diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-abbbi.dts b/arch/arm/boot/dts/am335x-abbbi.dts
--- a/arch/arm/boot/dts/am335x-abbbi.dts	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-abbbi.dts	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,169 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ * Copyright 2015 Konsulko Group
+ */
+/dts-v1/;
+
+#include "am33xx.dtsi"
+#include "am335x-bone-common.dtsi"
+
+&sgx {
+	status = "okay";
+};
+
+
+/ {
+	model = "Arrow BeagleBone Black Industrial";
+	compatible = "arrow,am335x-abbbi", "ti,am335x-bone", "ti,am33xx";
+
+	chosen {
+		base_dtb = "am335x-abbbi.dts";
+		base_dtb_timestamp = __TIMESTAMP__;
+	};
+};
+
+&ldo3_reg {
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-always-on;
+};
+
+&mmc1 {
+	vmmc-supply = <&vmmcsd_fixed>;
+};
+
+&mmc2 {
+	vmmc-supply = <&vmmcsd_fixed>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&emmc_pins>;
+	bus-width = <8>;
+	status = "okay";
+};
+
+&am33xx_pinmux {
+	adi_hdmi_bbbi_pins: adi_hdmi_bbbi_pins {
+		pinctrl-single,pins = <
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA0, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA1, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA2, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA3, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA4, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA5, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA6, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA7, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA8, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA9, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA10, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA11, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA12, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA13, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA14, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA15, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_VSYNC, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_HSYNC, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_PCLK, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_AC_BIAS_EN, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
+		>;
+	};
+
+	mcasp0_pins: mcasp0_pins {
+		pinctrl-single,pins = <
+			AM33XX_PADCONF(AM335X_PIN_MCASP0_AHCLKX, PIN_INPUT_PULLUP, MUX_MODE0) /* mcasp0_ahcklx.mcasp0_ahclkx */
+			AM33XX_PADCONF(AM335X_PIN_MCASP0_AHCLKR, PIN_OUTPUT_PULLDOWN, MUX_MODE2) /* mcasp0_ahclkr.mcasp0_axr2*/
+			AM33XX_PADCONF(AM335X_PIN_MCASP0_FSX, PIN_OUTPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MCASP0_ACLKX, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_GPMC_A11, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gpmc_a11.GPIO1_27 */
+		>;
+	};
+};
+
+&lcdc {
+	status = "okay";
+
+	/* If you want to get 24 bit RGB and 16 BGR mode instead of
+	 * current 16 bit RGB and 24 BGR modes, set the propety
+	 * below to "crossed" and uncomment the video-ports -property
+	 * in tda19988 node.
+	 */
+	blue-and-red-wiring = "straight";
+
+	port {
+		lcdc_0: endpoint@0 {
+			remote-endpoint = <&hdmi_0>;
+		};
+	};
+};
+
+&i2c0 {
+	adv7511: adv7511@39 {
+		compatible = "adi,adv7511";
+		reg = <0x39>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&adi_hdmi_bbbi_pins>;
+
+		/* Convert 24bit BGR to RGB, e.g. cross red and blue wiring */
+		/* video-ports = <0x234501>; */
+
+		#sound-dai-cells = <0>;
+
+		ports {
+			port@0 {
+				hdmi_0: endpoint@0 {
+					remote-endpoint = <&lcdc_0>;
+				};
+			};
+		};
+	};
+};
+
+&rtc {
+	system-power-controller;
+};
+
+&mcasp0	{
+	#sound-dai-cells = <0>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcasp0_pins>;
+	status = "okay";
+	op-mode = <0>;	/* MCASP_IIS_MODE */
+	tdm-slots = <2>;
+	serial-dir = <	/* 0: INACTIVE, 1: TX, 2: RX */
+			0 0 1 0
+		>;
+	tx-num-evt = <1>;
+	rx-num-evt = <1>;
+};
+
+/ {
+	clk_mcasp0_fixed: clk_mcasp0_fixed {
+		#clock-cells = <0>;
+		compatible = "fixed-clock";
+		clock-frequency = <24576000>;
+	};
+
+	clk_mcasp0: clk_mcasp0 {
+		#clock-cells = <0>;
+		compatible = "gpio-gate-clock";
+		clocks = <&clk_mcasp0_fixed>;
+		enable-gpios = <&gpio1 27 0>; /* BeagleBone Black Clk enable on GPIO1_27 */
+	};
+
+	sound {
+		compatible = "simple-audio-card";
+		simple-audio-card,name = "TI BeagleBone Black";
+		simple-audio-card,format = "i2s";
+		simple-audio-card,bitclock-master = <&dailink0_master>;
+		simple-audio-card,frame-master = <&dailink0_master>;
+
+		dailink0_master: simple-audio-card,cpu {
+			sound-dai = <&mcasp0>;
+			clocks = <&clk_mcasp0>;
+		};
+
+		simple-audio-card,codec {
+			sound-dai = <&adv7511>;
+		};
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-boneblack-audio.dts b/arch/arm/boot/dts/am335x-boneblack-audio.dts
--- a/arch/arm/boot/dts/am335x-boneblack-audio.dts	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-boneblack-audio.dts	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,50 @@
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+/dts-v1/;
+
+#include "am33xx.dtsi"
+#include "am335x-bone-common.dtsi"
+
+/ {
+	model = "TI AM335x BeagleBone Black";
+	compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx";
+
+	clk_mcasp0_fixed: clk_mcasp0_fixed {
+	      #clock-cells = <0>;
+	      compatible = "fixed-clock";
+	      clock-frequency = <24576000>;
+	};
+
+	clk_mcasp0: clk_mcasp0 {
+	      #clock-cells = <0>;
+	      compatible = "gpio-gate-clock";
+	      clocks = <&clk_mcasp0_fixed>;
+	      enable-gpios = <&gpio1 27 GPIO_ACTIVE_HIGH>; /* BeagleBone Black Clk enable on GPIO1_27 */
+	};
+};
+
+&ldo3_reg {
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-always-on;
+};
+
+&mmc1 {
+	vmmc-supply = <&vmmcsd_fixed>;
+};
+
+&cpu0_opp_table {
+	/*
+	 * All PG 2.0 silicon may not support 1GHz but some of the early
+	 * BeagleBone Blacks have PG 2.0 silicon which is guaranteed
+	 * to support 1GHz OPP so enable it for PG 2.0 on this board.
+	 */
+	oppnitro-1000000000 {
+		opp-supported-hw = <0x06 0x0100>;
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-boneblack-bbb-exp-c.dts b/arch/arm/boot/dts/am335x-boneblack-bbb-exp-c.dts
--- a/arch/arm/boot/dts/am335x-boneblack-bbb-exp-c.dts	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-boneblack-bbb-exp-c.dts	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,47 @@
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+/dts-v1/;
+
+#include "am33xx.dtsi"
+#include "am335x-bone-common-no-capemgr.dtsi"
+
+/ {
+	model = "TI AM335x BeagleBone Black";
+	compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx";
+};
+
+&ldo3_reg {
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-always-on;
+};
+
+&mmc1 {
+	vmmc-supply = <&vmmcsd_fixed>;
+};
+
+&mmc2 {
+	vmmc-supply = <&vmmcsd_fixed>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&emmc_pins>;
+	bus-width = <8>;
+	status = "okay";
+};
+
+&cpu0_opp_table {
+	/*
+	 * All PG 2.0 silicon may not support 1GHz but some of the early
+	 * BeagleBone Blacks have PG 2.0 silicon which is guaranteed
+	 * to support 1GHz OPP so enable it for PG 2.0 on this board.
+	 */
+	oppnitro-1000000000 {
+		opp-supported-hw = <0x06 0x0100>;
+	};
+};
+
+#include "am335x-cape-bbb-exp-c.dtsi"
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-boneblack-bbb-exp-r.dts b/arch/arm/boot/dts/am335x-boneblack-bbb-exp-r.dts
--- a/arch/arm/boot/dts/am335x-boneblack-bbb-exp-r.dts	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-boneblack-bbb-exp-r.dts	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,47 @@
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+/dts-v1/;
+
+#include "am33xx.dtsi"
+#include "am335x-bone-common-no-capemgr.dtsi"
+
+/ {
+	model = "TI AM335x BeagleBone Black";
+	compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx";
+};
+
+&ldo3_reg {
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-always-on;
+};
+
+&mmc1 {
+	vmmc-supply = <&vmmcsd_fixed>;
+};
+
+&mmc2 {
+	vmmc-supply = <&vmmcsd_fixed>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&emmc_pins>;
+	bus-width = <8>;
+	status = "okay";
+};
+
+&cpu0_opp_table {
+	/*
+	 * All PG 2.0 silicon may not support 1GHz but some of the early
+	 * BeagleBone Blacks have PG 2.0 silicon which is guaranteed
+	 * to support 1GHz OPP so enable it for PG 2.0 on this board.
+	 */
+	oppnitro-1000000000 {
+		opp-supported-hw = <0x06 0x0100>;
+	};
+};
+
+#include "am335x-cape-bbb-exp-r.dtsi"
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-boneblack-bbbmini.dts b/arch/arm/boot/dts/am335x-boneblack-bbbmini.dts
--- a/arch/arm/boot/dts/am335x-boneblack-bbbmini.dts	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-boneblack-bbbmini.dts	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,209 @@
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ *                    Modified by Mirko Denecke <mirkix@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+/dts-v1/;
+
+#include "am33xx.dtsi"
+#include "am335x-bone-common.dtsi"
+#include <dt-bindings/board/am335x-bbw-bbb-base.h>
+#include <dt-bindings/pinctrl/am33xx.h>
+
+/ {
+	model = "TI AM335x BeagleBone Black";
+	compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx";
+};
+
+&ldo3_reg {
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-always-on;
+};
+
+&mmc1 {
+	vmmc-supply = <&vmmcsd_fixed>;
+};
+
+&mmc2 {
+	vmmc-supply = <&vmmcsd_fixed>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&emmc_pins>;
+	bus-width = <8>;
+	status = "okay";
+};
+
+&cpu0_opp_table {
+	/*
+	 * All PG 2.0 silicon may not support 1GHz but some of the early
+	 * BeagleBone Blacks have PG 2.0 silicon which is guaranteed
+	 * to support 1GHz OPP so enable it for PG 2.0 on this board.
+	 */
+	oppnitro-1000000000 {
+		opp-supported-hw = <0x06 0x0100>;
+	};
+};
+
+&am33xx_pinmux {
+	dcan1_pins: pinmux_dcan1_pins {
+		pinctrl-single,pins = <
+			/* P9_26: uart1_rxd.d_can1_tx */
+			BONE_P9_26 (PIN_OUTPUT_PULLUP | MUX_MODE2)
+			/* P9_24: uart1_txd.d_can1_rx */
+			BONE_P9_24 (PIN_INPUT_PULLUP | MUX_MODE2)
+		>;
+	};
+
+	pru_pins: pinmux_pru_pins {
+		pinctrl-single,pins = <
+			0x03c 0x35	/* ecap0_in_pwm0_out.pr1_ecap0_ecap_capin, MODE5 | INPUT_PULLUP | PRU, PPM-sum, SBUS, DSM  */
+
+			0x0e8 0x25	/* lcd_pclk.pr1_pru1_pru_r30_10, MODE5 | OUTPUT | PRU, CH_1 */
+			0x0e0 0x25	/* lcd_vsync.pr1_pru1_pru_r30_8, MODE5 | OUTPUT | PRU, CH_2 */
+			0x0ec 0x25	/* lcd_ac_bias_en.pr1_pru1_pru_r30_11, MODE5 | OUTPUT | PRU, CH_3 */
+			0x0e4 0x25	/* lcd_hsync.pr1_pru1_pru_r30_9, MODE5 | OUTPUT | PRU, CH_4 */
+			0x0bc 0x25	/* lcd_data7.pr1_pru1_pru_r30_7, MODE5 | OUTPUT | PRU, CH_5 */
+			0x0b8 0x25	/* lcd_data6.pr1_pru1_pru_r30_6, MODE5 | OUTPUT | PRU, CH_6 */
+			0x0b4 0x25	/* lcd_data5.pr1_pru1_pru_r30_5, MODE5 | OUTPUT | PRU, CH_7 */
+			0x0b0 0x25	/* lcd_data4.pr1_pru1_pru_r30_4, MODE5 | OUTPUT | PRU, CH_8 */
+			0x0ac 0x25	/* lcd_data3.pr1_pru1_pru_r30_3, MODE5 | OUTPUT | PRU, CH_9 */
+			0x0a8 0x25	/* lcd_data2.pr1_pru1_pru_r30_2, MODE5 | OUTPUT | PRU, CH_10 */
+			0x0a4 0x25	/* lcd_data1.pr1_pru1_pru_r30_1, MODE5 | OUTPUT | PRU, CH_11 */
+			0x0a0 0x25	/* lcd_data0.pr1_pru1_pru_r30_0, MODE5 | OUTPUT | PRU, CH_12 */
+
+			BONE_P8_12 (PIN_OUTPUT_PULLDOWN | MUX_MODE6) /* HC-SR04 TRIG */
+			BONE_P8_16 (PIN_INPUT_PULLDOWN | MUX_MODE6) /* HC-SR04 ECHO */
+
+			BONE_P9_25 (PIN_INPUT_PULLDOWN | MUX_MODE6) /* MPU9250 INT */
+		>;
+	};
+
+	spi0_pins: pinmux_spi0_pins {
+		pinctrl-single,pins = <
+			/* P9_22: spi0_sclk.spi0_sclk */
+			BONE_P9_22 (PIN_INPUT_PULLUP | MUX_MODE0)
+			/* P9_21: spi0_d0.spi0_d0 */
+			BONE_P9_21 (PIN_INPUT_PULLUP | MUX_MODE0)
+			/* P9_18: spi0_d1.spi0_d1 */
+			BONE_P9_18 (PIN_OUTPUT_PULLUP | MUX_MODE0)
+			/* P9_17: spi0_cs0.spi0_cs0 */
+			BONE_P9_17 (PIN_OUTPUT_PULLUP | MUX_MODE0)
+		>;
+	};
+
+	spi1_pins: pinmux_spi1_pins {
+		pinctrl-single,pins = <
+			/* P9_31: mcasp0_aclkx.spi1_sclk */
+			BONE_P9_31 (PIN_INPUT_PULLUP | MUX_MODE3)
+
+			/* P9_29: mcasp0_fsx.spi1_d0 */
+			BONE_P9_29 (PIN_INPUT_PULLUP | MUX_MODE3)
+
+			/* P9_30: mcasp0_axr0.spi1_d1 */
+			BONE_P9_30 (PIN_OUTPUT_PULLUP | MUX_MODE3)
+
+			/* P9_28: mcasp0_ahclkr.spi1_cs0 */
+			BONE_P9_28 (PIN_OUTPUT_PULLUP | MUX_MODE3)
+
+			/* P9_19: uart1_rtsn.spi1_cs1 */
+/*			BONE_P9_19 (PIN_OUTPUT_PULLUP | MUX_MODE4)*/
+
+			/* P9_42: ecap0_in_pwm0_out.spi1_cs1 */
+			BONE_P9_42A (PIN_OUTPUT_PULLUP | MUX_MODE2)
+		>;
+	};
+
+	uart4_pins: pinmux_uart4_pins {
+		pinctrl-single,pins = <
+			/* P9_11: gpmc_wait0.uart4_rxd_mux2 */
+			BONE_P9_11 (PIN_INPUT_PULLUP | MUX_MODE6)
+			/* P9_13: gpmc_wpn.uart4_txd_mux2  */
+			BONE_P9_13 (PIN_OUTPUT_PULLDOWN | MUX_MODE6)
+		>;
+	};
+
+	uart5_pins: pinmux_uart5_pins {
+		pinctrl-single,pins = <
+			/* P8_38: lcd_data9.uart5_rxd */
+			BONE_P8_38 (PIN_INPUT_PULLUP | MUX_MODE4)
+			/* P8_37: lcd_data8.uart5_txd */
+			BONE_P8_37 (PIN_OUTPUT_PULLDOWN | MUX_MODE4)
+		>;
+	};
+};
+
+&dcan1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&dcan1_pins>;
+	status = "okay";
+};
+
+&i2c2 {
+	clock-frequency = <400000>;
+};
+
+&spi0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&spi0_pins>;
+	status = "okay";
+
+	spi0@0 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		spi-max-frequency = <24000000>;
+		reg = <0>;
+		compatible = "spidev";
+		symlink = "spi/0.0";
+	};
+};
+
+&spi1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&spi1_pins>;
+	status = "okay";
+
+	spi1@0 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		reg = <0>;
+		spi-max-frequency = <24000000>;
+		compatible = "spidev";
+		symlink = "spi/1.0";
+	};
+
+	spi1@1 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		reg = <1>;
+		spi-max-frequency = <24000000>;
+		compatible = "spidev";
+		symlink = "spi/1.1";
+	};
+};
+
+&tscadc {
+	adc {
+		ti,adc-channels = <0 1>;
+	};
+};
+
+//&pruss {
+//	pinctrl-names = "default";
+//	pinctrl-0 = <&pru_pins>;
+//	status = "okay";
+//};
+
+&uart4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart4_pins>;
+	status = "okay";
+};
+
+&uart5 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart5_pins>;
+	status = "okay";
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-boneblack-common.dtsi b/arch/arm/boot/dts/am335x-boneblack-common.dtsi
--- a/arch/arm/boot/dts/am335x-boneblack-common.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-boneblack-common.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -1,9 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include <dt-bindings/display/tda998x.h>
@@ -29,43 +26,36 @@
 &am33xx_pinmux {
 	nxp_hdmi_bonelt_pins: nxp_hdmi_bonelt_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x9b0, PIN_OUTPUT_PULLDOWN | MUX_MODE3)	/* xdma_event_intr0 */
-			AM33XX_IOPAD(0x8a0, PIN_OUTPUT | MUX_MODE0)		/* lcd_data0.lcd_data0 */
-			AM33XX_IOPAD(0x8a4, PIN_OUTPUT | MUX_MODE0)		/* lcd_data1.lcd_data1 */
-			AM33XX_IOPAD(0x8a8, PIN_OUTPUT | MUX_MODE0)		/* lcd_data2.lcd_data2 */
-			AM33XX_IOPAD(0x8ac, PIN_OUTPUT | MUX_MODE0)		/* lcd_data3.lcd_data3 */
-			AM33XX_IOPAD(0x8b0, PIN_OUTPUT | MUX_MODE0)		/* lcd_data4.lcd_data4 */
-			AM33XX_IOPAD(0x8b4, PIN_OUTPUT | MUX_MODE0)		/* lcd_data5.lcd_data5 */
-			AM33XX_IOPAD(0x8b8, PIN_OUTPUT | MUX_MODE0)		/* lcd_data6.lcd_data6 */
-			AM33XX_IOPAD(0x8bc, PIN_OUTPUT | MUX_MODE0)		/* lcd_data7.lcd_data7 */
-			AM33XX_IOPAD(0x8c0, PIN_OUTPUT | MUX_MODE0)		/* lcd_data8.lcd_data8 */
-			AM33XX_IOPAD(0x8c4, PIN_OUTPUT | MUX_MODE0)		/* lcd_data9.lcd_data9 */
-			AM33XX_IOPAD(0x8c8, PIN_OUTPUT | MUX_MODE0)		/* lcd_data10.lcd_data10 */
-			AM33XX_IOPAD(0x8cc, PIN_OUTPUT | MUX_MODE0)		/* lcd_data11.lcd_data11 */
-			AM33XX_IOPAD(0x8d0, PIN_OUTPUT | MUX_MODE0)		/* lcd_data12.lcd_data12 */
-			AM33XX_IOPAD(0x8d4, PIN_OUTPUT | MUX_MODE0)		/* lcd_data13.lcd_data13 */
-			AM33XX_IOPAD(0x8d8, PIN_OUTPUT | MUX_MODE0)		/* lcd_data14.lcd_data14 */
-			AM33XX_IOPAD(0x8dc, PIN_OUTPUT | MUX_MODE0)		/* lcd_data15.lcd_data15 */
-			AM33XX_IOPAD(0x8e0, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* lcd_vsync.lcd_vsync */
-			AM33XX_IOPAD(0x8e4, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* lcd_hsync.lcd_hsync */
-			AM33XX_IOPAD(0x8e8, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* lcd_pclk.lcd_pclk */
-			AM33XX_IOPAD(0x8ec, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* lcd_ac_bias_en.lcd_ac_bias_en */
-		>;
-	};
-
-	nxp_hdmi_bonelt_off_pins: nxp_hdmi_bonelt_off_pins {
-		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x9b0, PIN_OUTPUT_PULLDOWN | MUX_MODE3)	/* xdma_event_intr0 */
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA0, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA1, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA2, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA3, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA4, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA5, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA6, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA7, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA8, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA9, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA10, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA11, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA12, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA13, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA14, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA15, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_VSYNC, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_HSYNC, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_PCLK, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_AC_BIAS_EN, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
 		>;
 	};
 
 	mcasp0_pins: mcasp0_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x9ac, PIN_INPUT_PULLUP | MUX_MODE0) /* mcasp0_ahcklx.mcasp0_ahclkx */
-			AM33XX_IOPAD(0x99c, PIN_OUTPUT_PULLDOWN | MUX_MODE2) /* mcasp0_ahclkr.mcasp0_axr2*/
-			AM33XX_IOPAD(0x994, PIN_OUTPUT_PULLUP | MUX_MODE0) /* mcasp0_fsx.mcasp0_fsx */
-			AM33XX_IOPAD(0x990, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mcasp0_aclkx.mcasp0_aclkx */
-			AM33XX_IOPAD(0x86c, PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* gpmc_a11.GPIO1_27 */
+			AM33XX_PADCONF(AM335X_PIN_MCASP0_AHCLKX, PIN_INPUT_PULLUP, MUX_MODE0) /* mcasp0_ahcklx.mcasp0_ahclkx */
+			AM33XX_PADCONF(AM335X_PIN_MCASP0_AHCLKR, PIN_OUTPUT_PULLDOWN, MUX_MODE2) /* mcasp0_ahclkr.mcasp0_axr2*/
+			AM33XX_PADCONF(AM335X_PIN_MCASP0_FSX, PIN_OUTPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MCASP0_ACLKX, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_GPMC_A11, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gpmc_a11.GPIO1_27 */
 		>;
 	};
 };
@@ -92,9 +82,8 @@
 		compatible = "nxp,tda998x";
 		reg = <0x70>;
 
-		pinctrl-names = "default", "off";
+		pinctrl-names = "default";
 		pinctrl-0 = <&nxp_hdmi_bonelt_pins>;
-		pinctrl-1 = <&nxp_hdmi_bonelt_off_pins>;
 
 		/* Convert 24bit BGR to RGB, e.g. cross red and blue wiring */
 		/* video-ports = <0x234501>; */
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-boneblack.dts b/arch/arm/boot/dts/am335x-boneblack.dts
--- a/arch/arm/boot/dts/am335x-boneblack.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-boneblack.dts	2020-06-19 21:00:31.404227715 -0400
@@ -1,19 +1,26 @@
+// SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 /dts-v1/;
 
 #include "am33xx.dtsi"
 #include "am335x-bone-common.dtsi"
 #include "am335x-boneblack-common.dtsi"
+/* #include "am335x-bone-jtag.dtsi" */
 
 / {
 	model = "TI AM335x BeagleBone Black";
 	compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx";
+
+	chosen {
+		base_dtb = "am335x-boneblack.dts";
+		base_dtb_timestamp = __TIMESTAMP__;
+	};
+};
+
+&sgx {
+	status = "okay";
 };
 
 &cpu0_opp_table {
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-boneblack-prusuart.dts b/arch/arm/boot/dts/am335x-boneblack-prusuart.dts
--- a/arch/arm/boot/dts/am335x-boneblack-prusuart.dts	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-boneblack-prusuart.dts	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,108 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+#include "am335x-boneblack.dts"
+
+&am33xx_pinmux {
+	pru_uart0_bone_pins: pru_uart0bone_pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x990, PIN_OUTPUT_PULLUP | MUX_MODE5)	/* mcasp0_aclkx.pr1_pru0_pru_r30_0, port0 TXD, P9.31 */
+			AM33XX_IOPAD(0x994, PIN_INPUT_PULLUP | MUX_MODE6)	/* mcasp0_fsx.pr1_pru0_pru_r31_1, port0 RXD, P9.29 */
+
+			AM33XX_IOPAD(0x998, PIN_OUTPUT_PULLUP | MUX_MODE5)	/* mcasp0_axr0.pr1_pru0_pru_r30_2, port1 TXD, P9.30 */
+			AM33XX_IOPAD(0x99c, PIN_INPUT_PULLUP | MUX_MODE6)	/* mcasp0_ahclkr.pr1_pru0_pru_r31_3, port1 RXD, P9.28 */
+
+			AM33XX_IOPAD(0x9a4, PIN_OUTPUT_PULLUP | MUX_MODE5)	/* mcasp0_fsr.pr1_pru0_pru_r30_5, port2 TXD, P9.27 */
+			AM33XX_IOPAD(0x9ac, PIN_INPUT_PULLUP | MUX_MODE6)	/* mcasp0_ahclkx.pr1_pru0_pru_r31_7, port2 RXD, P9.25 */
+		>;
+	};
+	pru_uart1_bone_pins: pru_uart1bone_pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x8a0, PIN_OUTPUT_PULLUP | MUX_MODE5)	/* lcd_data0.pru_pru1_pru_r30_0, port0 TXD, P8.45 */
+			AM33XX_IOPAD(0x8a4, PIN_INPUT_PULLUP | MUX_MODE6)	/* lcd_data1.pru_pru1_pru_r31_1, port0 RXD, P8.46 */
+			AM33XX_IOPAD(0x8a8, PIN_INPUT_PULLDOWN | MUX_MODE6)	/* lcd_data2.pru_pru1_pru_r30_2, port0 CTS, P8.43 */
+			AM33XX_IOPAD(0x8ac, PIN_OUTPUT_PULLDOWN | MUX_MODE5)	/* lcd_data3.pru_pru1_pru_r31_3, port0 RTS, P8.44 */
+
+			AM33XX_IOPAD(0x8b0, PIN_OUTPUT_PULLUP | MUX_MODE5)	/* lcd_data4.pru_pru1_pru_r30_4, port1 TXD, P8.41 */
+			AM33XX_IOPAD(0x8b4, PIN_INPUT_PULLUP | MUX_MODE6)	/* lcd_data5.pru_pru1_pru_r31_5, port1 RXD, P8.42 */
+			AM33XX_IOPAD(0x8b8, PIN_INPUT_PULLDOWN | MUX_MODE6)	/* lcd_data6.pru_pru1_pru_r30_6, port1 CTS, P8.39 */
+			AM33XX_IOPAD(0x8bc, PIN_OUTPUT_PULLDOWN | MUX_MODE5)	/* lcd_data7.pru_pru1_pru_r31_7, port1 RTS, P8.40 */
+
+			AM33XX_IOPAD(0x8e0, PIN_OUTPUT_PULLUP | MUX_MODE5)	/* lcd_vsync.pru_pru1_pru_r30_8, port2 TXD, P8.27 */
+			AM33XX_IOPAD(0x8e4, PIN_INPUT_PULLUP | MUX_MODE6)	/* lcd_hsync.pru_pru1_pru_r31_9, port2 RXD, P8.29 */
+			AM33XX_IOPAD(0x8e8, PIN_INPUT_PULLDOWN | MUX_MODE6)	/* lcd_pclk.pru_pru1_pru_r30_10, port2 CTS, P8.28 */
+			AM33XX_IOPAD(0x8ec, PIN_OUTPUT_PULLDOWN | MUX_MODE5)	/* lcd_ac_bias_en.pru_pru1_pru_r31_11, port2 RTS, P8.30 */
+		>;
+	};
+};
+
+/{
+	pru_suart0 {
+		compatible = "ti,pru-soft-uart";
+		interrupt-parent = <&pruss_intc>;
+		prus = <&pru0>;
+		firmware-name = "ti-pruss/pru_swuart-fw.elf";
+		ti,pru-interrupt-map = <0 21 2 2>, <0 22 3 3>, <0 23 4 4>;
+		pinctrl-0 = <&pru_uart0_bone_pins>;
+		pinctrl-names = "default";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pru0_port0: port@0 {
+			reg = <0>;
+			interrupts = <21>;
+			ti,pru-suart-pins = /bits/ 8 <0 1>;
+		};
+		pru0_port1: port@1 {
+			reg = <1>;
+			interrupts = <22>;
+			ti,pru-suart-pins = /bits/ 8 <2 3>;
+		};
+		pru0_port2: port@2 {
+			reg = <2>;
+			interrupts = <23>;
+			ti,pru-suart-pins = /bits/ 8 <5 7>;
+		};
+	};
+
+	pru_suart1 {
+		compatible = "ti,pru-soft-uart";
+		interrupt-parent = <&pruss_intc>;
+		prus = <&pru1>;
+		firmware-name = "ti-pruss/pru_swuart-fw.elf";
+		ti,pru-interrupt-map = <0 24 5 5>, <0 25 6 6>, <0 26 7 7>;
+		pinctrl-0 = <&pru_uart1_bone_pins>;
+		pinctrl-names = "default";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pru1_port0: port@0 {
+			reg = <0>;
+			interrupts = <24>;
+			ti,pru-suart-pins = /bits/ 8 <0 1 2 3>;
+		};
+		pru1_port1: port@1 {
+			reg = <1>;
+			interrupts = <25>;
+			ti,pru-suart-pins = /bits/ 8 <4 5 6 7>;
+		};
+		pru1_port2: port@2 {
+			reg = <2>;
+			interrupts = <26>;
+			ti,pru-suart-pins = /bits/ 8 <8 9 10 11>;
+		};
+	};
+};
+
+/* Disable the following nodes due to pin mux conflicts with
+ * PRU Software UART signals
+ */
+&tda19988 {
+	status = "disabled";
+};
+
+&mcasp0 {
+	status = "disabled";
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-boneblack-roboticscape.dts b/arch/arm/boot/dts/am335x-boneblack-roboticscape.dts
--- a/arch/arm/boot/dts/am335x-boneblack-roboticscape.dts	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-boneblack-roboticscape.dts	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+/dts-v1/;
+
+#include "am33xx.dtsi"
+#include "am335x-bone-common-no-capemgr.dtsi"
+/* #include "am335x-boneblack-common.dtsi" */
+/* #include "am335x-bone-jtag.dtsi" */
+
+/* RoboticsCape */
+#include "am335x-bone-common-universal-pins.dtsi"
+#include "am335x-roboticscape.dtsi"
+
+/ {
+	model = "TI AM335x BeagleBone Black RoboticsCape";
+	compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx";
+};
+
+/* needed from: am335x-boneblack-common.dtsi */
+&ldo3_reg {
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-always-on;
+};
+
+&mmc1 {
+	vmmc-supply = <&vmmcsd_fixed>;
+};
+
+&mmc2 {
+	vmmc-supply = <&vmmcsd_fixed>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&emmc_pins>;
+	bus-width = <8>;
+	status = "okay";
+};
+
+&cpu0_opp_table {
+	/*
+	 * All PG 2.0 silicon may not support 1GHz but some of the early
+	 * BeagleBone Blacks have PG 2.0 silicon which is guaranteed
+	 * to support 1GHz OPP so enable it for PG 2.0 on this board.
+	 */
+	oppnitro-1000000000 {
+		opp-supported-hw = <0x06 0x0100>;
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-boneblack-uboot.dts b/arch/arm/boot/dts/am335x-boneblack-uboot.dts
--- a/arch/arm/boot/dts/am335x-boneblack-uboot.dts	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-boneblack-uboot.dts	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ */
+/dts-v1/;
+
+#include "am33xx.dtsi"
+#include "am335x-bone-common.dtsi"
+/* #include "am335x-bone-jtag.dtsi" */
+
+/ {
+	model = "TI AM335x BeagleBone Black";
+	compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx";
+
+	chosen {
+		base_dtb = "am335x-boneblack-uboot.dts";
+		base_dtb_timestamp = __TIMESTAMP__;
+	};
+};
+
+&sgx {
+	status = "okay";
+};
+
+&cpu0_opp_table {
+	/*
+	 * All PG 2.0 silicon may not support 1GHz but some of the early
+	 * BeagleBone Blacks have PG 2.0 silicon which is guaranteed
+	 * to support 1GHz OPP so enable it for PG 2.0 on this board.
+	 */
+	oppnitro-1000000000 {
+		opp-supported-hw = <0x06 0x0100>;
+	};
+};
+
+&ldo3_reg {
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-always-on;
+};
+
+&mmc1 {
+	vmmc-supply = <&vmmcsd_fixed>;
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-boneblack-uboot-univ.dts b/arch/arm/boot/dts/am335x-boneblack-uboot-univ.dts
--- a/arch/arm/boot/dts/am335x-boneblack-uboot-univ.dts	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-boneblack-uboot-univ.dts	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ */
+/dts-v1/;
+
+#include "am33xx.dtsi"
+#include "am335x-bone-common.dtsi"
+#include "am335x-bone-common-univ.dtsi"
+
+/ {
+	model = "TI AM335x BeagleBone Black";
+	compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx";
+
+	chosen {
+		base_dtb = "am335x-boneblack-uboot-univ.dts";
+		base_dtb_timestamp = __TIMESTAMP__;
+	};
+};
+
+&sgx {
+	status = "okay";
+};
+
+&cpu0_opp_table {
+	/*
+	 * All PG 2.0 silicon may not support 1GHz but some of the early
+	 * BeagleBone Blacks have PG 2.0 silicon which is guaranteed
+	 * to support 1GHz OPP so enable it for PG 2.0 on this board.
+	 */
+	oppnitro-1000000000 {
+		opp-supported-hw = <0x06 0x0100>;
+	};
+};
+
+&ldo3_reg {
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-always-on;
+};
+
+&mmc1 {
+	vmmc-supply = <&vmmcsd_fixed>;
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-boneblack-wireless.dts b/arch/arm/boot/dts/am335x-boneblack-wireless.dts
--- a/arch/arm/boot/dts/am335x-boneblack-wireless.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-boneblack-wireless.dts	2020-06-19 21:00:31.404227715 -0400
@@ -1,9 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 /dts-v1/;
 
@@ -11,11 +8,17 @@
 #include "am335x-bone-common.dtsi"
 #include "am335x-boneblack-common.dtsi"
 #include <dt-bindings/interrupt-controller/irq.h>
+/* #include "am335x-bone-jtag.dtsi" */
 
 / {
 	model = "TI AM335x BeagleBone Black Wireless";
 	compatible = "ti,am335x-bone-black-wireless", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx";
 
+	chosen {
+		base_dtb = "am335x-boneblack-wireless.dts";
+		base_dtb_timestamp = __TIMESTAMP__;
+	};
+
 	wlan_en_reg: fixedregulator@2 {
 		compatible = "regulator-fixed";
 		regulator-name = "wlan-en-regulator";
@@ -29,38 +32,42 @@
 	};
 };
 
+&sgx {
+	status = "okay";
+};
+
 &am33xx_pinmux {
 	bt_pins: pinmux_bt_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x928, PIN_OUTPUT_PULLUP | MUX_MODE7)	/* gmii1_txd0.gpio0_28 - BT_EN */
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_OUTPUT_PULLUP, MUX_MODE7)	/* gmii1_txd0.gpio0_28 - BT_EN */
 		>;
 	};
 
 	mmc3_pins: pinmux_mmc3_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x93c, PIN_INPUT_PULLUP | MUX_MODE6 ) /* (L15) gmii1_rxd1.mmc2_clk */
-			AM33XX_IOPAD(0x914, PIN_INPUT_PULLUP | MUX_MODE6 ) /* (J16) gmii1_txen.mmc2_cmd */
-			AM33XX_IOPAD(0x918, PIN_INPUT_PULLUP | MUX_MODE5 ) /* (J17) gmii1_rxdv.mmc2_dat0 */
-			AM33XX_IOPAD(0x91c, PIN_INPUT_PULLUP | MUX_MODE5 ) /* (J18) gmii1_txd3.mmc2_dat1 */
-			AM33XX_IOPAD(0x920, PIN_INPUT_PULLUP | MUX_MODE5 ) /* (K15) gmii1_txd2.mmc2_dat2 */
-			AM33XX_IOPAD(0x908, PIN_INPUT_PULLUP | MUX_MODE5 ) /* (H16) gmii1_col.mmc2_dat3 */
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLUP, MUX_MODE6 ) /* (L15) gmii1_rxd1.mmc2_clk */
+			AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_INPUT_PULLUP, MUX_MODE6 ) /* (J16) gmii1_txen.mmc2_cmd */
+			AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLUP, MUX_MODE5 ) /* (J17) gmii1_rxdv.mmc2_dat0 */
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_INPUT_PULLUP, MUX_MODE5 ) /* (J18) gmii1_txd3.mmc2_dat1 */
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_INPUT_PULLUP, MUX_MODE5 ) /* (K15) gmii1_txd2.mmc2_dat2 */
+			AM33XX_PADCONF(AM335X_PIN_MII1_COL, PIN_INPUT_PULLUP, MUX_MODE5 ) /* (H16) gmii1_col.mmc2_dat3 */
 		>;
 	};
 
 	uart3_pins: pinmux_uart3_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x934, PIN_INPUT_PULLUP | MUX_MODE1)	/* gmii1_rxd3.uart3_rxd */
-			AM33XX_IOPAD(0x938, PIN_OUTPUT_PULLDOWN | MUX_MODE1)	/* gmii1_rxd2.uart3_txd */
-			AM33XX_IOPAD(0x948, PIN_INPUT | MUX_MODE3)		/* mdio_data.uart3_ctsn */
-			AM33XX_IOPAD(0x94c, PIN_OUTPUT_PULLDOWN | MUX_MODE3)	/* mdio_clk.uart3_rtsn */
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLUP, MUX_MODE1)	/* gmii1_rxd3.uart3_rxd */
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_OUTPUT_PULLDOWN, MUX_MODE1)	/* gmii1_rxd2.uart3_txd */
+			AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT, MUX_MODE3)		/* mdio_data.uart3_ctsn */
+			AM33XX_PADCONF(AM335X_PIN_MDC, PIN_OUTPUT_PULLDOWN, MUX_MODE3)	/* mdio_clk.uart3_rtsn */
 		>;
 	};
 
 	wl18xx_pins: pinmux_wl18xx_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x92c, PIN_OUTPUT_PULLDOWN | MUX_MODE7)	/* gmii1_txclk.gpio3_9 WL_EN */
-			AM33XX_IOPAD(0x944, PIN_INPUT_PULLDOWN | MUX_MODE7)	/* rmii1_refclk.gpio0_29 WL_IRQ */
-			AM33XX_IOPAD(0x930, PIN_OUTPUT_PULLUP | MUX_MODE7)	/* gmii1_rxclk.gpio3_10 LS_BUF_EN */
+			AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_OUTPUT_PULLDOWN, MUX_MODE7)	/* gmii1_txclk.gpio3_9 WL_EN */
+			AM33XX_PADCONF(AM335X_PIN_RMII1_REF_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7)	/* rmii1_refclk.gpio0_29 WL_IRQ */
+			AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_OUTPUT_PULLUP, MUX_MODE7)	/* gmii1_rxclk.gpio3_10 LS_BUF_EN */
 		>;
 	};
 };
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-boneblack-wireless-roboticscape.dts b/arch/arm/boot/dts/am335x-boneblack-wireless-roboticscape.dts
--- a/arch/arm/boot/dts/am335x-boneblack-wireless-roboticscape.dts	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-boneblack-wireless-roboticscape.dts	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,149 @@
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+/dts-v1/;
+
+#include "am33xx.dtsi"
+#include "am335x-bone-common-no-capemgr.dtsi"
+/* #include "am335x-boneblack-common.dtsi" */
+/* #include "am335x-bone-jtag.dtsi" */
+#include <dt-bindings/interrupt-controller/irq.h>
+
+/* RoboticsCape */
+#include "am335x-bone-common-universal-pins.dtsi"
+#include "am335x-roboticscape.dtsi"
+
+/ {
+	model = "TI AM335x BeagleBone Black Wireless RoboticsCape";
+	compatible = "ti,am335x-bone-black-wireless", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx";
+
+	wlan_en_reg: fixedregulator@2 {
+		compatible = "regulator-fixed";
+		regulator-name = "wlan-en-regulator";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		startup-delay-us= <70000>;
+
+		/* WL_EN */
+		gpio = <&gpio3 9 0>;
+		enable-active-high;
+	};
+};
+
+&cpu0_opp_table {
+	/*
+	 * All PG 2.0 silicon may not support 1GHz but some of the early
+	 * BeagleBone Blacks have PG 2.0 silicon which is guaranteed
+	 * to support 1GHz OPP so enable it for PG 2.0 on this board.
+	 */
+	oppnitro-1000000000 {
+		opp-supported-hw = <0x06 0x0100>;
+	};
+};
+
+&am33xx_pinmux {
+	bt_pins: pinmux_bt_pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x928, PIN_OUTPUT_PULLUP | MUX_MODE7)	/* gmii1_txd0.gpio0_28 - BT_EN */
+		>;
+	};
+
+	mmc3_pins: pinmux_mmc3_pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x93c, PIN_INPUT_PULLUP | MUX_MODE6 ) /* (L15) gmii1_rxd1.mmc2_clk */
+			AM33XX_IOPAD(0x914, PIN_INPUT_PULLUP | MUX_MODE6 ) /* (J16) gmii1_txen.mmc2_cmd */
+			AM33XX_IOPAD(0x918, PIN_INPUT_PULLUP | MUX_MODE5 ) /* (J17) gmii1_rxdv.mmc2_dat0 */
+			AM33XX_IOPAD(0x91c, PIN_INPUT_PULLUP | MUX_MODE5 ) /* (J18) gmii1_txd3.mmc2_dat1 */
+			AM33XX_IOPAD(0x920, PIN_INPUT_PULLUP | MUX_MODE5 ) /* (K15) gmii1_txd2.mmc2_dat2 */
+			AM33XX_IOPAD(0x908, PIN_INPUT_PULLUP | MUX_MODE5 ) /* (H16) gmii1_col.mmc2_dat3 */
+		>;
+	};
+
+	uart3_pins: pinmux_uart3_pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x934, PIN_INPUT_PULLUP | MUX_MODE1)	/* gmii1_rxd3.uart3_rxd */
+			AM33XX_IOPAD(0x938, PIN_OUTPUT_PULLDOWN | MUX_MODE1)	/* gmii1_rxd2.uart3_txd */
+			AM33XX_IOPAD(0x948, PIN_INPUT | MUX_MODE3)		/* mdio_data.uart3_ctsn */
+			AM33XX_IOPAD(0x94c, PIN_OUTPUT_PULLDOWN | MUX_MODE3)	/* mdio_clk.uart3_rtsn */
+		>;
+	};
+
+	wl18xx_pins: pinmux_wl18xx_pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x92c, PIN_OUTPUT_PULLDOWN | MUX_MODE7)	/* gmii1_txclk.gpio3_9 WL_EN */
+			AM33XX_IOPAD(0x944, PIN_INPUT_PULLDOWN | MUX_MODE7)	/* rmii1_refclk.gpio0_29 WL_IRQ */
+			AM33XX_IOPAD(0x930, PIN_OUTPUT_PULLUP | MUX_MODE7)	/* gmii1_rxclk.gpio3_10 LS_BUF_EN */
+		>;
+	};
+};
+
+&mac {
+	status = "disabled";
+};
+
+&mmc3 {
+	dmas = <&edma_xbar 12 0 1
+		&edma_xbar 13 0 2>;
+	dma-names = "tx", "rx";
+	status = "okay";
+	vmmc-supply = <&wlan_en_reg>;
+	bus-width = <4>;
+	non-removable;
+	cap-power-off-card;
+	ti,needs-special-hs-handling;
+	keep-power-in-suspend;
+	pinctrl-names = "default";
+	pinctrl-0 = <&mmc3_pins &wl18xx_pins>;
+
+	#address-cells = <1>;
+	#size-cells = <0>;
+	wlcore: wlcore@2 {
+		compatible = "ti,wl1835";
+		reg = <2>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <29 IRQ_TYPE_EDGE_RISING>;
+	};
+};
+
+&uart3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart3_pins &bt_pins>;
+	status = "okay";
+
+	bluetooth {
+		compatible = "ti,wl1835-st";
+		enable-gpios = <&gpio0 28 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&gpio3 {
+	ls_buf_en {
+		gpio-hog;
+		gpios = <10 GPIO_ACTIVE_HIGH>;
+		output-high;
+		line-name = "LS_BUF_EN";
+	};
+};
+
+/* needed from: am335x-boneblack-common.dtsi */
+&ldo3_reg {
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-always-on;
+};
+
+&mmc1 {
+	vmmc-supply = <&vmmcsd_fixed>;
+};
+
+&mmc2 {
+	vmmc-supply = <&vmmcsd_fixed>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&emmc_pins>;
+	bus-width = <8>;
+	status = "okay";
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-boneblack-wl1835mod-cape.dtsi b/arch/arm/boot/dts/am335x-boneblack-wl1835mod-cape.dtsi
--- a/arch/arm/boot/dts/am335x-boneblack-wl1835mod-cape.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-boneblack-wl1835mod-cape.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,128 @@
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <dt-bindings/interrupt-controller/irq.h>
+
+/ {
+	wlan_en_reg: fixedregulator@2 {
+		compatible = "regulator-fixed";
+		regulator-name = "wlan-en-regulator";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+
+		/* WL_EN */
+		gpio = <&gpio0 26 0>;
+		enable-active-high;
+	};
+
+	kim {
+	        compatible = "kim";
+	        nshutdown_gpio = <44>; /* Bank1, pin12 */
+	        dev_name = "/dev/ttyO4";
+	        flow_cntrl = <1>;
+	        baud_rate = <3000000>;
+	};
+
+	btwilink {
+	        compatible = "btwilink";
+	};
+};
+
+&am33xx_pinmux {
+	bt_pins: pinmux_bt_pins {
+		pinctrl-single,pins = <
+			0x30 (PIN_OUTPUT_PULLUP | MUX_MODE7)	/* gpmc_ad12.gpio1_12 */
+		>;
+	};
+
+	mmc2_pins: pinmux_mmc2_pins {
+		pinctrl-single,pins = <
+			0x80 (PIN_INPUT_PULLUP | MUX_MODE2) /* gpmc_csn1.mmc1_clk */
+			0x84 (PIN_INPUT_PULLUP | MUX_MODE2) /* gpmc_csn2.mmc1_cmd */
+			0x00 (PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad0.mmc1_dat0 */
+			0x04 (PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad1.mmc1_dat1 */
+			0x08 (PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad2.mmc1_dat2 */
+			0x0c (PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad3.mmc1_dat3 */
+		>;
+	};
+
+	mmc2_pins_sleep: pinmux_mmc2_pins_sleep {
+		pinctrl-single,pins = <
+			0x80 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_csn1.mmc1_clk */
+			0x84 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_csn2.mmc1_cmd */
+			0x00 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_ad0.mmc1_dat0 */
+			0x04 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_ad1.mmc1_dat1 */
+			0x08 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_ad2.mmc1_dat2 */
+			0x0c (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_ad3.mmc1_dat3 */
+		>;
+	};
+
+	/* wl18xx card enable/irq GPIOs. */
+	wlan_pins: pinmux_wlan_pins {
+		pinctrl-single,pins = <
+			0x28 (PIN_OUTPUT_PULLDOWN | MUX_MODE7)	/* gpmc_ad10.gpio0_26 WL_EN*/
+			0x2C (PIN_INPUT_PULLUP | MUX_MODE7)	/* gpmc_ad11.gpio0_27 WL_IRQ*/
+			0x7C (PIN_OUTPUT_PULLUP | MUX_MODE0)	/* gpmc_csn0.gpio1_29 BF_EN*/
+		>;
+	};
+
+	/* wl18xx card enable/irq GPIOs. */
+	wlan_pins_sleep: pinmux_wlan_pins_sleep {
+		pinctrl-single,pins = <
+			0x28 (PIN_OUTPUT_PULLUP | MUX_MODE7)	/* gpmc_ad10.gpio0_26 WL_EN*/
+			0x2C (PIN_INPUT_PULLUP | MUX_MODE7)	/* gpmc_ad11.gpio0_27 WL_IRQ*/
+			0x7C (PIN_OUTPUT_PULLUP | MUX_MODE0)	/* gpmc_csn0.gpio1_29 BF_EN*/
+		>;
+	};
+
+	uart4_pins_default: pinmux_uart4_pins_default {
+		pinctrl-single,pins = <
+			0xD0 (PIN_INPUT | MUX_MODE6)		/* lcd_data12.uart4_cts */
+			0xD4 (PIN_OUTPUT_PULLDOWN | MUX_MODE6)	/* lcd_data13.uart4_rts */
+			0x70 (PIN_INPUT_PULLUP | MUX_MODE6)	/* gpmc_wait0.uart4_rxd */
+			0x74 (PIN_OUTPUT_PULLDOWN | MUX_MODE6)	/* gpmc_wpn.uart4_txd */
+		>;
+	};
+
+	uart4_pins_sleep: pinmux_uart4_pins_sleep {
+		pinctrl-single,pins = <
+			0xD0 (PIN_INPUT_PULLDOWN | MUX_MODE7)	/* lcd_data12.uart4_cts */
+			0xD4 (PIN_INPUT_PULLDOWN | MUX_MODE7)	/* lcd_data13.uart4_rts */
+			0x70 (PIN_INPUT_PULLDOWN | MUX_MODE7)	/* gpmc_wait0.uart4_rxd */
+			0x74 (PIN_INPUT_PULLDOWN | MUX_MODE7)	/* gpmc_wpn.uart4_txd */
+		>;
+	};
+};
+
+&mmc2 {
+	status = "okay";
+	vmmc-supply = <&wlan_en_reg>;
+	bus-width = <4>;
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&mmc2_pins &wlan_pins>;
+	pinctrl-1 = <&mmc2_pins_sleep &wlan_pins_sleep>;
+	ti,non-removable;
+	ti,needs-special-hs-handling;
+	cap-power-off-card;
+	keep-power-in-suspend;
+
+	#address-cells = <1>;
+	#size-cells = <0>;
+	wlcore: wlcore@0 {
+		compatible = "ti,wl1835";
+		reg = <2>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <27 IRQ_TYPE_LEVEL_HIGH>;
+	};
+};
+
+&uart4 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&uart4_pins_default>;
+	pinctrl-1 = <&uart4_pins_sleep>;
+	status = "okay";
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-boneblack-wl1835mod.dts b/arch/arm/boot/dts/am335x-boneblack-wl1835mod.dts
--- a/arch/arm/boot/dts/am335x-boneblack-wl1835mod.dts	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-boneblack-wl1835mod.dts	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,49 @@
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+/dts-v1/;
+
+#include "am33xx.dtsi"
+#include "am335x-bone-common-no-capemgr.dtsi"
+
+/ {
+	model = "TI AM335x BeagleBone Black";
+	compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx";
+};
+
+&cpu0_opp_table {
+	/*
+	 * All PG 2.0 silicon may not support 1GHz but some of the early
+	 * BeagleBone Blacks have PG 2.0 silicon which is guaranteed
+	 * to support 1GHz OPP so enable it for PG 2.0 on this board.
+	 */
+	oppnitro-1000000000 {
+		opp-supported-hw = <0x06 0x0100>;
+	};
+};
+
+&ldo3_reg {
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-always-on;
+};
+
+&mmc1 {
+	vmmc-supply = <&vmmcsd_fixed>;
+};
+
+/* EMMC in reset */
+&gpio1 {
+	emmc_rst {
+		gpio-hog;
+		gpios = <20 0>;
+		output-high;
+		line-name = "EMMC ResetN";
+	};
+};
+
+#include "am335x-boneblack-wl1835mod-cape.dtsi"
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-boneblue.dts b/arch/arm/boot/dts/am335x-boneblue.dts
--- a/arch/arm/boot/dts/am335x-boneblue.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-boneblue.dts	2020-06-19 21:00:31.404227715 -0400
@@ -1,19 +1,23 @@
+// SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 /dts-v1/;
 
 #include "am33xx.dtsi"
+#include "am335x-bone-common-universal-pins.dtsi"
 #include <dt-bindings/interrupt-controller/irq.h>
 
 / {
 	model = "TI AM335x BeagleBone Blue";
 	compatible = "ti,am335x-bone-blue", "ti,am33xx";
 
+	chosen {
+		stdout-path = &uart0;
+		base_dtb = "am335x-boneblue.dts";
+		base_dtb_timestamp = __TIMESTAMP__;
+	};
+
 	cpus {
 		cpu@0 {
 			cpu0-supply = <&dcdc2_reg>;
@@ -25,10 +29,6 @@
 		reg = <0x80000000 0x20000000>; /* 512 MB */
 	};
 
-	chosen {
-		stdout-path = &uart0;
-	};
-
 	leds {
 		pinctrl-names = "default";
 		pinctrl-0 = <&user_leds_s0>;
@@ -125,142 +125,329 @@
 		gpio = <&gpio3 9 0>;
 		enable-active-high;
 	};
+
+	bt_en {
+		pinctrl-names = "default";
+		pinctrl-0 = <&bt_pins>;
+		compatible = "gpio-leds";
+
+		wl18xx_bt_en {
+			label = "wl18xx_bt_en";
+			gpios = <&gpio0 28 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+	};
 };
 
-&am33xx_pinmux {
-	user_leds_s0: user_leds_s0 {
-		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x854, PIN_OUTPUT | MUX_MODE7) /* (V15) gpmc_a5.gpio1[21] - USR_LED_0 */
-			AM33XX_IOPAD(0x858, PIN_OUTPUT | MUX_MODE7) /* (U15) gpmc_a6.gpio1[22] - USR_LED_1 */
-			AM33XX_IOPAD(0x85c, PIN_OUTPUT | MUX_MODE7) /* (T15) gpmc_a7.gpio1[23] - USR_LED_2 */
-			AM33XX_IOPAD(0x860, PIN_OUTPUT | MUX_MODE7) /* (V16) gpmc_a8.gpio1[24] - USR_LED_3 */
-			AM33XX_IOPAD(0x9b0, PIN_OUTPUT | MUX_MODE7) /* (A15) xdma_event_intr0.gpio0[19] - WIFI_LED */
-			AM33XX_IOPAD(0x890, PIN_OUTPUT | MUX_MODE7) /* (R7) gpmc_advn_ale.gpio2[2] - P8.7, LED_RED, GP1_PIN_5 */
-			AM33XX_IOPAD(0x894, PIN_OUTPUT | MUX_MODE7) /* (T7) gpmc_oen_ren.gpio2[3] - P8.8, LED_GREEN, GP1_PIN_6 */
-			AM33XX_IOPAD(0x82c, PIN_OUTPUT | MUX_MODE7) /* (U12) gpmc_ad11.gpio0[27] - P8.17, BATT_LED_1 */
-			AM33XX_IOPAD(0x8dc, PIN_OUTPUT | MUX_MODE7) /* (T5) lcd_data15.gpio0[11] - P8.32, BATT_LED_2 */
-			AM33XX_IOPAD(0x87c, PIN_OUTPUT | MUX_MODE7) /* (V6) gpmc_csn0.gpio1[29] - P8.26, BATT_LED_3 */
-			AM33XX_IOPAD(0x828, PIN_OUTPUT | MUX_MODE7) /* (T11) gpmc_ad10.gpio0[26] - P8.14, BATT_LED_4 */
+&aliases {
+	i2c0 = &i2c0;
+	i2c1 = &i2c1;
+	i2c2 = &i2c2;
+	serial0 = &uart0;
+	serial1 = &uart1;
+	serial2 = &uart2;
+	serial3 = &uart3;
+	serial4 = &uart4;
+	serial5 = &uart5;
+	d-can0 = &dcan0;
+	d-can1 = &dcan1;
+	usb0 = &usb0;
+	usb1 = &usb1;
+	phy0 = &usb0_phy;
+	phy1 = &usb1_phy;
+	ethernet0 = &cpsw_emac0;
+	ethernet1 = &cpsw_emac1;
+	spi0 = &spi0;
+	spi1 = &spi1;
+};
+
+&sgx {
+	status = "okay";
+};
 
-		>;
+&cpu0_opp_table {
+	/*
+	* Octavo Systems:
+	* The EFUSE_SMA register is not programmed for any of the AM335x wafers
+	* we get and we are not programming them during our production test.
+	* Therefore, from a DEVICE_ID revision point of view, the silicon looks
+	* like it is Revision 2.1.  However, from an EFUSE_SMA point of view for
+	* the HW OPP table, the silicon looks like it is Revision 1.0 (ie the
+	* EFUSE_SMA register reads as all zeros).
+	*/
+	oppnitro-1000000000 {
+		opp-supported-hw = <0x06 0x0100>;
 	};
+};
 
-	i2c0_pins: pinmux_i2c0_pins {
+&am33xx_pinmux {
+	/***************************************************************************
+	* Static Pinmux
+	***************************************************************************/
+	mux_helper_pins: pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x988, PIN_INPUT_PULLUP | MUX_MODE0)	/* (C17) I2C0_SDA.I2C0_SDA */
-			AM33XX_IOPAD(0x98c, PIN_INPUT_PULLUP | MUX_MODE0)	/* (C16) I2C0_SCL.I2C0_SCL */
+
+			/* GPIO Inputs */
+			0x09c 0x37	/*P8.9  Pause BUTTON, input pullup*/
+			0x098 0x37	/*P8.10 MODE BUTTON input pullup*/
+			0x1AC 0x37	/*P9.25 MPU-9150 INTERRUPT IN*/
+
+			/* Motor Control GPIO Out*/
+			0x088 ( PIN_OUTPUT | MUX_MODE7 ) /* (T13) gpmc_csn3.gpio2[0] - MDIR_1A different from cape! */
+			0x074 ( PIN_OUTPUT | MUX_MODE7 ) /* (U17) gpmc_wpn.gpio0[31] - P9.13, MDIR_1B */
+			0x040 ( PIN_OUTPUT | MUX_MODE7 ) /* (R13) gpmc_a0.gpio1[16] - P9.15, MDIR_2A */
+			0x0D8 ( PIN_OUTPUT | MUX_MODE7 ) /* (V4) lcd_data14.gpio0[10] - P8.31, MDIR_2B different from cape! */
+			0x0AC ( PIN_OUTPUT | MUX_MODE7 ) /* (R4) lcd_data3.gpio2[9] - P8.44, MDIR_3A */
+			0x0A8 ( PIN_OUTPUT | MUX_MODE7 ) /* (R3) lcd_data2.gpio2[8] - P8.43, MDIR_3B */
+			0x0A0 ( PIN_OUTPUT | MUX_MODE7 ) /* (R1) lcd_data0.gpio2[6] - P8.45, MDIR_4A */
+			0x0A4 ( PIN_OUTPUT | MUX_MODE7 ) /* (R2) lcd_data1.gpio2[7] - P8.46, MDIR_4B */
+			0x1B4 ( PIN_OUTPUT | MUX_MODE7 ) /* (D14) xdma_event_intr1.gpio0[20] - P9.41, MOT_STBY */
+
+			/* PRU encoder input */
+			0x038 0x36	/* P8_16,PRU0_r31_16,MODE6 */
+
+			/* PRU Servo output */
+			0x0e0 0x05	/*pru1_pru_r30_8, MODE5*/
+			0x0e8 0x05	/*pru1_pru_r30_10, MODE5 */
+			0x0e4 0x05	/*pr1_pru1_pru_r30_9, MODE5 */
+			0x0ec 0x05	/*pru1_pru_r30_11, MODE5 */
+			0x0b8 0x05	/*pru1_pru_r30_6, MODE5 */
+			0x0bc 0x05	/*pru1_pru_r30_7, MODE5 */
+			0x0b0 0x05	/*pru1_pru_r30_4, MODE5 */
+			0x0b4 0x05	/*pru1_pru_r30_5, MODE5 */
+			0x0C8 0x0F	/*P8.36, SERVO_PWR GPIO OUT*/
+
+			/* WILINK 8 */
+			0x08c 0x0F	/*P8.18 V12 A2DP FSYNC */
+			0x078 0x0F	/*P9.12 A2DP_CLOCK*/
 		>;
 	};
 
-	i2c2_pins: pinmux_i2c2_pins {
+	user_leds_s0: user_leds_s0 {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x978, PIN_INPUT_PULLUP | MUX_MODE3)	/* (D18) uart1_ctsn.I2C2_SDA */
-			AM33XX_IOPAD(0x97c, PIN_INPUT_PULLUP | MUX_MODE3)	/* (D17) uart1_rtsn.I2C2_SCL */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_A5, PIN_OUTPUT, MUX_MODE7) /* (V15) gpmc_a5.gpio1[21] - USR_LED_0 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_A6, PIN_OUTPUT, MUX_MODE7) /* (U15) gpmc_a6.gpio1[22] - USR_LED_1 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_A7, PIN_OUTPUT, MUX_MODE7) /* (T15) gpmc_a7.gpio1[23] - USR_LED_2 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_A8, PIN_OUTPUT, MUX_MODE7) /* (V16) gpmc_a8.gpio1[24] - USR_LED_3 */
+			AM33XX_PADCONF(AM335X_PIN_XDMA_EVENT_INTR0, PIN_OUTPUT, MUX_MODE7) /* (A15) xdma_event_intr0.gpio0[19] - WIFI_LED */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_ADVN_ALE, PIN_OUTPUT, MUX_MODE7) /* (R7) gpmc_advn_ale.gpio2[2] - P8.7, LED_RED, GP1_PIN_5 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_OEN_REN, PIN_OUTPUT, MUX_MODE7) /* (T7) gpmc_oen_ren.gpio2[3] - P8.8, LED_GREEN, GP1_PIN_6 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD11, PIN_OUTPUT, MUX_MODE7) /* (U12) gpmc_ad11.gpio0[27] - P8.17, BATT_LED_1 */
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA15, PIN_OUTPUT, MUX_MODE7) /* (T5) lcd_data15.gpio0[11] - P8.32, BATT_LED_2 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_CSN0, PIN_OUTPUT, MUX_MODE7) /* (V6) gpmc_csn0.gpio1[29] - P8.26, BATT_LED_3 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD10, PIN_OUTPUT, MUX_MODE7) /* (T11) gpmc_ad10.gpio0[26] - P8.14, BATT_LED_4 */
+
 		>;
 	};
 
-	/* UT0 */
-	uart0_pins: pinmux_uart0_pins {
+	i2c0_pins: pinmux_i2c0_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x970, PIN_INPUT_PULLUP | MUX_MODE0)	/* (E15) uart0_rxd.uart0_rxd */
-			AM33XX_IOPAD(0x974, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* (E16) uart0_txd.uart0_txd */
+			AM33XX_IOPAD(0x988, PIN_INPUT_PULLUP | MUX_MODE0)	/* (C17) I2C0_SDA.I2C0_SDA */
+			AM33XX_IOPAD(0x98c, PIN_INPUT_PULLUP | MUX_MODE0)	/* (C16) I2C0_SCL.I2C0_SCL */
 		>;
 	};
 
-	/* UT1 */
-	uart1_pins: pinmux_uart1_pins {
+	i2c1_pins: pinmux_i2c1_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x980, PIN_INPUT_PULLUP | MUX_MODE0)	/* (D16) uart1_rxd.uart1_rxd */
-			AM33XX_IOPAD(0x984, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* (D15) uart1_txd.uart1_txd */
+			0x15C 0x32	/* P9.17,i2c1_scl,INPUT_PULLUP,MODE2 */
+			0x158 0x32	/* P9.18,i2c1_sda,INPUT_PULLUP,MODE2 */
 		>;
 	};
 
-	/* GPS */
-	uart2_pins: pinmux_uart2_pins {
+	i2c2_pins: pinmux_i2c2_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x950, PIN_INPUT_PULLUP | MUX_MODE1)	/* (A17) spi0_sclk.uart2_rxd */
-			AM33XX_IOPAD(0x954, PIN_OUTPUT_PULLDOWN | MUX_MODE1)	/* (B17) spi0_d0.uart2_txd */
+			AM33XX_PADCONF(AM335X_PIN_UART1_CTSN, PIN_INPUT_PULLUP, MUX_MODE3)	/* (D18) uart1_ctsn.I2C2_SDA */
+			AM33XX_PADCONF(AM335X_PIN_UART1_RTSN, PIN_INPUT_PULLUP, MUX_MODE3)	/* (D17) uart1_rtsn.I2C2_SCL */
 		>;
 	};
 
-	/* DSM2 */
-	uart4_pins: pinmux_uart4_pins {
+	/* UT0 */
+	uart0_pins: pinmux_uart0_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x870, PIN_INPUT_PULLUP | MUX_MODE6)	/* (T17) gpmc_wait0.uart4_rxd */
+			AM33XX_PADCONF(AM335X_PIN_UART0_RXD, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_UART0_TXD, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
 		>;
 	};
 
 	/* UT5 */
 	uart5_pins: pinmux_uart5_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x8C4, PIN_INPUT_PULLUP | MUX_MODE4)	/* (U2) lcd_data9.uart5_rxd */
-			AM33XX_IOPAD(0x8C0, PIN_OUTPUT_PULLDOWN | MUX_MODE4)	/* (U1) lcd_data8.uart5_txd */
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA9, PIN_INPUT_PULLUP, MUX_MODE4)	/* (U2) lcd_data9.uart5_rxd */
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA8, PIN_OUTPUT_PULLDOWN, MUX_MODE4)	/* (U1) lcd_data8.uart5_txd */
 		>;
 	};
 
 	mmc1_pins: pinmux_mmc1_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x960, PIN_INPUT | MUX_MODE7)		/* (C15) spi0_cs1.gpio0[6] */
+			AM33XX_PADCONF(AM335X_PIN_SPI0_CS1, PIN_INPUT, MUX_MODE7)		/* (C15) spi0_cs1.gpio0[6] */
 		>;
 	};
 
 	mmc2_pins: pinmux_mmc2_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x880, PIN_INPUT_PULLUP | MUX_MODE2)	/* (U9) gpmc_csn1.mmc1_clk */
-			AM33XX_IOPAD(0x884, PIN_INPUT_PULLUP | MUX_MODE2)	/* (V9) gpmc_csn2.mmc1_cmd */
-			AM33XX_IOPAD(0x800, PIN_INPUT_PULLUP | MUX_MODE1)	/* (U7) gpmc_ad0.mmc1_dat0 */
-			AM33XX_IOPAD(0x804, PIN_INPUT_PULLUP | MUX_MODE1)	/* (V7) gpmc_ad1.mmc1_dat1 */
-			AM33XX_IOPAD(0x808, PIN_INPUT_PULLUP | MUX_MODE1)	/* (R8) gpmc_ad2.mmc1_dat2 */
-			AM33XX_IOPAD(0x80c, PIN_INPUT_PULLUP | MUX_MODE1)	/* (T8) gpmc_ad3.mmc1_dat3 */
-			AM33XX_IOPAD(0x810, PIN_INPUT_PULLUP | MUX_MODE1)	/* (U8) gpmc_ad4.mmc1_dat4 */
-			AM33XX_IOPAD(0x814, PIN_INPUT_PULLUP | MUX_MODE1)	/* (V8) gpmc_ad5.mmc1_dat5 */
-			AM33XX_IOPAD(0x818, PIN_INPUT_PULLUP | MUX_MODE1)	/* (R9) gpmc_ad6.mmc1_dat6 */
-			AM33XX_IOPAD(0x81c, PIN_INPUT_PULLUP | MUX_MODE1)	/* (T9) gpmc_ad7.mmc1_dat7 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_CSN1, PIN_INPUT_PULLUP, MUX_MODE2)	/* (U9) gpmc_csn1.mmc1_clk */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_CSN2, PIN_INPUT_PULLUP, MUX_MODE2)	/* (V9) gpmc_csn2.mmc1_cmd */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD0, PIN_INPUT_PULLUP, MUX_MODE1)	/* (U7) gpmc_ad0.mmc1_dat0 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD1, PIN_INPUT_PULLUP, MUX_MODE1)	/* (V7) gpmc_ad1.mmc1_dat1 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD2, PIN_INPUT_PULLUP, MUX_MODE1)	/* (R8) gpmc_ad2.mmc1_dat2 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD3, PIN_INPUT_PULLUP, MUX_MODE1)	/* (T8) gpmc_ad3.mmc1_dat3 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD4, PIN_INPUT_PULLUP, MUX_MODE1)	/* (U8) gpmc_ad4.mmc1_dat4 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD5, PIN_INPUT_PULLUP, MUX_MODE1)	/* (V8) gpmc_ad5.mmc1_dat5 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD6, PIN_INPUT_PULLUP, MUX_MODE1)	/* (R9) gpmc_ad6.mmc1_dat6 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD7, PIN_INPUT_PULLUP, MUX_MODE1)	/* (T9) gpmc_ad7.mmc1_dat7 */
 		>;
 	};
 
 	mmc3_pins: pinmux_mmc3_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x93c, PIN_INPUT_PULLUP | MUX_MODE6)	/* (L15) gmii1_rxd1.mmc2_clk */
-			AM33XX_IOPAD(0x914, PIN_INPUT_PULLUP | MUX_MODE6)	/* (J16) gmii1_txen.mmc2_cmd */
-			AM33XX_IOPAD(0x918, PIN_INPUT_PULLUP | MUX_MODE5)	/* (J17) gmii1_rxdv.mmc2_dat0 */
-			AM33XX_IOPAD(0x91c, PIN_INPUT_PULLUP | MUX_MODE5)	/* (J18) gmii1_txd3.mmc2_dat1 */
-			AM33XX_IOPAD(0x920, PIN_INPUT_PULLUP | MUX_MODE5)	/* (K15) gmii1_txd2.mmc2_dat2 */
-			AM33XX_IOPAD(0x908, PIN_INPUT_PULLUP | MUX_MODE5)	/* (H16) gmii1_col.mmc2_dat3 */
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLUP, MUX_MODE6)	/* (L15) gmii1_rxd1.mmc2_clk */
+			AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_INPUT_PULLUP, MUX_MODE6)	/* (J16) gmii1_txen.mmc2_cmd */
+			AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLUP, MUX_MODE5)	/* (J17) gmii1_rxdv.mmc2_dat0 */
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_INPUT_PULLUP, MUX_MODE5)	/* (J18) gmii1_txd3.mmc2_dat1 */
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_INPUT_PULLUP, MUX_MODE5)	/* (K15) gmii1_txd2.mmc2_dat2 */
+			AM33XX_PADCONF(AM335X_PIN_MII1_COL, PIN_INPUT_PULLUP, MUX_MODE5)	/* (H16) gmii1_col.mmc2_dat3 */
 		>;
 	};
 
 	bt_pins: pinmux_bt_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x928, PIN_OUTPUT_PULLUP | MUX_MODE7)	/* (K17) gmii1_txd0.gpio0[28] - BT_EN */
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_OUTPUT_PULLUP, MUX_MODE7)	/* (K17) gmii1_txd0.gpio0[28] - BT_EN */
 		>;
 	};
 
 	uart3_pins: pinmux_uart3_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x934, PIN_INPUT_PULLUP | MUX_MODE1)	/* (L17) gmii1_rxd3.uart3_rxd */
-			AM33XX_IOPAD(0x938, PIN_OUTPUT_PULLDOWN | MUX_MODE1)	/* (L16) gmii1_rxd2.uart3_txd */
-			AM33XX_IOPAD(0x948, PIN_INPUT | MUX_MODE3)		/* (M17) mdio_data.uart3_ctsn */
-			AM33XX_IOPAD(0x94c, PIN_OUTPUT_PULLDOWN | MUX_MODE3)	/* (M18) mdio_clk.uart3_rtsn */
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLUP, MUX_MODE1)	/* (L17) gmii1_rxd3.uart3_rxd */
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_OUTPUT_PULLDOWN, MUX_MODE1)	/* (L16) gmii1_rxd2.uart3_txd */
+			AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT, MUX_MODE3)		/* (M17) mdio_data.uart3_ctsn */
+			AM33XX_PADCONF(AM335X_PIN_MDC, PIN_OUTPUT_PULLDOWN, MUX_MODE3)	/* (M18) mdio_clk.uart3_rtsn */
 		>;
 	};
 
 	wl18xx_pins: pinmux_wl18xx_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x92c, PIN_OUTPUT_PULLDOWN | MUX_MODE7)	/* (K18) gmii1_txclk.gpio3[9] - WL_EN */
-			AM33XX_IOPAD(0x924, PIN_INPUT_PULLDOWN | MUX_MODE7)	/* (K16) gmii1_txd1.gpio0[21] - WL_IRQ */
-			AM33XX_IOPAD(0x930, PIN_OUTPUT_PULLUP | MUX_MODE7)	/* (L18) gmii1_rxclk.gpio3[10] - LS_BUF_EN */
+			AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_OUTPUT_PULLDOWN, MUX_MODE7)	/* (K18) gmii1_txclk.gpio3[9] - WL_EN */
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD1, PIN_INPUT_PULLDOWN, MUX_MODE7)	/* (K16) gmii1_txd1.gpio0[21] - WL_IRQ */
+			AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_OUTPUT_PULLUP, MUX_MODE7)	/* (L18) gmii1_rxclk.gpio3[10] - LS_BUF_EN */
 		>;
 	};
 
 	/* DCAN */
 	dcan1_pins: pinmux_dcan1_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x96c, PIN_INPUT | MUX_MODE2)		/* (E17) uart0_rtsn.dcan1_rx */
-			AM33XX_IOPAD(0x968, PIN_OUTPUT | MUX_MODE2)		/* (E18) uart0_ctsn.dcan1_tx */
-			AM33XX_IOPAD(0x940, PIN_OUTPUT | MUX_MODE7)		/* (M16) gmii1_rxd0.gpio2[21] */
+			AM33XX_PADCONF(AM335X_PIN_UART0_RTSN, PIN_INPUT, MUX_MODE2)		/* (E17) uart0_rtsn.dcan1_rx */
+			AM33XX_PADCONF(AM335X_PIN_UART0_CTSN, PIN_OUTPUT, MUX_MODE2)		/* (E18) uart0_ctsn.dcan1_tx */
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD0, PIN_OUTPUT, MUX_MODE7)		/* (M16) gmii1_rxd0.gpio2[21] */
+		>;
+	};
+
+	/* E1 */
+	eqep0_pins: pinmux_eqep0_pins {
+		pinctrl-single,pins = <
+			AM33XX_PADCONF(AM335X_PIN_MCASP0_AXR0, PIN_INPUT, MUX_MODE1)		/* (B12) mcasp0_aclkr.eQEP0A_in */
+			AM33XX_PADCONF(AM335X_PIN_MCASP0_FSR, PIN_INPUT, MUX_MODE1)		/* (C13) mcasp0_fsr.eQEP0B_in */
+		>;
+	};
+
+	/* E2 */
+	eqep1_pins: pinmux_eqep1_pins {
+		pinctrl-single,pins = <
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA12, PIN_INPUT, MUX_MODE2)		/* (V2) lcd_data12.eQEP1A_in */
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA13, PIN_INPUT, MUX_MODE2)		/* (V3) lcd_data13.eQEP1B_in */
+		>;
+	};
+
+	/* E3 */
+	eqep2_pins: pinmux_eqep2_pins {
+		pinctrl-single,pins = <
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD12, PIN_INPUT, MUX_MODE4)		/* (T12) gpmc_ad12.eQEP2A_in */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD13, PIN_INPUT, MUX_MODE4)		/* (R12) gpmc_ad13.eQEP2B_in */
 		>;
 	};
+
+	ehrpwm1_pins: pinmux_ehrpwm1_pins {
+		pinctrl-single,pins = <
+			0x020  0x24 /* P8_19_pwm_pin */
+			0x024  0x24 /* P8_13_pwm_pin */
+		>;
+	};
+
+	ehrpwm2_pins: pinmux_ehrpwm2_pins {
+		pinctrl-single,pins = <
+			0x048  0x26 /* P9_14_pwm_pin */
+			0x04c  0x26 /* P9_16_pwm_pin */
+		>;
+	};
+
+	/* H18 SPI1_SS1     default SPI*/
+	H18_default_pin: pinmux_H18_default_pin {
+		pinctrl-single,pins = < 0x144 ( PIN_OUTPUT | MUX_MODE2 ) >; };
+	H18_gpio_pin: pinmux_H18_gpio_pin {
+		pinctrl-single,pins = < 0x144 ( PIN_OUTPUT | MUX_MODE7 ) >; };
+	H18_gpio_pu_pin: pinmux_H18_gpio_pu_pin {
+		pinctrl-single,pins = < 0x144 ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; };
+	H18_gpio_pd_pin: pinmux_H18_gpio_pd_pin {
+		pinctrl-single,pins = < 0x144 ( PIN_INPUT_PULLDOWN | MUX_MODE7 ) >; };
+	H18_spi_pin: pinmux_H18_spi_pin {
+		pinctrl-single,pins = < 0x144 ( PIN_OUTPUT | MUX_MODE2 ) >; };
+
+
+	/* C18 SPI1_SS2     default spi*/
+	C18_default_pin: pinmux_C18_default_pin {
+		pinctrl-single,pins = < 0x164 ( PIN_OUTPUT | MUX_MODE2 ) >; };
+	C18_gpio_pin: pinmux_C18_gpio_pin {
+		pinctrl-single,pins = < 0x164 ( PIN_OUTPUT | MUX_MODE7 ) >; };
+	C18_gpio_pu_pin: pinmux_C18_gpio_pu_pin {
+		pinctrl-single,pins = < 0x164 ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; };
+	C18_gpio_pd_pin: pinmux_C18_gpio_pd_pin {
+		pinctrl-single,pins = < 0x164 ( PIN_INPUT_PULLDOWN | MUX_MODE7 ) >; };
+	C18_spi_pin: pinmux_C18_spi_pin {
+		pinctrl-single,pins = < 0x164 ( PIN_OUTPUT | MUX_MODE2 ) >; };
+
+
+
+	/* D13 BLUE_GP0_PIN_5 gpio 3_20  */
+	D13_default_pin: pinmux_D13_default_pin {
+		pinctrl-single,pins = < 0x1A8 ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; };
+	D13_gpio_pin: pinmux_D13_gpio_pin {
+		pinctrl-single,pins = < 0x1A8 ( PIN_OUTPUT | MUX_MODE7 ) >; };
+	D13_gpio_pu_pin: pinmux_D13_gpio_pu_pin {
+		pinctrl-single,pins = < 0x1A8 ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; };
+	D13_gpio_pd_pin: pinmux_D13_gpio_pd_pin {
+		pinctrl-single,pins = < 0x1A8 ( PIN_INPUT_PULLDOWN | MUX_MODE7 ) >; };
+
+	/* H17 BLUE_GP1_PIN_4 gpio 3_1  */
+	H17_default_pin: pinmux_H17_default_pin {
+		pinctrl-single,pins = < 0x10C ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; };
+	H17_gpio_pin: pinmux_H17_gpio_pin {
+		pinctrl-single,pins = < 0x10C ( PIN_OUTPUT | MUX_MODE7 ) >; };
+	H17_gpio_pu_pin: pinmux_H17_gpio_pu_pin {
+		pinctrl-single,pins = < 0x10C ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; };
+	H17_gpio_pd_pin: pinmux_H17_gpio_pd_pin {
+		pinctrl-single,pins = < 0x10C ( PIN_INPUT_PULLDOWN | MUX_MODE7 ) >; };
+
+	/* J15 BLUE_GP1_PIN_3 gpio 3_2  */
+	J15_default_pin: pinmux_J15_default_pin {
+		pinctrl-single,pins = < 0x110 ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; };
+	J15_gpio_pin: pinmux_J15_gpio_pin {
+		pinctrl-single,pins = < 0x110 ( PIN_OUTPUT | MUX_MODE7 ) >; };
+	J15_gpio_pu_pin: pinmux_J15_gpio_pu_pin {
+		pinctrl-single,pins = < 0x110 ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; };
+	J15_gpio_pd_pin: pinmux_J15_gpio_pd_pin {
+		pinctrl-single,pins = < 0x110 ( PIN_INPUT_PULLDOWN | MUX_MODE7 ) >; };
+
+
+	/* U16 BLUE_GP0_PIN_3 gpio 1_25   */
+	U16_default_pin: pinmux_U16_default_pin {
+		pinctrl-single,pins = < 0x064 ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; };
+	U16_gpio_pin: pinmux_U16_gpio_pin {
+		pinctrl-single,pins = < 0x064 ( PIN_OUTPUT | MUX_MODE7 ) >; };
+	U16_gpio_pu_pin: pinmux_U16_gpio_pu_pin {
+		pinctrl-single,pins = < 0x064 ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; };
+	U16_gpio_pd_pin: pinmux_U16_gpio_pd_pin {
+		pinctrl-single,pins = < 0x064 ( PIN_INPUT_PULLDOWN | MUX_MODE7 ) >; };
+
+
 };
 
 &uart0 {
@@ -271,23 +458,16 @@
 };
 
 &uart1 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&uart1_pins>;
-
 	status = "okay";
 };
 
 &uart2 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&uart2_pins>;
 
 	status = "okay";
 };
 
 &uart4 {
 	pinctrl-names = "default";
-	pinctrl-0 = <&uart4_pins>;
-
 	status = "okay";
 };
 
@@ -353,32 +533,19 @@
 	};
 };
 
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_pins>;
+	status = "okay";
+	clock-frequency = <400000>;
+};
+
 &i2c2 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&i2c2_pins>;
 
 	status = "okay";
 	clock-frequency = <400000>;
-
-	mpu9250@68 {
-		compatible = "invensense,mpu9250";
-		reg = <0x68>;
-		interrupt-parent = <&gpio3>;
-		interrupts = <21 IRQ_TYPE_EDGE_RISING>;
-		i2c-gate {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			ax8975@c {
-				compatible = "ak,ak8975";
-				reg = <0x0c>;
-			};
-		};
-	};
-
-	pressure@76 {
-		compatible = "bosch,bmp280";
-		reg = <0x76>;
-	};
 };
 
 /include/ "tps65217.dtsi"
@@ -387,6 +554,8 @@
 	interrupts = <7>; /* NMI */
 	interrupt-parent = <&intc>;
 
+	ti,pmic-shutdown-controller;
+
 	charger {
 		interrupts = <0>, <1>;
 		interrupt-names = "USB", "AC";
@@ -487,22 +656,23 @@
 	};
 };
 
-&tscadc {
-	status = "okay";
-	adc {
-		ti,adc-channels = <0 1 2 3 4 5 6 7>;
-	};
-};
+//&tscadc {
+//	status = "okay";
+//	adc {
+//		ti,adc-channels = <0 1 2 3 4 5 6 7>;
+//	};
+//};
 
 &uart3 {
 	pinctrl-names = "default";
-	pinctrl-0 = <&uart3_pins &bt_pins>;
+	//pinctrl-0 = <&uart3_pins &bt_pins>;
+	pinctrl-0 = <&uart3_pins>;
 	status = "okay";
 
-	bluetooth {
-		compatible = "ti,wl1835-st";
-		enable-gpios = <&gpio0 28 GPIO_ACTIVE_HIGH>;
-	};
+//	bluetooth {
+//		compatible = "ti,wl1835-st";
+//		enable-gpios = <&gpio0 28 GPIO_ACTIVE_HIGH>;
+//	};
 };
 
 &aes {
@@ -515,10 +685,20 @@
 
 &rtc {
 	system-power-controller;
-	clocks = <&clk_32768_ck>, <&l4_per_clkctrl AM3_CLKDIV32K_CLKCTRL 0>;
-	clock-names = "ext-clk", "int-clk";
 };
 
+&wkup_m3_ipc {
+	ti,scale-data-fw = "am335x-bone-scale-data.bin";
+};
+
+//&pruss_soc_bus {
+//	status = "okay";
+//
+//	pruss: pruss@4a300000 {
+//		status = "okay";
+//	};
+//};
+
 &dcan1 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&dcan1_pins>;
@@ -533,3 +713,360 @@
 		line-name = "LS_BUF_EN";
 	};
 };
+
+&ocp {
+	/* activate the static pinmux helper list of pin modes above */
+	test_helper: helper {
+		compatible = "bone-pinmux-helper";
+		pinctrl-names = "default";
+		pinctrl-0 = <&mux_helper_pins>;
+
+		status = "okay";
+	};
+
+
+	/* UART 2 RX GPS */
+	P9_22_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "spi", "uart", "i2c", "pwm";
+		pinctrl-0 = <&P9_22_uart_pin>;
+		pinctrl-1 = <&P9_22_gpio_pin>;
+		pinctrl-2 = <&P9_22_gpio_pu_pin>;
+		pinctrl-3 = <&P9_22_gpio_pd_pin>;
+		pinctrl-4 = <&P9_22_spi_pin>;
+		pinctrl-5 = <&P9_22_uart_pin>;
+		pinctrl-6 = <&P9_22_i2c_pin>;
+		pinctrl-7 = <&P9_22_pwm_pin>;
+	};
+
+	/* UART 2 TX GPS*/
+	P9_21_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "spi", "uart", "i2c", "pwm";
+		pinctrl-0 = <&P9_21_uart_pin>;
+		pinctrl-1 = <&P9_21_gpio_pin>;
+		pinctrl-2 = <&P9_21_gpio_pu_pin>;
+		pinctrl-3 = <&P9_21_gpio_pd_pin>;
+		pinctrl-4 = <&P9_21_spi_pin>;
+		pinctrl-5 = <&P9_21_uart_pin>;
+		pinctrl-6 = <&P9_21_i2c_pin>;
+		pinctrl-7 = <&P9_21_pwm_pin>;
+	};
+
+	/* UART 1 RX */
+	P9_26_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "uart", "can", "i2c",  "pruin";
+		pinctrl-0 = <&P9_26_uart_pin>;
+		pinctrl-1 = <&P9_26_gpio_pin>;
+		pinctrl-2 = <&P9_26_gpio_pu_pin>;
+		pinctrl-3 = <&P9_26_gpio_pd_pin>;
+		pinctrl-4 = <&P9_26_uart_pin>;
+		pinctrl-5 = <&P9_26_can_pin>;
+		pinctrl-6 = <&P9_26_i2c_pin>;
+		pinctrl-7 = <&P9_26_pruin_pin>;
+	};
+
+	/* UART 1 TX */
+	 P9_24_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "uart", "can", "i2c",  "pruin";
+		pinctrl-0 = <&P9_24_uart_pin>;
+		pinctrl-1 = <&P9_24_gpio_pin>;
+		pinctrl-2 = <&P9_24_gpio_pu_pin>;
+		pinctrl-3 = <&P9_24_gpio_pd_pin>;
+		pinctrl-4 = <&P9_24_uart_pin>;
+		pinctrl-5 = <&P9_24_can_pin>;
+		pinctrl-6 = <&P9_24_i2c_pin>;
+		pinctrl-7 = <&P9_24_pruin_pin>;
+	};
+
+	/* SPI MOSI */
+	P9_30_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm", "spi", "pruout", "pruin";
+		pinctrl-0 = <&P9_30_spi_pin>;
+		pinctrl-1 = <&P9_30_gpio_pin>;
+		pinctrl-2 = <&P9_30_gpio_pu_pin>;
+		pinctrl-3 = <&P9_30_gpio_pd_pin>;
+		pinctrl-4 = <&P9_30_pwm_pin>;
+		pinctrl-5 = <&P9_30_spi_pin>;
+		pinctrl-6 = <&P9_30_pruout_pin>;
+		pinctrl-7 = <&P9_30_pruin_pin>;
+	};
+
+	/* SPI MISO */
+	P9_29_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm", "spi", "pruout", "pruin";
+		pinctrl-0 = <&P9_29_spi_pin>;
+		pinctrl-1 = <&P9_29_gpio_pin>;
+		pinctrl-2 = <&P9_29_gpio_pu_pin>;
+		pinctrl-3 = <&P9_29_gpio_pd_pin>;
+		pinctrl-4 = <&P9_29_pwm_pin>;
+		pinctrl-5 = <&P9_29_spi_pin>;
+		pinctrl-6 = <&P9_29_pruout_pin>;
+		pinctrl-7 = <&P9_29_pruin_pin>;
+	};
+
+	/* SPI SCK */
+	P9_31_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm", "spi", "spi_sclk", "pruout", "pruin";
+		pinctrl-0 = <&P9_31_spi_sclk_pin>;
+		pinctrl-1 = <&P9_31_gpio_pin>;
+		pinctrl-2 = <&P9_31_gpio_pu_pin>;
+		pinctrl-3 = <&P9_31_gpio_pd_pin>;
+		pinctrl-4 = <&P9_31_pwm_pin>;
+		pinctrl-5 = <&P9_31_spi_pin>;
+		pinctrl-6 = <&P9_31_spi_sclk_pin>;
+		pinctrl-7 = <&P9_31_pruout_pin>;
+		pinctrl-8 = <&P9_31_pruin_pin>;
+	};
+
+	/* Encoder 4 (U13) */
+	P8_15_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "pruin_pu", "gpio", "gpio_pu", "gpio_pd", "pruin", "qep", "pruecapin_pu";
+		pinctrl-0 = <&P8_15_pruin_pu_pin>;
+		pinctrl-1 = <&P8_15_pruin_pu_pin>;
+		pinctrl-2 = <&P8_15_gpio_pin>;
+		pinctrl-3 = <&P8_15_gpio_pu_pin>;
+		pinctrl-4 = <&P8_15_gpio_pd_pin>;
+		pinctrl-5 = <&P8_15_pruin_pin>;
+		pinctrl-6 = <&P8_15_qep_pin>;
+		pinctrl-7 = <&P8_15_pruecapin_pu_pin>;
+	};
+
+	/* UART4 RX DSM */
+	P9_11_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "uart", "gpio", "gpio_pu", "gpio_pd";
+		pinctrl-0 = <&P9_11_uart_pin>;
+		pinctrl-1 = <&P9_11_uart_pin>;
+		pinctrl-2 = <&P9_11_gpio_pin>;
+		pinctrl-3 = <&P9_11_gpio_pu_pin>;
+		pinctrl-4 = <&P9_11_gpio_pd_pin>;
+	};
+
+	/* U16 BLUE_GP0_PIN_3 gpio 1_25*/
+	U16_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd";
+		pinctrl-0 = <&U16_default_pin>;
+		pinctrl-1 = <&U16_gpio_pin>;
+		pinctrl-2 = <&U16_gpio_pu_pin>;
+		pinctrl-3 = <&U16_gpio_pd_pin>;
+	};
+
+
+	/* BLUE_GP0_PIN_3 gpio1_17*/
+	P9_23_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm";
+		pinctrl-0 = <&P9_23_default_pin>;
+		pinctrl-1 = <&P9_23_gpio_pin>;
+		pinctrl-2 = <&P9_23_gpio_pu_pin>;
+		pinctrl-3 = <&P9_23_gpio_pd_pin>;
+		pinctrl-4 = <&P9_23_pwm_pin>;
+	};
+
+	/* BLUE_GP0_PIN_5 gpio3_20 */
+	D13_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd";
+		pinctrl-0 = <&D13_default_pin>;
+		pinctrl-1 = <&D13_gpio_pin>;
+		pinctrl-2 = <&D13_gpio_pu_pin>;
+		pinctrl-3 = <&D13_gpio_pd_pin>;
+	};
+
+	/* BLUE_GP0_PIN_6 gpio3_17 */
+	P9_28_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm", "spi", "spi_cs", "pwm2", "pruout", "pruin";
+		pinctrl-0 = <&P9_28_default_pin>;
+		pinctrl-1 = <&P9_28_gpio_pin>;
+		pinctrl-2 = <&P9_28_gpio_pu_pin>;
+		pinctrl-3 = <&P9_28_gpio_pd_pin>;
+		pinctrl-4 = <&P9_28_pwm_pin>;
+		pinctrl-5 = <&P9_28_spi_pin>;
+		pinctrl-6 = <&P9_28_spi_cs_pin>;
+		pinctrl-7 = <&P9_28_pwm2_pin>;
+		pinctrl-8 = <&P9_28_pruout_pin>;
+		pinctrl-9 = <&P9_28_pruin_pin>;
+	};
+
+	/* BLUE_GP1_PIN_3 gpio3_2 */
+	J15_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd";
+		pinctrl-0 = <&J15_default_pin>;
+		pinctrl-1 = <&J15_gpio_pin>;
+		pinctrl-2 = <&J15_gpio_pu_pin>;
+		pinctrl-3 = <&J15_gpio_pd_pin>;
+	};
+
+	/* BLUE_GP1_PIN_4 gpio3_1 */
+	H17_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd";
+		pinctrl-0 = <&H17_default_pin>;
+		pinctrl-1 = <&H17_gpio_pin>;
+		pinctrl-2 = <&H17_gpio_pu_pin>;
+		pinctrl-3 = <&H17_gpio_pd_pin>;
+	};
+
+
+	////////////////////////////////////////////////////////////////////////
+	// Pinmux options only for BB BLue
+	////////////////////////////////////////////////////////////////////////
+
+
+
+	/* H18 SPI1_SS1     default SPI*/
+	H18_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "spi";
+		pinctrl-0 = <&H18_spi_pin>;
+		pinctrl-1 = <&H18_gpio_pin>;
+		pinctrl-2 = <&H18_gpio_pu_pin>;
+		pinctrl-3 = <&H18_gpio_pd_pin>;
+		pinctrl-4 = <&H18_spi_pin>;
+	};
+
+
+	/* C18 SPI1_SS2     default spi*/
+	C18_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "spi";
+		pinctrl-0 = <&C18_spi_pin>;
+		pinctrl-1 = <&C18_gpio_pin>;
+		pinctrl-2 = <&C18_gpio_pu_pin>;
+		pinctrl-3 = <&C18_gpio_pd_pin>;
+		pinctrl-4 = <&C18_spi_pin>;
+	};
+
+};
+
+/*******************************************************************************
+*	PWMSS
+*******************************************************************************/
+&epwmss0 {
+	status = "okay";
+};
+
+&epwmss1 {
+	status = "okay";
+};
+
+&epwmss2 {
+	status = "okay";
+};
+
+&ehrpwm0 {
+	status = "okay";
+};
+
+&ehrpwm1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&ehrpwm1_pins>;
+	status = "okay";
+};
+
+&ehrpwm2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&ehrpwm2_pins>;
+	status = "okay";
+};
+
+&ecap0 { status="okay"; };
+&ecap1 { status="okay"; };
+&ecap2 { status="okay"; };
+
+/*******************************************************************************
+* EQEP
+*******************************************************************************/
+&eqep0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&eqep0_pins>;
+
+	count_mode = <0>;  /* 0 - Quadrature mode, normal 90 phase offset cha & chb.  1 - Direction mode.  cha input = clock, chb input = direction */
+	swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */
+	invert_qa = <1>;   /* Should we invert the channel A input?  */
+	invert_qb = <1>;   /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */
+	invert_qi = <0>;   /* Should we invert the index input? */
+	invert_qs = <0>;   /* Should we invert the strobe input? */
+	status = "okay";
+};
+
+&eqep1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&eqep1_pins>;
+
+	count_mode = <0>;  /* 0 - Quadrature mode, normal 90 phase offset cha & chb.  1 - Direction mode.  cha input = clock, chb input = direction */
+	swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */
+	invert_qa = <1>;   /* Should we invert the channel A input?  */
+	invert_qb = <1>;   /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */
+	invert_qi = <0>;   /* Should we invert the index input? */
+	invert_qs = <0>;   /* Should we invert the strobe input? */
+	status = "okay";
+};
+
+&eqep2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&eqep2_pins>;
+
+	count_mode = <0>;  /* 0 - Quadrature mode, normal 90 phase offset cha & chb.  1 - Direction mode.  cha input = clock, chb input = direction */
+	swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */
+	invert_qa = <1>;   /* Should we invert the channel A input?  */
+	invert_qb = <1>;   /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */
+	invert_qi = <0>;   /* Should we invert the index input? */
+	invert_qs = <0>;   /* Should we invert the strobe input? */
+	status = "okay";
+};
+
+/*******************************************************************************
+	SPI
+*******************************************************************************/
+&spi1 {
+
+	#address-cells = <1>;
+	#size-cells = <0>;
+	status = "okay";
+
+	channel@0 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "spidev";
+		symlink = "spi/1.0";
+		reg = <0>;
+		spi-max-frequency = <24000000>;
+	};
+
+	channel@1 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "spidev";
+		symlink = "spi/1.1";
+		reg = <1>;
+		spi-max-frequency = <24000000>;
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-bone-common.dtsi b/arch/arm/boot/dts/am335x-bone-common.dtsi
--- a/arch/arm/boot/dts/am335x-bone-common.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-bone-common.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -1,9 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 / {
@@ -29,14 +26,14 @@
 		compatible = "gpio-leds";
 
 		led2 {
-			label = "beaglebone:green:heartbeat";
+			label = "beaglebone:green:usr0";
 			gpios = <&gpio1 21 GPIO_ACTIVE_HIGH>;
 			linux,default-trigger = "heartbeat";
 			default-state = "off";
 		};
 
 		led3 {
-			label = "beaglebone:green:mmc0";
+			label = "beaglebone:green:usr1";
 			gpios = <&gpio1 22 GPIO_ACTIVE_HIGH>;
 			linux,default-trigger = "mmc0";
 			default-state = "off";
@@ -66,125 +63,160 @@
 };
 
 &am33xx_pinmux {
-	pinctrl-names = "default";
-	pinctrl-0 = <&clkout2_pin>;
-
 	user_leds_s0: user_leds_s0 {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x854, PIN_OUTPUT_PULLDOWN | MUX_MODE7)	/* gpmc_a5.gpio1_21 */
-			AM33XX_IOPAD(0x858, PIN_OUTPUT_PULLUP | MUX_MODE7)	/* gpmc_a6.gpio1_22 */
-			AM33XX_IOPAD(0x85c, PIN_OUTPUT_PULLDOWN | MUX_MODE7)	/* gpmc_a7.gpio1_23 */
-			AM33XX_IOPAD(0x860, PIN_OUTPUT_PULLUP | MUX_MODE7)	/* gpmc_a8.gpio1_24 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_A5, PIN_OUTPUT_PULLDOWN, MUX_MODE7)	/* gpmc_a5.gpio1_21 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_A6, PIN_OUTPUT_PULLUP, MUX_MODE7)	/* gpmc_a6.gpio1_22 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_A7, PIN_OUTPUT_PULLDOWN, MUX_MODE7)	/* gpmc_a7.gpio1_23 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_A8, PIN_OUTPUT_PULLUP, MUX_MODE7)	/* gpmc_a8.gpio1_24 */
 		>;
 	};
 
 	i2c0_pins: pinmux_i2c0_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x988, PIN_INPUT_PULLUP | MUX_MODE0)	/* i2c0_sda.i2c0_sda */
-			AM33XX_IOPAD(0x98c, PIN_INPUT_PULLUP | MUX_MODE0)	/* i2c0_scl.i2c0_scl */
+			AM33XX_PADCONF(AM335X_PIN_I2C0_SDA, PIN_INPUT_PULLUP, MUX_MODE0)	/* i2c0_sda.i2c0_sda */
+			AM33XX_PADCONF(AM335X_PIN_I2C0_SCL, PIN_INPUT_PULLUP, MUX_MODE0)	/* i2c0_scl.i2c0_scl */
 		>;
 	};
 
-	i2c2_pins: pinmux_i2c2_pins {
-		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x978, PIN_INPUT_PULLUP | MUX_MODE3)	/* uart1_ctsn.i2c2_sda */
-			AM33XX_IOPAD(0x97c, PIN_INPUT_PULLUP | MUX_MODE3)	/* uart1_rtsn.i2c2_scl */
-		>;
-	};
+//	i2c2_pins: pinmux_i2c2_pins {
+//		pinctrl-single,pins = <
+//			AM33XX_PADCONF(AM335X_PIN_UART1_CTSN, PIN_INPUT_PULLUP, MUX_MODE3)	/* uart1_ctsn.i2c2_sda */
+//			AM33XX_PADCONF(AM335X_PIN_UART1_RTSN, PIN_INPUT_PULLUP, MUX_MODE3)	/* uart1_rtsn.i2c2_scl */
+//		>;
+//	};
 
 	uart0_pins: pinmux_uart0_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x970, PIN_INPUT_PULLUP | MUX_MODE0)	/* uart0_rxd.uart0_rxd */
-			AM33XX_IOPAD(0x974, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* uart0_txd.uart0_txd */
-		>;
-	};
-
-	clkout2_pin: pinmux_clkout2_pin {
-		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x9b4, PIN_OUTPUT_PULLDOWN | MUX_MODE3)	/* xdma_event_intr1.clkout2 */
+			AM33XX_PADCONF(AM335X_PIN_UART0_RXD, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_UART0_TXD, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
 		>;
 	};
 
 	cpsw_default: cpsw_default {
 		pinctrl-single,pins = <
 			/* Slave 1 */
-			AM33XX_IOPAD(0x910, PIN_INPUT_PULLUP | MUX_MODE0)	/* mii1_rxerr.mii1_rxerr */
-			AM33XX_IOPAD(0x914, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* mii1_txen.mii1_txen */
-			AM33XX_IOPAD(0x918, PIN_INPUT_PULLUP | MUX_MODE0)	/* mii1_rxdv.mii1_rxdv */
-			AM33XX_IOPAD(0x91c, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* mii1_txd3.mii1_txd3 */
-			AM33XX_IOPAD(0x920, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* mii1_txd2.mii1_txd2 */
-			AM33XX_IOPAD(0x924, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* mii1_txd1.mii1_txd1 */
-			AM33XX_IOPAD(0x928, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* mii1_txd0.mii1_txd0 */
-			AM33XX_IOPAD(0x92c, PIN_INPUT_PULLUP | MUX_MODE0)	/* mii1_txclk.mii1_txclk */
-			AM33XX_IOPAD(0x930, PIN_INPUT_PULLUP | MUX_MODE0)	/* mii1_rxclk.mii1_rxclk */
-			AM33XX_IOPAD(0x934, PIN_INPUT_PULLUP | MUX_MODE0)	/* mii1_rxd3.mii1_rxd3 */
-			AM33XX_IOPAD(0x938, PIN_INPUT_PULLUP | MUX_MODE0)	/* mii1_rxd2.mii1_rxd2 */
-			AM33XX_IOPAD(0x93c, PIN_INPUT_PULLUP | MUX_MODE0)	/* mii1_rxd1.mii1_rxd1 */
-			AM33XX_IOPAD(0x940, PIN_INPUT_PULLUP | MUX_MODE0)	/* mii1_rxd0.mii1_rxd0 */
+			AM33XX_PADCONF(AM335X_PIN_MII1_RX_ER, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD1, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD0, PIN_INPUT_PULLUP, MUX_MODE0)
 		>;
 	};
 
 	cpsw_sleep: cpsw_sleep {
 		pinctrl-single,pins = <
 			/* Slave 1 reset value */
-			AM33XX_IOPAD(0x910, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x914, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x918, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x91c, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x920, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x924, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x928, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x92c, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x930, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x934, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x938, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x93c, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x940, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RX_ER, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD1, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD0, PIN_INPUT_PULLDOWN, MUX_MODE7)
 		>;
 	};
 
 	davinci_mdio_default: davinci_mdio_default {
 		pinctrl-single,pins = <
 			/* MDIO */
-			AM33XX_IOPAD(0x948, PIN_INPUT_PULLUP | SLEWCTRL_FAST | MUX_MODE0)	/* mdio_data.mdio_data */
-			AM33XX_IOPAD(0x94c, PIN_OUTPUT_PULLUP | MUX_MODE0)			/* mdio_clk.mdio_clk */
+			AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT_PULLUP | SLEWCTRL_FAST, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MDC, PIN_OUTPUT_PULLUP, MUX_MODE0)
 		>;
 	};
 
 	davinci_mdio_sleep: davinci_mdio_sleep {
 		pinctrl-single,pins = <
 			/* MDIO reset value */
-			AM33XX_IOPAD(0x948, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x94c, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MDC, PIN_INPUT_PULLDOWN, MUX_MODE7)
 		>;
 	};
 
 	mmc1_pins: pinmux_mmc1_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x960, PIN_INPUT | MUX_MODE7)		/* spio0_cs1.gpio0_6 */
-			AM33XX_IOPAD(0x8fc, PIN_INPUT_PULLUP | MUX_MODE0)	/* mmc0_dat0.mmc0_dat0 */
-			AM33XX_IOPAD(0x8f8, PIN_INPUT_PULLUP | MUX_MODE0)	/* mmc0_dat1.mmc0_dat1 */
-			AM33XX_IOPAD(0x8f4, PIN_INPUT_PULLUP | MUX_MODE0)	/* mmc0_dat2.mmc0_dat2 */
-			AM33XX_IOPAD(0x8f0, PIN_INPUT_PULLUP | MUX_MODE0)	/* mmc0_dat3.mmc0_dat3 */
-			AM33XX_IOPAD(0x904, PIN_INPUT_PULLUP | MUX_MODE0)	/* mmc0_cmd.mmc0_cmd */
-			AM33XX_IOPAD(0x900, PIN_INPUT_PULLUP | MUX_MODE0)	/* mmc0_clk.mmc0_clk */
+			AM33XX_PADCONF(AM335X_PIN_SPI0_CS1, PIN_INPUT, MUX_MODE7)		/* spio0_cs1.gpio0_6 */
+			AM33XX_PADCONF(AM335X_PIN_MMC0_DAT0, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MMC0_DAT1, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MMC0_DAT2, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MMC0_DAT3, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MMC0_CMD, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MMC0_CLK, PIN_INPUT_PULLUP, MUX_MODE0)
 		>;
 	};
 
 	emmc_pins: pinmux_emmc_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x880, PIN_INPUT_PULLUP | MUX_MODE2) /* gpmc_csn1.mmc1_clk */
-			AM33XX_IOPAD(0x884, PIN_INPUT_PULLUP | MUX_MODE2) /* gpmc_csn2.mmc1_cmd */
-			AM33XX_IOPAD(0x800, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad0.mmc1_dat0 */
-			AM33XX_IOPAD(0x804, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad1.mmc1_dat1 */
-			AM33XX_IOPAD(0x808, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad2.mmc1_dat2 */
-			AM33XX_IOPAD(0x80c, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad3.mmc1_dat3 */
-			AM33XX_IOPAD(0x810, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad4.mmc1_dat4 */
-			AM33XX_IOPAD(0x814, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad5.mmc1_dat5 */
-			AM33XX_IOPAD(0x818, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad6.mmc1_dat6 */
-			AM33XX_IOPAD(0x81c, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad7.mmc1_dat7 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_CSN1, PIN_INPUT_PULLUP, MUX_MODE2) /* gpmc_csn1.mmc1_clk */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_CSN2, PIN_INPUT_PULLUP, MUX_MODE2) /* gpmc_csn2.mmc1_cmd */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD0, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad0.mmc1_dat0 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD1, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad1.mmc1_dat1 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD2, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad2.mmc1_dat2 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD3, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad3.mmc1_dat3 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD4, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad4.mmc1_dat4 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD5, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad5.mmc1_dat5 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD6, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad6.mmc1_dat6 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD7, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad7.mmc1_dat7 */
 		>;
 	};
+
+	/* P9_19 (ZCZ ball D17) i2c2_scl */
+	P9_19_default_pin: pinmux_P9_19_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_INPUT_PULLUP | MUX_MODE3) >; };	/* uart1_rtsn.i2c2_scl */
+	P9_19_gpio_pin: pinmux_P9_19_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* uart1_rtsn.gpio0_13 */
+	P9_19_gpio_pu_pin: pinmux_P9_19_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* uart1_rtsn.gpio0_13 */
+	P9_19_gpio_pd_pin: pinmux_P9_19_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* uart1_rtsn.gpio0_13 */
+	P9_19_gpio_input_pin: pinmux_P9_19_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_INPUT | MUX_MODE7) >; };			/* uart1_rtsn.gpio0_13 */
+	P9_19_timer_pin: pinmux_P9_19_timer_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* uart1_rtsn.timer5 */
+	P9_19_can_pin: pinmux_P9_19_can_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_INPUT_PULLUP | MUX_MODE2) >; };		/* uart1_rtsn.dcan0_rx */
+	P9_19_i2c_pin: pinmux_P9_19_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_INPUT_PULLUP | MUX_MODE3) >; };	/* uart1_rtsn.i2c2_scl */
+	P9_19_spi_cs_pin: pinmux_P9_19_spi_cs_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* uart1_rtsn.spi1_cs1 */
+	P9_19_pru_uart_pin: pinmux_P9_19_pru_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; };	/* uart1_rtsn.pr1_uart0_rts_n */
+
+	/* P9_20 (ZCZ ball D18) i2c2_sda */
+	P9_20_default_pin: pinmux_P9_20_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_INPUT_PULLUP | MUX_MODE3) >; };	/* uart1_ctsn.i2c2_sda */
+	P9_20_gpio_pin: pinmux_P9_20_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* uart1_ctsn.gpio0_12 */
+	P9_20_gpio_pu_pin: pinmux_P9_20_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* uart1_ctsn.gpio0_12 */
+	P9_20_gpio_pd_pin: pinmux_P9_20_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* uart1_ctsn.gpio0_12 */
+	P9_20_gpio_input_pin: pinmux_P9_20_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_INPUT | MUX_MODE7) >; };			/* uart1_ctsn.gpio0_12 */
+	P9_20_timer_pin: pinmux_P9_20_timer_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* uart1_ctsn.timer6 */
+	P9_20_can_pin: pinmux_P9_20_can_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | MUX_MODE2) >; };		/* uart1_ctsn.dcan0_tx */
+	P9_20_i2c_pin: pinmux_P9_20_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_INPUT_PULLUP | MUX_MODE3) >; };	/* uart1_ctsn.i2c2_sda */
+	P9_20_spi_cs_pin: pinmux_P9_20_spi_cs_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* uart1_ctsn.spi1_cs0 */
+	P9_20_pru_uart_pin: pinmux_P9_20_pru_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; };	/* uart1_ctsn.pr1_uart0_cts_n */
 };
 
 &uart0 {
@@ -251,7 +283,8 @@
 
 &i2c2 {
 	pinctrl-names = "default";
-	pinctrl-0 = <&i2c2_pins>;
+	//pinctrl-0 = <&i2c2_pins>;
+	pinctrl-0 = <>;
 
 	status = "okay";
 	clock-frequency = <100000>;
@@ -417,4 +450,76 @@
 &rtc {
 	clocks = <&clk_32768_ck>, <&l4_per_clkctrl AM3_CLKDIV32K_CLKCTRL 0>;
 	clock-names = "ext-clk", "int-clk";
+	system-power-controller;
+};
+
+&wkup_m3_ipc {
+	ti,scale-data-fw = "am335x-bone-scale-data.bin";
+};
+
+//&pruss_soc_bus {
+//	status = "okay";
+//
+//	pruss: pruss@4a300000 {
+//		status = "okay";
+//	};
+//};
+
+&ocp {
+	/* P9_19 (ZCZ ball D17) i2c */
+	P9_19_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart", "timer";
+		pinctrl-0 = <&P9_19_default_pin>;
+		pinctrl-1 = <&P9_19_gpio_pin>;
+		pinctrl-2 = <&P9_19_gpio_pu_pin>;
+		pinctrl-3 = <&P9_19_gpio_pd_pin>;
+		pinctrl-4 = <&P9_19_gpio_input_pin>;
+		pinctrl-5 = <&P9_19_spi_cs_pin>;
+		pinctrl-6 = <&P9_19_can_pin>;
+		pinctrl-7 = <&P9_19_i2c_pin>;
+		pinctrl-8 = <&P9_19_pru_uart_pin>;
+		pinctrl-9 = <&P9_19_timer_pin>;
+	};
+
+	/* P9_20 (ZCZ ball D18) i2c */
+	P9_20_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart", "timer";
+		pinctrl-0 = <&P9_20_default_pin>;
+		pinctrl-1 = <&P9_20_gpio_pin>;
+		pinctrl-2 = <&P9_20_gpio_pu_pin>;
+		pinctrl-3 = <&P9_20_gpio_pd_pin>;
+		pinctrl-4 = <&P9_20_gpio_input_pin>;
+		pinctrl-5 = <&P9_20_spi_cs_pin>;
+		pinctrl-6 = <&P9_20_can_pin>;
+		pinctrl-7 = <&P9_20_i2c_pin>;
+		pinctrl-8 = <&P9_20_pru_uart_pin>;
+		pinctrl-9 = <&P9_20_timer_pin>;
+	};
+
+
+	cape-universal {
+		compatible = "gpio-of-helper";
+		status = "okay";
+		pinctrl-names = "default";
+		pinctrl-0 = <>;
+
+		P9_19 {
+			gpio-name = "P9_19";
+			gpio = <&gpio0 13 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_20 {
+			gpio-name = "P9_20";
+			gpio = <&gpio0 12 0>;
+			input;
+			dir-changeable;
+		};
+
+	};
 };
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-bone-common-no-capemgr.dtsi b/arch/arm/boot/dts/am335x-bone-common-no-capemgr.dtsi
--- a/arch/arm/boot/dts/am335x-bone-common-no-capemgr.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-bone-common-no-capemgr.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,381 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/ {
+	cpus {
+		cpu@0 {
+			cpu0-supply = <&dcdc2_reg>;
+		};
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x80000000 0x10000000>; /* 256 MB */
+	};
+
+	chosen {
+		stdout-path = &uart0;
+	};
+
+	leds {
+		pinctrl-names = "default";
+		pinctrl-0 = <&user_leds_s0>;
+
+		compatible = "gpio-leds";
+
+		led2 {
+			label = "beaglebone:green:usr0";
+			gpios = <&gpio1 21 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "heartbeat";
+			default-state = "off";
+		};
+
+		led3 {
+			label = "beaglebone:green:usr1";
+			gpios = <&gpio1 22 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "mmc0";
+			default-state = "off";
+		};
+
+		led4 {
+			label = "beaglebone:green:usr2";
+			gpios = <&gpio1 23 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "cpu0";
+			default-state = "off";
+		};
+
+		led5 {
+			label = "beaglebone:green:usr3";
+			gpios = <&gpio1 24 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "mmc1";
+			default-state = "off";
+		};
+	};
+
+	vmmcsd_fixed: fixedregulator0 {
+		compatible = "regulator-fixed";
+		regulator-name = "vmmcsd_fixed";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+};
+
+&am33xx_pinmux {
+	user_leds_s0: user_leds_s0 {
+		pinctrl-single,pins = <
+			AM33XX_PADCONF(AM335X_PIN_GPMC_A5, PIN_OUTPUT_PULLDOWN, MUX_MODE7)	/* gpmc_a5.gpio1_21 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_A6, PIN_OUTPUT_PULLUP, MUX_MODE7)	/* gpmc_a6.gpio1_22 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_A7, PIN_OUTPUT_PULLDOWN, MUX_MODE7)	/* gpmc_a7.gpio1_23 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_A8, PIN_OUTPUT_PULLUP, MUX_MODE7)	/* gpmc_a8.gpio1_24 */
+		>;
+	};
+
+	i2c0_pins: pinmux_i2c0_pins {
+		pinctrl-single,pins = <
+			AM33XX_PADCONF(AM335X_PIN_I2C0_SDA, PIN_INPUT_PULLUP, MUX_MODE0)	/* i2c0_sda.i2c0_sda */
+			AM33XX_PADCONF(AM335X_PIN_I2C0_SCL, PIN_INPUT_PULLUP, MUX_MODE0)	/* i2c0_scl.i2c0_scl */
+		>;
+	};
+
+	i2c2_pins: pinmux_i2c2_pins {
+		pinctrl-single,pins = <
+			AM33XX_PADCONF(AM335X_PIN_UART1_CTSN, PIN_INPUT_PULLUP, MUX_MODE3)	/* uart1_ctsn.i2c2_sda */
+			AM33XX_PADCONF(AM335X_PIN_UART1_RTSN, PIN_INPUT_PULLUP, MUX_MODE3)	/* uart1_rtsn.i2c2_scl */
+		>;
+	};
+
+	uart0_pins: pinmux_uart0_pins {
+		pinctrl-single,pins = <
+			AM33XX_PADCONF(AM335X_PIN_UART0_RXD, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_UART0_TXD, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
+		>;
+	};
+
+	cpsw_default: cpsw_default {
+		pinctrl-single,pins = <
+			/* Slave 1 */
+			AM33XX_PADCONF(AM335X_PIN_MII1_RX_ER, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD1, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD0, PIN_INPUT_PULLUP, MUX_MODE0)
+		>;
+	};
+
+	cpsw_sleep: cpsw_sleep {
+		pinctrl-single,pins = <
+			/* Slave 1 reset value */
+			AM33XX_PADCONF(AM335X_PIN_MII1_RX_ER, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD1, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD0, PIN_INPUT_PULLDOWN, MUX_MODE7)
+		>;
+	};
+
+	davinci_mdio_default: davinci_mdio_default {
+		pinctrl-single,pins = <
+			/* MDIO */
+			AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT_PULLUP | SLEWCTRL_FAST, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MDC, PIN_OUTPUT_PULLUP, MUX_MODE0)
+		>;
+	};
+
+	davinci_mdio_sleep: davinci_mdio_sleep {
+		pinctrl-single,pins = <
+			/* MDIO reset value */
+			AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MDC, PIN_INPUT_PULLDOWN, MUX_MODE7)
+		>;
+	};
+
+	mmc1_pins: pinmux_mmc1_pins {
+		pinctrl-single,pins = <
+			AM33XX_PADCONF(AM335X_PIN_SPI0_CS1, PIN_INPUT, MUX_MODE7)		/* spio0_cs1.gpio0_6 */
+			AM33XX_PADCONF(AM335X_PIN_MMC0_DAT0, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MMC0_DAT1, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MMC0_DAT2, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MMC0_DAT3, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MMC0_CMD, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MMC0_CLK, PIN_INPUT_PULLUP, MUX_MODE0)
+		>;
+	};
+
+	emmc_pins: pinmux_emmc_pins {
+		pinctrl-single,pins = <
+			AM33XX_PADCONF(AM335X_PIN_GPMC_CSN1, PIN_INPUT_PULLUP, MUX_MODE2) /* gpmc_csn1.mmc1_clk */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_CSN2, PIN_INPUT_PULLUP, MUX_MODE2) /* gpmc_csn2.mmc1_cmd */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD0, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad0.mmc1_dat0 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD1, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad1.mmc1_dat1 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD2, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad2.mmc1_dat2 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD3, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad3.mmc1_dat3 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD4, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad4.mmc1_dat4 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD5, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad5.mmc1_dat5 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD6, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad6.mmc1_dat6 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD7, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad7.mmc1_dat7 */
+		>;
+	};
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_pins>;
+
+	status = "okay";
+};
+
+&usb {
+	status = "okay";
+};
+
+&usb_ctrl_mod {
+	status = "okay";
+};
+
+&usb0_phy {
+	status = "okay";
+};
+
+&usb1_phy {
+	status = "okay";
+};
+
+&usb0 {
+	status = "okay";
+	dr_mode = "peripheral";
+	interrupts-extended = <&intc 18 &tps 0>;
+	interrupt-names = "mc", "vbus";
+};
+
+&usb1 {
+	status = "okay";
+	dr_mode = "host";
+};
+
+&cppi41dma  {
+	status = "okay";
+};
+
+&i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c0_pins>;
+
+	status = "okay";
+	clock-frequency = <400000>;
+
+	tps: tps@24 {
+		reg = <0x24>;
+	};
+
+	baseboard_eeprom: baseboard_eeprom@50 {
+		compatible = "atmel,24c256";
+		reg = <0x50>;
+
+		#address-cells = <1>;
+		#size-cells = <1>;
+		baseboard_data: baseboard_data@0 {
+			reg = <0 0x100>;
+		};
+	};
+};
+
+&i2c2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c2_pins>;
+
+	status = "okay";
+	clock-frequency = <100000>;
+};
+
+
+/include/ "tps65217.dtsi"
+
+&tps {
+	/*
+	 * Configure pmic to enter OFF-state instead of SLEEP-state ("RTC-only
+	 * mode") at poweroff.  Most BeagleBone versions do not support RTC-only
+	 * mode and risk hardware damage if this mode is entered.
+	 *
+	 * For details, see linux-omap mailing list May 2015 thread
+	 *	[PATCH] ARM: dts: am335x-bone* enable pmic-shutdown-controller
+	 * In particular, messages:
+	 *	http://www.spinics.net/lists/linux-omap/msg118585.html
+	 *	http://www.spinics.net/lists/linux-omap/msg118615.html
+	 *
+	 * You can override this later with
+	 *	&tps {  /delete-property/ ti,pmic-shutdown-controller;  }
+	 * if you want to use RTC-only mode and made sure you are not affected
+	 * by the hardware problems. (Tip: double-check by performing a current
+	 * measurement after shutdown: it should be less than 1 mA.)
+	 */
+
+	interrupts = <7>; /* NMI */
+	interrupt-parent = <&intc>;
+
+	ti,pmic-shutdown-controller;
+
+	charger {
+		status = "okay";
+	};
+
+	pwrbutton {
+		status = "okay";
+	};
+
+	regulators {
+		dcdc1_reg: regulator@0 {
+			regulator-name = "vdds_dpr";
+			regulator-always-on;
+		};
+
+		dcdc2_reg: regulator@1 {
+			/* VDD_MPU voltage limits 0.95V - 1.26V with +/-4% tolerance */
+			regulator-name = "vdd_mpu";
+			regulator-min-microvolt = <925000>;
+			regulator-max-microvolt = <1351500>;
+			regulator-boot-on;
+			regulator-always-on;
+		};
+
+		dcdc3_reg: regulator@2 {
+			/* VDD_CORE voltage limits 0.95V - 1.1V with +/-4% tolerance */
+			regulator-name = "vdd_core";
+			regulator-min-microvolt = <925000>;
+			regulator-max-microvolt = <1150000>;
+			regulator-boot-on;
+			regulator-always-on;
+		};
+
+		ldo1_reg: regulator@3 {
+			regulator-name = "vio,vrtc,vdds";
+			regulator-always-on;
+		};
+
+		ldo2_reg: regulator@4 {
+			regulator-name = "vdd_3v3aux";
+			regulator-always-on;
+		};
+
+		ldo3_reg: regulator@5 {
+			regulator-name = "vdd_1v8";
+			regulator-always-on;
+		};
+
+		ldo4_reg: regulator@6 {
+			regulator-name = "vdd_3v3a";
+			regulator-always-on;
+		};
+	};
+};
+
+&cpsw_emac0 {
+	phy_id = <&davinci_mdio>, <0>;
+	phy-mode = "mii";
+};
+
+&mac {
+	slaves = <1>;
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&cpsw_default>;
+	pinctrl-1 = <&cpsw_sleep>;
+	status = "okay";
+};
+
+&davinci_mdio {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&davinci_mdio_default>;
+	pinctrl-1 = <&davinci_mdio_sleep>;
+	status = "okay";
+};
+
+&mmc1 {
+	status = "okay";
+	bus-width = <0x4>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&mmc1_pins>;
+	cd-gpios = <&gpio0 6 GPIO_ACTIVE_LOW>;
+};
+
+&aes {
+	status = "okay";
+};
+
+&sham {
+	status = "okay";
+};
+
+&rtc {
+	clocks = <&clk_32768_ck>, <&l4_per_clkctrl AM3_CLKDIV32K_CLKCTRL 0>;
+	clock-names = "ext-clk", "int-clk";
+	system-power-controller;
+};
+
+&wkup_m3_ipc {
+	ti,scale-data-fw = "am335x-bone-scale-data.bin";
+};
+
+//&pruss_soc_bus {
+//	status = "okay";
+//
+//	pruss: pruss@4a300000 {
+//		status = "okay";
+//	};
+//};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-bone-common-univ.dtsi b/arch/arm/boot/dts/am335x-bone-common-univ.dtsi
--- a/arch/arm/boot/dts/am335x-bone-common-univ.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-bone-common-univ.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,2961 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+&am33xx_pinmux {
+	/************************/
+	/* P8 Header */
+	/************************/
+
+	/* P8_01                GND */
+
+	/* P8_02                GND */
+
+
+	/* P8_03 (ZCZ ball R9) emmc */
+	P8_03_default_pin: pinmux_P8_03_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0818, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad6.gpio1_6 */
+	P8_03_gpio_pin: pinmux_P8_03_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0818, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad6.gpio1_6 */
+	P8_03_gpio_pu_pin: pinmux_P8_03_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0818, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad6.gpio1_6 */
+	P8_03_gpio_pd_pin: pinmux_P8_03_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0818, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad6.gpio1_6 */
+	P8_03_gpio_input_pin: pinmux_P8_03_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0818, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad6.gpio1_6 */
+
+	/* P8_04 (ZCZ ball T9) emmc */
+	P8_04_default_pin: pinmux_P8_04_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x081c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad7.gpio1_7 */
+	P8_04_gpio_pin: pinmux_P8_04_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x081c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad7.gpio1_7 */
+	P8_04_gpio_pu_pin: pinmux_P8_04_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x081c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad7.gpio1_7 */
+	P8_04_gpio_pd_pin: pinmux_P8_04_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x081c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad7.gpio1_7 */
+	P8_04_gpio_input_pin: pinmux_P8_04_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x081c, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad7.gpio1_7 */
+
+	/* P8_05 (ZCZ ball R8) emmc */
+	P8_05_default_pin: pinmux_P8_05_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0808, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad2.gpio1_2 */
+	P8_05_gpio_pin: pinmux_P8_05_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0808, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad2.gpio1_2 */
+	P8_05_gpio_pu_pin: pinmux_P8_05_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0808, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad2.gpio1_2 */
+	P8_05_gpio_pd_pin: pinmux_P8_05_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0808, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad2.gpio1_2 */
+	P8_05_gpio_input_pin: pinmux_P8_05_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0808, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad2.gpio1_2 */
+
+	/* P8_06 (ZCZ ball T8) emmc */
+	P8_06_default_pin: pinmux_P8_06_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x080c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad3.gpio1_3 */
+	P8_06_gpio_pin: pinmux_P8_06_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x080c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad3.gpio1_3 */
+	P8_06_gpio_pu_pin: pinmux_P8_06_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x080c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad3.gpio1_3 */
+	P8_06_gpio_pd_pin: pinmux_P8_06_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x080c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad3.gpio1_3 */
+	P8_06_gpio_input_pin: pinmux_P8_06_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x080c, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad3.gpio1_3 */
+
+	/* P8_07 (ZCZ ball R7) gpio2_2 */
+	P8_07_default_pin: pinmux_P8_07_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_advn_ale.gpio2_2 */
+	P8_07_gpio_pin: pinmux_P8_07_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0890, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_advn_ale.gpio2_2 */
+	P8_07_gpio_pu_pin: pinmux_P8_07_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_advn_ale.gpio2_2 */
+	P8_07_gpio_pd_pin: pinmux_P8_07_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_advn_ale.gpio2_2 */
+	P8_07_gpio_input_pin: pinmux_P8_07_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0890, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_advn_ale.gpio2_2 */
+	P8_07_timer_pin: pinmux_P8_07_timer_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* gpmc_advn_ale.timer4 */
+
+	/* P8_08 (ZCZ ball T7) gpio2_3 */
+	P8_08_default_pin: pinmux_P8_08_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_oen_ren.gpio2_3 */
+	P8_08_gpio_pin: pinmux_P8_08_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0894, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_oen_ren.gpio2_3 */
+	P8_08_gpio_pu_pin: pinmux_P8_08_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_oen_ren.gpio2_3 */
+	P8_08_gpio_pd_pin: pinmux_P8_08_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_oen_ren.gpio2_3 */
+	P8_08_gpio_input_pin: pinmux_P8_08_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0894, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_oen_ren.gpio2_3 */
+	P8_08_timer_pin: pinmux_P8_08_timer_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* gpmc_oen_ren.timer7 */
+
+	/* P8_09 (ZCZ ball T6) gpio2_5 */
+	P8_09_default_pin: pinmux_P8_09_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_be0n_cle.gpio2_5 */
+	P8_09_gpio_pin: pinmux_P8_09_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x089c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_be0n_cle.gpio2_5 */
+	P8_09_gpio_pu_pin: pinmux_P8_09_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_be0n_cle.gpio2_5 */
+	P8_09_gpio_pd_pin: pinmux_P8_09_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_be0n_cle.gpio2_5 */
+	P8_09_gpio_input_pin: pinmux_P8_09_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x089c, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_be0n_cle.gpio2_5 */
+	P8_09_timer_pin: pinmux_P8_09_timer_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* gpmc_be0n_cle.timer5 */
+
+	/* P8_10 (ZCZ ball U6) gpio2_4 */
+	P8_10_default_pin: pinmux_P8_10_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_wen.gpio2_4 */
+	P8_10_gpio_pin: pinmux_P8_10_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0898, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_wen.gpio2_4 */
+	P8_10_gpio_pu_pin: pinmux_P8_10_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_wen.gpio2_4 */
+	P8_10_gpio_pd_pin: pinmux_P8_10_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_wen.gpio2_4 */
+	P8_10_gpio_input_pin: pinmux_P8_10_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0898, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_wen.gpio2_4 */
+	P8_10_timer_pin: pinmux_P8_10_timer_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* gpmc_wen.timer6 */
+
+	/* P8_11 (ZCZ ball R12) gpio1_13 */
+	P8_11_default_pin: pinmux_P8_11_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad13.gpio1_13 */
+	P8_11_gpio_pin: pinmux_P8_11_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0834, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad13.gpio1_13 */
+	P8_11_gpio_pu_pin: pinmux_P8_11_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad13.gpio1_13 */
+	P8_11_gpio_pd_pin: pinmux_P8_11_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad13.gpio1_13 */
+	P8_11_gpio_input_pin: pinmux_P8_11_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0834, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad13.gpio1_13 */
+	P8_11_qep_pin: pinmux_P8_11_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* gpmc_ad13.eqep2b_in */
+	P8_11_pruout_pin: pinmux_P8_11_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; };	/* gpmc_ad13.pru0_out15 */
+
+	/* P8_12 (ZCZ ball T12) gpio1_12 */
+	P8_12_default_pin: pinmux_P8_12_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad12.gpio1_12 */
+	P8_12_gpio_pin: pinmux_P8_12_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0830, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad12.gpio1_12 */
+	P8_12_gpio_pu_pin: pinmux_P8_12_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad12.gpio1_12 */
+	P8_12_gpio_pd_pin: pinmux_P8_12_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad12.gpio1_12 */
+	P8_12_gpio_input_pin: pinmux_P8_12_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0830, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad12.gpio1_12 */
+	P8_12_qep_pin: pinmux_P8_12_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* gpmc_ad12.eqep2a_in */
+	P8_12_pruout_pin: pinmux_P8_12_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; };	/* gpmc_ad12.pru0_out14 */
+
+	/* P8_13 (ZCZ ball T10) gpio0_23 */
+	P8_13_default_pin: pinmux_P8_13_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad9.gpio0_23 */
+	P8_13_gpio_pin: pinmux_P8_13_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0824, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad9.gpio0_23 */
+	P8_13_gpio_pu_pin: pinmux_P8_13_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad9.gpio0_23 */
+	P8_13_gpio_pd_pin: pinmux_P8_13_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad9.gpio0_23 */
+	P8_13_gpio_input_pin: pinmux_P8_13_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0824, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad9.gpio0_23 */
+	P8_13_pwm_pin: pinmux_P8_13_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; };	/* gpmc_ad9.ehrpwm2b */
+
+	/* P8_14 (ZCZ ball T11) gpio0_26 */
+	P8_14_default_pin: pinmux_P8_14_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad10.gpio0_26 */
+	P8_14_gpio_pin: pinmux_P8_14_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0828, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad10.gpio0_26 */
+	P8_14_gpio_pu_pin: pinmux_P8_14_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad10.gpio0_26 */
+	P8_14_gpio_pd_pin: pinmux_P8_14_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad10.gpio0_26 */
+	P8_14_gpio_input_pin: pinmux_P8_14_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0828, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad10.gpio0_26 */
+	P8_14_pwm_pin: pinmux_P8_14_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; };	/* gpmc_ad10.ehrpwm2_tripzone_input */
+
+	/* P8_15 (ZCZ ball U13) gpio1_15 */
+	P8_15_default_pin: pinmux_P8_15_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad15.gpio1_15 */
+	P8_15_gpio_pin: pinmux_P8_15_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x083c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad15.gpio1_15 */
+	P8_15_gpio_pu_pin: pinmux_P8_15_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad15.gpio1_15 */
+	P8_15_gpio_pd_pin: pinmux_P8_15_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad15.gpio1_15 */
+	P8_15_gpio_input_pin: pinmux_P8_15_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x083c, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad15.gpio1_15 */
+	P8_15_qep_pin: pinmux_P8_15_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* gpmc_ad15.eqep2_strobe */
+	P8_15_pru_ecap_pin: pinmux_P8_15_pru_ecap_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* gpmc_ad15.pr1_ecap0_ecap_capin_apwm_o */
+	P8_15_pruin_pin: pinmux_P8_15_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x083c, PIN_INPUT | MUX_MODE6) >; };			/* gpmc_ad15.pru0_in15 */
+
+	/* P8_16 (ZCZ ball V13) gpio1_14 */
+	P8_16_default_pin: pinmux_P8_16_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad14.gpio1_14 */
+	P8_16_gpio_pin: pinmux_P8_16_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0838, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad14.gpio1_14 */
+	P8_16_gpio_pu_pin: pinmux_P8_16_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad14.gpio1_14 */
+	P8_16_gpio_pd_pin: pinmux_P8_16_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad14.gpio1_14 */
+	P8_16_gpio_input_pin: pinmux_P8_16_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0838, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad14.gpio1_14 */
+	P8_16_qep_pin: pinmux_P8_16_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* gpmc_ad14.eqep2_index */
+	P8_16_pruin_pin: pinmux_P8_16_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0838, PIN_INPUT | MUX_MODE6) >; };			/* gpmc_ad14.pru0_in14 */
+
+	/* P8_17 (ZCZ ball U12) gpio0_27 */
+	P8_17_default_pin: pinmux_P8_17_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad11.gpio0_27 */
+	P8_17_gpio_pin: pinmux_P8_17_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x082c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad11.gpio0_27 */
+	P8_17_gpio_pu_pin: pinmux_P8_17_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad11.gpio0_27 */
+	P8_17_gpio_pd_pin: pinmux_P8_17_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad11.gpio0_27 */
+	P8_17_gpio_input_pin: pinmux_P8_17_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x082c, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad11.gpio0_27 */
+	P8_17_pwm_pin: pinmux_P8_17_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; };	/* gpmc_ad11.ehrpwm0_synco */
+
+	/* P8_18 (ZCZ ball V12) gpio2_1 */
+	P8_18_default_pin: pinmux_P8_18_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_clk.gpio2_1 */
+	P8_18_gpio_pin: pinmux_P8_18_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x088c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_clk.gpio2_1 */
+	P8_18_gpio_pu_pin: pinmux_P8_18_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_clk.gpio2_1 */
+	P8_18_gpio_pd_pin: pinmux_P8_18_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_clk.gpio2_1 */
+	P8_18_gpio_input_pin: pinmux_P8_18_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x088c, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_clk.gpio2_1 */
+
+	/* P8_19 (ZCZ ball U10) gpio0_22 */
+	P8_19_default_pin: pinmux_P8_19_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad8.gpio0_22 */
+	P8_19_gpio_pin: pinmux_P8_19_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0820, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad8.gpio0_22 */
+	P8_19_gpio_pu_pin: pinmux_P8_19_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad8.gpio0_22 */
+	P8_19_gpio_pd_pin: pinmux_P8_19_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad8.gpio0_22 */
+	P8_19_gpio_input_pin: pinmux_P8_19_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0820, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad8.gpio0_22 */
+	P8_19_pwm_pin: pinmux_P8_19_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; };	/* gpmc_ad8.ehrpwm2a */
+
+	/* P8_20 (ZCZ ball V9) emmc */
+	P8_20_default_pin: pinmux_P8_20_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_csn2.gpio1_31 */
+	P8_20_gpio_pin: pinmux_P8_20_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0884, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_csn2.gpio1_31 */
+	P8_20_gpio_pu_pin: pinmux_P8_20_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_csn2.gpio1_31 */
+	P8_20_gpio_pd_pin: pinmux_P8_20_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_csn2.gpio1_31 */
+	P8_20_gpio_input_pin: pinmux_P8_20_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0884, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_csn2.gpio1_31 */
+	P8_20_pruout_pin: pinmux_P8_20_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* gpmc_csn2.pru1_out13 */
+	P8_20_pruin_pin: pinmux_P8_20_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0884, PIN_INPUT | MUX_MODE6) >; };			/* gpmc_csn2.pru1_in13 */
+
+	/* P8_21 (ZCZ ball U9) emmc */
+	P8_21_default_pin: pinmux_P8_21_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_csn1.gpio1_30 */
+	P8_21_gpio_pin: pinmux_P8_21_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0880, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_csn1.gpio1_30 */
+	P8_21_gpio_pu_pin: pinmux_P8_21_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_csn1.gpio1_30 */
+	P8_21_gpio_pd_pin: pinmux_P8_21_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_csn1.gpio1_30 */
+	P8_21_gpio_input_pin: pinmux_P8_21_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0880, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_csn1.gpio1_30 */
+	P8_21_pruout_pin: pinmux_P8_21_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* gpmc_csn1.pru1_out12 */
+	P8_21_pruin_pin: pinmux_P8_21_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0880, PIN_INPUT | MUX_MODE6) >; };			/* gpmc_csn1.pru1_in12 */
+
+	/* P8_22 (ZCZ ball V8) emmc */
+	P8_22_default_pin: pinmux_P8_22_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0814, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad5.gpio1_5 */
+	P8_22_gpio_pin: pinmux_P8_22_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0814, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad5.gpio1_5 */
+	P8_22_gpio_pu_pin: pinmux_P8_22_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0814, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad5.gpio1_5 */
+	P8_22_gpio_pd_pin: pinmux_P8_22_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0814, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad5.gpio1_5 */
+	P8_22_gpio_input_pin: pinmux_P8_22_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0814, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad5.gpio1_5 */
+
+	/* P8_23 (ZCZ ball U8) emmc */
+	P8_23_default_pin: pinmux_P8_23_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0810, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad4.gpio1_4 */
+	P8_23_gpio_pin: pinmux_P8_23_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0810, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad4.gpio1_4 */
+	P8_23_gpio_pu_pin: pinmux_P8_23_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0810, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad4.gpio1_4 */
+	P8_23_gpio_pd_pin: pinmux_P8_23_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0810, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad4.gpio1_4 */
+	P8_23_gpio_input_pin: pinmux_P8_23_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0810, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad4.gpio1_4 */
+
+	/* P8_24 (ZCZ ball V7) emmc */
+	P8_24_default_pin: pinmux_P8_24_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0804, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad1.gpio1_1 */
+	P8_24_gpio_pin: pinmux_P8_24_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0804, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad1.gpio1_1 */
+	P8_24_gpio_pu_pin: pinmux_P8_24_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0804, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad1.gpio1_1 */
+	P8_24_gpio_pd_pin: pinmux_P8_24_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0804, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad1.gpio1_1 */
+	P8_24_gpio_input_pin: pinmux_P8_24_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0804, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad1.gpio1_1 */
+
+	/* P8_25 (ZCZ ball U7) emmc */
+	P8_25_default_pin: pinmux_P8_25_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0800, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad0.gpio1_0 */
+	P8_25_gpio_pin: pinmux_P8_25_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0800, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad0.gpio1_0 */
+	P8_25_gpio_pu_pin: pinmux_P8_25_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0800, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad0.gpio1_0 */
+	P8_25_gpio_pd_pin: pinmux_P8_25_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0800, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad0.gpio1_0 */
+	P8_25_gpio_input_pin: pinmux_P8_25_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0800, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad0.gpio1_0 */
+
+	/* P8_26 (ZCZ ball V6) gpio1_29 */
+	P8_26_default_pin: pinmux_P8_26_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x087c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_csn0.gpio1_29 */
+	P8_26_gpio_pin: pinmux_P8_26_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x087c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_csn0.gpio1_29 */
+	P8_26_gpio_pu_pin: pinmux_P8_26_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x087c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_csn0.gpio1_29 */
+	P8_26_gpio_pd_pin: pinmux_P8_26_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x087c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_csn0.gpio1_29 */
+	P8_26_gpio_input_pin: pinmux_P8_26_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x087c, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_csn0.gpio1_29 */
+
+	/* P8_27 (ZCZ ball U5) hdmi */
+	P8_27_default_pin: pinmux_P8_27_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_vsync.gpio2_22 */
+	P8_27_gpio_pin: pinmux_P8_27_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_vsync.gpio2_22 */
+	P8_27_gpio_pu_pin: pinmux_P8_27_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_vsync.gpio2_22 */
+	P8_27_gpio_pd_pin: pinmux_P8_27_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_vsync.gpio2_22 */
+	P8_27_gpio_input_pin: pinmux_P8_27_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e0, PIN_INPUT | MUX_MODE7) >; };			/* lcd_vsync.gpio2_22 */
+	P8_27_pruout_pin: pinmux_P8_27_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* lcd_vsync.pru1_out8 */
+	P8_27_pruin_pin: pinmux_P8_27_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e0, PIN_INPUT | MUX_MODE6) >; };			/* lcd_vsync.pru1_in8 */
+
+	/* P8_28 (ZCZ ball V5) hdmi */
+	P8_28_default_pin: pinmux_P8_28_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_pclk.gpio2_24 */
+	P8_28_gpio_pin: pinmux_P8_28_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_pclk.gpio2_24 */
+	P8_28_gpio_pu_pin: pinmux_P8_28_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_pclk.gpio2_24 */
+	P8_28_gpio_pd_pin: pinmux_P8_28_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_pclk.gpio2_24 */
+	P8_28_gpio_input_pin: pinmux_P8_28_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e8, PIN_INPUT | MUX_MODE7) >; };			/* lcd_pclk.gpio2_24 */
+	P8_28_pruout_pin: pinmux_P8_28_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* lcd_pclk.pru1_out10 */
+	P8_28_pruin_pin: pinmux_P8_28_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e8, PIN_INPUT | MUX_MODE6) >; };			/* lcd_pclk.pru1_in10 */
+
+	/* P8_29 (ZCZ ball R5) hdmi */
+	P8_29_default_pin: pinmux_P8_29_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_hsync.gpio2_23 */
+	P8_29_gpio_pin: pinmux_P8_29_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_hsync.gpio2_23 */
+	P8_29_gpio_pu_pin: pinmux_P8_29_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_hsync.gpio2_23 */
+	P8_29_gpio_pd_pin: pinmux_P8_29_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_hsync.gpio2_23 */
+	P8_29_gpio_input_pin: pinmux_P8_29_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e4, PIN_INPUT | MUX_MODE7) >; };			/* lcd_hsync.gpio2_23 */
+	P8_29_pruout_pin: pinmux_P8_29_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* lcd_hsync.pru1_out9 */
+	P8_29_pruin_pin: pinmux_P8_29_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e4, PIN_INPUT | MUX_MODE6) >; };			/* lcd_hsync.pru1_in9 */
+
+	/* P8_30 (ZCZ ball R6) hdmi */
+	P8_30_default_pin: pinmux_P8_30_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_ac_bias_en.gpio2_25 */
+	P8_30_gpio_pin: pinmux_P8_30_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ec, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_ac_bias_en.gpio2_25 */
+	P8_30_gpio_pu_pin: pinmux_P8_30_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_ac_bias_en.gpio2_25 */
+	P8_30_gpio_pd_pin: pinmux_P8_30_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_ac_bias_en.gpio2_25 */
+	P8_30_gpio_input_pin: pinmux_P8_30_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ec, PIN_INPUT | MUX_MODE7) >; };			/* lcd_ac_bias_en.gpio2_25 */
+	P8_30_pruout_pin: pinmux_P8_30_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* lcd_ac_bias_en.pru1_out11 */
+	P8_30_pruin_pin: pinmux_P8_30_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ec, PIN_INPUT | MUX_MODE6) >; };			/* lcd_ac_bias_en.pru1_in11 */
+
+	/* P8_31 (ZCZ ball V4) hdmi */
+	P8_31_default_pin: pinmux_P8_31_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data14.gpio0_10 */
+	P8_31_gpio_pin: pinmux_P8_31_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data14.gpio0_10 */
+	P8_31_gpio_pu_pin: pinmux_P8_31_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data14.gpio0_10 */
+	P8_31_gpio_pd_pin: pinmux_P8_31_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data14.gpio0_10 */
+	P8_31_gpio_input_pin: pinmux_P8_31_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d8, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data14.gpio0_10 */
+	P8_31_qep_pin: pinmux_P8_31_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* lcd_data14.eqep1_index */
+	P8_31_uart_pin: pinmux_P8_31_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* lcd_data14.uart5_rxd */
+
+	/* P8_32 (ZCZ ball T5) hdmi */
+	P8_32_default_pin: pinmux_P8_32_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data15.gpio0_11 */
+	P8_32_gpio_pin: pinmux_P8_32_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08dc, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data15.gpio0_11 */
+	P8_32_gpio_pu_pin: pinmux_P8_32_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data15.gpio0_11 */
+	P8_32_gpio_pd_pin: pinmux_P8_32_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data15.gpio0_11 */
+	P8_32_gpio_input_pin: pinmux_P8_32_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08dc, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data15.gpio0_11 */
+	P8_32_qep_pin: pinmux_P8_32_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* lcd_data15.eqep1_strobe */
+
+	/* P8_33 (ZCZ ball V3) hdmi */
+	P8_33_default_pin: pinmux_P8_33_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data13.gpio0_9 */
+	P8_33_gpio_pin: pinmux_P8_33_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data13.gpio0_9 */
+	P8_33_gpio_pu_pin: pinmux_P8_33_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data13.gpio0_9 */
+	P8_33_gpio_pd_pin: pinmux_P8_33_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data13.gpio0_9 */
+	P8_33_gpio_input_pin: pinmux_P8_33_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d4, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data13.gpio0_9 */
+	P8_33_qep_pin: pinmux_P8_33_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* lcd_data13.eqep1b_in */
+
+	/* P8_34 (ZCZ ball U4) hdmi */
+	P8_34_default_pin: pinmux_P8_34_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data11.gpio2_17 */
+	P8_34_gpio_pin: pinmux_P8_34_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08cc, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data11.gpio2_17 */
+	P8_34_gpio_pu_pin: pinmux_P8_34_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data11.gpio2_17 */
+	P8_34_gpio_pd_pin: pinmux_P8_34_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data11.gpio2_17 */
+	P8_34_gpio_input_pin: pinmux_P8_34_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08cc, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data11.gpio2_17 */
+	P8_34_pwm_pin: pinmux_P8_34_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; };	/* lcd_data11.ehrpwm1b */
+
+	/* P8_35 (ZCZ ball V2) hdmi */
+	P8_35_default_pin: pinmux_P8_35_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data12.gpio0_8 */
+	P8_35_gpio_pin: pinmux_P8_35_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data12.gpio0_8 */
+	P8_35_gpio_pu_pin: pinmux_P8_35_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data12.gpio0_8 */
+	P8_35_gpio_pd_pin: pinmux_P8_35_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data12.gpio0_8 */
+	P8_35_gpio_input_pin: pinmux_P8_35_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d0, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data12.gpio0_8 */
+	P8_35_qep_pin: pinmux_P8_35_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* lcd_data12.eqep1a_in */
+
+	/* P8_36 (ZCZ ball U3) hdmi */
+	P8_36_default_pin: pinmux_P8_36_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data10.gpio2_16 */
+	P8_36_gpio_pin: pinmux_P8_36_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data10.gpio2_16 */
+	P8_36_gpio_pu_pin: pinmux_P8_36_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data10.gpio2_16 */
+	P8_36_gpio_pd_pin: pinmux_P8_36_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data10.gpio2_16 */
+	P8_36_gpio_input_pin: pinmux_P8_36_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c8, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data10.gpio2_16 */
+	P8_36_pwm_pin: pinmux_P8_36_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; };	/* lcd_data10.ehrpwm1a */
+
+	/* P8_37 (ZCZ ball U1) hdmi */
+	P8_37_default_pin: pinmux_P8_37_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data8.gpio2_14 */
+	P8_37_gpio_pin: pinmux_P8_37_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data8.gpio2_14 */
+	P8_37_gpio_pu_pin: pinmux_P8_37_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data8.gpio2_14 */
+	P8_37_gpio_pd_pin: pinmux_P8_37_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data8.gpio2_14 */
+	P8_37_gpio_input_pin: pinmux_P8_37_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c0, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data8.gpio2_14 */
+	P8_37_pwm_pin: pinmux_P8_37_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; };	/* lcd_data8.ehrpwm1_tripzone_input */
+	P8_37_uart_pin: pinmux_P8_37_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* lcd_data8.uart5_txd */
+
+	/* P8_38 (ZCZ ball U2) hdmi */
+	P8_38_default_pin: pinmux_P8_38_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data9.gpio2_15 */
+	P8_38_gpio_pin: pinmux_P8_38_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data9.gpio2_15 */
+	P8_38_gpio_pu_pin: pinmux_P8_38_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data9.gpio2_15 */
+	P8_38_gpio_pd_pin: pinmux_P8_38_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data9.gpio2_15 */
+	P8_38_gpio_input_pin: pinmux_P8_38_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c4, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data9.gpio2_15 */
+	P8_38_pwm_pin: pinmux_P8_38_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; };	/* lcd_data9.ehrpwm0_synco */
+	P8_38_uart_pin: pinmux_P8_38_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* lcd_data9.uart5_rxd */
+
+	/* P8_39 (ZCZ ball T3) hdmi */
+	P8_39_default_pin: pinmux_P8_39_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data6.gpio2_12 */
+	P8_39_gpio_pin: pinmux_P8_39_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data6.gpio2_12 */
+	P8_39_gpio_pu_pin: pinmux_P8_39_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data6.gpio2_12 */
+	P8_39_gpio_pd_pin: pinmux_P8_39_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data6.gpio2_12 */
+	P8_39_gpio_input_pin: pinmux_P8_39_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b8, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data6.gpio2_12 */
+	P8_39_qep_pin: pinmux_P8_39_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* lcd_data6.eqep2_index */
+	P8_39_pruout_pin: pinmux_P8_39_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* lcd_data6.pru1_out6 */
+	P8_39_pruin_pin: pinmux_P8_39_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b8, PIN_INPUT | MUX_MODE6) >; };			/* lcd_data6.pru1_in6 */
+
+	/* P8_40 (ZCZ ball T4) hdmi */
+	P8_40_default_pin: pinmux_P8_40_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data7.gpio2_13 */
+	P8_40_gpio_pin: pinmux_P8_40_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08bc, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data7.gpio2_13 */
+	P8_40_gpio_pu_pin: pinmux_P8_40_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data7.gpio2_13 */
+	P8_40_gpio_pd_pin: pinmux_P8_40_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data7.gpio2_13 */
+	P8_40_gpio_input_pin: pinmux_P8_40_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08bc, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data7.gpio2_13 */
+	P8_40_qep_pin: pinmux_P8_40_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* lcd_data7.eqep2_strobe */
+	P8_40_pruout_pin: pinmux_P8_40_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* lcd_data7.pru1_out7 */
+	P8_40_pruin_pin: pinmux_P8_40_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08bc, PIN_INPUT | MUX_MODE6) >; };			/* lcd_data7.pru1_in7 */
+
+	/* P8_41 (ZCZ ball T1) hdmi */
+	P8_41_default_pin: pinmux_P8_41_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data4.gpio2_10 */
+	P8_41_gpio_pin: pinmux_P8_41_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data4.gpio2_10 */
+	P8_41_gpio_pu_pin: pinmux_P8_41_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data4.gpio2_10 */
+	P8_41_gpio_pd_pin: pinmux_P8_41_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data4.gpio2_10 */
+	P8_41_gpio_input_pin: pinmux_P8_41_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b0, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data4.gpio2_10 */
+	P8_41_qep_pin: pinmux_P8_41_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* lcd_data4.eqep2a_in */
+	P8_41_pruout_pin: pinmux_P8_41_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* lcd_data4.pru1_out4 */
+	P8_41_pruin_pin: pinmux_P8_41_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b0, PIN_INPUT | MUX_MODE6) >; };			/* lcd_data4.pru1_in4 */
+
+	/* P8_42 (ZCZ ball T2) hdmi */
+	P8_42_default_pin: pinmux_P8_42_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data5.gpio2_11 */
+	P8_42_gpio_pin: pinmux_P8_42_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data5.gpio2_11 */
+	P8_42_gpio_pu_pin: pinmux_P8_42_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data5.gpio2_11 */
+	P8_42_gpio_pd_pin: pinmux_P8_42_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data5.gpio2_11 */
+	P8_42_gpio_input_pin: pinmux_P8_42_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b4, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data5.gpio2_11 */
+	P8_42_qep_pin: pinmux_P8_42_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* lcd_data5.eqep2b_in */
+	P8_42_pruout_pin: pinmux_P8_42_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* lcd_data5.pru1_out5 */
+	P8_42_pruin_pin: pinmux_P8_42_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b4, PIN_INPUT | MUX_MODE6) >; };			/* lcd_data5.pru1_in5 */
+
+	/* P8_43 (ZCZ ball R3) hdmi */
+	P8_43_default_pin: pinmux_P8_43_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data2.gpio2_8 */
+	P8_43_gpio_pin: pinmux_P8_43_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data2.gpio2_8 */
+	P8_43_gpio_pu_pin: pinmux_P8_43_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data2.gpio2_8 */
+	P8_43_gpio_pd_pin: pinmux_P8_43_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data2.gpio2_8 */
+	P8_43_gpio_input_pin: pinmux_P8_43_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a8, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data2.gpio2_8 */
+	P8_43_pwm_pin: pinmux_P8_43_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; };	/* lcd_data2.ehrpwm2_tripzone_input */
+	P8_43_pruout_pin: pinmux_P8_43_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* lcd_data2.pru1_out2 */
+	P8_43_pruin_pin: pinmux_P8_43_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a8, PIN_INPUT | MUX_MODE6) >; };			/* lcd_data2.pru1_in2 */
+
+	/* P8_44 (ZCZ ball R4) hdmi */
+	P8_44_default_pin: pinmux_P8_44_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data3.gpio2_9 */
+	P8_44_gpio_pin: pinmux_P8_44_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ac, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data3.gpio2_9 */
+	P8_44_gpio_pu_pin: pinmux_P8_44_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data3.gpio2_9 */
+	P8_44_gpio_pd_pin: pinmux_P8_44_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data3.gpio2_9 */
+	P8_44_gpio_input_pin: pinmux_P8_44_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ac, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data3.gpio2_9 */
+	P8_44_pwm_pin: pinmux_P8_44_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; };	/* lcd_data3.ehrpwm0_synco */
+	P8_44_pruout_pin: pinmux_P8_44_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* lcd_data3.pru1_out3 */
+	P8_44_pruin_pin: pinmux_P8_44_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ac, PIN_INPUT | MUX_MODE6) >; };			/* lcd_data3.pru1_in3 */
+
+	/* P8_45 (ZCZ ball R1) hdmi */
+	P8_45_default_pin: pinmux_P8_45_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data0.gpio2_6 */
+	P8_45_gpio_pin: pinmux_P8_45_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data0.gpio2_6 */
+	P8_45_gpio_pu_pin: pinmux_P8_45_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data0.gpio2_6 */
+	P8_45_gpio_pd_pin: pinmux_P8_45_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data0.gpio2_6 */
+	P8_45_gpio_input_pin: pinmux_P8_45_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a0, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data0.gpio2_6 */
+	P8_45_pwm_pin: pinmux_P8_45_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; };	/* lcd_data0.ehrpwm2a */
+	P8_45_pruout_pin: pinmux_P8_45_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* lcd_data0.pru1_out0 */
+	P8_45_pruin_pin: pinmux_P8_45_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a0, PIN_INPUT | MUX_MODE6) >; };			/* lcd_data0.pru1_in0 */
+
+	/* P8_46 (ZCZ ball R2) hdmi */
+	P8_46_default_pin: pinmux_P8_46_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data1.gpio2_7 */
+	P8_46_gpio_pin: pinmux_P8_46_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data1.gpio2_7 */
+	P8_46_gpio_pu_pin: pinmux_P8_46_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data1.gpio2_7 */
+	P8_46_gpio_pd_pin: pinmux_P8_46_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data1.gpio2_7 */
+	P8_46_gpio_input_pin: pinmux_P8_46_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a4, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data1.gpio2_7 */
+	P8_46_pwm_pin: pinmux_P8_46_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; };	/* lcd_data1.ehrpwm2b */
+	P8_46_pruout_pin: pinmux_P8_46_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* lcd_data1.pru1_out1 */
+	P8_46_pruin_pin: pinmux_P8_46_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a4, PIN_INPUT | MUX_MODE6) >; };			/* lcd_data1.pru1_in1 */
+
+	/************************/
+	/* P9 Header */
+	/************************/
+
+	/* P9_01                GND */
+
+	/* P9_02                GND */
+
+	/* P9_03                3V3 */
+
+	/* P9_04                3V3 */
+
+	/* P9_05                VDD_5V */
+
+	/* P9_06                VDD_5V */
+
+	/* P9_07                SYS_5V */
+
+	/* P9_08                SYS_5V */
+
+	/* P9_09                PWR_BUT */
+
+	/* P9_10                RSTn */
+
+	/* P9_11 (ZCZ ball T17) gpio0_30 */
+	P9_11_default_pin: pinmux_P9_11_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_wait0.gpio0_30 */
+	P9_11_gpio_pin: pinmux_P9_11_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0870, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_wait0.gpio0_30 */
+	P9_11_gpio_pu_pin: pinmux_P9_11_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_wait0.gpio0_30 */
+	P9_11_gpio_pd_pin: pinmux_P9_11_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_wait0.gpio0_30 */
+	P9_11_gpio_input_pin: pinmux_P9_11_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0870, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_wait0.gpio0_30 */
+	P9_11_uart_pin: pinmux_P9_11_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; };	/* gpmc_wait0.uart4_rxd */
+
+	/* P9_12 (ZCZ ball U18) gpio1_28 */
+	P9_12_default_pin: pinmux_P9_12_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_be1n.gpio1_28 */
+	P9_12_gpio_pin: pinmux_P9_12_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0878, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_be1n.gpio1_28 */
+	P9_12_gpio_pu_pin: pinmux_P9_12_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_be1n.gpio1_28 */
+	P9_12_gpio_pd_pin: pinmux_P9_12_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_be1n.gpio1_28 */
+	P9_12_gpio_input_pin: pinmux_P9_12_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0878, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_be1n.gpio1_28 */
+
+	/* P9_13 (ZCZ ball U17) gpio0_31 */
+	P9_13_default_pin: pinmux_P9_13_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_wpn.gpio0_31 */
+	P9_13_gpio_pin: pinmux_P9_13_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0874, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_wpn.gpio0_31 */
+	P9_13_gpio_pu_pin: pinmux_P9_13_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_wpn.gpio0_31 */
+	P9_13_gpio_pd_pin: pinmux_P9_13_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_wpn.gpio0_31 */
+	P9_13_gpio_input_pin: pinmux_P9_13_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0874, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_wpn.gpio0_31 */
+	P9_13_uart_pin: pinmux_P9_13_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; };	/* gpmc_wpn.uart4_txd */
+
+	/* P9_14 (ZCZ ball U14) gpio1_18 */
+	P9_14_default_pin: pinmux_P9_14_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a2.gpio1_18 */
+	P9_14_gpio_pin: pinmux_P9_14_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0848, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_a2.gpio1_18 */
+	P9_14_gpio_pu_pin: pinmux_P9_14_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a2.gpio1_18 */
+	P9_14_gpio_pd_pin: pinmux_P9_14_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a2.gpio1_18 */
+	P9_14_gpio_input_pin: pinmux_P9_14_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0848, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_a2.gpio1_18 */
+	P9_14_pwm_pin: pinmux_P9_14_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; };	/* gpmc_a2.ehrpwm1a */
+
+	/* P9_15 (ZCZ ball R13) gpio1_16 */
+	P9_15_default_pin: pinmux_P9_15_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a0.gpio1_16 */
+	P9_15_gpio_pin: pinmux_P9_15_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0840, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_a0.gpio1_16 */
+	P9_15_gpio_pu_pin: pinmux_P9_15_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a0.gpio1_16 */
+	P9_15_gpio_pd_pin: pinmux_P9_15_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a0.gpio1_16 */
+	P9_15_gpio_input_pin: pinmux_P9_15_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0840, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_a0.gpio1_16 */
+	P9_15_pwm_pin: pinmux_P9_15_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; };	/* gpmc_a0.ehrpwm1_tripzone_input */
+
+	/* P9_16 (ZCZ ball T14) gpio1_19 */
+	P9_16_default_pin: pinmux_P9_16_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a3.gpio1_19 */
+	P9_16_gpio_pin: pinmux_P9_16_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x084c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_a3.gpio1_19 */
+	P9_16_gpio_pu_pin: pinmux_P9_16_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a3.gpio1_19 */
+	P9_16_gpio_pd_pin: pinmux_P9_16_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a3.gpio1_19 */
+	P9_16_gpio_input_pin: pinmux_P9_16_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x084c, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_a3.gpio1_19 */
+	P9_16_pwm_pin: pinmux_P9_16_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; };	/* gpmc_a3.ehrpwm1b */
+
+	/* P9_17 (ZCZ ball A16) gpio0_5 */
+	P9_17_default_pin: pinmux_P9_17_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* spi0_cs0.gpio0_5 */
+	P9_17_gpio_pin: pinmux_P9_17_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x095c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* spi0_cs0.gpio0_5 */
+	P9_17_gpio_pu_pin: pinmux_P9_17_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* spi0_cs0.gpio0_5 */
+	P9_17_gpio_pd_pin: pinmux_P9_17_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* spi0_cs0.gpio0_5 */
+	P9_17_gpio_input_pin: pinmux_P9_17_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x095c, PIN_INPUT | MUX_MODE7) >; };			/* spi0_cs0.gpio0_5 */
+	P9_17_spi_cs_pin: pinmux_P9_17_spi_cs_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };	/* spi0_cs0.spi0_cs0 */
+	P9_17_i2c_pin: pinmux_P9_17_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* spi0_cs0.i2c1_scl */
+	P9_17_pwm_pin: pinmux_P9_17_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; };	/* spi0_cs0.ehrpwm0_synci */
+	P9_17_pru_uart_pin: pinmux_P9_17_pru_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* spi0_cs0.pr1_uart0_txd */
+
+	/* P9_18 (ZCZ ball B16) gpio0_4 */
+	P9_18_default_pin: pinmux_P9_18_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* spi0_d1.gpio0_4 */
+	P9_18_gpio_pin: pinmux_P9_18_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0958, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* spi0_d1.gpio0_4 */
+	P9_18_gpio_pu_pin: pinmux_P9_18_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* spi0_d1.gpio0_4 */
+	P9_18_gpio_pd_pin: pinmux_P9_18_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* spi0_d1.gpio0_4 */
+	P9_18_gpio_input_pin: pinmux_P9_18_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0958, PIN_INPUT | MUX_MODE7) >; };			/* spi0_d1.gpio0_4 */
+	P9_18_spi_pin: pinmux_P9_18_spi_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };	/* spi0_d1.spi0_d1 */
+	P9_18_i2c_pin: pinmux_P9_18_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* spi0_d1.i2c1_sda */
+	P9_18_pwm_pin: pinmux_P9_18_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; };	/* spi0_d1.ehrpwm0_tripzone_input */
+	P9_18_pru_uart_pin: pinmux_P9_18_pru_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* spi0_d1.pr1_uart0_rxd */
+
+	/* P9_19 (ZCZ ball D17) i2c2_scl */
+	P9_19_default_pin: pinmux_P9_19_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* uart1_rtsn.i2c2_scl */
+	P9_19_gpio_pin: pinmux_P9_19_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* uart1_rtsn.gpio0_13 */
+	P9_19_gpio_pu_pin: pinmux_P9_19_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* uart1_rtsn.gpio0_13 */
+	P9_19_gpio_pd_pin: pinmux_P9_19_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* uart1_rtsn.gpio0_13 */
+	P9_19_gpio_input_pin: pinmux_P9_19_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_INPUT | MUX_MODE7) >; };			/* uart1_rtsn.gpio0_13 */
+	P9_19_timer_pin: pinmux_P9_19_timer_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* uart1_rtsn.timer5 */
+	P9_19_can_pin: pinmux_P9_19_can_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_INPUT_PULLUP | MUX_MODE2) >; };		/* uart1_rtsn.dcan0_rx */
+	P9_19_i2c_pin: pinmux_P9_19_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* uart1_rtsn.i2c2_scl */
+	P9_19_spi_cs_pin: pinmux_P9_19_spi_cs_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* uart1_rtsn.spi1_cs1 */
+	P9_19_pru_uart_pin: pinmux_P9_19_pru_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; };	/* uart1_rtsn.pr1_uart0_rts_n */
+
+	/* P9_20 (ZCZ ball D18) i2c2_sda */
+	P9_20_default_pin: pinmux_P9_20_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* uart1_ctsn.i2c2_sda */
+	P9_20_gpio_pin: pinmux_P9_20_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* uart1_ctsn.gpio0_12 */
+	P9_20_gpio_pu_pin: pinmux_P9_20_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* uart1_ctsn.gpio0_12 */
+	P9_20_gpio_pd_pin: pinmux_P9_20_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* uart1_ctsn.gpio0_12 */
+	P9_20_gpio_input_pin: pinmux_P9_20_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_INPUT | MUX_MODE7) >; };			/* uart1_ctsn.gpio0_12 */
+	P9_20_timer_pin: pinmux_P9_20_timer_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* uart1_ctsn.timer6 */
+	P9_20_can_pin: pinmux_P9_20_can_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | MUX_MODE2) >; };		/* uart1_ctsn.dcan0_tx */
+	P9_20_i2c_pin: pinmux_P9_20_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* uart1_ctsn.i2c2_sda */
+	P9_20_spi_cs_pin: pinmux_P9_20_spi_cs_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* uart1_ctsn.spi1_cs0 */
+	P9_20_pru_uart_pin: pinmux_P9_20_pru_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; };	/* uart1_ctsn.pr1_uart0_cts_n */
+
+	/* P9_21 (ZCZ ball B17) gpio0_3 */
+	P9_21_default_pin: pinmux_P9_21_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* spi0_d0.gpio0_3 */
+	P9_21_gpio_pin: pinmux_P9_21_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* spi0_d0.gpio0_3 */
+	P9_21_gpio_pu_pin: pinmux_P9_21_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* spi0_d0.gpio0_3 */
+	P9_21_gpio_pd_pin: pinmux_P9_21_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* spi0_d0.gpio0_3 */
+	P9_21_gpio_input_pin: pinmux_P9_21_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_INPUT | MUX_MODE7) >; };			/* spi0_d0.gpio0_3 */
+	P9_21_spi_pin: pinmux_P9_21_spi_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };	/* spi0_d0.spi0_d0 */
+	P9_21_uart_pin: pinmux_P9_21_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* spi0_d0.uart2_txd */
+	P9_21_i2c_pin: pinmux_P9_21_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* spi0_d0.i2c2_scl */
+	P9_21_pwm_pin: pinmux_P9_21_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; };	/* spi0_d0.ehrpwm0b */
+	P9_21_pru_uart_pin: pinmux_P9_21_pru_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* spi0_d0.pr1_uart0_rts_n */
+
+	/* P9_22 (ZCZ ball A17) gpio0_2 */
+	P9_22_default_pin: pinmux_P9_22_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* spi0_sclk.gpio0_2 */
+	P9_22_gpio_pin: pinmux_P9_22_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* spi0_sclk.gpio0_2 */
+	P9_22_gpio_pu_pin: pinmux_P9_22_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* spi0_sclk.gpio0_2 */
+	P9_22_gpio_pd_pin: pinmux_P9_22_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* spi0_sclk.gpio0_2 */
+	P9_22_gpio_input_pin: pinmux_P9_22_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_INPUT | MUX_MODE7) >; };			/* spi0_sclk.gpio0_2 */
+	P9_22_spi_sclk_pin: pinmux_P9_22_spi_sclk_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };	/* spi0_sclk.spi0_sclk */
+	P9_22_uart_pin: pinmux_P9_22_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* spi0_sclk.uart2_rxd */
+	P9_22_i2c_pin: pinmux_P9_22_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* spi0_sclk.i2c2_sda */
+	P9_22_pwm_pin: pinmux_P9_22_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; };	/* spi0_sclk.ehrpwm0a */
+	P9_22_pru_uart_pin: pinmux_P9_22_pru_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* spi0_sclk.pr1_uart0_cts_n */
+
+	/* P9_23 (ZCZ ball V14) gpio1_17 */
+	P9_23_default_pin: pinmux_P9_23_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a1.gpio1_17 */
+	P9_23_gpio_pin: pinmux_P9_23_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0844, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_a1.gpio1_17 */
+	P9_23_gpio_pu_pin: pinmux_P9_23_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a1.gpio1_17 */
+	P9_23_gpio_pd_pin: pinmux_P9_23_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a1.gpio1_17 */
+	P9_23_gpio_input_pin: pinmux_P9_23_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0844, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_a1.gpio1_17 */
+	P9_23_pwm_pin: pinmux_P9_23_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; };	/* gpmc_a1.ehrpwm0_synco */
+
+	/* P9_24 (ZCZ ball D15) gpio0_15 */
+	P9_24_default_pin: pinmux_P9_24_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* uart1_txd.gpio0_15 */
+	P9_24_gpio_pin: pinmux_P9_24_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* uart1_txd.gpio0_15 */
+	P9_24_gpio_pu_pin: pinmux_P9_24_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* uart1_txd.gpio0_15 */
+	P9_24_gpio_pd_pin: pinmux_P9_24_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* uart1_txd.gpio0_15 */
+	P9_24_gpio_input_pin: pinmux_P9_24_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_INPUT | MUX_MODE7) >; };			/* uart1_txd.gpio0_15 */
+	P9_24_uart_pin: pinmux_P9_24_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };	/* uart1_txd.uart1_txd */
+	P9_24_can_pin: pinmux_P9_24_can_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_INPUT_PULLUP | MUX_MODE2) >; };		/* uart1_txd.dcan1_rx */
+	P9_24_i2c_pin: pinmux_P9_24_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* uart1_txd.i2c1_scl */
+	P9_24_pru_uart_pin: pinmux_P9_24_pru_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; };	/* uart1_txd.pr1_uart0_txd */
+	P9_24_pruin_pin: pinmux_P9_24_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_INPUT | MUX_MODE6) >; };			/* uart1_txd.pru0_in16 */
+
+	/* P9_25 (ZCZ ball A14) audio */
+	P9_25_default_pin: pinmux_P9_25_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_ahclkx.gpio3_21 */
+	P9_25_gpio_pin: pinmux_P9_25_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09ac, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* mcasp0_ahclkx.gpio3_21 */
+	P9_25_gpio_pu_pin: pinmux_P9_25_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_ahclkx.gpio3_21 */
+	P9_25_gpio_pd_pin: pinmux_P9_25_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_ahclkx.gpio3_21 */
+	P9_25_gpio_input_pin: pinmux_P9_25_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09ac, PIN_INPUT | MUX_MODE7) >; };			/* mcasp0_ahclkx.gpio3_21 */
+	P9_25_qep_pin: pinmux_P9_25_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* mcasp0_ahclkx.eqep0_strobe */
+	P9_25_pruout_pin: pinmux_P9_25_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* mcasp0_ahclkx.pru0_out7 */
+	P9_25_pruin_pin: pinmux_P9_25_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09ac, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_ahclkx.pru0_in7 */
+
+	/* P9_26 (ZCZ ball D16) gpio0_14 */
+	P9_26_default_pin: pinmux_P9_26_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* uart1_rxd.gpio0_14 */
+	P9_26_gpio_pin: pinmux_P9_26_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* uart1_rxd.gpio0_14 */
+	P9_26_gpio_pu_pin: pinmux_P9_26_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* uart1_rxd.gpio0_14 */
+	P9_26_gpio_pd_pin: pinmux_P9_26_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* uart1_rxd.gpio0_14 */
+	P9_26_gpio_input_pin: pinmux_P9_26_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_INPUT | MUX_MODE7) >; };			/* uart1_rxd.gpio0_14 */
+	P9_26_uart_pin: pinmux_P9_26_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };	/* uart1_rxd.uart1_rxd */
+	P9_26_can_pin: pinmux_P9_26_can_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | MUX_MODE2) >; };		/* uart1_rxd.dcan1_tx */
+	P9_26_i2c_pin: pinmux_P9_26_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* uart1_rxd.i2c1_sda */
+	P9_26_pru_uart_pin: pinmux_P9_26_pru_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; };	/* uart1_rxd.pr1_uart0_rxd */
+	P9_26_pruin_pin: pinmux_P9_26_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_INPUT | MUX_MODE6) >; };			/* uart1_rxd.pru1_in16 */
+
+	/* P9_27 (ZCZ ball C13) gpio3_19 */
+	P9_27_default_pin: pinmux_P9_27_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_fsr.gpio3_19 */
+	P9_27_gpio_pin: pinmux_P9_27_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* mcasp0_fsr.gpio3_19 */
+	P9_27_gpio_pu_pin: pinmux_P9_27_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_fsr.gpio3_19 */
+	P9_27_gpio_pd_pin: pinmux_P9_27_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_fsr.gpio3_19 */
+	P9_27_gpio_input_pin: pinmux_P9_27_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a4, PIN_INPUT | MUX_MODE7) >; };			/* mcasp0_fsr.gpio3_19 */
+	P9_27_qep_pin: pinmux_P9_27_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* mcasp0_fsr.eqep0b_in */
+	P9_27_pruout_pin: pinmux_P9_27_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* mcasp0_fsr.pru0_out5 */
+	P9_27_pruin_pin: pinmux_P9_27_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a4, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_fsr.pru0_in5 */
+
+	/* P9_28 (ZCZ ball C12) audio */
+	P9_28_default_pin: pinmux_P9_28_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_ahclkr.gpio3_17 */
+	P9_28_gpio_pin: pinmux_P9_28_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* mcasp0_ahclkr.gpio3_17 */
+	P9_28_gpio_pu_pin: pinmux_P9_28_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_ahclkr.gpio3_17 */
+	P9_28_gpio_pd_pin: pinmux_P9_28_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_ahclkr.gpio3_17 */
+	P9_28_gpio_input_pin: pinmux_P9_28_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_INPUT | MUX_MODE7) >; };			/* mcasp0_ahclkr.gpio3_17 */
+	P9_28_pwm_pin: pinmux_P9_28_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; };	/* mcasp0_ahclkr.ehrpwm0_synci */
+	P9_28_spi_cs_pin: pinmux_P9_28_spi_cs_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* mcasp0_ahclkr.spi1_cs0 */
+	P9_28_pwm2_pin: pinmux_P9_28_pwm2_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; };	/* mcasp0_ahclkr.ecap2_in_pwm2_out */
+	P9_28_pruout_pin: pinmux_P9_28_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* mcasp0_ahclkr.pru0_out3 */
+	P9_28_pruin_pin: pinmux_P9_28_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_ahclkr.pru0_in3 */
+
+	/* P9_29 (ZCZ ball B13) audio */
+	P9_29_default_pin: pinmux_P9_29_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_fsx.gpio3_15 */
+	P9_29_gpio_pin: pinmux_P9_29_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0994, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* mcasp0_fsx.gpio3_15 */
+	P9_29_gpio_pu_pin: pinmux_P9_29_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_fsx.gpio3_15 */
+	P9_29_gpio_pd_pin: pinmux_P9_29_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_fsx.gpio3_15 */
+	P9_29_gpio_input_pin: pinmux_P9_29_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0994, PIN_INPUT | MUX_MODE7) >; };			/* mcasp0_fsx.gpio3_15 */
+	P9_29_pwm_pin: pinmux_P9_29_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; };	/* mcasp0_fsx.ehrpwm0b */
+	P9_29_spi_pin: pinmux_P9_29_spi_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* mcasp0_fsx.spi1_d0 */
+	P9_29_pruout_pin: pinmux_P9_29_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* mcasp0_fsx.pru0_out1 */
+	P9_29_pruin_pin: pinmux_P9_29_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0994, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_fsx.pru0_in1 */
+
+	/* P9_30 (ZCZ ball D12) gpio3_16 */
+	P9_30_default_pin: pinmux_P9_30_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_axr0.gpio3_16 */
+	P9_30_gpio_pin: pinmux_P9_30_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0998, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* mcasp0_axr0.gpio3_16 */
+	P9_30_gpio_pu_pin: pinmux_P9_30_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_axr0.gpio3_16 */
+	P9_30_gpio_pd_pin: pinmux_P9_30_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_axr0.gpio3_16 */
+	P9_30_gpio_input_pin: pinmux_P9_30_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0998, PIN_INPUT | MUX_MODE7) >; };			/* mcasp0_axr0.gpio3_16 */
+	P9_30_pwm_pin: pinmux_P9_30_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; };	/* mcasp0_axr0.ehrpwm0_tripzone_input */
+	P9_30_spi_pin: pinmux_P9_30_spi_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* mcasp0_axr0.spi1_d1 */
+	P9_30_pruout_pin: pinmux_P9_30_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* mcasp0_axr0.pru0_out2 */
+	P9_30_pruin_pin: pinmux_P9_30_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0998, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_axr0.pru0_in2 */
+
+	/* P9_31 (ZCZ ball A13) audio */
+	P9_31_default_pin: pinmux_P9_31_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_aclkx.gpio3_14 */
+	P9_31_gpio_pin: pinmux_P9_31_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0990, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* mcasp0_aclkx.gpio3_14 */
+	P9_31_gpio_pu_pin: pinmux_P9_31_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_aclkx.gpio3_14 */
+	P9_31_gpio_pd_pin: pinmux_P9_31_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_aclkx.gpio3_14 */
+	P9_31_gpio_input_pin: pinmux_P9_31_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0990, PIN_INPUT | MUX_MODE7) >; };			/* mcasp0_aclkx.gpio3_14 */
+	P9_31_pwm_pin: pinmux_P9_31_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; };	/* mcasp0_aclkx.ehrpwm0a */
+	P9_31_spi_sclk_pin: pinmux_P9_31_spi_sclk_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* mcasp0_aclkx.spi1_sclk */
+	P9_31_pruout_pin: pinmux_P9_31_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* mcasp0_aclkx.pru0_out0 */
+	P9_31_pruin_pin: pinmux_P9_31_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0990, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_aclkx.pru0_in0 */
+
+	/* P9_32                VADC */
+
+	/* P9_33 (ZCZ ball C8)  AIN4         */
+
+	/* P9_34                AGND */
+
+	/* P9_35 (ZCZ ball A8)  AIN6         */
+
+	/* P9_36 (ZCZ ball B8)  AIN5         */
+
+	/* P9_37 (ZCZ ball B7)  AIN2         */
+
+	/* P9_38 (ZCZ ball A7)  AIN3         */
+
+	/* P9_39 (ZCZ ball B6)  AIN0         */
+
+	/* P9_40 (ZCZ ball C7)  AIN1         */
+
+	/* P9_41 (ZCZ ball D14) gpio0_20 */
+	P9_41_default_pin: pinmux_P9_41_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* xdma_event_intr1.gpio0_20 */
+	P9_41_gpio_pin: pinmux_P9_41_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09b4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* xdma_event_intr1.gpio0_20 */
+	P9_41_gpio_pu_pin: pinmux_P9_41_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* xdma_event_intr1.gpio0_20 */
+	P9_41_gpio_pd_pin: pinmux_P9_41_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* xdma_event_intr1.gpio0_20 */
+	P9_41_gpio_input_pin: pinmux_P9_41_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09b4, PIN_INPUT | MUX_MODE7) >; };			/* xdma_event_intr1.gpio0_20 */
+	P9_41_timer_pin: pinmux_P9_41_timer_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* xdma_event_intr1.timer7 */
+	P9_41_pruin_pin: pinmux_P9_41_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09b4, PIN_INPUT | MUX_MODE5) >; };			/* xdma_event_intr1.pru0_in16 */
+
+	/* P9_41.1 */
+	/* P9_91 (ZCZ ball D13) gpio3_20 */
+	P9_91_default_pin: pinmux_P9_91_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_axr1.gpio3_20 */
+	P9_91_gpio_pin: pinmux_P9_91_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* mcasp0_axr1.gpio3_20 */
+	P9_91_gpio_pu_pin: pinmux_P9_91_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_axr1.gpio3_20 */
+	P9_91_gpio_pd_pin: pinmux_P9_91_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_axr1.gpio3_20 */
+	P9_91_gpio_input_pin: pinmux_P9_91_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a8, PIN_INPUT | MUX_MODE7) >; };			/* mcasp0_axr1.gpio3_20 */
+	P9_91_qep_pin: pinmux_P9_91_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* mcasp0_axr1.eqep0_index */
+	P9_91_pruout_pin: pinmux_P9_91_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* mcasp0_axr1.pru0_out6 */
+	P9_91_pruin_pin: pinmux_P9_91_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a8, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_axr1.pru0_in6 */
+
+	/* P9_42 (ZCZ ball C18) gpio0_7 */
+	P9_42_default_pin: pinmux_P9_42_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* eCAP0_in_PWM0_out.gpio0_7 */
+	P9_42_gpio_pin: pinmux_P9_42_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* eCAP0_in_PWM0_out.gpio0_7 */
+	P9_42_gpio_pu_pin: pinmux_P9_42_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* eCAP0_in_PWM0_out.gpio0_7 */
+	P9_42_gpio_pd_pin: pinmux_P9_42_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* eCAP0_in_PWM0_out.gpio0_7 */
+	P9_42_gpio_input_pin: pinmux_P9_42_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_INPUT | MUX_MODE7) >; };			/* eCAP0_in_PWM0_out.gpio0_7 */
+	P9_42_pwm_pin: pinmux_P9_42_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE0) >; };	/* eCAP0_in_PWM0_out.ecap0_in_pwm0_out */
+	P9_42_uart_pin: pinmux_P9_42_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* eCAP0_in_PWM0_out.uart3_txd */
+	P9_42_spi_cs_pin: pinmux_P9_42_spi_cs_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* eCAP0_in_PWM0_out.spi1_cs1 */
+	P9_42_pru_ecap_pin: pinmux_P9_42_pru_ecap_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; };	/* eCAP0_in_PWM0_out.pr1_ecap0_ecap_capin_apwm_o */
+	P9_42_spi_sclk_pin: pinmux_P9_42_spi_sclk_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* eCAP0_in_PWM0_out.spi1_sclk */
+
+	/* P9_42.1 */
+	/* P9_92 (ZCZ ball B12) gpio3_18 */
+	P9_92_default_pin: pinmux_P9_92_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_aclkr.gpio3_18 */
+	P9_92_gpio_pin: pinmux_P9_92_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* mcasp0_aclkr.gpio3_18 */
+	P9_92_gpio_pu_pin: pinmux_P9_92_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_aclkr.gpio3_18 */
+	P9_92_gpio_pd_pin: pinmux_P9_92_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_aclkr.gpio3_18 */
+	P9_92_gpio_input_pin: pinmux_P9_92_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a0, PIN_INPUT | MUX_MODE7) >; };			/* mcasp0_aclkr.gpio3_18 */
+	P9_92_qep_pin: pinmux_P9_92_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* mcasp0_aclkr.eqep0a_in */
+	P9_92_pruout_pin: pinmux_P9_92_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* mcasp0_aclkr.pru0_out4 */
+	P9_92_pruin_pin: pinmux_P9_92_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a0, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_aclkr.pru0_in4 */
+
+	/* P9_43                GND */
+
+	/* P9_44                GND */
+
+	/* P9_45                GND */
+
+	/* P9_46                GND */
+
+	/*       (ZCZ ball A15) */
+	A15_default_pin: pinmux_A15_default_pin {
+		pinctrl-single,pins = <0x1b0  (PIN_OUTPUT_PULLUP | MUX_MODE7)>; };     /* Mode 7, Pull-Up */
+	A15_clkout_pin: pinmux_A15_clkout_pin {
+		pinctrl-single,pins = <0x1b0  0x0b>; };     /* Mode 3 */
+	A15_gpio_pin: pinmux_A15_gpio_pin {
+		pinctrl-single,pins = <0x1b0  0x2f>; };     /* Mode 7, RxActive */
+	A15_gpio_pu_pin: pinmux_A15_gpio_pu_pin {
+		pinctrl-single,pins = <0x1b0  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	A15_gpio_pd_pin: pinmux_A15_gpio_pd_pin {
+		pinctrl-single,pins = <0x1b0  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+};
+
+&i2c1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+
+	clock-frequency = <100000>;
+	symlink = "bone/i2c/1";
+};
+
+&i2c2 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+
+	clock-frequency = <100000>;
+	symlink = "bone/i2c/2";
+};
+
+&uart1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+	symlink = "bone/uart/1";
+};
+
+&uart2 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+	symlink = "bone/uart/2";
+};
+
+&uart3 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+	symlink = "bone/uart/3";
+};
+
+&uart4 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+	symlink = "bone/uart/4";
+};
+
+&uart5 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+	symlink = "bone/uart/5";
+};
+
+&dcan0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+	symlink = "bone/can/0";
+};
+
+&dcan1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+	symlink = "bone/can/1";
+};
+
+&eqep0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+
+	count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */
+	swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */
+	invert_qa = <1>; /* Should we invert the channel A input? */
+	invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */
+	invert_qi = <0>; /* Should we invert the index input? */
+	invert_qs = <0>; /* Should we invert the strobe input? */
+	symlink = "bone/eqep/0";
+};
+
+&eqep1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+
+	count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */
+	swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */
+	invert_qa = <1>; /* Should we invert the channel A input? */
+	invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */
+	invert_qi = <0>; /* Should we invert the index input? */
+	invert_qs = <0>; /* Should we invert the strobe input? */
+	symlink = "bone/eqep/1";
+};
+
+&eqep2 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+
+	count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */
+	swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */
+	invert_qa = <1>; /* Should we invert the channel A input? */
+	invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */
+	invert_qi = <0>; /* Should we invert the index input? */
+	invert_qs = <0>; /* Should we invert the strobe input? */
+	symlink = "bone/eqep/2";
+};
+
+&epwmss0 {
+	status = "okay";
+};
+
+&epwmss1 {
+	status = "okay";
+};
+
+&epwmss2 {
+	status = "okay";
+};
+
+&ehrpwm0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+};
+
+&ehrpwm1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+};
+
+&ehrpwm2 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+};
+
+&ecap0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+};
+
+&ecap1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+};
+
+&ecap2 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+};
+
+&spi0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+
+	channel@0 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		compatible = "spidev";
+		symlink = "spi/0.0";
+
+		reg = <0>;
+		spi-max-frequency = <16000000>;
+		spi-cpha;
+	};
+
+	channel@1 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		compatible = "spidev";
+		symlink = "spi/0.1";
+
+		reg = <1>;
+		spi-max-frequency = <16000000>;
+	};
+};
+
+&spi1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+
+	channel@0 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		compatible = "spidev";
+		symlink = "spi/1.0";
+
+		reg = <0>;
+		spi-max-frequency = <16000000>;
+		spi-cpha;
+	};
+
+	channel@1 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		compatible = "spidev";
+		symlink = "spi/1.1";
+
+		reg = <1>;
+		spi-max-frequency = <16000000>;
+	};
+};
+
+/**********************************************************************/
+/* Pin Multiplex Helpers                                              */
+/*                                                                    */
+/* These provide userspace runtime pin configuration for the          */
+/* BeagleBone cape expansion headers                                  */
+/**********************************************************************/
+
+&ocp {
+	/************************/
+	/* P8 Header */
+	/************************/
+
+	/* P8_01                GND */
+
+	/* P8_02                GND */
+
+
+	/* P8_03 (ZCZ ball R9) emmc */
+	P8_03_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input";
+		pinctrl-0 = <&P8_03_default_pin>;
+		pinctrl-1 = <&P8_03_gpio_pin>;
+		pinctrl-2 = <&P8_03_gpio_pu_pin>;
+		pinctrl-3 = <&P8_03_gpio_pd_pin>;
+		pinctrl-4 = <&P8_03_gpio_input_pin>;
+	};
+
+	/* P8_04 (ZCZ ball T9) emmc */
+	P8_04_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input";
+		pinctrl-0 = <&P8_04_default_pin>;
+		pinctrl-1 = <&P8_04_gpio_pin>;
+		pinctrl-2 = <&P8_04_gpio_pu_pin>;
+		pinctrl-3 = <&P8_04_gpio_pd_pin>;
+		pinctrl-4 = <&P8_04_gpio_input_pin>;
+	};
+
+	/* P8_05 (ZCZ ball R8) emmc */
+	P8_05_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input";
+		pinctrl-0 = <&P8_05_default_pin>;
+		pinctrl-1 = <&P8_05_gpio_pin>;
+		pinctrl-2 = <&P8_05_gpio_pu_pin>;
+		pinctrl-3 = <&P8_05_gpio_pd_pin>;
+		pinctrl-4 = <&P8_05_gpio_input_pin>;
+	};
+
+	/* P8_06 (ZCZ ball T8) emmc */
+	P8_06_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input";
+		pinctrl-0 = <&P8_06_default_pin>;
+		pinctrl-1 = <&P8_06_gpio_pin>;
+		pinctrl-2 = <&P8_06_gpio_pu_pin>;
+		pinctrl-3 = <&P8_06_gpio_pd_pin>;
+		pinctrl-4 = <&P8_06_gpio_input_pin>;
+	};
+
+	/* P8_07 (ZCZ ball R7) */
+	P8_07_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer";
+		pinctrl-0 = <&P8_07_default_pin>;
+		pinctrl-1 = <&P8_07_gpio_pin>;
+		pinctrl-2 = <&P8_07_gpio_pu_pin>;
+		pinctrl-3 = <&P8_07_gpio_pd_pin>;
+		pinctrl-4 = <&P8_07_gpio_input_pin>;
+		pinctrl-5 = <&P8_07_timer_pin>;
+	};
+
+	/* P8_08 (ZCZ ball T7) */
+	P8_08_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer";
+		pinctrl-0 = <&P8_08_default_pin>;
+		pinctrl-1 = <&P8_08_gpio_pin>;
+		pinctrl-2 = <&P8_08_gpio_pu_pin>;
+		pinctrl-3 = <&P8_08_gpio_pd_pin>;
+		pinctrl-4 = <&P8_08_gpio_input_pin>;
+		pinctrl-5 = <&P8_08_timer_pin>;
+	};
+
+	/* P8_09 (ZCZ ball T6) */
+	P8_09_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer";
+		pinctrl-0 = <&P8_09_default_pin>;
+		pinctrl-1 = <&P8_09_gpio_pin>;
+		pinctrl-2 = <&P8_09_gpio_pu_pin>;
+		pinctrl-3 = <&P8_09_gpio_pd_pin>;
+		pinctrl-4 = <&P8_09_gpio_input_pin>;
+		pinctrl-5 = <&P8_09_timer_pin>;
+	};
+
+	/* P8_10 (ZCZ ball U6) */
+	P8_10_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer";
+		pinctrl-0 = <&P8_10_default_pin>;
+		pinctrl-1 = <&P8_10_gpio_pin>;
+		pinctrl-2 = <&P8_10_gpio_pu_pin>;
+		pinctrl-3 = <&P8_10_gpio_pd_pin>;
+		pinctrl-4 = <&P8_10_gpio_input_pin>;
+		pinctrl-5 = <&P8_10_timer_pin>;
+	};
+
+	/* P8_11 (ZCZ ball R12) */
+	P8_11_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout";
+		pinctrl-0 = <&P8_11_default_pin>;
+		pinctrl-1 = <&P8_11_gpio_pin>;
+		pinctrl-2 = <&P8_11_gpio_pu_pin>;
+		pinctrl-3 = <&P8_11_gpio_pd_pin>;
+		pinctrl-4 = <&P8_11_gpio_input_pin>;
+		pinctrl-5 = <&P8_11_qep_pin>;
+		pinctrl-6 = <&P8_11_pruout_pin>;
+	};
+
+	/* P8_12 (ZCZ ball T12) */
+	P8_12_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout";
+		pinctrl-0 = <&P8_12_default_pin>;
+		pinctrl-1 = <&P8_12_gpio_pin>;
+		pinctrl-2 = <&P8_12_gpio_pu_pin>;
+		pinctrl-3 = <&P8_12_gpio_pd_pin>;
+		pinctrl-4 = <&P8_12_gpio_input_pin>;
+		pinctrl-5 = <&P8_12_qep_pin>;
+		pinctrl-6 = <&P8_12_pruout_pin>;
+	};
+
+	/* P8_13 (ZCZ ball T10) */
+	P8_13_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm";
+		pinctrl-0 = <&P8_13_default_pin>;
+		pinctrl-1 = <&P8_13_gpio_pin>;
+		pinctrl-2 = <&P8_13_gpio_pu_pin>;
+		pinctrl-3 = <&P8_13_gpio_pd_pin>;
+		pinctrl-4 = <&P8_13_gpio_input_pin>;
+		pinctrl-5 = <&P8_13_pwm_pin>;
+	};
+
+	/* P8_14 (ZCZ ball T11) */
+	P8_14_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm";
+		pinctrl-0 = <&P8_14_default_pin>;
+		pinctrl-1 = <&P8_14_gpio_pin>;
+		pinctrl-2 = <&P8_14_gpio_pu_pin>;
+		pinctrl-3 = <&P8_14_gpio_pd_pin>;
+		pinctrl-4 = <&P8_14_gpio_input_pin>;
+		pinctrl-5 = <&P8_14_pwm_pin>;
+	};
+
+	/* P8_15 (ZCZ ball U13) */
+	P8_15_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pru_ecap", "pruin";
+		pinctrl-0 = <&P8_15_default_pin>;
+		pinctrl-1 = <&P8_15_gpio_pin>;
+		pinctrl-2 = <&P8_15_gpio_pu_pin>;
+		pinctrl-3 = <&P8_15_gpio_pd_pin>;
+		pinctrl-4 = <&P8_15_gpio_input_pin>;
+		pinctrl-5 = <&P8_15_qep_pin>;
+		pinctrl-6 = <&P8_15_pru_ecap_pin>;
+		pinctrl-7 = <&P8_15_pruin_pin>;
+	};
+
+	/* P8_16 (ZCZ ball V13) */
+	P8_16_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruin";
+		pinctrl-0 = <&P8_16_default_pin>;
+		pinctrl-1 = <&P8_16_gpio_pin>;
+		pinctrl-2 = <&P8_16_gpio_pu_pin>;
+		pinctrl-3 = <&P8_16_gpio_pd_pin>;
+		pinctrl-4 = <&P8_16_gpio_input_pin>;
+		pinctrl-5 = <&P8_16_qep_pin>;
+		pinctrl-6 = <&P8_16_pruin_pin>;
+	};
+
+	/* P8_17 (ZCZ ball U12) */
+	P8_17_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm";
+		pinctrl-0 = <&P8_17_default_pin>;
+		pinctrl-1 = <&P8_17_gpio_pin>;
+		pinctrl-2 = <&P8_17_gpio_pu_pin>;
+		pinctrl-3 = <&P8_17_gpio_pd_pin>;
+		pinctrl-4 = <&P8_17_gpio_input_pin>;
+		pinctrl-5 = <&P8_17_pwm_pin>;
+	};
+
+	/* P8_18 (ZCZ ball V12) */
+	P8_18_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input";
+		pinctrl-0 = <&P8_18_default_pin>;
+		pinctrl-1 = <&P8_18_gpio_pin>;
+		pinctrl-2 = <&P8_18_gpio_pu_pin>;
+		pinctrl-3 = <&P8_18_gpio_pd_pin>;
+		pinctrl-4 = <&P8_18_gpio_input_pin>;
+	};
+
+	/* P8_19 (ZCZ ball U10) */
+	P8_19_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm";
+		pinctrl-0 = <&P8_19_default_pin>;
+		pinctrl-1 = <&P8_19_gpio_pin>;
+		pinctrl-2 = <&P8_19_gpio_pu_pin>;
+		pinctrl-3 = <&P8_19_gpio_pd_pin>;
+		pinctrl-4 = <&P8_19_gpio_input_pin>;
+		pinctrl-5 = <&P8_19_pwm_pin>;
+	};
+
+	/* P8_20 (ZCZ ball V9) emmc */
+	P8_20_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin";
+		pinctrl-0 = <&P8_20_default_pin>;
+		pinctrl-1 = <&P8_20_gpio_pin>;
+		pinctrl-2 = <&P8_20_gpio_pu_pin>;
+		pinctrl-3 = <&P8_20_gpio_pd_pin>;
+		pinctrl-4 = <&P8_20_gpio_input_pin>;
+		pinctrl-5 = <&P8_20_pruout_pin>;
+		pinctrl-6 = <&P8_20_pruin_pin>;
+	};
+
+	/* P8_21 (ZCZ ball U9) emmc */
+	P8_21_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin";
+		pinctrl-0 = <&P8_21_default_pin>;
+		pinctrl-1 = <&P8_21_gpio_pin>;
+		pinctrl-2 = <&P8_21_gpio_pu_pin>;
+		pinctrl-3 = <&P8_21_gpio_pd_pin>;
+		pinctrl-4 = <&P8_21_gpio_input_pin>;
+		pinctrl-5 = <&P8_21_pruout_pin>;
+		pinctrl-6 = <&P8_21_pruin_pin>;
+	};
+
+	/* P8_22 (ZCZ ball V8) emmc */
+	P8_22_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input";
+		pinctrl-0 = <&P8_22_default_pin>;
+		pinctrl-1 = <&P8_22_gpio_pin>;
+		pinctrl-2 = <&P8_22_gpio_pu_pin>;
+		pinctrl-3 = <&P8_22_gpio_pd_pin>;
+		pinctrl-4 = <&P8_22_gpio_input_pin>;
+	};
+
+	/* P8_23 (ZCZ ball U8) emmc */
+	P8_23_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input";
+		pinctrl-0 = <&P8_23_default_pin>;
+		pinctrl-1 = <&P8_23_gpio_pin>;
+		pinctrl-2 = <&P8_23_gpio_pu_pin>;
+		pinctrl-3 = <&P8_23_gpio_pd_pin>;
+		pinctrl-4 = <&P8_23_gpio_input_pin>;
+	};
+
+	/* P8_24 (ZCZ ball V7) emmc */
+	P8_24_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input";
+		pinctrl-0 = <&P8_24_default_pin>;
+		pinctrl-1 = <&P8_24_gpio_pin>;
+		pinctrl-2 = <&P8_24_gpio_pu_pin>;
+		pinctrl-3 = <&P8_24_gpio_pd_pin>;
+		pinctrl-4 = <&P8_24_gpio_input_pin>;
+	};
+
+	/* P8_25 (ZCZ ball U7) emmc */
+	P8_25_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input";
+		pinctrl-0 = <&P8_25_default_pin>;
+		pinctrl-1 = <&P8_25_gpio_pin>;
+		pinctrl-2 = <&P8_25_gpio_pu_pin>;
+		pinctrl-3 = <&P8_25_gpio_pd_pin>;
+		pinctrl-4 = <&P8_25_gpio_input_pin>;
+	};
+
+	/* P8_26 (ZCZ ball V6) */
+	P8_26_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input";
+		pinctrl-0 = <&P8_26_default_pin>;
+		pinctrl-1 = <&P8_26_gpio_pin>;
+		pinctrl-2 = <&P8_26_gpio_pu_pin>;
+		pinctrl-3 = <&P8_26_gpio_pd_pin>;
+		pinctrl-4 = <&P8_26_gpio_input_pin>;
+	};
+
+	/* P8_27 (ZCZ ball U5) hdmi */
+	P8_27_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin";
+		pinctrl-0 = <&P8_27_default_pin>;
+		pinctrl-1 = <&P8_27_gpio_pin>;
+		pinctrl-2 = <&P8_27_gpio_pu_pin>;
+		pinctrl-3 = <&P8_27_gpio_pd_pin>;
+		pinctrl-4 = <&P8_27_gpio_input_pin>;
+		pinctrl-5 = <&P8_27_pruout_pin>;
+		pinctrl-6 = <&P8_27_pruin_pin>;
+	};
+
+	/* P8_28 (ZCZ ball V5) hdmi */
+	P8_28_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin";
+		pinctrl-0 = <&P8_28_default_pin>;
+		pinctrl-1 = <&P8_28_gpio_pin>;
+		pinctrl-2 = <&P8_28_gpio_pu_pin>;
+		pinctrl-3 = <&P8_28_gpio_pd_pin>;
+		pinctrl-4 = <&P8_28_gpio_input_pin>;
+		pinctrl-5 = <&P8_28_pruout_pin>;
+		pinctrl-6 = <&P8_28_pruin_pin>;
+	};
+
+	/* P8_29 (ZCZ ball R5) hdmi */
+	P8_29_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin";
+		pinctrl-0 = <&P8_29_default_pin>;
+		pinctrl-1 = <&P8_29_gpio_pin>;
+		pinctrl-2 = <&P8_29_gpio_pu_pin>;
+		pinctrl-3 = <&P8_29_gpio_pd_pin>;
+		pinctrl-4 = <&P8_29_gpio_input_pin>;
+		pinctrl-5 = <&P8_29_pruout_pin>;
+		pinctrl-6 = <&P8_29_pruin_pin>;
+	};
+
+	/* P8_30 (ZCZ ball R6) hdmi */
+	P8_30_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin";
+		pinctrl-0 = <&P8_30_default_pin>;
+		pinctrl-1 = <&P8_30_gpio_pin>;
+		pinctrl-2 = <&P8_30_gpio_pu_pin>;
+		pinctrl-3 = <&P8_30_gpio_pd_pin>;
+		pinctrl-4 = <&P8_30_gpio_input_pin>;
+		pinctrl-5 = <&P8_30_pruout_pin>;
+		pinctrl-6 = <&P8_30_pruin_pin>;
+	};
+
+	/* P8_31 (ZCZ ball V4) hdmi */
+	P8_31_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "qep";
+		pinctrl-0 = <&P8_31_default_pin>;
+		pinctrl-1 = <&P8_31_gpio_pin>;
+		pinctrl-2 = <&P8_31_gpio_pu_pin>;
+		pinctrl-3 = <&P8_31_gpio_pd_pin>;
+		pinctrl-4 = <&P8_31_gpio_input_pin>;
+		pinctrl-5 = <&P8_31_uart_pin>;
+		pinctrl-6 = <&P8_31_qep_pin>;
+	};
+
+	/* P8_32 (ZCZ ball T5) hdmi */
+	P8_32_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep";
+		pinctrl-0 = <&P8_32_default_pin>;
+		pinctrl-1 = <&P8_32_gpio_pin>;
+		pinctrl-2 = <&P8_32_gpio_pu_pin>;
+		pinctrl-3 = <&P8_32_gpio_pd_pin>;
+		pinctrl-4 = <&P8_32_gpio_input_pin>;
+		pinctrl-5 = <&P8_32_qep_pin>;
+	};
+
+	/* P8_33 (ZCZ ball V3) hdmi */
+	P8_33_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep";
+		pinctrl-0 = <&P8_33_default_pin>;
+		pinctrl-1 = <&P8_33_gpio_pin>;
+		pinctrl-2 = <&P8_33_gpio_pu_pin>;
+		pinctrl-3 = <&P8_33_gpio_pd_pin>;
+		pinctrl-4 = <&P8_33_gpio_input_pin>;
+		pinctrl-5 = <&P8_33_qep_pin>;
+	};
+
+	/* P8_34 (ZCZ ball U4) hdmi */
+	P8_34_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm";
+		pinctrl-0 = <&P8_34_default_pin>;
+		pinctrl-1 = <&P8_34_gpio_pin>;
+		pinctrl-2 = <&P8_34_gpio_pu_pin>;
+		pinctrl-3 = <&P8_34_gpio_pd_pin>;
+		pinctrl-4 = <&P8_34_gpio_input_pin>;
+		pinctrl-5 = <&P8_34_pwm_pin>;
+	};
+
+	/* P8_35 (ZCZ ball V2) hdmi */
+	P8_35_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep";
+		pinctrl-0 = <&P8_35_default_pin>;
+		pinctrl-1 = <&P8_35_gpio_pin>;
+		pinctrl-2 = <&P8_35_gpio_pu_pin>;
+		pinctrl-3 = <&P8_35_gpio_pd_pin>;
+		pinctrl-4 = <&P8_35_gpio_input_pin>;
+		pinctrl-5 = <&P8_35_qep_pin>;
+	};
+
+	/* P8_36 (ZCZ ball U3) hdmi */
+	P8_36_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm";
+		pinctrl-0 = <&P8_36_default_pin>;
+		pinctrl-1 = <&P8_36_gpio_pin>;
+		pinctrl-2 = <&P8_36_gpio_pu_pin>;
+		pinctrl-3 = <&P8_36_gpio_pd_pin>;
+		pinctrl-4 = <&P8_36_gpio_input_pin>;
+		pinctrl-5 = <&P8_36_pwm_pin>;
+	};
+
+	/* P8_37 (ZCZ ball U1) hdmi */
+	P8_37_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "pwm";
+		pinctrl-0 = <&P8_37_default_pin>;
+		pinctrl-1 = <&P8_37_gpio_pin>;
+		pinctrl-2 = <&P8_37_gpio_pu_pin>;
+		pinctrl-3 = <&P8_37_gpio_pd_pin>;
+		pinctrl-4 = <&P8_37_gpio_input_pin>;
+		pinctrl-5 = <&P8_37_uart_pin>;
+		pinctrl-6 = <&P8_37_pwm_pin>;
+	};
+
+	/* P8_38 (ZCZ ball U2) hdmi */
+	P8_38_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "pwm";
+		pinctrl-0 = <&P8_38_default_pin>;
+		pinctrl-1 = <&P8_38_gpio_pin>;
+		pinctrl-2 = <&P8_38_gpio_pu_pin>;
+		pinctrl-3 = <&P8_38_gpio_pd_pin>;
+		pinctrl-4 = <&P8_38_gpio_input_pin>;
+		pinctrl-5 = <&P8_38_uart_pin>;
+		pinctrl-6 = <&P8_38_pwm_pin>;
+	};
+
+	/* P8_39 (ZCZ ball T3) hdmi */
+	P8_39_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin";
+		pinctrl-0 = <&P8_39_default_pin>;
+		pinctrl-1 = <&P8_39_gpio_pin>;
+		pinctrl-2 = <&P8_39_gpio_pu_pin>;
+		pinctrl-3 = <&P8_39_gpio_pd_pin>;
+		pinctrl-4 = <&P8_39_gpio_input_pin>;
+		pinctrl-5 = <&P8_39_qep_pin>;
+		pinctrl-6 = <&P8_39_pruout_pin>;
+		pinctrl-7 = <&P8_39_pruin_pin>;
+	};
+
+	/* P8_40 (ZCZ ball T4) hdmi */
+	P8_40_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin";
+		pinctrl-0 = <&P8_40_default_pin>;
+		pinctrl-1 = <&P8_40_gpio_pin>;
+		pinctrl-2 = <&P8_40_gpio_pu_pin>;
+		pinctrl-3 = <&P8_40_gpio_pd_pin>;
+		pinctrl-4 = <&P8_40_gpio_input_pin>;
+		pinctrl-5 = <&P8_40_qep_pin>;
+		pinctrl-6 = <&P8_40_pruout_pin>;
+		pinctrl-7 = <&P8_40_pruin_pin>;
+	};
+
+	/* P8_41 (ZCZ ball T1) hdmi */
+	P8_41_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin";
+		pinctrl-0 = <&P8_41_default_pin>;
+		pinctrl-1 = <&P8_41_gpio_pin>;
+		pinctrl-2 = <&P8_41_gpio_pu_pin>;
+		pinctrl-3 = <&P8_41_gpio_pd_pin>;
+		pinctrl-4 = <&P8_41_gpio_input_pin>;
+		pinctrl-5 = <&P8_41_qep_pin>;
+		pinctrl-6 = <&P8_41_pruout_pin>;
+		pinctrl-7 = <&P8_41_pruin_pin>;
+	};
+
+	/* P8_42 (ZCZ ball T2) hdmi */
+	P8_42_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin";
+		pinctrl-0 = <&P8_42_default_pin>;
+		pinctrl-1 = <&P8_42_gpio_pin>;
+		pinctrl-2 = <&P8_42_gpio_pu_pin>;
+		pinctrl-3 = <&P8_42_gpio_pd_pin>;
+		pinctrl-4 = <&P8_42_gpio_input_pin>;
+		pinctrl-5 = <&P8_42_qep_pin>;
+		pinctrl-6 = <&P8_42_pruout_pin>;
+		pinctrl-7 = <&P8_42_pruin_pin>;
+	};
+
+	/* P8_43 (ZCZ ball R3) hdmi */
+	P8_43_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin";
+		pinctrl-0 = <&P8_43_default_pin>;
+		pinctrl-1 = <&P8_43_gpio_pin>;
+		pinctrl-2 = <&P8_43_gpio_pu_pin>;
+		pinctrl-3 = <&P8_43_gpio_pd_pin>;
+		pinctrl-4 = <&P8_43_gpio_input_pin>;
+		pinctrl-5 = <&P8_43_pwm_pin>;
+		pinctrl-6 = <&P8_43_pruout_pin>;
+		pinctrl-7 = <&P8_43_pruin_pin>;
+	};
+
+	/* P8_44 (ZCZ ball R4) hdmi */
+	P8_44_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin";
+		pinctrl-0 = <&P8_44_default_pin>;
+		pinctrl-1 = <&P8_44_gpio_pin>;
+		pinctrl-2 = <&P8_44_gpio_pu_pin>;
+		pinctrl-3 = <&P8_44_gpio_pd_pin>;
+		pinctrl-4 = <&P8_44_gpio_input_pin>;
+		pinctrl-5 = <&P8_44_pwm_pin>;
+		pinctrl-6 = <&P8_44_pruout_pin>;
+		pinctrl-7 = <&P8_44_pruin_pin>;
+	};
+
+	/* P8_45 (ZCZ ball R1) hdmi */
+	P8_45_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin";
+		pinctrl-0 = <&P8_45_default_pin>;
+		pinctrl-1 = <&P8_45_gpio_pin>;
+		pinctrl-2 = <&P8_45_gpio_pu_pin>;
+		pinctrl-3 = <&P8_45_gpio_pd_pin>;
+		pinctrl-4 = <&P8_45_gpio_input_pin>;
+		pinctrl-5 = <&P8_45_pwm_pin>;
+		pinctrl-6 = <&P8_45_pruout_pin>;
+		pinctrl-7 = <&P8_45_pruin_pin>;
+	};
+
+	/* P8_46 (ZCZ ball R2) hdmi */
+	P8_46_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin";
+		pinctrl-0 = <&P8_46_default_pin>;
+		pinctrl-1 = <&P8_46_gpio_pin>;
+		pinctrl-2 = <&P8_46_gpio_pu_pin>;
+		pinctrl-3 = <&P8_46_gpio_pd_pin>;
+		pinctrl-4 = <&P8_46_gpio_input_pin>;
+		pinctrl-5 = <&P8_46_pwm_pin>;
+		pinctrl-6 = <&P8_46_pruout_pin>;
+		pinctrl-7 = <&P8_46_pruin_pin>;
+	};
+
+	/************************/
+	/* P9 Header */
+	/************************/
+
+	/* P9_01                GND */
+
+	/* P9_02                GND */
+
+	/* P9_03                3V3 */
+
+	/* P9_04                3V3 */
+
+	/* P9_05                VDD_5V */
+
+	/* P9_06                VDD_5V */
+
+	/* P9_07                SYS_5V */
+
+	/* P9_08                SYS_5V */
+
+	/* P9_09                PWR_BUT */
+
+	/* P9_10                RSTn */
+
+	/* P9_11 (ZCZ ball T17) */
+	P9_11_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart";
+		pinctrl-0 = <&P9_11_default_pin>;
+		pinctrl-1 = <&P9_11_gpio_pin>;
+		pinctrl-2 = <&P9_11_gpio_pu_pin>;
+		pinctrl-3 = <&P9_11_gpio_pd_pin>;
+		pinctrl-4 = <&P9_11_gpio_input_pin>;
+		pinctrl-5 = <&P9_11_uart_pin>;
+	};
+
+	/* P9_12 (ZCZ ball U18) */
+	P9_12_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input";
+		pinctrl-0 = <&P9_12_default_pin>;
+		pinctrl-1 = <&P9_12_gpio_pin>;
+		pinctrl-2 = <&P9_12_gpio_pu_pin>;
+		pinctrl-3 = <&P9_12_gpio_pd_pin>;
+		pinctrl-4 = <&P9_12_gpio_input_pin>;
+	};
+
+	/* P9_13 (ZCZ ball U17) */
+	P9_13_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart";
+		pinctrl-0 = <&P9_13_default_pin>;
+		pinctrl-1 = <&P9_13_gpio_pin>;
+		pinctrl-2 = <&P9_13_gpio_pu_pin>;
+		pinctrl-3 = <&P9_13_gpio_pd_pin>;
+		pinctrl-4 = <&P9_13_gpio_input_pin>;
+		pinctrl-5 = <&P9_13_uart_pin>;
+	};
+
+	/* P9_14 (ZCZ ball U14) */
+	P9_14_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm";
+		pinctrl-0 = <&P9_14_default_pin>;
+		pinctrl-1 = <&P9_14_gpio_pin>;
+		pinctrl-2 = <&P9_14_gpio_pu_pin>;
+		pinctrl-3 = <&P9_14_gpio_pd_pin>;
+		pinctrl-4 = <&P9_14_gpio_input_pin>;
+		pinctrl-5 = <&P9_14_pwm_pin>;
+	};
+
+	/* P9_15 (ZCZ ball R13) */
+	P9_15_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm";
+		pinctrl-0 = <&P9_15_default_pin>;
+		pinctrl-1 = <&P9_15_gpio_pin>;
+		pinctrl-2 = <&P9_15_gpio_pu_pin>;
+		pinctrl-3 = <&P9_15_gpio_pd_pin>;
+		pinctrl-4 = <&P9_15_gpio_input_pin>;
+		pinctrl-5 = <&P9_15_pwm_pin>;
+	};
+
+	/* P9_16 (ZCZ ball T14) */
+	P9_16_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm";
+		pinctrl-0 = <&P9_16_default_pin>;
+		pinctrl-1 = <&P9_16_gpio_pin>;
+		pinctrl-2 = <&P9_16_gpio_pu_pin>;
+		pinctrl-3 = <&P9_16_gpio_pd_pin>;
+		pinctrl-4 = <&P9_16_gpio_input_pin>;
+		pinctrl-5 = <&P9_16_pwm_pin>;
+	};
+
+	/* P9_17 (ZCZ ball A16) */
+	P9_17_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "i2c", "pwm", "pru_uart";
+		pinctrl-0 = <&P9_17_default_pin>;
+		pinctrl-1 = <&P9_17_gpio_pin>;
+		pinctrl-2 = <&P9_17_gpio_pu_pin>;
+		pinctrl-3 = <&P9_17_gpio_pd_pin>;
+		pinctrl-4 = <&P9_17_gpio_input_pin>;
+		pinctrl-5 = <&P9_17_spi_cs_pin>;
+		pinctrl-6 = <&P9_17_i2c_pin>;
+		pinctrl-7 = <&P9_17_pwm_pin>;
+		pinctrl-8 = <&P9_17_pru_uart_pin>;
+	};
+
+	/* P9_18 (ZCZ ball B16) */
+	P9_18_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "i2c", "pwm", "pru_uart";
+		pinctrl-0 = <&P9_18_default_pin>;
+		pinctrl-1 = <&P9_18_gpio_pin>;
+		pinctrl-2 = <&P9_18_gpio_pu_pin>;
+		pinctrl-3 = <&P9_18_gpio_pd_pin>;
+		pinctrl-4 = <&P9_18_gpio_input_pin>;
+		pinctrl-5 = <&P9_18_spi_pin>;
+		pinctrl-6 = <&P9_18_i2c_pin>;
+		pinctrl-7 = <&P9_18_pwm_pin>;
+		pinctrl-8 = <&P9_18_pru_uart_pin>;
+	};
+
+	/* P9_19 (ZCZ ball D17) i2c */
+	P9_19_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart", "timer";
+		pinctrl-0 = <&P9_19_default_pin>;
+		pinctrl-1 = <&P9_19_gpio_pin>;
+		pinctrl-2 = <&P9_19_gpio_pu_pin>;
+		pinctrl-3 = <&P9_19_gpio_pd_pin>;
+		pinctrl-4 = <&P9_19_gpio_input_pin>;
+		pinctrl-5 = <&P9_19_spi_cs_pin>;
+		pinctrl-6 = <&P9_19_can_pin>;
+		pinctrl-7 = <&P9_19_i2c_pin>;
+		pinctrl-8 = <&P9_19_pru_uart_pin>;
+		pinctrl-9 = <&P9_19_timer_pin>;
+	};
+
+	/* P9_20 (ZCZ ball D18) i2c */
+	P9_20_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart", "timer";
+		pinctrl-0 = <&P9_20_default_pin>;
+		pinctrl-1 = <&P9_20_gpio_pin>;
+		pinctrl-2 = <&P9_20_gpio_pu_pin>;
+		pinctrl-3 = <&P9_20_gpio_pd_pin>;
+		pinctrl-4 = <&P9_20_gpio_input_pin>;
+		pinctrl-5 = <&P9_20_spi_cs_pin>;
+		pinctrl-6 = <&P9_20_can_pin>;
+		pinctrl-7 = <&P9_20_i2c_pin>;
+		pinctrl-8 = <&P9_20_pru_uart_pin>;
+		pinctrl-9 = <&P9_20_timer_pin>;
+	};
+
+	/* P9_21 (ZCZ ball B17) */
+	P9_21_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "uart", "i2c", "pwm", "pru_uart";
+		pinctrl-0 = <&P9_21_default_pin>;
+		pinctrl-1 = <&P9_21_gpio_pin>;
+		pinctrl-2 = <&P9_21_gpio_pu_pin>;
+		pinctrl-3 = <&P9_21_gpio_pd_pin>;
+		pinctrl-4 = <&P9_21_gpio_input_pin>;
+		pinctrl-5 = <&P9_21_spi_pin>;
+		pinctrl-6 = <&P9_21_uart_pin>;
+		pinctrl-7 = <&P9_21_i2c_pin>;
+		pinctrl-8 = <&P9_21_pwm_pin>;
+		pinctrl-9 = <&P9_21_pru_uart_pin>;
+	};
+
+	/* P9_22 (ZCZ ball A17) */
+	P9_22_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_sclk", "uart", "i2c", "pwm", "pru_uart";
+		pinctrl-0 = <&P9_22_default_pin>;
+		pinctrl-1 = <&P9_22_gpio_pin>;
+		pinctrl-2 = <&P9_22_gpio_pu_pin>;
+		pinctrl-3 = <&P9_22_gpio_pd_pin>;
+		pinctrl-4 = <&P9_22_gpio_input_pin>;
+		pinctrl-5 = <&P9_22_spi_sclk_pin>;
+		pinctrl-6 = <&P9_22_uart_pin>;
+		pinctrl-7 = <&P9_22_i2c_pin>;
+		pinctrl-8 = <&P9_22_pwm_pin>;
+		pinctrl-9 = <&P9_22_pru_uart_pin>;
+	};
+
+	/* P9_23 (ZCZ ball V14) */
+	P9_23_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm";
+		pinctrl-0 = <&P9_23_default_pin>;
+		pinctrl-1 = <&P9_23_gpio_pin>;
+		pinctrl-2 = <&P9_23_gpio_pu_pin>;
+		pinctrl-3 = <&P9_23_gpio_pd_pin>;
+		pinctrl-4 = <&P9_23_gpio_input_pin>;
+		pinctrl-5 = <&P9_23_pwm_pin>;
+	};
+
+	/* P9_24 (ZCZ ball D15) */
+	P9_24_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "can", "i2c", "pru_uart", "pruin";
+		pinctrl-0 = <&P9_24_default_pin>;
+		pinctrl-1 = <&P9_24_gpio_pin>;
+		pinctrl-2 = <&P9_24_gpio_pu_pin>;
+		pinctrl-3 = <&P9_24_gpio_pd_pin>;
+		pinctrl-4 = <&P9_24_gpio_input_pin>;
+		pinctrl-5 = <&P9_24_uart_pin>;
+		pinctrl-6 = <&P9_24_can_pin>;
+		pinctrl-7 = <&P9_24_i2c_pin>;
+		pinctrl-8 = <&P9_24_pru_uart_pin>;
+		pinctrl-9 = <&P9_24_pruin_pin>;
+	};
+
+	/* P9_25 (ZCZ ball A14) audio */
+	P9_25_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin";
+		pinctrl-0 = <&P9_25_default_pin>;
+		pinctrl-1 = <&P9_25_gpio_pin>;
+		pinctrl-2 = <&P9_25_gpio_pu_pin>;
+		pinctrl-3 = <&P9_25_gpio_pd_pin>;
+		pinctrl-4 = <&P9_25_gpio_input_pin>;
+		pinctrl-5 = <&P9_25_qep_pin>;
+		pinctrl-6 = <&P9_25_pruout_pin>;
+		pinctrl-7 = <&P9_25_pruin_pin>;
+	};
+
+	/* P9_26 (ZCZ ball D16) */
+	P9_26_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "can", "i2c", "pru_uart", "pruin";
+		pinctrl-0 = <&P9_26_default_pin>;
+		pinctrl-1 = <&P9_26_gpio_pin>;
+		pinctrl-2 = <&P9_26_gpio_pu_pin>;
+		pinctrl-3 = <&P9_26_gpio_pd_pin>;
+		pinctrl-4 = <&P9_26_gpio_input_pin>;
+		pinctrl-5 = <&P9_26_uart_pin>;
+		pinctrl-6 = <&P9_26_can_pin>;
+		pinctrl-7 = <&P9_26_i2c_pin>;
+		pinctrl-8 = <&P9_26_pru_uart_pin>;
+		pinctrl-9 = <&P9_26_pruin_pin>;
+	};
+
+	/* P9_27 (ZCZ ball C13) */
+	P9_27_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin";
+		pinctrl-0 = <&P9_27_default_pin>;
+		pinctrl-1 = <&P9_27_gpio_pin>;
+		pinctrl-2 = <&P9_27_gpio_pu_pin>;
+		pinctrl-3 = <&P9_27_gpio_pd_pin>;
+		pinctrl-4 = <&P9_27_gpio_input_pin>;
+		pinctrl-5 = <&P9_27_qep_pin>;
+		pinctrl-6 = <&P9_27_pruout_pin>;
+		pinctrl-7 = <&P9_27_pruin_pin>;
+	};
+
+	/* P9_28 (ZCZ ball C12) audio */
+	P9_28_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "pwm", "pwm2", "pruout", "pruin";
+		pinctrl-0 = <&P9_28_default_pin>;
+		pinctrl-1 = <&P9_28_gpio_pin>;
+		pinctrl-2 = <&P9_28_gpio_pu_pin>;
+		pinctrl-3 = <&P9_28_gpio_pd_pin>;
+		pinctrl-4 = <&P9_28_gpio_input_pin>;
+		pinctrl-5 = <&P9_28_spi_cs_pin>;
+		pinctrl-6 = <&P9_28_pwm_pin>;
+		pinctrl-7 = <&P9_28_pwm2_pin>;
+		pinctrl-8 = <&P9_28_pruout_pin>;
+		pinctrl-9 = <&P9_28_pruin_pin>;
+	};
+
+	/* P9_29 (ZCZ ball B13) audio */
+	P9_29_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "pwm", "pruout", "pruin";
+		pinctrl-0 = <&P9_29_default_pin>;
+		pinctrl-1 = <&P9_29_gpio_pin>;
+		pinctrl-2 = <&P9_29_gpio_pu_pin>;
+		pinctrl-3 = <&P9_29_gpio_pd_pin>;
+		pinctrl-4 = <&P9_29_gpio_input_pin>;
+		pinctrl-5 = <&P9_29_spi_pin>;
+		pinctrl-6 = <&P9_29_pwm_pin>;
+		pinctrl-7 = <&P9_29_pruout_pin>;
+		pinctrl-8 = <&P9_29_pruin_pin>;
+	};
+
+	/* P9_30 (ZCZ ball D12) */
+	P9_30_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "pwm", "pruout", "pruin";
+		pinctrl-0 = <&P9_30_default_pin>;
+		pinctrl-1 = <&P9_30_gpio_pin>;
+		pinctrl-2 = <&P9_30_gpio_pu_pin>;
+		pinctrl-3 = <&P9_30_gpio_pd_pin>;
+		pinctrl-4 = <&P9_30_gpio_input_pin>;
+		pinctrl-5 = <&P9_30_spi_pin>;
+		pinctrl-6 = <&P9_30_pwm_pin>;
+		pinctrl-7 = <&P9_30_pruout_pin>;
+		pinctrl-8 = <&P9_30_pruin_pin>;
+	};
+
+	/* P9_31 (ZCZ ball A13) audio */
+	P9_31_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_sclk", "pwm", "pruout", "pruin";
+		pinctrl-0 = <&P9_31_default_pin>;
+		pinctrl-1 = <&P9_31_gpio_pin>;
+		pinctrl-2 = <&P9_31_gpio_pu_pin>;
+		pinctrl-3 = <&P9_31_gpio_pd_pin>;
+		pinctrl-4 = <&P9_31_gpio_input_pin>;
+		pinctrl-5 = <&P9_31_spi_sclk_pin>;
+		pinctrl-6 = <&P9_31_pwm_pin>;
+		pinctrl-7 = <&P9_31_pruout_pin>;
+		pinctrl-8 = <&P9_31_pruin_pin>;
+	};
+
+	/* P9_32                VADC */
+
+	/* P9_33 (ZCZ ball C8)  AIN4         */
+
+	/* P9_34                AGND */
+
+	/* P9_35 (ZCZ ball A8)  AIN6         */
+
+	/* P9_36 (ZCZ ball B8)  AIN5         */
+
+	/* P9_37 (ZCZ ball B7)  AIN2         */
+
+	/* P9_38 (ZCZ ball A7)  AIN3         */
+
+	/* P9_39 (ZCZ ball B6)  AIN0         */
+
+	/* P9_40 (ZCZ ball C7)  AIN1         */
+
+	/* P9_41 (ZCZ ball D14) */
+	P9_41_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer", "pruin";
+		pinctrl-0 = <&P9_41_default_pin>;
+		pinctrl-1 = <&P9_41_gpio_pin>;
+		pinctrl-2 = <&P9_41_gpio_pu_pin>;
+		pinctrl-3 = <&P9_41_gpio_pd_pin>;
+		pinctrl-4 = <&P9_41_gpio_input_pin>;
+		pinctrl-5 = <&P9_41_timer_pin>;
+		pinctrl-6 = <&P9_41_pruin_pin>;
+	};
+
+	/* P9_41.1 */
+	/* P9_91 (ZCZ ball D13) */
+	P9_91_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin";
+		pinctrl-0 = <&P9_91_default_pin>;
+		pinctrl-1 = <&P9_91_gpio_pin>;
+		pinctrl-2 = <&P9_91_gpio_pu_pin>;
+		pinctrl-3 = <&P9_91_gpio_pd_pin>;
+		pinctrl-4 = <&P9_91_gpio_input_pin>;
+		pinctrl-5 = <&P9_91_qep_pin>;
+		pinctrl-6 = <&P9_91_pruout_pin>;
+		pinctrl-7 = <&P9_91_pruin_pin>;
+	};
+
+	/* P9_42 (ZCZ ball C18) */
+	P9_42_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "spi_sclk", "uart", "pwm", "pru_ecap";
+		pinctrl-0 = <&P9_42_default_pin>;
+		pinctrl-1 = <&P9_42_gpio_pin>;
+		pinctrl-2 = <&P9_42_gpio_pu_pin>;
+		pinctrl-3 = <&P9_42_gpio_pd_pin>;
+		pinctrl-4 = <&P9_42_gpio_input_pin>;
+		pinctrl-5 = <&P9_42_spi_cs_pin>;
+		pinctrl-6 = <&P9_42_spi_sclk_pin>;
+		pinctrl-7 = <&P9_42_uart_pin>;
+		pinctrl-8 = <&P9_42_pwm_pin>;
+		pinctrl-9 = <&P9_42_pru_ecap_pin>;
+	};
+
+	/* P9_42.1 */
+	/* P9_92 (ZCZ ball B12) */
+	P9_92_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin";
+		pinctrl-0 = <&P9_92_default_pin>;
+		pinctrl-1 = <&P9_92_gpio_pin>;
+		pinctrl-2 = <&P9_92_gpio_pu_pin>;
+		pinctrl-3 = <&P9_92_gpio_pd_pin>;
+		pinctrl-4 = <&P9_92_gpio_input_pin>;
+		pinctrl-5 = <&P9_92_qep_pin>;
+		pinctrl-6 = <&P9_92_pruout_pin>;
+		pinctrl-7 = <&P9_92_pruin_pin>;
+	};
+
+	/* P9_43                GND */
+
+	/* P9_44                GND */
+
+	/* P9_45                GND */
+
+	/* P9_46                GND */
+
+	/*       (ZCZ ball A15) */
+	A15_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "clkout", "gpio", "gpio_pu", "gpio_pd";
+		pinctrl-0 = <&A15_default_pin>;
+		pinctrl-1 = <&A15_clkout_pin>;
+		pinctrl-2 = <&A15_gpio_pin>;
+		pinctrl-3 = <&A15_gpio_pu_pin>;
+		pinctrl-4 = <&A15_gpio_pd_pin>;
+	};
+
+	cape-universal {
+		compatible = "gpio-of-helper";
+		status = "okay";
+		pinctrl-names = "default";
+		pinctrl-0 = <>;
+
+		P8_03 {
+			gpio-name = "P8_03";
+			gpio = <&gpio1 6 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_04 {
+			gpio-name = "P8_04";
+			gpio = <&gpio1 7 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_05 {
+			gpio-name = "P8_05";
+			gpio = <&gpio1 2 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_06 {
+			gpio-name = "P8_06";
+			gpio = <&gpio1 3 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_07 {
+			gpio-name = "P8_07";
+			gpio = <&gpio2 2 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_08 {
+			gpio-name = "P8_08";
+			gpio = <&gpio2 3 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_09 {
+			gpio-name = "P8_09";
+			gpio = <&gpio2 5 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_10 {
+			gpio-name = "P8_10";
+			gpio = <&gpio2 4 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_11 {
+			gpio-name = "P8_11";
+			gpio = <&gpio1 13 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_12 {
+			gpio-name = "P8_12";
+			gpio = <&gpio1 12 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_13 {
+			gpio-name = "P8_13";
+			gpio = <&gpio0 23 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_14 {
+			gpio-name = "P8_14";
+			gpio = <&gpio0 26 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_15 {
+			gpio-name = "P8_15";
+			gpio = <&gpio1 15 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_16 {
+			gpio-name = "P8_16";
+			gpio = <&gpio1 14 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_17 {
+			gpio-name = "P8_17";
+			gpio = <&gpio0 27 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_18 {
+			gpio-name = "P8_18";
+			gpio = <&gpio2 1 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_19 {
+			gpio-name = "P8_19";
+			gpio = <&gpio0 22 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_20 {
+			gpio-name = "P8_20";
+			gpio = <&gpio1 31 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_21 {
+			gpio-name = "P8_21";
+			gpio = <&gpio1 30 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_22 {
+			gpio-name = "P8_22";
+			gpio = <&gpio1 5 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_23 {
+			gpio-name = "P8_23";
+			gpio = <&gpio1 4 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_24 {
+			gpio-name = "P8_24";
+			gpio = <&gpio1 1 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_25 {
+			gpio-name = "P8_25";
+			gpio = <&gpio1 0 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_26 {
+			gpio-name = "P8_26";
+			gpio = <&gpio1 29 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_27 {
+			gpio-name = "P8_27";
+			gpio = <&gpio2 22 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_28 {
+			gpio-name = "P8_28";
+			gpio = <&gpio2 24 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_29 {
+			gpio-name = "P8_29";
+			gpio = <&gpio2 23 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_30 {
+			gpio-name = "P8_30";
+			gpio = <&gpio2 25 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_31 {
+			gpio-name = "P8_31";
+			gpio = <&gpio0 10 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_32 {
+			gpio-name = "P8_32";
+			gpio = <&gpio0 11 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_33 {
+			gpio-name = "P8_33";
+			gpio = <&gpio0 9 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_34 {
+			gpio-name = "P8_34";
+			gpio = <&gpio2 17 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_35 {
+			gpio-name = "P8_35";
+			gpio = <&gpio0 8 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_36 {
+			gpio-name = "P8_36";
+			gpio = <&gpio2 16 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_37 {
+			gpio-name = "P8_37";
+			gpio = <&gpio2 14 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_38 {
+			gpio-name = "P8_38";
+			gpio = <&gpio2 15 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_39 {
+			gpio-name = "P8_39";
+			gpio = <&gpio2 12 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_40 {
+			gpio-name = "P8_40";
+			gpio = <&gpio2 13 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_41 {
+			gpio-name = "P8_41";
+			gpio = <&gpio2 10 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_42 {
+			gpio-name = "P8_42";
+			gpio = <&gpio2 11 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_43 {
+			gpio-name = "P8_43";
+			gpio = <&gpio2 8 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_44 {
+			gpio-name = "P8_44";
+			gpio = <&gpio2 9 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_45 {
+			gpio-name = "P8_45";
+			gpio = <&gpio2 6 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_46 {
+			gpio-name = "P8_46";
+			gpio = <&gpio2 7 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_11 {
+			gpio-name = "P9_11";
+			gpio = <&gpio0 30 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_12 {
+			gpio-name = "P9_12";
+			gpio = <&gpio1 28 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_13 {
+			gpio-name = "P9_13";
+			gpio = <&gpio0 31 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_14 {
+			gpio-name = "P9_14";
+			gpio = <&gpio1 18 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_15 {
+			gpio-name = "P9_15";
+			gpio = <&gpio1 16 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_16 {
+			gpio-name = "P9_16";
+			gpio = <&gpio1 19 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_17 {
+			gpio-name = "P9_17";
+			gpio = <&gpio0 5 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_18 {
+			gpio-name = "P9_18";
+			gpio = <&gpio0 4 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_19 {
+			gpio-name = "P9_19";
+			gpio = <&gpio0 13 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_20 {
+			gpio-name = "P9_20";
+			gpio = <&gpio0 12 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_21 {
+			gpio-name = "P9_21";
+			gpio = <&gpio0 3 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_22 {
+			gpio-name = "P9_22";
+			gpio = <&gpio0 2 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_23 {
+			gpio-name = "P9_23";
+			gpio = <&gpio1 17 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_24 {
+			gpio-name = "P9_24";
+			gpio = <&gpio0 15 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_25 {
+			gpio-name = "P9_25";
+			gpio = <&gpio3 21 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_26 {
+			gpio-name = "P9_26";
+			gpio = <&gpio0 14 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_27 {
+			gpio-name = "P9_27";
+			gpio = <&gpio3 19 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_28 {
+			gpio-name = "P9_28";
+			gpio = <&gpio3 17 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_29 {
+			gpio-name = "P9_29";
+			gpio = <&gpio3 15 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_30 {
+			gpio-name = "P9_30";
+			gpio = <&gpio3 16 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_31 {
+			gpio-name = "P9_31";
+			gpio = <&gpio3 14 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_41 {
+			gpio-name = "P9_41";
+			gpio = <&gpio0 20 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_91 {
+			gpio-name = "P9_91";
+			gpio = <&gpio3 20 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_42 {
+			gpio-name = "P9_42";
+			gpio = <&gpio0 7 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_92 {
+			gpio-name = "P9_92";
+			gpio = <&gpio3 18 0>;
+			input;
+			dir-changeable;
+		};
+
+		A15 {
+			gpio-name = "A15";
+			gpio = <&gpio0 19 0>;
+			input;
+			dir-changeable;
+		};
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-bone-common-universal-pins.dtsi b/arch/arm/boot/dts/am335x-bone-common-universal-pins.dtsi
--- a/arch/arm/boot/dts/am335x-bone-common-universal-pins.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-bone-common-universal-pins.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,967 @@
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+&am33xx_pinmux {
+	/************************/
+	/* P8 Header            */
+	/************************/
+
+	/* P8_01                GND     */
+	/* P8_02                GND     */
+	/* P8_03 (ZCZ ball R9 ) emmc    */
+	/* P8_04 (ZCZ ball T9 ) emmc    */
+	/* P8_05 (ZCZ ball R8 ) emmc    */
+	/* P8_06 (ZCZ ball T8 ) emmc    */
+
+	/* P8_07 (ZCZ ball R7 ) */
+	P8_07_default_pin: pinmux_P8_07_default_pin {
+		pinctrl-single,pins = <0x090  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_07_gpio_pin: pinmux_P8_07_gpio_pin {
+		pinctrl-single,pins = <0x090  0x2F>; };     /* Mode 7, RxActive */
+	P8_07_gpio_pu_pin: pinmux_P8_07_gpio_pu_pin {
+		pinctrl-single,pins = <0x090  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_07_gpio_pd_pin: pinmux_P8_07_gpio_pd_pin {
+		pinctrl-single,pins = <0x090  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_07_timer_pin: pinmux_P8_07_timer_pin {
+		pinctrl-single,pins = <0x090  0x32>; };     /* Mode 2, Pull-Up, RxActive */
+
+	/* P8_08 (ZCZ ball T7 ) */
+	P8_08_default_pin: pinmux_P8_08_default_pin {
+		pinctrl-single,pins = <0x094  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_08_gpio_pin: pinmux_P8_08_gpio_pin {
+		pinctrl-single,pins = <0x094  0x2F>; };     /* Mode 7, RxActive */
+	P8_08_gpio_pu_pin: pinmux_P8_08_gpio_pu_pin {
+		pinctrl-single,pins = <0x094  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_08_gpio_pd_pin: pinmux_P8_08_gpio_pd_pin {
+		pinctrl-single,pins = <0x094  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_08_timer_pin: pinmux_P8_08_timer_pin {
+		pinctrl-single,pins = <0x094  0x32>; };     /* Mode 2, Pull-Up, RxActive */
+
+	/* P8_09 (ZCZ ball T6 ) */
+	P8_09_default_pin: pinmux_P8_09_default_pin {
+		pinctrl-single,pins = <0x09c  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_09_gpio_pin: pinmux_P8_09_gpio_pin {
+		pinctrl-single,pins = <0x09c  0x2F>; };     /* Mode 7, RxActive */
+	P8_09_gpio_pu_pin: pinmux_P8_09_gpio_pu_pin {
+		pinctrl-single,pins = <0x09c  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_09_gpio_pd_pin: pinmux_P8_09_gpio_pd_pin {
+		pinctrl-single,pins = <0x09c  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_09_timer_pin: pinmux_P8_09_timer_pin {
+		pinctrl-single,pins = <0x09c  0x32>; };     /* Mode 2, Pull-Up, RxActive */
+
+	/* P8_10 (ZCZ ball U6 ) */
+	P8_10_default_pin: pinmux_P8_10_default_pin {
+		pinctrl-single,pins = <0x098  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_10_gpio_pin: pinmux_P8_10_gpio_pin {
+		pinctrl-single,pins = <0x098  0x2F>; };     /* Mode 7, RxActive */
+	P8_10_gpio_pu_pin: pinmux_P8_10_gpio_pu_pin {
+		pinctrl-single,pins = <0x098  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_10_gpio_pd_pin: pinmux_P8_10_gpio_pd_pin {
+		pinctrl-single,pins = <0x098  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_10_timer_pin: pinmux_P8_10_timer_pin {
+		pinctrl-single,pins = <0x098  0x32>; };     /* Mode 2, Pull-Up, RxActive */
+
+	/* P8_11 (ZCZ ball R12) */
+	P8_11_default_pin: pinmux_P8_11_default_pin {
+		pinctrl-single,pins = <0x034  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_11_gpio_pin: pinmux_P8_11_gpio_pin {
+		pinctrl-single,pins = <0x034  0x2F>; };     /* Mode 7, RxActive */
+	P8_11_gpio_pu_pin: pinmux_P8_11_gpio_pu_pin {
+		pinctrl-single,pins = <0x034  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_11_gpio_pd_pin: pinmux_P8_11_gpio_pd_pin {
+		pinctrl-single,pins = <0x034  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_11_pruout_pin: pinmux_P8_11_pruout_pin {
+		pinctrl-single,pins = <0x034  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+	P8_11_qep_pin: pinmux_P8_11_qep_pin {
+		pinctrl-single,pins = <0x034  0x24>; };     /* Mode 4, Pull-Down, RxActive */
+
+	/* P8_12 (ZCZ ball T12) */
+	P8_12_default_pin: pinmux_P8_12_default_pin {
+		pinctrl-single,pins = <0x030  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_12_gpio_pin: pinmux_P8_12_gpio_pin {
+		pinctrl-single,pins = <0x030  0x2F>; };     /* Mode 7, RxActive */
+	P8_12_gpio_pu_pin: pinmux_P8_12_gpio_pu_pin {
+		pinctrl-single,pins = <0x030  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_12_gpio_pd_pin: pinmux_P8_12_gpio_pd_pin {
+		pinctrl-single,pins = <0x030  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_12_pruout_pin: pinmux_P8_12_pruout_pin {
+		pinctrl-single,pins = <0x030  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+	P8_12_qep_pin: pinmux_P8_12_qep_pin {
+		pinctrl-single,pins = <0x030  0x24>; };     /* Mode 4, Pull-Down, RxActive */
+
+	/* P8_13 (ZCZ ball T10) */
+	P8_13_default_pin: pinmux_P8_13_default_pin {
+		pinctrl-single,pins = <0x024  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_13_gpio_pin: pinmux_P8_13_gpio_pin {
+		pinctrl-single,pins = <0x024  0x2F>; };     /* Mode 7, RxActive */
+	P8_13_gpio_pu_pin: pinmux_P8_13_gpio_pu_pin {
+		pinctrl-single,pins = <0x024  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_13_gpio_pd_pin: pinmux_P8_13_gpio_pd_pin {
+		pinctrl-single,pins = <0x024  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_13_pwm_pin: pinmux_P8_13_pwm_pin {
+		pinctrl-single,pins = <0x024  0x24>; };     /* Mode 4, Pull-Down, RxActive */
+
+	/* P8_14 (ZCZ ball T11) */
+	P8_14_default_pin: pinmux_P8_14_default_pin {
+		pinctrl-single,pins = <0x028  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_14_gpio_pin: pinmux_P8_14_gpio_pin {
+		pinctrl-single,pins = <0x028  0x2F>; };     /* Mode 7, RxActive */
+	P8_14_gpio_pu_pin: pinmux_P8_14_gpio_pu_pin {
+		pinctrl-single,pins = <0x028  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_14_gpio_pd_pin: pinmux_P8_14_gpio_pd_pin {
+		pinctrl-single,pins = <0x028  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_14_pwm_pin: pinmux_P8_14_pwm_pin {
+		pinctrl-single,pins = <0x028  0x24>; };     /* Mode 4, Pull-Down, RxActive */
+
+	/* P8_15 (ZCZ ball U13) */
+	P8_15_default_pin: pinmux_P8_15_default_pin {
+		pinctrl-single,pins = <0x03c  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_15_gpio_pin: pinmux_P8_15_gpio_pin {
+		pinctrl-single,pins = <0x03c  0x2F>; };     /* Mode 7, RxActive */
+	P8_15_gpio_pu_pin: pinmux_P8_15_gpio_pu_pin {
+		pinctrl-single,pins = <0x03c  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_15_gpio_pd_pin: pinmux_P8_15_gpio_pd_pin {
+		pinctrl-single,pins = <0x03c  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_15_pruin_pin: pinmux_P8_15_pruin_pin {
+		pinctrl-single,pins = <0x03c  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+	P8_15_qep_pin: pinmux_P8_15_qep_pin {
+		pinctrl-single,pins = <0x03c  0x24>; };     /* Mode 4, Pull-Down, RxActive */
+	P8_15_pruin_pu_pin: pinmux_P8_15_pruin_pu_pin {
+		pinctrl-single,pins = <0x03c  0x36>; };     /* Mode 6, Pull-Up, RxActive */
+	P8_15_pruecapin_pu_pin: pinmux_P8_15_pruecapin_pu_pin {
+		pinctrl-single,pins = <0x03c  0x35>; };     /* Mode 5, Pull-Up, RxActive */
+
+	/* P8_16 (ZCZ ball V13) */
+	P8_16_default_pin: pinmux_P8_16_default_pin {
+		pinctrl-single,pins = <0x038  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_16_gpio_pin: pinmux_P8_16_gpio_pin {
+		pinctrl-single,pins = <0x038  0x2F>; };     /* Mode 7, RxActive */
+	P8_16_gpio_pu_pin: pinmux_P8_16_gpio_pu_pin {
+		pinctrl-single,pins = <0x038  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_16_gpio_pd_pin: pinmux_P8_16_gpio_pd_pin {
+		pinctrl-single,pins = <0x038  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_16_pruin_pin: pinmux_P8_16_pruin_pin {
+		pinctrl-single,pins = <0x038  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+	P8_16_qep_pin: pinmux_P8_16_qep_pin {
+		pinctrl-single,pins = <0x038  0x24>; };     /* Mode 4, Pull-Down, RxActive */
+
+	/* P8_17 (ZCZ ball U12) */
+	P8_17_default_pin: pinmux_P8_17_default_pin {
+		pinctrl-single,pins = <0x02c  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_17_gpio_pin: pinmux_P8_17_gpio_pin {
+		pinctrl-single,pins = <0x02c  0x2F>; };     /* Mode 7, RxActive */
+	P8_17_gpio_pu_pin: pinmux_P8_17_gpio_pu_pin {
+		pinctrl-single,pins = <0x02c  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_17_gpio_pd_pin: pinmux_P8_17_gpio_pd_pin {
+		pinctrl-single,pins = <0x02c  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_17_pwm_pin: pinmux_P8_17_pwm_pin {
+		pinctrl-single,pins = <0x02c  0x24>; };     /* Mode 4, Pull-Down, RxActive */
+
+	/* P8_18 (ZCZ ball V12) */
+	P8_18_default_pin: pinmux_P8_18_default_pin {
+		pinctrl-single,pins = <0x08c  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_18_gpio_pin: pinmux_P8_18_gpio_pin {
+		pinctrl-single,pins = <0x08c  0x2F>; };     /* Mode 7, RxActive */
+	P8_18_gpio_pu_pin: pinmux_P8_18_gpio_pu_pin {
+		pinctrl-single,pins = <0x08c  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_18_gpio_pd_pin: pinmux_P8_18_gpio_pd_pin {
+		pinctrl-single,pins = <0x08c  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+
+	/* P8_19 (ZCZ ball U10) */
+	P8_19_default_pin: pinmux_P8_19_default_pin {
+		pinctrl-single,pins = <0x020  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_19_gpio_pin: pinmux_P8_19_gpio_pin {
+		pinctrl-single,pins = <0x020  0x2F>; };     /* Mode 7, RxActive */
+	P8_19_gpio_pu_pin: pinmux_P8_19_gpio_pu_pin {
+		pinctrl-single,pins = <0x020  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_19_gpio_pd_pin: pinmux_P8_19_gpio_pd_pin {
+		pinctrl-single,pins = <0x020  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_19_pwm_pin: pinmux_P8_19_pwm_pin {
+		pinctrl-single,pins = <0x020  0x24>; };     /* Mode 4, Pull-Down, RxActive */
+
+	/* P8_20 (ZCZ ball V9 ) emmc    */
+	/* P8_21 (ZCZ ball U9 ) emmc    */
+	/* P8_22 (ZCZ ball V8 ) emmc    */
+	/* P8_23 (ZCZ ball U8 ) emmc    */
+	/* P8_24 (ZCZ ball V7 ) emmc    */
+	/* P8_25 (ZCZ ball U7 ) emmc    */
+
+	/* P8_26 (ZCZ ball V6 ) */
+	P8_26_default_pin: pinmux_P8_26_default_pin {
+		pinctrl-single,pins = <0x07c  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_26_gpio_pin: pinmux_P8_26_gpio_pin {
+		pinctrl-single,pins = <0x07c  0x2F>; };     /* Mode 7, RxActive */
+	P8_26_gpio_pu_pin: pinmux_P8_26_gpio_pu_pin {
+		pinctrl-single,pins = <0x07c  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_26_gpio_pd_pin: pinmux_P8_26_gpio_pd_pin {
+		pinctrl-single,pins = <0x07c  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+
+	/* P8_27 (ZCZ ball U5 ) hdmi    */
+	P8_27_default_pin: pinmux_P8_27_default_pin {
+		pinctrl-single,pins = <0x0e0  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_27_gpio_pin: pinmux_P8_27_gpio_pin {
+		pinctrl-single,pins = <0x0e0  0x2F>; };     /* Mode 7, RxActive */
+	P8_27_gpio_pu_pin: pinmux_P8_27_gpio_pu_pin {
+		pinctrl-single,pins = <0x0e0  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_27_gpio_pd_pin: pinmux_P8_27_gpio_pd_pin {
+		pinctrl-single,pins = <0x0e0  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_27_pruout_pin: pinmux_P8_27_pruout_pin {
+		pinctrl-single,pins = <0x0e0  0x05>; };     /* Mode 5, Pull-Down*/
+	P8_27_pruin_pin: pinmux_P8_27_pruin_pin {
+		pinctrl-single,pins = <0x0e0  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+	P8_27_hdmi_pin: pinmux_P8_27_hdmi_pin {
+		pinctrl-single,pins = <0x0e0  0x00>; };     /* lcd_vsync.lcd_vsync, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT */
+
+	/* P8_28 (ZCZ ball V5 ) hdmi    */
+	P8_28_default_pin: pinmux_P8_28_default_pin {
+		pinctrl-single,pins = <0x0e8  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_28_gpio_pin: pinmux_P8_28_gpio_pin {
+		pinctrl-single,pins = <0x0e8  0x2F>; };     /* Mode 7, RxActive */
+	P8_28_gpio_pu_pin: pinmux_P8_28_gpio_pu_pin {
+		pinctrl-single,pins = <0x0e8  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_28_gpio_pd_pin: pinmux_P8_28_gpio_pd_pin {
+		pinctrl-single,pins = <0x0e8  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_28_pruout_pin: pinmux_P8_28_pruout_pin {
+		pinctrl-single,pins = <0x0e8  0x05>; };     /* Mode 5, Pull-Down */
+	P8_28_pruin_pin: pinmux_P8_28_pruin_pin {
+		pinctrl-single,pins = <0x0e8  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+	P8_28_hdmi_pin: pinmux_P8_28_hdmi_pin {
+		pinctrl-single,pins = <0x0e8  0x00>; };     /* lcd_pclk.lcd_pclk, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT */
+
+	/* P8_29 (ZCZ ball R5 ) hdmi    */
+	P8_29_default_pin: pinmux_P8_29_default_pin {
+		pinctrl-single,pins = <0x0e4  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_29_gpio_pin: pinmux_P8_29_gpio_pin {
+		pinctrl-single,pins = <0x0e4  0x2F>; };     /* Mode 7, RxActive */
+	P8_29_gpio_pu_pin: pinmux_P8_29_gpio_pu_pin {
+		pinctrl-single,pins = <0x0e4  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_29_gpio_pd_pin: pinmux_P8_29_gpio_pd_pin {
+		pinctrl-single,pins = <0x0e4  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_29_pruout_pin: pinmux_P8_29_pruout_pin {
+		pinctrl-single,pins = <0x0e4  0x05>; };     /* Mode 5, Pull-Down*/
+	P8_29_pruin_pin: pinmux_P8_29_pruin_pin {
+		pinctrl-single,pins = <0x0e4  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+	P8_29_hdmi_pin: pinmux_P8_29_hdmi_pin {
+		pinctrl-single,pins = <0x0e4  0x00>; };     /* lcd_hsync.lcd_hsync, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT */
+
+	/* P8_30 (ZCZ ball R6 ) hdmi    */
+	P8_30_default_pin: pinmux_P8_30_default_pin {
+		pinctrl-single,pins = <0x0ec  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_30_gpio_pin: pinmux_P8_30_gpio_pin {
+		pinctrl-single,pins = <0x0ec  0x2F>; };     /* Mode 7, RxActive */
+	P8_30_gpio_pu_pin: pinmux_P8_30_gpio_pu_pin {
+		pinctrl-single,pins = <0x0ec  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_30_gpio_pd_pin: pinmux_P8_30_gpio_pd_pin {
+		pinctrl-single,pins = <0x0ec  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_30_pruout_pin: pinmux_P8_30_pruout_pin {
+		pinctrl-single,pins = <0x0ec  0x05>; };     /* Mode 5, Pull-Down*/
+	P8_30_pruin_pin: pinmux_P8_30_pruin_pin {
+		pinctrl-single,pins = <0x0ec  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+	P8_30_hdmi_pin: pinmux_P8_30_hdmi_pin {
+		pinctrl-single,pins = <0x0ec  0x00>; };     /* lcd_ac_bias_en.lcd_ac_bias_en, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT */
+
+	/* P8_31 (ZCZ ball V4 ) hdmi    */
+	P8_31_default_pin: pinmux_P8_31_default_pin {
+		pinctrl-single,pins = <0x0d8  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_31_gpio_pin: pinmux_P8_31_gpio_pin {
+		pinctrl-single,pins = <0x0d8  0x2F>; };     /* Mode 7, RxActive */
+	P8_31_gpio_pu_pin: pinmux_P8_31_gpio_pu_pin {
+		pinctrl-single,pins = <0x0d8  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_31_gpio_pd_pin: pinmux_P8_31_gpio_pd_pin {
+		pinctrl-single,pins = <0x0d8  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_31_uart_pin: pinmux_P8_31_uart_pin {
+		pinctrl-single,pins = <0x0d8  0x24>; };     /* Mode 4, Pull-Down, RxActive */
+	P8_31_hdmi_pin: pinmux_P8_31_hdmi_pin {
+		pinctrl-single,pins = <0x0d8  0x08>; };     /* lcd_data14.lcd_data14, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */
+
+	/* P8_32 (ZCZ ball T5 ) hdmi    */
+	P8_32_default_pin: pinmux_P8_32_default_pin {
+		pinctrl-single,pins = <0x0dc  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_32_gpio_pin: pinmux_P8_32_gpio_pin {
+		pinctrl-single,pins = <0x0dc  0x2F>; };     /* Mode 7, RxActive */
+	P8_32_gpio_pu_pin: pinmux_P8_32_gpio_pu_pin {
+		pinctrl-single,pins = <0x0dc  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_32_gpio_pd_pin: pinmux_P8_32_gpio_pd_pin {
+		pinctrl-single,pins = <0x0dc  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_32_uart_pin: pinmux_P8_32_uart_pin {
+		pinctrl-single,pins = <0x0dc  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+	P8_32_hdmi_pin: pinmux_P8_32_hdmi_pin {
+		pinctrl-single,pins = <0x0dc  0x08>; };     /* lcd_data15.lcd_data15, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */
+
+	/* P8_33 (ZCZ ball V3 ) hdmi    */
+	P8_33_default_pin: pinmux_P8_33_default_pin {
+		pinctrl-single,pins = <0x0d4  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_33_gpio_pin: pinmux_P8_33_gpio_pin {
+		pinctrl-single,pins = <0x0d4  0x2F>; };     /* Mode 7, RxActive */
+	P8_33_gpio_pu_pin: pinmux_P8_33_gpio_pu_pin {
+		pinctrl-single,pins = <0x0d4  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_33_gpio_pd_pin: pinmux_P8_33_gpio_pd_pin {
+		pinctrl-single,pins = <0x0d4  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_33_hdmi_pin: pinmux_P8_33_hdmi_pin {
+		pinctrl-single,pins = <0x0d4  0x08>; };     /* lcd_data13.lcd_data13, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */
+	P8_33_qep_pin: pinmux_P8_33_qep_pin {
+		pinctrl-single,pins = <0x0d4  0x22>; };     /* Mode 2, Pull-Down, RxActive */
+
+	/* P8_34 (ZCZ ball U4 ) hdmi    */
+	P8_34_default_pin: pinmux_P8_34_default_pin {
+		pinctrl-single,pins = <0x0cc  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_34_gpio_pin: pinmux_P8_34_gpio_pin {
+		pinctrl-single,pins = <0x0cc  0x2F>; };     /* Mode 7, RxActive */
+	P8_34_gpio_pu_pin: pinmux_P8_34_gpio_pu_pin {
+		pinctrl-single,pins = <0x0cc  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_34_gpio_pd_pin: pinmux_P8_34_gpio_pd_pin {
+		pinctrl-single,pins = <0x0cc  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_34_pwm_pin: pinmux_P8_34_pwm_pin {
+		pinctrl-single,pins = <0x0cc  0x22>; };     /* Mode 2, Pull-Down, RxActive */
+	P8_34_hdmi_pin: pinmux_P8_34_hdmi_pin {
+		pinctrl-single,pins = <0x0cc  0x08>; };     /* lcd_data11.lcd_data11, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */
+
+	/* P8_35 (ZCZ ball V2 ) hdmi    */
+	P8_35_default_pin: pinmux_P8_35_default_pin {
+		pinctrl-single,pins = <0x0d0  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_35_gpio_pin: pinmux_P8_35_gpio_pin {
+		pinctrl-single,pins = <0x0d0  0x2F>; };     /* Mode 7, RxActive */
+	P8_35_gpio_pu_pin: pinmux_P8_35_gpio_pu_pin {
+		pinctrl-single,pins = <0x0d0  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_35_gpio_pd_pin: pinmux_P8_35_gpio_pd_pin {
+		pinctrl-single,pins = <0x0d0  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_35_hdmi_pin: pinmux_P8_35_hdmi_pin {
+		pinctrl-single,pins = <0x0d0  0x08>; };     /* lcd_data12.lcd_data12, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */
+	P8_35_qep_pin: pinmux_P8_35_qep_pin {
+		pinctrl-single,pins = <0x0d0  0x22>; };     /* Mode 2, Pull-Down, RxActive */
+
+	/* P8_36 (ZCZ ball U3 ) hdmi    */
+	P8_36_default_pin: pinmux_P8_36_default_pin {
+		pinctrl-single,pins = <0x0c8  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_36_gpio_pin: pinmux_P8_36_gpio_pin {
+		pinctrl-single,pins = <0x0c8  0x2F>; };     /* Mode 7, RxActive */
+	P8_36_gpio_pu_pin: pinmux_P8_36_gpio_pu_pin {
+		pinctrl-single,pins = <0x0c8  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_36_gpio_pd_pin: pinmux_P8_36_gpio_pd_pin {
+		pinctrl-single,pins = <0x0c8  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_36_pwm_pin: pinmux_P8_36_pwm_pin {
+		pinctrl-single,pins = <0x0c8  0x22>; };     /* Mode 2, Pull-Down, RxActive */
+	P8_36_hdmi_pin: pinmux_P8_36_hdmi_pin {
+		pinctrl-single,pins = <0x0c8  0x08>; };     /* lcd_data10.lcd_data10, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */
+
+	/* P8_37 (ZCZ ball U1 ) hdmi    */
+	P8_37_default_pin: pinmux_P8_37_default_pin {
+		pinctrl-single,pins = <0x0c0  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_37_gpio_pin: pinmux_P8_37_gpio_pin {
+		pinctrl-single,pins = <0x0c0  0x2F>; };     /* Mode 7, RxActive */
+	P8_37_gpio_pu_pin: pinmux_P8_37_gpio_pu_pin {
+		pinctrl-single,pins = <0x0c0  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_37_gpio_pd_pin: pinmux_P8_37_gpio_pd_pin {
+		pinctrl-single,pins = <0x0c0  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_37_uart_pin: pinmux_P8_37_uart_pin {
+		pinctrl-single,pins = <0x0c0  0x04>; };     /* Mode 4, Pull-Down*/
+	P8_37_pwm_pin: pinmux_P8_37_pwm_pin {
+		pinctrl-single,pins = <0x0c0  0x02>; };     /* Mode 2, Pull-Down*/
+	P8_37_hdmi_pin: pinmux_P8_37_hdmi_pin {
+		pinctrl-single,pins = <0x0c0  0x08>; };     /* lcd_data8.lcd_data8, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */
+
+
+	/* P8_38 (ZCZ ball U2 ) hdmi    */
+	P8_38_default_pin: pinmux_P8_38_default_pin {
+		pinctrl-single,pins = <0x0c4  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_38_gpio_pin: pinmux_P8_38_gpio_pin {
+		pinctrl-single,pins = <0x0c4  0x2F>; };     /* Mode 7, RxActive */
+	P8_38_gpio_pu_pin: pinmux_P8_38_gpio_pu_pin {
+		pinctrl-single,pins = <0x0c4  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_38_gpio_pd_pin: pinmux_P8_38_gpio_pd_pin {
+		pinctrl-single,pins = <0x0c4  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_38_uart_pin: pinmux_P8_38_uart_pin {
+		pinctrl-single,pins = <0x0c4  0x24>; };     /* Mode 4, Pull-Down, RxActive */
+	P8_38_pwm_pin: pinmux_P8_38_pwm_pin {
+		pinctrl-single,pins = <0x0c4  0x22>; };     /* Mode 2, Pull-Down, RxActive */
+	P8_38_hdmi_pin: pinmux_P8_38_hdmi_pin {
+		pinctrl-single,pins = <0x0c4  0x08>; };     /* lcd_data9.lcd_data9, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */
+
+
+	/* P8_39 (ZCZ ball T3 ) hdmi    */
+	P8_39_default_pin: pinmux_P8_39_default_pin {
+		pinctrl-single,pins = <0x0b8  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_39_gpio_pin: pinmux_P8_39_gpio_pin {
+		pinctrl-single,pins = <0x0b8  0x2F>; };     /* Mode 7, RxActive */
+	P8_39_gpio_pu_pin: pinmux_P8_39_gpio_pu_pin {
+		pinctrl-single,pins = <0x0b8  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_39_gpio_pd_pin: pinmux_P8_39_gpio_pd_pin {
+		pinctrl-single,pins = <0x0b8  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_39_pruout_pin: pinmux_P8_39_pruout_pin {
+		pinctrl-single,pins = <0x0b8  0x05>; };     /* Mode 5, Pull-Down*/
+	P8_39_pruin_pin: pinmux_P8_39_pruin_pin {
+		pinctrl-single,pins = <0x0b8  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+	P8_39_hdmi_pin: pinmux_P8_39_hdmi_pin {
+		pinctrl-single,pins = <0x0b8  0x08>; };     /* lcd_data6.lcd_data6, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */
+
+	/* P8_40 (ZCZ ball T4 ) hdmi    */
+	P8_40_default_pin: pinmux_P8_40_default_pin {
+		pinctrl-single,pins = <0x0bc  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_40_gpio_pin: pinmux_P8_40_gpio_pin {
+		pinctrl-single,pins = <0x0bc  0x2F>; };     /* Mode 7, RxActive */
+	P8_40_gpio_pu_pin: pinmux_P8_40_gpio_pu_pin {
+		pinctrl-single,pins = <0x0bc  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_40_gpio_pd_pin: pinmux_P8_40_gpio_pd_pin {
+		pinctrl-single,pins = <0x0bc  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_40_pruout_pin: pinmux_P8_40_pruout_pin {
+		pinctrl-single,pins = <0x0bc  0x05>; };     /* Mode 5, Pull-Down*/
+	P8_40_pruin_pin: pinmux_P8_40_pruin_pin {
+		pinctrl-single,pins = <0x0bc  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+	P8_40_hdmi_pin: pinmux_P8_40_hdmi_pin {
+		pinctrl-single,pins = <0x0bc  0x08>; };     /* lcd_data7.lcd_data7, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */
+
+	/* P8_41 (ZCZ ball T1 ) hdmi    */
+	P8_41_default_pin: pinmux_P8_41_default_pin {
+		pinctrl-single,pins = <0x0b0  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_41_gpio_pin: pinmux_P8_41_gpio_pin {
+		pinctrl-single,pins = <0x0b0  0x2F>; };     /* Mode 7, RxActive */
+	P8_41_gpio_pu_pin: pinmux_P8_41_gpio_pu_pin {
+		pinctrl-single,pins = <0x0b0  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_41_gpio_pd_pin: pinmux_P8_41_gpio_pd_pin {
+		pinctrl-single,pins = <0x0b0  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_41_pruout_pin: pinmux_P8_41_pruout_pin {
+		pinctrl-single,pins = <0x0b0  0x05>; };     /* Mode 5, Pull-Down*/
+	P8_41_pruin_pin: pinmux_P8_41_pruin_pin {
+		pinctrl-single,pins = <0x0b0  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+	P8_41_hdmi_pin: pinmux_P8_41_hdmi_pin {
+		pinctrl-single,pins = <0x0b0  0x08>; };     /* lcd_data4.lcd_data4, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */
+
+	/* P8_42 (ZCZ ball T2 ) hdmi    */
+	P8_42_default_pin: pinmux_P8_42_default_pin {
+		pinctrl-single,pins = <0x0b4  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_42_gpio_pin: pinmux_P8_42_gpio_pin {
+		pinctrl-single,pins = <0x0b4  0x2F>; };     /* Mode 7, RxActive */
+	P8_42_gpio_pu_pin: pinmux_P8_42_gpio_pu_pin {
+		pinctrl-single,pins = <0x0b4  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_42_gpio_pd_pin: pinmux_P8_42_gpio_pd_pin {
+		pinctrl-single,pins = <0x0b4  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_42_pruout_pin: pinmux_P8_42_pruout_pin {
+		pinctrl-single,pins = <0x0b4  0x05>; };     /* Mode 5, Pull-Down*/
+	P8_42_pruin_pin: pinmux_P8_42_pruin_pin {
+		pinctrl-single,pins = <0x0b4  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+	P8_42_hdmi_pin: pinmux_P8_42_hdmi_pin {
+		pinctrl-single,pins = <0x0b4  0x08>; };     /* lcd_data5.lcd_data5, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */
+
+	/* P8_43 (ZCZ ball R3 ) hdmi    */
+	P8_43_default_pin: pinmux_P8_43_default_pin {
+		pinctrl-single,pins = <0x0a8  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_43_gpio_pin: pinmux_P8_43_gpio_pin {
+		pinctrl-single,pins = <0x0a8  0x2F>; };     /* Mode 7, RxActive */
+	P8_43_gpio_pu_pin: pinmux_P8_43_gpio_pu_pin {
+		pinctrl-single,pins = <0x0a8  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_43_gpio_pd_pin: pinmux_P8_43_gpio_pd_pin {
+		pinctrl-single,pins = <0x0a8  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_43_pruout_pin: pinmux_P8_43_pruout_pin {
+		pinctrl-single,pins = <0x0a8  0x05>; };     /* Mode 5, Pull-Down*/
+	P8_43_pruin_pin: pinmux_P8_43_pruin_pin {
+		pinctrl-single,pins = <0x0a8  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+	P8_43_pwm_pin: pinmux_P8_43_pwm_pin {
+		pinctrl-single,pins = <0x0a8  0x03>; };     /* Mode 3, Pull-Down  */
+	P8_43_hdmi_pin: pinmux_P8_43_hdmi_pin {
+		pinctrl-single,pins = <0x0a8  0x08>; };     /* lcd_data2.lcd_data2, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */
+
+	/* P8_44 (ZCZ ball R4 ) hdmi    */
+	P8_44_default_pin: pinmux_P8_44_default_pin {
+		pinctrl-single,pins = <0x0ac  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_44_gpio_pin: pinmux_P8_44_gpio_pin {
+		pinctrl-single,pins = <0x0ac  0x2F>; };     /* Mode 7, RxActive */
+	P8_44_gpio_pu_pin: pinmux_P8_44_gpio_pu_pin {
+		pinctrl-single,pins = <0x0ac  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_44_gpio_pd_pin: pinmux_P8_44_gpio_pd_pin {
+		pinctrl-single,pins = <0x0ac  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_44_pruout_pin: pinmux_P8_44_pruout_pin {
+		pinctrl-single,pins = <0x0ac  0x05>; };     /* Mode 5, Pull-Down*/
+	P8_44_pruin_pin: pinmux_P8_44_pruin_pin {
+		pinctrl-single,pins = <0x0ac  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+	P8_44_pwm_pin: pinmux_P8_44_pwm_pin {
+		pinctrl-single,pins = <0x0ac  0x23>; };     /* Mode 3, Pull-Down, RxActive */
+	P8_44_hdmi_pin: pinmux_P8_44_hdmi_pin {
+		pinctrl-single,pins = <0x0ac  0x08>; };     /* lcd_data3.lcd_data3, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */
+
+	/* P8_45 (ZCZ ball R1 ) hdmi    */
+	P8_45_default_pin: pinmux_P8_45_default_pin {
+		pinctrl-single,pins = <0x0a0  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_45_gpio_pin: pinmux_P8_45_gpio_pin {
+		pinctrl-single,pins = <0x0a0  0x2F>; };     /* Mode 7, RxActive */
+	P8_45_gpio_pu_pin: pinmux_P8_45_gpio_pu_pin {
+		pinctrl-single,pins = <0x0a0  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_45_gpio_pd_pin: pinmux_P8_45_gpio_pd_pin {
+		pinctrl-single,pins = <0x0a0  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_45_pruout_pin: pinmux_P8_45_pruout_pin {
+		pinctrl-single,pins = <0x0a0  0x05>; };     /* Mode 5, Pull-Down*/
+	P8_45_pruin_pin: pinmux_P8_45_pruin_pin {
+		pinctrl-single,pins = <0x0a0  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+	P8_45_pwm_pin: pinmux_P8_45_pwm_pin {
+		pinctrl-single,pins = <0x0a0  0x03>; };     /* Mode 3, Pull-Down*/
+	P8_45_hdmi_pin: pinmux_P8_45_hdmi_pin {
+		pinctrl-single,pins = <0x0a0  0x08>; };     /* lcd_data0.lcd_data0, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */
+
+	/* P8_46 (ZCZ ball R2 ) hdmi    */
+	P8_46_default_pin: pinmux_P8_46_default_pin {
+		pinctrl-single,pins = <0x0a4  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_46_gpio_pin: pinmux_P8_46_gpio_pin {
+		pinctrl-single,pins = <0x0a4  0x2F>; };     /* Mode 7, RxActive */
+	P8_46_gpio_pu_pin: pinmux_P8_46_gpio_pu_pin {
+		pinctrl-single,pins = <0x0a4  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P8_46_gpio_pd_pin: pinmux_P8_46_gpio_pd_pin {
+		pinctrl-single,pins = <0x0a4  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P8_46_pruout_pin: pinmux_P8_46_pruout_pin {
+		pinctrl-single,pins = <0x0a4  0x05>; };     /* Mode 5, Pull-Down*/
+	P8_46_pruin_pin: pinmux_P8_46_pruin_pin {
+		pinctrl-single,pins = <0x0a4  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+	P8_46_pwm_pin: pinmux_P8_46_pwm_pin {
+		pinctrl-single,pins = <0x0a4  0x03>; };     /* Mode 3, Pull-Down*/
+	P8_46_hdmi_pin: pinmux_P8_46_hdmi_pin {
+		pinctrl-single,pins = <0x0a4  0x08>; };     /* lcd_data1.lcd_data1, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */
+
+	/************************/
+	/* P9 Header            */
+	/************************/
+
+	/* P9_01                GND     */
+	/* P9_02                GND     */
+	/* P9_03                3.3V    */
+	/* P9_04                3.3V    */
+	/* P9_05                VDD_5V  */
+	/* P9_06                VDD_5V  */
+	/* P9_07                SYS_5V  */
+	/* P9_08                SYS_5V  */
+	/* P9_09                PWR_BUT */
+	/* P9_10 (ZCZ ball A10) RESETn  */
+
+	/* P9_11 (ZCZ ball T17) */
+	P9_11_default_pin: pinmux_P9_11_default_pin {
+		pinctrl-single,pins = <0x070  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_11_gpio_pin: pinmux_P9_11_gpio_pin {
+		pinctrl-single,pins = <0x070  0x2F>; };     /* Mode 7, RxActive */
+	P9_11_gpio_pu_pin: pinmux_P9_11_gpio_pu_pin {
+		pinctrl-single,pins = <0x070  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_11_gpio_pd_pin: pinmux_P9_11_gpio_pd_pin {
+		pinctrl-single,pins = <0x070  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_11_uart_pin: pinmux_P9_11_uart_pin {
+		pinctrl-single,pins = <0x070  0x36>; };     /* Mode 6, Pull-Up, RxActive */
+
+	/* P9_12 (ZCZ ball U18) */
+	P9_12_default_pin: pinmux_P9_12_default_pin {
+		pinctrl-single,pins = <0x078  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_12_gpio_pin: pinmux_P9_12_gpio_pin {
+		pinctrl-single,pins = <0x078  0x2F>; };     /* Mode 7, RxActive */
+	P9_12_gpio_pu_pin: pinmux_P9_12_gpio_pu_pin {
+		pinctrl-single,pins = <0x078  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_12_gpio_pd_pin: pinmux_P9_12_gpio_pd_pin {
+		pinctrl-single,pins = <0x078  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+
+	/* P9_13 (ZCZ ball U17) */
+	P9_13_default_pin: pinmux_P9_13_default_pin {
+		pinctrl-single,pins = <0x074  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_13_gpio_pin: pinmux_P9_13_gpio_pin {
+		pinctrl-single,pins = <0x074  0x2F>; };     /* Mode 7, RxActive */
+	P9_13_gpio_pu_pin: pinmux_P9_13_gpio_pu_pin {
+		pinctrl-single,pins = <0x074  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_13_gpio_pd_pin: pinmux_P9_13_gpio_pd_pin {
+		pinctrl-single,pins = <0x074  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_13_uart_pin: pinmux_P9_13_uart_pin {
+		pinctrl-single,pins = <0x074  0x36>; };     /* Mode 6, Pull-Up, RxActive */
+
+	/* P9_14 (ZCZ ball U14) */
+	P9_14_default_pin: pinmux_P9_14_default_pin {
+		pinctrl-single,pins = <0x048  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_14_gpio_pin: pinmux_P9_14_gpio_pin {
+		pinctrl-single,pins = <0x048  0x2F>; };     /* Mode 7, RxActive */
+	P9_14_gpio_pu_pin: pinmux_P9_14_gpio_pu_pin {
+		pinctrl-single,pins = <0x048  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_14_gpio_pd_pin: pinmux_P9_14_gpio_pd_pin {
+		pinctrl-single,pins = <0x048  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_14_pwm_pin: pinmux_P9_14_pwm_pin {
+		pinctrl-single,pins = <0x048  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+
+	/* P9_15 (ZCZ ball R13) */
+	P9_15_default_pin: pinmux_P9_15_default_pin {
+		pinctrl-single,pins = <0x040  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_15_gpio_pin: pinmux_P9_15_gpio_pin {
+		pinctrl-single,pins = <0x040  0x2F>; };     /* Mode 7, RxActive */
+	P9_15_gpio_pu_pin: pinmux_P9_15_gpio_pu_pin {
+		pinctrl-single,pins = <0x040  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_15_gpio_pd_pin: pinmux_P9_15_gpio_pd_pin {
+		pinctrl-single,pins = <0x040  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_15_pwm_pin: pinmux_P9_15_pwm_pin {
+		pinctrl-single,pins = <0x040  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+
+	/* P9_16 (ZCZ ball T14) */
+	P9_16_default_pin: pinmux_P9_16_default_pin {
+		pinctrl-single,pins = <0x04c  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_16_gpio_pin: pinmux_P9_16_gpio_pin {
+		pinctrl-single,pins = <0x04c  0x2F>; };     /* Mode 7, RxActive */
+	P9_16_gpio_pu_pin: pinmux_P9_16_gpio_pu_pin {
+		pinctrl-single,pins = <0x04c  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_16_gpio_pd_pin: pinmux_P9_16_gpio_pd_pin {
+		pinctrl-single,pins = <0x04c  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_16_pwm_pin: pinmux_P9_16_pwm_pin {
+		pinctrl-single,pins = <0x04c  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+
+	/* P9_17 (ZCZ ball A16) */
+	P9_17_default_pin: pinmux_P9_17_default_pin {
+		pinctrl-single,pins = <0x15c  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_17_gpio_pin: pinmux_P9_17_gpio_pin {
+		pinctrl-single,pins = <0x15c  0x2F>; };     /* Mode 7, RxActive */
+	P9_17_gpio_pu_pin: pinmux_P9_17_gpio_pu_pin {
+		pinctrl-single,pins = <0x15c  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_17_gpio_pd_pin: pinmux_P9_17_gpio_pd_pin {
+		pinctrl-single,pins = <0x15c  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_17_spi_pin: pinmux_P9_17_spi_pin {
+		pinctrl-single,pins = <0x15c  0x30>; };     /* Mode 0, Pull-Up, RxActive */
+	P9_17_spi_cs_pin: pinmux_P9_17_spi_cs_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };	/* spi0_cs0.spi0_cs0 */
+	P9_17_i2c_pin: pinmux_P9_17_i2c_pin {
+		pinctrl-single,pins = <0x15c  0x32>; };     /* Mode 2, Pull-Up, RxActive */
+	P9_17_pwm_pin: pinmux_P9_17_pwm_pin {
+		pinctrl-single,pins = <0x15c  0x33>; };     /* Mode 3, Pull-Up, RxActive */
+
+	/* P9_18 (ZCZ ball B16) */
+	P9_18_default_pin: pinmux_P9_18_default_pin {
+		pinctrl-single,pins = <0x158  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_18_gpio_pin: pinmux_P9_18_gpio_pin {
+		pinctrl-single,pins = <0x158  0x2F>; };     /* Mode 7, RxActive */
+	P9_18_gpio_pu_pin: pinmux_P9_18_gpio_pu_pin {
+		pinctrl-single,pins = <0x158  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_18_gpio_pd_pin: pinmux_P9_18_gpio_pd_pin {
+		pinctrl-single,pins = <0x158  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_18_spi_pin: pinmux_P9_18_spi_pin {
+		pinctrl-single,pins = <0x158  0x30>; };     /* Mode 0, Pull-Up, RxActive */
+	P9_18_i2c_pin: pinmux_P9_18_i2c_pin {
+		pinctrl-single,pins = <0x158  0x32>; };     /* Mode 2, Pull-Up, RxActive */
+	P9_18_pwm_pin: pinmux_P9_18_pwm_pin {
+		pinctrl-single,pins = <0x158  0x33>; };     /* Mode 3, Pull-Up, RxActive */
+
+	/* P9_19 (ZCZ ball D17) */
+	P9_19_default_pin: pinmux_P9_19_default_pin {
+		pinctrl-single,pins = <0x17c  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_19_gpio_pin: pinmux_P9_19_gpio_pin {
+		pinctrl-single,pins = <0x17c  0x2F>; };     /* Mode 7, RxActive */
+	P9_19_gpio_pu_pin: pinmux_P9_19_gpio_pu_pin {
+		pinctrl-single,pins = <0x17c  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_19_gpio_pd_pin: pinmux_P9_19_gpio_pd_pin {
+		pinctrl-single,pins = <0x17c  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_19_can_pin: pinmux_P9_19_can_pin {
+		pinctrl-single,pins = <0x17c  0x32>; };     /* Mode 2, Pull-Up, RxActive */
+	P9_19_i2c_pin: pinmux_P9_19_i2c_pin {
+		pinctrl-single,pins = <0x17c  0x73>; };     /* (SLEWCTRL_SLOW | PIN_INPUT_PULLUP | MUX_MODE3) */
+
+	/* P9_20 (ZCZ ball D18) */
+	P9_20_default_pin: pinmux_P9_20_default_pin {
+		pinctrl-single,pins = <0x178  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_20_gpio_pin: pinmux_P9_20_gpio_pin {
+		pinctrl-single,pins = <0x178  0x2F>; };     /* Mode 7, RxActive */
+	P9_20_gpio_pu_pin: pinmux_P9_20_gpio_pu_pin {
+		pinctrl-single,pins = <0x178  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_20_gpio_pd_pin: pinmux_P9_20_gpio_pd_pin {
+		pinctrl-single,pins = <0x178  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_20_can_pin: pinmux_P9_20_can_pin {
+		pinctrl-single,pins = <0x178  0x12>; };     /* Mode 2, Pull-Up, RxActive */
+	P9_20_i2c_pin: pinmux_P9_20_i2c_pin {
+		pinctrl-single,pins = <0x178  0x73>; };     /* (SLEWCTRL_SLOW | PIN_INPUT_PULLUP | MUX_MODE3) */
+
+	/* P9_21 (ZCZ ball B17) */
+	P9_21_default_pin: pinmux_P9_21_default_pin {
+		pinctrl-single,pins = <0x154  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_21_gpio_pin: pinmux_P9_21_gpio_pin {
+		pinctrl-single,pins = <0x154  0x2F>; };     /* Mode 7, RxActive */
+	P9_21_gpio_pu_pin: pinmux_P9_21_gpio_pu_pin {
+		pinctrl-single,pins = <0x154  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_21_gpio_pd_pin: pinmux_P9_21_gpio_pd_pin {
+		pinctrl-single,pins = <0x154  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_21_spi_pin: pinmux_P9_21_spi_pin {
+		pinctrl-single,pins = <0x154  0x30>; };     /* Mode 0, Pull-Up, RxActive */
+	P9_21_uart_pin: pinmux_P9_21_uart_pin {
+		pinctrl-single,pins = <0x154  0x31>; };     /* Mode 1, Pull-Up, RxActive */
+	P9_21_i2c_pin: pinmux_P9_21_i2c_pin {
+		pinctrl-single,pins = <0x154  0x32>; };     /* Mode 2, Pull-Up, RxActive */
+	P9_21_pwm_pin: pinmux_P9_21_pwm_pin {
+		pinctrl-single,pins = <0x154  0x33>; };     /* Mode 3, Pull-Up, RxActive */
+
+	/* P9_22 (ZCZ ball A17) */
+	P9_22_default_pin: pinmux_P9_22_default_pin {
+		pinctrl-single,pins = <0x150  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_22_gpio_pin: pinmux_P9_22_gpio_pin {
+		pinctrl-single,pins = <0x150  0x2F>; };     /* Mode 7, RxActive */
+	P9_22_gpio_pu_pin: pinmux_P9_22_gpio_pu_pin {
+		pinctrl-single,pins = <0x150  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_22_gpio_pd_pin: pinmux_P9_22_gpio_pd_pin {
+		pinctrl-single,pins = <0x150  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_22_spi_pin: pinmux_P9_22_spi_pin {
+		pinctrl-single,pins = <0x150  0x30>; };     /* Mode 0, Pull-Up, RxActive */
+	P9_22_uart_pin: pinmux_P9_22_uart_pin {
+		pinctrl-single,pins = <0x150  0x31>; };     /* Mode 1, Pull-Up, RxActive */
+	P9_22_i2c_pin: pinmux_P9_22_i2c_pin {
+		pinctrl-single,pins = <0x150  0x32>; };     /* Mode 2, Pull-Up, RxActive */
+	P9_22_pwm_pin: pinmux_P9_22_pwm_pin {
+		pinctrl-single,pins = <0x150  0x33>; };     /* Mode 3, Pull-Up, RxActive */
+
+	/* P9_23 (ZCZ ball V14) */
+	P9_23_default_pin: pinmux_P9_23_default_pin {
+		pinctrl-single,pins = <0x044  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_23_gpio_pin: pinmux_P9_23_gpio_pin {
+		pinctrl-single,pins = <0x044  0x2F>; };     /* Mode 7, RxActive */
+	P9_23_gpio_pu_pin: pinmux_P9_23_gpio_pu_pin {
+		pinctrl-single,pins = <0x044  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_23_gpio_pd_pin: pinmux_P9_23_gpio_pd_pin {
+		pinctrl-single,pins = <0x044  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_23_pwm_pin: pinmux_P9_23_pwm_pin {
+		pinctrl-single,pins = <0x044  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+
+	/* P9_24 (ZCZ ball D15) */
+	P9_24_default_pin: pinmux_P9_24_default_pin {
+		pinctrl-single,pins = <0x184  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_24_gpio_pin: pinmux_P9_24_gpio_pin {
+		pinctrl-single,pins = <0x184  0x2F>; };     /* Mode 7, RxActive */
+	P9_24_gpio_pu_pin: pinmux_P9_24_gpio_pu_pin {
+		pinctrl-single,pins = <0x184  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_24_gpio_pd_pin: pinmux_P9_24_gpio_pd_pin {
+		pinctrl-single,pins = <0x184  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_24_uart_pin: pinmux_P9_24_uart_pin {
+		pinctrl-single,pins = <0x184  0x30>; };     /* Mode 0, Pull-Up, RxActive */
+	P9_24_can_pin: pinmux_P9_24_can_pin {
+		pinctrl-single,pins = <0x184  0x32>; };     /* Mode 2, Pull-Up, RxActive */
+	P9_24_i2c_pin: pinmux_P9_24_i2c_pin {
+		pinctrl-single,pins = <0x184  0x33>; };     /* Mode 3, Pull-Up, RxActive */
+	P9_24_pruin_pin: pinmux_P9_24_pruin_pin {
+		pinctrl-single,pins = <0x184  0x36>; };     /* Mode 6, Pull-Up, RxActive */
+
+	/* P9_25 (ZCZ ball A14) Audio   */
+	P9_25_default_pin: pinmux_P9_25_default_pin {
+		pinctrl-single,pins = <0x1ac  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_25_gpio_pin: pinmux_P9_25_gpio_pin {
+		pinctrl-single,pins = <0x1ac  0x2F>; };     /* Mode 7, RxActive */
+	P9_25_gpio_pu_pin: pinmux_P9_25_gpio_pu_pin {
+		pinctrl-single,pins = <0x1ac  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_25_gpio_pd_pin: pinmux_P9_25_gpio_pd_pin {
+		pinctrl-single,pins = <0x1ac  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_25_qep_pin: pinmux_P9_25_qep_pin {
+		pinctrl-single,pins = <0x1ac  0x21>; };     /* Mode 1, Pull-Down, RxActive */
+	P9_25_pruout_pin: pinmux_P9_25_pruout_pin {
+		pinctrl-single,pins = <0x1ac  0x25>; };     /* Mode 5, Pull-Down, RxActive */
+	P9_25_pruin_pin: pinmux_P9_25_pruin_pin {
+		pinctrl-single,pins = <0x1ac  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+	P9_25_audio_pin: pinmux_P9_25_audio_pin {
+		pinctrl-single,pins = <0x1ac  (PIN_INPUT_PULLUP | MUX_MODE0)>; };	/* mcasp0_ahclkx.mcasp0_ahclkx */
+
+	/* P9_26 (ZCZ ball D16) */
+	P9_26_default_pin: pinmux_P9_26_default_pin {
+		pinctrl-single,pins = <0x180  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_26_gpio_pin: pinmux_P9_26_gpio_pin {
+		pinctrl-single,pins = <0x180  0x2F>; };     /* Mode 7, RxActive */
+	P9_26_gpio_pu_pin: pinmux_P9_26_gpio_pu_pin {
+		pinctrl-single,pins = <0x180  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_26_gpio_pd_pin: pinmux_P9_26_gpio_pd_pin {
+		pinctrl-single,pins = <0x180  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_26_uart_pin: pinmux_P9_26_uart_pin {
+		pinctrl-single,pins = <0x180  0x30>; };     /* Mode 0, Pull-Up, RxActive */
+	P9_26_can_pin: pinmux_P9_26_can_pin {
+		pinctrl-single,pins = <0x180  0x12>; };     /* Mode 2, Pull-Up, RxActive */
+	P9_26_i2c_pin: pinmux_P9_26_i2c_pin {
+		pinctrl-single,pins = <0x180  0x33>; };     /* Mode 3, Pull-Up, RxActive */
+	P9_26_pruin_pin: pinmux_P9_26_pruin_pin {
+		pinctrl-single,pins = <0x180  0x36>; };     /* Mode 6, Pull-Up, RxActive */
+
+	/* P9_27 (ZCZ ball C13) */
+	P9_27_default_pin: pinmux_P9_27_default_pin {
+		pinctrl-single,pins = <0x1a4  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_27_gpio_pin: pinmux_P9_27_gpio_pin {
+		pinctrl-single,pins = <0x1a4  0x2F>; };     /* Mode 7, RxActive */
+	P9_27_gpio_pu_pin: pinmux_P9_27_gpio_pu_pin {
+		pinctrl-single,pins = <0x1a4  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_27_gpio_pd_pin: pinmux_P9_27_gpio_pd_pin {
+		pinctrl-single,pins = <0x1a4  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_27_qep_pin: pinmux_P9_27_qep_pin {
+		pinctrl-single,pins = <0x1a4  0x21>; };     /* Mode 1, Pull-Down, RxActive */
+	P9_27_pruout_pin: pinmux_P9_27_pruout_pin {
+		pinctrl-single,pins = <0x1a4  0x25>; };     /* Mode 5, Pull-Down, RxActive */
+	P9_27_pruin_pin: pinmux_P9_27_pruin_pin {
+		pinctrl-single,pins = <0x1a4  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+
+	/* P9_28 (ZCZ ball C12) Audio   */
+	P9_28_default_pin: pinmux_P9_28_default_pin {
+		pinctrl-single,pins = <0x19c  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_28_gpio_pin: pinmux_P9_28_gpio_pin {
+		pinctrl-single,pins = <0x19c  0x2F>; };     /* Mode 7, RxActive */
+	P9_28_gpio_pu_pin: pinmux_P9_28_gpio_pu_pin {
+		pinctrl-single,pins = <0x19c  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_28_gpio_pd_pin: pinmux_P9_28_gpio_pd_pin {
+		pinctrl-single,pins = <0x19c  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_28_pwm_pin: pinmux_P9_28_pwm_pin {
+		pinctrl-single,pins = <0x19c  0x21>; };     /* Mode 1, Pull-Down, RxActive */
+	P9_28_spi_pin: pinmux_P9_28_spi_pin {
+		pinctrl-single,pins = <0x19c  0x23>; };     /* Mode 3, Pull-Down, RxActive */
+	P9_28_spi_cs_pin: pinmux_P9_28_spi_cs_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* mcasp0_ahclkr.spi1_cs0 */
+	P9_28_pwm2_pin: pinmux_P9_28_pwm2_pin {
+		pinctrl-single,pins = <0x19c  0x24>; };     /* Mode 4, Pull-Down, RxActive */
+	P9_28_pruout_pin: pinmux_P9_28_pruout_pin {
+		pinctrl-single,pins = <0x19c  0x25>; };     /* Mode 5, Pull-Down, RxActive */
+	P9_28_pruin_pin: pinmux_P9_28_pruin_pin {
+		pinctrl-single,pins = <0x19c  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+	P9_28_audio_pin: pinmux_P9_28_audio_pin {
+		pinctrl-single,pins = <0x19c  (PIN_OUTPUT_PULLDOWN | MUX_MODE2)>; };	/* mcasp0_ahclkr.mcasp0_axr2 */
+
+	/* P9_29 (ZCZ ball B13) Audio   */
+	P9_29_default_pin: pinmux_P9_29_default_pin {
+		pinctrl-single,pins = <0x194  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_29_gpio_pin: pinmux_P9_29_gpio_pin {
+		pinctrl-single,pins = <0x194  0x2F>; };     /* Mode 7, RxActive */
+	P9_29_gpio_pu_pin: pinmux_P9_29_gpio_pu_pin {
+		pinctrl-single,pins = <0x194  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_29_gpio_pd_pin: pinmux_P9_29_gpio_pd_pin {
+		pinctrl-single,pins = <0x194  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_29_pwm_pin: pinmux_P9_29_pwm_pin {
+		pinctrl-single,pins = <0x194  0x21>; };     /* Mode 1, Pull-Down, RxActive */
+	P9_29_spi_pin: pinmux_P9_29_spi_pin {
+		pinctrl-single,pins = <0x194  0x23>; };     /* Mode 3, Pull-Down, RxActive */
+	P9_29_pruout_pin: pinmux_P9_29_pruout_pin {
+		pinctrl-single,pins = <0x194  0x25>; };     /* Mode 5, Pull-Down, RxActive */
+	P9_29_pruin_pin: pinmux_P9_29_pruin_pin {
+		pinctrl-single,pins = <0x194  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+	P9_29_audio_pin: pinmux_P9_29_audio_pin {
+		pinctrl-single,pins = <0x194  (PIN_OUTPUT_PULLUP | MUX_MODE0)>; };	/* mcasp0_fsx.mcasp0_fsx */
+
+	/* P9_30 (ZCZ ball D12) */
+	P9_30_default_pin: pinmux_P9_30_default_pin {
+		pinctrl-single,pins = <0x198  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_30_gpio_pin: pinmux_P9_30_gpio_pin {
+		pinctrl-single,pins = <0x198  0x2F>; };     /* Mode 7, RxActive */
+	P9_30_gpio_pu_pin: pinmux_P9_30_gpio_pu_pin {
+		pinctrl-single,pins = <0x198  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_30_gpio_pd_pin: pinmux_P9_30_gpio_pd_pin {
+		pinctrl-single,pins = <0x198  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_30_pwm_pin: pinmux_P9_30_pwm_pin {
+		pinctrl-single,pins = <0x198  0x21>; };     /* Mode 1, Pull-Down, RxActive */
+	P9_30_spi_pin: pinmux_P9_30_spi_pin {
+		pinctrl-single,pins = <0x198  0x23>; };     /* Mode 3, Pull-Down, RxActive */
+	P9_30_pruout_pin: pinmux_P9_30_pruout_pin {
+		pinctrl-single,pins = <0x198  0x25>; };     /* Mode 5, Pull-Down, RxActive */
+	P9_30_pruin_pin: pinmux_P9_30_pruin_pin {
+		pinctrl-single,pins = <0x198  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+
+	/* P9_31 (ZCZ ball A13) Audio   */
+	P9_31_default_pin: pinmux_P9_31_default_pin {
+		pinctrl-single,pins = <0x190  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_31_gpio_pin: pinmux_P9_31_gpio_pin {
+		pinctrl-single,pins = <0x190  0x2F>; };     /* Mode 7, RxActive */
+	P9_31_gpio_pu_pin: pinmux_P9_31_gpio_pu_pin {
+		pinctrl-single,pins = <0x190  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_31_gpio_pd_pin: pinmux_P9_31_gpio_pd_pin {
+		pinctrl-single,pins = <0x190  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_31_pwm_pin: pinmux_P9_31_pwm_pin {
+		pinctrl-single,pins = <0x190  0x21>; };     /* Mode 1, Pull-Down, RxActive */
+	P9_31_spi_pin: pinmux_P9_31_spi_pin {
+		pinctrl-single,pins = <0x190  0x23>; };     /* Mode 3, Pull-Down, RxActive */
+	P9_31_spi_sclk_pin: pinmux_P9_31_spi_sclk_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* mcasp0_aclkx.spi1_sclk */
+	P9_31_pruout_pin: pinmux_P9_31_pruout_pin {
+		pinctrl-single,pins = <0x190  0x25>; };     /* Mode 5, Pull-Down, RxActive */
+	P9_31_pruin_pin: pinmux_P9_31_pruin_pin {
+		pinctrl-single,pins = <0x190  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+	P9_31_audio_pin: pinmux_P9_31_audio_pin {
+		pinctrl-single,pins = <0x190  (PIN_OUTPUT_PULLDOWN | MUX_MODE0)>; };	/* mcasp0_aclkx.mcasp0_aclkx */
+
+	/* P9_32                VADC    */
+	/* P9_33 (ZCZ ball C8 ) AIN4    */
+	/* P9_34                AGND    */
+	/* P9_35 (ZCZ ball A8 ) AIN6    */
+	/* P9_36 (ZCZ ball B8 ) AIN5    */
+	/* P9_37 (ZCZ ball B7 ) AIN2    */
+	/* P9_38 (ZCZ ball A7 ) AIN3    */
+	/* P9_39 (ZCZ ball B6 ) AIN0    */
+	/* P9_40 (ZCZ ball C7 ) AIN1    */
+
+	/* P9_41 (ZCZ ball D14) */
+	P9_41_default_pin: pinmux_P9_41_default_pin {
+		pinctrl-single,pins = <0x1b4  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_41_gpio_pin: pinmux_P9_41_gpio_pin {
+		pinctrl-single,pins = <0x1b4  0x2F>; };     /* Mode 7, RxActive */
+	P9_41_gpio_pu_pin: pinmux_P9_41_gpio_pu_pin {
+		pinctrl-single,pins = <0x1b4  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_41_gpio_pd_pin: pinmux_P9_41_gpio_pd_pin {
+		pinctrl-single,pins = <0x1b4  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_41_timer_pin: pinmux_P9_41_timer_pin {
+		pinctrl-single,pins = <0x1b4  0x24>; };     /* Mode 4, Pull-Down, RxActive */
+	P9_41_pruin_pin: pinmux_P9_41_pruin_pin {
+		pinctrl-single,pins = <0x1b4  0x25>; };     /* Mode 5, Pull-Down, RxActive */
+
+	/* P9_41.1              */
+	/* P9_91 (ZCZ ball D13) */
+	P9_91_default_pin: pinmux_P9_91_default_pin {
+		pinctrl-single,pins = <0x1a8  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_91_gpio_pin: pinmux_P9_91_gpio_pin {
+		pinctrl-single,pins = <0x1a8  0x2F>; };     /* Mode 7, RxActive */
+	P9_91_gpio_pu_pin: pinmux_P9_91_gpio_pu_pin {
+		pinctrl-single,pins = <0x1a8  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_91_gpio_pd_pin: pinmux_P9_91_gpio_pd_pin {
+		pinctrl-single,pins = <0x1a8  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_91_qep_pin: pinmux_P9_91_qep_pin {
+		pinctrl-single,pins = <0x1a8  0x21>; };     /* Mode 1, Pull-Down, RxActive */
+	P9_91_pruout_pin: pinmux_P9_91_pruout_pin {
+		pinctrl-single,pins = <0x1a8  0x25>; };     /* Mode 5, Pull-Down, RxActive */
+	P9_91_pruin_pin: pinmux_P9_91_pruin_pin {
+		pinctrl-single,pins = <0x1a8  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+
+	/* P9_42 (ZCZ ball C18) */
+	P9_42_default_pin: pinmux_P9_42_default_pin {
+		pinctrl-single,pins = <0x164  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_42_gpio_pin: pinmux_P9_42_gpio_pin {
+		pinctrl-single,pins = <0x164  0x2F>; };     /* Mode 7, RxActive */
+	P9_42_gpio_pu_pin: pinmux_P9_42_gpio_pu_pin {
+		pinctrl-single,pins = <0x164  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_42_gpio_pd_pin: pinmux_P9_42_gpio_pd_pin {
+		pinctrl-single,pins = <0x164  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_42_pwm_pin: pinmux_P9_42_pwm_pin {
+		pinctrl-single,pins = <0x164  0x20>; };     /* Mode 0, Pull-Down, RxActive */
+	P9_42_uart_pin: pinmux_P9_42_uart_pin {
+		pinctrl-single,pins = <0x164  0x21>; };     /* Mode 1, Pull-Down, RxActive */
+	P9_42_spics_pin: pinmux_P9_42_spics_pin {
+		pinctrl-single,pins = <0x164  0x22>; };     /* Mode 2, Pull-Down, RxActive */
+	P9_42_spiclk_pin: pinmux_P9_42_spiclk_pin {
+		pinctrl-single,pins = <0x164  0x24>; };     /* Mode 4, Pull-Down, RxActive */
+	P9_42_spi_cs_pin: pinmux_P9_42_spi_cs_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* eCAP0_in_PWM0_out.spi1_cs1 */
+	P9_42_spi_sclk_pin: pinmux_P9_42_spi_sclk_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* eCAP0_in_PWM0_out.spi1_sclk */
+
+	/* P9_42.1              */
+	/* P9_92 (ZCZ ball B12) */
+	P9_92_default_pin: pinmux_P9_92_default_pin {
+		pinctrl-single,pins = <0x1a0  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_92_gpio_pin: pinmux_P9_92_gpio_pin {
+		pinctrl-single,pins = <0x1a0  0x2F>; };     /* Mode 7, RxActive */
+	P9_92_gpio_pu_pin: pinmux_P9_92_gpio_pu_pin {
+		pinctrl-single,pins = <0x1a0  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	P9_92_gpio_pd_pin: pinmux_P9_92_gpio_pd_pin {
+		pinctrl-single,pins = <0x1a0  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+	P9_92_qep_pin: pinmux_P9_92_qep_pin {
+		pinctrl-single,pins = <0x1a0  0x21>; };     /* Mode 1, Pull-Down, RxActive */
+	P9_92_pruout_pin: pinmux_P9_92_pruout_pin {
+		pinctrl-single,pins = <0x1a0  0x25>; };     /* Mode 5, Pull-Down, RxActive */
+	P9_92_pruin_pin: pinmux_P9_92_pruin_pin {
+		pinctrl-single,pins = <0x1a0  0x26>; };     /* Mode 6, Pull-Down, RxActive */
+
+	/* P9_43                GND     */
+	/* P9_44                GND     */
+	/* P9_45                GND     */
+	/* P9_46                GND     */
+
+	/*       (ZCZ ball A15) */
+	A15_default_pin: pinmux_A15_default_pin {
+		pinctrl-single,pins = <0x1b0 (PIN_OUTPUT_PULLUP | MUX_MODE7)>; };     /* Mode 7, Pull-Up */
+	A15_clkout_pin: pinmux_A15_clkout_pin {
+		pinctrl-single,pins = <0x1b0  0x0b>; };     /* Mode 3 */
+	A15_gpio_pin: pinmux_A15_gpio_pin {
+		pinctrl-single,pins = <0x1b0  0x2f>; };     /* Mode 7, RxActive */
+	A15_gpio_pu_pin: pinmux_A15_gpio_pu_pin {
+		pinctrl-single,pins = <0x1b0  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	A15_gpio_pd_pin: pinmux_A15_gpio_pd_pin {
+		pinctrl-single,pins = <0x1b0  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-bone.dts b/arch/arm/boot/dts/am335x-bone.dts
--- a/arch/arm/boot/dts/am335x-bone.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-bone.dts	2020-06-19 21:00:31.404227715 -0400
@@ -1,18 +1,21 @@
+// SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 /dts-v1/;
 
 #include "am33xx.dtsi"
 #include "am335x-bone-common.dtsi"
+/* #include "am335x-bone-jtag.dtsi" */
 
 / {
 	model = "TI AM335x BeagleBone";
 	compatible = "ti,am335x-bone", "ti,am33xx";
+
+	chosen {
+		base_dtb = "am335x-bone.dts";
+		base_dtb_timestamp = __TIMESTAMP__;
+	};
 };
 
 &ldo3_reg {
@@ -24,3 +27,7 @@
 &mmc1 {
 	vmmc-supply = <&ldo3_reg>;
 };
+
+&sgx {
+	status = "okay";
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-bonegreen-common.dtsi b/arch/arm/boot/dts/am335x-bonegreen-common.dtsi
--- a/arch/arm/boot/dts/am335x-bonegreen-common.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-bonegreen-common.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -1,9 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 &ldo3_reg {
@@ -27,8 +24,8 @@
 &am33xx_pinmux {
 	uart2_pins: uart2_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x950, PIN_INPUT | MUX_MODE1)	/* spi0_sclk.uart2_rxd */
-			AM33XX_IOPAD(0x954, PIN_OUTPUT | MUX_MODE1)	/* spi0_d0.uart2_txd */
+			AM33XX_PADCONF(AM335X_PIN_SPI0_SCLK, PIN_INPUT, MUX_MODE1)	/* spi0_sclk.uart2_rxd */
+			AM33XX_PADCONF(AM335X_PIN_SPI0_D0, PIN_OUTPUT, MUX_MODE1)	/* spi0_d0.uart2_txd */
 		>;
 	};
 };
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-bonegreen.dts b/arch/arm/boot/dts/am335x-bonegreen.dts
--- a/arch/arm/boot/dts/am335x-bonegreen.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-bonegreen.dts	2020-06-19 21:00:31.404227715 -0400
@@ -1,17 +1,24 @@
+// SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 /dts-v1/;
 
 #include "am33xx.dtsi"
 #include "am335x-bone-common.dtsi"
 #include "am335x-bonegreen-common.dtsi"
+/* #include "am335x-bone-jtag.dtsi" */
 
 / {
 	model = "TI AM335x BeagleBone Green";
 	compatible = "ti,am335x-bone-green", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx";
+
+	chosen {
+		base_dtb = "am335x-bonegreen.dts";
+		base_dtb_timestamp = __TIMESTAMP__;
+	};
+};
+
+&sgx {
+	status = "okay";
 };
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-bonegreen-gateway.dts b/arch/arm/boot/dts/am335x-bonegreen-gateway.dts
--- a/arch/arm/boot/dts/am335x-bonegreen-gateway.dts	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-bonegreen-gateway.dts	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,206 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ */
+/dts-v1/;
+
+#include "am33xx.dtsi"
+#include "am335x-bone-common.dtsi"
+#include "am335x-bonegreen-common.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+/* #include "am335x-bone-jtag.dtsi" */
+
+/ {
+	model = "SeeedStudio BeagleBone Green Gateway";
+	compatible = "ti,am335x-bone-green-gateway", "ti,am335x-bone-green", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx";
+
+	aliases {
+		rtc0 = &extrtc;
+		rtc1 = &rtc;
+	};
+
+	chosen {
+		base_dtb = "am335x-bonegreen-gateway.dts";
+		base_dtb_timestamp = __TIMESTAMP__;
+	};
+
+	wlan_en_reg: fixedregulator@2 {
+		compatible = "regulator-fixed";
+		regulator-name = "wlan-en-regulator";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		startup-delay-us= <70000>;
+
+		/* WL_EN */
+		gpio = <&gpio3 9 0>;
+		enable-active-high;
+	};
+
+	leds {
+		led6 {
+			label = "beaglebone:green:usr4";
+			gpios = <&gpio2 21 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "netdev";
+			default-state = "off";
+		};
+	};
+};
+
+&cpu0_opp_table {
+	/*
+	 * Octavo Systems:
+	 * The EFUSE_SMA register is not programmed for any of the AM335x wafers
+	 * we get and we are not programming them during our production test.
+	 * Therefore, from a DEVICE_ID revision point of view, the silicon looks
+	 * like it is Revision 2.1.  However, from an EFUSE_SMA point of view for
+	 * the HW OPP table, the silicon looks like it is Revision 1.0 (ie the
+	 * EFUSE_SMA register reads as all zeros).
+	 */
+	oppnitro-1000000000 {
+		opp-supported-hw = <0x06 0x0100>;
+	};
+};
+
+&sgx {
+	status = "okay";
+};
+
+&am33xx_pinmux {
+	pinctrl-names = "default";
+	pinctrl-0 = <&usbhost_pins>;
+
+	bt_pins: pinmux_bt_pins {
+		pinctrl-single,pins = <
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_OUTPUT_PULLUP, MUX_MODE7)	/* gmii1_txd0.gpio0_28 - BT_EN */
+		>;
+	};
+
+	mmc3_pins: pinmux_mmc3_pins {
+		pinctrl-single,pins = <
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLUP, MUX_MODE6 ) /* (L15) gmii1_rxd1.mmc2_clk */
+			AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_INPUT_PULLUP, MUX_MODE6 ) /* (J16) gmii1_txen.mmc2_cmd */
+			AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLUP, MUX_MODE5 ) /* (J17) gmii1_rxdv.mmc2_dat0 */
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_INPUT_PULLUP, MUX_MODE5 ) /* (J18) gmii1_txd3.mmc2_dat1 */
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_INPUT_PULLUP, MUX_MODE5 ) /* (K15) gmii1_txd2.mmc2_dat2 */
+			AM33XX_PADCONF(AM335X_PIN_MII1_COL, PIN_INPUT_PULLUP, MUX_MODE5 ) /* (H16) gmii1_col.mmc2_dat3 */
+		>;
+	};
+
+	uart2_grove_pins: pinmux_uart2_grove_pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x90c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6)
+			AM33XX_IOPAD(0x910, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6)
+		>;
+	};
+
+	uart3_pins: pinmux_uart3_pins {
+		pinctrl-single,pins = <
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLUP, MUX_MODE1)	/* gmii1_rxd3.uart3_rxd */
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_OUTPUT_PULLDOWN, MUX_MODE1)	/* gmii1_rxd2.uart3_txd */
+			AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT, MUX_MODE3)		/* mdio_data.uart3_ctsn */
+			AM33XX_PADCONF(AM335X_PIN_MDC, PIN_OUTPUT_PULLDOWN, MUX_MODE3)	/* mdio_clk.uart3_rtsn */
+		>;
+	};
+
+	wl18xx_pins: pinmux_wl18xx_pins {
+		pinctrl-single,pins = <
+			AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_OUTPUT_PULLDOWN, MUX_MODE7)	/* gmii1_txclk.gpio3_9 WL_EN */
+			AM33XX_PADCONF(AM335X_PIN_RMII1_REF_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7)	/* rmii1_refclk.gpio0_29 WL_IRQ */
+			AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_OUTPUT_PULLUP, MUX_MODE7)	/* gmii1_rxclk.gpio3_10 LS_BUF_EN */
+		>;
+	};
+
+	usbhost_pins: pinmux_usbhost_pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x924, PIN_OUTPUT_PULLUP | MUX_MODE7) /* gmii1_txd1.gpio0[21] */
+		>;
+	};
+};
+
+&mac {
+	status = "disabled";
+};
+
+&mmc3 {
+	dmas = <&edma_xbar 12 0 1
+		&edma_xbar 13 0 2>;
+	dma-names = "tx", "rx";
+	status = "okay";
+	vmmc-supply = <&wlan_en_reg>;
+	bus-width = <4>;
+	non-removable;
+	cap-power-off-card;
+	ti,needs-special-hs-handling;
+	keep-power-in-suspend;
+	pinctrl-names = "default";
+	pinctrl-0 = <&mmc3_pins &wl18xx_pins>;
+
+	#address-cells = <1>;
+	#size-cells = <0>;
+	wlcore: wlcore@2 {
+		compatible = "ti,wl1835";
+		reg = <2>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <29 IRQ_TYPE_EDGE_RISING>;
+	};
+};
+
+&uart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart2_grove_pins>;
+	status = "okay";
+};
+
+&uart3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart3_pins &bt_pins>;
+	status = "okay";
+
+	bluetooth {
+		compatible = "ti,wl1835-st";
+		enable-gpios = <&gpio0 28 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&i2c0 {
+	extrtc: rtc@68 {
+		compatible = "dallas,ds1340";
+		reg = <0x68>;
+	};
+};
+
+// (K16) gmii1_txd1.gpio0[21]
+&gpio0 {
+	usb_reset {
+		gpio-hog;
+		gpios = <21 GPIO_ACTIVE_HIGH>;
+		output-high;
+		line-name = "usb_reset";
+	};
+};
+
+&gpio3 {
+	ls_buf_en {
+		gpio-hog;
+		gpios = <10 GPIO_ACTIVE_HIGH>;
+		output-high;
+		line-name = "LS_BUF_EN";
+	};
+};
+
+&usb1 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	hub@1 {
+		compatible = "usb424,9512";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ethernet: ethernet@1 {
+			compatible = "usb424,ec00";
+			reg = <1>;
+		};
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-bonegreen-wireless-common-univ.dtsi b/arch/arm/boot/dts/am335x-bonegreen-wireless-common-univ.dtsi
--- a/arch/arm/boot/dts/am335x-bonegreen-wireless-common-univ.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-bonegreen-wireless-common-univ.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,2835 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+&am33xx_pinmux {
+	/************************/
+	/* P8 Header */
+	/************************/
+
+	/* P8_01                GND */
+
+	/* P8_02                GND */
+
+
+	/* P8_03 (ZCZ ball R9) emmc */
+	P8_03_default_pin: pinmux_P8_03_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0818, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad6.gpio1_6 */
+	P8_03_gpio_pin: pinmux_P8_03_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0818, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad6.gpio1_6 */
+	P8_03_gpio_pu_pin: pinmux_P8_03_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0818, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad6.gpio1_6 */
+	P8_03_gpio_pd_pin: pinmux_P8_03_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0818, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad6.gpio1_6 */
+	P8_03_gpio_input_pin: pinmux_P8_03_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0818, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad6.gpio1_6 */
+
+	/* P8_04 (ZCZ ball T9) emmc */
+	P8_04_default_pin: pinmux_P8_04_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x081c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad7.gpio1_7 */
+	P8_04_gpio_pin: pinmux_P8_04_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x081c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad7.gpio1_7 */
+	P8_04_gpio_pu_pin: pinmux_P8_04_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x081c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad7.gpio1_7 */
+	P8_04_gpio_pd_pin: pinmux_P8_04_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x081c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad7.gpio1_7 */
+	P8_04_gpio_input_pin: pinmux_P8_04_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x081c, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad7.gpio1_7 */
+
+	/* P8_05 (ZCZ ball R8) emmc */
+	P8_05_default_pin: pinmux_P8_05_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0808, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad2.gpio1_2 */
+	P8_05_gpio_pin: pinmux_P8_05_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0808, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad2.gpio1_2 */
+	P8_05_gpio_pu_pin: pinmux_P8_05_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0808, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad2.gpio1_2 */
+	P8_05_gpio_pd_pin: pinmux_P8_05_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0808, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad2.gpio1_2 */
+	P8_05_gpio_input_pin: pinmux_P8_05_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0808, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad2.gpio1_2 */
+
+	/* P8_06 (ZCZ ball T8) emmc */
+	P8_06_default_pin: pinmux_P8_06_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x080c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad3.gpio1_3 */
+	P8_06_gpio_pin: pinmux_P8_06_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x080c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad3.gpio1_3 */
+	P8_06_gpio_pu_pin: pinmux_P8_06_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x080c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad3.gpio1_3 */
+	P8_06_gpio_pd_pin: pinmux_P8_06_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x080c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad3.gpio1_3 */
+	P8_06_gpio_input_pin: pinmux_P8_06_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x080c, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad3.gpio1_3 */
+
+	/* P8_07 (ZCZ ball R7) gpio2_2 */
+	P8_07_default_pin: pinmux_P8_07_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_advn_ale.gpio2_2 */
+	P8_07_gpio_pin: pinmux_P8_07_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0890, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_advn_ale.gpio2_2 */
+	P8_07_gpio_pu_pin: pinmux_P8_07_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_advn_ale.gpio2_2 */
+	P8_07_gpio_pd_pin: pinmux_P8_07_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_advn_ale.gpio2_2 */
+	P8_07_gpio_input_pin: pinmux_P8_07_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0890, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_advn_ale.gpio2_2 */
+	P8_07_timer_pin: pinmux_P8_07_timer_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* gpmc_advn_ale.timer4 */
+
+	/* P8_08 (ZCZ ball T7) gpio2_3 */
+	P8_08_default_pin: pinmux_P8_08_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_oen_ren.gpio2_3 */
+	P8_08_gpio_pin: pinmux_P8_08_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0894, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_oen_ren.gpio2_3 */
+	P8_08_gpio_pu_pin: pinmux_P8_08_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_oen_ren.gpio2_3 */
+	P8_08_gpio_pd_pin: pinmux_P8_08_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_oen_ren.gpio2_3 */
+	P8_08_gpio_input_pin: pinmux_P8_08_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0894, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_oen_ren.gpio2_3 */
+	P8_08_timer_pin: pinmux_P8_08_timer_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* gpmc_oen_ren.timer7 */
+
+	/* P8_09 (ZCZ ball T6) gpio2_5 */
+	P8_09_default_pin: pinmux_P8_09_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_be0n_cle.gpio2_5 */
+	P8_09_gpio_pin: pinmux_P8_09_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x089c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_be0n_cle.gpio2_5 */
+	P8_09_gpio_pu_pin: pinmux_P8_09_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_be0n_cle.gpio2_5 */
+	P8_09_gpio_pd_pin: pinmux_P8_09_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_be0n_cle.gpio2_5 */
+	P8_09_gpio_input_pin: pinmux_P8_09_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x089c, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_be0n_cle.gpio2_5 */
+	P8_09_timer_pin: pinmux_P8_09_timer_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* gpmc_be0n_cle.timer5 */
+
+	/* P8_10 (ZCZ ball U6) gpio2_4 */
+	P8_10_default_pin: pinmux_P8_10_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_wen.gpio2_4 */
+	P8_10_gpio_pin: pinmux_P8_10_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0898, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_wen.gpio2_4 */
+	P8_10_gpio_pu_pin: pinmux_P8_10_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_wen.gpio2_4 */
+	P8_10_gpio_pd_pin: pinmux_P8_10_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_wen.gpio2_4 */
+	P8_10_gpio_input_pin: pinmux_P8_10_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0898, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_wen.gpio2_4 */
+	P8_10_timer_pin: pinmux_P8_10_timer_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* gpmc_wen.timer6 */
+
+	/* P8_11 (ZCZ ball R12) gpio1_13 */
+	P8_11_default_pin: pinmux_P8_11_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad13.gpio1_13 */
+	P8_11_gpio_pin: pinmux_P8_11_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0834, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad13.gpio1_13 */
+	P8_11_gpio_pu_pin: pinmux_P8_11_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad13.gpio1_13 */
+	P8_11_gpio_pd_pin: pinmux_P8_11_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad13.gpio1_13 */
+	P8_11_gpio_input_pin: pinmux_P8_11_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0834, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad13.gpio1_13 */
+	P8_11_qep_pin: pinmux_P8_11_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* gpmc_ad13.eqep2b_in */
+	P8_11_pruout_pin: pinmux_P8_11_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; };	/* gpmc_ad13.pru0_out15 */
+
+	/* P8_12 (ZCZ ball T12) gpio1_12 */
+	P8_12_default_pin: pinmux_P8_12_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad12.gpio1_12 */
+	P8_12_gpio_pin: pinmux_P8_12_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0830, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad12.gpio1_12 */
+	P8_12_gpio_pu_pin: pinmux_P8_12_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad12.gpio1_12 */
+	P8_12_gpio_pd_pin: pinmux_P8_12_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad12.gpio1_12 */
+	P8_12_gpio_input_pin: pinmux_P8_12_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0830, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad12.gpio1_12 */
+	P8_12_qep_pin: pinmux_P8_12_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* gpmc_ad12.eqep2a_in */
+	P8_12_pruout_pin: pinmux_P8_12_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; };	/* gpmc_ad12.pru0_out14 */
+
+	/* P8_13 (ZCZ ball T10) gpio0_23 */
+	P8_13_default_pin: pinmux_P8_13_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad9.gpio0_23 */
+	P8_13_gpio_pin: pinmux_P8_13_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0824, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad9.gpio0_23 */
+	P8_13_gpio_pu_pin: pinmux_P8_13_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad9.gpio0_23 */
+	P8_13_gpio_pd_pin: pinmux_P8_13_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad9.gpio0_23 */
+	P8_13_gpio_input_pin: pinmux_P8_13_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0824, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad9.gpio0_23 */
+	P8_13_pwm_pin: pinmux_P8_13_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; };	/* gpmc_ad9.ehrpwm2b */
+
+	/* P8_14 (ZCZ ball T11) wl1835: wl_en */
+
+	/* P8_15 (ZCZ ball U13) gpio1_15 */
+	P8_15_default_pin: pinmux_P8_15_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad15.gpio1_15 */
+	P8_15_gpio_pin: pinmux_P8_15_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x083c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad15.gpio1_15 */
+	P8_15_gpio_pu_pin: pinmux_P8_15_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad15.gpio1_15 */
+	P8_15_gpio_pd_pin: pinmux_P8_15_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad15.gpio1_15 */
+	P8_15_gpio_input_pin: pinmux_P8_15_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x083c, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad15.gpio1_15 */
+	P8_15_qep_pin: pinmux_P8_15_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* gpmc_ad15.eqep2_strobe */
+	P8_15_pru_ecap_pin: pinmux_P8_15_pru_ecap_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* gpmc_ad15.pr1_ecap0_ecap_capin_apwm_o */
+	P8_15_pruin_pin: pinmux_P8_15_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x083c, PIN_INPUT | MUX_MODE6) >; };			/* gpmc_ad15.pru0_in15 */
+
+	/* P8_16 (ZCZ ball V13) gpio1_14 */
+	P8_16_default_pin: pinmux_P8_16_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad14.gpio1_14 */
+	P8_16_gpio_pin: pinmux_P8_16_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0838, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad14.gpio1_14 */
+	P8_16_gpio_pu_pin: pinmux_P8_16_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad14.gpio1_14 */
+	P8_16_gpio_pd_pin: pinmux_P8_16_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad14.gpio1_14 */
+	P8_16_gpio_input_pin: pinmux_P8_16_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0838, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad14.gpio1_14 */
+	P8_16_qep_pin: pinmux_P8_16_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* gpmc_ad14.eqep2_index */
+	P8_16_pruin_pin: pinmux_P8_16_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0838, PIN_INPUT | MUX_MODE6) >; };			/* gpmc_ad14.pru0_in14 */
+
+	/* P8_17 (ZCZ ball U12) wl1835: wl_irq */
+
+	/* P8_18 (ZCZ ball V12) gpio2_1 */
+	P8_18_default_pin: pinmux_P8_18_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_clk.gpio2_1 */
+	P8_18_gpio_pin: pinmux_P8_18_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x088c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_clk.gpio2_1 */
+	P8_18_gpio_pu_pin: pinmux_P8_18_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_clk.gpio2_1 */
+	P8_18_gpio_pd_pin: pinmux_P8_18_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_clk.gpio2_1 */
+	P8_18_gpio_input_pin: pinmux_P8_18_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x088c, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_clk.gpio2_1 */
+
+	/* P8_19 (ZCZ ball U10) gpio0_22 */
+	P8_19_default_pin: pinmux_P8_19_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad8.gpio0_22 */
+	P8_19_gpio_pin: pinmux_P8_19_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0820, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad8.gpio0_22 */
+	P8_19_gpio_pu_pin: pinmux_P8_19_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad8.gpio0_22 */
+	P8_19_gpio_pd_pin: pinmux_P8_19_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad8.gpio0_22 */
+	P8_19_gpio_input_pin: pinmux_P8_19_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0820, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad8.gpio0_22 */
+	P8_19_pwm_pin: pinmux_P8_19_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; };	/* gpmc_ad8.ehrpwm2a */
+
+	/* P8_20 (ZCZ ball V9) emmc */
+	P8_20_default_pin: pinmux_P8_20_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_csn2.gpio1_31 */
+	P8_20_gpio_pin: pinmux_P8_20_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0884, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_csn2.gpio1_31 */
+	P8_20_gpio_pu_pin: pinmux_P8_20_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_csn2.gpio1_31 */
+	P8_20_gpio_pd_pin: pinmux_P8_20_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_csn2.gpio1_31 */
+	P8_20_gpio_input_pin: pinmux_P8_20_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0884, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_csn2.gpio1_31 */
+	P8_20_pruout_pin: pinmux_P8_20_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* gpmc_csn2.pru1_out13 */
+	P8_20_pruin_pin: pinmux_P8_20_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0884, PIN_INPUT | MUX_MODE6) >; };			/* gpmc_csn2.pru1_in13 */
+
+	/* P8_21 (ZCZ ball U9) emmc */
+	P8_21_default_pin: pinmux_P8_21_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_csn1.gpio1_30 */
+	P8_21_gpio_pin: pinmux_P8_21_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0880, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_csn1.gpio1_30 */
+	P8_21_gpio_pu_pin: pinmux_P8_21_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_csn1.gpio1_30 */
+	P8_21_gpio_pd_pin: pinmux_P8_21_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_csn1.gpio1_30 */
+	P8_21_gpio_input_pin: pinmux_P8_21_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0880, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_csn1.gpio1_30 */
+	P8_21_pruout_pin: pinmux_P8_21_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* gpmc_csn1.pru1_out12 */
+	P8_21_pruin_pin: pinmux_P8_21_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0880, PIN_INPUT | MUX_MODE6) >; };			/* gpmc_csn1.pru1_in12 */
+
+	/* P8_22 (ZCZ ball V8) emmc */
+	P8_22_default_pin: pinmux_P8_22_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0814, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad5.gpio1_5 */
+	P8_22_gpio_pin: pinmux_P8_22_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0814, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad5.gpio1_5 */
+	P8_22_gpio_pu_pin: pinmux_P8_22_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0814, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad5.gpio1_5 */
+	P8_22_gpio_pd_pin: pinmux_P8_22_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0814, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad5.gpio1_5 */
+	P8_22_gpio_input_pin: pinmux_P8_22_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0814, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad5.gpio1_5 */
+
+	/* P8_23 (ZCZ ball U8) emmc */
+	P8_23_default_pin: pinmux_P8_23_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0810, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad4.gpio1_4 */
+	P8_23_gpio_pin: pinmux_P8_23_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0810, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad4.gpio1_4 */
+	P8_23_gpio_pu_pin: pinmux_P8_23_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0810, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad4.gpio1_4 */
+	P8_23_gpio_pd_pin: pinmux_P8_23_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0810, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad4.gpio1_4 */
+	P8_23_gpio_input_pin: pinmux_P8_23_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0810, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad4.gpio1_4 */
+
+	/* P8_24 (ZCZ ball V7) emmc */
+	P8_24_default_pin: pinmux_P8_24_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0804, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad1.gpio1_1 */
+	P8_24_gpio_pin: pinmux_P8_24_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0804, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad1.gpio1_1 */
+	P8_24_gpio_pu_pin: pinmux_P8_24_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0804, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad1.gpio1_1 */
+	P8_24_gpio_pd_pin: pinmux_P8_24_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0804, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad1.gpio1_1 */
+	P8_24_gpio_input_pin: pinmux_P8_24_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0804, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad1.gpio1_1 */
+
+	/* P8_25 (ZCZ ball U7) emmc */
+	P8_25_default_pin: pinmux_P8_25_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0800, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad0.gpio1_0 */
+	P8_25_gpio_pin: pinmux_P8_25_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0800, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad0.gpio1_0 */
+	P8_25_gpio_pu_pin: pinmux_P8_25_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0800, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad0.gpio1_0 */
+	P8_25_gpio_pd_pin: pinmux_P8_25_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0800, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad0.gpio1_0 */
+	P8_25_gpio_input_pin: pinmux_P8_25_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0800, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad0.gpio1_0 */
+
+	/* P8_26 (ZCZ ball V6) gpio-hog wl1835 */
+
+	/* P8_27 (ZCZ ball U5) hdmi */
+	P8_27_default_pin: pinmux_P8_27_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_vsync.gpio2_22 */
+	P8_27_gpio_pin: pinmux_P8_27_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_vsync.gpio2_22 */
+	P8_27_gpio_pu_pin: pinmux_P8_27_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_vsync.gpio2_22 */
+	P8_27_gpio_pd_pin: pinmux_P8_27_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_vsync.gpio2_22 */
+	P8_27_gpio_input_pin: pinmux_P8_27_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e0, PIN_INPUT | MUX_MODE7) >; };			/* lcd_vsync.gpio2_22 */
+	P8_27_pruout_pin: pinmux_P8_27_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* lcd_vsync.pru1_out8 */
+	P8_27_pruin_pin: pinmux_P8_27_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e0, PIN_INPUT | MUX_MODE6) >; };			/* lcd_vsync.pru1_in8 */
+
+	/* P8_28 (ZCZ ball V5) hdmi */
+	P8_28_default_pin: pinmux_P8_28_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_pclk.gpio2_24 */
+	P8_28_gpio_pin: pinmux_P8_28_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_pclk.gpio2_24 */
+	P8_28_gpio_pu_pin: pinmux_P8_28_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_pclk.gpio2_24 */
+	P8_28_gpio_pd_pin: pinmux_P8_28_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_pclk.gpio2_24 */
+	P8_28_gpio_input_pin: pinmux_P8_28_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e8, PIN_INPUT | MUX_MODE7) >; };			/* lcd_pclk.gpio2_24 */
+	P8_28_pruout_pin: pinmux_P8_28_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* lcd_pclk.pru1_out10 */
+	P8_28_pruin_pin: pinmux_P8_28_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e8, PIN_INPUT | MUX_MODE6) >; };			/* lcd_pclk.pru1_in10 */
+
+	/* P8_29 (ZCZ ball R5) hdmi */
+	P8_29_default_pin: pinmux_P8_29_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_hsync.gpio2_23 */
+	P8_29_gpio_pin: pinmux_P8_29_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_hsync.gpio2_23 */
+	P8_29_gpio_pu_pin: pinmux_P8_29_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_hsync.gpio2_23 */
+	P8_29_gpio_pd_pin: pinmux_P8_29_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_hsync.gpio2_23 */
+	P8_29_gpio_input_pin: pinmux_P8_29_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e4, PIN_INPUT | MUX_MODE7) >; };			/* lcd_hsync.gpio2_23 */
+	P8_29_pruout_pin: pinmux_P8_29_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* lcd_hsync.pru1_out9 */
+	P8_29_pruin_pin: pinmux_P8_29_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e4, PIN_INPUT | MUX_MODE6) >; };			/* lcd_hsync.pru1_in9 */
+
+	/* P8_30 (ZCZ ball R6) hdmi */
+	P8_30_default_pin: pinmux_P8_30_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_ac_bias_en.gpio2_25 */
+	P8_30_gpio_pin: pinmux_P8_30_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ec, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_ac_bias_en.gpio2_25 */
+	P8_30_gpio_pu_pin: pinmux_P8_30_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_ac_bias_en.gpio2_25 */
+	P8_30_gpio_pd_pin: pinmux_P8_30_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_ac_bias_en.gpio2_25 */
+	P8_30_gpio_input_pin: pinmux_P8_30_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ec, PIN_INPUT | MUX_MODE7) >; };			/* lcd_ac_bias_en.gpio2_25 */
+	P8_30_pruout_pin: pinmux_P8_30_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* lcd_ac_bias_en.pru1_out11 */
+	P8_30_pruin_pin: pinmux_P8_30_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ec, PIN_INPUT | MUX_MODE6) >; };			/* lcd_ac_bias_en.pru1_in11 */
+
+	/* P8_31 (ZCZ ball V4) hdmi */
+	P8_31_default_pin: pinmux_P8_31_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data14.gpio0_10 */
+	P8_31_gpio_pin: pinmux_P8_31_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data14.gpio0_10 */
+	P8_31_gpio_pu_pin: pinmux_P8_31_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data14.gpio0_10 */
+	P8_31_gpio_pd_pin: pinmux_P8_31_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data14.gpio0_10 */
+	P8_31_gpio_input_pin: pinmux_P8_31_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d8, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data14.gpio0_10 */
+	P8_31_qep_pin: pinmux_P8_31_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* lcd_data14.eqep1_index */
+	P8_31_uart_pin: pinmux_P8_31_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* lcd_data14.uart5_rxd */
+
+	/* P8_32 (ZCZ ball T5) hdmi */
+	P8_32_default_pin: pinmux_P8_32_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data15.gpio0_11 */
+	P8_32_gpio_pin: pinmux_P8_32_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08dc, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data15.gpio0_11 */
+	P8_32_gpio_pu_pin: pinmux_P8_32_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data15.gpio0_11 */
+	P8_32_gpio_pd_pin: pinmux_P8_32_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data15.gpio0_11 */
+	P8_32_gpio_input_pin: pinmux_P8_32_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08dc, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data15.gpio0_11 */
+	P8_32_qep_pin: pinmux_P8_32_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* lcd_data15.eqep1_strobe */
+
+	/* P8_33 (ZCZ ball V3) hdmi */
+	P8_33_default_pin: pinmux_P8_33_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data13.gpio0_9 */
+	P8_33_gpio_pin: pinmux_P8_33_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data13.gpio0_9 */
+	P8_33_gpio_pu_pin: pinmux_P8_33_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data13.gpio0_9 */
+	P8_33_gpio_pd_pin: pinmux_P8_33_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data13.gpio0_9 */
+	P8_33_gpio_input_pin: pinmux_P8_33_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d4, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data13.gpio0_9 */
+	P8_33_qep_pin: pinmux_P8_33_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* lcd_data13.eqep1b_in */
+
+	/* P8_34 (ZCZ ball U4) hdmi */
+	P8_34_default_pin: pinmux_P8_34_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data11.gpio2_17 */
+	P8_34_gpio_pin: pinmux_P8_34_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08cc, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data11.gpio2_17 */
+	P8_34_gpio_pu_pin: pinmux_P8_34_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data11.gpio2_17 */
+	P8_34_gpio_pd_pin: pinmux_P8_34_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data11.gpio2_17 */
+	P8_34_gpio_input_pin: pinmux_P8_34_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08cc, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data11.gpio2_17 */
+	P8_34_pwm_pin: pinmux_P8_34_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; };	/* lcd_data11.ehrpwm1b */
+
+	/* P8_35 (ZCZ ball V2) hdmi */
+	P8_35_default_pin: pinmux_P8_35_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data12.gpio0_8 */
+	P8_35_gpio_pin: pinmux_P8_35_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data12.gpio0_8 */
+	P8_35_gpio_pu_pin: pinmux_P8_35_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data12.gpio0_8 */
+	P8_35_gpio_pd_pin: pinmux_P8_35_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data12.gpio0_8 */
+	P8_35_gpio_input_pin: pinmux_P8_35_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d0, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data12.gpio0_8 */
+	P8_35_qep_pin: pinmux_P8_35_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* lcd_data12.eqep1a_in */
+
+	/* P8_36 (ZCZ ball U3) hdmi */
+	P8_36_default_pin: pinmux_P8_36_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data10.gpio2_16 */
+	P8_36_gpio_pin: pinmux_P8_36_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data10.gpio2_16 */
+	P8_36_gpio_pu_pin: pinmux_P8_36_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data10.gpio2_16 */
+	P8_36_gpio_pd_pin: pinmux_P8_36_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data10.gpio2_16 */
+	P8_36_gpio_input_pin: pinmux_P8_36_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c8, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data10.gpio2_16 */
+	P8_36_pwm_pin: pinmux_P8_36_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; };	/* lcd_data10.ehrpwm1a */
+
+	/* P8_37 (ZCZ ball U1) hdmi */
+	P8_37_default_pin: pinmux_P8_37_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data8.gpio2_14 */
+	P8_37_gpio_pin: pinmux_P8_37_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data8.gpio2_14 */
+	P8_37_gpio_pu_pin: pinmux_P8_37_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data8.gpio2_14 */
+	P8_37_gpio_pd_pin: pinmux_P8_37_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data8.gpio2_14 */
+	P8_37_gpio_input_pin: pinmux_P8_37_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c0, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data8.gpio2_14 */
+	P8_37_pwm_pin: pinmux_P8_37_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; };	/* lcd_data8.ehrpwm1_tripzone_input */
+	P8_37_uart_pin: pinmux_P8_37_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* lcd_data8.uart5_txd */
+
+	/* P8_38 (ZCZ ball U2) hdmi */
+	P8_38_default_pin: pinmux_P8_38_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data9.gpio2_15 */
+	P8_38_gpio_pin: pinmux_P8_38_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data9.gpio2_15 */
+	P8_38_gpio_pu_pin: pinmux_P8_38_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data9.gpio2_15 */
+	P8_38_gpio_pd_pin: pinmux_P8_38_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data9.gpio2_15 */
+	P8_38_gpio_input_pin: pinmux_P8_38_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c4, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data9.gpio2_15 */
+	P8_38_pwm_pin: pinmux_P8_38_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; };	/* lcd_data9.ehrpwm0_synco */
+	P8_38_uart_pin: pinmux_P8_38_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* lcd_data9.uart5_rxd */
+
+	/* P8_39 (ZCZ ball T3) hdmi */
+	P8_39_default_pin: pinmux_P8_39_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data6.gpio2_12 */
+	P8_39_gpio_pin: pinmux_P8_39_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data6.gpio2_12 */
+	P8_39_gpio_pu_pin: pinmux_P8_39_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data6.gpio2_12 */
+	P8_39_gpio_pd_pin: pinmux_P8_39_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data6.gpio2_12 */
+	P8_39_gpio_input_pin: pinmux_P8_39_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b8, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data6.gpio2_12 */
+	P8_39_qep_pin: pinmux_P8_39_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* lcd_data6.eqep2_index */
+	P8_39_pruout_pin: pinmux_P8_39_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* lcd_data6.pru1_out6 */
+	P8_39_pruin_pin: pinmux_P8_39_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b8, PIN_INPUT | MUX_MODE6) >; };			/* lcd_data6.pru1_in6 */
+
+	/* P8_40 (ZCZ ball T4) hdmi */
+	P8_40_default_pin: pinmux_P8_40_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data7.gpio2_13 */
+	P8_40_gpio_pin: pinmux_P8_40_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08bc, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data7.gpio2_13 */
+	P8_40_gpio_pu_pin: pinmux_P8_40_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data7.gpio2_13 */
+	P8_40_gpio_pd_pin: pinmux_P8_40_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data7.gpio2_13 */
+	P8_40_gpio_input_pin: pinmux_P8_40_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08bc, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data7.gpio2_13 */
+	P8_40_qep_pin: pinmux_P8_40_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* lcd_data7.eqep2_strobe */
+	P8_40_pruout_pin: pinmux_P8_40_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* lcd_data7.pru1_out7 */
+	P8_40_pruin_pin: pinmux_P8_40_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08bc, PIN_INPUT | MUX_MODE6) >; };			/* lcd_data7.pru1_in7 */
+
+	/* P8_41 (ZCZ ball T1) hdmi */
+	P8_41_default_pin: pinmux_P8_41_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data4.gpio2_10 */
+	P8_41_gpio_pin: pinmux_P8_41_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data4.gpio2_10 */
+	P8_41_gpio_pu_pin: pinmux_P8_41_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data4.gpio2_10 */
+	P8_41_gpio_pd_pin: pinmux_P8_41_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data4.gpio2_10 */
+	P8_41_gpio_input_pin: pinmux_P8_41_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b0, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data4.gpio2_10 */
+	P8_41_qep_pin: pinmux_P8_41_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* lcd_data4.eqep2a_in */
+	P8_41_pruout_pin: pinmux_P8_41_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* lcd_data4.pru1_out4 */
+	P8_41_pruin_pin: pinmux_P8_41_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b0, PIN_INPUT | MUX_MODE6) >; };			/* lcd_data4.pru1_in4 */
+
+	/* P8_42 (ZCZ ball T2) hdmi */
+	P8_42_default_pin: pinmux_P8_42_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data5.gpio2_11 */
+	P8_42_gpio_pin: pinmux_P8_42_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data5.gpio2_11 */
+	P8_42_gpio_pu_pin: pinmux_P8_42_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data5.gpio2_11 */
+	P8_42_gpio_pd_pin: pinmux_P8_42_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data5.gpio2_11 */
+	P8_42_gpio_input_pin: pinmux_P8_42_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b4, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data5.gpio2_11 */
+	P8_42_qep_pin: pinmux_P8_42_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* lcd_data5.eqep2b_in */
+	P8_42_pruout_pin: pinmux_P8_42_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* lcd_data5.pru1_out5 */
+	P8_42_pruin_pin: pinmux_P8_42_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08b4, PIN_INPUT | MUX_MODE6) >; };			/* lcd_data5.pru1_in5 */
+
+	/* P8_43 (ZCZ ball R3) hdmi */
+	P8_43_default_pin: pinmux_P8_43_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data2.gpio2_8 */
+	P8_43_gpio_pin: pinmux_P8_43_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data2.gpio2_8 */
+	P8_43_gpio_pu_pin: pinmux_P8_43_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data2.gpio2_8 */
+	P8_43_gpio_pd_pin: pinmux_P8_43_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data2.gpio2_8 */
+	P8_43_gpio_input_pin: pinmux_P8_43_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a8, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data2.gpio2_8 */
+	P8_43_pwm_pin: pinmux_P8_43_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; };	/* lcd_data2.ehrpwm2_tripzone_input */
+	P8_43_pruout_pin: pinmux_P8_43_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* lcd_data2.pru1_out2 */
+	P8_43_pruin_pin: pinmux_P8_43_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a8, PIN_INPUT | MUX_MODE6) >; };			/* lcd_data2.pru1_in2 */
+
+	/* P8_44 (ZCZ ball R4) hdmi */
+	P8_44_default_pin: pinmux_P8_44_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data3.gpio2_9 */
+	P8_44_gpio_pin: pinmux_P8_44_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ac, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data3.gpio2_9 */
+	P8_44_gpio_pu_pin: pinmux_P8_44_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data3.gpio2_9 */
+	P8_44_gpio_pd_pin: pinmux_P8_44_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data3.gpio2_9 */
+	P8_44_gpio_input_pin: pinmux_P8_44_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ac, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data3.gpio2_9 */
+	P8_44_pwm_pin: pinmux_P8_44_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; };	/* lcd_data3.ehrpwm0_synco */
+	P8_44_pruout_pin: pinmux_P8_44_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* lcd_data3.pru1_out3 */
+	P8_44_pruin_pin: pinmux_P8_44_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ac, PIN_INPUT | MUX_MODE6) >; };			/* lcd_data3.pru1_in3 */
+
+	/* P8_45 (ZCZ ball R1) hdmi */
+	P8_45_default_pin: pinmux_P8_45_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data0.gpio2_6 */
+	P8_45_gpio_pin: pinmux_P8_45_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data0.gpio2_6 */
+	P8_45_gpio_pu_pin: pinmux_P8_45_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data0.gpio2_6 */
+	P8_45_gpio_pd_pin: pinmux_P8_45_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data0.gpio2_6 */
+	P8_45_gpio_input_pin: pinmux_P8_45_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a0, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data0.gpio2_6 */
+	P8_45_pwm_pin: pinmux_P8_45_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; };	/* lcd_data0.ehrpwm2a */
+	P8_45_pruout_pin: pinmux_P8_45_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* lcd_data0.pru1_out0 */
+	P8_45_pruin_pin: pinmux_P8_45_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a0, PIN_INPUT | MUX_MODE6) >; };			/* lcd_data0.pru1_in0 */
+
+	/* P8_46 (ZCZ ball R2) hdmi */
+	P8_46_default_pin: pinmux_P8_46_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data1.gpio2_7 */
+	P8_46_gpio_pin: pinmux_P8_46_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_data1.gpio2_7 */
+	P8_46_gpio_pu_pin: pinmux_P8_46_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_data1.gpio2_7 */
+	P8_46_gpio_pd_pin: pinmux_P8_46_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_data1.gpio2_7 */
+	P8_46_gpio_input_pin: pinmux_P8_46_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a4, PIN_INPUT | MUX_MODE7) >; };			/* lcd_data1.gpio2_7 */
+	P8_46_pwm_pin: pinmux_P8_46_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; };	/* lcd_data1.ehrpwm2b */
+	P8_46_pruout_pin: pinmux_P8_46_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* lcd_data1.pru1_out1 */
+	P8_46_pruin_pin: pinmux_P8_46_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08a4, PIN_INPUT | MUX_MODE6) >; };			/* lcd_data1.pru1_in1 */
+
+	/************************/
+	/* P9 Header */
+	/************************/
+
+	/* P9_01                GND */
+
+	/* P9_02                GND */
+
+	/* P9_03                3V3 */
+
+	/* P9_04                3V3 */
+
+	/* P9_05                VDD_5V */
+
+	/* P9_06                VDD_5V */
+
+	/* P9_07                SYS_5V */
+
+	/* P9_08                SYS_5V */
+
+	/* P9_09                PWR_BUT */
+
+	/* P9_10                RSTn */
+
+	/* P9_11 (ZCZ ball T17) gpio0_30 */
+	P9_11_default_pin: pinmux_P9_11_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_wait0.gpio0_30 */
+	P9_11_gpio_pin: pinmux_P9_11_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0870, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_wait0.gpio0_30 */
+	P9_11_gpio_pu_pin: pinmux_P9_11_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_wait0.gpio0_30 */
+	P9_11_gpio_pd_pin: pinmux_P9_11_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_wait0.gpio0_30 */
+	P9_11_gpio_input_pin: pinmux_P9_11_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0870, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_wait0.gpio0_30 */
+	P9_11_uart_pin: pinmux_P9_11_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; };	/* gpmc_wait0.uart4_rxd */
+
+	/* P9_12 (ZCZ ball U18) gpio1_28 */
+	P9_12_default_pin: pinmux_P9_12_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_be1n.gpio1_28 */
+	P9_12_gpio_pin: pinmux_P9_12_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0878, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_be1n.gpio1_28 */
+	P9_12_gpio_pu_pin: pinmux_P9_12_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_be1n.gpio1_28 */
+	P9_12_gpio_pd_pin: pinmux_P9_12_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_be1n.gpio1_28 */
+	P9_12_gpio_input_pin: pinmux_P9_12_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0878, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_be1n.gpio1_28 */
+
+	/* P9_13 (ZCZ ball U17) gpio0_31 */
+	P9_13_default_pin: pinmux_P9_13_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_wpn.gpio0_31 */
+	P9_13_gpio_pin: pinmux_P9_13_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0874, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_wpn.gpio0_31 */
+	P9_13_gpio_pu_pin: pinmux_P9_13_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_wpn.gpio0_31 */
+	P9_13_gpio_pd_pin: pinmux_P9_13_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_wpn.gpio0_31 */
+	P9_13_gpio_input_pin: pinmux_P9_13_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0874, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_wpn.gpio0_31 */
+	P9_13_uart_pin: pinmux_P9_13_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; };	/* gpmc_wpn.uart4_txd */
+
+	/* P9_14 (ZCZ ball U14) gpio1_18 */
+	P9_14_default_pin: pinmux_P9_14_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a2.gpio1_18 */
+	P9_14_gpio_pin: pinmux_P9_14_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0848, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_a2.gpio1_18 */
+	P9_14_gpio_pu_pin: pinmux_P9_14_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a2.gpio1_18 */
+	P9_14_gpio_pd_pin: pinmux_P9_14_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a2.gpio1_18 */
+	P9_14_gpio_input_pin: pinmux_P9_14_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0848, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_a2.gpio1_18 */
+	P9_14_pwm_pin: pinmux_P9_14_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; };	/* gpmc_a2.ehrpwm1a */
+
+	/* P9_15 (ZCZ ball R13) gpio1_16 */
+	P9_15_default_pin: pinmux_P9_15_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a0.gpio1_16 */
+	P9_15_gpio_pin: pinmux_P9_15_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0840, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_a0.gpio1_16 */
+	P9_15_gpio_pu_pin: pinmux_P9_15_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a0.gpio1_16 */
+	P9_15_gpio_pd_pin: pinmux_P9_15_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a0.gpio1_16 */
+	P9_15_gpio_input_pin: pinmux_P9_15_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0840, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_a0.gpio1_16 */
+	P9_15_pwm_pin: pinmux_P9_15_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; };	/* gpmc_a0.ehrpwm1_tripzone_input */
+
+	/* P9_16 (ZCZ ball T14) gpio1_19 */
+	P9_16_default_pin: pinmux_P9_16_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a3.gpio1_19 */
+	P9_16_gpio_pin: pinmux_P9_16_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x084c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_a3.gpio1_19 */
+	P9_16_gpio_pu_pin: pinmux_P9_16_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a3.gpio1_19 */
+	P9_16_gpio_pd_pin: pinmux_P9_16_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a3.gpio1_19 */
+	P9_16_gpio_input_pin: pinmux_P9_16_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x084c, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_a3.gpio1_19 */
+	P9_16_pwm_pin: pinmux_P9_16_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; };	/* gpmc_a3.ehrpwm1b */
+
+	/* P9_17 (ZCZ ball A16) gpio0_5 */
+	P9_17_default_pin: pinmux_P9_17_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* spi0_cs0.gpio0_5 */
+	P9_17_gpio_pin: pinmux_P9_17_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x095c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* spi0_cs0.gpio0_5 */
+	P9_17_gpio_pu_pin: pinmux_P9_17_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* spi0_cs0.gpio0_5 */
+	P9_17_gpio_pd_pin: pinmux_P9_17_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* spi0_cs0.gpio0_5 */
+	P9_17_gpio_input_pin: pinmux_P9_17_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x095c, PIN_INPUT | MUX_MODE7) >; };			/* spi0_cs0.gpio0_5 */
+	P9_17_spi_cs_pin: pinmux_P9_17_spi_cs_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };	/* spi0_cs0.spi0_cs0 */
+	P9_17_i2c_pin: pinmux_P9_17_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* spi0_cs0.i2c1_scl */
+	P9_17_pwm_pin: pinmux_P9_17_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; };	/* spi0_cs0.ehrpwm0_synci */
+	P9_17_pru_uart_pin: pinmux_P9_17_pru_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* spi0_cs0.pr1_uart0_txd */
+
+	/* P9_18 (ZCZ ball B16) gpio0_4 */
+	P9_18_default_pin: pinmux_P9_18_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* spi0_d1.gpio0_4 */
+	P9_18_gpio_pin: pinmux_P9_18_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0958, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* spi0_d1.gpio0_4 */
+	P9_18_gpio_pu_pin: pinmux_P9_18_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* spi0_d1.gpio0_4 */
+	P9_18_gpio_pd_pin: pinmux_P9_18_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* spi0_d1.gpio0_4 */
+	P9_18_gpio_input_pin: pinmux_P9_18_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0958, PIN_INPUT | MUX_MODE7) >; };			/* spi0_d1.gpio0_4 */
+	P9_18_spi_pin: pinmux_P9_18_spi_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };	/* spi0_d1.spi0_d1 */
+	P9_18_i2c_pin: pinmux_P9_18_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* spi0_d1.i2c1_sda */
+	P9_18_pwm_pin: pinmux_P9_18_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; };	/* spi0_d1.ehrpwm0_tripzone_input */
+	P9_18_pru_uart_pin: pinmux_P9_18_pru_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* spi0_d1.pr1_uart0_rxd */
+
+	/* P9_19 (ZCZ ball D17) i2c2_scl */
+	P9_19_default_pin: pinmux_P9_19_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* uart1_rtsn.i2c2_scl */
+	P9_19_gpio_pin: pinmux_P9_19_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* uart1_rtsn.gpio0_13 */
+	P9_19_gpio_pu_pin: pinmux_P9_19_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* uart1_rtsn.gpio0_13 */
+	P9_19_gpio_pd_pin: pinmux_P9_19_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* uart1_rtsn.gpio0_13 */
+	P9_19_gpio_input_pin: pinmux_P9_19_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_INPUT | MUX_MODE7) >; };			/* uart1_rtsn.gpio0_13 */
+	P9_19_timer_pin: pinmux_P9_19_timer_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* uart1_rtsn.timer5 */
+	P9_19_can_pin: pinmux_P9_19_can_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_INPUT_PULLUP | MUX_MODE2) >; };		/* uart1_rtsn.dcan0_rx */
+	P9_19_i2c_pin: pinmux_P9_19_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* uart1_rtsn.i2c2_scl */
+	P9_19_spi_cs_pin: pinmux_P9_19_spi_cs_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* uart1_rtsn.spi1_cs1 */
+	P9_19_pru_uart_pin: pinmux_P9_19_pru_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; };	/* uart1_rtsn.pr1_uart0_rts_n */
+
+	/* P9_20 (ZCZ ball D18) i2c2_sda */
+	P9_20_default_pin: pinmux_P9_20_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* uart1_ctsn.i2c2_sda */
+	P9_20_gpio_pin: pinmux_P9_20_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* uart1_ctsn.gpio0_12 */
+	P9_20_gpio_pu_pin: pinmux_P9_20_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* uart1_ctsn.gpio0_12 */
+	P9_20_gpio_pd_pin: pinmux_P9_20_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* uart1_ctsn.gpio0_12 */
+	P9_20_gpio_input_pin: pinmux_P9_20_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_INPUT | MUX_MODE7) >; };			/* uart1_ctsn.gpio0_12 */
+	P9_20_timer_pin: pinmux_P9_20_timer_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* uart1_ctsn.timer6 */
+	P9_20_can_pin: pinmux_P9_20_can_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | MUX_MODE2) >; };		/* uart1_ctsn.dcan0_tx */
+	P9_20_i2c_pin: pinmux_P9_20_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* uart1_ctsn.i2c2_sda */
+	P9_20_spi_cs_pin: pinmux_P9_20_spi_cs_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* uart1_ctsn.spi1_cs0 */
+	P9_20_pru_uart_pin: pinmux_P9_20_pru_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; };	/* uart1_ctsn.pr1_uart0_cts_n */
+
+	/* P9_21 (ZCZ ball B17) gpio0_3 */
+	P9_21_default_pin: pinmux_P9_21_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* spi0_d0.gpio0_3 */
+	P9_21_gpio_pin: pinmux_P9_21_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* spi0_d0.gpio0_3 */
+	P9_21_gpio_pu_pin: pinmux_P9_21_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* spi0_d0.gpio0_3 */
+	P9_21_gpio_pd_pin: pinmux_P9_21_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* spi0_d0.gpio0_3 */
+	P9_21_gpio_input_pin: pinmux_P9_21_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_INPUT | MUX_MODE7) >; };			/* spi0_d0.gpio0_3 */
+	P9_21_spi_pin: pinmux_P9_21_spi_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };	/* spi0_d0.spi0_d0 */
+	P9_21_uart_pin: pinmux_P9_21_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* spi0_d0.uart2_txd */
+	P9_21_i2c_pin: pinmux_P9_21_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* spi0_d0.i2c2_scl */
+	P9_21_pwm_pin: pinmux_P9_21_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; };	/* spi0_d0.ehrpwm0b */
+	P9_21_pru_uart_pin: pinmux_P9_21_pru_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* spi0_d0.pr1_uart0_rts_n */
+
+	/* P9_22 (ZCZ ball A17) gpio0_2 */
+	P9_22_default_pin: pinmux_P9_22_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* spi0_sclk.gpio0_2 */
+	P9_22_gpio_pin: pinmux_P9_22_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* spi0_sclk.gpio0_2 */
+	P9_22_gpio_pu_pin: pinmux_P9_22_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* spi0_sclk.gpio0_2 */
+	P9_22_gpio_pd_pin: pinmux_P9_22_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* spi0_sclk.gpio0_2 */
+	P9_22_gpio_input_pin: pinmux_P9_22_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_INPUT | MUX_MODE7) >; };			/* spi0_sclk.gpio0_2 */
+	P9_22_spi_sclk_pin: pinmux_P9_22_spi_sclk_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };	/* spi0_sclk.spi0_sclk */
+	P9_22_uart_pin: pinmux_P9_22_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* spi0_sclk.uart2_rxd */
+	P9_22_i2c_pin: pinmux_P9_22_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* spi0_sclk.i2c2_sda */
+	P9_22_pwm_pin: pinmux_P9_22_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; };	/* spi0_sclk.ehrpwm0a */
+	P9_22_pru_uart_pin: pinmux_P9_22_pru_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* spi0_sclk.pr1_uart0_cts_n */
+
+	/* P9_23 (ZCZ ball V14) gpio1_17 */
+	P9_23_default_pin: pinmux_P9_23_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a1.gpio1_17 */
+	P9_23_gpio_pin: pinmux_P9_23_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0844, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_a1.gpio1_17 */
+	P9_23_gpio_pu_pin: pinmux_P9_23_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a1.gpio1_17 */
+	P9_23_gpio_pd_pin: pinmux_P9_23_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a1.gpio1_17 */
+	P9_23_gpio_input_pin: pinmux_P9_23_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0844, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_a1.gpio1_17 */
+	P9_23_pwm_pin: pinmux_P9_23_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; };	/* gpmc_a1.ehrpwm0_synco */
+
+	/* P9_24 (ZCZ ball D15) gpio0_15 */
+	P9_24_default_pin: pinmux_P9_24_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* uart1_txd.gpio0_15 */
+	P9_24_gpio_pin: pinmux_P9_24_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* uart1_txd.gpio0_15 */
+	P9_24_gpio_pu_pin: pinmux_P9_24_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* uart1_txd.gpio0_15 */
+	P9_24_gpio_pd_pin: pinmux_P9_24_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* uart1_txd.gpio0_15 */
+	P9_24_gpio_input_pin: pinmux_P9_24_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_INPUT | MUX_MODE7) >; };			/* uart1_txd.gpio0_15 */
+	P9_24_uart_pin: pinmux_P9_24_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };	/* uart1_txd.uart1_txd */
+	P9_24_can_pin: pinmux_P9_24_can_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_INPUT_PULLUP | MUX_MODE2) >; };		/* uart1_txd.dcan1_rx */
+	P9_24_i2c_pin: pinmux_P9_24_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* uart1_txd.i2c1_scl */
+	P9_24_pru_uart_pin: pinmux_P9_24_pru_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; };	/* uart1_txd.pr1_uart0_txd */
+	P9_24_pruin_pin: pinmux_P9_24_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_INPUT | MUX_MODE6) >; };			/* uart1_txd.pru0_in16 */
+
+	/* P9_25 (ZCZ ball A14) audio */
+	P9_25_default_pin: pinmux_P9_25_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_ahclkx.gpio3_21 */
+	P9_25_gpio_pin: pinmux_P9_25_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09ac, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* mcasp0_ahclkx.gpio3_21 */
+	P9_25_gpio_pu_pin: pinmux_P9_25_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_ahclkx.gpio3_21 */
+	P9_25_gpio_pd_pin: pinmux_P9_25_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_ahclkx.gpio3_21 */
+	P9_25_gpio_input_pin: pinmux_P9_25_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09ac, PIN_INPUT | MUX_MODE7) >; };			/* mcasp0_ahclkx.gpio3_21 */
+	P9_25_qep_pin: pinmux_P9_25_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* mcasp0_ahclkx.eqep0_strobe */
+	P9_25_pruout_pin: pinmux_P9_25_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* mcasp0_ahclkx.pru0_out7 */
+	P9_25_pruin_pin: pinmux_P9_25_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09ac, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_ahclkx.pru0_in7 */
+
+	/* P9_26 (ZCZ ball D16) gpio0_14 */
+	P9_26_default_pin: pinmux_P9_26_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* uart1_rxd.gpio0_14 */
+	P9_26_gpio_pin: pinmux_P9_26_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* uart1_rxd.gpio0_14 */
+	P9_26_gpio_pu_pin: pinmux_P9_26_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* uart1_rxd.gpio0_14 */
+	P9_26_gpio_pd_pin: pinmux_P9_26_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* uart1_rxd.gpio0_14 */
+	P9_26_gpio_input_pin: pinmux_P9_26_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_INPUT | MUX_MODE7) >; };			/* uart1_rxd.gpio0_14 */
+	P9_26_uart_pin: pinmux_P9_26_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };	/* uart1_rxd.uart1_rxd */
+	P9_26_can_pin: pinmux_P9_26_can_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | MUX_MODE2) >; };		/* uart1_rxd.dcan1_tx */
+	P9_26_i2c_pin: pinmux_P9_26_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* uart1_rxd.i2c1_sda */
+	P9_26_pru_uart_pin: pinmux_P9_26_pru_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; };	/* uart1_rxd.pr1_uart0_rxd */
+	P9_26_pruin_pin: pinmux_P9_26_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_INPUT | MUX_MODE6) >; };			/* uart1_rxd.pru1_in16 */
+
+	/* P9_27 (ZCZ ball C13) gpio3_19 */
+	P9_27_default_pin: pinmux_P9_27_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_fsr.gpio3_19 */
+	P9_27_gpio_pin: pinmux_P9_27_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* mcasp0_fsr.gpio3_19 */
+	P9_27_gpio_pu_pin: pinmux_P9_27_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_fsr.gpio3_19 */
+	P9_27_gpio_pd_pin: pinmux_P9_27_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_fsr.gpio3_19 */
+	P9_27_gpio_input_pin: pinmux_P9_27_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a4, PIN_INPUT | MUX_MODE7) >; };			/* mcasp0_fsr.gpio3_19 */
+	P9_27_qep_pin: pinmux_P9_27_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* mcasp0_fsr.eqep0b_in */
+	P9_27_pruout_pin: pinmux_P9_27_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* mcasp0_fsr.pru0_out5 */
+	P9_27_pruin_pin: pinmux_P9_27_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a4, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_fsr.pru0_in5 */
+
+	/* P9_28 (ZCZ ball C12) audio */
+	P9_28_default_pin: pinmux_P9_28_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_ahclkr.gpio3_17 */
+	P9_28_gpio_pin: pinmux_P9_28_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* mcasp0_ahclkr.gpio3_17 */
+	P9_28_gpio_pu_pin: pinmux_P9_28_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_ahclkr.gpio3_17 */
+	P9_28_gpio_pd_pin: pinmux_P9_28_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_ahclkr.gpio3_17 */
+	P9_28_gpio_input_pin: pinmux_P9_28_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_INPUT | MUX_MODE7) >; };			/* mcasp0_ahclkr.gpio3_17 */
+	P9_28_pwm_pin: pinmux_P9_28_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; };	/* mcasp0_ahclkr.ehrpwm0_synci */
+	P9_28_spi_cs_pin: pinmux_P9_28_spi_cs_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* mcasp0_ahclkr.spi1_cs0 */
+	P9_28_pwm2_pin: pinmux_P9_28_pwm2_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; };	/* mcasp0_ahclkr.ecap2_in_pwm2_out */
+	P9_28_pruout_pin: pinmux_P9_28_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* mcasp0_ahclkr.pru0_out3 */
+	P9_28_pruin_pin: pinmux_P9_28_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_ahclkr.pru0_in3 */
+
+	/* P9_29 (ZCZ ball B13) audio */
+	P9_29_default_pin: pinmux_P9_29_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_fsx.gpio3_15 */
+	P9_29_gpio_pin: pinmux_P9_29_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0994, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* mcasp0_fsx.gpio3_15 */
+	P9_29_gpio_pu_pin: pinmux_P9_29_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_fsx.gpio3_15 */
+	P9_29_gpio_pd_pin: pinmux_P9_29_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_fsx.gpio3_15 */
+	P9_29_gpio_input_pin: pinmux_P9_29_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0994, PIN_INPUT | MUX_MODE7) >; };			/* mcasp0_fsx.gpio3_15 */
+	P9_29_pwm_pin: pinmux_P9_29_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; };	/* mcasp0_fsx.ehrpwm0b */
+	P9_29_spi_pin: pinmux_P9_29_spi_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* mcasp0_fsx.spi1_d0 */
+	P9_29_pruout_pin: pinmux_P9_29_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* mcasp0_fsx.pru0_out1 */
+	P9_29_pruin_pin: pinmux_P9_29_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0994, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_fsx.pru0_in1 */
+
+	/* P9_30 (ZCZ ball D12) gpio-hog wl1835 */
+
+	/* P9_31 (ZCZ ball A13) audio */
+	P9_31_default_pin: pinmux_P9_31_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_aclkx.gpio3_14 */
+	P9_31_gpio_pin: pinmux_P9_31_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0990, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* mcasp0_aclkx.gpio3_14 */
+	P9_31_gpio_pu_pin: pinmux_P9_31_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_aclkx.gpio3_14 */
+	P9_31_gpio_pd_pin: pinmux_P9_31_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_aclkx.gpio3_14 */
+	P9_31_gpio_input_pin: pinmux_P9_31_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0990, PIN_INPUT | MUX_MODE7) >; };			/* mcasp0_aclkx.gpio3_14 */
+	P9_31_pwm_pin: pinmux_P9_31_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; };	/* mcasp0_aclkx.ehrpwm0a */
+	P9_31_spi_sclk_pin: pinmux_P9_31_spi_sclk_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* mcasp0_aclkx.spi1_sclk */
+	P9_31_pruout_pin: pinmux_P9_31_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* mcasp0_aclkx.pru0_out0 */
+	P9_31_pruin_pin: pinmux_P9_31_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0990, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_aclkx.pru0_in0 */
+
+	/* P9_32                VADC */
+
+	/* P9_33 (ZCZ ball C8)  AIN4         */
+
+	/* P9_34                AGND */
+
+	/* P9_35 (ZCZ ball A8)  AIN6         */
+
+	/* P9_36 (ZCZ ball B8)  AIN5         */
+
+	/* P9_37 (ZCZ ball B7)  AIN2         */
+
+	/* P9_38 (ZCZ ball A7)  AIN3         */
+
+	/* P9_39 (ZCZ ball B6)  AIN0         */
+
+	/* P9_40 (ZCZ ball C7)  AIN1         */
+
+	/* P9_41 (ZCZ ball D14) gpio0_20 */
+	P9_41_default_pin: pinmux_P9_41_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* xdma_event_intr1.gpio0_20 */
+	P9_41_gpio_pin: pinmux_P9_41_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09b4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* xdma_event_intr1.gpio0_20 */
+	P9_41_gpio_pu_pin: pinmux_P9_41_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* xdma_event_intr1.gpio0_20 */
+	P9_41_gpio_pd_pin: pinmux_P9_41_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* xdma_event_intr1.gpio0_20 */
+	P9_41_gpio_input_pin: pinmux_P9_41_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09b4, PIN_INPUT | MUX_MODE7) >; };			/* xdma_event_intr1.gpio0_20 */
+	P9_41_timer_pin: pinmux_P9_41_timer_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* xdma_event_intr1.timer7 */
+	P9_41_pruin_pin: pinmux_P9_41_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09b4, PIN_INPUT | MUX_MODE5) >; };			/* xdma_event_intr1.pru0_in16 */
+
+	/* P9_41.1 */
+	/* P9_91 (ZCZ ball D13) gpio3_20 */
+	P9_91_default_pin: pinmux_P9_91_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_axr1.gpio3_20 */
+	P9_91_gpio_pin: pinmux_P9_91_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* mcasp0_axr1.gpio3_20 */
+	P9_91_gpio_pu_pin: pinmux_P9_91_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_axr1.gpio3_20 */
+	P9_91_gpio_pd_pin: pinmux_P9_91_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_axr1.gpio3_20 */
+	P9_91_gpio_input_pin: pinmux_P9_91_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a8, PIN_INPUT | MUX_MODE7) >; };			/* mcasp0_axr1.gpio3_20 */
+	P9_91_qep_pin: pinmux_P9_91_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* mcasp0_axr1.eqep0_index */
+	P9_91_pruout_pin: pinmux_P9_91_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* mcasp0_axr1.pru0_out6 */
+	P9_91_pruin_pin: pinmux_P9_91_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a8, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_axr1.pru0_in6 */
+
+	/* P9_42 (ZCZ ball C18) gpio0_7 */
+	P9_42_default_pin: pinmux_P9_42_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* eCAP0_in_PWM0_out.gpio0_7 */
+	P9_42_gpio_pin: pinmux_P9_42_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* eCAP0_in_PWM0_out.gpio0_7 */
+	P9_42_gpio_pu_pin: pinmux_P9_42_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* eCAP0_in_PWM0_out.gpio0_7 */
+	P9_42_gpio_pd_pin: pinmux_P9_42_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* eCAP0_in_PWM0_out.gpio0_7 */
+	P9_42_gpio_input_pin: pinmux_P9_42_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_INPUT | MUX_MODE7) >; };			/* eCAP0_in_PWM0_out.gpio0_7 */
+	P9_42_pwm_pin: pinmux_P9_42_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE0) >; };	/* eCAP0_in_PWM0_out.ecap0_in_pwm0_out */
+	P9_42_uart_pin: pinmux_P9_42_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* eCAP0_in_PWM0_out.uart3_txd */
+	P9_42_spi_cs_pin: pinmux_P9_42_spi_cs_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* eCAP0_in_PWM0_out.spi1_cs1 */
+	P9_42_pru_ecap_pin: pinmux_P9_42_pru_ecap_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; };	/* eCAP0_in_PWM0_out.pr1_ecap0_ecap_capin_apwm_o */
+	P9_42_spi_sclk_pin: pinmux_P9_42_spi_sclk_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* eCAP0_in_PWM0_out.spi1_sclk */
+
+	/* P9_42.1 */
+	/* P9_92 (ZCZ ball B12) gpio3_18 */
+	P9_92_default_pin: pinmux_P9_92_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_aclkr.gpio3_18 */
+	P9_92_gpio_pin: pinmux_P9_92_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* mcasp0_aclkr.gpio3_18 */
+	P9_92_gpio_pu_pin: pinmux_P9_92_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_aclkr.gpio3_18 */
+	P9_92_gpio_pd_pin: pinmux_P9_92_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_aclkr.gpio3_18 */
+	P9_92_gpio_input_pin: pinmux_P9_92_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a0, PIN_INPUT | MUX_MODE7) >; };			/* mcasp0_aclkr.gpio3_18 */
+	P9_92_qep_pin: pinmux_P9_92_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* mcasp0_aclkr.eqep0a_in */
+	P9_92_pruout_pin: pinmux_P9_92_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* mcasp0_aclkr.pru0_out4 */
+	P9_92_pruin_pin: pinmux_P9_92_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a0, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_aclkr.pru0_in4 */
+
+	/* P9_43                GND */
+
+	/* P9_44                GND */
+
+	/* P9_45                GND */
+
+	/* P9_46                GND */
+
+	/*       (ZCZ ball A15) */
+	A15_default_pin: pinmux_A15_default_pin {
+		pinctrl-single,pins = <0x1b0  (PIN_OUTPUT_PULLUP | MUX_MODE7)>; };     /* Mode 7, Pull-Up */
+	A15_clkout_pin: pinmux_A15_clkout_pin {
+		pinctrl-single,pins = <0x1b0  0x0b>; };     /* Mode 3 */
+	A15_gpio_pin: pinmux_A15_gpio_pin {
+		pinctrl-single,pins = <0x1b0  0x2f>; };     /* Mode 7, RxActive */
+	A15_gpio_pu_pin: pinmux_A15_gpio_pu_pin {
+		pinctrl-single,pins = <0x1b0  0x37>; };     /* Mode 7, Pull-Up, RxActive */
+	A15_gpio_pd_pin: pinmux_A15_gpio_pd_pin {
+		pinctrl-single,pins = <0x1b0  0x27>; };     /* Mode 7, Pull-Down, RxActive */
+};
+
+&i2c1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+
+	clock-frequency = <100000>;
+	symlink = "bone/i2c/1";
+};
+
+&i2c2 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+
+	clock-frequency = <100000>;
+	symlink = "bone/i2c/2";
+};
+
+&uart1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+	symlink = "bone/uart/1";
+};
+
+&uart2 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+	symlink = "bone/uart/2";
+};
+
+&uart3 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+	symlink = "bone/uart/3";
+};
+
+&uart4 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+	symlink = "bone/uart/4";
+};
+
+&uart5 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+	symlink = "bone/uart/5";
+};
+
+&dcan0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+	symlink = "bone/can/0";
+};
+
+&dcan1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+	symlink = "bone/can/1";
+};
+
+&eqep0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+
+	count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */
+	swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */
+	invert_qa = <1>; /* Should we invert the channel A input? */
+	invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */
+	invert_qi = <0>; /* Should we invert the index input? */
+	invert_qs = <0>; /* Should we invert the strobe input? */
+	symlink = "bone/eqep/0";
+};
+
+&eqep1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+
+	count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */
+	swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */
+	invert_qa = <1>; /* Should we invert the channel A input? */
+	invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */
+	invert_qi = <0>; /* Should we invert the index input? */
+	invert_qs = <0>; /* Should we invert the strobe input? */
+	symlink = "bone/eqep/1";
+};
+
+&eqep2 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+
+	count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */
+	swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */
+	invert_qa = <1>; /* Should we invert the channel A input? */
+	invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */
+	invert_qi = <0>; /* Should we invert the index input? */
+	invert_qs = <0>; /* Should we invert the strobe input? */
+	symlink = "bone/eqep/2";
+};
+
+&epwmss0 {
+	status = "okay";
+};
+
+&epwmss1 {
+	status = "okay";
+};
+
+&epwmss2 {
+	status = "okay";
+};
+
+&ehrpwm0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+};
+
+&ehrpwm1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+};
+
+&ehrpwm2 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+};
+
+&ecap0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+};
+
+&ecap1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+};
+
+&ecap2 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+};
+
+&spi0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+
+	channel@0 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		compatible = "spidev";
+		symlink = "spi/0.0";
+
+		reg = <0>;
+		spi-max-frequency = <16000000>;
+		spi-cpha;
+	};
+
+	channel@1 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		compatible = "spidev";
+		symlink = "spi/0.1";
+
+		reg = <1>;
+		spi-max-frequency = <16000000>;
+	};
+};
+
+&spi1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+
+	channel@0 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		compatible = "spidev";
+		symlink = "spi/1.0";
+
+		reg = <0>;
+		spi-max-frequency = <16000000>;
+		spi-cpha;
+	};
+
+	channel@1 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		compatible = "spidev";
+		symlink = "spi/1.1";
+
+		reg = <1>;
+		spi-max-frequency = <16000000>;
+	};
+};
+
+/**********************************************************************/
+/* Pin Multiplex Helpers                                              */
+/*                                                                    */
+/* These provide userspace runtime pin configuration for the          */
+/* BeagleBone cape expansion headers                                  */
+/**********************************************************************/
+
+&ocp {
+	/************************/
+	/* P8 Header */
+	/************************/
+
+	/* P8_01                GND */
+
+	/* P8_02                GND */
+
+
+	/* P8_03 (ZCZ ball R9) emmc */
+	P8_03_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input";
+		pinctrl-0 = <&P8_03_default_pin>;
+		pinctrl-1 = <&P8_03_gpio_pin>;
+		pinctrl-2 = <&P8_03_gpio_pu_pin>;
+		pinctrl-3 = <&P8_03_gpio_pd_pin>;
+		pinctrl-4 = <&P8_03_gpio_input_pin>;
+	};
+
+	/* P8_04 (ZCZ ball T9) emmc */
+	P8_04_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input";
+		pinctrl-0 = <&P8_04_default_pin>;
+		pinctrl-1 = <&P8_04_gpio_pin>;
+		pinctrl-2 = <&P8_04_gpio_pu_pin>;
+		pinctrl-3 = <&P8_04_gpio_pd_pin>;
+		pinctrl-4 = <&P8_04_gpio_input_pin>;
+	};
+
+	/* P8_05 (ZCZ ball R8) emmc */
+	P8_05_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input";
+		pinctrl-0 = <&P8_05_default_pin>;
+		pinctrl-1 = <&P8_05_gpio_pin>;
+		pinctrl-2 = <&P8_05_gpio_pu_pin>;
+		pinctrl-3 = <&P8_05_gpio_pd_pin>;
+		pinctrl-4 = <&P8_05_gpio_input_pin>;
+	};
+
+	/* P8_06 (ZCZ ball T8) emmc */
+	P8_06_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input";
+		pinctrl-0 = <&P8_06_default_pin>;
+		pinctrl-1 = <&P8_06_gpio_pin>;
+		pinctrl-2 = <&P8_06_gpio_pu_pin>;
+		pinctrl-3 = <&P8_06_gpio_pd_pin>;
+		pinctrl-4 = <&P8_06_gpio_input_pin>;
+	};
+
+	/* P8_07 (ZCZ ball R7) */
+	P8_07_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer";
+		pinctrl-0 = <&P8_07_default_pin>;
+		pinctrl-1 = <&P8_07_gpio_pin>;
+		pinctrl-2 = <&P8_07_gpio_pu_pin>;
+		pinctrl-3 = <&P8_07_gpio_pd_pin>;
+		pinctrl-4 = <&P8_07_gpio_input_pin>;
+		pinctrl-5 = <&P8_07_timer_pin>;
+	};
+
+	/* P8_08 (ZCZ ball T7) */
+	P8_08_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer";
+		pinctrl-0 = <&P8_08_default_pin>;
+		pinctrl-1 = <&P8_08_gpio_pin>;
+		pinctrl-2 = <&P8_08_gpio_pu_pin>;
+		pinctrl-3 = <&P8_08_gpio_pd_pin>;
+		pinctrl-4 = <&P8_08_gpio_input_pin>;
+		pinctrl-5 = <&P8_08_timer_pin>;
+	};
+
+	/* P8_09 (ZCZ ball T6) */
+	P8_09_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer";
+		pinctrl-0 = <&P8_09_default_pin>;
+		pinctrl-1 = <&P8_09_gpio_pin>;
+		pinctrl-2 = <&P8_09_gpio_pu_pin>;
+		pinctrl-3 = <&P8_09_gpio_pd_pin>;
+		pinctrl-4 = <&P8_09_gpio_input_pin>;
+		pinctrl-5 = <&P8_09_timer_pin>;
+	};
+
+	/* P8_10 (ZCZ ball U6) */
+	P8_10_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer";
+		pinctrl-0 = <&P8_10_default_pin>;
+		pinctrl-1 = <&P8_10_gpio_pin>;
+		pinctrl-2 = <&P8_10_gpio_pu_pin>;
+		pinctrl-3 = <&P8_10_gpio_pd_pin>;
+		pinctrl-4 = <&P8_10_gpio_input_pin>;
+		pinctrl-5 = <&P8_10_timer_pin>;
+	};
+
+	/* P8_11 (ZCZ ball R12) */
+	P8_11_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout";
+		pinctrl-0 = <&P8_11_default_pin>;
+		pinctrl-1 = <&P8_11_gpio_pin>;
+		pinctrl-2 = <&P8_11_gpio_pu_pin>;
+		pinctrl-3 = <&P8_11_gpio_pd_pin>;
+		pinctrl-4 = <&P8_11_gpio_input_pin>;
+		pinctrl-5 = <&P8_11_qep_pin>;
+		pinctrl-6 = <&P8_11_pruout_pin>;
+	};
+
+	/* P8_12 (ZCZ ball T12) */
+	P8_12_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout";
+		pinctrl-0 = <&P8_12_default_pin>;
+		pinctrl-1 = <&P8_12_gpio_pin>;
+		pinctrl-2 = <&P8_12_gpio_pu_pin>;
+		pinctrl-3 = <&P8_12_gpio_pd_pin>;
+		pinctrl-4 = <&P8_12_gpio_input_pin>;
+		pinctrl-5 = <&P8_12_qep_pin>;
+		pinctrl-6 = <&P8_12_pruout_pin>;
+	};
+
+	/* P8_13 (ZCZ ball T10) */
+	P8_13_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm";
+		pinctrl-0 = <&P8_13_default_pin>;
+		pinctrl-1 = <&P8_13_gpio_pin>;
+		pinctrl-2 = <&P8_13_gpio_pu_pin>;
+		pinctrl-3 = <&P8_13_gpio_pd_pin>;
+		pinctrl-4 = <&P8_13_gpio_input_pin>;
+		pinctrl-5 = <&P8_13_pwm_pin>;
+	};
+
+	/* P8_14 (ZCZ ball T11) wl1835: wl_en */
+
+	/* P8_15 (ZCZ ball U13) */
+	P8_15_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pru_ecap", "pruin";
+		pinctrl-0 = <&P8_15_default_pin>;
+		pinctrl-1 = <&P8_15_gpio_pin>;
+		pinctrl-2 = <&P8_15_gpio_pu_pin>;
+		pinctrl-3 = <&P8_15_gpio_pd_pin>;
+		pinctrl-4 = <&P8_15_gpio_input_pin>;
+		pinctrl-5 = <&P8_15_qep_pin>;
+		pinctrl-6 = <&P8_15_pru_ecap_pin>;
+		pinctrl-7 = <&P8_15_pruin_pin>;
+	};
+
+	/* P8_16 (ZCZ ball V13) */
+	P8_16_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruin";
+		pinctrl-0 = <&P8_16_default_pin>;
+		pinctrl-1 = <&P8_16_gpio_pin>;
+		pinctrl-2 = <&P8_16_gpio_pu_pin>;
+		pinctrl-3 = <&P8_16_gpio_pd_pin>;
+		pinctrl-4 = <&P8_16_gpio_input_pin>;
+		pinctrl-5 = <&P8_16_qep_pin>;
+		pinctrl-6 = <&P8_16_pruin_pin>;
+	};
+
+	/* P8_17 (ZCZ ball U12) wl1835: wl_irq */
+
+	/* P8_18 (ZCZ ball V12) */
+	P8_18_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input";
+		pinctrl-0 = <&P8_18_default_pin>;
+		pinctrl-1 = <&P8_18_gpio_pin>;
+		pinctrl-2 = <&P8_18_gpio_pu_pin>;
+		pinctrl-3 = <&P8_18_gpio_pd_pin>;
+		pinctrl-4 = <&P8_18_gpio_input_pin>;
+	};
+
+	/* P8_19 (ZCZ ball U10) */
+	P8_19_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm";
+		pinctrl-0 = <&P8_19_default_pin>;
+		pinctrl-1 = <&P8_19_gpio_pin>;
+		pinctrl-2 = <&P8_19_gpio_pu_pin>;
+		pinctrl-3 = <&P8_19_gpio_pd_pin>;
+		pinctrl-4 = <&P8_19_gpio_input_pin>;
+		pinctrl-5 = <&P8_19_pwm_pin>;
+	};
+
+	/* P8_20 (ZCZ ball V9) emmc */
+	P8_20_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin";
+		pinctrl-0 = <&P8_20_default_pin>;
+		pinctrl-1 = <&P8_20_gpio_pin>;
+		pinctrl-2 = <&P8_20_gpio_pu_pin>;
+		pinctrl-3 = <&P8_20_gpio_pd_pin>;
+		pinctrl-4 = <&P8_20_gpio_input_pin>;
+		pinctrl-5 = <&P8_20_pruout_pin>;
+		pinctrl-6 = <&P8_20_pruin_pin>;
+	};
+
+	/* P8_21 (ZCZ ball U9) emmc */
+	P8_21_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin";
+		pinctrl-0 = <&P8_21_default_pin>;
+		pinctrl-1 = <&P8_21_gpio_pin>;
+		pinctrl-2 = <&P8_21_gpio_pu_pin>;
+		pinctrl-3 = <&P8_21_gpio_pd_pin>;
+		pinctrl-4 = <&P8_21_gpio_input_pin>;
+		pinctrl-5 = <&P8_21_pruout_pin>;
+		pinctrl-6 = <&P8_21_pruin_pin>;
+	};
+
+	/* P8_22 (ZCZ ball V8) emmc */
+	P8_22_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input";
+		pinctrl-0 = <&P8_22_default_pin>;
+		pinctrl-1 = <&P8_22_gpio_pin>;
+		pinctrl-2 = <&P8_22_gpio_pu_pin>;
+		pinctrl-3 = <&P8_22_gpio_pd_pin>;
+		pinctrl-4 = <&P8_22_gpio_input_pin>;
+	};
+
+	/* P8_23 (ZCZ ball U8) emmc */
+	P8_23_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input";
+		pinctrl-0 = <&P8_23_default_pin>;
+		pinctrl-1 = <&P8_23_gpio_pin>;
+		pinctrl-2 = <&P8_23_gpio_pu_pin>;
+		pinctrl-3 = <&P8_23_gpio_pd_pin>;
+		pinctrl-4 = <&P8_23_gpio_input_pin>;
+	};
+
+	/* P8_24 (ZCZ ball V7) emmc */
+	P8_24_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input";
+		pinctrl-0 = <&P8_24_default_pin>;
+		pinctrl-1 = <&P8_24_gpio_pin>;
+		pinctrl-2 = <&P8_24_gpio_pu_pin>;
+		pinctrl-3 = <&P8_24_gpio_pd_pin>;
+		pinctrl-4 = <&P8_24_gpio_input_pin>;
+	};
+
+	/* P8_25 (ZCZ ball U7) emmc */
+	P8_25_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input";
+		pinctrl-0 = <&P8_25_default_pin>;
+		pinctrl-1 = <&P8_25_gpio_pin>;
+		pinctrl-2 = <&P8_25_gpio_pu_pin>;
+		pinctrl-3 = <&P8_25_gpio_pd_pin>;
+		pinctrl-4 = <&P8_25_gpio_input_pin>;
+	};
+
+	/* P8_26 (ZCZ ball V6) gpio-hog wl1835 */
+
+	/* P8_27 (ZCZ ball U5) hdmi */
+	P8_27_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin";
+		pinctrl-0 = <&P8_27_default_pin>;
+		pinctrl-1 = <&P8_27_gpio_pin>;
+		pinctrl-2 = <&P8_27_gpio_pu_pin>;
+		pinctrl-3 = <&P8_27_gpio_pd_pin>;
+		pinctrl-4 = <&P8_27_gpio_input_pin>;
+		pinctrl-5 = <&P8_27_pruout_pin>;
+		pinctrl-6 = <&P8_27_pruin_pin>;
+	};
+
+	/* P8_28 (ZCZ ball V5) hdmi */
+	P8_28_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin";
+		pinctrl-0 = <&P8_28_default_pin>;
+		pinctrl-1 = <&P8_28_gpio_pin>;
+		pinctrl-2 = <&P8_28_gpio_pu_pin>;
+		pinctrl-3 = <&P8_28_gpio_pd_pin>;
+		pinctrl-4 = <&P8_28_gpio_input_pin>;
+		pinctrl-5 = <&P8_28_pruout_pin>;
+		pinctrl-6 = <&P8_28_pruin_pin>;
+	};
+
+	/* P8_29 (ZCZ ball R5) hdmi */
+	P8_29_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin";
+		pinctrl-0 = <&P8_29_default_pin>;
+		pinctrl-1 = <&P8_29_gpio_pin>;
+		pinctrl-2 = <&P8_29_gpio_pu_pin>;
+		pinctrl-3 = <&P8_29_gpio_pd_pin>;
+		pinctrl-4 = <&P8_29_gpio_input_pin>;
+		pinctrl-5 = <&P8_29_pruout_pin>;
+		pinctrl-6 = <&P8_29_pruin_pin>;
+	};
+
+	/* P8_30 (ZCZ ball R6) hdmi */
+	P8_30_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin";
+		pinctrl-0 = <&P8_30_default_pin>;
+		pinctrl-1 = <&P8_30_gpio_pin>;
+		pinctrl-2 = <&P8_30_gpio_pu_pin>;
+		pinctrl-3 = <&P8_30_gpio_pd_pin>;
+		pinctrl-4 = <&P8_30_gpio_input_pin>;
+		pinctrl-5 = <&P8_30_pruout_pin>;
+		pinctrl-6 = <&P8_30_pruin_pin>;
+	};
+
+	/* P8_31 (ZCZ ball V4) hdmi */
+	P8_31_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "qep";
+		pinctrl-0 = <&P8_31_default_pin>;
+		pinctrl-1 = <&P8_31_gpio_pin>;
+		pinctrl-2 = <&P8_31_gpio_pu_pin>;
+		pinctrl-3 = <&P8_31_gpio_pd_pin>;
+		pinctrl-4 = <&P8_31_gpio_input_pin>;
+		pinctrl-5 = <&P8_31_uart_pin>;
+		pinctrl-6 = <&P8_31_qep_pin>;
+	};
+
+	/* P8_32 (ZCZ ball T5) hdmi */
+	P8_32_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep";
+		pinctrl-0 = <&P8_32_default_pin>;
+		pinctrl-1 = <&P8_32_gpio_pin>;
+		pinctrl-2 = <&P8_32_gpio_pu_pin>;
+		pinctrl-3 = <&P8_32_gpio_pd_pin>;
+		pinctrl-4 = <&P8_32_gpio_input_pin>;
+		pinctrl-5 = <&P8_32_qep_pin>;
+	};
+
+	/* P8_33 (ZCZ ball V3) hdmi */
+	P8_33_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep";
+		pinctrl-0 = <&P8_33_default_pin>;
+		pinctrl-1 = <&P8_33_gpio_pin>;
+		pinctrl-2 = <&P8_33_gpio_pu_pin>;
+		pinctrl-3 = <&P8_33_gpio_pd_pin>;
+		pinctrl-4 = <&P8_33_gpio_input_pin>;
+		pinctrl-5 = <&P8_33_qep_pin>;
+	};
+
+	/* P8_34 (ZCZ ball U4) hdmi */
+	P8_34_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm";
+		pinctrl-0 = <&P8_34_default_pin>;
+		pinctrl-1 = <&P8_34_gpio_pin>;
+		pinctrl-2 = <&P8_34_gpio_pu_pin>;
+		pinctrl-3 = <&P8_34_gpio_pd_pin>;
+		pinctrl-4 = <&P8_34_gpio_input_pin>;
+		pinctrl-5 = <&P8_34_pwm_pin>;
+	};
+
+	/* P8_35 (ZCZ ball V2) hdmi */
+	P8_35_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep";
+		pinctrl-0 = <&P8_35_default_pin>;
+		pinctrl-1 = <&P8_35_gpio_pin>;
+		pinctrl-2 = <&P8_35_gpio_pu_pin>;
+		pinctrl-3 = <&P8_35_gpio_pd_pin>;
+		pinctrl-4 = <&P8_35_gpio_input_pin>;
+		pinctrl-5 = <&P8_35_qep_pin>;
+	};
+
+	/* P8_36 (ZCZ ball U3) hdmi */
+	P8_36_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm";
+		pinctrl-0 = <&P8_36_default_pin>;
+		pinctrl-1 = <&P8_36_gpio_pin>;
+		pinctrl-2 = <&P8_36_gpio_pu_pin>;
+		pinctrl-3 = <&P8_36_gpio_pd_pin>;
+		pinctrl-4 = <&P8_36_gpio_input_pin>;
+		pinctrl-5 = <&P8_36_pwm_pin>;
+	};
+
+	/* P8_37 (ZCZ ball U1) hdmi */
+	P8_37_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "pwm";
+		pinctrl-0 = <&P8_37_default_pin>;
+		pinctrl-1 = <&P8_37_gpio_pin>;
+		pinctrl-2 = <&P8_37_gpio_pu_pin>;
+		pinctrl-3 = <&P8_37_gpio_pd_pin>;
+		pinctrl-4 = <&P8_37_gpio_input_pin>;
+		pinctrl-5 = <&P8_37_uart_pin>;
+		pinctrl-6 = <&P8_37_pwm_pin>;
+	};
+
+	/* P8_38 (ZCZ ball U2) hdmi */
+	P8_38_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "pwm";
+		pinctrl-0 = <&P8_38_default_pin>;
+		pinctrl-1 = <&P8_38_gpio_pin>;
+		pinctrl-2 = <&P8_38_gpio_pu_pin>;
+		pinctrl-3 = <&P8_38_gpio_pd_pin>;
+		pinctrl-4 = <&P8_38_gpio_input_pin>;
+		pinctrl-5 = <&P8_38_uart_pin>;
+		pinctrl-6 = <&P8_38_pwm_pin>;
+	};
+
+	/* P8_39 (ZCZ ball T3) hdmi */
+	P8_39_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin";
+		pinctrl-0 = <&P8_39_default_pin>;
+		pinctrl-1 = <&P8_39_gpio_pin>;
+		pinctrl-2 = <&P8_39_gpio_pu_pin>;
+		pinctrl-3 = <&P8_39_gpio_pd_pin>;
+		pinctrl-4 = <&P8_39_gpio_input_pin>;
+		pinctrl-5 = <&P8_39_qep_pin>;
+		pinctrl-6 = <&P8_39_pruout_pin>;
+		pinctrl-7 = <&P8_39_pruin_pin>;
+	};
+
+	/* P8_40 (ZCZ ball T4) hdmi */
+	P8_40_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin";
+		pinctrl-0 = <&P8_40_default_pin>;
+		pinctrl-1 = <&P8_40_gpio_pin>;
+		pinctrl-2 = <&P8_40_gpio_pu_pin>;
+		pinctrl-3 = <&P8_40_gpio_pd_pin>;
+		pinctrl-4 = <&P8_40_gpio_input_pin>;
+		pinctrl-5 = <&P8_40_qep_pin>;
+		pinctrl-6 = <&P8_40_pruout_pin>;
+		pinctrl-7 = <&P8_40_pruin_pin>;
+	};
+
+	/* P8_41 (ZCZ ball T1) hdmi */
+	P8_41_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin";
+		pinctrl-0 = <&P8_41_default_pin>;
+		pinctrl-1 = <&P8_41_gpio_pin>;
+		pinctrl-2 = <&P8_41_gpio_pu_pin>;
+		pinctrl-3 = <&P8_41_gpio_pd_pin>;
+		pinctrl-4 = <&P8_41_gpio_input_pin>;
+		pinctrl-5 = <&P8_41_qep_pin>;
+		pinctrl-6 = <&P8_41_pruout_pin>;
+		pinctrl-7 = <&P8_41_pruin_pin>;
+	};
+
+	/* P8_42 (ZCZ ball T2) hdmi */
+	P8_42_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin";
+		pinctrl-0 = <&P8_42_default_pin>;
+		pinctrl-1 = <&P8_42_gpio_pin>;
+		pinctrl-2 = <&P8_42_gpio_pu_pin>;
+		pinctrl-3 = <&P8_42_gpio_pd_pin>;
+		pinctrl-4 = <&P8_42_gpio_input_pin>;
+		pinctrl-5 = <&P8_42_qep_pin>;
+		pinctrl-6 = <&P8_42_pruout_pin>;
+		pinctrl-7 = <&P8_42_pruin_pin>;
+	};
+
+	/* P8_43 (ZCZ ball R3) hdmi */
+	P8_43_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin";
+		pinctrl-0 = <&P8_43_default_pin>;
+		pinctrl-1 = <&P8_43_gpio_pin>;
+		pinctrl-2 = <&P8_43_gpio_pu_pin>;
+		pinctrl-3 = <&P8_43_gpio_pd_pin>;
+		pinctrl-4 = <&P8_43_gpio_input_pin>;
+		pinctrl-5 = <&P8_43_pwm_pin>;
+		pinctrl-6 = <&P8_43_pruout_pin>;
+		pinctrl-7 = <&P8_43_pruin_pin>;
+	};
+
+	/* P8_44 (ZCZ ball R4) hdmi */
+	P8_44_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin";
+		pinctrl-0 = <&P8_44_default_pin>;
+		pinctrl-1 = <&P8_44_gpio_pin>;
+		pinctrl-2 = <&P8_44_gpio_pu_pin>;
+		pinctrl-3 = <&P8_44_gpio_pd_pin>;
+		pinctrl-4 = <&P8_44_gpio_input_pin>;
+		pinctrl-5 = <&P8_44_pwm_pin>;
+		pinctrl-6 = <&P8_44_pruout_pin>;
+		pinctrl-7 = <&P8_44_pruin_pin>;
+	};
+
+	/* P8_45 (ZCZ ball R1) hdmi */
+	P8_45_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin";
+		pinctrl-0 = <&P8_45_default_pin>;
+		pinctrl-1 = <&P8_45_gpio_pin>;
+		pinctrl-2 = <&P8_45_gpio_pu_pin>;
+		pinctrl-3 = <&P8_45_gpio_pd_pin>;
+		pinctrl-4 = <&P8_45_gpio_input_pin>;
+		pinctrl-5 = <&P8_45_pwm_pin>;
+		pinctrl-6 = <&P8_45_pruout_pin>;
+		pinctrl-7 = <&P8_45_pruin_pin>;
+	};
+
+	/* P8_46 (ZCZ ball R2) hdmi */
+	P8_46_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin";
+		pinctrl-0 = <&P8_46_default_pin>;
+		pinctrl-1 = <&P8_46_gpio_pin>;
+		pinctrl-2 = <&P8_46_gpio_pu_pin>;
+		pinctrl-3 = <&P8_46_gpio_pd_pin>;
+		pinctrl-4 = <&P8_46_gpio_input_pin>;
+		pinctrl-5 = <&P8_46_pwm_pin>;
+		pinctrl-6 = <&P8_46_pruout_pin>;
+		pinctrl-7 = <&P8_46_pruin_pin>;
+	};
+
+	/************************/
+	/* P9 Header */
+	/************************/
+
+	/* P9_01                GND */
+
+	/* P9_02                GND */
+
+	/* P9_03                3V3 */
+
+	/* P9_04                3V3 */
+
+	/* P9_05                VDD_5V */
+
+	/* P9_06                VDD_5V */
+
+	/* P9_07                SYS_5V */
+
+	/* P9_08                SYS_5V */
+
+	/* P9_09                PWR_BUT */
+
+	/* P9_10                RSTn */
+
+	/* P9_11 (ZCZ ball T17) */
+	P9_11_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart";
+		pinctrl-0 = <&P9_11_default_pin>;
+		pinctrl-1 = <&P9_11_gpio_pin>;
+		pinctrl-2 = <&P9_11_gpio_pu_pin>;
+		pinctrl-3 = <&P9_11_gpio_pd_pin>;
+		pinctrl-4 = <&P9_11_gpio_input_pin>;
+		pinctrl-5 = <&P9_11_uart_pin>;
+	};
+
+	/* P9_12 (ZCZ ball U18) */
+	P9_12_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input";
+		pinctrl-0 = <&P9_12_default_pin>;
+		pinctrl-1 = <&P9_12_gpio_pin>;
+		pinctrl-2 = <&P9_12_gpio_pu_pin>;
+		pinctrl-3 = <&P9_12_gpio_pd_pin>;
+		pinctrl-4 = <&P9_12_gpio_input_pin>;
+	};
+
+	/* P9_13 (ZCZ ball U17) */
+	P9_13_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart";
+		pinctrl-0 = <&P9_13_default_pin>;
+		pinctrl-1 = <&P9_13_gpio_pin>;
+		pinctrl-2 = <&P9_13_gpio_pu_pin>;
+		pinctrl-3 = <&P9_13_gpio_pd_pin>;
+		pinctrl-4 = <&P9_13_gpio_input_pin>;
+		pinctrl-5 = <&P9_13_uart_pin>;
+	};
+
+	/* P9_14 (ZCZ ball U14) */
+	P9_14_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm";
+		pinctrl-0 = <&P9_14_default_pin>;
+		pinctrl-1 = <&P9_14_gpio_pin>;
+		pinctrl-2 = <&P9_14_gpio_pu_pin>;
+		pinctrl-3 = <&P9_14_gpio_pd_pin>;
+		pinctrl-4 = <&P9_14_gpio_input_pin>;
+		pinctrl-5 = <&P9_14_pwm_pin>;
+	};
+
+	/* P9_15 (ZCZ ball R13) */
+	P9_15_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm";
+		pinctrl-0 = <&P9_15_default_pin>;
+		pinctrl-1 = <&P9_15_gpio_pin>;
+		pinctrl-2 = <&P9_15_gpio_pu_pin>;
+		pinctrl-3 = <&P9_15_gpio_pd_pin>;
+		pinctrl-4 = <&P9_15_gpio_input_pin>;
+		pinctrl-5 = <&P9_15_pwm_pin>;
+	};
+
+	/* P9_16 (ZCZ ball T14) */
+	P9_16_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm";
+		pinctrl-0 = <&P9_16_default_pin>;
+		pinctrl-1 = <&P9_16_gpio_pin>;
+		pinctrl-2 = <&P9_16_gpio_pu_pin>;
+		pinctrl-3 = <&P9_16_gpio_pd_pin>;
+		pinctrl-4 = <&P9_16_gpio_input_pin>;
+		pinctrl-5 = <&P9_16_pwm_pin>;
+	};
+
+	/* P9_17 (ZCZ ball A16) */
+	P9_17_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "i2c", "pwm", "pru_uart";
+		pinctrl-0 = <&P9_17_default_pin>;
+		pinctrl-1 = <&P9_17_gpio_pin>;
+		pinctrl-2 = <&P9_17_gpio_pu_pin>;
+		pinctrl-3 = <&P9_17_gpio_pd_pin>;
+		pinctrl-4 = <&P9_17_gpio_input_pin>;
+		pinctrl-5 = <&P9_17_spi_cs_pin>;
+		pinctrl-6 = <&P9_17_i2c_pin>;
+		pinctrl-7 = <&P9_17_pwm_pin>;
+		pinctrl-8 = <&P9_17_pru_uart_pin>;
+	};
+
+	/* P9_18 (ZCZ ball B16) */
+	P9_18_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "i2c", "pwm", "pru_uart";
+		pinctrl-0 = <&P9_18_default_pin>;
+		pinctrl-1 = <&P9_18_gpio_pin>;
+		pinctrl-2 = <&P9_18_gpio_pu_pin>;
+		pinctrl-3 = <&P9_18_gpio_pd_pin>;
+		pinctrl-4 = <&P9_18_gpio_input_pin>;
+		pinctrl-5 = <&P9_18_spi_pin>;
+		pinctrl-6 = <&P9_18_i2c_pin>;
+		pinctrl-7 = <&P9_18_pwm_pin>;
+		pinctrl-8 = <&P9_18_pru_uart_pin>;
+	};
+
+	/* P9_19 (ZCZ ball D17) i2c */
+	P9_19_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart", "timer";
+		pinctrl-0 = <&P9_19_default_pin>;
+		pinctrl-1 = <&P9_19_gpio_pin>;
+		pinctrl-2 = <&P9_19_gpio_pu_pin>;
+		pinctrl-3 = <&P9_19_gpio_pd_pin>;
+		pinctrl-4 = <&P9_19_gpio_input_pin>;
+		pinctrl-5 = <&P9_19_spi_cs_pin>;
+		pinctrl-6 = <&P9_19_can_pin>;
+		pinctrl-7 = <&P9_19_i2c_pin>;
+		pinctrl-8 = <&P9_19_pru_uart_pin>;
+		pinctrl-9 = <&P9_19_timer_pin>;
+	};
+
+	/* P9_20 (ZCZ ball D18) i2c */
+	P9_20_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart", "timer";
+		pinctrl-0 = <&P9_20_default_pin>;
+		pinctrl-1 = <&P9_20_gpio_pin>;
+		pinctrl-2 = <&P9_20_gpio_pu_pin>;
+		pinctrl-3 = <&P9_20_gpio_pd_pin>;
+		pinctrl-4 = <&P9_20_gpio_input_pin>;
+		pinctrl-5 = <&P9_20_spi_cs_pin>;
+		pinctrl-6 = <&P9_20_can_pin>;
+		pinctrl-7 = <&P9_20_i2c_pin>;
+		pinctrl-8 = <&P9_20_pru_uart_pin>;
+		pinctrl-9 = <&P9_20_timer_pin>;
+	};
+
+	/* P9_21 (ZCZ ball B17) */
+	P9_21_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "uart", "i2c", "pwm", "pru_uart";
+		pinctrl-0 = <&P9_21_default_pin>;
+		pinctrl-1 = <&P9_21_gpio_pin>;
+		pinctrl-2 = <&P9_21_gpio_pu_pin>;
+		pinctrl-3 = <&P9_21_gpio_pd_pin>;
+		pinctrl-4 = <&P9_21_gpio_input_pin>;
+		pinctrl-5 = <&P9_21_spi_pin>;
+		pinctrl-6 = <&P9_21_uart_pin>;
+		pinctrl-7 = <&P9_21_i2c_pin>;
+		pinctrl-8 = <&P9_21_pwm_pin>;
+		pinctrl-9 = <&P9_21_pru_uart_pin>;
+	};
+
+	/* P9_22 (ZCZ ball A17) */
+	P9_22_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_sclk", "uart", "i2c", "pwm", "pru_uart";
+		pinctrl-0 = <&P9_22_default_pin>;
+		pinctrl-1 = <&P9_22_gpio_pin>;
+		pinctrl-2 = <&P9_22_gpio_pu_pin>;
+		pinctrl-3 = <&P9_22_gpio_pd_pin>;
+		pinctrl-4 = <&P9_22_gpio_input_pin>;
+		pinctrl-5 = <&P9_22_spi_sclk_pin>;
+		pinctrl-6 = <&P9_22_uart_pin>;
+		pinctrl-7 = <&P9_22_i2c_pin>;
+		pinctrl-8 = <&P9_22_pwm_pin>;
+		pinctrl-9 = <&P9_22_pru_uart_pin>;
+	};
+
+	/* P9_23 (ZCZ ball V14) */
+	P9_23_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm";
+		pinctrl-0 = <&P9_23_default_pin>;
+		pinctrl-1 = <&P9_23_gpio_pin>;
+		pinctrl-2 = <&P9_23_gpio_pu_pin>;
+		pinctrl-3 = <&P9_23_gpio_pd_pin>;
+		pinctrl-4 = <&P9_23_gpio_input_pin>;
+		pinctrl-5 = <&P9_23_pwm_pin>;
+	};
+
+	/* P9_24 (ZCZ ball D15) */
+	P9_24_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "can", "i2c", "pru_uart", "pruin";
+		pinctrl-0 = <&P9_24_default_pin>;
+		pinctrl-1 = <&P9_24_gpio_pin>;
+		pinctrl-2 = <&P9_24_gpio_pu_pin>;
+		pinctrl-3 = <&P9_24_gpio_pd_pin>;
+		pinctrl-4 = <&P9_24_gpio_input_pin>;
+		pinctrl-5 = <&P9_24_uart_pin>;
+		pinctrl-6 = <&P9_24_can_pin>;
+		pinctrl-7 = <&P9_24_i2c_pin>;
+		pinctrl-8 = <&P9_24_pru_uart_pin>;
+		pinctrl-9 = <&P9_24_pruin_pin>;
+	};
+
+	/* P9_25 (ZCZ ball A14) audio */
+	P9_25_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin";
+		pinctrl-0 = <&P9_25_default_pin>;
+		pinctrl-1 = <&P9_25_gpio_pin>;
+		pinctrl-2 = <&P9_25_gpio_pu_pin>;
+		pinctrl-3 = <&P9_25_gpio_pd_pin>;
+		pinctrl-4 = <&P9_25_gpio_input_pin>;
+		pinctrl-5 = <&P9_25_qep_pin>;
+		pinctrl-6 = <&P9_25_pruout_pin>;
+		pinctrl-7 = <&P9_25_pruin_pin>;
+	};
+
+	/* P9_26 (ZCZ ball D16) */
+	P9_26_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "can", "i2c", "pru_uart", "pruin";
+		pinctrl-0 = <&P9_26_default_pin>;
+		pinctrl-1 = <&P9_26_gpio_pin>;
+		pinctrl-2 = <&P9_26_gpio_pu_pin>;
+		pinctrl-3 = <&P9_26_gpio_pd_pin>;
+		pinctrl-4 = <&P9_26_gpio_input_pin>;
+		pinctrl-5 = <&P9_26_uart_pin>;
+		pinctrl-6 = <&P9_26_can_pin>;
+		pinctrl-7 = <&P9_26_i2c_pin>;
+		pinctrl-8 = <&P9_26_pru_uart_pin>;
+		pinctrl-9 = <&P9_26_pruin_pin>;
+	};
+
+	/* P9_27 (ZCZ ball C13) */
+	P9_27_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin";
+		pinctrl-0 = <&P9_27_default_pin>;
+		pinctrl-1 = <&P9_27_gpio_pin>;
+		pinctrl-2 = <&P9_27_gpio_pu_pin>;
+		pinctrl-3 = <&P9_27_gpio_pd_pin>;
+		pinctrl-4 = <&P9_27_gpio_input_pin>;
+		pinctrl-5 = <&P9_27_qep_pin>;
+		pinctrl-6 = <&P9_27_pruout_pin>;
+		pinctrl-7 = <&P9_27_pruin_pin>;
+	};
+
+	/* P9_28 (ZCZ ball C12) audio */
+	P9_28_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "pwm", "pwm2", "pruout", "pruin";
+		pinctrl-0 = <&P9_28_default_pin>;
+		pinctrl-1 = <&P9_28_gpio_pin>;
+		pinctrl-2 = <&P9_28_gpio_pu_pin>;
+		pinctrl-3 = <&P9_28_gpio_pd_pin>;
+		pinctrl-4 = <&P9_28_gpio_input_pin>;
+		pinctrl-5 = <&P9_28_spi_cs_pin>;
+		pinctrl-6 = <&P9_28_pwm_pin>;
+		pinctrl-7 = <&P9_28_pwm2_pin>;
+		pinctrl-8 = <&P9_28_pruout_pin>;
+		pinctrl-9 = <&P9_28_pruin_pin>;
+	};
+
+	/* P9_29 (ZCZ ball B13) audio */
+	P9_29_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "pwm", "pruout", "pruin";
+		pinctrl-0 = <&P9_29_default_pin>;
+		pinctrl-1 = <&P9_29_gpio_pin>;
+		pinctrl-2 = <&P9_29_gpio_pu_pin>;
+		pinctrl-3 = <&P9_29_gpio_pd_pin>;
+		pinctrl-4 = <&P9_29_gpio_input_pin>;
+		pinctrl-5 = <&P9_29_spi_pin>;
+		pinctrl-6 = <&P9_29_pwm_pin>;
+		pinctrl-7 = <&P9_29_pruout_pin>;
+		pinctrl-8 = <&P9_29_pruin_pin>;
+	};
+
+	/* P9_30 (ZCZ ball D12) gpio-hog wl1835 */
+
+	/* P9_31 (ZCZ ball A13) audio */
+	P9_31_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_sclk", "pwm", "pruout", "pruin";
+		pinctrl-0 = <&P9_31_default_pin>;
+		pinctrl-1 = <&P9_31_gpio_pin>;
+		pinctrl-2 = <&P9_31_gpio_pu_pin>;
+		pinctrl-3 = <&P9_31_gpio_pd_pin>;
+		pinctrl-4 = <&P9_31_gpio_input_pin>;
+		pinctrl-5 = <&P9_31_spi_sclk_pin>;
+		pinctrl-6 = <&P9_31_pwm_pin>;
+		pinctrl-7 = <&P9_31_pruout_pin>;
+		pinctrl-8 = <&P9_31_pruin_pin>;
+	};
+
+	/* P9_32                VADC */
+
+	/* P9_33 (ZCZ ball C8)  AIN4         */
+
+	/* P9_34                AGND */
+
+	/* P9_35 (ZCZ ball A8)  AIN6         */
+
+	/* P9_36 (ZCZ ball B8)  AIN5         */
+
+	/* P9_37 (ZCZ ball B7)  AIN2         */
+
+	/* P9_38 (ZCZ ball A7)  AIN3         */
+
+	/* P9_39 (ZCZ ball B6)  AIN0         */
+
+	/* P9_40 (ZCZ ball C7)  AIN1         */
+
+	/* P9_41 (ZCZ ball D14) */
+	P9_41_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer", "pruin";
+		pinctrl-0 = <&P9_41_default_pin>;
+		pinctrl-1 = <&P9_41_gpio_pin>;
+		pinctrl-2 = <&P9_41_gpio_pu_pin>;
+		pinctrl-3 = <&P9_41_gpio_pd_pin>;
+		pinctrl-4 = <&P9_41_gpio_input_pin>;
+		pinctrl-5 = <&P9_41_timer_pin>;
+		pinctrl-6 = <&P9_41_pruin_pin>;
+	};
+
+	/* P9_41.1 */
+	/* P9_91 (ZCZ ball D13) */
+	P9_91_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin";
+		pinctrl-0 = <&P9_91_default_pin>;
+		pinctrl-1 = <&P9_91_gpio_pin>;
+		pinctrl-2 = <&P9_91_gpio_pu_pin>;
+		pinctrl-3 = <&P9_91_gpio_pd_pin>;
+		pinctrl-4 = <&P9_91_gpio_input_pin>;
+		pinctrl-5 = <&P9_91_qep_pin>;
+		pinctrl-6 = <&P9_91_pruout_pin>;
+		pinctrl-7 = <&P9_91_pruin_pin>;
+	};
+
+	/* P9_42 (ZCZ ball C18) */
+	P9_42_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "spi_sclk", "uart", "pwm", "pru_ecap";
+		pinctrl-0 = <&P9_42_default_pin>;
+		pinctrl-1 = <&P9_42_gpio_pin>;
+		pinctrl-2 = <&P9_42_gpio_pu_pin>;
+		pinctrl-3 = <&P9_42_gpio_pd_pin>;
+		pinctrl-4 = <&P9_42_gpio_input_pin>;
+		pinctrl-5 = <&P9_42_spi_cs_pin>;
+		pinctrl-6 = <&P9_42_spi_sclk_pin>;
+		pinctrl-7 = <&P9_42_uart_pin>;
+		pinctrl-8 = <&P9_42_pwm_pin>;
+		pinctrl-9 = <&P9_42_pru_ecap_pin>;
+	};
+
+	/* P9_42.1 */
+	/* P9_92 (ZCZ ball B12) */
+	P9_92_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin";
+		pinctrl-0 = <&P9_92_default_pin>;
+		pinctrl-1 = <&P9_92_gpio_pin>;
+		pinctrl-2 = <&P9_92_gpio_pu_pin>;
+		pinctrl-3 = <&P9_92_gpio_pd_pin>;
+		pinctrl-4 = <&P9_92_gpio_input_pin>;
+		pinctrl-5 = <&P9_92_qep_pin>;
+		pinctrl-6 = <&P9_92_pruout_pin>;
+		pinctrl-7 = <&P9_92_pruin_pin>;
+	};
+
+	/* P9_43                GND */
+
+	/* P9_44                GND */
+
+	/* P9_45                GND */
+
+	/* P9_46                GND */
+
+	/*       (ZCZ ball A15) */
+	A15_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "clkout", "gpio", "gpio_pu", "gpio_pd";
+		pinctrl-0 = <&A15_default_pin>;
+		pinctrl-1 = <&A15_clkout_pin>;
+		pinctrl-2 = <&A15_gpio_pin>;
+		pinctrl-3 = <&A15_gpio_pu_pin>;
+		pinctrl-4 = <&A15_gpio_pd_pin>;
+	};
+
+	cape-universal {
+		compatible = "gpio-of-helper";
+		status = "okay";
+		pinctrl-names = "default";
+		pinctrl-0 = <>;
+
+		P8_03 {
+			gpio-name = "P8_03";
+			gpio = <&gpio1 6 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_04 {
+			gpio-name = "P8_04";
+			gpio = <&gpio1 7 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_05 {
+			gpio-name = "P8_05";
+			gpio = <&gpio1 2 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_06 {
+			gpio-name = "P8_06";
+			gpio = <&gpio1 3 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_07 {
+			gpio-name = "P8_07";
+			gpio = <&gpio2 2 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_08 {
+			gpio-name = "P8_08";
+			gpio = <&gpio2 3 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_09 {
+			gpio-name = "P8_09";
+			gpio = <&gpio2 5 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_10 {
+			gpio-name = "P8_10";
+			gpio = <&gpio2 4 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_11 {
+			gpio-name = "P8_11";
+			gpio = <&gpio1 13 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_12 {
+			gpio-name = "P8_12";
+			gpio = <&gpio1 12 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_13 {
+			gpio-name = "P8_13";
+			gpio = <&gpio0 23 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_15 {
+			gpio-name = "P8_15";
+			gpio = <&gpio1 15 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_16 {
+			gpio-name = "P8_16";
+			gpio = <&gpio1 14 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_18 {
+			gpio-name = "P8_18";
+			gpio = <&gpio2 1 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_19 {
+			gpio-name = "P8_19";
+			gpio = <&gpio0 22 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_20 {
+			gpio-name = "P8_20";
+			gpio = <&gpio1 31 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_21 {
+			gpio-name = "P8_21";
+			gpio = <&gpio1 30 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_22 {
+			gpio-name = "P8_22";
+			gpio = <&gpio1 5 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_23 {
+			gpio-name = "P8_23";
+			gpio = <&gpio1 4 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_24 {
+			gpio-name = "P8_24";
+			gpio = <&gpio1 1 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_25 {
+			gpio-name = "P8_25";
+			gpio = <&gpio1 0 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_27 {
+			gpio-name = "P8_27";
+			gpio = <&gpio2 22 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_28 {
+			gpio-name = "P8_28";
+			gpio = <&gpio2 24 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_29 {
+			gpio-name = "P8_29";
+			gpio = <&gpio2 23 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_30 {
+			gpio-name = "P8_30";
+			gpio = <&gpio2 25 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_31 {
+			gpio-name = "P8_31";
+			gpio = <&gpio0 10 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_32 {
+			gpio-name = "P8_32";
+			gpio = <&gpio0 11 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_33 {
+			gpio-name = "P8_33";
+			gpio = <&gpio0 9 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_34 {
+			gpio-name = "P8_34";
+			gpio = <&gpio2 17 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_35 {
+			gpio-name = "P8_35";
+			gpio = <&gpio0 8 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_36 {
+			gpio-name = "P8_36";
+			gpio = <&gpio2 16 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_37 {
+			gpio-name = "P8_37";
+			gpio = <&gpio2 14 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_38 {
+			gpio-name = "P8_38";
+			gpio = <&gpio2 15 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_39 {
+			gpio-name = "P8_39";
+			gpio = <&gpio2 12 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_40 {
+			gpio-name = "P8_40";
+			gpio = <&gpio2 13 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_41 {
+			gpio-name = "P8_41";
+			gpio = <&gpio2 10 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_42 {
+			gpio-name = "P8_42";
+			gpio = <&gpio2 11 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_43 {
+			gpio-name = "P8_43";
+			gpio = <&gpio2 8 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_44 {
+			gpio-name = "P8_44";
+			gpio = <&gpio2 9 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_45 {
+			gpio-name = "P8_45";
+			gpio = <&gpio2 6 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_46 {
+			gpio-name = "P8_46";
+			gpio = <&gpio2 7 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_11 {
+			gpio-name = "P9_11";
+			gpio = <&gpio0 30 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_12 {
+			gpio-name = "P9_12";
+			gpio = <&gpio1 28 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_13 {
+			gpio-name = "P9_13";
+			gpio = <&gpio0 31 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_14 {
+			gpio-name = "P9_14";
+			gpio = <&gpio1 18 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_15 {
+			gpio-name = "P9_15";
+			gpio = <&gpio1 16 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_16 {
+			gpio-name = "P9_16";
+			gpio = <&gpio1 19 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_17 {
+			gpio-name = "P9_17";
+			gpio = <&gpio0 5 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_18 {
+			gpio-name = "P9_18";
+			gpio = <&gpio0 4 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_19 {
+			gpio-name = "P9_19";
+			gpio = <&gpio0 13 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_20 {
+			gpio-name = "P9_20";
+			gpio = <&gpio0 12 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_21 {
+			gpio-name = "P9_21";
+			gpio = <&gpio0 3 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_22 {
+			gpio-name = "P9_22";
+			gpio = <&gpio0 2 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_23 {
+			gpio-name = "P9_23";
+			gpio = <&gpio1 17 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_24 {
+			gpio-name = "P9_24";
+			gpio = <&gpio0 15 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_25 {
+			gpio-name = "P9_25";
+			gpio = <&gpio3 21 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_26 {
+			gpio-name = "P9_26";
+			gpio = <&gpio0 14 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_27 {
+			gpio-name = "P9_27";
+			gpio = <&gpio3 19 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_28 {
+			gpio-name = "P9_28";
+			gpio = <&gpio3 17 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_29 {
+			gpio-name = "P9_29";
+			gpio = <&gpio3 15 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_31 {
+			gpio-name = "P9_31";
+			gpio = <&gpio3 14 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_41 {
+			gpio-name = "P9_41";
+			gpio = <&gpio0 20 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_91 {
+			gpio-name = "P9_91";
+			gpio = <&gpio3 20 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_42 {
+			gpio-name = "P9_42";
+			gpio = <&gpio0 7 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_92 {
+			gpio-name = "P9_92";
+			gpio = <&gpio3 18 0>;
+			input;
+			dir-changeable;
+		};
+
+		A15 {
+			gpio-name = "A15";
+			gpio = <&gpio0 19 0>;
+			input;
+			dir-changeable;
+		};
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-bonegreen-wireless.dts b/arch/arm/boot/dts/am335x-bonegreen-wireless.dts
--- a/arch/arm/boot/dts/am335x-bonegreen-wireless.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-bonegreen-wireless.dts	2020-06-19 21:00:31.404227715 -0400
@@ -1,9 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 /dts-v1/;
 
@@ -11,11 +8,17 @@
 #include "am335x-bone-common.dtsi"
 #include "am335x-bonegreen-common.dtsi"
 #include <dt-bindings/interrupt-controller/irq.h>
+/* #include "am335x-bone-jtag.dtsi" */
 
 / {
 	model = "TI AM335x BeagleBone Green Wireless";
 	compatible = "ti,am335x-bone-green-wireless", "ti,am335x-bone-green", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx";
 
+	chosen {
+		base_dtb = "am335x-bonegreen-wireless.dts";
+		base_dtb_timestamp = __TIMESTAMP__;
+	};
+
 	wlan_en_reg: fixedregulator@2 {
 		compatible = "regulator-fixed";
 		regulator-name = "wlan-en-regulator";
@@ -29,38 +32,42 @@
 	};
 };
 
+&sgx {
+	status = "okay";
+};
+
 &am33xx_pinmux {
 	bt_pins: pinmux_bt_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x878, PIN_OUTPUT_PULLUP | MUX_MODE7)	/* gpmc_ad12.gpio1_28 BT_EN */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_BEN1, PIN_OUTPUT_PULLUP, MUX_MODE7)	/* gpmc_ad12.gpio1_28 BT_EN */
 		>;
 	};
 
 	mmc3_pins: pinmux_mmc3_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x830, PIN_INPUT_PULLUP | MUX_MODE3)	/* gpmc_ad12.mmc2_dat0 */
-			AM33XX_IOPAD(0x834, PIN_INPUT_PULLUP | MUX_MODE3)	/* gpmc_ad13.mmc2_dat1 */
-			AM33XX_IOPAD(0x838, PIN_INPUT_PULLUP | MUX_MODE3)	/* gpmc_ad14.mmc2_dat2 */
-			AM33XX_IOPAD(0x83c, PIN_INPUT_PULLUP | MUX_MODE3)	/* gpmc_ad15.mmc2_dat3 */
-			AM33XX_IOPAD(0x888, PIN_INPUT_PULLUP | MUX_MODE3)	/* gpmc_csn3.mmc2_cmd */
-			AM33XX_IOPAD(0x88c, PIN_INPUT_PULLUP | MUX_MODE3)	/* gpmc_clk.mmc2_clk */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD12, PIN_INPUT_PULLUP, MUX_MODE3)	/* gpmc_ad12.mmc2_dat0 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD13, PIN_INPUT_PULLUP, MUX_MODE3)	/* gpmc_ad13.mmc2_dat1 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD14, PIN_INPUT_PULLUP, MUX_MODE3)	/* gpmc_ad14.mmc2_dat2 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD15, PIN_INPUT_PULLUP, MUX_MODE3)	/* gpmc_ad15.mmc2_dat3 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_CSN3, PIN_INPUT_PULLUP, MUX_MODE3)	/* gpmc_csn3.mmc2_cmd */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_CLK, PIN_INPUT_PULLUP, MUX_MODE3)	/* gpmc_clk.mmc2_clk */
 		>;
 	};
 
 	uart3_pins: pinmux_uart3_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x934, PIN_INPUT_PULLUP | MUX_MODE1)	/* gmii1_rxd3.uart3_rxd */
-			AM33XX_IOPAD(0x938, PIN_OUTPUT_PULLDOWN | MUX_MODE1)	/* gmii1_rxd2.uart3_txd */
-			AM33XX_IOPAD(0x948, PIN_INPUT | MUX_MODE3)		/* mdio_data.uart3_ctsn */
-			AM33XX_IOPAD(0x94c, PIN_OUTPUT_PULLDOWN | MUX_MODE3)	/* mdio_clk.uart3_rtsn */
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLUP, MUX_MODE1)	/* gmii1_rxd3.uart3_rxd */
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_OUTPUT_PULLDOWN, MUX_MODE1)	/* gmii1_rxd2.uart3_txd */
+			AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT, MUX_MODE3)		/* mdio_data.uart3_ctsn */
+			AM33XX_PADCONF(AM335X_PIN_MDC, PIN_OUTPUT_PULLDOWN, MUX_MODE3)	/* mdio_clk.uart3_rtsn */
 		>;
 	};
 
 	wl18xx_pins: pinmux_wl18xx_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x828, PIN_OUTPUT_PULLDOWN | MUX_MODE7)	/* gpmc_ad10.gpio0_26 WL_EN */
-			AM33XX_IOPAD(0x82C, PIN_INPUT_PULLDOWN | MUX_MODE7)	/* gpmc_ad11.gpio0_27 WL_IRQ */
-			AM33XX_IOPAD(0x87C, PIN_OUTPUT_PULLUP | MUX_MODE7)	/* gpmc_csn0.gpio1_29 LS_BUF_EN */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD10, PIN_OUTPUT_PULLDOWN, MUX_MODE7)	/* gpmc_ad10.gpio0_26 WL_EN */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD11, PIN_INPUT_PULLDOWN, MUX_MODE7)	/* gpmc_ad11.gpio0_27 WL_IRQ */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_CSN0, PIN_OUTPUT_PULLUP, MUX_MODE7)	/* gpmc_csn0.gpio1_29 LS_BUF_EN */
 		>;
 	};
 };
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-bonegreen-wireless-uboot-univ.dts b/arch/arm/boot/dts/am335x-bonegreen-wireless-uboot-univ.dts
--- a/arch/arm/boot/dts/am335x-bonegreen-wireless-uboot-univ.dts	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-bonegreen-wireless-uboot-univ.dts	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,59 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ */
+/dts-v1/;
+
+#include "am33xx.dtsi"
+#include "am335x-bone-common.dtsi"
+#include "am335x-bonegreen-wireless-common-univ.dtsi"
+
+/ {
+	model = "TI AM335x BeagleBone Green Wireless";
+	compatible = "ti,am335x-bone-green-wireless", "ti,am335x-bone-green", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx";
+
+	chosen {
+		base_dtb = "am335x-bonegreen-wireless-uboot-univ.dts";
+		base_dtb_timestamp = __TIMESTAMP__;
+	};
+};
+
+&sgx {
+	status = "okay";
+};
+
+&ldo3_reg {
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-always-on;
+};
+
+&mmc1 {
+	vmmc-supply = <&vmmcsd_fixed>;
+};
+
+&gpio1 {
+	ls_buf_en {
+		gpio-hog;
+		gpios = <29 GPIO_ACTIVE_HIGH>;
+		output-high;
+		line-name = "LS_BUF_EN";
+	};
+};
+
+/* BT_AUD_OUT from wl1835 has to be pulled low when WL_EN is activated.*/
+/* in case it isn't, wilink8 ends up in one of the test modes that     */
+/* intruces various issues (elp wkaeup timeouts etc.)                  */
+/* On the BBGW this pin is routed through the level shifter (U21) that */
+/* introduces a pullup on the line and wilink8 ends up in a bad state. */
+/* use a gpio hog to force this pin low. An alternative may be adding  */
+/* an external pulldown on U21 pin 4.                                  */
+
+&gpio3 {
+	bt_aud_in {
+		gpio-hog;
+		gpios = <16 GPIO_ACTIVE_HIGH>;
+		output-low;
+		line-name = "MCASP0_AHCLKR";
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-bone-jtag.dtsi b/arch/arm/boot/dts/am335x-bone-jtag.dtsi
--- a/arch/arm/boot/dts/am335x-bone-jtag.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-bone-jtag.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2015 Robert Nelson <robertcnelson@gmail.com>
+ */
+
+&am33xx_pinmux {
+	pinctrl-names = "default";
+	pinctrl-0 = <&clkout2_pin>;
+
+	clkout2_pin: pinmux_clkout2_pin {
+		pinctrl-single,pins = <
+			AM33XX_PADCONF(AM335X_PIN_XDMA_EVENT_INTR1, PIN_OUTPUT_PULLDOWN, MUX_MODE3)     /* xdma_event_intr1.clkout2 */
+		>;
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-bone-pinmux-can0.dtsi b/arch/arm/boot/dts/am335x-bone-pinmux-can0.dtsi
--- a/arch/arm/boot/dts/am335x-bone-pinmux-can0.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-bone-pinmux-can0.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <dt-bindings/board/am335x-bbw-bbb-base.h>
+#include "am335x-peripheral-can0.dtsi"
+
+/* cape universal */
+
+/*
+ *&ocp {
+ *	P9_19_pinmux {
+ *		mode = "can";
+ *	};
+ *	P9_20_pinmux {
+ *		mode = "can";
+ *	};
+ *};
+ *
+ *&dcan0 {
+ *	pinctrl-0 = <>;
+ *};
+ *
+ */
+
+/* standard */
+
+&am33xx_pinmux {
+	dcan0_pins: pinmux_dcan0_pins {
+		pinctrl-single,pins = <
+			/* P9_20: uart1_ctsn.d_can0_tx */
+			BONE_P9_20 (PIN_OUTPUT_PULLUP | MUX_MODE2)
+			/* P9_19: uart1_rtsn.d_can0_rx */
+			BONE_P9_19 (PIN_INPUT_PULLUP | MUX_MODE2)
+		>;
+	};
+};
+
+&dcan0 {
+	pinctrl-0 = <&dcan0_pins>;
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-bone-pinmux-panel-1024x600-24bit.dtsi b/arch/arm/boot/dts/am335x-bone-pinmux-panel-1024x600-24bit.dtsi
--- a/arch/arm/boot/dts/am335x-bone-pinmux-panel-1024x600-24bit.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-bone-pinmux-panel-1024x600-24bit.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,151 @@
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <dt-bindings/board/am335x-bbw-bbb-base.h>
+#include "am335x-peripheral-panel-1024x600-24bit.dtsi"
+
+/* cape universal */
+
+/*
+ *&ocp {
+ *	P8_27_pinmux {
+ *		state = "disabled";
+ *	};
+ *	P8_28_pinmux {
+ *		state = "disabled";
+ *	};
+ *	P8_29_pinmux {
+ *		state = "disabled";
+ *	};
+ *	P8_30_pinmux {
+ *		state = "disabled";
+ *	};
+ *	P8_31_pinmux {
+ *		state = "disabled";
+ *	};
+ *	P8_32_pinmux {
+ *		state = "disabled";
+ *	};
+ *	P8_33_pinmux {
+ *		state = "disabled";
+ *	};
+ *	P8_34_pinmux {
+ *		state = "disabled";
+ *	};
+ *	P8_35_pinmux {
+ *		state = "disabled";
+ *	};
+ *	P8_36_pinmux {
+ *		state = "disabled";
+ *	};
+ *	P8_37_pinmux {
+ *		state = "disabled";
+ *	};
+ *	P8_38_pinmux {
+ *		state = "disabled";
+ *	};
+ *	P8_39_pinmux {
+ *		state = "disabled";
+ *	};
+ *	P8_40_pinmux {
+ *		state = "disabled";
+ *	};
+ *	P8_41_pinmux {
+ *		state = "disabled";
+ *	};
+ *	P8_42_pinmux {
+ *		state = "disabled";
+ *	};
+ *	P8_43_pinmux {
+ *		state = "disabled";
+ *	};
+ *	P8_44_pinmux {
+ *		state = "disabled";
+ *	};
+ *	P8_45_pinmux {
+ *		state = "disabled";
+ *	};
+ *	P8_46_pinmux {
+ *		state = "disabled";
+ *	};
+ *};
+ */
+
+/* standard */
+
+&am33xx_pinmux {
+	lcd_24bit_pins: pinmux_lcd_24bit_pins {
+		pinctrl-single,pins = <
+
+			/* P8_45: lcd_data0.lcd_data0 */
+			BONE_P8_45 (PIN_OUTPUT | MUX_MODE0)
+			/* P8_46: lcd_data1.lcd_data1 */
+			BONE_P8_46 (PIN_OUTPUT | MUX_MODE0)
+			/* P8_43: lcd_data2.lcd_data2 */
+			BONE_P8_43 (PIN_OUTPUT | MUX_MODE0)
+			/* P8_44: lcd_data3.lcd_data3 */
+			BONE_P8_44 (PIN_OUTPUT | MUX_MODE0)
+			/* P8_41: lcd_data4.lcd_data4 */
+			BONE_P8_41 (PIN_OUTPUT | MUX_MODE0)
+			/* P8_42: lcd_data5.lcd_data5 */
+			BONE_P8_42 (PIN_OUTPUT | MUX_MODE0)
+			/* P8_39: lcd_data6.lcd_data6 */
+			BONE_P8_39 (PIN_OUTPUT | MUX_MODE0)
+			/* P8_40: lcd_data7.lcd_data7 */
+			BONE_P8_40 (PIN_OUTPUT | MUX_MODE0)
+			/* P8_37: lcd_data8.lcd_data8 */
+			BONE_P8_37 (PIN_OUTPUT | MUX_MODE0)
+			/* P8_38: lcd_data9.lcd_data9 */
+			BONE_P8_38 (PIN_OUTPUT | MUX_MODE0)
+			/* P8_36: lcd_data10.lcd_data10 */
+			BONE_P8_36 (PIN_OUTPUT | MUX_MODE0)
+			/* P8_34: lcd_data11.lcd_data11 */
+			BONE_P8_34 (PIN_OUTPUT | MUX_MODE0)
+			/* P8_35: lcd_data12.lcd_data12 */
+			BONE_P8_35 (PIN_OUTPUT | MUX_MODE0)
+			/* P8_33: lcd_data13.lcd_data13 */
+			BONE_P8_33 (PIN_OUTPUT | MUX_MODE0)
+			/* P8_31: lcd_data14.lcd_data14 */
+			BONE_P8_31 (PIN_OUTPUT | MUX_MODE0)
+			/* P8_32: lcd_data15.lcd_data15 */
+			BONE_P8_32 (PIN_OUTPUT | MUX_MODE0)
+
+			/* gpmc_ad15.lcd_data16 */
+			BONE_P8_15 (PIN_OUTPUT | MUX_MODE1)
+			/* gpmc_ad14.lcd_data17 */
+			BONE_P8_16 (PIN_OUTPUT | MUX_MODE1)
+			/* gpmc_ad13.lcd_data18 */
+			BONE_P8_11 (PIN_OUTPUT | MUX_MODE1)
+			/* gpmc_ad12.lcd_data19 */
+			BONE_P8_12 (PIN_OUTPUT | MUX_MODE1)
+			/* gpmc_ad11.lcd_data20 */
+			BONE_P8_17 (PIN_OUTPUT | MUX_MODE1)
+			/* gpmc_ad10.lcd_data21 */
+			BONE_P8_14 (PIN_OUTPUT | MUX_MODE1)
+			/* gpmc_ad9.lcd_data22 */
+			BONE_P8_13 (PIN_OUTPUT | MUX_MODE1)
+			/* gpmc_ad8.lcd_data23 */
+			BONE_P8_19 (PIN_OUTPUT | MUX_MODE1)
+
+			/* P8_27: lcd_vsync.lcd_vsync */
+			BONE_P8_27 (PIN_OUTPUT | MUX_MODE0)
+			/* P8_29: lcd_hsync.lcd_hsync */
+			BONE_P8_29 (PIN_OUTPUT | MUX_MODE0)
+			/* P8_28: lcd_pclk.lcd_pclk*/
+			BONE_P8_28 (PIN_OUTPUT | MUX_MODE0)
+			/* P8_30: lcd_ac_bias_en.lcd_ac_bias_en */
+			BONE_P8_30 (PIN_OUTPUT | MUX_MODE0)
+		>;
+	};
+};
+
+/ {
+	panel {
+		pinctrl-0 = <&lcd_24bit_pins>;
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-bone-pinmux-ttyS1.dtsi b/arch/arm/boot/dts/am335x-bone-pinmux-ttyS1.dtsi
--- a/arch/arm/boot/dts/am335x-bone-pinmux-ttyS1.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-bone-pinmux-ttyS1.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/* Testing */
+/* sudo /sbin/getty -L ttyS1 115200 vt102 */
+
+#include <dt-bindings/board/am335x-bbw-bbb-base.h>
+#include "am335x-peripheral-ttyS1.dtsi"
+
+/* cape universal */
+
+/*
+ *&ocp {
+ *	P9_24_pinmux {
+ *		mode = "uart";
+ *	};
+ *	P9_26_pinmux {
+ *		mode = "uart";
+ *	};
+ *};
+ *
+ *&uart1 {
+ *	pinctrl-0 = <>;
+ *};
+ *
+ */
+
+/* standard */
+
+&am33xx_pinmux {
+	uart1_pins: pinmux_uart1_pins {
+		pinctrl-single,pins = <
+			/* P9_24: uart1_txd.uart1_txd */
+			BONE_P9_24 (PIN_OUTPUT_PULLDOWN | MUX_MODE0)
+			/* P9_26: uart1_rxd.uart1_rxd */
+			BONE_P9_26 (PIN_INPUT_PULLUP | MUX_MODE0)
+		>;
+	};
+};
+
+&uart1 {
+	pinctrl-0 = <&uart1_pins>;
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-bone-pinmux-ttyS2.dtsi b/arch/arm/boot/dts/am335x-bone-pinmux-ttyS2.dtsi
--- a/arch/arm/boot/dts/am335x-bone-pinmux-ttyS2.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-bone-pinmux-ttyS2.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/* Testing */
+/* sudo /sbin/getty -L ttyS2 115200 vt102 */
+
+#include <dt-bindings/board/am335x-bbw-bbb-base.h>
+#include "am335x-peripheral-ttyS2.dtsi"
+
+/* cape universal */
+
+/*
+ *&ocp {
+ *	P9_21_pinmux {
+ *		mode = "uart";
+ *	};
+ *	P9_22_pinmux {
+ *		mode = "uart";
+ *	};
+ *};
+ *
+ *&uart2 {
+ *	pinctrl-0 = <>;
+ *};
+ *
+ */
+
+/* standard */
+
+&am33xx_pinmux {
+	uart2_pins: pinmux_uart2_pins {
+		pinctrl-single,pins = <
+			/* P9_21: spi0_d0.uart2_txd */
+			BONE_P9_21 (PIN_OUTPUT_PULLDOWN | MUX_MODE1)
+			/* P9_22: spi0_sclk.uart2_rxd */
+			BONE_P9_22 (PIN_INPUT_PULLUP | MUX_MODE1)
+		>;
+	};
+};
+
+&uart2 {
+	pinctrl-0 = <&uart2_pins>;
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-bone-pinmux-ttyS4.dtsi b/arch/arm/boot/dts/am335x-bone-pinmux-ttyS4.dtsi
--- a/arch/arm/boot/dts/am335x-bone-pinmux-ttyS4.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-bone-pinmux-ttyS4.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/* Testing */
+/* sudo /sbin/getty -L ttyS4 115200 vt102 */
+
+#include <dt-bindings/board/am335x-bbw-bbb-base.h>
+#include "am335x-peripheral-ttyS4.dtsi"
+
+/* cape universal */
+
+/*
+ *&ocp {
+ *	P9_11_pinmux {
+ *		mode = "uart";
+ *	};
+ *	P9_13_pinmux {
+ *		mode = "uart";
+ *	};
+ *};
+ *
+ *&uart4 {
+ *	pinctrl-0 = <>;
+ *};
+ *
+ */
+
+/* standard */
+
+&am33xx_pinmux {
+	uart4_pins: pinmux_uart4_pins {
+		pinctrl-single,pins = <
+			/* P9_11: gpmc_wait0.uart4_rxd_mux2 */
+			BONE_P9_11 (PIN_INPUT_PULLUP | MUX_MODE6)
+			/* P9_13: gpmc_wpn.uart4_txd_mux2  */
+			BONE_P9_13 (PIN_OUTPUT_PULLDOWN | MUX_MODE6)
+		>;
+	};
+};
+
+&uart4 {
+	pinctrl-0 = <&uart4_pins>;
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-bone-uboot-univ.dts b/arch/arm/boot/dts/am335x-bone-uboot-univ.dts
--- a/arch/arm/boot/dts/am335x-bone-uboot-univ.dts	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-bone-uboot-univ.dts	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ */
+/dts-v1/;
+
+#include "am33xx.dtsi"
+#include "am335x-bone-common.dtsi"
+#include "am335x-bone-common-univ.dtsi"
+
+/ {
+	model = "TI AM335x BeagleBone";
+	compatible = "ti,am335x-bone", "ti,am33xx";
+
+	chosen {
+		base_dtb = "am335x-bone-uboot-univ.dts";
+		base_dtb_timestamp = __TIMESTAMP__;
+	};
+};
+
+&sgx {
+	status = "okay";
+};
+
+&ldo3_reg {
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <3300000>;
+	regulator-always-on;
+};
+
+&mmc1 {
+	vmmc-supply = <&ldo3_reg>;
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-cape-bbb-exp-c.dtsi b/arch/arm/boot/dts/am335x-cape-bbb-exp-c.dtsi
--- a/arch/arm/boot/dts/am335x-cape-bbb-exp-c.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-cape-bbb-exp-c.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,227 @@
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <dt-bindings/board/am335x-bbw-bbb-base.h>
+
+#include "am335x-peripheral-can0.dtsi"
+#include "am335x-bone-pinmux-can0.dtsi"
+
+#include "am335x-peripheral-ttyS1.dtsi"
+#include "am335x-bone-pinmux-ttyS1.dtsi"
+
+#include "am335x-peripheral-ttyS2.dtsi"
+#include "am335x-bone-pinmux-ttyS2.dtsi"
+
+#include "am335x-peripheral-ttyS4.dtsi"
+#include "am335x-bone-pinmux-ttyS4.dtsi"
+
+&am33xx_pinmux {
+	user_leds_s1: user_leds_s1 {
+		pinctrl-single,pins = <
+			0x98 0x7	/* gpmc_wen.gpio2_4, OUTPUT | MODE7 */
+			0x9c 0x7	/* gpmc_ben0_cle.gpio2_5, OUTPUT | MODE7 */
+		>;
+	};
+
+	bb_lcd_pwm_backlight_pins: pinmux_bb_lcd_pwm_backlight_pins {
+		pinctrl-single,pins = <
+			BONE_P9_14 (PIN_OUTPUT_PULLDOWN | MUX_MODE6) /* gpmc_a2.ehrpwm1a */
+		>;
+	};
+
+	keymap3_pins: pinmux_keymap3_pins {
+		pinctrl-single,pins = <
+			0x040 0x2f	/* KEY_UP gpmc_a0.gpio1_16, INPUT | PULLDIS | MODE7 */
+			0x04c 0x2f	/* KEY_DOWN gpmc_a3.gpio1_19, INPUT | PULLDIS | MODE7 */
+			0x078 0x2f	/* KEY_RIGHT gpmc_ben1.gpio1_28, INPUT | PULLDIS | MODE7 */
+			0x164 0x2f	/* KEY_LEFT ecap0_in_pwm0_out.gpio0_7, INPUT | PULLDIS | MODE7 */
+			0x1a4 0x2f	/* KEY_ENTER mcasp0_fxr.gpio3_19, INPUT | PULLDIS | MODE7 */
+		>;
+	};
+
+	edt_ft5306_ts_pins: pinmux_edt_ft5306_ts_pins {
+		pinctrl-single,pins = <
+			/* CAP_TSC gpmc_a1.gpio1_17, INPUT | MODE7 */
+			BONE_P9_23 (PIN_INPUT_PULLDOWN | MUX_MODE7)
+		>;
+	};
+
+	i2c1_pins: pinmux_i2c1_pins {
+		pinctrl-single,pins = <
+			/* spi0_d1.i2c1_sda, SLEWCTRL_SLOW | INPUT_PULLUP | MODE2 */
+			BONE_P9_18 (PIN_INPUT_PULLUP | SLEWCTRL_FAST | MUX_MODE2)
+			/* spi0_cs0.i2c1_scl, SLEWCTRL_SLOW | INPUT_PULLUP | MODE2 */
+			BONE_P9_17 (PIN_INPUT_PULLUP | SLEWCTRL_FAST | MUX_MODE2)
+		>;
+	};
+
+	mcasp0_pins: pinmux_mcasp0_pins {
+		pinctrl-single,pins = <
+			0x190 0x20      /* mcasp0_aclkx.mcasp0_aclkx, INPUT | MODE0 */
+			0x194 0x20      /* mcasp0_fsx.mcasp0_fsx, INPUT | MODE0 */
+			0x198 0x20      /* mcasp0_axr0.mcasp0_axr0, INPUT | MODE0 */
+			0x19c 0x22      /* mcasp0_ahclkr.mcasp0_axr2, INPUT | MODE2 */
+		>;
+	};
+};
+
+&epwmss1 {
+	status = "okay";
+};
+
+
+&ehrpwm1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&bb_lcd_pwm_backlight_pins>;
+	status = "okay";
+};
+
+&i2c1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_pins>;
+	clock-frequency = <400000>;
+
+	edt-ft5306@38 {
+		status = "okay";
+		compatible = "edt,edt-ft5306", "edt,edt-ft5x06";
+		pinctrl-names = "default";
+		pinctrl-0 = <&edt_ft5306_ts_pins>;
+
+		reg = <0x38>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <17 0>;
+
+		touchscreen-size-x = <600>;
+		touchscreen-size-y = <1024>;
+		touchscreen-inverted-x;
+		touchscreen-inverted-y;
+		touchscreen-swapped-x-y;
+	};
+
+	tlv320aic3x: tlv320aic3x@1b {
+		compatible = "ti,tlv320aic3x";
+		reg = <0x1b>;
+		status = "okay";
+	};
+};
+
+&mcasp0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcasp0_pins>;
+
+	status = "okay";
+
+	op-mode = <0>;          /* MCASP_IIS_MODE */
+	tdm-slots = <2>;
+	num-serializer = <16>;
+	serial-dir = <  /* 0: INACTIVE, 1: TX, 2: RX */
+		1 0 2 0
+		0 0 0 0
+		0 0 0 0
+		0 0 0 0
+	>;
+	tx-num-evt = <1>;
+	rx-num-evt = <1>;
+};
+
+/ {
+	backlight {
+		status = "okay";
+		compatible = "pwm-backlight";
+		pwms = <&ehrpwm1 0 50000 0>;
+		brightness-levels = <0 51 53 56 62 75 101 152 255>;
+		default-brightness-level = <8>;
+	};
+
+	gpio_keys {
+		compatible = "gpio-keys";
+		pinctrl-names = "default";
+		pinctrl-0 = <&keymap3_pins>;
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		button@1 {
+			debounce_interval = <50>;
+			linux,code = <105>;
+			label = "left";
+			gpios = <&gpio0 7 0x1>;
+			gpio-key,wakeup;
+			autorepeat;
+		};
+		button@2 {
+			debounce_interval = <50>;
+			linux,code = <106>;
+			label = "right";
+			gpios = <&gpio1 28 0x1>;
+			gpio-key,wakeup;
+			autorepeat;
+		};
+		button@3 {
+			debounce_interval = <50>;
+			linux,code = <103>;
+			label = "up";
+			gpios = <&gpio1 16 0x1>;
+			gpio-key,wakeup;
+			autorepeat;
+		};
+		button@4 {
+			debounce_interval = <50>;
+			linux,code = <108>;
+			label = "down";
+			gpios = <&gpio1 19 0x1>;
+			gpio-key,wakeup;
+			autorepeat;
+		};
+		button@5 {
+			debounce_interval = <50>;
+			linux,code = <28>;
+			label = "enter";
+			gpios = <&gpio3 19 0x1>;
+			gpio-key,wakeup;
+		};
+	};
+
+	gpio-leds-cape-lcd {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+
+		pinctrl-0 = <&user_leds_s1>;
+
+		lcd-led0 {
+			label = "lcd:green:usr0";
+			gpios = <&gpio2 4 0>;
+			linux,default-trigger = "heartbeat";
+			default-state = "off";
+		};
+
+		lcd-led1 {
+			label = "lcd:green:usr1";
+			gpios = <&gpio2 5 0>;
+			linux,default-trigger = "mmc0";
+			default-state = "off";
+		};
+	};
+
+	sound {
+		compatible = "ti,da830-evm-audio";
+		ti,model = "DA830 EVM";
+		ti,audio-codec = <&tlv320aic3x>;
+		ti,mcasp-controller = <&mcasp0>;
+		ti,codec-clock-rate = <12000000>;
+		ti,audio-routing =
+			"Headphone Jack",       "HPLOUT",
+			"Headphone Jack",       "HPROUT",
+			"MIC3L",                "Mic Jack",
+			"MIC3R",                "Mic Jack";
+	};
+};
+
+#include "am335x-peripheral-panel-1024x600-24bit.dtsi"
+#include "am335x-bone-pinmux-panel-1024x600-24bit.dtsi"
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-cape-bbb-exp-r.dtsi b/arch/arm/boot/dts/am335x-cape-bbb-exp-r.dtsi
--- a/arch/arm/boot/dts/am335x-cape-bbb-exp-r.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-cape-bbb-exp-r.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,217 @@
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <dt-bindings/board/am335x-bbw-bbb-base.h>
+
+#include "am335x-peripheral-can0.dtsi"
+#include "am335x-bone-pinmux-can0.dtsi"
+
+#include "am335x-peripheral-ttyS1.dtsi"
+#include "am335x-bone-pinmux-ttyS1.dtsi"
+
+#include "am335x-peripheral-ttyS2.dtsi"
+#include "am335x-bone-pinmux-ttyS2.dtsi"
+
+#include "am335x-peripheral-ttyS4.dtsi"
+#include "am335x-bone-pinmux-ttyS4.dtsi"
+
+&am33xx_pinmux {
+	user_leds_s1: user_leds_s1 {
+		pinctrl-single,pins = <
+			0x98 0x7	/* gpmc_wen.gpio2_4, OUTPUT | MODE7 */
+			0x9c 0x7	/* gpmc_ben0_cle.gpio2_5, OUTPUT | MODE7 */
+		>;
+	};
+
+	bb_lcd_pwm_backlight_pins: pinmux_bb_lcd_pwm_backlight_pins {
+		pinctrl-single,pins = <
+			BONE_P9_14 (PIN_OUTPUT_PULLDOWN | MUX_MODE6) /* gpmc_a2.ehrpwm1a */
+		>;
+	};
+
+	keymap3_pins: pinmux_keymap3_pins {
+		pinctrl-single,pins = <
+			0x040 0x2f	/* KEY_UP gpmc_a0.gpio1_16, INPUT | PULLDIS | MODE7 */
+			0x04c 0x2f	/* KEY_DOWN gpmc_a3.gpio1_19, INPUT | PULLDIS | MODE7 */
+			0x078 0x2f	/* KEY_RIGHT gpmc_ben1.gpio1_28, INPUT | PULLDIS | MODE7 */
+			0x164 0x2f	/* KEY_LEFT ecap0_in_pwm0_out.gpio0_7, INPUT | PULLDIS | MODE7 */
+			0x1a4 0x2f	/* KEY_ENTER mcasp0_fxr.gpio3_19, INPUT | PULLDIS | MODE7 */
+		>;
+	};
+
+	i2c1_pins: pinmux_i2c1_pins {
+		pinctrl-single,pins = <
+			/* spi0_d1.i2c1_sda, SLEWCTRL_SLOW | INPUT_PULLUP | MODE2 */
+			BONE_P9_18 (PIN_INPUT_PULLUP | SLEWCTRL_FAST | MUX_MODE2)
+			/* spi0_cs0.i2c1_scl, SLEWCTRL_SLOW | INPUT_PULLUP | MODE2 */
+			BONE_P9_17 (PIN_INPUT_PULLUP | SLEWCTRL_FAST | MUX_MODE2)
+		>;
+	};
+
+	mcasp0_pins: pinmux_mcasp0_pins {
+		pinctrl-single,pins = <
+			0x190 0x20      /* mcasp0_aclkx.mcasp0_aclkx, INPUT | MODE0 */
+			0x194 0x20      /* mcasp0_fsx.mcasp0_fsx, INPUT | MODE0 */
+			0x198 0x20      /* mcasp0_axr0.mcasp0_axr0, INPUT | MODE0 */
+			0x19c 0x22      /* mcasp0_ahclkr.mcasp0_axr2, INPUT | MODE2 */
+		>;
+	};
+};
+
+&epwmss1 {
+	status = "okay";
+};
+
+
+&ehrpwm1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&bb_lcd_pwm_backlight_pins>;
+	status = "okay";
+};
+
+&i2c1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_pins>;
+	clock-frequency = <400000>;
+
+	tlv320aic3x: tlv320aic3x@1b {
+		compatible = "ti,tlv320aic3x";
+		reg = <0x1b>;
+		status = "okay";
+	};
+};
+
+&mcasp0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcasp0_pins>;
+
+	status = "okay";
+
+	op-mode = <0>;          /* MCASP_IIS_MODE */
+	tdm-slots = <2>;
+	num-serializer = <16>;
+	serial-dir = <  /* 0: INACTIVE, 1: TX, 2: RX */
+		1 0 2 0
+		0 0 0 0
+		0 0 0 0
+		0 0 0 0
+	>;
+	tx-num-evt = <1>;
+	rx-num-evt = <1>;
+};
+
+&tscadc {
+	status = "okay";
+	tsc {
+		ti,wires = <4>;
+		ti,x-plate-resistance = <200>;
+		ti,coordinate-readouts = <5>;
+		ti,wire-config = <0x00 0x11 0x22 0x33>;
+	};
+
+	adc {
+		ti,adc-channels = <4 5 6 7>;
+	};
+};
+
+/ {
+	backlight {
+		status = "okay";
+		compatible = "pwm-backlight";
+		pwms = <&ehrpwm1 0 50000 0>;
+		brightness-levels = <0 51 53 56 62 75 101 152 255>;
+		default-brightness-level = <8>;
+	};
+
+	gpio_keys {
+		compatible = "gpio-keys";
+		pinctrl-names = "default";
+		pinctrl-0 = <&keymap3_pins>;
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		button@1 {
+			debounce_interval = <50>;
+			linux,code = <105>;
+			label = "left";
+			gpios = <&gpio0 7 0x1>;
+			gpio-key,wakeup;
+			autorepeat;
+		};
+		button@2 {
+			debounce_interval = <50>;
+			linux,code = <106>;
+			label = "right";
+			gpios = <&gpio1 28 0x1>;
+			gpio-key,wakeup;
+			autorepeat;
+		};
+		button@3 {
+			debounce_interval = <50>;
+			linux,code = <103>;
+			label = "up";
+			gpios = <&gpio1 16 0x1>;
+			gpio-key,wakeup;
+			autorepeat;
+		};
+		button@4 {
+			debounce_interval = <50>;
+			linux,code = <108>;
+			label = "down";
+			gpios = <&gpio1 19 0x1>;
+			gpio-key,wakeup;
+			autorepeat;
+		};
+		button@5 {
+			debounce_interval = <50>;
+			linux,code = <28>;
+			label = "enter";
+			gpios = <&gpio3 19 0x1>;
+			gpio-key,wakeup;
+		};
+	};
+
+	gpio-leds-cape-lcd {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+
+		pinctrl-0 = <&user_leds_s1>;
+
+		lcd-led0 {
+			label = "lcd:green:usr0";
+			gpios = <&gpio2 4 0>;
+			linux,default-trigger = "heartbeat";
+			default-state = "off";
+		};
+
+		lcd-led1 {
+			label = "lcd:green:usr1";
+			gpios = <&gpio2 5 0>;
+			linux,default-trigger = "mmc0";
+			default-state = "off";
+		};
+	};
+
+	sound {
+		compatible = "ti,da830-evm-audio";
+		ti,model = "DA830 EVM";
+		ti,audio-codec = <&tlv320aic3x>;
+		ti,mcasp-controller = <&mcasp0>;
+		ti,codec-clock-rate = <12000000>;
+		ti,audio-routing =
+			"Headphone Jack",       "HPLOUT",
+			"Headphone Jack",       "HPROUT",
+			"MIC3L",                "Mic Jack",
+			"MIC3R",                "Mic Jack";
+	};
+};
+
+#include "am335x-peripheral-panel-1024x600-24bit.dtsi"
+#include "am335x-bone-pinmux-panel-1024x600-24bit.dtsi"
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-evm.dts b/arch/arm/boot/dts/am335x-evm.dts
--- a/arch/arm/boot/dts/am335x-evm.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-evm.dts	2020-06-19 21:00:31.404227715 -0400
@@ -818,3 +818,19 @@
 	clocks = <&clk_32768_ck>, <&l4_per_clkctrl AM3_CLKDIV32K_CLKCTRL 0>;
 	clock-names = "ext-clk", "int-clk";
 };
+
+&wkup_m3_ipc {
+	ti,scale-data-fw = "am335x-evm-scale-data.bin";
+};
+
+&sgx {
+	status = "okay";
+};
+
+&pruss_soc_bus {
+	status = "okay";
+
+	pruss: pruss@4a300000 {
+		status = "okay";
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-evmsk.dts b/arch/arm/boot/dts/am335x-evmsk.dts
--- a/arch/arm/boot/dts/am335x-evmsk.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-evmsk.dts	2020-06-19 21:00:31.404227715 -0400
@@ -223,7 +223,13 @@
 
 &am33xx_pinmux {
 	pinctrl-names = "default";
-	pinctrl-0 = <&gpio_keys_s0 &clkout2_pin>;
+	pinctrl-0 = <&gpio_keys_s0 &clkout2_pin &ddr3_vtt_toggle>;
+
+	ddr3_vtt_toggle: ddr3_vtt_toggle {
+		pinctrl-single,pins = <
+			0x164 (PIN_OUTPUT | MUX_MODE7)	/* ecap0_in_pwm0_out.gpio0_7 */
+		>;
+	};
 
 	lcd_pins_default: lcd_pins_default {
 		pinctrl-single,pins = <
@@ -565,6 +571,12 @@
 	};
 };
 
+&wkup_m3_ipc {
+	ti,needs-vtt-toggle;
+	ti,vtt-gpio-pin = <7>;
+	ti,scale-data-fw = "am335x-evm-scale-data.bin";
+};
+
 #include "tps65910.dtsi"
 
 &tps {
@@ -747,7 +759,19 @@
 	blue-and-red-wiring = "crossed";
 };
 
+&sgx {
+	status = "okay";
+};
+
 &rtc {
 	clocks = <&clk_32768_ck>, <&l4_per_clkctrl AM3_CLKDIV32K_CLKCTRL 0>;
 	clock-names = "ext-clk", "int-clk";
 };
+
+&pruss_soc_bus {
+	status = "okay";
+
+	pruss: pruss@4a300000 {
+		status = "okay";
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-icev2-common.dtsi b/arch/arm/boot/dts/am335x-icev2-common.dtsi
--- a/arch/arm/boot/dts/am335x-icev2-common.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-icev2-common.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,405 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2016-2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+
+#include "am33xx.dtsi"
+
+/ {
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x80000000 0x10000000>; /* 256 MB */
+	};
+
+	chosen {
+		stdout-path = &uart3;
+	};
+
+	vbat: fixedregulator0 {
+		compatible = "regulator-fixed";
+		regulator-name = "vbat";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-boot-on;
+	};
+
+	vtt_fixed: fixedregulator1 {
+		compatible = "regulator-fixed";
+		regulator-name = "vtt";
+		regulator-min-microvolt = <1500000>;
+		regulator-max-microvolt = <1500000>;
+		gpio = <&gpio0 18 GPIO_ACTIVE_HIGH>;
+		regulator-always-on;
+		regulator-boot-on;
+		enable-active-high;
+	};
+
+	leds-iio {
+		status = "disabled";
+		compatible = "gpio-leds";
+		led-out0 {
+			label = "out0";
+			gpios = <&tpic2810 0 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-out1 {
+			label = "out1";
+			gpios = <&tpic2810 1 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-out2 {
+			label = "out2";
+			gpios = <&tpic2810 2 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-out3 {
+			label = "out3";
+			gpios = <&tpic2810 3 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-out4 {
+			label = "out4";
+			gpios = <&tpic2810 4 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-out5 {
+			label = "out5";
+			gpios = <&tpic2810 5 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-out6 {
+			label = "out6";
+			gpios = <&tpic2810 6 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-out7 {
+			label = "out7";
+			gpios = <&tpic2810 7 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+	};
+
+	/* Tricolor status LEDs */
+	leds1 {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&user_leds>;
+
+		led0 {
+			label = "status0:red:cpu0";
+			gpios = <&gpio0 17 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+			linux,default-trigger = "cpu0";
+		};
+
+		led1 {
+			label = "status0:green:usr";
+			gpios = <&gpio0 16 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led2 {
+			label = "status0:yellow:usr";
+			gpios = <&gpio3 9 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led3 {
+			label = "status1:red:mmc0";
+			gpios = <&gpio1 30 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+			linux,default-trigger = "mmc0";
+		};
+
+		led4 {
+			label = "status1:green:usr";
+			gpios = <&gpio0 20 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led5 {
+			label = "status1:yellow:usr";
+			gpios = <&gpio0 19 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+	};
+	gpio-decoder {
+		compatible = "gpio-decoder";
+		gpios = <&pca9536 3 GPIO_ACTIVE_HIGH>,
+			<&pca9536 2 GPIO_ACTIVE_HIGH>,
+			<&pca9536 1 GPIO_ACTIVE_HIGH>,
+			<&pca9536 0 GPIO_ACTIVE_HIGH>;
+		linux,axis = <0>; /* ABS_X */
+		decoder-max-value = <9>;
+	};
+};
+
+&am33xx_pinmux {
+	user_leds: user_leds {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x91c, PIN_OUTPUT | MUX_MODE7) /* (J18) gmii1_txd3.gpio0[16] */
+			AM33XX_IOPAD(0x920, PIN_OUTPUT | MUX_MODE7) /* (K15) gmii1_txd2.gpio0[17] */
+			AM33XX_IOPAD(0x9b0, PIN_OUTPUT | MUX_MODE7) /* (A15) xdma_event_intr0.gpio0[19] */
+			AM33XX_IOPAD(0x9b4, PIN_OUTPUT | MUX_MODE7) /* (D14) xdma_event_intr1.gpio0[20] */
+			AM33XX_IOPAD(0x880, PIN_OUTPUT | MUX_MODE7) /* (U9) gpmc_csn1.gpio1[30] */
+			AM33XX_IOPAD(0x92c, PIN_OUTPUT | MUX_MODE7) /* (K18) gmii1_txclk.gpio3[9] */
+		>;
+	};
+
+	mmc0_pins_default: mmc0_pins_default {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x8f0, PIN_INPUT_PULLUP | MUX_MODE0) /* (F17) mmc0_dat3.mmc0_dat3 */
+			AM33XX_IOPAD(0x8f4, PIN_INPUT_PULLUP | MUX_MODE0) /* (F18) mmc0_dat2.mmc0_dat2 */
+			AM33XX_IOPAD(0x8f8, PIN_INPUT_PULLUP | MUX_MODE0) /* (G15) mmc0_dat1.mmc0_dat1 */
+			AM33XX_IOPAD(0x8fc, PIN_INPUT_PULLUP | MUX_MODE0) /* (G16) mmc0_dat0.mmc0_dat0 */
+			AM33XX_IOPAD(0x900, PIN_INPUT_PULLUP | MUX_MODE0) /* (G17) mmc0_clk.mmc0_clk */
+			AM33XX_IOPAD(0x904, PIN_INPUT_PULLUP | MUX_MODE0) /* (G18) mmc0_cmd.mmc0_cmd */
+		>;
+	};
+
+	i2c0_pins_default: i2c0_pins_default {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x988, PIN_INPUT | MUX_MODE0) /* (C17) I2C0_SDA.I2C0_SDA */
+			AM33XX_IOPAD(0x98c, PIN_INPUT | MUX_MODE0) /* (C16) I2C0_SCL.I2C0_SCL */
+		>;
+	};
+
+	spi0_pins_default: spi0_pins_default {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x950, PIN_INPUT_PULLUP | MUX_MODE0) /* (A17) spi0_sclk.spi0_sclk */
+			AM33XX_IOPAD(0x954, PIN_INPUT_PULLUP | MUX_MODE0) /* (B17) spi0_d0.spi0_d0 */
+			AM33XX_IOPAD(0x958, PIN_INPUT_PULLUP | MUX_MODE0) /* (B16) spi0_d1.spi0_d1 */
+			AM33XX_IOPAD(0x95c, PIN_INPUT_PULLUP | MUX_MODE0) /* (A16) spi0_cs0.spi0_cs0 */
+			AM33XX_IOPAD(0x960, PIN_INPUT_PULLUP | MUX_MODE0) /* (C15) spi0_cs1.spi0_cs1 */
+			AM33XX_IOPAD(0x9a0, PIN_INPUT_PULLUP | MUX_MODE7) /* (B12) mcasp0_aclkr.gpio3[18] */
+		>;
+	};
+
+	uart3_pins_default: uart3_pins_default {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x934, PIN_INPUT_PULLUP | MUX_MODE1) /* (L17) gmii1_rxd3.uart3_rxd */
+			AM33XX_IOPAD(0x938, PIN_OUTPUT_PULLUP | MUX_MODE1) /* (L16) gmii1_rxd2.uart3_txd */
+		>;
+	};
+
+
+};
+
+&i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c0_pins_default>;
+
+	status = "okay";
+	clock-frequency = <400000>;
+
+	tps: power-controller@2d {
+		reg = <0x2d>;
+	};
+
+	tpic2810: gpio@60 {
+		compatible = "ti,tpic2810";
+		reg = <0x60>;
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
+
+	pca9536: gpio@41 {
+		compatible = "ti,pca9536";
+		reg = <0x41>;
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
+};
+
+&spi0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&spi0_pins_default>;
+
+	sn65hvs882@1 {
+		compatible = "pisosr-gpio";
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		load-gpios = <&gpio3 18 GPIO_ACTIVE_LOW>;
+
+		reg = <1>;
+		spi-max-frequency = <1000000>;
+		spi-cpol;
+	};
+
+	spi_nor: flash@0 {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		compatible = "winbond,w25q64", "jedec,spi-nor";
+		spi-max-frequency = <80000000>;
+		m25p,fast-read;
+		reg = <0>;
+
+		partition@0 {
+			label = "u-boot-spl";
+			reg = <0x0 0x80000>;
+			read-only;
+		};
+
+		partition@1 {
+			label = "u-boot";
+			reg = <0x80000 0x100000>;
+			read-only;
+		};
+
+		partition@2 {
+			label = "u-boot-env";
+			reg = <0x180000 0x20000>;
+			read-only;
+		};
+
+		partition@3 {
+			label = "misc";
+			reg = <0x1A0000 0x660000>;
+		};
+	};
+
+};
+
+&tscadc {
+	status = "okay";
+	adc {
+		ti,adc-channels = <1 2 3 4 5 6 7>;
+	};
+};
+
+#include "tps65910.dtsi"
+
+&tps {
+	vcc1-supply = <&vbat>;
+	vcc2-supply = <&vbat>;
+	vcc3-supply = <&vbat>;
+	vcc4-supply = <&vbat>;
+	vcc5-supply = <&vbat>;
+	vcc6-supply = <&vbat>;
+	vcc7-supply = <&vbat>;
+	vccio-supply = <&vbat>;
+
+	regulators {
+		vrtc_reg: regulator@0 {
+			regulator-always-on;
+		};
+
+		vio_reg: regulator@1 {
+			regulator-always-on;
+		};
+
+		vdd1_reg: regulator@2 {
+			regulator-name = "vdd_mpu";
+			regulator-min-microvolt = <912500>;
+			regulator-max-microvolt = <1326000>;
+			regulator-boot-on;
+			regulator-always-on;
+		};
+
+		vdd2_reg: regulator@3 {
+			regulator-name = "vdd_core";
+			regulator-min-microvolt = <912500>;
+			regulator-max-microvolt = <1144000>;
+			regulator-boot-on;
+			regulator-always-on;
+		};
+
+		vdd3_reg: regulator@4 {
+			regulator-always-on;
+		};
+
+		vdig1_reg: regulator@5 {
+			regulator-always-on;
+		};
+
+		vdig2_reg: regulator@6 {
+			regulator-always-on;
+		};
+
+		vpll_reg: regulator@7 {
+			regulator-always-on;
+		};
+
+		vdac_reg: regulator@8 {
+			regulator-always-on;
+		};
+
+		vaux1_reg: regulator@9 {
+			regulator-always-on;
+		};
+
+		vaux2_reg: regulator@10 {
+			regulator-always-on;
+		};
+
+		vaux33_reg: regulator@11 {
+			regulator-always-on;
+		};
+
+		vmmc_reg: regulator@12 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <3300000>;
+			regulator-always-on;
+		};
+	};
+};
+
+&mmc1 {
+	status = "okay";
+	vmmc-supply = <&vmmc_reg>;
+	bus-width = <4>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&mmc0_pins_default>;
+};
+
+&gpio0 {
+	/* Do not idle the GPIO used for holding the VTT regulator */
+	ti,no-reset-on-init;
+	ti,no-idle-on-init;
+};
+
+&uart3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart3_pins_default>;
+	status = "okay";
+};
+
+&gpio3 {
+	p4 {
+		gpio-hog;
+		gpios = <4 GPIO_ACTIVE_HIGH>;
+		output-high;
+		line-name = "PR1_MII_CTRL";
+	};
+
+	p10 {
+		gpio-hog;
+		gpios = <10 GPIO_ACTIVE_HIGH>;
+		/* ETH1 mux: Low for MII-PRU, high for RMII-CPSW */
+		output-high;
+		line-name = "MUX_MII_CTL1";
+	};
+};
+
+&phy_sel {
+	rmii-clock-ext;
+};
+
+&pruss_soc_bus {
+	status = "okay";
+
+	pruss: pruss@4a300000 {
+		status = "okay";
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-icev2.dts b/arch/arm/boot/dts/am335x-icev2.dts
--- a/arch/arm/boot/dts/am335x-icev2.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-icev2.dts	2020-06-19 21:00:31.404227715 -0400
@@ -1,207 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
- * Copyright (C) 2016 Texas Instruments Incorporated - http://www.ti.com/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-/*
- * AM335x ICE V2 board
- * http://www.ti.com/tool/tmdsice3359
+ * Copyright (C) 2016-2019 Texas Instruments Incorporated - http://www.ti.com/
  */
 
 /dts-v1/;
 
-#include "am33xx.dtsi"
+#include "am335x-icev2-common.dtsi"
 
 / {
-	model = "TI AM3359 ICE-V2";
+	model = "TI AM3359 ICE-V2 - dual CPSW";
 	compatible = "ti,am3359-icev2", "ti,am33xx";
 
-	memory@80000000 {
-		device_type = "memory";
-		reg = <0x80000000 0x10000000>; /* 256 MB */
-	};
-
-	chosen {
-		stdout-path = &uart3;
-	};
-
-	vbat: fixedregulator0 {
-		compatible = "regulator-fixed";
-		regulator-name = "vbat";
-		regulator-min-microvolt = <5000000>;
-		regulator-max-microvolt = <5000000>;
-		regulator-boot-on;
-	};
-
-	vtt_fixed: fixedregulator1 {
-		compatible = "regulator-fixed";
-		regulator-name = "vtt";
-		regulator-min-microvolt = <1500000>;
-		regulator-max-microvolt = <1500000>;
-		gpio = <&gpio0 18 GPIO_ACTIVE_HIGH>;
-		regulator-always-on;
-		regulator-boot-on;
-		enable-active-high;
-	};
-
-	leds-iio {
-		status = "disabled";
-		compatible = "gpio-leds";
-		led-out0 {
-			label = "out0";
-			gpios = <&tpic2810 0 GPIO_ACTIVE_HIGH>;
-			default-state = "off";
-		};
-
-		led-out1 {
-			label = "out1";
-			gpios = <&tpic2810 1 GPIO_ACTIVE_HIGH>;
-			default-state = "off";
-		};
-
-		led-out2 {
-			label = "out2";
-			gpios = <&tpic2810 2 GPIO_ACTIVE_HIGH>;
-			default-state = "off";
-		};
-
-		led-out3 {
-			label = "out3";
-			gpios = <&tpic2810 3 GPIO_ACTIVE_HIGH>;
-			default-state = "off";
-		};
-
-		led-out4 {
-			label = "out4";
-			gpios = <&tpic2810 4 GPIO_ACTIVE_HIGH>;
-			default-state = "off";
-		};
-
-		led-out5 {
-			label = "out5";
-			gpios = <&tpic2810 5 GPIO_ACTIVE_HIGH>;
-			default-state = "off";
-		};
-
-		led-out6 {
-			label = "out6";
-			gpios = <&tpic2810 6 GPIO_ACTIVE_HIGH>;
-			default-state = "off";
-		};
-
-		led-out7 {
-			label = "out7";
-			gpios = <&tpic2810 7 GPIO_ACTIVE_HIGH>;
-			default-state = "off";
-		};
-	};
-
-	/* Tricolor status LEDs */
-	leds1 {
-		compatible = "gpio-leds";
-		pinctrl-names = "default";
-		pinctrl-0 = <&user_leds>;
-
-		led0 {
-			label = "status0:red:cpu0";
-			gpios = <&gpio0 17 GPIO_ACTIVE_HIGH>;
-			default-state = "off";
-			linux,default-trigger = "cpu0";
-		};
-
-		led1 {
-			label = "status0:green:usr";
-			gpios = <&gpio0 16 GPIO_ACTIVE_HIGH>;
-			default-state = "off";
-		};
-
-		led2 {
-			label = "status0:yellow:usr";
-			gpios = <&gpio3 9 GPIO_ACTIVE_HIGH>;
-			default-state = "off";
-		};
-
-		led3 {
-			label = "status1:red:mmc0";
-			gpios = <&gpio1 30 GPIO_ACTIVE_HIGH>;
-			default-state = "off";
-			linux,default-trigger = "mmc0";
-		};
-
-		led4 {
-			label = "status1:green:usr";
-			gpios = <&gpio0 20 GPIO_ACTIVE_HIGH>;
-			default-state = "off";
-		};
-
-		led5 {
-			label = "status1:yellow:usr";
-			gpios = <&gpio0 19 GPIO_ACTIVE_HIGH>;
-			default-state = "off";
-		};
-	};
-	gpio-decoder {
-		compatible = "gpio-decoder";
-		gpios = <&pca9536 3 GPIO_ACTIVE_HIGH>,
-			<&pca9536 2 GPIO_ACTIVE_HIGH>,
-			<&pca9536 1 GPIO_ACTIVE_HIGH>,
-			<&pca9536 0 GPIO_ACTIVE_HIGH>;
-		linux,axis = <0>; /* ABS_X */
-		decoder-max-value = <9>;
+	aliases {
+		ethernet0 = &cpsw_emac0;
+		ethernet1 = &cpsw_emac1;
 	};
 };
 
 &am33xx_pinmux {
-	user_leds: user_leds {
-		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x91c, PIN_OUTPUT | MUX_MODE7) /* (J18) gmii1_txd3.gpio0[16] */
-			AM33XX_IOPAD(0x920, PIN_OUTPUT | MUX_MODE7) /* (K15) gmii1_txd2.gpio0[17] */
-			AM33XX_IOPAD(0x9b0, PIN_OUTPUT | MUX_MODE7) /* (A15) xdma_event_intr0.gpio0[19] */
-			AM33XX_IOPAD(0x9b4, PIN_OUTPUT | MUX_MODE7) /* (D14) xdma_event_intr1.gpio0[20] */
-			AM33XX_IOPAD(0x880, PIN_OUTPUT | MUX_MODE7) /* (U9) gpmc_csn1.gpio1[30] */
-			AM33XX_IOPAD(0x92c, PIN_OUTPUT | MUX_MODE7) /* (K18) gmii1_txclk.gpio3[9] */
-		>;
-	};
-
-	mmc0_pins_default: mmc0_pins_default {
-		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x8f0, PIN_INPUT_PULLUP | MUX_MODE0) /* (F17) mmc0_dat3.mmc0_dat3 */
-			AM33XX_IOPAD(0x8f4, PIN_INPUT_PULLUP | MUX_MODE0) /* (F18) mmc0_dat2.mmc0_dat2 */
-			AM33XX_IOPAD(0x8f8, PIN_INPUT_PULLUP | MUX_MODE0) /* (G15) mmc0_dat1.mmc0_dat1 */
-			AM33XX_IOPAD(0x8fc, PIN_INPUT_PULLUP | MUX_MODE0) /* (G16) mmc0_dat0.mmc0_dat0 */
-			AM33XX_IOPAD(0x900, PIN_INPUT_PULLUP | MUX_MODE0) /* (G17) mmc0_clk.mmc0_clk */
-			AM33XX_IOPAD(0x904, PIN_INPUT_PULLUP | MUX_MODE0) /* (G18) mmc0_cmd.mmc0_cmd */
-		>;
-	};
-
-	i2c0_pins_default: i2c0_pins_default {
-		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x988, PIN_INPUT | MUX_MODE0) /* (C17) I2C0_SDA.I2C0_SDA */
-			AM33XX_IOPAD(0x98c, PIN_INPUT | MUX_MODE0) /* (C16) I2C0_SCL.I2C0_SCL */
-		>;
-	};
-
-	spi0_pins_default: spi0_pins_default {
-		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x950, PIN_INPUT_PULLUP | MUX_MODE0) /* (A17) spi0_sclk.spi0_sclk */
-			AM33XX_IOPAD(0x954, PIN_INPUT_PULLUP | MUX_MODE0) /* (B17) spi0_d0.spi0_d0 */
-			AM33XX_IOPAD(0x958, PIN_INPUT_PULLUP | MUX_MODE0) /* (B16) spi0_d1.spi0_d1 */
-			AM33XX_IOPAD(0x95c, PIN_INPUT_PULLUP | MUX_MODE0) /* (A16) spi0_cs0.spi0_cs0 */
-			AM33XX_IOPAD(0x960, PIN_INPUT_PULLUP | MUX_MODE0) /* (C15) spi0_cs1.spi0_cs1 */
-			AM33XX_IOPAD(0x9a0, PIN_INPUT_PULLUP | MUX_MODE7) /* (B12) mcasp0_aclkr.gpio3[18] */
-		>;
-	};
-
-	uart3_pins_default: uart3_pins_default {
-		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x934, PIN_INPUT_PULLUP | MUX_MODE1) /* (L17) gmii1_rxd3.uart3_rxd */
-			AM33XX_IOPAD(0x938, PIN_OUTPUT_PULLUP | MUX_MODE1) /* (L16) gmii1_rxd2.uart3_txd */
-		>;
-	};
-
 	cpsw_default: cpsw_default {
 		pinctrl-single,pins = <
 			/* Slave 1, RMII mode */
@@ -290,6 +106,19 @@
 		gpio-controller;
 		#gpio-cells = <2>;
 	};
+
+	/* osd9616p0899-10 */
+	display@3c {
+		compatible = "solomon,ssd1306fb-i2c";
+		reg = <0x3c>;
+		solomon,height = <16>;
+		solomon,width = <96>;
+		solomon,com-seq;
+		solomon,com-invdir;
+		solomon,page-offset = <0>;
+		solomon,prechargep1 = <2>;
+		solomon,prechargep2 = <13>;
+	};
 };
 
 &spi0 {
@@ -448,13 +277,6 @@
 };
 
 &gpio3 {
-	p4 {
-		gpio-hog;
-		gpios = <4 GPIO_ACTIVE_HIGH>;
-		output-high;
-		line-name = "PR1_MII_CTRL";
-	};
-
 	p10 {
 		gpio-hog;
 		gpios = <10 GPIO_ACTIVE_HIGH>;
@@ -484,10 +306,6 @@
 	dual_emac;
 };
 
-&phy_sel {
-	rmii-clock-ext;
-};
-
 &davinci_mdio {
 	pinctrl-names = "default", "sleep";
 	pinctrl-0 = <&davinci_mdio_default>;
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-icev2-prueth.dts b/arch/arm/boot/dts/am335x-icev2-prueth.dts
--- a/arch/arm/boot/dts/am335x-icev2-prueth.dts	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-icev2-prueth.dts	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,139 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/*
+ * AM335x ICE V2 board
+ * http://www.ti.com/tool/tmdsice3359
+ */
+
+/dts-v1/;
+
+#include "am335x-icev2-common.dtsi"
+
+/ {
+	model = "TI AM3359 ICE-V2 - dual PRUETH";
+	compatible = "ti,am3359-icev2", "ti,am33xx";
+
+	aliases {
+		ethernet0 = &pruss_emac0;
+		ethernet1 = &pruss_emac1;
+	};
+
+	/* Dual mac ethernet application node on icss */
+	pruss_eth {
+		compatible = "ti,am3359-prueth";
+		prus = <&pru0>, <&pru1>;
+		firmware-name = "ti-pruss/am335x-pru0-prueth-fw.elf",
+				"ti-pruss/am335x-pru1-prueth-fw.elf";
+		sram = <&ocmcram>;
+		interrupt-parent = <&pruss_intc>;
+		mii-rt = <&pruss_mii_rt>;
+		iep = <&pruss_iep>;
+		pinctrl-0 = <&pruss_eth_default>;
+		pinctrl-names = "default";
+
+		pruss_emac0: ethernet-mii0 {
+			phy-handle = <&pruss_eth0_phy>;
+			phy-mode = "mii";
+			interrupts = <20>, <22>;
+			interrupt-names = "rx", "tx";
+			/* Filled in by bootloader */
+			local-mac-address = [00 00 00 00 00 00];
+		};
+
+		pruss_emac1: ethernet-mii1 {
+			phy-handle = <&pruss_eth1_phy>;
+			phy-mode = "mii";
+			interrupts = <21>, <23>;
+			interrupt-names = "rx", "tx";
+			/* Filled in by bootloader */
+			local-mac-address = [00 00 00 00 00 00];
+		};
+	};
+};
+
+&am33xx_pinmux {
+	pruss_mdio_default: pruss_mdio_default {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x88c, (PIN_OUTPUT | MUX_MODE5))		/* gpmc_clk.pr1_mdio_mdclk */
+			AM33XX_IOPAD(0x888, (PIN_INPUT  | MUX_MODE5))		/* gpmc_csn3.pr1_mdio_data */
+			AM33XX_IOPAD(0x89c, (PIN_INPUT_PULLUP | MUX_MODE7))	/* gpmc_ben0_cle.gpio2_5 */
+			/* disable CPSW MDIO */
+			AM33XX_IOPAD(0x948, (PIN_INPUT_PULLUP | MUX_MODE7))	/* mdio_data.gpio0_0 */
+			AM33XX_IOPAD(0x94c, (PIN_INPUT_PULLUP | MUX_MODE7))	/* mdio_clk.gpio0_1 */
+		>;
+	};
+
+	pruss_eth_default: pruss_eth_default {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x8a0, (PIN_INPUT  | MUX_MODE2))	/* dss_data0.pr1_mii_mt0_clk */
+			AM33XX_IOPAD(0x8b4, (PIN_OUTPUT | MUX_MODE2))	/* dss_data5.pr1_mii0_txd0 */
+			AM33XX_IOPAD(0x8b0, (PIN_OUTPUT | MUX_MODE2))	/* dss_data4.pr1_mii0_txd1 */
+			AM33XX_IOPAD(0x8ac, (PIN_OUTPUT | MUX_MODE2))	/* dss_data3.pr1_mii0_txd2 */
+			AM33XX_IOPAD(0x8a8, (PIN_OUTPUT | MUX_MODE2))	/* dss_data2.pr1_mii0_txd3 */
+			AM33XX_IOPAD(0x8cc, (PIN_INPUT  | MUX_MODE5))	/* dss_data11.pr1_mii0_rxd0 */
+			AM33XX_IOPAD(0x8c8, (PIN_INPUT  | MUX_MODE5))	/* dss_data10.pr1_mii0_rxd1 */
+			AM33XX_IOPAD(0x8c4, (PIN_INPUT  | MUX_MODE5))	/* dss_data9.pr1_mii0_rxd2 */
+			AM33XX_IOPAD(0x8c0, (PIN_INPUT  | MUX_MODE5))	/* dss_data8.pr1_mii0_rxd3 */
+			AM33XX_IOPAD(0x8a4, (PIN_OUTPUT | MUX_MODE2))	/* dss_data1.pr1_mii0_txen */
+			AM33XX_IOPAD(0x8d8, (PIN_INPUT  | MUX_MODE5))	/* dss_data14.pr1_mii_mr0_clk */
+			AM33XX_IOPAD(0x8dc, (PIN_INPUT  | MUX_MODE5))	/* dss_data15.pr1_mii0_rxdv */
+			AM33XX_IOPAD(0x8d4, (PIN_INPUT  | MUX_MODE5))	/* dss_data13.pr1_mii0_rxer */
+			AM33XX_IOPAD(0x8d0, (PIN_INPUT  | MUX_MODE5))	/* dss_data12.pr1_mii0_rxlink */
+			AM33XX_IOPAD(0x8e8, (PIN_INPUT  | MUX_MODE2))	/* dss_pclk.pr1_mii0_crs */
+
+			AM33XX_IOPAD(0x840, (PIN_INPUT  | MUX_MODE5))	/* gpmc_a0.pr1_mii_mt1_clk */
+			AM33XX_IOPAD(0x850, (PIN_OUTPUT | MUX_MODE5))	/* gpmc_a4.pr1_mii1_txd0 */
+			AM33XX_IOPAD(0x84c, (PIN_OUTPUT | MUX_MODE5))	/* gpmc_a3.pr1_mii1_txd1 */
+			AM33XX_IOPAD(0x848, (PIN_OUTPUT | MUX_MODE5))	/* gpmc_a2.pr1_mii1_txd2 */
+			AM33XX_IOPAD(0x844, (PIN_OUTPUT | MUX_MODE5))	/* gpmc_a1.pr1_mii1_txd3 */
+			AM33XX_IOPAD(0x860, (PIN_INPUT  | MUX_MODE5))	/* gpmc_a8.pr1_mii1_rxd0 */
+			AM33XX_IOPAD(0x85c, (PIN_INPUT  | MUX_MODE5))	/* gpmc_a7.pr1_mii1_rxd1 */
+			AM33XX_IOPAD(0x858, (PIN_INPUT  | MUX_MODE5))	/* gpmc_a6.pr1_mii1_rxd2 */
+			AM33XX_IOPAD(0x854, (PIN_INPUT  | MUX_MODE5))	/* gpmc_a5.pr1_mii1_rxd3 */
+			AM33XX_IOPAD(0x874, (PIN_OUTPUT | MUX_MODE5))	/* gpmc_wpn.pr1_mii1_txen */
+			AM33XX_IOPAD(0x864, (PIN_INPUT  | MUX_MODE5))	/* gpmc_a9.pr1_mii_mr1_clk */
+			AM33XX_IOPAD(0x868, (PIN_INPUT  | MUX_MODE5))	/* gpmc_a10.pr1_mii1_rxdv */
+			AM33XX_IOPAD(0x86c, (PIN_INPUT  | MUX_MODE5))	/* gpmc_a11.pr1_mii1_rxer */
+			AM33XX_IOPAD(0x878, (PIN_INPUT  | MUX_MODE5))	/* gpmc_ben1.pr1_mii1_rxlink */
+			AM33XX_IOPAD(0x8ec, (PIN_INPUT  | MUX_MODE2))	/* lcd_ac_bias_en.pr1_mii1_crs */
+			AM33XX_IOPAD(0x870, (PIN_INPUT  | MUX_MODE5))	/* gpmc_wait0.pr1_mii1_col */
+		>;
+	};
+};
+
+&gpio3 {
+	p4 {
+		gpio-hog;
+		gpios = <4 GPIO_ACTIVE_HIGH>;
+		output-high;
+		line-name = "PR1_MII_CTRL";
+	};
+
+	p10 {
+		gpio-hog;
+		gpios = <10 GPIO_ACTIVE_HIGH>;
+		/* ETH1 mux: Low for MII-PRU, high for RMII-CPSW */
+		output-low;
+		line-name = "MUX_MII_CTL1";
+	};
+};
+
+&pruss_mdio {
+	pinctrl-0 = <&pruss_mdio_default>;
+	pinctrl-names = "default";
+	reset-gpios = <&gpio2 5 GPIO_ACTIVE_LOW>;
+	reset-delay-us = <2>; /* PHY datasheet states 1uS min */
+	status = "okay";
+
+	pruss_eth0_phy: ethernet-phy@1 {
+		 reg = <1>;
+	};
+
+	pruss_eth1_phy: ethernet-phy@3 {
+		 reg = <3>;
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-lxm.dts b/arch/arm/boot/dts/am335x-lxm.dts
--- a/arch/arm/boot/dts/am335x-lxm.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-lxm.dts	2020-06-19 21:00:31.404227715 -0400
@@ -364,3 +364,6 @@
 	status = "okay";
 };
 
+&sgx {
+	status = "okay";
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-nano.dts b/arch/arm/boot/dts/am335x-nano.dts
--- a/arch/arm/boot/dts/am335x-nano.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-nano.dts	2020-06-19 21:00:31.404227715 -0400
@@ -467,3 +467,7 @@
 		};
 	};
 };
+
+&sgx {
+	status = "okay";
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-osd3358-sm-red.dts b/arch/arm/boot/dts/am335x-osd3358-sm-red.dts
--- a/arch/arm/boot/dts/am335x-osd3358-sm-red.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-osd3358-sm-red.dts	2020-06-19 21:00:31.404227715 -0400
@@ -13,10 +13,20 @@
 #include <dt-bindings/interrupt-controller/irq.h>
 
 #include <dt-bindings/display/tda998x.h>
+/* #include "am335x-bone-jtag.dtsi" */
+
+&sgx {
+	status = "okay";
+};
 
 / {
 	model = "Octavo Systems OSD3358-SM-RED";
 	compatible = "oct,osd3358-sm-refdesign", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx";
+
+	chosen {
+		base_dtb = "am335x-osd3358-sm-red.dts";
+		base_dtb_timestamp = __TIMESTAMP__;
+	};
 };
 
 &ldo3_reg {
@@ -40,61 +50,54 @@
 &am33xx_pinmux {
 	nxp_hdmi_bonelt_pins: nxp-hdmi-bonelt-pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x9b0, PIN_OUTPUT_PULLDOWN | MUX_MODE3)	/* xdma_event_intr0 */
-			AM33XX_IOPAD(0x8a0, PIN_OUTPUT | MUX_MODE0)		/* lcd_data0.lcd_data0 */
-			AM33XX_IOPAD(0x8a4, PIN_OUTPUT | MUX_MODE0)		/* lcd_data1.lcd_data1 */
-			AM33XX_IOPAD(0x8a8, PIN_OUTPUT | MUX_MODE0)		/* lcd_data2.lcd_data2 */
-			AM33XX_IOPAD(0x8ac, PIN_OUTPUT | MUX_MODE0)		/* lcd_data3.lcd_data3 */
-			AM33XX_IOPAD(0x8b0, PIN_OUTPUT | MUX_MODE0)		/* lcd_data4.lcd_data4 */
-			AM33XX_IOPAD(0x8b4, PIN_OUTPUT | MUX_MODE0)		/* lcd_data5.lcd_data5 */
-			AM33XX_IOPAD(0x8b8, PIN_OUTPUT | MUX_MODE0)		/* lcd_data6.lcd_data6 */
-			AM33XX_IOPAD(0x8bc, PIN_OUTPUT | MUX_MODE0)		/* lcd_data7.lcd_data7 */
-			AM33XX_IOPAD(0x8c0, PIN_OUTPUT | MUX_MODE0)		/* lcd_data8.lcd_data8 */
-			AM33XX_IOPAD(0x8c4, PIN_OUTPUT | MUX_MODE0)		/* lcd_data9.lcd_data9 */
-			AM33XX_IOPAD(0x8c8, PIN_OUTPUT | MUX_MODE0)		/* lcd_data10.lcd_data10 */
-			AM33XX_IOPAD(0x8cc, PIN_OUTPUT | MUX_MODE0)		/* lcd_data11.lcd_data11 */
-			AM33XX_IOPAD(0x8d0, PIN_OUTPUT | MUX_MODE0)		/* lcd_data12.lcd_data12 */
-			AM33XX_IOPAD(0x8d4, PIN_OUTPUT | MUX_MODE0)		/* lcd_data13.lcd_data13 */
-			AM33XX_IOPAD(0x8d8, PIN_OUTPUT | MUX_MODE0)		/* lcd_data14.lcd_data14 */
-			AM33XX_IOPAD(0x8dc, PIN_OUTPUT | MUX_MODE0)		/* lcd_data15.lcd_data15 */
-			AM33XX_IOPAD(0x8e0, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* lcd_vsync.lcd_vsync */
-			AM33XX_IOPAD(0x8e4, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* lcd_hsync.lcd_hsync */
-			AM33XX_IOPAD(0x8e8, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* lcd_pclk.lcd_pclk */
-			AM33XX_IOPAD(0x8ec, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* lcd_ac_bias_en.lcd_ac_bias_en */
-		>;
-	};
-
-	nxp_hdmi_bonelt_off_pins: nxp-hdmi-bonelt-off-pins {
-		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x9b0, PIN_OUTPUT_PULLDOWN | MUX_MODE3)	/* xdma_event_intr0 */
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA0, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA1, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA2, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA3, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA4, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA5, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA6, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA7, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA8, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA9, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA10, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA11, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA12, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA13, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA14, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_DATA15, PIN_OUTPUT, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_VSYNC, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_HSYNC, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_PCLK, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_LCD_AC_BIAS_EN, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
 		>;
 	};
 
 	mcasp0_pins: mcasp0-pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x9ac, PIN_INPUT_PULLUP | MUX_MODE0) /* mcasp0_ahcklx.mcasp0_ahclkx */
-			AM33XX_IOPAD(0x99c, PIN_OUTPUT_PULLDOWN | MUX_MODE2) /* mcasp0_ahclkr.mcasp0_axr2*/
-			AM33XX_IOPAD(0x994, PIN_OUTPUT_PULLUP | MUX_MODE0) /* mcasp0_fsx.mcasp0_fsx */
-			AM33XX_IOPAD(0x990, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mcasp0_aclkx.mcasp0_aclkx */
-			AM33XX_IOPAD(0x86c, PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* gpmc_a11.GPIO1_27 */
+			AM33XX_PADCONF(AM335X_PIN_MCASP0_AHCLKX, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MCASP0_AHCLKR, PIN_OUTPUT_PULLDOWN, MUX_MODE2) /* mcasp0_ahclkr.mcasp0_axr2*/
+			AM33XX_PADCONF(AM335X_PIN_MCASP0_FSX, PIN_OUTPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MCASP0_ACLKX, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_GPMC_A11, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gpmc_a11.GPIO1_27 */
 		>;
 	};
 
 	flash_enable: flash-enable {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x944, PIN_OUTPUT_PULLDOWN | MUX_MODE7) 	/* rmii1_ref_clk.gpio0_29 */
+			AM33XX_PADCONF(AM335X_PIN_RMII1_REF_CLK, PIN_OUTPUT_PULLDOWN, MUX_MODE7) 	/* rmii1_ref_clk.gpio0_29 */
 		>;
 	};
 
 	imu_interrupt: imu-interrupt {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x910, PIN_INPUT_PULLDOWN | MUX_MODE7) 		/* mii1_rx_er.gpio3_2 */
+			AM33XX_PADCONF(AM335X_PIN_MII1_RX_ER, PIN_INPUT_PULLDOWN, MUX_MODE7) 		/* mii1_rx_er.gpio3_2 */
 		>;
 	};
 
 	ethernet_interrupt: ethernet-interrupt{
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x908, PIN_INPUT_PULLDOWN | MUX_MODE7) 		/* mii1_col.gpio3_0 */
+			AM33XX_PADCONF(AM335X_PIN_MII1_COL, PIN_INPUT_PULLDOWN, MUX_MODE7) 		/* mii1_col.gpio3_0 */
 		>;
 	};
 };
@@ -124,9 +127,8 @@
 		compatible = "nxp,tda998x";
 		reg = <0x70>;
 
-		pinctrl-names = "default", "off";
+		pinctrl-names = "default";
 		pinctrl-0 = <&nxp_hdmi_bonelt_pins>;
-		pinctrl-1 = <&nxp_hdmi_bonelt_off_pins>;
 
 		/* Convert 24bit BGR to RGB, e.g. cross red and blue wiring */
 		/* video-ports = <0x234501>; */
@@ -264,114 +266,105 @@
 };
 
 &am33xx_pinmux {
-	pinctrl-names = "default";
-	pinctrl-0 = <&clkout2_pin>;
-
 	user_leds_s0: user-leds-s0 {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x854, PIN_OUTPUT_PULLDOWN | MUX_MODE7)	/* gpmc_a5.gpio1_21 */
-			AM33XX_IOPAD(0x858, PIN_OUTPUT_PULLUP | MUX_MODE7)	/* gpmc_a6.gpio1_22 */
-			AM33XX_IOPAD(0x85c, PIN_OUTPUT_PULLDOWN | MUX_MODE7)	/* gpmc_a7.gpio1_23 */
-			AM33XX_IOPAD(0x860, PIN_OUTPUT_PULLUP | MUX_MODE7)	/* gpmc_a8.gpio1_24 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_A5, PIN_OUTPUT_PULLDOWN, MUX_MODE7)	/* gpmc_a5.gpio1_21 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_A6, PIN_OUTPUT_PULLUP, MUX_MODE7)	/* gpmc_a6.gpio1_22 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_A7, PIN_OUTPUT_PULLDOWN, MUX_MODE7)	/* gpmc_a7.gpio1_23 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_A8, PIN_OUTPUT_PULLUP, MUX_MODE7)	/* gpmc_a8.gpio1_24 */
 		>;
 	};
 
 	i2c2_pins: pinmux-i2c2-pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x978, PIN_INPUT_PULLUP | MUX_MODE3)	/* uart1_ctsn.i2c2_sda */
-			AM33XX_IOPAD(0x97c, PIN_INPUT_PULLUP | MUX_MODE3)	/* uart1_rtsn.i2c2_scl */
+			AM33XX_PADCONF(AM335X_PIN_UART1_CTSN, PIN_INPUT_PULLUP, MUX_MODE3)	/* uart1_ctsn.i2c2_sda */
+			AM33XX_PADCONF(AM335X_PIN_UART1_RTSN, PIN_INPUT_PULLUP, MUX_MODE3)	/* uart1_rtsn.i2c2_scl */
 		>;
 	};
 
 	uart0_pins: pinmux-uart0-pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x970, PIN_INPUT_PULLUP | MUX_MODE0)	/* uart0_rxd.uart0_rxd */
-			AM33XX_IOPAD(0x974, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* uart0_txd.uart0_txd */
-		>;
-	};
-
-	clkout2_pin: pinmux-clkout2-pin {
-		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x9b4, PIN_OUTPUT_PULLDOWN | MUX_MODE3)	/* xdma_event_intr1.clkout2 */
+			AM33XX_PADCONF(AM335X_PIN_UART0_RXD, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_UART0_TXD, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
 		>;
 	};
 
 	cpsw_default: cpsw-default {
 		pinctrl-single,pins = <
 			/* Slave 1 */
-			AM33XX_IOPAD(0x914, PIN_OUTPUT_PULLDOWN | MUX_MODE2)	/* mii1_txen.rgmii1_tctl */
-			AM33XX_IOPAD(0x918, PIN_INPUT_PULLDOWN | MUX_MODE2)		/* mii1_rxdv.rgmii1_rctl */
-			AM33XX_IOPAD(0x91c, PIN_OUTPUT_PULLDOWN | MUX_MODE2)	/* mii1_txd3.rgmii1_txd3 */
-			AM33XX_IOPAD(0x920, PIN_OUTPUT_PULLDOWN | MUX_MODE2)	/* mii1_txd2.rgmii1_txd2 */
-			AM33XX_IOPAD(0x924, PIN_OUTPUT_PULLDOWN | MUX_MODE2)	/* mii1_txd1.rgmii1_txd1 */
-			AM33XX_IOPAD(0x928, PIN_OUTPUT_PULLDOWN | MUX_MODE2)	/* mii1_txd0.rgmii1_txd0 */
-			AM33XX_IOPAD(0x92c, PIN_OUTPUT_PULLDOWN | MUX_MODE2)	/* mii1_txclk.rgmii1_txclk */
-			AM33XX_IOPAD(0x930, PIN_INPUT_PULLDOWN | MUX_MODE2)		/* mii1_rxclk.rgmii1_rxclk */
-			AM33XX_IOPAD(0x934, PIN_INPUT_PULLDOWN | MUX_MODE2)		/* mii1_rxd3.rgmii1_rxd3 */
-			AM33XX_IOPAD(0x938, PIN_INPUT_PULLDOWN | MUX_MODE2)		/* mii1_rxd2.rgmii1_rxd2 */
-			AM33XX_IOPAD(0x93c, PIN_INPUT_PULLDOWN | MUX_MODE2)		/* mii1_rxd1.rgmii1_rxd1 */
-			AM33XX_IOPAD(0x940, PIN_INPUT_PULLDOWN | MUX_MODE2)		/* mii1_rxd0.rgmii1_rxd0 */
+			AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_OUTPUT_PULLDOWN, MUX_MODE2)	/* mii1_txen.rgmii1_tctl */
+			AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLDOWN, MUX_MODE2)		/* mii1_rxdv.rgmii1_rctl */
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_OUTPUT_PULLDOWN, MUX_MODE2)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_OUTPUT_PULLDOWN, MUX_MODE2)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD1, PIN_OUTPUT_PULLDOWN, MUX_MODE2)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_OUTPUT_PULLDOWN, MUX_MODE2)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_OUTPUT_PULLDOWN, MUX_MODE2)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE2)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLDOWN, MUX_MODE2)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_INPUT_PULLDOWN, MUX_MODE2)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLDOWN, MUX_MODE2)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD0, PIN_INPUT_PULLDOWN, MUX_MODE2)
 		>;
 	};
 
 	cpsw_sleep: cpsw-sleep {
 		pinctrl-single,pins = <
 			/* Slave 1 reset value */
-			AM33XX_IOPAD(0x914, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x918, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x91c, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x920, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x924, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x928, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x92c, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x930, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x934, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x938, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x93c, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x940, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD1, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD0, PIN_INPUT_PULLDOWN, MUX_MODE7)
 		>;
 	};
 
 	davinci_mdio_default: davinci-mdio-default {
 		pinctrl-single,pins = <
 			/* MDIO */
-			AM33XX_IOPAD(0x948, PIN_INPUT_PULLUP | SLEWCTRL_FAST | MUX_MODE0)	/* mdio_data.mdio_data */
-			AM33XX_IOPAD(0x94c, PIN_OUTPUT_PULLUP | MUX_MODE0)			/* mdio_clk.mdio_clk */
+			AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT_PULLUP | SLEWCTRL_FAST, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MDC, PIN_OUTPUT_PULLUP, MUX_MODE0)
 		>;
 	};
 
 	davinci_mdio_sleep: davinci-mdio-sleep {
 		pinctrl-single,pins = <
 			/* MDIO reset value */
-			AM33XX_IOPAD(0x948, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x94c, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MDC, PIN_INPUT_PULLDOWN, MUX_MODE7)
 		>;
 	};
 
 	mmc1_pins: pinmux-mmc1-pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x960, PIN_INPUT | MUX_MODE7) /* (C15) spi0_cs1.gpio0[6] */
-			AM33XX_IOPAD(0x8fc, PIN_INPUT_PULLUP | MUX_MODE0) /* (G16) mmc0_dat0.mmc0_dat0 */
-			AM33XX_IOPAD(0x8f8, PIN_INPUT_PULLUP | MUX_MODE0) /* (G15) mmc0_dat1.mmc0_dat1 */
-			AM33XX_IOPAD(0x8f4, PIN_INPUT_PULLUP | MUX_MODE0) /* (F18) mmc0_dat2.mmc0_dat2 */
-			AM33XX_IOPAD(0x8f0, PIN_INPUT_PULLUP | MUX_MODE0) /* (F17) mmc0_dat3.mmc0_dat3 */
-			AM33XX_IOPAD(0x904, PIN_INPUT_PULLUP | MUX_MODE0) /* (G18) mmc0_cmd.mmc0_cmd */
-			AM33XX_IOPAD(0x900, PIN_INPUT_PULLUP | MUX_MODE0) /* (G17) mmc0_clk.mmc0_clk */
+			AM33XX_PADCONF(AM335X_PIN_SPI0_CS1, PIN_INPUT, MUX_MODE7) /* (C15) spi0_cs1.gpio0[6] */
+			AM33XX_PADCONF(AM335X_PIN_MMC0_DAT0, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MMC0_DAT1, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MMC0_DAT2, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MMC0_DAT3, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MMC0_CMD, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MMC0_CLK, PIN_INPUT_PULLUP, MUX_MODE0)
 		>;
 	};
 
 	emmc_pins: pinmux-emmc-pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x880, PIN_INPUT_PULLUP | MUX_MODE2) /* gpmc_csn1.mmc1_clk */
-			AM33XX_IOPAD(0x884, PIN_INPUT_PULLUP | MUX_MODE2) /* gpmc_csn2.mmc1_cmd */
-			AM33XX_IOPAD(0x800, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad0.mmc1_dat0 */
-			AM33XX_IOPAD(0x804, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad1.mmc1_dat1 */
-			AM33XX_IOPAD(0x808, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad2.mmc1_dat2 */
-			AM33XX_IOPAD(0x80c, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad3.mmc1_dat3 */
-			AM33XX_IOPAD(0x810, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad4.mmc1_dat4 */
-			AM33XX_IOPAD(0x814, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad5.mmc1_dat5 */
-			AM33XX_IOPAD(0x818, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad6.mmc1_dat6 */
-			AM33XX_IOPAD(0x81c, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad7.mmc1_dat7 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_CSN1, PIN_INPUT_PULLUP, MUX_MODE2) /* gpmc_csn1.mmc1_clk */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_CSN2, PIN_INPUT_PULLUP, MUX_MODE2) /* gpmc_csn2.mmc1_cmd */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD0, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad0.mmc1_dat0 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD1, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad1.mmc1_dat1 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD2, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad2.mmc1_dat2 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD3, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad3.mmc1_dat3 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD4, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad4.mmc1_dat4 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD5, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad5.mmc1_dat5 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD6, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad6.mmc1_dat6 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_AD7, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad7.mmc1_dat7 */
 		>;
 	};
 };
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-osd335x-common.dtsi b/arch/arm/boot/dts/am335x-osd335x-common.dtsi
--- a/arch/arm/boot/dts/am335x-osd335x-common.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-osd335x-common.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -36,8 +36,8 @@
 &am33xx_pinmux {
 	i2c0_pins: pinmux-i2c0-pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x988, PIN_INPUT_PULLUP | MUX_MODE0)	/* (C17) I2C0_SDA.I2C0_SDA */
-			AM33XX_IOPAD(0x98c, PIN_INPUT_PULLUP | MUX_MODE0)	/* (C16) I2C0_SCL.I2C0_SCL */
+			AM33XX_PADCONF(AM335X_PIN_I2C0_SDA, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_I2C0_SCL, PIN_INPUT_PULLUP, MUX_MODE0)
 		>;
 	};
 };
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-pepper.dts b/arch/arm/boot/dts/am335x-pepper.dts
--- a/arch/arm/boot/dts/am335x-pepper.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-pepper.dts	2020-06-19 21:00:31.404227715 -0400
@@ -654,3 +654,7 @@
 		>;
 	};
 };
+
+&sgx {
+	status = "okay";
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-peripheral-can0.dtsi b/arch/arm/boot/dts/am335x-peripheral-can0.dtsi
--- a/arch/arm/boot/dts/am335x-peripheral-can0.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-peripheral-can0.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,13 @@
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+&dcan0 {
+	pinctrl-names = "default";
+
+	status = "okay";
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-peripheral-panel-1024x600-24bit.dtsi b/arch/arm/boot/dts/am335x-peripheral-panel-1024x600-24bit.dtsi
--- a/arch/arm/boot/dts/am335x-peripheral-panel-1024x600-24bit.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-peripheral-panel-1024x600-24bit.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,50 @@
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+&lcdc {
+	status = "okay";
+	blue-and-red-wiring = "crossed";
+};
+
+/ {
+	panel {
+		status = "okay";
+		compatible = "ti,tilcdc,panel";
+		pinctrl-names = "default";
+
+		panel-info {
+			ac-bias           = <255>;
+			ac-bias-intrpt    = <0>;
+			dma-burst-sz      = <16>;
+			bpp               = <32>;
+			fdd               = <0x80>;
+			sync-edge         = <0>;
+			sync-ctrl         = <0>;
+			raster-order      = <1>;
+			fifo-th           = <0>;
+		};
+		display-timings {
+			native-mode = <&timing0>;
+			timing0: 1024x600 {
+				clock-frequency = <36000000>;
+				hactive = <1024>;
+				vactive = <600>;
+				hfront-porch = <1>;
+				hback-porch = <45>;
+				hsync-len = <30>;
+				vback-porch = <22>;
+				vfront-porch = <12>;
+				vsync-len = <2>;
+				hsync-active = <1>;
+				vsync-active = <1>;
+				de-active = <1>;
+				pixelclk-active = <0>;
+			};
+		};
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-peripheral-ttyS1.dtsi b/arch/arm/boot/dts/am335x-peripheral-ttyS1.dtsi
--- a/arch/arm/boot/dts/am335x-peripheral-ttyS1.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-peripheral-ttyS1.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,13 @@
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+&uart1 {
+	pinctrl-names = "default";
+
+	status = "okay";
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-peripheral-ttyS2.dtsi b/arch/arm/boot/dts/am335x-peripheral-ttyS2.dtsi
--- a/arch/arm/boot/dts/am335x-peripheral-ttyS2.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-peripheral-ttyS2.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,13 @@
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+&uart2 {
+	pinctrl-names = "default";
+
+	status = "okay";
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-peripheral-ttyS4.dtsi b/arch/arm/boot/dts/am335x-peripheral-ttyS4.dtsi
--- a/arch/arm/boot/dts/am335x-peripheral-ttyS4.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-peripheral-ttyS4.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,13 @@
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+&uart4 {
+	pinctrl-names = "default";
+
+	status = "okay";
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-pocketbeagle.dts b/arch/arm/boot/dts/am335x-pocketbeagle.dts
--- a/arch/arm/boot/dts/am335x-pocketbeagle.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-pocketbeagle.dts	2020-06-19 21:00:31.404227715 -0400
@@ -9,12 +9,18 @@
 #include "am33xx.dtsi"
 #include "am335x-osd335x-common.dtsi"
 
+&sgx {
+	status = "okay";
+};
+
 / {
 	model = "TI AM335x PocketBeagle";
 	compatible = "ti,am335x-pocketbeagle", "ti,am335x-bone", "ti,am33xx";
 
 	chosen {
 		stdout-path = &uart0;
+		base_dtb = "am335x-pocketbeagle.dts";
+		base_dtb_timestamp = __TIMESTAMP__;
 	};
 
 	leds {
@@ -60,78 +66,904 @@
 };
 
 &am33xx_pinmux {
-	i2c2_pins: pinmux-i2c2-pins {
-		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x97c, PIN_INPUT_PULLUP | MUX_MODE3)	/* (D17) uart1_rtsn.I2C2_SCL */
-			AM33XX_IOPAD(0x978, PIN_INPUT_PULLUP | MUX_MODE3)	/* (D18) uart1_ctsn.I2C2_SDA */
-		>;
-	};
-
-	ehrpwm0_pins: pinmux-ehrpwm0-pins {
-		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x990, PIN_OUTPUT_PULLDOWN | MUX_MODE1)	/* (A13) mcasp0_aclkx.ehrpwm0A */
-		>;
-	};
-
-	ehrpwm1_pins: pinmux-ehrpwm1-pins {
-		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x848, PIN_OUTPUT_PULLDOWN | MUX_MODE6)	/* (U14) gpmc_a2.ehrpwm1A */
-		>;
-	};
+//	i2c2_pins: pinmux-i2c2-pins {
+//		pinctrl-single,pins = <
+//			AM33XX_PADCONF(AM335X_PIN_UART1_RTSN, PIN_INPUT_PULLUP, MUX_MODE3)	/* (D17) uart1_rtsn.I2C2_SCL */
+//			AM33XX_PADCONF(AM335X_PIN_UART1_CTSN, PIN_INPUT_PULLUP, MUX_MODE3)	/* (D18) uart1_ctsn.I2C2_SDA */
+//		>;
+//	};
+
+//	ehrpwm0_pins: pinmux-ehrpwm0-pins {
+//		pinctrl-single,pins = <
+//			AM33XX_PADCONF(AM335X_PIN_MCASP0_ACLKX, PIN_OUTPUT_PULLDOWN, MUX_MODE1)	/* (A13) mcasp0_aclkx.ehrpwm0A */
+//		>;
+//	};
+
+//	ehrpwm1_pins: pinmux-ehrpwm1-pins {
+//		pinctrl-single,pins = <
+//			AM33XX_PADCONF(AM335X_PIN_GPMC_A2, PIN_OUTPUT_PULLDOWN, MUX_MODE6)	/* (U14) gpmc_a2.ehrpwm1A */
+//		>;
+//	};
 
 	mmc0_pins: pinmux-mmc0-pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x960, PIN_INPUT | MUX_MODE7)		/* (C15) spi0_cs1.gpio0[6] */
-			AM33XX_IOPAD(0x8fc, PIN_INPUT_PULLUP | MUX_MODE0)	/* (G16) mmc0_dat0.mmc0_dat0 */
-			AM33XX_IOPAD(0x8f8, PIN_INPUT_PULLUP | MUX_MODE0)	/* (G15) mmc0_dat1.mmc0_dat1 */
-			AM33XX_IOPAD(0x8f4, PIN_INPUT_PULLUP | MUX_MODE0)	/* (F18) mmc0_dat2.mmc0_dat2 */
-			AM33XX_IOPAD(0x8f0, PIN_INPUT_PULLUP | MUX_MODE0)	/* (F17) mmc0_dat3.mmc0_dat3 */
-			AM33XX_IOPAD(0x904, PIN_INPUT_PULLUP | MUX_MODE0)	/* (G18) mmc0_cmd.mmc0_cmd */
-			AM33XX_IOPAD(0x900, PIN_INPUT_PULLUP | MUX_MODE0)	/* (G17) mmc0_clk.mmc0_clk */
-			AM33XX_IOPAD(0x9a0, PIN_INPUT | MUX_MODE4)		/* (B12) mcasp0_aclkr.mmc0_sdwp */
+			AM33XX_PADCONF(AM335X_PIN_SPI0_CS1, PIN_INPUT, MUX_MODE7)		/* (C15) spi0_cs1.gpio0[6] */
+			AM33XX_PADCONF(AM335X_PIN_MMC0_DAT0, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MMC0_DAT1, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MMC0_DAT2, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MMC0_DAT3, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MMC0_CMD, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MMC0_CLK, PIN_INPUT_PULLUP, MUX_MODE0)
 		>;
 	};
 
-	spi0_pins: pinmux-spi0-pins {
-		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x950, PIN_INPUT_PULLUP | MUX_MODE0)	/* (A17) spi0_sclk.spi0_sclk */
-			AM33XX_IOPAD(0x954, PIN_INPUT_PULLUP | MUX_MODE0)	/* (B17) spi0_d0.spi0_d0 */
-			AM33XX_IOPAD(0x958, PIN_INPUT_PULLUP | MUX_MODE0)	/* (B16) spi0_d1.spi0_d1 */
-			AM33XX_IOPAD(0x95c, PIN_INPUT_PULLUP | MUX_MODE0)	/* (A16) spi0_cs0.spi0_cs0 */
-		>;
-	};
-
-	spi1_pins: pinmux-spi1-pins {
-		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x964, PIN_INPUT_PULLUP | MUX_MODE4)	/* (C18) eCAP0_in_PWM0_out.spi1_sclk */
-			AM33XX_IOPAD(0x968, PIN_INPUT_PULLUP | MUX_MODE4)	/* (E18) uart0_ctsn.spi1_d0 */
-			AM33XX_IOPAD(0x96c, PIN_INPUT_PULLUP | MUX_MODE4)	/* (E17) uart0_rtsn.spi1_d1 */
-			AM33XX_IOPAD(0x9b0, PIN_INPUT_PULLUP | MUX_MODE4)	/* (A15) xdma_event_intr0.spi1_cs1 */
-		>;
-	};
+//	spi0_pins: pinmux-spi0-pins {
+//		pinctrl-single,pins = <
+//			AM33XX_PADCONF(AM335X_PIN_SPI0_SCLK, PIN_INPUT_PULLUP, MUX_MODE0)	/* (A17) spi0_sclk.spi0_sclk */
+//			AM33XX_PADCONF(AM335X_PIN_SPI0_D0, PIN_INPUT_PULLUP, MUX_MODE0)	/* (B17) spi0_d0.spi0_d0 */
+//			AM33XX_PADCONF(AM335X_PIN_SPI0_D1, PIN_INPUT_PULLUP, MUX_MODE0)	/* (B16) spi0_d1.spi0_d1 */
+//			AM33XX_PADCONF(AM335X_PIN_SPI0_CS0, PIN_INPUT_PULLUP, MUX_MODE0)	/* (A16) spi0_cs0.spi0_cs0 */
+//		>;
+//	};
+
+//	spi1_pins: pinmux-spi1-pins {
+//		pinctrl-single,pins = <
+//			AM33XX_PADCONF(AM335X_PIN_ECAP0_IN_PWM0_OUT, PIN_INPUT_PULLUP, MUX_MODE4)	/* (C18) eCAP0_in_PWM0_out.spi1_sclk */
+//			AM33XX_PADCONF(AM335X_PIN_UART0_CTSN, PIN_INPUT_PULLUP, MUX_MODE4)	/* (E18) uart0_ctsn.spi1_d0 */
+//			AM33XX_PADCONF(AM335X_PIN_UART0_RTSN, PIN_INPUT_PULLUP, MUX_MODE4)	/* (E17) uart0_rtsn.spi1_d1 */
+//			AM33XX_PADCONF(AM335X_PIN_XDMA_EVENT_INTR0, PIN_INPUT_PULLUP, MUX_MODE4)	/* (A15) xdma_event_intr0.spi1_cs1 */
+//		>;
+//	};
 
 	usr_leds_pins: pinmux-usr-leds-pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x854, PIN_OUTPUT | MUX_MODE7)		/* (V15) gpmc_a5.gpio1[21] - USR_LED_0 */
-			AM33XX_IOPAD(0x858, PIN_OUTPUT | MUX_MODE7)		/* (U15) gpmc_a6.gpio1[22] - USR_LED_1 */
-			AM33XX_IOPAD(0x85c, PIN_OUTPUT | MUX_MODE7)		/* (T15) gpmc_a7.gpio1[23] - USR_LED_2 */
-			AM33XX_IOPAD(0x860, PIN_OUTPUT | MUX_MODE7)		/* (V16) gpmc_a8.gpio1[24] - USR_LED_3 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_A5, PIN_OUTPUT, MUX_MODE7)		/* (V15) gpmc_a5.gpio1[21] - USR_LED_0 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_A6, PIN_OUTPUT, MUX_MODE7)		/* (U15) gpmc_a6.gpio1[22] - USR_LED_1 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_A7, PIN_OUTPUT, MUX_MODE7)		/* (T15) gpmc_a7.gpio1[23] - USR_LED_2 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_A8, PIN_OUTPUT, MUX_MODE7)		/* (V16) gpmc_a8.gpio1[24] - USR_LED_3 */
 		>;
 	};
 
 	uart0_pins: pinmux-uart0-pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x970, PIN_INPUT_PULLUP | MUX_MODE0)	/* (E15) uart0_rxd.uart0_rxd */
-			AM33XX_IOPAD(0x974, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* (E16) uart0_txd.uart0_txd */
+			AM33XX_PADCONF(AM335X_PIN_UART0_RXD, PIN_INPUT_PULLUP, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_UART0_TXD, PIN_OUTPUT_PULLDOWN, MUX_MODE0)
 		>;
 	};
 
-	uart4_pins: pinmux-uart4-pins {
-		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x870, PIN_INPUT_PULLUP | MUX_MODE6)	/* (T17) gpmc_wait0.uart4_rxd */
-			AM33XX_IOPAD(0x874, PIN_OUTPUT_PULLDOWN | MUX_MODE6)	/* (U17) gpmc_wpn.uart4_txd */
-		>;
-	};
+//	uart4_pins: pinmux-uart4-pins {
+//		pinctrl-single,pins = <
+//			AM33XX_PADCONF(AM335X_PIN_GPMC_WAIT0, PIN_INPUT_PULLUP, MUX_MODE6)	/* (T17) gpmc_wait0.uart4_rxd */
+//			AM33XX_PADCONF(AM335X_PIN_GPMC_WPN, PIN_OUTPUT_PULLDOWN, MUX_MODE6)	/* (U17) gpmc_wpn.uart4_txd */
+//		>;
+//	};
+
+	/************************/
+	/* P1 Header */
+	/************************/
+
+	/* P1_01                VIN-AC */
+
+	/* P1_02 (ZCZ ball R5) gpio2_23 */
+	P1_02_default_pin: pinmux_P1_02_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e4, PIN_INPUT | MUX_MODE7) >; };			/* lcd_hsync.gpio2_23 */
+	P1_02_gpio_pin: pinmux_P1_02_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_hsync.gpio2_23 */
+	P1_02_gpio_pu_pin: pinmux_P1_02_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_hsync.gpio2_23 */
+	P1_02_gpio_pd_pin: pinmux_P1_02_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_hsync.gpio2_23 */
+	P1_02_gpio_input_pin: pinmux_P1_02_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e4, PIN_INPUT | MUX_MODE7) >; };			/* lcd_hsync.gpio2_23 */
+	P1_02_pruout_pin: pinmux_P1_02_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* lcd_hsync.pru1_out9 */
+	P1_02_pruin_pin: pinmux_P1_02_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e4, PIN_INPUT | MUX_MODE6) >; };			/* lcd_hsync.pru1_in9 */
+
+	/* P1_03 (ZCZ ball F15)  usb1_vbus_out         */
+
+	/* P1_04 (ZCZ ball R6) gpio2_25 */
+	P1_04_default_pin: pinmux_P1_04_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_ac_bias_en.gpio2_25 */
+	P1_04_gpio_pin: pinmux_P1_04_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ec, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_ac_bias_en.gpio2_25 */
+	P1_04_gpio_pu_pin: pinmux_P1_04_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_ac_bias_en.gpio2_25 */
+	P1_04_gpio_pd_pin: pinmux_P1_04_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_ac_bias_en.gpio2_25 */
+	P1_04_gpio_input_pin: pinmux_P1_04_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ec, PIN_INPUT | MUX_MODE7) >; };			/* lcd_ac_bias_en.gpio2_25 */
+	P1_04_pruout_pin: pinmux_P1_04_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* lcd_ac_bias_en.pru1_out11 */
+	P1_04_pruin_pin: pinmux_P1_04_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08ec, PIN_INPUT | MUX_MODE6) >; };			/* lcd_ac_bias_en.pru1_in11 */
+
+	/* P1_05 (ZCZ ball T18)  usb1_vbus_in         */
+
+	/* P1_06 (ZCZ ball A16) spi0_cs0 */
+	P1_06_default_pin: pinmux_P1_06_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };	/* spi0_cs0.spi0_cs0 */
+	P1_06_gpio_pin: pinmux_P1_06_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x095c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* spi0_cs0.gpio0_5 */
+	P1_06_gpio_pu_pin: pinmux_P1_06_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* spi0_cs0.gpio0_5 */
+	P1_06_gpio_pd_pin: pinmux_P1_06_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* spi0_cs0.gpio0_5 */
+	P1_06_gpio_input_pin: pinmux_P1_06_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x095c, PIN_INPUT | MUX_MODE7) >; };			/* spi0_cs0.gpio0_5 */
+	P1_06_spi_cs_pin: pinmux_P1_06_spi_cs_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };	/* spi0_cs0.spi0_cs0 */
+	P1_06_i2c_pin: pinmux_P1_06_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* spi0_cs0.i2c1_scl */
+	P1_06_pwm_pin: pinmux_P1_06_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; };	/* spi0_cs0.ehrpwm0_synci */
+	P1_06_pru_uart_pin: pinmux_P1_06_pru_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* spi0_cs0.pr1_uart0_txd */
+
+	/* P1_07                VIN-USB */
+
+	/* P1_08 (ZCZ ball A17) spi0_sclk */
+	P1_08_default_pin: pinmux_P1_08_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };	/* spi0_sclk.spi0_sclk */
+	P1_08_gpio_pin: pinmux_P1_08_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* spi0_sclk.gpio0_2 */
+	P1_08_gpio_pu_pin: pinmux_P1_08_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* spi0_sclk.gpio0_2 */
+	P1_08_gpio_pd_pin: pinmux_P1_08_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* spi0_sclk.gpio0_2 */
+	P1_08_gpio_input_pin: pinmux_P1_08_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_INPUT | MUX_MODE7) >; };			/* spi0_sclk.gpio0_2 */
+	P1_08_spi_sclk_pin: pinmux_P1_08_spi_sclk_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };	/* spi0_sclk.spi0_sclk */
+	P1_08_uart_pin: pinmux_P1_08_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* spi0_sclk.uart2_rxd */
+	P1_08_i2c_pin: pinmux_P1_08_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* spi0_sclk.i2c2_sda */
+	P1_08_pwm_pin: pinmux_P1_08_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; };	/* spi0_sclk.ehrpwm0a */
+	P1_08_pru_uart_pin: pinmux_P1_08_pru_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* spi0_sclk.pr1_uart0_cts_n */
+
+	/* P1_09 (ZCZ ball R18)  USB1-DN         */
+
+	/* P1_10 (ZCZ ball B17) spi0_d0 */
+	P1_10_default_pin: pinmux_P1_10_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };	/* spi0_d0.spi0_d0 */
+	P1_10_gpio_pin: pinmux_P1_10_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* spi0_d0.gpio0_3 */
+	P1_10_gpio_pu_pin: pinmux_P1_10_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* spi0_d0.gpio0_3 */
+	P1_10_gpio_pd_pin: pinmux_P1_10_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* spi0_d0.gpio0_3 */
+	P1_10_gpio_input_pin: pinmux_P1_10_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_INPUT | MUX_MODE7) >; };			/* spi0_d0.gpio0_3 */
+	P1_10_spi_pin: pinmux_P1_10_spi_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };	/* spi0_d0.spi0_d0 */
+	P1_10_uart_pin: pinmux_P1_10_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* spi0_d0.uart2_txd */
+	P1_10_i2c_pin: pinmux_P1_10_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* spi0_d0.i2c2_scl */
+	P1_10_pwm_pin: pinmux_P1_10_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; };	/* spi0_d0.ehrpwm0b */
+	P1_10_pru_uart_pin: pinmux_P1_10_pru_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* spi0_d0.pr1_uart0_rts_n */
+
+	/* P1_11 (ZCZ ball R17)  USB1-DP         */
+
+	/* P1_12 (ZCZ ball B16) spi0_d1 */
+	P1_12_default_pin: pinmux_P1_12_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };	/* spi0_d1.spi0_d1 */
+	P1_12_gpio_pin: pinmux_P1_12_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0958, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* spi0_d1.gpio0_4 */
+	P1_12_gpio_pu_pin: pinmux_P1_12_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* spi0_d1.gpio0_4 */
+	P1_12_gpio_pd_pin: pinmux_P1_12_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* spi0_d1.gpio0_4 */
+	P1_12_gpio_input_pin: pinmux_P1_12_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0958, PIN_INPUT | MUX_MODE7) >; };			/* spi0_d1.gpio0_4 */
+	P1_12_spi_pin: pinmux_P1_12_spi_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };	/* spi0_d1.spi0_d1 */
+	P1_12_i2c_pin: pinmux_P1_12_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* spi0_d1.i2c1_sda */
+	P1_12_pwm_pin: pinmux_P1_12_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; };	/* spi0_d1.ehrpwm0_tripzone_input */
+	P1_12_pru_uart_pin: pinmux_P1_12_pru_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* spi0_d1.pr1_uart0_rxd */
+
+	/* P1_13 (ZCZ ball P17)  USB1-ID         */
+
+	/* P1_14                VOUT-3.3V */
+
+	/* P1_15                GND */
+
+	/* P1_16                GND */
+
+	/* P1_17 (ZCZ ball A9)  VREFN         */
+
+	/* P1_18 (ZCZ ball B9)  VREFP         */
+
+	/* P1_19 (ZCZ ball B6)  AIN0         */
+
+	/* P1_20 (ZCZ ball D14) gpio0_20 */
+	P1_20_default_pin: pinmux_P1_20_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* xdma_event_intr1.gpio0_20 */
+	P1_20_gpio_pin: pinmux_P1_20_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09b4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* xdma_event_intr1.gpio0_20 */
+	P1_20_gpio_pu_pin: pinmux_P1_20_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* xdma_event_intr1.gpio0_20 */
+	P1_20_gpio_pd_pin: pinmux_P1_20_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* xdma_event_intr1.gpio0_20 */
+	P1_20_gpio_input_pin: pinmux_P1_20_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09b4, PIN_INPUT | MUX_MODE7) >; };			/* xdma_event_intr1.gpio0_20 */
+	P1_20_pruin_pin: pinmux_P1_20_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09b4, PIN_INPUT | MUX_MODE5) >; };			/* xdma_event_intr1.pru0_in16 */
+
+	/* P1_21 (ZCZ ball C7)  AIN1         */
+
+	/* P1_22                GND */
+
+	/* P1_23 (ZCZ ball B7)  AIN2         */
+
+	/* P1_24                VOUT-5V */
+
+	/* P1_25 (ZCZ ball A7)  AIN3         */
+
+	/* P1_26 (ZCZ ball D18) i2c2_sda */
+	P1_26_default_pin: pinmux_P1_26_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* uart1_ctsn.i2c2_sda */
+	P1_26_gpio_pin: pinmux_P1_26_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* uart1_ctsn.gpio0_12 */
+	P1_26_gpio_pu_pin: pinmux_P1_26_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* uart1_ctsn.gpio0_12 */
+	P1_26_gpio_pd_pin: pinmux_P1_26_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* uart1_ctsn.gpio0_12 */
+	P1_26_gpio_input_pin: pinmux_P1_26_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_INPUT | MUX_MODE7) >; };			/* uart1_ctsn.gpio0_12 */
+	P1_26_can_pin: pinmux_P1_26_can_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | MUX_MODE2) >; };		/* uart1_ctsn.dcan0_tx */
+	P1_26_i2c_pin: pinmux_P1_26_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* uart1_ctsn.i2c2_sda */
+	P1_26_spi_cs_pin: pinmux_P1_26_spi_cs_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* uart1_ctsn.spi1_cs0 */
+	P1_26_pru_uart_pin: pinmux_P1_26_pru_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; };	/* uart1_ctsn.pr1_uart0_cts_n */
+
+	/* P1_27 (ZCZ ball C8)  AIN4         */
+
+	/* P1_28 (ZCZ ball D17) i2c2_scl */
+	P1_28_default_pin: pinmux_P1_28_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* uart1_rtsn.i2c2_scl */
+	P1_28_gpio_pin: pinmux_P1_28_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* uart1_rtsn.gpio0_13 */
+	P1_28_gpio_pu_pin: pinmux_P1_28_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* uart1_rtsn.gpio0_13 */
+	P1_28_gpio_pd_pin: pinmux_P1_28_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* uart1_rtsn.gpio0_13 */
+	P1_28_gpio_input_pin: pinmux_P1_28_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_INPUT | MUX_MODE7) >; };			/* uart1_rtsn.gpio0_13 */
+	P1_28_can_pin: pinmux_P1_28_can_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_INPUT_PULLUP | MUX_MODE2) >; };		/* uart1_rtsn.dcan0_rx */
+	P1_28_i2c_pin: pinmux_P1_28_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* uart1_rtsn.i2c2_scl */
+	P1_28_spi_cs_pin: pinmux_P1_28_spi_cs_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* uart1_rtsn.spi1_cs1 */
+	P1_28_pru_uart_pin: pinmux_P1_28_pru_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; };	/* uart1_rtsn.pr1_uart0_rts_n */
+
+	/* P1_29 (ZCZ ball A14) pru0_in7 */
+	P1_29_default_pin: pinmux_P1_29_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09ac, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_ahclkx.pru0_in7 */
+	P1_29_gpio_pin: pinmux_P1_29_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09ac, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* mcasp0_ahclkx.gpio3_21 */
+	P1_29_gpio_pu_pin: pinmux_P1_29_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_ahclkx.gpio3_21 */
+	P1_29_gpio_pd_pin: pinmux_P1_29_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_ahclkx.gpio3_21 */
+	P1_29_gpio_input_pin: pinmux_P1_29_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09ac, PIN_INPUT | MUX_MODE7) >; };			/* mcasp0_ahclkx.gpio3_21 */
+	P1_29_qep_pin: pinmux_P1_29_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* mcasp0_ahclkx.eqep0_strobe */
+	P1_29_pruout_pin: pinmux_P1_29_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* mcasp0_ahclkx.pru0_out7 */
+	P1_29_pruin_pin: pinmux_P1_29_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09ac, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_ahclkx.pru0_in7 */
+
+	/* P1_30 (ZCZ ball E16) uart0_txd */
+	P1_30_default_pin: pinmux_P1_30_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0974, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };	/* uart0_txd.uart0_txd */
+	P1_30_gpio_pin: pinmux_P1_30_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0974, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* uart0_txd.gpio1_11 */
+	P1_30_gpio_pu_pin: pinmux_P1_30_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0974, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* uart0_txd.gpio1_11 */
+	P1_30_gpio_pd_pin: pinmux_P1_30_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0974, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* uart0_txd.gpio1_11 */
+	P1_30_gpio_input_pin: pinmux_P1_30_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0974, PIN_INPUT | MUX_MODE7) >; };			/* uart0_txd.gpio1_11 */
+	P1_30_uart_pin: pinmux_P1_30_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0974, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };	/* uart0_txd.uart0_txd */
+	P1_30_spi_cs_pin: pinmux_P1_30_spi_cs_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0974, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* uart0_txd.spi1_cs1 */
+	P1_30_can_pin: pinmux_P1_30_can_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0974, PIN_INPUT_PULLUP | MUX_MODE2) >; };		/* uart0_txd.dcan0_rx */
+	P1_30_i2c_pin: pinmux_P1_30_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0974, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* uart0_txd.i2c2_scl */
+	P1_30_pruout_pin: pinmux_P1_30_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0974, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* uart0_txd.pru1_out15 */
+	P1_30_pruin_pin: pinmux_P1_30_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0974, PIN_INPUT | MUX_MODE6) >; };			/* uart0_txd.pru1_in15 */
+
+	/* P1_31 (ZCZ ball B12) pru0_in4 */
+	P1_31_default_pin: pinmux_P1_31_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a0, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_aclkr.pru0_in4 */
+	P1_31_gpio_pin: pinmux_P1_31_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* mcasp0_aclkr.gpio3_18 */
+	P1_31_gpio_pu_pin: pinmux_P1_31_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_aclkr.gpio3_18 */
+	P1_31_gpio_pd_pin: pinmux_P1_31_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_aclkr.gpio3_18 */
+	P1_31_gpio_input_pin: pinmux_P1_31_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a0, PIN_INPUT | MUX_MODE7) >; };			/* mcasp0_aclkr.gpio3_18 */
+	P1_31_qep_pin: pinmux_P1_31_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* mcasp0_aclkr.eqep0a_in */
+	P1_31_pruout_pin: pinmux_P1_31_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* mcasp0_aclkr.pru0_out4 */
+	P1_31_pruin_pin: pinmux_P1_31_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a0, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_aclkr.pru0_in4 */
+
+	/* P1_32 (ZCZ ball E15) uart0_rxd */
+	P1_32_default_pin: pinmux_P1_32_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };	/* uart0_rxd.uart0_rxd */
+	P1_32_gpio_pin: pinmux_P1_32_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0970, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* uart0_rxd.gpio1_10 */
+	P1_32_gpio_pu_pin: pinmux_P1_32_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* uart0_rxd.gpio1_10 */
+	P1_32_gpio_pd_pin: pinmux_P1_32_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* uart0_rxd.gpio1_10 */
+	P1_32_gpio_input_pin: pinmux_P1_32_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0970, PIN_INPUT | MUX_MODE7) >; };			/* uart0_rxd.gpio1_10 */
+	P1_32_uart_pin: pinmux_P1_32_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };	/* uart0_rxd.uart0_rxd */
+	P1_32_spi_cs_pin: pinmux_P1_32_spi_cs_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* uart0_rxd.spi1_cs0 */
+	P1_32_can_pin: pinmux_P1_32_can_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLUP | MUX_MODE2) >; };		/* uart0_rxd.dcan0_tx */
+	P1_32_i2c_pin: pinmux_P1_32_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* uart0_rxd.i2c2_sda */
+	P1_32_pruout_pin: pinmux_P1_32_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* uart0_rxd.pru1_out14 */
+	P1_32_pruin_pin: pinmux_P1_32_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0970, PIN_INPUT | MUX_MODE6) >; };			/* uart0_rxd.pru1_in14 */
+
+	/* P1_33 (ZCZ ball B13) pru0_in1 */
+	P1_33_default_pin: pinmux_P1_33_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0994, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_fsx.pru0_in1 */
+	P1_33_gpio_pin: pinmux_P1_33_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0994, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* mcasp0_fsx.gpio3_15 */
+	P1_33_gpio_pu_pin: pinmux_P1_33_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_fsx.gpio3_15 */
+	P1_33_gpio_pd_pin: pinmux_P1_33_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_fsx.gpio3_15 */
+	P1_33_gpio_input_pin: pinmux_P1_33_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0994, PIN_INPUT | MUX_MODE7) >; };			/* mcasp0_fsx.gpio3_15 */
+	P1_33_pwm_pin: pinmux_P1_33_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; };	/* mcasp0_fsx.ehrpwm0b */
+	P1_33_spi_pin: pinmux_P1_33_spi_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* mcasp0_fsx.spi1_d0 */
+	P1_33_pruout_pin: pinmux_P1_33_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* mcasp0_fsx.pru0_out1 */
+	P1_33_pruin_pin: pinmux_P1_33_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0994, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_fsx.pru0_in1 */
+
+	/* P1_34 (ZCZ ball T11) gpio0_26 */
+	P1_34_default_pin: pinmux_P1_34_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad10.gpio0_26 */
+	P1_34_gpio_pin: pinmux_P1_34_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0828, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad10.gpio0_26 */
+	P1_34_gpio_pu_pin: pinmux_P1_34_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad10.gpio0_26 */
+	P1_34_gpio_pd_pin: pinmux_P1_34_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad10.gpio0_26 */
+	P1_34_gpio_input_pin: pinmux_P1_34_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0828, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad10.gpio0_26 */
+	P1_34_pwm_pin: pinmux_P1_34_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; };	/* gpmc_ad10.ehrpwm2_tripzone_input */
+
+	/* P1_35 (ZCZ ball V5) pru1_in10 */
+	P1_35_default_pin: pinmux_P1_35_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e8, PIN_INPUT | MUX_MODE6) >; };			/* lcd_pclk.pru1_in10 */
+	P1_35_gpio_pin: pinmux_P1_35_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_pclk.gpio2_24 */
+	P1_35_gpio_pu_pin: pinmux_P1_35_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_pclk.gpio2_24 */
+	P1_35_gpio_pd_pin: pinmux_P1_35_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_pclk.gpio2_24 */
+	P1_35_gpio_input_pin: pinmux_P1_35_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e8, PIN_INPUT | MUX_MODE7) >; };			/* lcd_pclk.gpio2_24 */
+	P1_35_pruout_pin: pinmux_P1_35_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* lcd_pclk.pru1_out10 */
+	P1_35_pruin_pin: pinmux_P1_35_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e8, PIN_INPUT | MUX_MODE6) >; };			/* lcd_pclk.pru1_in10 */
+
+	/* P1_36 (ZCZ ball A13) ehrpwm0a */
+	P1_36_default_pin: pinmux_P1_36_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; };	/* mcasp0_aclkx.ehrpwm0a */
+	P1_36_gpio_pin: pinmux_P1_36_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0990, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* mcasp0_aclkx.gpio3_14 */
+	P1_36_gpio_pu_pin: pinmux_P1_36_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_aclkx.gpio3_14 */
+	P1_36_gpio_pd_pin: pinmux_P1_36_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_aclkx.gpio3_14 */
+	P1_36_gpio_input_pin: pinmux_P1_36_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0990, PIN_INPUT | MUX_MODE7) >; };			/* mcasp0_aclkx.gpio3_14 */
+	P1_36_pwm_pin: pinmux_P1_36_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; };	/* mcasp0_aclkx.ehrpwm0a */
+	P1_36_spi_sclk_pin: pinmux_P1_36_spi_sclk_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* mcasp0_aclkx.spi1_sclk */
+	P1_36_pruout_pin: pinmux_P1_36_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* mcasp0_aclkx.pru0_out0 */
+	P1_36_pruin_pin: pinmux_P1_36_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0990, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_aclkx.pru0_in0 */
+
+
+	/************************/
+	/* P2 Header */
+	/************************/
+
+	/* P2_01 (ZCZ ball U14) ehrpwm1a */
+	P2_01_default_pin: pinmux_P2_01_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; };	/* gpmc_a2.ehrpwm1a */
+	P2_01_gpio_pin: pinmux_P2_01_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0848, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_a2.gpio1_18 */
+	P2_01_gpio_pu_pin: pinmux_P2_01_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a2.gpio1_18 */
+	P2_01_gpio_pd_pin: pinmux_P2_01_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a2.gpio1_18 */
+	P2_01_gpio_input_pin: pinmux_P2_01_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0848, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_a2.gpio1_18 */
+	P2_01_pwm_pin: pinmux_P2_01_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; };	/* gpmc_a2.ehrpwm1a */
+
+	/* P2_02 (ZCZ ball V17) gpio1_27 */
+	P2_02_default_pin: pinmux_P2_02_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x086c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a11.gpio1_27 */
+	P2_02_gpio_pin: pinmux_P2_02_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x086c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_a11.gpio1_27 */
+	P2_02_gpio_pu_pin: pinmux_P2_02_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x086c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a11.gpio1_27 */
+	P2_02_gpio_pd_pin: pinmux_P2_02_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x086c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a11.gpio1_27 */
+	P2_02_gpio_input_pin: pinmux_P2_02_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x086c, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_a11.gpio1_27 */
+
+	/* P2_03 (ZCZ ball T10) gpio0_23 */
+	P2_03_default_pin: pinmux_P2_03_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad9.gpio0_23 */
+	P2_03_gpio_pin: pinmux_P2_03_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0824, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad9.gpio0_23 */
+	P2_03_gpio_pu_pin: pinmux_P2_03_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad9.gpio0_23 */
+	P2_03_gpio_pd_pin: pinmux_P2_03_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad9.gpio0_23 */
+	P2_03_gpio_input_pin: pinmux_P2_03_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0824, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad9.gpio0_23 */
+	P2_03_pwm_pin: pinmux_P2_03_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; };	/* gpmc_ad9.ehrpwm2b */
+
+	/* P2_04 (ZCZ ball T16) gpio1_26 */
+	P2_04_default_pin: pinmux_P2_04_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0868, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a10.gpio1_26 */
+	P2_04_gpio_pin: pinmux_P2_04_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0868, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_a10.gpio1_26 */
+	P2_04_gpio_pu_pin: pinmux_P2_04_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0868, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a10.gpio1_26 */
+	P2_04_gpio_pd_pin: pinmux_P2_04_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0868, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a10.gpio1_26 */
+	P2_04_gpio_input_pin: pinmux_P2_04_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0868, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_a10.gpio1_26 */
+
+	/* P2_05 (ZCZ ball T17) uart4_rxd */
+	P2_05_default_pin: pinmux_P2_05_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; };	/* gpmc_wait0.uart4_rxd */
+	P2_05_gpio_pin: pinmux_P2_05_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0870, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_wait0.gpio0_30 */
+	P2_05_gpio_pu_pin: pinmux_P2_05_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_wait0.gpio0_30 */
+	P2_05_gpio_pd_pin: pinmux_P2_05_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_wait0.gpio0_30 */
+	P2_05_gpio_input_pin: pinmux_P2_05_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0870, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_wait0.gpio0_30 */
+	P2_05_uart_pin: pinmux_P2_05_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; };	/* gpmc_wait0.uart4_rxd */
+
+	/* P2_06 (ZCZ ball U16) gpio1_25 */
+	P2_06_default_pin: pinmux_P2_06_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0864, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a9.gpio1_25 */
+	P2_06_gpio_pin: pinmux_P2_06_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0864, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_a9.gpio1_25 */
+	P2_06_gpio_pu_pin: pinmux_P2_06_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0864, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a9.gpio1_25 */
+	P2_06_gpio_pd_pin: pinmux_P2_06_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0864, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a9.gpio1_25 */
+	P2_06_gpio_input_pin: pinmux_P2_06_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0864, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_a9.gpio1_25 */
+
+	/* P2_07 (ZCZ ball U17) uart4_txd */
+	P2_07_default_pin: pinmux_P2_07_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; };	/* gpmc_wpn.uart4_txd */
+	P2_07_gpio_pin: pinmux_P2_07_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0874, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_wpn.gpio0_31 */
+	P2_07_gpio_pu_pin: pinmux_P2_07_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_wpn.gpio0_31 */
+	P2_07_gpio_pd_pin: pinmux_P2_07_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_wpn.gpio0_31 */
+	P2_07_gpio_input_pin: pinmux_P2_07_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0874, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_wpn.gpio0_31 */
+	P2_07_uart_pin: pinmux_P2_07_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; };	/* gpmc_wpn.uart4_txd */
+
+	/* P2_08 (ZCZ ball U18) gpio1_28 */
+	P2_08_default_pin: pinmux_P2_08_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_be1n.gpio1_28 */
+	P2_08_gpio_pin: pinmux_P2_08_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0878, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_be1n.gpio1_28 */
+	P2_08_gpio_pu_pin: pinmux_P2_08_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_be1n.gpio1_28 */
+	P2_08_gpio_pd_pin: pinmux_P2_08_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_be1n.gpio1_28 */
+	P2_08_gpio_input_pin: pinmux_P2_08_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0878, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_be1n.gpio1_28 */
+
+	/* P2_09 (ZCZ ball D15) i2c1_scl */
+	P2_09_default_pin: pinmux_P2_09_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* uart1_txd.i2c1_scl */
+	P2_09_gpio_pin: pinmux_P2_09_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* uart1_txd.gpio0_15 */
+	P2_09_gpio_pu_pin: pinmux_P2_09_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* uart1_txd.gpio0_15 */
+	P2_09_gpio_pd_pin: pinmux_P2_09_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* uart1_txd.gpio0_15 */
+	P2_09_gpio_input_pin: pinmux_P2_09_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_INPUT | MUX_MODE7) >; };			/* uart1_txd.gpio0_15 */
+	P2_09_uart_pin: pinmux_P2_09_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };	/* uart1_txd.uart1_txd */
+	P2_09_can_pin: pinmux_P2_09_can_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_INPUT_PULLUP | MUX_MODE2) >; };		/* uart1_txd.dcan1_rx */
+	P2_09_i2c_pin: pinmux_P2_09_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* uart1_txd.i2c1_scl */
+	P2_09_pru_uart_pin: pinmux_P2_09_pru_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; };	/* uart1_txd.pr1_uart0_txd */
+	P2_09_pruin_pin: pinmux_P2_09_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0984, PIN_INPUT | MUX_MODE6) >; };			/* uart1_txd.pru0_in16 */
+
+	/* P2_10 (ZCZ ball R14) gpio1_20 */
+	P2_10_default_pin: pinmux_P2_10_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0850, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a4.gpio1_20 */
+	P2_10_gpio_pin: pinmux_P2_10_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0850, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_a4.gpio1_20 */
+	P2_10_gpio_pu_pin: pinmux_P2_10_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0850, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a4.gpio1_20 */
+	P2_10_gpio_pd_pin: pinmux_P2_10_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0850, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_a4.gpio1_20 */
+	P2_10_gpio_input_pin: pinmux_P2_10_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0850, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_a4.gpio1_20 */
+	P2_10_qep_pin: pinmux_P2_10_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0850, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; };	/* gpmc_a4.eqep1a_in */
+
+	/* P2_11 (ZCZ ball D16) i2c1_sda */
+	P2_11_default_pin: pinmux_P2_11_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* uart1_rxd.i2c1_sda */
+	P2_11_gpio_pin: pinmux_P2_11_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* uart1_rxd.gpio0_14 */
+	P2_11_gpio_pu_pin: pinmux_P2_11_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* uart1_rxd.gpio0_14 */
+	P2_11_gpio_pd_pin: pinmux_P2_11_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* uart1_rxd.gpio0_14 */
+	P2_11_gpio_input_pin: pinmux_P2_11_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_INPUT | MUX_MODE7) >; };			/* uart1_rxd.gpio0_14 */
+	P2_11_uart_pin: pinmux_P2_11_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };	/* uart1_rxd.uart1_rxd */
+	P2_11_can_pin: pinmux_P2_11_can_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | MUX_MODE2) >; };		/* uart1_rxd.dcan1_tx */
+	P2_11_i2c_pin: pinmux_P2_11_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* uart1_rxd.i2c1_sda */
+	P2_11_pru_uart_pin: pinmux_P2_11_pru_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; };	/* uart1_rxd.pr1_uart0_rxd */
+	P2_11_pruin_pin: pinmux_P2_11_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0980, PIN_INPUT | MUX_MODE6) >; };			/* uart1_rxd.pru1_in16 */
+
+	/* P2_12                POWER_BUTTON */
+
+	/* P2_13                VOUT-5V */
+
+	/* P2_14                BAT-VIN */
+
+	/* P2_15                GND */
+
+	/* P2_16                BAT-TEMP */
+
+	/* P2_17 (ZCZ ball V12) gpio2_1 */
+	P2_17_default_pin: pinmux_P2_17_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_clk.gpio2_1 */
+	P2_17_gpio_pin: pinmux_P2_17_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x088c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_clk.gpio2_1 */
+	P2_17_gpio_pu_pin: pinmux_P2_17_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_clk.gpio2_1 */
+	P2_17_gpio_pd_pin: pinmux_P2_17_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_clk.gpio2_1 */
+	P2_17_gpio_input_pin: pinmux_P2_17_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x088c, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_clk.gpio2_1 */
+
+	/* P2_18 (ZCZ ball U13) gpio1_15 */
+	P2_18_default_pin: pinmux_P2_18_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad15.gpio1_15 */
+	P2_18_gpio_pin: pinmux_P2_18_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x083c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad15.gpio1_15 */
+	P2_18_gpio_pu_pin: pinmux_P2_18_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad15.gpio1_15 */
+	P2_18_gpio_pd_pin: pinmux_P2_18_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad15.gpio1_15 */
+	P2_18_gpio_input_pin: pinmux_P2_18_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x083c, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad15.gpio1_15 */
+	P2_18_qep_pin: pinmux_P2_18_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* gpmc_ad15.eqep2_strobe */
+	P2_18_pru_ecap_pin: pinmux_P2_18_pru_ecap_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* gpmc_ad15.pr1_ecap0_ecap_capin_apwm_o */
+	P2_18_pruin_pin: pinmux_P2_18_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x083c, PIN_INPUT | MUX_MODE6) >; };			/* gpmc_ad15.pru0_in15 */
+
+	/* P2_19 (ZCZ ball U12) gpio0_27 */
+	P2_19_default_pin: pinmux_P2_19_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad11.gpio0_27 */
+	P2_19_gpio_pin: pinmux_P2_19_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x082c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad11.gpio0_27 */
+	P2_19_gpio_pu_pin: pinmux_P2_19_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad11.gpio0_27 */
+	P2_19_gpio_pd_pin: pinmux_P2_19_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad11.gpio0_27 */
+	P2_19_gpio_input_pin: pinmux_P2_19_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x082c, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad11.gpio0_27 */
+	P2_19_pwm_pin: pinmux_P2_19_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; };	/* gpmc_ad11.ehrpwm0_synco */
+
+	/* P2_20 (ZCZ ball T13) gpio2_0 */
+	P2_20_default_pin: pinmux_P2_20_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0888, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_csn3.gpio2_0 */
+	P2_20_gpio_pin: pinmux_P2_20_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0888, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_csn3.gpio2_0 */
+	P2_20_gpio_pu_pin: pinmux_P2_20_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0888, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_csn3.gpio2_0 */
+	P2_20_gpio_pd_pin: pinmux_P2_20_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0888, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_csn3.gpio2_0 */
+	P2_20_gpio_input_pin: pinmux_P2_20_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0888, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_csn3.gpio2_0 */
+
+	/* P2_21                GND */
+
+	/* P2_22 (ZCZ ball V13) gpio1_14 */
+	P2_22_default_pin: pinmux_P2_22_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad14.gpio1_14 */
+	P2_22_gpio_pin: pinmux_P2_22_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0838, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad14.gpio1_14 */
+	P2_22_gpio_pu_pin: pinmux_P2_22_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad14.gpio1_14 */
+	P2_22_gpio_pd_pin: pinmux_P2_22_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad14.gpio1_14 */
+	P2_22_gpio_input_pin: pinmux_P2_22_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0838, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad14.gpio1_14 */
+	P2_22_qep_pin: pinmux_P2_22_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* gpmc_ad14.eqep2_index */
+	P2_22_pruin_pin: pinmux_P2_22_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0838, PIN_INPUT | MUX_MODE6) >; };			/* gpmc_ad14.pru0_in14 */
+
+	/* P2_23                VOUT-3.3V */
+
+	/* P2_24 (ZCZ ball T12) gpio1_12 */
+	P2_24_default_pin: pinmux_P2_24_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad12.gpio1_12 */
+	P2_24_gpio_pin: pinmux_P2_24_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0830, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad12.gpio1_12 */
+	P2_24_gpio_pu_pin: pinmux_P2_24_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad12.gpio1_12 */
+	P2_24_gpio_pd_pin: pinmux_P2_24_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad12.gpio1_12 */
+	P2_24_gpio_input_pin: pinmux_P2_24_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0830, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad12.gpio1_12 */
+	P2_24_qep_pin: pinmux_P2_24_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* gpmc_ad12.eqep2a_in */
+	P2_24_pruout_pin: pinmux_P2_24_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; };	/* gpmc_ad12.pru0_out14 */
+
+	/* P2_25 (ZCZ ball E17) spi1_d1 */
+	P2_25_default_pin: pinmux_P2_25_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x096c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* uart0_rtsn.spi1_d1 */
+	P2_25_gpio_pin: pinmux_P2_25_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x096c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* uart0_rtsn.gpio1_9 */
+	P2_25_gpio_pu_pin: pinmux_P2_25_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x096c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* uart0_rtsn.gpio1_9 */
+	P2_25_gpio_pd_pin: pinmux_P2_25_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x096c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* uart0_rtsn.gpio1_9 */
+	P2_25_gpio_input_pin: pinmux_P2_25_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x096c, PIN_INPUT | MUX_MODE7) >; };			/* uart0_rtsn.gpio1_9 */
+	P2_25_uart_pin: pinmux_P2_25_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x096c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* uart0_rtsn.uart4_txd */
+	P2_25_can_pin: pinmux_P2_25_can_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x096c, PIN_INPUT_PULLUP | MUX_MODE2) >; };		/* uart0_rtsn.dcan1_rx */
+	P2_25_i2c_pin: pinmux_P2_25_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x096c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* uart0_rtsn.i2c1_scl */
+	P2_25_spi_pin: pinmux_P2_25_spi_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x096c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* uart0_rtsn.spi1_d1 */
+	P2_25_spi_cs_pin: pinmux_P2_25_spi_cs_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x096c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; };	/* uart0_rtsn.spi1_cs0 */
+
+	/* P2_26                RESET# */
+
+	/* P2_27 (ZCZ ball E18) spi1_d0 */
+	P2_27_default_pin: pinmux_P2_27_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0968, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* uart0_ctsn.spi1_d0 */
+	P2_27_gpio_pin: pinmux_P2_27_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0968, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* uart0_ctsn.gpio1_8 */
+	P2_27_gpio_pu_pin: pinmux_P2_27_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0968, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* uart0_ctsn.gpio1_8 */
+	P2_27_gpio_pd_pin: pinmux_P2_27_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0968, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* uart0_ctsn.gpio1_8 */
+	P2_27_gpio_input_pin: pinmux_P2_27_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0968, PIN_INPUT | MUX_MODE7) >; };			/* uart0_ctsn.gpio1_8 */
+	P2_27_uart_pin: pinmux_P2_27_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0968, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* uart0_ctsn.uart4_rxd */
+	P2_27_can_pin: pinmux_P2_27_can_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0968, PIN_OUTPUT_PULLUP | MUX_MODE2) >; };		/* uart0_ctsn.dcan1_tx */
+	P2_27_i2c_pin: pinmux_P2_27_i2c_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0968, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* uart0_ctsn.i2c1_sda */
+	P2_27_spi_pin: pinmux_P2_27_spi_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0968, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* uart0_ctsn.spi1_d0 */
+
+	/* P2_28 (ZCZ ball D13) pru0_in6 */
+	P2_28_default_pin: pinmux_P2_28_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a8, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_axr1.pru0_in6 */
+	P2_28_gpio_pin: pinmux_P2_28_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* mcasp0_axr1.gpio3_20 */
+	P2_28_gpio_pu_pin: pinmux_P2_28_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_axr1.gpio3_20 */
+	P2_28_gpio_pd_pin: pinmux_P2_28_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_axr1.gpio3_20 */
+	P2_28_gpio_input_pin: pinmux_P2_28_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a8, PIN_INPUT | MUX_MODE7) >; };			/* mcasp0_axr1.gpio3_20 */
+	P2_28_qep_pin: pinmux_P2_28_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* mcasp0_axr1.eqep0_index */
+	P2_28_pruout_pin: pinmux_P2_28_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* mcasp0_axr1.pru0_out6 */
+	P2_28_pruin_pin: pinmux_P2_28_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a8, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_axr1.pru0_in6 */
+
+	/* P2_29 (ZCZ ball C18) spi1_sclk */
+	P2_29_default_pin: pinmux_P2_29_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* eCAP0_in_PWM0_out.spi1_sclk */
+	P2_29_gpio_pin: pinmux_P2_29_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* eCAP0_in_PWM0_out.gpio0_7 */
+	P2_29_gpio_pu_pin: pinmux_P2_29_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* eCAP0_in_PWM0_out.gpio0_7 */
+	P2_29_gpio_pd_pin: pinmux_P2_29_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* eCAP0_in_PWM0_out.gpio0_7 */
+	P2_29_gpio_input_pin: pinmux_P2_29_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_INPUT | MUX_MODE7) >; };			/* eCAP0_in_PWM0_out.gpio0_7 */
+	P2_29_pwm_pin: pinmux_P2_29_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE0) >; };	/* eCAP0_in_PWM0_out.ecap0_in_pwm0_out */
+	P2_29_uart_pin: pinmux_P2_29_uart_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* eCAP0_in_PWM0_out.uart3_txd */
+	P2_29_spi_cs_pin: pinmux_P2_29_spi_cs_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* eCAP0_in_PWM0_out.spi1_cs1 */
+	P2_29_pru_ecap_pin: pinmux_P2_29_pru_ecap_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; };	/* eCAP0_in_PWM0_out.pr1_ecap0_ecap_capin_apwm_o */
+	P2_29_spi_sclk_pin: pinmux_P2_29_spi_sclk_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* eCAP0_in_PWM0_out.spi1_sclk */
+
+	/* P2_30 (ZCZ ball C12) pru0_in3 */
+	P2_30_default_pin: pinmux_P2_30_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_ahclkr.pru0_in3 */
+	P2_30_gpio_pin: pinmux_P2_30_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* mcasp0_ahclkr.gpio3_17 */
+	P2_30_gpio_pu_pin: pinmux_P2_30_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_ahclkr.gpio3_17 */
+	P2_30_gpio_pd_pin: pinmux_P2_30_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_ahclkr.gpio3_17 */
+	P2_30_gpio_input_pin: pinmux_P2_30_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_INPUT | MUX_MODE7) >; };			/* mcasp0_ahclkr.gpio3_17 */
+	P2_30_pwm_pin: pinmux_P2_30_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; };	/* mcasp0_ahclkr.ehrpwm0_synci */
+	P2_30_spi_cs_pin: pinmux_P2_30_spi_cs_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* mcasp0_ahclkr.spi1_cs0 */
+	P2_30_pruout_pin: pinmux_P2_30_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* mcasp0_ahclkr.pru0_out3 */
+	P2_30_pruin_pin: pinmux_P2_30_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x099c, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_ahclkr.pru0_in3 */
+
+	/* P2_31 (ZCZ ball A15) spi1_cs1 */
+	P2_31_default_pin: pinmux_P2_31_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09b0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* xdma_event_intr0.spi1_cs1 */
+	P2_31_gpio_pin: pinmux_P2_31_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09b0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* xdma_event_intr0.gpio0_19 */
+	P2_31_gpio_pu_pin: pinmux_P2_31_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09b0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* xdma_event_intr0.gpio0_19 */
+	P2_31_gpio_pd_pin: pinmux_P2_31_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09b0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* xdma_event_intr0.gpio0_19 */
+	P2_31_gpio_input_pin: pinmux_P2_31_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09b0, PIN_INPUT | MUX_MODE7) >; };			/* xdma_event_intr0.gpio0_19 */
+	P2_31_spi_cs_pin: pinmux_P2_31_spi_cs_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09b0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* xdma_event_intr0.spi1_cs1 */
+	P2_31_pruin_pin: pinmux_P2_31_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09b0, PIN_INPUT | MUX_MODE5) >; };			/* xdma_event_intr0.pru1_in16 */
+
+	/* P2_32 (ZCZ ball D12) pru0_in2 */
+	P2_32_default_pin: pinmux_P2_32_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0998, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_axr0.pru0_in2 */
+	P2_32_gpio_pin: pinmux_P2_32_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0998, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* mcasp0_axr0.gpio3_16 */
+	P2_32_gpio_pu_pin: pinmux_P2_32_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_axr0.gpio3_16 */
+	P2_32_gpio_pd_pin: pinmux_P2_32_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_axr0.gpio3_16 */
+	P2_32_gpio_input_pin: pinmux_P2_32_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0998, PIN_INPUT | MUX_MODE7) >; };			/* mcasp0_axr0.gpio3_16 */
+	P2_32_pwm_pin: pinmux_P2_32_pwm_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; };	/* mcasp0_axr0.ehrpwm0_tripzone_input */
+	P2_32_spi_pin: pinmux_P2_32_spi_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* mcasp0_axr0.spi1_d1 */
+	P2_32_pruout_pin: pinmux_P2_32_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* mcasp0_axr0.pru0_out2 */
+	P2_32_pruin_pin: pinmux_P2_32_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0998, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_axr0.pru0_in2 */
+
+	/* P2_33 (ZCZ ball R12) gpio1_13 */
+	P2_33_default_pin: pinmux_P2_33_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad13.gpio1_13 */
+	P2_33_gpio_pin: pinmux_P2_33_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0834, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* gpmc_ad13.gpio1_13 */
+	P2_33_gpio_pu_pin: pinmux_P2_33_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad13.gpio1_13 */
+	P2_33_gpio_pd_pin: pinmux_P2_33_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* gpmc_ad13.gpio1_13 */
+	P2_33_gpio_input_pin: pinmux_P2_33_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0834, PIN_INPUT | MUX_MODE7) >; };			/* gpmc_ad13.gpio1_13 */
+	P2_33_qep_pin: pinmux_P2_33_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* gpmc_ad13.eqep2b_in */
+	P2_33_pruout_pin: pinmux_P2_33_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; };	/* gpmc_ad13.pru0_out15 */
+
+	/* P2_34 (ZCZ ball C13) pru0_in5 */
+	P2_34_default_pin: pinmux_P2_34_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a4, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_fsr.pru0_in5 */
+	P2_34_gpio_pin: pinmux_P2_34_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* mcasp0_fsr.gpio3_19 */
+	P2_34_gpio_pu_pin: pinmux_P2_34_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_fsr.gpio3_19 */
+	P2_34_gpio_pd_pin: pinmux_P2_34_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* mcasp0_fsr.gpio3_19 */
+	P2_34_gpio_input_pin: pinmux_P2_34_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a4, PIN_INPUT | MUX_MODE7) >; };			/* mcasp0_fsr.gpio3_19 */
+	P2_34_qep_pin: pinmux_P2_34_qep_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };	/* mcasp0_fsr.eqep0b_in */
+	P2_34_pruout_pin: pinmux_P2_34_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* mcasp0_fsr.pru0_out5 */
+	P2_34_pruin_pin: pinmux_P2_34_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x09a4, PIN_INPUT | MUX_MODE6) >; };			/* mcasp0_fsr.pru0_in5 */
+
+	/* P2_35 (ZCZ ball U5) gpio2_22 */
+	P2_35_default_pin: pinmux_P2_35_default_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e0, PIN_INPUT | MUX_MODE7) >; };			/* lcd_vsync.gpio2_22 */
+	P2_35_gpio_pin: pinmux_P2_35_gpio_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; };		/* lcd_vsync.gpio2_22 */
+	P2_35_gpio_pu_pin: pinmux_P2_35_gpio_pu_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };	/* lcd_vsync.gpio2_22 */
+	P2_35_gpio_pd_pin: pinmux_P2_35_gpio_pd_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; };	/* lcd_vsync.gpio2_22 */
+	P2_35_gpio_input_pin: pinmux_P2_35_gpio_input_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e0, PIN_INPUT | MUX_MODE7) >; };			/* lcd_vsync.gpio2_22 */
+	P2_35_pruout_pin: pinmux_P2_35_pruout_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; };	/* lcd_vsync.pru1_out8 */
+	P2_35_pruin_pin: pinmux_P2_35_pruin_pin { pinctrl-single,pins = <
+		AM33XX_IOPAD(0x08e0, PIN_INPUT | MUX_MODE6) >; };			/* lcd_vsync.pru1_in8 */
+
+	/* P2_36 (ZCZ ball C9)  AIN7         */
 };
 
 &epwmss0 {
@@ -141,7 +973,8 @@
 &ehrpwm0 {
 	status = "okay";
 	pinctrl-names = "default";
-	pinctrl-0 = <&ehrpwm0_pins>;
+	//pinctrl-0 = <&ehrpwm0_pins>;
+	pinctrl-0 = <>;
 };
 
 &epwmss1 {
@@ -151,7 +984,18 @@
 &ehrpwm1 {
 	status = "okay";
 	pinctrl-names = "default";
-	pinctrl-0 = <&ehrpwm1_pins>;
+	//pinctrl-0 = <&ehrpwm1_pins>;
+	pinctrl-0 = <>;
+};
+
+&epwmss2 {
+	status = "okay";
+};
+
+&ehrpwm2 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
 };
 
 &i2c0 {
@@ -161,9 +1005,18 @@
 	};
 };
 
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+
+	status = "okay";
+	clock-frequency = <400000>;
+};
+
 &i2c2 {
 	pinctrl-names = "default";
-	pinctrl-0 = <&i2c2_pins>;
+//	pinctrl-0 = <&i2c2_pins>;
+	pinctrl-0 = <>;
 
 	status = "okay";
 	clock-frequency = <400000>;
@@ -182,6 +1035,14 @@
 	system-power-controller;
 };
 
+//&pruss_soc_bus {
+//	status = "okay";
+//
+//	pruss: pruss@4a300000 {
+//		status = "okay";
+//	};
+//};
+
 &tscadc {
 	status = "okay";
 	adc {
@@ -194,14 +1055,30 @@
 
 &uart0 {
 	pinctrl-names = "default";
-	pinctrl-0 = <&uart0_pins>;
+	//pinctrl-0 = <&uart0_pins>;
+	pinctrl-0 = <>;
+
+	status = "okay";
+};
+
+&uart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+
+	status = "okay";
+};
+
+&uart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
 
 	status = "okay";
 };
 
 &uart4 {
 	pinctrl-names = "default";
-	pinctrl-0 = <&uart4_pins>;
+	//pinctrl-0 = <&uart4_pins>;
+	pinctrl-0 = <>;
 
 	status = "okay";
 };
@@ -235,3 +1112,1092 @@
 &cppi41dma  {
 	status = "okay";
 };
+
+&spi0 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	status = "okay";
+
+	channel@0 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "spidev";
+		symlink = "spi/0.0";
+		reg = <0>;
+		spi-max-frequency = <24000000>;
+	};
+
+	channel@1 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "spidev";
+		symlink = "spi/0.1";
+		reg = <1>;
+		spi-max-frequency = <24000000>;
+		status = "disabled";
+	};
+};
+
+&spi1 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	status = "okay";
+
+	channel@0 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "spidev";
+		symlink = "spi/1.0";
+		reg = <0>;
+		spi-max-frequency = <24000000>;
+	};
+
+	channel@1 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "spidev";
+		symlink = "spi/1.1";
+		reg = <1>;
+		spi-max-frequency = <24000000>;
+	};
+};
+
+&dcan0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+};
+
+&dcan1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <>;
+};
+
+&ocp {
+	/************************/
+	/* P1 Header */
+	/************************/
+
+	/* P1_01                VIN-AC */
+
+	/* P1_02 (ZCZ ball R5) gpio_input */
+	P1_02_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin";
+		pinctrl-0 = <&P1_02_default_pin>;
+		pinctrl-1 = <&P1_02_gpio_pin>;
+		pinctrl-2 = <&P1_02_gpio_pu_pin>;
+		pinctrl-3 = <&P1_02_gpio_pd_pin>;
+		pinctrl-4 = <&P1_02_gpio_input_pin>;
+		pinctrl-5 = <&P1_02_pruout_pin>;
+		pinctrl-6 = <&P1_02_pruin_pin>;
+	};
+
+	/* P1_03 (ZCZ ball F15)  usb1_vbus_out         */
+
+	/* P1_04 (ZCZ ball R6) */
+	P1_04_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin";
+		pinctrl-0 = <&P1_04_default_pin>;
+		pinctrl-1 = <&P1_04_gpio_pin>;
+		pinctrl-2 = <&P1_04_gpio_pu_pin>;
+		pinctrl-3 = <&P1_04_gpio_pd_pin>;
+		pinctrl-4 = <&P1_04_gpio_input_pin>;
+		pinctrl-5 = <&P1_04_pruout_pin>;
+		pinctrl-6 = <&P1_04_pruin_pin>;
+	};
+
+	/* P1_05 (ZCZ ball T18)  usb1_vbus_in         */
+
+	/* P1_06 (ZCZ ball A16) spi_cs */
+	P1_06_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "i2c", "pwm", "pru_uart";
+		pinctrl-0 = <&P1_06_default_pin>;
+		pinctrl-1 = <&P1_06_gpio_pin>;
+		pinctrl-2 = <&P1_06_gpio_pu_pin>;
+		pinctrl-3 = <&P1_06_gpio_pd_pin>;
+		pinctrl-4 = <&P1_06_gpio_input_pin>;
+		pinctrl-5 = <&P1_06_spi_cs_pin>;
+		pinctrl-6 = <&P1_06_i2c_pin>;
+		pinctrl-7 = <&P1_06_pwm_pin>;
+		pinctrl-8 = <&P1_06_pru_uart_pin>;
+	};
+
+	/* P1_07                VIN-USB */
+
+	/* P1_08 (ZCZ ball A17) spi_sclk */
+	P1_08_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_sclk", "uart", "i2c", "pwm", "pru_uart";
+		pinctrl-0 = <&P1_08_default_pin>;
+		pinctrl-1 = <&P1_08_gpio_pin>;
+		pinctrl-2 = <&P1_08_gpio_pu_pin>;
+		pinctrl-3 = <&P1_08_gpio_pd_pin>;
+		pinctrl-4 = <&P1_08_gpio_input_pin>;
+		pinctrl-5 = <&P1_08_spi_sclk_pin>;
+		pinctrl-6 = <&P1_08_uart_pin>;
+		pinctrl-7 = <&P1_08_i2c_pin>;
+		pinctrl-8 = <&P1_08_pwm_pin>;
+		pinctrl-9 = <&P1_08_pru_uart_pin>;
+	};
+
+	/* P1_09 (ZCZ ball R18)  USB1-DN         */
+
+	/* P1_10 (ZCZ ball B17) spi */
+	P1_10_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "uart", "i2c", "pwm", "pru_uart";
+		pinctrl-0 = <&P1_10_default_pin>;
+		pinctrl-1 = <&P1_10_gpio_pin>;
+		pinctrl-2 = <&P1_10_gpio_pu_pin>;
+		pinctrl-3 = <&P1_10_gpio_pd_pin>;
+		pinctrl-4 = <&P1_10_gpio_input_pin>;
+		pinctrl-5 = <&P1_10_spi_pin>;
+		pinctrl-6 = <&P1_10_uart_pin>;
+		pinctrl-7 = <&P1_10_i2c_pin>;
+		pinctrl-8 = <&P1_10_pwm_pin>;
+		pinctrl-9 = <&P1_10_pru_uart_pin>;
+	};
+
+	/* P1_11 (ZCZ ball R17)  USB1-DP         */
+
+	/* P1_12 (ZCZ ball B16) spi */
+	P1_12_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "i2c", "pwm", "pru_uart";
+		pinctrl-0 = <&P1_12_default_pin>;
+		pinctrl-1 = <&P1_12_gpio_pin>;
+		pinctrl-2 = <&P1_12_gpio_pu_pin>;
+		pinctrl-3 = <&P1_12_gpio_pd_pin>;
+		pinctrl-4 = <&P1_12_gpio_input_pin>;
+		pinctrl-5 = <&P1_12_spi_pin>;
+		pinctrl-6 = <&P1_12_i2c_pin>;
+		pinctrl-7 = <&P1_12_pwm_pin>;
+		pinctrl-8 = <&P1_12_pru_uart_pin>;
+	};
+
+	/* P1_13 (ZCZ ball P17)  USB1-ID         */
+
+	/* P1_14                VOUT-3.3V */
+
+	/* P1_15                GND */
+
+	/* P1_16                GND */
+
+	/* P1_17 (ZCZ ball A9)  VREFN         */
+
+	/* P1_18 (ZCZ ball B9)  VREFP         */
+
+	/* P1_19 (ZCZ ball B6)  AIN0         */
+
+	/* P1_20 (ZCZ ball D14) */
+	P1_20_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruin";
+		pinctrl-0 = <&P1_20_default_pin>;
+		pinctrl-1 = <&P1_20_gpio_pin>;
+		pinctrl-2 = <&P1_20_gpio_pu_pin>;
+		pinctrl-3 = <&P1_20_gpio_pd_pin>;
+		pinctrl-4 = <&P1_20_gpio_input_pin>;
+		pinctrl-5 = <&P1_20_pruin_pin>;
+	};
+
+	/* P1_21 (ZCZ ball C7)  AIN1         */
+
+	/* P1_22                GND */
+
+	/* P1_23 (ZCZ ball B7)  AIN2         */
+
+	/* P1_24                VOUT-5V */
+
+	/* P1_25 (ZCZ ball A7)  AIN3         */
+
+	/* P1_26 (ZCZ ball D18) i2c */
+	P1_26_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart";
+		pinctrl-0 = <&P1_26_default_pin>;
+		pinctrl-1 = <&P1_26_gpio_pin>;
+		pinctrl-2 = <&P1_26_gpio_pu_pin>;
+		pinctrl-3 = <&P1_26_gpio_pd_pin>;
+		pinctrl-4 = <&P1_26_gpio_input_pin>;
+		pinctrl-5 = <&P1_26_spi_cs_pin>;
+		pinctrl-6 = <&P1_26_can_pin>;
+		pinctrl-7 = <&P1_26_i2c_pin>;
+		pinctrl-8 = <&P1_26_pru_uart_pin>;
+	};
+
+	/* P1_27 (ZCZ ball C8)  AIN4         */
+
+	/* P1_28 (ZCZ ball D17) i2c */
+	P1_28_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart";
+		pinctrl-0 = <&P1_28_default_pin>;
+		pinctrl-1 = <&P1_28_gpio_pin>;
+		pinctrl-2 = <&P1_28_gpio_pu_pin>;
+		pinctrl-3 = <&P1_28_gpio_pd_pin>;
+		pinctrl-4 = <&P1_28_gpio_input_pin>;
+		pinctrl-5 = <&P1_28_spi_cs_pin>;
+		pinctrl-6 = <&P1_28_can_pin>;
+		pinctrl-7 = <&P1_28_i2c_pin>;
+		pinctrl-8 = <&P1_28_pru_uart_pin>;
+	};
+
+	/* P1_29 (ZCZ ball A14) pruin */
+	P1_29_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin";
+		pinctrl-0 = <&P1_29_default_pin>;
+		pinctrl-1 = <&P1_29_gpio_pin>;
+		pinctrl-2 = <&P1_29_gpio_pu_pin>;
+		pinctrl-3 = <&P1_29_gpio_pd_pin>;
+		pinctrl-4 = <&P1_29_gpio_input_pin>;
+		pinctrl-5 = <&P1_29_qep_pin>;
+		pinctrl-6 = <&P1_29_pruout_pin>;
+		pinctrl-7 = <&P1_29_pruin_pin>;
+	};
+
+	/* P1_30 (ZCZ ball E16) uart */
+	P1_30_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "uart", "can", "i2c", "pruout", "pruin";
+		pinctrl-0 = <&P1_30_default_pin>;
+		pinctrl-1 = <&P1_30_gpio_pin>;
+		pinctrl-2 = <&P1_30_gpio_pu_pin>;
+		pinctrl-3 = <&P1_30_gpio_pd_pin>;
+		pinctrl-4 = <&P1_30_gpio_input_pin>;
+		pinctrl-5 = <&P1_30_spi_cs_pin>;
+		pinctrl-6 = <&P1_30_uart_pin>;
+		pinctrl-7 = <&P1_30_can_pin>;
+		pinctrl-8 = <&P1_30_i2c_pin>;
+		pinctrl-9 = <&P1_30_pruout_pin>;
+		pinctrl-10 = <&P1_30_pruin_pin>;
+	};
+
+	/* P1_31 (ZCZ ball B12) pruin */
+	P1_31_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin";
+		pinctrl-0 = <&P1_31_default_pin>;
+		pinctrl-1 = <&P1_31_gpio_pin>;
+		pinctrl-2 = <&P1_31_gpio_pu_pin>;
+		pinctrl-3 = <&P1_31_gpio_pd_pin>;
+		pinctrl-4 = <&P1_31_gpio_input_pin>;
+		pinctrl-5 = <&P1_31_qep_pin>;
+		pinctrl-6 = <&P1_31_pruout_pin>;
+		pinctrl-7 = <&P1_31_pruin_pin>;
+	};
+
+	/* P1_32 (ZCZ ball E15) uart */
+	P1_32_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "uart", "can", "i2c", "pruout", "pruin";
+		pinctrl-0 = <&P1_32_default_pin>;
+		pinctrl-1 = <&P1_32_gpio_pin>;
+		pinctrl-2 = <&P1_32_gpio_pu_pin>;
+		pinctrl-3 = <&P1_32_gpio_pd_pin>;
+		pinctrl-4 = <&P1_32_gpio_input_pin>;
+		pinctrl-5 = <&P1_32_spi_cs_pin>;
+		pinctrl-6 = <&P1_32_uart_pin>;
+		pinctrl-7 = <&P1_32_can_pin>;
+		pinctrl-8 = <&P1_32_i2c_pin>;
+		pinctrl-9 = <&P1_32_pruout_pin>;
+		pinctrl-10 = <&P1_32_pruin_pin>;
+	};
+
+	/* P1_33 (ZCZ ball B13) pruin */
+	P1_33_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "pwm", "pruout", "pruin";
+		pinctrl-0 = <&P1_33_default_pin>;
+		pinctrl-1 = <&P1_33_gpio_pin>;
+		pinctrl-2 = <&P1_33_gpio_pu_pin>;
+		pinctrl-3 = <&P1_33_gpio_pd_pin>;
+		pinctrl-4 = <&P1_33_gpio_input_pin>;
+		pinctrl-5 = <&P1_33_spi_pin>;
+		pinctrl-6 = <&P1_33_pwm_pin>;
+		pinctrl-7 = <&P1_33_pruout_pin>;
+		pinctrl-8 = <&P1_33_pruin_pin>;
+	};
+
+	/* P1_34 (ZCZ ball T11) */
+	P1_34_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm";
+		pinctrl-0 = <&P1_34_default_pin>;
+		pinctrl-1 = <&P1_34_gpio_pin>;
+		pinctrl-2 = <&P1_34_gpio_pu_pin>;
+		pinctrl-3 = <&P1_34_gpio_pd_pin>;
+		pinctrl-4 = <&P1_34_gpio_input_pin>;
+		pinctrl-5 = <&P1_34_pwm_pin>;
+	};
+
+	/* P1_35 (ZCZ ball V5) pruin */
+	P1_35_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin";
+		pinctrl-0 = <&P1_35_default_pin>;
+		pinctrl-1 = <&P1_35_gpio_pin>;
+		pinctrl-2 = <&P1_35_gpio_pu_pin>;
+		pinctrl-3 = <&P1_35_gpio_pd_pin>;
+		pinctrl-4 = <&P1_35_gpio_input_pin>;
+		pinctrl-5 = <&P1_35_pruout_pin>;
+		pinctrl-6 = <&P1_35_pruin_pin>;
+	};
+
+	/* P1_36 (ZCZ ball A13) pwm */
+	P1_36_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_sclk", "pwm", "pruout", "pruin";
+		pinctrl-0 = <&P1_36_default_pin>;
+		pinctrl-1 = <&P1_36_gpio_pin>;
+		pinctrl-2 = <&P1_36_gpio_pu_pin>;
+		pinctrl-3 = <&P1_36_gpio_pd_pin>;
+		pinctrl-4 = <&P1_36_gpio_input_pin>;
+		pinctrl-5 = <&P1_36_spi_sclk_pin>;
+		pinctrl-6 = <&P1_36_pwm_pin>;
+		pinctrl-7 = <&P1_36_pruout_pin>;
+		pinctrl-8 = <&P1_36_pruin_pin>;
+	};
+
+
+	/************************/
+	/* P2 Header */
+	/************************/
+
+	/* P2_01 (ZCZ ball U14) pwm */
+	P2_01_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm";
+		pinctrl-0 = <&P2_01_default_pin>;
+		pinctrl-1 = <&P2_01_gpio_pin>;
+		pinctrl-2 = <&P2_01_gpio_pu_pin>;
+		pinctrl-3 = <&P2_01_gpio_pd_pin>;
+		pinctrl-4 = <&P2_01_gpio_input_pin>;
+		pinctrl-5 = <&P2_01_pwm_pin>;
+	};
+
+	/* P2_02 (ZCZ ball V17) */
+	P2_02_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input";
+		pinctrl-0 = <&P2_02_default_pin>;
+		pinctrl-1 = <&P2_02_gpio_pin>;
+		pinctrl-2 = <&P2_02_gpio_pu_pin>;
+		pinctrl-3 = <&P2_02_gpio_pd_pin>;
+		pinctrl-4 = <&P2_02_gpio_input_pin>;
+	};
+
+	/* P2_03 (ZCZ ball T10) */
+	P2_03_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm";
+		pinctrl-0 = <&P2_03_default_pin>;
+		pinctrl-1 = <&P2_03_gpio_pin>;
+		pinctrl-2 = <&P2_03_gpio_pu_pin>;
+		pinctrl-3 = <&P2_03_gpio_pd_pin>;
+		pinctrl-4 = <&P2_03_gpio_input_pin>;
+		pinctrl-5 = <&P2_03_pwm_pin>;
+	};
+
+	/* P2_04 (ZCZ ball T16) */
+	P2_04_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input";
+		pinctrl-0 = <&P2_04_default_pin>;
+		pinctrl-1 = <&P2_04_gpio_pin>;
+		pinctrl-2 = <&P2_04_gpio_pu_pin>;
+		pinctrl-3 = <&P2_04_gpio_pd_pin>;
+		pinctrl-4 = <&P2_04_gpio_input_pin>;
+	};
+
+	/* P2_05 (ZCZ ball T17) uart */
+	P2_05_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart";
+		pinctrl-0 = <&P2_05_default_pin>;
+		pinctrl-1 = <&P2_05_gpio_pin>;
+		pinctrl-2 = <&P2_05_gpio_pu_pin>;
+		pinctrl-3 = <&P2_05_gpio_pd_pin>;
+		pinctrl-4 = <&P2_05_gpio_input_pin>;
+		pinctrl-5 = <&P2_05_uart_pin>;
+	};
+
+	/* P2_06 (ZCZ ball U16) */
+	P2_06_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input";
+		pinctrl-0 = <&P2_06_default_pin>;
+		pinctrl-1 = <&P2_06_gpio_pin>;
+		pinctrl-2 = <&P2_06_gpio_pu_pin>;
+		pinctrl-3 = <&P2_06_gpio_pd_pin>;
+		pinctrl-4 = <&P2_06_gpio_input_pin>;
+	};
+
+	/* P2_07 (ZCZ ball U17) uart */
+	P2_07_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart";
+		pinctrl-0 = <&P2_07_default_pin>;
+		pinctrl-1 = <&P2_07_gpio_pin>;
+		pinctrl-2 = <&P2_07_gpio_pu_pin>;
+		pinctrl-3 = <&P2_07_gpio_pd_pin>;
+		pinctrl-4 = <&P2_07_gpio_input_pin>;
+		pinctrl-5 = <&P2_07_uart_pin>;
+	};
+
+	/* P2_08 (ZCZ ball U18) */
+	P2_08_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input";
+		pinctrl-0 = <&P2_08_default_pin>;
+		pinctrl-1 = <&P2_08_gpio_pin>;
+		pinctrl-2 = <&P2_08_gpio_pu_pin>;
+		pinctrl-3 = <&P2_08_gpio_pd_pin>;
+		pinctrl-4 = <&P2_08_gpio_input_pin>;
+	};
+
+	/* P2_09 (ZCZ ball D15) i2c */
+	P2_09_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "can", "i2c", "pru_uart", "pruin";
+		pinctrl-0 = <&P2_09_default_pin>;
+		pinctrl-1 = <&P2_09_gpio_pin>;
+		pinctrl-2 = <&P2_09_gpio_pu_pin>;
+		pinctrl-3 = <&P2_09_gpio_pd_pin>;
+		pinctrl-4 = <&P2_09_gpio_input_pin>;
+		pinctrl-5 = <&P2_09_uart_pin>;
+		pinctrl-6 = <&P2_09_can_pin>;
+		pinctrl-7 = <&P2_09_i2c_pin>;
+		pinctrl-8 = <&P2_09_pru_uart_pin>;
+		pinctrl-9 = <&P2_09_pruin_pin>;
+	};
+
+	/* P2_10 (ZCZ ball R14) */
+	P2_10_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep";
+		pinctrl-0 = <&P2_10_default_pin>;
+		pinctrl-1 = <&P2_10_gpio_pin>;
+		pinctrl-2 = <&P2_10_gpio_pu_pin>;
+		pinctrl-3 = <&P2_10_gpio_pd_pin>;
+		pinctrl-4 = <&P2_10_gpio_input_pin>;
+		pinctrl-5 = <&P2_10_qep_pin>;
+	};
+
+	/* P2_11 (ZCZ ball D16) i2c */
+	P2_11_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "can", "i2c", "pru_uart", "pruin";
+		pinctrl-0 = <&P2_11_default_pin>;
+		pinctrl-1 = <&P2_11_gpio_pin>;
+		pinctrl-2 = <&P2_11_gpio_pu_pin>;
+		pinctrl-3 = <&P2_11_gpio_pd_pin>;
+		pinctrl-4 = <&P2_11_gpio_input_pin>;
+		pinctrl-5 = <&P2_11_uart_pin>;
+		pinctrl-6 = <&P2_11_can_pin>;
+		pinctrl-7 = <&P2_11_i2c_pin>;
+		pinctrl-8 = <&P2_11_pru_uart_pin>;
+		pinctrl-9 = <&P2_11_pruin_pin>;
+	};
+
+	/* P2_12                POWER_BUTTON */
+
+	/* P2_13                VOUT-5V */
+
+	/* P2_14                BAT-VIN */
+
+	/* P2_15                GND */
+
+	/* P2_16                BAT-TEMP */
+
+	/* P2_17 (ZCZ ball V12) */
+	P2_17_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input";
+		pinctrl-0 = <&P2_17_default_pin>;
+		pinctrl-1 = <&P2_17_gpio_pin>;
+		pinctrl-2 = <&P2_17_gpio_pu_pin>;
+		pinctrl-3 = <&P2_17_gpio_pd_pin>;
+		pinctrl-4 = <&P2_17_gpio_input_pin>;
+	};
+
+	/* P2_18 (ZCZ ball U13) */
+	P2_18_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pru_ecap", "pruin";
+		pinctrl-0 = <&P2_18_default_pin>;
+		pinctrl-1 = <&P2_18_gpio_pin>;
+		pinctrl-2 = <&P2_18_gpio_pu_pin>;
+		pinctrl-3 = <&P2_18_gpio_pd_pin>;
+		pinctrl-4 = <&P2_18_gpio_input_pin>;
+		pinctrl-5 = <&P2_18_qep_pin>;
+		pinctrl-6 = <&P2_18_pru_ecap_pin>;
+		pinctrl-7 = <&P2_18_pruin_pin>;
+	};
+
+	/* P2_19 (ZCZ ball U12) */
+	P2_19_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm";
+		pinctrl-0 = <&P2_19_default_pin>;
+		pinctrl-1 = <&P2_19_gpio_pin>;
+		pinctrl-2 = <&P2_19_gpio_pu_pin>;
+		pinctrl-3 = <&P2_19_gpio_pd_pin>;
+		pinctrl-4 = <&P2_19_gpio_input_pin>;
+		pinctrl-5 = <&P2_19_pwm_pin>;
+	};
+
+	/* P2_20 (ZCZ ball T13) */
+	P2_20_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input";
+		pinctrl-0 = <&P2_20_default_pin>;
+		pinctrl-1 = <&P2_20_gpio_pin>;
+		pinctrl-2 = <&P2_20_gpio_pu_pin>;
+		pinctrl-3 = <&P2_20_gpio_pd_pin>;
+		pinctrl-4 = <&P2_20_gpio_input_pin>;
+	};
+
+	/* P2_21                GND */
+
+	/* P2_22 (ZCZ ball V13) */
+	P2_22_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruin";
+		pinctrl-0 = <&P2_22_default_pin>;
+		pinctrl-1 = <&P2_22_gpio_pin>;
+		pinctrl-2 = <&P2_22_gpio_pu_pin>;
+		pinctrl-3 = <&P2_22_gpio_pd_pin>;
+		pinctrl-4 = <&P2_22_gpio_input_pin>;
+		pinctrl-5 = <&P2_22_qep_pin>;
+		pinctrl-6 = <&P2_22_pruin_pin>;
+	};
+
+	/* P2_23                VOUT-3.3V */
+
+	/* P2_24 (ZCZ ball T12) */
+	P2_24_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout";
+		pinctrl-0 = <&P2_24_default_pin>;
+		pinctrl-1 = <&P2_24_gpio_pin>;
+		pinctrl-2 = <&P2_24_gpio_pu_pin>;
+		pinctrl-3 = <&P2_24_gpio_pd_pin>;
+		pinctrl-4 = <&P2_24_gpio_input_pin>;
+		pinctrl-5 = <&P2_24_qep_pin>;
+		pinctrl-6 = <&P2_24_pruout_pin>;
+	};
+
+	/* P2_25 (ZCZ ball E17) spi */
+	P2_25_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "spi_cs", "uart", "can", "i2c";
+		pinctrl-0 = <&P2_25_default_pin>;
+		pinctrl-1 = <&P2_25_gpio_pin>;
+		pinctrl-2 = <&P2_25_gpio_pu_pin>;
+		pinctrl-3 = <&P2_25_gpio_pd_pin>;
+		pinctrl-4 = <&P2_25_gpio_input_pin>;
+		pinctrl-5 = <&P2_25_spi_pin>;
+		pinctrl-6 = <&P2_25_spi_cs_pin>;
+		pinctrl-7 = <&P2_25_uart_pin>;
+		pinctrl-8 = <&P2_25_can_pin>;
+		pinctrl-9 = <&P2_25_i2c_pin>;
+	};
+
+	/* P2_26                RESET# */
+
+	/* P2_27 (ZCZ ball E18) spi */
+	P2_27_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "uart", "can", "i2c";
+		pinctrl-0 = <&P2_27_default_pin>;
+		pinctrl-1 = <&P2_27_gpio_pin>;
+		pinctrl-2 = <&P2_27_gpio_pu_pin>;
+		pinctrl-3 = <&P2_27_gpio_pd_pin>;
+		pinctrl-4 = <&P2_27_gpio_input_pin>;
+		pinctrl-5 = <&P2_27_spi_pin>;
+		pinctrl-6 = <&P2_27_uart_pin>;
+		pinctrl-7 = <&P2_27_can_pin>;
+		pinctrl-8 = <&P2_27_i2c_pin>;
+	};
+
+	/* P2_28 (ZCZ ball D13) pruin */
+	P2_28_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin";
+		pinctrl-0 = <&P2_28_default_pin>;
+		pinctrl-1 = <&P2_28_gpio_pin>;
+		pinctrl-2 = <&P2_28_gpio_pu_pin>;
+		pinctrl-3 = <&P2_28_gpio_pd_pin>;
+		pinctrl-4 = <&P2_28_gpio_input_pin>;
+		pinctrl-5 = <&P2_28_qep_pin>;
+		pinctrl-6 = <&P2_28_pruout_pin>;
+		pinctrl-7 = <&P2_28_pruin_pin>;
+	};
+
+	/* P2_29 (ZCZ ball C18) spi_sclk */
+	P2_29_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "spi_sclk", "uart", "pwm", "pru_ecap";
+		pinctrl-0 = <&P2_29_default_pin>;
+		pinctrl-1 = <&P2_29_gpio_pin>;
+		pinctrl-2 = <&P2_29_gpio_pu_pin>;
+		pinctrl-3 = <&P2_29_gpio_pd_pin>;
+		pinctrl-4 = <&P2_29_gpio_input_pin>;
+		pinctrl-5 = <&P2_29_spi_cs_pin>;
+		pinctrl-6 = <&P2_29_spi_sclk_pin>;
+		pinctrl-7 = <&P2_29_uart_pin>;
+		pinctrl-8 = <&P2_29_pwm_pin>;
+		pinctrl-9 = <&P2_29_pru_ecap_pin>;
+	};
+
+	/* P2_30 (ZCZ ball C12) pruin */
+	P2_30_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "pwm", "pruout", "pruin";
+		pinctrl-0 = <&P2_30_default_pin>;
+		pinctrl-1 = <&P2_30_gpio_pin>;
+		pinctrl-2 = <&P2_30_gpio_pu_pin>;
+		pinctrl-3 = <&P2_30_gpio_pd_pin>;
+		pinctrl-4 = <&P2_30_gpio_input_pin>;
+		pinctrl-5 = <&P2_30_spi_cs_pin>;
+		pinctrl-6 = <&P2_30_pwm_pin>;
+		pinctrl-7 = <&P2_30_pruout_pin>;
+		pinctrl-8 = <&P2_30_pruin_pin>;
+	};
+
+	/* P2_31 (ZCZ ball A15) spi_cs */
+	P2_31_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "pruin";
+		pinctrl-0 = <&P2_31_default_pin>;
+		pinctrl-1 = <&P2_31_gpio_pin>;
+		pinctrl-2 = <&P2_31_gpio_pu_pin>;
+		pinctrl-3 = <&P2_31_gpio_pd_pin>;
+		pinctrl-4 = <&P2_31_gpio_input_pin>;
+		pinctrl-5 = <&P2_31_spi_cs_pin>;
+		pinctrl-6 = <&P2_31_pruin_pin>;
+	};
+
+	/* P2_32 (ZCZ ball D12) pruin */
+	P2_32_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "pwm", "pruout", "pruin";
+		pinctrl-0 = <&P2_32_default_pin>;
+		pinctrl-1 = <&P2_32_gpio_pin>;
+		pinctrl-2 = <&P2_32_gpio_pu_pin>;
+		pinctrl-3 = <&P2_32_gpio_pd_pin>;
+		pinctrl-4 = <&P2_32_gpio_input_pin>;
+		pinctrl-5 = <&P2_32_spi_pin>;
+		pinctrl-6 = <&P2_32_pwm_pin>;
+		pinctrl-7 = <&P2_32_pruout_pin>;
+		pinctrl-8 = <&P2_32_pruin_pin>;
+	};
+
+	/* P2_33 (ZCZ ball R12) */
+	P2_33_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout";
+		pinctrl-0 = <&P2_33_default_pin>;
+		pinctrl-1 = <&P2_33_gpio_pin>;
+		pinctrl-2 = <&P2_33_gpio_pu_pin>;
+		pinctrl-3 = <&P2_33_gpio_pd_pin>;
+		pinctrl-4 = <&P2_33_gpio_input_pin>;
+		pinctrl-5 = <&P2_33_qep_pin>;
+		pinctrl-6 = <&P2_33_pruout_pin>;
+	};
+
+	/* P2_34 (ZCZ ball C13) pruin */
+	P2_34_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin";
+		pinctrl-0 = <&P2_34_default_pin>;
+		pinctrl-1 = <&P2_34_gpio_pin>;
+		pinctrl-2 = <&P2_34_gpio_pu_pin>;
+		pinctrl-3 = <&P2_34_gpio_pd_pin>;
+		pinctrl-4 = <&P2_34_gpio_input_pin>;
+		pinctrl-5 = <&P2_34_qep_pin>;
+		pinctrl-6 = <&P2_34_pruout_pin>;
+		pinctrl-7 = <&P2_34_pruin_pin>;
+	};
+
+	/* P2_35 (ZCZ ball U5) gpio_input */
+	P2_35_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin";
+		pinctrl-0 = <&P2_35_default_pin>;
+		pinctrl-1 = <&P2_35_gpio_pin>;
+		pinctrl-2 = <&P2_35_gpio_pu_pin>;
+		pinctrl-3 = <&P2_35_gpio_pd_pin>;
+		pinctrl-4 = <&P2_35_gpio_input_pin>;
+		pinctrl-5 = <&P2_35_pruout_pin>;
+		pinctrl-6 = <&P2_35_pruin_pin>;
+	};
+
+	/* P2_36 (ZCZ ball C9)  AIN7         */
+
+	cape-universal {
+		compatible = "gpio-of-helper";
+		status = "okay";
+		pinctrl-names = "default";
+		pinctrl-0 = <>;
+
+		P1_02 {
+			gpio-name = "P1_02";
+			gpio = <&gpio2 23 0>;
+			input;
+			dir-changeable;
+		};
+
+		P1_04 {
+			gpio-name = "P1_04";
+			gpio = <&gpio2 25 0>;
+			input;
+			dir-changeable;
+		};
+
+		P1_06 {
+			gpio-name = "P1_06";
+			gpio = <&gpio0 5 0>;
+			input;
+			dir-changeable;
+		};
+
+		P1_08 {
+			gpio-name = "P1_08";
+			gpio = <&gpio0 2 0>;
+			input;
+			dir-changeable;
+		};
+
+		P1_10 {
+			gpio-name = "P1_10";
+			gpio = <&gpio0 3 0>;
+			input;
+			dir-changeable;
+		};
+
+		P1_12 {
+			gpio-name = "P1_12";
+			gpio = <&gpio0 4 0>;
+			input;
+			dir-changeable;
+		};
+
+		P1_20 {
+			gpio-name = "P1_20";
+			gpio = <&gpio0 20 0>;
+			input;
+			dir-changeable;
+		};
+
+		P1_26 {
+			gpio-name = "P1_26";
+			gpio = <&gpio0 12 0>;
+			input;
+			dir-changeable;
+		};
+
+		P1_28 {
+			gpio-name = "P1_28";
+			gpio = <&gpio0 13 0>;
+			input;
+			dir-changeable;
+		};
+
+		P1_29 {
+			gpio-name = "P1_29";
+			gpio = <&gpio3 21 0>;
+			input;
+			dir-changeable;
+		};
+
+		P1_30 {
+			gpio-name = "P1_30";
+			gpio = <&gpio1 11 0>;
+			input;
+			dir-changeable;
+		};
+
+		P1_31 {
+			gpio-name = "P1_31";
+			gpio = <&gpio3 18 0>;
+			input;
+			dir-changeable;
+		};
+
+		P1_32 {
+			gpio-name = "P1_32";
+			gpio = <&gpio1 10 0>;
+			input;
+			dir-changeable;
+		};
+
+		P1_33 {
+			gpio-name = "P1_33";
+			gpio = <&gpio3 15 0>;
+			input;
+			dir-changeable;
+		};
+
+		P1_34 {
+			gpio-name = "P1_34";
+			gpio = <&gpio0 26 0>;
+			input;
+			dir-changeable;
+		};
+
+		P1_35 {
+			gpio-name = "P1_35";
+			gpio = <&gpio2 24 0>;
+			input;
+			dir-changeable;
+		};
+
+		P1_36 {
+			gpio-name = "P1_36";
+			gpio = <&gpio3 14 0>;
+			input;
+			dir-changeable;
+		};
+
+		P2_01 {
+			gpio-name = "P2_01";
+			gpio = <&gpio1 18 0>;
+			input;
+			dir-changeable;
+		};
+
+		P2_02 {
+			gpio-name = "P2_02";
+			gpio = <&gpio1 27 0>;
+			input;
+			dir-changeable;
+		};
+
+		P2_03 {
+			gpio-name = "P2_03";
+			gpio = <&gpio0 23 0>;
+			input;
+			dir-changeable;
+		};
+
+		P2_04 {
+			gpio-name = "P2_04";
+			gpio = <&gpio1 26 0>;
+			input;
+			dir-changeable;
+		};
+
+		P2_05 {
+			gpio-name = "P2_05";
+			gpio = <&gpio0 30 0>;
+			input;
+			dir-changeable;
+		};
+
+		P2_06 {
+			gpio-name = "P2_06";
+			gpio = <&gpio1 25 0>;
+			input;
+			dir-changeable;
+		};
+
+		P2_07 {
+			gpio-name = "P2_07";
+			gpio = <&gpio0 31 0>;
+			input;
+			dir-changeable;
+		};
+
+		P2_08 {
+			gpio-name = "P2_08";
+			gpio = <&gpio1 28 0>;
+			input;
+			dir-changeable;
+		};
+
+		P2_09 {
+			gpio-name = "P2_09";
+			gpio = <&gpio0 15 0>;
+			input;
+			dir-changeable;
+		};
+
+		P2_10 {
+			gpio-name = "P2_10";
+			gpio = <&gpio1 20 0>;
+			input;
+			dir-changeable;
+		};
+
+		P2_11 {
+			gpio-name = "P2_11";
+			gpio = <&gpio0 14 0>;
+			input;
+			dir-changeable;
+		};
+
+		P2_17 {
+			gpio-name = "P2_17";
+			gpio = <&gpio2 1 0>;
+			input;
+			dir-changeable;
+		};
+
+		P2_18 {
+			gpio-name = "P2_18";
+			gpio = <&gpio1 15 0>;
+			input;
+			dir-changeable;
+		};
+
+		P2_19 {
+			gpio-name = "P2_19";
+			gpio = <&gpio0 27 0>;
+			input;
+			dir-changeable;
+		};
+
+		P2_20 {
+			gpio-name = "P2_20";
+			gpio = <&gpio2 0 0>;
+			input;
+			dir-changeable;
+		};
+
+		P2_22 {
+			gpio-name = "P2_22";
+			gpio = <&gpio1 14 0>;
+			input;
+			dir-changeable;
+		};
+
+		P2_24 {
+			gpio-name = "P2_24";
+			gpio = <&gpio1 12 0>;
+			input;
+			dir-changeable;
+		};
+
+		P2_25 {
+			gpio-name = "P2_25";
+			gpio = <&gpio1 9 0>;
+			input;
+			dir-changeable;
+		};
+
+		P2_27 {
+			gpio-name = "P2_27";
+			gpio = <&gpio1 8 0>;
+			input;
+			dir-changeable;
+		};
+
+		P2_28 {
+			gpio-name = "P2_28";
+			gpio = <&gpio3 20 0>;
+			input;
+			dir-changeable;
+		};
+
+		P2_29 {
+			gpio-name = "P2_29";
+			gpio = <&gpio0 7 0>;
+			input;
+			dir-changeable;
+		};
+
+		P2_30 {
+			gpio-name = "P2_30";
+			gpio = <&gpio3 17 0>;
+			input;
+			dir-changeable;
+		};
+
+		P2_31 {
+			gpio-name = "P2_31";
+			gpio = <&gpio0 19 0>;
+			input;
+			dir-changeable;
+		};
+
+		P2_32 {
+			gpio-name = "P2_32";
+			gpio = <&gpio3 16 0>;
+			input;
+			dir-changeable;
+		};
+
+		P2_33 {
+			gpio-name = "P2_33";
+			gpio = <&gpio1 13 0>;
+			input;
+			dir-changeable;
+		};
+
+		P2_34 {
+			gpio-name = "P2_34";
+			gpio = <&gpio3 19 0>;
+			input;
+			dir-changeable;
+		};
+
+		P2_35 {
+			gpio-name = "P2_35";
+			gpio = <&gpio2 22 0>;
+			input;
+			dir-changeable;
+		};
+
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-revolve.dts b/arch/arm/boot/dts/am335x-revolve.dts
--- a/arch/arm/boot/dts/am335x-revolve.dts	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-revolve.dts	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,984 @@
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+/dts-v1/;
+
+#include "am33xx.dtsi"
+/* #include "am335x-bone-jtag.dtsi" */
+
+/ {
+
+	model = "Revolve:00A0";
+	compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx";
+
+	cpus {
+		cpu@0 {
+			cpu0-supply = <&dcdc2_reg>;
+		};
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x80000000 0x10000000>; /* 256 MB */
+	};
+
+	chosen {
+		stdout-path = &uart0;
+	};
+
+	vmmcsd_fixed: fixedregulator0 {
+		compatible = "regulator-fixed";
+		regulator-name = "vmmcsd_fixed";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+    leds {
+	    pinctrl-names = "default";
+	    pinctrl-0 = <&user_leds_s0>;
+
+	    compatible = "gpio-leds";
+
+	    led1 {
+		    label = "beaglebone:green:usr1";
+		    gpios = <&gpio1 22 GPIO_ACTIVE_HIGH>;
+            trigger-sources = <&usb1>;
+		    /*linux,default-trigger = "mmc0";*/
+		    default-state = "off";
+	    };
+
+	    led2 {
+		    label = "beaglebone:green:usr2";
+		    gpios = <&gpio1 23 GPIO_ACTIVE_HIGH>;
+		    linux,default-trigger = "cpu0";
+		    default-state = "off";
+	    };
+
+	    led3 {
+		    label = "beaglebone:green:usr3";
+		    gpios = <&gpio1 24 GPIO_ACTIVE_HIGH>;
+		    linux,default-trigger = "mmc1";
+		    default-state = "off";
+	    };
+	    led4 {
+		    label = "beaglebone:green:usr0";
+		    gpios = <&gpio1 25 GPIO_ACTIVE_HIGH>;
+		    linux,default-trigger = "heartbeat";
+		    default-state = "off";
+	    };
+
+    };
+
+
+    gpio_keys {
+        compatible = "gpio-keys";
+        pinctrl-names = "default";
+        pinctrl-0 = <&end_stop_pins>;
+        pinctrl-1 = <&stepper_fault_pins>;
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        switch_x1 {
+            label               = "Replic:End-stop-X1";
+            debounce_interval   = <50>;
+            linux,code          = <112>;
+            gpios               = <&gpio1 28 0x5>;
+            gpio-key,wakeup;
+        };
+        switch_x2 {
+            label               = "Replic:End-stop-X2";
+            debounce_interval   = <50>;
+            linux,code          = <113>;
+            gpios               = <&gpio1 17 0x5>;
+            gpio-key,wakeup;
+        };
+        switch_y1 {
+            label               = "Replic:End-stop-Y1";
+            debounce_interval   = <50>;
+            linux,code          = <114>;
+            gpios               = <&gpio1 27 0x5>;
+            gpio-key,wakeup;
+        };
+        switch_y2 {
+            label               = "Replic:End-stop-Y2";
+            debounce_interval   = <50>;
+            linux,code          = <115>;
+            gpios               = <&gpio1 29 0x5>;
+            gpio-key,wakeup;
+        };
+        switch_z1 {
+            label               = "Replic:End-stop-Z1";
+            debounce_interval   = <50>;
+            linux,code          = <116>;
+            gpios               = <&gpio1 14 0x5>;
+            gpio-key,wakeup;
+        };
+        switch_z2 {
+            label               = "Replic:End-stop-Z2";
+            debounce_interval   = <50>;
+            linux,code          = <117>;
+            gpios               = <&gpio1 15 0x5>;
+            gpio-key,wakeup;
+        };
+        fault_x {
+            label               = "Replic:Fault-X";
+            debounce_interval   = <50>;
+            linux,code          = <90>;
+            gpios               = <&gpio0 27 0x5>;
+            gpio-key,wakeup;
+        };
+        fault_y {
+            label               = "Replic:Fault-Y";
+            debounce_interval   = <50>;
+            linux,code          = <91>;
+            gpios               = <&gpio2 0 0x5>;
+            gpio-key,wakeup;
+        };
+        fault_z {
+            label               = "Replic:Fault-Z";
+            debounce_interval   = <50>;
+            linux,code          = <92>;
+            gpios               = <&gpio1 16 0x5>;
+            gpio-key,wakeup;
+        };
+        fault_e {
+            label               = "Replic:Fault-E";
+            debounce_interval   = <50>;
+            linux,code          = <93>;
+            gpios               = <&gpio2 1 0x5>;
+            gpio-key,wakeup;
+        };
+        fault_h {
+            label               = "Replic:Fault-H";
+            debounce_interval   = <50>;
+            linux,code          = <94>;
+            gpios               = <&gpio0 29 0x5>;
+            gpio-key,wakeup;
+        };
+        fault_a {
+            label               = "Replic:Fault-A";
+            debounce_interval   = <50>;
+            linux,code          = <94>;
+            gpios               = <&gpio0 26 0x5>;
+            gpio-key,wakeup;
+        };
+    };
+
+
+    pwm4: pwm-heater-e {
+        compatible = "ti,omap-dmtimer-pwm";
+        ti,timers = <&timer4>;
+        #pwm-cells = <3>;
+        pinctrl-names = "default";
+	    pinctrl-0 = <&timer4_pins>;
+
+    };
+
+    pwm5: pwm-heater-h {
+        compatible = "ti,omap-dmtimer-pwm";
+        ti,timers = <&timer5>;
+        #pwm-cells = <3>;
+        pinctrl-names = "default";
+	    pinctrl-0 = <&timer5_pins>;
+
+    };
+
+    pwm6: pwm-heater-hbp {
+        compatible = "ti,omap-dmtimer-pwm";
+        ti,timers = <&timer6>;
+        #pwm-cells = <3>;
+        pinctrl-names = "default";
+	    pinctrl-0 = <&timer6_pins>;
+
+    };
+
+    pwm7: pwm-heater-a {
+        compatible = "ti,omap-dmtimer-pwm";
+        ti,timers = <&timer7>;
+        #pwm-cells = <3>;
+        pinctrl-names = "default";
+	    pinctrl-0 = <&timer7_pins>;
+
+    };
+};
+
+
+&cpu0_opp_table {
+	/*
+	 * All PG 2.0 silicon may not support 1GHz but some of the early
+	 * BeagleBone Blacks have PG 2.0 silicon which is guaranteed
+	 * to support 1GHz OPP so enable it for PG 2.0 on this board.
+	 */
+	oppnitro-1000000000 {
+		opp-supported-hw = <0x06 0x0100>;
+	};
+};
+
+
+&am33xx_pinmux {
+	nxp_hdmi_bonelt_pins: nxp_hdmi_bonelt_pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x8a0, PIN_OUTPUT | MUX_MODE0)		/* lcd_data0.lcd_data0 */
+			AM33XX_IOPAD(0x8a4, PIN_OUTPUT | MUX_MODE0)		/* lcd_data1.lcd_data1 */
+			AM33XX_IOPAD(0x8a8, PIN_OUTPUT | MUX_MODE0)		/* lcd_data2.lcd_data2 */
+			AM33XX_IOPAD(0x8ac, PIN_OUTPUT | MUX_MODE0)		/* lcd_data3.lcd_data3 */
+			AM33XX_IOPAD(0x8b0, PIN_OUTPUT | MUX_MODE0)		/* lcd_data4.lcd_data4 */
+			AM33XX_IOPAD(0x8b4, PIN_OUTPUT | MUX_MODE0)		/* lcd_data5.lcd_data5 */
+			AM33XX_IOPAD(0x8b8, PIN_OUTPUT | MUX_MODE0)		/* lcd_data6.lcd_data6 */
+			AM33XX_IOPAD(0x8bc, PIN_OUTPUT | MUX_MODE0)		/* lcd_data7.lcd_data7 */
+			AM33XX_IOPAD(0x8c0, PIN_OUTPUT | MUX_MODE0)		/* lcd_data8.lcd_data8 */
+			AM33XX_IOPAD(0x8c4, PIN_OUTPUT | MUX_MODE0)		/* lcd_data9.lcd_data9 */
+			AM33XX_IOPAD(0x8c8, PIN_OUTPUT | MUX_MODE0)		/* lcd_data10.lcd_data10 */
+			AM33XX_IOPAD(0x8cc, PIN_OUTPUT | MUX_MODE0)		/* lcd_data11.lcd_data11 */
+			AM33XX_IOPAD(0x8d0, PIN_OUTPUT | MUX_MODE0)		/* lcd_data12.lcd_data12 */
+			AM33XX_IOPAD(0x8d4, PIN_OUTPUT | MUX_MODE0)		/* lcd_data13.lcd_data13 */
+			AM33XX_IOPAD(0x8d8, PIN_OUTPUT | MUX_MODE0)		/* lcd_data14.lcd_data14 */
+			AM33XX_IOPAD(0x8dc, PIN_OUTPUT | MUX_MODE0)		/* lcd_data15.lcd_data15 */
+			AM33XX_IOPAD(0x8e0, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* lcd_vsync.lcd_vsync */
+			AM33XX_IOPAD(0x8e4, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* lcd_hsync.lcd_hsync */
+			AM33XX_IOPAD(0x8e8, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* lcd_pclk.lcd_pclk */
+			AM33XX_IOPAD(0x8ec, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* lcd_ac_bias_en.lcd_ac_bias_en */
+		>;
+	};
+
+    ehrpwm1_pins: ehrpwm1_pins_default {
+	    pinctrl-single,pins = <
+		    0x20 ( PIN_OUTPUT | MUX_MODE4 ) /* (U10) gpmc_ad8.ehrpwm2A */
+		    0x24 ( PIN_OUTPUT | MUX_MODE4 ) /* (T10) gpmc_ad9.ehrpwm2B */
+	    >;
+    };
+
+    ehrpwm2_pins: ehrpwm2_pins_default {
+	    pinctrl-single,pins = <
+		    0x48 ( PIN_OUTPUT | MUX_MODE6 ) /* (U14) gpmc_a2.ehrpwm1A */
+		    0x4c ( PIN_OUTPUT | MUX_MODE6 ) /* (T14) gpmc_a3.ehrpwm1B */
+	    >;
+    };
+
+    spi0_pins_default: spi0_pins_deafult {
+	    pinctrl-single,pins = <
+		    0x150 ( PIN_INPUT | MUX_MODE0 ) /* (A17) spi0_sclk.spi0_sclk */
+		    0x154 ( PIN_INPUT | MUX_MODE0 ) /* (B17) spi0_d0.spi0_d0 */
+		    0x158 ( PIN_INPUT | MUX_MODE0 ) /* (B16) spi0_d1.spi0_d1 */
+		    0x15c ( PIN_OUTPUT | MUX_MODE0 ) /* (A16) spi0_cs0.spi0_cs0 */
+		    0x160 ( PIN_OUTPUT | MUX_MODE0 ) /* (C15) spi0_cs1.spi0_cs1 */
+	    >;
+    };
+
+    spi0_pins_reset: spi0_pins_reset {
+	    pinctrl-single,pins = <
+		    0x150 ( PIN_INPUT | MUX_MODE7 ) /* (A17) spi0_sclk.spi0_sclk */
+		    0x154 ( PIN_INPUT | MUX_MODE7 ) /* (B17) spi0_d0.spi0_d0 */
+		    0x158 ( PIN_INPUT | MUX_MODE7 ) /* (B16) spi0_d1.spi0_d1 */
+		    0x15c ( PIN_INPUT | MUX_MODE7 ) /* (A16) spi0_cs0.spi0_cs0 */
+		    0x160 ( PIN_INPUT | MUX_MODE7 ) /* (C15) spi0_cs1.spi0_cs1 */
+	    >;
+    };
+
+    spi1_pins: spi1_pins {
+	    pinctrl-single,pins = <
+		    0x164 ( PIN_INPUT | MUX_MODE4 ) /* (C18) eCAP0_in_PWM0_out.spi1_sclk */
+		    0x168 ( PIN_INPUT | MUX_MODE4 ) /* (E18) uart0_ctsn.spi1_d0 */
+		    0x16c ( PIN_INPUT | MUX_MODE4 ) /* (E17) uart0_rtsn.spi1_d1 */
+		    0x178 ( PIN_INPUT | MUX_MODE4 ) /* (D18) uart1_ctsn.spi1_cs0 */
+		    0x17c ( PIN_INPUT | MUX_MODE4 ) /* (D17) uart1_rtsn.spi1_cs1 */
+	    >;
+    };
+
+    pru_stepper_pins: pru_pruss11_pins {
+	    pinctrl-single,pins = <
+		    0x190 ( PIN_OUTPUT | MUX_MODE5 ) /* (A13) mcasp0_aclkx.pr1_pru0_pru_r30[0] */
+		    0x194 ( PIN_OUTPUT | MUX_MODE5 ) /* (B13) mcasp0_fsx.pr1_pru0_pru_r30[1] */
+		    0x198 ( PIN_OUTPUT | MUX_MODE5 ) /* (D12) mcasp0_axr0.pr1_pru0_pru_r30[2] */
+		    0x19c ( PIN_OUTPUT | MUX_MODE5 ) /* (C12) mcasp0_ahclkr.pr1_pru0_pru_r30[3] */
+		    0x1a0 ( PIN_OUTPUT | MUX_MODE5 ) /* (B12) mcasp0_aclkr.pr1_pru0_pru_r30[4] */
+		    0x1a4 ( PIN_OUTPUT | MUX_MODE5 ) /* (C13) mcasp0_fsr.pr1_pru0_pru_r30[5] */
+		    0x1a8 ( PIN_OUTPUT | MUX_MODE5 ) /* (D13) mcasp0_axr1.pr1_pru0_pru_r30[6] */
+		    0x1ac ( PIN_OUTPUT | MUX_MODE5 ) /* (A14) mcasp0_ahclkx.pr1_pru0_pru_r30[7] */
+		    0xf0 ( PIN_OUTPUT | MUX_MODE5 ) /* (F17) mmc0_dat3.pr1_pru0_pru_r30[8] */
+		    0xf4 ( PIN_OUTPUT | MUX_MODE5 ) /* (F18) mmc0_dat2.pr1_pru0_pru_r30[9] */
+		    0xf8 ( PIN_OUTPUT | MUX_MODE5 ) /* (G15) mmc0_dat1.pr1_pru0_pru_r30[10] */
+		    0xfc ( PIN_OUTPUT | MUX_MODE5 ) /* (G16) mmc0_dat0.pr1_pru0_pru_r30[11] */
+		    0x100 ( PIN_OUTPUT | MUX_MODE5 ) /* (G17) mmc0_clk.pr1_pru0_pru_r30[12] */
+		    0x104 ( PIN_OUTPUT | MUX_MODE5 ) /* (G18) mmc0_cmd.pr1_pru0_pru_r30[13] */
+		    0x30 ( PIN_OUTPUT | MUX_MODE6 ) /* (T12) gpmc_ad12.pr1_pru0_pru_r30[14] */
+		    0x34 ( PIN_OUTPUT | MUX_MODE6 ) /* (R12) gpmc_ad13.pr1_pru0_pru_r30[15] */
+		    /*0x38 ( PIN_INPUT | MUX_MODE6 )*/ /* (V13) gpmc_ad14.pr1_pru0_pru_r31[14] */
+		    /*0x3c ( PIN_INPUT | MUX_MODE6 )*/ /* (U13) gpmc_ad15.pr1_pru0_pru_r31[15] */
+		    0x1b4 ( PIN_INPUT | MUX_MODE5 ) /* (D14) xdma_event_intr1.pr1_pru0_pru_r31[16] */
+	    >;
+    };
+
+    pru_stepper_pins_gpio: pru_pruss11_pins_gpio {
+	    pinctrl-single,pins = <
+		    0x190 ( PIN_OUTPUT | MUX_MODE7 ) /* (A13) mcasp0_aclkx.pr1_pru0_pru_r30[0] */
+		    0x194 ( PIN_OUTPUT | MUX_MODE7 ) /* (B13) mcasp0_fsx.pr1_pru0_pru_r30[1] */
+		    0x198 ( PIN_OUTPUT | MUX_MODE7 ) /* (D12) mcasp0_axr0.pr1_pru0_pru_r30[2] */
+		    0x19c ( PIN_OUTPUT | MUX_MODE7 ) /* (C12) mcasp0_ahclkr.pr1_pru0_pru_r30[3] */
+		    0x1a0 ( PIN_OUTPUT | MUX_MODE7 ) /* (B12) mcasp0_aclkr.pr1_pru0_pru_r30[4] */
+		    0x1a4 ( PIN_OUTPUT | MUX_MODE7 ) /* (C13) mcasp0_fsr.pr1_pru0_pru_r30[5] */
+		    0x1a8 ( PIN_OUTPUT | MUX_MODE7 ) /* (D13) mcasp0_axr1.pr1_pru0_pru_r30[6] */
+		    0x1ac ( PIN_OUTPUT | MUX_MODE7 ) /* (A14) mcasp0_ahclkx.pr1_pru0_pru_r30[7] */
+		    0xf0 ( PIN_OUTPUT | MUX_MODE7 ) /* (F17) mmc0_dat3.pr1_pru0_pru_r30[8] */
+		    0xf4 ( PIN_OUTPUT | MUX_MODE7 ) /* (F18) mmc0_dat2.pr1_pru0_pru_r30[9] */
+		    0xf8 ( PIN_OUTPUT | MUX_MODE7 ) /* (G15) mmc0_dat1.pr1_pru0_pru_r30[10] */
+		    0xfc ( PIN_OUTPUT | MUX_MODE7 ) /* (G16) mmc0_dat0.pr1_pru0_pru_r30[11] */
+		    0x100 ( PIN_OUTPUT | MUX_MODE7 ) /* (G17) mmc0_clk.pr1_pru0_pru_r30[12] */
+		    0x104 ( PIN_OUTPUT | MUX_MODE7 ) /* (G18) mmc0_cmd.pr1_pru0_pru_r30[13] */
+		    0x30 ( PIN_OUTPUT | MUX_MODE7 ) /* (T12) gpmc_ad12.pr1_pru0_pru_r30[14] */
+		    0x34 ( PIN_OUTPUT | MUX_MODE7 ) /* (R12) gpmc_ad13.pr1_pru0_pru_r30[15] */
+	    >;
+    };
+
+    timer4_pins: timer4_pins {
+	    pinctrl-single,pins = <
+		    0x90 ( PIN_OUTPUT | MUX_MODE2 ) /* (R7) gpmc_advn_ale.timer4 */
+	    >;
+    };
+
+    timer5_pins: timer5_pins {
+	    pinctrl-single,pins = <
+		     0x9c ( PIN_OUTPUT | MUX_MODE2 ) /* (T6) gpmc_be0n_cle.timer5 */
+	    >;
+    };
+    timer6_pins: timer6_pins {
+	    pinctrl-single,pins = <
+		    0x98 ( PIN_OUTPUT | MUX_MODE2 ) /* (U6) gpmc_wen.timer6 */
+	    >;
+    };
+    timer7_pins: timer7_pins {
+	    pinctrl-single,pins = <
+		     0x94 ( PIN_OUTPUT | MUX_MODE2 ) /* (T7) gpmc_oen_ren.timer7 */
+	    >;
+    };
+
+    user_leds_s0: user_leds_s0 {
+		pinctrl-single,pins = <
+            0x64 ( PIN_OUTPUT | MUX_MODE7 ) /* (U16) gpmc_a9.gpio1[25] */
+			AM33XX_IOPAD(0x858, PIN_OUTPUT | MUX_MODE7)	/* gpmc_a6.gpio1_22 */
+			AM33XX_IOPAD(0x85c, PIN_OUTPUT | MUX_MODE7)	/* gpmc_a7.gpio1_23 */
+			AM33XX_IOPAD(0x860, PIN_OUTPUT | MUX_MODE7)	/* gpmc_a8.gpio1_24 */
+		>;
+	};
+
+    i2c0_pins: pinmux_i2c0_pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x988, PIN_INPUT_PULLUP | MUX_MODE0)	/* i2c0_sda.i2c0_sda */
+			AM33XX_IOPAD(0x98c, PIN_INPUT_PULLUP | MUX_MODE0)	/* i2c0_scl.i2c0_scl */
+		>;
+	};
+
+    i2c1_pins: i2c1_pins {
+        pinctrl-single,pins = <
+	        0x184 ( PIN_INPUT | MUX_MODE3 ) /* (D15) uart1_txd.I2C1_SCL */
+	        0x180 ( PIN_INPUT | MUX_MODE3 ) /* (D16) uart1_rxd.I2C1_SDA */
+        >;
+    };
+
+
+    uart0_pins: pinmux_uart0_pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x970, PIN_INPUT_PULLUP | MUX_MODE0)	/* uart0_rxd.uart0_rxd */
+			AM33XX_IOPAD(0x974, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* uart0_txd.uart0_txd */
+		>;
+	};
+
+    uart4_pins: pinmux_uart4_pins {
+	    pinctrl-single,pins = <
+		    0x70 ( PIN_INPUT | MUX_MODE6 ) /* (T17) gpmc_wait0.uart4_rxd */
+		    0x74 ( PIN_OUTPUT | MUX_MODE6 ) /* (U17) gpmc_wpn.uart4_txd */
+	    >;
+    };
+
+    cpsw_default: cpsw_default {
+		pinctrl-single,pins = <
+			/* Slave 1 */
+			0x108 (PIN_INPUT | MUX_MODE0)		/* mii1_col.mii1_col */
+			0x10c (PIN_INPUT | MUX_MODE0)		/* mii1_crs.mii1_crs */
+			AM33XX_IOPAD(0x910, PIN_INPUT_PULLUP | MUX_MODE0)	/* mii1_rxerr.mii1_rxerr */
+			AM33XX_IOPAD(0x914, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* mii1_txen.mii1_txen */
+			AM33XX_IOPAD(0x918, PIN_INPUT_PULLUP | MUX_MODE0)	/* mii1_rxdv.mii1_rxdv */
+			AM33XX_IOPAD(0x91c, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* mii1_txd3.mii1_txd3 */
+			AM33XX_IOPAD(0x920, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* mii1_txd2.mii1_txd2 */
+			AM33XX_IOPAD(0x924, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* mii1_txd1.mii1_txd1 */
+			AM33XX_IOPAD(0x928, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* mii1_txd0.mii1_txd0 */
+			AM33XX_IOPAD(0x92c, PIN_INPUT_PULLUP | MUX_MODE0)	/* mii1_txclk.mii1_txclk */
+			AM33XX_IOPAD(0x930, PIN_INPUT_PULLUP | MUX_MODE0)	/* mii1_rxclk.mii1_rxclk */
+			AM33XX_IOPAD(0x934, PIN_INPUT_PULLUP | MUX_MODE0)	/* mii1_rxd3.mii1_rxd3 */
+			AM33XX_IOPAD(0x938, PIN_INPUT_PULLUP | MUX_MODE0)	/* mii1_rxd2.mii1_rxd2 */
+			AM33XX_IOPAD(0x93c, PIN_INPUT_PULLUP | MUX_MODE0)	/* mii1_rxd1.mii1_rxd1 */
+			AM33XX_IOPAD(0x940, PIN_INPUT_PULLUP | MUX_MODE0)	/* mii1_rxd0.mii1_rxd0 */
+		>;
+	};
+
+	cpsw_sleep: cpsw_sleep {
+		pinctrl-single,pins = <
+			/* Slave 1 reset value */
+			0x108 (PIN_INPUT_PULLDOWN | MUX_MODE7)
+			0x10c (PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM33XX_IOPAD(0x910, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM33XX_IOPAD(0x914, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM33XX_IOPAD(0x918, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM33XX_IOPAD(0x91c, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM33XX_IOPAD(0x920, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM33XX_IOPAD(0x924, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM33XX_IOPAD(0x928, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM33XX_IOPAD(0x92c, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM33XX_IOPAD(0x930, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM33XX_IOPAD(0x934, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM33XX_IOPAD(0x938, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM33XX_IOPAD(0x93c, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM33XX_IOPAD(0x940, PIN_INPUT_PULLDOWN | MUX_MODE7)
+		>;
+	};
+
+	davinci_mdio_default: davinci_mdio_default {
+		pinctrl-single,pins = <
+			/* MDIO */
+			AM33XX_IOPAD(0x948, PIN_INPUT_PULLUP | SLEWCTRL_FAST | MUX_MODE0)	/* mdio_data.mdio_data */
+			AM33XX_IOPAD(0x94c, PIN_OUTPUT_PULLUP | MUX_MODE0)			/* mdio_clk.mdio_clk */
+		>;
+	};
+
+	davinci_mdio_sleep: davinci_mdio_sleep {
+		pinctrl-single,pins = <
+			/* MDIO reset value */
+			AM33XX_IOPAD(0x948, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM33XX_IOPAD(0x94c, PIN_INPUT_PULLDOWN | MUX_MODE7)
+		>;
+	};
+
+	emmc_pins: pinmux_emmc_pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x880, PIN_INPUT_PULLUP | MUX_MODE2) /* gpmc_csn1.mmc1_clk */
+			AM33XX_IOPAD(0x884, PIN_INPUT_PULLUP | MUX_MODE2) /* gpmc_csn2.mmc1_cmd */
+			AM33XX_IOPAD(0x800, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad0.mmc1_dat0 */
+			AM33XX_IOPAD(0x804, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad1.mmc1_dat1 */
+			AM33XX_IOPAD(0x808, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad2.mmc1_dat2 */
+			AM33XX_IOPAD(0x80c, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad3.mmc1_dat3 */
+			AM33XX_IOPAD(0x810, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad4.mmc1_dat4 */
+			AM33XX_IOPAD(0x814, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad5.mmc1_dat5 */
+			AM33XX_IOPAD(0x818, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad6.mmc1_dat6 */
+			AM33XX_IOPAD(0x81c, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad7.mmc1_dat7 */
+		>;
+	};
+
+    stepper_fault_pins: pinmux_stepper_fault_pins{
+        pinctrl-single,pins = <
+            0x2c  ( PIN_INPUT | MUX_MODE7 ) // FAULT X
+            0x88  ( PIN_INPUT | MUX_MODE7 ) // FAULT Y
+            0x40  ( PIN_INPUT | MUX_MODE7 ) // FAULT Z
+            0x8c  ( PIN_INPUT | MUX_MODE7 ) // FAULT E
+            0x144 ( PIN_INPUT | MUX_MODE7 ) // FAULT H
+            0x28  ( PIN_INPUT | MUX_MODE7 ) // FAULT A
+        >;
+    };
+
+    end_stop_pins: pinmux_end_stop_pins{
+        pinctrl-single,pins = <
+            0x78 ( PIN_INPUT | MUX_MODE7 ) // End stop X1  = GPIO3_21
+            0x44 ( PIN_INPUT | MUX_MODE7 ) // End stop X2  = GPIO0_30
+            0x6c ( PIN_INPUT | MUX_MODE7 ) // End stop Y1  = GPIO1_27
+            0x7c ( PIN_INPUT | MUX_MODE7 ) // End stop Z1  = GPIO1_14
+            0x38 ( PIN_INPUT | MUX_MODE7 ) // End stop Z2  = GPIO1_15
+            0x3c ( PIN_INPUT | MUX_MODE7 ) // End stop Y2  = GPIO3_17
+        >;
+    };
+
+    enable_stuff_pins: enable_stuff_pins{
+        pinctrl-single,pins = <
+	            0x21c ( PIN_INPUT | MUX_MODE7 ) /* (F16) USB0_DRVVBUS.gpio0[18] */
+        >;
+    };
+
+    eqep1_pins_default: myeqep1_pins_default {
+	    pinctrl-single,pins = <
+		    0x50 ( PIN_INPUT | MUX_MODE6 ) /* (R14) gpmc_a4.eQEP1A_in */
+		    0x54 ( PIN_INPUT | MUX_MODE6 ) /* (V15) gpmc_a5.eQEP1B_in */
+	    >;
+    };
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_pins>;
+	status = "okay";
+};
+
+&uart4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart4_pins>;
+	status = "okay";
+};
+
+&usb {
+	status = "okay";
+};
+
+&usb_ctrl_mod {
+	status = "okay";
+};
+
+&usb0_phy {
+	status = "okay";
+	#phy-cells = <0>;
+};
+
+&usb1_phy {
+	status = "okay";
+	#phy-cells = <0>;
+};
+
+&usb0 {
+	status = "okay";
+	dr_mode = "peripheral";
+	interrupts-extended = <&intc 18 &tps 0>;
+	interrupt-names = "mc", "vbus";
+};
+
+&usb1 {
+	status = "okay";
+	dr_mode = "host";
+};
+
+&cppi41dma  {
+	status = "okay";
+};
+
+&gpio0{
+	status = "okay";
+};
+
+&i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c0_pins>;
+
+	status = "okay";
+	clock-frequency = <400000>;
+
+	tps: tps@24 {
+		reg = <0x24>;
+	};
+
+	baseboard_eeprom: baseboard_eeprom@50 {
+		compatible = "atmel,24c256";
+		reg = <0x50>;
+
+		#address-cells = <1>;
+		#size-cells = <1>;
+		baseboard_data: baseboard_data@0 {
+			reg = <0 0x100>;
+		};
+	};
+
+	tda19988: tda19988@70 {
+		compatible = "nxp,tda998x";
+		reg = <0x70>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&nxp_hdmi_bonelt_pins>;
+
+		/* Convert 24bit BGR to RGB, e.g. cross red and blue wiring */
+		/* video-ports = <0x234501>; */
+		ports {
+			port@0 {
+				hdmi_0: endpoint@0 {
+					remote-endpoint = <&lcdc_0>;
+				};
+			};
+		};
+	};
+};
+
+&i2c1{
+    pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_pins>;
+	status = "okay";
+	clock-frequency = <400000>;
+};
+
+&epwmss1{
+    status = "okay";
+};
+
+&ehrpwm1{
+    status = "okay";
+    pinctrl-names = "default";
+	pinctrl-0 = <&ehrpwm1_pins>;
+};
+
+&eqep1{
+    status = "okay";
+    pinctrl-names = "default";
+    pinctrl-0 = <&eqep1_pins_default>;
+
+    count_mode = <0>;  /* 0 - Quadrature mode, normal 90 phase offset cha & chb.  1 - Direction mode.  cha input = clock, chb input = direction */
+    swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */
+    invert_qa = <1>;   /* Should we invert the channel A input?  */
+    invert_qb = <1>;   /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */
+};
+
+&epwmss2{
+    status = "okay";
+};
+
+&ehrpwm2{
+    status = "okay";
+    pinctrl-names = "default";
+	pinctrl-0 = <&ehrpwm2_pins>;
+};
+
+
+&timer4{
+    status = "okay";
+	ti,timer-pwm;
+};
+&timer5{
+    status = "okay";
+	ti,timer-pwm;
+};
+&timer6{
+    status = "okay";
+	ti,timer-pwm;
+};
+&timer7{
+    status = "okay";
+	ti,timer-pwm;
+};
+
+&spi0{
+    status = "okay";
+    pinctrl-names = "default";
+
+    stepper_control@0{
+        #address-cells 	  = <1>;
+        #size-cells 	  = <0>;
+        compatible 	  = "spidev";
+        reg 		  = <0>;
+        spi-max-frequency = <2000000>;
+    };
+    aux_control@1{
+        #address-cells 	  = <1>;
+        #size-cells 	  = <0>;
+        compatible 	  = "spidev";
+        reg 		  = <1>;
+        spi-max-frequency = <16000000>;
+    };
+};
+
+&spi1{
+    status = "okay";
+    pinctrl-names = "default";
+	pinctrl-0 = <&spi1_pins>;
+
+    ext_stepper_B@0{
+        #address-cells 	  = <1>;
+        #size-cells 	  = <0>;
+        compatible 	  = "spidev";
+        reg 		  = <0>;
+        spi-max-frequency = <16000000>;
+    };
+
+    ext_stepper_C@1{
+        #address-cells 	  = <1>;
+        #size-cells 	  = <0>;
+        compatible 	  = "spidev";
+        reg 		  = <1>;
+        spi-max-frequency = <16000000>;
+    };
+};
+
+&pruss{
+    status = "okay";
+};
+
+&tscadc{
+    status = "okay";
+    adc {
+        ti,adc-channels = <0 1 2 3 4 5 6 7>;
+	};
+};
+
+/include/ "tps65217.dtsi"
+
+&tps {
+	/*
+	 * Configure pmic to enter OFF-state instead of SLEEP-state ("RTC-only
+	 * mode") at poweroff.  Most BeagleBone versions do not support RTC-only
+	 * mode and risk hardware damage if this mode is entered.
+	 *
+	 * For details, see linux-omap mailing list May 2015 thread
+	 *	[PATCH] ARM: dts: am335x-bone* enable pmic-shutdown-controller
+	 * In particular, messages:
+	 *	http://www.spinics.net/lists/linux-omap/msg118585.html
+	 *	http://www.spinics.net/lists/linux-omap/msg118615.html
+	 *
+	 * You can override this later with
+	 *	&tps {  /delete-property/ ti,pmic-shutdown-controller;  }
+	 * if you want to use RTC-only mode and made sure you are not affected
+	 * by the hardware problems. (Tip: double-check by performing a current
+	 * measurement after shutdown: it should be less than 1 mA.)
+	 */
+
+	interrupts = <7>; /* NMI */
+	interrupt-parent = <&intc>;
+
+	ti,pmic-shutdown-controller;
+
+	charger {
+		status = "okay";
+	};
+
+	pwrbutton {
+		status = "okay";
+	};
+
+	regulators {
+		dcdc1_reg: regulator@0 {
+			regulator-name = "vdds_dpr";
+			regulator-always-on;
+		};
+
+		dcdc2_reg: regulator@1 {
+			/* VDD_MPU voltage limits 0.95V - 1.26V with +/-4% tolerance */
+			regulator-name = "vdd_mpu";
+			regulator-min-microvolt = <925000>;
+			regulator-max-microvolt = <1351500>;
+			regulator-boot-on;
+			regulator-always-on;
+		};
+
+		dcdc3_reg: regulator@2 {
+			/* VDD_CORE voltage limits 0.95V - 1.1V with +/-4% tolerance */
+			regulator-name = "vdd_core";
+			regulator-min-microvolt = <925000>;
+			regulator-max-microvolt = <1150000>;
+			regulator-boot-on;
+			regulator-always-on;
+		};
+
+		ldo1_reg: regulator@3 {
+			regulator-name = "vio,vrtc,vdds";
+			regulator-always-on;
+		};
+
+		ldo2_reg: regulator@4 {
+			regulator-name = "vdd_3v3aux";
+			regulator-always-on;
+		};
+
+		ldo3_reg: regulator@5 {
+			regulator-name = "vdd_1v8";
+			regulator-always-on;
+		};
+
+		ldo4_reg: regulator@6 {
+			regulator-name = "vdd_3v3a";
+			regulator-always-on;
+		};
+	};
+};
+
+&cpsw_emac0 {
+	phy_id = <&davinci_mdio>, <0>;
+	phy-mode = "mii";
+};
+
+&mac {
+	slaves = <1>;
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&cpsw_default>;
+	pinctrl-1 = <&cpsw_sleep>;
+	status = "okay";
+};
+
+&davinci_mdio {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&davinci_mdio_default>;
+	pinctrl-1 = <&davinci_mdio_sleep>;
+	status = "okay";
+};
+
+&mmc2 {
+	vmmc-supply = <&vmmcsd_fixed>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&emmc_pins>;
+	bus-width = <8>;
+	status = "okay";
+};
+
+
+&lcdc {
+	status = "okay";
+
+	/* If you want to get 24 bit RGB and 16 BGR mode instead of
+	 * current 16 bit RGB and 24 BGR modes, set the propety
+	 * below to "crossed" and uncomment the video-ports -property
+	 * in tda19988 node.
+	 */
+	blue-and-red-wiring = "straight";
+
+	port {
+		lcdc_0: endpoint@0 {
+			remote-endpoint = <&hdmi_0>;
+		};
+	};
+};
+
+&aes {
+	status = "okay";
+};
+
+&sham {
+	status = "okay";
+};
+
+&rtc {
+	system-power-controller;
+};
+
+
+/* the cape manager */
+/ {
+	bone_capemgr {
+		compatible = "ti,bone-capemgr";
+		status = "okay";
+
+		nvmem-cells = <&baseboard_data>;
+		nvmem-cell-names = "baseboard";
+		#slots = <0>;
+
+		/* map board revisions to compatible definitions */
+		baseboardmaps {
+			baseboard_revolve: board@0 {
+				board-name = "A335RVLV";
+				compatible-name = "ti,beaglebone";
+			};
+		};
+	};
+};
+
+
+&ocp {
+	enable_stuff_pinmux {
+		compatible = "gpio-of-helper";
+		status = "okay";
+		pinctrl-names = "default";
+		pinctrl-0 = <&enable_stuff_pins>;
+
+        enable_stuff{
+            gpio-name   = "enable_stuff";
+            gpio 		= <&gpio0 18 0>;
+			output;
+        };
+	};
+
+    sgx@0x56000000 {
+            compatible = "ti,sgx";
+            ti,hwmods = "gfx";
+            reg = <0x56000000 0x1000000>;
+            interrupts = <0x25>;
+            status = "okay";
+    };
+
+    pruss_pins {
+        compatible = "gpio-of-helper";
+		status = "okay";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pru_stepper_pins_gpio>;
+
+        stepper_x_step{
+            gpio-name   = "step_x";
+            gpio 		= <&gpio3 14 0>;
+			output;
+        };
+        stepper_x_dir{
+            gpio-name   = "dir_x";
+            gpio 		= <&gpio3 15 0>;
+			output;
+        };
+        stepper_y_step{
+            gpio-name   = "step_y";
+            gpio 		= <&gpio3 16 0>;
+			output;
+        };
+        stepper_y_dir{
+            gpio-name   = "dir_y";
+            gpio 		= <&gpio3 17 0>;
+			output;
+        };
+        stepper_z_step{
+            gpio-name   = "step_z";
+            gpio 		= <&gpio3 18 0>;
+			output;
+        };
+        stepper_z_dir{
+            gpio-name   = "dir_z";
+            gpio 		= <&gpio3 19 0>;
+			output;
+        };
+        stepper_e_step{
+            gpio-name   = "step_e";
+            gpio 		= <&gpio3 20 0>;
+			output;
+        };
+        stepper_e_dir{
+            gpio-name   = "dir_e";
+            gpio 		= <&gpio3 21 0>;
+			output;
+        };
+        stepper_h_step{
+            gpio-name   = "step_h";
+            gpio 		= <&gpio2 26 0>;
+			output;
+        };
+        stepper_h_dir{
+            gpio-name   = "dir_h";
+            gpio 		= <&gpio2 27 0>;
+			output;
+        };
+        stepper_a_step{
+            gpio-name   = "step_a";
+            gpio 		= <&gpio2 28 0>;
+			output;
+        };
+        stepper_a_dir{
+            gpio-name   = "dir_a";
+            gpio 		= <&gpio2 29 0>;
+			output;
+        };
+        stepper_b_step{
+            gpio-name   = "step_b";
+            gpio 		= <&gpio2 30 0>;
+			output;
+        };
+        stepper_b_dir{
+            gpio-name   = "dir_b";
+            gpio 		= <&gpio2 31 0>;
+			output;
+        };
+        stepper_c_step{
+            gpio-name   = "step_c";
+            gpio 		= <&gpio1 12 0>;
+			output;
+        };
+        stepper_c_dir{
+            gpio-name   = "dir_c";
+            gpio 		= <&gpio1 13 0>;
+			output;
+        };
+    };
+
+    spi0_pins {
+        compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "reset";
+		pinctrl-0 = <&spi0_pins_default>;
+        pinctrl-1 = <&spi0_pins_reset>;
+    };
+
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-roboticscape.dtsi b/arch/arm/boot/dts/am335x-roboticscape.dtsi
--- a/arch/arm/boot/dts/am335x-roboticscape.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-roboticscape.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,489 @@
+/*
+ * @file am335x-roboticscape.dtsi
+ *
+ * used by:
+ * am335x-boneblack-roboticscape.dts
+ * am335x-boneblack-wireless-roboticscape.dts
+ *
+ *
+ * @file am335x-boneblack-roboticscape.dtb
+ *
+ * based on am335x-boneblack.dtb
+ *
+ * This device tree serves to replace the need for an overlay when using the
+ * RoboticsCape. It is similar to the boneblue tree but preserves pin config for
+ * the black.
+ *
+ * This file was updated in April 2018 to support LED driver like in BB Blue
+ * This goes in sync with V0.4.0 of the Robotics Cape library
+ *
+ * @author     James Strawson
+ * @date       April 19, 2018
+ */
+
+&aliases {
+	i2c0 = &i2c0;
+	i2c1 = &i2c1;
+	i2c2 = &i2c2;
+	serial0 = &uart0;
+	serial1 = &uart1;
+	serial2 = &uart2;
+	serial3 = &uart3;
+	serial4 = &uart4;
+	serial5 = &uart5;
+	d-can0 = &dcan0;
+	d-can1 = &dcan1;
+	usb0 = &usb0;
+	usb1 = &usb1;
+	phy0 = &usb0_phy;
+	phy1 = &usb1_phy;
+	ethernet0 = &cpsw_emac0;
+	ethernet1 = &cpsw_emac1;
+	/* librobotcontrol expects /dev/spidev1 */
+	spi0 = &spi0;
+	spi1 = &spi1;
+	/delete-property/ spi2;
+};
+
+/*******************************************************************************
+* Pin Muxing
+*******************************************************************************/
+&am33xx_pinmux {
+
+/*******************************************************************************
+* Static Pinmux
+*******************************************************************************/
+	mux_helper_pins: pins {
+		pinctrl-single,pins = <
+
+			/* GPIO Input Pullup */
+			0x09c 0x37	/*P8.9  T6  PAUSE_BTN */
+			0x098 0x37	/*P8.10 U6  MODE_BTN  */
+			0x1AC 0x37	/*P9.25 A14 IMU_INT   */
+
+			/* Motor Control GPIO Out*/
+			0x0cc 0x0F	/*P8.34 MDIR_2B different from blue!!*/
+			0x0a8 0x0F	/*P8.43 MDIR_3B*/
+			0x0ac 0x0F	/*P8.44 MDIR_3A*/
+			0x0a0 0x0F	/*P8.45 MDIR_4A*/
+			0x0a4 0x0F	/*P8.46 MDIR_4B*/
+			0x078 0x0F	/*P9.12 MDIR_1A different from blue!!*/
+			0x074 0x0F	/*P9.13 MDIR_1B*/
+			0x040 0x0F	/*P9.15 MDIR_2A*/
+			0x1b4 0x0F	/*P9.41 MOT_STBY*/
+
+			/* PRU encoder input */
+			0x03c 0x36	/* P8_15,PRU0_r31_15,MODE6 */
+			0x038 0x36	/* P8_16,PRU0_r31_16,MODE6 */
+
+			/* PRU Servo output */
+			0x0e0 0x05	/*pru1_pru_r30_8, MODE5*/
+			0x0e8 0x05	/*pru1_pru_r30_10, MODE5 */
+			0x0e4 0x05	/*pr1_pru1_pru_r30_9, MODE5 */
+			0x0ec 0x05	/*pru1_pru_r30_11, MODE5 */
+			0x0b8 0x05	/*pru1_pru_r30_6, MODE5 */
+			0x0bc 0x05	/*pru1_pru_r30_7, MODE5 */
+			0x0b0 0x05	/*pru1_pru_r30_4, MODE5 */
+			0x0b4 0x05	/*pru1_pru_r30_5, MODE5 */
+			0x0C8 0x0F	/*P8.36, SERVO_PWR GPIO OUT*/
+
+			/* I2C1 */
+			0x15C 0x32	/* P9.17,i2c1_scl,INPUT_PULLUP,MODE2 */
+			0x158 0x32	/* P9.18,i2c1_sda,INPUT_PULLUP,MODE2 */
+
+			/* I2C2 */
+			//0x17c  0x73 /* P9.19, i2c2_sda, mode 3 */
+			//0x178  0x73 /* P9.20, i2c2_sda, mode 3 */
+
+			/* UART5 */
+			0x0C4 0x34	/* P8.38,uart5_rxd,MODE4 */
+			0x0C0 0x14	/* P8.37,uart5_txd,MODE4 */
+
+		>;
+	};
+
+
+	led_pins: pinmux_led_pins {
+		pinctrl-single,pins = <
+			0x090 (PIN_OUTPUT | MUX_MODE7) /* (R7) gpmc_advn_ale.gpio2[2] - P8.7, LED_RED, GP1_PIN_5 */
+			0x094 (PIN_OUTPUT | MUX_MODE7) /* (T7) gpmc_oen_ren.gpio2[3] - P8.8, LED_GREEN, GP1_PIN_6 */
+			0x02C (PIN_OUTPUT | MUX_MODE7) /* (U12) gpmc_ad11.gpio0[27] - P8.17, BATT_LED_1 */
+			0x08c (PIN_OUTPUT | MUX_MODE7) /* (V12) gpio2.1 - P8.18  BATT_LED_2 diff from BLUE! */
+			0x07C (PIN_OUTPUT | MUX_MODE7) /* (V6) gpmc_csn0.gpio1[29] - P8.26, BATT_LED_3 */
+			0x028 (PIN_OUTPUT | MUX_MODE7) /* (T11) gpmc_ad10.gpio0[26] - P8.14, BATT_LED_4 */
+		>;
+	};
+
+};
+
+
+/*******************************************************************************
+* apply static and dynamic pinmux modes listed above. Configurable pins get the
+* modes from am335x-boneblack-common-universal-pins.dtsi
+*******************************************************************************/
+&ocp {
+	/* activate the static pinmux helper list of pin modes above */
+	test_helper: helper {
+		compatible = "bone-pinmux-helper";
+		pinctrl-names = "default";
+		pinctrl-0 = <&mux_helper_pins>;
+
+		status = "okay";
+	};
+
+	/* UART4 RX DSM */
+	P9_11_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "uart";
+		pinctrl-0 = <&P9_11_default_pin>;
+		pinctrl-1 = <&P9_11_gpio_pin>;
+		pinctrl-2 = <&P9_11_gpio_pu_pin>;
+		pinctrl-3 = <&P9_11_gpio_pd_pin>;
+		pinctrl-4 = <&P9_11_uart_pin>;
+	};
+
+	/* UART 2 TX GPS*/
+	P9_21_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "spi", "uart", "i2c", "pwm";
+		pinctrl-0 = <&P9_21_uart_pin>;
+		pinctrl-1 = <&P9_21_gpio_pin>;
+		pinctrl-2 = <&P9_21_gpio_pu_pin>;
+		pinctrl-3 = <&P9_21_gpio_pd_pin>;
+		pinctrl-4 = <&P9_21_spi_pin>;
+		pinctrl-5 = <&P9_21_uart_pin>;
+		pinctrl-6 = <&P9_21_i2c_pin>;
+		pinctrl-7 = <&P9_21_pwm_pin>;
+	};
+
+	/* UART 2 RX GPS */
+	P9_22_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "spi", "uart", "i2c", "pwm";
+		pinctrl-0 = <&P9_22_uart_pin>;
+		pinctrl-1 = <&P9_22_gpio_pin>;
+		pinctrl-2 = <&P9_22_gpio_pu_pin>;
+		pinctrl-3 = <&P9_22_gpio_pd_pin>;
+		pinctrl-4 = <&P9_22_spi_pin>;
+		pinctrl-5 = <&P9_22_uart_pin>;
+		pinctrl-6 = <&P9_22_i2c_pin>;
+		pinctrl-7 = <&P9_22_pwm_pin>;
+	};
+
+	/* SPI MISO */
+	P9_29_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm", "spi", "pruout", "pruin";
+		pinctrl-0 = <&P9_29_spi_pin>;
+		pinctrl-1 = <&P9_29_gpio_pin>;
+		pinctrl-2 = <&P9_29_gpio_pu_pin>;
+		pinctrl-3 = <&P9_29_gpio_pd_pin>;
+		pinctrl-4 = <&P9_29_pwm_pin>;
+		pinctrl-5 = <&P9_29_spi_pin>;
+		pinctrl-6 = <&P9_29_pruout_pin>;
+		pinctrl-7 = <&P9_29_pruin_pin>;
+	};
+
+	/* SPI MOSI */
+	P9_30_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm", "spi", "pruout", "pruin";
+		pinctrl-0 = <&P9_30_spi_pin>;
+		pinctrl-1 = <&P9_30_gpio_pin>;
+		pinctrl-2 = <&P9_30_gpio_pu_pin>;
+		pinctrl-3 = <&P9_30_gpio_pd_pin>;
+		pinctrl-4 = <&P9_30_pwm_pin>;
+		pinctrl-5 = <&P9_30_spi_pin>;
+		pinctrl-6 = <&P9_30_pruout_pin>;
+		pinctrl-7 = <&P9_30_pruin_pin>;
+	};
+
+	/* SPI SCK */
+	P9_31_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm", "spi", "spi_sclk", "pruout", "pruin";
+		pinctrl-0 = <&P9_31_spi_sclk_pin>;
+		pinctrl-1 = <&P9_31_gpio_pin>;
+		pinctrl-2 = <&P9_31_gpio_pu_pin>;
+		pinctrl-3 = <&P9_31_gpio_pd_pin>;
+		pinctrl-4 = <&P9_31_pwm_pin>;
+		pinctrl-5 = <&P9_31_spi_pin>;
+		pinctrl-6 = <&P9_31_spi_sclk_pin>;
+		pinctrl-7 = <&P9_31_pruout_pin>;
+		pinctrl-8 = <&P9_31_pruin_pin>;
+	};
+
+	/* SPI SS1 GPIO3_17*/
+	P9_28_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm", "spi", "spi_cs", "pwm2", "pruout", "pruin";
+		pinctrl-0 = <&P9_28_spi_cs_pin>;
+		pinctrl-1 = <&P9_28_gpio_pin>;
+		pinctrl-2 = <&P9_28_gpio_pu_pin>;
+		pinctrl-3 = <&P9_28_gpio_pd_pin>;
+		pinctrl-4 = <&P9_28_pwm_pin>;
+		pinctrl-5 = <&P9_28_spi_pin>;
+		pinctrl-6 = <&P9_28_spi_cs_pin>;
+		pinctrl-7 = <&P9_28_pwm2_pin>;
+		pinctrl-8 = <&P9_28_pruout_pin>;
+		pinctrl-9 = <&P9_28_pruin_pin>;
+	};
+
+	/* SPI SS1  GPIO1_17*/
+	P9_23_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm";
+		pinctrl-0 = <&P9_23_gpio_pin>;
+		pinctrl-1 = <&P9_23_gpio_pin>;
+		pinctrl-2 = <&P9_23_gpio_pu_pin>;
+		pinctrl-3 = <&P9_23_gpio_pd_pin>;
+		pinctrl-4 = <&P9_23_pwm_pin>;
+	};
+
+	/* UART 1 TX */
+	 P9_24_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "uart", "can", "i2c",  "pruin";
+		pinctrl-0 = <&P9_24_uart_pin>;
+		pinctrl-1 = <&P9_24_gpio_pin>;
+		pinctrl-2 = <&P9_24_gpio_pu_pin>;
+		pinctrl-3 = <&P9_24_gpio_pd_pin>;
+		pinctrl-4 = <&P9_24_uart_pin>;
+		pinctrl-5 = <&P9_24_can_pin>;
+		pinctrl-6 = <&P9_24_i2c_pin>;
+		pinctrl-7 = <&P9_24_pruin_pin>;
+	};
+
+	/* UART 1 RX */
+	P9_26_pinmux {
+		compatible = "bone-pinmux-helper";
+		status = "okay";
+		pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "uart", "can", "i2c",  "pruin";
+		pinctrl-0 = <&P9_26_uart_pin>;
+		pinctrl-1 = <&P9_26_gpio_pin>;
+		pinctrl-2 = <&P9_26_gpio_pu_pin>;
+		pinctrl-3 = <&P9_26_gpio_pd_pin>;
+		pinctrl-4 = <&P9_26_uart_pin>;
+		pinctrl-5 = <&P9_26_can_pin>;
+		pinctrl-6 = <&P9_26_i2c_pin>;
+		pinctrl-7 = <&P9_26_pruin_pin>;
+	};
+
+
+};
+
+
+/*******************************************************************************
+* PWMSS
+*******************************************************************************/
+&epwmss0 {
+	status = "okay";
+};
+
+&epwmss1 {
+	status = "okay";
+};
+
+&epwmss2 {
+	status = "okay";
+};
+
+&ehrpwm0 {
+	status = "okay";
+};
+
+&ehrpwm1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&P8_19_pwm_pin &P8_13_pwm_pin>;
+	status = "okay";
+};
+
+&ehrpwm2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&P9_14_pwm_pin &P9_16_pwm_pin>;
+	status = "okay";
+};
+
+/*******************************************************************************
+* EQEP
+*******************************************************************************/
+&eqep0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&P9_92_qep_pin &P9_27_qep_pin>;
+
+	count_mode = <0>;  /* 0 - Quadrature mode, normal 90 phase offset cha & chb.  1 - Direction mode.  cha input = clock, chb input = direction */
+	swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */
+	invert_qa = <1>;   /* Should we invert the channel A input?  */
+	invert_qb = <1>;   /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */
+	invert_qi = <0>;   /* Should we invert the index input? */
+	invert_qs = <0>;   /* Should we invert the strobe input? */
+	status = "okay";
+};
+
+&eqep1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&P8_33_qep_pin &P8_35_qep_pin>;
+
+	count_mode = <0>;  /* 0 - Quadrature mode, normal 90 phase offset cha & chb.  1 - Direction mode.  cha input = clock, chb input = direction */
+	swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */
+	invert_qa = <1>;   /* Should we invert the channel A input?  */
+	invert_qb = <1>;   /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */
+	invert_qi = <0>;   /* Should we invert the index input? */
+	invert_qs = <0>;   /* Should we invert the strobe input? */
+	status = "okay";
+};
+
+&eqep2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&P8_12_qep_pin &P8_11_qep_pin>;
+
+	count_mode = <0>;  /* 0 - Quadrature mode, normal 90 phase offset cha & chb.  1 - Direction mode.  cha input = clock, chb input = direction */
+	swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */
+	invert_qa = <1>;   /* Should we invert the channel A input?  */
+	invert_qb = <1>;   /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */
+	invert_qi = <0>;   /* Should we invert the index input? */
+	invert_qs = <0>;   /* Should we invert the strobe input? */
+	status = "okay";
+};
+
+
+/*******************************************************************************
+* UART
+*******************************************************************************/
+&uart1 {
+	status = "okay";
+};
+
+&uart2 {
+	status = "okay";
+};
+
+&uart4 {
+	status = "okay";
+};
+
+&uart5 {
+	status = "okay";
+};
+
+
+/*******************************************************************************
+* PRU Encoder and Servos
+*******************************************************************************/
+&pruss_soc_bus {
+	status = "okay";
+
+	pruss: pruss@4a300000 {
+		status = "okay";
+	};
+};
+
+
+/*******************************************************************************
+* I2C
+*******************************************************************************/
+&i2c1 {
+	status = "okay";
+	clock-frequency = <400000>;
+	#address-cells = <1>;
+	#size-cells = <0>;
+};
+
+/*******************************************************************************
+* SPI
+*******************************************************************************/
+&spi1 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	status = "okay";
+
+	channel@0 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		compatible = "spidev";
+		symlink = "spi/1.0";
+
+		reg = <0>;
+		spi-max-frequency = <16000000>;
+		spi-cpha;
+	};
+
+	channel@1 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		compatible = "spidev";
+		symlink = "spi/1.1";
+
+		reg = <1>;
+		spi-max-frequency = <16000000>;
+	};
+};
+
+/*******************************************************************************
+* LEDs
+*******************************************************************************/
+/ {
+	leds {
+		pinctrl-names = "default";
+		pinctrl-0 = <&led_pins>;
+		compatible = "gpio-leds";
+
+		red_led {
+			label = "red";
+			gpios = <&gpio2 2 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		green_led {
+			label = "green";
+			gpios = <&gpio2 3 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		batt_1_led {
+			label = "bat25";
+			gpios = <&gpio0 27 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		batt_2_led {
+			label = "bat50";
+			gpios = <&gpio2 1 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		batt_3_led {
+			label = "bat75";
+			gpios = <&gpio1 29 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		batt_4_led {
+			label = "bat100";
+			gpios = <&gpio0 26 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+	};
+
+};
+
+&tscadc {
+	status = "okay";
+	adc {
+		ti,adc-channels = <0 1 2 3 4 5 6 7>;
+		ti,chan-step-avg = <0x16 0x16 0x16 0x16 0x16 0x16 0x16 0x16>;
+		ti,chan-step-opendelay = <0x98 0x98 0x98 0x98 0x98 0x98 0x98 0x98>;
+		ti,chan-step-sampledelay = <0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0>;
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am335x-sancloud-bbe.dts b/arch/arm/boot/dts/am335x-sancloud-bbe.dts
--- a/arch/arm/boot/dts/am335x-sancloud-bbe.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am335x-sancloud-bbe.dts	2020-06-19 21:00:31.404227715 -0400
@@ -1,9 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 /dts-v1/;
 
@@ -15,6 +12,15 @@
 / {
 	model = "SanCloud BeagleBone Enhanced";
 	compatible = "sancloud,am335x-boneenhanced", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx";
+
+	chosen {
+		base_dtb = "am335x-sancloud-bbe.dts";
+		base_dtb_timestamp = __TIMESTAMP__;
+	};
+};
+
+&sgx {
+	status = "okay";
 };
 
 &am33xx_pinmux {
@@ -23,70 +29,70 @@
 	cpsw_default: cpsw_default {
 		pinctrl-single,pins = <
 			/* Slave 1 */
-			AM33XX_IOPAD(0x914, PIN_OUTPUT_PULLDOWN | MUX_MODE2)	/* mii1_txen.rgmii1_tctl */
-			AM33XX_IOPAD(0x918, PIN_INPUT_PULLDOWN | MUX_MODE2)	/* mii1_rxdv.rgmii1_rctl */
-			AM33XX_IOPAD(0x91c, PIN_OUTPUT_PULLDOWN | MUX_MODE2)	/* mii1_txd3.rgmii1_td3 */
-			AM33XX_IOPAD(0x920, PIN_OUTPUT_PULLDOWN | MUX_MODE2)	/* mii1_txd2.rgmii1_td2 */
-			AM33XX_IOPAD(0x924, PIN_OUTPUT_PULLDOWN | MUX_MODE2)	/* mii1_txd1.rgmii1_td1 */
-			AM33XX_IOPAD(0x928, PIN_OUTPUT_PULLDOWN | MUX_MODE2)	/* mii1_txd0.rgmii1_td0 */
-			AM33XX_IOPAD(0x92c, PIN_OUTPUT_PULLDOWN | MUX_MODE2)	/* mii1_txclk.rgmii1_tclk */
-			AM33XX_IOPAD(0x930, PIN_INPUT_PULLDOWN | MUX_MODE2)	/* mii1_rxclk.rgmii1_rclk */
-			AM33XX_IOPAD(0x934, PIN_INPUT_PULLDOWN | MUX_MODE2)	/* mii1_rxd3.rgmii1_rd3 */
-			AM33XX_IOPAD(0x938, PIN_INPUT_PULLDOWN | MUX_MODE2)	/* mii1_rxd2.rgmii1_rd2 */
-			AM33XX_IOPAD(0x93c, PIN_INPUT_PULLDOWN | MUX_MODE2)	/* mii1_rxd1.rgmii1_rd1 */
-			AM33XX_IOPAD(0x940, PIN_INPUT_PULLDOWN | MUX_MODE2)	/* mii1_rxd0.rgmii1_rd0 */
+			AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_OUTPUT_PULLDOWN, MUX_MODE2)	/* mii1_txen.rgmii1_tctl */
+			AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLDOWN, MUX_MODE2)	/* mii1_rxdv.rgmii1_rctl */
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_OUTPUT_PULLDOWN, MUX_MODE2)	/* mii1_txd3.rgmii1_td3 */
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_OUTPUT_PULLDOWN, MUX_MODE2)	/* mii1_txd2.rgmii1_td2 */
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD1, PIN_OUTPUT_PULLDOWN, MUX_MODE2)	/* mii1_txd1.rgmii1_td1 */
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_OUTPUT_PULLDOWN, MUX_MODE2)	/* mii1_txd0.rgmii1_td0 */
+			AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_OUTPUT_PULLDOWN, MUX_MODE2)	/* mii1_txclk.rgmii1_tclk */
+			AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE2)	/* mii1_rxclk.rgmii1_rclk */
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLDOWN, MUX_MODE2)	/* mii1_rxd3.rgmii1_rd3 */
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_INPUT_PULLDOWN, MUX_MODE2)	/* mii1_rxd2.rgmii1_rd2 */
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLDOWN, MUX_MODE2)	/* mii1_rxd1.rgmii1_rd1 */
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD0, PIN_INPUT_PULLDOWN, MUX_MODE2)	/* mii1_rxd0.rgmii1_rd0 */
 		>;
 	};
 
 	cpsw_sleep: cpsw_sleep {
 		pinctrl-single,pins = <
 			/* Slave 1 reset value */
-			AM33XX_IOPAD(0x914, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x918, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x91c, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x920, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x924, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x928, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x92c, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x930, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x934, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x938, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x93c, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x940, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD1, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MII1_RXD0, PIN_INPUT_PULLDOWN, MUX_MODE7)
 		>;
 	};
 
 	davinci_mdio_default: davinci_mdio_default {
 		pinctrl-single,pins = <
 			/* MDIO */
-			AM33XX_IOPAD(0x948, PIN_INPUT_PULLUP | SLEWCTRL_FAST | MUX_MODE0)	/* mdio_data.mdio_data */
-			AM33XX_IOPAD(0x94c, PIN_OUTPUT_PULLUP | MUX_MODE0)			/* mdio_clk.mdio_clk */
+			AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT_PULLUP | SLEWCTRL_FAST, MUX_MODE0)
+			AM33XX_PADCONF(AM335X_PIN_MDC, PIN_OUTPUT_PULLUP, MUX_MODE0)
 		>;
 	};
 
 	davinci_mdio_sleep: davinci_mdio_sleep {
 		pinctrl-single,pins = <
 			/* MDIO reset value */
-			AM33XX_IOPAD(0x948, PIN_INPUT_PULLDOWN | MUX_MODE7)
-			AM33XX_IOPAD(0x94c, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT_PULLDOWN, MUX_MODE7)
+			AM33XX_PADCONF(AM335X_PIN_MDC, PIN_INPUT_PULLDOWN, MUX_MODE7)
 		>;
 	};
 
 	usb_hub_ctrl: usb_hub_ctrl {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x944, PIN_OUTPUT_PULLUP | MUX_MODE7)     /* rmii1_refclk.gpio0_29 */
+			AM33XX_PADCONF(AM335X_PIN_RMII1_REF_CLK, PIN_OUTPUT_PULLUP, MUX_MODE7)     /* rmii1_refclk.gpio0_29 */
 		>;
 	};
 
 	mpu6050_pins: pinmux_mpu6050_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x968, PIN_INPUT | MUX_MODE7)    /* uart0_ctsn.gpio1_8 */
+			AM33XX_PADCONF(AM335X_PIN_UART0_CTSN, PIN_INPUT, MUX_MODE7)    /* uart0_ctsn.gpio1_8 */
 		>;
 	};
 
 	lps3331ap_pins: pinmux_lps3331ap_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x868, PIN_INPUT | MUX_MODE7)     /* gpmc_a10.gpio1_26 */
+			AM33XX_PADCONF(AM335X_PIN_GPMC_A10, PIN_INPUT, MUX_MODE7)     /* gpmc_a10.gpio1_26 */
 		>;
 	};
 };
diff -Naur --no-dereference a/arch/arm/boot/dts/am33xx.dtsi b/arch/arm/boot/dts/am33xx.dtsi
--- a/arch/arm/boot/dts/am33xx.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am33xx.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -19,7 +19,7 @@
 	#size-cells = <1>;
 	chosen { };
 
-	aliases {
+	aliases: aliases {
 		i2c0 = &i2c0;
 		i2c1 = &i2c1;
 		i2c2 = &i2c2;
@@ -46,6 +46,7 @@
 		#size-cells = <0>;
 		cpu@0 {
 			compatible = "arm,cortex-a8";
+			enable-method = "ti,am3352";
 			device_type = "cpu";
 			reg = <0>;
 
@@ -55,6 +56,17 @@
 			clock-names = "cpu";
 
 			clock-latency = <300000>; /* From omap-cpufreq driver */
+			cpu-idle-states = <&mpu_gate>;
+		};
+
+		idle-states {
+			mpu_gate: mpu_gate {
+				compatible = "arm,idle-state";
+				entry-latency-us = <40>;
+				exit-latency-us = <90>;
+				min-residency-us = <300>;
+				ti,idle-wkup-m3;
+			};
 		};
 	};
 
@@ -159,7 +171,7 @@
 	 * for the moment, just use a fake OCP bus entry to represent
 	 * the whole bus hierarchy.
 	 */
-	ocp {
+	ocp: ocp {
 		compatible = "simple-bus";
 		#address-cells = <1>;
 		#size-cells = <1>;
@@ -306,6 +318,39 @@
 			#interrupt-cells = <2>;
 			reg = <0x44e07000 0x1000>;
 			interrupts = <96>;
+			gpio-line-names =
+				"MDIO_DATA",	// 0
+				"MDIO_CLK",	// 1
+				"SPI0_SCLK",	// 2
+				"SPI0_D0",	// 3
+				"SPI0_D1",	// 4
+				"SPI0_CS0",	// 5
+				"SPI0_CS1",	// 6
+				"ECAP0_IN_PWM0_OUT",	// 7
+				"LCD_DATA12",	// 8
+				"LCD_DATA13",	// 9
+				"LCD_DATA14",	// 10
+				"LCD_DATA15",	// 11
+				"UART1_CTSN",	// 12
+				"UART1_RTSN",	// 13
+				"UART1_RXD",	// 14
+				"UART1_TXD",	// 15
+				"GMII1_TXD3",	// 16
+				"GMII1_TXD2",	// 17
+				"USB0_DRVVBUS",	// 18
+				"XDMA_EVENT_INTR0",	// 19
+				"XDMA_EVENT_INTR1",	// 20
+				"GMII1_TXD1",	// 21
+				"GPMC_AD8",	// 22
+				"GPMC_AD9",	// 23
+				"NC",	// 24
+				"NC",	// 25
+				"GPMC_AD10",	// 26
+				"GPMC_AD11",	// 27
+				"GMII1_TXD0",	// 28
+				"RMII1_REFCLK",	// 29
+				"GPMC_WAIT0",	// 30
+				"GPMC_WPN";	// 31
 		};
 
 		gpio1: gpio@4804c000 {
@@ -317,6 +362,39 @@
 			#interrupt-cells = <2>;
 			reg = <0x4804c000 0x1000>;
 			interrupts = <98>;
+			gpio-line-names =
+				"GPMC_AD0",	// 0
+				"GPMC_AD1",	// 1
+				"GPMC_AD2",	// 2
+				"GPMC_AD3",	// 3
+				"GPMC_AD4",	// 4
+				"GPMC_AD5",	// 5
+				"GPMC_AD6",	// 6
+				"GPMC_AD7",	// 7
+				"UART0_CTSN",	// 8
+				"UART0_RTSN",	// 9
+				"UART0_RXD",	// 10
+				"UART0_TXD",	// 11
+				"GPMC_AD12",	// 12
+				"GPMC_AD13",	// 13
+				"GPMC_AD14",	// 14
+				"GPMC_AD15",	// 15
+				"GPMC_A0",	// 16
+				"GPMC_A1",	// 17
+				"GPMC_A2",	// 18
+				"GPMC_A3",	// 19
+				"GPMC_A4",	// 20
+				"GPMC_A5",	// 21
+				"GPMC_A6",	// 22
+				"GPMC_A7",	// 23
+				"GPMC_A8",	// 24
+				"GPMC_A9",	// 25
+				"GPMC_A10",	// 26
+				"GPMC_A11",	// 27
+				"GPMC_BE1N",	// 28
+				"GPMC_CSN0",	// 29
+				"GPMC_CSN1",	// 30
+				"GPMC_CSN2";	// 31
 		};
 
 		gpio2: gpio@481ac000 {
@@ -328,6 +406,39 @@
 			#interrupt-cells = <2>;
 			reg = <0x481ac000 0x1000>;
 			interrupts = <32>;
+			gpio-line-names =
+				"GPMC_CSN3",	// 0
+				"GPMC_CLK",	// 1
+				"GPMC_ADVN_ALE",	// 2
+				"GPMC_OEN_REN",	// 3
+				"GPMC_WEN",	// 4
+				"GPMC_BE0N_CLE",	// 5
+				"LCD_DATA0",	// 6
+				"LCD_DATA1",	// 7
+				"LCD_DATA2",	// 8
+				"LCD_DATA3",	// 9
+				"LCD_DATA4",	// 10
+				"LCD_DATA5",	// 11
+				"LCD_DATA6",	// 12
+				"LCD_DATA7",	// 13
+				"LCD_DATA8",	// 14
+				"LCD_DATA9",	// 15
+				"LCD_DATA10",	// 16
+				"LCD_DATA11",	// 17
+				"GMII1_RXD3",	// 18
+				"GMII1_RXD2",	// 19
+				"GMII1_RXD1",	// 20
+				"GMII1_RXD0",	// 21
+				"LCD_VSYNC",	// 22
+				"LCD_HSYNC",	// 23
+				"LCD_PCLK",	// 24
+				"LCD_AC_BIAS_EN",	// 25
+				"MMC0_DAT3",	// 26
+				"MMC0_DAT2",	// 27
+				"MMC0_DAT1",	// 28
+				"MMC0_DAT0",	// 29
+				"MMC0_CLK",	// 30
+				"MMC0_CMD";	// 31
 		};
 
 		gpio3: gpio@481ae000 {
@@ -339,6 +450,39 @@
 			#interrupt-cells = <2>;
 			reg = <0x481ae000 0x1000>;
 			interrupts = <62>;
+			gpio-line-names =
+				"GMII1_COL",	// 0
+				"GMII1_CRS",	// 1
+				"GMII1_RXER",	// 2
+				"GMII1_TXEN",	// 3
+				"GMII1_RXDV",	// 4
+				"I2C0_SDA",	// 5
+				"I2C0_SCL",	// 6
+				"EMU0",	// 7
+				"EMU1",	// 8
+				"GMII1_TXCLK",	// 9
+				"GMII1_RXCLK",	// 10
+				"NC",	// 11
+				"NC",	// 12
+				"USB1_DRVVBUS",	// 13
+				"MCASP0_ACLKX",	// 14
+				"MCASP0_FSX",	// 15
+				"MCASP0_AXR0",	// 16
+				"MCASP0_AHCLKR",	// 17
+				"MCASP0_ACLKR",	// 18
+				"MCASP0_FSR",	// 19
+				"MCASP0_AXR1",	// 20
+				"MCASP0_AHCLKX",	// 21
+				"NC",	// 22
+				"NC",	// 23
+				"NC",	// 24
+				"NC",	// 25
+				"NC",	// 26
+				"NC",	// 27
+				"NC",	// 28
+				"NC",	// 29
+				"NC",	// 30
+				"NC";	// 31
 		};
 
 		uart0: serial@44e09000 {
@@ -770,6 +914,16 @@
 				status = "disabled";
 			};
 
+			eqep0: eqep@0x48300180 {
+				compatible = "ti,am33xx-eqep";
+				reg = <0x48300180 0x80>;
+				clocks = <&l4ls_gclk>;
+				clock-names = "fck";
+				interrupt-parent = <&intc>;
+				interrupts = <79>;
+				status = "disabled";
+			};
+
 			ehrpwm0: pwm@48300200 {
 				compatible = "ti,am3352-ehrpwm",
 					     "ti,am33xx-ehrpwm";
@@ -804,6 +958,16 @@
 				status = "disabled";
 			};
 
+			eqep1: eqep@0x48302180 {
+				compatible = "ti,am33xx-eqep";
+				reg = <0x48302180 0x80>;
+				clocks = <&l4ls_gclk>;
+				clock-names = "fck";
+				interrupt-parent = <&intc>;
+				interrupts = <88>;
+				status = "disabled";
+			};
+
 			ehrpwm1: pwm@48302200 {
 				compatible = "ti,am3352-ehrpwm",
 					     "ti,am33xx-ehrpwm";
@@ -838,6 +1002,16 @@
 				status = "disabled";
 			};
 
+			eqep2: eqep@0x48304180 {
+				compatible = "ti,am33xx-eqep";
+				reg = <0x48304180 0x80>;
+				clocks = <&l4ls_gclk>;
+				clock-names = "fck";
+				interrupt-parent = <&intc>;
+				interrupts = <89>;
+				status = "disabled";
+			};
+
 			ehrpwm2: pwm@48304200 {
 				compatible = "ti,am3352-ehrpwm",
 					     "ti,am33xx-ehrpwm";
@@ -924,6 +1098,94 @@
 			};
 		};
 
+		pruss_soc_bus: pruss-soc-bus@4a326004 {
+			compatible = "ti,am3356-pruss-soc-bus";
+			reg = <0x4a326004 0x4>;
+			ti,hwmods = "pruss";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+			status = "disabled";
+
+			pruss: pruss@4a300000 {
+				compatible = "ti,am3356-pruss";
+				reg = <0x4a300000 0x80000>;
+				interrupts = <20 21 22 23 24 25 26 27>;
+				interrupt-names = "host2", "host3", "host4",
+						  "host5", "host6", "host7",
+						  "host8", "host9";
+				#address-cells = <1>;
+				#size-cells = <1>;
+				ranges;
+				status = "disabled";
+
+				pruss_mem: memories@4a300000 {
+					reg = <0x4a300000 0x2000>,
+					      <0x4a302000 0x2000>,
+					      <0x4a310000 0x3000>;
+					reg-names = "dram0", "dram1",
+						    "shrdram2";
+				};
+
+				pruss_cfg: cfg@4a326000 {
+					compatible = "syscon";
+					reg = <0x4a326000 0x2000>;
+				};
+
+				pruss_iep: iep@4a32e000 {
+					compatible = "syscon";
+					reg = <0x4a32e000 0x31c>;
+				};
+
+				pruss_mii_rt: mii-rt@4a332000 {
+					compatible = "syscon";
+					reg = <0x4a332000 0x58>;
+				};
+
+				pruss_intc: interrupt-controller@4a320000 {
+					compatible = "ti,am3356-pruss-intc";
+					reg = <0x4a320000 0x2000>;
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
+
+				pru0: pru@4a334000 {
+					compatible = "ti,am3356-pru";
+					reg = <0x4a334000 0x2000>,
+					      <0x4a322000 0x400>,
+					      <0x4a322400 0x100>;
+					reg-names = "iram", "control", "debug";
+					firmware-name = "am335x-pru0-fw";
+					interrupt-parent = <&pruss_intc>;
+					interrupts = <16>, <17>;
+					interrupt-names = "vring", "kick";
+				};
+
+				pru1: pru@4a338000 {
+					compatible = "ti,am3356-pru";
+					reg = <0x4a338000 0x2000>,
+					      <0x4a324000 0x400>,
+					      <0x4a324400 0x100>;
+					reg-names = "iram", "control", "debug";
+					firmware-name = "am335x-pru1-fw";
+					interrupt-parent = <&pruss_intc>;
+					interrupts = <18>, <19>;
+					interrupt-names = "vring", "kick";
+				};
+
+				pruss_mdio: mdio@4a332400 {
+					compatible = "ti,davinci_mdio";
+					reg = <0x4a332400 0x90>;
+					clocks = <&dpll_core_m4_ck>;
+					clock-names = "fck";
+					bus_freq = <1000000>;
+					#address-cells = <1>;
+					#size-cells = <0>;
+					status = "disabled";
+				};
+			};
+		};
+
 		elm: elm@48080000 {
 			compatible = "ti,am3352-elm";
 			reg = <0x48080000 0x2000>;
@@ -1040,6 +1302,16 @@
 			reg = <0x48310000 0x2000>;
 			interrupts = <111>;
 		};
+
+		sgx: sgx@56000000 {
+			compatible = "ti,am3352-sgx530", "img,sgx530";
+			ti,hwmods = "gfx";
+			reg = <0x56000000 0x10000>;
+			interrupts = <37>;
+			clocks = <&gfx_fck_div_ck>;
+			clock-names = "fclk";
+			status = "disabled";
+		};
 	};
 };
 
diff -Naur --no-dereference a/arch/arm/boot/dts/am4372.dtsi b/arch/arm/boot/dts/am4372.dtsi
--- a/arch/arm/boot/dts/am4372.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am4372.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -44,6 +44,7 @@
 		#size-cells = <0>;
 		cpu: cpu@0 {
 			compatible = "arm,cortex-a9";
+			enable-method = "ti,am4372";
 			device_type = "cpu";
 			reg = <0>;
 
@@ -53,6 +54,17 @@
 			operating-points-v2 = <&cpu0_opp_table>;
 
 			clock-latency = <300000>; /* From omap-cpufreq driver */
+			cpu-idle-states = <&mpu_gate>;
+		};
+
+		idle-states {
+			mpu_gate: mpu_gate {
+				compatible = "arm,idle-state";
+				entry-latency-us = <40>;
+				exit-latency-us = <100>;
+				min-residency-us = <300>;
+				local-timer-stop;
+			};
 		};
 	};
 
@@ -945,6 +957,171 @@
 			interrupts = <GIC_SPI 111 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
+		pruss_soc_bus: pruss-soc-bus@54426004 {
+			compatible = "ti,am4376-pruss-soc-bus";
+			reg = <0x54426004 0x4>;
+			ti,hwmods = "pruss";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+			status = "disabled";
+
+			pruss1: pruss@54400000 {
+				compatible = "ti,am4376-pruss";
+				reg = <0x54400000 0x40000>;
+				interrupts = <GIC_SPI 20 IRQ_TYPE_LEVEL_HIGH
+					      GIC_SPI 21 IRQ_TYPE_LEVEL_HIGH
+					      GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH
+					      GIC_SPI 23 IRQ_TYPE_LEVEL_HIGH
+					      GIC_SPI 24 IRQ_TYPE_LEVEL_HIGH
+					      GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH
+					      GIC_SPI 27 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-names = "host2", "host3", "host4",
+						  "host5", "host6", "host8",
+						  "host9";
+				#address-cells = <1>;
+				#size-cells = <1>;
+				ranges;
+				status = "disabled";
+
+				pruss1_mem: memories@54400000 {
+					reg = <0x54400000 0x2000>,
+					      <0x54402000 0x2000>,
+					      <0x54410000 0x8000>;
+					reg-names = "dram0", "dram1",
+						    "shrdram2";
+				};
+
+				pruss1_cfg: cfg@54426000 {
+					compatible = "syscon";
+					reg = <0x54426000 0x2000>;
+				};
+
+				pruss1_iep: iep@5442e000 {
+					compatible = "syscon";
+					reg = <0x5442e000 0x31c>;
+				};
+
+				pruss1_mii_rt: mii-rt@54432000 {
+					compatible = "syscon";
+					reg = <0x54432000 0x58>;
+				};
+
+				pruss1_intc: interrupt-controller@54420000 {
+					compatible = "ti,am4376-pruss-intc";
+					reg = <0x54420000 0x2000>;
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
+
+				pru1_0: pru@54434000 {
+					compatible = "ti,am4376-pru";
+					reg = <0x54434000 0x3000>,
+					      <0x54422000 0x400>,
+					      <0x54422400 0x100>;
+					reg-names = "iram", "control", "debug";
+					firmware-name = "am437x-pru1_0-fw";
+					interrupt-parent = <&pruss1_intc>;
+					interrupts = <16>, <17>;
+					interrupt-names = "vring", "kick";
+				};
+
+				pru1_1: pru@54438000 {
+					compatible = "ti,am4376-pru";
+					reg = <0x54438000 0x3000>,
+					      <0x54424000 0x400>,
+					      <0x54424400 0x100>;
+					reg-names = "iram", "control", "debug";
+					firmware-name = "am437x-pru1_1-fw";
+					interrupt-parent = <&pruss1_intc>;
+					interrupts = <18>, <19>;
+					interrupt-names = "vring", "kick";
+				};
+
+				pruss1_mdio: mdio@54432400 {
+					compatible = "ti,davinci_mdio";
+					reg = <0x54432400 0x90>;
+					clocks = <&dpll_core_m4_ck>;
+					clock-names = "fck";
+					bus_freq = <1000000>;
+					#address-cells = <1>;
+					#size-cells = <0>;
+					status = "disabled";
+				};
+			};
+
+			pruss0: pruss@54440000 {
+				compatible = "ti,am4376-pruss";
+				reg = <0x54440000 0x40000>;
+				interrupts = <GIC_SPI 159 IRQ_TYPE_LEVEL_HIGH
+					      GIC_SPI 160 IRQ_TYPE_LEVEL_HIGH
+					      GIC_SPI 161 IRQ_TYPE_LEVEL_HIGH
+					      GIC_SPI 162 IRQ_TYPE_LEVEL_HIGH
+					      GIC_SPI 163 IRQ_TYPE_LEVEL_HIGH
+					      GIC_SPI 164 IRQ_TYPE_LEVEL_HIGH
+					      GIC_SPI 165 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-names = "host2", "host3", "host4",
+						  "host5", "host6", "host8",
+						  "host9";
+				#address-cells = <1>;
+				#size-cells = <1>;
+				ranges;
+				status = "disabled";
+
+				pruss0_mem: memories@54440000 {
+					reg = <0x54440000 0x1000>,
+					      <0x54442000 0x1000>;
+					reg-names = "dram0", "dram1";
+				};
+
+				pruss0_cfg: cfg@54466000 {
+					compatible = "syscon";
+					reg = <0x54466000 0x2000>;
+				};
+
+				pruss0_iep: iep@5446e000 {
+					compatible = "syscon";
+					reg = <0x6e000 0x31c>;
+				};
+
+				pruss0_mii_rt: mii-rt@54472000 {
+					compatible = "syscon";
+					reg = <0x54472000 0x58>;
+				};
+
+				pruss0_intc: interrupt-controller@54460000 {
+					compatible = "ti,am4376-pruss-intc";
+					reg = <0x54460000 0x2000>;
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
+
+				pru0_0: pru@54474000 {
+					compatible = "ti,am4376-pru";
+					reg = <0x54474000 0x1000>,
+					      <0x54462000 0x400>,
+					      <0x54462400 0x100>;
+					reg-names = "iram", "control", "debug";
+					firmware-name = "am437x-pru0_0-fw";
+					interrupt-parent = <&pruss0_intc>;
+					interrupts = <16>, <17>;
+					interrupt-names = "vring", "kick";
+				};
+
+				pru0_1: pru@54478000 {
+					compatible = "ti,am4376-pru";
+					reg = <0x54478000 0x1000>,
+					      <0x54464000 0x400>,
+					      <0x54464400 0x100>;
+					reg-names = "iram", "control", "debug";
+					firmware-name = "am437x-pru0_1-fw";
+					interrupt-parent = <&pruss0_intc>;
+					interrupts = <18>, <19>;
+					interrupt-names = "vring", "kick";
+				};
+			};
+		};
+
 		mcasp0: mcasp@48038000 {
 			compatible = "ti,am33xx-mcasp-audio";
 			ti,hwmods = "mcasp0";
@@ -1213,6 +1390,16 @@
 			ti,hwmods = "vpfe1";
 			status = "disabled";
 		};
+
+		sgx: sgx@56000000 {
+			compatible = "ti,am4376-sgx530", "img,sgx530";
+			ti,hwmods = "gfx";
+			reg = <0x56000000 0x10000>;
+			interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&gfx_fck_div_ck>;
+			clock-names = "fclk";
+			status = "disabled";
+		};
 	};
 };
 
diff -Naur --no-dereference a/arch/arm/boot/dts/am437x-gp-evm.dts b/arch/arm/boot/dts/am437x-gp-evm.dts
--- a/arch/arm/boot/dts/am437x-gp-evm.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am437x-gp-evm.dts	2020-06-19 21:00:31.404227715 -0400
@@ -67,7 +67,13 @@
 		debounce-delay-ms = <5>;
 		col-scan-delay-us = <2>;
 
-		row-gpios = <&gpio3 21 GPIO_ACTIVE_HIGH /* Bank3, pin21 */
+		pinctrl-names = "default", "sleep";
+		pinctrl-0 = <&matrix_keypad_default>;
+		pinctrl-1 = <&matrix_keypad_sleep>;
+
+		linux,wakeup;
+
+		row-gpios = <&gpio0 3 GPIO_ACTIVE_HIGH /* Bank0, pin3 */
 				&gpio4 3 GPIO_ACTIVE_HIGH /* Bank4, pin3 */
 				&gpio4 2 GPIO_ACTIVE_HIGH>; /* Bank4, pin2 */
 
@@ -83,7 +89,7 @@
 		};
 
 	lcd0: display {
-		compatible = "osddisplays,osd057T0559-34ts", "panel-dpi";
+		compatible = "osddisplays,osd070t1718-19ts", "panel-dpi";
 		label = "lcd";
 
 		backlight = <&lcd_bl>;
@@ -155,16 +161,29 @@
 	beeper: beeper {
 		compatible = "gpio-beeper";
 		pinctrl-names = "default";
-		pinctrl-0 = <&beeper_pins>;
+		pinctrl-0 = <&beeper_pins_default>;
+		pinctrl-1 = <&beeper_pins_sleep>;
 		gpios = <&gpio4 12 GPIO_ACTIVE_HIGH>;
 	};
+
+	audio_mstrclk: mclk_osc {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <12000000>;
+	};
 };
 
 &am43xx_pinmux {
 	pinctrl-names = "default", "sleep";
-	pinctrl-0 = <&wlan_pins_default>;
+	pinctrl-0 = <&wlan_pins_default &ddr3_vtt_toggle_default &unused_pins &debugss_pins>;
 	pinctrl-1 = <&wlan_pins_sleep>;
 
+	ddr3_vtt_toggle_default: ddr_vtt_toggle_default {
+		pinctrl-single,pins = <
+			0x25C (DS0_PULL_UP_DOWN_EN | PIN_OUTPUT_PULLUP | DS0_FORCE_OFF_MODE | MUX_MODE7) /* spi0_cs0.gpio5_7 */
+		>;
+	};
+
 	i2c0_pins: i2c0_pins {
 		pinctrl-single,pins = <
 			AM4372_IOPAD(0x988, PIN_INPUT_PULLUP | SLEWCTRL_FAST | MUX_MODE0)  /* i2c0_sda.i2c0_sda */
@@ -511,21 +530,114 @@
 		>;
 	};
 
+	beeper_pins_default: beeper_pins_default {
+		pinctrl-single,pins = <
+			AM4372_IOPAD(0x9e0, PIN_OUTPUT_PULLUP | MUX_MODE7)	/* cam1_field.gpio4_12 */
+		>;
+	};
+
+	beeper_pins_sleep: beeper_pins_sleep {
+		pinctrl-single,pins = <
+			AM4372_IOPAD(0x9e0, PIN_INPUT_PULLDOWN | MUX_MODE7)	/* cam1_field.gpio4_12 */
+		>;
+	};
+
+	unused_pins: unused_pins {
+		pinctrl-single,pins = <
+			AM4372_IOPAD(0x854, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0x858, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0x860, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0x864, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0x868, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0x86c, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0x950, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0x990, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0x994, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0x998, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0x99c, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0x9a0, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0xa3c, PIN_INPUT | PULL_DISABLE | MUX_MODE7)
+			AM4372_IOPAD(0xa40, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0xa44, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0xa48, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0xa4c, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0xa50, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0xa54, PIN_INPUT | PULL_DISABLE | MUX_MODE7)
+			AM4372_IOPAD(0xa58, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0xa60, PIN_INPUT | PULL_DISABLE | MUX_MODE7)
+			AM4372_IOPAD(0xa68, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0xa70, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0xa78, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0xa7c, PIN_INPUT | PULL_DISABLE)
+			AM4372_IOPAD(0xac8, PIN_INPUT_PULLDOWN)
+			AM4372_IOPAD(0xad4, PIN_INPUT_PULLDOWN)
+			AM4372_IOPAD(0xad8, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0xadc, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0xae0, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0xae4, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0xae8, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0xaec, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0xaf0, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0xaf4, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0xaf8, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0xafc, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0xb00, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0xb04, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0xb08, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0xb0c, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0xb10, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0xb14, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0xb18, PIN_INPUT_PULLDOWN | MUX_MODE7)
+		>;
+	};
+
+	debugss_pins: pinmux_debugss_pins {
+		pinctrl-single,pins = <
+			AM4372_IOPAD(0xa90, PIN_INPUT_PULLDOWN)
+			AM4372_IOPAD(0xa94, PIN_INPUT_PULLDOWN)
+			AM4372_IOPAD(0xa98, PIN_INPUT_PULLDOWN)
+			AM4372_IOPAD(0xa9c, PIN_INPUT_PULLDOWN)
+			AM4372_IOPAD(0xaa0, PIN_INPUT_PULLDOWN)
+			AM4372_IOPAD(0xaa4, PIN_INPUT_PULLDOWN)
+			AM4372_IOPAD(0xaa8, PIN_INPUT_PULLDOWN)
+		>;
+	};
+
 	uart0_pins_default: uart0_pins_default {
 		pinctrl-single,pins = <
-			AM4372_IOPAD(0x968, PIN_INPUT | MUX_MODE0)		/* uart0_ctsn.uart0_ctsn */
-			AM4372_IOPAD(0x96C, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* uart0_rtsn.uart0_rtsn */
-			AM4372_IOPAD(0x970, PIN_INPUT_PULLUP | MUX_MODE0)	/* uart0_rxd.uart0_rxd */
-			AM4372_IOPAD(0x974, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* uart0_txd.uart0_txd */
+			AM4372_IOPAD(0x968, DS0_PULL_UP_DOWN_EN | INPUT_EN | MUX_MODE0) /* uart0_ctsn.uart0_ctsn */
+			AM4372_IOPAD(0x96C, DS0_PULL_UP_DOWN_EN | INPUT_EN | MUX_MODE0) /* uart0_rtsn.uart0_rtsn */
+			AM4372_IOPAD(0x970, PIN_INPUT_PULLUP | SLEWCTRL_FAST | DS0_PULL_UP_DOWN_EN | MUX_MODE0) /* uart0_rxd.uart0_rxd */
+			AM4372_IOPAD(0x974, PIN_INPUT | PULL_DISABLE | SLEWCTRL_FAST | DS0_PULL_UP_DOWN_EN | MUX_MODE0) /* uart0_txd.uart0_txd */
 		>;
 	};
 
-	beeper_pins: beeper_pins {
+	uart0_pins_sleep: uart0_pins_sleep {
 		pinctrl-single,pins = <
-			AM4372_IOPAD(0x9e0, PIN_OUTPUT_PULLUP | MUX_MODE7)	/* cam1_field.gpio4_12 */
+			AM4372_IOPAD(0x968, DS0_PULL_UP_DOWN_EN | INPUT_EN | MUX_MODE7) /* uart0_ctsn.uart0_ctsn */
+			AM4372_IOPAD(0x96C, DS0_PULL_UP_DOWN_EN | INPUT_EN | MUX_MODE7) /* uart0_rtsn.uart0_rtsn */
+			AM4372_IOPAD(0x970, PIN_INPUT_PULLUP | SLEWCTRL_FAST | DS0_PULL_UP_DOWN_EN | MUX_MODE0) /* uart0_rxd.uart0_rxd */
+			AM4372_IOPAD(0x974, PIN_INPUT_PULLDOWN | SLEWCTRL_FAST | DS0_PULL_UP_DOWN_EN | MUX_MODE0) /* uart0_txd.uart0_txd */
+		>;
+	};
+
+	matrix_keypad_default: matrix_keypad_default {
+		pinctrl-single,pins = <
+			AM4372_IOPAD(0x9a4, PIN_OUTPUT | MUX_MODE7)
+			AM4372_IOPAD(0x9a8, PIN_OUTPUT | MUX_MODE7)
+			AM4372_IOPAD(0x9ac, PIN_INPUT | PULL_DISABLE | MUX_MODE9)
+			AM4372_IOPAD(0x954, PIN_INPUT_PULLDOWN | MUX_MODE0)
 		>;
 	};
 
+	matrix_keypad_sleep: matrix_keypad_sleep {
+		pinctrl-single,pins = <
+			AM4372_IOPAD(0x9a4, PULL_UP | MUX_MODE7)
+			AM4372_IOPAD(0x9a8, PULL_UP | MUX_MODE7)
+			AM4372_IOPAD(0x9ac, PIN_INPUT | PULL_DISABLE | MUX_MODE9)
+			AM4372_IOPAD(0x954, PIN_INPUT_PULLDOWN | MUX_MODE0)
+		>;
+	};
 };
 
 &uart0 {
@@ -609,7 +721,7 @@
 		compatible = "ovti,ov2659";
 		reg = <0x30>;
 
-		clocks = <&refclk 0>;
+		clocks = <&audio_mstrclk>;
 		clock-names = "xvclk";
 
 		port {
@@ -650,7 +762,7 @@
 		compatible = "ovti,ov2659";
 		reg = <0x30>;
 
-		clocks = <&refclk 0>;
+		clocks = <&audio_mstrclk>;
 		clock-names = "xvclk";
 
 		port {
@@ -931,6 +1043,13 @@
 	};
 };
 
+&uart0 {
+	status = "okay";
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&uart0_pins_default>;
+	pinctrl-1 = <&uart0_pins_sleep>;
+};
+
 &dss {
 	status = "ok";
 
@@ -1020,3 +1139,24 @@
 &cpu {
 	cpu0-supply = <&dcdc2>;
 };
+
+&wkup_m3_ipc {
+	ti,set-io-isolation;
+	ti,scale-data-fw = "am43x-evm-scale-data.bin";
+};
+
+&sgx {
+	status = "okay";
+};
+
+&pruss_soc_bus {
+	status = "okay";
+
+	pruss1: pruss@54400000 {
+		status = "okay";
+	};
+
+	pruss0: pruss@54440000 {
+		status = "okay";
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am437x-gp-evm-hdmi.dts b/arch/arm/boot/dts/am437x-gp-evm-hdmi.dts
--- a/arch/arm/boot/dts/am437x-gp-evm-hdmi.dts	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am437x-gp-evm-hdmi.dts	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,122 @@
+/*
+ * Copyright (C) 2017 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/* AM437x GP EVM with HDMI output */
+
+#include "am437x-gp-evm.dts"
+
+/delete-node/ &lcd0;
+
+#include <dt-bindings/sound/sii902x-audio.h>
+
+/ {
+	aliases {
+		display0 = &hdmi;
+	};
+
+	hdmi: connector {
+		compatible = "hdmi-connector";
+		label = "hdmi";
+
+		type = "b";
+
+		port {
+			hdmi_connector_in: endpoint {
+				remote-endpoint = <&sii9022_out>;
+			};
+		};
+	};
+
+	sound@1 {
+		compatible = "simple-audio-card";
+		simple-audio-card,name = "HDMI";
+		simple-audio-card,format = "i2s";
+		simple-audio-card,bitclock-master = <&hdmi_dailink_master>;
+		simple-audio-card,frame-master = <&hdmi_dailink_master>;
+		hdmi_dailink_master: simple-audio-card,cpu {
+			sound-dai = <&mcasp1>;
+			system-clock-frequency = <24000000>;
+			system-clock-direction-out;
+		};
+
+		simple-audio-card,codec {
+			sound-dai = <&sii9022>;
+		};
+	};
+
+	sii9022_mclk: sii9022_mclk {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <12000000>;
+	};
+};
+
+&lcd_bl {
+	status = "disabled";
+};
+
+&sound0 {
+	status = "disabled";
+};
+
+&tlv320aic3106 {
+	status = "disabled";
+};
+
+&i2c1 {
+	sii9022: sii9022@3b {
+		#sound-dai-cells = <0>;
+		compatible = "sil,sii9022";
+		reg = <0x3b>;
+
+		interrupt-parent = <&gpio3>;
+		interrupts = <24 IRQ_TYPE_LEVEL_LOW>;
+
+		i2s-fifo-routing = <
+			(ENABLE_BIT|CONNECT_SD0)
+			0
+			0
+			0
+		>;
+		clocks = <&sii9022_mclk>;
+		clock-names = "mclk";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				sii9022_in: endpoint {
+					remote-endpoint = <&dpi_out>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				sii9022_out: endpoint {
+					remote-endpoint = <&hdmi_connector_in>;
+				};
+			};
+		};
+	};
+};
+
+&dpi_out {
+	remote-endpoint = <&sii9022_in>;
+	data-lines = <24>;
+};
+
+/* Override SelLCDorHDMI from am437x-gp-evm.dts to select HDMI */
+&gpio5 {
+	p8 {
+		output-low;
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am437x-idk-evm.dts b/arch/arm/boot/dts/am437x-idk-evm.dts
--- a/arch/arm/boot/dts/am437x-idk-evm.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am437x-idk-evm.dts	2020-06-19 21:00:31.404227715 -0400
@@ -22,6 +22,11 @@
 		stdout-path = &uart0;
 	};
 
+	aliases {
+		ethernet2 = &pruss1_emac0;
+		ethernet3 = &pruss1_emac1;
+	};
+
 	v24_0d: fixed-regulator-v24_0d {
 		compatible = "regulator-fixed";
 		regulator-name = "V24_0D";
@@ -173,6 +178,39 @@
 			default-state = "off";
 		};
 	};
+
+	/* Dual-MAC Ethernet application node on PRU-ICSS1 */
+	pruss1_eth {
+		compatible = "ti,am4376-prueth";
+		prus = <&pru1_0>, <&pru1_1>;
+		firmware-name = "ti-pruss/am437x-pru0-prueth-fw.elf",
+				"ti-pruss/am437x-pru1-prueth-fw.elf";
+		sram = <&ocmcram>;
+		interrupt-parent = <&pruss1_intc>;
+		mii-rt = <&pruss1_mii_rt>;
+		iep = <&pruss1_iep>;
+
+		pinctrl-0 = <&pruss1_eth_default>;
+		pinctrl-names = "default";
+
+		pruss1_emac0: ethernet-mii0 {
+			phy-handle = <&pruss1_eth0_phy>;
+			phy-mode = "mii";
+			interrupts = <20>, <22>;
+			interrupt-names = "rx", "tx";
+			/* Filled in by bootloader */
+			local-mac-address = [00 00 00 00 00 00];
+		};
+
+		pruss1_emac1: ethernet-mii1 {
+			phy-handle = <&pruss1_eth1_phy>;
+			phy-mode = "mii";
+			interrupts = <21>, <23>;
+			interrupt-names = "rx", "tx";
+			/* Filled in by bootloader */
+			local-mac-address = [00 00 00 00 00 00];
+		};
+	};
 };
 
 &am43xx_pinmux {
@@ -308,6 +346,51 @@
 		>;
 	};
 
+	pruss1_mdio_default: pruss1_mdio_default {
+		pinctrl-single,pins = <
+			AM4372_IOPAD(0x88c, (PIN_OUTPUT | MUX_MODE5))		/* gpmc_clk.pr1_mdio_mdclk */
+			AM4372_IOPAD(0xa70, (PIN_INPUT  | MUX_MODE8))		/* xdma_event_intr0.pr1_mdio_data */
+			AM4372_IOPAD(0xa00, (PIN_INPUT_PULLUP | MUX_MODE7))	/* cam1_data6.gpio4_20 */
+		>;
+	};
+
+	pruss1_eth_default: pruss1_eth_default {
+		pinctrl-single,pins = <
+			AM4372_IOPAD(0x8a0, (PIN_INPUT  | MUX_MODE2))	/* dss_data0.pr1_mii_mt0_clk */
+			AM4372_IOPAD(0x8b4, (PIN_OUTPUT | MUX_MODE2))	/* dss_data5.pr1_mii0_txd0 */
+			AM4372_IOPAD(0x8b0, (PIN_OUTPUT | MUX_MODE2))	/* dss_data4.pr1_mii0_txd1 */
+			AM4372_IOPAD(0x8ac, (PIN_OUTPUT | MUX_MODE2))	/* dss_data3.pr1_mii0_txd2 */
+			AM4372_IOPAD(0x8a8, (PIN_OUTPUT | MUX_MODE2))	/* dss_data2.pr1_mii0_txd3 */
+			AM4372_IOPAD(0x8cc, (PIN_INPUT  | MUX_MODE5))	/* dss_data11.pr1_mii0_rxd0 */
+			AM4372_IOPAD(0x8c8, (PIN_INPUT  | MUX_MODE5))	/* dss_data10.pr1_mii0_rxd1 */
+			AM4372_IOPAD(0x8c4, (PIN_INPUT  | MUX_MODE5))	/* dss_data9.pr1_mii0_rxd2 */
+			AM4372_IOPAD(0x8c0, (PIN_INPUT  | MUX_MODE5))	/* dss_data8.pr1_mii0_rxd3 */
+			AM4372_IOPAD(0x8a4, (PIN_OUTPUT | MUX_MODE2))	/* dss_data1.pr1_mii0_txen */
+			AM4372_IOPAD(0x8d8, (PIN_INPUT  | MUX_MODE5))	/* dss_data14.pr1_mii_mr0_clk */
+			AM4372_IOPAD(0x8dc, (PIN_INPUT  | MUX_MODE5))	/* dss_data15.pr1_mii0_rxdv */
+			AM4372_IOPAD(0x8d4, (PIN_INPUT  | MUX_MODE5))	/* dss_data13.pr1_mii0_rxer */
+			AM4372_IOPAD(0x8d0, (PIN_INPUT  | MUX_MODE5))	/* dss_data12.pr1_mii0_rxlink */
+			AM4372_IOPAD(0xa40, (PIN_INPUT  | MUX_MODE5))	/* gpio5_10.pr1_mii0_crs */
+			AM4372_IOPAD(0xa38, (PIN_INPUT  | MUX_MODE5))	/* gpio5_8.pr1_mii0_col */
+			AM4372_IOPAD(0x858, (PIN_INPUT  | MUX_MODE5))	/* gpmc_a6.pr1_mii_mt1_clk */
+			AM4372_IOPAD(0x854, (PIN_OUTPUT | MUX_MODE5))	/* gpmc_a5.pr1_mii1_txd0 */
+			AM4372_IOPAD(0x850, (PIN_OUTPUT | MUX_MODE5))	/* gpmc_a4.pr1_mii1_txd1 */
+			AM4372_IOPAD(0x84c, (PIN_OUTPUT | MUX_MODE5))	/* gpmc_a3.pr1_mii1_txd2 */
+			AM4372_IOPAD(0x848, (PIN_OUTPUT | MUX_MODE5))	/* gpmc_a2.pr1_mii1_txd3 */
+			AM4372_IOPAD(0x86c, (PIN_INPUT  | MUX_MODE5))	/* gpmc_a11.pr1_mii1_rxd0 */
+			AM4372_IOPAD(0x868, (PIN_INPUT  | MUX_MODE5))	/* gpmc_a10.pr1_mii1_rxd1 */
+			AM4372_IOPAD(0x864, (PIN_INPUT  | MUX_MODE5))	/* gpmc_a9.pr1_mii1_rxd2 */
+			AM4372_IOPAD(0x860, (PIN_INPUT  | MUX_MODE5))	/* gpmc_a8.pr1_mii1_rxd3 */
+			AM4372_IOPAD(0x840, (PIN_OUTPUT | MUX_MODE5))	/* gpmc_a0.pr1_mii1_txen */
+			AM4372_IOPAD(0x85c, (PIN_INPUT  | MUX_MODE5))	/* gpmc_a7.pr1_mii_mr1_clk */
+			AM4372_IOPAD(0x844, (PIN_INPUT  | MUX_MODE5))	/* gpmc_a1.pr1_mii1_rxdv */
+			AM4372_IOPAD(0x874, (PIN_INPUT  | MUX_MODE5))	/* gpmc_wpn.pr1_mii1_rxer */
+			AM4372_IOPAD(0xa4c, (PIN_INPUT  | MUX_MODE5))	/* gpio5_13.pr1_mii1_rxlink */
+			AM4372_IOPAD(0xa44, (PIN_INPUT  | MUX_MODE5))	/* gpio5_11.pr1_mii1_crs */
+			AM4372_IOPAD(0x878, (PIN_INPUT  | MUX_MODE5))	/* gpmc_be1n.pr1_mii1_col */
+		>;
+	};
+
 	qspi_pins_default: qspi_pins_default {
 		pinctrl-single,pins = <
 			AM4372_IOPAD(0x87c, PIN_OUTPUT_PULLUP | MUX_MODE3)	/* gpmc_csn0.qspi_csn */
@@ -533,3 +616,36 @@
 		opp-suspend;
 	};
 };
+
+&sgx {
+	status = "okay";
+};
+
+&pruss_soc_bus {
+	status = "okay";
+
+	pruss1: pruss@54400000 {
+		status = "okay";
+	};
+
+	pruss0: pruss@54440000 {
+		status = "okay";
+	};
+};
+
+&pruss1_mdio {
+	pinctrl-0 = <&pruss1_mdio_default>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	reset-gpios = <&gpio4 20 GPIO_ACTIVE_LOW>;
+	reset-delay-us = <2>;	/* PHY datasheet states 1uS min */
+
+	pruss1_eth0_phy: ethernet-phy@0 {
+		reg = <0>;
+	};
+
+	pruss1_eth1_phy: ethernet-phy@1 {
+		reg = <1>;
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am437x-sk-evm.dts b/arch/arm/boot/dts/am437x-sk-evm.dts
--- a/arch/arm/boot/dts/am437x-sk-evm.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am437x-sk-evm.dts	2020-06-19 21:00:31.404227715 -0400
@@ -275,6 +275,12 @@
 		>;
 	};
 
+	clkout1_pin: pinmux_clkout1_pin {
+		pinctrl-single,pins = <
+			0x270 (PIN_OUTPUT_PULLDOWN | MUX_MODE3)	/* XDMA_EVENT_INTR0/CLKOUT1 */
+		>;
+	};
+
 	cpsw_default: cpsw_default {
 		pinctrl-single,pins = <
 			/* Slave 1 */
@@ -596,6 +602,25 @@
 	pinctrl-0 = <&i2c1_pins>;
 	clock-frequency = <400000>;
 
+	ov2659@30 {
+		compatible = "ovti,ov2659";
+		reg = <0x30>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&clkout1_pin>;
+
+		clocks = <&clkout1_mux_ck>;
+		clock-names = "xvclk";
+		assigned-clocks = <&clkout1_mux_ck>;
+		assigned-clock-parents = <&clkout1_osc_div_ck>;
+
+		port {
+			ov2659_1: endpoint {
+				remote-endpoint = <&vpfe0_ep>;
+				link-frequencies = /bits/ 64 <70000000>;
+			};
+		};
+	};
+
 	edt-ft5306@38 {
 		status = "okay";
 		compatible = "edt,edt-ft5306", "edt,edt-ft5x06";
@@ -872,7 +897,7 @@
 	/* Camera port */
 	port {
 		vpfe0_ep: endpoint {
-			/* remote-endpoint = <&sensor>; add once we have it */
+			remote-endpoint = <&ov2659_1>;
 			ti,am437x-vpfe-interface = <0>;
 			bus-width = <8>;
 			hsync-active = <0>;
@@ -880,3 +905,23 @@
 		};
 	};
 };
+
+&wkup_m3_ipc {
+	ti,scale-data-fw = "am43x-evm-scale-data.bin";
+};
+
+&sgx {
+	status = "okay";
+};
+
+&pruss_soc_bus {
+	status = "okay";
+
+	pruss1: pruss@54400000 {
+		status = "okay";
+	};
+
+	pruss0: pruss@54440000 {
+		status = "okay";
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am43x-epos-evm.dts b/arch/arm/boot/dts/am43x-epos-evm.dts
--- a/arch/arm/boot/dts/am43x-epos-evm.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am43x-epos-evm.dts	2020-06-19 21:00:31.404227715 -0400
@@ -45,7 +45,7 @@
 	};
 
 	lcd0: display {
-		compatible = "osddisplays,osd057T0559-34ts", "panel-dpi";
+		compatible = "osddisplays,osd070t1718-19ts", "panel-dpi";
 		label = "lcd";
 
 		backlight = <&lcd_bl>;
@@ -147,6 +147,12 @@
 			system-clock-frequency = <12000000>;
 		};
 	};
+
+	audio_mstrclk: mclk_osc {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <12000000>;
+	};
 };
 
 &am43xx_pinmux {
@@ -155,7 +161,6 @@
 
 		unused_pins: unused_pins {
 			pinctrl-single,pins = <
-				AM4372_IOPAD(0x848, DS0_PIN_OUTPUT_PULLUP | PIN_OUTPUT_PULLUP | MUX_MODE7)
 				AM4372_IOPAD(0x850, DS0_PIN_INPUT | PIN_INPUT_PULLDOWN | MUX_MODE7)
 				AM4372_IOPAD(0x858, DS0_PIN_INPUT | PIN_INPUT_PULLDOWN | MUX_MODE7)
 				AM4372_IOPAD(0x860, DS0_PIN_INPUT | PIN_INPUT_PULLDOWN | MUX_MODE7)
@@ -590,7 +595,7 @@
 	status = "okay";
 	pinctrl-names = "default";
 	pinctrl-0 = <&i2c0_pins>;
-	clock-frequency = <400000>;
+	clock-frequency = <100000>;
 
 	tps65218: tps65218@24 {
 		reg = <0x24>;
@@ -687,6 +692,21 @@
 		IOVDD-supply = <&dcdc4>; /* V3_3D -> DCDC4 */
 		DVDD-supply = <&ldo1>; /* V1_8AUD -> V1_8D -> LDO1 */
 	};
+
+	ov2659@30 {
+		compatible = "ovti,ov2659";
+		reg = <0x30>;
+
+		clocks = <&audio_mstrclk>;
+		clock-names = "xvclk";
+
+		port {
+			ov2659_1: endpoint {
+				remote-endpoint = <&vpfe1_ep>;
+				link-frequencies = /bits/ 64 <70000000>;
+			};
+		};
+	};
 };
 
 &i2c2 {
@@ -953,7 +973,7 @@
 
 	port {
 		vpfe1_ep: endpoint {
-			/* remote-endpoint = <&sensor>; add once we have it */
+			remote-endpoint = <&ov2659_1>;
 			ti,am437x-vpfe-interface = <0>;
 			bus-width = <8>;
 			hsync-active = <0>;
@@ -995,3 +1015,11 @@
 &cpu {
 	cpu0-supply = <&dcdc2>;
 };
+
+&wkup_m3_ipc {
+	ti,scale-data-fw = "am43x-evm-scale-data.bin";
+};
+
+&sgx {
+	status = "okay";
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am43x-epos-evm-hdmi.dts b/arch/arm/boot/dts/am43x-epos-evm-hdmi.dts
--- a/arch/arm/boot/dts/am43x-epos-evm-hdmi.dts	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am43x-epos-evm-hdmi.dts	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,126 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/* AM43x EPOS EVM with HDMI output */
+
+#include "am43x-epos-evm.dts"
+
+/delete-node/ &lcd0;
+
+#include <dt-bindings/sound/sii902x-audio.h>
+
+/ {
+	aliases {
+		display0 = &hdmi;
+	};
+
+	hdmi: connector {
+		compatible = "hdmi-connector";
+		label = "hdmi";
+
+		type = "b";
+
+		port {
+			hdmi_connector_in: endpoint {
+				remote-endpoint = <&sii9022_out>;
+			};
+		};
+	};
+
+	sound@1 {
+		compatible = "simple-audio-card";
+		simple-audio-card,name = "HDMI";
+		simple-audio-card,format = "i2s";
+		simple-audio-card,bitclock-master = <&hdmi_dailink_master>;
+		simple-audio-card,frame-master = <&hdmi_dailink_master>;
+		hdmi_dailink_master: simple-audio-card,cpu {
+			sound-dai = <&mcasp1>;
+			system-clock-frequency = <24000000>;
+			system-clock-direction-out;
+		};
+
+		simple-audio-card,codec {
+			sound-dai = <&sii9022>;
+		};
+	};
+
+	sii9022_mclk: sii9022_mclk {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <12000000>;
+	};
+};
+
+&lcd_bl {
+	status = "disabled";
+};
+
+&sound0 {
+	status = "disabled";
+};
+
+&am43xx_pinmux {
+	sii9022_pins: sii9022_pins {
+		pinctrl-single,pins = <
+			AM4372_IOPAD(0x848, PIN_INPUT | MUX_MODE7)	/* gpmc_a2.gpio1_18 */
+		>;
+	};
+};
+
+&i2c2 {
+	sii9022: sii9022@3b {
+		#sound-dai-cells = <0>;
+		compatible = "sil,sii9022";
+		reg = <0x3b>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&sii9022_pins>;
+
+		interrupt-parent = <&gpio1>;
+		interrupts = <18 IRQ_TYPE_LEVEL_LOW>;
+
+		i2s-fifo-routing = <
+			(ENABLE_BIT|CONNECT_SD0)
+			0
+			0
+			0
+		>;
+		clocks = <&sii9022_mclk>;
+		clock-names = "mclk";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				sii9022_in: endpoint {
+					remote-endpoint = <&dpi_out>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				sii9022_out: endpoint {
+					remote-endpoint = <&hdmi_connector_in>;
+				};
+			};
+		};
+	};
+};
+
+&dpi_out {
+	remote-endpoint = <&sii9022_in>;
+	data-lines = <24>;
+};
+
+/* Override SelLCDorHDMI from am43x-epos-evm.dts to select HDMI */
+&gpio2 {
+	p1 {
+		output-low;
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am43xx-clocks.dtsi b/arch/arm/boot/dts/am43xx-clocks.dtsi
--- a/arch/arm/boot/dts/am43xx-clocks.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am43xx-clocks.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -707,6 +707,60 @@
 		ti,bit-shift = <8>;
 		reg = <0x2a48>;
 	};
+
+	clkout1_osc_div_ck: clkout1_osc_div_ck {
+		#clock-cells = <0>;
+		compatible = "ti,divider-clock";
+		clocks = <&sys_clkin_ck>;
+		ti,bit-shift = <20>;
+		ti,max-div = <4>;
+		reg = <0x4100>;
+	};
+
+	clkout1_src2_mux_ck: clkout1_src2_mux_ck {
+		#clock-cells = <0>;
+		compatible = "ti,mux-clock";
+		clocks = <&clk_rc32k_ck>, <&sysclk_div>, <&dpll_ddr_m2_ck>,
+			 <&dpll_per_m2_ck>, <&dpll_disp_m2_ck>,
+			 <&dpll_mpu_m2_ck>;
+		reg = <0x4100>;
+	};
+
+	clkout1_src2_pre_div_ck: clkout1_src2_pre_div_ck {
+		#clock-cells = <0>;
+		compatible = "ti,divider-clock";
+		clocks = <&clkout1_src2_mux_ck>;
+		ti,bit-shift = <4>;
+		ti,max-div = <8>;
+		reg = <0x4100>;
+	};
+
+	clkout1_src2_post_div_ck: clkout1_src2_post_div_ck {
+		#clock-cells = <0>;
+		compatible = "ti,divider-clock";
+		clocks = <&clkout1_src2_pre_div_ck>;
+		ti,bit-shift = <8>;
+		ti,max-div = <32>;
+		ti,index-power-of-two;
+		reg = <0x4100>;
+	};
+
+	clkout1_mux_ck: clkout1_mux_ck {
+		#clock-cells = <0>;
+		compatible = "ti,mux-clock";
+		clocks = <&clkout1_osc_div_ck>, <&clk_rc32k_ck>,
+			 <&clkout1_src2_post_div_ck>, <&dpll_extdev_m2_ck>;
+		ti,bit-shift = <16>;
+		reg = <0x4100>;
+	};
+
+	clkout1_ck: clkout1_ck {
+		#clock-cells = <0>;
+		compatible = "ti,gate-clock";
+		clocks = <&clkout1_mux_ck>;
+		ti,bit-shift = <23>;
+		reg = <0x4100>;
+	};
 };
 
 &prcm {
diff -Naur --no-dereference a/arch/arm/boot/dts/am571x-idk.dts b/arch/arm/boot/dts/am571x-idk.dts
--- a/arch/arm/boot/dts/am571x-idk.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am571x-idk.dts	2020-06-19 21:00:31.404227715 -0400
@@ -13,6 +13,7 @@
 #include "dra7-mmc-iodelay.dtsi"
 #include "dra72x-mmc-iodelay.dtsi"
 #include "am57xx-idk-common.dtsi"
+#include "dra7-ipu-dsp-common.dtsi"
 
 / {
 	model = "TI AM5718 IDK";
@@ -23,6 +24,33 @@
 		reg = <0x0 0x80000000 0x0 0x40000000>;
 	};
 
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		ipu2_memory_region: ipu2-memory@95800000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x95800000 0x0 0x3800000>;
+			reusable;
+			status = "okay";
+		};
+
+		dsp1_memory_region: dsp1-memory@99000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x99000000 0x0 0x4000000>;
+			reusable;
+			status = "okay";
+		};
+
+		ipu1_memory_region: ipu1-memory@9d000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x9d000000 0x0 0x2000000>;
+			reusable;
+			status = "okay";
+		};
+	};
+
 	leds {
 		compatible = "gpio-leds";
 		cpu0-led {
@@ -71,30 +99,24 @@
 	vbus-gpio = <&gpio7 22 GPIO_ACTIVE_HIGH>;
 };
 
-&mailbox5 {
+&ipu2 {
 	status = "okay";
-	mbox_ipu1_ipc3x: mbox_ipu1_ipc3x {
-		status = "okay";
-	};
-	mbox_dsp1_ipc3x: mbox_dsp1_ipc3x {
-		status = "okay";
-	};
+	memory-region = <&ipu2_memory_region>;
 };
 
-&mailbox6 {
+&ipu1 {
 	status = "okay";
-	mbox_ipu2_ipc3x: mbox_ipu2_ipc3x {
-		status = "okay";
-	};
+	memory-region = <&ipu1_memory_region>;
 };
 
-&pcie1_rc {
+&dsp1 {
 	status = "okay";
-	gpios = <&gpio3 23 GPIO_ACTIVE_HIGH>;
+	memory-region = <&dsp1_memory_region>;
 };
 
-&pcie1_ep {
-	gpios = <&gpio3 23 GPIO_ACTIVE_HIGH>;
+&pcie1_rc {
+	status = "okay";
+	gpios = <&gpio5 18 GPIO_ACTIVE_HIGH>;
 };
 
 &mmc1 {
@@ -110,6 +132,24 @@
 	pinctrl-2 = <&mmc2_pins_ddr_rev20 &mmc2_iodelay_ddr_conf>;
 };
 
-&cpu0 {
-	vdd-supply = <&smps12_reg>;
+&ov2659_1 {
+	remote-endpoint = <&vin1b_ep>;
 };
+
+&vin1b {
+	vin1b_ep: endpoint@2 {
+		slave-mode;
+		remote-endpoint = <&ov2659_1>;
+	};
+};
+
+&vip1 {
+	status = "okay";
+};
+
+&pruss2_mdio {
+	reset-gpios = <&gpio5 9 GPIO_ACTIVE_LOW>;
+	reset-delay-us = <2>;   /* PHY datasheet states 1uS min */
+};
+
+#include "dra7-ipu-common-early-boot.dtsi"
diff -Naur --no-dereference a/arch/arm/boot/dts/am5729-beagleboneai.dts b/arch/arm/boot/dts/am5729-beagleboneai.dts
--- a/arch/arm/boot/dts/am5729-beagleboneai.dts	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am5729-beagleboneai.dts	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,2883 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+
+#include "dra74x.dtsi"
+#include "am57xx-commercial-grade.dtsi"
+#include "dra74x-mmc-iodelay.dtsi"
+#include "dra74-ipu-dsp-common.dtsi"
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/pinctrl/dra.h>
+
+/ {
+	model = "BeagleBoard.org BeagleBone AI";
+	compatible = "beagle,am5729-beagleboneai", "ti,am5728",
+		     "ti,dra742", "ti,dra74", "ti,dra7";
+
+	aliases {
+		rtc0 = &tps659038_rtc;
+		rtc1 = &rtc;
+		display0 = &hdmi_conn;
+	};
+
+	chosen {
+		stdout-path = &uart1;
+		base_dtb = "am5729-beagleboneai.dts";
+		base_dtb_timestamp = __TIMESTAMP__;
+	};
+
+	memory@0 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 0x0 0x40000000>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		cmem_block_mem_1_ocmc3: cmem_block_mem@40500000 {
+			reg = <0x0 0x40500000 0x0 0x100000>;
+			no-map;
+			status = "okay";
+		};
+
+		ipu2_memory_region: ipu2-memory@95800000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x95800000 0x0 0x3800000>;
+			reusable;
+			status = "okay";
+		};
+
+		dsp1_memory_region: dsp1-memory@99000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x99000000 0x0 0x4000000>;
+			reusable;
+			status = "okay";
+		};
+
+		ipu1_memory_region: ipu1-memory@9d000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x9d000000 0x0 0x2000000>;
+			reusable;
+			status = "okay";
+		};
+
+		dsp2_memory_region: dsp2-memory@9f000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x9f000000 0x0 0x800000>;
+			reusable;
+			status = "okay";
+		};
+
+                cmem_block_mem_0: cmem_block_mem@a0000000 {
+                        reg = <0x0 0xa0000000 0x0 0x18000000>;
+                        no-map;
+                        status = "okay";
+                };
+	};
+
+        cmem {
+                compatible = "ti,cmem";
+                #address-cells = <1>;
+                #size-cells = <0>;
+		#pool-size-cells = <2>;
+                status = "okay";
+
+                cmem_block_0: cmem_block@0 {
+                        reg = <0>;
+                        memory-region = <&cmem_block_mem_0>;
+                        cmem-buf-pools = <1 0x0 0x18000000>;
+                };
+
+		cmem_block_1: cmem_block@1 {
+			reg = <1>;
+			memory-region = <&cmem_block_mem_1_ocmc3>;
+		};
+        };
+
+	vdd_adc: gpioregulator-vdd_adc {
+		compatible = "regulator-gpio";
+		regulator-name = "vdd_adc";
+		vin-supply = <&vdd_5v>;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+		gpios = <&gpio3 27 GPIO_ACTIVE_HIGH>;
+		states = <1800000 0
+			3300000 1>;
+	};
+
+	vdd_5v: fixedregulator-vdd_5v {
+		compatible = "regulator-fixed";
+		regulator-name = "vdd_5v";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vtt_fixed: fixedregulator-vtt {
+		/* TPS51200 */
+		compatible = "regulator-fixed";
+		regulator-name = "vtt_fixed";
+		vin-supply = <&vdd_ddr>;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&led_pins_default>;
+
+		led0 {
+			label = "beaglebone:green:usr0";
+			gpios = <&gpio3 17 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "heartbeat";
+			default-state = "off";
+		};
+
+		led1 {
+			label = "beaglebone:green:usr1";
+			gpios = <&gpio5 5 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "mmc0";
+			default-state = "off";
+		};
+
+		led2 {
+			label = "beaglebone:green:usr2";
+			gpios = <&gpio3 15 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "cpu";
+			default-state = "off";
+		};
+
+		led3 {
+			label = "beaglebone:green:usr3";
+			gpios = <&gpio3 14 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "mmc1";
+			default-state = "off";
+		};
+
+		led4 {
+			label = "beaglebone:green:usr4";
+			gpios = <&gpio3 7 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "netdev";
+			default-state = "off";
+		};
+	};
+
+	hdmi_conn: connector@0 {
+		compatible = "hdmi-connector";
+		label = "hdmi";
+		type = "a";
+
+		port {
+			hdmi_connector_in: endpoint {
+				remote-endpoint = <&hdmi_encoder_out>;
+			};
+		};
+	};
+
+	hdmi_enc: encoder@0 {
+		/* "ti,tpd12s016" software compatible with "ti,tpd12s015"
+		 *  no need for individual driver
+		 */
+		compatible = "ti,tpd12s015";
+		gpios = <0>,
+			<0>,
+			<&gpio7 12 GPIO_ACTIVE_HIGH>;
+
+		ports {
+			#address-cells = <0x1>;
+			#size-cells = <0x0>;
+
+			port@0 {
+				reg = <0x0>;
+
+				hdmi_encoder_in: endpoint@0 {
+					remote-endpoint = <&hdmi_out>;
+				};
+			};
+
+			port@1 {
+				reg = <0x1>;
+
+				hdmi_encoder_out: endpoint@0 {
+					remote-endpoint = <&hdmi_connector_in>;
+				};
+			};
+		};
+	};
+
+	emmc_pwrseq: emmc_pwrseq {
+		compatible = "mmc-pwrseq-emmc";
+		reset-gpios = <&gpio5 7 GPIO_ACTIVE_LOW>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&emmc_pwrseq_pins_default>;
+	};
+
+	brcmf_pwrseq: brcmf_pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		pinctrl-names = "default";
+		pinctrl-0 = <&brcmf_pwrseq_pins_default>;
+		reset-gpios = <&gpio3 22 GPIO_ACTIVE_LOW>,	/* BT-REG-ON */
+				<&gpio3 18 GPIO_ACTIVE_LOW>;	/* WL-REG-ON */
+	};
+
+	unused_pins: unused_pins {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&unused_pins_default>;
+	};
+
+	cape_pins: cape_pins {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&cape_pins_default>;
+	};
+
+	extcon_usb1: extcon_usb1 {
+		compatible = "linux,extcon-usb-gpio";
+		ti,enable-id-detection;
+		id-gpio = <&gpio3 13 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&extcon_usb1_pins_default>;
+	};
+};
+
+&ocp {
+	pruss1_shmem: pruss_shmem@4b200000 {
+		status = "okay";
+		compatible = "ti,pruss-shmem";
+		reg = <0x4b200000 0x020000>;
+	};
+
+	pruss2_shmem: pruss_shmem@4b280000 {
+		status = "okay";
+		compatible = "ti,pruss-shmem";
+		reg = <0x4b280000 0x020000>;
+	};
+
+	cape-universal {
+		compatible = "gpio-of-helper";
+		status = "okay";
+		pinctrl-names = "default";
+		pinctrl-0 = <>;
+
+		P8_03 {
+			gpio-name = "P8_03";
+			gpio = <&gpio1 24 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_04 {
+			gpio-name = "P8_04";
+			gpio = <&gpio1 25 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_05 {
+			gpio-name = "P8_05";
+			gpio = <&gpio7 1 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_06 {
+			gpio-name = "P8_06";
+			gpio = <&gpio7 2 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_07 {
+			gpio-name = "P8_07";
+			gpio = <&gpio6 5 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_08 {
+			gpio-name = "P8_08";
+			gpio = <&gpio6 6 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_09 {
+			gpio-name = "P8_09";
+			gpio = <&gpio6 18 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_10 {
+			gpio-name = "P8_10";
+			gpio = <&gpio6 4 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_11 {
+			gpio-name = "P8_11";
+			gpio = <&gpio3 11 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_12 {
+			gpio-name = "P8_12";
+			gpio = <&gpio3 10 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_13 {
+			gpio-name = "P8_13";
+			gpio = <&gpio4 11 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_14 {
+			gpio-name = "P8_14";
+			gpio = <&gpio4 13 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_15 {
+			gpio-name = "P8_15";
+			gpio = <&gpio4 3 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_16 {
+			gpio-name = "P8_16";
+			gpio = <&gpio4 29 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_17 {
+			gpio-name = "P8_17";
+			gpio = <&gpio8 18 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_18 {
+			gpio-name = "P8_18";
+			gpio = <&gpio4 9 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_19 {
+			gpio-name = "P8_19";
+			gpio = <&gpio4 10 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_20 {
+			gpio-name = "P8_20";
+			gpio = <&gpio6 30 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_21 {
+			gpio-name = "P8_21";
+			gpio = <&gpio6 29 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_22 {
+			gpio-name = "P8_22";
+			gpio = <&gpio1 23 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_23 {
+			gpio-name = "P8_23";
+			gpio = <&gpio1 22 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_24 {
+			gpio-name = "P8_24";
+			gpio = <&gpio7 0 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_25 {
+			gpio-name = "P8_25";
+			gpio = <&gpio6 31 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_26 {
+			gpio-name = "P8_26";
+			gpio = <&gpio4 28 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_27 {
+			gpio-name = "P8_27";
+			gpio = <&gpio4 23 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_28 {
+			gpio-name = "P8_28";
+			gpio = <&gpio4 19 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_29 {
+			gpio-name = "P8_29";
+			gpio = <&gpio4 22 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_30 {
+			gpio-name = "P8_30";
+			gpio = <&gpio4 20 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_31 {
+			gpio-name = "P8_31";
+			gpio = <&gpio8 14 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_32 {
+			gpio-name = "P8_32";
+			gpio = <&gpio8 15 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_33 {
+			gpio-name = "P8_33";
+			gpio = <&gpio8 13 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_34 {
+			gpio-name = "P8_34";
+			gpio = <&gpio8 11 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_35 {
+			gpio-name = "P8_35";
+			gpio = <&gpio8 12 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_36 {
+			gpio-name = "P8_36";
+			gpio = <&gpio8 10 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_37 {
+			gpio-name = "P8_37";
+			gpio = <&gpio8 8 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_38 {
+			gpio-name = "P8_38";
+			gpio = <&gpio8 9 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_39 {
+			gpio-name = "P8_39";
+			gpio = <&gpio8 6 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_40 {
+			gpio-name = "P8_40";
+			gpio = <&gpio8 7 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_41 {
+			gpio-name = "P8_41";
+			gpio = <&gpio8 4 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_42 {
+			gpio-name = "P8_42";
+			gpio = <&gpio8 5 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_43 {
+			gpio-name = "P8_43";
+			gpio = <&gpio8 2 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_44 {
+			gpio-name = "P8_44";
+			gpio = <&gpio8 3 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_45 {
+			gpio-name = "P8_45";
+			gpio = <&gpio8 0 0>;
+			input;
+			dir-changeable;
+		};
+
+		P8_46 {
+			gpio-name = "P8_46";
+			gpio = <&gpio8 1 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_11 {
+			gpio-name = "P9_11";
+			gpio = <&gpio8 17 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_12 {
+			gpio-name = "P9_12";
+			gpio = <&gpio5 0 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_13 {
+			gpio-name = "P9_13";
+			gpio = <&gpio6 12 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_14 {
+			gpio-name = "P9_14";
+			gpio = <&gpio4 25 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_15 {
+			gpio-name = "P9_15";
+			gpio = <&gpio3 12 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_16 {
+			gpio-name = "P9_16";
+			gpio = <&gpio4 26 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_17 {
+			gpio-name = "P9_17";
+			gpio = <&gpio7 17 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_18 {
+			gpio-name = "P9_18";
+			gpio = <&gpio7 16 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_19 {
+			gpio-name = "P9_19";
+			gpio = <&gpio7 3 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_20 {
+			gpio-name = "P9_20";
+			gpio = <&gpio7 4 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_21 {
+			gpio-name = "P9_21";
+			gpio = <&gpio3 3 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_22 {
+			gpio-name = "P9_22";
+			gpio = <&gpio6 19 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_23 {
+			gpio-name = "P9_23";
+			gpio = <&gpio7 11 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_24 {
+			gpio-name = "P9_24";
+			gpio = <&gpio6 15 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_25 {
+			gpio-name = "P9_25";
+			gpio = <&gpio6 17 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_26 {
+			gpio-name = "P9_26";
+			gpio = <&gpio6 14 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_27 {
+			gpio-name = "P9_27";
+			gpio = <&gpio6 15 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_28 {
+			gpio-name = "P9_28";
+			gpio = <&gpio4 17 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_29 {
+			gpio-name = "P9_29";
+			gpio = <&gpio5 11 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_30 {
+			gpio-name = "P9_30";
+			gpio = <&gpio5 12 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_31 {
+			gpio-name = "P9_31";
+			gpio = <&gpio5 10 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_41 {
+			gpio-name = "P9_41";
+			gpio = <&gpio6 20 0>;
+			input;
+			dir-changeable;
+		};
+
+		P9_42 {
+			gpio-name = "P9_42";
+			gpio = <&gpio4 18 0>;
+			input;
+			dir-changeable;
+		};
+	};
+};
+
+&dra7_pmx_core {
+    /************************/
+	/* P8 Header */
+	/************************/
+
+	/* P8_01                GND */
+
+	/* P8_02                GND */
+
+
+	/* P8_03 */
+	P8_03_default_pin: pinmux_P8_03_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x379C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_03_gpio_pin: pinmux_P8_03_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x379C, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_03_gpio_pu_pin: pinmux_P8_03_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x379C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_03_gpio_pd_pin: pinmux_P8_03_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x379C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_03_gpio_input_pin: pinmux_P8_03_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x379C, PIN_INPUT | MUX_MODE14) >; };
+
+	/* P8_04 */
+	P8_04_default_pin: pinmux_P8_04_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37A0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_04_gpio_pin: pinmux_P8_04_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37A0, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_04_gpio_pu_pin: pinmux_P8_04_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37A0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_04_gpio_pd_pin: pinmux_P8_04_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37A0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_04_gpio_input_pin: pinmux_P8_04_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37A0, PIN_INPUT | MUX_MODE14) >; };
+
+	/* P8_05 */
+	P8_05_default_pin: pinmux_P8_05_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x378C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_05_gpio_pin: pinmux_P8_05_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x378C, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_05_gpio_pu_pin: pinmux_P8_05_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x378C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_05_gpio_pd_pin: pinmux_P8_05_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x378C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_05_gpio_input_pin: pinmux_P8_05_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x378C, PIN_INPUT | MUX_MODE14) >; };
+
+	/* P8_06 */
+	P8_06_default_pin: pinmux_P8_06_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3790, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_06_gpio_pin: pinmux_P8_06_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3790, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_06_gpio_pu_pin: pinmux_P8_06_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3790, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_06_gpio_pd_pin: pinmux_P8_06_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3790, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_06_gpio_input_pin: pinmux_P8_06_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3790, PIN_INPUT | MUX_MODE14) >; };
+
+	/* P8_07 */
+	P8_07_default_pin: pinmux_P8_07_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36EC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_07_gpio_pin: pinmux_P8_07_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36EC, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_07_gpio_pu_pin: pinmux_P8_07_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36EC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_07_gpio_pd_pin: pinmux_P8_07_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36EC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_07_gpio_input_pin: pinmux_P8_07_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36EC, PIN_INPUT | MUX_MODE14) >; };
+	P8_07_timer_pin: pinmux_P8_07_timer_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36EC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE10) >; };
+
+	/* P8_08 */
+	P8_08_default_pin: pinmux_P8_08_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36F0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_08_gpio_pin: pinmux_P8_08_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36F0, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_08_gpio_pu_pin: pinmux_P8_08_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36F0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_08_gpio_pd_pin: pinmux_P8_08_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36F0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_08_gpio_input_pin: pinmux_P8_08_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36F0, PIN_INPUT | MUX_MODE14) >; };
+	P8_08_timer_pin: pinmux_P8_08_timer_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36F0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE10) >; };
+
+	/* P8_09 */
+	P8_09_default_pin: pinmux_P8_09_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3698, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_09_gpio_pin: pinmux_P8_09_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3698, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_09_gpio_pu_pin: pinmux_P8_09_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3698, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_09_gpio_pd_pin: pinmux_P8_09_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3698, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_09_gpio_input_pin: pinmux_P8_09_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3698, PIN_INPUT | MUX_MODE14) >; };
+	P8_09_timer_pin: pinmux_P8_09_timer_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3698, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE10) >; };
+
+	/* P8_10 */
+	P8_10_default_pin: pinmux_P8_10_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36E8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_10_gpio_pin: pinmux_P8_10_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36E8, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_10_gpio_pu_pin: pinmux_P8_10_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36E8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_10_gpio_pd_pin: pinmux_P8_10_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36E8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_10_gpio_input_pin: pinmux_P8_10_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36E8, PIN_INPUT | MUX_MODE14) >; };
+	P8_10_timer_pin: pinmux_P8_10_timer_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36E8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE10) >; };
+
+	/* P8_11 */
+	P8_11_default_pin: pinmux_P8_11_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3510, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_11_gpio_pin: pinmux_P8_11_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3510, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_11_gpio_pu_pin: pinmux_P8_11_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3510, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_11_gpio_pd_pin: pinmux_P8_11_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3510, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_11_gpio_input_pin: pinmux_P8_11_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3510, PIN_INPUT | MUX_MODE14) >; };
+	P8_11_qep_pin: pinmux_P8_11_qep_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3510, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE10) >; };
+	P8_11_pruout_pin: pinmux_P8_11_pruout_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3510, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; };
+
+	/* P8_12 */
+	P8_12_default_pin: pinmux_P8_12_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x350C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_12_gpio_pin: pinmux_P8_12_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x350C, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_12_gpio_pu_pin: pinmux_P8_12_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x350C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_12_gpio_pd_pin: pinmux_P8_12_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x350C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_12_gpio_input_pin: pinmux_P8_12_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x350C, PIN_INPUT | MUX_MODE14) >; };
+	P8_12_qep_pin: pinmux_P8_12_qep_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x350C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE10) >; };
+	P8_12_pruout_pin: pinmux_P8_12_pruout_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x350C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; };
+
+	/* P8_13 */
+	P8_13_default_pin: pinmux_P8_13_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3590, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_13_gpio_pin: pinmux_P8_13_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3590, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_13_gpio_pu_pin: pinmux_P8_13_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3590, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_13_gpio_pd_pin: pinmux_P8_13_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3590, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_13_gpio_input_pin: pinmux_P8_13_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3590, PIN_INPUT | MUX_MODE14) >; };
+	P8_13_pwm_pin: pinmux_P8_13_pwm_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3590, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE10) >; };
+
+	/* P8_14 */
+	P8_14_default_pin: pinmux_P8_14_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3598, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_14_gpio_pin: pinmux_P8_14_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3598, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_14_gpio_pu_pin: pinmux_P8_14_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3598, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_14_gpio_pd_pin: pinmux_P8_14_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3598, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_14_gpio_input_pin: pinmux_P8_14_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3598, PIN_INPUT | MUX_MODE14) >; };
+	P8_14_pwm_pin: pinmux_P8_14_pwm_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3598, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE10) >; };
+
+	/* P8_15a */
+	P8_15_default_pin: pinmux_P8_15_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3570, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_15_gpio_pin: pinmux_P8_15_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3570, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_15_gpio_pu_pin: pinmux_P8_15_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3570, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_15_gpio_pd_pin: pinmux_P8_15_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3570, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_15_gpio_input_pin: pinmux_P8_15_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3570, PIN_INPUT | MUX_MODE14) >; };
+	P8_15_qep_pin: pinmux_P8_15_qep_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3570, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE15) >; };	/* ?? eqep2_strobe available on P8.18*/
+	P8_15_pru_ecap_pin: pinmux_P8_15_pru_ecap_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3570, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE11) >; };
+
+	/* P8_15b */
+	P8_15_pruin_pin: pinmux_P8_15_pruin_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35B4, PIN_INPUT | MUX_MODE12) >; };
+
+	/* P8_16 */
+	P8_16_default_pin: pinmux_P8_16_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35BC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_16_gpio_pin: pinmux_P8_16_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35BC, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_16_gpio_pu_pin: pinmux_P8_16_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35BC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_16_gpio_pd_pin: pinmux_P8_16_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35BC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_16_gpio_input_pin: pinmux_P8_16_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35BC, PIN_INPUT | MUX_MODE14) >; };
+	P8_16_qep_pin: pinmux_P8_16_qep_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35BC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE15) >; };	/* ?? eqep2_index available on P8.15*/
+	P8_16_pruin_pin: pinmux_P8_16_pruin_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35BC, PIN_INPUT | MUX_MODE12) >; };
+
+	/* P8_17 */
+	P8_17_default_pin: pinmux_P8_17_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3624, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_17_gpio_pin: pinmux_P8_17_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3624, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_17_gpio_pu_pin: pinmux_P8_17_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3624, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_17_gpio_pd_pin: pinmux_P8_17_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3624, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_17_gpio_input_pin: pinmux_P8_17_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3624, PIN_INPUT | MUX_MODE14) >; };
+	P8_17_pwm_pin: pinmux_P8_17_pwm_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3624, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE15) >; };	/* ?? ehrpwm0_synco */
+
+	/* P8_18 */
+	P8_18_default_pin: pinmux_P8_18_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3588, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_18_gpio_pin: pinmux_P8_18_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3588, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_18_gpio_pu_pin: pinmux_P8_18_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3588, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_18_gpio_pd_pin: pinmux_P8_18_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3588, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_18_gpio_input_pin: pinmux_P8_18_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3588, PIN_INPUT | MUX_MODE14) >; };
+
+	/* P8_19 */
+	P8_19_default_pin: pinmux_P8_19_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x358C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_19_gpio_pin: pinmux_P8_19_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x358C, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_19_gpio_pu_pin: pinmux_P8_19_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x358C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_19_gpio_pd_pin: pinmux_P8_19_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x358C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_19_gpio_input_pin: pinmux_P8_19_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x358C, PIN_INPUT | MUX_MODE14) >; };
+	P8_19_pwm_pin: pinmux_P8_19_pwm_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x358C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE10) >; };
+
+	/* P8_20 (ZCZ ball V9) emmc */
+	P8_20_default_pin: pinmux_P8_20_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3780, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_20_gpio_pin: pinmux_P8_20_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3780, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_20_gpio_pu_pin: pinmux_P8_20_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3780, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_20_gpio_pd_pin: pinmux_P8_20_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3780, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_20_gpio_input_pin: pinmux_P8_20_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3780, PIN_INPUT | MUX_MODE14) >; };
+	P8_20_pruout_pin: pinmux_P8_20_pruout_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3780, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; };
+	P8_20_pruin_pin: pinmux_P8_20_pruin_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3780, PIN_INPUT | MUX_MODE12) >; };
+
+	/* P8_21 */
+	P8_21_default_pin: pinmux_P8_21_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x377C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_21_gpio_pin: pinmux_P8_21_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x377C, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_21_gpio_pu_pin: pinmux_P8_21_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x377C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_21_gpio_pd_pin: pinmux_P8_21_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x377C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_21_gpio_input_pin: pinmux_P8_21_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x377C, PIN_INPUT | MUX_MODE14) >; };
+	P8_21_pruout_pin: pinmux_P8_21_pruout_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x377C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; };
+	P8_21_pruin_pin: pinmux_P8_21_pruin_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x377C, PIN_INPUT | MUX_MODE12) >; };
+
+	/* P8_22 */
+	P8_22_default_pin: pinmux_P8_22_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3798, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_22_gpio_pin: pinmux_P8_22_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3798, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_22_gpio_pu_pin: pinmux_P8_22_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3798, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_22_gpio_pd_pin: pinmux_P8_22_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3798, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_22_gpio_input_pin: pinmux_P8_22_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3798, PIN_INPUT | MUX_MODE14) >; };
+
+	/* P8_23 */
+	P8_23_default_pin: pinmux_P8_23_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3794, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_23_gpio_pin: pinmux_P8_23_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3794, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_23_gpio_pu_pin: pinmux_P8_23_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3794, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_23_gpio_pd_pin: pinmux_P8_23_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3794, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_23_gpio_input_pin: pinmux_P8_23_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3794, PIN_INPUT | MUX_MODE14) >; };
+
+	/* P8_24 */
+	P8_24_default_pin: pinmux_P8_24_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3788, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_24_gpio_pin: pinmux_P8_24_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3788, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_24_gpio_pu_pin: pinmux_P8_24_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3788, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_24_gpio_pd_pin: pinmux_P8_24_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3788, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_24_gpio_input_pin: pinmux_P8_24_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3788, PIN_INPUT | MUX_MODE14) >; };
+
+	/* P8_25 */
+	P8_25_default_pin: pinmux_P8_25_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3784, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_25_gpio_pin: pinmux_P8_25_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3784, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_25_gpio_pu_pin: pinmux_P8_25_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3784, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_25_gpio_pd_pin: pinmux_P8_25_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3784, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_25_gpio_input_pin: pinmux_P8_25_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3784, PIN_INPUT | MUX_MODE14) >; };
+
+	/* P8_26 */
+	P8_26_default_pin: pinmux_P8_26_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35B8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_26_gpio_pin: pinmux_P8_26_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35B8, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_26_gpio_pu_pin: pinmux_P8_26_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35B8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_26_gpio_pd_pin: pinmux_P8_26_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35B8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_26_gpio_input_pin: pinmux_P8_26_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35B8, PIN_INPUT | MUX_MODE14) >; };
+
+	/* P8_27a */
+	P8_27_default_pin: pinmux_P8_27_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35D8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_27_gpio_pin: pinmux_P8_27_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35D8, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_27_gpio_pu_pin: pinmux_P8_27_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35D8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_27_gpio_pd_pin: pinmux_P8_27_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35D8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_27_gpio_input_pin: pinmux_P8_27_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35D8, PIN_INPUT | MUX_MODE14) >; };
+
+	/* P8_27b */
+	P8_27_pruout_pin: pinmux_P8_27_pruout_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3628, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; };
+	P8_27_pruin_pin: pinmux_P8_27_pruin_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3628, PIN_INPUT | MUX_MODE12) >; };
+
+	/* P8_28a */
+	P8_28_default_pin: pinmux_P8_28_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35C8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_28_gpio_pin: pinmux_P8_28_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35C8, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_28_gpio_pu_pin: pinmux_P8_28_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35C8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_28_gpio_pd_pin: pinmux_P8_28_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35C8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_28_gpio_input_pin: pinmux_P8_28_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35C8, PIN_INPUT | MUX_MODE14) >; };
+
+	/* P8_28b */
+	P8_28_pruout_pin: pinmux_P8_28_pruout_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x362C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; };
+	P8_28_pruin_pin: pinmux_P8_28_pruin_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x362C, PIN_INPUT | MUX_MODE12) >; };
+
+	/* P8_29a */
+	P8_29_default_pin: pinmux_P8_29_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35D4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_29_gpio_pin: pinmux_P8_29_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35D4, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_29_gpio_pu_pin: pinmux_P8_29_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35D4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_29_gpio_pd_pin: pinmux_P8_29_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35D4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_29_gpio_input_pin: pinmux_P8_29_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35D4, PIN_INPUT | MUX_MODE14) >; };
+
+	/* P8_29b */
+	P8_29_pruout_pin: pinmux_P8_29_pruout_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3630, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; };
+	P8_29_pruin_pin: pinmux_P8_29_pruin_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3630, PIN_INPUT | MUX_MODE12) >; };
+
+	/* P8_30a */
+	P8_30_default_pin: pinmux_P8_30_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35CC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_30_gpio_pin: pinmux_P8_30_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35CC, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_30_gpio_pu_pin: pinmux_P8_30_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35CC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_30_gpio_pd_pin: pinmux_P8_30_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35CC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_30_gpio_input_pin: pinmux_P8_30_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35CC, PIN_INPUT | MUX_MODE14) >; };
+
+	/* P8_30b */
+	P8_30_pruout_pin: pinmux_P8_30_pruout_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3634, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; };
+	P8_30_pruin_pin: pinmux_P8_30_pruin_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3634, PIN_INPUT | MUX_MODE12) >; };
+
+
+	/* P8_31a */
+	P8_31_default_pin: pinmux_P8_31_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3614, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_31_gpio_pin: pinmux_P8_31_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3614, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_31_gpio_pu_pin: pinmux_P8_31_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3614, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_31_gpio_pd_pin: pinmux_P8_31_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3614, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_31_gpio_input_pin: pinmux_P8_31_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3614, PIN_INPUT | MUX_MODE14) >; };
+
+	/* P8_31b */
+	P8_31_qep_pin: pinmux_P8_31_qep_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x373C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE15) >; };	/* ?? eqep1_index */
+	P8_31_uart_pin: pinmux_P8_31_uart_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x373C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };
+
+	/* P8_32a */
+	P8_32_default_pin: pinmux_P8_32_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3618, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_32_gpio_pin: pinmux_P8_32_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3618, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_32_gpio_pu_pin: pinmux_P8_32_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3618, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_32_gpio_pd_pin: pinmux_P8_32_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3618, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_32_gpio_input_pin: pinmux_P8_32_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3618, PIN_INPUT | MUX_MODE14) >; };
+
+	/* P8_32b */
+	P8_32_qep_pin: pinmux_P8_32_qep_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3740, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; };	/* ?? eqep1_strobe available on P9_21*/
+
+	/* P8_33a */
+	P8_33_default_pin: pinmux_P8_33_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3610, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_33_gpio_pin: pinmux_P8_33_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3610, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_33_gpio_pu_pin: pinmux_P8_33_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3610, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_33_gpio_pd_pin: pinmux_P8_33_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3610, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_33_gpio_input_pin: pinmux_P8_33_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3610, PIN_INPUT | MUX_MODE14) >; };
+
+	/* P8_33b */
+	P8_33_qep_pin: pinmux_P8_33_qep_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x34E8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE10) >; };
+
+
+	/* P8_34a */
+	P8_34_default_pin: pinmux_P8_34_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3608, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_34_gpio_pin: pinmux_P8_34_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3608, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_34_gpio_pu_pin: pinmux_P8_34_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3608, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_34_gpio_pd_pin: pinmux_P8_34_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3608, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_34_gpio_input_pin: pinmux_P8_34_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3608, PIN_INPUT | MUX_MODE14) >; };
+
+	/* P8_34b */
+	P8_34_pwm_pin: pinmux_P8_34_pwm_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3564, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE10) >; };	/* ?? ehrpwm1b instead we have ehrpwm1a here */
+
+	/* P8_35a */
+	P8_35_default_pin: pinmux_P8_35_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x360C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_35_gpio_pin: pinmux_P8_35_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x360C, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_35_gpio_pu_pin: pinmux_P8_35_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x360C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_35_gpio_pd_pin: pinmux_P8_35_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x360C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_35_gpio_input_pin: pinmux_P8_35_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x360C, PIN_INPUT | MUX_MODE14) >; };
+
+	/* P8_35b */
+	P8_35_qep_pin: pinmux_P8_35_qep_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x34E4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE10) >; };
+
+	/* P8_36a */
+	P8_36_default_pin: pinmux_P8_36_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3604, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_36_gpio_pin: pinmux_P8_36_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3604, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_36_gpio_pu_pin: pinmux_P8_36_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3604, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_36_gpio_pd_pin: pinmux_P8_36_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3604, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_36_gpio_input_pin: pinmux_P8_36_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3604, PIN_INPUT | MUX_MODE14) >; };
+
+	/* P8_36b */
+	P8_36_pwm_pin: pinmux_P8_36_pwm_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3568, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE10) >; };	/* ?? ehrpwm1a instead we have ehrpwm1b here */
+
+	/* P8_37a */
+	P8_37_default_pin: pinmux_P8_37_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35FC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_37_gpio_pin: pinmux_P8_37_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35FC, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_37_gpio_pu_pin: pinmux_P8_37_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35FC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_37_gpio_pd_pin: pinmux_P8_37_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35FC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_37_gpio_input_pin: pinmux_P8_37_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35FC, PIN_INPUT | MUX_MODE14) >; };
+	P8_37_pwm_pin: pinmux_P8_37_pwm_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35FC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE15) >; };	/* ?? ehrpwm1_tripzone_input */
+
+	/* P8_37b */
+	P8_37_uart_pin: pinmux_P8_37_uart_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3738, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* lcd_data8.uart5_txd */
+
+	/* P8_38a */
+	P8_38_default_pin: pinmux_P8_38_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3600, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_38_gpio_pin: pinmux_P8_38_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3600, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_38_gpio_pu_pin: pinmux_P8_38_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3600, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_38_gpio_pd_pin: pinmux_P8_38_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3600, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_38_gpio_input_pin: pinmux_P8_38_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3600, PIN_INPUT | MUX_MODE14) >; };
+	P8_38_pwm_pin: pinmux_P8_38_pwm_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3600, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE15) >; };	/* ?? ehrpwm0_synco */
+
+	/* P8_38b */
+	P8_38_uart_pin: pinmux_P8_38_uart_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3734, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };	/* lcd_data9.uart5_rxd */
+
+	/* P8_39 */
+	P8_39_default_pin: pinmux_P8_39_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35F4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_39_gpio_pin: pinmux_P8_39_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35F4, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_39_gpio_pu_pin: pinmux_P8_39_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35F4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_39_gpio_pd_pin: pinmux_P8_39_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35F4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_39_gpio_input_pin: pinmux_P8_39_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35F4, PIN_INPUT | MUX_MODE14) >; };
+	P8_39_qep_pin: pinmux_P8_39_qep_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35F4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE15) >; };	/* ?? eqep2_index */
+	P8_39_pruout_pin: pinmux_P8_39_pruout_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35F4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; };
+	P8_39_pruin_pin: pinmux_P8_39_pruin_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35F4, PIN_INPUT | MUX_MODE12) >; };
+
+	/* P8_40 */
+	P8_40_default_pin: pinmux_P8_40_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35F8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_40_gpio_pin: pinmux_P8_40_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35F8, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_40_gpio_pu_pin: pinmux_P8_40_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35F8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_40_gpio_pd_pin: pinmux_P8_40_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35F8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_40_gpio_input_pin: pinmux_P8_40_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35F8, PIN_INPUT | MUX_MODE14) >; };
+	P8_40_qep_pin: pinmux_P8_40_qep_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35F8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE15) >; };	/* ?? eqep2_strobe */
+	P8_40_pruout_pin: pinmux_P8_40_pruout_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35F8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; };
+	P8_40_pruin_pin: pinmux_P8_40_pruin_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35F8, PIN_INPUT | MUX_MODE12) >; };
+
+	/* P8_41 */
+	P8_41_default_pin: pinmux_P8_41_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35EC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_41_gpio_pin: pinmux_P8_41_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35EC, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_41_gpio_pu_pin: pinmux_P8_41_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35EC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_41_gpio_pd_pin: pinmux_P8_41_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35EC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_41_gpio_input_pin: pinmux_P8_41_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35EC, PIN_INPUT | MUX_MODE14) >; };
+	P8_41_qep_pin: pinmux_P8_41_qep_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35EC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE15) >; };	/* ?? eqep2a_in */
+	P8_41_pruout_pin: pinmux_P8_41_pruout_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35EC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; };
+	P8_41_pruin_pin: pinmux_P8_41_pruin_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35EC, PIN_INPUT | MUX_MODE12) >; };
+
+	/* P8_42 */
+	P8_42_default_pin: pinmux_P8_42_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35F0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_42_gpio_pin: pinmux_P8_42_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35F0, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_42_gpio_pu_pin: pinmux_P8_42_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35F0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_42_gpio_pd_pin: pinmux_P8_42_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35F0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_42_gpio_input_pin: pinmux_P8_42_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35F0, PIN_INPUT | MUX_MODE14) >; };
+	P8_42_qep_pin: pinmux_P8_42_qep_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35F0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE15) >; };	/* ?? eqep2b_in */
+	P8_42_pruout_pin: pinmux_P8_42_pruout_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35F0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; };
+	P8_42_pruin_pin: pinmux_P8_42_pruin_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35F0, PIN_INPUT | MUX_MODE12) >; };
+
+	/* P8_43 */
+	P8_43_default_pin: pinmux_P8_43_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35E4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_43_gpio_pin: pinmux_P8_43_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35E4, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_43_gpio_pu_pin: pinmux_P8_43_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35E4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_43_gpio_pd_pin: pinmux_P8_43_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35E4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_43_gpio_input_pin: pinmux_P8_43_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35E4, PIN_INPUT | MUX_MODE14) >; };
+	P8_43_pwm_pin: pinmux_P8_43_pwm_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35E4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE15) >; };	/* ?? ehrpwm2_tripzone_input */
+	P8_43_pruout_pin: pinmux_P8_43_pruout_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35E4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; };
+	P8_43_pruin_pin: pinmux_P8_43_pruin_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35E4, PIN_INPUT | MUX_MODE12) >; };
+
+	/* P8_44 */
+	P8_44_default_pin: pinmux_P8_44_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35E8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_44_gpio_pin: pinmux_P8_44_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35E8, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_44_gpio_pu_pin: pinmux_P8_44_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35E8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_44_gpio_pd_pin: pinmux_P8_44_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35E8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_44_gpio_input_pin: pinmux_P8_44_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35E8, PIN_INPUT | MUX_MODE14) >; };
+	P8_44_pwm_pin: pinmux_P8_44_pwm_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35E8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE15) >; };	/* ?? ehrpwm0_synco */
+	P8_44_pruout_pin: pinmux_P8_44_pruout_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35E8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; };
+	P8_44_pruin_pin: pinmux_P8_44_pruin_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35E8, PIN_INPUT | MUX_MODE12) >; };
+
+	/* P8_45a */
+	P8_45_default_pin: pinmux_P8_45_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35DC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_45_gpio_pin: pinmux_P8_45_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35DC, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_45_gpio_pu_pin: pinmux_P8_45_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35DC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_45_gpio_pd_pin: pinmux_P8_45_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35DC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_45_gpio_input_pin: pinmux_P8_45_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35DC, PIN_INPUT | MUX_MODE14) >; };
+	P8_45_pwm_pin: pinmux_P8_45_pwm_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35DC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE15) >; };	/* ?? ehrpwm2a */
+
+	/* P8_45b */
+	P8_45_pruout_pin: pinmux_P8_45_pruout_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x361C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; };
+	P8_45_pruin_pin: pinmux_P8_45_pruin_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x361C, PIN_INPUT | MUX_MODE12) >; };
+
+	/* P8_46a */
+	P8_46_default_pin: pinmux_P8_46_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35E0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_46_gpio_pin: pinmux_P8_46_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35E0, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P8_46_gpio_pu_pin: pinmux_P8_46_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35E0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P8_46_gpio_pd_pin: pinmux_P8_46_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35E0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P8_46_gpio_input_pin: pinmux_P8_46_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35E0, PIN_INPUT | MUX_MODE14) >; };
+	P8_46_pwm_pin: pinmux_P8_46_pwm_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35E0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; };	/* ?? ehrpwm2b */
+
+	/* P8_46b */
+	P8_46_pruout_pin: pinmux_P8_46_pruout_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3638, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; };
+	P8_46_pruin_pin: pinmux_P8_46_pruin_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3638, PIN_INPUT | MUX_MODE12) >; };
+
+	/************************/
+	/* P9 Header */
+	/************************/
+
+	/* P9_01                GND */
+
+	/* P9_02                GND */
+
+	/* P9_03                3V3 */
+
+	/* P9_04                3V3 */
+
+	/* P9_05                VDD_5V */
+
+	/* P9_06                VDD_5V */
+
+	/* P9_07                SYS_5V */
+
+	/* P9_08                SYS_5V */
+
+	/* P9_09                PWR_BUT */
+
+	/* P9_10                RSTn */
+
+	/* P9_11a */
+	P9_11_uart_pin: pinmux_P9_11_uart_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x372C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };
+
+	/* P9_11b */
+	P9_11_default_pin: pinmux_P9_11_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3620, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_11_gpio_pin: pinmux_P9_11_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3620, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P9_11_gpio_pu_pin: pinmux_P9_11_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3620, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_11_gpio_pd_pin: pinmux_P9_11_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3620, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_11_gpio_input_pin: pinmux_P9_11_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3620, PIN_INPUT | MUX_MODE14) >; };
+
+	/* P9_12 */
+	P9_12_default_pin: pinmux_P9_12_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36AC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_12_gpio_pin: pinmux_P9_12_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36AC, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P9_12_gpio_pu_pin: pinmux_P9_12_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36AC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_12_gpio_pd_pin: pinmux_P9_12_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36AC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_12_gpio_input_pin: pinmux_P9_12_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36AC, PIN_INPUT | MUX_MODE14) >; };
+
+	/* P9_13a */
+	P9_13_uart_pin: pinmux_P9_13_uart_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3730, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };
+
+	/* P9_13b */
+	P9_13_default_pin: pinmux_P9_13_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3680, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_13_gpio_pin: pinmux_P9_13_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3680, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P9_13_gpio_pu_pin: pinmux_P9_13_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3680, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_13_gpio_pd_pin: pinmux_P9_13_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3680, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_13_gpio_input_pin: pinmux_P9_13_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3680, PIN_INPUT | MUX_MODE14) >; };
+
+
+	/* P9_14 */
+	P9_14_default_pin: pinmux_P9_14_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35AC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_14_gpio_pin: pinmux_P9_14_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35AC, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P9_14_gpio_pu_pin: pinmux_P9_14_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35AC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_14_gpio_pd_pin: pinmux_P9_14_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35AC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_14_gpio_input_pin: pinmux_P9_14_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35AC, PIN_INPUT | MUX_MODE14) >; };
+	P9_14_pwm_pin: pinmux_P9_14_pwm_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35AC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE10) >; };
+
+	/* P9_15 */
+	P9_15_default_pin: pinmux_P9_15_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3514, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_15_gpio_pin: pinmux_P9_15_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3514, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P9_15_gpio_pu_pin: pinmux_P9_15_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3514, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_15_gpio_pd_pin: pinmux_P9_15_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3514, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_15_gpio_input_pin: pinmux_P9_15_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3514, PIN_INPUT | MUX_MODE14) >; };
+	P9_15_pwm_pin: pinmux_P9_15_pwm_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3514, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE15) >; };	/* ?? ehrpwm1_tripzone_input */
+
+	/* P9_16 */
+	P9_16_default_pin: pinmux_P9_16_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35B0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE10) >; };
+	P9_16_gpio_pin: pinmux_P9_16_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35B0, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P9_16_gpio_pu_pin: pinmux_P9_16_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35B0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_16_gpio_pd_pin: pinmux_P9_16_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35B0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_16_gpio_input_pin: pinmux_P9_16_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35B0, PIN_INPUT | MUX_MODE14) >; };
+	P9_16_pwm_pin: pinmux_P9_16_pwm_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35B0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE10) >; };
+
+	/* P9_17a */
+	P9_17_default_pin: pinmux_P9_17_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37CC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_17_gpio_pin: pinmux_P9_17_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37CC, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P9_17_gpio_pu_pin: pinmux_P9_17_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37CC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_17_gpio_pd_pin: pinmux_P9_17_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37CC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_17_gpio_input_pin: pinmux_P9_17_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37CC, PIN_INPUT | MUX_MODE14) >; };
+	P9_17_spi_cs_pin: pinmux_P9_17_spi_cs_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37CC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };
+
+	/* P9_17b */
+	P9_17_i2c_pin: pinmux_P9_17_i2c_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36B8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE10) >; };
+	P9_17_pwm_pin: pinmux_P9_17_pwm_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36B8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE15) >; };	/* ?? ehrpwm0_synci */
+	P9_17_pru_uart_pin: pinmux_P9_17_pru_uart_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36B8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };
+
+	/* P9_18 */
+	P9_18_default_pin: pinmux_P9_18_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37C8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_18_gpio_pin: pinmux_P9_18_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37C8, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P9_18_gpio_pu_pin: pinmux_P9_18_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37C8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_18_gpio_pd_pin: pinmux_P9_18_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37C8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_18_gpio_input_pin: pinmux_P9_18_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37C8, PIN_INPUT | MUX_MODE14) >; };
+	P9_18_spi_pin: pinmux_P9_18_spi_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37C8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };
+
+	P9_18_i2c_pin: pinmux_P9_18_i2c_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36B4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE10) >; };
+	P9_18_pwm_pin: pinmux_P9_18_pwm_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36B4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE15) >; };	/* ?? ehrpwm0_tripzone_input */
+	P9_18_pru_uart_pin: pinmux_P9_18_pru_uart_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36B4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };
+
+	/* P9_19a */
+	P9_19_default_pin: pinmux_P9_19_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3440, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };
+	P9_19_gpio_pin: pinmux_P9_19_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3440, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P9_19_gpio_pu_pin: pinmux_P9_19_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3440, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_19_gpio_pd_pin: pinmux_P9_19_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3440, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_19_gpio_input_pin: pinmux_P9_19_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3440, PIN_INPUT | MUX_MODE14) >; };
+	P9_19_timer_pin: pinmux_P9_19_timer_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3440, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE15) >; };	/* ?? timer5 */
+	P9_19_can_pin: pinmux_P9_19_can_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3440, PIN_INPUT_PULLUP | MUX_MODE15) >; };		/* ?? dcan0_rx */
+	P9_19_i2c_pin: pinmux_P9_19_i2c_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3440, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };
+
+	/* P9_19b */
+	P9_19_spi_cs_pin: pinmux_P9_19_spi_cs_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x357C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE15) >; };	/* ?? spi1_cs1 */
+	P9_19_pru_uart_pin: pinmux_P9_19_pru_uart_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x357C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE8) >; };
+
+	/* P9_20 */
+	P9_20_default_pin: pinmux_P9_20_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3444, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };
+	P9_20_gpio_pin: pinmux_P9_20_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3444, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P9_20_gpio_pu_pin: pinmux_P9_20_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3444, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_20_gpio_pd_pin: pinmux_P9_20_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3444, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_20_gpio_input_pin: pinmux_P9_20_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3444, PIN_INPUT | MUX_MODE14) >; };
+	P9_20_timer_pin: pinmux_P9_20_timer_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3444, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE15) >; };	/* ?? timer6 */
+	P9_20_can_pin: pinmux_P9_20_can_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3444, PIN_OUTPUT_PULLUP | MUX_MODE15) >; };		/* ?? dcan0_tx */
+	P9_20_i2c_pin: pinmux_P9_20_i2c_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3444, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; };
+
+	/* P9_20 */
+	P9_20_spi_cs_pin: pinmux_P9_20_spi_cs_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3578, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE15) >; };	/* ?? spi1_cs0 */
+	P9_20_pru_uart_pin: pinmux_P9_20_pru_uart_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3578, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE8) >; };
+
+	/* P9_21a */
+	P9_21_default_pin: pinmux_P9_21_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x34F0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_21_gpio_pin: pinmux_P9_21_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x34F0, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P9_21_gpio_pu_pin: pinmux_P9_21_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x34F0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_21_gpio_pd_pin: pinmux_P9_21_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x34F0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_21_gpio_input_pin: pinmux_P9_21_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x34F0, PIN_INPUT | MUX_MODE14) >; };
+	P9_21_pru_uart_pin: pinmux_P9_21_pru_uart_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x34F0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; };
+
+	/* P9_21b */
+	P9_21_spi_pin: pinmux_P9_21_spi_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37C4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };
+	P9_21_uart_pin: pinmux_P9_21_uart_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37C4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };
+	P9_21_i2c_pin: pinmux_P9_21_i2c_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37C4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE15) >; };	/* ?? i2c2_scl */
+	P9_21_pwm_pin: pinmux_P9_21_pwm_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37C4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE15) >; };	/* ?? ehrpwm0b */
+
+	/* P9_22a */
+	P9_22_default_pin: pinmux_P9_22_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x369C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_22_gpio_pin: pinmux_P9_22_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x369C, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P9_22_gpio_pu_pin: pinmux_P9_22_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x369C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_22_gpio_pd_pin: pinmux_P9_22_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x369C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_22_gpio_input_pin: pinmux_P9_22_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x369C, PIN_INPUT | MUX_MODE14) >; };
+
+	/* P9_22b */
+	P9_22_spi_sclk_pin: pinmux_P9_22_spi_sclk_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37C0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; };
+	P9_22_uart_pin: pinmux_P9_22_uart_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37C0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; };
+	P9_22_i2c_pin: pinmux_P9_22_i2c_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37C0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE15) >; };	/* ??i2c2_sda */
+	P9_22_pwm_pin: pinmux_P9_22_pwm_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37C0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE15) >; };	/* ?? ehrpwm0a */
+	P9_22_pru_uart_pin: pinmux_P9_22_pru_uart_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37C0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE15) >; };	/* ?? pr1_uart0_cts_n */
+
+	/* P9_23 */
+	P9_23_default_pin: pinmux_P9_23_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37B4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_23_gpio_pin: pinmux_P9_23_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37B4, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P9_23_gpio_pu_pin: pinmux_P9_23_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37B4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_23_gpio_pd_pin: pinmux_P9_23_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37B4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_23_gpio_input_pin: pinmux_P9_23_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37B4, PIN_INPUT | MUX_MODE14) >; };
+	P9_23_pwm_pin: pinmux_P9_23_pwm_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x37B4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE15) >; };	/* ?? ehrpwm0_synco */
+
+		/* P9_24 */
+	P9_24_default_pin: pinmux_P9_24_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x368C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_24_gpio_pin: pinmux_P9_24_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x368C, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P9_24_gpio_pu_pin: pinmux_P9_24_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x368C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_24_gpio_pd_pin: pinmux_P9_24_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x368C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_24_gpio_input_pin: pinmux_P9_24_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x368C, PIN_INPUT | MUX_MODE14) >; };
+	P9_24_uart_pin: pinmux_P9_24_uart_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x368C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };
+	P9_24_can_pin: pinmux_P9_24_can_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x368C, PIN_INPUT_PULLUP | MUX_MODE2) >; };
+	P9_24_i2c_pin: pinmux_P9_24_i2c_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x368C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE9) >; };
+	P9_24_pru_uart_pin: pinmux_P9_24_pru_uart_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x368C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE15) >; };	/* ?? pr1_uart0_txd */
+	P9_24_pruin_pin: pinmux_P9_24_pruin_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x368C, PIN_INPUT | MUX_MODE15) >; };			/* ?? pru0_in16 */
+
+
+	/* P9_25 */
+	P9_25_default_pin: pinmux_P9_25_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3694, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_25_gpio_pin: pinmux_P9_25_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3694, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P9_25_gpio_pu_pin: pinmux_P9_25_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3694, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_25_gpio_pd_pin: pinmux_P9_25_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3694, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_25_gpio_input_pin: pinmux_P9_25_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3694, PIN_INPUT | MUX_MODE14) >; };
+	P9_25_qep_pin: pinmux_P9_25_qep_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3694, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE15) >; };	/* ?? eqep0_strobe */
+	P9_25_pruout_pin: pinmux_P9_25_pruout_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3694, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; };
+	P9_25_pruin_pin: pinmux_P9_25_pruin_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3694, PIN_INPUT | MUX_MODE12) >; };
+
+		/* P9_26a */
+	P9_26_default_pin: pinmux_P9_26_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3688, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_26_gpio_pin: pinmux_P9_26_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3688, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P9_26_gpio_pu_pin: pinmux_P9_26_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3688, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_26_gpio_pd_pin: pinmux_P9_26_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3688, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_26_gpio_input_pin: pinmux_P9_26_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3688, PIN_INPUT | MUX_MODE14) >; };
+	P9_26_uart_pin: pinmux_P9_26_uart_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3688, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };
+	P9_26_can_pin: pinmux_P9_26_can_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3688, PIN_OUTPUT_PULLUP | MUX_MODE2) >; };
+	P9_26_i2c_pin: pinmux_P9_26_i2c_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3688, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE9) >; };
+	P9_26_pru_uart_pin: pinmux_P9_26_pru_uart_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3688, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE15) >; };	/* ?? pr1_uart0_rxd */
+
+	/* P9_26b */
+	P9_26_pruout_pin: pinmux_P9_26_pruout_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3544, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; };
+	P9_26_pruin_pin: pinmux_P9_26_pruin_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3544, PIN_INPUT | MUX_MODE12) >; };
+
+	/* P9_27a */
+	P9_27_default_pin: pinmux_P9_27_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35A0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_27_gpio_pin: pinmux_P9_27_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35A0, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P9_27_gpio_pu_pin: pinmux_P9_27_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35A0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_27_gpio_pd_pin: pinmux_P9_27_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35A0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_27_gpio_input_pin: pinmux_P9_27_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35A0, PIN_INPUT | MUX_MODE14) >; };
+	P9_27_qep_pin: pinmux_P9_27_qep_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35A0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE10) >; };
+	P9_27_pruout_pin: pinmux_P9_27_pruout_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35A0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; };
+	P9_27_pruin_pin: pinmux_P9_27_pruin_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35A0, PIN_INPUT | MUX_MODE12) >; };
+	P9_27_pruout_pin: pinmux_P9_27_pruout_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35A0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; };
+	P9_27_pruin_pin: pinmux_P9_27_pruin_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x35A0, PIN_INPUT | MUX_MODE12) >; };
+
+	/* P9_27b */
+
+
+	/* P9_28 */
+	P9_28_default_pin: pinmux_P9_28_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36E0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_28_gpio_pin: pinmux_P9_28_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36E0, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P9_28_gpio_pu_pin: pinmux_P9_28_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36E0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_28_gpio_pd_pin: pinmux_P9_28_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36E0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_28_gpio_input_pin: pinmux_P9_28_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36E0, PIN_INPUT | MUX_MODE14) >; };
+	P9_28_pwm_pin: pinmux_P9_28_pwm_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36E0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE15) >; };	/* ?? ehrpwm0_synci */
+	P9_28_spi_cs_pin: pinmux_P9_28_spi_cs_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36E0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };
+	P9_28_pwm2_pin: pinmux_P9_28_pwm2_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36E0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE15) >; };	/* ?? ecap2_in_pwm2_out */
+	P9_28_pruout_pin: pinmux_P9_28_pruout_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36E0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; };
+	P9_28_pruin_pin: pinmux_P9_28_pruin_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36E0, PIN_INPUT | MUX_MODE12) >; };
+
+	/* P9_29a */
+	P9_29_default_pin: pinmux_P9_29_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36D8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_29_gpio_pin: pinmux_P9_29_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36D8, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P9_29_gpio_pu_pin: pinmux_P9_29_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36D8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_29_gpio_pd_pin: pinmux_P9_29_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36D8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_29_gpio_input_pin: pinmux_P9_29_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36D8, PIN_INPUT | MUX_MODE14) >; };
+	P9_29_pwm_pin: pinmux_P9_29_pwm_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36D8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE15) >; };	/* ?? ehrpwm0b */
+	P9_29_spi_pin: pinmux_P9_29_spi_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36D8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };
+	P9_29_pruout_pin: pinmux_P9_29_pruout_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36D8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; };
+	P9_29_pruin_pin: pinmux_P9_29_pruin_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36D8, PIN_INPUT | MUX_MODE12) >; };
+
+	/* P9_29b */
+
+
+	/* P9_30 */
+	P9_30_default_pin: pinmux_P9_30_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36DC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_30_gpio_pin: pinmux_P9_30_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36DC, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P9_30_gpio_pu_pin: pinmux_P9_30_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36DC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_30_gpio_pd_pin: pinmux_P9_30_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36DC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_30_gpio_input_pin: pinmux_P9_30_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36DC, PIN_INPUT | MUX_MODE14) >; };
+	P9_30_pwm_pin: pinmux_P9_30_pwm_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36DC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE15) >; };	/* ?? ehrpwm0_tripzone_input */
+	P9_30_spi_pin: pinmux_P9_30_spi_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36DC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };
+	P9_30_pruout_pin: pinmux_P9_30_pruout_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36DC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; };
+	P9_30_pruin_pin: pinmux_P9_30_pruin_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36DC, PIN_INPUT | MUX_MODE12) >; };
+
+	/* P9_31b */
+	P9_31_default_pin: pinmux_P9_31_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36D4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_31_gpio_pin: pinmux_P9_31_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36D4, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P9_31_gpio_pu_pin: pinmux_P9_31_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36D4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_31_gpio_pd_pin: pinmux_P9_31_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36D4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_31_gpio_input_pin: pinmux_P9_31_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36D4, PIN_INPUT | MUX_MODE14) >; };
+	P9_31_pwm_pin: pinmux_P9_31_pwm_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36D4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; };	/* ?? ehrpwm0a */
+	P9_31_spi_sclk_pin: pinmux_P9_31_spi_sclk_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36D4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };
+	P9_31_pruout_pin: pinmux_P9_31_pruout_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36D4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; };
+	P9_31_pruin_pin: pinmux_P9_31_pruin_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36D4, PIN_INPUT | MUX_MODE12) >; };
+
+	/* P9_31b */
+
+
+	/* P9_32                VADC */
+
+	/* P9_33 	  			AIN4*/
+
+	/* P9_34                AGND */
+
+	/* P9_35 				AIN6 */
+
+	/* P9_36 				AIN5 */
+
+	/* P9_37  				AIN2 */
+
+	/* P9_38  				AIN3*/
+
+	/* P9_39  				AIN0*/
+
+	/* P9_40   				AIN1*/
+
+	/* P9_41a */
+	P9_41_default_pin: pinmux_P9_41_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36A0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_41_gpio_pin: pinmux_P9_41_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36A0, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P9_41_gpio_pu_pin: pinmux_P9_41_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36A0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_41_gpio_pd_pin: pinmux_P9_41_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36A0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_41_gpio_input_pin: pinmux_P9_41_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36A0, PIN_INPUT | MUX_MODE14) >; };
+	P9_41_timer_pin: pinmux_P9_41_timer_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36A0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; };	/* ?? timer7 */
+
+	/* P9_41b */
+	P9_41_pruout_pin: pinmux_P9_41_pruout_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3580, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; };
+	P9_41_pruin_pin: pinmux_P9_41_pruin_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x3580, PIN_INPUT | MUX_MODE12) >; };
+
+	/* P9_42a */
+	P9_42_default_pin: pinmux_P9_42_default_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36E4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_42_gpio_pin: pinmux_P9_42_gpio_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36E4, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; };
+	P9_42_gpio_pu_pin: pinmux_P9_42_gpio_pu_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36E4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; };
+	P9_42_gpio_pd_pin: pinmux_P9_42_gpio_pd_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36E4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; };
+	P9_42_gpio_input_pin: pinmux_P9_42_gpio_input_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36E4, PIN_INPUT | MUX_MODE14) >; };
+	P9_42_pwm_pin: pinmux_P9_42_pwm_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36E4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE15) >; };	/* ?? ecap0_in_pwm0_out */
+	P9_42_uart_pin: pinmux_P9_42_uart_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36E4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE15) >; };	/* ?? uart3_txd */
+	P9_42_spi_cs_pin: pinmux_P9_42_spi_cs_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36E4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; };
+	P9_42_pru_ecap_pin: pinmux_P9_42_pru_ecap_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36E4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE15) >; };	/* ?? pr1_ecap0_ecap_capin_apwm_o */
+	P9_42_spi_sclk_pin: pinmux_P9_42_spi_sclk_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x36E4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE15) >; };	/* ?? spi1_sclk */
+
+	/* P9_42b */
+	P9_42_pruout_pin: pinmux_P9_42_pruout_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x359C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; };
+	P9_42_pruin_pin: pinmux_P9_42_pruin_pin { pinctrl-single,pins = <
+		DRA7XX_CORE_IOPAD(0x359C, PIN_INPUT | MUX_MODE12) >; };
+
+	/* P9_43                GND */
+
+	/* P9_44                GND */
+
+	/* P9_45                GND */
+
+	/* P9_46                GND */
+
+	extcon_usb1_pins_default: extcon_usb1_pins_default {
+		pinctrl-single,pins = <
+			DRA7XX_CORE_IOPAD(0x3518, PIN_INPUT | MUX_MODE14) /* AG2: vin1a_d9.gpio3_13  - USR0 */
+		>;
+	};
+
+	led_pins_default: led_pins_default {
+		pinctrl-single,pins = <
+			DRA7XX_CORE_IOPAD(0x3528, PIN_OUTPUT | MUX_MODE14) /* AF6: vin1a_d13.gpio3_17  - USR0 */
+			DRA7XX_CORE_IOPAD(0x36c0, PIN_OUTPUT | MUX_MODE14) /* J11: mcasp1_axr3.gpio5_5 - USR1 */
+			DRA7XX_CORE_IOPAD(0x3520, PIN_OUTPUT | MUX_MODE14) /* AG5: vin1a_d12.gpio3_15  - USR2 */
+			DRA7XX_CORE_IOPAD(0x351c, PIN_OUTPUT | MUX_MODE14) /* AG3: vin1a_d10.gpio3_14  - USR3 */
+			DRA7XX_CORE_IOPAD(0x3500, PIN_OUTPUT | MUX_MODE14) /* AH6: vin1a_d3.gpio3_7    - USR4 */
+		>;
+	};
+
+	microsd_extra_pins_default: microsd_extra_pins_default {
+		pinctrl-single,pins = <
+			DRA7XX_CORE_IOPAD(0x376c, PIN_INPUT_PULLUP | MUX_MODE14) /* W7: mmc1_sdcd.gpio6_27 - MMC1_SDCD */
+		>;
+	};
+
+	ethphy_extra_pins_default: ethphy_extra_pins_default {
+		pinctrl-single,pins = <
+			DRA7XX_CORE_IOPAD(0x34c4, PIN_OUTPUT_PULLUP | MUX_MODE14) /* N1: gpmc_advn_ale.gpio2_23 - RGMII_RST */
+			DRA7XX_CORE_IOPAD(0x364c, PIN_INPUT | MUX_MODE14) /* Y1: uart3_txd.gpio5_19 - MII0_INT */
+		>;
+	};
+
+	emmc_pwrseq_pins_default: emmc_pwrseq_pins_default {
+		pinctrl-single,pins = <
+			DRA7XX_CORE_IOPAD(0x36c8, PIN_OUTPUT_PULLUP | MUX_MODE14) /* F13: mcasp1_axr5.gpio5_7 - eMMC_RSTn */
+		>;
+	};
+
+	brcmf_pwrseq_pins_default: brcmf_pwrseq_pins_default {
+		pinctrl-single,pins = <
+			DRA7XX_CORE_IOPAD(0x352c, PIN_OUTPUT_PULLUP | MUX_MODE14) /* AF3: vin1a_d14.gpio3_18 - WL_REG_ON */
+			DRA7XX_CORE_IOPAD(0x353c, PIN_OUTPUT_PULLUP | MUX_MODE14) /* AE5: vin1a_d18.gpio3_22 - BT_REG_ON */
+		>;
+	};
+
+	wifibt_extra_pins_default: wifibt_extra_pins_default {
+		pinctrl-single,pins = <
+			DRA7XX_CORE_IOPAD(0x3540, PIN_INPUT | MUX_MODE14) /* AE1: vin1a_d19.gpio3_23 - WL_HOST_WAKE */
+			DRA7XX_CORE_IOPAD(0x3450, PIN_INPUT | MUX_MODE8) /* P6: vin1a_d20.uart6_rxd - UART6_RXD */
+			DRA7XX_CORE_IOPAD(0x3454, PIN_INPUT | MUX_MODE8) /* R9: vin1a_d21.uart6_txd - UART6_TXD */
+			DRA7XX_CORE_IOPAD(0x3458, PIN_INPUT | MUX_MODE8) /* R5: vin1a_d22.uart6_ctsn - UART6_CTSN */
+			DRA7XX_CORE_IOPAD(0x345c, PIN_INPUT | MUX_MODE8) /* P5: vin1a_d23.uart6_rtsn - UART6_RTSN */
+			DRA7XX_CORE_IOPAD(0x3534, PIN_INPUT_PULLDOWN | MUX_MODE14) /* AF1: vin1a_d16.gpio3_20 - BT_HOST_WAKE */
+			DRA7XX_CORE_IOPAD(0x3538, PIN_OUTPUT_PULLDOWN | MUX_MODE14) /* AE3: vin1a_d6.gpio3_21 - BT_WAKE */
+		>;
+	};
+
+	adc_pins_default: adc_pins_default {
+		pinctrl-single,pins = <
+			DRA7XX_CORE_IOPAD(0x3550, PIN_OUTPUT | MUX_MODE14) /* AD3: vin1a_d23.gpio3_27 - VDD_ADC_SEL */
+			DRA7XX_CORE_IOPAD(0x34DC, PIN_INPUT_PULLUP | MUX_MODE14) /* AG8: vin1a_clk0.gpio2_30 - INT_ADC */
+		>;
+	};
+
+	pmic_pins_default: pmic_pins_default {
+		pinctrl-single,pins = <
+			DRA7XX_CORE_IOPAD(0x3690, PIN_INPUT_PULLUP | MUX_MODE14) /* F21: gpio6_16.gpio6_16 - PMIC_INT */
+		>;
+	};
+
+	hdmi_pins_default: hdmi_pins_default {
+		pinctrl-single,pins = <
+			DRA7XX_CORE_IOPAD(0x3808, PIN_INPUT | MUX_MODE1) /* C25: i2c2_sda.hdmi1_ddc_scl - HDMI_DDC_SCL */
+			DRA7XX_CORE_IOPAD(0x380C, PIN_INPUT | MUX_MODE1) /* F17: i2c2_scl.hdmi1_ddc_sda - HDMI_DDC_SDA */
+			DRA7XX_CORE_IOPAD(0x37BC, PIN_INPUT | MUX_MODE6) /* B20: spi1_cs3.hdmi1_cec - HDMI_DDC_CEC */
+#if 0
+			DRA7XX_CORE_IOPAD(0x37B8, PIN_INPUT | MUX_MODE6) /* B21: spi1_cs2.hdmi1_hpd - HDMI_DDC_HPD */
+#else
+			DRA7XX_CORE_IOPAD(0x37B8, PIN_INPUT | MUX_MODE14) /* B21: spi1_cs2.gpio7_12 - HDMI_DDC_HPD */
+#endif
+		>;
+	};
+
+	unused_pins_default: unused_pins_default {
+		pinctrl-single,pins = <
+			DRA7XX_CORE_IOPAD(0x3400, MUX_MODE15) /* M6: sysboot0 */
+			DRA7XX_CORE_IOPAD(0x3404, MUX_MODE15) /* M2: sysboot1 */
+			DRA7XX_CORE_IOPAD(0x3408, MUX_MODE15) /* L5: sysboot2 */
+			DRA7XX_CORE_IOPAD(0x340C, MUX_MODE15) /* M1: sysboot3 */
+			DRA7XX_CORE_IOPAD(0x3410, MUX_MODE15) /* L6: sysboot4 */
+			DRA7XX_CORE_IOPAD(0x3414, MUX_MODE15) /* L4: sysboot5 */
+			DRA7XX_CORE_IOPAD(0x3418, MUX_MODE15) /* L3: sysboot6 */
+			DRA7XX_CORE_IOPAD(0x341C, MUX_MODE15) /* L2: sysboot7 */
+			DRA7XX_CORE_IOPAD(0x3420, MUX_MODE15) /* L1: sysboot8 */
+			DRA7XX_CORE_IOPAD(0x3424, MUX_MODE15) /* K2: sysboot9 */
+			DRA7XX_CORE_IOPAD(0x3428, MUX_MODE15) /* J1: sysboot10 */
+			DRA7XX_CORE_IOPAD(0x342C, MUX_MODE15) /* J2: sysboot11 */
+			DRA7XX_CORE_IOPAD(0x3430, MUX_MODE15) /* H1: sysboot12 */
+			DRA7XX_CORE_IOPAD(0x3434, MUX_MODE15) /* J3: sysboot13 */
+			DRA7XX_CORE_IOPAD(0x3438, MUX_MODE15) /* H2: sysboot14 */
+			DRA7XX_CORE_IOPAD(0x343C, MUX_MODE15) /* H3: sysboot15 */
+
+			DRA7XX_CORE_IOPAD(0x3448, MUX_MODE15) /* T6: N/C */
+			DRA7XX_CORE_IOPAD(0x344C, MUX_MODE15) /* T7: N/C */
+
+			DRA7XX_CORE_IOPAD(0x3460, MUX_MODE15) /* N7: N/C */
+			DRA7XX_CORE_IOPAD(0x3464, MUX_MODE15) /* R4: N/C */
+			DRA7XX_CORE_IOPAD(0x3468, MUX_MODE15) /* N9: N/C */
+			DRA7XX_CORE_IOPAD(0x346C, MUX_MODE15) /* P9: N/C */
+			DRA7XX_CORE_IOPAD(0x3470, MUX_MODE15) /* P4: N/C */
+			DRA7XX_CORE_IOPAD(0x3474, MUX_MODE15) /* R3: N/C */
+			DRA7XX_CORE_IOPAD(0x3478, MUX_MODE15) /* T2: N/C */
+			DRA7XX_CORE_IOPAD(0x347C, MUX_MODE15) /* U2: N/C */
+			DRA7XX_CORE_IOPAD(0x3480, MUX_MODE15) /* U1: N/C */
+			DRA7XX_CORE_IOPAD(0x3484, MUX_MODE15) /* P3: N/C */
+			DRA7XX_CORE_IOPAD(0x3488, MUX_MODE15) /* R2: N/C */
+
+			DRA7XX_CORE_IOPAD(0x34B4, MUX_MODE15) /* T1: N/C */
+			DRA7XX_CORE_IOPAD(0x34B8, MUX_MODE15) /* P2: N/C */
+			DRA7XX_CORE_IOPAD(0x34BC, MUX_MODE15) /* P1: N/C */
+			DRA7XX_CORE_IOPAD(0x34C0, MUX_MODE15) /* P7: N/C */
+			DRA7XX_CORE_IOPAD(0x34C4, MUX_MODE15) /* N1: N/C */
+			DRA7XX_CORE_IOPAD(0x34C8, MUX_MODE15) /* M5: N/C */
+			DRA7XX_CORE_IOPAD(0x34CC, MUX_MODE15) /* M3: N/C */
+			DRA7XX_CORE_IOPAD(0x34D0, MUX_MODE15) /* N6: N/C */
+			DRA7XX_CORE_IOPAD(0x34D4, MUX_MODE15) /* M4: N/C */
+			DRA7XX_CORE_IOPAD(0x34D8, MUX_MODE15) /* N2: N/C */
+
+			DRA7XX_CORE_IOPAD(0x34E0, MUX_MODE15) /* AH7: N/C */
+
+			DRA7XX_CORE_IOPAD(0x34EC, MUX_MODE15) /* AE9: N/C */
+
+			DRA7XX_CORE_IOPAD(0x34F4, MUX_MODE15) /* AE8: N/C */
+			DRA7XX_CORE_IOPAD(0x34F8, MUX_MODE15) /* AD8: N/C */
+			DRA7XX_CORE_IOPAD(0x34FC, MUX_MODE15) /* AG7: N/C */
+
+			DRA7XX_CORE_IOPAD(0x3504, MUX_MODE15) /* AH3: N/C */
+			DRA7XX_CORE_IOPAD(0x3508, MUX_MODE15) /* AH5: N/C */
+
+			DRA7XX_CORE_IOPAD(0x3524, MUX_MODE15) /* AF2: N/C */
+
+			DRA7XX_CORE_IOPAD(0x3530, MUX_MODE15) /* AF4: N/C */
+
+			DRA7XX_CORE_IOPAD(0x354C, MUX_MODE15) /* AD2: N/C */
+
+			DRA7XX_CORE_IOPAD(0x3554, MUX_MODE15) /* E1: N/C */
+			DRA7XX_CORE_IOPAD(0x3558, MUX_MODE15) /* G2: N/C */
+			DRA7XX_CORE_IOPAD(0x355C, MUX_MODE15) /* H7: N/C */
+			DRA7XX_CORE_IOPAD(0x3560, MUX_MODE15) /* G1: N/C */
+
+			DRA7XX_CORE_IOPAD(0x356C, MUX_MODE15) /* F3: N/C */
+
+			DRA7XX_CORE_IOPAD(0x3574, MUX_MODE15) /* E2: N/C */
+
+			DRA7XX_CORE_IOPAD(0x3584, MUX_MODE15) /* E4: N/C */
+
+			DRA7XX_CORE_IOPAD(0x3594, MUX_MODE15) /* F6: N/C */
+
+			DRA7XX_CORE_IOPAD(0x35A4, MUX_MODE15) /* C4: N/C */
+			DRA7XX_CORE_IOPAD(0x35A8, MUX_MODE15) /* B2: N/C */
+
+			DRA7XX_CORE_IOPAD(0x35C0, MUX_MODE15) /* B5: N/C */
+			DRA7XX_CORE_IOPAD(0x35C4, MUX_MODE15) /* A4: N/C */
+
+			DRA7XX_CORE_IOPAD(0x35D0, MUX_MODE15) /* B11: N/C */
+
+			DRA7XX_CORE_IOPAD(0x3644, MUX_MODE15) /* U3: N/C */
+
+			DRA7XX_CORE_IOPAD(0x3680, MUX_MODE15) /* AB10: N/C */
+
+			DRA7XX_CORE_IOPAD(0x36BC, MUX_MODE15) /* G13: N/C */
+
+			DRA7XX_CORE_IOPAD(0x36C4, MUX_MODE15) /* E12: N/C */
+
+			DRA7XX_CORE_IOPAD(0x36CC, MUX_MODE15) /* C12: N/C */
+			DRA7XX_CORE_IOPAD(0x36D0, MUX_MODE15) /* D12: N/C */
+
+			DRA7XX_CORE_IOPAD(0x36FC, MUX_MODE15) /* E15: N/C */
+			DRA7XX_CORE_IOPAD(0x3700, MUX_MODE15) /* A20: N/C */
+
+			DRA7XX_CORE_IOPAD(0x370C, MUX_MODE15) /* C15: N/C */
+			DRA7XX_CORE_IOPAD(0x3710, MUX_MODE15) /* A16: N/C */
+			DRA7XX_CORE_IOPAD(0x3714, MUX_MODE15) /* D15: N/C */
+			DRA7XX_CORE_IOPAD(0x3718, MUX_MODE15) /* B16: N/C */
+			DRA7XX_CORE_IOPAD(0x371C, MUX_MODE15) /* B17: N/C */
+			DRA7XX_CORE_IOPAD(0x3720, MUX_MODE15) /* A17: N/C */
+			DRA7XX_CORE_IOPAD(0x3724, MUX_MODE15) /* B18: N/C */
+			DRA7XX_CORE_IOPAD(0x3728, MUX_MODE15) /* F15: N/C */
+
+			DRA7XX_CORE_IOPAD(0x3744, MUX_MODE15) /* AA3: N/C */
+			DRA7XX_CORE_IOPAD(0x3748, MUX_MODE15) /* AB9: N/C */
+			DRA7XX_CORE_IOPAD(0x374C, MUX_MODE15) /* AB3: N/C */
+			DRA7XX_CORE_IOPAD(0x3750, MUX_MODE15) /* AA4: N/C */
+
+			DRA7XX_CORE_IOPAD(0x3770, MUX_MODE15) /* Y9: N/C */
+			DRA7XX_CORE_IOPAD(0x3774, MUX_MODE15) /* AC5: N/C */
+			DRA7XX_CORE_IOPAD(0x3778, MUX_MODE15) /* AB4: N/C */
+
+			DRA7XX_CORE_IOPAD(0x37A4, MUX_MODE15) /* A25: N/C */
+			DRA7XX_CORE_IOPAD(0x37A8, MUX_MODE15) /* F16: N/C */
+			DRA7XX_CORE_IOPAD(0x37AC, MUX_MODE15) /* B25: N/C */
+			DRA7XX_CORE_IOPAD(0x37B0, MUX_MODE15) /* A24: N/C */
+
+			DRA7XX_CORE_IOPAD(0x37D0, MUX_MODE15) /* G20: N/C */
+			DRA7XX_CORE_IOPAD(0x37D4, MUX_MODE15) /* G19: N/C */
+
+			DRA7XX_CORE_IOPAD(0x3818, MUX_MODE15) /* AD17: N/C */
+			DRA7XX_CORE_IOPAD(0x381C, MUX_MODE15) /* AC17: N/C */
+			DRA7XX_CORE_IOPAD(0x3820, MUX_MODE15) /* AB16: N/C */
+			DRA7XX_CORE_IOPAD(0x3824, MUX_MODE15) /* AC16: N/C */
+		>;
+	};
+
+	cape_pins_default: cape_pins_default {
+		pinctrl-single,pins = <
+			DRA7XX_CORE_IOPAD(0x379C, MUX_MODE14) /* AB8: P8.3: mmc3_dat6.off */
+			DRA7XX_CORE_IOPAD(0x37A0, MUX_MODE14) /* AB5: P8.4: mmc3_dat7.off */
+			DRA7XX_CORE_IOPAD(0x378C, MUX_MODE14) /* AC9: P8.5: mmc3_dat2.off */
+			DRA7XX_CORE_IOPAD(0x3790, MUX_MODE14) /* AC3: P8.6: mmc3_dat3.off */
+			DRA7XX_CORE_IOPAD(0x36EC, MUX_MODE14) /* G14: P8.7: mcasp1_axr14.off */
+			DRA7XX_CORE_IOPAD(0x36F0, MUX_MODE14) /* F14: P8.8: mcasp1_axr15.off */
+			DRA7XX_CORE_IOPAD(0x3698, MUX_MODE14) /* E17: P8.9: xref_clk1.off */
+			DRA7XX_CORE_IOPAD(0x36E8, MUX_MODE14) /* A13: P8.10: mcasp1_axr13.off */
+			DRA7XX_CORE_IOPAD(0x3510, MUX_MODE14) /* AH4: P8.11: vin1a_d7.off */
+			DRA7XX_CORE_IOPAD(0x350C, MUX_MODE14) /* AG6: P8.12: vin1a_d6.off */
+			DRA7XX_CORE_IOPAD(0x3590, PIN_INPUT | MUX_MODE12) /* D3: P8.13: vin2a_d10.off */
+			DRA7XX_CORE_IOPAD(0x3598, MUX_MODE14) /* D5: P8.14: vin2a_d12.off */
+			DRA7XX_CORE_IOPAD(0x3570, MUX_MODE14) /* D1: P8.15a: vin2a_d2.off */
+			DRA7XX_CORE_IOPAD(0x35B4, MUX_MODE13) /* A3: P8.15b: vin2a_d19.off */
+			DRA7XX_CORE_IOPAD(0x35BC, MUX_MODE13) /* B4: P8.16: vin2a_d21.off */
+			DRA7XX_CORE_IOPAD(0x3624, MUX_MODE14) /* A7: P8.17: vout1_d18.off */
+			DRA7XX_CORE_IOPAD(0x3588, PIN_INPUT | MUX_MODE12) /* F5: P8.18: vin2a_d8.off */
+			DRA7XX_CORE_IOPAD(0x358C, PIN_INPUT | MUX_MODE12) /* E6: P8.19: vin2a_d9.off */
+			DRA7XX_CORE_IOPAD(0x3780, MUX_MODE14) /* AC4: P8.20: mmc3_cmd.off */
+			DRA7XX_CORE_IOPAD(0x377C, MUX_MODE14) /* AD4: P8.21: mmc3_clk.off */
+			DRA7XX_CORE_IOPAD(0x3798, MUX_MODE14) /* AD6: P8.22: mmc3_dat5.off */
+			DRA7XX_CORE_IOPAD(0x3794, MUX_MODE14) /* AC8: P8.23: mmc3_dat4.off */
+			DRA7XX_CORE_IOPAD(0x3788, MUX_MODE14) /* AC6: P8.24: mmc3_dat1.off */
+			DRA7XX_CORE_IOPAD(0x3784, MUX_MODE14) /* AC7: P8.25: mmc3_dat0.off */
+			DRA7XX_CORE_IOPAD(0x35B8, MUX_MODE13) /* B3: P8.26: vin2a_d20.off */
+			DRA7XX_CORE_IOPAD(0x35D8, MUX_MODE14) /* E11: P8.27a: vout1_vsync.off */
+			DRA7XX_CORE_IOPAD(0x3628, MUX_MODE14) /* A8: P8.27b: vout1_d19.off */
+			DRA7XX_CORE_IOPAD(0x35C8, MUX_MODE14) /* D11: P8.28a: vout1_clk.off */
+			DRA7XX_CORE_IOPAD(0x362C, MUX_MODE14) /* C9: P8.28b: vout1_d20.off */
+			DRA7XX_CORE_IOPAD(0x35D4, MUX_MODE14) /* C11: P8.29a: vout1_hsync.off */
+			DRA7XX_CORE_IOPAD(0x3630, MUX_MODE14) /* A9: P8.29b: vout1_d21.off */
+			DRA7XX_CORE_IOPAD(0x35CC, MUX_MODE14) /* B10: P8.30a: vout1_de.off */
+			DRA7XX_CORE_IOPAD(0x3634, MUX_MODE14) /* B9: P8.30b: vout1_d22.off */
+			DRA7XX_CORE_IOPAD(0x3614, MUX_MODE14) /* C8: P8.31a: vout1_d14.off */
+			DRA7XX_CORE_IOPAD(0x373C, MUX_MODE14) /* G16: P8.31b: mcasp4_axr0.off */
+			DRA7XX_CORE_IOPAD(0x3618, MUX_MODE14) /* C7: P8.32a: vout1_d15.off */
+			DRA7XX_CORE_IOPAD(0x3740, MUX_MODE14) /* D17: P8.32b: mcasp4_axr1.off */
+			DRA7XX_CORE_IOPAD(0x3610, MUX_MODE14) /* C6: P8.33a: vout1_d13.off */
+			DRA7XX_CORE_IOPAD(0x34E8, MUX_MODE14) /* AF9: P8.33b: vin1a_fld0.off */
+			DRA7XX_CORE_IOPAD(0x3608, MUX_MODE14) /* D8: P8.34a: vout1_d11.off */
+			DRA7XX_CORE_IOPAD(0x3564, MUX_MODE14) /* G6: P8.34b: vin2a_vsync0.off */
+			DRA7XX_CORE_IOPAD(0x360C, MUX_MODE14) /* A5: P8.35a: vout1_d12.off */
+			DRA7XX_CORE_IOPAD(0x34E4, MUX_MODE14) /* AD9: P8.35b: vin1a_de0.off */
+			DRA7XX_CORE_IOPAD(0x3604, MUX_MODE14) /* D7: P8.36a: vout1_d10.off */
+			DRA7XX_CORE_IOPAD(0x3568, MUX_MODE14) /* F2: P8.36b: vin2a_d0.off */
+			DRA7XX_CORE_IOPAD(0x35FC, MUX_MODE14) /* E8: P8.37a: vout1_d8.off */
+			DRA7XX_CORE_IOPAD(0x3738, MUX_MODE14) /* A21: P8.37b: mcasp4_fsx.off */
+			DRA7XX_CORE_IOPAD(0x3600, MUX_MODE14) /* D9: P8.38a: vout1_d9.off */
+			DRA7XX_CORE_IOPAD(0x3734, MUX_MODE14) /* C18: P8.38b: mcasp4_aclkx.off */
+			DRA7XX_CORE_IOPAD(0x35F4, MUX_MODE14) /* F8: P8.39: vout1_d6.off */
+			DRA7XX_CORE_IOPAD(0x35F8, MUX_MODE14) /* E7: P8.40: vout1_d7.off */
+			DRA7XX_CORE_IOPAD(0x35EC, MUX_MODE14) /* E9: P8.41: vout1_d4.off */
+			DRA7XX_CORE_IOPAD(0x35F0, MUX_MODE14) /* F9: P8.42: vout1_d5.off */
+			DRA7XX_CORE_IOPAD(0x35E4, MUX_MODE14) /* F10: P8.43: vout1_d2.off */
+			DRA7XX_CORE_IOPAD(0x35E8, MUX_MODE14) /* G11: P8.44: vout1_d3.off */
+			DRA7XX_CORE_IOPAD(0x35DC, MUX_MODE14) /* F11: P8.45a: vout1_d0.off */
+			DRA7XX_CORE_IOPAD(0x361C, MUX_MODE14) /* B7: P8.45b: vout1_d16.off */
+			DRA7XX_CORE_IOPAD(0x35E0, MUX_MODE14) /* G10: P8.46a: vout1_d1.off */
+			DRA7XX_CORE_IOPAD(0x3638, MUX_MODE14) /* A10: P8.46b: vout1_d23.off */
+			DRA7XX_CORE_IOPAD(0x372C, MUX_MODE14) /* B19: P9.11a: mcasp3_axr0.off */
+			DRA7XX_CORE_IOPAD(0x3620, MUX_MODE14) /* B8: P9.11b: vout1_d17.off */
+			DRA7XX_CORE_IOPAD(0x36AC, MUX_MODE14) /* B14: P9.12: mcasp1_aclkr.off */
+			DRA7XX_CORE_IOPAD(0x3730, MUX_MODE14) /* C17: P9.13: mcasp3_axr1.off */
+			DRA7XX_CORE_IOPAD(0x35AC, MUX_MODE10) /* D6: P9.14: vin2a_d17.off */
+			DRA7XX_CORE_IOPAD(0x3514, MUX_MODE14) /* AG4: P9.15: vin1a_d8.off */
+			DRA7XX_CORE_IOPAD(0x35B0, MUX_MODE13) /* C5: P9.16: vin2a_d18.off */
+			DRA7XX_CORE_IOPAD(0x37CC, MUX_MODE14) /* B24: P9.17a: spi2_cs0.off */
+			DRA7XX_CORE_IOPAD(0x36B8, MUX_MODE14) /* F12: P9.17b: mcasp1_axr1.off */
+			DRA7XX_CORE_IOPAD(0x37C8, MUX_MODE14) /* G17: P9.18a: spi2_d0.off */
+			DRA7XX_CORE_IOPAD(0x36B4, MUX_MODE14) /* G12: P9.18b: mcasp1_axr0.off */
+			DRA7XX_CORE_IOPAD(0x3440, PIN_INPUT_PULLUP | MUX_MODE7) /* R6: P9.19a: gpmc_a0.i2c4_scl */
+			DRA7XX_CORE_IOPAD(0x357C, PIN_INPUT_PULLUP | MUX_MODE12 ) /* F4: P9.19b: vin2a_d5.pr1_pru1_gpi2 */
+			DRA7XX_CORE_IOPAD(0x3444, PIN_INPUT_PULLUP | MUX_MODE7) /* T9: P9.20a: gpmc_a1.i2c4_sda */
+			DRA7XX_CORE_IOPAD(0x3578, PIN_INPUT_PULLUP | MUX_MODE12) /* D2: P9.20b: vin2a_d4.pr1_pru1_gpi1 */
+			DRA7XX_CORE_IOPAD(0x34F0, MUX_MODE14) /* AF8: P9.21a: vin1a_vsync0.off */
+			DRA7XX_CORE_IOPAD(0x37C4, MUX_MODE14) /* B22: P9.21b: spi2_d1.off */
+			DRA7XX_CORE_IOPAD(0x369C, MUX_MODE14) /* B26: P9.22a: xref_clk2.off */
+			DRA7XX_CORE_IOPAD(0x37C0, MUX_MODE14) /* A26: P9.22b: spi2_sclk.off */
+			DRA7XX_CORE_IOPAD(0x37B4, MUX_MODE14) /* A22: P9.23: spi1_cs1.off */
+			DRA7XX_CORE_IOPAD(0x368C, MUX_MODE14) /* F20: P9.24: gpio6_15.off */
+			DRA7XX_CORE_IOPAD(0x3694, MUX_MODE14) /* D18: P9.25: xref_clk0.off */
+			DRA7XX_CORE_IOPAD(0x3688, MUX_MODE14) /* E21: P9.26a: gpio6_14.off */
+			DRA7XX_CORE_IOPAD(0x3544, MUX_MODE14) /* AE2: P9.26b: vin1a_d20.off */
+			DRA7XX_CORE_IOPAD(0x35A0, MUX_MODE14) /* C3: P9.27a: vin2a_d14.off */
+			DRA7XX_CORE_IOPAD(0x36B0, MUX_MODE14) /* J14: P9.27b: mcasp1_fsr.off */
+			DRA7XX_CORE_IOPAD(0x36E0, MUX_MODE14) /* A12: P9.28: mcasp1_axr11.off */
+			DRA7XX_CORE_IOPAD(0x36D8, MUX_MODE14) /* A11: P9.29a: mcasp1_axr9.off */
+			DRA7XX_CORE_IOPAD(0x36A8, MUX_MODE14) /* D14: P9.29b: mcasp1_fsx.off */
+			DRA7XX_CORE_IOPAD(0x36DC, MUX_MODE14) /* B13: P9.30: mcasp1_axr10.off */
+			DRA7XX_CORE_IOPAD(0x36D4, MUX_MODE14) /* B12: P9.31a: mcasp1_axr8.off */
+			DRA7XX_CORE_IOPAD(0x36A4, MUX_MODE14) /* C14: P9.31b: mcasp1_aclkx.off */
+			DRA7XX_CORE_IOPAD(0x36A0, MUX_MODE14) /* C23: P9.41a: xref_clk3.off */
+			DRA7XX_CORE_IOPAD(0x3580, MUX_MODE14) /* C1: P9.41b: vin2a_d6.off */
+			DRA7XX_CORE_IOPAD(0x36E4, MUX_MODE14) /* E14: P9.42a: mcasp1_axr12.off */
+			DRA7XX_CORE_IOPAD(0x359C, MUX_MODE14) /* C2: P9.42b: vin2a_d13.off */
+		>;
+	};
+};
+
+&vip2 {
+	status = "okay";
+};
+
+&i2c1 {
+	status = "okay";
+	clock-frequency = <400000>;
+
+	tps659038: tps659038@58 {
+		compatible = "ti,tps659038";
+		reg = <0x58>;
+		interrupt-parent = <&gpio6>;
+		interrupts = <16 IRQ_TYPE_LEVEL_LOW>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&pmic_pins_default>;
+
+		#interrupt-cells = <2>;
+		interrupt-controller;
+
+		ti,system-power-controller;
+		ti,palmas-override-powerhold;
+
+		tps659038_pmic {
+			compatible = "ti,tps659038-pmic";
+
+			smps12-in-supply = <&vdd_5v>;
+			smps3-in-supply = <&vdd_5v>;
+			smps45-in-supply = <&vdd_5v>;
+			smps6-in-supply = <&vdd_5v>;
+			smps7-in-supply = <&vdd_5v>;
+			mps3-in-supply = <&vdd_5v>;
+			smps8-in-supply = <&vdd_5v>;
+			smps9-in-supply = <&vdd_5v>;
+			ldo1-in-supply = <&vdd_5v>;
+			ldo2-in-supply = <&vdd_5v>;
+			ldo3-in-supply = <&vdd_5v>;
+			ldo4-in-supply = <&vdd_5v>;
+			ldo9-in-supply = <&vdd_5v>;
+			ldoln-in-supply = <&vdd_5v>;
+			ldousb-in-supply = <&vdd_5v>;
+			ldortc-in-supply = <&vdd_5v>;
+
+			regulators {
+				vdd_mpu: smps12 {
+					/* VDD_MPU */
+					regulator-name = "smps12";
+					regulator-min-microvolt = <850000>;
+					regulator-max-microvolt = <1250000>;
+					regulator-always-on;
+					regulator-boot-on;
+				};
+
+				vdd_ddr: smps3 {
+					/* VDD_DDR EMIF1 EMIF2 */
+					regulator-name = "smps3";
+					regulator-min-microvolt = <1350000>;
+					regulator-max-microvolt = <1350000>;
+					regulator-always-on;
+					regulator-boot-on;
+				};
+
+				vdd_dspeve: smps45 {
+					/* VDD_DSPEVE on AM572 */
+					regulator-name = "smps45";
+					regulator-min-microvolt = < 850000>;
+					regulator-max-microvolt = <1250000>;
+					regulator-always-on;
+					regulator-boot-on;
+				};
+
+				vdd_gpu: smps6 {
+					/* VDD_GPU */
+					regulator-name = "smps6";
+					regulator-min-microvolt = < 850000>;
+					regulator-max-microvolt = <1250000>;
+					regulator-always-on;
+					regulator-boot-on;
+				};
+
+				vdd_core: smps7 {
+					/* VDD_CORE */
+					regulator-name = "smps7";
+					regulator-min-microvolt = < 850000>;	/*** 1.15V */
+					regulator-max-microvolt = <1150000>;
+					regulator-always-on;
+					regulator-boot-on;
+				};
+
+				vdd_iva: smps8 {
+					/* VDD_IVAHD */				/*** 1.06V */
+					regulator-name = "smps8";
+				};
+
+				vdd_3v3: smps9 {
+					/* VDD_3V3 */
+					regulator-name = "smps9";
+					regulator-min-microvolt = <3300000>;
+					regulator-max-microvolt = <3300000>;
+					regulator-always-on;
+					regulator-boot-on;
+				};
+
+				vdd_sd: ldo1 {
+					/* VDDSHV8 - VSDMMC  */
+					regulator-name = "ldo1";
+					regulator-min-microvolt = <1800000>;
+					regulator-max-microvolt = <3300000>;
+					regulator-boot-on;
+					regulator-always-on;
+				};
+
+				vdd_1v8: ldo2 {
+					/* VDDSH18V */
+					regulator-name = "ldo2";
+					regulator-min-microvolt = <1800000>;
+					regulator-max-microvolt = <1800000>;
+					regulator-always-on;
+					regulator-boot-on;
+				};
+
+				vdd_1v8_phy_ldo3: ldo3 {
+					/* R1.3a 572x V1_8PHY_LDO3: USB, SATA */
+					regulator-name = "ldo3";
+					regulator-min-microvolt = <1800000>;
+					regulator-max-microvolt = <1800000>;
+					regulator-always-on;
+					regulator-boot-on;
+				};
+
+				vdd_1v8_phy_ldo4: ldo4 {
+					/* R1.3a 572x V1_8PHY_LDO4: PCIE, HDMI*/
+					regulator-name = "ldo4";
+					regulator-min-microvolt = <1800000>;
+					regulator-max-microvolt = <1800000>;
+					regulator-always-on;
+					regulator-boot-on;
+				};
+
+				/* LDO5-8 unused */
+
+				vdd_rtc: ldo9 {
+					/* VDD_RTC  */
+					regulator-name = "ldo9";
+					regulator-min-microvolt = < 840000>;
+					regulator-max-microvolt = <1160000>;
+					regulator-always-on;
+					regulator-boot-on;
+				};
+
+				vdd_1v8_pll: ldoln {
+					/* VDDA_1V8_PLL */
+					regulator-name = "ldoln";
+					regulator-min-microvolt = <1800000>;
+					regulator-max-microvolt = <1800000>;
+					regulator-always-on;
+					regulator-boot-on;
+				};
+
+				ldousb_reg: ldousb {
+					/* VDDA_3V_USB: VDDA_USBHS33 */
+					regulator-name = "ldousb";
+					regulator-min-microvolt = <3300000>;
+					regulator-max-microvolt = <3300000>;
+					regulator-always-on;
+					regulator-boot-on;
+				};
+
+				ldortc_reg: ldortc {
+					/* VDDA_RTC  */
+					regulator-name = "ldortc";
+					regulator-min-microvolt = <1800000>;
+					regulator-max-microvolt = <1800000>;
+					regulator-always-on;
+					regulator-boot-on;
+				};
+
+				regen1: regen1 {
+					/* VDD_3V3_ON */
+					regulator-name = "regen1";
+					regulator-boot-on;
+					regulator-always-on;
+				};
+
+				regen2: regen2 {
+					/* Needed for PMIC internal resource */
+					regulator-name = "regen2";
+					regulator-boot-on;
+					regulator-always-on;
+				};
+			};
+		};
+
+		tps659038_rtc: tps659038_rtc {
+			compatible = "ti,palmas-rtc";
+			interrupt-parent = <&tps659038>;
+			interrupts = <8 IRQ_TYPE_EDGE_FALLING>;
+			wakeup-source;
+		};
+
+		tps659038_pwr_button: tps659038_pwr_button {
+			compatible = "ti,palmas-pwrbutton";
+			interrupt-parent = <&tps659038>;
+			interrupts = <1 IRQ_TYPE_EDGE_FALLING>;
+			wakeup-source;
+			ti,palmas-long-press-seconds = <12>;
+		};
+
+		tps659038_gpio: tps659038_gpio {
+			compatible = "ti,palmas-gpio";
+			gpio-controller;
+			#gpio-cells = <2>;
+		};
+	};
+
+	/* STMPE811 touch screen controller */
+	stmpe811@41 {
+		compatible = "st,stmpe811";
+		reg = <0x41>;
+		interrupts = <30 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-parent = <&gpio2>;
+		interrupt-controller;
+		id = <0>;
+		blocks = <0x5>;
+		irq-trigger = <0x1>;
+		st,mod-12b = <1>; /* 12-bit ADC */
+		st,ref-sel = <0>; /* internal ADC reference */
+		st,adc-freq = <1>; /* 3.25 MHz ADC clock speed */
+		st,sample-time = <4>; /* ADC converstion time: 80 clocks */
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&adc_pins_default>;
+
+		stmpe_adc {
+			compatible = "st,stmpe-adc";
+			st,norequest-mask = <0x00>; /* mask any channels to be used by touchscreen */
+			adc0: iio-device@0 {
+				#io-channel-cells = <1>;
+				iio-channels = <&adc0 4>, <&adc0 1>, <&adc0 2>, <&adc0 3>, <&adc0 4>, <&adc0 5>, <&adc0 6>;
+				iio-channel-names = "AIN0_P9_39", "AIN1_P9_40", "AIN2_P9_37", "AIN3_P9_38",
+					"AIN4_P9_33", "AIN5_P9_36", "AIN6_P9_35";
+			};
+		};
+
+		stmpe_touchscreen {
+			status = "disabled";
+			compatible = "st,stmpe-ts";
+			/* 8 sample average control */
+			st,ave-ctrl = <3>;
+			/* 7 length fractional part in z */
+			st,fraction-z = <7>;
+			/*
+			 * 50 mA typical 80 mA max touchscreen drivers
+			 * current limit value
+			 */
+			st,i-drive = <1>;
+			/* 1 ms panel driver settling time */
+			st,settling = <3>;
+			/* 5 ms touch detect interrupt delay */
+			st,touch-det-delay = <5>;
+		};
+
+		stmpe_gpio {
+			compatible = "st,stmpe-gpio";
+		};
+
+		stmpe_pwm {
+			compatible = "st,stmpe-pwm";
+			#pwm-cells = <2>;
+		};
+	};
+};
+
+&mcspi3 {
+	status = "okay";
+	ti,pindir-d0-out-d1-in;
+
+	sn65hvs882: sn65hvs882@0 {
+		compatible = "pisosr-gpio";
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		reg = <0>;
+		spi-max-frequency = <1000000>;
+		spi-cpol;
+	};
+};
+
+&cpu0 {
+	vdd-supply = <&vdd_mpu>;
+	voltage-tolerance = <1>;
+};
+
+&gpu {
+	status = "okay";
+};
+
+&pruss_soc_bus1 {
+	status = "okay";
+
+	pruss1: pruss@4b200000 {
+		status = "okay";
+	};
+};
+
+&pruss_soc_bus2 {
+	status = "okay";
+
+	pruss2: pruss@4b280000 {
+		status = "okay";
+	};
+};
+
+&uart1 {
+	status = "okay";
+};
+
+&davinci_mdio {
+	reset-gpios = <&gpio2 23 GPIO_ACTIVE_LOW>;
+	reset-delay-us = <2>;
+
+	phy0: ethernet-phy@1 {
+		reg = <4>;
+		compatible = "ethernet-phy-id004d.d072",
+			"ethernet-phy-ieee802.3-c22";
+		eee-broken-100tx;
+		eee-broken-1000t;
+	};
+};
+
+&mac {
+	slaves = <1>;
+	status = "okay";
+};
+
+&cpsw_emac0 {
+	phy-handle = <&phy0>;
+	phy-mode = "rgmii";
+};
+
+&ipu2 {
+	status = "okay";
+	memory-region = <&ipu2_memory_region>;
+};
+
+&ipu1 {
+	status = "okay";
+	memory-region = <&ipu1_memory_region>;
+};
+
+&dsp1 {
+	status = "okay";
+	memory-region = <&dsp1_memory_region>;
+};
+
+&dsp2 {
+	status = "okay";
+	memory-region = <&dsp2_memory_region>;
+};
+
+&mmc1 {
+	status = "okay";
+	vmmc-supply = <&vdd_3v3>;
+	vqmmc-supply = <&vdd_sd>;
+	bus-width = <4>;
+	cd-gpios = <&gpio6 27 GPIO_ACTIVE_LOW>; /* gpio 219 */
+
+	ti,needs-special-reset;
+	ti,dual-volt;
+	sd-uhs-sdr12;
+	sd-uhs-sdr25;
+	sd-uhs-sdr50;
+	sd-uhs-ddr50;
+	sd-uhs-sdr104;
+	mmc-hs200-1_8v;
+	cap-sd-highspeed;
+	cap-mmc-highspeed;
+	dmas = <&sdma_xbar 61>, <&sdma_xbar 62>;
+	dma-names = "tx", "rx";
+
+	pinctrl-names = "default", "hs", "sdr12", "sdr25", "sdr50", "ddr50", "sdr104";
+	pinctrl-0 = <&mmc1_pins_default &microsd_extra_pins_default>;
+	pinctrl-1 = <&mmc1_pins_hs &microsd_extra_pins_default>;
+	pinctrl-2 = <&mmc1_pins_sdr12 &microsd_extra_pins_default>;
+	pinctrl-3 = <&mmc1_pins_sdr25 &microsd_extra_pins_default>;
+	pinctrl-4 = <&mmc1_pins_sdr50 &microsd_extra_pins_default>;
+	pinctrl-5 = <&mmc1_pins_ddr50 &mmc1_iodelay_ddr_rev20_conf &microsd_extra_pins_default>;
+	pinctrl-6 = <&mmc1_pins_sdr104 &mmc1_iodelay_sdr104_rev20_conf &microsd_extra_pins_default>;
+};
+
+&mmc2 {
+	status = "okay";
+	vmmc-supply = <&vdd_1v8>;
+	vqmmc-supply = <&vdd_1v8>;
+	bus-width = <8>;
+	ti,non-removable;
+	non-removable;
+	mmc-pwrseq = <&emmc_pwrseq>;
+
+	ti,needs-special-reset;
+	dmas = <&sdma_xbar 47>, <&sdma_xbar 48>;
+	dma-names = "tx", "rx";
+	pinctrl-names = "default", "hs", "ddr_1_8v", "hs200_1_8v";
+	pinctrl-0 = <&mmc2_pins_default>;
+	pinctrl-1 = <&mmc2_pins_hs>;
+	pinctrl-2 = <&mmc2_pins_ddr_rev20>;
+	pinctrl-3 = <&mmc2_pins_hs200>;
+};
+
+&mmc4 {
+	/* DS: Default speed (DS) up to 25 MHz, including 1- and 4-bit modes (3.3 V signaling). */
+	/* HS: High speed up to 50 MHz (3.3 V signaling). */
+	/* SDR12: SDR up to 25 MHz (1.8 V signaling). */
+	/* SDR25: SDR up to 50 MHz (1.8 V signaling). */
+	/* SDR50: SDR up to 100 MHz (1.8 V signaling). */
+	/* SDR104: SDR up to 208 MHz (1.8 V signaling) */
+	/* DDR50: DDR up to 50 MHz (1.8 V signaling). */
+	status = "okay";
+
+	pinctrl-names = "default", "hs";
+	pinctrl-0 = <&mmc4_pins_default &wifibt_extra_pins_default>;
+	pinctrl-1 = <&mmc4_pins_hs &wifibt_extra_pins_default>;
+
+	ti,needs-special-reset;
+	vmmc-supply = <&vdd_3v3>;
+	cap-power-off-card;
+	keep-power-in-suspend;
+	bus-width = <4>;
+	ti,non-removable;
+	non-removable;
+	no-1-8-v;
+	max-frequency = <24000000>;
+
+	#address-cells = <1>;
+	#size-cells = <0>;
+	mmc-pwrseq = <&brcmf_pwrseq>;
+
+	brcmf: wifi@1 {
+		status = "okay";
+		reg = <1>;
+		compatible = "brcm,bcm4329-fmac";
+
+		//brcm,drive-strength = <10>;
+		//brcm,broken-sg-support;
+		//brcm,broken_sg_support;
+		brcm,sd-head-align = <4>;
+		brcm,sd_head_align = <4>;
+		brcm,sd_sgentry_align = <512>;
+
+		interrupt-parent = <&gpio3>;
+		interrupts = <23 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-names = "host-wake";
+	};
+};
+
+&usb2_phy1 {
+	phy-supply = <&ldousb_reg>;
+};
+
+&usb2_phy2 {
+	phy-supply = <&ldousb_reg>;
+};
+
+&usb1 {
+	status = "okay";
+	dr_mode = "otg";
+};
+
+&omap_dwc3_1 {
+	extcon = <&extcon_usb1>;
+};
+
+&usb2 {
+	status = "okay";
+	dr_mode = "host";
+};
+
+&bb2d {
+	status = "okay";
+};
+
+&dss {
+	status = "okay";
+	vdda_video-supply = <&vdd_1v8_pll>;
+};
+
+&hdmi {
+	status = "okay";
+	vdda-supply = <&vdd_1v8_phy_ldo4>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&hdmi_pins_default>;
+
+	port {
+		hdmi_out: endpoint {
+			remote-endpoint = <&hdmi_encoder_in>;
+		};
+	};
+};
+
+&bandgap {
+	status = "okay";
+};
+
+&mailbox1 {
+	status = "okay";
+};
+
+&mailbox2 {
+	status = "okay";
+};
+
+&mailbox3 {
+	status = "okay";
+};
+
+&mailbox4 {
+	status = "okay";
+};
+
+&mailbox5 {
+	status = "okay";
+};
+
+&mailbox6 {
+	status = "okay";
+};
+
+&mailbox7 {
+	status = "okay";
+};
+
+&mailbox8 {
+	status = "okay";
+};
+
+&mailbox9 {
+	status = "okay";
+};
+
+&mailbox10 {
+	status = "okay";
+};
+
+&mailbox11 {
+	status = "okay";
+};
+
+&mailbox12 {
+	status = "okay";
+};
+
+&mailbox13 {
+	status = "okay";
+};
+
+&cpu_alert0 {
+	temperature = <55000>; /* milliCelsius */
+};
+
+&cpu_crit {
+	temperature = <85000>; /* milliCelsius */
+};
+
+&gpu_crit {
+	temperature = <85000>; /* milliCelsius */
+};
+
+&core_crit {
+	temperature = <85000>; /* milliCelsius */
+};
+
+&dspeve_crit {
+	temperature = <85000>; /* milliCelsius */
+};
+
+&iva_crit {
+	temperature = <85000>; /* milliCelsius */
+};
+
+&sata {
+	status = "disabled";
+};
+
+&sata_phy {
+	status = "disabled";
+};
+
+/* bluetooth */
+&uart6 {
+	status = "okay";
+};
+
+/* cape header stuff */
+&i2c4 {
+	status = "okay";
+	clock-frequency = <100000>;
+	symlink = "bone-i2c2";
+};
+
+/* thermal hacks */
+&cpu0_opp_table {
+	opp_slow-400000000 {
+		opp-hz = /bits/ 64 <400000000>;
+		opp-microvolt = <860000 850000 1100000>,
+				<860000 850000 1100000>;
+		opp-supported-hw = <0xFF 0x01>;
+		opp-suspend;
+	};
+};
+
+#include "dra7-ipu-common-early-boot.dtsi"
diff -Naur --no-dereference a/arch/arm/boot/dts/am5729-beagleboneai-roboticscape.dts b/arch/arm/boot/dts/am5729-beagleboneai-roboticscape.dts
--- a/arch/arm/boot/dts/am5729-beagleboneai-roboticscape.dts	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am5729-beagleboneai-roboticscape.dts	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,242 @@
+/*
+ * Copyright (C) 2014-2019 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include "am5729-beagleboneai.dts"
+
+/ {
+	chosen {
+		base_dtb = "am5729-beagleboneai-roboticscape.dts";
+		base_dtb_timestamp = __TIMESTAMP__;
+	};
+
+	rcleds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&cape_pins_rc>;
+
+		red_led {
+			label = "red";
+			gpios = <&gpio6 5 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		green_led {
+			label = "green";
+			gpios = <&gpio6 6 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		batt_1_led {
+			label = "bat25";
+			gpios = <&gpio8 18 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		batt_2_led {
+			label = "bat50";
+			gpios = <&gpio4 9 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		batt_3_led {
+			label = "bat75";
+			gpios = <&gpio4 28 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		batt_4_led {
+			label = "bat100";
+			gpios = <&gpio4 13 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+	};
+};
+
+&ocp {
+	pruss1_shmem: pruss_shmem@4b200000 {
+		status = "okay";
+		compatible = "ti,pruss-shmem";
+		reg = <0x4b200000 0x020000>;
+	};
+
+	pruss2_shmem: pruss_shmem@4b280000 {
+		status = "okay";
+		compatible = "ti,pruss-shmem";
+		reg = <0x4b280000 0x020000>;
+	};
+};
+
+&cape_pins {
+	status = "disabled";
+};
+
+&dra7_pmx_core {
+	cape_pins_rc: cape_pins_rc {
+		pinctrl-single,pins = <
+			DRA7XX_CORE_IOPAD(0x379C, MUX_MODE15) /* AB8: P8.3: mmc3_dat6.off */
+			DRA7XX_CORE_IOPAD(0x37A0, MUX_MODE15) /* AB5: P8.4: mmc3_dat7.off */
+			DRA7XX_CORE_IOPAD(0x378C, MUX_MODE15) /* AC9: P8.5: mmc3_dat2.off */
+			DRA7XX_CORE_IOPAD(0x3790, MUX_MODE15) /* AC3: P8.6: mmc3_dat3.off */
+			DRA7XX_CORE_IOPAD(0x36EC, PIN_OUTPUT | MUX_MODE14) /* G14: mcasp1_axr14.gpio6_5 - P8_7 - RED */
+			DRA7XX_CORE_IOPAD(0x36F0, PIN_OUTPUT | MUX_MODE14) /* F14: mcasp1_axr15.gpio6_6 - P8_8 - GREEN */
+			DRA7XX_CORE_IOPAD(0x3698, PIN_INPUT_PULLUP | MUX_MODE14) /* E17: P8.9: xref_clk1.gpio6_18 - PAUSE */
+			DRA7XX_CORE_IOPAD(0x36E8, PIN_INPUT_PULLUP | MUX_MODE14) /* A13: P8.10: mcasp1_axr13.gpio6_4 - MODE */
+			DRA7XX_CORE_IOPAD(0x3510, PIN_INPUT | MUX_MODE10) /* AH4: P8.11: vin1a_d7.eQEP2B_in - QEP2B */
+			DRA7XX_CORE_IOPAD(0x350C, PIN_INPUT | MUX_MODE10) /* AG6: P8.12: vin1a_d6.eQEP2A_in - QEP2A */
+			DRA7XX_CORE_IOPAD(0x3590, PIN_OUTPUT | MUX_MODE10) /* D3: P8.13: vin2a_d10.ehrpwm2B - PWM2B */
+			DRA7XX_CORE_IOPAD(0x3598, PIN_OUTPUT | MUX_MODE14) /* D5: vin2a_d12.gpio4_13 - P8_14 - BAT100 */
+			DRA7XX_CORE_IOPAD(0x3570, MUX_MODE15) /* D1: P8.15a: vin2a_d2.off */
+			DRA7XX_CORE_IOPAD(0x35B4, PIN_INPUT | MUX_MODE12) /* A3: P8.15b: vin2a_d19.pr1_pru1_gpi16 - QEP4B */
+			DRA7XX_CORE_IOPAD(0x35BC, PIN_INPUT | MUX_MODE12) /* B4: P8.16: vin2a_d21.pr1_pru1_gpi18 - QEP4A */
+			DRA7XX_CORE_IOPAD(0x3624, PIN_OUTPUT | MUX_MODE14) /* A7: vout1_d18.gpio8_18 - P8_17 - BAT25 */
+			DRA7XX_CORE_IOPAD(0x3588, PIN_OUTPUT | MUX_MODE14) /* F5: vin2a_d8.gpio4_9 - P8_18 - BAT50 */
+			DRA7XX_CORE_IOPAD(0x358C, PIN_OUTPUT | MUX_MODE10) /* E6: P8.19: vin2a_d9.ehrpwm2A - PWM2A */
+			DRA7XX_CORE_IOPAD(0x3780, MUX_MODE15) /* AC4: P8.20: mmc3_cmd.off */
+			DRA7XX_CORE_IOPAD(0x377C, MUX_MODE15) /* AD4: P8.21: mmc3_clk.off */
+			DRA7XX_CORE_IOPAD(0x3798, MUX_MODE15) /* AD6: P8.22: mmc3_dat5.off */
+			DRA7XX_CORE_IOPAD(0x3794, MUX_MODE15) /* AC8: P8.23: mmc3_dat4.off */
+			DRA7XX_CORE_IOPAD(0x3788, MUX_MODE15) /* AC6: P8.24: mmc3_dat1.off */
+			DRA7XX_CORE_IOPAD(0x3784, MUX_MODE15) /* AC7: P8.25: mmc3_dat0.off */
+			DRA7XX_CORE_IOPAD(0x35B8, PIN_OUTPUT | MUX_MODE14) /* B3: vin2a_d20.gpio4_28 - P8_26 - BAT75 */
+			DRA7XX_CORE_IOPAD(0x35D8, MUX_MODE15) /* E11: P8.27a: vout1_vsync.off */
+			DRA7XX_CORE_IOPAD(0x3628, PIN_OUTPUT | MUX_MODE13) /* A8: P8.27b: vout1_d19.pr2_pru0_gpo16 - SVO1 */
+			DRA7XX_CORE_IOPAD(0x35C8, MUX_MODE15) /* D11: P8.28a: vout1_clk.off */
+			DRA7XX_CORE_IOPAD(0x362C, PIN_OUTPUT | MUX_MODE13) /* C9: P8.28b: vout1_d20.pr2_pru0_gpo17 - SVO2 */
+			DRA7XX_CORE_IOPAD(0x35D4, MUX_MODE15) /* C11: P8.29a: vout1_hsync.off */
+			DRA7XX_CORE_IOPAD(0x3630, PIN_OUTPUT | MUX_MODE13) /* A9: P8.29b: vout1_d21.pr2_pru0_gpo18 - SVO3 */
+			DRA7XX_CORE_IOPAD(0x35CC, MUX_MODE15) /* B10: P8.30a: vout1_de.off */
+			DRA7XX_CORE_IOPAD(0x3634, PIN_OUTPUT | MUX_MODE13) /* B9: P8.30b: vout1_d22.pr2_pru0_gpo19 - SVO4 */
+			DRA7XX_CORE_IOPAD(0x3614, MUX_MODE15) /* C8: P8.31a: vout1_d14.off */
+			DRA7XX_CORE_IOPAD(0x373C, MUX_MODE15) /* G16: P8.31b: mcasp4_axr0.off */
+			DRA7XX_CORE_IOPAD(0x3618, MUX_MODE15) /* C7: P8.32a: vout1_d15.off */
+			DRA7XX_CORE_IOPAD(0x3740, MUX_MODE15) /* D17: P8.32b: mcasp4_axr1.off */
+			DRA7XX_CORE_IOPAD(0x3610, MUX_MODE15) /* C6: P8.33a: vout1_d13.off */
+			DRA7XX_CORE_IOPAD(0x34E8, PIN_INPUT | MUX_MODE10) /* AF9: P8.33b: vin1a_fld0.eQEP1B_in - QEP1B */
+			DRA7XX_CORE_IOPAD(0x3608, PIN_OUTPUT | MUX_MODE14) /* D8: P8.34a: vout1_d11.gpio8_11 - MDIR2B */
+			DRA7XX_CORE_IOPAD(0x3564, MUX_MODE15) /* G6: P8.34b: vin2a_vsync0.off */
+			DRA7XX_CORE_IOPAD(0x360C, MUX_MODE15) /* A5: P8.35a: vout1_d12.off */
+			DRA7XX_CORE_IOPAD(0x34E4, PIN_INPUT | MUX_MODE10) /* AD9: P8.35b: vin1a_de0.eQEP1A_in - QEP1A */
+			DRA7XX_CORE_IOPAD(0x3604, PIN_OUTPUT | MUX_MODE14) /* D7: P8.36a: vout1_d10.gpio8_10 - SERVO_EN */
+			DRA7XX_CORE_IOPAD(0x3568, MUX_MODE15) /* F2: P8.36b: vin2a_d0.off */
+			DRA7XX_CORE_IOPAD(0x35FC, MUX_MODE15) /* E8: P8.37a: vout1_d8.off */
+			DRA7XX_CORE_IOPAD(0x3738, PIN_OUTPUT | MUX_MODE3) /* A21: P8.37b: mcasp4_fsx.uart8_txd - UART5_TX */
+			DRA7XX_CORE_IOPAD(0x3600, MUX_MODE15) /* D9: P8.38a: vout1_d9.off */
+			DRA7XX_CORE_IOPAD(0x3734, PIN_INPUT | MUX_MODE3) /* C18: P8.38b: mcasp4_aclkx.uart8_rxd - UART5_RX */
+			DRA7XX_CORE_IOPAD(0x35F4, PIN_OUTPUT | MUX_MODE13) /* F8: P8.39: vout1_d6.pr2_pru0_gpo3 - SVO5 */
+			DRA7XX_CORE_IOPAD(0x35F8, PIN_OUTPUT | MUX_MODE13) /* E7: P8.40: vout1_d7.pr2_pru0_gpo4 - SVO6 */
+			DRA7XX_CORE_IOPAD(0x35EC, PIN_OUTPUT | MUX_MODE13) /* E9: P8.41: vout1_d4.pr2_pru0_gpo1 - SVO7 */
+			DRA7XX_CORE_IOPAD(0x35F0, PIN_OUTPUT | MUX_MODE13) /* F9: P8.42: vout1_d5.pr2_pru0_gpo2 - SVO8 */
+			DRA7XX_CORE_IOPAD(0x35E4, PIN_OUTPUT | MUX_MODE14 ) /* F10: P8.43: vout1_d2.gpio8_2 - MDIR3B */
+			DRA7XX_CORE_IOPAD(0x35E8, PIN_OUTPUT | MUX_MODE14 ) /* G11: P8.44: vout1_d3.gpio8_3 - MDIR3A */
+			DRA7XX_CORE_IOPAD(0x35DC, PIN_OUTPUT | MUX_MODE14 ) /* F11: P8.45a: vout1_d0.gpio8_0 - MDIR4A */
+			DRA7XX_CORE_IOPAD(0x361C, MUX_MODE15) /* B7: P8.45b: vout1_d16.off */
+			DRA7XX_CORE_IOPAD(0x35E0, PIN_OUTPUT | MUX_MODE14 ) /* G10: P8.46a: vout1_d1.gpio8_1 - MDIR4B */
+			DRA7XX_CORE_IOPAD(0x3638, MUX_MODE15) /* A10: P8.46b: vout1_d23.off */
+			DRA7XX_CORE_IOPAD(0x372C, PIN_INPUT | MUX_MODE4) /* B19: P9.11a: mcasp3_axr0.uart5_rxd - UART4_RX */
+			DRA7XX_CORE_IOPAD(0x3620, MUX_MODE15) /* B8: P9.11b: vout1_d17.off */
+			DRA7XX_CORE_IOPAD(0x36AC, PIN_OUTPUT | MUX_MODE14) /* B14: P9.12: mcasp1_aclkr.gpio5_0 - MDIR1A */
+			DRA7XX_CORE_IOPAD(0x3730, PIN_OUTPUT | MUX_MODE14) /* C17: P9.13: mcasp3_axr1.off - MDIR1B */
+			DRA7XX_CORE_IOPAD(0x35AC, PIN_OUTPUT | MUX_MODE10) /* D6: P9.14: vin2a_d17.ehrpwm3A - PWM1A */
+			DRA7XX_CORE_IOPAD(0x3514, PIN_OUTPUT | MUX_MODE14) /* AG4: P9.15: vin1a_d8.gpio3_12 - MDIR2A */
+			DRA7XX_CORE_IOPAD(0x35B0, PIN_OUTPUT | MUX_MODE10) /* C5: P9.16: vin2a_d18.ehrpwm3B - PWM1B */
+			DRA7XX_CORE_IOPAD(0x37CC, PIN_INPUT | MUX_MODE14) /* B24: P9.17a: spi2_cs0.gpio7_17 */
+			DRA7XX_CORE_IOPAD(0x36B8, PIN_INPUT_PULLUP | SLEWCONTROL | MUX_MODE10) /* F12: P9.17b: mcasp1_axr1.i2c5_scl - I2C1_SCL */
+			DRA7XX_CORE_IOPAD(0x37C8, PIN_INPUT | MUX_MODE14) /* G17: P9.18a: spi2_d0.gpio7_16 */
+			DRA7XX_CORE_IOPAD(0x36B4, PIN_INPUT_PULLUP | SLEWCONTROL | MUX_MODE10) /* G12: P9.18b: mcasp1_axr0.i2c5_sda - I2C1_SDA */
+			DRA7XX_CORE_IOPAD(0x3440, PIN_INPUT | MUX_MODE7) /* R6: P9.19a: gpmc_a0.i2c4_scl - I2C2_SCL */
+			DRA7XX_CORE_IOPAD(0x357C, PIN_INPUT | MUX_MODE12 ) /* F4: P9.19b: vin2a_d5.pr1_pru1_gpi2 */
+			DRA7XX_CORE_IOPAD(0x3444, PIN_INPUT | MUX_MODE7) /* T9: P9.20a: gpmc_a1.i2c4_sda - I2C2_SDA */
+			DRA7XX_CORE_IOPAD(0x3578, PIN_INPUT | MUX_MODE12) /* D2: P9.20b: vin2a_d4.pr1_pru1_gpi1 */
+			DRA7XX_CORE_IOPAD(0x34F0, MUX_MODE15) /* AF8: P9.21a: vin1a_vsync0.off */
+			DRA7XX_CORE_IOPAD(0x37C4, PIN_OUTPUT | MUX_MODE1) /* B22: P9.21b: spi2_d1.uart3_txd - UART2_TX */
+			DRA7XX_CORE_IOPAD(0x369C, MUX_MODE15) /* B26: P9.22a: xref_clk2.off */
+			DRA7XX_CORE_IOPAD(0x37C0, PIN_INPUT | MUX_MODE1) /* A26: P9.22b: spi2_sclk.uart3_rxd - UART2_RX */
+			DRA7XX_CORE_IOPAD(0x37B4, PIN_OUTPUT | MUX_MODE14) /* A22: P9.23: spi1_cs1.gpio7_11 - SPI1_SS2 */
+			DRA7XX_CORE_IOPAD(0x368C, PIN_OUTPUT | MUX_MODE3) /* F20: P9.24: gpio6_15.uart10_txd - UART1_TX */
+			DRA7XX_CORE_IOPAD(0x3694, PIN_INPUT | MUX_MODE14) /* D18: P9.25: xref_clk0.gpio6_17 - IMU_INT */
+			DRA7XX_CORE_IOPAD(0x3688, PIN_INPUT | MUX_MODE3) /* E21: P9.26a: gpio6_14.uart10_rxd - UART1_RX */
+			DRA7XX_CORE_IOPAD(0x3544, MUX_MODE15) /* AE2: P9.26b: vin1a_d20.off */
+			DRA7XX_CORE_IOPAD(0x35A0, PIN_INPUT | MUX_MODE10) /* C3: P9.27a: vin2a_d14.eQEP3B_in - QEP0B */
+			DRA7XX_CORE_IOPAD(0x36B0, MUX_MODE15) /* J14: P9.27b: mcasp1_fsr.off */
+			DRA7XX_CORE_IOPAD(0x36E0, PIN_OUTPUT | MUX_MODE3) /* A12: P9.28: mcasp1_axr11.spi3_cs0 - SPI1_SS1 */
+			DRA7XX_CORE_IOPAD(0x36D8, PIN_INPUT | MUX_MODE3) /* A11: P9.29a: mcasp1_axr9.spi3_d1 - SPI1_MISO */
+			DRA7XX_CORE_IOPAD(0x36A8, MUX_MODE15) /* D14: P9.29b: mcasp1_fsx.off */
+			DRA7XX_CORE_IOPAD(0x36DC, PIN_OUTPUT | MUX_MODE3) /* B13: P9.30: mcasp1_axr10.spi3_d0 - SPI1_MOSI */
+			DRA7XX_CORE_IOPAD(0x36D4, PIN_OUTPUT | MUX_MODE3) /* B12: P9.31a: mcasp1_axr8.spi3_sclk - SPI1_SCK */
+			DRA7XX_CORE_IOPAD(0x36A4, MUX_MODE15) /* C14: P9.31b: mcasp1_aclkx.off */
+			DRA7XX_CORE_IOPAD(0x36A0, PIN_OUTPUT | MUX_MODE14) /* C23: P9.41a: xref_clk3.gpio6_20 - MOT_STBY */
+			DRA7XX_CORE_IOPAD(0x3580, MUX_MODE15) /* C1: P9.41b: vin2a_d6.off */
+			DRA7XX_CORE_IOPAD(0x36E4, MUX_MODE15) /* E14: P9.42a: mcasp1_axr12.off */
+			DRA7XX_CORE_IOPAD(0x359C, PIN_INPUT | MUX_MODE10) /* C2: P9.42b: vin2a_d13.eQEP3A_in - QEP0A */
+		>;
+	};
+};
+
+&i2c5 {
+	status = "okay";
+	clock-frequency = <100000>;
+};
+
+&epwmss0 {
+	status = "okay";
+};
+
+&epwmss1 {
+	status = "okay";
+};
+
+&epwmss2 {
+	status = "okay";
+};
+
+&ehrpwm1 {
+	status = "okay";
+};
+
+&ehrpwm2 {
+	status = "okay";
+};
+
+&uart8 {
+	status = "okay";
+};
+
+&uart5 {
+	status = "okay";
+};
+
+&uart3 {
+	status = "okay";
+};
+
+&uart10 {
+	status = "okay";
+};
+
+&mcspi1 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	status = "okay";
+
+	cs-gpios = <0>, <&gpio7 11 0>;
+
+	channel@0 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "spidev";
+		reg = <0>;
+		spi-max-frequency = <24000000>;
+	};
+
+	channel@1 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "spidev";
+		reg = <1>;
+		spi-max-frequency = <24000000>;
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am572x-idk-common.dtsi b/arch/arm/boot/dts/am572x-idk-common.dtsi
--- a/arch/arm/boot/dts/am572x-idk-common.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am572x-idk-common.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -6,6 +6,7 @@
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/interrupt-controller/irq.h>
 #include "am57xx-idk-common.dtsi"
+#include "dra74-ipu-dsp-common.dtsi"
 
 / {
 	memory@0 {
@@ -13,6 +14,40 @@
 		reg = <0x0 0x80000000 0x0 0x80000000>;
 	};
 
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		ipu2_memory_region: ipu2-memory@95800000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x95800000 0x0 0x3800000>;
+			reusable;
+			status = "okay";
+		};
+
+		dsp1_memory_region: dsp1-memory@99000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x99000000 0x0 0x4000000>;
+			reusable;
+			status = "okay";
+		};
+
+		ipu1_memory_region: ipu1-memory@9d000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x9d000000 0x0 0x2000000>;
+			reusable;
+			status = "okay";
+		};
+
+		dsp2_memory_region: dsp2-memory@9f000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x9f000000 0x0 0x800000>;
+			reusable;
+			status = "okay";
+		};
+	};
+
 	status-leds {
 		compatible = "gpio-leds";
 		cpu0-led {
@@ -71,26 +106,39 @@
 	gpios = <&gpio3 23 GPIO_ACTIVE_HIGH>;
 };
 
-&pcie1_ep {
-	gpios = <&gpio3 23 GPIO_ACTIVE_HIGH>;
+&ipu2 {
+	status = "okay";
+	memory-region = <&ipu2_memory_region>;
 };
 
-&mailbox5 {
+&ipu1 {
 	status = "okay";
-	mbox_ipu1_ipc3x: mbox_ipu1_ipc3x {
-		status = "okay";
-	};
-	mbox_dsp1_ipc3x: mbox_dsp1_ipc3x {
-		status = "okay";
-	};
+	memory-region = <&ipu1_memory_region>;
 };
 
-&mailbox6 {
+&dsp1 {
 	status = "okay";
-	mbox_ipu2_ipc3x: mbox_ipu2_ipc3x {
-		status = "okay";
-	};
-	mbox_dsp2_ipc3x: mbox_dsp2_ipc3x {
-		status = "okay";
+	memory-region = <&dsp1_memory_region>;
+};
+
+&dsp2 {
+	status = "okay";
+	memory-region = <&dsp2_memory_region>;
+};
+
+&ov2659_1 {
+	remote-endpoint = <&vin4b_ep>;
+};
+
+&vin4b {
+	vin4b_ep: endpoint@3 {
+		slave-mode;
+		remote-endpoint = <&ov2659_1>;
 	};
 };
+
+&vip2 {
+	status = "okay";
+};
+
+#include "dra7-ipu-common-early-boot.dtsi"
diff -Naur --no-dereference a/arch/arm/boot/dts/am572x-idk.dts b/arch/arm/boot/dts/am572x-idk.dts
--- a/arch/arm/boot/dts/am572x-idk.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am572x-idk.dts	2020-06-19 21:00:31.404227715 -0400
@@ -32,6 +32,8 @@
 	pinctrl-2 = <&mmc2_pins_ddr_rev20>;
 };
 
-&cpu0 {
-	vdd-supply = <&smps12_reg>;
+&pruss2_mdio {
+	reset-gpios = <&gpio5 8 GPIO_ACTIVE_LOW>,
+		      <&gpio5 9 GPIO_ACTIVE_LOW>;
+	reset-delay-us = <2>;	/* PHY datasheet states 1uS min */
 };
diff -Naur --no-dereference a/arch/arm/boot/dts/am574x-idk.dts b/arch/arm/boot/dts/am574x-idk.dts
--- a/arch/arm/boot/dts/am574x-idk.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am574x-idk.dts	2020-06-19 21:00:31.404227715 -0400
@@ -6,7 +6,7 @@
 
 /dts-v1/;
 
-#include "dra76x.dtsi"
+#include "dra74x-p.dtsi"
 #include "dra7-mmc-iodelay.dtsi"
 #include "dra76x-mmc-iodelay.dtsi"
 #include "am572x-idk-common.dtsi"
@@ -35,3 +35,13 @@
 	pinctrl-1 = <&mmc2_pins_default>;
 	pinctrl-2 = <&mmc2_pins_default>;
 };
+
+&emif1 {
+	status = "okay";
+};
+
+&pruss2_mdio {
+	reset-gpios = <&gpio5 8 GPIO_ACTIVE_LOW>,
+		      <&gpio5 9 GPIO_ACTIVE_LOW>;
+	reset-delay-us = <2>;   /* PHY datasheet states 1uS min */
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi b/arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi
--- a/arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -1,15 +1,13 @@
+// SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (C) 2014-2016 Texas Instruments Incorporated - http://www.ti.com/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 /dts-v1/;
 
 #include "dra74x.dtsi"
 #include "am57xx-commercial-grade.dtsi"
 #include "dra74x-mmc-iodelay.dtsi"
+#include "dra74-ipu-dsp-common.dtsi"
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/interrupt-controller/irq.h>
 
@@ -21,6 +19,9 @@
 		rtc1 = &tps659038_rtc;
 		rtc2 = &rtc;
 		display0 = &hdmi0;
+
+		sound0 = &sound0;
+		sound1 = &hdmi;
 	};
 
 	chosen {
@@ -32,6 +33,61 @@
 		reg = <0x0 0x80000000 0x0 0x80000000>;
 	};
 
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		ipu2_memory_region: ipu2-memory@95800000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x95800000 0x0 0x3800000>;
+			reusable;
+			status = "okay";
+		};
+
+		dsp1_memory_region: dsp1-memory@99000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x99000000 0x0 0x4000000>;
+			reusable;
+			status = "okay";
+		};
+
+		ipu1_memory_region: ipu1-memory@9d000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x9d000000 0x0 0x2000000>;
+			reusable;
+			status = "okay";
+		};
+
+		dsp2_memory_region: dsp2-memory@9f000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x9f000000 0x0 0x800000>;
+			reusable;
+			status = "okay";
+		};
+	};
+
+	main_12v0: fixedregulator-main_12v0 {
+		/* main supply */
+		compatible = "regulator-fixed";
+		regulator-name = "main_12v0";
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	evm_5v0: fixedregulator-evm_5v0 {
+		/* Output of TPS54531D */
+		compatible = "regulator-fixed";
+		regulator-name = "evm_5v0";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&main_12v0>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
 	vdd_3v3: fixedregulator-vdd_3v3 {
 		compatible = "regulator-fixed";
 		regulator-name = "vdd_3v3";
@@ -167,6 +223,10 @@
 	};
 };
 
+//&bb2d {
+//	status = "okay";
+//};
+
 &i2c1 {
 	status = "okay";
 	clock-frequency = <400000>;
@@ -528,12 +588,20 @@
        };
 };
 
+&gpu {
+	status = "ok";
+};
+
 &dss {
 	status = "ok";
 
 	vdda_video-supply = <&ldoln_reg>;
 };
 
+&bb2d {
+	status = "okay";
+};
+
 &hdmi {
 	status = "ok";
 	vdda-supply = <&ldo4_reg>;
@@ -550,10 +618,6 @@
 	gpios = <&gpio2 8 GPIO_ACTIVE_LOW>;
 };
 
-&pcie1_ep {
-	gpios = <&gpio2 8 GPIO_ACTIVE_LOW>;
-};
-
 &mcasp3 {
 	#sound-dai-cells = <0>;
 	assigned-clocks = <&l4per_clkctrl DRA7_MCASP3_CLKCTRL 24>;
@@ -570,22 +634,40 @@
 	rx-num-evt = <32>;
 };
 
-&mailbox5 {
+&pruss_soc_bus1 {
 	status = "okay";
-	mbox_ipu1_ipc3x: mbox_ipu1_ipc3x {
-		status = "okay";
-	};
-	mbox_dsp1_ipc3x: mbox_dsp1_ipc3x {
+
+	pruss1: pruss@4b200000 {
 		status = "okay";
 	};
 };
 
-&mailbox6 {
+&pruss_soc_bus2 {
 	status = "okay";
-	mbox_ipu2_ipc3x: mbox_ipu2_ipc3x {
-		status = "okay";
-	};
-	mbox_dsp2_ipc3x: mbox_dsp2_ipc3x {
+
+	pruss2: pruss@4b280000 {
 		status = "okay";
 	};
 };
+
+&ipu2 {
+	status = "okay";
+	memory-region = <&ipu2_memory_region>;
+};
+
+&ipu1 {
+	status = "okay";
+	memory-region = <&ipu1_memory_region>;
+};
+
+&dsp1 {
+	status = "okay";
+	memory-region = <&dsp1_memory_region>;
+};
+
+&dsp2 {
+	status = "okay";
+	memory-region = <&dsp2_memory_region>;
+};
+
+#include "dra7-ipu-common-early-boot.dtsi"
diff -Naur --no-dereference a/arch/arm/boot/dts/am57xx-beagle-x15.dts b/arch/arm/boot/dts/am57xx-beagle-x15.dts
--- a/arch/arm/boot/dts/am57xx-beagle-x15.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am57xx-beagle-x15.dts	2020-06-19 21:00:31.404227715 -0400
@@ -1,16 +1,19 @@
+// SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (C) 2014-2016 Texas Instruments Incorporated - http://www.ti.com/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include "am57xx-beagle-x15-common.dtsi"
+#include "am57xx-cmem.dtsi"
 
 / {
 	/* NOTE: This describes the "original" pre-production A2 revision */
 	model = "TI AM5728 BeagleBoard-X15";
+
+	chosen {
+		base_dtb = "am57xx-beagle-x15.dts";
+		base_dtb_timestamp = __TIMESTAMP__;
+	};
 };
 
 &tpd12s015 {
diff -Naur --no-dereference a/arch/arm/boot/dts/am57xx-beagle-x15-revb1.dts b/arch/arm/boot/dts/am57xx-beagle-x15-revb1.dts
--- a/arch/arm/boot/dts/am57xx-beagle-x15-revb1.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am57xx-beagle-x15-revb1.dts	2020-06-19 21:00:31.404227715 -0400
@@ -1,15 +1,18 @@
+// SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (C) 2014-2016 Texas Instruments Incorporated - http://www.ti.com/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include "am57xx-beagle-x15-common.dtsi"
+#include "am57xx-cmem.dtsi"
 
 / {
 	model = "TI AM5728 BeagleBoard-X15 rev B1";
+
+	chosen {
+		base_dtb = "am57xx-beagle-x15-revb1.dts";
+		base_dtb_timestamp = __TIMESTAMP__;
+	};
 };
 
 &tpd12s015 {
@@ -27,7 +30,7 @@
 };
 
 &mmc2 {
-	pinctrl-names = "default", "hs", "ddr_1_8v";
+	pinctrl-names = "default", "hs", "ddr_3_3v";
 	pinctrl-0 = <&mmc2_pins_default>;
 	pinctrl-1 = <&mmc2_pins_hs>;
 	pinctrl-2 = <&mmc2_pins_ddr_3_3v_rev11 &mmc2_iodelay_ddr_3_3v_rev11_conf>;
diff -Naur --no-dereference a/arch/arm/boot/dts/am57xx-beagle-x15-revc.dts b/arch/arm/boot/dts/am57xx-beagle-x15-revc.dts
--- a/arch/arm/boot/dts/am57xx-beagle-x15-revc.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am57xx-beagle-x15-revc.dts	2020-06-19 21:00:31.404227715 -0400
@@ -1,15 +1,18 @@
+// SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (C) 2014-2017 Texas Instruments Incorporated - http://www.ti.com/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include "am57xx-beagle-x15-common.dtsi"
+#include "am57xx-cmem.dtsi"
 
 / {
 	model = "TI AM5728 BeagleBoard-X15 rev C";
+
+	chosen {
+		base_dtb = "am57xx-beagle-x15-revc.dts";
+		base_dtb_timestamp = __TIMESTAMP__;
+	};
 };
 
 &tpd12s015 {
@@ -27,7 +30,7 @@
 };
 
 &mmc2 {
-	pinctrl-names = "default", "hs", "ddr_1_8v";
+	pinctrl-names = "default", "hs", "ddr_3_3v";
 	pinctrl-0 = <&mmc2_pins_default>;
 	pinctrl-1 = <&mmc2_pins_hs>;
 	pinctrl-2 = <&mmc2_pins_ddr_rev20>;
diff -Naur --no-dereference a/arch/arm/boot/dts/am57xx-cmem.dtsi b/arch/arm/boot/dts/am57xx-cmem.dtsi
--- a/arch/arm/boot/dts/am57xx-cmem.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/am57xx-cmem.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,43 @@
+/* https://git.yoctoproject.org/cgit/cgit.cgi/meta-ti/tree/recipes-kernel/linux/files/dra7xx/cmem.dtsi?h=thud */
+/* 0x0c000000 -> 0x18000000 for tild */
+
+/ {
+        reserved-memory {
+                #address-cells = <2>;
+                #size-cells = <2>;
+                ranges;
+
+                cmem_block_mem_0: cmem_block_mem@a0000000 {
+                        reg = <0x0 0xa0000000 0x0 0x18000000>;
+                        no-map;
+                        status = "okay";
+                };
+
+		cmem_block_mem_1_ocmc3: cmem_block_mem@40500000 {
+			reg = <0x0 0x40500000 0x0 0x100000>;
+			no-map;
+			status = "okay";
+		};
+        };
+
+        cmem {
+                compatible = "ti,cmem";
+                #address-cells = <1>;
+                #size-cells = <0>;
+
+		#pool-size-cells = <2>;
+
+                status = "okay";
+
+                cmem_block_0: cmem_block@0 {
+                        reg = <0>;
+                        memory-region = <&cmem_block_mem_0>;
+                        cmem-buf-pools = <1 0x0 0x18000000>;
+                };
+
+		cmem_block_1: cmem_block@1 {
+			reg = <1>;
+			memory-region = <&cmem_block_mem_1_ocmc3>;
+		};
+        };
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/am57xx-idk-common.dtsi b/arch/arm/boot/dts/am57xx-idk-common.dtsi
--- a/arch/arm/boot/dts/am57xx-idk-common.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/am57xx-idk-common.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -12,6 +12,9 @@
 	aliases {
 		rtc0 = &tps659038_rtc;
 		rtc1 = &rtc;
+		display0 = &hdmi0;
+		ethernet2 = &pruss2_emac0;
+		ethernet3 = &pruss2_emac1;
 	};
 
 	chosen {
@@ -48,6 +51,12 @@
 		regulator-boot-on;
 	};
 
+	src_clk_x1: src_clk_x1 {
+		#clock-cells = <0>;
+		compatible = "fixed-clock";
+		clock-frequency = <20000000>;
+	};
+
 	leds-iio {
 		status = "disabled";
 		compatible = "gpio-leds";
@@ -99,6 +108,84 @@
 			default-state = "off";
 		};
 	};
+
+	hdmi0: connector@0 {
+		compatible = "hdmi-connector";
+		label = "hdmi";
+
+		type = "a";
+
+		port {
+			hdmi_connector_in: endpoint {
+				remote-endpoint = <&tpd12s015_out>;
+			};
+		};
+	};
+
+	tpd12s015: encoder@0 {
+		compatible = "ti,tpd12s016", "ti,tpd12s015";
+
+		gpios = <0>, /* optional CT_CP_HPD */
+			<0>, /* optional LS_OE */
+			<&gpio7 12 GPIO_ACTIVE_HIGH>;	/* HPD */
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				tpd12s015_in: endpoint@0 {
+					remote-endpoint = <&hdmi_out>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				tpd12s015_out: endpoint@0 {
+					remote-endpoint = <&hdmi_connector_in>;
+				};
+			};
+		};
+	};
+
+	src_clk_x1: src_clk_x1 {
+		#clock-cells = <0>;
+		compatible = "fixed-clock";
+		clock-frequency = <20000000>;
+	};
+
+	/* Dual-MAC Ethernet application node on PRU-ICSS2 */
+	pruss2_eth: pruss2_eth {
+		compatible = "ti,am57-prueth";
+		prus = <&pru2_0>, <&pru2_1>;
+		firmware-name = "ti-pruss/am57xx-pru0-prueth-fw.elf",
+				"ti-pruss/am57xx-pru1-prueth-fw.elf";
+		sram = <&ocmcram1>;
+		interrupt-parent = <&pruss2_intc>;
+		mii-rt = <&pruss2_mii_rt>;
+		iep = <&pruss2_iep>;
+
+		pruss2_emac0: ethernet-mii0 {
+			phy-handle = <&pruss2_eth0_phy>;
+			phy-mode = "mii";
+			interrupts = <20>, <22>;
+			interrupt-names = "rx", "tx";
+			/* Filled in by bootloader */
+			local-mac-address = [00 00 00 00 00 00];
+		};
+
+		pruss2_emac1: ethernet-mii1 {
+			phy-handle = <&pruss2_eth1_phy>;
+			phy-mode = "mii";
+			interrupts = <21>, <23>;
+			interrupt-names = "rx", "tx";
+			/* Filled in by bootloader */
+			local-mac-address = [00 00 00 00 00 00];
+		};
+	};
 };
 
 &dra7_pmx_core {
@@ -338,6 +425,47 @@
 		gpio-controller;
 		#gpio-cells = <2>;
 	};
+
+	ov2659: ov2659@30 {
+		compatible = "ovti,ov2659";
+		reg = <0x30>;
+
+		clocks = <&src_clk_x1>;
+		clock-names = "xvclk";
+
+		pwrdn-gpios = <&gpio6 14 GPIO_ACTIVE_LOW>;
+
+		port {
+			ov2659_1: endpoint {
+				hsync-active = <1>;
+				vsync-active = <1>;
+				pclk-sample = <1>;
+				link-frequencies = /bits/ 64 <70000000>;
+			};
+		};
+	};
+
+	dsi_bridge: tc358778@0e {
+		compatible = "toshiba,tc358778", "toshiba,tc358768";
+		reg = <0x0e>;
+		status = "disabled";
+
+		clocks = <&src_clk_x1>;
+		clock-names = "refclk";
+
+		dsi_bridge_ports: ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				rgb_in: endpoint {
+					remote-endpoint = <&dpi_out>;
+					data-lines = <24>;
+				};
+			};
+		};
+	};
 };
 
 &mcspi3 {
@@ -479,3 +607,83 @@
 		};
 	};
 };
+
+&cpu0 {
+	vdd-supply = <&smps12_reg>;
+};
+
+&gpu {
+	status = "ok";
+};
+
+&hdmi {
+	status = "okay";
+	/*
+	 * XXX: Support AM572x-Rev 1.2a. this is wrong for AM571x-rev 1.3a,
+	 * AM572x-Rev1.3a - but thanks to always-on, they work.
+	 * TODO: SWITCH TO LDO4 once rev 1.2a is deprecated
+	 * (on rev 1.3a availability)
+	 */
+	vdda-supply = <&ldo3_reg>;
+
+	port {
+		hdmi_out: endpoint {
+			remote-endpoint = <&tpd12s015_in>;
+		};
+	};
+};
+
+&dss {
+	status = "okay";
+
+	vdda_video-supply = <&ldoln_reg>;
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+
+			dpi_out: endpoint {
+				remote-endpoint = <&rgb_in>;
+				data-lines = <24>;
+			};
+		};
+	};
+};
+
+&pruss_soc_bus1 {
+	status = "okay";
+
+	pruss1: pruss@4b200000 {
+		status = "okay";
+	};
+};
+
+&pruss_soc_bus2 {
+	status = "okay";
+
+	pruss2: pruss@4b280000 {
+		status = "okay";
+	};
+};
+
+&pruss2_mdio {
+	status = "okay";
+	pruss2_eth0_phy: ethernet-phy@0 {
+		reg = <0>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <30 IRQ_TYPE_EDGE_FALLING>;
+	};
+
+	pruss2_eth1_phy: ethernet-phy@1 {
+		reg = <1>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <31 IRQ_TYPE_EDGE_FALLING>;
+	};
+};
+
+&bb2d {
+	status = "ok";
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/da850.dtsi b/arch/arm/boot/dts/da850.dtsi
--- a/arch/arm/boot/dts/da850.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/da850.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -13,7 +13,9 @@
 	#address-cells = <1>;
 	#size-cells = <1>;
 	chosen { };
-	aliases { };
+	aliases {
+		rproc0 = &dsp;
+	};
 
 	memory@c0000000 {
 		device_type = "memory";
diff -Naur --no-dereference a/arch/arm/boot/dts/da850-lcdk.dts b/arch/arm/boot/dts/da850-lcdk.dts
--- a/arch/arm/boot/dts/da850-lcdk.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/da850-lcdk.dts	2020-06-19 21:00:31.404227715 -0400
@@ -74,12 +74,16 @@
 		simple-audio-card,name = "DA850-OMAPL138 LCDK";
 		simple-audio-card,widgets =
 			"Line", "Line In",
-			"Line", "Line Out";
+			"Line", "Line Out",
+			"Microphone", "Mic Jack";
 		simple-audio-card,routing =
 			"LINE1L", "Line In",
 			"LINE1R", "Line In",
 			"Line Out", "LLOUT",
-			"Line Out", "RLOUT";
+			"Line Out", "RLOUT",
+			"MIC3L", "Mic Jack",
+			"MIC3R", "Mic Jack",
+			"Mic Jack", "Mic Bias";
 		simple-audio-card,format = "dsp_b";
 		simple-audio-card,bitclock-master = <&link0_codec>;
 		simple-audio-card,frame-master = <&link0_codec>;
@@ -250,6 +254,8 @@
 		#sound-dai-cells = <0>;
 		compatible = "ti,tlv320aic3106";
 		reg = <0x18>;
+		adc-settle-ms = <40>;
+		ai3x-micbias-vg = <1>;		/* 2.0V */
 		status = "okay";
 
 		/* Regulators */
diff -Naur --no-dereference a/arch/arm/boot/dts/dra71-evm.dts b/arch/arm/boot/dts/dra71-evm.dts
--- a/arch/arm/boot/dts/dra71-evm.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/dra71-evm.dts	2020-06-19 21:00:31.404227715 -0400
@@ -20,6 +20,33 @@
 		reg = <0x0 0x80000000 0x0 0x80000000>; /* 2GB */
 	};
 
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		ipu2_memory_region: ipu2-memory@95800000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x95800000 0x0 0x3800000>;
+			reusable;
+			status = "okay";
+		};
+
+		dsp1_memory_region: dsp1-memory@99000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x99000000 0x0 0x4000000>;
+			reusable;
+			status = "okay";
+		};
+
+		ipu1_memory_region: ipu1-memory@9d000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x9d000000 0x0 0x2000000>;
+			reusable;
+			status = "okay";
+		};
+	};
+
 	vpo_sd_1v8_3v3: gpio-regulator-TPS74801 {
 		compatible = "regulator-gpio";
 
@@ -88,6 +115,8 @@
 				regulator-name = "lp8733-ldo0";
 				regulator-min-microvolt = <3300000>;
 				regulator-max-microvolt = <3300000>;
+				regulator-boot-on;
+				regulator-always-on;
 			};
 
 			lp8733_ldo1_reg: ldo1 {
@@ -150,6 +179,12 @@
 	};
 };
 
+&csi2_cam0 {
+	clock-lanes = <0>;
+	data-lanes = <1 2>;
+	remote-endpoint = <&csi2_phy0>;
+};
+
 &pcf_lcd {
 	interrupt-parent = <&gpio7>;
 	interrupts = <31 IRQ_TYPE_EDGE_FALLING>;
@@ -273,3 +308,20 @@
 &extcon_usb2 {
 	vbus-gpio = <&pcf_lcd 15 GPIO_ACTIVE_HIGH>;
 };
+
+&ipu2 {
+	status = "okay";
+	memory-region = <&ipu2_memory_region>;
+};
+
+&ipu1 {
+	status = "okay";
+	memory-region = <&ipu1_memory_region>;
+};
+
+&dsp1 {
+	status = "okay";
+	memory-region = <&dsp1_memory_region>;
+};
+
+#include "dra7-ipu-common-early-boot.dtsi"
diff -Naur --no-dereference a/arch/arm/boot/dts/dra72-evm-common.dtsi b/arch/arm/boot/dts/dra72-evm-common.dtsi
--- a/arch/arm/boot/dts/dra72-evm-common.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/dra72-evm-common.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -8,6 +8,7 @@
 /dts-v1/;
 
 #include "dra72x.dtsi"
+#include "dra7-ipu-dsp-common.dtsi"
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/clk/ti-dra7-atl.h>
 
@@ -16,6 +17,9 @@
 
 	aliases {
 		display0 = &hdmi0;
+
+		sound0 = &sound0;
+		sound1 = &hdmi;
 	};
 
 	chosen {
@@ -182,6 +186,12 @@
 		};
 	};
 
+	clk_ov10633_fixed: clk_ov10633_fixed {
+		#clock-cells = <0>;
+		compatible = "fixed-clock";
+		clock-frequency = <24000000>;
+	};
+
 	vmmcwl_fixed: fixedregulator-mmcwl {
 		compatible = "regulator-fixed";
 		regulator-name = "vmmcwl_fixed";
@@ -272,6 +282,54 @@
 			line-name = "vin6_sel_s0";
 		};
 	};
+
+	ov10633@37 {
+		compatible = "ovti,ov10633";
+		reg = <0x37>;
+
+		clocks = <&clk_ov10633_fixed>;
+		clock-names = "xvclk";
+
+		mux-gpios = <&pcf_hdmi 2	GPIO_ACTIVE_HIGH>, /* VIN2_S0 */
+			    <&pcf_hdmi 6	GPIO_ACTIVE_LOW>; /* VIN2_S2 */
+		port {
+			onboardLI: endpoint {
+				remote-endpoint = <&vin2a_ep>;
+				hsync-active = <1>;
+				vsync-active = <1>;
+				pclk-sample = <0>;
+			};
+		};
+	};
+
+	gpio_csi2_adap: tca6416@20 {
+		status = "okay";
+		compatible = "ti,tca6416";
+		reg = <0x20>;
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
+
+	ov490@24 {
+		compatible = "ovti,ov490";
+		reg = <0x24>;
+
+		mux-gpios = <&gpio_csi2_adap 0	GPIO_ACTIVE_LOW>,
+						/* CSI2_SEL_I2C_CLK */
+			    <&gpio_csi2_adap 1	GPIO_ACTIVE_HIGH>,
+						/* CSI2_SEL_REF_CLK */
+			    <&gpio_csi2_adap 3	GPIO_ACTIVE_HIGH>,
+						/* CSI2_CAM0_RESETn */
+			    <&gpio_csi2_adap 4	GPIO_ACTIVE_LOW>;
+						/* CSI2_CAM0_PWR_DWN */
+		port {
+			csi2_cam0: endpoint@0 {
+				clock-lanes = <0>;
+				data-lanes = <1 2 3 4>;
+				remote-endpoint = <&csi2_phy0>;
+			};
+		};
+	};
 };
 
 &uart1 {
@@ -514,10 +572,18 @@
 	};
 };
 
+&gpu {
+	status = "ok";
+};
+
 &dss {
 	status = "ok";
 };
 
+&bb2d {
+	status = "okay";
+};
+
 &hdmi {
 	status = "ok";
 
@@ -528,6 +594,28 @@
 	};
 };
 
+&vip1 {
+	status = "okay";
+};
+
+&vin2a {
+	vin2a_ep: endpoint@0 {
+		slave-mode;
+		remote-endpoint = <&onboardLI>;
+	};
+};
+
+&cal {
+	status = "okay";
+};
+
+&csi2_0 {
+	csi2_phy0: endpoint@0 {
+		slave-mode;
+		remote-endpoint = <&csi2_cam0>;
+	};
+};
+
 &atl {
 	assigned-clocks = <&abe_dpll_sys_clk_mux>,
 			  <&atl_clkctrl DRA7_ATL_CLKCTRL 26>,
@@ -563,23 +651,6 @@
 	rx-num-evt = <32>;
 };
 
-&mailbox5 {
-	status = "okay";
-	mbox_ipu1_ipc3x: mbox_ipu1_ipc3x {
-		status = "okay";
-	};
-	mbox_dsp1_ipc3x: mbox_dsp1_ipc3x {
-		status = "okay";
-	};
-};
-
-&mailbox6 {
-	status = "okay";
-	mbox_ipu2_ipc3x: mbox_ipu2_ipc3x {
-		status = "okay";
-	};
-};
-
 &pcie1_rc {
 	status = "okay";
 };
diff -Naur --no-dereference a/arch/arm/boot/dts/dra72-evm.dts b/arch/arm/boot/dts/dra72-evm.dts
--- a/arch/arm/boot/dts/dra72-evm.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/dra72-evm.dts	2020-06-19 21:00:31.404227715 -0400
@@ -15,6 +15,33 @@
 		reg = <0x0 0x80000000 0x0 0x40000000>; /* 1024 MB */
 	};
 
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		ipu2_memory_region: ipu2-memory@95800000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x95800000 0x0 0x3800000>;
+			reusable;
+			status = "okay";
+		};
+
+		dsp1_memory_region: dsp1-memory@99000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x99000000 0x0 0x4000000>;
+			reusable;
+			status = "okay";
+		};
+
+		ipu1_memory_region: ipu1-memory@9d000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x9d000000 0x0 0x2000000>;
+			reusable;
+			status = "okay";
+		};
+	};
+
 	evm_1v8_sw: fixedregulator-evm_1v8 {
 		compatible = "regulator-fixed";
 		regulator-name = "evm_1v8";
@@ -40,6 +67,11 @@
 	vdda-supply = <&ldo3_reg>;
 };
 
+&pcf_lcd {
+	interrupt-parent = <&gpio6>;
+	interrupts = <11 IRQ_TYPE_EDGE_FALLING>;
+};
+
 &pcf_gpio_21 {
 	interrupt-parent = <&gpio6>;
 	interrupts = <11 IRQ_TYPE_EDGE_FALLING>;
@@ -75,3 +107,20 @@
 	pinctrl-3 = <&mmc2_pins_hs200 &mmc2_iodelay_hs200_rev10_conf>;
 	vmmc-supply = <&evm_1v8_sw>;
 };
+
+&ipu2 {
+	status = "okay";
+	memory-region = <&ipu2_memory_region>;
+};
+
+&ipu1 {
+	status = "okay";
+	memory-region = <&ipu1_memory_region>;
+};
+
+&dsp1 {
+	status = "okay";
+	memory-region = <&dsp1_memory_region>;
+};
+
+#include "dra7-ipu-common-early-boot.dtsi"
diff -Naur --no-dereference a/arch/arm/boot/dts/dra72-evm-revc.dts b/arch/arm/boot/dts/dra72-evm-revc.dts
--- a/arch/arm/boot/dts/dra72-evm-revc.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/dra72-evm-revc.dts	2020-06-19 21:00:31.404227715 -0400
@@ -17,6 +17,33 @@
 		reg = <0x0 0x80000000 0x0 0x80000000>; /* 2GB */
 	};
 
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		ipu2_memory_region: ipu2-memory@95800000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x95800000 0x0 0x3800000>;
+			reusable;
+			status = "okay";
+		};
+
+		dsp1_memory_region: dsp1-memory@99000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x99000000 0x0 0x4000000>;
+			reusable;
+			status = "okay";
+		};
+
+		ipu1_memory_region: ipu1-memory@9d000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x9d000000 0x0 0x2000000>;
+			reusable;
+			status = "okay";
+		};
+	};
+
 	evm_1v8_sw: fixedregulator-evm_1v8 {
 		compatible = "regulator-fixed";
 		regulator-name = "evm_1v8";
@@ -48,6 +75,11 @@
 	vdda-supply = <&ldo2_reg>;
 };
 
+&pcf_lcd {
+	interrupt-parent = <&gpio3>;
+	interrupts = <30 IRQ_TYPE_EDGE_FALLING>;
+};
+
 &pcf_gpio_21 {
 	interrupt-parent = <&gpio3>;
 	interrupts = <30 IRQ_TYPE_EDGE_FALLING>;
@@ -116,3 +148,20 @@
 	pinctrl-3 = <&mmc2_pins_hs200 &mmc2_iodelay_hs200_rev20_conf>;
 	vmmc-supply = <&evm_1v8_sw>;
 };
+
+&ipu2 {
+	status = "okay";
+	memory-region = <&ipu2_memory_region>;
+};
+
+&ipu1 {
+	status = "okay";
+	memory-region = <&ipu1_memory_region>;
+};
+
+&dsp1 {
+	status = "okay";
+	memory-region = <&dsp1_memory_region>;
+};
+
+#include "dra7-ipu-common-early-boot.dtsi"
diff -Naur --no-dereference a/arch/arm/boot/dts/dra72x.dtsi b/arch/arm/boot/dts/dra72x.dtsi
--- a/arch/arm/boot/dts/dra72x.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/dra72x.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -12,11 +12,58 @@
 / {
 	compatible = "ti,dra722", "ti,dra72", "ti,dra7";
 
+	aliases {
+		rproc0 = &ipu1;
+		rproc1 = &ipu2;
+		rproc2 = &dsp1;
+	};
+
 	pmu {
 		compatible = "arm,cortex-a15-pmu";
 		interrupt-parent = <&wakeupgen>;
 		interrupts = <GIC_SPI 131 IRQ_TYPE_LEVEL_HIGH>;
 	};
+
+	ocp {
+		cal: cal@4845b000 {
+			compatible = "ti,dra72-cal";
+			ti,hwmods = "cal";
+			reg = <0x4845B000 0x400>,
+			      <0x4845B800 0x40>,
+			      <0x4845B900 0x40>;
+			reg-names = "cal_top",
+				    "cal_rx_core0",
+				    "cal_rx_core1";
+			interrupts = <GIC_SPI 119 IRQ_TYPE_LEVEL_HIGH>;
+			syscon-camerrx = <&scm_conf 0xE94>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				csi2_0: port@0 {
+					reg = <0>;
+				};
+				csi2_1: port@1 {
+					reg = <1>;
+				};
+			};
+		};
+	};
+};
+
+&scm {
+	dra72_vip_mux: pinmux@4a002e8c {
+		compatible = "pinctrl-single";
+		reg = <0xe8c 0x4>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		pinctrl-single,register-width = <32>;
+		pinctrl-single,function-mask = <0x7f>;
+	};
 };
 
 &dss {
diff -Naur --no-dereference a/arch/arm/boot/dts/dra74-ipu-dsp-common.dtsi b/arch/arm/boot/dts/dra74-ipu-dsp-common.dtsi
--- a/arch/arm/boot/dts/dra74-ipu-dsp-common.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/dra74-ipu-dsp-common.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Common IPU and DSP data for TI DRA74x/DRA76x/AM572x/AM574x platforms
+ */
+
+#include "dra7-ipu-dsp-common.dtsi"
+
+&mailbox6 {
+	mbox_dsp2_ipc3x: mbox_dsp2_ipc3x {
+		status = "okay";
+	};
+};
+
+&dsp2 {
+	mboxes = <&mailbox6 &mbox_dsp2_ipc3x>;
+	timers = <&timer6>;
+	watchdog-timers = <&timer13>;
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/dra74x.dtsi b/arch/arm/boot/dts/dra74x.dtsi
--- a/arch/arm/boot/dts/dra74x.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/dra74x.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -31,6 +31,13 @@
 		};
 	};
 
+	aliases {
+		rproc0 = &ipu1;
+		rproc1 = &ipu2;
+		rproc2 = &dsp1;
+		rproc3 = &dsp2;
+	};
+
 	pmu {
 		compatible = "arm,cortex-a15-pmu";
 		interrupt-parent = <&wakeupgen>;
@@ -75,7 +82,6 @@
 			ti,hwmods = "mmu0_dsp2";
 			#iommu-cells = <0>;
 			ti,syscon-mmuconfig = <&dsp2_system 0x0>;
-			status = "disabled";
 		};
 
 		mmu1_dsp2: mmu@41502000 {
@@ -85,7 +91,103 @@
 			ti,hwmods = "mmu1_dsp2";
 			#iommu-cells = <0>;
 			ti,syscon-mmuconfig = <&dsp2_system 0x1>;
+		};
+
+		dsp2: dsp@41000000 {
+			compatible = "ti,dra7-dsp";
+			reg = <0x41000000 0x48000>,
+			      <0x41600000 0x8000>,
+			      <0x41700000 0x8000>;
+			reg-names = "l2ram", "l1pram", "l1dram";
+			ti,hwmods = "dsp2";
+			syscon-bootreg = <&scm_conf 0x560>;
+			iommus = <&mmu0_dsp2>, <&mmu1_dsp2>;
+			ti,rproc-standby-info = <0x4a005620>;
+			status = "disabled";
+		};
+
+		vip2: vip@0x48990000 {
+			compatible = "ti,vip2";
+			reg = <0x48990000 0x114>,
+			      <0x48995500 0xD8>,
+			      <0x48995700 0x18>,
+			      <0x48995800 0x80>,
+			      <0x48995a00 0xD8>,
+			      <0x48995c00 0x18>,
+			      <0x48995d00 0x80>,
+			      <0x4899d000 0x400>;
+			reg-names = "vip",
+				    "parser0",
+				    "csc0",
+				    "sc0",
+				    "parser1",
+				    "csc1",
+				    "sc1",
+				    "vpdma";
+			ti,hwmods = "vip2";
+			interrupts = <GIC_SPI 352 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 393 IRQ_TYPE_LEVEL_HIGH>;
+			/* CTRL_CORE_SMA_SW_1 */
+			syscon-pol = <&scm_conf 0x534>;
+			#address-cells = <1>;
+			#size-cells = <0>;
 			status = "disabled";
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				vin3a: port@0 {
+					reg = <0>;
+				};
+				vin4a: port@1 {
+					reg = <1>;
+				};
+				vin3b: port@2 {
+					reg = <2>;
+				};
+				vin4b: port@3 {
+					reg = <3>;
+				};
+			};
+		};
+
+		vip3: vip@0x489b0000 {
+			compatible = "ti,vip3";
+			reg = <0x489b0000 0x114>,
+			      <0x489b5500 0xD8>,
+			      <0x489b5700 0x18>,
+			      <0x489b5800 0x80>,
+			      <0x489b5a00 0xD8>,
+			      <0x489b5c00 0x18>,
+			      <0x489b5d00 0x80>,
+			      <0x489bd000 0x400>;
+			reg-names = "vip",
+				    "parser0",
+				    "csc0",
+				    "sc0",
+				    "parser1",
+				    "csc1",
+				    "sc1",
+				    "vpdma";
+			ti,hwmods = "vip3";
+			interrupts = <GIC_SPI 353 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 394 IRQ_TYPE_LEVEL_HIGH>;
+			/* CTRL_CORE_SMA_SW_1 */
+			syscon-pol = <&scm_conf 0x534>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				vin5a: port@0 {
+					reg = <0>;
+				};
+				vin6a: port@1 {
+					reg = <1>;
+				};
+			};
 		};
 	};
 };
diff -Naur --no-dereference a/arch/arm/boot/dts/dra74x-p.dtsi b/arch/arm/boot/dts/dra74x-p.dtsi
--- a/arch/arm/boot/dts/dra74x-p.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/dra74x-p.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2017 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include "dra74x.dtsi"
+
+/ {
+	compatible = "ti,dra762", "ti,dra7";
+
+	ocp {
+		emif1: emif@4c000000 {
+			compatible = "ti,emif-dra7xx";
+			reg = <0x4c000000 0x200>;
+			interrupts = <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>;
+			status = "disabled";
+		};
+	};
+};
+
+/* MCAN interrupts are hard-wired to irqs 67, 68 */
+&crossbar_mpu {
+	ti,irqs-skip = <10 67 68 133 139 140>;
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/dra76-evm.dts b/arch/arm/boot/dts/dra76-evm.dts
--- a/arch/arm/boot/dts/dra76-evm.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/dra76-evm.dts	2020-06-19 21:00:31.404227715 -0400
@@ -16,11 +16,59 @@
 	model = "TI DRA762 EVM";
 	compatible = "ti,dra76-evm", "ti,dra762", "ti,dra7";
 
+	aliases {
+		display0 = &hdmi0;
+
+		sound0 = &sound0;
+		sound1 = &hdmi;
+	};
+
 	memory@0 {
 		device_type = "memory";
 		reg = <0x0 0x80000000 0x0 0x80000000>;
 	};
 
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		ipu2_memory_region: ipu2-memory@95800000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x95800000 0x0 0x3800000>;
+			reusable;
+			status = "okay";
+		};
+
+		dsp1_memory_region: dsp1-memory@99000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x99000000 0x0 0x4000000>;
+			reusable;
+			status = "okay";
+		};
+
+		ipu1_memory_region: ipu1-memory@9d000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x9d000000 0x0 0x2000000>;
+			reusable;
+			status = "okay";
+		};
+
+		dsp2_memory_region: dsp2-memory@9f000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x9f000000 0x0 0x800000>;
+			reusable;
+			status = "okay";
+		};
+
+		gpu_memory_region: gpu-memory@c0000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0xc0000000 0x0 0x10000000>;
+			reusable;
+			status = "okay";
+		};
+	};
+
 	vsys_12v0: fixedregulator-vsys12v0 {
 		/* main supply */
 		compatible = "regulator-fixed";
@@ -119,6 +167,48 @@
 		regulator-min-microvolt = <1800000>;
 		regulator-max-microvolt = <1800000>;
 	};
+
+	hdmi0: connector {
+		compatible = "hdmi-connector";
+		label = "hdmi";
+
+		type = "a";
+
+		port {
+			hdmi_connector_in: endpoint {
+				remote-endpoint = <&tpd12s015_out>;
+			};
+		};
+	};
+
+	tpd12s015: encoder {
+		compatible = "ti,tpd12s015";
+
+		gpios = <&gpio7 30 GPIO_ACTIVE_HIGH>,   /* gpio7_30, CT CP HPD */
+			<&gpio7 31 GPIO_ACTIVE_HIGH>,   /* gpio7_31, LS OE */
+			<&gpio7 12 GPIO_ACTIVE_HIGH>;   /* gpio7_12/sp1_cs2, HPD */
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				tpd12s015_in: endpoint {
+					remote-endpoint = <&hdmi_out>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				tpd12s015_out: endpoint {
+					remote-endpoint = <&hdmi_connector_in>;
+				};
+			};
+		};
+	};
 };
 
 &i2c1 {
@@ -320,6 +410,30 @@
 	};
 };
 
+&i2c5 {
+	status = "okay";
+	clock-frequency = <400000>;
+
+	ov10633@37 {
+		compatible = "ovti,ov10633";
+		reg = <0x37>;
+
+		clocks = <&clk_ov10633_fixed>;
+		clock-names = "xvclk";
+
+		mux-gpios = <&pcf_hdmi 2	GPIO_ACTIVE_HIGH>, /* VIN2_S0 */
+			    <&pcf_hdmi 6	GPIO_ACTIVE_LOW>; /* VIN2_S2 */
+		port {
+			onboardLI: endpoint {
+				remote-endpoint = <&vin2a_ep>;
+				hsync-active = <1>;
+				vsync-active = <1>;
+				pclk-sample = <0>;
+			};
+		};
+	};
+};
+
 &cpu0 {
 	vdd-supply = <&buck10_reg>;
 };
@@ -414,6 +528,23 @@
 	phy-supply = <&ldo3_reg>;
 };
 
+&dss {
+	status = "ok";
+	vdda_video-supply = <&ldo5_reg>;
+};
+
+&hdmi {
+	status = "ok";
+
+	vdda-supply = <&ldo1_reg>;
+
+	port {
+		hdmi_out: endpoint {
+			remote-endpoint = <&tpd12s015_in>;
+		};
+	};
+};
+
 &qspi {
 	spi-max-frequency = <96000000>;
 	m25p80@0 {
@@ -450,3 +581,36 @@
 		max-bitrate = <5000000>;
 	};
 };
+
+&vin2a {
+	vin2a_ep: endpoint@0 {
+		slave-mode;
+		remote-endpoint = <&onboardLI>;
+	};
+};
+
+&ipu2 {
+	status = "okay";
+	memory-region = <&ipu2_memory_region>;
+};
+
+&ipu1 {
+	status = "okay";
+	memory-region = <&ipu1_memory_region>;
+};
+
+&dsp1 {
+	status = "okay";
+	memory-region = <&dsp1_memory_region>;
+};
+
+&dsp2 {
+	status = "okay";
+	memory-region = <&dsp2_memory_region>;
+};
+
+&gpu {
+	memory-region = <&gpu_memory_region>;
+};
+
+#include "dra7-ipu-common-early-boot.dtsi"
diff -Naur --no-dereference a/arch/arm/boot/dts/dra76x.dtsi b/arch/arm/boot/dts/dra76x.dtsi
--- a/arch/arm/boot/dts/dra76x.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/dra76x.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -6,7 +6,7 @@
  * published by the Free Software Foundation.
  */
 
-#include "dra74x.dtsi"
+#include "dra74x-p.dtsi"
 
 / {
 	compatible = "ti,dra762", "ti,dra7";
@@ -40,13 +40,36 @@
 				bosch,mram-cfg = <0x0 0 0 32 0 0 1 1>;
 			};
 		};
-	};
 
-};
+		cal: cal@489b0000 {
+			compatible = "ti,dra76-cal";
+			ti,hwmods = "cal";
+			reg = <0x489B0000 0x400>,
+			      <0x489B0800 0x40>,
+			      <0x489B0900 0x40>;
+				reg-names = "cal_top",
+					    "cal_rx_core0",
+					    "cal_rx_core1";
+			interrupts = <GIC_SPI 353 IRQ_TYPE_LEVEL_HIGH>;
+			syscon-camerrx = <&scm_conf 0x6DC>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				csi2_0: port@0 {
+					reg = <0>;
+				};
+				csi2_1: port@1 {
+					reg = <1>;
+				};
+			};
+		};
+	};
 
-/* MCAN interrupts are hard-wired to irqs 67, 68 */
-&crossbar_mpu {
-	ti,irqs-skip = <10 67 68 133 139 140>;
 };
 
 &scm_conf_clocks {
@@ -81,3 +104,32 @@
 		reg = <0x3fc>;
 	};
 };
+
+&cpu0_opp_table {
+	opp_plus@1800000000 {
+		opp-hz = /bits/ 64 <1800000000>;
+		opp-microvolt = <1250000 950000 1250000>,
+				<1250000 950000 1250000>;
+		opp-supported-hw = <0xFF 0x08>;
+	};
+};
+
+&opp_supply_mpu {
+	ti,efuse-settings = <
+	/* uV   offset */
+	1060000 0x0
+	1160000 0x4
+	1210000 0x8
+	1250000 0xC
+	>;
+};
+
+&abb_mpu {
+	ti,abb_info = <
+	/*uV		ABB	efuse	rbb_m fbb_m	vset_m*/
+	1060000		0	0x0	0 0x02000000 0x01F00000
+	1160000		0	0x4	0 0x02000000 0x01F00000
+	1210000		0	0x8	0 0x02000000 0x01F00000
+	1250000		0	0xC	0 0x02000000 0x01F00000
+	>;
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/dra7.dtsi b/arch/arm/boot/dts/dra7.dtsi
--- a/arch/arm/boot/dts/dra7.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/dra7.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -145,7 +145,7 @@
 	 * the moment, just use a fake OCP bus entry to represent the whole bus
 	 * hierarchy.
 	 */
-	ocp {
+	ocp: ocp {
 		compatible = "ti,dra7-l3-noc", "simple-bus";
 		#address-cells = <1>;
 		#size-cells = <1>;
@@ -331,6 +331,7 @@
 				ti,hwmods = "pcie1";
 				phys = <&pcie1_phy>;
 				phy-names = "pcie-phy0";
+				ti,syscon-lane-sel = <&scm_conf_pcie 0x18>;
 				interrupt-map-mask = <0 0 0 7>;
 				interrupt-map = <0 0 0 1 &pcie1_intc 1>,
 						<0 0 0 2 &pcie1_intc 2>,
@@ -356,6 +357,7 @@
 				phys = <&pcie1_phy>;
 				phy-names = "pcie-phy0";
 				ti,syscon-unaligned-access = <&scm_conf1 0x14 1>;
+				ti,syscon-lane-sel = <&scm_conf_pcie 0x18>;
 				status = "disabled";
 			};
 		};
@@ -907,6 +909,8 @@
 			reg = <0x48032000 0x80>;
 			interrupts = <GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "timer2";
+			clocks = <&l4per_clkctrl DRA7_TIMER2_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		timer3: timer@48034000 {
@@ -914,6 +918,8 @@
 			reg = <0x48034000 0x80>;
 			interrupts = <GIC_SPI 34 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "timer3";
+			clocks = <&l4per_clkctrl DRA7_TIMER3_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		timer4: timer@48036000 {
@@ -921,6 +927,8 @@
 			reg = <0x48036000 0x80>;
 			interrupts = <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "timer4";
+			clocks = <&l4per_clkctrl DRA7_TIMER4_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		timer5: timer@48820000 {
@@ -928,6 +936,8 @@
 			reg = <0x48820000 0x80>;
 			interrupts = <GIC_SPI 36 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "timer5";
+			clocks = <&ipu_clkctrl DRA7_TIMER5_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		timer6: timer@48822000 {
@@ -935,6 +945,8 @@
 			reg = <0x48822000 0x80>;
 			interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "timer6";
+			clocks = <&ipu_clkctrl DRA7_TIMER6_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		timer7: timer@48824000 {
@@ -942,6 +954,8 @@
 			reg = <0x48824000 0x80>;
 			interrupts = <GIC_SPI 38 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "timer7";
+			clocks = <&ipu_clkctrl DRA7_TIMER7_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		timer8: timer@48826000 {
@@ -949,6 +963,8 @@
 			reg = <0x48826000 0x80>;
 			interrupts = <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "timer8";
+			clocks = <&ipu_clkctrl DRA7_TIMER8_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		timer9: timer@4803e000 {
@@ -956,6 +972,8 @@
 			reg = <0x4803e000 0x80>;
 			interrupts = <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "timer9";
+			clocks = <&l4per_clkctrl DRA7_TIMER9_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		timer10: timer@48086000 {
@@ -963,6 +981,8 @@
 			reg = <0x48086000 0x80>;
 			interrupts = <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "timer10";
+			clocks = <&l4per_clkctrl DRA7_TIMER10_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		timer11: timer@48088000 {
@@ -970,6 +990,8 @@
 			reg = <0x48088000 0x80>;
 			interrupts = <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "timer11";
+			clocks = <&l4per_clkctrl DRA7_TIMER11_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		timer12: timer@4ae20000 {
@@ -979,6 +1001,8 @@
 			ti,hwmods = "timer12";
 			ti,timer-alwon;
 			ti,timer-secure;
+			clocks = <&wkupaon_clkctrl DRA7_TIMER12_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		timer13: timer@48828000 {
@@ -986,6 +1010,8 @@
 			reg = <0x48828000 0x80>;
 			interrupts = <GIC_SPI 339 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "timer13";
+			clocks = <&l4per_clkctrl DRA7_TIMER13_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		timer14: timer@4882a000 {
@@ -993,6 +1019,8 @@
 			reg = <0x4882a000 0x80>;
 			interrupts = <GIC_SPI 340 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "timer14";
+			clocks = <&l4per_clkctrl DRA7_TIMER14_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		timer15: timer@4882c000 {
@@ -1000,6 +1028,8 @@
 			reg = <0x4882c000 0x80>;
 			interrupts = <GIC_SPI 341 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "timer15";
+			clocks = <&l4per_clkctrl DRA7_TIMER15_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		timer16: timer@4882e000 {
@@ -1007,6 +1037,8 @@
 			reg = <0x4882e000 0x80>;
 			interrupts = <GIC_SPI 342 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "timer16";
+			clocks = <&l4per_clkctrl DRA7_TIMER16_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		wdt2: wdt@4ae14000 {
@@ -1030,6 +1062,49 @@
 			ti,hwmods = "dmm";
 		};
 
+		ipu1: ipu@58820000 {
+			compatible = "ti,dra7-ipu";
+			reg = <0x58820000 0x10000>;
+			reg-names = "l2ram";
+			ti,hwmods = "ipu1";
+			iommus = <&mmu_ipu1>;
+			ti,rproc-standby-info = <0x4a005520>;
+			status = "disabled";
+		};
+
+		ipu2: ipu@55020000 {
+			compatible = "ti,dra7-ipu";
+			reg = <0x55020000 0x10000>;
+			reg-names = "l2ram";
+			ti,hwmods = "ipu2";
+			iommus = <&mmu_ipu2>;
+			ti,rproc-standby-info = <0x4a008920>;
+			status = "disabled";
+		};
+
+		dsp1: dsp@40800000 {
+			compatible = "ti,dra7-dsp";
+			reg = <0x40800000 0x48000>,
+			      <0x40e00000 0x8000>,
+			      <0x40f00000 0x8000>;
+			reg-names = "l2ram", "l1pram", "l1dram";
+			ti,hwmods = "dsp1";
+			syscon-bootreg = <&scm_conf 0x55c>;
+			iommus = <&mmu0_dsp1>, <&mmu1_dsp1>;
+			ti,rproc-standby-info = <0x4a005420>;
+			status = "disabled";
+		};
+
+//		bb2d: bb2d@59000000 {
+//			compatible = "ti,dra7-bb2d","vivante,gc";
+//			reg = <0x59000000 0x0700>;
+//			interrupts = <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>;
+//			ti,hwmods = "bb2d";
+//			clocks = <&dpll_core_h24x2_ck>;
+//			clock-names = "fclk";
+//			status = "disabled";
+//		};
+//s
 		i2c1: i2c@48070000 {
 			compatible = "ti,omap4-i2c";
 			reg = <0x48070000 0x100>;
@@ -1143,7 +1218,6 @@
 			ti,hwmods = "mmu0_dsp1";
 			#iommu-cells = <0>;
 			ti,syscon-mmuconfig = <&dsp1_system 0x0>;
-			status = "disabled";
 		};
 
 		mmu1_dsp1: mmu@40d02000 {
@@ -1153,7 +1227,6 @@
 			ti,hwmods = "mmu1_dsp1";
 			#iommu-cells = <0>;
 			ti,syscon-mmuconfig = <&dsp1_system 0x1>;
-			status = "disabled";
 		};
 
 		mmu_ipu1: mmu@58882000 {
@@ -1163,7 +1236,6 @@
 			ti,hwmods = "mmu_ipu1";
 			#iommu-cells = <0>;
 			ti,iommu-bus-err-back;
-			status = "disabled";
 		};
 
 		mmu_ipu2: mmu@55082000 {
@@ -1173,7 +1245,196 @@
 			ti,hwmods = "mmu_ipu2";
 			#iommu-cells = <0>;
 			ti,iommu-bus-err-back;
+		};
+
+		pruss_soc_bus1: pruss-soc-bus@4b226004 {
+			compatible = "ti,am5728-pruss-soc-bus";
+			reg = <0x4b226004 0x4>;
+			ti,hwmods = "pruss1";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+			status = "disabled";
+
+			pruss1: pruss@4b200000 {
+				compatible = "ti,am5728-pruss";
+				reg = <0x4b200000 0x80000>;
+				interrupts = <GIC_SPI 186 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 187 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 188 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 189 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 190 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 191 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 192 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 193 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-names = "host2", "host3", "host4",
+						  "host5", "host6", "host7",
+						  "host8", "host9";
+				#address-cells = <1>;
+				#size-cells = <1>;
+				ranges;
+				status = "disabled";
+
+				pruss1_mem: memories@4b200000 {
+					reg = <0x4b200000 0x2000>,
+					      <0x4b202000 0x2000>,
+					      <0x4b210000 0x8000>;
+					reg-names = "dram0", "dram1",
+						    "shrdram2";
+				};
+
+				pruss1_cfg: cfg@4b226000 {
+					compatible = "syscon";
+					reg = <0x4b226000 0x2000>;
+				};
+
+				pruss1_iep: iep@4b22e000 {
+					compatible = "syscon";
+					reg = <0x4b22e000 0x31c>;
+				};
+
+				pruss1_mii_rt: mii-rt@4b232000 {
+					compatible = "syscon";
+					reg = <0x4b232000 0x58>;
+				};
+
+				pruss1_intc: interrupt-controller@4b220000 {
+					compatible = "ti,am5728-pruss-intc";
+					reg = <0x4b220000 0x2000>;
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
+
+				pru1_0: pru@4b234000 {
+					compatible = "ti,am5728-pru";
+					reg = <0x4b234000 0x3000>,
+					      <0x4b222000 0x400>,
+					      <0x4b222400 0x100>;
+					reg-names = "iram", "control", "debug";
+					firmware-name = "am57xx-pru1_0-fw";
+					interrupt-parent = <&pruss1_intc>;
+					interrupts = <16>, <17>;
+					interrupt-names = "vring", "kick";
+				};
+
+				pru1_1: pru@4b238000 {
+					compatible = "ti,am5728-pru";
+					reg = <0x4b238000 0x3000>,
+					      <0x4b224000 0x400>,
+					      <0x4b224400 0x100>;
+					reg-names = "iram", "control", "debug";
+					firmware-name = "am57xx-pru1_1-fw";
+					interrupt-parent = <&pruss1_intc>;
+					interrupts = <18>, <19>;
+					interrupt-names = "vring", "kick";
+				};
+
+				pruss1_mdio: mdio@4b232400 {
+					compatible = "ti,davinci_mdio";
+					reg = <0x4b232400 0x90>;
+					#address-cells = <1>;
+					#size-cells = <0>;
+					clocks = <&dpll_gmac_h13x2_ck>;
+					clock-names = "fck";
+					bus_freq = <1000000>;
+					status = "disabled";
+				};
+			};
+		};
+
+		pruss_soc_bus2: pruss-soc-bus@4b2a6004 {
+			compatible = "ti,am5728-pruss-soc-bus";
+			reg = <0x4b2a6004 0x4>;
+			ti,hwmods = "pruss2";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
 			status = "disabled";
+
+			pruss2: pruss@4b280000 {
+				compatible = "ti,am5728-pruss";
+				reg = <0x4b280000 0x80000>;
+				interrupts = <GIC_SPI 196 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 198 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 199 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 200 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 201 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 202 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 203 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-names = "host2", "host3", "host4",
+						  "host5", "host6", "host7",
+						  "host8", "host9";
+				#address-cells = <1>;
+				#size-cells = <1>;
+				ranges;
+				status = "disabled";
+
+				pruss2_mem: memories@4b280000 {
+					reg = <0x4b280000 0x2000>,
+					      <0x4b282000 0x2000>,
+					      <0x4b290000 0x8000>;
+					reg-names = "dram0", "dram1",
+						    "shrdram2";
+				};
+
+				pruss2_cfg: cfg@4b2a6000 {
+					compatible = "syscon";
+					reg = <0x4b2a6000 0x2000>;
+				};
+
+				pruss2_iep: iep@4b2ae000 {
+					compatible = "syscon";
+					reg = <0x4b2ae000 0x31c>;
+				};
+
+				pruss2_mii_rt: mii-rt@4b2b2000 {
+					compatible = "syscon";
+					reg = <0x4b2b2000 0x58>;
+				};
+
+				pruss2_intc: interrupt-controller@4b2a0000 {
+					compatible = "ti,am5728-pruss-intc";
+					reg = <0x4b2a0000 0x2000>;
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
+
+				pru2_0: pru@4b2b4000 {
+					compatible = "ti,am5728-pru";
+					reg = <0x4b2b4000 0x3000>,
+					      <0x4b2a2000 0x400>,
+					      <0x4b2a2400 0x100>;
+					reg-names = "iram", "control", "debug";
+					firmware-name = "am57xx-pru2_0-fw";
+					interrupt-parent = <&pruss2_intc>;
+					interrupts = <16>, <17>;
+					interrupt-names = "vring", "kick";
+				};
+
+				pru2_1: pru@4b2b8000 {
+					compatible = "ti,am5728-pru";
+					reg = <0x4b2b8000 0x3000>,
+					      <0x4b2a4000 0x400>,
+					      <0x4b2a4400 0x100>;
+					reg-names = "iram", "control", "debug";
+					firmware-name = "am57xx-pru2_1-fw";
+					interrupt-parent = <&pruss2_intc>;
+					interrupts = <18>, <19>;
+					interrupt-names = "vring", "kick";
+				};
+
+				pruss2_mdio: mdio@4b2b2400 {
+					compatible = "ti,davinci_mdio";
+					reg = <0x4b2b2400 0x90>;
+					#address-cells = <1>;
+					#size-cells = <0>;
+					clocks = <&dpll_gmac_h13x2_ck>;
+					clock-names = "fck";
+					bus_freq = <1000000>;
+					status = "disabled";
+				};
+			};
 		};
 
 		abb_mpu: regulator-abb-mpu {
@@ -1915,6 +2176,28 @@
 			status = "disabled";
 		};
 
+		gpu: gpu@56000000 {
+			compatible = "ti,dra7-sgx544", "img,sgx544";
+			reg = <0x56000000 0x10000>;
+			reg-names = "gpu_ocp_base";
+			interrupts = <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>;
+			ti,hwmods = "gpu";
+			clocks = <&l3_iclk_div>, <&gpu_core_gclk_mux>,
+				<&gpu_hyd_gclk_mux>;
+			clock-names = "iclk", "fclk1", "fclk2";
+			status = "disabled";
+		};
+
+		bb2d: bb2d@59000000 {
+			compatible = "ti,dra7-bb2d";
+			reg = <0x59000000 0x0700>;
+			interrupts = <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>;
+			ti,hwmods = "bb2d";
+			clocks = <&dss_clkctrl DRA7_BB2D_CLKCTRL 0>;
+			clock-names = "fck";
+			status = "disabled";
+		};
+
 		dss: dss@58000000 {
 			compatible = "ti,dra7-dss";
 			/* 'reg' defined in dra72x.dtsi and dra74x.dtsi */
@@ -2111,6 +2394,68 @@
 			ti,absolute-max-voltage-uv = <1500000>;
 		};
 
+		vpe {
+			compatible = "ti,vpe";
+			ti,hwmods = "vpe";
+			clocks = <&dpll_core_h23x2_ck>;
+			clock-names = "fck";
+			reg = <0x489d0000 0x120>,
+			      <0x489d0700 0x80>,
+			      <0x489d5700 0x18>,
+			      <0x489dd000 0x400>;
+			reg-names = "vpe_top",
+				    "sc",
+				    "csc",
+				    "vpdma";
+			interrupts = <GIC_SPI 354 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+
+		vip1: vip@0x48970000 {
+			compatible = "ti,vip1";
+			reg = <0x48970000 0x114>,
+			      <0x48975500 0xD8>,
+			      <0x48975700 0x18>,
+			      <0x48975800 0x80>,
+			      <0x48975a00 0xD8>,
+			      <0x48975c00 0x18>,
+			      <0x48975d00 0x80>,
+			      <0x4897d000 0x400>;
+			reg-names = "vip",
+				    "parser0",
+				    "csc0",
+				    "sc0",
+				    "parser1",
+				    "csc1",
+				    "sc1",
+				    "vpdma";
+			ti,hwmods = "vip1";
+			interrupts = <GIC_SPI 351 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 392 IRQ_TYPE_LEVEL_HIGH>;
+			/* CTRL_CORE_SMA_SW_1 */
+			syscon-pol = <&scm_conf 0x534>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				vin1a: port@0 {
+					reg = <0>;
+				};
+				vin2a: port@1 {
+					reg = <1>;
+				};
+				vin1b: port@2 {
+					reg = <2>;
+				};
+				vin2b: port@3 {
+					reg = <3>;
+				};
+			};
+		};
 	};
 
 	thermal_zones: thermal-zones {
diff -Naur --no-dereference a/arch/arm/boot/dts/dra7-evm-common.dtsi b/arch/arm/boot/dts/dra7-evm-common.dtsi
--- a/arch/arm/boot/dts/dra7-evm-common.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/dra7-evm-common.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -6,6 +6,7 @@
  * published by the Free Software Foundation.
  */
 
+#include "dra74-ipu-dsp-common.dtsi"
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/clk/ti-dra7-atl.h>
 #include <dt-bindings/input/input.h>
@@ -104,6 +105,16 @@
 			gpios = <&pcf_lcd 3 GPIO_ACTIVE_LOW>;
 		};
 	};
+
+	clk_ov10633_fixed: clk_ov10633_fixed {
+		#clock-cells = <0>;
+		compatible = "fixed-clock";
+		clock-frequency = <24000000>;
+	};
+};
+
+&gpu {
+	status = "ok";
 };
 
 &i2c3 {
@@ -247,26 +258,6 @@
 	rx-num-evt = <32>;
 };
 
-&mailbox5 {
-	status = "okay";
-	mbox_ipu1_ipc3x: mbox_ipu1_ipc3x {
-		status = "okay";
-	};
-	mbox_dsp1_ipc3x: mbox_dsp1_ipc3x {
-		status = "okay";
-	};
-};
-
-&mailbox6 {
-	status = "okay";
-	mbox_ipu2_ipc3x: mbox_ipu2_ipc3x {
-		status = "okay";
-	};
-	mbox_dsp2_ipc3x: mbox_dsp2_ipc3x {
-		status = "okay";
-	};
-};
-
 &pcie1_rc {
 	status = "okay";
 };
@@ -285,3 +276,8 @@
 		interrupts = <7 IRQ_TYPE_EDGE_RISING>;
 	};
 };
+
+&vip1 {
+	status = "okay";
+};
+
diff -Naur --no-dereference a/arch/arm/boot/dts/dra7-evm.dts b/arch/arm/boot/dts/dra7-evm.dts
--- a/arch/arm/boot/dts/dra7-evm.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/dra7-evm.dts	2020-06-19 21:00:31.404227715 -0400
@@ -15,6 +15,12 @@
 	model = "TI DRA742";
 	compatible = "ti,dra7-evm", "ti,dra742", "ti,dra74", "ti,dra7";
 
+	aliases {
+		display0 = &hdmi0;
+		sound0 = &sound0;
+		sound1 = &hdmi;
+	};
+
 	memory@0 {
 		device_type = "memory";
 		reg = <0x0 0x80000000 0x0 0x60000000>; /* 1536 MB */
@@ -38,6 +44,47 @@
 		regulator-max-microvolt = <1800000>;
 	};
 
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		ipu2_memory_region: ipu2-memory@95800000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x95800000 0x0 0x3800000>;
+			reusable;
+			status = "okay";
+		};
+
+		dsp1_memory_region: dsp1-memory@99000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x99000000 0x0 0x4000000>;
+			reusable;
+			status = "okay";
+		};
+
+		ipu1_memory_region: ipu1-memory@9d000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x9d000000 0x0 0x2000000>;
+			reusable;
+			status = "okay";
+		};
+
+		dsp2_memory_region: dsp2-memory@9f000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x9f000000 0x0 0x800000>;
+			reusable;
+			status = "okay";
+		};
+
+		gpu_memory_region: gpu-memory@c0000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0xc0000000 0x0 0x10000000>;
+			reusable;
+			status = "okay";
+		};
+	};
+
 	evm_3v3_sd: fixedregulator-sd {
 		compatible = "regulator-fixed";
 		regulator-name = "evm_3v3_sd";
@@ -118,6 +165,54 @@
 		gpio = <&gpio7 11 GPIO_ACTIVE_HIGH>;
 	};
 
+	hdmi0: connector@1 {
+		compatible = "hdmi-connector";
+		label = "hdmi";
+
+		type = "a";
+
+		port {
+			hdmi_connector_in: endpoint {
+				remote-endpoint = <&tpd12s015_out>;
+			};
+		};
+	};
+
+	tpd12s015: encoder@1 {
+		compatible = "ti,dra7evm-tpd12s015";
+
+		pinctrl-names = "i2c", "ddc";
+		pinctrl-0 = <&hdmi_i2c_pins_i2c>;
+		pinctrl-1 = <&hdmi_i2c_pins_ddc>;
+
+		ddc-i2c-bus = <&i2c2>;
+
+		gpios = <&pcf_hdmi 4 0>,	/* P4, CT CP HPD */
+			<&pcf_hdmi 5 0>,	/* P5, LS OE */
+			<&gpio7 12 0>,	/* gpio7_12/sp1_cs2, HPD */
+			<&mcasp8 2 0>;	/* i2c2/ddc switch */
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				tpd12s015_in: endpoint@0 {
+					remote-endpoint = <&hdmi_out>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				tpd12s015_out: endpoint@0 {
+					remote-endpoint = <&hdmi_connector_in>;
+				};
+			};
+		};
+	};
 };
 
 &dra7_pmx_core {
@@ -134,6 +229,27 @@
 			DRA7XX_CORE_IOPAD(0x3818, MUX_MODE15 | PULL_UP)	/* wakeup0.off */
 		>;
 	};
+
+	mcasp8_axr2_pin: pinmux_mcasp8_axr2_pin {
+		pinctrl-single,pins = <
+			/* this pin is used as a GPIO via mcasp */
+			DRA7XX_CORE_IOPAD(0x36fc, PIN_OUTPUT | MUX_MODE1) /* mcasp2_aclkr.mcasp8_axr2 */
+		>;
+	};
+
+	hdmi_i2c_pins_i2c: pinmux_hdmi_i2c_pins_default {
+		pinctrl-single,pins = <
+			DRA7XX_CORE_IOPAD(0x3808, PIN_INPUT | MUX_MODE0) /* i2c2_sda.i2c2_sda */
+			DRA7XX_CORE_IOPAD(0x380c, PIN_INPUT | MUX_MODE0) /* i2c2_scl.i2c2_scl */
+		>;
+	};
+
+	hdmi_i2c_pins_ddc: pinmux_hdmi_i2c_pins_ddc {
+		pinctrl-single,pins = <
+			DRA7XX_CORE_IOPAD(0x3808, PIN_INPUT | MUX_MODE1) /* i2c2_sda.hdmi1_ddc_scl */
+			DRA7XX_CORE_IOPAD(0x380c, PIN_INPUT | MUX_MODE1) /* i2c2_scl.hdmi1_ddc_sda */
+		>;
+	};
 };
 
 &i2c1 {
@@ -343,6 +459,24 @@
 			line-name = "vin6_sel_s0";
 		};
 	};
+
+	ov10633@37 {
+		compatible = "ovti,ov10633";
+		reg = <0x37>;
+
+		clocks = <&clk_ov10633_fixed>;
+		clock-names = "xvclk";
+
+		mux-gpios = <&pcf_hdmi 3 GPIO_ACTIVE_LOW>; /* CAM_FPD_MUX_S0 */
+		port {
+			onboardLI: endpoint {
+				remote-endpoint = <&vin1a_ep>;
+				hsync-active = <1>;
+				vsync-active = <1>;
+				pclk-sample = <0>;
+			};
+		};
+	};
 };
 
 &mmc1 {
@@ -530,3 +664,68 @@
 	pinctrl-1 = <&dcan1_pins_sleep>;
 	pinctrl-2 = <&dcan1_pins_default>;
 };
+
+&vin1a {
+	vin1a_ep: endpoint@0 {
+		slave-mode;
+		remote-endpoint = <&onboardLI>;
+	};
+};
+
+&dss {
+	status = "ok";
+
+	vdda_video-supply = <&ldoln_reg>;
+};
+
+&hdmi {
+	status = "ok";
+	vdda-supply = <&ldo3_reg>;
+
+	port {
+		hdmi_out: endpoint {
+			remote-endpoint = <&tpd12s015_in>;
+		};
+	};
+};
+
+&mcasp8 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcasp8_axr2_pin>;
+
+	gpio-controller;
+	#gpio-cells = <2>;
+
+	/* Not used for audio, only AXR2 pin is used as GPIO */
+	status = "okay";
+};
+
+&ipu2 {
+	status = "okay";
+	memory-region = <&ipu2_memory_region>;
+};
+
+&ipu1 {
+	status = "okay";
+	memory-region = <&ipu1_memory_region>;
+};
+
+&dsp1 {
+	status = "okay";
+	memory-region = <&dsp1_memory_region>;
+};
+
+&dsp2 {
+	status = "okay";
+	memory-region = <&dsp2_memory_region>;
+};
+
+&gpu {
+	memory-region = <&gpu_memory_region>;
+};
+
+&bb2d {
+	status = "okay";
+};
+
+#include "dra7-ipu-common-early-boot.dtsi"
diff -Naur --no-dereference a/arch/arm/boot/dts/dra7-ipu-common-early-boot.dtsi b/arch/arm/boot/dts/dra7-ipu-common-early-boot.dtsi
--- a/arch/arm/boot/dts/dra7-ipu-common-early-boot.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/dra7-ipu-common-early-boot.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,49 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * Common dtsi file that needs to be included in corresponding TI DRA7xx
+ * and AM57xx board dts files that have the IPU1 _and_ IPU2 remote processors
+ * booted early from TI U-Boot/SPL.
+ */
+
+/ {
+	reserved-memory {
+		mmu-early-page-tables@95700000 {
+			/* address need to match the usage within U-Boot */
+			reg = <0x0 0x95700000 0x0 0x100000>;
+			no-map;
+		};
+	};
+};
+
+/* IPU1 */
+&timer11 {
+	ti,no-idle-on-init;
+	ti,no-reset-on-init;
+};
+
+&timer7 {
+	ti,no-idle-on-init;
+	ti,no-reset-on-init;
+};
+
+&timer8 {
+	ti,no-idle-on-init;
+	ti,no-reset-on-init;
+};
+
+&mmu_ipu1{
+	ti,no-idle-on-init;
+	ti,no-reset-on-init;
+};
+
+&ipu1_memory_region {
+	/delete-property/ reusable;
+	no-map;
+};
+
+&ipu1 {
+	ti,no-idle-on-init;
+	ti,no-reset-on-init;
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/dra7-ipu-dsp-common.dtsi b/arch/arm/boot/dts/dra7-ipu-dsp-common.dtsi
--- a/arch/arm/boot/dts/dra7-ipu-dsp-common.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/dra7-ipu-dsp-common.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Common IPU and DSP data for TI DRA7xx/AM57xx platforms
+ */
+
+&mailbox5 {
+	status = "okay";
+	mbox_ipu1_ipc3x: mbox_ipu1_ipc3x {
+		status = "okay";
+	};
+	mbox_dsp1_ipc3x: mbox_dsp1_ipc3x {
+		status = "okay";
+	};
+};
+
+&mailbox6 {
+	status = "okay";
+	mbox_ipu2_ipc3x: mbox_ipu2_ipc3x {
+		status = "okay";
+	};
+};
+
+&ipu2 {
+	mboxes = <&mailbox6 &mbox_ipu2_ipc3x>;
+	timers = <&timer3>;
+	watchdog-timers = <&timer4>, <&timer9>;
+};
+
+&ipu1 {
+	mboxes = <&mailbox5 &mbox_ipu1_ipc3x>;
+	timers = <&timer11>;
+	watchdog-timers = <&timer7>, <&timer8>;
+};
+
+&dsp1 {
+	mboxes = <&mailbox5 &mbox_dsp1_ipc3x>;
+	timers = <&timer5>;
+	watchdog-timers = <&timer10>;
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/dra7xx-clocks.dtsi b/arch/arm/boot/dts/dra7xx-clocks.dtsi
--- a/arch/arm/boot/dts/dra7xx-clocks.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/dra7xx-clocks.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -799,16 +799,6 @@
 		clock-div = <1>;
 	};
 
-	ipu1_gfclk_mux: ipu1_gfclk_mux@520 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&dpll_abe_m2x2_ck>, <&dpll_core_h22x2_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x0520>;
-		assigned-clocks = <&ipu1_gfclk_mux>;
-		assigned-clock-parents = <&dpll_core_h22x2_ck>;
-	};
-
 	dummy_ck: dummy_ck {
 		#clock-cells = <0>;
 		compatible = "fixed-clock";
@@ -1540,16 +1530,61 @@
 		};
 	};
 
-	ipu_cm: ipu_cm@500 {
+	dsp1_cm: dsp1_cm@400 {
+		compatible = "ti,omap4-cm";
+		reg = <0x400 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x400 0x100>;
+
+		dsp1_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	ipu1_cm: ipu1_cm@500 {
 		compatible = "ti,omap4-cm";
-		reg = <0x500 0x100>;
+		reg = <0x500 0x40>;
 		#address-cells = <1>;
 		#size-cells = <1>;
 		ranges = <0 0x500 0x100>;
 
-		ipu_clkctrl: clk@40 {
+		ipu1_clkctrl: clk@20 {
 			compatible = "ti,clkctrl";
-			reg = <0x40 0x44>;
+			reg = <0x20 0x20>;
+			#clock-cells = <2>;
+
+			assigned-clocks = <&ipu1_clkctrl DRA7_IPU1_CLKCTRL 24>;
+			assigned-clock-parents = <&dpll_core_h22x2_ck>;
+		};
+	};
+
+	ipu_cm: ipu_cm@540 {
+		compatible = "ti,omap4-cm";
+		reg = <0x540 0xc0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x540 0xc0>;
+
+		ipu_clkctrl: clk@0 {
+			compatible = "ti,clkctrl";
+			reg = <0x0 0x44>;
+			#clock-cells = <2>;
+		};
+	};
+
+	dsp2_cm: dsp2_cm@600 {
+		compatible = "ti,omap4-cm";
+		reg = <0x600 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x600 0x100>;
+
+		dsp2_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x4>;
 			#clock-cells = <2>;
 		};
 	};
@@ -1598,6 +1633,20 @@
 			#clock-cells = <2>;
 		};
 	};
+
+	ipu2_cm: ipu2_cm@900 {
+		compatible = "ti,omap4-cm";
+		reg = <0x900 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x900 0x100>;
+
+		ipu2_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x4>;
+			#clock-cells = <2>;
+		};
+	};
 
 	dma_cm: dma_cm@a00 {
 		compatible = "ti,omap4-cm";
diff -Naur --no-dereference a/arch/arm/boot/dts/keystone.dtsi b/arch/arm/boot/dts/keystone.dtsi
--- a/arch/arm/boot/dts/keystone.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/keystone.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -102,6 +102,16 @@
 				ti,syscon-dev = <&devctrl 0x2a0>;
 			};
 
+			pcie_devid: pcie-devid@128 {
+				compatible = "syscon";
+				reg = <0x00000128 0x4>;
+			};
+
+			pcie_mode: pcie-mode@14c {
+				compatible = "syscon";
+				reg = <0x0000014c 0x4>;
+			};
+
 			rstctrl: reset-controller@328 {
 				compatible = "ti,keystone-reset";
 				reg = <0x328 0x10>;
@@ -295,20 +305,32 @@
 				  1 0 0x21000A00 0x00000100>;
 		};
 
+		pcie0_phy: phy@2320000 {
+			#phy-cells = <0>;
+			compatible = "ti,keystone-serdes-pcie";
+			reg = <0x02320000 0x4000>;
+			link-rate-kbps = <5000000>;
+			num-lanes = <2>;
+			status = "disabled";
+		};
+
 		pcie0: pcie@21800000 {
 			compatible = "ti,keystone-pcie", "snps,dw-pcie";
+			reg =  <0x21800000 0x1000>, <0x21801000 0x1000>, <0x21802000 0x1000>;
+			reg-names = "app", "dbics", "config";
 			clocks = <&clkpcie>;
-			clock-names = "pcie";
+			clock-names = "fck";
 			#address-cells = <3>;
 			#size-cells = <2>;
-			reg =  <0x21801000 0x2000>, <0x21800000 0x1000>, <0x02620128 4>;
-			ranges = <0x82000000 0 0x50000000 0x50000000
-				  0 0x10000000>;
+			ranges = <0x81000000 0 0          0x50000000 0 0x100000
+				  0x82000000 0 0x50100000 0x50100000 0 0xFF00000>;
 
 			status = "disabled";
 			device_type = "pci";
 			num-lanes = <2>;
+			num-viewport = <32>;
 			bus-range = <0x00 0xff>;
+			phys = <&pcie0_phy>;
 
 			/* error interrupt */
 			interrupts = <GIC_SPI 38 IRQ_TYPE_EDGE_RISING>;
diff -Naur --no-dereference a/arch/arm/boot/dts/keystone-k2e.dtsi b/arch/arm/boot/dts/keystone-k2e.dtsi
--- a/arch/arm/boot/dts/keystone-k2e.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/keystone-k2e.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -93,6 +93,11 @@
 			#address-cells = <1>;
 			#size-cells = <1>;
 
+			sram-mpm@0 {
+				compatible = "ti,keystone-dsp-msm-ram";
+				reg = <0x0 0x80000>;
+			};
+
 			sram-bm@1f0000 {
 				reg = <0x001f0000 0x8000>;
 			};
@@ -135,21 +140,33 @@
 			status = "disabled";
 		};
 
+		pcie1_phy: phy@2326000 {
+			#phy-cells = <0>;
+			compatible = "ti,keystone-serdes-pcie";
+			reg = <0x02326000 0x4000>;
+			link-rate-kbps = <5000000>;
+			num-lanes = <2>;
+			status = "disabled";
+		};
+
 		pcie1: pcie@21020000 {
-			compatible = "ti,keystone-pcie","snps,dw-pcie";
+			compatible = "ti,keystone-pcie", "snps,dw-pcie";
+			reg =  <0x21020000 0x1000>, <0x21021000 0x1000>, <0x21022000 0x1000>;
+			reg-names = "app", "dbics", "config";
 			clocks = <&clkpcie1>;
-			clock-names = "pcie";
+			clock-names = "fck";
 			#address-cells = <3>;
 			#size-cells = <2>;
-			reg =  <0x21021000 0x2000>, <0x21020000 0x1000>, <0x02620128 4>;
-			ranges = <0x82000000 0 0x60000000 0x60000000
-				  0 0x10000000>;
-
+			ranges = <0x81000000 0 0          0x60000000 0 0x100000
+				  0x82000000 0 0x60100000 0x60100000 0 0xFF00000>;
+			ti,syscon-pcie-id = <&pcie_devid>;
+			ti,syscon-pcie-mode = <&pcie_mode>;
 			status = "disabled";
 			device_type = "pci";
 			num-lanes = <2>;
+			num-viewport = <32>;
 			bus-range = <0x00 0xff>;
-
+			phys = <&pcie1_phy>;
 			/* error interrupt */
 			interrupts = <GIC_SPI 385 IRQ_TYPE_EDGE_RISING>;
 			#interrupt-cells = <1>;
diff -Naur --no-dereference a/arch/arm/boot/dts/keystone-k2e-evm.dts b/arch/arm/boot/dts/keystone-k2e-evm.dts
--- a/arch/arm/boot/dts/keystone-k2e-evm.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/keystone-k2e-evm.dts	2020-06-19 21:00:31.404227715 -0400
@@ -24,6 +24,13 @@
 			reusable;
 			status = "okay";
 		};
+
+		dsp_common_mpm_memory: dsp-common-mpm-memory@820000000 {
+			compatible = "ti,keystone-dsp-mem-pool";
+			reg = <0x00000008 0x20000000 0x00000000 0x10000000>;
+			no-map;
+			status = "okay";
+		};
 	};
 };
 
@@ -175,3 +182,15 @@
 	memory-region = <&dsp_common_memory>;
 	status = "okay";
 };
+
+&gbe_serdes0 {
+	status = "okay";
+};
+
+&pcie1_phy {
+	status = "okay";
+};
+
+&pcie1 {
+	status = "okay";
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/keystone-k2e-netcp.dtsi b/arch/arm/boot/dts/keystone-k2e-netcp.dtsi
--- a/arch/arm/boot/dts/keystone-k2e-netcp.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/keystone-k2e-netcp.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -122,6 +122,106 @@
 			  <0x24189000 0x1000>;
 		reg-names = "global", "txchan", "rxchan",
 				"txsched", "rxflow";
+		ti,enable-all;
+	};
+};
+
+gbe_subsys: subsys@24200000 {
+	compatible = "syscon";
+	reg = <0x24200000 0x100>;
+};
+
+gbe_serdes0: phy@232a000 {
+	compatible		= "ti,keystone-serdes-gbe";
+	reg			= <0x0232a000 0x2000>;
+	status			= "disabled";
+	link-rate-kbps		= <1250000>;
+	num-lanes		= <4>;
+	#address-cells	= <1>;
+	#size-cells	= <0>;
+
+	serdes0_lane0: lane@0 {
+		#phy-cells	= <0>;
+		reg		= <0>;
+		status		= "ok";
+		control-rate	= <2>;
+		rx-start	= <7 5>;
+		rx-force	= <1 1>;
+		tx-coeff	= <0 0 0 12 4>;
+	};
+	serdes0_lane1: lane@1 {
+		#phy-cells	= <0>;
+		reg		= <1>;
+		status		= "ok";
+		control-rate	= <2>;
+		rx-start	= <7 5>;
+		rx-force	= <1 1>;
+		tx-coeff	= <0 0 0 12 4>;
+	};
+	serdes0_lane2: lane@2 {
+		#phy-cells	= <0>;
+		reg		= <2>;
+		status		= "disabled";
+		control-rate	= <2>;
+		rx-start	= <7 5>;
+		rx-force	= <1 1>;
+		tx-coeff	= <0 0 0 12 4>;
+	};
+	serdes0_lane3: lane@3 {
+		#phy-cells	= <0>;
+		reg		= <3>;
+		status		= "disabled";
+		control-rate	= <2>;
+		rx-start	= <7 5>;
+		rx-force	= <1 1>;
+		tx-coeff	= <0 0 0 12 4>;
+	};
+};
+
+gbe_serdes1: phy@2324000 {
+	compatible		= "ti,keystone-serdes-gbe";
+	reg			= <0x02324000 0x2000>;
+	status			= "disabled";
+	link-rate-kbps		= <1250000>;
+	num-lanes		= <4>;
+	#address-cells	= <1>;
+	#size-cells	= <0>;
+
+	serdes1_lane0: lane@0 {
+		#phy-cells	= <0>;
+		reg		= <0>;
+		status		= "disabled";
+		control-rate	= <2>;
+		rx-start	= <7 5>;
+		rx-force	= <1 1>;
+		tx-coeff	= <0 0 0 12 4>;
+	};
+	serdes1_lane1: lane@1 {
+		#phy-cells	= <0>;
+		reg		= <1>;
+		status		= "disabled";
+		control-rate	= <2>;
+		rx-start	= <7 5>;
+		rx-force	= <1 1>;
+		tx-coeff	= <0 0 0 12 4>;
+	};
+	serdes1_lane2: lane@2 {
+		#phy-cells	= <0>;
+		reg		= <2>;
+		status		= "disabled";
+		control-rate	= <2>;
+		rx-start	= <7 5>;
+		rx-force	= <1 1>;
+		tx-coeff	= <0 0 0 12 4>;
+	};
+	serdes1_lane3: lane@3 {
+		#phy-cells	= <0>;
+		reg		= <3>;
+		status		= "disabled";
+		control-rate	= <2>;
+		rx-start	= <7 5>;
+		rx-force	= <1 1>;
+		tx-coeff	= <0 0 0 12 4>;
 	};
 };
 
@@ -151,19 +251,21 @@
 		gbe@200000 { /* ETHSS */
 			label = "netcp-gbe";
 			compatible = "ti,netcp-gbe-9";
-			reg = <0x200000 0x900>, <0x220000 0x20000>;
-			/* enable-ale; */
+			syscon-subsys = <&gbe_subsys>;
+			reg = <0x200100 0x800>, <0x220000 0x20000>;
 			tx-queue = <896>;
 			tx-channel = "nettx";
 
 			interfaces {
 				gbe0: interface-0 {
-					slave-port = <0>;
+					phys		= <&serdes0_lane0>;
+					slave-port	= <0>;
 					link-interface	= <1>;
 					phy-handle	= <&ethphy0>;
 				};
 				gbe1: interface-1 {
-					slave-port = <1>;
+					phys		= <&serdes0_lane1>;
+					slave-port	= <1>;
 					link-interface	= <1>;
 					phy-handle	= <&ethphy1>;
 				};
@@ -171,26 +273,32 @@
 
 			secondary-slave-ports {
 				port-2 {
+					phys = <&serdes0_lane2>;
 					slave-port = <2>;
 					link-interface	= <2>;
 				};
 				port-3 {
+					phys = <&serdes0_lane3>;
 					slave-port = <3>;
 					link-interface	= <2>;
 				};
 				port-4 {
+					phys = <&serdes1_lane0>;
 					slave-port = <4>;
 					link-interface	= <2>;
 				};
 				port-5 {
+					phys = <&serdes1_lane1>;
 					slave-port = <5>;
 					link-interface	= <2>;
 				};
 				port-6 {
+					phys = <&serdes1_lane2>;
 					slave-port = <6>;
 					link-interface	= <2>;
 				};
 				port-7 {
+					phys = <&serdes1_lane3>;
 					slave-port = <7>;
 					link-interface	= <2>;
 				};
diff -Naur --no-dereference a/arch/arm/boot/dts/keystone-k2g.dtsi b/arch/arm/boot/dts/keystone-k2g.dtsi
--- a/arch/arm/boot/dts/keystone-k2g.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/keystone-k2g.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -102,6 +102,11 @@
 			#address-cells = <1>;
 			#size-cells = <1>;
 
+			sram-mpm@0 {
+				compatible = "ti,keystone-dsp-msm-ram";
+				reg = <0x0 0x80000>;
+			};
+
 			sram-bm@f7000 {
 				reg = <0x000f7000 0x8000>;
 			};
@@ -137,6 +142,48 @@
 				#gpio-cells = <2>;
 				gpio,syscon-dev = <&devctrl 0x240>;
 			};
+
+			prussgpio0: keystone-pruss-gpio@26c {
+				compatible = "ti,keystone-dsp-gpio";
+				reg = <0x26c 0x4>;
+				gpio-controller;
+				#gpio-cells = <2>;
+				gpio,syscon-dev = <&devctrl 0x26c>;
+			};
+
+			prussgpio1: keystone-pruss-gpio@270 {
+				compatible = "ti,keystone-dsp-gpio";
+				reg = <0x270 0x4>;
+				gpio-controller;
+				#gpio-cells = <2>;
+				gpio,syscon-dev = <&devctrl 0x270>;
+			};
+
+			prussgpio2: keystone-pruss-gpio@274 {
+				compatible = "ti,keystone-dsp-gpio";
+				reg = <0x274 0x4>;
+				gpio-controller;
+				#gpio-cells = <2>;
+				gpio,syscon-dev = <&devctrl 0x274>;
+			};
+
+			prussgpio3: keystone-pruss-gpio@278 {
+				compatible = "ti,keystone-dsp-gpio";
+				reg = <0x278 0x4>;
+				gpio-controller;
+				#gpio-cells = <2>;
+				gpio,syscon-dev = <&devctrl 0x278>;
+			};
+
+			pcie_devid: pcie-devid@128 {
+				compatible = "syscon";
+				reg = <0x00000128 0x4>;
+			};
+
+			pcie_mode: pcie-mode@14c {
+				compatible = "syscon";
+				reg = <0x0000014c 0x4>;
+			};
 		};
 
 		uart0: serial@2530c00 {
@@ -283,6 +330,14 @@
 			};
 		};
 
+		clock_event: timer@2210000 {
+			compatible = "ti,keystone-timer";
+			reg = <0x02210000 0x50>;
+			interrupts = <GIC_SPI 19 IRQ_TYPE_LEVEL_HIGH
+					GIC_SPI 18 IRQ_TYPE_LEVEL_LOW>;
+			clocks = <&k2g_clks 0x1e 0>;
+		};
+
 		gpio0: gpio@2603000 {
 			compatible = "ti,k2g-gpio", "ti,keystone-gpio";
 			reg = <0x02603000 0x100>;
@@ -324,6 +379,28 @@
 			clock-names = "gpio";
 		};
 
+		dss: dss@02540000 {
+			compatible = "ti,k2g-dss";
+			reg =	<0x02540000 0x400>,
+				<0x02550000 0x1000>,
+				<0x02557000 0x1000>,
+				<0x0255a800 0x100>,
+				<0x0255ac00 0x100>;
+			reg-names = "cfg", "common", "vid1", "ovr1", "vp1";
+			clocks =	<&k2g_clks 0x2 0>,
+					<&k2g_clks 0x2 1>;
+			clock-names = "fck", "vp1";
+			interrupts = <GIC_SPI 247 IRQ_TYPE_EDGE_RISING>;
+
+			power-domains = <&k2g_pds 0x2>;
+			status = "disabled";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+
+			max-memory-bandwidth = <230000000>;
+		};
+
 		edma0: edma@2700000 {
 			compatible = "ti,k2g-edma3-tpcc", "ti,edma3-tpcc";
 			reg =	<0x02700000 0x8000>;
@@ -606,6 +683,198 @@
 			interrupts = <GIC_SPI 123 IRQ_TYPE_EDGE_RISING>;
 		};
 
+		pruss_soc_bus0: pruss-soc-bus@20aa6004 {
+			compatible = "ti,k2g-pruss-soc-bus";
+			reg = <0x20aa6004 0x4>;
+			power-domains = <&k2g_pds 0x0014>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+			dma-ranges;
+			dma-coherent;
+
+			pruss0: pruss@20a80000 {
+				compatible = "ti,k2g-pruss";
+				reg = <0x20a80000 0x40000>;
+				interrupts = <GIC_SPI 224 IRQ_TYPE_EDGE_RISING>,
+					     <GIC_SPI 225 IRQ_TYPE_EDGE_RISING>,
+					     <GIC_SPI 226 IRQ_TYPE_EDGE_RISING>,
+					     <GIC_SPI 227 IRQ_TYPE_EDGE_RISING>,
+					     <GIC_SPI 228 IRQ_TYPE_EDGE_RISING>,
+					     <GIC_SPI 230 IRQ_TYPE_EDGE_RISING>,
+					     <GIC_SPI 231 IRQ_TYPE_EDGE_RISING>;
+				interrupt-names = "host2", "host3", "host4",
+						  "host5", "host6", "host8",
+						  "host9";
+				#address-cells = <1>;
+				#size-cells = <1>;
+				ranges;
+				dma-ranges;
+				dma-coherent;
+
+				pruss0_mem: memories@20a80000 {
+					reg = <0x20a80000 0x2000>,
+					      <0x20a82000 0x2000>,
+					      <0x20a90000 0x10000>;
+					reg-names = "dram0", "dram1",
+						    "shrdram2";
+				};
+
+				pruss0_cfg: cfg@20aa6000 {
+					compatible = "syscon";
+					reg = <0x20aa6000 0x2000>;
+				};
+
+				pruss0_iep: iep@20aae000 {
+					compatible = "syscon";
+					reg = <0x20aae000 0x31c>;
+				};
+
+				pruss0_mii_rt: mii-rt@20ab2000 {
+					compatible = "syscon";
+					reg = <0x20ab2000 0x70>;
+				};
+
+				pruss0_intc: interrupt-controller@20aa0000 {
+					compatible = "ti,k2g-pruss-intc";
+					reg = <0x20aa0000 0x2000>;
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
+
+				pru0_0: pru@20ab4000 {
+					compatible = "ti,k2g-pru";
+					reg = <0x20ab4000 0x3000>,
+					      <0x20aa2000 0x400>,
+					      <0x20aa2400 0x100>;
+					reg-names = "iram", "control", "debug";
+					firmware-name = "k2g-pru0_0-fw";
+					interrupt-parent = <&pruss0_intc>;
+					interrupts = <16>, <17>;
+					interrupt-names = "vring", "kick";
+				};
+
+				pru0_1: pru@20ab8000 {
+					compatible = "ti,k2g-pru";
+					reg = <0x20ab8000 0x3000>,
+					      <0x20aa4000 0x400>,
+					      <0x20aa4400 0x100>;
+					reg-names = "iram", "control", "debug";
+					firmware-name = "k2g-pru0_1-fw";
+					interrupt-parent = <&pruss0_intc>;
+					interrupts = <18>, <19>;
+					interrupt-names = "vring", "kick";
+				};
+
+				pruss0_mdio: mdio@20ab2400 {
+					compatible = "ti,davinci_mdio";
+					reg = <0x20ab2400 0x90>;
+					clocks = <&k2g_clks 0x0014 1>;
+					clock-names = "fck";
+					#address-cells = <1>;
+					#size-cells = <0>;
+					bus_freq = <2500000>;
+					status = "disabled";
+				};
+			};
+		};
+
+		pruss_soc_bus1: pruss-soc-bus@20ae6004 {
+			compatible = "ti,k2g-pruss-soc-bus";
+			reg = <0x20ae6004 0x4>;
+			power-domains = <&k2g_pds 0x0015>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+			dma-ranges;
+			dma-coherent;
+
+			pruss1: pruss@20ac0000 {
+				compatible = "ti,k2g-pruss";
+				reg = <0x20ac0000 0x40000>;
+				interrupts = <GIC_SPI 232 IRQ_TYPE_EDGE_RISING>,
+					     <GIC_SPI 233 IRQ_TYPE_EDGE_RISING>,
+					     <GIC_SPI 234 IRQ_TYPE_EDGE_RISING>,
+					     <GIC_SPI 235 IRQ_TYPE_EDGE_RISING>,
+					     <GIC_SPI 236 IRQ_TYPE_EDGE_RISING>,
+					     <GIC_SPI 238 IRQ_TYPE_EDGE_RISING>,
+					     <GIC_SPI 239 IRQ_TYPE_EDGE_RISING>;
+				interrupt-names = "host2", "host3", "host4",
+						  "host5", "host6", "host8",
+						  "host9";
+				#address-cells = <1>;
+				#size-cells = <1>;
+				ranges;
+				dma-ranges;
+				dma-coherent;
+
+				pruss1_mem: memories@20ac0000 {
+					reg = <0x20ac0000 0x2000>,
+					      <0x20ac2000 0x2000>,
+					      <0x20ad0000 0x10000>;
+					reg-names = "dram0", "dram1",
+						    "shrdram2";
+				};
+
+				pruss1_cfg: cfg@20ae6000 {
+					compatible = "syscon";
+					reg = <0x20ae6000 0x2000>;
+				};
+
+				pruss1_iep: iep@20aee000 {
+					compatible = "syscon";
+					reg = <0x20aee000 0x31c>;
+				};
+
+				pruss1_mii_rt: mii-rt@20af2000 {
+					compatible = "syscon";
+					reg = <0x20af2000 0x70>;
+				};
+
+				pruss1_intc: interrupt-controller@20ae0000 {
+					compatible = "ti,k2g-pruss-intc";
+					reg = <0x20ae0000 0x2000>;
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
+
+				pru1_0: pru@20af4000 {
+					compatible = "ti,k2g-pru";
+					reg = <0x20af4000 0x3000>,
+					      <0x20ae2000 0x400>,
+					      <0x20ae2400 0x100>;
+					reg-names = "iram", "control", "debug";
+					firmware-name = "k2g-pru1_0-fw";
+					interrupt-parent = <&pruss1_intc>;
+					interrupts = <16>, <17>;
+					interrupt-names = "vring", "kick";
+				};
+
+				pru1_1: pru@20af8000 {
+					compatible = "ti,k2g-pru";
+					reg = <0x20af8000 0x3000>,
+					      <0x20ae4000 0x400>,
+					      <0x20ae4400 0x100>;
+					reg-names = "iram", "control", "debug";
+					firmware-name = "k2g-pru1_1-fw";
+					interrupt-parent = <&pruss1_intc>;
+					interrupts = <18>, <19>;
+					interrupt-names = "vring", "kick";
+				};
+
+				pruss1_mdio: mdio@20af2400 {
+					compatible = "ti,davinci_mdio";
+					reg = <0x20af2400 0x90>;
+					clocks = <&k2g_clks 0x0015 1>;
+					clock-names = "fck";
+					#address-cells = <1>;
+					#size-cells = <0>;
+					bus_freq = <2500000>;
+					status = "disabled";
+				};
+			};
+		};
+
 		mdio: mdio@4200f00 {
 			compatible = "ti,keystone_mdio", "ti,davinci_mdio";
 			reg = <0x04200f00 0x100>;
@@ -618,5 +887,88 @@
 			bus_freq = <2500000>;
 		};
 		#include "keystone-k2g-netcp.dtsi"
+
+		pcie0_phy: phy@2320000 {
+			#phy-cells = <0>;
+			compatible = "ti,keystone-serdes-pcie";
+			reg = <0x02320000 0x4000>;
+			link-rate-kbps = <5000000>;
+			num-lanes = <1>;
+			status = "disabled";
+			power-domains = <&k2g_pds 0x0019>;
+			clocks = <&k2g_clks 0x0019 0>;
+			clock-names = "fck";
+		};
+
+		pcie0: pcie@21800000 {
+			compatible = "ti,keystone-pcie", "snps,dw-pcie";
+			reg =  <0x21800000 0x1000>, <0x21801000 0x1000>, <0x21802000 0x1000>;
+			reg-names = "app", "dbics", "config";
+			power-domains = <&k2g_pds 0x0019>;
+			clocks = <&k2g_clks 0x0019 0>;
+			clock-names = "fck";
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges = <0x81000000 0 0		0x23250000 0 0x4000
+				  0x82000000 0 0x70000000	0x70000000 0 0x10000000>;
+			ti,syscon-pcie-id = <&pcie_devid>;
+			ti,syscon-pcie-mode = <&pcie_mode>;
+			status = "disabled";
+			device_type = "pci";
+			num-lanes = <1>;
+			phys = <&pcie0_phy>;
+			dma-coherent;
+			num-ob-windows = <32>;
+			num-viewport = <32>;
+			/* error interrupt */
+			interrupts = <GIC_SPI 60 IRQ_TYPE_EDGE_RISING>;
+			#interrupt-cells = <1>;
+			interrupt-map-mask = <0 0 0 7>;
+			interrupt-map = <0 0 0 1 &pcie_intc0 0 IRQ_TYPE_EDGE_RISING>, /* INT A */
+					<0 0 0 2 &pcie_intc0 1 IRQ_TYPE_EDGE_RISING>, /* INT B */
+					<0 0 0 3 &pcie_intc0 2 IRQ_TYPE_EDGE_RISING>, /* INT C */
+					<0 0 0 4 &pcie_intc0 3 IRQ_TYPE_EDGE_RISING>; /* INT D */
+
+			pcie_msi_intc0: msi-interrupt-controller {
+				interrupt-controller;
+				#interrupt-cells = <1>;
+				interrupt-parent = <&gic>;
+				interrupts = <GIC_SPI 52 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 53 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 54 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 55 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 56 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 57 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 58 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 59 IRQ_TYPE_EDGE_RISING>;
+			};
+
+			pcie_intc0: legacy-interrupt-controller {
+				interrupt-controller;
+				#interrupt-cells = <2>;
+				interrupt-parent = <&gic>;
+				interrupts = <GIC_SPI 48 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 49 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 50 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 51 IRQ_TYPE_EDGE_RISING>;
+			};
+		};
+
+		pcie0_ep: pcie-ep@21800000 {
+			compatible = "ti,keystone-pcie-ep";
+			power-domains = <&k2g_pds 0x0019>;
+			clocks = <&k2g_clks 0x19 0x0>;
+			clock-names = "fck";
+			#size-cells = <2>;
+			reg =  <0x21800000 0x1000>, <0x21801000 0x2000>, <0x70000000 0x10000000>;
+			reg-names = "app", "dbics", "addr_space";
+			interrupts = <GIC_SPI 60 IRQ_TYPE_EDGE_RISING>;
+			num-lanes = <1>;
+			num-ib-windows = <4>;
+			num-ob-windows = <32>;
+			ti,syscon-pcie-mode = <&pcie_mode>;
+			dma-coherent;
+			status = "disabled";
+		};
 	};
 };
diff -Naur --no-dereference a/arch/arm/boot/dts/keystone-k2g-evm.dts b/arch/arm/boot/dts/keystone-k2g-evm.dts
--- a/arch/arm/boot/dts/keystone-k2g-evm.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/keystone-k2g-evm.dts	2020-06-19 21:00:31.404227715 -0400
@@ -6,6 +6,8 @@
  */
 /dts-v1/;
 
+#include <dt-bindings/sound/ti-mcasp.h>
+#include <dt-bindings/sound/sii902x-audio.h>
 #include "keystone-k2g.dtsi"
 
 / {
@@ -22,6 +24,13 @@
 		#size-cells = <2>;
 		ranges;
 
+		dsp_common_mpm_memory: dsp-common-mpm-memory@81d000000 {
+			compatible = "ti,keystone-dsp-mem-pool";
+			reg = <0x00000008 0x1d000000 0x00000000 0x2800000>;
+			no-map;
+			status = "okay";
+		};
+
 		dsp_common_memory: dsp-common-memory@81f800000 {
 			compatible = "shared-dma-pool";
 			reg = <0x00000008 0x1f800000 0x00000000 0x800000>;
@@ -41,10 +50,80 @@
 	vcc1v8_ldo1_reg: fixedregulator-vcc1v8-ldo1 {
 		compatible = "regulator-fixed";
 		regulator-name = "ldo1";
+	};
+
+	/* This is actually coming from TPS659118:LDO2_1V8 */
+	vcc1v8_aud_reg: fixedregulator-vcc1v8-aud {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc1v8_aud_fixed";
 		regulator-min-microvolt = <1800000>;
 		regulator-max-microvolt = <1800000>;
 		regulator-always-on;
 	};
+
+	hdmi: connector {
+		compatible = "hdmi-connector";
+		label = "hdmi";
+
+		type = "a";
+
+		port {
+			hdmi_connector_in: endpoint {
+				remote-endpoint = <&sii9022_out>;
+			};
+		};
+	};
+
+	sound0: sound@0 {
+		compatible = "simple-audio-card";
+		simple-audio-card,name = "K2G-EVM";
+		simple-audio-card,widgets =
+			"Headphone", "Headphone Jack",
+			"Line", "Line In";
+		simple-audio-card,routing =
+			"Headphone Jack",	"HPLOUT",
+			"Headphone Jack",	"HPROUT",
+			"LINE1L",		"Line In",
+			"LINE1R",		"Line In";
+
+		simple-audio-card,dai-link@0 {
+			format = "i2s";
+			bitclock-master = <&sound0_0_master>;
+			frame-master = <&sound0_0_master>;
+			sound0_0_master: cpu {
+				sound-dai = <&mcasp2>;
+				clocks = <&k2g_clks 0x6 1>;
+				system-clock-id = <MCASP_CLK_HCLK_AUXCLK>;
+			};
+
+			codec {
+				sound-dai = <&tlv320aic3106>;
+				system-clock-frequency = <12288000>;
+			};
+		};
+
+		simple-audio-card,dai-link@1 {
+			format = "i2s";
+			bitclock-master = <&sound0_1_master>;
+			frame-master = <&sound0_1_master>;
+			sound0_1_master: cpu {
+				sound-dai = <&mcasp2>;
+				clocks = <&k2g_clks 0x6 1>;
+				system-clock-id = <MCASP_CLK_HCLK_AUXCLK>;
+			};
+
+			codec {
+				sound-dai = <&sii9022>;
+				system-clock-frequency = <12288000>;
+			};
+		};
+	};
+
+	sii9022_mclk: sii9022_mclk {
+               compatible = "fixed-clock";
+               #clock-cells = <0>;
+               clock-frequency = <12288000>;
+       };
 };
 
 &k2g_pinctrl {
@@ -89,6 +168,13 @@
 		>;
 	};
 
+	i2c1_pins: pinmux_i2c1_pins {
+		pinctrl-single,pins = <
+			K2G_CORE_IOPAD(0x1384) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE0)	/* i2c1_scl.i2c1_scl */
+			K2G_CORE_IOPAD(0x1388) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE0)	/* i2c1_sda.i2c1_sda */
+		>;
+	};
+
 	ecap0_pins: ecap0_pins {
 		pinctrl-single,pins = <
 			K2G_CORE_IOPAD(0x1374) (BUFFER_CLASS_B | MUX_MODE4)	/* pr1_mdio_data.ecap0_in_apwm0_out */
@@ -160,6 +246,49 @@
 			K2G_CORE_IOPAD(0x1188) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0)	/* MDIO_DATA.MDIO_DATA */
 		>;
 	};
+
+	vout_pins: pinmux_vout_pins {
+		pinctrl-single,pins = <
+			K2G_CORE_IOPAD(0x1078) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dssdata23.dssdata23 */
+			K2G_CORE_IOPAD(0x107c) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dssdata22.dssdata22 */
+			K2G_CORE_IOPAD(0x1080) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dssdata21.dssdata21 */
+			K2G_CORE_IOPAD(0x1084) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dssdata20.dssdata20 */
+			K2G_CORE_IOPAD(0x1088) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dssdata19.dssdata19 */
+			K2G_CORE_IOPAD(0x108c) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dssdata18.dssdata18 */
+			K2G_CORE_IOPAD(0x1090) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dssdata17.dssdata17 */
+			K2G_CORE_IOPAD(0x1094) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dssdata16.dssdata16 */
+			K2G_CORE_IOPAD(0x1098) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dssdata15.dssdata15 */
+			K2G_CORE_IOPAD(0x109c) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dssdata14.dssdata14 */
+			K2G_CORE_IOPAD(0x10a0) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dssdata13.dssdata13 */
+			K2G_CORE_IOPAD(0x10a4) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dssdata12.dssdata12 */
+			K2G_CORE_IOPAD(0x10a8) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dssdata11.dssdata11 */
+			K2G_CORE_IOPAD(0x10ac) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dssdata10.dssdata10 */
+			K2G_CORE_IOPAD(0x10b0) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dssdata9.dssdata9 */
+			K2G_CORE_IOPAD(0x10b4) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dssdata8.dssdata8 */
+			K2G_CORE_IOPAD(0x10b8) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dssdata7.dssdata7 */
+			K2G_CORE_IOPAD(0x10bc) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dssdata6.dssdata6 */
+			K2G_CORE_IOPAD(0x10c0) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dssdata5.dssdata5 */
+			K2G_CORE_IOPAD(0x10c4) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dssdata4.dssdata4 */
+			K2G_CORE_IOPAD(0x10c8) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dssdata3.dssdata3 */
+			K2G_CORE_IOPAD(0x10cc) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dssdata2.dssdata2 */
+			K2G_CORE_IOPAD(0x10d0) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dssdata1.dssdata1 */
+			K2G_CORE_IOPAD(0x10d4) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dssdata0.dssdata0 */
+			K2G_CORE_IOPAD(0x10d8) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dssvsync.dssvsync */
+			K2G_CORE_IOPAD(0x10dc) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dsshsync.dsshsync */
+			K2G_CORE_IOPAD(0x10e0) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dsspclk.dsspclk */
+			K2G_CORE_IOPAD(0x10e4) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dssde.dssde */
+			K2G_CORE_IOPAD(0x10e8) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* dssfid.dssfid */
+		>;
+	};
+
+	mcasp2_pins: pinmux_mcasp2_pins {
+		pinctrl-single,pins = <
+			K2G_CORE_IOPAD(0x1234) (BUFFER_CLASS_B | PIN_PULLDOWN | MUX_MODE4)	/* pr0_pru_gpo2.mcasp2_axr2 */
+			K2G_CORE_IOPAD(0x1238) (BUFFER_CLASS_B | PIN_PULLDOWN | MUX_MODE4)	/* pr0_pru_gpo3.mcasp2_axr3 */
+			K2G_CORE_IOPAD(0x1254) (BUFFER_CLASS_B | PIN_PULLDOWN | MUX_MODE4)	/* pr0_pru_gpo10.mcasp2_afsx */
+			K2G_CORE_IOPAD(0x125c) (BUFFER_CLASS_B | PIN_PULLDOWN | MUX_MODE4)	/* pr0_pru_gpo12.mcasp2_aclkx */
+		>;
+	};
 };
 
 &uart0 {
@@ -190,6 +319,14 @@
 	status = "okay";
 };
 
+&pcie0_phy {
+	status = "okay";
+};
+
+&pcie0 {
+	status = "okay";
+};
+
 &dsp0 {
 	memory-region = <&dsp_common_memory>;
 	status = "okay";
@@ -357,3 +494,98 @@
 	pinctrl-0 = <&emac_pins>;
 	status = "okay";
 };
+
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_pins>;
+	status = "okay";
+	clock-frequency = <400000>;
+
+	sii9022: sii9022@3b {
+		#sound-dai-cells = <0>;
+		compatible = "sil,sii9022";
+		reg = <0x3b>;
+
+		i2s-fifo-routing = <
+			(ENABLE_BIT|CONNECT_SD0)
+			0
+			0
+			0
+		>;
+		clocks = <&sii9022_mclk>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				sii9022_in: endpoint {
+					remote-endpoint = <&dpi_out>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				sii9022_out: endpoint {
+					remote-endpoint = <&hdmi_connector_in>;
+				};
+			};
+		};
+	};
+
+	tlv320aic3106: tlv320aic3106@1b {
+		#sound-dai-cells = <0>;
+		compatible = "ti,tlv320aic3106";
+		reg = <0x1b>;
+		status = "okay";
+
+		/* Regulators */
+		AVDD-supply = <&vcc3v3_dcin_reg>;
+		IOVDD-supply = <&vcc3v3_dcin_reg>;
+		DRVDD-supply = <&vcc3v3_dcin_reg>;
+		DVDD-supply = <&vcc1v8_aud_reg>;
+	};
+};
+
+&dss {
+	pinctrl-names = "default";
+	pinctrl-0 = <&vout_pins>;
+	status = "ok";
+
+	port {
+		dpi_out: endpoint {
+			remote-endpoint = <&sii9022_in>;
+			data-lines = <24>;
+		};
+	};
+};
+
+&k2g_clks {
+	/* on the board 22.5792MHz is connected to AUDOSC_IN */
+	assigned-clocks = <&k2g_clks 0x4c 2>;
+	assigned-clock-rates = <22579200>;
+};
+
+&mcasp2 {
+	#sound-dai-cells = <0>;
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcasp2_pins>;
+
+	assigned-clocks = <&k2g_clks 0x6 1>;
+	assigned-clock-parents = <&k2g_clks 0x6 2>;
+
+	status = "okay";
+
+	op-mode = <0>;		/* MCASP_IIS_MODE */
+	tdm-slots = <2>;
+	/* 6 serializer */
+	serial-dir = <	/* 0: INACTIVE, 1: TX, 2: RX */
+		0 0 1 2 0 0 // AXR2: TX, AXR3: rx
+	>;
+	tx-num-evt = <32>;
+	rx-num-evt = <32>;
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/keystone-k2g-evm-lcd.dts b/arch/arm/boot/dts/keystone-k2g-evm-lcd.dts
--- a/arch/arm/boot/dts/keystone-k2g-evm-lcd.dts	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/keystone-k2g-evm-lcd.dts	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,91 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Source for K2G EVM with LCD Display
+ *
+ * Copyright (C) 2016-2018 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+#include "keystone-k2g-evm.dts"
+#include <dt-bindings/pwm/pwm.h>
+
+/delete-node/ &hdmi;
+/delete-node/ &sii9022;
+/delete-node/ &sound0;
+
+/ {
+	lcd0: display {
+		compatible = "newhaven,nhd-4.3-480272ef-atxl";
+		label = "lcd";
+
+		backlight = <&lcd_bl>;
+
+		port {
+			lcd_in: endpoint {
+				remote-endpoint = <&dpi_out>;
+			};
+		};
+	};
+
+	lcd_bl: backlight {
+		compatible = "pwm-backlight";
+		pwms = <&ecap0 0 50000 PWM_POLARITY_INVERTED>;
+		brightness-levels = <0 32 64 96 128 160 192 224 255>;
+		default-brightness-level = <8>;
+	};
+
+	sound1: sound@1 {
+		compatible = "simple-audio-card";
+		simple-audio-card,name = "K2G-EVM-LCD";
+		simple-audio-card,widgets =
+			"Headphone", "Headphone Jack",
+			"Line", "Line In";
+		simple-audio-card,routing =
+			"Headphone Jack",       "HPLOUT",
+			"Headphone Jack",       "HPROUT",
+			"LINE1L",               "Line In",
+			"LINE1R",               "Line In";
+		simple-audio-card,format = "i2s";
+		simple-audio-card,bitclock-master = <&sound1_master>;
+		simple-audio-card,frame-master = <&sound1_master>;
+
+		sound1_master: simple-audio-card,cpu {
+			sound-dai = <&mcasp2>;
+			clocks = <&k2g_clks 0x6 1>;
+			system-clock-id = <MCASP_CLK_HCLK_AUXCLK>;
+		};
+
+		simple-audio-card,codec {
+			sound-dai = <&tlv320aic3106>;
+			system-clock-frequency = <12288000>;
+		};
+	};
+};
+
+&i2c1 {
+	edt-ft5306@38 {
+		status = "okay";
+		compatible = "edt,edt-ft5306", "edt,edt-ft5x06";
+		reg = <0x38>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&edt_ft5306_ts_pins>;
+
+		interrupt-parent = <&gpio1>;
+		interrupts = <42 IRQ_TYPE_EDGE_FALLING>;
+
+		touchscreen-size-x = <480>;
+		touchscreen-size-y = <272>;
+	};
+};
+
+&k2g_pinctrl {
+	edt_ft5306_ts_pins: edt_ft5306_ts_pins {
+		pinctrl-single,pins = <
+			K2G_CORE_IOPAD(0x1364) (BUFFER_CLASS_B | PIN_PULLDOWN | MUX_MODE3)	/* pr1_pru1_gpo16.gpio1_42 */
+		>;
+	};
+};
+
+&dpi_out {
+	remote-endpoint = <&lcd_in>;
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/keystone-k2g-ice.dts b/arch/arm/boot/dts/keystone-k2g-ice.dts
--- a/arch/arm/boot/dts/keystone-k2g-ice.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/keystone-k2g-ice.dts	2020-06-19 21:00:31.404227715 -0400
@@ -23,6 +23,13 @@
 		#size-cells = <2>;
 		ranges;
 
+		dsp_common_mpm_memory: dsp-common-mpm-memory@81d000000 {
+			compatible = "ti,keystone-dsp-mem-pool";
+			reg = <0x00000008 0x1d000000 0x00000000 0x2800000>;
+			no-map;
+			status = "okay";
+		};
+
 		dsp_common_memory: dsp-common-memory@81f800000 {
 			compatible = "shared-dma-pool";
 			reg = <0x00000008 0x1f800000 0x00000000 0x800000>;
@@ -31,6 +38,13 @@
 		};
 	};
 
+	aliases {
+		ethernet1 = &pruss0_emac0;
+		ethernet2 = &pruss0_emac1;
+		ethernet3 = &pruss1_emac0;
+		ethernet4 = &pruss1_emac1;
+	};
+
 	vmain: fixedregulator-vmain {
 		compatible = "regulator-fixed";
 		regulator-name = "vmain_fixed";
@@ -215,6 +229,70 @@
 			linux,default-trigger = "heartbeat";
 		};
 	};
+
+	/* Dual-MAC Ethernet application node on PRU-ICSS0 */
+	pruss0_eth {
+		compatible = "ti,k2g-prueth";
+		prus = <&pru0_0>, <&pru0_1>;
+		firmware-name = "ti-pruss/k2g-pru0-prueth-fw.elf",
+				"ti-pruss/k2g-pru1-prueth-fw.elf";
+		ti,pruss-gp-mux-sel = <2>,	/* MII, needed for PR0_MII0 & PR0_MII1 */
+				      <2>;	/* MII, needed for PR0_MII1 & PR0_MII1 */
+		sram = <&msm_ram>;
+		interrupt-parent = <&pruss0_intc>;
+		mii-rt = <&pruss0_mii_rt>;
+		iep = <&pruss0_iep>;
+
+		pruss0_emac0: ethernet-mii0 {
+			phy-handle = <&pruss0_eth0_phy>;
+			phy-mode = "mii";
+			interrupts = <20>, <22>;
+			interrupt-names = "rx", "tx";
+			/* Filled in by bootloader */
+			local-mac-address = [00 00 00 00 00 00];
+		};
+
+		pruss0_emac1: ethernet-mii1 {
+			phy-handle = <&pruss0_eth1_phy>;
+			phy-mode = "mii";
+			interrupts = <21>, <23>;
+			interrupt-names = "rx", "tx";
+			/* Filled in by bootloader */
+			local-mac-address = [00 00 00 00 00 00];
+		};
+	};
+
+	/* Dual-MAC Ethernet application node on PRU-ICSS1 */
+	pruss1_eth {
+		compatible = "ti,k2g-prueth";
+		prus = <&pru1_0>, <&pru1_1>;
+		firmware-name = "ti-pruss/k2g-pru0-prueth-fw.elf",
+				"ti-pruss/k2g-pru1-prueth-fw.elf";
+		ti,pruss-gp-mux-sel = <2>,	/* MII, needed for PR1_MII0 & PR1_MII1 */
+				      <2>;	/* MII, needed for PR1_MII1 & PR1_MII1 */
+		sram = <&msm_ram>;
+		interrupt-parent = <&pruss1_intc>;
+		mii-rt = <&pruss1_mii_rt>;
+		iep = <&pruss1_iep>;
+
+		pruss1_emac0: ethernet-mii0 {
+			phy-handle = <&pruss1_eth0_phy>;
+			phy-mode = "mii";
+			interrupts = <20>, <22>;
+			interrupt-names = "rx", "tx";
+			/* Filled in by bootloader */
+			local-mac-address = [00 00 00 00 00 00];
+		};
+
+		pruss1_emac1: ethernet-mii1 {
+			phy-handle = <&pruss1_eth1_phy>;
+			phy-mode = "mii";
+			interrupts = <21>, <23>;
+			interrupt-names = "rx", "tx";
+			/* Filled in by bootloader */
+			local-mac-address = [00 00 00 00 00 00];
+		};
+	};
 };
 
 &k2g_pinctrl {
@@ -306,6 +384,123 @@
 			K2G_CORE_IOPAD(0x1188) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0)	/* MDIO_DATA.MDIO_DATA */
 		>;
 	};
+
+	pruss0_mdio_default: pruss0_mdio_default {
+		pinctrl-single,pins = <
+			K2G_CORE_IOPAD(0x12cc) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0)	/* pr0_mdio_data.pr0_mdio_data */
+			K2G_CORE_IOPAD(0x12d0) (BUFFER_CLASS_B | PIN_PULLDOWN | MUX_MODE0)	/* pr0_mdio_mdclk.pr0_mdio_mdclk */
+			K2G_CORE_IOPAD(0x105c) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE3)	/* gpmc_wait1.gpio0_23 (pr0_mii0_resetn) */
+			K2G_CORE_IOPAD(0x1070) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE3)	/* gpmc_csn2.gpio0_28 (pr0_mii0_intn) */
+			K2G_CORE_IOPAD(0x1054) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE3)	/* gpmc_be1n.gpio0_21 (pr0_mii1_resetn) */
+			K2G_CORE_IOPAD(0x1074) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE3)	/* gpmc_csn3.gpio0_29 (pr0_mii1_intn) */
+		>;
+	};
+
+	pruss1_mdio_default: pruss1_mdio_default {
+		pinctrl-single,pins = <
+			K2G_CORE_IOPAD(0x1374) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0)	/* pr1_mdio_data.pr1_mdio_data */
+			K2G_CORE_IOPAD(0x1378) (BUFFER_CLASS_B | PIN_PULLDOWN | MUX_MODE0)	/* pr1_mdio_mdclk.pr1_mdio_mdclk */
+			K2G_CORE_IOPAD(0x1050) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE3)	/* gpmc_be0ncle.gpio0_20 (pr1_mii0_resetn) */
+			K2G_CORE_IOPAD(0x1044) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE3)	/* gpmc_advnale.gpio0_17 (pr1_mii0_intn) */
+			K2G_CORE_IOPAD(0x1060) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE3)	/* gpmc_wpn.gpio0_24 (pr1_mii1_resetn) */
+			K2G_CORE_IOPAD(0x1088) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE3)	/* gpmc_wait0.gpio0_22 (pr1_mii1_intn) */
+		>;
+	};
+
+	pruss0_eth_default: pruss0_eth_default {
+		pinctrl-single,pins = <
+			/* PRUSS0 External Mux routing */
+			K2G_CORE_IOPAD(0x11d4) (BUFFER_CLASS_B | MUX_MODE3)			/* uart0_ctsn.gpio0_106 */
+			K2G_CORE_IOPAD(0x11d8) (BUFFER_CLASS_B | MUX_MODE3)			/* uart0_rtsn.gpio0_107 */
+			K2G_CORE_IOPAD(0x11fc) (BUFFER_CLASS_B | MUX_MODE3)			/* dcan0_rx.gpio1_57 */
+			K2G_CORE_IOPAD(0x1200) (BUFFER_CLASS_B | MUX_MODE3)			/* dcan0_tx.gpio1_56 */
+			K2G_CORE_IOPAD(0x1224) (BUFFER_CLASS_B | MUX_MODE3)			/* qspi_csn2.gpio1_66 */
+			K2G_CORE_IOPAD(0x1228) (BUFFER_CLASS_B | MUX_MODE3)			/* qspi_csn3.gpio1_67 */
+
+			/* PRUSS0 PRU0 Ethernet */
+			K2G_CORE_IOPAD(0x122c) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr0_pru0_gpo0.pr0_pru0_gpi0 (pr0_mii0_rxd0) */
+			K2G_CORE_IOPAD(0x1230) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr0_pru0_gpo1.pr0_pru0_gpi1 (pr0_mii0_rxd1) */
+			K2G_CORE_IOPAD(0x1234) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr0_pru0_gpo2.pr0_pru0_gpi2 (pr0_mii0_rxd2) */
+			K2G_CORE_IOPAD(0x1238) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr0_pru0_gpo3.pr0_pru0_gpi3 (pr0_mii0_rxd3) */
+			K2G_CORE_IOPAD(0x123c) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr0_pru0_gpo4.pr0_pru0_gpi4 (pr0_mii0_rxdv) */
+			K2G_CORE_IOPAD(0x1240) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr0_pru0_gpo5.pr0_pru0_gpi5 (pr0_mii0_rxer) */
+			K2G_CORE_IOPAD(0x1244) (BUFFER_CLASS_B | MUX_MODE1)			/* pr0_pru0_gpo6.pr0_pru0_gpi6 (pr0_mii_mr0_clk) */
+
+			K2G_CORE_IOPAD(0x124c) (BUFFER_CLASS_B | MUX_MODE1)			/* pr0_pru0_gpo8.pr0_pru0_gpi8 (pr0_mii0_rxlink) */
+			K2G_CORE_IOPAD(0x1250) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr0_pru0_gpo9.pr0_pru0_gpi9 (pr0_mii0_col) */
+			K2G_CORE_IOPAD(0x1254) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr0_pru0_gpo10.pr0_pru0_gpi10 (pr0_mii0_crs) */
+
+			K2G_CORE_IOPAD(0x12a8) (BUFFER_CLASS_B | MUX_MODE0)			/* pr0_pru1_gpo11.pr0_pru1_gpo11 (pr0_mii0_txd0) */
+			K2G_CORE_IOPAD(0x12ac) (BUFFER_CLASS_B | MUX_MODE0)			/* pr0_pru1_gpo12.pr0_pru1_gpo12 (pr0_mii0_txd1) */
+			K2G_CORE_IOPAD(0x12b0) (BUFFER_CLASS_B | MUX_MODE0)			/* pr0_pru1_gpo13.pr0_pru1_gpo13 (pr0_mii0_txd2) */
+			K2G_CORE_IOPAD(0x12b4) (BUFFER_CLASS_B | MUX_MODE0)			/* pr0_pru1_gpo14.pr0_pru1_gpo14 (pr0_mii0_txd3) */
+			K2G_CORE_IOPAD(0x12b8) (BUFFER_CLASS_B | MUX_MODE0)			/* pr0_pru1_gpo15.pr0_pru1_gpo15 (pr0_mii0_txen) */
+			K2G_CORE_IOPAD(0x12bc) (BUFFER_CLASS_B | MUX_MODE1)			/* pr0_pru1_gpo16.pr0_pru1_gpo16 (pr0_mii_mt0_clk) */
+
+			/* PRUSS0 PRU1 Ethernet */
+			K2G_CORE_IOPAD(0x127c) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr0_pru1_gpo0.pr0_pru1_gpi0 (pr0_mii1_rxd0) */
+			K2G_CORE_IOPAD(0x1280) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr0_pru1_gpo1.pr0_pru1_gpi1 (pr0_mii1_rxd1) */
+			K2G_CORE_IOPAD(0x1284) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr0_pru1_gpo2.pr0_pru1_gpi2 (pr0_mii1_rxd2) */
+			K2G_CORE_IOPAD(0x1288) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr0_pru1_gpo3.pr0_pru1_gpi3 (pr0_mii1_rxd3) */
+			K2G_CORE_IOPAD(0x128c) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr0_pru1_gpo4.pr0_pru1_gpi4 (pr0_mii1_rxdv) */
+			K2G_CORE_IOPAD(0x1290) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr0_pru1_gpo5.pr0_pru1_gpi5 (pr0_mii1_rxer) */
+			K2G_CORE_IOPAD(0x1294) (BUFFER_CLASS_B | MUX_MODE1)			/* pr0_pru1_gpo6.pr0_pru1_gpi6 (pr0_mii_mr1_clk) */
+
+			K2G_CORE_IOPAD(0x129c) (BUFFER_CLASS_B | MUX_MODE1)			/* pr0_pru1_gpo8.pr0_pru1_gpi8 (pr0_mii1_rxlink) */
+			K2G_CORE_IOPAD(0x12a0) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr0_pru1_gpo9.pr0_pru1_gpi9 (pr0_mii1_col) */
+			K2G_CORE_IOPAD(0x12a4) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr0_pru1_gpo10.pr0_pru1_gpi10 (pr0_mii1_crs) */
+
+			K2G_CORE_IOPAD(0x1258) (BUFFER_CLASS_B | MUX_MODE0)			/* pr0_pru0_gpo11.pr0_pru0_gpo11 (pr0_mii1_txd0) */
+			K2G_CORE_IOPAD(0x125c) (BUFFER_CLASS_B | MUX_MODE0)			/* pr0_pru0_gpo12.pr0_pru0_gpo12 (pr0_mii1_txd1) */
+			K2G_CORE_IOPAD(0x1260) (BUFFER_CLASS_B | MUX_MODE0)			/* pr0_pru0_gpo13.pr0_pru0_gpo13 (pr0_mii1_txd2) */
+			K2G_CORE_IOPAD(0x1264) (BUFFER_CLASS_B | MUX_MODE0)			/* pr0_pru0_gpo14.pr0_pru0_gpo14 (pr0_mii1_txd3) */
+			K2G_CORE_IOPAD(0x1268) (BUFFER_CLASS_B | MUX_MODE0)			/* pr0_pru0_gpo15.pr0_pru0_gpo15 (pr0_mii1_txen) */
+			K2G_CORE_IOPAD(0x126c) (BUFFER_CLASS_B | MUX_MODE1)			/* pr0_pru0_gpo16.pr0_pru0_gpo16 (pr0_mii_mt1_clk) */
+		>;
+	};
+
+	pruss1_eth_default: pruss1_eth_default {
+		pinctrl-single,pins = <
+			/* PRUSS1 PRU0 Ethernet */
+			K2G_CORE_IOPAD(0x12d4) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr1_pru0_gpo0.pr1_pru0_gpi0 (pr1_mii0_rxd0) */
+			K2G_CORE_IOPAD(0x12d8) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr1_pru0_gpo1.pr1_pru0_gpi1 (pr1_mii0_rxd1) */
+			K2G_CORE_IOPAD(0x12dc) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr1_pru0_gpo2.pr1_pru0_gpi2 (pr1_mii0_rxd2) */
+			K2G_CORE_IOPAD(0x12e0) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr1_pru0_gpo3.pr1_pru0_gpi3 (pr1_mii0_rxd3) */
+			K2G_CORE_IOPAD(0x12e4) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr1_pru0_gpo4.pr1_pru0_gpi4 (pr1_mii0_rxdv) */
+			K2G_CORE_IOPAD(0x12e8) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr1_pru0_gpo5.pr1_pru0_gpi5 (pr1_mii0_rxer) */
+			K2G_CORE_IOPAD(0x12ec) (BUFFER_CLASS_B | MUX_MODE1)			/* pr1_pru0_gpo6.pr1_pru0_gpi6 (pr1_mii_mr0_clk) */
+
+			K2G_CORE_IOPAD(0x12f4) (BUFFER_CLASS_B | MUX_MODE1)			/* pr1_pru0_gpo8.pr1_pru0_gpi8 (pr1_mii0_rxlink) */
+			K2G_CORE_IOPAD(0x12f8) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr1_pru0_gpo9.pr1_pru0_gpi9 (pr1_mii0_col) */
+			K2G_CORE_IOPAD(0x12fc) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr1_pru0_gpo10.pr1_pru0_gpi10 (pr1_mii0_crs) */
+
+			K2G_CORE_IOPAD(0x1350) (BUFFER_CLASS_B | MUX_MODE0)			/* pr1_pru1_gpo11.pr1_pru1_gpo11 (pr1_mii0_txd0) */
+			K2G_CORE_IOPAD(0x1354) (BUFFER_CLASS_B | MUX_MODE0)			/* pr1_pru1_gpo12.pr1_pru1_gpo12 (pr1_mii0_txd1) */
+			K2G_CORE_IOPAD(0x1358) (BUFFER_CLASS_B | MUX_MODE0)			/* pr1_pru1_gpo13.pr1_pru1_gpo13 (pr1_mii0_txd2) */
+			K2G_CORE_IOPAD(0x135c) (BUFFER_CLASS_B | MUX_MODE0)			/* pr1_pru1_gpo14.pr1_pru1_gpo14 (pr1_mii0_txd3) */
+			K2G_CORE_IOPAD(0x1360) (BUFFER_CLASS_B | MUX_MODE0)			/* pr1_pru1_gpo15.pr1_pru1_gpo15 (pr1_mii0_txen) */
+			K2G_CORE_IOPAD(0x1364) (BUFFER_CLASS_B | MUX_MODE1)			/* pr1_pru1_gpo16.pr1_pru1_gpo16 (pr1_mii_mt0_clk) */
+
+			/* PRUSS1 PRU1 Ethernet */
+			K2G_CORE_IOPAD(0x1324) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr1_pru1_gpo0.pr1_pru1_gpi0 (pr1_mii1_rxd0) */
+			K2G_CORE_IOPAD(0x132c) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr1_pru1_gpo2.pr1_pru1_gpi2 (pr1_mii1_rxd2) */
+			K2G_CORE_IOPAD(0x1330) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr1_pru1_gpo3.pr1_pru1_gpi3 (pr1_mii1_rxd3) */
+			K2G_CORE_IOPAD(0x1334) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr1_pru1_gpo4.pr1_pru1_gpi4 (pr1_mii1_rxdv) */
+			K2G_CORE_IOPAD(0x1338) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr1_pru1_gpo5.pr1_pru1_gpi5 (pr1_mii1_rxer) */
+			K2G_CORE_IOPAD(0x133c) (BUFFER_CLASS_B | MUX_MODE1)			/* pr1_pru1_gpo6.pr1_pru1_gpi6 (pr1_mii_mr1_clk) */
+
+			K2G_CORE_IOPAD(0x1344) (BUFFER_CLASS_B | MUX_MODE1)			/* pr1_pru1_gpo8.pr1_pru1_gpi8 (pr1_mii1_rxlink) */
+			K2G_CORE_IOPAD(0x1348) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr1_pru1_gpo9.pr1_pru1_gpi9 (pr1_mii1_col) */
+			K2G_CORE_IOPAD(0x134c) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE1)	/* pr1_pru1_gpo10.pr1_pru1_gpi10 (pr1_mii1_crs) */
+
+			K2G_CORE_IOPAD(0x1300) (BUFFER_CLASS_B | MUX_MODE0)			/* pr1_pru0_gpo11.pr1_pru0_gpo11 (pr1_mii1_txd0) */
+			K2G_CORE_IOPAD(0x1304) (BUFFER_CLASS_B | MUX_MODE0)			/* pr1_pru0_gpo12.pr1_pru0_gpo12 (pr1_mii1_txd1) */
+			K2G_CORE_IOPAD(0x1308) (BUFFER_CLASS_B | MUX_MODE0)			/* pr1_pru0_gpo13.pr1_pru0_gpo13 (pr1_mii1_txd2) */
+			K2G_CORE_IOPAD(0x130c) (BUFFER_CLASS_B | MUX_MODE0)			/* pr1_pru0_gpo14.pr1_pru0_gpo14 (pr1_mii1_txd3) */
+			K2G_CORE_IOPAD(0x1310) (BUFFER_CLASS_B | MUX_MODE0)			/* pr1_pru0_gpo15.pr1_pru0_gpo15 (pr1_mii1_txen) */
+			K2G_CORE_IOPAD(0x1314) (BUFFER_CLASS_B | MUX_MODE1)			/* pr1_pru0_gpo16.pr1_pru0_gpo16 (pr1_mii_mt1_clk) */
+		>;
+	};
 };
 
 &uart0 {
@@ -410,6 +605,21 @@
 		#gpio-cells = <2>;
 		vcc-supply = <&vdd_3v3>;
 	};
+
+	/* osd9616p0899-10 */
+	display@3c {
+		compatible = "solomon,ssd1306fb-i2c";
+		reg = <0x3c>;
+		reset-gpios = <&gpio1 52 GPIO_ACTIVE_HIGH>;
+		vbat-supply = <&vddb>;
+		solomon,height = <16>;
+		solomon,width = <96>;
+		solomon,com-seq;
+		solomon,com-invdir;
+		solomon,page-offset = <0>;
+		solomon,prechargep1 = <2>;
+		solomon,prechargep2 = <13>;
+	};
 };
 
 &qmss {
@@ -445,3 +655,63 @@
 	phy-mode = "rgmii-id";
 	status = "okay";
 };
+
+&pruss0 {
+	pinctrl-0 = <&pruss0_eth_default>;
+	pinctrl-names = "default";
+};
+
+&pruss1 {
+	pinctrl-0 = <&pruss1_eth_default>;
+	pinctrl-names = "default";
+};
+
+&pruss0_mdio {
+	status = "okay";
+	pinctrl-0 = <&pruss0_mdio_default>;
+	pinctrl-names = "default";
+	reset-gpios = <&gpio0 23 GPIO_ACTIVE_LOW>,
+		      <&gpio0 21 GPIO_ACTIVE_LOW>;
+	reset-delay-us = <20>;
+
+	pruss0_eth0_phy: ethernet-phy@0 {
+		reg = <0>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <28 IRQ_TYPE_EDGE_FALLING>;
+	};
+
+	pruss0_eth1_phy: ethernet-phy@1 {
+		reg = <1>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <29 IRQ_TYPE_EDGE_FALLING>;
+	};
+};
+
+&pruss1_mdio {
+	status = "okay";
+	pinctrl-0 = <&pruss1_mdio_default>;
+	pinctrl-names = "default";
+	reset-gpios = <&gpio0 20 GPIO_ACTIVE_LOW>,
+		      <&gpio0 24 GPIO_ACTIVE_LOW>;
+	reset-delay-us = <20>;
+
+	pruss1_eth0_phy: ethernet-phy@2 {
+		reg = <2>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <17 IRQ_TYPE_EDGE_FALLING>;
+	};
+
+	pruss1_eth1_phy: ethernet-phy@3 {
+		reg = <3>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <22 IRQ_TYPE_EDGE_FALLING>;
+	};
+};
+
+&pcie0_phy {
+	status = "okay";
+};
+
+&pcie0_ep {
+	status = "okay";
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/keystone-k2g-netcp.dtsi b/arch/arm/boot/dts/keystone-k2g-netcp.dtsi
--- a/arch/arm/boot/dts/keystone-k2g-netcp.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/keystone-k2g-netcp.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -89,6 +89,11 @@
 
 };
 
+gbe_subsys: subsys@4200000 {
+	compatible = "syscon";
+	reg = <0x4200000 0x20>;
+};
+
 netcp: netcp@4000000 {
 	reg = <0x2620110 0x8>;
 	reg-names = "efuse";
@@ -115,7 +120,8 @@
 		gbe: gbe@200000 {
 			label = "netcp-gbe";
 			compatible = "ti,netcp-gbe-2";
-			reg = <0x200000 0x20>, <0x220000 0x20000>;
+			syscon-subsys = <&gbe_subsys>;
+			reg = <0x220000 0x20000>;
 			enable-ale;
 			tx-queue = <5>;
 			tx-channel = "nettx";
diff -Naur --no-dereference a/arch/arm/boot/dts/keystone-k2hk.dtsi b/arch/arm/boot/dts/keystone-k2hk.dtsi
--- a/arch/arm/boot/dts/keystone-k2hk.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/keystone-k2hk.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -64,6 +64,11 @@
 			#address-cells = <1>;
 			#size-cells = <1>;
 
+			sram-mpm@0 {
+				compatible = "ti,keystone-dsp-msm-ram";
+				reg = <0x0 0x80000>;
+			};
+
 			sram-bm@5f0000 {
 				reg = <0x5f0000 0x8000>;
 			};
diff -Naur --no-dereference a/arch/arm/boot/dts/keystone-k2hk-evm.dts b/arch/arm/boot/dts/keystone-k2hk-evm.dts
--- a/arch/arm/boot/dts/keystone-k2hk-evm.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/keystone-k2hk-evm.dts	2020-06-19 21:00:31.404227715 -0400
@@ -24,6 +24,13 @@
 			reusable;
 			status = "okay";
 		};
+
+		dsp_common_mpm_memory: dsp-common-mpm-memory@820000000 {
+			compatible = "ti,keystone-dsp-mem-pool";
+			reg = <0x00000008 0x20000000 0x00000000 0x10000000>;
+			no-map;
+			status = "okay";
+		};
 	};
 
 	leds {
@@ -234,3 +241,7 @@
 	memory-region = <&dsp_common_memory>;
 	status = "okay";
 };
+
+&gbe_serdes {
+	status = "okay";
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/keystone-k2hk-netcp.dtsi b/arch/arm/boot/dts/keystone-k2hk-netcp.dtsi
--- a/arch/arm/boot/dts/keystone-k2hk-netcp.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/keystone-k2hk-netcp.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -139,6 +139,59 @@
 			  <0x2005000 0x400>;
 		reg-names = "global", "txchan", "rxchan",
 				"txsched", "rxflow";
+		ti,enable-all;
+	};
+};
+
+gbe_subsys: subsys@2090000 {
+	compatible = "syscon";
+	reg = <0x02090000 0x100>;
+};
+
+gbe_serdes: phy@232a000 {
+	compatible		= "ti,keystone-serdes-gbe";
+	reg			= <0x0232a000 0x2000>;
+	status			= "disabled";
+	link-rate-kbps		= <1250000>;
+	num-lanes		= <4>;
+	#address-cells	= <1>;
+	#size-cells	= <0>;
+
+	serdes_lane0: lane@0 {
+		#phy-cells	= <0>;
+		reg		= <0>;
+		status		= "ok";
+		control-rate	= <2>;
+		rx-start	= <7 5>;
+		rx-force	= <1 1>;
+		tx-coeff	= <0 0 0 12 4>;
+	};
+	serdes_lane1: lane@1 {
+		#phy-cells	= <0>;
+		reg		= <1>;
+		status		= "ok";
+		control-rate	= <2>;
+		rx-start	= <7 5>;
+		rx-force	= <1 1>;
+		tx-coeff	= <0 0 0 12 4>;
+	};
+	serdes_lane2: lane@2 {
+		#phy-cells	= <0>;
+		reg		= <2>;
+		status		= "disabled";
+		control-rate	= <2>;
+		rx-start	= <7 5>;
+		rx-force	= <1 1>;
+		tx-coeff	= <0 0 0 12 4>;
+	};
+	serdes_lane3: lane@3 {
+		#phy-cells	= <0>;
+		reg		= <3>;
+		status		= "disabled";
+		control-rate	= <2>;
+		rx-start	= <7 5>;
+		rx-force	= <1 1>;
+		tx-coeff	= <0 0 0 12 4>;
 	};
 };
 
@@ -170,18 +223,20 @@
 			#size-cells = <1>;
 			label = "netcp-gbe";
 			compatible = "ti,netcp-gbe";
-			reg = <0x90000 0x300>, <0x90400 0x400>, <0x90800 0x700>;
-			/* enable-ale; */
+			syscon-subsys = <&gbe_subsys>;
+			reg = <0x90100 0x200>, <0x90400 0x200>, <0x90800 0x700>;
 			tx-queue = <648>;
 			tx-channel = "nettx";
 
 			interfaces {
 				gbe0: interface-0 {
+					phys = <&serdes_lane0>;
 					slave-port = <0>;
 					link-interface = <1>;
 					phy-handle = <&ethphy0>;
 				};
 				gbe1: interface-1 {
+					phys = <&serdes_lane1>;
 					slave-port = <1>;
 					link-interface = <1>;
 					phy-handle = <&ethphy1>;
@@ -190,10 +245,12 @@
 
 			secondary-slave-ports {
 				port-2 {
+					phys = <&serdes_lane2>;
 					slave-port = <2>;
 					link-interface	= <2>;
 				};
 				port-3 {
+					phys = <&serdes_lane3>;
 					slave-port = <3>;
 					link-interface	= <2>;
 				};
diff -Naur --no-dereference a/arch/arm/boot/dts/keystone-k2l.dtsi b/arch/arm/boot/dts/keystone-k2l.dtsi
--- a/arch/arm/boot/dts/keystone-k2l.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/keystone-k2l.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -262,6 +262,11 @@
 			#address-cells = <1>;
 			#size-cells = <1>;
 
+			sram-mpm@0 {
+				compatible = "ti,keystone-dsp-msm-ram";
+				reg = <0x0 0x80000>;
+			};
+
 			sram-bm@1f8000 {
 				reg = <0x001f8000 0x8000>;
 			};
diff -Naur --no-dereference a/arch/arm/boot/dts/keystone-k2l-evm.dts b/arch/arm/boot/dts/keystone-k2l-evm.dts
--- a/arch/arm/boot/dts/keystone-k2l-evm.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/keystone-k2l-evm.dts	2020-06-19 21:00:31.404227715 -0400
@@ -24,6 +24,13 @@
 			reusable;
 			status = "okay";
 		};
+
+		dsp_common_mpm_memory: dsp-common-mpm-memory@820000000 {
+			compatible = "ti,keystone-dsp-mem-pool";
+			reg = <0x00000008 0x20000000 0x00000000 0x10000000>;
+			no-map;
+			status = "okay";
+		};
 	};
 };
 
@@ -163,3 +170,7 @@
 	memory-region = <&dsp_common_memory>;
 	status = "okay";
 };
+
+&gbe_serdes0 {
+	status = "okay";
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/keystone-k2l-netcp.dtsi b/arch/arm/boot/dts/keystone-k2l-netcp.dtsi
--- a/arch/arm/boot/dts/keystone-k2l-netcp.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/keystone-k2l-netcp.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -121,6 +121,70 @@
 			  <0x26189000 0x1000>;
 		reg-names = "global", "txchan", "rxchan",
 				"txsched", "rxflow";
+		ti,enable-all;
+	};
+};
+
+gbe_subsys: subsys@26200000 {
+	compatible = "syscon";
+	reg = <0x26200000 0x100>;
+};
+
+gbe_serdes0: phy@232a000 {
+	compatible		= "ti,keystone-serdes-gbe";
+	reg			= <0x0232a000 0x2000>;
+	status			= "disabled";
+	link-rate-kbps		= <1250000>;
+	num-lanes		= <2>;
+	#address-cells	= <1>;
+	#size-cells	= <0>;
+
+	serdes0_lane0: lane@0 {
+		#phy-cells	= <0>;
+		reg		= <0>;
+		status		= "ok";
+		control-rate	= <2>;
+		rx-start	= <7 5>;
+		rx-force	= <1 1>;
+		tx-coeff	= <0 0 0 12 4>;
+	};
+	serdes0_lane1: lane@1 {
+		#phy-cells	= <0>;
+		reg		= <1>;
+		status		= "ok";
+		control-rate	= <2>;
+		rx-start	= <7 5>;
+		rx-force	= <1 1>;
+		tx-coeff	= <0 0 0 12 4>;
+	};
+};
+
+gbe_serdes1: phy@2320000 {
+	compatible		= "ti,keystone-serdes-gbe";
+	reg			= <0x02320000 0x2000>;
+	status			= "disabled";
+	link-rate-kbps		= <1250000>;
+	num-lanes		= <2>;
+	#address-cells	= <1>;
+	#size-cells	= <0>;
+
+	serdes1_lane0: lane@0 {
+		#phy-cells	= <0>;
+		reg		= <0>;
+		status		= "disabled";
+		control-rate	= <2>;
+		rx-start	= <7 5>;
+		rx-force	= <1 1>;
+		tx-coeff	= <0 0 0 12 4>;
+	};
+	serdes1_lane1: lane@1 {
+		#phy-cells	= <0>;
+		reg		= <1>;
+		status		= "disabled";
+		control-rate	= <2>;
+		rx-start	= <7 5>;
+		rx-force	= <1 1>;
+		tx-coeff	= <0 0 0 12 4>;
 	};
 };
 
@@ -150,19 +214,21 @@
 		gbe@200000 { /* ETHSS */
 			label = "netcp-gbe";
 			compatible = "ti,netcp-gbe-5";
-			reg = <0x200000 0x900>, <0x220000 0x20000>;
-			/* enable-ale; */
+			syscon-subsys = <&gbe_subsys>;
+			reg = <0x200100 0x400>, <0x220000 0x20000>;
 			tx-queue = <896>;
 			tx-channel = "nettx";
 
 			interfaces {
 				gbe0: interface-0 {
-					slave-port = <0>;
+					phys		= <&serdes0_lane0>;
+					slave-port	= <0>;
 					link-interface	= <1>;
 					phy-handle	= <&ethphy0>;
 				};
 				gbe1: interface-1 {
-					slave-port = <1>;
+					phys		= <&serdes0_lane1>;
+					slave-port	= <1>;
 					link-interface	= <1>;
 					phy-handle	= <&ethphy1>;
 				};
@@ -170,10 +236,12 @@
 
 			secondary-slave-ports {
 				port-2 {
+					phys = <&serdes1_lane0>;
 					slave-port = <2>;
 					link-interface	= <2>;
 				};
 				port-3 {
+					phys = <&serdes1_lane1>;
 					slave-port = <3>;
 					link-interface	= <2>;
 				};
diff -Naur --no-dereference a/arch/arm/boot/dts/Makefile b/arch/arm/boot/dts/Makefile
--- a/arch/arm/boot/dts/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/Makefile	2020-06-19 21:00:31.404227715 -0400
@@ -1,4 +1,9 @@
 # SPDX-License-Identifier: GPL-2.0
+
+ifeq ($(CONFIG_OF_OVERLAY),y)
+DTC_FLAGS += -@
+endif
+
 dtb-$(CONFIG_ARCH_ALPINE) += \
 	alpine-db.dtb
 dtb-$(CONFIG_MACH_ARTPEC6) += \
@@ -230,6 +235,7 @@
 	keystone-k2l-evm.dtb \
 	keystone-k2e-evm.dtb \
 	keystone-k2g-evm.dtb \
+	keystone-k2g-evm-lcd.dtb \
 	keystone-k2g-ice.dtb
 dtb-$(CONFIG_MACH_KIRKWOOD) += \
 	kirkwood-b3.dtb \
@@ -696,7 +702,22 @@
 	am335x-base0033.dtb \
 	am335x-bone.dtb \
 	am335x-boneblack.dtb \
+	am335x-revolve.dtb \
+	am335x-boneblack-wireless-roboticscape.dtb \
+	am335x-boneblack-roboticscape.dtb \
+	am335x-bonegreen-gateway.dtb \
+	am335x-bonegreen-wireless-uboot-univ.dtb \
+	am335x-boneblack-uboot-univ.dtb \
+	am335x-boneblack-uboot.dtb \
+	am335x-bone-uboot-univ.dtb \
+	am335x-boneblack-audio.dtb \
+	am335x-boneblack-bbb-exp-r.dtb \
+	am335x-boneblack-bbb-exp-c.dtb \
+	am335x-boneblack-bbbmini.dtb \
+	am335x-boneblack-wl1835mod.dtb \
+	am335x-abbbi.dtb \
 	am335x-boneblack-wireless.dtb \
+	am335x-boneblack-prusuart.dtb \
 	am335x-boneblue.dtb \
 	am335x-bonegreen.dtb \
 	am335x-bonegreen-wireless.dtb \
@@ -705,6 +726,7 @@
 	am335x-evm.dtb \
 	am335x-evmsk.dtb \
 	am335x-icev2.dtb \
+	am335x-icev2-prueth.dtb \
 	am335x-lxm.dtb \
 	am335x-moxa-uc-8100-me-t.dtb \
 	am335x-nano.dtb \
@@ -731,8 +753,10 @@
 	omap4-var-stk-om44.dtb
 dtb-$(CONFIG_SOC_AM43XX) += \
 	am43x-epos-evm.dtb \
+	am43x-epos-evm-hdmi.dtb \
 	am437x-cm-t43.dtb \
 	am437x-gp-evm.dtb \
+	am437x-gp-evm-hdmi.dtb \
 	am437x-idk-evm.dtb \
 	am437x-sbc-t43.dtb \
 	am437x-sk-evm.dtb
@@ -743,18 +767,48 @@
 	omap5-uevm.dtb
 dtb-$(CONFIG_SOC_DRA7XX) += \
 	am57xx-beagle-x15.dtb \
+	am5729-beagleboneai-roboticscape.dtb \
 	am57xx-beagle-x15-revb1.dtb \
 	am57xx-beagle-x15-revc.dtb \
 	am57xx-cl-som-am57x.dtb \
 	am57xx-sbc-am57x.dtb \
 	am572x-idk.dtb \
-	am571x-idk.dtb \
+	am5729-beagleboneai.dtb \
 	am574x-idk.dtb \
 	dra7-evm.dtb \
 	dra72-evm.dtb \
 	dra72-evm-revc.dtb \
 	dra71-evm.dtb \
 	dra76-evm.dtb
+dtb-merge-$(CONFIG_SOC_DRA7XX) += \
+	am57xx-evm.dtb \
+	am57xx-evm-cam-mt9t111.dtb \
+	am57xx-evm-cam-ov10635.dtb \
+	am57xx-evm-reva3.dtb \
+	am57xx-evm-reva3-cam-mt9t111.dtb \
+	am57xx-evm-reva3-cam-ov10635.dtb \
+	am571x-idk.dtb \
+	am571x-idk-lcd-osd101t2045.dtb \
+	am571x-idk-lcd-osd101t2587.dtb \
+	am572x-idk-lcd-osd101t2045.dtb \
+	am572x-idk-lcd-osd101t2587.dtb \
+	am574x-idk-lcd-osd101t2587.dtb \
+	dra71-evm-nand.dtb \
+	dra71-evm-lcd-auo-g101evn01.0.dtb \
+	dra72-evm-lcd-osd101t2045.dtb \
+	dra72-evm-lcd-osd101t2587.dtb \
+	dra72-evm-revc-lcd-osd101t2045.dtb \
+	dra72-evm-revc-lcd-osd101t2587.dtb \
+	dra7-evm-lcd-osd101t2045.dtb \
+	dra7-evm-lcd-osd101t2587.dtb \
+	dra76-evm-ov490.dtb \
+	dra76-evm-ov5640.dtb \
+	dra76-evm-tfp410.dtb \
+	dra76-evm-tfp410-ov490.dtb \
+	dra76-evm-tfp410-ov5640.dtb
+ifeq ($(CONFIG_SOC_DRA7XX),y)
+dts-dirs += ti
+endif
 dtb-$(CONFIG_ARCH_ORION5X) += \
 	orion5x-kuroboxpro.dtb \
 	orion5x-lacie-d2-network.dtb \
@@ -1207,3 +1261,23 @@
 	aspeed-bmc-opp-zaius.dtb \
 	aspeed-bmc-portwell-neptune.dtb \
 	aspeed-bmc-quanta-q71l.dtb
+
+ifeq ($(BUILD_ITBS),y)
+dtb-y=
+dtb-merge-y=
+else
+dts-dirs=
+endif
+
+always		+= $(dtb-merge-y)
+
+$(addprefix $(obj)/,$(dtb-merge-y)): TI_DTBOS
+	@$(srctree)/scripts/dtb-merge $(srctree) $(objtree) $@ $(objtree)/scripts/dtc/fdtoverlay $(src)/ti
+
+TI_DTBOS:
+	$(Q)$(MAKE) $(build)=$(src)/ti
+
+PHONY += TI_DTBOS
+
+subdir-y	+= $(dts-dirs)
+subdir-		+= ti
diff -Naur --no-dereference a/arch/arm/boot/dts/omap4.dtsi b/arch/arm/boot/dts/omap4.dtsi
--- a/arch/arm/boot/dts/omap4.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/omap4.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -29,6 +29,8 @@
 		serial1 = &uart2;
 		serial2 = &uart3;
 		serial3 = &uart4;
+		rproc0 = &dsp;
+		rproc1 = &ipu;
 	};
 
 	cpus {
@@ -109,11 +111,6 @@
 			sram = <&ocmcram>;
 		};
 
-		dsp {
-			compatible = "ti,omap3-c64";
-			ti,hwmods = "dsp";
-		};
-
 		iva {
 			compatible = "ti,ivahd";
 			ti,hwmods = "iva";
@@ -424,6 +421,8 @@
 			interrupts = <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "timer5";
 			ti,timer-dsp;
+			clocks = <&abe_clkctrl OMAP4_TIMER5_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		timer6: timer@4013a000 {
@@ -433,6 +432,8 @@
 			interrupts = <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "timer6";
 			ti,timer-dsp;
+			clocks = <&abe_clkctrl OMAP4_TIMER6_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		timer7: timer@4013c000 {
@@ -442,6 +443,8 @@
 			interrupts = <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "timer7";
 			ti,timer-dsp;
+			clocks = <&abe_clkctrl OMAP4_TIMER7_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		timer8: timer@4013e000 {
@@ -452,6 +455,29 @@
 			ti,hwmods = "timer8";
 			ti,timer-pwm;
 			ti,timer-dsp;
+			clocks = <&abe_clkctrl OMAP4_TIMER8_CLKCTRL 24>;
+			clock-names = "fck";
+		};
+
+		dsp: dsp {
+			compatible = "ti,omap4-dsp";
+			ti,hwmods = "dsp";
+			syscon-bootreg = <&scm_conf 0x304>;
+			iommus = <&mmu_dsp>;
+			ti,rproc-standby-info = <0x4a004420>;
+			mboxes = <&mailbox &mbox_dsp>;
+			status = "disabled";
+		};
+
+		ipu: ipu@55020000 {
+			compatible = "ti,omap4-ipu";
+			reg = <0x55020000 0x10000>;
+			reg-names = "l2ram";
+			ti,hwmods = "ipu";
+			iommus = <&mmu_ipu>;
+			ti,rproc-standby-info = <0x4a008920>;
+			mboxes = <&mailbox &mbox_ipu>;
+			status = "disabled";
 		};
 
 		aes1: aes@4b501000 {
diff -Naur --no-dereference a/arch/arm/boot/dts/omap4-panda-common.dtsi b/arch/arm/boot/dts/omap4-panda-common.dtsi
--- a/arch/arm/boot/dts/omap4-panda-common.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/omap4-panda-common.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -14,6 +14,26 @@
 		reg = <0x80000000 0x40000000>; /* 1 GB */
 	};
 
+	reserved-memory {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		dsp_memory_region: dsp-memory@98000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x98000000 0x800000>;
+			reusable;
+			status = "okay";
+		};
+
+		ipu_memory_region: ipu-memory@98800000 {
+			compatible = "shared-dma-pool";
+			reg = <0x98800000 0x7000000>;
+			reusable;
+			status = "okay";
+		};
+	};
+
 	chosen {
 		stdout-path = &uart3;
 	};
@@ -591,3 +611,17 @@
 		};
 	};
 };
+
+&dsp {
+	status = "okay";
+	memory-region = <&dsp_memory_region>;
+	timers = <&timer5>;
+	watchdog-timers = <&timer6>;
+};
+
+&ipu {
+	status = "okay";
+	memory-region = <&ipu_memory_region>;
+	timers = <&timer3>;
+	watchdog-timers = <&timer9>, <&timer11>;
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/omap5-board-common.dtsi b/arch/arm/boot/dts/omap5-board-common.dtsi
--- a/arch/arm/boot/dts/omap5-board-common.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/omap5-board-common.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -12,6 +12,9 @@
 / {
 	aliases {
 		display0 = &hdmi0;
+
+		sound0 = &sound;
+		sound1 = &hdmi;
 	};
 
 	chosen {
diff -Naur --no-dereference a/arch/arm/boot/dts/omap5.dtsi b/arch/arm/boot/dts/omap5.dtsi
--- a/arch/arm/boot/dts/omap5.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/omap5.dtsi	2020-06-19 21:00:31.404227715 -0400
@@ -32,6 +32,8 @@
 		serial3 = &uart4;
 		serial4 = &uart5;
 		serial5 = &uart6;
+		rproc0 = &dsp;
+		rproc1 = &ipu;
 	};
 
 	cpus {
@@ -788,6 +790,8 @@
 			reg = <0x48032000 0x80>;
 			interrupts = <GIC_SPI 38 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "timer2";
+			clocks = <&l4per_clkctrl OMAP5_TIMER2_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		timer3: timer@48034000 {
@@ -795,6 +799,8 @@
 			reg = <0x48034000 0x80>;
 			interrupts = <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "timer3";
+			clocks = <&l4per_clkctrl OMAP5_TIMER3_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		timer4: timer@48036000 {
@@ -802,6 +808,8 @@
 			reg = <0x48036000 0x80>;
 			interrupts = <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "timer4";
+			clocks = <&l4per_clkctrl OMAP5_TIMER4_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		timer5: timer@40138000 {
@@ -812,6 +820,8 @@
 			ti,hwmods = "timer5";
 			ti,timer-dsp;
 			ti,timer-pwm;
+			clocks = <&abe_clkctrl OMAP5_TIMER5_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		timer6: timer@4013a000 {
@@ -822,6 +832,8 @@
 			ti,hwmods = "timer6";
 			ti,timer-dsp;
 			ti,timer-pwm;
+			clocks = <&abe_clkctrl OMAP5_TIMER6_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		timer7: timer@4013c000 {
@@ -831,6 +843,8 @@
 			interrupts = <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "timer7";
 			ti,timer-dsp;
+			clocks = <&abe_clkctrl OMAP5_TIMER7_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		timer8: timer@4013e000 {
@@ -841,6 +855,8 @@
 			ti,hwmods = "timer8";
 			ti,timer-dsp;
 			ti,timer-pwm;
+			clocks = <&abe_clkctrl OMAP5_TIMER8_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		timer9: timer@4803e000 {
@@ -849,6 +865,8 @@
 			interrupts = <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "timer9";
 			ti,timer-pwm;
+			clocks = <&l4per_clkctrl OMAP5_TIMER9_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		timer10: timer@48086000 {
@@ -857,6 +875,8 @@
 			interrupts = <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "timer10";
 			ti,timer-pwm;
+			clocks = <&l4per_clkctrl OMAP5_TIMER10_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		timer11: timer@48088000 {
@@ -865,6 +885,8 @@
 			interrupts = <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "timer11";
 			ti,timer-pwm;
+			clocks = <&l4per_clkctrl OMAP5_TIMER11_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		wdt2: wdt@4ae14000 {
@@ -874,6 +896,27 @@
 			ti,hwmods = "wd_timer2";
 		};
 
+		dsp: dsp {
+			compatible = "ti,omap5-dsp";
+			ti,hwmods = "dsp";
+			syscon-bootreg = <&scm_conf 0x304>;
+			iommus = <&mmu_dsp>;
+			ti,rproc-standby-info = <0x4a004420>;
+			mboxes = <&mailbox &mbox_dsp>;
+			status = "disabled";
+		};
+
+		ipu: ipu@55020000 {
+			compatible = "ti,omap5-ipu";
+			reg = <0x55020000 0x10000>;
+			reg-names = "l2ram";
+			ti,hwmods = "ipu";
+			iommus = <&mmu_ipu>;
+			ti,rproc-standby-info = <0x4a008920>;
+			mboxes = <&mailbox &mbox_ipu>;
+			status = "disabled";
+		};
+
 		dmm@4e000000 {
 			compatible = "ti,omap5-dmm";
 			reg = <0x4e000000 0x800>;
diff -Naur --no-dereference a/arch/arm/boot/dts/omap5-uevm.dts b/arch/arm/boot/dts/omap5-uevm.dts
--- a/arch/arm/boot/dts/omap5-uevm.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/dts/omap5-uevm.dts	2020-06-19 21:00:31.404227715 -0400
@@ -18,6 +18,26 @@
 		reg = <0 0x80000000 0 0x7f000000>; /* 2032 MB */
 	};
 
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		dsp_memory_region: dsp-memory@95000000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0x95000000 0 0x800000>;
+			reusable;
+			status = "okay";
+		};
+
+		ipu_memory_region: ipu-memory@95800000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0x95800000 0 0x3800000>;
+			reusable;
+			status = "okay";
+		};
+	};
+
 	aliases {
 		ethernet = &ethernet;
 	};
@@ -201,3 +221,17 @@
 &wlcore {
 	compatible = "ti,wl1837";
 };
+
+&dsp {
+	status = "okay";
+	memory-region = <&dsp_memory_region>;
+	timers = <&timer5>;
+	watchdog-timers = <&timer6>;
+};
+
+&ipu {
+	status = "okay";
+	memory-region = <&ipu_memory_region>;
+	timers = <&timer3>;
+	watchdog-timers = <&timer9>, <&timer11>;
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/ti/am571x-idk-icss1eth.dtso b/arch/arm/boot/dts/ti/am571x-idk-icss1eth.dtso
--- a/arch/arm/boot/dts/ti/am571x-idk-icss1eth.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/ti/am571x-idk-icss1eth.dtso	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,78 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+
+/ {
+  fragment@101 {
+	target-path = "/";
+
+	__overlay__ {
+
+		aliases {
+			ethernet4 = "/pruss1_eth/ethernet-mii0";
+			ethernet5 = "/pruss1_eth/ethernet-mii1";
+		};
+
+		/* Dual mac ethernet application node on icss1 */
+		pruss1_eth {
+			status = "okay";
+			compatible = "ti,am57-prueth";
+			prus = <&pru1_0>, <&pru1_1>;
+			firmware-name = "ti-pruss/am57xx-pru0-prueth-fw.elf",
+					"ti-pruss/am57xx-pru1-prueth-fw.elf";
+			ti,pruss-gp-mux-sel = <0>,	/* GP, default */
+					      <4>;	/* MII2, needed for PRUSS1_MII1 */
+			sram = <&ocmcram1>;
+			interrupt-parent = <&pruss1_intc>;
+			mii-rt = <&pruss1_mii_rt>;
+			iep = <&pruss1_iep>;
+
+			ethernet-mii0 {
+				phy-handle = <&pruss1_eth0_phy>;
+				phy-mode = "mii";
+				interrupts = <20>, <22>;
+				interrupt-names = "rx", "tx";
+				/* Filled in by bootloader */
+				local-mac-address = [00 00 00 00 00 00];
+			};
+
+			ethernet-mii1 {
+				phy-handle = <&pruss1_eth1_phy>;
+				phy-mode = "mii";
+				interrupts = <21>, <23>;
+				interrupt-names = "rx", "tx";
+				/* Filled in by bootloader */
+				local-mac-address = [00 00 00 00 00 00];
+			};
+		};
+	};
+  };
+};
+
+&pruss1_mdio {
+	status = "okay";
+	reset-gpios = <&gpio5 8 GPIO_ACTIVE_LOW>;
+	reset-delay-us = <2>;   /* PHY datasheet states 1uS min */
+	pruss1_eth0_phy: ethernet-phy@0 {
+		reg = <0>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <28 IRQ_TYPE_EDGE_FALLING>;
+	};
+
+	pruss1_eth1_phy: ethernet-phy@1 {
+		reg = <1>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <29 IRQ_TYPE_EDGE_FALLING>;
+	};
+};
+
+&pruss2_eth {
+	ti,pruss-gp-mux-sel = <4>,	/* MII2, needed for PRUSS1_MII0 */
+			      <4>;	/* MII2, needed for PRUSS1_MII1 */
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/ti/am571x-idk-touchscreen.dtso b/arch/arm/boot/dts/ti/am571x-idk-touchscreen.dtso
--- a/arch/arm/boot/dts/ti/am571x-idk-touchscreen.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/ti/am571x-idk-touchscreen.dtso	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+#include <dt-bindings/interrupt-controller/irq.h>
+
+&touchscreen {
+	interrupt-parent = <&gpio5>;
+	interrupts = <6 IRQ_TYPE_EDGE_FALLING>;
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/ti/am572x-idk-touchscreen.dtso b/arch/arm/boot/dts/ti/am572x-idk-touchscreen.dtso
--- a/arch/arm/boot/dts/ti/am572x-idk-touchscreen.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/ti/am572x-idk-touchscreen.dtso	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+#include <dt-bindings/interrupt-controller/irq.h>
+
+&touchscreen {
+	interrupt-parent = <&gpio3>;
+	interrupts = <14 IRQ_TYPE_EDGE_FALLING>;
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/ti/am57xx-evm-common.dtso b/arch/arm/boot/dts/ti/am57xx-evm-common.dtso
--- a/arch/arm/boot/dts/ti/am57xx-evm-common.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/ti/am57xx-evm-common.dtso	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,175 @@
+/dts-v1/;
+/plugin/;
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+
+/ {
+  fragment@101 {
+	target-path = "/";
+
+	__overlay__ {
+		compatible = "ti,am5728-evm", "ti,am572x-beagle-x15", "ti,am5728", "ti,dra742", "ti,dra74", "ti,dra7";
+		model = "TI AM5728 EVM";
+
+		aliases {
+			display0 = "/display";
+			display1 = "/connector";
+		};
+
+		gpio_keys {
+			compatible = "gpio-keys";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			autorepeat;
+
+			USER1 {
+				gpios = <&gpio2 23 GPIO_ACTIVE_LOW>;
+				label = "Up";
+				linux,code = <KEY_UP>;
+			};
+
+			USER2 {
+				gpios = <&gpio2 25 GPIO_ACTIVE_LOW>;
+				label = "Down";
+				linux,code = <KEY_DOWN>;
+			};
+
+			USER3 {
+				gpios = <&gpio2 28 GPIO_ACTIVE_LOW>;
+				label = "Left";
+				linux,code = <KEY_LEFT>;
+			};
+
+			USER4 {
+				gpios = <&gpio2 24 GPIO_ACTIVE_LOW>;
+				label = "Right";
+				linux,code = <KEY_RIGHT>;
+			};
+
+			USER5 {
+				gpios = <&gpio2 20 GPIO_ACTIVE_LOW>;
+				label = "Home";
+				linux,code = <KEY_HOME>;
+			};
+		};
+
+		lcd0: display {
+			compatible = "osddisplays,osd070t1718-19ts", "panel-dpi";
+			backlight = <&lcd_bl>;
+			enable-gpios = <&gpio2 5 GPIO_ACTIVE_HIGH>;
+			label = "lcd";
+
+			panel-timing {
+				clock-frequency = <33000000>;
+				de-active = <1>;
+				hactive = <800>;
+				hback-porch = <16>;
+				hfront-porch = <210>;
+				hsync-active = <0>;
+				hsync-len = <30>;
+				pixelclk-active = <1>;
+				vactive = <480>;
+				vback-porch = <10>;
+				vfront-porch = <22>;
+				vsync-active = <0>;
+				vsync-len = <13>;
+			};
+
+			port {
+				lcd_in: endpoint {
+					remote-endpoint = <&dpi_out>;
+				};
+			};
+		};
+
+		lcd_bl: backlight {
+			compatible = "pwm-backlight";
+			brightness-levels = <0 243 245 247 249 251 252 253 255>;
+			default-brightness-level = <8>;
+			pwms = <&ehrpwm1 0 50000 0>;
+		};
+
+		com_3v6: fixedregulator-com_3v6 {
+			compatible = "regulator-fixed";
+			regulator-name = "com_3v6";
+			regulator-min-microvolt = <3600000>;
+			regulator-max-microvolt = <3600000>;
+			vin-supply = <&evm_5v0>;
+			regulator-always-on;
+			regulator-boot-on;
+		};
+
+		vmmcwl_fixed: fixedregulator-mmcwl {
+			compatible = "regulator-fixed";
+			regulator-name = "vmmcwl_fixed";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			gpio = <&gpio5 8 GPIO_ACTIVE_HIGH>;
+			enable-active-high;
+		};
+	};
+  };
+};
+
+&ehrpwm1 {
+	status = "okay";
+};
+
+&epwmss1 {
+	status = "okay";
+};
+
+&i2c5 {
+	clock-frequency = <400000>;
+	status = "okay";
+
+	pixcir_ts@5c {
+		compatible = "pixcir,pixcir_tangoc";
+		attb-gpio = <&gpio2 4 GPIO_ACTIVE_HIGH>;
+		interrupt-parent = <&gpio2>;
+		interrupts = <4 0>;
+		reg = <0x5c>;
+		reset-gpio = <&gpio2 6 GPIO_ACTIVE_HIGH>;
+		touchscreen-size-x = <1024>;
+		touchscreen-size-y = <600>;
+	};
+};
+
+&uart8 {
+	status = "okay";
+};
+
+&dss {
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port {
+			reg = <0>;
+
+			dpi_out: endpoint {
+				data-lines = <24>;
+				remote-endpoint = <&lcd_in>;
+			};
+		};
+	};
+};
+
+&mmc3 {
+	status = "okay";
+	vmmc-supply = <&com_3v6>;
+	vqmmc-supply = <&vmmcwl_fixed>;
+	bus-width = <4>;
+	cap-power-off-card;
+	keep-power-in-suspend;
+	non-removable;
+	#address-cells = <1>;
+	#size-cells = <0>;
+	wlcore: wlcore@2 {
+		compatible = "ti,wl1835";
+		reg = <2>;
+		interrupt-parent = <&gpio5>;
+		interrupts = <7 IRQ_TYPE_EDGE_RISING>;
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/ti/am57xx-evm.dtso b/arch/arm/boot/dts/ti/am57xx-evm.dtso
--- a/arch/arm/boot/dts/ti/am57xx-evm.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/ti/am57xx-evm.dtso	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,11 @@
+/dts-v1/;
+/plugin/;
+
+&mmc3 {
+	pinctrl-names = "default", "hs", "sdr12", "sdr25", "sdr50";
+	pinctrl-0 = <&mmc3_pins_default>;
+	pinctrl-1 = <&mmc3_pins_hs>;
+	pinctrl-2 = <&mmc3_pins_sdr12>;
+	pinctrl-3 = <&mmc3_pins_sdr25>;
+	pinctrl-4 = <&mmc3_pins_sdr50 &mmc3_iodelay_manual1_rev11_conf>;
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/ti/am57xx-evm.its b/arch/arm/boot/dts/ti/am57xx-evm.its
--- a/arch/arm/boot/dts/ti/am57xx-evm.its	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/ti/am57xx-evm.its	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,107 @@
+/dts-v1/;
+
+/ {
+	description = "AM57xx-evm";
+	#address-cells = <1>;
+
+	images {
+		kernel@1 {
+			description = "LCPD kernel";
+			data = /incbin/("../../zImage");
+			type = "kernel";
+			arch = "arm";
+			os = "linux";
+			compression = "none";
+			load = <0x82000000>;
+			entry = <0x82000000>;
+		};
+		fdt@1 {
+			description = "AM57xx-beagle-x15";
+			data = /incbin/("am57xx-beagle-x15.dtb");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83000000>;
+		};
+		fdt@2 {
+			description = "AM57xx-beagle-x15-revc";
+			data = /incbin/("am57xx-beagle-x15-revc.dtb");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83000000>;
+		};
+		fdt@3 {
+			description = "AM57xx-evm-common overlay";
+			data = /incbin/("am57xx-evm-common.dtbo");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83080000>;
+		};
+		fdt@4 {
+			description = "MT9T111 camera support";
+			data = /incbin/("mt9t111.dtbo");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83100000>;
+		};
+		fdt@5 {
+			description = "OV10635 camera support";
+			data = /incbin/("ov10635.dtbo");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83100000>;
+		};
+		fdt@6 {
+			description = "AM57xx-evm overlay";
+			data = /incbin/("am57xx-evm.dtbo");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83090000>;
+		};
+		fdt@7 {
+			description = "AM57xx-evm-reva3 overlay";
+			data = /incbin/("am57xx-evm-reva3.dtbo");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83090000>;
+		};
+	};
+
+	configurations {
+		default = "am57xx-evm.dtb";
+		am57xx-evm.dtb {
+			description = "AM57xx EVM";
+			kernel = "kernel@1";
+			fdt = "fdt@1", "fdt@3", "fdt@6";
+		};
+		am57xx-evm-reva3.dtb {
+			description = "AM57xx EVM rev A3";
+			kernel = "kernel@1";
+			fdt = "fdt@2", "fdt@3", "fdt@7";
+		};
+		am57xx-beagle-x15.dtb {
+			description = "AM57xx beagle-x15";
+			kernel = "kernel@1";
+			fdt = "fdt@1";
+		};
+		am57xx-beagle-x15-revc.dtb {
+			description = "AM57xx beagle-x15 C";
+			kernel = "kernel@1";
+			fdt = "fdt@2";
+		};
+		cam-mt9t111 {
+			description = "MT9T111 camera";
+			fdt = "fdt@4";
+		};
+		cam-ov10635 {
+			description = "OV10635 camera";
+			fdt = "fdt@5";
+		};
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/ti/am57xx-evm-reva3.dtso b/arch/arm/boot/dts/ti/am57xx-evm-reva3.dtso
--- a/arch/arm/boot/dts/ti/am57xx-evm-reva3.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/ti/am57xx-evm-reva3.dtso	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,11 @@
+/dts-v1/;
+/plugin/;
+
+&mmc3 {
+	pinctrl-names = "default", "hs", "sdr12", "sdr25", "sdr50";
+	pinctrl-0 = <&mmc3_pins_default>;
+	pinctrl-1 = <&mmc3_pins_hs>;
+	pinctrl-2 = <&mmc3_pins_sdr12>;
+	pinctrl-3 = <&mmc3_pins_sdr25>;
+	pinctrl-4 = <&mmc3_pins_sdr50 &mmc3_iodelay_manual1_rev20_conf>;
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/ti/am57xx-idk.its b/arch/arm/boot/dts/ti/am57xx-idk.its
--- a/arch/arm/boot/dts/ti/am57xx-idk.its	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/ti/am57xx-idk.its	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,130 @@
+/dts-v1/;
+
+/ {
+	description = "DRA72-evm";
+	#address-cells = <1>;
+
+	images {
+		kernel@1 {
+			description = "LCPD kernel";
+			data = /incbin/("../../zImage");
+			type = "kernel";
+			arch = "arm";
+			os = "linux";
+			compression = "none";
+			load = <0x82000000>;
+			entry = <0x82000000>;
+		};
+		fdt@1 {
+			description = "am571x-idk";
+			data = /incbin/("am571x-idk.dtb");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83000000>;
+		};
+		fdt@2 {
+			description = "am572x-idk";
+			data = /incbin/("am572x-idk.dtb");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83000000>;
+		};
+		fdt@3 {
+			description = "am574x-idk";
+			data = /incbin/("am574x-idk.dtb");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83000000>;
+		};
+		fdt@4 {
+			description = "am57xx-idk OSD LCD common";
+			data = /incbin/("am57xx-idk-osd-lcd-common.dtbo");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83080000>;
+		};
+		fdt@5 {
+			description = "am571x-idk touchscreen";
+			data = /incbin/("am571x-idk-touchscreen.dtbo");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83100000>;
+		};
+		fdt@6 {
+			description = "am572x-idk touchscreen";
+			data = /incbin/("am572x-idk-touchscreen.dtbo");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83100000>;
+		};
+		fdt@7 {
+			description = "osd101t2045 LCD";
+			data = /incbin/("lcd-osd101t2045.dtbo");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83180000>;
+		};
+		fdt@8 {
+			description = "osd101t2587 LCD";
+			data = /incbin/("lcd-osd101t2587.dtbo");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83180000>;
+		};
+		fdt@9 {
+			description = "ICSS1 Ethernet";
+			data = /incbin/("am571x-idk-icss1eth.dtbo");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83080000>;
+		};
+	};
+
+	configurations {
+		default = "dra72-evm.dtb";
+		am571x-idk.dtb {
+			description = "am571x-idk 6 port Ethernet";
+			kernel = "kernel@1";
+			fdt = "fdt@1", "fdt@9";
+		};
+		am572x-idk.dtb {
+			description = "am572x-idk";
+			kernel = "kernel@1";
+			fdt = "fdt@2";
+		};
+		am574x-idk.dtb {
+			description = "am574x-idk";
+			kernel = "kernel@1";
+			fdt = "fdt@3";
+		};
+		am571x-idk-lcd-osd101t2045 {
+			description = "am571x-idk with osd101t2045 LCD";
+			fdt = "fdt@1", "fdt@4", "fdt@5", "fdt@7";
+		};
+		am571x-idk-lcd-osd101t2587 {
+			description = "am571x-idk with osd101t2587 LCD";
+			fdt = "fdt@1", "fdt@4", "fdt@5", "fdt@8";
+		};
+		am572x-idk-lcd-osd101t2045 {
+			description = "am572x-idk with osd101t2045 LCD";
+			fdt = "fdt@2", "fdt@4", "fdt@6", "fdt@7";
+		};
+		am572x-idk-lcd-osd101t2587 {
+			description = "am572x-idk with osd101t2587 LCD";
+			fdt = "fdt@2", "fdt@4", "fdt@6", "fdt@8";
+		};
+		am574x-idk-lcd-osd101t2587 {
+			description = "am574x-idk with osd101t2587 LCD";
+			fdt = "fdt@3", "fdt@4", "fdt@6", "fdt@8";
+		};
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/ti/am57xx-idk-osd-lcd-common.dtso b/arch/arm/boot/dts/ti/am57xx-idk-osd-lcd-common.dtso
--- a/arch/arm/boot/dts/ti/am57xx-idk-osd-lcd-common.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/ti/am57xx-idk-osd-lcd-common.dtso	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+
+/ {
+  fragment@101 {
+	target-path = "/";
+
+	__overlay__ {
+		aliases {
+			display0 = "/display";
+			display1 = "/connector";
+		};
+
+		lcd_bl: backlight {
+			compatible = "pwm-backlight";
+			pwms = <&ecap0 0 50000 1>;
+			brightness-levels = <0 51 53 56 62 75 101 152 255>;
+			default-brightness-level = <8>;
+		};
+	};
+  };
+};
+
+&dsi_bridge {
+	status = "okay";
+};
+
+&i2c1 {
+	touchscreen: edt-ft5506@38 {
+		status = "okay";
+		compatible = "edt,edt-ft5506", "edt,edt-ft5x06";
+
+		reg = <0x38>;
+
+		/* GPIO line is inverted before going to touch panel */
+		reset-gpios = <&gpio6 15 GPIO_ACTIVE_LOW>;
+
+		touchscreen-size-x = <1920>;
+		touchscreen-size-y = <1200>;
+
+		wakeup-source;
+	};
+};
+
+&epwmss0 {
+	status = "okay";
+};
+
+&ecap0 {
+	status = "okay";
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/ti/dra71-evm.its b/arch/arm/boot/dts/ti/dra71-evm.its
--- a/arch/arm/boot/dts/ti/dra71-evm.its	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/ti/dra71-evm.its	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,61 @@
+/dts-v1/;
+
+/ {
+	description = "DRA71x-evm";
+	#address-cells = <1>;
+
+	images {
+		kernel@1 {
+			description = "LCPD kernel";
+			data = /incbin/("../../zImage");
+			type = "kernel";
+			arch = "arm";
+			os = "linux";
+			compression = "none";
+			load = <0x82000000>;
+			entry = <0x82000000>;
+		};
+		fdt@1 {
+			description = "DRA71x-evm";
+			data = /incbin/("dra71-evm.dtb");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83000000>;
+		};
+		fdt@2 {
+			description = "DRA71x-evm LCD";
+			data = /incbin/("dra71-evm-lcd-auo-g101evn01.0.dtbo");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83100000>;
+		};
+		fdt@3 {
+			description = "DRA71x-evm NAND";
+			data = /incbin/("dra71-evm-nand.dtbo");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83200000>;
+		};
+	};
+
+	configurations {
+		default = "dra71-evm.dtb";
+		dra71-evm.dtb {
+			description = "DRA71x-evm";
+			kernel = "kernel@1";
+			fdt = "fdt@1";
+		};
+		lcd-auo-g101evn01.0 {
+			description = "DRA71x-evm with LCD overlay";
+			fdt = "fdt@2";
+		};
+		dra71-evm-nand {
+			description = "DRA71x-evm with NAND overlay";
+			kernel = "kernel@1";
+			fdt = "fdt@1", "fdt@3";
+		};
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/ti/dra71-evm-lcd-auo-g101evn01.0.dtso b/arch/arm/boot/dts/ti/dra71-evm-lcd-auo-g101evn01.0.dtso
--- a/arch/arm/boot/dts/ti/dra71-evm-lcd-auo-g101evn01.0.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/ti/dra71-evm-lcd-auo-g101evn01.0.dtso	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,106 @@
+/dts-v1/;
+/plugin/;
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+
+/ {
+  fragment@101 {
+	target-path = "/";
+
+	__overlay__ {
+		model = "TI DRA71 EVM-LCD-AUO-Display";
+
+		aliases {
+			display0 = "/display";
+			display1 = "/connector";
+		};
+
+		lcd_bl: backlight {
+			compatible = "led-backlight";
+			brightness-levels = <0 2 38 74 110 146 182 218 255>;
+			default-brightness-level = <8>;
+			enable-gpios = <&pcf_display_board 0 GPIO_ACTIVE_LOW>;
+			leds = <&backlight_led>;
+		};
+
+		lcd: display {
+			compatible = "auo,g101evn01.0";
+			enable-gpios = <&pcf_lcd 13 GPIO_ACTIVE_LOW>;
+			label = "lcd";
+			backlight = <&lcd_bl>;
+
+			port {
+				lcd_in: endpoint {
+					remote-endpoint = <&dpi_out>;
+				};
+			};
+		};
+	};
+  };
+};
+
+&pcf_gpio_21 {
+	p0 {
+		gpio-hog;
+		gpios = <0 GPIO_ACTIVE_HIGH>;
+		line-name = "sel_gpmc_ad_vid_s0";
+		output-low;
+	};
+
+	p7 {
+		gpio-hog;
+		gpios = <7 GPIO_ACTIVE_HIGH>;
+		line-name = "sel_gpmc_ad_vid_s2";
+		output-high;
+	};
+};
+
+&i2c1 {
+	pcf_display_board: gpio@27 {
+		compatible = "nxp,pcf8575";
+		#gpio-cells = <2>;
+		gpio-controller;
+		reg = <0x27>;
+	};
+
+	tlc59108@40 {
+		compatible = "ti,tlc59108";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		reg = <0x40>;
+
+		backlight_led: bl@2 {
+			label = "backlight";
+			reg = <0x2>;
+		};
+	};
+
+	touchscreen: goodix-gt9271@14 {
+		compatible = "goodix,gt9271";
+		interrupt-parent = <&gpio1>;
+		interrupts = <15 IRQ_TYPE_LEVEL_HIGH>;
+		irq-gpios = <&pcf_display_board 6 GPIO_ACTIVE_HIGH>;
+		reg = <0x14>;
+		reset-gpios = <&pcf_display_board 5 GPIO_ACTIVE_LOW>;
+		status = "okay";
+		touchscreen-inverted-y;
+		touchscreen-size-x = <1280>;
+		touchscreen-size-y = <800>;
+	};
+};
+
+&dss {
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		dss_port: port {
+			reg = <2>;
+
+			dpi_out: endpoint {
+				data-lines = <24>;
+				remote-endpoint = <&lcd_in>;
+			};
+		};
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/ti/dra71-evm-nand.dtso b/arch/arm/boot/dts/ti/dra71-evm-nand.dtso
--- a/arch/arm/boot/dts/ti/dra71-evm-nand.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/ti/dra71-evm-nand.dtso	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,6 @@
+/dts-v1/;
+/plugin/;
+
+&gpmc {
+	status = "okay";
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/ti/dra72-evm.its b/arch/arm/boot/dts/ti/dra72-evm.its
--- a/arch/arm/boot/dts/ti/dra72-evm.its	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/ti/dra72-evm.its	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,89 @@
+/dts-v1/;
+
+/ {
+	description = "DRA72-evm";
+	#address-cells = <1>;
+
+	images {
+		kernel@1 {
+			description = "LCPD kernel";
+			data = /incbin/("../../zImage");
+			type = "kernel";
+			arch = "arm";
+			os = "linux";
+			compression = "none";
+			load = <0x82000000>;
+			entry = <0x82000000>;
+		};
+		fdt@1 {
+			description = "DRA72-evm";
+			data = /incbin/("dra72-evm.dtb");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83000000>;
+		};
+		fdt@2 {
+			description = "DRA72-evm-revc";
+			data = /incbin/("dra72-evm-revc.dtb");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83000000>;
+		};
+		fdt@3 {
+			description = "DRA7x-evm OSD LCD common";
+			data = /incbin/("dra7x-evm-osd-lcd-common.dtbo");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83080000>;
+		};
+		fdt@4 {
+			description = "DRA72-evm touchscreen";
+			data = /incbin/("dra72-evm-touchscreen.dtbo");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83100000>;
+		};
+		fdt@5 {
+			description = "osd101t2045 LCD";
+			data = /incbin/("lcd-osd101t2045.dtbo");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83100000>;
+		};
+		fdt@6 {
+			description = "osd101t2587 LCD";
+			data = /incbin/("lcd-osd101t2587.dtbo");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83100000>;
+		};
+	};
+
+	configurations {
+		default = "dra72-evm.dtb";
+		dra72-evm.dtb {
+			description = "DRA72-evm";
+			kernel = "kernel@1";
+			fdt = "fdt@1";
+		};
+		dra72-evm-revc.dtb {
+			description = "DRA72-evm-revc";
+			kernel = "kernel@1";
+			fdt = "fdt@2";
+		};
+		lcd-osd101t2045 {
+			description = "osd101t2045 LCD overlay";
+			fdt = "fdt@3", "fdt@4", "fdt@5";
+		};
+		lcd-osd101t2587 {
+			description = "osd101t2587 LCD overlay";
+			fdt = "fdt@3", "fdt@4", "fdt@6";
+		};
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/ti/dra72-evm-touchscreen.dtso b/arch/arm/boot/dts/ti/dra72-evm-touchscreen.dtso
--- a/arch/arm/boot/dts/ti/dra72-evm-touchscreen.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/ti/dra72-evm-touchscreen.dtso	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+#include <dt-bindings/interrupt-controller/irq.h>
+
+&touchscreen {
+	interrupt-parent = <&gpio1>;
+	interrupts = <15 IRQ_TYPE_EDGE_FALLING>;
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/ti/dra74-evm.its b/arch/arm/boot/dts/ti/dra74-evm.its
--- a/arch/arm/boot/dts/ti/dra74-evm.its	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/ti/dra74-evm.its	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,76 @@
+/dts-v1/;
+
+/ {
+	description = "DRA74-evm";
+	#address-cells = <1>;
+
+	images {
+		kernel@1 {
+			description = "LCPD kernel";
+			data = /incbin/("../../zImage");
+			type = "kernel";
+			arch = "arm";
+			os = "linux";
+			compression = "none";
+			load = <0x82000000>;
+			entry = <0x82000000>;
+		};
+		fdt@1 {
+			description = "DRA74-evm";
+			data = /incbin/("dra7-evm.dtb");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83000000>;
+		};
+		fdt@2 {
+			description = "DRA7x-evm OSD LCD common";
+			data = /incbin/("dra7x-evm-osd-lcd-common.dtbo");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83080000>;
+		};
+		fdt@3 {
+			description = "DRA74-evm touchscreen";
+			data = /incbin/("dra74-evm-touchscreen.dtbo");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83100000>;
+		};
+		fdt@4 {
+			description = "osd101t2045 LCD";
+			data = /incbin/("lcd-osd101t2045.dtbo");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83100000>;
+		};
+		fdt@5 {
+			description = "osd101t2587 LCD";
+			data = /incbin/("lcd-osd101t2587.dtbo");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83100000>;
+		};
+	};
+
+	configurations {
+		default = "dra7-evm.dtb";
+		dra7-evm.dtb {
+			description = "DRA74-evm";
+			kernel = "kernel@1";
+			fdt = "fdt@1";
+		};
+		lcd-osd101t2045 {
+			description = "osd101t2045 LCD overlay";
+			fdt = "fdt@2", "fdt@3", "fdt@4";
+		};
+		lcd-osd101t2587 {
+			description = "osd101t2587 LCD overlay";
+			fdt = "fdt@2", "fdt@3", "fdt@5";
+		};
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/ti/dra74-evm-touchscreen.dtso b/arch/arm/boot/dts/ti/dra74-evm-touchscreen.dtso
--- a/arch/arm/boot/dts/ti/dra74-evm-touchscreen.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/ti/dra74-evm-touchscreen.dtso	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+#include <dt-bindings/interrupt-controller/irq.h>
+
+&touchscreen {
+	interrupt-parent = <&gpio1>;
+	interrupts = <2 IRQ_TYPE_EDGE_FALLING>;
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/ti/dra76-evm.its b/arch/arm/boot/dts/ti/dra76-evm.its
--- a/arch/arm/boot/dts/ti/dra76-evm.its	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/ti/dra76-evm.its	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,72 @@
+/dts-v1/;
+
+/ {
+	description = "DRA76-evm";
+	#address-cells = <1>;
+
+	images {
+		kernel@1 {
+			description = "LCPD kernel";
+			data = /incbin/("../../zImage");
+			type = "kernel";
+			arch = "arm";
+			os = "linux";
+			compression = "none";
+			load = <0x82000000>;
+			entry = <0x82000000>;
+		};
+		fdt@1 {
+			description = "DRA76-evm";
+			data = /incbin/("dra76-evm.dtb");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83000000>;
+		};
+		fdt@2 {
+			description = "tfp410 display support";
+			data = /incbin/("dra76-evm-tfp410.dtbo");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83080000>;
+		};
+		fdt@3 {
+			description = "OV490 4 data lanes camera";
+			data = /incbin/("dra7x-evm-ov490.dtbo");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83100000>;
+		};
+		fdt@4 {
+			description = "OV5640 camera";
+			data = /incbin/("dra7x-evm-ov5640.dtbo");
+			type = "flat_dt";
+			arch = "arm";
+			compression = "none";
+			load = <0x83100000>;
+		};
+	};
+
+	configurations {
+		default = "dra76-evm.dtb";
+		dra76-evm.dtb {
+			description = "DRA76-evm";
+			kernel = "kernel@1";
+			fdt = "fdt@1";
+		};
+		tfp410 {
+			description = "tfp410 overlay";
+			fdt = "fdt@2";
+		};
+		ov490 {
+			description = "OV490 overlay";
+			fdt = "fdt@3";
+		};
+		ov5640 {
+			description = "OV5640 overlay";
+			fdt = "fdt@4";
+		};
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/ti/dra76-evm-tfp410.dtso b/arch/arm/boot/dts/ti/dra76-evm-tfp410.dtso
--- a/arch/arm/boot/dts/ti/dra76-evm-tfp410.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/ti/dra76-evm-tfp410.dtso	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,117 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+
+/ {
+  fragment@101 {
+	target-path = "/";
+
+	__overlay__ {
+		aliases {
+			display0 = "/display";
+			display1 = "/connector";
+		};
+
+		tfp410: encoder@0 {
+			compatible = "ti,tfp410";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+					tfp410_in: endpoint@0 {
+						remote-endpoint = <&dpi_out>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+	
+					tfp410_out: endpoint@0 {
+						remote-endpoint = <&dvi_connector_in>;
+					};
+				};
+			};
+		};
+
+		dvi0: connector@0 {
+			compatible = "dvi-connector";
+			label = "dvi";
+
+			digital;
+
+			ddc-i2c-bus = <&i2c3>;
+
+			hpd-gpios = <&gpio1 2 GPIO_ACTIVE_HIGH>;	/* wakeup2/sys_nirq2/gpio1_2 HPD */
+
+			port {
+				dvi_connector_in: endpoint {
+					remote-endpoint = <&tfp410_out>;
+				};
+			};
+		};
+	};
+  };
+};
+
+&dss {
+	status = "ok";
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		status = "ok";
+
+		port {
+			reg = <0>;
+
+			dpi_out: endpoint {
+				remote-endpoint = <&tfp410_in>;
+				data-lines = <24>;
+			};
+		};
+	};
+};
+
+&gpio3 {
+	p1 {
+		/* GPIO3_1 CON_LCD_PWR_DN */
+		/* This affects the TFP410 and the USB */
+		gpio-hog;
+		gpios = <1 GPIO_ACTIVE_HIGH>;
+		output-low;
+		line-name = "CON_LCD_PWR_DN";
+	};
+};
+
+&i2c3 {
+	clock-frequency = <100000>;
+
+	pcf_tfp: pcf8757@20 {
+		compatible = "ti,pcf8575", "nxp,pcf8575";
+		reg = <0x27>;
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		p2 {
+			gpio-hog;
+			gpios = <2 GPIO_ACTIVE_HIGH>;
+			output-high;
+			line-name = "ct_hpd";
+		};
+
+		p3 {
+			gpio-hog;
+			gpios = <3 GPIO_ACTIVE_HIGH>;
+			output-high;
+			line-name = "ls_oe";
+		};
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/ti/dra7x-evm-osd-lcd-common.dtso b/arch/arm/boot/dts/ti/dra7x-evm-osd-lcd-common.dtso
--- a/arch/arm/boot/dts/ti/dra7x-evm-osd-lcd-common.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/ti/dra7x-evm-osd-lcd-common.dtso	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,115 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+
+/ {
+  fragment@101 {
+	target-path = "/";
+
+	__overlay__ {
+		aliases {
+			display0 = "/display";
+			display1 = "/connector";
+		};
+
+		lcd_bl: backlight {
+			compatible = "led-backlight";
+			leds = <&backlight_led>;
+			brightness-levels = <0 243 245 247 248 249 251 252 255>;
+			default-brightness-level = <8>;
+
+			enable-gpios = <&pcf_lcd 13 GPIO_ACTIVE_LOW>;
+		};
+
+		tc358768_refclk: tc358768_refclk {
+			#clock-cells = <0>;
+			compatible = "fixed-clock";
+			clock-frequency = <20000000>;
+		};
+	};
+  };
+};
+
+&i2c1 {
+	dsi_bridge: tc358768@0e {
+		compatible = "toshiba,tc358768";
+		reg = <0x0e>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		clocks = <&tc358768_refclk>;
+		clock-names = "refclk";
+
+		reset-gpios = <&pcf_display_board 0 GPIO_ACTIVE_LOW>;
+
+		dsi_bridge_ports: ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				rgb_in: endpoint {
+					remote-endpoint = <&dpi_out>;
+					data-lines = <24>;
+				};
+			};
+		};
+	};
+
+	tlc59108: tlc59116@40 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "ti,tlc59108";
+		reg = <0x40>;
+
+		backlight_led: bl@2 {
+			label = "backlight";
+			reg = <0x2>;
+		};
+	};
+
+	pcf_display_board: gpio@27 {
+		compatible = "nxp,pcf8575";
+		reg = <0x27>;
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
+
+	touchscreen: edt-ft5506@38 {
+		status = "okay";
+		compatible = "edt,edt-ft5506", "edt,edt-ft5x06";
+
+		reg = <0x38>;
+
+		/* GPIO line is inverted before going to touch panel */
+		reset-gpios = <&pcf_display_board 5 GPIO_ACTIVE_HIGH>;
+
+		touchscreen-size-x = <1920>;
+		touchscreen-size-y = <1200>;
+
+		wakeup-source;
+	};
+};
+
+&dss {
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+
+			dpi_out: endpoint {
+				remote-endpoint = <&rgb_in>;
+				data-lines = <24>;
+			};
+		};
+
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/ti/dra7x-evm-ov490.dtso b/arch/arm/boot/dts/ti/dra7x-evm-ov490.dtso
--- a/arch/arm/boot/dts/ti/dra7x-evm-ov490.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/ti/dra7x-evm-ov490.dtso	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,51 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2015-2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+#include <dt-bindings/gpio/gpio.h>
+
+&i2c5 {
+	gpio_csi2_adap: tca6416@20 {
+		status = "okay";
+		compatible = "ti,tca6416";
+		reg = <0x20>;
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
+
+	ov490@24 {
+		compatible = "ovti,ov490";
+		reg = <0x24>;
+
+		mux-gpios = <&gpio_csi2_adap 0	GPIO_ACTIVE_LOW>,
+						/* CSI2_SEL_I2C_CLK */
+			    <&gpio_csi2_adap 1	GPIO_ACTIVE_HIGH>,
+						/* CSI2_SEL_REF_CLK */
+			    <&gpio_csi2_adap 3	GPIO_ACTIVE_HIGH>,
+						/* CSI2_CAM0_RESETn */
+			    <&gpio_csi2_adap 4	GPIO_ACTIVE_LOW>;
+						/* CSI2_CAM0_PWR_DWN */
+		port {
+			csi2_cam0: endpoint@0 {
+				clock-lanes = <0>;
+				data-lanes = <1 2 3 4>;
+				remote-endpoint = <&csi2_phy0>;
+			};
+		};
+	};
+};
+
+&cal {
+	status = "okay";
+};
+
+&csi2_0 {
+	csi2_phy0: endpoint@0 {
+		clock-lanes = <0>;
+		data-lanes = <1 2 3 4>;
+		remote-endpoint = <&csi2_cam0>;
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/ti/dra7x-evm-ov5640.dtso b/arch/arm/boot/dts/ti/dra7x-evm-ov5640.dtso
--- a/arch/arm/boot/dts/ti/dra7x-evm-ov5640.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/ti/dra7x-evm-ov5640.dtso	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+#include <dt-bindings/gpio/gpio.h>
+
+/ {
+  fragment@101 {
+	target-path = "/";
+
+	__overlay__ {
+		clk_ov5640_fixed: clk_ov5640_fixed {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <24000000>;
+		};
+	};
+  };
+};
+
+&i2c5 {
+	ov5640@3C {
+		compatible = "ovti,ov5640";
+		reg = <0x3c>;
+
+		clocks = <&clk_ov5640_fixed>;
+		clock-names = "xclk";
+
+		port {
+			csi2_cam0: endpoint {
+				remote-endpoint = <&csi2_phy0>;
+				clock-lanes = <0>;
+				data-lanes = <1 2>;
+			};
+		};
+	};
+};
+
+&cal {
+	status = "okay";
+};
+
+&csi2_0 {
+	csi2_phy0: endpoint@0 {
+		remote-endpoint = <&csi2_cam0>;
+		clock-lanes = <0>;
+		data-lanes = <1 2>;
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/ti/lcd-osd101t2045.dtso b/arch/arm/boot/dts/ti/lcd-osd101t2045.dtso
--- a/arch/arm/boot/dts/ti/lcd-osd101t2045.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/ti/lcd-osd101t2045.dtso	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,34 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+
+
+&dsi_bridge {
+	lcd: display {
+		compatible = "osd,osd101t2045-53ts";
+		reg = <0>;
+
+		label = "lcd";
+
+		backlight = <&lcd_bl>;
+
+		port {
+			lcd_in: endpoint {
+				remote-endpoint = <&dsi_out>;
+			};
+		};
+	};
+};
+
+&dsi_bridge_ports {
+	port@1 {
+		reg = <1>;
+		dsi_out: endpoint {
+			remote-endpoint = <&lcd_in>;
+		};
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/ti/lcd-osd101t2587.dtso b/arch/arm/boot/dts/ti/lcd-osd101t2587.dtso
--- a/arch/arm/boot/dts/ti/lcd-osd101t2587.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/ti/lcd-osd101t2587.dtso	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,34 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+
+
+&dsi_bridge {
+	lcd: display {
+		compatible = "osd,osd101t2587-53ts";
+		reg = <0>;
+
+		label = "lcd";
+
+		backlight = <&lcd_bl>;
+
+		port {
+			lcd_in: endpoint {
+				remote-endpoint = <&dsi_out>;
+			};
+		};
+	};
+};
+
+&dsi_bridge_ports {
+	port@1 {
+		reg = <1>;
+		dsi_out: endpoint {
+			remote-endpoint = <&lcd_in>;
+		};
+	};
+};
diff -Naur --no-dereference a/arch/arm/boot/dts/ti/Makefile b/arch/arm/boot/dts/ti/Makefile
--- a/arch/arm/boot/dts/ti/Makefile	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/ti/Makefile	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,55 @@
+DTC_FLAGS += -@ -Wno-reg_format -Wno-avoid_default_addr_size
+DTC_FLAGS_MKIMAGE = -I dts -O dtb -p 500 -Wno-unit_address_vs_reg \
+	-i arch/arm/boot/dts/ti
+
+-include $(obj)/.itb-deps.d
+
+ifneq ($(DTC),)
+  DTC_PATH=$(dir $(DTC))
+else
+  DTC_PATH=$(objtree)/scripts/dtc
+endif
+
+quiet_cmd_mkfit = MKIMAGE -f $@
+cmd_mkfit = PATH="$(DTC_PATH):$(PATH)" $(MKIMAGE) -D "$(DTC_FLAGS_MKIMAGE)" -f $< $@
+
+define gen_itb_deps
+	@if [ -d $(obj) ] ; then \
+		echo >> $(obj)/.itb-deps.d ; \
+		echo "$(src)/$(patsubst %.its,%.itb,$(notdir $(1))): $(srctree)/$(src)/$(notdir $(1)) \\" >> $(obj)/.itb-deps.d ; \
+		cat $(1) | grep incbin | cut -d "\"" -f 2 | awk '{ printf " $(src)/%s \\\n", $$1 }' | sed '$$s/ \\$$//' >> $(obj)/.itb-deps.d ; \
+	fi
+
+endef
+
+$(obj)/.itb-deps.d:
+	@if [ -d  $(obj) ] ; then \
+		echo "$(src)/.itb-deps.d: $(src)/Makefile $(wildcard $(srctree)/$(src)/*.its)" > $(obj)/.itb-deps.d ; \
+		echo >> $(obj)/.itb-deps.d ; \
+	fi
+	$(foreach f,$(wildcard $(srctree)/$(src)/*.its),$(call gen_itb_deps,$f))
+
+$(obj)/%.dtb: $(src)/../%.dts FORCE
+	$(call if_changed_dep,dtc)
+
+$(obj)/%.dtbo: $(src)/%.dtso FORCE
+	$(call if_changed_dep,dtc)
+
+$(obj)/%.itb: $(src)/%.its FORCE
+	$(call cmd,mkfit)
+
+ifeq ($(BUILD_ITBS),y)
+dtb-$(CONFIG_SOC_DRA7XX) += \
+	am57xx-evm.itb \
+	am57xx-idk.itb \
+	dra71-evm.itb \
+	dra72-evm.itb \
+	dra74-evm.itb \
+	dra76-evm.itb
+else
+dtb-$(CONFIG_SOC_DRA7XX) += $(shell grep incbin $(srctree)/$(src)/*.its | grep dtb | cut -d "\"" -f 2)
+endif
+
+clean-files	:= *.dtb *.dtbo *.itb
+
+always		:= $(dtb-y)
diff -Naur --no-dereference a/arch/arm/boot/dts/ti/mt9t111.dtso b/arch/arm/boot/dts/ti/mt9t111.dtso
--- a/arch/arm/boot/dts/ti/mt9t111.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/ti/mt9t111.dtso	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,43 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2015-2018 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+#include <dt-bindings/gpio/gpio.h>
+
+&i2c5 {
+	mt9t11x@3C {
+		compatible = "aptina,mt9t111";
+		bufen-gpios = <&gpio5 12 GPIO_ACTIVE_LOW>;
+		camen-gpios = <&gpio6 11 GPIO_ACTIVE_LOW>;
+		oscen-gpios = <&gpio5 10 GPIO_ACTIVE_HIGH>;
+		powerdown-gpios = <&gpio5 11 GPIO_ACTIVE_LOW>;
+		reg = <0x3C>;
+		reset-gpios = <&gpio4 17 GPIO_ACTIVE_LOW>;
+
+		port {
+			cam: endpoint {
+				hsync-active = <1>;
+				input-clock-freq = <32000000>;
+				pclk-sample = <0>;
+				pixel-clock-freq = <96000000>;
+				remote-endpoint = <&vin3a_ep>;
+				vsync-active = <0>;
+			};
+		};
+	};
+};
+
+&vin3a {
+	vin3a_ep: endpoint {
+		remote-endpoint = <&cam>;
+		slave-mode;
+	};
+};
+
+&vip2 {
+	status = "okay";
+};
+
diff -Naur --no-dereference a/arch/arm/boot/dts/ti/ov10635.dtso b/arch/arm/boot/dts/ti/ov10635.dtso
--- a/arch/arm/boot/dts/ti/ov10635.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/boot/dts/ti/ov10635.dtso	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2015-2018 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+#include <dt-bindings/gpio/gpio.h>
+
+/ {
+  fragment@101 {
+	target-path = "/";
+
+	__overlay__ {
+		clk_ov10635: clk_ov10635 {
+			compatible = "gpio-gate-clock";
+			#clock-cells = <0>;
+			clocks = <&clk_ov10635_fixed>;
+			enable-gpios = <&gpio5 10 GPIO_ACTIVE_HIGH>;
+		};
+
+		clk_ov10635_fixed: clk_ov10635_fixed {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <24000000>;
+		};
+	};
+  };
+};
+
+&i2c5 {
+	ov10635@30 {
+		compatible = "ovti,ov10635";
+		clock-names = "xvclk";
+		clocks = <&clk_ov10635>;
+		powerdown-gpios = <&gpio5 11 GPIO_ACTIVE_HIGH>;
+		reg = <0x30>;
+		reset-gpios = <&gpio4 17 GPIO_ACTIVE_HIGH>;
+
+		port {
+			cam: endpoint {
+				hsync-active = <1>;
+				pclk-sample = <1>;
+				remote-endpoint = <&vin3a_ep>;
+				vsync-active = <1>;
+			};
+		};
+	};
+};
+
+&vin3a {
+	vin3a_ep: endpoint {
+		remote-endpoint = <&cam>;
+		slave-mode;
+	};
+};
+
+&vip2 {
+	status = "okay";
+};
+
+&gpio6 {
+	p11 {
+		gpio-hog;
+		gpios = <11 GPIO_ACTIVE_LOW>;
+		line-name = "cm-camen-gpio";
+		output-high;
+	};
+};
+
diff -Naur --no-dereference a/arch/arm/boot/.gitignore b/arch/arm/boot/.gitignore
--- a/arch/arm/boot/.gitignore	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/.gitignore	2020-06-19 21:00:31.404227715 -0400
@@ -3,3 +3,6 @@
 xipImage
 bootpImage
 uImage
+*.dtb
+*.dtbo
+*.itb
diff -Naur --no-dereference a/arch/arm/boot/Makefile b/arch/arm/boot/Makefile
--- a/arch/arm/boot/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/boot/Makefile	2020-06-19 21:00:31.404227715 -0400
@@ -29,6 +29,10 @@
 
 targets := Image zImage xipImage bootpImage uImage
 
+ifeq ($(CONFIG_OF_OVERLAY),y)
+DTC_FLAGS += -@
+endif
+
 ifeq ($(CONFIG_XIP_KERNEL),y)
 
 cmd_deflate_xip_data = $(CONFIG_SHELL) -c \
diff -Naur --no-dereference a/arch/arm/configs/davinci_all_defconfig b/arch/arm/configs/davinci_all_defconfig
--- a/arch/arm/configs/davinci_all_defconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/configs/davinci_all_defconfig	2020-06-19 21:00:31.404227715 -0400
@@ -167,8 +167,9 @@
 CONFIG_SND=m
 CONFIG_SND_USB_AUDIO=m
 CONFIG_SND_SOC=m
-CONFIG_SND_EDMA_SOC=m
-CONFIG_SND_DA850_SOC_EVM=m
+CONFIG_SND_SOC_TLV320AIC3X=m
+CONFIG_SND_SOC_DAVINCI_MCASP=m
+CONFIG_SND_SOC_DAVINCI_EVM=m
 CONFIG_SND_SIMPLE_CARD=m
 CONFIG_HID=m
 CONFIG_HID_A4TECH=m
diff -Naur --no-dereference a/arch/arm/configs/omap1_defconfig b/arch/arm/configs/omap1_defconfig
--- a/arch/arm/configs/omap1_defconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/configs/omap1_defconfig	2020-06-19 21:00:31.404227715 -0400
@@ -175,8 +175,6 @@
 # CONFIG_SND_VERBOSE_PROCFS is not set
 CONFIG_SND_DUMMY=y
 CONFIG_SND_USB_AUDIO=y
-CONFIG_SND_SOC=y
-CONFIG_SND_OMAP_SOC=y
 # CONFIG_USB_HID is not set
 CONFIG_USB=y
 CONFIG_USB_PHY=y
diff -Naur --no-dereference a/arch/arm/configs/omap2plus_defconfig b/arch/arm/configs/omap2plus_defconfig
--- a/arch/arm/configs/omap2plus_defconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/configs/omap2plus_defconfig	2020-06-19 21:00:31.404227715 -0400
@@ -381,13 +381,13 @@
 CONFIG_SND_DEBUG=y
 CONFIG_SND_USB_AUDIO=m
 CONFIG_SND_SOC=m
-CONFIG_SND_EDMA_SOC=m
-CONFIG_SND_AM33XX_SOC_EVM=m
-CONFIG_SND_OMAP_SOC=m
-CONFIG_SND_OMAP_SOC_HDMI_AUDIO=m
-CONFIG_SND_OMAP_SOC_OMAP_TWL4030=m
-CONFIG_SND_OMAP_SOC_OMAP_ABE_TWL6040=m
-CONFIG_SND_OMAP_SOC_OMAP3_PANDORA=m
+CONFIG_SND_SOC_TLV320AIC3X=m
+CONFIG_SND_SOC_DAVINCI_MCASP=m
+CONFIG_SND_SOC_NOKIA_RX51=m
+CONFIG_SND_SOC_OMAP_HDMI=m
+CONFIG_SND_SOC_OMAP_ABE_TWL6040=m
+CONFIG_SND_SOC_OMAP3_PANDORA=m
+CONFIG_SND_SOC_OMAP3_TWL4030=m
 CONFIG_SND_SOC_CPCAP=m
 CONFIG_SND_SIMPLE_CARD=m
 CONFIG_SND_AUDIO_GRAPH_CARD=m
diff -Naur --no-dereference a/arch/arm/configs/rcn-ee_defconfig b/arch/arm/configs/rcn-ee_defconfig
--- a/arch/arm/configs/rcn-ee_defconfig	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/configs/rcn-ee_defconfig	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,2518 @@
+# CONFIG_LOCALVERSION_AUTO is not set
+CONFIG_KERNEL_LZO=y
+CONFIG_SYSVIPC=y
+CONFIG_POSIX_MQUEUE=y
+CONFIG_NO_HZ_IDLE=y
+CONFIG_HIGH_RES_TIMERS=y
+CONFIG_PREEMPT=y
+CONFIG_BSD_PROCESS_ACCT=y
+CONFIG_BSD_PROCESS_ACCT_V3=y
+CONFIG_TASKSTATS=y
+CONFIG_TASK_DELAY_ACCT=y
+CONFIG_TASK_XACCT=y
+CONFIG_TASK_IO_ACCOUNTING=y
+CONFIG_IKCONFIG=y
+CONFIG_IKCONFIG_PROC=y
+CONFIG_MEMCG=y
+CONFIG_MEMCG_SWAP=y
+CONFIG_BLK_CGROUP=y
+CONFIG_CFS_BANDWIDTH=y
+CONFIG_RT_GROUP_SCHED=y
+CONFIG_CGROUP_PIDS=y
+CONFIG_CGROUP_FREEZER=y
+CONFIG_CPUSETS=y
+CONFIG_CGROUP_DEVICE=y
+CONFIG_CGROUP_CPUACCT=y
+CONFIG_CGROUP_PERF=y
+CONFIG_CGROUP_BPF=y
+CONFIG_NAMESPACES=y
+CONFIG_USER_NS=y
+CONFIG_CHECKPOINT_RESTORE=y
+CONFIG_SCHED_AUTOGROUP=y
+CONFIG_BLK_DEV_INITRD=y
+# CONFIG_SYSFS_SYSCALL is not set
+CONFIG_KALLSYMS_ALL=y
+CONFIG_BPF_SYSCALL=y
+CONFIG_USERFAULTFD=y
+CONFIG_EMBEDDED=y
+# CONFIG_COMPAT_BRK is not set
+CONFIG_SLAB_FREELIST_RANDOM=y
+CONFIG_SLAB_FREELIST_HARDENED=y
+CONFIG_PROFILING=y
+CONFIG_OMAP_RESET_CLOCKS=y
+CONFIG_SOC_OMAP5=y
+CONFIG_SOC_AM33XX=y
+CONFIG_SOC_AM43XX=y
+CONFIG_SOC_DRA7XX=y
+CONFIG_SOC_HAS_OMAP2_SDRC=y
+CONFIG_ARM_THUMBEE=y
+CONFIG_PL310_ERRATA_588369=y
+CONFIG_PL310_ERRATA_727915=y
+CONFIG_PL310_ERRATA_753970=y
+CONFIG_ARM_ERRATA_430973=y
+CONFIG_ARM_ERRATA_773022=y
+CONFIG_SMP=y
+CONFIG_NR_CPUS=2
+CONFIG_ARM_PSCI=y
+CONFIG_HZ_250=y
+CONFIG_SECCOMP=y
+# CONFIG_ATAGS is not set
+CONFIG_ZBOOT_ROM_TEXT=0x0
+CONFIG_ZBOOT_ROM_BSS=0x0
+CONFIG_EFI=y
+# CONFIG_DMI is not set
+CONFIG_CPU_FREQ=y
+CONFIG_CPU_FREQ_STAT=y
+CONFIG_CPU_FREQ_GOV_POWERSAVE=y
+CONFIG_CPU_FREQ_GOV_USERSPACE=y
+CONFIG_CPU_FREQ_GOV_ONDEMAND=y
+CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y
+CONFIG_CPU_FREQ_GOV_SCHEDUTIL=y
+CONFIG_CPUFREQ_DT=y
+# CONFIG_ARM_OMAP2PLUS_CPUFREQ is not set
+CONFIG_ARM_TI_CPUFREQ=y
+CONFIG_HIBERNATION=y
+CONFIG_PM_AUTOSLEEP=y
+CONFIG_PM_WAKELOCKS=y
+CONFIG_PM_DEBUG=y
+CONFIG_PM_ADVANCED_DEBUG=y
+CONFIG_APM_EMULATION=y
+CONFIG_ARM_CRYPTO=y
+CONFIG_CRYPTO_SHA1_ARM_NEON=m
+CONFIG_CRYPTO_SHA256_ARM=m
+CONFIG_CRYPTO_SHA512_ARM=m
+CONFIG_CRYPTO_AES_ARM=m
+CONFIG_CRYPTO_AES_ARM_BS=m
+CONFIG_CRYPTO_CHACHA20_NEON=m
+CONFIG_OPROFILE=m
+CONFIG_KPROBES=y
+CONFIG_JUMP_LABEL=y
+CONFIG_MODULES=y
+CONFIG_MODULE_FORCE_LOAD=y
+CONFIG_MODULE_UNLOAD=y
+CONFIG_MODULE_FORCE_UNLOAD=y
+CONFIG_MODVERSIONS=y
+CONFIG_MODULE_COMPRESS=y
+CONFIG_MODULE_COMPRESS_XZ=y
+CONFIG_BLK_DEV_THROTTLING=y
+CONFIG_BLK_WBT=y
+CONFIG_BLK_SED_OPAL=y
+CONFIG_PARTITION_ADVANCED=y
+CONFIG_KARMA_PARTITION=y
+CONFIG_CFQ_GROUP_IOSCHED=y
+CONFIG_MQ_IOSCHED_KYBER=m
+CONFIG_IOSCHED_BFQ=m
+CONFIG_BFQ_GROUP_IOSCHED=y
+CONFIG_BINFMT_MISC=m
+CONFIG_KSM=y
+CONFIG_FRONTSWAP=y
+CONFIG_ZSWAP=y
+CONFIG_ZBUD=y
+CONFIG_ZSMALLOC=m
+CONFIG_NET=y
+CONFIG_PACKET=y
+CONFIG_PACKET_DIAG=m
+CONFIG_UNIX=y
+CONFIG_UNIX_DIAG=m
+CONFIG_XFRM_USER=m
+CONFIG_XFRM_SUB_POLICY=y
+CONFIG_NET_KEY=m
+CONFIG_NET_KEY_MIGRATE=y
+CONFIG_INET=y
+CONFIG_IP_MULTICAST=y
+CONFIG_IP_ADVANCED_ROUTER=y
+CONFIG_IP_FIB_TRIE_STATS=y
+CONFIG_IP_MULTIPLE_TABLES=y
+CONFIG_IP_ROUTE_MULTIPATH=y
+CONFIG_IP_ROUTE_VERBOSE=y
+CONFIG_IP_PNP=y
+CONFIG_IP_PNP_DHCP=y
+CONFIG_IP_PNP_BOOTP=y
+CONFIG_IP_PNP_RARP=y
+CONFIG_NET_IPIP=m
+CONFIG_NET_IPGRE_DEMUX=m
+CONFIG_NET_IPGRE=m
+CONFIG_NET_IPGRE_BROADCAST=y
+CONFIG_IP_MROUTE=y
+CONFIG_IP_MROUTE_MULTIPLE_TABLES=y
+CONFIG_IP_PIMSM_V1=y
+CONFIG_IP_PIMSM_V2=y
+CONFIG_NET_IPVTI=m
+CONFIG_NET_FOU_IP_TUNNELS=y
+CONFIG_INET_AH=m
+CONFIG_INET_ESP=m
+CONFIG_INET_IPCOMP=m
+CONFIG_INET_XFRM_MODE_TRANSPORT=m
+CONFIG_INET_XFRM_MODE_TUNNEL=m
+CONFIG_INET_XFRM_MODE_BEET=m
+CONFIG_INET_DIAG=m
+CONFIG_INET_UDP_DIAG=m
+CONFIG_INET_DIAG_DESTROY=y
+CONFIG_TCP_CONG_ADVANCED=y
+CONFIG_TCP_CONG_HSTCP=m
+CONFIG_TCP_CONG_HYBLA=m
+CONFIG_TCP_CONG_NV=m
+CONFIG_TCP_CONG_SCALABLE=m
+CONFIG_TCP_CONG_LP=m
+CONFIG_TCP_CONG_VENO=m
+CONFIG_TCP_CONG_YEAH=m
+CONFIG_TCP_CONG_ILLINOIS=m
+CONFIG_TCP_CONG_DCTCP=m
+CONFIG_TCP_CONG_CDG=m
+CONFIG_TCP_CONG_BBR=m
+CONFIG_TCP_MD5SIG=y
+CONFIG_IPV6_ROUTER_PREF=y
+CONFIG_IPV6_ROUTE_INFO=y
+CONFIG_IPV6_OPTIMISTIC_DAD=y
+CONFIG_INET6_AH=m
+CONFIG_INET6_ESP=m
+CONFIG_INET6_ESP_OFFLOAD=m
+CONFIG_INET6_IPCOMP=m
+CONFIG_IPV6_MIP6=y
+CONFIG_IPV6_ILA=m
+CONFIG_INET6_XFRM_MODE_TRANSPORT=m
+CONFIG_INET6_XFRM_MODE_TUNNEL=m
+CONFIG_INET6_XFRM_MODE_BEET=m
+CONFIG_INET6_XFRM_MODE_ROUTEOPTIMIZATION=m
+CONFIG_IPV6_VTI=m
+CONFIG_IPV6_SIT=m
+CONFIG_IPV6_SIT_6RD=y
+CONFIG_IPV6_GRE=m
+CONFIG_IPV6_SUBTREES=y
+CONFIG_IPV6_MROUTE=y
+CONFIG_IPV6_MROUTE_MULTIPLE_TABLES=y
+CONFIG_IPV6_PIMSM_V2=y
+CONFIG_IPV6_SEG6_LWTUNNEL=y
+CONFIG_IPV6_SEG6_HMAC=y
+CONFIG_NETLABEL=y
+CONFIG_NETFILTER=y
+CONFIG_NF_CONNTRACK=y
+CONFIG_NF_LOG_NETDEV=m
+CONFIG_NF_CONNTRACK_SECMARK=y
+CONFIG_NF_CONNTRACK_ZONES=y
+CONFIG_NF_CONNTRACK_EVENTS=y
+CONFIG_NF_CONNTRACK_TIMEOUT=y
+CONFIG_NF_CONNTRACK_TIMESTAMP=y
+CONFIG_NF_CONNTRACK_AMANDA=m
+CONFIG_NF_CONNTRACK_FTP=m
+CONFIG_NF_CONNTRACK_H323=m
+CONFIG_NF_CONNTRACK_IRC=m
+CONFIG_NF_CONNTRACK_NETBIOS_NS=m
+CONFIG_NF_CONNTRACK_SNMP=m
+CONFIG_NF_CONNTRACK_PPTP=m
+CONFIG_NF_CONNTRACK_SANE=m
+CONFIG_NF_CONNTRACK_SIP=m
+CONFIG_NF_CONNTRACK_TFTP=m
+CONFIG_NF_CT_NETLINK=m
+CONFIG_NF_CT_NETLINK_TIMEOUT=m
+CONFIG_NF_CT_NETLINK_HELPER=m
+CONFIG_NETFILTER_NETLINK_GLUE_CT=y
+CONFIG_NF_TABLES=y
+CONFIG_NF_TABLES_SET=m
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_NETDEV=y
+CONFIG_NFT_NUMGEN=m
+CONFIG_NFT_CT=y
+CONFIG_NFT_COUNTER=m
+CONFIG_NFT_LOG=m
+CONFIG_NFT_LIMIT=m
+CONFIG_NFT_MASQ=m
+CONFIG_NFT_REDIR=m
+CONFIG_NFT_NAT=y
+CONFIG_NFT_OBJREF=m
+CONFIG_NFT_QUEUE=m
+CONFIG_NFT_QUOTA=m
+CONFIG_NFT_REJECT=m
+CONFIG_NFT_COMPAT=m
+CONFIG_NFT_HASH=m
+CONFIG_NFT_FIB_INET=m
+CONFIG_NFT_DUP_NETDEV=m
+CONFIG_NFT_FWD_NETDEV=m
+CONFIG_NFT_FIB_NETDEV=m
+CONFIG_NETFILTER_XT_SET=m
+CONFIG_NETFILTER_XT_TARGET_AUDIT=m
+CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m
+CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m
+CONFIG_NETFILTER_XT_TARGET_CONNMARK=m
+CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=m
+CONFIG_NETFILTER_XT_TARGET_CT=m
+CONFIG_NETFILTER_XT_TARGET_DSCP=m
+CONFIG_NETFILTER_XT_TARGET_HMARK=m
+CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m
+CONFIG_NETFILTER_XT_TARGET_LED=m
+CONFIG_NETFILTER_XT_TARGET_LOG=m
+CONFIG_NETFILTER_XT_TARGET_MARK=m
+CONFIG_NETFILTER_XT_TARGET_NFLOG=m
+CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m
+CONFIG_NETFILTER_XT_TARGET_TEE=m
+CONFIG_NETFILTER_XT_TARGET_TPROXY=m
+CONFIG_NETFILTER_XT_TARGET_TRACE=m
+CONFIG_NETFILTER_XT_TARGET_SECMARK=m
+CONFIG_NETFILTER_XT_TARGET_TCPMSS=m
+CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m
+CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m
+CONFIG_NETFILTER_XT_MATCH_BPF=m
+CONFIG_NETFILTER_XT_MATCH_CGROUP=m
+CONFIG_NETFILTER_XT_MATCH_CLUSTER=m
+CONFIG_NETFILTER_XT_MATCH_COMMENT=m
+CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m
+CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m
+CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m
+CONFIG_NETFILTER_XT_MATCH_CONNMARK=m
+CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m
+CONFIG_NETFILTER_XT_MATCH_CPU=m
+CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m
+CONFIG_NETFILTER_XT_MATCH_DSCP=m
+CONFIG_NETFILTER_XT_MATCH_ESP=m
+CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m
+CONFIG_NETFILTER_XT_MATCH_HELPER=m
+CONFIG_NETFILTER_XT_MATCH_IPCOMP=m
+CONFIG_NETFILTER_XT_MATCH_IPRANGE=m
+CONFIG_NETFILTER_XT_MATCH_IPVS=m
+CONFIG_NETFILTER_XT_MATCH_LENGTH=m
+CONFIG_NETFILTER_XT_MATCH_LIMIT=m
+CONFIG_NETFILTER_XT_MATCH_MAC=m
+CONFIG_NETFILTER_XT_MATCH_MARK=m
+CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m
+CONFIG_NETFILTER_XT_MATCH_NFACCT=m
+CONFIG_NETFILTER_XT_MATCH_OSF=m
+CONFIG_NETFILTER_XT_MATCH_OWNER=m
+CONFIG_NETFILTER_XT_MATCH_POLICY=m
+CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m
+CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m
+CONFIG_NETFILTER_XT_MATCH_QUOTA=m
+CONFIG_NETFILTER_XT_MATCH_RATEEST=m
+CONFIG_NETFILTER_XT_MATCH_REALM=m
+CONFIG_NETFILTER_XT_MATCH_RECENT=m
+CONFIG_NETFILTER_XT_MATCH_SOCKET=m
+CONFIG_NETFILTER_XT_MATCH_STATE=m
+CONFIG_NETFILTER_XT_MATCH_STATISTIC=m
+CONFIG_NETFILTER_XT_MATCH_STRING=m
+CONFIG_NETFILTER_XT_MATCH_TCPMSS=m
+CONFIG_NETFILTER_XT_MATCH_TIME=m
+CONFIG_NETFILTER_XT_MATCH_U32=m
+CONFIG_IP_SET=m
+CONFIG_IP_SET_BITMAP_IP=m
+CONFIG_IP_SET_BITMAP_IPMAC=m
+CONFIG_IP_SET_BITMAP_PORT=m
+CONFIG_IP_SET_HASH_IP=m
+CONFIG_IP_SET_HASH_IPMARK=m
+CONFIG_IP_SET_HASH_IPPORT=m
+CONFIG_IP_SET_HASH_IPPORTIP=m
+CONFIG_IP_SET_HASH_IPPORTNET=m
+CONFIG_IP_SET_HASH_IPMAC=m
+CONFIG_IP_SET_HASH_MAC=m
+CONFIG_IP_SET_HASH_NETPORTNET=m
+CONFIG_IP_SET_HASH_NET=m
+CONFIG_IP_SET_HASH_NETNET=m
+CONFIG_IP_SET_HASH_NETPORT=m
+CONFIG_IP_SET_HASH_NETIFACE=m
+CONFIG_IP_SET_LIST_SET=m
+CONFIG_IP_VS=m
+CONFIG_IP_VS_IPV6=y
+CONFIG_IP_VS_PROTO_TCP=y
+CONFIG_IP_VS_PROTO_UDP=y
+CONFIG_IP_VS_PROTO_ESP=y
+CONFIG_IP_VS_PROTO_AH=y
+CONFIG_IP_VS_PROTO_SCTP=y
+CONFIG_IP_VS_RR=m
+CONFIG_IP_VS_WRR=m
+CONFIG_IP_VS_LC=m
+CONFIG_IP_VS_WLC=m
+CONFIG_IP_VS_FO=m
+CONFIG_IP_VS_OVF=m
+CONFIG_IP_VS_LBLC=m
+CONFIG_IP_VS_LBLCR=m
+CONFIG_IP_VS_DH=m
+CONFIG_IP_VS_SH=m
+CONFIG_IP_VS_SED=m
+CONFIG_IP_VS_NQ=m
+CONFIG_IP_VS_FTP=m
+CONFIG_IP_VS_PE_SIP=m
+CONFIG_NFT_CHAIN_ROUTE_IPV4=m
+CONFIG_NFT_DUP_IPV4=m
+CONFIG_NFT_FIB_IPV4=m
+CONFIG_NF_TABLES_ARP=y
+CONFIG_NF_LOG_ARP=m
+CONFIG_NFT_CHAIN_NAT_IPV4=m
+CONFIG_NFT_MASQ_IPV4=m
+CONFIG_NFT_REDIR_IPV4=m
+CONFIG_IP_NF_IPTABLES=y
+CONFIG_IP_NF_MATCH_AH=m
+CONFIG_IP_NF_MATCH_ECN=m
+CONFIG_IP_NF_MATCH_RPFILTER=m
+CONFIG_IP_NF_MATCH_TTL=m
+CONFIG_IP_NF_FILTER=y
+CONFIG_IP_NF_TARGET_REJECT=m
+CONFIG_IP_NF_TARGET_SYNPROXY=m
+CONFIG_IP_NF_NAT=y
+CONFIG_IP_NF_TARGET_MASQUERADE=m
+CONFIG_IP_NF_TARGET_NETMAP=m
+CONFIG_IP_NF_TARGET_REDIRECT=m
+CONFIG_IP_NF_MANGLE=y
+CONFIG_IP_NF_TARGET_CLUSTERIP=m
+CONFIG_IP_NF_TARGET_ECN=m
+CONFIG_IP_NF_TARGET_TTL=m
+CONFIG_IP_NF_RAW=m
+CONFIG_IP_NF_SECURITY=m
+CONFIG_IP_NF_ARPTABLES=m
+CONFIG_IP_NF_ARPFILTER=m
+CONFIG_IP_NF_ARP_MANGLE=m
+CONFIG_NFT_CHAIN_ROUTE_IPV6=m
+CONFIG_NFT_CHAIN_NAT_IPV6=m
+CONFIG_NFT_MASQ_IPV6=m
+CONFIG_NFT_REDIR_IPV6=m
+CONFIG_NFT_DUP_IPV6=m
+CONFIG_NFT_FIB_IPV6=m
+CONFIG_IP6_NF_MATCH_AH=m
+CONFIG_IP6_NF_MATCH_EUI64=m
+CONFIG_IP6_NF_MATCH_FRAG=m
+CONFIG_IP6_NF_MATCH_OPTS=m
+CONFIG_IP6_NF_MATCH_HL=m
+CONFIG_IP6_NF_MATCH_IPV6HEADER=m
+CONFIG_IP6_NF_MATCH_MH=m
+CONFIG_IP6_NF_MATCH_RPFILTER=m
+CONFIG_IP6_NF_MATCH_RT=m
+CONFIG_IP6_NF_TARGET_HL=m
+CONFIG_IP6_NF_FILTER=m
+CONFIG_IP6_NF_TARGET_REJECT=m
+CONFIG_IP6_NF_TARGET_SYNPROXY=m
+CONFIG_IP6_NF_MANGLE=m
+CONFIG_IP6_NF_RAW=m
+CONFIG_IP6_NF_SECURITY=m
+CONFIG_IP6_NF_NAT=m
+CONFIG_IP6_NF_TARGET_MASQUERADE=m
+CONFIG_IP6_NF_TARGET_NPT=m
+CONFIG_NF_TABLES_BRIDGE=y
+CONFIG_NFT_BRIDGE_REJECT=m
+CONFIG_NF_LOG_BRIDGE=m
+CONFIG_BRIDGE_NF_EBTABLES=m
+CONFIG_BRIDGE_EBT_BROUTE=m
+CONFIG_BRIDGE_EBT_T_FILTER=m
+CONFIG_BRIDGE_EBT_T_NAT=m
+CONFIG_BRIDGE_EBT_802_3=m
+CONFIG_BRIDGE_EBT_AMONG=m
+CONFIG_BRIDGE_EBT_ARP=m
+CONFIG_BRIDGE_EBT_IP=m
+CONFIG_BRIDGE_EBT_IP6=m
+CONFIG_BRIDGE_EBT_LIMIT=m
+CONFIG_BRIDGE_EBT_MARK=m
+CONFIG_BRIDGE_EBT_PKTTYPE=m
+CONFIG_BRIDGE_EBT_STP=m
+CONFIG_BRIDGE_EBT_VLAN=m
+CONFIG_BRIDGE_EBT_ARPREPLY=m
+CONFIG_BRIDGE_EBT_DNAT=m
+CONFIG_BRIDGE_EBT_MARK_T=m
+CONFIG_BRIDGE_EBT_REDIRECT=m
+CONFIG_BRIDGE_EBT_SNAT=m
+CONFIG_BRIDGE_EBT_LOG=m
+CONFIG_BRIDGE_EBT_NFLOG=m
+CONFIG_IP_DCCP=m
+CONFIG_SCTP_COOKIE_HMAC_SHA1=y
+CONFIG_RDS=m
+CONFIG_RDS_TCP=m
+CONFIG_TIPC=m
+CONFIG_ATM=m
+CONFIG_ATM_CLIP=m
+CONFIG_ATM_LANE=m
+CONFIG_ATM_MPOA=m
+CONFIG_ATM_BR2684=m
+CONFIG_L2TP=m
+CONFIG_L2TP_DEBUGFS=m
+CONFIG_L2TP_V3=y
+CONFIG_L2TP_IP=m
+CONFIG_L2TP_ETH=m
+CONFIG_BRIDGE=m
+CONFIG_BRIDGE_VLAN_FILTERING=y
+CONFIG_VLAN_8021Q=y
+CONFIG_VLAN_8021Q_GVRP=y
+CONFIG_VLAN_8021Q_MVRP=y
+CONFIG_LLC2=m
+CONFIG_ATALK=m
+CONFIG_DEV_APPLETALK=m
+CONFIG_IPDDP=m
+CONFIG_IPDDP_ENCAP=y
+CONFIG_PHONET=m
+CONFIG_6LOWPAN=m
+CONFIG_6LOWPAN_GHC_EXT_HDR_HOP=m
+CONFIG_6LOWPAN_GHC_UDP=m
+CONFIG_6LOWPAN_GHC_ICMPV6=m
+CONFIG_6LOWPAN_GHC_EXT_HDR_DEST=m
+CONFIG_6LOWPAN_GHC_EXT_HDR_FRAG=m
+CONFIG_6LOWPAN_GHC_EXT_HDR_ROUTE=m
+CONFIG_IEEE802154=m
+CONFIG_IEEE802154_6LOWPAN=m
+CONFIG_MAC802154=m
+CONFIG_NET_SCHED=y
+CONFIG_NET_SCH_CBQ=m
+CONFIG_NET_SCH_HTB=m
+CONFIG_NET_SCH_HFSC=m
+CONFIG_NET_SCH_ATM=m
+CONFIG_NET_SCH_PRIO=m
+CONFIG_NET_SCH_MULTIQ=m
+CONFIG_NET_SCH_RED=m
+CONFIG_NET_SCH_SFB=m
+CONFIG_NET_SCH_SFQ=m
+CONFIG_NET_SCH_TEQL=m
+CONFIG_NET_SCH_TBF=m
+CONFIG_NET_SCH_GRED=m
+CONFIG_NET_SCH_DSMARK=m
+CONFIG_NET_SCH_NETEM=m
+CONFIG_NET_SCH_DRR=m
+CONFIG_NET_SCH_MQPRIO=m
+CONFIG_NET_SCH_CHOKE=m
+CONFIG_NET_SCH_QFQ=m
+CONFIG_NET_SCH_CODEL=m
+CONFIG_NET_SCH_FQ_CODEL=m
+CONFIG_NET_SCH_FQ=m
+CONFIG_NET_SCH_HHF=m
+CONFIG_NET_SCH_PIE=m
+CONFIG_NET_SCH_INGRESS=m
+CONFIG_NET_SCH_PLUG=m
+CONFIG_NET_CLS_BASIC=m
+CONFIG_NET_CLS_TCINDEX=m
+CONFIG_NET_CLS_ROUTE4=m
+CONFIG_NET_CLS_FW=m
+CONFIG_NET_CLS_U32=m
+CONFIG_CLS_U32_PERF=y
+CONFIG_CLS_U32_MARK=y
+CONFIG_NET_CLS_RSVP=m
+CONFIG_NET_CLS_RSVP6=m
+CONFIG_NET_CLS_FLOW=m
+CONFIG_NET_CLS_CGROUP=m
+CONFIG_NET_CLS_BPF=m
+CONFIG_NET_CLS_FLOWER=m
+CONFIG_NET_CLS_MATCHALL=m
+CONFIG_NET_EMATCH=y
+CONFIG_NET_EMATCH_CMP=m
+CONFIG_NET_EMATCH_NBYTE=m
+CONFIG_NET_EMATCH_U32=m
+CONFIG_NET_EMATCH_META=m
+CONFIG_NET_EMATCH_TEXT=m
+CONFIG_NET_EMATCH_CANID=m
+CONFIG_NET_EMATCH_IPSET=m
+CONFIG_NET_CLS_ACT=y
+CONFIG_NET_ACT_POLICE=m
+CONFIG_NET_ACT_GACT=m
+CONFIG_GACT_PROB=y
+CONFIG_NET_ACT_MIRRED=m
+CONFIG_NET_ACT_SAMPLE=m
+CONFIG_NET_ACT_IPT=m
+CONFIG_NET_ACT_NAT=m
+CONFIG_NET_ACT_PEDIT=m
+CONFIG_NET_ACT_SIMP=m
+CONFIG_NET_ACT_SKBEDIT=m
+CONFIG_NET_ACT_CSUM=m
+CONFIG_NET_ACT_VLAN=m
+CONFIG_NET_ACT_BPF=m
+CONFIG_NET_ACT_CONNMARK=m
+CONFIG_NET_ACT_SKBMOD=m
+CONFIG_NET_ACT_IFE=m
+CONFIG_NET_ACT_TUNNEL_KEY=m
+CONFIG_NET_IFE_SKBMARK=m
+CONFIG_NET_IFE_SKBPRIO=m
+CONFIG_NET_IFE_SKBTCINDEX=m
+CONFIG_NET_CLS_IND=y
+CONFIG_DCB=y
+CONFIG_BATMAN_ADV=m
+# CONFIG_BATMAN_ADV_BATMAN_V is not set
+CONFIG_BATMAN_ADV_NC=y
+CONFIG_BATMAN_ADV_MCAST=y
+CONFIG_BATMAN_ADV_DEBUGFS=y
+CONFIG_OPENVSWITCH=m
+CONFIG_VSOCKETS=m
+CONFIG_VIRTIO_VSOCKETS=m
+CONFIG_NETLINK_DIAG=m
+CONFIG_NET_MPLS_GSO=y
+CONFIG_MPLS_ROUTING=m
+CONFIG_MPLS_IPTUNNEL=m
+CONFIG_NET_SWITCHDEV=y
+CONFIG_CGROUP_NET_PRIO=y
+CONFIG_BPF_JIT=y
+CONFIG_NET_PKTGEN=m
+CONFIG_NET_DROP_MONITOR=m
+CONFIG_HAMRADIO=y
+CONFIG_AX25=m
+CONFIG_NETROM=m
+CONFIG_ROSE=m
+CONFIG_MKISS=m
+CONFIG_6PACK=m
+CONFIG_BPQETHER=m
+CONFIG_BAYCOM_SER_FDX=m
+CONFIG_BAYCOM_SER_HDX=m
+CONFIG_YAM=m
+CONFIG_CAN=y
+CONFIG_CAN_RAW=m
+CONFIG_CAN_BCM=m
+CONFIG_CAN_GW=m
+CONFIG_CAN_ISOTP=m
+CONFIG_CAN_VCAN=m
+CONFIG_CAN_SLCAN=m
+CONFIG_CAN_C_CAN=y
+CONFIG_CAN_C_CAN_PLATFORM=y
+CONFIG_CAN_HI311X=m
+CONFIG_CAN_MCP251X=m
+CONFIG_CAN_8DEV_USB=m
+CONFIG_CAN_EMS_USB=m
+CONFIG_CAN_ESD_USB2=m
+CONFIG_CAN_GS_USB=m
+CONFIG_CAN_KVASER_USB=m
+CONFIG_CAN_PEAK_USB=m
+CONFIG_BT=m
+CONFIG_BT_RFCOMM=m
+CONFIG_BT_RFCOMM_TTY=y
+CONFIG_BT_BNEP=m
+CONFIG_BT_BNEP_MC_FILTER=y
+CONFIG_BT_BNEP_PROTO_FILTER=y
+CONFIG_BT_HIDP=m
+CONFIG_BT_6LOWPAN=m
+CONFIG_BT_LEDS=y
+CONFIG_BT_HCIBTUSB=m
+CONFIG_BT_HCIBTSDIO=m
+CONFIG_BT_HCIUART=m
+CONFIG_BT_HCIUART_ATH3K=y
+CONFIG_BT_HCIUART_LL=y
+CONFIG_BT_HCIUART_3WIRE=y
+CONFIG_BT_HCIBCM203X=m
+CONFIG_BT_HCIBPA10X=m
+CONFIG_BT_HCIBFUSB=m
+CONFIG_BT_HCIVHCI=m
+CONFIG_BT_MRVL=m
+CONFIG_BT_MRVL_SDIO=m
+CONFIG_BT_ATH3K=m
+CONFIG_BT_WILINK=m
+CONFIG_AF_RXRPC_IPV6=y
+CONFIG_RXKAD=y
+CONFIG_CFG80211=m
+# CONFIG_CFG80211_DEFAULT_PS is not set
+CONFIG_CFG80211_WEXT=y
+CONFIG_MAC80211=m
+CONFIG_MAC80211_RC_MINSTREL_VHT=y
+CONFIG_MAC80211_MESH=y
+CONFIG_WIMAX=m
+CONFIG_RFKILL=y
+CONFIG_RFKILL_INPUT=y
+CONFIG_NET_9P=m
+CONFIG_NET_9P_VIRTIO=m
+CONFIG_NFC=m
+CONFIG_NFC_DIGITAL=m
+CONFIG_NFC_NCI=m
+CONFIG_NFC_NCI_SPI=m
+CONFIG_NFC_NCI_UART=m
+CONFIG_NFC_HCI=m
+CONFIG_NFC_SHDLC=y
+CONFIG_NFC_TRF7970A=m
+CONFIG_NFC_SIM=m
+CONFIG_NFC_PORT100=m
+CONFIG_NFC_PN544_I2C=m
+CONFIG_NFC_PN533_USB=m
+CONFIG_NFC_PN533_I2C=m
+CONFIG_NFC_MICROREAD_I2C=m
+CONFIG_NFC_ST21NFCA_I2C=m
+CONFIG_NFC_ST_NCI_I2C=m
+CONFIG_NFC_ST_NCI_SPI=m
+CONFIG_NFC_NXP_NCI=m
+CONFIG_NFC_NXP_NCI_I2C=m
+CONFIG_NFC_ST95HF=m
+CONFIG_NET_DEVLINK=m
+# CONFIG_UEVENT_HELPER is not set
+CONFIG_DEVTMPFS=y
+CONFIG_DEVTMPFS_MOUNT=y
+CONFIG_EXTRA_FIRMWARE="am335x-pm-firmware.elf am335x-bone-scale-data.bin am335x-evm-scale-data.bin am43x-evm-scale-data.bin"
+CONFIG_EXTRA_FIRMWARE_DIR="firmware"
+CONFIG_CMA_SIZE_MBYTES=48
+CONFIG_OMAP_OCP2SCP=y
+CONFIG_SIMPLE_PM_BUS=y
+CONFIG_CONNECTOR=y
+CONFIG_GNSS=y
+CONFIG_GNSS_SIRF_SERIAL=y
+CONFIG_GNSS_UBX_SERIAL=y
+CONFIG_MTD=y
+CONFIG_MTD_OF_PARTS=m
+CONFIG_MTD_AR7_PARTS=m
+CONFIG_MTD_BLOCK=m
+CONFIG_MTD_BLOCK_RO=m
+CONFIG_RFD_FTL=m
+CONFIG_SSFDC=m
+CONFIG_MTD_OOPS=m
+CONFIG_MTD_SWAP=m
+CONFIG_MTD_PHYSMAP=m
+CONFIG_MTD_PLATRAM=m
+CONFIG_MTD_DATAFLASH=m
+CONFIG_MTD_M25P80=m
+CONFIG_MTD_SST25L=m
+CONFIG_MTD_ONENAND=y
+CONFIG_MTD_ONENAND_VERIFY_WRITE=y
+CONFIG_MTD_ONENAND_2X_PROGRAM=y
+CONFIG_MTD_NAND=y
+CONFIG_MTD_NAND_ECC_BCH=y
+CONFIG_MTD_NAND_OMAP2=m
+CONFIG_MTD_NAND_NANDSIM=m
+CONFIG_MTD_LPDDR=m
+CONFIG_MTD_SPI_NOR=m
+CONFIG_MTD_UBI=y
+CONFIG_MTD_UBI_BLOCK=y
+CONFIG_OF_OVERLAY=y
+CONFIG_BLK_DEV_NULL_BLK=m
+CONFIG_ZRAM=m
+CONFIG_BLK_DEV_LOOP=m
+CONFIG_BLK_DEV_DRBD=m
+CONFIG_BLK_DEV_NBD=m
+CONFIG_BLK_DEV_RAM=m
+CONFIG_BLK_DEV_RAM_SIZE=16384
+CONFIG_ATA_OVER_ETH=m
+CONFIG_VIRTIO_BLK=m
+CONFIG_BLK_DEV_RBD=m
+CONFIG_AD525X_DPOT=m
+CONFIG_AD525X_DPOT_I2C=m
+CONFIG_AD525X_DPOT_SPI=m
+CONFIG_ICS932S401=m
+CONFIG_APDS9802ALS=m
+CONFIG_ISL29003=m
+CONFIG_ISL29020=m
+CONFIG_SENSORS_TSL2550=m
+CONFIG_SENSORS_BH1770=m
+CONFIG_SENSORS_APDS990X=m
+CONFIG_HMC6352=m
+CONFIG_DS1682=m
+CONFIG_SRAM=y
+CONFIG_DMA_BUF_PHYS=y
+CONFIG_TIEQEP=y
+CONFIG_EEPROM_AT24=y
+CONFIG_EEPROM_AT25=y
+CONFIG_EEPROM_LEGACY=m
+CONFIG_EEPROM_MAX6875=m
+CONFIG_EEPROM_93XX46=m
+CONFIG_TI_ST=m
+CONFIG_BEAGLEBONE_PINMUX_HELPER=y
+CONFIG_RAID_ATTRS=m
+# CONFIG_SCSI_PROC_FS is not set
+CONFIG_BLK_DEV_SD=y
+CONFIG_CHR_DEV_ST=m
+CONFIG_CHR_DEV_OSST=m
+CONFIG_BLK_DEV_SR=m
+CONFIG_BLK_DEV_SR_VENDOR=y
+CONFIG_CHR_DEV_SG=y
+CONFIG_CHR_DEV_SCH=m
+CONFIG_SCSI_CONSTANTS=y
+CONFIG_SCSI_LOGGING=y
+CONFIG_SCSI_SCAN_ASYNC=y
+CONFIG_SCSI_SPI_ATTRS=m
+CONFIG_SCSI_FC_ATTRS=m
+CONFIG_SCSI_SAS_LIBSAS=m
+CONFIG_SCSI_SAS_ATA=y
+CONFIG_SCSI_SRP_ATTRS=m
+CONFIG_ISCSI_TCP=m
+CONFIG_ISCSI_BOOT_SYSFS=m
+CONFIG_SCSI_UFSHCD=m
+CONFIG_LIBFC=m
+CONFIG_LIBFCOE=m
+CONFIG_SCSI_VIRTIO=m
+CONFIG_SCSI_DH=y
+CONFIG_SCSI_DH_RDAC=m
+CONFIG_SCSI_DH_HP_SW=m
+CONFIG_SCSI_DH_EMC=m
+CONFIG_SCSI_DH_ALUA=m
+CONFIG_SCSI_OSD_INITIATOR=m
+CONFIG_SCSI_OSD_ULD=m
+CONFIG_ATA=y
+CONFIG_SATA_AHCI_PLATFORM=y
+CONFIG_MD=y
+CONFIG_BLK_DEV_MD=m
+CONFIG_MD_LINEAR=m
+CONFIG_MD_RAID0=m
+CONFIG_MD_RAID1=m
+CONFIG_MD_RAID10=m
+CONFIG_MD_MULTIPATH=m
+CONFIG_MD_FAULTY=m
+CONFIG_BCACHE=m
+CONFIG_BLK_DEV_DM=m
+CONFIG_DM_MQ_DEFAULT=y
+CONFIG_DM_CRYPT=m
+CONFIG_DM_SNAPSHOT=m
+CONFIG_DM_THIN_PROVISIONING=m
+CONFIG_DM_CACHE=m
+CONFIG_DM_ERA=m
+CONFIG_DM_MIRROR=m
+CONFIG_DM_LOG_USERSPACE=m
+CONFIG_DM_ZERO=m
+CONFIG_DM_MULTIPATH=m
+CONFIG_DM_MULTIPATH_QL=m
+CONFIG_DM_MULTIPATH_ST=m
+CONFIG_DM_DELAY=m
+CONFIG_DM_UEVENT=y
+CONFIG_DM_FLAKEY=m
+CONFIG_DM_VERITY=m
+CONFIG_DM_SWITCH=m
+CONFIG_DM_LOG_WRITES=m
+CONFIG_DM_INTEGRITY=m
+CONFIG_TARGET_CORE=m
+CONFIG_TCM_IBLOCK=m
+CONFIG_TCM_FILEIO=m
+CONFIG_TCM_PSCSI=m
+CONFIG_TCM_USER2=m
+CONFIG_LOOPBACK_TARGET=m
+CONFIG_TCM_FC=m
+CONFIG_ISCSI_TARGET=m
+CONFIG_NETDEVICES=y
+CONFIG_BONDING=m
+CONFIG_DUMMY=m
+CONFIG_EQUALIZER=m
+CONFIG_IFB=m
+CONFIG_NET_TEAM=m
+CONFIG_NET_TEAM_MODE_BROADCAST=m
+CONFIG_NET_TEAM_MODE_ROUNDROBIN=m
+CONFIG_NET_TEAM_MODE_RANDOM=m
+CONFIG_NET_TEAM_MODE_ACTIVEBACKUP=m
+CONFIG_NET_TEAM_MODE_LOADBALANCE=m
+CONFIG_MACVLAN=m
+CONFIG_MACVTAP=m
+CONFIG_IPVLAN=m
+CONFIG_IPVTAP=m
+CONFIG_VXLAN=m
+CONFIG_GENEVE=m
+CONFIG_GTP=m
+CONFIG_MACSEC=m
+CONFIG_NETCONSOLE=m
+CONFIG_NETCONSOLE_DYNAMIC=y
+CONFIG_TUN=m
+CONFIG_VETH=m
+CONFIG_VIRTIO_NET=m
+CONFIG_NLMON=m
+CONFIG_NET_VRF=m
+CONFIG_ATM_DUMMY=m
+# CONFIG_NET_VENDOR_ALACRITECH is not set
+# CONFIG_NET_VENDOR_AMAZON is not set
+# CONFIG_NET_VENDOR_AQUANTIA is not set
+# CONFIG_NET_VENDOR_ARC is not set
+# CONFIG_NET_VENDOR_AURORA is not set
+# CONFIG_NET_VENDOR_BROADCOM is not set
+# CONFIG_NET_VENDOR_CADENCE is not set
+# CONFIG_NET_VENDOR_CAVIUM is not set
+# CONFIG_NET_VENDOR_CIRRUS is not set
+# CONFIG_NET_VENDOR_CORTINA is not set
+# CONFIG_NET_VENDOR_EZCHIP is not set
+# CONFIG_NET_VENDOR_FARADAY is not set
+# CONFIG_NET_VENDOR_HISILICON is not set
+# CONFIG_NET_VENDOR_HUAWEI is not set
+# CONFIG_NET_VENDOR_INTEL is not set
+# CONFIG_NET_VENDOR_MARVELL is not set
+# CONFIG_NET_VENDOR_MELLANOX is not set
+CONFIG_KS8851=m
+CONFIG_ENC28J60=y
+CONFIG_ENCX24J600=y
+# CONFIG_NET_VENDOR_NATSEMI is not set
+# CONFIG_NET_VENDOR_NETRONOME is not set
+# CONFIG_NET_VENDOR_NI is not set
+# CONFIG_NET_VENDOR_QUALCOMM is not set
+# CONFIG_NET_VENDOR_RENESAS is not set
+# CONFIG_NET_VENDOR_ROCKER is not set
+# CONFIG_NET_VENDOR_SAMSUNG is not set
+# CONFIG_NET_VENDOR_SEEQ is not set
+# CONFIG_NET_VENDOR_SOLARFLARE is not set
+CONFIG_SMC91X=m
+CONFIG_SMC911X=m
+CONFIG_SMSC911X=m
+# CONFIG_NET_VENDOR_SOCIONEXT is not set
+# CONFIG_NET_VENDOR_STMICRO is not set
+# CONFIG_NET_VENDOR_SYNOPSYS is not set
+CONFIG_TI_CPSW=y
+CONFIG_TI_CPTS=y
+# CONFIG_NET_VENDOR_VIA is not set
+CONFIG_WIZNET_W5100=y
+CONFIG_WIZNET_W5100_SPI=y
+CONFIG_AT803X_PHY=y
+CONFIG_DP83848_PHY=y
+CONFIG_DP83867_PHY=y
+CONFIG_MICREL_PHY=y
+CONFIG_MICROSEMI_PHY=y
+CONFIG_SMSC_PHY=y
+CONFIG_VITESSE_PHY=y
+CONFIG_PPP=m
+CONFIG_PPP_BSDCOMP=m
+CONFIG_PPP_DEFLATE=m
+CONFIG_PPP_FILTER=y
+CONFIG_PPP_MPPE=m
+CONFIG_PPP_MULTILINK=y
+CONFIG_PPPOATM=m
+CONFIG_PPPOE=m
+CONFIG_PPTP=m
+CONFIG_PPPOL2TP=m
+CONFIG_PPP_ASYNC=m
+CONFIG_PPP_SYNC_TTY=m
+CONFIG_SLIP=m
+CONFIG_SLIP_COMPRESSED=y
+CONFIG_SLIP_SMART=y
+CONFIG_SLIP_MODE_SLIP6=y
+CONFIG_USB_CATC=m
+CONFIG_USB_KAWETH=m
+CONFIG_USB_PEGASUS=m
+CONFIG_USB_RTL8150=m
+CONFIG_USB_RTL8152=m
+CONFIG_USB_LAN78XX=m
+CONFIG_USB_USBNET=y
+CONFIG_USB_NET_AX8817X=m
+CONFIG_USB_NET_AX88179_178A=m
+CONFIG_USB_NET_CDCETHER=m
+CONFIG_USB_NET_CDC_EEM=m
+CONFIG_USB_NET_CDC_NCM=m
+CONFIG_USB_NET_HUAWEI_CDC_NCM=m
+CONFIG_USB_NET_CDC_MBIM=m
+CONFIG_USB_NET_DM9601=m
+CONFIG_USB_NET_SR9700=m
+CONFIG_USB_NET_SR9800=m
+CONFIG_USB_NET_SMSC75XX=m
+CONFIG_USB_NET_SMSC95XX=y
+CONFIG_USB_NET_GL620A=m
+CONFIG_USB_NET_NET1080=m
+CONFIG_USB_NET_PLUSB=m
+CONFIG_USB_NET_MCS7830=m
+CONFIG_USB_NET_CDC_SUBSET=m
+CONFIG_USB_ALI_M5632=y
+CONFIG_USB_AN2720=y
+CONFIG_USB_EPSON2888=y
+CONFIG_USB_KC2190=y
+CONFIG_USB_NET_ZAURUS=m
+CONFIG_USB_NET_CX82310_ETH=m
+CONFIG_USB_NET_KALMIA=m
+CONFIG_USB_NET_QMI_WWAN=m
+CONFIG_USB_HSO=m
+CONFIG_USB_NET_INT51X1=m
+CONFIG_USB_CDC_PHONET=m
+CONFIG_USB_IPHETH=m
+CONFIG_USB_SIERRA_NET=m
+CONFIG_USB_VL600=m
+CONFIG_USB_NET_CH9200=m
+CONFIG_ATH9K=m
+CONFIG_ATH9K_HTC=m
+CONFIG_ATH9K_HWRNG=y
+CONFIG_CARL9170=m
+CONFIG_ATH6KL=m
+CONFIG_ATH6KL_SDIO=m
+CONFIG_ATH6KL_USB=m
+CONFIG_AR5523=m
+CONFIG_ATH10K=m
+CONFIG_ATH10K_USB=m
+CONFIG_AT76C50X_USB=m
+CONFIG_BRCMFMAC=m
+CONFIG_HOSTAP=m
+CONFIG_HOSTAP_FIRMWARE=y
+CONFIG_P54_COMMON=m
+CONFIG_P54_USB=m
+CONFIG_LIBERTAS=m
+CONFIG_LIBERTAS_USB=m
+CONFIG_LIBERTAS_SDIO=m
+CONFIG_LIBERTAS_MESH=y
+CONFIG_LIBERTAS_THINFIRM=m
+CONFIG_LIBERTAS_THINFIRM_USB=m
+CONFIG_MT7601U=m
+CONFIG_RT2X00=m
+CONFIG_RT2500USB=m
+CONFIG_RT73USB=m
+CONFIG_RT2800USB=m
+CONFIG_RT2800USB_RT3573=y
+CONFIG_RT2800USB_RT53XX=y
+CONFIG_RT2800USB_RT55XX=y
+CONFIG_RTL8187=m
+CONFIG_RTL8192CU=m
+# CONFIG_RTLWIFI_DEBUG is not set
+CONFIG_RTL8XXXU=m
+CONFIG_RSI_91X=m
+# CONFIG_RSI_SDIO is not set
+CONFIG_WL1251=m
+CONFIG_WL1251_SPI=m
+CONFIG_WL1251_SDIO=m
+CONFIG_WL12XX=m
+CONFIG_WL18XX=m
+CONFIG_WLCORE_SPI=m
+CONFIG_WLCORE_SDIO=m
+CONFIG_ZD1211RW=m
+CONFIG_MAC80211_HWSIM=m
+CONFIG_USB_NET_RNDIS_WLAN=m
+CONFIG_WIMAX_I2400M_USB=m
+CONFIG_IEEE802154_FAKELB=m
+CONFIG_IEEE802154_AT86RF230=m
+CONFIG_IEEE802154_MRF24J40=m
+CONFIG_IEEE802154_CC2520=m
+CONFIG_IEEE802154_ATUSB=m
+CONFIG_IEEE802154_ADF7242=m
+CONFIG_INPUT_SPARSEKMAP=m
+CONFIG_INPUT_JOYDEV=m
+CONFIG_INPUT_EVDEV=y
+CONFIG_KEYBOARD_ADP5588=m
+# CONFIG_KEYBOARD_ATKBD is not set
+CONFIG_KEYBOARD_QT2160=m
+CONFIG_KEYBOARD_GPIO=y
+CONFIG_KEYBOARD_LM8323=m
+CONFIG_KEYBOARD_MAX7359=m
+CONFIG_KEYBOARD_OPENCORES=m
+CONFIG_KEYBOARD_STOWAWAY=m
+# CONFIG_MOUSE_PS2 is not set
+CONFIG_MOUSE_APPLETOUCH=m
+CONFIG_MOUSE_ELAN_I2C=m
+CONFIG_MOUSE_SYNAPTICS_I2C=m
+CONFIG_MOUSE_SYNAPTICS_USB=m
+CONFIG_INPUT_JOYSTICK=y
+CONFIG_JOYSTICK_IFORCE=m
+CONFIG_JOYSTICK_IFORCE_USB=y
+CONFIG_JOYSTICK_IFORCE_232=y
+CONFIG_JOYSTICK_WARRIOR=m
+CONFIG_JOYSTICK_MAGELLAN=m
+CONFIG_JOYSTICK_SPACEORB=m
+CONFIG_JOYSTICK_SPACEBALL=m
+CONFIG_JOYSTICK_STINGER=m
+CONFIG_JOYSTICK_TWIDJOY=m
+CONFIG_JOYSTICK_ZHENHUA=m
+CONFIG_JOYSTICK_AS5011=m
+CONFIG_JOYSTICK_XPAD=m
+CONFIG_JOYSTICK_XPAD_FF=y
+CONFIG_JOYSTICK_XPAD_LEDS=y
+CONFIG_JOYSTICK_PSXPAD_SPI=y
+CONFIG_JOYSTICK_PSXPAD_SPI_FF=y
+CONFIG_JOYSTICK_PXRC=m
+CONFIG_INPUT_TABLET=y
+CONFIG_TABLET_USB_ACECAD=m
+CONFIG_TABLET_USB_AIPTEK=m
+CONFIG_TABLET_USB_GTCO=m
+CONFIG_TABLET_USB_HANWANG=m
+CONFIG_TABLET_USB_KBTAB=m
+CONFIG_TABLET_USB_PEGASUS=m
+CONFIG_TABLET_SERIAL_WACOM4=m
+CONFIG_INPUT_TOUCHSCREEN=y
+CONFIG_TOUCHSCREEN_ADS7846=m
+CONFIG_TOUCHSCREEN_AD7877=m
+CONFIG_TOUCHSCREEN_AD7879=m
+CONFIG_TOUCHSCREEN_AD7879_I2C=m
+CONFIG_TOUCHSCREEN_AR1021_I2C=y
+CONFIG_TOUCHSCREEN_ATMEL_MXT=m
+CONFIG_TOUCHSCREEN_DYNAPRO=m
+CONFIG_TOUCHSCREEN_HAMPSHIRE=m
+CONFIG_TOUCHSCREEN_FUJITSU=m
+CONFIG_TOUCHSCREEN_GOODIX=m
+CONFIG_TOUCHSCREEN_GUNZE=m
+CONFIG_TOUCHSCREEN_ELO=m
+CONFIG_TOUCHSCREEN_WACOM_W8001=m
+CONFIG_TOUCHSCREEN_MCS5000=m
+CONFIG_TOUCHSCREEN_MTOUCH=m
+CONFIG_TOUCHSCREEN_INEXIO=m
+CONFIG_TOUCHSCREEN_MK712=m
+CONFIG_TOUCHSCREEN_PENMOUNT=m
+CONFIG_TOUCHSCREEN_EDT_FT5X06=y
+CONFIG_TOUCHSCREEN_TOUCHRIGHT=m
+CONFIG_TOUCHSCREEN_TOUCHWIN=m
+CONFIG_TOUCHSCREEN_TI_AM335X_TSC=y
+CONFIG_TOUCHSCREEN_USB_COMPOSITE=m
+CONFIG_TOUCHSCREEN_TSC2005=m
+CONFIG_TOUCHSCREEN_TSC2007=m
+CONFIG_TOUCHSCREEN_SILEAD=y
+CONFIG_TOUCHSCREEN_STMPE=y
+CONFIG_TOUCHSCREEN_TPS6507X=m
+CONFIG_INPUT_MISC=y
+CONFIG_INPUT_MMA8450=m
+CONFIG_INPUT_ATI_REMOTE2=m
+CONFIG_INPUT_KEYSPAN_REMOTE=m
+CONFIG_INPUT_POWERMATE=m
+CONFIG_INPUT_YEALINK=m
+CONFIG_INPUT_CM109=m
+CONFIG_INPUT_TPS65218_PWRBUTTON=y
+CONFIG_INPUT_UINPUT=y
+CONFIG_INPUT_PALMAS_PWRBUTTON=y
+# CONFIG_LEGACY_PTYS is not set
+CONFIG_N_GSM=m
+CONFIG_SERIAL_8250=y
+# CONFIG_SERIAL_8250_DEPRECATED_OPTIONS is not set
+CONFIG_SERIAL_8250_CONSOLE=y
+# CONFIG_SERIAL_8250_DMA is not set
+CONFIG_SERIAL_8250_NR_UARTS=6
+CONFIG_SERIAL_8250_RUNTIME_UARTS=6
+CONFIG_SERIAL_8250_OMAP=y
+CONFIG_SERIAL_OF_PLATFORM=y
+CONFIG_SERIAL_MAX3100=m
+CONFIG_SERIAL_MAX310X=m
+CONFIG_SERIAL_SC16IS7XX=m
+CONFIG_SERIAL_SC16IS7XX_SPI=y
+CONFIG_SERIAL_DEV_BUS=y
+# CONFIG_SERIAL_DEV_CTRL_TTYPORT is not set
+CONFIG_TTY_PRINTK=m
+CONFIG_VIRTIO_CONSOLE=m
+CONFIG_HW_RANDOM=y
+CONFIG_HW_RANDOM_VIRTIO=m
+CONFIG_TCG_TPM=y
+CONFIG_TCG_TIS_I2C_ATMEL=y
+CONFIG_I2C_CHARDEV=y
+CONFIG_I2C_ARB_GPIO_CHALLENGE=m
+CONFIG_I2C_MUX_GPIO=y
+CONFIG_I2C_MUX_PCA954x=m
+CONFIG_I2C_MUX_PINCTRL=y
+CONFIG_I2C_GPIO=y
+CONFIG_I2C_DIOLAN_U2C=m
+CONFIG_I2C_ROBOTFUZZ_OSIF=m
+CONFIG_I2C_TAOS_EVM=m
+CONFIG_I2C_TINY_USB=m
+CONFIG_SPI=y
+CONFIG_SPI_GPIO=m
+CONFIG_SPI_OMAP24XX=y
+CONFIG_SPI_TI_QSPI=y
+CONFIG_SPI_SPIDEV=m
+CONFIG_SPI_SLAVE=y
+CONFIG_SPI_SLAVE_TIME=m
+CONFIG_SPI_SLAVE_SYSTEM_CONTROL=m
+CONFIG_PPS_CLIENT_KTIMER=m
+CONFIG_PPS_CLIENT_LDISC=m
+CONFIG_PPS_CLIENT_GPIO=m
+CONFIG_PINCTRL_MCP23S08=m
+CONFIG_PINCTRL_SINGLE=y
+CONFIG_PINCTRL_PALMAS=y
+CONFIG_GPIO_SYSFS=y
+CONFIG_GPIO_OF_HELPER=y
+CONFIG_GPIO_GENERIC_PLATFORM=y
+CONFIG_GPIO_SYSCON=y
+CONFIG_GPIO_ADP5588=m
+CONFIG_GPIO_ADNP=m
+CONFIG_GPIO_MAX7300=m
+CONFIG_GPIO_MAX732X=m
+CONFIG_GPIO_PCA953X=y
+CONFIG_GPIO_PCA953X_IRQ=y
+CONFIG_GPIO_PCF857X=m
+CONFIG_GPIO_TPIC2810=m
+CONFIG_GPIO_PALMAS=y
+CONFIG_GPIO_STMPE=y
+CONFIG_GPIO_TPS65218=y
+CONFIG_GPIO_74X164=m
+CONFIG_GPIO_MAX3191X=m
+CONFIG_GPIO_MAX7301=m
+CONFIG_GPIO_MC33880=m
+CONFIG_GPIO_PISOSR=m
+CONFIG_GPIO_XRA1403=m
+CONFIG_W1=m
+CONFIG_W1_MASTER_DS2490=m
+CONFIG_W1_MASTER_DS2482=m
+CONFIG_W1_MASTER_GPIO=m
+CONFIG_W1_SLAVE_THERM=m
+CONFIG_W1_SLAVE_SMEM=m
+CONFIG_W1_SLAVE_DS2405=m
+CONFIG_W1_SLAVE_DS2408=m
+CONFIG_W1_SLAVE_DS2413=m
+CONFIG_W1_SLAVE_DS2406=m
+CONFIG_W1_SLAVE_DS2423=m
+CONFIG_W1_SLAVE_DS2805=m
+CONFIG_W1_SLAVE_DS2431=m
+CONFIG_W1_SLAVE_DS2433=m
+CONFIG_W1_SLAVE_DS2433_CRC=y
+CONFIG_W1_SLAVE_DS2438=m
+CONFIG_W1_SLAVE_DS2780=m
+CONFIG_W1_SLAVE_DS2781=m
+CONFIG_W1_SLAVE_DS28E04=m
+CONFIG_W1_SLAVE_DS28E17=m
+CONFIG_POWER_AVS=y
+CONFIG_POWER_RESET=y
+CONFIG_POWER_RESET_GPIO=y
+CONFIG_POWER_RESET_GPIO_RESTART=y
+CONFIG_POWER_RESET_RESTART=y
+CONFIG_POWER_RESET_SYSCON=y
+CONFIG_POWER_RESET_SYSCON_POWEROFF=y
+CONFIG_GENERIC_ADC_BATTERY=m
+CONFIG_CHARGER_GPIO=m
+CONFIG_SENSORS_AD7314=m
+CONFIG_SENSORS_AD7414=m
+CONFIG_SENSORS_AD7418=m
+CONFIG_SENSORS_ADM1021=m
+CONFIG_SENSORS_ADM1025=m
+CONFIG_SENSORS_ADM1026=m
+CONFIG_SENSORS_ADM1029=m
+CONFIG_SENSORS_ADM1031=m
+CONFIG_SENSORS_ADM9240=m
+CONFIG_SENSORS_ADT7310=m
+CONFIG_SENSORS_ADT7410=m
+CONFIG_SENSORS_ADT7411=m
+CONFIG_SENSORS_ADT7462=m
+CONFIG_SENSORS_ADT7470=m
+CONFIG_SENSORS_ADT7475=m
+CONFIG_SENSORS_ASC7621=m
+CONFIG_SENSORS_ASPEED=m
+CONFIG_SENSORS_ATXP1=m
+CONFIG_SENSORS_DS620=m
+CONFIG_SENSORS_DS1621=m
+CONFIG_SENSORS_F71805F=m
+CONFIG_SENSORS_F71882FG=m
+CONFIG_SENSORS_F75375S=m
+CONFIG_SENSORS_FTSTEUTATES=m
+CONFIG_SENSORS_GL518SM=m
+CONFIG_SENSORS_GL520SM=m
+CONFIG_SENSORS_G760A=m
+CONFIG_SENSORS_G762=m
+CONFIG_SENSORS_GPIO_FAN=y
+CONFIG_SENSORS_HIH6130=m
+CONFIG_SENSORS_IIO_HWMON=m
+CONFIG_SENSORS_IT87=m
+CONFIG_SENSORS_JC42=m
+CONFIG_SENSORS_POWR1220=m
+CONFIG_SENSORS_LINEAGE=m
+CONFIG_SENSORS_LTC2945=m
+CONFIG_SENSORS_LTC2990=m
+CONFIG_SENSORS_LTC4151=m
+CONFIG_SENSORS_LTC4215=m
+CONFIG_SENSORS_LTC4222=m
+CONFIG_SENSORS_LTC4245=m
+CONFIG_SENSORS_LTC4260=m
+CONFIG_SENSORS_LTC4261=m
+CONFIG_SENSORS_MAX1111=m
+CONFIG_SENSORS_MAX16065=m
+CONFIG_SENSORS_MAX1619=m
+CONFIG_SENSORS_MAX1668=m
+CONFIG_SENSORS_MAX197=m
+CONFIG_SENSORS_MAX31722=m
+CONFIG_SENSORS_MAX6621=m
+CONFIG_SENSORS_MAX6639=m
+CONFIG_SENSORS_MAX6642=m
+CONFIG_SENSORS_MAX6650=m
+CONFIG_SENSORS_MAX6697=m
+CONFIG_SENSORS_MAX31790=m
+CONFIG_SENSORS_MCP3021=m
+CONFIG_SENSORS_TC654=m
+CONFIG_SENSORS_ADCXX=m
+CONFIG_SENSORS_LM63=m
+CONFIG_SENSORS_LM70=m
+CONFIG_SENSORS_LM73=m
+CONFIG_SENSORS_LM75=m
+CONFIG_SENSORS_LM77=m
+CONFIG_SENSORS_LM78=m
+CONFIG_SENSORS_LM80=m
+CONFIG_SENSORS_LM83=m
+CONFIG_SENSORS_LM85=m
+CONFIG_SENSORS_LM87=m
+CONFIG_SENSORS_LM90=m
+CONFIG_SENSORS_LM92=m
+CONFIG_SENSORS_LM93=m
+CONFIG_SENSORS_LM95234=m
+CONFIG_SENSORS_LM95241=m
+CONFIG_SENSORS_LM95245=m
+CONFIG_SENSORS_PC87360=m
+CONFIG_SENSORS_PC87427=m
+CONFIG_SENSORS_NTC_THERMISTOR=m
+CONFIG_SENSORS_NCT6683=m
+CONFIG_SENSORS_NCT6775=m
+CONFIG_SENSORS_NCT7802=m
+CONFIG_SENSORS_NCT7904=m
+CONFIG_SENSORS_NPCM7XX=m
+CONFIG_SENSORS_PCF8591=m
+CONFIG_PMBUS=m
+CONFIG_SENSORS_ADM1275=m
+CONFIG_SENSORS_IBM_CFFPS=m
+CONFIG_SENSORS_IR35221=m
+CONFIG_SENSORS_LM25066=m
+CONFIG_SENSORS_LTC2978=m
+CONFIG_SENSORS_LTC2978_REGULATOR=y
+CONFIG_SENSORS_LTC3815=m
+CONFIG_SENSORS_MAX16064=m
+CONFIG_SENSORS_MAX20751=m
+CONFIG_SENSORS_MAX31785=m
+CONFIG_SENSORS_MAX34440=m
+CONFIG_SENSORS_MAX8688=m
+CONFIG_SENSORS_TPS40422=m
+CONFIG_SENSORS_TPS53679=m
+CONFIG_SENSORS_UCD9000=m
+CONFIG_SENSORS_UCD9200=m
+CONFIG_SENSORS_ZL6100=m
+CONFIG_SENSORS_PWM_FAN=m
+CONFIG_SENSORS_SHT15=m
+CONFIG_SENSORS_SHT21=m
+CONFIG_SENSORS_SHTC1=m
+CONFIG_SENSORS_DME1737=m
+CONFIG_SENSORS_EMC1403=m
+CONFIG_SENSORS_EMC2103=m
+CONFIG_SENSORS_EMC6W201=m
+CONFIG_SENSORS_SMSC47M1=m
+CONFIG_SENSORS_SMSC47M192=m
+CONFIG_SENSORS_SMSC47B397=m
+CONFIG_SENSORS_SCH5627=m
+CONFIG_SENSORS_SCH5636=m
+CONFIG_SENSORS_STTS751=m
+CONFIG_SENSORS_SMM665=m
+CONFIG_SENSORS_ADC128D818=m
+CONFIG_SENSORS_ADS1015=m
+CONFIG_SENSORS_ADS7828=m
+CONFIG_SENSORS_ADS7871=m
+CONFIG_SENSORS_AMC6821=m
+CONFIG_SENSORS_INA209=m
+CONFIG_SENSORS_INA2XX=m
+CONFIG_SENSORS_INA3221=m
+CONFIG_SENSORS_TC74=m
+CONFIG_SENSORS_THMC50=m
+CONFIG_SENSORS_TMP102=y
+CONFIG_SENSORS_TMP103=m
+CONFIG_SENSORS_TMP108=m
+CONFIG_SENSORS_TMP401=m
+CONFIG_SENSORS_TMP421=m
+CONFIG_SENSORS_VT1211=m
+CONFIG_SENSORS_W83773G=m
+CONFIG_SENSORS_W83781D=m
+CONFIG_SENSORS_W83791D=m
+CONFIG_SENSORS_W83792D=m
+CONFIG_SENSORS_W83793=m
+CONFIG_SENSORS_W83795=m
+CONFIG_SENSORS_W83L785TS=m
+CONFIG_SENSORS_W83L786NG=m
+CONFIG_SENSORS_W83627HF=m
+CONFIG_SENSORS_W83627EHF=m
+CONFIG_THERMAL_GOV_FAIR_SHARE=y
+CONFIG_THERMAL_GOV_BANG_BANG=y
+CONFIG_CPU_THERMAL=y
+CONFIG_CLOCK_THERMAL=y
+CONFIG_DEVFREQ_THERMAL=y
+CONFIG_TI_THERMAL=y
+CONFIG_OMAP5_THERMAL=y
+CONFIG_DRA752_THERMAL=y
+CONFIG_GENERIC_ADC_THERMAL=m
+CONFIG_WATCHDOG=y
+CONFIG_WATCHDOG_SYSFS=y
+CONFIG_SOFT_WATCHDOG=y
+CONFIG_OMAP_WATCHDOG=y
+CONFIG_MFD_STMPE=y
+CONFIG_MFD_TI_AM335X_TSCADC=y
+CONFIG_MFD_PALMAS=y
+CONFIG_MFD_TPS65217=y
+CONFIG_MFD_TPS65218=y
+CONFIG_MFD_WL1273_CORE=m
+CONFIG_REGULATOR_USERSPACE_CONSUMER=y
+CONFIG_REGULATOR_GPIO=y
+CONFIG_REGULATOR_PALMAS=y
+CONFIG_REGULATOR_PBIAS=y
+CONFIG_REGULATOR_PWM=y
+CONFIG_REGULATOR_TI_ABB=y
+CONFIG_REGULATOR_TPS65217=y
+CONFIG_REGULATOR_TPS65218=y
+CONFIG_RC_CORE=m
+CONFIG_LIRC=y
+CONFIG_RC_DECODERS=y
+CONFIG_IR_NEC_DECODER=m
+CONFIG_IR_RC5_DECODER=m
+CONFIG_IR_RC6_DECODER=m
+CONFIG_IR_JVC_DECODER=m
+CONFIG_IR_SONY_DECODER=m
+CONFIG_IR_SANYO_DECODER=m
+CONFIG_IR_SHARP_DECODER=m
+CONFIG_IR_MCE_KBD_DECODER=m
+CONFIG_IR_XMP_DECODER=m
+CONFIG_RC_DEVICES=y
+CONFIG_RC_ATI_REMOTE=m
+CONFIG_IR_IMON=m
+CONFIG_IR_MCEUSB=m
+CONFIG_IR_REDRAT3=m
+CONFIG_IR_STREAMZAP=m
+CONFIG_IR_IGORPLUGUSB=m
+CONFIG_IR_IGUANA=m
+CONFIG_IR_TTUSBIR=m
+CONFIG_RC_LOOPBACK=m
+CONFIG_IR_GPIO_CIR=m
+CONFIG_MEDIA_SUPPORT=m
+CONFIG_MEDIA_CAMERA_SUPPORT=y
+CONFIG_MEDIA_ANALOG_TV_SUPPORT=y
+CONFIG_MEDIA_DIGITAL_TV_SUPPORT=y
+CONFIG_MEDIA_RADIO_SUPPORT=y
+CONFIG_MEDIA_SDR_SUPPORT=y
+CONFIG_MEDIA_CEC_SUPPORT=y
+CONFIG_MEDIA_CONTROLLER=y
+CONFIG_VIDEO_V4L2_SUBDEV_API=y
+CONFIG_DVB_DYNAMIC_MINORS=y
+CONFIG_MEDIA_USB_SUPPORT=y
+CONFIG_USB_VIDEO_CLASS=m
+CONFIG_USB_M5602=m
+CONFIG_USB_STV06XX=m
+CONFIG_USB_GL860=m
+CONFIG_USB_GSPCA_BENQ=m
+CONFIG_USB_GSPCA_CONEX=m
+CONFIG_USB_GSPCA_CPIA1=m
+CONFIG_USB_GSPCA_DTCS033=m
+CONFIG_USB_GSPCA_ETOMS=m
+CONFIG_USB_GSPCA_FINEPIX=m
+CONFIG_USB_GSPCA_JEILINJ=m
+CONFIG_USB_GSPCA_JL2005BCD=m
+CONFIG_USB_GSPCA_KINECT=m
+CONFIG_USB_GSPCA_KONICA=m
+CONFIG_USB_GSPCA_MARS=m
+CONFIG_USB_GSPCA_MR97310A=m
+CONFIG_USB_GSPCA_NW80X=m
+CONFIG_USB_GSPCA_OV519=m
+CONFIG_USB_GSPCA_OV534=m
+CONFIG_USB_GSPCA_OV534_9=m
+CONFIG_USB_GSPCA_PAC207=m
+CONFIG_USB_GSPCA_PAC7302=m
+CONFIG_USB_GSPCA_PAC7311=m
+CONFIG_USB_GSPCA_SE401=m
+CONFIG_USB_GSPCA_SN9C2028=m
+CONFIG_USB_GSPCA_SN9C20X=m
+CONFIG_USB_GSPCA_SONIXB=m
+CONFIG_USB_GSPCA_SONIXJ=m
+CONFIG_USB_GSPCA_SPCA500=m
+CONFIG_USB_GSPCA_SPCA501=m
+CONFIG_USB_GSPCA_SPCA505=m
+CONFIG_USB_GSPCA_SPCA506=m
+CONFIG_USB_GSPCA_SPCA508=m
+CONFIG_USB_GSPCA_SPCA561=m
+CONFIG_USB_GSPCA_SPCA1528=m
+CONFIG_USB_GSPCA_SQ905=m
+CONFIG_USB_GSPCA_SQ905C=m
+CONFIG_USB_GSPCA_SQ930X=m
+CONFIG_USB_GSPCA_STK014=m
+CONFIG_USB_GSPCA_STK1135=m
+CONFIG_USB_GSPCA_STV0680=m
+CONFIG_USB_GSPCA_SUNPLUS=m
+CONFIG_USB_GSPCA_T613=m
+CONFIG_USB_GSPCA_TOPRO=m
+CONFIG_USB_GSPCA_TOUPTEK=m
+CONFIG_USB_GSPCA_TV8532=m
+CONFIG_USB_GSPCA_VC032X=m
+CONFIG_USB_GSPCA_VICAM=m
+CONFIG_USB_GSPCA_XIRLINK_CIT=m
+CONFIG_USB_GSPCA_ZC3XX=m
+CONFIG_USB_PWC=m
+CONFIG_VIDEO_CPIA2=m
+CONFIG_USB_ZR364XX=m
+CONFIG_USB_STKWEBCAM=m
+CONFIG_USB_S2255=m
+CONFIG_VIDEO_USBTV=m
+CONFIG_VIDEO_PVRUSB2=m
+CONFIG_VIDEO_HDPVR=m
+CONFIG_VIDEO_USBVISION=m
+CONFIG_VIDEO_STK1160_COMMON=m
+CONFIG_VIDEO_GO7007=m
+CONFIG_VIDEO_GO7007_USB=m
+CONFIG_VIDEO_GO7007_USB_S2250_BOARD=m
+CONFIG_VIDEO_AU0828=m
+CONFIG_VIDEO_AU0828_RC=y
+CONFIG_VIDEO_CX231XX=m
+CONFIG_VIDEO_CX231XX_ALSA=m
+CONFIG_VIDEO_CX231XX_DVB=m
+CONFIG_DVB_USB=m
+CONFIG_DVB_USB_A800=m
+CONFIG_DVB_USB_DIBUSB_MB=m
+CONFIG_DVB_USB_DIBUSB_MB_FAULTY=y
+CONFIG_DVB_USB_DIBUSB_MC=m
+CONFIG_DVB_USB_DIB0700=m
+CONFIG_DVB_USB_UMT_010=m
+CONFIG_DVB_USB_CXUSB=m
+CONFIG_DVB_USB_M920X=m
+CONFIG_DVB_USB_DIGITV=m
+CONFIG_DVB_USB_VP7045=m
+CONFIG_DVB_USB_VP702X=m
+CONFIG_DVB_USB_GP8PSK=m
+CONFIG_DVB_USB_NOVA_T_USB2=m
+CONFIG_DVB_USB_TTUSB2=m
+CONFIG_DVB_USB_DTT200U=m
+CONFIG_DVB_USB_OPERA1=m
+CONFIG_DVB_USB_AF9005=m
+CONFIG_DVB_USB_AF9005_REMOTE=m
+CONFIG_DVB_USB_PCTV452E=m
+CONFIG_DVB_USB_DW2102=m
+CONFIG_DVB_USB_CINERGY_T2=m
+CONFIG_DVB_USB_DTV5100=m
+CONFIG_DVB_USB_AZ6027=m
+CONFIG_DVB_USB_TECHNISAT_USB2=m
+CONFIG_DVB_USB_V2=m
+CONFIG_DVB_USB_AF9015=m
+CONFIG_DVB_USB_AF9035=m
+CONFIG_DVB_USB_ANYSEE=m
+CONFIG_DVB_USB_AU6610=m
+CONFIG_DVB_USB_AZ6007=m
+CONFIG_DVB_USB_CE6230=m
+CONFIG_DVB_USB_EC168=m
+CONFIG_DVB_USB_GL861=m
+CONFIG_DVB_USB_LME2510=m
+CONFIG_DVB_USB_MXL111SF=m
+CONFIG_DVB_USB_RTL28XXU=m
+CONFIG_DVB_USB_DVBSKY=m
+CONFIG_SMS_USB_DRV=m
+CONFIG_DVB_B2C2_FLEXCOP_USB=m
+CONFIG_DVB_AS102=m
+CONFIG_VIDEO_EM28XX=m
+CONFIG_VIDEO_EM28XX_V4L2=m
+CONFIG_VIDEO_EM28XX_ALSA=m
+CONFIG_VIDEO_EM28XX_DVB=m
+CONFIG_USB_AIRSPY=m
+CONFIG_USB_HACKRF=m
+CONFIG_USB_MSI2500=m
+CONFIG_USB_PULSE8_CEC=m
+CONFIG_USB_RAINSHADOW_CEC=m
+CONFIG_V4L_PLATFORM_DRIVERS=y
+CONFIG_VIDEO_TI_CAL=m
+CONFIG_VIDEO_TI_VIP=m
+CONFIG_V4L_MEM2MEM_DRIVERS=y
+CONFIG_VIDEO_TI_VPE=m
+CONFIG_SMS_SDIO_DRV=m
+CONFIG_RADIO_SI470X=m
+CONFIG_USB_SI470X=m
+CONFIG_RADIO_SI4713=m
+CONFIG_I2C_SI4713=m
+CONFIG_USB_MR800=m
+CONFIG_RADIO_SHARK=m
+CONFIG_RADIO_SHARK2=m
+CONFIG_USB_KEENE=m
+CONFIG_USB_RAREMONO=m
+CONFIG_USB_MA901=m
+CONFIG_RADIO_WL128X=m
+CONFIG_DRM=y
+CONFIG_DRM_DP_AUX_CHARDEV=y
+CONFIG_DRM_LOAD_EDID_FIRMWARE=y
+CONFIG_DRM_I2C_NXP_TDA998X=y
+CONFIG_DRM_VGEM=m
+CONFIG_DRM_UDL=m
+CONFIG_DRM_OMAP=y
+# CONFIG_OMAP2_DSS_VENC is not set
+CONFIG_OMAP5_DSS_HDMI=y
+CONFIG_DRM_OMAP_ENCODER_TPD12S015=y
+CONFIG_DRM_OMAP_DRA7EVM_ENCODER_TPD12S015=y
+CONFIG_DRM_OMAP_CONNECTOR_HDMI=y
+CONFIG_DRM_TILCDC=y
+CONFIG_DRM_PANEL_SIMPLE=y
+CONFIG_DRM_PANEL_OSD_OSD101T2587_53TS=y
+CONFIG_DRM_SII902X=y
+CONFIG_DRM_TI_TFP410=y
+CONFIG_DRM_I2C_ADV7511=y
+CONFIG_DRM_I2C_ADV7511_AUDIO=y
+CONFIG_DRM_ETNAVIV=y
+CONFIG_DRM_TINYDRM=m
+CONFIG_TINYDRM_ILI9225=m
+CONFIG_TINYDRM_ILI9341=m
+CONFIG_TINYDRM_MI0283QT=m
+CONFIG_TINYDRM_REPAPER=m
+CONFIG_TINYDRM_ST7586=m
+CONFIG_TINYDRM_ST7735R=m
+CONFIG_DRM_LEGACY=y
+CONFIG_FIRMWARE_EDID=y
+CONFIG_FB_TILEBLITTING=y
+CONFIG_FB_SMSCUFX=m
+CONFIG_FB_UDL=m
+CONFIG_FB_SIMPLE=y
+CONFIG_FB_SSD1307=y
+# CONFIG_LCD_CLASS_DEVICE is not set
+CONFIG_BACKLIGHT_PWM=y
+CONFIG_BACKLIGHT_GPIO=y
+CONFIG_BACKLIGHT_LED=y
+CONFIG_FRAMEBUFFER_CONSOLE=y
+CONFIG_FRAMEBUFFER_CONSOLE_ROTATION=y
+CONFIG_LOGO=y
+CONFIG_SOUND=y
+CONFIG_SND=y
+CONFIG_SND_OSSEMUL=y
+CONFIG_SND_MIXER_OSS=m
+CONFIG_SND_PCM_OSS=m
+CONFIG_SND_HRTIMER=m
+CONFIG_SND_DYNAMIC_MINORS=y
+CONFIG_SND_SEQUENCER=m
+CONFIG_SND_SEQ_DUMMY=m
+CONFIG_SND_DUMMY=m
+CONFIG_SND_ALOOP=m
+CONFIG_SND_VIRMIDI=m
+CONFIG_SND_MTPAV=m
+CONFIG_SND_SERIAL_U16550=m
+CONFIG_SND_MPU401=m
+CONFIG_SND_HDA_PREALLOC_SIZE=2048
+CONFIG_SND_USB_AUDIO=m
+CONFIG_SND_USB_UA101=m
+CONFIG_SND_USB_CAIAQ=m
+CONFIG_SND_USB_CAIAQ_INPUT=y
+CONFIG_SND_USB_6FIRE=m
+CONFIG_SND_USB_HIFACE=m
+CONFIG_SND_BCD2000=m
+CONFIG_SND_USB_POD=m
+CONFIG_SND_USB_PODHD=m
+CONFIG_SND_USB_TONEPORT=m
+CONFIG_SND_USB_VARIAX=m
+CONFIG_SND_SOC=y
+CONFIG_SND_SOC_DAVINCI_MCASP=y
+CONFIG_SND_SOC_OMAP_DMIC=y
+CONFIG_SND_SOC_OMAP_MCBSP=y
+CONFIG_SND_SOC_OMAP_MCPDM=y
+CONFIG_SND_SOC_OMAP_HDMI=y
+CONFIG_SND_SOC_ADAU1701=m
+CONFIG_SND_SOC_ADAU7002=m
+CONFIG_SND_SOC_AK4554=m
+CONFIG_SND_SOC_CS4265=m
+CONFIG_SND_SOC_CS4271_I2C=m
+CONFIG_SND_SOC_PCM3168A_I2C=m
+CONFIG_SND_SOC_PCM512x_I2C=m
+CONFIG_SND_SOC_SPDIF=m
+CONFIG_SND_SOC_TLV320AIC23_I2C=y
+CONFIG_SND_SOC_TLV320AIC31XX=y
+CONFIG_SND_SOC_TLV320AIC3X=y
+CONFIG_SND_SOC_TS3A227E=m
+CONFIG_SND_SOC_WM8753=m
+CONFIG_SND_SOC_WM8804_I2C=m
+CONFIG_SND_SOC_WM8903=m
+CONFIG_SND_SOC_TPA6130A2=m
+CONFIG_SND_SIMPLE_CARD=y
+CONFIG_SND_SIMPLE_SCU_CARD=m
+CONFIG_SND_AUDIO_GRAPH_CARD=m
+CONFIG_SND_AUDIO_GRAPH_SCU_CARD=m
+CONFIG_HID_BATTERY_STRENGTH=y
+CONFIG_HIDRAW=y
+CONFIG_UHID=y
+CONFIG_HID_A4TECH=m
+CONFIG_HID_ACRUX=m
+CONFIG_HID_ACRUX_FF=y
+CONFIG_HID_APPLE=m
+CONFIG_HID_APPLEIR=m
+CONFIG_HID_ASUS=m
+CONFIG_HID_AUREAL=m
+CONFIG_HID_BELKIN=m
+CONFIG_HID_BETOP_FF=m
+CONFIG_HID_CHERRY=m
+CONFIG_HID_CHICONY=m
+CONFIG_HID_CORSAIR=m
+CONFIG_HID_PRODIKEYS=m
+CONFIG_HID_CMEDIA=m
+CONFIG_HID_CP2112=m
+CONFIG_HID_CYPRESS=m
+CONFIG_HID_DRAGONRISE=m
+CONFIG_DRAGONRISE_FF=y
+CONFIG_HID_EMS_FF=m
+CONFIG_HID_ELECOM=m
+CONFIG_HID_ELO=m
+CONFIG_HID_EZKEY=m
+CONFIG_HID_GEMBIRD=m
+CONFIG_HID_GFRM=m
+CONFIG_HID_HOLTEK=m
+CONFIG_HOLTEK_FF=y
+CONFIG_HID_GT683R=m
+CONFIG_HID_KEYTOUCH=m
+CONFIG_HID_KYE=m
+CONFIG_HID_UCLOGIC=m
+CONFIG_HID_WALTOP=m
+CONFIG_HID_GYRATION=m
+CONFIG_HID_ICADE=m
+CONFIG_HID_TWINHAN=m
+CONFIG_HID_KENSINGTON=m
+CONFIG_HID_LCPOWER=m
+CONFIG_HID_LENOVO=m
+CONFIG_HID_LOGITECH=y
+CONFIG_HID_LOGITECH_DJ=y
+CONFIG_LOGITECH_FF=y
+CONFIG_LOGIRUMBLEPAD2_FF=y
+CONFIG_LOGIG940_FF=y
+CONFIG_HID_MAGICMOUSE=m
+CONFIG_HID_MAYFLASH=m
+CONFIG_HID_MICROSOFT=m
+CONFIG_HID_MONTEREY=m
+CONFIG_HID_MULTITOUCH=m
+CONFIG_HID_NTI=m
+CONFIG_HID_NTRIG=m
+CONFIG_HID_ORTEK=m
+CONFIG_HID_PANTHERLORD=m
+CONFIG_PANTHERLORD_FF=y
+CONFIG_HID_PENMOUNT=m
+CONFIG_HID_PETALYNX=m
+CONFIG_HID_PICOLCD=m
+CONFIG_HID_PICOLCD_FB=y
+CONFIG_HID_PICOLCD_BACKLIGHT=y
+CONFIG_HID_PICOLCD_LEDS=y
+CONFIG_HID_PICOLCD_CIR=y
+CONFIG_HID_PLANTRONICS=m
+CONFIG_HID_PRIMAX=m
+CONFIG_HID_RETRODE=m
+CONFIG_HID_ROCCAT=m
+CONFIG_HID_SAITEK=m
+CONFIG_HID_SAMSUNG=m
+CONFIG_HID_SONY=m
+CONFIG_SONY_FF=y
+CONFIG_HID_SPEEDLINK=m
+CONFIG_HID_STEAM=m
+CONFIG_HID_STEELSERIES=m
+CONFIG_HID_SUNPLUS=m
+CONFIG_HID_RMI=m
+CONFIG_HID_GREENASIA=m
+CONFIG_GREENASIA_FF=y
+CONFIG_HID_SMARTJOYPLUS=m
+CONFIG_SMARTJOYPLUS_FF=y
+CONFIG_HID_TIVO=m
+CONFIG_HID_TOPSEED=m
+CONFIG_HID_THINGM=m
+CONFIG_HID_THRUSTMASTER=m
+CONFIG_THRUSTMASTER_FF=y
+CONFIG_HID_UDRAW_PS3=m
+CONFIG_HID_WACOM=m
+CONFIG_HID_WIIMOTE=m
+CONFIG_HID_XINMO=m
+CONFIG_HID_ZEROPLUS=m
+CONFIG_ZEROPLUS_FF=y
+CONFIG_HID_ZYDACRON=m
+CONFIG_HID_SENSOR_CUSTOM_SENSOR=m
+CONFIG_HID_ALPS=m
+CONFIG_USB_HID=m
+CONFIG_HID_PID=y
+CONFIG_USB_HIDDEV=y
+CONFIG_I2C_HID=m
+CONFIG_USB=y
+CONFIG_USB_ANNOUNCE_NEW_DEVICES=y
+CONFIG_USB_DYNAMIC_MINORS=y
+CONFIG_USB_OTG=y
+CONFIG_USB_LEDS_TRIGGER_USBPORT=m
+CONFIG_USB_MON=m
+CONFIG_USB_WUSB_CBAF=m
+CONFIG_USB_XHCI_HCD=y
+CONFIG_USB_EHCI_HCD=y
+CONFIG_USB_EHCI_ROOT_HUB_TT=y
+CONFIG_USB_EHCI_HCD_PLATFORM=y
+CONFIG_USB_PRINTER=m
+CONFIG_USB_TMC=m
+CONFIG_USB_STORAGE=y
+CONFIG_USB_STORAGE_REALTEK=m
+CONFIG_USB_STORAGE_DATAFAB=m
+CONFIG_USB_STORAGE_FREECOM=m
+CONFIG_USB_STORAGE_ISD200=m
+CONFIG_USB_STORAGE_USBAT=m
+CONFIG_USB_STORAGE_SDDR09=m
+CONFIG_USB_STORAGE_SDDR55=m
+CONFIG_USB_STORAGE_JUMPSHOT=m
+CONFIG_USB_STORAGE_ALAUDA=m
+CONFIG_USB_STORAGE_ONETOUCH=m
+CONFIG_USB_STORAGE_KARMA=m
+CONFIG_USB_STORAGE_CYPRESS_ATACB=m
+CONFIG_USB_STORAGE_ENE_UB6250=m
+CONFIG_USB_UAS=m
+CONFIG_USB_MDC800=m
+CONFIG_USB_MICROTEK=m
+CONFIG_USBIP_CORE=m
+CONFIG_USBIP_VHCI_HCD=m
+CONFIG_USBIP_VHCI_HC_PORTS=15
+CONFIG_USBIP_VHCI_NR_HCS=8
+CONFIG_USBIP_HOST=m
+CONFIG_USBIP_VUDC=m
+CONFIG_USB_MUSB_HDRC=y
+CONFIG_USB_MUSB_DSPS=y
+CONFIG_MUSB_PIO_ONLY=y
+CONFIG_USB_DWC3=y
+CONFIG_USB_SERIAL=m
+CONFIG_USB_SERIAL_GENERIC=y
+CONFIG_USB_SERIAL_SIMPLE=m
+CONFIG_USB_SERIAL_AIRCABLE=m
+CONFIG_USB_SERIAL_ARK3116=m
+CONFIG_USB_SERIAL_BELKIN=m
+CONFIG_USB_SERIAL_CH341=m
+CONFIG_USB_SERIAL_WHITEHEAT=m
+CONFIG_USB_SERIAL_DIGI_ACCELEPORT=m
+CONFIG_USB_SERIAL_CP210X=m
+CONFIG_USB_SERIAL_CYPRESS_M8=m
+CONFIG_USB_SERIAL_EMPEG=m
+CONFIG_USB_SERIAL_FTDI_SIO=m
+CONFIG_USB_SERIAL_VISOR=m
+CONFIG_USB_SERIAL_IPAQ=m
+CONFIG_USB_SERIAL_IR=m
+CONFIG_USB_SERIAL_EDGEPORT=m
+CONFIG_USB_SERIAL_EDGEPORT_TI=m
+CONFIG_USB_SERIAL_F81232=m
+CONFIG_USB_SERIAL_GARMIN=m
+CONFIG_USB_SERIAL_IPW=m
+CONFIG_USB_SERIAL_IUU=m
+CONFIG_USB_SERIAL_KEYSPAN_PDA=m
+CONFIG_USB_SERIAL_KEYSPAN=m
+CONFIG_USB_SERIAL_KLSI=m
+CONFIG_USB_SERIAL_KOBIL_SCT=m
+CONFIG_USB_SERIAL_MCT_U232=m
+CONFIG_USB_SERIAL_METRO=m
+CONFIG_USB_SERIAL_MOS7720=m
+CONFIG_USB_SERIAL_MOS7840=m
+CONFIG_USB_SERIAL_MXUPORT=m
+CONFIG_USB_SERIAL_NAVMAN=m
+CONFIG_USB_SERIAL_PL2303=m
+CONFIG_USB_SERIAL_OTI6858=m
+CONFIG_USB_SERIAL_QCAUX=m
+CONFIG_USB_SERIAL_QUALCOMM=m
+CONFIG_USB_SERIAL_SPCP8X5=m
+CONFIG_USB_SERIAL_SAFE=m
+CONFIG_USB_SERIAL_SIERRAWIRELESS=m
+CONFIG_USB_SERIAL_SYMBOL=m
+CONFIG_USB_SERIAL_TI=m
+CONFIG_USB_SERIAL_CYBERJACK=m
+CONFIG_USB_SERIAL_XIRCOM=m
+CONFIG_USB_SERIAL_OPTION=m
+CONFIG_USB_SERIAL_OMNINET=m
+CONFIG_USB_SERIAL_OPTICON=m
+CONFIG_USB_SERIAL_XSENS_MT=m
+CONFIG_USB_SERIAL_WISHBONE=m
+CONFIG_USB_SERIAL_SSU100=m
+CONFIG_USB_SERIAL_QT2=m
+CONFIG_USB_SERIAL_DEBUG=m
+CONFIG_USB_EMI62=m
+CONFIG_USB_EMI26=m
+CONFIG_USB_ADUTUX=m
+CONFIG_USB_SEVSEG=m
+CONFIG_USB_LEGOTOWER=m
+CONFIG_USB_LCD=m
+CONFIG_USB_CYPRESS_CY7C63=m
+CONFIG_USB_CYTHERM=m
+CONFIG_USB_IDMOUSE=m
+CONFIG_USB_FTDI_ELAN=m
+CONFIG_USB_APPLEDISPLAY=m
+CONFIG_USB_SISUSBVGA=m
+CONFIG_USB_SISUSBVGA_CON=y
+CONFIG_USB_LD=m
+CONFIG_USB_TRANCEVIBRATOR=m
+CONFIG_USB_IOWARRIOR=m
+CONFIG_USB_TEST=m
+CONFIG_USB_EHSET_TEST_FIXTURE=m
+CONFIG_USB_ISIGHTFW=m
+CONFIG_USB_YUREX=m
+CONFIG_USB_HSIC_USB3503=m
+CONFIG_USB_CHAOSKEY=m
+CONFIG_NOP_USB_XCEIV=y
+CONFIG_AM335X_PHY_USB=y
+CONFIG_USB_GPIO_VBUS=y
+CONFIG_USB_GADGET=y
+CONFIG_USB_GADGET_VBUS_DRAW=500
+CONFIG_USB_DUMMY_HCD=m
+CONFIG_USB_CONFIGFS=m
+CONFIG_USB_CONFIGFS_SERIAL=y
+CONFIG_USB_CONFIGFS_ACM=y
+CONFIG_USB_CONFIGFS_OBEX=y
+CONFIG_USB_CONFIGFS_NCM=y
+CONFIG_USB_CONFIGFS_ECM=y
+CONFIG_USB_CONFIGFS_ECM_SUBSET=y
+CONFIG_USB_CONFIGFS_RNDIS=y
+CONFIG_USB_CONFIGFS_EEM=y
+CONFIG_USB_CONFIGFS_PHONET=y
+CONFIG_USB_CONFIGFS_MASS_STORAGE=y
+CONFIG_USB_CONFIGFS_F_LB_SS=y
+CONFIG_USB_CONFIGFS_F_FS=y
+CONFIG_USB_CONFIGFS_F_UAC1=y
+CONFIG_USB_CONFIGFS_F_UAC2=y
+CONFIG_USB_CONFIGFS_F_MIDI=y
+CONFIG_USB_CONFIGFS_F_HID=y
+CONFIG_USB_CONFIGFS_F_UVC=y
+CONFIG_USB_CONFIGFS_F_PRINTER=y
+CONFIG_USB_ZERO=m
+CONFIG_USB_AUDIO=m
+CONFIG_USB_ETH=m
+CONFIG_USB_G_NCM=m
+CONFIG_USB_GADGETFS=m
+CONFIG_USB_FUNCTIONFS=m
+CONFIG_USB_FUNCTIONFS_ETH=y
+CONFIG_USB_FUNCTIONFS_RNDIS=y
+CONFIG_USB_FUNCTIONFS_GENERIC=y
+CONFIG_USB_MASS_STORAGE=m
+CONFIG_USB_G_SERIAL=m
+CONFIG_USB_MIDI_GADGET=m
+CONFIG_USB_G_PRINTER=m
+CONFIG_USB_CDC_COMPOSITE=m
+CONFIG_USB_G_NOKIA=m
+CONFIG_USB_G_ACM_MS=m
+CONFIG_USB_G_MULTI=m
+CONFIG_USB_G_HID=m
+CONFIG_USB_G_DBGP=m
+CONFIG_TYPEC=y
+CONFIG_TYPEC_TCPM=y
+CONFIG_TYPEC_UCSI=y
+CONFIG_USB_LED_TRIG=y
+CONFIG_MMC=y
+CONFIG_SDIO_UART=m
+CONFIG_MMC_SDHCI=y
+CONFIG_MMC_SDHCI_PLTFM=y
+CONFIG_MMC_OMAP_HS=y
+CONFIG_MMC_SPI=y
+CONFIG_MMC_VUB300=m
+CONFIG_MMC_USHC=m
+CONFIG_MMC_SDHCI_OMAP=y
+CONFIG_LEDS_CLASS=y
+CONFIG_LEDS_BRIGHTNESS_HW_CHANGED=y
+CONFIG_LEDS_LM3530=m
+CONFIG_LEDS_LM3642=m
+CONFIG_LEDS_PCA9532=m
+CONFIG_LEDS_PCA9532_GPIO=y
+CONFIG_LEDS_GPIO=y
+CONFIG_LEDS_LP3944=m
+CONFIG_LEDS_LP5521=m
+CONFIG_LEDS_LP5523=m
+CONFIG_LEDS_LP5562=m
+CONFIG_LEDS_LP8501=m
+CONFIG_LEDS_LP8860=m
+CONFIG_LEDS_PCA955X=m
+CONFIG_LEDS_PCA963X=m
+CONFIG_LEDS_DAC124S085=m
+CONFIG_LEDS_PWM=m
+CONFIG_LEDS_REGULATOR=m
+CONFIG_LEDS_BD2802=m
+CONFIG_LEDS_LT3593=m
+CONFIG_LEDS_TCA6507=m
+CONFIG_LEDS_TLC591XX=m
+CONFIG_LEDS_LM355x=m
+CONFIG_LEDS_IS31FL319X=m
+CONFIG_LEDS_IS31FL32XX=m
+CONFIG_LEDS_TRIGGER_TIMER=y
+CONFIG_LEDS_TRIGGER_ONESHOT=y
+CONFIG_LEDS_TRIGGER_DISK=y
+CONFIG_LEDS_TRIGGER_MTD=y
+CONFIG_LEDS_TRIGGER_HEARTBEAT=y
+CONFIG_LEDS_TRIGGER_BACKLIGHT=y
+CONFIG_LEDS_TRIGGER_CPU=y
+CONFIG_LEDS_TRIGGER_ACTIVITY=y
+CONFIG_LEDS_TRIGGER_GPIO=y
+CONFIG_LEDS_TRIGGER_DEFAULT_ON=y
+CONFIG_LEDS_TRIGGER_TRANSIENT=m
+CONFIG_LEDS_TRIGGER_CAMERA=m
+CONFIG_LEDS_TRIGGER_PANIC=y
+CONFIG_LEDS_TRIGGER_NETDEV=y
+CONFIG_ACCESSIBILITY=y
+CONFIG_A11Y_BRAILLE_CONSOLE=y
+CONFIG_RTC_CLASS=y
+CONFIG_RTC_DRV_ABB5ZES3=y
+CONFIG_RTC_DRV_ABX80X=y
+CONFIG_RTC_DRV_DS1307=y
+CONFIG_RTC_DRV_DS1374=y
+CONFIG_RTC_DRV_DS1374_WDT=y
+CONFIG_RTC_DRV_DS1672=y
+CONFIG_RTC_DRV_HYM8563=y
+CONFIG_RTC_DRV_MAX6900=y
+CONFIG_RTC_DRV_RS5C372=y
+CONFIG_RTC_DRV_ISL1208=y
+CONFIG_RTC_DRV_ISL12022=y
+CONFIG_RTC_DRV_ISL12026=y
+CONFIG_RTC_DRV_X1205=y
+CONFIG_RTC_DRV_PCF8523=y
+CONFIG_RTC_DRV_PCF85063=y
+CONFIG_RTC_DRV_PCF85363=y
+CONFIG_RTC_DRV_PCF8563=y
+CONFIG_RTC_DRV_PCF8583=y
+CONFIG_RTC_DRV_M41T80=y
+CONFIG_RTC_DRV_M41T80_WDT=y
+CONFIG_RTC_DRV_BQ32K=y
+CONFIG_RTC_DRV_PALMAS=y
+CONFIG_RTC_DRV_S35390A=y
+CONFIG_RTC_DRV_FM3130=y
+CONFIG_RTC_DRV_RX8010=y
+CONFIG_RTC_DRV_RX8581=y
+CONFIG_RTC_DRV_RX8025=y
+CONFIG_RTC_DRV_EM3027=y
+CONFIG_RTC_DRV_RV8803=y
+CONFIG_RTC_DRV_M41T93=y
+CONFIG_RTC_DRV_M41T94=y
+CONFIG_RTC_DRV_DS1302=y
+CONFIG_RTC_DRV_DS1305=y
+CONFIG_RTC_DRV_DS1343=y
+CONFIG_RTC_DRV_DS1347=y
+CONFIG_RTC_DRV_DS1390=y
+CONFIG_RTC_DRV_MAX6916=y
+CONFIG_RTC_DRV_R9701=y
+CONFIG_RTC_DRV_RX4581=y
+CONFIG_RTC_DRV_RX6110=y
+CONFIG_RTC_DRV_RS5C348=y
+CONFIG_RTC_DRV_MAX6902=y
+CONFIG_RTC_DRV_PCF2123=y
+CONFIG_RTC_DRV_MCP795=y
+CONFIG_RTC_DRV_DS3232=y
+CONFIG_RTC_DRV_PCF2127=y
+CONFIG_RTC_DRV_RV3029C2=y
+CONFIG_RTC_DRV_CMOS=m
+CONFIG_RTC_DRV_DS1286=m
+CONFIG_RTC_DRV_DS1511=m
+CONFIG_RTC_DRV_DS1553=m
+CONFIG_RTC_DRV_DS1685_FAMILY=m
+CONFIG_RTC_DRV_DS1742=m
+CONFIG_RTC_DRV_DS2404=m
+CONFIG_RTC_DRV_STK17TA8=m
+CONFIG_RTC_DRV_M48T86=m
+CONFIG_RTC_DRV_M48T35=m
+CONFIG_RTC_DRV_M48T59=m
+CONFIG_RTC_DRV_MSM6242=m
+CONFIG_RTC_DRV_BQ4802=m
+CONFIG_RTC_DRV_RP5C01=m
+CONFIG_RTC_DRV_V3020=m
+CONFIG_RTC_DRV_OMAP=y
+CONFIG_RTC_DRV_HID_SENSOR_TIME=m
+CONFIG_DMADEVICES=y
+CONFIG_TI_CPPI41=y
+CONFIG_ASYNC_TX_DMA=y
+CONFIG_AUXDISPLAY=y
+CONFIG_HD44780=m
+CONFIG_IMG_ASCII_LCD=m
+CONFIG_HT16K33=m
+CONFIG_UIO=m
+CONFIG_UIO_PDRV_GENIRQ=m
+CONFIG_UIO_DMEM_GENIRQ=m
+CONFIG_UIO_PRUSS=m
+CONFIG_UIO_PRUSS_SHMEM=m
+CONFIG_VIRT_DRIVERS=y
+CONFIG_VIRTIO_BALLOON=m
+CONFIG_VIRTIO_INPUT=m
+CONFIG_VIRTIO_MMIO=m
+CONFIG_STAGING=y
+CONFIG_RTLLIB=m
+CONFIG_RTL8723BS=m
+CONFIG_R8712U=m
+CONFIG_R8188EU=m
+CONFIG_ADIS16203=m
+CONFIG_ADIS16240=m
+CONFIG_AD7606=m
+CONFIG_AD7606_IFACE_SPI=m
+CONFIG_AD7780=m
+CONFIG_AD7816=m
+CONFIG_AD7192=m
+CONFIG_AD7280=m
+CONFIG_ADT7316=m
+CONFIG_ADT7316_I2C=m
+CONFIG_AD7150=m
+CONFIG_AD7152=m
+CONFIG_AD7746=m
+CONFIG_AD9832=m
+CONFIG_AD9834=m
+CONFIG_AD5933=m
+CONFIG_ADE7854=m
+CONFIG_AD2S90=m
+CONFIG_AD2S1210=m
+CONFIG_ION=y
+CONFIG_ION_SYSTEM_HEAP=y
+CONFIG_ION_CARVEOUT_HEAP=y
+CONFIG_ION_CHUNK_HEAP=y
+CONFIG_ION_TI=y
+CONFIG_STAGING_BOARD=y
+CONFIG_FB_TFT=m
+CONFIG_FB_TFT_AGM1264K_FL=m
+CONFIG_FB_TFT_BD663474=m
+CONFIG_FB_TFT_HX8340BN=m
+CONFIG_FB_TFT_HX8347D=m
+CONFIG_FB_TFT_HX8353D=m
+CONFIG_FB_TFT_HX8357D=m
+CONFIG_FB_TFT_ILI9163=m
+CONFIG_FB_TFT_ILI9320=m
+CONFIG_FB_TFT_ILI9325=m
+CONFIG_FB_TFT_ILI9340=m
+CONFIG_FB_TFT_ILI9341=m
+CONFIG_FB_TFT_ILI9481=m
+CONFIG_FB_TFT_ILI9486=m
+CONFIG_FB_TFT_PCD8544=m
+CONFIG_FB_TFT_RA8875=m
+CONFIG_FB_TFT_S6D02A1=m
+CONFIG_FB_TFT_S6D1121=m
+CONFIG_FB_TFT_SH1106=m
+CONFIG_FB_TFT_SSD1289=m
+CONFIG_FB_TFT_SSD1305=m
+CONFIG_FB_TFT_SSD1306=m
+CONFIG_FB_TFT_SSD1331=m
+CONFIG_FB_TFT_SSD1351=m
+CONFIG_FB_TFT_ST7735R=m
+CONFIG_FB_TFT_ST7789V=m
+CONFIG_FB_TFT_TINYLCD=m
+CONFIG_FB_TFT_TLS8204=m
+CONFIG_FB_TFT_UC1611=m
+CONFIG_FB_TFT_UC1701=m
+CONFIG_FB_TFT_UPD161704=m
+CONFIG_FB_TFT_WATTEROTT=m
+CONFIG_FB_FLEX=m
+CONFIG_FB_TFT_FBTFT_DEVICE=m
+CONFIG_GREYBUS=m
+CONFIG_GREYBUS_ES2=m
+CONFIG_GREYBUS_AUDIO=m
+CONFIG_GREYBUS_BOOTROM=m
+CONFIG_GREYBUS_FIRMWARE=m
+CONFIG_GREYBUS_HID=m
+CONFIG_GREYBUS_LIGHT=m
+CONFIG_GREYBUS_LOG=m
+CONFIG_GREYBUS_LOOPBACK=m
+CONFIG_GREYBUS_POWER=m
+CONFIG_GREYBUS_RAW=m
+CONFIG_GREYBUS_VIBRATOR=m
+CONFIG_GREYBUS_BRIDGED_PHY=m
+CONFIG_GREYBUS_GPIO=m
+CONFIG_GREYBUS_I2C=m
+CONFIG_GREYBUS_PWM=m
+CONFIG_GREYBUS_SDIO=m
+CONFIG_GREYBUS_SPI=m
+CONFIG_GREYBUS_UART=m
+CONFIG_GREYBUS_USB=m
+CONFIG_COMMON_CLK_PALMAS=y
+CONFIG_COMMON_CLK_TI_ADPLL=y
+CONFIG_HWSPINLOCK=y
+CONFIG_HWSPINLOCK_OMAP=y
+CONFIG_OMAP2PLUS_MBOX=y
+CONFIG_OMAP_IOMMU=y
+CONFIG_REMOTEPROC=m
+CONFIG_OMAP_REMOTEPROC=m
+CONFIG_OMAP_REMOTEPROC_WATCHDOG=y
+CONFIG_WKUP_M3_RPROC=m
+CONFIG_RPMSG_VIRTIO=m
+CONFIG_RPMSG_RPC=m
+CONFIG_RPMSG_PRU=m
+CONFIG_RPMSG_KDRV_DISPLAY=y
+CONFIG_SOC_TI=y
+CONFIG_AMX3_PM=m
+CONFIG_WKUP_M3_IPC=m
+CONFIG_TI_PRUSS=m
+CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND=y
+CONFIG_DEVFREQ_GOV_PERFORMANCE=y
+CONFIG_DEVFREQ_GOV_POWERSAVE=y
+CONFIG_DEVFREQ_GOV_USERSPACE=y
+CONFIG_DEVFREQ_GOV_PASSIVE=y
+CONFIG_PM_DEVFREQ_EVENT=y
+CONFIG_EXTCON_GPIO=y
+CONFIG_EXTCON_PALMAS=y
+CONFIG_EXTCON_USB_GPIO=y
+CONFIG_TI_EMIF=y
+CONFIG_TI_EMIF_SRAM=y
+CONFIG_IIO=y
+CONFIG_IIO_SW_DEVICE=m
+CONFIG_IIO_SW_TRIGGER=m
+CONFIG_ADIS16201=m
+CONFIG_ADIS16209=m
+CONFIG_ADXL345_I2C=m
+CONFIG_ADXL345_SPI=m
+CONFIG_BMA180=m
+CONFIG_BMA220=m
+CONFIG_BMC150_ACCEL=m
+CONFIG_DA280=m
+CONFIG_DA311=m
+CONFIG_DMARD06=m
+CONFIG_DMARD09=m
+CONFIG_DMARD10=m
+CONFIG_HID_SENSOR_ACCEL_3D=m
+CONFIG_IIO_ST_ACCEL_3AXIS=m
+CONFIG_KXSD9=m
+CONFIG_KXCJK1013=m
+CONFIG_MC3230=m
+CONFIG_MMA7455_I2C=m
+CONFIG_MMA7455_SPI=m
+CONFIG_MMA7660=m
+CONFIG_MMA8452=m
+CONFIG_MMA9551=m
+CONFIG_MMA9553=m
+CONFIG_MXC4005=m
+CONFIG_MXC6255=m
+CONFIG_SCA3000=m
+CONFIG_STK8312=m
+CONFIG_STK8BA50=m
+CONFIG_AD7266=m
+CONFIG_AD7291=m
+CONFIG_AD7298=m
+CONFIG_AD7476=m
+CONFIG_AD7766=m
+CONFIG_AD7791=m
+CONFIG_AD7793=m
+CONFIG_AD7887=m
+CONFIG_AD7923=m
+CONFIG_AD799X=m
+CONFIG_CC10001_ADC=m
+CONFIG_ENVELOPE_DETECTOR=m
+CONFIG_HI8435=m
+CONFIG_HX711=m
+CONFIG_INA2XX_ADC=m
+CONFIG_LTC2471=m
+CONFIG_LTC2485=m
+CONFIG_LTC2497=m
+CONFIG_MAX1027=m
+CONFIG_MAX11100=m
+CONFIG_MAX1118=m
+CONFIG_MAX1363=m
+CONFIG_MAX9611=m
+CONFIG_MCP320X=m
+CONFIG_MCP3422=m
+CONFIG_NAU7802=m
+CONFIG_PALMAS_GPADC=m
+CONFIG_STMPE_ADC=m
+CONFIG_TI_ADC081C=m
+CONFIG_TI_ADC0832=m
+CONFIG_TI_ADC084S021=m
+CONFIG_TI_ADC12138=m
+CONFIG_TI_ADC108S102=m
+CONFIG_TI_ADC128S052=m
+CONFIG_TI_ADC161S626=m
+CONFIG_TI_ADS1015=m
+CONFIG_TI_ADS7950=m
+CONFIG_TI_ADS8688=m
+CONFIG_TI_AM335X_ADC=y
+CONFIG_TI_TLC4541=m
+CONFIG_IIO_RESCALE=m
+CONFIG_AD8366=m
+CONFIG_ATLAS_PH_SENSOR=m
+CONFIG_BME680=m
+CONFIG_CCS811=m
+CONFIG_IAQCORE=m
+CONFIG_VZ89X=m
+CONFIG_AD5064=m
+CONFIG_AD5360=m
+CONFIG_AD5380=m
+CONFIG_AD5421=m
+CONFIG_AD5446=m
+CONFIG_AD5449=m
+CONFIG_AD5592R=m
+CONFIG_AD5593R=m
+CONFIG_AD5504=m
+CONFIG_AD5624R_SPI=m
+CONFIG_LTC2632=m
+CONFIG_AD5686_SPI=m
+CONFIG_AD5696_I2C=m
+CONFIG_AD5755=m
+CONFIG_AD5758=m
+CONFIG_AD5761=m
+CONFIG_AD5764=m
+CONFIG_AD5791=m
+CONFIG_AD7303=m
+CONFIG_AD8801=m
+CONFIG_DPOT_DAC=m
+CONFIG_DS4424=m
+CONFIG_M62332=m
+CONFIG_MAX517=m
+CONFIG_MAX5821=m
+CONFIG_MCP4725=m
+CONFIG_MCP4922=m
+CONFIG_TI_DAC082S085=m
+CONFIG_TI_DAC5571=m
+CONFIG_AD9523=m
+CONFIG_ADF4350=m
+CONFIG_ADIS16080=m
+CONFIG_ADIS16130=m
+CONFIG_ADIS16136=m
+CONFIG_ADIS16260=m
+CONFIG_ADXRS450=m
+CONFIG_BMG160=m
+CONFIG_HID_SENSOR_GYRO_3D=m
+CONFIG_MPU3050_I2C=m
+CONFIG_IIO_ST_GYRO_3AXIS=m
+CONFIG_ITG3200=m
+CONFIG_AFE4403=m
+CONFIG_AFE4404=m
+CONFIG_MAX30100=m
+CONFIG_MAX30102=m
+CONFIG_AM2315=m
+CONFIG_DHT11=m
+CONFIG_HDC100X=m
+CONFIG_HID_SENSOR_HUMIDITY=m
+CONFIG_HTS221=m
+CONFIG_HTU21=m
+CONFIG_SI7005=m
+CONFIG_SI7020=m
+CONFIG_ADIS16400=m
+CONFIG_ADIS16480=m
+CONFIG_BMI160_I2C=m
+CONFIG_BMI160_SPI=m
+CONFIG_KMX61=m
+CONFIG_INV_MPU6050_I2C=m
+CONFIG_INV_MPU6050_SPI=m
+CONFIG_IIO_ST_LSM6DSX=m
+CONFIG_ADJD_S311=m
+CONFIG_AL3320A=m
+CONFIG_APDS9300=m
+CONFIG_APDS9960=m
+CONFIG_BH1750=m
+CONFIG_BH1780=m
+CONFIG_CM32181=m
+CONFIG_CM3232=m
+CONFIG_CM3323=m
+CONFIG_CM3605=m
+CONFIG_CM36651=m
+CONFIG_GP2AP020A00F=m
+CONFIG_SENSORS_ISL29018=m
+CONFIG_SENSORS_ISL29028=m
+CONFIG_ISL29125=m
+CONFIG_HID_SENSOR_ALS=m
+CONFIG_HID_SENSOR_PROX=m
+CONFIG_JSA1212=m
+CONFIG_RPR0521=m
+CONFIG_LTR501=m
+CONFIG_LV0104CS=m
+CONFIG_MAX44000=m
+CONFIG_OPT3001=m
+CONFIG_PA12203001=m
+CONFIG_SI1133=m
+CONFIG_SI1145=m
+CONFIG_STK3310=m
+CONFIG_ST_UVIS25=m
+CONFIG_TCS3414=m
+CONFIG_TCS3472=m
+CONFIG_SENSORS_TSL2563=m
+CONFIG_TSL2583=m
+CONFIG_TSL2772=m
+CONFIG_TSL4531=m
+CONFIG_US5182D=m
+CONFIG_VCNL4000=m
+CONFIG_VEML6070=m
+CONFIG_VL6180=m
+CONFIG_ZOPT2201=m
+CONFIG_AK8974=m
+CONFIG_AK09911=m
+CONFIG_BMC150_MAGN_I2C=m
+CONFIG_BMC150_MAGN_SPI=m
+CONFIG_MAG3110=m
+CONFIG_HID_SENSOR_MAGNETOMETER_3D=m
+CONFIG_MMC35240=m
+CONFIG_IIO_ST_MAGN_3AXIS=m
+CONFIG_SENSORS_HMC5843_I2C=m
+CONFIG_SENSORS_HMC5843_SPI=m
+CONFIG_HID_SENSOR_INCLINOMETER_3D=m
+CONFIG_HID_SENSOR_DEVICE_ROTATION=m
+CONFIG_IIO_HRTIMER_TRIGGER=m
+CONFIG_IIO_INTERRUPT_TRIGGER=m
+CONFIG_IIO_TIGHTLOOP_TRIGGER=m
+CONFIG_IIO_SYSFS_TRIGGER=m
+CONFIG_AD5272=m
+CONFIG_DS1803=m
+CONFIG_MAX5481=m
+CONFIG_MAX5487=m
+CONFIG_MCP4018=m
+CONFIG_MCP4131=m
+CONFIG_MCP4531=m
+CONFIG_TPL0102=m
+CONFIG_LMP91000=m
+CONFIG_ABP060MG=m
+CONFIG_BMP280=m
+CONFIG_HID_SENSOR_PRESS=m
+CONFIG_HP03=m
+CONFIG_MPL115_I2C=m
+CONFIG_MPL115_SPI=m
+CONFIG_MPL3115=m
+CONFIG_MS5611=m
+CONFIG_MS5611_I2C=m
+CONFIG_MS5611_SPI=m
+CONFIG_MS5637=m
+CONFIG_IIO_ST_PRESS=m
+CONFIG_T5403=m
+CONFIG_HP206C=m
+CONFIG_ZPA2326=m
+CONFIG_AS3935=m
+CONFIG_ISL29501=m
+CONFIG_LIDAR_LITE_V2=m
+CONFIG_RFD77402=m
+CONFIG_SRF04=m
+CONFIG_SX9500=m
+CONFIG_SRF08=m
+CONFIG_VL53L0X_I2C=m
+CONFIG_AD2S1200=m
+CONFIG_MAXIM_THERMOCOUPLE=m
+CONFIG_HID_SENSOR_TEMP=m
+CONFIG_MLX90614=m
+CONFIG_MLX90632=m
+CONFIG_TMP006=m
+CONFIG_TMP007=m
+CONFIG_TSYS01=m
+CONFIG_TSYS02D=m
+CONFIG_PWM_OMAP_DMTIMER=y
+CONFIG_PWM_PCA9685=y
+CONFIG_PWM_STMPE=y
+CONFIG_PWM_TIECAP=y
+CONFIG_PWM_TIEHRPWM=y
+CONFIG_RESET_CONTROLLER=y
+CONFIG_RESET_TI_SYSCON=y
+CONFIG_OMAP_USB2=y
+CONFIG_TI_PIPE3=y
+CONFIG_RAS=y
+CONFIG_ANDROID=y
+CONFIG_FPGA=m
+CONFIG_ALTERA_PR_IP_CORE=m
+CONFIG_ALTERA_PR_IP_CORE_PLAT=m
+CONFIG_FPGA_MGR_ALTERA_PS_SPI=m
+CONFIG_FPGA_MGR_XILINX_SPI=m
+CONFIG_FPGA_MGR_ICE40_SPI=m
+CONFIG_FPGA_MGR_MACHXO2_SPI=m
+CONFIG_FPGA_BRIDGE=m
+CONFIG_XILINX_PR_DECOUPLER=m
+CONFIG_FPGA_REGION=m
+CONFIG_OF_FPGA_REGION=m
+CONFIG_EXT4_FS=y
+CONFIG_EXT4_FS_POSIX_ACL=y
+CONFIG_EXT4_FS_SECURITY=y
+CONFIG_EXT4_ENCRYPTION=y
+CONFIG_JFS_FS=m
+CONFIG_JFS_POSIX_ACL=y
+CONFIG_JFS_SECURITY=y
+CONFIG_GFS2_FS=m
+CONFIG_GFS2_FS_LOCKING_DLM=y
+CONFIG_BTRFS_FS=y
+CONFIG_BTRFS_FS_POSIX_ACL=y
+CONFIG_F2FS_FS=y
+CONFIG_F2FS_FS_SECURITY=y
+CONFIG_F2FS_FS_ENCRYPTION=y
+CONFIG_FANOTIFY=y
+CONFIG_FANOTIFY_ACCESS_PERMISSIONS=y
+CONFIG_QUOTA=y
+CONFIG_QUOTA_NETLINK_INTERFACE=y
+CONFIG_QFMT_V2=m
+CONFIG_AUTOFS4_FS=y
+CONFIG_FUSE_FS=y
+CONFIG_CUSE=m
+CONFIG_OVERLAY_FS=y
+CONFIG_FSCACHE=m
+CONFIG_FSCACHE_STATS=y
+CONFIG_CACHEFILES=m
+CONFIG_ISO9660_FS=m
+CONFIG_JOLIET=y
+CONFIG_ZISOFS=y
+CONFIG_UDF_FS=m
+CONFIG_MSDOS_FS=y
+CONFIG_VFAT_FS=y
+CONFIG_FAT_DEFAULT_IOCHARSET="ascii"
+CONFIG_FAT_DEFAULT_UTF8=y
+CONFIG_NTFS_FS=m
+CONFIG_TMPFS_POSIX_ACL=y
+CONFIG_CONFIGFS_FS=y
+CONFIG_ECRYPT_FS=m
+CONFIG_ECRYPT_FS_MESSAGING=y
+CONFIG_JFFS2_FS=m
+CONFIG_JFFS2_SUMMARY=y
+CONFIG_JFFS2_FS_XATTR=y
+CONFIG_JFFS2_COMPRESSION_OPTIONS=y
+CONFIG_JFFS2_LZO=y
+CONFIG_UBIFS_FS=y
+CONFIG_UBIFS_FS_ADVANCED_COMPR=y
+CONFIG_SQUASHFS=m
+CONFIG_SQUASHFS_XATTR=y
+CONFIG_SQUASHFS_LZ4=y
+CONFIG_SQUASHFS_LZO=y
+CONFIG_SQUASHFS_XZ=y
+CONFIG_SQUASHFS_ZSTD=y
+CONFIG_ROMFS_FS=m
+CONFIG_ROMFS_BACKED_BY_BOTH=y
+CONFIG_AUFS_FS=m
+CONFIG_AUFS_EXPORT=y
+CONFIG_AUFS_XATTR=y
+CONFIG_NFS_FS=y
+CONFIG_NFS_V3_ACL=y
+CONFIG_NFS_V4=y
+CONFIG_NFS_SWAP=y
+CONFIG_NFS_V4_1=y
+CONFIG_NFS_V4_2=y
+CONFIG_ROOT_NFS=y
+CONFIG_NFSD=m
+CONFIG_NFSD_V3_ACL=y
+CONFIG_NFSD_V4=y
+CONFIG_NFSD_BLOCKLAYOUT=y
+CONFIG_NFSD_V4_SECURITY_LABEL=y
+CONFIG_RPCSEC_GSS_KRB5=m
+CONFIG_SUNRPC_DEBUG=y
+CONFIG_CEPH_FS=m
+CONFIG_CEPH_FSCACHE=y
+CONFIG_CEPH_FS_POSIX_ACL=y
+CONFIG_CIFS=m
+CONFIG_CIFS_WEAK_PW_HASH=y
+CONFIG_CIFS_UPCALL=y
+CONFIG_CIFS_XATTR=y
+CONFIG_CIFS_POSIX=y
+CONFIG_CIFS_ACL=y
+CONFIG_CIFS_DFS_UPCALL=y
+CONFIG_CIFS_FSCACHE=y
+CONFIG_CODA_FS=m
+CONFIG_AFS_FS=m
+CONFIG_AFS_FSCACHE=y
+CONFIG_9P_FS=m
+CONFIG_9P_FSCACHE=y
+CONFIG_9P_FS_POSIX_ACL=y
+CONFIG_9P_FS_SECURITY=y
+CONFIG_NLS_DEFAULT="utf8"
+CONFIG_NLS_CODEPAGE_437=y
+CONFIG_NLS_CODEPAGE_737=m
+CONFIG_NLS_CODEPAGE_775=m
+CONFIG_NLS_CODEPAGE_850=m
+CONFIG_NLS_CODEPAGE_852=m
+CONFIG_NLS_CODEPAGE_855=m
+CONFIG_NLS_CODEPAGE_857=m
+CONFIG_NLS_CODEPAGE_860=m
+CONFIG_NLS_CODEPAGE_861=m
+CONFIG_NLS_CODEPAGE_862=m
+CONFIG_NLS_CODEPAGE_863=m
+CONFIG_NLS_CODEPAGE_864=m
+CONFIG_NLS_CODEPAGE_865=m
+CONFIG_NLS_CODEPAGE_866=m
+CONFIG_NLS_CODEPAGE_869=m
+CONFIG_NLS_CODEPAGE_936=m
+CONFIG_NLS_CODEPAGE_950=m
+CONFIG_NLS_CODEPAGE_932=m
+CONFIG_NLS_CODEPAGE_949=m
+CONFIG_NLS_CODEPAGE_874=m
+CONFIG_NLS_ISO8859_8=m
+CONFIG_NLS_CODEPAGE_1250=m
+CONFIG_NLS_CODEPAGE_1251=m
+CONFIG_NLS_ASCII=y
+CONFIG_NLS_ISO8859_1=m
+CONFIG_NLS_ISO8859_2=m
+CONFIG_NLS_ISO8859_3=m
+CONFIG_NLS_ISO8859_4=m
+CONFIG_NLS_ISO8859_5=m
+CONFIG_NLS_ISO8859_6=m
+CONFIG_NLS_ISO8859_7=m
+CONFIG_NLS_ISO8859_9=m
+CONFIG_NLS_ISO8859_13=m
+CONFIG_NLS_ISO8859_14=m
+CONFIG_NLS_ISO8859_15=m
+CONFIG_NLS_KOI8_R=m
+CONFIG_NLS_KOI8_U=m
+CONFIG_NLS_MAC_ROMAN=m
+CONFIG_NLS_MAC_CELTIC=m
+CONFIG_NLS_MAC_CENTEURO=m
+CONFIG_NLS_MAC_CROATIAN=m
+CONFIG_NLS_MAC_CYRILLIC=m
+CONFIG_NLS_MAC_GAELIC=m
+CONFIG_NLS_MAC_GREEK=m
+CONFIG_NLS_MAC_ICELAND=m
+CONFIG_NLS_MAC_INUIT=m
+CONFIG_NLS_MAC_ROMANIAN=m
+CONFIG_NLS_MAC_TURKISH=m
+CONFIG_NLS_UTF8=y
+CONFIG_DLM=m
+CONFIG_DLM_DEBUG=y
+CONFIG_SECURITY=y
+CONFIG_SECURITY_NETWORK_XFRM=y
+CONFIG_SECURITY_SELINUX=y
+CONFIG_SECURITY_TOMOYO=y
+CONFIG_SECURITY_APPARMOR=y
+CONFIG_SECURITY_YAMA=y
+CONFIG_INTEGRITY_SIGNATURE=y
+CONFIG_INTEGRITY_ASYMMETRIC_KEYS=y
+CONFIG_DEFAULT_SECURITY_APPARMOR=y
+CONFIG_CRYPTO_USER=m
+CONFIG_CRYPTO_TEST=m
+CONFIG_CRYPTO_CCM=y
+CONFIG_CRYPTO_GCM=y
+CONFIG_CRYPTO_CHACHA20POLY1305=m
+CONFIG_CRYPTO_LRW=m
+CONFIG_CRYPTO_CMAC=y
+CONFIG_CRYPTO_XCBC=m
+CONFIG_CRYPTO_VMAC=m
+CONFIG_CRYPTO_RMD128=m
+CONFIG_CRYPTO_RMD160=m
+CONFIG_CRYPTO_RMD256=m
+CONFIG_CRYPTO_RMD320=m
+CONFIG_CRYPTO_SHA3=m
+CONFIG_CRYPTO_TGR192=m
+CONFIG_CRYPTO_WP512=m
+CONFIG_CRYPTO_ANUBIS=m
+CONFIG_CRYPTO_ARC4=y
+CONFIG_CRYPTO_BLOWFISH=m
+CONFIG_CRYPTO_CAMELLIA=m
+CONFIG_CRYPTO_CAST5=m
+CONFIG_CRYPTO_CAST6=m
+CONFIG_CRYPTO_KHAZAD=m
+CONFIG_CRYPTO_SALSA20=m
+CONFIG_CRYPTO_SEED=m
+CONFIG_CRYPTO_SERPENT=m
+CONFIG_CRYPTO_TEA=m
+CONFIG_CRYPTO_TWOFISH=m
+CONFIG_CRYPTO_LZ4=m
+CONFIG_CRYPTO_LZ4HC=m
+CONFIG_CRYPTO_ANSI_CPRNG=m
+CONFIG_CRYPTO_USER_API_HASH=m
+CONFIG_CRYPTO_USER_API_SKCIPHER=m
+CONFIG_CRYPTO_USER_API_RNG=m
+CONFIG_CRYPTO_USER_API_AEAD=m
+CONFIG_CRYPTO_DEV_OMAP=y
+CONFIG_CRYPTO_DEV_OMAP_SHAM=y
+CONFIG_CRYPTO_DEV_OMAP_AES=y
+CONFIG_CRYPTO_DEV_OMAP_DES=y
+# CONFIG_RAID6_PQ_BENCHMARK is not set
+# CONFIG_XZ_DEC_X86 is not set
+# CONFIG_XZ_DEC_POWERPC is not set
+# CONFIG_XZ_DEC_IA64 is not set
+# CONFIG_XZ_DEC_SPARC is not set
+CONFIG_CORDIC=m
+CONFIG_IRQ_POLL=y
+CONFIG_PRINTK_TIME=y
+CONFIG_BOOT_PRINTK_DELAY=y
+CONFIG_DYNAMIC_DEBUG=y
+CONFIG_STRIP_ASM_SYMS=y
+CONFIG_MAGIC_SYSRQ=y
+CONFIG_MAGIC_SYSRQ_DEFAULT_ENABLE=0x01b6
+CONFIG_DETECT_HUNG_TASK=y
+CONFIG_SCHEDSTATS=y
+CONFIG_SCHED_STACK_END_CHECK=y
+CONFIG_NOTIFIER_ERROR_INJECTION=m
+CONFIG_FTRACE_SYSCALLS=y
+CONFIG_TRACER_SNAPSHOT=y
+CONFIG_STACK_TRACER=y
+CONFIG_BLK_DEV_IO_TRACE=y
+# CONFIG_RUNTIME_TESTING_MENU is not set
+CONFIG_BUG_ON_DATA_CORRUPTION=y
diff -Naur --no-dereference a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h
--- a/arch/arm/include/asm/dma-mapping.h	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/include/asm/dma-mapping.h	2020-06-19 21:00:31.404227715 -0400
@@ -14,6 +14,7 @@
 #include <asm/xen/hypervisor.h>
 
 extern const struct dma_map_ops arm_dma_ops;
+extern const struct dma_map_ops arm_dma_m_ops;
 extern const struct dma_map_ops arm_coherent_dma_ops;
 
 static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
diff -Naur --no-dereference a/arch/arm/include/asm/pgtable-3level.h b/arch/arm/include/asm/pgtable-3level.h
--- a/arch/arm/include/asm/pgtable-3level.h	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/include/asm/pgtable-3level.h	2020-06-19 21:00:31.404227715 -0400
@@ -78,7 +78,12 @@
 #define L_PTE_VALID		(_AT(pteval_t, 1) << 0)		/* Valid */
 #define L_PTE_PRESENT		(_AT(pteval_t, 3) << 0)		/* Present */
 #define L_PTE_USER		(_AT(pteval_t, 1) << 6)		/* AP[1] */
+#ifdef CONFIG_KEYSTONE2_DMA_COHERENT
+/* SH[1:0], outer shareable */
+#define L_PTE_SHARED		(_AT(pteval_t, 2) << 8)
+#else
 #define L_PTE_SHARED		(_AT(pteval_t, 3) << 8)		/* SH[1:0], inner shareable */
+#endif
 #define L_PTE_YOUNG		(_AT(pteval_t, 1) << 10)	/* AF */
 #define L_PTE_XN		(_AT(pteval_t, 1) << 54)	/* XN */
 #define L_PTE_DIRTY		(_AT(pteval_t, 1) << 55)
diff -Naur --no-dereference a/arch/arm/include/asm/pgtable-3level-hwdef.h b/arch/arm/include/asm/pgtable-3level-hwdef.h
--- a/arch/arm/include/asm/pgtable-3level-hwdef.h	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/include/asm/pgtable-3level-hwdef.h	2020-06-19 21:00:31.404227715 -0400
@@ -44,7 +44,11 @@
 #define PMD_SECT_CACHEABLE	(_AT(pmdval_t, 1) << 3)
 #define PMD_SECT_USER		(_AT(pmdval_t, 1) << 6)		/* AP[1] */
 #define PMD_SECT_AP2		(_AT(pmdval_t, 1) << 7)		/* read only */
+#ifdef CONFIG_KEYSTONE2_DMA_COHERENT
+#define PMD_SECT_S		(_AT(pmdval_t, 2) << 8)
+#else
 #define PMD_SECT_S		(_AT(pmdval_t, 3) << 8)
+#endif
 #define PMD_SECT_AF		(_AT(pmdval_t, 1) << 10)
 #define PMD_SECT_nG		(_AT(pmdval_t, 1) << 11)
 #define PMD_SECT_PXN		(_AT(pmdval_t, 1) << 53)
@@ -74,7 +78,12 @@
 #define PTE_BUFFERABLE		(_AT(pteval_t, 1) << 2)		/* AttrIndx[0] */
 #define PTE_CACHEABLE		(_AT(pteval_t, 1) << 3)		/* AttrIndx[1] */
 #define PTE_AP2			(_AT(pteval_t, 1) << 7)		/* AP[2] */
+#ifdef CONFIG_KEYSTONE2_DMA_COHERENT
+/* SH[1:0], outer shareable */
+#define PTE_EXT_SHARED		(_AT(pteval_t, 2) << 8)
+#else
 #define PTE_EXT_SHARED		(_AT(pteval_t, 3) << 8)		/* SH[1:0], inner shareable */
+#endif
 #define PTE_EXT_AF		(_AT(pteval_t, 1) << 10)	/* Access Flag */
 #define PTE_EXT_NG		(_AT(pteval_t, 1) << 11)	/* nG */
 #define PTE_EXT_PXN		(_AT(pteval_t, 1) << 53)	/* PXN */
diff -Naur --no-dereference a/arch/arm/include/asm/xor.h b/arch/arm/include/asm/xor.h
--- a/arch/arm/include/asm/xor.h	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/include/asm/xor.h	2020-06-19 21:00:31.404227715 -0400
@@ -212,3 +212,9 @@
 #else
 #define NEON_TEMPLATES
 #endif
+
+#ifdef CONFIG_KERNEL_MODE_NEON
+#define XOR_SELECT_TEMPLATE(FASTEST)	(&xor_block_neon)
+#else
+#define XOR_SELECT_TEMPLATE(FASTEST)	(&xor_block_arm4regs)
+#endif
diff -Naur --no-dereference a/arch/arm/kernel/armksyms.c b/arch/arm/kernel/armksyms.c
--- a/arch/arm/kernel/armksyms.c	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/kernel/armksyms.c	2020-06-19 21:00:31.404227715 -0400
@@ -182,3 +182,8 @@
 EXPORT_SYMBOL(__arm_smccc_smc);
 EXPORT_SYMBOL(__arm_smccc_hvc);
 #endif
+
+#ifdef CONFIG_ARM_VIRT_EXT
+extern char __hyp_stub_vectors[];
+EXPORT_SYMBOL(__hyp_stub_vectors);
+#endif
diff -Naur --no-dereference a/arch/arm/kernel/smccc-call.S b/arch/arm/kernel/smccc-call.S
--- a/arch/arm/kernel/smccc-call.S	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/kernel/smccc-call.S	2020-06-19 21:00:31.404227715 -0400
@@ -32,11 +32,12 @@
 	.macro SMCCC instr
 UNWIND(	.fnstart)
 	mov	r12, sp
-	push	{r4-r7}
-UNWIND(	.save	{r4-r7})
+	push	{r4-r11}
+UNWIND(	.save	{r4-r11})
 	ldm	r12, {r4-r7}
+	mov	r12, #0x200
 	\instr
-	pop	{r4-r7}
+	pop	{r4-r11}
 	ldr	r12, [sp, #(4 * 4)]
 	stm	r12, {r0-r3}
 	bx	lr
diff -Naur --no-dereference a/arch/arm/mach-davinci/board-dm365-evm.c b/arch/arm/mach-davinci/board-dm365-evm.c
--- a/arch/arm/mach-davinci/board-dm365-evm.c	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/mach-davinci/board-dm365-evm.c	2020-06-19 21:00:31.404227715 -0400
@@ -794,9 +794,9 @@
 	/* maybe setup mmc1/etc ... _after_ mmc0 */
 	evm_init_cpld();
 
-#ifdef CONFIG_SND_DM365_AIC3X_CODEC
+#ifdef CONFIG_SND_SOC_DM365_AIC3X_CODEC
 	dm365_init_asp();
-#elif defined(CONFIG_SND_DM365_VOICE_CODEC)
+#elif defined(CONFIG_SND_SOC_DM365_VOICE_CODEC)
 	dm365_init_vc();
 #endif
 	dm365_init_rtc();
diff -Naur --no-dereference a/arch/arm/mach-keystone/Kconfig b/arch/arm/mach-keystone/Kconfig
--- a/arch/arm/mach-keystone/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/mach-keystone/Kconfig	2020-06-19 21:00:31.404227715 -0400
@@ -11,6 +11,10 @@
 	select ZONE_DMA if ARM_LPAE
 	select PINCTRL
 	select PM_GENERIC_DOMAINS if PM
+	select KEYSTONE2_DMA_COHERENT
 	help
 	  Support for boards based on the Texas Instruments Keystone family of
 	  SoCs.
+
+config KEYSTONE2_DMA_COHERENT
+	bool
diff -Naur --no-dereference a/arch/arm/mach-omap1/Makefile b/arch/arm/mach-omap1/Makefile
--- a/arch/arm/mach-omap1/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/mach-omap1/Makefile	2020-06-19 21:00:31.404227715 -0400
@@ -8,7 +8,7 @@
 	 serial.o devices.o dma.o fb.o
 obj-y += clock.o clock_data.o opp_data.o reset.o pm_bus.o timer.o
 
-ifneq ($(CONFIG_SND_OMAP_SOC_MCBSP),)
+ifneq ($(CONFIG_SND_SOC_OMAP_MCBSP),)
 obj-y += mcbsp.o
 endif
 
diff -Naur --no-dereference a/arch/arm/mach-omap2/clock.c b/arch/arm/mach-omap2/clock.c
--- a/arch/arm/mach-omap2/clock.c	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/mach-omap2/clock.c	2020-06-19 21:00:31.404227715 -0400
@@ -119,6 +119,9 @@
 	if (cpu_is_omap343x())
 		features.flags |= TI_CLK_DPLL_HAS_FREQSEL;
 
+	if (omap_type() == OMAP2_DEVICE_TYPE_GP)
+		features.flags |= TI_CLK_DEVICE_TYPE_GP;
+
 	/* Idlest value for interface clocks.
 	 * 24xx uses 0 to indicate not ready, and 1 to indicate ready.
 	 * 34xx reverses this, just to keep us on our toes
diff -Naur --no-dereference a/arch/arm/mach-omap2/clockdomains43xx_data.c b/arch/arm/mach-omap2/clockdomains43xx_data.c
--- a/arch/arm/mach-omap2/clockdomains43xx_data.c	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/mach-omap2/clockdomains43xx_data.c	2020-06-19 21:00:31.404227715 -0400
@@ -87,6 +87,15 @@
 	.flags		  = CLKDM_CAN_SWSUP,
 };
 
+static struct clockdomain lcdc_43xx_clkdm = {
+	.name		  = "lcdc_clkdm",
+	.pwrdm		  = { .name = "per_pwrdm" },
+	.prcm_partition	  = AM43XX_CM_PARTITION,
+	.cm_inst	  = AM43XX_CM_PER_INST,
+	.clkdm_offs	  = AM43XX_CM_PER_LCDC_CDOFFS,
+	.flags		  = CLKDM_CAN_SWSUP,
+};
+
 static struct clockdomain dss_43xx_clkdm = {
 	.name		  = "dss_clkdm",
 	.pwrdm		  = { .name = "per_pwrdm" },
@@ -176,6 +185,7 @@
 	&pruss_ocp_43xx_clkdm,
 	&ocpwp_l3_43xx_clkdm,
 	&l3s_tsc_43xx_clkdm,
+	&lcdc_43xx_clkdm,
 	&dss_43xx_clkdm,
 	&l3_aon_43xx_clkdm,
 	&emif_43xx_clkdm,
diff -Naur --no-dereference a/arch/arm/mach-omap2/clockdomains7xx_data.c b/arch/arm/mach-omap2/clockdomains7xx_data.c
--- a/arch/arm/mach-omap2/clockdomains7xx_data.c	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/mach-omap2/clockdomains7xx_data.c	2020-06-19 21:00:31.404227715 -0400
@@ -609,7 +609,7 @@
 	.dep_bit	  = DRA7XX_CAM_STATDEP_SHIFT,
 	.wkdep_srcs	  = cam_wkup_sleep_deps,
 	.sleepdep_srcs	  = cam_wkup_sleep_deps,
-	.flags		  = CLKDM_CAN_HWSUP_SWSUP,
+	.flags		  = CLKDM_CAN_SWSUP,
 };
 
 static struct clockdomain l4per_7xx_clkdm = {
diff -Naur --no-dereference a/arch/arm/mach-omap2/display.c b/arch/arm/mach-omap2/display.c
--- a/arch/arm/mach-omap2/display.c	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/mach-omap2/display.c	2020-06-19 21:00:31.404227715 -0400
@@ -214,11 +214,61 @@
 
 	return 0;
 }
-#else
-static inline int omapdss_init_fbdev(void)
+
+static const char * const omapdss_compat_names[] __initconst = {
+	"ti,omap2-dss",
+	"ti,omap3-dss",
+	"ti,omap4-dss",
+	"ti,omap5-dss",
+	"ti,dra7-dss",
+};
+
+static struct device_node * __init omapdss_find_dss_of_node(void)
 {
-	return 0;
+	struct device_node *node;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(omapdss_compat_names); ++i) {
+		node = of_find_compatible_node(NULL, NULL,
+			omapdss_compat_names[i]);
+		if (node)
+			return node;
+	}
+
+	return NULL;
 }
+
+static int __init omapdss_init_of(void)
+{
+	int r;
+	struct device_node *node;
+	struct platform_device *pdev;
+
+	/* only create dss helper devices if dss is enabled in the .dts */
+
+	node = omapdss_find_dss_of_node();
+	if (!node)
+		return 0;
+
+	if (!of_device_is_available(node))
+		return 0;
+
+	pdev = of_find_device_by_node(node);
+
+	if (!pdev) {
+		pr_err("Unable to find DSS platform device\n");
+		return -ENODEV;
+	}
+
+	r = of_platform_populate(node, NULL, NULL, &pdev->dev);
+	if (r) {
+		pr_err("Unable to populate DSS submodule devices\n");
+		return r;
+	}
+
+	return omapdss_init_fbdev();
+}
+omap_device_initcall(omapdss_init_of);
 #endif /* CONFIG_FB_OMAP2 */
 
 static void dispc_disable_outputs(void)
@@ -366,58 +416,3 @@
 
 	return r;
 }
-
-static const char * const omapdss_compat_names[] __initconst = {
-	"ti,omap2-dss",
-	"ti,omap3-dss",
-	"ti,omap4-dss",
-	"ti,omap5-dss",
-	"ti,dra7-dss",
-};
-
-static struct device_node * __init omapdss_find_dss_of_node(void)
-{
-	struct device_node *node;
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(omapdss_compat_names); ++i) {
-		node = of_find_compatible_node(NULL, NULL,
-			omapdss_compat_names[i]);
-		if (node)
-			return node;
-	}
-
-	return NULL;
-}
-
-static int __init omapdss_init_of(void)
-{
-	int r;
-	struct device_node *node;
-	struct platform_device *pdev;
-
-	/* only create dss helper devices if dss is enabled in the .dts */
-
-	node = omapdss_find_dss_of_node();
-	if (!node)
-		return 0;
-
-	if (!of_device_is_available(node))
-		return 0;
-
-	pdev = of_find_device_by_node(node);
-
-	if (!pdev) {
-		pr_err("Unable to find DSS platform device\n");
-		return -ENODEV;
-	}
-
-	r = of_platform_populate(node, NULL, NULL, &pdev->dev);
-	if (r) {
-		pr_err("Unable to populate DSS submodule devices\n");
-		return r;
-	}
-
-	return omapdss_init_fbdev();
-}
-omap_device_initcall(omapdss_init_of);
diff -Naur --no-dereference a/arch/arm/mach-omap2/Makefile b/arch/arm/mach-omap2/Makefile
--- a/arch/arm/mach-omap2/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/mach-omap2/Makefile	2020-06-19 21:00:31.404227715 -0400
@@ -24,7 +24,7 @@
 obj-$(CONFIG_SOC_AM43XX) += $(hwmod-common) $(secure-common)
 obj-$(CONFIG_SOC_DRA7XX) += $(hwmod-common) $(secure-common)
 
-ifneq ($(CONFIG_SND_OMAP_SOC_MCBSP),)
+ifneq ($(CONFIG_SND_SOC_OMAP_MCBSP),)
 obj-y += mcbsp.o
 endif
 
@@ -236,10 +236,18 @@
 
 obj-$(CONFIG_MACH_OMAP2_TUSB6010)	+= usb-tusb6010.o
 
-arch/arm/mach-omap2/pm-asm-offsets.s: arch/arm/mach-omap2/pm-asm-offsets.c
+targets += pm-asm-offsets.s
+
+# Generate offset macros for use in ASM code
+arch/arm/mach-omap2/pm-asm-offsets.s: arch/arm/mach-omap2/pm-asm-offsets.c FORCE
 	$(call if_changed_dep,cc_s_c)
 
 include/generated/ti-pm-asm-offsets.h: arch/arm/mach-omap2/pm-asm-offsets.s FORCE
 	$(call filechk,offsets,__TI_PM_ASM_OFFSETS_H__)
 
 $(obj)/sleep33xx.o $(obj)/sleep43xx.o: include/generated/ti-pm-asm-offsets.h
+
+obj-$(CONFIG_OMAP_IOMMU)		+= omap-iommu.o
+ifneq ($(CONFIG_OMAP_REMOTEPROC),)
+obj-y					+= remoteproc.o
+endif
diff -Naur --no-dereference a/arch/arm/mach-omap2/omap_device.c b/arch/arm/mach-omap2/omap_device.c
--- a/arch/arm/mach-omap2/omap_device.c	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/mach-omap2/omap_device.c	2020-06-19 21:00:31.404227715 -0400
@@ -141,8 +141,8 @@
 	struct omap_hwmod *oh;
 	struct device_node *node = pdev->dev.of_node;
 	struct resource res;
-	const char *oh_name;
-	int oh_cnt, i, ret = 0;
+	const char *oh_name, *rst_name;
+	int oh_cnt, dstr_cnt, i, ret = 0;
 	bool device_active = false, skip_pm_domain = false;
 
 	oh_cnt = of_property_count_strings(node, "ti,hwmods");
@@ -189,6 +189,26 @@
 		ret = PTR_ERR(od);
 		goto odbfd_exit1;
 	}
+	dstr_cnt =
+		of_property_count_strings(node, "ti,deassert-hard-reset");
+	if (dstr_cnt > 0) {
+		for (i = 0; i < dstr_cnt; i += 2) {
+			of_property_read_string_index(
+				node, "ti,deassert-hard-reset", i,
+				&oh_name);
+			of_property_read_string_index(
+				node, "ti,deassert-hard-reset", i+1,
+				&rst_name);
+			oh = omap_hwmod_lookup(oh_name);
+			if (!oh) {
+				dev_warn(&pdev->dev,
+				"Cannot parse deassert property for '%s'\n",
+				oh_name);
+				break;
+			}
+			omap_hwmod_deassert_hardreset(oh, rst_name);
+		}
+	}
 
 	/* Fix up missing resource names */
 	for (i = 0; i < pdev->num_resources; i++) {
@@ -216,6 +236,27 @@
 	return ret;
 }
 
+/**
+ * _omap_device_check_reidle_hwmods - check all hwmods in device for reidle flag
+ * @od: struct omap_device *od
+ *
+ * Checks underlying hwmods for reidle flag, if present, remove from hwmod
+ * list and set flag in omap_device to keep track.  Returns 0.
+ */
+static int _omap_device_check_reidle_hwmods(struct omap_device *od)
+{
+	int i;
+
+	for (i = 0; i < od->hwmods_cnt; i++) {
+		if (od->hwmods[i]->flags & HWMOD_NEEDS_REIDLE) {
+			od->flags |= OMAP_DEVICE_HAS_REIDLE_HWMODS;
+			omap_hwmod_disable_reidle(od->hwmods[i]);
+		}
+	}
+
+	return 0;
+}
+
 static int _omap_device_notifier_call(struct notifier_block *nb,
 				      unsigned long event, void *dev)
 {
@@ -245,6 +286,13 @@
 			pm_runtime_set_active(dev);
 		}
 		break;
+	case BUS_NOTIFY_BOUND_DRIVER:
+		od = to_omap_device(pdev);
+		if (od) {
+			od->_driver_status = BUS_NOTIFY_BOUND_DRIVER;
+			_omap_device_check_reidle_hwmods(od);
+		}
+		break;
 	case BUS_NOTIFY_ADD_DEVICE:
 		if (pdev->dev.of_node)
 			omap_device_build_from_dt(pdev);
@@ -293,6 +341,24 @@
 	return ret;
 }
 
+/**
+ * _omap_device_reidle_hwmods - call omap_hwmod_enable_reidle on all hwmods
+ * @od: struct omap_device *od
+ *
+ * Add all underlying hwmods to hwmod reidle list.  Returns 0.
+ */
+static int _omap_device_reidle_hwmods(struct omap_device *od)
+{
+	int i;
+
+	for (i = 0; i < od->hwmods_cnt; i++)
+		if (od->hwmods[i]->flags | HWMOD_NEEDS_REIDLE)
+			omap_hwmod_enable_reidle(od->hwmods[i]);
+
+	/* XXX pass along return value here? */
+	return 0;
+}
+
 /* Public functions for use by core code */
 
 /**
@@ -378,6 +444,9 @@
 	if (!od)
 		return;
 
+	if (od->flags & OMAP_DEVICE_HAS_REIDLE_HWMODS)
+		_omap_device_reidle_hwmods(od);
+
 	od->pdev->archdata.od = NULL;
 	kfree(od->hwmods);
 	kfree(od);
@@ -829,6 +898,7 @@
 
 static int __init omap_device_init(void)
 {
+	omap_hwmod_setup_reidle();
 	bus_register_notifier(&platform_bus_type, &platform_nb);
 	return 0;
 }
diff -Naur --no-dereference a/arch/arm/mach-omap2/omap_device.h b/arch/arm/mach-omap2/omap_device.h
--- a/arch/arm/mach-omap2/omap_device.h	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/mach-omap2/omap_device.h	2020-06-19 21:00:31.404227715 -0400
@@ -39,6 +39,7 @@
 
 /* omap_device.flags values */
 #define OMAP_DEVICE_SUSPENDED		BIT(0)
+#define OMAP_DEVICE_HAS_REIDLE_HWMODS	BIT(1)
 
 /**
  * struct omap_device - omap_device wrapper for platform_devices
diff -Naur --no-dereference a/arch/arm/mach-omap2/omap_hwmod_33xx_43xx_ipblock_data.c b/arch/arm/mach-omap2/omap_hwmod_33xx_43xx_ipblock_data.c
--- a/arch/arm/mach-omap2/omap_hwmod_33xx_43xx_ipblock_data.c	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/mach-omap2/omap_hwmod_33xx_43xx_ipblock_data.c	2020-06-19 21:00:31.404227715 -0400
@@ -375,7 +375,8 @@
 	.name		= "cpgmac0",
 	.class		= &am33xx_cpgmac0_hwmod_class,
 	.clkdm_name	= "cpsw_125mhz_clkdm",
-	.flags		= (HWMOD_SWSUP_SIDLE | HWMOD_SWSUP_MSTANDBY),
+	.flags		= HWMOD_SWSUP_SIDLE | HWMOD_SWSUP_MSTANDBY |
+			  HWMOD_NEEDS_REIDLE,
 	.main_clk	= "cpsw_125mhz_gclk",
 	.mpu_rt_idx	= 1,
 	.prcm		= {
@@ -618,7 +619,7 @@
 	.class		= &am33xx_gpmc_hwmod_class,
 	.clkdm_name	= "l3s_clkdm",
 	/* Skip reset for CONFIG_OMAP_GPMC_DEBUG for bootloader timings */
-	.flags		= DEBUG_OMAP_GPMC_HWMOD_FLAGS,
+	.flags		= DEBUG_OMAP_GPMC_HWMOD_FLAGS | HWMOD_NEEDS_REIDLE,
 	.main_clk	= "l3s_gclk",
 	.prcm		= {
 		.omap4	= {
@@ -1096,7 +1097,8 @@
 	.name		= "tptc0",
 	.class		= &am33xx_tptc_hwmod_class,
 	.clkdm_name	= "l3_clkdm",
-	.flags		= HWMOD_SWSUP_SIDLE | HWMOD_SWSUP_MSTANDBY,
+	.flags		= HWMOD_SWSUP_SIDLE | HWMOD_SWSUP_MSTANDBY |
+			  HWMOD_NEEDS_REIDLE,
 	.main_clk	= "l3_gclk",
 	.prcm		= {
 		.omap4	= {
@@ -1110,7 +1112,8 @@
 	.name		= "tptc1",
 	.class		= &am33xx_tptc_hwmod_class,
 	.clkdm_name	= "l3_clkdm",
-	.flags		= (HWMOD_SWSUP_SIDLE | HWMOD_SWSUP_MSTANDBY),
+	.flags		= HWMOD_SWSUP_SIDLE | HWMOD_SWSUP_MSTANDBY |
+			  HWMOD_NEEDS_REIDLE,
 	.main_clk	= "l3_gclk",
 	.prcm		= {
 		.omap4	= {
@@ -1124,7 +1127,8 @@
 	.name		= "tptc2",
 	.class		= &am33xx_tptc_hwmod_class,
 	.clkdm_name	= "l3_clkdm",
-	.flags		= (HWMOD_SWSUP_SIDLE | HWMOD_SWSUP_MSTANDBY),
+	.flags		= HWMOD_SWSUP_SIDLE | HWMOD_SWSUP_MSTANDBY |
+			  HWMOD_NEEDS_REIDLE,
 	.main_clk	= "l3_gclk",
 	.prcm		= {
 		.omap4	= {
diff -Naur --no-dereference a/arch/arm/mach-omap2/omap_hwmod_33xx_data.c b/arch/arm/mach-omap2/omap_hwmod_33xx_data.c
--- a/arch/arm/mach-omap2/omap_hwmod_33xx_data.c	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/mach-omap2/omap_hwmod_33xx_data.c	2020-06-19 21:00:31.404227715 -0400
@@ -301,7 +301,8 @@
 	.name		= "usb_otg_hs",
 	.class		= &am33xx_usbotg_class,
 	.clkdm_name	= "l3s_clkdm",
-	.flags		= HWMOD_SWSUP_SIDLE | HWMOD_SWSUP_MSTANDBY,
+	.flags		= HWMOD_SWSUP_SIDLE | HWMOD_SWSUP_MSTANDBY |
+			  HWMOD_NEEDS_REIDLE,
 	.main_clk	= "usbotg_fck",
 	.prcm		= {
 		.omap4	= {
diff -Naur --no-dereference a/arch/arm/mach-omap2/omap_hwmod_44xx_data.c b/arch/arm/mach-omap2/omap_hwmod_44xx_data.c
--- a/arch/arm/mach-omap2/omap_hwmod_44xx_data.c	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/mach-omap2/omap_hwmod_44xx_data.c	2020-06-19 21:00:31.404227715 -0400
@@ -535,7 +535,6 @@
 			.clkctrl_offs = OMAP4_CM_TESLA_TESLA_CLKCTRL_OFFSET,
 			.rstctrl_offs = OMAP4_RM_TESLA_RSTCTRL_OFFSET,
 			.context_offs = OMAP4_RM_TESLA_TESLA_CONTEXT_OFFSET,
-			.modulemode   = MODULEMODE_HWCTRL,
 		},
 	},
 };
@@ -1469,7 +1468,6 @@
 			.clkctrl_offs = OMAP4_CM_DUCATI_DUCATI_CLKCTRL_OFFSET,
 			.rstctrl_offs = OMAP4_RM_DUCATI_RSTCTRL_OFFSET,
 			.context_offs = OMAP4_RM_DUCATI_DUCATI_CONTEXT_OFFSET,
-			.modulemode   = MODULEMODE_HWCTRL,
 		},
 	},
 };
diff -Naur --no-dereference a/arch/arm/mach-omap2/omap_hwmod_54xx_data.c b/arch/arm/mach-omap2/omap_hwmod_54xx_data.c
--- a/arch/arm/mach-omap2/omap_hwmod_54xx_data.c	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/mach-omap2/omap_hwmod_54xx_data.c	2020-06-19 21:00:31.408227715 -0400
@@ -321,6 +321,36 @@
 };
 
 /*
+ * 'dsp' class
+ * dsp sub-system
+ */
+
+static struct omap_hwmod_class omap54xx_dsp_hwmod_class = {
+	.name	= "dsp",
+};
+
+static struct omap_hwmod_rst_info omap54xx_dsp_resets[] = {
+	{ .name = "dsp", .rst_shift = 0 },
+};
+
+/* dsp */
+static struct omap_hwmod omap54xx_dsp_hwmod = {
+	.name		= "dsp",
+	.class		= &omap54xx_dsp_hwmod_class,
+	.clkdm_name	= "dsp_clkdm",
+	.rst_lines	= omap54xx_dsp_resets,
+	.rst_lines_cnt	= ARRAY_SIZE(omap54xx_dsp_resets),
+	.main_clk	= "dpll_iva_h11x2_ck",
+	.prcm = {
+		.omap4 = {
+			.clkctrl_offs = OMAP54XX_CM_DSP_DSP_CLKCTRL_OFFSET,
+			.rstctrl_offs = OMAP54XX_RM_DSP_RSTCTRL_OFFSET,
+			.context_offs = OMAP54XX_RM_DSP_DSP_CONTEXT_OFFSET,
+		},
+	},
+};
+
+/*
  * 'dss' class
  * display sub-system
  */
@@ -903,6 +933,37 @@
 };
 
 /*
+ * 'ipu' class
+ * imaging processor unit
+ */
+
+static struct omap_hwmod_class omap54xx_ipu_hwmod_class = {
+	.name	= "ipu",
+};
+
+static struct omap_hwmod_rst_info omap54xx_ipu_resets[] = {
+	{ .name = "cpu0", .rst_shift = 0 },
+	{ .name = "cpu1", .rst_shift = 1 },
+};
+
+/* ipu */
+static struct omap_hwmod omap54xx_ipu_hwmod = {
+	.name		= "ipu",
+	.class		= &omap54xx_ipu_hwmod_class,
+	.clkdm_name	= "ipu_clkdm",
+	.rst_lines	= omap54xx_ipu_resets,
+	.rst_lines_cnt	= ARRAY_SIZE(omap54xx_ipu_resets),
+	.main_clk	= "dpll_core_h22x2_ck",
+	.prcm = {
+		.omap4 = {
+			.clkctrl_offs = OMAP54XX_CM_IPU_IPU_CLKCTRL_OFFSET,
+			.rstctrl_offs = OMAP54XX_RM_IPU_RSTCTRL_OFFSET,
+			.context_offs = OMAP54XX_RM_IPU_IPU_CONTEXT_OFFSET,
+		},
+	},
+};
+
+/*
  * 'kbd' class
  * keyboard controller
  */
@@ -2074,6 +2135,14 @@
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
+/* dsp -> l3_main_1 */
+static struct omap_hwmod_ocp_if omap54xx_dsp__l3_main_1 = {
+	.master		= &omap54xx_dsp_hwmod,
+	.slave		= &omap54xx_l3_main_1_hwmod,
+	.clk		= "l3_iclk_div",
+	.user		= OCP_USER_MPU | OCP_USER_SDMA,
+};
+
 /* l4_cfg -> mmu_dsp */
 static struct omap_hwmod_ocp_if omap54xx_l4_cfg__mmu_dsp = {
 	.master		= &omap54xx_l4_cfg_hwmod,
@@ -2106,6 +2175,14 @@
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
+/* l3_main_2 -> ipu */
+static struct omap_hwmod_ocp_if omap54xx_l3_main_2__ipu = {
+	.master		= &omap54xx_l3_main_2_hwmod,
+	.slave		= &omap54xx_ipu_hwmod,
+	.clk		= "l3_iclk_div",
+	.user		= OCP_USER_MPU | OCP_USER_SDMA,
+};
+
 /* l3_main_2 -> mmu_ipu */
 static struct omap_hwmod_ocp_if omap54xx_l3_main_2__mmu_ipu = {
 	.master		= &omap54xx_l3_main_2_hwmod,
@@ -2695,7 +2772,9 @@
 	&omap54xx_l3_main_3__l3_instr,
 	&omap54xx_l3_main_2__l3_main_1,
 	&omap54xx_l4_cfg__l3_main_1,
+	&omap54xx_dsp__l3_main_1,
 	&omap54xx_mpu__l3_main_1,
+	&omap54xx_l3_main_2__ipu,
 	&omap54xx_l3_main_1__l3_main_2,
 	&omap54xx_l4_cfg__l3_main_2,
 	&omap54xx_l3_main_1__l3_main_3,
diff -Naur --no-dereference a/arch/arm/mach-omap2/omap_hwmod_7xx_data.c b/arch/arm/mach-omap2/omap_hwmod_7xx_data.c
--- a/arch/arm/mach-omap2/omap_hwmod_7xx_data.c	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/mach-omap2/omap_hwmod_7xx_data.c	2020-06-19 21:00:31.408227715 -0400
@@ -236,6 +236,160 @@
 };
 
 /*
+ * 'vpe' class
+ *
+ */
+
+static struct omap_hwmod_class_sysconfig dra7xx_vpe_sysc = {
+	.rev_offs	= -ENODEV,
+	.sysc_offs	= 0x0010,
+	.sysc_flags	= (SYSC_HAS_MIDLEMODE | SYSC_HAS_SIDLEMODE),
+	.idlemodes	= (SIDLE_FORCE | SIDLE_NO | SIDLE_SMART |
+			   MSTANDBY_FORCE | MSTANDBY_NO |
+			   MSTANDBY_SMART),
+	.sysc_fields	= &omap_hwmod_sysc_type2,
+};
+
+static struct omap_hwmod_class dra7xx_vpe_hwmod_class = {
+	.name	= "vpe",
+	.sysc	= &dra7xx_vpe_sysc,
+};
+
+/* vpe */
+static struct omap_hwmod dra7xx_vpe_hwmod = {
+	.name		= "vpe",
+	.class		= &dra7xx_vpe_hwmod_class,
+	.clkdm_name	= "vpe_clkdm",
+	.flags		= (HWMOD_SWSUP_SIDLE | HWMOD_SWSUP_MSTANDBY),
+	.prcm = {
+		.omap4 = {
+			.clkctrl_offs = DRA7XX_CM_VPE_VPE_CLKCTRL_OFFSET,
+			.context_offs = DRA7XX_RM_VPE_VPE_CONTEXT_OFFSET,
+			.modulemode   = MODULEMODE_HWCTRL,
+		},
+	},
+};
+
+/*
+ * 'vip' class
+ *
+ */
+
+static struct omap_hwmod_class_sysconfig dra7xx_vip_sysc = {
+	.rev_offs	= -ENODEV,
+	.sysc_offs	= 0x0010,
+	.sysc_flags	= (SYSC_HAS_MIDLEMODE | SYSC_HAS_SIDLEMODE),
+	.idlemodes	= (SIDLE_FORCE | SIDLE_NO | SIDLE_SMART |
+			   MSTANDBY_FORCE | MSTANDBY_NO |
+			   MSTANDBY_SMART),
+	.sysc_fields	= &omap_hwmod_sysc_type2,
+};
+
+static struct omap_hwmod_class dra7xx_vip_hwmod_class = {
+	.name	= "vip",
+	.sysc	= &dra7xx_vip_sysc,
+};
+
+/* vip1 */
+static struct omap_hwmod dra7xx_vip1_hwmod = {
+	.name		= "vip1",
+	.class		= &dra7xx_vip_hwmod_class,
+	.clkdm_name	= "cam_clkdm",
+	.main_clk	= "vip1_gclk_mux",
+	.flags		= (HWMOD_SWSUP_SIDLE | HWMOD_SWSUP_MSTANDBY),
+	.prcm = {
+		.omap4 = {
+			.clkctrl_offs = DRA7XX_CM_CAM_VIP1_CLKCTRL_OFFSET,
+			.context_offs = DRA7XX_RM_CAM_VIP1_CONTEXT_OFFSET,
+			.modulemode   = MODULEMODE_HWCTRL,
+		},
+	},
+};
+
+/* vip2 */
+static struct omap_hwmod dra7xx_vip2_hwmod = {
+	.name		= "vip2",
+	.class		= &dra7xx_vip_hwmod_class,
+	.clkdm_name	= "cam_clkdm",
+	.main_clk	= "vip2_gclk_mux",
+	.flags		= (HWMOD_SWSUP_SIDLE | HWMOD_SWSUP_MSTANDBY),
+	.prcm = {
+		.omap4 = {
+			.clkctrl_offs = DRA7XX_CM_CAM_VIP2_CLKCTRL_OFFSET,
+			.context_offs = DRA7XX_RM_CAM_VIP2_CONTEXT_OFFSET,
+			.modulemode   = MODULEMODE_HWCTRL,
+		},
+	},
+};
+
+/* vip3 */
+static struct omap_hwmod dra7xx_vip3_hwmod = {
+	.name		= "vip3",
+	.class		= &dra7xx_vip_hwmod_class,
+	.clkdm_name	= "cam_clkdm",
+	.main_clk	= "vip3_gclk_mux",
+	.flags		= (HWMOD_SWSUP_SIDLE | HWMOD_SWSUP_MSTANDBY),
+	.prcm = {
+		.omap4 = {
+			.clkctrl_offs = DRA7XX_CM_CAM_VIP3_CLKCTRL_OFFSET,
+			.context_offs = DRA7XX_RM_CAM_VIP3_CONTEXT_OFFSET,
+			.modulemode   = MODULEMODE_HWCTRL,
+		},
+	},
+};
+
+/*
+ * 'cal' class
+ *
+ */
+
+static struct omap_hwmod_class_sysconfig dra7xx_cal_sysc = {
+	.rev_offs	= 0x0000,
+	.sysc_offs	= 0x0010,
+	.sysc_flags	= (SYSC_HAS_SIDLEMODE | SYSC_HAS_RESET_STATUS |
+			   SYSC_HAS_SOFTRESET | SYSC_HAS_MIDLEMODE),
+	.idlemodes	= (SIDLE_FORCE | SIDLE_NO | SIDLE_SMART |
+			   MSTANDBY_FORCE | MSTANDBY_NO),
+	.sysc_fields	= &omap_hwmod_sysc_type2,
+};
+
+static struct omap_hwmod_class dra7xx_cal_hwmod_class = {
+	.name	= "cal",
+	.sysc	= &dra7xx_cal_sysc,
+};
+
+/* cal */
+static struct omap_hwmod dra7xx_cal_hwmod = {
+	.name		= "cal",
+	.class		= &dra7xx_cal_hwmod_class,
+	.clkdm_name	= "cam_clkdm",
+	.main_clk	= "vip2_gclk_mux",
+	.flags		= (HWMOD_SWSUP_SIDLE | HWMOD_SWSUP_MSTANDBY),
+	.prcm = {
+		.omap4 = {
+			.clkctrl_offs = DRA7XX_CM_CAM_VIP2_CLKCTRL_OFFSET,
+			.context_offs = DRA7XX_RM_CAM_VIP2_CONTEXT_OFFSET,
+			.modulemode   = MODULEMODE_HWCTRL,
+		},
+	},
+};
+
+static struct omap_hwmod dra76x_cal_hwmod = {
+	.name		= "cal",
+	.class		= &dra7xx_cal_hwmod_class,
+	.clkdm_name	= "cam_clkdm",
+	.main_clk	= "vip3_gclk_mux",
+	.flags		= (HWMOD_SWSUP_SIDLE | HWMOD_SWSUP_MSTANDBY),
+	.prcm = {
+		.omap4 = {
+			.clkctrl_offs = DRA7XX_CM_CAM_VIP3_CLKCTRL_OFFSET,
+			.context_offs = DRA7XX_RM_CAM_VIP3_CONTEXT_OFFSET,
+			.modulemode   = MODULEMODE_HWCTRL,
+		},
+	},
+};
+
+/*
  * 'counter' class
  *
  */
@@ -552,6 +706,53 @@
 };
 
 /*
+ * 'dsp' class
+ * dsp sub-system
+ */
+
+static struct omap_hwmod_class dra7xx_dsp_hwmod_class = {
+	.name   = "dsp",
+};
+
+static struct omap_hwmod_rst_info dra7xx_dsp_resets[] = {
+	{ .name = "dsp", .rst_shift = 0 },
+};
+
+/* dsp1 processor */
+static struct omap_hwmod dra7xx_dsp1_hwmod = {
+	.name		= "dsp1",
+	.class		= &dra7xx_dsp_hwmod_class,
+	.clkdm_name	= "dsp1_clkdm",
+	.rst_lines	= dra7xx_dsp_resets,
+	.rst_lines_cnt	= ARRAY_SIZE(dra7xx_dsp_resets),
+	.main_clk	= "dpll_dsp_m2_ck",
+	.prcm = {
+		.omap4 = {
+			.clkctrl_offs = DRA7XX_CM_DSP1_DSP1_CLKCTRL_OFFSET,
+			.rstctrl_offs = DRA7XX_RM_DSP1_RSTCTRL_OFFSET,
+			.context_offs = DRA7XX_RM_DSP1_DSP1_CONTEXT_OFFSET,
+		},
+	},
+};
+
+/* dsp2 processor */
+static struct omap_hwmod dra7xx_dsp2_hwmod = {
+	.name		= "dsp2",
+	.class		= &dra7xx_dsp_hwmod_class,
+	.clkdm_name	= "dsp2_clkdm",
+	.rst_lines	= dra7xx_dsp_resets,
+	.rst_lines_cnt	= ARRAY_SIZE(dra7xx_dsp_resets),
+	.main_clk	= "dpll_dsp_m2_ck",
+	.prcm = {
+		.omap4 = {
+			.clkctrl_offs = DRA7XX_CM_DSP2_DSP2_CLKCTRL_OFFSET,
+			.rstctrl_offs = DRA7XX_RM_DSP2_RSTCTRL_OFFSET,
+			.context_offs = DRA7XX_RM_DSP2_DSP2_CONTEXT_OFFSET,
+		},
+	},
+};
+
+/*
  * 'dss' class
  *
  */
@@ -1030,6 +1231,40 @@
 };
 
 /*
+ * 'gpu' class
+ * 3d graphics accelerator
+ */
+
+static struct omap_hwmod_class_sysconfig dra7xx_gpu_sysc = {
+	.rev_offs       = 0xfe00,
+	.sysc_offs      = 0xfe10,
+	.sysc_flags     = (SYSC_HAS_MIDLEMODE | SYSC_HAS_SIDLEMODE),
+	.idlemodes      = (SIDLE_FORCE | SIDLE_NO | SIDLE_SMART |
+			   SIDLE_SMART_WKUP | MSTANDBY_FORCE | MSTANDBY_NO |
+			   MSTANDBY_SMART | MSTANDBY_SMART_WKUP),
+	.sysc_fields    = &omap_hwmod_sysc_type2,
+};
+
+static struct omap_hwmod_class dra7xx_gpu_hwmod_class = {
+	.name   = "gpu",
+	.sysc   = &dra7xx_gpu_sysc,
+};
+
+static struct omap_hwmod dra7xx_gpu_hwmod = {
+	.name           = "gpu",
+	.class          = &dra7xx_gpu_hwmod_class,
+	.clkdm_name     = "gpu_clkdm",
+	.main_clk       = "gpu_core_gclk_mux",
+	.prcm = {
+		.omap4 = {
+			.clkctrl_offs = DRA7XX_CM_GPU_GPU_CLKCTRL_OFFSET,
+			.context_offs = DRA7XX_RM_GPU_GPU_CONTEXT_OFFSET,
+			.modulemode   = MODULEMODE_SWCTRL,
+		},
+	},
+};
+
+/*
  * 'hdq1w' class
  *
  */
@@ -1170,6 +1405,54 @@
 };
 
 /*
+ * 'ipu' class
+ * imaging processor unit
+ */
+
+static struct omap_hwmod_class dra7xx_ipu_hwmod_class = {
+	.name	= "ipu",
+};
+
+static struct omap_hwmod_rst_info dra7xx_ipu_resets[] = {
+	{ .name = "cpu0", .rst_shift = 0 },
+	{ .name = "cpu1", .rst_shift = 1 },
+};
+
+/* ipu1 processor */
+static struct omap_hwmod dra7xx_ipu1_hwmod = {
+	.name		= "ipu1",
+	.class		= &dra7xx_ipu_hwmod_class,
+	.clkdm_name	= "ipu1_clkdm",
+	.rst_lines	= dra7xx_ipu_resets,
+	.rst_lines_cnt	= ARRAY_SIZE(dra7xx_ipu_resets),
+	.main_clk	= "ipu1_gfclk_mux",
+	.prcm = {
+		.omap4 = {
+			.clkctrl_offs = DRA7XX_CM_IPU1_IPU1_CLKCTRL_OFFSET,
+			.rstctrl_offs = DRA7XX_RM_IPU1_RSTCTRL_OFFSET,
+			.context_offs = DRA7XX_RM_IPU1_IPU1_CONTEXT_OFFSET,
+		},
+	},
+};
+
+/* ipu2 processor */
+static struct omap_hwmod dra7xx_ipu2_hwmod = {
+	.name		= "ipu2",
+	.class		= &dra7xx_ipu_hwmod_class,
+	.clkdm_name	= "ipu2_clkdm",
+	.rst_lines	= dra7xx_ipu_resets,
+	.rst_lines_cnt	= ARRAY_SIZE(dra7xx_ipu_resets),
+	.main_clk	= "dpll_core_h22x2_ck",
+	.prcm = {
+		.omap4 = {
+			.clkctrl_offs = DRA7XX_CM_IPU2_IPU2_CLKCTRL_OFFSET,
+			.rstctrl_offs = DRA7XX_RM_IPU2_RSTCTRL_OFFSET,
+			.context_offs = DRA7XX_RM_IPU2_IPU2_CONTEXT_OFFSET,
+		},
+	},
+};
+
+/*
  * 'mailbox' class
  *
  */
@@ -1745,6 +2028,142 @@
 };
 
 /*
+ * 'mmu' class
+ * The memory management unit performs virtual to physical address translation
+ * for its requestors.
+ */
+
+static struct omap_hwmod_class_sysconfig dra7xx_mmu_sysc = {
+	.rev_offs	= 0x0000,
+	.sysc_offs	= 0x0010,
+	.syss_offs	= 0x0014,
+	.sysc_flags	= (SYSC_HAS_AUTOIDLE | SYSC_HAS_CLOCKACTIVITY |
+			   SYSC_HAS_SIDLEMODE | SYSC_HAS_SOFTRESET |
+			   SYSS_HAS_RESET_STATUS),
+	.idlemodes	= (SIDLE_FORCE | SIDLE_NO | SIDLE_SMART),
+	.sysc_fields	= &omap_hwmod_sysc_type1,
+};
+
+static struct omap_hwmod_class dra7xx_mmu_hwmod_class = {
+	.name = "mmu",
+	.sysc = &dra7xx_mmu_sysc,
+};
+
+/* DSP MMUs */
+static struct omap_hwmod_rst_info dra7xx_mmu_dsp_resets[] = {
+	{ .name = "mmu_cache", .rst_shift = 1 },
+};
+
+/* mmu0 - dsp1 */
+static struct omap_hwmod dra7xx_mmu0_dsp1_hwmod = {
+	.name		= "mmu0_dsp1",
+	.class		= &dra7xx_mmu_hwmod_class,
+	.clkdm_name	= "dsp1_clkdm",
+	.rst_lines	= dra7xx_mmu_dsp_resets,
+	.rst_lines_cnt	= ARRAY_SIZE(dra7xx_mmu_dsp_resets),
+	.main_clk	= "dpll_dsp_m2_ck",
+	.prcm = {
+		.omap4 = {
+			.clkctrl_offs = DRA7XX_CM_DSP1_DSP1_CLKCTRL_OFFSET,
+			.rstctrl_offs = DRA7XX_RM_DSP1_RSTCTRL_OFFSET,
+			.context_offs = DRA7XX_RM_DSP1_DSP1_CONTEXT_OFFSET,
+			.modulemode   = MODULEMODE_HWCTRL,
+		},
+	},
+};
+
+/* mmu1 - dsp1 */
+static struct omap_hwmod dra7xx_mmu1_dsp1_hwmod = {
+	.name		= "mmu1_dsp1",
+	.class		= &dra7xx_mmu_hwmod_class,
+	.clkdm_name	= "dsp1_clkdm",
+	.rst_lines	= dra7xx_mmu_dsp_resets,
+	.rst_lines_cnt	= ARRAY_SIZE(dra7xx_mmu_dsp_resets),
+	.main_clk	= "dpll_dsp_m2_ck",
+	.prcm = {
+		.omap4 = {
+			.rstctrl_offs = DRA7XX_RM_DSP1_RSTCTRL_OFFSET,
+			.flags = HWMOD_OMAP4_NO_CONTEXT_LOSS_BIT,
+		},
+	},
+};
+
+/* mmu0 - dsp2 */
+static struct omap_hwmod dra7xx_mmu0_dsp2_hwmod = {
+	.name		= "mmu0_dsp2",
+	.class		= &dra7xx_mmu_hwmod_class,
+	.clkdm_name	= "dsp2_clkdm",
+	.rst_lines	= dra7xx_mmu_dsp_resets,
+	.rst_lines_cnt	= ARRAY_SIZE(dra7xx_mmu_dsp_resets),
+	.main_clk	= "dpll_dsp_m2_ck",
+	.prcm = {
+		.omap4 = {
+			.clkctrl_offs = DRA7XX_CM_DSP2_DSP2_CLKCTRL_OFFSET,
+			.rstctrl_offs = DRA7XX_RM_DSP2_RSTCTRL_OFFSET,
+			.context_offs = DRA7XX_RM_DSP2_DSP2_CONTEXT_OFFSET,
+			.modulemode   = MODULEMODE_HWCTRL,
+		},
+	},
+};
+
+/* mmu1 - dsp2 */
+static struct omap_hwmod dra7xx_mmu1_dsp2_hwmod = {
+	.name		= "mmu1_dsp2",
+	.class		= &dra7xx_mmu_hwmod_class,
+	.clkdm_name	= "dsp2_clkdm",
+	.rst_lines	= dra7xx_mmu_dsp_resets,
+	.rst_lines_cnt	= ARRAY_SIZE(dra7xx_mmu_dsp_resets),
+	.main_clk	= "dpll_dsp_m2_ck",
+	.prcm = {
+		.omap4 = {
+			.rstctrl_offs = DRA7XX_RM_DSP2_RSTCTRL_OFFSET,
+			.flags = HWMOD_OMAP4_NO_CONTEXT_LOSS_BIT,
+		},
+	},
+};
+
+/* IPU MMUs */
+static struct omap_hwmod_rst_info dra7xx_mmu_ipu_resets[] = {
+	{ .name = "mmu_cache", .rst_shift = 2 },
+};
+
+/* mmu ipu1 */
+static struct omap_hwmod dra7xx_mmu_ipu1_hwmod = {
+	.name		= "mmu_ipu1",
+	.class		= &dra7xx_mmu_hwmod_class,
+	.clkdm_name	= "ipu1_clkdm",
+	.rst_lines	= dra7xx_mmu_ipu_resets,
+	.rst_lines_cnt	= ARRAY_SIZE(dra7xx_mmu_ipu_resets),
+	.main_clk	= "ipu1_gfclk_mux",
+	.prcm = {
+		.omap4 = {
+			.clkctrl_offs = DRA7XX_CM_IPU1_IPU1_CLKCTRL_OFFSET,
+			.rstctrl_offs = DRA7XX_RM_IPU1_RSTCTRL_OFFSET,
+			.context_offs = DRA7XX_RM_IPU1_IPU1_CONTEXT_OFFSET,
+			.modulemode   = MODULEMODE_HWCTRL,
+		},
+	},
+};
+
+/* mmu ipu2 */
+static struct omap_hwmod dra7xx_mmu_ipu2_hwmod = {
+	.name		= "mmu_ipu2",
+	.class		= &dra7xx_mmu_hwmod_class,
+	.clkdm_name	= "ipu2_clkdm",
+	.rst_lines	= dra7xx_mmu_ipu_resets,
+	.rst_lines_cnt	= ARRAY_SIZE(dra7xx_mmu_ipu_resets),
+	.main_clk	= "dpll_core_h22x2_ck",
+	.prcm = {
+		.omap4 = {
+			.clkctrl_offs = DRA7XX_CM_IPU2_IPU2_CLKCTRL_OFFSET,
+			.rstctrl_offs = DRA7XX_RM_IPU2_RSTCTRL_OFFSET,
+			.context_offs = DRA7XX_RM_IPU2_IPU2_CONTEXT_OFFSET,
+			.modulemode   = MODULEMODE_HWCTRL,
+		},
+	},
+};
+
+/*
  * 'mpu' class
  *
  */
@@ -1896,6 +2315,42 @@
 };
 
 /*
+ * 'pru-icss' class
+ * Programmable Real-Time Unit and Industrial Communication Subsystem
+ */
+static struct omap_hwmod_class dra7xx_pruss_hwmod_class = {
+	.name	= "pruss",
+};
+
+/* pru-icss1 */
+static struct omap_hwmod dra7xx_pruss1_hwmod = {
+	.name		= "pruss1",
+	.class		= &dra7xx_pruss_hwmod_class,
+	.clkdm_name	= "l4per2_clkdm",
+	.prcm		= {
+		.omap4	= {
+			.clkctrl_offs	= DRA7XX_CM_L4PER2_PRUSS1_CLKCTRL_OFFSET,
+			.context_offs	= DRA7XX_RM_L4PER2_PRUSS1_CONTEXT_OFFSET,
+			.modulemode	= MODULEMODE_SWCTRL,
+		},
+	},
+};
+
+/* pru-icss2 */
+static struct omap_hwmod dra7xx_pruss2_hwmod = {
+	.name		= "pruss2",
+	.class		= &dra7xx_pruss_hwmod_class,
+	.clkdm_name	= "l4per2_clkdm",
+	.prcm		= {
+		.omap4	= {
+			.clkctrl_offs	= DRA7XX_CM_L4PER2_PRUSS2_CLKCTRL_OFFSET,
+			.context_offs	= DRA7XX_RM_L4PER2_PRUSS2_CONTEXT_OFFSET,
+			.modulemode	= MODULEMODE_SWCTRL,
+		},
+	},
+};
+
+/*
  * 'qspi' class
  *
  */
@@ -2845,6 +3300,54 @@
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
+/* l3_main_1 -> mmu0_dsp1 */
+static struct omap_hwmod_ocp_if dra7xx_l3_main_1__mmu0_dsp1 = {
+	.master		= &dra7xx_l3_main_1_hwmod,
+	.slave		= &dra7xx_mmu0_dsp1_hwmod,
+	.clk		= "l3_iclk_div",
+	.user		= OCP_USER_MPU | OCP_USER_SDMA,
+};
+
+/* l3_main_1 -> mmu1_dsp1 */
+static struct omap_hwmod_ocp_if dra7xx_l3_main_1__mmu1_dsp1 = {
+	.master		= &dra7xx_l3_main_1_hwmod,
+	.slave		= &dra7xx_mmu1_dsp1_hwmod,
+	.clk		= "l3_iclk_div",
+	.user		= OCP_USER_MPU | OCP_USER_SDMA,
+};
+
+/* l3_main_1 -> mmu0_dsp2 */
+static struct omap_hwmod_ocp_if dra7xx_l3_main_1__mmu0_dsp2 = {
+	.master		= &dra7xx_l3_main_1_hwmod,
+	.slave		= &dra7xx_mmu0_dsp2_hwmod,
+	.clk		= "l3_iclk_div",
+	.user		= OCP_USER_MPU | OCP_USER_SDMA,
+};
+
+/* l3_main_1 -> mmu1_dsp2 */
+static struct omap_hwmod_ocp_if dra7xx_l3_main_1__mmu1_dsp2 = {
+	.master		= &dra7xx_l3_main_1_hwmod,
+	.slave		= &dra7xx_mmu1_dsp2_hwmod,
+	.clk		= "l3_iclk_div",
+	.user		= OCP_USER_MPU | OCP_USER_SDMA,
+};
+
+/* l3_main_1 -> mmu_ipu1 */
+static struct omap_hwmod_ocp_if dra7xx_l3_main_1__mmu_ipu1 = {
+	.master		= &dra7xx_l3_main_1_hwmod,
+	.slave		= &dra7xx_mmu_ipu1_hwmod,
+	.clk		= "l3_iclk_div",
+	.user		= OCP_USER_MPU | OCP_USER_SDMA,
+};
+
+/* l3_main_1 -> mmu_ipu2 */
+static struct omap_hwmod_ocp_if dra7xx_l3_main_1__mmu_ipu2 = {
+	.master		= &dra7xx_l3_main_1_hwmod,
+	.slave		= &dra7xx_mmu_ipu2_hwmod,
+	.clk		= "l3_iclk_div",
+	.user		= OCP_USER_MPU | OCP_USER_SDMA,
+};
+
 /* l3_main_1 -> l4_per1 */
 static struct omap_hwmod_ocp_if dra7xx_l3_main_1__l4_per1 = {
 	.master		= &dra7xx_l3_main_1_hwmod,
@@ -2970,6 +3473,22 @@
 	.user		= OCP_USER_MPU,
 };
 
+/* dsp1 -> l3_main_1 */
+static struct omap_hwmod_ocp_if dra7xx_dsp1__l3_main_1 = {
+	.master		= &dra7xx_dsp1_hwmod,
+	.slave		= &dra7xx_l3_main_1_hwmod,
+	.clk		= "l3_iclk_div",
+	.user		= OCP_USER_MPU | OCP_USER_SDMA,
+};
+
+/* dsp2 -> l3_main_1 */
+static struct omap_hwmod_ocp_if dra7xx_dsp2__l3_main_1 = {
+	.master		= &dra7xx_dsp2_hwmod,
+	.slave		= &dra7xx_l3_main_1_hwmod,
+	.clk		= "l3_iclk_div",
+	.user		= OCP_USER_MPU | OCP_USER_SDMA,
+};
+
 /* l3_main_1 -> dss */
 static struct omap_hwmod_ocp_if dra7xx_l3_main_1__dss = {
 	.master		= &dra7xx_l3_main_1_hwmod,
@@ -3186,6 +3705,14 @@
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
+/* l3_main_1 -> gpu */
+static struct omap_hwmod_ocp_if dra7xx_l3_main_1__gpu = {
+	.master         = &dra7xx_l3_main_1_hwmod,
+	.slave          = &dra7xx_gpu_hwmod,
+	.clk            = "l3_iclk_div",
+	.user           = OCP_USER_MPU | OCP_USER_SDMA,
+};
+
 /* l4_per1 -> hdq1w */
 static struct omap_hwmod_ocp_if dra7xx_l4_per1__hdq1w = {
 	.master		= &dra7xx_l4_per1_hwmod,
@@ -3234,6 +3761,22 @@
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
+/* ipu1 -> l3_main_1 */
+static struct omap_hwmod_ocp_if dra7xx_ipu1__l3_main_1 = {
+	.master		= &dra7xx_ipu1_hwmod,
+	.slave		= &dra7xx_l3_main_1_hwmod,
+	.clk		= "l3_iclk_div",
+	.user		= OCP_USER_MPU | OCP_USER_SDMA,
+};
+
+/* ipu2 -> l3_main_1 */
+static struct omap_hwmod_ocp_if dra7xx_ipu2__l3_main_1 = {
+	.master		= &dra7xx_ipu2_hwmod,
+	.slave		= &dra7xx_l3_main_1_hwmod,
+	.clk		= "l3_iclk_div",
+	.user		= OCP_USER_MPU | OCP_USER_SDMA,
+};
+
 /* l4_cfg -> mailbox1 */
 static struct omap_hwmod_ocp_if dra7xx_l4_cfg__mailbox1 = {
 	.master		= &dra7xx_l4_cfg_hwmod,
@@ -3458,6 +4001,22 @@
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
+/* l4_cfg -> pruss1 */
+static struct omap_hwmod_ocp_if dra7xx_l4_cfg__pruss1 = {
+	.master		= &dra7xx_l4_cfg_hwmod,
+	.slave		= &dra7xx_pruss1_hwmod,
+	.clk		= "dpll_gmac_h13x2_ck",
+	.user		= OCP_USER_MPU | OCP_USER_SDMA,
+};
+
+/* l4_cfg -> pruss2 */
+static struct omap_hwmod_ocp_if dra7xx_l4_cfg__pruss2 = {
+	.master		= &dra7xx_l4_cfg_hwmod,
+	.slave		= &dra7xx_pruss2_hwmod,
+	.clk		= "dpll_gmac_h13x2_ck",
+	.user		= OCP_USER_MPU | OCP_USER_SDMA,
+};
+
 /* l3_main_1 -> qspi */
 static struct omap_hwmod_ocp_if dra7xx_l3_main_1__qspi = {
 	.master		= &dra7xx_l3_main_1_hwmod,
@@ -3793,6 +4352,54 @@
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
+/* l4_per3 -> vpe */
+static struct omap_hwmod_ocp_if dra7xx_l4_per3__vpe = {
+	.master		= &dra7xx_l4_per3_hwmod,
+	.slave		= &dra7xx_vpe_hwmod,
+	.clk		= "l3_iclk_div",
+	.user		= OCP_USER_MPU | OCP_USER_SDMA,
+};
+
+/* l4_per3 -> vip1 */
+static struct omap_hwmod_ocp_if dra7xx_l4_per3__vip1 = {
+	.master		= &dra7xx_l4_per3_hwmod,
+	.slave		= &dra7xx_vip1_hwmod,
+	.clk		= "l3_iclk_div",
+	.user		= OCP_USER_MPU | OCP_USER_SDMA,
+};
+
+/* l4_per3 -> vip2 */
+static struct omap_hwmod_ocp_if dra7xx_l4_per3__vip2 = {
+	.master		= &dra7xx_l4_per3_hwmod,
+	.slave		= &dra7xx_vip2_hwmod,
+	.clk		= "l3_iclk_div",
+	.user		= OCP_USER_MPU | OCP_USER_SDMA,
+};
+
+/* l4_per3 -> vip3 */
+static struct omap_hwmod_ocp_if dra7xx_l4_per3__vip3 = {
+	.master		= &dra7xx_l4_per3_hwmod,
+	.slave		= &dra7xx_vip3_hwmod,
+	.clk		= "l3_iclk_div",
+	.user		= OCP_USER_MPU | OCP_USER_SDMA,
+};
+
+/* l4_per2 -> cal */
+static struct omap_hwmod_ocp_if dra7xx_l4_per2__cal = {
+	.master		= &dra7xx_l4_per2_hwmod,
+	.slave		= &dra7xx_cal_hwmod,
+	.clk		= "l3_iclk_div",
+	.user		= OCP_USER_MPU | OCP_USER_SDMA,
+};
+
+/* l4_per3 -> dra76x_cal */
+static struct omap_hwmod_ocp_if dra76x_l4_per3__cal = {
+	.master		= &dra7xx_l4_per3_hwmod,
+	.slave		= &dra76x_cal_hwmod,
+	.clk		= "l3_iclk_div",
+	.user		= OCP_USER_MPU | OCP_USER_SDMA,
+};
+
 /* l4_wkup -> wd_timer2 */
 static struct omap_hwmod_ocp_if dra7xx_l4_wkup__wd_timer2 = {
 	.master		= &dra7xx_l4_wkup_hwmod,
@@ -3862,6 +4469,7 @@
 	&dra7xx_l3_main_1__tptc1,
 	&dra7xx_l3_main_1__dss,
 	&dra7xx_l3_main_1__dispc,
+	&dra7xx_dsp1__l3_main_1,
 	&dra7xx_l3_main_1__hdmi,
 	&dra7xx_l3_main_1__aes1,
 	&dra7xx_l3_main_1__aes2,
@@ -3876,12 +4484,15 @@
 	&dra7xx_l4_per1__gpio7,
 	&dra7xx_l4_per1__gpio8,
 	&dra7xx_l3_main_1__gpmc,
+	&dra7xx_l3_main_1__gpu,
 	&dra7xx_l4_per1__hdq1w,
 	&dra7xx_l4_per1__i2c1,
 	&dra7xx_l4_per1__i2c2,
 	&dra7xx_l4_per1__i2c3,
 	&dra7xx_l4_per1__i2c4,
 	&dra7xx_l4_per1__i2c5,
+	&dra7xx_ipu1__l3_main_1,
+	&dra7xx_ipu2__l3_main_1,
 	&dra7xx_l4_cfg__mailbox1,
 	&dra7xx_l4_per3__mailbox2,
 	&dra7xx_l4_per3__mailbox3,
@@ -3903,6 +4514,10 @@
 	&dra7xx_l4_per1__mmc2,
 	&dra7xx_l4_per1__mmc3,
 	&dra7xx_l4_per1__mmc4,
+	&dra7xx_l3_main_1__mmu0_dsp1,
+	&dra7xx_l3_main_1__mmu1_dsp1,
+	&dra7xx_l3_main_1__mmu_ipu1,
+	&dra7xx_l3_main_1__mmu_ipu2,
 	&dra7xx_l4_cfg__mpu,
 	&dra7xx_l4_cfg__ocp2scp1,
 	&dra7xx_l4_cfg__ocp2scp3,
@@ -3910,6 +4525,8 @@
 	&dra7xx_l4_cfg__pciess1,
 	&dra7xx_l3_main_1__pciess2,
 	&dra7xx_l4_cfg__pciess2,
+	&dra7xx_l4_cfg__pruss1,
+	&dra7xx_l4_cfg__pruss2,
 	&dra7xx_l3_main_1__qspi,
 	&dra7xx_l4_cfg__sata,
 	&dra7xx_l4_cfg__smartreflex_core,
@@ -3948,6 +4565,8 @@
 	&dra7xx_l4_per2__vcp1,
 	&dra7xx_l3_main_1__vcp2,
 	&dra7xx_l4_per2__vcp2,
+	&dra7xx_l4_per3__vpe,
+	&dra7xx_l4_per3__vip1,
 	&dra7xx_l4_wkup__wd_timer2,
 	&dra7xx_l4_per2__epwmss0,
 	&dra7xx_l4_per2__epwmss1,
@@ -3964,20 +4583,31 @@
 
 /* SoC variant specific hwmod links */
 static struct omap_hwmod_ocp_if *dra76x_hwmod_ocp_ifs[] __initdata = {
+	&dra7xx_dsp2__l3_main_1,
+	&dra7xx_l3_main_1__mmu0_dsp2,
+	&dra7xx_l3_main_1__mmu1_dsp2,
 	&dra7xx_l4_per3__usb_otg_ss4,
+	&dra7xx_l4_per3__vip2,
 	NULL,
 };
 
 static struct omap_hwmod_ocp_if *acd_76x_hwmod_ocp_ifs[] __initdata = {
+	&dra76x_l4_per3__cal,
 	NULL,
 };
 
 static struct omap_hwmod_ocp_if *dra74x_hwmod_ocp_ifs[] __initdata = {
+	&dra7xx_dsp2__l3_main_1,
+	&dra7xx_l3_main_1__mmu0_dsp2,
+	&dra7xx_l3_main_1__mmu1_dsp2,
 	&dra7xx_l4_per3__usb_otg_ss4,
+	&dra7xx_l4_per3__vip2,
+	&dra7xx_l4_per3__vip3,
 	NULL,
 };
 
 static struct omap_hwmod_ocp_if *dra72x_hwmod_ocp_ifs[] __initdata = {
+	&dra7xx_l4_per2__cal,
 	NULL,
 };
 
diff -Naur --no-dereference a/arch/arm/mach-omap2/omap_hwmod.c b/arch/arm/mach-omap2/omap_hwmod.c
--- a/arch/arm/mach-omap2/omap_hwmod.c	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/mach-omap2/omap_hwmod.c	2020-06-19 21:00:31.404227715 -0400
@@ -139,8 +139,10 @@
 #include <linux/spinlock.h>
 #include <linux/slab.h>
 #include <linux/cpu.h>
+#include <linux/cpu_pm.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
+#include <linux/suspend.h>
 #include <linux/bootmem.h>
 
 #include <linux/platform_data/ti-sysc.h>
@@ -188,16 +190,16 @@
 
 /**
  * struct clkctrl_provider - clkctrl provider mapping data
- * @addr: base address for the provider
- * @size: size of the provider address space
- * @offset: offset of the provider from PRCM instance base
+ * @num_addrs: number of base address ranges for the provider
+ * @addr: base address(es) for the provider
+ * @size: size(s) of the provider address space(s)
  * @node: device node associated with the provider
  * @link: list link
  */
 struct clkctrl_provider {
-	u32			addr;
-	u32			size;
-	u16			offset;
+	int			num_addrs;
+	u32			*addr;
+	u32			*size;
 	struct device_node	*node;
 	struct list_head	link;
 };
@@ -236,6 +238,9 @@
 /* omap_hwmod_list contains all registered struct omap_hwmods */
 static LIST_HEAD(omap_hwmod_list);
 
+/* oh_reidle_list contains all omap_hwmods with HWMOD_NEEDS_REIDLE set */
+static LIST_HEAD(oh_reidle_list);
+
 /* mpu_oh: used to add/remove MPU initiator from sleepdep list */
 static struct omap_hwmod *mpu_oh;
 
@@ -724,23 +729,34 @@
 	const __be32 *addrp;
 	struct clkctrl_provider *provider;
 	u64 size;
+	int i;
 
 	provider = memblock_virt_alloc(sizeof(*provider), 0);
 	if (!provider)
 		return -ENOMEM;
 
-	addrp = of_get_address(np, 0, &size, NULL);
-	provider->addr = (u32)of_translate_address(np, addrp);
-	addrp = of_get_address(np->parent, 0, NULL, NULL);
-	provider->offset = provider->addr -
-			   (u32)of_translate_address(np->parent, addrp);
-	provider->addr &= ~0xff;
-	provider->size = size | 0xff;
 	provider->node = np;
 
-	pr_debug("%s: %s: %x...%x [+%x]\n", __func__, np->parent->name,
-		 provider->addr, provider->addr + provider->size,
-		 provider->offset);
+	provider->num_addrs =
+		of_property_count_elems_of_size(np, "reg", sizeof(u32)) / 2;
+
+	provider->addr =
+		memblock_virt_alloc(sizeof(void *) * provider->num_addrs, 0);
+	if (!provider->addr)
+		return -ENOMEM;
+
+	provider->size =
+		memblock_virt_alloc(sizeof(u32) * provider->num_addrs, 0);
+	if (!provider->size)
+		return -ENOMEM;
+
+	for (i = 0; i < provider->num_addrs; i++) {
+		addrp = of_get_address(np, i, &size, NULL);
+		provider->addr[i] = (u32)of_translate_address(np, addrp);
+		provider->size[i] = size;
+		pr_debug("%s: %pOF: %x...%x\n", __func__, np, provider->addr[i],
+			 provider->addr[i] + provider->size[i]);
+	}
 
 	list_add(&provider->link, &clkctrl_providers);
 
@@ -787,23 +803,26 @@
 	pr_debug("%s: %s: addr=%x\n", __func__, oh->name, addr);
 
 	list_for_each_entry(provider, &clkctrl_providers, link) {
-		if (provider->addr <= addr &&
-		    provider->addr + provider->size >= addr) {
-			struct of_phandle_args clkspec;
-
-			clkspec.np = provider->node;
-			clkspec.args_count = 2;
-			clkspec.args[0] = addr - provider->addr -
-					  provider->offset;
-			clkspec.args[1] = 0;
-
-			clk = of_clk_get_from_provider(&clkspec);
-
-			pr_debug("%s: %s got %p (offset=%x, provider=%s)\n",
-				 __func__, oh->name, clk, clkspec.args[0],
-				 provider->node->parent->name);
+		int i;
 
-			return clk;
+		for (i = 0; i < provider->num_addrs; i++) {
+			if (provider->addr[i] <= addr &&
+			    provider->addr[i] + provider->size[i] > addr) {
+				struct of_phandle_args clkspec;
+
+				clkspec.np = provider->node;
+				clkspec.args_count = 2;
+				clkspec.args[0] = addr - provider->addr[0];
+				clkspec.args[1] = 0;
+
+				clk = of_clk_get_from_provider(&clkspec);
+
+				pr_debug("%s: %s got %p (offset=%x, provider=%pOF)\n",
+					 __func__, oh->name, clk,
+					 clkspec.args[0], provider->node);
+
+				return clk;
+			}
 		}
 	}
 
@@ -2267,6 +2286,28 @@
 }
 
 /**
+ * _setup_reidle- check hwmod @oh and add to reidle list
+ * @oh: struct omap_hwmod *
+ * @n: (unused)
+ *
+ * Check hwmod for HWMOD_NEEDS_REIDLE flag and add to list if
+ * necessary. Return 0 on success.
+ */
+static int _setup_reidle(struct omap_hwmod *oh, void *data)
+{
+	int ret;
+
+	if (oh->flags & HWMOD_NEEDS_REIDLE) {
+		ret = omap_hwmod_enable_reidle(oh);
+
+		if (!ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+/**
  * _init_mpu_rt_base - populate the virtual address for a hwmod
  * @oh: struct omap_hwmod * to locate the virtual address
  * @data: (unused, caller should pass NULL)
@@ -2430,7 +2471,7 @@
  */
 static int _setup_reset(struct omap_hwmod *oh)
 {
-	int r;
+	int r = 0;
 
 	if (oh->_state != _HWMOD_STATE_INITIALIZED)
 		return -EINVAL;
@@ -2915,6 +2956,54 @@
 					   oh->prcm.omap4.rstst_offs);
 }
 
+/**
+ * _reidle - enable then idle a single hwmod
+ *
+ * enables and then immediately reidles an hwmod, as certain hwmods may
+ * not have their sysconfig registers programmed in an idle friendly state
+ * by default
+ */
+static void _reidle(struct omap_hwmod *oh)
+{
+	pr_debug("omap_hwmod: %s: %s\n", oh->name, __func__);
+
+	omap_hwmod_enable(oh);
+	omap_hwmod_softreset(oh);
+	omap_hwmod_idle(oh);
+}
+
+/**
+ * _reidle_all - enable then idle all hwmods in oh_reidle_list
+ *
+ * Called by pm_notifier to make sure flagged modules do not block suspend
+ * after context loss.
+ */
+static int _reidle_all(void)
+{
+	struct omap_hwmod_list *oh_list_item = NULL;
+
+	list_for_each_entry(oh_list_item, &oh_reidle_list, oh_list) {
+		_reidle(oh_list_item->oh);
+	}
+
+	return 0;
+}
+
+static int _omap_device_pm_notifier(struct notifier_block *self,
+				    unsigned long action, void *dev)
+{
+	switch (action) {
+	case PM_POST_SUSPEND:
+		_reidle_all();
+	}
+
+	return NOTIFY_DONE;
+}
+
+static struct notifier_block pm_nb = {
+	.notifier_call = _omap_device_pm_notifier,
+};
+
 /* Public functions */
 
 u32 omap_hwmod_read(struct omap_hwmod *oh, u16 reg_offs)
@@ -3409,6 +3498,7 @@
 	struct omap_hwmod_class *class;
 	void __iomem *regs = NULL;
 	unsigned long flags;
+	int ret = 0;
 
 	sysc = kzalloc(sizeof(*sysc), GFP_KERNEL);
 	if (!sysc)
@@ -3425,8 +3515,10 @@
 	if (!oh->_mpu_rt_va) {
 		regs = ioremap(data->module_pa,
 			       data->module_size);
-		if (!regs)
-			return -ENOMEM;
+		if (!regs) {
+			ret = -ENOMEM;
+			goto err;
+		}
 	}
 
 	/*
@@ -3434,8 +3526,10 @@
 	 * may not yet have ioremapped their registers.
 	 */
 	class = kmemdup(oh->class, sizeof(*oh->class), GFP_KERNEL);
-	if (!class)
-		return -ENOMEM;
+	if (!class) {
+		ret = -ENOMEM;
+		goto err;
+	}
 
 	class->sysc = sysc;
 
@@ -3448,6 +3542,9 @@
 	spin_unlock_irqrestore(&oh->_lock, flags);
 
 	return 0;
+err:
+	kfree(sysc);
+	return ret;
 }
 
 /**
@@ -3567,6 +3664,52 @@
 omap_postcore_initcall(omap_hwmod_setup_all);
 
 /**
+ * omap_hwmod_enable_reidle - add an omap_hwmod to reidle list
+ * @oh: struct omap_hwmod *
+ *
+ * Adds the omap_hwmod to the oh_reidle_list so it will gets enabled then idled
+ * after each suspend cycle. Returns 0 on success.
+ */
+int omap_hwmod_enable_reidle(struct omap_hwmod *oh)
+{
+	struct omap_hwmod_list *oh_list_item = NULL;
+
+	oh_list_item = kzalloc(sizeof(*oh_list_item), GFP_KERNEL);
+
+	if (!oh_list_item)
+		return -ENOMEM;
+
+	oh_list_item->oh = oh;
+	list_add(&oh_list_item->oh_list, &oh_reidle_list);
+
+	pr_debug("omap_hwmod: %s: added to reidle list\n", oh->name);
+
+	return 0;
+}
+
+/**
+ * omap_hwmod_disable_reidle - remove an omap_hwmod from reidle list
+ * @oh: struct omap_hwmod *
+ *
+ * Remove the omap_hwmod from the oh_reidle_list. Returns 0 on success.
+ */
+int omap_hwmod_disable_reidle(struct omap_hwmod *oh)
+{
+	struct omap_hwmod_list *li, *oh_list_item = NULL;
+
+	list_for_each_entry_safe(oh_list_item, li, &oh_reidle_list, oh_list) {
+		if (oh_list_item->oh == oh) {
+			list_del(&oh_list_item->oh_list);
+			pr_debug("omap_hwmod: %s: removed from reidle list\n",
+				 oh->name);
+			kfree(oh_list_item);
+		}
+	}
+
+	return 0;
+}
+
+/**
  * omap_hwmod_enable - enable an omap_hwmod
  * @oh: struct omap_hwmod *
  *
@@ -3654,6 +3797,7 @@
 	struct omap_hwmod_ocp_if *oi;
 	struct clockdomain *clkdm;
 	struct clk_hw_omap *clk;
+	struct clk_hw *hw;
 
 	if (!oh)
 		return NULL;
@@ -3670,7 +3814,14 @@
 		c = oi->_clk;
 	}
 
-	clk = to_clk_hw_omap(__clk_get_hw(c));
+	hw = __clk_get_hw(c);
+	if (!hw)
+		return NULL;
+
+	clk = to_clk_hw_omap(hw);
+	if (!clk)
+		return NULL;
+
 	clkdm = clk->clkdm;
 	if (!clkdm)
 		return NULL;
@@ -3933,6 +4084,22 @@
 	return ret;
 }
 
+static int cpu_notifier(struct notifier_block *nb, unsigned long cmd, void *v)
+{
+	switch (cmd) {
+	case CPU_CLUSTER_PM_ENTER:
+		if (enable_off_mode)
+			omap_hwmods_rst_save_context();
+		break;
+	case CPU_CLUSTER_PM_EXIT:
+		if (enable_off_mode)
+			omap_hwmods_rst_restore_context();
+		break;
+	}
+
+	return NOTIFY_OK;
+}
+
 /**
  * omap_hwmod_init - initialize the hwmod code
  *
@@ -3942,6 +4109,8 @@
  */
 void __init omap_hwmod_init(void)
 {
+	static struct notifier_block nb;
+
 	if (cpu_is_omap24xx()) {
 		soc_ops.wait_target_ready = _omap2xxx_3xxx_wait_target_ready;
 		soc_ops.assert_hardreset = _omap2_assert_hardreset;
@@ -3982,10 +4151,31 @@
 
 	_init_clkctrl_providers();
 
+	/* Only AM43XX can lose prm context during rtc-ddr suspend */
+	if (soc_is_am43xx()) {
+		nb.notifier_call = cpu_notifier;
+		cpu_pm_register_notifier(&nb);
+	}
+
 	inited = true;
 }
 
 /**
+ * omap_hwmod_setup_reidle - add hwmods to reidle list and register notifier
+ *
+ * Returns 0 on success.
+ */
+int omap_hwmod_setup_reidle(void)
+{
+	omap_hwmod_for_each(_setup_reidle, NULL);
+
+	if (!list_empty(&oh_reidle_list))
+		register_pm_notifier(&pm_nb);
+
+	return 0;
+}
+
+/**
  * omap_hwmod_get_main_clk - get pointer to main clock name
  * @oh: struct omap_hwmod *
  *
@@ -3999,3 +4189,61 @@
 
 	return oh->main_clk;
 }
+
+/**
+ * omap_hwmod_rst_save_context - Saves the HW reset line state of submodules
+ * @oh: struct omap_hwmod *
+ * @unused: (unused, caller should pass NULL)
+ *
+ * Saves the HW reset line state of all the submodules in the hwmod
+ */
+static int omap_hwmod_rst_save_context(struct omap_hwmod *oh, void *unused)
+{
+	int i;
+
+	for (i = 0; i < oh->rst_lines_cnt; i++)
+		oh->rst_lines[i].context =
+				_read_hardreset(oh, oh->rst_lines[i].name);
+	return 0;
+}
+
+/**
+ * omap_hwmod_rst_restore_context - Restores the HW reset line state of
+ *					submodules
+ * @oh: struct omap_hwmod *
+ * @unused: (unused, caller should pass NULL)
+ *
+ * Restores the HW reset line state of all the submodules in the hwmod
+ */
+static int omap_hwmod_rst_restore_context(struct omap_hwmod *oh, void *unused)
+{
+	int i;
+
+	for (i = 0; i < oh->rst_lines_cnt; i++)
+		if (oh->rst_lines[i].context)
+			_assert_hardreset(oh, oh->rst_lines[i].name);
+		else
+			_deassert_hardreset(oh, oh->rst_lines[i].name);
+
+	return 0;
+}
+
+/**
+ * omap_hwmods_rst_save_context - Saves the HW reset line state for all hwmods
+ *
+ * Saves the HW reset line state of all the registered hwmods
+ */
+void omap_hwmods_rst_save_context(void)
+{
+	omap_hwmod_for_each(omap_hwmod_rst_save_context, NULL);
+}
+
+/**
+ * omap_hwmods_rst_restore_context - Restores the HW reset line state for all hwmods
+ *
+ * Restores the HW reset line state of all the registered hwmods
+ */
+void omap_hwmods_rst_restore_context(void)
+{
+	omap_hwmod_for_each(omap_hwmod_rst_restore_context, NULL);
+}
diff -Naur --no-dereference a/arch/arm/mach-omap2/omap_hwmod.h b/arch/arm/mach-omap2/omap_hwmod.h
--- a/arch/arm/mach-omap2/omap_hwmod.h	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/mach-omap2/omap_hwmod.h	2020-06-19 21:00:31.404227715 -0400
@@ -168,6 +168,7 @@
 	const char	*name;
 	u8		rst_shift;
 	u8		st_shift;
+	u8		context;
 };
 
 /**
@@ -445,6 +446,10 @@
  *     entering HW_AUTO while hwmod is active. This is needed to workaround
  *     some modules which don't function correctly with HW_AUTO. For example,
  *     DCAN on DRA7x SoC needs this to workaround errata i893.
+ * HWMOD_NEEDS_REIDLE: Some devices do not assert their MSTANDBY signal by
+ *     default after losing context if no driver is present and using the
+ *     hwmod. This will break subsequent suspend cycles but can be fixed by
+ *     enabling then idling the unused hwmod after each suspend cycle.
  */
 #define HWMOD_SWSUP_SIDLE			(1 << 0)
 #define HWMOD_SWSUP_MSTANDBY			(1 << 1)
@@ -463,6 +468,7 @@
 #define HWMOD_OPT_CLKS_NEEDED			(1 << 14)
 #define HWMOD_NO_IDLE				(1 << 15)
 #define HWMOD_CLKDM_NOAUTO			(1 << 16)
+#define HWMOD_NEEDS_REIDLE			(1 << 17)
 
 /*
  * omap_hwmod._int_flags definitions
@@ -611,6 +617,14 @@
 
 struct device_node;
 
+/*
+ * omap_hwmod_list - simple generic container for omap_hwmod lists
+ */
+struct omap_hwmod_list {
+	struct omap_hwmod *oh;
+	struct list_head oh_list;
+};
+
 struct omap_hwmod *omap_hwmod_lookup(const char *name);
 int omap_hwmod_for_each(int (*fn)(struct omap_hwmod *oh, void *data),
 			void *data);
@@ -649,6 +663,10 @@
 int omap_hwmod_enable_wakeup(struct omap_hwmod *oh);
 int omap_hwmod_disable_wakeup(struct omap_hwmod *oh);
 
+int omap_hwmod_setup_reidle(void);
+int omap_hwmod_enable_reidle(struct omap_hwmod *oh);
+int omap_hwmod_disable_reidle(struct omap_hwmod *oh);
+
 int omap_hwmod_for_each_by_class(const char *classname,
 				 int (*fn)(struct omap_hwmod *oh,
 					   void *user),
@@ -661,6 +679,9 @@
 
 const char *omap_hwmod_get_main_clk(struct omap_hwmod *oh);
 
+void omap_hwmods_rst_save_context(void);
+void omap_hwmods_rst_restore_context(void);
+
 /*
  *
  */
diff -Naur --no-dereference a/arch/arm/mach-omap2/omap-iommu.c b/arch/arm/mach-omap2/omap-iommu.c
--- a/arch/arm/mach-omap2/omap-iommu.c	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/mach-omap2/omap-iommu.c	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,80 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * OMAP IOMMU quirks for various TI SoCs
+ *
+ * Copyright (C) 2015-2019 Texas Instruments Incorporated - http://www.ti.com/
+ *      Suman Anna <s-anna@ti.com>
+ */
+
+#include <linux/platform_device.h>
+#include <linux/err.h>
+
+#include "omap_hwmod.h"
+#include "omap_device.h"
+#include "clockdomain.h"
+#include "powerdomain.h"
+
+static void omap_iommu_dra7_emu_swsup_config(struct platform_device *pdev,
+					     bool enable)
+{
+	static struct clockdomain *emu_clkdm;
+	static DEFINE_SPINLOCK(emu_lock);
+	static atomic_t count;
+	struct device_node *np = pdev->dev.of_node;
+
+	if (!of_device_is_compatible(np, "ti,dra7-dsp-iommu"))
+		return;
+
+	if (!emu_clkdm) {
+		emu_clkdm = clkdm_lookup("emu_clkdm");
+		if (WARN_ON_ONCE(!emu_clkdm))
+			return;
+	}
+
+	spin_lock(&emu_lock);
+
+	if (enable && (atomic_inc_return(&count) == 1))
+		clkdm_deny_idle(emu_clkdm);
+	else if (!enable && (atomic_dec_return(&count) == 0))
+		clkdm_allow_idle(emu_clkdm);
+
+	spin_unlock(&emu_lock);
+}
+
+int omap_iommu_set_pwrdm_constraint(struct platform_device *pdev, bool request,
+				    u8 *pwrst)
+{
+	struct powerdomain *pwrdm;
+	struct omap_device *od;
+	u8 next_pwrst;
+	int ret = 0;
+
+	od = to_omap_device(pdev);
+	if (!od)
+		return -ENODEV;
+
+	if (od->hwmods_cnt != 1)
+		return -EINVAL;
+
+	pwrdm = omap_hwmod_get_pwrdm(od->hwmods[0]);
+	if (!pwrdm)
+		return -EINVAL;
+
+	if (request) {
+		*pwrst = pwrdm_read_next_pwrst(pwrdm);
+		omap_iommu_dra7_emu_swsup_config(pdev, true);
+	}
+
+	if (*pwrst > PWRDM_POWER_RET)
+		goto out;
+
+	next_pwrst = request ? PWRDM_POWER_ON : *pwrst;
+
+	ret = pwrdm_set_next_pwrst(pwrdm, next_pwrst);
+
+out:
+	if (!request)
+		omap_iommu_dra7_emu_swsup_config(pdev, false);
+
+	return ret;
+}
diff -Naur --no-dereference a/arch/arm/mach-omap2/omap-secure.h b/arch/arm/mach-omap2/omap-secure.h
--- a/arch/arm/mach-omap2/omap-secure.h	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/mach-omap2/omap-secure.h	2020-06-19 21:00:31.404227715 -0400
@@ -54,6 +54,9 @@
 #define OMAP4_PPA_L2_POR_INDEX		0x23
 #define OMAP4_PPA_CPU_ACTRL_SMP_INDEX	0x25
 
+#define AM43xx_PPA_SVC_PM_SUSPEND	0x71
+#define AM43xx_PPA_SVC_PM_RESUME	0x72
+
 /* Secure RX-51 PPA (Primary Protected Application) APIs */
 #define RX51_PPA_HWRNG			29
 #define RX51_PPA_L2_INVAL		40
diff -Naur --no-dereference a/arch/arm/mach-omap2/pdata-quirks.c b/arch/arm/mach-omap2/pdata-quirks.c
--- a/arch/arm/mach-omap2/pdata-quirks.c	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/mach-omap2/pdata-quirks.c	2020-06-19 21:00:31.408227715 -0400
@@ -24,9 +24,12 @@
 #include <linux/platform_data/pinctrl-single.h>
 #include <linux/platform_data/hsmmc-omap.h>
 #include <linux/platform_data/iommu-omap.h>
+#include <linux/platform_data/remoteproc-omap.h>
 #include <linux/platform_data/ti-sysc.h>
+#include <linux/platform_data/ti-pruss.h>
 #include <linux/platform_data/wkup_m3.h>
 #include <linux/platform_data/asoc-ti-mcbsp.h>
+#include <linux/platform_data/sgx-omap.h>
 
 #include "common.h"
 #include "common-board-devices.h"
@@ -35,6 +38,7 @@
 #include "omap-secure.h"
 #include "soc.h"
 #include "hsmmc.h"
+#include "remoteproc.h"
 
 static struct omap_hsmmc_platform_data __maybe_unused mmc_pdata[2];
 
@@ -46,6 +50,36 @@
 static struct of_dev_auxdata omap_auxdata_lookup[];
 static struct twl4030_gpio_platform_data twl_gpio_auxdata;
 
+#if defined(CONFIG_SOC_AM33XX) || defined(CONFIG_SOC_AM43XX)
+static struct gfx_sgx_platform_data sgx_pdata = {
+	.reset_name = "gfx",
+	.assert_reset = omap_device_assert_hardreset,
+	.deassert_reset = omap_device_deassert_hardreset,
+};
+#endif
+
+static bool __maybe_unused omap_device_is_enabled(struct platform_device *pdev)
+{
+	struct omap_device *od = to_omap_device(pdev);
+
+	if (od->_state == OMAP_DEVICE_STATE_ENABLED)
+		return true;
+	else
+		return false;
+}
+
+#if IS_ENABLED(CONFIG_OMAP_IOMMU)
+
+int omap_iommu_set_pwrdm_constraint(struct platform_device *pdev, bool request,
+				    u8 *pwrst);
+#else
+static inline int omap_iommu_set_pwrdm_constraint(struct platform_device *pdev,
+						  bool request, u8 *pwrst)
+{
+	return 0;
+}
+#endif
+
 #ifdef CONFIG_MACH_NOKIA_N8X0
 static void __init omap2420_n8x0_legacy_init(void)
 {
@@ -90,6 +124,13 @@
 	.reset_name = "mmu",
 	.assert_reset = omap_device_assert_hardreset,
 	.deassert_reset = omap_device_deassert_hardreset,
+	.device_enable = omap_device_enable,
+	.device_idle = omap_device_idle,
+};
+
+static struct iommu_platform_data omap3_iommu_isp_pdata = {
+	.device_enable = omap_device_enable,
+	.device_idle = omap_device_idle,
 };
 
 static int omap3_sbc_t3730_twl_callback(struct device *dev,
@@ -317,11 +358,24 @@
 }
 #endif /* CONFIG_ARCH_OMAP3 */
 
-#if defined(CONFIG_ARCH_OMAP4) || defined(CONFIG_SOC_OMAP5)
+#if defined(CONFIG_ARCH_OMAP4) || defined(CONFIG_SOC_OMAP5) || \
+	defined(CONFIG_SOC_DRA7XX)
+static struct omap_rproc_pdata omap4_ipu_dsp_pdata = {
+	.device_enable = omap_rproc_device_enable,
+	.device_shutdown = omap_rproc_device_shutdown,
+	.device_is_enabled = omap_device_is_enabled,
+};
+#endif
+
+#if defined(CONFIG_ARCH_OMAP4) || defined(CONFIG_SOC_OMAP5) || \
+	defined(CONFIG_SOC_DRA7XX)
 static struct iommu_platform_data omap4_iommu_pdata = {
 	.reset_name = "mmu_cache",
 	.assert_reset = omap_device_assert_hardreset,
 	.deassert_reset = omap_device_deassert_hardreset,
+	.device_enable = omap_device_enable,
+	.device_idle = omap_device_idle,
+	.device_is_enabled = omap_device_is_enabled,
 };
 #endif
 
@@ -331,6 +385,12 @@
 	.assert_reset = omap_device_assert_hardreset,
 	.deassert_reset = omap_device_deassert_hardreset,
 };
+
+static struct pruss_platform_data pruss_pdata = {
+	.reset_name = "pruss",
+	.assert_reset = omap_device_assert_hardreset,
+	.deassert_reset = omap_device_deassert_hardreset,
+};
 #endif
 
 #ifdef CONFIG_SOC_OMAP5
@@ -340,6 +400,21 @@
 #endif
 
 #ifdef CONFIG_SOC_DRA7XX
+static struct iommu_platform_data dra7_ipu1_dsp_iommu_pdata = {
+	.reset_name = "mmu_cache",
+	.assert_reset = omap_device_assert_hardreset,
+	.deassert_reset = omap_device_deassert_hardreset,
+	.device_enable = omap_device_enable,
+	.device_is_enabled = omap_device_is_enabled,
+	.device_idle = omap_device_idle,
+	.set_pwrdm_constraint = omap_iommu_set_pwrdm_constraint,
+};
+
+static struct iommu_platform_data dra7_dsp_mmu_edma_pdata = {
+	.device_enable = omap_device_enable,
+	.device_idle = omap_device_idle,
+};
+
 static struct omap_hsmmc_platform_data dra7_hsmmc_data_mmc1;
 static struct omap_hsmmc_platform_data dra7_hsmmc_data_mmc2;
 static struct omap_hsmmc_platform_data dra7_hsmmc_data_mmc3;
@@ -419,7 +494,7 @@
 	dev->platform_data = &twl_gpio_auxdata;
 }
 
-#if IS_ENABLED(CONFIG_SND_OMAP_SOC_MCBSP)
+#if IS_ENABLED(CONFIG_SND_SOC_OMAP_MCBSP)
 static struct omap_mcbsp_platform_data mcbsp_pdata;
 static void __init omap3_mcbsp_init(void)
 {
@@ -456,6 +531,8 @@
 #ifdef CONFIG_ARCH_OMAP3
 	OF_DEV_AUXDATA("ti,omap2-iommu", 0x5d000000, "5d000000.mmu",
 		       &omap3_iommu_pdata),
+	OF_DEV_AUXDATA("ti,omap2-iommu", 0x480bd400, "480bd400.mmu",
+		       &omap3_iommu_isp_pdata),
 	OF_DEV_AUXDATA("ti,omap3-smartreflex-core", 0x480cb000,
 		       "480cb000.smartreflex", &omap_sr_pdata[OMAP_SR_CORE]),
 	OF_DEV_AUXDATA("ti,omap3-smartreflex-mpu-iva", 0x480c9000,
@@ -467,7 +544,7 @@
 	OF_DEV_AUXDATA("ti,am3517-emac", 0x5c000000, "davinci_emac.0",
 		       &am35xx_emac_pdata),
 	/* McBSP modules with sidetone core */
-#if IS_ENABLED(CONFIG_SND_OMAP_SOC_MCBSP)
+#if IS_ENABLED(CONFIG_SND_SOC_OMAP_MCBSP)
 	OF_DEV_AUXDATA("ti,omap3-mcbsp", 0x49022000, "49022000.mcbsp", &mcbsp_pdata),
 	OF_DEV_AUXDATA("ti,omap3-mcbsp", 0x49024000, "49024000.mcbsp", &mcbsp_pdata),
 #endif
@@ -475,10 +552,26 @@
 #ifdef CONFIG_SOC_AM33XX
 	OF_DEV_AUXDATA("ti,am3352-wkup-m3", 0x44d00000, "44d00000.wkup_m3",
 		       &wkup_m3_data),
+	OF_DEV_AUXDATA("ti,am3356-pruss-soc-bus", 0x4a326004,
+		       "4a326004.pruss-soc-bus", &pruss_pdata),
+	OF_DEV_AUXDATA("ti,am3352-sgx530", 0x56000000, "56000000.sgx",
+		       &sgx_pdata),
 #endif
 #ifdef CONFIG_SOC_AM43XX
 	OF_DEV_AUXDATA("ti,am4372-wkup-m3", 0x44d00000, "44d00000.wkup_m3",
 		       &wkup_m3_data),
+	OF_DEV_AUXDATA("ti,am4376-pruss-soc-bus", 0x54426004,
+		       "54426004.pruss_soc_bus", &pruss_pdata),
+	OF_DEV_AUXDATA("ti,am4376-sgx530", 0x56000000, "56000000.sgx",
+		       &sgx_pdata),
+#endif
+#ifdef CONFIG_ARCH_OMAP4
+	OF_DEV_AUXDATA("ti,omap4-dsp", 0, "dsp", &omap4_ipu_dsp_pdata),
+	OF_DEV_AUXDATA("ti,omap4-ipu", 0x55020000, "ipu", &omap4_ipu_dsp_pdata),
+#endif
+#ifdef CONFIG_SOC_OMAP5
+	OF_DEV_AUXDATA("ti,omap5-dsp", 0, "dsp", &omap4_ipu_dsp_pdata),
+	OF_DEV_AUXDATA("ti,omap5-ipu", 0x55020000, "ipu", &omap4_ipu_dsp_pdata),
 #endif
 #if defined(CONFIG_ARCH_OMAP4) || defined(CONFIG_SOC_OMAP5)
 	OF_DEV_AUXDATA("ti,omap4-iommu", 0x4a066000, "4a066000.mmu",
@@ -499,6 +592,26 @@
 		       &dra7_hsmmc_data_mmc2),
 	OF_DEV_AUXDATA("ti,dra7-hsmmc", 0x480ad000, "480ad000.mmc",
 		       &dra7_hsmmc_data_mmc3),
+	OF_DEV_AUXDATA("ti,dra7-dsp-iommu", 0x40d01000, "40d01000.mmu",
+		       &dra7_ipu1_dsp_iommu_pdata),
+	OF_DEV_AUXDATA("ti,dra7-dsp-iommu", 0x41501000, "41501000.mmu",
+		       &dra7_ipu1_dsp_iommu_pdata),
+	OF_DEV_AUXDATA("ti,dra7-dsp-iommu", 0x40d02000, "40d02000.mmu",
+		       &dra7_dsp_mmu_edma_pdata),
+	OF_DEV_AUXDATA("ti,dra7-dsp-iommu", 0x41502000, "41502000.mmu",
+		       &dra7_dsp_mmu_edma_pdata),
+	OF_DEV_AUXDATA("ti,dra7-iommu", 0x55082000, "55082000.mmu",
+		       &omap4_iommu_pdata),
+	OF_DEV_AUXDATA("ti,dra7-iommu", 0x58882000, "58882000.mmu",
+		       &dra7_ipu1_dsp_iommu_pdata),
+	OF_DEV_AUXDATA("ti,dra7-ipu", 0x55020000, "55020000.ipu",
+		       &omap4_ipu_dsp_pdata),
+	OF_DEV_AUXDATA("ti,dra7-ipu", 0x58820000, "58820000.ipu",
+		       &omap4_ipu_dsp_pdata),
+	OF_DEV_AUXDATA("ti,dra7-dsp", 0x40800000, "40800000.dsp",
+		       &omap4_ipu_dsp_pdata),
+	OF_DEV_AUXDATA("ti,dra7-dsp", 0x41000000, "41000000.dsp",
+		       &omap4_ipu_dsp_pdata),
 #endif
 	/* Common auxdata */
 	OF_DEV_AUXDATA("ti,sysc", 0, NULL, &ti_sysc_pdata),
diff -Naur --no-dereference a/arch/arm/mach-omap2/pm33xx-core.c b/arch/arm/mach-omap2/pm33xx-core.c
--- a/arch/arm/mach-omap2/pm33xx-core.c	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/mach-omap2/pm33xx-core.c	2020-06-19 21:00:31.408227715 -0400
@@ -6,10 +6,18 @@
  *	Dave Gerlach
  */
 
+#include <linux/cpuidle.h>
+#include <linux/platform_data/pm33xx.h>
+#include <asm/cpuidle.h>
 #include <asm/smp_scu.h>
 #include <asm/suspend.h>
 #include <linux/errno.h>
-#include <linux/platform_data/pm33xx.h>
+#include <linux/clk.h>
+#include <linux/platform_data/gpio-omap.h>
+#include <linux/pinctrl/pinmux.h>
+#include <linux/wkup_m3_ipc.h>
+#include <linux/of.h>
+#include <linux/rtc.h>
 
 #include "cm33xx.h"
 #include "common.h"
@@ -22,12 +30,21 @@
 #include "prm33xx.h"
 #include "soc.h"
 #include "sram.h"
+#include "omap-secure.h"
 
 static struct powerdomain *cefuse_pwrdm, *gfx_pwrdm, *per_pwrdm, *mpu_pwrdm;
 static struct clockdomain *gfx_l4ls_clkdm;
 static void __iomem *scu_base;
 static struct omap_hwmod *rtc_oh;
 
+static int (*idle_fn)(u32 wfi_flags);
+
+struct amx3_idle_state {
+	int wfi_flags;
+};
+
+static struct amx3_idle_state *idle_states;
+
 static int __init am43xx_map_scu(void)
 {
 	scu_base = ioremap(scu_a9_get_base(), SZ_256);
@@ -38,7 +55,30 @@
 	return 0;
 }
 
-static int amx3_common_init(void)
+static int am33xx_check_off_mode_enable(void)
+{
+	if (enable_off_mode)
+		pr_warn("WARNING: This platform does not support off-mode, entering DeepSleep suspend.\n");
+
+	/* off mode not supported on am335x so return 0 always */
+	return 0;
+}
+
+static int am43xx_check_off_mode_enable(void)
+{
+	/*
+	 * Check for am437x-sk-evm which due to HW design cannot support
+	 * this mode reliably.
+	 */
+	if (of_machine_is_compatible("ti,am437x-sk-evm") && enable_off_mode) {
+		pr_warn("WARNING: This platform does not support off-mode, entering DeepSleep suspend.\n");
+		return 0;
+	}
+
+	return enable_off_mode;
+}
+
+static int amx3_common_init(int (*idle)(u32 wfi_flags))
 {
 	gfx_pwrdm = pwrdm_lookup("gfx_pwrdm");
 	per_pwrdm = pwrdm_lookup("per_pwrdm");
@@ -58,10 +98,12 @@
 	else
 		omap_set_pwrdm_state(cefuse_pwrdm, PWRDM_POWER_OFF);
 
+	idle_fn = idle;
+
 	return 0;
 }
 
-static int am33xx_suspend_init(void)
+static int am33xx_suspend_init(int (*idle)(u32 wfi_flags))
 {
 	int ret;
 
@@ -72,12 +114,12 @@
 		return -ENODEV;
 	}
 
-	ret = amx3_common_init();
+	ret = amx3_common_init(idle);
 
 	return ret;
 }
 
-static int am43xx_suspend_init(void)
+static int am43xx_suspend_init(int (*idle)(u32 wfi_flags))
 {
 	int ret = 0;
 
@@ -87,11 +129,17 @@
 		return ret;
 	}
 
-	ret = amx3_common_init();
+	ret = amx3_common_init(idle);
 
 	return ret;
 }
 
+static int amx3_suspend_deinit(void)
+{
+	idle_fn = NULL;
+	return 0;
+}
+
 static void amx3_pre_suspend_common(void)
 {
 	omap_set_pwrdm_state(gfx_pwrdm, PWRDM_POWER_OFF);
@@ -137,11 +185,51 @@
 {
 	int ret = 0;
 
+	/* Suspend secure side on HS devices */
+	if (omap_type() != OMAP2_DEVICE_TYPE_GP)
+		omap_secure_dispatcher(AM43xx_PPA_SVC_PM_SUSPEND,
+				       FLAG_START_CRITICAL,
+				       0, 0, 0, 0, 0);
+
 	amx3_pre_suspend_common();
 	scu_power_mode(scu_base, SCU_PM_POWEROFF);
 	ret = cpu_suspend(args, fn);
 	scu_power_mode(scu_base, SCU_PM_NORMAL);
-	amx3_post_suspend_common();
+
+	if (!am43xx_check_off_mode_enable())
+		amx3_post_suspend_common();
+
+	/* Resume secure side on HS devices */
+	if (omap_type() != OMAP2_DEVICE_TYPE_GP)
+		omap_secure_dispatcher(AM43xx_PPA_SVC_PM_RESUME,
+				       FLAG_START_CRITICAL,
+				       0, 0, 0, 0, 0);
+
+	return ret;
+}
+
+static int am33xx_cpu_suspend(int (*fn)(unsigned long), unsigned long args)
+{
+	int ret = 0;
+
+	if (omap_irq_pending() || need_resched())
+		return ret;
+
+	ret = cpu_suspend(args, fn);
+
+	return ret;
+}
+
+static int am43xx_cpu_suspend(int (*fn)(unsigned long), unsigned long args)
+{
+	int ret = 0;
+
+	if (!scu_base)
+		return 0;
+
+	scu_power_mode(scu_base, SCU_PM_DORMANT);
+	ret = cpu_suspend(args, fn);
+	scu_power_mode(scu_base, SCU_PM_NORMAL);
 
 	return ret;
 }
@@ -163,17 +251,64 @@
 	return omap_hwmod_get_mpu_rt_va(rtc_oh);
 }
 
+static void am43xx_save_context(void)
+{
+}
+
+static void am33xx_save_context(void)
+{
+	omap_intc_save_context();
+}
+
+static void am33xx_restore_context(void)
+{
+	omap_intc_restore_context();
+}
+
+static void am43xx_restore_context(void)
+{
+	/*
+	 * HACK: restore dpll_per_clkdcoldo register contents, to avoid
+	 * breaking suspend-resume
+	 */
+	writel_relaxed(0x0, AM33XX_L4_WK_IO_ADDRESS(0x44df2e14));
+}
+
+static void am43xx_prepare_rtc_suspend(void)
+{
+	omap_hwmod_enable(rtc_oh);
+}
+
+static void am43xx_prepare_rtc_resume(void)
+{
+	omap_hwmod_idle(rtc_oh);
+}
+
 static struct am33xx_pm_platform_data am33xx_ops = {
 	.init = am33xx_suspend_init,
+	.deinit = amx3_suspend_deinit,
 	.soc_suspend = am33xx_suspend,
+	.cpu_suspend = am33xx_cpu_suspend,
 	.get_sram_addrs = amx3_get_sram_addrs,
+	.save_context = am33xx_save_context,
+	.restore_context = am33xx_restore_context,
+	.prepare_rtc_suspend = am43xx_prepare_rtc_suspend,
+	.prepare_rtc_resume = am43xx_prepare_rtc_resume,
+	.check_off_mode_enable = am33xx_check_off_mode_enable,
 	.get_rtc_base_addr = am43xx_get_rtc_base_addr,
 };
 
 static struct am33xx_pm_platform_data am43xx_ops = {
 	.init = am43xx_suspend_init,
+	.deinit = amx3_suspend_deinit,
 	.soc_suspend = am43xx_suspend,
+	.cpu_suspend = am43xx_cpu_suspend,
 	.get_sram_addrs = amx3_get_sram_addrs,
+	.save_context = am43xx_save_context,
+	.restore_context = am43xx_restore_context,
+	.prepare_rtc_suspend = am43xx_prepare_rtc_suspend,
+	.prepare_rtc_resume = am43xx_prepare_rtc_resume,
+	.check_off_mode_enable = am43xx_check_off_mode_enable,
 	.get_rtc_base_addr = am43xx_get_rtc_base_addr,
 };
 
@@ -203,3 +338,64 @@
 
 	return 0;
 }
+
+static int __init amx3_idle_init(struct device_node *cpu_node, int cpu)
+{
+	struct device_node *state_node;
+	struct amx3_idle_state states[CPUIDLE_STATE_MAX];
+	int i;
+	int state_count = 1;
+
+	for (i = 0; ; i++) {
+		state_node = of_parse_phandle(cpu_node, "cpu-idle-states", i);
+		if (!state_node)
+			break;
+
+		if (!of_device_is_available(state_node))
+			continue;
+
+		if (i == CPUIDLE_STATE_MAX) {
+			pr_warn("%s: cpuidle states reached max possible\n",
+				__func__);
+			break;
+		}
+
+		states[state_count].wfi_flags = 0;
+
+		if (of_property_read_bool(state_node, "ti,idle-wkup-m3"))
+			states[state_count].wfi_flags |= WFI_FLAG_WAKE_M3 |
+							 WFI_FLAG_FLUSH_CACHE;
+
+		state_count++;
+	}
+
+	idle_states = kcalloc(state_count, sizeof(*idle_states), GFP_KERNEL);
+	if (!idle_states)
+		return -ENOMEM;
+
+	for (i = 1; i < state_count; i++)
+		idle_states[i].wfi_flags = states[i].wfi_flags;
+
+	return 0;
+}
+
+static int amx3_idle_enter(unsigned long index)
+{
+	struct amx3_idle_state *idle_state = &idle_states[index];
+
+	if (!idle_state)
+		return -EINVAL;
+
+	if (idle_fn)
+		idle_fn(idle_state->wfi_flags);
+
+	return 0;
+}
+
+static struct cpuidle_ops amx3_cpuidle_ops __initdata = {
+	.init = amx3_idle_init,
+	.suspend = amx3_idle_enter,
+};
+
+CPUIDLE_METHOD_OF_DECLARE(pm33xx_idle, "ti,am3352", &amx3_cpuidle_ops);
+CPUIDLE_METHOD_OF_DECLARE(pm43xx_idle, "ti,am4372", &amx3_cpuidle_ops);
diff -Naur --no-dereference a/arch/arm/mach-omap2/powerdomain.c b/arch/arm/mach-omap2/powerdomain.c
--- a/arch/arm/mach-omap2/powerdomain.c	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/mach-omap2/powerdomain.c	2020-06-19 21:00:31.408227715 -0400
@@ -1187,47 +1187,6 @@
 }
 
 /**
- * pwrdm_can_ever_lose_context - can this powerdomain ever lose context?
- * @pwrdm: struct powerdomain *
- *
- * Given a struct powerdomain * @pwrdm, returns 1 if the powerdomain
- * can lose either memory or logic context or if @pwrdm is invalid, or
- * returns 0 otherwise.  This function is not concerned with how the
- * powerdomain registers are programmed (i.e., to go off or not); it's
- * concerned with whether it's ever possible for this powerdomain to
- * go off while some other part of the chip is active.  This function
- * assumes that every powerdomain can go to either ON or INACTIVE.
- */
-bool pwrdm_can_ever_lose_context(struct powerdomain *pwrdm)
-{
-	int i;
-
-	if (!pwrdm) {
-		pr_debug("powerdomain: %s: invalid powerdomain pointer\n",
-			 __func__);
-		return 1;
-	}
-
-	if (pwrdm->pwrsts & PWRSTS_OFF)
-		return 1;
-
-	if (pwrdm->pwrsts & PWRSTS_RET) {
-		if (pwrdm->pwrsts_logic_ret & PWRSTS_OFF)
-			return 1;
-
-		for (i = 0; i < pwrdm->banks; i++)
-			if (pwrdm->pwrsts_mem_ret[i] & PWRSTS_OFF)
-				return 1;
-	}
-
-	for (i = 0; i < pwrdm->banks; i++)
-		if (pwrdm->pwrsts_mem_on[i] & PWRSTS_OFF)
-			return 1;
-
-	return 0;
-}
-
-/**
  * pwrdm_save_context - save powerdomain registers
  *
  * Register state is going to be lost due to a suspend or hibernate
diff -Naur --no-dereference a/arch/arm/mach-omap2/powerdomain.h b/arch/arm/mach-omap2/powerdomain.h
--- a/arch/arm/mach-omap2/powerdomain.h	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/mach-omap2/powerdomain.h	2020-06-19 21:00:31.408227715 -0400
@@ -247,7 +247,6 @@
 int pwrdm_pre_transition(struct powerdomain *pwrdm);
 int pwrdm_post_transition(struct powerdomain *pwrdm);
 int pwrdm_get_context_loss_count(struct powerdomain *pwrdm);
-bool pwrdm_can_ever_lose_context(struct powerdomain *pwrdm);
 
 extern int omap_set_pwrdm_state(struct powerdomain *pwrdm, u8 state);
 
diff -Naur --no-dereference a/arch/arm/mach-omap2/prcm43xx.h b/arch/arm/mach-omap2/prcm43xx.h
--- a/arch/arm/mach-omap2/prcm43xx.h	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/mach-omap2/prcm43xx.h	2020-06-19 21:00:31.408227715 -0400
@@ -68,6 +68,7 @@
 #define AM43XX_CM_PER_ICSS_CDOFFS			0x0300
 #define AM43XX_CM_PER_L4LS_CDOFFS			0x0400
 #define AM43XX_CM_PER_EMIF_CDOFFS			0x0700
+#define AM43XX_CM_PER_LCDC_CDOFFS			0x0800
 #define AM43XX_CM_PER_DSS_CDOFFS			0x0a00
 #define AM43XX_CM_PER_CPSW_CDOFFS			0x0b00
 #define AM43XX_CM_PER_OCPWP_L3_CDOFFS			0x0c00
diff -Naur --no-dereference a/arch/arm/mach-omap2/remoteproc.c b/arch/arm/mach-omap2/remoteproc.c
--- a/arch/arm/mach-omap2/remoteproc.c	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/mach-omap2/remoteproc.c	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,119 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Remote processor machine-specific module for OMAP4+ SoCs
+ *
+ * Copyright (C) 2011-2019 Texas Instruments Incorporated - http://www.ti.com/
+ *      Suman Anna <s-anna@ti.com>
+ */
+
+#include <linux/kernel.h>
+
+#include "omap_device.h"
+#include "remoteproc.h"
+
+/**
+ * omap_rproc_device_enable - enable the remoteproc device
+ * @pdev: the rproc platform device
+ *
+ * This function performs the necessary low-level functions to enable
+ * a remoteproc device to start executing. This typically includes
+ * releasing the reset lines, and enabling the clocks for the device.
+ * We do not usually expect this function to fail.
+ *
+ * Return: 0 on success, or the return code from the failed function
+ */
+int omap_rproc_device_enable(struct platform_device *pdev)
+{
+	int ret = -EINVAL;
+	struct omap_device *od = to_omap_device(pdev);
+
+	if (!od) {
+		dev_err(&pdev->dev, "device does not have a backing omap_device\n");
+		goto out;
+	}
+
+	/*
+	 * This reset management follows a device name check to differentiate
+	 * DSP and IPU processor subsystems. This check is weak and is ok for
+	 * now because of the dependencies against the pdata-quirks, where
+	 * the devices are given specific device names that satisfy the
+	 * criteria for the check. It can easily be replaced with a stronger
+	 * check like device node compatibility check, if needed.
+	 */
+	if (strstr(dev_name(&pdev->dev), "dsp")) {
+		ret = omap_device_deassert_hardreset(pdev, "dsp");
+		if (ret)
+			goto out;
+	} else if (strstr(dev_name(&pdev->dev), "ipu")) {
+		ret = omap_device_deassert_hardreset(pdev, "cpu0");
+		if (ret)
+			goto out;
+
+		ret = omap_device_deassert_hardreset(pdev, "cpu1");
+		if (ret)
+			goto out;
+	} else {
+		dev_err(&pdev->dev, "unsupported remoteproc\n");
+		goto out;
+	}
+
+	ret = omap_device_enable(pdev);
+
+out:
+	if (ret)
+		dev_err(&pdev->dev, "%s failed, ret = %d\n", __func__, ret);
+	return ret;
+}
+
+/**
+ * omap_rproc_device_shutdown - shutdown the remoteproc device
+ * @pdev: the rproc platform device
+ *
+ * This function performs the necessary low-level functions to shutdown
+ * a remoteproc device. This typically includes disabling the clocks
+ * for the device and asserting the associated reset lines. We do not
+ * usually expect this function to fail.
+ *
+ * Return: 0 on success, or the return code from the failed function
+ */
+int omap_rproc_device_shutdown(struct platform_device *pdev)
+{
+	int ret = -EINVAL;
+	struct omap_device *od = to_omap_device(pdev);
+
+	if (!od) {
+		dev_err(&pdev->dev, "device does not have a backing omap_device\n");
+		goto out;
+	}
+
+	ret = omap_device_idle(pdev);
+	if (ret)
+		goto out;
+
+	/*
+	 * This reset management follows a device name check to differentiate
+	 * DSP and IPU processor subsystems. This check is weak and is ok for
+	 * now because of the dependencies against the pdata-quirks, where
+	 * the devices are given specific device names that satisfy the
+	 * criteria for the check. It can easily be replaced with a stronger
+	 * check like device node compatibility check, if needed.
+	 */
+	if (strstr(dev_name(&pdev->dev), "dsp")) {
+		ret = omap_device_assert_hardreset(pdev, "dsp");
+	} else if (strstr(dev_name(&pdev->dev), "ipu")) {
+		ret = omap_device_assert_hardreset(pdev, "cpu1");
+		if (ret)
+			goto out;
+
+		ret = omap_device_assert_hardreset(pdev, "cpu0");
+		if (ret)
+			goto out;
+	} else {
+		dev_err(&pdev->dev, "unsupported remoteproc\n");
+	}
+
+out:
+	if (ret)
+		dev_err(&pdev->dev, "%s failed, ret = %d\n", __func__, ret);
+	return ret;
+}
diff -Naur --no-dereference a/arch/arm/mach-omap2/remoteproc.h b/arch/arm/mach-omap2/remoteproc.h
--- a/arch/arm/mach-omap2/remoteproc.h	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm/mach-omap2/remoteproc.h	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Remote processor machine-specific quirks for OMAP4+ SoCs
+ *
+ * Copyright (C) 2014-2019 Texas Instruments Incorporated - http://www.ti.com/
+ *      Suman Anna <s-anna@ti.com>
+ */
+
+#ifndef __ARCH_ARM_MACH_OMAP2_REMOTEPROC_H
+#define __ARCH_ARM_MACH_OMAP2_REMOTEPROC_H
+
+#include "linux/platform_device.h"
+
+#if IS_ENABLED(CONFIG_OMAP_REMOTEPROC)
+int omap_rproc_device_enable(struct platform_device *pdev);
+int omap_rproc_device_shutdown(struct platform_device *pdev);
+#else
+static inline int omap_rproc_device_enable(struct platform_device *pdev)
+{
+	return 0;
+}
+
+static inline int omap_rproc_device_shutdown(struct platform_device *pdev)
+{
+	return 0;
+}
+#endif
+
+#endif
diff -Naur --no-dereference a/arch/arm/mach-omap2/sleep43xx.S b/arch/arm/mach-omap2/sleep43xx.S
--- a/arch/arm/mach-omap2/sleep43xx.S	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/mach-omap2/sleep43xx.S	2020-06-19 21:00:31.408227715 -0400
@@ -368,6 +368,9 @@
 	mov     r1, #AM43XX_EMIF_POWEROFF_DISABLE
 	str     r1, [r2, #0x0]
 
+	ldr     r1, [r9, #EMIF_PM_RUN_HW_LEVELING]
+	blx     r1
+
 #ifdef CONFIG_CACHE_L2X0
 	ldr	r2, l2_cache_base
 	ldr	r0, [r2, #L2X0_CTRL]
diff -Naur --no-dereference a/arch/arm/mach-omap2/timer.c b/arch/arm/mach-omap2/timer.c
--- a/arch/arm/mach-omap2/timer.c	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/mach-omap2/timer.c	2020-06-19 21:00:31.408227715 -0400
@@ -33,6 +33,7 @@
 #include <linux/clk.h>
 #include <linux/delay.h>
 #include <linux/irq.h>
+#include <linux/irqchip/arm-gic.h>
 #include <linux/clocksource.h>
 #include <linux/clockchips.h>
 #include <linux/slab.h>
@@ -62,6 +63,10 @@
 #define INCREMENTER_DENUMERATOR_RELOAD_OFFSET		0x14
 #define NUMERATOR_DENUMERATOR_MASK			0xfffff000
 
+#define AM43XX_GIC_CPU_BASE				0x48240100
+
+static void __iomem *gic_cpu_base;
+
 /* Clockevent code */
 
 static struct omap_dm_timer clkev;
@@ -69,6 +74,8 @@
 
 /* Clockevent hwmod for am335x and am437x suspend */
 static struct omap_hwmod *clockevent_gpt_hwmod;
+static struct irq_chip *clkev_irq_chip;
+static struct irq_desc *clkev_irq_desc;
 
 /* Clockesource hwmod for am437x suspend */
 static struct omap_hwmod *clocksource_gpt_hwmod;
@@ -130,11 +137,50 @@
 	return 0;
 }
 
+static int omap_clkevt_late_ack_init(void)
+{
+	gic_cpu_base = ioremap(AM43XX_GIC_CPU_BASE, SZ_4K);
+
+	if (!gic_cpu_base)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static void omap_clkevt_late_ack(void)
+{
+	u32 val;
+
+	if (!clkev_irq_chip)
+		return;
+
+	/*
+	 * For the gic to properly clear an interrupt it must be read
+	 * from INTACK register
+	 */
+	if (gic_cpu_base)
+		val = readl_relaxed(gic_cpu_base + GIC_CPU_INTACK);
+	if (clkev_irq_chip->irq_ack)
+		clkev_irq_chip->irq_ack(&clkev_irq_desc->irq_data);
+	if (clkev_irq_chip->irq_eoi)
+		clkev_irq_chip->irq_eoi(&clkev_irq_desc->irq_data);
+
+	clkev_irq_chip->irq_unmask(&clkev_irq_desc->irq_data);
+}
+
 static void omap_clkevt_idle(struct clock_event_device *unused)
 {
 	if (!clockevent_gpt_hwmod)
 		return;
 
+	/*
+	 * It is possible for a late interrupt to be generated which will
+	 * cause a suspend failure. Let's ack it here both in the timer
+	 * and the interrupt controller to avoid this.
+	 */
+	__omap_dm_timer_write_status(&clkev, OMAP_TIMER_INT_OVERFLOW);
+	omap_clkevt_late_ack();
+
 	omap_hwmod_idle(clockevent_gpt_hwmod);
 }
 
@@ -400,8 +446,16 @@
 
 		clockevent_gpt_hwmod =
 			omap_hwmod_lookup(clockevent_gpt.name);
+
+		clkev_irq_desc = irq_to_desc(clkev.irq);
+		if (clkev_irq_desc)
+			clkev_irq_chip = irq_desc_get_chip(clkev_irq_desc);
+
 	}
 
+	if (soc_is_am437x())
+		omap_clkevt_late_ack_init();
+
 	pr_info("OMAP clockevent source: %s at %lu Hz\n", clockevent_gpt.name,
 		clkev.rate);
 }
diff -Naur --no-dereference a/arch/arm/Makefile b/arch/arm/Makefile
--- a/arch/arm/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/Makefile	2020-06-19 21:00:31.404227715 -0400
@@ -309,7 +309,7 @@
 
 # Build the DT binary blobs if we have OF configured
 ifeq ($(CONFIG_USE_OF),y)
-KBUILD_DTBS := dtbs
+KBUILD_DTBS := dtbs itbs
 endif
 
 all:	$(notdir $(KBUILD_IMAGE)) $(KBUILD_DTBS)
@@ -339,17 +339,28 @@
 $(INSTALL_TARGETS):
 	$(Q)$(MAKE) $(build)=$(boot) MACHINE=$(MACHINE) $@
 
-%.dtb: | scripts
-	$(Q)$(MAKE) $(build)=$(boot)/dts MACHINE=$(MACHINE) $(boot)/dts/$@
+%.dtb %.dtbo %.itb: dtbdir=$(patsubst %/,%,$(patsubst %./,%,$(boot)/dts/$(dir $@)))
 
-PHONY += dtbs dtbs_install
+%.dtb %.dtbo: | scripts
+	$(Q)$(MAKE) $(build)=$(dtbdir) MACHINE=$(MACHINE) $(boot)/dts/$@
+
+%.itb: | scripts zImage
+	$(Q)$(MAKE) $(build)=$(dtbdir) MACHINE=$(MACHINE) $(boot)/dts/$@
+
+PHONY += dtbs dtbs_install itbs_install
 
 dtbs: prepare scripts
 	$(Q)$(MAKE) $(build)=$(boot)/dts
 
+itbs: prepare scripts zImage
+	$(Q)$(MAKE) $(build)=$(boot)/dts BUILD_ITBS=y
+
 dtbs_install:
 	$(Q)$(MAKE) $(dtbinst)=$(boot)/dts
 
+itbs_install:
+	$(Q)$(MAKE) $(dtbinst)=$(boot)/dts BUILD_ITBS=y
+
 PHONY += vdso_install
 vdso_install:
 ifeq ($(CONFIG_VDSO),y)
@@ -373,6 +384,8 @@
   echo  '                  (supply initrd image via make variable INITRD=<path>)'
   echo  '* dtbs          - Build device tree blobs for enabled boards'
   echo  '  dtbs_install  - Install dtbs to $(INSTALL_DTBS_PATH)'
+  echo  '* itbs          - Build FIT images for enabled boards'
+  echo  '  itbs_install  - Install FIT images to $(INSTALL_DTBS_PATH)'
   echo  '  install       - Install uncompressed kernel'
   echo  '  zinstall      - Install compressed kernel'
   echo  '  uinstall      - Install U-Boot wrapped compressed kernel'
diff -Naur --no-dereference a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
--- a/arch/arm/mm/dma-mapping.c	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm/mm/dma-mapping.c	2020-06-19 21:00:31.408227715 -0400
@@ -50,6 +50,7 @@
 	const void *caller;
 	bool want_vaddr;
 	int coherent_flag;
+	bool zero;
 };
 
 struct arm_dma_free_args {
@@ -203,6 +204,27 @@
 };
 EXPORT_SYMBOL(arm_dma_ops);
 
+static void *arm_dma_malloc(struct device *dev, size_t size, dma_addr_t *handle,
+			    gfp_t gfp, unsigned long dma_attrs);
+
+const struct dma_map_ops arm_dma_m_ops = {
+	.alloc                  = arm_dma_malloc,
+	.free                   = arm_dma_free,
+	.mmap                   = arm_dma_mmap,
+	.get_sgtable            = arm_dma_get_sgtable,
+	.map_page               = arm_dma_map_page,
+	.unmap_page             = arm_dma_unmap_page,
+	.map_sg                 = arm_dma_map_sg,
+	.unmap_sg               = arm_dma_unmap_sg,
+	.sync_single_for_cpu    = arm_dma_sync_single_for_cpu,
+	.sync_single_for_device = arm_dma_sync_single_for_device,
+	.sync_sg_for_cpu        = arm_dma_sync_sg_for_cpu,
+	.sync_sg_for_device     = arm_dma_sync_sg_for_device,
+	.mapping_error		= arm_dma_mapping_error,
+	.dma_supported		= arm_dma_supported,
+};
+EXPORT_SYMBOL(arm_dma_m_ops);
+
 static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
 	dma_addr_t *handle, gfp_t gfp, unsigned long attrs);
 static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr,
@@ -356,7 +378,7 @@
 static void *__alloc_from_contiguous(struct device *dev, size_t size,
 				     pgprot_t prot, struct page **ret_page,
 				     const void *caller, bool want_vaddr,
-				     int coherent_flag, gfp_t gfp);
+				     int coherent_flag, gfp_t gfp, bool zero);
 
 static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
 				 pgprot_t prot, struct page **ret_page,
@@ -413,7 +435,7 @@
 	if (dev_get_cma_area(NULL))
 		ptr = __alloc_from_contiguous(NULL, atomic_pool_size, prot,
 				      &page, atomic_pool_init, true, NORMAL,
-				      GFP_KERNEL);
+				      GFP_KERNEL, true);
 	else
 		ptr = __alloc_remap_buffer(NULL, atomic_pool_size, gfp, prot,
 					   &page, atomic_pool_init, true);
@@ -587,7 +609,7 @@
 static void *__alloc_from_contiguous(struct device *dev, size_t size,
 				     pgprot_t prot, struct page **ret_page,
 				     const void *caller, bool want_vaddr,
-				     int coherent_flag, gfp_t gfp)
+				     int coherent_flag, gfp_t gfp, bool zero)
 {
 	unsigned long order = get_order(size);
 	size_t count = size >> PAGE_SHIFT;
@@ -598,7 +620,8 @@
 	if (!page)
 		return NULL;
 
-	__dma_clear_buffer(page, size, coherent_flag);
+	if (zero)
+		__dma_clear_buffer(page, size, coherent_flag);
 
 	if (!want_vaddr)
 		goto out;
@@ -675,7 +698,7 @@
 	return __alloc_from_contiguous(args->dev, args->size, args->prot,
 				       ret_page, args->caller,
 				       args->want_vaddr, args->coherent_flag,
-				       args->gfp);
+				       args->gfp, args->zero);
 }
 
 static void cma_allocator_free(struct arm_dma_free_args *args)
@@ -728,7 +751,7 @@
 
 static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
 			 gfp_t gfp, pgprot_t prot, bool is_coherent,
-			 unsigned long attrs, const void *caller)
+			 unsigned long attrs, const void *caller, bool zero)
 {
 	u64 mask = get_coherent_dma_mask(dev);
 	struct page *page = NULL;
@@ -743,6 +766,7 @@
 		.caller = caller,
 		.want_vaddr = ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) == 0),
 		.coherent_flag = is_coherent ? COHERENT : NORMAL,
+		.zero = zero,
 	};
 
 #ifdef CONFIG_DMA_API_DEBUG
@@ -816,14 +840,27 @@
 	pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
 
 	return __dma_alloc(dev, size, handle, gfp, prot, false,
-			   attrs, __builtin_return_address(0));
+			   attrs, __builtin_return_address(0), true);
+}
+
+/*
+ * Same as arm_dma_alloc except don't zero memory on alloc
+ */
+void *arm_dma_malloc(struct device *dev, size_t size, dma_addr_t *handle,
+		     gfp_t gfp, unsigned long attrs)
+{
+	pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
+
+	return __dma_alloc(dev, size, handle, gfp, prot, false,
+			   attrs, __builtin_return_address(0),
+			   false);
 }
 
 static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
 	dma_addr_t *handle, gfp_t gfp, unsigned long attrs)
 {
 	return __dma_alloc(dev, size, handle, gfp, PAGE_KERNEL, true,
-			   attrs, __builtin_return_address(0));
+			   attrs, __builtin_return_address(0), true);
 }
 
 static int __arm_dma_mmap(struct device *dev, struct vm_area_struct *vma,
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/am654-industrial-thermal.dtsi b/arch/arm64/boot/dts/ti/am654-industrial-thermal.dtsi
--- a/arch/arm64/boot/dts/ti/am654-industrial-thermal.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/am654-industrial-thermal.dtsi	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,45 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <dt-bindings/thermal/thermal.h>
+
+mpu0_thermal: mpu0_thermal {
+	polling-delay-passive = <250>; /* milliseconds */
+	polling-delay = <500>; /* milliseconds */
+	thermal-sensors = <&wkup_vtm0 0>;
+
+	trips {
+		mpu0_crit: mpu0_crit {
+			temperature = <125000>; /* milliCelsius */
+			hysteresis = <2000>; /* milliCelsius */
+			type = "critical";
+		};
+	};
+};
+
+mpu1_thermal: mpu1_thermal {
+	polling-delay-passive = <250>; /* milliseconds */
+	polling-delay = <500>; /* milliseconds */
+	thermal-sensors = <&wkup_vtm0 1>;
+
+	trips {
+		mpu1_crit: mpu1_crit {
+			temperature = <125000>; /* milliCelsius */
+			hysteresis = <2000>; /* milliCelsius */
+			type = "critical";
+		};
+	};
+};
+
+mcu_thermal: mcu_thermal {
+	polling-delay-passive = <250>; /* milliseconds */
+	polling-delay = <500>; /* milliseconds */
+	thermal-sensors = <&wkup_vtm0 2>;
+
+	trips {
+		mcu_crit: mcu_crit {
+			temperature = <125000>; /* milliCelsius */
+			hysteresis = <2000>; /* milliCelsius */
+			type = "critical";
+		};
+	};
+};
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/k3-am654-base-board.dts b/arch/arm64/boot/dts/ti/k3-am654-base-board.dts
--- a/arch/arm64/boot/dts/ti/k3-am654-base-board.dts	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/k3-am654-base-board.dts	2020-06-19 21:00:31.408227715 -0400
@@ -6,11 +6,18 @@
 /dts-v1/;
 
 #include "k3-am654.dtsi"
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/net/ti-dp83867.h>
 
 / {
 	compatible =  "ti,am654-evm", "ti,am654";
 	model = "Texas Instruments AM654 Base Board";
 
+	aliases {
+		ethernet1 = &pruss2_emac0;
+		ethernet2 = &pruss2_emac1;
+	};
+
 	chosen {
 		stdout-path = "serial2:115200n8";
 		bootargs = "earlycon=ns16550a,mmio32,0x02800000";
@@ -23,14 +30,521 @@
 		      <0x00000008 0x80000000 0x00000000 0x80000000>;
 	};
 
-	reserved-memory {
+	reserved_memory: reserved-memory {
 		#address-cells = <2>;
 		#size-cells = <2>;
 		ranges;
+
 		secure_ddr: secure_ddr@9e800000 {
 			reg = <0 0x9e800000 0 0x01800000>; /* for OP-TEE */
 			alignment = <0x1000>;
 			no-map;
 		};
+
+		mcu_r5fss0_core0_dma_memory_region: r5f-dma-memory@a0000000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0xa0000000 0 0x100000>;
+			no-map;
+		};
+
+		mcu_r5fss0_core0_memory_region: r5f-memory@a0100000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0xa0100000 0 0xf00000>;
+			no-map;
+		};
+
+		mcu_r5fss0_core1_dma_memory_region: r5f-dma-memory@a1000000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0xa1000000 0 0x100000>;
+			no-map;
+		};
+
+		mcu_r5fss0_core1_memory_region: r5f-memory@a1100000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0xa1100000 0 0xf00000>;
+			no-map;
+		};
+
+		rtos_ipc_memory_region: ipc-memories@a2000000 {
+			reg = <0x00 0xa2000000 0x00 0x00200000>;
+			alignment = <0x1000>;
+			no-map;
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		autorepeat;
+		pinctrl-names = "default";
+		pinctrl-0 = <&push_button_pins_default>;
+
+		sw5 {
+			label = "GPIO Key USER1";
+			linux,code = <BTN_0>;
+			gpios = <&wkup_gpio0 24 GPIO_ACTIVE_LOW>;
+		};
+
+		sw6 {
+			label = "GPIO Key USER2";
+			linux,code = <BTN_1>;
+			gpios = <&wkup_gpio0 27 GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	/* Dual Ethernet application node on PRU-ICSSG2 */
+	pruss2_eth {
+		compatible = "ti,am654-icssg-prueth";
+		pinctrl-names = "default";
+		pinctrl-0 = <&icssg2_rgmii_pins_default>;
+		sram = <&msmc_ram>;
+		interrupt-parent = <&main_udmass_inta>;
+
+		prus = <&pru2_0>, <&rtu2_0>, <&pru2_1>, <&rtu2_1>;
+		firmware-name = "ti-pruss/am65x-pru0-prueth-fw.elf",
+				"ti-pruss/am65x-rtu0-prueth-fw.elf",
+				"ti-pruss/am65x-pru1-prueth-fw.elf",
+				"ti-pruss/am65x-rtu1-prueth-fw.elf";
+		mii-g-rt = <&icssg2_mii_g_rt>;
+		mii-rt = <&icssg2_mii_rt>;
+		dma-coherent;
+		dmas = <&main_udmap &icssg2 0 UDMA_DIR_TX>, /* egress slice 0 */
+		       <&main_udmap &icssg2 1 UDMA_DIR_TX>, /* egress slice 0 */
+		       <&main_udmap &icssg2 2 UDMA_DIR_TX>, /* egress slice 0 */
+		       <&main_udmap &icssg2 3 UDMA_DIR_TX>, /* egress slice 0 */
+		       <&main_udmap &icssg2 4 UDMA_DIR_TX>, /* egress slice 1 */
+		       <&main_udmap &icssg2 5 UDMA_DIR_TX>, /* egress slice 1 */
+		       <&main_udmap &icssg2 6 UDMA_DIR_TX>, /* egress slice 1 */
+		       <&main_udmap &icssg2 7 UDMA_DIR_TX>, /* egress slice 1 */
+
+		       <&main_udmap &icssg2 0 UDMA_DIR_RX>, /* ingress slice 0 */
+		       <&main_udmap &icssg2 1 UDMA_DIR_RX>, /* ingress slice 1 */
+		       <&main_udmap &icssg2 2 UDMA_DIR_RX>, /* mgmnt rsp slice 0 */
+		       <&main_udmap &icssg2 3 UDMA_DIR_RX>; /* mgmnt rsp slice 1 */
+		dma-names = "tx0-0", "tx0-1", "tx0-2", "tx0-3",
+			    "tx1-0", "tx1-1", "tx1-2", "tx1-3",
+			    "rx0", "rx1",
+			    "rxmgm0", "rxmgm1";
+
+		pruss2_emac0: ethernet-mii0 {
+			phy-handle = <&pruss2_eth0_phy>;
+			phy-mode = "rgmii-rxid";
+			syscon-rgmii-delay = <&scm_conf 0x4120>;
+			iep = <&icssg2_iep0>;
+			/* Filled in by bootloader */
+			local-mac-address = [00 00 00 00 00 00];
+		};
+
+		pruss2_emac1: ethernet-mii1 {
+			phy-handle = <&pruss2_eth1_phy>;
+			phy-mode = "rgmii-rxid";
+			syscon-rgmii-delay = <&scm_conf 0x4124>;
+			iep = <&icssg2_iep1>;
+			/* Filled in by bootloader */
+			local-mac-address = [00 00 00 00 00 00];
+		};
+	};
+};
+
+&wkup_pmx0 {
+	wkup_i2c0_pins_default: wkup-i2c0-pins-default {
+		pinctrl-single,pins = <
+			AM65X_WKUP_IOPAD(0x00e0, PIN_INPUT, 0) /* (AC7) WKUP_I2C0_SCL */
+			AM65X_WKUP_IOPAD(0x00e4, PIN_INPUT, 0) /* (AD6) WKUP_I2C0_SDA */
+		>;
+	};
+
+	push_button_pins_default: push_button__pins_default {
+		pinctrl-single,pins = <
+			AM65X_WKUP_IOPAD(0x0030, PIN_INPUT, 7) /* (R5) WKUP_GPIO0_24 */
+			AM65X_WKUP_IOPAD(0x003c, PIN_INPUT, 7) /* (P2) WKUP_GPIO0_27 */
+		>;
+	};
+
+	mcu_cpsw_pins_default: mcu_cpsw_pins_default {
+		pinctrl-single,pins = <
+			AM65X_WKUP_IOPAD(0x0058, PIN_OUTPUT, 0) /* (N4) MCU_RGMII1_TX_CTL */
+			AM65X_WKUP_IOPAD(0x005c, PIN_INPUT, 0) /* (N5) MCU_RGMII1_RX_CTL */
+			AM65X_WKUP_IOPAD(0x0060, PIN_OUTPUT, 0) /* (M2) MCU_RGMII1_TD3 */
+			AM65X_WKUP_IOPAD(0x0064, PIN_OUTPUT, 0) /* (M3) MCU_RGMII1_TD2 */
+			AM65X_WKUP_IOPAD(0x0068, PIN_OUTPUT, 0) /* (M4) MCU_RGMII1_TD1 */
+			AM65X_WKUP_IOPAD(0x006c, PIN_OUTPUT, 0) /* (M5) MCU_RGMII1_TD0 */
+			AM65X_WKUP_IOPAD(0x0078, PIN_INPUT, 0) /* (L2) MCU_RGMII1_RD3 */
+			AM65X_WKUP_IOPAD(0x007c, PIN_INPUT, 0) /* (L5) MCU_RGMII1_RD2 */
+			AM65X_WKUP_IOPAD(0x0080, PIN_INPUT, 0) /* (M6) MCU_RGMII1_RD1 */
+			AM65X_WKUP_IOPAD(0x0084, PIN_INPUT, 0) /* (L6) MCU_RGMII1_RD0 */
+			AM65X_WKUP_IOPAD(0x0070, PIN_INPUT, 0) /* (N1) MCU_RGMII1_TXC */
+			AM65X_WKUP_IOPAD(0x0074, PIN_INPUT, 0) /* (M1) MCU_RGMII1_RXC */
+		>;
+	};
+
+	mcu_mdio_pins_default: mcu_mdio1_pins_default {
+		pinctrl-single,pins = <
+			AM65X_WKUP_IOPAD(0x008c, PIN_OUTPUT, 0) /* (L1) MCU_MDIO0_MDC */
+			AM65X_WKUP_IOPAD(0x0088, PIN_INPUT, 0) /* (L4) MCU_MDIO0_MDIO */
+		>;
+	};
+
+	wkup_pca554_default: wkup_pca554_default {
+		pinctrl-single,pins = <
+			AM65X_WKUP_IOPAD(0x0034, PIN_INPUT, 7) /* (T1) MCU_OSPI1_CLK.WKUP_GPIO0_25 */
+
+		>;
+	};
+
+	mcu_fss0_ospi0_pins_default: mcu-fss0-ospi0-pins_default {
+		pinctrl-single,pins = <
+			AM65X_WKUP_IOPAD(0x0000, PIN_OUTPUT, 0) /* (V1) MCU_OSPI0_CLK */
+			AM65X_WKUP_IOPAD(0x0008, PIN_INPUT, 0)	 /* (U2) MCU_OSPI0_DQS */
+			AM65X_WKUP_IOPAD(0x000c, PIN_INPUT, 0)  /* (U4) MCU_OSPI0_D0 */
+			AM65X_WKUP_IOPAD(0x0010, PIN_INPUT, 0)  /* (U5) MCU_OSPI0_D1 */
+			AM65X_WKUP_IOPAD(0x0014, PIN_INPUT, 0)  /* (T2) MCU_OSPI0_D2 */
+			AM65X_WKUP_IOPAD(0x0018, PIN_INPUT, 0)  /* (T3) MCU_OSPI0_D3 */
+			AM65X_WKUP_IOPAD(0x001c, PIN_INPUT, 0)  /* (T4) MCU_OSPI0_D4 */
+			AM65X_WKUP_IOPAD(0x0020, PIN_INPUT, 0)  /* (T5) MCU_OSPI0_D5 */
+			AM65X_WKUP_IOPAD(0x0024, PIN_INPUT, 0)  /* (R2) MCU_OSPI0_D6 */
+			AM65X_WKUP_IOPAD(0x0028, PIN_INPUT, 0)  /* (R3) MCU_OSPI0_D7 */
+			AM65X_WKUP_IOPAD(0x002c, PIN_OUTPUT, 0) /* (R4) MCU_OSPI0_CSn0 */
+		>;
+	};
+};
+
+&main_pmx0 {
+	main_uart0_pins_default: main-uart0-pins-default {
+		pinctrl-single,pins = <
+			AM65X_IOPAD(0x01e4, PIN_INPUT, 0)	/* (AF11) UART0_RXD */
+			AM65X_IOPAD(0x01e8, PIN_OUTPUT, 0)	/* (AE11) UART0_TXD */
+			AM65X_IOPAD(0x01ec, PIN_INPUT, 0)	/* (AG11) UART0_CTSn */
+			AM65X_IOPAD(0x01f0, PIN_OUTPUT, 0)	/* (AD11) UART0_RTSn */
+		>;
+	};
+
+	main_i2c2_pins_default: main-i2c2-pins-default {
+		pinctrl-single,pins = <
+			AM65X_IOPAD(0x0074, PIN_INPUT, 5) /* (T27) GPMC0_CSn3.I2C2_SCL */
+			AM65X_IOPAD(0x0070, PIN_INPUT, 5) /* (R25) GPMC0_CSn2.I2C2_SDA */
+		>;
+	};
+
+	main_mmc0_pins_default: main-mmc0-pins-default {
+		pinctrl-single,pins = <
+			AM65X_IOPAD(0x01a8, PIN_INPUT_PULLDOWN, 0) /* (B25) MMC0_CLK */
+			AM65X_IOPAD(0x01ac, PIN_INPUT_PULLUP, 0) /* (B27) MMC0_CMD */
+			AM65X_IOPAD(0x01a4, PIN_INPUT_PULLUP, 0) /* (A26) MMC0_DAT0 */
+			AM65X_IOPAD(0x01a0, PIN_INPUT_PULLUP, 0) /* (E25) MMC0_DAT1 */
+			AM65X_IOPAD(0x019c, PIN_INPUT_PULLUP, 0) /* (C26) MMC0_DAT2 */
+			AM65X_IOPAD(0x0198, PIN_INPUT_PULLUP, 0) /* (A25) MMC0_DAT3 */
+			AM65X_IOPAD(0x0194, PIN_INPUT_PULLUP, 0) /* (E24) MMC0_DAT4 */
+			AM65X_IOPAD(0x0190, PIN_INPUT_PULLUP, 0) /* (A24) MMC0_DAT5 */
+			AM65X_IOPAD(0x018c, PIN_INPUT_PULLUP, 0) /* (B26) MMC0_DAT6 */
+			AM65X_IOPAD(0x0188, PIN_INPUT_PULLUP, 0) /* (D25) MMC0_DAT7 */
+			AM65X_IOPAD(0x01b4, PIN_INPUT_PULLUP, 0) /* (A23) MMC0_SDCD */
+			AM65X_IOPAD(0x01b0, PIN_INPUT, 0) /* (C25) MMC0_DS */
+		>;
+	};
+
+	main_mmc1_pins_default: main_mmc1_pins_default {
+		pinctrl-single,pins = <
+			AM65X_IOPAD(0x02d4, PIN_INPUT_PULLDOWN, 0) /* (C27) MMC1_CLK */
+			AM65X_IOPAD(0x02d8, PIN_INPUT_PULLUP, 0) /* (C28) MMC1_CMD */
+			AM65X_IOPAD(0x02d0, PIN_INPUT_PULLUP, 0) /* (D28) MMC1_DAT0 */
+			AM65X_IOPAD(0x02cc, PIN_INPUT_PULLUP, 0) /* (E27) MMC1_DAT1 */
+			AM65X_IOPAD(0x02c8, PIN_INPUT_PULLUP, 0) /* (D26) MMC1_DAT2 */
+			AM65X_IOPAD(0x02c4, PIN_INPUT_PULLUP, 0) /* (D27) MMC1_DAT3 */
+			AM65X_IOPAD(0x02dc, PIN_INPUT_PULLUP, 0) /* (B24) MMC1_SDCD */
+			AM65X_IOPAD(0x02e0, PIN_INPUT, 0) /* (C24) MMC1_SDWP */
+		>;
+	};
+
+	usb1_pins_default: usb1_pins_default {
+		pinctrl-single,pins = <
+			AM65X_IOPAD(0x02c0, PIN_OUTPUT, 0) /* (AC8) USB1_DRVVBUS */
+		>;
+	};
+
+	main_spi0_pins_default: main-spi0-pins-default {
+		pinctrl-single,pins = <
+			AM65X_IOPAD(0x01c4, PIN_INPUT, 0) /* (AH13) SPI0_CLK */
+			AM65X_IOPAD(0x01c8, PIN_INPUT, 0) /* (AE13) SPI0_D0 */
+			AM65X_IOPAD(0x01cc, PIN_INPUT, 0) /* (AD13) SPI0_D1 */
+			AM65X_IOPAD(0x01bc, PIN_OUTPUT, 0) /* (AG13) SPI0_CS0 */
+		>;
+	};
+
+	icssg2_mdio_pins_default: icssg2_mdio_pins_default {
+		pinctrl-single,pins = <
+			AM65X_IOPAD(0x0094, PIN_INPUT, 2) /* (AC19) PRG2_PRU0_GPO7.PRG2_MDIO0_MDIO */
+			AM65X_IOPAD(0x00c8, PIN_OUTPUT, 2) /* (AE15) PRG2_PRU1_GPO7.PRG2_MDIO0_MDC */
+		>;
+	};
+
+	icssg2_rgmii_pins_default: icssg2_rgmii_pins_default {
+		pinctrl-single,pins = <
+			AM65X_IOPAD(0x00ac, PIN_INPUT, 2) /* (AH15) PRG2_PRU1_GPO0.PRG2_RGMII2_RD0 */
+			AM65X_IOPAD(0x00b0, PIN_INPUT, 2) /* (AC16) PRG2_PRU1_GPO1.PRG2_RGMII2_RD1 */
+			AM65X_IOPAD(0x00b4, PIN_INPUT, 2) /* (AD17) PRG2_PRU1_GPO2.PRG2_RGMII2_RD2 */
+			AM65X_IOPAD(0x00b8, PIN_INPUT, 2) /* (AH14) PRG2_PRU1_GPO3.PRG2_RGMII2_RD3 */
+			AM65X_IOPAD(0x00cc, PIN_OUTPUT, 2) /* (AD15) PRG2_PRU1_GPO8.PRG2_RGMII2_TD0 */
+			AM65X_IOPAD(0x00d0, PIN_OUTPUT, 2) /* (AF14) PRG2_PRU1_GPO9.PRG2_RGMII2_TD1 */
+			AM65X_IOPAD(0x00d4, PIN_OUTPUT, 2) /* (AC15) PRG2_PRU1_GPO10.PRG2_RGMII2_TD2 */
+			AM65X_IOPAD(0x00d8, PIN_OUTPUT, 2) /* (AD14) PRG2_PRU1_GPO11.PRG2_RGMII2_TD3 */
+			AM65X_IOPAD(0x00dc, PIN_INPUT, 2) /* (AE14) PRG2_PRU1_GPO16.PRG2_RGMII2_TXC */
+			AM65X_IOPAD(0x00c4, PIN_OUTPUT, 2) /* (AC17) PRG2_PRU1_GPO6.PRG2_RGMII2_TX_CTL */
+			AM65X_IOPAD(0x00c0, PIN_INPUT, 2) /* (AG15) PRG2_PRU1_GPO5.PRG2_RGMII2_RXC */
+			AM65X_IOPAD(0x00bc, PIN_INPUT, 2) /* (AG14) PRG2_PRU1_GPO4.PRG2_RGMII2_RX_CTL */
+
+			AM65X_IOPAD(0x0078, PIN_INPUT, 2) /* (AF18) PRG2_PRU0_GPO0.PRG2_RGMII1_RD0 */
+			AM65X_IOPAD(0x007c, PIN_INPUT, 2) /* (AE18) PRG2_PRU0_GPO1.PRG2_RGMII1_RD1 */
+			AM65X_IOPAD(0x0080, PIN_INPUT, 2) /* (AH17) PRG2_PRU0_GPO2.PRG2_RGMII1_RD2 */
+			AM65X_IOPAD(0x0084, PIN_INPUT, 2) /* (AG18) PRG2_PRU0_GPO3.PRG2_RGMII1_RD3 */
+			AM65X_IOPAD(0x0098, PIN_OUTPUT, 2) /* (AH16) PRG2_PRU0_GPO8.PRG2_RGMII1_TD0 */
+			AM65X_IOPAD(0x009c, PIN_OUTPUT, 2) /* (AG16) PRG2_PRU0_GPO9.PRG2_RGMII1_TD1 */
+			AM65X_IOPAD(0x00a0, PIN_OUTPUT, 2) /* (AF16) PRG2_PRU0_GPO10.PRG2_RGMII1_TD2 */
+			AM65X_IOPAD(0x00a4, PIN_OUTPUT, 2) /* (AE16) PRG2_PRU0_GPO11.PRG2_RGMII1_TD3 */
+			AM65X_IOPAD(0x00a8, PIN_INPUT, 2) /* (AD16) PRG2_PRU0_GPO16.PRG2_RGMII1_TXC */
+			AM65X_IOPAD(0x0090, PIN_OUTPUT, 2) /* (AE17) PRG2_PRU0_GPO6.PRG2_RGMII1_TX_CTL */
+			AM65X_IOPAD(0x008c, PIN_INPUT, 2) /* (AF17) PRG2_PRU0_GPO5.PRG2_RGMII1_RXC */
+			AM65X_IOPAD(0x0088, PIN_INPUT, 2) /* (AG17) PRG2_PRU0_GPO4.PRG2_RGMII1_RX_CTL */
+		>;
+	};
+};
+
+&main_pmx1 {
+	main_i2c0_pins_default: main-i2c0-pins-default {
+		pinctrl-single,pins = <
+			AM65X_IOPAD(0x0000, PIN_INPUT, 0) /* (D20) I2C0_SCL */
+			AM65X_IOPAD(0x0004, PIN_INPUT, 0) /* (C21) I2C0_SDA */
+		>;
+	};
+
+	main_i2c1_pins_default: main-i2c1-pins-default {
+		pinctrl-single,pins = <
+			AM65X_IOPAD(0x0008, PIN_INPUT, 0) /* (B21) I2C1_SCL */
+			AM65X_IOPAD(0x000c, PIN_INPUT, 0) /* (E21) I2C1_SDA */
+		>;
+	};
+
+	ecap0_pins_default: ecap0-pins-default {
+		pinctrl-single,pins = <
+			AM65X_IOPAD(0x0010, PIN_INPUT, 0) /* (D21) ECAP0_IN_APWM_OUT */
+		>;
+	};
+};
+
+&wkup_uart0 {
+	/* Wakeup UART is used by System firmware */
+	status = "disabled";
+};
+
+&main_uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_uart0_pins_default>;
+};
+
+&wkup_i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&wkup_i2c0_pins_default>;
+	clock-frequency = <400000>;
+
+	pca9554: gpio@39 {
+		compatible = "nxp,pca9554";
+		reg = <0x39>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&wkup_pca554_default>;
+		interrupt-parent = <&wkup_gpio0>;
+		interrupts = <25 IRQ_TYPE_EDGE_FALLING>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+};
+
+&main_i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_i2c0_pins_default>;
+	clock-frequency = <400000>;
+
+	pca9555: gpio@21 {
+		compatible = "nxp,pca9555";
+		reg = <0x21>;
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
+};
+
+&main_i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_i2c1_pins_default>;
+	clock-frequency = <400000>;
+};
+
+&main_i2c2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_i2c2_pins_default>;
+	clock-frequency = <400000>;
+};
+
+&mcu_cpsw {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcu_cpsw_pins_default &mcu_mdio_pins_default>;
+
+	cpts {
+		ti,pps = <3 1>;
+	};
+};
+
+&davinci_mdio {
+	phy0: ethernet-phy@0 {
+		reg = <0>;
+		/* TODO: phy reset: TCA9555RTWR(i2c:0x21)[p04].GPIO_MCU_RGMII_RSTN */
+		ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
+		ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_4_B_NIB>;
+	};
+};
+
+&cpsw_port1 {
+	phy-mode = "rgmii-rxid";
+	phy-handle = <&phy0>;
+};
+
+&ecap0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&ecap0_pins_default>;
+};
+
+&sdhci0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_mmc0_pins_default>;
+	bus-width = <8>;
+	non-removable;
+	ti,driver-strength-ohm = <50>;
+	disable-wp;
+};
+
+&sdhci1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_mmc1_pins_default>;
+	ti,driver-strength-ohm = <50>;
+	disable-wp;
+};
+
+&gpu {
+	status = "okay";
+};
+
+&dwc3_1 {
+	status = "okay";
+};
+
+&usb1_phy {
+	status = "okay";
+};
+
+&usb1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&usb1_pins_default>;
+	dr_mode = "otg";
+};
+
+&dwc3_0 {
+	status = "disabled";
+};
+
+&usb0_phy {
+	status = "disabled";
+};
+
+&main_spi0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_spi0_pins_default>;
+	#address-cells = <1>;
+	#size-cells= <0>;
+	ti,pindir-d0-out-d1-in = <1>;
+
+	flash@0{
+		compatible = "jedec,spi-nor";
+		reg = <0x0>;
+		spi-tx-bus-width = <1>;
+		spi-rx-bus-width = <1>;
+		spi-max-frequency = <48000000>;
+		#address-cells = <1>;
+		#size-cells= <1>;
+	};
+};
+
+&tscadc0 {
+	adc {
+		ti,adc-channels = <0 1 2 3 4 5 6 7>;
+	};
+};
+
+&tscadc1 {
+	adc {
+		ti,adc-channels = <0 1 2 3 4 5 6 7>;
+	};
+};
+
+&mcu_r5fss0_core0 {
+	memory-region = <&mcu_r5fss0_core0_dma_memory_region>,
+			<&mcu_r5fss0_core0_memory_region>;
+};
+
+&mcu_r5fss0_core1 {
+	memory-region = <&mcu_r5fss0_core1_dma_memory_region>,
+			<&mcu_r5fss0_core1_memory_region>;
+};
+
+&ospi0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcu_fss0_ospi0_pins_default>;
+
+	flash@0{
+		compatible = "jedec,spi-nor";
+		reg = <0x0>;
+		spi-tx-bus-width = <8>;
+		spi-rx-bus-width = <8>;
+		spi-max-frequency = <50000000>;
+		spi-dqs;
+		cdns,tshsl-ns = <60>;
+		cdns,tsd2d-ns = <60>;
+		cdns,tchsh-ns = <60>;
+		cdns,tslch-ns = <60>;
+		cdns,read-delay = <2>;
+		cdns,phy-mode;
+		#address-cells = <1>;
+		#size-cells = <1>;
+	};
+};
+
+&icssg2_mdio {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&icssg2_mdio_pins_default>;
+
+	pruss2_eth0_phy: ethernet-phy@0 {
+		reg = <0>;
+		ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
+		ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_4_B_NIB>;
+	};
+
+	pruss2_eth1_phy: ethernet-phy@3 {
+		reg = <3>;
+		ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
+		ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_4_B_NIB>;
+	};
+};
+
+#define TS_OFFSET(pa, val)     (0x4+(pa)*4) (0x80000000 | val)
+
+&timesync_router {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcu_cpts>;
+
+	/* Example of the timesync routing */
+	mcu_cpts: mcu_cpts {
+		pinctrl-single,pins = <
+			/* pps [cpts genf1] in13 -> out25 [cpts hw4_push] */
+			TS_OFFSET(25, 13)
+		>;
 	};
 };
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/k3-am654-base-board-jailhouse.dtso b/arch/arm64/boot/dts/ti/k3-am654-base-board-jailhouse.dtso
--- a/arch/arm64/boot/dts/ti/k3-am654-base-board-jailhouse.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/k3-am654-base-board-jailhouse.dtso	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2016-2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+
+&reserved_memory {
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	/*
+	 * Reserve for Jailhouse hypervisor firmware and inmates.
+	 * Must be in sync with jailhouse cell configurations.
+	 */
+	hyp_mem: jailhouse@8dfb00000 {
+		reg = <0x8 0xdfb00000 0x0 0x20500000>;
+		alignment = <0x1000>;
+		no-map;
+	};
+};
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/k3-am654-evm-csi2-ov490.dtso b/arch/arm64/boot/dts/ti/k3-am654-evm-csi2-ov490.dtso
--- a/arch/arm64/boot/dts/ti/k3-am654-evm-csi2-ov490.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/k3-am654-evm-csi2-ov490.dtso	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+#include <dt-bindings/gpio/gpio.h>
+
+&main_i2c1 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	gpio_csi2_adap: tca6416@20 {
+		status = "okay";
+		compatible = "ti,tca6416";
+		reg = <0x20>;
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
+
+	ov490@24 {
+		compatible = "ovti,ov490";
+		reg = <0x24>;
+
+		mux-gpios = <&gpio_csi2_adap 0  GPIO_ACTIVE_LOW>,
+						/* CSI2_SEL_I2C_CLK */
+			    <&gpio_csi2_adap 1  GPIO_ACTIVE_HIGH>,
+						/* CSI2_SEL_REF_CLK */
+			    <&gpio_csi2_adap 3  GPIO_ACTIVE_HIGH>,
+						/* CSI2_CAM0_RESETn */
+			    <&gpio_csi2_adap 4  GPIO_ACTIVE_LOW>;
+						/* CSI2_CAM0_PWR_DWN */
+		port {
+			csi2_cam0: endpoint@0 {
+				clock-lanes = <0>;
+				data-lanes = <1 2 3 4>;
+				remote-endpoint = <&csi2_phy0>;
+			};
+		};
+	};
+};
+
+&cal {
+	status = "okay";
+};
+
+&csi2_0 {
+	csi2_phy0: endpoint@0 {
+		slave-mode;
+		remote-endpoint = <&csi2_cam0>;
+	};
+};
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/k3-am654-evm-csi2-ov5640.dtso b/arch/arm64/boot/dts/ti/k3-am654-evm-csi2-ov5640.dtso
--- a/arch/arm64/boot/dts/ti/k3-am654-evm-csi2-ov5640.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/k3-am654-evm-csi2-ov5640.dtso	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,55 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+#include <dt-bindings/gpio/gpio.h>
+
+/ {
+  fragment@101 {
+	target-path = "/";
+
+	__overlay__ {
+		clk_ov5640_fixed: clk_ov5640_fixed {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <24000000>;
+		};
+	};
+  };
+};
+
+&main_i2c1 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	ov5640@3C {
+		compatible = "ovti,ov5640";
+		reg = <0x3c>;
+
+		clocks = <&clk_ov5640_fixed>;
+		clock-names = "xclk";
+
+		port {
+			csi2_cam0: endpoint {
+				remote-endpoint = <&csi2_phy0>;
+				clock-lanes = <0>;
+				data-lanes = <1 2>;
+			};
+		};
+	};
+};
+
+&cal {
+	status = "okay";
+};
+
+&csi2_0 {
+	csi2_phy0: endpoint@0 {
+		remote-endpoint = <&csi2_cam0>;
+		clock-lanes = <0>;
+		data-lanes = <1 2>;
+	};
+};
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/k3-am654-evm-hdmi.dtso b/arch/arm64/boot/dts/ti/k3-am654-evm-hdmi.dtso
--- a/arch/arm64/boot/dts/ti/k3-am654-evm-hdmi.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/k3-am654-evm-hdmi.dtso	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,128 @@
+// SPDX-License-Identifier: GPL-2.0
+/**
+ * J6 PLUS HDMI DISPLAY ADAPTER for am654-evm (connected to J36)
+ * Note: the USB is not connected on am654-evm
+ *
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/pinctrl/k3.h>
+
+/ {
+  fragment@101 {
+	target-path = "/";
+
+	__overlay__ {
+		dvi-connector {
+			compatible = "hdmi-connector";
+			ddc-i2c-bus = <&main_i2c1>;
+			digital;
+
+			/* the gpio doesn't support interrupt */
+			hpd-gpios = <&pca9555 11 GPIO_ACTIVE_HIGH>;
+
+			port {
+				dvi_connector_in: endpoint {
+					remote-endpoint = <&tfp410_out>;
+				};
+			};
+		};
+
+		dvi-bridge {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "ti,tfp410";
+			ti,deskew = <3>;
+
+			port@0 {
+				reg = <0>;
+
+				tfp410_in: endpoint {
+					remote-endpoint = <&dpi_out0>;
+					pclk-sample = <1>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				tfp410_out: endpoint {
+					remote-endpoint = <&dvi_connector_in>;
+				};
+			};
+		};
+	};
+  };
+};
+
+&main_pmx0 {
+	dss_vout1_pins_default: dss_vout1_pins_default {
+		pinctrl-single,pins = <
+			AM65X_IOPAD(0x0000, PIN_OUTPUT, 1)	/* VOUT1_DATA0 */
+			AM65X_IOPAD(0x0004, PIN_OUTPUT, 1)	/* VOUT1_DATA1 */
+			AM65X_IOPAD(0x0008, PIN_OUTPUT, 1)	/* VOUT1_DATA2 */
+			AM65X_IOPAD(0x000c, PIN_OUTPUT, 1)	/* VOUT1_DATA3 */
+			AM65X_IOPAD(0x0010, PIN_OUTPUT, 1)	/* VOUT1_DATA4 */
+			AM65X_IOPAD(0x0014, PIN_OUTPUT, 1)	/* VOUT1_DATA5 */
+			AM65X_IOPAD(0x0018, PIN_OUTPUT, 1)	/* VOUT1_DATA6 */
+			AM65X_IOPAD(0x001c, PIN_OUTPUT, 1)	/* VOUT1_DATA7 */
+			AM65X_IOPAD(0x0020, PIN_OUTPUT, 1)	/* VOUT1_DATA8 */
+			AM65X_IOPAD(0x0024, PIN_OUTPUT, 1)	/* VOUT1_DATA9 */
+			AM65X_IOPAD(0x0028, PIN_OUTPUT, 1)	/* VOUT1_DATA10 */
+			AM65X_IOPAD(0x002c, PIN_OUTPUT, 1)	/* VOUT1_DATA11 */
+			AM65X_IOPAD(0x0030, PIN_OUTPUT, 1)	/* VOUT1_DATA12 */
+			AM65X_IOPAD(0x0034, PIN_OUTPUT, 1)	/* VOUT1_DATA13 */
+			AM65X_IOPAD(0x0038, PIN_OUTPUT, 1)	/* VOUT1_DATA14 */
+			AM65X_IOPAD(0x003c, PIN_OUTPUT, 1)	/* VOUT1_DATA15 */
+			AM65X_IOPAD(0x0040, PIN_OUTPUT, 1)	/* VOUT1_DATA16 */
+			AM65X_IOPAD(0x0044, PIN_OUTPUT, 1)	/* VOUT1_DATA17 */
+			AM65X_IOPAD(0x0048, PIN_OUTPUT, 1)	/* VOUT1_DATA18 */
+			AM65X_IOPAD(0x004c, PIN_OUTPUT, 1)	/* VOUT1_DATA19 */
+			AM65X_IOPAD(0x0050, PIN_OUTPUT, 1)	/* VOUT1_DATA20 */
+			AM65X_IOPAD(0x0054, PIN_OUTPUT, 1)	/* VOUT1_DATA21 */
+			AM65X_IOPAD(0x0058, PIN_OUTPUT, 1)	/* VOUT1_DATA22 */
+			AM65X_IOPAD(0x005c, PIN_OUTPUT, 1)	/* VOUT1_DATA23 */
+			AM65X_IOPAD(0x0060, PIN_OUTPUT, 1)	/* VOUT1_VSYNC */
+			AM65X_IOPAD(0x0064, PIN_OUTPUT, 1)	/* VOUT1_HSYNC */
+			AM65X_IOPAD(0x0068, PIN_OUTPUT, 1)	/* VOUT1_PCLK */
+			AM65X_IOPAD(0x006c, PIN_OUTPUT, 1)	/* VOUT1_DE */
+		>;
+	};
+};
+
+&pca9555 {
+	p6 {
+		/* P06 - CON_LCD_PWR_DN */
+		gpio-hog;
+		gpios = <6 GPIO_ACTIVE_HIGH>;
+		output-low;
+		line-name = "CON_LCD_PWR_DN";
+	};
+};
+
+&dss {
+	status = "ok";
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&dss_vout1_pins_default>;
+
+	assigned-clocks = <&k3_clks 67 2>;
+	assigned-clock-parents = <&k3_clks 67 5>;
+};
+
+&dss_ports {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	port@1 {
+		reg = <1>;
+
+		dpi_out0: endpoint {
+			remote-endpoint = <&tfp410_in>;
+		};
+	};
+};
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/k3-am654-evm-oldi-lcd1evm.dtso b/arch/arm64/boot/dts/ti/k3-am654-evm-oldi-lcd1evm.dtso
--- a/arch/arm64/boot/dts/ti/k3-am654-evm-oldi-lcd1evm.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/k3-am654-evm-oldi-lcd1evm.dtso	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,74 @@
+// SPDX-License-Identifier: GPL-2.0
+/**
+ * OLDI-LCD1EVM Rocktech integrated panel and touch DT overlay for AM654-EVM.
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/pwm/pwm.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+
+/ {
+  fragment@101 {
+	target-path = "/";
+
+	__overlay__ {
+		display0 {
+			compatible = "rockteck,rk101ii01d-ct";
+			backlight = <&lcd_bl>;
+			enable-gpios = <&pca9555 8 GPIO_ACTIVE_HIGH>;
+
+			port {
+				lcd_in0: endpoint {
+					remote-endpoint = <&oldi_out0>;
+				};
+			};
+		};
+
+		lcd_bl: backlight {
+			compatible = "pwm-backlight";
+			pwms = <&ecap0 0 50000 PWM_POLARITY_INVERTED>;
+			brightness-levels = <0 32 64 96 128 160 192 224 255>;
+			default-brightness-level = <8>;
+		};
+	};
+  };
+};
+
+&dss {
+	status = "ok";
+};
+
+&dss_ports {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	port@0 {
+		reg = <0>;
+
+		oldi_out0: endpoint {
+			remote-endpoint = <&lcd_in0>;
+		};
+	};
+};
+
+&main_i2c1 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	gt928: touchscreen@14 {
+		status = "okay";
+		compatible = "goodix,gt928";
+		reg = <0x14>;
+
+		interrupt-parent = <&pca9554>;
+		interrupts = <3 IRQ_TYPE_EDGE_FALLING>;
+		touchscreen-size-x = <1280>;
+		touchscreen-size-y = <800>;
+
+		reset-gpios = <&pca9555 9 GPIO_ACTIVE_HIGH>;
+		irq-gpios = <&pca9554 3 GPIO_ACTIVE_HIGH>;
+	};
+};
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/k3-am654-evm-prupwm.dtso b/arch/arm64/boot/dts/ti/k3-am654-evm-prupwm.dtso
--- a/arch/arm64/boot/dts/ti/k3-am654-evm-prupwm.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/k3-am654-evm-prupwm.dtso	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,143 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * DT overlay for PRU breakout personality card on AM654 EVM
+ *
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+#include <dt-bindings/pinctrl/k3.h>
+
+/ {
+  fragment@101 {
+	target-path = "/";
+
+	__overlay__ {
+		pru_pwm0 {
+			compatible = "ti,pru-pwm";
+			prus = <&pru0_0>;
+			firmware-name = "ti-pruss/am65x-pru0-pwm-fw.elf";
+			assigned-clocks = <&icssg0_iepclk_mux>;
+			assigned-clock-parents = <&k3_clks 62 10>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			pwm_chip@0 {
+				compatible = "ti,pru-pwmchip";
+				reg = <0>;
+				#pwm-cells = <2>;
+				pinctrl-0 = <&prupwm0_chip0_pins_default>;
+				pinctrl-names = "default";
+			};
+
+			pwm_chip@1 {
+				compatible = "ti,pru-pwmchip";
+				reg = <1>;
+				#pwm-cells = <2>;
+				pinctrl-0 = <&prupwm0_chip1_pins_default>;
+				pinctrl-names = "default";
+			};
+		};
+		pru_pwm1 {
+			compatible = "ti,pru-pwm";
+			prus = <&pru1_0>;
+			firmware-name = "ti-pruss/am65x-pru0-pwm-fw.elf";
+			assigned-clocks = <&icssg1_iepclk_mux>;
+			assigned-clock-parents = <&k3_clks 63 10>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			pwm_chip@0 {
+				compatible = "ti,pru-pwmchip";
+				reg = <0>;
+				#pwm-cells = <2>;
+				pinctrl-0 = <&prupwm1_chip0_pins_default>;
+				pinctrl-names = "default";
+			};
+
+			pwm_chip@1 {
+				compatible = "ti,pru-pwmchip";
+				reg = <1>;
+				#pwm-cells = <2>;
+				pinctrl-0 = <&prupwm1_chip1_pins_default>;
+				pinctrl-names = "default";
+			};
+		};
+	};
+  };
+};
+
+&main_pmx0 {
+	prupwm0_chip0_pins_default: prupwm0_chip0_pins_default {
+		pinctrl-single,pins = <
+			AM65X_IOPAD(0x224, PIN_OUTPUT, 3) /* (AD27) PRG0_PRU0_GPO12.PRG0_PWM0_A0, pwm0 */
+			AM65X_IOPAD(0x228, PIN_OUTPUT, 3) /* (AC26) PRG0_PRU0_GPO13.PRG0_PWM0_B0, pwm1 */
+			AM65X_IOPAD(0x22c, PIN_OUTPUT, 3) /* (AD26) PRG0_PRU0_GPO14.PRG0_PWM0_A1, pwm2 */
+			AM65X_IOPAD(0x230, PIN_OUTPUT, 3) /* (AA24) PRG0_PRU0_GPO15.PRG0_PWM0_B1, pwm3 */
+			AM65X_IOPAD(0x234, PIN_OUTPUT, 3) /* (AD28) PRG0_PRU0_GPO16.PRG0_PWM0_A2, pwm4 */
+			AM65X_IOPAD(0x238, PIN_OUTPUT, 3) /*  (U26) PRG0_PRU0_GPO17.PRG0_PWM0_B2, pwm5 */
+
+			AM65X_IOPAD(0x274, PIN_OUTPUT, 3) /* (AC25) PRG0_PRU1_GPO12.PRG0_PWM1_A0, pwm6 */
+			AM65X_IOPAD(0x278, PIN_OUTPUT, 3) /* (AD25) PRG0_PRU1_GPO13.PRG0_PWM1_B0, pwm7 */
+			AM65X_IOPAD(0x27c, PIN_OUTPUT, 3) /* (AD24) PRG0_PRU1_GPO14.PRG0_PWM1_A1, pwm8 */
+			AM65X_IOPAD(0x280, PIN_OUTPUT, 3) /* (AE27) PRG0_PRU1_GPO15.PRG0_PWM1_B1, pwm9 */
+			AM65X_IOPAD(0x284, PIN_OUTPUT, 3) /* (AC24) PRG0_PRU1_GPO16.PRG0_PWM1_A2, pwm10 */
+			AM65X_IOPAD(0x288, PIN_OUTPUT, 3) /*  (Y27) PRG0_PRU1_GPO17.PRG0_PWM1_B2, pwm11 */
+		>;
+	};
+
+	prupwm0_chip1_pins_default: prupwm0_chip1_pins_default {
+		pinctrl-single,pins = <
+			AM65X_IOPAD(0x1fc, PIN_OUTPUT, 3) /*  (W24) PRG0_PRU0_GPO2.PRG0_PWM2_A0, pwm0 */
+			AM65X_IOPAD(0x204, PIN_OUTPUT, 3) /*  (Y24) PRG0_PRU0_GPO4.PRG0_PWM2_B0, pwm1 */
+			AM65X_IOPAD(0x214, PIN_OUTPUT, 3) /*  (V27) PRG0_PRU0_GPO8.PRG0_PWM2_A1, pwm2 */
+			AM65X_IOPAD(0x21c, PIN_OUTPUT, 3) /*  (U25) PRG0_PRU0_GPO10.PRG0_PWM2_B1, pwm3 */
+			AM65X_IOPAD(0x24c, PIN_OUTPUT, 3) /* (AC27) PRG0_PRU1_GPO2.PRG0_PWM2_A2, pwm4 */
+			AM65X_IOPAD(0x254, PIN_OUTPUT, 3) /* (AA25) PRG0_PRU1_GPO4.PRG0_PWM2_B2, pwm5 */
+
+			AM65X_IOPAD(0x1f4, PIN_OUTPUT, 3) /*  (V24) PRG0_PRU0_GPO0.PRG0_PWM3_A0, pwm6 */
+			AM65X_IOPAD(0x1f8, PIN_OUTPUT, 3) /*  (W25) PRG0_PRU0_GPO1.PRG0_PWM3_B0, pwm7 */
+			AM65X_IOPAD(0x20c, PIN_OUTPUT, 3) /*  (Y25) PRG0_PRU0_GPO6.PRG0_PWM3_A1, pwm8 */
+			AM65X_IOPAD(0x210, PIN_OUTPUT, 3) /*  (U27) PRG0_PRU0_GPO7.PRG0_PWM3_B1, pwm9 */
+			AM65X_IOPAD(0x200, PIN_OUTPUT, 3) /* (AA27) PRG0_PRU0_GPO3.PRG0_PWM3_A2, pwm10 */
+			AM65X_IOPAD(0x208, PIN_OUTPUT, 3) /*  (V28) PRG0_PRU0_GPO5.PRG0_PWM3_B2, pwm11 */
+		>;
+	};
+
+	prupwm1_chip0_pins_default: prupwm1_chip0_pins_default {
+		pinctrl-single,pins = <
+			AM65X_IOPAD(0x110, PIN_OUTPUT, 3) /* (AH20) PRG1_PRU0_GPO12.PRG1_PWM0_A0, pwm0 */
+			AM65X_IOPAD(0x114, PIN_OUTPUT, 3) /* (AH21) PRG1_PRU0_GPO13.PRG1_PWM0_B0, pwm1 */
+			AM65X_IOPAD(0x118, PIN_OUTPUT, 3) /* (AG20) PRG1_PRU0_GPO14.PRG1_PWM0_A1, pwm2 */
+			AM65X_IOPAD(0x11c, PIN_OUTPUT, 3) /* (AD19) PRG1_PRU0_GPO15.PRG1_PWM0_B1, pwm3 */
+			AM65X_IOPAD(0x120, PIN_OUTPUT, 3) /* (AD20) PRG1_PRU0_GPO16.PRG1_PWM0_A2, pwm4 */
+			AM65X_IOPAD(0x124, PIN_OUTPUT, 3) /* (AH26) PRG1_PRU0_GPO17.PRG1_PWM0_B2, pwm5 */
+
+			AM65X_IOPAD(0x160, PIN_OUTPUT, 3) /* (AE20) PRG1_PRU1_GPO12.PRG1_PWM1_A0, pwm6 */
+			AM65X_IOPAD(0x164, PIN_OUTPUT, 3) /* (AF19) PRG1_PRU1_GPO13.PRG1_PWM1_B0, pwm7 */
+			AM65X_IOPAD(0x168, PIN_OUTPUT, 3) /* (AH19) PRG1_PRU1_GPO14.PRG1_PWM1_A1, pwm8 */
+			AM65X_IOPAD(0x16c, PIN_OUTPUT, 3) /* (AG19) PRG1_PRU1_GPO15.PRG1_PWM1_B1, pwm9 */
+			AM65X_IOPAD(0x170, PIN_OUTPUT, 3) /* (AE19) PRG1_PRU1_GPO16.PRG1_PWM1_A2, pwm10 */
+			AM65X_IOPAD(0x174, PIN_OUTPUT, 3) /* (AE23) PRG1_PRU1_GPO17.PRG1_PWM1_B2, pwm11 */
+		>;
+	};
+
+	prupwm1_chip1_pins_default: prupwm1_chip1_pins_default {
+		pinctrl-single,pins = <
+			AM65X_IOPAD(0x0e8, PIN_OUTPUT, 3) /* (AF23) PRG1_PRU0_GPO2.PRG1_PWM2_A0, pwm0 */
+			AM65X_IOPAD(0x0f0, PIN_OUTPUT, 3) /* (AG23) PRG1_PRU0_GPO4.PRG1_PWM2_B0, pwm1 */
+			AM65X_IOPAD(0x100, PIN_OUTPUT, 3) /* (AF28) PRG1_PRU0_GPO8.PRG1_PWM2_A1, pwm2 */
+			AM65X_IOPAD(0x108, PIN_OUTPUT, 3) /* (AH25) PRG1_PRU0_GPO10.PRG1_PWM2_B1, pwm3 */
+			AM65X_IOPAD(0x138, PIN_OUTPUT, 3) /* (AG21) PRG1_PRU1_GPO2.PRG1_PWM2_A2, pwm4 */
+			AM65X_IOPAD(0x140, PIN_OUTPUT, 3) /* (AE21) PRG1_PRU1_GPO4.PRG1_PWM2_B2, pwm5 */
+
+			AM65X_IOPAD(0x0e0, PIN_OUTPUT, 3) /* (AE22) PRG1_PRU0_GPO0.PRG1_PWM3_A0, pwm6 */
+			AM65X_IOPAD(0x0e4, PIN_OUTPUT, 3) /* (AG24) PRG1_PRU0_GPO1.PRG1_PWM3_B0, pwm7 */
+			AM65X_IOPAD(0x0f8, PIN_OUTPUT, 3) /* (AF22) PRG1_PRU0_GPO6.PRG1_PWM3_A1, pwm8 */
+			AM65X_IOPAD(0x0fc, PIN_OUTPUT, 3) /* (AG27) PRG1_PRU0_GPO7.PRG1_PWM3_B1, pwm9 */
+			AM65X_IOPAD(0x0ec, PIN_OUTPUT, 3) /* (AD21) PRG1_PRU0_GPO3.PRG1_PWM3_A2, pwm10 */
+			AM65X_IOPAD(0x0f4, PIN_OUTPUT, 3) /* (AF27) PRG1_PRU0_GPO5.PRG1_PWM3_B2, pwm11 */
+		>;
+	};
+};
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/k3-am654-evm-tc358876.dtso b/arch/arm64/boot/dts/ti/k3-am654-evm-tc358876.dtso
--- a/arch/arm64/boot/dts/ti/k3-am654-evm-tc358876.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/k3-am654-evm-tc358876.dtso	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,114 @@
+// SPDX-License-Identifier: GPL-2.0
+/**
+ * Toshiba TC358867 expansion board for AM654-EVM.
+ *
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/pinctrl/k3.h>
+
+/ {
+  fragment@101 {
+	target-path = "/";
+
+	__overlay__ {
+		dp_refclk: clock {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <19200000>;
+		};
+	};
+  };
+};
+
+&main_pmx0 {
+	dss_vout1_pins_default: dss_vout1_pins_default {
+		pinctrl-single,pins = <
+			AM65X_IOPAD(0x0000, PIN_OUTPUT, 1)	/* VOUT1_DATA0 */
+			AM65X_IOPAD(0x0004, PIN_OUTPUT, 1)	/* VOUT1_DATA1 */
+			AM65X_IOPAD(0x0008, PIN_OUTPUT, 1)	/* VOUT1_DATA2 */
+			AM65X_IOPAD(0x000c, PIN_OUTPUT, 1)	/* VOUT1_DATA3 */
+			AM65X_IOPAD(0x0010, PIN_OUTPUT, 1)	/* VOUT1_DATA4 */
+			AM65X_IOPAD(0x0014, PIN_OUTPUT, 1)	/* VOUT1_DATA5 */
+			AM65X_IOPAD(0x0018, PIN_OUTPUT, 1)	/* VOUT1_DATA6 */
+			AM65X_IOPAD(0x001c, PIN_OUTPUT, 1)	/* VOUT1_DATA7 */
+			AM65X_IOPAD(0x0020, PIN_OUTPUT, 1)	/* VOUT1_DATA8 */
+			AM65X_IOPAD(0x0024, PIN_OUTPUT, 1)	/* VOUT1_DATA9 */
+			AM65X_IOPAD(0x0028, PIN_OUTPUT, 1)	/* VOUT1_DATA10 */
+			AM65X_IOPAD(0x002c, PIN_OUTPUT, 1)	/* VOUT1_DATA11 */
+			AM65X_IOPAD(0x0030, PIN_OUTPUT, 1)	/* VOUT1_DATA12 */
+			AM65X_IOPAD(0x0034, PIN_OUTPUT, 1)	/* VOUT1_DATA13 */
+			AM65X_IOPAD(0x0038, PIN_OUTPUT, 1)	/* VOUT1_DATA14 */
+			AM65X_IOPAD(0x003c, PIN_OUTPUT, 1)	/* VOUT1_DATA15 */
+			AM65X_IOPAD(0x0040, PIN_OUTPUT, 1)	/* VOUT1_DATA16 */
+			AM65X_IOPAD(0x0044, PIN_OUTPUT, 1)	/* VOUT1_DATA17 */
+			AM65X_IOPAD(0x0048, PIN_OUTPUT, 1)	/* VOUT1_DATA18 */
+			AM65X_IOPAD(0x004c, PIN_OUTPUT, 1)	/* VOUT1_DATA19 */
+			AM65X_IOPAD(0x0050, PIN_OUTPUT, 1)	/* VOUT1_DATA20 */
+			AM65X_IOPAD(0x0054, PIN_OUTPUT, 1)	/* VOUT1_DATA21 */
+			AM65X_IOPAD(0x0058, PIN_OUTPUT, 1)	/* VOUT1_DATA22 */
+			AM65X_IOPAD(0x005c, PIN_OUTPUT, 1)	/* VOUT1_DATA23 */
+			AM65X_IOPAD(0x0060, PIN_OUTPUT, 1)	/* VOUT1_VSYNC */
+			AM65X_IOPAD(0x0064, PIN_OUTPUT, 1)	/* VOUT1_HSYNC */
+			AM65X_IOPAD(0x0068, PIN_OUTPUT, 1)	/* VOUT1_PCLK */
+			AM65X_IOPAD(0x006c, PIN_OUTPUT, 1)	/* VOUT1_DE */
+		>;
+	};
+};
+
+&main_i2c1 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	edp-bridge@f {
+		compatible = "toshiba,tc358867", "toshiba,tc358767";
+		reg = <0x0f>;
+
+		reset-gpios = <&pca9555 6 GPIO_ACTIVE_HIGH>;
+
+		clock-names = "ref";
+		clocks = <&dp_refclk>;
+
+		toshiba,hpd-pin = <0>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@1 {
+				reg = <1>;
+
+				bridge_in: endpoint {
+					remote-endpoint = <&dpi_out>;
+				};
+			};
+		};
+	};
+};
+
+&dss {
+	status = "ok";
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&dss_vout1_pins_default>;
+
+	assigned-clocks = <&k3_clks 67 2>;
+	assigned-clock-parents = <&k3_clks 67 5>;
+};
+
+&dss_ports {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	port@1 {
+		reg = <1>;
+
+		dpi_out: endpoint {
+			remote-endpoint = <&bridge_in>;
+		};
+	};
+};
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/k3-am654-gp.dtso b/arch/arm64/boot/dts/ti/k3-am654-gp.dtso
--- a/arch/arm64/boot/dts/ti/k3-am654-gp.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/k3-am654-gp.dtso	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,133 @@
+// SPDX-License-Identifier: GPL-2.0
+/**
+ * DT overlay for GP application board on AM654 EVM
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+#include <dt-bindings/pinctrl/k3.h>
+#include <dt-bindings/gpio/gpio.h>
+
+/ {
+  fragment@101 {
+	target-path = "/";
+
+	__overlay__ {
+		gp_vcc_5v0: fixedregulator-gp_vcc_5v0 {
+			compatible = "regulator-fixed";
+			regulator-name = "gp_vcc_5v0";
+			regulator-min-microvolt = <5000000>;
+			regulator-max-microvolt = <5000000>;
+			regulator-always-on;
+			regulator-boot-on;
+		};
+
+		codec_vcc_3v3: fixedregulator-codec_vcc_3v3 {
+			/* LP5912-3.3DRVT */
+			compatible = "regulator-fixed";
+			regulator-name = "codec_vcc_3v3";
+			regulator-min-microvolt = <3300000>;
+			regulator-max-microvolt = <3300000>;
+			vin-supply = <&gp_vcc_5v0>;
+			regulator-always-on;
+			regulator-boot-on;
+		};
+
+		gp_vcc_1v8: fixedregulator-gp_vcc_1v8 {
+			compatible = "regulator-fixed";
+			regulator-name = "gp_vcc_1v8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-always-on;
+			regulator-boot-on;
+		};
+
+		sound {
+			compatible = "simple-audio-card";
+			simple-audio-card,name = "AM65x-GPEVM";
+			simple-audio-card,widgets =
+				"Headphone", "Headphone Jack",
+				"Line", "Line In";
+			simple-audio-card,routing =
+				"Headphone Jack",	"HPLOUT",
+				"Headphone Jack",	"HPROUT",
+				"LINE1L",		"Line In",
+				"LINE1R",		"Line In";
+			simple-audio-card,format = "dsp_b";
+			simple-audio-card,bitclock-master = <&sound_master>;
+			simple-audio-card,frame-master = <&sound_master>;
+			simple-audio-card,bitclock-inversion;
+
+			simple-audio-card,cpu {
+				sound-dai = <&mcasp0>;
+			};
+
+			sound_master: simple-audio-card,codec {
+				sound-dai = <&tlv320aic3106>;
+				system-clock-frequency = <12000000>;
+			};
+		};
+	};
+  };
+};
+
+&main_pmx0 {
+	mcasp0_pins: mcasp0_pins {
+		pinctrl-single,pins = <
+			AM65X_IOPAD(0x01f4, PIN_INPUT, 5) /* (V24) PRG0_PRU0_GPO0.MCASP0_ACLKX */
+			AM65X_IOPAD(0x01f8, PIN_INPUT, 5) /* (W25) PRG0_PRU0_GPO1.MCASP0_AFSX */
+			AM65X_IOPAD(0x0204, PIN_OUTPUT, 5) /* (Y24) PRG0_PRU0_GPO4.MCASP0_AXR0 */
+			AM65X_IOPAD(0x0208, PIN_INPUT, 5) /* (V28) PRG0_PRU0_GPO5.MCASP0_AXR1 */
+		>;
+	};
+
+	aic3106_pins: aic3106_pins {
+		pinctrl-single,pins = <
+			AM65X_IOPAD(0x011c, PIN_OUTPUT, 7) /* (AD19) PRG1_PRU0_GPO15.GPIO0_71 */
+		>;
+	};
+};
+
+&main_i2c0 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	tlv320aic3106: tlv320aic3106@1b {
+		#sound-dai-cells = <0>;
+		compatible = "ti,tlv320aic3106";
+		reg = <0x1b>;
+		status = "okay";
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&aic3106_pins>;
+
+		gpio-reset = <&main_gpio0 71 GPIO_ACTIVE_LOW>; /* gpio0_71 */
+
+		/* Regulators */
+		AVDD-supply = <&codec_vcc_3v3>;
+		IOVDD-supply = <&gp_vcc_1v8>;
+		DRVDD-supply = <&codec_vcc_3v3>;
+		DVDD-supply = <&gp_vcc_1v8>;
+	};
+};
+
+&mcasp0 {
+	#sound-dai-cells = <0>;
+	status = "okay";
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcasp0_pins>;
+
+	op-mode = <0>;          /* MCASP_IIS_MODE */
+	tdm-slots = <2>;
+	/* 16 serializers */
+	serial-dir = <  /* 0: INACTIVE, 1: TX, 2: RX */
+		1 2 0 0
+		0 0 0 0
+		0 0 0 0
+		0 0 0 0
+	>;
+	tx-num-evt = <32>;
+	rx-num-evt = <32>;
+};
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/k3-am654-idk.dtso b/arch/arm64/boot/dts/ti/k3-am654-idk.dtso
--- a/arch/arm64/boot/dts/ti/k3-am654-idk.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/k3-am654-idk.dtso	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,324 @@
+// SPDX-License-Identifier: GPL-2.0
+/**
+ * DT overlay for IDK application board on AM654 EVM
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+#include <dt-bindings/dma/k3-udma.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/net/ti-dp83867.h>
+#include <dt-bindings/phy/phy.h>
+#include <dt-bindings/pinctrl/k3.h>
+
+/ {
+   fragment@101 {
+	target-path = "/";
+
+	__overlay__ {
+		aliases {
+			ethernet3 = "/pruss0_eth/ethernet-mii0";
+			ethernet4 = "/pruss0_eth/ethernet-mii1";
+			ethernet5 = "/pruss1_eth/ethernet-mii0";
+			ethernet6 = "/pruss1_eth/ethernet-mii1";
+		};
+
+		/* Dual Ethernet application node on PRU-ICSSG0 */
+		pruss0_eth {
+			compatible = "ti,am654-icssg-prueth";
+			pinctrl-names = "default";
+			pinctrl-0 = <&icssg0_rgmii_pins_default>;
+			sram = <&msmc_ram>;
+			interrupt-parent = <&main_udmass_inta>;
+
+			prus = <&pru0_0>, <&rtu0_0>, <&pru0_1>, <&rtu0_1>;
+			firmware-name = "ti-pruss/am65x-pru0-prueth-fw.elf",
+					"ti-pruss/am65x-rtu0-prueth-fw.elf",
+					"ti-pruss/am65x-pru1-prueth-fw.elf",
+					"ti-pruss/am65x-rtu1-prueth-fw.elf";
+			mii-g-rt = <&icssg0_mii_g_rt>;
+			mii-rt = <&icssg0_mii_rt>;
+			dma-coherent;
+			dmas = <&main_udmap &icssg0 0 UDMA_DIR_TX>, /* egress slice 0 */
+			       <&main_udmap &icssg0 1 UDMA_DIR_TX>, /* egress slice 0 */
+			       <&main_udmap &icssg0 2 UDMA_DIR_TX>, /* egress slice 0 */
+			       <&main_udmap &icssg0 3 UDMA_DIR_TX>, /* egress slice 0 */
+			       <&main_udmap &icssg0 4 UDMA_DIR_TX>, /* egress slice 1 */
+			       <&main_udmap &icssg0 5 UDMA_DIR_TX>, /* egress slice 1 */
+			       <&main_udmap &icssg0 6 UDMA_DIR_TX>, /* egress slice 1 */
+			       <&main_udmap &icssg0 7 UDMA_DIR_TX>, /* egress slice 1 */
+
+			       <&main_udmap &icssg0 0 UDMA_DIR_RX>, /* ingress slice 0 */
+			       <&main_udmap &icssg0 1 UDMA_DIR_RX>, /* ingress slice 1 */
+			       <&main_udmap &icssg0 2 UDMA_DIR_RX>, /* mgmnt rsp slice 0 */
+			       <&main_udmap &icssg0 3 UDMA_DIR_RX>; /* mgmnt rsp slice 1 */
+			dma-names = "tx0-0", "tx0-1", "tx0-2", "tx0-3",
+				    "tx1-0", "tx1-1", "tx1-2", "tx1-3",
+				    "rx0", "rx1",
+				    "rxmgm0", "rxmgm1";
+
+			pruss0_emac0: ethernet-mii0 {
+				phy-handle = <&pruss0_eth0_phy>;
+				phy-mode = "rgmii-rxid";
+				syscon-rgmii-delay = <&scm_conf 0x4100>;
+				iep = <&icssg0_iep0>;
+				/* Filled in by bootloader */
+				local-mac-address = [00 00 00 00 00 00];
+			};
+
+			pruss0_emac1: ethernet-mii1 {
+				phy-handle = <&pruss0_eth1_phy>;
+				phy-mode = "rgmii-rxid";
+				syscon-rgmii-delay = <&scm_conf 0x4104>;
+				iep = <&icssg0_iep1>;
+				/* Filled in by bootloader */
+				local-mac-address = [00 00 00 00 00 00];
+			};
+		};
+
+		/* Dual Ethernet application node on PRU-ICSSG1 */
+		pruss1_eth {
+			compatible = "ti,am654-icssg-prueth";
+			pinctrl-names = "default";
+			pinctrl-0 = <&icssg1_rgmii_pins_default>;
+			sram = <&msmc_ram>;
+			interrupt-parent = <&main_udmass_inta>;
+
+			prus = <&pru1_0>, <&rtu1_0>, <&pru1_1>, <&rtu1_1>;
+			firmware-name = "ti-pruss/am65x-pru0-prueth-fw.elf",
+					"ti-pruss/am65x-rtu0-prueth-fw.elf",
+					"ti-pruss/am65x-pru1-prueth-fw.elf",
+					"ti-pruss/am65x-rtu1-prueth-fw.elf";
+			mii-g-rt = <&icssg1_mii_g_rt>;
+			mii-rt = <&icssg1_mii_rt>;
+			dma-coherent;
+			dmas = <&main_udmap &icssg1 0 UDMA_DIR_TX>, /* egress slice 0 */
+			       <&main_udmap &icssg1 1 UDMA_DIR_TX>, /* egress slice 0 */
+			       <&main_udmap &icssg1 2 UDMA_DIR_TX>, /* egress slice 0 */
+			       <&main_udmap &icssg1 3 UDMA_DIR_TX>, /* egress slice 0 */
+			       <&main_udmap &icssg1 4 UDMA_DIR_TX>, /* egress slice 1 */
+			       <&main_udmap &icssg1 5 UDMA_DIR_TX>, /* egress slice 1 */
+			       <&main_udmap &icssg1 6 UDMA_DIR_TX>, /* egress slice 1 */
+			       <&main_udmap &icssg1 7 UDMA_DIR_TX>, /* egress slice 1 */
+
+			       <&main_udmap &icssg1 0 UDMA_DIR_RX>, /* ingress slice 0 */
+			       <&main_udmap &icssg1 1 UDMA_DIR_RX>, /* ingress slice 1 */
+			       <&main_udmap &icssg1 2 UDMA_DIR_RX>, /* mgmnt rsp slice 0 */
+			       <&main_udmap &icssg1 3 UDMA_DIR_RX>; /* mgmnt rsp slice 1 */
+			dma-names = "tx0-0", "tx0-1", "tx0-2", "tx0-3",
+				    "tx1-0", "tx1-1", "tx1-2", "tx1-3",
+				    "rx0", "rx1",
+				    "rxmgm0", "rxmgm1";
+
+			pruss1_emac0: ethernet-mii0 {
+				phy-handle = <&pruss1_eth0_phy>;
+				phy-mode = "rgmii-rxid";
+				syscon-rgmii-delay = <&scm_conf 0x4110>;
+				iep = <&icssg1_iep0>;
+				/* Filled in by bootloader */
+				local-mac-address = [00 00 00 00 00 00];
+			};
+
+			pruss1_emac1: ethernet-mii1 {
+				phy-handle = <&pruss1_eth1_phy>;
+				phy-mode = "rgmii-rxid";
+				syscon-rgmii-delay = <&scm_conf 0x4114>;
+				iep = <&icssg1_iep1>;
+				/* Filled in by bootloader */
+				local-mac-address = [00 00 00 00 00 00];
+			};
+		};
+
+		gpio-decoder {
+			compatible = "gpio-decoder";
+			gpios = <&pca9536 3 GPIO_ACTIVE_HIGH>,
+				<&pca9536 2 GPIO_ACTIVE_HIGH>,
+				<&pca9536 1 GPIO_ACTIVE_HIGH>,
+				<&pca9536 0 GPIO_ACTIVE_HIGH>;
+			linux,axis = <0>; /* ABS_X */
+			decoder-max-value = <9>;
+		};
+	};
+  };
+};
+
+&main_pmx0 {
+	icssg0_mdio_pins_default: icssg0_mdio_pins_default {
+		pinctrl-single,pins = <
+			AM65X_IOPAD(0x0294, PIN_INPUT, 0) /* (AE26) PRG0_MDIO0_MDIO */
+			AM65X_IOPAD(0x0298, PIN_OUTPUT, 0) /* (AE28) PRG0_MDIO0_MDC */
+		>;
+	};
+
+	icssg0_rgmii_pins_default: icssg0_rgmii_pins_default {
+		pinctrl-single,pins = <
+			AM65X_IOPAD(0x0244, PIN_INPUT, 2) /* (AB28) PRG0_PRU1_GPO0.PRG0_RGMII2_RD0 */
+			AM65X_IOPAD(0x0248, PIN_INPUT, 2) /* (AC28) PRG0_PRU1_GPO1.PRG0_RGMII2_RD1 */
+			AM65X_IOPAD(0x024c, PIN_INPUT, 2) /* (AC27) PRG0_PRU1_GPO2.PRG0_RGMII2_RD2 */
+			AM65X_IOPAD(0x0250, PIN_INPUT, 2) /* (AB26) PRG0_PRU1_GPO3.PRG0_RGMII2_RD3 */
+			AM65X_IOPAD(0x0274, PIN_OUTPUT, 2) /* (AC25) PRG0_PRU1_GPO12.PRG0_RGMII2_TD0 */
+			AM65X_IOPAD(0x0278, PIN_OUTPUT, 2) /* (AD25) PRG0_PRU1_GPO13.PRG0_RGMII2_TD1 */
+			AM65X_IOPAD(0x027c, PIN_OUTPUT, 2) /* (AD24) PRG0_PRU1_GPO14.PRG0_RGMII2_TD2 */
+			AM65X_IOPAD(0x0280, PIN_OUTPUT, 2) /* (AE27) PRG0_PRU1_GPO15.PRG0_RGMII2_TD3 */
+			AM65X_IOPAD(0x0284, PIN_INPUT, 2) /* (AC24) PRG0_PRU1_GPO16.PRG0_RGMII2_TXC */
+			AM65X_IOPAD(0x0270, PIN_OUTPUT, 2) /* (AB24) PRG0_PRU1_GPO11.PRG0_RGMII2_TX_CTL */
+			AM65X_IOPAD(0x025c, PIN_INPUT, 2) /* (AB27) PRG0_PRU1_GPO6.PRG0_RGMII2_RXC */
+			AM65X_IOPAD(0x0254, PIN_INPUT, 2) /* (AA25) PRG0_PRU1_GPO4.PRG0_RGMII2_RX_CTL */
+
+			AM65X_IOPAD(0x01f4, PIN_INPUT, 2) /* (V24) PRG0_PRU0_GPO0.PRG0_RGMII1_RD0 */
+			AM65X_IOPAD(0x01f8, PIN_INPUT, 2) /* (W25) PRG0_PRU0_GPO1.PRG0_RGMII1_RD1 */
+			AM65X_IOPAD(0x01fc, PIN_INPUT, 2) /* (W24) PRG0_PRU0_GPO2.PRG0_RGMII1_RD2 */
+			AM65X_IOPAD(0x0200, PIN_INPUT, 2) /* (AA27) PRG0_PRU0_GPO3.PRG0_RGMII1_RD3 */
+			AM65X_IOPAD(0x0224, PIN_OUTPUT, 2) /* (AD27) PRG0_PRU0_GPO12.PRG0_RGMII1_TD0 */
+			AM65X_IOPAD(0x0228, PIN_OUTPUT, 2) /* (AC26) PRG0_PRU0_GPO13.PRG0_RGMII1_TD1 */
+			AM65X_IOPAD(0x022c, PIN_OUTPUT, 2) /* (AD26) PRG0_PRU0_GPO14.PRG0_RGMII1_TD2 */
+			AM65X_IOPAD(0x0230, PIN_OUTPUT, 2) /* (AA24) PRG0_PRU0_GPO15.PRG0_RGMII1_TD3 */
+			AM65X_IOPAD(0x0234, PIN_INPUT, 2) /* (AD28) PRG0_PRU0_GPO16.PRG0_RGMII1_TXC */
+			AM65X_IOPAD(0x0220, PIN_OUTPUT, 2) /* (AB25) PRG0_PRU0_GPO11.PRG0_RGMII1_TX_CTL */
+			AM65X_IOPAD(0x020c, PIN_INPUT, 2) /* (Y25) PRG0_PRU0_GPO6.PRG0_RGMII1_RXC */
+			AM65X_IOPAD(0x0204, PIN_INPUT, 2) /* (Y24) PRG0_PRU0_GPO4.PRG0_RGMII1_RX_CTL */
+		>;
+	};
+
+	icssg1_mdio_pins_default: icssg1_mdio_pins_default {
+		pinctrl-single,pins = <
+			AM65X_IOPAD(0x0180, PIN_INPUT, 0) /* (AD18) PRG1_MDIO0_MDIO */
+			AM65X_IOPAD(0x0184, PIN_OUTPUT, 0) /* (AH18) PRG1_MDIO0_MDC */
+		>;
+	};
+
+	icssg1_rgmii_pins_default: icssg1_rgmii_pins_default {
+		pinctrl-single,pins = <
+			AM65X_IOPAD(0x0130, PIN_INPUT, 2) /* (AH24) PRG1_PRU1_GPO0.PRG1_RGMII2_RD0 */
+			AM65X_IOPAD(0x0134, PIN_INPUT, 2) /* (AH23) PRG1_PRU1_GPO1.PRG1_RGMII2_RD1 */
+			AM65X_IOPAD(0x0138, PIN_INPUT, 2) /* (AG21) PRG1_PRU1_GPO2.PRG1_RGMII2_RD2 */
+			AM65X_IOPAD(0x013c, PIN_INPUT, 2) /* (AH22) PRG1_PRU1_GPO3.PRG1_RGMII2_RD3 */
+			AM65X_IOPAD(0x0160, PIN_OUTPUT, 2) /* (AE20) PRG1_PRU1_GPO12.PRG1_RGMII2_TD0 */
+			AM65X_IOPAD(0x0164, PIN_OUTPUT, 2) /* (AF19) PRG1_PRU1_GPO13.PRG1_RGMII2_TD1 */
+			AM65X_IOPAD(0x0168, PIN_OUTPUT, 2) /* (AH19) PRG1_PRU1_GPO14.PRG1_RGMII2_TD2 */
+			AM65X_IOPAD(0x016c, PIN_OUTPUT, 2) /* (AG19) PRG1_PRU1_GPO15.PRG1_RGMII2_TD3 */
+			AM65X_IOPAD(0x0170, PIN_INPUT, 2) /* (AE19) PRG1_PRU1_GPO16.PRG1_RGMII2_TXC */
+			AM65X_IOPAD(0x015c, PIN_OUTPUT, 2) /* (AC20) PRG1_PRU1_GPO11.PRG1_RGMII2_TX_CTL */
+			AM65X_IOPAD(0x0148, PIN_INPUT, 2) /* (AG22) PRG1_PRU1_GPO6.PRG1_RGMII2_RXC */
+			AM65X_IOPAD(0x0140, PIN_INPUT, 2) /* (AE21) PRG1_PRU1_GPO4.PRG1_RGMII2_RX_CTL */
+
+			AM65X_IOPAD(0x00e0, PIN_INPUT, 2) /* (AE22) PRG1_PRU0_GPO0.PRG1_RGMII1_RD0 */
+			AM65X_IOPAD(0x00e4, PIN_INPUT, 2) /* (AG24) PRG1_PRU0_GPO1.PRG1_RGMII1_RD1 */
+			AM65X_IOPAD(0x00e8, PIN_INPUT, 2) /* (AF23) PRG1_PRU0_GPO2.PRG1_RGMII1_RD2 */
+			AM65X_IOPAD(0x00ec, PIN_INPUT, 2) /* (AD21) PRG1_PRU0_GPO3.PRG1_RGMII1_RD3 */
+			AM65X_IOPAD(0x0110, PIN_OUTPUT, 2) /* (AH20) PRG1_PRU0_GPO12.PRG1_RGMII1_TD0 */
+			AM65X_IOPAD(0x0114, PIN_OUTPUT, 2) /* (AH21) PRG1_PRU0_GPO13.PRG1_RGMII1_TD1 */
+			AM65X_IOPAD(0x0118, PIN_OUTPUT, 2) /* (AG20) PRG1_PRU0_GPO14.PRG1_RGMII1_TD2 */
+			AM65X_IOPAD(0x011c, PIN_OUTPUT, 2) /* (AD19) PRG1_PRU0_GPO15.PRG1_RGMII1_TD3 */
+			AM65X_IOPAD(0x0120, PIN_INPUT, 2) /* (AD20) PRG1_PRU0_GPO16.PRG1_RGMII1_TXC */
+			AM65X_IOPAD(0x010c, PIN_OUTPUT, 2) /* (AF21) PRG1_PRU0_GPO11.PRG1_RGMII1_TX_CTL */
+			AM65X_IOPAD(0x00f8, PIN_INPUT, 2) /* (AF22) PRG1_PRU0_GPO6.PRG1_RGMII1_RXC */
+			AM65X_IOPAD(0x00f0, PIN_INPUT, 2) /* (AG23) PRG1_PRU0_GPO4.PRG1_RGMII1_RX_CTL */
+		>;
+	};
+
+	mcan0_gpio_pins_default: mcan0_gpio_pins_default {
+		pinctrl-single,pins = <
+			AM65X_IOPAD(0x023c, PIN_INPUT, 7) /* (V25) PRG0_PRU0_GPIO18:GPIO1_47 */
+		>;
+	};
+
+	mcan1_gpio_pins_default: mcan1_gpio_pins_default {
+		pinctrl-single,pins = <
+			AM65X_IOPAD(0x028c, PIN_INPUT, 7) /* (Y26) PRG0_PRU1_GPIO18.GPIO1_67 */
+		>;
+	};
+};
+
+&wkup_pmx0 {
+
+	mcu_mcan0_pins_default: mcu_mcan0_pins_default {
+		pinctrl-single,pins = <
+			AM65X_WKUP_IOPAD(0x00ac, PIN_INPUT_PULLUP, 0) /* (W2) MCU_MCAN0_RX */
+			AM65X_WKUP_IOPAD(0x00a8, PIN_OUTPUT_PULLUP, 0) /* (W1) MCU_MCAN0_TX */
+		>;
+	};
+
+	mcu_mcan1_pins_default: mcu_mcan1_pins_default {
+		pinctrl-single,pins = <
+			AM65X_WKUP_IOPAD(0x00c4, PIN_INPUT_PULLUP, 1) /* (AD3) WKUP_GPIO0_5.MCU_MCAN1_RX */
+			AM65X_WKUP_IOPAD(0x00c0, PIN_OUTPUT_PULLUP, 1) /* (AC3) WKUP_GPIO0_4.MCU_MCAN1_TX */
+		>;
+	};
+};
+
+&icssg0_mdio {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&icssg0_mdio_pins_default>;
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	pruss0_eth0_phy: ethernet-phy@0 {
+		reg = <0>;
+		ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
+		ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_4_B_NIB>;
+	};
+
+	pruss0_eth1_phy: ethernet-phy@3 {
+		reg = <3>;
+		ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
+		ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_4_B_NIB>;
+	};
+};
+
+&icssg1_mdio {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&icssg1_mdio_pins_default>;
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	pruss1_eth0_phy: ethernet-phy@0 {
+		reg = <0>;
+		ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
+		ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_4_B_NIB>;
+	};
+
+	pruss1_eth1_phy: ethernet-phy@3 {
+		reg = <3>;
+		ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
+		ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_4_B_NIB>;
+	};
+};
+
+&m_can0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcu_mcan0_pins_default &mcan0_gpio_pins_default>;
+	stb-gpios = <&main_gpio1 47 GPIO_ACTIVE_HIGH>;
+	can-transceiver {
+		max-bitrate = <5000000>;
+	};
+};
+
+&m_can1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcu_mcan1_pins_default &mcan1_gpio_pins_default>;
+	stb-gpios = <&main_gpio1 67 GPIO_ACTIVE_HIGH>;
+	can-transceiver {
+		max-bitrate = <5000000>;
+	};
+};
+
+&main_i2c0 {
+	status = "okay";
+	clock-frequency = <400000>;
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	pca9536: gpio@41 {
+		compatible = "ti,pca9536";
+		reg = <0x41>;
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
+};
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/k3-am654-pcie-usb2.dtso b/arch/arm64/boot/dts/ti/k3-am654-pcie-usb2.dtso
--- a/arch/arm64/boot/dts/ti/k3-am654-pcie-usb2.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/k3-am654-pcie-usb2.dtso	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,55 @@
+// SPDX-License-Identifier: GPL-2.0
+/**
+ * DT overlay for SERDES personality card: 2lane PCIe + USB2.0 Host on AM654 EVM
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/phy/phy.h>
+#include <dt-bindings/phy/phy-am654-serdes.h>
+#include <dt-bindings/pinctrl/k3.h>
+
+&serdes0 {
+	assigned-clocks = <&k3_clks 153 4>, <&serdes0 AM654_SERDES_CMU_REFCLK>, <&serdes0 AM654_SERDES_RO_REFCLK>;
+	assigned-clock-parents = <&k3_clks 153 8>, <&k3_clks 153 4>, <&k3_clks 153 4>;
+	status = "okay";
+};
+
+&serdes1 {
+	assigned-clocks = <&serdes1 AM654_SERDES_CMU_REFCLK>;
+	assigned-clock-parents = <&serdes0 AM654_SERDES_RO_REFCLK>;
+	status = "okay";
+};
+
+&pcie0_rc {
+	num-lanes = <2>;
+	phys = <&serdes0 PHY_TYPE_PCIE 1>, <&serdes1 PHY_TYPE_PCIE 1>;
+	phy-names = "pcie-phy0", "pcie-phy1";
+	reset-gpios = <&pca9555 5 GPIO_ACTIVE_HIGH>;
+	status = "okay";
+};
+
+&main_pmx0 {
+       usb0_pins_default: usb0_pins_default {
+               pinctrl-single,pins = <
+                       AM65X_IOPAD(0x02bc, PIN_OUTPUT, 0) /* (AD9) USB0_DRVVBUS */
+               >;
+       };
+};
+
+&dwc3_0 {
+	status = "okay";
+};
+
+&usb0_phy {
+	status = "okay";
+};
+
+&usb0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&usb0_pins_default>;
+	dr_mode = "host";
+};
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/k3-am654-pcie-usb3.dtso b/arch/arm64/boot/dts/ti/k3-am654-pcie-usb3.dtso
--- a/arch/arm64/boot/dts/ti/k3-am654-pcie-usb3.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/k3-am654-pcie-usb3.dtso	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: GPL-2.0
+/**
+ * DT overlay for SERDES personality card: 1lane PCIe + USB3.0 DRD on AM654 EVM
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/phy/phy.h>
+
+&serdes1 {
+	status = "okay";
+};
+
+&pcie1_rc {
+	phys = <&serdes1 PHY_TYPE_PCIE 0>;
+	phy-names = "pcie-phy0";
+	reset-gpios = <&pca9555 5 GPIO_ACTIVE_HIGH>;
+	status = "okay";
+};
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/k3-am65.dtsi b/arch/arm64/boot/dts/ti/k3-am65.dtsi
--- a/arch/arm64/boot/dts/ti/k3-am65.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/k3-am65.dtsi	2020-06-19 21:00:31.408227715 -0400
@@ -8,6 +8,10 @@
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/pinctrl/k3.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/dma/k3-udma.h>
+#include <dt-bindings/soc/ti,sci_pm_domain.h>
 
 / {
 	model = "Texas Instruments K3 AM654 SoC";
@@ -16,6 +20,23 @@
 	#address-cells = <2>;
 	#size-cells = <2>;
 
+	aliases {
+		serial0 = &wkup_uart0;
+		serial1 = &mcu_uart0;
+		serial2 = &main_uart0;
+		serial3 = &main_uart1;
+		serial4 = &main_uart2;
+		i2c0 = &wkup_i2c0;
+		i2c1 = &mcu_i2c0;
+		i2c2 = &main_i2c0;
+		i2c3 = &main_i2c1;
+		i2c4 = &main_i2c2;
+		i2c5 = &main_i2c3;
+		ethernet0 = &cpsw_port1;
+		rproc0 = &mcu_r5fss0_core0;
+		rproc1 = &mcu_r5fss0_core1;
+	};
+
 	chosen { };
 
 	firmware {
@@ -51,15 +72,25 @@
 		ranges = <0x00 0x00100000 0x00 0x00100000 0x00 0x00020000>, /* ctrl mmr */
 			 <0x00 0x00600000 0x00 0x00600000 0x00 0x00001100>, /* GPIO */
 			 <0x00 0x00900000 0x00 0x00900000 0x00 0x00012000>, /* serdes */
+			 <0x00 0x00A40000 0x00 0x00A40000 0x00 0x00000800>, /* timesync router */
 			 <0x00 0x01000000 0x00 0x01000000 0x00 0x0af02400>, /* Most peripherals */
 			 <0x00 0x30800000 0x00 0x30800000 0x00 0x0bc00000>, /* MAIN NAVSS */
+			 <0x00 0x10000000 0x00 0x10000000 0x00 0x10000000>, /* PCIe DAT */
 			 /* MCUSS Range */
 			 <0x00 0x28380000 0x00 0x28380000 0x00 0x03880000>,
 			 <0x00 0x40200000 0x00 0x40200000 0x00 0x00900100>,
+			 <0x00 0x40f00000 0x00 0x40f00000 0x00 0x00020000>, /* CTRL_MMR0 */
+			 <0x00 0x41000000 0x00 0x41000000 0x00 0x00020000>,
+			 <0x00 0x41400000 0x00 0x41400000 0x00 0x00020000>,
+			 <0x00 0x41c00000 0x00 0x41c00000 0x00 0x00080000>,
 			 <0x00 0x42040000 0x00 0x42040000 0x00 0x03ac2400>,
 			 <0x00 0x45100000 0x00 0x45100000 0x00 0x00c24000>,
 			 <0x00 0x46000000 0x00 0x46000000 0x00 0x00200000>,
-			 <0x00 0x47000000 0x00 0x47000000 0x00 0x00068400>;
+			 <0x00 0x47000000 0x00 0x47000000 0x00 0x00068400>,
+			 <0x00 0x50000000 0x00 0x50000000 0x00 0x8000000>,
+			 <0x00 0x70000000 0x00 0x70000000 0x00 0x200000>,
+			 <0x05 0x00000000 0x05 0x00000000 0x01 0x0000000>,
+			 <0x07 0x00000000 0x07 0x00000000 0x01 0x0000000>;
 
 		cbass_mcu: interconnect@28380000 {
 			compatible = "simple-bus";
@@ -67,10 +98,17 @@
 			#size-cells = <2>;
 			ranges = <0x00 0x28380000 0x00 0x28380000 0x00 0x03880000>, /* MCU NAVSS*/
 				 <0x00 0x40200000 0x00 0x40200000 0x00 0x00900100>, /* First peripheral window */
+				 <0x00 0x40f00000 0x00 0x40f00000 0x00 0x00020000>, /* CTRL_MMR0 */
+				 <0x00 0x41000000 0x00 0x41000000 0x00 0x00020000>, /* MCU R5F Core0 */
+				 <0x00 0x41400000 0x00 0x41400000 0x00 0x00020000>, /* MCU R5F Core1 */
+				 <0x00 0x41c00000 0x00 0x41c00000 0x00 0x00080000>, /* MCU SRAM */
 				 <0x00 0x42040000 0x00 0x42040000 0x00 0x03ac2400>, /* WKUP */
 				 <0x00 0x45100000 0x00 0x45100000 0x00 0x00c24000>, /* MMRs, remaining NAVSS */
 				 <0x00 0x46000000 0x00 0x46000000 0x00 0x00200000>, /* CPSW */
-				 <0x00 0x47000000 0x00 0x47000000 0x00 0x00068400>; /* OSPI space 1 */
+				 <0x00 0x47000000 0x00 0x47000000 0x00 0x00068400>, /* OSPI space 1 */
+				 <0x00 0x50000000 0x00 0x50000000 0x00 0x8000000>, /*  FSS OSPI0 data region 1 */
+				 <0x05 0x00000000 0x05 0x00000000 0x01 0x0000000>, /* FSS OSPI0 data region 3*/
+				 <0x07 0x00000000 0x07 0x00000000 0x01 0x0000000>; /* FSS OSPI1 data region 3*/
 
 			cbass_wakeup: interconnect@42040000 {
 				compatible = "simple-bus";
@@ -85,3 +123,5 @@
 
 /* Now include the peripherals for each bus segments */
 #include "k3-am65-main.dtsi"
+#include "k3-am65-mcu.dtsi"
+#include "k3-am65-wakeup.dtsi"
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/k3-am65-main.dtsi b/arch/arm64/boot/dts/ti/k3-am65-main.dtsi
--- a/arch/arm64/boot/dts/ti/k3-am65-main.dtsi	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/k3-am65-main.dtsi	2020-06-19 21:00:31.408227715 -0400
@@ -4,8 +4,68 @@
  *
  * Copyright (C) 2016-2018 Texas Instruments Incorporated - http://www.ti.com/
  */
+#include <dt-bindings/phy/phy-am654-serdes.h>
 
 &cbass_main {
+	msmc_ram: sram@70000000 {
+		compatible = "mmio-sram";
+		reg = <0x0 0x70000000 0x0 0x200000>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0x0 0x0 0x70000000 0x200000>;
+
+		atf-sram@0 {
+			reg = <0x0 0x20000>;
+		};
+
+		sysfw-sram@f0000 {
+			reg = <0xf0000 0x10000>;
+		};
+
+		l3cache-sram@100000 {
+			reg = <0x100000 0x100000>;
+		};
+	};
+
+	scm_conf: scm_conf@100000 {
+		compatible = "syscon", "simple-mfd";
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0x0 0x0 0x00100000 0x1c000>;
+
+		pcie0_mode: pcie-mode@4060 {
+			compatible = "syscon";
+			reg = <0x00004060 0x4>;
+		};
+
+		pcie1_mode: pcie-mode@4070 {
+			compatible = "syscon";
+			reg = <0x00004070 0x4>;
+		};
+
+		pcie_devid: pcie-devid@210 {
+			compatible = "syscon";
+			reg = <0x00000210 0x4>;
+		};
+
+		serdes0_clk: serdes_clk@4080 {
+			compatible = "syscon";
+			reg = <0x00004080 0x4>;
+		};
+
+		serdes1_clk: serdes_clk@4090 {
+			compatible = "syscon";
+			reg = <0x00004090 0x4>;
+		};
+
+		serdes_mux: mux-controller {
+			compatible = "mmio-mux";
+			#mux-control-cells = <1>;
+			mux-reg-masks = <0x4080 0x3>, /* SERDES0 lane select */
+					<0x4090 0x3>; /* SERDES1 lane select */
+		};
+	};
+
 	gic500: interrupt-controller@1800000 {
 		compatible = "arm,gic-v3";
 		#address-cells = <2>;
@@ -24,8 +84,1643 @@
 		gic_its: gic-its@1820000 {
 			compatible = "arm,gic-v3-its";
 			reg = <0x00 0x01820000 0x00 0x10000>;
+			socionext,synquacer-pre-its = <0x1000000 0x400000>;
 			msi-controller;
 			#msi-cells = <1>;
 		};
 	};
+
+	secure_proxy_main: mailbox@32c00000 {
+		compatible = "ti,am654-secure-proxy";
+		#mbox-cells = <1>;
+		reg-names = "target_data", "rt", "scfg";
+		reg = <0x00 0x32c00000 0x00 0x100000>,
+		      <0x00 0x32400000 0x00 0x100000>,
+		      <0x00 0x32800000 0x00 0x100000>;
+		interrupt-names = "rx_011";
+		interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	serdes0: serdes@900000 {
+		compatible = "ti,phy-am654-serdes";
+		reg = <0x0 0x900000 0x0 0x2000>;
+		reg-names = "serdes";
+		#phy-cells = <2>;
+		power-domains = <&k3_pds 153 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 153 4>, <&k3_clks 153 1>, <&serdes1 AM654_SERDES_LO_REFCLK>;
+		clock-output-names = "serdes0_cmu_refclk", "serdes0_lo_refclk", "serdes0_ro_refclk";
+		assigned-clocks = <&k3_clks 153 4>, <&serdes0 AM654_SERDES_CMU_REFCLK>;
+		assigned-clock-parents = <&k3_clks 153 8>, <&k3_clks 153 4>;
+		ti,serdes-clk = <&serdes0_clk>;
+		#clock-cells = <1>;
+		mux-controls = <&serdes_mux 0>;
+		status = "disabled";
+	};
+
+	serdes1: serdes@910000 {
+		compatible = "ti,phy-am654-serdes";
+		reg = <0x0 0x910000 0x0 0x2000>;
+		reg-names = "serdes";
+		#phy-cells = <2>;
+		power-domains = <&k3_pds 154 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&serdes0 AM654_SERDES_RO_REFCLK>, <&k3_clks 154 1>, <&k3_clks 154 5>;
+		clock-output-names = "serdes1_cmu_refclk", "serdes1_lo_refclk", "serdes1_ro_refclk";
+		assigned-clocks = <&k3_clks 154 5>, <&serdes1 AM654_SERDES_CMU_REFCLK>;
+		assigned-clock-parents = <&k3_clks 154 9>, <&k3_clks 154 5>;
+		ti,serdes-clk = <&serdes1_clk>;
+		#clock-cells = <1>;
+		mux-controls = <&serdes_mux 1>;
+		status = "disabled";
+	};
+
+	main_uart0: serial@2800000 {
+		compatible = "ti,am654-uart";
+		reg = <0x00 0x02800000 0x00 0x100>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		interrupts = <GIC_SPI 192 IRQ_TYPE_LEVEL_HIGH>;
+		clock-frequency = <48000000>;
+		current-speed = <115200>;
+		power-domains = <&k3_pds 146 TI_SCI_PD_EXCLUSIVE>;
+		dmas = <&main_udmap &pdma1 20 UDMA_DIR_TX>,
+		       <&main_udmap &pdma1 20 UDMA_DIR_RX>;
+		dma-names = "tx", "rx";
+	};
+
+	main_uart1: serial@2810000 {
+		compatible = "ti,am654-uart";
+		reg = <0x00 0x02810000 0x00 0x100>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		interrupts = <GIC_SPI 193 IRQ_TYPE_LEVEL_HIGH>;
+		clock-frequency = <48000000>;
+		power-domains = <&k3_pds 147 TI_SCI_PD_EXCLUSIVE>;
+		dmas = <&main_udmap &pdma1 21 UDMA_DIR_TX>,
+		       <&main_udmap &pdma1 21 UDMA_DIR_RX>;
+		dma-names = "tx", "rx";
+	};
+
+	main_uart2: serial@2820000 {
+		compatible = "ti,am654-uart";
+		reg = <0x00 0x02820000 0x00 0x100>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		interrupts = <GIC_SPI 194 IRQ_TYPE_LEVEL_HIGH>;
+		clock-frequency = <48000000>;
+		power-domains = <&k3_pds 148 TI_SCI_PD_EXCLUSIVE>;
+		dmas = <&main_udmap &pdma1 22 UDMA_DIR_TX>,
+		       <&main_udmap &pdma1 22 UDMA_DIR_RX>;
+		dma-names = "tx", "rx";
+	};
+
+	main_intr: interrupt-controller0 {
+		compatible = "ti,sci-intr";
+		interrupt-controller;
+		interrupt-parent = <&gic500>;
+		#interrupt-cells = <3>;
+		ti,sci = <&dmsc>;
+		ti,sci-dst-id = <56>;
+		ti,sci-rm-range-girq = <0x1>;
+	};
+
+	main_navss: main_navss {
+		compatible = "simple-bus";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		dma-coherent;
+		dma-ranges;
+		ranges;
+
+		ti,sci-dev-id = <118>;
+
+		main_navss_intr: interrupt-controller1 {
+			compatible = "ti,sci-intr";
+			interrupt-controller;
+			interrupt-parent = <&gic500>;
+			#interrupt-cells = <3>;
+			ti,sci = <&dmsc>;
+			ti,sci-dst-id = <56>;
+			ti,sci-rm-range-girq = <0x0>,
+					       <0x2>;
+		};
+
+		main_udmass_inta: interrupt-controller@33d00000 {
+			compatible = "ti,sci-inta";
+			reg = <0x0 0x33d00000 0x0 0x100000>;
+			interrupt-controller;
+			interrupt-parent = <&main_navss_intr>;
+			#interrupt-cells = <3>;
+			ti,sci = <&dmsc>;
+			ti,sci-dev-id = <179>;
+			ti,sci-rm-range-vint = <0x0>;
+			ti,sci-rm-range-global-event = <0x1>;
+		};
+
+		hwspinlock: spinlock@30e00000 {
+			compatible = "ti,am654-hwspinlock";
+			reg = <0x00 0x30e00000 0x00 0x1000>;
+			#hwlock-cells = <1>;
+		};
+
+		mailbox0_cluster0: mailbox@31f80000 {
+			compatible = "ti,am654-mailbox";
+			reg = <0x00 0x31f80000 0x00 0x200>;
+			#mbox-cells = <1>;
+			ti,mbox-num-users = <4>;
+			ti,mbox-num-fifos = <16>;
+			interrupt-parent = <&main_navss_intr>;
+			interrupts = <164 0 IRQ_TYPE_LEVEL_HIGH>;
+
+			mbox_mcu_r5fss0_core0: mbox-mcu-r5fss0-core0 {
+				ti,mbox-tx = <1 0 0>;
+				ti,mbox-rx = <0 0 0>;
+			};
+		};
+
+		mailbox0_cluster1: mailbox@31f81000 {
+			compatible = "ti,am654-mailbox";
+			reg = <0x00 0x31f81000 0x00 0x200>;
+			#mbox-cells = <1>;
+			ti,mbox-num-users = <4>;
+			ti,mbox-num-fifos = <16>;
+			interrupt-parent = <&main_navss_intr>;
+			interrupts = <165 0 IRQ_TYPE_LEVEL_HIGH>;
+
+			mbox_mcu_r5fss0_core1: mbox-mcu-r5fss0-core1 {
+				ti,mbox-tx = <1 0 0>;
+				ti,mbox-rx = <0 0 0>;
+			};
+		};
+
+		mailbox0_cluster2: mailbox@31f82000 {
+			compatible = "ti,am654-mailbox";
+			reg = <0x00 0x31f82000 0x00 0x200>;
+			#mbox-cells = <1>;
+			ti,mbox-num-users = <4>;
+			ti,mbox-num-fifos = <16>;
+			status = "disabled";
+		};
+
+		mailbox0_cluster3: mailbox@31f83000 {
+			compatible = "ti,am654-mailbox";
+			reg = <0x00 0x31f83000 0x00 0x200>;
+			#mbox-cells = <1>;
+			ti,mbox-num-users = <4>;
+			ti,mbox-num-fifos = <16>;
+			status = "disabled";
+		};
+
+		mailbox0_cluster4: mailbox@31f84000 {
+			compatible = "ti,am654-mailbox";
+			reg = <0x00 0x31f84000 0x00 0x200>;
+			#mbox-cells = <1>;
+			ti,mbox-num-users = <4>;
+			ti,mbox-num-fifos = <16>;
+			status = "disabled";
+		};
+
+		mailbox0_cluster5: mailbox@31f85000 {
+			compatible = "ti,am654-mailbox";
+			reg = <0x00 0x31f85000 0x00 0x200>;
+			#mbox-cells = <1>;
+			ti,mbox-num-users = <4>;
+			ti,mbox-num-fifos = <16>;
+			status = "disabled";
+		};
+
+		mailbox0_cluster6: mailbox@31f86000 {
+			compatible = "ti,am654-mailbox";
+			reg = <0x00 0x31f86000 0x00 0x200>;
+			#mbox-cells = <1>;
+			ti,mbox-num-users = <4>;
+			ti,mbox-num-fifos = <16>;
+			status = "disabled";
+		};
+
+		mailbox0_cluster7: mailbox@31f87000 {
+			compatible = "ti,am654-mailbox";
+			reg = <0x00 0x31f87000 0x00 0x200>;
+			#mbox-cells = <1>;
+			ti,mbox-num-users = <4>;
+			ti,mbox-num-fifos = <16>;
+			status = "disabled";
+		};
+
+		mailbox0_cluster8: mailbox@31f88000 {
+			compatible = "ti,am654-mailbox";
+			reg = <0x00 0x31f88000 0x00 0x200>;
+			#mbox-cells = <1>;
+			ti,mbox-num-users = <4>;
+			ti,mbox-num-fifos = <16>;
+			status = "disabled";
+		};
+
+		mailbox0_cluster9: mailbox@31f89000 {
+			compatible = "ti,am654-mailbox";
+			reg = <0x00 0x31f89000 0x00 0x200>;
+			#mbox-cells = <1>;
+			ti,mbox-num-users = <4>;
+			ti,mbox-num-fifos = <16>;
+			status = "disabled";
+		};
+
+		mailbox0_cluster10: mailbox@31f8a000 {
+			compatible = "ti,am654-mailbox";
+			reg = <0x00 0x31f8a000 0x00 0x200>;
+			#mbox-cells = <1>;
+			ti,mbox-num-users = <4>;
+			ti,mbox-num-fifos = <16>;
+			status = "disabled";
+		};
+
+		mailbox0_cluster11: mailbox@31f8b000 {
+			compatible = "ti,am654-mailbox";
+			reg = <0x00 0x31f8b000 0x00 0x200>;
+			#mbox-cells = <1>;
+			ti,mbox-num-users = <4>;
+			ti,mbox-num-fifos = <16>;
+			status = "disabled";
+		};
+
+		ringacc: ringacc@3c000000 {
+			compatible = "ti,am654-navss-ringacc";
+			reg =	<0x0 0x3c000000 0x0 0x400000>,
+				<0x0 0x38000000 0x0 0x400000>,
+				<0x0 0x31120000 0x0 0x100>,
+				<0x0 0x33000000 0x0 0x40000>;
+			reg-names = "rt", "fifos", "proxy_gcfg", "proxy_target";
+			ti,num-rings = <818>;
+			ti,sci-rm-range-gp-rings = <0x2>; /* GP ring range */
+			ti,dma-ring-reset-quirk;
+			ti,sci = <&dmsc>;
+			ti,sci-dev-id = <187>;
+			interrupt-parent = <&main_udmass_inta>;
+		};
+
+		main_udmap: udmap@31150000 {
+			compatible = "ti,am654-navss-main-udmap";
+			reg =	<0x0 0x31150000 0x0 0x100>,
+				<0x0 0x34000000 0x0 0x100000>,
+				<0x0 0x35000000 0x0 0x100000>;
+			reg-names = "gcfg", "rchanrt", "tchanrt";
+			#dma-cells = <3>;
+
+			ti,ringacc = <&ringacc>;
+			ti,psil-base = <0x1000>;
+
+			interrupt-parent = <&main_udmass_inta>;
+
+			ti,sci = <&dmsc>;
+			ti,sci-dev-id = <188>;
+
+			ti,sci-rm-range-tchan = <0x1>, /* TX_HCHAN */
+						<0x2>; /* TX_CHAN */
+			ti,sci-rm-range-rchan = <0x4>, /* RX_HCHAN */
+						<0x5>; /* RX_CHAN */
+			ti,sci-rm-range-rflow = <0x6>; /* GP RFLOW */
+		};
+
+		cpts@310d0000 {
+			compatible = "ti,am65-cpts";
+			reg = <0x0 0x310d0000 0x0 0x400>;
+			reg-names = "cpts";
+			clocks = <&main_cpts_mux>;
+			clock-names = "cpts";
+			interrupts-extended = <&main_navss_intr 163 0 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "cpts";
+			ti,cpts-periodic-outputs = <6>;
+			ti,cpts-ext-ts-inputs = <8>;
+
+			main_cpts_mux: cpts_refclk_mux {
+				#clock-cells = <0>;
+				clocks = <&k3_clks 118 5>, <&k3_clks 118 11>,
+					<&k3_clks 157 91>, <&k3_clks 157 77>,
+					<&k3_clks 157 102>, <&k3_clks 157 80>,
+					<&k3_clks 120 3>, <&k3_clks 121 3>;
+				cpts-mux-tbl = <0>, <1>;
+				assigned-clocks = <&main_cpts_mux>;
+				assigned-clock-parents = <&k3_clks 118 5>;
+			};
+		};
+	};
+
+	pdma0: pdma@2a41000 {
+		compatible = "ti,am654-pdma";
+		reg = <0x0 0x02A41000 0x0 0x400>;
+		reg-names = "eccaggr_cfg";
+
+		ti,psil-base = <0x4400>;
+
+		/* ti,psil-config0-2 */
+		UDMA_PDMA_TR_XY(0);
+		UDMA_PDMA_TR_XY(1);
+		UDMA_PDMA_TR_XY(2);
+	};
+
+	pdma1: pdma@2a42000 {
+		compatible = "ti,am654-pdma";
+		reg = <0x0 0x02A42000 0x0 0x400>;
+		reg-names = "eccaggr_cfg";
+
+		ti,psil-base = <0x4500>;
+
+		/* ti,psil-config0-22 */
+		UDMA_PDMA_PKT_XY(0);
+		UDMA_PDMA_TR_XY(1);
+		UDMA_PDMA_TR_XY(2);
+		UDMA_PDMA_TR_XY(3);
+		UDMA_PDMA_TR_XY(4);
+		UDMA_PDMA_TR_XY(5);
+		UDMA_PDMA_TR_XY(6);
+		UDMA_PDMA_TR_XY(7);
+		UDMA_PDMA_TR_XY(8);
+		UDMA_PDMA_TR_XY(9);
+		UDMA_PDMA_TR_XY(10);
+		UDMA_PDMA_TR_XY(11);
+		UDMA_PDMA_TR_XY(12);
+		UDMA_PDMA_TR_XY(13);
+		UDMA_PDMA_TR_XY(14);
+		UDMA_PDMA_TR_XY(15);
+		UDMA_PDMA_TR_XY(16);
+		UDMA_PDMA_TR_XY(17);
+		UDMA_PDMA_TR_XY(18);
+		UDMA_PDMA_TR_XY(19);
+		UDMA_PDMA_PKT_XY(20);
+		UDMA_PDMA_PKT_XY(21);
+		UDMA_PDMA_PKT_XY(22);
+	};
+
+	crypto: crypto@4E00000 {
+		compatible = "ti,sa2ul-crypto";
+		label = "crypto-aes-gbe";
+		reg = <0x0 0x4E00000 0x0 0x1200>;
+		power-domains = <&k3_pds 136 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 136 0>, <&k3_clks 136 1>, <&k3_clks 136 2>;
+		clock-names = "x2_clk", "pka_in_clk", "x1_clk";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges = <0x0 0x04E00000 0x00 0x04E00000 0x0 0x30000>;
+
+		status = "okay";
+		ti,psil-base = <0x4000>;
+
+		/* tx: crypto_pnp-1, rx: crypto_pnp-1 */
+		dmas = <&main_udmap &crypto 0 UDMA_DIR_TX>,
+				<&main_udmap &crypto 0 UDMA_DIR_RX>,
+				<&main_udmap &crypto 1 UDMA_DIR_RX>;
+		dma-names = "tx", "rx1", "rx2";
+
+		ti,psil-config0 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			ti,needs-epib;
+			ti,psd-size = <64>;
+			ti,notdpkt;
+		};
+
+		ti,psil-config1 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			ti,needs-epib;
+			ti,psd-size = <64>;
+			ti,notdpkt;
+		};
+
+		ti,psil-config2 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			ti,needs-epib;
+			ti,psd-size = <64>;
+			ti,notdpkt;
+		};
+
+		eip76d_trng: trng@4e10000 {
+			compatible = "inside-secure,safexcel-eip76";
+			reg = <0x0 0x4e10000 0x0 0x7d>;
+			interrupts = <GIC_SPI 24 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&k3_clks 136 1>;
+		};
+	};
+
+	main_pmx0: pinmux@11c000 {
+		compatible = "pinctrl-single";
+		reg = <0x0 0x11c000 0x0 0x2e4>;
+		#pinctrl-cells = <1>;
+		pinctrl-single,register-width = <32>;
+		pinctrl-single,function-mask = <0xffffffff>;
+	};
+
+	main_pmx1: pinmux@11c2e8 {
+		compatible = "pinctrl-single";
+		reg = <0x0 0x11c2e8 0x0 0x24>;
+		#pinctrl-cells = <1>;
+		pinctrl-single,register-width = <32>;
+		pinctrl-single,function-mask = <0xffffffff>;
+	};
+
+	main_i2c0: i2c@2000000 {
+		compatible = "ti,am654-i2c", "ti,omap4-i2c";
+		reg = <0x0 0x2000000 0x0 0x100>;
+		interrupts = <GIC_SPI 200 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clock-names = "fck";
+		clocks = <&k3_clks 110 1>;
+		power-domains = <&k3_pds 110 TI_SCI_PD_EXCLUSIVE>;
+	};
+
+	main_i2c1: i2c@2010000 {
+		compatible = "ti,am654-i2c", "ti,omap4-i2c";
+		reg = <0x0 0x2010000 0x0 0x100>;
+		interrupts = <GIC_SPI 201 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clock-names = "fck";
+		clocks = <&k3_clks 111 1>;
+		power-domains = <&k3_pds 111 TI_SCI_PD_EXCLUSIVE>;
+	};
+
+	main_i2c2: i2c@2020000 {
+		compatible = "ti,am654-i2c", "ti,omap4-i2c";
+		reg = <0x0 0x2020000 0x0 0x100>;
+		interrupts = <GIC_SPI 202 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clock-names = "fck";
+		clocks = <&k3_clks 112 1>;
+		power-domains = <&k3_pds 112 TI_SCI_PD_EXCLUSIVE>;
+	};
+
+	main_i2c3: i2c@2030000 {
+		compatible = "ti,am654-i2c", "ti,omap4-i2c";
+		reg = <0x0 0x2030000 0x0 0x100>;
+		interrupts = <GIC_SPI 203 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clock-names = "fck";
+		clocks = <&k3_clks 113 1>;
+		power-domains = <&k3_pds 113 TI_SCI_PD_EXCLUSIVE>;
+	};
+
+	main_gpio0:  main_gpio0@600000 {
+		compatible = "ti,k2g-gpio", "ti,keystone-gpio";
+		reg = <0x0 0x600000 0x0 0x100>;
+		gpio-controller;
+		#gpio-cells = <2>;
+			interrupt-parent = <&main_intr>;
+		interrupts = <57 256 IRQ_TYPE_EDGE_RISING>,
+				<57 257 IRQ_TYPE_EDGE_RISING>,
+				<57 258 IRQ_TYPE_EDGE_RISING>,
+				<57 259 IRQ_TYPE_EDGE_RISING>,
+				<57 260 IRQ_TYPE_EDGE_RISING>,
+				<57 261 IRQ_TYPE_EDGE_RISING>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		ti,ngpio = <96>;
+		ti,davinci-gpio-unbanked = <0>;
+		clocks = <&k3_clks 57 0>;
+		clock-names = "gpio";
+	};
+
+	main_gpio1:  main_gpio1@601000 {
+		compatible = "ti,k2g-gpio", "ti,keystone-gpio";
+		reg = <0x0 0x601000 0x0 0x100>;
+		gpio-controller;
+		#gpio-cells = <2>;
+			interrupt-parent = <&main_intr>;
+		interrupts = <58 256 IRQ_TYPE_EDGE_RISING>,
+				<58 257 IRQ_TYPE_EDGE_RISING>,
+				<58 258 IRQ_TYPE_EDGE_RISING>,
+				<58 259 IRQ_TYPE_EDGE_RISING>,
+				<58 260 IRQ_TYPE_EDGE_RISING>,
+				<58 261 IRQ_TYPE_EDGE_RISING>;
+		interrupt-controller;
+			#interrupt-cells = <2>;
+		ti,ngpio = <90>;
+		ti,davinci-gpio-unbanked = <0>;
+		clocks = <&k3_clks 58 0>;
+		clock-names = "gpio";
+	};
+
+	mcasp0: mcasp@02B00000 {
+		compatible = "ti,am33xx-mcasp-audio";
+		reg = <0x0 0x02B00000 0x0 0x2000>,
+			<0x0 0x02B08000 0x0 0x1000>;
+		reg-names = "mpu","dat";
+		interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>,
+				<GIC_SPI 209 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "tx", "rx";
+
+		/* tx: pdma0-0, rx: pdma0-0 */
+		dmas = <&main_udmap &pdma0 0 UDMA_DIR_TX>,
+			<&main_udmap &pdma0 0 UDMA_DIR_RX>;
+		dma-names = "tx", "rx";
+
+		clocks = <&k3_clks 104 0>;
+		clock-names = "fck";
+		power-domains = <&k3_pds 104 TI_SCI_PD_EXCLUSIVE>;
+
+		status = "disabled";
+	};
+
+	mcasp1: mcasp@02B10000 {
+		compatible = "ti,am33xx-mcasp-audio";
+		reg = <0x0 0x02B10000 0x0 0x2000>,
+			<0x0 0x02B18000 0x0 0x1000>;
+		reg-names = "mpu","dat";
+		interrupts = <GIC_SPI 210 IRQ_TYPE_LEVEL_HIGH>,
+				<GIC_SPI 211 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "tx", "rx";
+
+		/* tx: pdma0-1, rx: pdma0-1 */
+		dmas = <&main_udmap &pdma0 1 UDMA_DIR_TX>,
+			<&main_udmap &pdma0 1 UDMA_DIR_RX>;
+		dma-names = "tx", "rx";
+
+		clocks = <&k3_clks 105 0>;
+		clock-names = "fck";
+		power-domains = <&k3_pds 105 TI_SCI_PD_EXCLUSIVE>;
+
+		status = "disabled";
+	};
+
+	mcasp2: mcasp@02B20000 {
+		compatible = "ti,am33xx-mcasp-audio";
+		reg = <0x0 0x02B20000 0x0 0x2000>,
+			<0x0 0x02B28000 0x0 0x1000>;
+		reg-names = "mpu","dat";
+		interrupts = <GIC_SPI 212 IRQ_TYPE_LEVEL_HIGH>,
+				<GIC_SPI 213 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "tx", "rx";
+
+		/* tx: pdma0-1, rx: pdma0-1 */
+		dmas = <&main_udmap &pdma0 2 UDMA_DIR_TX>,
+			<&main_udmap &pdma0 2 UDMA_DIR_RX>;
+		dma-names = "tx", "rx";
+
+		clocks = <&k3_clks 106 0>;
+		clock-names = "fck";
+		power-domains = <&k3_pds 106 TI_SCI_PD_EXCLUSIVE>;
+
+		status = "disabled";
+	};
+
+	gpu: gpu@7000000 {
+		compatible = "ti,am654-sgx544", "img,sgx544";
+		reg = <0x0 0x7000000 0x0 0x10000>;
+		interrupts = <GIC_SPI 162 IRQ_TYPE_LEVEL_HIGH>;
+		power-domains = <&k3_pds 65 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 65 0>, <&k3_clks 65 1>, <&k3_clks 65 2>, <&k3_clks 65 3>;
+		clock-names = "mem_clk", "hyd_clk", "sgx_clk", "sys_clk";
+		status = "disabled";
+	};
+
+	icssg_soc_bus0: pruss-soc-bus@b026004 {
+		compatible = "ti,am654-icssg-soc-bus";
+		reg = <0x00 0x0b026004 0x00 0x4>;
+		power-domains = <&k3_pds 62 TI_SCI_PD_EXCLUSIVE>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0x0b000000 0x00 0x0b000000 0x100000>;
+		dma-ranges;
+
+		icssg0: icssg@b000000 {
+			compatible = "ti,am654-icssg";
+			reg = <0xb000000 0x80000>;
+			interrupts = <GIC_SPI 254 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 255 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 256 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 257 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 258 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 259 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 260 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 261 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "host2", "host3", "host4",
+					  "host5", "host6", "host7",
+					  "host8", "host9";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+			dma-ranges;
+
+			ti,psil-base = <0x4100>;	/* ICSSG0 PSIL thread start */
+			ti,psil-config0 {
+				linux,udma-mode = <UDMA_PKT_MODE>;
+				statictr-type = <PSIL_STATIC_TR_NONE>;
+				ti,needs-epib;
+				ti,psd-size = <16>;
+			};
+
+			ti,psil-config1 {
+				linux,udma-mode = <UDMA_PKT_MODE>;
+				statictr-type = <PSIL_STATIC_TR_NONE>;
+				ti,needs-epib;
+				ti,psd-size = <16>;
+			};
+
+			ti,psil-config2 {
+				linux,udma-mode = <UDMA_PKT_MODE>;
+				statictr-type = <PSIL_STATIC_TR_NONE>;
+				ti,needs-epib;
+				ti,psd-size = <16>;
+			};
+
+			ti,psil-config3 {
+				linux,udma-mode = <UDMA_PKT_MODE>;
+				statictr-type = <PSIL_STATIC_TR_NONE>;
+				ti,needs-epib;
+				ti,psd-size = <16>;
+			};
+
+			ti,psil-config4 {
+				linux,udma-mode = <UDMA_PKT_MODE>;
+				statictr-type = <PSIL_STATIC_TR_NONE>;
+				ti,needs-epib;
+				ti,psd-size = <16>;
+			};
+
+			ti,psil-config5 {
+				linux,udma-mode = <UDMA_PKT_MODE>;
+				statictr-type = <PSIL_STATIC_TR_NONE>;
+				ti,needs-epib;
+				ti,psd-size = <16>;
+			};
+
+			ti,psil-config6 {
+				linux,udma-mode = <UDMA_PKT_MODE>;
+				statictr-type = <PSIL_STATIC_TR_NONE>;
+				ti,needs-epib;
+				ti,psd-size = <16>;
+			};
+
+			ti,psil-config7 {
+				linux,udma-mode = <UDMA_PKT_MODE>;
+				statictr-type = <PSIL_STATIC_TR_NONE>;
+				ti,needs-epib;
+				ti,psd-size = <16>;
+			};
+
+			icssg0_mem: memories@b000000 {
+				reg = <0xb000000 0x2000>,
+				      <0xb002000 0x2000>,
+				      <0xb010000 0x10000>;
+				reg-names = "dram0", "dram1",
+					    "shrdram2";
+			};
+
+			icssg0_cfg: cfg@b026000 {
+				compatible = "syscon";
+				reg = <0xb026000 0x200>;
+			};
+
+			icssg0_coreclk_mux: coreclk_mux {
+				#clock-cells = <0>;
+				clocks = <&k3_clks 62 19>, /* icssg0_core_clk */
+					 <&k3_clks 62 3>;  /* icssg0_iclk */
+				assigned-clocks = <&icssg0_coreclk_mux>;
+				assigned-clock-parents = <&k3_clks 62 3>;
+			};
+
+			icssg0_iepclk_mux: iepclk_mux {
+				#clock-cells = <0>;
+				clocks = <&k3_clks 62 10>,	/* icssg0_iep_clk */
+					 <&icssg0_coreclk_mux>;	/* core_clk */
+				assigned-clocks = <&icssg0_iepclk_mux>;
+				assigned-clock-parents = <&icssg0_coreclk_mux>;
+			};
+
+			icssg0_iep0: iep@b02e000 {
+				compatible = "syscon";
+				reg = <0xb02e000 0x1000>;
+				clocks = <&icssg0_iepclk_mux>;
+			};
+
+			icssg0_iep1: iep@b02f000 {
+				compatible = "syscon";
+				reg = <0xb02f000 0x1000>;
+				clocks = <&icssg0_iepclk_mux>;
+			};
+
+			icssg0_mii_rt: mii-rt@b032000 {
+				compatible = "syscon";
+				reg = <0xb032000 0x100>;
+			};
+
+			icssg0_mii_g_rt: mii-g-rt@b033000 {
+				compatible = "syscon";
+				reg = <0xb033000 0x1000>;
+			};
+
+			icssg0_intc: interrupt-controller@b020000 {
+				compatible = "ti,am654-icssg-intc";
+				reg = <0xb020000 0x2000>;
+				interrupt-controller;
+				#interrupt-cells = <1>;
+			};
+
+			pru0_0: pru@b034000 {
+				compatible = "ti,am654-pru";
+				reg = <0xb034000 0x4000>,
+				      <0xb022000 0x100>,
+				      <0xb022400 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "am65x-pru0_0-fw";
+				interrupt-parent = <&icssg0_intc>;
+				interrupts = <16>, <17>;
+				interrupt-names = "vring", "kick";
+			};
+
+			rtu0_0: rtu@b004000 {
+				compatible = "ti,am654-rtu";
+				reg = <0xb004000 0x2000>,
+				      <0xb023000 0x100>,
+				      <0xb023400 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "am65x-rtu0_0-fw";
+				interrupt-parent = <&icssg0_intc>;
+				interrupts = <20>, <21>;
+				interrupt-names = "vring", "kick";
+			};
+
+			pru0_1: pru@b038000 {
+				compatible = "ti,am654-pru";
+				reg = <0xb038000 0x4000>,
+				      <0xb024000 0x100>,
+				      <0xb024400 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "am65x-pru0_1-fw";
+				interrupt-parent = <&icssg0_intc>;
+				interrupts = <18>, <19>;
+				interrupt-names = "vring", "kick";
+			};
+
+			rtu0_1: rtu@b006000 {
+				compatible = "ti,am654-rtu";
+				reg = <0xb006000 0x2000>,
+				      <0xb023800 0x100>,
+				      <0xb023c00 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "am65x-rtu0_1-fw";
+				interrupt-parent = <&icssg0_intc>;
+				interrupts = <22>, <23>;
+				interrupt-names = "vring", "kick";
+			};
+
+			icssg0_mdio: mdio@b032400 {
+				compatible = "ti,davinci_mdio";
+				reg = <0xb032400 0x100>;
+				clocks = <&k3_clks 62 3>;
+				clock-names = "fck";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				bus_freq = <1000000>;
+				status = "disabled";
+			};
+		};
+	};
+
+	icssg_soc_bus1: pruss-soc-bus@b126004 {
+		compatible = "ti,am654-icssg-soc-bus";
+		reg = <0x00 0x0b126004 0x00 0x4>;
+		power-domains = <&k3_pds 63 TI_SCI_PD_EXCLUSIVE>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0x0b100000 0x00 0x0b100000 0x100000>;
+		dma-ranges;
+
+		icssg1: icssg@b100000 {
+			compatible = "ti,am654-icssg";
+			reg = <0xb100000 0x80000>;
+			interrupts = <GIC_SPI 262 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 263 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 264 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 265 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 266 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 267 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 268 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 269 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "host2", "host3", "host4",
+					  "host5", "host6", "host7",
+					  "host8", "host9";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+			dma-ranges;
+
+			ti,psil-base = <0x4200>;	/* ICSSG1 PSIL thread start */
+			ti,psil-config0 {
+				linux,udma-mode = <UDMA_PKT_MODE>;
+				statictr-type = <PSIL_STATIC_TR_NONE>;
+				ti,needs-epib;
+				ti,psd-size = <16>;
+			};
+
+			ti,psil-config1 {
+				linux,udma-mode = <UDMA_PKT_MODE>;
+				statictr-type = <PSIL_STATIC_TR_NONE>;
+				ti,needs-epib;
+				ti,psd-size = <16>;
+			};
+
+			ti,psil-config2 {
+				linux,udma-mode = <UDMA_PKT_MODE>;
+				statictr-type = <PSIL_STATIC_TR_NONE>;
+				ti,needs-epib;
+				ti,psd-size = <16>;
+			};
+
+			ti,psil-config3 {
+				linux,udma-mode = <UDMA_PKT_MODE>;
+				statictr-type = <PSIL_STATIC_TR_NONE>;
+				ti,needs-epib;
+				ti,psd-size = <16>;
+			};
+
+			ti,psil-config4 {
+				linux,udma-mode = <UDMA_PKT_MODE>;
+				statictr-type = <PSIL_STATIC_TR_NONE>;
+				ti,needs-epib;
+				ti,psd-size = <16>;
+			};
+
+			ti,psil-config5 {
+				linux,udma-mode = <UDMA_PKT_MODE>;
+				statictr-type = <PSIL_STATIC_TR_NONE>;
+				ti,needs-epib;
+				ti,psd-size = <16>;
+			};
+
+			ti,psil-config6 {
+				linux,udma-mode = <UDMA_PKT_MODE>;
+				statictr-type = <PSIL_STATIC_TR_NONE>;
+				ti,needs-epib;
+				ti,psd-size = <16>;
+			};
+
+			ti,psil-config7 {
+				linux,udma-mode = <UDMA_PKT_MODE>;
+				statictr-type = <PSIL_STATIC_TR_NONE>;
+				ti,needs-epib;
+				ti,psd-size = <16>;
+			};
+
+			icssg1_mem: memories@b100000 {
+				reg = <0xb100000 0x2000>,
+				      <0xb102000 0x2000>,
+				      <0xb110000 0x10000>;
+				reg-names = "dram0", "dram1",
+					    "shrdram2";
+			};
+
+			icssg1_cfg: cfg@b126000 {
+				compatible = "syscon";
+				reg = <0xb126000 0x200>;
+			};
+
+			icssg1_coreclk_mux: coreclk_mux {
+				#clock-cells = <0>;
+				clocks = <&k3_clks 63 19>, /* icssg1_core_clk */
+					 <&k3_clks 63 3>;  /* icssg1_iclk */
+				assigned-clocks = <&icssg1_coreclk_mux>;
+				assigned-clock-parents = <&k3_clks 63 3>;
+			};
+
+			icssg1_iepclk_mux: iepclk_mux {
+				#clock-cells = <0>;
+				clocks = <&k3_clks 63 10>,	/* icssg1_iep_clk */
+					 <&icssg1_coreclk_mux>;	/* core_clk */
+				assigned-clocks = <&icssg1_iepclk_mux>;
+				assigned-clock-parents = <&icssg1_coreclk_mux>;
+			};
+
+			icssg1_iep0: iep@b12e000 {
+				compatible = "syscon";
+				reg = <0xb12e000 0x1000>;
+				clocks = <&icssg1_iepclk_mux>;
+			};
+
+			icssg1_iep1: iep@b12f000 {
+				compatible = "syscon";
+				reg = <0xb12f000 0x1000>;
+				clocks = <&icssg1_iepclk_mux>;
+			};
+
+			icssg1_mii_rt: mii-rt@b132000 {
+				compatible = "syscon";
+				reg = <0xb132000 0x100>;
+			};
+
+			icssg1_mii_g_rt: mii-g-rt@b133000 {
+				compatible = "syscon";
+				reg = <0xb133000 0x1000>;
+			};
+
+			icssg1_intc: interrupt-controller@b120000 {
+				compatible = "ti,am654-icssg-intc";
+				reg = <0xb120000 0x2000>;
+				interrupt-controller;
+				#interrupt-cells = <1>;
+			};
+
+			pru1_0: pru@b134000 {
+				compatible = "ti,am654-pru";
+				reg = <0xb134000 0x4000>,
+				      <0xb122000 0x100>,
+				      <0xb122400 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "am65x-pru1_0-fw";
+				interrupt-parent = <&icssg1_intc>;
+				interrupts = <16>, <17>;
+				interrupt-names = "vring", "kick";
+			};
+
+			rtu1_0: rtu@b104000 {
+				compatible = "ti,am654-rtu";
+				reg = <0xb104000 0x2000>,
+				      <0xb123000 0x100>,
+				      <0xb123400 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "am65x-rtu1_0-fw";
+				interrupt-parent = <&icssg1_intc>;
+				interrupts = <20>, <21>;
+				interrupt-names = "vring", "kick";
+			};
+
+			pru1_1: pru@b138000 {
+				compatible = "ti,am654-pru";
+				reg = <0xb138000 0x4000>,
+				      <0xb124000 0x100>,
+				      <0xb124400 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "am65x-pru1_1-fw";
+				interrupt-parent = <&icssg1_intc>;
+				interrupts = <18>, <19>;
+				interrupt-names = "vring", "kick";
+			};
+
+			rtu1_1: rtu@b106000 {
+				compatible = "ti,am654-rtu";
+				reg = <0xb106000 0x2000>,
+				      <0xb123800 0x100>,
+				      <0xb123c00 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "am65x-rtu1_1-fw";
+				interrupt-parent = <&icssg1_intc>;
+				interrupts = <22>, <23>;
+				interrupt-names = "vring", "kick";
+			};
+
+			icssg1_mdio: mdio@b132400 {
+				compatible = "ti,davinci_mdio";
+				reg = <0xb132400 0x100>;
+				clocks = <&k3_clks 63 3>;
+				clock-names = "fck";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				bus_freq = <1000000>;
+				status = "disabled";
+			};
+		};
+	};
+
+	icssg_soc_bus2: pruss-soc-bus@b226004 {
+		compatible = "ti,am654-icssg-soc-bus";
+		reg = <0x00 0x0b226004 0x00 0x4>;
+		power-domains = <&k3_pds 64 TI_SCI_PD_EXCLUSIVE>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0x0b200000 0x00 0x0b200000 0x100000>;
+		dma-ranges;
+
+		icssg2: icssg@b200000 {
+			compatible = "ti,am654-icssg";
+			reg = <0xb200000 0x80000>;
+			interrupts = <GIC_SPI 270 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 271 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 272 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 273 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 274 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 275 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 276 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 277 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "host2", "host3", "host4",
+					  "host5", "host6", "host7",
+					  "host8", "host9";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+			dma-ranges;
+
+			ti,psil-base = <0x4300>;	/* ICSSG2 PSIL thread start */
+			ti,psil-config0 {
+				linux,udma-mode = <UDMA_PKT_MODE>;
+				statictr-type = <PSIL_STATIC_TR_NONE>;
+				ti,needs-epib;
+				ti,psd-size = <16>;
+			};
+
+			ti,psil-config1 {
+				linux,udma-mode = <UDMA_PKT_MODE>;
+				statictr-type = <PSIL_STATIC_TR_NONE>;
+				ti,needs-epib;
+				ti,psd-size = <16>;
+			};
+
+			ti,psil-config2 {
+				linux,udma-mode = <UDMA_PKT_MODE>;
+				statictr-type = <PSIL_STATIC_TR_NONE>;
+				ti,needs-epib;
+				ti,psd-size = <16>;
+			};
+
+			ti,psil-config3 {
+				linux,udma-mode = <UDMA_PKT_MODE>;
+				statictr-type = <PSIL_STATIC_TR_NONE>;
+				ti,needs-epib;
+				ti,psd-size = <16>;
+			};
+
+			ti,psil-config4 {
+				linux,udma-mode = <UDMA_PKT_MODE>;
+				statictr-type = <PSIL_STATIC_TR_NONE>;
+				ti,needs-epib;
+				ti,psd-size = <16>;
+			};
+
+			ti,psil-config5 {
+				linux,udma-mode = <UDMA_PKT_MODE>;
+				statictr-type = <PSIL_STATIC_TR_NONE>;
+				ti,needs-epib;
+				ti,psd-size = <16>;
+			};
+
+			ti,psil-config6 {
+				linux,udma-mode = <UDMA_PKT_MODE>;
+				statictr-type = <PSIL_STATIC_TR_NONE>;
+				ti,needs-epib;
+				ti,psd-size = <16>;
+			};
+
+			ti,psil-config7 {
+				linux,udma-mode = <UDMA_PKT_MODE>;
+				statictr-type = <PSIL_STATIC_TR_NONE>;
+				ti,needs-epib;
+				ti,psd-size = <16>;
+			};
+
+			icssg2_mem: memories@b200000 {
+				reg = <0xb200000 0x2000>,
+				      <0xb202000 0x2000>,
+				      <0xb210000 0x10000>;
+				reg-names = "dram0", "dram1",
+					    "shrdram2";
+			};
+
+			icssg2_cfg: cfg@b226000 {
+				compatible = "syscon";
+				reg = <0xb226000 0x200>;
+			};
+
+			icssg2_coreclk_mux: coreclk_mux {
+				#clock-cells = <0>;
+				clocks = <&k3_clks 64 19>, /* icssg1_core_clk */
+					 <&k3_clks 64 3>;  /* icssg1_iclk */
+				assigned-clocks = <&icssg2_coreclk_mux>;
+				assigned-clock-parents = <&k3_clks 64 3>;
+			};
+
+			icssg2_iepclk_mux: iepclk_mux {
+				#clock-cells = <0>;
+				clocks = <&k3_clks 64 10>,	/* icssg1_iep_clk */
+					 <&icssg2_coreclk_mux>;	/* core_clk */
+				assigned-clocks = <&icssg2_iepclk_mux>;
+				assigned-clock-parents = <&icssg2_coreclk_mux>;
+			};
+
+			icssg2_iep0: iep@b22e000 {
+				compatible = "syscon";
+				reg = <0xb22e000 0x1000>;
+				clocks = <&icssg2_iepclk_mux>;
+			};
+
+			icssg2_iep1: iep@b22f000 {
+				compatible = "syscon";
+				reg = <0xb22f000 0x1000>;
+				clocks = <&icssg2_iepclk_mux>;
+			};
+
+			icssg2_mii_rt: mii-rt@b232000 {
+				compatible = "syscon";
+				reg = <0xb232000 0x100>;
+			};
+
+			icssg2_mii_g_rt: mii-g-rt@b233000 {
+				compatible = "syscon";
+				reg = <0xb233000 0x1000>;
+			};
+
+			icssg2_intc: interrupt-controller@b220000 {
+				compatible = "ti,am654-icssg-intc";
+				reg = <0xb220000 0x2000>;
+				interrupt-controller;
+				#interrupt-cells = <1>;
+			};
+
+			pru2_0: pru@b234000 {
+				compatible = "ti,am654-pru";
+				reg = <0xb234000 0x4000>,
+				      <0xb222000 0x100>,
+				      <0xb222400 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "am65x-pru2_0-fw";
+				interrupt-parent = <&icssg2_intc>;
+				interrupts = <16>, <17>;
+				interrupt-names = "vring", "kick";
+			};
+
+			rtu2_0: rtu@b204000 {
+				compatible = "ti,am654-rtu";
+				reg = <0xb204000 0x2000>,
+				      <0xb223000 0x100>,
+				      <0xb223400 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "am65x-rtu2_0-fw";
+				interrupt-parent = <&icssg2_intc>;
+				interrupts = <20>, <21>;
+				interrupt-names = "vring", "kick";
+			};
+
+			pru2_1: pru@b238000 {
+				compatible = "ti,am654-pru";
+				reg = <0xb238000 0x4000>,
+				      <0xb224000 0x100>,
+				      <0xb224400 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "am65x-pru2_1-fw";
+				interrupt-parent = <&icssg2_intc>;
+				interrupts = <18>, <19>;
+				interrupt-names = "vring", "kick";
+			};
+
+			rtu2_1: rtu@b206000 {
+				compatible = "ti,am654-rtu";
+				reg = <0xb206000 0x2000>,
+				      <0xb223800 0x100>,
+				      <0xb223c00 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "am65x-rtu2_1-fw";
+				interrupt-parent = <&icssg2_intc>;
+				interrupts = <22>, <23>;
+				interrupt-names = "vring", "kick";
+			};
+
+			icssg2_mdio: mdio@b232400 {
+				compatible = "ti,davinci_mdio";
+				reg = <0xb232400 0x100>;
+				clocks = <&k3_clks 64 3>;
+				clock-names = "fck";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				bus_freq = <1000000>;
+				status = "disabled";
+			};
+		};
+	};
+
+	ecap0: pwm@3100000 {
+		compatible = "ti,am654-ecap", "ti,am3352-ecap";
+		#pwm-cells = <3>;
+		reg = <0x0 0x03100000 0x0 0x60>;
+		power-domains = <&k3_pds 39 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 39 0>;
+		clock-names = "fck";
+	};
+
+	scm_conf: scm_conf@100000 {
+		compatible = "syscon", "simple-mfd";
+		reg = <0 0x00100000 0 0x1c000>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0x0 0x0 0x00100000 0x1c000>;
+	};
+
+	dss: dss@04a00000 {
+		compatible = "ti,am6-dss";
+		reg =	<0x0 0x04a00000 0x0 0x1000>, /* common */
+			<0x0 0x04a02000 0x0 0x1000>, /* vidl1 */
+			<0x0 0x04a06000 0x0 0x1000>, /* vid */
+			<0x0 0x04a07000 0x0 0x1000>, /* ovr1 */
+			<0x0 0x04a08000 0x0 0x1000>, /* ovr2 */
+			<0x0 0x04a0a000 0x0 0x1000>, /* vp1 */
+			<0x0 0x04a0b000 0x0 0x1000>; /* vp2 */
+		reg-names = "common", "vidl1", "vid",
+			"ovr1", "ovr2", "vp1", "vp2";
+
+		syscon = <&scm_conf>;
+
+		power-domains = <&k3_pds 67 TI_SCI_PD_EXCLUSIVE>;
+
+		clocks =	<&k3_clks 67 1>,
+				<&k3_clks 216 1>,
+				<&k3_clks 67 2>;
+		clock-names = "fck", "vp1", "vp2";
+
+		/*
+		 * Set vp2 clk (DPI_1_IN_CLK) mux to PLL4 via
+		 * DIV1. See "Figure 12-3365. DSS Integration"
+		 * in AM65x TRM for details.
+		 */
+		assigned-clocks = <&k3_clks 67 2>;
+		assigned-clock-parents = <&k3_clks 67 5>;
+
+		interrupts = <GIC_SPI 166 IRQ_TYPE_LEVEL_HIGH>;
+
+		status = "disabled";
+
+		dss_ports: ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+	};
+
+	cal: cal@6f03000 {
+		compatible = "ti,am654-cal";
+		reg = <0x0 0x06f03000 0x0 0x400>,
+		      <0x0 0x06f03800 0x0 0x40>;
+		reg-names = "cal_top",
+			    "cal_rx_core0";
+		interrupts = <GIC_SPI 152 IRQ_TYPE_LEVEL_HIGH>;
+		syscon-camerrx = <&scm_conf 0x40c0>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clock-names = "fck";
+		clocks = <&k3_clks 2 0>;
+		power-domains = <&k3_pds 2 TI_SCI_PD_EXCLUSIVE>;
+		status = "disabled";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			csi2_0: port@0 {
+				reg = <0>;
+			};
+		};
+	};
+
+	sdhci0: sdhci@4f80000 {
+		compatible = "ti,am654-sdhci-5.1";
+		reg = <0x0 0x4f80000 0x0 0x260>, <0x0 0x4f90000 0x0 0x134>;
+		power-domains = <&k3_pds 47 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 47 0>, <&k3_clks 47 1>;
+		clock-names = "clk_ahb", "clk_xin";
+		interrupts = <GIC_SPI 136 IRQ_TYPE_LEVEL_HIGH>;
+		mmc-ddr-1_8v;
+		mmc-hs200-1_8v;
+		ti,otap-del-sel-legacy = <0x0>;
+		ti,otap-del-sel-mmc-hs = <0x0>;
+		ti,otap-del-sel-sd-hs = <0x0>;
+		ti,otap-del-sel-sdr12 = <0x0>;
+		ti,otap-del-sel-sdr25 = <0x0>;
+		ti,otap-del-sel-sdr50 = <0x8>;
+		ti,otap-del-sel-sdr104 = <0x5>;
+		ti,otap-del-sel-ddr50 = <0x5>;
+		ti,otap-del-sel-ddr52 = <0x5>;
+		ti,otap-del-sel-hs200 = <0x5>;
+		ti,otap-del-sel-hs400 = <0x0>;
+		ti,trm-icp = <0x8>;
+		dma-coherent;
+	};
+
+	sdhci1: sdhci@4fa0000 {
+		compatible = "ti,am654-sdhci-5.1";
+		reg = <0x0 0x4fa0000 0x0 0x260>, <0x0 0x4fb0000 0x0 0x134>;
+		power-domains = <&k3_pds 48 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 48 0>, <&k3_clks 48 1>;
+		clock-names = "clk_ahb", "clk_xin";
+		interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>;
+		ti,otap-del-sel-legacy = <0x0>;
+		ti,otap-del-sel-mmc-hs = <0x0>;
+		ti,otap-del-sel-sd-hs = <0x0>;
+		ti,otap-del-sel-sdr12 = <0x0>;
+		ti,otap-del-sel-sdr25 = <0x0>;
+		ti,otap-del-sel-sdr50 = <0x8>;
+		ti,otap-del-sel-sdr104 = <0x7>;
+		ti,otap-del-sel-ddr50 = <0x4>;
+		ti,otap-del-sel-ddr52 = <0x4>;
+		ti,otap-del-sel-hs200 = <0x7>;
+		ti,trm-icp = <0x8>;
+		no-1-8-v;
+		dma-coherent;
+	};
+
+	dwc3_0: dwc3@4000000 {
+		compatible = "ti,am654-dwc3";
+		reg = <0x0 0x4000000 0x0 0x4000>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0x0 0x0 0x4000000 0x20000>;
+		interrupts = <GIC_SPI 97 IRQ_TYPE_LEVEL_HIGH>;
+		dma-coherent;
+		power-domains = <&k3_pds 151 TI_SCI_PD_EXCLUSIVE>;
+		assigned-clocks = <&k3_clks 151 2>, <&k3_clks 151 7>;
+		assigned-clock-parents = <&k3_clks 151 4>,	/* set REF_CLK to 20MHz i.e. PER0_PLL/48 */
+					 <&k3_clks 151 9>;	/* set PIPE3_TXB_CLK to CLK_12M_RC/256 (for HS only) */
+
+		usb0: usb@10000 {
+			compatible = "snps,dwc3";
+			reg = <0x10000 0x10000>;
+			interrupts = <GIC_SPI 100 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 100 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "peripheral",
+					  "host",
+					  "otg";
+			maximum-speed = "high-speed";
+			dr_mode = "otg";
+			phys = <&usb0_phy>;
+			phy-names = "usb2-phy";
+			snps,dis_u3_susphy_quirk;
+		};
+	};
+
+	usb0_phy: phy@4100000 {
+		compatible = "ti,am654-usb2", "ti,omap-usb2";
+		reg = <0x0 0x4100000 0x0 0x54>;
+		syscon-phy-power = <&scm_conf 0x4000>;
+		clocks = <&k3_clks 151 0>, <&k3_clks 151 1>;
+		clock-names = "wkupclk", "refclk";
+		#phy-cells = <0>;
+		ti,dis-chg-det-quirk;
+	};
+
+	dwc3_1: dwc3@4020000 {
+		compatible = "ti,am654-dwc3";
+		reg = <0x0 0x4020000 0x0 0x4000>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0x0 0x0 0x4020000 0x20000>;
+		interrupts = <GIC_SPI 117 IRQ_TYPE_LEVEL_HIGH>;
+		dma-coherent;
+		power-domains = <&k3_pds 152 TI_SCI_PD_EXCLUSIVE>;
+		assigned-clocks = <&k3_clks 152 2>;
+		assigned-clock-parents = <&k3_clks 152 4>;	/* set REF_CLK to 20MHz i.e. PER0_PLL/48 */
+
+		usb1: usb@10000 {
+			compatible = "snps,dwc3";
+			reg = <0x10000 0x10000>;
+			interrupts = <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 116 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "peripheral",
+					  "host",
+					  "otg";
+			maximum-speed = "high-speed";
+			dr_mode = "otg";
+			phys = <&usb1_phy>;
+			phy-names = "usb2-phy";
+		};
+	};
+
+	usb1_phy: phy@4110000 {
+		compatible = "ti,am654-usb2", "ti,omap-usb2";
+		reg = <0x0 0x4110000 0x0 0x54>;
+		syscon-phy-power = <&scm_conf 0x4020>;
+		clocks = <&k3_clks 152 0>, <&k3_clks 152 1>;
+		clock-names = "wkupclk", "refclk";
+		#phy-cells = <0>;
+		ti,dis-chg-det-quirk;
+	};
+
+	main_spi0: spi@2100000 {
+		compatible = "ti,am654-mcspi","ti,omap4-mcspi";
+		reg = <0x0 0x2100000 0x0 0x400>;
+		interrupts = <GIC_SPI 184 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&k3_clks 137 1>;
+		power-domains = <&k3_pds 137 TI_SCI_PD_EXCLUSIVE>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dmas = <&main_udmap &pdma1 0 UDMA_DIR_TX>,
+		       <&main_udmap &pdma1 0 UDMA_DIR_RX>;
+		dma-names = "tx0", "rx0";
+	};
+
+	main_spi1: spi@2110000 {
+		compatible = "ti,am654-mcspi","ti,omap4-mcspi";
+		reg = <0x0 0x2110000 0x0 0x400>;
+		interrupts = <GIC_SPI 185 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&k3_clks 138 1>;
+		power-domains = <&k3_pds 138 TI_SCI_PD_EXCLUSIVE>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		assigned-clocks = <&k3_clks 137 1>;
+		assigned-clock-rates = <48000000>;
+	};
+
+	main_spi2: spi@2120000 {
+		compatible = "ti,am654-mcspi","ti,omap4-mcspi";
+		reg = <0x0 0x2120000 0x0 0x400>;
+		interrupts = <GIC_SPI 186 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&k3_clks 139 1>;
+		power-domains = <&k3_pds 139 TI_SCI_PD_EXCLUSIVE>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+	};
+
+	main_spi3: spi@2130000 {
+		compatible = "ti,am654-mcspi","ti,omap4-mcspi";
+		reg = <0x0 0x2130000 0x0 0x400>;
+		interrupts = <GIC_SPI 187 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&k3_clks 140 1>;
+		power-domains = <&k3_pds 140 TI_SCI_PD_EXCLUSIVE>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+	};
+
+	main_spi4: spi@2140000 {
+		compatible = "ti,am654-mcspi","ti,omap4-mcspi";
+		reg = <0x0 0x2140000 0x0 0x400>;
+		interrupts = <GIC_SPI 188 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&k3_clks 141 1>;
+		power-domains = <&k3_pds 141 TI_SCI_PD_EXCLUSIVE>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+	};
+
+	pcie0_rc: pcie@5500000 {
+		compatible = "ti,am654-pcie-rc";
+		reg =  <0x0 0x5500000 0x0 0x1000>, <0x0 0x5501000 0x0 0x1000>, <0x0 0x10000000 0x0 0x2000>, <0x0 0x5506000 0x0 0x1000>;
+		reg-names = "app", "dbics", "config", "atu";
+		power-domains = <&k3_pds 120 TI_SCI_PD_EXCLUSIVE>;
+		#address-cells = <3>;
+		#size-cells = <2>;
+		ranges = <0x81000000 0 0          0x0 0x10020000 0 0x00010000
+			  0x82000000 0 0x10030000 0x0 0x10030000 0 0x07FD0000>;
+		ti,syscon-pcie-id = <&pcie_devid>;
+		ti,syscon-pcie-mode = <&pcie0_mode>;
+		bus-range = <0x0 0xff>;
+		device_type = "pci";
+		num-lanes = <1>;
+		num-ob-windows = <16>;
+		num-viewport = <16>;
+		max-link-speed = <3>;
+		dma-coherent;
+		interrupts = <GIC_SPI 340 IRQ_TYPE_EDGE_RISING>;
+		#interrupt-cells = <1>;
+		interrupt-map-mask = <0 0 0 7>;
+		interrupt-map = <0 0 0 1 &pcie0_intc 0>, /* INT A */
+				<0 0 0 2 &pcie0_intc 0>, /* INT B */
+				<0 0 0 3 &pcie0_intc 0>, /* INT C */
+				<0 0 0 4 &pcie0_intc 0>; /* INT D */
+		msi-map = <0x0 &gic_its 0x0 0x10000>;
+		status = "disabled";
+
+		pcie0_intc: legacy-interrupt-controller@1 {
+			interrupt-controller;
+			#interrupt-cells = <1>;
+			interrupt-parent = <&gic500>;
+			interrupts = <GIC_SPI 328 IRQ_TYPE_EDGE_RISING>;
+		};
+	};
+
+	pcie0_ep: pcie-ep@5500000 {
+		compatible = "ti,am654-pcie-ep";
+		reg =  <0x0 0x5500000 0x0 0x1000>, <0x0 0x5501000 0x0 0x1000>, <0x0 0x10000000 0x0 0x8000000>, <0x0 0x5506000 0x0 0x1000>;
+		reg-names = "app", "dbics", "addr_space", "atu";
+		power-domains = <&k3_pds 120 TI_SCI_PD_EXCLUSIVE>;
+		ti,syscon-pcie-mode = <&pcie0_mode>;
+		num-lanes = <1>;
+		num-ib-windows = <16>;
+		num-ob-windows = <16>;
+		max-link-speed = <3>;
+		dma-coherent;
+		interrupts = <GIC_SPI 340 IRQ_TYPE_EDGE_RISING>;
+		status = "disabled";
+	};
+
+	pcie1_rc: pcie@5600000 {
+		compatible = "ti,am654-pcie-rc";
+		reg =  <0x0 0x5600000 0x0 0x1000>, <0x0 0x5601000 0x0 0x1000>, <0x0 0x18000000 0x0 0x2000>, <0x0 0x5606000 0x0 0x1000>;
+		reg-names = "app", "dbics", "config", "atu";
+		power-domains = <&k3_pds 121 TI_SCI_PD_EXCLUSIVE>;
+		#address-cells = <3>;
+		#size-cells = <2>;
+		ranges = <0x81000000 0 0          0x0   0x18020000 0 0x00010000
+			  0x82000000 0 0x18030000 0x0   0x18030000 0 0x07FD0000>;
+		ti,syscon-pcie-id = <&pcie_devid>;
+		ti,syscon-pcie-mode = <&pcie1_mode>;
+		bus-range = <0x0 0xff>;
+		status = "disabled";
+		device_type = "pci";
+		num-lanes = <1>;
+		num-ob-windows = <16>;
+		num-viewport = <16>;
+		max-link-speed = <3>;
+		dma-coherent;
+		interrupts = <GIC_SPI 355 IRQ_TYPE_EDGE_RISING>;
+		#interrupt-cells = <1>;
+		interrupt-map-mask = <0 0 0 7>;
+		interrupt-map = <0 0 0 1 &pcie1_intc 0 IRQ_TYPE_EDGE_RISING>, /* INT A */
+				<0 0 0 2 &pcie1_intc 0 IRQ_TYPE_EDGE_RISING>, /* INT B */
+				<0 0 0 3 &pcie1_intc 0 IRQ_TYPE_EDGE_RISING>, /* INT C */
+				<0 0 0 4 &pcie1_intc 0 IRQ_TYPE_EDGE_RISING>; /* INT D */
+		msi-map = <0x0 &gic_its 0x10000 0x10000>;
+
+		pcie1_intc: legacy-interrupt-controller@1 {
+			interrupt-controller;
+			#interrupt-cells = <2>;
+			interrupt-parent = <&gic500>;
+			interrupts = <GIC_SPI 343 IRQ_TYPE_EDGE_RISING>;
+		};
+	};
+
+	pcie1_ep: pcie-ep@5600000 {
+		compatible = "ti,am654-pcie-ep";
+		reg =  <0x0 0x5600000 0x0 0x1000>, <0x0 0x5601000 0x0 0x1000>, <0x0 0x18000000 0x0 0x4000000>, <0x0 0x5606000 0x0 0x1000>;
+		reg-names = "app", "dbics", "addr_space", "atu";
+		power-domains = <&k3_pds 121 TI_SCI_PD_EXCLUSIVE>;
+		ti,syscon-pcie-mode = <&pcie1_mode>;
+		num-lanes = <1>;
+		num-ib-windows = <16>;
+		num-ob-windows = <16>;
+		max-link-speed = <3>;
+		dma-coherent;
+		interrupts = <GIC_SPI 355 IRQ_TYPE_EDGE_RISING>;
+		status = "disabled";
+	};
+
+	timesync_router: timesync_router@A40000 {
+		compatible = "pinctrl-single";
+		reg = <0x0 0xA40000 0x0 0x800>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		#pinctrl-cells = <1>;
+		pinctrl-single,register-width = <32>;
+		pinctrl-single,function-mask = <0x800007ff>;
+	};
+
+	ctrlmmr_epwm_ctrl: syscon@104140{
+		compatible = "syscon", "simple-mfd";
+		reg = <0x0 0x104140 0x0 0x18>;
+		ranges = <0x0 0x0 0x104140 0x18>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		ehrpwm_tbclk: clk@0 {
+			compatible = "ti,am654-ehrpwm-tbclk";
+			reg = <0x0 0x18>;
+			#clock-cells = <1>;
+		};
+	};
+
+	ehrpwm0: pwm@3000000 {
+		compatible = "ti,am654-ehrpwm", "ti,am3352-ehrpwm";
+		#pwm-cells = <3>;
+		reg = <0x0 0x3000000 0x0 0x100>;
+		power-domains = <&k3_pds 40 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&ehrpwm_tbclk 0>, <&k3_clks 40 0>;
+		clock-names = "tbclk", "fck";
+	};
+
+	ehrpwm1: pwm@3010000 {
+		compatible = "ti,am654-ehrpwm", "ti,am3352-ehrpwm";
+		#pwm-cells = <3>;
+		reg = <0x0 0x3010000 0x0 0x100>;
+		power-domains = <&k3_pds 41 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&ehrpwm_tbclk 1>, <&k3_clks 41 0>;
+		clock-names = "tbclk", "fck";
+	};
+
+	ehrpwm2: pwm@3020000 {
+		compatible = "ti,am654-ehrpwm", "ti,am3352-ehrpwm";
+		#pwm-cells = <3>;
+		reg = <0x0 0x3020000 0x0 0x100>;
+		power-domains = <&k3_pds 42 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&ehrpwm_tbclk 2>, <&k3_clks 42 0>;
+		clock-names = "tbclk", "fck";
+	};
+
+	ehrpwm3: pwm@3030000 {
+		compatible = "ti,am654-ehrpwm", "ti,am3352-ehrpwm";
+		#pwm-cells = <3>;
+		reg = <0x0 0x3030000 0x0 0x100>;
+		power-domains = <&k3_pds 43 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&ehrpwm_tbclk 3>, <&k3_clks 43 0>;
+		clock-names = "tbclk", "fck";
+	};
+
+	ehrpwm4: pwm@3040000 {
+		compatible = "ti,am654-ehrpwm", "ti,am3352-ehrpwm";
+		#pwm-cells = <3>;
+		reg = <0x0 0x3040000 0x0 0x100>;
+		power-domains = <&k3_pds 44 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&ehrpwm_tbclk 4>, <&k3_clks 44 0>;
+		clock-names = "tbclk", "fck";
+	};
+
+	ehrpwm5: pwm@3050000 {
+		compatible = "ti,am654-ehrpwm", "ti,am3352-ehrpwm";
+		#pwm-cells = <3>;
+		reg = <0x0 0x3050000 0x0 0x100>;
+		power-domains = <&k3_pds 45 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&ehrpwm_tbclk 5>, <&k3_clks 45 0>;
+		clock-names = "tbclk", "fck";
+	};
 };
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/k3-am65-mcu.dtsi b/arch/arm64/boot/dts/ti/k3-am65-mcu.dtsi
--- a/arch/arm64/boot/dts/ti/k3-am65-mcu.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/k3-am65-mcu.dtsi	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,442 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Source for AM6 SoC Family MCU Domain peripherals
+ *
+ * Copyright (C) 2016-2018 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+&cbass_mcu {
+	mcu_conf: scm_conf@40f00000 {
+		compatible = "syscon", "simple-mfd";
+		reg = <0x0 0x40f00000 0x0 0x20000>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0x0 0x0 0x40f00000 0x20000>;
+
+		phy_sel: cpsw-phy-sel@4040 {
+			compatible = "ti,am654-cpsw-phy-sel";
+			reg = <0x4040 0x4>;
+			reg-names = "gmii-sel";
+		};
+	};
+
+	mcu_uart0: serial@40a00000 {
+		compatible = "ti,am654-uart";
+			reg = <0x00 0x40a00000 0x00 0x100>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			interrupts = <GIC_SPI 565 IRQ_TYPE_LEVEL_HIGH>;
+			clock-frequency = <96000000>;
+			current-speed = <115200>;
+			power-domains = <&k3_pds 149 TI_SCI_PD_EXCLUSIVE>;
+			dmas = <&mcu_udmap &mcu_pdma1 18 UDMA_DIR_TX>,
+			       <&mcu_udmap &mcu_pdma1 18 UDMA_DIR_RX>;
+			dma-names = "tx", "rx";
+	};
+
+	mcu_ram: mcu-ram@41c00000 {
+		compatible = "mmio-sram";
+		reg = <0x00 0x41c00000 0x00 0x80000>;
+		ranges = <0x0 0x00 0x41c00000 0x80000>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		mcu_r5fss0_core0_sram: r5f-sram@0 {
+			reg = <0x0 0x40000>;
+		};
+	};
+
+	mcu_i2c0: i2c@40b00000 {
+		compatible = "ti,am654-i2c", "ti,omap4-i2c";
+		reg = <0x0 0x40b00000 0x0 0x100>;
+		interrupts = <GIC_SPI 564 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clock-names = "fck";
+		clocks = <&k3_clks 114 1>;
+		power-domains = <&k3_pds 114 TI_SCI_PD_EXCLUSIVE>;
+	};
+
+	mcu_navss: mcu_navss {
+		compatible = "simple-bus";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		dma-coherent;
+		dma-ranges;
+		ranges;
+
+		ti,sci-dev-id = <119>;
+
+		mcu_ringacc: ringacc@2b800000 {
+			compatible = "ti,am654-navss-ringacc";
+			reg =	<0x0 0x2b800000 0x0 0x400000>,
+				<0x0 0x2b000000 0x0 0x400000>,
+				<0x0 0x28590000 0x0 0x100>,
+				<0x0 0x2a500000 0x0 0x40000>;
+			reg-names = "rt", "fifos", "proxy_gcfg", "proxy_target";
+			ti,num-rings = <286>;
+			ti,sci-rm-range-gp-rings = <0x2>; /* GP ring range */
+			ti,dma-ring-reset-quirk;
+			ti,sci = <&dmsc>;
+			ti,sci-dev-id = <195>;
+			interrupt-parent = <&main_udmass_inta>;
+		};
+
+		mcu_udmap: udmap@31150000 {
+			compatible = "ti,am654-navss-mcu-udmap";
+			reg =	<0x0 0x285c0000 0x0 0x100>,
+				<0x0 0x2a800000 0x0 0x40000>,
+				<0x0 0x2aa00000 0x0 0x40000>;
+			reg-names = "gcfg", "rchanrt", "tchanrt";
+			#dma-cells = <3>;
+
+			ti,ringacc = <&mcu_ringacc>;
+			ti,psil-base = <0x6000>;
+
+			ti,sci = <&dmsc>;
+			ti,sci-dev-id = <194>;
+
+			ti,sci-rm-range-tchan = <0x1>, /* TX_HCHAN */
+						<0x2>; /* TX_CHAN */
+			ti,sci-rm-range-rchan = <0x3>, /* RX_HCHAN */
+						<0x4>; /* RX_CHAN */
+			ti,sci-rm-range-rflow = <0x5>; /* GP RFLOW */
+
+			interrupt-parent = <&main_udmass_inta>;
+		};
+	};
+
+	mcu_pdma0: pdma@40710000 {
+		compatible = "ti,am654-pdma";
+		reg = <0x0 0x40710000 0x0 0x400>;
+		reg-names = "eccaggr_cfg";
+
+		ti,psil-base = <0x7100>;
+
+		/* ti,psil-config0-3 */
+		UDMA_PDMA_TR_XY(0);
+		UDMA_PDMA_TR_XY(1);
+		UDMA_PDMA_TR_XY(2);
+		UDMA_PDMA_TR_XY(3);
+	};
+
+	mcu_pdma1: pdma@40711000 {
+		compatible = "ti,am654-pdma";
+		reg = <0x0 0x40711000 0x0 0x400>;
+		reg-names = "eccaggr_cfg";
+
+		ti,psil-base = <0x7200>;
+
+		/* ti,psil-config18 */
+		UDMA_PDMA_PKT_XY(18);
+	};
+
+	mcu_cpsw: ethernet@046000000 {
+		compatible = "ti,am654-cpsw-nuss";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		reg = <0x0 0x46000000 0x0 0x200000>;
+		reg-names = "cpsw_nuss";
+		ranges;
+		dma-coherent;
+		clocks = <&k3_clks 5 10>;
+		clock-names = "fck";
+		power-domains = <&k3_pds 5 TI_SCI_PD_EXCLUSIVE>;
+		ti,psil-base = <0x7000>;
+		cpsw-phy-sel = <&phy_sel>;
+
+		interrupt-parent = <&main_udmass_inta>;
+
+		dmas = <&mcu_udmap &mcu_cpsw 0 UDMA_DIR_TX>,
+		       <&mcu_udmap &mcu_cpsw 1 UDMA_DIR_TX>,
+		       <&mcu_udmap &mcu_cpsw 2 UDMA_DIR_TX>,
+		       <&mcu_udmap &mcu_cpsw 3 UDMA_DIR_TX>,
+		       <&mcu_udmap &mcu_cpsw 4 UDMA_DIR_TX>,
+		       <&mcu_udmap &mcu_cpsw 5 UDMA_DIR_TX>,
+		       <&mcu_udmap &mcu_cpsw 6 UDMA_DIR_TX>,
+		       <&mcu_udmap &mcu_cpsw 7 UDMA_DIR_TX>,
+		       <&mcu_udmap &mcu_cpsw 0 UDMA_DIR_RX>;
+		dma-names = "tx0", "tx1", "tx2", "tx3",
+			    "tx4", "tx5", "tx6", "tx7",
+			    "rx";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			host: host@0 {
+				reg = <0>;
+				ti,label = "host";
+			};
+
+			cpsw_port1: port@1 {
+				reg = <1>;
+				ti,mac-only;
+				ti,label = "port1";
+				ti,syscon-efuse = <&mcu_conf 0x200>;
+			};
+		};
+
+		davinci_mdio: mdio {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			bus_freq = <1000000>;
+		};
+
+		cpts {
+			clocks = <&mcu_cpsw_cpts_mux>;
+			clock-names = "cpts";
+			interrupts-extended = <&gic500 GIC_SPI 570 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "cpts";
+			ti,cpts-ext-ts-inputs = <4>;
+			ti,cpts-periodic-outputs = <2>;
+
+			mcu_cpsw_cpts_mux: cpts_refclk_mux {
+				#clock-cells = <0>;
+				clocks = <&k3_clks 118 5>, <&k3_clks 118 11>,
+					<&k3_clks 157 91>, <&k3_clks 157 77>,
+					<&k3_clks 157 102>, <&k3_clks 157 80>,
+					<&k3_clks 120 3>, <&k3_clks 121 3>;
+				assigned-clocks = <&mcu_cpsw_cpts_mux>;
+				assigned-clock-parents = <&k3_clks 118 5>;
+			};
+		};
+
+		ti,psil-config0 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+
+		ti,psil-config1 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+
+		ti,psil-config2 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+
+		ti,psil-config3 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+
+		ti,psil-config4 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+
+		ti,psil-config5 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+
+		ti,psil-config6 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+
+		ti,psil-config7 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+	};
+
+	mcu_spi0: spi@40300000 {
+		compatible = "ti,am654-mcspi","ti,omap4-mcspi";
+		reg = <0x0 0x40300000 0x0 0x400>;
+		interrupts = <GIC_SPI 560 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&k3_clks 142 1>;
+		power-domains = <&k3_pds 142 TI_SCI_PD_EXCLUSIVE>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+	};
+
+	mcu_spi1: spi@40310000 {
+		compatible = "ti,am654-mcspi","ti,omap4-mcspi";
+		reg = <0x0 0x40310000 0x0 0x400>;
+		interrupts = <GIC_SPI 561 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&k3_clks 143 1>;
+		power-domains = <&k3_pds 143 TI_SCI_PD_EXCLUSIVE>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+	};
+
+	mcu_spi2: spi@40320000 {
+		compatible = "ti,am654-mcspi","ti,omap4-mcspi";
+		reg = <0x0 0x40320000 0x0 0x400>;
+		interrupts = <GIC_SPI 562 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&k3_clks 144 1>;
+		power-domains = <&k3_pds 144 TI_SCI_PD_EXCLUSIVE>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+	};
+
+	tscadc0: tscadc@40200000 {
+		compatible = "ti,am654-tscadc", "ti,am3359-tscadc";
+		reg = <0x0 0x40200000 0x0 0x1000>;
+		interrupts = <GIC_SPI 580 IRQ_TYPE_LEVEL_HIGH>;
+		power-domains = <&k3_pds 0 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 0 2>;
+		assigned-clocks = <&k3_clks 0 2>;
+		assigned-clock-rates = <60000000>;
+		clock-names = "adc_tsc_fck";
+		dmas = <&mcu_udmap &mcu_pdma0 0 UDMA_DIR_RX>,
+			<&mcu_udmap &mcu_pdma0 1 UDMA_DIR_RX>;
+		dma-names = "fifo0", "fifo1";
+
+		adc {
+			#io-channel-cells = <1>;
+			compatible = "ti,am654-adc", "ti,am3359-adc";
+		};
+	};
+
+	tscadc1: tscadc@40210000 {
+		compatible = "ti,am654-tscadc", "ti,am3359-tscadc";
+		reg = <0x0 0x40210000 0x0 0x1000>;
+		interrupts = <GIC_SPI 581 IRQ_TYPE_LEVEL_HIGH>;
+		power-domains = <&k3_pds 1 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 1 2>;
+		assigned-clocks = <&k3_clks 1 2>;
+		assigned-clock-rates = <60000000>;
+		clock-names = "adc_tsc_fck";
+		dmas = <&mcu_udmap &mcu_pdma0 2 UDMA_DIR_RX>,
+			<&mcu_udmap &mcu_pdma0 3 UDMA_DIR_RX>;
+		dma-names = "fifo0", "fifo1";
+
+		adc {
+			#io-channel-cells = <1>;
+			compatible = "ti,am654-adc", "ti,am3359-adc";
+		};
+	};
+
+	mcu_r5fss0: r5fss@41000000 {
+		compatible = "ti,am654-r5fss";
+		lockstep-mode = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0x41000000 0x00 0x41000000 0x20000>,
+			 <0x41400000 0x00 0x41400000 0x20000>;
+		power-domains = <&k3_pds 129 TI_SCI_PD_EXCLUSIVE>;
+
+		mcu_r5fss0_core0: r5f@41000000 {
+			compatible = "ti,am654-r5f";
+			reg = <0x41000000 0x00008000>,
+			      <0x41010000 0x00008000>;
+			reg-names = "atcm", "btcm";
+			ti,sci = <&dmsc>;
+			ti,sci-dev-id = <159>;
+			ti,sci-proc-ids = <0x01 0xFF>;
+			resets = <&k3_reset 159 1>;
+			atcm-enable = <1>;
+			btcm-enable = <1>;
+			loczrama = <1>;
+			mboxes = <&mailbox0_cluster0 &mbox_mcu_r5fss0_core0>;
+			sram = <&mcu_r5fss0_core0_sram>;
+		};
+
+		mcu_r5fss0_core1: r5f@41400000 {
+			compatible = "ti,am654-r5f";
+			reg = <0x41400000 0x00008000>,
+			      <0x41410000 0x00008000>;
+			reg-names = "atcm", "btcm";
+			ti,sci = <&dmsc>;
+			ti,sci-dev-id = <245>;
+			ti,sci-proc-ids = <0x02 0xFF>;
+			resets = <&k3_reset 245 1>;
+			atcm-enable = <1>;
+			btcm-enable = <1>;
+			loczrama = <1>;
+			mboxes = <&mailbox0_cluster1 &mbox_mcu_r5fss0_core1>;
+		};
+	};
+
+	fss: fss@47000000 {
+		compatible = "simple-bus";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		ospi0: spi@47040000 {
+			compatible = "ti,am654-ospi", "cdns,qspi-nor";
+			reg = <0x0 0x47040000 0x0 0x100>,
+				<0x5 0x00000000 0x1 0x0000000>;
+			interrupts = <GIC_SPI 552 IRQ_TYPE_LEVEL_HIGH>;
+			cdns,fifo-depth = <256>;
+			cdns,fifo-width = <4>;
+			cdns,trigger-address = <0x50000000>;
+			clocks = <&k3_clks 248 0>;
+			assigned-clocks = <&k3_clks 248 0>;
+			assigned-clock-parents = <&k3_clks 248 2>;
+			assigned-clock-rates = <166666666>;
+			power-domains = <&k3_pds 248 TI_SCI_PD_EXCLUSIVE>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			dma-coherent;
+		};
+
+		ospi1: spi@47050000 {
+			compatible = "ti,am654-ospi", "cdns,qspi-nor";
+			reg = <0x0 0x47050000 0x0 0x100>,
+				<0x7 0x00000000 0x1 0x00000000>;
+			interrupts = <GIC_SPI 553 IRQ_TYPE_LEVEL_HIGH>;
+			cdns,fifo-depth = <256>;
+			cdns,fifo-width = <4>;
+			cdns,trigger-address = <0x58000000>;
+			clocks = <&k3_clks 249 6>;
+			power-domains = <&k3_pds 249 TI_SCI_PD_EXCLUSIVE>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			dma-coherent;
+		};
+	};
+
+	m_can0: mcan@40528000 {
+		compatible = "bosch,m_can";
+		reg = <0x0 0x40528000 0x0 0x400>,
+		      <0x0 0x40500000 0x0 0x4400>;
+		reg-names = "m_can", "message_ram";
+		power-domains = <&k3_pds 102 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 102 0>, <&k3_clks 102 5>;
+		clock-names = "cclk", "hclk";
+		interrupt-parent = <&gic500>;
+		interrupts = <GIC_SPI 544 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 545 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "int0", "int1";
+		bosch,mram-cfg = <0x0 0 0 32 0 0 1 1>;
+		status = "disabled";
+	};
+
+	m_can1: mcan@40568000 {
+		compatible = "bosch,m_can";
+		reg = <0x0 0x40568000 0x0 0x400>,
+		      <0x0 0x40540000 0x0 0x4400>;
+		reg-names = "m_can", "message_ram";
+		power-domains = <&k3_pds 103 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 103 0>, <&k3_clks 103 5>;
+		clock-names = "cclk", "hclk";
+		interrupt-parent = <&gic500>;
+		interrupts = <GIC_SPI 547 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 548 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "int0", "int1";
+		bosch,mram-cfg = <0x0 0 0 32 0 0 1 1>;
+		status = "disabled";
+	};
+};
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/k3-am65-wakeup.dtsi b/arch/arm64/boot/dts/ti/k3-am65-wakeup.dtsi
--- a/arch/arm64/boot/dts/ti/k3-am65-wakeup.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/k3-am65-wakeup.dtsi	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,105 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Source for AM6 SoC Family Wakeup Domain peripherals
+ *
+ * Copyright (C) 2016-2018 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+&cbass_wakeup {
+	dmsc: dmsc {
+		compatible = "ti,am654-sci";
+		ti,host-id = <12>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		mbox-names = "rx", "tx";
+
+		mboxes= <&secure_proxy_main 11>,
+			<&secure_proxy_main 13>;
+
+		k3_pds: power-controller {
+			compatible = "ti,sci-pm-domain";
+			#power-domain-cells = <2>;
+		};
+
+		k3_clks: clocks {
+			compatible = "ti,k2g-sci-clk";
+			#clock-cells = <2>;
+		};
+
+		k3_reset: reset-controller {
+			compatible = "ti,sci-reset";
+			#reset-cells = <2>;
+		};
+	};
+
+	wkup_pmx0: pinmux@4301c000 {
+		compatible = "pinctrl-single";
+		reg = <0x4301c000 0x118>;
+		#pinctrl-cells = <1>;
+		pinctrl-single,register-width = <32>;
+		pinctrl-single,function-mask = <0xffffffff>;
+	};
+
+	wkup_uart0: serial@42300000 {
+		compatible = "ti,am654-uart";
+		reg = <0x42300000 0x100>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		interrupts = <GIC_SPI 697 IRQ_TYPE_LEVEL_HIGH>;
+		clock-frequency = <48000000>;
+		current-speed = <115200>;
+		power-domains = <&k3_pds 150 TI_SCI_PD_EXCLUSIVE>;
+	};
+
+	wkup_intr: interrupt-controller2 {
+		compatible = "ti,sci-intr";
+		interrupt-controller;
+		interrupt-parent = <&gic500>;
+		#interrupt-cells = <3>;
+		ti,sci = <&dmsc>;
+		ti,sci-dst-id = <56>;
+		ti,sci-rm-range-girq = <0x4>;
+	};
+
+	wkup_i2c0: i2c@42120000 {
+		compatible = "ti,am654-i2c", "ti,omap4-i2c";
+		reg = <0x42120000 0x100>;
+		interrupts = <GIC_SPI 696 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clock-names = "fck";
+		clocks = <&k3_clks 115 1>;
+		power-domains = <&k3_pds 115 TI_SCI_PD_EXCLUSIVE>;
+	};
+
+	wkup_gpio0: wkup_gpio0@42110000 {
+		compatible = "ti,k2g-gpio", "ti,keystone-gpio";
+		reg = <0x42110000 0x100>;
+		gpio-controller;
+		#gpio-cells = <2>;
+			interrupt-parent = <&wkup_intr>;
+		interrupts = <59 128 IRQ_TYPE_EDGE_RISING>,
+				<59 129 IRQ_TYPE_EDGE_RISING>,
+				<59 130 IRQ_TYPE_EDGE_RISING>,
+				<59 131 IRQ_TYPE_EDGE_RISING>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		ti,ngpio = <56>;
+		ti,davinci-gpio-unbanked = <0>;
+		clocks = <&k3_clks 59 0>;
+		clock-names = "gpio";
+	};
+
+	wkup_vtm0: wkup_vtm0@42050000 {
+		compatible = "ti,am654-vtm";
+		reg = <0x42050000 0x25c>;
+		power-domains = <&k3_pds 80 TI_SCI_PD_EXCLUSIVE>;
+		#thermal-sensor-cells = <1>;
+	};
+
+	thermal_zones: thermal-zones {
+		#include "am654-industrial-thermal.dtsi"
+	};
+};
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/k3-j721e-common-proc-board.dts b/arch/arm64/boot/dts/ti/k3-j721e-common-proc-board.dts
--- a/arch/arm64/boot/dts/ti/k3-j721e-common-proc-board.dts	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/k3-j721e-common-proc-board.dts	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,849 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+
+#include "k3-j721e-som-p0.dtsi"
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/net/ti-dp83867.h>
+#include <dt-bindings/pci/pci.h>
+#include <dt-bindings/sound/ti-mcasp.h>
+
+/ {
+	chosen {
+		stdout-path = "serial2:115200n8";
+		bootargs = "console=ttyS2,115200n8 earlycon=ns16550a,mmio32,0x02800000";
+	};
+
+	evm_12v0: fixedregulator-evm12v0 {
+		/* main supply */
+		compatible = "regulator-fixed";
+		regulator-name = "evm_12v0";
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vsys_3v3: fixedregulator-vsys3v3 {
+		/* Output of LMS140 */
+		compatible = "regulator-fixed";
+		regulator-name = "vsys_3v3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&evm_12v0>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vsys_5v0: fixedregulator-vsys5v0 {
+		/* Output of LM5140 */
+		compatible = "regulator-fixed";
+		regulator-name = "vsys_5v0";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&evm_12v0>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	sound0: sound@0 {
+		compatible = "ti,j721e-cpb-audio";
+		ti,model = "j721e-cpb-analog";
+
+		ti,cpb-mcasp = <&mcasp10>;
+		ti,cpb-codec = <&pcm3168a_1>;
+
+		pll4-rate = <1179648000>; /* Used for 48KHz family */
+		pll15-rate = <1083801600>; /* Used for 44.1KHz family */
+
+		clocks = <&k3_clks 184 1>,
+			 <&k3_clks 184 2>, <&k3_clks 184 4>,
+			 <&k3_clks 157 371>,
+			 <&k3_clks 157 400>, <&k3_clks 157 401>;
+		clock-names = "cpb-mcasp",
+			      "cpb-mcasp-48000", "cpb-mcasp-44100",
+			      "audio-refclk2",
+			      "audio-refclk2-48000", "audio-refclk2-44100";
+	};
+
+	gpio_keys: gpio-keys {
+		compatible = "gpio-keys";
+		autorepeat;
+		pinctrl-names = "default";
+		pinctrl-0 = <&sw10_button_pins_default &sw11_button_pins_default>;
+
+		sw10: sw10 {
+			label = "GPIO Key USER1";
+			linux,code = <BTN_0>;
+			gpios = <&main_gpio0 0 GPIO_ACTIVE_LOW>;
+		};
+
+		sw11: sw11 {
+			label = "GPIO Key USER2";
+			linux,code = <BTN_1>;
+			gpios = <&wkup_gpio0 7 GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	vdd_mmc1: fixedregulator-sd {
+		compatible = "regulator-fixed";
+		regulator-name = "vdd_mmc1";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+		enable-active-high;
+		vin-supply = <&vsys_3v3>;
+		gpio = <&exp2 2 GPIO_ACTIVE_HIGH>;
+	};
+
+	dp0: connector {
+		compatible = "dp-connector";
+		label = "DP0";
+
+		port {
+			dp_connector_in: endpoint {
+				remote-endpoint = <&dp_bridge_output>;
+			};
+		};
+	};
+
+	cpsw9g_virt_mac: main_r5fss_cpsw9g_virt_mac0 {
+		compatible = "ti,j721e-cpsw-virt-mac";
+		dma-coherent;
+		ti,psil-base = <0x4a00>;
+		ti,remote-name = "mpu_1_0_ethswitch-device-0";
+		interrupt-parent = <&main_udmass_inta>;
+
+		dmas = <&main_udmap &cpsw9g_virt_mac 0 UDMA_DIR_TX>,
+		       <&main_udmap &cpsw9g_virt_mac 1 UDMA_DIR_TX>,
+		       <&main_udmap &cpsw9g_virt_mac 2 UDMA_DIR_TX>,
+		       <&main_udmap &cpsw9g_virt_mac 3 UDMA_DIR_TX>,
+		       <&main_udmap &cpsw9g_virt_mac 4 UDMA_DIR_TX>,
+		       <&main_udmap &cpsw9g_virt_mac 5 UDMA_DIR_TX>,
+		       <&main_udmap &cpsw9g_virt_mac 6 UDMA_DIR_TX>,
+		       <&main_udmap &cpsw9g_virt_mac 0 UDMA_DIR_RX>;
+		dma-names = "tx0", "tx1", "tx2", "tx3",
+			    "tx4", "tx5", "tx6",
+			    "rx";
+
+		virt_emac_port {
+			ti,label = "virt-port";
+			/* local-mac-address = [0 0 0 0 0 0]; */
+		};
+
+		ti,psil-config0 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+
+		ti,psil-config1 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+
+		ti,psil-config2 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+
+		ti,psil-config3 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+
+		ti,psil-config4 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+
+		ti,psil-config5 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+
+		ti,psil-config6 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+
+		ti,psil-config7 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+	};
+
+	vdd_sd_dv_alt: gpio-regulator-TLV71033 {
+		compatible = "regulator-gpio";
+		pinctrl-names = "default";
+		pinctrl-0 = <&vdd_sd_dv_alt_pins_default>;
+		regulator-name = "tlv71033";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+		vin-supply = <&vsys_5v0>;
+		gpios = <&main_gpio0 117 GPIO_ACTIVE_HIGH>;
+		states = <1800000 0x0
+			  3300000 0x1>;
+	};
+};
+
+&wkup_pmx0 {
+	mcu_cpsw_pins_default: mcu_cpsw_pins_default {
+		pinctrl-single,pins = <
+			J721E_WKUP_IOPAD(0x0058, PIN_OUTPUT, 0) /* MCU_RGMII1_TX_CTL */
+			J721E_WKUP_IOPAD(0x005c, PIN_INPUT, 0) /* MCU_RGMII1_RX_CTL */
+			J721E_WKUP_IOPAD(0x0060, PIN_OUTPUT, 0) /* MCU_RGMII1_TD3 */
+			J721E_WKUP_IOPAD(0x0064, PIN_OUTPUT, 0) /* MCU_RGMII1_TD2 */
+			J721E_WKUP_IOPAD(0x0068, PIN_OUTPUT, 0) /* MCU_RGMII1_TD1 */
+			J721E_WKUP_IOPAD(0x006c, PIN_OUTPUT, 0) /* MCU_RGMII1_TD0 */
+			J721E_WKUP_IOPAD(0x0078, PIN_INPUT, 0) /* MCU_RGMII1_RD3 */
+			J721E_WKUP_IOPAD(0x007c, PIN_INPUT, 0) /* MCU_RGMII1_RD2 */
+			J721E_WKUP_IOPAD(0x0080, PIN_INPUT, 0) /* MCU_RGMII1_RD1 */
+			J721E_WKUP_IOPAD(0x0084, PIN_INPUT, 0) /* MCU_RGMII1_RD0 */
+			J721E_WKUP_IOPAD(0x0070, PIN_INPUT, 0) /* MCU_RGMII1_TXC */
+			J721E_WKUP_IOPAD(0x0074, PIN_INPUT, 0) /* MCU_RGMII1_RXC */
+		>;
+	};
+
+	mcu_mdio_pins_default: mcu_mdio1_pins_default {
+		pinctrl-single,pins = <
+			J721E_WKUP_IOPAD(0x008c, PIN_OUTPUT, 0) /* MCU_MDIO0_MDC */
+			J721E_WKUP_IOPAD(0x0088, PIN_INPUT, 0) /* MCU_MDIO0_MDIO */
+		>;
+	};
+
+	mcu_fss0_ospi1_pins_default: mcu-fss0-ospi1-pins-default {
+		pinctrl-single,pins = <
+			J721E_WKUP_IOPAD(0x34, PIN_OUTPUT, 0) /* (F22) MCU_OSPI1_CLK */
+			J721E_WKUP_IOPAD(0x50, PIN_OUTPUT, 0) /* (C22) MCU_OSPI1_CSn0 */
+			J721E_WKUP_IOPAD(0x40, PIN_INPUT, 0) /* (D22) MCU_OSPI1_D0 */
+			J721E_WKUP_IOPAD(0x44, PIN_INPUT, 0) /* (G22) MCU_OSPI1_D1 */
+			J721E_WKUP_IOPAD(0x48, PIN_INPUT, 0) /* (D23) MCU_OSPI1_D2 */
+			J721E_WKUP_IOPAD(0x4c, PIN_INPUT, 0) /* (C23) MCU_OSPI1_D3 */
+			J721E_WKUP_IOPAD(0x3c, PIN_INPUT, 0) /* (B23) MCU_OSPI1_DQS */
+			J721E_WKUP_IOPAD(0x38, PIN_INPUT, 0) /* (A23) MCU_OSPI1_LBCLKO */
+		>;
+	};
+
+	sw11_button_pins_default: sw11_button_pins_default {
+		pinctrl-single,pins = <
+			J721E_WKUP_IOPAD(0xcc, PIN_INPUT, 7) /* (G28) WKUP_GPIO0_7 */
+		>;
+	};
+};
+
+&wkup_uart0 {
+	/* Wakeup UART is used by System firmware */
+	status = "disabled";
+};
+
+&main_uart3 {
+	/* UART not brought out */
+	status = "disabled";
+};
+
+&main_uart5 {
+	/* UART not brought out */
+	status = "disabled";
+};
+
+&main_uart6 {
+	/* UART not brought out */
+	status = "disabled";
+};
+
+&main_uart7 {
+	/* UART not brought out */
+	status = "disabled";
+};
+
+&main_uart8 {
+	/* UART not brought out */
+	status = "disabled";
+};
+
+&main_uart9 {
+	/* UART not brought out */
+	status = "disabled";
+};
+
+&main_gpio2 {
+	status = "disabled";
+};
+
+&main_gpio3 {
+	status = "disabled";
+};
+
+&main_gpio4 {
+	status = "disabled";
+};
+
+&main_gpio5 {
+	status = "disabled";
+};
+
+&main_gpio6 {
+	status = "disabled";
+};
+
+&main_gpio7 {
+	status = "disabled";
+};
+
+&wkup_gpio1 {
+	status = "disabled";
+};
+
+&main_pmx0 {
+	main_i2c1_exp4_pins_default: main-i2c1-exp4-pins-default {
+		pinctrl-single,pins = <
+			J721E_IOPAD(0x230, PIN_INPUT, 7) /* (U2) ECAP0_IN_APWM_OUT.GPIO1_11 */
+		>;
+	};
+
+	main_i2c0_pins_default: main-i2c0-pins-default {
+		pinctrl-single,pins = <
+			J721E_IOPAD(0x220, PIN_INPUT_PULLUP, 0) /* (AC5) I2C0_SCL */
+			J721E_IOPAD(0x224, PIN_INPUT_PULLUP, 0) /* (AA5) I2C0_SDA */
+		>;
+	};
+
+	main_i2c1_pins_default: main-i2c1-pins-default {
+		pinctrl-single,pins = <
+			J721E_IOPAD(0x228, PIN_INPUT_PULLUP, 0) /* (Y6) I2C1_SCL */
+			J721E_IOPAD(0x22c, PIN_INPUT_PULLUP, 0) /* (AA6) I2C1_SDA */
+		>;
+	};
+
+	main_i2c3_pins_default: main-i2c3-pins-default {
+		pinctrl-single,pins = <
+			J721E_IOPAD(0x270, PIN_INPUT_PULLUP, 4) /* (T26) MMC2_CLK.I2C3_SCL */
+			J721E_IOPAD(0x274, PIN_INPUT_PULLUP, 4) /* (T25) MMC2_CMD.I2C3_SDA */
+		>;
+	};
+
+	main_i2c6_pins_default: main-i2c6-pins-default {
+		pinctrl-single,pins = <
+			J721E_IOPAD(0x1d0, PIN_INPUT_PULLUP, 2) /* (AA3) SPI0_D1.I2C6_SCL */
+			J721E_IOPAD(0x1e4, PIN_INPUT_PULLUP, 2) /* (Y2) SPI1_D1.I2C6_SDA */
+		>;
+	};
+
+	mcasp10_pins_default: mcasp10_pins_default {
+		pinctrl-single,pins = <
+			J721E_IOPAD(0x158, PIN_OUTPUT_PULLDOWN, 12) /* (U23) RGMII5_TX_CTL.MCASP10_ACLKX */
+			J721E_IOPAD(0x15c, PIN_OUTPUT_PULLDOWN, 12) /* (U26) RGMII5_RX_CTL.MCASP10_AFSX */
+			J721E_IOPAD(0x160, PIN_OUTPUT_PULLDOWN, 12) /* (V28) RGMII5_TD3.MCASP10_AXR0 */
+			J721E_IOPAD(0x164, PIN_OUTPUT_PULLDOWN, 12) /* (V29) RGMII5_TD2.MCASP10_AXR1 */
+			J721E_IOPAD(0x170, PIN_OUTPUT_PULLDOWN, 12) /* (U29) RGMII5_TXC.MCASP10_AXR2 */
+			J721E_IOPAD(0x174, PIN_OUTPUT_PULLDOWN, 12) /* (U25) RGMII5_RXC.MCASP10_AXR3 */
+			J721E_IOPAD(0x198, PIN_INPUT_PULLDOWN, 12) /* (V25) RGMII6_TD1.MCASP10_AXR4 */
+			J721E_IOPAD(0x19c, PIN_INPUT_PULLDOWN, 12) /* (W27) RGMII6_TD0.MCASP10_AXR5 */
+			J721E_IOPAD(0x1a0, PIN_INPUT_PULLDOWN, 12) /* (W29) RGMII6_TXC.MCASP10_AXR6 */
+		>;
+	};
+
+	audi_ext_refclk2_pins_default: audi_ext_refclk2_pins_default {
+		pinctrl-single,pins = <
+			J721E_IOPAD(0x1a4, PIN_OUTPUT, 3) /* (W26) RGMII6_RXC.AUDIO_EXT_REFCLK2 */
+		>;
+	};
+
+	main_usbss0_pins_default: main_usbss0_pins_default {
+		pinctrl-single,pins = <
+			J721E_IOPAD(0x290, PIN_OUTPUT, 0) /* (U6) USB0_DRVVBUS */
+			J721E_IOPAD(0x210, PIN_INPUT, 7) /* (W3) MCAN1_RX.GPIO1_3 */
+		>;
+	};
+
+	main_usbss1_pins_default: main_usbss1_pins_default {
+		pinctrl-single,pins = <
+			J721E_IOPAD(0x214, PIN_OUTPUT, 4) /* (V4) MCAN1_TX.USB1_DRVVBUS */
+		>;
+	};
+
+	main_mmc1_pins_default: main_mmc1_pins_default {
+		pinctrl-single,pins = <
+			J721E_IOPAD(0x254, PIN_INPUT, 0) /* (R29) MMC1_CMD */
+			J721E_IOPAD(0x250, PIN_INPUT, 0) /* (P25) MMC1_CLK */
+			J721E_IOPAD(0x2ac, PIN_INPUT, 0) /* (P25) MMC1_CLKLB */
+			J721E_IOPAD(0x24c, PIN_INPUT, 0) /* (R24) MMC1_DAT0 */
+			J721E_IOPAD(0x248, PIN_INPUT, 0) /* (P24) MMC1_DAT1 */
+			J721E_IOPAD(0x244, PIN_INPUT, 0) /* (R25) MMC1_DAT2 */
+			J721E_IOPAD(0x240, PIN_INPUT, 0) /* (R26) MMC1_DAT3 */
+			J721E_IOPAD(0x258, PIN_INPUT, 0) /* (P23) MMC1_SDCD */
+			J721E_IOPAD(0x25c, PIN_INPUT, 0) /* (R28) MMC1_SDWP */
+		>;
+	};
+
+	dp0_pins_default: dp0_pins_default {
+		pinctrl-single,pins = <
+			J721E_IOPAD(0x1c4, PIN_INPUT, 5) /* SPI0_CS1.DP0_HPD */
+		>;
+	};
+
+	sw10_button_pins_default: sw10_button_pins_default {
+		pinctrl-single,pins = <
+			J721E_IOPAD(0x0, PIN_INPUT, 7) /* (AC18) EXTINTn.GPIO0_0 */
+		>;
+	};
+
+	vdd_sd_dv_alt_pins_default: vdd_sd_dv_alt_pins_default {
+		pinctrl-single,pins = <
+			J721E_IOPAD(0x1d8, PIN_INPUT, 7) /* (W4) SPI1_CS1.GPIO0_117 */
+		>;
+	};
+};
+
+&dss {
+	status = "ok";
+};
+
+&dss_ports {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	port@0 {
+		reg = <0>;
+
+		dpi_out_real0: endpoint {
+			remote-endpoint = <&dp_bridge_input>;
+		};
+	};
+};
+
+&serdes_wiz4 {
+	lane0-mode = <PHY_TYPE_DP>;
+	lane1-mode = <PHY_TYPE_DP>;
+	lane2-mode = <PHY_TYPE_DP>;
+	lane3-mode = <PHY_TYPE_DP>;
+};
+
+&mhdp {
+	status = "ok";
+	pinctrl-names = "default";
+	pinctrl-0 = <&dp0_pins_default>;
+};
+
+&dp0_ports {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	port@0 {
+		reg = <0>;
+		dp_bridge_input: endpoint {
+			remote-endpoint = <&dpi_out_real0>;
+		};
+	};
+
+	port@1 {
+		reg = <1>;
+		dp_bridge_output: endpoint {
+			remote-endpoint = <&dp_connector_in>;
+		};
+	};
+};
+
+&main_i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_i2c0_pins_default>;
+	clock-frequency = <400000>;
+
+	exp1: gpio@20 {
+		compatible = "ti,tca6416";
+		reg = <0x20>;
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
+
+	exp2: gpio@22 {
+		compatible = "ti,tca6424";
+		reg = <0x22>;
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		p09 {
+			/* P11 - MCASP/TRACE_MUX_S0 */
+			gpio-hog;
+			gpios = <9 GPIO_ACTIVE_HIGH>;
+			output-low;
+			line-name = "MCASP/TRACE_MUX_S0";
+		};
+
+		p10 {
+			/* P12 - MCASP/TRACE_MUX_S1 */
+			gpio-hog;
+			gpios = <10 GPIO_ACTIVE_HIGH>;
+			output-high;
+			line-name = "MCASP/TRACE_MUX_S1";
+		};
+	};
+};
+
+&main_i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_i2c1_pins_default>;
+	clock-frequency = <400000>;
+
+	exp4: gpio@20 {
+		compatible = "ti,tca6408";
+		reg = <0x20>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&main_i2c1_exp4_pins_default>;
+		interrupt-parent = <&main_gpio1>;
+		interrupts = <11 IRQ_TYPE_EDGE_FALLING>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+
+		p0 {
+			/* P0 - DP0_PWR_SW_EN */
+			gpio-hog;
+			gpios = <0 GPIO_ACTIVE_HIGH>;
+			output-high;
+			line-name = "DP0_PWR_SW_EN";
+		};
+	};
+};
+
+&main_i2c3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_i2c3_pins_default>;
+	clock-frequency = <400000>;
+
+	exp3: gpio@20 {
+		compatible = "ti,tca6408";
+		reg = <0x20>;
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		p0 {
+			/* P0 - CODEC_RSTz */
+			gpio-hog;
+			gpios = <0 GPIO_ACTIVE_HIGH>;
+			output-high;
+			line-name = "CODEC_RSTz";
+		};
+	};
+
+	pcm3168a_1: audio-codec@44 {
+		compatible = "ti,pcm3168a";
+		reg = <0x44>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&audi_ext_refclk2_pins_default>;
+
+		#sound-dai-cells = <1>;
+
+		/* C_AUDIO_REFCLK2 -> RGMII6_RXC (W26) */
+		clocks = <&k3_clks 157 371>;
+		clock-names = "scki";
+
+		/* HSDIV3_16FFT_MAIN_4_HSDIVOUT2_CLK -> REFCLK2 */
+		assigned-clocks = <&k3_clks 157 371>;
+		assigned-clock-parents = <&k3_clks 157 400>;
+		assigned-clock-rates = <24576000>; /* for 48KHz */
+
+		VDD1-supply = <&vsys_3v3>;
+		VDD2-supply = <&vsys_3v3>;
+		VCCAD1-supply = <&vsys_5v0>;
+		VCCAD2-supply = <&vsys_5v0>;
+		VCCDA1-supply = <&vsys_5v0>;
+		VCCDA2-supply = <&vsys_5v0>;
+	};
+};
+
+&main_i2c6 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_i2c6_pins_default>;
+	clock-frequency = <400000>;
+
+	exp5: gpio@20 {
+		compatible = "ti,tca6408";
+		reg = <0x20>;
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
+};
+
+&mcu_cpsw {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcu_cpsw_pins_default &mcu_mdio_pins_default>;
+
+	cpts {
+		ti,pps = <3 1>;
+	};
+};
+
+&davinci_mdio {
+	phy0: ethernet-phy@0 {
+		reg = <0>;
+		ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
+		ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_4_B_NIB>;
+	};
+};
+
+&cpsw_port1 {
+	phy-mode = "rgmii-rxid";
+	phy-handle = <&phy0>;
+};
+
+#define TS_OFFSET(pa, val)     (0x4 + (pa) * 4) (0x10000 | val)
+
+&timesync_router {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcu_cpts>;
+
+	mcu_cpts: mcu_cpts {
+		pinctrl-single,pins = <
+			/* pps [cpts genf1] in17 -> out25 [cpts hw4_push] */
+			TS_OFFSET(25, 17)
+		>;
+	};
+};
+
+&ospi1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcu_fss0_ospi1_pins_default>;
+
+	flash@0{
+		compatible = "jedec,spi-nor";
+		reg = <0x0>;
+		spi-tx-bus-width = <1>;
+		spi-rx-bus-width = <4>;
+		spi-max-frequency = <100000000>;
+		cdns,tshsl-ns = <60>;
+		cdns,tsd2d-ns = <60>;
+		cdns,tchsh-ns = <60>;
+		cdns,tslch-ns = <60>;
+		cdns,read-delay = <2>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+	};
+};
+
+&mcasp10 {
+	#sound-dai-cells = <0>;
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcasp10_pins_default>;
+
+	op-mode = <0>;		/* MCASP_IIS_MODE */
+	tdm-slots = <2>;
+	auxclk-fs-ratio = <256>;
+
+	serial-dir = <	/* 0: INACTIVE, 1: TX, 2: RX */
+		1 1 1 1
+		2 2 2 0
+	>;
+	tx-num-evt = <0>;
+	rx-num-evt = <0>;
+
+	status = "okay";
+};
+
+&usb_serdes_mux {
+	idle-states = <1>, <0>; /* USB0 to SERDES3, USB1 to SERDES1 */
+};
+
+&serdes_ln_ctrl {
+	idle-states = <SERDES0_LANE0_PCIE0_LANE0>, <SERDES0_LANE1_PCIE0_LANE1>,
+		      <SERDES1_LANE0_PCIE1_LANE0>, <SERDES1_LANE1_PCIE1_LANE1>,
+		      <SERDES2_LANE0_PCIE2_LANE0>, <SERDES2_LANE1_PCIE2_LANE1>,
+		      <SERDES3_LANE0_USB3_0_SWAP>, <SERDES3_LANE1_USB3_0>,
+		      <SERDES4_LANE0_EDP_LANE0>, <SERDES4_LANE1_EDP_LANE1>, <SERDES4_LANE2_EDP_LANE2>, <SERDES4_LANE3_EDP_LANE3>;
+};
+
+&serdes_wiz3 {
+	typec-dir-gpios = <&main_gpio1 3 GPIO_ACTIVE_HIGH>;
+	typec-dir-debounce = <300>;	/* TUSB321, tCCB_DEFAULT 133 ms */
+	lane0-mode = <PHY_TYPE_USB3>;
+	lane1-mode = <PHY_TYPE_USB3>;
+};
+
+&serdes3 {
+	serdes3_usb_link: link@0 {
+		reg = <0>;
+		cdns,num-lanes = <2>;
+		#phy-cells = <0>;
+		cdns,phy-type = <PHY_TYPE_USB3>;
+		resets = <&serdes_wiz3 1>, <&serdes_wiz3 2>;
+	};
+};
+
+&usbss0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_usbss0_pins_default>;
+	ti,vbus-divider;
+};
+
+&usb0 {
+	dr_mode = "otg";
+	maximum-speed = "super-speed";
+	phys = <&serdes3_usb_link>;
+	phy-names = "cdns3,usb3-phy";
+};
+
+&usbss1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_usbss1_pins_default>;
+	ti,usb2-only;
+};
+
+&usb1 {
+	dr_mode = "host";
+	maximum-speed = "high-speed";
+};
+
+&main_sdhci0 {
+	/* eMMC */
+	non-removable;
+	ti,driver-strength-ohm = <50>;
+	disable-wp;
+};
+
+&main_sdhci1 {
+	/* SD/MMC */
+	vmmc-supply = <&vdd_mmc1>;
+	vqmmc-supply = <&vdd_sd_dv_alt>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_mmc1_pins_default>;
+	disable-wp;
+};
+
+&main_sdhci2 {
+	/* Unused */
+	status = "disabled";
+};
+
+&serdes_wiz0 {
+	lane0-mode = <PHY_TYPE_PCIE>;
+	lane1-mode = <PHY_TYPE_PCIE>;
+};
+
+&serdes0 {
+	serdes0_pcie_link: link@0 {
+		reg = <0>;
+		cdns,num-lanes = <1>;
+		#phy-cells = <0>;
+		cdns,phy-type = <PHY_TYPE_PCIE>;
+		resets = <&serdes_wiz0 1>;
+	};
+};
+
+&serdes_wiz1 {
+	lane0-mode = <PHY_TYPE_PCIE>;
+	lane1-mode = <PHY_TYPE_PCIE>;
+};
+
+&serdes1 {
+	serdes1_pcie_link: link@0 {
+		reg = <0>;
+		cdns,num-lanes = <2>;
+		#phy-cells = <0>;
+		cdns,phy-type = <PHY_TYPE_PCIE>;
+		resets = <&serdes_wiz1 1>, <&serdes_wiz1 2>;
+	};
+};
+
+&serdes_wiz2 {
+	lane0-mode = <PHY_TYPE_PCIE>;
+	lane1-mode = <PHY_TYPE_PCIE>;
+};
+
+&serdes2 {
+	serdes2_pcie_link: link@0 {
+		reg = <0>;
+		cdns,num-lanes = <2>;
+		#phy-cells = <0>;
+		cdns,phy-type = <PHY_TYPE_PCIE>;
+		resets = <&serdes_wiz2 1>, <&serdes_wiz2 2>;
+	};
+};
+
+&pcie0 {
+	pci-mode = <PCI_MODE_RC>;
+	num-lanes = <1>;
+};
+
+&pcie1 {
+	pci-mode = <PCI_MODE_RC>;
+	num-lanes = <2>;
+};
+
+&pcie2 {
+	pci-mode = <PCI_MODE_RC>;
+	num-lanes = <2>;
+};
+
+&pcie0_rc {
+	reset-gpios = <&exp1 6 GPIO_ACTIVE_HIGH>;
+	phys = <&serdes0_pcie_link>;
+	phy-names = "pcie_phy";
+};
+
+&pcie1_rc {
+	reset-gpios = <&exp1 2 GPIO_ACTIVE_HIGH>;
+	phys = <&serdes1_pcie_link>;
+	phy-names = "pcie_phy";
+};
+
+&pcie2_rc {
+	reset-gpios = <&exp2 20 GPIO_ACTIVE_HIGH>;
+	phys = <&serdes2_pcie_link>;
+	phy-names = "pcie_phy";
+};
+
+&tscadc0 {
+	adc {
+		ti,adc-channels = <0 1 2 3 4 5 6 7>;
+	};
+};
+
+&tscadc1 {
+	adc {
+		ti,adc-channels = <0 1 2 3 4 5 6 7>;
+	};
+};
+
+/* uart2 assigned to cpsw9g eth-switch fw running on remote CPU core */
+&main_uart2 {
+	status = "disabled";
+};
+
+&pcie0_ep {
+	phys = <&serdes0_pcie_link>;
+	phy-names = "pcie_phy";
+};
+
+&pcie1_ep {
+	phys = <&serdes1_pcie_link>;
+	phy-names = "pcie_phy";
+};
+
+&pcie2_ep {
+	phys = <&serdes2_pcie_link>;
+	phy-names = "pcie_phy";
+};
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/k3-j721e-common-proc-board-infotainment-display-sharing.dtso b/arch/arm64/boot/dts/ti/k3-j721e-common-proc-board-infotainment-display-sharing.dtso
--- a/arch/arm64/boot/dts/ti/k3-j721e-common-proc-board-infotainment-display-sharing.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/k3-j721e-common-proc-board-infotainment-display-sharing.dtso	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,114 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Source for J721E ASTC VLAB Model with DSS partitioning support
+ *
+ * Copyright (C) 2016-2018 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+#include <dt-bindings/soc/ti,sci_pm_domain.h>
+
+/dts-v1/;
+/plugin/;
+
+/* Used by RTOS to power-up TFP410 via expander */
+&main_i2c1 {
+	status = "disabled";
+};
+
+&dss {
+	pinctrl-names = "none"; /* pinmux configured by RTOS */
+	power-domains = <&k3_pds 152 TI_SCI_PD_SHARED>; /* share IP among VMs and RTOS */
+
+	/* No changes to parents or rates for VP clocks
+	 * if the VP is not owned by us
+	 */
+	assigned-clocks = <&k3_clks 152 1>,
+		<&k3_clks 152 9>,
+		<&k3_clks 152 13>;
+
+	assigned-clock-parents = <&k3_clks 152 2>,	/* PLL16_HSDIV0 */
+		<&k3_clks 152 11>,			/* PLL18_HSDIV0 */
+		<&k3_clks 152 18>;			/* PLL23_HSDIV0 */
+
+	/* partition information */
+	dss_planes: dss-planes {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		/* vid1, marshalled to us by RTOS */
+		plane@0 {
+			reg = <0>;
+			managed = <0>;
+		};
+
+		/* vidl1, Reserved for jailhouse inmate */
+		plane@1 {
+			reg = <1>;
+			managed = <0>;
+		};
+
+		/* vid2, owned by RTOS */
+		plane@2 {
+			reg = <2>;
+			managed = <0>;
+		};
+
+		/* vidl2, marshalled to us by RTOS */
+		plane@3 {
+			reg = <3>;
+			managed = <0>;
+		};
+	};
+
+	dss_vps: dss-vps {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		/* Owned by jailhouse inmate */
+		vp@0 {
+			reg = <0>;
+			managed = <0>;
+		};
+
+		/* Owned by RTOS */
+		vp@1 {
+			reg = <1>;
+			managed = <0>;
+		};
+
+		/* The 2 below are not owned by anyone
+		 * else, so keeping here
+		 */
+		vp@2 {
+			reg = <2>;
+			managed = <1>;
+		};
+
+		vp@3 {
+			reg = <3>;
+			managed = <1>;
+		};
+	};
+
+	dss_commons: dss-commons {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		interrupt-common {
+			reg = <1>;
+		};
+
+		config-common {
+			status = "disabled";
+			reg = <0>;
+		};
+	};
+
+	dss_remote: dss-remote {
+		#address-cells = <0>;
+		#size-cells = <0>;
+
+		remote-name = "r5f-tidss";
+	};
+};
+
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/k3-j721e-common-proc-board-infotainment.dtso b/arch/arm64/boot/dts/ti/k3-j721e-common-proc-board-infotainment.dtso
--- a/arch/arm64/boot/dts/ti/k3-j721e-common-proc-board-infotainment.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/k3-j721e-common-proc-board-infotainment.dtso	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,172 @@
+// SPDX-License-Identifier: GPL-2.0
+/**
+ * Infotainment Expansion Board for j721e-evm
+ *
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/pinctrl/k3.h>
+
+/ {
+  fragment@101 {
+	target-path = "/";
+
+	__overlay__ {
+		dvi-connector {
+			compatible = "hdmi-connector";
+			ddc-i2c-bus = <&main_i2c1>;
+			digital;
+
+			/* P12 - HDMI_HPD */
+			hpd-gpios = <&vout_exp 10 GPIO_ACTIVE_HIGH>;
+
+			port {
+				dvi_connector_in: endpoint {
+					remote-endpoint = <&tfp410_out>;
+				};
+			};
+		};
+
+		dvi-bridge {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "ti,tfp410";
+
+			/* P10 - HDMI_PDn */
+			powerdown-gpios = <&vout_exp 8 GPIO_ACTIVE_LOW>;
+
+			port@0 {
+				reg = <0>;
+
+				tfp410_in: endpoint {
+					remote-endpoint = <&dpi_out0>;
+					pclk-sample = <1>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				tfp410_out: endpoint {
+					remote-endpoint = <&dvi_connector_in>;
+				};
+			};
+		};
+	};
+  };
+};
+
+&main_pmx0 {
+	main_i2c1_vout_exp_pins_default: main_i2c1_vout_exp_pins_default {
+		pinctrl-single,pins = <
+			J721E_IOPAD(0x264, PIN_INPUT, 7) /* (T29) MMC2_DAT2.GPIO1_24 */
+		>;
+	};
+
+	dss_vout0_pins_default: dss_vout0_pins_default {
+		pinctrl-single,pins = <
+			J721E_IOPAD(0x58, PIN_OUTPUT, 10) /* (AE22) PRG1_PRU1_GPO0.VOUT0_DATA0 */
+			J721E_IOPAD(0x5c, PIN_OUTPUT, 10) /* (AG23) PRG1_PRU1_GPO1.VOUT0_DATA1 */
+			J721E_IOPAD(0x60, PIN_OUTPUT, 10) /* (AF23) PRG1_PRU1_GPO2.VOUT0_DATA2 */
+			J721E_IOPAD(0x64, PIN_OUTPUT, 10) /* (AD23) PRG1_PRU1_GPO3.VOUT0_DATA3 */
+			J721E_IOPAD(0x68, PIN_OUTPUT, 10) /* (AH24) PRG1_PRU1_GPO4.VOUT0_DATA4 */
+			J721E_IOPAD(0x6c, PIN_OUTPUT, 10) /* (AG21) PRG1_PRU1_GPO5.VOUT0_DATA5 */
+			J721E_IOPAD(0x70, PIN_OUTPUT, 10) /* (AE23) PRG1_PRU1_GPO6.VOUT0_DATA6 */
+			J721E_IOPAD(0x74, PIN_OUTPUT, 10) /* (AC21) PRG1_PRU1_GPO7.VOUT0_DATA7 */
+			J721E_IOPAD(0x78, PIN_OUTPUT, 10) /* (Y23)  PRG1_PRU1_GPO8.VOUT0_DATA8 */
+			J721E_IOPAD(0x7c, PIN_OUTPUT, 10) /* (AF21) PRG1_PRU1_GPO9.VOUT0_DATA9 */
+			J721E_IOPAD(0x80, PIN_OUTPUT, 10) /* (AB23) PRG1_PRU1_GPO10.VOUT0_DATA10 */
+			J721E_IOPAD(0x84, PIN_OUTPUT, 10) /* (AJ25) PRG1_PRU1_GPO11.VOUT0_DATA11 */
+			J721E_IOPAD(0x88, PIN_OUTPUT, 10) /* (AH25) PRG1_PRU1_GPO12.VOUT0_DATA12 */
+			J721E_IOPAD(0x8c, PIN_OUTPUT, 10) /* (AG25) PRG1_PRU1_GPO13.VOUT0_DATA13 */
+			J721E_IOPAD(0x90, PIN_OUTPUT, 10) /* (AH26) PRG1_PRU1_GPO14.VOUT0_DATA14 */
+			J721E_IOPAD(0x94, PIN_OUTPUT, 10) /* (AJ27) PRG1_PRU1_GPO15.VOUT0_DATA15 */
+			J721E_IOPAD(0x30, PIN_OUTPUT, 10) /* (AF24) PRG1_PRU0_GPO11.VOUT0_DATA16 */
+			J721E_IOPAD(0x34, PIN_OUTPUT, 10) /* (AJ24) PRG1_PRU0_GPO12.VOUT0_DATA17 */
+			J721E_IOPAD(0x38, PIN_OUTPUT, 10) /* (AG24) PRG1_PRU0_GPO13.VOUT0_DATA18 */
+			J721E_IOPAD(0x3c, PIN_OUTPUT, 10) /* (AD24) PRG1_PRU0_GPO14.VOUT0_DATA19 */
+			J721E_IOPAD(0x40, PIN_OUTPUT, 10) /* (AC24) PRG1_PRU0_GPO15.VOUT0_DATA20 */
+			J721E_IOPAD(0x44, PIN_OUTPUT, 10) /* (AE24) PRG1_PRU0_GPO16.VOUT0_DATA21 */
+			J721E_IOPAD(0x24, PIN_OUTPUT, 10) /* (AJ20) PRG1_PRU0_GPO8.VOUT0_DATA22 */
+			J721E_IOPAD(0x28, PIN_OUTPUT, 10) /* (AG20) PRG1_PRU0_GPO9.VOUT0_DATA23 */
+			J721E_IOPAD(0x9c, PIN_OUTPUT, 10) /* (AC22) PRG1_PRU1_GPO17.VOUT0_DE */
+			J721E_IOPAD(0x98, PIN_OUTPUT, 10) /* (AJ26) PRG1_PRU1_GPO16.VOUT0_HSYNC */
+			J721E_IOPAD(0xa4, PIN_OUTPUT, 10) /* (AH22) PRG1_PRU1_GPO19.VOUT0_PCLK */
+			J721E_IOPAD(0xa0, PIN_OUTPUT, 10) /* (AJ22) PRG1_PRU1_GPO18.VOUT0_VSYNC */
+		>;
+	};
+};
+
+&main_i2c1 {
+	/* i2c1 is used for DVI DDC, so we need to use 100kHz */
+	clock-frequency = <100000>;
+
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	vout_exp: gpio@21 {
+		compatible = "ti,tca6416";
+		reg = <0x21>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&main_i2c1_vout_exp_pins_default>;
+		interrupt-parent = <&main_gpio1>;
+		interrupts = <24 IRQ_TYPE_EDGE_FALLING>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+};
+
+&vout_exp {
+	p11 {
+		/* P11 - HDMI_DDC_OE */
+		gpio-hog;
+		gpios = <9 GPIO_ACTIVE_HIGH>;
+		output-high;
+		line-name = "HDMI_DDC_OE";
+	};
+};
+
+&exp1 {
+	p14 {
+		/* P14 - VINOUT_MUX_SEL0 */
+		gpio-hog;
+		gpios = <12 GPIO_ACTIVE_HIGH>;
+		output-low;
+		line-name = "VINOUT_MUX_SEL0";
+	};
+
+	p15 {
+		/* P15 - VINOUT_MUX_SEL1 */
+		gpio-hog;
+		gpios = <13 GPIO_ACTIVE_HIGH>;
+		output-high;
+		line-name = "VINOUT_MUX_SEL1";
+	};
+};
+
+&dss {
+	status = "ok";
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&dss_vout0_pins_default>;
+};
+
+&dss_ports {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	port@1 {
+		reg = <1>;
+
+		dpi_out0: endpoint {
+			remote-endpoint = <&tfp410_in>;
+		};
+	};
+};
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/k3-j721e-common-proc-board-jailhouse.dtso b/arch/arm64/boot/dts/ti/k3-j721e-common-proc-board-jailhouse.dtso
--- a/arch/arm64/boot/dts/ti/k3-j721e-common-proc-board-jailhouse.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/k3-j721e-common-proc-board-jailhouse.dtso	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,154 @@
+// SPDX-License-Identifier: GPL-2.0
+/**
+ * Infotainment Expansion Board for j721e-evm
+ *
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/soc/ti,sci_pm_domain.h>
+#include <dt-bindings/pinctrl/k3.h>
+#include <dt-bindings/gpio/gpio.h>
+
+/dts-v1/;
+/plugin/;
+
+&reserved_memory {
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	/* Reserved for GPU firmware */
+	gpu_fw_mem: gpu-fw-mem@c0000000 {
+		reg = <0x0 0xc0000000 0x0 0x2000000>;
+		no-map;
+	};
+
+	/* Reserved for Jailhouse, IVSHMEM, baremetal apps */
+	hyp_mem: jailhouse@0x89fa00000 {
+		reg = <0x8 0x9fa00000 0x0 0x600000>;
+		alignment = <0x1000>;
+		no-map;
+	};
+
+	/* Reserved for Linux inmate */
+	inmate_mem: inmate@0x8a0000000 {
+		reg = <0x8 0xa0000000 0x0 0x60000000>;
+		alignment = <0x1000>;
+		no-map;
+	};
+};
+
+&main_i2c4 {
+	status = "disabled";
+};
+
+&main_uart1 {
+	status = "disabled";
+};
+
+&d5520 {
+	status = "disabled";
+};
+
+&main_sdhci0 {
+	status = "disabled";
+};
+
+&serdes_wiz4 {
+	status = "disabled";
+};
+
+&mhdp {
+	status = "disabled";
+};
+
+&sw10 {
+	status = "disabled";
+};
+
+&dss {
+	power-domains = <&k3_pds 152 TI_SCI_PD_SHARED>; /* share IP among VMs and RTOS */
+
+	/* No changes to parents or rates for VP 0 clocks
+	 * if the VP is not owned by us
+	 */
+
+	assigned-clocks = <&k3_clks 152 4>,
+		<&k3_clks 152 9>,
+		<&k3_clks 152 13>;
+
+	assigned-clock-parents = <&k3_clks 152 6>,	/* PLL19_HSDIV0 */
+		<&k3_clks 152 11>,			/* PLL18_HSDIV0 */
+		<&k3_clks 152 18>;			/* PLL23_HSDIV0 */
+
+	dss_planes: dss-planes {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		/*vid1*/
+		plane@0 {
+			reg = <0>;
+			managed = <1>;
+		};
+
+		/*vidl1*/
+		plane@1 {
+			reg = <1>;
+			managed = <0>;
+		};
+
+		/*vid2*/
+		plane@2 {
+			reg = <2>;
+			managed = <1>;
+		};
+
+		/*vidl2*/
+		plane@3 {
+			reg = <3>;
+			managed = <1>;
+		};
+	};
+
+	dss_vps: dss-vps {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		vp@0 {
+			reg = <0>;
+			managed = <0>;
+		};
+
+		vp@1 {
+			reg = <1>;
+			managed = <1>;
+		};
+
+		vp@2 {
+			reg = <2>;
+			managed = <1>;
+		};
+
+		vp@3 {
+			reg = <3>;
+			managed = <1>;
+		};
+	};
+
+	dss_commons: dss-commons {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		interrupt-common {
+			reg = <0>;
+		};
+
+		config-common {
+			status = "okay";
+			reg = <0>;
+		};
+	};
+};
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/k3-j721e.dtsi b/arch/arm64/boot/dts/ti/k3-j721e.dtsi
--- a/arch/arm64/boot/dts/ti/k3-j721e.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/k3-j721e.dtsi	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,192 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Source for J721E SoC Family
+ *
+ * Copyright (C) 2016-2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/pinctrl/k3.h>
+#include <dt-bindings/soc/ti,sci_pm_domain.h>
+#include <dt-bindings/dma/k3-udma.h>
+
+/ {
+	model = "Texas Instruments K3 J721E SoC";
+	compatible = "ti,j721e";
+	interrupt-parent = <&gic500>;
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	aliases {
+		serial0 = &wkup_uart0;
+		serial1 = &mcu_uart0;
+		serial2 = &main_uart0;
+		serial3 = &main_uart1;
+		serial4 = &main_uart2;
+		serial5 = &main_uart3;
+		serial6 = &main_uart4;
+		serial7 = &main_uart5;
+		serial8 = &main_uart6;
+		serial9 = &main_uart7;
+		serial10 = &main_uart8;
+		serial11 = &main_uart9;
+		spi0 = &ospi0;
+		spi1 = &ospi1;
+		rproc0 = &mcu_r5fss0_core0;
+		rproc1 = &mcu_r5fss0_core1;
+		rproc2 = &main_r5fss0_core0;
+		rproc3 = &main_r5fss0_core1;
+		rproc4 = &main_r5fss1_core0;
+		rproc5 = &main_r5fss1_core1;
+		rproc6 = &c66_0;
+		rproc7 = &c66_1;
+		rproc8 = &c71_0;
+	};
+
+	chosen { };
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		cpu-map {
+			cluster0: cluster0 {
+				core0 {
+					cpu = <&cpu0>;
+				};
+
+				core1 {
+					cpu = <&cpu1>;
+				};
+			};
+
+		};
+
+		cpu0: cpu@0 {
+			compatible = "arm,cortex-a72";
+			reg = <0x000>;
+			device_type = "cpu";
+			enable-method = "psci";
+			i-cache-size = <0xC000>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <0x8000>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <128>;
+			next-level-cache = <&L2_0>;
+		};
+
+		cpu1: cpu@1 {
+			compatible = "arm,cortex-a72";
+			reg = <0x001>;
+			device_type = "cpu";
+			enable-method = "psci";
+			i-cache-size = <0xC000>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <0x8000>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <128>;
+			next-level-cache = <&L2_0>;
+		};
+	};
+
+	L2_0: l2-cache0 {
+		compatible = "cache";
+		cache-level = <2>;
+		cache-size = <0x100000>;
+		cache-line-size = <64>;
+		cache-sets = <2048>;
+		next-level-cache = <&msmc_l3>;
+	};
+
+	msmc_l3: l3-cache0 {
+		compatible = "cache";
+		cache-level = <3>;
+	};
+
+	firmware {
+		optee {
+			compatible = "linaro,optee-tz";
+			method = "smc";
+		};
+
+		psci: psci {
+			compatible = "arm,psci-1.0";
+			method = "smc";
+		};
+	};
+
+	a72_timer0: timer-cl0-cpu0 {
+		compatible = "arm,armv8-timer";
+		interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_LOW>, /* cntpsirq */
+			     <GIC_PPI 14 IRQ_TYPE_LEVEL_LOW>, /* cntpnsirq */
+			     <GIC_PPI 11 IRQ_TYPE_LEVEL_LOW>, /* cntvirq */
+			     <GIC_PPI 10 IRQ_TYPE_LEVEL_LOW>; /* cnthpirq */
+	};
+
+	pmu: pmu {
+		compatible = "arm,armv8-pmuv3";
+		/* Recommendation from GIC500 TRM Table A.3 */
+		interrupts = <GIC_PPI 7 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	cbass_main: interconnect@100000 {
+		compatible = "simple-bus";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges = <0x00 0x00100000 0x00 0x00100000 0x00 0x00020000>, /* ctrl mmr */
+			 <0x00 0x00600000 0x00 0x00600000 0x00 0x00031100>, /* GPIO */
+			 <0x00 0x00900000 0x00 0x00900000 0x00 0x00012000>, /* serdes */
+			 <0x00 0x00A40000 0x00 0x00A40000 0x00 0x00000800>, /* timesync router */
+			 <0x00 0x06000000 0x00 0x06000000 0x00 0x00400000>, /* USBSS0 */
+			 <0x00 0x06400000 0x00 0x06400000 0x00 0x00400000>, /* USBSS1 */
+			 <0x00 0x01000000 0x00 0x01000000 0x00 0x0af02400>, /* Most peripherals */
+			 <0x00 0x30800000 0x00 0x30800000 0x00 0x0bc00000>, /* MAIN NAVSS */
+			 <0x00 0x0d000000 0x00 0x0d000000 0x00 0x01800000>, /* PCIe Core*/
+			 <0x00 0x10000000 0x00 0x10000000 0x00 0x10000000>, /* PCIe DAT */
+			 <0x00 0x64800000 0x00 0x64800000 0x00 0x00800000>, /* C71 */
+			 <0x44 0x00000000 0x44 0x00000000 0x00 0x08000000>, /* PCIe2 DAT */
+			 <0x4d 0x80800000 0x4d 0x80800000 0x00 0x00800000>, /* C66_0 */
+			 <0x4d 0x81800000 0x4d 0x81800000 0x00 0x00800000>, /* C66_1 */
+			 <0x4e 0x20000000 0x4e 0x20000000 0x00 0x00080000>, /* GPU */
+			 /* MCUSS_WKUP Range */
+			 <0x00 0x28380000 0x00 0x28380000 0x00 0x03880000>,
+			 <0x00 0x40200000 0x00 0x40200000 0x00 0x00998400>,
+			 <0x00 0x40f00000 0x00 0x40f00000 0x00 0x00020000>,
+			 <0x00 0x41000000 0x00 0x41000000 0x00 0x00020000>,
+			 <0x00 0x41400000 0x00 0x41400000 0x00 0x00020000>,
+			 <0x00 0x41c00000 0x00 0x41c00000 0x00 0x00100000>,
+			 <0x00 0x42040000 0x00 0x42040000 0x00 0x03ac2400>,
+			 <0x00 0x45100000 0x00 0x45100000 0x00 0x00c24000>,
+			 <0x00 0x46000000 0x00 0x46000000 0x00 0x00200000>,
+			 <0x00 0x47000000 0x00 0x47000000 0x00 0x00068400>,
+			 <0x00 0x50000000 0x00 0x50000000 0x00 0x10000000>,
+			 <0x00 0x70000000 0x00 0x70000000 0x00 0x00800000>,
+			 <0x05 0x00000000 0x05 0x00000000 0x01 0x00000000>,
+			 <0x07 0x00000000 0x07 0x00000000 0x01 0x00000000>;
+
+		cbass_mcu_wakeup: interconnect@28380000 {
+			compatible = "simple-bus";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges = <0x00 0x28380000 0x00 0x28380000 0x00 0x03880000>, /* MCU NAVSS*/
+				 <0x00 0x40200000 0x00 0x40200000 0x00 0x00998400>, /* First peripheral window */
+				 <0x00 0x40f00000 0x00 0x40f00000 0x00 0x00020000>, /* CTRL_MMR0 */
+				 <0x00 0x41000000 0x00 0x41000000 0x00 0x00020000>, /* MCU R5F Core0 */
+				 <0x00 0x41400000 0x00 0x41400000 0x00 0x00020000>, /* MCU R5F Core1 */
+				 <0x00 0x41c00000 0x00 0x41c00000 0x00 0x00100000>, /* MCU SRAM */
+				 <0x00 0x42040000 0x00 0x42040000 0x00 0x03ac2400>, /* WKUP peripheral window */
+				 <0x00 0x45100000 0x00 0x45100000 0x00 0x00c24000>, /* MMRs, remaining NAVSS */
+				 <0x00 0x46000000 0x00 0x46000000 0x00 0x00200000>, /* CPSW */
+				 <0x00 0x47000000 0x00 0x47000000 0x00 0x00068400>, /* OSPI register space */
+				 <0x00 0x50000000 0x00 0x50000000 0x00 0x10000000>, /* FSS OSPI0/1 data region 0 */
+				 <0x05 0x00000000 0x05 0x00000000 0x01 0x00000000>, /* FSS OSPI0 data region 3 */
+				 <0x07 0x00000000 0x07 0x00000000 0x01 0x00000000>; /* FSS OSPI1 data region 3*/
+		};
+	};
+};
+
+/* Now include the peripherals for each bus segments */
+#include "k3-j721e-main.dtsi"
+#include "k3-j721e-mcu-wakeup.dtsi"
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/k3-j721e-main.dtsi b/arch/arm64/boot/dts/ti/k3-j721e-main.dtsi
--- a/arch/arm64/boot/dts/ti/k3-j721e-main.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/k3-j721e-main.dtsi	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,2510 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Source for J721E SoC Family Main Domain peripherals
+ *
+ * Copyright (C) 2016-2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+#include <dt-bindings/phy/phy.h>
+#include <dt-bindings/mux/mux.h>
+#include <dt-bindings/mux/mux-j721e-wiz.h>
+
+&cbass_main {
+	msmc_ram: sram@70000000 {
+		compatible = "mmio-sram";
+		reg = <0x0 0x70000000 0x0 0x800000>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0x0 0x0 0x70000000 0x800000>;
+
+		atf-sram@0 {
+			reg = <0x0 0x20000>;
+		};
+	};
+
+	scm_conf: scm_conf@100000 {
+		compatible = "syscon", "simple-mfd";
+		reg = <0 0x00100000 0 0x1c000>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0x0 0x0 0x00100000 0x1c000>;
+
+		pcie0_ctrl: pcie-ctrl@4070 {
+			compatible = "syscon";
+			reg = <0x00004070 0x4>;
+		};
+
+		pcie1_ctrl: pcie-ctrl@4074 {
+			compatible = "syscon";
+			reg = <0x00004074 0x4>;
+		};
+
+		pcie2_ctrl: pcie-ctrl@4078 {
+			compatible = "syscon";
+			reg = <0x00004078 0x4>;
+		};
+
+		serdes_ln_ctrl: serdes_ln_ctrl@4080 {
+			compatible = "mmio-mux";
+			#mux-control-cells = <1>;
+			mux-reg-masks = <0x4080 0x3>, <0x4084 0x3>, /* SERDES0 lane0/1 select */
+					<0x4090 0x3>, <0x4094 0x3>, /* SERDES1 lane0/1 select */
+					<0x40a0 0x3>, <0x40a4 0x3>, /* SERDES2 lane0/1 select */
+					<0x40b0 0x3>, <0x40b4 0x3>, /* SERDES3 lane0/1 select */
+					<0x40c0 0x3>, <0x40c4 0x3>, <0x40c8 0x3>, <0x40cc 0x3>;
+					/* SERDES4 lane0/1/2/3 select */
+			idle-states = <SERDES0_LANE0_PCIE0_LANE0>, <SERDES0_LANE1_PCIE0_LANE1>,
+				      <SERDES1_LANE0_PCIE1_LANE0>, <SERDES1_LANE1_PCIE1_LANE1>,
+				      <SERDES2_LANE0_PCIE2_LANE0>, <SERDES2_LANE1_PCIE2_LANE1>,
+				      <MUX_IDLE_AS_IS>, <SERDES3_LANE1_USB3_0>,
+				      <SERDES4_LANE0_EDP_LANE0>, <SERDES4_LANE1_EDP_LANE1>, <SERDES4_LANE2_EDP_LANE2>, <SERDES4_LANE3_EDP_LANE3>;
+		};
+
+		usb_serdes_mux: mux-controller@4000 {
+			compatible = "mmio-mux";
+			#mux-control-cells = <1>;
+			mux-reg-masks = <0x4000 0x8000000>, /* USB0 to SERDES0/3 mux */
+					<0x4010 0x8000000>; /* USB1 to SERDES1/2 mux */
+	    };
+	};
+
+	gic500: interrupt-controller@1800000 {
+		compatible = "arm,gic-v3";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		#interrupt-cells = <3>;
+		interrupt-controller;
+		reg = <0x00 0x01800000 0x00 0x10000>,	/* GICD */
+		      <0x00 0x01900000 0x00 0x100000>;	/* GICR */
+
+		/* vcpumntirq: virtual CPU interface maintenance interrupt */
+		interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
+
+		gic_its: gic-its@1820000 {
+			compatible = "arm,gic-v3-its";
+			reg = <0x00 0x01820000 0x00 0x10000>;
+			socionext,synquacer-pre-its = <0x1000000 0x400000>;
+			msi-controller;
+			#msi-cells = <1>;
+		};
+	};
+
+	smmu0: smmu@36600000 {
+		compatible = "arm,smmu-v3";
+		reg = <0x0 0x36600000 0x0 0x100000>;
+		power-domains = <&k3_pds 229 TI_SCI_PD_EXCLUSIVE>;
+		interrupt-parent = <&gic500>;
+		interrupts = <GIC_SPI 772 IRQ_TYPE_EDGE_RISING>,
+			     <GIC_SPI 768 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "eventq", "gerror";
+		#iommu-cells = <1>;
+		dma-coherent;
+	};
+
+	main_gpio_intr: interrupt-controller0 {
+		compatible = "ti,sci-intr";
+		interrupt-controller;
+		interrupt-parent = <&gic500>;
+		#interrupt-cells = <3>;
+		ti,sci = <&dmsc>;
+		ti,sci-dst-id = <14>;
+		ti,sci-rm-range-girq = <0x1>;
+	};
+
+	cbass_main_navss: interconnect0 {
+		compatible = "simple-bus";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		dma-coherent;
+		dma-ranges;
+		ranges;
+
+		ti,sci-dev-id = <199>;
+
+		main_navss_intr: interrupt-controller1 {
+			compatible = "ti,sci-intr";
+			interrupt-controller;
+			interrupt-parent = <&gic500>;
+			#interrupt-cells = <3>;
+			ti,sci = <&dmsc>;
+			ti,sci-dst-id = <14>;
+			ti,sci-rm-range-girq = <0>, <2>;
+		};
+
+		main_udmass_inta: interrupt-controller@33d00000 {
+			compatible = "ti,sci-inta";
+			reg = <0x0 0x33d00000 0x0 0x100000>;
+			interrupt-controller;
+			interrupt-parent = <&main_navss_intr>;
+			#interrupt-cells = <3>;
+			ti,sci = <&dmsc>;
+			ti,sci-dev-id = <209>;
+			ti,sci-rm-range-vint = <0xa>;
+			ti,sci-rm-range-global-event = <0xd>;
+		};
+
+		hwspinlock: spinlock@30e00000 {
+			compatible = "ti,am654-hwspinlock";
+			reg = <0x00 0x30e00000 0x00 0x1000>;
+			#hwlock-cells = <1>;
+		};
+
+		mailbox0_cluster0: mailbox@31f80000 {
+			compatible = "ti,am654-mailbox";
+			reg = <0x00 0x31f80000 0x00 0x200>;
+			#mbox-cells = <1>;
+			ti,mbox-num-users = <4>;
+			ti,mbox-num-fifos = <16>;
+			interrupt-parent = <&main_navss_intr>;
+			interrupts = <214 0 IRQ_TYPE_LEVEL_HIGH>;
+
+			mbox_mcu_r5fss0_core0: mbox-mcu-r5fss0-core0 {
+				ti,mbox-rx = <0 0 0>;
+				ti,mbox-tx = <1 0 0>;
+			};
+
+			mbox_mcu_r5fss0_core1: mbox-mcu-r5fss0-core1 {
+				ti,mbox-rx = <2 0 0>;
+				ti,mbox-tx = <3 0 0>;
+			};
+		};
+
+		mailbox0_cluster1: mailbox@31f81000 {
+			compatible = "ti,am654-mailbox";
+			reg = <0x00 0x31f81000 0x00 0x200>;
+			#mbox-cells = <1>;
+			ti,mbox-num-users = <4>;
+			ti,mbox-num-fifos = <16>;
+			interrupt-parent = <&main_navss_intr>;
+			interrupts = <215 0 IRQ_TYPE_LEVEL_HIGH>;
+
+			mbox_main_r5fss0_core0: mbox-main-r5fss0-core0 {
+				ti,mbox-rx = <0 0 0>;
+				ti,mbox-tx = <1 0 0>;
+			};
+
+			mbox_main_r5fss0_core1: mbox-main-r5fss0-core1 {
+				ti,mbox-rx = <2 0 0>;
+				ti,mbox-tx = <3 0 0>;
+			};
+		};
+
+		mailbox0_cluster2: mailbox@31f82000 {
+			compatible = "ti,am654-mailbox";
+			reg = <0x00 0x31f82000 0x00 0x200>;
+			#mbox-cells = <1>;
+			ti,mbox-num-users = <4>;
+			ti,mbox-num-fifos = <16>;
+			interrupt-parent = <&main_navss_intr>;
+			interrupts = <216 0 IRQ_TYPE_LEVEL_HIGH>;
+
+			mbox_main_r5fss1_core0: mbox-main-r5fss1-core0 {
+				ti,mbox-rx = <0 0 0>;
+				ti,mbox-tx = <1 0 0>;
+			};
+
+			mbox_main_r5fss1_core1: mbox-main-r5fss1-core1 {
+				ti,mbox-rx = <2 0 0>;
+				ti,mbox-tx = <3 0 0>;
+			};
+		};
+
+		mailbox0_cluster3: mailbox@31f83000 {
+			compatible = "ti,am654-mailbox";
+			reg = <0x00 0x31f83000 0x00 0x200>;
+			#mbox-cells = <1>;
+			ti,mbox-num-users = <4>;
+			ti,mbox-num-fifos = <16>;
+			interrupt-parent = <&main_navss_intr>;
+			interrupts = <217 0 IRQ_TYPE_LEVEL_HIGH>;
+
+			mbox_c66_0: mbox-c66-0 {
+				ti,mbox-rx = <0 0 0>;
+				ti,mbox-tx = <1 0 0>;
+			};
+
+			mbox_c66_1: mbox-c66-1 {
+				ti,mbox-rx = <2 0 0>;
+				ti,mbox-tx = <3 0 0>;
+			};
+		};
+
+		mailbox0_cluster4: mailbox@31f84000 {
+			compatible = "ti,am654-mailbox";
+			reg = <0x00 0x31f84000 0x00 0x200>;
+			#mbox-cells = <1>;
+			ti,mbox-num-users = <4>;
+			ti,mbox-num-fifos = <16>;
+			interrupt-parent = <&main_navss_intr>;
+			interrupts = <218 0 IRQ_TYPE_LEVEL_HIGH>;
+
+			mbox_c71_0: mbox-c71-0 {
+				ti,mbox-rx = <0 0 0>;
+				ti,mbox-tx = <1 0 0>;
+			};
+		};
+
+		mailbox0_cluster5: mailbox@31f85000 {
+			compatible = "ti,am654-mailbox";
+			reg = <0x00 0x31f85000 0x00 0x200>;
+			#mbox-cells = <1>;
+			ti,mbox-num-users = <4>;
+			ti,mbox-num-fifos = <16>;
+			status = "disabled";
+		};
+
+		mailbox0_cluster6: mailbox@31f86000 {
+			compatible = "ti,am654-mailbox";
+			reg = <0x00 0x31f86000 0x00 0x200>;
+			#mbox-cells = <1>;
+			ti,mbox-num-users = <4>;
+			ti,mbox-num-fifos = <16>;
+			status = "disabled";
+		};
+
+		mailbox0_cluster7: mailbox@31f87000 {
+			compatible = "ti,am654-mailbox";
+			reg = <0x00 0x31f87000 0x00 0x200>;
+			#mbox-cells = <1>;
+			ti,mbox-num-users = <4>;
+			ti,mbox-num-fifos = <16>;
+			status = "disabled";
+		};
+
+		mailbox0_cluster8: mailbox@31f88000 {
+			compatible = "ti,am654-mailbox";
+			reg = <0x00 0x31f88000 0x00 0x200>;
+			#mbox-cells = <1>;
+			ti,mbox-num-users = <4>;
+			ti,mbox-num-fifos = <16>;
+			status = "disabled";
+		};
+
+		mailbox0_cluster9: mailbox@31f89000 {
+			compatible = "ti,am654-mailbox";
+			reg = <0x00 0x31f89000 0x00 0x200>;
+			#mbox-cells = <1>;
+			ti,mbox-num-users = <4>;
+			ti,mbox-num-fifos = <16>;
+			status = "disabled";
+		};
+
+		mailbox0_cluster10: mailbox@31f8a000 {
+			compatible = "ti,am654-mailbox";
+			reg = <0x00 0x31f8a000 0x00 0x200>;
+			#mbox-cells = <1>;
+			ti,mbox-num-users = <4>;
+			ti,mbox-num-fifos = <16>;
+			status = "disabled";
+		};
+
+		mailbox0_cluster11: mailbox@31f8b000 {
+			compatible = "ti,am654-mailbox";
+			reg = <0x00 0x31f8b000 0x00 0x200>;
+			#mbox-cells = <1>;
+			ti,mbox-num-users = <4>;
+			ti,mbox-num-fifos = <16>;
+			status = "disabled";
+		};
+
+		main_ringacc: ringacc@3c000000 {
+			compatible = "ti,am654-navss-ringacc";
+			reg =	<0x0 0x3c000000 0x0 0x400000>,
+				<0x0 0x38000000 0x0 0x400000>,
+				<0x0 0x31120000 0x0 0x100>,
+				<0x0 0x33000000 0x0 0x40000>;
+			reg-names = "rt", "fifos", "proxy_gcfg", "proxy_target";
+			ti,num-rings = <1024>;
+			ti,sci-rm-range-gp-rings = <0x1>; /* GP ring range */
+			ti,sci = <&dmsc>;
+			ti,sci-dev-id = <211>;
+			interrupt-parent = <&main_udmass_inta>;
+		};
+
+		main_udmap: udmap@31150000 {
+			compatible = "ti,j721e-navss-main-udmap";
+			reg =	<0x0 0x31150000 0x0 0x100>,
+				<0x0 0x34000000 0x0 0x100000>,
+				<0x0 0x35000000 0x0 0x100000>;
+			reg-names = "gcfg", "rchanrt", "tchanrt";
+			#dma-cells = <3>;
+
+			ti,ringacc = <&main_ringacc>;
+			ti,psil-base = <0x1000>;
+
+			interrupt-parent = <&main_udmass_inta>;
+
+			ti,sci = <&dmsc>;
+			ti,sci-dev-id = <212>;
+
+			ti,sci-rm-range-tchan = <0x0d>, /* TX_CHAN */
+						<0x0f>, /* TX_HCHAN */
+						<0x10>; /* TX_UHCHAN */
+			ti,sci-rm-range-rchan = <0x0a>, /* RX_CHAN */
+						<0x0b>, /* RX_HCHAN */
+						<0x0c>; /* RX_UHCHAN */
+			ti,sci-rm-range-rflow = <0x00>; /* GP RFLOW */
+		};
+
+		cpts@310d0000 {
+			compatible = "ti,j721e-cpts";
+			reg = <0x0 0x310d0000 0x0 0x400>;
+			reg-names = "cpts";
+			clocks = <&k3_clks 201 1>;
+			clock-names = "cpts";
+			interrupts-extended = <&main_navss_intr 201 0 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "cpts";
+			ti,cpts-periodic-outputs = <6>;
+			ti,cpts-ext-ts-inputs = <8>;
+		};
+
+		navss_pat0: pat@31010000 {
+			compatible = "ti,j721e-pat";
+			reg = <0x00 0x31010000 0x00 0x00000100>,
+			      <0x00 0x36400000 0x00 0x00040000>;
+			reg-names = "mmrs",
+				    "table";
+			ti,pat-window-base = <0x48 0x00000000>;
+			ti,pat-window-size = <0x00 0x40000000>;
+		};
+
+		navss_pat1: pat@31011000 {
+			compatible = "ti,j721e-pat";
+			reg = <0x00 0x31011000 0x00 0x00000100>,
+			      <0x00 0x36440000 0x00 0x00040000>;
+			reg-names = "mmrs",
+				    "table";
+			ti,pat-window-base = <0x48 0x40000000>;
+			ti,pat-window-size = <0x00 0x40000000>;
+		};
+
+		navss_pat2: pat@31012000 {
+			compatible = "ti,j721e-pat";
+			reg = <0x00 0x31012000 0x00 0x00000100>,
+			      <0x00 0x36480000 0x00 0x00040000>;
+			reg-names = "mmrs",
+				    "table";
+			ti,pat-window-base = <0x48 0x80000000>;
+			ti,pat-window-size = <0x00 0x40000000>;
+		};
+
+		navss_pat3: pat@31013000 {
+			compatible = "ti,j721e-pat";
+			reg = <0x00 0x31013000 0x00 0x00000100>,
+			      <0x00 0x364C0000 0x00 0x00008000>;
+			reg-names = "mmrs",
+				    "table";
+			ti,pat-window-base = <0x49 0x00000000>;
+			ti,pat-window-size = <0x00 0x80000000>;
+		};
+
+		navss_pat4: pat@31014000 {
+			compatible = "ti,j721e-pat";
+			reg = <0x00 0x31014000 0x00 0x00000100>,
+			      <0x00 0x36500000 0x00 0x00008000>;
+			reg-names = "mmrs",
+				    "table";
+			ti,pat-window-base = <0x49 0x80000000>;
+			ti,pat-window-size = <0x00 0x80000000>;
+		};
+	};
+
+	psilss@3400000 {
+		/* PSILSS16 USART */
+		compatible = "ti,j721e-psilss";
+		reg = <0x0 0x03400000 0x0 0x1000>;
+		reg-names = "config";
+
+		pdma_main_usart_g0: pdma_main_usart_g0 {
+			/* PDMA13 (PDMA_USART_G0) */
+			ti,psil-base = <0x4700>;
+
+			/* ti,psil-config0-1 */
+			UDMA_PDMA_PKT_XY(0);
+			UDMA_PDMA_PKT_XY(1);
+		};
+
+		pdma_main_usart_g1: pdma_main_usart_g1 {
+			/* PDMA14 (PDMA_USART_G1) */
+			ti,psil-base = <0x4702>;
+
+			/* ti,psil-config0-1 */
+			UDMA_PDMA_PKT_XY(0);
+			UDMA_PDMA_PKT_XY(1);
+		};
+
+		pdma_main_usart_g2: pdma_main_usart_g2 {
+			/* PDMA15 (PDMA_USART_G2) */
+			ti,psil-base = <0x4704>;
+
+			/* ti,psil-config0-5 */
+			UDMA_PDMA_PKT_XY(0);
+			UDMA_PDMA_PKT_XY(1);
+			UDMA_PDMA_PKT_XY(2);
+			UDMA_PDMA_PKT_XY(3);
+			UDMA_PDMA_PKT_XY(4);
+			UDMA_PDMA_PKT_XY(5);
+		};
+	};
+
+	psilss@3404000 {
+		/* PSILSS12 MISC */
+		compatible = "ti,j721e-psilss";
+		reg = <0x0 0x03404000 0x0 0x1000>;
+		reg-names = "config";
+
+		pdma_main_misc_g0: pdma_main_misc_g0 {
+			/* PDMA8 (PDMA_MISC_G0) */
+			ti,psil-base = <0x4600>;
+
+			/* ti,psil-config0-7 */
+			UDMA_PDMA_PKT_XY(0);
+			UDMA_PDMA_PKT_XY(1);
+			UDMA_PDMA_PKT_XY(2);
+			UDMA_PDMA_PKT_XY(3);
+			UDMA_PDMA_PKT_XY(4);
+			UDMA_PDMA_PKT_XY(5);
+			UDMA_PDMA_PKT_XY(6);
+			UDMA_PDMA_PKT_XY(7);
+		};
+
+		pdma_main_misc_g1: pdma_main_misc_g1 {
+			/* PDMA9 (PDMA_MISC_G1) */
+			ti,psil-base = <0x460c>;
+
+			/* ti,psil-config0-7 */
+			UDMA_PDMA_PKT_XY(0);
+			UDMA_PDMA_PKT_XY(1);
+			UDMA_PDMA_PKT_XY(2);
+			UDMA_PDMA_PKT_XY(3);
+			UDMA_PDMA_PKT_XY(4);
+			UDMA_PDMA_PKT_XY(5);
+			UDMA_PDMA_PKT_XY(6);
+			UDMA_PDMA_PKT_XY(7);
+		};
+
+		pdma_main_misc_g2: pdma_main_misc_g2 {
+			/* PDMA10 (PDMA_MISC_G2) */
+			ti,psil-base = <0x4618>;
+
+			/* ti,psil-config0-7 */
+			UDMA_PDMA_PKT_XY(0);
+			UDMA_PDMA_PKT_XY(1);
+			UDMA_PDMA_PKT_XY(2);
+			UDMA_PDMA_PKT_XY(3);
+			UDMA_PDMA_PKT_XY(4);
+			UDMA_PDMA_PKT_XY(5);
+			UDMA_PDMA_PKT_XY(6);
+			UDMA_PDMA_PKT_XY(7);
+		};
+
+		pdma_main_misc_g3: pdma_main_misc_g3 {
+			/* PDMA11 (PDMA_MISC_G3) */
+			ti,psil-base = <0x4624>;
+
+			/* ti,psil-config0-7 */
+			UDMA_PDMA_PKT_XY(0);
+			UDMA_PDMA_PKT_XY(1);
+			UDMA_PDMA_PKT_XY(2);
+			UDMA_PDMA_PKT_XY(3);
+			UDMA_PDMA_PKT_XY(4);
+			UDMA_PDMA_PKT_XY(5);
+			UDMA_PDMA_PKT_XY(6);
+			UDMA_PDMA_PKT_XY(7);
+		};
+	};
+
+	psilss@340c000 {
+		/* PSILSS1 AASRC */
+		compatible = "ti,j721e-psilss";
+		reg = <0x0 0x0340c000 0x0 0x1000>;
+		reg-names = "config";
+
+		pdma_main_mcasp_g0: pdma_main_mcasp_g0 {
+			/* PDMA6 (PDMA_MCASP_G0) */
+			ti,psil-base = <0x4400>;
+
+			/* ti,psil-config0-2 */
+			UDMA_PDMA_J721E_MCASP(0);
+			UDMA_PDMA_J721E_MCASP(1);
+			UDMA_PDMA_J721E_MCASP(2);
+		};
+	};
+
+	pdma_main_mcasp_g1: pdma_main_mcasp_g1 {
+		/* PDMA7 (PDMA_MCASP_G1) */
+		ti,psil-base = <0x4500>;
+
+		/* ti,psil-config0-8 */
+		UDMA_PDMA_J721E_MCASP(0);
+		UDMA_PDMA_J721E_MCASP(1);
+		UDMA_PDMA_J721E_MCASP(2);
+		UDMA_PDMA_J721E_MCASP(3);
+		UDMA_PDMA_J721E_MCASP(4);
+		UDMA_PDMA_J721E_MCASP(5);
+		UDMA_PDMA_J721E_MCASP(6);
+		UDMA_PDMA_J721E_MCASP(7);
+		UDMA_PDMA_J721E_MCASP(8);
+	};
+
+	secure_proxy_main: mailbox@32c00000 {
+		compatible = "ti,am654-secure-proxy";
+		#mbox-cells = <1>;
+		reg-names = "target_data", "rt", "scfg";
+		reg = <0x00 0x32c00000 0x00 0x100000>,
+		      <0x00 0x32400000 0x00 0x100000>,
+		      <0x00 0x32800000 0x00 0x100000>;
+		interrupt-names = "rx_011";
+		interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	main_crypto: crypto@4E00000 {
+		compatible = "ti,sa2ul-crypto";
+		label = "crypto-aes-gbe";
+		reg = <0x0 0x4E00000 0x0 0x1200>;
+		power-domains = <&k3_pds 264 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 264 0>, <&k3_clks 264 1>, <&k3_clks 264 2>;
+		clock-names = "x2_clk", "pka_in_clk", "x1_clk";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges = <0x0 0x04E00000 0x00 0x04E00000 0x0 0x30000>;
+
+		status = "okay";
+		ti,psil-base = <0x4000>;
+
+		/* tx: crypto_pnp-1, rx: crypto_pnp-1 */
+		dmas = <&main_udmap &main_crypto 0 UDMA_DIR_TX>,
+				<&main_udmap &main_crypto 0 UDMA_DIR_RX>,
+				<&main_udmap &main_crypto 1 UDMA_DIR_RX>;
+		dma-names = "tx", "rx1", "rx2";
+
+		ti,psil-config0 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			ti,needs-epib;
+			ti,psd-size = <64>;
+			ti,notdpkt;
+		};
+
+		ti,psil-config1 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			ti,needs-epib;
+			ti,psd-size = <64>;
+			ti,notdpkt;
+		};
+
+		ti,psil-config2 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			ti,needs-epib;
+			ti,psd-size = <64>;
+			ti,notdpkt;
+		};
+
+		main_eip76d_trng: trng@4e10000 {
+			compatible = "inside-secure,safexcel-eip76";
+			reg = <0x0 0x4e10000 0x0 0x7d>;
+			interrupts = <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&k3_clks 264 1>;
+		};
+	};
+
+	main_pmx0: pinmux@11c000 {
+		compatible = "pinctrl-single";
+		/* Proxy 0 addressing */
+		reg = <0x0 0x11c000 0x0 0x2b4>;
+		#pinctrl-cells = <1>;
+		pinctrl-single,register-width = <32>;
+		pinctrl-single,function-mask = <0xffffffff>;
+	};
+
+	dummy_cmn_refclk: dummy_cmn_refclk {
+		#clock-cells = <0>;
+		compatible = "fixed-clock";
+		clock-frequency = <0>;
+	};
+
+	dummy_cmn_refclk1: dummy_cmn_refclk1 {
+		#clock-cells = <0>;
+		compatible = "fixed-clock";
+		clock-frequency = <0>;
+	};
+
+	serdes_wiz0: wiz@5000000 {
+		compatible = "ti,j721e-wiz-16g";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		power-domains = <&k3_pds 292 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 292 5>, <&k3_clks 292 11>, <&dummy_cmn_refclk>;
+		clock-names = "fck", "core_ref_clk", "ext_ref_clk";
+		assigned-clocks = <&k3_clks 292 11>, <&k3_clks 292 0>;
+		assigned-clock-parents = <&k3_clks 292 15>, <&k3_clks 292 4>;
+		num-lanes = <2>;
+		#reset-cells = <1>;
+		ranges;
+
+		wiz0_pll0_refclk: pll0_refclk {
+			clocks = <&k3_clks 292 11>, <&dummy_cmn_refclk>;
+			clock-output-names = "wiz0_pll0_refclk";
+			#clock-cells = <0>;
+			assigned-clocks = <&wiz0_pll0_refclk>;
+			assigned-clock-parents = <&k3_clks 292 11>;
+		};
+
+		wiz0_pll1_refclk: pll1_refclk {
+			clocks = <&k3_clks 292 0>, <&dummy_cmn_refclk1>;
+			clock-output-names = "wiz0_pll1_refclk";
+			#clock-cells = <0>;
+			assigned-clocks = <&wiz0_pll1_refclk>;
+			assigned-clock-parents = <&k3_clks 292 0>;
+		};
+
+		wiz0_refclk_dig: refclk_dig {
+			clocks = <&k3_clks 292 11>, <&k3_clks 292 0>, <&dummy_cmn_refclk>, <&dummy_cmn_refclk1>;
+			clock-output-names = "wiz0_refclk_dig";
+			#clock-cells = <0>;
+			assigned-clocks = <&wiz0_refclk_dig>;
+			assigned-clock-parents = <&k3_clks 292 11>;
+		};
+
+		wiz0_cmn_refclk: cmn_refclk {
+			clocks = <&wiz0_refclk_dig>;
+			clock-output-names = "wiz0_cmn_refclk";
+			#clock-cells = <0>;
+		};
+
+		wiz0_cmn_refclk1: cmn_refclk1 {
+			clocks = <&wiz0_pll1_refclk>;
+			clock-output-names = "wiz0_cmn_refclk1";
+			#clock-cells = <0>;
+		};
+
+		serdes0: serdes@5000000 {
+			compatible = "cdns,ti,sierra-phy-t0";
+			reg-names = "serdes";
+			reg = <0x00 0x5000000 0x00 0x10000>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			resets = <&serdes_wiz0 0>;
+			reset-names = "sierra_reset";
+			clocks = <&wiz0_cmn_refclk>, <&wiz0_cmn_refclk1>;
+			clock-names = "cmn_refclk", "cmn_refclk1";
+		};
+	};
+
+	serdes_wiz1: wiz@5010000 {
+		compatible = "ti,j721e-wiz-16g";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		power-domains = <&k3_pds 293 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 293 5>, <&k3_clks 293 13>, <&dummy_cmn_refclk>;
+		clock-names = "fck", "core_ref_clk", "ext_ref_clk";
+		assigned-clocks = <&k3_clks 293 13>, <&k3_clks 293 0>;
+		assigned-clock-parents = <&k3_clks 293 17>, <&k3_clks 293 4>;
+		num-lanes = <2>;
+		#reset-cells = <1>;
+		ranges;
+
+		wiz1_pll0_refclk: pll0_refclk {
+			clocks = <&k3_clks 293 13>, <&dummy_cmn_refclk>;
+			clock-output-names = "wiz1_pll0_refclk";
+			#clock-cells = <0>;
+			assigned-clocks = <&wiz1_pll0_refclk>;
+			assigned-clock-parents = <&k3_clks 293 13>;
+		};
+
+		wiz1_pll1_refclk: pll1_refclk {
+			clocks = <&k3_clks 293 0>, <&dummy_cmn_refclk1>;
+			clock-output-names = "wiz1_pll1_refclk";
+			#clock-cells = <0>;
+			assigned-clocks = <&wiz1_pll1_refclk>;
+			assigned-clock-parents = <&k3_clks 293 0>;
+		};
+
+		wiz1_refclk_dig: refclk_dig {
+			clocks = <&k3_clks 293 13>, <&k3_clks 293 0>, <&dummy_cmn_refclk>, <&dummy_cmn_refclk1>;
+			clock-output-names = "wiz1_refclk_dig";
+			#clock-cells = <0>;
+			assigned-clocks = <&wiz1_refclk_dig>;
+			assigned-clock-parents = <&k3_clks 293 13>;
+		};
+
+		wiz1_cmn_refclk: cmn_refclk {
+			clocks = <&wiz1_refclk_dig>;
+			clock-output-names = "wiz1_cmn_refclk";
+			#clock-cells = <0>;
+		};
+
+		wiz1_cmn_refclk1: cmn_refclk1 {
+			clocks = <&wiz1_pll1_refclk>;
+			clock-output-names = "wiz1_cmn_refclk1";
+			#clock-cells = <0>;
+		};
+
+		serdes1: serdes@5010000 {
+			compatible = "cdns,ti,sierra-phy-t0";
+			reg-names = "serdes";
+			reg = <0x00 0x5010000 0x00 0x10000>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			resets = <&serdes_wiz1 0>;
+			reset-names = "sierra_reset";
+			clocks = <&wiz1_cmn_refclk>, <&wiz1_cmn_refclk1>;
+			clock-names = "cmn_refclk", "cmn_refclk1";
+		};
+	};
+
+	serdes_wiz2: wiz@5020000 {
+		compatible = "ti,j721e-wiz-16g";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		power-domains = <&k3_pds 294 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 294 5>, <&k3_clks 294 11>, <&dummy_cmn_refclk>;
+		clock-names = "fck", "core_ref_clk", "ext_ref_clk";
+		assigned-clocks = <&k3_clks 294 11>, <&k3_clks 294 0>;
+		assigned-clock-parents = <&k3_clks 294 15>, <&k3_clks 294 4>;
+		num-lanes = <2>;
+		#reset-cells = <1>;
+		ranges;
+
+		wiz2_pll0_refclk: pll0_refclk {
+			clocks = <&k3_clks 294 11>, <&dummy_cmn_refclk>;
+			clock-output-names = "wiz2_pll0_refclk";
+			#clock-cells = <0>;
+			assigned-clocks = <&wiz2_pll0_refclk>;
+			assigned-clock-parents = <&k3_clks 294 11>;
+		};
+
+		wiz2_pll1_refclk: pll1_refclk {
+			clocks = <&k3_clks 294 0>, <&dummy_cmn_refclk1>;
+			clock-output-names = "wiz2_pll1_refclk";
+			#clock-cells = <0>;
+			assigned-clocks = <&wiz2_pll1_refclk>;
+			assigned-clock-parents = <&k3_clks 294 0>;
+		};
+
+		wiz2_refclk_dig: refclk_dig {
+			clocks = <&k3_clks 294 11>, <&k3_clks 294 0>, <&dummy_cmn_refclk>, <&dummy_cmn_refclk1>;
+			clock-output-names = "wiz2_refclk_dig";
+			#clock-cells = <0>;
+			assigned-clocks = <&wiz2_refclk_dig>;
+			assigned-clock-parents = <&k3_clks 294 11>;
+		};
+
+		wiz2_cmn_refclk: cmn_refclk {
+			clocks = <&wiz2_refclk_dig>;
+			clock-output-names = "wiz2_cmn_refclk";
+			#clock-cells = <0>;
+		};
+
+		wiz2_cmn_refclk1: cmn_refclk1 {
+			clocks = <&wiz2_pll1_refclk>;
+			clock-output-names = "wiz2_cmn_refclk1";
+			#clock-cells = <0>;
+		};
+
+		serdes2: serdes@5020000 {
+			compatible = "cdns,ti,sierra-phy-t0";
+			reg-names = "serdes";
+			reg = <0x00 0x5020000 0x00 0x10000>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			resets = <&serdes_wiz2 0>;
+			reset-names = "sierra_reset";
+			clocks = <&wiz2_cmn_refclk>, <&wiz2_cmn_refclk1>;
+			clock-names = "cmn_refclk", "cmn_refclk1";
+		};
+	};
+
+	serdes_wiz3: wiz@5030000 {
+		compatible = "ti,j721e-wiz-16g";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		power-domains = <&k3_pds 295 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 295 5>, <&k3_clks 295 9>, <&dummy_cmn_refclk>;
+		clock-names = "fck", "core_ref_clk", "ext_ref_clk";
+		assigned-clocks = <&k3_clks 295 9>, <&k3_clks 295 0>;
+		assigned-clock-parents = <&k3_clks 295 13>, <&k3_clks 295 4>;
+		num-lanes = <2>;
+		#reset-cells = <1>;
+		ranges;
+
+		wiz3_pll0_refclk: pll0_refclk {
+			clocks = <&k3_clks 295 9>, <&dummy_cmn_refclk>;
+			clock-output-names = "wiz3_pll0_refclk";
+			#clock-cells = <0>;
+			assigned-clocks = <&wiz3_pll0_refclk>;
+			assigned-clock-parents = <&k3_clks 295 9>;
+		};
+
+		wiz3_pll1_refclk: pll1_refclk {
+			clocks = <&k3_clks 295 0>, <&dummy_cmn_refclk1>;
+			clock-output-names = "wiz3_pll1_refclk";
+			#clock-cells = <0>;
+			assigned-clocks = <&wiz3_pll1_refclk>;
+			assigned-clock-parents = <&k3_clks 295 0>;
+		};
+
+		wiz3_refclk_dig: refclk_dig {
+			clocks = <&k3_clks 295 9>, <&k3_clks 295 0>, <&dummy_cmn_refclk>, <&dummy_cmn_refclk1>;
+			clock-output-names = "wiz3_refclk_dig";
+			#clock-cells = <0>;
+			assigned-clocks = <&wiz3_refclk_dig>;
+			assigned-clock-parents = <&k3_clks 295 9>;
+		};
+
+		wiz3_cmn_refclk: cmn_refclk {
+			clocks = <&wiz3_refclk_dig>;
+			clock-output-names = "wiz3_cmn_refclk";
+			#clock-cells = <0>;
+		};
+
+		wiz3_cmn_refclk1: cmn_refclk1 {
+			clocks = <&wiz3_pll1_refclk>;
+			clock-output-names = "wiz3_cmn_refclk1";
+			#clock-cells = <0>;
+		};
+
+		serdes3: serdes@5030000 {
+			compatible = "cdns,ti,sierra-phy-t0";
+			reg-names = "serdes";
+			reg = <0x00 0x5030000 0x00 0x10000>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			resets = <&serdes_wiz3 0>;
+			reset-names = "sierra_reset";
+			clocks = <&wiz3_cmn_refclk>, <&wiz3_cmn_refclk1>;
+			clock-names = "cmn_refclk", "cmn_refclk1";
+		};
+	};
+
+	serdes_wiz4: wiz@5050000 {
+		compatible = "ti,j721e-wiz-10g";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		power-domains = <&k3_pds 297 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 297 1>, <&k3_clks 297 9>, <&dummy_cmn_refclk>;
+		clock-names = "fck", "core_ref_clk", "ext_ref_clk";
+		num-lanes = <4>;
+		#reset-cells = <1>;
+		ranges;
+
+		assigned-clocks = <&k3_clks 297 9>;
+		assigned-clock-parents = <&k3_clks 297 10>;
+		assigned-clock-rates = <19200000>;
+
+		wiz4_pll0_refclk: pll0_refclk {
+			clocks = <&k3_clks 297 9>, <&dummy_cmn_refclk>;
+			clock-output-names = "wiz4_pll0_refclk";
+			#clock-cells = <0>;
+			assigned-clocks = <&wiz4_pll0_refclk>;
+			assigned-clock-parents = <&k3_clks 297 9>;
+		};
+
+		wiz4_pll1_refclk: pll1_refclk {
+			clocks = <&k3_clks 297 9>, <&dummy_cmn_refclk>;
+			clock-output-names = "wiz4_pll1_refclk";
+			#clock-cells = <0>;
+			assigned-clocks = <&wiz4_pll1_refclk>;
+			assigned-clock-parents = <&k3_clks 297 9>;
+		};
+
+		wiz4_refclk_dig: refclk_dig {
+			clocks = <&k3_clks 297 9>, <&dummy_cmn_refclk>;
+			clock-output-names = "wiz4_refclk_dig";
+			#clock-cells = <0>;
+			assigned-clocks = <&wiz4_refclk_dig>;
+			assigned-clock-parents = <&k3_clks 297 9>;
+		};
+
+		wiz4_cmn_refclk: cmn_refclk {
+			clocks = <&wiz4_refclk_dig>;
+			clock-output-names = "wiz4_cmn_refclk";
+			#clock-cells = <0>;
+		};
+
+		wiz4_cmn_refclk1: cmn_refclk1 {
+			clocks = <&wiz4_pll1_refclk>;
+			clock-output-names = "wiz4_cmn_refclk1";
+			#clock-cells = <0>;
+		};
+
+		serdes4: serdes@5050000 {
+			/* XXX we also map EDP0 registers here as the PHY driver needs those... */
+			compatible = "cdns,torrent-phy";
+			reg = <0x00 0x05050000 0x0 0x00010000>, /* SERDES_10G0 */
+			      <0x00 0x0A030A00 0x0 0x00000040>; /* DSS_EDP0_V2A_CORE_VP_REGS_APB + 30A00 */
+
+			resets = <&serdes_wiz4 0>, <&serdes_wiz4 1>,
+				 <&serdes_wiz4 2>, <&serdes_wiz4 3>,
+				 <&serdes_wiz4 4>;
+
+			num_lanes = <4>;
+			max_bit_rate = <5400>;
+			#phy-cells = <0>;
+			clocks = <&wiz4_pll0_refclk>;
+			clock-names = "refclk";
+		};
+	};
+
+	mhdp: dp-bridge@000A000000 {
+		compatible = "ti,j721e-mhdp8546", "cdns,mhdp8546";
+		reg = <0x00 0x0A000000 0x0 0x30A00>, /* DSS_EDP0_V2A_CORE_VP_REGS_APB - upto PHY mapped area */
+		      <0x00 0x04F40000 0x0 0x20>;    /* DSS_EDP0_INTG_CFG_VP */
+
+		status = "disabled";
+
+		clocks = <&k3_clks 151 36>;
+
+		phys = <&serdes4>;
+		phy-names = "dpphy";
+
+		interrupt-parent = <&gic500>;
+		interrupts = <GIC_SPI 614 IRQ_TYPE_LEVEL_HIGH>;
+
+		power-domains = <&k3_pds 151 TI_SCI_PD_EXCLUSIVE>;
+
+		/* TODO: No audio config yet */
+
+		dp0_ports: ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+	};
+
+
+	pcie0: pcie@2900000 {
+		compatible = "ti,j721e-pcie";
+		reg = <0x00 0x02900000 0x00 0x1000>,
+		      <0x00 0x02907000 0x00 0x400>,
+		      <0x0 0x02905000 0x0 0x00000400>;
+		reg-names = "intd_cfg", "user_cfg", "vmap";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		ti,syscon-pcie-ctrl = <&pcie0_ctrl>;
+		max-link-speed = <3>;
+		num-lanes = <2>;
+		power-domains = <&k3_pds 239 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 239 1>;
+		clock-names = "fck";
+
+		pcie0_rc: pcie@d000000 {
+			compatible = "ti,j721e-cdns-pcie-host";
+			reg = <0x00 0x0d000000 0x00 0x00800000>,
+			      <0x00 0x10000000 0x00 0x00001000>;
+			reg-names = "reg", "cfg";
+			device_type = "pci";
+			#address-cells = <3>;
+			#size-cells = <2>;
+			bus-range = <0x0 0xf>;
+			cdns,max-outbound-regions = <16>;
+			cdns,no-bar-match-nbits = <64>;
+			vendor-id = /bits/ 16 <0x104c>;
+			device-id = /bits/ 16 <0xb00d>;
+			msi-map = <0x0 &gic_its 0x0 0x10000>;
+			iommu-map = <0x0 &smmu0 0x0 0x1000>;
+			dma-coherent;
+			ranges = <0x01000000 0x0 0x10001000  0x00 0x10001000  0x0 0x0010000>,
+				 <0x02000000 0x0 0x10011000  0x00 0x10011000  0x0 0x7fef000>;
+		};
+
+		pcie0_ep: pcie-ep@d000000 {
+			compatible = "ti,j721e-cdns-pcie-ep";
+			reg = <0x00 0x0d000000 0x00 0x00800000>,
+			      <0x00 0x10000000 0x00 0x08000000>;
+			reg-names = "reg", "addr_space";
+			cdns,max-outbound-regions = <16>;
+			max-functions = /bits/ 8 <6>;
+			max-virtual-functions = /bits/ 8 <0x4 0x4 0x4 0x4 0x0 0x0>;
+			dma-coherent;
+		};
+	};
+
+	pcie1: pcie@2910000 {
+		compatible = "ti,j721e-pcie";
+		reg = <0x00 0x02910000 0x00 0x1000>,
+		      <0x00 0x02917000 0x00 0x400>,
+		      <0x0 0x02915000 0x0 0x00000400>;
+		reg-names = "intd_cfg", "user_cfg", "vmap";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		ti,syscon-pcie-ctrl = <&pcie1_ctrl>;
+		max-link-speed = <3>;
+		num-lanes = <2>;
+		power-domains = <&k3_pds 240 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 240 1>;
+		clock-names = "fck";
+
+		pcie1_rc: pcie@d800000 {
+			compatible = "ti,j721e-cdns-pcie-host";
+			device_type = "pci";
+			#address-cells = <3>;
+			#size-cells = <2>;
+			bus-range = <0x0 0xf>;
+			cdns,max-outbound-regions = <16>;
+			cdns,no-bar-match-nbits = <64>;
+			vendor-id = /bits/ 16 <0x104c>;
+			device-id = /bits/ 16 <0xb00d>;
+			reg = <0x00 0x0d800000 0x00 0x00800000>,
+			      <0x00 0x18000000 0x00 0x00001000>;
+			reg-names = "reg", "cfg";
+			msi-map = <0x0 &gic_its 0x10000 0x10000>;
+			iommu-map = <0x0 &smmu0 0x04000 0x1000>;
+			dma-coherent;
+			ranges = <0x01000000 0x0 0x18001000  0x00 0x18001000  0x0 0x0010000>,
+				 <0x02000000 0x0 0x18011000  0x00 0x18011000  0x0 0x7fef000>;
+		};
+
+		pcie1_ep: pcie-ep@d800000 {
+			compatible = "ti,j721e-cdns-pcie-ep";
+			reg = <0x00 0x0d800000 0x00 0x00800000>,
+			      <0x00 0x18000000 0x00 0x08000000>;
+			reg-names = "reg", "addr_space";
+			cdns,max-outbound-regions = <16>;
+			max-functions = /bits/ 8 <6>;
+			max-virtual-functions = /bits/ 8 <0x4 0x4 0x4 0x4 0x0 0x0>;
+			dma-coherent;
+		};
+	};
+
+	pcie2: pcie@2920000 {
+		compatible = "ti,j721e-pcie";
+		reg = <0x00 0x02920000 0x00 0x1000>,
+		      <0x00 0x02927000 0x00 0x400>,
+		      <0x00 0x02925000 0x00 0x400>;
+		reg-names = "intd_cfg", "user_cfg", "vmap";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		ti,syscon-pcie-ctrl = <&pcie2_ctrl>;
+		max-link-speed = <3>;
+		num-lanes = <2>;
+		power-domains = <&k3_pds 241 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 241 1>;
+		clock-names = "fck";
+
+		pcie2_rc: pcie@e000000 {
+			compatible = "ti,j721e-cdns-pcie-host";
+			device_type = "pci";
+			#address-cells = <3>;
+			#size-cells = <2>;
+			bus-range = <0x0 0xf>;
+			cdns,max-outbound-regions = <16>;
+			cdns,no-bar-match-nbits = <64>;
+			vendor-id = /bits/ 16 <0x104c>;
+			device-id = /bits/ 16 <0xb00d>;
+			reg = <0x00 0x0e000000 0x00 0x00800000>,
+			      <0x44 0x00000000 0x00 0x00001000>;
+			reg-names = "reg", "cfg";
+			msi-map = <0x0 &gic_its 0x10000 0x10000>;
+			iommu-map = <0x0 &smmu0 0x08000 0x1000>;
+			dma-coherent;
+			ranges = <0x01000000 0x00 0x00001000  0x44 0x00001000  0x0 0x0010000>,
+				 <0x02000000 0x00 0x00011000  0x44 0x00011000  0x0 0x7fef000>;
+		};
+
+		pcie2_ep: pcie-ep@e000000 {
+			compatible = "ti,j721e-cdns-pcie-ep";
+			reg = <0x00 0x0e000000 0x00 0x00800000>,
+			      <0x44 0x00000000 0x00 0x08000000>;
+			reg-names = "reg", "addr_space";
+			cdns,max-outbound-regions = <16>;
+			max-functions = /bits/ 8 <6>;
+			max-virtual-functions = /bits/ 8 <0x4 0x4 0x4 0x4 0x0 0x0>;
+			dma-coherent;
+		};
+	};
+
+	main_uart0: serial@2800000 {
+		compatible = "ti,j721e-uart", "ti,am654-uart";
+		reg = <0x00 0x02800000 0x00 0x100>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		interrupts = <GIC_SPI 192 IRQ_TYPE_LEVEL_HIGH>;
+		clock-frequency = <48000000>;
+		current-speed = <115200>;
+		power-domains = <&k3_pds 146 TI_SCI_PD_SHARED>;
+		clocks = <&k3_clks 146 0>;
+		clock-names = "fclk";
+	};
+
+	main_uart1: serial@2810000 {
+		compatible = "ti,j721e-uart", "ti,am654-uart";
+		reg = <0x00 0x02810000 0x00 0x100>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		interrupts = <GIC_SPI 193 IRQ_TYPE_LEVEL_HIGH>;
+		clock-frequency = <48000000>;
+		current-speed = <115200>;
+		power-domains = <&k3_pds 278 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 278 0>;
+		clock-names = "fclk";
+	};
+
+	main_uart2: serial@2820000 {
+		compatible = "ti,j721e-uart", "ti,am654-uart";
+		reg = <0x00 0x02820000 0x00 0x100>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		interrupts = <GIC_SPI 194 IRQ_TYPE_LEVEL_HIGH>;
+		clock-frequency = <48000000>;
+		current-speed = <115200>;
+		power-domains = <&k3_pds 279 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 279 0>;
+		clock-names = "fclk";
+	};
+
+	main_uart3: serial@2830000 {
+		compatible = "ti,j721e-uart", "ti,am654-uart";
+		reg = <0x00 0x02830000 0x00 0x100>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		interrupts = <GIC_SPI 195 IRQ_TYPE_LEVEL_HIGH>;
+		clock-frequency = <48000000>;
+		current-speed = <115200>;
+		power-domains = <&k3_pds 280 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 280 0>;
+		clock-names = "fclk";
+	};
+
+	main_uart4: serial@2840000 {
+		compatible = "ti,j721e-uart", "ti,am654-uart";
+		reg = <0x00 0x02840000 0x00 0x100>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		interrupts = <GIC_SPI 196 IRQ_TYPE_LEVEL_HIGH>;
+		clock-frequency = <48000000>;
+		current-speed = <115200>;
+		power-domains = <&k3_pds 281 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 281 0>;
+		clock-names = "fclk";
+		dmas = <&main_udmap &pdma_main_usart_g2 0 UDMA_DIR_TX>,
+			<&main_udmap &pdma_main_usart_g2 0 UDMA_DIR_RX>;
+		dma-names = "tx", "rx";
+	};
+
+	main_uart5: serial@2850000 {
+		compatible = "ti,j721e-uart", "ti,am654-uart";
+		reg = <0x00 0x02850000 0x00 0x100>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		interrupts = <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH>;
+		clock-frequency = <48000000>;
+		current-speed = <115200>;
+		power-domains = <&k3_pds 282 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 282 0>;
+		clock-names = "fclk";
+	};
+
+	main_uart6: serial@2860000 {
+		compatible = "ti,j721e-uart", "ti,am654-uart";
+		reg = <0x00 0x02860000 0x00 0x100>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		interrupts = <GIC_SPI 198 IRQ_TYPE_LEVEL_HIGH>;
+		clock-frequency = <48000000>;
+		current-speed = <115200>;
+		power-domains = <&k3_pds 283 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 283 0>;
+		clock-names = "fclk";
+	};
+
+	main_uart7: serial@2870000 {
+		compatible = "ti,j721e-uart", "ti,am654-uart";
+		reg = <0x00 0x02870000 0x00 0x100>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		interrupts = <GIC_SPI 199 IRQ_TYPE_LEVEL_HIGH>;
+		clock-frequency = <48000000>;
+		current-speed = <115200>;
+		power-domains = <&k3_pds 284 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 284 0>;
+		clock-names = "fclk";
+	};
+
+	main_uart8: serial@2880000 {
+		compatible = "ti,j721e-uart", "ti,am654-uart";
+		reg = <0x00 0x02880000 0x00 0x100>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		interrupts = <GIC_SPI 248 IRQ_TYPE_LEVEL_HIGH>;
+		clock-frequency = <48000000>;
+		current-speed = <115200>;
+		power-domains = <&k3_pds 285 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 285 0>;
+		clock-names = "fclk";
+	};
+
+	main_uart9: serial@2890000 {
+		compatible = "ti,j721e-uart", "ti,am654-uart";
+		reg = <0x00 0x02890000 0x00 0x100>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		interrupts = <GIC_SPI 249 IRQ_TYPE_LEVEL_HIGH>;
+		clock-frequency = <48000000>;
+		current-speed = <115200>;
+		power-domains = <&k3_pds 286 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 286 0>;
+		clock-names = "fclk";
+	};
+
+	main_gpio0: gpio@600000 {
+		compatible = "ti,j721e-gpio", "ti,keystone-gpio";
+		reg = <0x0 0x00600000 0x0 0x100>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-parent = <&main_gpio_intr>;
+		interrupts = <105 0 IRQ_TYPE_EDGE_RISING>,
+			     <105 1 IRQ_TYPE_EDGE_RISING>,
+			     <105 2 IRQ_TYPE_EDGE_RISING>,
+			     <105 3 IRQ_TYPE_EDGE_RISING>,
+			     <105 4 IRQ_TYPE_EDGE_RISING>,
+			     <105 5 IRQ_TYPE_EDGE_RISING>,
+			     <105 6 IRQ_TYPE_EDGE_RISING>,
+			     <105 7 IRQ_TYPE_EDGE_RISING>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		ti,ngpio = <128>;
+		ti,davinci-gpio-unbanked = <0>;
+		power-domains = <&k3_pds 105 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 105 0>;
+		clock-names = "gpio";
+	};
+
+	main_gpio1: gpio@601000 {
+		compatible = "ti,j721e-gpio", "ti,keystone-gpio";
+		reg = <0x0 0x00601000 0x0 0x100>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-parent = <&main_gpio_intr>;
+		interrupts = <106 0 IRQ_TYPE_EDGE_RISING>,
+			     <106 1 IRQ_TYPE_EDGE_RISING>,
+			     <106 2 IRQ_TYPE_EDGE_RISING>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		ti,ngpio = <36>;
+		ti,davinci-gpio-unbanked = <0>;
+		power-domains = <&k3_pds 106 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 106 0>;
+		clock-names = "gpio";
+	};
+
+	main_gpio2: gpio@610000 {
+		compatible = "ti,j721e-gpio", "ti,keystone-gpio";
+		reg = <0x0 0x00610000 0x0 0x100>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-parent = <&main_gpio_intr>;
+		interrupts = <107 0 IRQ_TYPE_EDGE_RISING>,
+			     <107 1 IRQ_TYPE_EDGE_RISING>,
+			     <107 2 IRQ_TYPE_EDGE_RISING>,
+			     <107 3 IRQ_TYPE_EDGE_RISING>,
+			     <107 4 IRQ_TYPE_EDGE_RISING>,
+			     <107 5 IRQ_TYPE_EDGE_RISING>,
+			     <107 6 IRQ_TYPE_EDGE_RISING>,
+			     <107 7 IRQ_TYPE_EDGE_RISING>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		ti,ngpio = <128>;
+		ti,davinci-gpio-unbanked = <0>;
+		power-domains = <&k3_pds 107 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 107 0>;
+		clock-names = "gpio";
+	};
+
+	main_gpio3: gpio@611000 {
+		compatible = "ti,j721e-gpio", "ti,keystone-gpio";
+		reg = <0x0 0x00611000 0x0 0x100>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-parent = <&main_gpio_intr>;
+		interrupts = <108 0 IRQ_TYPE_EDGE_RISING>,
+			     <108 1 IRQ_TYPE_EDGE_RISING>,
+			     <108 2 IRQ_TYPE_EDGE_RISING>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		ti,ngpio = <36>;
+		ti,davinci-gpio-unbanked = <0>;
+		power-domains = <&k3_pds 108 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 108 0>;
+		clock-names = "gpio";
+	};
+
+	main_gpio4: gpio@620000 {
+		compatible = "ti,j721e-gpio", "ti,keystone-gpio";
+		reg = <0x0 0x00620000 0x0 0x100>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-parent = <&main_gpio_intr>;
+		interrupts = <109 0 IRQ_TYPE_EDGE_RISING>,
+			     <109 1 IRQ_TYPE_EDGE_RISING>,
+			     <109 2 IRQ_TYPE_EDGE_RISING>,
+			     <109 3 IRQ_TYPE_EDGE_RISING>,
+			     <109 4 IRQ_TYPE_EDGE_RISING>,
+			     <109 5 IRQ_TYPE_EDGE_RISING>,
+			     <109 6 IRQ_TYPE_EDGE_RISING>,
+			     <109 7 IRQ_TYPE_EDGE_RISING>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		ti,ngpio = <128>;
+		ti,davinci-gpio-unbanked = <0>;
+		power-domains = <&k3_pds 109 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 109 0>;
+		clock-names = "gpio";
+	};
+
+	main_gpio5: gpio@621000 {
+		compatible = "ti,j721e-gpio", "ti,keystone-gpio";
+		reg = <0x0 0x00621000 0x0 0x100>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-parent = <&main_gpio_intr>;
+		interrupts = <110 0 IRQ_TYPE_EDGE_RISING>,
+			     <110 1 IRQ_TYPE_EDGE_RISING>,
+			     <110 2 IRQ_TYPE_EDGE_RISING>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		ti,ngpio = <36>;
+		ti,davinci-gpio-unbanked = <0>;
+		power-domains = <&k3_pds 110 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 110 0>;
+		clock-names = "gpio";
+	};
+
+	main_gpio6: gpio@630000 {
+		compatible = "ti,j721e-gpio", "ti,keystone-gpio";
+		reg = <0x0 0x00630000 0x0 0x100>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-parent = <&main_gpio_intr>;
+		interrupts = <111 0 IRQ_TYPE_EDGE_RISING>,
+			     <111 1 IRQ_TYPE_EDGE_RISING>,
+			     <111 2 IRQ_TYPE_EDGE_RISING>,
+			     <111 3 IRQ_TYPE_EDGE_RISING>,
+			     <111 4 IRQ_TYPE_EDGE_RISING>,
+			     <111 5 IRQ_TYPE_EDGE_RISING>,
+			     <111 6 IRQ_TYPE_EDGE_RISING>,
+			     <111 7 IRQ_TYPE_EDGE_RISING>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		ti,ngpio = <128>;
+		ti,davinci-gpio-unbanked = <0>;
+		power-domains = <&k3_pds 111 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 111 0>;
+		clock-names = "gpio";
+	};
+
+	main_gpio7: gpio@631000 {
+		compatible = "ti,j721e-gpio", "ti,keystone-gpio";
+		reg = <0x0 0x00631000 0x0 0x100>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-parent = <&main_gpio_intr>;
+		interrupts = <112 0 IRQ_TYPE_EDGE_RISING>,
+			     <112 1 IRQ_TYPE_EDGE_RISING>,
+			     <112 2 IRQ_TYPE_EDGE_RISING>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		ti,ngpio = <36>;
+		ti,davinci-gpio-unbanked = <0>;
+		power-domains = <&k3_pds 112 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 112 0>;
+		clock-names = "gpio";
+	};
+
+	main_i2c0: i2c@2000000 {
+		compatible = "ti,j721e-i2c", "ti,omap4-i2c";
+		reg = <0x0 0x2000000 0x0 0x100>;
+		interrupts = <GIC_SPI 200 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clock-names = "fck";
+		clocks = <&k3_clks 187 0>;
+		power-domains = <&k3_pds 187 TI_SCI_PD_SHARED>;
+	};
+
+	main_i2c1: i2c@2010000 {
+		compatible = "ti,j721e-i2c", "ti,omap4-i2c";
+		reg = <0x0 0x2010000 0x0 0x100>;
+		interrupts = <GIC_SPI 201 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clock-names = "fck";
+		clocks = <&k3_clks 188 0>;
+		power-domains = <&k3_pds 188 TI_SCI_PD_EXCLUSIVE>;
+	};
+
+	main_i2c2: i2c@2020000 {
+		compatible = "ti,j721e-i2c", "ti,omap4-i2c";
+		reg = <0x0 0x2020000 0x0 0x100>;
+		interrupts = <GIC_SPI 202 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clock-names = "fck";
+		clocks = <&k3_clks 189 0>;
+		power-domains = <&k3_pds 189 TI_SCI_PD_EXCLUSIVE>;
+	};
+
+	main_i2c3: i2c@2030000 {
+		compatible = "ti,j721e-i2c", "ti,omap4-i2c";
+		reg = <0x0 0x2030000 0x0 0x100>;
+		interrupts = <GIC_SPI 203 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clock-names = "fck";
+		clocks = <&k3_clks 190 0>;
+		power-domains = <&k3_pds 190 TI_SCI_PD_EXCLUSIVE>;
+	};
+
+	main_i2c4: i2c@2040000 {
+		compatible = "ti,j721e-i2c", "ti,omap4-i2c";
+		reg = <0x0 0x2040000 0x0 0x100>;
+		interrupts = <GIC_SPI 204 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clock-names = "fck";
+		clocks = <&k3_clks 191 0>;
+		power-domains = <&k3_pds 191 TI_SCI_PD_EXCLUSIVE>;
+	};
+
+	main_i2c5: i2c@2050000 {
+		compatible = "ti,j721e-i2c", "ti,omap4-i2c";
+		reg = <0x0 0x2050000 0x0 0x100>;
+		interrupts = <GIC_SPI 205 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clock-names = "fck";
+		clocks = <&k3_clks 192 0>;
+		power-domains = <&k3_pds 192 TI_SCI_PD_EXCLUSIVE>;
+	};
+
+	main_i2c6: i2c@2060000 {
+		compatible = "ti,j721e-i2c", "ti,omap4-i2c";
+		reg = <0x0 0x2060000 0x0 0x100>;
+		interrupts = <GIC_SPI 206 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clock-names = "fck";
+		clocks = <&k3_clks 193 0>;
+		power-domains = <&k3_pds 193 TI_SCI_PD_EXCLUSIVE>;
+	};
+
+	timesync_router: timesync_router@A40000 {
+		compatible = "pinctrl-single";
+		reg = <0x0 0xa40000 0x0 0x800>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		#pinctrl-cells = <1>;
+		pinctrl-single,register-width = <32>;
+		pinctrl-single,function-mask = <0x000107ff>;
+	};
+
+	dss: dss@04a00000 {
+		compatible = "ti,j721e-dss";
+		reg =
+			<0x00 0x04a00000 0x00 0x10000>, /* common_m */
+			<0x00 0x04a10000 0x00 0x10000>, /* common_s0*/
+			<0x00 0x04b00000 0x00 0x10000>, /* common_s1*/
+			<0x00 0x04b10000 0x00 0x10000>, /* common_s2*/
+
+			<0x00 0x04a20000 0x00 0x10000>, /* vidl1 */
+			<0x00 0x04a30000 0x00 0x10000>, /* vidl2 */
+			<0x00 0x04a50000 0x00 0x10000>, /* vid1 */
+			<0x00 0x04a60000 0x00 0x10000>, /* vid2 */
+
+			<0x00 0x04a70000 0x00 0x10000>, /* ovr1 */
+			<0x00 0x04a90000 0x00 0x10000>, /* ovr2 */
+			<0x00 0x04ab0000 0x00 0x10000>, /* ovr3 */
+			<0x00 0x04ad0000 0x00 0x10000>, /* ovr4 */
+
+			<0x00 0x04a80000 0x00 0x10000>, /* vp1 */
+			<0x00 0x04aa0000 0x00 0x10000>, /* vp2 */
+			<0x00 0x04ac0000 0x00 0x10000>, /* vp3 */
+			<0x00 0x04ae0000 0x00 0x10000>, /* vp4 */
+			<0x00 0x04af0000 0x00 0x10000>; /* wb */
+
+		reg-names = "common_m", "common_s0",
+			"common_s1", "common_s2",
+			"vidl1", "vidl2","vid1","vid2",
+			"ovr1", "ovr2", "ovr3", "ovr4",
+			"vp1", "vp2", "vp3", "vp4",
+			"wb";
+
+		clocks =	<&k3_clks 152 0>,
+				<&k3_clks 152 1>,
+				<&k3_clks 152 4>,
+				<&k3_clks 152 9>,
+				<&k3_clks 152 13>;
+		clock-names = "fck", "vp1", "vp2", "vp3", "vp4";
+
+		assigned-clocks = <&k3_clks 152 1>,
+				  <&k3_clks 152 4>,
+				  <&k3_clks 152 9>,
+				  <&k3_clks 152 13>;
+		assigned-clock-parents = <&k3_clks 152 2>,	/* PLL16_HSDIV0 */
+					 <&k3_clks 152 6>,	/* PLL19_HSDIV0 */
+					 <&k3_clks 152 11>,	/* PLL18_HSDIV0 */
+					 <&k3_clks 152 18>;	/* PLL23_HSDIV0 */
+
+		power-domains = <&k3_pds 152 TI_SCI_PD_EXCLUSIVE>;
+
+		interrupts = <GIC_SPI 602 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 603 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 604 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 605 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "common_m",
+				  "common_s0",
+				  "common_s1",
+				  "common_s2";
+
+		status = "disabled";
+
+		dss_ports: ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+	};
+
+	gpu: gpu@4e20000000 {
+		compatible = "ti,j721e-pvr", "img,pvr-ge8430";
+		reg = <0x4e 0x20000000 0x00 0x80000>;
+		reg-names = "gpu_regs";
+		interrupts = <GIC_SPI 24 IRQ_TYPE_LEVEL_HIGH>;
+		power-domains = <&k3_pds 125 TI_SCI_PD_EXCLUSIVE>, <&k3_pds 126 TI_SCI_PD_EXCLUSIVE>;
+		power-domain-names = "gpu_0", "gpucore_0";
+		clocks = <&k3_clks 125 0>;
+		clock-names = "ctrl";
+	};
+
+	mcasp0: mcasp@02b00000 {
+		compatible = "ti,am33xx-mcasp-audio";
+		reg = <0x0 0x02b00000 0x0 0x2000>,
+			<0x0 0x02b08000 0x0 0x1000>;
+		reg-names = "mpu","dat";
+		interrupts = <GIC_SPI 544 IRQ_TYPE_LEVEL_HIGH>,
+				<GIC_SPI 545 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "tx", "rx";
+
+		/*
+		 * tx: PDMA_MAIN_MCASP_G0-0
+		 * rx: PDMA_MAIN_MCASP_G0-0
+		 */
+		dmas = <&main_udmap &pdma_main_mcasp_g0 0 UDMA_DIR_TX>,
+			<&main_udmap &pdma_main_mcasp_g0 0 UDMA_DIR_RX>;
+		dma-names = "tx", "rx";
+
+		clocks = <&k3_clks 174 1>;
+		clock-names = "fck";
+		power-domains = <&k3_pds 174 TI_SCI_PD_EXCLUSIVE>;
+
+		status = "disabled";
+	};
+
+	mcasp1: mcasp@02b10000 {
+		compatible = "ti,am33xx-mcasp-audio";
+		reg = <0x0 0x02b10000 0x0 0x2000>,
+			<0x0 0x02b18000 0x0 0x1000>;
+		reg-names = "mpu","dat";
+		interrupts = <GIC_SPI 546 IRQ_TYPE_LEVEL_HIGH>,
+				<GIC_SPI 547 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "tx", "rx";
+
+		/*
+		 * tx: PDMA_MAIN_MCASP_G0-1
+		 * rx: PDMA_MAIN_MCASP_G0-1
+		 */
+		dmas = <&main_udmap &pdma_main_mcasp_g0 1 UDMA_DIR_TX>,
+			<&main_udmap &pdma_main_mcasp_g0 1 UDMA_DIR_RX>;
+		dma-names = "tx", "rx";
+
+		clocks = <&k3_clks 175 1>;
+		clock-names = "fck";
+		power-domains = <&k3_pds 175 TI_SCI_PD_EXCLUSIVE>;
+
+		status = "disabled";
+	};
+
+	mcasp2: mcasp@02b20000 {
+		compatible = "ti,am33xx-mcasp-audio";
+		reg = <0x0 0x02b20000 0x0 0x2000>,
+			<0x0 0x02b28000 0x0 0x1000>;
+		reg-names = "mpu","dat";
+		interrupts = <GIC_SPI 548 IRQ_TYPE_LEVEL_HIGH>,
+				<GIC_SPI 549 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "tx", "rx";
+
+		/*
+		 * tx: PDMA_MAIN_MCASP_G0-2
+		 * rx: PDMA_MAIN_MCASP_G0-2
+		 */
+		dmas = <&main_udmap &pdma_main_mcasp_g0 2 UDMA_DIR_TX>,
+			<&main_udmap &pdma_main_mcasp_g0 2 UDMA_DIR_RX>;
+		dma-names = "tx", "rx";
+
+		clocks = <&k3_clks 176 1>;
+		clock-names = "fck";
+		power-domains = <&k3_pds 176 TI_SCI_PD_EXCLUSIVE>;
+
+		status = "disabled";
+	};
+
+	mcasp3: mcasp@02b30000 {
+		compatible = "ti,am33xx-mcasp-audio";
+		reg = <0x0 0x02b30000 0x0 0x2000>,
+			<0x0 0x02b38000 0x0 0x1000>;
+		reg-names = "mpu","dat";
+		interrupts = <GIC_SPI 550 IRQ_TYPE_LEVEL_HIGH>,
+				<GIC_SPI 551 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "tx", "rx";
+
+		/*
+		 * tx: PDMA_MAIN_MCASP_G1-0
+		 * rx: PDMA_MAIN_MCASP_G1-0
+		 */
+		dmas = <&main_udmap &pdma_main_mcasp_g1 0 UDMA_DIR_TX>,
+			<&main_udmap &pdma_main_mcasp_g1 0 UDMA_DIR_RX>;
+		dma-names = "tx", "rx";
+
+		clocks = <&k3_clks 177 1>;
+		clock-names = "fck";
+		power-domains = <&k3_pds 177 TI_SCI_PD_EXCLUSIVE>;
+
+		status = "disabled";
+	};
+
+	mcasp4: mcasp@02b40000 {
+		compatible = "ti,am33xx-mcasp-audio";
+		reg = <0x0 0x02b40000 0x0 0x2000>,
+			<0x0 0x02b48000 0x0 0x1000>;
+		reg-names = "mpu","dat";
+		interrupts = <GIC_SPI 552 IRQ_TYPE_LEVEL_HIGH>,
+				<GIC_SPI 553 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "tx", "rx";
+
+		/*
+		 * tx: PDMA_MAIN_MCASP_G1-1
+		 * rx: PDMA_MAIN_MCASP_G1-1
+		 */
+		dmas = <&main_udmap &pdma_main_mcasp_g1 1 UDMA_DIR_TX>,
+			<&main_udmap &pdma_main_mcasp_g1 1 UDMA_DIR_RX>;
+		dma-names = "tx", "rx";
+
+		clocks = <&k3_clks 178 1>;
+		clock-names = "fck";
+		power-domains = <&k3_pds 178 TI_SCI_PD_EXCLUSIVE>;
+
+		status = "disabled";
+	};
+
+	mcasp5: mcasp@02b50000 {
+		compatible = "ti,am33xx-mcasp-audio";
+		reg = <0x0 0x02b50000 0x0 0x2000>,
+			<0x0 0x02b58000 0x0 0x1000>;
+		reg-names = "mpu","dat";
+		interrupts = <GIC_SPI 554 IRQ_TYPE_LEVEL_HIGH>,
+				<GIC_SPI 555 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "tx", "rx";
+
+		/*
+		 * tx: PDMA_MAIN_MCASP_G1-2
+		 * rx: PDMA_MAIN_MCASP_G1-2
+		 */
+		dmas = <&main_udmap &pdma_main_mcasp_g1 2 UDMA_DIR_TX>,
+			<&main_udmap &pdma_main_mcasp_g1 2 UDMA_DIR_RX>;
+		dma-names = "tx", "rx";
+
+		clocks = <&k3_clks 179 1>;
+		clock-names = "fck";
+		power-domains = <&k3_pds 179 TI_SCI_PD_EXCLUSIVE>;
+
+		status = "disabled";
+	};
+
+	mcasp6: mcasp@02b60000 {
+		compatible = "ti,am33xx-mcasp-audio";
+		reg = <0x0 0x02b60000 0x0 0x2000>,
+			<0x0 0x02b68000 0x0 0x1000>;
+		reg-names = "mpu","dat";
+		interrupts = <GIC_SPI 556 IRQ_TYPE_LEVEL_HIGH>,
+				<GIC_SPI 557 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "tx", "rx";
+
+		/*
+		 * tx: PDMA_MAIN_MCASP_G1-3
+		 * rx: PDMA_MAIN_MCASP_G1-3
+		 */
+		dmas = <&main_udmap &pdma_main_mcasp_g1 3 UDMA_DIR_TX>,
+			<&main_udmap &pdma_main_mcasp_g1 3 UDMA_DIR_RX>;
+		dma-names = "tx", "rx";
+
+		clocks = <&k3_clks 180 1>;
+		clock-names = "fck";
+		power-domains = <&k3_pds 180 TI_SCI_PD_EXCLUSIVE>;
+
+		status = "disabled";
+	};
+
+	mcasp7: mcasp@02b70000 {
+		compatible = "ti,am33xx-mcasp-audio";
+		reg = <0x0 0x02b70000 0x0 0x2000>,
+			<0x0 0x02b78000 0x0 0x1000>;
+		reg-names = "mpu","dat";
+		interrupts = <GIC_SPI 558 IRQ_TYPE_LEVEL_HIGH>,
+				<GIC_SPI 559 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "tx", "rx";
+
+		/*
+		 * tx: PDMA_MAIN_MCASP_G1-4
+		 * rx: PDMA_MAIN_MCASP_G1-4
+		 */
+		dmas = <&main_udmap &pdma_main_mcasp_g1 4 UDMA_DIR_TX>,
+			<&main_udmap &pdma_main_mcasp_g1 4 UDMA_DIR_RX>;
+		dma-names = "tx", "rx";
+
+		clocks = <&k3_clks 181 1>;
+		clock-names = "fck";
+		power-domains = <&k3_pds 181 TI_SCI_PD_EXCLUSIVE>;
+
+		status = "disabled";
+	};
+
+	mcasp8: mcasp@02b80000 {
+		compatible = "ti,am33xx-mcasp-audio";
+		reg = <0x0 0x02b80000 0x0 0x2000>,
+			<0x0 0x02b88000 0x0 0x1000>;
+		reg-names = "mpu","dat";
+		interrupts = <GIC_SPI 560 IRQ_TYPE_LEVEL_HIGH>,
+				<GIC_SPI 561 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "tx", "rx";
+
+		/*
+		 * tx: PDMA_MAIN_MCASP_G1-5
+		 * rx: PDMA_MAIN_MCASP_G1-5
+		 */
+		dmas = <&main_udmap &pdma_main_mcasp_g1 5 UDMA_DIR_TX>,
+			<&main_udmap &pdma_main_mcasp_g1 5 UDMA_DIR_RX>;
+		dma-names = "tx", "rx";
+
+		clocks = <&k3_clks 182 1>;
+		clock-names = "fck";
+		power-domains = <&k3_pds 182 TI_SCI_PD_EXCLUSIVE>;
+
+		status = "disabled";
+	};
+
+	mcasp9: mcasp@02b90000 {
+		compatible = "ti,am33xx-mcasp-audio";
+		reg = <0x0 0x02b90000 0x0 0x2000>,
+			<0x0 0x02b98000 0x0 0x1000>;
+		reg-names = "mpu","dat";
+		interrupts = <GIC_SPI 562 IRQ_TYPE_LEVEL_HIGH>,
+				<GIC_SPI 563 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "tx", "rx";
+
+		/*
+		 * tx: PDMA_MAIN_MCASP_G1-6
+		 * rx: PDMA_MAIN_MCASP_G1-6
+		 */
+		dmas = <&main_udmap &pdma_main_mcasp_g1 6 UDMA_DIR_TX>,
+			<&main_udmap &pdma_main_mcasp_g1 6 UDMA_DIR_RX>;
+		dma-names = "tx", "rx";
+
+		clocks = <&k3_clks 183 1>;
+		clock-names = "fck";
+		power-domains = <&k3_pds 183 TI_SCI_PD_EXCLUSIVE>;
+
+		status = "disabled";
+	};
+
+	mcasp10: mcasp@02ba0000 {
+		compatible = "ti,am33xx-mcasp-audio";
+		reg = <0x0 0x02ba0000 0x0 0x2000>,
+			<0x0 0x02ba8000 0x0 0x1000>;
+		reg-names = "mpu","dat";
+		interrupts = <GIC_SPI 564 IRQ_TYPE_LEVEL_HIGH>,
+				<GIC_SPI 565 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "tx", "rx";
+
+		/*
+		 * tx: PDMA_MAIN_MCASP_G1-7
+		 * rx: PDMA_MAIN_MCASP_G1-7
+		 */
+		dmas = <&main_udmap &pdma_main_mcasp_g1 7 UDMA_DIR_TX>,
+			<&main_udmap &pdma_main_mcasp_g1 7 UDMA_DIR_RX>;
+		dma-names = "tx", "rx";
+
+		clocks = <&k3_clks 184 1>;
+		clock-names = "fck";
+		power-domains = <&k3_pds 184 TI_SCI_PD_EXCLUSIVE>;
+
+		status = "disabled";
+	};
+
+	mcasp11: mcasp@02bb0000 {
+		compatible = "ti,am33xx-mcasp-audio";
+		reg = <0x0 0x02bb0000 0x0 0x2000>,
+			<0x0 0x02bb8000 0x0 0x1000>;
+		reg-names = "mpu","dat";
+		interrupts = <GIC_SPI 566 IRQ_TYPE_LEVEL_HIGH>,
+				<GIC_SPI 567 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "tx", "rx";
+
+		/*
+		 * tx: PDMA_MAIN_MCASP_G1-8
+		 * rx: PDMA_MAIN_MCASP_G1-8
+		 */
+		dmas = <&main_udmap &pdma_main_mcasp_g1 8 UDMA_DIR_TX>,
+			<&main_udmap &pdma_main_mcasp_g1 8 UDMA_DIR_RX>;
+		dma-names = "tx", "rx";
+
+		clocks = <&k3_clks 185 1>;
+		clock-names = "fck";
+		power-domains = <&k3_pds 185 TI_SCI_PD_EXCLUSIVE>;
+
+		status = "disabled";
+	};
+
+	usbss0: cdns_usb@4104000 {
+		compatible = "ti,j721e-usb";
+		reg = <0x00 0x4104000 0x00 0x100>;
+		dma-coherent;
+		power-domains = <&k3_pds 288 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 288 15>, <&k3_clks 288 3>;
+		clock-names = "usb2_refclk", "lpm_clk";
+		assigned-clocks = <&k3_clks 288 15>;	/* USB2_REFCLK */
+		assigned-clock-parents = <&k3_clks 288 16>; /* HFOSC0 */
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		phy@4108000 {
+			compatible = "ti,j721e-usb2-phy";
+			reg = <0x00 0x4108000 0x00 0x400>;
+		};
+
+		usb0: usb@6000000 {
+			compatible = "cdns,usb3";
+			reg = <0x00 0x6000000 0x00 0x10000>,
+			      <0x00 0x6010000 0x00 0x10000>,
+			      <0x00 0x6020000 0x00 0x10000>;
+			reg-names = "otg", "xhci", "dev";
+			interrupts = <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>,	/* irq.0 */
+				     <GIC_SPI 102 IRQ_TYPE_LEVEL_HIGH>,	/* irq.6 */
+				     <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>;	/* otgirq.0 */
+			interrupt-names = "host",
+					  "peripheral",
+					  "otg";
+			maximum-speed = "super-speed";
+			dr_mode = "otg";
+		};
+	};
+
+	usbss1: cdns_usb@4114000 {
+		compatible = "ti,j721e-usb";
+		reg = <0x00 0x4114000 0x00 0x100>;
+		dma-coherent;
+		power-domains = <&k3_pds 289 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 289 15>, <&k3_clks 289 3>;
+		clock-names = "usb2_refclk", "lpm_clk";
+		assigned-clocks = <&k3_clks 289 15>;	/* USB2_REFCLK */
+		assigned-clock-parents = <&k3_clks 289 16>; /* HFOSC0 */
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		phy@4118000 {
+			compatible = "ti,j721e-usb2-phy";
+			reg = <0x00 0x4118000 0x00 0x400>;
+		};
+
+		usb1: usb@6400000 {
+			compatible = "cdns,usb3";
+			reg = <0x00 0x6400000 0x00 0x10000>,
+			      <0x00 0x6410000 0x00 0x10000>,
+			      <0x00 0x6420000 0x00 0x10000>;
+			reg-names = "otg", "xhci", "dev";
+			interrupts = <GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH>,	/* irq.0 */
+				     <GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>,	/* irq.6 */
+				     <GIC_SPI 121 IRQ_TYPE_LEVEL_HIGH>;	/* otgirq.0 */
+			interrupt-names = "host",
+					  "peripheral",
+					  "otg";
+			maximum-speed = "super-speed";
+			dr_mode = "otg";
+		};
+	};
+
+	main_sdhci0: sdhci@4f80000 {
+		compatible = "ti,j721e-sdhci-8bit";
+		reg = <0x0 0x4f80000 0x0 0x1000>, <0x0 0x4f88000 0x0 0x400>;
+		interrupts = <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>;
+		power-domains = <&k3_pds 91 TI_SCI_PD_EXCLUSIVE>;
+		clock-names = "clk_xin", "clk_ahb";
+		clocks = <&k3_clks 91 1>, <&k3_clks 91 0>;
+		assigned-clocks = <&k3_clks 91 1>;
+		assigned-clock-parents = <&k3_clks 91 2>;
+		bus-width = <8>;
+		mmc-hs400-1_8v;
+		mmc-ddr-1_8v;
+		ti,otap-del-sel-legacy = <0x0>;
+		ti,otap-del-sel-mmc-hs = <0x0>;
+		ti,otap-del-sel-ddr52 = <0x5>;
+		ti,otap-del-sel-hs200 = <0x6>;
+		ti,otap-del-sel-hs400 = <0x0>;
+		ti,trm-icp = <0x8>;
+		ti,strobe-sel = <0x77>;
+		dma-coherent;
+	};
+
+	main_sdhci1: sdhci@4fb0000 {
+		compatible = "ti,j721e-sdhci-4bit";
+		reg = <0x0 0x04fb0000 0x0 0x1000>, <0x0 0x4fb8000 0x0 0x400>;
+		interrupts = <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>;
+		power-domains = <&k3_pds 92 TI_SCI_PD_EXCLUSIVE>;
+		clock-names = "clk_xin", "clk_ahb";
+		clocks = <&k3_clks 92 0>, <&k3_clks 92 5>;
+		assigned-clocks = <&k3_clks 92 0>;
+		assigned-clock-parents = <&k3_clks 92 1>;
+		ti,otap-del-sel-legacy = <0x0>;
+		ti,otap-del-sel-sd-hs = <0xf>;
+		ti,otap-del-sel-sdr12 = <0xf>;
+		ti,otap-del-sel-sdr25 = <0xf>;
+		ti,otap-del-sel-sdr50 = <0xc>;
+		ti,otap-del-sel-sdr104 = <0x5>;
+		ti,otap-del-sel-ddr50 = <0xc>;
+		ti,trm-icp = <0x8>;
+		ti,clkbuf-sel = <0x7>;
+		dma-coherent;
+		sdhci-caps-mask = <0x2 0x0>;
+	};
+
+	main_sdhci2: sdhci@4f98000 {
+		compatible = "ti,j721e-sdhci-4bit";
+		reg = <0x0 0x4f98000 0x0 0x1000>, <0x0 0x4f90000 0x0 0x400>;
+		interrupts = <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>;
+		power-domains = <&k3_pds 93 TI_SCI_PD_EXCLUSIVE>;
+		clock-names = "clk_xin", "clk_ahb";
+		clocks = <&k3_clks 93 0>, <&k3_clks 93 5>;
+		assigned-clocks = <&k3_clks 93 0>;
+		assigned-clock-parents = <&k3_clks 93 1>;
+		ti,otap-del-sel-legacy = <0x0>;
+		ti,otap-del-sel-sd-hs = <0xf>;
+		ti,otap-del-sel-sdr12 = <0xf>;
+		ti,otap-del-sel-sdr25 = <0xf>;
+		ti,otap-del-sel-sdr50 = <0xc>;
+		ti,otap-del-sel-sdr104 = <0x5>;
+		ti,otap-del-sel-ddr50 = <0xc>;
+		ti,trm-icp = <0x8>;
+		ti,clkbuf-sel = <0x7>;
+		dma-coherent;
+		sdhci-caps-mask = <0x2 0x0>;
+	};
+
+	main_r5fss0: r5fss@5c00000 {
+		compatible = "ti,j721e-r5fss";
+		lockstep-mode = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0x5c00000 0x00 0x5c00000 0x20000>,
+			 <0x5d00000 0x00 0x5d00000 0x20000>;
+		power-domains = <&k3_pds 243 TI_SCI_PD_EXCLUSIVE>;
+
+		main_r5fss0_core0: r5f@5c00000 {
+			compatible = "ti,j721e-r5f";
+			reg = <0x5c00000 0x00008000>,
+			      <0x5c10000 0x00008000>;
+			reg-names = "atcm", "btcm";
+			ti,sci = <&dmsc>;
+			ti,sci-dev-id = <245>;
+			ti,sci-proc-ids = <0x06 0xFF>;
+			resets = <&k3_reset 245 1>;
+			atcm-enable = <1>;
+			btcm-enable = <1>;
+			loczrama = <1>;
+			mboxes = <&mailbox0_cluster1 &mbox_main_r5fss0_core0>;
+		};
+
+		main_r5fss0_core1: r5f@5d00000 {
+			compatible = "ti,j721e-r5f";
+			reg = <0x5d00000 0x00008000>,
+			      <0x5d10000 0x00008000>;
+			reg-names = "atcm", "btcm";
+			ti,sci = <&dmsc>;
+			ti,sci-dev-id = <246>;
+			ti,sci-proc-ids = <0x07 0xFF>;
+			resets = <&k3_reset 246 1>;
+			atcm-enable = <1>;
+			btcm-enable = <1>;
+			loczrama = <1>;
+			mboxes = <&mailbox0_cluster1 &mbox_main_r5fss0_core1>;
+		};
+	};
+
+	main_r5fss1: r5fss@5e00000 {
+		compatible = "ti,j721e-r5fss";
+		lockstep-mode = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0x5e00000 0x00 0x5e00000 0x20000>,
+			 <0x5f00000 0x00 0x5f00000 0x20000>;
+		power-domains = <&k3_pds 244 TI_SCI_PD_EXCLUSIVE>;
+
+		main_r5fss1_core0: r5f@5e00000 {
+			compatible = "ti,j721e-r5f";
+			reg = <0x5e00000 0x00008000>,
+			      <0x5e10000 0x00008000>;
+			reg-names = "atcm", "btcm";
+			ti,sci = <&dmsc>;
+			ti,sci-dev-id = <247>;
+			ti,sci-proc-ids = <0x08 0xFF>;
+			resets = <&k3_reset 247 1>;
+			atcm-enable = <1>;
+			btcm-enable = <1>;
+			loczrama = <1>;
+			mboxes = <&mailbox0_cluster2 &mbox_main_r5fss1_core0>;
+		};
+
+		main_r5fss1_core1: r5f@5f00000 {
+			compatible = "ti,j721e-r5f";
+			reg = <0x5f00000 0x00008000>,
+			      <0x5f10000 0x00008000>;
+			reg-names = "atcm", "btcm";
+			ti,sci = <&dmsc>;
+			ti,sci-dev-id = <248>;
+			ti,sci-proc-ids = <0x09 0xFF>;
+			resets = <&k3_reset 248 1>;
+			atcm-enable = <1>;
+			btcm-enable = <1>;
+			loczrama = <1>;
+			mboxes = <&mailbox0_cluster2 &mbox_main_r5fss1_core1>;
+		};
+	};
+
+	c66_0: dsp@4d80800000 {
+		compatible = "ti,j721e-c66-dsp";
+		reg = <0x4d 0x80800000 0x00 0x00048000>,
+		      <0x4d 0x80e00000 0x00 0x00008000>,
+		      <0x4d 0x80f00000 0x00 0x00008000>;
+		reg-names = "l2sram", "l1pram", "l1dram";
+		ti,sci = <&dmsc>;
+		ti,sci-dev-id = <142>;
+		ti,sci-proc-ids = <0x03 0xFF>;
+		resets = <&k3_reset 142 1>;
+		/*power-domains = <&k3_pds 142 TI_SCI_PD_EXCLUSIVE>;*/
+		mboxes = <&mailbox0_cluster3 &mbox_c66_0>;
+	};
+
+	c66_1: dsp@4d81800000 {
+		compatible = "ti,j721e-c66-dsp";
+		reg = <0x4d 0x81800000 0x00 0x00048000>,
+		      <0x4d 0x81e00000 0x00 0x00008000>,
+		      <0x4d 0x81f00000 0x00 0x00008000>;
+		reg-names = "l2sram", "l1pram", "l1dram";
+		ti,sci = <&dmsc>;
+		ti,sci-dev-id = <143>;
+		ti,sci-proc-ids = <0x04 0xFF>;
+		resets = <&k3_reset 143 1>;
+		/*power-domains = <&k3_pds 143 TI_SCI_PD_EXCLUSIVE>;*/
+		mboxes = <&mailbox0_cluster3 &mbox_c66_1>;
+	};
+
+	c71_0: dsp@64800000 {
+		compatible = "ti,j721e-c71-dsp";
+		reg = <0x00 0x64800000 0x00 0x00080000>,
+		      <0x00 0x64e00000 0x00 0x0000c000>;
+		reg-names = "l2sram", "l1dram";
+		ti,sci = <&dmsc>;
+		ti,sci-dev-id = <15>;
+		ti,sci-proc-ids = <0x30 0xFF>;
+		resets = <&k3_reset 15 1>;
+		/*power-domains = <&k3_pds 15 TI_SCI_PD_EXCLUSIVE>;*/
+		mboxes = <&mailbox0_cluster4 &mbox_c71_0>;
+	};
+
+	d5520: video-decoder@4300000 {
+		/* IMG D5520 driver configuration */
+		compatible = "img,d5500-vxd";
+		reg = <0x00 0x04300000 0x00 0x100000>;
+		power-domains = <&k3_pds 144 TI_SCI_PD_EXCLUSIVE>;
+		interrupts = <GIC_SPI 180 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	vxe384: video-encoder@4200000 {
+		compatible = "img,vxe384";
+		reg = <0x00 0x04200000 0x00 0x100000>;
+		power-domains = <&k3_pds 153 TI_SCI_PD_EXCLUSIVE>;
+		interrupts = <GIC_SPI 181 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	ufs_wrapper: ufs-wrapper@4e80000 {
+		compatible = "ti,j721e-ufs";
+		reg = <0x0 0x4e80000 0x0 0x100>;
+		power-domains = <&k3_pds 277 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 277 1>;
+		assigned-clocks = <&k3_clks 277 1>;
+		assigned-clock-parents = <&k3_clks 277 4>;
+		ranges;
+		#address-cells = <2>;
+		#size-cells = <2>;
+
+		ufs@4e84000 {
+			compatible = "cdns,ufshc-m31-16nm", "jedec,ufs-2.0";
+			reg = <0x0 0x4e84000 0x0 0x10000>;
+			interrupts = <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>;
+			freq-table-hz = <250000000 250000000>, <19200000 19200000>, <19200000 19200000>;
+			clocks = <&k3_clks 277 0>, <&k3_clks 277 1>, <&k3_clks 277 1>;
+			clock-names = "core_clk", "phy_clk", "ref_clk";
+			assigned-clocks = <&k3_clks 277 1>;
+			assigned-clock-parents = <&k3_clks 277 4>;
+			dma-coherent;
+		};
+	};
+
+	icssg_soc_bus0: pruss-soc-bus@b026004 {
+		compatible = "ti,j721e-icssg-soc-bus";
+		reg = <0x00 0x0b026004 0x00 0x4>;
+		power-domains = <&k3_pds 119 TI_SCI_PD_EXCLUSIVE>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0x0b000000 0x00 0x0b000000 0x100000>;
+		dma-ranges;
+
+		icssg0: icssg@b000000 {
+			compatible = "ti,j721e-icssg";
+			reg = <0xb000000 0x80000>;
+			interrupts = <GIC_SPI 254 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 255 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 256 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 257 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 258 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 259 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 260 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 261 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "host2", "host3", "host4",
+					  "host5", "host6", "host7",
+					  "host8", "host9";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+			dma-ranges;
+
+			icssg0_mem: memories@b000000 {
+				reg = <0xb000000 0x2000>,
+				      <0xb002000 0x2000>,
+				      <0xb010000 0x10000>;
+				reg-names = "dram0", "dram1",
+					    "shrdram2";
+			};
+
+			icssg0_cfg: cfg@b026000 {
+				compatible = "syscon";
+				reg = <0xb026000 0x200>;
+			};
+
+			icssg0_coreclk_mux: coreclk_mux {
+				#clock-cells = <0>;
+				clocks = <&k3_clks 119 24>, /* icssg0_core_clk */
+					 <&k3_clks 119 1>;  /* icssg0_iclk */
+				assigned-clocks = <&icssg0_coreclk_mux>;
+				assigned-clock-parents = <&k3_clks 119 1>;
+			};
+
+			icssg0_iepclk_mux: iepclk_mux {
+				#clock-cells = <0>;
+				clocks = <&k3_clks 119 3>,	/* icssg0_iep_clk */
+					 <&icssg0_coreclk_mux>;	/* core_clk */
+				assigned-clocks = <&icssg0_iepclk_mux>;
+				assigned-clock-parents = <&icssg0_coreclk_mux>;
+			};
+
+			icssg0_iep0: iep@b02e000 {
+				compatible = "syscon";
+				reg = <0xb02e000 0x1000>;
+				clocks = <&icssg0_iepclk_mux>;
+			};
+
+			icssg0_iep1: iep@b02f000 {
+				compatible = "syscon";
+				reg = <0xb02f000 0x1000>;
+				clocks = <&icssg0_iepclk_mux>;
+			};
+
+			icssg0_mii_rt: mii-rt@b032000 {
+				compatible = "syscon";
+				reg = <0xb032000 0x100>;
+			};
+
+			icssg0_mii_g_rt: mii-g-rt@b033000 {
+				compatible = "syscon";
+				reg = <0xb033000 0x1000>;
+			};
+
+			icssg0_intc: interrupt-controller@b020000 {
+				compatible = "ti,j721e-icssg-intc";
+				reg = <0xb020000 0x2000>;
+				interrupt-controller;
+				#interrupt-cells = <1>;
+			};
+
+			pru0_0: pru@b034000 {
+				compatible = "ti,j721e-pru";
+				reg = <0xb034000 0x3000>,
+				      <0xb022000 0x100>,
+				      <0xb022400 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "j7-pru0_0-fw";
+				interrupt-parent = <&icssg0_intc>;
+				interrupts = <16>, <17>;
+				interrupt-names = "vring", "kick";
+			};
+
+			rtu0_0: rtu@b004000 {
+				compatible = "ti,j721e-rtu";
+				reg = <0xb004000 0x2000>,
+				      <0xb023000 0x100>,
+				      <0xb023400 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "j7-rtu0_0-fw";
+				interrupt-parent = <&icssg0_intc>;
+				interrupts = <20>, <21>;
+				interrupt-names = "vring", "kick";
+			};
+
+			tx_pru0_0: txpru@b00a000 {
+				compatible = "ti,j721e-tx-pru";
+				reg = <0xb00a000 0x1800>,
+				      <0xb025000 0x100>,
+				      <0xb025400 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "j7-txpru0_0-fw";
+			};
+
+			pru0_1: pru@b038000 {
+				compatible = "ti,j721e-pru";
+				reg = <0xb038000 0x3000>,
+				      <0xb024000 0x100>,
+				      <0xb024400 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "j7-pru0_1-fw";
+				interrupt-parent = <&icssg0_intc>;
+				interrupts = <18>, <19>;
+				interrupt-names = "vring", "kick";
+			};
+
+			rtu0_1: rtu@b006000 {
+				compatible = "ti,j721e-rtu";
+				reg = <0xb006000 0x2000>,
+				      <0xb023800 0x100>,
+				      <0xb023c00 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "j7-rtu0_1-fw";
+				interrupt-parent = <&icssg0_intc>;
+				interrupts = <22>, <23>;
+				interrupt-names = "vring", "kick";
+			};
+
+			tx_pru0_1: txpru@b00c000 {
+				compatible = "ti,j721e-tx-pru";
+				reg = <0xb00c000 0x1800>,
+				      <0xb025800 0x100>,
+				      <0xb025c00 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "j7-txpru0_1-fw";
+			};
+
+			icssg0_mdio: mdio@b032400 {
+				compatible = "ti,davinci_mdio";
+				reg = <0xb032400 0x100>;
+				clocks = <&k3_clks 119 1>;
+				clock-names = "fck";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				bus_freq = <1000000>;
+				status = "disabled";
+			};
+		};
+	};
+
+	icssg_soc_bus1: pruss-soc-bus@b126004 {
+		compatible = "ti,j721e-icssg-soc-bus";
+		reg = <0x00 0x0b126004 0x00 0x4>;
+		power-domains = <&k3_pds 120 TI_SCI_PD_EXCLUSIVE>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0x0b100000 0x00 0x0b100000 0x100000>;
+		dma-ranges;
+
+		icssg1: icssg@b100000 {
+			compatible = "ti,j721e-icssg";
+			reg = <0xb100000 0x80000>;
+			interrupts = <GIC_SPI 262 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 263 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 264 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 265 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 266 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 267 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 268 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 269 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "host2", "host3", "host4",
+					  "host5", "host6", "host7",
+					  "host8", "host9";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+			dma-ranges;
+
+			icssg1_mem: memories@b100000 {
+				reg = <0xb100000 0x2000>,
+				      <0xb102000 0x2000>,
+				      <0xb110000 0x10000>;
+				reg-names = "dram0", "dram1",
+					    "shrdram2";
+			};
+
+			icssg1_cfg: cfg@b126000 {
+				compatible = "syscon";
+				reg = <0xb126000 0x200>;
+			};
+
+			icssg1_coreclk_mux: coreclk_mux {
+				#clock-cells = <0>;
+				clocks = <&k3_clks 120 54>, /* icssg1_core_clk */
+					 <&k3_clks 120 4>;  /* icssg1_iclk */
+				assigned-clocks = <&icssg1_coreclk_mux>;
+				assigned-clock-parents = <&k3_clks 120 4>;
+			};
+
+			icssg1_iepclk_mux: iepclk_mux {
+				#clock-cells = <0>;
+				clocks = <&k3_clks 120 9>,	/* icssg1_iep_clk */
+					 <&icssg1_coreclk_mux>;	/* core_clk */
+				assigned-clocks = <&icssg1_iepclk_mux>;
+				assigned-clock-parents = <&icssg1_coreclk_mux>;
+			};
+
+			icssg1_iep0: iep@b12e000 {
+				compatible = "syscon";
+				reg = <0xb12e000 0x1000>;
+				clocks = <&icssg1_iepclk_mux>;
+			};
+
+			icssg1_iep1: iep@b12f000 {
+				compatible = "syscon";
+				reg = <0xb12f000 0x1000>;
+				clocks = <&icssg1_iepclk_mux>;
+			};
+
+			icssg1_mii_rt: mii-rt@b132000 {
+				compatible = "syscon";
+				reg = <0xb132000 0x100>;
+			};
+
+			icssg1_mii_g_rt: mii-g-rt@b133000 {
+				compatible = "syscon";
+				reg = <0xb133000 0x1000>;
+			};
+
+			icssg1_intc: interrupt-controller@b120000 {
+				compatible = "ti,j721e-icssg-intc";
+				reg = <0xb120000 0x2000>;
+				interrupt-controller;
+				#interrupt-cells = <1>;
+			};
+
+			pru1_0: pru@b134000 {
+				compatible = "ti,j721e-pru";
+				reg = <0xb134000 0x4000>,
+				      <0xb122000 0x100>,
+				      <0xb122400 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "j7-pru1_0-fw";
+				interrupt-parent = <&icssg1_intc>;
+				interrupts = <16>, <17>;
+				interrupt-names = "vring", "kick";
+			};
+
+			rtu1_0: rtu@b104000 {
+				compatible = "ti,j721e-rtu";
+				reg = <0xb104000 0x2000>,
+				      <0xb123000 0x100>,
+				      <0xb123400 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "j7-rtu1_0-fw";
+				interrupt-parent = <&icssg1_intc>;
+				interrupts = <20>, <21>;
+				interrupt-names = "vring", "kick";
+			};
+
+			tx_pru1_0: txpru@b10a000 {
+				compatible = "ti,j721e-tx-pru";
+				reg = <0xb10a000 0x1800>,
+				      <0xb125000 0x100>,
+				      <0xb125400 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "j7-txpru1_0-fw";
+			};
+
+			pru1_1: pru@b138000 {
+				compatible = "ti,j721e-pru";
+				reg = <0xb138000 0x4000>,
+				      <0xb124000 0x100>,
+				      <0xb124400 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "j7-pru1_1-fw";
+				interrupt-parent = <&icssg1_intc>;
+				interrupts = <18>, <19>;
+				interrupt-names = "vring", "kick";
+			};
+
+			rtu1_1: rtu@b106000 {
+				compatible = "ti,j721e-rtu";
+				reg = <0xb106000 0x2000>,
+				      <0xb123800 0x100>,
+				      <0xb123c00 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "j7-rtu1_1-fw";
+				interrupt-parent = <&icssg1_intc>;
+				interrupts = <22>, <23>;
+				interrupt-names = "vring", "kick";
+			};
+
+			tx_pru1_1: txpru@b10c000 {
+				compatible = "ti,j721e-tx-pru";
+				reg = <0xb10c000 0x1800>,
+				      <0xb125800 0x100>,
+				      <0xb125c00 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "j7-txpru1_1-fw";
+			};
+
+			icssg1_mdio: mdio@b132400 {
+				compatible = "ti,davinci_mdio";
+				reg = <0xb132400 0x100>;
+				clocks = <&k3_clks 120 4>;
+				clock-names = "fck";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				bus_freq = <1000000>;
+				status = "disabled";
+			};
+		};
+	};
+
+	main_rti0: rti@2200000 {
+		compatible = "ti,rti-wdt";
+		reg = <0x0 0x2200000 0x0 0x100>;
+		clocks = <&k3_clks 252 1>;
+		power-domains = <&k3_pds 252 TI_SCI_PD_EXCLUSIVE>;
+		assigned-clocks = <&k3_clks 252 1>;
+		assigned-clock-parents = <&k3_clks 252 5>;
+	};
+
+	main_rti1: rti@2210000 {
+		compatible = "ti,rti-wdt";
+		reg = <0x0 0x2210000 0x0 0x100>;
+		clocks = <&k3_clks 253 1>;
+		power-domains = <&k3_pds 253 TI_SCI_PD_EXCLUSIVE>;
+		assigned-clocks = <&k3_clks 253 1>;
+		assigned-clock-parents = <&k3_clks 253 5>;
+	};
+};
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/k3-j721e-mcu-wakeup.dtsi b/arch/arm64/boot/dts/ti/k3-j721e-mcu-wakeup.dtsi
--- a/arch/arm64/boot/dts/ti/k3-j721e-mcu-wakeup.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/k3-j721e-mcu-wakeup.dtsi	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,526 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Source for J721E SoC Family MCU/WAKEUP Domain peripherals
+ *
+ * Copyright (C) 2016-2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+&cbass_mcu_wakeup {
+	dmsc: dmsc@44083000 {
+		compatible = "ti,k2g-sci";
+		ti,host-id = <12>;
+
+		mbox-names = "rx", "tx";
+
+		mboxes= <&secure_proxy_main 11>,
+			<&secure_proxy_main 13>;
+
+		reg-names = "debug_messages";
+		reg = <0x00 0x44083000 0x0 0x1000>;
+
+		k3_pds: power-controller {
+			compatible = "ti,sci-pm-domain";
+			#power-domain-cells = <2>;
+		};
+
+		k3_clks: clocks {
+			compatible = "ti,k2g-sci-clk";
+			#clock-cells = <2>;
+			ti,scan-clocks-from-dt;
+		};
+
+		k3_reset: reset-controller {
+			compatible = "ti,sci-reset";
+			#reset-cells = <2>;
+		};
+	};
+
+	mcu_conf: scm_conf@40f00000 {
+		compatible = "syscon", "simple-mfd";
+		reg = <0x0 0x40f00000 0x0 0x20000>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0x0 0x0 0x40f00000 0x20000>;
+
+		phy_sel: cpsw-phy-sel@4040 {
+			compatible = "ti,am654-cpsw-phy-sel";
+			reg = <0x4040 0x4>;
+			reg-names = "gmii-sel";
+		};
+	};
+
+	wkup_pmx0: pinmux@4301c000 {
+		compatible = "pinctrl-single";
+		/* Proxy 0 addressing */
+		reg = <0x00 0x4301c000 0x00 0x178>;
+		#pinctrl-cells = <1>;
+		pinctrl-single,register-width = <32>;
+		pinctrl-single,function-mask = <0xffffffff>;
+	};
+
+	wkup_gpio_intr: interrupt-controller2 {
+		compatible = "ti,sci-intr";
+		interrupt-controller;
+		interrupt-parent = <&gic500>;
+		#interrupt-cells = <3>;
+		ti,sci = <&dmsc>;
+		ti,sci-dst-id = <14>;
+		ti,sci-rm-range-girq = <0x5>;
+	};
+
+	mcu_ram: sram@41c00000 {
+		compatible = "mmio-sram";
+		reg = <0x00 0x41c00000 0x00 0x100000>;
+		ranges = <0x0 0x00 0x41c00000 0x100000>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+	};
+
+	wkup_uart0: serial@42300000 {
+		compatible = "ti,j721e-uart", "ti,am654-uart";
+		reg = <0x00 0x42300000 0x00 0x100>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		interrupts = <GIC_SPI 897 IRQ_TYPE_LEVEL_HIGH>;
+		clock-frequency = <48000000>;
+		current-speed = <115200>;
+		power-domains = <&k3_pds 287 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 287 0>;
+		clock-names = "fclk";
+	};
+
+	mcu_uart0: serial@40a00000 {
+		compatible = "ti,j721e-uart", "ti,am654-uart";
+		reg = <0x00 0x40a00000 0x00 0x100>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		interrupts = <GIC_SPI 846 IRQ_TYPE_LEVEL_HIGH>;
+		clock-frequency = <96000000>;
+		current-speed = <115200>;
+		power-domains = <&k3_pds 149 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 149 0>;
+		clock-names = "fclk";
+	};
+
+	wkup_gpio0: gpio@42110000 {
+		compatible = "ti,j721e-gpio", "ti,keystone-gpio";
+		reg = <0x0 0x42110000 0x0 0x100>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-parent = <&wkup_gpio_intr>;
+		interrupts = <113 0 IRQ_TYPE_EDGE_RISING>,
+			     <113 1 IRQ_TYPE_EDGE_RISING>,
+			     <113 2 IRQ_TYPE_EDGE_RISING>,
+			     <113 3 IRQ_TYPE_EDGE_RISING>,
+			     <113 4 IRQ_TYPE_EDGE_RISING>,
+			     <113 5 IRQ_TYPE_EDGE_RISING>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		ti,ngpio = <84>;
+		ti,davinci-gpio-unbanked = <0>;
+		power-domains = <&k3_pds 113 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 113 0>;
+		clock-names = "gpio";
+	};
+
+	wkup_gpio1: gpio@42100000 {
+		compatible = "ti,j721e-gpio", "ti,keystone-gpio";
+		reg = <0x0 0x42100000 0x0 0x100>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-parent = <&wkup_gpio_intr>;
+		interrupts = <114 0 IRQ_TYPE_EDGE_RISING>,
+			     <114 1 IRQ_TYPE_EDGE_RISING>,
+			     <114 2 IRQ_TYPE_EDGE_RISING>,
+			     <114 3 IRQ_TYPE_EDGE_RISING>,
+			     <114 4 IRQ_TYPE_EDGE_RISING>,
+			     <114 5 IRQ_TYPE_EDGE_RISING>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		ti,ngpio = <84>;
+		ti,davinci-gpio-unbanked = <0>;
+		power-domains = <&k3_pds 114 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 114 0>;
+		clock-names = "gpio";
+	};
+
+	cbass_mcu_navss: mcu_navss {
+		compatible = "simple-bus";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		dma-coherent;
+		dma-ranges;
+		ranges;
+
+		ti,sci-dev-id = <232>;
+
+		mcu_ringacc: ringacc@2b800000 {
+			compatible = "ti,am654-navss-ringacc";
+			reg =	<0x0 0x2b800000 0x0 0x400000>,
+				<0x0 0x2b000000 0x0 0x400000>,
+				<0x0 0x28590000 0x0 0x100>,
+				<0x0 0x2a500000 0x0 0x40000>;
+			reg-names = "rt", "fifos", "proxy_gcfg", "proxy_target";
+			ti,num-rings = <286>;
+			ti,sci-rm-range-gp-rings = <0x1>; /* GP ring range */
+			ti,sci = <&dmsc>;
+			ti,sci-dev-id = <235>;
+			interrupt-parent = <&main_udmass_inta>;
+		};
+
+		mcu_udmap: udmap@31150000 {
+			compatible = "ti,j721e-navss-mcu-udmap";
+			reg =	<0x0 0x285c0000 0x0 0x100>,
+				<0x0 0x2a800000 0x0 0x40000>,
+				<0x0 0x2aa00000 0x0 0x40000>;
+			reg-names = "gcfg", "rchanrt", "tchanrt";
+			#dma-cells = <3>;
+
+			ti,ringacc = <&mcu_ringacc>;
+			ti,psil-base = <0x6000>;
+
+			ti,sci = <&dmsc>;
+			ti,sci-dev-id = <236>;
+
+			ti,sci-rm-range-tchan = <0x0d>, /* TX_CHAN */
+						<0x0f>; /* TX_HCHAN */
+			ti,sci-rm-range-rchan = <0x0a>, /* RX_CHAN */
+						<0x0b>; /* RX_HCHAN */
+			ti,sci-rm-range-rflow = <0x00>; /* GP RFLOW */
+
+			interrupt-parent = <&main_udmass_inta>;
+		};
+	};
+
+	pdma_mcu_misc_g0: pdma_mcu_misc_g0 {
+		/* MCU_PDMA0 (MCU_PDMA_MISC_G0) */
+		ti,psil-base = <0x7100>;
+
+		/* ti,psil-config0-3 */
+		UDMA_PDMA_PKT_XY(0);
+		UDMA_PDMA_PKT_XY(1);
+		UDMA_PDMA_PKT_XY(2);
+		UDMA_PDMA_PKT_XY(3);
+	};
+
+	pdma_mcu_misc_g1: pdma_mcu_misc_g1 {
+		/* MCU_PDMA1 (MCU_PDMA_MISC_G1) */
+		ti,psil-base = <0x7200>;
+
+		/* ti,psil-config0-7 */
+		UDMA_PDMA_PKT_XY(0);
+		UDMA_PDMA_PKT_XY(1);
+		UDMA_PDMA_PKT_XY(2);
+		UDMA_PDMA_PKT_XY(3);
+		UDMA_PDMA_PKT_XY(4);
+		UDMA_PDMA_PKT_XY(5);
+		UDMA_PDMA_PKT_XY(6);
+		UDMA_PDMA_PKT_XY(7);
+	};
+
+	pdma_mcu_misc_g2: pdma_mcu_misc_g2 {
+		/* MCU_PDMA2 (MCU_PDMA_MISC_G2) */
+		ti,psil-base = <0x7300>;
+
+		/* ti,psil-config0 */
+		UDMA_PDMA_PKT_XY(0);
+	};
+
+	pdma_mcu_adc: pdma_mcu_adc {
+		/* MCU_PDMA_ADC */
+		ti,psil-base = <0x7400>;
+
+		/* ti,psil-config0 */
+		UDMA_PDMA_TR_XY(0);
+		UDMA_PDMA_TR_XY(1);
+		UDMA_PDMA_TR_XY(2);
+		UDMA_PDMA_TR_XY(3);
+	};
+
+	mcu_i2c0: i2c@40b00000 {
+		compatible = "ti,j721e-i2c", "ti,omap4-i2c";
+		reg = <0x0 0x40b00000 0x0 0x100>;
+		interrupts = <GIC_SPI 852 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clock-names = "fck";
+		clocks = <&k3_clks 194 0>;
+		power-domains = <&k3_pds 194 TI_SCI_PD_EXCLUSIVE>;
+	};
+
+	mcu_i2c1: i2c@40b10000 {
+		compatible = "ti,j721e-i2c", "ti,omap4-i2c";
+		reg = <0x0 0x40b10000 0x0 0x100>;
+		interrupts = <GIC_SPI 853 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clock-names = "fck";
+		clocks = <&k3_clks 195 0>;
+		power-domains = <&k3_pds 195 TI_SCI_PD_EXCLUSIVE>;
+	};
+
+	wkup_i2c0: i2c@42120000 {
+		compatible = "ti,j721e-i2c", "ti,omap4-i2c";
+		reg = <0x0 0x42120000 0x0 0x100>;
+		interrupts = <GIC_SPI 896 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clock-names = "fck";
+		clocks = <&k3_clks 197 0>;
+		power-domains = <&k3_pds 197 TI_SCI_PD_SHARED>;
+	};
+
+	mcu_cpsw: ethernet@046000000 {
+		compatible = "ti,j721e-cpsw-nuss";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		reg = <0x0 0x46000000 0x0 0x200000>;
+		reg-names = "cpsw_nuss";
+		ranges;
+		dma-coherent;
+		clocks = <&k3_clks 18 22>;
+		clock-names = "fck";
+		power-domains = <&k3_pds 18 TI_SCI_PD_EXCLUSIVE>;
+		ti,psil-base = <0x7000>;
+		cpsw-phy-sel = <&phy_sel>;
+
+		interrupt-parent = <&main_udmass_inta>;
+
+		dmas = <&mcu_udmap &mcu_cpsw 0 UDMA_DIR_TX>,
+		       <&mcu_udmap &mcu_cpsw 1 UDMA_DIR_TX>,
+		       <&mcu_udmap &mcu_cpsw 2 UDMA_DIR_TX>,
+		       <&mcu_udmap &mcu_cpsw 3 UDMA_DIR_TX>,
+		       <&mcu_udmap &mcu_cpsw 4 UDMA_DIR_TX>,
+		       <&mcu_udmap &mcu_cpsw 5 UDMA_DIR_TX>,
+		       <&mcu_udmap &mcu_cpsw 6 UDMA_DIR_TX>,
+		       <&mcu_udmap &mcu_cpsw 7 UDMA_DIR_TX>,
+		       <&mcu_udmap &mcu_cpsw 0 UDMA_DIR_RX>;
+		dma-names = "tx0", "tx1", "tx2", "tx3",
+			    "tx4", "tx5", "tx6", "tx7",
+			    "rx";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			host: host@0 {
+				reg = <0>;
+				ti,label = "host";
+			};
+
+			cpsw_port1: port@1 {
+				reg = <1>;
+				ti,mac-only;
+				ti,label = "port1";
+				ti,syscon-efuse = <&mcu_conf 0x200>;
+			};
+		};
+
+		davinci_mdio: mdio {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			bus_freq = <1000000>;
+		};
+
+		cpts {
+			clocks = <&k3_clks 18 2>;
+			clock-names = "cpts";
+			interrupts-extended = <&gic500 GIC_SPI 858 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "cpts";
+			ti,cpts-ext-ts-inputs = <4>;
+			ti,cpts-periodic-outputs = <2>;
+		};
+
+		ti,psil-config0 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+
+		ti,psil-config1 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+
+		ti,psil-config2 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+
+		ti,psil-config3 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+
+		ti,psil-config4 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+
+		ti,psil-config5 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+
+		ti,psil-config6 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+
+		ti,psil-config7 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+	};
+
+	fss: fss@47000000 {
+		compatible = "syscon", "simple-mfd";
+		reg = <0x0 0x47000000 0x0 0x100>;
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		ospi0: spi@47040000 {
+			compatible = "ti,j721e-ospi", "ti,am654-ospi";
+			reg = <0x0 0x47040000 0x0 0x100>,
+				<0x5 0x00000000 0x1 0x0000000>;
+			interrupts = <GIC_SPI 840 IRQ_TYPE_LEVEL_HIGH>;
+			cdns,fifo-depth = <256>;
+			cdns,fifo-width = <4>;
+			cdns,trigger-address = <0x0>;
+			clocks = <&k3_clks 103 0>;
+			assigned-clocks = <&k3_clks 103 0>;
+			assigned-clock-parents = <&k3_clks 103 2>;
+			assigned-clock-rates = <166666666>;
+			power-domains = <&k3_pds 103 TI_SCI_PD_EXCLUSIVE>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+
+		hbmc_mux: hbmc-mux {
+			compatible = "mmio-mux";
+			#mux-control-cells = <1>;
+			mux-reg-masks = <0x4 0x2>; /* HBMC select */
+		};
+
+		hbmc: hyperbus@47034000 {
+			compatible = "ti,j721e-hbmc", "ti,am654-hbmc";
+			reg = <0x0 0x47034000 0x0 0x100>,
+				<0x5 0x00000000 0x1 0x0000000>;
+			power-domains = <&k3_pds 102 TI_SCI_PD_EXCLUSIVE>;
+			clocks = <&k3_clks 102 0>;
+			assigned-clocks = <&k3_clks 102 0>;
+			assigned-clock-rates = <250000000>;
+			#address-cells = <2>;
+			#size-cells = <1>;
+			mux-controls = <&hbmc_mux 0>;
+		};
+
+		ospi1: spi@47050000 {
+			compatible = "ti,j721e-ospi", "ti,am654-ospi";
+			reg = <0x0 0x47050000 0x0 0x100>,
+				<0x7 0x00000000 0x1 0x00000000>;
+			interrupts = <GIC_SPI 841 IRQ_TYPE_LEVEL_HIGH>;
+			cdns,fifo-depth = <256>;
+			cdns,fifo-width = <4>;
+			cdns,trigger-address = <0x0>;
+			clocks = <&k3_clks 104 0>;
+			power-domains = <&k3_pds 104 TI_SCI_PD_EXCLUSIVE>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+	};
+
+	mcu_r5fss0: r5fss@41000000 {
+		compatible = "ti,j721e-r5fss";
+		lockstep-mode = <1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0x41000000 0x00 0x41000000 0x20000>,
+			 <0x41400000 0x00 0x41400000 0x20000>;
+		power-domains = <&k3_pds 249 TI_SCI_PD_EXCLUSIVE>;
+
+		mcu_r5fss0_core0: r5f@41000000 {
+			compatible = "ti,j721e-r5f";
+			reg = <0x41000000 0x00008000>,
+			      <0x41010000 0x00008000>;
+			reg-names = "atcm", "btcm";
+			ti,sci = <&dmsc>;
+			ti,sci-dev-id = <250>;
+			ti,sci-proc-ids = <0x01 0xFF>;
+			resets = <&k3_reset 250 1>;
+			atcm-enable = <1>;
+			btcm-enable = <1>;
+			loczrama = <1>;
+			mboxes = <&mailbox0_cluster0 &mbox_mcu_r5fss0_core0>;
+		};
+
+		mcu_r5fss0_core1: r5f@41400000 {
+			compatible = "ti,j721e-r5f";
+			reg = <0x41400000 0x00008000>,
+			      <0x41410000 0x00008000>;
+			reg-names = "atcm", "btcm";
+			ti,sci = <&dmsc>;
+			ti,sci-dev-id = <251>;
+			ti,sci-proc-ids = <0x02 0xFF>;
+			resets = <&k3_reset 251 1>;
+			atcm-enable = <1>;
+			btcm-enable = <1>;
+			loczrama = <1>;
+			mboxes = <&mailbox0_cluster0 &mbox_mcu_r5fss0_core1>;
+		};
+	};
+
+	tscadc0: tscadc@40200000 {
+		compatible = "ti,j721e-tscadc", "ti,am3359-tscadc";
+		reg = <0x0 0x40200000 0x0 0x1000>;
+		interrupts = <GIC_SPI 860 IRQ_TYPE_LEVEL_HIGH>;
+		power-domains = <&k3_pds 0 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 0 1>;
+		assigned-clocks = <&k3_clks 0 3>;
+		assigned-clock-rates = <60000000>;
+		clock-names = "adc_tsc_fck";
+		dmas = <&main_udmap &pdma_mcu_adc 0 UDMA_DIR_RX>,
+			<&main_udmap &pdma_mcu_adc 1 UDMA_DIR_RX>;
+		dma-names = "fifo0", "fifo1";
+
+		adc {
+			#io-channel-cells = <1>;
+			compatible = "ti,j721e-adc", "ti,am3359-adc";
+		};
+	};
+
+	tscadc1: tscadc@40210000 {
+		compatible = "ti,j721e-tscadc", "ti,am3359-tscadc";
+		reg = <0x0 0x40210000 0x0 0x1000>;
+		interrupts = <GIC_SPI 861 IRQ_TYPE_LEVEL_HIGH>;
+		power-domains = <&k3_pds 1 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 1 1>;
+		assigned-clocks = <&k3_clks 1 3>;
+		assigned-clock-rates = <60000000>;
+		clock-names = "adc_tsc_fck";
+		dmas = <&main_udmap &pdma_mcu_adc 2 UDMA_DIR_RX>,
+			<&main_udmap &pdma_mcu_adc 3 UDMA_DIR_RX>;
+		dma-names = "fifo0", "fifo1";
+
+		adc {
+			#io-channel-cells = <1>;
+			compatible = "ti,j721e-adc", "ti,am3359-adc";
+		};
+	};
+};
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/k3-j721e-pcie-backplane.dtso b/arch/arm64/boot/dts/ti/k3-j721e-pcie-backplane.dtso
--- a/arch/arm64/boot/dts/ti/k3-j721e-pcie-backplane.dtso	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/k3-j721e-pcie-backplane.dtso	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,41 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * J7 PCIe Backplane
+ *
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/pci/pci.h>
+
+/ {
+  fragment@101 {
+	target-path = "/";
+
+	__overlay__ {
+		epf_bus {
+			compatible = "pci-epf-bus";
+
+			ntb {
+				compatible = "pci-epf-ntb";
+				epcs = <&pcie0_ep>, <&pcie1_ep>;
+				epc-names = "primary", "secondary";
+				vendor-id = /bits/ 16 <0x104c>;
+				device-id = /bits/ 16 <0xb00d>;
+				num-mws = <4>;
+				mws-size = <0x100000>, <0x100000>, <0x100000>, <0x100000>;
+			};
+		};
+	};
+  };
+};
+
+&pcie0 {
+	pci-mode = <PCI_MODE_EP>;
+};
+
+&pcie1 {
+	pci-mode = <PCI_MODE_EP>;
+};
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/k3-j721e-proc-board-tps65917.dts b/arch/arm64/boot/dts/ti/k3-j721e-proc-board-tps65917.dts
--- a/arch/arm64/boot/dts/ti/k3-j721e-proc-board-tps65917.dts	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/k3-j721e-proc-board-tps65917.dts	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,829 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+
+#include "k3-j721e-som-tps65917.dtsi"
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/net/ti-dp83867.h>
+#include <dt-bindings/pci/pci.h>
+#include <dt-bindings/sound/ti-mcasp.h>
+
+/ {
+	chosen {
+		stdout-path = "serial2:115200n8";
+		bootargs = "console=ttyS2,115200n8 earlycon=ns16550a,mmio32,0x02800000";
+	};
+
+	evm_12v0: fixedregulator-evm12v0 {
+		/* main supply */
+		compatible = "regulator-fixed";
+		regulator-name = "evm_12v0";
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vsys_3v3: fixedregulator-vsys3v3 {
+		/* Output of LMS140 */
+		compatible = "regulator-fixed";
+		regulator-name = "vsys_3v3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&evm_12v0>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vsys_5v0: fixedregulator-vsys5v0 {
+		/* Output of LM5140 */
+		compatible = "regulator-fixed";
+		regulator-name = "vsys_5v0";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&evm_12v0>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	sound0: sound@0 {
+		compatible = "ti,j721e-cpb-audio";
+		ti,model = "j721e-cpb-analog";
+
+		ti,cpb-mcasp = <&mcasp10>;
+		ti,cpb-codec = <&pcm3168a_1>;
+
+		pll4-rate = <1179648000>; /* Used for 48KHz family */
+		pll15-rate = <1083801600>; /* Used for 44.1KHz family */
+
+		clocks = <&k3_clks 184 1>,
+			 <&k3_clks 184 2>, <&k3_clks 184 4>,
+			 <&k3_clks 157 371>,
+			 <&k3_clks 157 400>, <&k3_clks 157 401>;
+		clock-names = "cpb-mcasp",
+			      "cpb-mcasp-48000", "cpb-mcasp-44100",
+			      "audio-refclk2",
+			      "audio-refclk2-48000", "audio-refclk2-44100";
+	};
+
+	gpio_keys: gpio-keys {
+		compatible = "gpio-keys";
+		autorepeat;
+		pinctrl-names = "default";
+		pinctrl-0 = <&sw10_button_pins_default &sw11_button_pins_default>;
+
+		sw10: sw10 {
+			label = "GPIO Key USER1";
+			linux,code = <BTN_0>;
+			gpios = <&main_gpio0 0 GPIO_ACTIVE_LOW>;
+		};
+
+		sw11: sw11 {
+			label = "GPIO Key USER2";
+			linux,code = <BTN_1>;
+			gpios = <&wkup_gpio0 7 GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	vdd_mmc1: fixedregulator-sd {
+		compatible = "regulator-fixed";
+		regulator-name = "vdd_mmc1";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+		enable-active-high;
+		vin-supply = <&vsys_3v3>;
+		gpio = <&exp2 2 GPIO_ACTIVE_HIGH>;
+	};
+
+	dp0: connector {
+		compatible = "dp-connector";
+		label = "DP0";
+
+		port {
+			dp_connector_in: endpoint {
+				remote-endpoint = <&dp_bridge_output>;
+			};
+		};
+	};
+
+	cpsw9g_virt_mac: main_r5fss_cpsw9g_virt_mac0 {
+		compatible = "ti,j721e-cpsw-virt-mac";
+		dma-coherent;
+		ti,psil-base = <0x4a00>;
+		ti,remote-name = "mpu_1_0_ethswitch-device-0";
+		interrupt-parent = <&main_udmass_inta>;
+
+		dmas = <&main_udmap &cpsw9g_virt_mac 0 UDMA_DIR_TX>,
+		       <&main_udmap &cpsw9g_virt_mac 1 UDMA_DIR_TX>,
+		       <&main_udmap &cpsw9g_virt_mac 2 UDMA_DIR_TX>,
+		       <&main_udmap &cpsw9g_virt_mac 3 UDMA_DIR_TX>,
+		       <&main_udmap &cpsw9g_virt_mac 4 UDMA_DIR_TX>,
+		       <&main_udmap &cpsw9g_virt_mac 5 UDMA_DIR_TX>,
+		       <&main_udmap &cpsw9g_virt_mac 6 UDMA_DIR_TX>,
+		       <&main_udmap &cpsw9g_virt_mac 0 UDMA_DIR_RX>;
+		dma-names = "tx0", "tx1", "tx2", "tx3",
+			    "tx4", "tx5", "tx6",
+			    "rx";
+
+		virt_emac_port {
+			ti,label = "virt-port";
+			/* local-mac-address = [0 0 0 0 0 0]; */
+		};
+
+		ti,psil-config0 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+
+		ti,psil-config1 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+
+		ti,psil-config2 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+
+		ti,psil-config3 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+
+		ti,psil-config4 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+
+		ti,psil-config5 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+
+		ti,psil-config6 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+
+		ti,psil-config7 {
+			linux,udma-mode = <UDMA_PKT_MODE>;
+			statictr-type = <PSIL_STATIC_TR_NONE>;
+			ti,needs-epib;
+			ti,psd-size = <16>;
+		};
+	};
+};
+
+&wkup_pmx0 {
+	mcu_cpsw_pins_default: mcu_cpsw_pins_default {
+		pinctrl-single,pins = <
+			J721E_WKUP_IOPAD(0x0058, PIN_OUTPUT, 0) /* MCU_RGMII1_TX_CTL */
+			J721E_WKUP_IOPAD(0x005c, PIN_INPUT, 0) /* MCU_RGMII1_RX_CTL */
+			J721E_WKUP_IOPAD(0x0060, PIN_OUTPUT, 0) /* MCU_RGMII1_TD3 */
+			J721E_WKUP_IOPAD(0x0064, PIN_OUTPUT, 0) /* MCU_RGMII1_TD2 */
+			J721E_WKUP_IOPAD(0x0068, PIN_OUTPUT, 0) /* MCU_RGMII1_TD1 */
+			J721E_WKUP_IOPAD(0x006c, PIN_OUTPUT, 0) /* MCU_RGMII1_TD0 */
+			J721E_WKUP_IOPAD(0x0078, PIN_INPUT, 0) /* MCU_RGMII1_RD3 */
+			J721E_WKUP_IOPAD(0x007c, PIN_INPUT, 0) /* MCU_RGMII1_RD2 */
+			J721E_WKUP_IOPAD(0x0080, PIN_INPUT, 0) /* MCU_RGMII1_RD1 */
+			J721E_WKUP_IOPAD(0x0084, PIN_INPUT, 0) /* MCU_RGMII1_RD0 */
+			J721E_WKUP_IOPAD(0x0070, PIN_INPUT, 0) /* MCU_RGMII1_TXC */
+			J721E_WKUP_IOPAD(0x0074, PIN_INPUT, 0) /* MCU_RGMII1_RXC */
+		>;
+	};
+
+	mcu_mdio_pins_default: mcu_mdio1_pins_default {
+		pinctrl-single,pins = <
+			J721E_WKUP_IOPAD(0x008c, PIN_OUTPUT, 0) /* MCU_MDIO0_MDC */
+			J721E_WKUP_IOPAD(0x0088, PIN_INPUT, 0) /* MCU_MDIO0_MDIO */
+		>;
+	};
+
+	mcu_fss0_ospi1_pins_default: mcu-fss0-ospi1-pins-default {
+		pinctrl-single,pins = <
+			J721E_WKUP_IOPAD(0x34, PIN_OUTPUT, 0) /* (F22) MCU_OSPI1_CLK */
+			J721E_WKUP_IOPAD(0x50, PIN_OUTPUT, 0) /* (C22) MCU_OSPI1_CSn0 */
+			J721E_WKUP_IOPAD(0x40, PIN_INPUT, 0) /* (D22) MCU_OSPI1_D0 */
+			J721E_WKUP_IOPAD(0x44, PIN_INPUT, 0) /* (G22) MCU_OSPI1_D1 */
+			J721E_WKUP_IOPAD(0x48, PIN_INPUT, 0) /* (D23) MCU_OSPI1_D2 */
+			J721E_WKUP_IOPAD(0x4c, PIN_INPUT, 0) /* (C23) MCU_OSPI1_D3 */
+			J721E_WKUP_IOPAD(0x3c, PIN_INPUT, 0) /* (B23) MCU_OSPI1_DQS */
+			J721E_WKUP_IOPAD(0x38, PIN_INPUT, 0) /* (A23) MCU_OSPI1_LBCLKO */
+		>;
+	};
+
+	sw11_button_pins_default: sw11_button_pins_default {
+		pinctrl-single,pins = <
+			J721E_WKUP_IOPAD(0xcc, PIN_INPUT, 7) /* (G28) WKUP_GPIO0_7 */
+		>;
+	};
+};
+
+&wkup_uart0 {
+	/* Wakeup UART is used by System firmware */
+	status = "disabled";
+};
+
+&main_uart3 {
+	/* UART not brought out */
+	status = "disabled";
+};
+
+&main_uart5 {
+	/* UART not brought out */
+	status = "disabled";
+};
+
+&main_uart6 {
+	/* UART not brought out */
+	status = "disabled";
+};
+
+&main_uart7 {
+	/* UART not brought out */
+	status = "disabled";
+};
+
+&main_uart8 {
+	/* UART not brought out */
+	status = "disabled";
+};
+
+&main_uart9 {
+	/* UART not brought out */
+	status = "disabled";
+};
+
+&main_gpio2 {
+	status = "disabled";
+};
+
+&main_gpio3 {
+	status = "disabled";
+};
+
+&main_gpio4 {
+	status = "disabled";
+};
+
+&main_gpio5 {
+	status = "disabled";
+};
+
+&main_gpio6 {
+	status = "disabled";
+};
+
+&main_gpio7 {
+	status = "disabled";
+};
+
+&wkup_gpio1 {
+	status = "disabled";
+};
+
+&main_pmx0 {
+	main_i2c1_exp4_pins_default: main-i2c1-exp4-pins-default {
+		pinctrl-single,pins = <
+			J721E_IOPAD(0x230, PIN_INPUT, 7) /* (U2) ECAP0_IN_APWM_OUT.GPIO1_11 */
+		>;
+	};
+
+	main_i2c0_pins_default: main-i2c0-pins-default {
+		pinctrl-single,pins = <
+			J721E_IOPAD(0x220, PIN_INPUT_PULLUP, 0) /* (AC5) I2C0_SCL */
+			J721E_IOPAD(0x224, PIN_INPUT_PULLUP, 0) /* (AA5) I2C0_SDA */
+		>;
+	};
+
+	main_i2c1_pins_default: main-i2c1-pins-default {
+		pinctrl-single,pins = <
+			J721E_IOPAD(0x228, PIN_INPUT_PULLUP, 0) /* (Y6) I2C1_SCL */
+			J721E_IOPAD(0x22c, PIN_INPUT_PULLUP, 0) /* (AA6) I2C1_SDA */
+		>;
+	};
+
+	main_i2c3_pins_default: main-i2c3-pins-default {
+		pinctrl-single,pins = <
+			J721E_IOPAD(0x270, PIN_INPUT_PULLUP, 4) /* (T26) MMC2_CLK.I2C3_SCL */
+			J721E_IOPAD(0x274, PIN_INPUT_PULLUP, 4) /* (T25) MMC2_CMD.I2C3_SDA */
+		>;
+	};
+
+	main_i2c6_pins_default: main-i2c6-pins-default {
+		pinctrl-single,pins = <
+			J721E_IOPAD(0x1d0, PIN_INPUT_PULLUP, 2) /* (AA3) SPI0_D1.I2C6_SCL */
+			J721E_IOPAD(0x1e4, PIN_INPUT_PULLUP, 2) /* (Y2) SPI1_D1.I2C6_SDA */
+		>;
+	};
+
+	mcasp10_pins_default: mcasp10_pins_default {
+		pinctrl-single,pins = <
+			J721E_IOPAD(0x158, PIN_OUTPUT_PULLDOWN, 12) /* (U23) RGMII5_TX_CTL.MCASP10_ACLKX */
+			J721E_IOPAD(0x15c, PIN_OUTPUT_PULLDOWN, 12) /* (U26) RGMII5_RX_CTL.MCASP10_AFSX */
+			J721E_IOPAD(0x160, PIN_INPUT_PULLDOWN, 12) /* (V28) RGMII5_TD3.MCASP10_AXR0 */
+			J721E_IOPAD(0x164, PIN_INPUT_PULLDOWN, 12) /* (V29) RGMII5_TD2.MCASP10_AXR1 */
+			J721E_IOPAD(0x170, PIN_INPUT_PULLDOWN, 12) /* (U29) RGMII5_TXC.MCASP10_AXR2 */
+			J721E_IOPAD(0x174, PIN_OUTPUT_PULLDOWN, 12) /* (U25) RGMII5_RXC.MCASP10_AXR3 */
+			J721E_IOPAD(0x198, PIN_OUTPUT_PULLDOWN, 12) /* (V25) RGMII6_TD1.MCASP10_AXR4 */
+			J721E_IOPAD(0x19c, PIN_OUTPUT_PULLDOWN, 12) /* (W27) RGMII6_TD0.MCASP10_AXR5 */
+			J721E_IOPAD(0x1a0, PIN_OUTPUT_PULLDOWN, 12) /* (W29) RGMII6_TXC.MCASP10_AXR6 */
+		>;
+	};
+
+	audi_ext_refclk2_pins_default: audi_ext_refclk2_pins_default {
+		pinctrl-single,pins = <
+			J721E_IOPAD(0x1a4, PIN_OUTPUT, 3) /* (W26) RGMII6_RXC.AUDIO_EXT_REFCLK2 */
+		>;
+	};
+
+	main_usbss0_pins_default: main_usbss0_pins_default {
+		pinctrl-single,pins = <
+			J721E_IOPAD(0x290, PIN_OUTPUT, 0) /* (U6) USB0_DRVVBUS */
+			J721E_IOPAD(0x210, PIN_INPUT, 7) /* (W3) MCAN1_RX.GPIO1_3 */
+		>;
+	};
+
+	main_usbss1_pins_default: main_usbss1_pins_default {
+		pinctrl-single,pins = <
+			J721E_IOPAD(0x214, PIN_OUTPUT, 4) /* (V4) MCAN1_TX.USB1_DRVVBUS */
+		>;
+	};
+
+	main_mmc1_pins_default: main_mmc1_pins_default {
+		pinctrl-single,pins = <
+			J721E_IOPAD(0x254, PIN_INPUT, 0) /* (R29) MMC1_CMD */
+			J721E_IOPAD(0x250, PIN_INPUT, 0) /* (P25) MMC1_CLK */
+			J721E_IOPAD(0x2ac, PIN_INPUT, 0) /* (P25) MMC1_CLKLB */
+			J721E_IOPAD(0x24c, PIN_INPUT, 0) /* (R24) MMC1_DAT0 */
+			J721E_IOPAD(0x248, PIN_INPUT, 0) /* (P24) MMC1_DAT1 */
+			J721E_IOPAD(0x244, PIN_INPUT, 0) /* (R25) MMC1_DAT2 */
+			J721E_IOPAD(0x240, PIN_INPUT, 0) /* (R26) MMC1_DAT3 */
+			J721E_IOPAD(0x258, PIN_INPUT, 0) /* (P23) MMC1_SDCD */
+			J721E_IOPAD(0x25c, PIN_INPUT, 0) /* (R28) MMC1_SDWP */
+		>;
+	};
+
+	dp0_pins_default: dp0_pins_default {
+		pinctrl-single,pins = <
+			J721E_IOPAD(0x1c4, PIN_INPUT, 5) /* SPI0_CS1.DP0_HPD */
+		>;
+	};
+
+	sw10_button_pins_default: sw10_button_pins_default {
+		pinctrl-single,pins = <
+			J721E_IOPAD(0x0, PIN_INPUT, 7) /* (AC18) EXTINTn.GPIO0_0 */
+		>;
+	};
+};
+
+&dss {
+	status = "ok";
+};
+
+&dss_ports {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	port@0 {
+		reg = <0>;
+
+		dpi_out_real0: endpoint {
+			remote-endpoint = <&dp_bridge_input>;
+		};
+	};
+};
+
+&serdes_wiz4 {
+	lane0-mode = <PHY_TYPE_DP>;
+	lane1-mode = <PHY_TYPE_DP>;
+	lane2-mode = <PHY_TYPE_DP>;
+	lane3-mode = <PHY_TYPE_DP>;
+};
+
+&mhdp {
+	status = "ok";
+	pinctrl-names = "default";
+	pinctrl-0 = <&dp0_pins_default>;
+};
+
+&dp0_ports {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	port@0 {
+		reg = <0>;
+		dp_bridge_input: endpoint {
+			remote-endpoint = <&dpi_out_real0>;
+		};
+	};
+
+	port@1 {
+		reg = <1>;
+		dp_bridge_output: endpoint {
+			remote-endpoint = <&dp_connector_in>;
+		};
+	};
+};
+
+&main_i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_i2c0_pins_default>;
+	clock-frequency = <400000>;
+
+	exp1: gpio@20 {
+		compatible = "ti,tca6416";
+		reg = <0x20>;
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
+
+	exp2: gpio@22 {
+		compatible = "ti,tca6424";
+		reg = <0x22>;
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		p09 {
+			/* P11 - MCASP/TRACE_MUX_S0 */
+			gpio-hog;
+			gpios = <9 GPIO_ACTIVE_HIGH>;
+			output-low;
+			line-name = "MCASP/TRACE_MUX_S0";
+		};
+
+		p10 {
+			/* P12 - MCASP/TRACE_MUX_S1 */
+			gpio-hog;
+			gpios = <10 GPIO_ACTIVE_HIGH>;
+			output-high;
+			line-name = "MCASP/TRACE_MUX_S1";
+		};
+	};
+};
+
+&main_i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_i2c1_pins_default>;
+	clock-frequency = <400000>;
+
+	exp4: gpio@20 {
+		compatible = "ti,tca6408";
+		reg = <0x20>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&main_i2c1_exp4_pins_default>;
+		interrupt-parent = <&main_gpio1>;
+		interrupts = <11 IRQ_TYPE_EDGE_FALLING>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+
+		p0 {
+			/* P0 - DP0_PWR_SW_EN */
+			gpio-hog;
+			gpios = <0 GPIO_ACTIVE_HIGH>;
+			output-high;
+			line-name = "DP0_PWR_SW_EN";
+		};
+	};
+};
+
+&main_i2c3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_i2c3_pins_default>;
+	clock-frequency = <400000>;
+
+	exp3: gpio@20 {
+		compatible = "ti,tca6408";
+		reg = <0x20>;
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		p0 {
+			/* P0 - CODEC_RSTz */
+			gpio-hog;
+			gpios = <0 GPIO_ACTIVE_HIGH>;
+			output-high;
+			line-name = "CODEC_RSTz";
+		};
+	};
+
+	pcm3168a_1: audio-codec@44 {
+		compatible = "ti,pcm3168a";
+		reg = <0x44>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&audi_ext_refclk2_pins_default>;
+
+		#sound-dai-cells = <1>;
+
+		/* C_AUDIO_REFCLK2 -> RGMII6_RXC (W26) */
+		clocks = <&k3_clks 157 371>;
+		clock-names = "scki";
+
+		/* HSDIV3_16FFT_MAIN_4_HSDIVOUT2_CLK -> REFCLK2 */
+		assigned-clocks = <&k3_clks 157 371>;
+		assigned-clock-parents = <&k3_clks 157 400>;
+		assigned-clock-rates = <24576000>; /* for 48KHz */
+
+		VDD1-supply = <&vsys_3v3>;
+		VDD2-supply = <&vsys_3v3>;
+		VCCAD1-supply = <&vsys_5v0>;
+		VCCAD2-supply = <&vsys_5v0>;
+		VCCDA1-supply = <&vsys_5v0>;
+		VCCDA2-supply = <&vsys_5v0>;
+	};
+};
+
+&main_i2c6 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_i2c6_pins_default>;
+	clock-frequency = <400000>;
+
+	exp5: gpio@20 {
+		compatible = "ti,tca6408";
+		reg = <0x20>;
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
+};
+
+&mcu_cpsw {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcu_cpsw_pins_default &mcu_mdio_pins_default>;
+
+	cpts {
+		ti,pps = <3 1>;
+	};
+};
+
+&davinci_mdio {
+	phy0: ethernet-phy@0 {
+		reg = <0>;
+		ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
+		ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_4_B_NIB>;
+	};
+};
+
+&cpsw_port1 {
+	phy-mode = "rgmii-rxid";
+	phy-handle = <&phy0>;
+};
+
+#define TS_OFFSET(pa, val)     (0x4 + (pa) * 4) (0x10000 | val)
+
+&timesync_router {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcu_cpts>;
+
+	mcu_cpts: mcu_cpts {
+		pinctrl-single,pins = <
+			/* pps [cpts genf1] in17 -> out25 [cpts hw4_push] */
+			TS_OFFSET(25, 17)
+		>;
+	};
+};
+
+&ospi1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcu_fss0_ospi1_pins_default>;
+
+	flash@0{
+		compatible = "jedec,spi-nor";
+		reg = <0x0>;
+		spi-tx-bus-width = <1>;
+		spi-rx-bus-width = <4>;
+		spi-max-frequency = <100000000>;
+		cdns,tshsl-ns = <60>;
+		cdns,tsd2d-ns = <60>;
+		cdns,tchsh-ns = <60>;
+		cdns,tslch-ns = <60>;
+		cdns,read-delay = <2>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+	};
+};
+
+&mcasp10 {
+	#sound-dai-cells = <0>;
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcasp10_pins_default>;
+
+	op-mode = <0>;		/* MCASP_IIS_MODE */
+	tdm-slots = <2>;
+	auxclk-fs-ratio = <256>;
+
+	serial-dir = <	/* 0: INACTIVE, 1: TX, 2: RX */
+		2 2 2 1
+		1 1 1 0
+	>;
+	tx-num-evt = <0>;
+	rx-num-evt = <0>;
+
+	status = "okay";
+};
+
+&usb_serdes_mux {
+	idle-states = <1>, <0>; /* USB0 to SERDES3, USB1 to SERDES1 */
+};
+
+&serdes_ln_ctrl {
+	idle-states = <SERDES0_LANE0_PCIE0_LANE0>, <SERDES0_LANE1_PCIE0_LANE1>,
+		      <SERDES1_LANE0_PCIE1_LANE0>, <SERDES1_LANE1_PCIE1_LANE1>,
+		      <SERDES2_LANE0_PCIE2_LANE0>, <SERDES2_LANE1_PCIE2_LANE1>,
+		      <SERDES3_LANE0_USB3_0_SWAP>, <SERDES3_LANE1_USB3_0>,
+		      <SERDES4_LANE0_EDP_LANE0>, <SERDES4_LANE1_EDP_LANE1>, <SERDES4_LANE2_EDP_LANE2>, <SERDES4_LANE3_EDP_LANE3>;
+};
+
+&serdes_wiz3 {
+	typec-dir-gpios = <&main_gpio1 3 GPIO_ACTIVE_HIGH>;
+	typec-dir-debounce = <300>;	/* TUSB321, tCCB_DEFAULT 133 ms */
+	lane0-mode = <PHY_TYPE_USB3>;
+	lane1-mode = <PHY_TYPE_USB3>;
+};
+
+&serdes3 {
+	serdes3_usb_link: link@0 {
+		reg = <0>;
+		cdns,num-lanes = <2>;
+		#phy-cells = <0>;
+		cdns,phy-type = <PHY_TYPE_USB3>;
+		resets = <&serdes_wiz3 1>, <&serdes_wiz3 2>;
+	};
+};
+
+&usbss0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_usbss0_pins_default>;
+	ti,vbus-divider;
+};
+
+&usb0 {
+	dr_mode = "otg";
+	maximum-speed = "super-speed";
+	phys = <&serdes3_usb_link>;
+	phy-names = "cdns3,usb3-phy";
+};
+
+&usbss1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_usbss1_pins_default>;
+	ti,usb2-only;
+};
+
+&usb1 {
+	dr_mode = "host";
+	maximum-speed = "high-speed";
+};
+
+&main_sdhci0 {
+	/* eMMC */
+	non-removable;
+	ti,driver-strength-ohm = <50>;
+	disable-wp;
+};
+
+&main_sdhci1 {
+	/* SD/MMC */
+	vmmc-supply = <&vdd_mmc1>;
+	vqmmc-supply = <&ldo1_reg>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_mmc1_pins_default>;
+	disable-wp;
+};
+
+&main_sdhci2 {
+	/* Unused */
+	status = "disabled";
+};
+
+&serdes_wiz0 {
+	lane0-mode = <PHY_TYPE_PCIE>;
+	lane1-mode = <PHY_TYPE_PCIE>;
+};
+
+&serdes0 {
+	serdes0_pcie_link: link@0 {
+		reg = <0>;
+		cdns,num-lanes = <1>;
+		#phy-cells = <0>;
+		cdns,phy-type = <PHY_TYPE_PCIE>;
+		resets = <&serdes_wiz0 1>;
+	};
+};
+
+&serdes_wiz1 {
+	lane0-mode = <PHY_TYPE_PCIE>;
+	lane1-mode = <PHY_TYPE_PCIE>;
+};
+
+&serdes1 {
+	serdes1_pcie_link: link@0 {
+		reg = <0>;
+		cdns,num-lanes = <2>;
+		#phy-cells = <0>;
+		cdns,phy-type = <PHY_TYPE_PCIE>;
+		resets = <&serdes_wiz1 1>, <&serdes_wiz1 2>;
+	};
+};
+
+&serdes_wiz2 {
+	lane0-mode = <PHY_TYPE_PCIE>;
+	lane1-mode = <PHY_TYPE_PCIE>;
+};
+
+&serdes2 {
+	serdes2_pcie_link: link@0 {
+		reg = <0>;
+		cdns,num-lanes = <2>;
+		#phy-cells = <0>;
+		cdns,phy-type = <PHY_TYPE_PCIE>;
+		resets = <&serdes_wiz2 1>, <&serdes_wiz2 2>;
+	};
+};
+
+&pcie0 {
+	pci-mode = <PCI_MODE_RC>;
+	num-lanes = <1>;
+};
+
+&pcie1 {
+	pci-mode = <PCI_MODE_RC>;
+	num-lanes = <2>;
+};
+
+&pcie2 {
+	pci-mode = <PCI_MODE_RC>;
+	num-lanes = <2>;
+};
+
+&pcie0_rc {
+	reset-gpios = <&exp1 6 GPIO_ACTIVE_HIGH>;
+	phys = <&serdes0_pcie_link>;
+	phy-names = "pcie_phy";
+};
+
+&pcie1_rc {
+	reset-gpios = <&exp1 2 GPIO_ACTIVE_HIGH>;
+	phys = <&serdes1_pcie_link>;
+	phy-names = "pcie_phy";
+};
+
+&pcie2_rc {
+	reset-gpios = <&exp2 20 GPIO_ACTIVE_HIGH>;
+	phys = <&serdes2_pcie_link>;
+	phy-names = "pcie_phy";
+};
+
+&tscadc0 {
+	adc {
+		ti,adc-channels = <0 1 2 3 4 5 6 7>;
+	};
+};
+
+&tscadc1 {
+	adc {
+		ti,adc-channels = <0 1 2 3 4 5 6 7>;
+	};
+};
+
+&pcie0_ep {
+	phys = <&serdes0_pcie_link>;
+	phy-names = "pcie_phy";
+};
+
+&pcie1_ep {
+	phys = <&serdes1_pcie_link>;
+	phy-names = "pcie_phy";
+};
+
+&pcie2_ep {
+	phys = <&serdes2_pcie_link>;
+	phy-names = "pcie_phy";
+};
+
+/* uart2 assigned to cpsw9g eth-switch fw running on remote CPU core */
+&main_uart2 {
+	status = "disabled";
+};
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/k3-j721e-som-p0.dtsi b/arch/arm64/boot/dts/ti/k3-j721e-som-p0.dtsi
--- a/arch/arm64/boot/dts/ti/k3-j721e-som-p0.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/k3-j721e-som-p0.dtsi	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,270 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+
+#include "k3-j721e.dtsi"
+
+/ {
+	memory@80000000 {
+		device_type = "memory";
+		/* 4G RAM */
+		reg = <0x00000000 0x80000000 0x00000000 0x80000000>,
+		      <0x00000008 0x80000000 0x00000000 0x80000000>;
+	};
+
+	reserved_memory: reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		secure_ddr: optee@9e800000 {
+			reg = <0x00 0x9e800000 0x00 0x01800000>;
+			alignment = <0x1000>;
+			no-map;
+		};
+
+		mcu_r5fss0_core0_dma_memory_region: r5f-dma-memory@a0000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa0000000 0x00 0x100000>;
+			no-map;
+		};
+
+		mcu_r5fss0_core0_memory_region: r5f-memory@a0100000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa0100000 0x00 0xf00000>;
+			no-map;
+		};
+
+		mcu_r5fss0_core1_dma_memory_region: r5f-dma-memory@a1000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa1000000 0x00 0x100000>;
+			no-map;
+		};
+
+		mcu_r5fss0_core1_memory_region: r5f-memory@a1100000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa1100000 0x00 0xf00000>;
+			no-map;
+		};
+
+		main_r5fss0_core0_dma_memory_region: r5f-dma-memory@a2000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa2000000 0x00 0x100000>;
+			no-map;
+		};
+
+		main_r5fss0_core0_memory_region: r5f-memory@a2100000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa2100000 0x00 0xf00000>;
+			no-map;
+		};
+
+		main_r5fss0_core1_dma_memory_region: r5f-dma-memory@a3000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa3000000 0x00 0x100000>;
+			no-map;
+		};
+
+		main_r5fss0_core1_memory_region: r5f-memory@a3100000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa3100000 0x00 0xf00000>;
+			no-map;
+		};
+
+		main_r5fss1_core0_dma_memory_region: r5f-dma-memory@a4000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa4000000 0x00 0x100000>;
+			no-map;
+		};
+
+		main_r5fss1_core0_memory_region: r5f-memory@a4100000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa4100000 0x00 0xf00000>;
+			no-map;
+		};
+
+		main_r5fss1_core1_dma_memory_region: r5f-dma-memory@a5000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa5000000 0x00 0x100000>;
+			no-map;
+		};
+
+		main_r5fss1_core1_memory_region: r5f-memory@a5100000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa5100000 0x00 0xf00000>;
+			no-map;
+		};
+
+		c66_1_dma_memory_region: c66-dma-memory@a6000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa6000000 0x00 0x100000>;
+			no-map;
+		};
+
+		c66_0_memory_region: c66-memory@a6100000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa6100000 0x00 0xf00000>;
+			no-map;
+		};
+
+		c66_0_dma_memory_region: c66-dma-memory@a7000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa7000000 0x00 0x100000>;
+			no-map;
+		};
+
+		c66_1_memory_region: c66-memory@a7100000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa7100000 0x00 0xf00000>;
+			no-map;
+		};
+
+		c71_0_dma_memory_region: c71-dma-memory@a8000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa8000000 0x00 0x100000>;
+			no-map;
+		};
+
+		c71_0_memory_region: c71-memory@a8100000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa8100000 0x00 0xf00000>;
+			no-map;
+		};
+
+		rtos_ipc_memory_region: ipc-memories@aa000000 {
+			reg = <0x00 0xaa000000 0x00 0x01c00000>;
+			alignment = <0x1000>;
+			no-map;
+		};
+	};
+};
+
+&wkup_pmx0 {
+	wkup_i2c0_pins_default: wkup_i2c0_pins_default {
+		pinctrl-single,pins = <
+			J721E_WKUP_IOPAD(0xf8, PIN_INPUT_PULLUP, 0) /* (J25) WKUP_I2C0_SCL */
+			J721E_WKUP_IOPAD(0xfc, PIN_INPUT_PULLUP, 0) /* (H24) WKUP_I2C0_SDA */
+		>;
+	};
+
+	mcu_fss0_ospi0_pins_default: mcu-fss0-ospi0-pins-default {
+		pinctrl-single,pins = <
+			J721E_WKUP_IOPAD(0x0000, PIN_OUTPUT, 0) /* MCU_OSPI0_CLK */
+			J721E_WKUP_IOPAD(0x0008, PIN_INPUT, 0)  /* MCU_OSPI0_DQS */
+			J721E_WKUP_IOPAD(0x000c, PIN_INPUT, 0)  /* MCU_OSPI0_D0 */
+			J721E_WKUP_IOPAD(0x0010, PIN_INPUT, 0)  /* MCU_OSPI0_D1 */
+			J721E_WKUP_IOPAD(0x0014, PIN_INPUT, 0)  /* MCU_OSPI0_D2 */
+			J721E_WKUP_IOPAD(0x0018, PIN_INPUT, 0)  /* MCU_OSPI0_D3 */
+			J721E_WKUP_IOPAD(0x001c, PIN_INPUT, 0)  /* MCU_OSPI0_D4 */
+			J721E_WKUP_IOPAD(0x0020, PIN_INPUT, 0)  /* MCU_OSPI0_D5 */
+			J721E_WKUP_IOPAD(0x0024, PIN_INPUT, 0)  /* MCU_OSPI0_D6 */
+			J721E_WKUP_IOPAD(0x0028, PIN_INPUT, 0)  /* MCU_OSPI0_D7 */
+			J721E_WKUP_IOPAD(0x002c, PIN_OUTPUT, 0) /* MCU_OSPI0_CSn0 */
+		>;
+	};
+
+	mcu_fss0_hpb0_pins_default: mcu-fss0-hpb0-pins-default {
+		pinctrl-single,pins = <
+			J721E_WKUP_IOPAD(0x0, PIN_OUTPUT, 1) /* (E20) MCU_OSPI0_CLK.MCU_HYPERBUS0_CK */
+			J721E_WKUP_IOPAD(0x4, PIN_OUTPUT, 1) /* (C21) MCU_OSPI0_LBCLKO.MCU_HYPERBUS0_CKn */
+			J721E_WKUP_IOPAD(0x2c, PIN_OUTPUT, 1) /* (F19) MCU_OSPI0_CSn0.MCU_HYPERBUS0_CSn0 */
+			J721E_WKUP_IOPAD(0x54, PIN_OUTPUT, 3) /* (E22) MCU_OSPI1_CSn1.MCU_HYPERBUS0_CSn1 */
+			J721E_WKUP_IOPAD(0x30, PIN_OUTPUT, 1) /* (E19) MCU_OSPI0_CSn1.MCU_HYPERBUS0_RESETn */
+			J721E_WKUP_IOPAD(0x8, PIN_INPUT, 1) /* (D21) MCU_OSPI0_DQS.MCU_HYPERBUS0_RWDS */
+			J721E_WKUP_IOPAD(0xc, PIN_INPUT, 1) /* (D20) MCU_OSPI0_D0.MCU_HYPERBUS0_DQ0 */
+			J721E_WKUP_IOPAD(0x10, PIN_INPUT, 1) /* (G19) MCU_OSPI0_D1.MCU_HYPERBUS0_DQ1 */
+			J721E_WKUP_IOPAD(0x14, PIN_INPUT, 1) /* (G20) MCU_OSPI0_D2.MCU_HYPERBUS0_DQ2 */
+			J721E_WKUP_IOPAD(0x18, PIN_INPUT, 1) /* (F20) MCU_OSPI0_D3.MCU_HYPERBUS0_DQ3 */
+			J721E_WKUP_IOPAD(0x1c, PIN_INPUT, 1) /* (F21) MCU_OSPI0_D4.MCU_HYPERBUS0_DQ4 */
+			J721E_WKUP_IOPAD(0x20, PIN_INPUT, 1) /* (E21) MCU_OSPI0_D5.MCU_HYPERBUS0_DQ5 */
+			J721E_WKUP_IOPAD(0x24, PIN_INPUT, 1) /* (B22) MCU_OSPI0_D6.MCU_HYPERBUS0_DQ6 */
+			J721E_WKUP_IOPAD(0x28, PIN_INPUT, 1) /* (G21) MCU_OSPI0_D7.MCU_HYPERBUS0_DQ7 */
+		>;
+	};
+};
+
+&ospi0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcu_fss0_ospi0_pins_default>;
+
+	flash@0{
+		compatible = "jedec,spi-nor";
+		reg = <0x0>;
+		spi-tx-bus-width = <8>;
+		spi-rx-bus-width = <8>;
+		spi-max-frequency = <50000000>;
+		spi-dqs;
+		cdns,tshsl-ns = <60>;
+		cdns,tsd2d-ns = <60>;
+		cdns,tchsh-ns = <60>;
+		cdns,tslch-ns = <60>;
+		cdns,read-delay = <0>;
+		cdns,phy-mode;
+		#address-cells = <1>;
+		#size-cells = <1>;
+	};
+};
+
+&mcu_r5fss0_core0 {
+	memory-region = <&mcu_r5fss0_core0_dma_memory_region>,
+			<&mcu_r5fss0_core0_memory_region>;
+};
+
+&mcu_r5fss0_core1 {
+	memory-region = <&mcu_r5fss0_core1_dma_memory_region>,
+			<&mcu_r5fss0_core1_memory_region>;
+};
+
+&main_r5fss0_core0 {
+	memory-region = <&main_r5fss0_core0_dma_memory_region>,
+			<&main_r5fss0_core0_memory_region>;
+};
+
+&main_r5fss0_core1 {
+	memory-region = <&main_r5fss0_core1_dma_memory_region>,
+			<&main_r5fss0_core1_memory_region>;
+};
+
+&main_r5fss1_core0 {
+	memory-region = <&main_r5fss1_core0_dma_memory_region>,
+			<&main_r5fss1_core0_memory_region>;
+};
+
+&main_r5fss1_core1 {
+	memory-region = <&main_r5fss1_core1_dma_memory_region>,
+			<&main_r5fss1_core1_memory_region>;
+};
+
+&c66_0 {
+	memory-region = <&c66_0_dma_memory_region>,
+			<&c66_0_memory_region>;
+};
+
+&c66_1 {
+	memory-region = <&c66_1_dma_memory_region>,
+			<&c66_1_memory_region>;
+};
+
+&c71_0 {
+	memory-region = <&c71_0_dma_memory_region>,
+			<&c71_0_memory_region>;
+};
+
+&hbmc {
+	/* OSPI and HBMC are muxed inside FSS, Bootloader will enable
+	 * appropriate node based on board detection
+	 */
+	status = "disabled";
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcu_fss0_hpb0_pins_default>;
+	ranges = <0x0 0x0 0x5 0x00000000 0x4000000>, /* 64MB Flash on CS0 */
+		 <0x1 0x0 0x5 0x04000000 0x800000>; /* 8MB RAM on CS1 */
+
+	flash@0,0 {
+		compatible = "cypress,hyperflash", "cfi-flash";
+		reg = <0x0 0x0 0x4000000>;
+	};
+};
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/k3-j721e-som-tps65917.dtsi b/arch/arm64/boot/dts/ti/k3-j721e-som-tps65917.dtsi
--- a/arch/arm64/boot/dts/ti/k3-j721e-som-tps65917.dtsi	1969-12-31 19:00:00.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/k3-j721e-som-tps65917.dtsi	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,307 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/dts-v1/;
+
+#include "k3-j721e.dtsi"
+
+/ {
+	memory@80000000 {
+		device_type = "memory";
+		/* 4G RAM */
+		reg = <0x00000000 0x80000000 0x00000000 0x80000000>,
+		      <0x00000008 0x80000000 0x00000000 0x80000000>;
+	};
+
+	reserved_memory: reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		secure_ddr: optee@9e800000 {
+			reg = <0x00 0x9e800000 0x00 0x01800000>;
+			alignment = <0x1000>;
+			no-map;
+		};
+
+		mcu_r5fss0_core0_dma_memory_region: r5f-dma-memory@a0000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa0000000 0x00 0x100000>;
+			no-map;
+		};
+
+		mcu_r5fss0_core0_memory_region: r5f-memory@a0100000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa0100000 0x00 0xf00000>;
+			no-map;
+		};
+
+		mcu_r5fss0_core1_dma_memory_region: r5f-dma-memory@a1000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa1000000 0x00 0x100000>;
+			no-map;
+		};
+
+		mcu_r5fss0_core1_memory_region: r5f-memory@a1100000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa1100000 0x00 0xf00000>;
+			no-map;
+		};
+
+		main_r5fss0_core0_dma_memory_region: r5f-dma-memory@a2000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa2000000 0x00 0x100000>;
+			no-map;
+		};
+
+		main_r5fss0_core0_memory_region: r5f-memory@a2100000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa2100000 0x00 0xf00000>;
+			no-map;
+		};
+
+		main_r5fss0_core1_dma_memory_region: r5f-dma-memory@a3000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa3000000 0x00 0x100000>;
+			no-map;
+		};
+
+		main_r5fss0_core1_memory_region: r5f-memory@a3100000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa3100000 0x00 0xf00000>;
+			no-map;
+		};
+
+		main_r5fss1_core0_dma_memory_region: r5f-dma-memory@a4000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa4000000 0x00 0x100000>;
+			no-map;
+		};
+
+		main_r5fss1_core0_memory_region: r5f-memory@a4100000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa4100000 0x00 0xf00000>;
+			no-map;
+		};
+
+		main_r5fss1_core1_dma_memory_region: r5f-dma-memory@a5000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa5000000 0x00 0x100000>;
+			no-map;
+		};
+
+		main_r5fss1_core1_memory_region: r5f-memory@a5100000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa5100000 0x00 0xf00000>;
+			no-map;
+		};
+
+		c66_1_dma_memory_region: c66-dma-memory@a6000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa6000000 0x00 0x100000>;
+			no-map;
+		};
+
+		c66_0_memory_region: c66-memory@a6100000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa6100000 0x00 0xf00000>;
+			no-map;
+		};
+
+		c66_0_dma_memory_region: c66-dma-memory@a7000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa7000000 0x00 0x100000>;
+			no-map;
+		};
+
+		c66_1_memory_region: c66-memory@a7100000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa7100000 0x00 0xf00000>;
+			no-map;
+		};
+
+		c71_0_dma_memory_region: c71-dma-memory@a8000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa8000000 0x00 0x100000>;
+			no-map;
+		};
+
+		c71_0_memory_region: c71-memory@a8100000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa8100000 0x00 0xf00000>;
+			no-map;
+		};
+
+		rtos_ipc_memory_region: ipc-memories@aa000000 {
+			reg = <0x00 0xaa000000 0x00 0x01c00000>;
+			alignment = <0x1000>;
+			no-map;
+		};
+	};
+};
+
+&wkup_pmx0 {
+	wkup_i2c0_pins_default: wkup_i2c0_pins_default {
+		pinctrl-single,pins = <
+			J721E_WKUP_IOPAD(0xf8, PIN_INPUT_PULLUP, 0) /* (J25) WKUP_I2C0_SCL */
+			J721E_WKUP_IOPAD(0xfc, PIN_INPUT_PULLUP, 0) /* (H24) WKUP_I2C0_SDA */
+		>;
+	};
+
+	mcu_fss0_ospi0_pins_default: mcu-fss0-ospi0-pins-default {
+		pinctrl-single,pins = <
+			J721E_WKUP_IOPAD(0x0000, PIN_OUTPUT, 0) /* MCU_OSPI0_CLK */
+			J721E_WKUP_IOPAD(0x0008, PIN_INPUT, 0)  /* MCU_OSPI0_DQS */
+			J721E_WKUP_IOPAD(0x000c, PIN_INPUT, 0)  /* MCU_OSPI0_D0 */
+			J721E_WKUP_IOPAD(0x0010, PIN_INPUT, 0)  /* MCU_OSPI0_D1 */
+			J721E_WKUP_IOPAD(0x0014, PIN_INPUT, 0)  /* MCU_OSPI0_D2 */
+			J721E_WKUP_IOPAD(0x0018, PIN_INPUT, 0)  /* MCU_OSPI0_D3 */
+			J721E_WKUP_IOPAD(0x001c, PIN_INPUT, 0)  /* MCU_OSPI0_D4 */
+			J721E_WKUP_IOPAD(0x0020, PIN_INPUT, 0)  /* MCU_OSPI0_D5 */
+			J721E_WKUP_IOPAD(0x0024, PIN_INPUT, 0)  /* MCU_OSPI0_D6 */
+			J721E_WKUP_IOPAD(0x0028, PIN_INPUT, 0)  /* MCU_OSPI0_D7 */
+			J721E_WKUP_IOPAD(0x002c, PIN_OUTPUT, 0) /* MCU_OSPI0_CSn0 */
+		>;
+	};
+
+	mcu_fss0_hpb0_pins_default: mcu-fss0-hpb0-pins-default {
+		pinctrl-single,pins = <
+			J721E_WKUP_IOPAD(0x0, PIN_OUTPUT, 1) /* (E20) MCU_OSPI0_CLK.MCU_HYPERBUS0_CK */
+			J721E_WKUP_IOPAD(0x4, PIN_OUTPUT, 1) /* (C21) MCU_OSPI0_LBCLKO.MCU_HYPERBUS0_CKn */
+			J721E_WKUP_IOPAD(0x2c, PIN_OUTPUT, 1) /* (F19) MCU_OSPI0_CSn0.MCU_HYPERBUS0_CSn0 */
+			J721E_WKUP_IOPAD(0x54, PIN_OUTPUT, 3) /* (E22) MCU_OSPI1_CSn1.MCU_HYPERBUS0_CSn1 */
+			J721E_WKUP_IOPAD(0x30, PIN_OUTPUT, 1) /* (E19) MCU_OSPI0_CSn1.MCU_HYPERBUS0_RESETn */
+			J721E_WKUP_IOPAD(0x8, PIN_INPUT, 1) /* (D21) MCU_OSPI0_DQS.MCU_HYPERBUS0_RWDS */
+			J721E_WKUP_IOPAD(0xc, PIN_INPUT, 1) /* (D20) MCU_OSPI0_D0.MCU_HYPERBUS0_DQ0 */
+			J721E_WKUP_IOPAD(0x10, PIN_INPUT, 1) /* (G19) MCU_OSPI0_D1.MCU_HYPERBUS0_DQ1 */
+			J721E_WKUP_IOPAD(0x14, PIN_INPUT, 1) /* (G20) MCU_OSPI0_D2.MCU_HYPERBUS0_DQ2 */
+			J721E_WKUP_IOPAD(0x18, PIN_INPUT, 1) /* (F20) MCU_OSPI0_D3.MCU_HYPERBUS0_DQ3 */
+			J721E_WKUP_IOPAD(0x1c, PIN_INPUT, 1) /* (F21) MCU_OSPI0_D4.MCU_HYPERBUS0_DQ4 */
+			J721E_WKUP_IOPAD(0x20, PIN_INPUT, 1) /* (E21) MCU_OSPI0_D5.MCU_HYPERBUS0_DQ5 */
+			J721E_WKUP_IOPAD(0x24, PIN_INPUT, 1) /* (B22) MCU_OSPI0_D6.MCU_HYPERBUS0_DQ6 */
+			J721E_WKUP_IOPAD(0x28, PIN_INPUT, 1) /* (G21) MCU_OSPI0_D7.MCU_HYPERBUS0_DQ7 */
+		>;
+	};
+};
+
+&wkup_i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&wkup_i2c0_pins_default>;
+	clock-frequency = <400000>;
+
+	tps65917: tps65917@58 {
+		reg = <0x58>;
+		compatible = "ti,tps65917";
+
+		tps65917_pmic {
+			compatible = "ti,tps65917-pmic";
+
+			ldo1-in-supply = <&vsys_3v3>;
+
+			tps65917_regulators: regulators {
+				ldo1_reg: ldo1 {
+					/* LDO1_OUT --> VDD_SD_DV_REG  */
+					regulator-name = "ldo1";
+					regulator-min-microvolt = <1800000>;
+					regulator-max-microvolt = <3300000>;
+					regulator-allow-bypass;
+				};
+
+				ldo2_reg: ldo2 {
+					/* LDO2_OUT --> VDA_USB_3V3_REG  */
+					regulator-name = "ldo2";
+					regulator-min-microvolt = <3300000>;
+					regulator-max-microvolt = <3300000>;
+					regulator-allow-bypass;
+					regulator-always-on;
+					regulator-boot-on;
+				};
+			};
+		};
+	};
+};
+
+&ospi0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcu_fss0_ospi0_pins_default>;
+
+	flash@0{
+		compatible = "jedec,spi-nor";
+		reg = <0x0>;
+		spi-tx-bus-width = <8>;
+		spi-rx-bus-width = <8>;
+		spi-max-frequency = <50000000>;
+		spi-dqs;
+		cdns,tshsl-ns = <60>;
+		cdns,tsd2d-ns = <60>;
+		cdns,tchsh-ns = <60>;
+		cdns,tslch-ns = <60>;
+		cdns,read-delay = <0>;
+		cdns,phy-mode;
+		#address-cells = <1>;
+		#size-cells = <1>;
+	};
+};
+
+&mcu_r5fss0_core0 {
+	memory-region = <&mcu_r5fss0_core0_dma_memory_region>,
+			<&mcu_r5fss0_core0_memory_region>;
+};
+
+&mcu_r5fss0_core1 {
+	memory-region = <&mcu_r5fss0_core1_dma_memory_region>,
+			<&mcu_r5fss0_core1_memory_region>;
+};
+
+&main_r5fss0_core0 {
+	memory-region = <&main_r5fss0_core0_dma_memory_region>,
+			<&main_r5fss0_core0_memory_region>;
+};
+
+&main_r5fss0_core1 {
+	memory-region = <&main_r5fss0_core1_dma_memory_region>,
+			<&main_r5fss0_core1_memory_region>;
+};
+
+&main_r5fss1_core0 {
+	memory-region = <&main_r5fss1_core0_dma_memory_region>,
+			<&main_r5fss1_core0_memory_region>;
+};
+
+&main_r5fss1_core1 {
+	memory-region = <&main_r5fss1_core1_dma_memory_region>,
+			<&main_r5fss1_core1_memory_region>;
+};
+
+&c66_0 {
+	memory-region = <&c66_0_dma_memory_region>,
+			<&c66_0_memory_region>;
+};
+
+&c66_1 {
+	memory-region = <&c66_1_dma_memory_region>,
+			<&c66_1_memory_region>;
+};
+
+&c71_0 {
+	memory-region = <&c71_0_dma_memory_region>,
+			<&c71_0_memory_region>;
+};
+
+&hbmc {
+	/* OSPI and HBMC are muxed inside FSS, Bootloader will enable
+	 * appropriate node based on board detection
+	 */
+	status = "disabled";
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcu_fss0_hpb0_pins_default>;
+	ranges = <0x0 0x0 0x5 0x00000000 0x4000000>, /* 64MB Flash on CS0 */
+		 <0x1 0x0 0x5 0x04000000 0x800000>; /* 8MB RAM on CS1 */
+
+	flash@0,0 {
+		compatible = "cypress,hyperflash", "cfi-flash";
+		reg = <0x0 0x0 0x4000000>;
+	};
+};
diff -Naur --no-dereference a/arch/arm64/boot/dts/ti/Makefile b/arch/arm64/boot/dts/ti/Makefile
--- a/arch/arm64/boot/dts/ti/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm64/boot/dts/ti/Makefile	2020-06-19 21:00:31.408227715 -0400
@@ -6,4 +6,30 @@
 # Copyright (C) 2016-2018 Texas Instruments Incorporated - http://www.ti.com/
 #
 
-dtb-$(CONFIG_ARCH_K3_AM6_SOC) += k3-am654-base-board.dtb
+DTC_FLAGS += -@
+
+dtb-$(CONFIG_ARCH_K3_AM6_SOC) += k3-am654-base-board.dtb \
+	k3-am654-gp.dtbo \
+	k3-am654-evm-hdmi.dtbo \
+	k3-am654-evm-oldi-lcd1evm.dtbo \
+	k3-am654-evm-tc358876.dtbo \
+	k3-am654-pcie-usb3.dtbo \
+	k3-am654-pcie-usb2.dtbo \
+	k3-am654-evm-csi2-ov490.dtbo \
+	k3-am654-evm-csi2-ov5640.dtbo \
+	k3-am654-idk.dtbo \
+	k3-am654-base-board-jailhouse.dtbo \
+	k3-am654-evm-prupwm.dtbo
+
+dtb-$(CONFIG_ARCH_K3_J721E_SOC) += k3-j721e-common-proc-board.dtb \
+				   k3-j721e-proc-board-tps65917.dtb \
+				   k3-j721e-common-proc-board-infotainment.dtbo \
+				   k3-j721e-common-proc-board-infotainment-display-sharing.dtbo \
+				   k3-j721e-common-proc-board-jailhouse.dtbo \
+				   k3-j721e-pcie-backplane.dtbo
+
+$(obj)/%.dtbo: $(src)/%.dtso FORCE
+	$(call if_changed_dep,dtc)
+
+always		:= $(dtb-y)
+clean-files	:= *.dtb *.dtbo
diff -Naur --no-dereference a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig
--- a/arch/arm64/configs/defconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm64/configs/defconfig	2020-06-19 21:00:31.408227715 -0400
@@ -115,6 +115,7 @@
 CONFIG_ARM_BIG_LITTLE_CPUFREQ=y
 CONFIG_ARM_SCPI_CPUFREQ=y
 CONFIG_ARM_TEGRA186_CPUFREQ=y
+CONFIG_TI_SCI_PROTOCOL=y
 CONFIG_NET=y
 CONFIG_PACKET=y
 CONFIG_UNIX=y
@@ -279,6 +280,7 @@
 CONFIG_SERIAL_8250_SHARE_IRQ=y
 CONFIG_SERIAL_8250_BCM2835AUX=y
 CONFIG_SERIAL_8250_DW=y
+CONFIG_SERIAL_8250_OMAP=y
 CONFIG_SERIAL_8250_MT6577=y
 CONFIG_SERIAL_8250_UNIPHIER=y
 CONFIG_SERIAL_OF_PLATFORM=y
@@ -577,6 +579,7 @@
 CONFIG_COMMON_CLK_S2MPS11=y
 CONFIG_CLK_QORIQ=y
 CONFIG_COMMON_CLK_PWM=y
+CONFIG_TI_SCI_CLK=y
 CONFIG_COMMON_CLK_QCOM=y
 CONFIG_QCOM_CLK_SMD_RPM=y
 CONFIG_IPQ_GCC_8074=y
@@ -588,6 +591,7 @@
 CONFIG_ARM_MHU=y
 CONFIG_PLATFORM_MHU=y
 CONFIG_BCM2835_MBOX=y
+CONFIG_TI_MESSAGE_MANAGER=y
 CONFIG_QCOM_APCS_IPC=y
 CONFIG_ROCKCHIP_IOMMU=y
 CONFIG_TEGRA_IOMMU_SMMU=y
@@ -607,7 +611,9 @@
 CONFIG_ARCH_TEGRA_186_SOC=y
 CONFIG_ARCH_TEGRA_194_SOC=y
 CONFIG_ARCH_K3_AM6_SOC=y
+CONFIG_ARCH_K3_J721E_SOC=y
 CONFIG_SOC_TI=y
+CONFIG_TI_SCI_PM_DOMAINS=y
 CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND=y
 CONFIG_EXTCON_USB_GPIO=y
 CONFIG_EXTCON_USBC_CROS_EC=y
@@ -627,6 +633,7 @@
 CONFIG_PWM_ROCKCHIP=y
 CONFIG_PWM_SAMSUNG=y
 CONFIG_PWM_TEGRA=m
+CONFIG_RESET_TI_SCI=y
 CONFIG_PHY_XGENE=y
 CONFIG_PHY_SUN4I_USB=y
 CONFIG_PHY_HI6220_USB=y
diff -Naur --no-dereference a/arch/arm64/Kconfig.platforms b/arch/arm64/Kconfig.platforms
--- a/arch/arm64/Kconfig.platforms	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm64/Kconfig.platforms	2020-06-19 21:00:31.408227715 -0400
@@ -74,6 +74,11 @@
 config ARCH_K3
 	bool "Texas Instruments Inc. K3 multicore SoC architecture"
 	select PM_GENERIC_DOMAINS if PM
+	select MAILBOX
+	select TI_MESSAGE_MANAGER
+	select TI_SCI_PROTOCOL
+	select TI_SCI_INTR_IRQCHIP
+	select TI_SCI_INTA_IRQCHIP
 	help
 	  This enables support for Texas Instruments' K3 multicore SoC
 	  architecture.
diff -Naur --no-dereference a/arch/arm64/kernel/arm64ksyms.c b/arch/arm64/kernel/arm64ksyms.c
--- a/arch/arm64/kernel/arm64ksyms.c	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm64/kernel/arm64ksyms.c	2020-06-19 21:00:31.408227715 -0400
@@ -86,3 +86,7 @@
 EXPORT_SYMBOL(__ashrti3);
 extern long long __lshrti3(long long a, int b);
 EXPORT_SYMBOL(__lshrti3);
+
+	/* hyp-stub.S */
+extern char __hyp_stub_vectors[];
+EXPORT_SYMBOL(__hyp_stub_vectors);
diff -Naur --no-dereference a/arch/arm64/Makefile b/arch/arm64/Makefile
--- a/arch/arm64/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/arch/arm64/Makefile	2020-06-19 21:00:31.408227715 -0400
@@ -128,8 +128,10 @@
 zinstall install:
 	$(Q)$(MAKE) $(build)=$(boot) $@
 
-%.dtb: scripts
-	$(Q)$(MAKE) $(build)=$(boot)/dts $(boot)/dts/$@
+%.dtb %.dtbo: dtbdir=$(patsubst %/,%,$(patsubst %./,%,$(boot)/dts/$(dir $@)))
+
+%.dtb %.dtbo: scripts
+	$(Q)$(MAKE) $(build)=$(dtbdir) $(boot)/dts/$@
 
 PHONY += dtbs dtbs_install
 
diff -Naur --no-dereference a/block/bsg-lib.c b/block/bsg-lib.c
--- a/block/bsg-lib.c	2020-01-09 04:19:10.000000000 -0500
+++ b/block/bsg-lib.c	2020-06-19 21:00:31.408227715 -0400
@@ -296,6 +296,15 @@
 	kfree(job->reply);
 }
 
+void bsg_remove_queue(struct request_queue *q)
+{
+	if (q) {
+		bsg_unregister_queue(q);
+		blk_cleanup_queue(q);
+	}
+}
+EXPORT_SYMBOL_GPL(bsg_remove_queue);
+
 /**
  * bsg_setup_queue - Create and add the bsg hooks so we can receive requests
  * @dev: device to attach bsg device to
@@ -304,7 +313,7 @@
  * @dd_job_size: size of LLD data needed for each job
  */
 struct request_queue *bsg_setup_queue(struct device *dev, const char *name,
-		bsg_job_fn *job_fn, int dd_job_size)
+		bsg_job_fn *job_fn, rq_timed_out_fn *timeout, int dd_job_size)
 {
 	struct request_queue *q;
 	int ret;
@@ -327,6 +336,7 @@
 	blk_queue_flag_set(QUEUE_FLAG_BIDI, q);
 	blk_queue_softirq_done(q, bsg_softirq_done);
 	blk_queue_rq_timeout(q, BLK_DEFAULT_SG_TIMEOUT);
+	blk_queue_rq_timed_out(q, timeout);
 
 	ret = bsg_register_queue(q, dev, name, &bsg_transport_ops);
 	if (ret) {
diff -Naur --no-dereference a/crypto/sha256_generic.c b/crypto/sha256_generic.c
--- a/crypto/sha256_generic.c	2020-01-09 04:19:10.000000000 -0500
+++ b/crypto/sha256_generic.c	2020-06-19 21:00:31.408227715 -0400
@@ -68,7 +68,7 @@
 	W[I] = s1(W[I-2]) + W[I-7] + s0(W[I-15]) + W[I-16];
 }
 
-static void sha256_transform(u32 *state, const u8 *input)
+void sha256_transform(u32 *state, const u8 *input)
 {
 	u32 a, b, c, d, e, f, g, h, t1, t2;
 	u32 W[64];
@@ -230,6 +230,7 @@
 	a = b = c = d = e = f = g = h = t1 = t2 = 0;
 	memzero_explicit(W, 64 * sizeof(u32));
 }
+EXPORT_SYMBOL(sha256_transform);
 
 static void sha256_generic_block_fn(struct sha256_state *sst, u8 const *src,
 				    int blocks)
diff -Naur --no-dereference a/crypto/tcrypt.c b/crypto/tcrypt.c
--- a/crypto/tcrypt.c	2020-01-09 04:19:10.000000000 -0500
+++ b/crypto/tcrypt.c	2020-06-19 21:00:31.408227715 -0400
@@ -1110,6 +1110,7 @@
 			"(%5u byte blocks,%5u bytes per update,%4u updates): ",
 			i, speed[i].blen, speed[i].plen, speed[i].blen / speed[i].plen);
 
+		sg_set_buf(sg, tvmem[0], speed[i].plen);
 		ahash_request_set_crypt(req, sg, output, speed[i].plen);
 
 		if (secs) {
diff -Naur --no-dereference a/Documentation/ABI/testing/debugfs-aufs b/Documentation/ABI/testing/debugfs-aufs
--- a/Documentation/ABI/testing/debugfs-aufs	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/ABI/testing/debugfs-aufs	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,55 @@
+What:		/debug/aufs/si_<id>/
+Date:		March 2009
+Contact:	J. R. Okajima <hooanon05g@gmail.com>
+Description:
+		Under /debug/aufs, a directory named si_<id> is created
+		per aufs mount, where <id> is a unique id generated
+		internally.
+
+What:		/debug/aufs/si_<id>/plink
+Date:		Apr 2013
+Contact:	J. R. Okajima <hooanon05g@gmail.com>
+Description:
+		It has three lines and shows the information about the
+		pseudo-link. The first line is a single number
+		representing a number of buckets. The second line is a
+		number of pseudo-links per buckets (separated by a
+		blank). The last line is a single number representing a
+		total number of psedo-links.
+		When the aufs mount option 'noplink' is specified, it
+		will show "1\n0\n0\n".
+
+What:		/debug/aufs/si_<id>/xib
+Date:		March 2009
+Contact:	J. R. Okajima <hooanon05g@gmail.com>
+Description:
+		It shows the consumed blocks by xib (External Inode Number
+		Bitmap), its block size and file size.
+		When the aufs mount option 'noxino' is specified, it
+		will be empty. About XINO files, see the aufs manual.
+
+What:		/debug/aufs/si_<id>/xi0, xi1 ... xiN and xiN-N
+Date:		March 2009
+Contact:	J. R. Okajima <hooanon05g@gmail.com>
+Description:
+		It shows the consumed blocks by xino (External Inode Number
+		Translation Table), its link count, block size and file
+		size.
+		Due to the file size limit, there may exist multiple
+		xino files per branch.  In this case, "-N" is added to
+		the filename and it corresponds to the index of the
+		internal xino array.  "-0" is omitted.
+		When the aufs mount option 'noxino' is specified, Those
+		entries won't exist.  About XINO files, see the aufs
+		manual.
+
+What:		/debug/aufs/si_<id>/xigen
+Date:		March 2009
+Contact:	J. R. Okajima <hooanon05g@gmail.com>
+Description:
+		It shows the consumed blocks by xigen (External Inode
+		Generation Table), its block size and file size.
+		If CONFIG_AUFS_EXPORT is disabled, this entry will not
+		be created.
+		When the aufs mount option 'noxino' is specified, it
+		will be empty. About XINO files, see the aufs manual.
diff -Naur --no-dereference a/Documentation/ABI/testing/sysfs-aufs b/Documentation/ABI/testing/sysfs-aufs
--- a/Documentation/ABI/testing/sysfs-aufs	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/ABI/testing/sysfs-aufs	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,31 @@
+What:		/sys/fs/aufs/si_<id>/
+Date:		March 2009
+Contact:	J. R. Okajima <hooanon05g@gmail.com>
+Description:
+		Under /sys/fs/aufs, a directory named si_<id> is created
+		per aufs mount, where <id> is a unique id generated
+		internally.
+
+What:		/sys/fs/aufs/si_<id>/br0, br1 ... brN
+Date:		March 2009
+Contact:	J. R. Okajima <hooanon05g@gmail.com>
+Description:
+		It shows the abolute path of a member directory (which
+		is called branch) in aufs, and its permission.
+
+What:		/sys/fs/aufs/si_<id>/brid0, brid1 ... bridN
+Date:		July 2013
+Contact:	J. R. Okajima <hooanon05g@gmail.com>
+Description:
+		It shows the id of a member directory (which is called
+		branch) in aufs.
+
+What:		/sys/fs/aufs/si_<id>/xi_path
+Date:		March 2009
+Contact:	J. R. Okajima <hooanon05g@gmail.com>
+Description:
+		It shows the abolute path of XINO (External Inode Number
+		Bitmap, Translation Table and Generation Table) file
+		even if it is the default path.
+		When the aufs mount option 'noxino' is specified, it
+		will be empty. About XINO files, see the aufs manual.
diff -Naur --no-dereference a/Documentation/ABI/testing/sysfs-bus-rpmsg b/Documentation/ABI/testing/sysfs-bus-rpmsg
--- a/Documentation/ABI/testing/sysfs-bus-rpmsg	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/ABI/testing/sysfs-bus-rpmsg	2020-06-19 21:00:31.400227715 -0400
@@ -93,3 +93,32 @@
 		This sysfs entry allows the rpmsg driver for a rpmsg device
 		to be specified which will override standard OF, ID table
 		and name matching.
+
+What:		/sys/bus/rpmsg/devices/.../desc
+Date:		September 2018
+KernelVersion:	4.19
+Contact:	Suman Anna <s-anna@ti.com>
+Description:
+		Every rpmsg device is a communication channel with a remote
+		processor. Channels are identified by a textual name (see
+		/sys/bus/rpmsg/devices/.../name above) and have a local
+		("source") rpmsg address, and remote ("destination") rpmsg
+		address.
+
+		A channel is first created when an entity, whether local
+		or remote, starts listening on it for messages (and is thus
+		called an rpmsg server). When that happens, a "name service"
+		announcement is sent to the other processor, in order to let
+		it know about the creation of the channel (this way remote
+		clients know they can start sending messages).
+
+		The listening entity (or client) which communicates with a
+		remote processor is referred as rpmsg driver. The rpmsg device
+		and rpmsg driver are matched based on rpmsg device name (see
+		/sys/bus/rpmsg/devices/.../name above) and rpmsg driver ID table.
+
+		This sysfs entry contains an additional optional description of
+		the rpmsg device that can be optionally included as part of the
+		"name service" announcement. This description is then passed on
+		to the corresponding rpmsg drivers to further distinguish multiple
+		devices associated with the same rpmsg driver.
diff -Naur --no-dereference a/Documentation/ABI/testing/sysfs-class-remoteproc b/Documentation/ABI/testing/sysfs-class-remoteproc
--- a/Documentation/ABI/testing/sysfs-class-remoteproc	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/ABI/testing/sysfs-class-remoteproc	2020-06-19 21:00:31.400227715 -0400
@@ -48,3 +48,13 @@
 
 		Writing "stop" will attempt to halt the remote processor and
 		return it to the "offline" state.
+
+What:		/sys/class/remoteproc/.../name
+Date:		August 2019
+KernelVersion:	5.4
+Contact:	Suman Anna <s-anna@ti.com>
+Description:	Remote processor name
+
+		Reports the name of the remote processor. This can be used by
+		userspace in exactly identifying a remote processor and ease
+		up the usage in modifying the 'firmware' or 'state' files.
diff -Naur --no-dereference a/Documentation/devicetree/bindings/arm/cpus.txt b/Documentation/devicetree/bindings/arm/cpus.txt
--- a/Documentation/devicetree/bindings/arm/cpus.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/arm/cpus.txt	2020-06-19 21:00:31.400227715 -0400
@@ -224,6 +224,8 @@
 			    "rockchip,rk3036-smp"
 			    "rockchip,rk3066-smp"
 			    "ste,dbx500-smp"
+			    "ti,am3352"
+			    "ti,am4372"
 
 	- cpu-release-addr
 		Usage: required for systems that have an "enable-method"
diff -Naur --no-dereference a/Documentation/devicetree/bindings/arm/keystone/ti,sci.txt b/Documentation/devicetree/bindings/arm/keystone/ti,sci.txt
--- a/Documentation/devicetree/bindings/arm/keystone/ti,sci.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/arm/keystone/ti,sci.txt	2020-06-19 21:00:31.400227715 -0400
@@ -24,7 +24,8 @@
 
 Required properties:
 -------------------
-- compatible: should be "ti,k2g-sci"
+- compatible:	should be "ti,k2g-sci" for TI 66AK2G SoC
+		should be "ti,am654-sci" for for TI AM654 SoC
 - mbox-names:
 	"rx" - Mailbox corresponding to receive path
 	"tx" - Mailbox corresponding to transmit path
@@ -45,11 +46,15 @@
 	debug_messages - Map the Debug message region
 - reg:  register space corresponding to the debug_messages
 - ti,system-reboot-controller: If system reboot can be triggered by SoC reboot
+- ti,host-id: Integer value corresponding to the host ID assigned by Firmware
+	for identification of host processing entities such as virtual
+	machines
 
 Example (K2G):
 -------------
 	pmmc: pmmc {
 		compatible = "ti,k2g-sci";
+		ti,host-id = <2>;
 		mbox-names = "rx", "tx";
 		mboxes= <&msgmgr &msgmgr_proxy_pmmc_rx>,
 			<&msgmgr &msgmgr_proxy_pmmc_tx>;
diff -Naur --no-dereference a/Documentation/devicetree/bindings/arm/omap/omap.txt b/Documentation/devicetree/bindings/arm/omap/omap.txt
--- a/Documentation/devicetree/bindings/arm/omap/omap.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/arm/omap/omap.txt	2020-06-19 21:00:31.400227715 -0400
@@ -24,6 +24,8 @@
 - ti,no-reset-on-init: When present, the module should not be reset at init
 - ti,no-idle-on-init: When present, the module should not be idled at init
 - ti,no-idle: When present, the module is never allowed to idle.
+- ti,deassert-hard-reset: list of hwmod and hardware reset line name pairs
+  (ascii strings) to be deasserted upon device instantiation.
 
 Example:
 
diff -Naur --no-dereference a/Documentation/devicetree/bindings/arm/ti/k3.txt b/Documentation/devicetree/bindings/arm/ti/k3.txt
--- a/Documentation/devicetree/bindings/arm/ti/k3.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/arm/ti/k3.txt	2020-06-19 21:00:31.400227715 -0400
@@ -13,6 +13,9 @@
 - AM654
   compatible = "ti,am654";
 
+- J721E
+  compatible = "ti,j721e";
+
 Boards
 ------
 
diff -Naur --no-dereference a/Documentation/devicetree/bindings/clock/ti,sci-clk.txt b/Documentation/devicetree/bindings/clock/ti,sci-clk.txt
--- a/Documentation/devicetree/bindings/clock/ti,sci-clk.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/clock/ti,sci-clk.txt	2020-06-19 21:00:31.400227715 -0400
@@ -18,6 +18,13 @@
   and clocks IDs for 66AK2G SoC are documented at
   http://processors.wiki.ti.com/index.php/TISCI#66AK2G02_Data
 
+Optional properties:
+-------------------
+- ti,scan-clocks-from-dt: Scan clock tree info from DT. By default,
+  clocks are queried from firmware, which can be rather slow operation,
+  especially if there is a really large number of clocks available out
+  of which only a handful are ever used by kernel.
+
 Examples:
 --------
 
diff -Naur --no-dereference a/Documentation/devicetree/bindings/clock/ti,syscon-gate-clock.txt b/Documentation/devicetree/bindings/clock/ti,syscon-gate-clock.txt
--- a/Documentation/devicetree/bindings/clock/ti,syscon-gate-clock.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/clock/ti,syscon-gate-clock.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,31 @@
+TI syscon gate clock driver
+
+The gate clock node must be provided inside a system controller node.
+
+Required:
+- comaptible: Must be "ti,am654-ehrpwm-tbclk" for EHRPWM tbclk on AM654 SoC
+- #clock-cells: must be <1>. It represents the index of clk instance
+  within the syscon regmap. Index starts from 0.
+
+Example:
+	ctrlmmr_epwm_ctrl: syscon@104140 {
+		compatible = "syscon", "simple-mfd";
+		reg = <0x0 0x104140 0x0 0x18>;
+		ranges = <0x0 0x0 0x104140 0x18>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		ehrpwm_tbclk: clk@0 {
+			compatible = "ti,am654-ehrpwm-tbclk";
+			#clock-cells = <1>;
+		};
+	};
+
+	ehrpwm0: pwm@3000000 {
+		compatible = "ti,am654-ehrpwm", "ti,am3352-ehrpwm";
+		#pwm-cells = <3>;
+		reg = <0x0 0x3000000 0x0 0x100>;
+		power-domains = <&k3_pds 40>;
+		clocks = <&ehrpwm_tbclk 0>;
+		clock-names = "tbclk";
+	};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/crypto/sa2ul.txt b/Documentation/devicetree/bindings/crypto/sa2ul.txt
--- a/Documentation/devicetree/bindings/crypto/sa2ul.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/crypto/sa2ul.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,47 @@
+K3 SoC SA2UL crypto module
+
+Required properties:
+
+- compatible : Should be:
+  - "ti,sa2ul-crypto"
+- reg : Offset and length of the register set for the module
+
+- dmas: DMA specifiers for tx and rx dma. sa2ul needs one tx channel
+	and 2 rx channels. First rx channel for < 256 bytes and
+	the other one for >=256 bytes. See the DMA client binding,
+        Documentation/devicetree/bindings/dma/dma.txt
+- dma-names: DMA request names has to have one tx and 2 rx names
+	corresponding to dmas abive.
+- ti,psil-config* - UDMA PSIL native Peripheral using packet mode.
+	SA2UL must have EPIB(Extended protocal information block)
+	and PSDATA(protocol specific data) properties.
+
+Example AM654 SA2UL:
+crypto: crypto@4E00000 {
+	compatible = "ti,sa2ul-crypto";
+	reg = <0x0 0x4E00000 0x0 0x1200>;
+	ti,psil-base = <0x4000>;
+
+	dmas = <&main_udmap &crypto 0 UDMA_DIR_TX>,
+		<&main_udmap &crypto 0 UDMA_DIR_RX>,
+		<&main_udmap &crypto 1 UDMA_DIR_RX>;
+	dma-names = "tx", "rx1", "rx2";
+
+	ti,psil-config0 {
+		linux,udma-mode = <UDMA_PKT_MODE>;
+		ti,needs-epib;
+		ti,psd-size = <64>;
+	};
+
+	ti,psil-config1 {
+		linux,udma-mode = <UDMA_PKT_MODE>;
+		ti,needs-epib;
+		ti,psd-size = <64>;
+	};
+
+	ti,psil-config2 {
+		linux,udma-mode = <UDMA_PKT_MODE>;
+		ti,needs-epib;
+		ti,psd-size = <64>;
+	};
+};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/display/bridge/cdns,mhdp.txt b/Documentation/devicetree/bindings/display/bridge/cdns,mhdp.txt
--- a/Documentation/devicetree/bindings/display/bridge/cdns,mhdp.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/display/bridge/cdns,mhdp.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,48 @@
+Cadence MHDP bridge
+==========================
+
+The Cadence MHDP bridge is a DPI to DP bridge.
+
+Required properties:
+- compatible: should be "cdns,mhdp8546",
+  Use "ti,j721e-mhdp8546" for TI J7 SoCs.
+- reg: physical base address and length of the controller's registers,
+- clocks: DP bridge clock, it's used by the IP to know how to translate
+	a number of clock cycles into a time (which is used to comply
+	with DP standard timings and delays),
+- phys: see the Documentation/devicetree/bindings/phy/phy-cadence-torrent.txt
+- phy-names: must be "dpphy"
+
+Required subnodes:
+- ports: Ports as described in Documentation/devictree/bindings/graph.txt
+	Port 0 - input port representing the DP bridge input
+	Port 1 - output port representing the DP bridge output
+
+Example:
+
+	mhdp: dp-bridge@f0fb000000 {
+		compatible = "cdns,mhdp8546";
+		reg = <0xf0 0xfb000000 0x0 0x1000000>;
+		clocks = <&mhdp_clock>;
+		phys = <&dp_phy>;
+		phy-names = "dpphy";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				dp_bridge_input: endpoint {
+					remote-endpoint = <&xxx_dpi_output>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+				dp_bridge_output: endpoint {
+					remote-endpoint = <&xxx_dp_connector_input>;
+				};
+			};
+		};
+	};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/display/bridge/sii902x.txt b/Documentation/devicetree/bindings/display/bridge/sii902x.txt
--- a/Documentation/devicetree/bindings/display/bridge/sii902x.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/display/bridge/sii902x.txt	2020-06-19 21:00:31.400227715 -0400
@@ -8,6 +8,21 @@
 	- interrupts: describe the interrupt line used to inform the host 
 	  about hotplug events.
 	- reset-gpios: OF device-tree gpio specification for RST_N pin.
+	- i2s-fifo-routing: Array of exactly 4 integers indicating i2s
+	  pins for audio fifo routing. First integer defines routing to
+	  fifo 0 and second to fifo 1, etc. Integers can be filled with
+	  definitions from: include/dt-bindings/sound/sii902x-audio.h
+	  The available definitions are:
+	  - ENABLE_BIT:	enable this audio fifo
+	  - CONNECT_SD#: route audio input from SD0, SD1, SD2, or SD3 i2s
+			 data input pin
+	  - LEFT_RIGHT_SWAP_BIT: swap i2s input channels for this fifo
+	  I2S HDMI audio is configured only if this property is found.
+	- clocks: describes SII902x MCLK input. MCLK is used to produce
+	  HDMI audio CTS values. This property is required if
+	  "i2s-fifo-routing"-property is present. This property follows
+	  Documentation/devicetree/bindings/clock/clock-bindings.txt
+	  consumer binding.
 
 Optional subnodes:
 	- video input: this subnode can contain a video input port node
@@ -21,6 +36,15 @@
 		compatible = "sil,sii9022";
 		reg = <0x39>;
 		reset-gpios = <&pioA 1 0>;
+
+		i2s-fifo-routing = <
+			(ENABLE_BIT|CONNECT_SD0)
+			0
+			0
+			0
+		>;
+		clocks = <&mclk>;
+
 		ports {
 			#address-cells = <1>;
 			#size-cells = <0>;
diff -Naur --no-dereference a/Documentation/devicetree/bindings/display/bridge/ti,tfp410.txt b/Documentation/devicetree/bindings/display/bridge/ti,tfp410.txt
--- a/Documentation/devicetree/bindings/display/bridge/ti,tfp410.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/display/bridge/ti,tfp410.txt	2020-06-19 21:00:31.400227715 -0400
@@ -6,15 +6,32 @@
 
 Optional properties:
 - powerdown-gpios: power-down gpio
-- reg: I2C address. If and only if present the device node
-       should be placed into the i2c controller node where the
-       tfp410 i2c is connected to.
+- reg: I2C address. If and only if present the device node should be placed
+  into the I2C controller node where the TFP410 I2C is connected to.
+- ti,deskew: data de-skew in 350ps increments, from -4 to +3, as configured
+  through th DK[3:1] pins. This property shall be present only if the TFP410
+  is not connected through I2C.
 
 Required nodes:
-- Video port 0 for DPI input [1].
-- Video port 1 for DVI output [1].
 
-[1]: Documentation/devicetree/bindings/media/video-interfaces.txt
+This device has two video ports. Their connections are modeled using the OF
+graph bindings specified in [1]. Each port node shall have a single endpoint.
+
+- Port 0 is the DPI input port. Its endpoint subnode shall contain a
+  pclk-sample and bus-width property and a remote-endpoint property as specified
+  in [1].
+  - If pclk-sample is not defined, pclk-sample = 0 should be assumed for
+    backward compatibility.
+  - If bus-width is not defined then bus-width = 24 should be assumed for
+    backward compatibility.
+    bus-width = 24: 24 data lines are connected and single-edge mode
+    bus-width = 12: 12 data lines are connected and dual-edge mode
+
+- Port 1 is the DVI output port. Its endpoint subnode shall contain a
+  remote-endpoint property is specified in [1].
+
+[1] Documentation/devicetree/bindings/media/video-interfaces.txt
+
 
 Example
 -------
@@ -22,6 +39,7 @@
 tfp410: encoder@0 {
 	compatible = "ti,tfp410";
 	powerdown-gpios = <&twl_gpio 2 GPIO_ACTIVE_LOW>;
+	ti,deskew = <4>;
 
 	ports {
 		#address-cells = <1>;
@@ -31,6 +49,8 @@
 			reg = <0>;
 
 			tfp410_in: endpoint@0 {
+				pclk-sample = <1>;
+				bus-width = <24>;
 				remote-endpoint = <&dpi_out>;
 			};
 		};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/display/bridge/toshiba,tc358767.txt b/Documentation/devicetree/bindings/display/bridge/toshiba,tc358767.txt
--- a/Documentation/devicetree/bindings/display/bridge/toshiba,tc358767.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/display/bridge/toshiba,tc358767.txt	2020-06-19 21:00:31.400227715 -0400
@@ -12,6 +12,7 @@
                    (active high shutdown input)
  - reset-gpios: OF device-tree gpio specification for RSTX pin
                 (active low system reset)
+ - toshiba,hpd-pin: TC358767 GPIO pin number to which HPD is connected to (0 or 1)
  - ports: the ports node can contain video interface port nodes to connect
    to a DPI/DSI source and to an eDP/DP sink according to [1][2]:
     - port@0: DSI input port
diff -Naur --no-dereference a/Documentation/devicetree/bindings/display/panel/osddisplays,osd070t1718-19ts.txt b/Documentation/devicetree/bindings/display/panel/osddisplays,osd070t1718-19ts.txt
--- a/Documentation/devicetree/bindings/display/panel/osddisplays,osd070t1718-19ts.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/display/panel/osddisplays,osd070t1718-19ts.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,12 @@
+OSD Displays OSD070T1718-19TS 7" WVGA TFT LCD panel
+
+Required properties:
+- compatible: shall be "osddisplays,osd070t1718-19ts"
+- power-supply: see simple-panel.txt
+
+Optional properties:
+- backlight: see simple-panel.txt
+
+This binding is compatible with the simple-panel binding, which is specified
+in simple-panel.txt in this directory. No other simple-panel properties than
+the ones specified herein are valid.
diff -Naur --no-dereference a/Documentation/devicetree/bindings/display/panel/rockteck,rk101ii01d-ct.txt b/Documentation/devicetree/bindings/display/panel/rockteck,rk101ii01d-ct.txt
--- a/Documentation/devicetree/bindings/display/panel/rockteck,rk101ii01d-ct.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/display/panel/rockteck,rk101ii01d-ct.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,8 @@
+Rockteck 10.1" TFT 1280x800 Pixels
+- with LVDS interface, LED Backlight and capacitive touch panel
+
+Required properties:
+- compatible: should be "rockteck,rk101ii01d-ct"
+
+This binding is compatible with the simple-panel binding, which is specified
+in simple-panel.txt in this directory.
diff -Naur --no-dereference a/Documentation/devicetree/bindings/display/ti/ti,am6-dss.txt b/Documentation/devicetree/bindings/display/ti/ti,am6-dss.txt
--- a/Documentation/devicetree/bindings/display/ti/ti,am6-dss.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/display/ti/ti,am6-dss.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,16 @@
+Texas Instruments AM65x Display Subsystem
+==========================================
+
+Required properties:
+- compatible: "ti,am65x-dss", "ti,am6-dss"
+- reg: address and length of the register spaces for DSS submodules
+- reg-names: "common", "vidl1", "vid", "ovr1", "ovr2", "vp1", "vp2"
+- clocks: phandle to fclk, vp1, and vp2 clocks
+- clock-names: "fck", "vp1", "vp2"
+- interrupts: phandle to the DISPC interrupt
+- syscon: phandle to syscon device handling OLDI_PWRDN_TX (partition 1 of
+  AM654 CTRL MMR0)
+
+The DSS outputs are described using the device graphs as documented in
+Documentation/devicetree/bindings/graph.txt. AM6 DSS has a DPI output as port 0
+and an OLDI output as port 1.
diff -Naur --no-dereference a/Documentation/devicetree/bindings/display/ti/ti,dra7evm-tpd12s015.txt b/Documentation/devicetree/bindings/display/ti/ti,dra7evm-tpd12s015.txt
--- a/Documentation/devicetree/bindings/display/ti/ti,dra7evm-tpd12s015.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/display/ti/ti,dra7evm-tpd12s015.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,49 @@
+TPD12S015 HDMI level shifter and ESD protection chip customized for DRA7 EVM
+============================================================================
+
+Required properties:
+- compatible: "ti,dra7evm-tpd12s015"
+
+Optional properties:
+- gpios: CT CP HPD, LS OE and HPD gpios
+- disable-hpd: Disables hot plug detect, required for old DRA7
+  EVMs(Rev D and older)
+
+Required nodes:
+- Video port 0 for HDMI input
+- Video port 1 for HDMI output
+
+Example
+-------
+
+tpd12s015: encoder@1 {
+	compatible = "ti,dra7evm-tpd12s015";
+
+	gpios = <&gpio2 28 GPIO_ACTIVE_HIGH>,	/* 60, CT CP HPD */
+		<&gpio2 9 GPIO_ACTIVE_HIGH>,	/* 41, LS OE */
+		<&gpio2 31 GPIO_ACTIVE_HIGH>	/* 63, HPD */
+		<&mcasp8 2 GPIO_ACTIVE_HIGH>;	/* mcasp8.axr2, i2c2/ddc switch */
+
+	disable-hpd;
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+
+			tpd12s015_in: endpoint@0 {
+				remote-endpoint = <&hdmi_out>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+
+			tpd12s015_out: endpoint@0 {
+				remote-endpoint = <&hdmi_connector_in>;
+			};
+		};
+	};
+};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/display/ti/ti,j7-dss.txt b/Documentation/devicetree/bindings/display/ti/ti,j7-dss.txt
--- a/Documentation/devicetree/bindings/display/ti/ti,j7-dss.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/display/ti/ti,j7-dss.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,119 @@
+Texas Instruments J721E Display Subsystem
+==========================================
+
+Required properties:
+- compatible: "ti,j721e-dss"
+- reg: address and length of the register spaces for DSS submodules
+- reg-names: "common_m", "common_s0", "common_s1", "common_s2",
+	     "vidl1", "vid", "ovr1", "ovr2", "vp1", "vp2"
+- clocks: phandle to fclk, vp1, vp2, vp3, and vp4 clocks
+- clock-names: "fck", "vp1", "vp2", "vp3", "vp4"
+- interrupts: phandle to DISPC interrupts
+- interrupt-names: "common_m", "common_s0", "common_s1", and "common_s2"
+
+Optional properties:
+- dss-planes: partitioning information for pipes, if display sharing is required
+- dss-vps: partitioning information for video-ports, if display sharing is required
+- dss-commons: partitioning information for common areas, if display sharing is required
+- dss-remote: remote name that is expected to be advertised by the display sharing firmware
+
+The DSS outputs are described using the device graphs as documented in
+Documentation/devicetree/bindings/graph.txt.
+
+Display Sharing on Texas Instruments J721E
+=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
+
+Partitioning the pipes
+-----------------------
+
+In J721E, multiple video-ports, pipes, and common regions can be driven by separate
+compute entities, and therefore should not be accessed by tidss driver. To ensure this,
+optional properties like dss-planes, dss-vps and dss-commons should be used as required
+by the setup.
+
+Pipe partitioning is described by the 'dss-planes' node. Each pipe is repesented by a 'plane'
+node inside the 'dss-planes' node.
+
+Each 'plane' node must have a 'reg' property to identify the pipe, and a 'managed' property to
+indicate whether the pipe is managed by the driver. the managed property must be 0 if the
+pipe is not to be accessed by the driver.
+
+Since the children of the 'dss-planes' node uses 'reg' properties to identify pipes, it must have '#address-cells'
+and '#size-cells' properties, set to 1 and 0 respectively.
+
+dss-planes {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	/* vid1, managed by tidss */
+	plane@0 {
+		reg = <0>;
+		managed = <1>;
+	};
+
+	/* vidl1, not managed by tidss */
+	plane@1 {
+		reg = <1>;
+		managed = <0>;
+	};
+
+	...
+};
+
+Partitioning the video-ports
+-----------------------------
+
+Video port partitioning is described by the 'dss-vps' node. Each video-port is repesented by a 'vp'
+node inside the 'dss-vps' node.
+
+Each 'vp' node must have a 'reg' property to identify the video-port, and a 'managed' property to
+indicate whether the video-port is managed by the driver. the managed property must be 0 if the
+video-port is not to be accessed by the driver.
+
+Since the children of the 'dss-vps' node uses 'reg' properties to identify pipes, it must have '#address-cells'
+and '#size-cells' properties, set to 1 and 0 respectively.
+
+dss-vps {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	/* VP1, managed by tidss */
+	vp@0 {
+		reg = <0>;
+		managed = <1>;
+	};
+
+	/* VP2, not managed by tidss */
+	vp@1 {
+		reg = <1>;
+		managed = <0>;
+	};
+
+	...
+};
+
+Partitioning the common regions
+--------------------------------
+
+Tidss can use at most two common regions
+- mandatory interrupt region
+- optional config region
+
+The common region partitioning is described by the 'dss-commons' node. It must have a child node called
+'interrupt-common' and it may contain an optional node called 'config-common'
+
+'interrupt-common' node must contain a 'reg' property to indicate which common region to be used for interrupt
+handling
+
+'config-common' node must contain a 'reg' property to indicate which common region to be used for device
+configuration. It may contain a 'status' property which can be set to "disabled" when the configuration is
+not to be done.
+
+Since the children of the  'dss-commons' node uses 'reg' properties to identify common regions, it must have
+'#address-cells' and '#size-cells' properties, set to 1 and 0 respectively.
+
+Display sharing
+----------------
+
+To enable display sharing with another compute entity, there must be a 'dss-remote' node. The node must have a
+string property called 'remote-name' which indicates the remote-device name to look for and attach
diff -Naur --no-dereference a/Documentation/devicetree/bindings/display/ti/ti,k2g-dss.txt b/Documentation/devicetree/bindings/display/ti/ti,k2g-dss.txt
--- a/Documentation/devicetree/bindings/display/ti/ti,k2g-dss.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/display/ti/ti,k2g-dss.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,15 @@
+Texas Instruments K2G Display Subsystem
+=======================================
+
+Required properties:
+- compatible: "ti,k2g-dss"
+- reg: address and length of the register spaces for DSS submodules
+- reg-names: "cfg", "common", "vid1", "ovr1", "vp1"
+- clocks: phandle to fclk and vp1 clocks
+- clock-names: "fck", "vp1"
+- interrupts: phandle to the DISPC interrupt
+
+The DSS outputs are described using the device graphs as documented in
+Documentation/devicetree/bindings/graph.txt. K2G DSS has a single DPI output as
+port 0.
+
diff -Naur --no-dereference a/Documentation/devicetree/bindings/dma/ti/k3-udma.txt b/Documentation/devicetree/bindings/dma/ti/k3-udma.txt
--- a/Documentation/devicetree/bindings/dma/ti/k3-udma.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/dma/ti/k3-udma.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,145 @@
+* Texas Instruments K3 NAVSS Unified DMA – Peripheral Root Complex (UDMA-P)
+
+The UDMA-P is intended to perform similar (but significantly upgraded) functions
+as the packet-oriented DMA used on previous SoC devices. The UDMA-P module
+supports the transmission and reception of various packet types. The UDMA-P is
+architected to facilitate the segmentation and reassembly of SoC DMA data
+structure compliant packets to/from smaller data blocks that are natively
+compatible with the specific requirements of each connected peripheral. Multiple
+Tx and Rx channels are provided within the DMA which allow multiple segmentation
+or reassembly operations to be ongoing. The DMA controller maintains state
+information for each of the channels which allows packet segmentation and
+reassembly operations to be time division multiplexed between channels in order
+to share the underlying DMA hardware. An external DMA scheduler is used to
+control the ordering and rate at which this multiplexing occurs for Transmit
+operations. The ordering and rate of Receive operations is indirectly controlled
+by the order in which blocks are pushed into the DMA on the Rx PSI-L interface.
+
+The UDMA-P also supports acting as both a UTC and UDMA-C for its internal
+channels. Channels in the UDMA-P can be configured to be either Packet-Based or
+Third-Party channels on a channel by channel basis.
+
+Required properties:
+--------------------
+- compatible:		Should be
+			"ti,am654-navss-main-udmap" for am654 main NAVSS UDMAP
+			"ti,am654-navss-mcu-udmap" for am654 mcu NAVSS UDMAP
+			"ti,j721e-navss-main-udmap" for j721e main NAVSS UDMAP
+			"ti,j721e-navss-mcu-udmap" for j721e mcu NAVSS UDMAP
+- #dma-cells:		Should be set to <3>.
+			- The first parameter is a phandle to the remote PSI-L
+			  endpoint
+			- The second parameter is the thread offset within the
+			  remote thread ID range
+			- The third parameter is the channel direction.
+- reg:			Memory map of UDMAP
+- reg-names:		"gcfg", "rchanrt", "tchanrt"
+- interrupt-parent:	phandle for "ti,sci-irq" irqchip
+- ti,ringacc:		phandle for the ring accelerator node
+- ti,psil-base:		PSI-L thread ID base of the UDMAP channels
+- ti,sci:		phandle on TI-SCI compatible System controller node
+- ti,sci-dev-id:	TI-SCI device id
+- ti,sci-rm-range-tchan: UDMA tchan resource list in pairs of type and subtype
+- ti,sci-rm-range-rchan: UDMA rchan resource list in pairs of type and subtype
+- ti,sci-rm-range-rflow: UDMA rflow resource list in pairs of type and subtype
+
+For PSI-L thread management the parent NAVSS node must have:
+- ti,sci:		phandle on TI-SCI compatible System controller node
+- ti,sci-dev-id:	TI-SCI device id of the NAVSS instance
+
+Remote PSI-L endpoint
+
+Required properties:
+--------------------
+- ti,psil-base:		PSI-L thread ID base of the endpoint
+
+Within the PSI-L endpoint node thread configuration subnodes must present with:
+ti,psil-configX naming convention, where X is the thread ID offset.
+
+Configuration node Required properties:
+--------------------
+- linux,udma-mode:	Channel mode, can be:
+			- UDMA_PKT_MODE: for Packet mode channels (peripherals)
+			- UDMA_TR_MODE: for Third-Party mode
+
+Configuration node Optional properties:
+--------------------
+- statictr-type:	In case the remote endpoint requires StaticTR
+			configuration:
+			- PSIL_STATIC_TR_XY: XY type of StaticTR
+			- PSIL_STATIC_TR_MCAN: MCAN type of StaticTR
+- enable-acc32:		Force 32 bit access on peripheral port. Only valid for
+			XY type StaticTR, not supported on am654.
+			Must be enabled for threads servicing McASP with AFIFO
+			bypass mode.
+- enable-burst:		Enable burst access on peripheral port. Only valid for
+			XY type StaticTR, not supported on am654.
+- ti,channel-tpl:	Channel Throughput level:
+			0 / or not present - normal channel
+			1 - High Throughput channel
+			2 - Ultra High Throughput channel (j721e only)
+- ti,needs-epib:	If the endpoint require EPIB to be present in the
+			descriptor.
+- ti,psd-size:		Size of the Protocol Specific Data section of the
+			descriptor.
+- ti,notdpkt:		The Teardown Completion Message on the thread must be
+			suppressed.
+
+Example:
+
+main_navss: main_navss {
+	compatible = "simple-bus";
+	#address-cells = <2>;
+	#size-cells = <2>;
+	dma-coherent;
+	dma-ranges;
+	ranges;
+
+	ti,sci = <&dmsc>;
+	ti,sci-dev-id = <118>;
+
+	main_udmap: udmap@31150000 {
+		compatible = "ti,am654-navss-main-udmap";
+		reg =	<0x0 0x31150000 0x0 0x100>,
+			<0x0 0x34000000 0x0 0x100000>,
+			<0x0 0x35000000 0x0 0x100000>;
+		reg-names = "gcfg", "rchanrt", "tchanrt";
+		#dma-cells = <3>;
+
+		ti,ringacc = <&ringacc>;
+		ti,psil-base = <0x1000>;
+
+		interrupt-parent = <&main_udmass_inta>;
+
+		ti,sci = <&dmsc>;
+		ti,sci-dev-id = <188>;
+
+		ti,sci-rm-range-tchan = <0x6 0x1>, /* TX_HCHAN */
+					<0x6 0x2>; /* TX_CHAN */
+		ti,sci-rm-range-rchan = <0x6 0x4>, /* RX_HCHAN */
+					<0x6 0x5>; /* RX_CHAN */
+		ti,sci-rm-range-rflow = <0x6 0x6>; /* GP RFLOW */
+	};
+};
+
+pdma0: pdma@2a41000 {
+	compatible = "ti,am654-pdma";
+	reg = <0x0 0x02A41000 0x0 0x400>;
+	reg-names = "eccaggr_cfg";
+
+	ti,psil-base = <0x4400>;
+
+	/* ti,psil-config0-2 */
+	UDMA_PDMA_TR_XY(0);
+	UDMA_PDMA_TR_XY(1);
+	UDMA_PDMA_TR_XY(2);
+};
+
+mcasp0: mcasp@02B00000 {
+...
+	/* tx: pdma0-0, rx: pdma0-0 */
+	dmas = <&main_udmap &pdma0 0 UDMA_DIR_TX>,
+	       <&main_udmap &pdma0 0 UDMA_DIR_RX>;
+	dma-names = "tx", "rx";
+...
+};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/gpio/gpio-davinci.txt b/Documentation/devicetree/bindings/gpio/gpio-davinci.txt
--- a/Documentation/devicetree/bindings/gpio/gpio-davinci.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/gpio/gpio-davinci.txt	2020-06-19 21:00:31.400227715 -0400
@@ -5,6 +5,7 @@
 			"ti,keystone-gpio": for Keystone 2 66AK2H/K, 66AK2L,
 						66AK2E SoCs
 			"ti,k2g-gpio", "ti,keystone-gpio": for 66AK2G
+			"ti,j721e-gpio", "ti,keystone-gpio": for J721E SoCs
 
 - reg: Physical base address of the controller and the size of memory mapped
        registers.
diff -Naur --no-dereference a/Documentation/devicetree/bindings/gpu/ti-bb2d.txt b/Documentation/devicetree/bindings/gpu/ti-bb2d.txt
--- a/Documentation/devicetree/bindings/gpu/ti-bb2d.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/gpu/ti-bb2d.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,23 @@
+* Texas Instruments BB2D blitter module
+
+This binding describes the 2D BitBlit (BB2D) graphics accelerator
+subsystem based on the GC320 core from Vivante Corporation available
+in Texas Instruments SoCs.
+
+Required properties:
+  - compatible: should be "ti,dra7-bb2d" for TI DRA7xx/AM57xx SoCs
+  - reg : base address and length of BB2D IP registers
+  - interrupts : BB2D interrupt line number
+  - ti,hwmods : name of the hwmod associated with BB2D module
+  - clocks : handle to BB2D functional clock
+  - clock-names : fck
+
+Example:
+        bb2d: bb2d@59000000 {
+            compatible = "ti,dra7-bb2d";
+            reg = <0x59000000 0x0700>;
+            interrupts = <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>;
+            ti,hwmods = "bb2d";
+            clocks = <&dss_clkctrl DRA7_BB2D_CLKCTRL 0>;
+            clock-names = "fck";
+        };
diff -Naur --no-dereference a/Documentation/devicetree/bindings/gpu/ti-pvr.txt b/Documentation/devicetree/bindings/gpu/ti-pvr.txt
--- a/Documentation/devicetree/bindings/gpu/ti-pvr.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/gpu/ti-pvr.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,74 @@
+Texas Instruments PowerVR 3D GPU
+================================
+
+PowerVR is a 3D Graphics Processing Unit from Imagination Technologies. Texas
+Instruments SoCs have integrated different generations of PowerVR 3D GPUs. This
+binding describes the GPU's integrated in Texas Instruments SoCs.
+
+Required properties:
+--------------------
+  - compatible:
+      accepted values:
+	(a) "ti,am3352-sgx530", "img,sgx530" for TI AM33x
+	(b) "ti,am4376-sgx530", "img,sgx530" for TI AM43x
+	(c) "ti,dra7-sgx544", "img,sgx544" for TI DRA7xx / AM57x
+	(d) "ti,am654-sgx544", "img,sgx544" for TI AM654
+	(e) "ti,j721e-pvr", "img,pvr-ge8430" for TI J721E SoC with PowerVR GE8430 GPU
+
+  - reg: base address and length of registers
+  - interrupts: phandle to GPU interrupts
+  - clocks: from SoC clock bindings
+
+Optional properties:
+--------------------
+  - reg-names: names of registers listed in reg property in same order
+  - clock-names: names of clocks listed in clocks property in same order
+
+SoC-specific properties:
+------------------------
+The following properties are mandatory for OMAP architecture based SoC's
+  - ti,hwmods: name of hwmod associated with AM33x, AM43x, DRA7xx devices
+
+The following properties are mandatory for K3 AM65x and J721E SoCs only:
+  - power-domains: power domain parameters from SoC device description
+
+The following properties are optional for K3 AM65x and J721E SoCs only:
+  - power-domain-names: names of domains listed in power-domains property in
+    same order
+
+Examples:
+--------
+
+1. /* DRA7 */
+	sgx@56000000 {
+		compatible = "ti,dra7-sgx544", "img,sgx544";
+		reg = <0x56000000 0x10000>;
+		reg-names = "gpu_ocp_base";
+		interrupts = <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>;
+		ti,hwmods = "gpu";
+		clocks = <&l3_iclk_div>, <&gpu_core_gclk_mux>,
+			 <&gpu_hyd_gclk_mux>;
+		clock-names = "iclk", "fclk1", "fclk2";
+	};
+
+2. /* AM65x */
+	gpu@7000000 {
+		compatible = "ti,am654-sgx544", "img,sgx544";
+		reg = <0x0 0x7000000 0x0 0x10000>;
+		interrupts = <GIC_SPI 162 IRQ_TYPE_LEVEL_HIGH>;
+		power-domains = <&k3_pds 65>;
+		clocks = <&k3_clks 65 0>, <&k3_clks 65 1>, <&k3_clks 65 2>, <&k3_clks 65 3>;
+		clock-names = "mem_clk", "hyd_clk", "sgx_clk", "sys_clk";
+	};
+
+3. /* J721e */
+	gpu: gpu@4e20000000 {
+		compatible = "ti,j721e-pvr", "img,pvr-ge8430";
+		reg = <0x4e 0x20000000 0x00 0x80000>;
+		reg-names = "gpu_regs";
+		interrupts = <GIC_SPI 24 IRQ_TYPE_LEVEL_HIGH>;
+		power-domains = <&k3_pds 125 TI_SCI_PD_EXCLUSIVE>, <&k3_pds 126 TI_SCI_PD_EXCLUSIVE>;
+		power-domain-names = "gpu_0", "gpucore_0";
+		clocks = <&k3_clks 125 0>;
+		clock-names = "ctrl";
+	};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/hwlock/omap-hwspinlock.txt b/Documentation/devicetree/bindings/hwlock/omap-hwspinlock.txt
--- a/Documentation/devicetree/bindings/hwlock/omap-hwspinlock.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/hwlock/omap-hwspinlock.txt	2020-06-19 21:00:31.400227715 -0400
@@ -1,12 +1,16 @@
-OMAP4+ HwSpinlock Driver
-========================
+TI HwSpinlock for OMAP and K3 based SoCs
+=========================================
 
 Required properties:
-- compatible:		Should be "ti,omap4-hwspinlock" for
-			    OMAP44xx, OMAP54xx, AM33xx, AM43xx, DRA7xx SoCs
+- compatible:		Should be one of the following,
+			  "ti,omap4-hwspinlock" for
+				OMAP44xx, OMAP54xx, AM33xx, AM43xx, DRA7xx SoCs
+			  "ti,am654-hwspinlock" for
+				K3 AM65x and J721E SoCs
 - reg:			Contains the hwspinlock module register address space
 			(base address and length)
 - ti,hwmods:		Name of the hwmod associated with the hwspinlock device
+			(for OMAP architecture based SoCs only)
 - #hwlock-cells:	Should be 1. The OMAP hwspinlock users will use a
 			0-indexed relative hwlock number as the argument
 			specifier value for requesting a specific hwspinlock
@@ -17,10 +21,21 @@
 
 Example:
 
-/* OMAP4 */
+1. OMAP4 SoCs
 hwspinlock: spinlock@4a0f6000 {
 	compatible = "ti,omap4-hwspinlock";
 	reg = <0x4a0f6000 0x1000>;
 	ti,hwmods = "spinlock";
 	#hwlock-cells = <1>;
 };
+
+2. AM65x SoCs and J721E SoCs
+&cbass_main {
+	cbass_main_navss: interconnect0 {
+		hwspinlock: spinlock@30e00000 {
+			compatible = "ti,am654-hwspinlock";
+			reg = <0x00 0x30e00000 0x00 0x1000>;
+			#hwlock-cells = <1>;
+		};
+	};
+};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/i2c/i2c-omap.txt b/Documentation/devicetree/bindings/i2c/i2c-omap.txt
--- a/Documentation/devicetree/bindings/i2c/i2c-omap.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/i2c/i2c-omap.txt	2020-06-19 21:00:31.400227715 -0400
@@ -1,8 +1,13 @@
 I2C for OMAP platforms
 
 Required properties :
-- compatible : Must be "ti,omap2420-i2c", "ti,omap2430-i2c", "ti,omap3-i2c"
-  or "ti,omap4-i2c"
+- compatible : Must be
+	"ti,omap2420-i2c" for OMAP2420 SoCs
+	"ti,omap2430-i2c" for OMAP2430 SoCs
+	"ti,omap3-i2c" for OMAP3 SoCs
+	"ti,omap4-i2c" for OMAP4+ SoCs
+	"ti,am654-i2c", "ti,omap4-i2c" for AM654 SoCs
+	"ti,j721e-i2c", "ti,omap4-i2c" for j721e SoCs
 - ti,hwmods : Must be "i2c<n>", n being the instance number (1-based)
 - #address-cells = <1>;
 - #size-cells = <0>;
diff -Naur --no-dereference a/Documentation/devicetree/bindings/input/touchscreen/ti-tsc-adc.txt b/Documentation/devicetree/bindings/input/touchscreen/ti-tsc-adc.txt
--- a/Documentation/devicetree/bindings/input/touchscreen/ti-tsc-adc.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/input/touchscreen/ti-tsc-adc.txt	2020-06-19 21:00:31.400227715 -0400
@@ -2,7 +2,13 @@
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Required properties:
+- mfd
+	compatible: Should be
+		"ti,am3359-tscadc" for AM335x/AM437x SoCs
+		"ti,am654-tscadc", "ti,am3359-tscadc" for AM654 SoCs
+		"ti,j721e-tscadc", "ti,am3359-tscadc" for J721E SoCs
 - child "tsc"
+	compatible: Should be "ti,am3359-tsc".
 	ti,wires: Wires refer to application modes i.e. 4/5/8 wire touchscreen
 		  support on the platform.
 	ti,x-plate-resistance: X plate resistance
@@ -25,6 +31,10 @@
 			AIN0 = 0, AIN1 = 1 and so on till AIN7 = 7.
 			XP  = 0, XN = 1, YP = 2, YN = 3.
 - child "adc"
+	compatible: Should be
+		    "ti,am3359-adc" for AM335x/AM437x SoCs
+		    "ti,am654-adc", "ti,am3359-adc" for AM654 SoCs
+		    "ti,j721e-adc", "ti,am3359-adc" for J721E SoCs
 	ti,adc-channels: List of analog inputs available for ADC.
 			 AIN0 = 0, AIN1 = 1 and so on till AIN7 = 7.
 
diff -Naur --no-dereference a/Documentation/devicetree/bindings/interrupt-controller/ti,pruss-intc.txt b/Documentation/devicetree/bindings/interrupt-controller/ti,pruss-intc.txt
--- a/Documentation/devicetree/bindings/interrupt-controller/ti,pruss-intc.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/interrupt-controller/ti,pruss-intc.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,63 @@
+PRU ICSS INTC on TI SoCs
+========================
+
+Each PRUSS has a single interrupt controller instance that is common to both
+the PRU cores. Each interrupt controller can detect 64 input events which are
+then mapped to 10 possible output interrupts through two levels of mapping. The
+input events can be triggered by either the PRUs and/or various other PRUSS
+internal and external peripherals. The first 2 output interrupts are fed
+exclusively to the internal PRU cores, with the remaining 8 connected to
+external interrupt controllers including the MPU.
+
+The K3 family of SoCs can handle 160 input events that can be mapped to 20
+different possible output interrupts. The additional output interrupts are
+connected to new sub-modules within the ICSSG instances.
+
+This interrupt-controller node should be defined as a child node of the
+corresponding PRUSS node. The node should be named "interrupt-controller".
+Please see the overall PRUSS bindings document for additional details
+including a complete example,
+    Documentation/devicetree/bindings/soc/ti/ti,pruss.txt
+
+Required Properties:
+--------------------
+- compatible           : should be one of,
+                             "ti,am3356-pruss-intc" for AM335x family of SoCs
+                             "ti,am4376-pruss-intc" for AM437x family of SoCs
+                             "ti,am5728-pruss-intc" for AM57xx family of SoCs
+                             "ti,k2g-pruss-intc" for 66AK2G family of SoCs
+                             "ti,am654-icssg-intc" for K3 AM65x family of SoCs
+                             "ti,j721e-icssg-intc" for K3 J721E family of SoCs
+- reg                  : base address and size for the PRUSS INTC sub-module
+- interrupt-controller : mark this node as an interrupt controller
+- #interrupt-cells     : should be 1. Client users shall use the PRU System
+                         event number (the interrupt source that the client
+                         is interested in) as the value of the interrupts
+                         property in their node
+
+Example:
+--------
+
+1.	/* AM33xx PRU-ICSS */
+	pruss_soc_bus: pruss-soc-bus@4a326004 {
+		compatible = "ti,am3356-pruss-soc-bus";
+		...
+
+		pruss: pruss@4a300000 {
+			compatible = "ti,am3356-pruss";
+			reg = <0x4a300000 0x80000>;
+			interrupts = <20 21 22 23 24 25 26 27>;
+			interrupt-names = "host2", "host3", "host4",
+					  "host5", "host6", "host7",
+					  "host8", "host9";
+			#address-cells = <1>;
+			...
+
+			pruss_intc: interrupt-controller@4a320000 {
+				compatible = "ti,am3356-pruss-intc";
+				reg = <0x4a320000 0x2000>;
+				interrupt-controller;
+				#interrupt-cells = <1>;
+			};
+		};
+	};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/interrupt-controller/ti,sci-inta.txt b/Documentation/devicetree/bindings/interrupt-controller/ti,sci-inta.txt
--- a/Documentation/devicetree/bindings/interrupt-controller/ti,sci-inta.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/interrupt-controller/ti,sci-inta.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,74 @@
+Texas Instruments K3 Interrupt Aggregator
+=========================================
+
+The Interrupt Aggregator (INTA) provides a centralized machine
+which handles the termination of system events to that they can
+be coherently processed by the host(s) in the system. A maximum
+of 64 events can be mapped to a single interrupt.
+
+
+                              Interrupt Aggregator
+                     +-----------------------------------------+
+                     |      Intmap            VINT             |
+                     | +--------------+  +------------+        |
+            m ------>| | vint  | bit  |  | 0 |.....|63| vint0  |
+               .     | +--------------+  +------------+        |       +------+
+               .     |         .               .               |       | HOST |
+Globalevents  ------>|         .               .               |------>| IRQ  |
+               .     |         .               .               |       | CTRL |
+               .     |         .               .               |       +------+
+            n ------>| +--------------+  +------------+        |
+                     | | vint  | bit  |  | 0 |.....|63| vintx  |
+                     | +--------------+  +------------+        |
+                     |                                         |
+                     +-----------------------------------------+
+
+Configuration of these Intmap registers that maps global events to vint is done
+by a system controller (like the Device Memory and Security Controller on K3
+AM654 SoC). Driver should request the system controller to get the range
+of global events and vints assigned to the requesting host. Management
+of these requested resources should be handled by driver and requests
+system controller to map specific global event to vint, bit pair.
+
+Communication between the host processor running an OS and the system
+controller happens through a protocol called TI System Control Interface
+(TISCI protocol). For more details refer:
+Documentation/devicetree/bindings/arm/keystone/ti,sci.txt
+
+TISCI Interrupt Aggregator Node:
+-------------------------------
+- compatible:		Must be "ti,sci-inta".
+- reg:			Should contain registers location and length.
+- interrupt-controller:	Identifies the node as an interrupt controller
+- #interrupt-cells:	Specifies the number of cells needed to encode an
+			interrupt source. The value should be 4.
+			First cell should contain the TISCI device ID of source
+			Second cell should contain the event source offset
+			within the device
+			Third cell specified the interrupt number(vint)
+			reaching Interrupt aggregator.
+			Fourth cell specifies the trigger type as defined
+			in interrupts.txt in this directory.
+- interrupt-parent:	phandle of irq parent for TISCI intr.
+- ti,sci:		Phandle to TI-SCI compatible System controller node.
+- ti,sci-dev-id:	TISCI device ID of the Interrupt Aggregator.
+- ti,sci-rm-range-vint:	TISCI subtype id representing the virtual interrupts
+			(vints) range within this IA, assigned to the
+			requesting host context.
+- ti,sci-rm-range-global-event:	TISCI subtype id representing the global
+			events range reaching this IA and are assigned
+			to the requesting host context.
+
+Example:
+--------
+main_udmass_inta: interrupt-controller@33d00000 {
+	compatible = "ti,sci-inta";
+	reg = <0x0 0x33d00000 0x0 0x100000>;
+	interrupt-controller;
+	interrupt-parent = <&main_navss_intr>;
+	#interrupt-cells = <4>;
+	ti,sci = <&dmsc>;
+	ti,sci-dev-id = <179>;
+	ti,sci-rm-range-vint = <0x0>;
+	ti,sci-rm-range-global-event = <0x1>;
+};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/interrupt-controller/ti,sci-intr.txt b/Documentation/devicetree/bindings/interrupt-controller/ti,sci-intr.txt
--- a/Documentation/devicetree/bindings/interrupt-controller/ti,sci-intr.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/interrupt-controller/ti,sci-intr.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,81 @@
+Texas Instruments K3 Interrupt Router
+=====================================
+
+The Interrupt Router (INTR) module provides a mechanism to mux M
+interrupt inputs to N interrupt outputs, where all M inputs are selectable
+to be driven per N output. There is one register per output (MUXCNTL_N) that
+controls the selection.
+
+
+                                 Interrupt Router
+                             +----------------------+
+                             |  Inputs     Outputs  |
+        +-------+            | +------+             |
+        | GPIO  |----------->| | irq0 |             |       Host IRQ
+        +-------+            | +------+             |      controller
+                             |    .        +-----+  |      +-------+
+        +-------+            |    .        |  0  |  |----->|  IRQ  |
+        | INTA  |----------->|    .        +-----+  |      +-------+
+        +-------+            |    .          .      |
+                             | +------+      .      |
+                             | | irqM |    +-----+  |
+                             | +------+    |  N  |  |
+                             |             +-----+  |
+                             +----------------------+
+
+Configuration of these MUXCNTL_N registers is done by a system controller
+(like the Device Memory and Security Controller on K3 AM654 SoC). System
+controller will keep track of the used and unused registers within the Router.
+Driver should request the system controller to get the range of GIC IRQs
+assigned to the requesting hosts. It is the drivers responsibility to keep
+track of Host IRQs.
+
+Communication between the host processor running an OS and the system
+controller happens through a protocol called TI System Control Interface
+(TISCI protocol). For more details refer:
+Documentation/devicetree/bindings/arm/keystone/ti,sci.txt
+
+TISCI Interrupt Router Node:
+----------------------------
+- compatible:		Must be "ti,sci-intr".
+- interrupt-controller:	Identifies the node as an interrupt controller
+- #interrupt-cells:	Specifies the number of cells needed to encode an
+			interrupt source. The value should be 3.
+			First cell should contain the TISCI device ID of source
+			Second cell should contain the interrupt source offset
+			within the device
+			Third cell specifies the trigger type as defined
+			in interrupts.txt in this directory. Only level
+			sensitive trigger types are supported.
+- interrupt-parent:	phandle of irq parent for TISCI intr.
+- ti,sci:		Phandle to TI-SCI compatible System controller node.
+- ti,sci-dst-id:	TISCI device ID of the destination IRQ controller.
+- ti,sci-rm-range-girq:	TISCI subtype id representing the host irqs assigned
+			to this interrupt router.
+
+Example:
+--------
+The following example demonstrates both interrupt router node and the consumer
+node(main gpio) on the AM654 SoC:
+
+main_intr: interrupt-controller@1 {
+	compatible = "ti,sci-intr";
+	interrupt-controller;
+	interrupt-parent = <&gic>;
+	#interrupt-cells = <3>;
+	ti,sci = <&dmsc>;
+	ti,sci-dst-id = <56>;
+	ti,sci-rm-range-girq = <0x1>;
+};
+
+main_gpio0:  main_gpio0@600000 {
+	...
+	interrupt-parent = <&main_intr>;
+	interrupts = <57 256 IRQ_TYPE_EDGE_RISING>,
+			<57 257 IRQ_TYPE_EDGE_RISING>,
+			<57 258 IRQ_TYPE_EDGE_RISING>,
+			<57 259 IRQ_TYPE_EDGE_RISING>,
+			<57 260 IRQ_TYPE_EDGE_RISING>,
+			<57 261 IRQ_TYPE_EDGE_RISING>;
+	...
+};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/mailbox/omap-mailbox.txt b/Documentation/devicetree/bindings/mailbox/omap-mailbox.txt
--- a/Documentation/devicetree/bindings/mailbox/omap-mailbox.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/mailbox/omap-mailbox.txt	2020-06-19 21:00:31.400227715 -0400
@@ -1,4 +1,4 @@
-OMAP2+ Mailbox Driver
+OMAP2+ and K3 Mailbox
 =====================
 
 The OMAP mailbox hardware facilitates communication between different processors
@@ -7,7 +7,7 @@
 communication is achieved through a set of registers for message storage and
 interrupt configuration registers.
 
-Each mailbox IP block has a certain number of h/w fifo queues and output
+Each mailbox IP block/cluster has a certain number of h/w fifo queues and output
 interrupt lines. An output interrupt line is routed to an interrupt controller
 within a processor subsystem, and there can be more than one line going to a
 specific processor's interrupt controller. The interrupt line connections are
@@ -23,12 +23,16 @@
 instance. DRA7xx has multiple instances with different number of h/w fifo queues
 and interrupt lines between different instances. The interrupt lines can also be
 routed to different processor sub-systems on DRA7xx as they are routed through
-the Crossbar, a kind of interrupt router/multiplexer.
+the Crossbar, a kind of interrupt router/multiplexer. The K3 AM65x and J721E
+SoCs has each of these instances form a cluster and combine multiple clusters
+into a single IP block present within the Main NavSS. The interrupt lines from
+all these clusters are multiplexed and routed to different processor subsystems
+over a limited number of common interrupt output lines of an Interrupt Router.
 
 Mailbox Device Node:
 ====================
-A Mailbox device node is used to represent a Mailbox IP instance within a SoC.
-The sub-mailboxes are represented as child nodes of this parent node.
+A Mailbox device node is used to represent a Mailbox IP instance/cluster within
+a SoC. The sub-mailboxes are represented as child nodes of this parent node.
 
 Required properties:
 --------------------
@@ -37,12 +41,12 @@
 			    "ti,omap3-mailbox" for OMAP3430, OMAP3630 SoCs
 			    "ti,omap4-mailbox" for OMAP44xx, OMAP54xx, AM33xx,
 						   AM43xx and DRA7xx SoCs
+			    "ti,am654-mailbox" for K3 AM65x and J721E SoCs
 - reg:			Contains the mailbox register address range (base
 			address and length)
 - interrupts:		Contains the interrupt information for the mailbox
 			device. The format is dependent on which interrupt
-			controller the OMAP device uses
-- ti,hwmods:		Name of the hwmod associated with the mailbox
+			controller the Mailbox device uses
 - #mbox-cells:		Common mailbox binding property to identify the number
 			of cells required for the mailbox specifier. Should be
 			1
@@ -50,6 +54,23 @@
 			device can interrupt
 - ti,mbox-num-fifos:	Number of h/w fifo queues within the mailbox IP block
 
+SoC-specific Required properties:
+---------------------------------
+The following are mandatory properties for the OMAP architecture based SoCs
+only:
+- ti,hwmods:		Name of the hwmod associated with the mailbox. This
+			should be defined in the mailbox node only if the node
+			is not defined as a child node of a corresponding sysc
+			interconnect node.
+
+The following are mandatory properties for the K3 AM65x and J721E SoCs only:
+- interrupt-parent:	Should contain a phandle to the TI-SCI interrupt
+			controller node that is used to dynamically program
+			the interrupt routes between the IP and the main GIC
+			controllers. See the following binding for additional
+			details,
+			Documentation/devicetree/bindings/interrupt-controller/ti,sci-intr.txt
+
 Child Nodes:
 ============
 A child node is used for representing the actual sub-mailbox device that is
@@ -98,7 +119,7 @@
 Example:
 --------
 
-/* OMAP4 */
+1. /* OMAP4 */
 mailbox: mailbox@4a0f4000 {
 	compatible = "ti,omap4-mailbox";
 	reg = <0x4a0f4000 0x200>;
@@ -123,7 +144,7 @@
 	...
 };
 
-/* AM33xx */
+2. /* AM33xx */
 mailbox: mailbox@480c8000 {
 	compatible = "ti,omap4-mailbox";
 	reg = <0x480C8000 0x200>;
@@ -137,3 +158,23 @@
 		ti,mbox-rx = <0 0 3>;
 	};
 };
+
+3. /* AM65x */
+&cbass_main {
+	cbass_main_navss: interconnect0 {
+		mailbox0_cluster0: mailbox@31f80000 {
+			compatible = "ti,am654-mailbox";
+			reg = <0x00 0x31f80000 0x00 0x200>;
+			#mbox-cells = <1>;
+			ti,mbox-num-users = <4>;
+			ti,mbox-num-fifos = <16>;
+			interrupt-parent = <&main_navss_intr>;
+			interrupts = <164 0 IRQ_TYPE_LEVEL_HIGH>;
+
+			mbox_mcu_r5fss0_core0: mbox-mcu-r5fss0-core0 {
+				ti,mbox-tx = <1 0 0>;
+				ti,mbox-rx = <0 0 0>;
+			};
+		};
+	};
+};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/media/i2c/mt9t11x.txt b/Documentation/devicetree/bindings/media/i2c/mt9t11x.txt
--- a/Documentation/devicetree/bindings/media/i2c/mt9t11x.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/media/i2c/mt9t11x.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,49 @@
+* Aptina 1/4-Inch QXGA CMOS Digital Image Sensor
+
+The Aptina MT9T111 is a 1/4-inch CMOS active pixel digital image sensor with
+an active array size of 2048H x 1536V. It is programmable through a simple
+two-wire serial interface.
+
+Required Properties:
+
+- compatible: value should be either one among the following
+	(a) "aptina,mt9t111" for MT9T111 color sensor
+	(b) "aptina,mt9t112" for MT9T112 color sensor
+
+- input-clock-freq: Source clock freqquency (Hz)
+- pixel-clock-freq: Target pixel clock freqquency (Hz)
+
+Optional Properties:
+
+- reset-gpios: Toggle when the module is enabled or disabled.
+- powerdown-gpios: Toggle when the module is enabled or disabled.
+- oscen-gpios: Enabled once at probe time.
+- bufen-gpios: Enabled once at probe time.
+- camen-gpios: Enabled once at probe time.
+
+For further reading on port node refer to
+Documentation/devicetree/bindings/media/video-interfaces.txt.
+
+Example:
+
+	mt9t111@3C {
+		compatible = "aptina,mt9t111";
+		reg = <0x3C>;
+
+		reset-gpios = <&gpio4 17 GPIO_ACTIVE_LOW>;
+		oscen-gpios = <&gpio5 10 GPIO_ACTIVE_HIGH>;
+		powerdown-gpios = <&gpio5 11 GPIO_ACTIVE_LOW>;
+		bufen-gpios = <&gpio5 12 GPIO_ACTIVE_LOW>;
+		camen-gpios = <&gpio6 11 GPIO_ACTIVE_LOW>;
+
+		port {
+			cam: endpoint {
+				remote-endpoint = <&vin3a>;
+				hsync-active = <1>;
+				vsync-active = <1>;
+				pclk-sample = <1>;
+				input-clock-freq = <32000000>;
+				pixel-clock-freq = <96000000>;
+			};
+		};
+	};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/media/i2c/ov1063x.txt b/Documentation/devicetree/bindings/media/i2c/ov1063x.txt
--- a/Documentation/devicetree/bindings/media/i2c/ov1063x.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/media/i2c/ov1063x.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,50 @@
+* OmniVision 8/10 bit digital camera
+
+The Omnivision digital camera is a 720p camera which is configurable at 8/10bit
+YUYV output and can be configured for various resolutions.
+
+Required Properties:
+- compatible : Must be one of the following
+  - "ovti,ov10633": For OV10633 camera
+  - "ovti,ov10635": For OV10635 camera
+- clocks: reference to the xvclk input clock.
+- clock-names: should be "xvclk".
+
+Optional Properties:
+- reset-gpios: reference to the GPIO connected to the reset pin, if any.
+- powerdown-gpios: reference to the GPIO connected to the pwdn pin, if any.
+- mux-gpios : A list of gpios with active high/low flags for enabling the
+              sensor. This may contain gpios for power, board muxes, etc.
+              Driver would set all of them as specified by the active
+              high/low flag
+
+  There are no custom optional properties supported for this device.
+  Although all the endpoint properties documented in
+  Documentation/devicetree/bindings/media/video-interfaces.txt are supported.
+
+Example:
+
+	&i2c2 {
+		...
+		...
+		ovcamera@30 {
+			compatible = "ovti,ov10635";
+			reg = <0x30>;
+
+			clocks = <&clk_ov10635>;
+			clock-names = "xvclk";
+
+			mux-gpios = <&pcf_hdmi 3 GPIO_ACTIVE_LOW>;
+			reset-gpios = <&gpio4 17 GPIO_ACTIVE_HIGH>;
+			powerdown-gpios = <&gpio5 11 GPIO_ACTIVE_HIGH>;
+
+			port {
+				onboard_cam: endpoint {
+					hsync-active = <1>;
+					vsync-active = <1>;
+					pclk-sample = <1>;
+				};
+			};
+		};
+		...
+	};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/media/i2c/ov490.txt b/Documentation/devicetree/bindings/media/i2c/ov490.txt
--- a/Documentation/devicetree/bindings/media/i2c/ov490.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/media/i2c/ov490.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,46 @@
+* OmniVision OV490 CSI2 Image Signal Processor (ISP)
+
+The Omnivision OV490 is a Image Signal Processor which is a companion
+processor for the Raw image sensors. This device can process the raw images
+and apply multiple algorithms to correct/transform/enhance the image quality.
+
+It needs programming lot of registers are most of them are written from
+a flash memory. The output is a CSI2 MIPI interface which can work with
+configurable number of lanes.
+
+Required Properties:
+- compatible : "ovti,ov490"
+
+Optional Properties:
+- mux-gpios : A list of gpios with active high/low flags for enabling the sensor.
+         This may contain gpios for power, board muxes, etc. Driver would
+         set all of them as specified by the active high/low flag.
+
+  There are no custom optional properties supported for this device.
+  Although all the CSI2 endpoint properties documented in
+  Documentation/devicetree/bindings/media/video-interfaces.txt are supported.
+
+Example:
+
+	&i2c2 {
+		...
+		...
+		ov490@cam0 {
+			compatible = "ovti,ov490";
+			reg = <0x24>;
+
+			mux-gpios = <&gpio_csi2_adap 0  GPIO_ACTIVE_LOW>,
+				    <&gpio_csi2_adap 1  GPIO_ACTIVE_HIGH>,
+				    <&gpio_csi2_adap 3  GPIO_ACTIVE_HIGH>,
+				    <&gpio_csi2_adap 4  GPIO_ACTIVE_LOW>;
+
+			port {
+				csi2_cam0: endpoint@0 {
+					clock-lanes = <0>;
+					data-lanes = <1 2 3 4>;
+					remote-endpoint = <&csi2_phy0>;
+				};
+			};
+		};
+		...
+	};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/media/img,d5520-vxd.txt b/Documentation/devicetree/bindings/media/img,d5520-vxd.txt
--- a/Documentation/devicetree/bindings/media/img,d5520-vxd.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/media/img,d5520-vxd.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,22 @@
+Imagination D5520-VXD Driver
+----------------------------
+
+The IMG VXD video decode driver for the D5500-VXD is a video decoder for
+multiple video formats including H.264 and HEVC on the TI J721E family
+of SoCs.
+
+Required properties:
+- compatible: must be "img,d5500-vxd"
+- reg: physical base address and length of the registers for the D5520
+- interrupts: should contain interrupt number for the D5520
+
+Example for J721E:
+==================
+
+       d5520: video-decoder@4300000 {
+               /* IMG D5520 driver configuration */
+               compatible = "img,d5500-vxd";
+               reg = <0x00 0x04300000 0x00 0x100000>;
+               power-domains = <&k3_pds 144>;
+               interrupts = <GIC_SPI 180 IRQ_TYPE_LEVEL_HIGH>;
+       };
diff -Naur --no-dereference a/Documentation/devicetree/bindings/media/img,vxe384.txt b/Documentation/devicetree/bindings/media/img,vxe384.txt
--- a/Documentation/devicetree/bindings/media/img,vxe384.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/media/img,vxe384.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,20 @@
+Imagination VXE384 Driver
+-------------------------
+
+The IMG VXE384 video encode driver for the VXE384 is a video encoder for
+multiple video formats including H.264 on the TI J721E family of SoCs.
+
+Required properties:
+- compatible: must be "img,vxe384"
+- reg: physical base address and length of the registers for the VXE384
+- interrupts: should contain interrupt number for the VXE384
+
+Example for J721E:
+==================
+
+       vxe384: video-encoder@4200000 {
+               compatible = "img,vxe384";
+               reg = <0x00 0x04200000 0x00 0x100000>;
+               power-domains = <&k3_pds 153 TI_SCI_PD_EXCLUSIVE>;
+               interrupts = <GIC_SPI 181 IRQ_TYPE_LEVEL_HIGH>;
+       };
diff -Naur --no-dereference a/Documentation/devicetree/bindings/media/ti-cal.txt b/Documentation/devicetree/bindings/media/ti-cal.txt
--- a/Documentation/devicetree/bindings/media/ti-cal.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/media/ti-cal.txt	2020-06-19 21:00:31.400227715 -0400
@@ -7,12 +7,21 @@
 DRA72x device.
 
 Required properties:
-- compatible: must be "ti,dra72-cal"
+- compatible:
+ Should be "ti,dra72-cal", for DRA72 controllers
+ Should be "ti,dra72-pre-es2-cal", for DRA72 controllers pre ES2.0
+ Should be "ti,dra76-cal", for DRA76 controllers
+ Should be "ti,am654-cal", for AM654 controllers
 - reg:	CAL Top level, Receiver Core #0, Receiver Core #1 and Camera RX
 	control address space
-- reg-names: cal_top, cal_rx_core0, cal_rx_core1, and camerrx_control
+- reg-names: cal_top, cal_rx_core0, cal_rx_core1 and camerrx_control
 	     registers
 - interrupts: should contain IRQ line for the CAL;
+- syscon-camerrx: phandle to the device control module and offset to the
+		  control_camerarx_core register
+		  This node is meant to replace the "camerrx_control" reg
+		  entry above but "camerrx_control" is still handled
+		  for backward compatibility.
 
 CAL supports 2 camera port nodes on MIPI bus. Each CSI2 camera port nodes
 should contain a 'port' child node with child 'endpoint' node. Please
@@ -25,13 +34,12 @@
 		ti,hwmods = "cal";
 		reg = <0x4845B000 0x400>,
 		      <0x4845B800 0x40>,
-		      <0x4845B900 0x40>,
-		      <0x4A002e94 0x4>;
+		      <0x4845B900 0x40>;
 		reg-names = "cal_top",
 			    "cal_rx_core0",
-			    "cal_rx_core1",
-			    "camerrx_control";
+			    "cal_rx_core1";
 		interrupts = <GIC_SPI 119 IRQ_TYPE_LEVEL_HIGH>;
+		syscon-camerrx = <&scm_conf 0xE94>;
 		#address-cells = <1>;
 		#size-cells = <0>;
 
diff -Naur --no-dereference a/Documentation/devicetree/bindings/media/ti-vip.txt b/Documentation/devicetree/bindings/media/ti-vip.txt
--- a/Documentation/devicetree/bindings/media/ti-vip.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/media/ti-vip.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,94 @@
+Texas Instruments DRA7x VIDEO INPUT PORT (VIP)
+----------------------------------------------
+
+The Video Input Port (VIP) is a key component for image capture
+applications. The capture module provides the system interface and the
+processing capability to connect parallel image-sensor as well as
+BT.656/1120 capable encoder chip.
+
+Required properties:
+- compatible: must be "ti,vip1", "ti,vip2" or "ti,vip3".
+- reg:	VIP top level, parser, colorspace converter, scaler for slice 0 and 1
+	and vpdma memory address space;
+- reg-names: vip, parser0, csc0, sc0, parser1, csc1, sc1 and vpdma registers;
+- interrupts: should contain IRQ line for VIP;
+- syscon-pol: phandle to the device control module;
+
+VIP supports 2 slices. Each slice can handle up to 2 camera port nodes.
+Each port nodes should contain a 'port' child node with child 'endpoint'
+node. Please refer to the bindings defined in
+Documentation/devicetree/bindings/media/video-interfaces.txt.
+
+Example:
+	vip1: vip@0x48970000 {
+		compatible = "ti,vip1";
+		reg = <0x48970000 0x114>,
+		      <0x48975500 0xD8>,
+		      <0x48975700 0x18>,
+		      <0x48975800 0x80>,
+		      <0x48975a00 0xD8>,
+		      <0x48975c00 0x18>,
+		      <0x48975d00 0x80>,
+		      <0x4897d000 0x400>;
+		reg-names = "vip",
+			    "parser0",
+			    "csc0",
+			    "sc0",
+			    "parser1",
+			    "csc1",
+			    "sc1",
+			    "vpdma";
+		ti,hwmods = "vip1";
+		interrupts = <GIC_SPI 351 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 392 IRQ_TYPE_LEVEL_HIGH>;
+		/* CTRL_CORE_SMA_SW_1 */
+		syscon-pol = <&scm_conf 0x534>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		status = "disabled";
+		vin1a: port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			status = "disabled";
+			endpoint@0 {
+				slave-mode;
+				remote-endpoint = <&camera1>;
+			};
+		};
+		vin2a: port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			status = "disabled";
+		};
+		vin1b: port@2 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <2>;
+			status = "disabled";
+		};
+		vin2b: port@3 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <3>;
+			status = "disabled";
+		};
+	};
+
+	i2c5: i2c@4807c000 {
+		ov10633@37 {
+			compatible = "ovti,ov10633";
+			reg = <0x37>;
+
+			mux-gpios = <&pcf_hdmi 3 GPIO_ACTIVE_LOW>;
+			port {
+				camera1: endpoint {
+					remote-endpoint = <&vin1a>;
+					hsync-active = <1>;
+					vsync-active = <1>;
+					pclk-sample = <0>;
+				};
+			};
+		};
+	};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/media/ti-vpe.txt b/Documentation/devicetree/bindings/media/ti-vpe.txt
--- a/Documentation/devicetree/bindings/media/ti-vpe.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/media/ti-vpe.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,33 @@
+Texas Instruments DRA7x VIDEO PROCESSING ENGINE (VPE)
+------------------------------------------------------
+
+The Video Processing Engine (VPE) is a key component for image post
+processing applications. VPE consist of a single memory to memory
+path which can perform chroma up/down sampling, deinterlacing,
+scaling and color space conversion.
+
+Required properties:
+- compatible: must be "ti,vpe"
+- reg:	physical base address and length of the registers set for the 8
+	memory regions required;
+- reg-names: name associated with the memory regions described is <reg>;
+- interrupts: should contain IRQ line for VPE;
+
+Example:
+	vpe {
+		compatible = "ti,vpe";
+		ti,hwmods = "vpe";
+		clocks = <&dpll_core_h23x2_ck>;
+		clock-names = "fck";
+		reg =	<0x489d0000 0x120>,
+			<0x489d0700 0x80>,
+			<0x489d5700 0x18>,
+			<0x489dd000 0x400>;
+		reg-names =	"vpe_top",
+				"sc",
+				"csc",
+				"vpdma";
+		interrupts = <GIC_SPI 354 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+	};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/misc/ti,dma_buf_phys.txt b/Documentation/devicetree/bindings/misc/ti,dma_buf_phys.txt
--- a/Documentation/devicetree/bindings/misc/ti,dma_buf_phys.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/misc/ti,dma_buf_phys.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,17 @@
+DMA-BUF contiguous buffer physical address user-space exporter binding
+
+Driver allowing user-space attaching of DMA-BUFs returning CPU physical
+addresses. The reasoning for making this act like a regular device
+described by DT is so the virtual device that binds the buffer can be
+made to act as if it is out on a bus or behind an IOMMU, for example.
+
+Required Properties:
+ - compatible:		Must be "ti,dma_buf_phys"
+
+Example:
+
+dma_buf_phys {
+	compatible = "ti,dma_buf_phys";
+	iommus = <&some_iommu>;
+	dma-coherent;
+};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/misc/ti,pat.txt b/Documentation/devicetree/bindings/misc/ti,pat.txt
--- a/Documentation/devicetree/bindings/misc/ti,pat.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/misc/ti,pat.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,35 @@
+Texas Instruments Page-based Address Translator (PAT) driver binding
+--------------------------------------------------------------------
+
+A Page-based Address Translator (PAT) device performs address translation
+using tables stored in an internal SRAM. Each PAT supports a set number of
+pages, each occupying a programmable 4KB, 16KB, 64KB, or 1MB of addresses
+in a window for which an incoming transaction will be translated.
+
+TI-PAT controller Device Node
+=============================
+
+The TI-PAT node describes the Texas Instrument's Page-based Address
+Translator (PAT).
+
+Required properties:
+-------------------
+- compatible: should be "ti,j721e-pat"
+- reg-names:
+	mmrs - Memory mapped registers region
+	table - Location of the table of translation pages
+- reg: register addresses corresponding to the above
+- ti,pat-window-base: base address of window translated by this PAT
+- ti,pat-window-size: size of window translated by this PAT
+
+Example:
+
+navss_pat0: pat@31010000 {
+	compatible = "ti,j721e-pat";
+	reg = <0x00 0x31010000 0x00 0x00000100>,
+	      <0x00 0x36400000 0x00 0x00040000>;
+	reg-names = "mmrs",
+	            "table";
+	ti,pat-window-base = <0x48 0x00000000>;
+	ti,pat-window-size = <0x00 0x40000000>;
+};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/mmc/sdhci-am654.txt b/Documentation/devicetree/bindings/mmc/sdhci-am654.txt
--- a/Documentation/devicetree/bindings/mmc/sdhci-am654.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/mmc/sdhci-am654.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,61 @@
+Device Tree Bindings for the SDHCI Controllers present on TI's AM654 SOCs
+
+The bindings follow the mmc[1], clock[2] and interrupt[3] bindings.
+Only deviations are documented here.
+
+  [1] Documentation/devicetree/bindings/mmc/mmc.txt
+  [2] Documentation/devicetree/bindings/clock/clock-bindings.txt
+  [3] Documentation/devicetree/bindings/interrupt-controller/interrupts.txt
+
+Required Properties:
+	- compatible: should be one of:
+			"ti,am654-sdhci-5.1": SDHCI on AM654 device.
+			"ti,j721e-sdhci-8bit": 8 bit SDHCI on J721E device.
+			"ti,j721e-sdhci-4bit": 4 bit SDHCI on J721E device.
+	- reg: Must be two entries.
+		- The first should be the sdhci register space
+		- The second should the subsystem/phy register space
+	- clocks: Handles to the clock inputs.
+	- clock-names: Tuple including "clk_xin" and "clk_ahb"
+	- interrupts: Interrupt specifiers
+	Output tap delay for each speed mode:
+	- ti,otap-del-sel-legacy
+	- ti,otap-del-sel-mmc-hs
+	- ti,otap-del-sel-sd-hs
+	- ti,otap-del-sel-sdr12
+	- ti,otap-del-sel-sdr25
+	- ti,otap-del-sel-sdr50
+	- ti,otap-del-sel-sdr104
+	- ti,otap-del-sel-ddr50
+	- ti,otap-del-sel-ddr52
+	- ti,otap-del-sel-hs200
+	- ti,otap-del-sel-hs400
+	  These bindings must be provided otherwise the driver will disable the
+	  corresponding speed mode (i.e. all nodes must provide at least -legacy)
+
+Optional Properties (Required for ti,am654-sdhci-5.1 and ti,j721e-sdhci-8bit):
+	- ti,trm-icp: DLL trim select
+	- ti,driver-strength-ohm: driver strength in ohms.
+				  Valid values are 33, 40, 50, 66 and 100 ohms.
+Optional Properties:
+	- ti,strobe-sel: strobe select delay for HS400 speed mode. Default value: 0x0.
+	- ti,clkbuf-sel: clock delay buffer select. Default value: 0x0
+
+Example:
+
+	sdhci0: sdhci@4f80000 {
+		compatible = "ti,am654-sdhci-5.1";
+		reg = <0x0 0x4f80000 0x0 0x260>, <0x0 0x4f90000 0x0 0x134>;
+		power-domains = <&k3_pds 47>;
+		clocks = <&k3_clks 47 0>, <&k3_clks 47 1>;
+		clock-names = "clk_ahb", "clk_xin";
+		interrupts = <GIC_SPI 136 IRQ_TYPE_LEVEL_HIGH>;
+		sdhci-caps-mask = <0x80000007 0x0>;
+		mmc-ddr-1_8v;
+		ti,otap-del-sel-legacy = <0x0>;
+		ti,otap-del-sel-mmc-hs = <0x0>;
+		ti,otap-del-sel-ddr52 = <0x5>;
+		ti,otap-del-sel-hs200 = <0x5>;
+		ti,otap-del-sel-hs400 = <0x0>;
+		ti,trm-icp = <0x8>;
+	};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/mmc/sdhci-omap.txt b/Documentation/devicetree/bindings/mmc/sdhci-omap.txt
--- a/Documentation/devicetree/bindings/mmc/sdhci-omap.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/mmc/sdhci-omap.txt	2020-06-19 21:00:31.400227715 -0400
@@ -2,6 +2,8 @@
 
 Refer to mmc.txt for standard MMC bindings.
 
+For UHS devices which require tuning, the device tree should have a "cpu_thermal" node which maps to the appropriate thermal zone. This is used to get the temperature of the zone during tuning.
+
 Required properties:
 - compatible: Should be "ti,dra7-sdhci" for DRA7 and DRA72 controllers
 	      Should be "ti,k2g-sdhci" for K2G
diff -Naur --no-dereference a/Documentation/devicetree/bindings/mtd/cadence-quadspi.txt b/Documentation/devicetree/bindings/mtd/cadence-quadspi.txt
--- a/Documentation/devicetree/bindings/mtd/cadence-quadspi.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/mtd/cadence-quadspi.txt	2020-06-19 21:00:31.400227715 -0400
@@ -4,6 +4,9 @@
 - compatible : should be one of the following:
 	Generic default - "cdns,qspi-nor".
 	For TI 66AK2G SoC - "ti,k2g-qspi", "cdns,qspi-nor".
+	For TI AM654 SoC  - "ti,am654-ospi", "cdns,qspi-nor".
+	For TI J721e SoC - "ti,j721e-ospi", "ti,am654-ospi".
+
 - reg : Contains two entries, each of which is a tuple consisting of a
 	physical address and length. The first entry is the address and
 	length of the controller register set. The second entry is the
@@ -19,6 +22,8 @@
 - cdns,rclk-en : Flag to indicate that QSPI return clock is used to latch
   the read data rather than the QSPI clock. Make sure that QSPI return
   clock is populated on the board before using this property.
+- cdns,delay-elem-ps: Size of single PHY DLL delay element in ps.
+  Defaults to 80ps if property is absent.
 
 Optional subnodes:
 Subnodes of the Cadence Quad SPI controller are spi slave nodes with additional
@@ -34,6 +39,8 @@
 		  (qspi_n_ss_out).
 - cdns,tslch-ns : Delay in nanoseconds between setting qspi_n_ss_out low
                   and first bit transfer.
+- cdns,phy-mode : Flag to indicate whether PHY needs to be enabled when
+		  reading from flash
 
 Example:
 
diff -Naur --no-dereference a/Documentation/devicetree/bindings/mtd/cypress,hyperflash.txt b/Documentation/devicetree/bindings/mtd/cypress,hyperflash.txt
--- a/Documentation/devicetree/bindings/mtd/cypress,hyperflash.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/mtd/cypress,hyperflash.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,13 @@
+Bindings for HyperFlash NOR flash chips compliant with Cypress HyperBus
+specification and supports Cypress CFI specification 1.5 command set.
+
+Required properties:
+- compatible : "cypress,hyperflash", "cfi-flash" for HyperFlash NOR chips
+- reg : Address of flash's memory map
+
+Example:
+
+	flash@0 {
+		compatible = "cypress,hyperflash", "cfi-flash";
+		reg = <0x0 0x4000000>;
+	};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/mtd/ti,am654-hbmc.txt b/Documentation/devicetree/bindings/mtd/ti,am654-hbmc.txt
--- a/Documentation/devicetree/bindings/mtd/ti,am654-hbmc.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/mtd/ti,am654-hbmc.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,52 @@
+Bindings for HyperBus Memory Controller (HBMC) on TI's K3 family of SoCs
+
+Required properties:
+- compatible : "ti,am654-hbmc" for AM654 SoC
+- compatible : "ti,j721e-hbmc", "ti,am654-hbmc" for J721E SoC
+- reg : Two entries:
+	First entry pointed to the register space of HBMC controller
+	Second entry pointing to the memory map region dedicated for
+	MMIO access to attached flash devices
+- ranges : Address translation from offset within CS to allocated MMIO
+	   space in SoC
+
+Optional properties:
+- mux-controls : phandle to the multiplexer that controls selection of
+		 HBMC vs OSPI inside Flash SubSystem (FSS). Default is OSPI,
+		 if property is absent.
+		 See Documentation/devicetree/bindings/mux/reg-mux.txt
+		 for mmio-mux binding details
+
+Example:
+
+	fss: system-controller@47000000 {
+		compatible = "syscon", "simple-mfd";
+		reg = <0x0 0x47000000 0x0 0x100>;
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		hbmc_mux: multiplexer {
+			compatible = "mmio-mux";
+			#mux-control-cells = <1>;
+			mux-reg-masks = <0x4 0x2>; /* 0: reg 0x4, bit 1 */
+		};
+
+		hbmc: hyperbus@47034000 {
+			compatible = "ti,am654-hbmc";
+			reg = <0x0 0x47034000 0x0 0x100>,
+				<0x5 0x00000000 0x1 0x0000000>;
+			power-domains = <&k3_pds 55>;
+			#address-cells = <2>;
+			#size-cells = <1>;
+			ranges = <0x0 0x0 0x5 0x00000000 0x4000000>, /* CS0 - 64MB */
+				 <0x1 0x0 0x5 0x04000000 0x4000000>; /* CS1 - 64MB */
+			mux-controls = <&hbmc_mux 0>;
+
+			/* Slave flash node */
+			flash@0,0 {
+				compatible = "cypress,hyperflash", "cfi-flash";
+				reg = <0x0 0x0 0x4000000>;
+			};
+		};
+	};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/net/can/m_can.txt b/Documentation/devicetree/bindings/net/can/m_can.txt
--- a/Documentation/devicetree/bindings/net/can/m_can.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/net/can/m_can.txt	2020-06-19 21:00:31.400227715 -0400
@@ -48,6 +48,8 @@
 			  that can be used for CAN/CAN-FD modes. See
 			  Documentation/devicetree/bindings/net/can/can-transceiver.txt
 			  for details.
+stb-gpios		: gpio node to toggle the STB (standby) signal on the transceiver
+
 Example:
 SoC dtsi:
 m_can1: can@20e8000 {
diff -Naur --no-dereference a/Documentation/devicetree/bindings/net/cpsw-phy-sel.txt b/Documentation/devicetree/bindings/net/cpsw-phy-sel.txt
--- a/Documentation/devicetree/bindings/net/cpsw-phy-sel.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/net/cpsw-phy-sel.txt	2020-06-19 21:00:31.400227715 -0400
@@ -5,6 +5,7 @@
 - compatible		: Should be "ti,am3352-cpsw-phy-sel" for am335x platform and
 			  "ti,dra7xx-cpsw-phy-sel" for dra7xx platform
 			  "ti,am43xx-cpsw-phy-sel" for am43xx platform
+			  "ti,am654-cpsw-phy-sel" for am654x platform
 - reg			: physical base address and size of the cpsw
 			  registers map
 - reg-names		: names of the register map given in "reg" node
diff -Naur --no-dereference a/Documentation/devicetree/bindings/net/cpsw.txt b/Documentation/devicetree/bindings/net/cpsw.txt
--- a/Documentation/devicetree/bindings/net/cpsw.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/net/cpsw.txt	2020-06-19 21:00:31.400227715 -0400
@@ -40,6 +40,10 @@
 			  Mult and shift will be calculated basing on CPTS
 			  rftclk frequency if both cpts_clock_shift and
 			  cpts_clock_mult properties are not provided.
+- cpts-ext-ts-inputs	: The number of external time stamp channels.
+			  The different CPTS versions might support up 8
+			  external time stamp channels.
+			  if absent - unsupported.
 
 Slave Properties:
 Required properties:
diff -Naur --no-dereference a/Documentation/devicetree/bindings/net/keystone-netcp.txt b/Documentation/devicetree/bindings/net/keystone-netcp.txt
--- a/Documentation/devicetree/bindings/net/keystone-netcp.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/net/keystone-netcp.txt	2020-06-19 21:00:31.400227715 -0400
@@ -62,32 +62,42 @@
 		big endian mode with the DSP in little endian.
 
 NetCP device properties: Device specification for NetCP sub-modules.
+
 1Gb/10Gb (gbe/xgbe) ethernet switch sub-module specifications.
 Required properties:
 - label:	Must be "netcp-gbe" for 1Gb & "netcp-xgbe" for 10Gb.
 - compatible:	Must be one of below:-
 		"ti,netcp-gbe" for 1GbE on NetCP 1.4
-		"ti,netcp-gbe-5" for 1GbE N NetCP 1.5 (N=5)
-		"ti,netcp-gbe-9" for 1GbE N NetCP 1.5 (N=9)
-		"ti,netcp-gbe-2" for 1GbE N NetCP 1.5 (N=2)
+		"ti,netcp-gbe-5" for 1GbE on NetCP 1.5 (N=5)
+		"ti,netcp-gbe-9" for 1GbE on NetCP 1.5 (N=9)
+		"ti,netcp-gbe-2" for 1GbE on NSS (Network Subsystem)
 		"ti,netcp-xgbe" for 10 GbE
 
+- syscon-subsys:	phandle to syscon node of the switch
+			subsystem registers of NetCP 1.4/1.5
+			or EMAC registers of NSS
+
+- syscon-pcsr:		(10gbe only) phandle to syscon node of the
+			switch PCSR registers.
+
 - reg:		register location and the size for the following register
 		regions in the specified order.
 		- switch subsystem registers
+		- sgmii module registers
 		- sgmii port3/4 module registers (only for NetCP 1.4)
 		- switch module registers
-		- serdes registers (only for 10G)
 
 		NetCP 1.4 ethss, here is the order
-			index #0 - switch subsystem registers
+			index #0 - sgmii module registers
 			index #1 - sgmii port3/4 module registers
 			index #2 - switch module registers
 
-		NetCP 1.5 ethss 9 port, 5 port and 2 port
-			index #0 - switch subsystem registers
+		NetCP 1.5 ethss 9 port, 5 port
+			index #0 - sgmii module registers
 			index #1 - switch module registers
-			index #2 - serdes registers
+
+		NSS of K2G SoC
+			index #0 sgmii module registers
 
 - tx-channel:	the navigator packet dma channel name for tx.
 - tx-queue:	the navigator queue number associated with the tx dma channel.
@@ -104,6 +114,10 @@
 			- 10Gb mac<->mac forced mode : 11
 ----phy-handle:	phandle to PHY device
 
+----phys:	phandles to serdes PHY devices
+		see Documentation/devicetree/bindings/phy/ti-phy.txt
+		for Keystone SerDes device specificcations.
+
 Optional properties:
 - enable-ale:	NetCP driver keeps the address learning feature in the ethernet
 		switch module disabled. This attribute is to enable the address
@@ -113,6 +127,21 @@
 				will only initialize these ports and attach PHY
 				driver to them if needed.
 
+ Properties related to cpts configurations.
+	- cpts-rftclk-sel: selects one of up to 32 clocks for time sync
+		reference.  Default = 0.
+	- cpts_clock_mult/cpts_clock_shift:
+		used for converting time counter cycles to ns as in
+
+		ns = (cycles * clock_mult) >> _shift
+
+		Defaults: clock_mult, clock_shift = calculated from
+		CPTS refclk
+	- cpts-ext-ts-inputs:
+		The number of external time stamp channels.
+		The different CPTS versions might support up 8
+		external time stamp channels. if absent - unsupported.
+
 NetCP interface properties: Interface specification for NetCP sub-modules.
 Required properties:
 - rx-channel:	the navigator packet dma channel name for rx.
@@ -145,6 +174,16 @@
 
 Example binding:
 
+gbe_subsys: subsys@2090000 {
+	compatible = "syscon";
+	reg = <0x02090000 0x100>;
+};
+
+gbe_serdes: phy@232a000 {
+	compatible		= "ti,keystone-serdes-gbe";
+	...
+};
+
 netcp: netcp@2000000 {
 	reg = <0x2620110 0x8>;
 	reg-names = "efuse";
@@ -154,7 +193,6 @@
 	ranges  = <0 0x2000000 0xfffff>;
 	clocks = <&papllclk>, <&clkcpgmac>, <&chipclk12>;
 	dma-coherent;
-	/* big-endian; */
 	dma-id = <0>;
 
 	netcp-devices {
@@ -163,17 +201,19 @@
 		ranges;
 		gbe@90000 {
 			label = "netcp-gbe";
-			reg = <0x90000 0x300>, <0x90400 0x400>, <0x90800 0x700>;
-			/* enable-ale; */
+			syscon-subsys = <&gbe_subsys>;
+			reg = <0x90100 0x200>, <0x90400 0x200>, <0x90800 0x700>;
 			tx-queue = <648>;
 			tx-channel = <8>;
 
 			interfaces {
 				gbe0: interface-0 {
+					phys = <&serdes_lane0>;
 					slave-port = <0>;
 					link-interface	= <4>;
 				};
 				gbe1: interface-1 {
+					phys = <&serdes_lane1>;
 					slave-port = <1>;
 					link-interface	= <4>;
 				};
@@ -181,10 +221,12 @@
 
 			secondary-slave-ports {
 				port-2 {
+					phys = <&serdes_lane2>;
 					slave-port = <2>;
 					link-interface	= <2>;
 				};
 				port-3 {
+					phys = <&serdes_lane3>;
 					slave-port = <3>;
 					link-interface	= <2>;
 				};
@@ -219,3 +261,81 @@
 		};
 	};
 };
+
+Example bindings (10gbe):
+
+xgbe_serdes: phy@231e000 {
+	compatible		= "ti,keystone-serdes-xgbe";
+	...
+};
+
+netcpx: netcpx@2f00000 {
+	status		= "disabled";
+	compatible	= "ti,netcp-1.0";
+	#address-cells	= <1>;
+	#size-cells	= <1>;
+	ranges;
+
+	clocks		= <&clkxge>;
+	clock-names	= "clk_xge";
+	dma-coherent;
+
+	ti,navigator-dmas = <&dma_xgbe 0>,
+			<&dma_xgbe 8>,
+			<&dma_xgbe 0>;
+	ti,navigator-dma-names = "xnetrx0", "xnetrx1", "xnettx";
+
+	netcp-devices {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+		xgbe@2f00000 {
+			label = "netcp-xgbe";
+			compatible = "ti,netcp-xgbe";
+			syscon-subsys = <&xgbe_subsys>;
+			syscon-pcsr = <&xgbe_pcsr>;
+			reg = <0x02f00100 0x200>, <0x02f01000 0xb00>;
+			tx-queue = <692>;
+			tx-channel = "xnettx";
+
+			interfaces {
+				xgbe0: interface-0 {
+					phys = <&xserdes_lane0>;
+					slave-port = <0>;
+					link-interface	= <11>;
+				};
+				xgbe1: interface-1 {
+					phys = <&xserdes_lane1>;
+					slave-port = <1>;
+					link-interface	= <11>;
+				};
+			};
+		};
+	};
+
+	netcp-interfaces {
+		interface-0 {
+			rx-channel = "xnetrx0";
+			rx-pool = <2048 12>;
+			tx-pool = <1024 12>;
+			rx-queue-depth = <1024 1024 0 0>;
+			rx-buffer-size = <1536 4096 0 0>;
+			rx-queue = <532>;
+			tx-completion-queue = <534>;
+			efuse-mac = <0>;
+			netcp-xgbe = <&xgbe0>;
+
+		};
+		interface-1 {
+			rx-channel = "xnetrx1";
+			rx-pool = <2048 12>;
+			tx-pool = <1024 12>;
+			rx-queue-depth = <1024 1024 0 0>;
+			rx-buffer-size = <1536 4096 0 0>;
+			rx-queue = <533>;
+			tx-completion-queue = <535>;
+			efuse-mac = <0>;
+			netcp-xgbe = <&xgbe1>;
+		};
+	};
+};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/net/ti,am654-cpsw-nuss.txt b/Documentation/devicetree/bindings/net/ti,am654-cpsw-nuss.txt
--- a/Documentation/devicetree/bindings/net/ti,am654-cpsw-nuss.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/net/ti,am654-cpsw-nuss.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,214 @@
+TI AM654 SoC Gigabit Ethernet Switch (CPSW) subsystem Device Tree Bindings
+------------------------------------------------------
+
+The TI am654 SoC Gigabit Ethernet Switch subsystem (CPSW NUSS) has two ports and
+provides Ethernet packet communication for the device and can be configured
+as an Ethernet switch. CPSW NUSS features: the Reduced Gigabit Media Independent
+Interface (RGMII), Reduced Media Independent Interface (RMII), and the
+Management Data Input/Output (MDIO) interface for physical layer device
+(PHY) management.
+
+The TI am654 SoC has integrated two-port Gigabit Ethernet Switch subsystem
+into device MCU domain named MCU_CPSW0. One Ethernet port (port 1) with
+selectable RGMII and RMII interfaces and an internal Communications
+Port Programming Interface (CPPI) port (Host port 0).
+
+Host Port 0 CPPI Packet Streaming Interface interface supports 8 TX channels
+and on RX channels operating by TI am654 NAVSS Unified DMA Peripheral Root
+Complex (UDMA-P) controller.
+
+Required properties:
+- compatible	: Should be "ti,am654-cpsw-nuss" or
+		  "ti,j721e-cpsw-nuss"
+- reg		: physical base address and size of the CPSW NUSS registers map
+- reg-names	: should be "cpsw_nuss"
+
+- clocks	: CPSW NUSS functional clock
+- clock-names	: should be "fck"
+		See Documentation/devicetree/bindings/clock/ti,sci-clk.txt
+- power-domains : CPSW NUSS power domain
+		See Documentation/devicetree/bindings/soc/ti/sci-pm-domain.txt
+
+- interrupt-parent : The parent interrupt controller - should point on TI
+		System Control Interface (TI-SCI) IRQCHIP phandle
+		See Documentation/devicetree/bindings/interrupt-controller/ti,sci-irq.txt
+
+- dma-coherent	: indicates that CPSW NUSS operates with coherent memory.
+- dmas		: list of UDMA-P controller channels specifiers
+- dma-names	: should be "tx0", "tx1", "tx2", "tx3", "tx4", "tx5", "tx6",
+		"tx7", "rx";
+- ti,psil-base : CPSW NUSS PSI-L endpoint thread ID base of the UDMA-P channels
+		Within the PSI-L endpoint node thread configuration subnodes
+		must present with ti,psil-configX naming convention,
+		where X is the thread ID offset.
+		See Documentation/devicetree/bindings/dma/ti/k3-udma.txt
+
+Optional properties:
+- ti,rx-flow-id-base : specifies fixed UDMA-P RX flow id offset to be used
+		by RX CPPI DMA channel. It will be selected dynamically
+		if not specified.
+
+- syscon	: Phandle to the system control device node, which is
+		the control module device of the am33x
+
+Required Sub-nodes:
+- ti,psil-config0..ti,psil-config7 : PSI-L endpoint node threads configuration
+		subnodes
+
+	Required properties:
+	- linux,udma-mode : should be UDMA_PKT_MODE
+	- statictr-type : should be PSIL_STATIC_TR_NONE
+	- ti,needs-epib : CPSW NUSS provides/accepts EPIB with each packet
+	- ti,psd-size	: CPSW NUSS provides/accepts 16 bytes of PS data with
+			each packet
+
+- ports	: contains CPSW NUSS ports descriptions
+	Required properties - all ports:
+	- reg : CPSW NUSS port number. Should be 0,1 for TI am654 SoC
+		0 - Host port 0
+		1.. - external Ethernet ports
+
+	Optional properties - all ports:
+	- ti,label : Describes the label associated with this port
+
+	Required properties for - external Ethernet ports:
+	- phy-mode : operation mode of the PHY interface [1]
+	- phy-handle : phandle, specifies a reference to a node representing
+		a PHY device [1]
+
+	Optional properties for - external Ethernet ports:
+	- ti,mac-only: port operates in MAC only mode
+	- mac-address : array of 6 bytes, specifies the MAC address. Always
+		accounted first if present [1]
+	- ti,syscon-efuse: tuple of two cells. First is phandle on syscon System
+		Control Module (SCM) node. Second is offset inside SCM points on
+		efuse registers containing Ethernet MAC address.
+		Accounted second if "mac-address" doesn't present.
+
+- mdio : CPSW NUSS MDIO bus description
+	- bus_freq : MDIO Bus frequency
+	See Documentation/devicetree/bindings/net/mdio.txt
+
+- cpsw-phy-sel: CPSW NUSS PHY mode Selection device description
+	Documentation/devicetree/bindings/net/cpsw-phy-sel.txt
+
+- cpts : The Common Platform Time Sync (CPTS) module description
+	Documentation/devicetree/bindings/net/ti,am654-cpts.txt
+
+[1] See Documentation/devicetree/bindings/net/ethernet.txt
+
+Examples:
+
+mcu_cpsw: cpsw_nuss@046000000 {
+	compatible = "ti,am654-cpsw-nuss";
+	#address-cells = <2>;
+	#size-cells = <2>;
+	reg = <0x0 0x46000000 0x0 0x200000>;
+	reg-names = "cpsw_nuss";
+	ranges;
+	dma-coherent;
+	clocks = <&k3_clks 5 10>;
+	clock-names = "fck";
+	power-domains = <&k3_pds 5>;
+	ti,psil-base = <0x7000>;
+
+	interrupt-parent = <&k3_irq>;
+
+	dmas = <&mcu_udmap &mcu_cpsw 0 UDMA_DIR_TX>,
+	       <&mcu_udmap &mcu_cpsw 1 UDMA_DIR_TX>,
+	       <&mcu_udmap &mcu_cpsw 2 UDMA_DIR_TX>,
+	       <&mcu_udmap &mcu_cpsw 3 UDMA_DIR_TX>,
+	       <&mcu_udmap &mcu_cpsw 4 UDMA_DIR_TX>,
+	       <&mcu_udmap &mcu_cpsw 5 UDMA_DIR_TX>,
+	       <&mcu_udmap &mcu_cpsw 6 UDMA_DIR_TX>,
+	       <&mcu_udmap &mcu_cpsw 7 UDMA_DIR_TX>,
+	       <&mcu_udmap &mcu_cpsw 0 UDMA_DIR_RX>;
+	dma-names = "tx0", "tx1", "tx2", "tx3",
+		    "tx4", "tx5", "tx6", "tx7",
+		    "rx";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		host: host@0 {
+			reg = <0>;
+			ti,label = "host";
+		};
+
+		cpsw_port1: port@1 {
+			reg = <1>;
+			ti,mac-only;
+			ti,label = "port1";
+			ti,syscon-efuse = <&mcu_conf 0x200>;
+		};
+	};
+
+	davinci_mdio: mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		bus_freq = <1000000>;
+	};
+
+	phy_sel: cpsw-phy-sel@40f04040 {
+		compatible = "ti,am654-cpsw-phy-sel";
+		reg= <0x0 0x40f04040 0x0 0x4>;
+		reg-names = "gmii-sel";
+	};
+
+	ti,psil-config0 {
+		linux,udma-mode = <UDMA_PKT_MODE>;
+		statictr-type = <PSIL_STATIC_TR_NONE>;
+		ti,needs-epib;
+		ti,psd-size = <16>;
+	};
+
+	ti,psil-config1 {
+		linux,udma-mode = <UDMA_PKT_MODE>;
+		statictr-type = <PSIL_STATIC_TR_NONE>;
+		ti,needs-epib;
+		ti,psd-size = <16>;
+	};
+
+	ti,psil-config2 {
+		linux,udma-mode = <UDMA_PKT_MODE>;
+		statictr-type = <PSIL_STATIC_TR_NONE>;
+		ti,needs-epib;
+		ti,psd-size = <16>;
+	};
+
+	ti,psil-config3 {
+		linux,udma-mode = <UDMA_PKT_MODE>;
+		statictr-type = <PSIL_STATIC_TR_NONE>;
+		ti,needs-epib;
+		ti,psd-size = <16>;
+	};
+
+	ti,psil-config4 {
+		linux,udma-mode = <UDMA_PKT_MODE>;
+		statictr-type = <PSIL_STATIC_TR_NONE>;
+		ti,needs-epib;
+		ti,psd-size = <16>;
+	};
+
+	ti,psil-config5 {
+		linux,udma-mode = <UDMA_PKT_MODE>;
+		statictr-type = <PSIL_STATIC_TR_NONE>;
+		ti,needs-epib;
+		ti,psd-size = <16>;
+	};
+
+	ti,psil-config6 {
+		linux,udma-mode = <UDMA_PKT_MODE>;
+		statictr-type = <PSIL_STATIC_TR_NONE>;
+		ti,needs-epib;
+		ti,psd-size = <16>;
+	};
+
+	ti,psil-config7 {
+		linux,udma-mode = <UDMA_PKT_MODE>;
+		statictr-type = <PSIL_STATIC_TR_NONE>;
+		ti,needs-epib;
+		ti,psd-size = <16>;
+	};
+};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/net/ti,am654-cpts.txt b/Documentation/devicetree/bindings/net/ti,am654-cpts.txt
--- a/Documentation/devicetree/bindings/net/ti,am654-cpts.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/net/ti,am654-cpts.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,111 @@
+TI AM65x SoC The Common Platform Time Sync (CPTS) module Device Tree Bindings
+------------------------------------------------------
+
+The CPTS module is used to facilitate host control of time sync operations.
+Main features of CPTS module are:
+- Supports the selection of multiple external clock sources
+- Software control of time sync events via interrupt or polling
+- Supports hardware timestamp push inputs (HWx_TS_PUSH)
+- Supports timestamp counter compare output (TS_COMP)
+- Supports timestamp counter bit output (TS_SYNC)
+- Supports timestamp Generator function outputs (TS_GENFx)
+- Supports 64-bit timestamp mode in ns with ppm and nudge adjustment.
+Depending on integration it enables compliance with the IEEE 1588-2008 standard
+for a precision clock synchronization protocol, Ethernet Enhanced Scheduled
+Traffic Operations (CPTS_ESTFn) and PCIe Subsystem Precision Time
+Measurement (PTM)
+
+TI AM65x SoC has several similar Time Sync (CPTS) modules integrated into the
+different parts of the system which could be synchronized with each other:
+- Main CPTS
+- MCU CPSW CPTS with IEEE 1588-2008 support
+- PCIe subsystem CPTS for PCIe Subsystem Precision Time Measurement (PTM)
+  support
+
+Properties:
+- compatible	: Should be "ti,am65-cpts"  or
+		  "ti,j721e-cpts"
+- reg		: physical base address and size of the CPTS registers map
+- reg-names	: should be "cpts"
+	Depending on CPTS module integration and when CPTS is integral part of
+	another module (MCU CPSW for example) "compatible" and "reg" can
+	be omitted - parent module is fully responsible for CPTS enabling and
+	configuration.
+- clocks	: CPTS reference clock. Should point on cpts_refclk_mux clock.
+- clock-names	: should be "cpts"
+- interrupts-extended : CPTS events interrupt specifier
+- interrupt-names : should be "cpts"
+
+Required Sub-nodes:
+- cpts_refclk_mux: CPTS reference clock multiplexer clock
+	Required properties:
+	- clocks : List of phandle and clock specifier pairs, one pair
+		for each possible CPTS reference clock input
+	- assigned-clocks : phandle on cpts_refclk_mux
+	- assigned-clock-parents : phandle on selected CPTS reference clock
+	Selection of CPTS reference clock expected to be done in board files.
+	By default first listed clock will be selected as CPTS reference clock.
+
+Optional properties:
+- ti,cpts-periodic-outputs : Number of timestamp Generator function
+		 outputs (TS_GENFx)
+- ti,cpts-ext-ts-inputs : Number of hardware timestamp push inputs (HWx_TS_PUSH)
+- ti,pps : pair of HWx_TS_PUSH input and TS_GENFy output indexes used for
+	PPS events generation. Platform/board specific.
+
+References:
+ Documentation/devicetree/bindings/clock/ti,sci-clk.txt
+ Documentation/devicetree/bindings/clock/clock-bindings.txt
+ Documentation/devicetree/bindings/interrupt-controller/interrupts.txt
+ Documentation/devicetree/bindings/interrupt-controller/ti,sci-irq.txt
+
+Examples:
+
+Main CPTS module:
+cpts@310d0000 {
+	compatible = "ti,am65-cpts";
+	reg = <0x0 0x310d0000 0x0 0x400>;
+	reg-names = "cpts";
+	clocks = <&main_cpts_mux>;
+	clock-names = "cpts";
+	interrupts-extended = <&k3_irq 163 0 IRQ_TYPE_LEVEL_HIGH>;
+	interrupt-names = "cpts";
+	ti,cpts-periodic-outputs = <6>;
+	ti,cpts-ext-ts-inputs = <8>;
+
+	main_cpts_mux: cpts_refclk_mux {
+		#clock-cells = <0>;
+		clocks = <&k3_clks 118 5>, <&k3_clks 118 11>,
+			<&k3_clks 157 91>, <&k3_clks 157 77>,
+			<&k3_clks 157 102>, <&k3_clks 157 80>,
+			<&k3_clks 120 3>, <&k3_clks 121 3>;
+		cpts-mux-tbl = <0>, <1>;
+		assigned-clocks = <&main_cpts_mux>;
+		assigned-clock-parents = <&k3_clks 118 11>;
+	};
+};
+
+MCU CPSW CPTS module:
+mcu_cpsw: cpsw_nuss@046000000 {
+	compatible = "ti,am654-cpsw-nuss";
+	[...]
+
+	cpts {
+		clocks = <&mcu_cpsw_cpts_mux>;
+		clock-names = "cpts";
+		interrupts-extended = <&gic GIC_SPI 570 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "cpts";
+		ti,cpts-ext-ts-inputs = <4>;
+		ti,cpts-periodic-outputs = <2>;
+
+		mcu_cpsw_cpts_mux: cpts_refclk_mux {
+			#clock-cells = <0>;
+			clocks = <&k3_clks 118 5>, <&k3_clks 118 11>,
+				<&k3_clks 157 91>, <&k3_clks 157 77>,
+				<&k3_clks 157 102>, <&k3_clks 157 80>,
+				<&k3_clks 120 3>, <&k3_clks 121 3>;
+			assigned-clocks = <&mcu_cpsw_cpts_mux>;
+			assigned-clock-parents = <&k3_clks 118 11>;
+		};
+	};
+};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/net/ti,cpsw-virt-mac.txt b/Documentation/devicetree/bindings/net/ti,cpsw-virt-mac.txt
--- a/Documentation/devicetree/bindings/net/ti,cpsw-virt-mac.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/net/ti,cpsw-virt-mac.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,126 @@
+TI J721E VIRT CPSWxg Ethernet mac
+======================================
+
+Required properties:
+- compatible       : Should be "ti,j721e-cpsw-virt-mac" for J721E Family SoCs
+- interrupt-parent : The parent interrupt controller - should point on TI
+		System Control Interface (TI-SCI) IRQCHIP phandle [1]
+- dma-coherent	: indicates that device operates with coherent memory.
+- dmas		: list of UDMA controller channel specifiers [2]
+- dma-names	: should be "tx0", "tx1", "tx2", "tx3", "tx4", "tx5", "tx6",
+		"tx7", "rx";
+- ti,psil-base : CPSWxg NUSS PSI-L endpoint thread ID base of the UDMA-P
+		channels. The PSI-L endpoint node thread configuration
+		subnodes must be present present with ti,psil-configX naming
+		convention, where X is the thread ID offset [2].
+- ti,remote-name: Name of the connected rpmsg-kdrv device represented
+		by Eth switch FW running on the on of R5F cores.
+		Should be "mpu_1_0_ethswitch-device-0" for J721E Family SoCs.
+
+Required Sub-nodes:
+- ti,psil-config0..ti,psil-config7 : PSI-L endpoint node threads configuration
+		subnodes [2]
+
+	Required properties:
+	- linux,udma-mode : should be UDMA_PKT_MODE
+	- statictr-type : should be PSIL_STATIC_TR_NONE
+	- ti,needs-epib : CPSW NUSS provides/accepts EPIB with each packet
+	- ti,psd-size	: CPSW NUSS provides/accepts 16 bytes of PS data with
+			each packet
+
+- virt_emac_port: contains VIRT CPSWxg Ethernet mac port descriptions
+	Optional properties - all ports:
+	- ti,label : Describes the label associated with this port
+	- local-mac-address: array of 6 bytes, the assigned MAC address [3].
+	- ti,rx-flow-id-base: Debug only. Defines the RX flow id to forcibly
+			    used by driver.
+
+The MAC address provided by Eth switch FW will be used if neither of
+"mac-address" and "local-mac-address" is defined.
+
+References:
+ [1] Documentation/devicetree/bindings/interrupt-controller/ti,sci-irq.txt
+ [2] Documentation/devicetree/bindings/dma/ti/k3-udma.txt
+ [3] See Documentation/devicetree/bindings/net/ethernet.txt
+
+Examples:
+cpsw9g_virt_mac: main_r5fss_cpsw9g_virt_mac0 {
+	compatible = "ti,j721e-cpsw-virt-mac";
+	dma-coherent;
+	ti,psil-base = <0x4a00>;
+	ti,remote-name = "mpu_1_0_ethswitch-device-0";
+	interrupt-parent = <&main_udmass_inta>;
+
+	dmas = <&main_udmap &cpsw9g_virt_mac 0 UDMA_DIR_TX>,
+	       <&main_udmap &cpsw9g_virt_mac 1 UDMA_DIR_TX>,
+	       <&main_udmap &cpsw9g_virt_mac 2 UDMA_DIR_TX>,
+	       <&main_udmap &cpsw9g_virt_mac 3 UDMA_DIR_TX>,
+	       <&main_udmap &cpsw9g_virt_mac 4 UDMA_DIR_TX>,
+	       <&main_udmap &cpsw9g_virt_mac 5 UDMA_DIR_TX>,
+	       <&main_udmap &cpsw9g_virt_mac 6 UDMA_DIR_TX>,
+	       <&main_udmap &cpsw9g_virt_mac 0 UDMA_DIR_RX>;
+	dma-names = "tx0", "tx1", "tx2", "tx3",
+		    "tx4", "tx5", "tx6",
+		    "rx";
+
+	virt_emac_port {
+		ti,label = "virt-port";
+		/* local-mac-address = [0 0 0 0 0 0]; */
+	};
+
+	ti,psil-config0 {
+		linux,udma-mode = <UDMA_PKT_MODE>;
+		statictr-type = <PSIL_STATIC_TR_NONE>;
+		ti,needs-epib;
+		ti,psd-size = <16>;
+	};
+
+	ti,psil-config1 {
+		linux,udma-mode = <UDMA_PKT_MODE>;
+		statictr-type = <PSIL_STATIC_TR_NONE>;
+		ti,needs-epib;
+		ti,psd-size = <16>;
+	};
+
+	ti,psil-config2 {
+		linux,udma-mode = <UDMA_PKT_MODE>;
+		statictr-type = <PSIL_STATIC_TR_NONE>;
+		ti,needs-epib;
+		ti,psd-size = <16>;
+	};
+
+	ti,psil-config3 {
+		linux,udma-mode = <UDMA_PKT_MODE>;
+		statictr-type = <PSIL_STATIC_TR_NONE>;
+		ti,needs-epib;
+		ti,psd-size = <16>;
+	};
+
+	ti,psil-config4 {
+		linux,udma-mode = <UDMA_PKT_MODE>;
+		statictr-type = <PSIL_STATIC_TR_NONE>;
+		ti,needs-epib;
+		ti,psd-size = <16>;
+	};
+
+	ti,psil-config5 {
+		linux,udma-mode = <UDMA_PKT_MODE>;
+		statictr-type = <PSIL_STATIC_TR_NONE>;
+		ti,needs-epib;
+		ti,psd-size = <16>;
+	};
+
+	ti,psil-config6 {
+		linux,udma-mode = <UDMA_PKT_MODE>;
+		statictr-type = <PSIL_STATIC_TR_NONE>;
+		ti,needs-epib;
+		ti,psd-size = <16>;
+	};
+
+	ti,psil-config7 {
+		linux,udma-mode = <UDMA_PKT_MODE>;
+		statictr-type = <PSIL_STATIC_TR_NONE>;
+		ti,needs-epib;
+		ti,psd-size = <16>;
+	};
+};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/net/ti,icssg-prueth.txt b/Documentation/devicetree/bindings/net/ti,icssg-prueth.txt
--- a/Documentation/devicetree/bindings/net/ti,icssg-prueth.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/net/ti,icssg-prueth.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,91 @@
+Texas Instruments ICSSG PRUSS Ethernet
+======================================
+
+Required properties:
+- compatible       : Should be "ti,am654-icssg-prueth" for AM65x Family SoCs
+- prus             : list of pHandles to the PRU and RTU nodes
+- firmware-name    : should contain the name of the firmware image
+			file located in the firmware search path
+- sram	           : phandle to MSMC SRAM node
+- dmas		   : list of phandles and specifiers to UDMA as specified in
+			bindings/dma/ti/k3-udma.txt.
+- dma-names	   : Names for the DMA channels.
+			Should be "tx0-0", "tx0-1", "tx0-2", "0-3",
+			    "tx1-0", "tx1-1", "tx1-2", "tx1-3",
+			    "rx0", "rx1", "rxmgm0", "rxmgm1"
+- mii-g-rt	   : phandle to MII_G_RT module's syscon regmap.
+- mii-rt	   : phandle to MII_RT module's syscon regmap.
+
+Must contain children, one for each of the MAC ports.
+Children must be named ethernet-mii0 and ethernet-mii1.
+Either one or both children can be present. If only one
+child is present driver operates in single EMAC mode.
+
+For single mode operation with the 2nd SLICE, you still need
+to provide both PRUs and RTUs and firmware-names but the firmware-name
+for the first PRU & RTU can be NULL.
+
+Required properties for children:
+- phy-handle       : See ethernet.txt file in the same directory.
+- phy-mode         : See ethernet.txt file in the same directory.
+- syscon-rgmii-delay : phandle to system controller node and register offset
+			to ICSSG control register for RGMII transmit delay.
+- iep		   : phandle to IEP module's syscon regmap.
+
+Optional properties for children:
+- local-mac-address	: mac address for the port.
+
+Example (k3-am654 base board, dual-emac):
+========================================
+
+	/* Dual Ethernet application node on PRU-ICSSG2 */
+	pruss2_eth: pruss2_eth {
+		compatible = "ti,am654-icssg-prueth";
+		pinctrl-names = "default";
+		pinctrl-0 = <&icssg2_rgmii_pins_default>;
+		sram = <&msmc_ram>;
+		interrupt-parent = <&main_udmass_inta>;
+
+		prus = <&pru2_0>, <&rtu2_0>, <&pru2_1>, <&rtu2_1>;
+		firmware-name = "ti-pruss/am65x-pru0-prueth-fw.elf",
+				"ti-pruss/am65x-rtu0-prueth-fw.elf",
+				"ti-pruss/am65x-pru1-prueth-fw.elf",
+				"ti-pruss/am65x-rtu1-prueth-fw.elf";
+		mii-g-rt = <&icssg2_mii_g_rt>;
+		dma-coherent;
+		dmas = <&main_udmap &pruss2_eth 0 UDMA_DIR_TX>,	/* egress slice 0 */
+		       <&main_udmap &pruss2_eth 1 UDMA_DIR_TX>, /* egress slice 0 */
+		       <&main_udmap &pruss2_eth 2 UDMA_DIR_TX>, /* egress slice 0 */
+		       <&main_udmap &pruss2_eth 3 UDMA_DIR_TX>, /* egress slice 0 */
+		       <&main_udmap &pruss2_eth 4 UDMA_DIR_TX>, /* egress slice 1 */
+		       <&main_udmap &pruss2_eth 5 UDMA_DIR_TX>, /* egress slice 1 */
+		       <&main_udmap &pruss2_eth 6 UDMA_DIR_TX>, /* egress slice 1 */
+		       <&main_udmap &pruss2_eth 7 UDMA_DIR_TX>, /* egress slice 1 */
+
+		       <&main_udmap &pruss2_eth 0 UDMA_DIR_RX>, /* ingress slice 0 */
+		       <&main_udmap &pruss2_eth 1 UDMA_DIR_RX>, /* ingress slice 1 */
+		       <&main_udmap &pruss2_eth 2 UDMA_DIR_RX>, /* mgmnt rsp slice 0 */
+		       <&main_udmap &pruss2_eth 3 UDMA_DIR_RX>; /* mgmnt rsp slice 1 */
+		dma-names = "tx0-0", "tx0-1", "tx0-2", "tx0-3",
+			    "tx1-0", "tx1-1", "tx1-2", "tx1-3",
+			    "rx0", "rx1",
+			    "rxmgm0", "rxmgm1";
+
+		pruss2_emac0: ethernet-mii0 {
+			phy-handle = <&pruss2_eth0_phy>;
+			phy-mode = "rgmii-id";
+			syscon-rgmii-delay = <&scm_conf 0x4120>;
+			iep = <&icssg2_iep0>;
+			/* Filled in by bootloader */
+			local-mac-address = [00 00 00 00 00 00];
+		};
+
+		pruss2_emac1: ethernet-mii1 {
+			phy-handle = <&pruss2_eth1_phy>;
+			phy-mode = "rgmii-id";
+			syscon-rgmii-delay = <&scm_conf 0x4124>;
+			iep = <&icssg2_iep1>;
+			/* Filled in by bootloader */
+			local-mac-address = [00 00 00 00 00 00];
+		};
+	};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/net/ti-prueth.txt b/Documentation/devicetree/bindings/net/ti-prueth.txt
--- a/Documentation/devicetree/bindings/net/ti-prueth.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/net/ti-prueth.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,89 @@
+Texas Instruments PRUSS Ethernet MAC
+====================================
+
+Required properties:
+- compatible       : Should be one of the following,
+                        "ti,am3359-prueth" for AM335x SoCs
+                        "ti,am4376-prueth" for AM437x SoCs
+                        "ti,am57-prueth" for AM57xx SoCs
+                        "ti,k2g-prueth" for 66AK2G SoCs
+
+- prus             : list of pHandles to the PRU nodes
+- firmware-name	   : list of strings containing firmware path.
+- sram	           : pHandle to OCMC SRAM node
+- interrupt-parent : pHandle to the PRUSS INTC node
+- mii-rt	   : pHandle to MII_RT module's syscon regmap
+- iep		   : pHandle to IEP module's syscon regmap
+
+Must contain children, one for each of the MAC ports.
+Children must be named ethernet-mii0 and ethernet-mii1.
+Either one or both children can be present. If only one
+child is present driver operates in single EMAC mode.
+
+For single mode operation with the 2nd PRU, you still need
+to provide both PRUs and firmware-names but the firmware-name
+for the first PRU can be NULL. See 2nd example.
+
+Required properties for children:
+- phy-handle       : See ethernet.txt file in the same directory.
+- phy-mode         : See ethernet.txt file in the same directory.
+- interrupt-names  : should be "rx" and "tx"
+- interrupts       : should contain an array of PRUSS system event
+                     numbers used as the interrupt sources for Rx
+                     and Tx respectively.
+
+Optional properties for children:
+- local-mac-address	: mac address for the port.
+
+Example (am572x-idk board, dual-emac):
+======================================
+	pruss2_eth {
+		compatible = "ti,am57-prueth";
+		prus = <&pru2_0>, <&pru2_1>;
+		firmware-name = "ti-pruss/am57xx-pru0-prueth-fw.elf",
+				"ti-pruss/am57xx-pru1-prueth-fw.elf";
+		sram = <&ocmcram1>;
+		interrupt-parent = <&pruss2_intc>;
+		mii-rt = <&pruss2_mii_rt>;
+		iep = <&pruss2_iep>;
+
+		pruss2_emac0: ethernet-mii0 {
+			phy-handle = <&pruss2_eth0_phy>;
+			phy-mode = "mii";
+			interrupts = <20>, <22>;
+			interrupt-names = "rx", "tx";
+			/* Filled in by bootloader */
+			local-mac-address = [00 00 00 00 00 00];
+		};
+
+		pruss2_emac1: ethernet-mii1 {
+			phy-handle = <&pruss2_eth1_phy>;
+			phy-mode = "mii";
+			interrupts = <21>, <23>;
+			interrupt-names = "rx", "tx";
+			/* Filled in by bootloader */
+			local-mac-address = [00 00 00 00 00 00];
+		};
+	};
+
+Example (am572x-idk board, single-emac):
+=======================================
+	pruss2_eth {
+		compatible = "ti,am57-prueth";
+		prus = <&pru2_0>, <&pru2_1>;
+		firmware-name = "",
+				"ti-pruss/am57xx-pru1-prueth-fw.elf";
+		sram = <&ocmcram1>;
+		interrupt-parent = <&pruss2_intc>;
+		mii-rt = <&pruss2_mii_rt>;
+		iep = <&pruss2_iep>;
+
+		pruss2_emac1: ethernet-mii1 {
+			phy-handle = <&pruss2_eth1_phy>;
+			phy-mode = "mii";
+			interrupts = <21>, <23>;
+			interrupt-names = "rx", "tx";
+			/* Filled in by bootloader */
+			local-mac-address = [00 00 00 00 00 00];
+		};
+	};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/pci/cdns,cdns-pcie-ep.txt b/Documentation/devicetree/bindings/pci/cdns,cdns-pcie-ep.txt
--- a/Documentation/devicetree/bindings/pci/cdns,cdns-pcie-ep.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/pci/cdns,cdns-pcie-ep.txt	2020-06-19 21:00:31.400227715 -0400
@@ -2,6 +2,7 @@
 
 Required properties:
 - compatible: Should contain "cdns,cdns-pcie-ep" to identify the IP used.
+	      Should contain "ti,j721e-cdns-pcie-ep" for TI platforms.
 - reg: Should contain the controller register base address and AXI interface
   region base address respectively.
 - reg-names: Must be "reg" and "mem" respectively.
@@ -9,6 +10,8 @@
 
 Optional properties:
 - max-functions: Maximum number of functions that can be configured (default 1).
+- max-virtual-functions: Array to specify the maximum number of virtual
+  functions that can be configured for each physical function (default is 0).
 - phys: From PHY bindings: List of Generic PHY phandles. One per lane if more
   than one in the list.  If only one PHY listed it must manage all lanes. 
 - phy-names:  List of names to identify the PHY.
diff -Naur --no-dereference a/Documentation/devicetree/bindings/pci/cdns,cdns-pcie-host.txt b/Documentation/devicetree/bindings/pci/cdns,cdns-pcie-host.txt
--- a/Documentation/devicetree/bindings/pci/cdns,cdns-pcie-host.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/pci/cdns,cdns-pcie-host.txt	2020-06-19 21:00:31.400227715 -0400
@@ -5,6 +5,7 @@
 
 Required properties:
 - compatible: Should contain "cdns,cdns-pcie-host" to identify the IP used.
+	      Should contain "ti,j721e-cdns-pcie-host" for TI platforms.
 - reg: Should contain the controller register base address, PCIe configuration
   window base address, and AXI interface region base address respectively.
 - reg-names: Must be "reg", "cfg" and "mem" respectively.
@@ -27,6 +28,8 @@
 - phys: From PHY bindings: List of Generic PHY phandles. One per lane if more
   than one in the list.  If only one PHY listed it must manage all lanes. 
 - phy-names:  List of names to identify the PHY.
+- iommu-map: As specified in Documentation/devicetree/bindings/pci/pci-iommu.txt
+- reset-gpios: GPIO specifier for the PERST# signal
 
 Example:
 
diff -Naur --no-dereference a/Documentation/devicetree/bindings/pci/designware-pcie.txt b/Documentation/devicetree/bindings/pci/designware-pcie.txt
--- a/Documentation/devicetree/bindings/pci/designware-pcie.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/pci/designware-pcie.txt	2020-06-19 21:00:31.400227715 -0400
@@ -4,8 +4,11 @@
 - compatible:
 	"snps,dw-pcie" for RC mode;
 	"snps,dw-pcie-ep" for EP mode;
-- reg: Should contain the configuration address space.
-- reg-names: Must be "config" for the PCIe configuration space.
+- reg: For designware cores version < 4.80 contains the configuration
+       address space. For designware core version >= 4.80, contains
+       the configuration and ATU address space
+- reg-names: Must be "config" for the PCIe configuration space and "atu" for
+	     the ATU address space.
     (The old way of getting the configuration address space from "ranges"
     is deprecated and should be avoided.)
 - num-lanes: number of lanes to use
diff -Naur --no-dereference a/Documentation/devicetree/bindings/pci/endpoint/pci-epf-bus.txt b/Documentation/devicetree/bindings/pci/endpoint/pci-epf-bus.txt
--- a/Documentation/devicetree/bindings/pci/endpoint/pci-epf-bus.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/pci/endpoint/pci-epf-bus.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,27 @@
+PCI Endpoint Function Bus
+
+This describes the bindings for endpoint function bus to which endpoint
+function devices should be attached.
+
+Required Properties:
+ - compatible: Should be "pci-epf-bus"
+
+One or more subnodes representing PCIe endpoint function device exposed
+to the remote host.
+
+Example:
+Following is an example of NTB device exposed to the remote host.
+
+epf_bus {
+	compatible = "pci-epf-bus";
+
+	ntb {
+		compatible = "pci-epf-ntb";
+		epcs = <&pcie0_ep>, <&pcie1_ep>;
+		epc-names = "primary", "secondary";
+		vendor-id = /bits/ 16 <0x104c>;
+		device-id = /bits/ 16 <0xb00d>;
+		num-mws = <4>;
+		mws-size = <0x100000>, <0x100000>, <0x100000>, <0x100000>;
+	};
+};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/pci/endpoint/pci-epf-ntb.txt b/Documentation/devicetree/bindings/pci/endpoint/pci-epf-ntb.txt
--- a/Documentation/devicetree/bindings/pci/endpoint/pci-epf-ntb.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/pci/endpoint/pci-epf-ntb.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,31 @@
+PCI Endpoint NTB Function Device
+
+This describes the bindings to be used when a NTB device has to be
+exposed to the remote host over PCIe.
+
+Required Properties:
+ - compatible: Should be "pci-epf-ntb"
+ - epcs: As defined in generic pci-epf bindings defined in pci-epf.txt
+ - epc-names: As defined in generic pci-epf bindings defined in pci-epf.txt
+ - vendor-id: As defined in generic pci-epf bindings defined in pci-epf.txt
+ - device-id: As defined in generic pci-epf bindings defined in pci-epf.txt
+ - num-mws: Specify the number of memory windows. Should not be more than 4.
+ - mws-size: List of 'num-mws' entries containing size of each memory window.
+
+Optional Properties:
+ - spad-count: Specify the number of scratchpad registers to be supported
+ - db-count: Specify the number of doorbell interrupts to be supported. Must
+	     not be greater than 32.
+
+Example:
+Following is an example of NTB device exposed to the remote host.
+
+ntb {
+	compatible = "pci-epf-ntb";
+	epcs = <&pcie0_ep>, <&pcie1_ep>;
+	epc-names = "primary", "secondary";
+	vendor-id = /bits/ 16 <0x104c>;
+	device-id = /bits/ 16 <0xb00d>;
+	num-mws = <4>;
+	mws-size = <0x100000>, <0x100000>, <0x100000>, <0x100000>;
+};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/pci/endpoint/pci-epf.txt b/Documentation/devicetree/bindings/pci/endpoint/pci-epf.txt
--- a/Documentation/devicetree/bindings/pci/endpoint/pci-epf.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/pci/endpoint/pci-epf.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,28 @@
+PCI Endpoint Function Device
+
+This describes the generic bindings to be used when a device has to be
+exposed to the remote host over PCIe. The device could be an actual
+peripheral in the platform or a virtual device created by the software.
+
+epcs : phandle to the endpoint controller device
+epc-names : the names of the endpoint controller device corresponding
+	    to the EPCs present in the *epcs* phandle
+vendor-id: used to identify device manufacturer
+device-id: used to identify a particular device
+baseclass-code: used to classify the type of function the device performs
+subclass-code: used to identify more specifically the function of the device
+subsys-vendor-id: used to identify vendor of the add-in card or subsystem
+subsys-id: used to specify an id that is specific to a vendor
+
+Example:
+Following is an example of NTB device exposed to the remote host.
+
+ntb {
+	compatible = "pci-epf-ntb";
+	epcs = <&pcie0_ep>, <&pcie1_ep>;
+	epc-names = "primary", "secondary";
+	vendor-id = /bits/ 16 <0x104c>;
+	device-id = /bits/ 16 <0xb00d>;
+	num-mws = <4>;
+	mws-size = <0x100000>, <0x100000>, <0x100000>, <0x100000>;
+};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/pci/pci-keystone.txt b/Documentation/devicetree/bindings/pci/pci-keystone.txt
--- a/Documentation/devicetree/bindings/pci/pci-keystone.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/pci/pci-keystone.txt	2020-06-19 21:00:31.400227715 -0400
@@ -11,13 +11,24 @@
 
 Required Properties:-
 
-compatibility: "ti,keystone-pcie"
-reg:	index 1 is the base address and length of DW application registers.
-	index 2 is the base address and length of PCI device ID register.
+compatibility: Should be "ti,keystone-pcie" for RC on Keystone2 SoC
+	       Should be "ti,am654-pcie-rc" for RC on AM654x SoC
+reg: Three register ranges as listed in the reg-names property
+reg-names: "dbics" for the DesignWare PCIe registers, "app" for the
+	   TI specific application registers, "config" for the
+	   configuration space address
 
 pcie_msi_intc : Interrupt controller device node for MSI IRQ chip
 	interrupt-cells: should be set to 1
 	interrupts: GIC interrupt lines connected to PCI MSI interrupt lines
+	(required if the compatible is "ti,keystone-pcie")
+msi-map: As specified in Documentation/devicetree/bindings/pci/pci-msi.txt
+	 (required if the compatible is "ti,am654-pcie-rc".
+
+ti,syscon-pcie-id : phandle to the device control module required to set device
+		    id and vendor id.
+ti,syscon-pcie-mode : phandle to the device control module required to configure
+		      PCI in either RC mode or EP mode.
 
  Example:
 	pcie_msi_intc: msi-interrupt-controller {
@@ -35,12 +46,12 @@
 	};
 
 pcie_intc: Interrupt controller device node for Legacy IRQ chip
-	interrupt-cells: should be set to 1
+	interrupt-cells: should be set to 2
 
  Example:
 	pcie_intc: legacy-interrupt-controller {
 		interrupt-controller;
-		#interrupt-cells = <1>;
+		#interrupt-cells = <2>;
 		interrupt-parent = <&gic>;
 		interrupts = <GIC_SPI 26 IRQ_TYPE_EDGE_RISING>,
 			<GIC_SPI 27 IRQ_TYPE_EDGE_RISING>,
@@ -58,3 +69,7 @@
 DesignWare DT Properties not applicable for Keystone PCI
 
 1. pcie_bus clock-names not used.  Instead, a phandle to phys is used.
+
+Optional Properties:
+max-link-speed: As defined in Documentation/devicetree/bindings/pci/pci.txt
+reset-gpios: GPIO spec for the PERST# signal (only host mode)
diff -Naur --no-dereference a/Documentation/devicetree/bindings/pci/ti,j721e-pci.txt b/Documentation/devicetree/bindings/pci/ti,j721e-pci.txt
--- a/Documentation/devicetree/bindings/pci/ti,j721e-pci.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/pci/ti,j721e-pci.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,63 @@
+PCI Controller in J721E:
+Required Properties:
+ - compatible: Should be "ti,j721e-pcie"
+ - reg : Two register ranges as listed in the reg-names property
+ - reg-names : The first entry must be "intd_cfg" used for handling legacy
+     interrupts. The second entry must be "user_cfg" and it has registers
+     for enabling the LTSSM. The third entry must be "vmap" and it has
+     registers for programming the requestor ID to VIRTID mapping and ATYPE.
+ - #address-cells : should be 1 to indicate the child node should use 1 cell
+     for address
+ - #size-cells : should be 1 to indicate the child node should use 1 cell for
+     size
+ - mode : should be <PCI_MODE_RC> for RC mode and should be <PCI_MODE_EP> for
+     EP mode
+
+Optional Properties:
+ - max-link-speed: As defined in Documentation/devicetree/bindings/pci/pci.txt
+     Default is GEN3
+ - num-lanes: number of lanes to use. Default is '1' lane
+It should have two child nodes, one for RC mode (pcie) and the other for EP
+mode (pcie-ep). The child node for RC should follow the binding in
+cdns,cdns-pcie-host.txt. The child node for the EP should follow the binding
+in cdns,cdns-pcie-ep.txt
+
+Example:
+	pcie0: pcie@2900000 {
+		compatible = "ti,j721e-pcie";
+		reg = <0x02900000 0x1000>,
+		      <0x02907000 0x400>;
+		reg-names = "intd_cfg", "user_cfg";
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		pcie0_rc: pcie@d000000 {
+			compatible = "cdns,cdns-pcie-host";
+			reg = <0x0d000000 0x00800000>,
+			      <0x10000000 0x00001000>,
+			      <0x10000000 0x08000000>;
+			reg-names = "reg", "cfg", "mem";
+			device_type = "pci";
+			#address-cells = <3>;
+			#size-cells = <2>;
+			bus-range = <0x0 0xff>;
+			linux,pci-domain = <0>;
+			cdns,max-outbound-regions = <16>;
+			cdns,no-bar-match-nbits = <32>;
+			vendor-id = /bits/ 16 <0x17cd>;
+			device-id = /bits/ 16 <0x0200>;
+			msi-map = <0x0 &gic_its 0x0 0x1000>;
+			ranges = <0x01000000 0x0 0x10001000  0x10001000  0x0 0x0010000>,
+				 <0x02000000 0x0 0x10011000  0x10011000  0x0 0x7fef000>;
+		};
+
+		pcie0_ep: pcie-ep@d000000 {
+			compatible = "cdns,cdns-pcie-ep";
+			reg = <0x0d000000 0x00800000>,
+			      <0x10000000 0x08000000>;
+			reg-names = "reg", "mem";
+			cdns,max-outbound-regions = <16>;
+			max-functions = /bits/ 8 <2>;
+		};
+	};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/pci/ti-pci.txt b/Documentation/devicetree/bindings/pci/ti-pci.txt
--- a/Documentation/devicetree/bindings/pci/ti-pci.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/pci/ti-pci.txt	2020-06-19 21:00:31.400227715 -0400
@@ -1,14 +1,21 @@
 TI PCI Controllers
 
 PCIe DesignWare Controller
- - compatible: Should be "ti,dra7-pcie" for RC
-	       Should be "ti,dra7-pcie-ep" for EP
+ - compatible: Should be "ti,dra7-pcie" for RC (deprecated)
+	       Should be "ti,dra7-pcie-ep" for EP (deprecated)
+	       Should be "ti,dra746-pcie-rc" for dra74x/dra76 in RC mode
+	       Should be "ti,dra746-pcie-ep" for dra74x/dra76 in EP mode
+	       Should be "ti,dra726-pcie-rc" for dra72x in RC mode
+	       Should be "ti,dra726-pcie-ep" for dra72x in EP mode
  - phys : list of PHY specifiers (used by generic PHY framework)
  - phy-names : must be "pcie-phy0", "pcie-phy1", "pcie-phyN".. based on the
 	       number of PHYs as specified in *phys* property.
  - ti,hwmods : Name of the hwmod associated to the pcie, "pcie<X>",
 	       where <X> is the instance number of the pcie from the HW spec.
  - num-lanes as specified in ../designware-pcie.txt
+ - ti,syscon-lane-sel : phandle/offset pair. Phandle to the system control
+			module and the register offset to specify lane
+			selection.
 
 HOST MODE
 =========
diff -Naur --no-dereference a/Documentation/devicetree/bindings/phy/phy-cadence-sierra.txt b/Documentation/devicetree/bindings/phy/phy-cadence-sierra.txt
--- a/Documentation/devicetree/bindings/phy/phy-cadence-sierra.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/phy/phy-cadence-sierra.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,67 @@
+Cadence Sierra PHY
+-----------------------
+
+Required properties:
+- compatible:	cdns,sierra-phy-t0
+- clocks:	Must contain an entry in clock-names.
+		See ../clocks/clock-bindings.txt for details.
+- clock-names:	Must be "phy_clk"
+- resets:	Must contain an entry for each in reset-names.
+		See ../reset/reset.txt for details.
+- reset-names:	Must include "sierra_reset" and "sierra_apb".
+		"sierra_reset" must control the reset line to the PHY.
+		"sierra_apb" must control the reset line to the APB PHY
+		interface.
+- reg:		register range for the PHY.
+- #address-cells: Must be 1
+- #size-cells:	Must be 0
+
+Optional properties:
+- cdns,autoconf:	A boolean property whose presence indicates that the
+			PHY registers will be configured by hardware. If not
+			present, all sub-node optional properties must be
+			provided.
+
+Sub-nodes:
+  Each group of PHY lanes with a single master lane should be represented as
+  a sub-node. Note that the actual configuration of each lane is determined by
+  hardware strapping, and must match the configuration specified here.
+
+Sub-node required properties:
+- #phy-cells:	Generic PHY binding; must be 0.
+- reg:		The master lane number.  This is the lowest numbered lane
+		in the lane group.
+- resets:	Must contain one entry which controls the reset line for the
+		master lane of the sub-node.
+		See ../reset/reset.txt for details.
+
+Sub-node optional properties:
+- cdns,num-lanes:	Number of lanes in this group.  From 1 to 4.  The
+			group is made up of consecutive lanes.
+- cdns,phy-type:	Can be PHY_TYPE_PCIE or PHY_TYPE_USB3, depending on
+			configuration of lanes.
+
+Example:
+	pcie_phy4: pcie-phy@fd240000 {
+		compatible = "cdns,sierra-phy-t0";
+		reg = <0x0 0xfd240000 0x0 0x40000>;
+		resets = <&phyrst 0>, <&phyrst 1>;
+		reset-names = "sierra_reset", "sierra_apb";
+		clocks = <&phyclock>;
+		clock-names = "phy_clk";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		pcie0_phy0: pcie-phy@0 {
+				reg = <0>;
+				resets = <&phyrst 2>;
+				cdns,num-lanes = <2>;
+				#phy-cells = <0>;
+				cdns,phy-type = <PHY_TYPE_PCIE>;
+		};
+		pcie0_phy1: pcie-phy@2 {
+				reg = <2>;
+				resets = <&phyrst 4>;
+				cdns,num-lanes = <1>;
+				#phy-cells = <0>;
+				cdns,phy-type = <PHY_TYPE_PCIE>;
+		};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/phy/phy-cadence-torrent.txt b/Documentation/devicetree/bindings/phy/phy-cadence-torrent.txt
--- a/Documentation/devicetree/bindings/phy/phy-cadence-torrent.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/phy/phy-cadence-torrent.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,33 @@
+Cadence Torrent SD0801 PHY binding for DisplayPort
+===================================================
+
+This binding describes the Cadence SD0801 PHY hardware included with
+the Cadence MHDP DisplayPort controller.
+
+-------------------------------------------------------------------------------
+Required properties (controller (parent) node):
+- compatible	: Should be "cdns,torrent-phy"
+- clocks	: PHY reference clock. Must contain an entry in clock-names.
+		  See ../clocks/clock-bindings.txt for details.
+- clock-names	: Must be "refclk"
+- reg		: Defines the following sets of registers in the parent
+		  mhdp device:
+			- Offset of the DPTX PHY configuration registers
+			- Offset of the SD0801 PHY configuration registers
+- #phy-cells	: from the generic PHY bindings, must be 0.
+
+Optional properties:
+- num_lanes	: Number of DisplayPort lanes to use (1, 2 or 4)
+- max_bit_rate	: Maximum DisplayPort link bit rate to use, in Mbps (2160,
+		  2430, 2700, 3240, 4320, 5400 or 8100)
+-------------------------------------------------------------------------------
+
+Example:
+	dp_phy: phy@f0fb030a00 {
+		compatible = "cdns,torrent-phy";
+		reg = <0xf0 0xfb030a00 0x0 0x00000040>,
+		      <0xf0 0xfb500000 0x0 0x00100000>;
+		num_lanes = <4>;
+		max_bit_rate = <8100>;
+		#phy-cells = <0>;
+	};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/phy/ti,k2-serdes-phy.txt b/Documentation/devicetree/bindings/phy/ti,k2-serdes-phy.txt
--- a/Documentation/devicetree/bindings/phy/ti,k2-serdes-phy.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/phy/ti,k2-serdes-phy.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,273 @@
+TI Keystone SerDes PHY DT Documentation
+---------------------------------------
+Required properties:
+ - compatible: should be one of
+	* "ti,keystone-serdes-gbe"
+	* "ti,keystone-serdes-xgbe"
+	* "ti,keystone-serdes-pcie"
+ - reg:
+	* base address and length of the SerDes register set
+ - num-lanes:
+	* Number of lanes in SerDes.
+
+Optional properties:
+ - syscon-peripheral:
+	* Handle to the subsystem register region of the peripheral
+	  inside which the SerDes exists.  Required for 10gbe.
+ - syscon-link:
+	* Handle to the Link register region of the peripheral inside
+	  which the SerDes exists.  Example: it is the PCSR register
+	  region in the case of 10gbe.  Required for 10gbe.
+ - rx-force-enable:
+	* Include this property if receiver attenuation and boost are
+	  to be configured with specific values defined in rx-force.
+ - link-rate-kbps:
+	* SerDes link rate to be configured, in kbps.
+
+
+For gbe and 10gbe SerDes, it is optional to represent each lane as
+a sub-node, which can be enabled or disabled individually using
+the "status" property.  If a lane is not represented by a node, the
+lane is disabled.
+
+Required properties (lane sub-node):
+ - #phy-cells:
+	* From the generic phy bindings, must be 0;
+ - reg:
+	* lane number
+
+Optional properties (lane sub-node):
+ - control-rate:
+	* Lane control rate
+		0: full rate
+		1: half rate
+		2: quarter rate
+ - rx-start:
+	* Initial lane rx equalizer attenuation and boost configurations.
+	* Must be array of 2 integers.
+ - rx-force:
+	* Forced lane rx equalizer attenuation and boost configurations.
+	* Must be array of 2 integers.
+ - tx-coeff:
+	* Lane c1, c2, cm, attenuation and regulator output voltage
+	  configurations.
+	* Must be array of 5 integers.
+ - loopback:
+	* Include this property to enable loopback at the SerDes
+	  lane level.
+
+Example for Keystone K2E GBE:
+-----------------------------
+
+gbe_serdes0: phy@232a000 {
+	compatible		= "ti,keystone-serdes-gbe";
+	reg			= <0x0232a000 0x2000>;
+	link-rate-kbps		= <1250000>;
+	num-lanes		= <4>;
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	serdes0_lane0: lane@0 {
+		status		= "ok";
+		#phy-cells	= <0>;
+		reg		= <0>;
+		control-rate	= <2>;
+		rx-start	= <7 5>;
+		rx-force	= <1 1>;
+		tx-coeff	= <0 0 0 12 4>;
+	};
+	serdes0_lane1: lane@1 {
+		status		= "ok";
+		#phy-cells	= <0>;
+		reg		= <1>;
+		control-rate	= <2>;
+		rx-start	= <7 5>;
+		rx-force	= <1 1>;
+		tx-coeff	= <0 0 0 12 4>;
+	};
+	serdes0_lane2: lane@2 {
+		status		= "disabled";
+		#phy-cells	= <0>;
+		reg		= <2>;
+		control-rate	= <2>;
+		rx-start	= <7 5>;
+		rx-force	= <1 1>;
+		tx-coeff	= <0 0 0 12 4>;
+	};
+	serdes0_lane3: lane@3 {
+		status		= "disabled";
+		#phy-cells	= <0>;
+		reg		= <3>;
+		control-rate	= <2>;
+		rx-start	= <7 5>;
+		rx-force	= <1 1>;
+		tx-coeff	= <0 0 0 12 4>;
+	};
+};
+
+gbe_serdes1: phy@2324000 {
+	compatible		= "ti,keystone-serdes-gbe";
+	reg			= <0x02324000 0x2000>;
+	link-rate-kbps		= <1250000>;
+	num-lanes		= <4>;
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	serdes1_lane0: lane@0 {
+		status		= "disabled";
+		#phy-cells	= <0>;
+		reg		= <0>;
+		control-rate	= <2>;
+		rx-start	= <7 5>;
+		rx-force	= <1 1>;
+		tx-coeff	= <0 0 0 12 4>;
+	};
+	serdes1_lane1: lane@1 {
+		status		= "disabled";
+		#phy-cells	= <0>;
+		reg		= <1>;
+		control-rate	= <2>;
+		rx-start	= <7 5>;
+		rx-force	= <1 1>;
+		tx-coeff	= <0 0 0 12 4>;
+	};
+	serdes1_lane2: lane@2 {
+		status		= "disabled";
+		#phy-cells	= <0>;
+		reg		= <2>;
+		control-rate	= <2>;
+		rx-start	= <7 5>;
+		rx-force	= <1 1>;
+		tx-coeff	= <0 0 0 12 4>;
+	};
+	serdes1_lane3: lane@3 {
+		status		= "disabled";
+		#phy-cells	= <0>;
+		reg		= <3>;
+		control-rate	= <2>;
+		rx-start	= <7 5>;
+		rx-force	= <1 1>;
+		tx-coeff	= <0 0 0 12 4>;
+	};
+};
+
+netcp: netcp@24000000 {
+	...
+
+	netcp-devices {
+		...
+
+		gbe@200000 {
+			...
+
+			gbe0: interface-0 {
+				phys = <&serdes0_lane0>;
+				...
+			};
+			gbe1: interface-1 {
+				phys = <&serdes0_lane1>;
+				...
+			};
+
+			...
+		};
+
+		...
+	};
+};
+
+Example for Keystone PCIE:
+--------------------------
+
+	pcie0_phy: phy@2320000 {
+		compatible = "ti,keystone-serdes-pcie";
+		reg = <0x02320000 0x4000>;
+		link-rate-kbps = <5000000>;
+		num-lanes = <2>;
+	};
+
+No SerDes phy handle needed in PCIe controller node
+since the SerDes PHY provider driver performs all
+the configurations needed for PCIe.
+
+Example for K2E 10GBE:
+----------------------
+
+Define the syscon regmaps for 10gbe subsystem:
+
+xgbe_subsys: subsys@2f00000 {
+	status		= "ok";
+	compatible	= "syscon";
+	reg		= <0x02f00000 0x100>;
+};
+
+Define the syscon regmaps for 10gbe pcsr:
+
+xgbe_pcsr: pcsr@2f00000 {
+	status		= "ok";
+	compatible	= "syscon";
+	reg		= <0x02f00600 0x100>;
+};
+
+Define the 10gbe SerDes node:
+
+xgbe_serdes: phy@231e000 {
+	status			= "ok";
+	compatible		= "ti,keystone-serdes-xgbe";
+	reg			= <0x0231e000 0x2000>;
+	link-rate-kbps		= <10312500>;
+	num-lanes		= <2>;
+	syscon-peripheral	= <&xgbe_subsys>;
+	syscon-link		= <&xgbe_pcsr>;
+	#address-cells  = <1>;
+	#size-cells     = <0>;
+
+	xserdes_lane0: lane@0 {
+		status		= "ok";
+		#phy-cells	= <0>;
+		reg		= <0>;
+		control-rate	= <0>;
+		rx-start	= <7 5>;
+		rx-force	= <1 1>;
+		tx-coeff	= <2 0 0 12 4>;
+	};
+	xserdes_lane1: lane@1 {
+		status		= "ok";
+		#phy-cells	= <0>;
+		reg		= <1>;
+		control-rate	= <0>;
+		rx-start	= <7 5>;
+		rx-force	= <1 1>;
+		tx-coeff	= <2 0 0 12 4>;
+	};
+};
+
+Then the 10gbe SerDes PHY can be used in the 10gbe switch node:
+
+netcpx: netcpx@2f00000 {
+
+	...
+
+	netcp-devices {
+
+		...
+
+		xgbe@2f00000 {
+
+			...
+
+			xgbe0: interface-0 {
+				phys = <&xserdes_lane0>;
+				...
+			};
+			xgbe1: interface-1 {
+				phys = <&xserdes_lane1>;
+				...
+			};
+
+			...
+		};
+	};
+
+	...
+};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.txt b/Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.txt
--- a/Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,112 @@
+TI J721E WIZ (SERDES Wrapper)
+
+Required properties:
+ - compatible: Should be "ti,j721e-wiz-16g" for Sierra phy wrapper,
+	       or "ti,j721e-wiz-10g" for Torrent phy wrapper.
+ - #address-cells : should be 2 to indicate the child node should use 2 cell
+     for address
+ - #size-cells: should be 2 to indicate the child node should use 2 cell for
+     size
+ - power-domains: As documented by the generic PM domain bindings in
+     Documentation/devicetree/bindings/power/power_domain.txt.
+ - clocks: clock-specifier to represent input to the WIZ required for WIZ
+     module to be functional
+ - num-lanes: Represents thenumber of lanes enabled in the SoC
+     Should be '2' for Sierra wrapper in J721E
+     Should be '4' for Torrent wrapper in J721E
+ - #reset-cells: As documented by the generic reset bindings in
+     Documentation/devicetree/bindings/reset/reset.txt
+     Should be '1'
+ - ranges: Empty ranges property to describe 1:1 translation between parent
+     address space and child address space
+
+Optional properties:
+assigned-clocks and assigned-clock-parents: As documented in the generic
+clock bindings in Documentation/devicetree/bindings/clock/clock-bindings.txt
+
+ - typec-dir-gpios: GPIO to signal Type-C cable orientation for lane swap.
+     If GPIO is active, lane 0 and lane 1 of SERDES will be swapped to
+     achieve the funtionality of an exernal type-C plug flip mux.
+
+ - typec-dir-debounce: Number of milliseconds to wait before sampling
+     typec-dir-gpio. If not specified, the GPIO will be sampled ASAP.
+     Type-C spec states minimum CC pin debounce of 100 ms and maximum
+     of 200 ms.
+
+ - lane<n>-mode, : Integer describing static lane usage for lane n. For
+     Sierra there may be properties for n = 0 and n = 1, for Torrent n = 0,
+     n = 1, n = 2, and n = 3. The constants to indicate the lane usage 
+     are defined in "include/dt-bindings/phy/phy.h". The lane is assumed
+     to be unused if the lane<n>-use property does not exist.
+
+Required subnodes:
+ - Clock Subnode: WIZ node should have '3' subnodes for each of the clock
+     selects it supports. The clock subnodes should have the following names
+	1) pll0_refclk
+	2) pll1_refclk
+	3) refclk_dig
+     Each of these subnodes should clocks, clock-output-names, #clock-cells,
+     assigned-clocks and assigned-clock-parents. All these properties are
+     documented in the generic clock bindings in
+     Documentation/devicetree/bindings/clock/clock-bindings.txt
+ - SERDES Subnode: WIZ node should have '1' subnode for the SERDES
+     *) Sierra SERDES should follow the bindings specified in
+        Documentation/devicetree/bindings/phy/phy-cadence-sierra.txt
+     *) Torrent SERDES should follow the bindings specified in
+        Documentation/devicetree/bindings/phy/phy-cadence-dp.txt
+
+Example: Example shows binding for SERDES_16G (Sierra SERDES with WIZ wrapper)
+serdes_wiz0: wiz@5000000 {
+	compatible = "ti,j721e-wiz-16g";
+	#address-cells = <2>;
+	#size-cells = <2>;
+	power-domains = <&k3_pds 292 TI_SCI_PD_EXCLUSIVE>;
+	clocks = <&k3_clks 292 5>;
+	num-lanes = <2>;
+	#reset-cells = <1>;
+	ranges;
+	lane0-mode = <PHY_TYPE_PCIE>;
+	lane1-mode = <PHY_TYPE_PCIE>;
+
+	pll0_refclk: pll0_refclk {
+		clocks = <&k3_clks 292 11>, <&cmn_refclk>;
+		clock-output-names = "pll0_refclk";
+		#clock-cells = <0>;
+		assigned-clocks = <&pll0_refclk>;
+		assigned-clock-parents = <&k3_clks 292 11>;
+	};
+
+	pll1_refclk: pll1_refclk {
+		clocks = <&k3_clks 292 0>, <&cmn_refclk1>;
+		clock-output-names = "pll1_refclk";
+		#clock-cells = <0>;
+		assigned-clocks = <&pll1_refclk>;
+		assigned-clock-parents = <&k3_clks 292 0>;
+	};
+
+	refclk_dig: refclk_dig {
+		clocks = <&k3_clks 292 11>, <&k3_clks 292 0>,
+			 <&cmn_refclk>, <&cmn_refclk1>;
+		clock-output-names = "refclk_dig";
+		#clock-cells = <0>;
+		assigned-clocks = <&refclk_dig>;
+		assigned-clock-parents = <&k3_clks 292 11>;
+	};
+
+	serdes0: serdes@5000000 {
+		compatible = "cdns,ti,sierra-phy-t0";
+		reg-names = "serdes";
+		reg = <0x00 0x5000000 0x00 0x10000>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		resets = <&serdes_wiz0 0>;
+		reset-names = "sierra_reset";
+		pcie0_phy0: link@0 {
+			reg = <0>;
+			cdns,num-lanes = <2>;
+			#phy-cells = <0>;
+			cdns,phy-type = <PHY_TYPE_PCIE>;
+			resets = <&serdes_wiz0 1>, <&serdes_wiz0 2>;
+		};
+	};
+};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/phy/ti-phy.txt b/Documentation/devicetree/bindings/phy/ti-phy.txt
--- a/Documentation/devicetree/bindings/phy/ti-phy.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/phy/ti-phy.txt	2020-06-19 21:00:31.400227715 -0400
@@ -35,6 +35,7 @@
 	       DRA7x
 	       Should be "ti,dra7x-usb2-phy2" for the 2nd instance of USB2 PHY
 	       in DRA7x
+	       Should be "ti,am654-usb2" for the USB2 PHYs on AM654.
  - reg : Address and length of the register set for the device.
  - #phy-cells: determine the number of cells that should be given in the
    phandle while referencing this phy.
@@ -52,6 +53,9 @@
 - syscon-phy-power : phandle/offset pair. Phandle to the system control
   module and the register offset to power on/off the PHY.
 
+Optional properties:
+ - ti,dis-chg-det-quirk : when set the phy will disable CHG_DET logic
+
 This is usually a subnode of ocp2scp to which it is connected.
 
 usb2phy@4a0ad080 {
@@ -132,3 +136,80 @@
 	syscon-pllreset = <&scm_conf 0x3fc>;
 	#phy-cells = <0>;
 };
+
+
+TI AM654 SERDES
+
+Required properties:
+ - compatible: Should be "ti,phy-am654-serdes"
+ - reg : Address and length of the register set for the device.
+ - reg-names: Should be "serdes" which corresponds to the register space
+	populated in "reg".
+ - #phy-cells: determine the number of cells that should be given in the
+	phandle while referencing this phy. Should be "2". The 1st cell
+	corresponds to the phy type (should be one of the types specified in
+	include/dt-bindings/phy/phy.h) and the 2nd cell should be the serdes
+	lane function.
+	If SERDES0 is referenced 2nd cell should be:
+		0 - USB3
+		1 - PCIe0 Lane0
+		2 - ICSS2 SGMII Lane0
+	If SERDES1 is referenced 2nd cell should be:
+		0 - PCIe1 Lane0
+		1 - PCIe0 Lane1
+		2 - ICSS2 SGMII Lane1
+ - clocks: List of clock-specifiers representing the input to the SERDES.
+	Should have 3 items representing the left input clock, external
+	reference clock and right input clock in that order.
+ - clock-output-names: List of clock names for each of the clock outputs of
+	SERDES. Should have 3 items for CMU reference clock,
+	left output clock and right output clock in that order.
+ - assigned-clocks: As defined in
+	Documentation/devicetree/bindings/clock/clock-bindings.txt
+ - assigned-clock-parents: As defined in
+	Documentation/devicetree/bindings/clock/clock-bindings.txt
+ - #clock-cells: Should be <1> to choose between the 3 output clocks.
+	Defined in Documentation/devicetree/bindings/clock/clock-bindings.txt
+
+   The following macros are defined in dt-bindings/phy/phy-am654-serdes.h
+   for selecting the correct reference clock. This can be used while
+   specifying the clocks created by SERDES.
+	=> AM654_SERDES_CMU_REFCLK
+	=> AM654_SERDES_LO_REFCLK
+	=> AM654_SERDES_RO_REFCLK
+
+ - mux-controls: phandle to the multiplexer
+
+Example:
+
+Example for SERDES0 is given below. It has 3 clock inputs;
+left input reference clock as indicated by <&k3_clks 153 4>, external
+reference clock as indicated by <&k3_clks 153 1> and right input
+reference clock as indicated by <&serdes1 AM654_SERDES_LO_REFCLK>. (The
+right input of SERDES0 is connected to the left output of SERDES1).
+
+SERDES0 registers 3 clock outputs as indicated in clock-output-names. The
+first refers to the CMU reference clock, second refers to the left output
+reference clock and the third refers to the right output reference clock.
+
+The assigned-clocks and assigned-clock-parents is used here to set the
+parent of left input reference clock to MAINHSDIV_CLKOUT4 and parent of
+CMU reference clock to left input reference clock.
+
+serdes0: serdes@900000 {
+	compatible = "ti,phy-am654-serdes";
+	reg = <0x0 0x900000 0x0 0x2000>;
+	reg-names = "serdes";
+	#phy-cells = <2>;
+	power-domains = <&k3_pds 153>;
+	clocks = <&k3_clks 153 4>, <&k3_clks 153 1>,
+			<&serdes1 AM654_SERDES_LO_REFCLK>;
+	clock-output-names = "serdes0_cmu_refclk", "serdes0_lo_refclk",
+				"serdes0_ro_refclk";
+	assigned-clocks = <&k3_clks 153 4>, <&serdes0 AM654_SERDES_CMU_REFCLK>;
+	assigned-clock-parents = <&k3_clks 153 8>, <&k3_clks 153 4>;
+	ti,serdes-clk = <&serdes0_clk>;
+	mux-controls = <&serdes_mux 0>;
+	#clock-cells = <1>;
+	status = "disabled";
+};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/pwm/pwm-pru.txt b/Documentation/devicetree/bindings/pwm/pwm-pru.txt
--- a/Documentation/devicetree/bindings/pwm/pwm-pru.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/pwm/pwm-pru.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,52 @@
+PRUSS PWM controller on TI SoCs
+
+The PRUSS can be programmed as multi IEP-based PWM controllers.
+
+Each PRU PWM node should define the PRU Application node properties as detailed
+in bindings ti,pru-rproc.txt and one or more PWM chip node(s) describing the
+IEP-based PWM controller(s).
+
+Required properties:
+--------------------
+- compatible        : should be "ti,pru-pwm".
+- prus              : phandle to the PRU node used.
+- firmware-name     : firmware for the PRU core.
+
+PWM Chip Node
+==============
+Required properties:
+--------------------
+- compatible        : should be "ti,pru-pwmchip".
+- reg               : index of the pwmchip corresponding to the index of the
+		      IEP, 0-based.
+- #pwm-cells        : should be 2. See pwm.txt in this directory for a
+		      description of the cell formats.
+
+Example (AM654x EVM board):
+---------------------------
+
+	pru_pwm0 {
+		compatible = "ti,pru-pwm";
+		prus = <&pru0_0>;
+		firmware-name = "ti-pruss/am65x-pru0-pwm-fw.elf";
+		assigned-clocks = <&icssg0_iepclk_mux>;
+		assigned-clock-parents = <&k3_clks 62 10>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pwm_chip@0 {
+			compatible = "ti,pru-pwmchip";
+			reg = <0>;
+			#pwm-cells = <2>;
+			pinctrl-0 = <&prupwm0_chip0_pins_default>;
+			pinctrl-names = "default";
+		};
+
+		pwm_chip@1 {
+			compatible = "ti,pru-pwmchip";
+			reg = <1>;
+			#pwm-cells = <2>;
+			pinctrl-0 = <&prupwm0_chip1_pins_default>;
+			pinctrl-names = "default";
+		};
+	};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/pwm/pwm-tiecap.txt b/Documentation/devicetree/bindings/pwm/pwm-tiecap.txt
--- a/Documentation/devicetree/bindings/pwm/pwm-tiecap.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/pwm/pwm-tiecap.txt	2020-06-19 21:00:31.400227715 -0400
@@ -7,6 +7,7 @@
   for da850  - compatible = "ti,da850-ecap", "ti,am3352-ecap", "ti,am33xx-ecap";
   for dra746 - compatible = "ti,dra746-ecap", "ti,am3352-ecap";
   for 66ak2g - compatible = "ti,k2g-ecap", "ti,am3352-ecap";
+  for am654  - compatible = "ti,am654-ecap", "ti,am3352-ecap";
 - #pwm-cells: should be 3. See pwm.txt in this directory for a description of
   the cells format. The PWM channel index ranges from 0 to 4. The only third
   cell flag supported by this binding is PWM_POLARITY_INVERTED.
diff -Naur --no-dereference a/Documentation/devicetree/bindings/pwm/pwm-tiehrpwm.txt b/Documentation/devicetree/bindings/pwm/pwm-tiehrpwm.txt
--- a/Documentation/devicetree/bindings/pwm/pwm-tiehrpwm.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/pwm/pwm-tiehrpwm.txt	2020-06-19 21:00:31.400227715 -0400
@@ -4,6 +4,7 @@
 - compatible: Must be "ti,<soc>-ehrpwm".
   for am33xx  - compatible = "ti,am3352-ehrpwm", "ti,am33xx-ehrpwm";
   for am4372  - compatible = "ti,am4372-ehrpwm", "ti-am3352-ehrpwm", "ti,am33xx-ehrpwm";
+  for am654   - compatible = "ti,am654-ehrpwm", "ti-am3352-ehrpwm";
   for da850   - compatible = "ti,da850-ehrpwm", "ti-am3352-ehrpwm", "ti,am33xx-ehrpwm";
   for dra746 - compatible = "ti,dra746-ehrpwm", "ti-am3352-ehrpwm";
 - #pwm-cells: should be 3. See pwm.txt in this directory for a description of
diff -Naur --no-dereference a/Documentation/devicetree/bindings/remoteproc/ti,k3-dsp-rproc.txt b/Documentation/devicetree/bindings/remoteproc/ti,k3-dsp-rproc.txt
--- a/Documentation/devicetree/bindings/remoteproc/ti,k3-dsp-rproc.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/remoteproc/ti,k3-dsp-rproc.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,179 @@
+TI K3 DSP devices
+=================
+
+The TI K3 family of SoCs usually have one or more TI DSP Core sub-systems that
+are used to offload some of the processor-intensive tasks or algorithms, for
+achieving various system level goals.
+
+These processor sub-systems usually contain additional sub-modules like L1
+and/or L2 caches/SRAMs, an Interrupt Controller, an external memory controller,
+a dedicated local power/sleep controller etc. The DSP processor cores in the
+K3 SoCs is usually either a TMS320C66x CorePac processor or a TMS320C71x CorePac
+processor.
+
+DSP Device Node:
+================
+Each DSP Core sub-system is represented as a single DT node. Each node has a
+number of required or optional properties that enable the OS running on the
+host processor (Arm CorePac) to perform the device management of the remote
+processor and to communicate with the remote processor.
+
+Required properties:
+--------------------
+The following are the mandatory properties:
+
+- compatible:		Should be one of the following,
+			    "ti,j721e-c66-dsp" for C66x DSPs on K3 J721E SoCs
+			    "ti,j721e-c71-dsp" for C71x DSPs on K3 J721E SoCs
+
+- reg:			Should contain an entry for each value in 'reg-names'.
+			Each entry should have the memory region's start address
+			and the size of the region, the representation matching
+			the parent node's '#address-cells' and '#size-cells' values.
+
+- reg-names:		Should contain strings with the following names, each
+			representing a specific internal memory region (if
+			present), and should be defined in this order,
+			     "l2sram", "l1pram", "l1dram"
+			NOTE: C71x DSPs do not have a "l1pram" memory.
+
+- ti,sci:		Should be a phandle to the TI-SCI System Controller node
+
+- ti,sci-dev-id:	Should contain the TI-SCI device id corresponding to the
+			DSP Core. Please refer to the corresponding System
+			Controller documentation for valid values for the DSP
+			cores.
+
+- ti,sci-proc-ids:	Should contain 2 integer values. The first cell should
+			contain the TI-SCI processor id for the DSP core device
+			and the second cell should contain the TI-SCI host id to
+			which the processor control ownership should be
+			transferred to.
+
+- resets:		Should contain the phandle to the reset controller node
+			managing the resets for this device, and a reset
+			specifier. Please refer to the following reset bindings
+			for the reset argument specifier,
+			Documentation/devicetree/bindings/reset/ti,sci-reset.txt
+
+- power-domains:	Should contain a phandle to a PM domain provider node
+			and an args specifier containing the DSP device id
+			value. This property is as per the binding,
+			Documentation/devicetree/bindings/soc/ti/sci-pm-domain.txt
+
+- mboxes:		OMAP Mailbox specifier denoting the sub-mailbox, to be
+			used for communication with the remote processor. The
+			specifier format is as per the bindings,
+			Documentation/devicetree/bindings/mailbox/omap-mailbox.txt
+			This property should match with the sub-mailbox node
+			used in the firmware image.
+
+- memory-region:	phandle to the reserved memory nodes to be associated
+			with the remoteproc device. There should be atleast two
+			reserved memory nodes defined - the first one would be
+			used for dynamic DMA allocations like vrings and vring
+			buffers, and the remaining ones used for the firmware
+			image sections. The reserved memory nodes should be
+			carveout nodes, and should be defined as per the
+			bindings in
+			Documentation/devicetree/bindings/reserved-memory/reserved-memory.txt
+
+Optional properties:
+--------------------
+
+- sram:			pHandle to a reserved on-chip SRAM region. The region
+			should be defined as a child node of the respective
+			SRAM node, and should be defined as per the generic
+			bindings in,
+			Documentation/devicetree/bindings/misc/sram.txt
+
+
+Example:
+---------
+
+1. J721E SoC
+	/* J721E remoteproc alias */
+	aliases {
+		rproc0 = &mcu_r5fss0_core0;
+		rproc1 = &mcu_r5fss0_core1;
+		rproc2 = &main_r5fss0_core0;
+		rproc3 = &main_r5fss0_core1;
+		rproc4 = &main_r5fss1_core0;
+		rproc5 = &main_r5fss1_core1;
+		rproc6 = &c66_0;
+		rproc7 = &c66_1;
+		rproc8 = &c71_0;
+	};
+
+	/* DSP Carveout reserved memory nodes */
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		c66_0_dma_memory_region: c66-dma-memory@a6000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa6000000 0x00 0x100000>;
+			no-map;
+		};
+
+		c66_0_memory_region: c66-memory@a6100000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa6100000 0x00 0xf00000>;
+			no-map;
+		};
+
+		c71_0_dma_memory_region: c71-dma-memory@a8000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa8000000 0x00 0x100000>;
+			no-map;
+		};
+
+		c71_0_memory_region: c71-memory@a8100000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa8100000 0x00 0xf00000>;
+			no-map;
+		};
+	};
+
+	cbass_main: interconnect@100000 {
+		compatible = "simple-bus";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges = <0x00 0x64800000 0x00 0x64800000 0x00 0x00800000>, /* C71_0 */
+			 <0x4d 0x80800000 0x4d 0x80800000 0x00 0x00800000>, /* C66_0 */
+			 <0x4d 0x81800000 0x4d 0x81800000 0x00 0x00800000>; /* C66_1 */
+
+		/* J721E C66_0 DSP node */
+		c66_0: dsp@4d80800000 {
+			compatible = "ti,j721e-c66-dsp";
+			reg = <0x4d 0x80800000 0x00 0x00048000>,
+			      <0x4d 0x80e00000 0x00 0x00008000>,
+			      <0x4d 0x80f00000 0x00 0x00008000>;
+			reg-names = "l2sram", "l1pram", "l1dram";
+			ti,sci = <&dmsc>;
+			ti,sci-dev-id = <142>;
+			ti,sci-proc-ids = <0x03 0xFF>;
+			resets = <&k3_reset 142 1>;
+			power-domains = <&k3_pds 142 TI_SCI_PD_EXCLUSIVE>;
+			memory-region = <&c66_0_dma_memory_region>,
+					<&c66_0_memory_region>;
+			mboxes = <&mailbox0_cluster3 &mbox_c66_0>;
+		};
+
+		/* J721E C71_0 DSP node */
+		c71_0: dsp@64800000 {
+			compatible = "ti,j721e-c71-dsp";
+			reg = <0x00 0x64800000 0x00 0x00080000>,
+			      <0x00 0x64e00000 0x00 0x0000c000>;
+			reg-names = "l2sram", "l1dram";
+			ti,sci = <&dmsc>;
+			ti,sci-dev-id = <15>;
+			ti,sci-proc-ids = <0x30 0xFF>;
+			resets = <&k3_reset 15 1>;
+			power-domains = <&k3_pds 15 TI_SCI_PD_EXCLUSIVE>;
+			memory-region = <&c71_0_dma_memory_region>,
+					<&c71_0_memory_region>;
+			mboxes = <&mailbox0_cluster4 &mbox_c71_0>;
+		};
+	};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/remoteproc/ti,k3-r5f-rproc.txt b/Documentation/devicetree/bindings/remoteproc/ti,k3-r5f-rproc.txt
--- a/Documentation/devicetree/bindings/remoteproc/ti,k3-r5f-rproc.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/remoteproc/ti,k3-r5f-rproc.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,239 @@
+TI K3 R5F processor subsystems
+==============================
+
+The TI K3 family of SoCs usually have one or more dual-core Arm Cortex
+R5F processor subsystems/clusters (R5FSS). The dual core cluster can be
+used either in a LockStep mode providing safety/fault tolerance features
+or in a Split mode providing two individual compute cores for doubling
+the compute capacity. These are used together with other processors
+present on the SoC to achieve various system level goals.
+
+R5F Sub-System Device Node:
+===========================
+Each Dual-Core R5F sub-system is represented as a single DTS node representing
+the cluster, with a pair of child DT nodes representing the individual R5F
+cores. Each node has a number of required or optional properties that enable
+the OS running on the host processor to perform the device management of the
+remote processor and to communicate with the remote processor.
+
+Required properties:
+--------------------
+The following are the mandatory properties:
+
+- compatible:		Should be one of the following,
+			    "ti,am654-r5fss" for R5F clusters/subsystems on
+			                       K3 AM65x SoCs
+			    "ti,j721e-r5fss" for R5F clusters/subsystems on
+			                       K3 J721E SoCs
+- power-domains:	Should contain a phandle to a PM domain provider node
+			and an args specifier containing the R5FSS device id
+			value. This property is as per the binding,
+			Documentation/devicetree/bindings/soc/ti/sci-pm-domain.txt
+- #address-cells:	Should be 1
+- #size-cells:		Should be 1
+- ranges:		Standard ranges definition providing translations for
+			R5F TCM address spaces
+
+Optional properties:
+--------------------
+- lockstep-mode:	Configuration Mode for the Dual R5F cores within the R5F
+			cluster. Should be either a value of 1 (LockStep mode) or
+			0 (Split mode), default is LockStep mode if omitted.
+
+
+R5F Processor Child Nodes:
+==========================
+The R5F Sub-System device node should define two R5F child nodes, each node
+representing a TI instantiation of the Arm Cortex R5F core. There are some
+specific integration differences for the IP like the usage of a Region Address
+Translator (RAT) for translating the larger SoC bus addresses into a 32-bit
+address space for the processor.
+
+Required properties:
+--------------------
+The following are the mandatory properties:
+
+- compatible:		Should be one of the following,
+			    "ti,am654-r5f" for the R5F cores in K3 AM65x SoCs
+			    "ti,j721e-r5f" for the R5F cores in K3 J721E SOCs
+- reg:			Should contain an entry for each value in 'reg-names'.
+			Each entry should have the memory region's start address
+			and the size of the region, the representation matching
+			the parent node's '#address-cells' and '#size-cells' values.
+- reg-names:		Should contain strings with the following names, each
+			representing a specific internal memory region, and
+			should be defined in this order,
+			     "atcm", "btcm"
+- ti,sci:		Should be a phandle to the TI-SCI System Controller node
+- ti,sci-dev-id:	Should contain the TI-SCI device id corresponding to the
+			R5F Core. Please refer to the corresponding System
+			Controller documentation for valid values for the R5F
+			cores.
+- ti,sci-proc-ids:	Should contain 2 integer values. The first cell should
+			contain the TI-SCI processor id for the R5F core device
+			and the second cell should contain the TI-SCI host id to
+			which the processor control ownership should be
+			transferred to.
+- resets:		Should contain the phandle to the reset controller node
+			managing the resets for this device, and a reset
+			specifier. Please refer to the following reset bindings
+			for the reset argument specifier,
+			Documentation/devicetree/bindings/reset/ti,sci-reset.txt
+			    for AM65x and J721E SoCs
+
+The following properties are mandatory for R5F Core0 in both LockStep and Split
+modes, and are mandatory for R5F Core1 _only_ in Split mode. They are unused for
+R5F Core1 in LockStep mode:
+
+- mboxes:		OMAP Mailbox specifier denoting the sub-mailbox, to be
+			used for communication with the remote processor. The
+			specifier format is as per the bindings,
+			Documentation/devicetree/bindings/mailbox/omap-mailbox.txt
+			This property should match with the sub-mailbox node
+			used in the firmware image.
+- memory-region:	phandle to the reserved memory nodes to be associated
+			with the remoteproc device. There should be atleast two
+			reserved memory nodes defined - the first one would be
+			used for dynamic DMA allocations like vrings and vring
+			buffers, and the remaining ones used for the firmware
+			image sections. The reserved memory nodes should be
+			carveout nodes, and should be defined as per the
+			bindings in
+			Documentation/devicetree/bindings/reserved-memory/reserved-memory.txt
+
+Optional properties:
+--------------------
+The following properties are optional properties for each of the R5F cores:
+
+- atcm-enable:		R5F core configuration mode dictating if ATCM should be
+			enabled. Should be either a value of 1 (enabled) or
+			0 (disabled), default is disabled if omitted. R5F view
+			of ATCM dictated by loczrama property.
+- btcm-enable:		R5F core configuration mode dictating if BTCM should be
+			enabled. Should be either a value of 1 (enabled) or
+			0 (disabled), default is enabled if omitted. R5F view
+			of BTCM dictated by loczrama property.
+- loczrama:		R5F core configuration mode dictating which TCM should
+			appear at address 0 (from core's view). Should be either
+			a value of 1 (ATCM at 0x0) or 0 (BTCM at 0x0), default
+			value is 1 if omitted.
+- sram:			pHandle to a reserved on-chip SRAM region. The region
+			should be defined as a child node of the respective
+			SRAM node, and should be defined as per the generic
+			bindings in,
+			Documentation/devicetree/bindings/misc/sram.txt
+
+
+Example:
+--------
+1. AM654 SoC
+	/* AM65x remoteproc alias */
+	aliases {
+		rproc0 = &mcu_r5fss0_core0;
+	};
+
+	/* R5F DDR Carveout reserved memory nodes */
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		mcu_r5fss0_core1_dma_memory_region: r5f-dma-memory@9b000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0x9b000000 0x00 0x100000>;
+			no-map;
+		};
+
+		mcu_r5fss0_core1_memory_region: r5f-memory@9b100000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0x9b100000 0x00 0xf00000>;
+			no-map;
+		};
+
+		mcu_r5fss0_core0_dma_memory_region: r5f-dma-memory@9c000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0x9c000000 0x00 0x100000>;
+			no-map;
+		};
+
+		mcu_r5fss0_core0_memory_region: r5f-memory@9c100000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0x9c100000 0x00 0x700000>;
+			no-map;
+		};
+	};
+
+	cbass_main: interconnect@100000 {
+		compatible = "simple-bus";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges = <0x00 0x41000000 0x00 0x41000000 0x00 0x00020000>,
+			 <0x00 0x41400000 0x00 0x41400000 0x00 0x00020000>,
+			 <0x00 0x41c00000 0x00 0x41c00000 0x00 0x00080000>;
+
+		cbass_mcu: interconnect@28380000 {
+			compatible = "simple-bus";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges = <0x00 0x41000000 0x00 0x41000000 0x00 0x00020000>, /* MCU R5F Core0 */
+				 <0x00 0x41400000 0x00 0x41400000 0x00 0x00020000>, /* MCU R5F Core1 */
+				 <0x00 0x41c00000 0x00 0x41c00000 0x00 0x00080000>; /* MCU SRAM */
+
+			/* MCU domain SRAM node */
+			mcu_ram: mcu-ram@41c00000 {
+				compatible = "mmio-sram";
+				reg = <0x00 0x41c00000 0x00 0x80000>;
+				ranges = <0x0 0x00 0x41c00000 0x80000>;
+				#address-cells = <1>;
+				#size-cells = <1>;
+
+				mcu_r5fss0_core0_sram: r5f-sram@0 {
+					reg = <0x0 0x40000>;
+				};
+			};
+
+			/* AM65x MCU R5FSS node */
+			mcu_r5fss0: r5fss@41000000 {
+				compatible = "ti,am654-r5fss";
+				power-domains = <&k3_pds 129>;
+				lockstep-mode = <1>;
+				#address-cells = <1>;
+				#size-cells = <1>;
+				ranges = <0x41000000 0x00 0x41000000 0x20000>,
+					 <0x41400000 0x00 0x41400000 0x20000>;
+
+				mcu_r5f0: r5f@41000000 {
+					compatible = "ti,am654-r5f";
+					reg = <0x41000000 0x00008000>,
+					      <0x41010000 0x00008000>;
+					reg-names = "atcm", "btcm";
+					ti,sci = <&dmsc>;
+					ti,sci-dev-id = <159>;
+					ti,sci-proc-ids = <0x01 0xFF>;
+					resets = <&k3_reset 159 1>;
+					atcm-enable = <1>;
+					btcm-enable = <1>;
+					loczrama = <1>;
+					mboxes = <&mailbox0 &mbox_mcu_r5fss0_core0>;
+					memory-region = <&mcu_r5fss0_core0_dma_memory_region>,
+							<&mcu_r5fss0_core0_memory_region>;
+					sram = <&mcu_r5fss0_core0_sram>;
+				};
+
+				mcu_r5f1: r5f@41400000 {
+					compatible = "ti,am654-r5f";
+					reg = <0x41400000 0x00008000>,
+					      <0x41410000 0x00008000>;
+					reg-names = "atcm", "btcm";
+					ti,sci = <&dmsc>;
+					ti,sci-dev-id = <245>;
+					ti,sci-proc-ids = <0x02 0xFF>;
+					resets = <&k3_reset 245 1>;
+					atcm-enable = <1>;
+					btcm-enable = <1>;
+					loczrama = <1>;
+					mboxes = <&mailbox1 &mbox_mcu_r5fss0_core1>;
+				};
+			};
+		};
+	};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/remoteproc/ti,omap-remoteproc.txt b/Documentation/devicetree/bindings/remoteproc/ti,omap-remoteproc.txt
--- a/Documentation/devicetree/bindings/remoteproc/ti,omap-remoteproc.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/remoteproc/ti,omap-remoteproc.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,218 @@
+OMAP4+ Remoteproc Devices
+=========================
+
+The OMAP family of SoCs usually have one or more slave processor sub-systems
+that are used to offload some of the processor-intensive tasks, or to manage
+other hardware accelerators, for achieving various system level goals.
+
+The processor cores in the sub-system are usually behind an IOMMU, and may
+contain additional sub-modules like Internal RAM and/or ROMs, L1 and/or L2
+caches, an Interrupt Controller, a Cache Controller etc.
+
+The OMAP SoCs usually have a DSP processor sub-system and/or an IPU processor
+sub-system. The DSP processor sub-system can contain any of the TI's C64x,
+C66x or C67x family of DSP cores as the main execution unit. The IPU processor
+sub-system usually contains either a Dual-Core Cortex-M3 or Dual-Core Cortex-M4
+processors.
+
+Remote Processor Node:
+======================
+Each remote processor sub-system is represented as a single DT node. Each node
+has a number of required or optional properties that enable the OS running on
+the host processor (MPU) to perform the device management of the remote
+processor and to communicate with the remote processor. The various properties
+can be classified as constant or variable. The constant properties are dictated
+by the SoC and does not change from one board to another having the same SoC.
+Examples of constant properties include 'iommus', 'reg'. The variable properties
+are dictated by the system integration aspects such as memory on the board, or
+configuration used within the corresponding firmware image. Examples of variable
+properties include 'mboxes', 'memory-region', 'timers', 'watchdog-timers' etc.
+
+Required properties:
+--------------------
+The following are the mandatory properties:
+
+- compatible:	Should be one of the following,
+		    "ti,omap4-dsp" for DSPs on OMAP4 SoCs
+		    "ti,omap5-dsp" for DSPs on OMAP5 SoCs
+		    "ti,dra7-dsp" for DSPs on DRA7xx/AM57xx SoCs
+		    "ti,omap4-ipu" for IPUs on OMAP4 SoCs
+		    "ti,omap5-ipu" for IPUs on OMAP5 SoCs
+		    "ti,dra7-ipu" for IPUs on DRA7xx/AM57xx SoCs
+
+- ti,hwmods:	Name of the hwmod associated with the remoteproc device
+
+- iommus:	phandles to OMAP IOMMU nodes, that need to be programmed
+		for this remote processor to access any external RAM memory or
+		other peripheral device address spaces. This property usually
+		has only a single phandle. Multiple phandles are used only in
+		cases where the sub-system has different ports for different
+		sub-modules within the processor sub-system (eg: DRA7 DSPs),
+		and need the same programming in both the MMUs.
+
+- mboxes:	OMAP Mailbox specifier denoting the sub-mailbox, to be used for
+		communication with the remote processor. The specifier format is
+		as per the bindings,
+		Documentation/devicetree/bindings/mailbox/omap-mailbox.txt
+		This property should match with the sub-mailbox node used in
+		the firmware image.
+
+Optional properties:
+--------------------
+Some of these properties are mandatory on some SoCs, and some are optional
+depending on the configuration of the firmware image to be executed on the
+remote processor. The conditions are mentioned for each property.
+
+The following are the optional properties:
+- reg:			Address space for any remoteproc memories present on
+			the SoC. Should contain an entry for each value in
+			'reg-names'. These are mandatory for all DSP and IPU
+			processors that have them (OMAP4/OMAP5 DSPs do not have
+			any RAMs)
+
+- reg-names:		Required names for each of the address spaces defined in
+			the 'reg' property. Should contain a string from among
+			the following names, each representing the corresponding
+			internal RAM memory region,
+			   "l2ram" for L2 RAM,
+			   "l1pram" for L1 Program RAM Memory/Cache,
+			   "l1dram" for L1 Data RAM Memory/Cache,
+
+			All devices may not have all the above memories.
+
+- syscon-bootreg:	Should be a pair of the phandle to the System Control
+			Configuration region that contains the boot address
+			register, and the register offset of the boot address
+			register within the System Control module. This property
+			is required for all the DSP instances on OMAP4, OMAP5
+			and DRA7xx SoCs.
+
+- memory-region:	phandle to the reserved memory node to be associated
+			with the remoteproc device. The reserved memory node
+			can be a CMA memory node, and should be defined as
+			per the bindings,
+			Documentation/devicetree/bindings/reserved-memory/reserved-memory.txt
+
+- mbox-names:		Optional names for the OMAP mailbox specifiers mentioned
+			in the 'mboxes' property, one per specifier value
+
+- timers:		One or more phandles to OMAP DMTimer nodes, that serve
+			as System/Tick timers for the OS running on the remote
+			processors. This will usually be a single timer if the
+			processor sub-system is running in SMP mode, or one per
+			core in the processor sub-system. This can also be used
+			to reserve specific timers to be dedicated to the
+			remote processors.
+
+			This property is mandatory on remote processors requiring
+			external tick wakeup, and to support Power Management
+			features. The timers to be used should match with the
+			timers used in the firmware image.
+
+- watchdog-timers:	One or more phandles to OMAP DMTimer nodes, used to
+			serve as Watchdog timers for the processor cores. This
+			will usually be one per executing processor core, even
+			if the processor sub-system is running a SMP OS.
+
+			The timers to be used should match with the watchdog
+			timers used in the firmware image.
+
+- ti,rproc-standby-info: Standby data for the remote processor. This is
+			 mandatory to support Power Management for the OMAP
+			 remoteprocs, and should contain the address containing
+			 the module standby status.
+
+Example:
+--------
+
+1. OMAP4 DSP
+	/* DSP Reserved Memory node */
+	reserved-memory {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		dsp_memory_region: dsp-memory@98000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x98000000 0x800000>;
+			reusable;
+		};
+	};
+
+	/* DSP node */
+	ocp {
+		dsp: dsp {
+			compatible = "ti,omap4-dsp";
+			ti,hwmods = "dsp";
+			syscon-bootreg = <&scm_conf 0x304>;
+			iommus = <&mmu_dsp>;
+			ti,rproc-standby-info = <0x4a004420>;
+			mboxes = <&mailbox &mbox_dsp>;
+			memory-region = <&dsp_memory_region>;
+			timers = <&timer5>;
+			watchdog-timers = <&timer6>;
+		};
+	};
+
+2. OMAP5 IPU
+	/* IPU Reserved Memory node */
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		ipu_memory_region: ipu-memory@95800000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0x95800000 0 0x3800000>;
+			reusable;
+		};
+	};
+
+	/* IPU node */
+	ocp {
+		ipu: ipu@55020000 {
+			compatible = "ti,omap5-ipu";
+			reg = <0x55020000 0x10000>;
+			reg-names = "l2ram";
+			ti,hwmods = "ipu";
+			iommus = <&mmu_ipu>;
+			ti,rproc-standby-info = <0x4a008920>;
+			mboxes = <&mailbox &mbox_ipu>;
+			memory-region = <&ipu_memory_region>;
+			timers = <&timer3>, <&timer4>;
+			watchdog-timers = <&timer9>, <&timer11>;
+		};
+	};
+
+3. DRA7xx/AM57xx DSP
+	/* DSP1 Reserved Memory node */
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		dsp1_memory_region: dsp1-memory@99000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x99000000 0x0 0x4000000>;
+			reusable;
+		};
+	};
+
+	/* DSP1 node */
+	ocp {
+		dsp1: dsp@40800000 {
+			compatible = "ti,dra7-dsp";
+			reg = <0x40800000 0x48000>,
+			      <0x40e00000 0x8000>,
+			      <0x40f00000 0x8000>;
+			reg-names = "l2ram", "l1pram", "l1dram";
+			ti,hwmods = "dsp1";
+			syscon-bootreg = <&scm_conf 0x55c>;
+			iommus = <&mmu0_dsp1>, <&mmu1_dsp1>;
+			ti,rproc-standby-info = <0x4a005420>;
+			mboxes = <&mailbox5 &mbox_dsp1_ipc3x>;
+			memory-region = <&dsp1_memory_region>;
+			timers = <&timer5>;
+			watchdog-timers = <&timer10>;
+		};
+	};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/remoteproc/ti,pru-rproc.txt b/Documentation/devicetree/bindings/remoteproc/ti,pru-rproc.txt
--- a/Documentation/devicetree/bindings/remoteproc/ti,pru-rproc.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/remoteproc/ti,pru-rproc.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,168 @@
+PRU/RTU/Tx_PRU Core on TI SoCs
+==============================
+
+Each PRUSS has dual PRU cores, each represented by a PRU node. Each PRU core
+has a dedicated Instruction RAM, Control and Debug register sets, and use
+the Data RAMs present within the PRUSS for code execution.
+
+The K3 SoCs containing ICSSG v1.0 (eg: AM65x) have two additional PRU cores
+called RTUs with slightly different IP integration. The K3 SoCs containing
+the revised ICSSG v1.1 (eg: J721E) have an extra two auxiliary PRU cores
+called Tx_PRUs that augment the PRUs. Each RTU or Tx_PRU core can be used
+independently like a PRU, or alongside a corresponding PRU core to
+provide/implement auxiliary functionality/support.
+
+Each PRU, RTU or Tx_PRU core node should be defined as a child node of the
+corresponding PRUSS node. Each node can optionally be rendered inactive by
+using the standard DT string property, "status".
+
+Please see the overall PRUSS bindings document for additional details
+including a complete example,
+    Documentation/devicetree/bindings/soc/ti/ti,pruss.txt
+
+Required Properties:
+--------------------
+- compatible     : should be
+                       "ti,am3356-pru" for AM335x family of SoCs
+                       "ti,am4376-pru" for AM437x family of SoCs
+                       "ti,am5728-pru" for AM57xx family of SoCs
+                       "ti,k2g-pru" for 66AK2G family of SoCs
+                       "ti,am654-pru" for PRUs in K3 AM65x family of SoCs
+                       "ti,am654-rtu" for RTUs in K3 AM65x family of SoCs
+                       "ti,j721e-pru" for PRUs in K3 J721E family of SoCs
+                       "ti,j721e-rtu" for RTUs in K3 J721E family of SoCs
+                       "ti,j721e-tx-pru" for Tx_PRUs in K3 J721E family of SoCs
+- reg            : base address and size for each of the 3 sub-module address
+                   spaces as mentioned in reg-names, and in the same order as
+                   the reg-names
+- reg-names      : should contain each of the following 3 names, the binding is
+                   agnostic of the order of these reg-names, preferable to have
+                   the "iram" entry as the first one
+                       "iram" for Instruction RAM,
+                       "control" for the CTRL sub-module registers,
+                       "debug" for the Debug sub-module registers,
+- firmware-name  : should contain the name of the default firmware image file
+                   located on the firmware search path
+
+Optional Properties:
+--------------------
+The virtio based communication between the MPU and a PRU core _requires_
+either the 'mboxes' property, or the set of 'interrupt-parent', 'interrupts'
+and 'interrupt-names' properties to be defined. The latter option is the
+preferred choice. The 'mboxes' property is not applicable for 66AK2G and
+DA850/OMAP-L138 SoCs.
+
+- mboxes           : OMAP Mailbox specifier denoting the sub-mailbox, if using
+                     a mailbox for IPC signalling between host and a PRU core.
+                     The specifier format is as per the bindings,
+                         Documentation/devicetree/bindings/mailbox/omap-mailbox.txt
+                     This property should match with the sub-mailbox node used
+                     in the corresponding firmware image.
+- interrupt-parent : phandle to the PRUSS INTC node. Should be defined if
+                     interrupts property is to be used.
+- interrupts       : array of interrupt specifiers if using PRU system events
+                     for IPC signalling between host and a PRU core. This
+                     property should match with the PRU system event used in
+                     the corresponding firmware image.
+- interrupt-names  : should use one of the following names for each interrupt,
+                     the name should match the corresponding PRU system event
+                     number,
+                         "vring" - for PRU to HOST virtqueue signalling
+                         "kick"  - for HOST to PRU virtqueue signalling
+
+
+Application/Consumer Nodes
+==========================
+A PRU application/consumer/user node typically uses one or more PRU device nodes
+to implement a PRU application/functionality. Each application/client node would
+need a reference to at least a PRU node, and optionally define some properties
+needed for hardware/firmware configuration. The below properties are a list of
+common properties supported by the PRU remoteproc infrastructure.
+
+The application nodes shall define their own bindings like regular platform
+devices, so below are in addition to each node's bindings.
+
+Required Properties:
+--------------------
+- prus                 : phandles to the PRU or RTU nodes used
+
+Optional Properties:
+--------------------
+- firmware-name        : firmwares for the PRU cores, the default firmware
+                         for the core from the PRU node will be used if not
+                         provided. The firmware names should correspond to
+                         the PRU cores listed in the 'prus' property
+- ti,pruss-gp-mux-sel  : array of values for the GP_MUX_SEL under PRUSS_GPCFG
+                         register for a PRU. This selects the internal muxing
+                         scheme for the PRU instance. If not provided, the
+                         default out-of-reset value (0) for the PRU core is
+                         used. Values should correspond to the PRU cores listed
+                         in the 'prus' property
+- ti,pru-interrupt-map : PRU interrupt mappings, containing an array of entries
+                         with each entry consisting of 4 cell-values. First one
+                         is an index towards the "prus" property to identify the
+                         PRU core for the interrupt map, second is the PRU
+                         System Event id, third is the PRU interrupt channel id
+                         and fourth is the PRU host interrupt id. If provided,
+                         this map will supercede any other configuration
+                         provided through firmware
+
+
+Example:
+--------
+
+1.	/* AM33xx PRU-ICSS */
+	pruss_soc_bus: pruss-soc-bus@4a326004 {
+		compatible = "ti,am3356-pruss-soc-bus";
+		...
+
+		pruss: pruss@4a300000 {
+			compatible = "ti,am3356-pruss";
+			reg = <0x4a300000 0x80000>;
+			...
+
+			pruss_mem: memories@4a300000 {
+				reg = <0x4a300000 0x2000>,
+				      <0x4a302000 0x2000>,
+				      <0x4a310000 0x3000>;
+				reg-names = "dram0", "dram1", "shrdram2";
+			};
+
+			pru0: pru@4a334000 {
+				compatible = "ti,am3356-pru";
+				reg = <0x4a334000 0x2000>,
+				      <0x4a322000 0x400>,
+				      <0x4a322400 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "am335x-pru0-fw";
+				interrupt-parent = <&pruss_intc>;
+				interrupts = <16>, <17>;
+				interrupt-names = "vring", "kick";
+			};
+
+			pru1: pru@4a338000 {
+				compatible = "ti,am3356-pru";
+				reg = <0x4a338000 0x2000>,
+				      <0x4a324000 0x400>,
+				      <0x4a324400 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "am335x-pru1-fw";
+				interrupt-parent = <&pruss_intc>;
+				interrupts = <18>, <19>;
+				interrupt-names = "vring", "kick";
+				/* mboxes = <&mailbox &mbox_pru1>; */
+			};
+		};
+	};
+
+3:	/* PRU application node example */
+	app_node: app_node {
+		...
+		prus = <&pru0>, <&pru1>;
+		firmware-name = "pruss-app-fw", "pruss-app-fw-2";
+		ti,pruss-gp-mux-sel = <2>, <1>;
+		/* setup interrupts for prus:
+		   prus[0] => pru1_0: ev=16, chnl=2, host-irq=7,
+		   prus[1] => pru1_1: ev=19, chnl=1, host-irq=3 */
+		ti,pru-interrupt-map = <0 16 2 7 >, <1 19 1 3>;
+	}
diff -Naur --no-dereference a/Documentation/devicetree/bindings/serial/omap_serial.txt b/Documentation/devicetree/bindings/serial/omap_serial.txt
--- a/Documentation/devicetree/bindings/serial/omap_serial.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/serial/omap_serial.txt	2020-06-19 21:00:31.400227715 -0400
@@ -1,6 +1,7 @@
 OMAP UART controller
 
 Required properties:
+- compatible : should be "ti,j721e-uart", "ti,am654-uart" for J721E controllers
 - compatible : should be "ti,am654-uart" for AM654 controllers
 - compatible : should be "ti,omap2-uart" for OMAP2 controllers
 - compatible : should be "ti,omap3-uart" for OMAP3 controllers
@@ -22,6 +23,8 @@
 - dma-names : "rx" for receive channel, "tx" for transmit channel.
 - rs485-rts-delay, rs485-rx-during-tx, linux,rs485-enabled-at-boot-time: see rs485.txt
 - rs485-rts-active-high: drive RTS high when sending (default is low).
+- clocks: phandle to the functional clock as per
+  Documentation/devicetree/bindings/clock/clock-bindings.txt
 
 Example:
 
diff -Naur --no-dereference a/Documentation/devicetree/bindings/serial/pru-suart.txt b/Documentation/devicetree/bindings/serial/pru-suart.txt
--- a/Documentation/devicetree/bindings/serial/pru-suart.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/serial/pru-suart.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,62 @@
+PRU Software UART on TI SoCs
+
+The PRU can be programmed as multi-ports UART.
+
+Each PRU SUART node should define the PRU Application node properties as
+detailed in bindings ti,pru-rproc.txt and one or more port nodes describing
+the emulated UART port(s).
+
+Required properties:
+--------------------
+- compatible           : should be "ti,pru-soft-uart".
+- interrupt-parent     : phandle to the PRUSS INTC node.
+- prus                 : phandle to the PRU node used.
+- firmware-name        : software UART firmware for the PRU core.
+- ti,pru-interrupt-map : PRU interrupt mappings, see ti,pru-rproc.txt for
+                         details. The number of the entries in the array
+                         should match the number of the port nodes.
+
+UART Port Node
+==============
+Required properties:
+--------------------
+- reg               : index of the port, 0-based.
+- interrupts        : interrupt specifier for PRU signaling the host. The
+                      property should match the event defined in
+                      ti,pru-interrupt-map property.
+- ti,pru-suart-pins : PRU pin mapping for UART signals, containing a single
+                      entry of 2 or 4 byte cell-values. The first two values
+		      are for txd and rxd pins, the next two values are
+		      optional which are for cts and rts pins. The value is
+		      the index of the PRU GPIOs defined in PRU R30 and R31.
+
+Example (AM335x BeagleBone Black board):
+---------------------------------------
+
+	pru_suart0 {
+		compatible = "ti,pru-soft-uart";
+		interrupt-parent = <&pruss_intc>;
+		prus = <&pru0>;
+		firmware-name = "ti-pruss/pru0_swuart-fw.out";
+		ti,pru-interrupt-map = <0 21 2 2 >, <0 22 3 3>, <0 23 4 4>;
+		pinctrl-0 = <&pru_uart0_bone_pins>;
+		pinctrl-names = "default";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pru0_port0: port@0 {
+			reg = <0>;
+			interrupts = <21>;
+			ti,pru-suart-pins = /bits/ 8 <0 1>;
+		};
+		pru0_port1: port@1 {
+			reg = <1>;
+			interrupts = <22>;
+			ti,pru-suart-pins = /bits/ 8 <2 3 4 5>;
+		};
+		pru0_port2: port@2 {
+			reg = <2>;
+			interrupts = <23>;
+			ti,pru-suart-pins = /bits/ 8 <8 9 10 11>;
+		};
+	};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/soc/ti/k3-navss-ringacc.txt b/Documentation/devicetree/bindings/soc/ti/k3-navss-ringacc.txt
--- a/Documentation/devicetree/bindings/soc/ti/k3-navss-ringacc.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/soc/ti/k3-navss-ringacc.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,57 @@
+* Texas Instruments K3 NavigatorSS Ring Accelerator
+
+The Ring Accelerator (RA) is a machine which converts read/write accesses
+from/to a constant address into corresponding read/write accesses from/to a
+circular data structure in memory. The RA eliminates the need for each DMA
+controller which needs to access ring elements from having to know the current
+state of the ring (base address, current offset). The DMA controller
+performs a read or write access to a specific address range (which maps to the
+source interface on the RA) and the RA replaces the address for the transaction
+with a new address which corresponds to the head or tail element of the ring
+(head for reads, tail for writes).
+
+The Ring Accelerator is a hardware module that is responsible for accelerating
+management of the packet queues. The K3 SoCs can have more than one RA instances
+
+Required properties:
+- compatible	: Must be "ti,am654-navss-ringacc";
+- reg		: Should contain register location and length of the following
+		  named register regions.
+- reg-names	: should be
+		  "rt" - The RA Ring Real-time Control/Status Registers
+		  "fifos" - The RA Queues Registers
+		  "proxy_gcfg" - The RA Proxy Global Config Registers
+		  "proxy_target" - The RA Proxy Datapath Registers
+- ti,num-rings	: Number of rings supported by RA
+- ti,sci-rm-range-gp-rings : TI-SCI RM subtype for GP ring range
+
+Optional properties:
+ -- ti,dma-ring-reset-quirk : enable ringacc / udma ring state interoperability
+		  issue software w/a
+ -- ti,sci : phandle on TI-SCI compatible System controller node
+ -- ti,sci-dev-id : TI-SCI device id
+
+Example:
+
+ringacc: ringacc@3c000000 {
+	compatible = "ti,am654-navss-ringacc";
+	reg =	<0x0 0x3c000000 0x0 0x400000>,
+		<0x0 0x38000000 0x0 0x400000>,
+		<0x0 0x31120000 0x0 0x100>,
+		<0x0 0x33000000 0x0 0x40000>;
+	reg-names = "rt", "fifos",
+		    "proxy_gcfg", "proxy_target";
+	ti,num-rings = <818>;
+	ti,sci-rm-range-gp-rings = <0x2>; /* GP ring range */
+	ti,dma-ring-reset-quirk;
+	ti,sci = <&dmsc>;
+	ti,sci-dev-id = <187>;
+};
+
+client:
+
+dma_ipx: dma_ipx@<addr> {
+	...
+	ti,ringacc = <&ringacc>;
+	...
+}
diff -Naur --no-dereference a/Documentation/devicetree/bindings/soc/ti/keystone-dsp-mem.txt b/Documentation/devicetree/bindings/soc/ti/keystone-dsp-mem.txt
--- a/Documentation/devicetree/bindings/soc/ti/keystone-dsp-mem.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/soc/ti/keystone-dsp-mem.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,73 @@
+TI Keystone DSP Memory Mapping
+==============================
+
+Binding status: Unstable - ABI compatibility may be broken in the future
+
+The Keystone DSP Memory Mapping binding defines the memory regions that
+are reserved to be used with DSPs. These regions can be mapped into
+userspace for providing direct user-mode access to these regions for the
+purposes of shared memory communication with the DSP remote processor
+devices on the SoC. These memory regions can also be used for supporting
+user-space based loading of the DSP remoteproc devices.
+
+The memory regions can either be from regular DDR memory or from the
+On-chip RAM, and there can be one or more regions of each memory type.
+
+DDR Memory usage (Optional):
+----------------------------
+One or more memory regions from DDR memory can be reserved specifically
+to be used by the DSPs on the SoC. Each region should be defined as a
+child node of the reserved-memory node with the following required
+properties:
+
+- compatible : Should be "ti,keystone-dsp-mem-pool"
+- reg        : Should contain the region's start address and size following
+               the #address-cells and #size-cells defined in the parent
+               reserved-memory node.
+- no-map     : Should be defined to remove this region from kernel
+
+Please see Documentation/devicetree/bindings/reserved-memory/reserved-memory.txt
+for more details on reserved-memory node.
+
+SRAM usage (Optional):
+----------------------
+The On-chip Multicore Shared Memory (MSM) RAM can also be exposed to
+userspace by defining specific child nodes under the corresponding parent
+SRAM node. The generic SRAM binding is as per the binding document
+Documentation/devicetree/bindings/misc/sram.txt. Following properties
+should be used in each corresponding child node for the userspace mapping
+usecase:
+
+- compatible : Should be "ti,keystone-dsp-msm-ram"
+- reg        : Should contain a pair of values for the address and size
+               of the region, following the parent-child ranges convention.
+
+Example:
+--------
+	/* 66AK2H EVM */
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		dsp_common_mpm_memory: dsp-common-mpm-memory@820000000 {
+			compatible = "ti,keystone-dsp-mem-pool";
+			reg = <0x00000008 0x20000000 0x00000000 0x20000000>;
+			no-map;
+		};
+	};
+
+	soc {
+		msm_ram: msmram@c000000 {
+			compatible = "mmio-sram";
+			reg = <0x0c000000 0x600000>;
+			ranges = <0x0 0x0c000000 0x600000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			sram-mpm@0 {
+				compatible = "ti,keystone-dsp-msm-ram";
+				reg = <0x0 0x80000>;
+			};
+		};
+	};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/soc/ti/sci-pm-domain.txt b/Documentation/devicetree/bindings/soc/ti/sci-pm-domain.txt
--- a/Documentation/devicetree/bindings/soc/ti/sci-pm-domain.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/soc/ti/sci-pm-domain.txt	2020-06-19 21:00:31.400227715 -0400
@@ -19,8 +19,15 @@
 Required Properties:
 --------------------
 - compatible: should be "ti,sci-pm-domain"
-- #power-domain-cells: Must be 1 so that an id can be provided in each
-		       device node.
+- #power-domain-cells: Can be one of the following:
+			1: Containing the device id of each node
+			2: First entry should be device id
+			   Second entry should be one of the floowing:
+			   TI_SCI_PD_EXCLUSIVE: To allow device to be
+						exclusively controlled by
+						the requesting hosts.
+			   TI_SCI_PD_SHARED: To allow device to be shared
+					     by multiple hosts.
 
 Example (K2G):
 -------------
diff -Naur --no-dereference a/Documentation/devicetree/bindings/soc/ti/ti,pruss.txt b/Documentation/devicetree/bindings/soc/ti/ti,pruss.txt
--- a/Documentation/devicetree/bindings/soc/ti/ti,pruss.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/soc/ti/ti,pruss.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,374 @@
+PRU-ICSS on TI SoCs
+===================
+
+Binding status: Unstable - Subject to changes for undocumented sub-modules
+
+The Programmable Real-Time Unit and Industrial Communication Subsystem
+(PRU-ICSS) is present on various TI SoCs such as AM335x, AM437x, Keystone
+66AK2G, OMAP-L138/DA850 etc. A PRUSS consists of dual 32-bit RISC cores
+(Programmable Real-Time Units, or PRUs), shared RAM, data and instruction
+RAMs, some internal peripheral modules to facilitate industrial communication,
+and an interrupt controller.
+
+The programmable nature of the PRUs provide flexibility to implement custom
+peripheral interfaces, fast real-time responses, or specialized data handling.
+The common peripheral modules include the following,
+  - an Ethernet MII_RT module with two MII ports
+  - an MDIO port to control external Ethernet PHYs
+  - an Industrial Ethernet Peripheral (IEP) to manage/generate Industrial
+    Ethernet functions
+  - an Enhanced Capture Module (eCAP)
+  - an Industrial Ethernet Timer with 7/9 capture and 16 compare events
+  - a 16550-compatible UART to support PROFIBUS
+  - Enhanced GPIO with async capture and serial support
+
+A PRU-ICSS subsystem can have up to three shared data memories. A PRU core
+acts on a primary Data RAM (there are usually 2 Data RAMs) at its address
+0x0, but also has access to a secondary Data RAM (primary to the other PRU
+core) at its address 0x2000. A shared Data RAM, if present, can be accessed
+by both the PRU cores. The Interrupt Controller (INTC) and a CFG module are
+common to both the PRU cores. Each PRU core also has a private instruction RAM,
+and specific register spaces for Control and Debug functionalities.
+
+Various sub-modules within a PRU-ICSS subsystem are represented as individual
+nodes and are defined using a parent-child hierarchy depending on their
+integration within the IP and the SoC. These nodes are described in the
+following sections.
+
+
+PRU-ICSS SoC Bus Node
+======================
+This node represents the integration of the PRU-ICSS IP into a SoC, and is
+required for all SoCs. The PRU-ICSS parent nodes need to be defined as child
+nodes of this node.
+
+Required Properties:
+--------------------
+- compatible     : should be one of,
+                       "ti,am3356-pruss-soc-bus" for AM335x family of SoCs
+                       "ti,am4376-pruss-soc-bus" for AM437x family of SoCs
+                       "ti,am5728-pruss-soc-bus" for AM57xx family of SoCs
+                       "ti,k2g-pruss-soc-bus" for 66AK2G family of SoCs
+                       "ti,am654-icssg-soc-bus" for K3 AM65x family of SoCs
+                       "ti,j721e-icssg-soc-bus" for K3 J721E family of SoCs
+- reg            : address and size of the PRUSS CFG sub-module register
+                   dictating the interconnect configuration
+- #address-cells : should be 1
+- #size-cells    : should be 1
+- ranges         : standard ranges definition
+
+SoC-specific Required properties:
+---------------------------------
+
+The following are mandatory properties for all OMAP-architecture based SoCs:
+- ti,hwmods      : name of the hwmod associated with the PRUSS instance
+
+The following properties are _required_ only for Keystone 2 66AK2G SoCs,
+K3 AM65x SoCs and K3 J721E SoCs only:
+- power-domains  : Should contain a phandle to a PM domain provider node and an
+                   args specifier containing the PRUSS SCI device id value. This
+                   property is as per the binding,
+                       Documentation/devicetree/bindings/soc/ti/sci-pm-domain.txt
+
+
+PRU-ICSS Node
+==============
+Each PRU-ICSS subsystem instance is represented as a child node of
+the PRUSS SoC bus node, with the individual PRU processor cores, a
+memories node, an INTC node and an MDIO node represented as child nodes
+within this parent PRUSS node.
+
+Required Properties:
+--------------------
+- compatible     : should be one of,
+                       "ti,am3356-pruss" for AM335x family of SoCs
+                       "ti,am4376-pruss" for AM437x family of SoCs
+                       "ti,am5728-pruss" for AM57xx family of SoCs
+                       "ti,k2g-pruss" for 66AK2G family of SoCs
+                       "ti,am654-icssg" for K3 AM65x family of SoCs
+                       "ti,j721e-icssg" for K3 J721E family of SoCs
+- reg            : base address and size of the entire PRU-ICSS space
+- interrupts     : all the interrupts generated towards the main host
+                   processor in the SoC. The format depends on the
+                   interrupt specifier for the particular SoC's MPU
+                   parent interrupt controller
+- interrupt-names: should use one of the following names for each interrupt,
+                   the name should match the corresponding host interrupt
+                   number,
+                       "host2", "host3", "host4", "host5", "host6",
+                       "host7", "host8" or "host9"
+                   NOTE: AM437x and 66AK2G SoCs do not have "host7" interrupt
+                         connected to MPU
+- #address-cells : should be 1
+- #size-cells    : should be 1
+- ranges         : no specific range translations required, child nodes have the
+                   same address view as the parent, so should be mentioned without
+                   any value for the property
+
+Optional Properties: (Applies only to "ti,am654-icssg" and "ti,j721e-icssg")
+--------------------
+- ti,psil-base	    : PSIL thread start number
+- ti,psil-config<n> :   PSIL configuration for each thread number
+
+PRU-ICSS Memories Node
+=======================
+The various Data RAMs within a PRU-ICSS are represented as a single
+node with the name 'memories'.
+
+Required Properties:
+--------------------
+- reg            : base address and size for each of the Data RAMs as
+                   mentioned in reg-names, and in the same order as the
+                   reg-names
+- reg-names      : should contain a string(s) from among the following names,
+                   each representing a specific Data RAM region. A PRU-ICSS may
+                   not have all of the Data RAMs. The binding is agnostic
+                   of the order of these reg-names
+                       "dram0" for Data RAM0,
+                       "dram1" for Data RAM1,
+                       "shrdram2" for Shared Data RAM,
+
+
+PRU-ICSS CFG, IEP, MII_RT Nodes
+================================
+The individual sub-modules CFG, IEP and MII_RT are represented as a syscon
+node each for now with specific node names as below:
+                  "cfg" for CFG sub-module,
+                  "iep" for IEP sub-module,
+                  "mii-rt" for MII-RT sub-module,
+
+The following sub-modules are new and present only on ICSSG IP instances and
+are applicable only for K3 SoCs. They are represented as follows:
+	MII_G_RT sub-module: represent as a syscon node with name "mii-g-rt"
+
+See Documentation/devicetree/bindings/mfd/syscon.txt for generic syscon
+binding details.
+
+Required Properties:
+--------------------
+The following are required properties for IEP nodes
+- clocks        : phandle to IEP core clock.
+
+
+IEPCLK MUX Clock node
+=====================
+The IEP module can get its clock from 2 sources i.e. IEP_CLK input to
+the PRU-ICSS module or the Bus clock (ICLK). This node models this clock
+mux and should have the name "iepclk_mux".
+
+Required Properties:
+--------------------
+-#clock-cells   : should be 0
+-clocks         : phandles to the 2 input clocks to the IEPCLK MUX.
+
+CORECLK MUX Clock node
+======================
+This is applicable only for ICSSG (K3 SoCs). The ICSSG modules core clock
+can be set to one of the 2 sources i.e. CORE_CLK input to the ICSSG module
+or VBUS clock. This node models this clock mux and should have the name
+"coreclk_mux".
+
+Required Properties:
+--------------------
+-#clock-cells   : should be 0
+-clocks         : phandles to the 2 input clocks to the CORECLK MUX.
+
+
+PRUSS INTC Node
+================
+Each PRUSS has a single interrupt controller instance that is common to all
+the PRU cores. This should be represented as an interrupt-controller node. The
+bindings for this shall be defined in the file,
+    Documentation/devicetree/bindings/interrupt-controller/ti,pruss-intc.txt
+
+
+PRU Node
+=========
+Each PRUSS has dual PRU cores, each represented as a remoteproc device through
+a PRU child node each. Each node can optionally be rendered inactive by using
+the standard DT string property, "status". The ICSSG IP present on K3 SoCs have
+additional auxiliary PRU cores with slightly different IP integration. The
+bindings for this shall be defined in the file,
+    Documentation/devicetree/bindings/remoteproc/ti,pru-rproc.txt
+
+
+MDIO Node
+==========
+Each PRUSS has an MDIO module that can be used to control external PHYs. The
+MDIO module used within the PRU-ICSS is an instance of the MDIO Controller
+used in TI Davinci SoCs. Please refer to the corresponding binding document,
+Documentation/devicetree/bindings/net/davinci-mdio.txt for details.
+
+
+Example:
+========
+1.	/* AM33xx PRU-ICSS */
+	pruss_soc_bus: pruss-soc-bus@4a326004 {
+		compatible = "ti,am3356-pruss-soc-bus";
+		ti,hwmods = "pruss";
+		reg = <0x4a326004 0x4>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		pruss: pruss@4a300000 {
+			compatible = "ti,am3356-pruss";
+			reg = <0x4a300000 0x80000>;
+			interrupts = <20 21 22 23 24 25 26 27>;
+			interrupt-names = "host2", "host3", "host4",
+					  "host5", "host6", "host7",
+					  "host8", "host9";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+
+			pruss_mem: memories@4a300000 {
+				reg = <0x4a300000 0x2000>,
+				      <0x4a302000 0x2000>,
+				      <0x4a310000 0x3000>;
+				reg-names = "dram0", "dram1", "shrdram2";
+			};
+
+			pruss_cfg: cfg@4a326000 {
+				compatible = "syscon";
+				reg = <0x4a326000 0x2000>;
+			};
+
+			pruss_iep: iep@4a32e000 {
+				compatible = "syscon";
+				reg = <0x4a32e000 0x31c>;
+			};
+
+			pruss_mii_rt: mii-rt@4a332000 {
+				compatible = "syscon";
+				reg = <0x4a332000 0x58>;
+			};
+
+			pruss_intc: interrupt-controller@4a320000 {
+				compatible = "ti,am3356-pruss-intc";
+				reg = <0x4a320000 0x2000>;
+				interrupt-controller;
+				#interrupt-cells = <1>;
+			};
+
+			pru0: pru@4a334000 {
+				compatible = "ti,am3356-pru";
+				reg = <0x4a334000 0x2000>,
+				      <0x4a322000 0x400>,
+				      <0x4a322400 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "am335x-pru0-fw";
+			};
+
+			pru1: pru@4a338000 {
+				compatible = "ti,am3356-pru";
+				reg = <0x4a338000 0x2000>,
+				      <0x4a324000 0x400>,
+				      <0x4a324400 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "am335x-pru1-fw";
+			};
+
+			pruss_mdio: mdio@4a332400 {
+				compatible = "ti,davinci_mdio";
+				reg = <0x4a332400 0x90>;
+				clocks = <&dpll_core_m4_ck>;
+				clock-names = "fck";
+				bus_freq = <1000000>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+			};
+		};
+	};
+
+2.	/* AM43xx PRU-ICSS with PRUSS1 node (PRUSS0 not shown completely) */
+	pruss_soc_bus: pruss-soc-bus@54426004 {
+		compatible = "ti,am4376-pruss-soc-bus";
+		reg = <0x54426004 0x4>;
+		ti,hwmods = "pruss";
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		pruss1: pruss@54400000 {
+			compatible = "ti,am4376-pruss";
+			reg = <0x54400000 0x40000>;
+			interrupts = <GIC_SPI 20 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 21 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 23 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 24 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 27 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "host2", "host3", "host4",
+					  "host5", "host6", "host8",
+					  "host9";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+
+			pruss1_mem: memories@54400000 {
+				reg = <0x54400000 0x2000>,
+				      <0x54402000 0x2000>,
+				      <0x54410000 0x8000>;
+				reg-names = "dram0", "dram1", "shrdram2";
+			};
+
+			pruss1_cfg: cfg@54426000 {
+				compatible = "syscon";
+				reg = <0x54426000 0x2000>;
+			};
+
+			pruss1_iep: iep@5442e000 {
+				compatible = "syscon";
+				reg = <0x5442e000 0x31c>;
+			};
+
+			pruss1_mii_rt: mii-rt@54432000 {
+				compatible = "syscon";
+				reg = <0x54432000 0x58>;
+			};
+
+			pruss1_intc: interrupt-controller@54420000 {
+				compatible = "ti,am4376-pruss-intc";
+				reg = <0x54420000 0x2000>;
+				interrupt-controller;
+				#interrupt-cells = <1>;
+			};
+
+			pru1_0: pru@54434000 {
+				compatible = "ti,am4376-pru";
+				reg = <0x54434000 0x3000>,
+				      <0x54422000 0x400>,
+				      <0x54422400 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "am437x-pru1_0-fw";
+			};
+
+			pru1_1: pru@54438000 {
+				compatible = "ti,am4376-pru";
+				reg = <0x54438000 0x3000>,
+				      <0x54424000 0x400>,
+				      <0x54424400 0x100>;
+				reg-names = "iram", "control", "debug";
+				firmware-name = "am437x-pru1_1-fw";
+			};
+
+			pruss1_mdio: mdio@54432400 {
+				compatible = "ti,davinci_mdio";
+				reg = <0x54432400 0x90>;
+				clocks = <&dpll_core_m4_ck>;
+				clock-names = "fck";
+				bus_freq = <1000000>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+				status = "disabled";
+			};
+		};
+
+		pruss0: pruss@54440000 {
+			compatible = "ti,am4376-pruss";
+			reg = <0x54440000 0x40000>;
+			...
+		};
+	};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/soc/ti/wkup_m3_ipc.txt b/Documentation/devicetree/bindings/soc/ti/wkup_m3_ipc.txt
--- a/Documentation/devicetree/bindings/soc/ti/wkup_m3_ipc.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/soc/ti/wkup_m3_ipc.txt	2020-06-19 21:00:31.400227715 -0400
@@ -55,3 +55,96 @@
 			...
 		};
 	};
+
+Support for I2C PMIC Voltage Scaling
+--------------------
+It is possible to pass the name of a binary file to laod to the CM3 firmware
+in order to provide I2C sequences for the CM3 to send out to the PMIC during
+low power mode entry.
+
+Optional properties:
+--------------------
+- scale-data-fw:	Name of the firmware binary in /lib/firmware to copy to m3
+			aux data.
+
+Support for VTT Toggle
+==================================
+In order to enable the support for VTT toggle during Suspend/Resume
+sequence needed by some boards (like AM335x EVM-SK & AM437x GP EVM),
+the below DT properties are required. It is possible to toggle VTT
+using one of two methods depending on the SoC being used, either
+GPIO0 toggle (AM335x and AM437x), or any GPIO with DS_PAD_CONFIG
+bits in the control module (AM437x only).
+
+VTT Toggle using GPIO0
+----------------------------------
+Supported by: AM335x and AM437x
+Used on: AM335x EVM-SK
+
+Optional properties:
+- ti,needs-vtt-toggle:	Indicates that the boards requires VTT toggling
+			during suspend/resume.
+- ti,vtt-gpio-pin:	Specifies the GPIO0 pin used for VTT toggle.
+
+Important Note:
+- Here it is assumed that VTT Toggle will be done using a pin on GPIO-0 Instance.
+  It will not work on any other GPIO using the above properties, regardless of
+  which part is being used.
+
+Example:
+	wkup_m3_ipc: wkup_m3_ipc@1324 {
+		compatible = "ti,am3352-wkup-m3-ipc";
+		...
+		...
+		ti,needs-vtt-toggle;
+		ti,vtt-gpio-pin = <7>;
+		...
+	};
+
+VTT Toggle using any GPIO
+----------------------------------
+Supported by: AM437x ONLY
+Used on: AM437x GP EVM
+
+Many of the pins on AM437x have the ability to configure both normal and
+sleep states. Because of this it is possible to use any pin with a
+corresponding CTRL_CONF_* register in the control module and the
+DS_PAD_CONFIG bits to toggle the VTT regulator enable pin. The DS state of
+the pin must be configured such that the pin disables the VTT regulator. The
+normal state of the pin must be configured such that the VTT regulator is
+enabled by the state alone. This is because the VTT regulator must be enabled
+before context is restored to the controlling GPIO.
+
+Optional properties:
+- ti,set-io-isolation:	Indicates that the IO's should be placed into
+			isolation and the DS_PAD_CONFIG values should be
+			used during suspend.
+
+Example:
+
+On the AM437x GP EVM, the VTT enable line must be held low to disable VTT
+regulator and held high to enable, so the following pinctrl entry is used.
+The DS pull is enabled which uses a pull down by default and DS off mode is
+used which outputs a low by default. For the normal state, a pull up is
+specified so that the VTT enable line gets pulled high immediately after
+the DS states are removed upon exit from DeepSleep0.
+
+	&am43xx_pinmux {
+		pinctrl-names = "default";
+		pinctrl-0 = <&ddr3_vtt_toggle_default>;
+
+		ddr3_vtt_toggle_default: ddr_vtt_toggle_default {
+		pinctrl-single,pins = <
+			0x25C (DS0_PULL_UP_DOWN_EN | PIN_OUTPUT_PULLUP |
+			       DS0_FORCE_OFF_MODE | MUX_MODE7)>;
+		};
+		...
+	};
+
+	wkup_m3_ipc: wkup_m3_ipc@1324 {
+		compatible = "ti,am4372-wkup-m3-ipc";
+		...
+		...
+		ti,set-io-isolation;
+		...
+	};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/sound/davinci-mcasp-audio.txt b/Documentation/devicetree/bindings/sound/davinci-mcasp-audio.txt
--- a/Documentation/devicetree/bindings/sound/davinci-mcasp-audio.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/sound/davinci-mcasp-audio.txt	2020-06-19 21:00:31.400227715 -0400
@@ -30,6 +30,11 @@
 - ti,hwmods : Must be "mcasp<n>", n is controller instance starting 0
 - tx-num-evt : FIFO levels.
 - rx-num-evt : FIFO levels.
+- dismod : Specify the drive on TX pin during inactive slots
+	0 : 3-state
+	2 : logic low
+	3 : logic high
+	Defaults to 'logic low' when the property is not present
 - sram-size-playback : size of sram to be allocated during playback
 - sram-size-capture  : size of sram to be allocated during capture
 - interrupts : Interrupt numbers for McASP
@@ -39,6 +44,26 @@
   		 please refer to pinctrl-bindings.txt
 - fck_parent : Should contain a valid clock name which will be used as parent
 	       for the McASP fck
+- auxclk-fs-ratio: When McASP is bus master indicates the ratio between AUCLK
+		   and FS rate if applicable:
+		   AUCLK rate = auxclk-fs-ratio * FS rate
+
+Optional GPIO support:
+If any McASP pin need to be used as GPIO then the McASP node must have:
+...
+  gpio-controller
+  #gpio-cells = <2>;
+...
+
+When requesting a GPIO, the first parameter is the PIN index in McASP_P*
+registers.
+For example to request the AXR2 pin of mcasp8:
+function-gpios = <&mcasp8 2 0>;
+
+Or to request the ACLKR pin of mcasp8:
+function-gpios = <&mcasp8 29 0>;
+
+For generic gpio information, please refer to bindings/gpio/gpio.txt
 
 Example:
 
diff -Naur --no-dereference a/Documentation/devicetree/bindings/sound/omap-mcpdm.txt b/Documentation/devicetree/bindings/sound/omap-mcpdm.txt
--- a/Documentation/devicetree/bindings/sound/omap-mcpdm.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/sound/omap-mcpdm.txt	2020-06-19 21:00:31.400227715 -0400
@@ -7,6 +7,8 @@
        <L3 interconnect address, size>;
 - interrupts: Interrupt number for McPDM
 - ti,hwmods: Name of the hwmod associated to the McPDM
+- clocks:  phandle for the pdmclk provider, likely <&twl6040>
+- clock-names: Must be "pdmclk"
 
 Example:
 
@@ -18,3 +20,11 @@
 	interrupt-parent = <&gic>;
 	ti,hwmods = "mcpdm";
 };
+
+In board DTS file the pdmclk needs to be added:
+
+&mcpdm {
+	clocks = <&twl6040>;
+	clock-names = "pdmclk";
+	status = "okay";
+};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/sound/simple-card.txt b/Documentation/devicetree/bindings/sound/simple-card.txt
--- a/Documentation/devicetree/bindings/sound/simple-card.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/sound/simple-card.txt	2020-06-19 21:00:31.400227715 -0400
@@ -94,6 +94,8 @@
 - system-clock-direction-out		: specifies clock direction as 'out' on
 					  initialization. It is useful for some aCPUs with
 					  fixed clocks.
+- system-clock-id			: Numberic ID of the system clock to
+					  select within the dai, default is 0.
 
 Example 1 - single DAI link:
 
diff -Naur --no-dereference a/Documentation/devicetree/bindings/sound/ti,j721e-cpb-audio.txt b/Documentation/devicetree/bindings/sound/ti,j721e-cpb-audio.txt
--- a/Documentation/devicetree/bindings/sound/ti,j721e-cpb-audio.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/sound/ti,j721e-cpb-audio.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,53 @@
+* Audio support for j721e Common Processor Board
+
+The audio support on the board is using pcm3168a codec connected to McASP10
+serializers in parallel setup.
+The pcm3168a SCKI clock is coming via the j721e AUDIO_REFCLK2 pin.
+In order to support 48KHz and 44.1KHz family of sampling rates the parent clock
+for AUDIO_REFCLK2 needs to be changed between PLL4 (for 48KHz) and PLL15 (for
+44.1KHz). The same PLLs are used for McASP10's AUXCLK clock via different
+HSDIVIDER.
+
+Required properties:
+- compatible : "ti,j721e-cpb-audio"
+- ti,model : Sound card name, should be "j721e-cpb-analog"
+- ti,cpb-mcasp : phandle to McASP10
+- ti,cpb-codec : phandle to the pcm3168a codec on the CPB
+- pll4-rate : Rate of the PLL4 in Hz. Must be chacked agains how it is
+	      configured by sysfw.
+- pll15-rate : Rate of the PLL15 in Hz. Must be chacked agains how it is
+	       configured by sysfw.
+- clocks : Must contain an entry for each entry in clock-names.
+- clock-names : Must include the following entries:
+		"cpb-mcasp" (McASP10 auxclk clock)
+		"cpb-mcasp-48000" (PLL4_HSDIV0 parent for McASP10 auxclk)
+		"cpb-mcasp-44100" (PLL15_HSDIV0 parent for McASP10 auxclk)
+		"audio-refclk2" (AUDIO_REFCLK2 clock)
+		"audio-refclk2-48000" (PLL4_HSDIV2 parent for AUDIO_REFCLK2
+				       clock)
+		"audio-refclk2-44100" (PLL15_HSDIV2 parent for AUDIO_REFCLK2
+				       clock)
+
+Example:
+
+sound0: sound@0 {
+	compatible = "ti,j721e-cpb-audio";
+	ti,model = "j721e-cpb-analog";
+
+	status = "okay";
+
+	ti,cpb-mcasp = <&mcasp10>;
+	ti,cpb-codec = <&pcm3168a_1>;
+
+	pll4-rate = <1179648000>; /* Used for 48KHz family */
+	pll15-rate = <1083801600>; /* Used for 44.1KHz family */
+
+	clocks = <&k3_clks 184 1>,
+		 <&k3_clks 184 2>, <&k3_clks 184 4>,
+		 <&k3_clks 157 371>,
+		 <&k3_clks 157 400>, <&k3_clks 157 401>;
+	clock-names = "cpb-mcasp",
+		      "cpb-mcasp-48000", "cpb-mcasp-44100",
+		      "audio-refclk2",
+		      "audio-refclk2-48000", "audio-refclk2-44100";
+};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/spi/omap-spi.txt b/Documentation/devicetree/bindings/spi/omap-spi.txt
--- a/Documentation/devicetree/bindings/spi/omap-spi.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/spi/omap-spi.txt	2020-06-19 21:00:31.400227715 -0400
@@ -2,6 +2,7 @@
 
 Required properties:
 - compatible :
+  - "ti,am654-mcspi" for AM654.
   - "ti,omap2-mcspi" for OMAP2 & OMAP3.
   - "ti,omap4-mcspi" for OMAP4+.
 - ti,spi-num-cs : Number of chipselect supported  by the instance.
diff -Naur --no-dereference a/Documentation/devicetree/bindings/spi/spi-bus.txt b/Documentation/devicetree/bindings/spi/spi-bus.txt
--- a/Documentation/devicetree/bindings/spi/spi-bus.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/spi/spi-bus.txt	2020-06-19 21:00:31.400227715 -0400
@@ -77,6 +77,8 @@
 		    Defaults to 1 if not present.
 - spi-rx-delay-us - Microsecond delay after a read transfer.
 - spi-tx-delay-us - Microsecond delay after a write transfer.
+- spi-dqs         - Empty property indicating device has Data Strobe
+		    line (DQS)
 
 Some SPI controllers and devices support Dual and Quad SPI transfer mode.
 It allows data in the SPI system to be transferred using 2 wires (DUAL) or 4
diff -Naur --no-dereference a/Documentation/devicetree/bindings/staging/android/ion/ti,ion.txt b/Documentation/devicetree/bindings/staging/android/ion/ti,ion.txt
--- a/Documentation/devicetree/bindings/staging/android/ion/ti,ion.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/staging/android/ion/ti,ion.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,37 @@
+Texas Instruments ION heap declaration binding
+
+Describes heaps and the associated memory. Each child node provides
+information about a single heap. Only chunk and carveout heaps are currently
+supported. CMA and system heaps are automatically generated when enabled.
+
+Required Properties:
+ - compatible:		Must be "ti,ion"
+
+Required child node properties:
+ - memory-region:	A phandle to the reserved memory node that backs this
+			heap. The reserved memory node can be a CMA memory
+			node, and should be defined as per the bindings in
+			Documentation/devicetree/bindings/reserved-memory/reserved-memory.txt
+
+ - ion-heap-type:	Heap type ID as defined by uapi/ion.h
+
+Required child node properties for chunk heaps:
+ - ion-chunk-size:	Size of each allocation pool, allocation will only
+			be made in granules of this size to prevent
+			fragmentation
+
+Example:
+
+ion {
+	compatible = "ti,ion";
+	ti_chunk_heap_0 {
+		memory-region = <&ion_block_mem_0>;
+		ion-heap-type = <3>; // ION_HEAP_TYPE_CHUNK
+		ion-chunk-size = <0x00010000>; // 64KB
+	};
+
+	ti_carveout_heap_0 {
+		memory-region = <&ion_block_mem_1>;
+		ion-heap-type = <2>; // ION_HEAP_TYPE_CARVEOUT
+	};
+};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/thermal/ti_k3_thermal.txt b/Documentation/devicetree/bindings/thermal/ti_k3_thermal.txt
--- a/Documentation/devicetree/bindings/thermal/ti_k3_thermal.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/thermal/ti_k3_thermal.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,29 @@
+* Texas Instrument K3 VTM bandgap bindings
+
+In the Voltage Thermal Management Module(VTM), K3 supplies a voltage
+reference and a temperature sensor feature that are gathered in the band
+gap voltage and temperature sensor (VBGAPTS) module. The band
+gap provides current and voltage reference for its internal
+circuits and other analog IP blocks. The analog-to-digital
+converter (ADC) produces an output value that is proportional
+to the silicon temperature.
+
+Required properties:
+- compatible : Should be:
+  - "ti,am654-vtm" : for K3 AM654 VTM bandgap
+- reg: Base address and address range of the VTM registers
+- power-domains  : Should contain a phandle to a PM domain provider node and an
+                   args specifier containing the VTM device id value. This
+                   property is as per the binding,
+                       Documentation/devicetree/bindings/soc/ti/sci-pm-domain.txt
+- #thermal-sensor-cells: Should be 1. See ./thermal.txt for a description
+
+Example:
+
+AM654:
+wkup_vtm0: wkup_vtm0@42050000 {
+	compatible = "ti,am654-vtm";
+	reg = <0x0 0x42050000 0x0 0x25c>;
+	power-domains = <&k3_pds 80>;
+	#thermal-sensor-cells = <1>;
+};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/ufs/cdns,ufshc.txt b/Documentation/devicetree/bindings/ufs/cdns,ufshc.txt
--- a/Documentation/devicetree/bindings/ufs/cdns,ufshc.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/ufs/cdns,ufshc.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,32 @@
+* Cadence Universal Flash Storage (UFS) Controller
+
+UFS nodes are defined to describe on-chip UFS host controllers.
+Each UFS controller instance should have its own node.
+Please see the ufshcd-pltfrm.txt for a list of all available properties.
+
+Required properties:
+- compatible	: Compatible list, contains one of the following controllers:
+			"cdns,ufshc" - Generic CDNS HCI,
+			"cdns,ufshc-m31-16nm" - CDNS UFS HC + M31 16nm PHY
+		  complemented with the JEDEC version:
+			"jedec,ufs-2.0"
+
+- reg		: Address and length of the UFS register set.
+- interrupts	: One interrupt mapping.
+- freq-table-hz	: Clock frequency table.
+		  See the ufshcd-pltfrm.txt for details.
+- clocks	: List of phandle and clock specifier pairs.
+- clock-names	: List of clock input name strings sorted in the same
+		  order as the clocks property. "core_clk" is mandatory.
+		  Depending on a type of a PHY,
+		  the "phy_clk" clock can also be added, if needed.
+
+Example:
+	ufs@fd030000 {
+		compatible = "cdns,ufshc", "jedec,ufs-2.0";
+		reg = <0xfd030000 0x10000>;
+		interrupts = <0 1 IRQ_TYPE_LEVEL_HIGH>;
+		freq-table-hz = <0 0>, <0 0>;
+		clocks = <&ufs_core_clk>, <&ufs_phy_clk>;
+		clock-names = "core_clk", "phy_clk";
+	};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/ufs/ti-j721e-ufs.txt b/Documentation/devicetree/bindings/ufs/ti-j721e-ufs.txt
--- a/Documentation/devicetree/bindings/ufs/ti-j721e-ufs.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/ufs/ti-j721e-ufs.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,39 @@
+TI J721e UFS Host Controller Glue Driver
+
+Required properties:
+- compatible : Must be ""ti,j721e-ufs" for TI J721e SoC
+- reg : address of TI UFS glue registers
+- clocks : phandle to the M-PHY clock
+
+Subnodes:
+
+Cadence UFS Host Controller node needs to be added as a subnode to glue
+driver node. See Documentation/devicetree/bindings/ufs/cdns,ufshc.txt
+for subnode bindings
+
+Example :
+
+	ufs_wrapper: ufs-wrapper@4e80000 {
+		compatible = "ti,j721e-ufs";
+		reg = <0x0 0x4e80000 0x0 0x100>;
+		power-domains = <&k3_pds 277>;
+		clocks = <&k3_clks 277 1>;
+		assigned-clocks = <&k3_clks 277 1>;
+		assigned-clock-parents = <&k3_clks 277 4>;
+		ranges;
+		#address-cells = <2>;
+		#size-cells = <2>;
+
+		ufs@4e84000 {
+			compatible = "cdns,ufshc-m31-16nm", "jedec,ufs-2.0";
+			reg = <0x0 0x4e84000 0x0 0x10000>;
+			interrupts = <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>;
+			power-domains = <&k3_pds 277>;
+			freq-table-hz = <0 0>, <0 0>;
+			clocks = <&k3_clks 277 0>, <&k3_clks 277 1>;
+			clock-names = "core_clk", "phy_clk";
+			assigned-clocks = <&k3_clks 277 1>;
+			assigned-clock-parents = <&k3_clks 277 4>;
+			dma-coherent;
+		};
+	};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/ufs/ufshcd-pltfrm.txt b/Documentation/devicetree/bindings/ufs/ufshcd-pltfrm.txt
--- a/Documentation/devicetree/bindings/ufs/ufshcd-pltfrm.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/ufs/ufshcd-pltfrm.txt	2020-06-19 21:00:31.400227715 -0400
@@ -33,6 +33,12 @@
 - clocks                : List of phandle and clock specifier pairs
 - clock-names           : List of clock input name strings sorted in the same
                           order as the clocks property.
+			  "ref_clk" indicates reference clock frequency.
+			  UFS host supplies reference clock to UFS device and UFS device
+			  specification allows host to provide one of the 4 frequencies (19.2 MHz,
+			  26 MHz, 38.4 MHz, 52MHz) for reference clock. This "ref_clk" entry is
+			  parsed and used to update the reference clock setting in device.
+			  Defaults to 26 MHz(as per specification) if not specified by host.
 - freq-table-hz		: Array of <min max> operating frequencies stored in the same
                           order as the clocks property. If this property is not
 			  defined or a value in the array is "0" then it is assumed
diff -Naur --no-dereference a/Documentation/devicetree/bindings/usb/cdns-usb3-ti.txt b/Documentation/devicetree/bindings/usb/cdns-usb3-ti.txt
--- a/Documentation/devicetree/bindings/usb/cdns-usb3-ti.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/usb/cdns-usb3-ti.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,59 @@
+Binding for the TI specific wrapper for the Cadence USBSS-DRD controller
+
+Required properties:
+  - compatible: Should contain "ti,j721e-usb"
+  - reg: Physical base address and size of the wrappers register area.
+  - power-domains: Should contain a phandle to a PM domain provider node
+                   and an args specifier containing the USB device id
+                   value. This property is as per the binding documentation:
+                   Documentation/devicetree/bindings/soc/ti/sci-pm-domain.txt
+  - clocks: Clock phandles to usb2_refclk and lpm_clk
+  - clock-names: Should contain "usb2_refclk" and "lpm_clk"
+
+Optional properties:
+ - ti,usb2-only: If present, it restricts the controller to USB2.0 mode of
+		 operation. Must be present if USB3 PHY is not available
+		 for USB.
+ - ti,modestrap-host: Set controller modestrap to HOST mode.
+ - ti,modestrap-peripheral: Set controller modestrap to PERIPHERAL mode.
+ - ti,vbus-divider: Should be present if USB VBUS line is connected to the
+		 VBUS pin of the SoC via a 1/3 voltage divider.
+
+Sub-nodes:
+The USB2 PHY and the Cadence USB3 controller should be the sub-nodes.
+
+Example:
+
+	ti_usb0: cdns_usb@4104000 {
+		compatible = "ti,j721e-usb";
+		reg = <0x00 0x4104000 0x00 0x100>;
+		power-domains = <&k3_pds 288 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 288 15>, <&k3_clks 288 3>;
+		clock-names = "usb2_refclk", "lpm_clk";
+		assigned-clocks = <&k3_clks 288 15>;	/* USB2_REFCLK */
+		assigned-clock-parents = <&k3_clks 288 16>; /* HFOSC0 */
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		phy@4108000 {
+			compatible = "ti,j721e-usb2-phy";
+			reg = <0x00 0x4108000 0x00 0x400>;
+		};
+
+		usb0: usb@6000000 {
+			compatible = "cdns,usb3-1.0.1";
+			reg = <0x00 0x6000000 0x00 0x10000>,
+			      <0x00 0x6010000 0x00 0x10000>,
+			      <0x00 0x6020000 0x00 0x10000>;
+			reg-names = "otg", "xhci", "dev";
+			interrupts = <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>,	/* irq.0 */
+				     <GIC_SPI 102 IRQ_TYPE_LEVEL_HIGH>,	/* irq.6 */
+				     <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>;	/* otgirq.0 */
+			interrupt-names = "host",
+					  "peripheral",
+					  "otg";
+			maximum-speed = "super-speed";
+			dr_mode = "otg";
+		};
+	};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/usb/cdns-usb3.txt b/Documentation/devicetree/bindings/usb/cdns-usb3.txt
--- a/Documentation/devicetree/bindings/usb/cdns-usb3.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/usb/cdns-usb3.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,45 @@
+Binding for the Cadence USBSS-DRD controller
+
+Required properties:
+ - reg: Physical base address and size of the controller's register areas.
+	 Controller has 3 different regions:
+	 - HOST registers area
+	 - DEVICE registers area
+	 - OTG/DRD registers area
+ - reg-names - register memory area names:
+	"xhci" - for HOST registers space
+	"dev" - for DEVICE registers space
+	"otg" - for OTG/DRD registers space
+ - compatible: Should contain: "cdns,usb3"
+ - interrupts: Interrupts used by cdns3 controller:
+	"host" - interrupt used by XHCI driver.
+	"peripheral" - interrupt used by device driver
+	"otg" - interrupt used by DRD/OTG  part of driver
+
+Optional properties:
+ - maximum-speed : valid arguments are "super-speed", "high-speed" and
+                   "full-speed"; refer to usb/generic.txt
+ - dr_mode: Should be one of "host", "peripheral" or "otg".
+ - phys: reference to the USB PHY
+ - phy-names: from the *Generic PHY* bindings;
+	Supported names are:
+	- cdns3,usb2-phy
+	- cdns3,usb3-phy
+
+ - cdns,on-chip-buff-size : size of memory intended as internal memory for endpoints
+	buffers expressed in KB
+
+Example:
+	usb@f3000000 {
+		compatible = "cdns,usb3";
+		interrupts = <GIC_USB_IRQ 7 IRQ_TYPE_LEVEL_HIGH>,
+				<GIC_USB_IRQ  7 IRQ_TYPE_LEVEL_HIGH>,
+				<GIC_USB_IRQ  8 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "host", "peripheral", "otg";
+		reg = <0xf3000000 0x10000>,	/* memory area for HOST registers */
+			<0xf3010000 0x10000>,	/* memory area for DEVICE registers */
+			<0xf3020000 0x10000>;	/* memory area for OTG/DRD registers */
+		reg-names = "xhci", "dev", "otg";
+		phys = <&usb2_phy>, <&usb3_phy>;
+		phy-names = "cdns3,usb2-phy", "cnds3,usb3-phy";
+	};
diff -Naur --no-dereference a/Documentation/devicetree/bindings/usb/keystone-usb.txt b/Documentation/devicetree/bindings/usb/keystone-usb.txt
--- a/Documentation/devicetree/bindings/usb/keystone-usb.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/usb/keystone-usb.txt	2020-06-19 21:00:31.400227715 -0400
@@ -3,7 +3,9 @@
 DWC3 GLUE
 
 Required properties:
- - compatible: should be "ti,keystone-dwc3".
+ - compatible: should be
+		"ti,keystone-dwc3" for Keystone 2 SoCs
+		"ti,am654-dwc3" for AM654 SoC
  - #address-cells, #size-cells : should be '1' if the device has sub-nodes
    with 'reg' property.
  - reg : Address and length of the register set for the USB subsystem on
@@ -21,7 +23,7 @@
 - clock-names:		Must be "usb".
 
 
-The following are mandatory properties for Keystone 2 66AK2G SoCs only:
+The following are mandatory properties for 66AK2G and AM654:
 
 - power-domains:	Should contain a phandle to a PM domain provider node
 			and an args specifier containing the USB device id
diff -Naur --no-dereference a/Documentation/devicetree/bindings/vendor-prefixes.txt b/Documentation/devicetree/bindings/vendor-prefixes.txt
--- a/Documentation/devicetree/bindings/vendor-prefixes.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/devicetree/bindings/vendor-prefixes.txt	2020-06-19 21:00:31.400227715 -0400
@@ -54,6 +54,7 @@
 axentia	Axentia Technologies AB
 axis	Axis Communications AB
 bananapi BIPAI KEJI LIMITED
+beagle	BeagleBoard.org Foundation
 bhf	Beckhoff Automation GmbH & Co. KG
 bitmain	Bitmain Technologies
 boe	BOE Technology Group Co., Ltd.
@@ -285,6 +286,7 @@
 ORCL	Oracle Corporation
 orisetech	Orise Technology
 ortustech	Ortus Technology Co., Ltd.
+osddisplays	OSD Displays
 ovti	OmniVision Technologies
 oxsemi	Oxford Semiconductor, Ltd.
 panasonic	Panasonic Corporation
@@ -321,6 +323,7 @@
 rikomagic	Rikomagic Tech Corp. Ltd
 riscv	RISC-V Foundation
 rockchip	Fuzhou Rockchip Electronics Co., Ltd
+rocktech	Rocktech Displays Limited
 rohm	ROHM Semiconductor Co., Ltd
 roofull	Shenzhen Roofull Technology Co, Ltd
 samsung	Samsung Semiconductor
diff -Naur --no-dereference a/Documentation/devicetree/bindings/watchdog/ti,rti-wdt.txt b/Documentation/devicetree/bindings/watchdog/ti,rti-wdt.txt
--- a/Documentation/devicetree/bindings/watchdog/ti,rti-wdt.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/devicetree/bindings/watchdog/ti,rti-wdt.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,18 @@
+Texas Instruments RTI (Real Time Interrupt) module. RTI is used for
+implementing a watchdog on TI SoCs.
+
+Required properties:
+	compatible	= "ti,rti-wdt";
+	reg		= Physical IO space for RTI
+	clocks		= Functional clock for the RTI
+
+Example:
+
+main_rti0: rti@2200000 {
+	compatible = "ti,rti-wdt";
+	reg = <0x0 0x2200000 0x0 0x100>;
+	clocks = <&k3_clks 252 1>;
+	power-domains = <&k3_pds 252 TI_SCI_PD_EXCLUSIVE>;
+	assigned-clocks = <&k3_clks 252 1>;
+	assigned-clock-parents = <&k3_clks 252 5>;
+};
diff -Naur --no-dereference a/Documentation/driver-api/dmaengine/client.rst b/Documentation/driver-api/dmaengine/client.rst
--- a/Documentation/driver-api/dmaengine/client.rst	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/driver-api/dmaengine/client.rst	2020-06-19 21:00:31.400227715 -0400
@@ -151,6 +151,81 @@
      Note that callbacks will always be invoked from the DMA
      engines tasklet, never from interrupt context.
 
+  Optional: per descriptor metadata
+  ---------------------------------
+  DMAengine provides two ways for metadata support.
+
+  DESC_METADATA_CLIENT
+
+    The metadata buffer is allocated/provided by the client driver and it is
+    attached to the descriptor.
+
+  .. code-block:: c
+
+     int dmaengine_desc_attach_metadata(struct dma_async_tx_descriptor *desc,
+				   void *data, size_t len);
+
+  DESC_METADATA_ENGINE
+
+    The metadata buffer is allocated/managed by the DMA driver. The client
+    driver can ask for the pointer, maximum size and the currently used size of
+    the metadata and can directly update or read it.
+
+  .. code-block:: c
+
+     void *dmaengine_desc_get_metadata_ptr(struct dma_async_tx_descriptor *desc,
+		size_t *payload_len, size_t *max_len);
+
+     int dmaengine_desc_set_metadata_len(struct dma_async_tx_descriptor *desc,
+		size_t payload_len);
+
+  Client drivers can query if a given mode is supported with:
+
+  .. code-block:: c
+
+     bool dmaengine_is_metadata_mode_supported(struct dma_chan *chan,
+		enum dma_desc_metadata_mode mode);
+
+  Depending on the used mode client drivers must follow different flow.
+
+  DESC_METADATA_CLIENT
+
+    - DMA_MEM_TO_DEV / DEV_MEM_TO_MEM:
+      1. prepare the descriptor (dmaengine_prep_*)
+         construct the metadata in the client's buffer
+      2. use dmaengine_desc_attach_metadata() to attach the buffer to the
+         descriptor
+      3. submit the transfer
+    - DMA_DEV_TO_MEM:
+      1. prepare the descriptor (dmaengine_prep_*)
+      2. use dmaengine_desc_attach_metadata() to attach the buffer to the
+         descriptor
+      3. submit the transfer
+      4. when the transfer is completed, the metadata should be available in the
+         attached buffer
+
+  DESC_METADATA_ENGINE
+
+    - DMA_MEM_TO_DEV / DEV_MEM_TO_MEM:
+      1. prepare the descriptor (dmaengine_prep_*)
+      2. use dmaengine_desc_get_metadata_ptr() to get the pointer to the
+         engine's metadata area
+      3. update the metadata at the pointer
+      4. use dmaengine_desc_set_metadata_len()  to tell the DMA engine the
+         amount of data the client has placed into the metadata buffer
+      5. submit the transfer
+    - DMA_DEV_TO_MEM:
+      1. prepare the descriptor (dmaengine_prep_*)
+      2. submit the transfer
+      3. on transfer completion, use dmaengine_desc_get_metadata_ptr() to get the
+         pointer to the engine's metadata are
+      4. Read out the metadate from the pointer
+
+  .. note::
+
+     Mixed use of DESC_METADATA_CLIENT / DESC_METADATA_ENGINE is not allowed,
+     client drivers must use either of the modes per descriptor.
+
 4. Submit the transaction
 
    Once the descriptor has been prepared and the callback information
diff -Naur --no-dereference a/Documentation/driver-api/dmaengine/provider.rst b/Documentation/driver-api/dmaengine/provider.rst
--- a/Documentation/driver-api/dmaengine/provider.rst	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/driver-api/dmaengine/provider.rst	2020-06-19 21:00:31.400227715 -0400
@@ -247,6 +247,52 @@
 (DMA_CYCLIC). Addresses pointing to a device's register (e.g. a FIFO)
 are typically fixed.
 
+Per descriptor metadata support
+-------------------------------
+Some data movement architecure (DMA controller and peripherals) uses metadata
+associated with a transaction. The DMA controller role is to transfer the
+payload and the metadata alongside.
+The metadata itself is not used by the DMA engine itself, but it contains
+parameters, keys, vectors, etc for peripheral or from the peripheral.
+
+The DMAengine framework provides a generic ways to facilitate the metadata for
+descriptors. Depending on the architecture the DMA driver can implement either
+or both of the methods and it is up to the client driver to choose which one
+to use.
+
+- DESC_METADATA_CLIENT
+
+  The metadata buffer is allocated/provided by the client driver and it is
+  attached (via the dmaengine_desc_attach_metadata() helper to the descriptor.
+
+  From the DMA driver the following is expected for this mode:
+  - DMA_MEM_TO_DEV / DEV_MEM_TO_MEM
+    The data from the provided metadata buffer should be prepared for the DMA
+    controller to be sent alongside of the payload data. Either by copying to a
+    hardware descriptor, or highly coupled packet.
+  - DMA_DEV_TO_MEM
+    On transfer completion the DMA driver must copy the metadata to the client
+    provided metadata buffer.
+
+- DESC_METADATA_ENGINE
+
+  The metadata buffer is allocated/managed by the DMA driver. The client driver
+  can ask for the pointer, maximum size and the currently used size of the
+  metadata and can directly update or read it. dmaengine_desc_get_metadata_ptr()
+  and dmaengine_desc_set_metadata_len() is provided as helper functions.
+
+  From the DMA driver the following is expected for this mode:
+  - get_metadata_ptr
+    Should return a pointer for the metadata buffer, the maximum size of the
+    metadata buffer and the currently used / valid (if any) bytes in the buffer.
+  - set_metadata_len
+    It is called by the clients after it have placed the metadata to the buffer
+    to let the DMA driver know the number of valid bytes provided.
+
+  Note: since the client will ask for the metadata pointer in the completion
+  callback (in DMA_DEV_TO_MEM case) the DMA driver must ensure that the
+  descriptor is not freed up prior the callback is called.
+
 Device operations
 -----------------
 
diff -Naur --no-dereference a/Documentation/driver-model/devres.txt b/Documentation/driver-model/devres.txt
--- a/Documentation/driver-model/devres.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/driver-model/devres.txt	2020-06-19 21:00:31.400227715 -0400
@@ -235,6 +235,7 @@
 
 CLOCK
   devm_clk_get()
+  devm_clk_get_optional()
   devm_clk_put()
   devm_clk_hw_register()
   devm_of_clk_add_hw_provider()
diff -Naur --no-dereference a/Documentation/filesystems/aufs/design/01intro.txt b/Documentation/filesystems/aufs/design/01intro.txt
--- a/Documentation/filesystems/aufs/design/01intro.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/filesystems/aufs/design/01intro.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,171 @@
+
+# Copyright (C) 2005-2020 Junjiro R. Okajima
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Introduction
+----------------------------------------
+
+aufs [ei ju: ef es] | /ey-yoo-ef-es/ | [a u f s]
+1. abbrev. for "advanced multi-layered unification filesystem".
+2. abbrev. for "another unionfs".
+3. abbrev. for "auf das" in German which means "on the" in English.
+   Ex. "Butter aufs Brot"(G) means "butter onto bread"(E).
+   But "Filesystem aufs Filesystem" is hard to understand.
+4. abbrev. for "African Urban Fashion Show".
+
+AUFS is a filesystem with features:
+- multi layered stackable unification filesystem, the member directory
+  is called as a branch.
+- branch permission and attribute, 'readonly', 'real-readonly',
+  'readwrite', 'whiteout-able', 'link-able whiteout', etc. and their
+  combination.
+- internal "file copy-on-write".
+- logical deletion, whiteout.
+- dynamic branch manipulation, adding, deleting and changing permission.
+- allow bypassing aufs, user's direct branch access.
+- external inode number translation table and bitmap which maintains the
+  persistent aufs inode number.
+- seekable directory, including NFS readdir.
+- file mapping, mmap and sharing pages.
+- pseudo-link, hardlink over branches.
+- loopback mounted filesystem as a branch.
+- several policies to select one among multiple writable branches.
+- revert a single systemcall when an error occurs in aufs.
+- and more...
+
+
+Multi Layered Stackable Unification Filesystem
+----------------------------------------------------------------------
+Most people already knows what it is.
+It is a filesystem which unifies several directories and provides a
+merged single directory. When users access a file, the access will be
+passed/re-directed/converted (sorry, I am not sure which English word is
+correct) to the real file on the member filesystem. The member
+filesystem is called 'lower filesystem' or 'branch' and has a mode
+'readonly' and 'readwrite.' And the deletion for a file on the lower
+readonly branch is handled by creating 'whiteout' on the upper writable
+branch.
+
+On LKML, there have been discussions about UnionMount (Jan Blunck,
+Bharata B Rao and Valerie Aurora) and Unionfs (Erez Zadok). They took
+different approaches to implement the merged-view.
+The former tries putting it into VFS, and the latter implements as a
+separate filesystem.
+(If I misunderstand about these implementations, please let me know and
+I shall correct it. Because it is a long time ago when I read their
+source files last time).
+
+UnionMount's approach will be able to small, but may be hard to share
+branches between several UnionMount since the whiteout in it is
+implemented in the inode on branch filesystem and always
+shared. According to Bharata's post, readdir does not seems to be
+finished yet.
+There are several missing features known in this implementations such as
+- for users, the inode number may change silently. eg. copy-up.
+- link(2) may break by copy-up.
+- read(2) may get an obsoleted filedata (fstat(2) too).
+- fcntl(F_SETLK) may be broken by copy-up.
+- unnecessary copy-up may happen, for example mmap(MAP_PRIVATE) after
+  open(O_RDWR).
+
+In linux-3.18, "overlay" filesystem (formerly known as "overlayfs") was
+merged into mainline. This is another implementation of UnionMount as a
+separated filesystem. All the limitations and known problems which
+UnionMount are equally inherited to "overlay" filesystem.
+
+Unionfs has a longer history. When I started implementing a stackable
+filesystem (Aug 2005), it already existed. It has virtual super_block,
+inode, dentry and file objects and they have an array pointing lower
+same kind objects. After contributing many patches for Unionfs, I
+re-started my project AUFS (Jun 2006).
+
+In AUFS, the structure of filesystem resembles to Unionfs, but I
+implemented my own ideas, approaches and enhancements and it became
+totally different one.
+
+Comparing DM snapshot and fs based implementation
+- the number of bytes to be copied between devices is much smaller.
+- the type of filesystem must be one and only.
+- the fs must be writable, no readonly fs, even for the lower original
+  device. so the compression fs will not be usable. but if we use
+  loopback mount, we may address this issue.
+  for instance,
+	mount /cdrom/squashfs.img /sq
+	losetup /sq/ext2.img
+	losetup /somewhere/cow
+	dmsetup "snapshot /dev/loop0 /dev/loop1 ..."
+- it will be difficult (or needs more operations) to extract the
+  difference between the original device and COW.
+- DM snapshot-merge may help a lot when users try merging. in the
+  fs-layer union, users will use rsync(1).
+
+You may want to read my old paper "Filesystems in LiveCD"
+(http://aufs.sourceforge.net/aufs2/report/sq/sq.pdf).
+
+
+Several characters/aspects/persona of aufs
+----------------------------------------------------------------------
+
+Aufs has several characters, aspects or persona.
+1. a filesystem, callee of VFS helper
+2. sub-VFS, caller of VFS helper for branches
+3. a virtual filesystem which maintains persistent inode number
+4. reader/writer of files on branches such like an application
+
+1. Callee of VFS Helper
+As an ordinary linux filesystem, aufs is a callee of VFS. For instance,
+unlink(2) from an application reaches sys_unlink() kernel function and
+then vfs_unlink() is called. vfs_unlink() is one of VFS helper and it
+calls filesystem specific unlink operation. Actually aufs implements the
+unlink operation but it behaves like a redirector.
+
+2. Caller of VFS Helper for Branches
+aufs_unlink() passes the unlink request to the branch filesystem as if
+it were called from VFS. So the called unlink operation of the branch
+filesystem acts as usual. As a caller of VFS helper, aufs should handle
+every necessary pre/post operation for the branch filesystem.
+- acquire the lock for the parent dir on a branch
+- lookup in a branch
+- revalidate dentry on a branch
+- mnt_want_write() for a branch
+- vfs_unlink() for a branch
+- mnt_drop_write() for a branch
+- release the lock on a branch
+
+3. Persistent Inode Number
+One of the most important issue for a filesystem is to maintain inode
+numbers. This is particularly important to support exporting a
+filesystem via NFS. Aufs is a virtual filesystem which doesn't have a
+backend block device for its own. But some storage is necessary to
+keep and maintain the inode numbers. It may be a large space and may not
+suit to keep in memory. Aufs rents some space from its first writable
+branch filesystem (by default) and creates file(s) on it. These files
+are created by aufs internally and removed soon (currently) keeping
+opened.
+Note: Because these files are removed, they are totally gone after
+      unmounting aufs. It means the inode numbers are not persistent
+      across unmount or reboot. I have a plan to make them really
+      persistent which will be important for aufs on NFS server.
+
+4. Read/Write Files Internally (copy-on-write)
+Because a branch can be readonly, when you write a file on it, aufs will
+"copy-up" it to the upper writable branch internally. And then write the
+originally requested thing to the file. Generally kernel doesn't
+open/read/write file actively. In aufs, even a single write may cause a
+internal "file copy". This behaviour is very similar to cp(1) command.
+
+Some people may think it is better to pass such work to user space
+helper, instead of doing in kernel space. Actually I am still thinking
+about it. But currently I have implemented it in kernel space.
diff -Naur --no-dereference a/Documentation/filesystems/aufs/design/02struct.txt b/Documentation/filesystems/aufs/design/02struct.txt
--- a/Documentation/filesystems/aufs/design/02struct.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/filesystems/aufs/design/02struct.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,258 @@
+
+# Copyright (C) 2005-2020 Junjiro R. Okajima
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Basic Aufs Internal Structure
+
+Superblock/Inode/Dentry/File Objects
+----------------------------------------------------------------------
+As like an ordinary filesystem, aufs has its own
+superblock/inode/dentry/file objects. All these objects have a
+dynamically allocated array and store the same kind of pointers to the
+lower filesystem, branch.
+For example, when you build a union with one readwrite branch and one
+readonly, mounted /au, /rw and /ro respectively.
+- /au = /rw + /ro
+- /ro/fileA exists but /rw/fileA
+
+Aufs lookup operation finds /ro/fileA and gets dentry for that. These
+pointers are stored in a aufs dentry. The array in aufs dentry will be,
+- [0] = NULL (because /rw/fileA doesn't exist)
+- [1] = /ro/fileA
+
+This style of an array is essentially same to the aufs
+superblock/inode/dentry/file objects.
+
+Because aufs supports manipulating branches, ie. add/delete/change
+branches dynamically, these objects has its own generation. When
+branches are changed, the generation in aufs superblock is
+incremented. And a generation in other object are compared when it is
+accessed. When a generation in other objects are obsoleted, aufs
+refreshes the internal array.
+
+
+Superblock
+----------------------------------------------------------------------
+Additionally aufs superblock has some data for policies to select one
+among multiple writable branches, XIB files, pseudo-links and kobject.
+See below in detail.
+About the policies which supports copy-down a directory, see
+wbr_policy.txt too.
+
+
+Branch and XINO(External Inode Number Translation Table)
+----------------------------------------------------------------------
+Every branch has its own xino (external inode number translation table)
+file. The xino file is created and unlinked by aufs internally. When two
+members of a union exist on the same filesystem, they share the single
+xino file.
+The struct of a xino file is simple, just a sequence of aufs inode
+numbers which is indexed by the lower inode number.
+In the above sample, assume the inode number of /ro/fileA is i111 and
+aufs assigns the inode number i999 for fileA. Then aufs writes 999 as
+4(8) bytes at 111 * 4(8) bytes offset in the xino file.
+
+When the inode numbers are not contiguous, the xino file will be sparse
+which has a hole in it and doesn't consume as much disk space as it
+might appear. If your branch filesystem consumes disk space for such
+holes, then you should specify 'xino=' option at mounting aufs.
+
+Aufs has a mount option to free the disk blocks for such holes in XINO
+files on tmpfs or ramdisk. But it is not so effective actually. If you
+meet a problem of disk shortage due to XINO files, then you should try
+"tmpfs-ino.patch" (and "vfs-ino.patch" too) in aufs4-standalone.git.
+The patch localizes the assignment inumbers per tmpfs-mount and avoid
+the holes in XINO files.
+
+Also a writable branch has three kinds of "whiteout bases". All these
+are existed when the branch is joined to aufs, and their names are
+whiteout-ed doubly, so that users will never see their names in aufs
+hierarchy.
+1. a regular file which will be hardlinked to all whiteouts.
+2. a directory to store a pseudo-link.
+3. a directory to store an "orphan"-ed file temporary.
+
+1. Whiteout Base
+   When you remove a file on a readonly branch, aufs handles it as a
+   logical deletion and creates a whiteout on the upper writable branch
+   as a hardlink of this file in order not to consume inode on the
+   writable branch.
+2. Pseudo-link Dir
+   See below, Pseudo-link.
+3. Step-Parent Dir
+   When "fileC" exists on the lower readonly branch only and it is
+   opened and removed with its parent dir, and then user writes
+   something into it, then aufs copies-up fileC to this
+   directory. Because there is no other dir to store fileC. After
+   creating a file under this dir, the file is unlinked.
+
+Because aufs supports manipulating branches, ie. add/delete/change
+dynamically, a branch has its own id. When the branch order changes,
+aufs finds the new index by searching the branch id.
+
+
+Pseudo-link
+----------------------------------------------------------------------
+Assume "fileA" exists on the lower readonly branch only and it is
+hardlinked to "fileB" on the branch. When you write something to fileA,
+aufs copies-up it to the upper writable branch. Additionally aufs
+creates a hardlink under the Pseudo-link Directory of the writable
+branch. The inode of a pseudo-link is kept in aufs super_block as a
+simple list. If fileB is read after unlinking fileA, aufs returns
+filedata from the pseudo-link instead of the lower readonly
+branch. Because the pseudo-link is based upon the inode, to keep the
+inode number by xino (see above) is essentially necessary.
+
+All the hardlinks under the Pseudo-link Directory of the writable branch
+should be restored in a proper location later. Aufs provides a utility
+to do this. The userspace helpers executed at remounting and unmounting
+aufs by default.
+During this utility is running, it puts aufs into the pseudo-link
+maintenance mode. In this mode, only the process which began the
+maintenance mode (and its child processes) is allowed to operate in
+aufs. Some other processes which are not related to the pseudo-link will
+be allowed to run too, but the rest have to return an error or wait
+until the maintenance mode ends. If a process already acquires an inode
+mutex (in VFS), it has to return an error.
+
+
+XIB(external inode number bitmap)
+----------------------------------------------------------------------
+Addition to the xino file per a branch, aufs has an external inode number
+bitmap in a superblock object. It is also an internal file such like a
+xino file.
+It is a simple bitmap to mark whether the aufs inode number is in-use or
+not.
+To reduce the file I/O, aufs prepares a single memory page to cache xib.
+
+As well as XINO files, aufs has a feature to truncate/refresh XIB to
+reduce the number of consumed disk blocks for these files.
+
+
+Virtual or Vertical Dir, and Readdir in Userspace
+----------------------------------------------------------------------
+In order to support multiple layers (branches), aufs readdir operation
+constructs a virtual dir block on memory. For readdir, aufs calls
+vfs_readdir() internally for each dir on branches, merges their entries
+with eliminating the whiteout-ed ones, and sets it to file (dir)
+object. So the file object has its entry list until it is closed. The
+entry list will be updated when the file position is zero and becomes
+obsoleted. This decision is made in aufs automatically.
+
+The dynamically allocated memory block for the name of entries has a
+unit of 512 bytes (by default) and stores the names contiguously (no
+padding). Another block for each entry is handled by kmem_cache too.
+During building dir blocks, aufs creates hash list and judging whether
+the entry is whiteouted by its upper branch or already listed.
+The merged result is cached in the corresponding inode object and
+maintained by a customizable life-time option.
+
+Some people may call it can be a security hole or invite DoS attack
+since the opened and once readdir-ed dir (file object) holds its entry
+list and becomes a pressure for system memory. But I'd say it is similar
+to files under /proc or /sys. The virtual files in them also holds a
+memory page (generally) while they are opened. When an idea to reduce
+memory for them is introduced, it will be applied to aufs too.
+For those who really hate this situation, I've developed readdir(3)
+library which operates this merging in userspace. You just need to set
+LD_PRELOAD environment variable, and aufs will not consume no memory in
+kernel space for readdir(3).
+
+
+Workqueue
+----------------------------------------------------------------------
+Aufs sometimes requires privilege access to a branch. For instance,
+in copy-up/down operation. When a user process is going to make changes
+to a file which exists in the lower readonly branch only, and the mode
+of one of ancestor directories may not be writable by a user
+process. Here aufs copy-up the file with its ancestors and they may
+require privilege to set its owner/group/mode/etc.
+This is a typical case of a application character of aufs (see
+Introduction).
+
+Aufs uses workqueue synchronously for this case. It creates its own
+workqueue. The workqueue is a kernel thread and has privilege. Aufs
+passes the request to call mkdir or write (for example), and wait for
+its completion. This approach solves a problem of a signal handler
+simply.
+If aufs didn't adopt the workqueue and changed the privilege of the
+process, then the process may receive the unexpected SIGXFSZ or other
+signals.
+
+Also aufs uses the system global workqueue ("events" kernel thread) too
+for asynchronous tasks, such like handling inotify/fsnotify, re-creating a
+whiteout base and etc. This is unrelated to a privilege.
+Most of aufs operation tries acquiring a rw_semaphore for aufs
+superblock at the beginning, at the same time waits for the completion
+of all queued asynchronous tasks.
+
+
+Whiteout
+----------------------------------------------------------------------
+The whiteout in aufs is very similar to Unionfs's. That is represented
+by its filename. UnionMount takes an approach of a file mode, but I am
+afraid several utilities (find(1) or something) will have to support it.
+
+Basically the whiteout represents "logical deletion" which stops aufs to
+lookup further, but also it represents "dir is opaque" which also stop
+further lookup.
+
+In aufs, rmdir(2) and rename(2) for dir uses whiteout alternatively.
+In order to make several functions in a single systemcall to be
+revertible, aufs adopts an approach to rename a directory to a temporary
+unique whiteouted name.
+For example, in rename(2) dir where the target dir already existed, aufs
+renames the target dir to a temporary unique whiteouted name before the
+actual rename on a branch, and then handles other actions (make it opaque,
+update the attributes, etc). If an error happens in these actions, aufs
+simply renames the whiteouted name back and returns an error. If all are
+succeeded, aufs registers a function to remove the whiteouted unique
+temporary name completely and asynchronously to the system global
+workqueue.
+
+
+Copy-up
+----------------------------------------------------------------------
+It is a well-known feature or concept.
+When user modifies a file on a readonly branch, aufs operate "copy-up"
+internally and makes change to the new file on the upper writable branch.
+When the trigger systemcall does not update the timestamps of the parent
+dir, aufs reverts it after copy-up.
+
+
+Move-down (aufs3.9 and later)
+----------------------------------------------------------------------
+"Copy-up" is one of the essential feature in aufs. It copies a file from
+the lower readonly branch to the upper writable branch when a user
+changes something about the file.
+"Move-down" is an opposite action of copy-up. Basically this action is
+ran manually instead of automatically and internally.
+For desgin and implementation, aufs has to consider these issues.
+- whiteout for the file may exist on the lower branch.
+- ancestor directories may not exist on the lower branch.
+- diropq for the ancestor directories may exist on the upper branch.
+- free space on the lower branch will reduce.
+- another access to the file may happen during moving-down, including
+  UDBA (see "Revalidate Dentry and UDBA").
+- the file should not be hard-linked nor pseudo-linked. they should be
+  handled by auplink utility later.
+
+Sometimes users want to move-down a file from the upper writable branch
+to the lower readonly or writable branch. For instance,
+- the free space of the upper writable branch is going to run out.
+- create a new intermediate branch between the upper and lower branch.
+- etc.
+
+For this purpose, use "aumvdown" command in aufs-util.git.
diff -Naur --no-dereference a/Documentation/filesystems/aufs/design/03atomic_open.txt b/Documentation/filesystems/aufs/design/03atomic_open.txt
--- a/Documentation/filesystems/aufs/design/03atomic_open.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/filesystems/aufs/design/03atomic_open.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,85 @@
+
+# Copyright (C) 2015-2020 Junjiro R. Okajima
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Support for a branch who has its ->atomic_open()
+----------------------------------------------------------------------
+The filesystems who implement its ->atomic_open() are not majority. For
+example NFSv4 does, and aufs should call NFSv4 ->atomic_open,
+particularly for open(O_CREAT|O_EXCL, 0400) case. Other than
+->atomic_open(), NFSv4 returns an error for this open(2). While I am not
+sure whether all filesystems who have ->atomic_open() behave like this,
+but NFSv4 surely returns the error.
+
+In order to support ->atomic_open() for aufs, there are a few
+approaches.
+
+A. Introduce aufs_atomic_open()
+   - calls one of VFS:do_last(), lookup_open() or atomic_open() for
+     branch fs.
+B. Introduce aufs_atomic_open() calling create, open and chmod. this is
+   an aufs user Pip Cet's approach
+   - calls aufs_create(), VFS finish_open() and notify_change().
+   - pass fake-mode to finish_open(), and then correct the mode by
+     notify_change().
+C. Extend aufs_open() to call branch fs's ->atomic_open()
+   - no aufs_atomic_open().
+   - aufs_lookup() registers the TID to an aufs internal object.
+   - aufs_create() does nothing when the matching TID is registered, but
+     registers the mode.
+   - aufs_open() calls branch fs's ->atomic_open() when the matching
+     TID is registered.
+D. Extend aufs_open() to re-try branch fs's ->open() with superuser's
+   credential
+   - no aufs_atomic_open().
+   - aufs_create() registers the TID to an internal object. this info
+     represents "this process created this file just now."
+   - when aufs gets EACCES from branch fs's ->open(), then confirm the
+     registered TID and re-try open() with superuser's credential.
+
+Pros and cons for each approach.
+
+A.
+   - straightforward but highly depends upon VFS internal.
+   - the atomic behavaiour is kept.
+   - some of parameters such as nameidata are hard to reproduce for
+     branch fs.
+   - large overhead.
+B.
+   - easy to implement.
+   - the atomic behavaiour is lost.
+C.
+   - the atomic behavaiour is kept.
+   - dirty and tricky.
+   - VFS checks whether the file is created correctly after calling
+     ->create(), which means this approach doesn't work.
+D.
+   - easy to implement.
+   - the atomic behavaiour is lost.
+   - to open a file with superuser's credential and give it to a user
+     process is a bad idea, since the file object keeps the credential
+     in it. It may affect LSM or something. This approach doesn't work
+     either.
+
+The approach A is ideal, but it hard to implement. So here is a
+variation of A, which is to be implemented.
+
+A-1. Introduce aufs_atomic_open()
+     - calls branch fs ->atomic_open() if exists. otherwise calls
+       vfs_create() and finish_open().
+     - the demerit is that the several checks after branch fs
+       ->atomic_open() are lost. in the ordinary case, the checks are
+       done by VFS:do_last(), lookup_open() and atomic_open(). some can
+       be implemented in aufs, but not all I am afraid.
diff -Naur --no-dereference a/Documentation/filesystems/aufs/design/03lookup.txt b/Documentation/filesystems/aufs/design/03lookup.txt
--- a/Documentation/filesystems/aufs/design/03lookup.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/filesystems/aufs/design/03lookup.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,113 @@
+
+# Copyright (C) 2005-2020 Junjiro R. Okajima
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Lookup in a Branch
+----------------------------------------------------------------------
+Since aufs has a character of sub-VFS (see Introduction), it operates
+lookup for branches as VFS does. It may be a heavy work. But almost all
+lookup operation in aufs is the simplest case, ie. lookup only an entry
+directly connected to its parent. Digging down the directory hierarchy
+is unnecessary. VFS has a function lookup_one_len() for that use, and
+aufs calls it.
+
+When a branch is a remote filesystem, aufs basically relies upon its
+->d_revalidate(), also aufs forces the hardest revalidate tests for
+them.
+For d_revalidate, aufs implements three levels of revalidate tests. See
+"Revalidate Dentry and UDBA" in detail.
+
+
+Test Only the Highest One for the Directory Permission (dirperm1 option)
+----------------------------------------------------------------------
+Let's try case study.
+- aufs has two branches, upper readwrite and lower readonly.
+  /au = /rw + /ro
+- "dirA" exists under /ro, but /rw. and its mode is 0700.
+- user invoked "chmod a+rx /au/dirA"
+- the internal copy-up is activated and "/rw/dirA" is created and its
+  permission bits are set to world readable.
+- then "/au/dirA" becomes world readable?
+
+In this case, /ro/dirA is still 0700 since it exists in readonly branch,
+or it may be a natively readonly filesystem. If aufs respects the lower
+branch, it should not respond readdir request from other users. But user
+allowed it by chmod. Should really aufs rejects showing the entries
+under /ro/dirA?
+
+To be honest, I don't have a good solution for this case. So aufs
+implements 'dirperm1' and 'nodirperm1' mount options, and leave it to
+users.
+When dirperm1 is specified, aufs checks only the highest one for the
+directory permission, and shows the entries. Otherwise, as usual, checks
+every dir existing on all branches and rejects the request.
+
+As a side effect, dirperm1 option improves the performance of aufs
+because the number of permission check is reduced when the number of
+branch is many.
+
+
+Revalidate Dentry and UDBA (User's Direct Branch Access)
+----------------------------------------------------------------------
+Generally VFS helpers re-validate a dentry as a part of lookup.
+0. digging down the directory hierarchy.
+1. lock the parent dir by its i_mutex.
+2. lookup the final (child) entry.
+3. revalidate it.
+4. call the actual operation (create, unlink, etc.)
+5. unlock the parent dir
+
+If the filesystem implements its ->d_revalidate() (step 3), then it is
+called. Actually aufs implements it and checks the dentry on a branch is
+still valid.
+But it is not enough. Because aufs has to release the lock for the
+parent dir on a branch at the end of ->lookup() (step 2) and
+->d_revalidate() (step 3) while the i_mutex of the aufs dir is still
+held by VFS.
+If the file on a branch is changed directly, eg. bypassing aufs, after
+aufs released the lock, then the subsequent operation may cause
+something unpleasant result.
+
+This situation is a result of VFS architecture, ->lookup() and
+->d_revalidate() is separated. But I never say it is wrong. It is a good
+design from VFS's point of view. It is just not suitable for sub-VFS
+character in aufs.
+
+Aufs supports such case by three level of revalidation which is
+selectable by user.
+1. Simple Revalidate
+   Addition to the native flow in VFS's, confirm the child-parent
+   relationship on the branch just after locking the parent dir on the
+   branch in the "actual operation" (step 4). When this validation
+   fails, aufs returns EBUSY. ->d_revalidate() (step 3) in aufs still
+   checks the validation of the dentry on branches.
+2. Monitor Changes Internally by Inotify/Fsnotify
+   Addition to above, in the "actual operation" (step 4) aufs re-lookup
+   the dentry on the branch, and returns EBUSY if it finds different
+   dentry.
+   Additionally, aufs sets the inotify/fsnotify watch for every dir on branches
+   during it is in cache. When the event is notified, aufs registers a
+   function to kernel 'events' thread by schedule_work(). And the
+   function sets some special status to the cached aufs dentry and inode
+   private data. If they are not cached, then aufs has nothing to
+   do. When the same file is accessed through aufs (step 0-3) later,
+   aufs will detect the status and refresh all necessary data.
+   In this mode, aufs has to ignore the event which is fired by aufs
+   itself.
+3. No Extra Validation
+   This is the simplest test and doesn't add any additional revalidation
+   test, and skip the revalidation in step 4. It is useful and improves
+   aufs performance when system surely hide the aufs branches from user,
+   by over-mounting something (or another method).
diff -Naur --no-dereference a/Documentation/filesystems/aufs/design/04branch.txt b/Documentation/filesystems/aufs/design/04branch.txt
--- a/Documentation/filesystems/aufs/design/04branch.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/filesystems/aufs/design/04branch.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,74 @@
+
+# Copyright (C) 2005-2020 Junjiro R. Okajima
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Branch Manipulation
+
+Since aufs supports dynamic branch manipulation, ie. add/remove a branch
+and changing its permission/attribute, there are a lot of works to do.
+
+
+Add a Branch
+----------------------------------------------------------------------
+o Confirm the adding dir exists outside of aufs, including loopback
+  mount, and its various attributes.
+o Initialize the xino file and whiteout bases if necessary.
+  See struct.txt.
+
+o Check the owner/group/mode of the directory
+  When the owner/group/mode of the adding directory differs from the
+  existing branch, aufs issues a warning because it may impose a
+  security risk.
+  For example, when a upper writable branch has a world writable empty
+  top directory, a malicious user can create any files on the writable
+  branch directly, like copy-up and modify manually. If something like
+  /etc/{passwd,shadow} exists on the lower readonly branch but the upper
+  writable branch, and the writable branch is world-writable, then a
+  malicious guy may create /etc/passwd on the writable branch directly
+  and the infected file will be valid in aufs.
+  I am afraid it can be a security issue, but aufs can do nothing except
+  producing a warning.
+
+
+Delete a Branch
+----------------------------------------------------------------------
+o Confirm the deleting branch is not busy
+  To be general, there is one merit to adopt "remount" interface to
+  manipulate branches. It is to discard caches. At deleting a branch,
+  aufs checks the still cached (and connected) dentries and inodes. If
+  there are any, then they are all in-use. An inode without its
+  corresponding dentry can be alive alone (for example, inotify/fsnotify case).
+
+  For the cached one, aufs checks whether the same named entry exists on
+  other branches.
+  If the cached one is a directory, because aufs provides a merged view
+  to users, as long as one dir is left on any branch aufs can show the
+  dir to users. In this case, the branch can be removed from aufs.
+  Otherwise aufs rejects deleting the branch.
+
+  If any file on the deleting branch is opened by aufs, then aufs
+  rejects deleting.
+
+
+Modify the Permission of a Branch
+----------------------------------------------------------------------
+o Re-initialize or remove the xino file and whiteout bases if necessary.
+  See struct.txt.
+
+o rw --> ro: Confirm the modifying branch is not busy
+  Aufs rejects the request if any of these conditions are true.
+  - a file on the branch is mmap-ed.
+  - a regular file on the branch is opened for write and there is no
+    same named entry on the upper branch.
diff -Naur --no-dereference a/Documentation/filesystems/aufs/design/05wbr_policy.txt b/Documentation/filesystems/aufs/design/05wbr_policy.txt
--- a/Documentation/filesystems/aufs/design/05wbr_policy.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/filesystems/aufs/design/05wbr_policy.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,64 @@
+
+# Copyright (C) 2005-2020 Junjiro R. Okajima
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Policies to Select One among Multiple Writable Branches
+----------------------------------------------------------------------
+When the number of writable branch is more than one, aufs has to decide
+the target branch for file creation or copy-up. By default, the highest
+writable branch which has the parent (or ancestor) dir of the target
+file is chosen (top-down-parent policy).
+By user's request, aufs implements some other policies to select the
+writable branch, for file creation several policies, round-robin,
+most-free-space, and other policies. For copy-up, top-down-parent,
+bottom-up-parent, bottom-up and others.
+
+As expected, the round-robin policy selects the branch in circular. When
+you have two writable branches and creates 10 new files, 5 files will be
+created for each branch. mkdir(2) systemcall is an exception. When you
+create 10 new directories, all will be created on the same branch.
+And the most-free-space policy selects the one which has most free
+space among the writable branches. The amount of free space will be
+checked by aufs internally, and users can specify its time interval.
+
+The policies for copy-up is more simple,
+top-down-parent is equivalent to the same named on in create policy,
+bottom-up-parent selects the writable branch where the parent dir
+exists and the nearest upper one from the copyup-source,
+bottom-up selects the nearest upper writable branch from the
+copyup-source, regardless the existence of the parent dir.
+
+There are some rules or exceptions to apply these policies.
+- If there is a readonly branch above the policy-selected branch and
+  the parent dir is marked as opaque (a variation of whiteout), or the
+  target (creating) file is whiteout-ed on the upper readonly branch,
+  then the result of the policy is ignored and the target file will be
+  created on the nearest upper writable branch than the readonly branch.
+- If there is a writable branch above the policy-selected branch and
+  the parent dir is marked as opaque or the target file is whiteouted
+  on the branch, then the result of the policy is ignored and the target
+  file will be created on the highest one among the upper writable
+  branches who has diropq or whiteout. In case of whiteout, aufs removes
+  it as usual.
+- link(2) and rename(2) systemcalls are exceptions in every policy.
+  They try selecting the branch where the source exists as possible
+  since copyup a large file will take long time. If it can't be,
+  ie. the branch where the source exists is readonly, then they will
+  follow the copyup policy.
+- There is an exception for rename(2) when the target exists.
+  If the rename target exists, aufs compares the index of the branches
+  where the source and the target exists and selects the higher
+  one. If the selected branch is readonly, then aufs follows the
+  copyup policy.
diff -Naur --no-dereference a/Documentation/filesystems/aufs/design/06dirren.dot b/Documentation/filesystems/aufs/design/06dirren.dot
--- a/Documentation/filesystems/aufs/design/06dirren.dot	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/filesystems/aufs/design/06dirren.dot	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,31 @@
+
+// to view this graph, run dot(1) command in GRAPHVIZ.
+
+digraph G {
+node [shape=box];
+whinfo [label="detailed info file\n(lower_brid_root-hinum, h_inum, namelen, old name)"];
+
+node [shape=oval];
+
+aufs_rename -> whinfo [label="store/remove"];
+
+node [shape=oval];
+inode_list [label="h_inum list in branch\ncache"];
+
+node [shape=box];
+whinode [label="h_inum list file"];
+
+node [shape=oval];
+brmgmt [label="br_add/del/mod/umount"];
+
+brmgmt -> inode_list [label="create/remove"];
+brmgmt -> whinode [label="load/store"];
+
+inode_list -> whinode [style=dashed,dir=both];
+
+aufs_rename -> inode_list [label="add/del"];
+
+aufs_lookup -> inode_list [label="search"];
+
+aufs_lookup -> whinfo [label="load/remove"];
+}
diff -Naur --no-dereference a/Documentation/filesystems/aufs/design/06dirren.txt b/Documentation/filesystems/aufs/design/06dirren.txt
--- a/Documentation/filesystems/aufs/design/06dirren.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/filesystems/aufs/design/06dirren.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,102 @@
+
+# Copyright (C) 2017-2020 Junjiro R. Okajima
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Special handling for renaming a directory (DIRREN)
+----------------------------------------------------------------------
+First, let's assume we have a simple usecase.
+
+- /u = /rw + /ro
+- /rw/dirA exists
+- /ro/dirA and /ro/dirA/file exist too
+- there is no dirB on both branches
+- a user issues rename("dirA", "dirB")
+
+Now, what should aufs behave against this rename(2)?
+There are a few possible cases.
+
+A. returns EROFS.
+   since dirA exists on a readonly branch which cannot be renamed.
+B. returns EXDEV.
+   it is possible to copy-up dirA (only the dir itself), but the child
+   entries ("file" in this case) should not be. it must be a bad
+   approach to copy-up recursively.
+C. returns a success.
+   even the branch /ro is readonly, aufs tries renaming it. Obviously it
+   is a violation of aufs' policy.
+D. construct an extra information which indicates that /ro/dirA should
+   be handled as the name of dirB.
+   overlayfs has a similar feature called REDIRECT.
+
+Until now, aufs implements the case B only which returns EXDEV, and
+expects the userspace application behaves like mv(1) which tries
+issueing rename(2) recursively.
+
+A new aufs feature called DIRREN is introduced which implements the case
+D. There are several "extra information" added.
+
+1. detailed info per renamed directory
+   path: /rw/dirB/$AUFS_WH_DR_INFO_PFX.<lower branch-id>
+2. the inode-number list of directories on a branch
+   path: /rw/dirB/$AUFS_WH_DR_BRHINO
+
+The filename of "detailed info per directory" represents the lower
+branch, and its format is
+- a type of the branch id
+  one of these.
+  + uuid (not implemented yet)
+  + fsid
+  + dev
+- the inode-number of the branch root dir
+
+And it contains these info in a single regular file.
+- magic number
+- branch's inode-number of the logically renamed dir
+- the name of the before-renamed dir
+
+The "detailed info per directory" file is created in aufs rename(2), and
+loaded in any lookup.
+The info is considered in lookup for the matching case only. Here
+"matching" means that the root of branch (in the info filename) is same
+to the current looking-up branch. After looking-up the before-renamed
+name, the inode-number is compared. And the matched dentry is used.
+
+The "inode-number list of directories" is a regular file which contains
+simply the inode-numbers on the branch. The file is created or updated
+in removing the branch, and loaded in adding the branch. Its lifetime is
+equal to the branch.
+The list is refered in lookup, and when the current target inode is
+found in the list, the aufs tries loading the "detailed info per
+directory" and get the changed and valid name of the dir.
+
+Theoretically these "extra informaiton" may be able to be put into XATTR
+in the dir inode. But aufs doesn't choose this way because
+1. XATTR may not be supported by the branch (or its configuration)
+2. XATTR may have its size limit.
+3. XATTR may be less easy to convert than a regular file, when the
+   format of the info is changed in the future.
+At the same time, I agree that the regular file approach is much slower
+than XATTR approach. So, in the future, aufs may take the XATTR or other
+better approach.
+
+This DIRREN feature is enabled by aufs configuration, and is activated
+by a new mount option.
+
+For the more complicated case, there is a work with UDBA option, which
+is to dected the direct access to the branches (by-passing aufs) and to
+maintain the cashes in aufs. Since a single cached aufs dentry may
+contains two names, before- and after-rename, the name comparision in
+UDBA handler may not work correctly. In this case, the behaviour will be
+equivalen to udba=reval case.
diff -Naur --no-dereference a/Documentation/filesystems/aufs/design/06fhsm.txt b/Documentation/filesystems/aufs/design/06fhsm.txt
--- a/Documentation/filesystems/aufs/design/06fhsm.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/filesystems/aufs/design/06fhsm.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,120 @@
+
+# Copyright (C) 2011-2020 Junjiro R. Okajima
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+
+File-based Hierarchical Storage Management (FHSM)
+----------------------------------------------------------------------
+Hierarchical Storage Management (or HSM) is a well-known feature in the
+storage world. Aufs provides this feature as file-based with multiple
+writable branches, based upon the principle of "Colder, the Lower".
+Here the word "colder" means that the less used files, and "lower" means
+that the position in the order of the stacked branches vertically.
+These multiple writable branches are prioritized, ie. the topmost one
+should be the fastest drive and be used heavily.
+
+o Characters in aufs FHSM story
+- aufs itself and a new branch attribute.
+- a new ioctl interface to move-down and to establish a connection with
+  the daemon ("move-down" is a converse of "copy-up").
+- userspace tool and daemon.
+
+The userspace daemon establishes a connection with aufs and waits for
+the notification. The notified information is very similar to struct
+statfs containing the number of consumed blocks and inodes.
+When the consumed blocks/inodes of a branch exceeds the user-specified
+upper watermark, the daemon activates its move-down process until the
+consumed blocks/inodes reaches the user-specified lower watermark.
+
+The actual move-down is done by aufs based upon the request from
+user-space since we need to maintain the inode number and the internal
+pointer arrays in aufs.
+
+Currently aufs FHSM handles the regular files only. Additionally they
+must not be hard-linked nor pseudo-linked.
+
+
+o Cowork of aufs and the user-space daemon
+  During the userspace daemon established the connection, aufs sends a
+  small notification to it whenever aufs writes something into the
+  writable branch. But it may cost high since aufs issues statfs(2)
+  internally. So user can specify a new option to cache the
+  info. Actually the notification is controlled by these factors.
+  + the specified cache time.
+  + classified as "force" by aufs internally.
+  Until the specified time expires, aufs doesn't send the info
+  except the forced cases. When aufs decide forcing, the info is always
+  notified to userspace.
+  For example, the number of free inodes is generally large enough and
+  the shortage of it happens rarely. So aufs doesn't force the
+  notification when creating a new file, directory and others. This is
+  the typical case which aufs doesn't force.
+  When aufs writes the actual filedata and the files consumes any of new
+  blocks, the aufs forces notifying.
+
+
+o Interfaces in aufs
+- New branch attribute.
+  + fhsm
+    Specifies that the branch is managed by FHSM feature. In other word,
+    participant in the FHSM.
+    When nofhsm is set to the branch, it will not be the source/target
+    branch of the move-down operation. This attribute is set
+    independently from coo and moo attributes, and if you want full
+    FHSM, you should specify them as well.
+- New mount option.
+  + fhsm_sec
+    Specifies a second to suppress many less important info to be
+    notified.
+- New ioctl.
+  + AUFS_CTL_FHSM_FD
+    create a new file descriptor which userspace can read the notification
+    (a subset of struct statfs) from aufs.
+- Module parameter 'brs'
+  It has to be set to 1. Otherwise the new mount option 'fhsm' will not
+  be set.
+- mount helpers /sbin/mount.aufs and /sbin/umount.aufs
+  When there are two or more branches with fhsm attributes,
+  /sbin/mount.aufs invokes the user-space daemon and /sbin/umount.aufs
+  terminates it. As a result of remounting and branch-manipulation, the
+  number of branches with fhsm attribute can be one. In this case,
+  /sbin/mount.aufs will terminate the user-space daemon.
+
+
+Finally the operation is done as these steps in kernel-space.
+- make sure that,
+  + no one else is using the file.
+  + the file is not hard-linked.
+  + the file is not pseudo-linked.
+  + the file is a regular file.
+  + the parent dir is not opaqued.
+- find the target writable branch.
+- make sure the file is not whiteout-ed by the upper (than the target)
+  branch.
+- make the parent dir on the target branch.
+- mutex lock the inode on the branch.
+- unlink the whiteout on the target branch (if exists).
+- lookup and create the whiteout-ed temporary name on the target branch.
+- copy the file as the whiteout-ed temporary name on the target branch.
+- rename the whiteout-ed temporary name to the original name.
+- unlink the file on the source branch.
+- maintain the internal pointer array and the external inode number
+  table (XINO).
+- maintain the timestamps and other attributes of the parent dir and the
+  file.
+
+And of course, in every step, an error may happen. So the operation
+should restore the original file state after an error happens.
diff -Naur --no-dereference a/Documentation/filesystems/aufs/design/06mmap.txt b/Documentation/filesystems/aufs/design/06mmap.txt
--- a/Documentation/filesystems/aufs/design/06mmap.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/filesystems/aufs/design/06mmap.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,72 @@
+
+# Copyright (C) 2005-2020 Junjiro R. Okajima
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+mmap(2) -- File Memory Mapping
+----------------------------------------------------------------------
+In aufs, the file-mapped pages are handled by a branch fs directly, no
+interaction with aufs. It means aufs_mmap() calls the branch fs's
+->mmap().
+This approach is simple and good, but there is one problem.
+Under /proc, several entries show the mmapped files by its path (with
+device and inode number), and the printed path will be the path on the
+branch fs's instead of virtual aufs's.
+This is not a problem in most cases, but some utilities lsof(1) (and its
+user) may expect the path on aufs.
+
+To address this issue, aufs adds a new member called vm_prfile in struct
+vm_area_struct (and struct vm_region). The original vm_file points to
+the file on the branch fs in order to handle everything correctly as
+usual. The new vm_prfile points to a virtual file in aufs, and the
+show-functions in procfs refers to vm_prfile if it is set.
+Also we need to maintain several other places where touching vm_file
+such like
+- fork()/clone() copies vma and the reference count of vm_file is
+  incremented.
+- merging vma maintains the ref count too.
+
+This is not a good approach. It just fakes the printed path. But it
+leaves all behaviour around f_mapping unchanged. This is surely an
+advantage.
+Actually aufs had adopted another complicated approach which calls
+generic_file_mmap() and handles struct vm_operations_struct. In this
+approach, aufs met a hard problem and I could not solve it without
+switching the approach.
+
+There may be one more another approach which is
+- bind-mount the branch-root onto the aufs-root internally
+- grab the new vfsmount (ie. struct mount)
+- lazy-umount the branch-root internally
+- in open(2) the aufs-file, open the branch-file with the hidden
+  vfsmount (instead of the original branch's vfsmount)
+- ideally this "bind-mount and lazy-umount" should be done atomically,
+  but it may be possible from userspace by the mount helper.
+
+Adding the internal hidden vfsmount and using it in opening a file, the
+file path under /proc will be printed correctly. This approach looks
+smarter, but is not possible I am afraid.
+- aufs-root may be bind-mount later. when it happens, another hidden
+  vfsmount will be required.
+- it is hard to get the chance to bind-mount and lazy-umount
+  + in kernel-space, FS can have vfsmount in open(2) via
+    file->f_path, and aufs can know its vfsmount. But several locks are
+    already acquired, and if aufs tries to bind-mount and lazy-umount
+    here, then it may cause a deadlock.
+  + in user-space, bind-mount doesn't invoke the mount helper.
+- since /proc shows dev and ino, aufs has to give vma these info. it
+  means a new member vm_prinode will be necessary. this is essentially
+  equivalent to vm_prfile described above.
+
+I have to give up this "looks-smater" approach.
diff -Naur --no-dereference a/Documentation/filesystems/aufs/design/06xattr.txt b/Documentation/filesystems/aufs/design/06xattr.txt
--- a/Documentation/filesystems/aufs/design/06xattr.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/filesystems/aufs/design/06xattr.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,96 @@
+
+# Copyright (C) 2014-2020 Junjiro R. Okajima
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+
+Listing XATTR/EA and getting the value
+----------------------------------------------------------------------
+For the inode standard attributes (owner, group, timestamps, etc.), aufs
+shows the values from the topmost existing file. This behaviour is good
+for the non-dir entries since the bahaviour exactly matches the shown
+information. But for the directories, aufs considers all the same named
+entries on the lower branches. Which means, if one of the lower entry
+rejects readdir call, then aufs returns an error even if the topmost
+entry allows it. This behaviour is necessary to respect the branch fs's
+security, but can make users confused since the user-visible standard
+attributes don't match the behaviour.
+To address this issue, aufs has a mount option called dirperm1 which
+checks the permission for the topmost entry only, and ignores the lower
+entry's permission.
+
+A similar issue can happen around XATTR.
+getxattr(2) and listxattr(2) families behave as if dirperm1 option is
+always set. Otherwise these very unpleasant situation would happen.
+- listxattr(2) may return the duplicated entries.
+- users may not be able to remove or reset the XATTR forever,
+
+
+XATTR/EA support in the internal (copy,move)-(up,down)
+----------------------------------------------------------------------
+Generally the extended attributes of inode are categorized as these.
+- "security" for LSM and capability.
+- "system" for posix ACL, 'acl' mount option is required for the branch
+  fs generally.
+- "trusted" for userspace, CAP_SYS_ADMIN is required.
+- "user" for userspace, 'user_xattr' mount option is required for the
+  branch fs generally.
+
+Moreover there are some other categories. Aufs handles these rather
+unpopular categories as the ordinary ones, ie. there is no special
+condition nor exception.
+
+In copy-up, the support for XATTR on the dst branch may differ from the
+src branch. In this case, the copy-up operation will get an error and
+the original user operation which triggered the copy-up will fail. It
+can happen that even all copy-up will fail.
+When both of src and dst branches support XATTR and if an error occurs
+during copying XATTR, then the copy-up should fail obviously. That is a
+good reason and aufs should return an error to userspace. But when only
+the src branch support that XATTR, aufs should not return an error.
+For example, the src branch supports ACL but the dst branch doesn't
+because the dst branch may natively un-support it or temporary
+un-support it due to "noacl" mount option. Of course, the dst branch fs
+may NOT return an error even if the XATTR is not supported. It is
+totally up to the branch fs.
+
+Anyway when the aufs internal copy-up gets an error from the dst branch
+fs, then aufs tries removing the just copied entry and returns the error
+to the userspace. The worst case of this situation will be all copy-up
+will fail.
+
+For the copy-up operation, there two basic approaches.
+- copy the specified XATTR only (by category above), and return the
+  error unconditionally if it happens.
+- copy all XATTR, and ignore the error on the specified category only.
+
+In order to support XATTR and to implement the correct behaviour, aufs
+chooses the latter approach and introduces some new branch attributes,
+"icexsec", "icexsys", "icextr", "icexusr", and "icexoth".
+They correspond to the XATTR namespaces (see above). Additionally, to be
+convenient, "icex" is also provided which means all "icex*" attributes
+are set (here the word "icex" stands for "ignore copy-error on XATTR").
+
+The meaning of these attributes is to ignore the error from setting
+XATTR on that branch.
+Note that aufs tries copying all XATTR unconditionally, and ignores the
+error from the dst branch according to the specified attributes.
+
+Some XATTR may have its default value. The default value may come from
+the parent dir or the environment. If the default value is set at the
+file creating-time, it will be overwritten by copy-up.
+Some contradiction may happen I am afraid.
+Do we need another attribute to stop copying XATTR? I am unsure. For
+now, aufs implements the branch attributes to ignore the error.
diff -Naur --no-dereference a/Documentation/filesystems/aufs/design/07export.txt b/Documentation/filesystems/aufs/design/07export.txt
--- a/Documentation/filesystems/aufs/design/07export.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/filesystems/aufs/design/07export.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,58 @@
+
+# Copyright (C) 2005-2020 Junjiro R. Okajima
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Export Aufs via NFS
+----------------------------------------------------------------------
+Here is an approach.
+- like xino/xib, add a new file 'xigen' which stores aufs inode
+  generation.
+- iget_locked(): initialize aufs inode generation for a new inode, and
+  store it in xigen file.
+- destroy_inode(): increment aufs inode generation and store it in xigen
+  file. it is necessary even if it is not unlinked, because any data of
+  inode may be changed by UDBA.
+- encode_fh(): for a root dir, simply return FILEID_ROOT. otherwise
+  build file handle by
+  + branch id (4 bytes)
+  + superblock generation (4 bytes)
+  + inode number (4 or 8 bytes)
+  + parent dir inode number (4 or 8 bytes)
+  + inode generation (4 bytes))
+  + return value of exportfs_encode_fh() for the parent on a branch (4
+    bytes)
+  + file handle for a branch (by exportfs_encode_fh())
+- fh_to_dentry():
+  + find the index of a branch from its id in handle, and check it is
+    still exist in aufs.
+  + 1st level: get the inode number from handle and search it in cache.
+  + 2nd level: if not found in cache, get the parent inode number from
+    the handle and search it in cache. and then open the found parent
+    dir, find the matching inode number by vfs_readdir() and get its
+    name, and call lookup_one_len() for the target dentry.
+  + 3rd level: if the parent dir is not cached, call
+    exportfs_decode_fh() for a branch and get the parent on a branch,
+    build a pathname of it, convert it a pathname in aufs, call
+    path_lookup(). now aufs gets a parent dir dentry, then handle it as
+    the 2nd level.
+  + to open the dir, aufs needs struct vfsmount. aufs keeps vfsmount
+    for every branch, but not itself. to get this, (currently) aufs
+    searches in current->nsproxy->mnt_ns list. it may not be a good
+    idea, but I didn't get other approach.
+  + test the generation of the gotten inode.
+- every inode operation: they may get EBUSY due to UDBA. in this case,
+  convert it into ESTALE for NFSD.
+- readdir(): call lockdep_on/off() because filldir in NFSD calls
+  lookup_one_len(), vfs_getattr(), encode_fh() and others.
diff -Naur --no-dereference a/Documentation/filesystems/aufs/design/08shwh.txt b/Documentation/filesystems/aufs/design/08shwh.txt
--- a/Documentation/filesystems/aufs/design/08shwh.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/filesystems/aufs/design/08shwh.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,52 @@
+
+# Copyright (C) 2005-2020 Junjiro R. Okajima
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Show Whiteout Mode (shwh)
+----------------------------------------------------------------------
+Generally aufs hides the name of whiteouts. But in some cases, to show
+them is very useful for users. For instance, creating a new middle layer
+(branch) by merging existing layers.
+
+(borrowing aufs1 HOW-TO from a user, Michael Towers)
+When you have three branches,
+- Bottom: 'system', squashfs (underlying base system), read-only
+- Middle: 'mods', squashfs, read-only
+- Top: 'overlay', ram (tmpfs), read-write
+
+The top layer is loaded at boot time and saved at shutdown, to preserve
+the changes made to the system during the session.
+When larger changes have been made, or smaller changes have accumulated,
+the size of the saved top layer data grows. At this point, it would be
+nice to be able to merge the two overlay branches ('mods' and 'overlay')
+and rewrite the 'mods' squashfs, clearing the top layer and thus
+restoring save and load speed.
+
+This merging is simplified by the use of another aufs mount, of just the
+two overlay branches using the 'shwh' option.
+# mount -t aufs -o ro,shwh,br:/livesys/overlay=ro+wh:/livesys/mods=rr+wh \
+	aufs /livesys/merge_union
+
+A merged view of these two branches is then available at
+/livesys/merge_union, and the new feature is that the whiteouts are
+visible!
+Note that in 'shwh' mode the aufs mount must be 'ro', which will disable
+writing to all branches. Also the default mode for all branches is 'ro'.
+It is now possible to save the combined contents of the two overlay
+branches to a new squashfs, e.g.:
+# mksquashfs /livesys/merge_union /path/to/newmods.squash
+
+This new squashfs archive can be stored on the boot device and the
+initramfs will use it to replace the old one at the next boot.
diff -Naur --no-dereference a/Documentation/filesystems/aufs/design/10dynop.txt b/Documentation/filesystems/aufs/design/10dynop.txt
--- a/Documentation/filesystems/aufs/design/10dynop.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/filesystems/aufs/design/10dynop.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,47 @@
+
+# Copyright (C) 2010-2020 Junjiro R. Okajima
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Dynamically customizable FS operations
+----------------------------------------------------------------------
+Generally FS operations (struct inode_operations, struct
+address_space_operations, struct file_operations, etc.) are defined as
+"static const", but it never means that FS have only one set of
+operation. Some FS have multiple sets of them. For instance, ext2 has
+three sets, one for XIP, for NOBH, and for normal.
+Since aufs overrides and redirects these operations, sometimes aufs has
+to change its behaviour according to the branch FS type. More importantly
+VFS acts differently if a function (member in the struct) is set or
+not. It means aufs should have several sets of operations and select one
+among them according to the branch FS definition.
+
+In order to solve this problem and not to affect the behaviour of VFS,
+aufs defines these operations dynamically. For instance, aufs defines
+dummy direct_IO function for struct address_space_operations, but it may
+not be set to the address_space_operations actually. When the branch FS
+doesn't have it, aufs doesn't set it to its address_space_operations
+while the function definition itself is still alive. So the behaviour
+itself will not change, and it will return an error when direct_IO is
+not set.
+
+The lifetime of these dynamically generated operation object is
+maintained by aufs branch object. When the branch is removed from aufs,
+the reference counter of the object is decremented. When it reaches
+zero, the dynamically generated operation object will be freed.
+
+This approach is designed to support AIO (io_submit), Direct I/O and
+XIP (DAX) mainly.
+Currently this approach is applied to address_space_operations for
+regular files only.
diff -Naur --no-dereference a/Documentation/filesystems/aufs/README b/Documentation/filesystems/aufs/README
--- a/Documentation/filesystems/aufs/README	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/filesystems/aufs/README	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,404 @@
+
+Aufs4 -- advanced multi layered unification filesystem version 4.x
+http://aufs.sf.net
+Junjiro R. Okajima
+
+
+0. Introduction
+----------------------------------------
+In the early days, aufs was entirely re-designed and re-implemented
+Unionfs Version 1.x series. Adding many original ideas, approaches,
+improvements and implementations, it becomes totally different from
+Unionfs while keeping the basic features.
+Recently, Unionfs Version 2.x series begin taking some of the same
+approaches to aufs1's.
+Unionfs is being developed by Professor Erez Zadok at Stony Brook
+University and his team.
+
+Aufs4 supports linux-4.0 and later, and for linux-3.x series try aufs3.
+If you want older kernel version support, try aufs2-2.6.git or
+aufs2-standalone.git repository, aufs1 from CVS on SourceForge.
+
+Note: it becomes clear that "Aufs was rejected. Let's give it up."
+      According to Christoph Hellwig, linux rejects all union-type
+      filesystems but UnionMount.
+<http://marc.info/?l=linux-kernel&m=123938533724484&w=2>
+
+PS. Al Viro seems have a plan to merge aufs as well as overlayfs and
+    UnionMount, and he pointed out an issue around a directory mutex
+    lock and aufs addressed it. But it is still unsure whether aufs will
+    be merged (or any other union solution).
+<http://marc.info/?l=linux-kernel&m=136312705029295&w=1>
+
+
+1. Features
+----------------------------------------
+- unite several directories into a single virtual filesystem. The member
+  directory is called as a branch.
+- you can specify the permission flags to the branch, which are 'readonly',
+  'readwrite' and 'whiteout-able.'
+- by upper writable branch, internal copyup and whiteout, files/dirs on
+  readonly branch are modifiable logically.
+- dynamic branch manipulation, add, del.
+- etc...
+
+Also there are many enhancements in aufs, such as:
+- test only the highest one for the directory permission (dirperm1)
+- copyup on open (coo=)
+- 'move' policy for copy-up between two writable branches, after
+  checking free space.
+- xattr, acl
+- readdir(3) in userspace.
+- keep inode number by external inode number table
+- keep the timestamps of file/dir in internal copyup operation
+- seekable directory, supporting NFS readdir.
+- whiteout is hardlinked in order to reduce the consumption of inodes
+  on branch
+- do not copyup, nor create a whiteout when it is unnecessary
+- revert a single systemcall when an error occurs in aufs
+- remount interface instead of ioctl
+- maintain /etc/mtab by an external command, /sbin/mount.aufs.
+- loopback mounted filesystem as a branch
+- kernel thread for removing the dir who has a plenty of whiteouts
+- support copyup sparse file (a file which has a 'hole' in it)
+- default permission flags for branches
+- selectable permission flags for ro branch, whether whiteout can
+  exist or not
+- export via NFS.
+- support <sysfs>/fs/aufs and <debugfs>/aufs.
+- support multiple writable branches, some policies to select one
+  among multiple writable branches.
+- a new semantics for link(2) and rename(2) to support multiple
+  writable branches.
+- no glibc changes are required.
+- pseudo hardlink (hardlink over branches)
+- allow a direct access manually to a file on branch, e.g. bypassing aufs.
+  including NFS or remote filesystem branch.
+- userspace wrapper for pathconf(3)/fpathconf(3) with _PC_LINK_MAX.
+- and more...
+
+Currently these features are dropped temporary from aufs4.
+See design/08plan.txt in detail.
+- nested mount, i.e. aufs as readonly no-whiteout branch of another aufs
+  (robr)
+- statistics of aufs thread (/sys/fs/aufs/stat)
+
+Features or just an idea in the future (see also design/*.txt),
+- reorder the branch index without del/re-add.
+- permanent xino files for NFSD
+- an option for refreshing the opened files after add/del branches
+- light version, without branch manipulation. (unnecessary?)
+- copyup in userspace
+- inotify in userspace
+- readv/writev
+
+
+2. Download
+----------------------------------------
+There are three GIT trees for aufs4, aufs4-linux.git,
+aufs4-standalone.git, and aufs-util.git. Note that there is no "4" in
+"aufs-util.git."
+While the aufs-util is always necessary, you need either of aufs4-linux
+or aufs4-standalone.
+
+The aufs4-linux tree includes the whole linux mainline GIT tree,
+git://git.kernel.org/.../torvalds/linux.git.
+And you cannot select CONFIG_AUFS_FS=m for this version, eg. you cannot
+build aufs4 as an external kernel module.
+Several extra patches are not included in this tree. Only
+aufs4-standalone tree contains them. They are described in the later
+section "Configuration and Compilation."
+
+On the other hand, the aufs4-standalone tree has only aufs source files
+and necessary patches, and you can select CONFIG_AUFS_FS=m.
+But you need to apply all aufs patches manually.
+
+You will find GIT branches whose name is in form of "aufs4.x" where "x"
+represents the linux kernel version, "linux-4.x". For instance,
+"aufs4.0" is for linux-4.0. For latest "linux-4.x-rcN", use
+"aufs4.x-rcN" branch.
+
+o aufs4-linux tree
+$ git clone --reference /your/linux/git/tree \
+	git://github.com/sfjro/aufs4-linux.git aufs4-linux.git
+- if you don't have linux GIT tree, then remove "--reference ..."
+$ cd aufs4-linux.git
+$ git checkout origin/aufs4.0
+
+Or You may want to directly git-pull aufs into your linux GIT tree, and
+leave the patch-work to GIT.
+$ cd /your/linux/git/tree
+$ git remote add aufs4 git://github.com/sfjro/aufs4-linux.git
+$ git fetch aufs4
+$ git checkout -b my4.0 v4.0
+$ (add your local change...)
+$ git pull aufs4 aufs4.0
+- now you have v4.0 + your_changes + aufs4.0 in you my4.0 branch.
+- you may need to solve some conflicts between your_changes and
+  aufs4.0. in this case, git-rerere is recommended so that you can
+  solve the similar conflicts automatically when you upgrade to 4.1 or
+  later in the future.
+
+o aufs4-standalone tree
+$ git clone git://github.com/sfjro/aufs4-standalone.git aufs4-standalone.git
+$ cd aufs4-standalone.git
+$ git checkout origin/aufs4.0
+
+o aufs-util tree
+$ git clone git://git.code.sf.net/p/aufs/aufs-util aufs-util.git
+- note that the public aufs-util.git is on SourceForge instead of
+  GitHUB.
+$ cd aufs-util.git
+$ git checkout origin/aufs4.0
+
+Note: The 4.x-rcN branch is to be used with `rc' kernel versions ONLY.
+The minor version number, 'x' in '4.x', of aufs may not always
+follow the minor version number of the kernel.
+Because changes in the kernel that cause the use of a new
+minor version number do not always require changes to aufs-util.
+
+Since aufs-util has its own minor version number, you may not be
+able to find a GIT branch in aufs-util for your kernel's
+exact minor version number.
+In this case, you should git-checkout the branch for the
+nearest lower number.
+
+For (an unreleased) example:
+If you are using "linux-4.10" and the "aufs4.10" branch
+does not exist in aufs-util repository, then "aufs4.9", "aufs4.8"
+or something numerically smaller is the branch for your kernel.
+
+Also you can view all branches by
+	$ git branch -a
+
+
+3. Configuration and Compilation
+----------------------------------------
+Make sure you have git-checkout'ed the correct branch.
+
+For aufs4-linux tree,
+- enable CONFIG_AUFS_FS.
+- set other aufs configurations if necessary.
+
+For aufs4-standalone tree,
+There are several ways to build.
+
+1.
+- apply ./aufs4-kbuild.patch to your kernel source files.
+- apply ./aufs4-base.patch too.
+- apply ./aufs4-mmap.patch too.
+- apply ./aufs4-standalone.patch too, if you have a plan to set
+  CONFIG_AUFS_FS=m. otherwise you don't need ./aufs4-standalone.patch.
+- copy ./{Documentation,fs,include/uapi/linux/aufs_type.h} files to your
+  kernel source tree. Never copy $PWD/include/uapi/linux/Kbuild.
+- enable CONFIG_AUFS_FS, you can select either
+  =m or =y.
+- and build your kernel as usual.
+- install the built kernel.
+  Note: Since linux-3.9, every filesystem module requires an alias
+  "fs-<fsname>". You should make sure that "fs-aufs" is listed in your
+  modules.aliases file if you set CONFIG_AUFS_FS=m.
+- install the header files too by "make headers_install" to the
+  directory where you specify. By default, it is $PWD/usr.
+  "make help" shows a brief note for headers_install.
+- and reboot your system.
+
+2.
+- module only (CONFIG_AUFS_FS=m).
+- apply ./aufs4-base.patch to your kernel source files.
+- apply ./aufs4-mmap.patch too.
+- apply ./aufs4-standalone.patch too.
+- build your kernel, don't forget "make headers_install", and reboot.
+- edit ./config.mk and set other aufs configurations if necessary.
+  Note: You should read $PWD/fs/aufs/Kconfig carefully which describes
+  every aufs configurations.
+- build the module by simple "make".
+  Note: Since linux-3.9, every filesystem module requires an alias
+  "fs-<fsname>". You should make sure that "fs-aufs" is listed in your
+  modules.aliases file.
+- you can specify ${KDIR} make variable which points to your kernel
+  source tree.
+- install the files
+  + run "make install" to install the aufs module, or copy the built
+    $PWD/aufs.ko to /lib/modules/... and run depmod -a (or reboot simply).
+  + run "make install_headers" (instead of headers_install) to install
+    the modified aufs header file (you can specify DESTDIR which is
+    available in aufs standalone version's Makefile only), or copy
+    $PWD/usr/include/linux/aufs_type.h to /usr/include/linux or wherever
+    you like manually. By default, the target directory is $PWD/usr.
+- no need to apply aufs4-kbuild.patch, nor copying source files to your
+  kernel source tree.
+
+Note: The header file aufs_type.h is necessary to build aufs-util
+      as well as "make headers_install" in the kernel source tree.
+      headers_install is subject to be forgotten, but it is essentially
+      necessary, not only for building aufs-util.
+      You may not meet problems without headers_install in some older
+      version though.
+
+And then,
+- read README in aufs-util, build and install it
+- note that your distribution may contain an obsoleted version of
+  aufs_type.h in /usr/include/linux or something. When you build aufs
+  utilities, make sure that your compiler refers the correct aufs header
+  file which is built by "make headers_install."
+- if you want to use readdir(3) in userspace or pathconf(3) wrapper,
+  then run "make install_ulib" too. And refer to the aufs manual in
+  detail.
+
+There several other patches in aufs4-standalone.git. They are all
+optional. When you meet some problems, they will help you.
+- aufs4-loopback.patch
+  Supports a nested loopback mount in a branch-fs. This patch is
+  unnecessary until aufs produces a message like "you may want to try
+  another patch for loopback file".
+- proc_mounts.patch
+  When there are many mountpoints and many mount(2)/umount(2) are
+  running, then /proc/mounts may not show the all mountpoints.  This
+  patch makes /proc/mounts always show the full mountpoints list.
+  If you don't want to apply this patch and meet such problem, then you
+  need to increase the value of 'ProcMounts_Times' make-variable in
+  aufs-util.git as a second best solution.
+- vfs-ino.patch
+  Modifies a system global kernel internal function get_next_ino() in
+  order to stop assigning 0 for an inode-number. Not directly related to
+  aufs, but recommended generally.
+- tmpfs-idr.patch
+  Keeps the tmpfs inode number as the lowest value. Effective to reduce
+  the size of aufs XINO files for tmpfs branch. Also it prevents the
+  duplication of inode number, which is important for backup tools and
+  other utilities. When you find aufs XINO files for tmpfs branch
+  growing too much, try this patch.
+- lockdep-debug.patch
+  Because aufs is not only an ordinary filesystem (callee of VFS), but
+  also a caller of VFS functions for branch filesystems, subclassing of
+  the internal locks for LOCKDEP is necessary. LOCKDEP is a debugging
+  feature of linux kernel. If you enable CONFIG_LOCKDEP, then you will
+  need to apply this debug patch to expand several constant values.
+  If you don't know what LOCKDEP is, then you don't have apply this
+  patch.
+
+
+4. Usage
+----------------------------------------
+At first, make sure aufs-util are installed, and please read the aufs
+manual, aufs.5 in aufs-util.git tree.
+$ man -l aufs.5
+
+And then,
+$ mkdir /tmp/rw /tmp/aufs
+# mount -t aufs -o br=/tmp/rw:${HOME} none /tmp/aufs
+
+Here is another example. The result is equivalent.
+# mount -t aufs -o br=/tmp/rw=rw:${HOME}=ro none /tmp/aufs
+  Or
+# mount -t aufs -o br:/tmp/rw none /tmp/aufs
+# mount -o remount,append:${HOME} /tmp/aufs
+
+Then, you can see whole tree of your home dir through /tmp/aufs. If
+you modify a file under /tmp/aufs, the one on your home directory is
+not affected, instead the same named file will be newly created under
+/tmp/rw. And all of your modification to a file will be applied to
+the one under /tmp/rw. This is called the file based Copy on Write
+(COW) method.
+Aufs mount options are described in aufs.5.
+If you run chroot or something and make your aufs as a root directory,
+then you need to customize the shutdown script. See the aufs manual in
+detail.
+
+Additionally, there are some sample usages of aufs which are a
+diskless system with network booting, and LiveCD over NFS.
+See sample dir in CVS tree on SourceForge.
+
+
+5. Contact
+----------------------------------------
+When you have any problems or strange behaviour in aufs, please let me
+know with:
+- /proc/mounts (instead of the output of mount(8))
+- /sys/module/aufs/*
+- /sys/fs/aufs/* (if you have them)
+- /debug/aufs/* (if you have them)
+- linux kernel version
+  if your kernel is not plain, for example modified by distributor,
+  the url where i can download its source is necessary too.
+- aufs version which was printed at loading the module or booting the
+  system, instead of the date you downloaded.
+- configuration (define/undefine CONFIG_AUFS_xxx)
+- kernel configuration or /proc/config.gz (if you have it)
+- LSM (linux security module, if you are using)
+- behaviour which you think to be incorrect
+- actual operation, reproducible one is better
+- mailto: aufs-users at lists.sourceforge.net
+
+Usually, I don't watch the Public Areas(Bugs, Support Requests, Patches,
+and Feature Requests) on SourceForge. Please join and write to
+aufs-users ML.
+
+
+6. Acknowledgements
+----------------------------------------
+Thanks to everyone who have tried and are using aufs, whoever
+have reported a bug or any feedback.
+
+Especially donators:
+Tomas Matejicek(slax.org) made a donation (much more than once).
+	Since Apr 2010, Tomas M (the author of Slax and Linux Live
+	scripts) is making "doubling" donations.
+	Unfortunately I cannot list all of the donators, but I really
+	appreciate.
+	It ends Aug 2010, but the ordinary donation URL is still available.
+	<http://sourceforge.net/donate/index.php?group_id=167503>
+Dai Itasaka made a donation (2007/8).
+Chuck Smith made a donation (2008/4, 10 and 12).
+Henk Schoneveld made a donation (2008/9).
+Chih-Wei Huang, ASUS, CTC donated Eee PC 4G (2008/10).
+Francois Dupoux made a donation (2008/11).
+Bruno Cesar Ribas and Luis Carlos Erpen de Bona, C3SL serves public
+	aufs2 GIT tree (2009/2).
+William Grant made a donation (2009/3).
+Patrick Lane made a donation (2009/4).
+The Mail Archive (mail-archive.com) made donations (2009/5).
+Nippy Networks (Ed Wildgoose) made a donation (2009/7).
+New Dream Network, LLC (www.dreamhost.com) made a donation (2009/11).
+Pavel Pronskiy made a donation (2011/2).
+Iridium and Inmarsat satellite phone retailer (www.mailasail.com), Nippy
+	Networks (Ed Wildgoose) made a donation for hardware (2011/3).
+Max Lekomcev (DOM-TV project) made a donation (2011/7, 12, 2012/3, 6 and
+11).
+Sam Liddicott made a donation (2011/9).
+Era Scarecrow made a donation (2013/4).
+Bor Ratajc made a donation (2013/4).
+Alessandro Gorreta made a donation (2013/4).
+POIRETTE Marc made a donation (2013/4).
+Alessandro Gorreta made a donation (2013/4).
+lauri kasvandik made a donation (2013/5).
+"pemasu from Finland" made a donation (2013/7).
+The Parted Magic Project made a donation (2013/9 and 11).
+Pavel Barta made a donation (2013/10).
+Nikolay Pertsev made a donation (2014/5).
+James B made a donation (2014/7 and 2015/7).
+Stefano Di Biase made a donation (2014/8).
+Daniel Epellei made a donation (2015/1).
+OmegaPhil made a donation (2016/1, 2018/4).
+Tomasz Szewczyk made a donation (2016/4).
+James Burry made a donation (2016/12).
+Carsten Rose made a donation (2018/9).
+Porteus Kiosk made a donation (2018/10).
+
+Thank you very much.
+Donations are always, including future donations, very important and
+helpful for me to keep on developing aufs.
+
+
+7.
+----------------------------------------
+If you are an experienced user, no explanation is needed. Aufs is
+just a linux filesystem.
+
+
+Enjoy!
+
+# Local variables: ;
+# mode: text;
+# End: ;
diff -Naur --no-dereference a/Documentation/PCI/endpoint/pci-test-ntb.txt b/Documentation/PCI/endpoint/pci-test-ntb.txt
--- a/Documentation/PCI/endpoint/pci-test-ntb.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/Documentation/PCI/endpoint/pci-test-ntb.txt	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,315 @@
+			       PCI NTB FUNCTION
+		    Kishon Vijay Abraham I <kishon@ti.com>
+
+PCI NTB Function allows two different systems (or hosts) to communicate
+with each other by configurig the endpoint instances in such a way that
+transactions from one system is routed to the other system.
+
+In the below diagram, PCI NTB function configures the SoC with multiple
+PCIe Endpoint (EP) instances in such a way that transaction from one EP
+controller is routed to the other EP controller. Once PCI NTB function
+configures the SoC with multiple EP instances, HOST1 and HOST2 can
+communicate with each other using SoC as a bridge.
+
+   +-------------+                                   +-------------+
+   |             |                                   |             |
+   |    HOST1    |                                   |    HOST2    |
+   |             |                                   |             |
+   +------^------+                                   +------^------+
+          |                                                 |
+          |                                                 |
++---------|-------------------------------------------------|---------+
+|  +------v------+                                   +------v------+  |
+|  |             |                                   |             |  |
+|  |     EP      |                                   |     EP      |  |
+|  | CONTROLLER1 |                                   | CONTROLLER2 |  |
+|  |             <----------------------------------->             |  |
+|  |             |                                   |             |  |
+|  |             |                                   |             |  |
+|  |             |  SoC With Multiple EP Instances   |             |  |
+|  |             |  (Configured using NTB Function)  |             |  |
+|  +-------------+                                   +-------------+  |
++---------------------------------------------------------------------+
+
+Constructs used for Implementing NTB:
+
+	*) Config Region
+	*) Self Scratchpad Registers
+	*) Peer Scratchpad Registers
+	*) Doorbell Registers
+	*) Memory Window
+
+Modeling Constructs:
+
+  There are 5 or more distinct regions (config, self scratchpad, peer
+scratchpad, doorbell, one or more memory windows) to be modeled to achieve
+NTB functionality. Atleast one memory window is required while more than
+one is permitted. All these regions should be mapped to BAR for hosts to
+access these regions.
+
+If one 32-bit BAR is allocated for each of these regions, the scheme would
+look like
+	BAR0 -> Config Region
+	BAR1 -> Self Scratchpad
+	BAR2 -> Peer Scratchpad
+	BAR3 -> Doorbell
+	BAR4 -> Memory Window 1
+	BAR5 -> Memory Window 2
+
+However if we allocate a separate BAR for each of the region, there would not
+be enough BARs for all the regions in a platform that supports only 64-bit
+BAR.
+
+In order to be be supported by most of the platforms, the regions should be
+packed and mapped to BARs in a way that provides NTB functionality and
+also making sure the hosts doesn't access any region that it is not supposed
+to.
+
+The following scheme is used in EPF NTB Function
+
+	BAR0 -> Config Region + Self Scratchpad
+	BAR1 -> Peer Scratchpad
+	BAR2 -> Doorbell + Memory Window 1
+	BAR3 -> Memory Window 2
+	BAR4 -> Memory Window 3
+	BAR4 -> Memory Window 4
+
+With this scheme, for the basic NTB functionality 3 BARs should be sufficient.
+
+Modeling Config/Scratchpad Region:
+
++-----------------+------->+------------------+        +-----------------+
+|       BAR0      |        |  CONFIG REGION   |        |       BAR0      |
++-----------------+----+   +------------------+<-------+-----------------+
+|       BAR1      |    |   |SCRATCHPAD REGION |        |       BAR1      |
++-----------------+    +-->+------------------+<-------+-----------------+
+|       BAR2      |            Local Memory            |       BAR2      |
++-----------------+                                    +-----------------+
+|       BAR3      |                                    |       BAR3      |
++-----------------+                                    +-----------------+
+|       BAR4      |                                    |       BAR4      |
++-----------------+                                    +-----------------+
+|       BAR5      |                                    |       BAR5      |
++-----------------+                                    +-----------------+
+  EP CONTROLLER 1                                        EP CONTROLLER 2
+
+Above diagram shows Config region + Scratchpad region for HOST1 (connected to
+EP controller 1) allocated in local memory. The HOST1 can access the config
+region and scratchpad region (self scratchpad) using BAR0 of EP controller 1.
+The peer host (HOST2 connected to EP controller 2) can also access this
+scratchpad region (peer scratchpad) using BAR1 of EP controller 2. This
+diagram shows the case where Config region and Scratchpad region is allocated
+for HOST1, however the same is applicable for HOST2.
+
+Modeling Doorbell/Memory Window 1:
+
++-----------------+    +----->+----------------+-----------+-----------------+
+|       BAR0      |    |      |   Doorbell 1   +-----------> MSI|X ADDRESS 1 |
++-----------------+    |      +----------------+           +-----------------+
+|       BAR1      |    |      |   Doorbell 2   +---------+ |                 |
++-----------------+    |      +----------------+         | |                 |
+|       BAR2      |    |      |   Doorbell 3   +-------+ | +-----------------+
++-----------------+    |      +----------------+       | +-> MSI|X ADDRESS 2 |
+|       BAR3      |    |      |   Doorbell 4   +-----+ |   +-----------------+
++----------------------+      +----------------+     | |   |                 |
+|       BAR4      |           |                |     | |   +-----------------+
++----------------------+      |      MW1       +---+ | +-->+ MSI|X ADDRESS 3||
+|       BAR5      |    |      |                |   | |     +-----------------+
++-----------------+    +----->-----------------+   | |     |                 |
+  EP CONTROLLER 1             |                |   | |     +-----------------+
+                              |                |   | +---->+ MSI|X ADDRESS 4 |
+                              +----------------+   |       +-----------------+
+                               EP CONTROLLER 2     |       |                 |
+                                 (OB SPACE)        |       |                 |
+                                                   +------->      MW1        |
+                                                           |                 |
+                                                           |                 |
+                                                           +-----------------+
+                                                           |                 |
+                                                           |                 |
+                                                           |                 |
+                                                           |                 |
+                                                           |                 |
+                                                           +-----------------+
+							    PCI Address Space
+							    (Managed by HOST2)
+
+Above diagram shows how the doorbell and memory window 1 is mapped so that
+HOST1 can raise doorbell interrupt on HOST2 and also how HOST1 can access
+buffers exposed by HOST2 using memory window1 (MW1). Here doorbell and
+memory window 1 regions are allocated in EP controller 2 outbound (OB) address
+space. Allocating and configuring BARs for doorbell and memory window1
+is done during the initialization phase of NTB endpoint function driver.
+Mapping from EP controller 2 OB space to PCI address space is done when HOST2
+sends CMD_CONFIGURE_MW/CMD_CONFIGURE_DOORBELL. The commands are explained
+below.
+
+Modeling Optional Memory Windows:
+
+This is modeled the same was as MW1 but each of the additional memory windows
+is mapped to separate BARs.
+
+Config Region:
+
+Config Region is a construct that is specific to NTB implemented using NTB
+Endpoint Function Driver. The host and endpoint side NTB function driver will
+exchange informatio with each other using this region. Config Region has
+Control/Status Registers for configuring the Endpoint Controller. Host can
+write into this region for configuring the outbound ATU and to indicate the
+link status. Endpoint can indicate the status of commands issued be host in
+this region. Endpoint can also indicate the scratchpad offset, number of
+memory windows to the host using this region.
+
+The format of Config Region is given below. Each of the fields here are 32
+bits.
+
+	+------------------------+
+	|         COMMAND        |
+	+------------------------+
+	|         ARGUMENT       |
+	+------------------------+
+	|         STATUS         |
+	+------------------------+
+	|         TOPOLOGY       |
+	+------------------------+
+	|    ADDRESS (LOWER 32)  |
+	+------------------------+
+	|    ADDRESS (UPPER 32)  |
+	+------------------------+
+	|           SIZE         |
+	+------------------------+
+	|  MEMORY WINDOW1 OFFSET |
+	+------------------------+
+	|   NO OF MEMORY WINDOW  |
+	+------------------------+
+	|       SPAD OFFSET      |
+	+------------------------+
+	|        SPAD COUNT      |
+	+------------------------+
+	|      DB ENTRY SIZE     |
+	+------------------------+
+	|         DB DATA        |
+	+------------------------+
+	|            :           |
+	+------------------------+
+	|            :           |
+	+------------------------+
+	|         DB DATA        |
+	+------------------------+
+
+
+  COMMAND:
+
+	NTB function supports three commands:
+
+	  CMD_CONFIGURE_DOORBELL (0x1): Command to configure doorbell. Before
+	invoking this command, the host should allocate and initialize
+	MSI/MSI-X vectors (i.e initialize the MSI/MSI-X capability in the
+	Endpoint). The endpoint on receiving this command will configure
+	the outbound ATU such that transaction to DB BAR will be routed
+	to the MSI/MSI-X address programmed by the host. The ARGUMENT
+	register should be populated with number of DBs to configure (in the
+	lower 16 bits) and if MSI or MSI-X should be configured (BIT 16).
+	(TODO: Add support for MSI-X).
+
+	  CMD_CONFIGURE_MW (0x2): Command to configure memory window. The
+	host invokes this command after allocating a buffer that can be
+	accessed by remote host. The allocated address should be programmed
+	in the ADDRESS register (64 bit), the size should be programmed in
+	the SIZE register and the memory window index should be programmed
+	in the ARGUMENT register. The endpoint on receiving this command
+	will configure the outbound ATU such that trasaction to MW BAR
+	will be routed to the address provided by the host.
+
+	  CMD_LINK_UP (0x3): Command to indicate an NTB application is
+	bound to the EP device on the host side. Once the endpoint
+	receives this command from both the hosts, the endpoint will
+	raise an LINK_UP event to both the hosts to indicate the hosts
+	can start communicating with each other.
+
+  ARGUMENT:
+
+	The value of this register is based on the commands issued in
+	command register. See COMMAND section for more information.
+
+  configuring memory window and to indicate the host side NTB application
+  has initialized.
+
+  TOPOLOGY:
+
+	Set to NTB_TOPO_B2B_USD for Primary interface
+	Set to NTB_TOPO_B2B_DSD for Secondary interface
+
+  ADDRESS/SIZE:
+
+	Address and Size to be used while configuring the memory window.
+	See "CMD_CONFIGURE_MW" for more info.
+
+  MEMORY WINDOW1 OFFSET:
+
+	Memory Window 1 and Doorbell registers are packed together in the
+	same BAR. The initial portion of the region will have doorbell
+	registers and the latter portion of the region is for memory window 1.
+	This register will specify the offset of the memory window 1.
+
+  NO OF MEMORY WINDOW:
+
+	Specifies the number of memory windows supported by the NTB device.
+
+  SPAD OFFSET:
+
+	Self scratchpad region and config region are packed together in the
+	same BAR. The initial portion of the will have config region and
+	the latter portion of the region is for self scratchpad. This
+	register will specify the offset of the self scratchpad registers.
+
+  SPAD COUNT:
+
+	Specifies the number of scratchpad registers supported by the NTB
+	device.
+
+  DB ENTRY SIZE:
+
+	Used to determine the offset within the DB BAR that should be written
+	in order to raise doorbell. EPF NTB can use either MSI/MSI-X to
+	ring doorbell (MSI-X support will be added later). MSI uses same
+	address for all the interrupts and MSI-X can provide different
+	addresses for different interrupts. The MSI/MSI-X address is provided
+	by the host and the address it gives is based on the MSI/MSI-X
+	implementation supported by the host. For instance, ARM platform
+	using GIC ITS will have same MSI-X address for all the interrupts.
+	In order to support all the combinations and use the same mechanism
+	for both MSI and MSI-X, EPF NTB allocates separate region in the
+	Outbound Address Space for each of the interrupts. This region will
+	be mapped to the MSI/MSI-X address provided by the host. If a host
+	provides the same address for all the interrupts, all the regions
+	will be translated to the same address. If a host provides different
+	address, the regions will be translated to different address. This
+	will ensure there is no difference while raising the doorbell.
+
+  DB DATA:
+
+	EPF NTB supports 32 interrupts. So there are 32 DB DATA registers.
+	This holds the MSI/MSI-X data that has to be written to MSI address
+	for raising doorbell interrupt. This will be populated by EPF NTB
+	while invoking CMD_CONFIGURE_MW.
+
+Scratchpad Registers:
+
+  Each host has it's own register space allocated in the memory of NTB EPC.
+  They are both readable and writable from both sides of the bridge. They
+  are used by applications built over NTB and can be used to pass control
+  and status information between both sides of a device.
+
+  Scratchpad registers has 2 parts
+	1) Self Scratchpad: Host's own register space
+	2) Peer Scratchpad: Remote host's register space.
+
+Doorbell Registers:
+
+  Registers using which one host can interrupt the other host.
+
+Memory Window:
+
+  Actual transfer of data between the two hosts will happen using the
+  memory window.
diff -Naur --no-dereference a/Documentation/pwm.txt b/Documentation/pwm.txt
--- a/Documentation/pwm.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/pwm.txt	2020-06-19 21:00:31.400227715 -0400
@@ -85,9 +85,11 @@
 
 The PWM channels are numbered using a per-chip index from 0 to npwm-1.
 
-When a PWM channel is exported a pwmX directory will be created in the
+When a PWM channel is exported a pwm-N:X directory will be created in the
 pwmchipN directory it is associated with, where X is the number of the
-channel that was exported. The following properties will then be available:
+channel that was exported. It will also be exposed at /sys/class/pwm/ and
+can be identified by the pwm_channel device type.
+The following properties will then be available:
 
   period
     The total period of the PWM signal (read/write).
diff -Naur --no-dereference a/Documentation/scsi/ufs.txt b/Documentation/scsi/ufs.txt
--- a/Documentation/scsi/ufs.txt	2020-01-09 04:19:10.000000000 -0500
+++ b/Documentation/scsi/ufs.txt	2020-06-19 21:00:31.400227715 -0400
@@ -128,6 +128,37 @@
 In this version of UFSHCD Query requests and power management
 functionality are not implemented.
 
+4. BSG Support
+------------------
+
+This transport driver supports exchanging UFS protocol information units
+(UPIUs) with a UFS device. Typically, user space will allocate
+struct ufs_bsg_request and struct ufs_bsg_reply (see ufs_bsg.h) as
+request_upiu and reply_upiu respectively.  Filling those UPIUs should
+be done in accordance with JEDEC spec UFS2.1 paragraph 10.7.
+*Caveat emptor*: The driver makes no further input validations and sends the
+UPIU to the device as it is.  Open the bsg device in /dev/ufs-bsg and
+send SG_IO with the applicable sg_io_v4:
+
+	io_hdr_v4.guard = 'Q';
+	io_hdr_v4.protocol = BSG_PROTOCOL_SCSI;
+	io_hdr_v4.subprotocol = BSG_SUB_PROTOCOL_SCSI_TRANSPORT;
+	io_hdr_v4.response = (__u64)reply_upiu;
+	io_hdr_v4.max_response_len = reply_len;
+	io_hdr_v4.request_len = request_len;
+	io_hdr_v4.request = (__u64)request_upiu;
+	if (dir == SG_DXFER_TO_DEV) {
+		io_hdr_v4.dout_xfer_len = (uint32_t)byte_cnt;
+		io_hdr_v4.dout_xferp = (uintptr_t)(__u64)buff;
+	} else {
+		io_hdr_v4.din_xfer_len = (uint32_t)byte_cnt;
+		io_hdr_v4.din_xferp = (uintptr_t)(__u64)buff;
+	}
+
+If you wish to read or write a descriptor, use the appropriate xferp of
+sg_io_v4.
+
+
 UFS Specifications can be found at,
 UFS - http://www.jedec.org/sites/default/files/docs/JESD220.pdf
 UFSHCI - http://www.jedec.org/sites/default/files/docs/JESD223.pdf
diff -Naur --no-dereference a/drivers/ata/ahci_platform.c b/drivers/ata/ahci_platform.c
--- a/drivers/ata/ahci_platform.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/ata/ahci_platform.c	2020-06-19 21:00:31.408227715 -0400
@@ -70,6 +70,9 @@
 	if (!port)
 		port = &ahci_port_info;
 
+	if (of_device_is_compatible(dev->of_node, "snps,dwc-ahci"))
+		hpriv->flags |= AHCI_HFLAG_32BIT_ONLY;
+
 	rc = ahci_platform_init_host(pdev, hpriv, port,
 				     &ahci_platform_sht);
 	if (rc)
diff -Naur --no-dereference a/drivers/base/platform.c b/drivers/base/platform.c
--- a/drivers/base/platform.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/base/platform.c	2020-06-19 21:00:31.408227715 -0400
@@ -80,6 +80,26 @@
 EXPORT_SYMBOL_GPL(platform_get_resource);
 
 /**
+ * devm_platform_ioremap_resource - call devm_ioremap_resource() for a platform
+ *				    device
+ *
+ * @pdev: platform device to use both for memory resource lookup as well as
+ *        resource managemend
+ * @index: resource index
+ */
+#ifdef CONFIG_HAS_IOMEM
+void __iomem *devm_platform_ioremap_resource(struct platform_device *pdev,
+					     unsigned int index)
+{
+	struct resource *res;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, index);
+	return devm_ioremap_resource(&pdev->dev, res);
+}
+EXPORT_SYMBOL_GPL(devm_platform_ioremap_resource);
+#endif /* CONFIG_HAS_IOMEM */
+
+/**
  * platform_get_irq - get an IRQ for a device
  * @dev: platform device
  * @num: IRQ number index
diff -Naur --no-dereference a/drivers/block/loop.c b/drivers/block/loop.c
--- a/drivers/block/loop.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/block/loop.c	2020-06-19 21:00:31.408227715 -0400
@@ -749,6 +749,24 @@
 	return error;
 }
 
+/*
+ * for AUFS
+ * no get/put for file.
+ */
+struct file *loop_backing_file(struct super_block *sb)
+{
+	struct file *ret;
+	struct loop_device *l;
+
+	ret = NULL;
+	if (MAJOR(sb->s_dev) == LOOP_MAJOR) {
+		l = sb->s_bdev->bd_disk->private_data;
+		ret = l->lo_backing_file;
+	}
+	return ret;
+}
+EXPORT_SYMBOL_GPL(loop_backing_file);
+
 /* loop sysfs attributes */
 
 static ssize_t loop_attr_show(struct device *dev, char *page,
diff -Naur --no-dereference a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig
--- a/drivers/char/hw_random/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/char/hw_random/Kconfig	2020-06-19 21:00:31.408227715 -0400
@@ -154,7 +154,7 @@
 
 config HW_RANDOM_OMAP
 	tristate "OMAP Random Number Generator support"
-	depends on ARCH_OMAP16XX || ARCH_OMAP2PLUS || ARCH_MVEBU
+	depends on ARCH_OMAP16XX || ARCH_OMAP2PLUS || ARCH_MVEBU || (ARCH_K3 && CRYPTO_DEV_SA2UL)
 	default HW_RANDOM
  	---help---
  	  This driver provides kernel-side support for the Random Number
diff -Naur --no-dereference a/drivers/char/hw_random/omap-rng.c b/drivers/char/hw_random/omap-rng.c
--- a/drivers/char/hw_random/omap-rng.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/char/hw_random/omap-rng.c	2020-06-19 21:00:31.408227715 -0400
@@ -29,6 +29,7 @@
 #include <linux/of_address.h>
 #include <linux/interrupt.h>
 #include <linux/clk.h>
+#include <linux/k3_sa2ul.h>
 
 #include <asm/io.h>
 
@@ -447,6 +448,11 @@
 	if (!priv)
 		return -ENOMEM;
 
+#if defined(CONFIG_CRYPTO_DEV_SA2UL_MODULE) || defined(CONFIG_CRYPTO_DEV_SA2UL)
+	ret = sa2ul_trng_enable(dev->parent);
+	if (ret)
+		return ret;
+#endif
 	priv->rng.read = omap_rng_do_read;
 	priv->rng.init = omap_rng_init;
 	priv->rng.cleanup = omap_rng_cleanup;
diff -Naur --no-dereference a/drivers/clk/clk.c b/drivers/clk/clk.c
--- a/drivers/clk/clk.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/clk/clk.c	2020-06-19 21:00:31.408227715 -0400
@@ -924,6 +924,99 @@
 }
 
 /**
+ * clk_gate_restore_context - restore context for poweroff
+ * @hw: the clk_hw pointer of clock whose state is to be restored
+ *
+ * The gate clocks restore context function enables or disables
+ * the clock based on the enable_count. This is done in cases
+ * where the clock context is lost and based on the enable_count
+ * the clock either needs to be enabled/disabled. This
+ * helps restore the state of gate clocks.
+ */
+void clk_gate_restore_context(struct clk_hw *hw)
+{
+	if (hw->clk->core->enable_count)
+		hw->clk->core->ops->enable(hw);
+	else
+		hw->clk->core->ops->disable(hw);
+}
+EXPORT_SYMBOL_GPL(clk_gate_restore_context);
+
+static int _clk_save_context(struct clk_core *clk)
+{
+	struct clk_core *child;
+	int ret = 0;
+
+	hlist_for_each_entry(child, &clk->children, child_node) {
+		ret = _clk_save_context(child);
+		if (ret < 0)
+			return ret;
+	}
+
+	if (clk->ops && clk->ops->save_context)
+		ret = clk->ops->save_context(clk->hw);
+
+	return ret;
+}
+
+static void _clk_restore_context(struct clk_core *clk)
+{
+	struct clk_core *child;
+
+	if (clk->ops && clk->ops->restore_context)
+		clk->ops->restore_context(clk->hw);
+
+	hlist_for_each_entry(child, &clk->children, child_node)
+		_clk_restore_context(child);
+}
+
+/**
+ * clk_save_context - save clock context for poweroff
+ *
+ * Saves the context of the clock register for powerstates in which the
+ * contents of the registers will be lost. Occurs deep within the suspend
+ * code.  Returns 0 on success.
+ */
+int clk_save_context(void)
+{
+	struct clk_core *clk;
+	int ret;
+
+	hlist_for_each_entry(clk, &clk_root_list, child_node) {
+		ret = _clk_save_context(clk);
+		if (ret < 0)
+			return ret;
+	}
+
+	hlist_for_each_entry(clk, &clk_orphan_list, child_node) {
+		ret = _clk_save_context(clk);
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(clk_save_context);
+
+/**
+ * clk_restore_context - restore clock context after poweroff
+ *
+ * Restore the saved clock context upon resume.
+ *
+ */
+void clk_restore_context(void)
+{
+	struct clk_core *clk;
+
+	hlist_for_each_entry(clk, &clk_root_list, child_node)
+		_clk_restore_context(clk);
+
+	hlist_for_each_entry(clk, &clk_orphan_list, child_node)
+		_clk_restore_context(clk);
+}
+EXPORT_SYMBOL_GPL(clk_restore_context);
+
+/**
  * clk_enable - ungate a clock
  * @clk: the clk being ungated
  *
@@ -3322,6 +3415,7 @@
 void clk_unregister(struct clk *clk)
 {
 	unsigned long flags;
+	struct clk_hw *hw;
 
 	if (!clk || WARN_ON_ONCE(IS_ERR(clk)))
 		return;
@@ -3363,7 +3457,11 @@
 		pr_warn("%s: unregistering protected clock: %s\n",
 					__func__, clk->core->name);
 
+	hw = clk->core->hw;
 	kref_put(&clk->core->ref, __clk_release);
+	__clk_free_clk(clk);
+	hw->clk = NULL;
+
 unlock:
 	clk_prepare_unlock();
 }
diff -Naur --no-dereference a/drivers/clk/clk-devres.c b/drivers/clk/clk-devres.c
--- a/drivers/clk/clk-devres.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/clk/clk-devres.c	2020-06-19 21:00:31.408227715 -0400
@@ -34,6 +34,17 @@
 }
 EXPORT_SYMBOL(devm_clk_get);
 
+struct clk *devm_clk_get_optional(struct device *dev, const char *id)
+{
+	struct clk *clk = devm_clk_get(dev, id);
+
+	if (clk == ERR_PTR(-ENOENT))
+		return NULL;
+
+	return clk;
+}
+EXPORT_SYMBOL(devm_clk_get_optional);
+
 struct clk_bulk_devres {
 	struct clk_bulk_data *clks;
 	int num_clks;
diff -Naur --no-dereference a/drivers/clk/keystone/Kconfig b/drivers/clk/keystone/Kconfig
--- a/drivers/clk/keystone/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/clk/keystone/Kconfig	2020-06-19 21:00:31.408227715 -0400
@@ -14,3 +14,11 @@
 	  This adds the clock driver support over TI System Control Interface.
 	  If you wish to use clock resources from the PMMC firmware, say Y.
 	  Otherwise, say N.
+
+config TI_SYSCON_CLK
+	tristate "Syscon based clock driver for K2/K3 SoCs"
+	depends on (ARCH_KEYSTONE || ARCH_K3 || COMPILE_TEST) && OF
+	default (ARCH_KEYSTONE || ARCH_K3)
+	help
+	  This adds clock driver support for syscon based gate
+	  clocks on TI's K2 and K3 SoCs.
diff -Naur --no-dereference a/drivers/clk/keystone/Makefile b/drivers/clk/keystone/Makefile
--- a/drivers/clk/keystone/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/clk/keystone/Makefile	2020-06-19 21:00:31.408227715 -0400
@@ -1,2 +1,3 @@
 obj-$(CONFIG_COMMON_CLK_KEYSTONE)	+= pll.o gate.o
 obj-$(CONFIG_TI_SCI_CLK)		+= sci-clk.o
+obj-$(CONFIG_TI_SYSCON_CLK)		+= syscon-clk.o
diff -Naur --no-dereference a/drivers/clk/keystone/sci-clk.c b/drivers/clk/keystone/sci-clk.c
--- a/drivers/clk/keystone/sci-clk.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/clk/keystone/sci-clk.c	2020-06-19 21:00:31.408227715 -0400
@@ -23,6 +23,7 @@
 #include <linux/slab.h>
 #include <linux/soc/ti/ti_sci_protocol.h>
 #include <linux/bsearch.h>
+#include <linux/list_sort.h>
 
 #define SCI_CLK_SSC_ENABLE		BIT(0)
 #define SCI_CLK_ALLOW_FREQ_CHANGE	BIT(1)
@@ -52,14 +53,16 @@
  * @num_parents: Number of parents for this clock
  * @provider:	 Master clock provider
  * @flags:	 Flags for the clock
+ * @node:	 Link for handling clocks probed via DT
  */
 struct sci_clk {
 	struct clk_hw hw;
 	u16 dev_id;
-	u8 clk_id;
-	u8 num_parents;
+	u32 clk_id;
+	u32 num_parents;
 	struct sci_clk_provider *provider;
 	u8 flags;
+	struct list_head node;
 };
 
 #define to_sci_clk(_hw) container_of(_hw, struct sci_clk, hw)
@@ -206,7 +209,7 @@
 	struct sci_clk *clk = to_sci_clk(hw);
 
 	return clk->provider->ops->set_freq(clk->provider->sci, clk->dev_id,
-					    clk->clk_id, rate, rate, rate);
+					    clk->clk_id, 0, rate, ULONG_MAX);
 }
 
 /**
@@ -218,11 +221,11 @@
 static u8 sci_clk_get_parent(struct clk_hw *hw)
 {
 	struct sci_clk *clk = to_sci_clk(hw);
-	u8 parent_id;
+	u32 parent_id = 0;
 	int ret;
 
 	ret = clk->provider->ops->get_parent(clk->provider->sci, clk->dev_id,
-					     clk->clk_id, &parent_id);
+					     clk->clk_id, (void *)&parent_id);
 	if (ret) {
 		dev_err(clk->provider->dev,
 			"get-parent failed for dev=%d, clk=%d, ret=%d\n",
@@ -230,7 +233,9 @@
 		return 0;
 	}
 
-	return parent_id - clk->clk_id - 1;
+	parent_id = parent_id - clk->clk_id - 1;
+
+	return (u8)parent_id;
 }
 
 /**
@@ -280,8 +285,8 @@
 	int i;
 	int ret = 0;
 
-	name = kasprintf(GFP_KERNEL, "%s:%d:%d", dev_name(provider->dev),
-			 sci_clk->dev_id, sci_clk->clk_id);
+	name = kasprintf(GFP_KERNEL, "clk:%d:%d", sci_clk->dev_id,
+			 sci_clk->clk_id);
 
 	init.name = name;
 
@@ -306,8 +311,7 @@
 		for (i = 0; i < sci_clk->num_parents; i++) {
 			char *parent_name;
 
-			parent_name = kasprintf(GFP_KERNEL, "%s:%d:%d",
-						dev_name(provider->dev),
+			parent_name = kasprintf(GFP_KERNEL, "clk:%d:%d",
 						sci_clk->dev_id,
 						sci_clk->clk_id + 1 + i);
 			if (!parent_name) {
@@ -353,6 +357,15 @@
 	return -1;
 }
 
+static int _cmp_sci_clk_list(void *priv, struct list_head *a,
+			     struct list_head *b)
+{
+	struct sci_clk *ca = container_of(a, struct sci_clk, node);
+	struct sci_clk *cb = container_of(b, struct sci_clk, node);
+
+	return _cmp_sci_clk(ca, &cb);
+}
+
 /**
  * sci_clk_get - Xlate function for getting clock handles
  * @clkspec: device tree clock specifier
@@ -404,22 +417,8 @@
 };
 MODULE_DEVICE_TABLE(of, ti_sci_clk_of_match);
 
-/**
- * ti_sci_clk_probe - Probe function for the TI SCI clock driver
- * @pdev: platform device pointer to be probed
- *
- * Probes the TI SCI clock device. Allocates a new clock provider
- * and registers this to the common clock framework. Also applies
- * any required flags to the identified clocks via clock lists
- * supplied from DT. Returns 0 for success, negative error value
- * for failure.
- */
-static int ti_sci_clk_probe(struct platform_device *pdev)
+static int ti_sci_scan_clocks_from_fw(struct sci_clk_provider *provider)
 {
-	struct device *dev = &pdev->dev;
-	struct device_node *np = dev->of_node;
-	struct sci_clk_provider *provider;
-	const struct ti_sci_handle *handle;
 	int ret;
 	int num_clks = 0;
 	struct sci_clk **clks = NULL;
@@ -428,28 +427,18 @@
 	int max_clks = 0;
 	int clk_id = 0;
 	int dev_id = 0;
-	u8 num_parents;
+	u32 num_parents = 0;
 	int gap_size = 0;
-
-	handle = devm_ti_sci_get_handle(dev);
-	if (IS_ERR(handle))
-		return PTR_ERR(handle);
-
-	provider = devm_kzalloc(dev, sizeof(*provider), GFP_KERNEL);
-	if (!provider)
-		return -ENOMEM;
-
-	provider->sci = handle;
-	provider->ops = &handle->ops.clk_ops;
-	provider->dev = dev;
+	struct device *dev = provider->dev;
 
 	while (1) {
 		ret = provider->ops->get_num_parents(provider->sci, dev_id,
-						     clk_id, &num_parents);
+						     clk_id,
+						     (void *)&num_parents);
 		if (ret) {
 			gap_size++;
 			if (!clk_id) {
-				if (gap_size >= 5)
+				if (gap_size >= 50)
 					break;
 				dev_id++;
 			} else {
@@ -502,13 +491,173 @@
 
 	devm_kfree(dev, clks);
 
+	return 0;
+}
+
+static int ti_sci_scan_clocks_from_dt(struct sci_clk_provider *provider)
+{
+	struct device *dev = provider->dev;
+	struct device_node *np = NULL;
+	int ret;
+	int index;
+	struct of_phandle_args args;
+	struct list_head clks;
+	struct sci_clk *sci_clk, *prev;
+	int num_clks = 0;
+	int num_parents;
+	int clk_id;
+
+	INIT_LIST_HEAD(&clks);
+
+	while (1) {
+		np = of_find_node_with_property(np, "clocks");
+		if (!np)
+			break;
+
+		index = 0;
+
+		do {
+			ret = of_parse_phandle_with_args(np, "clocks",
+							 "#clock-cells", index,
+							 &args);
+			if (ret)
+				break;
+
+			if (args.args_count == 2 && args.np == dev->of_node) {
+				sci_clk = devm_kzalloc(dev, sizeof(*sci_clk),
+						       GFP_KERNEL);
+				if (!sci_clk)
+					return -ENOMEM;
+
+				sci_clk->dev_id = args.args[0];
+				sci_clk->clk_id = args.args[1];
+				sci_clk->provider = provider;
+				provider->ops->
+					get_num_parents(provider->sci,
+							sci_clk->dev_id,
+							sci_clk->clk_id,
+							(void *)&sci_clk->num_parents);
+				list_add_tail(&sci_clk->node, &clks);
+
+				num_clks++;
+
+				num_parents = sci_clk->num_parents;
+				if (num_parents == 1)
+					num_parents = 0;
+
+				/*
+				 * Linux kernel has inherent limitation
+				 * of 255 clock parents at the moment.
+				 * Right now, it is not expected that
+				 * any mux clock from sci-clk driver
+				 * would exceed that limit either, but
+				 * the ABI basically provides that
+				 * possibility. Print out a warning if
+				 * this happens for any clock.
+				 */
+				if (num_parents >= 255) {
+					dev_warn(dev, "too many parents for dev=%d, clk=%d (%d), cropping to 255.\n",
+						 sci_clk->dev_id,
+						 sci_clk->clk_id, num_parents);
+					num_parents = 255;
+				}
+
+				clk_id = args.args[1] + 1;
+
+				while (num_parents--) {
+					sci_clk = devm_kzalloc(dev,
+							       sizeof(*sci_clk),
+							       GFP_KERNEL);
+					if (!sci_clk)
+						return -ENOMEM;
+					sci_clk->dev_id = args.args[0];
+					sci_clk->clk_id = clk_id++;
+					sci_clk->provider = provider;
+					list_add_tail(&sci_clk->node, &clks);
+
+					num_clks++;
+				}
+			}
+
+			index++;
+		} while (args.np);
+	}
+
+	list_sort(NULL, &clks, _cmp_sci_clk_list);
+
+	provider->clocks = devm_kmalloc_array(dev, num_clks, sizeof(sci_clk),
+					      GFP_KERNEL);
+	if (!provider->clocks)
+		return -ENOMEM;
+
+	num_clks = 0;
+	prev = NULL;
+
+	list_for_each_entry(sci_clk, &clks, node) {
+		if (prev && prev->dev_id == sci_clk->dev_id &&
+		    prev->clk_id == sci_clk->clk_id)
+			continue;
+
+		provider->clocks[num_clks++] = sci_clk;
+		prev = sci_clk;
+	}
+
+	provider->num_clocks = num_clks;
+
+	return 0;
+}
+
+/**
+ * ti_sci_clk_probe - Probe function for the TI SCI clock driver
+ * @pdev: platform device pointer to be probed
+ *
+ * Probes the TI SCI clock device. Allocates a new clock provider
+ * and registers this to the common clock framework. Also applies
+ * any required flags to the identified clocks via clock lists
+ * supplied from DT. Returns 0 for success, negative error value
+ * for failure.
+ */
+static int ti_sci_clk_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct sci_clk_provider *provider;
+	const struct ti_sci_handle *handle;
+	int ret;
+
+	handle = devm_ti_sci_get_handle(dev);
+	if (IS_ERR(handle))
+		return PTR_ERR(handle);
+
+	provider = devm_kzalloc(dev, sizeof(*provider), GFP_KERNEL);
+	if (!provider)
+		return -ENOMEM;
+
+	provider->sci = handle;
+	provider->ops = &handle->ops.clk_ops;
+	provider->dev = dev;
+
+	if (of_property_read_bool(np, "ti,scan-clocks-from-dt")) {
+		ret = ti_sci_scan_clocks_from_dt(provider);
+		if (ret) {
+			dev_err(dev, "scan clocks from DT failed: %d\n", ret);
+			return ret;
+		}
+	} else {
+		ret = ti_sci_scan_clocks_from_fw(provider);
+		if (ret) {
+			dev_err(dev, "scan clocks from FW failed: %d\n", ret);
+			return ret;
+		}
+	}
+
 	ret = ti_sci_init_clocks(provider);
 	if (ret) {
-		pr_err("ti-sci-init-clocks failed.\n");
+		dev_err(dev, "init-clocks failed: %d\n", ret);
 		return ret;
 	}
 
-	return of_clk_add_hw_provider(np, sci_clk_get, provider);
+	return of_clk_add_hw_provider(dev->of_node, sci_clk_get, provider);
 }
 
 /**
diff -Naur --no-dereference a/drivers/clk/keystone/syscon-clk.c b/drivers/clk/keystone/syscon-clk.c
--- a/drivers/clk/keystone/syscon-clk.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/clk/keystone/syscon-clk.c	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,179 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+//
+
+#include <linux/clk-provider.h>
+#include <linux/clk.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+struct ti_syscon_gate_clk_priv {
+	struct clk_hw hw;
+	struct regmap *regmap;
+	u32 reg;
+	u32 idx;
+};
+
+struct ti_syscon_gate_clk_data {
+	char *name;
+	u32 offset;
+	u32 bit_idx;
+};
+
+static struct
+ti_syscon_gate_clk_priv *to_ti_syscon_gate_clk_priv(struct clk_hw *hw)
+{
+	return container_of(hw, struct ti_syscon_gate_clk_priv, hw);
+}
+
+static int ti_syscon_gate_clk_enable(struct clk_hw *hw)
+{
+	struct ti_syscon_gate_clk_priv *priv = to_ti_syscon_gate_clk_priv(hw);
+
+	return regmap_write_bits(priv->regmap, priv->reg, priv->idx,
+				 priv->idx);
+}
+
+static void ti_syscon_gate_clk_disable(struct clk_hw *hw)
+{
+	struct ti_syscon_gate_clk_priv *priv = to_ti_syscon_gate_clk_priv(hw);
+
+	regmap_write_bits(priv->regmap, priv->reg, priv->idx, 0);
+}
+
+static int ti_syscon_gate_clk_is_enabled(struct clk_hw *hw)
+{
+	unsigned int val;
+	struct ti_syscon_gate_clk_priv *priv = to_ti_syscon_gate_clk_priv(hw);
+
+	regmap_read(priv->regmap, priv->reg, &val);
+
+	return !!(val & priv->idx);
+}
+
+static const struct clk_ops ti_syscon_gate_clk_ops = {
+	.enable		= ti_syscon_gate_clk_enable,
+	.disable	= ti_syscon_gate_clk_disable,
+	.is_enabled	= ti_syscon_gate_clk_is_enabled,
+};
+
+static struct clk_hw
+*ti_syscon_gate_clk_register(struct device *dev, struct regmap *regmap,
+			     const struct ti_syscon_gate_clk_data *data)
+{
+	struct ti_syscon_gate_clk_priv *priv;
+	struct clk_init_data init;
+	int ret;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = data->name;
+	init.ops = &ti_syscon_gate_clk_ops;
+	init.parent_names = NULL;
+	init.num_parents = 0;
+	init.flags = 0;
+
+	priv->regmap = regmap;
+	priv->reg = data->offset;
+	priv->idx = BIT(data->bit_idx);
+	priv->hw.init = &init;
+
+	ret = devm_clk_hw_register(dev, &priv->hw);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return &priv->hw;
+}
+
+static int ti_syscon_gate_clk_probe(struct platform_device *pdev)
+{
+	const struct ti_syscon_gate_clk_data *data, *p;
+	struct clk_hw_onecell_data *hw_data;
+	struct device *dev = &pdev->dev;
+	struct device_node *parent;
+	struct regmap *regmap;
+	int num_clks = 0;
+	int i;
+
+	data = of_device_get_match_data(dev);
+	if (!data)
+		return -EINVAL;
+
+	parent = of_get_parent(dev->of_node);
+	regmap = syscon_node_to_regmap(parent);
+	of_node_put(parent);
+	if (IS_ERR(regmap)) {
+		if (PTR_ERR(regmap) == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
+		dev_err(dev, "failed to find parent regmap\n");
+		return PTR_ERR(regmap);
+	}
+
+	for (p = data; p->name; p++)
+		num_clks++;
+
+	hw_data = devm_kzalloc(dev, struct_size(hw_data, hws, num_clks),
+			       GFP_KERNEL);
+	if (!hw_data)
+		return -ENOMEM;
+
+	hw_data->num = num_clks;
+
+	for (i = 0; i < num_clks; i++) {
+		hw_data->hws[i] = ti_syscon_gate_clk_register(dev, regmap,
+							      &data[i]);
+		if (IS_ERR(hw_data->hws[i]))
+			dev_err(dev, "failed to register %s",
+				data[i].name);
+	}
+
+	return devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get,
+					   hw_data);
+}
+
+#define TI_SYSCON_CLK_GATE(_name, _offset, _bit_idx)	\
+	{						\
+		.name = _name,				\
+		.offset = (_offset),			\
+		.bit_idx = (_bit_idx),			\
+	}
+
+static const struct ti_syscon_gate_clk_data am654_clk_data[] = {
+	TI_SYSCON_CLK_GATE("ehrpwm_tbclk0", 0x0, 0),
+	TI_SYSCON_CLK_GATE("ehrpwm_tbclk1", 0x4, 0),
+	TI_SYSCON_CLK_GATE("ehrpwm_tbclk2", 0x8, 0),
+	TI_SYSCON_CLK_GATE("ehrpwm_tbclk3", 0xc, 0),
+	TI_SYSCON_CLK_GATE("ehrpwm_tbclk4", 0x10, 0),
+	TI_SYSCON_CLK_GATE("ehrpwm_tbclk5", 0x14, 0),
+	{ /* Sentinel */ },
+};
+
+static const struct of_device_id ti_syscon_gate_clk_ids[] = {
+	{
+		.compatible = "ti,am654-ehrpwm-tbclk",
+		.data = &am654_clk_data,
+	},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, ti_syscon_gate_clk_ids);
+
+static struct platform_driver ti_syscon_gate_clk_driver = {
+	.probe = ti_syscon_gate_clk_probe,
+	.driver = {
+		.name = "ti-syscon-gate-clk",
+		.of_match_table = ti_syscon_gate_clk_ids,
+	},
+};
+
+module_platform_driver(ti_syscon_gate_clk_driver);
+
+MODULE_AUTHOR("Vignesh Raghavendra <vigneshr@ti.com>");
+MODULE_DESCRIPTION("Syscon backed gate-clock driver");
+MODULE_LICENSE("GPL");
diff -Naur --no-dereference a/drivers/clk/ti/autoidle.c b/drivers/clk/ti/autoidle.c
--- a/drivers/clk/ti/autoidle.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/clk/ti/autoidle.c	2020-06-19 21:00:31.408227715 -0400
@@ -47,6 +47,9 @@
 {
 	struct clk_hw_omap *c;
 
+	if (!clk)
+		return -EINVAL;
+
 	c = to_clk_hw_omap(__clk_get_hw(clk));
 	if (c->ops && c->ops->deny_idle)
 		c->ops->deny_idle(c);
@@ -63,6 +66,9 @@
 {
 	struct clk_hw_omap *c;
 
+	if (!clk)
+		return -EINVAL;
+
 	c = to_clk_hw_omap(__clk_get_hw(clk));
 	if (c->ops && c->ops->allow_idle)
 		c->ops->allow_idle(c);
diff -Naur --no-dereference a/drivers/clk/ti/clk-33xx.c b/drivers/clk/ti/clk-33xx.c
--- a/drivers/clk/ti/clk-33xx.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/clk/ti/clk-33xx.c	2020-06-19 21:00:31.408227715 -0400
@@ -83,7 +83,7 @@
 	{ AM3_EPWMSS2_CLKCTRL, NULL, CLKF_SW_SUP, "l4ls_gclk" },
 	{ AM3_L3_INSTR_CLKCTRL, NULL, CLKF_SW_SUP, "l3_gclk", "l3_clkdm" },
 	{ AM3_L3_MAIN_CLKCTRL, NULL, CLKF_SW_SUP, "l3_gclk", "l3_clkdm" },
-	{ AM3_PRUSS_CLKCTRL, NULL, CLKF_SW_SUP, "pruss_ocp_gclk", "pruss_ocp_clkdm" },
+	{ AM3_PRUSS_CLKCTRL, NULL, CLKF_SW_SUP | CLKF_NO_IDLEST, "pruss_ocp_gclk", "pruss_ocp_clkdm" },
 	{ AM3_TIMER5_CLKCTRL, NULL, CLKF_SW_SUP, "timer5_fck" },
 	{ AM3_TIMER6_CLKCTRL, NULL, CLKF_SW_SUP, "timer6_fck" },
 	{ AM3_MMC2_CLKCTRL, NULL, CLKF_SW_SUP, "mmc_clk" },
@@ -181,7 +181,7 @@
 };
 
 static const struct omap_clkctrl_reg_data am3_gfx_l3_clkctrl_regs[] __initconst = {
-	{ AM3_GFX_CLKCTRL, NULL, CLKF_SW_SUP, "gfx_fck_div_ck" },
+	{ AM3_GFX_CLKCTRL, NULL, CLKF_SW_SUP | CLKF_NO_IDLEST, "gfx_fck_div_ck" },
 	{ 0 },
 };
 
diff -Naur --no-dereference a/drivers/clk/ti/clk-43xx.c b/drivers/clk/ti/clk-43xx.c
--- a/drivers/clk/ti/clk-43xx.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/clk/ti/clk-43xx.c	2020-06-19 21:00:31.408227715 -0400
@@ -65,7 +65,7 @@
 };
 
 static const struct omap_clkctrl_reg_data am4_gfx_l3_clkctrl_regs[] __initconst = {
-	{ AM4_GFX_CLKCTRL, NULL, CLKF_SW_SUP, "gfx_fck_div_ck" },
+	{ AM4_GFX_CLKCTRL, NULL, CLKF_SW_SUP | CLKF_NO_IDLEST, "gfx_fck_div_ck" },
 	{ 0 },
 };
 
@@ -140,7 +140,7 @@
 	{ AM4_QSPI_CLKCTRL, NULL, CLKF_SW_SUP, "l3s_gclk", "l3s_clkdm" },
 	{ AM4_USB_OTG_SS0_CLKCTRL, am4_usb_otg_ss0_bit_data, CLKF_SW_SUP, "l3s_gclk", "l3s_clkdm" },
 	{ AM4_USB_OTG_SS1_CLKCTRL, am4_usb_otg_ss1_bit_data, CLKF_SW_SUP, "l3s_gclk", "l3s_clkdm" },
-	{ AM4_PRUSS_CLKCTRL, NULL, CLKF_SW_SUP, "pruss_ocp_gclk", "pruss_ocp_clkdm" },
+	{ AM4_PRUSS_CLKCTRL, NULL, CLKF_SW_SUP | CLKF_NO_IDLEST, "pruss_ocp_gclk", "pruss_ocp_clkdm" },
 	{ AM4_L4_LS_CLKCTRL, NULL, CLKF_SW_SUP, "l4ls_gclk" },
 	{ AM4_D_CAN0_CLKCTRL, NULL, CLKF_SW_SUP, "dcan0_fck" },
 	{ AM4_D_CAN1_CLKCTRL, NULL, CLKF_SW_SUP, "dcan1_fck" },
diff -Naur --no-dereference a/drivers/clk/ti/clk-44xx.c b/drivers/clk/ti/clk-44xx.c
--- a/drivers/clk/ti/clk-44xx.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/clk/ti/clk-44xx.c	2020-06-19 21:00:31.408227715 -0400
@@ -40,7 +40,7 @@
 };
 
 static const struct omap_clkctrl_reg_data omap4_tesla_clkctrl_regs[] __initconst = {
-	{ OMAP4_DSP_CLKCTRL, NULL, CLKF_HW_SUP, "dpll_iva_m4x2_ck" },
+	{ OMAP4_DSP_CLKCTRL, NULL, CLKF_HW_SUP | CLKF_NO_IDLEST, "dpll_iva_m4x2_ck" },
 	{ 0 },
 };
 
@@ -222,7 +222,7 @@
 };
 
 static const struct omap_clkctrl_reg_data omap4_ducati_clkctrl_regs[] __initconst = {
-	{ OMAP4_IPU_CLKCTRL, NULL, CLKF_HW_SUP, "ducati_clk_mux_ck" },
+	{ OMAP4_IPU_CLKCTRL, NULL, CLKF_HW_SUP | CLKF_NO_IDLEST, "ducati_clk_mux_ck" },
 	{ 0 },
 };
 
@@ -773,6 +773,17 @@
 	DT_CLK(NULL, "usb_tll_hs_usb_ch2_clk", "l3_init_cm:0048:10"),
 	DT_CLK(NULL, "utmi_p1_gfclk", "l3_init_cm:0038:24"),
 	DT_CLK(NULL, "utmi_p2_gfclk", "l3_init_cm:0038:25"),
+	DT_CLK("4a318000.timer", "timer_sys_ck", "sys_clkin_ck"),
+	DT_CLK("48032000.timer", "timer_sys_ck", "sys_clkin_ck"),
+	DT_CLK("48034000.timer", "timer_sys_ck", "sys_clkin_ck"),
+	DT_CLK("48036000.timer", "timer_sys_ck", "sys_clkin_ck"),
+	DT_CLK("4803e000.timer", "timer_sys_ck", "sys_clkin_ck"),
+	DT_CLK("48086000.timer", "timer_sys_ck", "sys_clkin_ck"),
+	DT_CLK("48088000.timer", "timer_sys_ck", "sys_clkin_ck"),
+	DT_CLK("40138000.timer", "timer_sys_ck", "syc_clk_div_ck"),
+	DT_CLK("4013a000.timer", "timer_sys_ck", "syc_clk_div_ck"),
+	DT_CLK("4013c000.timer", "timer_sys_ck", "syc_clk_div_ck"),
+	DT_CLK("4013e000.timer", "timer_sys_ck", "syc_clk_div_ck"),
 	{ .node_name = NULL },
 };
 
diff -Naur --no-dereference a/drivers/clk/ti/clk-54xx.c b/drivers/clk/ti/clk-54xx.c
--- a/drivers/clk/ti/clk-54xx.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/clk/ti/clk-54xx.c	2020-06-19 21:00:31.408227715 -0400
@@ -34,7 +34,7 @@
 };
 
 static const struct omap_clkctrl_reg_data omap5_dsp_clkctrl_regs[] __initconst = {
-	{ OMAP5_MMU_DSP_CLKCTRL, NULL, CLKF_HW_SUP, "dpll_iva_h11x2_ck" },
+	{ OMAP5_MMU_DSP_CLKCTRL, NULL, CLKF_HW_SUP | CLKF_NO_IDLEST, "dpll_iva_h11x2_ck" },
 	{ 0 },
 };
 
@@ -148,7 +148,7 @@
 };
 
 static const struct omap_clkctrl_reg_data omap5_ipu_clkctrl_regs[] __initconst = {
-	{ OMAP5_MMU_IPU_CLKCTRL, NULL, CLKF_HW_SUP, "dpll_core_h22x2_ck" },
+	{ OMAP5_MMU_IPU_CLKCTRL, NULL, CLKF_HW_SUP | CLKF_NO_IDLEST, "dpll_core_h22x2_ck" },
 	{ 0 },
 };
 
@@ -533,6 +533,17 @@
 	DT_CLK(NULL, "usb_tll_hs_usb_ch2_clk", "l3init_cm:0048:10"),
 	DT_CLK(NULL, "utmi_p1_gfclk", "l3init_cm:0038:24"),
 	DT_CLK(NULL, "utmi_p2_gfclk", "l3init_cm:0038:25"),
+	DT_CLK("4ae18000.timer", "timer_sys_ck", "sys_clkin"),
+	DT_CLK("48032000.timer", "timer_sys_ck", "sys_clkin"),
+	DT_CLK("48034000.timer", "timer_sys_ck", "sys_clkin"),
+	DT_CLK("48036000.timer", "timer_sys_ck", "sys_clkin"),
+	DT_CLK("4803e000.timer", "timer_sys_ck", "sys_clkin"),
+	DT_CLK("48086000.timer", "timer_sys_ck", "sys_clkin"),
+	DT_CLK("48088000.timer", "timer_sys_ck", "sys_clkin"),
+	DT_CLK("40138000.timer", "timer_sys_ck", "dss_syc_gfclk_div"),
+	DT_CLK("4013a000.timer", "timer_sys_ck", "dss_syc_gfclk_div"),
+	DT_CLK("4013c000.timer", "timer_sys_ck", "dss_syc_gfclk_div"),
+	DT_CLK("4013e000.timer", "timer_sys_ck", "dss_syc_gfclk_div"),
 	{ .node_name = NULL },
 };
 
diff -Naur --no-dereference a/drivers/clk/ti/clk-7xx.c b/drivers/clk/ti/clk-7xx.c
--- a/drivers/clk/ti/clk-7xx.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/clk/ti/clk-7xx.c	2020-06-19 21:00:31.408227715 -0400
@@ -27,6 +27,32 @@
 	{ 0 },
 };
 
+static const struct omap_clkctrl_reg_data dra7_dsp1_clkctrl_regs[] __initconst = {
+	{ DRA7_DSP1_CLKCTRL, NULL, CLKF_HW_SUP | CLKF_NO_IDLEST, "dpll_dsp_m2_ck" },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data dra7_dsp2_clkctrl_regs[] __initconst = {
+	{ DRA7_DSP2_CLKCTRL, NULL, CLKF_HW_SUP | CLKF_NO_IDLEST, "dpll_dsp_m2_ck" },
+	{ 0 },
+};
+
+static const char * const dra7_ipu1_gfclk_mux_parents[] __initconst = {
+	"dpll_abe_m2x2_ck",
+	"dpll_core_h22x2_ck",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data dra7_ipu1_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_ipu1_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data dra7_ipu1_clkctrl_regs[] __initconst = {
+	{ DRA7_IPU1_CLKCTRL, dra7_ipu1_bit_data, CLKF_HW_SUP | CLKF_NO_IDLEST, "ipu1_cm:clk:0000:24", "ipu1_clkdm" },
+	{ 0 },
+};
+
 static const char * const dra7_mcasp1_aux_gfclk_mux_parents[] __initconst = {
 	"per_abe_x1_gfclk2_div",
 	"video1_clk2_div",
@@ -140,6 +166,11 @@
 	{ 0 },
 };
 
+static const struct omap_clkctrl_reg_data dra7_ipu2_clkctrl_regs[] __initconst = {
+	{ DRA7_IPU2_CLKCTRL, NULL, CLKF_HW_SUP | CLKF_NO_IDLEST, "dpll_core_h22x2_ck" },
+	{ 0 },
+};
+
 static const struct omap_clkctrl_reg_data dra7_dma_clkctrl_regs[] __initconst = {
 	{ DRA7_DMA_SYSTEM_CLKCTRL, NULL, 0, "l3_iclk_div" },
 	{ 0 },
@@ -362,7 +393,7 @@
 	{ DRA7_MMC2_CLKCTRL, dra7_mmc2_bit_data, CLKF_SW_SUP, "l3init_cm:clk:0010:25" },
 	{ DRA7_USB_OTG_SS2_CLKCTRL, dra7_usb_otg_ss2_bit_data, CLKF_HW_SUP, "dpll_core_h13x2_ck" },
 	{ DRA7_USB_OTG_SS3_CLKCTRL, NULL, CLKF_HW_SUP, "dpll_core_h13x2_ck" },
-	{ DRA7_USB_OTG_SS4_CLKCTRL, NULL, CLKF_HW_SUP, "dpll_core_h13x2_ck" },
+	{ DRA7_USB_OTG_SS4_CLKCTRL, NULL, CLKF_HW_SUP | CLKF_SOC_DRA74 | CLKF_SOC_DRA76, "dpll_core_h13x2_ck" },
 	{ DRA7_SATA_CLKCTRL, dra7_sata_bit_data, CLKF_SW_SUP, "func_48m_fclk" },
 	{ DRA7_PCIE1_CLKCTRL, dra7_pcie1_bit_data, CLKF_SW_SUP, "l4_root_clk_div", "pcie_clkdm" },
 	{ DRA7_PCIE2_CLKCTRL, dra7_pcie2_bit_data, CLKF_SW_SUP, "l4_root_clk_div", "pcie_clkdm" },
@@ -615,6 +646,8 @@
 static const struct omap_clkctrl_reg_data dra7_l4per_clkctrl_regs[] __initconst = {
 	{ DRA7_L4_PER2_CLKCTRL, NULL, 0, "l3_iclk_div", "l4per2_clkdm" },
 	{ DRA7_L4_PER3_CLKCTRL, NULL, 0, "l3_iclk_div", "l4per3_clkdm" },
+	{ DRA7_PRUSS1_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_gmac_h13x2_ck" },
+	{ DRA7_PRUSS2_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_gmac_h13x2_ck" },
 	{ DRA7_TIMER10_CLKCTRL, dra7_timer10_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0028:24" },
 	{ DRA7_TIMER11_CLKCTRL, dra7_timer11_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0030:24" },
 	{ DRA7_TIMER2_CLKCTRL, dra7_timer2_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0038:24" },
@@ -662,7 +695,7 @@
 	{ DRA7_AES1_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div", "l4sec_clkdm" },
 	{ DRA7_AES2_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div", "l4sec_clkdm" },
 	{ DRA7_DES_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div", "l4sec_clkdm" },
-	{ DRA7_RNG_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div", "l4sec_clkdm" },
+	{ DRA7_RNG_CLKCTRL, NULL, CLKF_HW_SUP | CLKF_SOC_NONSEC, "l3_iclk_div", "l4sec_clkdm" },
 	{ DRA7_SHAM_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div", "l4sec_clkdm" },
 	{ DRA7_UART7_CLKCTRL, dra7_uart7_bit_data, CLKF_SW_SUP, "l4per_cm:clk:01d0:24", "l4per2_clkdm" },
 	{ DRA7_UART8_CLKCTRL, dra7_uart8_bit_data, CLKF_SW_SUP, "l4per_cm:clk:01e0:24", "l4per2_clkdm" },
@@ -704,7 +737,7 @@
 	{ DRA7_WD_TIMER2_CLKCTRL, NULL, CLKF_SW_SUP, "sys_32k_ck" },
 	{ DRA7_GPIO1_CLKCTRL, dra7_gpio1_bit_data, CLKF_HW_SUP, "wkupaon_iclk_mux" },
 	{ DRA7_TIMER1_CLKCTRL, dra7_timer1_bit_data, CLKF_SW_SUP, "wkupaon_cm:clk:0020:24" },
-	{ DRA7_TIMER12_CLKCTRL, NULL, 0, "secure_32k_clk_src_ck" },
+	{ DRA7_TIMER12_CLKCTRL, NULL, CLKF_SOC_NONSEC, "secure_32k_clk_src_ck" },
 	{ DRA7_COUNTER_32K_CLKCTRL, NULL, 0, "wkupaon_iclk_mux" },
 	{ DRA7_UART10_CLKCTRL, dra7_uart10_bit_data, CLKF_SW_SUP, "wkupaon_cm:clk:0060:24" },
 	{ DRA7_DCAN1_CLKCTRL, dra7_dcan1_bit_data, CLKF_SW_SUP, "wkupaon_cm:clk:0068:24" },
@@ -714,10 +747,14 @@
 
 const struct omap_clkctrl_data dra7_clkctrl_data[] __initconst = {
 	{ 0x4a005320, dra7_mpu_clkctrl_regs },
+	{ 0x4a005420, dra7_dsp1_clkctrl_regs },
+	{ 0x4a005520, dra7_ipu1_clkctrl_regs },
 	{ 0x4a005540, dra7_ipu_clkctrl_regs },
+	{ 0x4a005620, dra7_dsp2_clkctrl_regs },
 	{ 0x4a005740, dra7_rtc_clkctrl_regs },
 	{ 0x4a008620, dra7_coreaon_clkctrl_regs },
 	{ 0x4a008720, dra7_l3main1_clkctrl_regs },
+	{ 0x4a008920, dra7_ipu2_clkctrl_regs },
 	{ 0x4a008a20, dra7_dma_clkctrl_regs },
 	{ 0x4a008b20, dra7_emif_clkctrl_regs },
 	{ 0x4a008c00, dra7_atl_clkctrl_regs },
@@ -734,6 +771,21 @@
 	DT_CLK(NULL, "timer_32k_ck", "sys_32k_ck"),
 	DT_CLK(NULL, "sys_clkin_ck", "timer_sys_clk_div"),
 	DT_CLK(NULL, "sys_clkin", "sys_clkin1"),
+	DT_CLK("4ae18000.timer", "timer_sys_ck", "timer_sys_clk_div"),
+	DT_CLK("48032000.timer", "timer_sys_ck", "timer_sys_clk_div"),
+	DT_CLK("48034000.timer", "timer_sys_ck", "timer_sys_clk_div"),
+	DT_CLK("48036000.timer", "timer_sys_ck", "timer_sys_clk_div"),
+	DT_CLK("4803e000.timer", "timer_sys_ck", "timer_sys_clk_div"),
+	DT_CLK("48086000.timer", "timer_sys_ck", "timer_sys_clk_div"),
+	DT_CLK("48088000.timer", "timer_sys_ck", "timer_sys_clk_div"),
+	DT_CLK("48820000.timer", "timer_sys_ck", "timer_sys_clk_div"),
+	DT_CLK("48822000.timer", "timer_sys_ck", "timer_sys_clk_div"),
+	DT_CLK("48824000.timer", "timer_sys_ck", "timer_sys_clk_div"),
+	DT_CLK("48826000.timer", "timer_sys_ck", "timer_sys_clk_div"),
+	DT_CLK("48828000.timer", "timer_sys_ck", "timer_sys_clk_div"),
+	DT_CLK("4882a000.timer", "timer_sys_ck", "timer_sys_clk_div"),
+	DT_CLK("4882c000.timer", "timer_sys_ck", "timer_sys_clk_div"),
+	DT_CLK("4882e000.timer", "timer_sys_ck", "timer_sys_clk_div"),
 	DT_CLK(NULL, "atl_dpll_clk_mux", "atl_cm:0000:24"),
 	DT_CLK(NULL, "atl_gfclk_mux", "atl_cm:0000:26"),
 	DT_CLK(NULL, "dcan1_sys_clk_mux", "wkupaon_cm:0068:24"),
@@ -752,6 +804,7 @@
 	DT_CLK(NULL, "gpio6_dbclk", "l4per_cm:0080:8"),
 	DT_CLK(NULL, "gpio7_dbclk", "l4per_cm:0110:8"),
 	DT_CLK(NULL, "gpio8_dbclk", "l4per_cm:0118:8"),
+	DT_CLK(NULL, "ipu1_gfclk_mux", "ipu1_cm:0000:24"),
 	DT_CLK(NULL, "mcasp1_ahclkr_mux", "ipu_cm:0010:28"),
 	DT_CLK(NULL, "mcasp1_ahclkx_mux", "ipu_cm:0010:24"),
 	DT_CLK(NULL, "mcasp1_aux_gfclk_mux", "ipu_cm:0010:22"),
diff -Naur --no-dereference a/drivers/clk/ti/clkctrl.c b/drivers/clk/ti/clkctrl.c
--- a/drivers/clk/ti/clkctrl.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/clk/ti/clkctrl.c	2020-06-19 21:00:31.408227715 -0400
@@ -444,6 +444,7 @@
 	const __be32 *addrp;
 	u32 addr;
 	int ret;
+	u16 soc_mask = 0;
 
 	addrp = of_get_address(node, 0, NULL, NULL);
 	addr = (u32)of_translate_address(node, addrp);
@@ -459,6 +460,12 @@
 #ifdef CONFIG_SOC_DRA7XX
 	if (of_machine_is_compatible("ti,dra7"))
 		data = dra7_clkctrl_data;
+	if (of_machine_is_compatible("ti,dra72"))
+		soc_mask = CLKF_SOC_DRA72;
+	if (of_machine_is_compatible("ti,dra74"))
+		soc_mask = CLKF_SOC_DRA74;
+	if (of_machine_is_compatible("ti,dra76"))
+		soc_mask = CLKF_SOC_DRA76;
 #endif
 #ifdef CONFIG_SOC_AM33XX
 	if (of_machine_is_compatible("ti,am33xx"))
@@ -478,6 +485,9 @@
 		data = dm816_clkctrl_data;
 #endif
 
+	if (ti_clk_get_features()->flags & TI_CLK_DEVICE_TYPE_GP)
+		soc_mask |= CLKF_SOC_NONSEC;
+
 	while (data->addr) {
 		if (addr == data->addr)
 			break;
@@ -517,6 +527,12 @@
 	reg_data = data->regs;
 
 	while (reg_data->parent) {
+		if ((reg_data->flags & CLKF_SOC_MASK) &&
+		    (reg_data->flags & soc_mask) == 0) {
+			reg_data++;
+			continue;
+		}
+
 		hw = kzalloc(sizeof(*hw), GFP_KERNEL);
 		if (!hw)
 			return;
diff -Naur --no-dereference a/drivers/clk/ti/clockdomain.c b/drivers/clk/ti/clockdomain.c
--- a/drivers/clk/ti/clockdomain.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/clk/ti/clockdomain.c	2020-06-19 21:00:31.408227715 -0400
@@ -146,10 +146,12 @@
 		if (clk_hw_get_flags(clk_hw) & CLK_IS_BASIC) {
 			pr_warn("can't setup clkdm for basic clk %s\n",
 				__clk_get_name(clk));
+			clk_put(clk);
 			continue;
 		}
 		to_clk_hw_omap(clk_hw)->clkdm_name = clkdm_name;
 		omap2_init_clk_clkdm(clk_hw);
+		clk_put(clk);
 	}
 }
 
diff -Naur --no-dereference a/drivers/clk/ti/clock.h b/drivers/clk/ti/clock.h
--- a/drivers/clk/ti/clock.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/clk/ti/clock.h	2020-06-19 21:00:31.408227715 -0400
@@ -24,6 +24,7 @@
 	u8			flags;
 	s8			latch;
 	const struct clk_div_table	*table;
+	u32		context;
 };
 
 #define to_clk_omap_divider(_hw) container_of(_hw, struct clk_omap_divider, hw)
@@ -36,6 +37,7 @@
 	u8			shift;
 	s8			latch;
 	u8			flags;
+	u8			saved_parent;
 };
 
 #define to_clk_omap_mux(_hw) container_of(_hw, struct clk_omap_mux, hw)
@@ -81,6 +83,13 @@
 #define CLKF_HW_SUP			BIT(6)
 #define CLKF_NO_IDLEST			BIT(7)
 
+#define CLKF_SOC_MASK			GENMASK(11, 8)
+
+#define CLKF_SOC_NONSEC			BIT(8)
+#define CLKF_SOC_DRA72			BIT(9)
+#define CLKF_SOC_DRA74			BIT(10)
+#define CLKF_SOC_DRA76			BIT(11)
+
 #define CLK(dev, con, ck)		\
 	{				\
 		.lk = {			\
diff -Naur --no-dereference a/drivers/clk/ti/divider.c b/drivers/clk/ti/divider.c
--- a/drivers/clk/ti/divider.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/clk/ti/divider.c	2020-06-19 21:00:31.408227715 -0400
@@ -268,10 +268,46 @@
 	return 0;
 }
 
+/**
+ * clk_divider_save_context - Save the divider value
+ * @hw: pointer  struct clk_hw
+ *
+ * Save the divider value
+ */
+static int clk_divider_save_context(struct clk_hw *hw)
+{
+	struct clk_omap_divider *divider = to_clk_omap_divider(hw);
+	u32 val;
+
+	val = ti_clk_ll_ops->clk_readl(&divider->reg) >> divider->shift;
+	divider->context = val & div_mask(divider);
+
+	return 0;
+}
+
+/**
+ * clk_divider_restore_context - restore the saved the divider value
+ * @hw: pointer  struct clk_hw
+ *
+ * Restore the saved the divider value
+ */
+static void clk_divider_restore_context(struct clk_hw *hw)
+{
+	struct clk_omap_divider *divider = to_clk_omap_divider(hw);
+	u32 val;
+
+	val = ti_clk_ll_ops->clk_readl(&divider->reg);
+	val &= ~(div_mask(divider) << divider->shift);
+	val |= divider->context << divider->shift;
+	ti_clk_ll_ops->clk_writel(val, &divider->reg);
+}
+
 const struct clk_ops ti_clk_divider_ops = {
 	.recalc_rate = ti_clk_divider_recalc_rate,
 	.round_rate = ti_clk_divider_round_rate,
 	.set_rate = ti_clk_divider_set_rate,
+	.save_context = clk_divider_save_context,
+	.restore_context = clk_divider_restore_context,
 };
 
 static struct clk *_register_divider(struct device *dev, const char *name,
diff -Naur --no-dereference a/drivers/clk/ti/dpll3xxx.c b/drivers/clk/ti/dpll3xxx.c
--- a/drivers/clk/ti/dpll3xxx.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/clk/ti/dpll3xxx.c	2020-06-19 21:00:31.408227715 -0400
@@ -782,6 +782,130 @@
 	return rate;
 }
 
+/**
+ * omap3_core_dpll_save_context - Save the m and n values of the divider
+ * @hw: pointer  struct clk_hw
+ *
+ * Before the dpll registers are lost save the last rounded rate m and n
+ * and the enable mask.
+ */
+int omap3_core_dpll_save_context(struct clk_hw *hw)
+{
+	struct clk_hw_omap *clk = to_clk_hw_omap(hw);
+	struct dpll_data *dd;
+	u32 v;
+
+	dd = clk->dpll_data;
+
+	v = ti_clk_ll_ops->clk_readl(&dd->control_reg);
+	clk->context = (v & dd->enable_mask) >> __ffs(dd->enable_mask);
+
+	if (clk->context == DPLL_LOCKED) {
+		v = ti_clk_ll_ops->clk_readl(&dd->mult_div1_reg);
+		dd->last_rounded_m = (v & dd->mult_mask) >>
+						__ffs(dd->mult_mask);
+		dd->last_rounded_n = ((v & dd->div1_mask) >>
+						__ffs(dd->div1_mask)) + 1;
+	}
+
+	return 0;
+}
+
+/**
+ * omap3_core_dpll_restore_context - restore the m and n values of the divider
+ * @hw: pointer  struct clk_hw
+ *
+ * Restore the last rounded rate m and n
+ * and the enable mask.
+ */
+void omap3_core_dpll_restore_context(struct clk_hw *hw)
+{
+	struct clk_hw_omap *clk = to_clk_hw_omap(hw);
+	const struct dpll_data *dd;
+	u32 v;
+
+	dd = clk->dpll_data;
+
+	if (clk->context == DPLL_LOCKED) {
+		_omap3_dpll_write_clken(clk, 0x4);
+		_omap3_wait_dpll_status(clk, 0);
+
+		v = ti_clk_ll_ops->clk_readl(&dd->mult_div1_reg);
+		v &= ~(dd->mult_mask | dd->div1_mask);
+		v |= dd->last_rounded_m << __ffs(dd->mult_mask);
+		v |= (dd->last_rounded_n - 1) << __ffs(dd->div1_mask);
+		ti_clk_ll_ops->clk_writel(v, &dd->mult_div1_reg);
+
+		_omap3_dpll_write_clken(clk, DPLL_LOCKED);
+		_omap3_wait_dpll_status(clk, 1);
+	} else {
+		_omap3_dpll_write_clken(clk, clk->context);
+	}
+}
+
+/**
+ * omap3_non_core_dpll_save_context - Save the m and n values of the divider
+ * @hw: pointer  struct clk_hw
+ *
+ * Before the dpll registers are lost save the last rounded rate m and n
+ * and the enable mask.
+ */
+int omap3_noncore_dpll_save_context(struct clk_hw *hw)
+{
+	struct clk_hw_omap *clk = to_clk_hw_omap(hw);
+	struct dpll_data *dd;
+	u32 v;
+
+	dd = clk->dpll_data;
+
+	v = ti_clk_ll_ops->clk_readl(&dd->control_reg);
+	clk->context = (v & dd->enable_mask) >> __ffs(dd->enable_mask);
+
+	if (clk->context == DPLL_LOCKED) {
+		v = ti_clk_ll_ops->clk_readl(&dd->mult_div1_reg);
+		dd->last_rounded_m = (v & dd->mult_mask) >>
+						__ffs(dd->mult_mask);
+		dd->last_rounded_n = ((v & dd->div1_mask) >>
+						__ffs(dd->div1_mask)) + 1;
+	}
+
+	return 0;
+}
+
+/**
+ * omap3_core_dpll_restore_context - restore the m and n values of the divider
+ * @hw: pointer  struct clk_hw
+ *
+ * Restore the last rounded rate m and n
+ * and the enable mask.
+ */
+void omap3_noncore_dpll_restore_context(struct clk_hw *hw)
+{
+	struct clk_hw_omap *clk = to_clk_hw_omap(hw);
+	const struct dpll_data *dd;
+	u32 ctrl, mult_div1;
+
+	dd = clk->dpll_data;
+
+	ctrl = ti_clk_ll_ops->clk_readl(&dd->control_reg);
+	mult_div1 = ti_clk_ll_ops->clk_readl(&dd->mult_div1_reg);
+
+	if (clk->context == ((ctrl & dd->enable_mask) >>
+			     __ffs(dd->enable_mask)) &&
+	    dd->last_rounded_m == ((mult_div1 & dd->mult_mask) >>
+				   __ffs(dd->mult_mask)) &&
+	    dd->last_rounded_n == ((mult_div1 & dd->div1_mask) >>
+				   __ffs(dd->div1_mask)) + 1) {
+		/* nothing to be done */
+		return;
+	}
+
+	if (clk->context == DPLL_LOCKED)
+		omap3_noncore_dpll_program(clk, 0);
+	else
+		_omap3_dpll_write_clken(clk, clk->context);
+}
+
 /* OMAP3/4 non-CORE DPLL clkops */
 const struct clk_hw_omap_ops clkhwops_omap3_dpll = {
 	.allow_idle	= omap3_dpll_allow_idle,
diff -Naur --no-dereference a/drivers/clk/ti/dpll.c b/drivers/clk/ti/dpll.c
--- a/drivers/clk/ti/dpll.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/clk/ti/dpll.c	2020-06-19 21:00:31.408227715 -0400
@@ -39,6 +39,8 @@
 	.set_rate_and_parent	= &omap3_noncore_dpll_set_rate_and_parent,
 	.determine_rate	= &omap4_dpll_regm4xen_determine_rate,
 	.get_parent	= &omap2_init_dpll_parent,
+	.save_context	= &omap3_core_dpll_save_context,
+	.restore_context = &omap3_core_dpll_restore_context,
 };
 #else
 static const struct clk_ops dpll_m4xen_ck_ops = {};
@@ -62,6 +64,8 @@
 	.set_rate_and_parent	= &omap3_noncore_dpll_set_rate_and_parent,
 	.determine_rate	= &omap3_noncore_dpll_determine_rate,
 	.get_parent	= &omap2_init_dpll_parent,
+	.save_context	= &omap3_noncore_dpll_save_context,
+	.restore_context = &omap3_noncore_dpll_restore_context,
 };
 
 static const struct clk_ops dpll_no_gate_ck_ops = {
@@ -72,6 +76,8 @@
 	.set_parent	= &omap3_noncore_dpll_set_parent,
 	.set_rate_and_parent	= &omap3_noncore_dpll_set_rate_and_parent,
 	.determine_rate	= &omap3_noncore_dpll_determine_rate,
+	.save_context	= &omap3_noncore_dpll_save_context,
+	.restore_context = &omap3_noncore_dpll_restore_context
 };
 #else
 static const struct clk_ops dpll_core_ck_ops = {};
diff -Naur --no-dereference a/drivers/clk/ti/gate.c b/drivers/clk/ti/gate.c
--- a/drivers/clk/ti/gate.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/clk/ti/gate.c	2020-06-19 21:00:31.408227715 -0400
@@ -33,6 +33,7 @@
 	.init		= &omap2_init_clk_clkdm,
 	.enable		= &omap2_clkops_enable_clkdm,
 	.disable	= &omap2_clkops_disable_clkdm,
+	.restore_context = clk_gate_restore_context,
 };
 
 const struct clk_ops omap_gate_clk_ops = {
@@ -40,6 +41,7 @@
 	.enable		= &omap2_dflt_clk_enable,
 	.disable	= &omap2_dflt_clk_disable,
 	.is_enabled	= &omap2_dflt_clk_is_enabled,
+	.restore_context = clk_gate_restore_context,
 };
 
 static const struct clk_ops omap_gate_clk_hsdiv_restore_ops = {
@@ -47,6 +49,7 @@
 	.enable		= &omap36xx_gate_clk_enable_with_hsdiv_restore,
 	.disable	= &omap2_dflt_clk_disable,
 	.is_enabled	= &omap2_dflt_clk_is_enabled,
+	.restore_context = clk_gate_restore_context,
 };
 
 /**
diff -Naur --no-dereference a/drivers/clk/ti/mux.c b/drivers/clk/ti/mux.c
--- a/drivers/clk/ti/mux.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/clk/ti/mux.c	2020-06-19 21:00:31.408227715 -0400
@@ -91,10 +91,39 @@
 	return 0;
 }
 
+/**
+ * clk_mux_save_context - Save the parent selcted in the mux
+ * @hw: pointer  struct clk_hw
+ *
+ * Save the parent mux value.
+ */
+static int clk_mux_save_context(struct clk_hw *hw)
+{
+	struct clk_omap_mux *mux = to_clk_omap_mux(hw);
+
+	mux->saved_parent = ti_clk_mux_get_parent(hw);
+	return 0;
+}
+
+/**
+ * clk_mux_restore_context - Restore the parent in the mux
+ * @hw: pointer  struct clk_hw
+ *
+ * Restore the saved parent mux value.
+ */
+static void clk_mux_restore_context(struct clk_hw *hw)
+{
+	struct clk_omap_mux *mux = to_clk_omap_mux(hw);
+
+	ti_clk_mux_set_parent(hw, mux->saved_parent);
+}
+
 const struct clk_ops ti_clk_mux_ops = {
 	.get_parent = ti_clk_mux_get_parent,
 	.set_parent = ti_clk_mux_set_parent,
 	.determine_rate = __clk_mux_determine_rate,
+	.save_context = clk_mux_save_context,
+	.restore_context = clk_mux_restore_context,
 };
 
 static struct clk *_register_mux(struct device *dev, const char *name,
diff -Naur --no-dereference a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig
--- a/drivers/clocksource/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/clocksource/Kconfig	2020-06-19 21:00:31.408227715 -0400
@@ -377,7 +377,7 @@
 	  allwinner,erratum-unknown1 property is found in the timer node.
 
 config ARM_GLOBAL_TIMER
-	bool "Support for the ARM global timer" if COMPILE_TEST
+	bool "Support for the ARM global timer"
 	select TIMER_OF if OF
 	depends on ARM
 	help
diff -Naur --no-dereference a/drivers/clocksource/timer-keystone.c b/drivers/clocksource/timer-keystone.c
--- a/drivers/clocksource/timer-keystone.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/clocksource/timer-keystone.c	2020-06-19 21:00:31.408227715 -0400
@@ -17,6 +17,7 @@
 #include <linux/interrupt.h>
 #include <linux/of_address.h>
 #include <linux/of_irq.h>
+#include <linux/platform_device.h>
 
 #define TIMER_NAME			"timer-keystone"
 
@@ -42,11 +43,13 @@
  * @base: timer memory base address
  * @hz_period: cycles per HZ period
  * @event_dev: event device based on timer
+ * @registered: Flag to keep a track of registration status
  */
 static struct keystone_timer {
 	void __iomem *base;
 	unsigned long hz_period;
 	struct clock_event_device event_dev;
+	bool registered;
 } timer;
 
 static inline u32 keystone_timer_readl(unsigned long rg)
@@ -144,13 +147,14 @@
 	return 0;
 }
 
-static int __init keystone_timer_init(struct device_node *np)
+static int keystone_timer_init(struct device_node *np)
 {
 	struct clock_event_device *event_dev = &timer.event_dev;
 	unsigned long rate;
 	struct clk *clk;
 	int irq, error;
 
+	timer.registered = false;
 	irq  = irq_of_parse_and_map(np, 0);
 	if (!irq) {
 		pr_err("%s: failed to map interrupts\n", __func__);
@@ -219,6 +223,7 @@
 	clockevents_config_and_register(event_dev, rate, 1, ULONG_MAX);
 
 	pr_info("keystone timer clock @%lu Hz\n", rate);
+	timer.registered = true;
 	return 0;
 err:
 	clk_put(clk);
@@ -228,3 +233,44 @@
 
 TIMER_OF_DECLARE(keystone_timer, "ti,keystone-timer",
 			   keystone_timer_init);
+
+static const struct of_device_id keystone_clocksource_of_match[] = {
+	{.compatible = "ti,keystone-timer", },
+	{},
+};
+
+static int keystone_clocksource_probe(struct platform_device *pdev)
+{
+	struct clk *clk;
+
+	if (timer.registered)
+		return 0;
+
+	clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(clk)) {
+		if (PTR_ERR(clk) != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "failed to get clock\n");
+		return PTR_ERR(clk);
+	}
+
+	clk_put(clk);
+	keystone_timer_init(pdev->dev.of_node);
+	if (!timer.registered)
+		return -EINVAL;
+
+	return 0;
+}
+
+static struct platform_driver keystone_clocksource_driver = {
+	.probe		= keystone_clocksource_probe,
+	.driver		= {
+		.name	= "keystone_clocksource",
+		.of_match_table = keystone_clocksource_of_match,
+	},
+};
+
+static int __init keystone_clocksource_init_driver(void)
+{
+	return platform_driver_register(&keystone_clocksource_driver);
+}
+device_initcall(keystone_clocksource_init_driver);
diff -Naur --no-dereference a/drivers/clocksource/timer-ti-dm.c b/drivers/clocksource/timer-ti-dm.c
--- a/drivers/clocksource/timer-ti-dm.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/clocksource/timer-ti-dm.c	2020-06-19 21:00:31.408227715 -0400
@@ -94,6 +94,13 @@
 
 static void omap_timer_restore_context(struct omap_dm_timer *timer)
 {
+	/*
+	 * Do not restore the context during late attach. Kernel data
+	 * structure is not in sync with the register settings of the timer.
+	 */
+	if (timer->late_attach)
+		return;
+
 	omap_dm_timer_write_reg(timer, OMAP_TIMER_WAKEUP_EN_REG,
 				timer->context.twer);
 	omap_dm_timer_write_reg(timer, OMAP_TIMER_COUNTER_REG,
@@ -138,35 +145,6 @@
 	return 0;
 }
 
-static int omap_dm_timer_of_set_source(struct omap_dm_timer *timer)
-{
-	int ret;
-	struct clk *parent;
-
-	/*
-	 * FIXME: OMAP1 devices do not use the clock framework for dmtimers so
-	 * do not call clk_get() for these devices.
-	 */
-	if (!timer->fclk)
-		return -ENODEV;
-
-	parent = clk_get(&timer->pdev->dev, NULL);
-	if (IS_ERR(parent))
-		return -ENODEV;
-
-	/* Bail out if both clocks point to fck */
-	if (clk_is_match(parent, timer->fclk))
-		return 0;
-
-	ret = clk_set_parent(timer->fclk, parent);
-	if (ret < 0)
-		pr_err("%s: failed to set parent\n", __func__);
-
-	clk_put(parent);
-
-	return ret;
-}
-
 static int omap_dm_timer_set_source(struct omap_dm_timer *timer, int source)
 {
 	int ret;
@@ -223,6 +201,20 @@
 	return ret;
 }
 
+static int omap_dm_timer_is_enabled(struct omap_dm_timer *timer)
+{
+	u32 val;
+
+	val = omap_dm_timer_read_reg(timer, OMAP_TIMER_CTRL_REG);
+
+	/* Check if timer ST bit is set or the Counter register is loaded */
+	if (val & OMAP_TIMER_CTRL_ST ||
+	    omap_dm_timer_read_reg(timer, OMAP_TIMER_COUNTER_REG))
+		return 1;
+	else
+		return 0;
+}
+
 static void omap_dm_timer_enable(struct omap_dm_timer *timer)
 {
 	int c;
@@ -276,9 +268,15 @@
 	__omap_dm_timer_enable_posted(timer);
 	omap_dm_timer_disable(timer);
 
-	rc = omap_dm_timer_of_set_source(timer);
-	if (rc == -ENODEV)
-		return omap_dm_timer_set_source(timer, OMAP_TIMER_SRC_32_KHZ);
+	/*
+	 * During late attach, do not set the timer source during prepare
+	 * as the timer might be clocked from a different source. It will
+	 * be set properly from remoteproc.
+	 */
+	if (timer->late_attach)
+		return 0;
+
+	rc = omap_dm_timer_set_source(timer, OMAP_TIMER_SRC_32_KHZ);
 
 	return rc;
 }
@@ -534,6 +532,16 @@
 
 	/* Save the context */
 	timer->context.tclr = l;
+
+	/*
+	 * Now that timer has been started, call pm_runtime_put_noidle to
+	 * balance the pm_runtime device usage count to the proper value as
+	 * the regular case, and reset the late_attach flag.
+	 */
+	if (timer->late_attach)
+		pm_runtime_put_noidle(&timer->pdev->dev);
+	timer->late_attach = 0;
+
 	return 0;
 }
 
@@ -574,10 +582,18 @@
 		l |= OMAP_TIMER_CTRL_AR;
 	else
 		l &= ~OMAP_TIMER_CTRL_AR;
-	omap_dm_timer_write_reg(timer, OMAP_TIMER_CTRL_REG, l);
-	omap_dm_timer_write_reg(timer, OMAP_TIMER_LOAD_REG, load);
 
-	omap_dm_timer_write_reg(timer, OMAP_TIMER_TRIGGER_REG, 0);
+	/*
+	 * If late attach is enabled, do not modify the dmtimer registers.
+	 * The registers would have been configured already.
+	 */
+	if (!timer->late_attach) {
+		omap_dm_timer_write_reg(timer, OMAP_TIMER_CTRL_REG, l);
+		omap_dm_timer_write_reg(timer, OMAP_TIMER_LOAD_REG, load);
+
+		omap_dm_timer_write_reg(timer, OMAP_TIMER_TRIGGER_REG, 0);
+	}
+
 	/* Save the context */
 	timer->context.tclr = l;
 	timer->context.tldr = load;
@@ -599,13 +615,21 @@
 	l = omap_dm_timer_read_reg(timer, OMAP_TIMER_CTRL_REG);
 	if (autoreload) {
 		l |= OMAP_TIMER_CTRL_AR;
-		omap_dm_timer_write_reg(timer, OMAP_TIMER_LOAD_REG, load);
+		/*
+		 * If late attach is enabled, do not modify the dmtimer
+		 * registers. The registers would have been configured
+		 * already.
+		 */
+		if (!timer->late_attach)
+			omap_dm_timer_write_reg(timer, OMAP_TIMER_LOAD_REG,
+						load);
 	} else {
 		l &= ~OMAP_TIMER_CTRL_AR;
 	}
 	l |= OMAP_TIMER_CTRL_ST;
 
-	__omap_dm_timer_load_start(timer, l, load, timer->posted);
+	if (!timer->late_attach)
+		__omap_dm_timer_load_start(timer, l, load, timer->posted);
 
 	/* Save the context */
 	timer->context.tclr = l;
@@ -878,6 +902,16 @@
 			goto err_get_sync;
 		}
 		__omap_dm_timer_init_regs(timer);
+
+		if (omap_dm_timer_is_enabled(timer))
+			timer->late_attach = 1;
+		/*
+		 * Increase the pm_runtime usage count and prevent kernel power
+		 * management from idling or disabling the timer.
+		 */
+		if (timer->late_attach)
+			pm_runtime_get_noresume(dev);
+
 		pm_runtime_put(dev);
 	}
 
@@ -915,6 +949,12 @@
 		if (!strcmp(dev_name(&timer->pdev->dev),
 			    dev_name(&pdev->dev))) {
 			list_del(&timer->node);
+			/*
+			 * Reset device usage counter if late_attach is still
+			 * set
+			 */
+			if (timer->late_attach)
+				pm_runtime_put_noidle(&timer->pdev->dev);
 			ret = 0;
 			break;
 		}
@@ -936,6 +976,7 @@
 	.free = omap_dm_timer_free,
 	.enable = omap_dm_timer_enable,
 	.disable = omap_dm_timer_disable,
+	.is_enabled = omap_dm_timer_is_enabled,
 	.get_fclk = omap_dm_timer_get_fclk,
 	.start = omap_dm_timer_start,
 	.stop = omap_dm_timer_stop,
diff -Naur --no-dereference a/drivers/cpufreq/ti-cpufreq.c b/drivers/cpufreq/ti-cpufreq.c
--- a/drivers/cpufreq/ti-cpufreq.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/cpufreq/ti-cpufreq.c	2020-06-19 21:00:31.408227715 -0400
@@ -33,11 +33,14 @@
 
 #define DRA7_EFUSE_HAS_OD_MPU_OPP		11
 #define DRA7_EFUSE_HAS_HIGH_MPU_OPP		15
+#define DRA76_EFUSE_HAS_PLUS_MPU_OPP		18
 #define DRA7_EFUSE_HAS_ALL_MPU_OPP		23
+#define DRA76_EFUSE_HAS_ALL_MPU_OPP		24
 
 #define DRA7_EFUSE_NOM_MPU_OPP			BIT(0)
 #define DRA7_EFUSE_OD_MPU_OPP			BIT(1)
 #define DRA7_EFUSE_HIGH_MPU_OPP			BIT(2)
+#define DRA76_EFUSE_PLUS_MPU_OPP		BIT(3)
 
 #define VERSION_COUNT				2
 
@@ -82,6 +85,9 @@
 	 */
 
 	switch (efuse) {
+	case DRA76_EFUSE_HAS_PLUS_MPU_OPP:
+	case DRA76_EFUSE_HAS_ALL_MPU_OPP:
+		calculated_efuse |= DRA76_EFUSE_PLUS_MPU_OPP;
 	case DRA7_EFUSE_HAS_ALL_MPU_OPP:
 	case DRA7_EFUSE_HAS_HIGH_MPU_OPP:
 		calculated_efuse |= DRA7_EFUSE_HIGH_MPU_OPP;
diff -Naur --no-dereference a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
--- a/drivers/crypto/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/crypto/Kconfig	2020-06-19 21:00:31.408227715 -0400
@@ -762,4 +762,24 @@
 
 source "drivers/crypto/hisilicon/Kconfig"
 
+config CRYPTO_DEV_SA2UL
+	tristate "Support for TI security accelerator"
+	depends on ARCH_K3 || COMPILE_TEST
+	select ARM64_CRYPTO
+	select CRYPTO_AES
+	select CRYPTO_AES_ARM64
+	select CRYPTO_SHA1
+	select CRYPTO_SHA512
+	select CRYPTO_SHA256
+	select CRYPTO_MD5
+	select CRYPTO_ALGAPI
+	select CRYPTO_AUTHENC
+	select CRYPTO_HMAC
+	select HW_RANDOM
+	default m if ARCH_K3
+	help
+	  Keystone devices include a security accelerator engine that may be
+	  used for crypto offload.  Select this if you want to use hardware
+	  acceleration for cryptographic algorithms on these devices.
+
 endif # CRYPTO_HW
diff -Naur --no-dereference a/drivers/crypto/Makefile b/drivers/crypto/Makefile
--- a/drivers/crypto/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/crypto/Makefile	2020-06-19 21:00:31.408227715 -0400
@@ -46,4 +46,5 @@
 obj-$(CONFIG_CRYPTO_DEV_BCM_SPU) += bcm/
 obj-$(CONFIG_CRYPTO_DEV_SAFEXCEL) += inside-secure/
 obj-$(CONFIG_CRYPTO_DEV_ARTPEC6) += axis/
+obj-$(CONFIG_CRYPTO_DEV_SA2UL) += sa2ul.o
 obj-y += hisilicon/
diff -Naur --no-dereference a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c
--- a/drivers/crypto/omap-aes.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/crypto/omap-aes.c	2020-06-19 21:00:31.408227715 -0400
@@ -1312,7 +1312,8 @@
 	tasklet_kill(&dd->done_task);
 	omap_aes_dma_cleanup(dd);
 	pm_runtime_disable(dd->dev);
-	dd = NULL;
+
+	sysfs_remove_group(&dd->dev->kobj, &omap_aes_attr_group);
 
 	return 0;
 }
diff -Naur --no-dereference a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c
--- a/drivers/crypto/omap-sham.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/crypto/omap-sham.c	2020-06-19 21:00:31.408227715 -0400
@@ -115,6 +115,8 @@
 #define FLAGS_BE32_SHA1		8
 #define FLAGS_SGS_COPIED	9
 #define FLAGS_SGS_ALLOCED	10
+#define FLAGS_HUGE		11
+
 /* context flags */
 #define FLAGS_FINUP		16
 
@@ -139,6 +141,8 @@
 #define BUFLEN			SHA512_BLOCK_SIZE
 #define OMAP_SHA_DMA_THRESHOLD	256
 
+#define OMAP_SHA_MAX_DMA_LEN	(1024 * 2048)
+
 struct omap_sham_dev;
 
 struct omap_sham_reqctx {
@@ -692,21 +696,20 @@
 
 	set_bit(FLAGS_SGS_ALLOCED, &ctx->dd->flags);
 
+	ctx->offset += new_len - ctx->bufcnt;
 	ctx->bufcnt = 0;
 
 	return 0;
 }
 
 static int omap_sham_copy_sgs(struct omap_sham_reqctx *ctx,
-			      struct scatterlist *sg, int bs, int new_len)
+			      struct scatterlist *sg, int bs,
+			      unsigned int new_len)
 {
 	int pages;
 	void *buf;
-	int len;
-
-	len = new_len + ctx->bufcnt;
 
-	pages = get_order(ctx->total);
+	pages = get_order(new_len);
 
 	buf = (void *)__get_free_pages(GFP_ATOMIC, pages);
 	if (!buf) {
@@ -718,14 +721,14 @@
 		memcpy(buf, ctx->dd->xmit_buf, ctx->bufcnt);
 
 	scatterwalk_map_and_copy(buf + ctx->bufcnt, sg, ctx->offset,
-				 ctx->total - ctx->bufcnt, 0);
+				 min(new_len, ctx->total) - ctx->bufcnt, 0);
 	sg_init_table(ctx->sgl, 1);
-	sg_set_buf(ctx->sgl, buf, len);
+	sg_set_buf(ctx->sgl, buf, new_len);
 	ctx->sg = ctx->sgl;
 	set_bit(FLAGS_SGS_COPIED, &ctx->dd->flags);
 	ctx->sg_len = 1;
+	ctx->offset += new_len - ctx->bufcnt;
 	ctx->bufcnt = 0;
-	ctx->offset = 0;
 
 	return 0;
 }
@@ -744,7 +747,7 @@
 	if (!sg || !sg->length || !nbytes)
 		return 0;
 
-	new_len = nbytes;
+	new_len = nbytes - offset;
 
 	if (offset)
 		list_ok = false;
@@ -754,6 +757,9 @@
 	else
 		new_len = (new_len - 1) / bs * bs;
 
+	if (!new_len)
+		return 0;
+
 	if (nbytes != new_len)
 		list_ok = false;
 
@@ -797,10 +803,17 @@
 		}
 	}
 
+	if (new_len > OMAP_SHA_MAX_DMA_LEN) {
+		new_len = OMAP_SHA_MAX_DMA_LEN;
+		aligned = false;
+	}
+
 	if (!aligned)
 		return omap_sham_copy_sgs(rctx, sg, bs, new_len);
 	else if (!list_ok)
 		return omap_sham_copy_sg_lists(rctx, sg, bs, new_len);
+	else
+		rctx->offset += new_len;
 
 	rctx->sg_len = n;
 	rctx->sg = sg;
@@ -824,7 +837,12 @@
 	else
 		nbytes = 0;
 
-	rctx->total = nbytes + rctx->bufcnt;
+	rctx->total = nbytes + rctx->bufcnt - rctx->offset;
+
+	dev_dbg(rctx->dd->dev,
+		"%s: nbytes=%d, bs=%d, total=%d, offset=%d, bufcnt=%d\n",
+		__func__, nbytes, bs, rctx->total, rctx->offset,
+		rctx->bufcnt);
 
 	if (!rctx->total)
 		return 0;
@@ -850,12 +868,15 @@
 
 	xmit_len = rctx->total;
 
+	if (xmit_len > OMAP_SHA_MAX_DMA_LEN)
+		xmit_len = OMAP_SHA_MAX_DMA_LEN;
+
 	if (!IS_ALIGNED(xmit_len, bs)) {
 		if (final)
 			xmit_len = DIV_ROUND_UP(xmit_len, bs) * bs;
 		else
 			xmit_len = xmit_len / bs * bs;
-	} else if (!final) {
+	} else if (!final && rctx->total == xmit_len) {
 		xmit_len -= bs;
 	}
 
@@ -883,7 +904,7 @@
 		rctx->sg_len = 1;
 	}
 
-	if (hash_later) {
+	if (hash_later && hash_later <= rctx->buflen) {
 		int offset = 0;
 
 		if (hash_later > req->nbytes) {
@@ -904,6 +925,9 @@
 		rctx->bufcnt = 0;
 	}
 
+	if (hash_later > rctx->buflen)
+		set_bit(FLAGS_HUGE, &rctx->dd->flags);
+
 	if (!final)
 		rctx->total = xmit_len;
 
@@ -1001,10 +1025,11 @@
 	struct ahash_request *req = dd->req;
 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
 	int err;
-	bool final = ctx->flags & BIT(FLAGS_FINUP);
+	bool final = (ctx->flags & BIT(FLAGS_FINUP)) &&
+			!(dd->flags & BIT(FLAGS_HUGE));
 
-	dev_dbg(dd->dev, "update_req: total: %u, digcnt: %d, finup: %d\n",
-		 ctx->total, ctx->digcnt, (ctx->flags & BIT(FLAGS_FINUP)) != 0);
+	dev_dbg(dd->dev, "update_req: total: %u, digcnt: %d, final: %d",
+		ctx->total, ctx->digcnt, final);
 
 	if (ctx->total < get_block_size(ctx) ||
 	    ctx->total < dd->fallback_sz)
@@ -1027,6 +1052,9 @@
 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
 	int err = 0, use_dma = 1;
 
+	if (dd->flags & BIT(FLAGS_HUGE))
+		return 0;
+
 	if ((ctx->total <= get_block_size(ctx)) || dd->polling_mode)
 		/*
 		 * faster to handle last block with cpu or
@@ -1087,7 +1115,7 @@
 
 	if (test_bit(FLAGS_SGS_COPIED, &dd->flags))
 		free_pages((unsigned long)sg_virt(ctx->sg),
-			   get_order(ctx->sg->length + ctx->bufcnt));
+			   get_order(ctx->sg->length));
 
 	if (test_bit(FLAGS_SGS_ALLOCED, &dd->flags))
 		kfree(ctx->sg);
@@ -1096,6 +1124,21 @@
 
 	dd->flags &= ~(BIT(FLAGS_SGS_ALLOCED) | BIT(FLAGS_SGS_COPIED));
 
+	if (dd->flags & BIT(FLAGS_HUGE)) {
+		dd->flags &= ~(BIT(FLAGS_CPU) | BIT(FLAGS_DMA_READY) |
+				BIT(FLAGS_OUTPUT_READY) | BIT(FLAGS_HUGE));
+		omap_sham_prepare_request(req, ctx->op == OP_UPDATE);
+		if (ctx->op == OP_UPDATE || (dd->flags & BIT(FLAGS_HUGE))) {
+			err = omap_sham_update_req(dd);
+			if (err != -EINPROGRESS &&
+			    (ctx->flags & BIT(FLAGS_FINUP)))
+				err = omap_sham_final_req(dd);
+		} else if (ctx->op == OP_FINAL) {
+			omap_sham_final_req(dd);
+		}
+		return;
+	}
+
 	if (!err) {
 		dd->pdata->copy_hash(req, 1);
 		if (test_bit(FLAGS_FINAL, &dd->flags))
@@ -1111,6 +1154,8 @@
 	pm_runtime_mark_last_busy(dd->dev);
 	pm_runtime_put_autosuspend(dd->dev);
 
+	ctx->offset = 0;
+
 	if (req->base.complete)
 		req->base.complete(&req->base, err);
 }
@@ -1162,7 +1207,7 @@
 		/* request has changed - restore hash */
 		dd->pdata->copy_hash(req, 0);
 
-	if (ctx->op == OP_UPDATE) {
+	if (ctx->op == OP_UPDATE || (dd->flags & BIT(FLAGS_HUGE))) {
 		err = omap_sham_update_req(dd);
 		if (err != -EINPROGRESS && (ctx->flags & BIT(FLAGS_FINUP)))
 			/* no final() after finup() */
@@ -1735,6 +1780,8 @@
 	struct omap_sham_dev *dd = (struct omap_sham_dev *)data;
 	int err = 0;
 
+	dev_dbg(dd->dev, "%s: flags=%lx\n", __func__, dd->flags);
+
 	if (!test_bit(FLAGS_BUSY, &dd->flags)) {
 		omap_sham_handle_queue(dd, NULL);
 		return;
@@ -2229,6 +2276,8 @@
 	if (!dd->polling_mode)
 		dma_release_channel(dd->dma_lch);
 
+	sysfs_remove_group(&dd->dev->kobj, &omap_sham_attr_group);
+
 	return 0;
 }
 
diff -Naur --no-dereference a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c
--- a/drivers/crypto/sa2ul.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/crypto/sa2ul.c	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,2556 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * AM6 SA2UL crypto accelerator driver
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
+ *
+ * Authors:	Keerthy
+ *              Vitaly Andrianov
+ */
+#include <linux/clk.h>
+#include <linux/module.h>
+#include <linux/dmapool.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/dmaengine.h>
+#include <linux/cryptohash.h>
+#include <linux/mod_devicetable.h>
+
+#include <crypto/authenc.h>
+#include <crypto/des.h>
+#include <crypto/internal/aead.h>
+#include <crypto/internal/skcipher.h>
+#include <crypto/internal/hash.h>
+#include <crypto/scatterwalk.h>
+#include <crypto/sha.h>
+
+#include "sa2ul.h"
+
+/* Byte offset for key in encryption security context */
+#define SC_ENC_KEY_OFFSET (1 + 27 + 4)
+/* Byte offset for Aux-1 in encryption security context */
+#define SC_ENC_AUX1_OFFSET (1 + 27 + 4 + 32)
+
+#define SA_CMDL_UPD_ENC         0x0001
+#define SA_CMDL_UPD_AUTH        0x0002
+#define SA_CMDL_UPD_ENC_IV      0x0004
+#define SA_CMDL_UPD_AUTH_IV     0x0008
+#define SA_CMDL_UPD_AUX_KEY     0x0010
+
+#define SA_AUTH_SUBKEY_LEN	16
+#define SA_CMDL_PAYLOAD_LENGTH_MASK	0xFFFF
+#define SA_CMDL_SOP_BYPASS_LEN_MASK	0xFF000000
+
+#define MODE_CONTROL_BYTES	27
+#define SA_HASH_PROCESSING	0
+#define SA_CRYPTO_PROCESSING	0
+#define SA_UPLOAD_HASH_TO_TLR	BIT(6)
+
+#define SA_SW0_FLAGS_MASK	0xF0000
+#define SA_SW0_CMDL_INFO_MASK	0x1F00000
+#define SA_SW0_CMDL_PRESENT	BIT(4)
+#define SA_SW0_ENG_ID_MASK	0x3E000000
+#define SA_SW0_DEST_INFO_PRESENT	BIT(30)
+#define SA_SW2_EGRESS_LENGTH		0xFF000000
+#define SA_BASIC_HASH		0x10
+
+#define SHA256_DIGEST_WORDS    8
+/* Make 32-bit word from 4 bytes */
+#define SA_MK_U32(b0, b1, b2, b3) (((b0) << 24) | ((b1) << 16) | \
+				   ((b2) << 8) | (b3))
+
+/* size of SCCTL structure in bytes */
+#define SA_SCCTL_SZ 16
+
+/* Max Authentication tag size */
+#define SA_MAX_AUTH_TAG_SZ 64
+
+#define PRIV_ID	0x1
+#define PRIV	0x1
+
+static struct device *sa_k3_dev;
+
+/**
+ * struct sa_cmdl_cfg - Command label configuration descriptor
+ * @enc1st: If the iteration needs encryption before authentication
+ * @aalg: authentication algorithm ID
+ * @enc_eng_id: Encryption Engine ID supported by the SA hardware
+ * @auth_eng_id: authentication Engine ID
+ * @iv_size: Initialization Vector size
+ * @akey: Authentication key
+ * @akey_len: Authentication key length
+ */
+struct sa_cmdl_cfg {
+	int enc1st;
+	int aalg;
+	u8 enc_eng_id;
+	u8 auth_eng_id;
+	u8 iv_size;
+	const u8 *akey;
+	u16 akey_len;
+	u16 auth_subkey_len;
+};
+
+/**
+ * struct algo_data - Crypto algorithm specific data
+ * @enc_eng: Encryption engine info structure
+ * @auth_eng: Authentication engine info structure
+ * @auth_ctrl: Authentication control word
+ * @hash_size: Size of Digest
+ * @ealg_id: Encryption Algorithm ID
+ * @aalg_id: Authentication algorithm ID
+ * @mci_enc: Mode Control Instruction for Encryption algorithm
+ * @mci_dec: Mode Control Instruction for Decryption
+ * @inv_key: Whether the encryption algorithm demands key inversion
+ * @keyed_mac: Whether the Authentication algorithm has Key
+ * @prep_iopad: Function pointer to generate intermediate ipad/opad
+ */
+struct algo_data {
+	struct sa_eng_info enc_eng;
+	struct sa_eng_info auth_eng;
+	u8 auth_ctrl;
+	u8 hash_size;
+	u8 ealg_id;
+	u8 aalg_id;
+	u8 *mci_enc;
+	u8 *mci_dec;
+	bool inv_key;
+	bool keyed_mac;
+	void (*prep_iopad)(const u8 *key, u16 key_sz, u32 *ipad, u32 *opad);
+};
+
+/**
+ * struct sa_alg_tmpl: A generic template encompassing crypto/aead algorithms
+ * @alg: A union of aead/crypto algorithm type.
+ * @registered: Flag indicating if the crypto algorithm is already registered
+ */
+struct sa_alg_tmpl {
+	u32 type;		/* CRYPTO_ALG_TYPE from <linux/crypto.h> */
+	union {
+		struct crypto_alg crypto;
+		struct aead_alg aead;
+	} alg;
+	int registered;
+};
+
+/**
+ * struct sa_rx_data: RX Packet miscellaneous data place holder
+ * @req: crypto request data pointer
+ * @tx_in: dma_async_tx_descriptor pointer for rx channel
+ * @enc: Flag indicating either encryption or decryption
+ */
+struct sa_rx_data {
+	void *req;
+	struct device *ddev;
+	struct dma_async_tx_descriptor *tx_in;
+	u8 enc;
+};
+
+/*
+ * Mode Control Instructions for various Key lengths 128, 192, 256
+ * For CBC (Cipher Block Chaining) mode for encryption
+ */
+static u8 mci_cbc_enc_array[3][MODE_CONTROL_BYTES] = {
+	{	0x21, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00	},
+	{	0x21, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00	},
+	{	0x21, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00	},
+};
+
+/*
+ * Mode Control Instructions for various Key lengths 128, 192, 256
+ * For CBC (Cipher Block Chaining) mode for decryption
+ */
+static u8 mci_cbc_dec_array[3][MODE_CONTROL_BYTES] = {
+	{	0x31, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00	},
+	{	0x31, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00	},
+	{	0x31, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00	},
+};
+
+/*
+ * Mode Control Instructions for various Key lengths 128, 192, 256
+ * For ECB (Electronic Code Book) mode for encryption
+ */
+static u8 mci_ecb_enc_array[3][27] = {
+	{	0x21, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00	},
+	{	0x21, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00	},
+	{	0x21, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00	},
+};
+
+/*
+ * Mode Control Instructions for various Key lengths 128, 192, 256
+ * For ECB (Electronic Code Book) mode for decryption
+ */
+static u8 mci_ecb_dec_array[3][27] = {
+	{	0x31, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00	},
+	{	0x31, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00	},
+	{	0x31, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00	},
+};
+
+/*
+ * Mode Control Instructions for DES algorithm
+ * For CBC (Cipher Block Chaining) mode and ECB mode
+ * encryption and for decryption respectively
+ */
+static u8 mci_cbc_3des_enc_array[MODE_CONTROL_BYTES] = {
+	0x20, 0x00, 0x00, 0x18, 0x88, 0x52, 0xaa, 0x4b, 0x7e, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00,
+};
+
+static u8 mci_cbc_3des_dec_array[MODE_CONTROL_BYTES] = {
+	0x30, 0x00, 0x00, 0x85, 0x0a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00,
+};
+
+static u8 mci_ecb_3des_enc_array[MODE_CONTROL_BYTES] = {
+	0x20, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00,
+};
+
+static u8 mci_ecb_3des_dec_array[MODE_CONTROL_BYTES] = {
+	0x30, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00,
+};
+
+/*
+ * Perform 16 byte or 128 bit swizzling
+ * The SA2UL Expects the security context to
+ * be in little Endian and the bus width is 128 bits or 16 bytes
+ * Hence swap 16 bytes at a time from higher to lower address
+ */
+static void sa_swiz_128(u8 *in, u16 len)
+{
+	u8 data[16];
+	int i, j;
+
+	for (i = 0; i < len; i += 16) {
+		memcpy(data, &in[i], 16);
+		for (j = 0; j < 16; j++)
+			in[i + j] = data[15 - j];
+	}
+}
+
+/* Prepare the ipad and opad from key as per SHA algorithm step 1*/
+static void prepare_kiopad(u8 *k_ipad, u8 *k_opad, const u8 *key, u16 key_sz)
+{
+	int i;
+
+	for (i = 0; i < key_sz; i++) {
+		k_ipad[i] = key[i] ^ 0x36;
+		k_opad[i] = key[i] ^ 0x5c;
+	}
+
+	/* Instead of XOR with 0 */
+	for (; i < SHA_MESSAGE_BYTES; i++) {
+		k_ipad[i] = 0x36;
+		k_opad[i] = 0x5c;
+	}
+}
+
+/* Generate HMAC-SHA1 intermediate Hash */
+static
+void sa_hmac_sha1_get_pad(const u8 *key, u16 key_sz, u32 *ipad, u32 *opad)
+{
+	u32 ws[SHA_WORKSPACE_WORDS];
+	u8 k_ipad[SHA_MESSAGE_BYTES];
+	u8 k_opad[SHA_MESSAGE_BYTES];
+	int i;
+
+	prepare_kiopad(k_ipad, k_opad, key, key_sz);
+
+	/* SHA-1 on k_ipad */
+	sha_init(ipad);
+	sha_transform(ipad, k_ipad, ws);
+
+	for (i = 0; i < SHA_DIGEST_WORDS; i++)
+		ipad[i] = cpu_to_be32(ipad[i]);
+
+	/* SHA-1 on k_opad */
+	sha_init(opad);
+	sha_transform(opad, k_opad, ws);
+
+	for (i = 0; i < SHA_DIGEST_WORDS; i++)
+		opad[i] = cpu_to_be32(opad[i]);
+}
+
+void sha256_init(u32 *buf)
+{
+	buf[0] = SHA256_H0;
+	buf[1] = SHA256_H1;
+	buf[2] = SHA256_H2;
+	buf[3] = SHA256_H3;
+	buf[4] = SHA256_H4;
+	buf[5] = SHA256_H5;
+	buf[6] = SHA256_H6;
+	buf[7] = SHA256_H7;
+}
+
+static void sa_hmac_sha256_get_pad(const u8 *key, u16 key_sz, u32 *ipad,
+				   u32 *opad)
+{
+	u8 k_ipad[SHA_MESSAGE_BYTES];
+	u8 k_opad[SHA_MESSAGE_BYTES];
+	int i;
+
+	prepare_kiopad(k_ipad, k_opad, key, key_sz);
+
+	/* SHA-256 on k_ipad */
+	sha256_init(ipad);
+	sha256_transform(ipad, k_ipad);
+
+	for (i = 0; i < SHA256_DIGEST_WORDS; i++)
+		ipad[i] = cpu_to_be32(ipad[i]);
+
+	/* SHA-256 on k_opad */
+	sha256_init(opad);
+	sha256_transform(opad, k_opad);
+
+	for (i = 0; i < SHA256_DIGEST_WORDS; i++)
+		opad[i] = cpu_to_be32(opad[i]);
+}
+
+/* Derive the inverse key used in AES-CBC decryption operation */
+static inline int sa_aes_inv_key(u8 *inv_key, const u8 *key, u16 key_sz)
+{
+	struct crypto_aes_ctx ctx;
+	int key_pos;
+
+	if (crypto_aes_expand_key(&ctx, key, key_sz)) {
+		pr_err("%s: bad key len(%d)\n", __func__, key_sz);
+		return -EINVAL;
+	}
+
+	/* Based crypto_aes_expand_key logic */
+	switch (key_sz) {
+	case AES_KEYSIZE_128:
+	case AES_KEYSIZE_192:
+		key_pos = key_sz + 24;
+		break;
+
+	case AES_KEYSIZE_256:
+		key_pos = key_sz + 24 - 4;
+		break;
+
+	default:
+		pr_err("%s: bad key len(%d)\n", __func__, key_sz);
+		return -EINVAL;
+	}
+
+	memcpy(inv_key, &ctx.key_enc[key_pos], key_sz);
+	return 0;
+}
+
+/* Set Security context for the encryption engine */
+static int sa_set_sc_enc(struct algo_data *ad, const u8 *key, u16 key_sz,
+			 u16 aad_len, u8 enc, u8 *sc_buf)
+{
+	const u8 *mci = NULL;
+
+	/* Set Encryption mode selector to crypto processing */
+	sc_buf[0] = SA_CRYPTO_PROCESSING;
+
+	if (enc)
+		mci = ad->mci_enc;
+	else
+		mci = ad->mci_dec;
+	/* Set the mode control instructions in security context */
+	if (mci)
+		memcpy(&sc_buf[1], mci, MODE_CONTROL_BYTES);
+
+	/* For AES-CBC decryption get the inverse key */
+	if (ad->inv_key && !enc) {
+		if (sa_aes_inv_key(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz))
+			return -EINVAL;
+	/* For all other cases: key is used */
+	} else {
+		memcpy(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz);
+	}
+
+	return 0;
+}
+
+/* Set Security context for the authentication engine */
+static void sa_set_sc_auth(struct algo_data *ad, const u8 *key, u16 key_sz,
+			   u8 *sc_buf)
+{
+	u32 ipad[64], opad[64];
+
+	/* Set Authentication mode selector to hash processing */
+	sc_buf[0] = SA_HASH_PROCESSING;
+	/* Auth SW ctrl word: bit[6]=1 (upload computed hash to TLR section) */
+	sc_buf[1] = SA_UPLOAD_HASH_TO_TLR;
+	sc_buf[1] |= ad->auth_ctrl;
+
+	/* Copy the keys or ipad/opad */
+	if (ad->keyed_mac) {
+		ad->prep_iopad(key, key_sz, ipad, opad);
+		/* Copy ipad to AuthKey */
+		memcpy(&sc_buf[32], ipad, ad->hash_size);
+		/* Copy opad to Aux-1 */
+		memcpy(&sc_buf[64], opad, ad->hash_size);
+	} else {
+		/* basic hash */
+		sc_buf[1] |= SA_BASIC_HASH;
+	}
+}
+
+static inline void sa_copy_iv(u32 *out, const u8 *iv, bool size16)
+{
+	int j;
+
+	for (j = 0; j < ((size16) ? 4 : 2); j++) {
+		*out = cpu_to_be32(*((u32 *)iv));
+		iv += 4;
+		out++;
+	}
+}
+
+/* Format general command label */
+static int sa_format_cmdl_gen(struct sa_cmdl_cfg *cfg, u8 *cmdl,
+			      struct sa_cmdl_upd_info *upd_info)
+{
+	u8 enc_offset = 0, auth_offset = 0, total = 0;
+	u8 enc_next_eng = SA_ENG_ID_OUTPORT2;
+	u8 auth_next_eng = SA_ENG_ID_OUTPORT2;
+	u32 *word_ptr = (u32 *)cmdl;
+	int i;
+
+	/* Clear the command label */
+	memzero_explicit(cmdl, (SA_MAX_CMDL_WORDS * sizeof(u32)));
+
+	/* Iniialize the command update structure */
+	memzero_explicit(upd_info, sizeof(*upd_info));
+
+	if (cfg->enc1st) {
+		if (cfg->enc_eng_id != SA_ENG_ID_NONE)
+			auth_offset = SA_CMDL_HEADER_SIZE_BYTES;
+
+		if (cfg->iv_size)
+			auth_offset += cfg->iv_size;
+
+		if (cfg->auth_eng_id != SA_ENG_ID_NONE)
+			enc_next_eng = cfg->auth_eng_id;
+		else
+			enc_next_eng = SA_ENG_ID_OUTPORT2;
+	} else {
+		if (cfg->auth_eng_id != SA_ENG_ID_NONE)
+			enc_offset = SA_CMDL_HEADER_SIZE_BYTES;
+
+		if (cfg->auth_subkey_len)
+			enc_offset += cfg->auth_subkey_len;
+
+		if (cfg->enc_eng_id != SA_ENG_ID_NONE)
+			auth_next_eng = cfg->enc_eng_id;
+		else
+			auth_next_eng = SA_ENG_ID_OUTPORT2;
+	}
+
+	if (cfg->enc_eng_id != SA_ENG_ID_NONE) {
+		upd_info->flags |= SA_CMDL_UPD_ENC;
+		upd_info->enc_size.index = enc_offset >> 2;
+		upd_info->enc_offset.index = upd_info->enc_size.index + 1;
+		/* Encryption command label */
+		cmdl[enc_offset + SA_CMDL_OFFSET_NESC] = enc_next_eng;
+
+		/* Encryption modes requiring IV */
+		if (cfg->iv_size) {
+			upd_info->flags |= SA_CMDL_UPD_ENC_IV;
+			upd_info->enc_iv.index =
+				(enc_offset + SA_CMDL_HEADER_SIZE_BYTES) >> 2;
+			upd_info->enc_iv.size = cfg->iv_size;
+
+			cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] =
+				SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size;
+
+			cmdl[enc_offset + SA_CMDL_OFFSET_OPTION_CTRL1] =
+				(SA_CTX_ENC_AUX2_OFFSET | (cfg->iv_size >> 3));
+			enc_offset += SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size;
+		} else {
+			cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] =
+						SA_CMDL_HEADER_SIZE_BYTES;
+			enc_offset += SA_CMDL_HEADER_SIZE_BYTES;
+		}
+	}
+
+	if (cfg->auth_eng_id != SA_ENG_ID_NONE) {
+		upd_info->flags |= SA_CMDL_UPD_AUTH;
+		upd_info->auth_size.index = auth_offset >> 2;
+		upd_info->auth_offset.index = upd_info->auth_size.index + 1;
+		cmdl[auth_offset + SA_CMDL_OFFSET_NESC] = auth_next_eng;
+
+		/* Algorithm with subkeys */
+		if (cfg->aalg == SA_AALG_ID_AES_XCBC ||
+		    cfg->aalg == SA_AALG_ID_CMAC) {
+			upd_info->flags |= SA_CMDL_UPD_AUX_KEY;
+			upd_info->aux_key_info.index =
+				(auth_offset + SA_CMDL_HEADER_SIZE_BYTES) >> 2;
+			cmdl[auth_offset + SA_CMDL_OFFSET_LABEL_LEN] =
+						SA_CMDL_HEADER_SIZE_BYTES +
+							cfg->auth_subkey_len;
+			cmdl[auth_offset + SA_CMDL_OFFSET_OPTION_CTRL1] =
+						(SA_CTX_ENC_AUX1_OFFSET |
+						(cfg->auth_subkey_len  >> 3));
+
+			auth_offset += SA_CMDL_HEADER_SIZE_BYTES +
+							cfg->auth_subkey_len;
+		} else {
+			cmdl[auth_offset + SA_CMDL_OFFSET_LABEL_LEN] =
+						SA_CMDL_HEADER_SIZE_BYTES;
+			auth_offset += SA_CMDL_HEADER_SIZE_BYTES;
+		}
+	}
+
+	if (cfg->enc1st)
+		total = auth_offset;
+	else
+		total = enc_offset;
+
+	total = roundup(total, 8);
+
+	for (i = 0; i < total / 4; i++)
+		word_ptr[i] = be32_to_cpu(word_ptr[i]);
+
+	return total;
+}
+
+/* Update Command label */
+static inline void
+sa_update_cmdl(struct device *dev, u8 enc_offset, u16 enc_size, u8 *enc_iv,
+	       u8 auth_offset, u16 auth_size, u8 *auth_iv, u8 aad_size,
+	       u8 *aad, struct sa_cmdl_upd_info *upd_info, u32 *cmdl)
+{
+	int i = 0, j;
+
+	if (upd_info->submode != SA_MODE_GEN) {
+		dev_err(dev, "unsupported mode(%d)\n", upd_info->submode);
+		return;
+	}
+
+	if (likely(upd_info->flags & SA_CMDL_UPD_ENC)) {
+		cmdl[upd_info->enc_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK;
+		cmdl[upd_info->enc_size.index] |= enc_size;
+		cmdl[upd_info->enc_offset.index] &=
+						~SA_CMDL_SOP_BYPASS_LEN_MASK;
+		cmdl[upd_info->enc_offset.index] |=
+			((u32)enc_offset << __ffs(SA_CMDL_SOP_BYPASS_LEN_MASK));
+
+		if (likely(upd_info->flags & SA_CMDL_UPD_ENC_IV)) {
+			u32 *data = &cmdl[upd_info->enc_iv.index];
+
+			for (j = 0; i < upd_info->enc_iv.size; i += 4, j++) {
+				data[j] = cpu_to_be32(*((u32 *)enc_iv));
+				enc_iv += 4;
+			}
+		}
+	}
+
+	if (likely(upd_info->flags & SA_CMDL_UPD_AUTH)) {
+		cmdl[upd_info->auth_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK;
+		cmdl[upd_info->auth_size.index] |= auth_size;
+		cmdl[upd_info->auth_offset.index] &=
+						~SA_CMDL_SOP_BYPASS_LEN_MASK;
+		cmdl[upd_info->auth_offset.index] |= ((u32)auth_offset <<
+					__ffs(SA_CMDL_SOP_BYPASS_LEN_MASK));
+		if (upd_info->flags & SA_CMDL_UPD_AUTH_IV) {
+			sa_copy_iv(&cmdl[upd_info->auth_iv.index], auth_iv,
+				   (upd_info->auth_iv.size > 8));
+		}
+
+		if (upd_info->flags & SA_CMDL_UPD_AUX_KEY) {
+			int offset = (auth_size & 0xF) ? 4 : 0;
+
+			memcpy(&cmdl[upd_info->aux_key_info.index],
+			       &upd_info->aux_key[offset], 16);
+		}
+	}
+}
+
+/* Format SWINFO words to be sent to SA */
+static
+void sa_set_swinfo(u8 eng_id, u16 sc_id, dma_addr_t sc_phys,
+		   u8 cmdl_present, u8 cmdl_offset, u8 flags,
+		   u8 hash_size, u32 *swinfo)
+{
+	swinfo[0] = sc_id;
+	swinfo[0] |= (flags << __ffs(SA_SW0_FLAGS_MASK));
+	if (likely(cmdl_present))
+		swinfo[0] |= ((cmdl_offset | SA_SW0_CMDL_PRESENT) <<
+						__ffs(SA_SW0_CMDL_INFO_MASK));
+	swinfo[0] |= (eng_id << __ffs(SA_SW0_ENG_ID_MASK));
+
+	swinfo[0] |= SA_SW0_DEST_INFO_PRESENT;
+	swinfo[1] = (u32)(sc_phys & 0xFFFFFFFFULL);
+	swinfo[2] = (u32)((sc_phys & 0xFFFFFFFF00000000ULL) >> 32);
+	swinfo[2] |= (hash_size << __ffs(SA_SW2_EGRESS_LENGTH));
+}
+
+/* Dump the security context */
+static void sa_dump_sc(u8 *buf, dma_addr_t dma_addr)
+{
+#ifdef DEBUG
+	dev_info(sa_k3_dev, "Security context dump:: 0x%pad\n", &dma_addr);
+	print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET,
+		       16, 1, buf, SA_CTX_MAX_SZ, false);
+#endif
+}
+
+static
+int sa_init_sc(struct sa_ctx_info *ctx, const u8 *enc_key,
+	       u16 enc_key_sz, const u8 *auth_key, u16 auth_key_sz,
+	       struct algo_data *ad, u8 enc, u32 *swinfo, bool auth_req)
+{
+	int use_enc = 0;
+	int enc_sc_offset = 0, auth_sc_offset = 0;
+	u8 *sc_buf = ctx->sc;
+	u16 sc_id = ctx->sc_id;
+	u16 aad_len = 0;	/* Currently not supporting AEAD algo */
+	u8 first_engine;
+
+	memzero_explicit(sc_buf, SA_CTX_MAX_SZ);
+
+	if (ad->auth_eng.eng_id <= SA_ENG_ID_EM2 || !auth_req)
+		use_enc = 1;
+
+	/* Determine the order of encryption & Authentication contexts */
+	if (enc || !use_enc) {
+		if (auth_req) {
+			enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ;
+			auth_sc_offset = enc_sc_offset + ad->enc_eng.sc_size;
+		} else {
+			enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ;
+		}
+	} else {
+		auth_sc_offset = SA_CTX_PHP_PE_CTX_SZ;
+		enc_sc_offset = auth_sc_offset + ad->auth_eng.sc_size;
+	}
+
+	/* SCCTL Owner info: 0=host, 1=CP_ACE */
+	sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0;
+	/* SCCTL F/E control */
+	if (auth_req)
+		sc_buf[1] = SA_SCCTL_FE_AUTH_ENC;
+	else
+		sc_buf[1] = SA_SCCTL_FE_ENC;
+	memcpy(&sc_buf[2], &sc_id, 2);
+	sc_buf[4] = 0x0;
+	sc_buf[5] = PRIV_ID;
+	sc_buf[6] = PRIV;
+	sc_buf[7] = 0x0;
+
+	/* Initialize the rest of PHP context */
+	memzero_explicit(sc_buf + SA_SCCTL_SZ, SA_CTX_PHP_PE_CTX_SZ -
+			 SA_SCCTL_SZ);
+
+	/* Prepare context for encryption engine */
+	if (ad->enc_eng.sc_size) {
+		if (sa_set_sc_enc(ad, enc_key, enc_key_sz, aad_len,
+				  enc, &sc_buf[enc_sc_offset]))
+			return -EINVAL;
+	}
+
+	/* Prepare context for authentication engine */
+	if (ad->auth_eng.sc_size) {
+		if (use_enc) {
+			if (sa_set_sc_enc(ad, auth_key, auth_key_sz,
+					  aad_len, 0, &sc_buf[auth_sc_offset]))
+				return -EINVAL;
+		} else {
+			sa_set_sc_auth(ad, auth_key, auth_key_sz,
+				       &sc_buf[auth_sc_offset]);
+		}
+	}
+
+	/* Set the ownership of context to CP_ACE */
+	sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0x80;
+
+	/* swizzle the security context */
+	sa_swiz_128(sc_buf, SA_CTX_MAX_SZ);
+	/* Setup SWINFO */
+	if (!auth_req)
+		first_engine = ad->enc_eng.eng_id;
+	else
+		first_engine = enc ? ad->enc_eng.eng_id : ad->auth_eng.eng_id;
+
+	if (auth_req) {
+		if (!ad->hash_size)
+			return -EINVAL;
+		/* Round up the tag size to multiple of 4 */
+		ad->hash_size = roundup(ad->hash_size, 8);
+	}
+
+	sa_set_swinfo(first_engine, ctx->sc_id, ctx->sc_phys, 1, 0,
+		      SA_SW_INFO_FLAG_EVICT, ad->hash_size, swinfo);
+
+	sa_dump_sc(sc_buf, ctx->sc_phys);
+
+	return 0;
+}
+
+/* Free the per direction context memory */
+static void sa_free_ctx_info(struct sa_ctx_info *ctx,
+			     struct sa_crypto_data *data)
+{
+	unsigned long bn;
+
+	bn = ctx->sc_id - data->sc_id_start;
+	spin_lock(&data->scid_lock);
+	__clear_bit(bn, data->ctx_bm);
+	data->sc_id--;
+	spin_unlock(&data->scid_lock);
+
+	if (ctx->sc) {
+		dma_pool_free(data->sc_pool, ctx->sc, ctx->sc_phys);
+		ctx->sc = NULL;
+	}
+}
+
+static int sa_init_ctx_info(struct sa_ctx_info *ctx,
+			    struct sa_crypto_data *data)
+{
+	unsigned long bn;
+	int err;
+
+	spin_lock(&data->scid_lock);
+	bn = find_first_zero_bit(data->ctx_bm, SA_MAX_NUM_CTX);
+	__set_bit(bn, data->ctx_bm);
+	data->sc_id++;
+	spin_unlock(&data->scid_lock);
+
+	ctx->sc_id = (u16)(data->sc_id_start + bn);
+
+	ctx->sc = dma_pool_alloc(data->sc_pool, GFP_KERNEL, &ctx->sc_phys);
+	if (!ctx->sc) {
+		dev_err(&data->pdev->dev, "Failed to allocate SC memory\n");
+		err = -ENOMEM;
+		goto scid_rollback;
+	}
+
+	return 0;
+
+scid_rollback:
+	spin_lock(&data->scid_lock);
+	__clear_bit(bn, data->ctx_bm);
+	data->sc_id--;
+	spin_unlock(&data->scid_lock);
+
+	return err;
+}
+
+static void sa_aes_cra_exit(struct crypto_tfm *tfm)
+{
+	struct crypto_alg *alg = tfm->__crt_alg;
+	struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
+
+	dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
+		__func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
+		ctx->dec.sc_id, &ctx->dec.sc_phys);
+
+	if ((alg->cra_flags & CRYPTO_ALG_TYPE_ABLKCIPHER)
+	    == CRYPTO_ALG_TYPE_ABLKCIPHER) {
+		sa_free_ctx_info(&ctx->enc, data);
+		sa_free_ctx_info(&ctx->dec, data);
+	}
+}
+
+static int sa_aes_cra_init(struct crypto_tfm *tfm)
+{
+	struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct crypto_alg *alg = tfm->__crt_alg;
+	struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
+	int ret;
+
+	if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) ==
+	    CRYPTO_ALG_TYPE_ABLKCIPHER) {
+		memzero_explicit(ctx, sizeof(*ctx));
+		ctx->dev_data = data;
+
+		ret = sa_init_ctx_info(&ctx->enc, data);
+		if (ret)
+			return ret;
+		ret = sa_init_ctx_info(&ctx->dec, data);
+		if (ret) {
+			sa_free_ctx_info(&ctx->enc, data);
+			return ret;
+		}
+	}
+
+	dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
+		__func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
+		ctx->dec.sc_id, &ctx->dec.sc_phys);
+	return 0;
+}
+
+static int sa_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
+			 unsigned int keylen, struct algo_data *ad)
+{
+	struct sa_tfm_ctx *ctx = crypto_ablkcipher_ctx(tfm);
+
+	const char *cra_name;
+	int cmdl_len;
+	struct sa_cmdl_cfg cfg;
+
+	if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
+	    keylen != AES_KEYSIZE_256)
+		return -EINVAL;
+
+	cra_name = crypto_tfm_alg_name(&tfm->base);
+
+	memzero_explicit(&cfg, sizeof(cfg));
+	cfg.enc1st = 1;
+	cfg.enc_eng_id = ad->enc_eng.eng_id;
+	cfg.iv_size = crypto_ablkcipher_ivsize(tfm);
+	cfg.auth_eng_id = SA_ENG_ID_NONE;
+	cfg.auth_subkey_len = 0;
+
+	/* Setup Encryption Security Context & Command label template */
+	if (sa_init_sc(&ctx->enc, key, keylen,
+		       NULL, 0, ad, 1, &ctx->enc.epib[1], false))
+		goto badkey;
+
+	cmdl_len = sa_format_cmdl_gen(&cfg,
+				      (u8 *)ctx->enc.cmdl,
+				      &ctx->enc.cmdl_upd_info);
+	if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
+		goto badkey;
+
+	ctx->enc.cmdl_size = cmdl_len;
+
+	/* Setup Decryption Security Context & Command label template */
+	if (sa_init_sc(&ctx->dec, key, keylen,
+		       NULL, 0, ad, 0, &ctx->dec.epib[1], false))
+		goto badkey;
+
+	cfg.enc1st = 0;
+	cfg.enc_eng_id = ad->enc_eng.eng_id;
+	cfg.auth_eng_id = SA_ENG_ID_NONE;
+	cfg.auth_subkey_len = 0;
+	cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl,
+				      &ctx->dec.cmdl_upd_info);
+
+	if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
+		goto badkey;
+
+	ctx->dec.cmdl_size = cmdl_len;
+
+	kfree(ad);
+
+	return 0;
+
+badkey:
+	dev_err(sa_k3_dev, "%s: badkey\n", __func__);
+	return -EINVAL;
+}
+
+static int sa_aes_cbc_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
+			     unsigned int keylen)
+{
+	struct algo_data *ad = kzalloc(sizeof(*ad), GFP_KERNEL);
+	/* Convert the key size (16/24/32) to the key size index (0/1/2) */
+	int key_idx = (keylen >> 3) - 2;
+
+	ad->enc_eng.eng_id = SA_ENG_ID_EM1;
+	ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
+	ad->auth_eng.eng_id = SA_ENG_ID_NONE;
+	ad->auth_eng.sc_size = 0;
+	ad->mci_enc = mci_cbc_enc_array[key_idx];
+	ad->mci_dec = mci_cbc_dec_array[key_idx];
+	ad->inv_key = true;
+	ad->ealg_id = SA_EALG_ID_AES_CBC;
+	ad->aalg_id = SA_AALG_ID_NONE;
+
+	return sa_aes_setkey(tfm, key, keylen, ad);
+}
+
+static int sa_aes_ecb_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
+			     unsigned int keylen)
+{
+	struct algo_data *ad = kzalloc(sizeof(*ad), GFP_KERNEL);
+	/* Convert the key size (16/24/32) to the key size index (0/1/2) */
+	int key_idx = (keylen >> 3) - 2;
+
+	ad->enc_eng.eng_id = SA_ENG_ID_EM1;
+	ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
+	ad->auth_eng.eng_id = SA_ENG_ID_NONE;
+	ad->auth_eng.sc_size = 0;
+	ad->mci_enc = mci_ecb_enc_array[key_idx];
+	ad->mci_dec = mci_ecb_dec_array[key_idx];
+	ad->inv_key = true;
+	ad->ealg_id = SA_EALG_ID_AES_ECB;
+	ad->aalg_id = SA_AALG_ID_NONE;
+
+	return sa_aes_setkey(tfm, key, keylen, ad);
+}
+
+static int sa_3des_cbc_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
+			      unsigned int keylen)
+{
+	struct algo_data *ad = kzalloc(sizeof(*ad), GFP_KERNEL);
+
+	ad->enc_eng.eng_id = SA_ENG_ID_EM1;
+	ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
+	ad->auth_eng.eng_id = SA_ENG_ID_NONE;
+	ad->auth_eng.sc_size = 0;
+	ad->mci_enc = mci_cbc_3des_enc_array;
+	ad->mci_dec = mci_cbc_3des_dec_array;
+	ad->ealg_id = SA_EALG_ID_3DES_CBC;
+	ad->aalg_id = SA_AALG_ID_NONE;
+
+	return sa_aes_setkey(tfm, key, keylen, ad);
+}
+
+static int sa_3des_ecb_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
+			      unsigned int keylen)
+{
+	struct algo_data *ad = kzalloc(sizeof(*ad), GFP_KERNEL);
+
+	ad->enc_eng.eng_id = SA_ENG_ID_EM1;
+	ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
+	ad->auth_eng.eng_id = SA_ENG_ID_NONE;
+	ad->auth_eng.sc_size = 0;
+	ad->mci_enc = mci_ecb_3des_enc_array;
+	ad->mci_dec = mci_ecb_3des_dec_array;
+	ad->aalg_id = SA_AALG_ID_NONE;
+
+	return sa_aes_setkey(tfm, key, keylen, ad);
+}
+
+static void sa_aes_dma_in_callback(void *data)
+{
+	struct sa_rx_data *rxd = (struct sa_rx_data *)data;
+	struct ablkcipher_request *req = (struct ablkcipher_request *)rxd->req;
+
+	int sglen = sg_nents_for_len(req->dst, req->nbytes);
+
+	dma_unmap_sg(rxd->ddev, req->src, sglen, DMA_TO_DEVICE);
+	if (req->src != req->dst)
+		dma_unmap_sg(rxd->ddev, req->dst, sglen, DMA_FROM_DEVICE);
+
+	kfree(rxd);
+
+	ablkcipher_request_complete(req, 0);
+}
+
+static void sa_aead_dma_in_callback(void *data)
+{
+	struct sa_rx_data *rxd = (struct sa_rx_data *)data;
+	struct aead_request *req = (struct aead_request *)rxd->req;
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	u32 *mdptr;
+	unsigned int start = req->assoclen + req->cryptlen;
+	unsigned int authsize = crypto_aead_authsize(tfm);
+	u8 auth_tag[SA_MAX_AUTH_TAG_SZ];
+	int i, sglen, err = 0;
+	size_t pl, ml;
+
+	mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml);
+	for (i = 0; i < (authsize / 4); i++)
+		mdptr[i + 4] = htonl(mdptr[i + 4]);
+
+	if (rxd->enc) {
+		scatterwalk_map_and_copy((void *)&mdptr[4], req->dst,
+					 start, crypto_aead_authsize(tfm), 1);
+	} else {
+		start -= authsize;
+		scatterwalk_map_and_copy(auth_tag, req->src,
+					 start, crypto_aead_authsize(tfm), 0);
+
+		err = memcmp((void *)&mdptr[4],
+			     auth_tag, authsize) ? -EBADMSG : 0;
+	}
+
+	sglen = sg_nents_for_len(req->dst, req->cryptlen + authsize);
+	dma_unmap_sg(rxd->ddev, req->dst, sglen, DMA_FROM_DEVICE);
+
+	sglen =  sg_nents_for_len(req->src, req->assoclen + req->cryptlen);
+	dma_unmap_sg(rxd->ddev, req->src, sglen, DMA_TO_DEVICE);
+
+	kfree(rxd);
+
+	aead_request_complete(req, err);
+}
+
+static void
+sa_prepare_tx_desc(u32 *mdptr, u32 pslen, u32 *psdata, u32 epiblen, u32 *epib)
+{
+	u32 *out, *in;
+	int i;
+
+	for (out = mdptr, in = epib, i = 0; i < epiblen / sizeof(u32); i++)
+		*out++ = *in++;
+
+	mdptr[4] = (0xFFFF << 16);
+	for (out = &mdptr[5], in = psdata, i = 0;
+	     i < pslen / sizeof(u32); i++)
+		*out++ = *in++;
+}
+
+static int sa_aes_run(struct ablkcipher_request *req, u8 *iv, int enc)
+{
+	struct sa_tfm_ctx *ctx =
+	    crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
+	struct sa_ctx_info *sa_ctx = enc ? &ctx->enc : &ctx->dec;
+	struct sa_crypto_data *pdata = dev_get_drvdata(sa_k3_dev);
+	struct sa_dma_req_ctx req_ctx;
+	struct dma_async_tx_descriptor *tx_in, *tx_out;
+	struct sa_rx_data *rxd;
+	u8 enc_offset;
+	int sg_nents, dst_nents;
+	int psdata_offset;
+	u8 auth_offset = 0;
+	u8 *auth_iv = NULL;
+	u8 *aad = NULL;
+	u8 aad_len = 0;
+	u16 enc_len;
+	u16 auth_len = 0;
+	u32 req_type;
+	u32 *mdptr;
+	size_t pl, ml;
+	struct device *ddev;
+	struct dma_chan *dma_rx;
+	gfp_t flags;
+
+	flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+		GFP_KERNEL : GFP_ATOMIC;
+
+	enc_offset = 0x0;
+	enc_len = req->nbytes;
+
+	if (enc_len >= 256)
+		dma_rx = pdata->dma_rx2;
+	else
+		dma_rx = pdata->dma_rx1;
+	/* Allocate descriptor & submit packet */
+	sg_nents = sg_nents_for_len(req->src, enc_len);
+	dst_nents = sg_nents_for_len(req->dst, enc_len);
+
+	memcpy(req_ctx.cmdl, sa_ctx->cmdl, sa_ctx->cmdl_size);
+
+	/* Update Command Label */
+	sa_update_cmdl(sa_k3_dev, enc_offset, enc_len,
+		       iv, auth_offset, auth_len,
+		       auth_iv, aad_len, aad,
+		       &sa_ctx->cmdl_upd_info, req_ctx.cmdl);
+
+	/*
+	 * Last 2 words in PSDATA will have the crypto alg type &
+	 * crypto request pointer
+	 */
+	req_type = CRYPTO_ALG_TYPE_ABLKCIPHER;
+	if (enc)
+		req_type |= (SA_REQ_SUBTYPE_ENC << SA_REQ_SUBTYPE_SHIFT);
+	else
+		req_type |= (SA_REQ_SUBTYPE_DEC << SA_REQ_SUBTYPE_SHIFT);
+
+	psdata_offset = sa_ctx->cmdl_size / sizeof(u32);
+	req_ctx.cmdl[psdata_offset++] = req_type;
+
+	ddev = dma_rx->device->dev;
+	/* map the packet */
+	req_ctx.src = req->src;
+	req_ctx.src_nents = dma_map_sg(ddev, req->src, sg_nents, DMA_TO_DEVICE);
+	if (req->src != req->dst)
+		dst_nents = dma_map_sg(ddev, req->dst,
+				       sg_nents, DMA_FROM_DEVICE);
+	else
+		dst_nents = req_ctx.src_nents;
+
+	if (unlikely(req_ctx.src_nents != sg_nents)) {
+		dev_warn_ratelimited(sa_k3_dev, "failed to map tx pkt\n");
+		return -EIO;
+	}
+
+	req_ctx.dev_data = pdata;
+	req_ctx.pkt = true;
+
+	dma_sync_sg_for_device(pdata->dev, req->src, req_ctx.src_nents,
+			       DMA_TO_DEVICE);
+
+	tx_in = dmaengine_prep_slave_sg(dma_rx, req->dst, dst_nents,
+					DMA_DEV_TO_MEM,
+					DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+	if (!tx_in) {
+		dev_err(pdata->dev, "IN prep_slave_sg() failed\n");
+		return -EINVAL;
+	}
+
+	rxd = kzalloc(sizeof(*rxd), GFP_KERNEL);
+	rxd->req = (void *)req;
+	rxd->ddev = ddev;
+
+	/* IN */
+	tx_in->callback = sa_aes_dma_in_callback;
+	tx_in->callback_param = rxd;
+
+	tx_out = dmaengine_prep_slave_sg(pdata->dma_tx, req->src,
+					 req_ctx.src_nents, DMA_MEM_TO_DEV,
+					DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+	if (!tx_out) {
+		dev_err(pdata->dev, "OUT prep_slave_sg() failed\n");
+		return -EINVAL;
+	}
+
+	mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(tx_out, &pl, &ml);
+
+	sa_prepare_tx_desc(mdptr, (sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS *
+			   sizeof(u32))), req_ctx.cmdl,
+			   sizeof(sa_ctx->epib), sa_ctx->epib);
+
+	ml = sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS * sizeof(u32));
+	dmaengine_desc_set_metadata_len(tx_out, 44);
+
+	dmaengine_submit(tx_out);
+	dmaengine_submit(tx_in);
+
+	dma_async_issue_pending(dma_rx);
+	dma_async_issue_pending(pdata->dma_tx);
+
+	return -EINPROGRESS;
+}
+
+static int sa_aes_cbc_encrypt(struct ablkcipher_request *req)
+{
+	return sa_aes_run(req, req->info, 1);
+}
+
+static int sa_aes_cbc_decrypt(struct ablkcipher_request *req)
+{
+	return sa_aes_run(req, req->info, 0);
+}
+
+static int sa_init_tfm(struct crypto_tfm *tfm)
+{
+	struct crypto_alg *alg = tfm->__crt_alg;
+	struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
+	int ret;
+
+	if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AEAD) {
+		memset(ctx, 0, sizeof(*ctx));
+		ctx->dev_data = data;
+
+		ret = sa_init_ctx_info(&ctx->enc, data);
+		if (ret)
+			return ret;
+		ret = sa_init_ctx_info(&ctx->dec, data);
+		if (ret) {
+			sa_free_ctx_info(&ctx->enc, data);
+			return ret;
+		}
+	}
+
+	dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
+		__func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
+		ctx->dec.sc_id, &ctx->dec.sc_phys);
+	return 0;
+}
+
+/* Algorithm init */
+static int sa_cra_init_aead(struct crypto_aead *tfm)
+{
+	return sa_init_tfm(crypto_aead_tfm(tfm));
+}
+
+/* Algorithm context teardown */
+static void sa_exit_tfm(struct crypto_tfm *tfm)
+{
+	struct crypto_alg *alg = tfm->__crt_alg;
+	struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
+
+	dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
+		__func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
+		ctx->dec.sc_id, &ctx->dec.sc_phys);
+
+	if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK)
+		== CRYPTO_ALG_TYPE_AEAD) {
+		sa_free_ctx_info(&ctx->enc, data);
+		sa_free_ctx_info(&ctx->dec, data);
+	}
+}
+
+static void sa_exit_tfm_aead(struct crypto_aead *tfm)
+{
+	return sa_exit_tfm(crypto_aead_tfm(tfm));
+}
+
+/* AEAD algorithm configuration interface function */
+static int sa_aead_setkey(struct crypto_aead *authenc,
+			  const u8 *key, unsigned int keylen,
+			  struct algo_data *ad)
+{
+	struct sa_tfm_ctx *ctx = crypto_aead_ctx(authenc);
+	struct crypto_authenc_keys keys;
+
+	const char *cra_name;
+	int cmdl_len;
+	struct sa_cmdl_cfg cfg;
+
+	if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
+		goto badkey;
+
+	cra_name = crypto_tfm_alg_name(crypto_aead_tfm(authenc));
+
+	memset(&cfg, 0, sizeof(cfg));
+	cfg.enc1st = 1;
+	cfg.aalg = ad->aalg_id;
+	cfg.enc_eng_id = ad->enc_eng.eng_id;
+	cfg.auth_eng_id = ad->auth_eng.eng_id;
+	cfg.iv_size = crypto_aead_ivsize(authenc);
+	cfg.akey = keys.authkey;
+	cfg.akey_len = keys.authkeylen;
+
+	/* Setup Encryption Security Context & Command label template */
+	if (sa_init_sc(&ctx->enc, keys.enckey, keys.enckeylen,
+		       keys.authkey, keys.authkeylen,
+		       ad, 1, &ctx->enc.epib[1], true))
+		goto badkey;
+
+	cmdl_len = sa_format_cmdl_gen(&cfg,
+				      (u8 *)ctx->enc.cmdl,
+				      &ctx->enc.cmdl_upd_info);
+	if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
+		goto badkey;
+
+	ctx->enc.cmdl_size = cmdl_len;
+
+	/* Setup Decryption Security Context & Command label template */
+	if (sa_init_sc(&ctx->dec, keys.enckey, keys.enckeylen,
+		       keys.authkey, keys.authkeylen,
+		       ad, 0, &ctx->dec.epib[1], true))
+		goto badkey;
+
+	cfg.enc1st = 0;
+	cfg.enc_eng_id = ad->enc_eng.eng_id;
+	cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl,
+				      &ctx->dec.cmdl_upd_info);
+
+	if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
+		goto badkey;
+
+	ctx->dec.cmdl_size = cmdl_len;
+
+	kfree(ad);
+
+	return 0;
+
+badkey:
+	dev_err(sa_k3_dev, "%s: badkey\n", __func__);
+	return -EINVAL;
+}
+
+static int sa_aead_cbc_sha1_setkey(struct crypto_aead *authenc,
+				   const u8 *key, unsigned int keylen)
+{
+	struct algo_data *ad = kzalloc(sizeof(*ad), GFP_KERNEL);
+	struct crypto_authenc_keys keys;
+	int ret = 0, key_idx;
+
+	ret = crypto_authenc_extractkeys(&keys, key, keylen);
+	if (ret)
+		return ret;
+
+	/* Convert the key size (16/24/32) to the key size index (0/1/2) */
+	key_idx = (keys.enckeylen >> 3) - 2;
+	ad->enc_eng.eng_id = SA_ENG_ID_EM1;
+	ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
+	ad->auth_eng.eng_id = SA_ENG_ID_AM1;
+	ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ;
+	ad->mci_enc = mci_cbc_enc_array[key_idx];
+	ad->mci_dec = mci_cbc_dec_array[key_idx];
+	ad->inv_key = true;
+	ad->keyed_mac = true;
+	ad->ealg_id = SA_EALG_ID_AES_CBC;
+	ad->aalg_id = SA_AALG_ID_HMAC_SHA1;
+	ad->hash_size = SHA1_DIGEST_SIZE;
+	ad->auth_ctrl = 0x2;
+	ad->prep_iopad = sa_hmac_sha1_get_pad;
+
+	return sa_aead_setkey(authenc, key, keylen, ad);
+}
+
+static int sa_aead_cbc_sha256_setkey(struct crypto_aead *authenc,
+				     const u8 *key, unsigned int keylen)
+{
+	struct algo_data *ad = kzalloc(sizeof(*ad), GFP_KERNEL);
+	struct crypto_authenc_keys keys;
+	int ret = 0, key_idx;
+
+	ret = crypto_authenc_extractkeys(&keys, key, keylen);
+	if (ret)
+		return ret;
+
+	/* Convert the key size (16/24/32) to the key size index (0/1/2) */
+	key_idx = (keys.enckeylen >> 3) - 2;
+
+	ad->enc_eng.eng_id = SA_ENG_ID_EM1;
+	ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
+	ad->auth_eng.eng_id = SA_ENG_ID_AM1;
+	ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ;
+	ad->mci_enc = mci_cbc_enc_array[key_idx];
+	ad->mci_dec = mci_cbc_dec_array[key_idx];
+	ad->inv_key = true;
+	ad->keyed_mac = true;
+	ad->ealg_id = SA_EALG_ID_AES_CBC;
+	ad->aalg_id = SA_AALG_ID_HMAC_SHA2_256;
+	ad->hash_size = SHA256_DIGEST_SIZE;
+	ad->auth_ctrl = 0x4;
+	ad->prep_iopad = sa_hmac_sha256_get_pad;
+
+	return sa_aead_setkey(authenc, key, keylen, ad);
+}
+
+static int sa_aead_run(struct aead_request *req, u8 *iv, int enc)
+{
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm);
+	struct sa_ctx_info *sa_ctx = enc ? &ctx->enc : &ctx->dec;
+	struct sa_rx_data *rxd;
+	struct dma_async_tx_descriptor *tx_in, *tx_out;
+	struct sa_crypto_data *pdata = dev_get_drvdata(sa_k3_dev);
+	struct sa_dma_req_ctx req_ctx;
+	u8 enc_offset;
+	int sg_nents, dst_nents;
+	int psdata_offset;
+	u8 auth_offset = 0;
+	u8 *auth_iv = NULL;
+	u8 *aad = NULL;
+	u8 aad_len = 0;
+	u16 enc_len;
+	u16 auth_len = 0;
+	u32 *mdptr;
+	u32 req_type;
+	struct dma_chan *dma_rx;
+	gfp_t flags;
+	size_t pl, ml;
+	struct device *ddev;
+
+	flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+			GFP_KERNEL : GFP_ATOMIC;
+
+	if (enc) {
+		iv = (u8 *)(req->iv);
+		enc_offset = req->assoclen;
+		enc_len = req->cryptlen;
+		auth_len = req->assoclen + req->cryptlen;
+	} else {
+		enc_offset = req->assoclen;
+		enc_len = req->cryptlen - crypto_aead_authsize(tfm);
+		auth_len = req->assoclen + req->cryptlen -
+			crypto_aead_authsize(tfm);
+	}
+
+	if (enc_len >= 256)
+		dma_rx = pdata->dma_rx2;
+	else
+		dma_rx = pdata->dma_rx1;
+
+	ddev = dma_rx->device->dev;
+	/* Allocate descriptor & submit packet */
+	sg_nents = sg_nents_for_len(req->src, enc_len + req->assoclen);
+	dst_nents = sg_nents_for_len(req->dst, enc_len +
+				     crypto_aead_authsize(tfm));
+
+	memcpy(req_ctx.cmdl, sa_ctx->cmdl, sa_ctx->cmdl_size);
+	/* Update Command Label */
+	sa_update_cmdl(sa_k3_dev, enc_offset, enc_len,
+		       iv, auth_offset, auth_len,
+		       auth_iv, aad_len, aad,
+		       &sa_ctx->cmdl_upd_info, req_ctx.cmdl);
+
+	/*
+	 * Last 2 words in PSDATA will have the crypto alg type &
+	 * crypto request pointer
+	 */
+	req_type = CRYPTO_ALG_TYPE_AEAD;
+	if (enc)
+		req_type |= (SA_REQ_SUBTYPE_ENC << SA_REQ_SUBTYPE_SHIFT);
+	else
+		req_type |= (SA_REQ_SUBTYPE_DEC << SA_REQ_SUBTYPE_SHIFT);
+
+	psdata_offset = sa_ctx->cmdl_size / sizeof(u32);
+
+	/* map the packet */
+	req_ctx.src = req->src;
+	req_ctx.src_nents = dma_map_sg(ddev, req->src,
+				       sg_nents, DMA_TO_DEVICE);
+	dst_nents = dma_map_sg(ddev, req->dst,
+			       dst_nents, DMA_FROM_DEVICE);
+
+	if (unlikely(req_ctx.src_nents != sg_nents)) {
+		dev_warn_ratelimited(sa_k3_dev, "failed to map tx pkt\n");
+		return -EIO;
+	}
+
+	req_ctx.dev_data = pdata;
+	req_ctx.pkt = true;
+
+	dma_sync_sg_for_device(pdata->dev, req->src, req_ctx.src_nents,
+			       DMA_TO_DEVICE);
+
+	tx_in = dmaengine_prep_slave_sg(dma_rx, req->dst, dst_nents,
+					DMA_DEV_TO_MEM,
+					DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+	if (!tx_in) {
+		dev_err(pdata->dev, "IN prep_slave_sg() failed\n");
+		return -EINVAL;
+	}
+
+	rxd = kzalloc(sizeof(*rxd), GFP_KERNEL);
+	rxd->req = (void *)req;
+	rxd->enc = enc;
+	rxd->tx_in = tx_in;
+
+	/* IN */
+	tx_in->callback = sa_aead_dma_in_callback;
+	tx_in->callback_param = rxd;
+
+	tx_out = dmaengine_prep_slave_sg(pdata->dma_tx, req->src,
+					 req_ctx.src_nents, DMA_MEM_TO_DEV,
+					 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+	if (!tx_out) {
+		dev_err(pdata->dev, "OUT prep_slave_sg() failed\n");
+		return -EINVAL;
+	}
+
+	mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(tx_out, &pl, &ml);
+
+	sa_prepare_tx_desc(mdptr, (sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS *
+			   sizeof(u32))), req_ctx.cmdl,
+			   sizeof(sa_ctx->epib), sa_ctx->epib);
+
+	ml = sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS * sizeof(u32));
+	dmaengine_desc_set_metadata_len(tx_out, 52);
+
+	dmaengine_submit(tx_out);
+	dmaengine_submit(tx_in);
+
+	dma_async_issue_pending(dma_rx);
+	dma_async_issue_pending(pdata->dma_tx);
+	return -EINPROGRESS;
+}
+
+/* AEAD algorithm encrypt interface function */
+static int sa_aead_encrypt(struct aead_request *req)
+{
+	return sa_aead_run(req, req->iv, 1);
+}
+
+/* AEAD algorithm decrypt interface function */
+static int sa_aead_decrypt(struct aead_request *req)
+{
+	return sa_aead_run(req, req->iv, 0);
+}
+
+static void sa_sham_dma_in_callback(void *data)
+{
+	struct sa_rx_data *rxd = (struct sa_rx_data *)data;
+	struct ahash_request *req = (struct ahash_request *)rxd->req;
+	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+	unsigned int authsize = crypto_ahash_digestsize(tfm);
+	int i, sg_nents;
+	size_t ml, pl;
+	u32 *mdptr, *result;
+
+	mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml);
+	result = (u32 *)req->result;
+
+	for (i = 0; i < (authsize / 4); i++)
+		result[i] = htonl(mdptr[i + 4]);
+
+	sg_nents = sg_nents_for_len(req->src, req->nbytes);
+	dma_unmap_sg(rxd->ddev, req->src, sg_nents, DMA_FROM_DEVICE);
+
+	kfree(rxd);
+
+	ahash_request_complete(req, 0);
+}
+
+/*
+ * IC can not process zero message hash,
+ * so we put the fixed hash out when met zero message.
+ */
+
+static int zero_message_process(struct ahash_request *req)
+{
+	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+	int sa_digest_size = crypto_ahash_digestsize(tfm);
+
+	switch (sa_digest_size) {
+	case SHA1_DIGEST_SIZE:
+		memcpy(req->result, sha1_zero_message_hash, sa_digest_size);
+		break;
+	case SHA256_DIGEST_SIZE:
+		memcpy(req->result, sha256_zero_message_hash, sa_digest_size);
+		break;
+	case SHA512_DIGEST_SIZE:
+		memcpy(req->result, sha512_zero_message_hash, sa_digest_size);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int sa_sham_digest(struct ahash_request *req)
+{
+	struct sa_tfm_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
+	struct sa_ctx_info *sa_ctx = &ctx->enc;
+	struct dma_async_tx_descriptor *tx_in, *tx_out;
+	struct sa_crypto_data *pdata = dev_get_drvdata(sa_k3_dev);
+	struct sa_dma_req_ctx req_ctx;
+	struct sa_rx_data *rxd;
+	u8 enc_offset;
+	int sg_nents;
+	int psdata_offset;
+	u8 auth_offset = 0;
+	u8 *auth_iv = NULL;
+	u8 *aad = NULL;
+	u8 aad_len = 0;
+	u16 enc_len;
+	u16 auth_len = 0;
+	u32 req_type;
+	u32 *mdptr;
+	struct dma_chan *dma_rx;
+	gfp_t flags;
+	size_t pl, ml;
+	struct device *ddev;
+
+	flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+			GFP_KERNEL : GFP_ATOMIC;
+	enc_len = 0;
+	auth_len = req->nbytes;
+	enc_offset = 0;
+
+	if (!req->nbytes)
+		return zero_message_process(req);
+
+	if (auth_len >= 256)
+		dma_rx = pdata->dma_rx2;
+	else
+		dma_rx = pdata->dma_rx1;
+
+	ddev = dma_rx->device->dev;
+
+	/* Allocate descriptor & submit packet */
+	sg_nents = sg_nents_for_len(req->src, req->nbytes);
+
+	memcpy(req_ctx.cmdl, sa_ctx->cmdl, sa_ctx->cmdl_size);
+	/* Update Command Label */
+	sa_update_cmdl(sa_k3_dev, enc_offset, enc_len,
+		       NULL, auth_offset, auth_len,
+		       auth_iv, aad_len, aad,
+		       &sa_ctx->cmdl_upd_info, req_ctx.cmdl);
+
+	/*
+	 * Last 2 words in PSDATA will have the crypto alg type &
+	 * crypto request pointer
+	 */
+	req_type = CRYPTO_ALG_TYPE_AHASH;
+
+	psdata_offset = sa_ctx->cmdl_size / sizeof(u32);
+	req_ctx.cmdl[psdata_offset++] = req_type;
+
+	/* map the packet */
+	req_ctx.src = req->src;
+	req_ctx.src_nents = dma_map_sg(ddev, req->src, sg_nents, DMA_TO_DEVICE);
+
+	if (unlikely(req_ctx.src_nents != sg_nents)) {
+		dev_warn_ratelimited(sa_k3_dev, "failed to map tx pkt\n");
+		return -EIO;
+	}
+
+	req_ctx.dev_data = pdata;
+	req_ctx.pkt = true;
+
+	dma_sync_sg_for_device(pdata->dev, req->src, req_ctx.src_nents,
+			       DMA_TO_DEVICE);
+
+	tx_in = dmaengine_prep_slave_sg(dma_rx, req->src, req_ctx.src_nents,
+					DMA_DEV_TO_MEM,
+					DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+	if (!tx_in) {
+		dev_err(pdata->dev, "IN prep_slave_sg() failed\n");
+		return -EINVAL;
+	}
+
+	rxd = kzalloc(sizeof(*rxd), GFP_KERNEL);
+	rxd->req = (void *)req;
+	rxd->tx_in = tx_in;
+	tx_in->callback = sa_sham_dma_in_callback;
+	tx_in->callback_param = rxd;
+
+	tx_out = dmaengine_prep_slave_sg(pdata->dma_tx, req->src,
+					 req_ctx.src_nents, DMA_MEM_TO_DEV,
+					 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+	if (!tx_out) {
+		dev_err(pdata->dev, "OUT prep_slave_sg() failed\n");
+		return -EINVAL;
+	}
+
+	mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(tx_out, &pl, &ml);
+	sa_prepare_tx_desc(mdptr, (sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS *
+			   sizeof(u32))), req_ctx.cmdl,
+			   sizeof(sa_ctx->epib), sa_ctx->epib);
+
+	dmaengine_desc_set_metadata_len(tx_out, 28);
+
+	dmaengine_submit(tx_out);
+	dmaengine_submit(tx_in);
+
+	dma_async_issue_pending(dma_rx);
+	dma_async_issue_pending(pdata->dma_tx);
+
+	return -EINPROGRESS;
+}
+
+static int sa_sha_setup(struct sa_tfm_ctx *ctx, struct  algo_data *ad)
+{
+	int bs = crypto_shash_blocksize(ctx->shash);
+	int cmdl_len;
+	struct sa_cmdl_cfg cfg;
+
+	memset(ctx->authkey, 0, bs);
+	memset(&cfg, 0, sizeof(cfg));
+	cfg.enc1st = 0;
+	cfg.aalg = ad->aalg_id;
+	cfg.enc_eng_id = ad->enc_eng.eng_id;
+	cfg.auth_eng_id = ad->auth_eng.eng_id;
+	cfg.iv_size = 0;
+	cfg.akey = NULL;
+	cfg.akey_len = 0;
+
+	/* Setup Encryption Security Context & Command label template */
+	if (sa_init_sc(&ctx->enc, NULL, 0, NULL, 0, ad, 0,
+		       &ctx->enc.epib[1], true))
+		goto badkey;
+
+	cmdl_len = sa_format_cmdl_gen(&cfg,
+				      (u8 *)ctx->enc.cmdl,
+				      &ctx->enc.cmdl_upd_info);
+	if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
+		goto badkey;
+
+	ctx->enc.cmdl_size = cmdl_len;
+
+	return 0;
+badkey:
+	dev_err(sa_k3_dev, "%s: badkey\n", __func__);
+	return -EINVAL;
+}
+
+static int sa_sham_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base)
+{
+	struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct crypto_alg *alg = tfm->__crt_alg;
+	struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
+	int ret;
+
+	if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) ==
+	    CRYPTO_ALG_TYPE_AHASH) {
+		memset(ctx, 0, sizeof(*ctx));
+		ctx->dev_data = data;
+		ret = sa_init_ctx_info(&ctx->enc, data);
+		if (ret)
+			return ret;
+	}
+
+	if (alg_base) {
+		ctx->shash = crypto_alloc_shash(alg_base, 0,
+						CRYPTO_ALG_NEED_FALLBACK);
+		if (IS_ERR(ctx->shash)) {
+			pr_err("base driver %s couldn't be loaded\n", alg_base);
+			return PTR_ERR(ctx->shash);
+		}
+		/* for fallback */
+		ctx->fallback_tfm = crypto_alloc_ahash(alg_base, 0,
+						CRYPTO_ALG_NEED_FALLBACK);
+		if (IS_ERR(ctx->fallback_tfm)) {
+			dev_err(ctx->dev_data->dev,
+				"Could not load fallback driver\n");
+			return PTR_ERR(ctx->fallback_tfm);
+		}
+	}
+
+	dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
+		__func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
+		ctx->dec.sc_id, &ctx->dec.sc_phys);
+
+	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+				 sizeof(struct sa_dma_req_ctx) +
+				 crypto_ahash_reqsize(ctx->fallback_tfm));
+
+	return 0;
+}
+
+static int sa_sha1_digest(struct ahash_request *req)
+{
+	return sa_sham_digest(req);
+}
+
+static int sa_sha256_digest(struct ahash_request *req)
+{
+	return sa_sham_digest(req);
+}
+
+static int sa_sha512_digest(struct ahash_request *req)
+{
+	return sa_sham_digest(req);
+}
+
+static int sa_sham_init(struct ahash_request *req)
+{
+	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+	struct sa_dma_req_ctx *rctx = ahash_request_ctx(req);
+	struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
+
+	dev_dbg(sa_k3_dev, "init: digest size: %d\n",
+		crypto_ahash_digestsize(tfm));
+
+	ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
+	rctx->fallback_req.base.flags = req->base.flags &
+					CRYPTO_TFM_REQ_MAY_SLEEP;
+
+	return crypto_ahash_init(&rctx->fallback_req);
+}
+
+static int sa_sham_shash_digest(struct crypto_shash *tfm, u32 flags,
+				const u8 *data, unsigned int len, u8 *out)
+{
+	SHASH_DESC_ON_STACK(shash, tfm);
+
+	shash->tfm = tfm;
+	shash->flags = flags & CRYPTO_TFM_REQ_MAY_SLEEP;
+
+	return crypto_shash_digest(shash, data, len, out);
+}
+
+static int sa_sham_setkey(struct crypto_ahash *tfm, const u8 *key,
+			  unsigned int keylen, struct  algo_data *ad)
+{
+	struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
+	int bs = crypto_shash_blocksize(ctx->shash);
+	int ds = crypto_shash_digestsize(ctx->shash);
+	int cmdl_len;
+	struct sa_cmdl_cfg cfg;
+	int err;
+
+	if (keylen > bs) {
+		err = sa_sham_shash_digest(ctx->shash,
+					   crypto_shash_get_flags(ctx->shash),
+					   key, keylen, ctx->authkey);
+		if (err)
+			return err;
+		keylen = ds;
+	} else {
+		memcpy(ctx->authkey, key, keylen);
+	}
+
+	/* Set up the fallback algorithm with key as well */
+	crypto_ahash_setkey(ctx->fallback_tfm, key, keylen);
+
+	memset(ctx->authkey + keylen, 0, bs - keylen);
+	memset(&cfg, 0, sizeof(cfg));
+	cfg.enc1st = 0;
+	cfg.aalg = ad->aalg_id;
+	cfg.enc_eng_id = ad->enc_eng.eng_id;
+	cfg.auth_eng_id = ad->auth_eng.eng_id;
+	cfg.iv_size = 0;
+	cfg.akey = ctx->authkey;
+	cfg.akey_len = keylen;
+
+	/* Setup Encryption Security Context & Command label template */
+	if (sa_init_sc(&ctx->enc, NULL, 0, ctx->authkey, keylen, ad, 0,
+		       &ctx->enc.epib[1], true))
+		goto badkey;
+
+	cmdl_len = sa_format_cmdl_gen(&cfg,
+				      (u8 *)ctx->enc.cmdl,
+				      &ctx->enc.cmdl_upd_info);
+	if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
+		goto badkey;
+
+	ctx->enc.cmdl_size = cmdl_len;
+
+	kfree(ad);
+
+	return 0;
+badkey:
+	dev_err(sa_k3_dev, "%s: badkey\n", __func__);
+	return -EINVAL;
+}
+
+static int sa_sham_sha1_setkey(struct crypto_ahash *tfm, const u8 *key,
+			       unsigned int keylen)
+{
+	struct algo_data *ad = kzalloc(sizeof(*ad), GFP_KERNEL);
+
+	ad->enc_eng.eng_id = SA_ENG_ID_NONE;
+	ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
+	ad->auth_eng.eng_id = SA_ENG_ID_AM1;
+	ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ;
+	ad->mci_enc = NULL;
+	ad->mci_dec = NULL;
+	ad->inv_key = false;
+	ad->keyed_mac = true;
+	ad->ealg_id = SA_EALG_ID_NONE;
+	ad->aalg_id = SA_AALG_ID_HMAC_SHA1;
+	ad->hash_size = SHA1_DIGEST_SIZE;
+	ad->auth_ctrl = 0x2;
+	ad->prep_iopad = sa_hmac_sha1_get_pad;
+
+	return sa_sham_setkey(tfm, key, keylen, ad);
+}
+
+static int sa_sham_sha256_setkey(struct crypto_ahash *tfm, const u8 *key,
+				 unsigned int keylen)
+{
+	struct algo_data *ad = kzalloc(sizeof(*ad), GFP_KERNEL);
+
+	ad->enc_eng.eng_id = SA_ENG_ID_NONE;
+	ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
+	ad->auth_eng.eng_id = SA_ENG_ID_AM1;
+	ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ;
+	ad->mci_enc = NULL;
+	ad->mci_dec = NULL;
+	ad->inv_key = false;
+	ad->keyed_mac = true;
+	ad->ealg_id = SA_EALG_ID_NONE;
+	ad->aalg_id = SA_AALG_ID_HMAC_SHA2_256;
+	ad->hash_size = SHA256_DIGEST_SIZE;
+	ad->auth_ctrl = 0x4;
+	ad->prep_iopad = sa_hmac_sha256_get_pad;
+
+	return sa_sham_setkey(tfm, key, keylen, ad);
+}
+
+static int sa_sham_cra_sha1_init(struct crypto_tfm *tfm)
+{
+	struct algo_data *ad = kzalloc(sizeof(*ad), GFP_KERNEL);
+	struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	sa_sham_cra_init_alg(tfm, "sha1");
+
+	ad->enc_eng.eng_id = SA_ENG_ID_NONE;
+	ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
+	ad->auth_eng.eng_id = SA_ENG_ID_AM1;
+	ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ;
+	ad->mci_enc = NULL;
+	ad->mci_dec = NULL;
+	ad->inv_key = false;
+	ad->keyed_mac = false;
+	ad->ealg_id = SA_EALG_ID_NONE;
+	ad->aalg_id = SA_AALG_ID_SHA1;
+	ad->hash_size = SHA1_DIGEST_SIZE;
+	ad->auth_ctrl = 0x2;
+
+	sa_sha_setup(ctx, ad);
+
+	kfree(ad);
+
+	return 0;
+}
+
+static int sa_sham_cra_sha256_init(struct crypto_tfm *tfm)
+{
+	struct algo_data *ad = kzalloc(sizeof(*ad), GFP_KERNEL);
+	struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	sa_sham_cra_init_alg(tfm, "sha256");
+
+	ad->enc_eng.eng_id = SA_ENG_ID_NONE;
+	ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
+	ad->auth_eng.eng_id = SA_ENG_ID_AM1;
+	ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ;
+	ad->mci_enc = NULL;
+	ad->mci_dec = NULL;
+	ad->inv_key = false;
+	ad->keyed_mac = false;
+	ad->ealg_id = SA_EALG_ID_NONE;
+	ad->aalg_id = SA_AALG_ID_SHA2_256;
+	ad->hash_size = SHA256_DIGEST_SIZE;
+	ad->auth_ctrl = 0x4;
+
+	sa_sha_setup(ctx, ad);
+
+	kfree(ad);
+
+	return 0;
+}
+
+static int sa_sham_cra_sha512_init(struct crypto_tfm *tfm)
+{
+	struct algo_data *ad = kzalloc(sizeof(*ad), GFP_KERNEL);
+	struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	sa_sham_cra_init_alg(tfm, "sha512");
+
+	ad->enc_eng.eng_id = SA_ENG_ID_NONE;
+	ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
+	ad->auth_eng.eng_id = SA_ENG_ID_AM1;
+	ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ;
+	ad->mci_enc = NULL;
+	ad->mci_dec = NULL;
+	ad->inv_key = false;
+	ad->keyed_mac = false;
+	ad->ealg_id = SA_EALG_ID_NONE;
+	ad->aalg_id = SA_AALG_ID_SHA2_512;
+	ad->hash_size = SHA512_DIGEST_SIZE;
+	ad->auth_ctrl = 0x6;
+
+	sa_sha_setup(ctx, ad);
+
+	kfree(ad);
+
+	return 0;
+}
+
+static void sa_sham_cra_exit(struct crypto_tfm *tfm)
+{
+	struct crypto_alg *alg = tfm->__crt_alg;
+	struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
+
+	dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
+		__func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
+		ctx->dec.sc_id, &ctx->dec.sc_phys);
+
+	if ((alg->cra_flags & CRYPTO_ALG_TYPE_AHASH)
+	    == CRYPTO_ALG_TYPE_AHASH) {
+		sa_free_ctx_info(&ctx->enc, data);
+	}
+}
+
+static int sa_sham_update(struct ahash_request *req)
+{
+	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+	struct sa_dma_req_ctx *rctx = ahash_request_ctx(req);
+	struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
+
+	ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
+	rctx->fallback_req.base.flags = req->base.flags &
+					CRYPTO_TFM_REQ_MAY_SLEEP;
+	rctx->fallback_req.nbytes = req->nbytes;
+	rctx->fallback_req.src = req->src;
+
+	return crypto_ahash_update(&rctx->fallback_req);
+}
+
+static int sa_sham_final(struct ahash_request *req)
+{
+	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+	struct sa_dma_req_ctx *rctx = ahash_request_ctx(req);
+	struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
+
+	ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
+	rctx->fallback_req.base.flags = req->base.flags &
+					CRYPTO_TFM_REQ_MAY_SLEEP;
+	rctx->fallback_req.result = req->result;
+
+	return crypto_ahash_final(&rctx->fallback_req);
+}
+
+static int sa_sham_finup(struct ahash_request *req)
+{
+	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+	struct sa_dma_req_ctx *rctx = ahash_request_ctx(req);
+	struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
+
+	ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
+	rctx->fallback_req.base.flags = req->base.flags &
+					CRYPTO_TFM_REQ_MAY_SLEEP;
+
+	rctx->fallback_req.nbytes = req->nbytes;
+	rctx->fallback_req.src = req->src;
+	rctx->fallback_req.result = req->result;
+
+	return crypto_ahash_finup(&rctx->fallback_req);
+}
+
+static int sa_sham_import(struct ahash_request *req, const void *in)
+{
+	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+	struct sa_dma_req_ctx *rctx = ahash_request_ctx(req);
+	struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
+
+	ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
+	rctx->fallback_req.base.flags = req->base.flags &
+					CRYPTO_TFM_REQ_MAY_SLEEP;
+
+	return crypto_ahash_import(&rctx->fallback_req, in);
+}
+
+static int sa_sham_export(struct ahash_request *req, void *out)
+{
+	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+	struct sa_dma_req_ctx *rctx = ahash_request_ctx(req);
+	struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
+
+	ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
+	rctx->fallback_req.base.flags = req->base.flags &
+					CRYPTO_TFM_REQ_MAY_SLEEP;
+
+	return crypto_ahash_export(&rctx->fallback_req, out);
+}
+
+static struct sa_alg_tmpl sa_algs[] = {
+	{.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+	 .alg.crypto = {
+			.cra_name = "cbc(aes)",
+			.cra_driver_name = "cbc-aes-sa2ul",
+			.cra_priority = 30000,
+			.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
+				     CRYPTO_ALG_KERN_DRIVER_ONLY |
+				     CRYPTO_ALG_ASYNC |
+				     CRYPTO_ALG_NEED_FALLBACK,
+			.cra_blocksize = AES_BLOCK_SIZE,
+			.cra_ctxsize = sizeof(struct sa_tfm_ctx),
+			.cra_alignmask = 0,
+			.cra_type = &crypto_ablkcipher_type,
+			.cra_module = THIS_MODULE,
+			.cra_init = sa_aes_cra_init,
+			.cra_exit = sa_aes_cra_exit,
+			.cra_u.ablkcipher = {
+					     .min_keysize = AES_MIN_KEY_SIZE,
+					     .max_keysize = AES_MAX_KEY_SIZE,
+					     .ivsize = AES_BLOCK_SIZE,
+					     .setkey = sa_aes_cbc_setkey,
+					     .encrypt = sa_aes_cbc_encrypt,
+					     .decrypt = sa_aes_cbc_decrypt,
+					}
+			}
+	},
+	{	.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+		.alg.crypto = {
+			.cra_name = "ecb(aes)",
+			.cra_driver_name = "ecb-aes-sa2ul",
+			.cra_priority = 30000,
+			.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
+					CRYPTO_ALG_KERN_DRIVER_ONLY |
+				CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
+			.cra_blocksize = AES_BLOCK_SIZE,
+			.cra_ctxsize = sizeof(struct sa_tfm_ctx),
+			.cra_alignmask = 0,
+			.cra_type = &crypto_ablkcipher_type,
+			.cra_module = THIS_MODULE,
+			.cra_init = sa_aes_cra_init,
+			.cra_exit = sa_aes_cra_exit,
+			.cra_u.ablkcipher = {
+				.min_keysize    = AES_MIN_KEY_SIZE,
+				.max_keysize    = AES_MAX_KEY_SIZE,
+				.setkey		= sa_aes_ecb_setkey,
+				.encrypt	= sa_aes_cbc_encrypt,
+				.decrypt	= sa_aes_cbc_decrypt,
+			}
+		}
+	},
+	{	.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+		.alg.crypto = {
+			.cra_name = "cbc(des3_ede)",
+			.cra_driver_name = "cbc-des3-sa2ul",
+			.cra_priority = 30000,
+			.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
+					CRYPTO_ALG_KERN_DRIVER_ONLY |
+				CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
+			.cra_blocksize = DES_BLOCK_SIZE,
+			.cra_ctxsize = sizeof(struct sa_tfm_ctx),
+			.cra_alignmask = 0,
+			.cra_type = &crypto_ablkcipher_type,
+			.cra_module = THIS_MODULE,
+			.cra_init = sa_aes_cra_init,
+			.cra_exit = sa_aes_cra_exit,
+			.cra_u.ablkcipher = {
+				.min_keysize    = 3 * DES_KEY_SIZE,
+				.max_keysize    = 3 * DES_KEY_SIZE,
+				.ivsize		= DES_BLOCK_SIZE,
+				.setkey		= sa_3des_cbc_setkey,
+				.encrypt	= sa_aes_cbc_encrypt,
+				.decrypt	= sa_aes_cbc_decrypt,
+			}
+		}
+	},
+	{	.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+		.alg.crypto = {
+			.cra_name = "ecb(des3_ede)",
+			.cra_driver_name = "ecb-des3-sa2ul",
+			.cra_priority = 30000,
+			.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
+					CRYPTO_ALG_KERN_DRIVER_ONLY |
+				CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
+			.cra_blocksize = DES_BLOCK_SIZE,
+			.cra_ctxsize = sizeof(struct sa_tfm_ctx),
+			.cra_alignmask = 0,
+			.cra_type = &crypto_ablkcipher_type,
+			.cra_module = THIS_MODULE,
+			.cra_init = sa_aes_cra_init,
+			.cra_exit = sa_aes_cra_exit,
+			.cra_u.ablkcipher = {
+				.min_keysize    = 3 * DES_KEY_SIZE,
+				.max_keysize    = 3 * DES_KEY_SIZE,
+				.ivsize		= DES_BLOCK_SIZE,
+				.setkey		= sa_3des_ecb_setkey,
+				.encrypt	= sa_aes_cbc_encrypt,
+				.decrypt	= sa_aes_cbc_decrypt,
+			}
+		}
+	},
+	/* AEAD algorithms */
+	{.type	= CRYPTO_ALG_TYPE_AEAD,
+		.alg.aead = {
+				.base = {
+				.cra_name = "authenc(hmac(sha1),cbc(aes))",
+				.cra_driver_name =
+					"authenc(hmac(sha1),cbc(aes))-keystone-sa",
+				.cra_blocksize = AES_BLOCK_SIZE,
+				.cra_flags = CRYPTO_ALG_TYPE_AEAD |
+					CRYPTO_ALG_KERN_DRIVER_ONLY |
+					CRYPTO_ALG_ASYNC,
+				.cra_ctxsize = sizeof(struct sa_tfm_ctx),
+				.cra_module = THIS_MODULE,
+				.cra_alignmask = 0,
+				.cra_priority = 3000,
+			},
+			.ivsize = AES_BLOCK_SIZE,
+			.maxauthsize = SHA1_DIGEST_SIZE,
+
+			.init = sa_cra_init_aead,
+			.exit = sa_exit_tfm_aead,
+			.setkey = sa_aead_cbc_sha1_setkey,
+			.encrypt = sa_aead_encrypt,
+			.decrypt = sa_aead_decrypt,
+		}
+	},
+	{.type	= CRYPTO_ALG_TYPE_AEAD,
+		.alg.aead = {
+				.base = {
+				.cra_name = "authenc(hmac(sha256),cbc(aes))",
+				.cra_driver_name =
+					"authenc(hmac(sha256),cbc(aes))-keystone-sa",
+				.cra_blocksize = AES_BLOCK_SIZE,
+				.cra_flags = CRYPTO_ALG_TYPE_AEAD |
+					CRYPTO_ALG_KERN_DRIVER_ONLY |
+					CRYPTO_ALG_ASYNC,
+				.cra_ctxsize = sizeof(struct sa_tfm_ctx),
+				.cra_module = THIS_MODULE,
+				.cra_alignmask = 0,
+				.cra_priority = 3000,
+			},
+			.ivsize = AES_BLOCK_SIZE,
+			.maxauthsize = SHA256_DIGEST_SIZE,
+
+			.init = sa_cra_init_aead,
+			.exit = sa_exit_tfm_aead,
+			.setkey = sa_aead_cbc_sha256_setkey,
+			.encrypt = sa_aead_encrypt,
+			.decrypt = sa_aead_decrypt,
+		}
+	},
+};
+
+static struct ahash_alg algs_sha[] = {
+{
+	.init		= sa_sham_init,
+	.update		= sa_sham_update,
+	.final		= sa_sham_final,
+	.finup		= sa_sham_finup,
+	.digest		= sa_sha1_digest,
+	.export		= sa_sham_export,
+	.import		= sa_sham_import,
+	.halg.digestsize	= SHA1_DIGEST_SIZE,
+	.halg.statesize		= 128,
+	.halg.base	= {
+		.cra_name		= "sha1",
+		.cra_driver_name	= "sa-sha1",
+		.cra_priority		= 400,
+		.cra_flags		= CRYPTO_ALG_ASYNC |
+						CRYPTO_ALG_NEED_FALLBACK,
+		.cra_blocksize		= SHA1_BLOCK_SIZE,
+		.cra_ctxsize		= sizeof(struct sa_tfm_ctx),
+		.cra_alignmask		= SA_ALIGN_MASK,
+		.cra_module		= THIS_MODULE,
+		.cra_init		= sa_sham_cra_sha1_init,
+		.cra_exit		= sa_sham_cra_exit,
+	}
+},
+{
+	.init		= sa_sham_init,
+	.update		= sa_sham_update,
+	.final		= sa_sham_final,
+	.finup		= sa_sham_finup,
+	.digest		= sa_sha256_digest,
+	.export		= sa_sham_export,
+	.import		= sa_sham_import,
+	.halg.digestsize	= SHA256_DIGEST_SIZE,
+	.halg.statesize		= 128,
+	.halg.base	= {
+		.cra_name		= "sha256",
+		.cra_driver_name	= "sa-sha256",
+		.cra_priority		= 400,
+		.cra_flags		= CRYPTO_ALG_ASYNC |
+						CRYPTO_ALG_NEED_FALLBACK,
+		.cra_blocksize		= SHA256_BLOCK_SIZE,
+		.cra_ctxsize		= sizeof(struct sa_tfm_ctx),
+		.cra_alignmask		= SA_ALIGN_MASK,
+		.cra_module		= THIS_MODULE,
+		.cra_init		= sa_sham_cra_sha256_init,
+		.cra_exit		= sa_sham_cra_exit,
+	}
+},
+{
+	.init		= sa_sham_init,
+	.update		= sa_sham_update,
+	.final		= sa_sham_final,
+	.finup		= sa_sham_finup,
+	.digest		= sa_sha512_digest,
+	.export		= sa_sham_export,
+	.import		= sa_sham_import,
+	.halg.digestsize	= SHA512_DIGEST_SIZE,
+	.halg.statesize		= 256,
+	.halg.base	= {
+		.cra_name		= "sha512",
+		.cra_driver_name	= "sa-sha512",
+		.cra_priority		= 400,
+		.cra_flags		= CRYPTO_ALG_ASYNC |
+						CRYPTO_ALG_NEED_FALLBACK,
+		.cra_blocksize		= SHA512_BLOCK_SIZE,
+		.cra_ctxsize		= sizeof(struct sa_tfm_ctx),
+		.cra_alignmask		= SA_ALIGN_MASK,
+		.cra_module		= THIS_MODULE,
+		.cra_init		= sa_sham_cra_sha512_init,
+		.cra_exit		= sa_sham_cra_exit,
+	}
+},
+{
+	.init		= sa_sham_init,
+	.update		= sa_sham_update,
+	.final		= sa_sham_final,
+	.finup		= sa_sham_finup,
+	.digest		= sa_sham_digest,
+	.setkey		= sa_sham_sha1_setkey,
+	.export		= sa_sham_export,
+	.import		= sa_sham_import,
+	.halg.digestsize	= SHA1_DIGEST_SIZE,
+	.halg.statesize		= 128,
+	.halg.base	= {
+		.cra_name		= "hmac(sha1)",
+		.cra_driver_name	= "sa-hmac-sha1",
+		.cra_priority		= 400,
+		.cra_flags		= CRYPTO_ALG_TYPE_AHASH |
+						CRYPTO_ALG_ASYNC |
+						CRYPTO_ALG_KERN_DRIVER_ONLY |
+						CRYPTO_ALG_NEED_FALLBACK,
+		.cra_blocksize		= SHA1_BLOCK_SIZE,
+		.cra_ctxsize		= sizeof(struct sa_tfm_ctx),
+		.cra_alignmask		= SA_ALIGN_MASK,
+		.cra_module		= THIS_MODULE,
+		.cra_init		= sa_sham_cra_sha1_init,
+		.cra_exit		= sa_sham_cra_exit,
+	}
+},
+{
+	.init		= sa_sham_init,
+	.update		= sa_sham_update,
+	.final		= sa_sham_final,
+	.finup		= sa_sham_finup,
+	.digest		= sa_sham_digest,
+	.setkey		= sa_sham_sha256_setkey,
+	.export		= sa_sham_export,
+	.import		= sa_sham_import,
+	.halg.digestsize	= SHA256_DIGEST_SIZE,
+	.halg.statesize		= 128,
+	.halg.base	= {
+		.cra_name		= "hmac(sha256)",
+		.cra_driver_name	= "sa-hmac-sha256",
+		.cra_priority		= 400,
+		.cra_flags		= CRYPTO_ALG_TYPE_AHASH |
+						CRYPTO_ALG_ASYNC |
+						CRYPTO_ALG_KERN_DRIVER_ONLY |
+						CRYPTO_ALG_NEED_FALLBACK,
+		.cra_blocksize		= SHA256_BLOCK_SIZE,
+		.cra_ctxsize		= sizeof(struct sa_tfm_ctx),
+		.cra_alignmask		= SA_ALIGN_MASK,
+		.cra_module		= THIS_MODULE,
+		.cra_init		= sa_sham_cra_sha256_init,
+		.cra_exit		= sa_sham_cra_exit,
+	}
+},
+};
+
+/* Register the algorithms in crypto framework */
+void sa_register_algos(const struct device *dev)
+{
+	char *alg_name;
+	u32 type;
+	int i, err, num_algs = ARRAY_SIZE(sa_algs);
+
+	for (i = 0; i < num_algs; i++) {
+		type = sa_algs[i].type;
+		if (type == CRYPTO_ALG_TYPE_AEAD) {
+			alg_name = sa_algs[i].alg.aead.base.cra_name;
+			err = crypto_register_aead(&sa_algs[i].alg.aead);
+		} else if (type == CRYPTO_ALG_TYPE_ABLKCIPHER) {
+			alg_name = sa_algs[i].alg.crypto.cra_name;
+			err = crypto_register_alg(&sa_algs[i].alg.crypto);
+		} else {
+			dev_err(dev,
+				"un-supported crypto algorithm (%d)",
+				sa_algs[i].type);
+			continue;
+		}
+
+		if (err)
+			dev_err(dev, "Failed to register '%s'\n", alg_name);
+		else
+			sa_algs[i].registered = 1;
+	}
+
+	num_algs = ARRAY_SIZE(algs_sha);
+	for (i = 0; i < num_algs; i++) {
+		alg_name =  algs_sha[i].halg.base.cra_name;
+		err = crypto_register_ahash(&algs_sha[i]);
+		if (err)
+			dev_err(dev, "Failed to register '%s'\n",
+				alg_name);
+	}
+}
+
+/* Unregister the algorithms in crypto framework */
+void sa_unregister_algos(const struct device *dev)
+{
+	char *alg_name;
+	u32 type;
+	int i, err = 0, num_algs = ARRAY_SIZE(sa_algs);
+
+	for (i = 0; i < num_algs; i++) {
+		type = sa_algs[i].type;
+		if (type == CRYPTO_ALG_TYPE_AEAD) {
+			alg_name = sa_algs[i].alg.aead.base.cra_name;
+			crypto_unregister_aead(&sa_algs[i].alg.aead);
+		} else {
+			alg_name = sa_algs[i].alg.crypto.cra_name;
+			err = crypto_unregister_alg(&sa_algs[i].alg.crypto);
+		}
+
+		sa_algs[i].registered = 0;
+	}
+
+	num_algs = ARRAY_SIZE(algs_sha);
+	for (i = 0; i < num_algs; i++) {
+		alg_name =  algs_sha[i].halg.base.cra_name;
+		err = crypto_unregister_ahash(&algs_sha[i]);
+		if (err)
+			dev_err(dev, "Failed to register '%s'\n",
+				alg_name);
+	}
+}
+
+static int sa_init_mem(struct sa_crypto_data *dev_data)
+{
+	struct device *dev = &dev_data->pdev->dev;
+	/* Setup dma pool for security context buffers */
+	dev_data->sc_pool = dma_pool_create("keystone-sc", dev,
+					    SA_CTX_MAX_SZ, 64, 0);
+	if (!dev_data->sc_pool) {
+		dev_err(dev, "Failed to create dma pool");
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static int sa_dma_init(struct sa_crypto_data *dd)
+{
+	int ret;
+	struct dma_slave_config cfg;
+
+	dd->dma_rx1 = NULL;
+	dd->dma_tx = NULL;
+	dd->dma_rx2 = NULL;
+
+	ret = dma_coerce_mask_and_coherent(dd->dev, DMA_BIT_MASK(48));
+	if (ret)
+		return ret;
+
+	dd->dma_rx1 = dma_request_chan(dd->dev, "rx1");
+	if (IS_ERR(dd->dma_rx1)) {
+		if (PTR_ERR(dd->dma_rx1) != -EPROBE_DEFER)
+			dev_err(dd->dev, "Unable to request rx1 DMA channel\n");
+		return PTR_ERR(dd->dma_rx1);
+	}
+
+	dd->dma_rx2 = dma_request_chan(dd->dev, "rx2");
+	if (IS_ERR(dd->dma_rx2)) {
+		dma_release_channel(dd->dma_rx1);
+		if (PTR_ERR(dd->dma_rx1) != -EPROBE_DEFER)
+			dev_err(dd->dev, "Unable to request rx2 DMA channel\n");
+		return PTR_ERR(dd->dma_rx2);
+	}
+
+	dd->dma_tx = dma_request_chan(dd->dev, "tx");
+	if (IS_ERR(dd->dma_tx)) {
+		if (PTR_ERR(dd->dma_rx1) != -EPROBE_DEFER)
+			dev_err(dd->dev, "Unable to request tx DMA channel\n");
+		ret = PTR_ERR(dd->dma_tx);
+		goto err_dma_tx;
+	}
+
+	memzero_explicit(&cfg, sizeof(cfg));
+
+	cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+	cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+	cfg.src_maxburst = 4;
+	cfg.dst_maxburst = 4;
+
+	ret = dmaengine_slave_config(dd->dma_rx1, &cfg);
+	if (ret) {
+		dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",
+			ret);
+		return ret;
+	}
+
+	ret = dmaengine_slave_config(dd->dma_rx2, &cfg);
+	if (ret) {
+		dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",
+			ret);
+		return ret;
+	}
+
+	ret = dmaengine_slave_config(dd->dma_tx, &cfg);
+	if (ret) {
+		dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n",
+			ret);
+		return ret;
+	}
+
+	return 0;
+
+err_dma_tx:
+	dma_release_channel(dd->dma_rx1);
+	dma_release_channel(dd->dma_rx2);
+
+	return ret;
+}
+
+int sa2ul_trng_enable(struct device *dev)
+{
+	struct sa_crypto_data *dev_data;
+	u32 val;
+
+	if (!dev)
+		return -ENODEV;
+
+	dev_data = dev_get_drvdata(dev);
+
+	val = readl_relaxed(dev_data->base + SA_ENGINE_ENABLE_CONTROL);
+	writel_relaxed(val | SA_EEC_TRNG_EN, dev_data->base +
+		       SA_ENGINE_ENABLE_CONTROL);
+
+	return 0;
+}
+EXPORT_SYMBOL(sa2ul_trng_enable);
+
+static int sa_ul_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *node = dev->of_node;
+	struct resource *res;
+	static void __iomem *saul_base;
+	struct sa_crypto_data *dev_data;
+	u32 val;
+	int ret;
+
+	dev_data = devm_kzalloc(dev, sizeof(*dev_data), GFP_KERNEL);
+	if (!dev_data)
+		return -ENOMEM;
+
+	sa_k3_dev = dev;
+	dev_data->dev = dev;
+	dev_data->pdev = pdev;
+	platform_set_drvdata(pdev, dev_data);
+	dev_set_drvdata(sa_k3_dev, dev_data);
+
+	sa_init_mem(dev_data);
+	ret = sa_dma_init(dev_data);
+	if (ret)
+		return ret;
+
+	spin_lock_init(&dev_data->scid_lock);
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	saul_base = devm_ioremap_resource(dev, res);
+
+	dev_data->base = saul_base;
+	val = SA_EEC_ENCSS_EN | SA_EEC_AUTHSS_EN | SA_EEC_CTXCACH_EN |
+	    SA_EEC_CPPI_PORT_IN_EN | SA_EEC_CPPI_PORT_OUT_EN;
+
+	writel_relaxed(val, saul_base + SA_ENGINE_ENABLE_CONTROL);
+
+	sa_register_algos(dev);
+
+	ret = of_platform_populate(node, NULL, NULL, &pdev->dev);
+	if (ret)
+		dev_err(dev, "of_platform_populate failed\n");
+	return 0;
+}
+
+static int sa_ul_remove(struct platform_device *pdev)
+{
+	struct sa_crypto_data *dev_data = platform_get_drvdata(pdev);
+
+	sa_unregister_algos(&pdev->dev);
+
+	dma_release_channel(dev_data->dma_rx2);
+	dma_release_channel(dev_data->dma_rx1);
+	dma_release_channel(dev_data->dma_tx);
+
+	dma_pool_destroy(dev_data->sc_pool);
+
+	platform_set_drvdata(pdev, NULL);
+
+	return 0;
+}
+
+static const struct of_device_id of_match[] = {
+	{.compatible = "ti,sa2ul-crypto",},
+	{},
+};
+MODULE_DEVICE_TABLE(of, of_match);
+
+static struct platform_driver sa_ul_driver = {
+	.probe = sa_ul_probe,
+	.remove = sa_ul_remove,
+	.driver = {
+		   .name = "saul-crypto",
+		   .of_match_table = of_match,
+		   },
+};
+module_platform_driver(sa_ul_driver);
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/crypto/sa2ul.h b/drivers/crypto/sa2ul.h
--- a/drivers/crypto/sa2ul.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/crypto/sa2ul.h	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,392 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * AM6 SA2UL crypto accelerator driver
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
+ *
+ * Authors:	Keerthy
+ *              Vitaly Andrianov
+ */
+
+#ifndef _K3_SA2UL_
+#define _K3_SA2UL_
+
+#include <linux/interrupt.h>
+#include <linux/skbuff.h>
+#include <linux/hw_random.h>
+#include <crypto/aes.h>
+
+#define SA_ENGINE_ENABLE_CONTROL	0x1000
+
+struct sa_tfm_ctx;
+/*
+ * SA_ENGINE_ENABLE_CONTROL register bits
+ */
+#define SA_EEC_ENCSS_EN			0x00000001
+#define SA_EEC_AUTHSS_EN		0x00000002
+#define SA_EEC_TRNG_EN			0x00000008
+#define SA_EEC_PKA_EN			0x00000010
+#define SA_EEC_CTXCACH_EN		0x00000080
+#define SA_EEC_CPPI_PORT_IN_EN		0x00000200
+#define SA_EEC_CPPI_PORT_OUT_EN		0x00000800
+
+/*
+ * Encoding used to identify the typo of crypto operation
+ * performed on the packet when the packet is returned
+ * by SA
+ */
+#define SA_REQ_SUBTYPE_ENC	0x0001
+#define SA_REQ_SUBTYPE_DEC	0x0002
+#define SA_REQ_SUBTYPE_SHIFT	16
+#define SA_REQ_SUBTYPE_MASK	0xffff
+
+/* Number of 32 bit words in EPIB  */
+#define SA_DMA_NUM_EPIB_WORDS   4
+
+/* Number of 32 bit words in PS data  */
+#define SA_DMA_NUM_PS_WORDS     16
+#define NKEY_SZ			3
+#define MCI_SZ			27
+
+/*
+ * Maximum number of simultaeneous security contexts
+ * supported by the driver
+ */
+#define SA_MAX_NUM_CTX	512
+
+/*
+ * Assumption: CTX size is multiple of 32
+ */
+#define SA_CTX_SIZE_TO_DMA_SIZE(ctx_sz) \
+		((ctx_sz) ? ((ctx_sz) / 32 - 1) : 0)
+
+#define SA_CTX_ENC_KEY_OFFSET   32
+#define SA_CTX_ENC_AUX1_OFFSET  64
+#define SA_CTX_ENC_AUX2_OFFSET  96
+#define SA_CTX_ENC_AUX3_OFFSET  112
+#define SA_CTX_ENC_AUX4_OFFSET  128
+
+/* Next Engine Select code in CP_ACE */
+#define SA_ENG_ID_EM1   2       /* Enc/Dec engine with AES/DEC core */
+#define SA_ENG_ID_EM2   3       /* Encryption/Decryption enginefor pass 2 */
+#define SA_ENG_ID_AM1   4       /* Auth. engine with SHA1/MD5/SHA2 core */
+#define SA_ENG_ID_AM2   5       /*  Authentication engine for pass 2 */
+#define SA_ENG_ID_OUTPORT2 20   /*  Egress module 2  */
+#define SA_ENG_ID_NONE  0xff
+
+/*
+ * Command Label Definitions
+ */
+#define SA_CMDL_OFFSET_NESC           0      /* Next Engine Select Code */
+#define SA_CMDL_OFFSET_LABEL_LEN      1      /* Engine Command Label Length */
+/* 16-bit Length of Data to be processed */
+#define SA_CMDL_OFFSET_DATA_LEN       2
+#define SA_CMDL_OFFSET_DATA_OFFSET    4      /* Stat Data Offset */
+#define SA_CMDL_OFFSET_OPTION_CTRL1   5      /* Option Control Byte 1 */
+#define SA_CMDL_OFFSET_OPTION_CTRL2   6      /* Option Control Byte 2 */
+#define SA_CMDL_OFFSET_OPTION_CTRL3   7      /* Option Control Byte 3 */
+#define SA_CMDL_OFFSET_OPTION_BYTE    8
+
+#define SA_CMDL_HEADER_SIZE_BYTES	8
+
+#define SA_CMDL_OPTION_BYTES_MAX_SIZE     72
+#define SA_CMDL_MAX_SIZE_BYTES (SA_CMDL_HEADER_SIZE_BYTES + \
+				SA_CMDL_OPTION_BYTES_MAX_SIZE)
+
+/* SWINFO word-0 flags */
+#define SA_SW_INFO_FLAG_EVICT   0x0001
+#define SA_SW_INFO_FLAG_TEAR    0x0002
+#define SA_SW_INFO_FLAG_NOPD    0x0004
+
+/*
+ * This type represents the various packet types to be processed
+ * by the PHP engine in SA.
+ * It is used to identify the corresponding PHP processing function.
+ */
+#define SA_CTX_PE_PKT_TYPE_3GPP_AIR    0    /* 3GPP Air Cipher */
+#define SA_CTX_PE_PKT_TYPE_SRTP        1    /* SRTP */
+#define SA_CTX_PE_PKT_TYPE_IPSEC_AH    2    /* IPSec Authentication Header */
+/* IPSec Encapsulating Security Payload */
+#define SA_CTX_PE_PKT_TYPE_IPSEC_ESP   3
+/* Indicates that it is in data mode, It may not be used by PHP */
+#define SA_CTX_PE_PKT_TYPE_NONE        4
+#define SA_CTX_ENC_TYPE1_SZ     64      /* Encryption SC with Key only */
+#define SA_CTX_ENC_TYPE2_SZ     96      /* Encryption SC with Key and Aux1 */
+
+#define SA_CTX_AUTH_TYPE1_SZ    64      /* Auth SC with Key only */
+#define SA_CTX_AUTH_TYPE2_SZ    96      /* Auth SC with Key and Aux1 */
+/* Size of security context for PHP engine */
+#define SA_CTX_PHP_PE_CTX_SZ    64
+
+#define SA_CTX_MAX_SZ (64 + SA_CTX_ENC_TYPE2_SZ + SA_CTX_AUTH_TYPE2_SZ)
+
+/*
+ * Encoding of F/E control in SCCTL
+ *  Bit 0-1: Fetch PHP Bytes
+ *  Bit 2-3: Fetch Encryption/Air Ciphering Bytes
+ *  Bit 4-5: Fetch Authentication Bytes or Encr pass 2
+ *  Bit 6-7: Evict PHP Bytes
+ *
+ *  where   00 = 0 bytes
+ *          01 = 64 bytes
+ *          10 = 96 bytes
+ *          11 = 128 bytes
+ */
+#define SA_CTX_DMA_SIZE_0       0
+#define SA_CTX_DMA_SIZE_64      1
+#define SA_CTX_DMA_SIZE_96      2
+#define SA_CTX_DMA_SIZE_128     3
+
+/*
+ * Byte offset of the owner word in SCCTL
+ * in the security context
+ */
+#define SA_CTX_SCCTL_OWNER_OFFSET 0
+
+#define SA_CTX_ENC_KEY_OFFSET   32
+#define SA_CTX_ENC_AUX1_OFFSET  64
+#define SA_CTX_ENC_AUX2_OFFSET  96
+#define SA_CTX_ENC_AUX3_OFFSET  112
+#define SA_CTX_ENC_AUX4_OFFSET  128
+
+#define SA_SCCTL_FE_AUTH_ENC	0x65
+#define SA_SCCTL_FE_ENC		0x8D
+
+#define SA_ALIGN_MASK		(sizeof(u32) - 1)
+#define SA_ALIGNED		__aligned(32)
+
+/**
+ * struct sa_crypto_data - Crypto driver instance data
+ * @base: Base address of the register space
+ * @pdev: Platform device pointer
+ * @sc_pool: security context pool
+ * @dev: Device pointer
+ * @scid_lock: secure context ID lock
+ * @sc_id_start: starting index for SC ID
+ * @sc_id_end: Ending index for SC ID
+ * @sc_id: Security Context ID
+ * @ctx_bm: Bitmap to keep track of Security context ID's
+ * @ctx: SA tfm context pointer
+ * @dma_rx1: Pointer to DMA rx channel for sizes < 256 Bytes
+ * @dma_rx2: Pointer to DMA rx channel for sizes > 256 Bytes
+ * @dma_tx: Pointer to DMA TX channel
+ */
+struct sa_crypto_data {
+	void __iomem *base;
+	struct platform_device	*pdev;
+	struct dma_pool		*sc_pool;
+	struct device *dev;
+	spinlock_t	scid_lock; /* lock for SC-ID allocation */
+	/* Security context data */
+	u16		sc_id_start;
+	u16		sc_id_end;
+	u16		sc_id;
+	unsigned long	ctx_bm[DIV_ROUND_UP(SA_MAX_NUM_CTX,
+				BITS_PER_LONG)];
+	struct sa_tfm_ctx	*ctx;
+	struct dma_chan		*dma_rx1;
+	struct dma_chan		*dma_rx2;
+	struct dma_chan		*dma_tx;
+};
+
+/**
+ * struct sa_cmdl_param_info: Command label parameters info
+ * @index: Index of the parameter in the command label format
+ * @offset: the offset of the parameter
+ * @size: Size of the parameter
+ */
+struct sa_cmdl_param_info {
+	u16	index;
+	u16	offset;
+	u16	size;
+};
+
+/* Maximum length of Auxiliary data in 32bit words */
+#define SA_MAX_AUX_DATA_WORDS	8
+
+/**
+ * struct sa_cmdl_upd_info: Command label updation info
+ * @flags: flags in command label
+ * @submode: Encryption submodes
+ * @enc_size: Size of first pass encryption size
+ * @enc_size2: Size of second pass encryption size
+ * @enc_offset: Encryption payload offset in the packet
+ * @enc_iv: Encryption initialization vector for pass2
+ * @enc_iv2: Encryption initialization vector for pass2
+ * @aad: Associated data
+ * @payload: Payload info
+ * @auth_size: Authentication size for pass 1
+ * @auth_size2: Authentication size for pass 2
+ * @auth_offset: Authentication payload offset
+ * @auth_iv: Authentication initialization vector
+ * @aux_key_info: Authentication aux key information
+ * @aux_key: Aux key for authentication
+ */
+struct sa_cmdl_upd_info {
+	u16	flags;
+	u16	submode;
+	struct sa_cmdl_param_info	enc_size;
+	struct sa_cmdl_param_info	enc_size2;
+	struct sa_cmdl_param_info	enc_offset;
+	struct sa_cmdl_param_info	enc_iv;
+	struct sa_cmdl_param_info	enc_iv2;
+	struct sa_cmdl_param_info	aad;
+	struct sa_cmdl_param_info	payload;
+	struct sa_cmdl_param_info	auth_size;
+	struct sa_cmdl_param_info	auth_size2;
+	struct sa_cmdl_param_info	auth_offset;
+	struct sa_cmdl_param_info	auth_iv;
+	struct sa_cmdl_param_info	aux_key_info;
+	u32				aux_key[SA_MAX_AUX_DATA_WORDS];
+};
+
+/*
+ * Number of 32bit words appended after the command label
+ * in PSDATA to identify the crypto request context.
+ * word-0: Request type
+ * word-1: pointer to request
+ */
+#define SA_PSDATA_CTX_WORDS 4
+
+/* Maximum size of Command label in 32 words */
+#define SA_MAX_CMDL_WORDS (SA_DMA_NUM_PS_WORDS - SA_PSDATA_CTX_WORDS)
+
+/**
+ * struct sa_ctx_info: SA context information
+ * @sc: Pointer to security context
+ * @sc_phys: Security context physical address that is passed on to SA2UL
+ * @cmdl_size: Command label size
+ * @cmdl: Command label for a particular iteration
+ * @cmdl_upd_info: structure holding command label updation info
+ * @epib: Extended protocol information block words
+ */
+struct sa_ctx_info {
+	u8		*sc;
+	dma_addr_t	sc_phys;
+	u16		sc_id;
+	u16		cmdl_size;
+	u32		cmdl[SA_MAX_CMDL_WORDS];
+	struct sa_cmdl_upd_info cmdl_upd_info;
+	/* Store Auxiliary data such as K2/K3 subkeys in AES-XCBC */
+	u32		epib[SA_DMA_NUM_EPIB_WORDS];
+};
+
+struct sa_sham_hmac_ctx {
+	struct crypto_shash	*shash;
+	u8			ipad[SHA512_BLOCK_SIZE] SA_ALIGNED;
+	u8			opad[SHA512_BLOCK_SIZE] SA_ALIGNED;
+};
+
+/**
+ * struct sa_tfm_ctx: TFM context structure
+ * @dev_data: struct sa_crypto_data pointer
+ * @enc: struct sa_ctx_info for encryption
+ * @dec: struct sa_ctx_info for decryption
+ * @auth: struct sa_ctx_info for authentication
+ * @keylen: encrption/decryption keylength
+ * @key: encryption key
+ * @shash: software hash crypto_hash
+ * @authkey: authentication key
+ * @fallback_tfm: SW fallback ahash algorithm
+ */
+struct sa_tfm_ctx {
+	struct sa_crypto_data *dev_data;
+	struct sa_ctx_info enc;
+	struct sa_ctx_info dec;
+	struct sa_ctx_info auth;
+	int keylen;
+	u32 key[AES_KEYSIZE_256 / sizeof(u32)];
+	struct sa_sham_hmac_ctx base[0];
+	struct crypto_shash	*shash;
+	u8 authkey[SHA512_BLOCK_SIZE];
+	/* for fallback */
+	struct crypto_ahash	*fallback_tfm;
+};
+
+/**
+ * struct sa_dma_req_ctx: Structure used for tx dma request
+ * @dev_data: struct sa_crypto_data pointer
+ * @cmdl: Complete command label with psdata and epib included
+ * @src: source payload scatterlist pointer
+ * @src_nents: Number of nodes in source scatterlist
+ * @pkt: packet dma
+ */
+struct sa_dma_req_ctx {
+	struct sa_crypto_data *dev_data;
+	u32		cmdl[SA_MAX_CMDL_WORDS + SA_PSDATA_CTX_WORDS];
+	struct scatterlist *src;
+	unsigned int	src_nents;
+	bool		pkt;
+	struct ahash_request fallback_req;
+	u32 mode;
+};
+
+enum sa_submode {
+	SA_MODE_GEN = 0,
+	SA_MODE_CCM,
+	SA_MODE_GCM,
+	SA_MODE_GMAC
+};
+
+/* Encryption algorithms */
+enum sa_ealg_id {
+	SA_EALG_ID_NONE = 0,        /* No encryption */
+	SA_EALG_ID_NULL,            /* NULL encryption */
+	SA_EALG_ID_AES_CTR,         /* AES Counter mode */
+	SA_EALG_ID_AES_F8,          /* AES F8 mode */
+	SA_EALG_ID_AES_CBC,         /* AES CBC mode */
+	SA_EALG_ID_DES_CBC,         /* DES CBC mode */
+	SA_EALG_ID_3DES_CBC,        /* 3DES CBC mode */
+	SA_EALG_ID_CCM,             /* Counter with CBC-MAC mode */
+	SA_EALG_ID_GCM,             /* Galois Counter mode */
+	SA_EALG_ID_AES_ECB,
+	SA_EALG_ID_LAST
+};
+
+/* Authentication algorithms */
+enum sa_aalg_id {
+	SA_AALG_ID_NONE = 0,      /* No Authentication  */
+	SA_AALG_ID_NULL = SA_EALG_ID_LAST, /* NULL Authentication  */
+	SA_AALG_ID_MD5,           /* MD5 mode */
+	SA_AALG_ID_SHA1,          /* SHA1 mode */
+	SA_AALG_ID_SHA2_224,      /* 224-bit SHA2 mode */
+	SA_AALG_ID_SHA2_256,      /* 256-bit SHA2 mode */
+	SA_AALG_ID_SHA2_512,      /* 512-bit SHA2 mode */
+	SA_AALG_ID_HMAC_MD5,      /* HMAC with MD5 mode */
+	SA_AALG_ID_HMAC_SHA1,     /* HMAC with SHA1 mode */
+	SA_AALG_ID_HMAC_SHA2_224, /* HMAC with 224-bit SHA2 mode */
+	SA_AALG_ID_HMAC_SHA2_256, /* HMAC with 256-bit SHA2 mode */
+	SA_AALG_ID_GMAC,          /* Galois Message Auth. Code mode */
+	SA_AALG_ID_CMAC,          /* Cipher-based Mes. Auth. Code mode */
+	SA_AALG_ID_CBC_MAC,       /* Cipher Block Chaining */
+	SA_AALG_ID_AES_XCBC       /* AES Extended Cipher Block Chaining */
+};
+
+/*
+ * Mode control engine algorithms used to index the
+ * mode control instruction tables
+ */
+enum sa_eng_algo_id {
+	SA_ENG_ALGO_ECB = 0,
+	SA_ENG_ALGO_CBC,
+	SA_ENG_ALGO_CFB,
+	SA_ENG_ALGO_OFB,
+	SA_ENG_ALGO_CTR,
+	SA_ENG_ALGO_F8,
+	SA_ENG_ALGO_F8F9,
+	SA_ENG_ALGO_GCM,
+	SA_ENG_ALGO_GMAC,
+	SA_ENG_ALGO_CCM,
+	SA_ENG_ALGO_CMAC,
+	SA_ENG_ALGO_CBCMAC,
+	SA_NUM_ENG_ALGOS
+};
+
+struct sa_eng_info {
+	u8	eng_id;
+	u16	sc_size;
+};
+
+extern struct device *sa_ks2_dev;
+
+#endif /* _K3_SA2UL_ */
diff -Naur --no-dereference a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
--- a/drivers/dma/dmaengine.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/dma/dmaengine.c	2020-06-19 21:00:31.408227715 -0400
@@ -615,7 +615,8 @@
 }
 EXPORT_SYMBOL_GPL(dma_get_slave_channel);
 
-struct dma_chan *dma_get_any_slave_channel(struct dma_device *device)
+struct dma_chan *dmadev_get_slave_channel(struct dma_device *device,
+					  dma_filter_fn fn, void *fn_param)
 {
 	dma_cap_mask_t mask;
 	struct dma_chan *chan;
@@ -626,13 +627,13 @@
 	/* lock against __dma_request_channel */
 	mutex_lock(&dma_list_mutex);
 
-	chan = find_candidate(device, &mask, NULL, NULL);
+	chan = find_candidate(device, &mask, fn, fn_param);
 
 	mutex_unlock(&dma_list_mutex);
 
 	return IS_ERR(chan) ? NULL : chan;
 }
-EXPORT_SYMBOL_GPL(dma_get_any_slave_channel);
+EXPORT_SYMBOL_GPL(dmadev_get_slave_channel);
 
 /**
  * __dma_request_channel - try to allocate an exclusive channel
@@ -1306,6 +1307,79 @@
 }
 EXPORT_SYMBOL(dma_async_tx_descriptor_init);
 
+static inline int desc_check_and_set_metadata_mode(
+	struct dma_async_tx_descriptor *desc, enum dma_desc_metadata_mode mode)
+{
+	/* Make sure that the metadata mode is not mixed */
+	if (!desc->desc_metadata_mode) {
+		if (dmaengine_is_metadata_mode_supported(desc->chan, mode))
+			desc->desc_metadata_mode = mode;
+		else
+			return -ENOTSUPP;
+	} else if (desc->desc_metadata_mode != mode) {
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int dmaengine_desc_attach_metadata(struct dma_async_tx_descriptor *desc,
+				   void *data, size_t len)
+{
+	int ret;
+
+	if (!desc)
+		return -EINVAL;
+
+	ret = desc_check_and_set_metadata_mode(desc, DESC_METADATA_CLIENT);
+	if (ret)
+		return ret;
+
+	if (!desc->metadata_ops || !desc->metadata_ops->attach)
+		return -ENOTSUPP;
+
+	return desc->metadata_ops->attach(desc, data, len);
+}
+EXPORT_SYMBOL_GPL(dmaengine_desc_attach_metadata);
+
+void *dmaengine_desc_get_metadata_ptr(struct dma_async_tx_descriptor *desc,
+				      size_t *payload_len, size_t *max_len)
+{
+	int ret;
+
+	if (!desc)
+		return ERR_PTR(-EINVAL);
+
+	ret = desc_check_and_set_metadata_mode(desc, DESC_METADATA_ENGINE);
+	if (ret)
+		return ERR_PTR(ret);
+
+	if (!desc->metadata_ops || !desc->metadata_ops->get_ptr)
+		return ERR_PTR(-ENOTSUPP);
+
+	return desc->metadata_ops->get_ptr(desc, payload_len, max_len);
+}
+EXPORT_SYMBOL_GPL(dmaengine_desc_get_metadata_ptr);
+
+int dmaengine_desc_set_metadata_len(struct dma_async_tx_descriptor *desc,
+				    size_t payload_len)
+{
+	int ret;
+
+	if (!desc)
+		return -EINVAL;
+
+	ret = desc_check_and_set_metadata_mode(desc, DESC_METADATA_ENGINE);
+	if (ret)
+		return ret;
+
+	if (!desc->metadata_ops || !desc->metadata_ops->set_len)
+		return -ENOTSUPP;
+
+	return desc->metadata_ops->set_len(desc, payload_len);
+}
+EXPORT_SYMBOL_GPL(dmaengine_desc_set_metadata_len);
+
 /* dma_wait_for_async_tx - spin wait for a transaction to complete
  * @tx: in-flight transaction to wait on
  */
diff -Naur --no-dereference a/drivers/dma/dmaengine.h b/drivers/dma/dmaengine.h
--- a/drivers/dma/dmaengine.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/dma/dmaengine.h	2020-06-19 21:00:31.408227715 -0400
@@ -77,6 +77,7 @@
 		state->last = complete;
 		state->used = used;
 		state->residue = 0;
+		state->in_flight_bytes = 0;
 	}
 	return dma_async_is_complete(cookie, complete, used);
 }
@@ -87,6 +88,13 @@
 		state->residue = residue;
 }
 
+static inline void dma_set_in_flight_bytes(struct dma_tx_state *state,
+					   u32 in_flight_bytes)
+{
+	if (state)
+		state->in_flight_bytes = in_flight_bytes;
+}
+
 struct dmaengine_desc_callback {
 	dma_async_tx_callback callback;
 	dma_async_tx_callback_result callback_result;
diff -Naur --no-dereference a/drivers/dma/ti/k3-navss-udma.c b/drivers/dma/ti/k3-navss-udma.c
--- a/drivers/dma/ti/k3-navss-udma.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/dma/ti/k3-navss-udma.c	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,1302 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * K3 NAVSS DMA glue interface
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
+ *
+ */
+
+#include <linux/atomic.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <dt-bindings/dma/k3-udma.h>
+#include <linux/irqchip/irq-ti-sci-inta.h>
+#include <linux/soc/ti/k3-ringacc.h>
+#include <linux/dma/ti-cppi5.h>
+#include <linux/dma/k3-navss-udma.h>
+
+#include "k3-udma.h"
+
+struct k3_nav_udmax_common {
+	struct device *dev;
+	struct udma_dev *udmax;
+	const struct udma_tisci_rm *tisci_rm;
+	struct k3_ringacc *ringacc;
+	u32 src_thread;
+	u32 dst_thread;
+
+	u32  hdesc_size;
+	bool epib;
+	u32  psdata_size;
+	u32  swdata_size;
+};
+
+struct k3_nav_udmax_tx_channel {
+	struct k3_nav_udmax_common common;
+
+	struct udma_tchan *udma_tchanx;
+	int udma_tchan_id;
+
+	struct k3_ring *ringtx;
+	struct k3_ring *ringtxcq;
+
+	bool psil_paired;
+
+	unsigned int virq;
+
+	atomic_t free_pkts;
+	bool tx_pause_on_err;
+	bool tx_filt_einfo;
+	bool tx_filt_pswords;
+	bool tx_supr_tdpkt;
+};
+
+/**
+ * k3_nav_udmax_rx_flow - UDMA RX flow context data
+ *
+ */
+struct k3_nav_udmax_rx_flow {
+	struct udma_rflow *udma_rflow;
+	int udma_rflow_id;
+	struct k3_ring *ringrx;
+	struct k3_ring *ringrxfdq;
+
+	unsigned int virq;
+};
+
+struct k3_nav_udmax_rx_channel {
+	struct k3_nav_udmax_common common;
+
+	struct udma_rchan *udma_rchanx;
+	int udma_rchan_id;
+	bool remote;
+
+	bool psil_paired;
+
+	u32  swdata_size;
+	int  flow_id_base;
+
+	struct k3_nav_udmax_rx_flow *flows;
+	u32 flow_num;
+	u32 flows_ready;
+};
+
+#define K3_UDMAX_TDOWN_TIMEOUT_US 1000
+
+static int of_k3_nav_udmax_parse(struct device_node *udmax_np,
+				 struct k3_nav_udmax_common *common)
+{
+	common->ringacc = of_k3_ringacc_get_by_phandle(udmax_np, "ti,ringacc");
+	if (IS_ERR(common->ringacc))
+		return PTR_ERR(common->ringacc);
+
+	common->udmax = of_xudma_dev_get(udmax_np, NULL);
+	if (IS_ERR(common->udmax))
+		return PTR_ERR(common->udmax);
+
+	common->tisci_rm = xudma_dev_get_tisci_rm(common->udmax);
+
+	return 0;
+}
+
+static int of_k3_nav_udmax_parse_chn(struct device_node *chn_np,
+				     const char *name,
+				     struct k3_nav_udmax_common *common,
+				     bool tx_chn)
+{
+	struct device_node *psil_cfg_node;
+	struct device_node *ch_cfg_node;
+	struct of_phandle_args dma_spec;
+	int index, ret = 0;
+	char prop[50];
+	u32 val;
+
+	if (unlikely(!name))
+		return -EINVAL;
+
+	index = of_property_match_string(chn_np, "dma-names", name);
+	if (index < 0)
+		return index;
+
+	if (of_parse_phandle_with_args(chn_np, "dmas", "#dma-cells", index,
+				       &dma_spec))
+		return -ENOENT;
+
+	if (tx_chn && dma_spec.args[2] != UDMA_DIR_TX) {
+		ret = -EINVAL;
+		goto out_put_spec;
+	}
+
+	if (!tx_chn && dma_spec.args[2] != UDMA_DIR_RX) {
+		ret = -EINVAL;
+		goto out_put_spec;
+	}
+
+	/* get psil cfg node */
+	psil_cfg_node = of_find_node_by_phandle(dma_spec.args[0]);
+	if (!psil_cfg_node) {
+		ret = -ENOENT;
+		goto out_put_spec;
+	}
+
+	snprintf(prop, sizeof(prop), "ti,psil-config%u", dma_spec.args[1]);
+	ch_cfg_node = of_find_node_by_name(psil_cfg_node, prop);
+	if (!ch_cfg_node) {
+		dev_err(common->dev,
+			"Channel %u configuration node is missing\n",
+			dma_spec.args[1]);
+		goto out_put_psil_cfg;
+	}
+
+	common->epib = of_property_read_bool(ch_cfg_node, "ti,needs-epib");
+
+	if (!of_property_read_u32(ch_cfg_node, "ti,psd-size", &val))
+		common->psdata_size = val;
+
+	ret = of_property_read_u32(psil_cfg_node, "ti,psil-base", &val);
+	if (ret) {
+		dev_err(common->dev, "ti,psil-base is missing %d\n", ret);
+		goto out_ch_cfg;
+	}
+
+	if (tx_chn)
+		common->dst_thread = val + dma_spec.args[1];
+	else
+		common->src_thread = val + dma_spec.args[1];
+	ret = of_k3_nav_udmax_parse(dma_spec.np, common);
+
+out_ch_cfg:
+	of_node_put(ch_cfg_node);
+out_put_psil_cfg:
+	of_node_put(psil_cfg_node);
+out_put_spec:
+	of_node_put(dma_spec.np);
+	return ret;
+};
+
+static void k3_nav_udmax_dump_tx_chn(struct k3_nav_udmax_tx_channel *tx_chn)
+{
+	struct device *dev = tx_chn->common.dev;
+
+	dev_dbg(dev, "dump_tx_chn:\n"
+		"udma_tchan_id: %d\n"
+		"src_thread: %08x\n"
+		"dst_thread: %08x\n",
+		tx_chn->udma_tchan_id,
+		tx_chn->common.src_thread,
+		tx_chn->common.dst_thread);
+}
+
+static void k3_nav_udmax_dump_tx_rt_chn(struct k3_nav_udmax_tx_channel *chn,
+					char *mark)
+{
+	struct device *dev = chn->common.dev;
+
+	dev_dbg(dev, "=== dump ===> %s\n", mark);
+	dev_dbg(dev, "0x%08X: %08X\n", UDMA_TCHAN_RT_CTL_REG,
+		xudma_tchanrt_read(chn->udma_tchanx, UDMA_TCHAN_RT_CTL_REG));
+	dev_dbg(dev, "0x%08X: %08X\n", UDMA_TCHAN_RT_PEER_RT_EN_REG,
+		xudma_tchanrt_read(chn->udma_tchanx,
+				   UDMA_TCHAN_RT_PEER_RT_EN_REG));
+	dev_dbg(dev, "0x%08X: %08X\n", UDMA_TCHAN_RT_PCNT_REG,
+		xudma_tchanrt_read(chn->udma_tchanx, UDMA_TCHAN_RT_PCNT_REG));
+	dev_dbg(dev, "0x%08X: %08X\n", UDMA_TCHAN_RT_BCNT_REG,
+		xudma_tchanrt_read(chn->udma_tchanx, UDMA_TCHAN_RT_BCNT_REG));
+	dev_dbg(dev, "0x%08X: %08X\n", UDMA_TCHAN_RT_SBCNT_REG,
+		xudma_tchanrt_read(chn->udma_tchanx, UDMA_TCHAN_RT_SBCNT_REG));
+}
+
+static int k3_nav_udmax_cfg_tx_chn(struct k3_nav_udmax_tx_channel *tx_chn)
+{
+	const struct udma_tisci_rm *tisci_rm = tx_chn->common.tisci_rm;
+	struct ti_sci_msg_rm_udmap_tx_ch_cfg req;
+
+	memset(&req, 0, sizeof(req));
+
+	req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID;
+	req.nav_id = tisci_rm->tisci_dev_id;
+	req.index = tx_chn->udma_tchan_id;
+	if (tx_chn->tx_pause_on_err)
+		req.tx_pause_on_err = 1;
+	if (tx_chn->tx_filt_einfo)
+		req.tx_filt_einfo = 1;
+	if (tx_chn->tx_filt_pswords)
+		req.tx_filt_pswords = 1;
+	req.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
+	if (tx_chn->tx_supr_tdpkt)
+		req.tx_supr_tdpkt = 1;
+	req.tx_fetch_size = tx_chn->common.hdesc_size >> 2;
+	req.txcq_qnum = k3_ringacc_get_ring_id(tx_chn->ringtxcq);
+
+	return tisci_rm->tisci_udmap_ops->tx_ch_cfg(tisci_rm->tisci, &req);
+}
+
+struct k3_nav_udmax_tx_channel *k3_nav_udmax_request_tx_chn(struct device *dev,
+		const char *name, struct k3_nav_udmax_tx_channel_cfg *cfg)
+{
+	struct k3_nav_udmax_tx_channel *tx_chn;
+	int ret;
+
+	tx_chn = devm_kzalloc(dev, sizeof(*tx_chn), GFP_KERNEL);
+	if (!tx_chn)
+		return ERR_PTR(-ENOMEM);
+
+	tx_chn->common.dev = dev;
+	tx_chn->common.swdata_size = cfg->swdata_size;
+	tx_chn->tx_pause_on_err = cfg->tx_pause_on_err;
+	tx_chn->tx_filt_einfo = cfg->tx_filt_einfo;
+	tx_chn->tx_filt_pswords = cfg->tx_filt_pswords;
+	tx_chn->tx_supr_tdpkt = cfg->tx_supr_tdpkt;
+
+	/* parse of udmap channel */
+	ret = of_k3_nav_udmax_parse_chn(dev->of_node, name,
+					&tx_chn->common, true);
+	if (ret)
+		goto err;
+
+	tx_chn->common.hdesc_size = cppi5_hdesc_calc_size(tx_chn->common.epib,
+						tx_chn->common.psdata_size,
+						tx_chn->common.swdata_size);
+
+	/* request and cfg UDMAP TX channel */
+	tx_chn->udma_tchanx = xudma_tchan_get(tx_chn->common.udmax, -1);
+	if (IS_ERR(tx_chn->udma_tchanx)) {
+		ret = PTR_ERR(tx_chn->udma_tchanx);
+		dev_err(dev, "UDMAX tchanx get err %d\n", ret);
+		goto err;
+	}
+	tx_chn->udma_tchan_id = xudma_tchan_get_id(tx_chn->udma_tchanx);
+
+	atomic_set(&tx_chn->free_pkts, cfg->txcq_cfg.size);
+
+	/* request and cfg rings */
+	tx_chn->ringtx = k3_ringacc_request_ring(tx_chn->common.ringacc,
+						 tx_chn->udma_tchan_id, 0);
+	if (!tx_chn->ringtx) {
+		ret = -ENODEV;
+		dev_err(dev, "Failed to get TX ring %u\n",
+			tx_chn->udma_tchan_id);
+		goto err;
+	}
+
+	tx_chn->ringtxcq = k3_ringacc_request_ring(tx_chn->common.ringacc,
+						   -1, 0);
+	if (!tx_chn->ringtxcq) {
+		ret = -ENODEV;
+		dev_err(dev, "Failed to get TXCQ ring\n");
+		goto err;
+	}
+
+	ret = k3_ringacc_ring_cfg(tx_chn->ringtx, &cfg->tx_cfg);
+	if (ret) {
+		dev_err(dev, "Failed to cfg ringtx %d\n", ret);
+		goto err;
+	}
+
+	ret = k3_ringacc_ring_cfg(tx_chn->ringtxcq, &cfg->txcq_cfg);
+	if (ret) {
+		dev_err(dev, "Failed to cfg ringtx %d\n", ret);
+		goto err;
+	}
+
+	/* request and cfg psi-l */
+	tx_chn->common.src_thread =
+			xudma_dev_get_psil_base(tx_chn->common.udmax) +
+			tx_chn->udma_tchan_id;
+
+	ret = k3_nav_udmax_cfg_tx_chn(tx_chn);
+	if (ret) {
+		dev_err(dev, "Failed to cfg tchan %d\n", ret);
+		goto err;
+	}
+
+	ret = xudma_navss_psil_pair(tx_chn->common.udmax,
+				    tx_chn->common.src_thread,
+				    tx_chn->common.dst_thread);
+	if (ret) {
+		dev_err(dev, "PSI-L request err %d\n", ret);
+		goto err;
+	}
+
+	tx_chn->psil_paired = true;
+
+	/* reset TX RT registers */
+	k3_nav_udmax_disable_tx_chn(tx_chn);
+
+	k3_nav_udmax_dump_tx_chn(tx_chn);
+
+	return tx_chn;
+
+err:
+	k3_nav_udmax_release_tx_chn(tx_chn);
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(k3_nav_udmax_request_tx_chn);
+
+void k3_nav_udmax_release_tx_chn(struct k3_nav_udmax_tx_channel *tx_chn)
+{
+	if (tx_chn->psil_paired) {
+		xudma_navss_psil_unpair(tx_chn->common.udmax,
+					tx_chn->common.src_thread,
+					tx_chn->common.dst_thread);
+		tx_chn->psil_paired = false;
+	}
+
+	if (!IS_ERR_OR_NULL(tx_chn->common.udmax)) {
+		if (!IS_ERR_OR_NULL(tx_chn->udma_tchanx))
+			xudma_tchan_put(tx_chn->common.udmax,
+					tx_chn->udma_tchanx);
+
+		xudma_dev_put(tx_chn->common.udmax);
+	}
+
+	if (tx_chn->ringtxcq)
+		k3_ringacc_ring_free(tx_chn->ringtxcq);
+
+	if (tx_chn->ringtx)
+		k3_ringacc_ring_free(tx_chn->ringtx);
+}
+EXPORT_SYMBOL_GPL(k3_nav_udmax_release_tx_chn);
+
+int k3_nav_udmax_push_tx_chn(struct k3_nav_udmax_tx_channel *tx_chn,
+			     struct cppi5_host_desc_t *desc_tx,
+			     dma_addr_t desc_dma)
+{
+	u32 ringtxcq_id;
+
+	if (!atomic_add_unless(&tx_chn->free_pkts, -1, 0))
+		return -ENOMEM;
+
+	ringtxcq_id = k3_ringacc_get_ring_id(tx_chn->ringtxcq);
+	cppi5_desc_set_retpolicy(&desc_tx->hdr, 0, ringtxcq_id);
+
+	return k3_ringacc_ring_push(tx_chn->ringtx, &desc_dma);
+}
+EXPORT_SYMBOL_GPL(k3_nav_udmax_push_tx_chn);
+
+int k3_nav_udmax_pop_tx_chn(struct k3_nav_udmax_tx_channel *tx_chn,
+			    dma_addr_t *desc_dma)
+{
+	int ret;
+
+	ret = k3_ringacc_ring_pop(tx_chn->ringtxcq, desc_dma);
+	if (!ret)
+		atomic_inc(&tx_chn->free_pkts);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(k3_nav_udmax_pop_tx_chn);
+
+int k3_nav_udmax_enable_tx_chn(struct k3_nav_udmax_tx_channel *tx_chn)
+{
+	u32 txrt_ctl;
+
+	txrt_ctl = UDMA_PEER_RT_EN_ENABLE;
+	xudma_tchanrt_write(tx_chn->udma_tchanx,
+			    UDMA_TCHAN_RT_PEER_RT_EN_REG,
+			    txrt_ctl);
+
+	txrt_ctl = xudma_tchanrt_read(tx_chn->udma_tchanx,
+				      UDMA_TCHAN_RT_CTL_REG);
+	txrt_ctl |= UDMA_CHAN_RT_CTL_EN;
+	xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_TCHAN_RT_CTL_REG,
+			    txrt_ctl);
+
+	k3_nav_udmax_dump_tx_rt_chn(tx_chn, "txchn en");
+	return 0;
+}
+EXPORT_SYMBOL_GPL(k3_nav_udmax_enable_tx_chn);
+
+void k3_nav_udmax_disable_tx_chn(struct k3_nav_udmax_tx_channel *tx_chn)
+{
+	k3_nav_udmax_dump_tx_rt_chn(tx_chn, "txchn dis1");
+
+	xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_TCHAN_RT_CTL_REG, 0);
+
+	xudma_tchanrt_write(tx_chn->udma_tchanx,
+			    UDMA_TCHAN_RT_PEER_RT_EN_REG, 0);
+	k3_nav_udmax_dump_tx_rt_chn(tx_chn, "txchn dis2");
+}
+EXPORT_SYMBOL_GPL(k3_nav_udmax_disable_tx_chn);
+
+void k3_nav_udmax_tdown_tx_chn(struct k3_nav_udmax_tx_channel *tx_chn,
+			       bool sync)
+{
+	int i = 0;
+	u32 val;
+
+	k3_nav_udmax_dump_tx_rt_chn(tx_chn, "txchn tdown1");
+
+	xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_TCHAN_RT_CTL_REG,
+			    UDMA_CHAN_RT_CTL_EN | UDMA_CHAN_RT_CTL_TDOWN);
+
+	val = xudma_tchanrt_read(tx_chn->udma_tchanx, UDMA_TCHAN_RT_CTL_REG);
+
+	while (sync && (val & UDMA_CHAN_RT_CTL_EN)) {
+		val = xudma_tchanrt_read(tx_chn->udma_tchanx,
+					 UDMA_TCHAN_RT_CTL_REG);
+		udelay(1);
+		if (i > K3_UDMAX_TDOWN_TIMEOUT_US) {
+			dev_err(tx_chn->common.dev, "TX tdown timeout\n");
+			break;
+		}
+		i++;
+	}
+
+	val = xudma_tchanrt_read(tx_chn->udma_tchanx,
+				 UDMA_TCHAN_RT_PEER_RT_EN_REG);
+	if (sync && (val & UDMA_PEER_RT_EN_ENABLE))
+		dev_err(tx_chn->common.dev, "TX tdown peer not stopped\n");
+	k3_nav_udmax_dump_tx_rt_chn(tx_chn, "txchn tdown2");
+}
+EXPORT_SYMBOL_GPL(k3_nav_udmax_tdown_tx_chn);
+
+void k3_nav_udmax_reset_tx_chn(struct k3_nav_udmax_tx_channel *tx_chn,
+			       void *data,
+			       void (*cleanup)(void *data, dma_addr_t desc_dma))
+{
+	dma_addr_t desc_dma;
+	int occ_tx, i, ret;
+
+	/* reset TXCQ as it is not input for udma - expected to be empty */
+	if (tx_chn->ringtxcq)
+		k3_ringacc_ring_reset(tx_chn->ringtxcq);
+
+	/*
+	 * TXQ reset need to be special way as it is input for udma and its
+	 * state cached by udma, so:
+	 * 1) save TXQ occ
+	 * 2) clean up TXQ and call callback .cleanup() for each desc
+	 * 3) reset TXQ in a special way
+	 */
+	occ_tx = k3_ringacc_ring_get_occ(tx_chn->ringtx);
+	dev_dbg(tx_chn->common.dev, "TX reset occ_tx %u\n", occ_tx);
+
+	for (i = 0; i < occ_tx; i++) {
+		ret = k3_ringacc_ring_pop(tx_chn->ringtx, &desc_dma);
+		if (ret) {
+			dev_err(tx_chn->common.dev, "TX reset pop %d\n", ret);
+			break;
+		}
+		cleanup(data, desc_dma);
+	}
+
+	k3_ringacc_ring_reset_dma(tx_chn->ringtx, occ_tx);
+}
+EXPORT_SYMBOL_GPL(k3_nav_udmax_reset_tx_chn);
+
+u32 k3_nav_udmax_tx_get_hdesc_size(struct k3_nav_udmax_tx_channel *tx_chn)
+{
+	return tx_chn->common.hdesc_size;
+}
+EXPORT_SYMBOL_GPL(k3_nav_udmax_tx_get_hdesc_size);
+
+u32 k3_nav_udmax_tx_get_txcq_id(struct k3_nav_udmax_tx_channel *tx_chn)
+{
+	return k3_ringacc_get_ring_id(tx_chn->ringtxcq);
+}
+EXPORT_SYMBOL_GPL(k3_nav_udmax_tx_get_txcq_id);
+
+int k3_nav_udmax_tx_get_irq(struct k3_nav_udmax_tx_channel *tx_chn,
+			    unsigned int *irq, u32 flags, bool share,
+			    struct k3_nav_udmax_tx_channel *tx_chn_share)
+{
+	unsigned int virq = 0;
+
+	if (share && tx_chn_share)
+		virq = tx_chn_share->virq;
+
+	tx_chn->virq = ti_sci_inta_register_event(tx_chn->common.dev,
+				k3_ringacc_get_tisci_dev_id(tx_chn->ringtxcq),
+				k3_ringacc_get_ring_id(tx_chn->ringtxcq), virq,
+				false, flags);
+	if (tx_chn->virq <= 0)
+		return -ENODEV;
+
+	*irq = tx_chn->virq;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(k3_nav_udmax_tx_get_irq);
+
+void k3_nav_udmax_tx_put_irq(struct k3_nav_udmax_tx_channel *tx_chn)
+{
+	if (tx_chn->virq <= 0)
+		return;
+
+	ti_sci_inta_unregister_event(tx_chn->common.dev,
+				k3_ringacc_get_tisci_dev_id(tx_chn->ringtxcq),
+				k3_ringacc_get_ring_id(tx_chn->ringtxcq),
+				tx_chn->virq);
+}
+EXPORT_SYMBOL_GPL(k3_nav_udmax_tx_put_irq);
+
+static int k3_nav_udmax_cfg_rx_chn(struct k3_nav_udmax_rx_channel *rx_chn)
+{
+	const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
+	struct ti_sci_msg_rm_udmap_rx_ch_cfg req;
+	int ret;
+
+	memset(&req, 0, sizeof(req));
+
+	req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID;
+
+	req.nav_id = tisci_rm->tisci_dev_id;
+	req.index = rx_chn->udma_rchan_id;
+	req.rx_fetch_size = rx_chn->common.hdesc_size >> 2;
+	/*
+	 * TODO: we can't support rxcq_qnum/RCHAN[a]_RCQ cfg with current sysfw
+	 * and udmax impl, so just configure it to invalid value.
+	 * req.rxcq_qnum = k3_ringacc_get_ring_id(rx_chn->flows[0].ringrx);
+	 */
+	req.rxcq_qnum = 0xFFFF;
+	if (rx_chn->flow_num && rx_chn->flow_id_base != rx_chn->udma_rchan_id) {
+		/* Default flow + extra ones */
+		req.flowid_start = rx_chn->flow_id_base;
+		req.flowid_cnt = rx_chn->flow_num;
+	}
+	req.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
+
+	ret = tisci_rm->tisci_udmap_ops->rx_ch_cfg(tisci_rm->tisci, &req);
+	if (ret)
+		dev_err(rx_chn->common.dev, "rchan%d cfg failed %d\n",
+			rx_chn->udma_rchan_id, ret);
+
+	return ret;
+}
+
+static void k3_nav_udmax_release_rx_flow(struct k3_nav_udmax_rx_channel *rx_chn,
+					 u32 flow_num)
+{
+	struct k3_nav_udmax_rx_flow *flow = &rx_chn->flows[flow_num];
+
+	if (IS_ERR_OR_NULL(flow->udma_rflow))
+		return;
+
+	if (flow->ringrxfdq)
+		k3_ringacc_ring_free(flow->ringrxfdq);
+
+	if (flow->ringrx)
+		k3_ringacc_ring_free(flow->ringrx);
+
+	xudma_rflow_put(rx_chn->common.udmax, flow->udma_rflow);
+	flow->udma_rflow = NULL;
+	rx_chn->flows_ready--;
+}
+
+static int k3_nav_udmax_cfg_rx_flow(struct k3_nav_udmax_rx_channel *rx_chn,
+				    u32 flow_idx,
+				    struct k3_nav_udmax_rx_flow_cfg *flow_cfg)
+{
+	struct k3_nav_udmax_rx_flow *flow = &rx_chn->flows[flow_idx];
+	const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
+	struct device *dev = rx_chn->common.dev;
+	struct ti_sci_msg_rm_udmap_flow_cfg req;
+	int rx_ring_id;
+	int rx_ringfdq_id;
+	int ret = 0;
+
+	flow->udma_rflow = xudma_rflow_get(rx_chn->common.udmax,
+					   flow->udma_rflow_id);
+	if (IS_ERR(flow->udma_rflow)) {
+		ret = PTR_ERR(flow->udma_rflow);
+		dev_err(dev, "UDMAX rflow get err %d\n", ret);
+		goto err;
+	}
+
+	if (flow->udma_rflow_id != xudma_rflow_get_id(flow->udma_rflow)) {
+		xudma_rflow_put(rx_chn->common.udmax, flow->udma_rflow);
+		return -ENODEV;
+	}
+
+	/* request and cfg rings */
+	flow->ringrx = k3_ringacc_request_ring(rx_chn->common.ringacc,
+					       flow_cfg->ring_rxq_id, 0);
+	if (!flow->ringrx) {
+		ret = -ENODEV;
+		dev_err(dev, "Failed to get RX ring\n");
+		goto err;
+	}
+
+	flow->ringrxfdq = k3_ringacc_request_ring(rx_chn->common.ringacc,
+						  flow_cfg->ring_rxfdq0_id, 0);
+	if (!flow->ringrxfdq) {
+		ret = -ENODEV;
+		dev_err(dev, "Failed to get RXFDQ ring\n");
+		goto err;
+	}
+
+	ret = k3_ringacc_ring_cfg(flow->ringrx, &flow_cfg->rx_cfg);
+	if (ret) {
+		dev_err(dev, "Failed to cfg ringrx %d\n", ret);
+		goto err;
+	}
+
+	ret = k3_ringacc_ring_cfg(flow->ringrxfdq, &flow_cfg->rxfdq_cfg);
+	if (ret) {
+		dev_err(dev, "Failed to cfg ringrxfdq %d\n", ret);
+		goto err;
+	}
+
+	if (rx_chn->remote) {
+		rx_ring_id = TI_SCI_RESOURCE_NULL;
+		rx_ringfdq_id = TI_SCI_RESOURCE_NULL;
+	} else {
+		rx_ring_id = k3_ringacc_get_ring_id(flow->ringrx);
+		rx_ringfdq_id = k3_ringacc_get_ring_id(flow->ringrxfdq);
+	}
+
+	memset(&req, 0, sizeof(req));
+
+	req.valid_params =
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
+	req.nav_id = tisci_rm->tisci_dev_id;
+	req.flow_index = flow->udma_rflow_id;
+	if (rx_chn->common.epib)
+		req.rx_einfo_present = 1;
+	if (rx_chn->common.psdata_size)
+		req.rx_psinfo_present = 1;
+	if (flow_cfg->rx_error_handling)
+		req.rx_error_handling = 1;
+	req.rx_desc_type = 0;
+	req.rx_dest_qnum = rx_ring_id;
+	req.rx_src_tag_hi_sel = 0;
+	req.rx_src_tag_lo_sel = flow_cfg->src_tag_lo_sel;
+	req.rx_dest_tag_hi_sel = 0;
+	req.rx_dest_tag_lo_sel = 0;
+	req.rx_fdq0_sz0_qnum = rx_ringfdq_id;
+	req.rx_fdq1_qnum = rx_ringfdq_id;
+	req.rx_fdq2_qnum = rx_ringfdq_id;
+	req.rx_fdq3_qnum = rx_ringfdq_id;
+
+	ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req);
+	if (ret) {
+		dev_err(dev, "flow%d config failed: %d\n", flow->udma_rflow_id,
+			ret);
+		goto err;
+	}
+
+	rx_chn->flows_ready++;
+	dev_dbg(dev, "flow%d config done. ready:%d\n",
+		flow->udma_rflow_id, rx_chn->flows_ready);
+
+	return 0;
+err:
+	k3_nav_udmax_release_rx_flow(rx_chn, flow_idx);
+	return ret;
+}
+
+static void k3_nav_udmax_dump_rx_chn(struct k3_nav_udmax_rx_channel *chn)
+{
+	struct device *dev = chn->common.dev;
+
+	dev_dbg(dev, "dump_rx_chn:\n"
+		"udma_rchan_id: %d\n"
+		"src_thread: %08x\n"
+		"dst_thread: %08x\n"
+		"epib: %d\n"
+		"hdesc_size: %u\n"
+		"psdata_size: %u\n"
+		"swdata_size: %u\n"
+		"flow_id_base: %d\n"
+		"flow_num: %d\n",
+		chn->udma_rchan_id,
+		chn->common.src_thread,
+		chn->common.dst_thread,
+		chn->common.epib,
+		chn->common.hdesc_size,
+		chn->common.psdata_size,
+		chn->common.swdata_size,
+		chn->flow_id_base,
+		chn->flow_num);
+}
+
+static void k3_nav_udmax_dump_rx_rt_chn(struct k3_nav_udmax_rx_channel *chn,
+					char *mark)
+{
+	struct device *dev = chn->common.dev;
+
+	dev_dbg(dev, "=== dump ===> %s\n", mark);
+
+	dev_dbg(dev, "0x%08X: %08X\n", UDMA_RCHAN_RT_CTL_REG,
+		xudma_rchanrt_read(chn->udma_rchanx, UDMA_RCHAN_RT_CTL_REG));
+	dev_dbg(dev, "0x%08X: %08X\n", UDMA_RCHAN_RT_PEER_RT_EN_REG,
+		xudma_rchanrt_read(chn->udma_rchanx,
+				   UDMA_RCHAN_RT_PEER_RT_EN_REG));
+	dev_dbg(dev, "0x%08X: %08X\n", UDMA_RCHAN_RT_PCNT_REG,
+		xudma_rchanrt_read(chn->udma_rchanx, UDMA_RCHAN_RT_PCNT_REG));
+	dev_dbg(dev, "0x%08X: %08X\n", UDMA_RCHAN_RT_BCNT_REG,
+		xudma_rchanrt_read(chn->udma_rchanx, UDMA_RCHAN_RT_BCNT_REG));
+	dev_dbg(dev, "0x%08X: %08X\n", UDMA_RCHAN_RT_SBCNT_REG,
+		xudma_rchanrt_read(chn->udma_rchanx, UDMA_RCHAN_RT_SBCNT_REG));
+}
+
+static int
+k3_nav_udmax_allocate_rx_flows(struct k3_nav_udmax_rx_channel *rx_chn,
+			       struct k3_nav_udmax_rx_channel_cfg *cfg)
+{
+	int ret;
+
+	/* default rflow */
+	if (cfg->flow_id_use_rxchan_id)
+		return 0;
+
+	/* not a GP rflows */
+	if (rx_chn->flow_id_base != -1 &&
+	    !xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base))
+		return 0;
+
+	/* Allocate range of GP rflows */
+	ret = xudma_alloc_gp_rflow_range(rx_chn->common.udmax,
+					 rx_chn->flow_id_base,
+					 rx_chn->flow_num);
+	if (ret < 0) {
+		dev_err(rx_chn->common.dev, "UDMAX reserve_rflow %d cnt:%d err: %d\n",
+			rx_chn->flow_id_base, rx_chn->flow_num, ret);
+		return ret;
+	}
+	rx_chn->flow_id_base = ret;
+
+	return 0;
+}
+
+static struct k3_nav_udmax_rx_channel *
+k3_nav_udmax_request_rx_chn_priv(struct device *dev,
+				 const char *name,
+				 struct k3_nav_udmax_rx_channel_cfg *cfg)
+{
+	struct k3_nav_udmax_rx_channel *rx_chn;
+	int ret, i;
+
+	if (cfg->flow_id_num <= 0)
+		return ERR_PTR(-EINVAL);
+
+	if (cfg->flow_id_num != 1 &&
+	    (cfg->def_flow_cfg || cfg->flow_id_use_rxchan_id))
+		return ERR_PTR(-EINVAL);
+
+	rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL);
+	if (!rx_chn)
+		return ERR_PTR(-ENOMEM);
+
+	rx_chn->common.dev = dev;
+	rx_chn->common.swdata_size = cfg->swdata_size;
+	rx_chn->remote = false;
+
+	/* parse of udmap channel */
+	ret = of_k3_nav_udmax_parse_chn(dev->of_node, name,
+					&rx_chn->common, false);
+	if (ret)
+		goto err;
+
+	rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib,
+						rx_chn->common.psdata_size,
+						rx_chn->common.swdata_size);
+
+	/* request and cfg UDMAP RX channel */
+	rx_chn->udma_rchanx = xudma_rchan_get(rx_chn->common.udmax, -1);
+	if (IS_ERR(rx_chn->udma_rchanx)) {
+		ret = PTR_ERR(rx_chn->udma_rchanx);
+		dev_err(dev, "UDMAX rchanx get err %d\n", ret);
+		goto err;
+	}
+	rx_chn->udma_rchan_id = xudma_rchan_get_id(rx_chn->udma_rchanx);
+
+	rx_chn->flow_num = cfg->flow_id_num;
+	rx_chn->flow_id_base = cfg->flow_id_base;
+
+	/* Use RX channel id as flow id: target dev can't generate flow_id */
+	if (cfg->flow_id_use_rxchan_id)
+		rx_chn->flow_id_base = rx_chn->udma_rchan_id;
+
+	rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num,
+				     sizeof(*rx_chn->flows), GFP_KERNEL);
+	if (!rx_chn->flows) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	ret = k3_nav_udmax_allocate_rx_flows(rx_chn, cfg);
+	if (ret)
+		goto err;
+
+	for (i = 0; i < rx_chn->flow_num; i++)
+		rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i;
+
+	/* request and cfg psi-l */
+	rx_chn->common.dst_thread =
+			xudma_dev_get_psil_base(rx_chn->common.udmax) +
+			rx_chn->udma_rchan_id;
+
+	ret = k3_nav_udmax_cfg_rx_chn(rx_chn);
+	if (ret) {
+		dev_err(dev, "Failed to cfg rchan %d\n", ret);
+		goto err;
+	}
+
+	/* init default RX flow only if flow_num = 1 */
+	if (cfg->def_flow_cfg) {
+		ret = k3_nav_udmax_cfg_rx_flow(rx_chn, 0, cfg->def_flow_cfg);
+		if (ret)
+			goto err;
+	}
+
+	ret = xudma_navss_psil_pair(rx_chn->common.udmax,
+				    rx_chn->common.src_thread,
+				    rx_chn->common.dst_thread);
+	if (ret) {
+		dev_err(dev, "PSI-L request err %d\n", ret);
+		goto err;
+	}
+
+	rx_chn->psil_paired = true;
+
+	/* reset RX RT registers */
+	k3_nav_udmax_disable_rx_chn(rx_chn);
+
+	k3_nav_udmax_dump_rx_chn(rx_chn);
+
+	return rx_chn;
+
+err:
+	k3_nav_udmax_release_rx_chn(rx_chn);
+	return ERR_PTR(ret);
+}
+
+static struct k3_nav_udmax_rx_channel *
+k3_nav_udmax_request_remote_rx_chn(struct device *dev,
+				   const char *name,
+				   struct k3_nav_udmax_rx_channel_cfg *cfg)
+{
+	struct k3_nav_udmax_rx_channel *rx_chn;
+	int ret, i;
+
+	if (cfg->flow_id_num <= 0 ||
+	    cfg->flow_id_use_rxchan_id ||
+	    cfg->def_flow_cfg ||
+	    cfg->flow_id_base < 0)
+		return ERR_PTR(-EINVAL);
+
+	/*
+	 * Remote RX channel is under control of Remote CPU core, so
+	 * Linux can only request and manipulate by dedicated RX flows
+	 */
+
+	rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL);
+	if (!rx_chn)
+		return ERR_PTR(-ENOMEM);
+
+	rx_chn->common.dev = dev;
+	rx_chn->common.swdata_size = cfg->swdata_size;
+	rx_chn->remote = true;
+	rx_chn->udma_rchan_id = -1;
+	rx_chn->flow_num = cfg->flow_id_num;
+	rx_chn->flow_id_base = cfg->flow_id_base;
+	rx_chn->psil_paired = false;
+
+	/* parse of udmap channel */
+	ret = of_k3_nav_udmax_parse_chn(dev->of_node, name,
+					&rx_chn->common, false);
+	if (ret)
+		goto err;
+
+	rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib,
+						rx_chn->common.psdata_size,
+						rx_chn->common.swdata_size);
+
+	rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num,
+				     sizeof(*rx_chn->flows), GFP_KERNEL);
+	if (!rx_chn->flows) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	ret = k3_nav_udmax_allocate_rx_flows(rx_chn, cfg);
+	if (ret)
+		goto err;
+
+	for (i = 0; i < rx_chn->flow_num; i++)
+		rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i;
+
+	k3_nav_udmax_dump_rx_chn(rx_chn);
+
+	return rx_chn;
+
+err:
+	k3_nav_udmax_release_rx_chn(rx_chn);
+	return ERR_PTR(ret);
+}
+
+struct k3_nav_udmax_rx_channel *
+k3_nav_udmax_request_rx_chn(struct device *dev,
+			    const char *name,
+			    struct k3_nav_udmax_rx_channel_cfg *cfg)
+{
+	if (cfg->remote)
+		return k3_nav_udmax_request_remote_rx_chn(dev, name, cfg);
+	else
+		return k3_nav_udmax_request_rx_chn_priv(dev, name, cfg);
+}
+EXPORT_SYMBOL_GPL(k3_nav_udmax_request_rx_chn);
+
+void k3_nav_udmax_release_rx_chn(struct k3_nav_udmax_rx_channel *rx_chn)
+{
+	int i;
+
+	if (IS_ERR_OR_NULL(rx_chn->common.udmax))
+		return;
+
+	if (rx_chn->psil_paired) {
+		xudma_navss_psil_unpair(rx_chn->common.udmax,
+					rx_chn->common.src_thread,
+					rx_chn->common.dst_thread);
+		rx_chn->psil_paired = false;
+	}
+
+	for (i = 0; i < rx_chn->flow_num; i++)
+		k3_nav_udmax_release_rx_flow(rx_chn, i);
+
+	if (xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base))
+		xudma_free_gp_rflow_range(rx_chn->common.udmax,
+					  rx_chn->flow_id_base,
+					  rx_chn->flow_num);
+
+	if (!IS_ERR_OR_NULL(rx_chn->udma_rchanx))
+		xudma_rchan_put(rx_chn->common.udmax,
+				rx_chn->udma_rchanx);
+
+	xudma_dev_put(rx_chn->common.udmax);
+}
+EXPORT_SYMBOL_GPL(k3_nav_udmax_release_rx_chn);
+
+int k3_nav_udmax_rx_flow_init(struct k3_nav_udmax_rx_channel *rx_chn,
+			      u32 flow_idx,
+			      struct k3_nav_udmax_rx_flow_cfg *flow_cfg)
+{
+	if (flow_idx >= rx_chn->flow_num)
+		return -EINVAL;
+
+	return k3_nav_udmax_cfg_rx_flow(rx_chn, flow_idx, flow_cfg);
+}
+EXPORT_SYMBOL_GPL(k3_nav_udmax_rx_flow_init);
+
+u32 k3_nav_udmax_rx_flow_get_fdq_id(struct k3_nav_udmax_rx_channel *rx_chn,
+				    u32 flow_idx)
+{
+	struct k3_nav_udmax_rx_flow *flow;
+
+	if (flow_idx >= rx_chn->flow_num)
+		return -EINVAL;
+
+	flow = &rx_chn->flows[flow_idx];
+
+	return k3_ringacc_get_ring_id(flow->ringrxfdq);
+}
+EXPORT_SYMBOL_GPL(k3_nav_udmax_rx_flow_get_fdq_id);
+
+u32 k3_nav_udmax_rx_get_flow_id_base(struct k3_nav_udmax_rx_channel *rx_chn)
+{
+	return rx_chn->flow_id_base;
+}
+EXPORT_SYMBOL_GPL(k3_nav_udmax_rx_get_flow_id_base);
+
+int k3_nav_udmax_rx_flow_enable(struct k3_nav_udmax_rx_channel *rx_chn,
+				u32 flow_idx)
+{
+	struct k3_nav_udmax_rx_flow *flow = &rx_chn->flows[flow_idx];
+	const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
+	struct device *dev = rx_chn->common.dev;
+	struct ti_sci_msg_rm_udmap_flow_cfg req;
+	int rx_ring_id;
+	int rx_ringfdq_id;
+	int ret = 0;
+
+	if (!rx_chn->remote)
+		return -EINVAL;
+
+	rx_ring_id = k3_ringacc_get_ring_id(flow->ringrx);
+	rx_ringfdq_id = k3_ringacc_get_ring_id(flow->ringrxfdq);
+
+	memset(&req, 0, sizeof(req));
+
+	req.valid_params =
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
+	req.nav_id = tisci_rm->tisci_dev_id;
+	req.flow_index = flow->udma_rflow_id;
+	req.rx_dest_qnum = rx_ring_id;
+	req.rx_fdq0_sz0_qnum = rx_ringfdq_id;
+	req.rx_fdq1_qnum = rx_ringfdq_id;
+	req.rx_fdq2_qnum = rx_ringfdq_id;
+	req.rx_fdq3_qnum = rx_ringfdq_id;
+
+	ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req);
+	if (ret) {
+		dev_err(dev, "flow%d enable failed: %d\n", flow->udma_rflow_id,
+			ret);
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(k3_nav_udmax_rx_flow_enable);
+
+int k3_nav_udmax_rx_flow_disable(struct k3_nav_udmax_rx_channel *rx_chn,
+				 u32 flow_idx)
+{
+	struct k3_nav_udmax_rx_flow *flow = &rx_chn->flows[flow_idx];
+	const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
+	struct device *dev = rx_chn->common.dev;
+	struct ti_sci_msg_rm_udmap_flow_cfg req;
+	int ret = 0;
+
+	if (!rx_chn->remote)
+		return -EINVAL;
+
+	memset(&req, 0, sizeof(req));
+	req.valid_params =
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
+	req.nav_id = tisci_rm->tisci_dev_id;
+	req.flow_index = flow->udma_rflow_id;
+	req.rx_dest_qnum = TI_SCI_RESOURCE_NULL;
+	req.rx_fdq0_sz0_qnum = TI_SCI_RESOURCE_NULL;
+	req.rx_fdq1_qnum = TI_SCI_RESOURCE_NULL;
+	req.rx_fdq2_qnum = TI_SCI_RESOURCE_NULL;
+	req.rx_fdq3_qnum = TI_SCI_RESOURCE_NULL;
+
+	ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req);
+	if (ret) {
+		dev_err(dev, "flow%d disable failed: %d\n", flow->udma_rflow_id,
+			ret);
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(k3_nav_udmax_rx_flow_disable);
+
+int k3_nav_udmax_enable_rx_chn(struct k3_nav_udmax_rx_channel *rx_chn)
+{
+	u32 rxrt_ctl;
+
+	if (rx_chn->remote)
+		return -EINVAL;
+
+	if (rx_chn->flows_ready < rx_chn->flow_num)
+		return -EINVAL;
+
+	rxrt_ctl = xudma_rchanrt_read(rx_chn->udma_rchanx,
+				      UDMA_RCHAN_RT_CTL_REG);
+	rxrt_ctl |= UDMA_CHAN_RT_CTL_EN;
+	xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_RCHAN_RT_CTL_REG,
+			    rxrt_ctl);
+
+	xudma_rchanrt_write(rx_chn->udma_rchanx,
+			    UDMA_RCHAN_RT_PEER_RT_EN_REG,
+			    UDMA_PEER_RT_EN_ENABLE);
+
+	k3_nav_udmax_dump_rx_rt_chn(rx_chn, "rxrt en");
+	return 0;
+}
+EXPORT_SYMBOL_GPL(k3_nav_udmax_enable_rx_chn);
+
+void k3_nav_udmax_disable_rx_chn(struct k3_nav_udmax_rx_channel *rx_chn)
+{
+	if (rx_chn->remote)
+		return;
+
+	k3_nav_udmax_dump_rx_rt_chn(rx_chn, "rxrt dis1");
+
+	xudma_rchanrt_write(rx_chn->udma_rchanx,
+			    UDMA_RCHAN_RT_PEER_RT_EN_REG,
+			    0);
+	xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_RCHAN_RT_CTL_REG, 0);
+
+	k3_nav_udmax_dump_rx_rt_chn(rx_chn, "rxrt dis2");
+}
+EXPORT_SYMBOL_GPL(k3_nav_udmax_disable_rx_chn);
+
+void k3_nav_udmax_tdown_rx_chn(struct k3_nav_udmax_rx_channel *rx_chn,
+			       bool sync)
+{
+	int i = 0;
+	u32 val;
+
+	if (rx_chn->remote)
+		return;
+
+	k3_nav_udmax_dump_rx_rt_chn(rx_chn, "rxrt tdown1");
+
+	xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_RCHAN_RT_PEER_RT_EN_REG,
+			    UDMA_PEER_RT_EN_ENABLE | UDMA_PEER_RT_EN_TEARDOWN);
+
+	val = xudma_rchanrt_read(rx_chn->udma_rchanx, UDMA_RCHAN_RT_CTL_REG);
+
+	while (sync && (val & UDMA_CHAN_RT_CTL_EN)) {
+		val = xudma_rchanrt_read(rx_chn->udma_rchanx,
+					 UDMA_RCHAN_RT_CTL_REG);
+		udelay(1);
+		if (i > K3_UDMAX_TDOWN_TIMEOUT_US) {
+			dev_err(rx_chn->common.dev, "RX tdown timeout\n");
+			break;
+		}
+		i++;
+	}
+
+	val = xudma_rchanrt_read(rx_chn->udma_rchanx,
+				 UDMA_RCHAN_RT_PEER_RT_EN_REG);
+	if (sync && (val & UDMA_PEER_RT_EN_ENABLE))
+		dev_err(rx_chn->common.dev, "TX tdown peer not stopped\n");
+	k3_nav_udmax_dump_rx_rt_chn(rx_chn, "rxrt tdown2");
+}
+EXPORT_SYMBOL_GPL(k3_nav_udmax_tdown_rx_chn);
+
+void k3_nav_udmax_reset_rx_chn(struct k3_nav_udmax_rx_channel *rx_chn,
+			       u32 flow_num, void *data,
+			       void (*cleanup)(void *data, dma_addr_t desc_dma),
+			       bool skip_fdq)
+{
+	struct k3_nav_udmax_rx_flow *flow = &rx_chn->flows[flow_num];
+	struct device *dev = rx_chn->common.dev;
+	dma_addr_t desc_dma;
+	int occ_rx, i, ret;
+
+	/* reset RXCQ as it is not input for udma - expected to be empty */
+	occ_rx = k3_ringacc_ring_get_occ(flow->ringrx);
+	dev_dbg(dev, "RX reset flow %u occ_rx %u\n", flow_num, occ_rx);
+	if (flow->ringrx)
+		k3_ringacc_ring_reset(flow->ringrx);
+
+	/* Skip RX FDQ in case one FDQ is used for the set of flows */
+	if (skip_fdq)
+		return;
+
+	/*
+	 * RX FDQ reset need to be special way as it is input for udma and its
+	 * state cached by udma, so:
+	 * 1) save RX FDQ occ
+	 * 2) clean up RX FDQ and call callback .cleanup() for each desc
+	 * 3) reset RX FDQ in a special way
+	 */
+	occ_rx = k3_ringacc_ring_get_occ(flow->ringrxfdq);
+	dev_dbg(dev, "RX reset flow %u occ_rx_fdq %u\n", flow_num, occ_rx);
+
+	for (i = 0; i < occ_rx; i++) {
+		ret = k3_ringacc_ring_pop(flow->ringrxfdq, &desc_dma);
+		if (ret) {
+			dev_err(dev, "RX reset pop %d\n", ret);
+			break;
+		}
+		cleanup(data, desc_dma);
+	}
+
+	k3_ringacc_ring_reset_dma(flow->ringrxfdq, occ_rx);
+}
+EXPORT_SYMBOL_GPL(k3_nav_udmax_reset_rx_chn);
+
+int k3_nav_udmax_push_rx_chn(struct k3_nav_udmax_rx_channel *rx_chn,
+			     u32 flow_num, struct cppi5_host_desc_t *desc_rx,
+			     dma_addr_t desc_dma)
+{
+	struct k3_nav_udmax_rx_flow *flow = &rx_chn->flows[flow_num];
+
+	return k3_ringacc_ring_push(flow->ringrxfdq, &desc_dma);
+}
+EXPORT_SYMBOL_GPL(k3_nav_udmax_push_rx_chn);
+
+int k3_nav_udmax_pop_rx_chn(struct k3_nav_udmax_rx_channel *rx_chn,
+			    u32 flow_num, dma_addr_t *desc_dma)
+{
+	struct k3_nav_udmax_rx_flow *flow = &rx_chn->flows[flow_num];
+
+	return k3_ringacc_ring_pop(flow->ringrx, desc_dma);
+}
+EXPORT_SYMBOL_GPL(k3_nav_udmax_pop_rx_chn);
+
+int k3_nav_udmax_rx_get_irq(struct k3_nav_udmax_rx_channel *rx_chn,
+			    u32 flow_num,
+			    unsigned int *irq, u32 flags, bool share,
+			    u32 flow_num_share)
+{
+	struct k3_nav_udmax_rx_flow *flow, *flow_share;
+	unsigned int virq = 0;
+
+	if (flow_num >= rx_chn->flow_num ||
+	    (flow_num_share != -1 && flow_num_share >= rx_chn->flow_num))
+		return -EINVAL;
+
+	flow = &rx_chn->flows[flow_num];
+
+	if (share && flow_num_share != -1) {
+		flow_share = &rx_chn->flows[flow_num_share];
+		virq = flow_share->virq;
+	}
+
+	flow->virq = ti_sci_inta_register_event(rx_chn->common.dev,
+				k3_ringacc_get_tisci_dev_id(flow->ringrx),
+				k3_ringacc_get_ring_id(flow->ringrx), virq,
+				false, flags);
+	if (flow->virq <= 0)
+		return -ENODEV;
+
+	*irq = flow->virq;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(k3_nav_udmax_rx_get_irq);
+
+void k3_nav_udmax_rx_put_irq(struct k3_nav_udmax_rx_channel *rx_chn,
+			     u32 flow_num)
+{
+	struct k3_nav_udmax_rx_flow *flow;
+
+	if (flow_num >= rx_chn->flow_num)
+		return;
+
+	flow = &rx_chn->flows[flow_num];
+
+	if (flow->virq <= 0)
+		return;
+
+	ti_sci_inta_unregister_event(rx_chn->common.dev,
+				     k3_ringacc_get_tisci_dev_id(flow->ringrx),
+				     k3_ringacc_get_ring_id(flow->ringrx),
+				     flow->virq);
+}
+EXPORT_SYMBOL_GPL(k3_nav_udmax_rx_put_irq);
+
+MODULE_DESCRIPTION("TI K3 UDMA glue layer for non DMAengine clients");
+MODULE_AUTHOR("Grygorii Strashko <grygorii.strashko@ti.com>");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c
--- a/drivers/dma/ti/k3-udma.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/dma/ti/k3-udma.c	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,3677 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
+ *  Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/delay.h>
+#include <linux/dmaengine.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmapool.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/of.h>
+#include <linux/of_dma.h>
+#include <linux/of_device.h>
+#include <linux/of_irq.h>
+#include <linux/workqueue.h>
+#include <linux/completion.h>
+#include <dt-bindings/dma/k3-udma.h>
+#include <linux/soc/ti/k3-ringacc.h>
+#include <linux/soc/ti/ti_sci_protocol.h>
+#include <linux/irqchip/irq-ti-sci-inta.h>
+#include <linux/dma/ti-cppi5.h>
+
+#include "../virt-dma.h"
+#include "k3-udma.h"
+
+struct udma_static_tr {
+	u8 elsize; /* RPSTR0 */
+	u16 elcnt; /* RPSTR0 */
+	u16 bstcnt; /* RPSTR1 */
+};
+
+#define K3_UDMA_MAX_RFLOWS		1024
+#define K3_UDMA_DEFAULT_RING_SIZE	16
+
+struct udma_chan;
+
+enum udma_mmr {
+	MMR_GCFG = 0,
+	MMR_RCHANRT,
+	MMR_TCHANRT,
+	MMR_LAST,
+};
+
+static const char * const mmr_names[] = { "gcfg", "rchanrt", "tchanrt" };
+
+struct udma_tchan {
+	void __iomem *reg_rt;
+
+	int id;
+	struct k3_ring *t_ring; /* Transmit ring */
+	struct k3_ring *tc_ring; /* Transmit Completion ring */
+};
+
+struct udma_rchan {
+	void __iomem *reg_rt;
+
+	int id;
+	struct k3_ring *fd_ring; /* Free Descriptor ring */
+	struct k3_ring *r_ring; /* Receive ring*/
+};
+
+struct udma_rflow {
+	void __iomem *reg_rflow;
+
+	int id;
+};
+
+struct udma_match_data {
+	bool enable_memcpy_support;
+	bool have_acc32;
+	bool have_burst;
+	u32 statictr_z_mask;
+	u32 rchan_oes_offset;
+
+	u8 tpl_levels;
+	u32 level_start_idx[];
+};
+
+struct udma_hwdesc {
+	size_t cppi5_desc_size;
+	void *cppi5_desc_vaddr;
+	dma_addr_t cppi5_desc_paddr;
+
+	/* TR descriptor internal pointers */
+	void *tr_req_base;
+	struct cppi5_tr_resp_t *tr_resp_base;
+};
+
+struct udma_rx_flush {
+	struct udma_hwdesc hwdescs[2];
+
+	size_t buffer_size;
+	void *buffer_vaddr;
+	dma_addr_t buffer_paddr;
+};
+
+struct udma_dev {
+	struct dma_device ddev;
+	struct device *dev;
+	void __iomem *mmrs[MMR_LAST];
+	const struct udma_match_data *match_data;
+
+	size_t desc_align; /* alignment to use for descriptors */
+
+	struct udma_tisci_rm tisci_rm;
+
+	struct k3_ringacc *ringacc;
+
+	struct irq_domain *irq_domain;
+
+	struct work_struct purge_work;
+	struct list_head desc_to_purge;
+	spinlock_t lock;
+
+	struct udma_rx_flush rx_flush;
+
+	int tchan_cnt;
+	int echan_cnt;
+	int rchan_cnt;
+	int rflow_cnt;
+	unsigned long *tchan_map;
+	unsigned long *rchan_map;
+	unsigned long *rflow_gp_map;
+	unsigned long *rflow_gp_map_allocated;
+	unsigned long *rflow_in_use;
+
+	struct udma_tchan *tchans;
+	struct udma_rchan *rchans;
+	struct udma_rflow *rflows;
+
+	struct udma_chan *channels;
+	u32 psil_base;
+};
+
+struct udma_desc {
+	struct virt_dma_desc vd;
+
+	bool terminated;
+
+	enum dma_transfer_direction dir;
+
+	struct udma_static_tr static_tr;
+	u32 residue;
+
+	unsigned int sglen;
+	unsigned int desc_idx; /* Only used for cyclic in packet mode */
+	unsigned int tr_idx;
+
+	u32 metadata_size;
+	void *metadata; /* pointer to provided metadata buffer (EPIP, PSdata) */
+
+	unsigned int hwdesc_count;
+	struct udma_hwdesc hwdesc[0];
+};
+
+enum udma_chan_state {
+	UDMA_CHAN_IS_IDLE = 0, /* not active, no teardown is in progress */
+	UDMA_CHAN_IS_ACTIVE, /* Normal operation */
+	UDMA_CHAN_IS_TERMINATING, /* channel is being terminated */
+};
+
+struct udma_tx_drain {
+	struct delayed_work work;
+	ktime_t tstamp;
+	u32 residue;
+};
+
+struct udma_chan {
+	struct virt_dma_chan vc;
+	struct dma_slave_config	cfg;
+	struct udma_dev *ud;
+	struct udma_desc *desc;
+	struct udma_desc *terminated_desc;
+	struct udma_static_tr static_tr;
+	char *name;
+
+	struct udma_tchan *tchan;
+	struct udma_rchan *rchan;
+	struct udma_rflow *rflow;
+
+	bool psil_paired;
+	u32 irq_ra_tisci;
+	u32 irq_ra_idx;
+	u32 irq_udma_idx;
+
+	int irq_num_ring;
+	int irq_num_udma;
+
+	bool cyclic;
+	bool paused;
+
+	enum udma_chan_state state;
+	struct completion teardown_completed;
+
+	struct udma_tx_drain tx_drain;
+
+	u32 bcnt; /* number of bytes completed since the start of the channel */
+	u32 in_ring_cnt; /* number of descriptors in flight */
+
+	bool pkt_mode; /* TR or packet */
+	bool needs_epib; /* EPIB is needed for the communication or not */
+	u32 psd_size; /* size of Protocol Specific Data */
+	u32 metadata_size; /* (needs_epib ? 16:0) + psd_size */
+	u32 hdesc_size; /* Size of a packet descriptor in packet mode */
+	bool notdpkt; /* Suppress sending TDC packet */
+	int remote_thread_id;
+	u32 src_thread;
+	u32 dst_thread;
+	u32 static_tr_type;
+	bool enable_acc32;
+	bool enable_burst;
+	enum udma_tp_level channel_tpl; /* Channel Throughput Level */
+
+	/* dmapool for packet mode descriptors */
+	bool use_dma_pool;
+	struct dma_pool *hdesc_pool;
+
+	u32 id;
+	enum dma_transfer_direction dir;
+};
+
+static inline struct udma_dev *to_udma_dev(struct dma_device *d)
+{
+	return container_of(d, struct udma_dev, ddev);
+}
+
+static inline struct udma_chan *to_udma_chan(struct dma_chan *c)
+{
+	return container_of(c, struct udma_chan, vc.chan);
+}
+
+static inline struct udma_desc *to_udma_desc(struct dma_async_tx_descriptor *t)
+{
+	return container_of(t, struct udma_desc, vd.tx);
+}
+
+/* Generic register access functions */
+static inline u32 udma_read(void __iomem *base, int reg)
+{
+	return __raw_readl(base + reg);
+}
+
+static inline void udma_write(void __iomem *base, int reg, u32 val)
+{
+	__raw_writel(val, base + reg);
+}
+
+static inline void udma_update_bits(void __iomem *base, int reg,
+				    u32 mask, u32 val)
+{
+	u32 tmp, orig;
+
+	orig = __raw_readl(base + reg);
+	tmp = orig & ~mask;
+	tmp |= (val & mask);
+
+	if (tmp != orig)
+		__raw_writel(tmp, base + reg);
+}
+
+/* TCHANRT */
+static inline u32 udma_tchanrt_read(struct udma_tchan *tchan, int reg)
+{
+	if (!tchan)
+		return 0;
+	return udma_read(tchan->reg_rt, reg);
+}
+
+static inline void udma_tchanrt_write(struct udma_tchan *tchan, int reg,
+				      u32 val)
+{
+	if (!tchan)
+		return;
+	udma_write(tchan->reg_rt, reg, val);
+}
+
+static inline void udma_tchanrt_update_bits(struct udma_tchan *tchan, int reg,
+					    u32 mask, u32 val)
+{
+	if (!tchan)
+		return;
+	udma_update_bits(tchan->reg_rt, reg, mask, val);
+}
+
+/* RCHANRT */
+static inline u32 udma_rchanrt_read(struct udma_rchan *rchan, int reg)
+{
+	if (!rchan)
+		return 0;
+	return udma_read(rchan->reg_rt, reg);
+}
+
+static inline void udma_rchanrt_write(struct udma_rchan *rchan, int reg,
+				      u32 val)
+{
+	if (!rchan)
+		return;
+	udma_write(rchan->reg_rt, reg, val);
+}
+
+static inline void udma_rchanrt_update_bits(struct udma_rchan *rchan, int reg,
+					    u32 mask, u32 val)
+{
+	if (!rchan)
+		return;
+	udma_update_bits(rchan->reg_rt, reg, mask, val);
+}
+
+static inline int navss_psil_pair(struct udma_dev *ud, u32 src_thread,
+				  u32 dst_thread)
+{
+	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
+
+	dst_thread |= UDMA_PSIL_DST_THREAD_ID_OFFSET;
+	return tisci_rm->tisci_psil_ops->pair(tisci_rm->tisci,
+					      tisci_rm->tisci_navss_dev_id,
+					      src_thread, dst_thread);
+}
+
+static inline int navss_psil_unpair(struct udma_dev *ud, u32 src_thread,
+				    u32 dst_thread)
+{
+	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
+
+	dst_thread |= UDMA_PSIL_DST_THREAD_ID_OFFSET;
+	return tisci_rm->tisci_psil_ops->unpair(tisci_rm->tisci,
+						tisci_rm->tisci_navss_dev_id,
+						src_thread, dst_thread);
+}
+
+static inline char *udma_get_dir_text(enum dma_transfer_direction dir)
+{
+	switch (dir) {
+	case DMA_DEV_TO_MEM:
+		return "DEV_TO_MEM";
+	case DMA_MEM_TO_DEV:
+		return "MEM_TO_DEV";
+	case DMA_MEM_TO_MEM:
+		return "MEM_TO_MEM";
+	case DMA_DEV_TO_DEV:
+		return "DEV_TO_DEV";
+	default:
+		break;
+	}
+
+	return "invalid";
+}
+
+static void udma_reset_uchan(struct udma_chan *uc)
+{
+	uc->state = UDMA_CHAN_IS_IDLE;
+	uc->remote_thread_id = -1;
+	uc->dir = DMA_MEM_TO_MEM;
+	uc->pkt_mode = false;
+	uc->static_tr_type = 0;
+	uc->enable_acc32 = 0;
+	uc->enable_burst = 0;
+	uc->channel_tpl = 0;
+	uc->psd_size = 0;
+	uc->metadata_size = 0;
+	uc->hdesc_size = 0;
+	uc->notdpkt = 0;
+}
+
+static inline void udma_dump_chan_stdata(struct udma_chan *uc)
+{
+	struct device *dev = uc->ud->dev;
+	u32 offset;
+	int i;
+
+	if (uc->dir == DMA_MEM_TO_DEV || uc->dir == DMA_MEM_TO_MEM) {
+		dev_dbg(dev, "TCHAN State data:\n");
+		for (i = 0; i < 32; i++) {
+			offset = UDMA_TCHAN_RT_STDATA_REG + i * 4;
+			dev_dbg(dev, "TRT_STDATA[%02d]: 0x%08x\n", i,
+				udma_tchanrt_read(uc->tchan, offset));
+		}
+	}
+
+	if (uc->dir == DMA_DEV_TO_MEM || uc->dir == DMA_MEM_TO_MEM) {
+		dev_dbg(dev, "RCHAN State data:\n");
+		for (i = 0; i < 32; i++) {
+			offset = UDMA_RCHAN_RT_STDATA_REG + i * 4;
+			dev_dbg(dev, "RRT_STDATA[%02d]: 0x%08x\n", i,
+				udma_rchanrt_read(uc->rchan, offset));
+		}
+	}
+}
+
+static inline dma_addr_t udma_curr_cppi5_desc_paddr(struct udma_desc *d,
+						    int idx)
+{
+	return d->hwdesc[idx].cppi5_desc_paddr;
+}
+
+static inline void *udma_curr_cppi5_desc_vaddr(struct udma_desc *d, int idx)
+{
+	return d->hwdesc[idx].cppi5_desc_vaddr;
+}
+
+static inline struct udma_desc *udma_udma_desc_from_paddr(struct udma_chan *uc,
+							  dma_addr_t paddr)
+{
+	struct udma_desc *d = uc->terminated_desc;
+
+	if (d) {
+		dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d,
+								   d->desc_idx);
+
+		if (desc_paddr != paddr)
+			d = NULL;
+	}
+
+	if (!d) {
+		d = uc->desc;
+		if (d) {
+			dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d,
+								d->desc_idx);
+
+			if (desc_paddr != paddr)
+				d = NULL;
+		}
+	}
+
+	return d;
+}
+
+static void udma_free_hwdesc(struct udma_chan *uc, struct udma_desc *d)
+{
+	if (uc->use_dma_pool) {
+		int i;
+
+		for (i = 0; i < d->hwdesc_count; i++) {
+			if (!d->hwdesc[i].cppi5_desc_vaddr)
+				continue;
+
+			dma_pool_free(uc->hdesc_pool,
+				      d->hwdesc[i].cppi5_desc_vaddr,
+				      d->hwdesc[i].cppi5_desc_paddr);
+
+			d->hwdesc[i].cppi5_desc_vaddr = NULL;
+		}
+	} else if (d->hwdesc[0].cppi5_desc_vaddr) {
+		struct udma_dev *ud = uc->ud;
+
+		dma_free_coherent(ud->dev, d->hwdesc[0].cppi5_desc_size,
+				  d->hwdesc[0].cppi5_desc_vaddr,
+				  d->hwdesc[0].cppi5_desc_paddr);
+
+		d->hwdesc[0].cppi5_desc_vaddr = NULL;
+	}
+}
+
+static void udma_purge_desc_work(struct work_struct *work)
+{
+	struct udma_dev *ud = container_of(work, typeof(*ud), purge_work);
+	struct virt_dma_desc *vd, *_vd;
+	unsigned long flags;
+	LIST_HEAD(head);
+
+	spin_lock_irqsave(&ud->lock, flags);
+	list_splice_tail_init(&ud->desc_to_purge, &head);
+	spin_unlock_irqrestore(&ud->lock, flags);
+
+	list_for_each_entry_safe(vd, _vd, &head, node) {
+		struct udma_chan *uc = to_udma_chan(vd->tx.chan);
+		struct udma_desc *d = to_udma_desc(&vd->tx);
+
+		udma_free_hwdesc(uc, d);
+		list_del(&vd->node);
+		kfree(d);
+	}
+
+	/* If more to purge, schedule the work again */
+	if (!list_empty(&ud->desc_to_purge))
+		schedule_work(&ud->purge_work);
+}
+
+static void udma_desc_free(struct virt_dma_desc *vd)
+{
+	struct udma_dev *ud = to_udma_dev(vd->tx.chan->device);
+	struct udma_chan *uc = to_udma_chan(vd->tx.chan);
+	struct udma_desc *d = to_udma_desc(&vd->tx);
+	unsigned long flags;
+
+	if (uc->terminated_desc == d)
+		uc->terminated_desc = NULL;
+
+	if (uc->use_dma_pool) {
+		udma_free_hwdesc(uc, d);
+		kfree(d);
+		return;
+	}
+
+	spin_lock_irqsave(&ud->lock, flags);
+	list_add_tail(&vd->node, &ud->desc_to_purge);
+	spin_unlock_irqrestore(&ud->lock, flags);
+
+	schedule_work(&ud->purge_work);
+}
+
+static inline bool udma_is_chan_running(struct udma_chan *uc)
+{
+	u32 trt_ctl = 0;
+	u32 rrt_ctl = 0;
+
+	switch (uc->dir) {
+	case DMA_DEV_TO_MEM:
+		rrt_ctl = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_CTL_REG);
+		break;
+	case DMA_MEM_TO_DEV:
+		trt_ctl = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_CTL_REG);
+		break;
+	case DMA_MEM_TO_MEM:
+		trt_ctl = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_CTL_REG);
+		rrt_ctl = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_CTL_REG);
+		break;
+	default:
+		break;
+	}
+
+	if (trt_ctl & UDMA_CHAN_RT_CTL_EN || rrt_ctl & UDMA_CHAN_RT_CTL_EN)
+		return true;
+
+	return false;
+}
+
+static void udma_sync_for_device(struct udma_chan *uc, int idx)
+{
+	struct udma_desc *d = uc->desc;
+
+	if (uc->cyclic && uc->pkt_mode) {
+		dma_sync_single_for_device(uc->ud->dev,
+					   d->hwdesc[idx].cppi5_desc_paddr,
+					   d->hwdesc[idx].cppi5_desc_size,
+					   DMA_TO_DEVICE);
+	} else {
+		int i;
+
+		for (i = 0; i < d->hwdesc_count; i++) {
+			if (!d->hwdesc[i].cppi5_desc_vaddr)
+				continue;
+
+			dma_sync_single_for_device(uc->ud->dev,
+						d->hwdesc[i].cppi5_desc_paddr,
+						d->hwdesc[i].cppi5_desc_size,
+						DMA_TO_DEVICE);
+		}
+	}
+}
+
+static inline dma_addr_t udma_get_rx_flush_hwdesc_paddr(struct udma_chan *uc)
+{
+	return uc->ud->rx_flush.hwdescs[uc->pkt_mode].cppi5_desc_paddr;
+}
+
+static int udma_push_to_ring(struct udma_chan *uc, int idx)
+{
+	struct udma_desc *d = uc->desc;
+	struct k3_ring *ring = NULL;
+	dma_addr_t paddr;
+	int ret;
+
+	switch (uc->dir) {
+	case DMA_DEV_TO_MEM:
+		ring = uc->rchan->fd_ring;
+		break;
+	case DMA_MEM_TO_DEV:
+		ring = uc->tchan->t_ring;
+		break;
+	case DMA_MEM_TO_MEM:
+		ring = uc->tchan->t_ring;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* RX flush packet: idx == -1 is only passed in case of DEV_TO_MEM */
+	if (idx == -1) {
+		paddr = udma_get_rx_flush_hwdesc_paddr(uc);
+	} else {
+		paddr = udma_curr_cppi5_desc_paddr(d, idx);
+
+		wmb(); /* Ensure that writes are not moved over this point */
+		udma_sync_for_device(uc, idx);
+	}
+
+	ret = k3_ringacc_ring_push(ring, &paddr);
+	if (!ret)
+		uc->in_ring_cnt++;
+
+	return ret;
+}
+
+static bool udma_desc_is_rx_flush(struct udma_chan *uc, dma_addr_t addr)
+{
+	if (uc->dir != DMA_DEV_TO_MEM)
+		return false;
+
+	if (addr == udma_get_rx_flush_hwdesc_paddr(uc))
+		return true;
+
+	return false;
+}
+
+static int udma_pop_from_ring(struct udma_chan *uc, dma_addr_t *addr)
+{
+	struct k3_ring *ring = NULL;
+	int ret = -ENOENT;
+
+	switch (uc->dir) {
+	case DMA_DEV_TO_MEM:
+		ring = uc->rchan->r_ring;
+		break;
+	case DMA_MEM_TO_DEV:
+		ring = uc->tchan->tc_ring;
+		break;
+	case DMA_MEM_TO_MEM:
+		ring = uc->tchan->tc_ring;
+		break;
+	default:
+		break;
+	}
+
+	if (ring && k3_ringacc_ring_get_occ(ring)) {
+		struct udma_desc *d = NULL;
+
+		ret = k3_ringacc_ring_pop(ring, addr);
+		if (ret)
+			return ret;
+
+		/* Teardown completion */
+		if (*addr & 0x1)
+			return ret;
+
+		/* Check for flush descriptor */
+		if (udma_desc_is_rx_flush(uc, *addr))
+			return -ENOENT;
+
+		d = udma_udma_desc_from_paddr(uc, *addr);
+
+		if (d)
+			dma_sync_single_for_cpu(uc->ud->dev, *addr,
+						d->hwdesc[0].cppi5_desc_size,
+						DMA_FROM_DEVICE);
+		rmb(); /* Ensure that reads are not moved before this point */
+
+		if (!ret)
+			uc->in_ring_cnt--;
+	}
+
+	return ret;
+}
+
+static void udma_reset_rings(struct udma_chan *uc)
+{
+	struct k3_ring *ring1 = NULL;
+	struct k3_ring *ring2 = NULL;
+
+	switch (uc->dir) {
+	case DMA_DEV_TO_MEM:
+		if (uc->rchan) {
+			ring1 = uc->rchan->fd_ring;
+			ring2 = uc->rchan->r_ring;
+		}
+		break;
+	case DMA_MEM_TO_DEV:
+		if (uc->tchan) {
+			ring1 = uc->tchan->t_ring;
+			ring2 = uc->tchan->tc_ring;
+		}
+		break;
+	case DMA_MEM_TO_MEM:
+		if (uc->tchan) {
+			ring1 = uc->tchan->t_ring;
+			ring2 = uc->tchan->tc_ring;
+		}
+		break;
+	default:
+		break;
+	}
+
+	if (ring1)
+		k3_ringacc_ring_reset_dma(ring1,
+					  k3_ringacc_ring_get_occ(ring1));
+	if (ring2)
+		k3_ringacc_ring_reset(ring2);
+
+	/* make sure we are not leaking memory by stalled descriptor */
+	if (uc->terminated_desc) {
+		udma_desc_free(&uc->terminated_desc->vd);
+		uc->terminated_desc = NULL;
+	}
+
+	uc->in_ring_cnt = 0;
+}
+
+static inline void udma_reset_counters(struct udma_chan *uc)
+{
+	u32 val;
+
+	if (uc->tchan) {
+		val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_BCNT_REG);
+		udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_BCNT_REG, val);
+
+		val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_SBCNT_REG);
+		udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_SBCNT_REG, val);
+
+		val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_PCNT_REG);
+		udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PCNT_REG, val);
+
+		val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_PEER_BCNT_REG);
+		udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_BCNT_REG, val);
+	}
+
+	if (uc->rchan) {
+		val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_BCNT_REG);
+		udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_BCNT_REG, val);
+
+		val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_SBCNT_REG);
+		udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_SBCNT_REG, val);
+
+		val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_PCNT_REG);
+		udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PCNT_REG, val);
+
+		val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_PEER_BCNT_REG);
+		udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_BCNT_REG, val);
+	}
+
+	uc->bcnt = 0;
+}
+
+static inline int udma_reset_chan(struct udma_chan *uc, bool hard)
+{
+	switch (uc->dir) {
+	case DMA_DEV_TO_MEM:
+		udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG, 0);
+		udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 0);
+		break;
+	case DMA_MEM_TO_DEV:
+		udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 0);
+		udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_RT_EN_REG, 0);
+		break;
+	case DMA_MEM_TO_MEM:
+		udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 0);
+		udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 0);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* Reset all counters */
+	udma_reset_counters(uc);
+
+	/* Hard reset: re-initialize the channel to reset */
+	if (hard) {
+		struct udma_chan uc_backup = *uc;
+		int ret;
+
+		uc->ud->ddev.device_free_chan_resources(&uc->vc.chan);
+		/* restore the channel configuration */
+		uc->dir = uc_backup.dir;
+		uc->remote_thread_id = uc_backup.remote_thread_id;
+		uc->pkt_mode = uc_backup.pkt_mode;
+		uc->static_tr_type = uc_backup.static_tr_type;
+		uc->enable_acc32 = uc_backup.enable_acc32;
+		uc->enable_burst = uc_backup.enable_burst;
+		uc->channel_tpl = uc_backup.channel_tpl;
+		uc->psd_size = uc_backup.psd_size;
+		uc->metadata_size = uc_backup.metadata_size;
+		uc->hdesc_size = uc_backup.hdesc_size;
+		uc->notdpkt = uc_backup.notdpkt;
+
+		ret = uc->ud->ddev.device_alloc_chan_resources(&uc->vc.chan);
+		if (ret)
+			return ret;
+	}
+	uc->state = UDMA_CHAN_IS_IDLE;
+
+	return 0;
+}
+
+static inline void udma_start_desc(struct udma_chan *uc)
+{
+	if (uc->pkt_mode && (uc->cyclic || uc->dir == DMA_DEV_TO_MEM)) {
+		int i;
+
+		/* Push all descriptors to ring for packet mode cyclic or RX */
+		for (i = 0; i < uc->desc->sglen; i++)
+			udma_push_to_ring(uc, i);
+	} else {
+		udma_push_to_ring(uc, 0);
+	}
+}
+
+static inline bool udma_chan_needs_reconfiguration(struct udma_chan *uc)
+{
+	/* Only PDMAs have staticTR */
+	if (!uc->static_tr_type)
+		return false;
+
+	/* Check if the staticTR configuration has changed for TX */
+	if (memcmp(&uc->static_tr, &uc->desc->static_tr, sizeof(uc->static_tr)))
+		return true;
+
+	return false;
+}
+
+static int udma_start(struct udma_chan *uc)
+{
+	const struct udma_match_data *match_data = uc->ud->match_data;
+	struct virt_dma_desc *vd = vchan_next_desc(&uc->vc);
+
+	if (!vd) {
+		uc->desc = NULL;
+		return -ENOENT;
+	}
+
+	list_del(&vd->node);
+
+	uc->desc = to_udma_desc(&vd->tx);
+
+	/* Channel is already running and does not need reconfiguration */
+	if (udma_is_chan_running(uc) && !udma_chan_needs_reconfiguration(uc)) {
+		udma_start_desc(uc);
+		goto out;
+	}
+
+	/* Make sure that we clear the teardown bit, if it is set */
+	udma_reset_chan(uc, false);
+
+	/* Push descriptors before we start the channel */
+	udma_start_desc(uc);
+
+	switch (uc->desc->dir) {
+	case DMA_DEV_TO_MEM:
+		/* Config remote TR */
+		if (uc->static_tr_type) {
+			u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) |
+				  PDMA_STATIC_TR_X(uc->desc->static_tr.elsize);
+
+			if (uc->enable_acc32 && match_data->have_acc32)
+				val |= PDMA_STATIC_TR_XY_ACC32;
+			if (uc->enable_burst && match_data->have_burst)
+				val |= PDMA_STATIC_TR_XY_BURST;
+
+			udma_rchanrt_write(uc->rchan,
+				UDMA_RCHAN_RT_PEER_STATIC_TR_XY_REG, val);
+
+			udma_rchanrt_write(uc->rchan,
+				UDMA_RCHAN_RT_PEER_STATIC_TR_Z_REG,
+				PDMA_STATIC_TR_Z(uc->desc->static_tr.bstcnt,
+						 match_data->statictr_z_mask));
+
+			/* save the current staticTR configuration */
+			memcpy(&uc->static_tr, &uc->desc->static_tr,
+			       sizeof(uc->static_tr));
+		}
+
+		udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG,
+				   UDMA_CHAN_RT_CTL_EN);
+
+		/* Enable remote */
+		udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG,
+				   UDMA_PEER_RT_EN_ENABLE);
+
+		break;
+	case DMA_MEM_TO_DEV:
+		/* Config remote TR */
+		if (uc->static_tr_type) {
+			u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) |
+				  PDMA_STATIC_TR_X(uc->desc->static_tr.elsize);
+
+			if (uc->enable_acc32 && match_data->have_acc32)
+				val |= PDMA_STATIC_TR_XY_ACC32;
+			if (uc->enable_burst && match_data->have_burst)
+				val |= PDMA_STATIC_TR_XY_BURST;
+
+			udma_tchanrt_write(uc->tchan,
+				UDMA_TCHAN_RT_PEER_STATIC_TR_XY_REG, val);
+
+			/* save the current staticTR configuration */
+			memcpy(&uc->static_tr, &uc->desc->static_tr,
+			       sizeof(uc->static_tr));
+		}
+
+		/* Enable remote */
+		udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_RT_EN_REG,
+				   UDMA_PEER_RT_EN_ENABLE);
+
+		udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG,
+				   UDMA_CHAN_RT_CTL_EN);
+
+		break;
+	case DMA_MEM_TO_MEM:
+		udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG,
+				   UDMA_CHAN_RT_CTL_EN);
+		udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG,
+				   UDMA_CHAN_RT_CTL_EN);
+
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	uc->state = UDMA_CHAN_IS_ACTIVE;
+out:
+
+	return 0;
+}
+
+static inline int udma_stop(struct udma_chan *uc)
+{
+	enum udma_chan_state old_state = uc->state;
+
+	uc->state = UDMA_CHAN_IS_TERMINATING;
+	reinit_completion(&uc->teardown_completed);
+
+	switch (uc->dir) {
+	case DMA_DEV_TO_MEM:
+		if (!uc->cyclic && !uc->desc)
+			udma_push_to_ring(uc, -1);
+
+		udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG,
+				   UDMA_PEER_RT_EN_ENABLE |
+				   UDMA_PEER_RT_EN_TEARDOWN);
+		break;
+	case DMA_MEM_TO_DEV:
+		udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_RT_EN_REG,
+				   UDMA_PEER_RT_EN_ENABLE |
+				   UDMA_PEER_RT_EN_FLUSH);
+		udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG,
+				   UDMA_CHAN_RT_CTL_EN |
+				   UDMA_CHAN_RT_CTL_TDOWN);
+		break;
+	case DMA_MEM_TO_MEM:
+		udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG,
+				   UDMA_CHAN_RT_CTL_EN |
+				   UDMA_CHAN_RT_CTL_TDOWN);
+		break;
+	default:
+		uc->state = old_state;
+		complete_all(&uc->teardown_completed);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void udma_cyclic_packet_elapsed(struct udma_chan *uc)
+{
+	struct udma_desc *d = uc->desc;
+	struct cppi5_host_desc_t *h_desc;
+
+	h_desc = d->hwdesc[d->desc_idx].cppi5_desc_vaddr;
+	cppi5_hdesc_reset_to_original(h_desc);
+	udma_push_to_ring(uc, d->desc_idx);
+	d->desc_idx = (d->desc_idx + 1) % d->sglen;
+}
+
+static inline void udma_fetch_epib(struct udma_chan *uc, struct udma_desc *d)
+{
+	struct cppi5_host_desc_t *h_desc = d->hwdesc[0].cppi5_desc_vaddr;
+
+	memcpy(d->metadata, h_desc->epib, d->metadata_size);
+}
+
+static inline bool udma_is_desc_really_done(struct udma_chan *uc,
+					    struct udma_desc *d)
+{
+	u32 peer_bcnt, bcnt;
+
+	/* Only TX towards PDMA is affected */
+	if (!uc->static_tr_type || uc->dir != DMA_MEM_TO_DEV)
+		return true;
+
+	peer_bcnt = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_PEER_BCNT_REG);
+	bcnt = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_BCNT_REG);
+
+	/* Transfer is incomplete, store current residue and time stamp */
+	if (peer_bcnt < bcnt) {
+		uc->tx_drain.residue = bcnt - peer_bcnt;
+		uc->tx_drain.tstamp = ktime_get();
+		return false;
+	}
+
+	return true;
+}
+
+static void udma_check_tx_completion(struct work_struct *work)
+{
+	struct udma_chan *uc = container_of(work, typeof(*uc),
+					    tx_drain.work.work);
+	bool desc_done = true;
+	u32 residue_diff;
+	ktime_t time_diff;
+	unsigned long delay;
+
+	while (1) {
+		if (uc->desc) {
+			/* Get previous residue and time stamp */
+			residue_diff = uc->tx_drain.residue;
+			time_diff = uc->tx_drain.tstamp;
+			/*
+			 * Get current residue and time stamp or see if
+			 * transfer is complete
+			 */
+			desc_done = udma_is_desc_really_done(uc, uc->desc);
+		}
+
+		if (!desc_done) {
+			/*
+			 * Find the time delta and residue delta w.r.t
+			 * previous poll
+			 */
+			time_diff = ktime_sub(uc->tx_drain.tstamp,
+					      time_diff) + 1;
+			residue_diff -= uc->tx_drain.residue;
+			if (residue_diff) {
+				/*
+				 * Try to guess when we should check
+				 * next time by calculating rate at
+				 * which data is being drained at the
+				 * peer device
+				 */
+				delay = (time_diff / residue_diff) *
+					uc->tx_drain.residue;
+			} else {
+				/* No progress, check again in 1 second  */
+				schedule_delayed_work(&uc->tx_drain.work, HZ);
+				break;
+			}
+
+			usleep_range(ktime_to_us(delay),
+				     ktime_to_us(delay) + 10);
+			continue;
+		}
+
+		if (uc->desc) {
+			struct udma_desc *d = uc->desc;
+
+			uc->bcnt += d->residue;
+			udma_start(uc);
+			vchan_cookie_complete(&d->vd);
+			break;
+		}
+
+		break;
+	}
+}
+
+static void udma_ring_callback(struct udma_chan *uc, dma_addr_t paddr)
+{
+	struct udma_desc *d;
+	unsigned long flags;
+
+	if (!paddr)
+		return;
+
+	spin_lock_irqsave(&uc->vc.lock, flags);
+
+	/* Teardown completion message */
+	if (paddr & 0x1) {
+		/* Compensate our internal pop/push counter */
+		uc->in_ring_cnt++;
+
+		complete_all(&uc->teardown_completed);
+
+		if (uc->terminated_desc) {
+			udma_desc_free(&uc->terminated_desc->vd);
+			uc->terminated_desc = NULL;
+		}
+
+		if (!uc->desc)
+			udma_start(uc);
+
+		goto out;
+	}
+
+	d = udma_udma_desc_from_paddr(uc, paddr);
+
+	if (d) {
+		dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d,
+								   d->desc_idx);
+		if (desc_paddr != paddr) {
+			dev_err(uc->ud->dev, "not matching descriptors!\n");
+			goto out;
+		}
+
+		if (uc->cyclic) {
+			/* push the descriptor back to the ring */
+			if (d == uc->desc) {
+				udma_cyclic_packet_elapsed(uc);
+				vchan_cyclic_callback(&d->vd);
+			}
+		} else {
+			bool desc_done = false;
+
+			if (d == uc->desc) {
+				desc_done = udma_is_desc_really_done(uc, d);
+
+				if (desc_done) {
+					uc->bcnt += d->residue;
+					udma_start(uc);
+				} else {
+					schedule_delayed_work(&uc->tx_drain.work,
+							      0);
+				}
+			}
+
+			if (desc_done)
+				vchan_cookie_complete(&d->vd);
+		}
+	}
+out:
+	spin_unlock_irqrestore(&uc->vc.lock, flags);
+}
+
+static void udma_tr_event_callback(struct udma_chan *uc)
+{
+	struct udma_desc *d;
+	unsigned long flags;
+
+	spin_lock_irqsave(&uc->vc.lock, flags);
+	d = uc->desc;
+	if (d) {
+		d->tr_idx = (d->tr_idx + 1) % d->sglen;
+
+		if (uc->cyclic) {
+			vchan_cyclic_callback(&d->vd);
+		} else {
+			/* TODO: figure out the real amount of data */
+			uc->bcnt += d->residue;
+			udma_start(uc);
+			vchan_cookie_complete(&d->vd);
+		}
+	}
+
+	spin_unlock_irqrestore(&uc->vc.lock, flags);
+}
+
+static irqreturn_t udma_ring_irq_handler(int irq, void *data)
+{
+	struct udma_chan *uc = data;
+	dma_addr_t paddr = 0;
+
+	if (!udma_pop_from_ring(uc, &paddr))
+		udma_ring_callback(uc, paddr);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t udma_udma_irq_handler(int irq, void *data)
+{
+	struct udma_chan *uc = data;
+
+	udma_tr_event_callback(uc);
+
+	return IRQ_HANDLED;
+}
+
+/**
+ * __udma_alloc_gp_rflow_range - alloc range of GP RX flows
+ * @ud: UDMA device
+ * @from: Start the search from this flow id number
+ * @cnt: Number of consecutive flow ids to allocate
+ *
+ * Allocate range of RX flow ids for future use, those flows can be requested
+ * only using explicit flow id number. if @from is set to -1 it will try to find
+ * first free range. if @from is positive value it will force allocation only
+ * of the specified range of flows.
+ *
+ * Returns -ENOMEM if can't find free range.
+ * -EEXIST if requested range is busy.
+ * -EINVAL if wrong input values passed.
+ * Returns flow id on success.
+ */
+static int __udma_alloc_gp_rflow_range(struct udma_dev *ud, int from, int cnt)
+{
+	int start, tmp_from;
+	DECLARE_BITMAP(tmp, K3_UDMA_MAX_RFLOWS);
+
+	tmp_from = from;
+	if (tmp_from < 0)
+		tmp_from = ud->rchan_cnt;
+	/* default flows can't be allocated and accessible only by id */
+	if (tmp_from < ud->rchan_cnt)
+		return -EINVAL;
+
+	if (tmp_from + cnt > ud->rflow_cnt)
+		return -EINVAL;
+
+	bitmap_or(tmp, ud->rflow_gp_map, ud->rflow_gp_map_allocated,
+		  ud->rflow_cnt);
+
+	start = bitmap_find_next_zero_area(tmp,
+					   ud->rflow_cnt,
+					   tmp_from, cnt, 0);
+	if (start >= ud->rflow_cnt)
+		return -ENOMEM;
+
+	if (from >= 0 && start != from)
+		return -EEXIST;
+
+	bitmap_set(ud->rflow_gp_map_allocated, start, cnt);
+	return start;
+}
+
+static int __udma_free_gp_rflow_range(struct udma_dev *ud, int from, int cnt)
+{
+	if (from < ud->rchan_cnt)
+		return -EINVAL;
+	if (from + cnt > ud->rflow_cnt)
+		return -EINVAL;
+
+	bitmap_clear(ud->rflow_gp_map_allocated, from, cnt);
+	return 0;
+}
+
+static struct udma_rflow *__udma_get_rflow(struct udma_dev *ud, int id)
+{
+	/*
+	 * Attempt to request rflow by ID can be made for any rflow
+	 * if not in use with assumption that caller knows what's doing.
+	 * TI-SCI FW will perform additional permission check ant way, it's
+	 * safe
+	 */
+
+	if (id < 0 || id >= ud->rflow_cnt)
+		return ERR_PTR(-ENOENT);
+
+	if (test_bit(id, ud->rflow_in_use))
+		return ERR_PTR(-ENOENT);
+
+	/* GP rflow has to be allocated first */
+	if (!test_bit(id, ud->rflow_gp_map) &&
+	    !test_bit(id, ud->rflow_gp_map_allocated))
+		return ERR_PTR(-EINVAL);
+
+	dev_dbg(ud->dev, "get rflow%d\n", id);
+	set_bit(id, ud->rflow_in_use);
+	return &ud->rflows[id];
+}
+
+static void __udma_put_rflow(struct udma_dev *ud, struct udma_rflow *rflow)
+{
+	if (!test_bit(rflow->id, ud->rflow_in_use)) {
+		dev_err(ud->dev, "attempt to put unused rflow%d\n", rflow->id);
+		return;
+	}
+
+	dev_dbg(ud->dev, "put rflow%d\n", rflow->id);
+	clear_bit(rflow->id, ud->rflow_in_use);
+}
+
+#define UDMA_RESERVE_RESOURCE(res)					\
+static struct udma_##res *__udma_reserve_##res(struct udma_dev *ud,	\
+					       enum udma_tp_level tpl,	\
+					       int id)			\
+{									\
+	if (id >= 0) {							\
+		if (test_bit(id, ud->res##_map)) {			\
+			dev_err(ud->dev, "res##%d is in use\n", id);	\
+			return ERR_PTR(-ENOENT);			\
+		}							\
+	} else {							\
+		int start;						\
+									\
+		if (tpl >= ud->match_data->tpl_levels)			\
+			tpl = ud->match_data->tpl_levels - 1;		\
+									\
+		start = ud->match_data->level_start_idx[tpl];		\
+									\
+		id = find_next_zero_bit(ud->res##_map, ud->res##_cnt,	\
+					start);				\
+		if (id == ud->res##_cnt) {				\
+			return ERR_PTR(-ENOENT);			\
+		}							\
+	}								\
+									\
+	set_bit(id, ud->res##_map);					\
+	return &ud->res##s[id];						\
+}
+
+UDMA_RESERVE_RESOURCE(tchan);
+UDMA_RESERVE_RESOURCE(rchan);
+
+static int udma_get_tchan(struct udma_chan *uc)
+{
+	struct udma_dev *ud = uc->ud;
+
+	if (uc->tchan) {
+		dev_dbg(ud->dev, "chan%d: already have tchan%d allocated\n",
+			uc->id, uc->tchan->id);
+		return 0;
+	}
+
+	uc->tchan = __udma_reserve_tchan(ud, uc->channel_tpl, -1);
+	if (IS_ERR(uc->tchan))
+		return PTR_ERR(uc->tchan);
+
+	return 0;
+}
+
+static int udma_get_rchan(struct udma_chan *uc)
+{
+	struct udma_dev *ud = uc->ud;
+
+	if (uc->rchan) {
+		dev_dbg(ud->dev, "chan%d: already have rchan%d allocated\n",
+			uc->id, uc->rchan->id);
+		return 0;
+	}
+
+	uc->rchan = __udma_reserve_rchan(ud, uc->channel_tpl, -1);
+	if (IS_ERR(uc->rchan))
+		return PTR_ERR(uc->rchan);
+
+	return 0;
+}
+
+static int udma_get_chan_pair(struct udma_chan *uc)
+{
+	struct udma_dev *ud = uc->ud;
+	const struct udma_match_data *match_data = ud->match_data;
+	int chan_id, end;
+
+	if ((uc->tchan && uc->rchan) && uc->tchan->id == uc->rchan->id) {
+		dev_info(ud->dev, "chan%d: already have %d pair allocated\n",
+			 uc->id, uc->tchan->id);
+		return 0;
+	}
+
+	if (uc->tchan) {
+		dev_err(ud->dev, "chan%d: already have tchan%d allocated\n",
+			uc->id, uc->tchan->id);
+		return -EBUSY;
+	} else if (uc->rchan) {
+		dev_err(ud->dev, "chan%d: already have rchan%d allocated\n",
+			uc->id, uc->rchan->id);
+		return -EBUSY;
+	}
+
+	/* Can be optimized, but let's have it like this for now */
+	end = min(ud->tchan_cnt, ud->rchan_cnt);
+	/* Try to use the highest TPL channel pair for MEM_TO_MEM channels */
+	chan_id = match_data->level_start_idx[match_data->tpl_levels - 1];
+	for (; chan_id < end; chan_id++) {
+		if (!test_bit(chan_id, ud->tchan_map) &&
+		    !test_bit(chan_id, ud->rchan_map))
+			break;
+	}
+
+	if (chan_id == end)
+		return -ENOENT;
+
+	set_bit(chan_id, ud->tchan_map);
+	set_bit(chan_id, ud->rchan_map);
+	uc->tchan = &ud->tchans[chan_id];
+	uc->rchan = &ud->rchans[chan_id];
+
+	return 0;
+}
+
+static int udma_get_rflow(struct udma_chan *uc, int flow_id)
+{
+	struct udma_dev *ud = uc->ud;
+
+	if (uc->rflow) {
+		dev_dbg(ud->dev, "chan%d: already have rflow%d allocated\n",
+			uc->id, uc->rflow->id);
+		return 0;
+	}
+
+	if (!uc->rchan)
+		dev_warn(ud->dev, "chan%d: does not have rchan??\n", uc->id);
+
+	uc->rflow = __udma_get_rflow(ud, flow_id);
+	if (IS_ERR(uc->rflow))
+		return PTR_ERR(uc->rflow);
+
+	return 0;
+}
+
+static void udma_put_rchan(struct udma_chan *uc)
+{
+	struct udma_dev *ud = uc->ud;
+
+	if (uc->rchan) {
+		dev_dbg(ud->dev, "chan%d: put rchan%d\n", uc->id,
+			uc->rchan->id);
+		clear_bit(uc->rchan->id, ud->rchan_map);
+		uc->rchan = NULL;
+	}
+}
+
+static void udma_put_tchan(struct udma_chan *uc)
+{
+	struct udma_dev *ud = uc->ud;
+
+	if (uc->tchan) {
+		dev_dbg(ud->dev, "chan%d: put tchan%d\n", uc->id,
+			uc->tchan->id);
+		clear_bit(uc->tchan->id, ud->tchan_map);
+		uc->tchan = NULL;
+	}
+}
+
+static void udma_put_rflow(struct udma_chan *uc)
+{
+	struct udma_dev *ud = uc->ud;
+
+	if (uc->rflow) {
+		dev_dbg(ud->dev, "chan%d: put rflow%d\n", uc->id,
+			uc->rflow->id);
+		__udma_put_rflow(ud, uc->rflow);
+		uc->rflow = NULL;
+	}
+}
+
+static void udma_free_tx_resources(struct udma_chan *uc)
+{
+	if (!uc->tchan)
+		return;
+
+	k3_ringacc_ring_free(uc->tchan->t_ring);
+	k3_ringacc_ring_free(uc->tchan->tc_ring);
+	uc->tchan->t_ring = NULL;
+	uc->tchan->tc_ring = NULL;
+
+	udma_put_tchan(uc);
+}
+
+static int udma_alloc_tx_resources(struct udma_chan *uc)
+{
+	struct k3_ring_cfg ring_cfg;
+	struct udma_dev *ud = uc->ud;
+	int ret;
+
+	ret = udma_get_tchan(uc);
+	if (ret)
+		return ret;
+
+	uc->tchan->t_ring = k3_ringacc_request_ring(ud->ringacc,
+						    uc->tchan->id, 0);
+	if (!uc->tchan->t_ring) {
+		ret = -EBUSY;
+		goto err_tx_ring;
+	}
+
+	uc->tchan->tc_ring = k3_ringacc_request_ring(ud->ringacc, -1, 0);
+	if (!uc->tchan->tc_ring) {
+		ret = -EBUSY;
+		goto err_txc_ring;
+	}
+
+	memset(&ring_cfg, 0, sizeof(ring_cfg));
+	ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;
+	ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8;
+	ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE;
+
+	ret = k3_ringacc_ring_cfg(uc->tchan->t_ring, &ring_cfg);
+	ret |= k3_ringacc_ring_cfg(uc->tchan->tc_ring, &ring_cfg);
+
+	if (ret)
+		goto err_ringcfg;
+
+	return 0;
+
+err_ringcfg:
+	k3_ringacc_ring_free(uc->tchan->tc_ring);
+	uc->tchan->tc_ring = NULL;
+err_txc_ring:
+	k3_ringacc_ring_free(uc->tchan->t_ring);
+	uc->tchan->t_ring = NULL;
+err_tx_ring:
+	udma_put_tchan(uc);
+
+	return ret;
+}
+
+static void udma_free_rx_resources(struct udma_chan *uc)
+{
+	if (!uc->rchan)
+		return;
+
+	if (uc->dir != DMA_MEM_TO_MEM) {
+		k3_ringacc_ring_free(uc->rchan->fd_ring);
+		k3_ringacc_ring_free(uc->rchan->r_ring);
+		uc->rchan->fd_ring = NULL;
+		uc->rchan->r_ring = NULL;
+
+		udma_put_rflow(uc);
+	}
+
+	udma_put_rchan(uc);
+}
+
+static int udma_alloc_rx_resources(struct udma_chan *uc)
+{
+	struct k3_ring_cfg ring_cfg;
+	struct udma_dev *ud = uc->ud;
+	int fd_ring_id;
+	int ret;
+
+	ret = udma_get_rchan(uc);
+	if (ret)
+		return ret;
+
+	/* For MEM_TO_MEM we don't need rflow or rings */
+	if (uc->dir == DMA_MEM_TO_MEM)
+		return 0;
+
+	ret = udma_get_rflow(uc, uc->rchan->id);
+	if (ret) {
+		ret = -EBUSY;
+		goto err_rflow;
+	}
+
+	fd_ring_id = ud->tchan_cnt + ud->echan_cnt + uc->rchan->id;
+	uc->rchan->fd_ring = k3_ringacc_request_ring(ud->ringacc, fd_ring_id,
+						     0);
+	if (!uc->rchan->fd_ring) {
+		ret = -EBUSY;
+		goto err_rx_ring;
+	}
+
+	uc->rchan->r_ring = k3_ringacc_request_ring(ud->ringacc, -1, 0);
+	if (!uc->rchan->r_ring) {
+		ret = -EBUSY;
+		goto err_rxc_ring;
+	}
+
+	memset(&ring_cfg, 0, sizeof(ring_cfg));
+
+	if (uc->pkt_mode)
+		ring_cfg.size = SG_MAX_SEGMENTS;
+	else
+		ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;
+
+	ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8;
+	ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE;
+
+	ret = k3_ringacc_ring_cfg(uc->rchan->fd_ring, &ring_cfg);
+	ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;
+	ret |= k3_ringacc_ring_cfg(uc->rchan->r_ring, &ring_cfg);
+
+	if (ret)
+		goto err_ringcfg;
+
+	return 0;
+
+err_ringcfg:
+	k3_ringacc_ring_free(uc->rchan->r_ring);
+	uc->rchan->r_ring = NULL;
+err_rxc_ring:
+	k3_ringacc_ring_free(uc->rchan->fd_ring);
+	uc->rchan->fd_ring = NULL;
+err_rx_ring:
+	udma_put_rflow(uc);
+err_rflow:
+	udma_put_rchan(uc);
+
+	return ret;
+}
+
+static int udma_alloc_chan_resources(struct dma_chan *chan)
+{
+	struct udma_chan *uc = to_udma_chan(chan);
+	struct udma_dev *ud = to_udma_dev(chan->device);
+	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
+	const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
+	const struct udma_match_data *match_data = uc->ud->match_data;
+	struct udma_tchan *tchan;
+	struct udma_rchan *rchan;
+	int ret;
+
+	if (uc->pkt_mode || uc->dir == DMA_MEM_TO_MEM) {
+		uc->use_dma_pool = true;
+		/* in case of MEM_TO_MEM we have maximum of two TRs */
+		if (uc->dir == DMA_MEM_TO_MEM)
+			uc->hdesc_size = cppi5_trdesc_calc_size(
+					sizeof(struct cppi5_tr_type15_t), 2);
+	}
+
+	if (uc->use_dma_pool) {
+		uc->hdesc_pool = dma_pool_create(uc->name, ud->ddev.dev,
+						 uc->hdesc_size, ud->desc_align,
+						 0);
+		if (!uc->hdesc_pool) {
+			dev_err(ud->ddev.dev,
+				"Descriptor pool allocation failed\n");
+			uc->use_dma_pool = false;
+			return -ENOMEM;
+		}
+	}
+
+	pm_runtime_get_sync(ud->ddev.dev);
+
+	/*
+	 * Make sure that the completion is in a known state:
+	 * No teardown, the channel is idle
+	 */
+	reinit_completion(&uc->teardown_completed);
+	complete_all(&uc->teardown_completed);
+	uc->state = UDMA_CHAN_IS_IDLE;
+
+	switch (uc->dir) {
+	case DMA_MEM_TO_MEM:
+		/* Non synchronized - mem to mem type of transfer */
+		dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-MEM\n", __func__,
+			uc->id);
+
+		ret = udma_get_chan_pair(uc);
+		if (ret)
+			return ret;
+
+		ret = udma_alloc_tx_resources(uc);
+		if (ret)
+			return ret;
+
+		ret = udma_alloc_rx_resources(uc);
+		if (ret) {
+			udma_free_tx_resources(uc);
+			return ret;
+		}
+
+		uc->src_thread = ud->psil_base + uc->tchan->id;
+		uc->dst_thread = (ud->psil_base + uc->rchan->id) | 0x8000;
+
+		break;
+	case DMA_MEM_TO_DEV:
+		/* Slave transfer synchronized - mem to dev (TX) trasnfer */
+		dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__,
+			uc->id);
+
+		ret = udma_alloc_tx_resources(uc);
+		if (ret) {
+			uc->remote_thread_id = -1;
+			return ret;
+		}
+
+		uc->src_thread = ud->psil_base + uc->tchan->id;
+		uc->dst_thread = uc->remote_thread_id;
+		if (!(uc->dst_thread & 0x8000))
+			uc->dst_thread |= 0x8000;
+
+		break;
+	case DMA_DEV_TO_MEM:
+		/* Slave transfer synchronized - dev to mem (RX) trasnfer */
+		dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__,
+			uc->id);
+
+		ret = udma_alloc_rx_resources(uc);
+		if (ret) {
+			uc->remote_thread_id = -1;
+			return ret;
+		}
+
+		uc->src_thread = uc->remote_thread_id;
+		uc->dst_thread = (ud->psil_base + uc->rchan->id) | 0x8000;
+
+		break;
+	default:
+		/* Can not happen */
+		dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n",
+			__func__, uc->id, uc->dir);
+		return -EINVAL;
+	}
+
+	tchan = uc->tchan;
+	rchan = uc->rchan;
+
+	/*
+	 * Configure Tx and Rx channel type to:
+	 * Third Party DMA control transfers using pass by reference rings
+	 */
+	if (uc->dir == DMA_MEM_TO_MEM) {
+		/* Non synchronized - mem to mem type of transfer */
+		int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring);
+		struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 };
+		struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 };
+
+		req_tx.valid_params =
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID;
+
+		req_tx.nav_id = tisci_rm->tisci_dev_id;
+		req_tx.index = tchan->id;
+		req_tx.tx_pause_on_err = 0;
+		req_tx.tx_filt_einfo = 0;
+		req_tx.tx_filt_pswords = 0;
+		req_tx.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR;
+		req_tx.tx_supr_tdpkt = uc->notdpkt;
+		req_tx.tx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2;
+		req_tx.txcq_qnum = tc_ring;
+
+		ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);
+		if (ret) {
+			dev_err(ud->dev, "tchan%d cfg failed %d\n",
+				tchan->id, ret);
+			goto err_res_free;
+		}
+
+		req_rx.valid_params =
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_SHORT_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_LONG_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID;
+
+		req_rx.nav_id = tisci_rm->tisci_dev_id;
+		req_rx.index = rchan->id;
+		req_rx.rx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2;
+		req_rx.rxcq_qnum = tc_ring;
+		req_rx.rx_pause_on_err = 0;
+		req_rx.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR;
+		req_rx.rx_ignore_short = 0;
+		req_rx.rx_ignore_long = 0;
+		req_rx.flowid_start = 0;
+		req_rx.flowid_cnt = 0;
+
+		ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);
+		if (ret) {
+			dev_err(ud->dev, "rchan%d alloc failed %d\n",
+				rchan->id, ret);
+			goto err_res_free;
+		}
+
+		uc->irq_ra_tisci = k3_ringacc_get_tisci_dev_id(tchan->tc_ring);
+		uc->irq_ra_idx = tc_ring;
+		uc->irq_udma_idx = tchan->id;
+	} else {
+		/* Slave transfer */
+		u32 mode, fetch_size;
+
+		if (uc->pkt_mode) {
+			mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
+			fetch_size = cppi5_hdesc_calc_size(uc->needs_epib,
+							   uc->psd_size, 0);
+		} else {
+			mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBRR;
+			fetch_size = sizeof(struct cppi5_desc_hdr_t);
+		}
+
+		if (uc->dir == DMA_MEM_TO_DEV) {
+			/* TX */
+			int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring);
+			struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 };
+
+			req_tx.valid_params =
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID;
+
+			req_tx.nav_id = tisci_rm->tisci_dev_id;
+			req_tx.index = tchan->id;
+			req_tx.tx_pause_on_err = 0;
+			req_tx.tx_filt_einfo = 0;
+			req_tx.tx_filt_pswords = 0;
+			req_tx.tx_chan_type = mode;
+			req_tx.tx_supr_tdpkt = uc->notdpkt;
+			req_tx.tx_fetch_size = fetch_size >> 2;
+			req_tx.txcq_qnum = tc_ring;
+
+			ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);
+			if (ret) {
+				dev_err(ud->dev, "tchan%d cfg failed %d\n",
+					tchan->id, ret);
+				goto err_res_free;
+			}
+
+			uc->irq_ra_tisci = k3_ringacc_get_tisci_dev_id(
+								tchan->tc_ring);
+			uc->irq_ra_idx = tc_ring;
+			uc->irq_udma_idx = tchan->id;
+		} else {
+			/* RX */
+			int fd_ring = k3_ringacc_get_ring_id(rchan->fd_ring);
+			int rx_ring = k3_ringacc_get_ring_id(rchan->r_ring);
+			struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 };
+			struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 };
+
+			req_rx.valid_params =
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_SHORT_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_LONG_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID;
+
+			req_rx.nav_id = tisci_rm->tisci_dev_id;
+			req_rx.index = rchan->id;
+			req_rx.rx_fetch_size =  fetch_size >> 2;
+			req_rx.rxcq_qnum = rx_ring;
+			req_rx.rx_pause_on_err = 0;
+			req_rx.rx_chan_type = mode;
+			req_rx.rx_ignore_short = 0;
+			req_rx.rx_ignore_long = 0;
+			req_rx.flowid_start = 0;
+			req_rx.flowid_cnt = 0;
+
+			ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);
+			if (ret) {
+				dev_err(ud->dev, "rchan%d cfg failed %d\n",
+					rchan->id, ret);
+				goto err_res_free;
+			}
+
+			flow_req.valid_params =
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
+			TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
+
+			flow_req.nav_id = tisci_rm->tisci_dev_id;
+			flow_req.flow_index = rchan->id;
+
+			if (uc->needs_epib)
+				flow_req.rx_einfo_present = 1;
+			else
+				flow_req.rx_einfo_present = 0;
+			if (uc->psd_size)
+				flow_req.rx_psinfo_present = 1;
+			else
+				flow_req.rx_psinfo_present = 0;
+			flow_req.rx_error_handling = 1;
+			flow_req.rx_desc_type = 0;
+			flow_req.rx_dest_qnum = rx_ring;
+			flow_req.rx_src_tag_hi_sel = 2;
+			flow_req.rx_src_tag_lo_sel = 4;
+			flow_req.rx_dest_tag_hi_sel = 5;
+			flow_req.rx_dest_tag_lo_sel = 4;
+			flow_req.rx_fdq0_sz0_qnum = fd_ring;
+			flow_req.rx_fdq1_qnum = fd_ring;
+			flow_req.rx_fdq2_qnum = fd_ring;
+			flow_req.rx_fdq3_qnum = fd_ring;
+
+			ret = tisci_ops->rx_flow_cfg(tisci_rm->tisci,
+						      &flow_req);
+
+			if (ret) {
+				dev_err(ud->dev, "flow%d config failed: %d\n",
+					rchan->id, ret);
+				goto err_chan_free;
+			}
+
+			uc->irq_ra_tisci = k3_ringacc_get_tisci_dev_id(
+								rchan->r_ring);
+			uc->irq_ra_idx = rx_ring;
+			uc->irq_udma_idx = match_data->rchan_oes_offset +
+					   rchan->id;
+		}
+	}
+
+	if (udma_is_chan_running(uc)) {
+		dev_warn(ud->dev, "chan%d: tchan%d is running!\n", uc->id,
+			 uc->tchan->id);
+		udma_stop(uc);
+		if (udma_is_chan_running(uc)) {
+			dev_err(ud->dev, "chan%d: won't stop!\n", uc->id);
+			goto err_chan_free;
+		}
+	}
+
+	/* PSI-L pairing */
+	ret = navss_psil_pair(ud, uc->src_thread, uc->dst_thread);
+	if (ret) {
+		dev_err(ud->dev, "PSI-L pairing failed: 0x%04x -> 0x%04x\n",
+			uc->src_thread, uc->dst_thread);
+		goto err_chan_free;
+	}
+
+	uc->psil_paired = true;
+
+	/* Get the interrupts... */
+	uc->irq_num_ring = ti_sci_inta_register_event(ud->dev, uc->irq_ra_tisci,
+						      uc->irq_ra_idx, 0, false,
+						      IRQF_TRIGGER_HIGH);
+	if (uc->irq_num_ring <= 0) {
+		dev_err(ud->dev, "Failed to get ring irq (index: %u) %d\n",
+			uc->irq_ra_idx, uc->irq_num_ring);
+		ret = -EINVAL;
+		goto err_psi_free;
+	}
+
+	ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler, 0, uc->name,
+			  uc);
+	if (ret) {
+		dev_err(ud->dev, "%s: chan%d: Failed to request ring irq\n",
+			__func__, uc->id);
+		goto err_irq_free;
+	}
+
+	/* Event from UDMA (TR events) only needed for slave TR mode channels */
+	if (is_slave_direction(uc->dir) && !uc->pkt_mode) {
+		uc->irq_num_udma = ti_sci_inta_register_event(ud->dev,
+						tisci_rm->tisci_dev_id,
+						uc->irq_udma_idx, 0, true,
+						IRQF_TRIGGER_HIGH);
+		if (uc->irq_num_udma <= 0) {
+			dev_err(ud->dev,
+				"Failed to get udma irq (index: %u) %d\n",
+				uc->irq_udma_idx, uc->irq_num_udma);
+
+			free_irq(uc->irq_num_ring, uc);
+
+			ret = -EINVAL;
+			goto err_irq_free;
+		}
+
+		ret = request_irq(uc->irq_num_udma, udma_udma_irq_handler, 0,
+				  uc->name, uc);
+		if (ret) {
+			dev_err(ud->dev,
+				"%s: chan%d: Failed to request UDMA irq\n",
+				__func__, uc->id);
+			free_irq(uc->irq_num_ring, uc);
+			goto err_irq_free;
+		}
+	} else {
+		uc->irq_num_udma = 0;
+	}
+
+	udma_reset_rings(uc);
+
+	INIT_DELAYED_WORK_ONSTACK(&uc->tx_drain.work,
+				  udma_check_tx_completion);
+	return 0;
+
+err_irq_free:
+	if (uc->irq_num_ring > 0) {
+		ti_sci_inta_unregister_event(ud->dev, uc->irq_ra_tisci,
+					     uc->irq_ra_idx, uc->irq_num_ring);
+		uc->irq_num_ring = 0;
+	}
+
+	if (uc->irq_num_udma > 0) {
+		ti_sci_inta_unregister_event(ud->dev, tisci_rm->tisci_dev_id,
+					     uc->irq_udma_idx,
+					     uc->irq_num_udma);
+		uc->irq_num_udma = 0;
+	}
+err_psi_free:
+	navss_psil_unpair(ud, uc->src_thread, uc->dst_thread);
+	uc->psil_paired = false;
+err_chan_free:
+err_res_free:
+	udma_free_tx_resources(uc);
+	udma_free_rx_resources(uc);
+	udma_reset_uchan(uc);
+
+	if (uc->use_dma_pool) {
+		dma_pool_destroy(uc->hdesc_pool);
+		uc->use_dma_pool = false;
+	}
+
+	return ret;
+}
+
+static int udma_slave_config(struct dma_chan *chan,
+			     struct dma_slave_config *cfg)
+{
+	struct udma_chan *uc = to_udma_chan(chan);
+
+	memcpy(&uc->cfg, cfg, sizeof(uc->cfg));
+
+	return 0;
+}
+
+static struct udma_desc *udma_alloc_tr_desc(struct udma_chan *uc,
+					    size_t tr_size, int tr_count,
+					    enum dma_transfer_direction dir)
+{
+	struct udma_hwdesc *hwdesc;
+	struct cppi5_desc_hdr_t *tr_desc;
+	struct udma_desc *d;
+	u32 reload_count = 0;
+	u32 ring_id;
+
+	switch (tr_size) {
+	case 16:
+	case 32:
+	case 64:
+	case 128:
+		break;
+	default:
+		dev_err(uc->ud->dev, "Unsupported TR size of %zu\n", tr_size);
+		return NULL;
+	}
+
+	/* We have only one descriptor containing multiple TRs */
+	d = kzalloc(sizeof(*d) + sizeof(d->hwdesc[0]), GFP_ATOMIC);
+	if (!d)
+		return NULL;
+
+	d->sglen = tr_count;
+
+	d->hwdesc_count = 1;
+	hwdesc = &d->hwdesc[0];
+
+	/* Allocate memory for DMA ring descriptor */
+	if (uc->use_dma_pool) {
+		hwdesc->cppi5_desc_size = uc->hdesc_size;
+		hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,
+						GFP_ATOMIC,
+						&hwdesc->cppi5_desc_paddr);
+	} else {
+		hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size,
+								 tr_count);
+		hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size,
+						uc->ud->desc_align);
+		hwdesc->cppi5_desc_vaddr = dma_zalloc_coherent(uc->ud->dev,
+						hwdesc->cppi5_desc_size,
+						&hwdesc->cppi5_desc_paddr,
+						GFP_ATOMIC);
+	}
+
+	if (!hwdesc->cppi5_desc_vaddr) {
+		kfree(d);
+		return NULL;
+	}
+
+	/* Start of the TR req records */
+	hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size;
+	/* Start address of the TR response array */
+	hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size * tr_count;
+
+	tr_desc = hwdesc->cppi5_desc_vaddr;
+
+	if (uc->cyclic)
+		reload_count = CPPI5_INFO0_TRDESC_RLDCNT_INFINITE;
+
+	if (dir == DMA_DEV_TO_MEM)
+		ring_id = k3_ringacc_get_ring_id(uc->rchan->r_ring);
+	else
+		ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);
+
+	cppi5_trdesc_init(tr_desc, tr_count, tr_size, 0, reload_count);
+	cppi5_desc_set_pktids(tr_desc, uc->id, 0x3fff);
+	cppi5_desc_set_retpolicy(tr_desc, 0, ring_id);
+
+	return d;
+}
+
+static struct udma_desc *udma_prep_slave_sg_tr(
+	struct udma_chan *uc, struct scatterlist *sgl, unsigned int sglen,
+	enum dma_transfer_direction dir, unsigned long tx_flags, void *context)
+{
+	enum dma_slave_buswidth dev_width;
+	struct scatterlist *sgent;
+	struct udma_desc *d;
+	size_t tr_size;
+	struct cppi5_tr_type1_t *tr_req = NULL;
+	unsigned int i;
+	u32 burst;
+
+	if (dir == DMA_DEV_TO_MEM) {
+		dev_width = uc->cfg.src_addr_width;
+		burst = uc->cfg.src_maxburst;
+	} else if (dir == DMA_MEM_TO_DEV) {
+		dev_width = uc->cfg.dst_addr_width;
+		burst = uc->cfg.dst_maxburst;
+	} else {
+		dev_err(uc->ud->dev, "%s: bad direction?\n", __func__);
+		return NULL;
+	}
+
+	if (!burst)
+		burst = 1;
+
+	/* Now allocate and setup the descriptor. */
+	tr_size = sizeof(struct cppi5_tr_type1_t);
+	d = udma_alloc_tr_desc(uc, tr_size, sglen, dir);
+	if (!d)
+		return NULL;
+
+	d->sglen = sglen;
+
+	tr_req = (struct cppi5_tr_type1_t *)d->hwdesc[0].tr_req_base;
+	for_each_sg(sgl, sgent, sglen, i) {
+		d->residue += sg_dma_len(sgent);
+
+		cppi5_tr_init(&tr_req[i].flags, CPPI5_TR_TYPE1, false, false,
+			      CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
+		cppi5_tr_csf_set(&tr_req[i].flags, CPPI5_TR_CSF_SUPR_EVT);
+
+		tr_req[i].addr = sg_dma_address(sgent);
+		tr_req[i].icnt0 = burst * dev_width;
+		tr_req[i].dim1 = burst * dev_width;
+		tr_req[i].icnt1 = sg_dma_len(sgent) / tr_req[i].icnt0;
+	}
+
+	cppi5_tr_csf_set(&tr_req[i - 1].flags, CPPI5_TR_CSF_EOP);
+
+	return d;
+}
+
+static inline int udma_configure_statictr(struct udma_chan *uc,
+					  struct udma_desc *d,
+					  enum dma_slave_buswidth dev_width,
+					  u16 elcnt)
+{
+	if (!uc->static_tr_type)
+		return 0;
+
+	/* Bus width translates to the element size (ES) */
+	switch (dev_width) {
+	case DMA_SLAVE_BUSWIDTH_1_BYTE:
+		d->static_tr.elsize = 0;
+		break;
+	case DMA_SLAVE_BUSWIDTH_2_BYTES:
+		d->static_tr.elsize = 1;
+		break;
+	case DMA_SLAVE_BUSWIDTH_3_BYTES:
+		d->static_tr.elsize = 2;
+		break;
+	case DMA_SLAVE_BUSWIDTH_4_BYTES:
+		d->static_tr.elsize = 3;
+		break;
+	case DMA_SLAVE_BUSWIDTH_8_BYTES:
+		d->static_tr.elsize = 4;
+		break;
+	default: /* not reached */
+		return -EINVAL;
+	}
+
+	d->static_tr.elcnt = elcnt;
+
+	/*
+	 * PDMA must to close the packet when the channel is in packet mode.
+	 * For TR mode when the channel is not cyclic we also need PDMA to close
+	 * the packet otherwise the transfer will stall because PDMA holds on
+	 * the data it has received from the peripheral.
+	 */
+	if (uc->pkt_mode || !uc->cyclic) {
+		unsigned int div = dev_width * elcnt;
+
+		if (uc->cyclic)
+			d->static_tr.bstcnt = d->residue / d->sglen / div;
+		else
+			d->static_tr.bstcnt = d->residue / div;
+
+		if (uc->dir == DMA_DEV_TO_MEM &&
+		    d->static_tr.bstcnt > uc->ud->match_data->statictr_z_mask)
+			return -EINVAL;
+	} else {
+		d->static_tr.bstcnt = 0;
+	}
+
+	return 0;
+}
+
+static struct udma_desc *udma_prep_slave_sg_pkt(
+	struct udma_chan *uc, struct scatterlist *sgl, unsigned int sglen,
+	enum dma_transfer_direction dir, unsigned long tx_flags, void *context)
+{
+	struct scatterlist *sgent;
+	struct cppi5_host_desc_t *h_desc = NULL;
+	struct udma_desc *d;
+	u32 ring_id;
+	unsigned int i;
+
+	d = kzalloc(sizeof(*d) + sglen * sizeof(d->hwdesc[0]), GFP_ATOMIC);
+	if (!d)
+		return NULL;
+
+	d->sglen = sglen;
+	d->hwdesc_count = sglen;
+
+	if (dir == DMA_DEV_TO_MEM)
+		ring_id = k3_ringacc_get_ring_id(uc->rchan->r_ring);
+	else
+		ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);
+
+	for_each_sg(sgl, sgent, sglen, i) {
+		struct udma_hwdesc *hwdesc = &d->hwdesc[i];
+		dma_addr_t sg_addr = sg_dma_address(sgent);
+		struct cppi5_host_desc_t *desc;
+		size_t sg_len = sg_dma_len(sgent);
+
+		hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,
+						GFP_ATOMIC,
+						&hwdesc->cppi5_desc_paddr);
+		if (!hwdesc->cppi5_desc_vaddr) {
+			dev_err(uc->ud->dev,
+				"descriptor%d allocation failed\n", i);
+
+			udma_free_hwdesc(uc, d);
+			kfree(d);
+			return NULL;
+		}
+
+		d->residue += sg_len;
+		hwdesc->cppi5_desc_size = uc->hdesc_size;
+		desc = hwdesc->cppi5_desc_vaddr;
+
+		if (i == 0) {
+			cppi5_hdesc_init(desc, 0, 0);
+			/* Flow and Packed ID ??? */
+			cppi5_desc_set_pktids(&desc->hdr, uc->id, 0x3fff);
+			cppi5_desc_set_retpolicy(&desc->hdr, 0, ring_id);
+		} else {
+			cppi5_hdesc_reset_hbdesc(desc);
+			cppi5_desc_set_retpolicy(&desc->hdr, 0, 0xffff);
+		}
+
+		/* attach the sg buffer to the descriptor */
+		cppi5_hdesc_attach_buf(desc, sg_addr, sg_len, sg_addr, sg_len);
+
+		/* Attach link as host buffer descriptor */
+		if (h_desc)
+			cppi5_hdesc_link_hbdesc(h_desc,
+						hwdesc->cppi5_desc_paddr);
+
+		if (dir == DMA_MEM_TO_DEV)
+			h_desc = desc;
+	}
+
+	if (d->residue >= SZ_4M) {
+		dev_err(uc->ud->dev,
+			"%s: Transfer size %u is over the supported 4M range\n",
+			__func__, d->residue);
+		udma_free_hwdesc(uc, d);
+		kfree(d);
+		return NULL;
+	}
+
+	h_desc = d->hwdesc[0].cppi5_desc_vaddr;
+	cppi5_hdesc_set_pktlen(h_desc, d->residue);
+
+	return d;
+}
+
+static int udma_attach_metadata(struct dma_async_tx_descriptor *desc,
+				void *data, size_t len)
+{
+	struct udma_desc *d = to_udma_desc(desc);
+	struct udma_chan *uc = to_udma_chan(desc->chan);
+	struct cppi5_host_desc_t *h_desc;
+	u32 psd_size = len;
+	u32 flags = 0;
+
+	if (!uc->pkt_mode || !uc->metadata_size)
+		return -ENOTSUPP;
+
+	if (!data || len > uc->metadata_size)
+		return -EINVAL;
+
+	if (uc->needs_epib && len < CPPI5_INFO0_HDESC_EPIB_SIZE)
+		return -EINVAL;
+
+	h_desc = d->hwdesc[0].cppi5_desc_vaddr;
+	if (d->dir == DMA_MEM_TO_DEV)
+		memcpy(h_desc->epib, data, len);
+
+	if (uc->needs_epib)
+		psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE;
+
+	d->metadata = data;
+	d->metadata_size = len;
+	if (uc->needs_epib)
+		flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT;
+
+	cppi5_hdesc_update_flags(h_desc, flags);
+	cppi5_hdesc_update_psdata_size(h_desc, psd_size);
+
+	return 0;
+}
+
+static void *udma_get_metadata_ptr(struct dma_async_tx_descriptor *desc,
+				   size_t *payload_len, size_t *max_len)
+{
+	struct udma_desc *d = to_udma_desc(desc);
+	struct udma_chan *uc = to_udma_chan(desc->chan);
+	struct cppi5_host_desc_t *h_desc;
+
+	if (!uc->pkt_mode || !uc->metadata_size)
+		return ERR_PTR(-ENOTSUPP);
+
+	h_desc = d->hwdesc[0].cppi5_desc_vaddr;
+
+	*max_len = uc->metadata_size;
+
+	*payload_len = cppi5_hdesc_epib_present(&h_desc->hdr) ?
+		       CPPI5_INFO0_HDESC_EPIB_SIZE : 0;
+	*payload_len += cppi5_hdesc_get_psdata_size(h_desc);
+
+	return h_desc->epib;
+}
+
+static int udma_set_metadata_len(struct dma_async_tx_descriptor *desc,
+				 size_t payload_len)
+{
+	struct udma_desc *d = to_udma_desc(desc);
+	struct udma_chan *uc = to_udma_chan(desc->chan);
+	struct cppi5_host_desc_t *h_desc;
+	u32 psd_size = payload_len;
+	u32 flags = 0;
+
+	if (!uc->pkt_mode || !uc->metadata_size)
+		return -ENOTSUPP;
+
+	if (payload_len > uc->metadata_size)
+		return -EINVAL;
+
+	if (uc->needs_epib && payload_len < CPPI5_INFO0_HDESC_EPIB_SIZE)
+		return -EINVAL;
+
+	h_desc = d->hwdesc[0].cppi5_desc_vaddr;
+
+	if (uc->needs_epib) {
+		psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE;
+		flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT;
+	}
+
+	cppi5_hdesc_update_flags(h_desc, flags);
+	cppi5_hdesc_update_psdata_size(h_desc, psd_size);
+
+	return 0;
+}
+
+static struct dma_descriptor_metadata_ops metadata_ops = {
+	.attach = udma_attach_metadata,
+	.get_ptr = udma_get_metadata_ptr,
+	.set_len = udma_set_metadata_len,
+};
+
+static struct dma_async_tx_descriptor *udma_prep_slave_sg(
+	struct dma_chan *chan, struct scatterlist *sgl, unsigned int sglen,
+	enum dma_transfer_direction dir, unsigned long tx_flags, void *context)
+{
+	struct udma_chan *uc = to_udma_chan(chan);
+	enum dma_slave_buswidth dev_width;
+	struct udma_desc *d;
+	u32 burst;
+
+	if (dir != uc->dir) {
+		dev_err(chan->device->dev,
+			"%s: chan%d is for %s, not supporting %s\n",
+			__func__, uc->id, udma_get_dir_text(uc->dir),
+			udma_get_dir_text(dir));
+		return NULL;
+	}
+
+	if (dir == DMA_DEV_TO_MEM) {
+		dev_width = uc->cfg.src_addr_width;
+		burst = uc->cfg.src_maxburst;
+	} else if (dir == DMA_MEM_TO_DEV) {
+		dev_width = uc->cfg.dst_addr_width;
+		burst = uc->cfg.dst_maxburst;
+	} else {
+		dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
+		return NULL;
+	}
+
+	if (!burst)
+		burst = 1;
+
+	if (uc->pkt_mode)
+		d = udma_prep_slave_sg_pkt(uc, sgl, sglen, dir, tx_flags,
+					   context);
+	else
+		d = udma_prep_slave_sg_tr(uc, sgl, sglen, dir, tx_flags,
+					  context);
+
+	if (!d)
+		return NULL;
+
+	d->dir = dir;
+	d->desc_idx = 0;
+	d->tr_idx = 0;
+
+	/* static TR for remote PDMA */
+	if (udma_configure_statictr(uc, d, dev_width, burst)) {
+		dev_err(uc->ud->dev,
+			"%s: StaticTR Z is limted to maximum 4095 (%u)\n",
+			__func__, d->static_tr.bstcnt);
+
+		udma_free_hwdesc(uc, d);
+		kfree(d);
+		return NULL;
+	}
+
+	if (uc->metadata_size)
+		d->vd.tx.metadata_ops = &metadata_ops;
+
+	return vchan_tx_prep(&uc->vc, &d->vd, tx_flags);
+}
+
+static struct udma_desc *udma_prep_dma_cyclic_tr(
+	struct udma_chan *uc, dma_addr_t buf_addr, size_t buf_len,
+	size_t period_len, enum dma_transfer_direction dir, unsigned long flags)
+{
+	enum dma_slave_buswidth dev_width;
+	struct udma_desc *d;
+	size_t tr_size;
+	struct cppi5_tr_type1_t *tr_req;
+	unsigned int i;
+	unsigned int periods = buf_len / period_len;
+	u32 burst;
+
+	if (dir == DMA_DEV_TO_MEM) {
+		dev_width = uc->cfg.src_addr_width;
+		burst = uc->cfg.src_maxburst;
+	} else if (dir == DMA_MEM_TO_DEV) {
+		dev_width = uc->cfg.dst_addr_width;
+		burst = uc->cfg.dst_maxburst;
+	} else {
+		dev_err(uc->ud->dev, "%s: bad direction?\n", __func__);
+		return NULL;
+	}
+
+	if (!burst)
+		burst = 1;
+
+	/* Now allocate and setup the descriptor. */
+	tr_size = sizeof(struct cppi5_tr_type1_t);
+	d = udma_alloc_tr_desc(uc, tr_size, periods, dir);
+	if (!d)
+		return NULL;
+
+	tr_req = (struct cppi5_tr_type1_t *)d->hwdesc[0].tr_req_base;
+	for (i = 0; i < periods; i++) {
+		cppi5_tr_init(&tr_req[i].flags, CPPI5_TR_TYPE1, false, false,
+			      CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
+
+		tr_req[i].addr = buf_addr + period_len * i;
+		tr_req[i].icnt0 = dev_width;
+		tr_req[i].icnt1 = period_len / dev_width;
+		tr_req[i].dim1 = dev_width;
+
+		if (!(flags & DMA_PREP_INTERRUPT))
+			cppi5_tr_csf_set(&tr_req[i].flags,
+					 CPPI5_TR_CSF_SUPR_EVT);
+	}
+
+	return d;
+}
+
+static struct udma_desc *udma_prep_dma_cyclic_pkt(
+	struct udma_chan *uc, dma_addr_t buf_addr, size_t buf_len,
+	size_t period_len, enum dma_transfer_direction dir, unsigned long flags)
+{
+	struct udma_desc *d;
+	u32 ring_id;
+	int i;
+	int periods = buf_len / period_len;
+
+	if (periods > (K3_UDMA_DEFAULT_RING_SIZE - 1))
+		return NULL;
+
+	if (period_len > 0x3FFFFF)
+		return NULL;
+
+	d = kzalloc(sizeof(*d) + periods * sizeof(d->hwdesc[0]), GFP_ATOMIC);
+	if (!d)
+		return NULL;
+
+	d->hwdesc_count = periods;
+
+	/* TODO: re-check this... */
+	if (dir == DMA_DEV_TO_MEM)
+		ring_id = k3_ringacc_get_ring_id(uc->rchan->r_ring);
+	else
+		ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);
+
+	for (i = 0; i < periods; i++) {
+		struct udma_hwdesc *hwdesc = &d->hwdesc[i];
+		dma_addr_t period_addr = buf_addr + (period_len * i);
+		struct cppi5_host_desc_t *h_desc;
+
+		hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,
+						GFP_ATOMIC,
+						&hwdesc->cppi5_desc_paddr);
+		if (!hwdesc->cppi5_desc_vaddr) {
+			dev_err(uc->ud->dev,
+				"descriptor%d allocation failed\n", i);
+
+			udma_free_hwdesc(uc, d);
+			kfree(d);
+			return NULL;
+		}
+
+		hwdesc->cppi5_desc_size = uc->hdesc_size;
+		h_desc = hwdesc->cppi5_desc_vaddr;
+
+		cppi5_hdesc_init(h_desc, 0, 0);
+		cppi5_hdesc_set_pktlen(h_desc, period_len);
+
+		/* Flow and Packed ID ??? */
+		cppi5_desc_set_pktids(&h_desc->hdr, uc->id, 0x3fff);
+		cppi5_desc_set_retpolicy(&h_desc->hdr, 0, ring_id);
+
+		/* attach each period to a new descriptor */
+		cppi5_hdesc_attach_buf(h_desc,
+				       period_addr, period_len,
+				       period_addr, period_len);
+	}
+
+	return d;
+}
+
+static struct dma_async_tx_descriptor *udma_prep_dma_cyclic(
+	struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
+	size_t period_len, enum dma_transfer_direction dir, unsigned long flags)
+{
+	struct udma_chan *uc = to_udma_chan(chan);
+	enum dma_slave_buswidth dev_width;
+	struct udma_desc *d;
+	u32 burst;
+
+	if (dir != uc->dir) {
+		dev_err(chan->device->dev,
+			"%s: chan%d is for %s, not supporting %s\n",
+			__func__, uc->id, udma_get_dir_text(uc->dir),
+			udma_get_dir_text(dir));
+		return NULL;
+	}
+
+	uc->cyclic = true;
+
+	if (dir == DMA_DEV_TO_MEM) {
+		dev_width = uc->cfg.src_addr_width;
+		burst = uc->cfg.src_maxburst;
+	} else if (dir == DMA_MEM_TO_DEV) {
+		dev_width = uc->cfg.dst_addr_width;
+		burst = uc->cfg.dst_maxburst;
+	} else {
+		dev_err(uc->ud->dev, "%s: bad direction?\n", __func__);
+		return NULL;
+	}
+
+	if (!burst)
+		burst = 1;
+
+	if (uc->pkt_mode)
+		d = udma_prep_dma_cyclic_pkt(uc, buf_addr, buf_len, period_len,
+					     dir, flags);
+	else
+		d = udma_prep_dma_cyclic_tr(uc, buf_addr, buf_len, period_len,
+					    dir, flags);
+
+	if (!d)
+		return NULL;
+
+	d->sglen = buf_len / period_len;
+
+	d->dir = dir;
+	d->residue = buf_len;
+
+	/* static TR for remote PDMA */
+	if (udma_configure_statictr(uc, d, dev_width, burst)) {
+		dev_err(uc->ud->dev,
+			"%s: StaticTR Z is limted to maximum 4095 (%u)\n",
+			__func__, d->static_tr.bstcnt);
+
+		udma_free_hwdesc(uc, d);
+		kfree(d);
+		return NULL;
+	}
+
+	if (uc->metadata_size)
+		d->vd.tx.metadata_ops = &metadata_ops;
+
+	return vchan_tx_prep(&uc->vc, &d->vd, flags);
+}
+
+static struct dma_async_tx_descriptor *udma_prep_dma_memcpy(
+	struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
+	size_t len, unsigned long tx_flags)
+{
+	struct udma_chan *uc = to_udma_chan(chan);
+	struct udma_desc *d;
+	struct cppi5_tr_type15_t *tr_req;
+	int num_tr;
+	size_t tr_size = sizeof(struct cppi5_tr_type15_t);
+	u16 tr0_cnt0, tr0_cnt1, tr1_cnt0;
+
+	if (uc->dir != DMA_MEM_TO_MEM) {
+		dev_err(chan->device->dev,
+			"%s: chan%d is for %s, not supporting %s\n",
+			__func__, uc->id, udma_get_dir_text(uc->dir),
+			udma_get_dir_text(DMA_MEM_TO_MEM));
+		return NULL;
+	}
+
+	if (len < SZ_64K) {
+		num_tr = 1;
+		tr0_cnt0 = len;
+		tr0_cnt1 = 1;
+	} else {
+		unsigned long align_to = __ffs(src | dest);
+
+		if (align_to > 3)
+			align_to = 3;
+		/*
+		 * Keep simple: tr0: SZ_64K-alignment blocks,
+		 *		tr1: the remaining
+		 */
+		num_tr = 2;
+		tr0_cnt0 = (SZ_64K - BIT(align_to));
+		if (len / tr0_cnt0 >= SZ_64K) {
+			dev_err(uc->ud->dev, "size %zu is not supported\n",
+				len);
+			return NULL;
+		}
+
+		tr0_cnt1 = len / tr0_cnt0;
+		tr1_cnt0 = len % tr0_cnt0;
+	}
+
+	d = udma_alloc_tr_desc(uc, tr_size, num_tr, DMA_MEM_TO_MEM);
+	if (!d)
+		return NULL;
+
+	d->dir = DMA_MEM_TO_MEM;
+	d->desc_idx = 0;
+	d->tr_idx = 0;
+	d->residue = len;
+
+	tr_req = (struct cppi5_tr_type15_t *)d->hwdesc[0].tr_req_base;
+
+	cppi5_tr_init(&tr_req[0].flags, CPPI5_TR_TYPE15, false, true,
+		      CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
+	cppi5_tr_csf_set(&tr_req[0].flags, CPPI5_TR_CSF_SUPR_EVT);
+
+	tr_req[0].addr = src;
+	tr_req[0].icnt0 = tr0_cnt0;
+	tr_req[0].icnt1 = tr0_cnt1;
+	tr_req[0].icnt2 = 1;
+	tr_req[0].icnt3 = 1;
+	tr_req[0].dim1 = tr0_cnt0;
+
+	tr_req[0].daddr = dest;
+	tr_req[0].dicnt0 = tr0_cnt0;
+	tr_req[0].dicnt1 = tr0_cnt1;
+	tr_req[0].dicnt2 = 1;
+	tr_req[0].dicnt3 = 1;
+	tr_req[0].ddim1 = tr0_cnt0;
+
+	if (num_tr == 2) {
+		cppi5_tr_init(&tr_req[1].flags, CPPI5_TR_TYPE15, false, true,
+			      CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
+		cppi5_tr_csf_set(&tr_req[1].flags, CPPI5_TR_CSF_SUPR_EVT);
+
+		tr_req[1].addr = src + tr0_cnt1 * tr0_cnt0;
+		tr_req[1].icnt0 = tr1_cnt0;
+		tr_req[1].icnt1 = 1;
+		tr_req[1].icnt2 = 1;
+		tr_req[1].icnt3 = 1;
+
+		tr_req[1].daddr = dest + tr0_cnt1 * tr0_cnt0;
+		tr_req[1].dicnt0 = tr1_cnt0;
+		tr_req[1].dicnt1 = 1;
+		tr_req[1].dicnt2 = 1;
+		tr_req[1].dicnt3 = 1;
+	}
+
+	cppi5_tr_csf_set(&tr_req[num_tr - 1].flags, CPPI5_TR_CSF_EOP);
+
+	if (uc->metadata_size)
+		d->vd.tx.metadata_ops = &metadata_ops;
+
+	return vchan_tx_prep(&uc->vc, &d->vd, tx_flags);
+}
+
+static void udma_issue_pending(struct dma_chan *chan)
+{
+	struct udma_chan *uc = to_udma_chan(chan);
+	unsigned long flags;
+
+	spin_lock_irqsave(&uc->vc.lock, flags);
+
+	/* If we have something pending and no active descriptor, then */
+	if (vchan_issue_pending(&uc->vc) && !uc->desc) {
+		/*
+		 * start a descriptor if the channel is NOT [marked as
+		 * terminating _and_ it is still running (teardown has not
+		 * completed yet)].
+		 */
+		if (!(uc->state == UDMA_CHAN_IS_TERMINATING &&
+		      udma_is_chan_running(uc)))
+			udma_start(uc);
+	}
+
+	spin_unlock_irqrestore(&uc->vc.lock, flags);
+}
+
+/* Not much yet */
+static enum dma_status udma_tx_status(struct dma_chan *chan,
+				      dma_cookie_t cookie,
+				      struct dma_tx_state *txstate)
+{
+	struct udma_chan *uc = to_udma_chan(chan);
+	enum dma_status ret;
+
+	ret = dma_cookie_status(chan, cookie, txstate);
+
+	if (!udma_is_chan_running(uc))
+		ret = DMA_COMPLETE;
+
+	if (ret == DMA_COMPLETE || !txstate)
+		return ret;
+
+	if (uc->desc && uc->desc->vd.tx.cookie == cookie) {
+		u32 pdma_bcnt = 0;
+		u32 bcnt = 0;
+		u32 pcnt = 0;
+		u32 residue = uc->desc->residue;
+		u32 delay = 0;
+
+		if (uc->desc->dir == DMA_MEM_TO_DEV) {
+			bcnt = udma_tchanrt_read(uc->tchan,
+						 UDMA_TCHAN_RT_SBCNT_REG);
+			pdma_bcnt = udma_tchanrt_read(uc->tchan,
+						UDMA_TCHAN_RT_PEER_BCNT_REG);
+			pcnt = udma_tchanrt_read(uc->tchan,
+						 UDMA_TCHAN_RT_PCNT_REG);
+
+			if (bcnt > pdma_bcnt)
+				delay = bcnt - pdma_bcnt;
+		} else if (uc->desc->dir == DMA_DEV_TO_MEM) {
+			bcnt = udma_rchanrt_read(uc->rchan,
+						 UDMA_RCHAN_RT_BCNT_REG);
+			pdma_bcnt = udma_rchanrt_read(uc->rchan,
+						UDMA_RCHAN_RT_PEER_BCNT_REG);
+			pcnt = udma_rchanrt_read(uc->rchan,
+						 UDMA_RCHAN_RT_PCNT_REG);
+
+			if (pdma_bcnt > bcnt)
+				delay = pdma_bcnt - bcnt;
+		} else {
+			u32 sbcnt;
+
+			sbcnt = udma_tchanrt_read(uc->tchan,
+						  UDMA_TCHAN_RT_BCNT_REG);
+			bcnt = udma_tchanrt_read(uc->tchan,
+						 UDMA_TCHAN_RT_PEER_BCNT_REG);
+			pcnt = udma_tchanrt_read(uc->tchan,
+						 UDMA_TCHAN_RT_PCNT_REG);
+
+			if (sbcnt > bcnt)
+				delay = sbcnt - bcnt;
+		}
+
+		bcnt -= uc->bcnt;
+		if (bcnt && !(bcnt % uc->desc->residue))
+			residue = 0;
+		else
+			residue -= bcnt % uc->desc->residue;
+
+		if (!residue && (uc->dir == DMA_DEV_TO_MEM || !delay)) {
+			ret = DMA_COMPLETE;
+			delay = 0;
+		}
+
+		dma_set_residue(txstate, residue);
+		dma_set_in_flight_bytes(txstate, delay);
+
+	} else {
+		ret = DMA_COMPLETE;
+	}
+
+	return ret;
+}
+
+static int udma_pause(struct dma_chan *chan)
+{
+	struct udma_chan *uc = to_udma_chan(chan);
+
+	if (!uc->desc)
+		return -EINVAL;
+
+	/* pause the channel */
+	switch (uc->desc->dir) {
+	case DMA_DEV_TO_MEM:
+		udma_rchanrt_update_bits(uc->rchan,
+					 UDMA_RCHAN_RT_PEER_RT_EN_REG,
+					 UDMA_PEER_RT_EN_PAUSE,
+					 UDMA_PEER_RT_EN_PAUSE);
+		break;
+	case DMA_MEM_TO_DEV:
+		udma_tchanrt_update_bits(uc->tchan,
+					 UDMA_TCHAN_RT_PEER_RT_EN_REG,
+					 UDMA_PEER_RT_EN_PAUSE,
+					 UDMA_PEER_RT_EN_PAUSE);
+		break;
+	case DMA_MEM_TO_MEM:
+		udma_tchanrt_update_bits(uc->tchan, UDMA_TCHAN_RT_CTL_REG,
+					 UDMA_CHAN_RT_CTL_PAUSE,
+					 UDMA_CHAN_RT_CTL_PAUSE);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int udma_resume(struct dma_chan *chan)
+{
+	struct udma_chan *uc = to_udma_chan(chan);
+
+	if (!uc->desc)
+		return -EINVAL;
+
+	/* resume the channel */
+	switch (uc->desc->dir) {
+	case DMA_DEV_TO_MEM:
+		udma_rchanrt_update_bits(uc->rchan,
+					 UDMA_RCHAN_RT_PEER_RT_EN_REG,
+					 UDMA_PEER_RT_EN_PAUSE, 0);
+
+		break;
+	case DMA_MEM_TO_DEV:
+		udma_tchanrt_update_bits(uc->tchan,
+					 UDMA_TCHAN_RT_PEER_RT_EN_REG,
+					 UDMA_PEER_RT_EN_PAUSE, 0);
+		break;
+	case DMA_MEM_TO_MEM:
+		udma_tchanrt_update_bits(uc->tchan, UDMA_TCHAN_RT_CTL_REG,
+					 UDMA_CHAN_RT_CTL_PAUSE, 0);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int udma_terminate_all(struct dma_chan *chan)
+{
+	struct udma_chan *uc = to_udma_chan(chan);
+	unsigned long flags;
+	LIST_HEAD(head);
+
+	spin_lock_irqsave(&uc->vc.lock, flags);
+
+	if (udma_is_chan_running(uc))
+		udma_stop(uc);
+
+	if (uc->desc) {
+		uc->terminated_desc = uc->desc;
+		uc->desc = NULL;
+		uc->terminated_desc->terminated = true;
+		cancel_delayed_work(&uc->tx_drain.work);
+	}
+
+	uc->paused = false;
+
+	vchan_get_all_descriptors(&uc->vc, &head);
+	spin_unlock_irqrestore(&uc->vc.lock, flags);
+	vchan_dma_desc_free_list(&uc->vc, &head);
+
+	return 0;
+}
+
+static void udma_synchronize(struct dma_chan *chan)
+{
+	struct udma_chan *uc = to_udma_chan(chan);
+	unsigned long timeout = msecs_to_jiffies(1000);
+
+	vchan_synchronize(&uc->vc);
+
+	if (uc->state == UDMA_CHAN_IS_TERMINATING) {
+		timeout = wait_for_completion_timeout(&uc->teardown_completed,
+						      timeout);
+		if (!timeout) {
+			dev_warn(uc->ud->dev, "chan%d teardown timeout!\n",
+				 uc->id);
+			udma_dump_chan_stdata(uc);
+			udma_reset_chan(uc, true);
+		}
+	}
+
+	udma_reset_chan(uc, false);
+	if (udma_is_chan_running(uc))
+		dev_warn(uc->ud->dev, "chan%d refused to stop!\n", uc->id);
+
+	cancel_delayed_work_sync(&uc->tx_drain.work);
+	udma_reset_rings(uc);
+}
+
+static void udma_desc_pre_callback(struct virt_dma_chan *vc,
+				   struct virt_dma_desc *vd,
+				   struct dmaengine_result *result)
+{
+	struct udma_chan *uc = to_udma_chan(&vc->chan);
+	struct udma_desc *d;
+
+	if (!vd)
+		return;
+
+	d = to_udma_desc(&vd->tx);
+
+	if (d->metadata_size)
+		udma_fetch_epib(uc, d);
+
+	/* Provide residue information for the client */
+	if (result) {
+		void *desc_vaddr = udma_curr_cppi5_desc_vaddr(d, d->desc_idx);
+
+		if (cppi5_desc_get_type(desc_vaddr) ==
+		    CPPI5_INFO0_DESC_TYPE_VAL_HOST) {
+			result->residue = d->residue -
+					  cppi5_hdesc_get_pktlen(desc_vaddr);
+			if (result->residue)
+				result->result = DMA_TRANS_ABORTED;
+			else
+				result->result = DMA_TRANS_NOERROR;
+		} else {
+			result->residue = 0;
+			result->result = DMA_TRANS_NOERROR;
+		}
+	}
+}
+
+/*
+ * This tasklet handles the completion of a DMA descriptor by
+ * calling its callback and freeing it.
+ */
+static void udma_vchan_complete(unsigned long arg)
+{
+	struct virt_dma_chan *vc = (struct virt_dma_chan *)arg;
+	struct virt_dma_desc *vd, *_vd;
+	struct dmaengine_desc_callback cb;
+	LIST_HEAD(head);
+
+	spin_lock_irq(&vc->lock);
+	list_splice_tail_init(&vc->desc_completed, &head);
+	vd = vc->cyclic;
+	if (vd) {
+		vc->cyclic = NULL;
+		dmaengine_desc_get_callback(&vd->tx, &cb);
+	} else {
+		memset(&cb, 0, sizeof(cb));
+	}
+	spin_unlock_irq(&vc->lock);
+
+	udma_desc_pre_callback(vc, vd, NULL);
+	dmaengine_desc_callback_invoke(&cb, NULL);
+
+	list_for_each_entry_safe(vd, _vd, &head, node) {
+		struct dmaengine_result result;
+
+		dmaengine_desc_get_callback(&vd->tx, &cb);
+
+		list_del(&vd->node);
+
+		udma_desc_pre_callback(vc, vd, &result);
+		dmaengine_desc_callback_invoke(&cb, &result);
+
+		vchan_vdesc_fini(vd);
+	}
+}
+
+static void udma_free_chan_resources(struct dma_chan *chan)
+{
+	struct udma_chan *uc = to_udma_chan(chan);
+	struct udma_dev *ud = to_udma_dev(chan->device);
+	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
+
+	udma_terminate_all(chan);
+	if (uc->terminated_desc) {
+		udma_reset_chan(uc, false);
+		udma_reset_rings(uc);
+	}
+
+	cancel_delayed_work_sync(&uc->tx_drain.work);
+	destroy_delayed_work_on_stack(&uc->tx_drain.work);
+
+	if (uc->irq_num_ring > 0) {
+		free_irq(uc->irq_num_ring, uc);
+
+		ti_sci_inta_unregister_event(ud->dev, uc->irq_ra_tisci,
+					     uc->irq_ra_idx, uc->irq_num_ring);
+		uc->irq_num_ring = 0;
+	}
+	if (uc->irq_num_udma > 0) {
+		free_irq(uc->irq_num_udma, uc);
+
+		ti_sci_inta_unregister_event(ud->dev, tisci_rm->tisci_dev_id,
+					     uc->irq_udma_idx,
+					     uc->irq_num_udma);
+		uc->irq_num_udma = 0;
+	}
+
+	/* Release PSI-L pairing */
+	if (uc->psil_paired) {
+		navss_psil_unpair(ud, uc->src_thread, uc->dst_thread);
+		uc->psil_paired = false;
+	}
+
+	vchan_free_chan_resources(&uc->vc);
+	tasklet_kill(&uc->vc.task);
+
+	pm_runtime_put(ud->ddev.dev);
+
+	udma_free_tx_resources(uc);
+	udma_free_rx_resources(uc);
+	udma_reset_uchan(uc);
+
+	if (uc->use_dma_pool) {
+		dma_pool_destroy(uc->hdesc_pool);
+		uc->use_dma_pool = false;
+	}
+}
+
+static struct platform_driver udma_driver;
+
+static bool udma_dma_filter_fn(struct dma_chan *chan, void *param)
+{
+	u32 *args;
+	struct udma_chan *uc;
+	struct udma_dev *ud;
+	struct device_node *chconf_node, *slave_node;
+	char prop[50];
+	u32 val;
+
+	if (chan->device->dev->driver != &udma_driver.driver)
+		return false;
+
+	uc = to_udma_chan(chan);
+	ud = uc->ud;
+	args = param;
+
+	if (args[2] == UDMA_DIR_TX) {
+		uc->dir = DMA_MEM_TO_DEV;
+	} else if (args[2] == UDMA_DIR_RX) {
+		uc->dir = DMA_DEV_TO_MEM;
+	} else {
+		dev_err(ud->dev, "Invalid direction (%u)\n", args[2]);
+		return false;
+	}
+
+	slave_node = of_find_node_by_phandle(args[0]);
+	if (!slave_node) {
+		dev_err(ud->dev, "Slave node is missing\n");
+		uc->dir = DMA_MEM_TO_MEM;
+		return false;
+	}
+
+	if (of_property_read_u32(slave_node, "ti,psil-base", &val)) {
+		dev_err(ud->dev, "ti,psil-base is missing\n");
+		uc->dir = DMA_MEM_TO_MEM;
+		return false;
+	}
+
+	uc->remote_thread_id = val + args[1];
+
+	snprintf(prop, sizeof(prop), "ti,psil-config%u", args[1]);
+	/* Does of_node_put on slave_node */
+	chconf_node = of_find_node_by_name(slave_node, prop);
+	if (!chconf_node) {
+		dev_err(ud->dev, "Channel configuration node is missing\n");
+		uc->dir = DMA_MEM_TO_MEM;
+		uc->remote_thread_id = -1;
+		return false;
+	}
+
+	if (!of_property_read_u32(chconf_node, "linux,udma-mode", &val)) {
+		if (val == UDMA_PKT_MODE)
+			uc->pkt_mode = true;
+	}
+
+	if (!of_property_read_u32(chconf_node, "statictr-type", &val))
+		uc->static_tr_type = val;
+
+	if (uc->static_tr_type == PSIL_STATIC_TR_XY) {
+		uc->enable_acc32 = of_property_read_bool(chconf_node,
+							 "enable-acc32");
+		uc->enable_burst = of_property_read_bool(chconf_node,
+							 "enable-burst");
+	}
+
+	if (!of_property_read_u32(chconf_node, "ti,channel-tpl", &val))
+		uc->channel_tpl = val;
+
+	uc->notdpkt = of_property_read_bool(chconf_node, "ti,notdpkt");
+
+	uc->needs_epib = of_property_read_bool(chconf_node, "ti,needs-epib");
+	if (!of_property_read_u32(chconf_node, "ti,psd-size", &val))
+		uc->psd_size = val;
+	uc->metadata_size = (uc->needs_epib ? CPPI5_INFO0_HDESC_EPIB_SIZE : 0) +
+			    uc->psd_size;
+
+	if (uc->pkt_mode)
+		uc->hdesc_size = ALIGN(sizeof(struct cppi5_host_desc_t) +
+				 uc->metadata_size, ud->desc_align);
+
+	of_node_put(chconf_node);
+
+	dev_dbg(ud->dev, "chan%d: Remote thread: 0x%04x (%s)\n", uc->id,
+		uc->remote_thread_id, udma_get_dir_text(uc->dir));
+
+	return true;
+}
+
+static struct dma_chan *udma_of_xlate(struct of_phandle_args *dma_spec,
+				      struct of_dma *ofdma)
+{
+	struct udma_dev *ud = ofdma->of_dma_data;
+	struct dma_chan *chan;
+
+	if (dma_spec->args_count != 3)
+		return NULL;
+
+	chan = dmadev_get_slave_channel(&ud->ddev, udma_dma_filter_fn,
+					&dma_spec->args[0]);
+	if (!chan) {
+		dev_err(ud->dev, "get channel fail in %s.\n", __func__);
+		return ERR_PTR(-EINVAL);
+	}
+
+	return chan;
+}
+
+static struct udma_match_data am654_main_data = {
+	.enable_memcpy_support = true,
+	.have_acc32 = false,
+	.have_burst = false,
+	.statictr_z_mask = GENMASK(11, 0),
+	.rchan_oes_offset = 0x2000,
+	.tpl_levels = 2,
+	.level_start_idx = {
+		[0] = 8, /* Normal channels */
+		[1] = 0, /* High Throughput channels */
+	},
+};
+
+static struct udma_match_data am654_mcu_data = {
+	.enable_memcpy_support = false, /* MEM_TO_MEM is slow via MCU UDMA */
+	.have_acc32 = false,
+	.have_burst = false,
+	.statictr_z_mask = GENMASK(11, 0),
+	.rchan_oes_offset = 0x2000,
+	.tpl_levels = 2,
+	.level_start_idx = {
+		[0] = 2, /* Normal channels */
+		[1] = 0, /* High Throughput channels */
+	},
+};
+
+static struct udma_match_data j721e_main_data = {
+	.enable_memcpy_support = true,
+	.have_acc32 = true,
+	.have_burst = true,
+	.statictr_z_mask = GENMASK(23, 0),
+	.rchan_oes_offset = 0x400,
+	.tpl_levels = 3,
+	.level_start_idx = {
+		[0] = 16, /* Normal channels */
+		[1] = 4, /* High Throughput channels */
+		[2] = 0, /* Ultra High Throughput channels */
+	},
+};
+
+static struct udma_match_data j721e_mcu_data = {
+	.enable_memcpy_support = false, /* MEM_TO_MEM is slow via MCU UDMA */
+	.have_acc32 = true,
+	.have_burst = true,
+	.statictr_z_mask = GENMASK(23, 0),
+	.rchan_oes_offset = 0x400,
+	.tpl_levels = 2,
+	.level_start_idx = {
+		[0] = 2, /* Normal channels */
+		[1] = 0, /* High Throughput channels */
+	},
+};
+
+static const struct of_device_id udma_of_match[] = {
+	{
+		.compatible = "ti,am654-navss-main-udmap",
+		.data = &am654_main_data,
+	}, {
+		.compatible = "ti,am654-navss-mcu-udmap",
+		.data = &am654_mcu_data,
+	}, {
+		.compatible = "ti,j721e-navss-main-udmap",
+		.data = &j721e_main_data,
+	}, {
+		.compatible = "ti,j721e-navss-mcu-udmap",
+		.data = &j721e_mcu_data,
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(of, udma_of_match);
+
+static int udma_get_mmrs(struct platform_device *pdev, struct udma_dev *ud)
+{
+	struct resource *res;
+	int i;
+
+	for (i = 0; i < MMR_LAST; i++) {
+		res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+						   mmr_names[i]);
+		ud->mmrs[i] = devm_ioremap_resource(&pdev->dev, res);
+		if (IS_ERR(ud->mmrs[i]))
+			return PTR_ERR(ud->mmrs[i]);
+	}
+
+	return 0;
+}
+
+static int udma_setup_resources(struct udma_dev *ud)
+{
+	struct device *dev = ud->dev;
+	int ch_count, i;
+	u32 cap2, cap3;
+	struct ti_sci_resource_desc *rm_desc;
+	struct ti_sci_resource *rm_res;
+	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
+	char *range_names[] = { "ti,sci-rm-range-tchan",
+				"ti,sci-rm-range-rchan",
+				"ti,sci-rm-range-rflow" };
+
+	cap2 = udma_read(ud->mmrs[MMR_GCFG], 0x28);
+	cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c);
+
+	ud->rflow_cnt = cap3 & 0x3fff;
+	ud->tchan_cnt = cap2 & 0x1ff;
+	ud->echan_cnt = (cap2 >> 9) & 0x1ff;
+	ud->rchan_cnt = (cap2 >> 18) & 0x1ff;
+	ch_count  = ud->tchan_cnt + ud->rchan_cnt;
+
+	ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt),
+					   sizeof(unsigned long), GFP_KERNEL);
+	ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans),
+				  GFP_KERNEL);
+	ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt),
+					   sizeof(unsigned long), GFP_KERNEL);
+	ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans),
+				  GFP_KERNEL);
+	ud->rflow_gp_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rflow_cnt),
+					      sizeof(unsigned long),
+					      GFP_KERNEL);
+	ud->rflow_gp_map_allocated = devm_kcalloc(dev,
+						  BITS_TO_LONGS(ud->rflow_cnt),
+						  sizeof(unsigned long),
+						  GFP_KERNEL);
+	ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt),
+					sizeof(unsigned long),
+					GFP_KERNEL);
+	ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows),
+				  GFP_KERNEL);
+
+	if (!ud->tchan_map || !ud->rchan_map || !ud->rflow_gp_map ||
+	    !ud->rflow_gp_map_allocated || !ud->tchans || !ud->rchans ||
+	    !ud->rflows || !ud->rflow_in_use)
+		return -ENOMEM;
+
+	/*
+	 * RX flows with the same Ids as RX channels are reserved to be used
+	 * as default flows if remote HW can't generate flow_ids. Those
+	 * RX flows can be requested only explicitly by id.
+	 */
+	bitmap_set(ud->rflow_gp_map_allocated, 0, ud->rchan_cnt);
+
+	/* by default no GP rflows are assigned to Linux */
+	bitmap_set(ud->rflow_gp_map, 0, ud->rflow_cnt);
+
+	/* Get resource ranges from tisci */
+	for (i = 0; i < RM_RANGE_LAST; i++)
+		tisci_rm->rm_ranges[i] = devm_ti_sci_get_of_resource(
+							tisci_rm->tisci, dev,
+							tisci_rm->tisci_dev_id,
+							range_names[i]);
+
+	/* tchan ranges */
+	rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
+	if (IS_ERR(rm_res)) {
+		bitmap_zero(ud->tchan_map, ud->tchan_cnt);
+	} else {
+		bitmap_fill(ud->tchan_map, ud->tchan_cnt);
+		for (i = 0; i < rm_res->sets; i++) {
+			rm_desc = &rm_res->desc[i];
+			bitmap_clear(ud->tchan_map, rm_desc->start,
+				     rm_desc->num);
+			dev_dbg(dev, "ti-sci-res: tchan: %d:%d\n",
+				rm_desc->start, rm_desc->num);
+		}
+	}
+
+	/* rchan and matching default flow ranges */
+	rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
+	if (IS_ERR(rm_res)) {
+		bitmap_zero(ud->rchan_map, ud->rchan_cnt);
+	} else {
+		bitmap_fill(ud->rchan_map, ud->rchan_cnt);
+		for (i = 0; i < rm_res->sets; i++) {
+			rm_desc = &rm_res->desc[i];
+			bitmap_clear(ud->rchan_map, rm_desc->start,
+				     rm_desc->num);
+			dev_dbg(dev, "ti-sci-res: rchan: %d:%d\n",
+				rm_desc->start, rm_desc->num);
+		}
+	}
+
+	/* GP rflow ranges */
+	rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW];
+	if (IS_ERR(rm_res)) {
+		/* all gp flows are assigned exclusively to Linux */
+		bitmap_clear(ud->rflow_gp_map, ud->rchan_cnt,
+			     ud->rflow_cnt - ud->rchan_cnt);
+	} else {
+		for (i = 0; i < rm_res->sets; i++) {
+			rm_desc = &rm_res->desc[i];
+			bitmap_clear(ud->rflow_gp_map, rm_desc->start,
+				     rm_desc->num);
+			dev_dbg(dev, "ti-sci-res: rflow: %d:%d\n",
+				rm_desc->start, rm_desc->num);
+		}
+	}
+
+	ch_count -= bitmap_weight(ud->tchan_map, ud->tchan_cnt);
+	ch_count -= bitmap_weight(ud->rchan_map, ud->rchan_cnt);
+	if (!ch_count)
+		return -ENODEV;
+
+	ud->channels = devm_kcalloc(dev, ch_count, sizeof(*ud->channels),
+				    GFP_KERNEL);
+	if (!ud->channels)
+		return -ENOMEM;
+
+	dev_info(dev,
+		 "Channels: %d (tchan: %u, echan: %u, rchan: %u, rflow: %u)\n",
+		 ch_count, ud->tchan_cnt, ud->echan_cnt, ud->rchan_cnt,
+		 ud->rflow_cnt);
+
+	return ch_count;
+}
+
+static int udma_setup_rx_flush(struct udma_dev *ud)
+{
+	struct udma_rx_flush *rx_flush = &ud->rx_flush;
+	struct cppi5_desc_hdr_t *tr_desc;
+	struct cppi5_tr_type1_t *tr_req;
+	struct cppi5_host_desc_t *desc;
+	struct device *dev = ud->dev;
+	struct udma_hwdesc *hwdesc;
+	size_t tr_size;
+
+	/* Allocate 1K buffer for discarded data on RX channel teardown */
+	rx_flush->buffer_size = SZ_1K;
+	rx_flush->buffer_vaddr = devm_kzalloc(dev, rx_flush->buffer_size,
+					      GFP_KERNEL);
+	if (!rx_flush->buffer_vaddr)
+		return -ENOMEM;
+
+	rx_flush->buffer_paddr = dma_map_single(dev, rx_flush->buffer_vaddr,
+						rx_flush->buffer_size,
+						DMA_TO_DEVICE);
+	if (dma_mapping_error(dev, rx_flush->buffer_paddr))
+		return -ENOMEM;
+
+	/* Set up descriptor to be used for TR mode */
+	hwdesc = &rx_flush->hwdescs[0];
+	tr_size = sizeof(struct cppi5_tr_type1_t);
+	hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size, 1);
+	hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size,
+					ud->desc_align);
+
+	hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size,
+						GFP_KERNEL);
+	if (!hwdesc->cppi5_desc_vaddr)
+		return -ENOMEM;
+
+	hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr,
+						  hwdesc->cppi5_desc_size,
+						  DMA_TO_DEVICE);
+	if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr))
+		return -ENOMEM;
+
+	/* Start of the TR req records */
+	hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size;
+	/* Start address of the TR response array */
+	hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size;
+
+	tr_desc = hwdesc->cppi5_desc_vaddr;
+	cppi5_trdesc_init(tr_desc, 1, tr_size, 0, 0);
+	cppi5_desc_set_pktids(tr_desc, 0, 0x3fff);
+	cppi5_desc_set_retpolicy(tr_desc, 0, 0);
+
+	tr_req = hwdesc->tr_req_base;
+	cppi5_tr_init(&tr_req->flags, CPPI5_TR_TYPE1, false, false,
+		      CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
+	cppi5_tr_csf_set(&tr_req->flags, CPPI5_TR_CSF_SUPR_EVT);
+
+	tr_req->addr = rx_flush->buffer_paddr;
+	tr_req->icnt0 = rx_flush->buffer_size;
+	tr_req->icnt1 = 1;
+
+	/* Set up descriptor to be used for packet mode */
+	hwdesc = &rx_flush->hwdescs[1];
+	hwdesc->cppi5_desc_size = ALIGN(sizeof(struct cppi5_host_desc_t) +
+					CPPI5_INFO0_HDESC_EPIB_SIZE +
+					CPPI5_INFO0_HDESC_PSDATA_MAX_SIZE,
+					ud->desc_align);
+
+	hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size,
+						GFP_KERNEL);
+	if (!hwdesc->cppi5_desc_vaddr)
+		return -ENOMEM;
+
+	hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr,
+						  hwdesc->cppi5_desc_size,
+						  DMA_TO_DEVICE);
+	if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr))
+		return -ENOMEM;
+
+	desc = hwdesc->cppi5_desc_vaddr;
+	cppi5_hdesc_init(desc, 0, 0);
+	cppi5_desc_set_pktids(&desc->hdr, 0, 0x3fff);
+	cppi5_desc_set_retpolicy(&desc->hdr, 0, 0);
+
+	cppi5_hdesc_attach_buf(desc,
+			       rx_flush->buffer_paddr, rx_flush->buffer_size,
+			       rx_flush->buffer_paddr, rx_flush->buffer_size);
+
+	dma_sync_single_for_device(dev, hwdesc->cppi5_desc_paddr,
+				   hwdesc->cppi5_desc_size, DMA_TO_DEVICE);
+	return 0;
+}
+
+#define TI_UDMAC_BUSWIDTHS	(BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
+				 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
+				 BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
+				 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
+				 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES))
+
+static int udma_probe(struct platform_device *pdev)
+{
+	struct device_node *parent_irq_node;
+	struct device_node *navss_node = pdev->dev.parent->of_node;
+	struct device *dev = &pdev->dev;
+	struct udma_dev *ud;
+	const struct of_device_id *match;
+	int i, ret;
+	int ch_count;
+
+	ret = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(48));
+	if (ret)
+		dev_err(dev, "failed to set dma mask stuff\n");
+
+	ud = devm_kzalloc(dev, sizeof(*ud), GFP_KERNEL);
+	if (!ud)
+		return -ENOMEM;
+
+	ret = udma_get_mmrs(pdev, ud);
+	if (ret)
+		return ret;
+
+	ud->tisci_rm.tisci = ti_sci_get_by_phandle(dev->of_node, "ti,sci");
+	if (IS_ERR(ud->tisci_rm.tisci))
+		return PTR_ERR(ud->tisci_rm.tisci);
+
+	ret = of_property_read_u32(dev->of_node, "ti,sci-dev-id",
+				   &ud->tisci_rm.tisci_dev_id);
+	if (ret) {
+		dev_err(dev, "ti,sci-dev-id read failure %d\n", ret);
+		return ret;
+	}
+
+	ret = of_property_read_u32(navss_node, "ti,sci-dev-id",
+				   &ud->tisci_rm.tisci_navss_dev_id);
+	if (ret) {
+		dev_err(dev, "NAVSS ti,sci-dev-id read failure %d\n", ret);
+		return ret;
+	}
+
+	ud->tisci_rm.tisci_udmap_ops = &ud->tisci_rm.tisci->ops.rm_udmap_ops;
+	ud->tisci_rm.tisci_psil_ops = &ud->tisci_rm.tisci->ops.rm_psil_ops;
+
+	ud->ringacc = of_k3_ringacc_get_by_phandle(dev->of_node, "ti,ringacc");
+	if (IS_ERR(ud->ringacc))
+		return PTR_ERR(ud->ringacc);
+
+	parent_irq_node = of_irq_find_parent(dev->of_node);
+	if (!parent_irq_node) {
+		dev_err(dev, "Failed to get IRQ parent node\n");
+		return -ENODEV;
+	}
+
+	ud->irq_domain = irq_find_host(parent_irq_node);
+	if (!ud->irq_domain)
+		return -EPROBE_DEFER;
+
+	match = of_match_node(udma_of_match, dev->of_node);
+	if (!match) {
+		dev_err(dev, "No compatible match found\n");
+		return -ENODEV;
+	}
+	ud->match_data = match->data;
+
+	dma_cap_set(DMA_SLAVE, ud->ddev.cap_mask);
+	dma_cap_set(DMA_CYCLIC, ud->ddev.cap_mask);
+
+	ud->ddev.device_alloc_chan_resources = udma_alloc_chan_resources;
+	ud->ddev.device_config = udma_slave_config;
+	ud->ddev.device_prep_slave_sg = udma_prep_slave_sg;
+	ud->ddev.device_prep_dma_cyclic = udma_prep_dma_cyclic;
+	ud->ddev.device_issue_pending = udma_issue_pending;
+	ud->ddev.device_tx_status = udma_tx_status;
+	ud->ddev.device_pause = udma_pause;
+	ud->ddev.device_resume = udma_resume;
+	ud->ddev.device_terminate_all = udma_terminate_all;
+	ud->ddev.device_synchronize = udma_synchronize;
+
+	ud->ddev.device_free_chan_resources = udma_free_chan_resources;
+	ud->ddev.src_addr_widths = TI_UDMAC_BUSWIDTHS;
+	ud->ddev.dst_addr_widths = TI_UDMAC_BUSWIDTHS;
+	ud->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
+	ud->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
+	ud->ddev.copy_align = DMAENGINE_ALIGN_8_BYTES;
+	ud->ddev.desc_metadata_modes = DESC_METADATA_CLIENT |
+				       DESC_METADATA_ENGINE;
+	if (ud->match_data->enable_memcpy_support) {
+		dma_cap_set(DMA_MEMCPY, ud->ddev.cap_mask);
+		ud->ddev.device_prep_dma_memcpy = udma_prep_dma_memcpy;
+		ud->ddev.directions |= BIT(DMA_MEM_TO_MEM);
+	}
+
+	ud->ddev.dev = dev;
+	ud->dev = dev;
+
+	INIT_LIST_HEAD(&ud->ddev.channels);
+	INIT_LIST_HEAD(&ud->desc_to_purge);
+
+	ret = of_property_read_u32(dev->of_node, "ti,psil-base",
+				   &ud->psil_base);
+	if (ret) {
+		dev_info(dev, "Missing ti,psil-base property, using %d.\n",
+			 ret);
+		return ret;
+	}
+
+	ch_count = udma_setup_resources(ud);
+	if (ch_count <= 0)
+		return ch_count;
+
+	spin_lock_init(&ud->lock);
+	INIT_WORK(&ud->purge_work, udma_purge_desc_work);
+
+	ud->desc_align = 64;
+	if (ud->desc_align < dma_get_cache_alignment())
+		ud->desc_align = dma_get_cache_alignment();
+
+	ret = udma_setup_rx_flush(ud);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < ud->tchan_cnt; i++) {
+		struct udma_tchan *tchan = &ud->tchans[i];
+
+		tchan->id = i;
+		tchan->reg_rt = ud->mmrs[MMR_TCHANRT] + i * 0x1000;
+	}
+
+	for (i = 0; i < ud->rchan_cnt; i++) {
+		struct udma_rchan *rchan = &ud->rchans[i];
+
+		rchan->id = i;
+		rchan->reg_rt = ud->mmrs[MMR_RCHANRT] + i * 0x1000;
+	}
+
+	for (i = 0; i < ud->rflow_cnt; i++) {
+		struct udma_rflow *rflow = &ud->rflows[i];
+
+		rflow->id = i;
+	}
+
+	for (i = 0; i < ch_count; i++) {
+		struct udma_chan *uc = &ud->channels[i];
+
+		uc->ud = ud;
+		uc->vc.desc_free = udma_desc_free;
+		uc->id = i;
+		uc->remote_thread_id = -1;
+		uc->tchan = NULL;
+		uc->rchan = NULL;
+		uc->dir = DMA_MEM_TO_MEM;
+		uc->name = devm_kasprintf(dev, GFP_KERNEL, "%s chan%d",
+					  dev_name(dev), i);
+
+		vchan_init(&uc->vc, &ud->ddev);
+		/* Use custom vchan completion handling */
+		tasklet_init(&uc->vc.task, udma_vchan_complete,
+			     (unsigned long)&uc->vc);
+		init_completion(&uc->teardown_completed);
+	}
+
+	ret = dma_async_device_register(&ud->ddev);
+	if (ret) {
+		dev_err(dev, "failed to register slave DMA engine: %d\n", ret);
+		return ret;
+	}
+
+	platform_set_drvdata(pdev, ud);
+
+	ret = of_dma_controller_register(dev->of_node, udma_of_xlate, ud);
+	if (ret) {
+		dev_err(dev, "failed to register of_dma controller\n");
+		dma_async_device_unregister(&ud->ddev);
+	}
+
+	return ret;
+}
+
+static int udma_remove(struct platform_device *pdev)
+{
+	struct udma_dev *ud = platform_get_drvdata(pdev);
+
+	of_dma_controller_free(pdev->dev.of_node);
+	dma_async_device_unregister(&ud->ddev);
+
+	/* Make sure that we did proper cleanup */
+	cancel_work_sync(&ud->purge_work);
+	udma_purge_desc_work(&ud->purge_work);
+
+	pm_runtime_put_sync(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+
+	return 0;
+}
+
+static struct platform_driver udma_driver = {
+	.driver = {
+		.name	= "ti-udma",
+		.of_match_table = udma_of_match,
+	},
+	.probe		= udma_probe,
+	.remove		= udma_remove,
+};
+
+module_platform_driver(udma_driver);
+
+/* Private interfaces to UDMA */
+#include "k3-udma-private.c"
+
+MODULE_ALIAS("platform:ti-udma");
+MODULE_DESCRIPTION("TI K3 DMA driver for CPPI 5.0 compliant devices");
+MODULE_AUTHOR("Peter Ujfalusi <peter.ujfalusi@ti.com>");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/dma/ti/k3-udma.h b/drivers/dma/ti/k3-udma.h
--- a/drivers/dma/ti/k3-udma.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/dma/ti/k3-udma.h	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,164 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ *  Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
+ */
+
+#ifndef K3_UDMA_H_
+#define K3_UDMA_H_
+
+#define UDMA_PSIL_DST_THREAD_ID_OFFSET 0x8000
+
+/* Global registers */
+#define UDMA_REV_REG			0x0
+#define UDMA_PERF_CTL_REG		0x4
+#define UDMA_EMU_CTL_REG		0x8
+#define UDMA_PSIL_TO_REG		0x10
+#define UDMA_UTC_CTL_REG		0x1c
+#define UDMA_CAP_REG(i)			(0x20 + ((i) * 4))
+#define UDMA_RX_FLOW_ID_FW_OES_REG	0x80
+#define UDMA_RX_FLOW_ID_FW_STATUS_REG	0x88
+
+/* TX chan RT regs */
+#define UDMA_TCHAN_RT_CTL_REG		0x0
+#define UDMA_TCHAN_RT_SWTRIG_REG	0x8
+#define UDMA_TCHAN_RT_STDATA_REG	0x80
+
+#define UDMA_TCHAN_RT_PEERn_REG(i)	(0x200 + ((i) * 0x4))
+#define UDMA_TCHAN_RT_PEER_STATIC_TR_XY_REG	\
+	UDMA_TCHAN_RT_PEERn_REG(0)	/* PSI-L: 0x400 */
+#define UDMA_TCHAN_RT_PEER_STATIC_TR_Z_REG	\
+	UDMA_TCHAN_RT_PEERn_REG(1)	/* PSI-L: 0x401 */
+#define UDMA_TCHAN_RT_PEER_BCNT_REG		\
+	UDMA_TCHAN_RT_PEERn_REG(4)	/* PSI-L: 0x404 */
+#define UDMA_TCHAN_RT_PEER_RT_EN_REG		\
+	UDMA_TCHAN_RT_PEERn_REG(8)	/* PSI-L: 0x408 */
+
+#define UDMA_TCHAN_RT_PCNT_REG		0x400
+#define UDMA_TCHAN_RT_BCNT_REG		0x408
+#define UDMA_TCHAN_RT_SBCNT_REG		0x410
+
+/* RX chan RT regs */
+#define UDMA_RCHAN_RT_CTL_REG		0x0
+#define UDMA_RCHAN_RT_SWTRIG_REG	0x8
+#define UDMA_RCHAN_RT_STDATA_REG	0x80
+
+#define UDMA_RCHAN_RT_PEERn_REG(i)	(0x200 + ((i) * 0x4))
+#define UDMA_RCHAN_RT_PEER_STATIC_TR_XY_REG	\
+	UDMA_RCHAN_RT_PEERn_REG(0)	/* PSI-L: 0x400 */
+#define UDMA_RCHAN_RT_PEER_STATIC_TR_Z_REG	\
+	UDMA_RCHAN_RT_PEERn_REG(1)	/* PSI-L: 0x401 */
+#define UDMA_RCHAN_RT_PEER_BCNT_REG		\
+	UDMA_RCHAN_RT_PEERn_REG(4)	/* PSI-L: 0x404 */
+#define UDMA_RCHAN_RT_PEER_RT_EN_REG		\
+	UDMA_RCHAN_RT_PEERn_REG(8)	/* PSI-L: 0x408 */
+
+#define UDMA_RCHAN_RT_PCNT_REG		0x400
+#define UDMA_RCHAN_RT_BCNT_REG		0x408
+#define UDMA_RCHAN_RT_SBCNT_REG		0x410
+
+/* UDMA_TCHAN_RT_CTL_REG/UDMA_RCHAN_RT_CTL_REG */
+#define UDMA_CHAN_RT_CTL_EN	BIT(31)
+#define UDMA_CHAN_RT_CTL_TDOWN	BIT(30)
+#define UDMA_CHAN_RT_CTL_PAUSE	BIT(29)
+#define UDMA_CHAN_RT_CTL_FTDOWN	BIT(28)
+#define UDMA_CHAN_RT_CTL_ERROR	BIT(0)
+
+/* UDMA_TCHAN_RT_PEER_RT_EN_REG/UDMA_RCHAN_RT_PEER_RT_EN_REG (PSI-L: 0x408) */
+#define UDMA_PEER_RT_EN_ENABLE		BIT(31)
+#define UDMA_PEER_RT_EN_TEARDOWN	BIT(30)
+#define UDMA_PEER_RT_EN_PAUSE		BIT(29)
+#define UDMA_PEER_RT_EN_FLUSH		BIT(28)
+#define UDMA_PEER_RT_EN_IDLE		BIT(1)
+
+/*
+ * UDMA_TCHAN_RT_PEER_STATIC_TR_XY_REG /
+ * UDMA_RCHAN_RT_PEER_STATIC_TR_XY_REG
+ */
+#define PDMA_STATIC_TR_X_MASK		GENMASK(26, 24)
+#define PDMA_STATIC_TR_X_SHIFT		(24)
+#define PDMA_STATIC_TR_Y_MASK		GENMASK(11, 0)
+#define PDMA_STATIC_TR_Y_SHIFT		(0)
+
+#define PDMA_STATIC_TR_Y(x)	\
+	(((x) << PDMA_STATIC_TR_Y_SHIFT) & PDMA_STATIC_TR_Y_MASK)
+#define PDMA_STATIC_TR_X(x)	\
+	(((x) << PDMA_STATIC_TR_X_SHIFT) & PDMA_STATIC_TR_X_MASK)
+
+#define PDMA_STATIC_TR_XY_ACC32		BIT(30)
+#define PDMA_STATIC_TR_XY_BURST		BIT(31)
+
+/*
+ * UDMA_TCHAN_RT_PEER_STATIC_TR_Z_REG /
+ * UDMA_RCHAN_RT_PEER_STATIC_TR_Z_REG
+ */
+#define PDMA_STATIC_TR_Z(x, mask)	((x) & (mask))
+
+/* Private API to UDMA via k3-udma-private */
+#include <linux/of.h>
+#include <linux/dmaengine.h>
+#include <linux/soc/ti/ti_sci_protocol.h>
+
+struct udma_dev;
+struct udma_tchan;
+struct udma_rchan;
+struct udma_rflow;
+
+enum udma_rm_range {
+	RM_RANGE_TCHAN = 0,
+	RM_RANGE_RCHAN,
+	RM_RANGE_RFLOW,
+	RM_RANGE_LAST,
+};
+
+/* Channel Throughput Levels */
+enum udma_tp_level {
+	UDMA_TP_NORMAL = 0,
+	UDMA_TP_HIGH = 1,
+	UDMA_TP_ULTRAHIGH = 2,
+	UDMA_TP_LAST,
+};
+
+struct udma_tisci_rm {
+	const struct ti_sci_handle *tisci;
+	const struct ti_sci_rm_udmap_ops *tisci_udmap_ops;
+	u32  tisci_dev_id;
+
+	/* tisci information for PSI-L thread pairing/unpairing */
+	const struct ti_sci_rm_psil_ops *tisci_psil_ops;
+	u32  tisci_navss_dev_id;
+
+	struct ti_sci_resource *rm_ranges[RM_RANGE_LAST];
+};
+
+/* HACK: Direct access to UDMA resources */
+int xudma_navss_psil_pair(struct udma_dev *ud, u32 src_thread, u32 dst_thread);
+int xudma_navss_psil_unpair(struct udma_dev *ud, u32 src_thread,
+			    u32 dst_thread);
+
+struct udma_dev *of_xudma_dev_get(struct device_node *np, const char *property);
+void xudma_dev_put(struct udma_dev *ud);
+u32 xudma_dev_get_psil_base(struct udma_dev *ud);
+struct udma_tisci_rm *xudma_dev_get_tisci_rm(struct udma_dev *ud);
+
+int xudma_alloc_gp_rflow_range(struct udma_dev *ud, int from, int cnt);
+int xudma_free_gp_rflow_range(struct udma_dev *ud, int from, int cnt);
+
+struct udma_tchan *xudma_tchan_get(struct udma_dev *ud, int id);
+struct udma_rchan *xudma_rchan_get(struct udma_dev *ud, int id);
+struct udma_rflow *xudma_rflow_get(struct udma_dev *ud, int id);
+
+void xudma_tchan_put(struct udma_dev *ud, struct udma_tchan *p);
+void xudma_rchan_put(struct udma_dev *ud, struct udma_rchan *p);
+void xudma_rflow_put(struct udma_dev *ud, struct udma_rflow *p);
+
+int xudma_tchan_get_id(struct udma_tchan *p);
+int xudma_rchan_get_id(struct udma_rchan *p);
+int xudma_rflow_get_id(struct udma_rflow *p);
+
+u32 xudma_tchanrt_read(struct udma_tchan *tchan, int reg);
+void xudma_tchanrt_write(struct udma_tchan *tchan, int reg, u32 val);
+u32 xudma_rchanrt_read(struct udma_rchan *rchan, int reg);
+void xudma_rchanrt_write(struct udma_rchan *rchan, int reg, u32 val);
+bool xudma_rflow_is_gp(struct udma_dev *ud, int id);
+
+#endif /* K3_UDMA_H_ */
diff -Naur --no-dereference a/drivers/dma/ti/k3-udma-private.c b/drivers/dma/ti/k3-udma-private.c
--- a/drivers/dma/ti/k3-udma-private.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/dma/ti/k3-udma-private.c	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,141 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
+ *  Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
+ */
+
+int xudma_navss_psil_pair(struct udma_dev *ud, u32 src_thread, u32 dst_thread)
+{
+	return navss_psil_pair(ud, src_thread, dst_thread);
+}
+EXPORT_SYMBOL(xudma_navss_psil_pair);
+
+int xudma_navss_psil_unpair(struct udma_dev *ud, u32 src_thread, u32 dst_thread)
+{
+	return navss_psil_unpair(ud, src_thread, dst_thread);
+}
+EXPORT_SYMBOL(xudma_navss_psil_unpair);
+
+struct udma_dev *of_xudma_dev_get(struct device_node *np, const char *property)
+{
+	struct device_node *udma_node = np;
+	struct platform_device *pdev;
+	struct udma_dev *ud;
+
+	if (property) {
+		udma_node = of_parse_phandle(np, property, 0);
+		if (!udma_node) {
+			pr_err("UDMA node is not found\n");
+			return ERR_PTR(-ENODEV);
+		}
+	}
+
+	pdev = of_find_device_by_node(udma_node);
+	if (!pdev) {
+		pr_err("UDMA device not found\n");
+		return ERR_PTR(-EPROBE_DEFER);
+	}
+
+	if (np != udma_node)
+		of_node_put(udma_node);
+
+	ud = platform_get_drvdata(pdev);
+	if (!ud) {
+		pr_err("UDMA has not been proped\n");
+		return ERR_PTR(-EPROBE_DEFER);
+	}
+
+	pm_runtime_get_sync(&pdev->dev);
+
+	return ud;
+}
+EXPORT_SYMBOL(of_xudma_dev_get);
+
+void xudma_dev_put(struct udma_dev *ud)
+{
+	pm_runtime_put_sync(ud->ddev.dev);
+}
+EXPORT_SYMBOL(xudma_dev_put);
+
+u32 xudma_dev_get_psil_base(struct udma_dev *ud)
+{
+	return ud->psil_base;
+}
+EXPORT_SYMBOL(xudma_dev_get_psil_base);
+
+struct udma_tisci_rm *xudma_dev_get_tisci_rm(struct udma_dev *ud)
+{
+	return &ud->tisci_rm;
+}
+EXPORT_SYMBOL(xudma_dev_get_tisci_rm);
+
+int xudma_alloc_gp_rflow_range(struct udma_dev *ud, int from, int cnt)
+{
+	return __udma_alloc_gp_rflow_range(ud, from, cnt);
+}
+EXPORT_SYMBOL(xudma_alloc_gp_rflow_range);
+
+int xudma_free_gp_rflow_range(struct udma_dev *ud, int from, int cnt)
+{
+	return __udma_free_gp_rflow_range(ud, from, cnt);
+}
+EXPORT_SYMBOL(xudma_free_gp_rflow_range);
+
+bool xudma_rflow_is_gp(struct udma_dev *ud, int id)
+{
+	return !test_bit(id, ud->rflow_gp_map);
+}
+EXPORT_SYMBOL(xudma_rflow_is_gp);
+
+#define XUDMA_GET_PUT_RESOURCE(res)					\
+struct udma_##res *xudma_##res##_get(struct udma_dev *ud, int id)	\
+{									\
+	return __udma_reserve_##res(ud, false, id);			\
+}									\
+EXPORT_SYMBOL(xudma_##res##_get);					\
+									\
+void xudma_##res##_put(struct udma_dev *ud, struct udma_##res *p)	\
+{									\
+	clear_bit(p->id, ud->res##_map);				\
+}									\
+EXPORT_SYMBOL(xudma_##res##_put)
+XUDMA_GET_PUT_RESOURCE(tchan);
+XUDMA_GET_PUT_RESOURCE(rchan);
+
+struct udma_rflow *xudma_rflow_get(struct udma_dev *ud, int id)
+{
+	return __udma_get_rflow(ud, id);
+}
+EXPORT_SYMBOL(xudma_rflow_get);
+
+void xudma_rflow_put(struct udma_dev *ud, struct udma_rflow *p)
+{
+	__udma_put_rflow(ud, p);
+}
+EXPORT_SYMBOL(xudma_rflow_put);
+
+#define XUDMA_GET_RESOURCE_ID(res)					\
+int xudma_##res##_get_id(struct udma_##res *p)				\
+{									\
+	return p->id;							\
+}									\
+EXPORT_SYMBOL(xudma_##res##_get_id)
+XUDMA_GET_RESOURCE_ID(tchan);
+XUDMA_GET_RESOURCE_ID(rchan);
+XUDMA_GET_RESOURCE_ID(rflow);
+
+/* Exported register access functions */
+#define XUDMA_RT_IO_FUNCTIONS(res)					\
+u32 xudma_##res##rt_read(struct udma_##res *p, int reg)			\
+{									\
+	return udma_##res##rt_read(p, reg);				\
+}									\
+EXPORT_SYMBOL(xudma_##res##rt_read);					\
+									\
+void xudma_##res##rt_write(struct udma_##res *p, int reg, u32 val)	\
+{									\
+	udma_##res##rt_write(p, reg, val);				\
+}									\
+EXPORT_SYMBOL(xudma_##res##rt_write)
+XUDMA_RT_IO_FUNCTIONS(tchan);
+XUDMA_RT_IO_FUNCTIONS(rchan);
diff -Naur --no-dereference a/drivers/dma/ti/Kconfig b/drivers/dma/ti/Kconfig
--- a/drivers/dma/ti/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/dma/ti/Kconfig	2020-06-19 21:00:31.408227715 -0400
@@ -33,5 +33,26 @@
 	  Enable support for the TI sDMA (System DMA or DMA4) controller. This
 	  DMA engine is found on OMAP and DRA7xx parts.
 
+config TI_K3_UDMA
+	tristate "Texas Instruments UDMA support"
+	depends on ARCH_K3 || COMPILE_TEST
+	depends on TI_SCI_PROTOCOL
+	select DMA_ENGINE
+	select DMA_VIRTUAL_CHANNELS
+	select TI_K3_RINGACC
+	default y
+        help
+	  Enable support for the TI UDMA (Unified DMA) controller. This
+	  DMA engine is used in AM65x.
+
+config TI_K3_UDMA_GLUE_LAYER
+	tristate "Texas Instruments UDMA Glue layer for non DMAengine users"
+	depends on ARCH_K3 || COMPILE_TEST
+	depends on TI_K3_UDMA
+	default y
+	help
+	  Say y here to support the K3 NAVSS DMA glue interface
+	  If unsure, say N.
+
 config TI_DMA_CROSSBAR
 	bool
diff -Naur --no-dereference a/drivers/dma/ti/Makefile b/drivers/dma/ti/Makefile
--- a/drivers/dma/ti/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/dma/ti/Makefile	2020-06-19 21:00:31.408227715 -0400
@@ -2,4 +2,6 @@
 obj-$(CONFIG_TI_CPPI41) += cppi41.o
 obj-$(CONFIG_TI_EDMA) += edma.o
 obj-$(CONFIG_DMA_OMAP) += omap-dma.o
+obj-$(CONFIG_TI_K3_UDMA) += k3-udma.o
+obj-$(CONFIG_TI_K3_UDMA_GLUE_LAYER) += k3-navss-udma.o
 obj-$(CONFIG_TI_DMA_CROSSBAR) += dma-crossbar.o
diff -Naur --no-dereference a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
--- a/drivers/dma-buf/dma-buf.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/dma-buf/dma-buf.c	2020-06-19 21:00:31.408227715 -0400
@@ -90,6 +90,10 @@
 
 	dmabuf = file->private_data;
 
+	/* check if buffer supports mmap */
+	if (!dmabuf->ops->mmap)
+		return -EINVAL;
+
 	/* check for overflowing the buffer's size */
 	if (vma->vm_pgoff + vma_pages(vma) >
 	    dmabuf->size >> PAGE_SHIFT)
@@ -404,9 +408,7 @@
 			  || !exp_info->ops
 			  || !exp_info->ops->map_dma_buf
 			  || !exp_info->ops->unmap_dma_buf
-			  || !exp_info->ops->release
-			  || !exp_info->ops->map
-			  || !exp_info->ops->mmap)) {
+			  || !exp_info->ops->release)) {
 		return ERR_PTR(-EINVAL);
 	}
 
@@ -907,6 +909,10 @@
 	if (WARN_ON(!dmabuf || !vma))
 		return -EINVAL;
 
+	/* check if buffer supports mmap */
+	if (!dmabuf->ops->mmap)
+		return -EINVAL;
+
 	/* check for offset overflow */
 	if (pgoff + vma_pages(vma) < pgoff)
 		return -EOVERFLOW;
diff -Naur --no-dereference a/drivers/firmware/ti_sci.c b/drivers/firmware/ti_sci.c
--- a/drivers/firmware/ti_sci.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/firmware/ti_sci.c	2020-06-19 21:00:31.408227715 -0400
@@ -65,18 +65,36 @@
 };
 
 /**
+ * struct ti_sci_rm_type_map - Structure representing TISCI Resource
+ *				management representation of dev_ids.
+ * @dev_id:	TISCI device ID
+ * @type:	Corresponding id as identified by TISCI RM.
+ *
+ * Note: This is used only as a work around for using RM range apis
+ *	for AM654 SoC. For future SoCs dev_id will be used as type
+ *	for RM range APIs. In order to maintain ABI backward compatibility
+ *	type is not being changed for AM654 SoC.
+ */
+struct ti_sci_rm_type_map {
+	u32 dev_id;
+	u16 type;
+};
+
+/**
  * struct ti_sci_desc - Description of SoC integration
- * @host_id:		Host identifier representing the compute entity
+ * @default_host_id:	Host identifier representing the compute entity
  * @max_rx_timeout_ms:	Timeout for communication with SoC (in Milliseconds)
  * @max_msgs: Maximum number of messages that can be pending
  *		  simultaneously in the system
  * @max_msg_size: Maximum size of data per message that can be handled.
+ * @rm_type_map: RM resource type mapping structure.
  */
 struct ti_sci_desc {
-	u8 host_id;
+	u8 default_host_id;
 	int max_rx_timeout_ms;
 	int max_msgs;
 	int max_msg_size;
+	struct ti_sci_rm_type_map *rm_type_map;
 };
 
 /**
@@ -94,6 +112,7 @@
  * @chan_rx:	Receive mailbox channel
  * @minfo:	Message info
  * @node:	list head
+ * @host_id:	Host ID
  * @users:	Number of users of this instance
  */
 struct ti_sci_info {
@@ -110,6 +129,7 @@
 	struct mbox_chan *chan_rx;
 	struct ti_sci_xfers_info minfo;
 	struct list_head node;
+	u8 host_id;
 	/* protected by ti_sci_list_mutex */
 	int users;
 
@@ -370,7 +390,7 @@
 
 	hdr->seq = xfer_id;
 	hdr->type = msg_type;
-	hdr->host = info->desc->host_id;
+	hdr->host = info->host_id;
 	hdr->flags = msg_flags;
 
 	return xfer;
@@ -632,6 +652,7 @@
 
 /**
  * ti_sci_cmd_get_device() - command to request for device managed by TISCI
+ *			     that can be shared with other hosts.
  * @handle:	Pointer to TISCI handle as retrieved by *ti_sci_get_handle
  * @id:		Device Identifier
  *
@@ -639,12 +660,30 @@
  * usage count by balancing get_device with put_device. No refcounting is
  * managed by driver for that purpose.
  *
- * NOTE: The request is for exclusive access for the processor.
- *
  * Return: 0 if all went fine, else return appropriate error.
  */
 static int ti_sci_cmd_get_device(const struct ti_sci_handle *handle, u32 id)
 {
+	return ti_sci_set_device_state(handle, id, 0,
+				       MSG_DEVICE_SW_STATE_ON);
+}
+
+/**
+ * ti_sci_cmd_get_device_exclusive() - command to request for device managed by
+ *				       TISCI that is exclusively owned by the
+ *				       requesting host.
+ * @handle:	Pointer to TISCI handle as retrieved by *ti_sci_get_handle
+ * @id:		Device Identifier
+ *
+ * Request for the device - NOTE: the client MUST maintain integrity of
+ * usage count by balancing get_device with put_device. No refcounting is
+ * managed by driver for that purpose.
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static int ti_sci_cmd_get_device_exclusive(const struct ti_sci_handle *handle,
+					   u32 id)
+{
 	return ti_sci_set_device_state(handle, id,
 				       MSG_FLAG_DEVICE_EXCLUSIVE,
 				       MSG_DEVICE_SW_STATE_ON);
@@ -663,6 +702,26 @@
  */
 static int ti_sci_cmd_idle_device(const struct ti_sci_handle *handle, u32 id)
 {
+	return ti_sci_set_device_state(handle, id, 0,
+				       MSG_DEVICE_SW_STATE_RETENTION);
+}
+
+/**
+ * ti_sci_cmd_idle_device_exclusive() - Command to idle a device managed by
+ *					TISCI that is exclusively owned by
+ *					requesting host.
+ * @handle:	Pointer to TISCI handle as retrieved by *ti_sci_get_handle
+ * @id:		Device Identifier
+ *
+ * Request for the device - NOTE: the client MUST maintain integrity of
+ * usage count by balancing get_device with put_device. No refcounting is
+ * managed by driver for that purpose.
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static int ti_sci_cmd_idle_device_exclusive(const struct ti_sci_handle *handle,
+					    u32 id)
+{
 	return ti_sci_set_device_state(handle, id,
 				       MSG_FLAG_DEVICE_EXCLUSIVE,
 				       MSG_DEVICE_SW_STATE_RETENTION);
@@ -913,7 +972,7 @@
  * Return: 0 if all went well, else returns appropriate error value.
  */
 static int ti_sci_set_clock_state(const struct ti_sci_handle *handle,
-				  u32 dev_id, u8 clk_id,
+				  u32 dev_id, u32 clk_id,
 				  u32 flags, u8 state)
 {
 	struct ti_sci_info *info;
@@ -941,7 +1000,12 @@
 	}
 	req = (struct ti_sci_msg_req_set_clock_state *)xfer->xfer_buf;
 	req->dev_id = dev_id;
-	req->clk_id = clk_id;
+	if (clk_id < 255) {
+		req->clk_id = clk_id;
+	} else {
+		req->clk_id = 255;
+		req->clk_id_32 = clk_id;
+	}
 	req->request_state = state;
 
 	ret = ti_sci_do_xfer(info, xfer);
@@ -973,7 +1037,7 @@
  * Return: 0 if all went well, else returns appropriate error value.
  */
 static int ti_sci_cmd_get_clock_state(const struct ti_sci_handle *handle,
-				      u32 dev_id, u8 clk_id,
+				      u32 dev_id, u32 clk_id,
 				      u8 *programmed_state, u8 *current_state)
 {
 	struct ti_sci_info *info;
@@ -1004,7 +1068,12 @@
 	}
 	req = (struct ti_sci_msg_req_get_clock_state *)xfer->xfer_buf;
 	req->dev_id = dev_id;
-	req->clk_id = clk_id;
+	if (clk_id < 255) {
+		req->clk_id = clk_id;
+	} else {
+		req->clk_id = 255;
+		req->clk_id_32 = clk_id;
+	}
 
 	ret = ti_sci_do_xfer(info, xfer);
 	if (ret) {
@@ -1044,8 +1113,8 @@
  * Return: 0 if all went well, else returns appropriate error value.
  */
 static int ti_sci_cmd_get_clock(const struct ti_sci_handle *handle, u32 dev_id,
-				u8 clk_id, bool needs_ssc, bool can_change_freq,
-				bool enable_input_term)
+				u32 clk_id, bool needs_ssc,
+				bool can_change_freq, bool enable_input_term)
 {
 	u32 flags = 0;
 
@@ -1070,9 +1139,10 @@
  * Return: 0 if all went well, else returns appropriate error value.
  */
 static int ti_sci_cmd_idle_clock(const struct ti_sci_handle *handle,
-				 u32 dev_id, u8 clk_id)
+				 u32 dev_id, u32 clk_id)
 {
-	return ti_sci_set_clock_state(handle, dev_id, clk_id, 0,
+	return ti_sci_set_clock_state(handle, dev_id, clk_id,
+				      MSG_FLAG_CLOCK_ALLOW_FREQ_CHANGE,
 				      MSG_CLOCK_SW_STATE_UNREQ);
 }
 
@@ -1089,9 +1159,10 @@
  * Return: 0 if all went well, else returns appropriate error value.
  */
 static int ti_sci_cmd_put_clock(const struct ti_sci_handle *handle,
-				u32 dev_id, u8 clk_id)
+				u32 dev_id, u32 clk_id)
 {
-	return ti_sci_set_clock_state(handle, dev_id, clk_id, 0,
+	return ti_sci_set_clock_state(handle, dev_id, clk_id,
+				      MSG_FLAG_CLOCK_ALLOW_FREQ_CHANGE,
 				      MSG_CLOCK_SW_STATE_AUTO);
 }
 
@@ -1107,7 +1178,7 @@
  * Return: 0 if all went well, else returns appropriate error value.
  */
 static int ti_sci_cmd_clk_is_auto(const struct ti_sci_handle *handle,
-				  u32 dev_id, u8 clk_id, bool *req_state)
+				  u32 dev_id, u32 clk_id, bool *req_state)
 {
 	u8 state = 0;
 	int ret;
@@ -1136,7 +1207,7 @@
  * Return: 0 if all went well, else returns appropriate error value.
  */
 static int ti_sci_cmd_clk_is_on(const struct ti_sci_handle *handle, u32 dev_id,
-				u8 clk_id, bool *req_state, bool *curr_state)
+				u32 clk_id, bool *req_state, bool *curr_state)
 {
 	u8 c_state = 0, r_state = 0;
 	int ret;
@@ -1169,7 +1240,7 @@
  * Return: 0 if all went well, else returns appropriate error value.
  */
 static int ti_sci_cmd_clk_is_off(const struct ti_sci_handle *handle, u32 dev_id,
-				 u8 clk_id, bool *req_state, bool *curr_state)
+				 u32 clk_id, bool *req_state, bool *curr_state)
 {
 	u8 c_state = 0, r_state = 0;
 	int ret;
@@ -1201,7 +1272,7 @@
  * Return: 0 if all went well, else returns appropriate error value.
  */
 static int ti_sci_cmd_clk_set_parent(const struct ti_sci_handle *handle,
-				     u32 dev_id, u8 clk_id, u8 parent_id)
+				     u32 dev_id, u32 clk_id, u32 parent_id)
 {
 	struct ti_sci_info *info;
 	struct ti_sci_msg_req_set_clock_parent *req;
@@ -1228,8 +1299,18 @@
 	}
 	req = (struct ti_sci_msg_req_set_clock_parent *)xfer->xfer_buf;
 	req->dev_id = dev_id;
-	req->clk_id = clk_id;
-	req->parent_id = parent_id;
+	if (clk_id < 255) {
+		req->clk_id = clk_id;
+	} else {
+		req->clk_id = 255;
+		req->clk_id_32 = clk_id;
+	}
+	if (parent_id < 255) {
+		req->parent_id = parent_id;
+	} else {
+		req->parent_id = 255;
+		req->parent_id_32 = parent_id;
+	}
 
 	ret = ti_sci_do_xfer(info, xfer);
 	if (ret) {
@@ -1259,7 +1340,7 @@
  * Return: 0 if all went well, else returns appropriate error value.
  */
 static int ti_sci_cmd_clk_get_parent(const struct ti_sci_handle *handle,
-				     u32 dev_id, u8 clk_id, u8 *parent_id)
+				     u32 dev_id, u32 clk_id, u32 *parent_id)
 {
 	struct ti_sci_info *info;
 	struct ti_sci_msg_req_get_clock_parent *req;
@@ -1286,7 +1367,12 @@
 	}
 	req = (struct ti_sci_msg_req_get_clock_parent *)xfer->xfer_buf;
 	req->dev_id = dev_id;
-	req->clk_id = clk_id;
+	if (clk_id < 255) {
+		req->clk_id = clk_id;
+	} else {
+		req->clk_id = 255;
+		req->clk_id_32 = clk_id;
+	}
 
 	ret = ti_sci_do_xfer(info, xfer);
 	if (ret) {
@@ -1296,10 +1382,14 @@
 
 	resp = (struct ti_sci_msg_resp_get_clock_parent *)xfer->xfer_buf;
 
-	if (!ti_sci_is_response_ack(resp))
+	if (!ti_sci_is_response_ack(resp)) {
 		ret = -ENODEV;
-	else
-		*parent_id = resp->parent_id;
+	} else {
+		if (resp->parent_id < 255)
+			*parent_id = resp->parent_id;
+		else
+			*parent_id = resp->parent_id_32;
+	}
 
 fail:
 	ti_sci_put_one_xfer(&info->minfo, xfer);
@@ -1319,8 +1409,8 @@
  * Return: 0 if all went well, else returns appropriate error value.
  */
 static int ti_sci_cmd_clk_get_num_parents(const struct ti_sci_handle *handle,
-					  u32 dev_id, u8 clk_id,
-					  u8 *num_parents)
+					  u32 dev_id, u32 clk_id,
+					  u32 *num_parents)
 {
 	struct ti_sci_info *info;
 	struct ti_sci_msg_req_get_clock_num_parents *req;
@@ -1347,7 +1437,12 @@
 	}
 	req = (struct ti_sci_msg_req_get_clock_num_parents *)xfer->xfer_buf;
 	req->dev_id = dev_id;
-	req->clk_id = clk_id;
+	if (clk_id < 255) {
+		req->clk_id = clk_id;
+	} else {
+		req->clk_id = 255;
+		req->clk_id_32 = clk_id;
+	}
 
 	ret = ti_sci_do_xfer(info, xfer);
 	if (ret) {
@@ -1357,10 +1452,14 @@
 
 	resp = (struct ti_sci_msg_resp_get_clock_num_parents *)xfer->xfer_buf;
 
-	if (!ti_sci_is_response_ack(resp))
+	if (!ti_sci_is_response_ack(resp)) {
 		ret = -ENODEV;
-	else
-		*num_parents = resp->num_parents;
+	} else {
+		if (resp->num_parents < 255)
+			*num_parents = resp->num_parents;
+		else
+			*num_parents = resp->num_parents_32;
+	}
 
 fail:
 	ti_sci_put_one_xfer(&info->minfo, xfer);
@@ -1388,7 +1487,7 @@
  * Return: 0 if all went well, else returns appropriate error value.
  */
 static int ti_sci_cmd_clk_get_match_freq(const struct ti_sci_handle *handle,
-					 u32 dev_id, u8 clk_id, u64 min_freq,
+					 u32 dev_id, u32 clk_id, u64 min_freq,
 					 u64 target_freq, u64 max_freq,
 					 u64 *match_freq)
 {
@@ -1417,7 +1516,12 @@
 	}
 	req = (struct ti_sci_msg_req_query_clock_freq *)xfer->xfer_buf;
 	req->dev_id = dev_id;
-	req->clk_id = clk_id;
+	if (clk_id < 255) {
+		req->clk_id = clk_id;
+	} else {
+		req->clk_id = 255;
+		req->clk_id_32 = clk_id;
+	}
 	req->min_freq_hz = min_freq;
 	req->target_freq_hz = target_freq;
 	req->max_freq_hz = max_freq;
@@ -1460,7 +1564,7 @@
  * Return: 0 if all went well, else returns appropriate error value.
  */
 static int ti_sci_cmd_clk_set_freq(const struct ti_sci_handle *handle,
-				   u32 dev_id, u8 clk_id, u64 min_freq,
+				   u32 dev_id, u32 clk_id, u64 min_freq,
 				   u64 target_freq, u64 max_freq)
 {
 	struct ti_sci_info *info;
@@ -1488,7 +1592,12 @@
 	}
 	req = (struct ti_sci_msg_req_set_clock_freq *)xfer->xfer_buf;
 	req->dev_id = dev_id;
-	req->clk_id = clk_id;
+	if (clk_id < 255) {
+		req->clk_id = clk_id;
+	} else {
+		req->clk_id = 255;
+		req->clk_id_32 = clk_id;
+	}
 	req->min_freq_hz = min_freq;
 	req->target_freq_hz = target_freq;
 	req->max_freq_hz = max_freq;
@@ -1521,7 +1630,7 @@
  * Return: 0 if all went well, else returns appropriate error value.
  */
 static int ti_sci_cmd_clk_get_freq(const struct ti_sci_handle *handle,
-				   u32 dev_id, u8 clk_id, u64 *freq)
+				   u32 dev_id, u32 clk_id, u64 *freq)
 {
 	struct ti_sci_info *info;
 	struct ti_sci_msg_req_get_clock_freq *req;
@@ -1548,7 +1657,12 @@
 	}
 	req = (struct ti_sci_msg_req_get_clock_freq *)xfer->xfer_buf;
 	req->dev_id = dev_id;
-	req->clk_id = clk_id;
+	if (clk_id < 255) {
+		req->clk_id = clk_id;
+	} else {
+		req->clk_id = 255;
+		req->clk_id_32 = clk_id;
+	}
 
 	ret = ti_sci_do_xfer(info, xfer);
 	if (ret) {
@@ -1615,6 +1729,1351 @@
 	return ret;
 }
 
+static int ti_sci_get_resource_type(struct ti_sci_info *info, u16 dev_id,
+				    u16 *type)
+{
+	struct ti_sci_rm_type_map *rm_type_map = info->desc->rm_type_map;
+	bool found = false;
+	int i;
+
+	/* If map is not provided then assume dev_id is used as type */
+	if (!rm_type_map) {
+		*type = dev_id;
+		return 0;
+	}
+
+	for (i = 0; rm_type_map[i].dev_id; i++) {
+		if (rm_type_map[i].dev_id == dev_id) {
+			*type = rm_type_map[i].type;
+			found = true;
+			break;
+		}
+	}
+
+	if (!found)
+		return -EINVAL;
+
+	return 0;
+}
+
+/**
+ * ti_sci_get_resource_range - Helper to get a range of resources assigned
+ *			       to a host. Resource is uniquely identified by
+ *			       type and subtype.
+ * @handle:		Pointer to TISCI handle.
+ * @dev_id:		TISCI device ID.
+ * @subtype:		Resource assignment subtype that is being requested
+ *			from the given device.
+ * @s_host:		Host processor ID to which the resources are allocated
+ * @range_start:	Start index of the resource range
+ * @range_num:		Number of resources in the range
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static int ti_sci_get_resource_range(const struct ti_sci_handle *handle,
+				     u32 dev_id, u8 subtype, u8 s_host,
+				     u16 *range_start, u16 *range_num)
+{
+	struct ti_sci_msg_resp_get_resource_range *resp;
+	struct ti_sci_msg_req_get_resource_range *req;
+	struct ti_sci_xfer *xfer;
+	struct ti_sci_info *info;
+	struct device *dev;
+	u16 type;
+	int ret = 0;
+
+	if (IS_ERR(handle))
+		return PTR_ERR(handle);
+	if (!handle)
+		return -EINVAL;
+
+	info = handle_to_ti_sci_info(handle);
+	dev = info->dev;
+
+	xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_GET_RESOURCE_RANGE,
+				   TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+				   sizeof(*req), sizeof(*resp));
+	if (IS_ERR(xfer)) {
+		ret = PTR_ERR(xfer);
+		dev_err(dev, "Message alloc failed(%d)\n", ret);
+		return ret;
+	}
+
+	ret = ti_sci_get_resource_type(info, dev_id, &type);
+	if (ret) {
+		dev_err(dev, "rm type lookup failed for %u\n", dev_id);
+		goto fail;
+	}
+
+	req = (struct ti_sci_msg_req_get_resource_range *)xfer->xfer_buf;
+	req->secondary_host = s_host;
+	req->type = type & MSG_RM_RESOURCE_TYPE_MASK;
+	req->subtype = subtype & MSG_RM_RESOURCE_SUBTYPE_MASK;
+
+	ret = ti_sci_do_xfer(info, xfer);
+	if (ret) {
+		dev_err(dev, "Mbox send fail %d\n", ret);
+		goto fail;
+	}
+
+	resp = (struct ti_sci_msg_resp_get_resource_range *)xfer->xfer_buf;
+
+	if (!ti_sci_is_response_ack(resp)) {
+		ret = -ENODEV;
+	} else if (!resp->range_start && !resp->range_num) {
+		ret = -ENODEV;
+	} else {
+		*range_start = resp->range_start;
+		*range_num = resp->range_num;
+	};
+
+fail:
+	ti_sci_put_one_xfer(&info->minfo, xfer);
+
+	return ret;
+}
+
+/**
+ * ti_sci_cmd_get_resource_range - Get a range of resources assigned to host
+ *				   that is same as ti sci interface host.
+ * @handle:		Pointer to TISCI handle.
+ * @dev_id:		TISCI device ID.
+ * @subtype:		Resource assignment subtype that is being requested
+ *			from the given device.
+ * @range_start:	Start index of the resource range
+ * @range_num:		Number of resources in the range
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static int ti_sci_cmd_get_resource_range(const struct ti_sci_handle *handle,
+					 u32 dev_id, u8 subtype,
+					 u16 *range_start, u16 *range_num)
+{
+	return ti_sci_get_resource_range(handle, dev_id, subtype,
+					 TI_SCI_IRQ_SECONDARY_HOST_INVALID,
+					 range_start, range_num);
+}
+
+/**
+ * ti_sci_cmd_get_resource_range_from_shost - Get a range of resources
+ *					      assigned to a specified host.
+ * @handle:		Pointer to TISCI handle.
+ * @dev_id:		TISCI device ID.
+ * @subtype:		Resource assignment subtype that is being requested
+ *			from the given device.
+ * @s_host:		Host processor ID to which the resources are allocated
+ * @range_start:	Start index of the resource range
+ * @range_num:		Number of resources in the range
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static
+int ti_sci_cmd_get_resource_range_from_shost(const struct ti_sci_handle *handle,
+					     u32 dev_id, u8 subtype, u8 s_host,
+					     u16 *range_start, u16 *range_num)
+{
+	return ti_sci_get_resource_range(handle, dev_id, subtype, s_host,
+					 range_start, range_num);
+}
+
+/**
+ * ti_sci_manage_irq() - Helper api to configure/release the irq route between
+ *			 the requested source and destination
+ * @handle:		Pointer to TISCI handle.
+ * @valid_params:	Bit fields defining the validity of certain params
+ * @src_id:		Device ID of the IRQ source
+ * @src_index:		IRQ source index within the source device
+ * @dst_id:		Device ID of the IRQ destination
+ * @dt_host_irq:	IRQ number of the destination device
+ * @ia_id:		Device ID of the IA, if the IRQ flows through this IA
+ * @vint:		Virtual interrupt to be used within the IA
+ * @global_event:	Global event number to be used for the requesting event
+ * @vint_status_bit:	Virtual interrupt status bit to be used for the event
+ * @s_host:		Secondary host ID to which the irq/event is being
+ *			requested for.
+ * @type:		Request type irq set or release.
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static int ti_sci_manage_irq(const struct ti_sci_handle *handle,
+			     u32 valid_params, u16 src_id, u16 src_index,
+			     u16 dst_id, u16 dst_host_irq, u16 ia_id, u16 vint,
+			     u16 global_event, u8 vint_status_bit, u8 s_host,
+			     u16 type)
+{
+	struct ti_sci_msg_req_manage_irq *req;
+	struct ti_sci_msg_hdr *resp;
+	struct ti_sci_xfer *xfer;
+	struct ti_sci_info *info;
+	struct device *dev;
+	int ret = 0;
+
+	if (IS_ERR(handle))
+		return PTR_ERR(handle);
+	if (!handle)
+		return -EINVAL;
+
+	info = handle_to_ti_sci_info(handle);
+	dev = info->dev;
+
+	xfer = ti_sci_get_one_xfer(info, type, TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+				   sizeof(*req), sizeof(*resp));
+	if (IS_ERR(xfer)) {
+		ret = PTR_ERR(xfer);
+		dev_err(dev, "Message alloc failed(%d)\n", ret);
+		return ret;
+	}
+	req = (struct ti_sci_msg_req_manage_irq *)xfer->xfer_buf;
+	req->valid_params = valid_params;
+	req->src_id = src_id;
+	req->src_index = src_index;
+	req->dst_id = dst_id;
+	req->dst_host_irq = dst_host_irq;
+	req->ia_id = ia_id;
+	req->vint = vint;
+	req->global_event = global_event;
+	req->vint_status_bit = vint_status_bit;
+	req->secondary_host = s_host;
+
+	ret = ti_sci_do_xfer(info, xfer);
+	if (ret) {
+		dev_err(dev, "Mbox send fail %d\n", ret);
+		goto fail;
+	}
+
+	resp = (struct ti_sci_msg_hdr *)xfer->xfer_buf;
+
+	ret = ti_sci_is_response_ack(resp) ? 0 : -ENODEV;
+
+fail:
+	ti_sci_put_one_xfer(&info->minfo, xfer);
+
+	return ret;
+}
+
+/**
+ * ti_sci_set_irq() - Helper api to configure the irq route between the
+ *		      requested source and destination
+ * @handle:		Pointer to TISCI handle.
+ * @valid_params:	Bit fields defining the validity of certain params
+ * @src_id:		Device ID of the IRQ source
+ * @src_index:		IRQ source index within the source device
+ * @dst_id:		Device ID of the IRQ destination
+ * @dt_host_irq:	IRQ number of the destination device
+ * @ia_id:		Device ID of the IA, if the IRQ flows through this IA
+ * @vint:		Virtual interrupt to be used within the IA
+ * @global_event:	Global event number to be used for the requesting event
+ * @vint_status_bit:	Virtual interrupt status bit to be used for the event
+ * @s_host:		Secondary host ID to which the irq/event is being
+ *			requested for.
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static int ti_sci_set_irq(const struct ti_sci_handle *handle, u32 valid_params,
+			  u16 src_id, u16 src_index, u16 dst_id,
+			  u16 dst_host_irq, u16 ia_id, u16 vint,
+			  u16 global_event, u8 vint_status_bit, u8 s_host)
+{
+	pr_debug("%s: IRQ set with valid_params = 0x%x from src = %d, index = %d, to dst = %d, irq = %d,via ia_id = %d, vint = %d, global event = %d,status_bit = %d\n",
+		 __func__, valid_params, src_id, src_index,
+		 dst_id, dst_host_irq, ia_id, vint, global_event,
+		 vint_status_bit);
+
+	return ti_sci_manage_irq(handle, valid_params, src_id, src_index,
+				 dst_id, dst_host_irq, ia_id, vint,
+				 global_event, vint_status_bit, s_host,
+				 TI_SCI_MSG_SET_IRQ);
+}
+
+/**
+ * ti_sci_free_irq() - Helper api to free the irq route between the
+ *			   requested source and destination
+ * @handle:		Pointer to TISCI handle.
+ * @valid_params:	Bit fields defining the validity of certain params
+ * @src_id:		Device ID of the IRQ source
+ * @src_index:		IRQ source index within the source device
+ * @dst_id:		Device ID of the IRQ destination
+ * @dt_host_irq:	IRQ number of the destination device
+ * @ia_id:		Device ID of the IA, if the IRQ flows through this IA
+ * @vint:		Virtual interrupt to be used within the IA
+ * @global_event:	Global event number to be used for the requesting event
+ * @vint_status_bit:	Virtual interrupt status bit to be used for the event
+ * @s_host:		Secondary host ID to which the irq/event is being
+ *			requested for.
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static int ti_sci_free_irq(const struct ti_sci_handle *handle, u32 valid_params,
+			   u16 src_id, u16 src_index, u16 dst_id,
+			   u16 dst_host_irq, u16 ia_id, u16 vint,
+			   u16 global_event, u8 vint_status_bit, u8 s_host)
+{
+	pr_debug("%s: IRQ release with valid_params = 0x%x from src = %d, index = %d, to dst = %d, irq = %d,via ia_id = %d, vint = %d, global event = %d,status_bit = %d\n",
+		 __func__, valid_params, src_id, src_index,
+		 dst_id, dst_host_irq, ia_id, vint, global_event,
+		 vint_status_bit);
+
+	return ti_sci_manage_irq(handle, valid_params, src_id, src_index,
+				 dst_id, dst_host_irq, ia_id, vint,
+				 global_event, vint_status_bit, s_host,
+				 TI_SCI_MSG_FREE_IRQ);
+}
+
+/**
+ * ti_sci_cmd_set_direct_irq() - Configure a non-event based direct irq route
+ *				 between the requested source and destination.
+ * @handle:		Pointer to TISCI handle.
+ * @src_id:		Device ID of the IRQ source
+ * @src_index:		IRQ source index within the source device
+ * @dst_id:		Device ID of the IRQ destination
+ * @dt_host_irq:	IRQ number of the destination device
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static int ti_sci_cmd_set_direct_irq(const struct ti_sci_handle *handle,
+				     u16 src_id, u16 src_index, u16 dst_id,
+				     u16 dst_host_irq)
+{
+	u32 valid_params = MSG_FLAG_DST_ID_VALID | MSG_FLAG_DST_HOST_IRQ_VALID;
+
+	return ti_sci_set_irq(handle, valid_params, src_id, src_index,
+			      dst_id, dst_host_irq, 0, 0, 0, 0, 0);
+}
+
+/**
+ * ti_sci_cmd_set_event_irq() - Configure an event based irq route between the
+ *				requested source and destination
+ * @handle:		Pointer to TISCI handle.
+ * @src_id:		Device ID of the IRQ source
+ * @src_index:		IRQ source index within the source device
+ * @dst_id:		Device ID of the IRQ destination
+ * @dt_host_irq:	IRQ number of the destination device
+ * @ia_id:		Device ID of the IA, if the IRQ flows through this IA
+ * @vint:		Virtual interrupt to be used within the IA
+ * @global_event:	Global event number to be used for the requesting event
+ * @vint_status_bit:	Virtual interrupt status bit to be used for the event
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static int ti_sci_cmd_set_event_irq(const struct ti_sci_handle *handle,
+				    u16 src_id, u16 src_index, u16 dst_id,
+				    u16 dst_host_irq, u16 ia_id, u16 vint,
+				    u16 global_event, u8 vint_status_bit)
+{
+	u32 valid_params = MSG_FLAG_DST_ID_VALID |
+			   MSG_FLAG_DST_HOST_IRQ_VALID | MSG_FLAG_IA_ID_VALID |
+			   MSG_FLAG_VINT_VALID | MSG_FLAG_GLB_EVNT_VALID |
+			   MSG_FLAG_VINT_STS_BIT_VALID;
+
+	return ti_sci_set_irq(handle, valid_params, src_id, src_index, dst_id,
+			      dst_host_irq, ia_id, vint, global_event,
+			      vint_status_bit, 0);
+}
+
+/**
+ * ti_sci_cmd_set_direct_irq_from_shost() - Configure a non-event based direct
+ *					    irq route between the source and
+ *					    destination belonging to a
+ *					    specified host.
+ * @handle:		Pointer to TISCI handle.
+ * @src_id:		Device ID of the IRQ source
+ * @src_index:		IRQ source index within the source device
+ * @dst_id:		Device ID of the IRQ destination
+ * @dt_host_irq:	IRQ number of the destination device
+ * @s_host:		Secondary host ID to which the irq/event is being
+ *			requested for.
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static
+int ti_sci_cmd_set_direct_irq_from_shost(const struct ti_sci_handle *handle,
+					 u16 src_id, u16 src_index, u16 dst_id,
+					 u16 dst_host_irq, u8 s_host)
+{
+	u32 valid_params = MSG_FLAG_DST_ID_VALID | MSG_FLAG_DST_HOST_IRQ_VALID |
+			   MSG_FLAG_SHOST_VALID;
+
+	return ti_sci_set_irq(handle, valid_params, src_id, src_index,
+			      dst_id, dst_host_irq, 0, 0, 0, 0, s_host);
+}
+
+/**
+ * ti_sci_cmd_set_event_irq_from_shost() - Configure an event based irq
+ *					   route between the source and
+ *					   destination belonging to a
+ *					   specified host.
+ * @handle:		Pointer to TISCI handle.
+ * @src_id:		Device ID of the IRQ source
+ * @src_index:		IRQ source index within the source device
+ * @dst_id:		Device ID of the IRQ destination
+ * @dt_host_irq:	IRQ number of the destination device
+ * @ia_id:		Device ID of the IA, if the IRQ flows through this IA
+ * @vint:		Virtual interrupt to be used within the IA
+ * @global_event:	Global event number to be used for the requesting event
+ * @vint_status_bit:	Virtual interrupt status bit to be used for the event
+ * @s_host:		Secondary host ID to which the irq/event is being
+ *			requested for.
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static
+int ti_sci_cmd_set_event_irq_from_shost(const struct ti_sci_handle *handle,
+					u16 src_id, u16 src_index, u16 dst_id,
+					u16 dst_host_irq, u16 ia_id, u16 vint,
+					u16 global_event, u8 vint_status_bit,
+					u8 s_host)
+{
+	u32 valid_params = MSG_FLAG_DST_ID_VALID |
+			   MSG_FLAG_DST_HOST_IRQ_VALID | MSG_FLAG_IA_ID_VALID |
+			   MSG_FLAG_VINT_VALID | MSG_FLAG_GLB_EVNT_VALID |
+			   MSG_FLAG_VINT_STS_BIT_VALID | MSG_FLAG_SHOST_VALID;
+
+	return ti_sci_set_irq(handle, valid_params, src_id, src_index,
+			      dst_id, dst_host_irq, ia_id, vint,
+			      global_event, vint_status_bit, s_host);
+}
+
+/**
+ * ti_sci_cmd_set_event_irq_to_poll() - Configure an event based irq
+ *					in polling mode
+ * @handle:		Pointer to TISCI handle.
+ * @src_id:		Device ID of the IRQ source
+ * @src_index:		IRQ source index within the source device
+ * @dst_id:		Device ID of the IRQ destination
+ * @dt_host_irq:	IRQ number of the destination device
+ * @ia_id:		Device ID of the IA, if the IRQ flows through this IA
+ * @vint:		Virtual interrupt to be used within the IA
+ * @global_event:	Global event number to be used for the requesting event
+ * @vint_status_bit:	Virtual interrupt status bit to be used for the event
+ * @s_host:		Secondary host ID to which the irq/event is being
+ *			requested for.
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static int ti_sci_cmd_set_event_irq_to_poll(const struct ti_sci_handle *handle,
+					    u16 src_id, u16 src_index,
+					    u16 ia_id, u16 vint,
+					    u16 global_event,
+					    u8 vint_status_bit)
+{
+	u32 valid_params = MSG_FLAG_IA_ID_VALID | MSG_FLAG_VINT_VALID |
+			   MSG_FLAG_GLB_EVNT_VALID |
+			   MSG_FLAG_VINT_STS_BIT_VALID;
+
+	return ti_sci_set_irq(handle, valid_params, src_id, src_index, 0, 0,
+			      ia_id, vint, global_event, vint_status_bit, 0);
+}
+
+/**
+ * ti_sci_cmd_free_direct_irq() - Free a non-event based direct irq route
+ *				  between the requested source and destination.
+ * @handle:		Pointer to TISCI handle.
+ * @src_id:		Device ID of the IRQ source
+ * @src_index:		IRQ source index within the source device
+ * @dst_id:		Device ID of the IRQ destination
+ * @dt_host_irq:	IRQ number of the destination device
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static int ti_sci_cmd_free_direct_irq(const struct ti_sci_handle *handle,
+				      u16 src_id, u16 src_index, u16 dst_id,
+				      u16 dst_host_irq)
+{
+	u32 valid_params = MSG_FLAG_DST_ID_VALID | MSG_FLAG_DST_HOST_IRQ_VALID;
+
+	return ti_sci_free_irq(handle, valid_params, src_id, src_index,
+			       dst_id, dst_host_irq, 0, 0, 0, 0, 0);
+}
+
+/**
+ * ti_sci_cmd_free_event_irq() - Free an event based irq route between the
+ *				 requested source and destination
+ * @handle:		Pointer to TISCI handle.
+ * @src_id:		Device ID of the IRQ source
+ * @src_index:		IRQ source index within the source device
+ * @dst_id:		Device ID of the IRQ destination
+ * @dt_host_irq:	IRQ number of the destination device
+ * @ia_id:		Device ID of the IA, if the IRQ flows through this IA
+ * @vint:		Virtual interrupt to be used within the IA
+ * @global_event:	Global event number to be used for the requesting event
+ * @vint_status_bit:	Virtual interrupt status bit to be used for the event
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static int ti_sci_cmd_free_event_irq(const struct ti_sci_handle *handle,
+				     u16 src_id, u16 src_index, u16 dst_id,
+				     u16 dst_host_irq, u16 ia_id, u16 vint,
+				     u16 global_event, u8 vint_status_bit)
+{
+	u32 valid_params = MSG_FLAG_DST_ID_VALID |
+			   MSG_FLAG_DST_HOST_IRQ_VALID | MSG_FLAG_IA_ID_VALID |
+			   MSG_FLAG_VINT_VALID | MSG_FLAG_GLB_EVNT_VALID |
+			   MSG_FLAG_VINT_STS_BIT_VALID;
+
+	return ti_sci_free_irq(handle, valid_params, src_id, src_index,
+			       dst_id, dst_host_irq, ia_id, vint,
+			       global_event, vint_status_bit, 0);
+}
+
+/**
+ * ti_sci_cmd_free_direct_irq_from_shost() - Free a non-event based direct irq
+ *					     route between the source and
+ *					     destination belonging to a
+ *					     specified host.
+ * @handle:		Pointer to TISCI handle.
+ * @src_id:		Device ID of the IRQ source
+ * @src_index:		IRQ source index within the source device
+ * @dst_id:		Device ID of the IRQ destination
+ * @dt_host_irq:	IRQ number of the destination device
+ * @s_host:		Secondary host ID to which the irq/event is being
+ *			requested for.
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static
+int ti_sci_cmd_free_direct_irq_from_shost(const struct ti_sci_handle *handle,
+					  u16 src_id, u16 src_index, u16 dst_id,
+					  u16 dst_host_irq, u8 s_host)
+{
+	u32 valid_params = MSG_FLAG_DST_ID_VALID | MSG_FLAG_DST_HOST_IRQ_VALID |
+			   MSG_FLAG_SHOST_VALID;
+
+	return ti_sci_free_irq(handle, valid_params, src_id, src_index,
+			       dst_id, dst_host_irq, 0, 0, 0, 0, s_host);
+}
+
+/**
+ * ti_sci_cmd_free_event_irq_from_shost() - Free an event based irq
+ *					    route between the source and
+ *					    destination belonging to a
+ *					    specified host.
+ * @handle:		Pointer to TISCI handle.
+ * @src_id:		Device ID of the IRQ source
+ * @src_index:		IRQ source index within the source device
+ * @dst_id:		Device ID of the IRQ destination
+ * @dt_host_irq:	IRQ number of the destination device
+ * @ia_id:		Device ID of the IA, if the IRQ flows through this IA
+ * @vint:		Virtual interrupt to be used within the IA
+ * @global_event:	Global event number to be used for the requesting event
+ * @vint_status_bit:	Virtual interrupt status bit to be used for the event
+ * @s_host:		Secondary host ID to which the irq/event is being
+ *			requested for.
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static
+int ti_sci_cmd_free_event_irq_from_shost(const struct ti_sci_handle *handle,
+					 u16 src_id, u16 src_index, u16 dst_id,
+					 u16 dst_host_irq, u16 ia_id, u16 vint,
+					 u16 global_event, u8 vint_status_bit,
+					 u8 s_host)
+{
+	u32 valid_params = MSG_FLAG_DST_ID_VALID |
+			   MSG_FLAG_DST_HOST_IRQ_VALID | MSG_FLAG_IA_ID_VALID |
+			   MSG_FLAG_VINT_VALID | MSG_FLAG_GLB_EVNT_VALID |
+			   MSG_FLAG_VINT_STS_BIT_VALID | MSG_FLAG_SHOST_VALID;
+
+	return ti_sci_free_irq(handle, valid_params, src_id, src_index,
+			       dst_id, dst_host_irq, ia_id, vint,
+			       global_event, vint_status_bit, s_host);
+}
+
+/**
+ * ti_sci_cmd_free_event_irq_to_poll() - Free an event based irq
+ *					 in polling mode
+ * @handle:		Pointer to TISCI handle.
+ * @src_id:		Device ID of the IRQ source
+ * @src_index:		IRQ source index within the source device
+ * @dst_id:		Device ID of the IRQ destination
+ * @dt_host_irq:	IRQ number of the destination device
+ * @ia_id:		Device ID of the IA, if the IRQ flows through this IA
+ * @vint:		Virtual interrupt to be used within the IA
+ * @global_event:	Global event number to be used for the requesting event
+ * @vint_status_bit:	Virtual interrupt status bit to be used for the event
+ * @s_host:		Secondary host ID to which the irq/event is being
+ *			requested for.
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static int ti_sci_cmd_free_event_irq_to_poll(const struct ti_sci_handle *handle,
+					     u16 src_id, u16 src_index,
+					     u16 ia_id, u16 vint,
+					     u16 global_event,
+					     u8 vint_status_bit)
+{
+	u32 valid_params = MSG_FLAG_IA_ID_VALID | MSG_FLAG_VINT_VALID |
+			   MSG_FLAG_GLB_EVNT_VALID |
+			   MSG_FLAG_VINT_STS_BIT_VALID;
+
+	return ti_sci_free_irq(handle, valid_params, src_id, src_index, 0, 0,
+			       ia_id, vint, global_event, vint_status_bit, 0);
+}
+
+/**
+ * ti_sci_cmd_ring_config() - configure RA ring
+ * @handle:	pointer to TI SCI handle
+ * @valid_params: Bitfield defining validity of ring configuration parameters.
+ * @nav_id: Device ID of Navigator Subsystem from which the ring is allocated
+ * @index: Ring index.
+ * @addr_lo: The ring base address lo 32 bits
+ * @addr_hi: The ring base address hi 32 bits
+ * @count: Number of ring elements.
+ * @mode: The mode of the ring
+ * @size: The ring element size.
+ * @order_id: Specifies the ring's bus order ID.
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ *
+ * See @ti_sci_msg_rm_ring_cfg_req for more info.
+ */
+static int ti_sci_cmd_ring_config(const struct ti_sci_handle *handle,
+				  u32 valid_params, u16 nav_id, u16 index,
+				  u32 addr_lo, u32 addr_hi, u32 count,
+				  u8 mode, u8 size, u8 order_id)
+{
+	struct ti_sci_msg_rm_ring_cfg_resp *resp;
+	struct ti_sci_msg_rm_ring_cfg_req *req;
+	struct ti_sci_xfer *xfer;
+	struct ti_sci_info *info;
+	struct device *dev;
+	int ret = 0;
+
+	if (IS_ERR_OR_NULL(handle))
+		return -EINVAL;
+
+	info = handle_to_ti_sci_info(handle);
+	dev = info->dev;
+
+	xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_RM_RING_CFG,
+				   TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+				   sizeof(*req), sizeof(*resp));
+	if (IS_ERR(xfer)) {
+		ret = PTR_ERR(xfer);
+		dev_err(info->dev, "RM_RA:Message config failed(%d)\n", ret);
+		return ret;
+	}
+	req = (struct ti_sci_msg_rm_ring_cfg_req *)xfer->xfer_buf;
+	req->valid_params = valid_params;
+	req->nav_id = nav_id;
+	req->index = index;
+	req->addr_lo = addr_lo;
+	req->addr_hi = addr_hi;
+	req->count = count;
+	req->mode = mode;
+	req->size = size;
+	req->order_id = order_id;
+
+	ret = ti_sci_do_xfer(info, xfer);
+	if (ret) {
+		dev_err(info->dev, "RM_RA:Mbox config send fail %d\n", ret);
+		goto fail;
+	}
+
+	resp = (struct ti_sci_msg_rm_ring_cfg_resp *)xfer->xfer_buf;
+
+	ret = ti_sci_is_response_ack(resp) ? 0 : -ENODEV;
+
+fail:
+	ti_sci_put_one_xfer(&info->minfo, xfer);
+	dev_dbg(info->dev, "RM_RA:config ring %u ret:%d\n", index, ret);
+	return ret;
+}
+
+/**
+ * ti_sci_cmd_ring_get_config() - get RA ring configuration
+ * @handle:	pointer to TI SCI handle
+ * @nav_id: Device ID of Navigator Subsystem from which the ring is allocated
+ * @index: Ring index.
+ * @addr_lo: returns ring's base address lo 32 bits
+ * @addr_hi: returns ring's base address hi 32 bits
+ * @count: returns number of ring elements.
+ * @mode: returns mode of the ring
+ * @size: returns ring element size.
+ * @order_id: returns ring's bus order ID.
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ *
+ * See @ti_sci_msg_rm_ring_get_cfg_req for more info.
+ */
+static int ti_sci_cmd_ring_get_config(const struct ti_sci_handle *handle,
+				      u32 nav_id, u32 index, u8 *mode,
+				      u32 *addr_lo, u32 *addr_hi,
+				      u32 *count, u8 *size, u8 *order_id)
+{
+	struct ti_sci_msg_rm_ring_get_cfg_resp *resp;
+	struct ti_sci_msg_rm_ring_get_cfg_req *req;
+	struct ti_sci_xfer *xfer;
+	struct ti_sci_info *info;
+	struct device *dev;
+	int ret = 0;
+
+	if (IS_ERR_OR_NULL(handle))
+		return -EINVAL;
+
+	info = handle_to_ti_sci_info(handle);
+	dev = info->dev;
+
+	xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_RM_RING_GET_CFG,
+				   TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+				   sizeof(*req), sizeof(*resp));
+	if (IS_ERR(xfer)) {
+		ret = PTR_ERR(xfer);
+		dev_err(info->dev,
+			"RM_RA:Message get config failed(%d)\n", ret);
+		return ret;
+	}
+	req = (struct ti_sci_msg_rm_ring_get_cfg_req *)xfer->xfer_buf;
+	req->nav_id = nav_id;
+	req->index = index;
+
+	ret = ti_sci_do_xfer(info, xfer);
+	if (ret) {
+		dev_err(info->dev, "RM_RA:Mbox get config send fail %d\n", ret);
+		goto fail;
+	}
+
+	resp = (struct ti_sci_msg_rm_ring_get_cfg_resp *)xfer->xfer_buf;
+
+	if (!ti_sci_is_response_ack(resp)) {
+		ret = -ENODEV;
+	} else {
+		if (mode)
+			*mode = resp->mode;
+		if (addr_lo)
+			*addr_lo = resp->addr_lo;
+		if (addr_hi)
+			*addr_hi = resp->addr_hi;
+		if (count)
+			*count = resp->count;
+		if (size)
+			*size = resp->size;
+		if (order_id)
+			*order_id = resp->order_id;
+	};
+
+fail:
+	ti_sci_put_one_xfer(&info->minfo, xfer);
+	dev_dbg(info->dev, "RM_RA:get config ring %u ret:%d\n", index, ret);
+	return ret;
+}
+
+static int ti_sci_cmd_rm_psil_pair(const struct ti_sci_handle *handle,
+				   u32 nav_id, u32 src_thread, u32 dst_thread)
+{
+	struct ti_sci_msg_hdr *resp;
+	struct ti_sci_msg_psil_pair *req;
+	struct ti_sci_xfer *xfer;
+	struct ti_sci_info *info;
+	struct device *dev;
+	int ret = 0;
+
+	if (IS_ERR(handle))
+		return PTR_ERR(handle);
+	if (!handle)
+		return -EINVAL;
+
+	info = handle_to_ti_sci_info(handle);
+	dev = info->dev;
+
+	xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_RM_PSIL_PAIR,
+				   TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+				   sizeof(*req), sizeof(*resp));
+	if (IS_ERR(xfer)) {
+		ret = PTR_ERR(xfer);
+		dev_err(dev, "RM_PSIL:Message reconfig failed(%d)\n", ret);
+		return ret;
+	}
+	req = (struct ti_sci_msg_psil_pair *)xfer->xfer_buf;
+	req->nav_id = nav_id;
+	req->src_thread = src_thread;
+	req->dst_thread = dst_thread;
+
+	ret = ti_sci_do_xfer(info, xfer);
+	if (ret) {
+		dev_err(dev, "RM_PSIL:Mbox send fail %d\n", ret);
+		goto fail;
+	}
+
+	resp = (struct ti_sci_msg_hdr *)xfer->xfer_buf;
+	ret = ti_sci_is_response_ack(resp) ? 0 : -EINVAL;
+
+fail:
+	ti_sci_put_one_xfer(&info->minfo, xfer);
+
+	return ret;
+}
+
+static int ti_sci_cmd_rm_psil_unpair(const struct ti_sci_handle *handle,
+				     u32 nav_id, u32 src_thread, u32 dst_thread)
+{
+	struct ti_sci_msg_hdr *resp;
+	struct ti_sci_msg_psil_unpair *req;
+	struct ti_sci_xfer *xfer;
+	struct ti_sci_info *info;
+	struct device *dev;
+	int ret = 0;
+
+	if (IS_ERR(handle))
+		return PTR_ERR(handle);
+	if (!handle)
+		return -EINVAL;
+
+	info = handle_to_ti_sci_info(handle);
+	dev = info->dev;
+
+	xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_RM_PSIL_UNPAIR,
+				   TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+				   sizeof(*req), sizeof(*resp));
+	if (IS_ERR(xfer)) {
+		ret = PTR_ERR(xfer);
+		dev_err(dev, "RM_PSIL:Message reconfig failed(%d)\n", ret);
+		return ret;
+	}
+	req = (struct ti_sci_msg_psil_unpair *)xfer->xfer_buf;
+	req->nav_id = nav_id;
+	req->src_thread = src_thread;
+	req->dst_thread = dst_thread;
+
+	ret = ti_sci_do_xfer(info, xfer);
+	if (ret) {
+		dev_err(dev, "RM_PSIL:Mbox send fail %d\n", ret);
+		goto fail;
+	}
+
+	resp = (struct ti_sci_msg_hdr *)xfer->xfer_buf;
+	ret = ti_sci_is_response_ack(resp) ? 0 : -EINVAL;
+
+fail:
+	ti_sci_put_one_xfer(&info->minfo, xfer);
+
+	return ret;
+}
+
+static int ti_sci_cmd_rm_udmap_tx_ch_cfg(const struct ti_sci_handle *handle,
+			const struct ti_sci_msg_rm_udmap_tx_ch_cfg *params)
+{
+	struct ti_sci_msg_rm_udmap_tx_ch_cfg_resp *resp;
+	struct ti_sci_msg_rm_udmap_tx_ch_cfg_req *req;
+	struct ti_sci_xfer *xfer;
+	struct ti_sci_info *info;
+	struct device *dev;
+	int ret = 0;
+
+	if (IS_ERR_OR_NULL(handle))
+		return -EINVAL;
+
+	info = handle_to_ti_sci_info(handle);
+	dev = info->dev;
+
+	xfer = ti_sci_get_one_xfer(info, TISCI_MSG_RM_UDMAP_TX_CH_CFG,
+				   TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+				   sizeof(*req), sizeof(*resp));
+	if (IS_ERR(xfer)) {
+		ret = PTR_ERR(xfer);
+		dev_err(info->dev, "Message TX_CH_CFG alloc failed(%d)\n", ret);
+		return ret;
+	}
+	req = (struct ti_sci_msg_rm_udmap_tx_ch_cfg_req *)xfer->xfer_buf;
+	req->valid_params = params->valid_params;
+	req->nav_id = params->nav_id;
+	req->index = params->index;
+	req->tx_pause_on_err = params->tx_pause_on_err;
+	req->tx_filt_einfo = params->tx_filt_einfo;
+	req->tx_filt_pswords = params->tx_filt_pswords;
+	req->tx_atype = params->tx_atype;
+	req->tx_chan_type = params->tx_chan_type;
+	req->tx_supr_tdpkt = params->tx_supr_tdpkt;
+	req->tx_fetch_size = params->tx_fetch_size;
+	req->tx_credit_count = params->tx_credit_count;
+	req->txcq_qnum = params->txcq_qnum;
+	req->tx_priority = params->tx_priority;
+	req->tx_qos = params->tx_qos;
+	req->tx_orderid = params->tx_orderid;
+	req->fdepth = params->fdepth;
+	req->tx_sched_priority = params->tx_sched_priority;
+	req->tx_burst_size = params->tx_burst_size;
+
+	ret = ti_sci_do_xfer(info, xfer);
+	if (ret) {
+		dev_err(info->dev, "Mbox send TX_CH_CFG fail %d\n", ret);
+		goto fail;
+	}
+
+	resp =
+	      (struct ti_sci_msg_rm_udmap_tx_ch_cfg_resp *)xfer->xfer_buf;
+	ret = ti_sci_is_response_ack(resp) ? 0 : -EINVAL;
+
+fail:
+	ti_sci_put_one_xfer(&info->minfo, xfer);
+	dev_dbg(info->dev, "TX_CH_CFG: chn %u ret:%u\n", params->index, ret);
+	return ret;
+}
+
+static int ti_sci_cmd_rm_udmap_rx_ch_cfg(const struct ti_sci_handle *handle,
+			const struct ti_sci_msg_rm_udmap_rx_ch_cfg *params)
+{
+	struct ti_sci_msg_rm_udmap_rx_ch_cfg_resp *resp;
+	struct ti_sci_msg_rm_udmap_rx_ch_cfg_req *req;
+	struct ti_sci_xfer *xfer;
+	struct ti_sci_info *info;
+	struct device *dev;
+	int ret = 0;
+
+	if (IS_ERR_OR_NULL(handle))
+		return -EINVAL;
+
+	info = handle_to_ti_sci_info(handle);
+	dev = info->dev;
+
+	xfer = ti_sci_get_one_xfer(info, TISCI_MSG_RM_UDMAP_RX_CH_CFG,
+				   TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+				   sizeof(*req), sizeof(*resp));
+	if (IS_ERR(xfer)) {
+		ret = PTR_ERR(xfer);
+		dev_err(info->dev, "Message RX_CH_CFG alloc failed(%d)\n", ret);
+		return ret;
+	}
+	req = (struct ti_sci_msg_rm_udmap_rx_ch_cfg_req *)xfer->xfer_buf;
+	req->valid_params = params->valid_params;
+	req->nav_id = params->nav_id;
+	req->index = params->index;
+	req->rx_fetch_size = params->rx_fetch_size;
+	req->rxcq_qnum = params->rxcq_qnum;
+	req->rx_priority = params->rx_priority;
+	req->rx_qos = params->rx_qos;
+	req->rx_orderid = params->rx_orderid;
+	req->rx_sched_priority = params->rx_sched_priority;
+	req->flowid_start = params->flowid_start;
+	req->flowid_cnt = params->flowid_cnt;
+	req->rx_pause_on_err = params->rx_pause_on_err;
+	req->rx_atype = params->rx_atype;
+	req->rx_chan_type = params->rx_chan_type;
+	req->rx_ignore_short = params->rx_ignore_short;
+	req->rx_ignore_long = params->rx_ignore_long;
+	req->rx_burst_size = params->rx_burst_size;
+
+	ret = ti_sci_do_xfer(info, xfer);
+	if (ret) {
+		dev_err(info->dev, "Mbox send RX_CH_CFG fail %d\n", ret);
+		goto fail;
+	}
+
+	resp =
+	      (struct ti_sci_msg_rm_udmap_rx_ch_cfg_resp *)xfer->xfer_buf;
+	ret = ti_sci_is_response_ack(resp) ? 0 : -EINVAL;
+
+fail:
+	ti_sci_put_one_xfer(&info->minfo, xfer);
+	dev_dbg(info->dev, "RX_CH_CFG: chn %u ret:%d\n", params->index, ret);
+	return ret;
+}
+
+static int ti_sci_cmd_rm_udmap_rx_flow_cfg(const struct ti_sci_handle *handle,
+			const struct ti_sci_msg_rm_udmap_flow_cfg *params)
+{
+	struct ti_sci_msg_rm_udmap_flow_cfg_resp *resp;
+	struct ti_sci_msg_rm_udmap_flow_cfg_req *req;
+	struct ti_sci_xfer *xfer;
+	struct ti_sci_info *info;
+	struct device *dev;
+	int ret = 0;
+
+	if (IS_ERR_OR_NULL(handle))
+		return -EINVAL;
+
+	info = handle_to_ti_sci_info(handle);
+	dev = info->dev;
+
+	xfer = ti_sci_get_one_xfer(info, TISCI_MSG_RM_UDMAP_FLOW_CFG,
+				   TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+				   sizeof(*req), sizeof(*resp));
+	if (IS_ERR(xfer)) {
+		ret = PTR_ERR(xfer);
+		dev_err(dev, "RX_FL_CFG: Message alloc failed(%d)\n", ret);
+		return ret;
+	}
+	req = (struct ti_sci_msg_rm_udmap_flow_cfg_req *)xfer->xfer_buf;
+	req->valid_params = params->valid_params;
+	req->nav_id = params->nav_id;
+	req->flow_index = params->flow_index;
+	req->rx_einfo_present = params->rx_einfo_present;
+	req->rx_psinfo_present = params->rx_psinfo_present;
+	req->rx_error_handling = params->rx_error_handling;
+	req->rx_desc_type = params->rx_desc_type;
+	req->rx_sop_offset = params->rx_sop_offset;
+	req->rx_dest_qnum = params->rx_dest_qnum;
+	req->rx_src_tag_hi = params->rx_src_tag_hi;
+	req->rx_src_tag_lo = params->rx_src_tag_lo;
+	req->rx_dest_tag_hi = params->rx_dest_tag_hi;
+	req->rx_dest_tag_lo = params->rx_dest_tag_lo;
+	req->rx_src_tag_hi_sel = params->rx_src_tag_hi_sel;
+	req->rx_src_tag_lo_sel = params->rx_src_tag_lo_sel;
+	req->rx_dest_tag_hi_sel = params->rx_dest_tag_hi_sel;
+	req->rx_dest_tag_lo_sel = params->rx_dest_tag_lo_sel;
+	req->rx_fdq0_sz0_qnum = params->rx_fdq0_sz0_qnum;
+	req->rx_fdq1_qnum = params->rx_fdq1_qnum;
+	req->rx_fdq2_qnum = params->rx_fdq2_qnum;
+	req->rx_fdq3_qnum = params->rx_fdq3_qnum;
+	req->rx_ps_location = params->rx_ps_location;
+
+	ret = ti_sci_do_xfer(info, xfer);
+	if (ret) {
+		dev_err(dev, "RX_FL_CFG: Mbox send fail %d\n", ret);
+		goto fail;
+	}
+
+	resp =
+	       (struct ti_sci_msg_rm_udmap_flow_cfg_resp *)xfer->xfer_buf;
+	ret = ti_sci_is_response_ack(resp) ? 0 : -EINVAL;
+
+fail:
+	ti_sci_put_one_xfer(&info->minfo, xfer);
+	dev_dbg(info->dev, "RX_FL_CFG: %u ret:%d\n", params->flow_index, ret);
+	return ret;
+}
+
+/**
+ * ti_sci_cmd_proc_request() - Command to request a physical processor control
+ * @handle:	Pointer to TI SCI handle
+ * @proc_id:	Processor ID this request is for
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_cmd_proc_request(const struct ti_sci_handle *handle,
+				   u8 proc_id)
+{
+	struct ti_sci_msg_req_proc_request *req;
+	struct ti_sci_msg_hdr *resp;
+	struct ti_sci_info *info;
+	struct ti_sci_xfer *xfer;
+	struct device *dev;
+	int ret = 0;
+
+	if (!handle)
+		return -EINVAL;
+	if (IS_ERR(handle))
+		return PTR_ERR(handle);
+
+	info = handle_to_ti_sci_info(handle);
+	dev = info->dev;
+
+	xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_PROC_REQUEST,
+				   TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+				   sizeof(*req), sizeof(*resp));
+	if (IS_ERR(xfer)) {
+		ret = PTR_ERR(xfer);
+		dev_err(dev, "Message alloc failed(%d)\n", ret);
+		return ret;
+	}
+	req = (struct ti_sci_msg_req_proc_request *)xfer->xfer_buf;
+	req->processor_id = proc_id;
+
+	ret = ti_sci_do_xfer(info, xfer);
+	if (ret) {
+		dev_err(dev, "Mbox send fail %d\n", ret);
+		goto fail;
+	}
+
+	resp = (struct ti_sci_msg_hdr *)xfer->tx_message.buf;
+
+	ret = ti_sci_is_response_ack(resp) ? 0 : -ENODEV;
+
+fail:
+	ti_sci_put_one_xfer(&info->minfo, xfer);
+
+	return ret;
+}
+
+/**
+ * ti_sci_cmd_proc_release() - Command to release a physical processor control
+ * @handle:	Pointer to TI SCI handle
+ * @proc_id:	Processor ID this request is for
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_cmd_proc_release(const struct ti_sci_handle *handle,
+				   u8 proc_id)
+{
+	struct ti_sci_msg_req_proc_release *req;
+	struct ti_sci_msg_hdr *resp;
+	struct ti_sci_info *info;
+	struct ti_sci_xfer *xfer;
+	struct device *dev;
+	int ret = 0;
+
+	if (!handle)
+		return -EINVAL;
+	if (IS_ERR(handle))
+		return PTR_ERR(handle);
+
+	info = handle_to_ti_sci_info(handle);
+	dev = info->dev;
+
+	xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_PROC_RELEASE,
+				   TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+				   sizeof(*req), sizeof(*resp));
+	if (IS_ERR(xfer)) {
+		ret = PTR_ERR(xfer);
+		dev_err(dev, "Message alloc failed(%d)\n", ret);
+		return ret;
+	}
+	req = (struct ti_sci_msg_req_proc_release *)xfer->xfer_buf;
+	req->processor_id = proc_id;
+
+	ret = ti_sci_do_xfer(info, xfer);
+	if (ret) {
+		dev_err(dev, "Mbox send fail %d\n", ret);
+		goto fail;
+	}
+
+	resp = (struct ti_sci_msg_hdr *)xfer->tx_message.buf;
+
+	ret = ti_sci_is_response_ack(resp) ? 0 : -ENODEV;
+
+fail:
+	ti_sci_put_one_xfer(&info->minfo, xfer);
+
+	return ret;
+}
+
+/**
+ * ti_sci_cmd_proc_handover() - Command to handover a physical processor
+ *				control to a host in the processor's access
+ *				control list.
+ * @handle:	Pointer to TI SCI handle
+ * @proc_id:	Processor ID this request is for
+ * @host_id:	Host ID to get the control of the processor
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_cmd_proc_handover(const struct ti_sci_handle *handle,
+				    u8 proc_id, u8 host_id)
+{
+	struct ti_sci_msg_req_proc_handover *req;
+	struct ti_sci_msg_hdr *resp;
+	struct ti_sci_info *info;
+	struct ti_sci_xfer *xfer;
+	struct device *dev;
+	int ret = 0;
+
+	if (!handle)
+		return -EINVAL;
+	if (IS_ERR(handle))
+		return PTR_ERR(handle);
+
+	info = handle_to_ti_sci_info(handle);
+	dev = info->dev;
+
+	xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_PROC_HANDOVER,
+				   TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+				   sizeof(*req), sizeof(*resp));
+	if (IS_ERR(xfer)) {
+		ret = PTR_ERR(xfer);
+		dev_err(dev, "Message alloc failed(%d)\n", ret);
+		return ret;
+	}
+	req = (struct ti_sci_msg_req_proc_handover *)xfer->xfer_buf;
+	req->processor_id = proc_id;
+	req->host_id = host_id;
+
+	ret = ti_sci_do_xfer(info, xfer);
+	if (ret) {
+		dev_err(dev, "Mbox send fail %d\n", ret);
+		goto fail;
+	}
+
+	resp = (struct ti_sci_msg_hdr *)xfer->tx_message.buf;
+
+	ret = ti_sci_is_response_ack(resp) ? 0 : -ENODEV;
+
+fail:
+	ti_sci_put_one_xfer(&info->minfo, xfer);
+
+	return ret;
+}
+
+/**
+ * ti_sci_cmd_proc_set_config() - Command to set the processor boot
+ *				    configuration flags
+ * @handle:		Pointer to TI SCI handle
+ * @proc_id:		Processor ID this request is for
+ * @config_flags_set:	Configuration flags to be set
+ * @config_flags_clear:	Configuration flags to be cleared.
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_cmd_proc_set_config(const struct ti_sci_handle *handle,
+				      u8 proc_id, u64 bootvector,
+				      u32 config_flags_set,
+				      u32 config_flags_clear)
+{
+	struct ti_sci_msg_req_set_config *req;
+	struct ti_sci_msg_hdr *resp;
+	struct ti_sci_info *info;
+	struct ti_sci_xfer *xfer;
+	struct device *dev;
+	int ret = 0;
+
+	if (!handle)
+		return -EINVAL;
+	if (IS_ERR(handle))
+		return PTR_ERR(handle);
+
+	info = handle_to_ti_sci_info(handle);
+	dev = info->dev;
+
+	xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_SET_CONFIG,
+				   TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+				   sizeof(*req), sizeof(*resp));
+	if (IS_ERR(xfer)) {
+		ret = PTR_ERR(xfer);
+		dev_err(dev, "Message alloc failed(%d)\n", ret);
+		return ret;
+	}
+	req = (struct ti_sci_msg_req_set_config *)xfer->xfer_buf;
+	req->processor_id = proc_id;
+	req->bootvector_low = bootvector & TI_SCI_ADDR_LOW_MASK;
+	req->bootvector_high = (bootvector & TI_SCI_ADDR_HIGH_MASK) >>
+				TI_SCI_ADDR_HIGH_SHIFT;
+	req->config_flags_set = config_flags_set;
+	req->config_flags_clear = config_flags_clear;
+
+	ret = ti_sci_do_xfer(info, xfer);
+	if (ret) {
+		dev_err(dev, "Mbox send fail %d\n", ret);
+		goto fail;
+	}
+
+	resp = (struct ti_sci_msg_hdr *)xfer->tx_message.buf;
+
+	ret = ti_sci_is_response_ack(resp) ? 0 : -ENODEV;
+
+fail:
+	ti_sci_put_one_xfer(&info->minfo, xfer);
+
+	return ret;
+}
+
+/**
+ * ti_sci_cmd_proc_set_control() - Command to set the processor boot
+ *				     control flags
+ * @handle:			Pointer to TI SCI handle
+ * @proc_id:			Processor ID this request is for
+ * @control_flags_set:		Control flags to be set
+ * @control_flags_clear:	Control flags to be cleared
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_cmd_proc_set_control(const struct ti_sci_handle *handle,
+				       u8 proc_id, u32 control_flags_set,
+				       u32 control_flags_clear)
+{
+	struct ti_sci_msg_req_set_ctrl *req;
+	struct ti_sci_msg_hdr *resp;
+	struct ti_sci_info *info;
+	struct ti_sci_xfer *xfer;
+	struct device *dev;
+	int ret = 0;
+
+	if (!handle)
+		return -EINVAL;
+	if (IS_ERR(handle))
+		return PTR_ERR(handle);
+
+	info = handle_to_ti_sci_info(handle);
+	dev = info->dev;
+
+	xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_SET_CTRL,
+				   TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+				   sizeof(*req), sizeof(*resp));
+	if (IS_ERR(xfer)) {
+		ret = PTR_ERR(xfer);
+		dev_err(dev, "Message alloc failed(%d)\n", ret);
+		return ret;
+	}
+	req = (struct ti_sci_msg_req_set_ctrl *)xfer->xfer_buf;
+	req->processor_id = proc_id;
+	req->control_flags_set = control_flags_set;
+	req->control_flags_clear = control_flags_clear;
+
+	ret = ti_sci_do_xfer(info, xfer);
+	if (ret) {
+		dev_err(dev, "Mbox send fail %d\n", ret);
+		goto fail;
+	}
+
+	resp = (struct ti_sci_msg_hdr *)xfer->tx_message.buf;
+
+	ret = ti_sci_is_response_ack(resp) ? 0 : -ENODEV;
+
+fail:
+	ti_sci_put_one_xfer(&info->minfo, xfer);
+
+	return ret;
+}
+
+/**
+ * ti_sci_cmd_get_boot_status() - Command to get the processor boot status
+ * @handle:	Pointer to TI SCI handle
+ * @proc_id:	Processor ID this request is for
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_cmd_proc_get_status(const struct ti_sci_handle *handle,
+				      u8 proc_id, u64 *bv, u32 *cfg_flags,
+				      u32 *ctrl_flags, u32 *sts_flags)
+{
+	struct ti_sci_msg_resp_get_status *resp;
+	struct ti_sci_msg_req_get_status *req;
+	struct ti_sci_info *info;
+	struct ti_sci_xfer *xfer;
+	struct device *dev;
+	int ret = 0;
+
+	if (!handle)
+		return -EINVAL;
+	if (IS_ERR(handle))
+		return PTR_ERR(handle);
+
+	info = handle_to_ti_sci_info(handle);
+	dev = info->dev;
+
+	xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_GET_STATUS,
+				   TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+				   sizeof(*req), sizeof(*resp));
+	if (IS_ERR(xfer)) {
+		ret = PTR_ERR(xfer);
+		dev_err(dev, "Message alloc failed(%d)\n", ret);
+		return ret;
+	}
+	req = (struct ti_sci_msg_req_get_status *)xfer->xfer_buf;
+	req->processor_id = proc_id;
+
+	ret = ti_sci_do_xfer(info, xfer);
+	if (ret) {
+		dev_err(dev, "Mbox send fail %d\n", ret);
+		goto fail;
+	}
+
+	resp = (struct ti_sci_msg_resp_get_status *)xfer->tx_message.buf;
+
+	if (!ti_sci_is_response_ack(resp)) {
+		ret = -ENODEV;
+	} else {
+		*bv = (resp->bootvector_low & TI_SCI_ADDR_LOW_MASK) |
+		      (((u64)resp->bootvector_high << TI_SCI_ADDR_HIGH_SHIFT) &
+		       TI_SCI_ADDR_HIGH_MASK);
+		*cfg_flags = resp->config_flags;
+		*ctrl_flags = resp->control_flags;
+		*sts_flags = resp->status_flags;
+	}
+
+fail:
+	ti_sci_put_one_xfer(&info->minfo, xfer);
+
+	return ret;
+}
+
 /*
  * ti_sci_setup_ops() - Setup the operations structures
  * @info:	pointer to TISCI pointer
@@ -1625,11 +3084,19 @@
 	struct ti_sci_core_ops *core_ops = &ops->core_ops;
 	struct ti_sci_dev_ops *dops = &ops->dev_ops;
 	struct ti_sci_clk_ops *cops = &ops->clk_ops;
+	struct ti_sci_rm_core_ops *rm_core_ops = &ops->rm_core_ops;
+	struct ti_sci_rm_irq_ops *iops = &ops->rm_irq_ops;
+	struct ti_sci_proc_ops *pops = &ops->proc_ops;
+	struct ti_sci_rm_ringacc_ops *rops = &ops->rm_ring_ops;
+	struct ti_sci_rm_psil_ops *psilops = &ops->rm_psil_ops;
+	struct ti_sci_rm_udmap_ops *udmap_ops = &ops->rm_udmap_ops;
 
 	core_ops->reboot_device = ti_sci_cmd_core_reboot;
 
 	dops->get_device = ti_sci_cmd_get_device;
+	dops->get_device_exclusive = ti_sci_cmd_get_device_exclusive;
 	dops->idle_device = ti_sci_cmd_idle_device;
+	dops->idle_device_exclusive = ti_sci_cmd_idle_device_exclusive;
 	dops->put_device = ti_sci_cmd_put_device;
 
 	dops->is_valid = ti_sci_cmd_dev_is_valid;
@@ -1655,6 +3122,39 @@
 	cops->get_best_match_freq = ti_sci_cmd_clk_get_match_freq;
 	cops->set_freq = ti_sci_cmd_clk_set_freq;
 	cops->get_freq = ti_sci_cmd_clk_get_freq;
+
+	rm_core_ops->get_range = ti_sci_cmd_get_resource_range;
+	rm_core_ops->get_range_from_shost =
+				ti_sci_cmd_get_resource_range_from_shost;
+
+	iops->set_direct_irq = ti_sci_cmd_set_direct_irq;
+	iops->set_event_irq = ti_sci_cmd_set_event_irq;
+	iops->set_direct_irq_from_shost = ti_sci_cmd_set_direct_irq_from_shost;
+	iops->set_event_irq_from_shost = ti_sci_cmd_set_event_irq_from_shost;
+	iops->set_event_irq_to_poll = ti_sci_cmd_set_event_irq_to_poll;
+	iops->free_direct_irq = ti_sci_cmd_free_direct_irq;
+	iops->free_event_irq = ti_sci_cmd_free_event_irq;
+	iops->free_direct_irq_from_shost =
+					ti_sci_cmd_free_direct_irq_from_shost;
+	iops->free_event_irq_from_shost = ti_sci_cmd_free_event_irq_from_shost;
+	iops->free_event_irq_to_poll = ti_sci_cmd_free_event_irq_to_poll;
+
+	pops->request = ti_sci_cmd_proc_request;
+	pops->release = ti_sci_cmd_proc_release;
+	pops->handover = ti_sci_cmd_proc_handover;
+	pops->set_config = ti_sci_cmd_proc_set_config;
+	pops->set_control = ti_sci_cmd_proc_set_control;
+	pops->get_status = ti_sci_cmd_proc_get_status;
+
+	rops->config = ti_sci_cmd_ring_config;
+	rops->get_config = ti_sci_cmd_ring_get_config;
+
+	psilops->pair = ti_sci_cmd_rm_psil_pair;
+	psilops->unpair = ti_sci_cmd_rm_psil_unpair;
+
+	udmap_ops->tx_ch_cfg = ti_sci_cmd_rm_udmap_tx_ch_cfg;
+	udmap_ops->rx_ch_cfg = ti_sci_cmd_rm_udmap_rx_ch_cfg;
+	udmap_ops->rx_flow_cfg = ti_sci_cmd_rm_udmap_rx_flow_cfg;
 }
 
 /**
@@ -1779,6 +3279,224 @@
 }
 EXPORT_SYMBOL_GPL(devm_ti_sci_get_handle);
 
+/**
+ * ti_sci_get_by_phandle() - Get the TI SCI handle using DT phandle
+ * @np:		device node
+ * @propname:	property name containing phandle on TISCI node
+ *
+ * NOTE: The function does not track individual clients of the framework
+ * and is expected to be maintained by caller of TI SCI protocol library.
+ * ti_sci_put_handle must be balanced with successful ti_sci_get_by_phandle
+ * Return: pointer to handle if successful, else:
+ * -EPROBE_DEFER if the instance is not ready
+ * -ENODEV if the required node handler is missing
+ * -EINVAL if invalid conditions are encountered.
+ */
+const struct ti_sci_handle *ti_sci_get_by_phandle(struct device_node *np,
+						  const char *property)
+{
+	struct ti_sci_handle *handle = NULL;
+	struct device_node *ti_sci_np;
+	struct ti_sci_info *info;
+	struct list_head *p;
+
+	if (!np) {
+		pr_err("I need a device pointer\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	ti_sci_np = of_parse_phandle(np, property, 0);
+	if (!ti_sci_np)
+		return ERR_PTR(-ENODEV);
+
+	mutex_lock(&ti_sci_list_mutex);
+	list_for_each(p, &ti_sci_list) {
+		info = list_entry(p, struct ti_sci_info, node);
+		if (ti_sci_np == info->dev->of_node) {
+			handle = &info->handle;
+			info->users++;
+			break;
+		}
+	}
+	mutex_unlock(&ti_sci_list_mutex);
+	of_node_put(ti_sci_np);
+
+	if (!handle)
+		return ERR_PTR(-EPROBE_DEFER);
+
+	return handle;
+}
+EXPORT_SYMBOL_GPL(ti_sci_get_by_phandle);
+
+/**
+ * devm_ti_sci_get_by_phandle() - Managed get handle using phandle
+ * @dev:	Device pointer requesting TISCI handle
+ * @propname:	property name containing phandle on TISCI node
+ *
+ * NOTE: This releases the handle once the device resources are
+ * no longer needed. MUST NOT BE released with ti_sci_put_handle.
+ * The function does not track individual clients of the framework
+ * and is expected to be maintained by caller of TI SCI protocol library.
+ *
+ * Return: 0 if all went fine, else corresponding error.
+ */
+const struct ti_sci_handle *devm_ti_sci_get_by_phandle(struct device *dev,
+						       const char *property)
+{
+	const struct ti_sci_handle *handle;
+	const struct ti_sci_handle **ptr;
+
+	ptr = devres_alloc(devm_ti_sci_release, sizeof(*ptr), GFP_KERNEL);
+	if (!ptr)
+		return ERR_PTR(-ENOMEM);
+	handle = ti_sci_get_by_phandle(dev_of_node(dev), property);
+
+	if (!IS_ERR(handle)) {
+		*ptr = handle;
+		devres_add(dev, ptr);
+	} else {
+		devres_free(ptr);
+	}
+
+	return handle;
+}
+EXPORT_SYMBOL_GPL(devm_ti_sci_get_by_phandle);
+
+/*
+ * ti_sci_get_free_resource() - Get a free resource from TISCI resource.
+ * @res:	Pointer to the TISCI resource
+ *
+ * Return: resource num if all went ok else TI_SCI_RESOURCE_NULL.
+ */
+u16 ti_sci_get_free_resource(struct ti_sci_resource *res)
+{
+	unsigned long flags;
+	u16 set, free_bit;
+
+	raw_spin_lock_irqsave(&res->lock, flags);
+	for (set = 0; set < res->sets; set++) {
+		free_bit = find_first_zero_bit(res->desc[set].res_map,
+					       res->desc[set].num);
+		if (free_bit != res->desc[set].num) {
+			set_bit(free_bit, res->desc[set].res_map);
+			raw_spin_unlock_irqrestore(&res->lock, flags);
+			return res->desc[set].start + free_bit;
+		}
+	}
+	raw_spin_unlock_irqrestore(&res->lock, flags);
+
+	return TI_SCI_RESOURCE_NULL;
+}
+EXPORT_SYMBOL_GPL(ti_sci_get_free_resource);
+
+/**
+ * ti_sci_release_resource() - Release a resource from TISCI resource.
+ * @res:	Pointer to the TISCI resource
+ */
+void ti_sci_release_resource(struct ti_sci_resource *res, u16 id)
+{
+	unsigned long flags;
+	u16 set;
+
+	raw_spin_lock_irqsave(&res->lock, flags);
+	for (set = 0; set < res->sets; set++) {
+		if (res->desc[set].start <= id &&
+		    (res->desc[set].num + res->desc[set].start) > id)
+			clear_bit(id - res->desc[set].start,
+				  res->desc[set].res_map);
+	}
+	raw_spin_unlock_irqrestore(&res->lock, flags);
+}
+EXPORT_SYMBOL_GPL(ti_sci_release_resource);
+
+/**
+ * devm_ti_sci_get_of_resource() - Get a TISCI resource assigned to a device
+ * @handle:	TISCI handle
+ * @dev:	Device pointer to which the resource is assigned
+ * @of_prop:	property name by which the resource are represented
+ *
+ * Note: This function expects of_prop to be in the form of tuples
+ *	<type, subtype>. Allocates and initializes ti_sci_resource structure
+ *	for each of_prop. Client driver can directly call
+ *	ti_sci_(get_free, release)_resource apis for handling the resource.
+ *
+ * Return: Pointer to ti_sci_resource if all went well else appropriate
+ *	   error pointer.
+ */
+struct ti_sci_resource *
+devm_ti_sci_get_of_resource(const struct ti_sci_handle *handle,
+			    struct device *dev, u32 dev_id, char *of_prop)
+{
+	u32 resource_subtype;
+	u16 resource_type;
+	struct ti_sci_resource *res;
+	bool valid_set = false;
+	int sets, i, ret;
+
+	res = devm_kzalloc(dev, sizeof(*res), GFP_KERNEL);
+	if (!res)
+		return ERR_PTR(-ENOMEM);
+
+	sets = of_property_count_elems_of_size(dev_of_node(dev), of_prop,
+					       sizeof(u32));
+	if (sets < 0) {
+		dev_err(dev, "%s resource type ids not available\n", of_prop);
+		return ERR_PTR(sets);
+	}
+
+	res->sets = sets;
+
+	res->desc = devm_kcalloc(dev, res->sets, sizeof(*res->desc),
+				 GFP_KERNEL);
+	if (!res->desc)
+		return ERR_PTR(-ENOMEM);
+
+	ret = ti_sci_get_resource_type(handle_to_ti_sci_info(handle), dev_id,
+				       &resource_type);
+	if (ret) {
+		dev_err(dev, "No valid resource type for %u\n", dev_id);
+		return ERR_PTR(-EINVAL);
+	}
+
+	for (i = 0; i < res->sets; i++) {
+		ret = of_property_read_u32_index(dev_of_node(dev), of_prop, i,
+						 &resource_subtype);
+		if (ret)
+			return ERR_PTR(-EINVAL);
+
+		ret = handle->ops.rm_core_ops.get_range(handle, dev_id,
+							resource_subtype,
+							&res->desc[i].start,
+							&res->desc[i].num);
+		if (ret) {
+			dev_dbg(dev, "type %d subtype %d not allocated for host %d\n",
+				resource_type, resource_subtype,
+				handle_to_ti_sci_info(handle)->host_id);
+			res->desc[i].start = 0;
+			res->desc[i].num = 0;
+			continue;
+		}
+
+		valid_set = true;
+		dev_dbg(dev, "res type = %d, subtype = %d, start = %d, num = %d\n",
+			resource_type, resource_subtype, res->desc[i].start,
+			res->desc[i].num);
+
+		res->desc[i].res_map =
+			devm_kzalloc(dev, BITS_TO_LONGS(res->desc[i].num) *
+				     sizeof(*res->desc[i].res_map), GFP_KERNEL);
+		if (!res->desc[i].res_map)
+			return ERR_PTR(-ENOMEM);
+	}
+	raw_spin_lock_init(&res->lock);
+
+	if (valid_set)
+		return res;
+
+	return ERR_PTR(-EINVAL);
+}
+EXPORT_SYMBOL_GPL(devm_ti_sci_get_of_resource);
+
 static int tisci_reboot_handler(struct notifier_block *nb, unsigned long mode,
 				void *cmd)
 {
@@ -1793,16 +3511,39 @@
 
 /* Description for K2G */
 static const struct ti_sci_desc ti_sci_pmmc_k2g_desc = {
-	.host_id = 2,
+	.default_host_id = 2,
 	/* Conservative duration */
-	.max_rx_timeout_ms = 1000,
+	.max_rx_timeout_ms = 10000,
 	/* Limited by MBOX_TX_QUEUE_LEN. K2G can handle upto 128 messages! */
 	.max_msgs = 20,
 	.max_msg_size = 64,
+	.rm_type_map = NULL,
+};
+
+static struct ti_sci_rm_type_map ti_sci_am654_rm_type_map[] = {
+	{.dev_id = 56, .type = 0x00b}, /* GIC_IRQ */
+	{.dev_id = 179, .type = 0x000}, /* MAIN_NAV_UDMASS_IA0 */
+	{.dev_id = 187, .type = 0x009}, /* MAIN_NAV_RA */
+	{.dev_id = 188, .type = 0x006}, /* MAIN_NAV_UDMAP */
+	{.dev_id = 194, .type = 0x007}, /* MCU_NAV_UDMAP */
+	{.dev_id = 195, .type = 0x00a}, /* MCU_NAV_RA */
+	{.dev_id = 0, .type = 0x000}, /* end of table */
+};
+
+/* Description for AM654 */
+static const struct ti_sci_desc ti_sci_pmmc_am654_desc = {
+	.default_host_id = 12,
+	/* Conservative duration */
+	.max_rx_timeout_ms = 10000,
+	/* Limited by MBOX_TX_QUEUE_LEN. K2G can handle upto 128 messages! */
+	.max_msgs = 20,
+	.max_msg_size = 60,
+	.rm_type_map = ti_sci_am654_rm_type_map,
 };
 
 static const struct of_device_id ti_sci_of_match[] = {
 	{.compatible = "ti,k2g-sci", .data = &ti_sci_pmmc_k2g_desc},
+	{.compatible = "ti,am654-sci", .data = &ti_sci_pmmc_am654_desc},
 	{ /* Sentinel */ },
 };
 MODULE_DEVICE_TABLE(of, ti_sci_of_match);
@@ -1819,6 +3560,7 @@
 	int ret = -EINVAL;
 	int i;
 	int reboot = 0;
+	u32 h_id;
 
 	of_id = of_match_device(ti_sci_of_match, dev);
 	if (!of_id) {
@@ -1833,6 +3575,19 @@
 
 	info->dev = dev;
 	info->desc = desc;
+	ret = of_property_read_u32(dev->of_node, "ti,host-id", &h_id);
+	/* if the property is not present in DT, use a default from desc */
+	if (ret < 0) {
+		info->host_id = info->desc->default_host_id;
+	} else {
+		if (!h_id) {
+			dev_warn(dev, "Host ID 0 is reserved for firmware\n");
+			info->host_id = info->desc->default_host_id;
+		} else {
+			info->host_id = h_id;
+		}
+	}
+
 	reboot = of_property_read_bool(dev->of_node,
 				       "ti,system-reboot-controller");
 	INIT_LIST_HEAD(&info->node);
diff -Naur --no-dereference a/drivers/firmware/ti_sci.h b/drivers/firmware/ti_sci.h
--- a/drivers/firmware/ti_sci.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/firmware/ti_sci.h	2020-06-19 21:00:31.408227715 -0400
@@ -35,6 +35,50 @@
 #define TI_SCI_MSG_QUERY_CLOCK_FREQ	0x010d
 #define TI_SCI_MSG_GET_CLOCK_FREQ	0x010e
 
+/* Resource Management Requests */
+#define TI_SCI_MSG_GET_RESOURCE_RANGE	0x1500
+
+/* IRQ requests */
+#define TI_SCI_MSG_SET_IRQ		0x1000
+#define TI_SCI_MSG_FREE_IRQ		0x1001
+
+/* Processor Control requests */
+#define TI_SCI_MSG_PROC_REQUEST		0xc000
+#define TI_SCI_MSG_PROC_RELEASE		0xc001
+#define TI_SCI_MSG_PROC_HANDOVER	0xc005
+#define TI_SCI_MSG_SET_CONFIG		0xc100
+#define TI_SCI_MSG_SET_CTRL		0xc101
+#define TI_SCI_MSG_GET_STATUS		0xc400
+
+/* NAVSS resource management */
+/* Ringacc requests */
+#define TI_SCI_MSG_RM_RING_ALLOCATE		0x1100
+#define TI_SCI_MSG_RM_RING_FREE			0x1101
+#define TI_SCI_MSG_RM_RING_RECONFIG		0x1102
+#define TI_SCI_MSG_RM_RING_RESET		0x1103
+#define TI_SCI_MSG_RM_RING_CFG			0x1110
+#define TI_SCI_MSG_RM_RING_GET_CFG		0x1111
+
+/* PSI-L requests */
+#define TI_SCI_MSG_RM_PSIL_PAIR			0x1280
+#define TI_SCI_MSG_RM_PSIL_UNPAIR		0x1281
+
+#define TI_SCI_MSG_RM_UDMAP_TX_ALLOC		0x1200
+#define TI_SCI_MSG_RM_UDMAP_TX_FREE		0x1201
+#define TI_SCI_MSG_RM_UDMAP_RX_ALLOC		0x1210
+#define TI_SCI_MSG_RM_UDMAP_RX_FREE		0x1211
+#define TI_SCI_MSG_RM_UDMAP_FLOW_CFG		0x1220
+#define TI_SCI_MSG_RM_UDMAP_OPT_FLOW_CFG	0x1221
+
+#define TISCI_MSG_RM_UDMAP_TX_CH_CFG		0x1205
+#define TISCI_MSG_RM_UDMAP_TX_CH_GET_CFG	0x1206
+#define TISCI_MSG_RM_UDMAP_RX_CH_CFG		0x1215
+#define TISCI_MSG_RM_UDMAP_RX_CH_GET_CFG	0x1216
+#define TISCI_MSG_RM_UDMAP_FLOW_CFG		0x1230
+#define TISCI_MSG_RM_UDMAP_FLOW_SIZE_THRESH_CFG	0x1231
+#define TISCI_MSG_RM_UDMAP_FLOW_GET_CFG		0x1232
+#define TISCI_MSG_RM_UDMAP_FLOW_SIZE_THRESH_GET_CFG	0x1233
+
 /**
  * struct ti_sci_msg_hdr - Generic Message Header for All messages and responses
  * @type:	Type of messages: One of TI_SCI_MSG* values
@@ -195,7 +239,8 @@
  * @dev_id:	Device identifier this request is for
  * @clk_id:	Clock identifier for the device for this request.
  *		Each device has it's own set of clock inputs. This indexes
- *		which clock input to modify.
+ *		which clock input to modify. Set to 255 if clock ID is
+ *		greater than or equal to 255.
  * @request_state: Request the state for the clock to be set to.
  *		MSG_CLOCK_SW_STATE_UNREQ: The IP does not require this clock,
  *		it can be disabled, regardless of the state of the device
@@ -206,6 +251,9 @@
  *		being required by the device.(default)
  *		MSG_CLOCK_SW_STATE_REQ:  Configure the clock to be enabled,
  *		regardless of the state of the device.
+ * @clk_id_32:	Clock identifier for the device for this request.
+ *		Only to be used if the clock ID is greater than or equal to
+ *		255.
  *
  * Normally, all required clocks are managed by TISCI entity, this is used
  * only for specific control *IF* required. Auto managed state is
@@ -227,6 +275,7 @@
 #define MSG_CLOCK_SW_STATE_AUTO		1
 #define MSG_CLOCK_SW_STATE_REQ		2
 	u8 request_state;
+	u32 clk_id_32;
 } __packed;
 
 /**
@@ -235,7 +284,11 @@
  * @dev_id:	Device identifier this request is for
  * @clk_id:	Clock identifier for the device for this request.
  *		Each device has it's own set of clock inputs. This indexes
- *		which clock input to get state of.
+ *		which clock input to get state of. Set to 255 if the clock
+ *		ID is greater than or equal to 255.
+ * @clk_id_32:	Clock identifier for the device for the request.
+ *		Only to be used if the clock ID is greater than or equal to
+ *		255.
  *
  * Request type is TI_SCI_MSG_GET_CLOCK_STATE, response is state
  * of the clock
@@ -244,6 +297,7 @@
 	struct ti_sci_msg_hdr hdr;
 	u32 dev_id;
 	u8 clk_id;
+	u32 clk_id_32;
 } __packed;
 
 /**
@@ -271,9 +325,13 @@
  * @dev_id:	Device identifier this request is for
  * @clk_id:	Clock identifier for the device for this request.
  *		Each device has it's own set of clock inputs. This indexes
- *		which clock input to modify.
+ *		which clock input to modify. Set to 255 if clock ID is
+ *		greater than or equal to 255.
  * @parent_id:	The new clock parent is selectable by an index via this
- *		parameter.
+ *		parameter. Set to 255 if clock ID is greater than or
+ *		equal to 255.
+ * @clk_id_32:	Clock identifier if @clk_id field is 255.
+ * @parent_id_32:	Parent identifier if @parent_id is 255.
  *
  * Request type is TI_SCI_MSG_SET_CLOCK_PARENT, response is generic
  * ACK / NACK message.
@@ -283,6 +341,8 @@
 	u32 dev_id;
 	u8 clk_id;
 	u8 parent_id;
+	u32 clk_id_32;
+	u32 parent_id_32;
 } __packed;
 
 /**
@@ -291,7 +351,10 @@
  * @dev_id:	Device identifier this request is for
  * @clk_id:	Clock identifier for the device for this request.
  *		Each device has it's own set of clock inputs. This indexes
- *		which clock input to get the parent for.
+ *		which clock input to get the parent for. If this field
+ *		contains 255, the actual clock identifier is stored in
+ *		@clk_id_32.
+ * @clk_id_32:	Clock identifier if the @clk_id field contains 255.
  *
  * Request type is TI_SCI_MSG_GET_CLOCK_PARENT, response is parent information
  */
@@ -299,25 +362,32 @@
 	struct ti_sci_msg_hdr hdr;
 	u32 dev_id;
 	u8 clk_id;
+	u32 clk_id_32;
 } __packed;
 
 /**
  * struct ti_sci_msg_resp_get_clock_parent - Response with clock parent
  * @hdr:	Generic Header
- * @parent_id:	The current clock parent
+ * @parent_id:	The current clock parent. If set to 255, the current parent
+ *		ID can be found from the @parent_id_32 field.
+ * @parent_id_32:	Current clock parent if @parent_id field is set to
+ *			255.
  *
  * Response to TI_SCI_MSG_GET_CLOCK_PARENT.
  */
 struct ti_sci_msg_resp_get_clock_parent {
 	struct ti_sci_msg_hdr hdr;
 	u8 parent_id;
+	u32 parent_id_32;
 } __packed;
 
 /**
  * struct ti_sci_msg_req_get_clock_num_parents - Request to get clock parents
  * @hdr:	Generic header
  * @dev_id:	Device identifier this request is for
- * @clk_id:	Clock identifier for the device for this request.
+ * @clk_id:	Clock identifier for the device for this request. Set to
+ *		255 if clock ID is greater than or equal to 255.
+ * @clk_id_32:	Clock identifier if the @clk_id field contains 255.
  *
  * This request provides information about how many clock parent options
  * are available for a given clock to a device. This is typically used
@@ -330,18 +400,24 @@
 	struct ti_sci_msg_hdr hdr;
 	u32 dev_id;
 	u8 clk_id;
+	u32 clk_id_32;
 } __packed;
 
 /**
  * struct ti_sci_msg_resp_get_clock_num_parents - Response for get clk parents
  * @hdr:		Generic header
- * @num_parents:	Number of clock parents
+ * @num_parents:	Number of clock parents. If set to 255, the actual
+ *			number of parents is stored into @num_parents_32
+ *			field instead.
+ * @num_parents_32:	Number of clock parents if @num_parents field is
+ *			set to 255.
  *
  * Response to TI_SCI_MSG_GET_NUM_CLOCK_PARENTS
  */
 struct ti_sci_msg_resp_get_clock_num_parents {
 	struct ti_sci_msg_hdr hdr;
 	u8 num_parents;
+	u32 num_parents_32;
 } __packed;
 
 /**
@@ -356,7 +432,9 @@
  * @max_freq_hz: The maximum allowable frequency in Hz. This is the maximum
  *		allowable programmed frequency and does not account for clock
  *		tolerances and jitter.
- * @clk_id:	Clock identifier for the device for this request.
+ * @clk_id:	Clock identifier for the device for this request. Set to
+ *		255 if clock identifier is greater than or equal to 255.
+ * @clk_id_32:	Clock identifier if @clk_id is set to 255.
  *
  * NOTE: Normally clock frequency management is automatically done by TISCI
  * entity. In case of specific requests, TISCI evaluates capability to achieve
@@ -373,6 +451,7 @@
 	u64 target_freq_hz;
 	u64 max_freq_hz;
 	u8 clk_id;
+	u32 clk_id_32;
 } __packed;
 
 /**
@@ -400,7 +479,9 @@
  * @max_freq_hz: The maximum allowable frequency in Hz. This is the maximum
  *		allowable programmed frequency and does not account for clock
  *		tolerances and jitter.
- * @clk_id:	Clock identifier for the device for this request.
+ * @clk_id:	Clock identifier for the device for this request. Set to
+ *		255 if clock ID is greater than or equal to 255.
+ * @clk_id_32:	Clock identifier if @clk_id field is set to 255.
  *
  * NOTE: Normally clock frequency management is automatically done by TISCI
  * entity. In case of specific requests, TISCI evaluates capability to achieve
@@ -429,13 +510,16 @@
 	u64 target_freq_hz;
 	u64 max_freq_hz;
 	u8 clk_id;
+	u32 clk_id_32;
 } __packed;
 
 /**
  * struct ti_sci_msg_req_get_clock_freq - Request to get the clock frequency
  * @hdr:	Generic Header
  * @dev_id:	Device identifier this request is for
- * @clk_id:	Clock identifier for the device for this request.
+ * @clk_id:	Clock identifier for the device for this request. Set to
+ *		255 if clock ID is greater than or equal to 255.
+ * @clk_id_32:	Clock identifier if @clk_id field is set to 255.
  *
  * NOTE: Normally clock frequency management is automatically done by TISCI
  * entity. In some cases, clock frequencies are configured by host.
@@ -447,6 +531,7 @@
 	struct ti_sci_msg_hdr hdr;
 	u32 dev_id;
 	u8 clk_id;
+	u32 clk_id_32;
 } __packed;
 
 /**
@@ -461,4 +546,902 @@
 	u64 freq_hz;
 } __packed;
 
+#define TI_SCI_IRQ_SECONDARY_HOST_INVALID	0xff
+
+/**
+ * struct ti_sci_msg_req_get_resource_range - Request to get a host's assigned
+ *					      range of resources.
+ * @hdr:		Generic Header
+ * @type:		Unique resource assignment type
+ * @subtype:		Resource assignment subtype within the resource type.
+ * @secondary_host:	Host processing entity to which the resources are
+ *			allocated. This is required only when the destination
+ *			host id id different from ti sci interface host id,
+ *			else TI_SCI_IRQ_SECONDARY_HOST_INVALID can be passed.
+ *
+ * Request type is TI_SCI_MSG_GET_RESOURCE_RANGE. Responded with requested
+ * resource range which is of type TI_SCI_MSG_GET_RESOURCE_RANGE.
+ */
+struct ti_sci_msg_req_get_resource_range {
+	struct ti_sci_msg_hdr hdr;
+#define MSG_RM_RESOURCE_TYPE_MASK	GENMASK(9, 0)
+#define MSG_RM_RESOURCE_SUBTYPE_MASK	GENMASK(5, 0)
+	u16 type;
+	u8 subtype;
+	u8 secondary_host;
+} __packed;
+
+/**
+ * struct ti_sci_msg_resp_get_resource_range - Response to resource get range.
+ * @hdr:		Generic Header
+ * @range_start:	Start index of the resource range.
+ * @range_num:		Number of resources in the range.
+ *
+ * Response to request TI_SCI_MSG_GET_RESOURCE_RANGE.
+ */
+struct ti_sci_msg_resp_get_resource_range {
+	struct ti_sci_msg_hdr hdr;
+	u16 range_start;
+	u16 range_num;
+} __packed;
+
+/**
+ * struct ti_sci_msg_req_manage_irq - Request to configure/release the route
+ *					between the dev and the host.
+ * @hdr:		Generic Header
+ * @valid_params:	Bit fields defining the validity of interrupt source
+ *			parameters. If a bit is not set, then corresponding
+ *			field is not valid and will not be used for route set.
+ *			Bit field definitions:
+ *			0 - Valid bit for @dst_id
+ *			1 - Valid bit for @dst_host_irq
+ *			2 - Valid bit for @ia_id
+ *			3 - Valid bit for @vint
+ *			4 - Valid bit for @global_event
+ *			5 - Valid bit for @vint_status_bit_index
+ *			31 - Valid bit for @secondary_host
+ * @src_id:		IRQ source peripheral ID.
+ * @src_index:		IRQ source index within the peripheral
+ * @dst_id:		IRQ Destination ID. Based on the architecture it can be
+ *			IRQ controller or host processor ID.
+ * @dst_host_irq:	IRQ number of the destination host IRQ controller
+ * @ia_id:		Device ID of the interrupt aggregator in which the
+ *			vint resides.
+ * @vint:		Virtual interrupt number if the interrupt route
+ *			is through an interrupt aggregator.
+ * @global_event:	Global event that is to be mapped to interrupt
+ *			aggregator virtual interrupt status bit.
+ * @vint_status_bit:	Virtual interrupt status bit if the interrupt route
+ *			utilizes an interrupt aggregator status bit.
+ * @secondary_host:	Host ID of the IRQ destination computing entity. This is
+ *			required only when destination host id is different
+ *			from ti sci interface host id.
+ *
+ * Request type is TI_SCI_MSG_SET/RELEASE_IRQ.
+ * Response is generic ACK / NACK message.
+ */
+struct ti_sci_msg_req_manage_irq {
+	struct ti_sci_msg_hdr hdr;
+#define MSG_FLAG_DST_ID_VALID			TI_SCI_MSG_FLAG(0)
+#define MSG_FLAG_DST_HOST_IRQ_VALID		TI_SCI_MSG_FLAG(1)
+#define MSG_FLAG_IA_ID_VALID			TI_SCI_MSG_FLAG(2)
+#define MSG_FLAG_VINT_VALID			TI_SCI_MSG_FLAG(3)
+#define MSG_FLAG_GLB_EVNT_VALID			TI_SCI_MSG_FLAG(4)
+#define MSG_FLAG_VINT_STS_BIT_VALID		TI_SCI_MSG_FLAG(5)
+#define MSG_FLAG_SHOST_VALID			TI_SCI_MSG_FLAG(31)
+	u32 valid_params;
+	u16 src_id;
+	u16 src_index;
+	u16 dst_id;
+	u16 dst_host_irq;
+	u16 ia_id;
+	u16 vint;
+	u16 global_event;
+	u8 vint_status_bit;
+	u8 secondary_host;
+} __packed;
+
+/**
+ * struct ti_sci_msg_req_proc_request - Request a processor
+ * @hdr:		Generic Header
+ * @processor_id:	ID of processor being requested
+ *
+ * Request type is TI_SCI_MSG_PROC_REQUEST, response is a generic ACK/NACK
+ * message.
+ */
+struct ti_sci_msg_req_proc_request {
+	struct ti_sci_msg_hdr hdr;
+	u8 processor_id;
+} __packed;
+
+/**
+ * struct ti_sci_msg_req_proc_release - Release a processor
+ * @hdr:		Generic Header
+ * @processor_id:	ID of processor being released
+ *
+ * Request type is TI_SCI_MSG_PROC_RELEASE, response is a generic ACK/NACK
+ * message.
+ */
+struct ti_sci_msg_req_proc_release {
+	struct ti_sci_msg_hdr hdr;
+	u8 processor_id;
+} __packed;
+
+/**
+ * struct ti_sci_msg_req_proc_handover - Handover a processor to a host
+ * @hdr:		Generic Header
+ * @processor_id:	ID of processor being handed over
+ * @host_id:		Host ID the control needs to be transferred to
+ *
+ * Request type is TI_SCI_MSG_PROC_HANDOVER, response is a generic ACK/NACK
+ * message.
+ */
+struct ti_sci_msg_req_proc_handover {
+	struct ti_sci_msg_hdr hdr;
+	u8 processor_id;
+	u8 host_id;
+} __packed;
+
+/* Boot Vector masks */
+#define TI_SCI_ADDR_LOW_MASK			GENMASK_ULL(31, 0)
+#define TI_SCI_ADDR_HIGH_MASK			GENMASK_ULL(63, 32)
+#define TI_SCI_ADDR_HIGH_SHIFT			32
+
+/**
+ * struct ti_sci_msg_req_set_config - Set Processor boot configuration
+ * @hdr:		Generic Header
+ * @processor_id:	ID of processor being configured
+ * @bootvector_low:	Lower 32 bit address (Little Endian) of boot vector
+ * @bootvector_high:	Higher 32 bit address (Little Endian) of boot vector
+ * @config_flags_set:	Optional Processor specific Config Flags to set.
+ *			Setting a bit here implies the corresponding mode
+ *			will be set
+ * @config_flags_clear:	Optional Processor specific Config Flags to clear.
+ *			Setting a bit here implies the corresponding mode
+ *			will be cleared
+ *
+ * Request type is TI_SCI_MSG_PROC_HANDOVER, response is a generic ACK/NACK
+ * message.
+ */
+struct ti_sci_msg_req_set_config {
+	struct ti_sci_msg_hdr hdr;
+	u8 processor_id;
+	u32 bootvector_low;
+	u32 bootvector_high;
+	u32 config_flags_set;
+	u32 config_flags_clear;
+} __packed;
+
+/**
+ * struct ti_sci_msg_req_set_ctrl - Set Processor boot control flags
+ * @hdr:		Generic Header
+ * @processor_id:	ID of processor being configured
+ * @control_flags_set:	Optional Processor specific Control Flags to set.
+ *			Setting a bit here implies the corresponding mode
+ *			will be set
+ * @control_flags_clear:Optional Processor specific Control Flags to clear.
+ *			Setting a bit here implies the corresponding mode
+ *			will be cleared
+ *
+ * Request type is TI_SCI_MSG_SET_CTRL, response is a generic ACK/NACK
+ * message.
+ */
+struct ti_sci_msg_req_set_ctrl {
+	struct ti_sci_msg_hdr hdr;
+	u8 processor_id;
+	u32 control_flags_set;
+	u32 control_flags_clear;
+} __packed;
+
+/**
+ * struct ti_sci_msg_req_get_status - Processor boot status request
+ * @hdr:		Generic Header
+ * @processor_id:	ID of processor whose status is being requested
+ *
+ * Request type is TI_SCI_MSG_GET_STATUS, response is an appropriate
+ * message, or NACK in case of inability to satisfy request.
+ */
+struct ti_sci_msg_req_get_status {
+	struct ti_sci_msg_hdr hdr;
+	u8 processor_id;
+} __packed;
+
+/**
+ * struct ti_sci_msg_resp_get_status - Processor boot status response
+ * @hdr:		Generic Header
+ * @processor_id:	ID of processor whose status is returned
+ * @bootvector_low:	Lower 32 bit address (Little Endian) of boot vector
+ * @bootvector_high:	Higher 32 bit address (Little Endian) of boot vector
+ * @config_flags:	Optional Processor specific Config Flags set currently
+ * @control_flags:	Optional Processor specific Control Flags set currently
+ * @status_flags:	Optional Processor specific Status Flags set currently
+ *
+ * Response structure to a TI_SCI_MSG_GET_STATUS request.
+ */
+struct ti_sci_msg_resp_get_status {
+	struct ti_sci_msg_hdr hdr;
+	u8 processor_id;
+	u32 bootvector_low;
+	u32 bootvector_high;
+	u32 config_flags;
+	u32 control_flags;
+	u32 status_flags;
+} __packed;
+
+/**
+ * struct ti_sci_msg_rm_ring_cfg_req - Configure a Navigator Subsystem ring
+ *
+ * Configures the non-real-time registers of a Navigator Subsystem ring.
+ * @hdr:	Generic Header
+ * @valid_params: Bitfield defining validity of ring configuration parameters.
+ *	The ring configuration fields are not valid, and will not be used for
+ *	ring configuration, if their corresponding valid bit is zero.
+ *	Valid bit usage:
+ *	0 - Valid bit for @tisci_msg_rm_ring_cfg_req addr_lo
+ *	1 - Valid bit for @tisci_msg_rm_ring_cfg_req addr_hi
+ *	2 - Valid bit for @tisci_msg_rm_ring_cfg_req count
+ *	3 - Valid bit for @tisci_msg_rm_ring_cfg_req mode
+ *	4 - Valid bit for @tisci_msg_rm_ring_cfg_req size
+ *	5 - Valid bit for @tisci_msg_rm_ring_cfg_req order_id
+ * @nav_id: Device ID of Navigator Subsystem from which the ring is allocated
+ * @index: ring index to be configured.
+ * @addr_lo: 32 LSBs of ring base address to be programmed into the ring's
+ *	RING_BA_LO register
+ * @addr_hi: 16 MSBs of ring base address to be programmed into the ring's
+ *	RING_BA_HI register.
+ * @count: Number of ring elements. Must be even if mode is CREDENTIALS or QM
+ *	modes.
+ * @mode: Specifies the mode the ring is to be configured.
+ * @size: Specifies encoded ring element size. To calculate the encoded size use
+ *	the formula (log2(size_bytes) - 2), where size_bytes cannot be
+ *	greater than 256.
+ * @order_id: Specifies the ring's bus order ID.
+ */
+struct ti_sci_msg_rm_ring_cfg_req {
+	struct ti_sci_msg_hdr hdr;
+	u32 valid_params;
+	u16 nav_id;
+	u16 index;
+	u32 addr_lo;
+	u32 addr_hi;
+	u32 count;
+	u8 mode;
+	u8 size;
+	u8 order_id;
+} __packed;
+
+/**
+ * struct ti_sci_msg_rm_ring_cfg_resp - Response to configuring a ring.
+ *
+ * @hdr:	Generic Header
+ */
+struct ti_sci_msg_rm_ring_cfg_resp {
+	struct ti_sci_msg_hdr hdr;
+} __packed;
+
+/**
+ * struct ti_sci_msg_rm_ring_get_cfg_req - Get RA ring's configuration
+ *
+ * Gets the configuration of the non-real-time register fields of a ring.  The
+ * host, or a supervisor of the host, who owns the ring must be the requesting
+ * host.  The values of the non-real-time registers are returned in
+ * @ti_sci_msg_rm_ring_get_cfg_resp.
+ *
+ * @hdr: Generic Header
+ * @nav_id: Device ID of Navigator Subsystem from which the ring is allocated
+ * @index: ring index.
+ */
+struct ti_sci_msg_rm_ring_get_cfg_req {
+	struct ti_sci_msg_hdr hdr;
+	u16 nav_id;
+	u16 index;
+} __packed;
+
+/**
+ * struct ti_sci_msg_rm_ring_get_cfg_resp -  Ring get configuration response
+ *
+ * Response received by host processor after RM has handled
+ * @ti_sci_msg_rm_ring_get_cfg_req. The response contains the ring's
+ * non-real-time register values.
+ *
+ * @hdr: Generic Header
+ * @addr_lo: Ring 32 LSBs of base address
+ * @addr_hi: Ring 16 MSBs of base address.
+ * @count: Ring number of elements.
+ * @mode: Ring mode.
+ * @size: encoded Ring element size
+ * @order_id: ing order ID.
+ */
+struct ti_sci_msg_rm_ring_get_cfg_resp {
+	struct ti_sci_msg_hdr hdr;
+	u32 addr_lo;
+	u32 addr_hi;
+	u32 count;
+	u8 mode;
+	u8 size;
+	u8 order_id;
+} __packed;
+
+/**
+ * struct ti_sci_msg_psil_pair - Pairs a PSI-L source thread to a destination
+ *				 thread
+ * @hdr:	Generic Header
+ * @nav_id:	SoC Navigator Subsystem device ID whose PSI-L config proxy is
+ *		used to pair the source and destination threads.
+ * @src_thread:	PSI-L source thread ID within the PSI-L System thread map.
+ *
+ * UDMAP transmit channels mapped to source threads will have their
+ * TCHAN_THRD_ID register programmed with the destination thread if the pairing
+ * is successful.
+
+ * @dst_thread: PSI-L destination thread ID within the PSI-L System thread map.
+ * PSI-L destination threads start at index 0x8000.  The request is NACK'd if
+ * the destination thread is not greater than or equal to 0x8000.
+ *
+ * UDMAP receive channels mapped to destination threads will have their
+ * RCHAN_THRD_ID register programmed with the source thread if the pairing
+ * is successful.
+ *
+ * Request type is TI_SCI_MSG_RM_PSIL_PAIR, response is a generic ACK or NACK
+ * message.
+ */
+struct ti_sci_msg_psil_pair {
+	struct ti_sci_msg_hdr hdr;
+	u32 nav_id;
+	u32 src_thread;
+	u32 dst_thread;
+} __packed;
+
+/**
+ * struct ti_sci_msg_psil_unpair - Unpairs a PSI-L source thread from a
+ *				   destination thread
+ * @hdr:	Generic Header
+ * @nav_id:	SoC Navigator Subsystem device ID whose PSI-L config proxy is
+ *		used to unpair the source and destination threads.
+ * @src_thread:	PSI-L source thread ID within the PSI-L System thread map.
+ *
+ * UDMAP transmit channels mapped to source threads will have their
+ * TCHAN_THRD_ID register cleared if the unpairing is successful.
+ *
+ * @dst_thread: PSI-L destination thread ID within the PSI-L System thread map.
+ * PSI-L destination threads start at index 0x8000.  The request is NACK'd if
+ * the destination thread is not greater than or equal to 0x8000.
+ *
+ * UDMAP receive channels mapped to destination threads will have their
+ * RCHAN_THRD_ID register cleared if the unpairing is successful.
+ *
+ * Request type is TI_SCI_MSG_RM_PSIL_UNPAIR, response is a generic ACK or NACK
+ * message.
+ */
+struct ti_sci_msg_psil_unpair {
+	struct ti_sci_msg_hdr hdr;
+	u32 nav_id;
+	u32 src_thread;
+	u32 dst_thread;
+} __packed;
+
+/**
+ * struct ti_sci_msg_udmap_rx_flow_cfg -  UDMAP receive flow configuration
+ *					  message
+ * @hdr: Generic Header
+ * @nav_id: SoC Navigator Subsystem device ID from which the receive flow is
+ *	allocated
+ * @flow_index: UDMAP receive flow index for non-optional configuration.
+ * @rx_ch_index: Specifies the index of the receive channel using the
+ *	flow_index.
+ * @rx_einfo_present: UDMAP receive flow extended packet info present.
+ * @rx_psinfo_present: UDMAP receive flow PS words present.
+ * @rx_error_handling: UDMAP receive flow error handling configuration. Valid
+ *	values are TI_SCI_RM_UDMAP_RX_FLOW_ERR_DROP/RETRY.
+ * @rx_desc_type: UDMAP receive flow descriptor type. It can be one of
+ *	TI_SCI_RM_UDMAP_RX_FLOW_DESC_HOST/MONO.
+ * @rx_sop_offset: UDMAP receive flow start of packet offset.
+ * @rx_dest_qnum: UDMAP receive flow destination queue number.
+ * @rx_ps_location: UDMAP receive flow PS words location.
+ *	0 - end of packet descriptor
+ *	1 - Beginning of the data buffer
+ * @rx_src_tag_hi: UDMAP receive flow source tag high byte constant
+ * @rx_src_tag_lo: UDMAP receive flow source tag low byte constant
+ * @rx_dest_tag_hi: UDMAP receive flow destination tag high byte constant
+ * @rx_dest_tag_lo: UDMAP receive flow destination tag low byte constant
+ * @rx_src_tag_hi_sel: UDMAP receive flow source tag high byte selector
+ * @rx_src_tag_lo_sel: UDMAP receive flow source tag low byte selector
+ * @rx_dest_tag_hi_sel: UDMAP receive flow destination tag high byte selector
+ * @rx_dest_tag_lo_sel: UDMAP receive flow destination tag low byte selector
+ * @rx_size_thresh_en: UDMAP receive flow packet size based free buffer queue
+ *	enable. If enabled, the ti_sci_rm_udmap_rx_flow_opt_cfg also need to be
+ *	configured and sent.
+ * @rx_fdq0_sz0_qnum: UDMAP receive flow free descriptor queue 0.
+ * @rx_fdq1_qnum: UDMAP receive flow free descriptor queue 1.
+ * @rx_fdq2_qnum: UDMAP receive flow free descriptor queue 2.
+ * @rx_fdq3_qnum: UDMAP receive flow free descriptor queue 3.
+ *
+ * For detailed information on the settings, see the UDMAP section of the TRM.
+ */
+struct ti_sci_msg_udmap_rx_flow_cfg {
+	struct ti_sci_msg_hdr hdr;
+	u32 nav_id;
+	u32 flow_index;
+	u32 rx_ch_index;
+	u8 rx_einfo_present;
+	u8 rx_psinfo_present;
+	u8 rx_error_handling;
+	u8 rx_desc_type;
+	u16 rx_sop_offset;
+	u16 rx_dest_qnum;
+	u8 rx_ps_location;
+	u8 rx_src_tag_hi;
+	u8 rx_src_tag_lo;
+	u8 rx_dest_tag_hi;
+	u8 rx_dest_tag_lo;
+	u8 rx_src_tag_hi_sel;
+	u8 rx_src_tag_lo_sel;
+	u8 rx_dest_tag_hi_sel;
+	u8 rx_dest_tag_lo_sel;
+	u8 rx_size_thresh_en;
+	u16 rx_fdq0_sz0_qnum;
+	u16 rx_fdq1_qnum;
+	u16 rx_fdq2_qnum;
+	u16 rx_fdq3_qnum;
+} __packed;
+
+/**
+ * struct rm_ti_sci_msg_udmap_rx_flow_opt_cfg - parameters for UDMAP receive
+ *						flow optional configuration
+ * @hdr: Generic Header
+ * @nav_id: SoC Navigator Subsystem device ID from which the receive flow is
+ *	allocated
+ * @flow_index: UDMAP receive flow index for optional configuration.
+ * @rx_ch_index: Specifies the index of the receive channel using the flow_index
+ * @rx_size_thresh0: UDMAP receive flow packet size threshold 0.
+ * @rx_size_thresh1: UDMAP receive flow packet size threshold 1.
+ * @rx_size_thresh2: UDMAP receive flow packet size threshold 2.
+ * @rx_fdq0_sz1_qnum: UDMAP receive flow free descriptor queue for size
+ *	threshold 1.
+ * @rx_fdq0_sz2_qnum: UDMAP receive flow free descriptor queue for size
+ *	threshold 2.
+ * @rx_fdq0_sz3_qnum: UDMAP receive flow free descriptor queue for size
+ *	threshold 3.
+ *
+ * For detailed information on the settings, see the UDMAP section of the TRM.
+ */
+struct rm_ti_sci_msg_udmap_rx_flow_opt_cfg {
+	struct ti_sci_msg_hdr hdr;
+	u32 nav_id;
+	u32 flow_index;
+	u32 rx_ch_index;
+	u16 rx_size_thresh0;
+	u16 rx_size_thresh1;
+	u16 rx_size_thresh2;
+	u16 rx_fdq0_sz1_qnum;
+	u16 rx_fdq0_sz2_qnum;
+	u16 rx_fdq0_sz3_qnum;
+} __attribute__((__packed__));
+
+/**
+ * Configures a Navigator Subsystem UDMAP transmit channel
+ *
+ * Configures the non-real-time registers of a Navigator Subsystem UDMAP
+ * transmit channel.  The channel index must be assigned to the host defined
+ * in the TISCI header via the RM board configuration resource assignment
+ * range list.
+ *
+ * @hdr: Generic Header
+ *
+ * @valid_params: Bitfield defining validity of tx channel configuration
+ * parameters. The tx channel configuration fields are not valid, and will not
+ * be used for ch configuration, if their corresponding valid bit is zero.
+ * Valid bit usage:
+ *    0 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::tx_pause_on_err
+ *    1 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::tx_atype
+ *    2 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::tx_chan_type
+ *    3 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::tx_fetch_size
+ *    4 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::txcq_qnum
+ *    5 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::tx_priority
+ *    6 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::tx_qos
+ *    7 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::tx_orderid
+ *    8 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::tx_sched_priority
+ *    9 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::tx_filt_einfo
+ *   10 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::tx_filt_pswords
+ *   11 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::tx_supr_tdpkt
+ *   12 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::tx_credit_count
+ *   13 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::fdepth
+ *
+ * @nav_id: SoC device ID of Navigator Subsystem where tx channel is located
+ *
+ * @index: UDMAP transmit channel index.
+ *
+ * @tx_pause_on_err: UDMAP transmit channel pause on error configuration to
+ * be programmed into the tx_pause_on_err field of the channel's TCHAN_TCFG
+ * register.
+ *
+ * @tx_filt_einfo: UDMAP transmit channel extended packet information passing
+ * configuration to be programmed into the tx_filt_einfo field of the
+ * channel's TCHAN_TCFG register.
+ *
+ * @tx_filt_pswords: UDMAP transmit channel protocol specific word passing
+ * configuration to be programmed into the tx_filt_pswords field of the
+ * channel's TCHAN_TCFG register.
+ *
+ * @tx_atype: UDMAP transmit channel non Ring Accelerator access pointer
+ * interpretation configuration to be programmed into the tx_atype field of
+ * the channel's TCHAN_TCFG register.
+ *
+ * @tx_chan_type: UDMAP transmit channel functional channel type and work
+ * passing mechanism configuration to be programmed into the tx_chan_type
+ * field of the channel's TCHAN_TCFG register.
+ *
+ * @tx_supr_tdpkt: UDMAP transmit channel teardown packet generation suppression
+ * configuration to be programmed into the tx_supr_tdpkt field of the channel's
+ * TCHAN_TCFG register.
+ *
+ * @tx_fetch_size: UDMAP transmit channel number of 32-bit descriptor words to
+ * fetch configuration to be programmed into the tx_fetch_size field of the
+ * channel's TCHAN_TCFG register.  The user must make sure to set the maximum
+ * word count that can pass through the channel for any allowed descriptor type.
+ *
+ * @tx_credit_count: UDMAP transmit channel transfer request credit count
+ * configuration to be programmed into the count field of the TCHAN_TCREDIT
+ * register.  Specifies how many credits for complete TRs are available.
+ *
+ * @txcq_qnum: UDMAP transmit channel completion queue configuration to be
+ * programmed into the txcq_qnum field of the TCHAN_TCQ register. The specified
+ * completion queue must be assigned to the host, or a subordinate of the host,
+ * requesting configuration of the transmit channel.
+ *
+ * @tx_priority: UDMAP transmit channel transmit priority value to be programmed
+ * into the priority field of the channel's TCHAN_TPRI_CTRL register.
+ *
+ * @tx_qos: UDMAP transmit channel transmit qos value to be programmed into the
+ * qos field of the channel's TCHAN_TPRI_CTRL register.
+ *
+ * @tx_orderid: UDMAP transmit channel bus order id value to be programmed into
+ * the orderid field of the channel's TCHAN_TPRI_CTRL register.
+ *
+ * @fdepth: UDMAP transmit channel FIFO depth configuration to be programmed
+ * into the fdepth field of the TCHAN_TFIFO_DEPTH register. Sets the number of
+ * Tx FIFO bytes which are allowed to be stored for the channel. Check the UDMAP
+ * section of the TRM for restrictions regarding this parameter.
+ *
+ * @tx_sched_priority: UDMAP transmit channel tx scheduling priority
+ * configuration to be programmed into the priority field of the channel's
+ * TCHAN_TST_SCHED register.
+ */
+struct ti_sci_msg_rm_udmap_tx_ch_cfg_req {
+	struct ti_sci_msg_hdr hdr;
+	u32 valid_params;
+	u16 nav_id;
+	u16 index;
+	u8 tx_pause_on_err;
+	u8 tx_filt_einfo;
+	u8 tx_filt_pswords;
+	u8 tx_atype;
+	u8 tx_chan_type;
+	u8 tx_supr_tdpkt;
+	u16 tx_fetch_size;
+	u8 tx_credit_count;
+	u16 txcq_qnum;
+	u8 tx_priority;
+	u8 tx_qos;
+	u8 tx_orderid;
+	u16 fdepth;
+	u8 tx_sched_priority;
+	u8 tx_burst_size;
+} __packed;
+
+/**
+ *  Response to configuring a UDMAP transmit channel.
+ *
+ * @hdr: Standard TISCI header
+ */
+struct ti_sci_msg_rm_udmap_tx_ch_cfg_resp {
+	struct ti_sci_msg_hdr hdr;
+} __packed;
+
+/**
+ * Configures a Navigator Subsystem UDMAP receive channel
+ *
+ * Configures the non-real-time registers of a Navigator Subsystem UDMAP
+ * receive channel.  The channel index must be assigned to the host defined
+ * in the TISCI header via the RM board configuration resource assignment
+ * range list.
+ *
+ * @hdr: Generic Header
+ *
+ * @valid_params: Bitfield defining validity of rx channel configuration
+ * parameters.
+ * The rx channel configuration fields are not valid, and will not be used for
+ * ch configuration, if their corresponding valid bit is zero.
+ * Valid bit usage:
+ *    0 - Valid bit for @ti_sci_msg_rm_udmap_rx_ch_cfg_req::rx_pause_on_err
+ *    1 - Valid bit for @ti_sci_msg_rm_udmap_rx_ch_cfg_req::rx_atype
+ *    2 - Valid bit for @ti_sci_msg_rm_udmap_rx_ch_cfg_req::rx_chan_type
+ *    3 - Valid bit for @ti_sci_msg_rm_udmap_rx_ch_cfg_req::rx_fetch_size
+ *    4 - Valid bit for @ti_sci_msg_rm_udmap_rx_ch_cfg_req::rxcq_qnum
+ *    5 - Valid bit for @ti_sci_msg_rm_udmap_rx_ch_cfg_req::rx_priority
+ *    6 - Valid bit for @ti_sci_msg_rm_udmap_rx_ch_cfg_req::rx_qos
+ *    7 - Valid bit for @ti_sci_msg_rm_udmap_rx_ch_cfg_req::rx_orderid
+ *    8 - Valid bit for @ti_sci_msg_rm_udmap_rx_ch_cfg_req::rx_sched_priority
+ *    9 - Valid bit for @ti_sci_msg_rm_udmap_rx_ch_cfg_req::flowid_start
+ *   10 - Valid bit for @ti_sci_msg_rm_udmap_rx_ch_cfg_req::flowid_cnt
+ *   11 - Valid bit for @ti_sci_msg_rm_udmap_rx_ch_cfg_req::rx_ignore_short
+ *   12 - Valid bit for @ti_sci_msg_rm_udmap_rx_ch_cfg_req::rx_ignore_long
+ *
+ * @nav_id: SoC device ID of Navigator Subsystem where rx channel is located
+ *
+ * @index: UDMAP receive channel index.
+ *
+ * @rx_fetch_size: UDMAP receive channel number of 32-bit descriptor words to
+ * fetch configuration to be programmed into the rx_fetch_size field of the
+ * channel's RCHAN_RCFG register.
+ *
+ * @rxcq_qnum: UDMAP receive channel completion queue configuration to be
+ * programmed into the rxcq_qnum field of the RCHAN_RCQ register.
+ * The specified completion queue must be assigned to the host, or a subordinate
+ * of the host, requesting configuration of the receive channel.
+ *
+ * @rx_priority: UDMAP receive channel receive priority value to be programmed
+ * into the priority field of the channel's RCHAN_RPRI_CTRL register.
+ *
+ * @rx_qos: UDMAP receive channel receive qos value to be programmed into the
+ * qos field of the channel's RCHAN_RPRI_CTRL register.
+ *
+ * @rx_orderid: UDMAP receive channel bus order id value to be programmed into
+ * the orderid field of the channel's RCHAN_RPRI_CTRL register.
+ *
+ * @rx_sched_priority: UDMAP receive channel rx scheduling priority
+ * configuration to be programmed into the priority field of the channel's
+ * RCHAN_RST_SCHED register.
+ *
+ * @flowid_start: UDMAP receive channel additional flows starting index
+ * configuration to program into the flow_start field of the RCHAN_RFLOW_RNG
+ * register. Specifies the starting index for flow IDs the receive channel is to
+ * make use of beyond the default flow. flowid_start and @ref flowid_cnt must be
+ * set as valid and configured together. The starting flow ID set by
+ * @ref flowid_cnt must be a flow index within the Navigator Subsystem's subset
+ * of flows beyond the default flows statically mapped to receive channels.
+ * The additional flows must be assigned to the host, or a subordinate of the
+ * host, requesting configuration of the receive channel.
+ *
+ * @flowid_cnt: UDMAP receive channel additional flows count configuration to
+ * program into the flowid_cnt field of the RCHAN_RFLOW_RNG register.
+ * This field specifies how many flow IDs are in the additional contiguous range
+ * of legal flow IDs for the channel.  @ref flowid_start and flowid_cnt must be
+ * set as valid and configured together. Disabling the valid_params field bit
+ * for flowid_cnt indicates no flow IDs other than the default are to be
+ * allocated and used by the receive channel. @ref flowid_start plus flowid_cnt
+ * cannot be greater than the number of receive flows in the receive channel's
+ * Navigator Subsystem.  The additional flows must be assigned to the host, or a
+ * subordinate of the host, requesting configuration of the receive channel.
+ *
+ * @rx_pause_on_err: UDMAP receive channel pause on error configuration to be
+ * programmed into the rx_pause_on_err field of the channel's RCHAN_RCFG
+ * register.
+ *
+ * @rx_atype: UDMAP receive channel non Ring Accelerator access pointer
+ * interpretation configuration to be programmed into the rx_atype field of the
+ * channel's RCHAN_RCFG register.
+ *
+ * @rx_chan_type: UDMAP receive channel functional channel type and work passing
+ * mechanism configuration to be programmed into the rx_chan_type field of the
+ * channel's RCHAN_RCFG register.
+ *
+ * @rx_ignore_short: UDMAP receive channel short packet treatment configuration
+ * to be programmed into the rx_ignore_short field of the RCHAN_RCFG register.
+ *
+ * @rx_ignore_long: UDMAP receive channel long packet treatment configuration to
+ * be programmed into the rx_ignore_long field of the RCHAN_RCFG register.
+ */
+struct ti_sci_msg_rm_udmap_rx_ch_cfg_req {
+	struct ti_sci_msg_hdr hdr;
+	u32 valid_params;
+	u16 nav_id;
+	u16 index;
+	u16 rx_fetch_size;
+	u16 rxcq_qnum;
+	u8 rx_priority;
+	u8 rx_qos;
+	u8 rx_orderid;
+	u8 rx_sched_priority;
+	u16 flowid_start;
+	u16 flowid_cnt;
+	u8 rx_pause_on_err;
+	u8 rx_atype;
+	u8 rx_chan_type;
+	u8 rx_ignore_short;
+	u8 rx_ignore_long;
+	u8 rx_burst_size;
+
+} __packed;
+
+/**
+ * Response to configuring a UDMAP receive channel.
+ *
+ * @hdr: Standard TISCI header
+ */
+struct ti_sci_msg_rm_udmap_rx_ch_cfg_resp {
+	struct ti_sci_msg_hdr hdr;
+} __packed;
+
+/**
+ * Configures a Navigator Subsystem UDMAP receive flow
+ *
+ * Configures a Navigator Subsystem UDMAP receive flow's registers.
+ * Configuration does not include the flow registers which handle size-based
+ * free descriptor queue routing.
+ *
+ * The flow index must be assigned to the host defined in the TISCI header via
+ * the RM board configuration resource assignment range list.
+ *
+ * @hdr: Standard TISCI header
+ *
+ * @valid_params
+ * Bitfield defining validity of rx flow configuration parameters.  The
+ * rx flow configuration fields are not valid, and will not be used for flow
+ * configuration, if their corresponding valid bit is zero.  Valid bit usage:
+ *     0 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_einfo_present
+ *     1 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_psinfo_present
+ *     2 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_error_handling
+ *     3 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_desc_type
+ *     4 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_sop_offset
+ *     5 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_dest_qnum
+ *     6 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_src_tag_hi
+ *     7 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_src_tag_lo
+ *     8 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_dest_tag_hi
+ *     9 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_dest_tag_lo
+ *    10 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_src_tag_hi_sel
+ *    11 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_src_tag_lo_sel
+ *    12 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_dest_tag_hi_sel
+ *    13 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_dest_tag_lo_sel
+ *    14 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_fdq0_sz0_qnum
+ *    15 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_fdq1_sz0_qnum
+ *    16 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_fdq2_sz0_qnum
+ *    17 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_fdq3_sz0_qnum
+ *    18 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_ps_location
+ *
+ * @nav_id: SoC device ID of Navigator Subsystem from which the receive flow is
+ * allocated
+ *
+ * @flow_index: UDMAP receive flow index for non-optional configuration.
+ *
+ * @rx_einfo_present:
+ * UDMAP receive flow extended packet info present configuration to be
+ * programmed into the rx_einfo_present field of the flow's RFLOW_RFA register.
+ *
+ * @rx_psinfo_present:
+ * UDMAP receive flow PS words present configuration to be programmed into the
+ * rx_psinfo_present field of the flow's RFLOW_RFA register.
+ *
+ * @rx_error_handling:
+ * UDMAP receive flow error handling configuration to be programmed into the
+ * rx_error_handling field of the flow's RFLOW_RFA register.
+ *
+ * @rx_desc_type:
+ * UDMAP receive flow descriptor type configuration to be programmed into the
+ * rx_desc_type field field of the flow's RFLOW_RFA register.
+ *
+ * @rx_sop_offset:
+ * UDMAP receive flow start of packet offset configuration to be programmed
+ * into the rx_sop_offset field of the RFLOW_RFA register.  See the UDMAP
+ * section of the TRM for more information on this setting.  Valid values for
+ * this field are 0-255 bytes.
+ *
+ * @rx_dest_qnum:
+ * UDMAP receive flow destination queue configuration to be programmed into the
+ * rx_dest_qnum field of the flow's RFLOW_RFA register.  The specified
+ * destination queue must be valid within the Navigator Subsystem and must be
+ * owned by the host, or a subordinate of the host, requesting allocation and
+ * configuration of the receive flow.
+ *
+ * @rx_src_tag_hi:
+ * UDMAP receive flow source tag high byte constant configuration to be
+ * programmed into the rx_src_tag_hi field of the flow's RFLOW_RFB register.
+ * See the UDMAP section of the TRM for more information on this setting.
+ *
+ * @rx_src_tag_lo:
+ * UDMAP receive flow source tag low byte constant configuration to be
+ * programmed into the rx_src_tag_lo field of the flow's RFLOW_RFB register.
+ * See the UDMAP section of the TRM for more information on this setting.
+ *
+ * @rx_dest_tag_hi:
+ * UDMAP receive flow destination tag high byte constant configuration to be
+ * programmed into the rx_dest_tag_hi field of the flow's RFLOW_RFB register.
+ * See the UDMAP section of the TRM for more information on this setting.
+ *
+ * @rx_dest_tag_lo:
+ * UDMAP receive flow destination tag low byte constant configuration to be
+ * programmed into the rx_dest_tag_lo field of the flow's RFLOW_RFB register.
+ * See the UDMAP section of the TRM for more information on this setting.
+ *
+ * @rx_src_tag_hi_sel:
+ * UDMAP receive flow source tag high byte selector configuration to be
+ * programmed into the rx_src_tag_hi_sel field of the RFLOW_RFC register.  See
+ * the UDMAP section of the TRM for more information on this setting.
+ *
+ * @rx_src_tag_lo_sel:
+ * UDMAP receive flow source tag low byte selector configuration to be
+ * programmed into the rx_src_tag_lo_sel field of the RFLOW_RFC register.  See
+ * the UDMAP section of the TRM for more information on this setting.
+ *
+ * @rx_dest_tag_hi_sel:
+ * UDMAP receive flow destination tag high byte selector configuration to be
+ * programmed into the rx_dest_tag_hi_sel field of the RFLOW_RFC register.  See
+ * the UDMAP section of the TRM for more information on this setting.
+ *
+ * @rx_dest_tag_lo_sel:
+ * UDMAP receive flow destination tag low byte selector configuration to be
+ * programmed into the rx_dest_tag_lo_sel field of the RFLOW_RFC register.  See
+ * the UDMAP section of the TRM for more information on this setting.
+ *
+ * @rx_fdq0_sz0_qnum:
+ * UDMAP receive flow free descriptor queue 0 configuration to be programmed
+ * into the rx_fdq0_sz0_qnum field of the flow's RFLOW_RFD register.  See the
+ * UDMAP section of the TRM for more information on this setting. The specified
+ * free queue must be valid within the Navigator Subsystem and must be owned
+ * by the host, or a subordinate of the host, requesting allocation and
+ * configuration of the receive flow.
+ *
+ * @rx_fdq1_qnum:
+ * UDMAP receive flow free descriptor queue 1 configuration to be programmed
+ * into the rx_fdq1_qnum field of the flow's RFLOW_RFD register.  See the
+ * UDMAP section of the TRM for more information on this setting.  The specified
+ * free queue must be valid within the Navigator Subsystem and must be owned
+ * by the host, or a subordinate of the host, requesting allocation and
+ * configuration of the receive flow.
+ *
+ * @rx_fdq2_qnum:
+ * UDMAP receive flow free descriptor queue 2 configuration to be programmed
+ * into the rx_fdq2_qnum field of the flow's RFLOW_RFE register.  See the
+ * UDMAP section of the TRM for more information on this setting.  The specified
+ * free queue must be valid within the Navigator Subsystem and must be owned
+ * by the host, or a subordinate of the host, requesting allocation and
+ * configuration of the receive flow.
+ *
+ * @rx_fdq3_qnum:
+ * UDMAP receive flow free descriptor queue 3 configuration to be programmed
+ * into the rx_fdq3_qnum field of the flow's RFLOW_RFE register.  See the
+ * UDMAP section of the TRM for more information on this setting.  The specified
+ * free queue must be valid within the Navigator Subsystem and must be owned
+ * by the host, or a subordinate of the host, requesting allocation and
+ * configuration of the receive flow.
+ *
+ * @rx_ps_location:
+ * UDMAP receive flow PS words location configuration to be programmed into the
+ * rx_ps_location field of the flow's RFLOW_RFA register.
+ */
+struct ti_sci_msg_rm_udmap_flow_cfg_req {
+	struct ti_sci_msg_hdr hdr;
+	u32 valid_params;
+	u16 nav_id;
+	u16 flow_index;
+	u8 rx_einfo_present;
+	u8 rx_psinfo_present;
+	u8 rx_error_handling;
+	u8 rx_desc_type;
+	u16 rx_sop_offset;
+	u16 rx_dest_qnum;
+	u8 rx_src_tag_hi;
+	u8 rx_src_tag_lo;
+	u8 rx_dest_tag_hi;
+	u8 rx_dest_tag_lo;
+	u8 rx_src_tag_hi_sel;
+	u8 rx_src_tag_lo_sel;
+	u8 rx_dest_tag_hi_sel;
+	u8 rx_dest_tag_lo_sel;
+	u16 rx_fdq0_sz0_qnum;
+	u16 rx_fdq1_qnum;
+	u16 rx_fdq2_qnum;
+	u16 rx_fdq3_qnum;
+	u8 rx_ps_location;
+} __packed;
+
+/**
+ *  Response to configuring a Navigator Subsystem UDMAP receive flow
+ *
+ * @hdr: Standard TISCI header
+ */
+struct ti_sci_msg_rm_udmap_flow_cfg_resp {
+	struct ti_sci_msg_hdr hdr;
+} __packed;
+
 #endif /* __TI_SCI_H */
diff -Naur --no-dereference a/drivers/gpio/gpio-davinci.c b/drivers/gpio/gpio-davinci.c
--- a/drivers/gpio/gpio-davinci.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpio/gpio-davinci.c	2020-06-19 21:00:31.408227715 -0400
@@ -292,7 +292,7 @@
 static void gpio_irq_disable(struct irq_data *d)
 {
 	struct davinci_gpio_regs __iomem *g = irq2regs(d);
-	u32 mask = (u32) irq_data_get_irq_handler_data(d);
+	uintptr_t mask = (uintptr_t)irq_data_get_irq_handler_data(d);
 
 	writel_relaxed(mask, &g->clr_falling);
 	writel_relaxed(mask, &g->clr_rising);
@@ -301,7 +301,7 @@
 static void gpio_irq_enable(struct irq_data *d)
 {
 	struct davinci_gpio_regs __iomem *g = irq2regs(d);
-	u32 mask = (u32) irq_data_get_irq_handler_data(d);
+	uintptr_t mask = (uintptr_t)irq_data_get_irq_handler_data(d);
 	unsigned status = irqd_get_trigger_type(d);
 
 	status &= IRQ_TYPE_EDGE_FALLING | IRQ_TYPE_EDGE_RISING;
@@ -442,7 +442,7 @@
 				"davinci_gpio");
 	irq_set_irq_type(irq, IRQ_TYPE_NONE);
 	irq_set_chip_data(irq, (__force void *)g);
-	irq_set_handler_data(irq, (void *)__gpio_mask(hw));
+	irq_set_handler_data(irq, (void *)(uintptr_t)__gpio_mask(hw));
 
 	return 0;
 }
diff -Naur --no-dereference a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c
--- a/drivers/gpio/gpiolib.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpio/gpiolib.c	2020-06-19 21:00:31.408227715 -0400
@@ -2312,10 +2312,10 @@
 	if (test_and_set_bit(FLAG_REQUESTED, &desc->flags) == 0) {
 		desc_set_label(desc, label ? : "?");
 		status = 0;
-	} else {
-		kfree_const(label);
-		status = -EBUSY;
-		goto done;
+//	} else {
+//		kfree_const(label);
+//		status = -EBUSY;
+//		goto done;
 	}
 
 	if (chip->request) {
diff -Naur --no-dereference a/drivers/gpio/gpiolib-sysfs.c b/drivers/gpio/gpiolib-sysfs.c
--- a/drivers/gpio/gpiolib-sysfs.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpio/gpiolib-sysfs.c	2020-06-19 21:00:31.408227715 -0400
@@ -37,10 +37,10 @@
 /*
  * /sys/class/gpio/gpioN... only for GPIOs that are exported
  *   /direction
- *      * MAY BE OMITTED if kernel won't allow direction changes
  *      * is read/write as "in" or "out"
  *      * may also be written as "high" or "low", initializing
  *        output value as specified ("out" implies "low")
+ *      * read-only if kernel won't allow direction changes
  *   /value
  *      * always readable, subject to hardware behavior
  *      * may be writable, as zero/nonzero
@@ -53,6 +53,8 @@
  *      * is read/write as zero/nonzero
  *      * also affects existing and subsequent "falling" and "rising"
  *        /edge configuration
+ *   /label
+ *      * descriptor label
  */
 
 static ssize_t direction_show(struct device *dev,
@@ -83,7 +85,9 @@
 
 	mutex_lock(&data->mutex);
 
-	if (sysfs_streq(buf, "high"))
+	if (!data->direction_can_change)
+		status = -EPERM;
+	else if (sysfs_streq(buf, "high"))
 		status = gpiod_direction_output_raw(desc, 1);
 	else if (sysfs_streq(buf, "out") || sysfs_streq(buf, "low"))
 		status = gpiod_direction_output_raw(desc, 0);
@@ -362,6 +366,23 @@
 }
 static DEVICE_ATTR_RW(active_low);
 
+static ssize_t label_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	struct gpiod_data *data = dev_get_drvdata(dev);
+	struct gpio_desc *desc = data->desc;
+	ssize_t			status;
+
+	mutex_lock(&data->mutex);
+
+	status = sprintf(buf, "%s\n", desc->label);
+
+	mutex_unlock(&data->mutex);
+
+	return status;
+}
+static DEVICE_ATTR_RO(label);
+
 static umode_t gpio_is_visible(struct kobject *kobj, struct attribute *attr,
 			       int n)
 {
@@ -373,12 +394,15 @@
 
 	if (attr == &dev_attr_direction.attr) {
 		if (!show_direction)
-			mode = 0;
+			mode &= 0444;
 	} else if (attr == &dev_attr_edge.attr) {
 		if (gpiod_to_irq(desc) < 0)
 			mode = 0;
 		if (!show_direction && test_bit(FLAG_IS_OUT, &desc->flags))
 			mode = 0;
+	} else if (attr == &dev_attr_value.attr) {
+		if (!show_direction && !test_bit(FLAG_IS_OUT, &desc->flags))
+			mode &= 0444;
 	}
 
 	return mode;
@@ -389,6 +413,7 @@
 	&dev_attr_edge.attr,
 	&dev_attr_value.attr,
 	&dev_attr_active_low.attr,
+	&dev_attr_label.attr,
 	NULL,
 };
 
@@ -402,6 +427,10 @@
 	NULL
 };
 
+/* bwlegh, a second device in the same file... get out of my namespace! */
+#define dev_attr_label dev_attr_chip_label
+#define label_show chip_label_show
+
 /*
  * /sys/class/gpio/gpiochipN/
  *   /base ... matching gpio_chip.base (N)
diff -Naur --no-dereference a/drivers/gpio/gpio-of-helper.c b/drivers/gpio/gpio-of-helper.c
--- a/drivers/gpio/gpio-of-helper.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpio/gpio-of-helper.c	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,435 @@
+/*
+ * GPIO OF based helper
+ *
+ * A simple DT based driver to provide access to GPIO functionality
+ * to user-space via sysfs.
+ *
+ * Copyright (C) 2013 Pantelis Antoniou <panto@antoniou-consulting.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/string.h>
+#include <linux/timer.h>
+#include <linux/errno.h>
+#include <linux/init.h>
+#include <linux/delay.h>
+#include <linux/bitops.h>
+#include <linux/err.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/of_gpio.h>
+#include <linux/pinctrl/pinctrl.h>
+#include <linux/pinctrl/pinmux.h>
+#include <linux/pinctrl/consumer.h>
+#include <linux/atomic.h>
+#include <linux/clk.h>
+#include <linux/interrupt.h>
+#include <linux/math64.h>
+#include <linux/atomic.h>
+#include <linux/idr.h>
+
+/* fwd decl. */
+struct gpio_of_helper_info;
+
+enum gpio_type {
+	GPIO_TYPE_INPUT = 0,
+	GPIO_TYPE_OUTPUT = 1,
+};
+
+struct gpio_of_entry {
+	int id;
+	struct gpio_of_helper_info *info;
+	struct device_node *node;
+	enum gpio_type type;
+	int gpio;
+	int irq;
+	const char *name;
+	atomic64_t counter;
+	unsigned int count_flags;
+#define COUNT_RISING_EDGE	(1 << 0)
+#define COUNT_FALLING_EDGE	(1 << 1)
+};
+
+struct gpio_of_helper_info {
+	struct platform_device *pdev;
+	struct idr idr;
+};
+
+static const struct of_device_id gpio_of_helper_of_match[] = {
+	{
+		.compatible = "gpio-of-helper",
+	},
+	{ },
+};
+MODULE_DEVICE_TABLE(of, gpio_of_helper_of_match);
+
+static ssize_t gpio_of_helper_show_status(struct device *dev,
+				struct device_attribute *attr, char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct gpio_of_helper_info *info = platform_get_drvdata(pdev);
+	struct gpio_of_entry *entry;
+	char *p, *e;
+	int id, n;
+
+	p = buf;
+	e = p + PAGE_SIZE;
+	n = 0;
+	idr_for_each_entry(&info->idr, entry, id) {
+		switch (entry->type) {
+		case GPIO_TYPE_INPUT:
+			n = snprintf(p, e - p, "%2d %-24s %3d %-3s %llu\n",
+				entry->id, entry->name, entry->gpio, "IN",
+				(unsigned long long)
+					atomic64_read(&entry->counter));
+			break;
+		case GPIO_TYPE_OUTPUT:
+			n = snprintf(p, e - p, "%2d %-24s %3d %-3s\n",
+				entry->id, entry->name, entry->gpio, "OUT");
+			break;
+		}
+		p += n;
+	}
+
+	return p - buf;
+}
+
+static DEVICE_ATTR(status, S_IRUGO,
+		gpio_of_helper_show_status, NULL);
+
+static irqreturn_t gpio_of_helper_handler(int irq, void *ptr)
+{
+	struct gpio_of_entry *entry = ptr;
+
+	/* caution - low speed interfaces only! */
+	atomic64_inc(&entry->counter);
+
+	return IRQ_HANDLED;
+}
+
+static struct gpio_of_entry *
+gpio_of_entry_create(struct gpio_of_helper_info *info,
+		struct device_node *node)
+{
+	struct platform_device *pdev = info->pdev;
+	struct device *dev = &pdev->dev;
+	struct gpio_of_entry *entry;
+	int err, gpio, irq;
+	unsigned int req_flags, count_flags, irq_flags;
+	enum gpio_type type;
+	enum of_gpio_flags gpio_flags;
+	const char *name;
+
+	/* get the type of the node first */
+	if (of_property_read_bool(node, "input"))
+		type = GPIO_TYPE_INPUT;
+	else if (of_property_read_bool(node, "output")
+			|| of_property_read_bool(node, "init-low")
+			|| of_property_read_bool(node, "init-high"))
+		type = GPIO_TYPE_OUTPUT;
+	else {
+		dev_err(dev, "Not valid gpio node type\n");
+		err = -EINVAL;
+		goto err_bad_node;
+	}
+
+	/* get the name */
+	if (of_property_read_string(node, "line-name", &name))
+		if (of_property_read_string(node, "gpio-name", &name))
+			name = node->name;
+
+	err = of_get_named_gpio_flags(node, "gpio", 0, &gpio_flags);
+	if (IS_ERR_VALUE(err)) {
+		dev_err(dev, "Failed to get gpio property of '%s'\n", name);
+		goto err_bad_node;
+	}
+	gpio = err;
+
+	req_flags = 0;
+	count_flags = 0;
+
+	/* set the request flags */
+	switch (type) {
+		case GPIO_TYPE_INPUT:
+			req_flags = GPIOF_DIR_IN | GPIOF_EXPORT;
+			if (of_property_read_bool(node, "count-falling-edge"))
+				count_flags |= COUNT_FALLING_EDGE;
+			if (of_property_read_bool(node, "count-rising-edge"))
+				count_flags |= COUNT_RISING_EDGE;
+			break;
+		case GPIO_TYPE_OUTPUT:
+			req_flags = GPIOF_DIR_OUT | GPIOF_EXPORT;
+			if (of_property_read_bool(node, "init-high"))
+				req_flags |= GPIOF_OUT_INIT_HIGH;
+			else if (of_property_read_bool(node, "init-low"))
+				req_flags |= GPIOF_OUT_INIT_LOW;
+			break;
+	}
+	if (of_property_read_bool(node, "dir-changeable"))
+		req_flags |= GPIOF_EXPORT_CHANGEABLE;
+	if (gpio_flags & OF_GPIO_ACTIVE_LOW)
+		req_flags |= GPIOF_ACTIVE_LOW;
+	if (gpio_flags & OF_GPIO_SINGLE_ENDED) {
+		if (gpio_flags & OF_GPIO_ACTIVE_LOW)
+			req_flags |= GPIOF_OPEN_DRAIN;
+		else
+			req_flags |= GPIOF_OPEN_SOURCE;
+	}
+
+	/* request the gpio */
+	err = devm_gpio_request_one(dev, gpio, req_flags, name);
+	if (err != 0) {
+		dev_err(dev, "Failed to request gpio '%s'\n", name);
+		goto err_bad_node;
+	}
+
+	irq = -1;
+	irq_flags = 0;
+
+	/* counter mode requested - need an interrupt */
+	if (count_flags != 0) {
+		irq = gpio_to_irq(gpio);
+		if (IS_ERR_VALUE(irq)) {
+			dev_err(dev, "Failed to request gpio '%s'\n", name);
+			goto err_bad_node;
+		}
+
+		if (count_flags & COUNT_RISING_EDGE)
+			irq_flags |= IRQF_TRIGGER_RISING;
+		if (count_flags & COUNT_FALLING_EDGE)
+			irq_flags |= IRQF_TRIGGER_FALLING;
+	}
+
+//	if (!idr_pre_get(&info->idr, GFP_KERNEL)) {
+//		dev_err(dev, "Failed on idr_pre_get of '%s'\n", name);
+//		err = -ENOMEM;
+//		goto err_no_mem;
+//	}
+
+	idr_preload(GFP_KERNEL);
+
+	entry = devm_kzalloc(dev, sizeof(*entry), GFP_KERNEL);
+	if (entry == NULL) {
+		dev_err(dev, "Failed to allocate gpio entry of '%s'\n", name);
+		err = -ENOMEM;
+		goto err_no_mem;
+	}
+
+	entry->id = -1;
+	entry->info = info;
+	entry->node = of_node_get(node);	/* get node reference */
+	entry->type = type;
+	entry->gpio = gpio;
+	entry->irq = irq;
+	entry->name = name;
+
+	/* interrupt enable is last thing done */
+	if (irq >= 0) {
+		atomic64_set(&entry->counter, 0);
+		entry->count_flags = count_flags;
+		err = devm_request_irq(dev, irq, gpio_of_helper_handler,
+				irq_flags, name, entry);
+		if (err != 0) {
+			dev_err(dev, "Failed to request irq of '%s'\n", name);
+			goto err_no_irq;
+		}
+	}
+
+	/* all done; insert */
+//	err = idr_get_new(&info->idr, entry, &entry->id);
+//	if (IS_ERR_VALUE(err)) {
+//		dev_err(dev, "Failed to idr_get_new  of '%s'\n", name);
+//		goto err_fail_idr;
+//	}
+
+	err = idr_alloc(&info->idr, entry, 0, 0, GFP_NOWAIT);
+	if (err >= 0)
+		entry->id = err;
+
+	idr_preload_end();
+
+	if (err < 0) {
+		dev_err(dev, "Failed to idr_get_new  of '%s'\n", name);
+		goto err_fail_idr;
+	}
+
+	dev_dbg(dev, "Allocated GPIO id=%d name='%s'\n", entry->id, name);
+
+	return entry;
+
+err_fail_idr:
+	/* nothing to do */
+err_no_irq:
+	/* release node ref */
+	of_node_put(node);
+	/* nothing else needs to be done, devres handles it */
+err_no_mem:
+err_bad_node:
+	return ERR_PTR(err);
+}
+
+static int gpio_of_entry_destroy(struct gpio_of_entry *entry)
+{
+	struct gpio_of_helper_info *info = entry->info;
+	struct platform_device *pdev = info->pdev;
+	struct device *dev = &pdev->dev;
+
+	dev_dbg(dev, "Destroying GPIO id=%d\n", entry->id);
+
+	/* remove from the IDR */
+	idr_remove(&info->idr, entry->id);
+
+	/* remove node ref */
+	of_node_put(entry->node);
+
+	/* free gpio */
+	devm_gpio_free(dev, entry->gpio);
+
+	/* gree irq */
+	if (entry->irq >= 0)
+		devm_free_irq(dev, entry->irq, entry);
+
+	/* and free */
+	devm_kfree(dev, entry);
+
+	return 0;
+}
+
+static int gpio_of_helper_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct gpio_of_helper_info *info;
+	struct gpio_of_entry *entry;
+	struct device_node *pnode = pdev->dev.of_node;
+	struct device_node *cnode;
+	struct pinctrl *pinctrl;
+	int err;
+
+	/* we only support OF */
+	if (pnode == NULL) {
+		dev_err(&pdev->dev, "No platform of_node!\n");
+		return -ENODEV;
+	}
+
+	pinctrl = devm_pinctrl_get_select_default(&pdev->dev);
+	if (IS_ERR(pinctrl)) {
+		/* special handling for probe defer */
+		if (PTR_ERR(pinctrl) == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
+
+		dev_warn(&pdev->dev,
+			"pins are not configured from the driver\n");
+	}
+
+	info = devm_kzalloc(&pdev->dev, sizeof(*info), GFP_KERNEL);
+	if (info == NULL) {
+		dev_err(&pdev->dev, "Failed to allocate info\n");
+		err = -ENOMEM;
+		goto err_no_mem;
+	}
+	platform_set_drvdata(pdev, info);
+	info->pdev = pdev;
+
+	idr_init(&info->idr);
+
+	err = device_create_file(dev, &dev_attr_status);
+	if (err != 0) {
+		dev_err(dev, "Failed to create status sysfs attribute\n");
+		goto err_no_sysfs;
+	}
+
+	for_each_child_of_node(pnode, cnode) {
+
+		entry = gpio_of_entry_create(info, cnode);
+		if (IS_ERR_OR_NULL(entry)) {
+			dev_err(dev, "Failed to create gpio entry\n");
+			err = PTR_ERR(entry);
+			goto err_fail_entry;
+		}
+	}
+
+	dev_info(&pdev->dev, "ready\n");
+
+	return 0;
+err_fail_entry:
+	device_remove_file(&pdev->dev, &dev_attr_status);
+err_no_sysfs:
+err_no_mem:
+	return err;
+}
+
+static int gpio_of_helper_remove(struct platform_device *pdev)
+{
+	struct gpio_of_helper_info *info = platform_get_drvdata(pdev);
+	struct gpio_of_entry *entry;
+	int id;
+
+	dev_info(&pdev->dev, "removing\n");
+
+	device_remove_file(&pdev->dev, &dev_attr_status);
+
+	id = 0;
+	idr_for_each_entry(&info->idr, entry, id) {
+		/* destroy each and every one */
+		gpio_of_entry_destroy(entry);
+	}
+
+	return 0;
+}
+
+#ifdef CONFIG_PM
+//#ifdef CONFIG_PM_RUNTIME
+static int gpio_of_helper_runtime_suspend(struct device *dev)
+{
+	/* place holder */
+	return 0;
+}
+
+static int gpio_of_helper_runtime_resume(struct device *dev)
+{
+	/* place holder */
+	return 0;
+}
+//#endif /* CONFIG_PM_RUNTIME */
+
+static struct dev_pm_ops gpio_of_helper_pm_ops = {
+	SET_RUNTIME_PM_OPS(gpio_of_helper_runtime_suspend,
+			   gpio_of_helper_runtime_resume, NULL)
+};
+#define GPIO_OF_HELPER_PM_OPS (&gpio_of_helper_pm_ops)
+#else
+#define GPIO_OF_HELPER_PM_OPS NULL
+#endif /* CONFIG_PM */
+
+struct platform_driver gpio_of_helper_driver = {
+	.probe		= gpio_of_helper_probe,
+	.remove		= gpio_of_helper_remove,
+	.driver = {
+		.name		= "gpio-of-helper",
+		.owner		= THIS_MODULE,
+		.pm		= GPIO_OF_HELPER_PM_OPS,
+		.of_match_table	= gpio_of_helper_of_match,
+	},
+};
+
+module_platform_driver(gpio_of_helper_driver);
+
+MODULE_AUTHOR("Pantelis Antoniou <panto@antoniou-consulting.com>");
+MODULE_DESCRIPTION("GPIO OF Helper driver");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:gpio-of-helper");
diff -Naur --no-dereference a/drivers/gpio/gpio-omap.c b/drivers/gpio/gpio-omap.c
--- a/drivers/gpio/gpio-omap.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpio/gpio-omap.c	2020-06-19 21:00:31.408227715 -0400
@@ -1582,6 +1582,8 @@
 			continue;
 
 		pm_runtime_get_sync(bank->chip.parent);
+
+		bank->power_mode = 0;
 	}
 }
 #endif
diff -Naur --no-dereference a/drivers/gpio/gpio-pca953x.c b/drivers/gpio/gpio-pca953x.c
--- a/drivers/gpio/gpio-pca953x.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpio/gpio-pca953x.c	2020-06-19 21:00:31.408227715 -0400
@@ -653,6 +653,7 @@
 			     int irq_base)
 {
 	struct i2c_client *client = chip->client;
+	unsigned long irqflags;
 	int ret, i;
 
 	if (client->irq && irq_base != -1
@@ -671,12 +672,16 @@
 			chip->irq_stat[i] &= chip->reg_direction[i];
 		mutex_init(&chip->irq_lock);
 
+		irqflags = irq_get_trigger_type(client->irq);
+		if (irqflags == IRQF_TRIGGER_NONE)
+			irqflags = IRQF_TRIGGER_LOW;
+		irqflags |= IRQF_ONESHOT | IRQF_SHARED;
+
 		ret = devm_request_threaded_irq(&client->dev,
 					client->irq,
 					   NULL,
 					   pca953x_irq_handler,
-					   IRQF_TRIGGER_LOW | IRQF_ONESHOT |
-						   IRQF_SHARED,
+					   irqflags,
 					   dev_name(&client->dev), chip);
 		if (ret) {
 			dev_err(&client->dev, "failed to request irq %d\n",
diff -Naur --no-dereference a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig
--- a/drivers/gpio/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpio/Kconfig	2020-06-19 21:00:31.408227715 -0400
@@ -73,6 +73,20 @@
 	  Kernel drivers may also request that a particular GPIO be
 	  exported to userspace; this can be useful when debugging.
 
+config GPIO_OF_HELPER
+	bool "GPIO OF helper device (EXPERIMENTAL)"
+	depends on OF_GPIO
+	help
+	  Say Y here to add an GPIO OF helper driver
+
+	  Allows you specify a GPIO helper based on OF
+	  which allows simple export of GPIO functionality
+	  in user-space.
+
+	  Features include, value set/get, direction control,
+	  interrupt/value change poll support, event counting
+	  and others.
+
 config GPIO_GENERIC
 	depends on HAS_IOMEM # Only for IOMEM drivers
 	tristate
@@ -170,7 +184,7 @@
 config GPIO_DAVINCI
 	bool "TI Davinci/Keystone GPIO support"
 	default y if ARCH_DAVINCI
-	depends on ARM && (ARCH_DAVINCI || ARCH_KEYSTONE)
+	depends on (ARM || ARM64) && (ARCH_DAVINCI || ARCH_KEYSTONE || ARCH_K3)
 	help
 	  Say yes here to enable GPIO support for TI Davinci/Keystone SoCs.
 
diff -Naur --no-dereference a/drivers/gpio/Makefile b/drivers/gpio/Makefile
--- a/drivers/gpio/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpio/Makefile	2020-06-19 21:00:31.408227715 -0400
@@ -10,6 +10,7 @@
 obj-$(CONFIG_OF_GPIO)		+= gpiolib-of.o
 obj-$(CONFIG_GPIO_SYSFS)	+= gpiolib-sysfs.o
 obj-$(CONFIG_GPIO_ACPI)		+= gpiolib-acpi.o
+obj-$(CONFIG_GPIO_OF_HELPER)	+= gpio-of-helper.o
 
 # Device drivers. Generally keep list sorted alphabetically
 obj-$(CONFIG_GPIO_GENERIC)	+= gpio-generic.o
diff -Naur --no-dereference a/drivers/gpu/drm/bridge/cdns-mhdp.c b/drivers/gpu/drm/bridge/cdns-mhdp.c
--- a/drivers/gpu/drm/bridge/cdns-mhdp.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/bridge/cdns-mhdp.c	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,2208 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Cadence MHDP DP bridge driver.
+ *
+ * Copyright: 2018 Cadence Design Systems, Inc.
+ *
+ * Author: Quentin Schulz <quentin.schulz@free-electrons.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/firmware.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/phy/phy.h>
+#include <linux/phy/phy-dp.h>
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_bridge.h>
+#include <drm/drm_connector.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_dp_helper.h>
+#include <drm/drm_modeset_helper_vtables.h>
+#include <drm/drm_print.h>
+#include <drm/drm_crtc_helper.h>
+
+#include <linux/irq.h>
+#include <linux/of_irq.h>
+#include <linux/of_device.h>
+
+#include <asm/unaligned.h>
+
+#include "cdns-mhdp.h"
+#include "cdns-mhdp-j721e.h"
+
+/* CDNS MHDP Helpers */
+#define MAILBOX_RETRY_US		1000
+#define MAILBOX_TIMEOUT_US		5000000
+
+/* mailbox */
+#define MB_OPCODE_ID			0
+#define MB_MODULE_ID			1
+#define MB_SIZE_MSB_ID			2
+#define MB_SIZE_LSB_ID			3
+#define MB_DATA_ID			4
+
+#define MB_MODULE_ID_DP_TX		0x01
+#define MB_MODULE_ID_HDCP_TX		0x07
+#define MB_MODULE_ID_HDCP_RX		0x08
+#define MB_MODULE_ID_HDCP_GENERAL	0x09
+#define MB_MODULE_ID_GENERAL		0x0a
+
+/* general opcode */
+#define GENERAL_MAIN_CONTROL		0x01
+#define GENERAL_TEST_ECHO		0x02
+#define GENERAL_BUS_SETTINGS		0x03
+#define GENERAL_TEST_ACCESS		0x04
+#define GENERAL_REGISTER_READ		0x07
+
+#define DPTX_SET_POWER_MNG			0x00
+#define DPTX_SET_HOST_CAPABILITIES		0x01
+#define DPTX_GET_EDID				0x02
+#define DPTX_READ_DPCD				0x03
+#define DPTX_WRITE_DPCD				0x04
+#define DPTX_ENABLE_EVENT			0x05
+#define DPTX_WRITE_REGISTER			0x06
+#define DPTX_READ_REGISTER			0x07
+#define DPTX_WRITE_FIELD			0x08
+#define DPTX_TRAINING_CONTROL			0x09
+#define DPTX_READ_EVENT				0x0a
+#define DPTX_READ_LINK_STAT			0x0b
+#define DPTX_SET_VIDEO				0x0c
+#define DPTX_SET_AUDIO				0x0d
+#define DPTX_GET_LAST_AUX_STAUS			0x0e
+#define DPTX_SET_LINK_BREAK_POINT		0x0f
+#define DPTX_FORCE_LANES			0x10
+#define DPTX_HPD_STATE				0x11
+#define DPTX_ADJUST_LT				0x12
+
+#define FW_STANDBY				0
+#define FW_ACTIVE				1
+
+#define DPTX_READ_EVENT_HPD_TO_HIGH            BIT(0)
+#define DPTX_READ_EVENT_HPD_TO_LOW             BIT(1)
+#define DPTX_READ_EVENT_HPD_PULSE              BIT(2)
+#define DPTX_READ_EVENT_HPD_STATE              BIT(3)
+
+static inline u32 get_unaligned_be24(const void *p)
+{
+	const u8 *_p = p;
+
+	return _p[0] << 16 | _p[1] << 8 | _p[2];
+}
+
+static inline void put_unaligned_be24(u32 val, void *p)
+{
+	u8 *_p = p;
+
+	_p[0] = val >> 16;
+	_p[1] = val >> 8;
+	_p[2] = val;
+}
+
+static int cdns_mhdp_mailbox_read(struct cdns_mhdp_device *mhdp)
+{
+	int val, ret;
+
+	WARN_ON(!mutex_is_locked(&mhdp->mbox_mutex));
+
+	ret = readx_poll_timeout(readl, mhdp->regs + CDNS_MAILBOX_EMPTY,
+				 val, !val, MAILBOX_RETRY_US,
+				 MAILBOX_TIMEOUT_US);
+	if (ret < 0)
+		return ret;
+
+	return readl(mhdp->regs + CDNS_MAILBOX_RX_DATA) & 0xff;
+}
+
+static int cdp_dp_mailbox_write(struct cdns_mhdp_device *mhdp, u8 val)
+{
+	int ret, full;
+
+	WARN_ON(!mutex_is_locked(&mhdp->mbox_mutex));
+
+	ret = readx_poll_timeout(readl, mhdp->regs + CDNS_MAILBOX_FULL,
+				 full, !full, MAILBOX_RETRY_US,
+				 MAILBOX_TIMEOUT_US);
+	if (ret < 0)
+		return ret;
+
+	writel(val, mhdp->regs + CDNS_MAILBOX_TX_DATA);
+
+	return 0;
+}
+
+static int cdns_mhdp_mailbox_validate_receive(struct cdns_mhdp_device *mhdp,
+					      u8 module_id, u8 opcode,
+					      u16 req_size)
+{
+	u32 mbox_size, i;
+	u8 header[4];
+	int ret;
+
+	/* read the header of the message */
+	for (i = 0; i < 4; i++) {
+		ret = cdns_mhdp_mailbox_read(mhdp);
+		if (ret < 0)
+			return ret;
+
+		header[i] = ret;
+	}
+
+	mbox_size = get_unaligned_be16(header + 2);
+
+	if (opcode != header[0] || module_id != header[1] ||
+	    req_size != mbox_size) {
+		/*
+		 * If the message in mailbox is not what we want, we need to
+		 * clear the mailbox by reading its contents.
+		 */
+		for (i = 0; i < mbox_size; i++)
+			if (cdns_mhdp_mailbox_read(mhdp) < 0)
+				break;
+
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int cdns_mhdp_mailbox_read_receive(struct cdns_mhdp_device *mhdp,
+					  u8 *buff, u16 buff_size)
+{
+	u32 i;
+	int ret;
+
+	for (i = 0; i < buff_size; i++) {
+		ret = cdns_mhdp_mailbox_read(mhdp);
+		if (ret < 0)
+			return ret;
+
+		buff[i] = ret;
+	}
+
+	return 0;
+}
+
+static int cdns_mhdp_mailbox_send(struct cdns_mhdp_device *mhdp, u8 module_id,
+				  u8 opcode, u16 size, u8 *message)
+{
+	u8 header[4];
+	int ret, i;
+
+	header[0] = opcode;
+	header[1] = module_id;
+	put_unaligned_be16(size, header + 2);
+
+	for (i = 0; i < 4; i++) {
+		ret = cdp_dp_mailbox_write(mhdp, header[i]);
+		if (ret)
+			return ret;
+	}
+
+	for (i = 0; i < size; i++) {
+		ret = cdp_dp_mailbox_write(mhdp, message[i]);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static
+int cdns_mhdp_reg_read(struct cdns_mhdp_device *mhdp, u32 addr, u32 *value)
+{
+	u8 msg[4], resp[8];
+	int ret;
+
+	if (addr == 0) {
+		ret = -EINVAL;
+		goto err_reg_read;
+	}
+
+	put_unaligned_be32(addr, msg);
+
+	mutex_lock(&mhdp->mbox_mutex);
+
+	ret = cdns_mhdp_mailbox_send(mhdp, MB_MODULE_ID_GENERAL,
+				     GENERAL_REGISTER_READ,
+				     sizeof(msg), msg);
+	if (ret)
+		goto err_reg_read;
+
+	ret = cdns_mhdp_mailbox_validate_receive(mhdp, MB_MODULE_ID_GENERAL,
+						 GENERAL_REGISTER_READ,
+						 sizeof(resp));
+	if (ret)
+		goto err_reg_read;
+
+	ret = cdns_mhdp_mailbox_read_receive(mhdp, resp, sizeof(resp));
+	if (ret)
+		goto err_reg_read;
+
+	/* Returned address value should be the same as requested */
+	if (memcmp(msg, resp, sizeof(msg))) {
+		ret = -EINVAL;
+		goto err_reg_read;
+	}
+
+	*value = get_unaligned_be32(resp + 4);
+
+err_reg_read:
+	mutex_unlock(&mhdp->mbox_mutex);
+	if (ret) {
+		DRM_DEV_ERROR(mhdp->dev, "Failed to read register.\n");
+		*value = 0;
+	}
+
+	return ret;
+}
+
+static
+int cdns_mhdp_reg_write(struct cdns_mhdp_device *mhdp, u16 addr, u32 val)
+{
+	u8 msg[6];
+	int ret;
+
+	put_unaligned_be16(addr, msg);
+	put_unaligned_be32(val, msg + 2);
+
+	mutex_lock(&mhdp->mbox_mutex);
+
+	ret = cdns_mhdp_mailbox_send(mhdp, MB_MODULE_ID_DP_TX,
+				     DPTX_WRITE_REGISTER, sizeof(msg), msg);
+
+	mutex_unlock(&mhdp->mbox_mutex);
+
+	return ret;
+}
+
+static
+int cdns_mhdp_reg_write_bit(struct cdns_mhdp_device *mhdp, u16 addr,
+			    u8 start_bit, u8 bits_no, u32 val)
+{
+	u8 field[8];
+	int ret;
+
+	put_unaligned_be16(addr, field);
+	field[2] = start_bit;
+	field[3] = bits_no;
+	put_unaligned_be32(val, field + 4);
+
+	mutex_lock(&mhdp->mbox_mutex);
+
+	ret = cdns_mhdp_mailbox_send(mhdp, MB_MODULE_ID_DP_TX,
+				     DPTX_WRITE_FIELD, sizeof(field), field);
+
+	mutex_unlock(&mhdp->mbox_mutex);
+
+	return ret;
+}
+
+static
+int cdns_mhdp_dpcd_read(struct cdns_mhdp_device *mhdp,
+			u32 addr, u8 *data, u16 len)
+{
+	u8 msg[5], reg[5];
+	int ret;
+
+	put_unaligned_be16(len, msg);
+	put_unaligned_be24(addr, msg + 2);
+
+	mutex_lock(&mhdp->mbox_mutex);
+
+	ret = cdns_mhdp_mailbox_send(mhdp, MB_MODULE_ID_DP_TX,
+				     DPTX_READ_DPCD, sizeof(msg), msg);
+	if (ret)
+		goto err_dpcd_read;
+
+	ret = cdns_mhdp_mailbox_validate_receive(mhdp, MB_MODULE_ID_DP_TX,
+						 DPTX_READ_DPCD,
+						 sizeof(reg) + len);
+	if (ret)
+		goto err_dpcd_read;
+
+	ret = cdns_mhdp_mailbox_read_receive(mhdp, reg, sizeof(reg));
+	if (ret)
+		goto err_dpcd_read;
+
+	ret = cdns_mhdp_mailbox_read_receive(mhdp, data, len);
+
+err_dpcd_read:
+	mutex_unlock(&mhdp->mbox_mutex);
+
+	return ret;
+}
+
+static
+int cdns_mhdp_dpcd_write(struct cdns_mhdp_device *mhdp, u32 addr, u8 value)
+{
+	u8 msg[6], reg[5];
+	int ret;
+
+	put_unaligned_be16(1, msg);
+	put_unaligned_be24(addr, msg + 2);
+	msg[5] = value;
+
+	mutex_lock(&mhdp->mbox_mutex);
+
+	ret = cdns_mhdp_mailbox_send(mhdp, MB_MODULE_ID_DP_TX,
+				     DPTX_WRITE_DPCD, sizeof(msg), msg);
+	if (ret)
+		goto err_dpcd_write;
+
+	ret = cdns_mhdp_mailbox_validate_receive(mhdp, MB_MODULE_ID_DP_TX,
+						 DPTX_WRITE_DPCD, sizeof(reg));
+	if (ret)
+		goto err_dpcd_write;
+
+	ret = cdns_mhdp_mailbox_read_receive(mhdp, reg, sizeof(reg));
+	if (ret)
+		goto err_dpcd_write;
+
+	if (addr != get_unaligned_be24(reg + 2))
+		ret = -EINVAL;
+
+err_dpcd_write:
+	mutex_unlock(&mhdp->mbox_mutex);
+
+	if (ret)
+		DRM_DEV_ERROR(mhdp->dev, "dpcd write failed: %d\n", ret);
+	return ret;
+}
+
+static
+int cdns_mhdp_set_firmware_active(struct cdns_mhdp_device *mhdp, bool enable)
+{
+	u8 msg[5];
+	int ret, i;
+
+	msg[0] = GENERAL_MAIN_CONTROL;
+	msg[1] = MB_MODULE_ID_GENERAL;
+	msg[2] = 0;
+	msg[3] = 1;
+	msg[4] = enable ? FW_ACTIVE : FW_STANDBY;
+
+	mutex_lock(&mhdp->mbox_mutex);
+
+	for (i = 0; i < sizeof(msg); i++) {
+		ret = cdp_dp_mailbox_write(mhdp, msg[i]);
+		if (ret)
+			goto err_set_firmware_active;
+	}
+
+	/* read the firmware state */
+	for (i = 0; i < sizeof(msg); i++)  {
+		ret = cdns_mhdp_mailbox_read(mhdp);
+		if (ret < 0)
+			goto err_set_firmware_active;
+
+		msg[i] = ret;
+	}
+
+	ret = 0;
+
+err_set_firmware_active:
+	mutex_unlock(&mhdp->mbox_mutex);
+
+	if (ret < 0)
+		DRM_DEV_ERROR(mhdp->dev, "set firmware active failed\n");
+	return ret;
+}
+
+static
+int cdns_mhdp_get_hpd_status(struct cdns_mhdp_device *mhdp)
+{
+	u8 status;
+	int ret;
+
+	mutex_lock(&mhdp->mbox_mutex);
+
+	ret = cdns_mhdp_mailbox_send(mhdp, MB_MODULE_ID_DP_TX,
+				     DPTX_HPD_STATE, 0, NULL);
+	if (ret)
+		goto err_get_hpd;
+
+	ret = cdns_mhdp_mailbox_validate_receive(mhdp, MB_MODULE_ID_DP_TX,
+						 DPTX_HPD_STATE,
+						 sizeof(status));
+	if (ret)
+		goto err_get_hpd;
+
+	ret = cdns_mhdp_mailbox_read_receive(mhdp, &status, sizeof(status));
+	if (ret)
+		goto err_get_hpd;
+
+	mutex_unlock(&mhdp->mbox_mutex);
+
+	return status;
+
+err_get_hpd:
+	mutex_unlock(&mhdp->mbox_mutex);
+
+	DRM_DEV_ERROR(mhdp->dev, "get hpd status failed: %d\n", ret);
+	return ret;
+}
+
+static
+int cdns_mhdp_get_edid_block(void *data, u8 *edid,
+			     unsigned int block, size_t length)
+{
+	struct cdns_mhdp_device *mhdp = data;
+	u8 msg[2], reg[2], i;
+	int ret;
+
+	mutex_lock(&mhdp->mbox_mutex);
+
+	for (i = 0; i < 4; i++) {
+		msg[0] = block / 2;
+		msg[1] = block % 2;
+
+		ret = cdns_mhdp_mailbox_send(mhdp, MB_MODULE_ID_DP_TX,
+					     DPTX_GET_EDID, sizeof(msg), msg);
+		if (ret)
+			continue;
+
+		ret = cdns_mhdp_mailbox_validate_receive(mhdp,
+							 MB_MODULE_ID_DP_TX,
+							 DPTX_GET_EDID,
+							 sizeof(reg) + length);
+		if (ret)
+			continue;
+
+		ret = cdns_mhdp_mailbox_read_receive(mhdp, reg, sizeof(reg));
+		if (ret)
+			continue;
+
+		ret = cdns_mhdp_mailbox_read_receive(mhdp, edid, length);
+		if (ret)
+			continue;
+
+		if (reg[0] == length && reg[1] == block / 2)
+			break;
+	}
+
+	mutex_unlock(&mhdp->mbox_mutex);
+
+	if (ret)
+		DRM_DEV_ERROR(mhdp->dev, "get block[%d] edid failed: %d\n",
+			      block, ret);
+
+	return ret;
+}
+
+static
+int cdns_mhdp_read_event(struct cdns_mhdp_device *mhdp)
+{
+	u8 event = 0;
+	int ret;
+
+	mutex_lock(&mhdp->mbox_mutex);
+
+	ret = cdns_mhdp_mailbox_send(mhdp, MB_MODULE_ID_DP_TX,
+				     DPTX_READ_EVENT, 0, NULL);
+	if (ret)
+		goto out;
+
+	ret = cdns_mhdp_mailbox_validate_receive(mhdp,
+						 MB_MODULE_ID_DP_TX,
+						 DPTX_READ_EVENT,
+						 sizeof(event));
+	if (ret < 0)
+		goto out;
+
+	ret = cdns_mhdp_mailbox_read_receive(mhdp, &event,
+					     sizeof(event));
+out:
+	mutex_unlock(&mhdp->mbox_mutex);
+
+	if (ret < 0)
+		return ret;
+
+	return event;
+}
+
+static
+int cdns_mhdp_adjust_lt(struct cdns_mhdp_device *mhdp,
+			u8 nlanes, u16 udelay, u8 *lanes_data, u8 *link_status)
+{
+	u8 payload[7];
+	u8 hdr[5]; /* For DPCD read response header */
+	u32 addr;
+	u8 const nregs = 6; /* Registers 0x202-0x207 */
+	int ret;
+
+	if (nlanes != 4 && nlanes != 2 && nlanes != 1) {
+		DRM_DEV_ERROR(mhdp->dev, "invalid number of lanes: %d\n",
+			      nlanes);
+		ret = -EINVAL;
+		goto err_adjust_lt;
+	}
+
+	payload[0] = nlanes;
+	put_unaligned_be16(udelay, payload + 1);
+	memcpy(payload + 3, lanes_data, nlanes);
+
+	mutex_lock(&mhdp->mbox_mutex);
+
+	ret = cdns_mhdp_mailbox_send(mhdp, MB_MODULE_ID_DP_TX,
+				     DPTX_ADJUST_LT,
+				     sizeof(payload), payload);
+	if (ret)
+		goto err_adjust_lt;
+
+	/* Yes, read the DPCD read command response */
+	ret = cdns_mhdp_mailbox_validate_receive(mhdp, MB_MODULE_ID_DP_TX,
+						 DPTX_READ_DPCD,
+						 sizeof(hdr) + nregs);
+	if (ret)
+		goto err_adjust_lt;
+
+	ret = cdns_mhdp_mailbox_read_receive(mhdp, hdr, sizeof(hdr));
+	if (ret)
+		goto err_adjust_lt;
+
+	addr = get_unaligned_be24(hdr + 2);
+	if (addr != DP_LANE0_1_STATUS)
+		goto err_adjust_lt;
+
+	ret = cdns_mhdp_mailbox_read_receive(mhdp, link_status, nregs);
+
+err_adjust_lt:
+	mutex_unlock(&mhdp->mbox_mutex);
+
+	if (ret)
+		DRM_DEV_ERROR(mhdp->dev, "Failed to adjust Link Training.\n");
+
+	return ret;
+}
+
+/* EOF CDNS MHDP Helpers */
+
+#define FW_NAME					"cadence/mhdp8546.bin"
+#define CDNS_MHDP_IMEM				0x10000
+
+#define CDNS_DP_TRAINING_PATTERN_4		0x7
+
+#define CDNS_KEEP_ALIVE_TIMEOUT			2000
+
+#ifdef CONFIG_DRM_CDNS_MHDP_J721E
+static const struct mhdp_platform_ops mhdp_ti_j721e_ops = {
+	.init = cdns_mhdp_j721e_init,
+	.exit = cdns_mhdp_j721e_fini,
+	.enable = cdns_mhdp_j721e_enable,
+	.disable = cdns_mhdp_j721e_disable,
+};
+#endif
+
+static const struct of_device_id mhdp_ids[] = {
+	{ .compatible = "cdns,mhdp8546", },
+#ifdef CONFIG_DRM_CDNS_MHDP_J721E
+	{ .compatible = "ti,j721e-mhdp8546", .data = &mhdp_ti_j721e_ops },
+#endif
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, mhdp_ids);
+
+#define CDNS_LANE_1				BIT(0)
+#define CDNS_LANE_2				BIT(1)
+#define CDNS_LANE_4				BIT(2)
+
+#define CDNS_VOLT_SWING(x)			((x) & GENMASK(1, 0))
+#define CDNS_FORCE_VOLT_SWING			BIT(2)
+
+#define CDNS_PRE_EMPHASIS(x)			((x) & GENMASK(1, 0))
+#define CDNS_FORCE_PRE_EMPHASIS			BIT(2)
+
+#define CDNS_SUPPORT_TPS(x)			BIT((x) - 1)
+
+#define CDNS_FAST_LINK_TRAINING			BIT(0)
+
+#define CDNS_LANE_MAPPING_TYPE_C_LANE_0(x)	((x) & GENMASK(1, 0))
+#define CDNS_LANE_MAPPING_TYPE_C_LANE_1(x)	((x) & GENMASK(3, 2))
+#define CDNS_LANE_MAPPING_TYPE_C_LANE_2(x)	((x) & GENMASK(5, 4))
+#define CDNS_LANE_MAPPING_TYPE_C_LANE_3(x)	((x) & GENMASK(7, 6))
+#define CDNS_LANE_MAPPING_NORMAL		0xe4
+#define CDNS_LANE_MAPPING_FLIPPED		0x1b
+
+#define CDNS_DP_MAX_NUM_LANES			4
+#define CDNS_DP_TEST_VSC_SDP			(1 << 6) /* 1.3+ */
+#define CDNS_DP_TEST_COLOR_FORMAT_RAW_Y_ONLY	(1 << 7)
+
+static unsigned int max_link_rate(struct cdns_mhdp_host host,
+				  struct cdns_mhdp_sink sink)
+{
+	return min(host.link_rate, sink.link_rate);
+}
+
+static u8 mhdp_max_num_lanes(struct cdns_mhdp_host host,
+			     struct cdns_mhdp_sink sink)
+{
+	return min_t(u8, sink.lanes_cnt, host.lanes_cnt);
+}
+
+static u8 eq_training_pattern_supported(struct cdns_mhdp_host host,
+					struct cdns_mhdp_sink sink)
+{
+	return fls(host.pattern_supp & sink.pattern_supp);
+}
+
+static bool mhdp_get_ssc_supported(struct cdns_mhdp_device *mhdp)
+{
+	/* Check if SSC is supported by both sides */
+	return (mhdp->host.ssc) && (mhdp->sink.ssc);
+}
+
+static int mhdp_fw_activate(const struct firmware *fw,
+			    struct cdns_mhdp_device *mhdp)
+{
+	unsigned int reg;
+	int ret = 0;
+
+	dev_dbg(mhdp->dev, "%s\n", __func__);
+
+	if (!fw || !fw->data) {
+		dev_err(mhdp->dev, "%s: No firmware.\n", __func__);
+		return -EINVAL;
+	}
+
+	spin_lock(&mhdp->start_lock);
+	if (mhdp->hw_state != MHDP_HW_INACTIVE) {
+		spin_unlock(&mhdp->start_lock);
+		if (mhdp->hw_state != MHDP_HW_STOPPED)
+			dev_err(mhdp->dev, "%s: Bad HW state: %d\n",
+				__func__, mhdp->hw_state);
+		return -EBUSY;
+	}
+	mhdp->hw_state = MHDP_HW_LOADING;
+	spin_unlock(&mhdp->start_lock);
+
+	/* Release uCPU reset and stall it. */
+	writel(CDNS_CPU_STALL, mhdp->regs + CDNS_APB_CTRL);
+
+	memcpy_toio(mhdp->regs + CDNS_MHDP_IMEM, fw->data, fw->size);
+
+	/* Leave debug mode, release stall */
+	writel(0, mhdp->regs + CDNS_APB_CTRL);
+
+	/*
+	 * Wait for the KEEP_ALIVE "message" on the first 8 bits.
+	 * Updated each sched "tick" (~2ms)
+	 */
+	ret = readl_poll_timeout(mhdp->regs + CDNS_KEEP_ALIVE, reg,
+				 reg & CDNS_KEEP_ALIVE_MASK, 500,
+				 CDNS_KEEP_ALIVE_TIMEOUT);
+	if (ret) {
+		dev_err(mhdp->dev,
+			"device didn't give any life sign: reg %d\n", reg);
+		goto error;
+	}
+
+	/* Init events to 0 as it's not cleared by FW at boot but on read */
+	readl(mhdp->regs + CDNS_SW_EVENT0);
+	readl(mhdp->regs + CDNS_SW_EVENT1);
+	readl(mhdp->regs + CDNS_SW_EVENT2);
+	readl(mhdp->regs + CDNS_SW_EVENT3);
+
+	/* Activate uCPU */
+	ret = cdns_mhdp_set_firmware_active(mhdp, true);
+	if (ret) {
+		dev_err(mhdp->dev, "%s: Failed to activate FW: %d\n",
+			__func__, ret);
+		goto error;
+	}
+
+	spin_lock(&mhdp->start_lock);
+
+	mhdp->hw_state = MHDP_HW_READY;
+
+	/*
+	 * Here we must keep the lock while enabling the interrupts
+	 * since it would otherwise be possible that interrupt enable
+	 * code is executed after the bridge is detached. The similar
+	 * situation is not possible in attach()/detach() callbacks
+	 * since the hw_state changes from MHDP_HW_READY to
+	 * MHDP_HW_STOPPED happens only due to driver removal when
+	 * bridge should already be detached.
+	 */
+	if (mhdp->bridge_attached) {
+		/* enable interrupts */
+		writel(0, mhdp->regs + CDNS_APB_INT_MASK);
+		writel(0, mhdp->regs + CDNS_MB_INT_MASK);
+	}
+
+	spin_unlock(&mhdp->start_lock);
+
+	dev_dbg(mhdp->dev, "DP FW activated\n");
+
+	return 0;
+error:
+	spin_lock(&mhdp->start_lock);
+	mhdp->hw_state = MHDP_HW_INACTIVE;
+	spin_unlock(&mhdp->start_lock);
+
+	return ret;
+}
+
+static void mhdp_fw_cb(const struct firmware *fw, void *context)
+{
+	struct cdns_mhdp_device *mhdp = context;
+	bool bridge_attached;
+	int ret;
+
+	dev_dbg(mhdp->dev, "firmware callback\n");
+
+	ret = mhdp_fw_activate(fw, mhdp);
+
+	release_firmware(fw);
+
+	if (ret)
+		return;
+
+	/*
+	 *  XXX how to make sure the bridge is still attached when
+	 *      calling drm_kms_helper_hotplug_event() after releasing
+	 *      the lock? We should not hold the spin lock when
+	 *      calling drm_kms_helper_hotplug_event() since it may
+	 *      cause a dead lock. FB-dev console calls detect from the
+	 *      same thread just down the call stack started here.
+	 */
+	spin_lock(&mhdp->start_lock);
+	bridge_attached = mhdp->bridge_attached;
+	spin_unlock(&mhdp->start_lock);
+	if (bridge_attached)
+		drm_kms_helper_hotplug_event(mhdp->bridge.dev);
+}
+
+static int load_firmware(struct cdns_mhdp_device *mhdp)
+{
+	const struct firmware *fw;
+	int ret;
+
+	/* First try if we can get firware directly */
+	ret = request_firmware_direct(&fw, FW_NAME, mhdp->dev);
+	if (ret == 0) {
+		ret = mhdp_fw_activate(fw, mhdp);
+
+		release_firmware(fw);
+
+		return ret;
+	}
+
+	/* Direct loading failed, activate delayed loading */
+	ret = request_firmware_nowait(THIS_MODULE, true, FW_NAME, mhdp->dev,
+				      GFP_KERNEL, mhdp, mhdp_fw_cb);
+	if (ret) {
+		dev_err(mhdp->dev, "failed to load firmware (%s), ret: %d\n",
+			FW_NAME, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void mhdp_check_link(struct cdns_mhdp_device *mhdp)
+{
+	struct drm_connector *conn = &mhdp->connector;
+	u8 status[DP_LINK_STATUS_SIZE];
+	bool hpd_state;
+	int hpd_event;
+	int ret;
+
+	/* Nothing to check if there is no link */
+	if (!mhdp->link_up)
+		return;
+
+	hpd_event = cdns_mhdp_read_event(mhdp);
+
+	/* Geting event bits failed, bail out */
+	if (hpd_event < 0) {
+		dev_warn(mhdp->dev, "%s: read event failed: %d\n",
+			 __func__, hpd_event);
+		return;
+	}
+
+	hpd_state = !!(hpd_event & DPTX_READ_EVENT_HPD_STATE);
+
+	/* No point the check the link if HPD is down (cable is unplugged) */
+	if (!hpd_state)
+		return;
+
+	/*
+	 * Prevent display reconfiguration between link check and link
+	 * status property setting. We must use the legacy giant-lock
+	 * since drm_connector_set_link_status_property()'s fine
+	 * grained DRM locking implementation is broken.
+	 */
+	mutex_lock(&conn->dev->mode_config.mutex);
+
+	/* Check if the link is still up */
+	ret = drm_dp_dpcd_read_link_status(&mhdp->aux, status);
+
+	if (ret < 0 || /* If dpcd read fails, assume the link is down too */
+	    !drm_dp_channel_eq_ok(status, mhdp->link.num_lanes) ||
+	    !drm_dp_clock_recovery_ok(status, mhdp->link.num_lanes))
+		/* Link is broken, indicate it with the link status property */
+		drm_connector_set_link_status_property(conn, DRM_MODE_LINK_STATUS_BAD);
+
+	mutex_unlock(&conn->dev->mode_config.mutex);
+}
+
+static irqreturn_t mhdp_irq_handler(int irq, void *data)
+{
+	struct cdns_mhdp_device *mhdp = (struct cdns_mhdp_device *)data;
+	u32 mbox_stat, apb_stat, sw_ev0, sw_ev1, sw_ev2, sw_ev3;
+	bool bridge_attached;
+
+	apb_stat = readl(mhdp->regs + CDNS_APB_INT_STATUS);
+	mbox_stat = readl(mhdp->regs + CDNS_MB_INT_STATUS);
+	sw_ev0 = readl(mhdp->regs + CDNS_SW_EVENT0);
+	sw_ev1 = readl(mhdp->regs + CDNS_SW_EVENT1);
+	sw_ev2 = readl(mhdp->regs + CDNS_SW_EVENT2);
+	sw_ev3 = readl(mhdp->regs + CDNS_SW_EVENT3);
+
+	//dev_dbg(mhdp->dev, "MHDP IRQ apb %x, mbox %x, sw_ev %x/%x/%x/%x\n", apb_stat, mbox_stat, sw_ev0, sw_ev1, sw_ev2, sw_ev3);
+
+	/*
+	 *  Calling drm_kms_helper_hotplug_event() when not attached
+	 *  to drm device causes an oops because the drm_bridge->dev
+	 *  is NULL. See mhdp_fw_cb() comments for details about the
+	 *  problems related drm_kms_helper_hotplug_event() call.
+	 */
+	spin_lock(&mhdp->start_lock);
+	bridge_attached = mhdp->bridge_attached;
+	spin_unlock(&mhdp->start_lock);
+
+	if (bridge_attached && (sw_ev0 & CDNS_DPTX_HPD)) {
+		mhdp_check_link(mhdp);
+
+		drm_kms_helper_hotplug_event(mhdp->bridge.dev);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static ssize_t mhdp_transfer(struct drm_dp_aux *aux,
+			     struct drm_dp_aux_msg *msg)
+{
+	struct cdns_mhdp_device *mhdp = dev_get_drvdata(aux->dev);
+	int ret;
+
+	if (msg->request != DP_AUX_NATIVE_WRITE &&
+	    msg->request != DP_AUX_NATIVE_READ)
+		return -ENOTSUPP;
+
+	if (msg->request == DP_AUX_NATIVE_WRITE) {
+		const u8 *buf = msg->buffer;
+		int i;
+
+		for (i = 0; i < msg->size; ++i) {
+			ret = cdns_mhdp_dpcd_write(mhdp,
+						   msg->address + i, buf[i]);
+			if (!ret)
+				continue;
+
+			DRM_DEV_ERROR(mhdp->dev, "Failed to write DPCD\n");
+
+			return ret;
+		}
+	} else {
+		ret = cdns_mhdp_dpcd_read(mhdp, msg->address,
+					  msg->buffer, msg->size);
+		if (ret) {
+			DRM_DEV_ERROR(mhdp->dev, "Failed to read DPCD\n");
+
+			return ret;
+		}
+	}
+
+	return msg->size;
+}
+
+static int cdns_mhdp_get_modes(struct drm_connector *connector)
+{
+	struct cdns_mhdp_device *mhdp = connector_to_mhdp(connector);
+	struct edid *edid;
+	int num_modes;
+
+	edid = drm_do_get_edid(connector, cdns_mhdp_get_edid_block, mhdp);
+	if (!edid) {
+		DRM_DEV_ERROR(mhdp->dev, "Failed to read EDID\n");
+
+		return 0;
+	}
+
+	drm_connector_update_edid_property(connector, edid);
+	num_modes = drm_add_edid_modes(connector, edid);
+	kfree(edid);
+
+	/*
+	 * HACK: Warn about unsupported display formats until we deal
+	 *       with them correctly.
+	 */
+	if (connector->display_info.color_formats &&
+	    !(connector->display_info.color_formats &
+	      mhdp->display_fmt.color_format))
+		dev_warn(mhdp->dev,
+			 "%s: No supported color_format found (0x%08x)\n",
+			__func__, connector->display_info.color_formats);
+
+	if (connector->display_info.bpc &&
+	    connector->display_info.bpc < mhdp->display_fmt.bpc)
+		dev_warn(mhdp->dev, "%s: Display bpc only %d < %d\n",
+			 __func__, connector->display_info.bpc,
+			 mhdp->display_fmt.bpc);
+
+	return num_modes;
+}
+
+static int cdns_mhdp_detect(struct drm_connector *conn,
+			    struct drm_modeset_acquire_ctx *ctx,
+			    bool force)
+{
+	struct cdns_mhdp_device *mhdp = connector_to_mhdp(conn);
+	bool hw_ready;
+	int ret;
+
+	dev_dbg(mhdp->dev, "%s\n", __func__);
+
+	spin_lock(&mhdp->start_lock);
+
+	hw_ready = mhdp->hw_state == MHDP_HW_READY;
+
+	spin_unlock(&mhdp->start_lock);
+
+	if (!hw_ready || WARN_ON(!mhdp->bridge_attached))
+		return connector_status_disconnected;
+
+	ret = cdns_mhdp_get_hpd_status(mhdp);
+	if (ret > 0) {
+		mhdp->plugged = true;
+		return connector_status_connected;
+	}
+	if (ret < 0)
+		dev_err(mhdp->dev, "Failed to obtain HPD state\n");
+
+	mhdp->plugged = false;
+
+	return connector_status_disconnected;
+}
+
+static
+bool cdns_mhdp_bandwidth_ok(struct cdns_mhdp_device *mhdp,
+			    const struct drm_display_mode *mode,
+			    int lanes, int rate)
+{
+	u32 max_bw, req_bw, bpp;
+
+	bpp = cdns_mhdp_get_bpp(&mhdp->display_fmt);
+	req_bw = mode->clock * bpp / 8;
+
+	max_bw = lanes * rate;
+
+	if (req_bw > max_bw) {
+		dev_dbg(mhdp->dev, "%s: %s (%u * %u/8 =) %u > %u (= %u * %u)\n",
+			__func__, mode->name, mode->clock, bpp, req_bw,
+			max_bw, lanes, rate);
+
+		return false;
+	}
+
+	return true;
+}
+
+static
+enum drm_mode_status cdns_mhdp_mode_valid(struct drm_connector *conn,
+					  struct drm_display_mode *mode)
+{
+	struct cdns_mhdp_device *mhdp = connector_to_mhdp(conn);
+
+	if (!cdns_mhdp_bandwidth_ok(mhdp, mode, mhdp->host.lanes_cnt,
+				    mhdp->host.link_rate))
+		return MODE_CLOCK_HIGH;
+
+	return MODE_OK;
+}
+
+static const struct drm_connector_helper_funcs cdns_mhdp_conn_helper_funcs = {
+	.detect_ctx = cdns_mhdp_detect,
+	.get_modes = cdns_mhdp_get_modes,
+	.mode_valid = cdns_mhdp_mode_valid,
+};
+
+static const struct drm_connector_funcs cdns_mhdp_conn_funcs = {
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+	.reset = drm_atomic_helper_connector_reset,
+	.destroy = drm_connector_cleanup,
+};
+
+static int cdns_mhdp_attach(struct drm_bridge *bridge)
+{
+	struct cdns_mhdp_device *mhdp = bridge_to_mhdp(bridge);
+	u32 bus_format = MEDIA_BUS_FMT_RGB121212_1X36;
+	struct drm_connector *conn = &mhdp->connector;
+	bool hw_ready;
+	int ret;
+
+	dev_dbg(mhdp->dev, "%s\n", __func__);
+
+	if (&mhdp->bridge != bridge)
+		return -ENODEV;
+
+	conn->polled = DRM_CONNECTOR_POLL_HPD;
+
+	ret = drm_connector_init(bridge->dev, conn, &cdns_mhdp_conn_funcs,
+				 DRM_MODE_CONNECTOR_DisplayPort);
+	if (ret) {
+		dev_err(mhdp->dev, "failed to init connector\n");
+		return ret;
+	}
+
+	drm_connector_helper_add(conn, &cdns_mhdp_conn_helper_funcs);
+
+	ret = drm_display_info_set_bus_formats(&conn->display_info,
+					       &bus_format, 1);
+	if (ret)
+		return ret;
+
+	conn->display_info.bus_flags = DRM_BUS_FLAG_DE_HIGH;
+	/*
+	 * HACK: DP is internal to J7 SoC and we need to use DRIVE_POSEDGE
+	 * in the display controller. This is achieved for the time being
+	 * by defining SAMPLE_NEGEDGE here.
+	 */
+	conn->display_info.bus_flags |= DRM_BUS_FLAG_PIXDATA_SAMPLE_NEGEDGE |
+		DRM_BUS_FLAG_SYNC_SAMPLE_NEGEDGE;
+
+	ret = drm_connector_attach_encoder(conn, bridge->encoder);
+	if (ret) {
+		dev_err(mhdp->dev, "failed to attach connector to encoder\n");
+		return ret;
+	}
+
+	spin_lock(&mhdp->start_lock);
+
+	mhdp->bridge_attached = true;
+	hw_ready = mhdp->hw_state == MHDP_HW_READY;
+
+	spin_unlock(&mhdp->start_lock);
+
+	if (hw_ready) {
+		/* enable interrupts */
+		writel(0, mhdp->regs + CDNS_APB_INT_MASK);
+		writel(0, mhdp->regs + CDNS_MB_INT_MASK);
+	}
+
+	//writel(~CDNS_APB_INT_MASK_SW_EVENT_INT, mhdp->regs + CDNS_APB_INT_MASK);
+
+	return 0;
+}
+
+static int mhdp_link_training_init(struct cdns_mhdp_device *mhdp)
+{
+	u32 reg32;
+	u8 i;
+	union phy_configure_opts phy_cfg;
+	int ret;
+
+	drm_dp_dpcd_writeb(&mhdp->aux, DP_TRAINING_PATTERN_SET,
+			   DP_TRAINING_PATTERN_DISABLE);
+
+	/* Reset PHY configuration */
+	reg32 = CDNS_PHY_COMMON_CONFIG | CDNS_PHY_TRAINING_TYPE(1);
+	if (!mhdp->host.scrambler)
+		reg32 |= CDNS_PHY_SCRAMBLER_BYPASS;
+
+	cdns_mhdp_reg_write(mhdp, CDNS_DPTX_PHY_CONFIG, reg32);
+
+	cdns_mhdp_reg_write(mhdp, CDNS_DP_ENHNCD,
+			    mhdp->sink.enhanced & mhdp->host.enhanced);
+
+	cdns_mhdp_reg_write(mhdp, CDNS_DP_LANE_EN,
+			    CDNS_DP_LANE_EN_LANES(mhdp->link.num_lanes));
+
+	drm_dp_link_configure(&mhdp->aux, &mhdp->link);
+	phy_cfg.dp.link_rate = (mhdp->link.rate / 100);
+	phy_cfg.dp.lanes = (mhdp->link.num_lanes);
+	for (i = 0; i < 4; i++) {
+		phy_cfg.dp.voltage[i] = 0;
+		phy_cfg.dp.pre[i] = 0;
+	}
+	phy_cfg.dp.ssc = mhdp_get_ssc_supported(mhdp);
+	phy_cfg.dp.set_lanes = true;
+	phy_cfg.dp.set_rate = true;
+	phy_cfg.dp.set_voltages = true;
+	ret = phy_configure(mhdp->phy,  &phy_cfg);
+	if (ret) {
+		dev_err(mhdp->dev, "%s: phy_configure() failed: %d\n",
+			__func__, ret);
+		return ret;
+	}
+
+	cdns_mhdp_reg_write(mhdp, CDNS_DPTX_PHY_CONFIG,
+			    CDNS_PHY_COMMON_CONFIG |
+			    CDNS_PHY_TRAINING_EN |
+			    CDNS_PHY_TRAINING_TYPE(1) |
+			    CDNS_PHY_SCRAMBLER_BYPASS);
+
+	drm_dp_dpcd_writeb(&mhdp->aux, DP_TRAINING_PATTERN_SET,
+			   DP_TRAINING_PATTERN_1 | DP_LINK_SCRAMBLING_DISABLE);
+
+	return 0;
+}
+
+static void mhdp_get_adjust_train(struct cdns_mhdp_device *mhdp,
+				  u8 link_status[DP_LINK_STATUS_SIZE],
+				  u8 lanes_data[CDNS_DP_MAX_NUM_LANES],
+				  union phy_configure_opts *phy_cfg)
+{
+	unsigned int i;
+	u8 adjust, max_pre_emphasis, max_volt_swing;
+	u8 set_volt, set_pre;
+
+	max_pre_emphasis = CDNS_PRE_EMPHASIS(mhdp->host.pre_emphasis)
+			   << DP_TRAIN_PRE_EMPHASIS_SHIFT;
+	max_volt_swing = CDNS_VOLT_SWING(mhdp->host.volt_swing);
+
+	for (i = 0; i < mhdp->link.num_lanes; i++) {
+		/* Check if Voltage swing and pre-emphasis are within limits */
+		adjust = drm_dp_get_adjust_request_voltage(link_status, i);
+		set_volt = min_t(u8, adjust, max_volt_swing);
+
+		adjust = drm_dp_get_adjust_request_pre_emphasis(link_status, i);
+		set_pre = min_t(u8, adjust, max_pre_emphasis) >> DP_TRAIN_PRE_EMPHASIS_SHIFT;
+
+		/* Voltage swing level and pre-emphasis level combination is not allowed:
+		 * leaving pre-emphasis as-is, and adjusting voltage swing.
+		 */
+		if (set_volt + set_pre > 3)
+			set_volt = 3 - set_pre;
+
+		phy_cfg->dp.voltage[i] = set_volt;
+		lanes_data[i] = set_volt;
+
+		if (set_volt == max_volt_swing)
+			lanes_data[i] |= DP_TRAIN_MAX_SWING_REACHED;
+
+		phy_cfg->dp.pre[i] = set_pre;
+		lanes_data[i] |= (set_pre << DP_TRAIN_PRE_EMPHASIS_SHIFT);
+
+		if (set_pre == (max_pre_emphasis >> DP_TRAIN_PRE_EMPHASIS_SHIFT))
+			lanes_data[i] |= DP_TRAIN_MAX_PRE_EMPHASIS_REACHED;
+	}
+}
+
+static void mhdp_set_adjust_request_voltage(
+	u8 link_status[DP_LINK_STATUS_SIZE], int lane, u8 volt)
+{
+	int i = DP_ADJUST_REQUEST_LANE0_1 + (lane >> 1);
+	int s = ((lane & 1) ?
+		 DP_ADJUST_VOLTAGE_SWING_LANE1_SHIFT :
+		 DP_ADJUST_VOLTAGE_SWING_LANE0_SHIFT);
+	int idx = i - DP_LANE0_1_STATUS;
+
+	link_status[idx] &= ~(DP_ADJUST_VOLTAGE_SWING_LANE0_MASK << s);
+	link_status[idx] |= volt << s;
+}
+
+static void mhdp_set_adjust_request_pre_emphasis(
+	u8 link_status[DP_LINK_STATUS_SIZE], int lane, u8 pre_emphasis)
+{
+	int i = DP_ADJUST_REQUEST_LANE0_1 + (lane >> 1);
+	int s = ((lane & 1) ?
+		 DP_ADJUST_PRE_EMPHASIS_LANE1_SHIFT :
+		 DP_ADJUST_PRE_EMPHASIS_LANE0_SHIFT);
+	int idx = i - DP_LANE0_1_STATUS;
+
+	link_status[idx] &= ~(DP_ADJUST_PRE_EMPHASIS_LANE0_MASK << s);
+	link_status[idx] |= pre_emphasis << s;
+}
+
+static void mhdp_adjust_requested_eq(struct cdns_mhdp_device *mhdp,
+				     u8 link_status[DP_LINK_STATUS_SIZE])
+{
+	unsigned int i;
+	u8 volt, pre, max_volt = CDNS_VOLT_SWING(mhdp->host.volt_swing),
+		      max_pre = CDNS_PRE_EMPHASIS(mhdp->host.pre_emphasis);
+
+	for (i = 0; i < mhdp->link.num_lanes; i++) {
+		volt = drm_dp_get_adjust_request_voltage(link_status, i);
+		pre = drm_dp_get_adjust_request_pre_emphasis(link_status, i);
+		if (volt + pre > 3)
+			mhdp_set_adjust_request_voltage(link_status, i,
+							3 - pre);
+		if (mhdp->host.volt_swing & CDNS_FORCE_VOLT_SWING)
+			mhdp_set_adjust_request_voltage(link_status, i,
+							max_volt);
+		if (mhdp->host.pre_emphasis & CDNS_FORCE_PRE_EMPHASIS)
+			mhdp_set_adjust_request_pre_emphasis(link_status, i,
+							     max_pre);
+	}
+}
+
+static bool mhdp_link_training_channel_eq(struct cdns_mhdp_device *mhdp,
+					  u8 eq_tps,
+					  unsigned int training_interval)
+{
+	u8 lanes_data[CDNS_DP_MAX_NUM_LANES], fail_counter_short = 0;
+	u8 link_status[DP_LINK_STATUS_SIZE];
+	u32 reg32;
+	union phy_configure_opts phy_cfg;
+	int ret;
+
+	dev_dbg(mhdp->dev, "Starting EQ phase\n");
+
+	/* Enable link training TPS[eq_tps] in PHY */
+	reg32 = CDNS_PHY_COMMON_CONFIG | CDNS_PHY_TRAINING_EN |
+		CDNS_PHY_TRAINING_TYPE(eq_tps);
+	if (eq_tps != 4)
+		reg32 |= CDNS_PHY_SCRAMBLER_BYPASS;
+	cdns_mhdp_reg_write(mhdp, CDNS_DPTX_PHY_CONFIG, reg32);
+
+	drm_dp_dpcd_writeb(&mhdp->aux, DP_TRAINING_PATTERN_SET,
+			   (eq_tps != 4) ? eq_tps | DP_LINK_SCRAMBLING_DISABLE :
+			   CDNS_DP_TRAINING_PATTERN_4);
+
+	drm_dp_dpcd_read_link_status(&mhdp->aux, link_status);
+
+	do {
+		mhdp_get_adjust_train(mhdp, link_status, lanes_data, &phy_cfg);
+		phy_cfg.dp.lanes = (mhdp->link.num_lanes);
+		phy_cfg.dp.ssc = mhdp_get_ssc_supported(mhdp);
+		phy_cfg.dp.set_lanes = false;
+		phy_cfg.dp.set_rate = false;
+		phy_cfg.dp.set_voltages = true;
+		ret = phy_configure(mhdp->phy,  &phy_cfg);
+		if (ret) {
+			dev_err(mhdp->dev, "%s: phy_configure() failed: %d\n",
+				__func__, ret);
+			goto err;
+		}
+
+		cdns_mhdp_adjust_lt(mhdp, mhdp->link.num_lanes,
+				    training_interval, lanes_data, link_status);
+
+		if (!drm_dp_clock_recovery_ok(link_status, mhdp->link.num_lanes))
+			goto err;
+
+		if (drm_dp_channel_eq_ok(link_status, mhdp->link.num_lanes)) {
+			dev_dbg(mhdp->dev, "EQ phase succeeded\n");
+			return true;
+		}
+
+		fail_counter_short++;
+
+		mhdp_adjust_requested_eq(mhdp, link_status);
+	} while (fail_counter_short < 5);
+
+err:
+	dev_dbg(mhdp->dev, "EQ phase failed for %d lanes and %d rate\n",
+		mhdp->link.num_lanes, mhdp->link.rate);
+
+	return false;
+}
+
+static void mhdp_adjust_requested_cr(struct cdns_mhdp_device *mhdp,
+				     u8 link_status[DP_LINK_STATUS_SIZE],
+				     u8 *req_volt, u8 *req_pre)
+{
+	const u32 max_volt = CDNS_VOLT_SWING(mhdp->host.volt_swing),
+		  max_pre = CDNS_PRE_EMPHASIS(mhdp->host.pre_emphasis);
+	unsigned int i;
+
+	for (i = 0; i < mhdp->link.num_lanes; i++) {
+		unsigned int val;
+
+		val = mhdp->host.volt_swing & CDNS_FORCE_VOLT_SWING ?
+		      max_volt : req_volt[i];
+		mhdp_set_adjust_request_voltage(link_status, i, val);
+
+		val = mhdp->host.pre_emphasis & CDNS_FORCE_PRE_EMPHASIS ?
+		      max_pre : req_pre[i];
+		mhdp_set_adjust_request_pre_emphasis(link_status, i, val);
+	}
+}
+
+static void mhdp_validate_cr(struct cdns_mhdp_device *mhdp, bool *cr_done,
+			     bool *same_before_adjust, bool *max_swing_reached,
+			     u8 before_cr[DP_LINK_STATUS_SIZE],
+			     u8 after_cr[DP_LINK_STATUS_SIZE], u8 *req_volt,
+			     u8 *req_pre)
+{
+	const u8 max_volt = CDNS_VOLT_SWING(mhdp->host.volt_swing),
+		 max_pre = CDNS_PRE_EMPHASIS(mhdp->host.pre_emphasis);
+	bool same_pre, same_volt;
+	unsigned int i;
+
+	*same_before_adjust = false;
+	*max_swing_reached = false;
+	*cr_done = drm_dp_clock_recovery_ok(after_cr, mhdp->link.num_lanes);
+
+	for (i = 0; i < mhdp->link.num_lanes; i++) {
+		u8 tmp;
+
+		tmp = drm_dp_get_adjust_request_voltage(after_cr, i);
+		req_volt[i] = min_t(u8, tmp, max_volt);
+
+		tmp = drm_dp_get_adjust_request_pre_emphasis(after_cr, i) >>
+		      DP_TRAIN_PRE_EMPHASIS_SHIFT;
+		req_pre[i] = min_t(u8, tmp, max_pre);
+
+		same_pre = (before_cr[i] & DP_TRAIN_PRE_EMPHASIS_MASK) ==
+			   req_pre[i] << DP_TRAIN_PRE_EMPHASIS_SHIFT;
+		same_volt = (before_cr[i] & DP_TRAIN_VOLTAGE_SWING_MASK) ==
+			    req_volt[i];
+		if (same_pre && same_volt)
+			*same_before_adjust = true;
+
+		/* 3.1.5.2 in DP Standard v1.4. Table 3-1 */
+		if (!*cr_done && req_volt[i] + req_pre[i] >= 3) {
+			*max_swing_reached = true;
+			return;
+		}
+	}
+}
+
+static bool mhdp_link_training_cr(struct cdns_mhdp_device *mhdp)
+{
+	u8 lanes_data[CDNS_DP_MAX_NUM_LANES],
+	fail_counter_short = 0, fail_counter_cr_long = 0;
+	u8 link_status[DP_LINK_STATUS_SIZE];
+	bool cr_done;
+	union phy_configure_opts phy_cfg;
+	int ret;
+
+	dev_dbg(mhdp->dev, "Starting CR phase\n");
+
+	ret = mhdp_link_training_init(mhdp);
+	if (ret)
+		goto err;
+
+	drm_dp_dpcd_read_link_status(&mhdp->aux, link_status);
+
+	do {
+		u8 requested_adjust_volt_swing[CDNS_DP_MAX_NUM_LANES] = {};
+		u8 requested_adjust_pre_emphasis[CDNS_DP_MAX_NUM_LANES] = {};
+		bool same_before_adjust, max_swing_reached;
+
+		mhdp_get_adjust_train(mhdp, link_status, lanes_data, &phy_cfg);
+		phy_cfg.dp.lanes = (mhdp->link.num_lanes);
+		phy_cfg.dp.ssc = mhdp_get_ssc_supported(mhdp);
+		phy_cfg.dp.set_lanes = false;
+		phy_cfg.dp.set_rate = false;
+		phy_cfg.dp.set_voltages = true;
+		ret = phy_configure(mhdp->phy,  &phy_cfg);
+		if (ret) {
+			dev_err(mhdp->dev, "%s: phy_configure() failed: %d\n",
+				__func__, ret);
+			goto err;
+		}
+
+		cdns_mhdp_adjust_lt(mhdp, mhdp->link.num_lanes, 100,
+				    lanes_data, link_status);
+
+		mhdp_validate_cr(mhdp, &cr_done, &same_before_adjust,
+				 &max_swing_reached, lanes_data, link_status,
+				 requested_adjust_volt_swing,
+				 requested_adjust_pre_emphasis);
+
+		if (max_swing_reached) {
+			dev_err(mhdp->dev, "CR: max swing reached\n");
+			goto err;
+		}
+
+		if (cr_done) {
+			dev_dbg(mhdp->dev, "CR phase succeeded\n");
+			return true;
+		}
+
+		/* Not all CR_DONE bits set */
+		fail_counter_cr_long++;
+
+		if (same_before_adjust) {
+			fail_counter_short++;
+			continue;
+		}
+
+		fail_counter_short = 0;
+		/*
+		 * Voltage swing/pre-emphasis adjust requested
+		 * during CR phase
+		 */
+		mhdp_adjust_requested_cr(mhdp, link_status,
+					 requested_adjust_volt_swing,
+					 requested_adjust_pre_emphasis);
+	} while (fail_counter_short < 5 && fail_counter_cr_long < 10);
+
+err:
+	dev_dbg(mhdp->dev, "CR phase failed for %d lanes and %d rate\n",
+		mhdp->link.num_lanes, mhdp->link.rate);
+
+	return false;
+}
+
+static void lower_link_rate(struct drm_dp_link *link)
+{
+	switch (drm_dp_link_rate_to_bw_code(link->rate)) {
+	case DP_LINK_BW_2_7:
+		link->rate = drm_dp_bw_code_to_link_rate(DP_LINK_BW_1_62);
+		break;
+	case DP_LINK_BW_5_4:
+		link->rate = drm_dp_bw_code_to_link_rate(DP_LINK_BW_2_7);
+		break;
+	case DP_LINK_BW_8_1:
+		link->rate = drm_dp_bw_code_to_link_rate(DP_LINK_BW_5_4);
+		break;
+	}
+}
+
+static int mhdp_link_training(struct cdns_mhdp_device *mhdp,
+			      unsigned int training_interval)
+{
+	u32 reg32;
+	const u8 eq_tps = eq_training_pattern_supported(mhdp->host, mhdp->sink);
+
+	while (1) {
+		if (!mhdp_link_training_cr(mhdp)) {
+			if (drm_dp_link_rate_to_bw_code(mhdp->link.rate) !=
+			    DP_LINK_BW_1_62) {
+				dev_dbg(mhdp->dev,
+					"Reducing link rate during CR phase\n");
+				lower_link_rate(&mhdp->link);
+
+				continue;
+			} else if (mhdp->link.num_lanes > 1) {
+				dev_dbg(mhdp->dev,
+					"Reducing lanes number during CR phase\n");
+				mhdp->link.num_lanes >>= 1;
+				mhdp->link.rate = max_link_rate(mhdp->host,
+								mhdp->sink);
+
+				continue;
+			}
+
+			dev_dbg(mhdp->dev,
+				"Link training failed during CR phase\n");
+			goto err;
+		}
+
+		if (mhdp_link_training_channel_eq(mhdp, eq_tps,
+						  training_interval))
+			break;
+
+		if (mhdp->link.num_lanes > 1) {
+			dev_dbg(mhdp->dev,
+				"Reducing lanes number during EQ phase\n");
+			mhdp->link.num_lanes >>= 1;
+
+			continue;
+		} else if (drm_dp_link_rate_to_bw_code(mhdp->link.rate) !=
+			   DP_LINK_BW_1_62) {
+			dev_dbg(mhdp->dev,
+				"Reducing link rate during EQ phase\n");
+			lower_link_rate(&mhdp->link);
+			mhdp->link.num_lanes = mhdp_max_num_lanes(mhdp->host, mhdp->sink);
+
+			continue;
+		}
+
+		dev_dbg(mhdp->dev, "Link training failed during EQ phase\n");
+		goto err;
+	}
+
+	dev_dbg(mhdp->dev, "Link training successful\n");
+
+	drm_dp_dpcd_writeb(&mhdp->aux, DP_TRAINING_PATTERN_SET,
+			   mhdp->host.scrambler ? 0 :
+			   DP_LINK_SCRAMBLING_DISABLE);
+
+	cdns_mhdp_reg_read(mhdp, CDNS_DP_FRAMER_GLOBAL_CONFIG, &reg32);
+	reg32 &= ~GENMASK(1, 0);
+	reg32 |= CDNS_DP_NUM_LANES(mhdp->link.num_lanes);
+	reg32 |= CDNS_DP_WR_FAILING_EDGE_VSYNC;
+	cdns_mhdp_reg_write(mhdp, CDNS_DP_FRAMER_GLOBAL_CONFIG, reg32);
+
+	return 0;
+err:
+	/* Reset PHY config */
+	reg32 = CDNS_PHY_COMMON_CONFIG | CDNS_PHY_TRAINING_TYPE(1);
+	if (!mhdp->host.scrambler)
+		reg32 |= CDNS_PHY_SCRAMBLER_BYPASS;
+	cdns_mhdp_reg_write(mhdp, CDNS_DPTX_PHY_CONFIG, reg32);
+
+	drm_dp_dpcd_writeb(&mhdp->aux, DP_TRAINING_PATTERN_SET,
+			   DP_TRAINING_PATTERN_DISABLE);
+
+	return -EIO;
+}
+
+static void cdns_mhdp_disable(struct drm_bridge *bridge)
+{
+	struct cdns_mhdp_device *mhdp = bridge_to_mhdp(bridge);
+	u32 resp;
+
+	dev_dbg(mhdp->dev, "%s\n", __func__);
+
+	cdns_mhdp_reg_read(mhdp, CDNS_DP_FRAMER_GLOBAL_CONFIG, &resp);
+	resp &= ~CDNS_DP_FRAMER_EN;
+	resp |= CDNS_DP_NO_VIDEO_MODE;
+	cdns_mhdp_reg_write(mhdp, CDNS_DP_FRAMER_GLOBAL_CONFIG, resp);
+
+	mhdp->link_up = false;
+
+	if (mhdp->plugged)
+		drm_dp_link_power_down(&mhdp->aux, &mhdp->link);
+
+	/* Disable VIF clock for stream 0 */
+	cdns_mhdp_reg_read(mhdp, CDNS_DPTX_CAR, &resp);
+	cdns_mhdp_reg_write(mhdp, CDNS_DPTX_CAR,
+			    resp & ~(CDNS_VIF_CLK_EN | CDNS_VIF_CLK_RSTN));
+
+	if (mhdp->ops && mhdp->ops->disable)
+		mhdp->ops->disable(mhdp);
+}
+
+static u32 get_training_interval_us(struct cdns_mhdp_device *mhdp,
+				    u32 interval)
+{
+	if (interval == 0)
+		return 400;
+	if (interval < 5)
+		return 4000 << (interval - 1);
+	dev_err(mhdp->dev,
+		"wrong training interval returned by DPCD: %d\n", interval);
+	return 0;
+}
+
+static void mhdp_fill_sink_caps(struct cdns_mhdp_device *mhdp,
+				u8 dpcd[DP_RECEIVER_CAP_SIZE])
+{
+	mhdp->sink.link_rate = mhdp->link.rate;
+	mhdp->sink.lanes_cnt = mhdp->link.num_lanes;
+	mhdp->sink.enhanced = !!(mhdp->link.capabilities &
+				 DP_LINK_CAP_ENHANCED_FRAMING);
+
+	/* Set SSC support */
+	mhdp->sink.ssc = !!(dpcd[DP_MAX_DOWNSPREAD] &
+				  DP_MAX_DOWNSPREAD_0_5);
+
+	/* Set TPS support */
+	mhdp->sink.pattern_supp = CDNS_SUPPORT_TPS(1) | CDNS_SUPPORT_TPS(2);
+	if (drm_dp_tps3_supported(dpcd))
+		mhdp->sink.pattern_supp |= CDNS_SUPPORT_TPS(3);
+	if (drm_dp_tps4_supported(dpcd))
+		mhdp->sink.pattern_supp |= CDNS_SUPPORT_TPS(4);
+
+	/* Set fast link support */
+	mhdp->sink.fast_link = !!(dpcd[DP_MAX_DOWNSPREAD] &
+				  DP_NO_AUX_HANDSHAKE_LINK_TRAINING);
+}
+
+static int cdns_mhdp_link_up(struct cdns_mhdp_device *mhdp)
+{
+	u32 resp;
+	u8 reg0[DP_RECEIVER_CAP_SIZE], amp[2];
+	u8 ext_cap_chk = 0;
+	unsigned int addr;
+	int err;
+
+	drm_dp_link_probe(&mhdp->aux, &mhdp->link);
+
+	dev_dbg(mhdp->dev, "Set sink device power state via DPCD\n");
+	drm_dp_link_power_up(&mhdp->aux, &mhdp->link);
+	/* FIXME (CDNS): do we have to wait for 100ms before going on? */
+	mdelay(100);
+
+	drm_dp_dpcd_readb(&mhdp->aux, DP_TRAINING_AUX_RD_INTERVAL, &ext_cap_chk);
+
+	if (ext_cap_chk & DP_EXTENDED_RECEIVER_CAP_FIELD_PRESENT)
+		addr = DP_DP13_DPCD_REV;
+	else
+		addr = DP_DPCD_REV;
+
+	err = drm_dp_dpcd_read(&mhdp->aux, addr, reg0, DP_RECEIVER_CAP_SIZE);
+	if (err < 0) {
+		dev_err(mhdp->dev, "Failed to read receiver capabilities\n");
+		return err;
+	}
+
+	mhdp_fill_sink_caps(mhdp, reg0);
+
+	mhdp->link.rate = max_link_rate(mhdp->host, mhdp->sink);
+	mhdp->link.num_lanes = mhdp_max_num_lanes(mhdp->host, mhdp->sink);
+
+	/* Disable framer for link training */
+	cdns_mhdp_reg_read(mhdp, CDNS_DP_FRAMER_GLOBAL_CONFIG, &resp);
+	resp &= ~CDNS_DP_FRAMER_EN;
+	cdns_mhdp_reg_write(mhdp, CDNS_DP_FRAMER_GLOBAL_CONFIG, resp);
+
+	/* Spread AMP if required, enable 8b/10b coding */
+	amp[0] = mhdp_get_ssc_supported(mhdp) ? DP_SPREAD_AMP_0_5 : 0;
+	amp[1] = DP_SET_ANSI_8B10B;
+	drm_dp_dpcd_write(&mhdp->aux, DP_DOWNSPREAD_CTRL, amp, 2);
+
+	if (mhdp->host.fast_link & mhdp->sink.fast_link) {
+		/* FIXME: implement fastlink */
+		dev_err(mhdp->dev, "fastlink not supported\n");
+		return -ENOTSUPP;
+	} else {
+		const u32 interval = reg0[DP_TRAINING_AUX_RD_INTERVAL] &
+				     DP_TRAINING_AUX_RD_MASK;
+		const u32 interval_us = get_training_interval_us(mhdp,
+								 interval);
+		if (!interval_us ||
+		    mhdp_link_training(mhdp, interval_us)) {
+			dev_err(mhdp->dev, "Link training failed. Exiting.\n");
+			return -EIO;
+		}
+	}
+
+	mhdp->link_up = true;
+
+	return 0;
+}
+
+u32 cdns_mhdp_get_bpp(struct cdns_mhdp_display_fmt *fmt)
+{
+	u32 bpp;
+
+	if (fmt->y_only)
+		return fmt->bpc;
+
+	switch (fmt->color_format) {
+	case DRM_COLOR_FORMAT_RGB444:
+	case DRM_COLOR_FORMAT_YCRCB444:
+		bpp = fmt->bpc * 3;
+		break;
+	case DRM_COLOR_FORMAT_YCRCB422:
+		bpp = fmt->bpc * 2;
+		break;
+	case DRM_COLOR_FORMAT_YCRCB420:
+		bpp = fmt->bpc * 3 / 2;
+		break;
+	default:
+		bpp = fmt->bpc * 3;
+		WARN_ON(1);
+	}
+	return bpp;
+}
+
+static int cdns_mhdp_sst_enable(struct drm_bridge *bridge)
+{
+	struct cdns_mhdp_device *mhdp = bridge_to_mhdp(bridge);
+	u32 rate, vs, vs_f, required_bandwidth, available_bandwidth;
+	u32 tu_size = 30, line_thresh1, line_thresh2, line_thresh = 0;
+	struct drm_display_mode *mode;
+	int pxlclock;
+	u32 bpp, bpc, pxlfmt;
+
+	pxlfmt = mhdp->display_fmt.color_format;
+	bpc = mhdp->display_fmt.bpc;
+
+	mode = &bridge->encoder->crtc->state->mode;
+	pxlclock = mode->crtc_clock;
+
+	mhdp->stream_id = 0;
+
+	rate = mhdp->link.rate / 1000;
+
+	bpp = cdns_mhdp_get_bpp(&mhdp->display_fmt);
+
+	if (!cdns_mhdp_bandwidth_ok(mhdp, mode, mhdp->link.num_lanes,
+				    mhdp->link.rate)) {
+		dev_err(mhdp->dev, "%s: Not enough BW for %s (%u lanes at %u Mbps)\n",
+			__func__, mode->name, mhdp->link.num_lanes,
+			mhdp->link.rate / 100);
+		return -EINVAL;
+	}
+
+	/* find optimal tu_size */
+	required_bandwidth = pxlclock * bpp / 8;
+	available_bandwidth = mhdp->link.num_lanes * rate;
+	do {
+		tu_size += 2;
+
+		vs_f = tu_size * required_bandwidth / available_bandwidth;
+		vs = vs_f / 1000;
+		vs_f = vs_f % 1000;
+		/*
+		 * FIXME (CDNS): downspreading?
+		 * It's unused is what I've been told.
+		 */
+	} while ((vs == 1 || ((vs_f > 850 || vs_f < 100) && vs_f != 0) ||
+		  tu_size - vs < 2) && tu_size < 64);
+
+	if (vs > 64) {
+		dev_err(mhdp->dev,
+			"%s: No space for framing %s (%u lanes at %u Mbps)\n",
+			__func__, mode->name, mhdp->link.num_lanes,
+			mhdp->link.rate / 100);
+		return -EINVAL;
+	}
+
+	cdns_mhdp_reg_write(mhdp, CDNS_DP_FRAMER_TU,
+			    CDNS_DP_FRAMER_TU_VS(vs) |
+			    CDNS_DP_FRAMER_TU_SIZE(tu_size) |
+			    CDNS_DP_FRAMER_TU_CNT_RST_EN);
+
+	line_thresh1 = ((vs + 1) << 5) * 8 / bpp;
+	line_thresh2 = (pxlclock << 5) / 1000 / rate * (vs + 1) - (1 << 5);
+	line_thresh = line_thresh1 - line_thresh2 / mhdp->link.num_lanes;
+	line_thresh = (line_thresh >> 5) + 2;
+	cdns_mhdp_reg_write(mhdp, CDNS_DP_LINE_THRESH(0),
+			    line_thresh & GENMASK(5, 0));
+
+	cdns_mhdp_reg_write(mhdp, CDNS_DP_STREAM_CONFIG_2(0),
+			    CDNS_DP_SC2_TU_VS_DIFF((tu_size - vs > 3) ?
+						   0 : tu_size - vs));
+
+	cdns_mhdp_configure_video(bridge);
+
+	return 0;
+}
+
+void cdns_mhdp_configure_video(struct drm_bridge *bridge)
+{
+	struct cdns_mhdp_device *mhdp = bridge_to_mhdp(bridge);
+	unsigned int dp_framer_sp = 0, msa_horizontal_1,
+			   msa_vertical_1, bnd_hsync2vsync, hsync2vsync_pol_ctrl,
+			   misc0 = 0, misc1 = 0, pxl_repr,
+			   front_porch, back_porch, msa_h0, msa_v0, hsync, vsync,
+			   dp_vertical_1;
+	struct drm_display_mode *mode;
+	u32 bpp, bpc, pxlfmt;
+	u32 tmp;
+	u8 stream_id = mhdp->stream_id;
+
+	mode = &bridge->encoder->crtc->state->mode;
+
+	pxlfmt = mhdp->display_fmt.color_format;
+	bpc = mhdp->display_fmt.bpc;
+
+	/* if YCBCR supported and stream not SD, use ITU709 */
+	/* FIXME: handle ITU version with YCBCR420 when supported */
+	if ((pxlfmt == DRM_COLOR_FORMAT_YCRCB444 ||
+	     pxlfmt == DRM_COLOR_FORMAT_YCRCB422) && mode->crtc_vdisplay >= 720)
+		misc0 = DP_YCBCR_COEFFICIENTS_ITU709;
+
+	bpp = cdns_mhdp_get_bpp(&mhdp->display_fmt);
+
+	switch (pxlfmt) {
+	case DRM_COLOR_FORMAT_RGB444:
+		pxl_repr = CDNS_DP_FRAMER_RGB << CDNS_DP_FRAMER_PXL_FORMAT;
+		misc0 |= DP_COLOR_FORMAT_RGB;
+		break;
+	case DRM_COLOR_FORMAT_YCRCB444:
+		pxl_repr = CDNS_DP_FRAMER_YCBCR444 << CDNS_DP_FRAMER_PXL_FORMAT;
+		misc0 |= DP_COLOR_FORMAT_YCbCr444 | DP_TEST_DYNAMIC_RANGE_CEA;
+		break;
+	case DRM_COLOR_FORMAT_YCRCB422:
+		pxl_repr = CDNS_DP_FRAMER_YCBCR422 << CDNS_DP_FRAMER_PXL_FORMAT;
+		misc0 |= DP_COLOR_FORMAT_YCbCr422 | DP_TEST_DYNAMIC_RANGE_CEA;
+		break;
+	case DRM_COLOR_FORMAT_YCRCB420:
+		pxl_repr = CDNS_DP_FRAMER_YCBCR420 << CDNS_DP_FRAMER_PXL_FORMAT;
+		break;
+	default:
+		pxl_repr = CDNS_DP_FRAMER_Y_ONLY << CDNS_DP_FRAMER_PXL_FORMAT;
+	}
+
+	switch (bpc) {
+	case 6:
+		misc0 |= DP_TEST_BIT_DEPTH_6;
+		pxl_repr |= CDNS_DP_FRAMER_6_BPC;
+		break;
+	case 8:
+		misc0 |= DP_TEST_BIT_DEPTH_8;
+		pxl_repr |= CDNS_DP_FRAMER_8_BPC;
+		break;
+	case 10:
+		misc0 |= DP_TEST_BIT_DEPTH_10;
+		pxl_repr |= CDNS_DP_FRAMER_10_BPC;
+		break;
+	case 12:
+		misc0 |= DP_TEST_BIT_DEPTH_12;
+		pxl_repr |= CDNS_DP_FRAMER_12_BPC;
+		break;
+	case 16:
+		misc0 |= DP_TEST_BIT_DEPTH_16;
+		pxl_repr |= CDNS_DP_FRAMER_16_BPC;
+		break;
+	}
+
+	bnd_hsync2vsync = CDNS_IP_BYPASS_V_INTERFACE;
+	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
+		bnd_hsync2vsync |= CDNS_IP_DET_INTERLACE_FORMAT;
+
+	cdns_mhdp_reg_write(mhdp, CDNS_BND_HSYNC2VSYNC(stream_id),
+			    bnd_hsync2vsync);
+
+	hsync2vsync_pol_ctrl = 0;
+	if (mode->flags & DRM_MODE_FLAG_NHSYNC)
+		hsync2vsync_pol_ctrl |= CDNS_H2V_HSYNC_POL_ACTIVE_LOW;
+	if (mode->flags & DRM_MODE_FLAG_NVSYNC)
+		hsync2vsync_pol_ctrl |= CDNS_H2V_VSYNC_POL_ACTIVE_LOW;
+	cdns_mhdp_reg_write(mhdp, CDNS_HSYNC2VSYNC_POL_CTRL(stream_id),
+			    hsync2vsync_pol_ctrl);
+
+	cdns_mhdp_reg_write(mhdp, CDNS_DP_FRAMER_PXL_REPR(stream_id), pxl_repr);
+
+	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
+		dp_framer_sp |= CDNS_DP_FRAMER_INTERLACE;
+	if (mode->flags & DRM_MODE_FLAG_NHSYNC)
+		dp_framer_sp |= CDNS_DP_FRAMER_HSYNC_POL_LOW;
+	if (mode->flags & DRM_MODE_FLAG_NVSYNC)
+		dp_framer_sp |= CDNS_DP_FRAMER_VSYNC_POL_LOW;
+	cdns_mhdp_reg_write(mhdp, CDNS_DP_FRAMER_SP(stream_id), dp_framer_sp);
+
+	front_porch = mode->crtc_hsync_start - mode->crtc_hdisplay;
+	back_porch = mode->crtc_htotal - mode->crtc_hsync_end;
+	cdns_mhdp_reg_write(mhdp, CDNS_DP_FRONT_BACK_PORCH(stream_id),
+			    CDNS_DP_FRONT_PORCH(front_porch) |
+			    CDNS_DP_BACK_PORCH(back_porch));
+
+	cdns_mhdp_reg_write(mhdp, CDNS_DP_BYTE_COUNT(stream_id),
+			    mode->crtc_hdisplay * bpp / 8);
+
+	msa_h0 = mode->crtc_htotal - mode->crtc_hsync_start;
+	cdns_mhdp_reg_write(mhdp, CDNS_DP_MSA_HORIZONTAL_0(stream_id),
+			    CDNS_DP_MSAH0_H_TOTAL(mode->crtc_htotal) |
+			    CDNS_DP_MSAH0_HSYNC_START(msa_h0));
+
+	hsync = mode->crtc_hsync_end - mode->crtc_hsync_start;
+	msa_horizontal_1 = CDNS_DP_MSAH1_HSYNC_WIDTH(hsync) |
+			   CDNS_DP_MSAH1_HDISP_WIDTH(mode->crtc_hdisplay);
+	if (mode->flags & DRM_MODE_FLAG_NHSYNC)
+		msa_horizontal_1 |= CDNS_DP_MSAH1_HSYNC_POL_LOW;
+	cdns_mhdp_reg_write(mhdp, CDNS_DP_MSA_HORIZONTAL_1(stream_id),
+			    msa_horizontal_1);
+
+	msa_v0 = mode->crtc_vtotal - mode->crtc_vsync_start;
+	cdns_mhdp_reg_write(mhdp, CDNS_DP_MSA_VERTICAL_0(stream_id),
+			    CDNS_DP_MSAV0_V_TOTAL(mode->crtc_vtotal) |
+			    CDNS_DP_MSAV0_VSYNC_START(msa_v0));
+
+	vsync = mode->crtc_vsync_end - mode->crtc_vsync_start;
+	msa_vertical_1 = CDNS_DP_MSAV1_VSYNC_WIDTH(vsync) |
+			 CDNS_DP_MSAV1_VDISP_WIDTH(mode->crtc_vdisplay);
+	if (mode->flags & DRM_MODE_FLAG_NVSYNC)
+		msa_vertical_1 |= CDNS_DP_MSAV1_VSYNC_POL_LOW;
+	cdns_mhdp_reg_write(mhdp, CDNS_DP_MSA_VERTICAL_1(stream_id),
+			    msa_vertical_1);
+
+	if ((mode->flags & DRM_MODE_FLAG_INTERLACE) &&
+	    mode->crtc_vtotal % 2 == 0)
+		misc1 = DP_TEST_INTERLACED;
+	if (mhdp->display_fmt.y_only)
+		misc1 |= CDNS_DP_TEST_COLOR_FORMAT_RAW_Y_ONLY;
+	/* FIXME: use VSC SDP for Y420 */
+	/* FIXME: (CDNS) no code for Y420 in bare metal test */
+	if (pxlfmt == DRM_COLOR_FORMAT_YCRCB420)
+		misc1 = CDNS_DP_TEST_VSC_SDP;
+
+	cdns_mhdp_reg_write(mhdp, CDNS_DP_MSA_MISC(stream_id),
+			    misc0 | (misc1 << 8));
+
+	cdns_mhdp_reg_write(mhdp, CDNS_DP_HORIZONTAL(stream_id),
+			    CDNS_DP_H_HSYNC_WIDTH(hsync) |
+			    CDNS_DP_H_H_TOTAL(mode->crtc_hdisplay));
+
+	cdns_mhdp_reg_write(mhdp, CDNS_DP_VERTICAL_0(stream_id),
+			    CDNS_DP_V0_VHEIGHT(mode->crtc_vdisplay) |
+			    CDNS_DP_V0_VSTART(msa_v0));
+
+	dp_vertical_1 = CDNS_DP_V1_VTOTAL(mode->crtc_vtotal);
+	if ((mode->flags & DRM_MODE_FLAG_INTERLACE) &&
+	    mode->crtc_vtotal % 2 == 0)
+		dp_vertical_1 |= CDNS_DP_V1_VTOTAL_EVEN;
+
+	cdns_mhdp_reg_write(mhdp, CDNS_DP_VERTICAL_1(stream_id), dp_vertical_1);
+
+	cdns_mhdp_reg_write_bit(mhdp, CDNS_DP_VB_ID(stream_id), 2, 1,
+				(mode->flags & DRM_MODE_FLAG_INTERLACE) ?
+				CDNS_DP_VB_ID_INTERLACED : 0);
+
+
+	cdns_mhdp_reg_read(mhdp, CDNS_DP_FRAMER_GLOBAL_CONFIG, &tmp);
+	tmp |= CDNS_DP_FRAMER_EN;
+	tmp &= ~CDNS_DP_NO_VIDEO_MODE;
+	cdns_mhdp_reg_write(mhdp, CDNS_DP_FRAMER_GLOBAL_CONFIG, tmp);
+}
+
+void cdns_mhdp_enable(struct drm_bridge *bridge)
+{
+	struct cdns_mhdp_device *mhdp = bridge_to_mhdp(bridge);
+	u32 resp;
+	u32 reg32;
+
+	dev_dbg(mhdp->dev, "bridge enable\n");
+
+	if (mhdp->ops && mhdp->ops->enable)
+		mhdp->ops->enable(mhdp);
+
+	/* Enable VIF clock for stream 0 */
+	cdns_mhdp_reg_read(mhdp, CDNS_DPTX_CAR, &resp);
+	cdns_mhdp_reg_write(mhdp, CDNS_DPTX_CAR,
+			    resp | CDNS_VIF_CLK_EN | CDNS_VIF_CLK_RSTN);
+
+	if (!mhdp->link_up)
+		cdns_mhdp_link_up(mhdp);
+
+	cdns_mhdp_sst_enable(bridge);
+
+	/* Reset PHY config */
+	reg32 = CDNS_PHY_COMMON_CONFIG | CDNS_PHY_TRAINING_TYPE(1);
+	if (!mhdp->host.scrambler)
+		reg32 |= CDNS_PHY_SCRAMBLER_BYPASS;
+	cdns_mhdp_reg_write(mhdp, CDNS_DPTX_PHY_CONFIG, reg32);
+}
+
+static void cdns_mhdp_detach(struct drm_bridge *bridge)
+{
+	struct cdns_mhdp_device *mhdp = bridge_to_mhdp(bridge);
+
+	dev_dbg(mhdp->dev, "%s\n", __func__);
+
+	spin_lock(&mhdp->start_lock);
+
+	mhdp->bridge_attached = false;
+
+	spin_unlock(&mhdp->start_lock);
+
+	writel(~0, mhdp->regs + CDNS_APB_INT_MASK);
+	writel(~0, mhdp->regs + CDNS_MB_INT_MASK);
+}
+
+static const struct drm_bridge_funcs cdns_mhdp_bridge_funcs = {
+	.enable = cdns_mhdp_enable,
+	.disable = cdns_mhdp_disable,
+	.attach = cdns_mhdp_attach,
+	.detach = cdns_mhdp_detach,
+};
+
+static int mhdp_probe(struct platform_device *pdev)
+{
+	const struct of_device_id *match;
+	struct resource *regs;
+	struct cdns_mhdp_device *mhdp;
+	struct clk *clk;
+	int ret;
+	unsigned long rate;
+	int irq;
+	u32 lanes_prop;
+
+	mhdp = devm_kzalloc(&pdev->dev, sizeof(struct cdns_mhdp_device),
+			    GFP_KERNEL);
+	if (!mhdp)
+		return -ENOMEM;
+
+	clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(clk)) {
+		dev_err(&pdev->dev, "couldn't get clk: %ld\n", PTR_ERR(clk));
+		return PTR_ERR(clk);
+	}
+
+	mhdp->clk = clk;
+	mhdp->dev = &pdev->dev;
+	mutex_init(&mhdp->mbox_mutex);
+	spin_lock_init(&mhdp->start_lock);
+	dev_set_drvdata(&pdev->dev, mhdp);
+
+	drm_dp_aux_init(&mhdp->aux);
+	mhdp->aux.dev = &pdev->dev;
+	mhdp->aux.transfer = mhdp_transfer;
+
+	regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	mhdp->regs = devm_ioremap_resource(&pdev->dev, regs);
+	if (IS_ERR(mhdp->regs))
+		return PTR_ERR(mhdp->regs);
+
+	mhdp->phy = devm_phy_get(&pdev->dev, "dpphy");
+	if (IS_ERR(mhdp->phy)) {
+		dev_err(&pdev->dev, "no PHY configured\n");
+		return PTR_ERR(mhdp->phy);
+	}
+
+	platform_set_drvdata(pdev, mhdp);
+
+	clk_prepare_enable(clk);
+
+	match = of_match_device(mhdp_ids, &pdev->dev);
+	if (!match)
+		return -ENODEV;
+	mhdp->ops = (struct mhdp_platform_ops *)match->data;
+
+	pm_runtime_enable(&pdev->dev);
+	ret = pm_runtime_get_sync(&pdev->dev);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "pm_runtime_get_sync failed\n");
+		pm_runtime_disable(&pdev->dev);
+		goto clk_disable;
+	}
+
+	if (mhdp->ops && mhdp->ops->init) {
+		ret = mhdp->ops->init(mhdp);
+		if (ret != 0) {
+			dev_err(&pdev->dev, "MHDP platform initialization failed: %d\n",
+				ret);
+			goto runtime_put;
+		}
+	}
+
+	rate = clk_get_rate(clk);
+	writel(rate % 1000000, mhdp->regs + CDNS_SW_CLK_L);
+	writel(rate / 1000000, mhdp->regs + CDNS_SW_CLK_H);
+
+	dev_dbg(&pdev->dev, "func clk rate %lu Hz\n", rate);
+
+	writel(~0, mhdp->regs + CDNS_MB_INT_MASK);
+	writel(~0, mhdp->regs + CDNS_APB_INT_MASK);
+
+	irq = platform_get_irq(pdev, 0);
+	ret = devm_request_threaded_irq(mhdp->dev, irq, NULL, mhdp_irq_handler,
+					IRQF_ONESHOT, "mhdp8546", mhdp);
+	if (ret) {
+		dev_err(&pdev->dev, "cannot install IRQ %d\n", irq);
+		ret = -EIO;
+		goto j721e_fini;
+	}
+
+	/* Read source capabilities, based on PHY's device tree properties. */
+	ret = device_property_read_u32(&(mhdp->phy->dev), "num_lanes",
+				       &(lanes_prop));
+	if (ret)
+		mhdp->host.lanes_cnt = CDNS_LANE_4;
+	else
+		mhdp->host.lanes_cnt = lanes_prop;
+
+	ret = device_property_read_u32(&(mhdp->phy->dev), "max_bit_rate",
+				       &(mhdp->host.link_rate));
+	if (ret)
+		mhdp->host.link_rate = drm_dp_bw_code_to_link_rate(DP_LINK_BW_8_1);
+	else
+		/* PHY uses Mb/s, DRM uses tens of kb/s. */
+		mhdp->host.link_rate *= 100;
+
+	mhdp->host.volt_swing = CDNS_VOLT_SWING(3);
+	mhdp->host.pre_emphasis = CDNS_PRE_EMPHASIS(3);
+	mhdp->host.pattern_supp = CDNS_SUPPORT_TPS(1) |
+				  CDNS_SUPPORT_TPS(2) | CDNS_SUPPORT_TPS(3) |
+				  CDNS_SUPPORT_TPS(4);
+	mhdp->host.lane_mapping = CDNS_LANE_MAPPING_NORMAL;
+	mhdp->host.fast_link = false;
+	mhdp->host.enhanced = true;
+	mhdp->host.scrambler = true;
+	mhdp->host.ssc = false;
+
+	/* The only currently supported format */
+	mhdp->display_fmt.y_only = false;
+	mhdp->display_fmt.color_format = DRM_COLOR_FORMAT_RGB444;
+	mhdp->display_fmt.bpc = 8;
+
+	mhdp->bridge.of_node = pdev->dev.of_node;
+	mhdp->bridge.funcs = &cdns_mhdp_bridge_funcs;
+
+	ret = phy_init(mhdp->phy);
+	if (ret) {
+		dev_err(mhdp->dev, "Failed to initialize PHY: %d\n", ret);
+		goto runtime_put;
+	}
+
+	drm_bridge_add(&mhdp->bridge);
+
+	ret = load_firmware(mhdp);
+	if (ret)
+		goto phy_exit;
+
+	return 0;
+
+phy_exit:
+	phy_exit(mhdp->phy);
+j721e_fini:
+	cdns_mhdp_j721e_fini(mhdp);
+runtime_put:
+	pm_runtime_put_sync(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+clk_disable:
+	clk_disable_unprepare(mhdp->clk);
+
+	return ret;
+}
+
+MODULE_FIRMWARE(FW_NAME);
+
+static int mhdp_remove(struct platform_device *pdev)
+{
+	struct cdns_mhdp_device *mhdp = dev_get_drvdata(&pdev->dev);
+	unsigned int timeout = 10;
+	bool stop_fw = false;
+	int ret = 0;
+
+	if (mhdp->ops && mhdp->ops->exit)
+		mhdp->ops->exit(mhdp);
+
+	drm_bridge_remove(&mhdp->bridge);
+
+wait_loading:
+	spin_lock(&mhdp->start_lock);
+	if (mhdp->hw_state == MHDP_HW_LOADING && timeout-- > 0) {
+		spin_unlock(&mhdp->start_lock);
+		msleep(100);
+		goto wait_loading;
+	} else if (mhdp->hw_state == MHDP_HW_READY) {
+		stop_fw = true;
+		timeout = 1; /* We were succesful even if counter reached 0 */
+	}
+	mhdp->hw_state = MHDP_HW_STOPPED;
+	spin_unlock(&mhdp->start_lock);
+
+	if (timeout == 0)
+		dev_err(mhdp->dev, "%s: Timeout waiting for fw loading\n",
+			__func__);
+
+	if (stop_fw) {
+		ret = cdns_mhdp_set_firmware_active(mhdp, false);
+		if (ret)
+			dev_err(mhdp->dev, "%s: De-activate FW failed: %d\n",
+				__func__, ret);
+	}
+
+	phy_exit(mhdp->phy);
+
+	cdns_mhdp_j721e_fini(mhdp);
+
+	pm_runtime_put_sync(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+
+	clk_disable_unprepare(mhdp->clk);
+
+	/* FIXME: check for missing functions */
+
+	return ret;
+}
+
+static struct platform_driver mhdp_driver = {
+	.driver	= {
+		.name		= "cdns-mhdp",
+		.of_match_table	= of_match_ptr(mhdp_ids),
+	},
+	.probe	= mhdp_probe,
+	.remove	= mhdp_remove,
+};
+module_platform_driver(mhdp_driver);
+
+MODULE_AUTHOR("Quentin Schulz <quentin.schulz@free-electrons.com>");
+MODULE_AUTHOR("Przemyslaw Gaj <pgaj@cadence.com>");
+MODULE_AUTHOR("Damian Kos <dkos@cadence.com>");
+MODULE_AUTHOR("Piotr Sroka <piotrs@cadence.com>");
+MODULE_DESCRIPTION("Cadence MHDP DP bridge driver");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:cdns-mhdp");
diff -Naur --no-dereference a/drivers/gpu/drm/bridge/cdns-mhdp.h b/drivers/gpu/drm/bridge/cdns-mhdp.h
--- a/drivers/gpu/drm/bridge/cdns-mhdp.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/bridge/cdns-mhdp.h	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,286 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Cadence MHDP DP MST bridge driver.
+ *
+ * Copyright: 2018 Cadence Design Systems, Inc.
+ *
+ * Author: Quentin Schulz <quentin.schulz@free-electrons.com>
+ */
+
+
+#ifndef CDNS_MHDP_H
+#define CDNS_MHDP_H
+
+#include <drm/drm_bridge.h>
+#include <drm/drm_connector.h>
+#include <drm/drm_dp_helper.h>
+
+#define CDNS_APB_CFG				0x00000
+#define CDNS_APB_CTRL				(CDNS_APB_CFG + 0x00)
+#define CDNS_CPU_STALL				BIT(3)
+
+#define CDNS_MAILBOX_FULL			(CDNS_APB_CFG + 0x08)
+#define CDNS_MAILBOX_EMPTY			(CDNS_APB_CFG + 0x0c)
+#define CDNS_MAILBOX_TX_DATA			(CDNS_APB_CFG + 0x10)
+#define CDNS_MAILBOX_RX_DATA			(CDNS_APB_CFG + 0x14)
+#define CDNS_KEEP_ALIVE				(CDNS_APB_CFG + 0x18)
+#define CDNS_KEEP_ALIVE_MASK			GENMASK(7, 0)
+
+#define CDNS_MB_INT_MASK			(CDNS_APB_CFG + 0x34)
+#define CDNS_MB_INT_STATUS			(CDNS_APB_CFG + 0x38)
+
+#define CDNS_SW_CLK_L				(CDNS_APB_CFG + 0x3c)
+#define CDNS_SW_CLK_H				(CDNS_APB_CFG + 0x40)
+#define CDNS_SW_EVENT0				(CDNS_APB_CFG + 0x44)
+#define CDNS_DPTX_HPD				BIT(0)
+
+#define CDNS_SW_EVENT1				(CDNS_APB_CFG + 0x48)
+#define CDNS_SW_EVENT2				(CDNS_APB_CFG + 0x4c)
+#define CDNS_SW_EVENT3				(CDNS_APB_CFG + 0x50)
+
+#define CDNS_APB_INT_MASK			(CDNS_APB_CFG + 0x6C)
+#define CDNS_APB_INT_MASK_MAILBOX_INT		BIT(0)
+#define CDNS_APB_INT_MASK_SW_EVENT_INT		BIT(1)
+#define CDNS_APB_INT_STATUS			(CDNS_APB_CFG + 0x70)
+
+#define CDNS_DPTX_CAR				(CDNS_APB_CFG + 0x904)
+#define CDNS_VIF_CLK_EN				BIT(0)
+#define CDNS_VIF_CLK_RSTN			BIT(1)
+
+#define CDNS_SOURCE_VIDEO_IF(s)			(0x00b00 + (s * 0x20))
+#define CDNS_BND_HSYNC2VSYNC(s)			(CDNS_SOURCE_VIDEO_IF(s) + \
+						 0x00)
+#define CDNS_IP_DTCT_WIN			GENMASK(11, 0)
+#define CDNS_IP_DET_INTERLACE_FORMAT		BIT(12)
+#define CDNS_IP_BYPASS_V_INTERFACE		BIT(13)
+
+#define CDNS_HSYNC2VSYNC_POL_CTRL(s)		(CDNS_SOURCE_VIDEO_IF(s) + \
+						 0x10)
+#define CDNS_H2V_HSYNC_POL_ACTIVE_LOW		BIT(1)
+#define CDNS_H2V_VSYNC_POL_ACTIVE_LOW		BIT(2)
+
+#define CDNS_DPTX_PHY_CONFIG			0x02000
+#define CDNS_PHY_TRAINING_EN			BIT(0)
+#define CDNS_PHY_TRAINING_TYPE(x)		(((x) & GENMASK(3, 0)) << 1)
+#define CDNS_PHY_SCRAMBLER_BYPASS		BIT(5)
+#define CDNS_PHY_ENCODER_BYPASS			BIT(6)
+#define CDNS_PHY_SKEW_BYPASS			BIT(7)
+#define CDNS_PHY_TRAINING_AUTO			BIT(8)
+#define CDNS_PHY_LANE0_SKEW(x)			(((x) & GENMASK(2, 0)) << 9)
+#define CDNS_PHY_LANE1_SKEW(x)			(((x) & GENMASK(2, 0)) << 12)
+#define CDNS_PHY_LANE2_SKEW(x)			(((x) & GENMASK(2, 0)) << 15)
+#define CDNS_PHY_LANE3_SKEW(x)			(((x) & GENMASK(2, 0)) << 18)
+#define CDNS_PHY_COMMON_CONFIG			(CDNS_PHY_LANE1_SKEW(1) | \
+						CDNS_PHY_LANE2_SKEW(2) |  \
+						CDNS_PHY_LANE3_SKEW(3))
+#define CDNS_PHY_10BIT_EN			BIT(21)
+
+#define CDNS_DPTX_FRAMER			0x02200
+#define CDNS_DP_FRAMER_GLOBAL_CONFIG		(CDNS_DPTX_FRAMER + 0x00)
+#define CDNS_DP_NUM_LANES(x)			(x - 1)
+#define CDNS_DP_MST_EN				BIT(2)
+#define CDNS_DP_FRAMER_EN			BIT(3)
+#define CDNS_DP_RATE_GOVERNOR_EN		BIT(4)
+#define CDNS_DP_NO_VIDEO_MODE			BIT(5)
+#define CDNS_DP_DISABLE_PHY_RST			BIT(6)
+#define CDNS_DP_WR_FAILING_EDGE_VSYNC		BIT(7)
+
+#define CDNS_DP_FRAMER_TU			(CDNS_DPTX_FRAMER + 0x08)
+#define CDNS_DP_FRAMER_TU_SIZE(x)		(((x) & GENMASK(6, 0)) << 8)
+#define CDNS_DP_FRAMER_TU_VS(x)			((x) & GENMASK(5, 0))
+#define CDNS_DP_FRAMER_TU_CNT_RST_EN		BIT(15)
+
+#define CDNS_DPTX_STREAM(s)			(0x03000 + s * 0x80)
+#define CDNS_DP_MSA_HORIZONTAL_0(s)		(CDNS_DPTX_STREAM(s) + 0x00)
+#define CDNS_DP_MSAH0_H_TOTAL(x)		(x)
+#define CDNS_DP_MSAH0_HSYNC_START(x)		((x) << 16)
+
+#define CDNS_DP_MSA_HORIZONTAL_1(s)		(CDNS_DPTX_STREAM(s) + 0x04)
+#define CDNS_DP_MSAH1_HSYNC_WIDTH(x)		(x)
+#define CDNS_DP_MSAH1_HSYNC_POL_LOW		BIT(15)
+#define CDNS_DP_MSAH1_HDISP_WIDTH(x)		((x) << 16)
+
+#define CDNS_DP_MSA_VERTICAL_0(s)		(CDNS_DPTX_STREAM(s) + 0x08)
+#define CDNS_DP_MSAV0_V_TOTAL(x)		(x)
+#define CDNS_DP_MSAV0_VSYNC_START(x)		((x) << 16)
+
+#define CDNS_DP_MSA_VERTICAL_1(s)		(CDNS_DPTX_STREAM(s) + 0x0c)
+#define CDNS_DP_MSAV1_VSYNC_WIDTH(x)		(x)
+#define CDNS_DP_MSAV1_VSYNC_POL_LOW		BIT(15)
+#define CDNS_DP_MSAV1_VDISP_WIDTH(x)		((x) << 16)
+
+#define CDNS_DP_MSA_MISC(s)			(CDNS_DPTX_STREAM(s) + 0x10)
+#define CDNS_DP_STREAM_CONFIGs(s)		(CDNS_DPTX_STREAM(s) + 0x14)
+#define CDNS_DP_STREAM_CONFIG_2(s)		(CDNS_DPTX_STREAM(s) + 0x2c)
+#define CDNS_DP_SC2_TU_VS_DIFF(x)		((x) << 8)
+
+#define CDNS_DP_HORIZONTAL(s)			(CDNS_DPTX_STREAM(s) + 0x30)
+#define CDNS_DP_H_HSYNC_WIDTH(x)		(x)
+#define CDNS_DP_H_H_TOTAL(x)			((x) << 16)
+
+#define CDNS_DP_VERTICAL_0(s)			(CDNS_DPTX_STREAM(s) + 0x34)
+#define CDNS_DP_V0_VHEIGHT(x)			(x)
+#define CDNS_DP_V0_VSTART(x)			((x) << 16)
+
+#define CDNS_DP_VERTICAL_1(s)			(CDNS_DPTX_STREAM(s) + 0x38)
+#define CDNS_DP_V1_VTOTAL(x)			(x)
+#define CDNS_DP_V1_VTOTAL_EVEN			BIT(16)
+
+#define CDNS_DP_FRAMER_PXL_REPR(s)		(CDNS_DPTX_STREAM(s) + 0x4c)
+#define CDNS_DP_FRAMER_6_BPC			BIT(0)
+#define CDNS_DP_FRAMER_8_BPC			BIT(1)
+#define CDNS_DP_FRAMER_10_BPC			BIT(2)
+#define CDNS_DP_FRAMER_12_BPC			BIT(3)
+#define CDNS_DP_FRAMER_16_BPC			BIT(4)
+#define CDNS_DP_FRAMER_PXL_FORMAT		0x8
+#define CDNS_DP_FRAMER_RGB			BIT(0)
+#define CDNS_DP_FRAMER_YCBCR444			BIT(1)
+#define CDNS_DP_FRAMER_YCBCR422			BIT(2)
+#define CDNS_DP_FRAMER_YCBCR420			BIT(3)
+#define CDNS_DP_FRAMER_Y_ONLY			BIT(4)
+
+#define CDNS_DP_FRAMER_SP(s)			(CDNS_DPTX_STREAM(s) + 0x50)
+#define CDNS_DP_FRAMER_VSYNC_POL_LOW		BIT(0)
+#define CDNS_DP_FRAMER_HSYNC_POL_LOW		BIT(1)
+#define CDNS_DP_FRAMER_INTERLACE		BIT(2)
+
+#define CDNS_DP_LINE_THRESH(s)			(CDNS_DPTX_STREAM(s) + 0x64)
+#define CDNS_DP_ACTIVE_LINE_THRESH(x)		(x)
+
+#define CDNS_DP_VB_ID(s)			(CDNS_DPTX_STREAM(s) + 0x68)
+#define CDNS_DP_VB_ID_INTERLACED		BIT(2)
+#define CDNS_DP_VB_ID_COMPRESSED		BIT(6)
+
+#define CDNS_DP_FRONT_BACK_PORCH(s)		(CDNS_DPTX_STREAM(s) + 0x78)
+#define CDNS_DP_BACK_PORCH(x)			(x)
+#define CDNS_DP_FRONT_PORCH(x)			((x) << 16)
+
+#define CDNS_DP_BYTE_COUNT(s)			(CDNS_DPTX_STREAM(s) + 0x7c)
+#define CDNS_DP_BYTE_COUNT_BYTES_IN_CHUNK_SHIFT	16
+
+#define CDNS_DP_MST_STREAM_CONFIG(s)		(CDNS_DPTX_STREAM(s) + 0x14)
+#define CDNS_DP_MST_STRM_CFG_STREAM_EN		BIT(0)
+#define CDNS_DP_MST_STRM_CFG_NO_VIDEO		BIT(1)
+
+#define CDNS_DP_MST_SLOT_ALLOCATE(s)		(CDNS_DPTX_STREAM(s) + 0x44)
+#define CDNS_DP_S_ALLOC_START_SLOT(x)		(x)
+#define CDNS_DP_S_ALLOC_END_SLOT(x)		((x) << 8)
+
+#define CDNS_DP_RATE_GOVERNING(s)		(CDNS_DPTX_STREAM(s) + 0x48)
+#define CDNS_DP_RG_TARG_AV_SLOTS_Y(x)		(x)
+#define CDNS_DP_RG_TARG_AV_SLOTS_X(x)		(x << 4)
+#define CDNS_DP_RG_ENABLE			BIT(10)
+
+#define CDNS_DP_MTPH_CONTROL			0x2264
+#define CDNS_DP_MTPH_ECF_EN			BIT(0)
+#define CDNS_DP_MTPH_ACT_EN			BIT(1)
+#define CDNS_DP_MTPH_LVP_EN			BIT(2)
+
+#define CDNS_DP_MTPH_STATUS			0x226C
+#define CDNS_DP_MTPH_ACT_STATUS			BIT(0)
+
+
+#define CDNS_DPTX_GLOBAL			0x02300
+#define CDNS_DP_LANE_EN				(CDNS_DPTX_GLOBAL + 0x00)
+#define CDNS_DP_LANE_EN_LANES(x)		GENMASK(x - 1, 0)
+#define CDNS_DP_ENHNCD				(CDNS_DPTX_GLOBAL + 0x04)
+
+struct cdns_mhdp_host {
+	unsigned int link_rate;
+	u8 lanes_cnt;
+	u8 volt_swing;
+	u8 pre_emphasis;
+	u8 pattern_supp;
+	u8 lane_mapping;
+	u8 fast_link : 1;
+	u8 enhanced : 1;
+	u8 scrambler : 1;
+	u8 ssc : 1;
+};
+
+struct cdns_mhdp_sink {
+	unsigned int link_rate;
+	u8 lanes_cnt;
+	u8 pattern_supp;
+	u8 fast_link : 1;
+	u8 enhanced : 1;
+	u8 ssc : 1;
+};
+
+struct cdns_mhdp_display_fmt {
+	u32 color_format;
+	u32 bpc;
+	u8 y_only : 1;
+};
+
+/*
+ * These enums present MHDP hw initialization state
+ * Legal state transitions are:
+ * MHDP_HW_INACTIVE <-> MHDP_HW_LOADING -> MHDP_HW_READY
+ *        |                                     |
+ *        '----------> MHDP_HW_STOPPED <--------'
+ */
+enum mhdp_hw_state { MHDP_HW_INACTIVE = 0, /* HW not initialized */
+		     MHDP_HW_LOADING,	   /* HW initialization in progress */
+		     MHDP_HW_READY,	   /* HW ready, FW active*/
+		     MHDP_HW_STOPPED };	   /* Driver removal FW to be stopped */
+
+struct cdns_mhdp_device;
+
+struct mhdp_platform_ops {
+	int (*init)(struct cdns_mhdp_device *mhdp);
+	void (*exit)(struct cdns_mhdp_device *mhdp);
+	void (*enable)(struct cdns_mhdp_device *mhdp);
+	void (*disable)(struct cdns_mhdp_device *mhdp);
+};
+
+struct cdns_mhdp_device {
+	void __iomem *regs;
+	void __iomem *j721e_regs;
+
+	struct device *dev;
+	struct clk *clk;
+	struct phy *phy;
+
+	const struct mhdp_platform_ops *ops;
+
+	/* This is to protect mailbox communications with the firmware */
+	struct mutex mbox_mutex;
+
+	struct drm_connector connector;
+	struct drm_bridge bridge;
+
+	struct drm_dp_link link;
+	struct drm_dp_aux aux;
+
+	struct cdns_mhdp_host host;
+	struct cdns_mhdp_sink sink;
+	struct cdns_mhdp_display_fmt display_fmt;
+	s8 stream_id;
+
+	u8 link_up : 1;
+	u8 plugged : 1;
+
+	/*
+	 * "start_lock" protects the access to bridge_attached and
+	 * hw_state data members that control the delayed firmware
+	 * loading and attaching the bridge. They are accessed from
+	 * both the DRM core and mhdp_fw_cb(). In most cases just
+	 * protecting the data members is enough, but the irq mask
+	 * setting needs to be protected when enabling the FW.
+	 */
+	spinlock_t start_lock;
+	u8 bridge_attached : 1;
+	enum mhdp_hw_state hw_state;
+};
+
+#define connector_to_mhdp(x) container_of(x, struct cdns_mhdp_device, connector)
+#define bridge_to_mhdp(x) container_of(x, struct cdns_mhdp_device, bridge)
+
+#define CDNS_MHDP_MAX_STREAMS   4
+
+u32 cdns_mhdp_get_bpp(struct cdns_mhdp_display_fmt *fmt);
+void cdns_mhdp_configure_video(struct drm_bridge *bridge);
+void cdns_mhdp_enable(struct drm_bridge *bridge);
+
+#endif
diff -Naur --no-dereference a/drivers/gpu/drm/bridge/cdns-mhdp-j721e.c b/drivers/gpu/drm/bridge/cdns-mhdp-j721e.c
--- a/drivers/gpu/drm/bridge/cdns-mhdp-j721e.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/bridge/cdns-mhdp-j721e.c	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * TI j721e Cadence MHDP DP wrapper
+ *
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Jyri Sarha <jsarha@ti.com
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/device.h>
+#include <linux/io.h>
+
+#include "cdns-mhdp-j721e.h"
+
+#define	REVISION			0x00
+#define	DPTX_IPCFG			0x04
+#define	ECC_MEM_CFG			0x08
+#define	DPTX_DSC_CFG			0x0c
+#define	DPTX_SRC_CFG			0x10
+#define	DPTX_VIF_SECURE_MODE_CFG	0x14
+#define	DPTX_VIF_CONN_STATUS		0x18
+#define	PHY_CLK_STATUS			0x1c
+
+#define DPTX_SRC_AIF_EN			BIT(16)
+#define DPTX_SRC_VIF_3_IN30B		BIT(11)
+#define DPTX_SRC_VIF_2_IN30B		BIT(10)
+#define DPTX_SRC_VIF_1_IN30B		BIT(9)
+#define DPTX_SRC_VIF_0_IN30B		BIT(8)
+#define DPTX_SRC_VIF_3_SEL_DPI5		BIT(7)
+#define DPTX_SRC_VIF_3_SEL_DPI3		0
+#define DPTX_SRC_VIF_2_SEL_DPI4		BIT(6)
+#define DPTX_SRC_VIF_2_SEL_DPI2		0
+#define DPTX_SRC_VIF_1_SEL_DPI3		BIT(5)
+#define DPTX_SRC_VIF_1_SEL_DPI1		0
+#define DPTX_SRC_VIF_0_SEL_DPI2		BIT(4)
+#define DPTX_SRC_VIF_0_SEL_DPI0		0
+#define DPTX_SRC_VIF_3_EN		BIT(3)
+#define DPTX_SRC_VIF_2_EN		BIT(2)
+#define DPTX_SRC_VIF_1_EN		BIT(1)
+#define DPTX_SRC_VIF_0_EN		BIT(0)
+
+/* TODO turn DPTX_IPCFG fw_mem_clk_en at pm_runtime_suspend. */
+
+int cdns_mhdp_j721e_init(struct cdns_mhdp_device *mhdp)
+{
+	struct platform_device *pdev = to_platform_device(mhdp->dev);
+	struct resource *regs;
+
+	regs = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+	mhdp->j721e_regs = devm_ioremap_resource(&pdev->dev, regs);
+	if (IS_ERR(mhdp->j721e_regs))
+		return PTR_ERR(mhdp->j721e_regs);
+
+	return 0;
+}
+
+void cdns_mhdp_j721e_fini(struct cdns_mhdp_device *mhdp)
+{
+}
+
+void cdns_mhdp_j721e_enable(struct cdns_mhdp_device *mhdp)
+{
+	/*
+	 * Eneble VIF_0 and select DPI2 as its input. DSS0 DPI0 is connected
+	 * to eDP DPI2. This is the only supported SST configuration on
+	 * J721E.
+	 */
+	writel(DPTX_SRC_VIF_0_EN | DPTX_SRC_VIF_0_SEL_DPI2,
+	       mhdp->j721e_regs + DPTX_SRC_CFG);
+}
+
+void cdns_mhdp_j721e_disable(struct cdns_mhdp_device *mhdp)
+{
+	/* HACK: always disable everything and put everything to defaults  */
+	writel(0, mhdp->j721e_regs + DPTX_DSC_CFG);
+}
diff -Naur --no-dereference a/drivers/gpu/drm/bridge/cdns-mhdp-j721e.h b/drivers/gpu/drm/bridge/cdns-mhdp-j721e.h
--- a/drivers/gpu/drm/bridge/cdns-mhdp-j721e.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/bridge/cdns-mhdp-j721e.h	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,55 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * TI j721e Cadence MHDP DP wrapper
+ *
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Jyri Sarha <jsarha@ti.com
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef CDNS_MHDP_J721E_H
+#define CDNS_MHDP_J721E_H
+
+#include <linux/platform_device.h>
+#include "cdns-mhdp.h"
+
+struct cdns_mhdp_j721e_wrap;
+
+#ifdef CONFIG_DRM_CDNS_MHDP_J721E
+
+int cdns_mhdp_j721e_init(struct cdns_mhdp_device *mhdp);
+
+void cdns_mhdp_j721e_fini(struct cdns_mhdp_device *mhdp);
+
+void cdns_mhdp_j721e_enable(struct cdns_mhdp_device *mhdp);
+
+void cdns_mhdp_j721e_disable(struct cdns_mhdp_device *mhdp);
+
+#else
+
+static inline
+int cdns_mhdp_j721e_init(struct cdns_mhdp_device *mhdp)
+{
+	return 0;
+}
+
+static inline
+void cdns_mhdp_j721e_fini(struct cdns_mhdp_device *mhdp)
+{
+}
+
+static inline
+void cdns_mhdp_j721e_sst_enable(struct cdns_mhdp_device *mhdp);
+{
+}
+
+static inline
+void cdns_mhdp_j721e_sst_disable(struct cdns_mhdp_device *mhdp)
+{
+}
+#endif /* CONFIG_DRM_CDNS_MHDP_J721E */
+
+#endif /* !CDNS_MHDP_J721E_H */
diff -Naur --no-dereference a/drivers/gpu/drm/bridge/dumb-vga-dac.c b/drivers/gpu/drm/bridge/dumb-vga-dac.c
--- a/drivers/gpu/drm/bridge/dumb-vga-dac.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/bridge/dumb-vga-dac.c	2020-06-19 21:00:31.408227715 -0400
@@ -234,7 +234,7 @@
  */
 static const struct drm_bridge_timings default_dac_timings = {
 	/* Timing specifications, datasheet page 7 */
-	.sampling_edge = DRM_BUS_FLAG_PIXDATA_POSEDGE,
+	.input_bus_flags = DRM_BUS_FLAG_PIXDATA_SAMPLE_POSEDGE,
 	.setup_time_ps = 500,
 	.hold_time_ps = 1500,
 };
@@ -245,7 +245,7 @@
  */
 static const struct drm_bridge_timings ti_ths8134_dac_timings = {
 	/* From timing diagram, datasheet page 9 */
-	.sampling_edge = DRM_BUS_FLAG_PIXDATA_POSEDGE,
+	.input_bus_flags = DRM_BUS_FLAG_PIXDATA_SAMPLE_POSEDGE,
 	/* From datasheet, page 12 */
 	.setup_time_ps = 3000,
 	/* I guess this means latched input */
@@ -258,7 +258,7 @@
  */
 static const struct drm_bridge_timings ti_ths8135_dac_timings = {
 	/* From timing diagram, datasheet page 14 */
-	.sampling_edge = DRM_BUS_FLAG_PIXDATA_POSEDGE,
+	.input_bus_flags = DRM_BUS_FLAG_PIXDATA_SAMPLE_POSEDGE,
 	/* From datasheet, page 16 */
 	.setup_time_ps = 2000,
 	.hold_time_ps = 500,
diff -Naur --no-dereference a/drivers/gpu/drm/bridge/Kconfig b/drivers/gpu/drm/bridge/Kconfig
--- a/drivers/gpu/drm/bridge/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/bridge/Kconfig	2020-06-19 21:00:31.408227715 -0400
@@ -35,6 +35,25 @@
 	  Support Cadence DPI to DSI bridge. This is an internal
 	  bridge and is meant to be directly embedded in a SoC.
 
+config DRM_CDNS_MHDP
+	tristate "Cadence DPI/DP bridge"
+	select DRM_KMS_HELPER
+	select DRM_PANEL_BRIDGE
+	depends on OF
+	help
+	  Support Cadence DPI to DP bridge. This is an internal
+	  bridge and is meant to be directly embedded in a SoC.
+
+if DRM_CDNS_MHDP
+
+config DRM_CDNS_MHDP_J721E
+	bool "J721E Cadence DPI/DP wrapper support"
+	default y
+	help
+	  Support J721E Cadence DPI/DP wrapper
+
+endif
+
 config DRM_DUMB_VGA_DAC
 	tristate "Dumb VGA DAC Bridge support"
 	depends on OF
@@ -96,6 +115,7 @@
 	depends on OF
 	select DRM_KMS_HELPER
 	select REGMAP_I2C
+	select I2C_MUX
 	---help---
 	  Silicon Image sii902x bridge chip driver.
 
@@ -122,6 +142,16 @@
 	---help---
 	  Toshiba TC358767 eDP bridge chip driver.
 
+config DRM_TOSHIBA_TC358768
+	tristate "Toshiba TC358768 MIPI DSI bridge"
+	depends on OF
+	select DRM_KMS_HELPER
+	select REGMAP_I2C
+	select DRM_PANEL
+	select DRM_MIPI_DSI
+	---help---
+	  Toshiba TC358768AXBG/TC358778XBG DSI bridge chip driver.
+
 config DRM_TI_TFP410
 	tristate "TI TFP410 DVI/HDMI bridge"
 	depends on OF
diff -Naur --no-dereference a/drivers/gpu/drm/bridge/Makefile b/drivers/gpu/drm/bridge/Makefile
--- a/drivers/gpu/drm/bridge/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/bridge/Makefile	2020-06-19 21:00:31.408227715 -0400
@@ -11,7 +11,14 @@
 obj-$(CONFIG_DRM_SII9234) += sii9234.o
 obj-$(CONFIG_DRM_THINE_THC63LVD1024) += thc63lvd1024.o
 obj-$(CONFIG_DRM_TOSHIBA_TC358767) += tc358767.o
+obj-$(CONFIG_DRM_TOSHIBA_TC358768) += tc358768.o
 obj-$(CONFIG_DRM_ANALOGIX_DP) += analogix/
 obj-$(CONFIG_DRM_I2C_ADV7511) += adv7511/
 obj-$(CONFIG_DRM_TI_TFP410) += ti-tfp410.o
+obj-$(CONFIG_DRM_CDNS_MHDP) += mhdp8546.o
 obj-y += synopsys/
+
+mhdp8546-objs := cdns-mhdp.o
+ifeq ($(CONFIG_DRM_CDNS_MHDP_J721E),y)
+	mhdp8546-objs += cdns-mhdp-j721e.o
+endif
diff -Naur --no-dereference a/drivers/gpu/drm/bridge/sii902x.c b/drivers/gpu/drm/bridge/sii902x.c
--- a/drivers/gpu/drm/bridge/sii902x.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/bridge/sii902x.c	2020-06-19 21:00:31.408227715 -0400
@@ -1,4 +1,6 @@
 /*
+ * Copyright (C) 2018 Renesas Electronics
+ *
  * Copyright (C) 2016 Atmel
  *		      Bo Shen <voice.shen@atmel.com>
  *
@@ -21,15 +23,20 @@
  */
 
 #include <linux/gpio/consumer.h>
+#include <linux/i2c-mux.h>
 #include <linux/i2c.h>
 #include <linux/module.h>
 #include <linux/regmap.h>
+#include <linux/clk.h>
 
 #include <drm/drmP.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_edid.h>
 
+#include <sound/hdmi-codec.h>
+#include <dt-bindings/sound/sii902x-audio.h>
+
 #define SII902X_TPI_VIDEO_DATA			0x0
 
 #define SII902X_TPI_PIXEL_REPETITION		0x8
@@ -71,6 +78,80 @@
 #define SII902X_AVI_POWER_STATE_MSK		GENMASK(1, 0)
 #define SII902X_AVI_POWER_STATE_D(l)		((l) & SII902X_AVI_POWER_STATE_MSK)
 
+/* Audio  */
+#define SII902X_TPI_I2S_ENABLE_MAPPING_REG	0x1f
+#define SII902X_TPI_I2S_CONFIG_FIFO0			(0 << 0)
+#define SII902X_TPI_I2S_CONFIG_FIFO1			(1 << 0)
+#define SII902X_TPI_I2S_CONFIG_FIFO2			(2 << 0)
+#define SII902X_TPI_I2S_CONFIG_FIFO3			(3 << 0)
+#define SII902X_TPI_I2S_LEFT_RIGHT_SWAP			(1 << 2)
+#define SII902X_TPI_I2S_AUTO_DOWNSAMPLE			(1 << 3)
+#define SII902X_TPI_I2S_SELECT_SD0			(0 << 4)
+#define SII902X_TPI_I2S_SELECT_SD1			(1 << 4)
+#define SII902X_TPI_I2S_SELECT_SD2			(2 << 4)
+#define SII902X_TPI_I2S_SELECT_SD3			(3 << 4)
+#define SII902X_TPI_I2S_FIFO_ENABLE			(1 << 7)
+
+#define SII902X_TPI_I2S_INPUT_CONFIG_REG	0x20
+#define SII902X_TPI_I2S_FIRST_BIT_SHIFT_YES		(0 << 0)
+#define SII902X_TPI_I2S_FIRST_BIT_SHIFT_NO		(1 << 0)
+#define SII902X_TPI_I2S_SD_DIRECTION_MSB_FIRST		(0 << 1)
+#define SII902X_TPI_I2S_SD_DIRECTION_LSB_FIRST		(1 << 1)
+#define SII902X_TPI_I2S_SD_JUSTIFY_LEFT			(0 << 2)
+#define SII902X_TPI_I2S_SD_JUSTIFY_RIGHT		(1 << 2)
+#define SII902X_TPI_I2S_WS_POLARITY_LOW			(0 << 3)
+#define SII902X_TPI_I2S_WS_POLARITY_HIGH		(1 << 3)
+#define SII902X_TPI_I2S_MCLK_MULTIPLIER_128		(0 << 4)
+#define SII902X_TPI_I2S_MCLK_MULTIPLIER_256		(1 << 4)
+#define SII902X_TPI_I2S_MCLK_MULTIPLIER_384		(2 << 4)
+#define SII902X_TPI_I2S_MCLK_MULTIPLIER_512		(3 << 4)
+#define SII902X_TPI_I2S_MCLK_MULTIPLIER_768		(4 << 4)
+#define SII902X_TPI_I2S_MCLK_MULTIPLIER_1024		(5 << 4)
+#define SII902X_TPI_I2S_MCLK_MULTIPLIER_1152		(6 << 4)
+#define SII902X_TPI_I2S_MCLK_MULTIPLIER_192		(7 << 4)
+#define SII902X_TPI_I2S_SCK_EDGE_FALLING		(0 << 7)
+#define SII902X_TPI_I2S_SCK_EDGE_RISING			(1 << 7)
+
+#define SII902X_TPI_I2S_STRM_HDR_BASE	0x21
+#define SII902X_TPI_I2S_STRM_HDR_SIZE	5
+
+#define SII902X_TPI_AUDIO_CONFIG_BYTE2_REG	0x26
+#define SII902X_TPI_AUDIO_CODING_STREAM_HEADER		(0 << 0)
+#define SII902X_TPI_AUDIO_CODING_PCM			(1 << 0)
+#define SII902X_TPI_AUDIO_CODING_AC3			(2 << 0)
+#define SII902X_TPI_AUDIO_CODING_MPEG1			(3 << 0)
+#define SII902X_TPI_AUDIO_CODING_MP3			(4 << 0)
+#define SII902X_TPI_AUDIO_CODING_MPEG2			(5 << 0)
+#define SII902X_TPI_AUDIO_CODING_AAC			(6 << 0)
+#define SII902X_TPI_AUDIO_CODING_DTS			(7 << 0)
+#define SII902X_TPI_AUDIO_CODING_ATRAC			(8 << 0)
+#define SII902X_TPI_AUDIO_MUTE_DISABLE			(0 << 4)
+#define SII902X_TPI_AUDIO_MUTE_ENABLE			(1 << 4)
+#define SII902X_TPI_AUDIO_LAYOUT_2_CHANNELS		(0 << 5)
+#define SII902X_TPI_AUDIO_LAYOUT_8_CHANNELS		(1 << 5)
+#define SII902X_TPI_AUDIO_INTERFACE_DISABLE		(0 << 6)
+#define SII902X_TPI_AUDIO_INTERFACE_SPDIF		(1 << 6)
+#define SII902X_TPI_AUDIO_INTERFACE_I2S			(2 << 6)
+
+#define SII902X_TPI_AUDIO_CONFIG_BYTE3_REG	0x27
+#define SII902X_TPI_AUDIO_CHANNEL_STREAM		(0 << 0)
+#define SII902X_TPI_AUDIO_2_CHANNEL			(1 << 0)
+#define SII902X_TPI_AUDIO_8_CHANNEL			(7 << 0)
+#define SII902X_TPI_AUDIO_FREQ_STREAM			(0 << 3)
+#define SII902X_TPI_AUDIO_FREQ_32KHZ			(1 << 3)
+#define SII902X_TPI_AUDIO_FREQ_44KHZ			(2 << 3)
+#define SII902X_TPI_AUDIO_FREQ_48KHZ			(3 << 3)
+#define SII902X_TPI_AUDIO_FREQ_88KHZ			(4 << 3)
+#define SII902X_TPI_AUDIO_FREQ_96KHZ			(5 << 3)
+#define SII902X_TPI_AUDIO_FREQ_176KHZ			(6 << 3)
+#define SII902X_TPI_AUDIO_FREQ_192KHZ			(7 << 3)
+#define SII902X_TPI_AUDIO_SAMPLE_SIZE_STREAM		(0 << 6)
+#define SII902X_TPI_AUDIO_SAMPLE_SIZE_16		(1 << 6)
+#define SII902X_TPI_AUDIO_SAMPLE_SIZE_20		(2 << 6)
+#define SII902X_TPI_AUDIO_SAMPLE_SIZE_24		(3 << 6)
+
+#define SII902X_TPI_AUDIO_CONFIG_BYTE4_REG	0x28
+
 #define SII902X_INT_ENABLE			0x3c
 #define SII902X_INT_STATUS			0x3d
 #define SII902X_HOTPLUG_EVENT			BIT(0)
@@ -78,6 +159,16 @@
 
 #define SII902X_REG_TPI_RQB			0xc7
 
+/* Indirect internal register access */
+#define SII902X_IND_SET_PAGE			0xbc
+#define SII902X_IND_OFFSET			0xbd
+#define SII902X_IND_VALUE			0xbe
+
+#define SII902X_TPI_MISC_INFOFRAME_BASE		0xbf
+#define SII902X_TPI_MISC_INFOFRAME_END		0xde
+#define SII902X_TPI_MISC_INFOFRAME_SIZE	\
+	(SII902X_TPI_MISC_INFOFRAME_END - SII902X_TPI_MISC_INFOFRAME_BASE)
+
 #define SII902X_I2C_BUS_ACQUISITION_TIMEOUT_MS	500
 
 struct sii902x {
@@ -86,8 +177,57 @@
 	struct drm_bridge bridge;
 	struct drm_connector connector;
 	struct gpio_desc *reset_gpio;
+	struct i2c_mux_core *i2cmux;
+	struct mutex mutex;
+	struct sii902x_audio {
+		struct platform_device *pdev;
+		u8 channels;
+		struct clk *mclk;
+		bool mclk_enabled;
+		u32 i2s_fifo_routing[4];
+	} audio;
 };
 
+static int sii902x_read_unlocked(struct i2c_client *i2c, u8 reg, u8 *val)
+{
+	union i2c_smbus_data data;
+	int ret;
+
+	ret = __i2c_smbus_xfer(i2c->adapter, i2c->addr, i2c->flags,
+			       I2C_SMBUS_READ, reg, I2C_SMBUS_BYTE_DATA, &data);
+
+	if (ret < 0)
+		return ret;
+
+	*val = data.byte;
+	return 0;
+}
+
+static int sii902x_write_unlocked(struct i2c_client *i2c, u8 reg, u8 val)
+{
+	union i2c_smbus_data data;
+
+	data.byte = val;
+
+	return __i2c_smbus_xfer(i2c->adapter, i2c->addr, i2c->flags,
+				I2C_SMBUS_WRITE, reg, I2C_SMBUS_BYTE_DATA,
+				&data);
+}
+
+static int sii902x_update_bits_unlocked(struct i2c_client *i2c, u8 reg, u8 mask,
+					u8 val)
+{
+	int ret;
+	u8 status;
+
+	ret = sii902x_read_unlocked(i2c, reg, &status);
+	if (ret)
+		return ret;
+	status &= ~mask;
+	status |= val & mask;
+	return sii902x_write_unlocked(i2c, reg, status);
+}
+
 static inline struct sii902x *bridge_to_sii902x(struct drm_bridge *bridge)
 {
 	return container_of(bridge, struct sii902x, bridge);
@@ -117,8 +257,12 @@
 	struct sii902x *sii902x = connector_to_sii902x(connector);
 	unsigned int status;
 
+	mutex_lock(&sii902x->mutex);
+
 	regmap_read(sii902x->regmap, SII902X_INT_STATUS, &status);
 
+	mutex_unlock(&sii902x->mutex);
+
 	return (status & SII902X_PLUGGED_STATUS) ?
 	       connector_status_connected : connector_status_disconnected;
 }
@@ -135,43 +279,19 @@
 static int sii902x_get_modes(struct drm_connector *connector)
 {
 	struct sii902x *sii902x = connector_to_sii902x(connector);
-	struct regmap *regmap = sii902x->regmap;
 	u32 bus_format = MEDIA_BUS_FMT_RGB888_1X24;
-	struct device *dev = &sii902x->i2c->dev;
-	unsigned long timeout;
-	unsigned int retries;
-	unsigned int status;
 	struct edid *edid;
-	int num = 0;
-	int ret;
+	u8 output_mode = SII902X_SYS_CTRL_OUTPUT_DVI;
+	int num = 0, ret;
 
-	ret = regmap_update_bits(regmap, SII902X_SYS_CTRL_DATA,
-				 SII902X_SYS_CTRL_DDC_BUS_REQ,
-				 SII902X_SYS_CTRL_DDC_BUS_REQ);
-	if (ret)
-		return ret;
-
-	timeout = jiffies +
-		  msecs_to_jiffies(SII902X_I2C_BUS_ACQUISITION_TIMEOUT_MS);
-	do {
-		ret = regmap_read(regmap, SII902X_SYS_CTRL_DATA, &status);
-		if (ret)
-			return ret;
-	} while (!(status & SII902X_SYS_CTRL_DDC_BUS_GRTD) &&
-		 time_before(jiffies, timeout));
-
-	if (!(status & SII902X_SYS_CTRL_DDC_BUS_GRTD)) {
-		dev_err(dev, "failed to acquire the i2c bus\n");
-		return -ETIMEDOUT;
-	}
-
-	ret = regmap_write(regmap, SII902X_SYS_CTRL_DATA, status);
-	if (ret)
-		return ret;
+	mutex_lock(&sii902x->mutex);
 
-	edid = drm_get_edid(connector, sii902x->i2c->adapter);
+	edid = drm_get_edid(connector, sii902x->i2cmux->adapter[0]);
 	drm_connector_update_edid_property(connector, edid);
 	if (edid) {
+	        if (drm_detect_hdmi_monitor(edid))
+			output_mode = SII902X_SYS_CTRL_OUTPUT_HDMI;
+
 		num = drm_add_edid_modes(connector, edid);
 		kfree(edid);
 	}
@@ -179,45 +299,19 @@
 	ret = drm_display_info_set_bus_formats(&connector->display_info,
 					       &bus_format, 1);
 	if (ret)
-		return ret;
+		goto error_out;
 
-	/*
-	 * Sometimes the I2C bus can stall after failure to use the
-	 * EDID channel. Retry a few times to see if things clear
-	 * up, else continue anyway.
-	 */
-	retries = 5;
-	do {
-		ret = regmap_read(regmap, SII902X_SYS_CTRL_DATA,
-				  &status);
-		retries--;
-	} while (ret && retries);
+	ret = regmap_update_bits(sii902x->regmap, SII902X_SYS_CTRL_DATA,
+				 SII902X_SYS_CTRL_OUTPUT_MODE, output_mode);
 	if (ret)
-		dev_err(dev, "failed to read status (%d)\n", ret);
+		goto error_out;
 
-	ret = regmap_update_bits(regmap, SII902X_SYS_CTRL_DATA,
-				 SII902X_SYS_CTRL_DDC_BUS_REQ |
-				 SII902X_SYS_CTRL_DDC_BUS_GRTD, 0);
-	if (ret)
-		return ret;
+	ret = num;
 
-	timeout = jiffies +
-		  msecs_to_jiffies(SII902X_I2C_BUS_ACQUISITION_TIMEOUT_MS);
-	do {
-		ret = regmap_read(regmap, SII902X_SYS_CTRL_DATA, &status);
-		if (ret)
-			return ret;
-	} while (status & (SII902X_SYS_CTRL_DDC_BUS_REQ |
-			   SII902X_SYS_CTRL_DDC_BUS_GRTD) &&
-		 time_before(jiffies, timeout));
+error_out:
+	mutex_unlock(&sii902x->mutex);
 
-	if (status & (SII902X_SYS_CTRL_DDC_BUS_REQ |
-		      SII902X_SYS_CTRL_DDC_BUS_GRTD)) {
-		dev_err(dev, "failed to release the i2c bus\n");
-		return -ETIMEDOUT;
-	}
-
-	return num;
+	return ret;
 }
 
 static enum drm_mode_status sii902x_mode_valid(struct drm_connector *connector,
@@ -237,20 +331,28 @@
 {
 	struct sii902x *sii902x = bridge_to_sii902x(bridge);
 
+	mutex_lock(&sii902x->mutex);
+
 	regmap_update_bits(sii902x->regmap, SII902X_SYS_CTRL_DATA,
 			   SII902X_SYS_CTRL_PWR_DWN,
 			   SII902X_SYS_CTRL_PWR_DWN);
+
+	mutex_unlock(&sii902x->mutex);
 }
 
 static void sii902x_bridge_enable(struct drm_bridge *bridge)
 {
 	struct sii902x *sii902x = bridge_to_sii902x(bridge);
 
+	mutex_lock(&sii902x->mutex);
+
 	regmap_update_bits(sii902x->regmap, SII902X_PWR_STATE_CTRL,
 			   SII902X_AVI_POWER_STATE_MSK,
 			   SII902X_AVI_POWER_STATE_D(0));
 	regmap_update_bits(sii902x->regmap, SII902X_SYS_CTRL_DATA,
 			   SII902X_SYS_CTRL_PWR_DWN, 0);
+
+	mutex_unlock(&sii902x->mutex);
 }
 
 static void sii902x_bridge_mode_set(struct drm_bridge *bridge,
@@ -277,26 +379,30 @@
 	buf[9] = SII902X_TPI_AVI_INPUT_RANGE_AUTO |
 		 SII902X_TPI_AVI_INPUT_COLORSPACE_RGB;
 
+	mutex_lock(&sii902x->mutex);
+
 	ret = regmap_bulk_write(regmap, SII902X_TPI_VIDEO_DATA, buf, 10);
 	if (ret)
-		return;
+		goto out;
 
 	ret = drm_hdmi_avi_infoframe_from_display_mode(&frame, adj, false);
 	if (ret < 0) {
 		DRM_ERROR("couldn't fill AVI infoframe\n");
-		return;
+		goto out;
 	}
 
 	ret = hdmi_avi_infoframe_pack(&frame, buf, sizeof(buf));
 	if (ret < 0) {
 		DRM_ERROR("failed to pack AVI infoframe: %d\n", ret);
-		return;
+		goto out;
 	}
 
 	/* Do not send the infoframe header, but keep the CRC field. */
 	regmap_bulk_write(regmap, SII902X_TPI_AVI_INFOFRAME,
 			  buf + HDMI_INFOFRAME_HEADER_SIZE - 1,
 			  HDMI_AVI_INFOFRAME_SIZE + 1);
+out:
+	mutex_unlock(&sii902x->mutex);
 }
 
 static int sii902x_bridge_attach(struct drm_bridge *bridge)
@@ -337,6 +443,312 @@
 	.enable = sii902x_bridge_enable,
 };
 
+static int sii902x_mute(struct sii902x *sii902x, int mute)
+{
+	u8 channel_layout = sii902x->audio.channels > 2 ?
+		SII902X_TPI_AUDIO_LAYOUT_8_CHANNELS :
+		SII902X_TPI_AUDIO_LAYOUT_2_CHANNELS;
+	struct device *dev = &sii902x->i2c->dev;
+	int ret;
+
+	if (mute) {
+		dev_dbg(dev, "%s: Muted (%d)\n", __func__, mute);
+		ret = regmap_write(sii902x->regmap,
+				   SII902X_TPI_AUDIO_CONFIG_BYTE2_REG,
+				   SII902X_TPI_AUDIO_INTERFACE_I2S |
+				   channel_layout |
+				   SII902X_TPI_AUDIO_MUTE_ENABLE |
+				   SII902X_TPI_AUDIO_CODING_PCM);
+	} else {
+		dev_dbg(dev, "%s: Unmuted (%d)\n", __func__, mute);
+		ret = regmap_write(sii902x->regmap,
+				   SII902X_TPI_AUDIO_CONFIG_BYTE2_REG,
+				   SII902X_TPI_AUDIO_INTERFACE_I2S |
+				   channel_layout |
+				   SII902X_TPI_AUDIO_MUTE_DISABLE |
+				   SII902X_TPI_AUDIO_CODING_PCM);
+	}
+	return ret;
+}
+
+static const unsigned int sii902x_mclk_div_table[] = {
+	128, 256, 384, 512, 768, 1024, 1152, 192 };
+
+static int sii902x_select_mclk_div(u8 *i2s_config_reg, unsigned int rate,
+				   unsigned int mclk)
+{
+	int distance = 100000;
+	u8 i, nearest = 0;
+
+	for (i = 0; i < ARRAY_SIZE(sii902x_mclk_div_table); i++) {
+		unsigned int div = mclk / rate;
+
+		if (abs(div - sii902x_mclk_div_table[i]) < distance) {
+			nearest = i;
+			distance = abs(div - sii902x_mclk_div_table[i]);
+		}
+		if (div == sii902x_mclk_div_table[i])
+			break;
+	}
+
+	*i2s_config_reg |= nearest << 4;
+
+	if (i == ARRAY_SIZE(sii902x_mclk_div_table))
+		return sii902x_mclk_div_table[nearest];
+
+	return 0;
+}
+
+static int sii902x_audio_hw_params(struct device *dev, void *data,
+				   struct hdmi_codec_daifmt *daifmt,
+				   struct hdmi_codec_params *params)
+{
+	struct sii902x *sii902x = dev_get_drvdata(dev);
+	u8 i2s_config_reg = SII902X_TPI_I2S_SD_DIRECTION_MSB_FIRST;
+	u8 config_byte3_reg = SII902X_TPI_AUDIO_CHANNEL_STREAM;
+	u8 infoframe_buf[HDMI_INFOFRAME_SIZE(AUDIO)];
+	unsigned long mclk_rate;
+	int i, ret;
+
+	if (daifmt->bit_clk_master || daifmt->frame_clk_master) {
+		dev_dbg(dev, "%s: I2S master mode not supported\n", __func__);
+		return -EINVAL;
+	}
+
+	switch (daifmt->fmt) {
+	case HDMI_I2S:
+		i2s_config_reg |= SII902X_TPI_I2S_FIRST_BIT_SHIFT_YES |
+			SII902X_TPI_I2S_SD_JUSTIFY_LEFT;
+		break;
+	case HDMI_RIGHT_J:
+		i2s_config_reg |= SII902X_TPI_I2S_SD_JUSTIFY_RIGHT;
+		break;
+	case HDMI_LEFT_J:
+		i2s_config_reg |= SII902X_TPI_I2S_SD_JUSTIFY_LEFT;
+		break;
+	default:
+		dev_dbg(dev, "%s: Unsupported i2s format %u\n", __func__,
+			daifmt->fmt);
+		return -EINVAL;
+	}
+
+	if (daifmt->bit_clk_inv)
+		i2s_config_reg |= SII902X_TPI_I2S_SCK_EDGE_FALLING;
+	else
+		i2s_config_reg |= SII902X_TPI_I2S_SCK_EDGE_RISING;
+
+	if (daifmt->frame_clk_inv)
+		i2s_config_reg |= SII902X_TPI_I2S_WS_POLARITY_LOW;
+	else
+		i2s_config_reg |= SII902X_TPI_I2S_WS_POLARITY_HIGH;
+
+	switch (params->sample_width) {
+	case 16:
+		config_byte3_reg |= SII902X_TPI_AUDIO_SAMPLE_SIZE_16;
+		break;
+	case 20:
+		config_byte3_reg |= SII902X_TPI_AUDIO_SAMPLE_SIZE_20;
+		break;
+	case 24:
+	case 32:
+		config_byte3_reg |= SII902X_TPI_AUDIO_SAMPLE_SIZE_24;
+		break;
+	default:
+		dev_err(dev, "%s: Unsupported sample width %u\n", __func__,
+			params->sample_width);
+		return -EINVAL;
+	};
+
+	ret = clk_prepare_enable(sii902x->audio.mclk);
+	if (ret) {
+		dev_err(dev, "Enabling mclk failed: %d\n", ret);
+		return ret;
+	}
+	sii902x->audio.mclk_enabled = true;
+
+	mclk_rate = clk_get_rate(sii902x->audio.mclk);
+
+	ret = sii902x_select_mclk_div(&i2s_config_reg, params->sample_rate,
+				      mclk_rate);
+	if (ret)
+		dev_dbg(dev, "Inaccurate reference clock (%ld/%d != %u)\n",
+			mclk_rate, ret, params->sample_rate);
+
+	sii902x->audio.channels = params->channels;
+
+	mutex_lock(&sii902x->mutex);
+
+	ret = sii902x_mute(sii902x, true);
+	if (ret < 0)
+		goto out;
+
+	ret = regmap_write(sii902x->regmap, SII902X_TPI_I2S_INPUT_CONFIG_REG,
+			   i2s_config_reg);
+	if (ret)
+		goto out;
+
+	for (i = 0; i < ARRAY_SIZE(sii902x->audio.i2s_fifo_routing); i++)
+		regmap_write(sii902x->regmap,
+			     SII902X_TPI_I2S_ENABLE_MAPPING_REG,
+			     sii902x->audio.i2s_fifo_routing[i]);
+
+	ret = regmap_write(sii902x->regmap, SII902X_TPI_AUDIO_CONFIG_BYTE3_REG,
+			   config_byte3_reg);
+	if (ret)
+		goto out;
+
+	ret = regmap_bulk_write(sii902x->regmap, SII902X_TPI_I2S_STRM_HDR_BASE,
+				params->iec.status,
+				min((size_t) SII902X_TPI_I2S_STRM_HDR_SIZE,
+				    sizeof(params->iec.status)));
+	if (ret)
+		goto out;
+
+	ret = hdmi_audio_infoframe_pack(&params->cea, infoframe_buf,
+					sizeof(infoframe_buf));
+	if (ret < 0) {
+		dev_err(dev, "%s: Failed to pack audio infoframe: %d\n",
+			__func__, ret);
+		goto out;
+	}
+
+	ret = regmap_bulk_write(sii902x->regmap,
+				SII902X_TPI_MISC_INFOFRAME_BASE,
+				infoframe_buf,
+				min(ret, SII902X_TPI_MISC_INFOFRAME_SIZE));
+	if (ret)
+		goto out;
+
+	/* Decode Level 0 Packets */
+	ret = regmap_write(sii902x->regmap, SII902X_IND_SET_PAGE, 0x02);
+	if (ret)
+		goto out;
+
+	ret = regmap_write(sii902x->regmap, SII902X_IND_OFFSET, 0x24);
+	if (ret)
+		goto out;
+
+	ret = regmap_write(sii902x->regmap, SII902X_IND_VALUE, 0x02);
+	if (ret)
+		goto out;
+
+	dev_dbg(dev, "%s: hdmi audio enabled\n", __func__);
+out:
+	mutex_unlock(&sii902x->mutex);
+
+	if (ret) {
+		clk_disable_unprepare(sii902x->audio.mclk);
+		dev_err(dev, "%s: hdmi audio enable failed: %d\n", __func__,
+			ret);
+	}
+
+	return ret;
+}
+
+static void sii902x_audio_shutdown(struct device *dev, void *data)
+{
+	struct sii902x *sii902x = dev_get_drvdata(dev);
+
+	mutex_lock(&sii902x->mutex);
+
+	regmap_write(sii902x->regmap, SII902X_TPI_AUDIO_CONFIG_BYTE2_REG,
+		     SII902X_TPI_AUDIO_INTERFACE_DISABLE);
+
+	mutex_unlock(&sii902x->mutex);
+
+	if (sii902x->audio.mclk_enabled) {
+		clk_disable_unprepare(sii902x->audio.mclk);
+		sii902x->audio.mclk_enabled = false;
+	}
+}
+
+static int sii902x_audio_digital_mute(struct device *dev, void *data, bool enable)
+{
+	struct sii902x *sii902x = dev_get_drvdata(dev);
+
+	mutex_lock(&sii902x->mutex);
+
+	sii902x_mute(sii902x, enable);
+
+	mutex_unlock(&sii902x->mutex);
+
+	return 0;
+}
+
+static int sii902x_audio_get_eld(struct device *dev, void *data,
+				 uint8_t *buf, size_t len)
+{
+	struct sii902x *sii902x = dev_get_drvdata(dev);
+
+	mutex_lock(&sii902x->mutex);
+
+	memcpy(buf, sii902x->connector.eld,
+	       min(sizeof(sii902x->connector.eld), len));
+
+	mutex_unlock(&sii902x->mutex);
+
+	return 0;
+}
+
+static const struct hdmi_codec_ops sii902x_audio_codec_ops = {
+	.hw_params = sii902x_audio_hw_params,
+	.audio_shutdown = sii902x_audio_shutdown,
+	.digital_mute = sii902x_audio_digital_mute,
+	.get_eld = sii902x_audio_get_eld,
+};
+
+static int sii902x_audio_codec_init(struct sii902x *sii902x,
+				    struct device *dev)
+{
+	static const u8 i2s_fifo_defaults[] = {
+		SII902X_TPI_I2S_CONFIG_FIFO0,
+		SII902X_TPI_I2S_CONFIG_FIFO1,
+		SII902X_TPI_I2S_CONFIG_FIFO2,
+		SII902X_TPI_I2S_CONFIG_FIFO3,
+	};
+	struct hdmi_codec_pdata codec_data = {
+		.ops = &sii902x_audio_codec_ops,
+		.i2s = 1, /* Only i2s support for now. */
+		.spdif = 0,
+		.max_i2s_channels = 0,
+	};
+	int ret, i;
+
+	ret = of_property_read_u32_array(dev->of_node, "i2s-fifo-routing",
+					 sii902x->audio.i2s_fifo_routing,
+					 ARRAY_SIZE(sii902x->audio.i2s_fifo_routing));
+
+	if (ret != 0) {
+		if (ret == -EINVAL)
+			dev_dbg(dev, "%s: No \"i2s-fifo-routing\", no audio\n",
+			__func__);
+		else
+			dev_err(dev,
+				"%s: Error gettin \"i2s-fifo-routing\": %d\n",
+				__func__, ret);
+		return 0;
+	}
+
+	sii902x->audio.mclk = devm_clk_get(dev, NULL);
+	if (IS_ERR(sii902x->audio.mclk)) {
+		dev_err(dev, "%s: No clock (audio mclk) found: %ld\n",
+			__func__, PTR_ERR(sii902x->audio.mclk));
+		return 0;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(sii902x->audio.i2s_fifo_routing); i++) {
+		if (sii902x->audio.i2s_fifo_routing[i] & ENABLE_BIT)
+			codec_data.max_i2s_channels += 2;
+		sii902x->audio.i2s_fifo_routing[i] |= i2s_fifo_defaults[i];
+	}
+
+	sii902x->audio.pdev = platform_device_register_data(
+		dev, HDMI_CODEC_DRV_NAME, PLATFORM_DEVID_AUTO,
+		&codec_data, sizeof(codec_data));
+
+	return PTR_ERR_OR_ZERO(sii902x->audio.pdev);
+}
+
 static const struct regmap_range sii902x_volatile_ranges[] = {
 	{ .range_min = 0, .range_max = 0xff },
 };
@@ -349,6 +761,7 @@
 static const struct regmap_config sii902x_regmap_config = {
 	.reg_bits = 8,
 	.val_bits = 8,
+	.max_register = SII902X_TPI_MISC_INFOFRAME_END,
 	.volatile_table = &sii902x_volatile_table,
 	.cache_type = REGCACHE_NONE,
 };
@@ -367,6 +780,127 @@
 	return IRQ_HANDLED;
 }
 
+/*
+ * The purpose of sii902x_i2c_bypass_select is to enable the pass through
+ * mode of the HDMI transmitter. Do not use regmap from within this function,
+ * only use sii902x_*_unlocked functions to read/modify/write registers.
+ * We are holding the parent adapter lock here, keep this in mind before
+ * adding more i2c transactions.
+ *
+ * Also, since SII902X_SYS_CTRL_DATA is used with regmap_update_bits elsewhere
+ * in this driver, we need to make sure that we only touch 0x1A[2:1] from
+ * within sii902x_i2c_bypass_select and sii902x_i2c_bypass_deselect, and that
+ * we leave the remaining bits as we have found them.
+ */
+static int sii902x_i2c_bypass_select(struct i2c_mux_core *mux, u32 chan_id)
+{
+	struct sii902x *sii902x = i2c_mux_priv(mux);
+	struct device *dev = &sii902x->i2c->dev;
+	unsigned long timeout;
+	u8 status;
+	int ret;
+
+	ret = sii902x_update_bits_unlocked(sii902x->i2c, SII902X_SYS_CTRL_DATA,
+					   SII902X_SYS_CTRL_DDC_BUS_REQ,
+					   SII902X_SYS_CTRL_DDC_BUS_REQ);
+	if (ret)
+		return ret;
+
+	timeout = jiffies +
+		  msecs_to_jiffies(SII902X_I2C_BUS_ACQUISITION_TIMEOUT_MS);
+	do {
+		ret = sii902x_read_unlocked(sii902x->i2c, SII902X_SYS_CTRL_DATA,
+					    &status);
+		if (ret)
+			return ret;
+	} while (!(status & SII902X_SYS_CTRL_DDC_BUS_GRTD) &&
+		 time_before(jiffies, timeout));
+
+	if (!(status & SII902X_SYS_CTRL_DDC_BUS_GRTD)) {
+		dev_err(dev, "Failed to acquire the i2c bus\n");
+		return -ETIMEDOUT;
+	}
+
+	return sii902x_write_unlocked(sii902x->i2c, SII902X_SYS_CTRL_DATA,
+				      status);
+}
+
+/*
+ * The purpose of sii902x_i2c_bypass_deselect is to disable the pass through
+ * mode of the HDMI transmitter. Do not use regmap from within this function,
+ * only use sii902x_*_unlocked functions to read/modify/write registers.
+ * We are holding the parent adapter lock here, keep this in mind before
+ * adding more i2c transactions.
+ *
+ * Also, since SII902X_SYS_CTRL_DATA is used with regmap_update_bits elsewhere
+ * in this driver, we need to make sure that we only touch 0x1A[2:1] from
+ * within sii902x_i2c_bypass_select and sii902x_i2c_bypass_deselect, and that
+ * we leave the remaining bits as we have found them.
+ */
+static int sii902x_i2c_bypass_deselect(struct i2c_mux_core *mux, u32 chan_id)
+{
+	struct sii902x *sii902x = i2c_mux_priv(mux);
+	struct device *dev = &sii902x->i2c->dev;
+	unsigned long timeout;
+	unsigned int retries;
+	u8 status;
+	int ret;
+
+	/*
+	 * When the HDMI transmitter is in pass through mode, we need an
+	 * (undocumented) additional delay between STOP and START conditions
+	 * to guarantee the bus won't get stuck.
+	 */
+	udelay(30);
+
+	/*
+	 * Sometimes the I2C bus can stall after failure to use the
+	 * EDID channel. Retry a few times to see if things clear
+	 * up, else continue anyway.
+	 */
+	retries = 5;
+	do {
+		ret = sii902x_read_unlocked(sii902x->i2c, SII902X_SYS_CTRL_DATA,
+					    &status);
+		retries--;
+	} while (ret && retries);
+	if (ret) {
+		dev_err(dev, "failed to read status (%d)\n", ret);
+		return ret;
+	}
+
+	ret = sii902x_update_bits_unlocked(sii902x->i2c, SII902X_SYS_CTRL_DATA,
+					   SII902X_SYS_CTRL_DDC_BUS_REQ |
+					   SII902X_SYS_CTRL_DDC_BUS_GRTD, 0);
+	if (ret)
+		return ret;
+
+	timeout = jiffies +
+		  msecs_to_jiffies(SII902X_I2C_BUS_ACQUISITION_TIMEOUT_MS);
+	do {
+		ret = sii902x_read_unlocked(sii902x->i2c, SII902X_SYS_CTRL_DATA,
+					    &status);
+		if (ret)
+			return ret;
+	} while (status & (SII902X_SYS_CTRL_DDC_BUS_REQ |
+			   SII902X_SYS_CTRL_DDC_BUS_GRTD) &&
+		 time_before(jiffies, timeout));
+
+	if (status & (SII902X_SYS_CTRL_DDC_BUS_REQ |
+		      SII902X_SYS_CTRL_DDC_BUS_GRTD)) {
+		dev_err(dev, "failed to release the i2c bus\n");
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+static const struct drm_bridge_timings default_sii902x_timings = {
+	.input_bus_flags = DRM_BUS_FLAG_PIXDATA_SAMPLE_NEGEDGE
+		 | DRM_BUS_FLAG_SYNC_SAMPLE_NEGEDGE
+		 | DRM_BUS_FLAG_DE_HIGH,
+};
+
 static int sii902x_probe(struct i2c_client *client,
 			 const struct i2c_device_id *id)
 {
@@ -376,6 +910,13 @@
 	u8 chipid[4];
 	int ret;
 
+	ret = i2c_check_functionality(client->adapter,
+				      I2C_FUNC_SMBUS_BYTE_DATA);
+	if (!ret) {
+		dev_err(dev, "I2C adapter not suitable\n");
+		return -EIO;
+	}
+
 	sii902x = devm_kzalloc(dev, sizeof(*sii902x), GFP_KERNEL);
 	if (!sii902x)
 		return -ENOMEM;
@@ -393,6 +934,9 @@
 		return PTR_ERR(sii902x->reset_gpio);
 	}
 
+	/* protect audio and video functions from each other */
+	mutex_init(&sii902x->mutex);
+
 	sii902x_reset(sii902x);
 
 	ret = regmap_write(sii902x->regmap, SII902X_REG_TPI_RQB, 0x0);
@@ -430,11 +974,22 @@
 
 	sii902x->bridge.funcs = &sii902x_bridge_funcs;
 	sii902x->bridge.of_node = dev->of_node;
+	sii902x->bridge.timings = &default_sii902x_timings;
 	drm_bridge_add(&sii902x->bridge);
 
+	sii902x_audio_codec_init(sii902x, dev);
+
 	i2c_set_clientdata(client, sii902x);
 
-	return 0;
+	sii902x->i2cmux = i2c_mux_alloc(client->adapter, dev,
+					1, 0, I2C_MUX_GATE,
+					sii902x_i2c_bypass_select,
+					sii902x_i2c_bypass_deselect);
+	if (!sii902x->i2cmux)
+		return -ENOMEM;
+
+	sii902x->i2cmux->priv = sii902x;
+	return i2c_mux_add_adapter(sii902x->i2cmux, 0, 0, 0);
 }
 
 static int sii902x_remove(struct i2c_client *client)
@@ -442,6 +997,7 @@
 {
 	struct sii902x *sii902x = i2c_get_clientdata(client);
 
+	i2c_mux_del_adapters(sii902x->i2cmux);
 	drm_bridge_remove(&sii902x->bridge);
 
 	return 0;
diff -Naur --no-dereference a/drivers/gpu/drm/bridge/tc358767.c b/drivers/gpu/drm/bridge/tc358767.c
--- a/drivers/gpu/drm/bridge/tc358767.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/bridge/tc358767.c	2020-06-19 21:00:31.408227715 -0400
@@ -71,6 +71,7 @@
 
 /* System */
 #define TC_IDREG		0x0500
+#define SYSSTAT			0x0508
 #define SYSCTRL			0x0510
 #define DP0_AUDSRC_NO_INPUT		(0 << 3)
 #define DP0_AUDSRC_I2S_RX		(1 << 3)
@@ -78,6 +79,19 @@
 #define DP0_VIDSRC_DSI_RX		(1 << 0)
 #define DP0_VIDSRC_DPI_RX		(2 << 0)
 #define DP0_VIDSRC_COLOR_BAR		(3 << 0)
+#define GPIOM			0x0540
+#define GPIOC			0x0544
+#define GPIOO			0x0548
+#define GPIOI			0x054c
+#define INTCTL_G		0x0560
+#define INTSTS_G		0x0564
+
+#define INT_SYSERR		BIT(16)
+#define INT_GPIO_H(x)		(1 << (x == 0 ? 2 : 10))
+#define INT_GPIO_LC(x)		(1 << (x == 0 ? 3 : 11))
+
+#define INT_GP0_LCNT		0x0584
+#define INT_GP1_LCNT		0x0588
 
 /* Control */
 #define DP0CTL			0x0600
@@ -186,11 +200,8 @@
 struct tc_edp_link {
 	struct drm_dp_link	base;
 	u8			assr;
-	int			scrambler_dis;
-	int			spread;
-	int			coding8b10b;
-	u8			swing;
-	u8			preemp;
+	bool			scrambler_dis;
+	bool			spread;
 };
 
 struct tc_data {
@@ -208,7 +219,7 @@
 	/* display edid */
 	struct edid		*edid;
 	/* current mode */
-	struct drm_display_mode	*mode;
+	struct drm_display_mode	mode;
 
 	u32			rev;
 	u8			assr;
@@ -216,6 +227,12 @@
 	struct gpio_desc	*sd_gpio;
 	struct gpio_desc	*reset_gpio;
 	struct clk		*refclk;
+
+	/* do we have IRQ */
+	bool			have_irq;
+
+	/* HPD pin number (0 or 1) or -ENODEV */
+	int			hpd_pin;
 };
 
 static inline struct tc_data *aux_to_tc(struct drm_dp_aux *a)
@@ -286,14 +303,17 @@
 	ret = regmap_read(tc->regmap, DP0_AUXSTATUS, &value);
 	if (ret < 0)
 		return ret;
+
 	if (value & AUX_BUSY) {
-		if (value & AUX_TIMEOUT) {
-			dev_err(tc->dev, "i2c access timeout!\n");
-			return -ETIMEDOUT;
-		}
+		dev_err(tc->dev, "aux busy!\n");
 		return -EBUSY;
 	}
 
+	if (value & AUX_TIMEOUT) {
+		dev_err(tc->dev, "aux access timeout!\n");
+		return -ETIMEDOUT;
+	}
+
 	*reply = (value & AUX_STATUS_MASK) >> AUX_STATUS_SHIFT;
 	return 0;
 }
@@ -387,13 +407,10 @@
 	 * No training pattern, skew lane 1 data by two LSCLK cycles with
 	 * respect to lane 0 data, AutoCorrect Mode = 0
 	 */
-	u32 reg = DP0_SRCCTRL_NOTP | DP0_SRCCTRL_LANESKEW;
+	u32 reg = DP0_SRCCTRL_NOTP | DP0_SRCCTRL_LANESKEW | DP0_SRCCTRL_EN810B;
 
 	if (tc->link.scrambler_dis)
 		reg |= DP0_SRCCTRL_SCRMBLDIS;	/* Scrambler Disabled */
-	if (tc->link.coding8b10b)
-		/* Enable 8/10B Encoder (TxData[19:16] not used) */
-		reg |= DP0_SRCCTRL_EN810B;
 	if (tc->link.spread)
 		reg |= DP0_SRCCTRL_SSCG;	/* Spread Spectrum Enable */
 	if (tc->link.base.num_lanes == 2)
@@ -545,7 +562,6 @@
 	unsigned long rate;
 	u32 value;
 	int ret;
-	u32 dp_phy_ctrl;
 
 	rate = clk_get_rate(tc->refclk);
 	switch (rate) {
@@ -570,10 +586,7 @@
 	value |= SYSCLK_SEL_LSCLK | LSCLK_DIV_2;
 	tc_write(SYS_PLLPARAM, value);
 
-	dp_phy_ctrl = BGREN | PWR_SW_EN | PHY_A0_EN;
-	if (tc->link.base.num_lanes == 2)
-		dp_phy_ctrl |= PHY_2LANE;
-	tc_write(DP_PHY_CTRL, dp_phy_ctrl);
+	tc_write(DP_PHY_CTRL, BGREN | PWR_SW_EN | PHY_A0_EN);
 
 	/*
 	 * Initially PLLs are in bypass. Force PLL parameter update,
@@ -590,8 +603,9 @@
 	if (ret == -ETIMEDOUT) {
 		dev_err(tc->dev, "Timeout waiting for PHY to become ready");
 		return ret;
-	} else if (ret)
+	} else if (ret) {
 		goto err;
+	}
 
 	/* Setup AUX link */
 	tc_write(DP0_AUXCFG1, AUX_RX_FILTER_EN |
@@ -627,13 +641,13 @@
 	ret = drm_dp_dpcd_readb(&tc->aux, DP_MAX_DOWNSPREAD, tmp);
 	if (ret < 0)
 		goto err_dpcd_read;
-	tc->link.spread = tmp[0] & BIT(0); /* 0.5% down spread */
+	tc->link.spread = tmp[0] & DP_MAX_DOWNSPREAD_0_5;
 
 	ret = drm_dp_dpcd_readb(&tc->aux, DP_MAIN_LINK_CHANNEL_CODING, tmp);
 	if (ret < 0)
 		goto err_dpcd_read;
-	tc->link.coding8b10b = tmp[0] & BIT(0);
-	tc->link.scrambler_dis = 0;
+
+	tc->link.scrambler_dis = false;
 	/* read assr */
 	ret = drm_dp_dpcd_readb(&tc->aux, DP_EDP_CONFIGURATION_SET, tmp);
 	if (ret < 0)
@@ -646,7 +660,9 @@
 		tc->link.base.num_lanes,
 		(tc->link.base.capabilities & DP_LINK_CAP_ENHANCED_FRAMING) ?
 		"enhanced" : "non-enhanced");
-	dev_dbg(tc->dev, "ANSI 8B/10B: %d\n", tc->link.coding8b10b);
+	dev_dbg(tc->dev, "Downspread: %s, scrambler: %s\n",
+		tc->link.spread ? "0.5%" : "0.0%",
+		tc->link.scrambler_dis ? "disabled" : "enabled");
 	dev_dbg(tc->dev, "Display ASSR: %d, TC358767 ASSR: %d\n",
 		tc->link.assr, tc->assr);
 
@@ -657,7 +673,8 @@
 	return ret;
 }
 
-static int tc_set_video_mode(struct tc_data *tc, struct drm_display_mode *mode)
+static int tc_set_video_mode(struct tc_data *tc,
+			     const struct drm_display_mode *mode)
 {
 	int ret;
 	int vid_sync_dly;
@@ -669,6 +686,8 @@
 	int upper_margin = mode->vtotal - mode->vsync_end;
 	int lower_margin = mode->vsync_start - mode->vdisplay;
 	int vsync_len = mode->vsync_end - mode->vsync_start;
+	u32 bits_per_pixel = 24;
+	u32 in_bw, out_bw;
 
 	/*
 	 * Recommended maximum number of symbols transferred in a transfer unit:
@@ -676,7 +695,10 @@
 	 *              (output active video bandwidth in bytes))
 	 * Must be less than tu_size.
 	 */
-	max_tu_symbol = TU_SIZE_RECOMMENDED - 1;
+
+	in_bw = mode->clock * bits_per_pixel / 8;
+	out_bw = tc->link.base.num_lanes * tc->link.base.rate;
+	max_tu_symbol = DIV_ROUND_UP(in_bw * TU_SIZE_RECOMMENDED, out_bw);
 
 	dev_dbg(tc->dev, "set mode %dx%d\n",
 		mode->hdisplay, mode->vdisplay);
@@ -743,89 +765,31 @@
 	return ret;
 }
 
-static int tc_link_training(struct tc_data *tc, int pattern)
+static int tc_wait_link_training(struct tc_data *tc, u32 *error)
 {
-	const char * const *errors;
-	u32 srcctrl = tc_srcctrl(tc) | DP0_SRCCTRL_SCRMBLDIS |
-		      DP0_SRCCTRL_AUTOCORRECT;
-	int timeout;
-	int retry;
+	u32 timeout = 1000;
 	u32 value;
 	int ret;
 
-	if (pattern == DP_TRAINING_PATTERN_1) {
-		srcctrl |= DP0_SRCCTRL_TP1;
-		errors = training_pattern1_errors;
-	} else {
-		srcctrl |= DP0_SRCCTRL_TP2;
-		errors = training_pattern2_errors;
-	}
-
-	/* Set DPCD 0x102 for Training Part 1 or 2 */
-	tc_write(DP0_SNKLTCTRL, DP_LINK_SCRAMBLING_DISABLE | pattern);
-
-	tc_write(DP0_LTLOOPCTRL,
-		 (0x0f << 28) |	/* Defer Iteration Count */
-		 (0x0f << 24) |	/* Loop Iteration Count */
-		 (0x0d << 0));	/* Loop Timer Delay */
-
-	retry = 5;
 	do {
-		/* Set DP0 Training Pattern */
-		tc_write(DP0_SRCCTRL, srcctrl);
-
-		/* Enable DP0 to start Link Training */
-		tc_write(DP0CTL, DP_EN);
+		udelay(1);
+		tc_read(DP0_LTSTAT, &value);
+	} while ((!(value & LT_LOOPDONE)) && (--timeout));
 
-		/* wait */
-		timeout = 1000;
-		do {
-			tc_read(DP0_LTSTAT, &value);
-			udelay(1);
-		} while ((!(value & LT_LOOPDONE)) && (--timeout));
-		if (timeout == 0) {
-			dev_err(tc->dev, "Link training timeout!\n");
-		} else {
-			int pattern = (value >> 11) & 0x3;
-			int error = (value >> 8) & 0x7;
-
-			dev_dbg(tc->dev,
-				"Link training phase %d done after %d uS: %s\n",
-				pattern, 1000 - timeout, errors[error]);
-			if (pattern == DP_TRAINING_PATTERN_1 && error == 0)
-				break;
-			if (pattern == DP_TRAINING_PATTERN_2) {
-				value &= LT_CHANNEL1_EQ_BITS |
-					 LT_INTERLANE_ALIGN_DONE |
-					 LT_CHANNEL0_EQ_BITS;
-				/* in case of two lanes */
-				if ((tc->link.base.num_lanes == 2) &&
-				    (value == (LT_CHANNEL1_EQ_BITS |
-					       LT_INTERLANE_ALIGN_DONE |
-					       LT_CHANNEL0_EQ_BITS)))
-					break;
-				/* in case of one line */
-				if ((tc->link.base.num_lanes == 1) &&
-				    (value == (LT_INTERLANE_ALIGN_DONE |
-					       LT_CHANNEL0_EQ_BITS)))
-					break;
-			}
-		}
-		/* restart */
-		tc_write(DP0CTL, 0);
-		usleep_range(10, 20);
-	} while (--retry);
-	if (retry == 0) {
-		dev_err(tc->dev, "Failed to finish training phase %d\n",
-			pattern);
+	if (timeout == 0) {
+		dev_err(tc->dev, "Link training timeout waiting for LT_LOOPDONE!\n");
+		ret = -ETIMEDOUT;
+		goto err;
 	}
 
+	*error = (value >> 8) & 0x7;
+
 	return 0;
 err:
 	return ret;
 }
 
-static int tc_main_link_setup(struct tc_data *tc)
+static int tc_main_link_enable(struct tc_data *tc)
 {
 	struct drm_dp_aux *aux = &tc->aux;
 	struct device *dev = tc->dev;
@@ -835,10 +799,11 @@
 	u32 value;
 	int ret;
 	u8 tmp[8];
+	u32 error;
 
-	/* display mode should be set at this point */
-	if (!tc->mode)
-		return -EINVAL;
+	dev_dbg(tc->dev, "link enable\n");
+
+	tc_write(DP0CTL, 0);
 
 	tc_write(DP0_SRCCTRL, tc_srcctrl(tc));
 	/* SSCG and BW27 on DP1 must be set to the same as on DP0 */
@@ -871,7 +836,6 @@
 	if (tc->link.base.num_lanes == 2)
 		dp_phy_ctrl |= PHY_2LANE;
 	tc_write(DP_PHY_CTRL, dp_phy_ctrl);
-	msleep(100);
 
 	/* PLL setup */
 	tc_write(DP0_PLLCTRL, PLLUPDATE | PLLEN);
@@ -880,14 +844,6 @@
 	tc_write(DP1_PLLCTRL, PLLUPDATE | PLLEN);
 	tc_wait_pll_lock(tc);
 
-	/* PXL PLL setup */
-	if (tc_test_pattern) {
-		ret = tc_pxl_pll_en(tc, clk_get_rate(tc->refclk),
-				    1000 * tc->mode->clock);
-		if (ret)
-			goto err;
-	}
-
 	/* Reset/Enable Main Links */
 	dp_phy_ctrl |= DP_PHY_RST | PHY_M1_RST | PHY_M0_RST;
 	tc_write(DP_PHY_CTRL, dp_phy_ctrl);
@@ -935,7 +891,7 @@
 			dev_dbg(dev, "Failed to switch display ASSR to %d, falling back to unscrambled mode\n",
 				 tc->assr);
 			/* trying with disabled scrambler */
-			tc->link.scrambler_dis = 1;
+			tc->link.scrambler_dis = true;
 		}
 	}
 
@@ -947,19 +903,70 @@
 	/* DOWNSPREAD_CTRL */
 	tmp[0] = tc->link.spread ? DP_SPREAD_AMP_0_5 : 0x00;
 	/* MAIN_LINK_CHANNEL_CODING_SET */
-	tmp[1] =  tc->link.coding8b10b ? DP_SET_ANSI_8B10B : 0x00;
+	tmp[1] =  DP_SET_ANSI_8B10B;
 	ret = drm_dp_dpcd_write(aux, DP_DOWNSPREAD_CTRL, tmp, 2);
 	if (ret < 0)
 		goto err_dpcd_write;
 
-	ret = tc_link_training(tc, DP_TRAINING_PATTERN_1);
+	// Reset voltage-swing & pre-emphasis
+	tmp[0] = tmp[1] = DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_0;
+	ret = drm_dp_dpcd_write(aux, DP_TRAINING_LANE0_SET, tmp, 2);
+	if (ret < 0)
+		goto err_dpcd_write;
+
+	/* LINK TRAINING PATTERN 1 */
+
+	/* Set DPCD 0x102 for Training Pattern 1 */
+	tc_write(DP0_SNKLTCTRL, DP_LINK_SCRAMBLING_DISABLE | DP_TRAINING_PATTERN_1);
+
+	tc_write(DP0_LTLOOPCTRL,
+		 (15 << 28) |	/* Defer Iteration Count */
+		 (15 << 24) |	/* Loop Iteration Count */
+		 (0xd << 0));	/* Loop Timer Delay */
+
+	tc_write(DP0_SRCCTRL, tc_srcctrl(tc) | DP0_SRCCTRL_SCRMBLDIS | DP0_SRCCTRL_AUTOCORRECT |
+		 DP0_SRCCTRL_TP1);
+
+	/* Enable DP0 to start Link Training */
+	tc_write(DP0CTL,
+		 ((tc->link.base.capabilities & DP_LINK_CAP_ENHANCED_FRAMING) ? EF_EN : 0) |
+		 DP_EN);
+
+	/* wait */
+	ret = tc_wait_link_training(tc, &error);
 	if (ret)
 		goto err;
 
-	ret = tc_link_training(tc, DP_TRAINING_PATTERN_2);
+	if (error) {
+		dev_err(tc->dev, "Link training phase 1 failed: %s\n",
+			training_pattern1_errors[error]);
+		ret = -ENODEV;
+		goto err;
+	}
+
+	/* LINK TRAINING PATTERN 2 */
+
+	/* Set DPCD 0x102 for Training Pattern 2 */
+	tc_write(DP0_SNKLTCTRL, DP_LINK_SCRAMBLING_DISABLE | DP_TRAINING_PATTERN_2);
+
+	tc_write(DP0_SRCCTRL, tc_srcctrl(tc) | DP0_SRCCTRL_SCRMBLDIS | DP0_SRCCTRL_AUTOCORRECT |
+		 DP0_SRCCTRL_TP2);
+
+	/* wait */
+	ret = tc_wait_link_training(tc, &error);
 	if (ret)
 		goto err;
 
+	if (error) {
+		dev_err(tc->dev, "Link training phase 2 failed: %s\n",
+			training_pattern2_errors[error]);
+		ret = -ENODEV;
+		goto err;
+	}
+
+	/* Clear Training Pattern, set AutoCorrect Mode = 1 */
+	tc_write(DP0_SRCCTRL, tc_srcctrl(tc) | DP0_SRCCTRL_AUTOCORRECT);
+
 	/* Clear DPCD 0x102 */
 	/* Note: Can Not use DP0_SNKLTCTRL (0x06E4) short cut */
 	tmp[0] = tc->link.scrambler_dis ? DP_LINK_SCRAMBLING_DISABLE : 0x00;
@@ -967,47 +974,43 @@
 	if (ret < 0)
 		goto err_dpcd_write;
 
-	/* Clear Training Pattern, set AutoCorrect Mode = 1 */
-	tc_write(DP0_SRCCTRL, tc_srcctrl(tc) | DP0_SRCCTRL_AUTOCORRECT);
+	/* Check link status */
+	ret = drm_dp_dpcd_read_link_status(aux, tmp);
+	if (ret < 0)
+		goto err_dpcd_read;
 
-	/* Wait */
-	timeout = 100;
-	do {
-		udelay(1);
-		/* Read DPCD 0x202-0x207 */
-		ret = drm_dp_dpcd_read_link_status(aux, tmp + 2);
-		if (ret < 0)
-			goto err_dpcd_read;
-	} while ((--timeout) &&
-		 !(drm_dp_channel_eq_ok(tmp + 2,  tc->link.base.num_lanes)));
+	ret = 0;
 
-	if (timeout == 0) {
-		/* Read DPCD 0x200-0x201 */
-		ret = drm_dp_dpcd_read(aux, DP_SINK_COUNT, tmp, 2);
-		if (ret < 0)
-			goto err_dpcd_read;
-		dev_err(dev, "channel(s) EQ not ok\n");
-		dev_info(dev, "0x0200 SINK_COUNT: 0x%02x\n", tmp[0]);
-		dev_info(dev, "0x0201 DEVICE_SERVICE_IRQ_VECTOR: 0x%02x\n",
-			 tmp[1]);
-		dev_info(dev, "0x0202 LANE0_1_STATUS: 0x%02x\n", tmp[2]);
-		dev_info(dev, "0x0204 LANE_ALIGN_STATUS_UPDATED: 0x%02x\n",
-			 tmp[4]);
-		dev_info(dev, "0x0205 SINK_STATUS: 0x%02x\n", tmp[5]);
-		dev_info(dev, "0x0206 ADJUST_REQUEST_LANE0_1: 0x%02x\n",
-			 tmp[6]);
+	value = tmp[0] & DP_CHANNEL_EQ_BITS;
 
-		return -EAGAIN;
+	if (value != DP_CHANNEL_EQ_BITS) {
+		dev_err(tc->dev, "Lane 0 failed: %x\n", value);
+		ret = -ENODEV;
 	}
 
-	ret = tc_set_video_mode(tc, tc->mode);
-	if (ret)
-		goto err;
+	if (tc->link.base.num_lanes == 2) {
+		value = (tmp[0] >> 4) & DP_CHANNEL_EQ_BITS;
 
-	/* Set M/N */
-	ret = tc_stream_clock_calc(tc);
-	if (ret)
+		if (value != DP_CHANNEL_EQ_BITS) {
+			dev_err(tc->dev, "Lane 1 failed: %x\n", value);
+			ret = -ENODEV;
+		}
+
+		if (!(tmp[2] & DP_INTERLANE_ALIGN_DONE)) {
+			dev_err(tc->dev, "Interlane align failed\n");
+			ret = -ENODEV;
+		}
+	}
+
+	if (ret) {
+		dev_err(dev, "0x0202 LANE0_1_STATUS:            0x%02x\n", tmp[0]);
+		dev_err(dev, "0x0203 LANE2_3_STATUS             0x%02x\n", tmp[1]);
+		dev_err(dev, "0x0204 LANE_ALIGN_STATUS_UPDATED: 0x%02x\n", tmp[2]);
+		dev_err(dev, "0x0205 SINK_STATUS:               0x%02x\n", tmp[3]);
+		dev_err(dev, "0x0206 ADJUST_REQUEST_LANE0_1:    0x%02x\n", tmp[4]);
+		dev_err(dev, "0x0207 ADJUST_REQUEST_LANE2_3:    0x%02x\n", tmp[5]);
 		goto err;
+	}
 
 	return 0;
 err_dpcd_read:
@@ -1019,39 +1022,81 @@
 	return ret;
 }
 
-static int tc_main_link_stream(struct tc_data *tc, int state)
+static int tc_main_link_disable(struct tc_data *tc)
+{
+	int ret;
+
+	dev_dbg(tc->dev, "link disable\n");
+
+	tc_write(DP0_SRCCTRL, 0);
+	tc_write(DP0CTL, 0);
+
+	return 0;
+err:
+	return ret;
+}
+
+static int tc_stream_enable(struct tc_data *tc)
 {
 	int ret;
 	u32 value;
 
-	dev_dbg(tc->dev, "stream: %d\n", state);
+	dev_dbg(tc->dev, "stream enable\n");
 
-	if (state) {
-		value = VID_MN_GEN | DP_EN;
-		if (tc->link.base.capabilities & DP_LINK_CAP_ENHANCED_FRAMING)
-			value |= EF_EN;
-		tc_write(DP0CTL, value);
-		/*
-		 * VID_EN assertion should be delayed by at least N * LSCLK
-		 * cycles from the time VID_MN_GEN is enabled in order to
-		 * generate stable values for VID_M. LSCLK is 270 MHz or
-		 * 162 MHz, VID_N is set to 32768 in  tc_stream_clock_calc(),
-		 * so a delay of at least 203 us should suffice.
-		 */
-		usleep_range(500, 1000);
-		value |= VID_EN;
-		tc_write(DP0CTL, value);
-		/* Set input interface */
-		value = DP0_AUDSRC_NO_INPUT;
-		if (tc_test_pattern)
-			value |= DP0_VIDSRC_COLOR_BAR;
-		else
-			value |= DP0_VIDSRC_DPI_RX;
-		tc_write(SYSCTRL, value);
-	} else {
-		tc_write(DP0CTL, 0);
+	/* PXL PLL setup */
+	if (tc_test_pattern) {
+		ret = tc_pxl_pll_en(tc, clk_get_rate(tc->refclk),
+				    1000 * tc->mode.clock);
+		if (ret)
+			goto err;
 	}
 
+	ret = tc_set_video_mode(tc, &tc->mode);
+	if (ret)
+		goto err;
+
+	/* Set M/N */
+	ret = tc_stream_clock_calc(tc);
+	if (ret)
+		goto err;
+
+	value = VID_MN_GEN | DP_EN;
+	if (tc->link.base.capabilities & DP_LINK_CAP_ENHANCED_FRAMING)
+		value |= EF_EN;
+	tc_write(DP0CTL, value);
+	/*
+	 * VID_EN assertion should be delayed by at least N * LSCLK
+	 * cycles from the time VID_MN_GEN is enabled in order to
+	 * generate stable values for VID_M. LSCLK is 270 MHz or
+	 * 162 MHz, VID_N is set to 32768 in  tc_stream_clock_calc(),
+	 * so a delay of at least 203 us should suffice.
+	 */
+	usleep_range(500, 1000);
+	value |= VID_EN;
+	tc_write(DP0CTL, value);
+	/* Set input interface */
+	value = DP0_AUDSRC_NO_INPUT;
+	if (tc_test_pattern)
+		value |= DP0_VIDSRC_COLOR_BAR;
+	else
+		value |= DP0_VIDSRC_DPI_RX;
+	tc_write(SYSCTRL, value);
+
+	return 0;
+err:
+	return ret;
+}
+
+static int tc_stream_disable(struct tc_data *tc)
+{
+	int ret;
+
+	dev_dbg(tc->dev, "stream disable\n");
+
+	tc_write(DP0CTL, 0);
+
+	tc_pxl_pll_dis(tc);
+
 	return 0;
 err:
 	return ret;
@@ -1069,15 +1114,22 @@
 	struct tc_data *tc = bridge_to_tc(bridge);
 	int ret;
 
-	ret = tc_main_link_setup(tc);
+	ret = tc_get_display_props(tc);
+	if (ret < 0) {
+		dev_err(tc->dev, "failed to read display props: %d\n", ret);
+		return;
+	}
+
+	ret = tc_main_link_enable(tc);
 	if (ret < 0) {
-		dev_err(tc->dev, "main link setup error: %d\n", ret);
+		dev_err(tc->dev, "main link enable error: %d\n", ret);
 		return;
 	}
 
-	ret = tc_main_link_stream(tc, 1);
+	ret = tc_stream_enable(tc);
 	if (ret < 0) {
 		dev_err(tc->dev, "main link stream start error: %d\n", ret);
+		tc_main_link_disable(tc);
 		return;
 	}
 
@@ -1091,9 +1143,13 @@
 
 	drm_panel_disable(tc->panel);
 
-	ret = tc_main_link_stream(tc, 0);
+	ret = tc_stream_disable(tc);
 	if (ret < 0)
 		dev_err(tc->dev, "main link stream stop error: %d\n", ret);
+
+	ret = tc_main_link_disable(tc);
+	if (ret < 0)
+		dev_err(tc->dev, "main link disable error: %d\n", ret);
 }
 
 static void tc_bridge_post_disable(struct drm_bridge *bridge)
@@ -1115,10 +1171,10 @@
 	return true;
 }
 
-static enum drm_mode_status tc_connector_mode_valid(struct drm_connector *connector,
-				   struct drm_display_mode *mode)
+static enum drm_mode_status tc_mode_valid(struct drm_bridge *bridge,
+					  const struct drm_display_mode *mode)
 {
-	struct tc_data *tc = connector_to_tc(connector);
+	struct tc_data *tc = bridge_to_tc(bridge);
 	u32 req, avail;
 	u32 bits_per_pixel = 24;
 
@@ -1141,7 +1197,7 @@
 {
 	struct tc_data *tc = bridge_to_tc(bridge);
 
-	tc->mode = mode;
+	tc->mode = *mode;
 }
 
 static int tc_connector_get_modes(struct drm_connector *connector)
@@ -1176,29 +1232,42 @@
 	return count;
 }
 
-static void tc_connector_set_polling(struct tc_data *tc,
-				     struct drm_connector *connector)
-{
-	/* TODO: add support for HPD */
-	connector->polled = DRM_CONNECTOR_POLL_CONNECT |
-			    DRM_CONNECTOR_POLL_DISCONNECT;
-}
+static const struct drm_connector_helper_funcs tc_connector_helper_funcs = {
+	.get_modes = tc_connector_get_modes,
+};
 
-static struct drm_encoder *
-tc_connector_best_encoder(struct drm_connector *connector)
+static enum drm_connector_status tc_connector_detect(struct drm_connector *connector, bool force)
 {
 	struct tc_data *tc = connector_to_tc(connector);
+	bool conn;
+	u32 val;
+	int ret;
 
-	return tc->bridge.encoder;
-}
+	if (tc->hpd_pin < 0) {
+		if (!tc->panel)
+			return connector_status_unknown;
 
-static const struct drm_connector_helper_funcs tc_connector_helper_funcs = {
-	.get_modes = tc_connector_get_modes,
-	.mode_valid = tc_connector_mode_valid,
-	.best_encoder = tc_connector_best_encoder,
-};
+		conn = true;
+	} else {
+		tc_read(GPIOI, &val);
+
+		conn = val & BIT(tc->hpd_pin);
+	}
+
+	if (force && conn)
+		tc_get_display_props(tc);
+
+	if (conn)
+		return connector_status_connected;
+	else
+		return connector_status_disconnected;
+
+err:
+	return connector_status_unknown;
+}
 
 static const struct drm_connector_funcs tc_connector_funcs = {
+	.detect = tc_connector_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = drm_connector_cleanup,
 	.reset = drm_atomic_helper_connector_reset,
@@ -1213,13 +1282,23 @@
 	struct drm_device *drm = bridge->dev;
 	int ret;
 
-	/* Create eDP connector */
+	/* Create DP/eDP connector */
 	drm_connector_helper_add(&tc->connector, &tc_connector_helper_funcs);
 	ret = drm_connector_init(drm, &tc->connector, &tc_connector_funcs,
-				 DRM_MODE_CONNECTOR_eDP);
+				 tc->panel ? DRM_MODE_CONNECTOR_eDP :
+				 DRM_MODE_CONNECTOR_DisplayPort);
 	if (ret)
 		return ret;
 
+	/* Don't poll if don't have HPD connected */
+	if (tc->hpd_pin >= 0) {
+		if (tc->have_irq)
+			tc->connector.polled = DRM_CONNECTOR_POLL_HPD;
+		else
+			tc->connector.polled = DRM_CONNECTOR_POLL_CONNECT |
+					       DRM_CONNECTOR_POLL_DISCONNECT;
+	}
+
 	if (tc->panel)
 		drm_panel_attach(tc->panel, &tc->connector);
 
@@ -1236,6 +1315,7 @@
 
 static const struct drm_bridge_funcs tc_bridge_funcs = {
 	.attach = tc_bridge_attach,
+	.mode_valid = tc_mode_valid,
 	.mode_set = tc_bridge_mode_set,
 	.pre_enable = tc_bridge_pre_enable,
 	.enable = tc_bridge_enable,
@@ -1255,6 +1335,8 @@
 	regmap_reg_range(DP_PHY_CTRL, DP_PHY_CTRL),
 	regmap_reg_range(DP0_PLLCTRL, PXL_PLLCTRL),
 	regmap_reg_range(VFUEN0, VFUEN0),
+	regmap_reg_range(INTSTS_G, INTSTS_G),
+	regmap_reg_range(GPIOI, GPIOI),
 };
 
 static const struct regmap_access_table tc_volatile_table = {
@@ -1283,6 +1365,43 @@
 	.val_format_endian = REGMAP_ENDIAN_LITTLE,
 };
 
+static irqreturn_t tc_irq_handler(int irq, void *arg)
+{
+	struct tc_data *tc = arg;
+	u32 val;
+	int r;
+
+	r = regmap_read(tc->regmap, INTSTS_G, &val);
+	if (r)
+		return IRQ_NONE;
+
+	if (!val)
+		return IRQ_NONE;
+
+	if (val & INT_SYSERR) {
+		u32 stat = 0;
+
+		regmap_read(tc->regmap, SYSSTAT, &stat);
+
+		dev_err(tc->dev, "syserr %x\n", stat);
+	}
+
+	if (tc->hpd_pin >= 0 && tc->bridge.dev) {
+		bool h = val & INT_GPIO_H(tc->hpd_pin);
+		bool lc = val & INT_GPIO_LC(tc->hpd_pin);
+
+		dev_dbg(tc->dev, "GPIO%d: %s %s\n", tc->hpd_pin,
+			h ? "H" : "", lc ? "LC" : "");
+
+		if (h || lc)
+			drm_kms_helper_hotplug_event(tc->bridge.dev);
+	}
+
+	regmap_write(tc->regmap, INTSTS_G, val);
+
+	return IRQ_HANDLED;
+}
+
 static int tc_probe(struct i2c_client *client, const struct i2c_device_id *id)
 {
 	struct device *dev = &client->dev;
@@ -1334,6 +1453,31 @@
 		return ret;
 	}
 
+	ret = of_property_read_u32(dev->of_node, "toshiba,hpd-pin", &tc->hpd_pin);
+	if (ret) {
+		tc->hpd_pin = -ENODEV;
+	} else {
+		if (tc->hpd_pin < 0 || tc->hpd_pin > 1) {
+			dev_err(dev, "failed to parse HPD number\n");
+			return ret;
+		}
+	}
+
+	if (client->irq > 0) {
+		/* enable SysErr */
+		regmap_write(tc->regmap, INTCTL_G, INT_SYSERR);
+
+		ret = devm_request_threaded_irq(dev, client->irq,
+						NULL, tc_irq_handler, IRQF_ONESHOT,
+						"tc358767-irq", tc);
+		if (ret) {
+			dev_err(dev, "failed to register dp interrupt\n");
+			return ret;
+		}
+
+		tc->have_irq = true;
+	}
+
 	ret = regmap_read(tc->regmap, TC_IDREG, &tc->rev);
 	if (ret) {
 		dev_err(tc->dev, "can not read device ID: %d\n", ret);
@@ -1347,6 +1491,22 @@
 
 	tc->assr = (tc->rev == 0x6601); /* Enable ASSR for eDP panels */
 
+	if (tc->hpd_pin >= 0) {
+		u32 lcnt_reg = tc->hpd_pin == 0 ? INT_GP0_LCNT : INT_GP1_LCNT;
+		u32 h_lc = INT_GPIO_H(tc->hpd_pin) | INT_GPIO_LC(tc->hpd_pin);
+
+		/* Set LCNT to 2ms */
+		regmap_write(tc->regmap, lcnt_reg,
+			     clk_get_rate(tc->refclk) * 2 / 1000);
+		/* We need the "alternate" mode for HPD */
+		regmap_write(tc->regmap, GPIOM, BIT(tc->hpd_pin));
+
+		if (tc->have_irq) {
+			/* enable H & LC */
+			regmap_update_bits(tc->regmap, INTCTL_G, h_lc, h_lc);
+		}
+	}
+
 	ret = tc_aux_link_setup(tc);
 	if (ret)
 		return ret;
@@ -1359,12 +1519,6 @@
 	if (ret)
 		return ret;
 
-	ret = tc_get_display_props(tc);
-	if (ret)
-		goto err_unregister_aux;
-
-	tc_connector_set_polling(tc, &tc->connector);
-
 	tc->bridge.funcs = &tc_bridge_funcs;
 	tc->bridge.of_node = dev->of_node;
 	drm_bridge_add(&tc->bridge);
@@ -1372,9 +1526,6 @@
 	i2c_set_clientdata(client, tc);
 
 	return 0;
-err_unregister_aux:
-	drm_dp_aux_unregister(&tc->aux);
-	return ret;
 }
 
 static int tc_remove(struct i2c_client *client)
@@ -1384,8 +1535,6 @@
 	drm_bridge_remove(&tc->bridge);
 	drm_dp_aux_unregister(&tc->aux);
 
-	tc_pxl_pll_dis(tc);
-
 	return 0;
 }
 
diff -Naur --no-dereference a/drivers/gpu/drm/bridge/tc358768.c b/drivers/gpu/drm/bridge/tc358768.c
--- a/drivers/gpu/drm/bridge/tc358768.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/bridge/tc358768.c	2020-06-19 21:00:31.408227715 -0400
@@ -0,0 +1,917 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
+ *  Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/device.h>
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_of.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_mipi_dsi.h>
+#include <video/mipi_display.h>
+#include <video/videomode.h>
+
+/* Global (16-bit addressable) */
+#define TC358768_CHIPID			0x0000
+#define TC358768_SYSCTL			0x0002
+#define TC358768_CONFCTL		0x0004
+#define TC358768_VSDLY			0x0006
+#define TC358768_DATAFMT		0x0008
+#define TC358768_GPIOEN			0x000E
+#define TC358768_GPIODIR		0x0010
+#define TC358768_GPIOIN			0x0012
+#define TC358768_GPIOOUT		0x0014
+#define TC358768_PLLCTL0		0x0016
+#define TC358768_PLLCTL1		0x0018
+#define TC358768_CMDBYTE		0x0022
+#define TC358768_PP_MISC		0x0032
+#define TC358768_DSITX_DT		0x0050
+#define TC358768_FIFOSTATUS		0x00F8
+
+/* Debug (16-bit addressable) */
+#define TC358768_VBUFCTRL		0x00E0
+#define TC358768_DBG_WIDTH		0x00E2
+#define TC358768_DBG_VBLANK		0x00E4
+#define TC358768_DBG_DATA		0x00E8
+
+/* TX PHY (32-bit addressable) */
+#define TC358768_CLW_DPHYCONTTX		0x0100
+#define TC358768_D0W_DPHYCONTTX		0x0104
+#define TC358768_D1W_DPHYCONTTX		0x0108
+#define TC358768_D2W_DPHYCONTTX		0x010C
+#define TC358768_D3W_DPHYCONTTX		0x0110
+#define TC358768_CLW_CNTRL		0x0140
+#define TC358768_D0W_CNTRL		0x0144
+#define TC358768_D1W_CNTRL		0x0148
+#define TC358768_D2W_CNTRL		0x014C
+#define TC358768_D3W_CNTRL		0x0150
+
+/* TX PPI (32-bit addressable) */
+#define TC358768_STARTCNTRL		0x0204
+#define TC358768_DSITXSTATUS		0x0208
+#define TC358768_LINEINITCNT		0x0210
+#define TC358768_LPTXTIMECNT		0x0214
+#define TC358768_TCLK_HEADERCNT		0x0218
+#define TC358768_TCLK_TRAILCNT		0x021C
+#define TC358768_THS_HEADERCNT		0x0220
+#define TC358768_TWAKEUP		0x0224
+#define TC358768_TCLK_POSTCNT		0x0228
+#define TC358768_THS_TRAILCNT		0x022C
+#define TC358768_HSTXVREGCNT		0x0230
+#define TC358768_HSTXVREGEN		0x0234
+#define TC358768_TXOPTIONCNTRL		0x0238
+#define TC358768_BTACNTRL1		0x023C
+
+/* TX CTRL (32-bit addressable) */
+#define TC358768_DSI_CONTROL		0x040C
+#define TC358768_DSI_STATUS		0x0410
+#define TC358768_DSI_INT		0x0414
+#define TC358768_DSI_INT_ENA		0x0418
+#define TC358768_DSICMD_RDFIFO		0x0430
+#define TC358768_DSI_ACKERR		0x0434
+#define TC358768_DSI_ACKERR_INTENA	0x0438
+#define TC358768_DSI_ACKERR_HALT	0x043c
+#define TC358768_DSI_RXERR		0x0440
+#define TC358768_DSI_RXERR_INTENA	0x0444
+#define TC358768_DSI_RXERR_HALT		0x0448
+#define TC358768_DSI_ERR		0x044C
+#define TC358768_DSI_ERR_INTENA		0x0450
+#define TC358768_DSI_ERR_HALT		0x0454
+#define TC358768_DSI_CONFW		0x0500
+#define TC358768_DSI_LPCMD		0x0500
+#define TC358768_DSI_RESET		0x0504
+#define TC358768_DSI_INT_CLR		0x050C
+#define TC358768_DSI_START		0x0518
+
+/* DSITX CTRL (16-bit addressable) */
+#define TC358768_DSICMD_TX		0x0600
+#define TC358768_DSICMD_TYPE		0x0602
+#define TC358768_DSICMD_WC		0x0604
+#define TC358768_DSICMD_WD0		0x0610
+#define TC358768_DSICMD_WD1		0x0612
+#define TC358768_DSICMD_WD2		0x0614
+#define TC358768_DSICMD_WD3		0x0616
+#define TC358768_DSI_EVENT		0x0620
+#define TC358768_DSI_VSW		0x0622
+#define TC358768_DSI_VBPR		0x0624
+#define TC358768_DSI_VACT		0x0626
+#define TC358768_DSI_HSW		0x0628
+#define TC358768_DSI_HBPR		0x062A
+#define TC358768_DSI_HACT		0x062C
+
+struct tc358768_dsi_output {
+	struct mipi_dsi_device *dev;
+	struct drm_panel *panel;
+	struct drm_bridge *bridge;
+};
+
+struct tc358768_priv {
+	struct device *dev;
+	struct regmap *regmap;
+	struct gpio_desc *reset_gpio;
+	int enabled;
+	struct clk *refclk;
+
+	struct mipi_dsi_host dsi_host;
+	struct drm_bridge bridge;
+	struct tc358768_dsi_output output;
+
+	const struct drm_display_mode *mode;
+	u32 pd_lines; /* number of Parallel Port Input Data Lines */
+	u32 dsi_lanes; /* number of DSI Lanes */
+
+	u32 fbd;
+	u32 prd;
+	u32 frs;
+
+	u32 dsiclk; /* pll_clk / 2 */
+};
+
+static inline struct tc358768_priv *dsi_host_to_tc358768(struct mipi_dsi_host
+							 *host)
+{
+	return container_of(host, struct tc358768_priv, dsi_host);
+}
+
+static inline struct tc358768_priv *bridge_to_tc358768(struct drm_bridge
+						       *bridge)
+{
+	return container_of(bridge, struct tc358768_priv, bridge);
+}
+
+static int tc358768_write(struct tc358768_priv *priv, u32 reg, u32 val)
+{
+	size_t count = 2;
+
+	/* 16-bit register? */
+	if (reg < 0x100 || reg >= 0x600)
+		count = 1;
+
+	return regmap_bulk_write(priv->regmap, reg, &val, count);
+}
+
+static int tc358768_read(struct tc358768_priv *priv, u32 reg, u32 *val)
+{
+	size_t count = 2;
+
+	/* 16-bit register? */
+	if (reg < 0x100 || reg >= 0x600) {
+		*val = 0;
+		count = 1;
+	}
+
+	return regmap_bulk_read(priv->regmap, reg, val, count);
+}
+
+static int tc358768_update_bits(struct tc358768_priv *priv, u32 reg, u32 mask,
+				u32 val)
+{
+	int ret;
+	u32 tmp, orig;
+
+	ret = tc358768_read(priv, reg, &orig);
+	if (ret != 0)
+		return ret;
+
+	tmp = orig & ~mask;
+	tmp |= val & mask;
+
+	if (tmp != orig)
+		ret = tc358768_write(priv, reg, tmp);
+
+	return ret;
+}
+
+static void tc358768_sw_reset(struct tc358768_priv *priv)
+{
+	/* Assert Reset */
+	tc358768_write(priv, TC358768_SYSCTL, 1);
+	/* Release Reset, Exit Sleep */
+	tc358768_write(priv, TC358768_SYSCTL, 0);
+}
+
+static void tc358768_hw_enable(struct tc358768_priv *priv, bool enable)
+{
+	if (!priv->reset_gpio)
+		return;
+
+	if (enable && priv->enabled++ > 0)
+		return;
+
+	if (!enable && --priv->enabled != 0)
+		return;
+
+	gpiod_set_value_cansleep(priv->reset_gpio, enable);
+	regcache_cache_only(priv->regmap, !enable);
+	if (enable) {
+		/* wait for encoder clocks to stabilize */
+		usleep_range(1000, 2000);
+
+		regcache_sync(priv->regmap);
+	}
+}
+
+static u32 tc358768_pll_to_pclk(struct tc358768_priv *priv, u32 pll_clk)
+{
+	return (u32)div_u64((u64)pll_clk * priv->dsi_lanes, priv->pd_lines);
+}
+
+static u32 tc358768_pclk_to_pll(struct tc358768_priv *priv, u32 pclk)
+{
+	return (u32)div_u64((u64)pclk * priv->pd_lines, priv->dsi_lanes);
+}
+
+static int tc358768_calc_pll(struct tc358768_priv *priv)
+{
+	const u32 frs_limits[] = {
+		1000000000,
+		500000000,
+		250000000,
+		125000000,
+		62500000
+	};
+	unsigned long refclk;
+	u32 prd, target_pll, i, max_pll, min_pll;
+	u32 frs, best_diff, best_pll, best_prd, best_fbd;
+
+	target_pll = tc358768_pclk_to_pll(priv, priv->mode->clock * 1000);
+
+	/* pll_clk = RefClk * [(FBD + 1)/ (PRD + 1)] * [1 / (2^FRS)] */
+
+	frs = UINT_MAX;
+
+	for (i = 0; i < ARRAY_SIZE(frs_limits) - 1; i++) {
+		if (target_pll < frs_limits[i] &&
+		    target_pll >= frs_limits[i + 1]) {
+			frs = i;
+			max_pll = frs_limits[i];
+			min_pll = frs_limits[i + 1];
+			break;
+		}
+	}
+
+	if (frs == UINT_MAX)
+		return -EINVAL;
+
+	refclk = clk_get_rate(priv->refclk);
+
+	best_diff = UINT_MAX;
+	best_pll = 0;
+	best_prd = 0;
+	best_fbd = 0;
+
+	for (prd = 0; prd < 16; ++prd) {
+		u32 divisor = (prd + 1) * (1 << frs);
+		u32 fbd;
+
+		for (fbd = 0; fbd < 512; ++fbd) {
+			u32 pll, diff;
+
+			pll = (u32)div_u64((u64)refclk * (fbd + 1), divisor);
+
+			if (pll >= max_pll || pll < min_pll)
+				continue;
+
+			diff = max(pll, target_pll) - min(pll, target_pll);
+
+			if (diff < best_diff) {
+				best_diff = diff;
+				best_pll = pll;
+				best_prd = prd;
+				best_fbd = fbd;
+			}
+
+			if (best_diff == 0)
+				break;
+		}
+
+		if (best_diff == 0)
+			break;
+	}
+
+	if (best_diff == UINT_MAX) {
+		dev_err(priv->dev, "could not find suitable PLL setup\n");
+		return -EINVAL;
+	}
+
+	priv->fbd = best_fbd;
+	priv->prd = best_prd;
+	priv->frs = frs;
+	priv->dsiclk = best_pll / 2;
+
+	return 0;
+}
+
+static int tc358768_dsi_host_attach(struct mipi_dsi_host *host,
+				    struct mipi_dsi_device *dev)
+{
+	struct tc358768_priv *priv = dsi_host_to_tc358768(host);
+	struct drm_bridge *bridge;
+	struct drm_panel *panel;
+	struct device_node *ep;
+	int ret;
+
+	if (dev->lanes > 4) {
+		dev_err(priv->dev, "unsupported number of data lanes(%u)\n",
+			dev->lanes);
+		return -EINVAL;
+	}
+
+	/*
+	 * tc358768 supports both Video and Pulse mode, but the driver only
+	 * implements Video (event) mode currently
+	 */
+	if (!(dev->mode_flags & MIPI_DSI_MODE_VIDEO)) {
+		dev_err(priv->dev, "Only MIPI_DSI_MODE_VIDEO is supported\n");
+		return -ENOTSUPP;
+	}
+
+	/*
+	 * tc358768 supports RGB888, RGB666, RGB666_PACKED and RGB565, but only
+	 * RGB888 is verified.
+	 */
+	if (dev->format != MIPI_DSI_FMT_RGB888) {
+		dev_warn(priv->dev, "Only MIPI_DSI_FMT_RGB888 tested!\n");
+		return -ENOTSUPP;
+	}
+
+	ret = drm_of_find_panel_or_bridge(host->dev->of_node, 1, 0, &panel,
+					  &bridge);
+	if (ret)
+		return ret;
+
+	if (panel) {
+		bridge = drm_panel_bridge_add(panel, DRM_MODE_CONNECTOR_DSI);
+		if (IS_ERR(bridge))
+			return PTR_ERR(bridge);
+	}
+
+	priv->output.dev = dev;
+	priv->output.bridge = bridge;
+	priv->output.panel = panel;
+
+	priv->dsi_lanes = dev->lanes;
+
+	/* get input ep (port0/endpoint0) */
+	ret = -EINVAL;
+	ep = of_graph_get_endpoint_by_regs(host->dev->of_node, 0, 0);
+	if (ep) {
+		ret = of_property_read_u32(ep, "data-lines", &priv->pd_lines);
+
+		of_node_put(ep);
+	}
+
+	if (ret)
+		priv->pd_lines = mipi_dsi_pixel_format_to_bpp(dev->format);
+
+	drm_bridge_add(&priv->bridge);
+
+	return 0;
+}
+
+static int tc358768_dsi_host_detach(struct mipi_dsi_host *host,
+				    struct mipi_dsi_device *dev)
+{
+	struct tc358768_priv *priv = dsi_host_to_tc358768(host);
+
+	drm_bridge_remove(&priv->bridge);
+	if (priv->output.panel)
+		drm_panel_bridge_remove(priv->output.bridge);
+
+	return 0;
+}
+
+static ssize_t tc358768_dsi_host_transfer(struct mipi_dsi_host *host,
+					  const struct mipi_dsi_msg *msg)
+{
+	struct tc358768_priv *priv = dsi_host_to_tc358768(host);
+	struct mipi_dsi_packet packet;
+	int ret;
+
+	if (msg->rx_len) {
+		dev_warn(priv->dev, "MIPI rx is not supported\n");
+		return -ENOTSUPP;
+	}
+
+	if (msg->tx_len > 8) {
+		dev_warn(priv->dev, "Maximum 8 byte MIPI tx is supported\n");
+		return -ENOTSUPP;
+	}
+
+	ret = mipi_dsi_create_packet(&packet, msg);
+	if (ret)
+		return ret;
+
+	tc358768_hw_enable(priv, true);
+
+	if (mipi_dsi_packet_format_is_short(msg->type)) {
+		tc358768_write(priv, TC358768_DSICMD_TYPE,
+			       (0x10 << 8) | (packet.header[0] & 0x3f));
+		tc358768_write(priv, TC358768_DSICMD_WC, 0);
+		tc358768_write(priv, TC358768_DSICMD_WD0,
+			       (packet.header[2] << 8) | packet.header[1]);
+	} else {
+		int i, j;
+
+		tc358768_write(priv, TC358768_DSICMD_TYPE,
+			       (0x40 << 8) | (packet.header[0] & 0x3f));
+		tc358768_write(priv, TC358768_DSICMD_WC, packet.payload_length);
+		for (i = 0; i < packet.payload_length; i += 2) {
+			u16 val = 0;
+
+			for (j = 0; j < 2 && i + j < packet.payload_length; j++)
+				val |= packet.payload[i + j] << (8 * j);
+
+			tc358768_write(priv, TC358768_DSICMD_WD0 + i, val);
+		}
+	}
+
+	/* start transfer */
+	tc358768_write(priv, TC358768_DSICMD_TX, 1);
+
+	tc358768_hw_enable(priv, false);
+
+	return 0;
+}
+
+static const struct mipi_dsi_host_ops tc358768_dsi_host_ops = {
+	.attach = tc358768_dsi_host_attach,
+	.detach = tc358768_dsi_host_detach,
+	.transfer = tc358768_dsi_host_transfer,
+};
+
+static int tc358768_bridge_attach(struct drm_bridge *bridge)
+{
+	struct tc358768_priv *priv = bridge_to_tc358768(bridge);
+
+	if (!drm_core_check_feature(bridge->dev, DRIVER_ATOMIC)) {
+		dev_err(priv->dev, "needs atomic updates support\n");
+		return -ENOTSUPP;
+	}
+
+	return drm_bridge_attach(bridge->encoder, priv->output.bridge, bridge);
+}
+
+static enum drm_mode_status
+tc358768_bridge_mode_valid(struct drm_bridge *bridge,
+			   const struct drm_display_mode *mode)
+{
+	struct tc358768_priv *priv = bridge_to_tc358768(bridge);
+
+	priv->mode = mode;
+
+	if (tc358768_calc_pll(priv))
+		return MODE_CLOCK_RANGE;
+
+	return MODE_OK;
+}
+
+static void tc358768_bridge_disable(struct drm_bridge *bridge)
+{
+	struct tc358768_priv *priv = bridge_to_tc358768(bridge);
+
+	/* set FrmStop */
+	tc358768_update_bits(priv, TC358768_PP_MISC, BIT(15), BIT(15));
+
+	/* wait at least for one frame */
+	msleep(50);
+
+	/* clear PP_en */
+	tc358768_update_bits(priv, TC358768_CONFCTL, BIT(6), 0);
+
+	/* set RstPtr */
+	tc358768_update_bits(priv, TC358768_PP_MISC, BIT(14), BIT(14));
+
+	tc358768_hw_enable(priv, false);
+}
+
+static void tc358768_setup_pll(struct tc358768_priv *priv)
+{
+	u32 fbd, prd, frs;
+
+	fbd = priv->fbd;
+	prd = priv->prd;
+	frs = priv->frs;
+
+	dev_dbg(priv->dev, "PLL: refclk %lu, fbd %u, prd %u, frs %u\n",
+		clk_get_rate(priv->refclk), fbd, prd, frs);
+	dev_dbg(priv->dev, "PLL: pll_clk: %u, DSIClk %u, DSIByteClk %u\n",
+		priv->dsiclk * 2, priv->dsiclk, priv->dsiclk / 4);
+	dev_dbg(priv->dev, "PLL: pclk %u (panel: %u)\n",
+		tc358768_pll_to_pclk(priv, priv->dsiclk * 2),
+		priv->mode->clock * 1000);
+
+	/* PRD[15:12] FBD[8:0] */
+	tc358768_write(priv, TC358768_PLLCTL0, (prd << 12) | fbd);
+
+	/* FRS[11:10] LBWS[9:8] CKEN[4] RESETB[1] EN[0] */
+	tc358768_write(priv, TC358768_PLLCTL1,
+		       (frs << 10) | (0x2 << 8) | BIT(1) | BIT(0));
+
+	/* wait for lock */
+	usleep_range(1000, 2000);
+
+	/* FRS[11:10] LBWS[9:8] CKEN[4] PLL_CKEN[4] RESETB[1] EN[0] */
+	tc358768_write(priv, TC358768_PLLCTL1,
+		       (frs << 10) | (0x2 << 8) | BIT(4) | BIT(1) | BIT(0));
+}
+
+#define TC358768_PRECISION	1000
+static u32 tc358768_ns_to_cnt(u32 ns, u32 period_nsk)
+{
+	return (ns * TC358768_PRECISION + period_nsk) / period_nsk;
+}
+
+static u32 tc358768_to_ns(u32 nsk)
+{
+	return (nsk / TC358768_PRECISION);
+}
+
+static void tc358768_bridge_enable(struct drm_bridge *bridge)
+{
+	struct tc358768_priv *priv = bridge_to_tc358768(bridge);
+	const struct drm_display_mode *mode = priv->mode;
+	struct mipi_dsi_device *dsi_dev = priv->output.dev;
+	u32 val, val2, lptxcnt, hact, data_type;
+	u32 dsiclk = priv->dsiclk;
+	u32 dsibclk = dsiclk / 4;
+	u32 dsibclk_nsk, dsiclk_nsk, ui_nsk, phy_delay_nsk;
+	int i;
+
+	tc358768_hw_enable(priv, true);
+
+	tc358768_sw_reset(priv);
+
+	tc358768_setup_pll(priv);
+
+	/* VSDly[9:0] */
+	tc358768_write(priv, TC358768_VSDLY, 1);
+
+	/* Data Format Control Register */
+	val = BIT(2) | BIT(1) | BIT(0); /* rdswap_en | dsitx_en | txdt_en */
+	switch (dsi_dev->format) {
+	case MIPI_DSI_FMT_RGB888:
+		val |= (0x3 << 4);
+		hact = mode->hdisplay * 3;
+		data_type = MIPI_DSI_PACKED_PIXEL_STREAM_24;
+		break;
+	case MIPI_DSI_FMT_RGB666:
+		val |= (0x4 << 4);
+		hact = mode->hdisplay * 3;
+		data_type = MIPI_DSI_PACKED_PIXEL_STREAM_18;
+		break;
+
+	case MIPI_DSI_FMT_RGB666_PACKED:
+		val |= (0x4 << 4) | BIT(3);
+		hact = mode->hdisplay * 18 / 8;
+		data_type = MIPI_DSI_PIXEL_STREAM_3BYTE_18;
+		break;
+
+	case MIPI_DSI_FMT_RGB565:
+		val |= (0x5 << 4);
+		hact = mode->hdisplay * 2;
+		data_type = MIPI_DSI_PACKED_PIXEL_STREAM_16;
+		break;
+	default:
+		dev_err(priv->dev, "Invalid data format (%u)\n",
+			dsi_dev->format);
+		return;
+	}
+
+	tc358768_write(priv, TC358768_DATAFMT, val);
+	tc358768_write(priv, TC358768_DSITX_DT, data_type);
+
+	/* Enable D-PHY (HiZ->LP11) */
+	tc358768_write(priv, TC358768_CLW_CNTRL, 0x0000);
+	/* Enable lanes */
+	for (i = 0; i < dsi_dev->lanes; i++)
+		tc358768_write(priv, TC358768_D0W_CNTRL + i * 4, 0x0000);
+
+	/* DSI Timings */
+	dsibclk_nsk = (u32)div_u64((u64)1000000000 * TC358768_PRECISION,
+				  dsibclk);
+	dsiclk_nsk = (u32)div_u64((u64)1000000000 * TC358768_PRECISION, dsiclk);
+	ui_nsk = dsiclk_nsk / 2;
+	phy_delay_nsk = dsibclk_nsk + 2 * dsiclk_nsk;
+	dev_dbg(priv->dev, "dsiclk_nsk: %u\n", dsiclk_nsk);
+	dev_dbg(priv->dev, "ui_nsk: %u\n", ui_nsk);
+	dev_dbg(priv->dev, "dsibclk_nsk: %u\n", dsibclk_nsk);
+	dev_dbg(priv->dev, "phy_delay_nsk: %u\n", phy_delay_nsk);
+
+	/* LP11 > 100us for D-PHY Rx Init */
+	val = tc358768_ns_to_cnt(100 * 1000, dsibclk_nsk) - 1;
+	dev_dbg(priv->dev, "LINEINITCNT: 0x%x\n", val);
+	tc358768_write(priv, TC358768_LINEINITCNT, val);
+
+	/* LPTimeCnt > 50ns */
+	val = tc358768_ns_to_cnt(50, dsibclk_nsk) - 1;
+	lptxcnt = val;
+	dev_dbg(priv->dev, "LPTXTIMECNT: 0x%x\n", val);
+	tc358768_write(priv, TC358768_LPTXTIMECNT, val);
+
+	/* 38ns < TCLK_PREPARE < 95ns */
+	val = tc358768_ns_to_cnt(65, dsibclk_nsk) - 1;
+	/* TCLK_PREPARE > 300ns */
+	val2 = tc358768_ns_to_cnt(300 + tc358768_to_ns(3 * ui_nsk),
+				  dsibclk_nsk);
+	val |= (val2 - tc358768_to_ns(phy_delay_nsk - dsibclk_nsk)) << 8;
+	dev_dbg(priv->dev, "TCLK_HEADERCNT: 0x%x\n", val);
+	tc358768_write(priv, TC358768_TCLK_HEADERCNT, val);
+
+	/* TCLK_TRAIL > 60ns + 3*UI */
+	val = 60 + tc358768_to_ns(3 * ui_nsk);
+	val = tc358768_ns_to_cnt(val, dsibclk_nsk) - 5;
+	dev_dbg(priv->dev, "TCLK_TRAILCNT: 0x%x\n", val);
+	tc358768_write(priv, TC358768_TCLK_TRAILCNT, val);
+
+	/* 40ns + 4*UI < THS_PREPARE < 85ns + 6*UI */
+	val = 50 + tc358768_to_ns(4 * ui_nsk);
+	val = tc358768_ns_to_cnt(val, dsibclk_nsk) - 1;
+	/* THS_ZERO > 145ns + 10*UI */
+	val2 = tc358768_ns_to_cnt(145 - tc358768_to_ns(ui_nsk), dsibclk_nsk);
+	val |= (val2 - tc358768_to_ns(phy_delay_nsk)) << 8;
+	dev_dbg(priv->dev, "THS_HEADERCNT: 0x%x\n", val);
+	tc358768_write(priv, TC358768_THS_HEADERCNT, val);
+
+	/* TWAKEUP > 1ms in lptxcnt steps */
+	val = tc358768_ns_to_cnt(1020000, dsibclk_nsk);
+	val = val / (lptxcnt + 1) - 1;
+	dev_dbg(priv->dev, "TWAKEUP: 0x%x\n", val);
+	tc358768_write(priv, TC358768_TWAKEUP, val);
+
+	/* TCLK_POSTCNT > 60ns + 52*UI */
+	val = tc358768_ns_to_cnt(60 + tc358768_to_ns(52 * ui_nsk),
+				 dsibclk_nsk) - 3;
+	dev_dbg(priv->dev, "TCLK_POSTCNT: 0x%x\n", val);
+	tc358768_write(priv, TC358768_TCLK_POSTCNT, val);
+
+	/* 60ns + 4*UI < THS_PREPARE < 105ns + 12*UI */
+	val = tc358768_ns_to_cnt(60 + tc358768_to_ns(15 * ui_nsk),
+				 dsibclk_nsk) - 5;
+	dev_dbg(priv->dev, "THS_TRAILCNT: 0x%x\n", val);
+	tc358768_write(priv, TC358768_THS_TRAILCNT, val);
+
+	val = BIT(0);
+	for (i = 0; i < dsi_dev->lanes; i++)
+		val |= BIT(i + 1);
+	tc358768_write(priv, TC358768_HSTXVREGEN, val);
+
+	if (!(dsi_dev->mode_flags & MIPI_DSI_CLOCK_NON_CONTINUOUS))
+		tc358768_write(priv, TC358768_TXOPTIONCNTRL, 0x1);
+
+	/* TXTAGOCNT[26:16] RXTASURECNT[10:0] */
+	val = tc358768_to_ns((lptxcnt + 1) * dsibclk_nsk * 4);
+	val = tc358768_ns_to_cnt(val, dsibclk_nsk) - 1;
+	val2 = tc358768_ns_to_cnt(tc358768_to_ns((lptxcnt + 1) * dsibclk_nsk),
+				  dsibclk_nsk) - 2;
+	val |= val2 << 16;
+	dev_dbg(priv->dev, "BTACNTRL1: 0x%x\n", val);
+	tc358768_write(priv, TC358768_BTACNTRL1, val);
+
+	/* START[0] */
+	tc358768_write(priv, TC358768_STARTCNTRL, 1);
+
+	/* Set event mode */
+	tc358768_write(priv, TC358768_DSI_EVENT, 1);
+
+	/* vsw (+ vbp) */
+	tc358768_write(priv, TC358768_DSI_VSW,
+		       mode->vtotal - mode->vsync_start);
+	/* vbp (not used in event mode) */
+	tc358768_write(priv, TC358768_DSI_VBPR, 0);
+	/* vact */
+	tc358768_write(priv, TC358768_DSI_VACT, mode->vdisplay);
+
+	/* (hsw + hbp) * byteclk * ndl / pclk */
+	val = (u32)div_u64((mode->htotal - mode->hsync_start) *
+			   ((u64)priv->dsiclk / 4) * priv->dsi_lanes,
+			   mode->clock * 1000);
+	tc358768_write(priv, TC358768_DSI_HSW, val);
+	/* hbp (not used in event mode) */
+	tc358768_write(priv, TC358768_DSI_HBPR, 0);
+	/* hact (bytes) */
+	tc358768_write(priv, TC358768_DSI_HACT, hact);
+
+	/* VSYNC polarity */
+	if (!(mode->flags & DRM_MODE_FLAG_NVSYNC))
+		tc358768_update_bits(priv, TC358768_CONFCTL, BIT(5), BIT(5));
+	/* HSYNC polarity */
+	if (mode->flags & DRM_MODE_FLAG_PHSYNC)
+		tc358768_update_bits(priv, TC358768_PP_MISC, BIT(0), BIT(0));
+
+	/* Start DSI Tx */
+	tc358768_write(priv, TC358768_DSI_START, 0x1);
+
+	/* Configure DSI_Control register */
+	val = (6 << 29) | (0x3 << 24); /* CLEAR, DSI_Control */
+	val |= BIT(7) | BIT(5) | 0x3 << 1 | BIT(0);
+	tc358768_write(priv, TC358768_DSI_CONFW, val);
+
+	val = (5 << 29) | (0x3 << 24); /* SET, DSI_Control */
+	if (!(dsi_dev->mode_flags & MIPI_DSI_MODE_LPM))
+		val |= BIT(7);
+	if (!(dsi_dev->mode_flags & MIPI_DSI_CLOCK_NON_CONTINUOUS))
+		val |= BIT(5);
+	val |= (dsi_dev->lanes - 1) << 1;
+	if (dsi_dev->mode_flags & MIPI_DSI_MODE_EOT_PACKET)
+		val |= BIT(0);
+	tc358768_write(priv, TC358768_DSI_CONFW, val);
+
+	val = (6 << 29) | (0x3 << 24); /* CLEAR, DSI_Control */
+	val |= 0x8000; /* DSI mode */
+	tc358768_write(priv, TC358768_DSI_CONFW, val);
+
+	/* clear FrmStop and RstPtr */
+	tc358768_update_bits(priv, TC358768_PP_MISC, 0x3 << 14, 0);
+
+	/* set PP_en */
+	tc358768_update_bits(priv, TC358768_CONFCTL, BIT(6), BIT(6));
+}
+
+static const struct drm_bridge_funcs tc358768_bridge_funcs = {
+	.attach = tc358768_bridge_attach,
+	.mode_valid = tc358768_bridge_mode_valid,
+	.disable = tc358768_bridge_disable,
+	.enable = tc358768_bridge_enable,
+};
+
+static const struct drm_bridge_timings default_tc358768_timings = {
+	.input_bus_flags = DRM_BUS_FLAG_PIXDATA_SAMPLE_POSEDGE
+		 | DRM_BUS_FLAG_SYNC_SAMPLE_NEGEDGE
+		 | DRM_BUS_FLAG_DE_HIGH,
+};
+
+static bool tc358768_is_reserved_reg(unsigned int reg)
+{
+	switch (reg) {
+	case 0x114 ... 0x13f:
+	case 0x200:
+	case 0x20c:
+	case 0x400 ... 0x408:
+	case 0x41c ... 0x42f:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool tc358768_writeable_reg(struct device *dev, unsigned int reg)
+{
+	if (tc358768_is_reserved_reg(reg))
+		return false;
+
+	switch (reg) {
+	case TC358768_CHIPID:
+	case TC358768_FIFOSTATUS:
+	case TC358768_DSITXSTATUS ... (TC358768_DSITXSTATUS + 2):
+	case TC358768_DSI_CONTROL ... (TC358768_DSI_INT_ENA + 2):
+	case TC358768_DSICMD_RDFIFO ... (TC358768_DSI_ERR_HALT + 2):
+		return false;
+	default:
+		return true;
+	}
+}
+
+static bool tc358768_readable_reg(struct device *dev, unsigned int reg)
+{
+	if (tc358768_is_reserved_reg(reg))
+		return false;
+
+	switch (reg) {
+	case TC358768_STARTCNTRL:
+	case TC358768_DSI_CONFW ... (TC358768_DSI_CONFW + 2):
+	case TC358768_DSI_INT_CLR ... (TC358768_DSI_INT_CLR + 2):
+	case TC358768_DSI_START ... (TC358768_DSI_START + 2):
+	case TC358768_DBG_DATA:
+		return false;
+	default:
+		return true;
+	}
+}
+
+static bool tc358768_volatile_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case TC358768_FIFOSTATUS:
+	case TC358768_STARTCNTRL ... (TC358768_DSITXSTATUS + 2):
+	case TC358768_DSI_CONTROL ... (TC358768_DSI_INT_ENA + 2):
+	case TC358768_DSICMD_RDFIFO ... (TC358768_DSI_ERR_HALT + 2):
+	case TC358768_DSICMD_TX:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static const struct regmap_config tc358768_regmap_config = {
+	.name = "tc358768",
+	.reg_bits = 16,
+	.val_bits = 16,
+	.max_register = TC358768_DSI_HACT,
+	.cache_type = REGCACHE_RBTREE,
+	.writeable_reg = tc358768_writeable_reg,
+	.readable_reg = tc358768_readable_reg,
+	.volatile_reg = tc358768_volatile_reg,
+	.reg_format_endian = REGMAP_ENDIAN_BIG,
+	.val_format_endian = REGMAP_ENDIAN_BIG,
+};
+
+static const struct i2c_device_id tc358768_i2c_ids[] = {
+	{ "tc358768", 0 },
+	{ "tc358778", 0 },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, tc358768_i2c_ids);
+
+static const struct of_device_id tc358768_of_ids[] = {
+	{ .compatible = "toshiba,tc358768", },
+	{ .compatible = "toshiba,tc358778", },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, tc358768_of_ids);
+
+static int tc358768_i2c_probe(struct i2c_client *client,
+			      const struct i2c_device_id *id)
+{
+	struct tc358768_priv *priv;
+	struct device *dev = &client->dev;
+	struct device_node *np = dev->of_node;
+	int ret;
+
+	if (!np)
+		return -ENODEV;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	dev_set_drvdata(dev, priv);
+	priv->dev = dev;
+
+	priv->refclk = devm_clk_get(dev, "refclk");
+	if (IS_ERR(priv->refclk))
+		return PTR_ERR(priv->refclk);
+
+	priv->reset_gpio  = devm_gpiod_get_optional(dev, "reset",
+						    GPIOD_OUT_LOW);
+	if (IS_ERR(priv->reset_gpio))
+		return PTR_ERR(priv->reset_gpio);
+
+	priv->regmap = devm_regmap_init_i2c(client, &tc358768_regmap_config);
+	if (IS_ERR(priv->regmap)) {
+		dev_err(dev, "Failed to init regmap\n");
+		return PTR_ERR(priv->regmap);
+	}
+
+	if (priv->reset_gpio)
+		regcache_cache_only(priv->regmap, true);
+
+	priv->dsi_host.dev = dev;
+	priv->dsi_host.ops = &tc358768_dsi_host_ops;
+
+	priv->bridge.funcs = &tc358768_bridge_funcs;
+	priv->bridge.timings = &default_tc358768_timings;
+	priv->bridge.of_node = np;
+
+	i2c_set_clientdata(client, priv);
+
+	ret = mipi_dsi_host_register(&priv->dsi_host);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int tc358768_i2c_remove(struct i2c_client *client)
+{
+	struct tc358768_priv *priv = i2c_get_clientdata(client);
+
+	mipi_dsi_host_unregister(&priv->dsi_host);
+
+	return 0;
+}
+
+static struct i2c_driver tc358768_driver = {
+	.driver = {
+		.name = "tc358768",
+		.of_match_table = tc358768_of_ids,
+	},
+	.id_table = tc358768_i2c_ids,
+	.probe = tc358768_i2c_probe,
+	.remove	= tc358768_i2c_remove,
+};
+module_i2c_driver(tc358768_driver);
+
+MODULE_AUTHOR("Peter Ujfalusi <peter.ujfalusi@ti.com>");
+MODULE_DESCRIPTION("TC358768AXBG/TC358778XBG DSI bridge");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/gpu/drm/bridge/ti-tfp410.c b/drivers/gpu/drm/bridge/ti-tfp410.c
--- a/drivers/gpu/drm/bridge/ti-tfp410.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/bridge/ti-tfp410.c	2020-06-19 21:00:31.408227715 -0400
@@ -27,10 +27,16 @@
 struct tfp410 {
 	struct drm_bridge	bridge;
 	struct drm_connector	connector;
+	unsigned int		connector_type;
 
+	u32			bus_format;
 	struct i2c_adapter	*ddc;
 	struct gpio_desc	*hpd;
+	int			hpd_irq;
 	struct delayed_work	hpd_work;
+	struct gpio_desc	*powerdown;
+
+	struct drm_bridge_timings timings;
 
 	struct device *dev;
 };
@@ -125,26 +131,57 @@
 		return -ENODEV;
 	}
 
-	if (dvi->hpd)
+	if (dvi->hpd_irq >= 0)
 		dvi->connector.polled = DRM_CONNECTOR_POLL_HPD;
+	else
+		dvi->connector.polled = DRM_CONNECTOR_POLL_CONNECT | DRM_CONNECTOR_POLL_DISCONNECT;
 
 	drm_connector_helper_add(&dvi->connector,
 				 &tfp410_con_helper_funcs);
 	ret = drm_connector_init(bridge->dev, &dvi->connector,
-				 &tfp410_con_funcs, DRM_MODE_CONNECTOR_HDMIA);
+				 &tfp410_con_funcs, dvi->connector_type);
 	if (ret) {
 		dev_err(dvi->dev, "drm_connector_init() failed: %d\n", ret);
 		return ret;
 	}
 
+	drm_display_info_set_bus_formats(&dvi->connector.display_info,
+					 &dvi->bus_format, 1);
+
 	drm_connector_attach_encoder(&dvi->connector,
 					  bridge->encoder);
 
 	return 0;
 }
 
+static void tfp410_enable(struct drm_bridge *bridge)
+{
+	struct tfp410 *dvi = drm_bridge_to_tfp410(bridge);
+
+	gpiod_set_value_cansleep(dvi->powerdown, 0);
+}
+
+static void tfp410_disable(struct drm_bridge *bridge)
+{
+	struct tfp410 *dvi = drm_bridge_to_tfp410(bridge);
+
+	gpiod_set_value_cansleep(dvi->powerdown, 1);
+}
+
+static enum drm_mode_status tfp410_mode_valid(struct drm_bridge *bridge,
+					      const struct drm_display_mode *mode)
+{
+	if (mode->clock < 25000 || mode->clock > 165000)
+		return MODE_BAD;
+
+	return MODE_OK;
+}
+
 static const struct drm_bridge_funcs tfp410_bridge_funcs = {
 	.attach		= tfp410_attach,
+	.enable		= tfp410_enable,
+	.disable	= tfp410_disable,
+	.mode_valid	= tfp410_mode_valid,
 };
 
 static void tfp410_hpd_work_func(struct work_struct *work)
@@ -167,6 +204,83 @@
 	return IRQ_HANDLED;
 }
 
+static const struct drm_bridge_timings tfp410_default_timings = {
+	.input_bus_flags = DRM_BUS_FLAG_PIXDATA_SAMPLE_POSEDGE
+			 | DRM_BUS_FLAG_DE_HIGH,
+	.setup_time_ps = 1200,
+	.hold_time_ps = 1300,
+};
+
+static int tfp410_parse_timings(struct tfp410 *dvi, bool i2c)
+{
+	struct drm_bridge_timings *timings = &dvi->timings;
+	struct device_node *ep;
+	u32 pclk_sample = 0;
+	u32 bus_width = 24;
+	s32 deskew = 0;
+
+	/* Start with defaults. */
+	*timings = tfp410_default_timings;
+
+	if (i2c)
+		/*
+		 * In I2C mode timings are configured through the I2C interface.
+		 * As the driver doesn't support I2C configuration yet, we just
+		 * go with the defaults (BSEL=1, DSEL=1, DKEN=0, EDGE=1).
+		 */
+		return 0;
+
+	/*
+	 * In non-I2C mode, timings are configured through the BSEL, DSEL, DKEN
+	 * and EDGE pins. They are specified in DT through endpoint properties
+	 * and vendor-specific properties.
+	 */
+	ep = of_graph_get_endpoint_by_regs(dvi->dev->of_node, 0, 0);
+	if (!ep)
+		return -EINVAL;
+
+	/* Get the sampling edge from the endpoint. */
+	of_property_read_u32(ep, "pclk-sample", &pclk_sample);
+	of_property_read_u32(ep, "bus-width", &bus_width);
+	of_node_put(ep);
+
+	timings->input_bus_flags = DRM_BUS_FLAG_DE_HIGH;
+
+	switch (pclk_sample) {
+	case 0:
+		timings->input_bus_flags |= DRM_BUS_FLAG_PIXDATA_SAMPLE_NEGEDGE
+					 |  DRM_BUS_FLAG_SYNC_SAMPLE_NEGEDGE;
+		break;
+	case 1:
+		timings->input_bus_flags |= DRM_BUS_FLAG_PIXDATA_SAMPLE_POSEDGE
+					 |  DRM_BUS_FLAG_SYNC_SAMPLE_POSEDGE;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	switch (bus_width) {
+	case 12:
+		dvi->bus_format = MEDIA_BUS_FMT_RGB888_2X12_LE;
+		break;
+	case 24:
+		dvi->bus_format = MEDIA_BUS_FMT_RGB888_1X24;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* Get the setup and hold time from vendor-specific properties. */
+	of_property_read_u32(dvi->dev->of_node, "ti,deskew", (u32 *)&deskew);
+	if (deskew < -4 || deskew > 3)
+		return -EINVAL;
+
+	timings->setup_time_ps = min(0, 1200 - 350 * deskew);
+	timings->hold_time_ps = min(0, 1300 + 350 * deskew);
+
+	return 0;
+}
+
 static int tfp410_get_connector_properties(struct tfp410 *dvi)
 {
 	struct device_node *connector_node, *ddc_phandle;
@@ -177,6 +291,11 @@
 	if (!connector_node)
 		return -ENODEV;
 
+	if (of_device_is_compatible(connector_node, "hdmi-connector"))
+		dvi->connector_type = DRM_MODE_CONNECTOR_HDMIA;
+	else
+		dvi->connector_type = DRM_MODE_CONNECTOR_DVID;
+
 	dvi->hpd = fwnode_get_named_gpiod(&connector_node->fwnode,
 					"hpd-gpios", 0, GPIOD_IN, "hpd");
 	if (IS_ERR(dvi->hpd)) {
@@ -205,7 +324,7 @@
 	return ret;
 }
 
-static int tfp410_init(struct device *dev)
+static int tfp410_init(struct device *dev, bool i2c)
 {
 	struct tfp410 *dvi;
 	int ret;
@@ -222,16 +341,33 @@
 
 	dvi->bridge.funcs = &tfp410_bridge_funcs;
 	dvi->bridge.of_node = dev->of_node;
+	dvi->bridge.timings = &dvi->timings;
 	dvi->dev = dev;
 
+	ret = tfp410_parse_timings(dvi, i2c);
+	if (ret)
+		goto fail;
+
 	ret = tfp410_get_connector_properties(dvi);
 	if (ret)
 		goto fail;
 
-	if (dvi->hpd) {
+	dvi->powerdown = devm_gpiod_get_optional(dev, "powerdown",
+						 GPIOD_OUT_HIGH);
+	if (IS_ERR(dvi->powerdown)) {
+		dev_err(dev, "failed to parse powerdown gpio\n");
+		return PTR_ERR(dvi->powerdown);
+	}
+
+	if (dvi->hpd)
+		dvi->hpd_irq = gpiod_to_irq(dvi->hpd);
+	else
+		dvi->hpd_irq = -ENXIO;
+
+	if (dvi->hpd_irq >= 0) {
 		INIT_DELAYED_WORK(&dvi->hpd_work, tfp410_hpd_work_func);
 
-		ret = devm_request_threaded_irq(dev, gpiod_to_irq(dvi->hpd),
+		ret = devm_request_threaded_irq(dev, dvi->hpd_irq,
 			NULL, tfp410_hpd_irq_thread, IRQF_TRIGGER_RISING |
 			IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
 			"hdmi-hpd", dvi);
@@ -255,7 +391,8 @@
 {
 	struct tfp410 *dvi = dev_get_drvdata(dev);
 
-	cancel_delayed_work_sync(&dvi->hpd_work);
+	if (dvi->hpd_irq >= 0)
+		cancel_delayed_work_sync(&dvi->hpd_work);
 
 	drm_bridge_remove(&dvi->bridge);
 
@@ -269,7 +406,7 @@
 
 static int tfp410_probe(struct platform_device *pdev)
 {
-	return tfp410_init(&pdev->dev);
+	return tfp410_init(&pdev->dev, false);
 }
 
 static int tfp410_remove(struct platform_device *pdev)
@@ -306,7 +443,7 @@
 		return -ENXIO;
 	}
 
-	return tfp410_init(&client->dev);
+	return tfp410_init(&client->dev, true);
 }
 
 static int tfp410_i2c_remove(struct i2c_client *client)
diff -Naur --no-dereference a/drivers/gpu/drm/drm_atomic.c b/drivers/gpu/drm/drm_atomic.c
--- a/drivers/gpu/drm/drm_atomic.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/drm_atomic.c	2020-06-19 21:00:31.408227715 -0400
@@ -1164,6 +1164,7 @@
 
 /**
  * drm_atomic_private_obj_init - initialize private object
+ * @dev: DRM device this object will be attached to
  * @obj: private object
  * @state: initial private object state
  * @funcs: pointer to the struct of function pointers that identify the object
@@ -1173,14 +1174,18 @@
  * driver private object that needs its own atomic state.
  */
 void
-drm_atomic_private_obj_init(struct drm_private_obj *obj,
+drm_atomic_private_obj_init(struct drm_device *dev,
+			    struct drm_private_obj *obj,
 			    struct drm_private_state *state,
 			    const struct drm_private_state_funcs *funcs)
 {
 	memset(obj, 0, sizeof(*obj));
 
+	drm_modeset_lock_init(&obj->lock);
+
 	obj->state = state;
 	obj->funcs = funcs;
+	list_add_tail(&obj->head, &dev->mode_config.privobj_list);
 }
 EXPORT_SYMBOL(drm_atomic_private_obj_init);
 
@@ -1193,7 +1198,9 @@
 void
 drm_atomic_private_obj_fini(struct drm_private_obj *obj)
 {
+	list_del(&obj->head);
 	obj->funcs->atomic_destroy_state(obj, obj->state);
+	drm_modeset_lock_fini(&obj->lock);
 }
 EXPORT_SYMBOL(drm_atomic_private_obj_fini);
 
@@ -1203,8 +1210,8 @@
  * @obj: private object to get the state for
  *
  * This function returns the private object state for the given private object,
- * allocating the state if needed. It does not grab any locks as the caller is
- * expected to care of any required locking.
+ * allocating the state if needed. It will also grab the relevant private
+ * object lock to make sure that the state is consistent.
  *
  * RETURNS:
  *
@@ -1214,7 +1221,7 @@
 drm_atomic_get_private_obj_state(struct drm_atomic_state *state,
 				 struct drm_private_obj *obj)
 {
-	int index, num_objs, i;
+	int index, num_objs, i, ret;
 	size_t size;
 	struct __drm_private_objs_state *arr;
 	struct drm_private_state *obj_state;
@@ -1223,6 +1230,10 @@
 		if (obj == state->private_objs[i].ptr)
 			return state->private_objs[i].state;
 
+	ret = drm_modeset_lock(&obj->lock, state->acquire_ctx);
+	if (ret)
+		return ERR_PTR(ret);
+
 	num_objs = state->num_private_objs + 1;
 	size = sizeof(*state->private_objs) * num_objs;
 	arr = krealloc(state->private_objs, size, GFP_KERNEL);
diff -Naur --no-dereference a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c
--- a/drivers/gpu/drm/drm_atomic_helper.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/drm_atomic_helper.c	2020-06-19 21:00:31.412227715 -0400
@@ -457,7 +457,7 @@
 static enum drm_mode_status mode_valid_path(struct drm_connector *connector,
 					    struct drm_encoder *encoder,
 					    struct drm_crtc *crtc,
-					    struct drm_display_mode *mode)
+					    const struct drm_display_mode *mode)
 {
 	enum drm_mode_status ret;
 
@@ -496,7 +496,7 @@
 		struct drm_crtc *crtc = conn_state->crtc;
 		struct drm_crtc_state *crtc_state;
 		enum drm_mode_status mode_status;
-		struct drm_display_mode *mode;
+		const struct drm_display_mode *mode;
 
 		if (!crtc || !encoder)
 			continue;
@@ -3219,12 +3219,17 @@
 	struct drm_connector_state *new_conn_state;
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *new_crtc_state;
+	struct drm_private_obj *privobj;
+	struct drm_private_state *new_priv_state;
 
 	state->acquire_ctx = ctx;
 
 	for_each_new_plane_in_state(state, plane, new_plane_state, i)
 		state->planes[i].old_state = plane->state;
 
+	for_each_new_private_obj_in_state(state, privobj, new_priv_state, i)
+		state->private_objs[i].old_state = privobj->state;
+
 	for_each_new_crtc_in_state(state, crtc, new_crtc_state, i)
 		state->crtcs[i].old_state = crtc->state;
 
@@ -3814,6 +3819,7 @@
 	struct drm_connector_list_iter conn_iter;
 	struct drm_plane *plane;
 	struct drm_crtc *crtc;
+	struct drm_private_obj *privobj;
 	int err = 0;
 
 	state = drm_atomic_state_alloc(dev);
@@ -3841,6 +3847,16 @@
 			goto free;
 		}
 	}
+
+	drm_for_each_privobj(privobj, dev) {
+		struct drm_private_state *priv_state;
+
+		priv_state = drm_atomic_get_private_obj_state(state, privobj);
+		if (IS_ERR(priv_state)) {
+			err = PTR_ERR(priv_state);
+			goto free;
+		}
+	}
 
 	drm_connector_list_iter_begin(dev, &conn_iter);
 	drm_for_each_connector_iter(conn, &conn_iter) {
diff -Naur --no-dereference a/drivers/gpu/drm/drm_dp_helper.c b/drivers/gpu/drm/drm_dp_helper.c
--- a/drivers/gpu/drm/drm_dp_helper.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/drm_dp_helper.c	2020-06-19 21:00:31.412227715 -0400
@@ -370,10 +370,38 @@
 {
 	u8 values[3];
 	int err;
+	unsigned int addr;
 
 	memset(link, 0, sizeof(*link));
 
-	err = drm_dp_dpcd_read(aux, DP_DPCD_REV, values, sizeof(values));
+	/*
+	 * DP 1.4 introduced a DP_EXTENDED_RECEIVER_CAP_FIELD_PRESENT bit in
+	 * DP_TRAINING_AUX_RD_INTERVAL register. If set, DPCD registers from
+	 * DP_DPCD_REV to DP_ADAPTER_CAP should be retrieved starting from
+	 * DP_DPCD_REV_EXTENDED. All registers are copied except DP_DPCD_REV,
+	 * DP_MAX_LINK_RATE and DP_DOWNSTREAMPORT_PRESENT which represent the
+	 * "true capabilities" of DPRX device.
+	 *
+	 * Original DP_DPCD_REV, DP_MAX_LINK_RATE and DP_DOWNSTREAMPORT_PRESENT
+	 * might falsely return lower capabilities to "avoid interoperability
+	 * issues with some of the existing DP Source devices that malfunction
+	 * when they discover the higher capabilities within those three
+	 * registers.".
+	 *
+	 * Before DP 1.4, DP_EXTENDED_RECEIVER_CAP_FIELD_PRESENT bit was
+	 * reserved and read 0 so it's safe to check against it even if
+	 * DP revision is <1.4
+	 */
+	err = drm_dp_dpcd_readb(aux, DP_TRAINING_AUX_RD_INTERVAL, values);
+	if (err < 0)
+		return err;
+
+	if (values[0] & DP_EXTENDED_RECEIVER_CAP_FIELD_PRESENT)
+		addr = DP_DP13_DPCD_REV;
+	else
+		addr = DP_DPCD_REV;
+
+	err = drm_dp_dpcd_read(aux, addr, values, sizeof(values));
 	if (err < 0)
 		return err;
 
diff -Naur --no-dereference a/drivers/gpu/drm/drm_dp_mst_topology.c b/drivers/gpu/drm/drm_dp_mst_topology.c
--- a/drivers/gpu/drm/drm_dp_mst_topology.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/drm_dp_mst_topology.c	2020-06-19 21:00:31.412227715 -0400
@@ -3216,7 +3216,7 @@
 	/* max. time slots - one slot for MTP header */
 	mst_state->avail_slots = 63;
 
-	drm_atomic_private_obj_init(&mgr->base,
+	drm_atomic_private_obj_init(dev, &mgr->base,
 				    &mst_state->base,
 				    &mst_state_funcs);
 
diff -Naur --no-dereference a/drivers/gpu/drm/drm_mode_config.c b/drivers/gpu/drm/drm_mode_config.c
--- a/drivers/gpu/drm/drm_mode_config.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/drm_mode_config.c	2020-06-19 21:00:31.412227715 -0400
@@ -382,6 +382,7 @@
 	INIT_LIST_HEAD(&dev->mode_config.property_list);
 	INIT_LIST_HEAD(&dev->mode_config.property_blob_list);
 	INIT_LIST_HEAD(&dev->mode_config.plane_list);
+	INIT_LIST_HEAD(&dev->mode_config.privobj_list);
 	idr_init(&dev->mode_config.crtc_idr);
 	idr_init(&dev->mode_config.tile_idr);
 	ida_init(&dev->mode_config.connector_ida);
diff -Naur --no-dereference a/drivers/gpu/drm/drm_modes.c b/drivers/gpu/drm/drm_modes.c
--- a/drivers/gpu/drm/drm_modes.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/drm_modes.c	2020-06-19 21:00:31.412227715 -0400
@@ -662,22 +662,22 @@
  * @bus_flags: information about pixelclk, sync and DE polarity will be stored
  * here
  *
- * Sets DRM_BUS_FLAG_DE_(LOW|HIGH),  DRM_BUS_FLAG_PIXDATA_(POS|NEG)EDGE and
- * DISPLAY_FLAGS_SYNC_(POS|NEG)EDGE in @bus_flags according to DISPLAY_FLAGS
+ * Sets DRM_BUS_FLAG_DE_(LOW|HIGH),  DRM_BUS_FLAG_PIXDATA_DRIVE_(POS|NEG)EDGE
+ * and DISPLAY_FLAGS_SYNC_(POS|NEG)EDGE in @bus_flags according to DISPLAY_FLAGS
  * found in @vm
  */
 void drm_bus_flags_from_videomode(const struct videomode *vm, u32 *bus_flags)
 {
 	*bus_flags = 0;
 	if (vm->flags & DISPLAY_FLAGS_PIXDATA_POSEDGE)
-		*bus_flags |= DRM_BUS_FLAG_PIXDATA_POSEDGE;
+		*bus_flags |= DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE;
 	if (vm->flags & DISPLAY_FLAGS_PIXDATA_NEGEDGE)
-		*bus_flags |= DRM_BUS_FLAG_PIXDATA_NEGEDGE;
+		*bus_flags |= DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE;
 
 	if (vm->flags & DISPLAY_FLAGS_SYNC_POSEDGE)
-		*bus_flags |= DRM_BUS_FLAG_SYNC_POSEDGE;
+		*bus_flags |= DRM_BUS_FLAG_SYNC_DRIVE_POSEDGE;
 	if (vm->flags & DISPLAY_FLAGS_SYNC_NEGEDGE)
-		*bus_flags |= DRM_BUS_FLAG_SYNC_NEGEDGE;
+		*bus_flags |= DRM_BUS_FLAG_SYNC_DRIVE_NEGEDGE;
 
 	if (vm->flags & DISPLAY_FLAGS_DE_LOW)
 		*bus_flags |= DRM_BUS_FLAG_DE_LOW;
diff -Naur --no-dereference a/drivers/gpu/drm/drm_modeset_lock.c b/drivers/gpu/drm/drm_modeset_lock.c
--- a/drivers/gpu/drm/drm_modeset_lock.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/drm_modeset_lock.c	2020-06-19 21:00:31.412227715 -0400
@@ -22,6 +22,7 @@
  */
 
 #include <drm/drmP.h>
+#include <drm/drm_atomic.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_modeset_lock.h>
 
@@ -388,6 +389,7 @@
 int drm_modeset_lock_all_ctx(struct drm_device *dev,
 			     struct drm_modeset_acquire_ctx *ctx)
 {
+	struct drm_private_obj *privobj;
 	struct drm_crtc *crtc;
 	struct drm_plane *plane;
 	int ret;
@@ -407,6 +409,12 @@
 		if (ret)
 			return ret;
 	}
+
+	drm_for_each_privobj(privobj, dev) {
+		ret = drm_modeset_lock(&privobj->lock, ctx);
+		if (ret)
+			return ret;
+	}
 
 	return 0;
 }
diff -Naur --no-dereference a/drivers/gpu/drm/etnaviv/Kconfig b/drivers/gpu/drm/etnaviv/Kconfig
--- a/drivers/gpu/drm/etnaviv/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/etnaviv/Kconfig	2020-06-19 21:00:31.412227715 -0400
@@ -2,7 +2,7 @@
 config DRM_ETNAVIV
 	tristate "ETNAVIV (DRM support for Vivante GPU IP cores)"
 	depends on DRM
-	depends on ARCH_MXC || ARCH_DOVE || (ARM && COMPILE_TEST)
+	depends on ARCH_MXC || ARCH_DOVE || ARCH_OMAP2PLUS || (ARM && COMPILE_TEST)
 	depends on MMU
 	select SHMEM
 	select SYNC_FILE
diff -Naur --no-dereference a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c	2020-06-19 21:00:31.412227715 -0400
@@ -99,7 +99,7 @@
 	vsw = mode->vsync_end - mode->vsync_start;
 
 	/* INV_PXCK as default (most display sample data on rising edge) */
-	if (!(con->display_info.bus_flags & DRM_BUS_FLAG_PIXDATA_POSEDGE))
+	if (!(con->display_info.bus_flags & DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE))
 		pol |= DCU_SYN_POL_INV_PXCK;
 
 	if (mode->flags & DRM_MODE_FLAG_NHSYNC)
diff -Naur --no-dereference a/drivers/gpu/drm/imx/ipuv3-crtc.c b/drivers/gpu/drm/imx/ipuv3-crtc.c
--- a/drivers/gpu/drm/imx/ipuv3-crtc.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/imx/ipuv3-crtc.c	2020-06-19 21:00:31.412227715 -0400
@@ -277,7 +277,7 @@
 	sig_cfg.enable_pol = !(imx_crtc_state->bus_flags & DRM_BUS_FLAG_DE_LOW);
 	/* Default to driving pixel data on negative clock edges */
 	sig_cfg.clk_pol = !!(imx_crtc_state->bus_flags &
-			     DRM_BUS_FLAG_PIXDATA_POSEDGE);
+			     DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE);
 	sig_cfg.bus_format = imx_crtc_state->bus_format;
 	sig_cfg.v_to_h_sync = 0;
 	sig_cfg.hsync_pin = imx_crtc_state->di_hsync_pin;
diff -Naur --no-dereference a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig
--- a/drivers/gpu/drm/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/Kconfig	2020-06-19 21:00:31.408227715 -0400
@@ -335,6 +335,8 @@
 
 source "drivers/gpu/drm/xen/Kconfig"
 
+source "drivers/gpu/drm/tidss/Kconfig"
+
 # Keep legacy drivers last
 
 menuconfig DRM_LEGACY
diff -Naur --no-dereference a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
--- a/drivers/gpu/drm/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/Makefile	2020-06-19 21:00:31.408227715 -0400
@@ -107,3 +107,4 @@
 obj-$(CONFIG_DRM_PL111) += pl111/
 obj-$(CONFIG_DRM_TVE200) += tve200/
 obj-$(CONFIG_DRM_XEN) += xen/
+obj-$(CONFIG_DRM_TIDSS) += tidss/
diff -Naur --no-dereference a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c	2020-06-19 21:00:31.412227715 -0400
@@ -144,7 +144,7 @@
 
 	state->mdp5_kms = mdp5_kms;
 
-	drm_atomic_private_obj_init(&mdp5_kms->glob_state,
+	drm_atomic_private_obj_init(mdp5_kms->dev, &mdp5_kms->glob_state,
 				    &state->base,
 				    &mdp5_global_state_funcs);
 	return 0;
diff -Naur --no-dereference a/drivers/gpu/drm/mxsfb/mxsfb_crtc.c b/drivers/gpu/drm/mxsfb/mxsfb_crtc.c
--- a/drivers/gpu/drm/mxsfb/mxsfb_crtc.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/mxsfb/mxsfb_crtc.c	2020-06-19 21:00:31.412227715 -0400
@@ -242,12 +242,12 @@
 	if (!(bus_flags & DRM_BUS_FLAG_DE_LOW))
 		vdctrl0 |= VDCTRL0_ENABLE_ACT_HIGH;
 	/*
-	 * DRM_BUS_FLAG_PIXDATA_ defines are controller centric,
+	 * DRM_BUS_FLAG_PIXDATA_DRIVE_ defines are controller centric,
 	 * controllers VDCTRL0_DOTCLK is display centric.
 	 * Drive on positive edge       -> display samples on falling edge
-	 * DRM_BUS_FLAG_PIXDATA_POSEDGE -> VDCTRL0_DOTCLK_ACT_FALLING
+	 * DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE -> VDCTRL0_DOTCLK_ACT_FALLING
 	 */
-	if (bus_flags & DRM_BUS_FLAG_PIXDATA_POSEDGE)
+	if (bus_flags & DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE)
 		vdctrl0 |= VDCTRL0_DOTCLK_ACT_FALLING;
 
 	writel(vdctrl0, mxsfb->base + LCDC_VDCTRL0);
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c b/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c
--- a/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c	2020-06-19 21:00:31.412227715 -0400
@@ -18,178 +18,32 @@
 
 struct panel_drv_data {
 	struct omap_dss_device dssdev;
-	struct omap_dss_device *in;
 
 	struct device *dev;
-
-	struct videomode vm;
-};
-
-static const struct videomode tvc_pal_vm = {
-	.hactive	= 720,
-	.vactive	= 574,
-	.pixelclock	= 13500000,
-	.hsync_len	= 64,
-	.hfront_porch	= 12,
-	.hback_porch	= 68,
-	.vsync_len	= 5,
-	.vfront_porch	= 5,
-	.vback_porch	= 41,
-
-	.flags		= DISPLAY_FLAGS_INTERLACED | DISPLAY_FLAGS_HSYNC_LOW |
-			  DISPLAY_FLAGS_VSYNC_LOW,
 };
 
 #define to_panel_data(x) container_of(x, struct panel_drv_data, dssdev)
 
-static int tvc_connect(struct omap_dss_device *dssdev)
+static int tvc_connect(struct omap_dss_device *src,
+		       struct omap_dss_device *dst)
 {
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in;
-	int r;
-
-	dev_dbg(ddata->dev, "connect\n");
-
-	if (omapdss_device_is_connected(dssdev))
-		return 0;
-
-	in = omapdss_of_find_source_for_first_ep(ddata->dev->of_node);
-	if (IS_ERR(in)) {
-		dev_err(ddata->dev, "failed to find video source\n");
-		return PTR_ERR(in);
-	}
-
-	r = in->ops.atv->connect(in, dssdev);
-	if (r) {
-		omap_dss_put_device(in);
-		return r;
-	}
-
-	ddata->in = in;
 	return 0;
 }
 
-static void tvc_disconnect(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	dev_dbg(ddata->dev, "disconnect\n");
-
-	if (!omapdss_device_is_connected(dssdev))
-		return;
-
-	in->ops.atv->disconnect(in, dssdev);
-
-	omap_dss_put_device(in);
-	ddata->in = NULL;
-}
-
-static int tvc_enable(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-	int r;
-
-	dev_dbg(ddata->dev, "enable\n");
-
-	if (!omapdss_device_is_connected(dssdev))
-		return -ENODEV;
-
-	if (omapdss_device_is_enabled(dssdev))
-		return 0;
-
-	in->ops.atv->set_timings(in, &ddata->vm);
-
-	r = in->ops.atv->enable(in);
-	if (r)
-		return r;
-
-	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
-
-	return r;
-}
-
-static void tvc_disable(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	dev_dbg(ddata->dev, "disable\n");
-
-	if (!omapdss_device_is_enabled(dssdev))
-		return;
-
-	in->ops.atv->disable(in);
-
-	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
-}
-
-static void tvc_set_timings(struct omap_dss_device *dssdev,
-			    struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	ddata->vm = *vm;
-	dssdev->panel.vm = *vm;
-
-	in->ops.atv->set_timings(in, vm);
-}
-
-static void tvc_get_timings(struct omap_dss_device *dssdev,
-			    struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-
-	*vm = ddata->vm;
-}
-
-static int tvc_check_timings(struct omap_dss_device *dssdev,
-			     struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	return in->ops.atv->check_timings(in, vm);
-}
-
-static u32 tvc_get_wss(struct omap_dss_device *dssdev)
+static void tvc_disconnect(struct omap_dss_device *src,
+			   struct omap_dss_device *dst)
 {
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	return in->ops.atv->get_wss(in);
 }
 
-static int tvc_set_wss(struct omap_dss_device *dssdev, u32 wss)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	return in->ops.atv->set_wss(in, wss);
-}
-
-static struct omap_dss_driver tvc_driver = {
+static const struct omap_dss_device_ops tvc_ops = {
 	.connect		= tvc_connect,
 	.disconnect		= tvc_disconnect,
-
-	.enable			= tvc_enable,
-	.disable		= tvc_disable,
-
-	.set_timings		= tvc_set_timings,
-	.get_timings		= tvc_get_timings,
-	.check_timings		= tvc_check_timings,
-
-	.get_wss		= tvc_get_wss,
-	.set_wss		= tvc_set_wss,
 };
 
 static int tvc_probe(struct platform_device *pdev)
 {
 	struct panel_drv_data *ddata;
 	struct omap_dss_device *dssdev;
-	int r;
 
 	ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);
 	if (!ddata)
@@ -198,20 +52,16 @@
 	platform_set_drvdata(pdev, ddata);
 	ddata->dev = &pdev->dev;
 
-	ddata->vm = tvc_pal_vm;
-
 	dssdev = &ddata->dssdev;
-	dssdev->driver = &tvc_driver;
+	dssdev->ops = &tvc_ops;
 	dssdev->dev = &pdev->dev;
 	dssdev->type = OMAP_DISPLAY_TYPE_VENC;
+	dssdev->display = true;
 	dssdev->owner = THIS_MODULE;
-	dssdev->panel.vm = tvc_pal_vm;
+	dssdev->of_ports = BIT(0);
 
-	r = omapdss_register_display(dssdev);
-	if (r) {
-		dev_err(&pdev->dev, "Failed to register panel\n");
-		return r;
-	}
+	omapdss_display_init(dssdev);
+	omapdss_device_register(dssdev);
 
 	return 0;
 }
@@ -219,12 +69,8 @@
 static int __exit tvc_remove(struct platform_device *pdev)
 {
 	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-	struct omap_dss_device *dssdev = &ddata->dssdev;
-
-	omapdss_unregister_display(&ddata->dssdev);
 
-	tvc_disable(dssdev);
-	tvc_disconnect(dssdev);
+	omapdss_device_unregister(&ddata->dssdev);
 
 	return 0;
 }
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/displays/connector-dvi.c b/drivers/gpu/drm/omapdrm/displays/connector-dvi.c
--- a/drivers/gpu/drm/omapdrm/displays/connector-dvi.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/displays/connector-dvi.c	1969-12-31 19:00:00.000000000 -0500
@@ -1,461 +0,0 @@
-/*
- * Generic DVI Connector driver
- *
- * Copyright (C) 2013 Texas Instruments Incorporated - http://www.ti.com/
- * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- */
-
-#include <linux/gpio/consumer.h>
-#include <linux/i2c.h>
-#include <linux/module.h>
-#include <linux/platform_device.h>
-#include <linux/slab.h>
-
-#include <drm/drm_edid.h>
-
-#include "../dss/omapdss.h"
-
-static const struct videomode dvic_default_vm = {
-	.hactive	= 640,
-	.vactive	= 480,
-
-	.pixelclock	= 23500000,
-
-	.hfront_porch	= 48,
-	.hsync_len	= 32,
-	.hback_porch	= 80,
-
-	.vfront_porch	= 3,
-	.vsync_len	= 4,
-	.vback_porch	= 7,
-
-	.flags		= DISPLAY_FLAGS_HSYNC_HIGH | DISPLAY_FLAGS_VSYNC_HIGH |
-			  DISPLAY_FLAGS_SYNC_NEGEDGE | DISPLAY_FLAGS_DE_HIGH |
-			  DISPLAY_FLAGS_PIXDATA_POSEDGE,
-};
-
-struct panel_drv_data {
-	struct omap_dss_device dssdev;
-	struct omap_dss_device *in;
-
-	struct videomode vm;
-
-	struct i2c_adapter *i2c_adapter;
-
-	struct gpio_desc *hpd_gpio;
-
-	void (*hpd_cb)(void *cb_data, enum drm_connector_status status);
-	void *hpd_cb_data;
-	bool hpd_enabled;
-	/* mutex for hpd fields above */
-	struct mutex hpd_lock;
-};
-
-#define to_panel_data(x) container_of(x, struct panel_drv_data, dssdev)
-
-static int dvic_connect(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in;
-	int r;
-
-	if (omapdss_device_is_connected(dssdev))
-		return 0;
-
-	in = omapdss_of_find_source_for_first_ep(dssdev->dev->of_node);
-	if (IS_ERR(in)) {
-		dev_err(dssdev->dev, "failed to find video source\n");
-		return PTR_ERR(in);
-	}
-
-	r = in->ops.dvi->connect(in, dssdev);
-	if (r) {
-		omap_dss_put_device(in);
-		return r;
-	}
-
-	ddata->in = in;
-	return 0;
-}
-
-static void dvic_disconnect(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	if (!omapdss_device_is_connected(dssdev))
-		return;
-
-	in->ops.dvi->disconnect(in, dssdev);
-
-	omap_dss_put_device(in);
-	ddata->in = NULL;
-}
-
-static int dvic_enable(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-	int r;
-
-	if (!omapdss_device_is_connected(dssdev))
-		return -ENODEV;
-
-	if (omapdss_device_is_enabled(dssdev))
-		return 0;
-
-	in->ops.dvi->set_timings(in, &ddata->vm);
-
-	r = in->ops.dvi->enable(in);
-	if (r)
-		return r;
-
-	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
-
-	return 0;
-}
-
-static void dvic_disable(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	if (!omapdss_device_is_enabled(dssdev))
-		return;
-
-	in->ops.dvi->disable(in);
-
-	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
-}
-
-static void dvic_set_timings(struct omap_dss_device *dssdev,
-			     struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	ddata->vm = *vm;
-	dssdev->panel.vm = *vm;
-
-	in->ops.dvi->set_timings(in, vm);
-}
-
-static void dvic_get_timings(struct omap_dss_device *dssdev,
-			     struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-
-	*vm = ddata->vm;
-}
-
-static int dvic_check_timings(struct omap_dss_device *dssdev,
-			      struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	return in->ops.dvi->check_timings(in, vm);
-}
-
-static int dvic_ddc_read(struct i2c_adapter *adapter,
-		unsigned char *buf, u16 count, u8 offset)
-{
-	int r, retries;
-
-	for (retries = 3; retries > 0; retries--) {
-		struct i2c_msg msgs[] = {
-			{
-				.addr   = DDC_ADDR,
-				.flags  = 0,
-				.len    = 1,
-				.buf    = &offset,
-			}, {
-				.addr   = DDC_ADDR,
-				.flags  = I2C_M_RD,
-				.len    = count,
-				.buf    = buf,
-			}
-		};
-
-		r = i2c_transfer(adapter, msgs, 2);
-		if (r == 2)
-			return 0;
-
-		if (r != -EAGAIN)
-			break;
-	}
-
-	return r < 0 ? r : -EIO;
-}
-
-static int dvic_read_edid(struct omap_dss_device *dssdev,
-		u8 *edid, int len)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	int r, l, bytes_read;
-
-	if (ddata->hpd_gpio && !gpiod_get_value_cansleep(ddata->hpd_gpio))
-		return -ENODEV;
-
-	if (!ddata->i2c_adapter)
-		return -ENODEV;
-
-	l = min(EDID_LENGTH, len);
-	r = dvic_ddc_read(ddata->i2c_adapter, edid, l, 0);
-	if (r)
-		return r;
-
-	bytes_read = l;
-
-	/* if there are extensions, read second block */
-	if (len > EDID_LENGTH && edid[0x7e] > 0) {
-		l = min(EDID_LENGTH, len - EDID_LENGTH);
-
-		r = dvic_ddc_read(ddata->i2c_adapter, edid + EDID_LENGTH,
-				l, EDID_LENGTH);
-		if (r)
-			return r;
-
-		bytes_read += l;
-	}
-
-	return bytes_read;
-}
-
-static bool dvic_detect(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	unsigned char out;
-	int r;
-
-	if (ddata->hpd_gpio)
-		return gpiod_get_value_cansleep(ddata->hpd_gpio);
-
-	if (!ddata->i2c_adapter)
-		return true;
-
-	r = dvic_ddc_read(ddata->i2c_adapter, &out, 1, 0);
-
-	return r == 0;
-}
-
-static int dvic_register_hpd_cb(struct omap_dss_device *dssdev,
-				 void (*cb)(void *cb_data,
-					    enum drm_connector_status status),
-				 void *cb_data)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-
-	if (!ddata->hpd_gpio)
-		return -ENOTSUPP;
-
-	mutex_lock(&ddata->hpd_lock);
-	ddata->hpd_cb = cb;
-	ddata->hpd_cb_data = cb_data;
-	mutex_unlock(&ddata->hpd_lock);
-	return 0;
-}
-
-static void dvic_unregister_hpd_cb(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-
-	if (!ddata->hpd_gpio)
-		return;
-
-	mutex_lock(&ddata->hpd_lock);
-	ddata->hpd_cb = NULL;
-	ddata->hpd_cb_data = NULL;
-	mutex_unlock(&ddata->hpd_lock);
-}
-
-static void dvic_enable_hpd(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-
-	if (!ddata->hpd_gpio)
-		return;
-
-	mutex_lock(&ddata->hpd_lock);
-	ddata->hpd_enabled = true;
-	mutex_unlock(&ddata->hpd_lock);
-}
-
-static void dvic_disable_hpd(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-
-	if (!ddata->hpd_gpio)
-		return;
-
-	mutex_lock(&ddata->hpd_lock);
-	ddata->hpd_enabled = false;
-	mutex_unlock(&ddata->hpd_lock);
-}
-
-static struct omap_dss_driver dvic_driver = {
-	.connect	= dvic_connect,
-	.disconnect	= dvic_disconnect,
-
-	.enable		= dvic_enable,
-	.disable	= dvic_disable,
-
-	.set_timings	= dvic_set_timings,
-	.get_timings	= dvic_get_timings,
-	.check_timings	= dvic_check_timings,
-
-	.read_edid	= dvic_read_edid,
-	.detect		= dvic_detect,
-
-	.register_hpd_cb	= dvic_register_hpd_cb,
-	.unregister_hpd_cb	= dvic_unregister_hpd_cb,
-	.enable_hpd		= dvic_enable_hpd,
-	.disable_hpd		= dvic_disable_hpd,
-};
-
-static irqreturn_t dvic_hpd_isr(int irq, void *data)
-{
-	struct panel_drv_data *ddata = data;
-
-	mutex_lock(&ddata->hpd_lock);
-	if (ddata->hpd_enabled && ddata->hpd_cb) {
-		enum drm_connector_status status;
-
-		if (dvic_detect(&ddata->dssdev))
-			status = connector_status_connected;
-		else
-			status = connector_status_disconnected;
-
-		ddata->hpd_cb(ddata->hpd_cb_data, status);
-	}
-	mutex_unlock(&ddata->hpd_lock);
-
-	return IRQ_HANDLED;
-}
-
-static int dvic_probe_of(struct platform_device *pdev)
-{
-	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-	struct device_node *node = pdev->dev.of_node;
-	struct device_node *adapter_node;
-	struct i2c_adapter *adapter;
-	struct gpio_desc *gpio;
-	int r;
-
-	gpio = devm_gpiod_get_optional(&pdev->dev, "hpd", GPIOD_IN);
-	if (IS_ERR(gpio)) {
-		dev_err(&pdev->dev, "failed to parse HPD gpio\n");
-		return PTR_ERR(gpio);
-	}
-
-	ddata->hpd_gpio = gpio;
-
-	mutex_init(&ddata->hpd_lock);
-
-	if (ddata->hpd_gpio) {
-		r = devm_request_threaded_irq(&pdev->dev,
-			gpiod_to_irq(ddata->hpd_gpio), NULL, dvic_hpd_isr,
-			IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
-			"DVI HPD", ddata);
-		if (r)
-			return r;
-	}
-
-	adapter_node = of_parse_phandle(node, "ddc-i2c-bus", 0);
-	if (adapter_node) {
-		adapter = of_get_i2c_adapter_by_node(adapter_node);
-		of_node_put(adapter_node);
-		if (adapter == NULL) {
-			dev_err(&pdev->dev, "failed to parse ddc-i2c-bus\n");
-			return -EPROBE_DEFER;
-		}
-
-		ddata->i2c_adapter = adapter;
-	}
-
-	return 0;
-}
-
-static int dvic_probe(struct platform_device *pdev)
-{
-	struct panel_drv_data *ddata;
-	struct omap_dss_device *dssdev;
-	int r;
-
-	ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);
-	if (!ddata)
-		return -ENOMEM;
-
-	platform_set_drvdata(pdev, ddata);
-
-	r = dvic_probe_of(pdev);
-	if (r)
-		return r;
-
-	ddata->vm = dvic_default_vm;
-
-	dssdev = &ddata->dssdev;
-	dssdev->driver = &dvic_driver;
-	dssdev->dev = &pdev->dev;
-	dssdev->type = OMAP_DISPLAY_TYPE_DVI;
-	dssdev->owner = THIS_MODULE;
-	dssdev->panel.vm = dvic_default_vm;
-
-	r = omapdss_register_display(dssdev);
-	if (r) {
-		dev_err(&pdev->dev, "Failed to register panel\n");
-		goto err_reg;
-	}
-
-	return 0;
-
-err_reg:
-	i2c_put_adapter(ddata->i2c_adapter);
-	mutex_destroy(&ddata->hpd_lock);
-
-	return r;
-}
-
-static int __exit dvic_remove(struct platform_device *pdev)
-{
-	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-	struct omap_dss_device *dssdev = &ddata->dssdev;
-
-	omapdss_unregister_display(&ddata->dssdev);
-
-	dvic_disable(dssdev);
-	dvic_disconnect(dssdev);
-
-	i2c_put_adapter(ddata->i2c_adapter);
-
-	mutex_destroy(&ddata->hpd_lock);
-
-	return 0;
-}
-
-static const struct of_device_id dvic_of_match[] = {
-	{ .compatible = "omapdss,dvi-connector", },
-	{},
-};
-
-MODULE_DEVICE_TABLE(of, dvic_of_match);
-
-static struct platform_driver dvi_connector_driver = {
-	.probe	= dvic_probe,
-	.remove	= __exit_p(dvic_remove),
-	.driver	= {
-		.name	= "connector-dvi",
-		.of_match_table = dvic_of_match,
-		.suppress_bind_attrs = true,
-	},
-};
-
-module_platform_driver(dvi_connector_driver);
-
-MODULE_AUTHOR("Tomi Valkeinen <tomi.valkeinen@ti.com>");
-MODULE_DESCRIPTION("Generic DVI Connector driver");
-MODULE_LICENSE("GPL");
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c b/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c
--- a/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c	2020-06-19 21:00:31.412227715 -0400
@@ -10,284 +10,74 @@
  */
 
 #include <linux/gpio/consumer.h>
-#include <linux/slab.h>
 #include <linux/module.h>
-#include <linux/platform_device.h>
-#include <linux/of.h>
-#include <linux/of_gpio.h>
 #include <linux/mutex.h>
-
-#include <drm/drm_edid.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
 
 #include "../dss/omapdss.h"
 
-static const struct videomode hdmic_default_vm = {
-	.hactive	= 640,
-	.vactive	= 480,
-	.pixelclock	= 25175000,
-	.hsync_len	= 96,
-	.hfront_porch	= 16,
-	.hback_porch	= 48,
-	.vsync_len	= 2,
-	.vfront_porch	= 11,
-	.vback_porch	= 31,
-
-	.flags		= DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW,
-};
-
 struct panel_drv_data {
 	struct omap_dss_device dssdev;
-	struct omap_dss_device *in;
 	void (*hpd_cb)(void *cb_data, enum drm_connector_status status);
 	void *hpd_cb_data;
-	bool hpd_enabled;
 	struct mutex hpd_lock;
 
 	struct device *dev;
 
-	struct videomode vm;
-
-	int hpd_gpio;
+	struct gpio_desc *hpd_gpio;
 };
 
 #define to_panel_data(x) container_of(x, struct panel_drv_data, dssdev)
 
-static int hdmic_connect(struct omap_dss_device *dssdev)
+static int hdmic_connect(struct omap_dss_device *src,
+			 struct omap_dss_device *dst)
 {
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in;
-	int r;
-
-	dev_dbg(ddata->dev, "connect\n");
-
-	if (omapdss_device_is_connected(dssdev))
-		return 0;
-
-	in = omapdss_of_find_source_for_first_ep(ddata->dev->of_node);
-	if (IS_ERR(in)) {
-		dev_err(ddata->dev, "failed to find video source\n");
-		return PTR_ERR(in);
-	}
-
-	r = in->ops.hdmi->connect(in, dssdev);
-	if (r) {
-		omap_dss_put_device(in);
-		return r;
-	}
-
-	ddata->in = in;
 	return 0;
 }
 
-static void hdmic_disconnect(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	dev_dbg(ddata->dev, "disconnect\n");
-
-	if (!omapdss_device_is_connected(dssdev))
-		return;
-
-	in->ops.hdmi->disconnect(in, dssdev);
-
-	omap_dss_put_device(in);
-	ddata->in = NULL;
-}
-
-static int hdmic_enable(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-	int r;
-
-	dev_dbg(ddata->dev, "enable\n");
-
-	if (!omapdss_device_is_connected(dssdev))
-		return -ENODEV;
-
-	if (omapdss_device_is_enabled(dssdev))
-		return 0;
-
-	in->ops.hdmi->set_timings(in, &ddata->vm);
-
-	r = in->ops.hdmi->enable(in);
-	if (r)
-		return r;
-
-	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
-
-	return r;
-}
-
-static void hdmic_disable(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	dev_dbg(ddata->dev, "disable\n");
-
-	if (!omapdss_device_is_enabled(dssdev))
-		return;
-
-	in->ops.hdmi->disable(in);
-
-	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
-}
-
-static void hdmic_set_timings(struct omap_dss_device *dssdev,
-			      struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	ddata->vm = *vm;
-	dssdev->panel.vm = *vm;
-
-	in->ops.hdmi->set_timings(in, vm);
-}
-
-static void hdmic_get_timings(struct omap_dss_device *dssdev,
-			      struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-
-	*vm = ddata->vm;
-}
-
-static int hdmic_check_timings(struct omap_dss_device *dssdev,
-			       struct videomode *vm)
+static void hdmic_disconnect(struct omap_dss_device *src,
+			     struct omap_dss_device *dst)
 {
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	return in->ops.hdmi->check_timings(in, vm);
-}
-
-static int hdmic_read_edid(struct omap_dss_device *dssdev,
-		u8 *edid, int len)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	return in->ops.hdmi->read_edid(in, edid, len);
 }
 
 static bool hdmic_detect(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-	bool connected;
 
-	if (gpio_is_valid(ddata->hpd_gpio))
-		connected = gpio_get_value_cansleep(ddata->hpd_gpio);
-	else
-		connected = in->ops.hdmi->detect(in);
-	if (!connected && in->ops.hdmi->lost_hotplug)
-		in->ops.hdmi->lost_hotplug(in);
-	return connected;
+	return gpiod_get_value_cansleep(ddata->hpd_gpio);
 }
 
-static int hdmic_register_hpd_cb(struct omap_dss_device *dssdev,
-				 void (*cb)(void *cb_data,
+static void hdmic_register_hpd_cb(struct omap_dss_device *dssdev,
+				  void (*cb)(void *cb_data,
 					    enum drm_connector_status status),
-				 void *cb_data)
+				  void *cb_data)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	if (gpio_is_valid(ddata->hpd_gpio)) {
-		mutex_lock(&ddata->hpd_lock);
-		ddata->hpd_cb = cb;
-		ddata->hpd_cb_data = cb_data;
-		mutex_unlock(&ddata->hpd_lock);
-		return 0;
-	} else if (in->ops.hdmi->register_hpd_cb) {
-		return in->ops.hdmi->register_hpd_cb(in, cb, cb_data);
-	}
 
-	return -ENOTSUPP;
+	mutex_lock(&ddata->hpd_lock);
+	ddata->hpd_cb = cb;
+	ddata->hpd_cb_data = cb_data;
+	mutex_unlock(&ddata->hpd_lock);
 }
 
 static void hdmic_unregister_hpd_cb(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	if (gpio_is_valid(ddata->hpd_gpio)) {
-		mutex_lock(&ddata->hpd_lock);
-		ddata->hpd_cb = NULL;
-		ddata->hpd_cb_data = NULL;
-		mutex_unlock(&ddata->hpd_lock);
-	} else if (in->ops.hdmi->unregister_hpd_cb) {
-		in->ops.hdmi->unregister_hpd_cb(in);
-	}
-}
-
-static void hdmic_enable_hpd(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	if (gpio_is_valid(ddata->hpd_gpio)) {
-		mutex_lock(&ddata->hpd_lock);
-		ddata->hpd_enabled = true;
-		mutex_unlock(&ddata->hpd_lock);
-	} else if (in->ops.hdmi->enable_hpd) {
-		in->ops.hdmi->enable_hpd(in);
-	}
-}
-
-static void hdmic_disable_hpd(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	if (gpio_is_valid(ddata->hpd_gpio)) {
-		mutex_lock(&ddata->hpd_lock);
-		ddata->hpd_enabled = false;
-		mutex_unlock(&ddata->hpd_lock);
-	} else if (in->ops.hdmi->disable_hpd) {
-		in->ops.hdmi->disable_hpd(in);
-	}
-}
-
-static int hdmic_set_hdmi_mode(struct omap_dss_device *dssdev, bool hdmi_mode)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	return in->ops.hdmi->set_hdmi_mode(in, hdmi_mode);
-}
-
-static int hdmic_set_infoframe(struct omap_dss_device *dssdev,
-		const struct hdmi_avi_infoframe *avi)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
 
-	return in->ops.hdmi->set_infoframe(in, avi);
+	mutex_lock(&ddata->hpd_lock);
+	ddata->hpd_cb = NULL;
+	ddata->hpd_cb_data = NULL;
+	mutex_unlock(&ddata->hpd_lock);
 }
 
-static struct omap_dss_driver hdmic_driver = {
+static const struct omap_dss_device_ops hdmic_ops = {
 	.connect		= hdmic_connect,
 	.disconnect		= hdmic_disconnect,
 
-	.enable			= hdmic_enable,
-	.disable		= hdmic_disable,
-
-	.set_timings		= hdmic_set_timings,
-	.get_timings		= hdmic_get_timings,
-	.check_timings		= hdmic_check_timings,
-
-	.read_edid		= hdmic_read_edid,
 	.detect			= hdmic_detect,
 	.register_hpd_cb	= hdmic_register_hpd_cb,
 	.unregister_hpd_cb	= hdmic_unregister_hpd_cb,
-	.enable_hpd		= hdmic_enable_hpd,
-	.disable_hpd		= hdmic_disable_hpd,
-	.set_hdmi_mode		= hdmic_set_hdmi_mode,
-	.set_hdmi_infoframe	= hdmic_set_infoframe,
 };
 
 static irqreturn_t hdmic_hpd_isr(int irq, void *data)
@@ -295,7 +85,7 @@
 	struct panel_drv_data *ddata = data;
 
 	mutex_lock(&ddata->hpd_lock);
-	if (ddata->hpd_enabled && ddata->hpd_cb) {
+	if (ddata->hpd_cb) {
 		enum drm_connector_status status;
 
 		if (hdmic_detect(&ddata->dssdev))
@@ -310,26 +100,11 @@
 	return IRQ_HANDLED;
 }
 
-static int hdmic_probe_of(struct platform_device *pdev)
-{
-	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-	struct device_node *node = pdev->dev.of_node;
-	int gpio;
-
-	/* HPD GPIO */
-	gpio = of_get_named_gpio(node, "hpd-gpios", 0);
-	if (gpio_is_valid(gpio))
-		ddata->hpd_gpio = gpio;
-	else
-		ddata->hpd_gpio = -ENODEV;
-
-	return 0;
-}
-
 static int hdmic_probe(struct platform_device *pdev)
 {
 	struct panel_drv_data *ddata;
 	struct omap_dss_device *dssdev;
+	struct gpio_desc *gpio;
 	int r;
 
 	ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);
@@ -339,20 +114,20 @@
 	platform_set_drvdata(pdev, ddata);
 	ddata->dev = &pdev->dev;
 
-	r = hdmic_probe_of(pdev);
-	if (r)
-		return r;
-
 	mutex_init(&ddata->hpd_lock);
 
-	if (gpio_is_valid(ddata->hpd_gpio)) {
-		r = devm_gpio_request_one(&pdev->dev, ddata->hpd_gpio,
-				GPIOF_DIR_IN, "hdmi_hpd");
-		if (r)
-			return r;
+	/* HPD GPIO */
+	gpio = devm_gpiod_get_optional(&pdev->dev, "hpd", GPIOD_IN);
+	if (IS_ERR(gpio)) {
+		dev_err(&pdev->dev, "failed to parse HPD gpio\n");
+		return PTR_ERR(gpio);
+	}
+
+	ddata->hpd_gpio = gpio;
 
+	if (ddata->hpd_gpio) {
 		r = devm_request_threaded_irq(&pdev->dev,
-				gpio_to_irq(ddata->hpd_gpio),
+				gpiod_to_irq(ddata->hpd_gpio),
 				NULL, hdmic_hpd_isr,
 				IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING |
 				IRQF_ONESHOT,
@@ -361,20 +136,19 @@
 			return r;
 	}
 
-	ddata->vm = hdmic_default_vm;
-
 	dssdev = &ddata->dssdev;
-	dssdev->driver = &hdmic_driver;
+	dssdev->ops = &hdmic_ops;
 	dssdev->dev = &pdev->dev;
 	dssdev->type = OMAP_DISPLAY_TYPE_HDMI;
+	dssdev->display = true;
 	dssdev->owner = THIS_MODULE;
-	dssdev->panel.vm = hdmic_default_vm;
+	dssdev->of_ports = BIT(0);
+	dssdev->ops_flags = ddata->hpd_gpio
+			  ? OMAP_DSS_DEVICE_OP_DETECT | OMAP_DSS_DEVICE_OP_HPD
+			  : 0;
 
-	r = omapdss_register_display(dssdev);
-	if (r) {
-		dev_err(&pdev->dev, "Failed to register panel\n");
-		return r;
-	}
+	omapdss_display_init(dssdev);
+	omapdss_device_register(dssdev);
 
 	return 0;
 }
@@ -382,12 +156,8 @@
 static int __exit hdmic_remove(struct platform_device *pdev)
 {
 	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-	struct omap_dss_device *dssdev = &ddata->dssdev;
-
-	omapdss_unregister_display(&ddata->dssdev);
 
-	hdmic_disable(dssdev);
-	hdmic_disconnect(dssdev);
+	omapdss_device_unregister(&ddata->dssdev);
 
 	return 0;
 }
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/displays/dra7-evm-encoder-tpd12s015.c b/drivers/gpu/drm/omapdrm/displays/dra7-evm-encoder-tpd12s015.c
--- a/drivers/gpu/drm/omapdrm/displays/dra7-evm-encoder-tpd12s015.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/displays/dra7-evm-encoder-tpd12s015.c	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,369 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * DRA7 EVM TPD12S015 HDMI ESD protection & level shifter chip driver
+ *
+ * Copyright (C) 2013-2018 Texas Instruments Incorporated -  http://www.ti.com/
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ */
+
+#include <linux/completion.h>
+#include <linux/delay.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/gpio.h>
+#include <linux/i2c.h>
+#include <linux/io.h>
+#include <linux/platform_device.h>
+#include <linux/gpio/consumer.h>
+#include <linux/suspend.h>
+#include <linux/of_platform.h>
+#include <linux/pinctrl/consumer.h>
+#include <linux/spinlock.h>
+
+#include "../dss/omapdss.h"
+
+#define SEL_I2C2	0
+#define SEL_HDMI	1
+
+/* HPD gpio debounce time in microseconds */
+#define HPD_DEBOUNCE_TIME	1000
+
+struct i2c_mux_wa {
+	struct gpio_desc *i2c_ddc_gpio;
+
+	struct pinctrl *pins;
+	struct pinctrl_state *pin_state_i2c;
+	struct pinctrl_state *pin_state_ddc;
+
+	struct i2c_adapter *ddc_i2c_adapter;
+};
+
+struct panel_drv_data {
+	struct omap_dss_device dssdev;
+	struct omap_dss_device *src;
+	void (*hpd_cb)(void *cb_data, enum drm_connector_status status);
+	void *hpd_cb_data;
+	struct mutex hpd_lock;
+
+	struct gpio_desc *ct_cp_hpd_gpio;
+	struct gpio_desc *ls_oe_gpio;
+	struct gpio_desc *hpd_gpio;
+
+	struct i2c_mux_wa i2c_wa;
+};
+
+/*
+ * use SEL_I2C2 to configure pcf8575@26 to set/unset LS_OE and CT_HPD, and use
+ * SEL_HDMI to read edid via the HDMI ddc lines
+ */
+static void tpd_i2c_ddc_demux(struct i2c_mux_wa *i2c_wa, int sel)
+{
+	/*
+	 * switch to I2C2 or HDMI DDC internal pinmux gpio to low or high to
+	 * select I2C2 or HDMI path respectively
+	 */
+	if (sel == SEL_I2C2) {
+		pinctrl_select_state(i2c_wa->pins, i2c_wa->pin_state_i2c);
+		gpiod_set_value(i2c_wa->i2c_ddc_gpio, 0);
+	} else {
+		pinctrl_select_state(i2c_wa->pins, i2c_wa->pin_state_ddc);
+		gpiod_set_value(i2c_wa->i2c_ddc_gpio, 1);
+	}
+
+	/* let it propagate */
+	udelay(5);
+}
+
+#define to_panel_data(x) container_of(x, struct panel_drv_data, dssdev)
+
+static int tpd_connect(struct omap_dss_device *src,
+		       struct omap_dss_device *dst)
+{
+	struct panel_drv_data *ddata = to_panel_data(dst);
+	int r;
+
+	r = omapdss_device_connect(dst->dss, dst, dst->next);
+	if (r)
+		return r;
+
+	gpiod_set_value_cansleep(ddata->ct_cp_hpd_gpio, 1);
+	gpiod_set_value_cansleep(ddata->ls_oe_gpio, 1);
+
+	/* DC-DC converter needs at max 300us to get to 90% of 5V */
+	udelay(300);
+
+	/*
+	 * The HPD GPIO debounce causes a delay until we see the real HPD state.
+	 * If tpd_read_edid() or tpd_detect() are called very soon after setting
+	 * the ct_cp_hpd-gpio, we could observe wrong HPD value. So sleep here
+	 * until the GPIO values has become valid.
+	 */
+	msleep(DIV_ROUND_UP(HPD_DEBOUNCE_TIME, 1000));
+
+	ddata->src = src;
+
+	return 0;
+}
+
+static void tpd_disconnect(struct omap_dss_device *src,
+			   struct omap_dss_device *dst)
+{
+	struct panel_drv_data *ddata = to_panel_data(dst);
+
+	gpiod_set_value_cansleep(ddata->ct_cp_hpd_gpio, 0);
+	gpiod_set_value_cansleep(ddata->ls_oe_gpio, 0);
+
+	omapdss_device_disconnect(dst, dst->next);
+	ddata->src = NULL;
+}
+
+static int tpd_read_edid(struct omap_dss_device *dssdev,
+		u8 *edid, int len)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *src = ddata->src;
+	struct i2c_mux_wa *i2c_wa = &ddata->i2c_wa;
+	int r = 0;
+
+	if (!gpiod_get_value_cansleep(ddata->hpd_gpio))
+		return -ENODEV;
+
+	i2c_lock_bus(i2c_wa->ddc_i2c_adapter, I2C_LOCK_ROOT_ADAPTER);
+
+	tpd_i2c_ddc_demux(i2c_wa, SEL_HDMI);
+
+	r = src->ops->read_edid(src, edid, len);
+
+	tpd_i2c_ddc_demux(i2c_wa, SEL_I2C2);
+
+	i2c_unlock_bus(i2c_wa->ddc_i2c_adapter, I2C_LOCK_ROOT_ADAPTER);
+
+	return r;
+}
+
+static bool tpd_detect(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+
+	return gpiod_get_value_cansleep(ddata->hpd_gpio);
+}
+
+static void tpd_register_hpd_cb(struct omap_dss_device *dssdev,
+				void (*cb)(void *cb_data,
+					  enum drm_connector_status status),
+				void *cb_data)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+
+	mutex_lock(&ddata->hpd_lock);
+	ddata->hpd_cb = cb;
+	ddata->hpd_cb_data = cb_data;
+	mutex_unlock(&ddata->hpd_lock);
+}
+
+static void tpd_unregister_hpd_cb(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+
+	mutex_lock(&ddata->hpd_lock);
+	ddata->hpd_cb = NULL;
+	ddata->hpd_cb_data = NULL;
+	mutex_unlock(&ddata->hpd_lock);
+}
+
+static const struct omap_dss_device_ops tpd_ops = {
+	.connect		= tpd_connect,
+	.disconnect		= tpd_disconnect,
+	.detect			= tpd_detect,
+	.register_hpd_cb	= tpd_register_hpd_cb,
+	.unregister_hpd_cb	= tpd_unregister_hpd_cb,
+	.read_edid		= tpd_read_edid,
+};
+
+static irqreturn_t tpd_hpd_isr(int irq, void *data)
+{
+	struct panel_drv_data *ddata = data;
+
+	mutex_lock(&ddata->hpd_lock);
+	if (ddata->hpd_cb) {
+		enum drm_connector_status status;
+
+		if (tpd_detect(&ddata->dssdev))
+			status = connector_status_connected;
+		else
+			status = connector_status_disconnected;
+
+		ddata->hpd_cb(ddata->hpd_cb_data, status);
+	}
+	mutex_unlock(&ddata->hpd_lock);
+
+	return IRQ_HANDLED;
+}
+
+static int tpd_init_i2c_mux(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct i2c_mux_wa *i2c_wa = &ddata->i2c_wa;
+	struct device_node *node;
+
+	i2c_wa->i2c_ddc_gpio = devm_gpiod_get_index(&pdev->dev, NULL,
+						    3, GPIOD_OUT_LOW);
+	if (IS_ERR(i2c_wa->i2c_ddc_gpio))
+		return PTR_ERR(i2c_wa->i2c_ddc_gpio);
+
+	i2c_wa->pins = devm_pinctrl_get(&pdev->dev);
+	if (IS_ERR(i2c_wa->pins))
+		return PTR_ERR(i2c_wa->pins);
+
+	i2c_wa->pin_state_i2c = pinctrl_lookup_state(i2c_wa->pins, "i2c");
+	if (IS_ERR(i2c_wa->pin_state_i2c))
+		return PTR_ERR(i2c_wa->pin_state_i2c);
+
+	i2c_wa->pin_state_ddc = pinctrl_lookup_state(i2c_wa->pins, "ddc");
+	if (IS_ERR(i2c_wa->pin_state_ddc))
+		return PTR_ERR(i2c_wa->pin_state_ddc);
+
+	node = of_parse_phandle(pdev->dev.of_node, "ddc-i2c-bus", 0);
+	if (!node)
+		return -ENODEV;
+
+	i2c_wa->ddc_i2c_adapter = of_find_i2c_adapter_by_node(node);
+	if (!i2c_wa->ddc_i2c_adapter)
+		return -ENODEV;
+
+	tpd_i2c_ddc_demux(i2c_wa, SEL_I2C2);
+
+	return 0;
+}
+
+static void tpd_uninit_i2c_mux(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+
+	tpd_i2c_ddc_demux(&ddata->i2c_wa, SEL_I2C2);
+
+	i2c_put_adapter(ddata->i2c_wa.ddc_i2c_adapter);
+}
+
+static int tpd_probe(struct platform_device *pdev)
+{
+	struct omap_dss_device *dssdev;
+	struct panel_drv_data *ddata;
+	int r;
+	struct gpio_desc *gpio;
+
+	ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);
+	if (!ddata)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, ddata);
+
+	gpio = devm_gpiod_get_index_optional(&pdev->dev, NULL, 0,
+		 GPIOD_OUT_LOW);
+	if (IS_ERR(gpio))
+		return PTR_ERR(gpio);
+
+	ddata->ct_cp_hpd_gpio = gpio;
+
+	gpio = devm_gpiod_get_index_optional(&pdev->dev, NULL, 1,
+		 GPIOD_OUT_LOW);
+	if (IS_ERR(gpio))
+		return PTR_ERR(gpio);
+
+	ddata->ls_oe_gpio = gpio;
+
+	gpio = devm_gpiod_get_index(&pdev->dev, NULL, 2,
+		GPIOD_IN);
+	if (IS_ERR(gpio))
+		return PTR_ERR(gpio);
+
+	ddata->hpd_gpio = gpio;
+
+	mutex_init(&ddata->hpd_lock);
+
+	r = tpd_init_i2c_mux(pdev);
+	if (r)
+		return r;
+
+	/*
+	 * we see some low voltage glitches on the HPD_B line before it
+	 * stabalizes to around 5V. We see the effects of this glitch on the
+	 * HPD_A side, and hence on the gpio on DRA7x. The glitch is quite short
+	 * in duration, but it takes a while for the voltage to go down back to
+	 * 0 volts, we set a debounce value of 1 millisecond to prevent this,
+	 * the reason for the glitch not being taken care of by the TPD chip
+	 * needs to be investigated
+	 */
+	r = gpiod_set_debounce(ddata->hpd_gpio, HPD_DEBOUNCE_TIME);
+	if (r)
+		goto err;
+
+	r = devm_request_threaded_irq(&pdev->dev, gpiod_to_irq(ddata->hpd_gpio),
+		NULL, tpd_hpd_isr,
+		IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
+		"tpd12s015 hpd", ddata);
+	if (r)
+		goto err;
+
+	dssdev = &ddata->dssdev;
+	dssdev->ops = &tpd_ops;
+	dssdev->dev = &pdev->dev;
+	dssdev->type = OMAP_DISPLAY_TYPE_HDMI;
+	dssdev->owner = THIS_MODULE;
+	dssdev->of_ports = BIT(1) | BIT(0);
+	dssdev->ops_flags = OMAP_DSS_DEVICE_OP_DETECT
+			  | OMAP_DSS_DEVICE_OP_EDID
+			  | OMAP_DSS_DEVICE_OP_HPD;
+
+	dssdev->next = omapdss_of_find_connected_device(pdev->dev.of_node, 1);
+	if (IS_ERR(dssdev->next)) {
+		if (PTR_ERR(dssdev->next) != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "failed to find video sink\n");
+		r = PTR_ERR(dssdev->next);
+		goto err;
+	}
+
+	omapdss_device_register(dssdev);
+
+	return 0;
+err:
+	tpd_uninit_i2c_mux(pdev);
+	return r;
+}
+
+static int __exit tpd_remove(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct omap_dss_device *dssdev = &ddata->dssdev;
+
+	tpd_uninit_i2c_mux(pdev);
+
+	if (dssdev->next)
+		omapdss_device_put(dssdev->next);
+	omapdss_device_unregister(&ddata->dssdev);
+
+	return 0;
+}
+
+static const struct of_device_id tpd_of_match[] = {
+	{ .compatible = "omapdss,ti,dra7evm-tpd12s015", },
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, tpd_of_match);
+
+static struct platform_driver tpd_driver = {
+	.probe	= tpd_probe,
+	.remove	= __exit_p(tpd_remove),
+	.driver	= {
+		.name	= "dra7evm-tpd12s015",
+		.of_match_table = tpd_of_match,
+		.suppress_bind_attrs = true,
+	},
+};
+
+module_platform_driver(tpd_driver);
+
+MODULE_AUTHOR("Tomi Valkeinen <tomi.valkeinen@ti.com>");
+MODULE_DESCRIPTION("TPD12S015 driver");
+MODULE_LICENSE("GPL");
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/displays/encoder-opa362.c b/drivers/gpu/drm/omapdrm/displays/encoder-opa362.c
--- a/drivers/gpu/drm/omapdrm/displays/encoder-opa362.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/displays/encoder-opa362.c	2020-06-19 21:00:31.412227715 -0400
@@ -23,162 +23,45 @@
 
 struct panel_drv_data {
 	struct omap_dss_device dssdev;
-	struct omap_dss_device *in;
 
 	struct gpio_desc *enable_gpio;
-
-	struct videomode vm;
 };
 
 #define to_panel_data(x) container_of(x, struct panel_drv_data, dssdev)
 
-static int opa362_connect(struct omap_dss_device *dssdev,
-		struct omap_dss_device *dst)
+static int opa362_connect(struct omap_dss_device *src,
+			  struct omap_dss_device *dst)
 {
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in;
-	int r;
-
-	dev_dbg(dssdev->dev, "connect\n");
-
-	if (omapdss_device_is_connected(dssdev))
-		return -EBUSY;
-
-	in = omapdss_of_find_source_for_first_ep(dssdev->dev->of_node);
-	if (IS_ERR(in)) {
-		dev_err(dssdev->dev, "failed to find video source\n");
-		return PTR_ERR(in);
-	}
-
-	r = in->ops.atv->connect(in, dssdev);
-	if (r) {
-		omap_dss_put_device(in);
-		return r;
-	}
-
-	dst->src = dssdev;
-	dssdev->dst = dst;
-
-	ddata->in = in;
-	return 0;
+	return omapdss_device_connect(dst->dss, dst, dst->next);
 }
 
-static void opa362_disconnect(struct omap_dss_device *dssdev,
-		struct omap_dss_device *dst)
+static void opa362_disconnect(struct omap_dss_device *src,
+			      struct omap_dss_device *dst)
 {
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	dev_dbg(dssdev->dev, "disconnect\n");
-
-	WARN_ON(!omapdss_device_is_connected(dssdev));
-	if (!omapdss_device_is_connected(dssdev))
-		return;
-
-	WARN_ON(dst != dssdev->dst);
-	if (dst != dssdev->dst)
-		return;
-
-	dst->src = NULL;
-	dssdev->dst = NULL;
-
-	in->ops.atv->disconnect(in, &ddata->dssdev);
-
-	omap_dss_put_device(in);
-	ddata->in = NULL;
+	omapdss_device_disconnect(dst, dst->next);
 }
 
-static int opa362_enable(struct omap_dss_device *dssdev)
+static void opa362_enable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-	int r;
-
-	dev_dbg(dssdev->dev, "enable\n");
-
-	if (!omapdss_device_is_connected(dssdev))
-		return -ENODEV;
-
-	if (omapdss_device_is_enabled(dssdev))
-		return 0;
-
-	in->ops.atv->set_timings(in, &ddata->vm);
-
-	r = in->ops.atv->enable(in);
-	if (r)
-		return r;
 
 	if (ddata->enable_gpio)
 		gpiod_set_value_cansleep(ddata->enable_gpio, 1);
-
-	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
-
-	return 0;
 }
 
 static void opa362_disable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	dev_dbg(dssdev->dev, "disable\n");
-
-	if (!omapdss_device_is_enabled(dssdev))
-		return;
 
 	if (ddata->enable_gpio)
 		gpiod_set_value_cansleep(ddata->enable_gpio, 0);
-
-	in->ops.atv->disable(in);
-
-	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
-}
-
-static void opa362_set_timings(struct omap_dss_device *dssdev,
-			       struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	dev_dbg(dssdev->dev, "set_timings\n");
-
-	ddata->vm = *vm;
-	dssdev->panel.vm = *vm;
-
-	in->ops.atv->set_timings(in, vm);
-}
-
-static void opa362_get_timings(struct omap_dss_device *dssdev,
-			       struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-
-	dev_dbg(dssdev->dev, "get_timings\n");
-
-	*vm = ddata->vm;
 }
 
-static int opa362_check_timings(struct omap_dss_device *dssdev,
-				struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	dev_dbg(dssdev->dev, "check_timings\n");
-
-	return in->ops.atv->check_timings(in, vm);
-}
-
-static const struct omapdss_atv_ops opa362_atv_ops = {
+static const struct omap_dss_device_ops opa362_ops = {
 	.connect	= opa362_connect,
 	.disconnect	= opa362_disconnect,
-
 	.enable		= opa362_enable,
 	.disable	= opa362_disable,
-
-	.check_timings	= opa362_check_timings,
-	.set_timings	= opa362_set_timings,
-	.get_timings	= opa362_get_timings,
 };
 
 static int opa362_probe(struct platform_device *pdev)
@@ -186,7 +69,6 @@
 	struct panel_drv_data *ddata;
 	struct omap_dss_device *dssdev;
 	struct gpio_desc *gpio;
-	int r;
 
 	dev_dbg(&pdev->dev, "probe\n");
 
@@ -203,18 +85,21 @@
 	ddata->enable_gpio = gpio;
 
 	dssdev = &ddata->dssdev;
-	dssdev->ops.atv = &opa362_atv_ops;
+	dssdev->ops = &opa362_ops;
 	dssdev->dev = &pdev->dev;
 	dssdev->type = OMAP_DISPLAY_TYPE_VENC;
-	dssdev->output_type = OMAP_DISPLAY_TYPE_VENC;
 	dssdev->owner = THIS_MODULE;
+	dssdev->of_ports = BIT(1) | BIT(0);
 
-	r = omapdss_register_output(dssdev);
-	if (r) {
-		dev_err(&pdev->dev, "Failed to register output\n");
-		return r;
+	dssdev->next = omapdss_of_find_connected_device(pdev->dev.of_node, 1);
+	if (IS_ERR(dssdev->next)) {
+		if (PTR_ERR(dssdev->next) != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "failed to find video sink\n");
+		return PTR_ERR(dssdev->next);
 	}
 
+	omapdss_device_register(dssdev);
+
 	return 0;
 }
 
@@ -223,15 +108,11 @@
 	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
 	struct omap_dss_device *dssdev = &ddata->dssdev;
 
-	omapdss_unregister_output(&ddata->dssdev);
+	if (dssdev->next)
+		omapdss_device_put(dssdev->next);
+	omapdss_device_unregister(&ddata->dssdev);
 
-	WARN_ON(omapdss_device_is_enabled(dssdev));
-	if (omapdss_device_is_enabled(dssdev))
-		opa362_disable(dssdev);
-
-	WARN_ON(omapdss_device_is_connected(dssdev));
-	if (omapdss_device_is_connected(dssdev))
-		opa362_disconnect(dssdev, dssdev->dst);
+	opa362_disable(dssdev);
 
 	return 0;
 }
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/displays/encoder-tfp410.c b/drivers/gpu/drm/omapdrm/displays/encoder-tfp410.c
--- a/drivers/gpu/drm/omapdrm/displays/encoder-tfp410.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/displays/encoder-tfp410.c	1969-12-31 19:00:00.000000000 -0500
@@ -1,277 +0,0 @@
-/*
- * TFP410 DPI-to-DVI encoder driver
- *
- * Copyright (C) 2013 Texas Instruments Incorporated - http://www.ti.com/
- * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- */
-
-#include <linux/gpio/consumer.h>
-#include <linux/module.h>
-#include <linux/platform_device.h>
-#include <linux/slab.h>
-#include <linux/of_gpio.h>
-
-#include "../dss/omapdss.h"
-
-struct panel_drv_data {
-	struct omap_dss_device dssdev;
-	struct omap_dss_device *in;
-
-	int pd_gpio;
-
-	struct videomode vm;
-};
-
-#define to_panel_data(x) container_of(x, struct panel_drv_data, dssdev)
-
-static int tfp410_connect(struct omap_dss_device *dssdev,
-		struct omap_dss_device *dst)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in;
-	int r;
-
-	if (omapdss_device_is_connected(dssdev))
-		return -EBUSY;
-
-	in = omapdss_of_find_source_for_first_ep(dssdev->dev->of_node);
-	if (IS_ERR(in)) {
-		dev_err(dssdev->dev, "failed to find video source\n");
-		return PTR_ERR(in);
-	}
-
-	r = in->ops.dpi->connect(in, dssdev);
-	if (r) {
-		omap_dss_put_device(in);
-		return r;
-	}
-
-	dst->src = dssdev;
-	dssdev->dst = dst;
-
-	ddata->in = in;
-	return 0;
-}
-
-static void tfp410_disconnect(struct omap_dss_device *dssdev,
-		struct omap_dss_device *dst)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	WARN_ON(!omapdss_device_is_connected(dssdev));
-	if (!omapdss_device_is_connected(dssdev))
-		return;
-
-	WARN_ON(dst != dssdev->dst);
-	if (dst != dssdev->dst)
-		return;
-
-	dst->src = NULL;
-	dssdev->dst = NULL;
-
-	in->ops.dpi->disconnect(in, &ddata->dssdev);
-
-	omap_dss_put_device(in);
-	ddata->in = NULL;
-}
-
-static int tfp410_enable(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-	int r;
-
-	if (!omapdss_device_is_connected(dssdev))
-		return -ENODEV;
-
-	if (omapdss_device_is_enabled(dssdev))
-		return 0;
-
-	in->ops.dpi->set_timings(in, &ddata->vm);
-
-	r = in->ops.dpi->enable(in);
-	if (r)
-		return r;
-
-	if (gpio_is_valid(ddata->pd_gpio))
-		gpio_set_value_cansleep(ddata->pd_gpio, 1);
-
-	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
-
-	return 0;
-}
-
-static void tfp410_disable(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	if (!omapdss_device_is_enabled(dssdev))
-		return;
-
-	if (gpio_is_valid(ddata->pd_gpio))
-		gpio_set_value_cansleep(ddata->pd_gpio, 0);
-
-	in->ops.dpi->disable(in);
-
-	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
-}
-
-static void tfp410_fix_timings(struct videomode *vm)
-{
-	vm->flags |= DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_PIXDATA_POSEDGE |
-		     DISPLAY_FLAGS_SYNC_POSEDGE;
-}
-
-static void tfp410_set_timings(struct omap_dss_device *dssdev,
-			       struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	tfp410_fix_timings(vm);
-
-	ddata->vm = *vm;
-	dssdev->panel.vm = *vm;
-
-	in->ops.dpi->set_timings(in, vm);
-}
-
-static void tfp410_get_timings(struct omap_dss_device *dssdev,
-			       struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-
-	*vm = ddata->vm;
-}
-
-static int tfp410_check_timings(struct omap_dss_device *dssdev,
-				struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	tfp410_fix_timings(vm);
-
-	return in->ops.dpi->check_timings(in, vm);
-}
-
-static const struct omapdss_dvi_ops tfp410_dvi_ops = {
-	.connect	= tfp410_connect,
-	.disconnect	= tfp410_disconnect,
-
-	.enable		= tfp410_enable,
-	.disable	= tfp410_disable,
-
-	.check_timings	= tfp410_check_timings,
-	.set_timings	= tfp410_set_timings,
-	.get_timings	= tfp410_get_timings,
-};
-
-static int tfp410_probe_of(struct platform_device *pdev)
-{
-	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-	struct device_node *node = pdev->dev.of_node;
-	int gpio;
-
-	gpio = of_get_named_gpio(node, "powerdown-gpios", 0);
-
-	if (gpio_is_valid(gpio) || gpio == -ENOENT) {
-		ddata->pd_gpio = gpio;
-	} else {
-		if (gpio != -EPROBE_DEFER)
-			dev_err(&pdev->dev, "failed to parse PD gpio\n");
-		return gpio;
-	}
-
-	return 0;
-}
-
-static int tfp410_probe(struct platform_device *pdev)
-{
-	struct panel_drv_data *ddata;
-	struct omap_dss_device *dssdev;
-	int r;
-
-	ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);
-	if (!ddata)
-		return -ENOMEM;
-
-	platform_set_drvdata(pdev, ddata);
-
-	r = tfp410_probe_of(pdev);
-	if (r)
-		return r;
-
-	if (gpio_is_valid(ddata->pd_gpio)) {
-		r = devm_gpio_request_one(&pdev->dev, ddata->pd_gpio,
-				GPIOF_OUT_INIT_LOW, "tfp410 PD");
-		if (r) {
-			dev_err(&pdev->dev, "Failed to request PD GPIO %d\n",
-					ddata->pd_gpio);
-			return r;
-		}
-	}
-
-	dssdev = &ddata->dssdev;
-	dssdev->ops.dvi = &tfp410_dvi_ops;
-	dssdev->dev = &pdev->dev;
-	dssdev->type = OMAP_DISPLAY_TYPE_DPI;
-	dssdev->output_type = OMAP_DISPLAY_TYPE_DVI;
-	dssdev->owner = THIS_MODULE;
-	dssdev->port_num = 1;
-
-	r = omapdss_register_output(dssdev);
-	if (r) {
-		dev_err(&pdev->dev, "Failed to register output\n");
-		return r;
-	}
-
-	return 0;
-}
-
-static int __exit tfp410_remove(struct platform_device *pdev)
-{
-	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-	struct omap_dss_device *dssdev = &ddata->dssdev;
-
-	omapdss_unregister_output(&ddata->dssdev);
-
-	WARN_ON(omapdss_device_is_enabled(dssdev));
-	if (omapdss_device_is_enabled(dssdev))
-		tfp410_disable(dssdev);
-
-	WARN_ON(omapdss_device_is_connected(dssdev));
-	if (omapdss_device_is_connected(dssdev))
-		tfp410_disconnect(dssdev, dssdev->dst);
-
-	return 0;
-}
-
-static const struct of_device_id tfp410_of_match[] = {
-	{ .compatible = "omapdss,ti,tfp410", },
-	{},
-};
-
-MODULE_DEVICE_TABLE(of, tfp410_of_match);
-
-static struct platform_driver tfp410_driver = {
-	.probe	= tfp410_probe,
-	.remove	= __exit_p(tfp410_remove),
-	.driver	= {
-		.name	= "tfp410",
-		.of_match_table = tfp410_of_match,
-		.suppress_bind_attrs = true,
-	},
-};
-
-module_platform_driver(tfp410_driver);
-
-MODULE_AUTHOR("Tomi Valkeinen <tomi.valkeinen@ti.com>");
-MODULE_DESCRIPTION("TFP410 DPI to DVI encoder driver");
-MODULE_LICENSE("GPL");
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c b/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c
--- a/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c	2020-06-19 21:00:31.412227715 -0400
@@ -21,42 +21,26 @@
 
 struct panel_drv_data {
 	struct omap_dss_device dssdev;
-	struct omap_dss_device *in;
 	void (*hpd_cb)(void *cb_data, enum drm_connector_status status);
 	void *hpd_cb_data;
-	bool hpd_enabled;
 	struct mutex hpd_lock;
 
 	struct gpio_desc *ct_cp_hpd_gpio;
 	struct gpio_desc *ls_oe_gpio;
 	struct gpio_desc *hpd_gpio;
-
-	struct videomode vm;
 };
 
 #define to_panel_data(x) container_of(x, struct panel_drv_data, dssdev)
 
-static int tpd_connect(struct omap_dss_device *dssdev,
-		struct omap_dss_device *dst)
+static int tpd_connect(struct omap_dss_device *src,
+		       struct omap_dss_device *dst)
 {
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in;
+	struct panel_drv_data *ddata = to_panel_data(dst);
 	int r;
 
-	in = omapdss_of_find_source_for_first_ep(dssdev->dev->of_node);
-	if (IS_ERR(in)) {
-		dev_err(dssdev->dev, "failed to find video source\n");
-		return PTR_ERR(in);
-	}
-
-	r = in->ops.hdmi->connect(in, dssdev);
-	if (r) {
-		omap_dss_put_device(in);
+	r = omapdss_device_connect(dst->dss, dst, dst->next);
+	if (r)
 		return r;
-	}
-
-	dst->src = dssdev;
-	dssdev->dst = dst;
 
 	gpiod_set_value_cansleep(ddata->ct_cp_hpd_gpio, 1);
 	gpiod_set_value_cansleep(ddata->ls_oe_gpio, 1);
@@ -64,125 +48,31 @@
 	/* DC-DC converter needs at max 300us to get to 90% of 5V */
 	udelay(300);
 
-	ddata->in = in;
 	return 0;
 }
 
-static void tpd_disconnect(struct omap_dss_device *dssdev,
-		struct omap_dss_device *dst)
+static void tpd_disconnect(struct omap_dss_device *src,
+			   struct omap_dss_device *dst)
 {
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	WARN_ON(dst != dssdev->dst);
-
-	if (dst != dssdev->dst)
-		return;
+	struct panel_drv_data *ddata = to_panel_data(dst);
 
 	gpiod_set_value_cansleep(ddata->ct_cp_hpd_gpio, 0);
 	gpiod_set_value_cansleep(ddata->ls_oe_gpio, 0);
 
-	dst->src = NULL;
-	dssdev->dst = NULL;
-
-	in->ops.hdmi->disconnect(in, &ddata->dssdev);
-
-	omap_dss_put_device(in);
-	ddata->in = NULL;
-}
-
-static int tpd_enable(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-	int r;
-
-	if (dssdev->state == OMAP_DSS_DISPLAY_ACTIVE)
-		return 0;
-
-	in->ops.hdmi->set_timings(in, &ddata->vm);
-
-	r = in->ops.hdmi->enable(in);
-	if (r)
-		return r;
-
-	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
-
-	return r;
-}
-
-static void tpd_disable(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	if (dssdev->state != OMAP_DSS_DISPLAY_ACTIVE)
-		return;
-
-	in->ops.hdmi->disable(in);
-
-	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
-}
-
-static void tpd_set_timings(struct omap_dss_device *dssdev,
-			    struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	ddata->vm = *vm;
-	dssdev->panel.vm = *vm;
-
-	in->ops.hdmi->set_timings(in, vm);
-}
-
-static void tpd_get_timings(struct omap_dss_device *dssdev,
-			    struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-
-	*vm = ddata->vm;
-}
-
-static int tpd_check_timings(struct omap_dss_device *dssdev,
-			     struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-	int r;
-
-	r = in->ops.hdmi->check_timings(in, vm);
-
-	return r;
-}
-
-static int tpd_read_edid(struct omap_dss_device *dssdev,
-		u8 *edid, int len)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	if (!gpiod_get_value_cansleep(ddata->hpd_gpio))
-		return -ENODEV;
-
-	return in->ops.hdmi->read_edid(in, edid, len);
+	omapdss_device_disconnect(dst, dst->next);
 }
 
 static bool tpd_detect(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-	bool connected = gpiod_get_value_cansleep(ddata->hpd_gpio);
 
-	if (!connected && in->ops.hdmi->lost_hotplug)
-		in->ops.hdmi->lost_hotplug(in);
-	return connected;
+	return gpiod_get_value_cansleep(ddata->hpd_gpio);
 }
 
-static int tpd_register_hpd_cb(struct omap_dss_device *dssdev,
-			       void (*cb)(void *cb_data,
+static void tpd_register_hpd_cb(struct omap_dss_device *dssdev,
+				void (*cb)(void *cb_data,
 					  enum drm_connector_status status),
-			       void *cb_data)
+				void *cb_data)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
@@ -190,8 +80,6 @@
 	ddata->hpd_cb = cb;
 	ddata->hpd_cb_data = cb_data;
 	mutex_unlock(&ddata->hpd_lock);
-
-	return 0;
 }
 
 static void tpd_unregister_hpd_cb(struct omap_dss_device *dssdev)
@@ -204,61 +92,12 @@
 	mutex_unlock(&ddata->hpd_lock);
 }
 
-static void tpd_enable_hpd(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-
-	mutex_lock(&ddata->hpd_lock);
-	ddata->hpd_enabled = true;
-	mutex_unlock(&ddata->hpd_lock);
-}
-
-static void tpd_disable_hpd(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-
-	mutex_lock(&ddata->hpd_lock);
-	ddata->hpd_enabled = false;
-	mutex_unlock(&ddata->hpd_lock);
-}
-
-static int tpd_set_infoframe(struct omap_dss_device *dssdev,
-		const struct hdmi_avi_infoframe *avi)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	return in->ops.hdmi->set_infoframe(in, avi);
-}
-
-static int tpd_set_hdmi_mode(struct omap_dss_device *dssdev,
-		bool hdmi_mode)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	return in->ops.hdmi->set_hdmi_mode(in, hdmi_mode);
-}
-
-static const struct omapdss_hdmi_ops tpd_hdmi_ops = {
+static const struct omap_dss_device_ops tpd_ops = {
 	.connect		= tpd_connect,
 	.disconnect		= tpd_disconnect,
-
-	.enable			= tpd_enable,
-	.disable		= tpd_disable,
-
-	.check_timings		= tpd_check_timings,
-	.set_timings		= tpd_set_timings,
-	.get_timings		= tpd_get_timings,
-
-	.read_edid		= tpd_read_edid,
 	.detect			= tpd_detect,
 	.register_hpd_cb	= tpd_register_hpd_cb,
 	.unregister_hpd_cb	= tpd_unregister_hpd_cb,
-	.enable_hpd		= tpd_enable_hpd,
-	.disable_hpd		= tpd_disable_hpd,
-	.set_infoframe		= tpd_set_infoframe,
-	.set_hdmi_mode		= tpd_set_hdmi_mode,
 };
 
 static irqreturn_t tpd_hpd_isr(int irq, void *data)
@@ -266,7 +105,7 @@
 	struct panel_drv_data *ddata = data;
 
 	mutex_lock(&ddata->hpd_lock);
-	if (ddata->hpd_enabled && ddata->hpd_cb) {
+	if (ddata->hpd_cb) {
 		enum drm_connector_status status;
 
 		if (tpd_detect(&ddata->dssdev))
@@ -283,7 +122,7 @@
 
 static int tpd_probe(struct platform_device *pdev)
 {
-	struct omap_dss_device *in, *dssdev;
+	struct omap_dss_device *dssdev;
 	struct panel_drv_data *ddata;
 	int r;
 	struct gpio_desc *gpio;
@@ -325,21 +164,23 @@
 		return r;
 
 	dssdev = &ddata->dssdev;
-	dssdev->ops.hdmi = &tpd_hdmi_ops;
+	dssdev->ops = &tpd_ops;
 	dssdev->dev = &pdev->dev;
 	dssdev->type = OMAP_DISPLAY_TYPE_HDMI;
-	dssdev->output_type = OMAP_DISPLAY_TYPE_HDMI;
 	dssdev->owner = THIS_MODULE;
-	dssdev->port_num = 1;
-
-	in = ddata->in;
-
-	r = omapdss_register_output(dssdev);
-	if (r) {
-		dev_err(&pdev->dev, "Failed to register output\n");
-		return r;
+	dssdev->of_ports = BIT(1) | BIT(0);
+	dssdev->ops_flags = OMAP_DSS_DEVICE_OP_DETECT
+			  | OMAP_DSS_DEVICE_OP_HPD;
+
+	dssdev->next = omapdss_of_find_connected_device(pdev->dev.of_node, 1);
+	if (IS_ERR(dssdev->next)) {
+		if (PTR_ERR(dssdev->next) != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "failed to find video sink\n");
+		return PTR_ERR(dssdev->next);
 	}
 
+	omapdss_device_register(dssdev);
+
 	return 0;
 }
 
@@ -348,15 +189,9 @@
 	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
 	struct omap_dss_device *dssdev = &ddata->dssdev;
 
-	omapdss_unregister_output(&ddata->dssdev);
-
-	WARN_ON(omapdss_device_is_enabled(dssdev));
-	if (omapdss_device_is_enabled(dssdev))
-		tpd_disable(dssdev);
-
-	WARN_ON(omapdss_device_is_connected(dssdev));
-	if (omapdss_device_is_connected(dssdev))
-		tpd_disconnect(dssdev, dssdev->dst);
+	if (dssdev->next)
+		omapdss_device_put(dssdev->next);
+	omapdss_device_unregister(&ddata->dssdev);
 
 	return 0;
 }
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/displays/Kconfig b/drivers/gpu/drm/omapdrm/displays/Kconfig
--- a/drivers/gpu/drm/omapdrm/displays/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/displays/Kconfig	2020-06-19 21:00:31.412227715 -0400
@@ -6,22 +6,18 @@
 	  Driver for OPA362 external analog TV amplifier controlled
 	  through a GPIO.
 
-config DRM_OMAP_ENCODER_TFP410
-        tristate "TFP410 DPI to DVI Encoder"
-	help
-	  Driver for TFP410 DPI to DVI encoder.
-
 config DRM_OMAP_ENCODER_TPD12S015
         tristate "TPD12S015 HDMI ESD protection and level shifter"
 	help
 	  Driver for TPD12S015, which offers HDMI ESD protection and level
 	  shifting.
 
-config DRM_OMAP_CONNECTOR_DVI
-        tristate "DVI Connector"
-	depends on I2C
+config DRM_OMAP_DRA7EVM_ENCODER_TPD12S015
+        tristate "DRA7 EVM TPD12S015 HDMI ESD protection and level shifter"
+	depends on SND_SOC_DAVINCI_MCASP
 	help
-	  Driver for a generic DVI connector.
+	  A custom TPD12S015 driver for the DRA7 EVM board, it contains some
+	  hacks required for HDMI to work properly on J6 EVM.
 
 config DRM_OMAP_CONNECTOR_HDMI
         tristate "HDMI Connector"
@@ -33,12 +29,6 @@
 	help
 	  Driver for a generic analog TV connector.
 
-config DRM_OMAP_PANEL_DPI
-	tristate "Generic DPI panel"
-	depends on BACKLIGHT_CLASS_DEVICE
-	help
-	  Driver for generic DPI panels.
-
 config DRM_OMAP_PANEL_DSI_CM
 	tristate "Generic DSI Command Mode Panel"
 	depends on BACKLIGHT_CLASS_DEVICE
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/displays/Makefile b/drivers/gpu/drm/omapdrm/displays/Makefile
--- a/drivers/gpu/drm/omapdrm/displays/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/displays/Makefile	2020-06-19 21:00:31.412227715 -0400
@@ -1,11 +1,9 @@
 # SPDX-License-Identifier: GPL-2.0
 obj-$(CONFIG_DRM_OMAP_ENCODER_OPA362) += encoder-opa362.o
-obj-$(CONFIG_DRM_OMAP_ENCODER_TFP410) += encoder-tfp410.o
 obj-$(CONFIG_DRM_OMAP_ENCODER_TPD12S015) += encoder-tpd12s015.o
-obj-$(CONFIG_DRM_OMAP_CONNECTOR_DVI) += connector-dvi.o
+obj-$(CONFIG_DRM_OMAP_DRA7EVM_ENCODER_TPD12S015) += dra7-evm-encoder-tpd12s015.o
 obj-$(CONFIG_DRM_OMAP_CONNECTOR_HDMI) += connector-hdmi.o
 obj-$(CONFIG_DRM_OMAP_CONNECTOR_ANALOG_TV) += connector-analog-tv.o
-obj-$(CONFIG_DRM_OMAP_PANEL_DPI) += panel-dpi.o
 obj-$(CONFIG_DRM_OMAP_PANEL_DSI_CM) += panel-dsi-cm.o
 obj-$(CONFIG_DRM_OMAP_PANEL_SONY_ACX565AKM) += panel-sony-acx565akm.o
 obj-$(CONFIG_DRM_OMAP_PANEL_LGPHILIPS_LB035Q02) += panel-lgphilips-lb035q02.o
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/displays/panel-dpi.c b/drivers/gpu/drm/omapdrm/displays/panel-dpi.c
--- a/drivers/gpu/drm/omapdrm/displays/panel-dpi.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/displays/panel-dpi.c	1969-12-31 19:00:00.000000000 -0500
@@ -1,278 +0,0 @@
-/*
- * Generic MIPI DPI Panel Driver
- *
- * Copyright (C) 2013 Texas Instruments Incorporated - http://www.ti.com/
- * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- */
-
-#include <linux/gpio/consumer.h>
-#include <linux/module.h>
-#include <linux/platform_device.h>
-#include <linux/slab.h>
-#include <linux/of.h>
-#include <linux/regulator/consumer.h>
-#include <linux/backlight.h>
-
-#include <video/of_display_timing.h>
-
-#include "../dss/omapdss.h"
-
-struct panel_drv_data {
-	struct omap_dss_device dssdev;
-	struct omap_dss_device *in;
-
-	struct videomode vm;
-
-	struct backlight_device *backlight;
-
-	struct gpio_desc *enable_gpio;
-	struct regulator *vcc_supply;
-};
-
-#define to_panel_data(p) container_of(p, struct panel_drv_data, dssdev)
-
-static int panel_dpi_connect(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in;
-	int r;
-
-	if (omapdss_device_is_connected(dssdev))
-		return 0;
-
-	in = omapdss_of_find_source_for_first_ep(dssdev->dev->of_node);
-	if (IS_ERR(in)) {
-		dev_err(dssdev->dev, "failed to find video source\n");
-		return PTR_ERR(in);
-	}
-
-	r = in->ops.dpi->connect(in, dssdev);
-	if (r) {
-		omap_dss_put_device(in);
-		return r;
-	}
-
-	ddata->in = in;
-	return 0;
-}
-
-static void panel_dpi_disconnect(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	if (!omapdss_device_is_connected(dssdev))
-		return;
-
-	in->ops.dpi->disconnect(in, dssdev);
-
-	omap_dss_put_device(in);
-	ddata->in = NULL;
-}
-
-static int panel_dpi_enable(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-	int r;
-
-	if (!omapdss_device_is_connected(dssdev))
-		return -ENODEV;
-
-	if (omapdss_device_is_enabled(dssdev))
-		return 0;
-
-	in->ops.dpi->set_timings(in, &ddata->vm);
-
-	r = in->ops.dpi->enable(in);
-	if (r)
-		return r;
-
-	r = regulator_enable(ddata->vcc_supply);
-	if (r) {
-		in->ops.dpi->disable(in);
-		return r;
-	}
-
-	gpiod_set_value_cansleep(ddata->enable_gpio, 1);
-	backlight_enable(ddata->backlight);
-
-	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
-
-	return 0;
-}
-
-static void panel_dpi_disable(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	if (!omapdss_device_is_enabled(dssdev))
-		return;
-
-	backlight_disable(ddata->backlight);
-
-	gpiod_set_value_cansleep(ddata->enable_gpio, 0);
-	regulator_disable(ddata->vcc_supply);
-
-	in->ops.dpi->disable(in);
-
-	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
-}
-
-static void panel_dpi_set_timings(struct omap_dss_device *dssdev,
-				  struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	ddata->vm = *vm;
-	dssdev->panel.vm = *vm;
-
-	in->ops.dpi->set_timings(in, vm);
-}
-
-static void panel_dpi_get_timings(struct omap_dss_device *dssdev,
-				  struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-
-	*vm = ddata->vm;
-}
-
-static int panel_dpi_check_timings(struct omap_dss_device *dssdev,
-				   struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	return in->ops.dpi->check_timings(in, vm);
-}
-
-static struct omap_dss_driver panel_dpi_ops = {
-	.connect	= panel_dpi_connect,
-	.disconnect	= panel_dpi_disconnect,
-
-	.enable		= panel_dpi_enable,
-	.disable	= panel_dpi_disable,
-
-	.set_timings	= panel_dpi_set_timings,
-	.get_timings	= panel_dpi_get_timings,
-	.check_timings	= panel_dpi_check_timings,
-};
-
-static int panel_dpi_probe_of(struct platform_device *pdev)
-{
-	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-	struct device_node *node = pdev->dev.of_node;
-	int r;
-	struct display_timing timing;
-	struct gpio_desc *gpio;
-
-	gpio = devm_gpiod_get_optional(&pdev->dev, "enable", GPIOD_OUT_LOW);
-	if (IS_ERR(gpio))
-		return PTR_ERR(gpio);
-
-	ddata->enable_gpio = gpio;
-
-	/*
-	 * Many different panels are supported by this driver and there are
-	 * probably very different needs for their reset pins in regards to
-	 * timing and order relative to the enable gpio. So for now it's just
-	 * ensured that the reset line isn't active.
-	 */
-	gpio = devm_gpiod_get_optional(&pdev->dev, "reset", GPIOD_OUT_LOW);
-	if (IS_ERR(gpio))
-		return PTR_ERR(gpio);
-
-	ddata->vcc_supply = devm_regulator_get(&pdev->dev, "vcc");
-	if (IS_ERR(ddata->vcc_supply))
-		return PTR_ERR(ddata->vcc_supply);
-
-	ddata->backlight = devm_of_find_backlight(&pdev->dev);
-
-	if (IS_ERR(ddata->backlight))
-		return PTR_ERR(ddata->backlight);
-
-	r = of_get_display_timing(node, "panel-timing", &timing);
-	if (r) {
-		dev_err(&pdev->dev, "failed to get video timing\n");
-		return r;
-	}
-
-	videomode_from_timing(&timing, &ddata->vm);
-
-	return 0;
-}
-
-static int panel_dpi_probe(struct platform_device *pdev)
-{
-	struct panel_drv_data *ddata;
-	struct omap_dss_device *dssdev;
-	int r;
-
-	ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);
-	if (ddata == NULL)
-		return -ENOMEM;
-
-	platform_set_drvdata(pdev, ddata);
-
-	r = panel_dpi_probe_of(pdev);
-	if (r)
-		return r;
-
-	dssdev = &ddata->dssdev;
-	dssdev->dev = &pdev->dev;
-	dssdev->driver = &panel_dpi_ops;
-	dssdev->type = OMAP_DISPLAY_TYPE_DPI;
-	dssdev->owner = THIS_MODULE;
-	dssdev->panel.vm = ddata->vm;
-
-	r = omapdss_register_display(dssdev);
-	if (r) {
-		dev_err(&pdev->dev, "Failed to register panel\n");
-		return r;
-	}
-
-	return 0;
-}
-
-static int __exit panel_dpi_remove(struct platform_device *pdev)
-{
-	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-	struct omap_dss_device *dssdev = &ddata->dssdev;
-
-	omapdss_unregister_display(dssdev);
-
-	panel_dpi_disable(dssdev);
-	panel_dpi_disconnect(dssdev);
-
-	return 0;
-}
-
-static const struct of_device_id panel_dpi_of_match[] = {
-	{ .compatible = "omapdss,panel-dpi", },
-	{},
-};
-
-MODULE_DEVICE_TABLE(of, panel_dpi_of_match);
-
-static struct platform_driver panel_dpi_driver = {
-	.probe = panel_dpi_probe,
-	.remove = __exit_p(panel_dpi_remove),
-	.driver = {
-		.name = "panel-dpi",
-		.of_match_table = panel_dpi_of_match,
-		.suppress_bind_attrs = true,
-	},
-};
-
-module_platform_driver(panel_dpi_driver);
-
-MODULE_AUTHOR("Tomi Valkeinen <tomi.valkeinen@ti.com>");
-MODULE_DESCRIPTION("Generic MIPI DPI Panel Driver");
-MODULE_LICENSE("GPL");
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c b/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c
--- a/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c	2020-06-19 21:00:31.412227715 -0400
@@ -24,6 +24,8 @@
 #include <linux/of_device.h>
 #include <linux/regulator/consumer.h>
 
+#include <drm/drm_connector.h>
+
 #include <video/mipi_display.h>
 #include <video/of_display_timing.h>
 
@@ -41,7 +43,7 @@
 
 struct panel_drv_data {
 	struct omap_dss_device dssdev;
-	struct omap_dss_device *in;
+	struct omap_dss_device *src;
 
 	struct videomode vm;
 
@@ -142,11 +144,11 @@
 
 static int dsicm_dcs_read_1(struct panel_drv_data *ddata, u8 dcs_cmd, u8 *data)
 {
-	struct omap_dss_device *in = ddata->in;
+	struct omap_dss_device *src = ddata->src;
 	int r;
 	u8 buf[1];
 
-	r = in->ops.dsi->dcs_read(in, ddata->channel, dcs_cmd, buf, 1);
+	r = src->ops->dsi.dcs_read(src, ddata->channel, dcs_cmd, buf, 1);
 
 	if (r < 0)
 		return r;
@@ -158,29 +160,30 @@
 
 static int dsicm_dcs_write_0(struct panel_drv_data *ddata, u8 dcs_cmd)
 {
-	struct omap_dss_device *in = ddata->in;
-	return in->ops.dsi->dcs_write(in, ddata->channel, &dcs_cmd, 1);
+	struct omap_dss_device *src = ddata->src;
+
+	return src->ops->dsi.dcs_write(src, ddata->channel, &dcs_cmd, 1);
 }
 
 static int dsicm_dcs_write_1(struct panel_drv_data *ddata, u8 dcs_cmd, u8 param)
 {
-	struct omap_dss_device *in = ddata->in;
+	struct omap_dss_device *src = ddata->src;
 	u8 buf[2] = { dcs_cmd, param };
 
-	return in->ops.dsi->dcs_write(in, ddata->channel, buf, 2);
+	return src->ops->dsi.dcs_write(src, ddata->channel, buf, 2);
 }
 
 static int dsicm_sleep_in(struct panel_drv_data *ddata)
 
 {
-	struct omap_dss_device *in = ddata->in;
+	struct omap_dss_device *src = ddata->src;
 	u8 cmd;
 	int r;
 
 	hw_guard_wait(ddata);
 
 	cmd = MIPI_DCS_ENTER_SLEEP_MODE;
-	r = in->ops.dsi->dcs_write_nosync(in, ddata->channel, &cmd, 1);
+	r = src->ops->dsi.dcs_write_nosync(src, ddata->channel, &cmd, 1);
 	if (r)
 		return r;
 
@@ -228,7 +231,7 @@
 static int dsicm_set_update_window(struct panel_drv_data *ddata,
 		u16 x, u16 y, u16 w, u16 h)
 {
-	struct omap_dss_device *in = ddata->in;
+	struct omap_dss_device *src = ddata->src;
 	int r;
 	u16 x1 = x;
 	u16 x2 = x + w - 1;
@@ -242,7 +245,7 @@
 	buf[3] = (x2 >> 8) & 0xff;
 	buf[4] = (x2 >> 0) & 0xff;
 
-	r = in->ops.dsi->dcs_write_nosync(in, ddata->channel, buf, sizeof(buf));
+	r = src->ops->dsi.dcs_write_nosync(src, ddata->channel, buf, sizeof(buf));
 	if (r)
 		return r;
 
@@ -252,11 +255,11 @@
 	buf[3] = (y2 >> 8) & 0xff;
 	buf[4] = (y2 >> 0) & 0xff;
 
-	r = in->ops.dsi->dcs_write_nosync(in, ddata->channel, buf, sizeof(buf));
+	r = src->ops->dsi.dcs_write_nosync(src, ddata->channel, buf, sizeof(buf));
 	if (r)
 		return r;
 
-	in->ops.dsi->bta_sync(in, ddata->channel);
+	src->ops->dsi.bta_sync(src, ddata->channel);
 
 	return r;
 }
@@ -275,7 +278,7 @@
 
 static int dsicm_enter_ulps(struct panel_drv_data *ddata)
 {
-	struct omap_dss_device *in = ddata->in;
+	struct omap_dss_device *src = ddata->src;
 	int r;
 
 	if (ddata->ulps_enabled)
@@ -290,7 +293,7 @@
 	if (ddata->ext_te_gpio)
 		disable_irq(gpiod_to_irq(ddata->ext_te_gpio));
 
-	in->ops.dsi->disable(in, false, true);
+	src->ops->dsi.disable(src, false, true);
 
 	ddata->ulps_enabled = true;
 
@@ -309,19 +312,14 @@
 
 static int dsicm_exit_ulps(struct panel_drv_data *ddata)
 {
-	struct omap_dss_device *in = ddata->in;
+	struct omap_dss_device *src = ddata->src;
 	int r;
 
 	if (!ddata->ulps_enabled)
 		return 0;
 
-	r = in->ops.dsi->enable(in);
-	if (r) {
-		dev_err(&ddata->pdev->dev, "failed to enable DSI\n");
-		goto err1;
-	}
-
-	in->ops.dsi->enable_hs(in, ddata->channel, true);
+	src->ops->enable(src);
+	src->ops->dsi.enable_hs(src, ddata->channel, true);
 
 	r = _dsicm_enable_te(ddata, true);
 	if (r) {
@@ -347,7 +345,7 @@
 			enable_irq(gpiod_to_irq(ddata->ext_te_gpio));
 		ddata->ulps_enabled = false;
 	}
-err1:
+
 	dsicm_queue_ulps_work(ddata);
 
 	return r;
@@ -366,7 +364,7 @@
 static int dsicm_bl_update_status(struct backlight_device *dev)
 {
 	struct panel_drv_data *ddata = dev_get_drvdata(&dev->dev);
-	struct omap_dss_device *in = ddata->in;
+	struct omap_dss_device *src = ddata->src;
 	int r = 0;
 	int level;
 
@@ -381,13 +379,13 @@
 	mutex_lock(&ddata->lock);
 
 	if (ddata->enabled) {
-		in->ops.dsi->bus_lock(in);
+		src->ops->dsi.bus_lock(src);
 
 		r = dsicm_wake_up(ddata);
 		if (!r)
 			r = dsicm_dcs_write_1(ddata, DCS_BRIGHTNESS, level);
 
-		in->ops.dsi->bus_unlock(in);
+		src->ops->dsi.bus_unlock(src);
 	}
 
 	mutex_unlock(&ddata->lock);
@@ -414,21 +412,21 @@
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-	struct omap_dss_device *in = ddata->in;
+	struct omap_dss_device *src = ddata->src;
 	u8 errors = 0;
 	int r;
 
 	mutex_lock(&ddata->lock);
 
 	if (ddata->enabled) {
-		in->ops.dsi->bus_lock(in);
+		src->ops->dsi.bus_lock(src);
 
 		r = dsicm_wake_up(ddata);
 		if (!r)
 			r = dsicm_dcs_read_1(ddata, DCS_READ_NUM_ERRORS,
 					&errors);
 
-		in->ops.dsi->bus_unlock(in);
+		src->ops->dsi.bus_unlock(src);
 	} else {
 		r = -ENODEV;
 	}
@@ -446,20 +444,20 @@
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-	struct omap_dss_device *in = ddata->in;
+	struct omap_dss_device *src = ddata->src;
 	u8 id1, id2, id3;
 	int r;
 
 	mutex_lock(&ddata->lock);
 
 	if (ddata->enabled) {
-		in->ops.dsi->bus_lock(in);
+		src->ops->dsi.bus_lock(src);
 
 		r = dsicm_wake_up(ddata);
 		if (!r)
 			r = dsicm_get_id(ddata, &id1, &id2, &id3);
 
-		in->ops.dsi->bus_unlock(in);
+		src->ops->dsi.bus_unlock(src);
 	} else {
 		r = -ENODEV;
 	}
@@ -478,7 +476,7 @@
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-	struct omap_dss_device *in = ddata->in;
+	struct omap_dss_device *src = ddata->src;
 	unsigned long t;
 	int r;
 
@@ -489,14 +487,14 @@
 	mutex_lock(&ddata->lock);
 
 	if (ddata->enabled) {
-		in->ops.dsi->bus_lock(in);
+		src->ops->dsi.bus_lock(src);
 
 		if (t)
 			r = dsicm_enter_ulps(ddata);
 		else
 			r = dsicm_wake_up(ddata);
 
-		in->ops.dsi->bus_unlock(in);
+		src->ops->dsi.bus_unlock(src);
 	}
 
 	mutex_unlock(&ddata->lock);
@@ -528,7 +526,7 @@
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-	struct omap_dss_device *in = ddata->in;
+	struct omap_dss_device *src = ddata->src;
 	unsigned long t;
 	int r;
 
@@ -541,9 +539,9 @@
 
 	if (ddata->enabled) {
 		/* dsicm_wake_up will restart the timer */
-		in->ops.dsi->bus_lock(in);
+		src->ops->dsi.bus_lock(src);
 		r = dsicm_wake_up(ddata);
-		in->ops.dsi->bus_unlock(in);
+		src->ops->dsi.bus_unlock(src);
 	}
 
 	mutex_unlock(&ddata->lock);
@@ -603,7 +601,7 @@
 
 static int dsicm_power_on(struct panel_drv_data *ddata)
 {
-	struct omap_dss_device *in = ddata->in;
+	struct omap_dss_device *src = ddata->src;
 	u8 id1, id2, id3;
 	int r;
 	struct omap_dss_dsi_config dsi_config = {
@@ -635,7 +633,7 @@
 	}
 
 	if (ddata->pin_config.num_pins > 0) {
-		r = in->ops.dsi->configure_pins(in, &ddata->pin_config);
+		r = src->ops->dsi.configure_pins(src, &ddata->pin_config);
 		if (r) {
 			dev_err(&ddata->pdev->dev,
 				"failed to configure DSI pins\n");
@@ -643,21 +641,17 @@
 		}
 	}
 
-	r = in->ops.dsi->set_config(in, &dsi_config);
+	r = src->ops->dsi.set_config(src, &dsi_config);
 	if (r) {
 		dev_err(&ddata->pdev->dev, "failed to configure DSI\n");
 		goto err_vddi;
 	}
 
-	r = in->ops.dsi->enable(in);
-	if (r) {
-		dev_err(&ddata->pdev->dev, "failed to enable DSI\n");
-		goto err_vddi;
-	}
+	src->ops->enable(src);
 
 	dsicm_hw_reset(ddata);
 
-	in->ops.dsi->enable_hs(in, ddata->channel, false);
+	src->ops->dsi.enable_hs(src, ddata->channel, false);
 
 	r = dsicm_sleep_out(ddata);
 	if (r)
@@ -689,7 +683,7 @@
 	if (r)
 		goto err;
 
-	r = in->ops.dsi->enable_video_output(in, ddata->channel);
+	r = src->ops->dsi.enable_video_output(src, ddata->channel);
 	if (r)
 		goto err;
 
@@ -701,7 +695,7 @@
 		ddata->intro_printed = true;
 	}
 
-	in->ops.dsi->enable_hs(in, ddata->channel, true);
+	src->ops->dsi.enable_hs(src, ddata->channel, true);
 
 	return 0;
 err:
@@ -709,7 +703,7 @@
 
 	dsicm_hw_reset(ddata);
 
-	in->ops.dsi->disable(in, true, false);
+	src->ops->dsi.disable(src, true, false);
 err_vddi:
 	if (ddata->vddi)
 		regulator_disable(ddata->vddi);
@@ -722,10 +716,10 @@
 
 static void dsicm_power_off(struct panel_drv_data *ddata)
 {
-	struct omap_dss_device *in = ddata->in;
+	struct omap_dss_device *src = ddata->src;
 	int r;
 
-	in->ops.dsi->disable_video_output(in, ddata->channel);
+	src->ops->dsi.disable_video_output(src, ddata->channel);
 
 	r = dsicm_dcs_write_0(ddata, MIPI_DCS_SET_DISPLAY_OFF);
 	if (!r)
@@ -737,7 +731,7 @@
 		dsicm_hw_reset(ddata);
 	}
 
-	in->ops.dsi->disable(in, true, false);
+	src->ops->dsi.disable(src, true, false);
 
 	if (ddata->vddi)
 		regulator_disable(ddata->vddi);
@@ -756,134 +750,85 @@
 	return dsicm_power_on(ddata);
 }
 
-static int dsicm_connect(struct omap_dss_device *dssdev)
+static int dsicm_connect(struct omap_dss_device *src,
+			 struct omap_dss_device *dst)
 {
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct panel_drv_data *ddata = to_panel_data(dst);
 	struct device *dev = &ddata->pdev->dev;
-	struct omap_dss_device *in;
 	int r;
 
-	if (omapdss_device_is_connected(dssdev))
-		return 0;
-
-	in = omapdss_of_find_source_for_first_ep(dssdev->dev->of_node);
-	if (IS_ERR(in)) {
-		dev_err(dssdev->dev, "failed to find video source\n");
-		return PTR_ERR(in);
-	}
-
-	r = in->ops.dsi->connect(in, dssdev);
-	if (r) {
-		dev_err(dev, "Failed to connect to video source\n");
-		goto err_connect;
-	}
-
-	r = in->ops.dsi->request_vc(in, &ddata->channel);
+	r = src->ops->dsi.request_vc(src, &ddata->channel);
 	if (r) {
 		dev_err(dev, "failed to get virtual channel\n");
-		goto err_req_vc;
+		return r;
 	}
 
-	r = in->ops.dsi->set_vc_id(in, ddata->channel, TCH);
+	r = src->ops->dsi.set_vc_id(src, ddata->channel, TCH);
 	if (r) {
 		dev_err(dev, "failed to set VC_ID\n");
-		goto err_vc_id;
+		src->ops->dsi.release_vc(src, ddata->channel);
+		return r;
 	}
 
-	ddata->in = in;
+	ddata->src = src;
 	return 0;
-
-err_vc_id:
-	in->ops.dsi->release_vc(in, ddata->channel);
-err_req_vc:
-	in->ops.dsi->disconnect(in, dssdev);
-err_connect:
-	omap_dss_put_device(in);
-	return r;
 }
 
-static void dsicm_disconnect(struct omap_dss_device *dssdev)
+static void dsicm_disconnect(struct omap_dss_device *src,
+			     struct omap_dss_device *dst)
 {
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	if (!omapdss_device_is_connected(dssdev))
-		return;
-
-	in->ops.dsi->release_vc(in, ddata->channel);
-	in->ops.dsi->disconnect(in, dssdev);
+	struct panel_drv_data *ddata = to_panel_data(dst);
 
-	omap_dss_put_device(in);
-	ddata->in = NULL;
+	src->ops->dsi.release_vc(src, ddata->channel);
+	ddata->src = NULL;
 }
 
-static int dsicm_enable(struct omap_dss_device *dssdev)
+static void dsicm_enable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
+	struct omap_dss_device *src = ddata->src;
 	int r;
 
-	dev_dbg(&ddata->pdev->dev, "enable\n");
-
 	mutex_lock(&ddata->lock);
 
-	if (!omapdss_device_is_connected(dssdev)) {
-		r = -ENODEV;
-		goto err;
-	}
-
-	if (omapdss_device_is_enabled(dssdev)) {
-		r = 0;
-		goto err;
-	}
-
-	in->ops.dsi->bus_lock(in);
+	src->ops->dsi.bus_lock(src);
 
 	r = dsicm_power_on(ddata);
 
-	in->ops.dsi->bus_unlock(in);
+	src->ops->dsi.bus_unlock(src);
 
 	if (r)
 		goto err;
 
-	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
-
 	mutex_unlock(&ddata->lock);
 
 	dsicm_bl_power(ddata, true);
 
-	return 0;
+	return;
 err:
-	dev_dbg(&ddata->pdev->dev, "enable failed\n");
+	dev_dbg(&ddata->pdev->dev, "enable failed (%d)\n", r);
 	mutex_unlock(&ddata->lock);
-	return r;
 }
 
 static void dsicm_disable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
+	struct omap_dss_device *src = ddata->src;
 	int r;
 
-	dev_dbg(&ddata->pdev->dev, "disable\n");
-
 	dsicm_bl_power(ddata, false);
 
 	mutex_lock(&ddata->lock);
 
 	dsicm_cancel_ulps_work(ddata);
 
-	in->ops.dsi->bus_lock(in);
+	src->ops->dsi.bus_lock(src);
 
-	if (omapdss_device_is_enabled(dssdev)) {
-		r = dsicm_wake_up(ddata);
-		if (!r)
-			dsicm_power_off(ddata);
-	}
-
-	in->ops.dsi->bus_unlock(in);
+	r = dsicm_wake_up(ddata);
+	if (!r)
+		dsicm_power_off(ddata);
 
-	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
+	src->ops->dsi.bus_unlock(src);
 
 	mutex_unlock(&ddata->lock);
 }
@@ -891,16 +836,16 @@
 static void dsicm_framedone_cb(int err, void *data)
 {
 	struct panel_drv_data *ddata = data;
-	struct omap_dss_device *in = ddata->in;
+	struct omap_dss_device *src = ddata->src;
 
 	dev_dbg(&ddata->pdev->dev, "framedone, err %d\n", err);
-	in->ops.dsi->bus_unlock(ddata->in);
+	src->ops->dsi.bus_unlock(src);
 }
 
 static irqreturn_t dsicm_te_isr(int irq, void *data)
 {
 	struct panel_drv_data *ddata = data;
-	struct omap_dss_device *in = ddata->in;
+	struct omap_dss_device *src = ddata->src;
 	int old;
 	int r;
 
@@ -909,7 +854,7 @@
 	if (old) {
 		cancel_delayed_work(&ddata->te_timeout_work);
 
-		r = in->ops.dsi->update(in, ddata->channel, dsicm_framedone_cb,
+		r = src->ops->dsi.update(src, ddata->channel, dsicm_framedone_cb,
 				ddata);
 		if (r)
 			goto err;
@@ -918,7 +863,7 @@
 	return IRQ_HANDLED;
 err:
 	dev_err(&ddata->pdev->dev, "start update failed\n");
-	in->ops.dsi->bus_unlock(in);
+	src->ops->dsi.bus_unlock(src);
 	return IRQ_HANDLED;
 }
 
@@ -926,25 +871,25 @@
 {
 	struct panel_drv_data *ddata = container_of(work, struct panel_drv_data,
 					te_timeout_work.work);
-	struct omap_dss_device *in = ddata->in;
+	struct omap_dss_device *src = ddata->src;
 
 	dev_err(&ddata->pdev->dev, "TE not received for 250ms!\n");
 
 	atomic_set(&ddata->do_update, 0);
-	in->ops.dsi->bus_unlock(in);
+	src->ops->dsi.bus_unlock(src);
 }
 
 static int dsicm_update(struct omap_dss_device *dssdev,
 				    u16 x, u16 y, u16 w, u16 h)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
+	struct omap_dss_device *src = ddata->src;
 	int r;
 
 	dev_dbg(&ddata->pdev->dev, "update %d, %d, %d x %d\n", x, y, w, h);
 
 	mutex_lock(&ddata->lock);
-	in->ops.dsi->bus_lock(in);
+	src->ops->dsi.bus_lock(src);
 
 	r = dsicm_wake_up(ddata);
 	if (r)
@@ -956,9 +901,8 @@
 	}
 
 	/* XXX no need to send this every frame, but dsi break if not done */
-	r = dsicm_set_update_window(ddata, 0, 0,
-			dssdev->panel.vm.hactive,
-			dssdev->panel.vm.vactive);
+	r = dsicm_set_update_window(ddata, 0, 0, ddata->vm.hactive,
+				    ddata->vm.vactive);
 	if (r)
 		goto err;
 
@@ -967,17 +911,17 @@
 				msecs_to_jiffies(250));
 		atomic_set(&ddata->do_update, 1);
 	} else {
-		r = in->ops.dsi->update(in, ddata->channel, dsicm_framedone_cb,
+		r = src->ops->dsi.update(src, ddata->channel, dsicm_framedone_cb,
 				ddata);
 		if (r)
 			goto err;
 	}
 
-	/* note: no bus_unlock here. unlock is in framedone_cb */
+	/* note: no bus_unlock here. unlock is src framedone_cb */
 	mutex_unlock(&ddata->lock);
 	return 0;
 err:
-	in->ops.dsi->bus_unlock(in);
+	src->ops->dsi.bus_unlock(src);
 	mutex_unlock(&ddata->lock);
 	return r;
 }
@@ -985,13 +929,13 @@
 static int dsicm_sync(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
+	struct omap_dss_device *src = ddata->src;
 
 	dev_dbg(&ddata->pdev->dev, "sync\n");
 
 	mutex_lock(&ddata->lock);
-	in->ops.dsi->bus_lock(in);
-	in->ops.dsi->bus_unlock(in);
+	src->ops->dsi.bus_lock(src);
+	src->ops->dsi.bus_unlock(src);
 	mutex_unlock(&ddata->lock);
 
 	dev_dbg(&ddata->pdev->dev, "sync done\n");
@@ -1001,7 +945,7 @@
 
 static int _dsicm_enable_te(struct panel_drv_data *ddata, bool enable)
 {
-	struct omap_dss_device *in = ddata->in;
+	struct omap_dss_device *src = ddata->src;
 	int r;
 
 	if (enable)
@@ -1010,7 +954,7 @@
 		r = dsicm_dcs_write_0(ddata, MIPI_DCS_SET_TEAR_OFF);
 
 	if (!ddata->ext_te_gpio)
-		in->ops.dsi->enable_te(in, enable);
+		src->ops->dsi.enable_te(src, enable);
 
 	/* possible panel bug */
 	msleep(100);
@@ -1021,7 +965,7 @@
 static int dsicm_enable_te(struct omap_dss_device *dssdev, bool enable)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
+	struct omap_dss_device *src = ddata->src;
 	int r;
 
 	mutex_lock(&ddata->lock);
@@ -1029,7 +973,7 @@
 	if (ddata->te_enabled == enable)
 		goto end;
 
-	in->ops.dsi->bus_lock(in);
+	src->ops->dsi.bus_lock(src);
 
 	if (ddata->enabled) {
 		r = dsicm_wake_up(ddata);
@@ -1043,13 +987,13 @@
 
 	ddata->te_enabled = enable;
 
-	in->ops.dsi->bus_unlock(in);
+	src->ops->dsi.bus_unlock(src);
 end:
 	mutex_unlock(&ddata->lock);
 
 	return 0;
 err:
-	in->ops.dsi->bus_unlock(in);
+	src->ops->dsi.bus_unlock(src);
 	mutex_unlock(&ddata->lock);
 
 	return r;
@@ -1072,7 +1016,7 @@
 		u16 x, u16 y, u16 w, u16 h)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
+	struct omap_dss_device *src = ddata->src;
 	int r;
 	int first = 1;
 	int plen;
@@ -1089,9 +1033,9 @@
 	}
 
 	size = min((u32)w * h * 3,
-		   dssdev->panel.vm.hactive * dssdev->panel.vm.vactive * 3);
+		   ddata->vm.hactive * ddata->vm.vactive * 3);
 
-	in->ops.dsi->bus_lock(in);
+	src->ops->dsi.bus_lock(src);
 
 	r = dsicm_wake_up(ddata);
 	if (r)
@@ -1107,7 +1051,7 @@
 
 	dsicm_set_update_window(ddata, x, y, w, h);
 
-	r = in->ops.dsi->set_max_rx_packet_size(in, ddata->channel, plen);
+	r = src->ops->dsi.set_max_rx_packet_size(src, ddata->channel, plen);
 	if (r)
 		goto err2;
 
@@ -1115,7 +1059,7 @@
 		u8 dcs_cmd = first ? 0x2e : 0x3e;
 		first = 0;
 
-		r = in->ops.dsi->dcs_read(in, ddata->channel, dcs_cmd,
+		r = src->ops->dsi.dcs_read(src, ddata->channel, dcs_cmd,
 				buf + buf_used, size - buf_used);
 
 		if (r < 0) {
@@ -1141,9 +1085,9 @@
 	r = buf_used;
 
 err3:
-	in->ops.dsi->set_max_rx_packet_size(in, ddata->channel, 1);
+	src->ops->dsi.set_max_rx_packet_size(src, ddata->channel, 1);
 err2:
-	in->ops.dsi->bus_unlock(in);
+	src->ops->dsi.bus_unlock(src);
 err1:
 	mutex_unlock(&ddata->lock);
 	return r;
@@ -1154,7 +1098,7 @@
 	struct panel_drv_data *ddata = container_of(work, struct panel_drv_data,
 			ulps_work.work);
 	struct omap_dss_device *dssdev = &ddata->dssdev;
-	struct omap_dss_device *in = ddata->in;
+	struct omap_dss_device *src = ddata->src;
 
 	mutex_lock(&ddata->lock);
 
@@ -1163,37 +1107,40 @@
 		return;
 	}
 
-	in->ops.dsi->bus_lock(in);
+	src->ops->dsi.bus_lock(src);
 
 	dsicm_enter_ulps(ddata);
 
-	in->ops.dsi->bus_unlock(in);
+	src->ops->dsi.bus_unlock(src);
 	mutex_unlock(&ddata->lock);
 }
 
-static void dsicm_get_timings(struct omap_dss_device *dssdev,
-			      struct videomode *vm)
+static int dsicm_get_modes(struct omap_dss_device *dssdev,
+			   struct drm_connector *connector)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
-	*vm = ddata->vm;
+	connector->display_info.width_mm = ddata->width_mm;
+	connector->display_info.height_mm = ddata->height_mm;
+
+	return omapdss_display_get_modes(connector, &ddata->vm);
 }
 
 static int dsicm_check_timings(struct omap_dss_device *dssdev,
-			       struct videomode *vm)
+			       struct drm_display_mode *mode)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 	int ret = 0;
 
-	if (vm->hactive != ddata->vm.hactive)
+	if (mode->hdisplay != ddata->vm.hactive)
 		ret = -EINVAL;
 
-	if (vm->vactive != ddata->vm.vactive)
+	if (mode->vdisplay != ddata->vm.vactive)
 		ret = -EINVAL;
 
 	if (ret) {
 		dev_warn(dssdev->dev, "wrong resolution: %d x %d",
-			 vm->hactive, vm->vactive);
+			 mode->hdisplay, mode->vdisplay);
 		dev_warn(dssdev->dev, "panel resolution: %d x %d",
 			 ddata->vm.hactive, ddata->vm.vactive);
 	}
@@ -1201,29 +1148,21 @@
 	return ret;
 }
 
-static void dsicm_get_size(struct omap_dss_device *dssdev,
-			  unsigned int *width, unsigned int *height)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-
-	*width = ddata->width_mm;
-	*height = ddata->height_mm;
-}
-
-static struct omap_dss_driver dsicm_ops = {
+static const struct omap_dss_device_ops dsicm_ops = {
 	.connect	= dsicm_connect,
 	.disconnect	= dsicm_disconnect,
 
 	.enable		= dsicm_enable,
 	.disable	= dsicm_disable,
 
+	.get_modes	= dsicm_get_modes,
+	.check_timings	= dsicm_check_timings,
+};
+
+static const struct omap_dss_driver dsicm_dss_driver = {
 	.update		= dsicm_update,
 	.sync		= dsicm_sync,
 
-	.get_timings	= dsicm_get_timings,
-	.check_timings	= dsicm_check_timings,
-	.get_size	= dsicm_get_size,
-
 	.enable_te	= dsicm_enable_te,
 	.get_te		= dsicm_get_te,
 
@@ -1330,20 +1269,19 @@
 
 	dssdev = &ddata->dssdev;
 	dssdev->dev = dev;
-	dssdev->driver = &dsicm_ops;
-	dssdev->panel.vm = ddata->vm;
+	dssdev->ops = &dsicm_ops;
+	dssdev->driver = &dsicm_dss_driver;
 	dssdev->type = OMAP_DISPLAY_TYPE_DSI;
+	dssdev->display = true;
 	dssdev->owner = THIS_MODULE;
+	dssdev->of_ports = BIT(0);
+	dssdev->ops_flags = OMAP_DSS_DEVICE_OP_MODES;
 
-	dssdev->panel.dsi_pix_fmt = OMAP_DSS_DSI_FMT_RGB888;
 	dssdev->caps = OMAP_DSS_DISPLAY_CAP_MANUAL_UPDATE |
 		OMAP_DSS_DISPLAY_CAP_TEAR_ELIM;
 
-	r = omapdss_register_display(dssdev);
-	if (r) {
-		dev_err(dev, "Failed to register panel\n");
-		goto err_reg;
-	}
+	omapdss_display_init(dssdev);
+	omapdss_device_register(dssdev);
 
 	mutex_init(&ddata->lock);
 
@@ -1414,10 +1352,11 @@
 
 	dev_dbg(&pdev->dev, "remove\n");
 
-	omapdss_unregister_display(dssdev);
+	omapdss_device_unregister(dssdev);
 
-	dsicm_disable(dssdev);
-	dsicm_disconnect(dssdev);
+	if (omapdss_device_is_enabled(dssdev))
+		dsicm_disable(dssdev);
+	omapdss_device_disconnect(ddata->src, dssdev);
 
 	sysfs_remove_group(&pdev->dev.kobj, &dsicm_attr_group);
 
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c b/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c
--- a/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c	2020-06-19 21:00:31.412227715 -0400
@@ -33,19 +33,11 @@
 	.vfront_porch	= 4,
 	.vback_porch	= 18,
 
-	.flags		= DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW |
-			  DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_SYNC_NEGEDGE |
-			  DISPLAY_FLAGS_PIXDATA_POSEDGE,
-	/*
-	 * Note: According to the panel documentation:
-	 * DE is active LOW
-	 * DATA needs to be driven on the FALLING edge
-	 */
+	.flags		= DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW,
 };
 
 struct panel_drv_data {
 	struct omap_dss_device dssdev;
-	struct omap_dss_device *in;
 
 	struct spi_device *spi;
 
@@ -116,128 +108,53 @@
 	lb035q02_write_reg(spi, 0x3b, 0x0806);
 }
 
-static int lb035q02_connect(struct omap_dss_device *dssdev)
+static int lb035q02_connect(struct omap_dss_device *src,
+			    struct omap_dss_device *dst)
 {
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in;
-	int r;
-
-	if (omapdss_device_is_connected(dssdev))
-		return 0;
-
-	in = omapdss_of_find_source_for_first_ep(dssdev->dev->of_node);
-	if (IS_ERR(in)) {
-		dev_err(dssdev->dev, "failed to find video source\n");
-		return PTR_ERR(in);
-	}
-
-	r = in->ops.dpi->connect(in, dssdev);
-	if (r) {
-		omap_dss_put_device(in);
-		return r;
-	}
+	struct panel_drv_data *ddata = to_panel_data(dst);
 
 	init_lb035q02_panel(ddata->spi);
 
-	ddata->in = in;
 	return 0;
 }
 
-static void lb035q02_disconnect(struct omap_dss_device *dssdev)
+static void lb035q02_disconnect(struct omap_dss_device *src,
+				struct omap_dss_device *dst)
 {
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	if (!omapdss_device_is_connected(dssdev))
-		return;
-
-	in->ops.dpi->disconnect(in, dssdev);
-
-	omap_dss_put_device(in);
-	ddata->in = NULL;
 }
 
-static int lb035q02_enable(struct omap_dss_device *dssdev)
+static void lb035q02_enable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-	int r;
-
-	if (!omapdss_device_is_connected(dssdev))
-		return -ENODEV;
-
-	if (omapdss_device_is_enabled(dssdev))
-		return 0;
-
-	in->ops.dpi->set_timings(in, &ddata->vm);
-
-	r = in->ops.dpi->enable(in);
-	if (r)
-		return r;
 
 	if (ddata->enable_gpio)
 		gpiod_set_value_cansleep(ddata->enable_gpio, 1);
-
-	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
-
-	return 0;
 }
 
 static void lb035q02_disable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	if (!omapdss_device_is_enabled(dssdev))
-		return;
 
 	if (ddata->enable_gpio)
 		gpiod_set_value_cansleep(ddata->enable_gpio, 0);
-
-	in->ops.dpi->disable(in);
-
-	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
-}
-
-static void lb035q02_set_timings(struct omap_dss_device *dssdev,
-				 struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	ddata->vm = *vm;
-	dssdev->panel.vm = *vm;
-
-	in->ops.dpi->set_timings(in, vm);
-}
-
-static void lb035q02_get_timings(struct omap_dss_device *dssdev,
-				 struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-
-	*vm = ddata->vm;
 }
 
-static int lb035q02_check_timings(struct omap_dss_device *dssdev,
-				  struct videomode *vm)
+static int lb035q02_get_modes(struct omap_dss_device *dssdev,
+			      struct drm_connector *connector)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
 
-	return in->ops.dpi->check_timings(in, vm);
+	return omapdss_display_get_modes(connector, &ddata->vm);
 }
 
-static struct omap_dss_driver lb035q02_ops = {
+static const struct omap_dss_device_ops lb035q02_ops = {
 	.connect	= lb035q02_connect,
 	.disconnect	= lb035q02_disconnect,
 
 	.enable		= lb035q02_enable,
 	.disable	= lb035q02_disable,
 
-	.set_timings	= lb035q02_set_timings,
-	.get_timings	= lb035q02_get_timings,
-	.check_timings	= lb035q02_check_timings,
+	.get_modes	= lb035q02_get_modes,
 };
 
 static int lb035q02_probe_of(struct spi_device *spi)
@@ -278,16 +195,24 @@
 
 	dssdev = &ddata->dssdev;
 	dssdev->dev = &spi->dev;
-	dssdev->driver = &lb035q02_ops;
+	dssdev->ops = &lb035q02_ops;
 	dssdev->type = OMAP_DISPLAY_TYPE_DPI;
+	dssdev->display = true;
 	dssdev->owner = THIS_MODULE;
-	dssdev->panel.vm = ddata->vm;
+	dssdev->of_ports = BIT(0);
+	dssdev->ops_flags = OMAP_DSS_DEVICE_OP_MODES;
 
-	r = omapdss_register_display(dssdev);
-	if (r) {
-		dev_err(&spi->dev, "Failed to register panel\n");
-		return r;
-	}
+	/*
+	 * Note: According to the panel documentation:
+	 * DE is active LOW
+	 * DATA needs to be driven on the FALLING edge
+	 */
+	dssdev->bus_flags = DRM_BUS_FLAG_DE_HIGH
+			  | DRM_BUS_FLAG_SYNC_DRIVE_NEGEDGE
+			  | DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE;
+
+	omapdss_display_init(dssdev);
+	omapdss_device_register(dssdev);
 
 	return 0;
 }
@@ -297,10 +222,9 @@
 	struct panel_drv_data *ddata = dev_get_drvdata(&spi->dev);
 	struct omap_dss_device *dssdev = &ddata->dssdev;
 
-	omapdss_unregister_display(dssdev);
+	omapdss_device_unregister(dssdev);
 
 	lb035q02_disable(dssdev);
-	lb035q02_disconnect(dssdev);
 
 	return 0;
 }
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c b/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c
--- a/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c	2020-06-19 21:00:31.412227715 -0400
@@ -11,22 +11,19 @@
  * (at your option) any later version.
  */
 
-#include <linux/module.h>
 #include <linux/delay.h>
-#include <linux/spi/spi.h>
 #include <linux/gpio/consumer.h>
-#include <linux/of_gpio.h>
+#include <linux/module.h>
+#include <linux/spi/spi.h>
 
 #include "../dss/omapdss.h"
 
 struct panel_drv_data {
 	struct omap_dss_device	dssdev;
-	struct omap_dss_device *in;
 
 	struct videomode vm;
 
-	int res_gpio;
-	int qvga_gpio;
+	struct gpio_desc *res_gpio;
 
 	struct spi_device *spi;
 };
@@ -74,9 +71,7 @@
 	.vsync_len	= 1,
 	.vback_porch	= 4,
 
-	.flags		= DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW |
-			  DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_SYNC_POSEDGE |
-			  DISPLAY_FLAGS_PIXDATA_POSEDGE,
+	.flags		= DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW,
 };
 
 #define to_panel_data(p) container_of(p, struct panel_drv_data, dssdev)
@@ -112,151 +107,54 @@
 	return 0;
 }
 
-static int nec_8048_connect(struct omap_dss_device *dssdev)
+static int nec_8048_connect(struct omap_dss_device *src,
+			    struct omap_dss_device *dst)
 {
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in;
-	int r;
-
-	if (omapdss_device_is_connected(dssdev))
-		return 0;
-
-	in = omapdss_of_find_source_for_first_ep(dssdev->dev->of_node);
-	if (IS_ERR(in)) {
-		dev_err(dssdev->dev, "failed to find video source\n");
-		return PTR_ERR(in);
-	}
-
-	r = in->ops.dpi->connect(in, dssdev);
-	if (r) {
-		omap_dss_put_device(in);
-		return r;
-	}
-
-	ddata->in = in;
 	return 0;
 }
 
-static void nec_8048_disconnect(struct omap_dss_device *dssdev)
+static void nec_8048_disconnect(struct omap_dss_device *src,
+				struct omap_dss_device *dst)
 {
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	if (!omapdss_device_is_connected(dssdev))
-		return;
-
-	in->ops.dpi->disconnect(in, dssdev);
-
-	omap_dss_put_device(in);
-	ddata->in = NULL;
 }
 
-static int nec_8048_enable(struct omap_dss_device *dssdev)
+static void nec_8048_enable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-	int r;
-
-	if (!omapdss_device_is_connected(dssdev))
-		return -ENODEV;
-
-	if (omapdss_device_is_enabled(dssdev))
-		return 0;
-
-	in->ops.dpi->set_timings(in, &ddata->vm);
 
-	r = in->ops.dpi->enable(in);
-	if (r)
-		return r;
-
-	if (gpio_is_valid(ddata->res_gpio))
-		gpio_set_value_cansleep(ddata->res_gpio, 1);
-
-	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
-
-	return 0;
+	gpiod_set_value_cansleep(ddata->res_gpio, 1);
 }
 
 static void nec_8048_disable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	if (!omapdss_device_is_enabled(dssdev))
-		return;
-
-	if (gpio_is_valid(ddata->res_gpio))
-		gpio_set_value_cansleep(ddata->res_gpio, 0);
-
-	in->ops.dpi->disable(in);
-
-	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
-}
-
-static void nec_8048_set_timings(struct omap_dss_device *dssdev,
-				 struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	ddata->vm = *vm;
-	dssdev->panel.vm = *vm;
-
-	in->ops.dpi->set_timings(in, vm);
-}
-
-static void nec_8048_get_timings(struct omap_dss_device *dssdev,
-				 struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
-	*vm = ddata->vm;
+	gpiod_set_value_cansleep(ddata->res_gpio, 0);
 }
 
-static int nec_8048_check_timings(struct omap_dss_device *dssdev,
-				  struct videomode *vm)
+static int nec_8048_get_modes(struct omap_dss_device *dssdev,
+			      struct drm_connector *connector)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
 
-	return in->ops.dpi->check_timings(in, vm);
+	return omapdss_display_get_modes(connector, &ddata->vm);
 }
 
-static struct omap_dss_driver nec_8048_ops = {
+static const struct omap_dss_device_ops nec_8048_ops = {
 	.connect	= nec_8048_connect,
 	.disconnect	= nec_8048_disconnect,
 
 	.enable		= nec_8048_enable,
 	.disable	= nec_8048_disable,
 
-	.set_timings	= nec_8048_set_timings,
-	.get_timings	= nec_8048_get_timings,
-	.check_timings	= nec_8048_check_timings,
+	.get_modes	= nec_8048_get_modes,
 };
 
-static int nec_8048_probe_of(struct spi_device *spi)
-{
-	struct device_node *node = spi->dev.of_node;
-	struct panel_drv_data *ddata = dev_get_drvdata(&spi->dev);
-	int gpio;
-
-	gpio = of_get_named_gpio(node, "reset-gpios", 0);
-	if (!gpio_is_valid(gpio)) {
-		dev_err(&spi->dev, "failed to parse enable gpio\n");
-		return gpio;
-	}
-	ddata->res_gpio = gpio;
-
-	/* XXX the panel spec doesn't mention any QVGA pin?? */
-	ddata->qvga_gpio = -ENOENT;
-
-	return 0;
-}
-
 static int nec_8048_probe(struct spi_device *spi)
 {
 	struct panel_drv_data *ddata;
 	struct omap_dss_device *dssdev;
+	struct gpio_desc *gpio;
 	int r;
 
 	dev_dbg(&spi->dev, "%s\n", __func__);
@@ -280,38 +178,30 @@
 
 	ddata->spi = spi;
 
-	r = nec_8048_probe_of(spi);
-	if (r)
-		return r;
-
-	if (gpio_is_valid(ddata->qvga_gpio)) {
-		r = devm_gpio_request_one(&spi->dev, ddata->qvga_gpio,
-				GPIOF_OUT_INIT_HIGH, "lcd QVGA");
-		if (r)
-			return r;
+	gpio = devm_gpiod_get(&spi->dev, "reset", GPIOD_OUT_LOW);
+	if (IS_ERR(gpio)) {
+		dev_err(&spi->dev, "failed to get reset gpio\n");
+		return PTR_ERR(gpio);
 	}
 
-	if (gpio_is_valid(ddata->res_gpio)) {
-		r = devm_gpio_request_one(&spi->dev, ddata->res_gpio,
-				GPIOF_OUT_INIT_LOW, "lcd RES");
-		if (r)
-			return r;
-	}
+	ddata->res_gpio = gpio;
 
 	ddata->vm = nec_8048_panel_vm;
 
 	dssdev = &ddata->dssdev;
 	dssdev->dev = &spi->dev;
-	dssdev->driver = &nec_8048_ops;
+	dssdev->ops = &nec_8048_ops;
 	dssdev->type = OMAP_DISPLAY_TYPE_DPI;
+	dssdev->display = true;
 	dssdev->owner = THIS_MODULE;
-	dssdev->panel.vm = ddata->vm;
+	dssdev->of_ports = BIT(0);
+	dssdev->ops_flags = OMAP_DSS_DEVICE_OP_MODES;
+	dssdev->bus_flags = DRM_BUS_FLAG_DE_HIGH
+			  | DRM_BUS_FLAG_SYNC_DRIVE_POSEDGE
+			  | DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE;
 
-	r = omapdss_register_display(dssdev);
-	if (r) {
-		dev_err(&spi->dev, "Failed to register panel\n");
-		return r;
-	}
+	omapdss_display_init(dssdev);
+	omapdss_device_register(dssdev);
 
 	return 0;
 }
@@ -323,10 +213,9 @@
 
 	dev_dbg(&ddata->spi->dev, "%s\n", __func__);
 
-	omapdss_unregister_display(dssdev);
+	omapdss_device_unregister(dssdev);
 
 	nec_8048_disable(dssdev);
-	nec_8048_disconnect(dssdev);
 
 	return 0;
 }
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/displays/panel-sharp-ls037v7dw01.c b/drivers/gpu/drm/omapdrm/displays/panel-sharp-ls037v7dw01.c
--- a/drivers/gpu/drm/omapdrm/displays/panel-sharp-ls037v7dw01.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/displays/panel-sharp-ls037v7dw01.c	2020-06-19 21:00:31.412227715 -0400
@@ -21,7 +21,6 @@
 
 struct panel_drv_data {
 	struct omap_dss_device dssdev;
-	struct omap_dss_device *in;
 	struct regulator *vcc;
 
 	struct videomode vm;
@@ -47,81 +46,38 @@
 	.vfront_porch	= 1,
 	.vback_porch	= 1,
 
-	.flags		= DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW |
-			  DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_SYNC_NEGEDGE |
-			  DISPLAY_FLAGS_PIXDATA_POSEDGE,
-	/*
-	 * Note: According to the panel documentation:
-	 * DATA needs to be driven on the FALLING edge
-	 */
+	.flags		= DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW,
 };
 
 #define to_panel_data(p) container_of(p, struct panel_drv_data, dssdev)
 
-static int sharp_ls_connect(struct omap_dss_device *dssdev)
+static int sharp_ls_connect(struct omap_dss_device *src,
+			    struct omap_dss_device *dst)
 {
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in;
-	int r;
-
-	if (omapdss_device_is_connected(dssdev))
-		return 0;
-
-	in = omapdss_of_find_source_for_first_ep(dssdev->dev->of_node);
-	if (IS_ERR(in)) {
-		dev_err(dssdev->dev, "failed to find video source\n");
-		return PTR_ERR(in);
-	}
-
-	r = in->ops.dpi->connect(in, dssdev);
-	if (r) {
-		omap_dss_put_device(in);
-		return r;
-	}
-
-	ddata->in = in;
 	return 0;
 }
 
-static void sharp_ls_disconnect(struct omap_dss_device *dssdev)
+static void sharp_ls_disconnect(struct omap_dss_device *src,
+				struct omap_dss_device *dst)
 {
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	if (!omapdss_device_is_connected(dssdev))
-		return;
-
-	in->ops.dpi->disconnect(in, dssdev);
-
-	omap_dss_put_device(in);
-	ddata->in = NULL;
 }
 
-static int sharp_ls_enable(struct omap_dss_device *dssdev)
+static void sharp_ls_pre_enable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
 	int r;
 
-	if (!omapdss_device_is_connected(dssdev))
-		return -ENODEV;
-
-	if (omapdss_device_is_enabled(dssdev))
-		return 0;
-
-	in->ops.dpi->set_timings(in, &ddata->vm);
-
 	if (ddata->vcc) {
 		r = regulator_enable(ddata->vcc);
-		if (r != 0)
-			return r;
+		if (r)
+			dev_err(dssdev->dev, "%s: failed to enable regulator\n",
+				__func__);
 	}
+}
 
-	r = in->ops.dpi->enable(in);
-	if (r) {
-		regulator_disable(ddata->vcc);
-		return r;
-	}
+static void sharp_ls_enable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
 	/* wait couple of vsyncs until enabling the LCD */
 	msleep(50);
@@ -131,19 +87,11 @@
 
 	if (ddata->ini_gpio)
 		gpiod_set_value_cansleep(ddata->ini_gpio, 1);
-
-	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
-
-	return 0;
 }
 
 static void sharp_ls_disable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	if (!omapdss_device_is_enabled(dssdev))
-		return;
 
 	if (ddata->ini_gpio)
 		gpiod_set_value_cansleep(ddata->ini_gpio, 0);
@@ -152,56 +100,35 @@
 		gpiod_set_value_cansleep(ddata->resb_gpio, 0);
 
 	/* wait at least 5 vsyncs after disabling the LCD */
-
 	msleep(100);
-
-	in->ops.dpi->disable(in);
-
-	if (ddata->vcc)
-		regulator_disable(ddata->vcc);
-
-	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
-}
-
-static void sharp_ls_set_timings(struct omap_dss_device *dssdev,
-				 struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	ddata->vm = *vm;
-	dssdev->panel.vm = *vm;
-
-	in->ops.dpi->set_timings(in, vm);
 }
 
-static void sharp_ls_get_timings(struct omap_dss_device *dssdev,
-				 struct videomode *vm)
+static void sharp_ls_post_disable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
-	*vm = ddata->vm;
+	if (ddata->vcc)
+		regulator_disable(ddata->vcc);
 }
 
-static int sharp_ls_check_timings(struct omap_dss_device *dssdev,
-				  struct videomode *vm)
+static int sharp_ls_get_modes(struct omap_dss_device *dssdev,
+			      struct drm_connector *connector)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
 
-	return in->ops.dpi->check_timings(in, vm);
+	return omapdss_display_get_modes(connector, &ddata->vm);
 }
 
-static struct omap_dss_driver sharp_ls_ops = {
+static const struct omap_dss_device_ops sharp_ls_ops = {
 	.connect	= sharp_ls_connect,
 	.disconnect	= sharp_ls_disconnect,
 
+	.pre_enable	= sharp_ls_pre_enable,
 	.enable		= sharp_ls_enable,
 	.disable	= sharp_ls_disable,
+	.post_disable	= sharp_ls_post_disable,
 
-	.set_timings	= sharp_ls_set_timings,
-	.get_timings	= sharp_ls_get_timings,
-	.check_timings	= sharp_ls_check_timings,
+	.get_modes	= sharp_ls_get_modes,
 };
 
 static  int sharp_ls_get_gpio_of(struct device *dev, int index, int val,
@@ -278,16 +205,23 @@
 
 	dssdev = &ddata->dssdev;
 	dssdev->dev = &pdev->dev;
-	dssdev->driver = &sharp_ls_ops;
+	dssdev->ops = &sharp_ls_ops;
 	dssdev->type = OMAP_DISPLAY_TYPE_DPI;
+	dssdev->display = true;
 	dssdev->owner = THIS_MODULE;
-	dssdev->panel.vm = ddata->vm;
+	dssdev->of_ports = BIT(0);
+	dssdev->ops_flags = OMAP_DSS_DEVICE_OP_MODES;
 
-	r = omapdss_register_display(dssdev);
-	if (r) {
-		dev_err(&pdev->dev, "Failed to register panel\n");
-		return r;
-	}
+	/*
+	 * Note: According to the panel documentation:
+	 * DATA needs to be driven on the FALLING edge
+	 */
+	dssdev->bus_flags = DRM_BUS_FLAG_DE_HIGH
+			  | DRM_BUS_FLAG_SYNC_DRIVE_NEGEDGE
+			  | DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE;
+
+	omapdss_display_init(dssdev);
+	omapdss_device_register(dssdev);
 
 	return 0;
 }
@@ -297,10 +231,12 @@
 	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
 	struct omap_dss_device *dssdev = &ddata->dssdev;
 
-	omapdss_unregister_display(dssdev);
+	omapdss_device_unregister(dssdev);
 
-	sharp_ls_disable(dssdev);
-	sharp_ls_disconnect(dssdev);
+	if (omapdss_device_is_enabled(dssdev)) {
+		sharp_ls_disable(dssdev);
+		sharp_ls_post_disable(dssdev);
+	}
 
 	return 0;
 }
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c b/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c
--- a/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c	2020-06-19 21:00:31.412227715 -0400
@@ -20,17 +20,15 @@
  * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <linux/backlight.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/jiffies.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
-#include <linux/delay.h>
-#include <linux/spi/spi.h>
-#include <linux/jiffies.h>
 #include <linux/sched.h>
-#include <linux/backlight.h>
-#include <linux/gpio/consumer.h>
-#include <linux/of.h>
-#include <linux/of_gpio.h>
+#include <linux/spi/spi.h>
 
 #include "../dss/omapdss.h"
 
@@ -64,9 +62,8 @@
 
 struct panel_drv_data {
 	struct omap_dss_device	dssdev;
-	struct omap_dss_device *in;
 
-	int reset_gpio;
+	struct gpio_desc *reset_gpio;
 
 	struct videomode vm;
 
@@ -100,9 +97,7 @@
 	.vsync_len	= 3,
 	.vback_porch	= 4,
 
-	.flags		= DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW |
-			  DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_SYNC_NEGEDGE |
-			  DISPLAY_FLAGS_PIXDATA_POSEDGE,
+	.flags		= DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW,
 };
 
 #define to_panel_data(p) container_of(p, struct panel_drv_data, dssdev)
@@ -507,66 +502,28 @@
 	.attrs = bldev_attrs,
 };
 
-static int acx565akm_connect(struct omap_dss_device *dssdev)
+static int acx565akm_connect(struct omap_dss_device *src,
+			     struct omap_dss_device *dst)
 {
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in;
-	int r;
-
-	if (omapdss_device_is_connected(dssdev))
-		return 0;
-
-	in = omapdss_of_find_source_for_first_ep(dssdev->dev->of_node);
-	if (IS_ERR(in)) {
-		dev_err(dssdev->dev, "failed to find video source\n");
-		return PTR_ERR(in);
-	}
-
-	r = in->ops.sdi->connect(in, dssdev);
-	if (r) {
-		omap_dss_put_device(in);
-		return r;
-	}
-
-	ddata->in = in;
 	return 0;
 }
 
-static void acx565akm_disconnect(struct omap_dss_device *dssdev)
+static void acx565akm_disconnect(struct omap_dss_device *src,
+				 struct omap_dss_device *dst)
 {
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	if (!omapdss_device_is_connected(dssdev))
-		return;
-
-	in->ops.sdi->disconnect(in, dssdev);
-
-	omap_dss_put_device(in);
-	ddata->in = NULL;
 }
 
 static int acx565akm_panel_power_on(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-	int r;
 
 	dev_dbg(&ddata->spi->dev, "%s\n", __func__);
 
-	in->ops.sdi->set_timings(in, &ddata->vm);
-
-	r = in->ops.sdi->enable(in);
-	if (r) {
-		pr_err("%s sdi enable failed\n", __func__);
-		return r;
-	}
-
 	/*FIXME tweak me */
 	msleep(50);
 
-	if (gpio_is_valid(ddata->reset_gpio))
-		gpio_set_value(ddata->reset_gpio, 1);
+	if (ddata->reset_gpio)
+		gpiod_set_value(ddata->reset_gpio, 1);
 
 	if (ddata->enabled) {
 		dev_dbg(&ddata->spi->dev, "panel already enabled\n");
@@ -597,7 +554,6 @@
 static void acx565akm_panel_power_off(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
 
 	dev_dbg(dssdev->dev, "%s\n", __func__);
 
@@ -615,106 +571,49 @@
 	 */
 	msleep(50);
 
-	if (gpio_is_valid(ddata->reset_gpio))
-		gpio_set_value(ddata->reset_gpio, 0);
+	if (ddata->reset_gpio)
+		gpiod_set_value(ddata->reset_gpio, 0);
 
 	/* FIXME need to tweak this delay */
 	msleep(100);
-
-	in->ops.sdi->disable(in);
 }
 
-static int acx565akm_enable(struct omap_dss_device *dssdev)
+static void acx565akm_enable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	int r;
-
-	dev_dbg(dssdev->dev, "%s\n", __func__);
-
-	if (!omapdss_device_is_connected(dssdev))
-		return -ENODEV;
-
-	if (omapdss_device_is_enabled(dssdev))
-		return 0;
 
 	mutex_lock(&ddata->mutex);
-	r = acx565akm_panel_power_on(dssdev);
+	acx565akm_panel_power_on(dssdev);
 	mutex_unlock(&ddata->mutex);
-	if (r)
-		return r;
-
-	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
-
-	return 0;
 }
 
 static void acx565akm_disable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
-	dev_dbg(dssdev->dev, "%s\n", __func__);
-
-	if (!omapdss_device_is_enabled(dssdev))
-		return;
-
 	mutex_lock(&ddata->mutex);
 	acx565akm_panel_power_off(dssdev);
 	mutex_unlock(&ddata->mutex);
-
-	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
-}
-
-static void acx565akm_set_timings(struct omap_dss_device *dssdev,
-				  struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	ddata->vm = *vm;
-	dssdev->panel.vm = *vm;
-
-	in->ops.sdi->set_timings(in, vm);
-}
-
-static void acx565akm_get_timings(struct omap_dss_device *dssdev,
-				  struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-
-	*vm = ddata->vm;
 }
 
-static int acx565akm_check_timings(struct omap_dss_device *dssdev,
-				   struct videomode *vm)
+static int acx565akm_get_modes(struct omap_dss_device *dssdev,
+			       struct drm_connector *connector)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
 
-	return in->ops.sdi->check_timings(in, vm);
+	return omapdss_display_get_modes(connector, &ddata->vm);
 }
 
-static struct omap_dss_driver acx565akm_ops = {
+static const struct omap_dss_device_ops acx565akm_ops = {
 	.connect	= acx565akm_connect,
 	.disconnect	= acx565akm_disconnect,
 
 	.enable		= acx565akm_enable,
 	.disable	= acx565akm_disable,
 
-	.set_timings	= acx565akm_set_timings,
-	.get_timings	= acx565akm_get_timings,
-	.check_timings	= acx565akm_check_timings,
+	.get_modes	= acx565akm_get_modes,
 };
 
-static int acx565akm_probe_of(struct spi_device *spi)
-{
-	struct panel_drv_data *ddata = dev_get_drvdata(&spi->dev);
-	struct device_node *np = spi->dev.of_node;
-
-	ddata->reset_gpio = of_get_named_gpio(np, "reset-gpios", 0);
-
-	return 0;
-}
-
 static int acx565akm_probe(struct spi_device *spi)
 {
 	struct panel_drv_data *ddata;
@@ -722,6 +621,7 @@
 	struct backlight_device *bldev;
 	int max_brightness, brightness;
 	struct backlight_properties props;
+	struct gpio_desc *gpio;
 	int r;
 
 	dev_dbg(&spi->dev, "%s\n", __func__);
@@ -738,19 +638,16 @@
 
 	mutex_init(&ddata->mutex);
 
-	r = acx565akm_probe_of(spi);
-	if (r)
-		return r;
-
-	if (gpio_is_valid(ddata->reset_gpio)) {
-		r = devm_gpio_request_one(&spi->dev, ddata->reset_gpio,
-				GPIOF_OUT_INIT_LOW, "lcd reset");
-		if (r)
-			goto err_gpio;
+	gpio = devm_gpiod_get_optional(&spi->dev, "reset", GPIOD_OUT_LOW);
+	if (IS_ERR(gpio)) {
+		dev_err(&spi->dev, "failed to parse reset gpio\n");
+		return PTR_ERR(gpio);
 	}
 
-	if (gpio_is_valid(ddata->reset_gpio))
-		gpio_set_value(ddata->reset_gpio, 1);
+	ddata->reset_gpio = gpio;
+
+	if (ddata->reset_gpio)
+		gpiod_set_value(ddata->reset_gpio, 1);
 
 	/*
 	 * After reset we have to wait 5 msec before the first
@@ -762,12 +659,12 @@
 
 	r = panel_detect(ddata);
 
-	if (!ddata->enabled && gpio_is_valid(ddata->reset_gpio))
-		gpio_set_value(ddata->reset_gpio, 0);
+	if (!ddata->enabled && ddata->reset_gpio)
+		gpiod_set_value(ddata->reset_gpio, 0);
 
 	if (r) {
 		dev_err(&spi->dev, "%s panel detect error\n", __func__);
-		goto err_detect;
+		return r;
 	}
 
 	memset(&props, 0, sizeof(props));
@@ -777,17 +674,15 @@
 
 	bldev = backlight_device_register("acx565akm", &ddata->spi->dev,
 			ddata, &acx565akm_bl_ops, &props);
-	if (IS_ERR(bldev)) {
-		r = PTR_ERR(bldev);
-		goto err_reg_bl;
-	}
+	if (IS_ERR(bldev))
+		return PTR_ERR(bldev);
 	ddata->bl_dev = bldev;
 	if (ddata->has_cabc) {
 		r = sysfs_create_group(&bldev->dev.kobj, &bldev_attr_group);
 		if (r) {
 			dev_err(&bldev->dev,
 				"%s failed to create sysfs files\n", __func__);
-			goto err_sysfs;
+			goto err_backlight_unregister;
 		}
 		ddata->cabc_mode = get_hw_cabc_mode(ddata);
 	}
@@ -809,26 +704,23 @@
 
 	dssdev = &ddata->dssdev;
 	dssdev->dev = &spi->dev;
-	dssdev->driver = &acx565akm_ops;
+	dssdev->ops = &acx565akm_ops;
 	dssdev->type = OMAP_DISPLAY_TYPE_SDI;
+	dssdev->display = true;
 	dssdev->owner = THIS_MODULE;
-	dssdev->panel.vm = ddata->vm;
+	dssdev->of_ports = BIT(0);
+	dssdev->ops_flags = OMAP_DSS_DEVICE_OP_MODES;
+	dssdev->bus_flags = DRM_BUS_FLAG_DE_HIGH
+			  | DRM_BUS_FLAG_SYNC_DRIVE_NEGEDGE
+			  | DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE;
 
-	r = omapdss_register_display(dssdev);
-	if (r) {
-		dev_err(&spi->dev, "Failed to register panel\n");
-		goto err_reg;
-	}
+	omapdss_display_init(dssdev);
+	omapdss_device_register(dssdev);
 
 	return 0;
 
-err_reg:
-	sysfs_remove_group(&bldev->dev.kobj, &bldev_attr_group);
-err_sysfs:
+err_backlight_unregister:
 	backlight_device_unregister(bldev);
-err_reg_bl:
-err_detect:
-err_gpio:
 	return r;
 }
 
@@ -842,10 +734,10 @@
 	sysfs_remove_group(&ddata->bl_dev->dev.kobj, &bldev_attr_group);
 	backlight_device_unregister(ddata->bl_dev);
 
-	omapdss_unregister_display(dssdev);
+	omapdss_device_unregister(dssdev);
 
-	acx565akm_disable(dssdev);
-	acx565akm_disconnect(dssdev);
+	if (omapdss_device_is_enabled(dssdev))
+		acx565akm_disable(dssdev);
 
 	return 0;
 }
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c
--- a/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c	2020-06-19 21:00:31.412227715 -0400
@@ -27,13 +27,11 @@
 #include <linux/module.h>
 #include <linux/delay.h>
 #include <linux/spi/spi.h>
-#include <linux/gpio.h>
 
 #include "../dss/omapdss.h"
 
 struct panel_drv_data {
 	struct omap_dss_device dssdev;
-	struct omap_dss_device *in;
 
 	struct videomode vm;
 
@@ -51,13 +49,7 @@
 	.vsync_len	= 2,
 	.vback_porch	= 2,
 
-	.flags		= DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW |
-			  DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_SYNC_POSEDGE |
-			  DISPLAY_FLAGS_PIXDATA_NEGEDGE,
-	/*
-	 * Note: According to the panel documentation:
-	 * SYNC needs to be driven on the FALLING edge
-	 */
+	.flags		= DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW,
 };
 
 #define JBT_COMMAND	0x000
@@ -166,65 +158,23 @@
 
 #define to_panel_data(p) container_of(p, struct panel_drv_data, dssdev)
 
-static int td028ttec1_panel_connect(struct omap_dss_device *dssdev)
+static int td028ttec1_panel_connect(struct omap_dss_device *src,
+				    struct omap_dss_device *dst)
 {
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in;
-	int r;
-
-	if (omapdss_device_is_connected(dssdev))
-		return 0;
-
-	in = omapdss_of_find_source_for_first_ep(dssdev->dev->of_node);
-	if (IS_ERR(in)) {
-		dev_err(dssdev->dev, "failed to find video source\n");
-		return PTR_ERR(in);
-	}
-
-	r = in->ops.dpi->connect(in, dssdev);
-	if (r) {
-		omap_dss_put_device(in);
-		return r;
-	}
-
-	ddata->in = in;
 	return 0;
 }
 
-static void td028ttec1_panel_disconnect(struct omap_dss_device *dssdev)
+static void td028ttec1_panel_disconnect(struct omap_dss_device *src,
+					struct omap_dss_device *dst)
 {
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	if (!omapdss_device_is_connected(dssdev))
-		return;
-
-	in->ops.dpi->disconnect(in, dssdev);
-
-	omap_dss_put_device(in);
-	ddata->in = NULL;
 }
 
-static int td028ttec1_panel_enable(struct omap_dss_device *dssdev)
+static void td028ttec1_panel_enable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-	int r;
-
-	if (!omapdss_device_is_connected(dssdev))
-		return -ENODEV;
+	int r = 0;
 
-	if (omapdss_device_is_enabled(dssdev))
-		return 0;
-
-	in->ops.dpi->set_timings(in, &ddata->vm);
-
-	r = in->ops.dpi->enable(in);
-	if (r)
-		return r;
-
-	dev_dbg(dssdev->dev, "td028ttec1_panel_enable() - state %d\n",
-		dssdev->state);
+	dev_dbg(dssdev->dev, "%s: state %d\n", __func__, dssdev->state);
 
 	/* three times command zero */
 	r |= jbt_ret_write_0(ddata, 0x00);
@@ -235,8 +185,8 @@
 	usleep_range(1000, 2000);
 
 	if (r) {
-		dev_warn(dssdev->dev, "transfer error\n");
-		goto transfer_err;
+		dev_warn(dssdev->dev, "%s: transfer error\n", __func__);
+		return;
 	}
 
 	/* deep standby out */
@@ -306,20 +256,13 @@
 
 	r |= jbt_ret_write_0(ddata, JBT_REG_DISPLAY_ON);
 
-	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
-
-transfer_err:
-
-	return r ? -EIO : 0;
+	if (r)
+		dev_err(dssdev->dev, "%s: write error\n", __func__);
 }
 
 static void td028ttec1_panel_disable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	if (!omapdss_device_is_enabled(dssdev))
-		return;
 
 	dev_dbg(dssdev->dev, "td028ttec1_panel_disable()\n");
 
@@ -327,51 +270,24 @@
 	jbt_reg_write_2(ddata, JBT_REG_OUTPUT_CONTROL, 0x8002);
 	jbt_ret_write_0(ddata, JBT_REG_SLEEP_IN);
 	jbt_reg_write_1(ddata, JBT_REG_POWER_ON_OFF, 0x00);
-
-	in->ops.dpi->disable(in);
-
-	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
-}
-
-static void td028ttec1_panel_set_timings(struct omap_dss_device *dssdev,
-					 struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	ddata->vm = *vm;
-	dssdev->panel.vm = *vm;
-
-	in->ops.dpi->set_timings(in, vm);
 }
 
-static void td028ttec1_panel_get_timings(struct omap_dss_device *dssdev,
-					 struct videomode *vm)
+static int td028ttec1_panel_get_modes(struct omap_dss_device *dssdev,
+				      struct drm_connector *connector)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
-	*vm = ddata->vm;
+	return omapdss_display_get_modes(connector, &ddata->vm);
 }
 
-static int td028ttec1_panel_check_timings(struct omap_dss_device *dssdev,
-					  struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	return in->ops.dpi->check_timings(in, vm);
-}
-
-static struct omap_dss_driver td028ttec1_ops = {
+static const struct omap_dss_device_ops td028ttec1_ops = {
 	.connect	= td028ttec1_panel_connect,
 	.disconnect	= td028ttec1_panel_disconnect,
 
 	.enable		= td028ttec1_panel_enable,
 	.disable	= td028ttec1_panel_disable,
 
-	.set_timings	= td028ttec1_panel_set_timings,
-	.get_timings	= td028ttec1_panel_get_timings,
-	.check_timings	= td028ttec1_panel_check_timings,
+	.get_modes	= td028ttec1_panel_get_modes,
 };
 
 static int td028ttec1_panel_probe(struct spi_device *spi)
@@ -403,16 +319,23 @@
 
 	dssdev = &ddata->dssdev;
 	dssdev->dev = &spi->dev;
-	dssdev->driver = &td028ttec1_ops;
+	dssdev->ops = &td028ttec1_ops;
 	dssdev->type = OMAP_DISPLAY_TYPE_DPI;
+	dssdev->display = true;
 	dssdev->owner = THIS_MODULE;
-	dssdev->panel.vm = ddata->vm;
+	dssdev->of_ports = BIT(0);
+	dssdev->ops_flags = OMAP_DSS_DEVICE_OP_MODES;
 
-	r = omapdss_register_display(dssdev);
-	if (r) {
-		dev_err(&spi->dev, "Failed to register panel\n");
-		return r;
-	}
+	/*
+	 * Note: According to the panel documentation:
+	 * SYNC needs to be driven on the FALLING edge
+	 */
+	dssdev->bus_flags = DRM_BUS_FLAG_DE_HIGH
+			  | DRM_BUS_FLAG_SYNC_DRIVE_POSEDGE
+			  | DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE;
+
+	omapdss_display_init(dssdev);
+	omapdss_device_register(dssdev);
 
 	return 0;
 }
@@ -424,10 +347,9 @@
 
 	dev_dbg(&ddata->spi_dev->dev, "%s\n", __func__);
 
-	omapdss_unregister_display(dssdev);
+	omapdss_device_unregister(dssdev);
 
 	td028ttec1_panel_disable(dssdev);
-	td028ttec1_panel_disconnect(dssdev);
 
 	return 0;
 }
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/displays/panel-tpo-td043mtea1.c b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td043mtea1.c
--- a/drivers/gpu/drm/omapdrm/displays/panel-tpo-td043mtea1.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td043mtea1.c	2020-06-19 21:00:31.412227715 -0400
@@ -10,14 +10,13 @@
  * (at your option) any later version.
  */
 
-#include <linux/module.h>
 #include <linux/delay.h>
-#include <linux/spi/spi.h>
-#include <linux/regulator/consumer.h>
-#include <linux/gpio/consumer.h>
 #include <linux/err.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/regulator/consumer.h>
 #include <linux/slab.h>
-#include <linux/of_gpio.h>
+#include <linux/spi/spi.h>
 
 #include "../dss/omapdss.h"
 
@@ -54,16 +53,14 @@
 
 struct panel_drv_data {
 	struct omap_dss_device	dssdev;
-	struct omap_dss_device *in;
 
 	struct videomode vm;
 
 	struct spi_device *spi;
 	struct regulator *vcc_reg;
-	int nreset_gpio;
+	struct gpio_desc *reset_gpio;
 	u16 gamma[12];
 	u32 mode;
-	u32 hmirror:1;
 	u32 vmirror:1;
 	u32 powered_on:1;
 	u32 spi_suspended:1;
@@ -84,13 +81,7 @@
 	.vfront_porch	= 39,
 	.vback_porch	= 34,
 
-	.flags		= DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW |
-			  DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_SYNC_POSEDGE |
-			  DISPLAY_FLAGS_PIXDATA_NEGEDGE,
-	/*
-	 * Note: According to the panel documentation:
-	 * SYNC needs to be driven on the FALLING edge
-	 */
+	.flags		= DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW,
 };
 
 #define to_panel_data(p) container_of(p, struct panel_drv_data, dssdev)
@@ -152,22 +143,6 @@
 	return tpo_td043_write(spi, 4, reg4);
 }
 
-static int tpo_td043_set_hmirror(struct omap_dss_device *dssdev, bool enable)
-{
-	struct panel_drv_data *ddata = dev_get_drvdata(dssdev->dev);
-
-	ddata->hmirror = enable;
-	return tpo_td043_write_mirror(ddata->spi, ddata->hmirror,
-			ddata->vmirror);
-}
-
-static bool tpo_td043_get_hmirror(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = dev_get_drvdata(dssdev->dev);
-
-	return ddata->hmirror;
-}
-
 static ssize_t tpo_td043_vmirror_show(struct device *dev,
 	struct device_attribute *attr, char *buf)
 {
@@ -189,7 +164,7 @@
 
 	val = !!val;
 
-	ret = tpo_td043_write_mirror(ddata->spi, ddata->hmirror, val);
+	ret = tpo_td043_write_mirror(ddata->spi, false, val);
 	if (ret < 0)
 		return ret;
 
@@ -300,16 +275,14 @@
 	/* wait for panel to stabilize */
 	msleep(160);
 
-	if (gpio_is_valid(ddata->nreset_gpio))
-		gpio_set_value(ddata->nreset_gpio, 1);
+	gpiod_set_value(ddata->reset_gpio, 0);
 
 	tpo_td043_write(ddata->spi, 2,
 			TPO_R02_MODE(ddata->mode) | TPO_R02_NCLK_RISING);
 	tpo_td043_write(ddata->spi, 3, TPO_R03_VAL_NORMAL);
 	tpo_td043_write(ddata->spi, 0x20, 0xf0);
 	tpo_td043_write(ddata->spi, 0x21, 0xf0);
-	tpo_td043_write_mirror(ddata->spi, ddata->hmirror,
-			ddata->vmirror);
+	tpo_td043_write_mirror(ddata->spi, false, ddata->vmirror);
 	tpo_td043_write_gamma(ddata->spi, ddata->gamma);
 
 	ddata->powered_on = 1;
@@ -324,8 +297,7 @@
 	tpo_td043_write(ddata->spi, 3,
 			TPO_R03_VAL_STANDBY | TPO_R03_EN_PWM);
 
-	if (gpio_is_valid(ddata->nreset_gpio))
-		gpio_set_value(ddata->nreset_gpio, 0);
+	gpiod_set_value(ddata->reset_gpio, 1);
 
 	/* wait for at least 2 vsyncs before cutting off power */
 	msleep(50);
@@ -337,63 +309,22 @@
 	ddata->powered_on = 0;
 }
 
-static int tpo_td043_connect(struct omap_dss_device *dssdev)
+static int tpo_td043_connect(struct omap_dss_device *src,
+			     struct omap_dss_device *dst)
 {
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in;
-	int r;
-
-	if (omapdss_device_is_connected(dssdev))
-		return 0;
-
-	in = omapdss_of_find_source_for_first_ep(dssdev->dev->of_node);
-	if (IS_ERR(in)) {
-		dev_err(dssdev->dev, "failed to find video source\n");
-		return PTR_ERR(in);
-	}
-
-	r = in->ops.dpi->connect(in, dssdev);
-	if (r) {
-		omap_dss_put_device(in);
-		return r;
-	}
-
-	ddata->in = in;
 	return 0;
 }
 
-static void tpo_td043_disconnect(struct omap_dss_device *dssdev)
+static void tpo_td043_disconnect(struct omap_dss_device *src,
+				 struct omap_dss_device *dst)
 {
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	if (!omapdss_device_is_connected(dssdev))
-		return;
-
-	in->ops.dpi->disconnect(in, dssdev);
-
-	omap_dss_put_device(in);
-	ddata->in = NULL;
 }
 
-static int tpo_td043_enable(struct omap_dss_device *dssdev)
+static void tpo_td043_enable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
 	int r;
 
-	if (!omapdss_device_is_connected(dssdev))
-		return -ENODEV;
-
-	if (omapdss_device_is_enabled(dssdev))
-		return 0;
-
-	in->ops.dpi->set_timings(in, &ddata->vm);
-
-	r = in->ops.dpi->enable(in);
-	if (r)
-		return r;
-
 	/*
 	 * If we are resuming from system suspend, SPI clocks might not be
 	 * enabled yet, so we'll program the LCD from SPI PM resume callback.
@@ -401,96 +332,44 @@
 	if (!ddata->spi_suspended) {
 		r = tpo_td043_power_on(ddata);
 		if (r) {
-			in->ops.dpi->disable(in);
-			return r;
+			dev_err(&ddata->spi->dev, "%s: power on failed (%d)\n",
+				__func__, r);
+			return;
 		}
 	}
-
-	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
-
-	return 0;
 }
 
 static void tpo_td043_disable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	if (!omapdss_device_is_enabled(dssdev))
-		return;
-
-	in->ops.dpi->disable(in);
 
 	if (!ddata->spi_suspended)
 		tpo_td043_power_off(ddata);
-
-	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
-}
-
-static void tpo_td043_set_timings(struct omap_dss_device *dssdev,
-				  struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	ddata->vm = *vm;
-	dssdev->panel.vm = *vm;
-
-	in->ops.dpi->set_timings(in, vm);
 }
 
-static void tpo_td043_get_timings(struct omap_dss_device *dssdev,
-				  struct videomode *vm)
+static int tpo_td043_get_modes(struct omap_dss_device *dssdev,
+			       struct drm_connector *connector)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
-	*vm = ddata->vm;
+	return omapdss_display_get_modes(connector, &ddata->vm);
 }
 
-static int tpo_td043_check_timings(struct omap_dss_device *dssdev,
-				   struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *in = ddata->in;
-
-	return in->ops.dpi->check_timings(in, vm);
-}
-
-static struct omap_dss_driver tpo_td043_ops = {
+static const struct omap_dss_device_ops tpo_td043_ops = {
 	.connect	= tpo_td043_connect,
 	.disconnect	= tpo_td043_disconnect,
 
 	.enable		= tpo_td043_enable,
 	.disable	= tpo_td043_disable,
 
-	.set_timings	= tpo_td043_set_timings,
-	.get_timings	= tpo_td043_get_timings,
-	.check_timings	= tpo_td043_check_timings,
-
-	.set_mirror	= tpo_td043_set_hmirror,
-	.get_mirror	= tpo_td043_get_hmirror,
+	.get_modes	= tpo_td043_get_modes,
 };
 
-static int tpo_td043_probe_of(struct spi_device *spi)
-{
-	struct device_node *node = spi->dev.of_node;
-	struct panel_drv_data *ddata = dev_get_drvdata(&spi->dev);
-	int gpio;
-
-	gpio = of_get_named_gpio(node, "reset-gpios", 0);
-	if (!gpio_is_valid(gpio)) {
-		dev_err(&spi->dev, "failed to parse enable gpio\n");
-		return gpio;
-	}
-	ddata->nreset_gpio = gpio;
-
-	return 0;
-}
-
 static int tpo_td043_probe(struct spi_device *spi)
 {
 	struct panel_drv_data *ddata;
 	struct omap_dss_device *dssdev;
+	struct gpio_desc *gpio;
 	int r;
 
 	dev_dbg(&spi->dev, "%s\n", __func__);
@@ -512,59 +391,52 @@
 
 	ddata->spi = spi;
 
-	r = tpo_td043_probe_of(spi);
-	if (r)
-		return r;
-
 	ddata->mode = TPO_R02_MODE_800x480;
 	memcpy(ddata->gamma, tpo_td043_def_gamma, sizeof(ddata->gamma));
 
 	ddata->vcc_reg = devm_regulator_get(&spi->dev, "vcc");
 	if (IS_ERR(ddata->vcc_reg)) {
 		dev_err(&spi->dev, "failed to get LCD VCC regulator\n");
-		r = PTR_ERR(ddata->vcc_reg);
-		goto err_regulator;
+		return PTR_ERR(ddata->vcc_reg);
 	}
 
-	if (gpio_is_valid(ddata->nreset_gpio)) {
-		r = devm_gpio_request_one(&spi->dev,
-				ddata->nreset_gpio, GPIOF_OUT_INIT_LOW,
-				"lcd reset");
-		if (r < 0) {
-			dev_err(&spi->dev, "couldn't request reset GPIO\n");
-			goto err_gpio_req;
-		}
+	gpio = devm_gpiod_get(&spi->dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(gpio)) {
+		dev_err(&spi->dev, "failed to get reset gpio\n");
+		return PTR_ERR(gpio);
 	}
 
+	ddata->reset_gpio = gpio;
+
 	r = sysfs_create_group(&spi->dev.kobj, &tpo_td043_attr_group);
 	if (r) {
 		dev_err(&spi->dev, "failed to create sysfs files\n");
-		goto err_sysfs;
+		return r;
 	}
 
 	ddata->vm = tpo_td043_vm;
 
 	dssdev = &ddata->dssdev;
 	dssdev->dev = &spi->dev;
-	dssdev->driver = &tpo_td043_ops;
+	dssdev->ops = &tpo_td043_ops;
 	dssdev->type = OMAP_DISPLAY_TYPE_DPI;
+	dssdev->display = true;
 	dssdev->owner = THIS_MODULE;
-	dssdev->panel.vm = ddata->vm;
+	dssdev->of_ports = BIT(0);
+	dssdev->ops_flags = OMAP_DSS_DEVICE_OP_MODES;
 
-	r = omapdss_register_display(dssdev);
-	if (r) {
-		dev_err(&spi->dev, "Failed to register panel\n");
-		goto err_reg;
-	}
+	/*
+	 * Note: According to the panel documentation:
+	 * SYNC needs to be driven on the FALLING edge
+	 */
+	dssdev->bus_flags = DRM_BUS_FLAG_DE_HIGH
+			  | DRM_BUS_FLAG_SYNC_DRIVE_POSEDGE
+			  | DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE;
 
-	return 0;
+	omapdss_display_init(dssdev);
+	omapdss_device_register(dssdev);
 
-err_reg:
-	sysfs_remove_group(&spi->dev.kobj, &tpo_td043_attr_group);
-err_sysfs:
-err_gpio_req:
-err_regulator:
-	return r;
+	return 0;
 }
 
 static int tpo_td043_remove(struct spi_device *spi)
@@ -574,10 +446,10 @@
 
 	dev_dbg(&ddata->spi->dev, "%s\n", __func__);
 
-	omapdss_unregister_display(dssdev);
+	omapdss_device_unregister(dssdev);
 
-	tpo_td043_disable(dssdev);
-	tpo_td043_disconnect(dssdev);
+	if (omapdss_device_is_enabled(dssdev))
+		tpo_td043_disable(dssdev);
 
 	sysfs_remove_group(&spi->dev.kobj, &tpo_td043_attr_group);
 
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/dss/base.c b/drivers/gpu/drm/omapdrm/dss/base.c
--- a/drivers/gpu/drm/omapdrm/dss/base.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/dss/base.c	2020-06-19 21:00:31.412227715 -0400
@@ -14,24 +14,18 @@
  */
 
 #include <linux/kernel.h>
+#include <linux/list.h>
 #include <linux/module.h>
+#include <linux/mutex.h>
 #include <linux/of.h>
 #include <linux/of_graph.h>
-#include <linux/list.h>
+#include <linux/platform_device.h>
 
 #include "dss.h"
 #include "omapdss.h"
 
 static struct dss_device *dss_device;
 
-static struct list_head omapdss_comp_list;
-
-struct omapdss_comp_node {
-	struct list_head list;
-	struct device_node *node;
-	bool dss_core_component;
-};
-
 struct dss_device *omapdss_get_dss(void)
 {
 	return dss_device;
@@ -56,6 +50,262 @@
 }
 EXPORT_SYMBOL(dispc_get_ops);
 
+
+/* -----------------------------------------------------------------------------
+ * OMAP DSS Devices Handling
+ */
+
+static LIST_HEAD(omapdss_devices_list);
+static DEFINE_MUTEX(omapdss_devices_lock);
+
+void omapdss_device_register(struct omap_dss_device *dssdev)
+{
+	mutex_lock(&omapdss_devices_lock);
+	list_add_tail(&dssdev->list, &omapdss_devices_list);
+	mutex_unlock(&omapdss_devices_lock);
+}
+EXPORT_SYMBOL_GPL(omapdss_device_register);
+
+void omapdss_device_unregister(struct omap_dss_device *dssdev)
+{
+	mutex_lock(&omapdss_devices_lock);
+	list_del(&dssdev->list);
+	mutex_unlock(&omapdss_devices_lock);
+}
+EXPORT_SYMBOL_GPL(omapdss_device_unregister);
+
+static bool omapdss_device_is_registered(struct device_node *node)
+{
+	struct omap_dss_device *dssdev;
+	bool found = false;
+
+	mutex_lock(&omapdss_devices_lock);
+
+	list_for_each_entry(dssdev, &omapdss_devices_list, list) {
+		if (dssdev->dev->of_node == node) {
+			found = true;
+			break;
+		}
+	}
+
+	mutex_unlock(&omapdss_devices_lock);
+	return found;
+}
+
+struct omap_dss_device *omapdss_device_get(struct omap_dss_device *dssdev)
+{
+	if (!try_module_get(dssdev->owner))
+		return NULL;
+
+	if (get_device(dssdev->dev) == NULL) {
+		module_put(dssdev->owner);
+		return NULL;
+	}
+
+	return dssdev;
+}
+EXPORT_SYMBOL(omapdss_device_get);
+
+void omapdss_device_put(struct omap_dss_device *dssdev)
+{
+	put_device(dssdev->dev);
+	module_put(dssdev->owner);
+}
+EXPORT_SYMBOL(omapdss_device_put);
+
+struct omap_dss_device *omapdss_find_device_by_node(struct device_node *node)
+{
+	struct omap_dss_device *dssdev;
+
+	list_for_each_entry(dssdev, &omapdss_devices_list, list) {
+		if (dssdev->dev->of_node == node)
+			return omapdss_device_get(dssdev);
+	}
+
+	return NULL;
+}
+
+/*
+ * Search for the next output device starting at @from. Release the reference to
+ * the @from device, and acquire a reference to the returned device if found.
+ */
+struct omap_dss_device *omapdss_device_next_output(struct omap_dss_device *from)
+{
+	struct omap_dss_device *dssdev;
+	struct list_head *list;
+
+	mutex_lock(&omapdss_devices_lock);
+
+	if (list_empty(&omapdss_devices_list)) {
+		dssdev = NULL;
+		goto done;
+	}
+
+	/*
+	 * Start from the from entry if given or from omapdss_devices_list
+	 * otherwise.
+	 */
+	list = from ? &from->list : &omapdss_devices_list;
+
+	list_for_each_entry(dssdev, list, list) {
+		/*
+		 * Stop if we reach the omapdss_devices_list, that's the end of
+		 * the list.
+		 */
+		if (&dssdev->list == &omapdss_devices_list) {
+			dssdev = NULL;
+			goto done;
+		}
+
+		if (dssdev->id &&
+		    (dssdev->next || dssdev->bridge || dssdev->panel))
+			goto done;
+	}
+
+	dssdev = NULL;
+
+done:
+	if (from)
+		omapdss_device_put(from);
+	if (dssdev)
+		omapdss_device_get(dssdev);
+
+	mutex_unlock(&omapdss_devices_lock);
+	return dssdev;
+}
+EXPORT_SYMBOL(omapdss_device_next_output);
+
+static bool omapdss_device_is_connected(struct omap_dss_device *dssdev)
+{
+	return dssdev->dss;
+}
+
+int omapdss_device_connect(struct dss_device *dss,
+			   struct omap_dss_device *src,
+			   struct omap_dss_device *dst)
+{
+	int ret;
+
+	dev_dbg(&dss->pdev->dev, "connect(%s, %s)\n",
+		src ? dev_name(src->dev) : "NULL",
+		dst ? dev_name(dst->dev) : "NULL");
+
+	if (!dst) {
+		/*
+		 * The destination is NULL when the source is connected to a
+		 * bridge or panel instead of a DSS device. Stop here, we will
+		 * attach the bridge or panel later when we will have a DRM
+		 * encoder.
+		 */
+		return src && (src->bridge || src->panel) ? 0 : -EINVAL;
+	}
+
+	if (omapdss_device_is_connected(dst))
+		return -EBUSY;
+
+	dst->dss = dss;
+
+	ret = dst->ops->connect(src, dst);
+	if (ret < 0) {
+		dst->dss = NULL;
+		return ret;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(omapdss_device_connect);
+
+void omapdss_device_disconnect(struct omap_dss_device *src,
+			       struct omap_dss_device *dst)
+{
+	struct dss_device *dss = src ? src->dss : dst->dss;
+
+	dev_dbg(&dss->pdev->dev, "disconnect(%s, %s)\n",
+		src ? dev_name(src->dev) : "NULL",
+		dst ? dev_name(dst->dev) : "NULL");
+
+	if (!dst) {
+		WARN_ON(!src->bridge && !src->panel);
+		return;
+	}
+
+	if (!dst->id && !omapdss_device_is_connected(dst)) {
+		WARN_ON(!dst->display);
+		return;
+	}
+
+	WARN_ON(dst->state != OMAP_DSS_DISPLAY_DISABLED);
+
+	dst->ops->disconnect(src, dst);
+	dst->dss = NULL;
+}
+EXPORT_SYMBOL_GPL(omapdss_device_disconnect);
+
+void omapdss_device_pre_enable(struct omap_dss_device *dssdev)
+{
+	if (!dssdev)
+		return;
+
+	omapdss_device_pre_enable(dssdev->next);
+
+	if (dssdev->ops->pre_enable)
+		dssdev->ops->pre_enable(dssdev);
+}
+EXPORT_SYMBOL_GPL(omapdss_device_pre_enable);
+
+void omapdss_device_enable(struct omap_dss_device *dssdev)
+{
+	if (!dssdev)
+		return;
+
+	if (dssdev->ops->enable)
+		dssdev->ops->enable(dssdev);
+
+	omapdss_device_enable(dssdev->next);
+
+	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
+}
+EXPORT_SYMBOL_GPL(omapdss_device_enable);
+
+void omapdss_device_disable(struct omap_dss_device *dssdev)
+{
+	if (!dssdev)
+		return;
+
+	omapdss_device_disable(dssdev->next);
+
+	if (dssdev->ops->disable)
+		dssdev->ops->disable(dssdev);
+}
+EXPORT_SYMBOL_GPL(omapdss_device_disable);
+
+void omapdss_device_post_disable(struct omap_dss_device *dssdev)
+{
+	if (!dssdev)
+		return;
+
+	if (dssdev->ops->post_disable)
+		dssdev->ops->post_disable(dssdev);
+
+	omapdss_device_post_disable(dssdev->next);
+
+	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
+}
+EXPORT_SYMBOL_GPL(omapdss_device_post_disable);
+
+/* -----------------------------------------------------------------------------
+ * Components Handling
+ */
+
+static struct list_head omapdss_comp_list;
+
+struct omapdss_comp_node {
+	struct list_head list;
+	struct device_node *node;
+	bool dss_core_component;
+	const char *compat;
+};
+
 static bool omapdss_list_contains(const struct device_node *node)
 {
 	struct omapdss_comp_node *comp;
@@ -71,13 +321,20 @@
 static void omapdss_walk_device(struct device *dev, struct device_node *node,
 				bool dss_core)
 {
+	struct omapdss_comp_node *comp;
 	struct device_node *n;
-	struct omapdss_comp_node *comp = devm_kzalloc(dev, sizeof(*comp),
-						      GFP_KERNEL);
+	const char *compat;
+	int ret;
+
+	ret = of_property_read_string(node, "compatible", &compat);
+	if (ret < 0)
+		return;
 
+	comp = devm_kzalloc(dev, sizeof(*comp), GFP_KERNEL);
 	if (comp) {
 		comp->node = node;
 		comp->dss_core_component = dss_core;
+		comp->compat = compat;
 		list_add(&comp->list, &omapdss_comp_list);
 	}
 
@@ -117,12 +374,8 @@
 
 	omapdss_walk_device(dev, dev->of_node, true);
 
-	for_each_available_child_of_node(dev->of_node, child) {
-		if (!of_find_property(child, "compatible", NULL))
-			continue;
-
+	for_each_available_child_of_node(dev->of_node, child)
 		omapdss_walk_device(dev, child, true);
-	}
 }
 EXPORT_SYMBOL(omapdss_gather_components);
 
@@ -130,9 +383,9 @@
 {
 	if (comp->dss_core_component)
 		return true;
-	if (omapdss_component_is_display(comp->node))
+	if (!strstarts(comp->compat, "omapdss,"))
 		return true;
-	if (omapdss_component_is_output(comp->node))
+	if (omapdss_device_is_registered(comp->node))
 		return true;
 
 	return false;
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/dss/core.c b/drivers/gpu/drm/omapdrm/dss/core.c
--- a/drivers/gpu/drm/omapdrm/dss/core.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/dss/core.c	1969-12-31 19:00:00.000000000 -0500
@@ -1,88 +0,0 @@
-/*
- * Copyright (C) 2009 Nokia Corporation
- * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
- *
- * Some code and ideas taken from drivers/video/omap/ driver
- * by Imre Deak.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define DSS_SUBSYS_NAME "CORE"
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/platform_device.h>
-
-#include "omapdss.h"
-#include "dss.h"
-
-/* INIT */
-static struct platform_driver * const omap_dss_drivers[] = {
-	&omap_dsshw_driver,
-	&omap_dispchw_driver,
-#ifdef CONFIG_OMAP2_DSS_DSI
-	&omap_dsihw_driver,
-#endif
-#ifdef CONFIG_OMAP2_DSS_VENC
-	&omap_venchw_driver,
-#endif
-#ifdef CONFIG_OMAP4_DSS_HDMI
-	&omapdss_hdmi4hw_driver,
-#endif
-#ifdef CONFIG_OMAP5_DSS_HDMI
-	&omapdss_hdmi5hw_driver,
-#endif
-};
-
-static struct platform_device *omap_drm_device;
-
-static int __init omap_dss_init(void)
-{
-	int r;
-
-	r = platform_register_drivers(omap_dss_drivers,
-				      ARRAY_SIZE(omap_dss_drivers));
-	if (r)
-		goto err_reg;
-
-	omap_drm_device = platform_device_register_simple("omapdrm", 0, NULL, 0);
-	if (IS_ERR(omap_drm_device)) {
-		r = PTR_ERR(omap_drm_device);
-		goto err_reg;
-	}
-
-	return 0;
-
-err_reg:
-	platform_unregister_drivers(omap_dss_drivers,
-				    ARRAY_SIZE(omap_dss_drivers));
-
-	return r;
-}
-
-static void __exit omap_dss_exit(void)
-{
-	platform_device_unregister(omap_drm_device);
-
-	platform_unregister_drivers(omap_dss_drivers,
-				    ARRAY_SIZE(omap_dss_drivers));
-}
-
-module_init(omap_dss_init);
-module_exit(omap_dss_exit);
-
-MODULE_AUTHOR("Tomi Valkeinen <tomi.valkeinen@ti.com>");
-MODULE_DESCRIPTION("OMAP2/3 Display Subsystem");
-MODULE_LICENSE("GPL v2");
-
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/dss/dispc.c b/drivers/gpu/drm/omapdrm/dss/dispc.c
--- a/drivers/gpu/drm/omapdrm/dss/dispc.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/dss/dispc.c	2020-06-19 21:00:31.412227715 -0400
@@ -103,6 +103,8 @@
 	u8 mgr_height_start;
 	u16 mgr_width_max;
 	u16 mgr_height_max;
+	u16 ovl_width_max;
+	u16 ovl_height_max;
 	unsigned long max_lcd_pclk;
 	unsigned long max_tv_pclk;
 	unsigned int max_downscale;
@@ -125,6 +127,7 @@
 	const unsigned int num_reg_fields;
 	const enum omap_overlay_caps *overlay_caps;
 	const u32 **supported_color_modes;
+	const u32 *supported_scaler_color_modes;
 	unsigned int num_mgrs;
 	unsigned int num_ovls;
 	unsigned int buffer_size_unit;
@@ -914,32 +917,91 @@
 #undef CVAL
 }
 
-static void dispc_setup_color_conv_coef(struct dispc_device *dispc)
-{
-	int i;
-	int num_ovl = dispc_get_num_ovls(dispc);
+/* YUV -> RGB, ITU-R BT.601, full range */
+const static struct csc_coef_yuv2rgb coefs_yuv2rgb_bt601_full = {
+	256,   0,  358,		/* ry, rcb, rcr |1.000  0.000  1.402|*/
+	256, -88, -182,		/* gy, gcb, gcr |1.000 -0.344 -0.714|*/
+	256, 452,    0,		/* by, bcb, bcr |1.000  1.772  0.000|*/
+	true,			/* full range */
+};
 
-	/* YUV -> RGB, ITU-R BT.601, limited range */
-	const struct csc_coef_yuv2rgb coefs_yuv2rgb_bt601_lim = {
-		298,    0,  409,	/* ry, rcb, rcr */
-		298, -100, -208,	/* gy, gcb, gcr */
-		298,  516,    0,	/* by, bcb, bcr */
-		false,			/* limited range */
-	};
+/* YUV -> RGB, ITU-R BT.601, limited range */
+const static struct csc_coef_yuv2rgb coefs_yuv2rgb_bt601_lim = {
+	298,    0,  409,	/* ry, rcb, rcr |1.164  0.000  1.596|*/
+	298, -100, -208,	/* gy, gcb, gcr |1.164 -0.392 -0.813|*/
+	298,  516,    0,	/* by, bcb, bcr |1.164  2.017  0.000|*/
+	false,			/* limited range */
+};
 
-	/* RGB -> YUV, ITU-R BT.601, limited range */
-	const struct csc_coef_rgb2yuv coefs_rgb2yuv_bt601_lim = {
-		 66, 129,  25,		/* yr,   yg,  yb */
-		-38, -74, 112,		/* cbr, cbg, cbb */
-		112, -94, -18,		/* crr, crg, crb */
-		false,			/* limited range */
-	};
+/* YUV -> RGB, ITU-R BT.709, full range */
+const static struct csc_coef_yuv2rgb coefs_yuv2rgb_bt709_full = {
+	256,    0,  402,        /* ry, rcb, rcr |1.000  0.000  1.570|*/
+	256,  -48, -120,        /* gy, gcb, gcr |1.000 -0.187 -0.467|*/
+	256,  475,    0,        /* by, bcb, bcr |1.000  1.856  0.000|*/
+	true,                   /* full range */
+};
 
-	for (i = 1; i < num_ovl; i++)
-		dispc_ovl_write_color_conv_coef(dispc, i, &coefs_yuv2rgb_bt601_lim);
+/* YUV -> RGB, ITU-R BT.709, limited range */
+const static struct csc_coef_yuv2rgb coefs_yuv2rgb_bt709_lim = {
+	298,    0,  459,	/* ry, rcb, rcr |1.164  0.000  1.793|*/
+	298,  -55, -136,	/* gy, gcb, gcr |1.164 -0.213 -0.533|*/
+	298,  541,    0,	/* by, bcb, bcr |1.164  2.112  0.000|*/
+	false,			/* limited range */
+};
 
-	if (dispc->feat->has_writeback)
-		dispc_wb_write_color_conv_coef(dispc, &coefs_rgb2yuv_bt601_lim);
+/* RGB -> YUV, ITU-R BT.601, limited range */
+const static struct csc_coef_rgb2yuv coefs_rgb2yuv_bt601_lim = {
+	 66, 129,  25,		/* yr,   yg,  yb | 0.257  0.504  0.098|*/
+	-38, -74, 112,		/* cbr, cbg, cbb |-0.148 -0.291  0.439|*/
+	112, -94, -18,		/* crr, crg, crb | 0.439 -0.368 -0.071|*/
+	false,			/* limited range */
+};
+
+/* RGB -> YUV, ITU-R BT.601, full range */
+const static struct csc_coef_rgb2yuv coefs_rgb2yuv_bt601_full = {
+	 77,  150,  29,		/* yr,   yg,  yb | 0.299  0.587  0.114|*/
+	-43,  -85, 128,		/* cbr, cbg, cbb |-0.173 -0.339  0.511|*/
+	128, -107, -21,		/* crr, crg, crb | 0.511 -0.428 -0.083|*/
+	true,			/* full range */
+};
+
+/* RGB -> YUV, ITU-R BT.709, limited range */
+const static struct csc_coef_rgb2yuv coefs_rgb2yuv_bt701_lim = {
+	 47,  157,   16,	/* yr,   yg,  yb | 0.1826  0.6142  0.0620|*/
+	-26,  -87,  112,	/* cbr, cbg, cbb |-0.1006 -0.3386  0.4392|*/
+	112, -102,  -10,	/* crr, crg, crb | 0.4392 -0.3989 -0.0403|*/
+	false,			/* limited range */
+};
+
+static int dispc_ovl_set_csc(struct dispc_device *dispc,
+			     enum omap_plane_id plane,
+			     enum drm_color_encoding color_encoding,
+			     enum drm_color_range color_range)
+{
+	const struct csc_coef_yuv2rgb *csc;
+
+	switch (color_encoding) {
+	case DRM_COLOR_YCBCR_BT601:
+		if (color_range == DRM_COLOR_YCBCR_FULL_RANGE)
+			csc = &coefs_yuv2rgb_bt601_full;
+		else
+			csc = &coefs_yuv2rgb_bt601_lim;
+		break;
+	case DRM_COLOR_YCBCR_BT709:
+		if (color_range == DRM_COLOR_YCBCR_FULL_RANGE)
+			csc = &coefs_yuv2rgb_bt709_full;
+		else
+			csc = &coefs_yuv2rgb_bt709_lim;
+		break;
+	default:
+		DSSERR("Unsupported CSC mode %d for plane %d\n",
+		       color_encoding, plane);
+		return -EINVAL;
+	}
+
+	dispc_ovl_write_color_conv_coef(dispc, plane, csc);
+
+	return 0;
 }
 
 static void dispc_ovl_set_ba0(struct dispc_device *dispc,
@@ -1140,18 +1202,6 @@
 	REG_FLD_MOD(dispc, DISPC_OVL_ATTRIBUTES(plane), m, 4, 1);
 }
 
-static bool format_is_yuv(u32 fourcc)
-{
-	switch (fourcc) {
-	case DRM_FORMAT_YUYV:
-	case DRM_FORMAT_UYVY:
-	case DRM_FORMAT_NV12:
-		return true;
-	default:
-		return false;
-	}
-}
-
 static void dispc_ovl_configure_burst_type(struct dispc_device *dispc,
 					   enum omap_plane_id plane,
 					   enum omap_dss_rotation_type rotation)
@@ -1910,11 +1960,14 @@
 	int scale_x = out_width != orig_width;
 	int scale_y = out_height != orig_height;
 	bool chroma_upscale = plane != OMAP_DSS_WB;
+	const struct drm_format_info *info;
+
+	info = drm_format_info(fourcc);
 
 	if (!dispc_has_feature(dispc, FEAT_HANDLE_UV_SEPARATE))
 		return;
 
-	if (!format_is_yuv(fourcc)) {
+	if (!info->is_yuv) {
 		/* reset chroma resampling for RGB formats  */
 		if (plane != OMAP_DSS_WB)
 			REG_FLD_MOD(dispc, DISPC_OVL_ATTRIBUTES2(plane),
@@ -2506,6 +2559,12 @@
 	return 0;
 }
 
+static enum omap_overlay_caps dispc_ovl_get_caps(struct dispc_device *dispc,
+						 enum omap_plane_id plane)
+{
+	return dispc->feat->overlay_caps[plane];
+}
+
 #define DIV_FRAC(dividend, divisor) \
 	((dividend) * 100 / (divisor) - ((dividend) / (divisor) * 100))
 
@@ -2530,6 +2589,19 @@
 	if (width == out_width && height == out_height)
 		return 0;
 
+	if (dispc->feat->supported_scaler_color_modes) {
+		const u32 *modes = dispc->feat->supported_scaler_color_modes;
+		int i;
+
+		for (i = 0; modes[i]; ++i) {
+			if (modes[i] == fourcc)
+				break;
+		}
+
+		if (modes[i] == 0)
+			return -EINVAL;
+	}
+
 	if (plane == OMAP_DSS_WB) {
 		switch (fourcc) {
 		case DRM_FORMAT_NV12:
@@ -2605,6 +2677,13 @@
 	return 0;
 }
 
+static void dispc_ovl_get_max_size(struct dispc_device *dispc,
+				   u16 *width, u16 *height)
+{
+	*width = dispc->feat->ovl_width_max;
+	*height = dispc->feat->ovl_height_max;
+}
+
 static int dispc_ovl_setup_common(struct dispc_device *dispc,
 				  enum omap_plane_id plane,
 				  enum omap_overlay_caps caps,
@@ -2616,7 +2695,9 @@
 				  u8 pre_mult_alpha, u8 global_alpha,
 				  enum omap_dss_rotation_type rotation_type,
 				  bool replication, const struct videomode *vm,
-				  bool mem_to_mem)
+				  bool mem_to_mem,
+				  enum drm_color_encoding color_encoding,
+				  enum drm_color_range color_range)
 {
 	bool five_taps = true;
 	bool fieldmode = false;
@@ -2624,7 +2705,7 @@
 	unsigned int offset0, offset1;
 	s32 row_inc;
 	s32 pix_inc;
-	u16 frame_width, frame_height;
+	u16 frame_width;
 	unsigned int field_offset = 0;
 	u16 in_height = height;
 	u16 in_width = width;
@@ -2632,6 +2713,9 @@
 	bool ilace = !!(vm->flags & DISPLAY_FLAGS_INTERLACED);
 	unsigned long pclk = dispc_plane_pclk_rate(dispc, plane);
 	unsigned long lclk = dispc_plane_lclk_rate(dispc, plane);
+	const struct drm_format_info *info;
+
+	info = drm_format_info(fourcc);
 
 	/* when setting up WB, dispc_plane_pclk_rate() returns 0 */
 	if (plane == OMAP_DSS_WB)
@@ -2640,7 +2724,7 @@
 	if (paddr == 0 && rotation_type != OMAP_DSS_ROT_TILER)
 		return -EINVAL;
 
-	if (format_is_yuv(fourcc) && (in_width & 1)) {
+	if (info->is_yuv && (in_width & 1)) {
 		DSSERR("input width %d is not even for YUV format\n", in_width);
 		return -EINVAL;
 	}
@@ -2680,7 +2764,7 @@
 		DSSDBG("predecimation %d x %x, new input size %d x %d\n",
 			x_predecim, y_predecim, in_width, in_height);
 
-	if (format_is_yuv(fourcc) && (in_width & 1)) {
+	if (info->is_yuv && (in_width & 1)) {
 		DSSDBG("predecimated input width is not even for YUV format\n");
 		DSSDBG("adjusting input width %d -> %d\n",
 			in_width, in_width & ~1);
@@ -2688,7 +2772,7 @@
 		in_width &= ~1;
 	}
 
-	if (format_is_yuv(fourcc))
+	if (info->is_yuv)
 		cconv = 1;
 
 	if (ilace && !fieldmode) {
@@ -2714,13 +2798,10 @@
 	row_inc = 0;
 	pix_inc = 0;
 
-	if (plane == OMAP_DSS_WB) {
+	if (plane == OMAP_DSS_WB)
 		frame_width = out_width;
-		frame_height = out_height;
-	} else {
+	else
 		frame_width = in_width;
-		frame_height = height;
-	}
 
 	calc_offset(screen_width, frame_width,
 			fourcc, fieldmode, field_offset,
@@ -2765,6 +2846,9 @@
 				      fieldmode, fourcc, rotation);
 		dispc_ovl_set_output_size(dispc, plane, out_width, out_height);
 		dispc_ovl_set_vid_color_conv(dispc, plane, cconv);
+
+		if (plane != OMAP_DSS_WB)
+			dispc_ovl_set_csc(dispc, plane, color_encoding, color_range);
 	}
 
 	dispc_ovl_set_rotation_attrs(dispc, plane, rotation, rotation_type,
@@ -2801,7 +2885,8 @@
 		oi->screen_width, oi->pos_x, oi->pos_y, oi->width, oi->height,
 		oi->out_width, oi->out_height, oi->fourcc, oi->rotation,
 		oi->zorder, oi->pre_mult_alpha, oi->global_alpha,
-		oi->rotation_type, replication, vm, mem_to_mem);
+		oi->rotation_type, replication, vm, mem_to_mem,
+		oi->color_encoding, oi->color_range);
 
 	return r;
 }
@@ -2834,7 +2919,8 @@
 		wi->buf_width, pos_x, pos_y, in_width, in_height, wi->width,
 		wi->height, wi->fourcc, wi->rotation, zorder,
 		wi->pre_mult_alpha, global_alpha, wi->rotation_type,
-		replication, vm, mem_to_mem);
+		replication, vm, mem_to_mem, DRM_COLOR_YCBCR_BT601,
+		DRM_COLOR_YCBCR_LIMITED_RANGE);
 	if (r)
 		return r;
 
@@ -2904,13 +2990,6 @@
 	return 0;
 }
 
-static enum omap_dss_output_id
-dispc_mgr_get_supported_outputs(struct dispc_device *dispc,
-				enum omap_channel channel)
-{
-	return dss_get_supported_outputs(dispc->dss, channel);
-}
-
 static void dispc_lcd_enable_signal_polarity(struct dispc_device *dispc,
 					     bool act_high)
 {
@@ -3001,7 +3080,7 @@
 				info->trans_key);
 	dispc_mgr_enable_trans_key(dispc, channel, info->trans_enabled);
 	dispc_mgr_enable_alpha_fixed_zorder(dispc, channel,
-			info->partial_alpha_enabled);
+			info->alpha_blender_enabled);
 	if (dispc_has_feature(dispc, FEAT_CPR)) {
 		dispc_mgr_enable_cpr(dispc, channel, info->cpr_enable);
 		dispc_mgr_set_cpr_coef(dispc, channel, &info->cpr_coefs);
@@ -3120,28 +3199,29 @@
 		return pclk <= dispc->feat->max_tv_pclk;
 }
 
-bool dispc_mgr_timings_ok(struct dispc_device *dispc, enum omap_channel channel,
-			  const struct videomode *vm)
+static int dispc_mgr_check_timings(struct dispc_device *dispc,
+				   enum omap_channel channel,
+				   const struct videomode *vm)
 {
 	if (!_dispc_mgr_size_ok(dispc, vm->hactive, vm->vactive))
-		return false;
+		return MODE_BAD;
 
 	if (!_dispc_mgr_pclk_ok(dispc, channel, vm->pixelclock))
-		return false;
+		return MODE_BAD;
 
 	if (dss_mgr_is_lcd(channel)) {
 		/* TODO: OMAP4+ supports interlace for LCD outputs */
 		if (vm->flags & DISPLAY_FLAGS_INTERLACED)
-			return false;
+			return MODE_BAD;
 
 		if (!_dispc_lcd_timings_ok(dispc, vm->hsync_len,
 				vm->hfront_porch, vm->hback_porch,
 				vm->vsync_len, vm->vfront_porch,
 				vm->vback_porch))
-			return false;
+			return MODE_BAD;
 	}
 
-	return true;
+	return MODE_OK;
 }
 
 static void _dispc_mgr_set_lcd_timings(struct dispc_device *dispc,
@@ -3243,7 +3323,7 @@
 
 	DSSDBG("channel %d xres %u yres %u\n", channel, t.hactive, t.vactive);
 
-	if (!dispc_mgr_timings_ok(dispc, channel, &t)) {
+	if (dispc_mgr_check_timings(dispc, channel, &t)) {
 		BUG();
 		return;
 	}
@@ -3972,7 +4052,8 @@
 	    dispc->feat->has_gamma_table)
 		REG_FLD_MOD(dispc, DISPC_CONFIG, 1, 9, 9);
 
-	dispc_setup_color_conv_coef(dispc);
+	if (dispc->feat->has_writeback)
+		dispc_wb_write_color_conv_coef(dispc, &coefs_rgb2yuv_bt601_full);
 
 	dispc_set_loadmode(dispc, OMAP_DSS_LOAD_FRAME_ONLY);
 
@@ -4251,6 +4332,12 @@
 	DRM_FORMAT_RGBX8888),
 };
 
+static const u32 omap3_dispc_supported_scaler_color_modes[] = {
+	DRM_FORMAT_XRGB8888, DRM_FORMAT_RGB565, DRM_FORMAT_YUYV,
+	DRM_FORMAT_UYVY,
+	0,
+};
+
 static const struct dispc_features omap24xx_dispc_feats = {
 	.sw_start		=	5,
 	.fp_start		=	15,
@@ -4262,6 +4349,8 @@
 	.mgr_height_start	=	26,
 	.mgr_width_max		=	2048,
 	.mgr_height_max		=	2048,
+	.ovl_width_max		=	2048,
+	.ovl_height_max		=	2048,
 	.max_lcd_pclk		=	66500000,
 	.max_downscale		=	2,
 	/*
@@ -4279,6 +4368,7 @@
 	.num_reg_fields		=	ARRAY_SIZE(omap2_dispc_reg_fields),
 	.overlay_caps		=	omap2_dispc_overlay_caps,
 	.supported_color_modes	=	omap2_dispc_supported_color_modes,
+	.supported_scaler_color_modes = COLOR_ARRAY(DRM_FORMAT_XRGB8888),
 	.num_mgrs		=	2,
 	.num_ovls		=	3,
 	.buffer_size_unit	=	1,
@@ -4299,6 +4389,8 @@
 	.mgr_height_start	=	26,
 	.mgr_width_max		=	2048,
 	.mgr_height_max		=	2048,
+	.ovl_width_max		=	2048,
+	.ovl_height_max		=	2048,
 	.max_lcd_pclk		=	173000000,
 	.max_tv_pclk		=	59000000,
 	.max_downscale		=	4,
@@ -4313,6 +4405,7 @@
 	.num_reg_fields		=	ARRAY_SIZE(omap3_dispc_reg_fields),
 	.overlay_caps		=	omap3430_dispc_overlay_caps,
 	.supported_color_modes	=	omap3_dispc_supported_color_modes,
+	.supported_scaler_color_modes = omap3_dispc_supported_scaler_color_modes,
 	.num_mgrs		=	2,
 	.num_ovls		=	3,
 	.buffer_size_unit	=	1,
@@ -4333,6 +4426,8 @@
 	.mgr_height_start	=	26,
 	.mgr_width_max		=	2048,
 	.mgr_height_max		=	2048,
+	.ovl_width_max		=	2048,
+	.ovl_height_max		=	2048,
 	.max_lcd_pclk		=	173000000,
 	.max_tv_pclk		=	59000000,
 	.max_downscale		=	4,
@@ -4347,6 +4442,7 @@
 	.num_reg_fields		=	ARRAY_SIZE(omap3_dispc_reg_fields),
 	.overlay_caps		=	omap3430_dispc_overlay_caps,
 	.supported_color_modes	=	omap3_dispc_supported_color_modes,
+	.supported_scaler_color_modes = omap3_dispc_supported_scaler_color_modes,
 	.num_mgrs		=	2,
 	.num_ovls		=	3,
 	.buffer_size_unit	=	1,
@@ -4367,6 +4463,8 @@
 	.mgr_height_start	=	26,
 	.mgr_width_max		=	2048,
 	.mgr_height_max		=	2048,
+	.ovl_width_max		=	2048,
+	.ovl_height_max		=	2048,
 	.max_lcd_pclk		=	173000000,
 	.max_tv_pclk		=	59000000,
 	.max_downscale		=	4,
@@ -4381,6 +4479,7 @@
 	.num_reg_fields		=	ARRAY_SIZE(omap3_dispc_reg_fields),
 	.overlay_caps		=	omap3630_dispc_overlay_caps,
 	.supported_color_modes	=	omap3_dispc_supported_color_modes,
+	.supported_scaler_color_modes = omap3_dispc_supported_scaler_color_modes,
 	.num_mgrs		=	2,
 	.num_ovls		=	3,
 	.buffer_size_unit	=	1,
@@ -4401,6 +4500,8 @@
 	.mgr_height_start	=	26,
 	.mgr_width_max		=	2048,
 	.mgr_height_max		=	2048,
+	.ovl_width_max		=	2048,
+	.ovl_height_max		=	2048,
 	.max_lcd_pclk		=	173000000,
 	.max_tv_pclk		=	59000000,
 	.max_downscale		=	4,
@@ -4415,6 +4516,7 @@
 	.num_reg_fields		=	ARRAY_SIZE(omap3_dispc_reg_fields),
 	.overlay_caps		=	omap3430_dispc_overlay_caps,
 	.supported_color_modes	=	omap3_dispc_supported_color_modes,
+	.supported_scaler_color_modes = omap3_dispc_supported_scaler_color_modes,
 	.num_mgrs		=	1,
 	.num_ovls		=	3,
 	.buffer_size_unit	=	1,
@@ -4435,6 +4537,8 @@
 	.mgr_height_start	=	26,
 	.mgr_width_max		=	2048,
 	.mgr_height_max		=	2048,
+	.ovl_width_max		=	2048,
+	.ovl_height_max		=	2048,
 	.max_lcd_pclk		=	170000000,
 	.max_tv_pclk		=	185625000,
 	.max_downscale		=	4,
@@ -4474,8 +4578,10 @@
 	.mgr_height_start	=	27,
 	.mgr_width_max		=	4096,
 	.mgr_height_max		=	4096,
+	.ovl_width_max		=	2048,
+	.ovl_height_max		=	4096,
 	.max_lcd_pclk		=	170000000,
-	.max_tv_pclk		=	186000000,
+	.max_tv_pclk		=	192000000,
 	.max_downscale		=	4,
 	.max_line_width		=	2048,
 	.min_pcd		=	1,
@@ -4605,7 +4711,7 @@
 	.mgri = {
 		.default_color = 0,
 		.trans_enabled = false,
-		.partial_alpha_enabled = false,
+		.alpha_blender_enabled = false,
 		.cpr_enable = false,
 	},
 	.lcd_conf = {
@@ -4740,15 +4846,18 @@
 	.mgr_go_busy = dispc_mgr_go_busy,
 	.mgr_go = dispc_mgr_go,
 	.mgr_set_lcd_config = dispc_mgr_set_lcd_config,
+	.mgr_check_timings = dispc_mgr_check_timings,
 	.mgr_set_timings = dispc_mgr_set_timings,
 	.mgr_setup = dispc_mgr_setup,
-	.mgr_get_supported_outputs = dispc_mgr_get_supported_outputs,
 	.mgr_gamma_size = dispc_mgr_gamma_size,
 	.mgr_set_gamma = dispc_mgr_set_gamma,
 
 	.ovl_enable = dispc_ovl_enable,
 	.ovl_setup = dispc_ovl_setup,
 	.ovl_get_color_modes = dispc_ovl_get_color_modes,
+	.ovl_color_mode_supported = dispc_ovl_color_mode_supported,
+	.ovl_get_caps = dispc_ovl_get_caps,
+	.ovl_get_max_size = dispc_ovl_get_max_size,
 
 	.wb_get_framedone_irq = dispc_wb_get_framedone_irq,
 	.wb_setup = dispc_wb_setup,
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/dss/display.c b/drivers/gpu/drm/omapdrm/dss/display.c
--- a/drivers/gpu/drm/omapdrm/dss/display.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/dss/display.c	2020-06-19 21:00:31.412227715 -0400
@@ -21,27 +21,17 @@
 #define DSS_SUBSYS_NAME "DISPLAY"
 
 #include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/jiffies.h>
-#include <linux/platform_device.h>
 #include <linux/of.h>
 
-#include "omapdss.h"
+#include <drm/drm_connector.h>
+#include <drm/drm_modes.h>
 
-static void omapdss_default_get_timings(struct omap_dss_device *dssdev,
-					struct videomode *vm)
-{
-	*vm = dssdev->panel.vm;
-}
+#include "omapdss.h"
 
-static LIST_HEAD(panel_list);
-static DEFINE_MUTEX(panel_list_mutex);
 static int disp_num_counter;
 
-int omapdss_register_display(struct omap_dss_device *dssdev)
+void omapdss_display_init(struct omap_dss_device *dssdev)
 {
-	struct omap_dss_driver *drv = dssdev->driver;
-	struct list_head *cur;
 	int id;
 
 	/*
@@ -52,123 +42,39 @@
 	if (id < 0)
 		id = disp_num_counter++;
 
-	snprintf(dssdev->alias, sizeof(dssdev->alias), "display%d", id);
-
 	/* Use 'label' property for name, if it exists */
 	of_property_read_string(dssdev->dev->of_node, "label", &dssdev->name);
 
 	if (dssdev->name == NULL)
-		dssdev->name = dssdev->alias;
-
-	if (drv && drv->get_timings == NULL)
-		drv->get_timings = omapdss_default_get_timings;
-
-	mutex_lock(&panel_list_mutex);
-	list_for_each(cur, &panel_list) {
-		struct omap_dss_device *ldev = list_entry(cur,
-							 struct omap_dss_device,
-							 panel_list);
-		if (strcmp(ldev->alias, dssdev->alias) > 0)
-			break;
-	}
-	list_add_tail(&dssdev->panel_list, cur);
-	mutex_unlock(&panel_list_mutex);
-	return 0;
-}
-EXPORT_SYMBOL(omapdss_register_display);
-
-void omapdss_unregister_display(struct omap_dss_device *dssdev)
-{
-	mutex_lock(&panel_list_mutex);
-	list_del(&dssdev->panel_list);
-	mutex_unlock(&panel_list_mutex);
+		dssdev->name = devm_kasprintf(dssdev->dev, GFP_KERNEL,
+					      "display%u", id);
 }
-EXPORT_SYMBOL(omapdss_unregister_display);
+EXPORT_SYMBOL_GPL(omapdss_display_init);
 
-bool omapdss_component_is_display(struct device_node *node)
+struct omap_dss_device *omapdss_display_get(struct omap_dss_device *output)
 {
-	struct omap_dss_device *dssdev;
-	bool found = false;
+	while (output->next)
+		output = output->next;
 
-	mutex_lock(&panel_list_mutex);
-	list_for_each_entry(dssdev, &panel_list, panel_list) {
-		if (dssdev->dev->of_node == node) {
-			found = true;
-			goto out;
-		}
-	}
-out:
-	mutex_unlock(&panel_list_mutex);
-	return found;
+	return omapdss_device_get(output);
 }
-EXPORT_SYMBOL(omapdss_component_is_display);
+EXPORT_SYMBOL_GPL(omapdss_display_get);
 
-struct omap_dss_device *omap_dss_get_device(struct omap_dss_device *dssdev)
+int omapdss_display_get_modes(struct drm_connector *connector,
+			      const struct videomode *vm)
 {
-	if (!try_module_get(dssdev->owner))
-		return NULL;
-
-	if (get_device(dssdev->dev) == NULL) {
-		module_put(dssdev->owner);
-		return NULL;
-	}
-
-	return dssdev;
-}
-EXPORT_SYMBOL(omap_dss_get_device);
+	struct drm_display_mode *mode;
 
-void omap_dss_put_device(struct omap_dss_device *dssdev)
-{
-	put_device(dssdev->dev);
-	module_put(dssdev->owner);
-}
-EXPORT_SYMBOL(omap_dss_put_device);
+	mode = drm_mode_create(connector->dev);
+	if (!mode)
+		return 0;
 
-/*
- * ref count of the found device is incremented.
- * ref count of from-device is decremented.
- */
-struct omap_dss_device *omap_dss_get_next_device(struct omap_dss_device *from)
-{
-	struct list_head *l;
-	struct omap_dss_device *dssdev;
+	drm_display_mode_from_videomode(vm, mode);
 
-	mutex_lock(&panel_list_mutex);
+	mode->type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED;
+	drm_mode_set_name(mode);
+	drm_mode_probed_add(connector, mode);
 
-	if (list_empty(&panel_list)) {
-		dssdev = NULL;
-		goto out;
-	}
-
-	if (from == NULL) {
-		dssdev = list_first_entry(&panel_list, struct omap_dss_device,
-				panel_list);
-		omap_dss_get_device(dssdev);
-		goto out;
-	}
-
-	omap_dss_put_device(from);
-
-	list_for_each(l, &panel_list) {
-		dssdev = list_entry(l, struct omap_dss_device, panel_list);
-		if (dssdev == from) {
-			if (list_is_last(l, &panel_list)) {
-				dssdev = NULL;
-				goto out;
-			}
-
-			dssdev = list_entry(l->next, struct omap_dss_device,
-					panel_list);
-			omap_dss_get_device(dssdev);
-			goto out;
-		}
-	}
-
-	WARN(1, "'from' dssdev not found\n");
-
-	dssdev = NULL;
-out:
-	mutex_unlock(&panel_list_mutex);
-	return dssdev;
+	return 1;
 }
-EXPORT_SYMBOL(omap_dss_get_next_device);
+EXPORT_SYMBOL_GPL(omapdss_display_get_modes);
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/dss/dpi.c b/drivers/gpu/drm/omapdrm/dss/dpi.c
--- a/drivers/gpu/drm/omapdrm/dss/dpi.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/dss/dpi.c	2020-06-19 21:00:31.412227715 -0400
@@ -39,6 +39,7 @@
 	struct platform_device *pdev;
 	enum dss_model dss_model;
 	struct dss_device *dss;
+	unsigned int id;
 
 	struct regulator *vdds_dsi_reg;
 	enum dss_clk_source clk_src;
@@ -46,8 +47,8 @@
 
 	struct mutex lock;
 
-	struct videomode vm;
 	struct dss_lcd_mgr_config mgr_config;
+	unsigned long pixelclock;
 	int data_lines;
 
 	struct omap_dss_device output;
@@ -346,32 +347,19 @@
 
 static int dpi_set_mode(struct dpi_data *dpi)
 {
-	struct videomode *vm = &dpi->vm;
 	int lck_div = 0, pck_div = 0;
 	unsigned long fck = 0;
-	unsigned long pck;
 	int r = 0;
 
 	if (dpi->pll)
 		r = dpi_set_pll_clk(dpi, dpi->output.dispc_channel,
-				    vm->pixelclock, &fck, &lck_div, &pck_div);
+				    dpi->pixelclock, &fck, &lck_div, &pck_div);
 	else
-		r = dpi_set_dispc_clk(dpi, vm->pixelclock, &fck,
+		r = dpi_set_dispc_clk(dpi, dpi->pixelclock, &fck,
 				&lck_div, &pck_div);
 	if (r)
 		return r;
 
-	pck = fck / lck_div / pck_div;
-
-	if (pck != vm->pixelclock) {
-		DSSWARN("Could not find exact pixel clock. Requested %lu Hz, got %lu Hz\n",
-			vm->pixelclock, pck);
-
-		vm->pixelclock = pck;
-	}
-
-	dss_mgr_set_timings(&dpi->output, vm);
-
 	return 0;
 }
 
@@ -389,7 +377,7 @@
 	dss_mgr_set_lcd_config(&dpi->output, &dpi->mgr_config);
 }
 
-static int dpi_display_enable(struct omap_dss_device *dssdev)
+static void dpi_display_enable(struct omap_dss_device *dssdev)
 {
 	struct dpi_data *dpi = dpi_get_data_from_dssdev(dssdev);
 	struct omap_dss_device *out = &dpi->output;
@@ -397,12 +385,6 @@
 
 	mutex_lock(&dpi->lock);
 
-	if (!out->dispc_channel_connected) {
-		DSSERR("failed to enable display: no output/manager\n");
-		r = -ENODEV;
-		goto err_no_out_mgr;
-	}
-
 	if (dpi->vdds_dsi_reg) {
 		r = regulator_enable(dpi->vdds_dsi_reg);
 		if (r)
@@ -413,7 +395,7 @@
 	if (r)
 		goto err_get_dispc;
 
-	r = dss_dpi_select_source(dpi->dss, out->port_num, out->dispc_channel);
+	r = dss_dpi_select_source(dpi->dss, dpi->id, out->dispc_channel);
 	if (r)
 		goto err_src_sel;
 
@@ -437,7 +419,7 @@
 
 	mutex_unlock(&dpi->lock);
 
-	return 0;
+	return;
 
 err_mgr_enable:
 err_set_mode:
@@ -450,9 +432,7 @@
 	if (dpi->vdds_dsi_reg)
 		regulator_disable(dpi->vdds_dsi_reg);
 err_reg_enable:
-err_no_out_mgr:
 	mutex_unlock(&dpi->lock);
-	return r;
 }
 
 static void dpi_display_disable(struct omap_dss_device *dssdev)
@@ -478,7 +458,7 @@
 }
 
 static void dpi_set_timings(struct omap_dss_device *dssdev,
-			    struct videomode *vm)
+			    const struct drm_display_mode *mode)
 {
 	struct dpi_data *dpi = dpi_get_data_from_dssdev(dssdev);
 
@@ -486,51 +466,35 @@
 
 	mutex_lock(&dpi->lock);
 
-	dpi->vm = *vm;
-
-	mutex_unlock(&dpi->lock);
-}
-
-static void dpi_get_timings(struct omap_dss_device *dssdev,
-			    struct videomode *vm)
-{
-	struct dpi_data *dpi = dpi_get_data_from_dssdev(dssdev);
-
-	mutex_lock(&dpi->lock);
-
-	*vm = dpi->vm;
+	dpi->pixelclock = mode->clock * 1000;
 
 	mutex_unlock(&dpi->lock);
 }
 
 static int dpi_check_timings(struct omap_dss_device *dssdev,
-			     struct videomode *vm)
+			     struct drm_display_mode *mode)
 {
 	struct dpi_data *dpi = dpi_get_data_from_dssdev(dssdev);
-	enum omap_channel channel = dpi->output.dispc_channel;
 	int lck_div, pck_div;
 	unsigned long fck;
 	unsigned long pck;
 	struct dpi_clk_calc_ctx ctx;
 	bool ok;
 
-	if (vm->hactive % 8 != 0)
-		return -EINVAL;
-
-	if (!dispc_mgr_timings_ok(dpi->dss->dispc, channel, vm))
+	if (mode->hdisplay % 8 != 0)
 		return -EINVAL;
 
-	if (vm->pixelclock == 0)
+	if (mode->clock == 0)
 		return -EINVAL;
 
 	if (dpi->pll) {
-		ok = dpi_pll_clk_calc(dpi, vm->pixelclock, &ctx);
+		ok = dpi_pll_clk_calc(dpi, mode->clock * 1000, &ctx);
 		if (!ok)
 			return -EINVAL;
 
 		fck = ctx.pll_cinfo.clkout[ctx.clkout_idx];
 	} else {
-		ok = dpi_dss_clk_calc(dpi, vm->pixelclock, &ctx);
+		ok = dpi_dss_clk_calc(dpi, mode->clock * 1000, &ctx);
 		if (!ok)
 			return -EINVAL;
 
@@ -542,7 +506,7 @@
 
 	pck = fck / lck_div / pck_div;
 
-	vm->pixelclock = pck;
+	mode->clock = pck / 1000;
 
 	return 0;
 }
@@ -562,38 +526,6 @@
 	return 0;
 }
 
-static const struct soc_device_attribute dpi_soc_devices[] = {
-	{ .machine = "OMAP3[456]*" },
-	{ .machine = "[AD]M37*" },
-	{ /* sentinel */ }
-};
-
-static int dpi_init_regulator(struct dpi_data *dpi)
-{
-	struct regulator *vdds_dsi;
-
-	/*
-	 * The DPI uses the DSI VDDS on OMAP34xx, OMAP35xx, OMAP36xx, AM37xx and
-	 * DM37xx only.
-	 */
-	if (!soc_device_match(dpi_soc_devices))
-		return 0;
-
-	if (dpi->vdds_dsi_reg)
-		return 0;
-
-	vdds_dsi = devm_regulator_get(&dpi->pdev->dev, "vdds_dsi");
-	if (IS_ERR(vdds_dsi)) {
-		if (PTR_ERR(vdds_dsi) != -EPROBE_DEFER)
-			DSSERR("can't get VDDS_DSI regulator\n");
-		return PTR_ERR(vdds_dsi);
-	}
-
-	dpi->vdds_dsi_reg = vdds_dsi;
-
-	return 0;
-}
-
 static void dpi_init_pll(struct dpi_data *dpi)
 {
 	struct dss_pll *pll;
@@ -621,7 +553,7 @@
  * the channel in some more dynamic manner, or get the channel as a user
  * parameter.
  */
-static enum omap_channel dpi_get_channel(struct dpi_data *dpi, int port_num)
+static enum omap_channel dpi_get_channel(struct dpi_data *dpi)
 {
 	switch (dpi->dss_model) {
 	case DSS_MODEL_OMAP2:
@@ -629,7 +561,7 @@
 		return OMAP_DSS_CHANNEL_LCD;
 
 	case DSS_MODEL_DRA7:
-		switch (port_num) {
+		switch (dpi->id) {
 		case 2:
 			return OMAP_DSS_CHANNEL_LCD3;
 		case 1:
@@ -651,49 +583,23 @@
 	}
 }
 
-static int dpi_connect(struct omap_dss_device *dssdev,
-		struct omap_dss_device *dst)
+static int dpi_connect(struct omap_dss_device *src,
+		       struct omap_dss_device *dst)
 {
-	struct dpi_data *dpi = dpi_get_data_from_dssdev(dssdev);
-	int r;
-
-	r = dpi_init_regulator(dpi);
-	if (r)
-		return r;
+	struct dpi_data *dpi = dpi_get_data_from_dssdev(dst);
 
 	dpi_init_pll(dpi);
 
-	r = dss_mgr_connect(&dpi->output, dssdev);
-	if (r)
-		return r;
-
-	r = omapdss_output_set_device(dssdev, dst);
-	if (r) {
-		DSSERR("failed to connect output to new device: %s\n",
-				dst->name);
-		dss_mgr_disconnect(&dpi->output, dssdev);
-		return r;
-	}
-
-	return 0;
+	return omapdss_device_connect(dst->dss, dst, dst->next);
 }
 
-static void dpi_disconnect(struct omap_dss_device *dssdev,
-		struct omap_dss_device *dst)
+static void dpi_disconnect(struct omap_dss_device *src,
+			   struct omap_dss_device *dst)
 {
-	struct dpi_data *dpi = dpi_get_data_from_dssdev(dssdev);
-
-	WARN_ON(dst != dssdev->dst);
-
-	if (dst != dssdev->dst)
-		return;
-
-	omapdss_output_unset_device(dssdev);
-
-	dss_mgr_disconnect(&dpi->output, dssdev);
+	omapdss_device_disconnect(dst, dst->next);
 }
 
-static const struct omapdss_dpi_ops dpi_ops = {
+static const struct omap_dss_device_ops dpi_ops = {
 	.connect = dpi_connect,
 	.disconnect = dpi_disconnect,
 
@@ -702,18 +608,16 @@
 
 	.check_timings = dpi_check_timings,
 	.set_timings = dpi_set_timings,
-	.get_timings = dpi_get_timings,
 };
 
-static void dpi_init_output_port(struct dpi_data *dpi, struct device_node *port)
+static int dpi_init_output_port(struct dpi_data *dpi, struct device_node *port)
 {
 	struct omap_dss_device *out = &dpi->output;
+	u32 port_num = 0;
 	int r;
-	u32 port_num;
 
-	r = of_property_read_u32(port, "reg", &port_num);
-	if (r)
-		port_num = 0;
+	of_property_read_u32(port, "reg", &port_num);
+	dpi->id = port_num <= 2 ? port_num : 0;
 
 	switch (port_num) {
 	case 2:
@@ -730,13 +634,19 @@
 
 	out->dev = &dpi->pdev->dev;
 	out->id = OMAP_DSS_OUTPUT_DPI;
-	out->output_type = OMAP_DISPLAY_TYPE_DPI;
-	out->dispc_channel = dpi_get_channel(dpi, port_num);
-	out->port_num = port_num;
-	out->ops.dpi = &dpi_ops;
+	out->type = OMAP_DISPLAY_TYPE_DPI;
+	out->dispc_channel = dpi_get_channel(dpi);
+	out->of_ports = BIT(port_num);
+	out->ops = &dpi_ops;
 	out->owner = THIS_MODULE;
 
-	omapdss_register_output(out);
+	r = omapdss_device_init_output(out);
+	if (r < 0)
+		return r;
+
+	omapdss_device_register(out);
+
+	return 0;
 }
 
 static void dpi_uninit_output_port(struct device_node *port)
@@ -744,7 +654,37 @@
 	struct dpi_data *dpi = port->data;
 	struct omap_dss_device *out = &dpi->output;
 
-	omapdss_unregister_output(out);
+	omapdss_device_unregister(out);
+	omapdss_device_cleanup_output(out);
+}
+
+static const struct soc_device_attribute dpi_soc_devices[] = {
+	{ .machine = "OMAP3[456]*" },
+	{ .machine = "[AD]M37*" },
+	{ /* sentinel */ }
+};
+
+static int dpi_init_regulator(struct dpi_data *dpi)
+{
+	struct regulator *vdds_dsi;
+
+	/*
+	 * The DPI uses the DSI VDDS on OMAP34xx, OMAP35xx, OMAP36xx, AM37xx and
+	 * DM37xx only.
+	 */
+	if (!soc_device_match(dpi_soc_devices))
+		return 0;
+
+	vdds_dsi = devm_regulator_get(&dpi->pdev->dev, "vdds_dsi");
+	if (IS_ERR(vdds_dsi)) {
+		if (PTR_ERR(vdds_dsi) != -EPROBE_DEFER)
+			DSSERR("can't get VDDS_DSI regulator\n");
+		return PTR_ERR(vdds_dsi);
+	}
+
+	dpi->vdds_dsi_reg = vdds_dsi;
+
+	return 0;
 }
 
 int dpi_init_port(struct dss_device *dss, struct platform_device *pdev,
@@ -764,15 +704,14 @@
 		return 0;
 
 	r = of_property_read_u32(ep, "data-lines", &datalines);
+	of_node_put(ep);
 	if (r) {
 		DSSERR("failed to parse datalines\n");
-		goto err_datalines;
+		return r;
 	}
 
 	dpi->data_lines = datalines;
 
-	of_node_put(ep);
-
 	dpi->pdev = pdev;
 	dpi->dss_model = dss_model;
 	dpi->dss = dss;
@@ -780,14 +719,11 @@
 
 	mutex_init(&dpi->lock);
 
-	dpi_init_output_port(dpi, port);
-
-	return 0;
-
-err_datalines:
-	of_node_put(ep);
+	r = dpi_init_regulator(dpi);
+	if (r)
+		return r;
 
-	return r;
+	return dpi_init_output_port(dpi, port);
 }
 
 void dpi_uninit_port(struct device_node *port)
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/dss/dsi.c b/drivers/gpu/drm/omapdrm/dss/dsi.c
--- a/drivers/gpu/drm/omapdrm/dss/dsi.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/dss/dsi.c	2020-06-19 21:00:31.412227715 -0400
@@ -403,6 +403,7 @@
 	struct {
 		struct dss_debugfs_entry *irqs;
 		struct dss_debugfs_entry *regs;
+		struct dss_debugfs_entry *clks;
 	} debugfs;
 
 #ifdef CONFIG_OMAP2_DSS_COLLECT_IRQ_STATS
@@ -442,27 +443,6 @@
 	return dev_get_drvdata(dssdev->dev);
 }
 
-static struct dsi_data *dsi_get_dsi_from_id(int module)
-{
-	struct omap_dss_device *out;
-	enum omap_dss_output_id	id;
-
-	switch (module) {
-	case 0:
-		id = OMAP_DSS_OUTPUT_DSI1;
-		break;
-	case 1:
-		id = OMAP_DSS_OUTPUT_DSI2;
-		break;
-	default:
-		return NULL;
-	}
-
-	out = omap_dss_get_output(id);
-
-	return out ? to_dsi_data(out) : NULL;
-}
-
 static inline void dsi_write_reg(struct dsi_data *dsi,
 				 const struct dsi_reg idx, u32 val)
 {
@@ -1157,26 +1137,6 @@
 	WARN_ON(r < 0 && r != -ENOSYS);
 }
 
-static int dsi_regulator_init(struct dsi_data *dsi)
-{
-	struct regulator *vdds_dsi;
-
-	if (dsi->vdds_dsi_reg != NULL)
-		return 0;
-
-	vdds_dsi = devm_regulator_get(dsi->dev, "vdd");
-
-	if (IS_ERR(vdds_dsi)) {
-		if (PTR_ERR(vdds_dsi) != -EPROBE_DEFER)
-			DSSERR("can't get DSI VDD regulator\n");
-		return PTR_ERR(vdds_dsi);
-	}
-
-	dsi->vdds_dsi_reg = vdds_dsi;
-
-	return 0;
-}
-
 static void _dsi_print_reset_status(struct dsi_data *dsi)
 {
 	u32 l;
@@ -1373,10 +1333,6 @@
 
 	DSSDBG("PLL init\n");
 
-	r = dsi_regulator_init(dsi);
-	if (r)
-		return r;
-
 	r = dsi_runtime_get(dsi);
 	if (r)
 		return r;
@@ -1434,8 +1390,9 @@
 	DSSDBG("PLL disable done\n");
 }
 
-static void dsi_dump_dsi_clocks(struct dsi_data *dsi, struct seq_file *s)
+static int dsi_dump_dsi_clocks(struct seq_file *s, void *p)
 {
+	struct dsi_data *dsi = s->private;
 	struct dss_pll_clock_info *cinfo = &dsi->pll.cinfo;
 	enum dss_clk_source dispc_clk_src, dsi_clk_src;
 	int dsi_module = dsi->module_id;
@@ -1445,7 +1402,7 @@
 	dsi_clk_src = dss_get_dsi_clk_source(dsi->dss, dsi_module);
 
 	if (dsi_runtime_get(dsi))
-		return;
+		return 0;
 
 	seq_printf(s,	"- DSI%d PLL -\n", dsi_module + 1);
 
@@ -1489,23 +1446,14 @@
 	seq_printf(s,	"LP_CLK\t\t%lu\n", dsi->current_lp_cinfo.lp_clk);
 
 	dsi_runtime_put(dsi);
-}
-
-void dsi_dump_clocks(struct seq_file *s)
-{
-	struct dsi_data *dsi;
-	int i;
 
-	for  (i = 0; i < MAX_NUM_DSI; i++) {
-		dsi = dsi_get_dsi_from_id(i);
-		if (dsi)
-			dsi_dump_dsi_clocks(dsi, s);
-	}
+	return 0;
 }
 
 #ifdef CONFIG_OMAP2_DSS_COLLECT_IRQ_STATS
-static void dsi_dump_dsi_irqs(struct dsi_data *dsi, struct seq_file *s)
+static int dsi_dump_dsi_irqs(struct seq_file *s, void *p)
 {
+	struct dsi_data *dsi = s->private;
 	unsigned long flags;
 	struct dsi_irq_stats stats;
 
@@ -1589,33 +1537,20 @@
 	PIS(ULPSACTIVENOT_ALL0);
 	PIS(ULPSACTIVENOT_ALL1);
 #undef PIS
-}
 
-static int dsi1_dump_irqs(struct seq_file *s, void *p)
-{
-	struct dsi_data *dsi = dsi_get_dsi_from_id(0);
-
-	dsi_dump_dsi_irqs(dsi, s);
-	return 0;
-}
-
-static int dsi2_dump_irqs(struct seq_file *s, void *p)
-{
-	struct dsi_data *dsi = dsi_get_dsi_from_id(1);
-
-	dsi_dump_dsi_irqs(dsi, s);
 	return 0;
 }
 #endif
 
-static void dsi_dump_dsi_regs(struct dsi_data *dsi, struct seq_file *s)
+static int dsi_dump_dsi_regs(struct seq_file *s, void *p)
 {
-#define DUMPREG(r) seq_printf(s, "%-35s %08x\n", #r, dsi_read_reg(dsi, r))
+	struct dsi_data *dsi = s->private;
 
 	if (dsi_runtime_get(dsi))
-		return;
+		return 0;
 	dsi_enable_scp_clk(dsi);
 
+#define DUMPREG(r) seq_printf(s, "%-35s %08x\n", #r, dsi_read_reg(dsi, r))
 	DUMPREG(DSI_REVISION);
 	DUMPREG(DSI_SYSCONFIG);
 	DUMPREG(DSI_SYSSTATUS);
@@ -1685,25 +1620,11 @@
 	DUMPREG(DSI_PLL_GO);
 	DUMPREG(DSI_PLL_CONFIGURATION1);
 	DUMPREG(DSI_PLL_CONFIGURATION2);
+#undef DUMPREG
 
 	dsi_disable_scp_clk(dsi);
 	dsi_runtime_put(dsi);
-#undef DUMPREG
-}
-
-static int dsi1_dump_regs(struct seq_file *s, void *p)
-{
-	struct dsi_data *dsi = dsi_get_dsi_from_id(0);
 
-	dsi_dump_dsi_regs(dsi, s);
-	return 0;
-}
-
-static int dsi2_dump_regs(struct seq_file *s, void *p)
-{
-	struct dsi_data *dsi = dsi_get_dsi_from_id(1);
-
-	dsi_dump_dsi_regs(dsi, s);
 	return 0;
 }
 
@@ -3330,7 +3251,7 @@
 
 	if (dsi->mode == OMAP_DSS_DSI_VIDEO_MODE) {
 		int bpp = dsi_get_pixel_size(dsi->pix_fmt);
-		struct videomode *vm = &dsi->vm;
+		const struct videomode *vm = &dsi->vm;
 		/*
 		 * Don't use line buffers if width is greater than the video
 		 * port's line buffer size
@@ -3459,7 +3380,7 @@
 	int ddr_clk_pre, ddr_clk_post, enter_hs_mode_lat, exit_hs_mode_lat;
 	int tclk_trail, ths_exit, exiths_clk;
 	bool ddr_alwon;
-	struct videomode *vm = &dsi->vm;
+	const struct videomode *vm = &dsi->vm;
 	int bpp = dsi_get_pixel_size(dsi->pix_fmt);
 	int ndl = dsi->num_lanes_used - 1;
 	int dsi_fclk_hsdiv = dsi->user_dsi_cinfo.mX[HSDIV_DSI] + 1;
@@ -3709,7 +3630,7 @@
 		int vbp = dsi->vm_timings.vbp;
 		int window_sync = dsi->vm_timings.window_sync;
 		bool hsync_end;
-		struct videomode *vm = &dsi->vm;
+		const struct videomode *vm = &dsi->vm;
 		int bpp = dsi_get_pixel_size(dsi->pix_fmt);
 		int tl, t_he, width_bytes;
 
@@ -3818,19 +3739,13 @@
 {
 	struct dsi_data *dsi = to_dsi_data(dssdev);
 	int bpp = dsi_get_pixel_size(dsi->pix_fmt);
-	struct omap_dss_device *out = &dsi->output;
 	u8 data_type;
 	u16 word_count;
 	int r;
 
-	if (!out->dispc_channel_connected) {
-		DSSERR("failed to enable display: no output/manager\n");
-		return -ENODEV;
-	}
-
 	r = dsi_display_init_dispc(dsi);
 	if (r)
-		goto err_init_dispc;
+		return r;
 
 	if (dsi->mode == OMAP_DSS_DSI_VIDEO_MODE) {
 		switch (dsi->pix_fmt) {
@@ -3879,7 +3794,6 @@
 	}
 err_pix_fmt:
 	dsi_display_uninit_dispc(dsi);
-err_init_dispc:
 	return r;
 }
 
@@ -3966,8 +3880,6 @@
 		msecs_to_jiffies(250));
 	BUG_ON(r == 0);
 
-	dss_mgr_set_timings(&dsi->output, &dsi->vm);
-
 	dss_mgr_start_update(&dsi->output);
 
 	if (dsi->te_enabled) {
@@ -4109,24 +4021,6 @@
 		dsi->mgr_config.fifohandcheck = false;
 	}
 
-	/*
-	 * override interlace, logic level and edge related parameters in
-	 * videomode with default values
-	 */
-	dsi->vm.flags &= ~DISPLAY_FLAGS_INTERLACED;
-	dsi->vm.flags &= ~DISPLAY_FLAGS_HSYNC_LOW;
-	dsi->vm.flags |= DISPLAY_FLAGS_HSYNC_HIGH;
-	dsi->vm.flags &= ~DISPLAY_FLAGS_VSYNC_LOW;
-	dsi->vm.flags |= DISPLAY_FLAGS_VSYNC_HIGH;
-	dsi->vm.flags &= ~DISPLAY_FLAGS_PIXDATA_NEGEDGE;
-	dsi->vm.flags |= DISPLAY_FLAGS_PIXDATA_POSEDGE;
-	dsi->vm.flags &= ~DISPLAY_FLAGS_DE_LOW;
-	dsi->vm.flags |= DISPLAY_FLAGS_DE_HIGH;
-	dsi->vm.flags &= ~DISPLAY_FLAGS_SYNC_POSEDGE;
-	dsi->vm.flags |= DISPLAY_FLAGS_SYNC_NEGEDGE;
-
-	dss_mgr_set_timings(&dsi->output, &dsi->vm);
-
 	r = dsi_configure_dispc_clocks(dsi);
 	if (r)
 		goto err1;
@@ -4262,10 +4156,10 @@
 	}
 }
 
-static int dsi_display_enable(struct omap_dss_device *dssdev)
+static void dsi_display_enable(struct omap_dss_device *dssdev)
 {
 	struct dsi_data *dsi = to_dsi_data(dssdev);
-	int r = 0;
+	int r;
 
 	DSSDBG("dsi_display_enable\n");
 
@@ -4285,14 +4179,13 @@
 
 	mutex_unlock(&dsi->lock);
 
-	return 0;
+	return;
 
 err_init_dsi:
 	dsi_runtime_put(dsi);
 err_get_dsi:
 	mutex_unlock(&dsi->lock);
 	DSSDBG("dsi_display_enable FAILED\n");
-	return r;
 }
 
 static void dsi_display_disable(struct omap_dss_device *dssdev,
@@ -4842,6 +4735,30 @@
 	dsi->user_dispc_cinfo = ctx.dispc_cinfo;
 
 	dsi->vm = ctx.vm;
+
+	/*
+	 * override interlace, logic level and edge related parameters in
+	 * videomode with default values
+	 */
+	dsi->vm.flags &= ~DISPLAY_FLAGS_INTERLACED;
+	dsi->vm.flags &= ~DISPLAY_FLAGS_HSYNC_LOW;
+	dsi->vm.flags |= DISPLAY_FLAGS_HSYNC_HIGH;
+	dsi->vm.flags &= ~DISPLAY_FLAGS_VSYNC_LOW;
+	dsi->vm.flags |= DISPLAY_FLAGS_VSYNC_HIGH;
+	/*
+	 * HACK: These flags should be handled through the omap_dss_device bus
+	 * flags, but this will only be possible when the DSI encoder will be
+	 * converted to the omapdrm-managed encoder model.
+	 */
+	dsi->vm.flags &= ~DISPLAY_FLAGS_PIXDATA_NEGEDGE;
+	dsi->vm.flags |= DISPLAY_FLAGS_PIXDATA_POSEDGE;
+	dsi->vm.flags &= ~DISPLAY_FLAGS_DE_LOW;
+	dsi->vm.flags |= DISPLAY_FLAGS_DE_HIGH;
+	dsi->vm.flags &= ~DISPLAY_FLAGS_SYNC_POSEDGE;
+	dsi->vm.flags |= DISPLAY_FLAGS_SYNC_NEGEDGE;
+
+	dss_mgr_set_timings(&dsi->output, &dsi->vm);
+
 	dsi->vm_timings = ctx.dsi_vm;
 
 	mutex_unlock(&dsi->lock);
@@ -4962,163 +4879,62 @@
 	return 0;
 }
 
-static int dsi_connect(struct omap_dss_device *dssdev,
-		struct omap_dss_device *dst)
+static int dsi_connect(struct omap_dss_device *src,
+		       struct omap_dss_device *dst)
 {
-	struct dsi_data *dsi = to_dsi_data(dssdev);
-	int r;
-
-	r = dsi_regulator_init(dsi);
-	if (r)
-		return r;
-
-	r = dss_mgr_connect(&dsi->output, dssdev);
-	if (r)
-		return r;
-
-	r = omapdss_output_set_device(dssdev, dst);
-	if (r) {
-		DSSERR("failed to connect output to new device: %s\n",
-				dssdev->name);
-		dss_mgr_disconnect(&dsi->output, dssdev);
-		return r;
-	}
-
-	return 0;
+	return omapdss_device_connect(dst->dss, dst, dst->next);
 }
 
-static void dsi_disconnect(struct omap_dss_device *dssdev,
-		struct omap_dss_device *dst)
+static void dsi_disconnect(struct omap_dss_device *src,
+			   struct omap_dss_device *dst)
 {
-	struct dsi_data *dsi = to_dsi_data(dssdev);
-
-	WARN_ON(dst != dssdev->dst);
-
-	if (dst != dssdev->dst)
-		return;
-
-	omapdss_output_unset_device(dssdev);
-
-	dss_mgr_disconnect(&dsi->output, dssdev);
+	omapdss_device_disconnect(dst, dst->next);
 }
 
-static const struct omapdss_dsi_ops dsi_ops = {
+static const struct omap_dss_device_ops dsi_ops = {
 	.connect = dsi_connect,
 	.disconnect = dsi_disconnect,
-
-	.bus_lock = dsi_bus_lock,
-	.bus_unlock = dsi_bus_unlock,
-
 	.enable = dsi_display_enable,
-	.disable = dsi_display_disable,
-
-	.enable_hs = dsi_vc_enable_hs,
-
-	.configure_pins = dsi_configure_pins,
-	.set_config = dsi_set_config,
-
-	.enable_video_output = dsi_enable_video_output,
-	.disable_video_output = dsi_disable_video_output,
-
-	.update = dsi_update,
-
-	.enable_te = dsi_enable_te,
-
-	.request_vc = dsi_request_vc,
-	.set_vc_id = dsi_set_vc_id,
-	.release_vc = dsi_release_vc,
-
-	.dcs_write = dsi_vc_dcs_write,
-	.dcs_write_nosync = dsi_vc_dcs_write_nosync,
-	.dcs_read = dsi_vc_dcs_read,
-
-	.gen_write = dsi_vc_generic_write,
-	.gen_write_nosync = dsi_vc_generic_write_nosync,
-	.gen_read = dsi_vc_generic_read,
-
-	.bta_sync = dsi_vc_send_bta_sync,
-
-	.set_max_rx_packet_size = dsi_vc_set_max_rx_packet_size,
-};
-
-static void dsi_init_output(struct dsi_data *dsi)
-{
-	struct omap_dss_device *out = &dsi->output;
-
-	out->dev = dsi->dev;
-	out->id = dsi->module_id == 0 ?
-			OMAP_DSS_OUTPUT_DSI1 : OMAP_DSS_OUTPUT_DSI2;
-
-	out->output_type = OMAP_DISPLAY_TYPE_DSI;
-	out->name = dsi->module_id == 0 ? "dsi.0" : "dsi.1";
-	out->dispc_channel = dsi_get_channel(dsi);
-	out->ops.dsi = &dsi_ops;
-	out->owner = THIS_MODULE;
-
-	omapdss_register_output(out);
-}
 
-static void dsi_uninit_output(struct dsi_data *dsi)
-{
-	struct omap_dss_device *out = &dsi->output;
+	.dsi = {
+		.bus_lock = dsi_bus_lock,
+		.bus_unlock = dsi_bus_unlock,
 
-	omapdss_unregister_output(out);
-}
+		.disable = dsi_display_disable,
 
-static int dsi_probe_of(struct dsi_data *dsi)
-{
-	struct device_node *node = dsi->dev->of_node;
-	struct property *prop;
-	u32 lane_arr[10];
-	int len, num_pins;
-	int r, i;
-	struct device_node *ep;
-	struct omap_dsi_pin_config pin_cfg;
+		.enable_hs = dsi_vc_enable_hs,
 
-	ep = of_graph_get_endpoint_by_regs(node, 0, 0);
-	if (!ep)
-		return 0;
+		.configure_pins = dsi_configure_pins,
+		.set_config = dsi_set_config,
 
-	prop = of_find_property(ep, "lanes", &len);
-	if (prop == NULL) {
-		dev_err(dsi->dev, "failed to find lane data\n");
-		r = -EINVAL;
-		goto err;
-	}
+		.enable_video_output = dsi_enable_video_output,
+		.disable_video_output = dsi_disable_video_output,
 
-	num_pins = len / sizeof(u32);
+		.update = dsi_update,
 
-	if (num_pins < 4 || num_pins % 2 != 0 ||
-		num_pins > dsi->num_lanes_supported * 2) {
-		dev_err(dsi->dev, "bad number of lanes\n");
-		r = -EINVAL;
-		goto err;
-	}
+		.enable_te = dsi_enable_te,
 
-	r = of_property_read_u32_array(ep, "lanes", lane_arr, num_pins);
-	if (r) {
-		dev_err(dsi->dev, "failed to read lane data\n");
-		goto err;
-	}
+		.request_vc = dsi_request_vc,
+		.set_vc_id = dsi_set_vc_id,
+		.release_vc = dsi_release_vc,
 
-	pin_cfg.num_pins = num_pins;
-	for (i = 0; i < num_pins; ++i)
-		pin_cfg.pins[i] = (int)lane_arr[i];
+		.dcs_write = dsi_vc_dcs_write,
+		.dcs_write_nosync = dsi_vc_dcs_write_nosync,
+		.dcs_read = dsi_vc_dcs_read,
 
-	r = dsi_configure_pins(&dsi->output, &pin_cfg);
-	if (r) {
-		dev_err(dsi->dev, "failed to configure pins");
-		goto err;
-	}
+		.gen_write = dsi_vc_generic_write,
+		.gen_write_nosync = dsi_vc_generic_write_nosync,
+		.gen_read = dsi_vc_generic_read,
 
-	of_node_put(ep);
+		.bta_sync = dsi_vc_send_bta_sync,
 
-	return 0;
+		.set_max_rx_packet_size = dsi_vc_set_max_rx_packet_size,
+	},
+};
 
-err:
-	of_node_put(ep);
-	return r;
-}
+/* -----------------------------------------------------------------------------
+ * PLL
+ */
 
 static const struct dss_pll_ops dsi_pll_ops = {
 	.enable = dsi_pll_enable,
@@ -5233,7 +5049,162 @@
 	return 0;
 }
 
-/* DSI1 HW IP initialisation */
+/* -----------------------------------------------------------------------------
+ * Component Bind & Unbind
+ */
+
+static int dsi_bind(struct device *dev, struct device *master, void *data)
+{
+	struct dss_device *dss = dss_get_device(master);
+	struct dsi_data *dsi = dev_get_drvdata(dev);
+	char name[10];
+	u32 rev;
+	int r;
+
+	dsi->dss = dss;
+
+	dsi_init_pll_data(dss, dsi);
+
+	r = dsi_runtime_get(dsi);
+	if (r)
+		return r;
+
+	rev = dsi_read_reg(dsi, DSI_REVISION);
+	dev_dbg(dev, "OMAP DSI rev %d.%d\n",
+	       FLD_GET(rev, 7, 4), FLD_GET(rev, 3, 0));
+
+	dsi->line_buffer_size = dsi_get_line_buf_size(dsi);
+
+	dsi_runtime_put(dsi);
+
+	snprintf(name, sizeof(name), "dsi%u_regs", dsi->module_id + 1);
+	dsi->debugfs.regs = dss_debugfs_create_file(dss, name,
+						    dsi_dump_dsi_regs, dsi);
+#ifdef CONFIG_OMAP2_DSS_COLLECT_IRQ_STATS
+	snprintf(name, sizeof(name), "dsi%u_irqs", dsi->module_id + 1);
+	dsi->debugfs.irqs = dss_debugfs_create_file(dss, name,
+						    dsi_dump_dsi_irqs, dsi);
+#endif
+	snprintf(name, sizeof(name), "dsi%u_clks", dsi->module_id + 1);
+	dsi->debugfs.clks = dss_debugfs_create_file(dss, name,
+						    dsi_dump_dsi_clocks, dsi);
+
+	return 0;
+}
+
+static void dsi_unbind(struct device *dev, struct device *master, void *data)
+{
+	struct dsi_data *dsi = dev_get_drvdata(dev);
+
+	dss_debugfs_remove_file(dsi->debugfs.clks);
+	dss_debugfs_remove_file(dsi->debugfs.irqs);
+	dss_debugfs_remove_file(dsi->debugfs.regs);
+
+	WARN_ON(dsi->scp_clk_refcount > 0);
+
+	dss_pll_unregister(&dsi->pll);
+}
+
+static const struct component_ops dsi_component_ops = {
+	.bind	= dsi_bind,
+	.unbind	= dsi_unbind,
+};
+
+/* -----------------------------------------------------------------------------
+ * Probe & Remove, Suspend & Resume
+ */
+
+static int dsi_init_output(struct dsi_data *dsi)
+{
+	struct omap_dss_device *out = &dsi->output;
+	int r;
+
+	out->dev = dsi->dev;
+	out->id = dsi->module_id == 0 ?
+			OMAP_DSS_OUTPUT_DSI1 : OMAP_DSS_OUTPUT_DSI2;
+
+	out->type = OMAP_DISPLAY_TYPE_DSI;
+	out->name = dsi->module_id == 0 ? "dsi.0" : "dsi.1";
+	out->dispc_channel = dsi_get_channel(dsi);
+	out->ops = &dsi_ops;
+	out->owner = THIS_MODULE;
+	out->of_ports = BIT(0);
+	out->bus_flags = DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE
+		       | DRM_BUS_FLAG_DE_HIGH
+		       | DRM_BUS_FLAG_SYNC_DRIVE_NEGEDGE;
+
+	r = omapdss_device_init_output(out);
+	if (r < 0)
+		return r;
+
+	omapdss_device_register(out);
+
+	return 0;
+}
+
+static void dsi_uninit_output(struct dsi_data *dsi)
+{
+	struct omap_dss_device *out = &dsi->output;
+
+	omapdss_device_unregister(out);
+	omapdss_device_cleanup_output(out);
+}
+
+static int dsi_probe_of(struct dsi_data *dsi)
+{
+	struct device_node *node = dsi->dev->of_node;
+	struct property *prop;
+	u32 lane_arr[10];
+	int len, num_pins;
+	int r, i;
+	struct device_node *ep;
+	struct omap_dsi_pin_config pin_cfg;
+
+	ep = of_graph_get_endpoint_by_regs(node, 0, 0);
+	if (!ep)
+		return 0;
+
+	prop = of_find_property(ep, "lanes", &len);
+	if (prop == NULL) {
+		dev_err(dsi->dev, "failed to find lane data\n");
+		r = -EINVAL;
+		goto err;
+	}
+
+	num_pins = len / sizeof(u32);
+
+	if (num_pins < 4 || num_pins % 2 != 0 ||
+		num_pins > dsi->num_lanes_supported * 2) {
+		dev_err(dsi->dev, "bad number of lanes\n");
+		r = -EINVAL;
+		goto err;
+	}
+
+	r = of_property_read_u32_array(ep, "lanes", lane_arr, num_pins);
+	if (r) {
+		dev_err(dsi->dev, "failed to read lane data\n");
+		goto err;
+	}
+
+	pin_cfg.num_pins = num_pins;
+	for (i = 0; i < num_pins; ++i)
+		pin_cfg.pins[i] = (int)lane_arr[i];
+
+	r = dsi_configure_pins(&dsi->output, &pin_cfg);
+	if (r) {
+		dev_err(dsi->dev, "failed to configure pins");
+		goto err;
+	}
+
+	of_node_put(ep);
+
+	return 0;
+
+err:
+	of_node_put(ep);
+	return r;
+}
+
 static const struct dsi_of_data dsi_of_data_omap34xx = {
 	.model = DSI_MODEL_OMAP3,
 	.pll_hw = &dss_omap3_dsi_pll_hw,
@@ -5299,23 +5270,21 @@
 	{ /* sentinel */ }
 };
 
-static int dsi_bind(struct device *dev, struct device *master, void *data)
+static int dsi_probe(struct platform_device *pdev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct dss_device *dss = dss_get_device(master);
 	const struct soc_device_attribute *soc;
 	const struct dsi_module_id_data *d;
-	u32 rev;
-	int r, i;
+	struct device *dev = &pdev->dev;
 	struct dsi_data *dsi;
 	struct resource *dsi_mem;
 	struct resource *res;
+	unsigned int i;
+	int r;
 
 	dsi = devm_kzalloc(dev, sizeof(*dsi), GFP_KERNEL);
 	if (!dsi)
 		return -ENOMEM;
 
-	dsi->dss = dss;
 	dsi->dev = dev;
 	dev_set_drvdata(dev, dsi);
 
@@ -5366,6 +5335,13 @@
 		return r;
 	}
 
+	dsi->vdds_dsi_reg = devm_regulator_get(dev, "vdd");
+	if (IS_ERR(dsi->vdds_dsi_reg)) {
+		if (PTR_ERR(dsi->vdds_dsi_reg) != -EPROBE_DEFER)
+			DSSERR("can't get DSI VDD regulator\n");
+		return PTR_ERR(dsi->vdds_dsi_reg);
+	}
+
 	soc = soc_device_match(dsi_soc_devices);
 	if (soc)
 		dsi->data = soc->data;
@@ -5412,108 +5388,67 @@
 	if (r)
 		return r;
 
-	dsi_init_pll_data(dss, dsi);
-
 	pm_runtime_enable(dev);
 
-	r = dsi_runtime_get(dsi);
-	if (r)
-		goto err_runtime_get;
-
-	rev = dsi_read_reg(dsi, DSI_REVISION);
-	dev_dbg(dev, "OMAP DSI rev %d.%d\n",
-	       FLD_GET(rev, 7, 4), FLD_GET(rev, 3, 0));
-
 	/* DSI on OMAP3 doesn't have register DSI_GNQ, set number
 	 * of data to 3 by default */
-	if (dsi->data->quirks & DSI_QUIRK_GNQ)
+	if (dsi->data->quirks & DSI_QUIRK_GNQ) {
+		dsi_runtime_get(dsi);
 		/* NB_DATA_LANES */
 		dsi->num_lanes_supported = 1 + REG_GET(dsi, DSI_GNQ, 11, 9);
-	else
+		dsi_runtime_put(dsi);
+	} else {
 		dsi->num_lanes_supported = 3;
+	}
 
-	dsi->line_buffer_size = dsi_get_line_buf_size(dsi);
+	r = of_platform_populate(dev->of_node, NULL, NULL, dev);
+	if (r) {
+		DSSERR("Failed to populate DSI child devices: %d\n", r);
+		goto err_pm_disable;
+	}
 
-	dsi_init_output(dsi);
+	r = dsi_init_output(dsi);
+	if (r)
+		goto err_of_depopulate;
 
 	r = dsi_probe_of(dsi);
 	if (r) {
 		DSSERR("Invalid DSI DT data\n");
-		goto err_probe_of;
+		goto err_uninit_output;
 	}
 
-	r = of_platform_populate(dev->of_node, NULL, NULL, dev);
+	r = component_add(&pdev->dev, &dsi_component_ops);
 	if (r)
-		DSSERR("Failed to populate DSI child devices: %d\n", r);
-
-	dsi_runtime_put(dsi);
-
-	if (dsi->module_id == 0)
-		dsi->debugfs.regs = dss_debugfs_create_file(dss, "dsi1_regs",
-							    dsi1_dump_regs,
-							    &dsi);
-	else
-		dsi->debugfs.regs = dss_debugfs_create_file(dss, "dsi2_regs",
-							    dsi2_dump_regs,
-							    &dsi);
-#ifdef CONFIG_OMAP2_DSS_COLLECT_IRQ_STATS
-	if (dsi->module_id == 0)
-		dsi->debugfs.irqs = dss_debugfs_create_file(dss, "dsi1_irqs",
-							    dsi1_dump_irqs,
-							    &dsi);
-	else
-		dsi->debugfs.irqs = dss_debugfs_create_file(dss, "dsi2_irqs",
-							    dsi2_dump_irqs,
-							    &dsi);
-#endif
+		goto err_uninit_output;
 
 	return 0;
 
-err_probe_of:
+err_uninit_output:
 	dsi_uninit_output(dsi);
-	dsi_runtime_put(dsi);
-
-err_runtime_get:
+err_of_depopulate:
+	of_platform_depopulate(dev);
+err_pm_disable:
 	pm_runtime_disable(dev);
 	return r;
 }
 
-static void dsi_unbind(struct device *dev, struct device *master, void *data)
+static int dsi_remove(struct platform_device *pdev)
 {
-	struct dsi_data *dsi = dev_get_drvdata(dev);
-
-	dss_debugfs_remove_file(dsi->debugfs.irqs);
-	dss_debugfs_remove_file(dsi->debugfs.regs);
+	struct dsi_data *dsi = platform_get_drvdata(pdev);
 
-	of_platform_depopulate(dev);
-
-	WARN_ON(dsi->scp_clk_refcount > 0);
-
-	dss_pll_unregister(&dsi->pll);
+	component_del(&pdev->dev, &dsi_component_ops);
 
 	dsi_uninit_output(dsi);
 
-	pm_runtime_disable(dev);
+	of_platform_depopulate(&pdev->dev);
+
+	pm_runtime_disable(&pdev->dev);
 
 	if (dsi->vdds_dsi_reg != NULL && dsi->vdds_dsi_enabled) {
 		regulator_disable(dsi->vdds_dsi_reg);
 		dsi->vdds_dsi_enabled = false;
 	}
-}
-
-static const struct component_ops dsi_component_ops = {
-	.bind	= dsi_bind,
-	.unbind	= dsi_unbind,
-};
 
-static int dsi_probe(struct platform_device *pdev)
-{
-	return component_add(&pdev->dev, &dsi_component_ops);
-}
-
-static int dsi_remove(struct platform_device *pdev)
-{
-	component_del(&pdev->dev, &dsi_component_ops);
 	return 0;
 }
 
@@ -5527,19 +5462,12 @@
 	/* wait for current handler to finish before turning the DSI off */
 	synchronize_irq(dsi->irq);
 
-	dispc_runtime_put(dsi->dss->dispc);
-
 	return 0;
 }
 
 static int dsi_runtime_resume(struct device *dev)
 {
 	struct dsi_data *dsi = dev_get_drvdata(dev);
-	int r;
-
-	r = dispc_runtime_get(dsi->dss->dispc);
-	if (r)
-		return r;
 
 	dsi->is_enabled = true;
 	/* ensure the irq handler sees the is_enabled value */
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/dss/dss.c b/drivers/gpu/drm/omapdrm/dss/dss.c
--- a/drivers/gpu/drm/omapdrm/dss/dss.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/dss/dss.c	2020-06-19 21:00:31.412227715 -0400
@@ -394,9 +394,6 @@
 
 	dss_dump_clocks(dss, s);
 	dispc_dump_clocks(dss->dispc, s);
-#ifdef CONFIG_OMAP2_DSS_DSI
-	dsi_dump_clocks(s);
-#endif
 	return 0;
 }
 
@@ -681,12 +678,6 @@
 	return dss->feat->fck_freq_max;
 }
 
-enum omap_dss_output_id dss_get_supported_outputs(struct dss_device *dss,
-						  enum omap_channel channel)
-{
-	return dss->feat->outputs[channel];
-}
-
 static int dss_setup_default_clock(struct dss_device *dss)
 {
 	unsigned long max_dss_fck, prate;
@@ -956,7 +947,7 @@
 				&dss_debug_fops);
 	if (IS_ERR(d)) {
 		kfree(entry);
-		return ERR_PTR(PTR_ERR(d));
+		return ERR_CAST(d);
 	}
 
 	entry->dentry = d;
@@ -1183,7 +1174,8 @@
 	struct platform_device *pdev = dss->pdev;
 	struct device_node *parent = pdev->dev.of_node;
 	struct device_node *port;
-	int i;
+	unsigned int i;
+	int r;
 
 	for (i = 0; i < dss->feat->num_ports; i++) {
 		port = of_graph_get_port_by_id(parent, i);
@@ -1192,11 +1184,17 @@
 
 		switch (dss->feat->ports[i]) {
 		case OMAP_DISPLAY_TYPE_DPI:
-			dpi_init_port(dss, pdev, port, dss->feat->model);
+			r = dpi_init_port(dss, pdev, port, dss->feat->model);
+			if (r)
+				return r;
 			break;
+
 		case OMAP_DISPLAY_TYPE_SDI:
-			sdi_init_port(dss, pdev, port);
+			r = sdi_init_port(dss, pdev, port);
+			if (r)
+				return r;
 			break;
+
 		default:
 			break;
 		}
@@ -1315,6 +1313,7 @@
 static int dss_bind(struct device *dev)
 {
 	struct dss_device *dss = dev_get_drvdata(dev);
+	struct platform_device *drm_pdev;
 	int r;
 
 	r = component_bind_all(dev, NULL);
@@ -1323,14 +1322,25 @@
 
 	pm_set_vt_switch(0);
 
-	omapdss_gather_components(dev);
 	omapdss_set_dss(dss);
 
+	drm_pdev = platform_device_register_simple("omapdrm", 0, NULL, 0);
+	if (IS_ERR(drm_pdev)) {
+		component_unbind_all(dev, NULL);
+		return PTR_ERR(drm_pdev);
+	}
+
+	dss->drm_pdev = drm_pdev;
+
 	return 0;
 }
 
 static void dss_unbind(struct device *dev)
 {
+	struct dss_device *dss = dev_get_drvdata(dev);
+
+	platform_device_unregister(dss->drm_pdev);
+
 	omapdss_set_dss(NULL);
 
 	component_unbind_all(dev, NULL);
@@ -1474,14 +1484,23 @@
 						   dss);
 
 	/* Add all the child devices as components. */
+	r = of_platform_populate(pdev->dev.of_node, NULL, NULL, &pdev->dev);
+	if (r)
+		goto err_uninit_debugfs;
+
+	omapdss_gather_components(&pdev->dev);
+
 	device_for_each_child(&pdev->dev, &match, dss_add_child_component);
 
 	r = component_master_add_with_match(&pdev->dev, &dss_component_ops, match);
 	if (r)
-		goto err_uninit_debugfs;
+		goto err_of_depopulate;
 
 	return 0;
 
+err_of_depopulate:
+	of_platform_depopulate(&pdev->dev);
+
 err_uninit_debugfs:
 	dss_debugfs_remove_file(dss->debugfs.clk);
 	dss_debugfs_remove_file(dss->debugfs.dss);
@@ -1510,6 +1529,8 @@
 {
 	struct dss_device *dss = platform_get_drvdata(pdev);
 
+	of_platform_depopulate(&pdev->dev);
+
 	component_master_del(&pdev->dev, &dss_component_ops);
 
 	dss_debugfs_remove_file(dss->debugfs.clk);
@@ -1539,12 +1560,9 @@
 
 	DSSDBG("shutdown\n");
 
-	for_each_dss_dev(dssdev) {
-		if (!dssdev->driver)
-			continue;
-
+	for_each_dss_output(dssdev) {
 		if (dssdev->state == OMAP_DSS_DISPLAY_ACTIVE)
-			dssdev->driver->disable(dssdev);
+			dssdev->ops->disable(dssdev);
 	}
 }
 
@@ -1598,3 +1616,40 @@
 		.suppress_bind_attrs = true,
 	},
 };
+
+/* INIT */
+static struct platform_driver * const omap_dss_drivers[] = {
+	&omap_dsshw_driver,
+	&omap_dispchw_driver,
+#ifdef CONFIG_OMAP2_DSS_DSI
+	&omap_dsihw_driver,
+#endif
+#ifdef CONFIG_OMAP2_DSS_VENC
+	&omap_venchw_driver,
+#endif
+#ifdef CONFIG_OMAP4_DSS_HDMI
+	&omapdss_hdmi4hw_driver,
+#endif
+#ifdef CONFIG_OMAP5_DSS_HDMI
+	&omapdss_hdmi5hw_driver,
+#endif
+};
+
+static int __init omap_dss_init(void)
+{
+	return platform_register_drivers(omap_dss_drivers,
+					 ARRAY_SIZE(omap_dss_drivers));
+}
+
+static void __exit omap_dss_exit(void)
+{
+	platform_unregister_drivers(omap_dss_drivers,
+				    ARRAY_SIZE(omap_dss_drivers));
+}
+
+module_init(omap_dss_init);
+module_exit(omap_dss_exit);
+
+MODULE_AUTHOR("Tomi Valkeinen <tomi.valkeinen@nokia.com>");
+MODULE_DESCRIPTION("OMAP2/3 Display Subsystem");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/dss/dss.h b/drivers/gpu/drm/omapdrm/dss/dss.h
--- a/drivers/gpu/drm/omapdrm/dss/dss.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/dss/dss.h	2020-06-19 21:00:31.412227715 -0400
@@ -238,6 +238,8 @@
 	struct regmap	*syscon_pll_ctrl;
 	u32		syscon_pll_ctrl_offset;
 
+	struct platform_device *drm_pdev;
+
 	struct clk	*parent_clk;
 	struct clk	*dss_clk;
 	unsigned long	dss_clk_rate;
@@ -267,6 +269,8 @@
 
 	struct dispc_device *dispc;
 	const struct dispc_ops *dispc_ops;
+	const struct dss_mgr_ops *mgr_ops;
+	struct omap_drm_private *mgr_ops_priv;
 };
 
 /* core */
@@ -313,8 +317,6 @@
 
 unsigned long dss_get_dispc_clk_rate(struct dss_device *dss);
 unsigned long dss_get_max_fck_rate(struct dss_device *dss);
-enum omap_dss_output_id dss_get_supported_outputs(struct dss_device *dss,
-						  enum omap_channel channel);
 int dss_dpi_select_source(struct dss_device *dss, int port,
 			  enum omap_channel channel);
 void dss_select_hdmi_venc_clk_source(struct dss_device *dss,
@@ -374,8 +376,6 @@
 
 #ifdef CONFIG_OMAP2_DSS_DSI
 
-void dsi_dump_clocks(struct seq_file *s);
-
 void dsi_irq_handler(void);
 
 #endif
@@ -417,9 +417,6 @@
 		    unsigned long pck_min, unsigned long pck_max,
 		    dispc_div_calc_func func, void *data);
 
-bool dispc_mgr_timings_ok(struct dispc_device *dispc,
-			  enum omap_channel channel,
-			  const struct videomode *vm);
 int dispc_calc_clock_rates(struct dispc_device *dispc,
 			   unsigned long dispc_fclk_rate,
 			   struct dispc_clock_info *cinfo);
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/dss/dss-of.c b/drivers/gpu/drm/omapdrm/dss/dss-of.c
--- a/drivers/gpu/drm/omapdrm/dss/dss-of.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/dss/dss-of.c	2020-06-19 21:00:31.412227715 -0400
@@ -12,74 +12,25 @@
  * more details.
  */
 
-#include <linux/device.h>
 #include <linux/err.h>
-#include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_graph.h>
-#include <linux/seq_file.h>
 
 #include "omapdss.h"
 
-struct device_node *dss_of_port_get_parent_device(struct device_node *port)
-{
-	struct device_node *np;
-	int i;
-
-	if (!port)
-		return NULL;
-
-	np = of_get_parent(port);
-
-	for (i = 0; i < 2 && np; ++i) {
-		struct property *prop;
-
-		prop = of_find_property(np, "compatible", NULL);
-
-		if (prop)
-			return np;
-
-		np = of_get_next_parent(np);
-	}
-
-	return NULL;
-}
-
-u32 dss_of_port_get_port_number(struct device_node *port)
-{
-	int r;
-	u32 reg;
-
-	r = of_property_read_u32(port, "reg", &reg);
-	if (r)
-		reg = 0;
-
-	return reg;
-}
-
 struct omap_dss_device *
-omapdss_of_find_source_for_first_ep(struct device_node *node)
+omapdss_of_find_connected_device(struct device_node *node, unsigned int port)
 {
-	struct device_node *ep;
-	struct device_node *src_port;
-	struct omap_dss_device *src;
-
-	ep = of_graph_get_endpoint_by_regs(node, 0, 0);
-	if (!ep)
-		return ERR_PTR(-EINVAL);
-
-	src_port = of_graph_get_remote_port(ep);
-	if (!src_port) {
-		of_node_put(ep);
-		return ERR_PTR(-EINVAL);
-	}
+	struct device_node *remote_node;
+	struct omap_dss_device *dssdev;
 
-	of_node_put(ep);
-
-	src = omap_dss_find_output_by_port_node(src_port);
+	remote_node = of_graph_get_remote_node(node, port, 0);
+	if (!remote_node)
+		return NULL;
 
-	of_node_put(src_port);
+	dssdev = omapdss_find_device_by_node(remote_node);
+	of_node_put(remote_node);
 
-	return src ? src : ERR_PTR(-EPROBE_DEFER);
+	return dssdev ? dssdev : ERR_PTR(-EPROBE_DEFER);
 }
-EXPORT_SYMBOL_GPL(omapdss_of_find_source_for_first_ep);
+EXPORT_SYMBOL_GPL(omapdss_of_find_connected_device);
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/dss/hdmi4.c b/drivers/gpu/drm/omapdrm/dss/hdmi4.c
--- a/drivers/gpu/drm/omapdrm/dss/hdmi4.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi4.c	2020-06-19 21:00:31.412227715 -0400
@@ -108,26 +108,6 @@
 	return IRQ_HANDLED;
 }
 
-static int hdmi_init_regulator(struct omap_hdmi *hdmi)
-{
-	struct regulator *reg;
-
-	if (hdmi->vdda_reg != NULL)
-		return 0;
-
-	reg = devm_regulator_get(&hdmi->pdev->dev, "vdda");
-
-	if (IS_ERR(reg)) {
-		if (PTR_ERR(reg) != -EPROBE_DEFER)
-			DSSERR("can't get VDDA regulator\n");
-		return PTR_ERR(reg);
-	}
-
-	hdmi->vdda_reg = reg;
-
-	return 0;
-}
-
 static int hdmi_power_on_core(struct omap_hdmi *hdmi)
 {
 	int r;
@@ -174,7 +154,7 @@
 static int hdmi_power_on_full(struct omap_hdmi *hdmi)
 {
 	int r;
-	struct videomode *vm;
+	const struct videomode *vm;
 	struct hdmi_wp_data *wp = &hdmi->wp;
 	struct dss_pll_clock_info hdmi_cinfo = { 0 };
 	unsigned int pc;
@@ -227,9 +207,6 @@
 
 	hdmi4_configure(&hdmi->core, &hdmi->wp, &hdmi->cfg);
 
-	/* tv size */
-	dss_mgr_set_timings(&hdmi->output, vm);
-
 	r = dss_mgr_enable(&hdmi->output);
 	if (r)
 		goto err_mgr_enable;
@@ -271,39 +248,20 @@
 	hdmi_power_off_core(hdmi);
 }
 
-static int hdmi_display_check_timing(struct omap_dss_device *dssdev,
-				     struct videomode *vm)
-{
-	struct omap_hdmi *hdmi = dssdev_to_hdmi(dssdev);
-
-	if (!dispc_mgr_timings_ok(hdmi->dss->dispc, dssdev->dispc_channel, vm))
-		return -EINVAL;
-
-	return 0;
-}
-
-static void hdmi_display_set_timing(struct omap_dss_device *dssdev,
-				    struct videomode *vm)
+static void hdmi_display_set_timings(struct omap_dss_device *dssdev,
+				     const struct drm_display_mode *mode)
 {
 	struct omap_hdmi *hdmi = dssdev_to_hdmi(dssdev);
 
 	mutex_lock(&hdmi->lock);
 
-	hdmi->cfg.vm = *vm;
+	drm_display_mode_to_videomode(mode, &hdmi->cfg.vm);
 
-	dispc_set_tv_pclk(hdmi->dss->dispc, vm->pixelclock);
+	dispc_set_tv_pclk(hdmi->dss->dispc, mode->clock * 1000);
 
 	mutex_unlock(&hdmi->lock);
 }
 
-static void hdmi_display_get_timings(struct omap_dss_device *dssdev,
-				     struct videomode *vm)
-{
-	struct omap_hdmi *hdmi = dssdev_to_hdmi(dssdev);
-
-	*vm = hdmi->cfg.vm;
-}
-
 static int hdmi_dump_regs(struct seq_file *s, void *p)
 {
 	struct omap_hdmi *hdmi = s->private;
@@ -354,26 +312,20 @@
 	hdmi_wp_audio_enable(&hd->wp, false);
 }
 
-static int hdmi_display_enable(struct omap_dss_device *dssdev)
+static void hdmi_display_enable(struct omap_dss_device *dssdev)
 {
 	struct omap_hdmi *hdmi = dssdev_to_hdmi(dssdev);
 	unsigned long flags;
-	int r = 0;
+	int r;
 
 	DSSDBG("ENTER hdmi_display_enable\n");
 
 	mutex_lock(&hdmi->lock);
 
-	if (!dssdev->dispc_channel_connected) {
-		DSSERR("failed to enable display: no output/manager\n");
-		r = -ENODEV;
-		goto err0;
-	}
-
 	r = hdmi_power_on_full(hdmi);
 	if (r) {
 		DSSERR("failed to power on device\n");
-		goto err0;
+		goto done;
 	}
 
 	if (hdmi->audio_configured) {
@@ -393,12 +345,8 @@
 	hdmi->display_enabled = true;
 	spin_unlock_irqrestore(&hdmi->audio_playing_lock, flags);
 
+done:
 	mutex_unlock(&hdmi->lock);
-	return 0;
-
-err0:
-	mutex_unlock(&hdmi->lock);
-	return r;
 }
 
 static void hdmi_display_disable(struct omap_dss_device *dssdev)
@@ -456,44 +404,16 @@
 	mutex_unlock(&hdmi->lock);
 }
 
-static int hdmi_connect(struct omap_dss_device *dssdev,
-		struct omap_dss_device *dst)
+static int hdmi_connect(struct omap_dss_device *src,
+			struct omap_dss_device *dst)
 {
-	struct omap_hdmi *hdmi = dssdev_to_hdmi(dssdev);
-	int r;
-
-	r = hdmi_init_regulator(hdmi);
-	if (r)
-		return r;
-
-	r = dss_mgr_connect(&hdmi->output, dssdev);
-	if (r)
-		return r;
-
-	r = omapdss_output_set_device(dssdev, dst);
-	if (r) {
-		DSSERR("failed to connect output to new device: %s\n",
-				dst->name);
-		dss_mgr_disconnect(&hdmi->output, dssdev);
-		return r;
-	}
-
-	return 0;
+	return omapdss_device_connect(dst->dss, dst, dst->next);
 }
 
-static void hdmi_disconnect(struct omap_dss_device *dssdev,
-		struct omap_dss_device *dst)
+static void hdmi_disconnect(struct omap_dss_device *src,
+			    struct omap_dss_device *dst)
 {
-	struct omap_hdmi *hdmi = dssdev_to_hdmi(dssdev);
-
-	WARN_ON(dst != dssdev->dst);
-
-	if (dst != dssdev->dst)
-		return;
-
-	omapdss_output_unset_device(dssdev);
-
-	dss_mgr_disconnect(&hdmi->output, dssdev);
+	omapdss_device_disconnect(dst, dst->next);
 }
 
 static int hdmi_read_edid(struct omap_dss_device *dssdev,
@@ -548,69 +468,28 @@
 	return 0;
 }
 
-static const struct omapdss_hdmi_ops hdmi_ops = {
+static const struct omap_dss_device_ops hdmi_ops = {
 	.connect		= hdmi_connect,
 	.disconnect		= hdmi_disconnect,
 
 	.enable			= hdmi_display_enable,
 	.disable		= hdmi_display_disable,
 
-	.check_timings		= hdmi_display_check_timing,
-	.set_timings		= hdmi_display_set_timing,
-	.get_timings		= hdmi_display_get_timings,
+	.set_timings		= hdmi_display_set_timings,
 
 	.read_edid		= hdmi_read_edid,
-	.lost_hotplug		= hdmi_lost_hotplug,
-	.set_infoframe		= hdmi_set_infoframe,
-	.set_hdmi_mode		= hdmi_set_hdmi_mode,
-};
-
-static void hdmi_init_output(struct omap_hdmi *hdmi)
-{
-	struct omap_dss_device *out = &hdmi->output;
 
-	out->dev = &hdmi->pdev->dev;
-	out->id = OMAP_DSS_OUTPUT_HDMI;
-	out->output_type = OMAP_DISPLAY_TYPE_HDMI;
-	out->name = "hdmi.0";
-	out->dispc_channel = OMAP_DSS_CHANNEL_DIGIT;
-	out->ops.hdmi = &hdmi_ops;
-	out->owner = THIS_MODULE;
-
-	omapdss_register_output(out);
-}
-
-static void hdmi_uninit_output(struct omap_hdmi *hdmi)
-{
-	struct omap_dss_device *out = &hdmi->output;
-
-	omapdss_unregister_output(out);
-}
-
-static int hdmi_probe_of(struct omap_hdmi *hdmi)
-{
-	struct platform_device *pdev = hdmi->pdev;
-	struct device_node *node = pdev->dev.of_node;
-	struct device_node *ep;
-	int r;
-
-	ep = of_graph_get_endpoint_by_regs(node, 0, 0);
-	if (!ep)
-		return 0;
-
-	r = hdmi_parse_lanes_of(pdev, ep, &hdmi->phy);
-	if (r)
-		goto err;
-
-	of_node_put(ep);
-	return 0;
+	.hdmi = {
+		.lost_hotplug		= hdmi_lost_hotplug,
+		.set_infoframe		= hdmi_set_infoframe,
+		.set_hdmi_mode		= hdmi_set_hdmi_mode,
+	},
+};
 
-err:
-	of_node_put(ep);
-	return r;
-}
+/* -----------------------------------------------------------------------------
+ * Audio Callbacks
+ */
 
-/* Audio callbacks */
 static int hdmi_audio_startup(struct device *dev,
 			      void (*abort_cb)(struct device *dev))
 {
@@ -725,86 +604,49 @@
 	return 0;
 }
 
-/* HDMI HW IP initialisation */
+/* -----------------------------------------------------------------------------
+ * Component Bind & Unbind
+ */
+
 static int hdmi4_bind(struct device *dev, struct device *master, void *data)
 {
-	struct platform_device *pdev = to_platform_device(dev);
 	struct dss_device *dss = dss_get_device(master);
-	struct omap_hdmi *hdmi;
+	struct omap_hdmi *hdmi = dev_get_drvdata(dev);
 	int r;
-	int irq;
-
-	hdmi = kzalloc(sizeof(*hdmi), GFP_KERNEL);
-	if (!hdmi)
-		return -ENOMEM;
 
-	hdmi->pdev = pdev;
 	hdmi->dss = dss;
-	dev_set_drvdata(&pdev->dev, hdmi);
 
-	mutex_init(&hdmi->lock);
-	spin_lock_init(&hdmi->audio_playing_lock);
-
-	r = hdmi_probe_of(hdmi);
-	if (r)
-		goto err_free;
-
-	r = hdmi_wp_init(pdev, &hdmi->wp, 4);
-	if (r)
-		goto err_free;
-
-	r = hdmi_pll_init(dss, pdev, &hdmi->pll, &hdmi->wp);
+	r = hdmi_runtime_get(hdmi);
 	if (r)
-		goto err_free;
+		return r;
 
-	r = hdmi_phy_init(pdev, &hdmi->phy, 4);
+	r = hdmi_pll_init(dss, hdmi->pdev, &hdmi->pll, &hdmi->wp);
 	if (r)
-		goto err_pll;
+		goto err_runtime_put;
 
-	r = hdmi4_core_init(pdev, &hdmi->core);
+	r = hdmi4_cec_init(hdmi->pdev, &hdmi->core, &hdmi->wp);
 	if (r)
-		goto err_pll;
-
-	r = hdmi4_cec_init(pdev, &hdmi->core, &hdmi->wp);
-	if (r)
-		goto err_pll;
-
-	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		DSSERR("platform_get_irq failed\n");
-		r = -ENODEV;
-		goto err_pll;
-	}
-
-	r = devm_request_threaded_irq(&pdev->dev, irq,
-			NULL, hdmi_irq_handler,
-			IRQF_ONESHOT, "OMAP HDMI", hdmi);
-	if (r) {
-		DSSERR("HDMI IRQ request failed\n");
-		goto err_pll;
-	}
-
-	pm_runtime_enable(&pdev->dev);
-
-	hdmi_init_output(hdmi);
+		goto err_pll_uninit;
 
 	r = hdmi_audio_register(hdmi);
 	if (r) {
 		DSSERR("Registering HDMI audio failed\n");
-		hdmi_uninit_output(hdmi);
-		pm_runtime_disable(&pdev->dev);
-		return r;
+		goto err_cec_uninit;
 	}
 
 	hdmi->debugfs = dss_debugfs_create_file(dss, "hdmi", hdmi_dump_regs,
 					       hdmi);
 
+	hdmi_runtime_put(hdmi);
+
 	return 0;
 
-err_pll:
+err_cec_uninit:
+	hdmi4_cec_uninit(&hdmi->core);
+err_pll_uninit:
 	hdmi_pll_uninit(&hdmi->pll);
-err_free:
-	kfree(hdmi);
+err_runtime_put:
+	hdmi_runtime_put(hdmi);
 	return r;
 }
 
@@ -817,15 +659,8 @@
 	if (hdmi->audio_pdev)
 		platform_device_unregister(hdmi->audio_pdev);
 
-	hdmi_uninit_output(hdmi);
-
 	hdmi4_cec_uninit(&hdmi->core);
-
 	hdmi_pll_uninit(&hdmi->pll);
-
-	pm_runtime_disable(dev);
-
-	kfree(hdmi);
 }
 
 static const struct component_ops hdmi4_component_ops = {
@@ -833,42 +668,148 @@
 	.unbind	= hdmi4_unbind,
 };
 
-static int hdmi4_probe(struct platform_device *pdev)
+/* -----------------------------------------------------------------------------
+ * Probe & Remove, Suspend & Resume
+ */
+
+static int hdmi4_init_output(struct omap_hdmi *hdmi)
 {
-	return component_add(&pdev->dev, &hdmi4_component_ops);
+	struct omap_dss_device *out = &hdmi->output;
+	int r;
+
+	out->dev = &hdmi->pdev->dev;
+	out->id = OMAP_DSS_OUTPUT_HDMI;
+	out->type = OMAP_DISPLAY_TYPE_HDMI;
+	out->name = "hdmi.0";
+	out->dispc_channel = OMAP_DSS_CHANNEL_DIGIT;
+	out->ops = &hdmi_ops;
+	out->owner = THIS_MODULE;
+	out->of_ports = BIT(0);
+	out->ops_flags = OMAP_DSS_DEVICE_OP_EDID;
+
+	r = omapdss_device_init_output(out);
+	if (r < 0)
+		return r;
+
+	omapdss_device_register(out);
+
+	return 0;
 }
 
-static int hdmi4_remove(struct platform_device *pdev)
+static void hdmi4_uninit_output(struct omap_hdmi *hdmi)
 {
-	component_del(&pdev->dev, &hdmi4_component_ops);
-	return 0;
+	struct omap_dss_device *out = &hdmi->output;
+
+	omapdss_device_unregister(out);
+	omapdss_device_cleanup_output(out);
 }
 
-static int hdmi_runtime_suspend(struct device *dev)
+static int hdmi4_probe_of(struct omap_hdmi *hdmi)
 {
-	struct omap_hdmi *hdmi = dev_get_drvdata(dev);
+	struct platform_device *pdev = hdmi->pdev;
+	struct device_node *node = pdev->dev.of_node;
+	struct device_node *ep;
+	int r;
 
-	dispc_runtime_put(hdmi->dss->dispc);
+	ep = of_graph_get_endpoint_by_regs(node, 0, 0);
+	if (!ep)
+		return 0;
 
-	return 0;
+	r = hdmi_parse_lanes_of(pdev, ep, &hdmi->phy);
+	of_node_put(ep);
+	return r;
 }
 
-static int hdmi_runtime_resume(struct device *dev)
+static int hdmi4_probe(struct platform_device *pdev)
 {
-	struct omap_hdmi *hdmi = dev_get_drvdata(dev);
+	struct omap_hdmi *hdmi;
+	int irq;
 	int r;
 
-	r = dispc_runtime_get(hdmi->dss->dispc);
-	if (r < 0)
-		return r;
+	hdmi = kzalloc(sizeof(*hdmi), GFP_KERNEL);
+	if (!hdmi)
+		return -ENOMEM;
+
+	hdmi->pdev = pdev;
+
+	dev_set_drvdata(&pdev->dev, hdmi);
+
+	mutex_init(&hdmi->lock);
+	spin_lock_init(&hdmi->audio_playing_lock);
+
+	r = hdmi4_probe_of(hdmi);
+	if (r)
+		goto err_free;
+
+	r = hdmi_wp_init(pdev, &hdmi->wp, 4);
+	if (r)
+		goto err_free;
+
+	r = hdmi_phy_init(pdev, &hdmi->phy, 4);
+	if (r)
+		goto err_free;
+
+	r = hdmi4_core_init(pdev, &hdmi->core);
+	if (r)
+		goto err_free;
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		DSSERR("platform_get_irq failed\n");
+		r = -ENODEV;
+		goto err_free;
+	}
+
+	r = devm_request_threaded_irq(&pdev->dev, irq,
+			NULL, hdmi_irq_handler,
+			IRQF_ONESHOT, "OMAP HDMI", hdmi);
+	if (r) {
+		DSSERR("HDMI IRQ request failed\n");
+		goto err_free;
+	}
+
+	hdmi->vdda_reg = devm_regulator_get(&pdev->dev, "vdda");
+	if (IS_ERR(hdmi->vdda_reg)) {
+		r = PTR_ERR(hdmi->vdda_reg);
+		if (r != -EPROBE_DEFER)
+			DSSERR("can't get VDDA regulator\n");
+		goto err_free;
+	}
+
+	pm_runtime_enable(&pdev->dev);
+
+	r = hdmi4_init_output(hdmi);
+	if (r)
+		goto err_pm_disable;
+
+	r = component_add(&pdev->dev, &hdmi4_component_ops);
+	if (r)
+		goto err_uninit_output;
 
 	return 0;
+
+err_uninit_output:
+	hdmi4_uninit_output(hdmi);
+err_pm_disable:
+	pm_runtime_disable(&pdev->dev);
+err_free:
+	kfree(hdmi);
+	return r;
 }
 
-static const struct dev_pm_ops hdmi_pm_ops = {
-	.runtime_suspend = hdmi_runtime_suspend,
-	.runtime_resume = hdmi_runtime_resume,
-};
+static int hdmi4_remove(struct platform_device *pdev)
+{
+	struct omap_hdmi *hdmi = platform_get_drvdata(pdev);
+
+	component_del(&pdev->dev, &hdmi4_component_ops);
+
+	hdmi4_uninit_output(hdmi);
+
+	pm_runtime_disable(&pdev->dev);
+
+	kfree(hdmi);
+	return 0;
+}
 
 static const struct of_device_id hdmi_of_match[] = {
 	{ .compatible = "ti,omap4-hdmi", },
@@ -880,7 +821,6 @@
 	.remove		= hdmi4_remove,
 	.driver         = {
 		.name   = "omapdss_hdmi",
-		.pm	= &hdmi_pm_ops,
 		.of_match_table = hdmi_of_match,
 		.suppress_bind_attrs = true,
 	},
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/dss/hdmi4_core.c b/drivers/gpu/drm/omapdrm/dss/hdmi4_core.c
--- a/drivers/gpu/drm/omapdrm/dss/hdmi4_core.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi4_core.c	2020-06-19 21:00:31.412227715 -0400
@@ -553,8 +553,9 @@
 	}
 
 	/* Set ACR clock divisor */
-	REG_FLD_MOD(av_base,
-			HDMI_CORE_AV_FREQ_SVAL, cfg->mclk_mode, 2, 0);
+	if (cfg->use_mclk)
+		REG_FLD_MOD(av_base, HDMI_CORE_AV_FREQ_SVAL,
+			    cfg->mclk_mode, 2, 0);
 
 	r = hdmi_read_reg(av_base, HDMI_CORE_AV_ACR_CTRL);
 	/*
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/dss/hdmi5.c b/drivers/gpu/drm/omapdrm/dss/hdmi5.c
--- a/drivers/gpu/drm/omapdrm/dss/hdmi5.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi5.c	2020-06-19 21:00:31.412227715 -0400
@@ -117,24 +117,6 @@
 	return IRQ_HANDLED;
 }
 
-static int hdmi_init_regulator(struct omap_hdmi *hdmi)
-{
-	struct regulator *reg;
-
-	if (hdmi->vdda_reg != NULL)
-		return 0;
-
-	reg = devm_regulator_get(&hdmi->pdev->dev, "vdda");
-	if (IS_ERR(reg)) {
-		DSSERR("can't get VDDA regulator\n");
-		return PTR_ERR(reg);
-	}
-
-	hdmi->vdda_reg = reg;
-
-	return 0;
-}
-
 static int hdmi_power_on_core(struct omap_hdmi *hdmi)
 {
 	int r;
@@ -171,7 +153,7 @@
 static int hdmi_power_on_full(struct omap_hdmi *hdmi)
 {
 	int r;
-	struct videomode *vm;
+	const struct videomode *vm;
 	struct dss_pll_clock_info hdmi_cinfo = { 0 };
 	unsigned int pc;
 
@@ -224,9 +206,6 @@
 
 	hdmi5_configure(&hdmi->core, &hdmi->wp, &hdmi->cfg);
 
-	/* tv size */
-	dss_mgr_set_timings(&hdmi->output, vm);
-
 	r = dss_mgr_enable(&hdmi->output);
 	if (r)
 		goto err_mgr_enable;
@@ -268,39 +247,20 @@
 	hdmi_power_off_core(hdmi);
 }
 
-static int hdmi_display_check_timing(struct omap_dss_device *dssdev,
-				     struct videomode *vm)
-{
-	struct omap_hdmi *hdmi = dssdev_to_hdmi(dssdev);
-
-	if (!dispc_mgr_timings_ok(hdmi->dss->dispc, dssdev->dispc_channel, vm))
-		return -EINVAL;
-
-	return 0;
-}
-
-static void hdmi_display_set_timing(struct omap_dss_device *dssdev,
-				    struct videomode *vm)
+static void hdmi_display_set_timings(struct omap_dss_device *dssdev,
+				     const struct drm_display_mode *mode)
 {
 	struct omap_hdmi *hdmi = dssdev_to_hdmi(dssdev);
 
 	mutex_lock(&hdmi->lock);
 
-	hdmi->cfg.vm = *vm;
+	drm_display_mode_to_videomode(mode, &hdmi->cfg.vm);
 
-	dispc_set_tv_pclk(hdmi->dss->dispc, vm->pixelclock);
+	dispc_set_tv_pclk(hdmi->dss->dispc, mode->clock * 1000);
 
 	mutex_unlock(&hdmi->lock);
 }
 
-static void hdmi_display_get_timings(struct omap_dss_device *dssdev,
-				     struct videomode *vm)
-{
-	struct omap_hdmi *hdmi = dssdev_to_hdmi(dssdev);
-
-	*vm = hdmi->cfg.vm;
-}
-
 static int hdmi_dump_regs(struct seq_file *s, void *p)
 {
 	struct omap_hdmi *hdmi = s->private;
@@ -360,26 +320,20 @@
 	REG_FLD_MOD(hd->wp.base, HDMI_WP_SYSCONFIG, hd->wp_idlemode, 3, 2);
 }
 
-static int hdmi_display_enable(struct omap_dss_device *dssdev)
+static void hdmi_display_enable(struct omap_dss_device *dssdev)
 {
 	struct omap_hdmi *hdmi = dssdev_to_hdmi(dssdev);
 	unsigned long flags;
-	int r = 0;
+	int r;
 
 	DSSDBG("ENTER hdmi_display_enable\n");
 
 	mutex_lock(&hdmi->lock);
 
-	if (!dssdev->dispc_channel_connected) {
-		DSSERR("failed to enable display: no output/manager\n");
-		r = -ENODEV;
-		goto err0;
-	}
-
 	r = hdmi_power_on_full(hdmi);
 	if (r) {
 		DSSERR("failed to power on device\n");
-		goto err0;
+		goto done;
 	}
 
 	if (hdmi->audio_configured) {
@@ -399,12 +353,8 @@
 	hdmi->display_enabled = true;
 	spin_unlock_irqrestore(&hdmi->audio_playing_lock, flags);
 
+done:
 	mutex_unlock(&hdmi->lock);
-	return 0;
-
-err0:
-	mutex_unlock(&hdmi->lock);
-	return r;
 }
 
 static void hdmi_display_disable(struct omap_dss_device *dssdev)
@@ -459,44 +409,16 @@
 	mutex_unlock(&hdmi->lock);
 }
 
-static int hdmi_connect(struct omap_dss_device *dssdev,
-		struct omap_dss_device *dst)
+static int hdmi_connect(struct omap_dss_device *src,
+			struct omap_dss_device *dst)
 {
-	struct omap_hdmi *hdmi = dssdev_to_hdmi(dssdev);
-	int r;
-
-	r = hdmi_init_regulator(hdmi);
-	if (r)
-		return r;
-
-	r = dss_mgr_connect(&hdmi->output, dssdev);
-	if (r)
-		return r;
-
-	r = omapdss_output_set_device(dssdev, dst);
-	if (r) {
-		DSSERR("failed to connect output to new device: %s\n",
-				dst->name);
-		dss_mgr_disconnect(&hdmi->output, dssdev);
-		return r;
-	}
-
-	return 0;
+	return omapdss_device_connect(dst->dss, dst, dst->next);
 }
 
-static void hdmi_disconnect(struct omap_dss_device *dssdev,
-		struct omap_dss_device *dst)
+static void hdmi_disconnect(struct omap_dss_device *src,
+			    struct omap_dss_device *dst)
 {
-	struct omap_hdmi *hdmi = dssdev_to_hdmi(dssdev);
-
-	WARN_ON(dst != dssdev->dst);
-
-	if (dst != dssdev->dst)
-		return;
-
-	omapdss_output_unset_device(dssdev);
-
-	dss_mgr_disconnect(&hdmi->output, dssdev);
+	omapdss_device_disconnect(dst, dst->next);
 }
 
 static int hdmi_read_edid(struct omap_dss_device *dssdev,
@@ -540,68 +462,27 @@
 	return 0;
 }
 
-static const struct omapdss_hdmi_ops hdmi_ops = {
+static const struct omap_dss_device_ops hdmi_ops = {
 	.connect		= hdmi_connect,
 	.disconnect		= hdmi_disconnect,
 
 	.enable			= hdmi_display_enable,
 	.disable		= hdmi_display_disable,
 
-	.check_timings		= hdmi_display_check_timing,
-	.set_timings		= hdmi_display_set_timing,
-	.get_timings		= hdmi_display_get_timings,
+	.set_timings		= hdmi_display_set_timings,
 
 	.read_edid		= hdmi_read_edid,
-	.set_infoframe		= hdmi_set_infoframe,
-	.set_hdmi_mode		= hdmi_set_hdmi_mode,
-};
-
-static void hdmi_init_output(struct omap_hdmi *hdmi)
-{
-	struct omap_dss_device *out = &hdmi->output;
-
-	out->dev = &hdmi->pdev->dev;
-	out->id = OMAP_DSS_OUTPUT_HDMI;
-	out->output_type = OMAP_DISPLAY_TYPE_HDMI;
-	out->name = "hdmi.0";
-	out->dispc_channel = OMAP_DSS_CHANNEL_DIGIT;
-	out->ops.hdmi = &hdmi_ops;
-	out->owner = THIS_MODULE;
-
-	omapdss_register_output(out);
-}
-
-static void hdmi_uninit_output(struct omap_hdmi *hdmi)
-{
-	struct omap_dss_device *out = &hdmi->output;
-
-	omapdss_unregister_output(out);
-}
-
-static int hdmi_probe_of(struct omap_hdmi *hdmi)
-{
-	struct platform_device *pdev = hdmi->pdev;
-	struct device_node *node = pdev->dev.of_node;
-	struct device_node *ep;
-	int r;
-
-	ep = of_graph_get_endpoint_by_regs(node, 0, 0);
-	if (!ep)
-		return 0;
 
-	r = hdmi_parse_lanes_of(pdev, ep, &hdmi->phy);
-	if (r)
-		goto err;
-
-	of_node_put(ep);
-	return 0;
+	.hdmi = {
+		.set_infoframe		= hdmi_set_infoframe,
+		.set_hdmi_mode		= hdmi_set_hdmi_mode,
+	},
+};
 
-err:
-	of_node_put(ep);
-	return r;
-}
+/* -----------------------------------------------------------------------------
+ * Audio Callbacks
+ */
 
-/* Audio callbacks */
 static int hdmi_audio_startup(struct device *dev,
 			      void (*abort_cb)(struct device *dev))
 {
@@ -722,27 +603,125 @@
 	return 0;
 }
 
-/* HDMI HW IP initialisation */
+/* -----------------------------------------------------------------------------
+ * Component Bind & Unbind
+ */
+
 static int hdmi5_bind(struct device *dev, struct device *master, void *data)
 {
-	struct platform_device *pdev = to_platform_device(dev);
 	struct dss_device *dss = dss_get_device(master);
-	struct omap_hdmi *hdmi;
+	struct omap_hdmi *hdmi = dev_get_drvdata(dev);
 	int r;
+
+	hdmi->dss = dss;
+
+	r = hdmi_pll_init(dss, hdmi->pdev, &hdmi->pll, &hdmi->wp);
+	if (r)
+		return r;
+
+	r = hdmi_audio_register(hdmi);
+	if (r) {
+		DSSERR("Registering HDMI audio failed %d\n", r);
+		goto err_pll_uninit;
+	}
+
+	hdmi->debugfs = dss_debugfs_create_file(dss, "hdmi", hdmi_dump_regs,
+						hdmi);
+
+	return 0;
+
+err_pll_uninit:
+	hdmi_pll_uninit(&hdmi->pll);
+	return r;
+}
+
+static void hdmi5_unbind(struct device *dev, struct device *master, void *data)
+{
+	struct omap_hdmi *hdmi = dev_get_drvdata(dev);
+
+	dss_debugfs_remove_file(hdmi->debugfs);
+
+	if (hdmi->audio_pdev)
+		platform_device_unregister(hdmi->audio_pdev);
+
+	hdmi_pll_uninit(&hdmi->pll);
+}
+
+static const struct component_ops hdmi5_component_ops = {
+	.bind	= hdmi5_bind,
+	.unbind	= hdmi5_unbind,
+};
+
+/* -----------------------------------------------------------------------------
+ * Probe & Remove, Suspend & Resume
+ */
+
+static int hdmi5_init_output(struct omap_hdmi *hdmi)
+{
+	struct omap_dss_device *out = &hdmi->output;
+	int r;
+
+	out->dev = &hdmi->pdev->dev;
+	out->id = OMAP_DSS_OUTPUT_HDMI;
+	out->type = OMAP_DISPLAY_TYPE_HDMI;
+	out->name = "hdmi.0";
+	out->dispc_channel = OMAP_DSS_CHANNEL_DIGIT;
+	out->ops = &hdmi_ops;
+	out->owner = THIS_MODULE;
+	out->of_ports = BIT(0);
+	out->ops_flags = OMAP_DSS_DEVICE_OP_EDID;
+
+	r = omapdss_device_init_output(out);
+	if (r < 0)
+		return r;
+
+	omapdss_device_register(out);
+
+	return 0;
+}
+
+static void hdmi5_uninit_output(struct omap_hdmi *hdmi)
+{
+	struct omap_dss_device *out = &hdmi->output;
+
+	omapdss_device_unregister(out);
+	omapdss_device_cleanup_output(out);
+}
+
+static int hdmi5_probe_of(struct omap_hdmi *hdmi)
+{
+	struct platform_device *pdev = hdmi->pdev;
+	struct device_node *node = pdev->dev.of_node;
+	struct device_node *ep;
+	int r;
+
+	ep = of_graph_get_endpoint_by_regs(node, 0, 0);
+	if (!ep)
+		return 0;
+
+	r = hdmi_parse_lanes_of(pdev, ep, &hdmi->phy);
+	of_node_put(ep);
+	return r;
+}
+
+static int hdmi5_probe(struct platform_device *pdev)
+{
+	struct omap_hdmi *hdmi;
 	int irq;
+	int r;
 
 	hdmi = kzalloc(sizeof(*hdmi), GFP_KERNEL);
 	if (!hdmi)
 		return -ENOMEM;
 
 	hdmi->pdev = pdev;
-	hdmi->dss = dss;
+
 	dev_set_drvdata(&pdev->dev, hdmi);
 
 	mutex_init(&hdmi->lock);
 	spin_lock_init(&hdmi->audio_playing_lock);
 
-	r = hdmi_probe_of(hdmi);
+	r = hdmi5_probe_of(hdmi);
 	if (r)
 		goto err_free;
 
@@ -750,23 +729,19 @@
 	if (r)
 		goto err_free;
 
-	r = hdmi_pll_init(dss, pdev, &hdmi->pll, &hdmi->wp);
-	if (r)
-		goto err_free;
-
 	r = hdmi_phy_init(pdev, &hdmi->phy, 5);
 	if (r)
-		goto err_pll;
+		goto err_free;
 
 	r = hdmi5_core_init(pdev, &hdmi->core);
 	if (r)
-		goto err_pll;
+		goto err_free;
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0) {
 		DSSERR("platform_get_irq failed\n");
 		r = -ENODEV;
-		goto err_pll;
+		goto err_free;
 	}
 
 	r = devm_request_threaded_irq(&pdev->dev, irq,
@@ -774,93 +749,52 @@
 			IRQF_ONESHOT, "OMAP HDMI", hdmi);
 	if (r) {
 		DSSERR("HDMI IRQ request failed\n");
-		goto err_pll;
+		goto err_free;
 	}
 
-	pm_runtime_enable(&pdev->dev);
+	hdmi->vdda_reg = devm_regulator_get(&pdev->dev, "vdda");
+	if (IS_ERR(hdmi->vdda_reg)) {
+		r = PTR_ERR(hdmi->vdda_reg);
+		if (r != -EPROBE_DEFER)
+			DSSERR("can't get VDDA regulator\n");
+		goto err_free;
+	}
 
-	hdmi_init_output(hdmi);
+	pm_runtime_enable(&pdev->dev);
 
-	r = hdmi_audio_register(hdmi);
-	if (r) {
-		DSSERR("Registering HDMI audio failed %d\n", r);
-		hdmi_uninit_output(hdmi);
-		pm_runtime_disable(&pdev->dev);
-		return r;
-	}
+	r = hdmi5_init_output(hdmi);
+	if (r)
+		goto err_pm_disable;
 
-	hdmi->debugfs = dss_debugfs_create_file(dss, "hdmi", hdmi_dump_regs,
-						hdmi);
+	r = component_add(&pdev->dev, &hdmi5_component_ops);
+	if (r)
+		goto err_uninit_output;
 
 	return 0;
 
-err_pll:
-	hdmi_pll_uninit(&hdmi->pll);
+err_uninit_output:
+	hdmi5_uninit_output(hdmi);
+err_pm_disable:
+	pm_runtime_disable(&pdev->dev);
 err_free:
 	kfree(hdmi);
 	return r;
 }
 
-static void hdmi5_unbind(struct device *dev, struct device *master, void *data)
-{
-	struct omap_hdmi *hdmi = dev_get_drvdata(dev);
-
-	dss_debugfs_remove_file(hdmi->debugfs);
-
-	if (hdmi->audio_pdev)
-		platform_device_unregister(hdmi->audio_pdev);
-
-	hdmi_uninit_output(hdmi);
-
-	hdmi_pll_uninit(&hdmi->pll);
-
-	pm_runtime_disable(dev);
-
-	kfree(hdmi);
-}
-
-static const struct component_ops hdmi5_component_ops = {
-	.bind	= hdmi5_bind,
-	.unbind	= hdmi5_unbind,
-};
-
-static int hdmi5_probe(struct platform_device *pdev)
-{
-	return component_add(&pdev->dev, &hdmi5_component_ops);
-}
-
 static int hdmi5_remove(struct platform_device *pdev)
 {
-	component_del(&pdev->dev, &hdmi5_component_ops);
-	return 0;
-}
+	struct omap_hdmi *hdmi = platform_get_drvdata(pdev);
 
-static int hdmi_runtime_suspend(struct device *dev)
-{
-	struct omap_hdmi *hdmi = dev_get_drvdata(dev);
-
-	dispc_runtime_put(hdmi->dss->dispc);
-
-	return 0;
-}
+	component_del(&pdev->dev, &hdmi5_component_ops);
 
-static int hdmi_runtime_resume(struct device *dev)
-{
-	struct omap_hdmi *hdmi = dev_get_drvdata(dev);
-	int r;
+	hdmi5_uninit_output(hdmi);
 
-	r = dispc_runtime_get(hdmi->dss->dispc);
-	if (r < 0)
-		return r;
+	pm_runtime_disable(&pdev->dev);
 
+	kfree(hdmi);
 	return 0;
 }
 
-static const struct dev_pm_ops hdmi_pm_ops = {
-	.runtime_suspend = hdmi_runtime_suspend,
-	.runtime_resume = hdmi_runtime_resume,
-};
-
 static const struct of_device_id hdmi_of_match[] = {
 	{ .compatible = "ti,omap5-hdmi", },
 	{ .compatible = "ti,dra7-hdmi", },
@@ -872,7 +806,6 @@
 	.remove		= hdmi5_remove,
 	.driver         = {
 		.name   = "omapdss_hdmi5",
-		.pm	= &hdmi_pm_ops,
 		.of_match_table = hdmi_of_match,
 		.suppress_bind_attrs = true,
 	},
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/dss/hdmi5_core.c b/drivers/gpu/drm/omapdrm/dss/hdmi5_core.c
--- a/drivers/gpu/drm/omapdrm/dss/hdmi5_core.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi5_core.c	2020-06-19 21:00:31.412227715 -0400
@@ -34,24 +34,12 @@
 
 #include "hdmi5_core.h"
 
-/* only 24 bit color depth used for now */
-static const struct csc_table csc_table_deepcolor[] = {
-	/* HDMI_DEEP_COLOR_24BIT */
-	[0] = { 7036, 0, 0, 32, 0, 7036, 0, 32, 0, 0, 7036, 32, },
-	/* HDMI_DEEP_COLOR_30BIT */
-	[1] = { 7015, 0, 0, 128, 0, 7015, 0, 128, 0, 0, 7015, 128, },
-	/* HDMI_DEEP_COLOR_36BIT */
-	[2] = { 7010, 0, 0, 512, 0, 7010, 0, 512, 0, 0, 7010, 512, },
-	/* FULL RANGE */
-	[3] = { 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, },
-};
-
 static void hdmi_core_ddc_init(struct hdmi_core_data *core)
 {
 	void __iomem *base = core->base;
 	const unsigned long long iclk = 266000000;	/* DSS L3 ICLK */
-	const unsigned int ss_scl_high = 4600;		/* ns */
-	const unsigned int ss_scl_low = 5400;		/* ns */
+	const unsigned int ss_scl_high = 4700;		/* ns */
+	const unsigned int ss_scl_low = 5500;		/* ns */
 	const unsigned int fs_scl_high = 600;		/* ns */
 	const unsigned int fs_scl_low = 1300;		/* ns */
 	const unsigned int sda_hold = 1000;		/* ns */
@@ -287,7 +275,7 @@
 }
 
 static void hdmi_core_init(struct hdmi_core_vid_config *video_cfg,
-			struct hdmi_config *cfg)
+			   const struct hdmi_config *cfg)
 {
 	DSSDBG("hdmi_core_init\n");
 
@@ -325,10 +313,10 @@
 
 /* DSS_HDMI_CORE_VIDEO_CONFIG */
 static void hdmi_core_video_config(struct hdmi_core_data *core,
-			struct hdmi_core_vid_config *cfg)
+			const struct hdmi_core_vid_config *cfg)
 {
 	void __iomem *base = core->base;
-	struct videomode *vm = &cfg->v_fc_config.vm;
+	const struct videomode *vm = &cfg->v_fc_config.vm;
 	unsigned char r = 0;
 	bool vsync_pol, hsync_pol;
 
@@ -408,14 +396,6 @@
 	REG_FLD_MOD(base, HDMI_CORE_VP_CONF, clr_depth ? 0 : 2, 1, 0);
 }
 
-static void hdmi_core_config_csc(struct hdmi_core_data *core)
-{
-	int clr_depth = 0;	/* 24 bit color depth */
-
-	/* CSC_COLORDEPTH */
-	REG_FLD_MOD(core->base, HDMI_CORE_CSC_SCALE, clr_depth, 7, 4);
-}
-
 static void hdmi_core_config_video_sampler(struct hdmi_core_data *core)
 {
 	int video_mapping = 1;	/* for 24 bit color depth */
@@ -480,47 +460,67 @@
 	REG_FLD_MOD(base, HDMI_CORE_FC_PRCONF, pr, 3, 0);
 }
 
-static void hdmi_core_csc_config(struct hdmi_core_data *core,
-		struct csc_table csc_coeff)
+static void hdmi_core_write_csc(struct hdmi_core_data *core,
+		const struct csc_table *csc_coeff)
 {
 	void __iomem *base = core->base;
 
-	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_A1_MSB, csc_coeff.a1 >> 8 , 6, 0);
-	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_A1_LSB, csc_coeff.a1, 7, 0);
-	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_A2_MSB, csc_coeff.a2 >> 8, 6, 0);
-	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_A2_LSB, csc_coeff.a2, 7, 0);
-	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_A3_MSB, csc_coeff.a3 >> 8, 6, 0);
-	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_A3_LSB, csc_coeff.a3, 7, 0);
-	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_A4_MSB, csc_coeff.a4 >> 8, 6, 0);
-	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_A4_LSB, csc_coeff.a4, 7, 0);
-	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_B1_MSB, csc_coeff.b1 >> 8, 6, 0);
-	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_B1_LSB, csc_coeff.b1, 7, 0);
-	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_B2_MSB, csc_coeff.b2 >> 8, 6, 0);
-	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_B2_LSB, csc_coeff.b2, 7, 0);
-	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_B3_MSB, csc_coeff.b3 >> 8, 6, 0);
-	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_B3_LSB, csc_coeff.b3, 7, 0);
-	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_B4_MSB, csc_coeff.b4 >> 8, 6, 0);
-	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_B4_LSB, csc_coeff.b4, 7, 0);
-	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_C1_MSB, csc_coeff.c1 >> 8, 6, 0);
-	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_C1_LSB, csc_coeff.c1, 7, 0);
-	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_C2_MSB, csc_coeff.c2 >> 8, 6, 0);
-	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_C2_LSB, csc_coeff.c2, 7, 0);
-	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_C3_MSB, csc_coeff.c3 >> 8, 6, 0);
-	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_C3_LSB, csc_coeff.c3, 7, 0);
-	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_C4_MSB, csc_coeff.c4 >> 8, 6, 0);
-	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_C4_LSB, csc_coeff.c4, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_A1_MSB, csc_coeff->a1 >> 8 , 6, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_A1_LSB, csc_coeff->a1, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_A2_MSB, csc_coeff->a2 >> 8, 6, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_A2_LSB, csc_coeff->a2, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_A3_MSB, csc_coeff->a3 >> 8, 6, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_A3_LSB, csc_coeff->a3, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_A4_MSB, csc_coeff->a4 >> 8, 6, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_A4_LSB, csc_coeff->a4, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_B1_MSB, csc_coeff->b1 >> 8, 6, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_B1_LSB, csc_coeff->b1, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_B2_MSB, csc_coeff->b2 >> 8, 6, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_B2_LSB, csc_coeff->b2, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_B3_MSB, csc_coeff->b3 >> 8, 6, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_B3_LSB, csc_coeff->b3, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_B4_MSB, csc_coeff->b4 >> 8, 6, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_B4_LSB, csc_coeff->b4, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_C1_MSB, csc_coeff->c1 >> 8, 6, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_C1_LSB, csc_coeff->c1, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_C2_MSB, csc_coeff->c2 >> 8, 6, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_C2_LSB, csc_coeff->c2, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_C3_MSB, csc_coeff->c3 >> 8, 6, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_C3_LSB, csc_coeff->c3, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_C4_MSB, csc_coeff->c4 >> 8, 6, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_C4_LSB, csc_coeff->c4, 7, 0);
 
+	/* enable CSC */
 	REG_FLD_MOD(base, HDMI_CORE_MC_FLOWCTRL, 0x1, 0, 0);
 }
 
-static void hdmi_core_configure_range(struct hdmi_core_data *core)
+static void hdmi_core_configure_range(struct hdmi_core_data *core,
+                                      enum hdmi_quantization_range range)
 {
-	struct csc_table csc_coeff = { 0 };
-
-	/* support limited range with 24 bit color depth for now */
-	csc_coeff = csc_table_deepcolor[0];
+	static const struct csc_table csc_limited_range = {
+		7036, 0, 0, 32, 0, 7036, 0, 32, 0, 0, 7036, 32
+	};
+	static const struct csc_table csc_full_range = {
+		8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0
+	};
+	const struct csc_table *csc_coeff;
+
+	/* CSC_COLORDEPTH  = 24 bits*/
+	REG_FLD_MOD(core->base, HDMI_CORE_CSC_SCALE, 0, 7, 4);
+
+	switch (range) {
+		case HDMI_QUANTIZATION_RANGE_FULL:
+			csc_coeff = &csc_full_range;
+			break;
+
+		case HDMI_QUANTIZATION_RANGE_DEFAULT:
+		case HDMI_QUANTIZATION_RANGE_LIMITED:
+		default:
+			csc_coeff = &csc_limited_range;
+			break;
+	}
 
-	hdmi_core_csc_config(core, csc_coeff);
+	hdmi_core_write_csc(core, csc_coeff);
 }
 
 static void hdmi_core_enable_video_path(struct hdmi_core_data *core)
@@ -611,9 +611,20 @@
 	struct videomode vm;
 	struct hdmi_video_format video_format;
 	struct hdmi_core_vid_config v_core_cfg;
+	enum hdmi_quantization_range range;
 
 	hdmi_core_mask_interrupts(core);
 
+	if (cfg->hdmi_dvi_mode == HDMI_HDMI) {
+		char vic = cfg->infoframe.video_code;
+
+		/* All CEA modes other than VIC 1 use limited quantization range. */
+		range = vic > 1 ? HDMI_QUANTIZATION_RANGE_LIMITED:
+			HDMI_QUANTIZATION_RANGE_FULL;
+	} else {
+		range = HDMI_QUANTIZATION_RANGE_FULL;
+	}
+
 	hdmi_core_init(&v_core_cfg, cfg);
 
 	hdmi_wp_init_vid_fmt_timings(&video_format, &vm, cfg);
@@ -627,9 +638,8 @@
 
 	hdmi_wp_video_config_interface(wp, &vm);
 
-	/* support limited range with 24 bit color depth for now */
-	hdmi_core_configure_range(core);
-	cfg->infoframe.quantization_range = HDMI_QUANTIZATION_RANGE_LIMITED;
+	hdmi_core_configure_range(core, range);
+	cfg->infoframe.quantization_range = range;
 
 	/*
 	 * configure core video part, set software reset in the core
@@ -639,7 +649,6 @@
 	hdmi_core_video_config(core, &v_core_cfg);
 
 	hdmi_core_config_video_packetizer(core);
-	hdmi_core_config_csc(core);
 	hdmi_core_config_video_sampler(core);
 
 	if (cfg->hdmi_dvi_mode == HDMI_HDMI)
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/dss/hdmi.h b/drivers/gpu/drm/omapdrm/dss/hdmi.h
--- a/drivers/gpu/drm/omapdrm/dss/hdmi.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi.h	2020-06-19 21:00:31.412227715 -0400
@@ -313,13 +313,13 @@
 int hdmi_wp_set_phy_pwr(struct hdmi_wp_data *wp, enum hdmi_phy_pwr val);
 int hdmi_wp_set_pll_pwr(struct hdmi_wp_data *wp, enum hdmi_pll_pwr val);
 void hdmi_wp_video_config_format(struct hdmi_wp_data *wp,
-		struct hdmi_video_format *video_fmt);
+		const struct hdmi_video_format *video_fmt);
 void hdmi_wp_video_config_interface(struct hdmi_wp_data *wp,
-		struct videomode *vm);
+		const struct videomode *vm);
 void hdmi_wp_video_config_timing(struct hdmi_wp_data *wp,
-		struct videomode *vm);
+		const struct videomode *vm);
 void hdmi_wp_init_vid_fmt_timings(struct hdmi_video_format *video_fmt,
-		struct videomode *vm, struct hdmi_config *param);
+		struct videomode *vm, const struct hdmi_config *param);
 int hdmi_wp_init(struct platform_device *pdev, struct hdmi_wp_data *wp,
 		 unsigned int version);
 phys_addr_t hdmi_wp_get_audio_dma_addr(struct hdmi_wp_data *wp);
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/dss/hdmi_wp.c b/drivers/gpu/drm/omapdrm/dss/hdmi_wp.c
--- a/drivers/gpu/drm/omapdrm/dss/hdmi_wp.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi_wp.c	2020-06-19 21:00:31.412227715 -0400
@@ -131,7 +131,7 @@
 }
 
 void hdmi_wp_video_config_format(struct hdmi_wp_data *wp,
-		struct hdmi_video_format *video_fmt)
+		const struct hdmi_video_format *video_fmt)
 {
 	u32 l = 0;
 
@@ -144,7 +144,7 @@
 }
 
 void hdmi_wp_video_config_interface(struct hdmi_wp_data *wp,
-				    struct videomode *vm)
+				    const struct videomode *vm)
 {
 	u32 r;
 	bool vsync_inv, hsync_inv;
@@ -164,7 +164,7 @@
 }
 
 void hdmi_wp_video_config_timing(struct hdmi_wp_data *wp,
-				 struct videomode *vm)
+				 const struct videomode *vm)
 {
 	u32 timing_h = 0;
 	u32 timing_v = 0;
@@ -193,7 +193,7 @@
 }
 
 void hdmi_wp_init_vid_fmt_timings(struct hdmi_video_format *video_fmt,
-		struct videomode *vm, struct hdmi_config *param)
+		struct videomode *vm, const struct hdmi_config *param)
 {
 	DSSDBG("Enter hdmi_wp_video_init_format\n");
 
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/dss/Makefile b/drivers/gpu/drm/omapdrm/dss/Makefile
--- a/drivers/gpu/drm/omapdrm/dss/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/dss/Makefile	2020-06-19 21:00:31.412227715 -0400
@@ -6,7 +6,7 @@
 
 obj-$(CONFIG_OMAP2_DSS) += omapdss.o
 # Core DSS files
-omapdss-y := core.o dss.o dispc.o dispc_coefs.o \
+omapdss-y := dss.o dispc.o dispc_coefs.o \
 	pll.o video-pll.o
 omapdss-$(CONFIG_OMAP2_DSS_DPI) += dpi.o
 omapdss-$(CONFIG_OMAP2_DSS_VENC) += venc.o
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/dss/omapdss-boot-init.c b/drivers/gpu/drm/omapdrm/dss/omapdss-boot-init.c
--- a/drivers/gpu/drm/omapdrm/dss/omapdss-boot-init.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/dss/omapdss-boot-init.c	2020-06-19 21:00:31.412227715 -0400
@@ -184,6 +184,23 @@
 	{},
 };
 
+static const struct of_device_id omapdss_of_fixups_whitelist[] __initconst = {
+	{ .compatible = "composite-video-connector" },
+	{ .compatible = "hdmi-connector" },
+	{ .compatible = "lgphilips,lb035q02" },
+	{ .compatible = "nec,nl8048hl11" },
+	{ .compatible = "panel-dsi-cm" },
+	{ .compatible = "sharp,ls037v7dw01" },
+	{ .compatible = "sony,acx565akm" },
+	{ .compatible = "svideo-connector" },
+	{ .compatible = "ti,opa362" },
+	{ .compatible = "ti,tpd12s015" },
+	{ .compatible = "ti,dra7evm-tpd12s015" },
+	{ .compatible = "toppoly,td028ttec1" },
+	{ .compatible = "tpo,td028ttec1" },
+	{ .compatible = "tpo,td043mtea1" },
+};
+
 static int __init omapdss_boot_init(void)
 {
 	struct device_node *dss, *child;
@@ -210,7 +227,7 @@
 		n = list_first_entry(&dss_conv_list, struct dss_conv_node,
 			list);
 
-		if (!n->root)
+		if (of_match_node(omapdss_of_fixups_whitelist, n->node))
 			omapdss_omapify_node(n->node);
 
 		list_del(&n->list);
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/dss/omapdss.h b/drivers/gpu/drm/omapdrm/dss/omapdss.h
--- a/drivers/gpu/drm/omapdrm/dss/omapdss.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/dss/omapdss.h	2020-06-19 21:00:31.412227715 -0400
@@ -19,13 +19,13 @@
 #define __OMAP_DRM_DSS_H
 
 #include <linux/list.h>
-#include <linux/kobject.h>
 #include <linux/device.h>
 #include <linux/interrupt.h>
 #include <video/videomode.h>
 #include <linux/platform_data/omapdss.h>
 #include <uapi/drm/drm_mode.h>
 #include <drm/drm_crtc.h>
+#include <drm/drm_color_mgmt.h>
 
 #define DISPC_IRQ_FRAMEDONE		(1 << 0)
 #define DISPC_IRQ_VSYNC			(1 << 1)
@@ -68,6 +68,7 @@
 struct snd_aes_iec958;
 struct snd_cea_861_aud_if;
 struct hdmi_avi_infoframe;
+struct drm_connector;
 
 enum omap_display_type {
 	OMAP_DISPLAY_TYPE_NONE		= 0,
@@ -254,6 +255,9 @@
 	u8 global_alpha;
 	u8 pre_mult_alpha;
 	u8 zorder;
+
+	enum drm_color_encoding color_encoding;
+	enum drm_color_range color_range;
 };
 
 struct omap_overlay_manager_info {
@@ -263,7 +267,7 @@
 	u32 trans_key;
 	bool trans_enabled;
 
-	bool partial_alpha_enabled;
+	bool alpha_blender_enabled;
 
 	bool cpr_enable;
 	struct omap_dss_cpr_coefs cpr_coefs;
@@ -296,117 +300,14 @@
 	u8 pre_mult_alpha;
 };
 
-struct omapdss_dpi_ops {
-	int (*connect)(struct omap_dss_device *dssdev,
-			struct omap_dss_device *dst);
-	void (*disconnect)(struct omap_dss_device *dssdev,
-			struct omap_dss_device *dst);
-
-	int (*enable)(struct omap_dss_device *dssdev);
-	void (*disable)(struct omap_dss_device *dssdev);
-
-	int (*check_timings)(struct omap_dss_device *dssdev,
-			     struct videomode *vm);
-	void (*set_timings)(struct omap_dss_device *dssdev,
-			    struct videomode *vm);
-	void (*get_timings)(struct omap_dss_device *dssdev,
-			    struct videomode *vm);
-};
-
-struct omapdss_sdi_ops {
-	int (*connect)(struct omap_dss_device *dssdev,
-			struct omap_dss_device *dst);
-	void (*disconnect)(struct omap_dss_device *dssdev,
-			struct omap_dss_device *dst);
-
-	int (*enable)(struct omap_dss_device *dssdev);
-	void (*disable)(struct omap_dss_device *dssdev);
-
-	int (*check_timings)(struct omap_dss_device *dssdev,
-			     struct videomode *vm);
-	void (*set_timings)(struct omap_dss_device *dssdev,
-			    struct videomode *vm);
-	void (*get_timings)(struct omap_dss_device *dssdev,
-			    struct videomode *vm);
-};
-
-struct omapdss_dvi_ops {
-	int (*connect)(struct omap_dss_device *dssdev,
-			struct omap_dss_device *dst);
-	void (*disconnect)(struct omap_dss_device *dssdev,
-			struct omap_dss_device *dst);
-
-	int (*enable)(struct omap_dss_device *dssdev);
-	void (*disable)(struct omap_dss_device *dssdev);
-
-	int (*check_timings)(struct omap_dss_device *dssdev,
-			     struct videomode *vm);
-	void (*set_timings)(struct omap_dss_device *dssdev,
-			    struct videomode *vm);
-	void (*get_timings)(struct omap_dss_device *dssdev,
-			    struct videomode *vm);
-};
-
-struct omapdss_atv_ops {
-	int (*connect)(struct omap_dss_device *dssdev,
-			struct omap_dss_device *dst);
-	void (*disconnect)(struct omap_dss_device *dssdev,
-			struct omap_dss_device *dst);
-
-	int (*enable)(struct omap_dss_device *dssdev);
-	void (*disable)(struct omap_dss_device *dssdev);
-
-	int (*check_timings)(struct omap_dss_device *dssdev,
-			     struct videomode *vm);
-	void (*set_timings)(struct omap_dss_device *dssdev,
-			    struct videomode *vm);
-	void (*get_timings)(struct omap_dss_device *dssdev,
-			    struct videomode *vm);
-
-	int (*set_wss)(struct omap_dss_device *dssdev, u32 wss);
-	u32 (*get_wss)(struct omap_dss_device *dssdev);
-};
-
 struct omapdss_hdmi_ops {
-	int (*connect)(struct omap_dss_device *dssdev,
-			struct omap_dss_device *dst);
-	void (*disconnect)(struct omap_dss_device *dssdev,
-			struct omap_dss_device *dst);
-
-	int (*enable)(struct omap_dss_device *dssdev);
-	void (*disable)(struct omap_dss_device *dssdev);
-
-	int (*check_timings)(struct omap_dss_device *dssdev,
-			     struct videomode *vm);
-	void (*set_timings)(struct omap_dss_device *dssdev,
-			    struct videomode *vm);
-	void (*get_timings)(struct omap_dss_device *dssdev,
-			    struct videomode *vm);
-
-	int (*read_edid)(struct omap_dss_device *dssdev, u8 *buf, int len);
 	void (*lost_hotplug)(struct omap_dss_device *dssdev);
-	bool (*detect)(struct omap_dss_device *dssdev);
-
-	int (*register_hpd_cb)(struct omap_dss_device *dssdev,
-			       void (*cb)(void *cb_data,
-					  enum drm_connector_status status),
-			       void *cb_data);
-	void (*unregister_hpd_cb)(struct omap_dss_device *dssdev);
-	void (*enable_hpd)(struct omap_dss_device *dssdev);
-	void (*disable_hpd)(struct omap_dss_device *dssdev);
-
 	int (*set_hdmi_mode)(struct omap_dss_device *dssdev, bool hdmi_mode);
 	int (*set_infoframe)(struct omap_dss_device *dssdev,
 		const struct hdmi_avi_infoframe *avi);
 };
 
 struct omapdss_dsi_ops {
-	int (*connect)(struct omap_dss_device *dssdev,
-			struct omap_dss_device *dst);
-	void (*disconnect)(struct omap_dss_device *dssdev,
-			struct omap_dss_device *dst);
-
-	int (*enable)(struct omap_dss_device *dssdev);
 	void (*disable)(struct omap_dss_device *dssdev, bool disconnect_lanes,
 			bool enter_ulps);
 
@@ -457,78 +358,104 @@
 			int channel, u16 plen);
 };
 
+struct omap_dss_device_ops {
+	int (*connect)(struct omap_dss_device *dssdev,
+			struct omap_dss_device *dst);
+	void (*disconnect)(struct omap_dss_device *dssdev,
+			struct omap_dss_device *dst);
+
+	void (*pre_enable)(struct omap_dss_device *dssdev);
+	void (*enable)(struct omap_dss_device *dssdev);
+	void (*disable)(struct omap_dss_device *dssdev);
+	void (*post_disable)(struct omap_dss_device *dssdev);
+
+	int (*check_timings)(struct omap_dss_device *dssdev,
+			     struct drm_display_mode *mode);
+	void (*set_timings)(struct omap_dss_device *dssdev,
+			    const struct drm_display_mode *mode);
+
+	bool (*detect)(struct omap_dss_device *dssdev);
+
+	void (*register_hpd_cb)(struct omap_dss_device *dssdev,
+				void (*cb)(void *cb_data,
+					  enum drm_connector_status status),
+				void *cb_data);
+	void (*unregister_hpd_cb)(struct omap_dss_device *dssdev);
+
+	int (*read_edid)(struct omap_dss_device *dssdev, u8 *buf, int len);
+
+	int (*get_modes)(struct omap_dss_device *dssdev,
+			 struct drm_connector *connector);
+
+	union {
+		const struct omapdss_hdmi_ops hdmi;
+		const struct omapdss_dsi_ops dsi;
+	};
+};
+
+/**
+ * enum omap_dss_device_ops_flag - Indicates which device ops are supported
+ * @OMAP_DSS_DEVICE_OP_DETECT: The device supports output connection detection
+ * @OMAP_DSS_DEVICE_OP_HPD: The device supports all hot-plug-related operations
+ * @OMAP_DSS_DEVICE_OP_EDID: The device supports reading EDID
+ * @OMAP_DSS_DEVICE_OP_MODES: The device supports reading modes
+ */
+enum omap_dss_device_ops_flag {
+	OMAP_DSS_DEVICE_OP_DETECT = BIT(0),
+	OMAP_DSS_DEVICE_OP_HPD = BIT(1),
+	OMAP_DSS_DEVICE_OP_EDID = BIT(2),
+	OMAP_DSS_DEVICE_OP_MODES = BIT(3),
+};
+
 struct omap_dss_device {
-	struct kobject kobj;
 	struct device *dev;
 
 	struct module *owner;
 
-	struct list_head panel_list;
+	struct dss_device *dss;
+	struct omap_dss_device *next;
+	struct drm_bridge *bridge;
+	struct drm_panel *panel;
 
-	/* alias in the form of "display%d" */
-	char alias[16];
+	struct list_head list;
 
+	/*
+	 * DSS type that this device generates (for DSS internal devices) or
+	 * requires (for external encoders, connectors and panels). Must be a
+	 * non-zero (different than OMAP_DISPLAY_TYPE_NONE) value.
+	 */
 	enum omap_display_type type;
-	enum omap_display_type output_type;
-
-	struct {
-		struct videomode vm;
 
-		enum omap_dss_dsi_pixel_format dsi_pix_fmt;
-		enum omap_dss_dsi_mode dsi_mode;
-	} panel;
+	/*
+	 * True if the device is a display (panel or connector) at the end of
+	 * the pipeline, false otherwise.
+	 */
+	bool display;
 
 	const char *name;
 
-	struct omap_dss_driver *driver;
-
-	union {
-		const struct omapdss_dpi_ops *dpi;
-		const struct omapdss_sdi_ops *sdi;
-		const struct omapdss_dvi_ops *dvi;
-		const struct omapdss_hdmi_ops *hdmi;
-		const struct omapdss_atv_ops *atv;
-		const struct omapdss_dsi_ops *dsi;
-	} ops;
-
-	/* helper variable for driver suspend/resume */
-	bool activate_after_resume;
+	const struct omap_dss_driver *driver;
+	const struct omap_dss_device_ops *ops;
+	unsigned long ops_flags;
+	u32 bus_flags;
 
 	enum omap_display_caps caps;
 
-	struct omap_dss_device *src;
-
 	enum omap_dss_display_state state;
 
 	/* OMAP DSS output specific fields */
 
-	struct list_head list;
-
 	/* DISPC channel for this output */
 	enum omap_channel dispc_channel;
-	bool dispc_channel_connected;
 
 	/* output instance */
 	enum omap_dss_output_id id;
 
-	/* the port number in the DT node */
-	int port_num;
-
-	/* dynamic fields */
-	struct omap_dss_device *dst;
+	/* bitmask of port numbers in DT */
+	unsigned int of_ports;
 };
 
 struct omap_dss_driver {
-	int (*probe)(struct omap_dss_device *);
-	void (*remove)(struct omap_dss_device *);
-
-	int (*connect)(struct omap_dss_device *dssdev);
-	void (*disconnect)(struct omap_dss_device *dssdev);
-
-	int (*enable)(struct omap_dss_device *display);
-	void (*disable)(struct omap_dss_device *display);
-	int (*run_test)(struct omap_dss_device *display, int test);
-
 	int (*update)(struct omap_dss_device *dssdev,
 			       u16 x, u16 y, u16 w, u16 h);
 	int (*sync)(struct omap_dss_device *dssdev);
@@ -536,42 +463,9 @@
 	int (*enable_te)(struct omap_dss_device *dssdev, bool enable);
 	int (*get_te)(struct omap_dss_device *dssdev);
 
-	u8 (*get_rotate)(struct omap_dss_device *dssdev);
-	int (*set_rotate)(struct omap_dss_device *dssdev, u8 rotate);
-
-	bool (*get_mirror)(struct omap_dss_device *dssdev);
-	int (*set_mirror)(struct omap_dss_device *dssdev, bool enable);
-
 	int (*memory_read)(struct omap_dss_device *dssdev,
 			void *buf, size_t size,
 			u16 x, u16 y, u16 w, u16 h);
-
-	int (*check_timings)(struct omap_dss_device *dssdev,
-			     struct videomode *vm);
-	void (*set_timings)(struct omap_dss_device *dssdev,
-			    struct videomode *vm);
-	void (*get_timings)(struct omap_dss_device *dssdev,
-			    struct videomode *vm);
-	void (*get_size)(struct omap_dss_device *dssdev,
-			 unsigned int *width, unsigned int *height);
-
-	int (*set_wss)(struct omap_dss_device *dssdev, u32 wss);
-	u32 (*get_wss)(struct omap_dss_device *dssdev);
-
-	int (*read_edid)(struct omap_dss_device *dssdev, u8 *buf, int len);
-	bool (*detect)(struct omap_dss_device *dssdev);
-
-	int (*register_hpd_cb)(struct omap_dss_device *dssdev,
-			       void (*cb)(void *cb_data,
-					  enum drm_connector_status status),
-			       void *cb_data);
-	void (*unregister_hpd_cb)(struct omap_dss_device *dssdev);
-	void (*enable_hpd)(struct omap_dss_device *dssdev);
-	void (*disable_hpd)(struct omap_dss_device *dssdev);
-
-	int (*set_hdmi_mode)(struct omap_dss_device *dssdev, bool hdmi_mode);
-	int (*set_hdmi_infoframe)(struct omap_dss_device *dssdev,
-		const struct hdmi_avi_infoframe *avi);
 };
 
 struct dss_device *omapdss_get_dss(void);
@@ -581,27 +475,35 @@
 	return !!omapdss_get_dss();
 }
 
-int omapdss_register_display(struct omap_dss_device *dssdev);
-void omapdss_unregister_display(struct omap_dss_device *dssdev);
-
-struct omap_dss_device *omap_dss_get_device(struct omap_dss_device *dssdev);
-void omap_dss_put_device(struct omap_dss_device *dssdev);
-#define for_each_dss_dev(d) while ((d = omap_dss_get_next_device(d)) != NULL)
-struct omap_dss_device *omap_dss_get_next_device(struct omap_dss_device *from);
+void omapdss_display_init(struct omap_dss_device *dssdev);
+struct omap_dss_device *omapdss_display_get(struct omap_dss_device *output);
+int omapdss_display_get_modes(struct drm_connector *connector,
+			      const struct videomode *vm);
+
+void omapdss_device_register(struct omap_dss_device *dssdev);
+void omapdss_device_unregister(struct omap_dss_device *dssdev);
+struct omap_dss_device *omapdss_device_get(struct omap_dss_device *dssdev);
+void omapdss_device_put(struct omap_dss_device *dssdev);
+struct omap_dss_device *omapdss_find_device_by_node(struct device_node *node);
+int omapdss_device_connect(struct dss_device *dss,
+			   struct omap_dss_device *src,
+			   struct omap_dss_device *dst);
+void omapdss_device_disconnect(struct omap_dss_device *src,
+			       struct omap_dss_device *dst);
+void omapdss_device_pre_enable(struct omap_dss_device *dssdev);
+void omapdss_device_enable(struct omap_dss_device *dssdev);
+void omapdss_device_disable(struct omap_dss_device *dssdev);
+void omapdss_device_post_disable(struct omap_dss_device *dssdev);
 
 int omap_dss_get_num_overlay_managers(void);
 
 int omap_dss_get_num_overlays(void);
 
-int omapdss_register_output(struct omap_dss_device *output);
-void omapdss_unregister_output(struct omap_dss_device *output);
-struct omap_dss_device *omap_dss_get_output(enum omap_dss_output_id id);
-struct omap_dss_device *omap_dss_find_output_by_port_node(struct device_node *port);
-int omapdss_output_set_device(struct omap_dss_device *out,
-		struct omap_dss_device *dssdev);
-int omapdss_output_unset_device(struct omap_dss_device *out);
-
-struct omap_dss_device *omapdss_find_output_from_display(struct omap_dss_device *dssdev);
+#define for_each_dss_output(d) \
+	while ((d = omapdss_device_next_output(d)) != NULL)
+struct omap_dss_device *omapdss_device_next_output(struct omap_dss_device *from);
+int omapdss_device_init_output(struct omap_dss_device *out);
+void omapdss_device_cleanup_output(struct omap_dss_device *out);
 
 typedef void (*omap_dispc_isr_t) (void *arg, u32 mask);
 int omap_dispc_register_isr(omap_dispc_isr_t isr, void *arg, u32 mask);
@@ -610,21 +512,13 @@
 int omapdss_compat_init(void);
 void omapdss_compat_uninit(void);
 
-static inline bool omapdss_device_is_connected(struct omap_dss_device *dssdev)
-{
-	return dssdev->src;
-}
-
 static inline bool omapdss_device_is_enabled(struct omap_dss_device *dssdev)
 {
 	return dssdev->state == OMAP_DSS_DISPLAY_ACTIVE;
 }
 
 struct omap_dss_device *
-omapdss_of_find_source_for_first_ep(struct device_node *node);
-
-struct device_node *dss_of_port_get_parent_device(struct device_node *port);
-u32 dss_of_port_get_port_number(struct device_node *port);
+omapdss_of_find_connected_device(struct device_node *node, unsigned int port);
 
 enum dss_writeback_channel {
 	DSS_WB_LCD1_MGR =	0,
@@ -638,13 +532,6 @@
 };
 
 struct dss_mgr_ops {
-	int (*connect)(struct omap_drm_private *priv,
-		       enum omap_channel channel,
-		       struct omap_dss_device *dst);
-	void (*disconnect)(struct omap_drm_private *priv,
-			   enum omap_channel channel,
-			   struct omap_dss_device *dst);
-
 	void (*start_update)(struct omap_drm_private *priv,
 			     enum omap_channel channel);
 	int (*enable)(struct omap_drm_private *priv,
@@ -665,14 +552,11 @@
 			void (*handler)(void *), void *data);
 };
 
-int dss_install_mgr_ops(const struct dss_mgr_ops *mgr_ops,
+int dss_install_mgr_ops(struct dss_device *dss,
+			const struct dss_mgr_ops *mgr_ops,
 			struct omap_drm_private *priv);
-void dss_uninstall_mgr_ops(void);
+void dss_uninstall_mgr_ops(struct dss_device *dss);
 
-int dss_mgr_connect(struct omap_dss_device *dssdev,
-		    struct omap_dss_device *dst);
-void dss_mgr_disconnect(struct omap_dss_device *dssdev,
-			struct omap_dss_device *dst);
 void dss_mgr_set_timings(struct omap_dss_device *dssdev,
 		const struct videomode *vm);
 void dss_mgr_set_lcd_config(struct omap_dss_device *dssdev,
@@ -720,13 +604,14 @@
 	void (*mgr_set_lcd_config)(struct dispc_device *dispc,
 				   enum omap_channel channel,
 				   const struct dss_lcd_mgr_config *config);
+	int (*mgr_check_timings)(struct dispc_device *dispc,
+				 enum omap_channel channel,
+				 const struct videomode *vm);
 	void (*mgr_set_timings)(struct dispc_device *dispc,
 				enum omap_channel channel,
 				const struct videomode *vm);
 	void (*mgr_setup)(struct dispc_device *dispc, enum omap_channel channel,
 			  const struct omap_overlay_manager_info *info);
-	enum omap_dss_output_id (*mgr_get_supported_outputs)(
-			struct dispc_device *dispc, enum omap_channel channel);
 	u32 (*mgr_gamma_size)(struct dispc_device *dispc,
 			      enum omap_channel channel);
 	void (*mgr_set_gamma)(struct dispc_device *dispc,
@@ -743,6 +628,12 @@
 
 	const u32 *(*ovl_get_color_modes)(struct dispc_device *dispc,
 					  enum omap_plane_id plane);
+	bool (*ovl_color_mode_supported)(struct dispc_device *dispc,
+					 enum omap_plane_id plane, u32 fourcc);
+	enum omap_overlay_caps (*ovl_get_caps)(struct dispc_device *dispc,
+					       enum omap_plane_id plane);
+	void (*ovl_get_max_size)(struct dispc_device *dispc,
+				 u16 *width, u16 *height);
 
 	u32 (*wb_get_framedone_irq)(struct dispc_device *dispc);
 	int (*wb_setup)(struct dispc_device *dispc,
@@ -757,9 +648,6 @@
 struct dispc_device *dispc_get_dispc(struct dss_device *dss);
 const struct dispc_ops *dispc_get_ops(struct dss_device *dss);
 
-bool omapdss_component_is_display(struct device_node *node);
-bool omapdss_component_is_output(struct device_node *node);
-
 bool omapdss_stack_is_ready(void);
 void omapdss_gather_components(struct device *dev);
 
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/dss/output.c b/drivers/gpu/drm/omapdrm/dss/output.c
--- a/drivers/gpu/drm/omapdrm/dss/output.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/dss/output.c	2020-06-19 21:00:31.412227715 -0400
@@ -20,239 +20,132 @@
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 #include <linux/of.h>
+#include <linux/of_graph.h>
 
-#include "omapdss.h"
-
-static LIST_HEAD(output_list);
-static DEFINE_MUTEX(output_lock);
-
-int omapdss_output_set_device(struct omap_dss_device *out,
-		struct omap_dss_device *dssdev)
-{
-	int r;
-
-	mutex_lock(&output_lock);
-
-	if (out->dst) {
-		dev_err(out->dev,
-			"output already has device %s connected to it\n",
-			out->dst->name);
-		r = -EINVAL;
-		goto err;
-	}
-
-	if (out->output_type != dssdev->type) {
-		dev_err(out->dev, "output type and display type don't match\n");
-		r = -EINVAL;
-		goto err;
-	}
-
-	out->dst = dssdev;
-	dssdev->src = out;
-
-	mutex_unlock(&output_lock);
-
-	return 0;
-err:
-	mutex_unlock(&output_lock);
-
-	return r;
-}
-EXPORT_SYMBOL(omapdss_output_set_device);
+#include <drm/drm_panel.h>
 
-int omapdss_output_unset_device(struct omap_dss_device *out)
-{
-	int r;
-
-	mutex_lock(&output_lock);
-
-	if (!out->dst) {
-		dev_err(out->dev,
-			"output doesn't have a device connected to it\n");
-		r = -EINVAL;
-		goto err;
-	}
-
-	if (out->dst->state != OMAP_DSS_DISPLAY_DISABLED) {
-		dev_err(out->dev,
-			"device %s is not disabled, cannot unset device\n",
-			out->dst->name);
-		r = -EINVAL;
-		goto err;
-	}
-
-	out->dst->src = NULL;
-	out->dst = NULL;
-
-	mutex_unlock(&output_lock);
-
-	return 0;
-err:
-	mutex_unlock(&output_lock);
-
-	return r;
-}
-EXPORT_SYMBOL(omapdss_output_unset_device);
-
-int omapdss_register_output(struct omap_dss_device *out)
-{
-	list_add_tail(&out->list, &output_list);
-	return 0;
-}
-EXPORT_SYMBOL(omapdss_register_output);
-
-void omapdss_unregister_output(struct omap_dss_device *out)
-{
-	list_del(&out->list);
-}
-EXPORT_SYMBOL(omapdss_unregister_output);
+#include "dss.h"
+#include "omapdss.h"
 
-bool omapdss_component_is_output(struct device_node *node)
+int omapdss_device_init_output(struct omap_dss_device *out)
 {
-	struct omap_dss_device *out;
-
-	list_for_each_entry(out, &output_list, list) {
-		if (out->dev->of_node == node)
-			return true;
-	}
-
-	return false;
-}
-EXPORT_SYMBOL(omapdss_component_is_output);
+	struct device_node *remote_node;
+	u32 port_num;
 
-struct omap_dss_device *omap_dss_get_output(enum omap_dss_output_id id)
-{
-	struct omap_dss_device *out;
+	/*
+	 * FIXME: DSS outputs have only a single bit set, so we can use __ffs.
+	 * This is a temporary fix until the port management has been cleaned.
+	 */
+	port_num = __ffs(out->of_ports);
 
-	list_for_each_entry(out, &output_list, list) {
-		if (out->id == id)
-			return out;
+	remote_node = of_graph_get_remote_node(out->dev->of_node, port_num, 0);
+	if (!remote_node) {
+		dev_dbg(out->dev, "failed to find video sink\n");
+		return 0;
 	}
 
-	return NULL;
-}
-EXPORT_SYMBOL(omap_dss_get_output);
-
-struct omap_dss_device *omap_dss_find_output_by_port_node(struct device_node *port)
-{
-	struct device_node *src_node;
-	struct omap_dss_device *out;
-	u32 reg;
-
-	src_node = dss_of_port_get_parent_device(port);
-	if (!src_node)
-		return NULL;
+	out->next = omapdss_find_device_by_node(remote_node);
+	out->bridge = of_drm_find_bridge(remote_node);
+	out->panel = of_drm_find_panel(remote_node);
+	if (IS_ERR(out->panel))
+		out->panel = NULL;
 
-	reg = dss_of_port_get_port_number(port);
+	of_node_put(remote_node);
 
-	list_for_each_entry(out, &output_list, list) {
-		if (out->dev->of_node == src_node && out->port_num == reg) {
-			of_node_put(src_node);
-			return omap_dss_get_device(out);
-		}
+	if (out->next && out->type != out->next->type) {
+		dev_err(out->dev, "output type and display type don't match\n");
+		omapdss_device_put(out->next);
+		out->next = NULL;
+		return -EINVAL;
 	}
 
-	of_node_put(src_node);
-
-	return NULL;
+	return out->next || out->bridge || out->panel ? 0 : -EPROBE_DEFER;
 }
+EXPORT_SYMBOL(omapdss_device_init_output);
 
-struct omap_dss_device *omapdss_find_output_from_display(struct omap_dss_device *dssdev)
+void omapdss_device_cleanup_output(struct omap_dss_device *out)
 {
-	while (dssdev->src)
-		dssdev = dssdev->src;
-
-	if (dssdev->id != 0)
-		return omap_dss_get_device(dssdev);
-
-	return NULL;
+	if (out->next)
+		omapdss_device_put(out->next);
 }
-EXPORT_SYMBOL(omapdss_find_output_from_display);
-
-static const struct dss_mgr_ops *dss_mgr_ops;
-static struct omap_drm_private *dss_mgr_ops_priv;
+EXPORT_SYMBOL(omapdss_device_cleanup_output);
 
-int dss_install_mgr_ops(const struct dss_mgr_ops *mgr_ops,
+int dss_install_mgr_ops(struct dss_device *dss,
+			const struct dss_mgr_ops *mgr_ops,
 			struct omap_drm_private *priv)
 {
-	if (dss_mgr_ops)
+	if (dss->mgr_ops)
 		return -EBUSY;
 
-	dss_mgr_ops = mgr_ops;
-	dss_mgr_ops_priv = priv;
+	dss->mgr_ops = mgr_ops;
+	dss->mgr_ops_priv = priv;
 
 	return 0;
 }
 EXPORT_SYMBOL(dss_install_mgr_ops);
 
-void dss_uninstall_mgr_ops(void)
+void dss_uninstall_mgr_ops(struct dss_device *dss)
 {
-	dss_mgr_ops = NULL;
-	dss_mgr_ops_priv = NULL;
+	dss->mgr_ops = NULL;
+	dss->mgr_ops_priv = NULL;
 }
 EXPORT_SYMBOL(dss_uninstall_mgr_ops);
 
-int dss_mgr_connect(struct omap_dss_device *dssdev, struct omap_dss_device *dst)
-{
-	return dss_mgr_ops->connect(dss_mgr_ops_priv,
-				    dssdev->dispc_channel, dst);
-}
-EXPORT_SYMBOL(dss_mgr_connect);
-
-void dss_mgr_disconnect(struct omap_dss_device *dssdev,
-			struct omap_dss_device *dst)
-{
-	dss_mgr_ops->disconnect(dss_mgr_ops_priv, dssdev->dispc_channel, dst);
-}
-EXPORT_SYMBOL(dss_mgr_disconnect);
-
 void dss_mgr_set_timings(struct omap_dss_device *dssdev,
 			 const struct videomode *vm)
 {
-	dss_mgr_ops->set_timings(dss_mgr_ops_priv, dssdev->dispc_channel, vm);
+	dssdev->dss->mgr_ops->set_timings(dssdev->dss->mgr_ops_priv,
+					  dssdev->dispc_channel, vm);
 }
 EXPORT_SYMBOL(dss_mgr_set_timings);
 
 void dss_mgr_set_lcd_config(struct omap_dss_device *dssdev,
 		const struct dss_lcd_mgr_config *config)
 {
-	dss_mgr_ops->set_lcd_config(dss_mgr_ops_priv,
-				    dssdev->dispc_channel, config);
+	dssdev->dss->mgr_ops->set_lcd_config(dssdev->dss->mgr_ops_priv,
+					     dssdev->dispc_channel, config);
 }
 EXPORT_SYMBOL(dss_mgr_set_lcd_config);
 
 int dss_mgr_enable(struct omap_dss_device *dssdev)
 {
-	return dss_mgr_ops->enable(dss_mgr_ops_priv, dssdev->dispc_channel);
+	return dssdev->dss->mgr_ops->enable(dssdev->dss->mgr_ops_priv,
+					    dssdev->dispc_channel);
 }
 EXPORT_SYMBOL(dss_mgr_enable);
 
 void dss_mgr_disable(struct omap_dss_device *dssdev)
 {
-	dss_mgr_ops->disable(dss_mgr_ops_priv, dssdev->dispc_channel);
+	dssdev->dss->mgr_ops->disable(dssdev->dss->mgr_ops_priv,
+				      dssdev->dispc_channel);
 }
 EXPORT_SYMBOL(dss_mgr_disable);
 
 void dss_mgr_start_update(struct omap_dss_device *dssdev)
 {
-	dss_mgr_ops->start_update(dss_mgr_ops_priv, dssdev->dispc_channel);
+	dssdev->dss->mgr_ops->start_update(dssdev->dss->mgr_ops_priv,
+					   dssdev->dispc_channel);
 }
 EXPORT_SYMBOL(dss_mgr_start_update);
 
 int dss_mgr_register_framedone_handler(struct omap_dss_device *dssdev,
 		void (*handler)(void *), void *data)
 {
-	return dss_mgr_ops->register_framedone_handler(dss_mgr_ops_priv,
-						       dssdev->dispc_channel,
-						       handler, data);
+	struct dss_device *dss = dssdev->dss;
+
+	return dss->mgr_ops->register_framedone_handler(dss->mgr_ops_priv,
+							dssdev->dispc_channel,
+							handler, data);
 }
 EXPORT_SYMBOL(dss_mgr_register_framedone_handler);
 
 void dss_mgr_unregister_framedone_handler(struct omap_dss_device *dssdev,
 		void (*handler)(void *), void *data)
 {
-	dss_mgr_ops->unregister_framedone_handler(dss_mgr_ops_priv,
-						  dssdev->dispc_channel,
-						  handler, data);
+	struct dss_device *dss = dssdev->dss;
+
+	dss->mgr_ops->unregister_framedone_handler(dss->mgr_ops_priv,
+						   dssdev->dispc_channel,
+						   handler, data);
 }
 EXPORT_SYMBOL(dss_mgr_unregister_framedone_handler);
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/dss/sdi.c b/drivers/gpu/drm/omapdrm/dss/sdi.c
--- a/drivers/gpu/drm/omapdrm/dss/sdi.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/dss/sdi.c	2020-06-19 21:00:31.412227715 -0400
@@ -37,7 +37,7 @@
 	struct regulator *vdds_sdi_reg;
 
 	struct dss_lcd_mgr_config mgr_config;
-	struct videomode vm;
+	unsigned long pixelclock;
 	int datapairs;
 
 	struct omap_dss_device output;
@@ -129,49 +129,27 @@
 	dss_mgr_set_lcd_config(&sdi->output, &sdi->mgr_config);
 }
 
-static int sdi_display_enable(struct omap_dss_device *dssdev)
+static void sdi_display_enable(struct omap_dss_device *dssdev)
 {
 	struct sdi_device *sdi = dssdev_to_sdi(dssdev);
-	struct videomode *vm = &sdi->vm;
-	unsigned long fck;
 	struct dispc_clock_info dispc_cinfo;
-	unsigned long pck;
+	unsigned long fck;
 	int r;
 
-	if (!sdi->output.dispc_channel_connected) {
-		DSSERR("failed to enable display: no output/manager\n");
-		return -ENODEV;
-	}
-
 	r = regulator_enable(sdi->vdds_sdi_reg);
 	if (r)
-		goto err_reg_enable;
+		return;
 
 	r = dispc_runtime_get(sdi->dss->dispc);
 	if (r)
 		goto err_get_dispc;
 
-	/* 15.5.9.1.2 */
-	vm->flags |= DISPLAY_FLAGS_PIXDATA_POSEDGE | DISPLAY_FLAGS_SYNC_POSEDGE;
-
-	r = sdi_calc_clock_div(sdi, vm->pixelclock, &fck, &dispc_cinfo);
+	r = sdi_calc_clock_div(sdi, sdi->pixelclock, &fck, &dispc_cinfo);
 	if (r)
 		goto err_calc_clock_div;
 
 	sdi->mgr_config.clock_info = dispc_cinfo;
 
-	pck = fck / dispc_cinfo.lck_div / dispc_cinfo.pck_div;
-
-	if (pck != vm->pixelclock) {
-		DSSWARN("Could not find exact pixel clock. Requested %lu Hz, got %lu Hz\n",
-			vm->pixelclock, pck);
-
-		vm->pixelclock = pck;
-	}
-
-
-	dss_mgr_set_timings(&sdi->output, vm);
-
 	r = dss_set_fck_rate(sdi->dss, fck);
 	if (r)
 		goto err_set_dss_clock_div;
@@ -202,7 +180,7 @@
 	if (r)
 		goto err_mgr_enable;
 
-	return 0;
+	return;
 
 err_mgr_enable:
 	dss_sdi_disable(sdi->dss);
@@ -212,8 +190,6 @@
 	dispc_runtime_put(sdi->dss->dispc);
 err_get_dispc:
 	regulator_disable(sdi->vdds_sdi_reg);
-err_reg_enable:
-	return r;
 }
 
 static void sdi_display_disable(struct omap_dss_device *dssdev)
@@ -230,96 +206,55 @@
 }
 
 static void sdi_set_timings(struct omap_dss_device *dssdev,
-			    struct videomode *vm)
+			    const struct drm_display_mode *mode)
 {
 	struct sdi_device *sdi = dssdev_to_sdi(dssdev);
 
-	sdi->vm = *vm;
-}
-
-static void sdi_get_timings(struct omap_dss_device *dssdev,
-			    struct videomode *vm)
-{
-	struct sdi_device *sdi = dssdev_to_sdi(dssdev);
-
-	*vm = sdi->vm;
+	sdi->pixelclock = mode->clock * 1000;
 }
 
 static int sdi_check_timings(struct omap_dss_device *dssdev,
-			     struct videomode *vm)
+			     struct drm_display_mode *mode)
 {
 	struct sdi_device *sdi = dssdev_to_sdi(dssdev);
-	enum omap_channel channel = dssdev->dispc_channel;
-
-	if (!dispc_mgr_timings_ok(sdi->dss->dispc, channel, vm))
-		return -EINVAL;
+	struct dispc_clock_info dispc_cinfo;
+	unsigned long pixelclock = mode->clock * 1000;
+	unsigned long fck;
+	unsigned long pck;
+	int r;
 
-	if (vm->pixelclock == 0)
+	if (pixelclock == 0)
 		return -EINVAL;
 
-	return 0;
-}
+	r = sdi_calc_clock_div(sdi, pixelclock, &fck, &dispc_cinfo);
+	if (r)
+		return r;
 
-static int sdi_init_regulator(struct sdi_device *sdi)
-{
-	struct regulator *vdds_sdi;
+	pck = fck / dispc_cinfo.lck_div / dispc_cinfo.pck_div;
 
-	if (sdi->vdds_sdi_reg)
-		return 0;
+	if (pck != pixelclock) {
+		DSSWARN("Pixel clock adjusted from %lu Hz to %lu Hz\n",
+			pixelclock, pck);
 
-	vdds_sdi = devm_regulator_get(&sdi->pdev->dev, "vdds_sdi");
-	if (IS_ERR(vdds_sdi)) {
-		if (PTR_ERR(vdds_sdi) != -EPROBE_DEFER)
-			DSSERR("can't get VDDS_SDI regulator\n");
-		return PTR_ERR(vdds_sdi);
+		mode->clock = pck / 1000;
 	}
 
-	sdi->vdds_sdi_reg = vdds_sdi;
-
 	return 0;
 }
 
-static int sdi_connect(struct omap_dss_device *dssdev,
-		struct omap_dss_device *dst)
+static int sdi_connect(struct omap_dss_device *src,
+		       struct omap_dss_device *dst)
 {
-	struct sdi_device *sdi = dssdev_to_sdi(dssdev);
-	int r;
-
-	r = sdi_init_regulator(sdi);
-	if (r)
-		return r;
-
-	r = dss_mgr_connect(&sdi->output, dssdev);
-	if (r)
-		return r;
-
-	r = omapdss_output_set_device(dssdev, dst);
-	if (r) {
-		DSSERR("failed to connect output to new device: %s\n",
-				dst->name);
-		dss_mgr_disconnect(&sdi->output, dssdev);
-		return r;
-	}
-
-	return 0;
+	return omapdss_device_connect(dst->dss, dst, dst->next);
 }
 
-static void sdi_disconnect(struct omap_dss_device *dssdev,
-		struct omap_dss_device *dst)
+static void sdi_disconnect(struct omap_dss_device *src,
+			   struct omap_dss_device *dst)
 {
-	struct sdi_device *sdi = dssdev_to_sdi(dssdev);
-
-	WARN_ON(dst != dssdev->dst);
-
-	if (dst != dssdev->dst)
-		return;
-
-	omapdss_output_unset_device(dssdev);
-
-	dss_mgr_disconnect(&sdi->output, dssdev);
+	omapdss_device_disconnect(dst, dst->next);
 }
 
-static const struct omapdss_sdi_ops sdi_ops = {
+static const struct omap_dss_device_ops sdi_ops = {
 	.connect = sdi_connect,
 	.disconnect = sdi_disconnect,
 
@@ -328,29 +263,38 @@
 
 	.check_timings = sdi_check_timings,
 	.set_timings = sdi_set_timings,
-	.get_timings = sdi_get_timings,
 };
 
-static void sdi_init_output(struct sdi_device *sdi)
+static int sdi_init_output(struct sdi_device *sdi)
 {
 	struct omap_dss_device *out = &sdi->output;
+	int r;
 
 	out->dev = &sdi->pdev->dev;
 	out->id = OMAP_DSS_OUTPUT_SDI;
-	out->output_type = OMAP_DISPLAY_TYPE_SDI;
+	out->type = OMAP_DISPLAY_TYPE_SDI;
 	out->name = "sdi.0";
 	out->dispc_channel = OMAP_DSS_CHANNEL_LCD;
 	/* We have SDI only on OMAP3, where it's on port 1 */
-	out->port_num = 1;
-	out->ops.sdi = &sdi_ops;
+	out->of_ports = BIT(1);
+	out->ops = &sdi_ops;
 	out->owner = THIS_MODULE;
+	out->bus_flags = DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE	/* 15.5.9.1.2 */
+		       | DRM_BUS_FLAG_SYNC_DRIVE_POSEDGE;
+
+	r = omapdss_device_init_output(out);
+	if (r < 0)
+		return r;
+
+	omapdss_device_register(out);
 
-	omapdss_register_output(out);
+	return 0;
 }
 
 static void sdi_uninit_output(struct sdi_device *sdi)
 {
-	omapdss_unregister_output(&sdi->output);
+	omapdss_device_unregister(&sdi->output);
+	omapdss_device_cleanup_output(&sdi->output);
 }
 
 int sdi_init_port(struct dss_device *dss, struct platform_device *pdev,
@@ -372,25 +316,32 @@
 	}
 
 	r = of_property_read_u32(ep, "datapairs", &datapairs);
+	of_node_put(ep);
 	if (r) {
 		DSSERR("failed to parse datapairs\n");
-		goto err_datapairs;
+		goto err_free;
 	}
 
 	sdi->datapairs = datapairs;
 	sdi->dss = dss;
 
-	of_node_put(ep);
-
 	sdi->pdev = pdev;
 	port->data = sdi;
 
-	sdi_init_output(sdi);
+	sdi->vdds_sdi_reg = devm_regulator_get(&pdev->dev, "vdds_sdi");
+	if (IS_ERR(sdi->vdds_sdi_reg)) {
+		r = PTR_ERR(sdi->vdds_sdi_reg);
+		if (r != -EPROBE_DEFER)
+			DSSERR("can't get VDDS_SDI regulator\n");
+		goto err_free;
+	}
+
+	r = sdi_init_output(sdi);
+	if (r)
+		goto err_free;
 
 	return 0;
 
-err_datapairs:
-	of_node_put(ep);
 err_free:
 	kfree(sdi);
 
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/dss/venc.c b/drivers/gpu/drm/omapdrm/dss/venc.c
--- a/drivers/gpu/drm/omapdrm/dss/venc.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/dss/venc.c	2020-06-19 21:00:31.412227715 -0400
@@ -267,63 +267,40 @@
 	VENC_MODE_NTSC,
 };
 
-static const struct videomode omap_dss_pal_vm = {
-	.hactive	= 720,
-	.vactive	= 574,
-	.pixelclock	= 13500000,
-	.hsync_len	= 64,
-	.hfront_porch	= 12,
-	.hback_porch	= 68,
-	.vsync_len	= 5,
-	.vfront_porch	= 5,
-	.vback_porch	= 41,
-
-	.flags		= DISPLAY_FLAGS_INTERLACED | DISPLAY_FLAGS_HSYNC_LOW |
-			  DISPLAY_FLAGS_VSYNC_LOW | DISPLAY_FLAGS_DE_HIGH |
-			  DISPLAY_FLAGS_PIXDATA_POSEDGE |
-			  DISPLAY_FLAGS_SYNC_NEGEDGE,
-};
+static const struct drm_display_mode omap_dss_pal_mode = {
+	.hdisplay	= 720,
+	.hsync_start	= 732,
+	.hsync_end	= 796,
+	.htotal		= 864,
+	.vdisplay	= 574,
+	.vsync_start	= 579,
+	.vsync_end	= 584,
+	.vtotal		= 625,
+	.clock		= 13500,
 
-static const struct videomode omap_dss_ntsc_vm = {
-	.hactive	= 720,
-	.vactive	= 482,
-	.pixelclock	= 13500000,
-	.hsync_len	= 64,
-	.hfront_porch	= 16,
-	.hback_porch	= 58,
-	.vsync_len	= 6,
-	.vfront_porch	= 6,
-	.vback_porch	= 31,
-
-	.flags		= DISPLAY_FLAGS_INTERLACED | DISPLAY_FLAGS_HSYNC_LOW |
-			  DISPLAY_FLAGS_VSYNC_LOW | DISPLAY_FLAGS_DE_HIGH |
-			  DISPLAY_FLAGS_PIXDATA_POSEDGE |
-			  DISPLAY_FLAGS_SYNC_NEGEDGE,
+	.flags		= DRM_MODE_FLAG_INTERLACE | DRM_MODE_FLAG_NHSYNC |
+			  DRM_MODE_FLAG_NVSYNC,
 };
 
-static enum venc_videomode venc_get_videomode(const struct videomode *vm)
-{
-	if (!(vm->flags & DISPLAY_FLAGS_INTERLACED))
-		return VENC_MODE_UNKNOWN;
-
-	if (vm->pixelclock == omap_dss_pal_vm.pixelclock &&
-	    vm->hactive == omap_dss_pal_vm.hactive &&
-	    vm->vactive == omap_dss_pal_vm.vactive)
-		return VENC_MODE_PAL;
-
-	if (vm->pixelclock == omap_dss_ntsc_vm.pixelclock &&
-	    vm->hactive == omap_dss_ntsc_vm.hactive &&
-	    vm->vactive == omap_dss_ntsc_vm.vactive)
-		return VENC_MODE_NTSC;
+static const struct drm_display_mode omap_dss_ntsc_mode = {
+	.hdisplay	= 720,
+	.hsync_start	= 736,
+	.hsync_end	= 800,
+	.htotal		= 858,
+	.vdisplay	= 482,
+	.vsync_start	= 488,
+	.vsync_end	= 494,
+	.vtotal		= 525,
+	.clock		= 13500,
 
-	return VENC_MODE_UNKNOWN;
-}
+	.flags		= DRM_MODE_FLAG_INTERLACE | DRM_MODE_FLAG_NHSYNC |
+			  DRM_MODE_FLAG_NVSYNC,
+};
 
 struct venc_device {
 	struct platform_device *pdev;
 	void __iomem *base;
 	struct mutex venc_lock;
-	u32 wss_data;
 	struct regulator *vdda_dac_reg;
 	struct dss_device *dss;
 
@@ -331,7 +308,7 @@
 
 	struct clk	*tv_dac_clk;
 
-	struct videomode vm;
+	const struct venc_config *config;
 	enum omap_dss_venc_type type;
 	bool invert_polarity;
 	bool requires_tv_dac_clk;
@@ -367,8 +344,7 @@
 	venc_write_reg(venc, VENC_BLACK_LEVEL, config->black_level);
 	venc_write_reg(venc, VENC_BLANK_LEVEL, config->blank_level);
 	venc_write_reg(venc, VENC_M_CONTROL, config->m_control);
-	venc_write_reg(venc, VENC_BSTAMP_WSS_DATA, config->bstamp_wss_data |
-		       venc->wss_data);
+	venc_write_reg(venc, VENC_BSTAMP_WSS_DATA, config->bstamp_wss_data);
 	venc_write_reg(venc, VENC_S_CARR, config->s_carr);
 	venc_write_reg(venc, VENC_L21__WC_CTL, config->l21__wc_ctl);
 	venc_write_reg(venc, VENC_SAVID__EAVID, config->savid__eavid);
@@ -452,18 +428,6 @@
 	WARN_ON(r < 0 && r != -ENOSYS);
 }
 
-static const struct venc_config *venc_timings_to_config(struct videomode *vm)
-{
-	switch (venc_get_videomode(vm)) {
-	default:
-		WARN_ON_ONCE(1);
-	case VENC_MODE_PAL:
-		return &venc_config_pal_trm;
-	case VENC_MODE_NTSC:
-		return &venc_config_ntsc_trm;
-	}
-}
-
 static int venc_power_on(struct venc_device *venc)
 {
 	u32 l;
@@ -474,7 +438,7 @@
 		goto err0;
 
 	venc_reset(venc);
-	venc_write_config(venc, venc_timings_to_config(&venc->vm));
+	venc_write_config(venc, venc->config);
 
 	dss_set_venc_output(venc->dss, venc->type);
 	dss_set_dac_pwrdn_bgz(venc->dss, 1);
@@ -491,8 +455,6 @@
 
 	venc_write_reg(venc, VENC_OUTPUT_CONTROL, l);
 
-	dss_mgr_set_timings(&venc->output, &venc->vm);
-
 	r = regulator_enable(venc->vdda_dac_reg);
 	if (r)
 		goto err1;
@@ -526,33 +488,17 @@
 	venc_runtime_put(venc);
 }
 
-static int venc_display_enable(struct omap_dss_device *dssdev)
+static void venc_display_enable(struct omap_dss_device *dssdev)
 {
 	struct venc_device *venc = dssdev_to_venc(dssdev);
-	int r;
 
 	DSSDBG("venc_display_enable\n");
 
 	mutex_lock(&venc->venc_lock);
 
-	if (!dssdev->dispc_channel_connected) {
-		DSSERR("Failed to enable display: no output/manager\n");
-		r = -ENODEV;
-		goto err0;
-	}
-
-	r = venc_power_on(venc);
-	if (r)
-		goto err0;
-
-	venc->wss_data = 0;
+	venc_power_on(venc);
 
 	mutex_unlock(&venc->venc_lock);
-
-	return 0;
-err0:
-	mutex_unlock(&venc->venc_lock);
-	return r;
 }
 
 static void venc_display_disable(struct omap_dss_device *dssdev)
@@ -568,118 +514,96 @@
 	mutex_unlock(&venc->venc_lock);
 }
 
-static void venc_set_timings(struct omap_dss_device *dssdev,
-			     struct videomode *vm)
+static int venc_get_modes(struct omap_dss_device *dssdev,
+			  struct drm_connector *connector)
 {
-	struct venc_device *venc = dssdev_to_venc(dssdev);
-	struct videomode actual_vm;
+	static const struct drm_display_mode *modes[] = {
+		&omap_dss_pal_mode,
+		&omap_dss_ntsc_mode,
+	};
+	unsigned int i;
 
-	DSSDBG("venc_set_timings\n");
+	for (i = 0; i < ARRAY_SIZE(modes); ++i) {
+		struct drm_display_mode *mode;
 
-	mutex_lock(&venc->venc_lock);
+		mode = drm_mode_duplicate(connector->dev, modes[i]);
+		if (!mode)
+			return i;
 
-	switch (venc_get_videomode(vm)) {
-	default:
-		WARN_ON_ONCE(1);
-	case VENC_MODE_PAL:
-		actual_vm = omap_dss_pal_vm;
-		break;
-	case VENC_MODE_NTSC:
-		actual_vm = omap_dss_ntsc_vm;
-		break;
+		mode->type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED;
+		drm_mode_set_name(mode);
+		drm_mode_probed_add(connector, mode);
 	}
 
-	/* Reset WSS data when the TV standard changes. */
-	if (memcmp(&venc->vm, &actual_vm, sizeof(actual_vm)))
-		venc->wss_data = 0;
-
-	venc->vm = actual_vm;
-
-	dispc_set_tv_pclk(venc->dss->dispc, 13500000);
-
-	mutex_unlock(&venc->venc_lock);
-}
-
-static int venc_check_timings(struct omap_dss_device *dssdev,
-			      struct videomode *vm)
-{
-	DSSDBG("venc_check_timings\n");
-
-	switch (venc_get_videomode(vm)) {
-	case VENC_MODE_PAL:
-	case VENC_MODE_NTSC:
-		return 0;
-	default:
-		return -EINVAL;
-	}
+	return ARRAY_SIZE(modes);
 }
 
-static void venc_get_timings(struct omap_dss_device *dssdev,
-			     struct videomode *vm)
+static enum venc_videomode venc_get_videomode(const struct drm_display_mode *mode)
 {
-	struct venc_device *venc = dssdev_to_venc(dssdev);
-
-	mutex_lock(&venc->venc_lock);
-
-	*vm = venc->vm;
+	if (!(mode->flags & DRM_MODE_FLAG_INTERLACE))
+		return VENC_MODE_UNKNOWN;
 
-	mutex_unlock(&venc->venc_lock);
-}
+	if (mode->clock == omap_dss_pal_mode.clock &&
+	    mode->hdisplay == omap_dss_pal_mode.hdisplay &&
+	    mode->vdisplay == omap_dss_pal_mode.vdisplay)
+		return VENC_MODE_PAL;
 
-static u32 venc_get_wss(struct omap_dss_device *dssdev)
-{
-	struct venc_device *venc = dssdev_to_venc(dssdev);
+	if (mode->clock == omap_dss_ntsc_mode.clock &&
+	    mode->hdisplay == omap_dss_ntsc_mode.hdisplay &&
+	    mode->vdisplay == omap_dss_ntsc_mode.vdisplay)
+		return VENC_MODE_NTSC;
 
-	/* Invert due to VENC_L21_WC_CTL:INV=1 */
-	return (venc->wss_data >> 8) ^ 0xfffff;
+	return VENC_MODE_UNKNOWN;
 }
 
-static int venc_set_wss(struct omap_dss_device *dssdev, u32 wss)
+static void venc_set_timings(struct omap_dss_device *dssdev,
+			     const struct drm_display_mode *mode)
 {
 	struct venc_device *venc = dssdev_to_venc(dssdev);
-	const struct venc_config *config;
-	int r;
+	enum venc_videomode venc_mode = venc_get_videomode(mode);
 
-	DSSDBG("venc_set_wss\n");
+	DSSDBG("venc_set_timings\n");
 
 	mutex_lock(&venc->venc_lock);
 
-	config = venc_timings_to_config(&venc->vm);
-
-	/* Invert due to VENC_L21_WC_CTL:INV=1 */
-	venc->wss_data = (wss ^ 0xfffff) << 8;
-
-	r = venc_runtime_get(venc);
-	if (r)
-		goto err;
+	switch (venc_mode) {
+	default:
+		WARN_ON_ONCE(1);
+		/* Fall-through */
+	case VENC_MODE_PAL:
+		venc->config = &venc_config_pal_trm;
+		break;
 
-	venc_write_reg(venc, VENC_BSTAMP_WSS_DATA, config->bstamp_wss_data |
-		       venc->wss_data);
+	case VENC_MODE_NTSC:
+		venc->config = &venc_config_ntsc_trm;
+		break;
+	}
 
-	venc_runtime_put(venc);
+	dispc_set_tv_pclk(venc->dss->dispc, 13500000);
 
-err:
 	mutex_unlock(&venc->venc_lock);
-
-	return r;
 }
 
-static int venc_init_regulator(struct venc_device *venc)
+static int venc_check_timings(struct omap_dss_device *dssdev,
+			      struct drm_display_mode *mode)
 {
-	struct regulator *vdda_dac;
+	DSSDBG("venc_check_timings\n");
 
-	if (venc->vdda_dac_reg != NULL)
-		return 0;
+	switch (venc_get_videomode(mode)) {
+	case VENC_MODE_PAL:
+		drm_mode_copy(mode, &omap_dss_pal_mode);
+		break;
 
-	vdda_dac = devm_regulator_get(&venc->pdev->dev, "vdda");
-	if (IS_ERR(vdda_dac)) {
-		if (PTR_ERR(vdda_dac) != -EPROBE_DEFER)
-			DSSERR("can't get VDDA_DAC regulator\n");
-		return PTR_ERR(vdda_dac);
-	}
+	case VENC_MODE_NTSC:
+		drm_mode_copy(mode, &omap_dss_ntsc_mode);
+		break;
 
-	venc->vdda_dac_reg = vdda_dac;
+	default:
+		return -EINVAL;
+	}
 
+	drm_mode_set_crtcinfo(mode, CRTC_INTERLACE_HALVE_V);
+	drm_mode_set_name(mode);
 	return 0;
 }
 
@@ -760,47 +684,19 @@
 	return 0;
 }
 
-static int venc_connect(struct omap_dss_device *dssdev,
-		struct omap_dss_device *dst)
+static int venc_connect(struct omap_dss_device *src,
+			struct omap_dss_device *dst)
 {
-	struct venc_device *venc = dssdev_to_venc(dssdev);
-	int r;
-
-	r = venc_init_regulator(venc);
-	if (r)
-		return r;
-
-	r = dss_mgr_connect(&venc->output, dssdev);
-	if (r)
-		return r;
-
-	r = omapdss_output_set_device(dssdev, dst);
-	if (r) {
-		DSSERR("failed to connect output to new device: %s\n",
-				dst->name);
-		dss_mgr_disconnect(&venc->output, dssdev);
-		return r;
-	}
-
-	return 0;
+	return omapdss_device_connect(dst->dss, dst, dst->next);
 }
 
-static void venc_disconnect(struct omap_dss_device *dssdev,
-		struct omap_dss_device *dst)
+static void venc_disconnect(struct omap_dss_device *src,
+			    struct omap_dss_device *dst)
 {
-	struct venc_device *venc = dssdev_to_venc(dssdev);
-
-	WARN_ON(dst != dssdev->dst);
-
-	if (dst != dssdev->dst)
-		return;
-
-	omapdss_output_unset_device(dssdev);
-
-	dss_mgr_disconnect(&venc->output, dssdev);
+	omapdss_device_disconnect(dst, dst->next);
 }
 
-static const struct omapdss_atv_ops venc_ops = {
+static const struct omap_dss_device_ops venc_ops = {
 	.connect = venc_connect,
 	.disconnect = venc_disconnect,
 
@@ -809,30 +705,82 @@
 
 	.check_timings = venc_check_timings,
 	.set_timings = venc_set_timings,
-	.get_timings = venc_get_timings,
 
-	.set_wss = venc_set_wss,
-	.get_wss = venc_get_wss,
+	.get_modes = venc_get_modes,
 };
 
-static void venc_init_output(struct venc_device *venc)
+/* -----------------------------------------------------------------------------
+ * Component Bind & Unbind
+ */
+
+static int venc_bind(struct device *dev, struct device *master, void *data)
+{
+	struct dss_device *dss = dss_get_device(master);
+	struct venc_device *venc = dev_get_drvdata(dev);
+	u8 rev_id;
+	int r;
+
+	venc->dss = dss;
+
+	r = venc_runtime_get(venc);
+	if (r)
+		return r;
+
+	rev_id = (u8)(venc_read_reg(venc, VENC_REV_ID) & 0xff);
+	dev_dbg(dev, "OMAP VENC rev %d\n", rev_id);
+
+	venc_runtime_put(venc);
+
+	venc->debugfs = dss_debugfs_create_file(dss, "venc", venc_dump_regs,
+						venc);
+
+	return 0;
+}
+
+static void venc_unbind(struct device *dev, struct device *master, void *data)
+{
+	struct venc_device *venc = dev_get_drvdata(dev);
+
+	dss_debugfs_remove_file(venc->debugfs);
+}
+
+static const struct component_ops venc_component_ops = {
+	.bind	= venc_bind,
+	.unbind	= venc_unbind,
+};
+
+/* -----------------------------------------------------------------------------
+ * Probe & Remove, Suspend & Resume
+ */
+
+static int venc_init_output(struct venc_device *venc)
 {
 	struct omap_dss_device *out = &venc->output;
+	int r;
 
 	out->dev = &venc->pdev->dev;
 	out->id = OMAP_DSS_OUTPUT_VENC;
-	out->output_type = OMAP_DISPLAY_TYPE_VENC;
+	out->type = OMAP_DISPLAY_TYPE_VENC;
 	out->name = "venc.0";
 	out->dispc_channel = OMAP_DSS_CHANNEL_DIGIT;
-	out->ops.atv = &venc_ops;
+	out->ops = &venc_ops;
 	out->owner = THIS_MODULE;
+	out->of_ports = BIT(0);
+	out->ops_flags = OMAP_DSS_DEVICE_OP_MODES;
+
+	r = omapdss_device_init_output(out);
+	if (r < 0)
+		return r;
+
+	omapdss_device_register(out);
 
-	omapdss_register_output(out);
+	return 0;
 }
 
 static void venc_uninit_output(struct venc_device *venc)
 {
-	omapdss_unregister_output(&venc->output);
+	omapdss_device_unregister(&venc->output);
+	omapdss_device_cleanup_output(&venc->output);
 }
 
 static int venc_probe_of(struct venc_device *venc)
@@ -878,19 +826,15 @@
 	return r;
 }
 
-/* VENC HW IP initialisation */
 static const struct soc_device_attribute venc_soc_devices[] = {
 	{ .machine = "OMAP3[45]*" },
 	{ .machine = "AM35*" },
 	{ /* sentinel */ }
 };
 
-static int venc_bind(struct device *dev, struct device *master, void *data)
+static int venc_probe(struct platform_device *pdev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct dss_device *dss = dss_get_device(master);
 	struct venc_device *venc;
-	u8 rev_id;
 	struct resource *venc_mem;
 	int r;
 
@@ -899,8 +843,8 @@
 		return -ENOMEM;
 
 	venc->pdev = pdev;
-	venc->dss = dss;
-	dev_set_drvdata(dev, venc);
+
+	platform_set_drvdata(pdev, venc);
 
 	/* The OMAP34xx, OMAP35xx and AM35xx VENC require the TV DAC clock. */
 	if (soc_device_match(venc_soc_devices))
@@ -908,7 +852,7 @@
 
 	mutex_init(&venc->venc_lock);
 
-	venc->wss_data = 0;
+	venc->config = &venc_config_pal_trm;
 
 	venc_mem = platform_get_resource(venc->pdev, IORESOURCE_MEM, 0);
 	venc->base = devm_ioremap_resource(&pdev->dev, venc_mem);
@@ -917,68 +861,54 @@
 		goto err_free;
 	}
 
+	venc->vdda_dac_reg = devm_regulator_get(&pdev->dev, "vdda");
+	if (IS_ERR(venc->vdda_dac_reg)) {
+		r = PTR_ERR(venc->vdda_dac_reg);
+		if (r != -EPROBE_DEFER)
+			DSSERR("can't get VDDA_DAC regulator\n");
+		goto err_free;
+	}
+
 	r = venc_get_clocks(venc);
 	if (r)
 		goto err_free;
 
-	pm_runtime_enable(&pdev->dev);
-
-	r = venc_runtime_get(venc);
+	r = venc_probe_of(venc);
 	if (r)
-		goto err_runtime_get;
-
-	rev_id = (u8)(venc_read_reg(venc, VENC_REV_ID) & 0xff);
-	dev_dbg(&pdev->dev, "OMAP VENC rev %d\n", rev_id);
-
-	venc_runtime_put(venc);
+		goto err_free;
 
-	r = venc_probe_of(venc);
-	if (r) {
-		DSSERR("Invalid DT data\n");
-		goto err_probe_of;
-	}
+	pm_runtime_enable(&pdev->dev);
 
-	venc->debugfs = dss_debugfs_create_file(dss, "venc", venc_dump_regs,
-						venc);
+	r = venc_init_output(venc);
+	if (r)
+		goto err_pm_disable;
 
-	venc_init_output(venc);
+	r = component_add(&pdev->dev, &venc_component_ops);
+	if (r)
+		goto err_uninit_output;
 
 	return 0;
 
-err_probe_of:
-err_runtime_get:
+err_uninit_output:
+	venc_uninit_output(venc);
+err_pm_disable:
 	pm_runtime_disable(&pdev->dev);
 err_free:
 	kfree(venc);
 	return r;
 }
 
-static void venc_unbind(struct device *dev, struct device *master, void *data)
+static int venc_remove(struct platform_device *pdev)
 {
-	struct venc_device *venc = dev_get_drvdata(dev);
+	struct venc_device *venc = platform_get_drvdata(pdev);
 
-	dss_debugfs_remove_file(venc->debugfs);
+	component_del(&pdev->dev, &venc_component_ops);
 
 	venc_uninit_output(venc);
 
-	pm_runtime_disable(dev);
+	pm_runtime_disable(&pdev->dev);
 
 	kfree(venc);
-}
-
-static const struct component_ops venc_component_ops = {
-	.bind	= venc_bind,
-	.unbind	= venc_unbind,
-};
-
-static int venc_probe(struct platform_device *pdev)
-{
-	return component_add(&pdev->dev, &venc_component_ops);
-}
-
-static int venc_remove(struct platform_device *pdev)
-{
-	component_del(&pdev->dev, &venc_component_ops);
 	return 0;
 }
 
@@ -989,19 +919,12 @@
 	if (venc->tv_dac_clk)
 		clk_disable_unprepare(venc->tv_dac_clk);
 
-	dispc_runtime_put(venc->dss->dispc);
-
 	return 0;
 }
 
 static int venc_runtime_resume(struct device *dev)
 {
 	struct venc_device *venc = dev_get_drvdata(dev);
-	int r;
-
-	r = dispc_runtime_get(venc->dss->dispc);
-	if (r < 0)
-		return r;
 
 	if (venc->tv_dac_clk)
 		clk_prepare_enable(venc->tv_dac_clk);
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/Kconfig b/drivers/gpu/drm/omapdrm/Kconfig
--- a/drivers/gpu/drm/omapdrm/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/Kconfig	2020-06-19 21:00:31.412227715 -0400
@@ -10,6 +10,17 @@
 
 if DRM_OMAP
 
+config DRM_OMAP_WB
+	bool "Enable writeback support for OMAP DRM driver"
+	depends on DRM_OMAP
+	depends on (VIDEO_V4L2 = y) || (VIDEO_V4L2 = m && DRM_OMAP = m)
+	depends on VIDEO_DEV && HAS_DMA
+	select VIDEOBUF2_DMA_CONTIG
+	select V4L2_MEM2MEM_DEV
+	default n
+	help
+	  Select this to enable memory-to-memory/capture writeback support.
+
 source "drivers/gpu/drm/omapdrm/dss/Kconfig"
 source "drivers/gpu/drm/omapdrm/displays/Kconfig"
 
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/Makefile b/drivers/gpu/drm/omapdrm/Makefile
--- a/drivers/gpu/drm/omapdrm/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/Makefile	2020-06-19 21:00:31.412227715 -0400
@@ -12,6 +12,7 @@
 	omap_debugfs.o \
 	omap_crtc.o \
 	omap_plane.o \
+	omap_overlay.o \
 	omap_encoder.o \
 	omap_connector.o \
 	omap_fb.o \
@@ -22,4 +23,6 @@
 
 omapdrm-$(CONFIG_DRM_FBDEV_EMULATION) += omap_fbdev.o
 
+omapdrm-$(CONFIG_DRM_OMAP_WB) += omap_wb.o omap_wb_cap.o omap_wb_m2m.o
+
 obj-$(CONFIG_DRM_OMAP)	+= omapdrm.o
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/omap_connector.c b/drivers/gpu/drm/omapdrm/omap_connector.c
--- a/drivers/gpu/drm/omapdrm/omap_connector.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/omap_connector.c	2020-06-19 21:00:31.412227715 -0400
@@ -18,6 +18,7 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_crtc_helper.h>
+#include <drm/drm_panel.h>
 
 #include "omap_drv.h"
 
@@ -29,10 +30,31 @@
 
 struct omap_connector {
 	struct drm_connector base;
-	struct omap_dss_device *dssdev;
+	struct omap_dss_device *output;
+	struct omap_dss_device *hpd;
 	bool hdmi_mode;
 };
 
+static void omap_connector_hpd_notify(struct drm_connector *connector,
+				      enum drm_connector_status status)
+{
+	struct omap_connector *omap_connector = to_omap_connector(connector);
+	struct omap_dss_device *dssdev;
+
+	if (status != connector_status_disconnected)
+		return;
+
+	/*
+	 * Notify all devics in the pipeline of disconnection. This is required
+	 * to let the HDMI encoders reset their internal state related to
+	 * connection status, such as the CEC address.
+	 */
+	for (dssdev = omap_connector->output; dssdev; dssdev = dssdev->next) {
+		if (dssdev->ops && dssdev->ops->hdmi.lost_hotplug)
+			dssdev->ops->hdmi.lost_hotplug(dssdev);
+	}
+}
+
 static void omap_connector_hpd_cb(void *cb_data,
 				  enum drm_connector_status status)
 {
@@ -46,8 +68,31 @@
 	connector->status = status;
 	mutex_unlock(&dev->mode_config.mutex);
 
-	if (old_status != status)
-		drm_kms_helper_hotplug_event(dev);
+	if (old_status == status)
+		return;
+
+	omap_connector_hpd_notify(connector, status);
+
+	drm_kms_helper_hotplug_event(dev);
+}
+
+void omap_connector_enable_hpd(struct drm_connector *connector)
+{
+	struct omap_connector *omap_connector = to_omap_connector(connector);
+	struct omap_dss_device *hpd = omap_connector->hpd;
+
+	if (hpd)
+		hpd->ops->register_hpd_cb(hpd, omap_connector_hpd_cb,
+					  omap_connector);
+}
+
+void omap_connector_disable_hpd(struct drm_connector *connector)
+{
+	struct omap_connector *omap_connector = to_omap_connector(connector);
+	struct omap_dss_device *hpd = omap_connector->hpd;
+
+	if (hpd)
+		hpd->ops->unregister_hpd_cb(hpd);
 }
 
 bool omap_connector_get_hdmi_mode(struct drm_connector *connector)
@@ -57,176 +102,196 @@
 	return omap_connector->hdmi_mode;
 }
 
+static struct omap_dss_device *
+omap_connector_find_device(struct drm_connector *connector,
+			   enum omap_dss_device_ops_flag op)
+{
+	struct omap_connector *omap_connector = to_omap_connector(connector);
+	struct omap_dss_device *dssdev = NULL;
+	struct omap_dss_device *d;
+
+	for (d = omap_connector->output; d; d = d->next) {
+		if (d->ops_flags & op)
+			dssdev = d;
+	}
+
+	return dssdev;
+}
+
 static enum drm_connector_status omap_connector_detect(
 		struct drm_connector *connector, bool force)
 {
-	struct omap_connector *omap_connector = to_omap_connector(connector);
-	struct omap_dss_device *dssdev = omap_connector->dssdev;
-	struct omap_dss_driver *dssdrv = dssdev->driver;
-	enum drm_connector_status ret;
-
-	if (dssdrv->detect) {
-		if (dssdrv->detect(dssdev))
-			ret = connector_status_connected;
-		else
-			ret = connector_status_disconnected;
-	} else if (dssdev->type == OMAP_DISPLAY_TYPE_DPI ||
-			dssdev->type == OMAP_DISPLAY_TYPE_DBI ||
-			dssdev->type == OMAP_DISPLAY_TYPE_SDI ||
-			dssdev->type == OMAP_DISPLAY_TYPE_DSI) {
-		ret = connector_status_connected;
+	struct omap_dss_device *dssdev;
+	enum drm_connector_status status;
+
+	dssdev = omap_connector_find_device(connector,
+					    OMAP_DSS_DEVICE_OP_DETECT);
+
+	if (dssdev) {
+		status = dssdev->ops->detect(dssdev)
+		       ? connector_status_connected
+		       : connector_status_disconnected;
+
+		omap_connector_hpd_notify(connector, status);
 	} else {
-		ret = connector_status_unknown;
+		switch (connector->connector_type) {
+		case DRM_MODE_CONNECTOR_DPI:
+		case DRM_MODE_CONNECTOR_LVDS:
+		case DRM_MODE_CONNECTOR_DSI:
+			status = connector_status_connected;
+			break;
+		default:
+			status = connector_status_unknown;
+			break;
+		}
 	}
 
-	VERB("%s: %d (force=%d)", omap_connector->dssdev->name, ret, force);
+	VERB("%s: %d (force=%d)", connector->name, status, force);
 
-	return ret;
+	return status;
 }
 
 static void omap_connector_destroy(struct drm_connector *connector)
 {
 	struct omap_connector *omap_connector = to_omap_connector(connector);
-	struct omap_dss_device *dssdev = omap_connector->dssdev;
 
-	DBG("%s", omap_connector->dssdev->name);
-	if (connector->polled == DRM_CONNECTOR_POLL_HPD &&
-	    dssdev->driver->unregister_hpd_cb) {
-		dssdev->driver->unregister_hpd_cb(dssdev);
+	DBG("%s", connector->name);
+
+	if (omap_connector->hpd) {
+		struct omap_dss_device *hpd = omap_connector->hpd;
+
+		hpd->ops->unregister_hpd_cb(hpd);
+		omapdss_device_put(hpd);
+		omap_connector->hpd = NULL;
 	}
+
 	drm_connector_unregister(connector);
 	drm_connector_cleanup(connector);
-	kfree(omap_connector);
 
-	omap_dss_put_device(dssdev);
+	omapdss_device_put(omap_connector->output);
+
+	kfree(omap_connector);
 }
 
 #define MAX_EDID  512
 
-static int omap_connector_get_modes(struct drm_connector *connector)
+static int omap_connector_get_modes_edid(struct drm_connector *connector,
+					 struct omap_dss_device *dssdev)
 {
 	struct omap_connector *omap_connector = to_omap_connector(connector);
-	struct omap_dss_device *dssdev = omap_connector->dssdev;
-	struct omap_dss_driver *dssdrv = dssdev->driver;
-	struct drm_device *dev = connector->dev;
-	int n = 0;
-
-	DBG("%s", omap_connector->dssdev->name);
-
-	/* if display exposes EDID, then we parse that in the normal way to
-	 * build table of supported modes.. otherwise (ie. fixed resolution
-	 * LCD panels) we just return a single mode corresponding to the
-	 * currently configured timings:
-	 */
-	if (dssdrv->read_edid) {
-		void *edid = kzalloc(MAX_EDID, GFP_KERNEL);
-
-		if (!edid)
-			return 0;
-
-		if ((dssdrv->read_edid(dssdev, edid, MAX_EDID) > 0) &&
-				drm_edid_is_valid(edid)) {
-			drm_connector_update_edid_property(
-					connector, edid);
-			n = drm_add_edid_modes(connector, edid);
-
-			omap_connector->hdmi_mode =
-				drm_detect_hdmi_monitor(edid);
-		} else {
-			drm_connector_update_edid_property(
-					connector, NULL);
-		}
+	enum drm_connector_status status;
+	void *edid;
+	int n;
+
+	status = omap_connector_detect(connector, false);
+	if (status != connector_status_connected)
+		goto no_edid;
+
+	edid = kzalloc(MAX_EDID, GFP_KERNEL);
+	if (!edid)
+		goto no_edid;
 
+	if (dssdev->ops->read_edid(dssdev, edid, MAX_EDID) <= 0 ||
+	    !drm_edid_is_valid(edid)) {
 		kfree(edid);
-	} else {
-		struct drm_display_mode *mode = drm_mode_create(dev);
-		struct videomode vm = {0};
-
-		if (!mode)
-			return 0;
-
-		dssdrv->get_timings(dssdev, &vm);
-
-		drm_display_mode_from_videomode(&vm, mode);
-
-		mode->type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED;
-		drm_mode_set_name(mode);
-		drm_mode_probed_add(connector, mode);
+		goto no_edid;
+	}
 
-		if (dssdrv->get_size) {
-			dssdrv->get_size(dssdev,
-					 &connector->display_info.width_mm,
-					 &connector->display_info.height_mm);
-		}
+	drm_connector_update_edid_property(connector, edid);
+	n = drm_add_edid_modes(connector, edid);
 
-		n = 1;
-	}
+	omap_connector->hdmi_mode = drm_detect_hdmi_monitor(edid);
 
+	kfree(edid);
 	return n;
+
+no_edid:
+	drm_connector_update_edid_property(connector, NULL);
+	return 0;
 }
 
-static int omap_connector_mode_valid(struct drm_connector *connector,
-				 struct drm_display_mode *mode)
+static int omap_connector_get_modes(struct drm_connector *connector)
 {
 	struct omap_connector *omap_connector = to_omap_connector(connector);
-	struct omap_dss_device *dssdev = omap_connector->dssdev;
-	struct omap_dss_driver *dssdrv = dssdev->driver;
-	struct videomode vm = {0};
-	struct drm_device *dev = connector->dev;
-	struct drm_display_mode *new_mode;
-	int r, ret = MODE_BAD;
+	struct omap_dss_device *dssdev;
 
-	drm_display_mode_to_videomode(mode, &vm);
-	mode->vrefresh = drm_mode_vrefresh(mode);
+	DBG("%s", connector->name);
 
 	/*
-	 * if the panel driver doesn't have a check_timings, it's most likely
-	 * a fixed resolution panel, check if the timings match with the
-	 * panel's timings
+	 * If display exposes EDID, then we parse that in the normal way to
+	 * build table of supported modes.
 	 */
-	if (dssdrv->check_timings) {
-		r = dssdrv->check_timings(dssdev, &vm);
-	} else {
-		struct videomode t = {0};
+	dssdev = omap_connector_find_device(connector,
+					    OMAP_DSS_DEVICE_OP_EDID);
+	if (dssdev)
+		return omap_connector_get_modes_edid(connector, dssdev);
 
-		dssdrv->get_timings(dssdev, &t);
+	/*
+	 * Otherwise if the display pipeline reports modes (e.g. with a fixed
+	 * resolution panel or an analog TV output), query it.
+	 */
+	dssdev = omap_connector_find_device(connector,
+					    OMAP_DSS_DEVICE_OP_MODES);
+	if (dssdev)
+		return dssdev->ops->get_modes(dssdev, connector);
 
-		/*
-		 * Ignore the flags, as we don't get them from
-		 * drm_display_mode_to_videomode.
-		 */
-		t.flags = 0;
-
-		if (memcmp(&vm, &t, sizeof(vm)))
-			r = -EINVAL;
-		else
-			r = 0;
-	}
+	/*
+	 * Otherwise if the display pipeline uses a drm_panel, we delegate the
+	 * operation to the panel API.
+	 */
+	if (omap_connector->output->panel)
+		return drm_panel_get_modes(omap_connector->output->panel);
+
+	/*
+	 * We can't retrieve modes, which can happen for instance for a DVI or
+	 * VGA output with the DDC bus unconnected. The KMS core will add the
+	 * default modes.
+	 */
+	return 0;
+}
 
-	if (!r) {
-		/* check if vrefresh is still valid */
-		new_mode = drm_mode_duplicate(dev, mode);
+enum drm_mode_status omap_connector_mode_fixup(struct omap_dss_device *dssdev,
+					const struct drm_display_mode *mode,
+					struct drm_display_mode *adjusted_mode)
+{
+	int ret;
 
-		if (!new_mode)
-			return MODE_BAD;
+	drm_mode_copy(adjusted_mode, mode);
+
+	for (; dssdev; dssdev = dssdev->next) {
+		if (!dssdev->ops->check_timings)
+			continue;
 
-		new_mode->clock = vm.pixelclock / 1000;
-		new_mode->vrefresh = 0;
-		if (mode->vrefresh == drm_mode_vrefresh(new_mode))
-			ret = MODE_OK;
-		drm_mode_destroy(dev, new_mode);
+		ret = dssdev->ops->check_timings(dssdev, adjusted_mode);
+		if (ret)
+			return MODE_BAD;
 	}
 
-	DBG("connector: mode %s: "
-			"%d:\"%s\" %d %d %d %d %d %d %d %d %d %d 0x%x 0x%x",
-			(ret == MODE_OK) ? "valid" : "invalid",
-			mode->base.id, mode->name, mode->vrefresh, mode->clock,
-			mode->hdisplay, mode->hsync_start,
-			mode->hsync_end, mode->htotal,
-			mode->vdisplay, mode->vsync_start,
-			mode->vsync_end, mode->vtotal, mode->type, mode->flags);
+	return MODE_OK;
+}
+
+static enum drm_mode_status omap_connector_mode_valid(struct drm_connector *connector,
+				 struct drm_display_mode *mode)
+{
+	struct omap_connector *omap_connector = to_omap_connector(connector);
+	struct drm_display_mode new_mode = { { 0 } };
+	enum drm_mode_status status;
+
+	status = omap_connector_mode_fixup(omap_connector->output, mode,
+					   &new_mode);
+	if (status != MODE_OK)
+		goto done;
+
+	/* Check if vrefresh is still valid. */
+	if (drm_mode_vrefresh(mode) != drm_mode_vrefresh(&new_mode))
+		status = MODE_NOCLOCK;
+
+done:
+	DBG("connector: mode %s: " DRM_MODE_FMT,
+			(status == MODE_OK) ? "valid" : "invalid",
+			DRM_MODE_ARG(mode));
 
-	return ret;
+	return status;
 }
 
 static const struct drm_connector_funcs omap_connector_funcs = {
@@ -243,52 +308,77 @@
 	.mode_valid = omap_connector_mode_valid,
 };
 
+static int omap_connector_get_type(struct omap_dss_device *output)
+{
+	struct omap_dss_device *display;
+	enum omap_display_type type;
+
+	display = omapdss_display_get(output);
+	type = display->type;
+	omapdss_device_put(display);
+
+	switch (type) {
+	case OMAP_DISPLAY_TYPE_HDMI:
+		return DRM_MODE_CONNECTOR_HDMIA;
+	case OMAP_DISPLAY_TYPE_DVI:
+		return DRM_MODE_CONNECTOR_DVID;
+	case OMAP_DISPLAY_TYPE_DSI:
+		return DRM_MODE_CONNECTOR_DSI;
+	case OMAP_DISPLAY_TYPE_DPI:
+	case OMAP_DISPLAY_TYPE_DBI:
+		return DRM_MODE_CONNECTOR_DPI;
+	case OMAP_DISPLAY_TYPE_VENC:
+		/* TODO: This could also be composite */
+		return DRM_MODE_CONNECTOR_SVIDEO;
+	case OMAP_DISPLAY_TYPE_SDI:
+		return DRM_MODE_CONNECTOR_LVDS;
+	default:
+		return DRM_MODE_CONNECTOR_Unknown;
+	}
+}
+
 /* initialize connector */
 struct drm_connector *omap_connector_init(struct drm_device *dev,
-		int connector_type, struct omap_dss_device *dssdev,
-		struct drm_encoder *encoder)
+					  struct omap_dss_device *output,
+					  struct drm_encoder *encoder)
 {
 	struct drm_connector *connector = NULL;
 	struct omap_connector *omap_connector;
-	bool hpd_supported = false;
-
-	DBG("%s", dssdev->name);
+	struct omap_dss_device *dssdev;
 
-	omap_dss_get_device(dssdev);
+	DBG("%s", output->name);
 
 	omap_connector = kzalloc(sizeof(*omap_connector), GFP_KERNEL);
 	if (!omap_connector)
 		goto fail;
 
-	omap_connector->dssdev = dssdev;
+	omap_connector->output = omapdss_device_get(output);
 
 	connector = &omap_connector->base;
+	connector->interlace_allowed = 1;
+	connector->doublescan_allowed = 0;
 
 	drm_connector_init(dev, connector, &omap_connector_funcs,
-				connector_type);
+			   omap_connector_get_type(output));
 	drm_connector_helper_add(connector, &omap_connector_helper_funcs);
 
-	if (dssdev->driver->register_hpd_cb) {
-		int ret = dssdev->driver->register_hpd_cb(dssdev,
-							  omap_connector_hpd_cb,
-							  omap_connector);
-		if (!ret)
-			hpd_supported = true;
-		else if (ret != -ENOTSUPP)
-			DBG("%s: Failed to register HPD callback (%d).",
-			    dssdev->name, ret);
-	}
-
-	if (hpd_supported)
+	/*
+	 * Initialize connector status handling. First try to find a device that
+	 * supports hot-plug reporting. If it fails, fall back to a device that
+	 * support polling. If that fails too, we don't support hot-plug
+	 * detection at all.
+	 */
+	dssdev = omap_connector_find_device(connector, OMAP_DSS_DEVICE_OP_HPD);
+	if (dssdev) {
+		omap_connector->hpd = omapdss_device_get(dssdev);
 		connector->polled = DRM_CONNECTOR_POLL_HPD;
-	else if (dssdev->driver->detect)
-		connector->polled = DRM_CONNECTOR_POLL_CONNECT |
-				    DRM_CONNECTOR_POLL_DISCONNECT;
-	else
-		connector->polled = 0;
-
-	connector->interlace_allowed = 1;
-	connector->doublescan_allowed = 0;
+	} else {
+		dssdev = omap_connector_find_device(connector,
+						    OMAP_DSS_DEVICE_OP_DETECT);
+		if (dssdev)
+			connector->polled = DRM_CONNECTOR_POLL_CONNECT |
+					    DRM_CONNECTOR_POLL_DISCONNECT;
+	}
 
 	return connector;
 
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/omap_connector.h b/drivers/gpu/drm/omapdrm/omap_connector.h
--- a/drivers/gpu/drm/omapdrm/omap_connector.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/omap_connector.h	2020-06-19 21:00:31.412227715 -0400
@@ -22,16 +22,21 @@
 
 #include <linux/types.h>
 
+enum drm_mode_status;
+
 struct drm_connector;
 struct drm_device;
 struct drm_encoder;
 struct omap_dss_device;
 
 struct drm_connector *omap_connector_init(struct drm_device *dev,
-		int connector_type, struct omap_dss_device *dssdev,
-		struct drm_encoder *encoder);
-struct drm_encoder *omap_connector_attached_encoder(
-		struct drm_connector *connector);
+					  struct omap_dss_device *output,
+					  struct drm_encoder *encoder);
 bool omap_connector_get_hdmi_mode(struct drm_connector *connector);
+void omap_connector_enable_hpd(struct drm_connector *connector);
+void omap_connector_disable_hpd(struct drm_connector *connector);
+enum drm_mode_status omap_connector_mode_fixup(struct omap_dss_device *dssdev,
+					const struct drm_display_mode *mode,
+					struct drm_display_mode *adjusted_mode);
 
 #endif /* __OMAPDRM_CONNECTOR_H__ */
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/omap_crtc.c b/drivers/gpu/drm/omapdrm/omap_crtc.c
--- a/drivers/gpu/drm/omapdrm/omap_crtc.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/omap_crtc.c	2020-06-19 21:00:31.412227715 -0400
@@ -33,6 +33,11 @@
 	/* Shadow values for legacy userspace support. */
 	unsigned int rotation;
 	unsigned int zpos;
+
+	u32 default_color;
+	unsigned int trans_key_mode;
+	unsigned int trans_key;
+	bool alpha_blender_enabled;
 };
 
 #define to_omap_crtc(x) container_of(x, struct omap_crtc, base)
@@ -41,6 +46,7 @@
 	struct drm_crtc base;
 
 	const char *name;
+	struct omap_drm_pipeline *pipe;
 	enum omap_channel channel;
 
 	struct videomode vm;
@@ -108,38 +114,7 @@
  * job of sequencing the setup of the video pipe in the proper order
  */
 
-/* ovl-mgr-id -> crtc */
-static struct omap_crtc *omap_crtcs[8];
-static struct omap_dss_device *omap_crtc_output[8];
-
 /* we can probably ignore these until we support command-mode panels: */
-static int omap_crtc_dss_connect(struct omap_drm_private *priv,
-		enum omap_channel channel,
-		struct omap_dss_device *dst)
-{
-	const struct dispc_ops *dispc_ops = priv->dispc_ops;
-	struct dispc_device *dispc = priv->dispc;
-
-	if (omap_crtc_output[channel])
-		return -EINVAL;
-
-	if (!(dispc_ops->mgr_get_supported_outputs(dispc, channel) & dst->id))
-		return -EINVAL;
-
-	omap_crtc_output[channel] = dst;
-	dst->dispc_channel_connected = true;
-
-	return 0;
-}
-
-static void omap_crtc_dss_disconnect(struct omap_drm_private *priv,
-		enum omap_channel channel,
-		struct omap_dss_device *dst)
-{
-	omap_crtc_output[channel] = NULL;
-	dst->dispc_channel_connected = false;
-}
-
 static void omap_crtc_dss_start_update(struct omap_drm_private *priv,
 				       enum omap_channel channel)
 {
@@ -159,7 +134,7 @@
 	if (WARN_ON(omap_crtc->enabled == enable))
 		return;
 
-	if (omap_crtc_output[channel]->output_type == OMAP_DISPLAY_TYPE_HDMI) {
+	if (omap_crtc->pipe->output->type == OMAP_DISPLAY_TYPE_HDMI) {
 		priv->dispc_ops->mgr_enable(priv->dispc, channel, enable);
 		omap_crtc->enabled = enable;
 		return;
@@ -215,7 +190,8 @@
 static int omap_crtc_dss_enable(struct omap_drm_private *priv,
 				enum omap_channel channel)
 {
-	struct omap_crtc *omap_crtc = omap_crtcs[channel];
+	struct drm_crtc *crtc = priv->channels[channel]->crtc;
+	struct omap_crtc *omap_crtc = to_omap_crtc(crtc);
 
 	priv->dispc_ops->mgr_set_timings(priv->dispc, omap_crtc->channel,
 					 &omap_crtc->vm);
@@ -227,7 +203,8 @@
 static void omap_crtc_dss_disable(struct omap_drm_private *priv,
 				  enum omap_channel channel)
 {
-	struct omap_crtc *omap_crtc = omap_crtcs[channel];
+	struct drm_crtc *crtc = priv->channels[channel]->crtc;
+	struct omap_crtc *omap_crtc = to_omap_crtc(crtc);
 
 	omap_crtc_set_enabled(&omap_crtc->base, false);
 }
@@ -236,7 +213,9 @@
 		enum omap_channel channel,
 		const struct videomode *vm)
 {
-	struct omap_crtc *omap_crtc = omap_crtcs[channel];
+	struct drm_crtc *crtc = priv->channels[channel]->crtc;
+	struct omap_crtc *omap_crtc = to_omap_crtc(crtc);
+
 	DBG("%s", omap_crtc->name);
 	omap_crtc->vm = *vm;
 }
@@ -245,7 +224,8 @@
 		enum omap_channel channel,
 		const struct dss_lcd_mgr_config *config)
 {
-	struct omap_crtc *omap_crtc = omap_crtcs[channel];
+	struct drm_crtc *crtc = priv->channels[channel]->crtc;
+	struct omap_crtc *omap_crtc = to_omap_crtc(crtc);
 
 	DBG("%s", omap_crtc->name);
 	priv->dispc_ops->mgr_set_lcd_config(priv->dispc, omap_crtc->channel,
@@ -266,8 +246,6 @@
 }
 
 static const struct dss_mgr_ops mgr_ops = {
-	.connect = omap_crtc_dss_connect,
-	.disconnect = omap_crtc_dss_disconnect,
 	.start_update = omap_crtc_dss_start_update,
 	.enable = omap_crtc_dss_enable,
 	.disable = omap_crtc_dss_disable,
@@ -330,18 +308,72 @@
 	DBG("%s: apply done", omap_crtc->name);
 }
 
+static s16 omap_crtc_S31_32_to_s2_8(s64 coef)
+{
+	uint64_t sign_bit = 1ULL << 63;
+	uint64_t cbits = (uint64_t) coef;
+	s16 ret = clamp_val(((cbits & ~sign_bit) >> 24), 0, 0x1FF);
+
+	if (cbits & sign_bit)
+		ret = -ret;
+
+	return ret;
+}
+
+static void omap_crtc_cpr_coefs_from_ctm(const struct drm_color_ctm *ctm,
+					 struct omap_dss_cpr_coefs *cpr)
+{
+	cpr->rr = omap_crtc_S31_32_to_s2_8(ctm->matrix[0]);
+	cpr->rg = omap_crtc_S31_32_to_s2_8(ctm->matrix[1]);
+	cpr->rb = omap_crtc_S31_32_to_s2_8(ctm->matrix[2]);
+	cpr->gr = omap_crtc_S31_32_to_s2_8(ctm->matrix[3]);
+	cpr->gg = omap_crtc_S31_32_to_s2_8(ctm->matrix[4]);
+	cpr->gb = omap_crtc_S31_32_to_s2_8(ctm->matrix[5]);
+	cpr->br = omap_crtc_S31_32_to_s2_8(ctm->matrix[6]);
+	cpr->bg = omap_crtc_S31_32_to_s2_8(ctm->matrix[7]);
+	cpr->bb = omap_crtc_S31_32_to_s2_8(ctm->matrix[8]);
+}
+
 static void omap_crtc_write_crtc_properties(struct drm_crtc *crtc)
 {
 	struct omap_drm_private *priv = crtc->dev->dev_private;
 	struct omap_crtc *omap_crtc = to_omap_crtc(crtc);
 	struct omap_overlay_manager_info info;
+	const struct omap_crtc_state *omap_state =
+		to_omap_crtc_state(crtc->state);
 
 	memset(&info, 0, sizeof(info));
 
-	info.default_color = 0x000000;
-	info.trans_enabled = false;
-	info.partial_alpha_enabled = false;
-	info.cpr_enable = false;
+	info.default_color = omap_state->default_color;
+
+	info.trans_key = omap_state->trans_key;
+
+	switch (omap_state->trans_key_mode) {
+	case 0:
+	default:
+		info.trans_enabled = false;
+		break;
+	case 1:
+		info.trans_enabled = true;
+		info.trans_key_type = OMAP_DSS_COLOR_KEY_GFX_DST;
+		break;
+	case 2:
+		info.trans_enabled = true;
+		info.trans_key_type = OMAP_DSS_COLOR_KEY_VID_SRC;
+		break;
+	}
+
+	info.alpha_blender_enabled = omap_state->alpha_blender_enabled;
+
+	if (crtc->state->ctm) {
+		struct drm_color_ctm *ctm =
+			(struct drm_color_ctm *) crtc->state->ctm->data;
+
+		info.cpr_enable = true;
+		omap_crtc_cpr_coefs_from_ctm(ctm, &info.cpr_coefs);
+	} else {
+		info.cpr_enable = false;
+	}
 
 	priv->dispc_ops->mgr_setup(priv->dispc, omap_crtc->channel, &info);
 }
@@ -377,11 +409,14 @@
 static void omap_crtc_atomic_enable(struct drm_crtc *crtc,
 				    struct drm_crtc_state *old_state)
 {
+	struct omap_drm_private *priv = crtc->dev->dev_private;
 	struct omap_crtc *omap_crtc = to_omap_crtc(crtc);
 	int ret;
 
 	DBG("%s", omap_crtc->name);
 
+	priv->dispc_ops->runtime_get(priv->dispc);
+
 	spin_lock_irq(&crtc->dev->event_lock);
 	drm_crtc_vblank_on(crtc);
 	ret = drm_crtc_vblank_get(crtc);
@@ -394,6 +429,7 @@
 static void omap_crtc_atomic_disable(struct drm_crtc *crtc,
 				     struct drm_crtc_state *old_state)
 {
+	struct omap_drm_private *priv = crtc->dev->dev_private;
 	struct omap_crtc *omap_crtc = to_omap_crtc(crtc);
 
 	DBG("%s", omap_crtc->name);
@@ -406,12 +442,23 @@
 	spin_unlock_irq(&crtc->dev->event_lock);
 
 	drm_crtc_vblank_off(crtc);
+
+	priv->dispc_ops->runtime_put(priv->dispc);
 }
 
 static enum drm_mode_status omap_crtc_mode_valid(struct drm_crtc *crtc,
 					const struct drm_display_mode *mode)
 {
 	struct omap_drm_private *priv = crtc->dev->dev_private;
+	struct omap_crtc *omap_crtc = to_omap_crtc(crtc);
+	struct videomode vm = {0};
+	int r;
+
+	drm_display_mode_to_videomode(mode, &vm);
+	r = priv->dispc_ops->mgr_check_timings(priv->dispc, omap_crtc->channel,
+					       &vm);
+	if (r)
+		return r;
 
 	/* Check for bandwidth limit */
 	if (priv->max_bandwidth) {
@@ -447,57 +494,17 @@
 {
 	struct omap_crtc *omap_crtc = to_omap_crtc(crtc);
 	struct drm_display_mode *mode = &crtc->state->adjusted_mode;
-	struct omap_drm_private *priv = crtc->dev->dev_private;
-	const u32 flags_mask = DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_DE_LOW |
-		DISPLAY_FLAGS_PIXDATA_POSEDGE | DISPLAY_FLAGS_PIXDATA_NEGEDGE |
-		DISPLAY_FLAGS_SYNC_POSEDGE | DISPLAY_FLAGS_SYNC_NEGEDGE;
-	unsigned int i;
-
-	DBG("%s: set mode: %d:\"%s\" %d %d %d %d %d %d %d %d %d %d 0x%x 0x%x",
-	    omap_crtc->name, mode->base.id, mode->name,
-	    mode->vrefresh, mode->clock,
-	    mode->hdisplay, mode->hsync_start, mode->hsync_end, mode->htotal,
-	    mode->vdisplay, mode->vsync_start, mode->vsync_end, mode->vtotal,
-	    mode->type, mode->flags);
-
-	drm_display_mode_to_videomode(mode, &omap_crtc->vm);
-
-	/*
-	 * HACK: This fixes the vm flags.
-	 * struct drm_display_mode does not contain the VSYNC/HSYNC/DE flags
-	 * and they get lost when converting back and forth between
-	 * struct drm_display_mode and struct videomode. The hack below
-	 * goes and fetches the missing flags from the panel drivers.
-	 *
-	 * Correct solution would be to use DRM's bus-flags, but that's not
-	 * easily possible before the omapdrm's panel/encoder driver model
-	 * has been changed to the DRM model.
-	 */
-
-	for (i = 0; i < priv->num_encoders; ++i) {
-		struct drm_encoder *encoder = priv->encoders[i];
-
-		if (encoder->crtc == crtc) {
-			struct omap_dss_device *dssdev;
 
-			dssdev = omap_encoder_get_dssdev(encoder);
+	DBG("%s: set mode: " DRM_MODE_FMT,
+	    omap_crtc->name, DRM_MODE_ARG(mode));
 
-			if (dssdev) {
-				struct videomode vm = {0};
-
-				dssdev->driver->get_timings(dssdev, &vm);
-
-				omap_crtc->vm.flags |= vm.flags & flags_mask;
-			}
-
-			break;
-		}
-	}
+	drm_display_mode_to_videomode(mode, &omap_crtc->vm);
 }
 
 static int omap_crtc_atomic_check(struct drm_crtc *crtc,
 				struct drm_crtc_state *state)
 {
+	const struct omap_crtc_state *omap_state = to_omap_crtc_state(state);
 	struct drm_plane_state *pri_state;
 
 	if (state->color_mgmt_changed && state->gamma_lut) {
@@ -508,6 +515,25 @@
 			return -EINVAL;
 	}
 
+	if (omap_state->trans_key_mode) {
+		struct drm_plane *plane;
+		struct drm_plane_state *plane_state;
+		u32 zpos_mask = 0;
+
+		drm_for_each_plane_mask(plane, crtc->dev, state->plane_mask) {
+			plane_state = drm_atomic_get_plane_state(state->state,
+								 plane);
+			if (IS_ERR(plane_state))
+				return PTR_ERR(plane_state);
+
+			if (zpos_mask & BIT(plane_state->zpos))
+				return -EINVAL;
+
+			zpos_mask |= BIT(plane_state->zpos);
+			plane_state->normalized_zpos = plane_state->zpos;
+		}
+	}
+
 	pri_state = drm_atomic_get_new_plane_state(state->state, crtc->primary);
 	if (pri_state) {
 		struct omap_crtc_state *omap_crtc_state =
@@ -571,6 +597,7 @@
 {
 	struct omap_drm_private *priv = crtc->dev->dev_private;
 	struct drm_plane_state *plane_state;
+	struct omap_crtc_state *omap_state = to_omap_crtc_state(state);
 
 	/*
 	 * Delegate property set to the primary plane. Get the plane state and
@@ -586,6 +613,14 @@
 		plane_state->rotation = val;
 	else if (property == priv->zorder_prop)
 		plane_state->zpos = val;
+	else if (property == priv->background_color_prop)
+		omap_state->default_color = val;
+	else if (property == priv->trans_key_mode_prop)
+		omap_state->trans_key_mode = val;
+	else if (property == priv->trans_key_prop)
+		omap_state->trans_key = val;
+	else if (property == priv->alpha_blender_prop)
+		omap_state->alpha_blender_enabled = !!val;
 	else
 		return -EINVAL;
 
@@ -604,12 +639,28 @@
 		*val = omap_state->rotation;
 	else if (property == priv->zorder_prop)
 		*val = omap_state->zpos;
+	else if (property == priv->background_color_prop)
+		*val = omap_state->default_color;
+	else if (property == priv->trans_key_mode_prop)
+		*val = omap_state->trans_key_mode;
+	else if (property == priv->trans_key_prop)
+		*val = omap_state->trans_key;
+	else if (property == priv->alpha_blender_prop)
+		*val = omap_state->alpha_blender_enabled;
 	else
 		return -EINVAL;
 
 	return 0;
 }
 
+int omap_crtc_atomic_get_trans_key_mode(struct drm_crtc *crtc,
+					const struct drm_crtc_state *state)
+{
+	struct omap_crtc_state *omap_state = to_omap_crtc_state(state);
+
+	return omap_state->trans_key_mode;
+}
+
 static void omap_crtc_reset(struct drm_crtc *crtc)
 {
 	if (crtc->state)
@@ -641,6 +692,12 @@
 	state->zpos = current_state->zpos;
 	state->rotation = current_state->rotation;
 
+	state->default_color = current_state->default_color;
+
+	state->trans_key_mode = current_state->trans_key_mode;
+	state->trans_key = current_state->trans_key;
+	state->alpha_blender_enabled = current_state->alpha_blender_enabled;
+
 	return &state->base;
 }
 
@@ -681,37 +738,41 @@
 
 void omap_crtc_pre_init(struct omap_drm_private *priv)
 {
-	memset(omap_crtcs, 0, sizeof(omap_crtcs));
+	dss_install_mgr_ops(priv->dss, &mgr_ops, priv);
+}
 
-	dss_install_mgr_ops(&mgr_ops, priv);
+void omap_crtc_pre_uninit(struct omap_drm_private *priv)
+{
+	dss_uninstall_mgr_ops(priv->dss);
 }
 
-void omap_crtc_pre_uninit(void)
+static void omap_crtc_install_properties(struct drm_crtc *crtc)
 {
-	dss_uninstall_mgr_ops();
+	struct drm_device *dev = crtc->dev;
+	struct drm_mode_object *obj = &crtc->base;
+	struct omap_drm_private *priv = dev->dev_private;
+
+	drm_object_attach_property(obj, priv->background_color_prop, 0);
+	drm_object_attach_property(obj, priv->trans_key_mode_prop, 0);
+	drm_object_attach_property(obj, priv->trans_key_prop, 0);
+	drm_object_attach_property(obj, priv->alpha_blender_prop, 0);
 }
 
 /* initialize crtc */
 struct drm_crtc *omap_crtc_init(struct drm_device *dev,
-		struct drm_plane *plane, struct omap_dss_device *dssdev)
+				struct omap_drm_pipeline *pipe,
+				struct drm_plane *plane)
 {
 	struct omap_drm_private *priv = dev->dev_private;
 	struct drm_crtc *crtc = NULL;
 	struct omap_crtc *omap_crtc;
 	enum omap_channel channel;
-	struct omap_dss_device *out;
 	int ret;
 
-	out = omapdss_find_output_from_display(dssdev);
-	channel = out->dispc_channel;
-	omap_dss_put_device(out);
+	channel = pipe->output->dispc_channel;
 
 	DBG("%s", channel_names[channel]);
 
-	/* Multiple displays on same channel is not allowed */
-	if (WARN_ON(omap_crtcs[channel] != NULL))
-		return ERR_PTR(-EINVAL);
-
 	omap_crtc = kzalloc(sizeof(*omap_crtc), GFP_KERNEL);
 	if (!omap_crtc)
 		return ERR_PTR(-ENOMEM);
@@ -720,6 +781,7 @@
 
 	init_waitqueue_head(&omap_crtc->pending_wait);
 
+	omap_crtc->pipe = pipe;
 	omap_crtc->channel = channel;
 	omap_crtc->name = channel_names[channel];
 
@@ -727,7 +789,7 @@
 					&omap_crtc_funcs, NULL);
 	if (ret < 0) {
 		dev_err(dev->dev, "%s(): could not init crtc for: %s\n",
-			__func__, dssdev->name);
+			__func__, pipe->output->name);
 		kfree(omap_crtc);
 		return ERR_PTR(ret);
 	}
@@ -744,13 +806,12 @@
 	if (priv->dispc_ops->mgr_gamma_size(priv->dispc, channel)) {
 		unsigned int gamma_lut_size = 256;
 
-		drm_crtc_enable_color_mgmt(crtc, 0, false, gamma_lut_size);
+		drm_crtc_enable_color_mgmt(crtc, 0, true, gamma_lut_size);
 		drm_mode_crtc_set_gamma_size(crtc, gamma_lut_size);
 	}
 
+	omap_crtc_install_properties(crtc);
 	omap_plane_install_properties(crtc->primary, &crtc->base);
 
-	omap_crtcs[channel] = omap_crtc;
-
 	return crtc;
 }
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/omap_crtc.h b/drivers/gpu/drm/omapdrm/omap_crtc.h
--- a/drivers/gpu/drm/omapdrm/omap_crtc.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/omap_crtc.h	2020-06-19 21:00:31.412227715 -0400
@@ -27,17 +27,22 @@
 struct drm_crtc;
 struct drm_device;
 struct drm_plane;
+struct omap_drm_pipeline;
 struct omap_dss_device;
 struct videomode;
 
 struct videomode *omap_crtc_timings(struct drm_crtc *crtc);
 enum omap_channel omap_crtc_channel(struct drm_crtc *crtc);
 void omap_crtc_pre_init(struct omap_drm_private *priv);
-void omap_crtc_pre_uninit(void);
+void omap_crtc_pre_uninit(struct omap_drm_private *priv);
 struct drm_crtc *omap_crtc_init(struct drm_device *dev,
-		struct drm_plane *plane, struct omap_dss_device *dssdev);
+				struct omap_drm_pipeline *pipe,
+				struct drm_plane *plane);
 int omap_crtc_wait_pending(struct drm_crtc *crtc);
 void omap_crtc_error_irq(struct drm_crtc *crtc, u32 irqstatus);
 void omap_crtc_vblank_irq(struct drm_crtc *crtc);
+int omap_crtc_atomic_get_trans_key_mode(struct drm_crtc *crtc,
+					const struct drm_crtc_state *state);
+
 
 #endif /* __OMAPDRM_CRTC_H__ */
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/omap_dmm_priv.h b/drivers/gpu/drm/omapdrm/omap_dmm_priv.h
--- a/drivers/gpu/drm/omapdrm/omap_dmm_priv.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/omap_dmm_priv.h	2020-06-19 21:00:31.412227715 -0400
@@ -159,6 +159,7 @@
 
 struct dmm {
 	struct device *dev;
+	dma_addr_t phys_base;
 	void __iomem *base;
 	int irq;
 
@@ -189,6 +190,12 @@
 	struct list_head alloc_head;
 
 	const struct dmm_platform_data *plat_data;
+
+	bool dmm_workaround;
+	spinlock_t wa_lock;
+	u32 *wa_dma_data;
+	dma_addr_t wa_dma_handle;
+	struct dma_chan *wa_dma_chan;
 };
 
 #endif
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/omap_dmm_tiler.c b/drivers/gpu/drm/omapdrm/omap_dmm_tiler.c
--- a/drivers/gpu/drm/omapdrm/omap_dmm_tiler.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/omap_dmm_tiler.c	2020-06-19 21:00:31.412227715 -0400
@@ -18,6 +18,7 @@
 #include <linux/completion.h>
 #include <linux/delay.h>
 #include <linux/dma-mapping.h>
+#include <linux/dmaengine.h>
 #include <linux/errno.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
@@ -79,14 +80,138 @@
 			DMM_PAT_DESCR__2, DMM_PAT_DESCR__3},
 };
 
+static int dmm_dma_copy(struct dmm *dmm, dma_addr_t src, dma_addr_t dst)
+{
+	struct dma_device *dma_dev = dmm->wa_dma_chan->device;
+	struct dma_async_tx_descriptor *tx;
+	enum dma_status status;
+	dma_cookie_t cookie;
+
+	tx = dma_dev->device_prep_dma_memcpy(dmm->wa_dma_chan, dst, src, 4, 0);
+	if (!tx) {
+		dev_err(dmm->dev, "Failed to prepare DMA memcpy\n");
+		return -EIO;
+	}
+
+	cookie = tx->tx_submit(tx);
+	if (dma_submit_error(cookie)) {
+		dev_err(dmm->dev, "Failed to do DMA tx_submit\n");
+		return -EIO;
+	}
+
+	dma_async_issue_pending(dmm->wa_dma_chan);
+	status = dma_sync_wait(dmm->wa_dma_chan, cookie);
+	if (status != DMA_COMPLETE)
+		dev_err(dmm->dev, "i878 wa DMA copy failure\n");
+
+	dmaengine_terminate_all(dmm->wa_dma_chan);
+	return 0;
+}
+
+static u32 dmm_read_wa(struct dmm *dmm, u32 reg)
+{
+	dma_addr_t src, dst;
+	int r;
+
+	src = dmm->phys_base + reg;
+	dst = dmm->wa_dma_handle;
+
+	r = dmm_dma_copy(dmm, src, dst);
+	if (r) {
+		dev_err(dmm->dev, "sDMA read transfer timeout\n");
+		return readl(dmm->base + reg);
+	}
+
+	/*
+	 * As per i878 workaround, the DMA is used to access the DMM registers.
+	 * Make sure that the readl is not moved by the compiler or the CPU
+	 * earlier than the DMA finished writing the value to memory.
+	 */
+	rmb();
+	return readl(dmm->wa_dma_data);
+}
+
+static void dmm_write_wa(struct dmm *dmm, u32 val, u32 reg)
+{
+	dma_addr_t src, dst;
+	int r;
+
+	writel(val, dmm->wa_dma_data);
+	/*
+	 * As per i878 workaround, the DMA is used to access the DMM registers.
+	 * Make sure that the writel is not moved by the compiler or the CPU, so
+	 * the data will be in place before we start the DMA to do the actual
+	 * register write.
+	 */
+	wmb();
+
+	src = dmm->wa_dma_handle;
+	dst = dmm->phys_base + reg;
+
+	r = dmm_dma_copy(dmm, src, dst);
+	if (r) {
+		dev_err(dmm->dev, "sDMA write transfer timeout\n");
+		writel(val, dmm->base + reg);
+	}
+}
+
 static u32 dmm_read(struct dmm *dmm, u32 reg)
 {
-	return readl(dmm->base + reg);
+	if (dmm->dmm_workaround) {
+		u32 v;
+		unsigned long flags;
+
+		spin_lock_irqsave(&dmm->wa_lock, flags);
+		v = dmm_read_wa(dmm, reg);
+		spin_unlock_irqrestore(&dmm->wa_lock, flags);
+
+		return v;
+	} else {
+		return readl(dmm->base + reg);
+	}
 }
 
 static void dmm_write(struct dmm *dmm, u32 val, u32 reg)
 {
-	writel(val, dmm->base + reg);
+	if (dmm->dmm_workaround) {
+		unsigned long flags;
+
+		spin_lock_irqsave(&dmm->wa_lock, flags);
+		dmm_write_wa(dmm, val, reg);
+		spin_unlock_irqrestore(&dmm->wa_lock, flags);
+	} else {
+		writel(val, dmm->base + reg);
+	}
+}
+
+static int dmm_workaround_init(struct dmm *dmm)
+{
+	dma_cap_mask_t mask;
+
+	spin_lock_init(&dmm->wa_lock);
+
+	dmm->wa_dma_data = dma_alloc_coherent(dmm->dev,  sizeof(u32),
+					      &dmm->wa_dma_handle, GFP_KERNEL);
+	if (!dmm->wa_dma_data)
+		return -ENOMEM;
+
+	dma_cap_zero(mask);
+	dma_cap_set(DMA_MEMCPY, mask);
+
+	dmm->wa_dma_chan = dma_request_channel(mask, NULL, NULL);
+	if (!dmm->wa_dma_chan) {
+		dma_free_coherent(dmm->dev, 4, dmm->wa_dma_data, dmm->wa_dma_handle);
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+static void dmm_workaround_uninit(struct dmm *dmm)
+{
+	dma_release_channel(dmm->wa_dma_chan);
+
+	dma_free_coherent(dmm->dev, 4, dmm->wa_dma_data, dmm->wa_dma_handle);
 }
 
 /* simple allocator to grab next 16 byte aligned memory from txn */
@@ -614,6 +739,10 @@
 	unsigned long flags;
 
 	if (omap_dmm) {
+		/* Disable all enabled interrupts */
+		dmm_write(omap_dmm, 0x7e7e7e7e, DMM_PAT_IRQENABLE_CLR);
+		free_irq(omap_dmm->irq, omap_dmm);
+
 		/* free all area regions */
 		spin_lock_irqsave(&list_lock, flags);
 		list_for_each_entry_safe(block, _block, &omap_dmm->alloc_head,
@@ -636,8 +765,8 @@
 		if (omap_dmm->dummy_page)
 			__free_page(omap_dmm->dummy_page);
 
-		if (omap_dmm->irq > 0)
-			free_irq(omap_dmm->irq, omap_dmm);
+		if (omap_dmm->dmm_workaround)
+			dmm_workaround_uninit(omap_dmm);
 
 		iounmap(omap_dmm->base);
 		kfree(omap_dmm);
@@ -684,6 +813,7 @@
 		goto fail;
 	}
 
+	omap_dmm->phys_base = mem->start;
 	omap_dmm->base = ioremap(mem->start, SZ_2K);
 
 	if (!omap_dmm->base) {
@@ -699,6 +829,22 @@
 
 	omap_dmm->dev = &dev->dev;
 
+	if (of_machine_is_compatible("ti,dra7")) {
+		/*
+		 * DRA7 Errata i878 says that MPU should not be used to access
+		 * RAM and DMM at the same time. As it's not possible to prevent
+		 * MPU accessing RAM, we need to access DMM via a proxy.
+		 */
+		if (!dmm_workaround_init(omap_dmm)) {
+			omap_dmm->dmm_workaround = true;
+			dev_info(&dev->dev,
+				"workaround for errata i878 in use\n");
+		} else {
+			dev_warn(&dev->dev,
+				 "failed to initialize work-around for i878\n");
+		}
+	}
+
 	hwinfo = dmm_read(omap_dmm, DMM_PAT_HWINFO);
 	omap_dmm->num_engines = (hwinfo >> 24) & 0x1F;
 	omap_dmm->num_lut = (hwinfo >> 16) & 0x1F;
@@ -725,24 +871,6 @@
 	dmm_write(omap_dmm, 0x88888888, DMM_TILER_OR__0);
 	dmm_write(omap_dmm, 0x88888888, DMM_TILER_OR__1);
 
-	ret = request_irq(omap_dmm->irq, omap_dmm_irq_handler, IRQF_SHARED,
-				"omap_dmm_irq_handler", omap_dmm);
-
-	if (ret) {
-		dev_err(&dev->dev, "couldn't register IRQ %d, error %d\n",
-			omap_dmm->irq, ret);
-		omap_dmm->irq = -1;
-		goto fail;
-	}
-
-	/* Enable all interrupts for each refill engine except
-	 * ERR_LUT_MISS<n> (which is just advisory, and we don't care
-	 * about because we want to be able to refill live scanout
-	 * buffers for accelerated pan/scroll) and FILL_DSC<n> which
-	 * we just generally don't care about.
-	 */
-	dmm_write(omap_dmm, 0x7e7e7e7e, DMM_PAT_IRQENABLE_SET);
-
 	omap_dmm->dummy_page = alloc_page(GFP_KERNEL | __GFP_DMA32);
 	if (!omap_dmm->dummy_page) {
 		dev_err(&dev->dev, "could not allocate dummy page\n");
@@ -834,6 +962,24 @@
 		.p1.y = omap_dmm->container_height - 1,
 	};
 
+	ret = request_irq(omap_dmm->irq, omap_dmm_irq_handler, IRQF_SHARED,
+				"omap_dmm_irq_handler", omap_dmm);
+
+	if (ret) {
+		dev_err(&dev->dev, "couldn't register IRQ %d, error %d\n",
+			omap_dmm->irq, ret);
+		omap_dmm->irq = -1;
+		goto fail;
+	}
+
+	/* Enable all interrupts for each refill engine except
+	 * ERR_LUT_MISS<n> (which is just advisory, and we don't care
+	 * about because we want to be able to refill live scanout
+	 * buffers for accelerated pan/scroll) and FILL_DSC<n> which
+	 * we just generally don't care about.
+	 */
+	dmm_write(omap_dmm, 0x7e7e7e7e, DMM_PAT_IRQENABLE_SET);
+
 	/* initialize all LUTs to dummy page entries */
 	for (i = 0; i < omap_dmm->num_lut; i++) {
 		area.tcm = omap_dmm->tcm[i];
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/omap_dmm_tiler.h b/drivers/gpu/drm/omapdrm/omap_dmm_tiler.h
--- a/drivers/gpu/drm/omapdrm/omap_dmm_tiler.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/omap_dmm_tiler.h	2020-06-19 21:00:31.412227715 -0400
@@ -113,7 +113,7 @@
 /* GEM bo flags -> tiler fmt */
 static inline enum tiler_fmt gem2fmt(u32 flags)
 {
-	switch (flags & OMAP_BO_TILED) {
+	switch (flags & OMAP_BO_TILED_MASK) {
 	case OMAP_BO_TILED_8:
 		return TILFMT_8BIT;
 	case OMAP_BO_TILED_16:
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/omap_drv.c b/drivers/gpu/drm/omapdrm/omap_drv.c
--- a/drivers/gpu/drm/omapdrm/omap_drv.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/omap_drv.c	2020-06-19 21:00:31.412227715 -0400
@@ -15,12 +15,15 @@
  * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <linux/of.h>
+#include <linux/sort.h>
 #include <linux/sys_soc.h>
 
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_fb_helper.h>
+#include <drm/drm_panel.h>
 
 #include "omap_dmm_tiler.h"
 #include "omap_drv.h"
@@ -32,6 +35,14 @@
 #define DRIVER_MINOR		0
 #define DRIVER_PATCHLEVEL	0
 
+#define MAX_NR_DISPLAYS		8
+static int display_order[MAX_NR_DISPLAYS];
+static int display_order_nelm;
+module_param_array_named(displays, display_order, int, &display_order_nelm,
+			 0444);
+MODULE_PARM_DESC(displays,
+		 "ID array to specify the order of the active displays");
+
 /*
  * mode config funcs
  */
@@ -116,6 +127,98 @@
 	priv->dispc_ops->runtime_put(priv->dispc);
 }
 
+static int drm_atomic_state_normalized_zpos_cmp(const void *a, const void *b)
+{
+	const struct drm_plane_state *sa = *(struct drm_plane_state **)a;
+	const struct drm_plane_state *sb = *(struct drm_plane_state **)b;
+
+	if (sa->normalized_zpos != sb->normalized_zpos)
+		return sa->normalized_zpos - sb->normalized_zpos;
+	else
+		return sa->plane->base.id - sb->plane->base.id;
+}
+
+static int omap_atomic_update_normalize_zpos(struct drm_device *dev,
+					     struct drm_atomic_state *state)
+{
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *old_state, *new_state;
+	struct drm_plane *plane;
+	int c, i, n, inc;
+	int total_planes = dev->mode_config.num_total_plane;
+	struct drm_plane_state **states;
+	int ret = 0;
+
+	states = kmalloc_array(total_planes, sizeof(*states), GFP_KERNEL);
+	if (!states)
+		return -ENOMEM;
+
+	for_each_oldnew_crtc_in_state(state, crtc, old_state, new_state, c) {
+		if (old_state->plane_mask == new_state->plane_mask &&
+		    !new_state->zpos_changed)
+			continue;
+
+		if (omap_crtc_atomic_get_trans_key_mode(crtc, new_state))
+			continue;
+
+		/* Reset plane increment and index value for every crtc */
+		n = 0;
+
+		/*
+		 * Normalization process might create new states for planes
+		 * which normalized_zpos has to be recalculated.
+		 */
+		drm_for_each_plane_mask(plane, dev, new_state->plane_mask) {
+			struct drm_plane_state *plane_state =
+				drm_atomic_get_plane_state(new_state->state,
+							   plane);
+			if (IS_ERR(plane_state)) {
+				ret = PTR_ERR(plane_state);
+				goto done;
+			}
+			states[n++] = plane_state;
+		}
+
+		sort(states, n, sizeof(*states),
+		     drm_atomic_state_normalized_zpos_cmp, NULL);
+
+		for (i = 0, inc = 0; i < n; i++) {
+			plane = states[i]->plane;
+
+			states[i]->normalized_zpos = i + inc;
+			DRM_DEBUG_ATOMIC("[PLANE:%d:%s] updated normalized zpos value %d\n",
+					 plane->base.id, plane->name,
+					 states[i]->normalized_zpos);
+
+			if (is_omap_plane_dual_overlay(states[i]))
+				inc++;
+		}
+		new_state->zpos_changed = true;
+	}
+
+done:
+	kfree(states);
+	return ret;
+}
+
+static int omap_atomic_check(struct drm_device *dev,
+			     struct drm_atomic_state *state)
+{
+	int ret;
+
+	ret = drm_atomic_helper_check(dev, state);
+	if (ret)
+		return ret;
+
+	if (dev->mode_config.normalize_zpos) {
+		ret = omap_atomic_update_normalize_zpos(dev, state);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
 static const struct drm_mode_config_helper_funcs omap_mode_config_helper_funcs = {
 	.atomic_commit_tail = omap_atomic_commit_tail,
 };
@@ -123,69 +226,141 @@
 static const struct drm_mode_config_funcs omap_mode_config_funcs = {
 	.fb_create = omap_framebuffer_create,
 	.output_poll_changed = drm_fb_helper_output_poll_changed,
-	.atomic_check = drm_atomic_helper_check,
+	.atomic_check = omap_atomic_check,
 	.atomic_commit = drm_atomic_helper_commit,
 };
 
-static int get_connector_type(struct omap_dss_device *dssdev)
+/* Global/shared object state funcs */
+
+/*
+ * This is a helper that returns the private state currently in operation.
+ * Note that this would return the "old_state" if called in the atomic check
+ * path, and the "new_state" after the atomic swap has been done.
+ */
+struct omap_global_state *
+omap_get_existing_global_state(struct omap_drm_private *priv)
 {
-	switch (dssdev->type) {
-	case OMAP_DISPLAY_TYPE_HDMI:
-		return DRM_MODE_CONNECTOR_HDMIA;
-	case OMAP_DISPLAY_TYPE_DVI:
-		return DRM_MODE_CONNECTOR_DVID;
-	case OMAP_DISPLAY_TYPE_DSI:
-		return DRM_MODE_CONNECTOR_DSI;
-	case OMAP_DISPLAY_TYPE_DPI:
-	case OMAP_DISPLAY_TYPE_DBI:
-		return DRM_MODE_CONNECTOR_DPI;
-	case OMAP_DISPLAY_TYPE_VENC:
-		/* TODO: This could also be composite */
-		return DRM_MODE_CONNECTOR_SVIDEO;
-	case OMAP_DISPLAY_TYPE_SDI:
-		return DRM_MODE_CONNECTOR_LVDS;
-	default:
-		return DRM_MODE_CONNECTOR_Unknown;
-	}
+	return to_omap_global_state(priv->glob_obj.state);
+}
+
+/*
+ * This acquires the modeset lock set aside for global state, creates
+ * a new duplicated private object state.
+ */
+struct omap_global_state *__must_check
+omap_get_global_state(struct drm_atomic_state *s)
+{
+	struct omap_drm_private *priv = s->dev->dev_private;
+	struct drm_private_state *priv_state;
+
+	priv_state = drm_atomic_get_private_obj_state(s, &priv->glob_obj);
+	if (IS_ERR(priv_state))
+		return ERR_CAST(priv_state);
+
+	return to_omap_global_state(priv_state);
 }
 
-static void omap_disconnect_dssdevs(void)
+static struct drm_private_state *
+omap_global_duplicate_state(struct drm_private_obj *obj)
 {
-	struct omap_dss_device *dssdev = NULL;
+	struct omap_global_state *state;
+
+	state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL);
+	if (!state)
+		return NULL;
 
-	for_each_dss_dev(dssdev)
-		dssdev->driver->disconnect(dssdev);
+	__drm_atomic_helper_private_obj_duplicate_state(obj, &state->base);
+
+	return &state->base;
 }
 
-static int omap_connect_dssdevs(void)
+static void omap_global_destroy_state(struct drm_private_obj *obj,
+				      struct drm_private_state *state)
 {
+	struct omap_global_state *omap_state = to_omap_global_state(state);
+
+	kfree(omap_state);
+}
+
+static const struct drm_private_state_funcs omap_global_state_funcs = {
+	.atomic_duplicate_state = omap_global_duplicate_state,
+	.atomic_destroy_state = omap_global_destroy_state,
+};
+
+static int omap_global_obj_init(struct drm_device *dev)
+{
+	struct omap_drm_private *priv = dev->dev_private;
+	struct omap_global_state *state;
+
+	state = kzalloc(sizeof(*state), GFP_KERNEL);
+	if (!state)
+		return -ENOMEM;
+
+	drm_atomic_private_obj_init(dev, &priv->glob_obj, &state->base,
+				    &omap_global_state_funcs);
+	return 0;
+}
+
+static void omap_global_obj_fini(struct omap_drm_private *priv)
+{
+	drm_atomic_private_obj_fini(&priv->glob_obj);
+}
+
+static void omap_disconnect_pipelines(struct drm_device *ddev)
+{
+	struct omap_drm_private *priv = ddev->dev_private;
+	unsigned int i;
+
+	for (i = 0; i < priv->num_pipes; i++) {
+		struct omap_drm_pipeline *pipe = &priv->pipes[i];
+
+		if (pipe->output->panel)
+			drm_panel_detach(pipe->output->panel);
+
+		omapdss_device_disconnect(NULL, pipe->output);
+
+		omapdss_device_put(pipe->output);
+		pipe->output = NULL;
+	}
+
+	memset(&priv->channels, 0, sizeof(priv->channels));
+
+	priv->num_pipes = 0;
+}
+
+static int omap_connect_pipelines(struct drm_device *ddev)
+{
+	struct omap_drm_private *priv = ddev->dev_private;
+	struct omap_dss_device *output = NULL;
 	int r;
-	struct omap_dss_device *dssdev = NULL;
 
-	if (!omapdss_stack_is_ready())
-		return -EPROBE_DEFER;
+	/* No displays should be enabled */
+	if (display_order_nelm == 1 && display_order[0] < 0)
+		return 0;
 
-	for_each_dss_dev(dssdev) {
-		r = dssdev->driver->connect(dssdev);
+	for_each_dss_output(output) {
+		r = omapdss_device_connect(priv->dss, NULL, output);
 		if (r == -EPROBE_DEFER) {
-			omap_dss_put_device(dssdev);
-			goto cleanup;
+			omapdss_device_put(output);
+			return r;
 		} else if (r) {
-			dev_warn(dssdev->dev, "could not connect display: %s\n",
-				dssdev->name);
+			dev_warn(output->dev, "could not connect output %s\n",
+				 output->name);
+		} else {
+			struct omap_drm_pipeline *pipe;
+
+			pipe = &priv->pipes[priv->num_pipes++];
+			pipe->output = omapdss_device_get(output);
+
+			if (priv->num_pipes == ARRAY_SIZE(priv->pipes)) {
+				/* To balance the 'for_each_dss_output' loop */
+				omapdss_device_put(output);
+				break;
+			}
 		}
 	}
 
 	return 0;
-
-cleanup:
-	/*
-	 * if we are deferring probe, we disconnect the devices we previously
-	 * connected
-	 */
-	omap_disconnect_dssdevs();
-
-	return r;
 }
 
 static int omap_modeset_init_properties(struct drm_device *dev)
@@ -193,25 +368,161 @@
 	struct omap_drm_private *priv = dev->dev_private;
 	unsigned int num_planes = priv->dispc_ops->get_num_ovls(priv->dispc);
 
+	static const struct drm_prop_enum_list trans_key_mode_list[] = {
+		{ 0, "disable"},
+		{ 1, "gfx-dst"},
+		{ 2, "vid-src"},
+	};
+
 	priv->zorder_prop = drm_property_create_range(dev, 0, "zorder", 0,
 						      num_planes - 1);
 	if (!priv->zorder_prop)
 		return -ENOMEM;
 
+	priv->global_alpha_prop = drm_property_create_range(dev, 0,
+		"global_alpha", 0, 255);
+	if (!priv->global_alpha_prop)
+		return -ENOMEM;
+
+	priv->pre_mult_alpha_prop = drm_property_create_bool(dev, 0,
+		"pre_mult_alpha");
+	if (!priv->pre_mult_alpha_prop)
+		return -ENOMEM;
+
+	/* crtc properties */
+
+	priv->background_color_prop = drm_property_create_range(dev, 0,
+		"background", 0, 0xffffff);
+	if (!priv->background_color_prop)
+		return -ENOMEM;
+
+	priv->trans_key_mode_prop = drm_property_create_enum(dev, 0,
+		"trans-key-mode",
+		trans_key_mode_list, ARRAY_SIZE(trans_key_mode_list));
+	if (!priv->trans_key_mode_prop)
+		return -ENOMEM;
+
+	priv->trans_key_prop = drm_property_create_range(dev, 0, "trans-key",
+		0, 0xffffff);
+	if (!priv->trans_key_prop)
+		return -ENOMEM;
+
+	priv->alpha_blender_prop = drm_property_create_bool(dev, 0,
+		"alpha_blender");
+	if (!priv->alpha_blender_prop)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static int omap_display_id(struct omap_dss_device *output)
+{
+	struct device_node *node = NULL;
+
+	if (output->next) {
+		struct omap_dss_device *display;
+
+		display = omapdss_display_get(output);
+		node = display->dev->of_node;
+		omapdss_device_put(display);
+	} else if (output->bridge) {
+		struct drm_bridge *bridge = output->bridge;
+
+		while (bridge->next)
+			bridge = bridge->next;
+
+		node = bridge->of_node;
+	} else if (output->panel) {
+		node = output->panel->dev->of_node;
+	}
+
+	return node ? of_alias_get_id(node, "display") : -ENODEV;
+}
+
+static int omap_compare_pipelines(const void *a, const void *b)
+{
+	const struct omap_drm_pipeline *pipe1 = a;
+	const struct omap_drm_pipeline *pipe2 = b;
+
+	if (pipe1->alias_id > pipe2->alias_id)
+		return 1;
+	else if (pipe1->alias_id < pipe2->alias_id)
+		return -1;
 	return 0;
 }
 
+static void omap_sort_pipes(struct drm_device *ddev)
+{
+	struct omap_drm_private *priv = ddev->dev_private;
+	struct omap_drm_pipeline pipes[ARRAY_SIZE(priv->pipes)];
+	unsigned int num_pipes = 0;
+	unsigned long pipes_mask = 0;
+	int i;
+
+	if (!priv->num_pipes)
+		return;
+
+	/* Sort the pipelines by DT aliases. */
+	sort(priv->pipes, priv->num_pipes, sizeof(priv->pipes[0]),
+	     omap_compare_pipelines, NULL);
+
+	if (!display_order_nelm)
+		return;
+
+	bitmap_set(&pipes_mask, 0, priv->num_pipes);
+
+	/* Do ordering based on the display_order parameter array */
+	for (i = 0; i < display_order_nelm; i++) {
+		int old_index = display_order[i];
+
+		if ((old_index >= 0 && old_index < priv->num_pipes) &&
+		    (pipes_mask & BIT(old_index))) {
+			pipes[num_pipes++] = priv->pipes[old_index];
+			clear_bit(old_index, &pipes_mask);
+		} else {
+			dev_err(ddev->dev,
+				"Ignoring invalid displays module parameter\n");
+			num_pipes = 0;
+			break;
+		}
+	}
+
+	if (num_pipes > 0) {
+		u32 idx;
+
+		/* check if we have dssdev which is not carried over */
+		for_each_set_bit(idx, &pipes_mask, ARRAY_SIZE(priv->pipes)) {
+			struct omap_drm_pipeline *pipe = &priv->pipes[idx];
+
+			if (pipe->output->panel)
+				drm_panel_detach(pipe->output->panel);
+
+			omapdss_device_disconnect(NULL, pipe->output);
+
+			omapdss_device_put(pipe->output);
+			pipe->output = NULL;
+		}
+
+		for (i = 0; i < num_pipes; i++)
+			priv->pipes[i] = pipes[i];
+		for (i = num_pipes; i < priv->num_pipes; i++)
+			priv->pipes[i].output = NULL;
+
+		priv->num_pipes = num_pipes;
+	}
+}
+
 static int omap_modeset_init(struct drm_device *dev)
 {
 	struct omap_drm_private *priv = dev->dev_private;
-	struct omap_dss_device *dssdev = NULL;
 	int num_ovls = priv->dispc_ops->get_num_ovls(priv->dispc);
 	int num_mgrs = priv->dispc_ops->get_num_mgrs(priv->dispc);
-	int num_crtcs, crtc_idx, plane_idx;
+	unsigned int i;
 	int ret;
 	u32 plane_crtc_mask;
 
-	drm_mode_config_init(dev);
+	if (!omapdss_stack_is_ready())
+		return -EPROBE_DEFER;
 
 	ret = omap_modeset_init_properties(dev);
 	if (ret < 0)
@@ -225,87 +536,106 @@
 	 * configuration does not match the expectations or exceeds
 	 * the available resources, the configuration is rejected.
 	 */
-	num_crtcs = 0;
-	for_each_dss_dev(dssdev)
-		if (omapdss_device_is_connected(dssdev))
-			num_crtcs++;
-
-	if (num_crtcs > num_mgrs || num_crtcs > num_ovls ||
-	    num_crtcs > ARRAY_SIZE(priv->crtcs) ||
-	    num_crtcs > ARRAY_SIZE(priv->planes) ||
-	    num_crtcs > ARRAY_SIZE(priv->encoders) ||
-	    num_crtcs > ARRAY_SIZE(priv->connectors)) {
+	ret = omap_connect_pipelines(dev);
+	if (ret < 0)
+		return ret;
+
+	if (priv->num_pipes > num_mgrs || priv->num_pipes > num_ovls) {
 		dev_err(dev->dev, "%s(): Too many connected displays\n",
 			__func__);
 		return -EINVAL;
 	}
 
-	/* All planes can be put to any CRTC */
-	plane_crtc_mask = (1 << num_crtcs) - 1;
-
-	dssdev = NULL;
+	/* Create all planes first. They can all be put to any CRTC. */
+	plane_crtc_mask = (1 << priv->num_pipes) - 1;
 
-	crtc_idx = 0;
-	plane_idx = 0;
-	for_each_dss_dev(dssdev) {
-		struct drm_connector *connector;
-		struct drm_encoder *encoder;
+	for (i = 0; i < num_ovls; i++) {
+		enum drm_plane_type type = i < priv->num_pipes
+					 ? DRM_PLANE_TYPE_PRIMARY
+					 : DRM_PLANE_TYPE_OVERLAY;
 		struct drm_plane *plane;
-		struct drm_crtc *crtc;
-
-		if (!omapdss_device_is_connected(dssdev))
-			continue;
 
-		encoder = omap_encoder_init(dev, dssdev);
-		if (!encoder)
-			return -ENOMEM;
-
-		connector = omap_connector_init(dev,
-				get_connector_type(dssdev), dssdev, encoder);
-		if (!connector)
-			return -ENOMEM;
+		if (WARN_ON(priv->num_planes >= ARRAY_SIZE(priv->planes)))
+			return -EINVAL;
 
-		plane = omap_plane_init(dev, plane_idx, DRM_PLANE_TYPE_PRIMARY,
-					plane_crtc_mask);
+		plane = omap_plane_init(dev, i, type, plane_crtc_mask);
 		if (IS_ERR(plane))
 			return PTR_ERR(plane);
 
-		crtc = omap_crtc_init(dev, plane, dssdev);
-		if (IS_ERR(crtc))
-			return PTR_ERR(crtc);
+		priv->planes[priv->num_planes++] = plane;
+	}
 
-		drm_connector_attach_encoder(connector, encoder);
-		encoder->possible_crtcs = (1 << crtc_idx);
+	/*
+	 * Create the encoders, attach the bridges and get the pipeline alias
+	 * IDs.
+	 */
+	for (i = 0; i < priv->num_pipes; i++) {
+		struct omap_drm_pipeline *pipe = &priv->pipes[i];
+		int id;
 
-		priv->crtcs[priv->num_crtcs++] = crtc;
-		priv->planes[priv->num_planes++] = plane;
-		priv->encoders[priv->num_encoders++] = encoder;
-		priv->connectors[priv->num_connectors++] = connector;
+		pipe->encoder = omap_encoder_init(dev, pipe->output);
+		if (!pipe->encoder)
+			return -ENOMEM;
+
+		if (pipe->output->bridge) {
+			ret = drm_bridge_attach(pipe->encoder,
+						pipe->output->bridge, NULL);
+			if (ret < 0)
+				return ret;
+		}
 
-		plane_idx++;
-		crtc_idx++;
+		id = omap_display_id(pipe->output);
+		pipe->alias_id = id >= 0 ? id : i;
 	}
 
+	omap_sort_pipes(dev);
+
 	/*
-	 * Create normal planes for the remaining overlays:
+	 * Populate the pipeline lookup table by DISPC channel. Only one display
+	 * is allowed per channel.
 	 */
-	for (; plane_idx < num_ovls; plane_idx++) {
-		struct drm_plane *plane;
+	for (i = 0; i < priv->num_pipes; ++i) {
+		struct omap_drm_pipeline *pipe = &priv->pipes[i];
+		enum omap_channel channel = pipe->output->dispc_channel;
 
-		if (WARN_ON(priv->num_planes >= ARRAY_SIZE(priv->planes)))
+		if (WARN_ON(priv->channels[channel] != NULL))
 			return -EINVAL;
 
-		plane = omap_plane_init(dev, plane_idx, DRM_PLANE_TYPE_OVERLAY,
-			plane_crtc_mask);
-		if (IS_ERR(plane))
-			return PTR_ERR(plane);
+		priv->channels[channel] = pipe;
+	}
 
-		priv->planes[priv->num_planes++] = plane;
+	/* Create the connectors and CRTCs. */
+	for (i = 0; i < priv->num_pipes; i++) {
+		struct omap_drm_pipeline *pipe = &priv->pipes[i];
+		struct drm_encoder *encoder = pipe->encoder;
+		struct drm_crtc *crtc;
+
+		if (!pipe->output->bridge) {
+			pipe->connector = omap_connector_init(dev, pipe->output,
+							      encoder);
+			if (!pipe->connector)
+				return -ENOMEM;
+
+			drm_connector_attach_encoder(pipe->connector, encoder);
+
+			if (pipe->output->panel) {
+				ret = drm_panel_attach(pipe->output->panel,
+						       pipe->connector);
+				if (ret < 0)
+					return ret;
+			}
+		}
+
+		crtc = omap_crtc_init(dev, pipe, priv->planes[i]);
+		if (IS_ERR(crtc))
+			return PTR_ERR(crtc);
+
+		encoder->possible_crtcs = 1 << i;
+		pipe->crtc = crtc;
 	}
 
-	DBG("registered %d planes, %d crtcs, %d encoders and %d connectors\n",
-		priv->num_planes, priv->num_crtcs, priv->num_encoders,
-		priv->num_connectors);
+	DBG("registered %u planes, %u crtcs/encoders/connectors\n",
+	    priv->num_planes, priv->num_pipes);
 
 	dev->mode_config.min_width = 8;
 	dev->mode_config.min_height = 2;
@@ -335,26 +665,28 @@
 /*
  * Enable the HPD in external components if supported
  */
-static void omap_modeset_enable_external_hpd(void)
+static void omap_modeset_enable_external_hpd(struct drm_device *ddev)
 {
-	struct omap_dss_device *dssdev = NULL;
+	struct omap_drm_private *priv = ddev->dev_private;
+	unsigned int i;
 
-	for_each_dss_dev(dssdev) {
-		if (dssdev->driver->enable_hpd)
-			dssdev->driver->enable_hpd(dssdev);
+	for (i = 0; i < priv->num_pipes; i++) {
+		if (priv->pipes[i].connector)
+			omap_connector_enable_hpd(priv->pipes[i].connector);
 	}
 }
 
 /*
  * Disable the HPD in external components if supported
  */
-static void omap_modeset_disable_external_hpd(void)
+static void omap_modeset_disable_external_hpd(struct drm_device *ddev)
 {
-	struct omap_dss_device *dssdev = NULL;
+	struct omap_drm_private *priv = ddev->dev_private;
+	unsigned int i;
 
-	for_each_dss_dev(dssdev) {
-		if (dssdev->driver->disable_hpd)
-			dssdev->driver->disable_hpd(dssdev);
+	for (i = 0; i < priv->num_pipes; i++) {
+		if (priv->pipes[i].connector)
+			omap_connector_disable_hpd(priv->pipes[i].connector);
 	}
 }
 
@@ -428,7 +760,7 @@
 	args->size = omap_gem_mmap_size(obj);
 	args->offset = omap_gem_mmap_offset(obj);
 
-	drm_gem_object_unreference_unlocked(obj);
+	drm_gem_object_put_unlocked(obj);
 
 	return ret;
 }
@@ -525,6 +857,14 @@
 
 	DBG("%s", dev_name(dev));
 
+	/* Allocate and initialize the DRM device. */
+	ddev = drm_dev_alloc(&omap_drm_driver, dev);
+	if (IS_ERR(ddev))
+		return PTR_ERR(ddev);
+
+	priv->ddev = ddev;
+	ddev->dev_private = priv;
+
 	priv->dev = dev;
 	priv->dss = omapdss_get_dss();
 	priv->dispc = dispc_get_dispc(priv->dss);
@@ -532,10 +872,6 @@
 
 	omap_crtc_pre_init(priv);
 
-	ret = omap_connect_dssdevs();
-	if (ret)
-		goto err_crtc_uninit;
-
 	soc = soc_device_match(omapdrm_soc_devices);
 	priv->omaprev = soc ? (unsigned int)soc->data : 0;
 	priv->wq = alloc_ordered_workqueue("omapdrm", 0);
@@ -543,16 +879,6 @@
 	mutex_init(&priv->list_lock);
 	INIT_LIST_HEAD(&priv->obj_list);
 
-	/* Allocate and initialize the DRM device. */
-	ddev = drm_dev_alloc(&omap_drm_driver, priv->dev);
-	if (IS_ERR(ddev)) {
-		ret = PTR_ERR(ddev);
-		goto err_destroy_wq;
-	}
-
-	priv->ddev = ddev;
-	ddev->dev_private = priv;
-
 	/* Get memory bandwidth limits */
 	if (priv->dispc_ops->get_memory_bandwidth_limit)
 		priv->max_bandwidth =
@@ -560,26 +886,44 @@
 
 	omap_gem_init(ddev);
 
+	drm_mode_config_init(ddev);
+
+	ret = omap_global_obj_init(ddev);
+	if (ret)
+		goto err_gem_deinit;
+
+	ret = omap_hwoverlays_init(priv);
+	if (ret)
+		goto err_free_priv_obj;
+
 	ret = omap_modeset_init(ddev);
 	if (ret) {
 		dev_err(priv->dev, "omap_modeset_init failed: ret=%d\n", ret);
-		goto err_free_drm_dev;
+		goto err_free_overlays;
 	}
 
 	/* Initialize vblank handling, start with all CRTCs disabled. */
-	ret = drm_vblank_init(ddev, priv->num_crtcs);
+	ret = drm_vblank_init(ddev, priv->num_pipes);
 	if (ret) {
 		dev_err(priv->dev, "could not init vblank\n");
 		goto err_cleanup_modeset;
 	}
 
-	for (i = 0; i < priv->num_crtcs; i++)
-		drm_crtc_vblank_off(priv->crtcs[i]);
+	for (i = 0; i < priv->num_pipes; i++)
+		drm_crtc_vblank_off(priv->pipes[i].crtc);
 
 	omap_fbdev_init(ddev);
 
 	drm_kms_helper_poll_init(ddev);
-	omap_modeset_enable_external_hpd();
+	omap_modeset_enable_external_hpd(ddev);
+
+	if (priv->dispc_ops->has_writeback(priv->dispc)) {
+		ret = omap_wb_init(ddev);
+		if (ret)
+			dev_warn(priv->dev, "failed to initialize writeback\n");
+		else
+			priv->wb_initialized = true;
+	}
 
 	/*
 	 * Register the DRM device with the core and the connectors with
@@ -592,21 +936,27 @@
 	return 0;
 
 err_cleanup_helpers:
-	omap_modeset_disable_external_hpd();
+	if (priv->wb_initialized)
+		omap_wb_cleanup(ddev);
+
+	omap_modeset_disable_external_hpd(ddev);
+
 	drm_kms_helper_poll_fini(ddev);
 
 	omap_fbdev_fini(ddev);
 err_cleanup_modeset:
-	drm_mode_config_cleanup(ddev);
 	omap_drm_irq_uninstall(ddev);
-err_free_drm_dev:
+err_free_overlays:
+	omap_hwoverlays_destroy(priv);
+err_free_priv_obj:
+	omap_global_obj_fini(priv);
+err_gem_deinit:
+	drm_mode_config_cleanup(ddev);
 	omap_gem_deinit(ddev);
-	drm_dev_unref(ddev);
-err_destroy_wq:
 	destroy_workqueue(priv->wq);
-	omap_disconnect_dssdevs();
-err_crtc_uninit:
-	omap_crtc_pre_uninit();
+	omap_disconnect_pipelines(ddev);
+	omap_crtc_pre_uninit(priv);
+	drm_dev_put(ddev);
 	return ret;
 }
 
@@ -618,24 +968,28 @@
 
 	drm_dev_unregister(ddev);
 
-	omap_modeset_disable_external_hpd();
+	if (priv->wb_initialized)
+		omap_wb_cleanup(ddev);
+
+	omap_modeset_disable_external_hpd(ddev);
 	drm_kms_helper_poll_fini(ddev);
 
 	omap_fbdev_fini(ddev);
 
 	drm_atomic_helper_shutdown(ddev);
 
-	drm_mode_config_cleanup(ddev);
-
 	omap_drm_irq_uninstall(ddev);
+	omap_hwoverlays_destroy(priv);
+	omap_global_obj_fini(priv);
+	drm_mode_config_cleanup(ddev);
 	omap_gem_deinit(ddev);
 
-	drm_dev_unref(ddev);
-
 	destroy_workqueue(priv->wq);
 
-	omap_disconnect_dssdevs();
-	omap_crtc_pre_uninit();
+	omap_disconnect_pipelines(ddev);
+	omap_crtc_pre_uninit(priv);
+
+	drm_dev_put(ddev);
 }
 
 static int pdev_probe(struct platform_device *pdev)
@@ -677,54 +1031,12 @@
 }
 
 #ifdef CONFIG_PM_SLEEP
-static int omap_drm_suspend_all_displays(void)
-{
-	struct omap_dss_device *dssdev = NULL;
-
-	for_each_dss_dev(dssdev) {
-		if (!dssdev->driver)
-			continue;
-
-		if (dssdev->state == OMAP_DSS_DISPLAY_ACTIVE) {
-			dssdev->driver->disable(dssdev);
-			dssdev->activate_after_resume = true;
-		} else {
-			dssdev->activate_after_resume = false;
-		}
-	}
-
-	return 0;
-}
-
-static int omap_drm_resume_all_displays(void)
-{
-	struct omap_dss_device *dssdev = NULL;
-
-	for_each_dss_dev(dssdev) {
-		if (!dssdev->driver)
-			continue;
-
-		if (dssdev->activate_after_resume) {
-			dssdev->driver->enable(dssdev);
-			dssdev->activate_after_resume = false;
-		}
-	}
-
-	return 0;
-}
-
 static int omap_drm_suspend(struct device *dev)
 {
 	struct omap_drm_private *priv = dev_get_drvdata(dev);
 	struct drm_device *drm_dev = priv->ddev;
 
-	drm_kms_helper_poll_disable(drm_dev);
-
-	drm_modeset_lock_all(drm_dev);
-	omap_drm_suspend_all_displays();
-	drm_modeset_unlock_all(drm_dev);
-
-	return 0;
+	return drm_mode_config_helper_suspend(drm_dev);
 }
 
 static int omap_drm_resume(struct device *dev)
@@ -732,11 +1044,7 @@
 	struct omap_drm_private *priv = dev_get_drvdata(dev);
 	struct drm_device *drm_dev = priv->ddev;
 
-	drm_modeset_lock_all(drm_dev);
-	omap_drm_resume_all_displays();
-	drm_modeset_unlock_all(drm_dev);
-
-	drm_kms_helper_poll_enable(drm_dev);
+	drm_mode_config_helper_resume(drm_dev);
 
 	return omap_gem_resume(drm_dev);
 }
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/omap_drv.h b/drivers/gpu/drm/omapdrm/omap_drv.h
--- a/drivers/gpu/drm/omapdrm/omap_drv.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/omap_drv.h	2020-06-19 21:00:31.412227715 -0400
@@ -23,6 +23,7 @@
 #include <linux/workqueue.h>
 
 #include <drm/drmP.h>
+#include <drm/drm_atomic.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_gem.h>
 #include <drm/omap_drm.h>
@@ -37,6 +38,7 @@
 #include "omap_gem.h"
 #include "omap_irq.h"
 #include "omap_plane.h"
+#include "omap_overlay.h"
 
 #define DBG(fmt, ...) DRM_DEBUG(fmt"\n", ##__VA_ARGS__)
 #define VERB(fmt, ...) if (0) DRM_DEBUG(fmt, ##__VA_ARGS__) /* verbose debug */
@@ -45,6 +47,28 @@
 
 struct omap_drm_usergart;
 
+struct omap_drm_pipeline {
+	struct drm_crtc *crtc;
+	struct drm_encoder *encoder;
+	struct drm_connector *connector;
+	struct omap_dss_device *output;
+	unsigned int alias_id;
+};
+
+/*
+ * Global private object state for tracking resources that are shared across
+ * multiple kms objects (planes/crtcs/etc).
+ */
+#define to_omap_global_state(x) container_of(x, struct omap_global_state, base)
+struct omap_global_state {
+	struct drm_private_state base;
+
+	struct drm_atomic_state *state;
+
+	/* global atomic state of assignment between overlays and planes */
+	struct drm_plane *hwoverlay_to_plane[8];
+};
+
 struct omap_drm_private {
 	struct drm_device *ddev;
 	struct device *dev;
@@ -54,17 +78,22 @@
 	struct dispc_device *dispc;
 	const struct dispc_ops *dispc_ops;
 
-	unsigned int num_crtcs;
-	struct drm_crtc *crtcs[8];
+	unsigned int num_pipes;
+	struct omap_drm_pipeline pipes[8];
+	struct omap_drm_pipeline *channels[8];
 
 	unsigned int num_planes;
 	struct drm_plane *planes[8];
 
-	unsigned int num_encoders;
-	struct drm_encoder *encoders[8];
+	unsigned int num_ovls;
+	struct omap_hw_overlay *overlays[8];
 
-	unsigned int num_connectors;
-	struct drm_connector *connectors[8];
+	/*
+	 * Global private object state, Do not access directly, use
+	 * omap_global_get_state()
+	 */
+	struct drm_modeset_lock glob_obj_lock;
+	struct drm_private_obj glob_obj;
 
 	struct drm_fb_helper *fbdev;
 
@@ -81,6 +110,14 @@
 
 	/* properties: */
 	struct drm_property *zorder_prop;
+	struct drm_property *global_alpha_prop;
+	struct drm_property *pre_mult_alpha_prop;
+
+	/* crtc properties */
+	struct drm_property *background_color_prop;
+	struct drm_property *trans_key_mode_prop;
+	struct drm_property *trans_key_prop;
+	struct drm_property *alpha_blender_prop;
 
 	/* irq handling: */
 	spinlock_t wait_lock;		/* protects the wait_list */
@@ -89,9 +126,36 @@
 
 	/* memory bandwidth limit if it is needed on the platform */
 	unsigned int max_bandwidth;
+
+	void *wb_private;	      /* Write-back private data */
+	bool wb_initialized;
 };
 
 
 int omap_debugfs_init(struct drm_minor *minor);
+struct omap_global_state *__must_check
+omap_get_global_state(struct drm_atomic_state *s);
+struct omap_global_state *
+omap_get_existing_global_state(struct omap_drm_private *priv);
+
+#if IS_ENABLED(CONFIG_DRM_OMAP_WB)
+
+#define OMAP_WB_IRQ_MASK (DISPC_IRQ_FRAMEDONEWB | \
+			  DISPC_IRQ_WBBUFFEROVERFLOW | \
+			  DISPC_IRQ_WBUNCOMPLETEERROR)
+
+int omap_wb_init(struct drm_device *drmdev);
+void omap_wb_cleanup(struct drm_device *drmdev);
+void omap_wb_irq(void *priv, u32 irqstatus);
+
+#else
+
+#define OMAP_WB_IRQ_MASK (0)
+
+static inline int omap_wb_init(struct drm_device *drmdev) { return 0; }
+static inline void omap_wb_cleanup(struct drm_device *drmdev) { }
+static inline void omap_wb_irq(void *priv, u32 irqstatus) { }
+
+#endif
 
 #endif /* __OMAPDRM_DRV_H__ */
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/omap_encoder.c b/drivers/gpu/drm/omapdrm/omap_encoder.c
--- a/drivers/gpu/drm/omapdrm/omap_encoder.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/omap_encoder.c	2020-06-19 21:00:31.412227715 -0400
@@ -20,6 +20,7 @@
 #include <drm/drm_crtc.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_panel.h>
 
 #include "omap_drv.h"
 
@@ -36,16 +37,9 @@
  */
 struct omap_encoder {
 	struct drm_encoder base;
-	struct omap_dss_device *dssdev;
+	struct omap_dss_device *output;
 };
 
-struct omap_dss_device *omap_encoder_get_dssdev(struct drm_encoder *encoder)
-{
-	struct omap_encoder *omap_encoder = to_omap_encoder(encoder);
-
-	return omap_encoder->dssdev;
-}
-
 static void omap_encoder_destroy(struct drm_encoder *encoder)
 {
 	struct omap_encoder *omap_encoder = to_omap_encoder(encoder);
@@ -58,102 +52,200 @@
 	.destroy = omap_encoder_destroy,
 };
 
-static void omap_encoder_mode_set(struct drm_encoder *encoder,
-				struct drm_display_mode *mode,
-				struct drm_display_mode *adjusted_mode)
+static void omap_encoder_update_videomode_flags(struct videomode *vm,
+						u32 bus_flags)
+{
+	if (!(vm->flags & (DISPLAY_FLAGS_DE_LOW |
+			   DISPLAY_FLAGS_DE_HIGH))) {
+		if (bus_flags & DRM_BUS_FLAG_DE_LOW)
+			vm->flags |= DISPLAY_FLAGS_DE_LOW;
+		else if (bus_flags & DRM_BUS_FLAG_DE_HIGH)
+			vm->flags |= DISPLAY_FLAGS_DE_HIGH;
+	}
+
+	if (!(vm->flags & (DISPLAY_FLAGS_PIXDATA_POSEDGE |
+			   DISPLAY_FLAGS_PIXDATA_NEGEDGE))) {
+		if (bus_flags & DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE)
+			vm->flags |= DISPLAY_FLAGS_PIXDATA_POSEDGE;
+		else if (bus_flags & DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE)
+			vm->flags |= DISPLAY_FLAGS_PIXDATA_NEGEDGE;
+	}
+
+	if (!(vm->flags & (DISPLAY_FLAGS_SYNC_POSEDGE |
+			   DISPLAY_FLAGS_SYNC_NEGEDGE))) {
+		if (bus_flags & DRM_BUS_FLAG_SYNC_DRIVE_POSEDGE)
+			vm->flags |= DISPLAY_FLAGS_SYNC_POSEDGE;
+		else if (bus_flags & DRM_BUS_FLAG_SYNC_DRIVE_NEGEDGE)
+			vm->flags |= DISPLAY_FLAGS_SYNC_NEGEDGE;
+	}
+}
+
+static void omap_encoder_hdmi_mode_set(struct drm_connector *connector,
+				       struct drm_encoder *encoder,
+				       struct drm_display_mode *adjusted_mode)
 {
-	struct drm_device *dev = encoder->dev;
 	struct omap_encoder *omap_encoder = to_omap_encoder(encoder);
-	struct omap_dss_device *dssdev = omap_encoder->dssdev;
-	struct drm_connector *connector;
+	struct omap_dss_device *dssdev = omap_encoder->output;
 	bool hdmi_mode;
-	int r;
 
-	hdmi_mode = false;
-	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-		if (connector->encoder == encoder) {
-			hdmi_mode = omap_connector_get_hdmi_mode(connector);
-			break;
-		}
-	}
+	hdmi_mode = omap_connector_get_hdmi_mode(connector);
 
-	if (dssdev->driver->set_hdmi_mode)
-		dssdev->driver->set_hdmi_mode(dssdev, hdmi_mode);
+	if (dssdev->ops->hdmi.set_hdmi_mode)
+		dssdev->ops->hdmi.set_hdmi_mode(dssdev, hdmi_mode);
 
-	if (hdmi_mode && dssdev->driver->set_hdmi_infoframe) {
+	if (hdmi_mode && dssdev->ops->hdmi.set_infoframe) {
 		struct hdmi_avi_infoframe avi;
+		int r;
 
 		r = drm_hdmi_avi_infoframe_from_display_mode(&avi, adjusted_mode,
 							     false);
 		if (r == 0)
-			dssdev->driver->set_hdmi_infoframe(dssdev, &avi);
+			dssdev->ops->hdmi.set_infoframe(dssdev, &avi);
 	}
 }
 
-static void omap_encoder_disable(struct drm_encoder *encoder)
+static void omap_encoder_mode_set(struct drm_encoder *encoder,
+				  struct drm_display_mode *mode,
+				  struct drm_display_mode *adjusted_mode)
 {
 	struct omap_encoder *omap_encoder = to_omap_encoder(encoder);
-	struct omap_dss_device *dssdev = omap_encoder->dssdev;
-	struct omap_dss_driver *dssdrv = dssdev->driver;
+	struct omap_dss_device *output = omap_encoder->output;
+	struct omap_dss_device *dssdev;
+	struct drm_device *dev = encoder->dev;
+	struct drm_connector *connector;
+	struct drm_bridge *bridge;
+	struct videomode vm = { 0 };
+	u32 bus_flags;
+
+	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+		if (connector->encoder == encoder)
+			break;
+	}
+
+	drm_display_mode_to_videomode(adjusted_mode, &vm);
+
+	/*
+	 * HACK: This fixes the vm flags.
+	 * struct drm_display_mode does not contain the VSYNC/HSYNC/DE flags and
+	 * they get lost when converting back and forth between struct
+	 * drm_display_mode and struct videomode. The hack below goes and
+	 * fetches the missing flags.
+	 *
+	 * A better solution is to use DRM's bus-flags through the whole driver.
+	 */
+	for (dssdev = output; dssdev; dssdev = dssdev->next)
+		omap_encoder_update_videomode_flags(&vm, dssdev->bus_flags);
+
+	for (bridge = output->bridge; bridge; bridge = bridge->next) {
+		if (!bridge->timings)
+			continue;
+
+		bus_flags = bridge->timings->input_bus_flags;
+		omap_encoder_update_videomode_flags(&vm, bus_flags);
+	}
+
+	bus_flags = connector->display_info.bus_flags;
+	omap_encoder_update_videomode_flags(&vm, bus_flags);
 
-	dssdrv->disable(dssdev);
+	/* Set timings for all devices in the display pipeline. */
+	dss_mgr_set_timings(output, &vm);
+
+	for (dssdev = output; dssdev; dssdev = dssdev->next) {
+		if (dssdev->ops->set_timings)
+			dssdev->ops->set_timings(dssdev, adjusted_mode);
+	}
+
+	/* Set the HDMI mode and HDMI infoframe if applicable. */
+	if (output->type == OMAP_DISPLAY_TYPE_HDMI)
+		omap_encoder_hdmi_mode_set(connector, encoder, adjusted_mode);
 }
 
-static int omap_encoder_update(struct drm_encoder *encoder,
-			       enum omap_channel channel,
-			       struct videomode *vm)
+static void omap_encoder_disable(struct drm_encoder *encoder)
 {
-	struct drm_device *dev = encoder->dev;
 	struct omap_encoder *omap_encoder = to_omap_encoder(encoder);
-	struct omap_dss_device *dssdev = omap_encoder->dssdev;
-	struct omap_dss_driver *dssdrv = dssdev->driver;
-	int ret;
-
-	if (dssdrv->check_timings) {
-		ret = dssdrv->check_timings(dssdev, vm);
-	} else {
-		struct videomode t = {0};
+	struct omap_dss_device *dssdev = omap_encoder->output;
+	struct drm_device *dev = encoder->dev;
 
-		dssdrv->get_timings(dssdev, &t);
+	dev_dbg(dev->dev, "disable(%s)\n", dssdev->name);
 
-		if (memcmp(vm, &t, sizeof(*vm)))
-			ret = -EINVAL;
-		else
-			ret = 0;
+	/* Disable the panel if present. */
+	if (dssdev->panel) {
+		drm_panel_disable(dssdev->panel);
+		drm_panel_unprepare(dssdev->panel);
 	}
 
-	if (ret) {
-		dev_err(dev->dev, "could not set timings: %d\n", ret);
-		return ret;
+	/*
+	 * Disable the chain of external devices, starting at the one at the
+	 * internal encoder's output.
+	 */
+	omapdss_device_disable(dssdev->next);
+
+	/*
+	 * Disable the internal encoder. This will disable the DSS output. The
+	 * DSI is treated as an exception as DSI pipelines still use the legacy
+	 * flow where the pipeline output controls the encoder.
+	 */
+	if (dssdev->type != OMAP_DISPLAY_TYPE_DSI) {
+		dssdev->ops->disable(dssdev);
+		dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
 	}
 
-	if (dssdrv->set_timings)
-		dssdrv->set_timings(dssdev, vm);
-
-	return 0;
+	/*
+	 * Perform the post-disable operations on the chain of external devices
+	 * to complete the display pipeline disable.
+	 */
+	omapdss_device_post_disable(dssdev->next);
 }
 
 static void omap_encoder_enable(struct drm_encoder *encoder)
 {
 	struct omap_encoder *omap_encoder = to_omap_encoder(encoder);
-	struct omap_dss_device *dssdev = omap_encoder->dssdev;
-	struct omap_dss_driver *dssdrv = dssdev->driver;
-	int r;
-
-	omap_encoder_update(encoder, omap_crtc_channel(encoder->crtc),
-			    omap_crtc_timings(encoder->crtc));
-
-	r = dssdrv->enable(dssdev);
-	if (r)
-		dev_err(encoder->dev->dev,
-			"Failed to enable display '%s': %d\n",
-			dssdev->name, r);
+	struct omap_dss_device *dssdev = omap_encoder->output;
+	struct drm_device *dev = encoder->dev;
+
+	dev_dbg(dev->dev, "enable(%s)\n", dssdev->name);
+
+	/* Prepare the chain of external devices for pipeline enable. */
+	omapdss_device_pre_enable(dssdev->next);
+
+	/*
+	 * Enable the internal encoder. This will enable the DSS output. The
+	 * DSI is treated as an exception as DSI pipelines still use the legacy
+	 * flow where the pipeline output controls the encoder.
+	 */
+	if (dssdev->type != OMAP_DISPLAY_TYPE_DSI) {
+		dssdev->ops->enable(dssdev);
+		dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
+	}
+
+	/*
+	 * Enable the chain of external devices, starting at the one at the
+	 * internal encoder's output.
+	 */
+	omapdss_device_enable(dssdev->next);
+
+	/* Enable the panel if present. */
+	if (dssdev->panel) {
+		drm_panel_prepare(dssdev->panel);
+		drm_panel_enable(dssdev->panel);
+	}
 }
 
 static int omap_encoder_atomic_check(struct drm_encoder *encoder,
 				     struct drm_crtc_state *crtc_state,
 				     struct drm_connector_state *conn_state)
 {
+	struct omap_encoder *omap_encoder = to_omap_encoder(encoder);
+	enum drm_mode_status status;
+
+	status = omap_connector_mode_fixup(omap_encoder->output,
+					   &crtc_state->mode,
+					   &crtc_state->adjusted_mode);
+	if (status != MODE_OK) {
+		dev_err(encoder->dev->dev, "invalid timings: %d\n", status);
+		return -EINVAL;
+	}
+
 	return 0;
 }
 
@@ -166,7 +258,7 @@
 
 /* initialize encoder */
 struct drm_encoder *omap_encoder_init(struct drm_device *dev,
-		struct omap_dss_device *dssdev)
+				      struct omap_dss_device *output)
 {
 	struct drm_encoder *encoder = NULL;
 	struct omap_encoder *omap_encoder;
@@ -175,7 +267,7 @@
 	if (!omap_encoder)
 		goto fail;
 
-	omap_encoder->dssdev = dssdev;
+	omap_encoder->output = output;
 
 	encoder = &omap_encoder->base;
 
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/omap_encoder.h b/drivers/gpu/drm/omapdrm/omap_encoder.h
--- a/drivers/gpu/drm/omapdrm/omap_encoder.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/omap_encoder.h	2020-06-19 21:00:31.412227715 -0400
@@ -25,9 +25,6 @@
 struct omap_dss_device;
 
 struct drm_encoder *omap_encoder_init(struct drm_device *dev,
-		struct omap_dss_device *dssdev);
-
-/* map crtc to vblank mask */
-struct omap_dss_device *omap_encoder_get_dssdev(struct drm_encoder *encoder);
+				      struct omap_dss_device *output);
 
 #endif /* __OMAPDRM_ENCODER_H__ */
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/omap_fb.c b/drivers/gpu/drm/omapdrm/omap_fb.c
--- a/drivers/gpu/drm/omapdrm/omap_fb.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/omap_fb.c	2020-06-19 21:00:31.412227715 -0400
@@ -87,7 +87,7 @@
 
 bool omap_framebuffer_supports_rotation(struct drm_framebuffer *fb)
 {
-	return omap_gem_flags(fb->obj[0]) & OMAP_BO_TILED;
+	return omap_gem_flags(fb->obj[0]) & OMAP_BO_TILED_MASK;
 }
 
 /* Note: DRM rotates counter-clockwise, TILER & DSS rotates clockwise */
@@ -123,7 +123,9 @@
 /* update ovl info for scanout, handles cases of multi-planar fb's, etc.
  */
 void omap_framebuffer_update_scanout(struct drm_framebuffer *fb,
-		struct drm_plane_state *state, struct omap_overlay_info *info)
+		struct drm_plane_state *state,
+		struct omap_overlay_info *info,
+		struct omap_overlay_info *r_info)
 {
 	struct omap_framebuffer *omap_fb = to_omap_framebuffer(fb);
 	const struct drm_format_info *format = omap_fb->format;
@@ -146,7 +148,7 @@
 	x = state->src_x >> 16;
 	y = state->src_y >> 16;
 
-	if (omap_gem_flags(fb->obj[0]) & OMAP_BO_TILED) {
+	if (omap_gem_flags(fb->obj[0]) & OMAP_BO_TILED_MASK) {
 		u32 w = state->src_w >> 16;
 		u32 h = state->src_h >> 16;
 
@@ -204,7 +206,7 @@
 		plane = &omap_fb->planes[1];
 
 		if (info->rotation_type == OMAP_DSS_ROT_TILER) {
-			WARN_ON(!(omap_gem_flags(fb->obj[1]) & OMAP_BO_TILED));
+			WARN_ON(!(omap_gem_flags(fb->obj[1]) & OMAP_BO_TILED_MASK));
 			omap_gem_rotated_dma_addr(fb->obj[1], orient, x/2, y/2,
 						  &info->p_uv_addr);
 		} else {
@@ -213,6 +215,35 @@
 	} else {
 		info->p_uv_addr = 0;
 	}
+
+	if (r_info) {
+		info->width /= 2;
+		info->out_width /= 2;
+
+		*r_info = *info;
+
+		if (fb->format->is_yuv) {
+			if (info->width & 1) {
+				info->width++;
+				r_info->width--;
+			}
+
+			if (info->out_width & 1) {
+				info->out_width++;
+				r_info->out_width--;
+			}
+		}
+
+		r_info->pos_x = info->pos_x + info->out_width;
+
+		r_info->paddr =	get_linear_addr(fb, format, 0,
+						x + info->width, y);
+		if (fb->format->format == DRM_FORMAT_NV12) {
+			r_info->p_uv_addr =
+				get_linear_addr(fb, format, 1,
+						x + info->width, y);
+		}
+	}
 }
 
 /* pin, prepare for scanout: */
@@ -319,7 +350,7 @@
 
 error:
 	while (--i >= 0)
-		drm_gem_object_unreference_unlocked(bos[i]);
+		drm_gem_object_put_unlocked(bos[i]);
 
 	return fb;
 }
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/omap_fbdev.c b/drivers/gpu/drm/omapdrm/omap_fbdev.c
--- a/drivers/gpu/drm/omapdrm/omap_fbdev.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/omap_fbdev.c	2020-06-19 21:00:31.412227715 -0400
@@ -150,7 +150,7 @@
 		/* note: if fb creation failed, we can't rely on fb destroy
 		 * to unref the bo:
 		 */
-		drm_gem_object_unreference_unlocked(fbdev->bo);
+		drm_gem_object_put_unlocked(fbdev->bo);
 		ret = PTR_ERR(fb);
 		goto fail;
 	}
@@ -243,7 +243,7 @@
 	struct drm_fb_helper *helper;
 	int ret = 0;
 
-	if (!priv->num_crtcs || !priv->num_connectors)
+	if (!priv->num_pipes)
 		return;
 
 	fbdev = kzalloc(sizeof(*fbdev), GFP_KERNEL);
@@ -256,7 +256,7 @@
 
 	drm_fb_helper_prepare(dev, helper, &omap_fb_helper_funcs);
 
-	ret = drm_fb_helper_init(dev, helper, priv->num_connectors);
+	ret = drm_fb_helper_init(dev, helper, priv->num_pipes);
 	if (ret)
 		goto fail;
 
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/omap_fb.h b/drivers/gpu/drm/omapdrm/omap_fb.h
--- a/drivers/gpu/drm/omapdrm/omap_fb.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/omap_fb.h	2020-06-19 21:00:31.412227715 -0400
@@ -37,7 +37,9 @@
 int omap_framebuffer_pin(struct drm_framebuffer *fb);
 void omap_framebuffer_unpin(struct drm_framebuffer *fb);
 void omap_framebuffer_update_scanout(struct drm_framebuffer *fb,
-		struct drm_plane_state *state, struct omap_overlay_info *info);
+		struct drm_plane_state *state,
+		struct omap_overlay_info *info,
+		struct omap_overlay_info *r_info);
 bool omap_framebuffer_supports_rotation(struct drm_framebuffer *fb);
 void omap_framebuffer_describe(struct drm_framebuffer *fb, struct seq_file *m);
 
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/omap_gem.c b/drivers/gpu/drm/omapdrm/omap_gem.c
--- a/drivers/gpu/drm/omapdrm/omap_gem.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/omap_gem.c	2020-06-19 21:00:31.412227715 -0400
@@ -205,7 +205,7 @@
 	struct omap_gem_object *omap_obj = to_omap_bo(obj);
 	struct omap_drm_private *priv = obj->dev->dev_private;
 
-	if (omap_obj->flags & OMAP_BO_TILED) {
+	if (omap_obj->flags & OMAP_BO_TILED_MASK) {
 		enum tiler_fmt fmt = gem2fmt(omap_obj->flags);
 		int i;
 
@@ -333,7 +333,7 @@
 	struct omap_gem_object *omap_obj = to_omap_bo(obj);
 	size_t size = obj->size;
 
-	if (omap_obj->flags & OMAP_BO_TILED) {
+	if (omap_obj->flags & OMAP_BO_TILED_MASK) {
 		/* for tiled buffers, the virtual size has stride rounded up
 		 * to 4kb.. (to hide the fact that row n+1 might start 16kb or
 		 * 32kb later!).  But we don't back the entire buffer with
@@ -522,7 +522,7 @@
 	 * probably trigger put_pages()?
 	 */
 
-	if (omap_obj->flags & OMAP_BO_TILED)
+	if (omap_obj->flags & OMAP_BO_TILED_MASK)
 		ret = omap_gem_fault_2d(obj, vma, vmf);
 	else
 		ret = omap_gem_fault_1d(obj, vma, vmf);
@@ -638,7 +638,7 @@
 
 	*offset = omap_gem_mmap_offset(obj);
 
-	drm_gem_object_unreference_unlocked(obj);
+	drm_gem_object_put_unlocked(obj);
 
 fail:
 	return ret;
@@ -793,7 +793,7 @@
 			if (ret)
 				goto fail;
 
-			if (omap_obj->flags & OMAP_BO_TILED) {
+			if (omap_obj->flags & OMAP_BO_TILED_MASK) {
 				block = tiler_reserve_2d(fmt,
 						omap_obj->width,
 						omap_obj->height, 0);
@@ -826,9 +826,11 @@
 
 		omap_obj->dma_addr_cnt++;
 
-		*dma_addr = omap_obj->dma_addr;
+		if (dma_addr)
+			*dma_addr = omap_obj->dma_addr;
 	} else if (omap_gem_is_contiguous(omap_obj)) {
-		*dma_addr = omap_obj->dma_addr;
+		if (dma_addr)
+			*dma_addr = omap_obj->dma_addr;
 	} else {
 		ret = -EINVAL;
 		goto fail;
@@ -841,20 +843,16 @@
 }
 
 /**
- * omap_gem_unpin() - Unpin a GEM object from memory
+ * omap_gem_unpin_locked() - Unpin a GEM object from memory
  * @obj: the GEM object
  *
- * Unpin the given GEM object previously pinned with omap_gem_pin(). Pins are
- * reference-counted, the actualy unpin will only be performed when the number
- * of calls to this function matches the number of calls to omap_gem_pin().
+ * omap_gem_unpin() without locking.
  */
-void omap_gem_unpin(struct drm_gem_object *obj)
+static void omap_gem_unpin_locked(struct drm_gem_object *obj)
 {
 	struct omap_gem_object *omap_obj = to_omap_bo(obj);
 	int ret;
 
-	mutex_lock(&omap_obj->lock);
-
 	if (omap_obj->dma_addr_cnt > 0) {
 		omap_obj->dma_addr_cnt--;
 		if (omap_obj->dma_addr_cnt == 0) {
@@ -872,7 +870,22 @@
 			omap_obj->block = NULL;
 		}
 	}
+}
+
+/**
+ * omap_gem_unpin() - Unpin a GEM object from memory
+ * @obj: the GEM object
+ *
+ * Unpin the given GEM object previously pinned with omap_gem_pin(). Pins are
+ * reference-counted, the actual unpin will only be performed when the number
+ * of calls to this function matches the number of calls to omap_gem_pin().
+ */
+void omap_gem_unpin(struct drm_gem_object *obj)
+{
+	struct omap_gem_object *omap_obj = to_omap_bo(obj);
 
+	mutex_lock(&omap_obj->lock);
+	omap_gem_unpin_locked(obj);
 	mutex_unlock(&omap_obj->lock);
 }
 
@@ -889,7 +902,7 @@
 	mutex_lock(&omap_obj->lock);
 
 	if ((omap_obj->dma_addr_cnt > 0) && omap_obj->block &&
-			(omap_obj->flags & OMAP_BO_TILED)) {
+			(omap_obj->flags & OMAP_BO_TILED_MASK)) {
 		*dma_addr = tiler_tsptr(omap_obj->block, orient, x, y);
 		ret = 0;
 	}
@@ -904,7 +917,7 @@
 {
 	struct omap_gem_object *omap_obj = to_omap_bo(obj);
 	int ret = -EINVAL;
-	if (omap_obj->flags & OMAP_BO_TILED)
+	if (omap_obj->flags & OMAP_BO_TILED_MASK)
 		ret = tiler_stride(gem2fmt(omap_obj->flags), orient);
 	return ret;
 }
@@ -1042,7 +1055,7 @@
 			off, &omap_obj->dma_addr, omap_obj->dma_addr_cnt,
 			omap_obj->vaddr, omap_obj->roll);
 
-	if (omap_obj->flags & OMAP_BO_TILED) {
+	if (omap_obj->flags & OMAP_BO_TILED_MASK) {
 		seq_printf(m, " %dx%d", omap_obj->width, omap_obj->height);
 		if (omap_obj->block) {
 			struct tcm_area *area = &omap_obj->block->area;
@@ -1093,6 +1106,9 @@
 	list_del(&omap_obj->mm_list);
 	mutex_unlock(&priv->list_lock);
 
+	if (omap_obj->flags & OMAP_BO_MEM_PIN)
+		omap_gem_unpin_locked(obj);
+
 	/*
 	 * We own the sole reference to the object at this point, but to keep
 	 * lockdep happy, we must still take the omap_obj_lock to call
@@ -1129,6 +1145,47 @@
 	kfree(omap_obj);
 }
 
+static bool omap_gem_validate_flags(struct drm_device *dev, u32 flags)
+{
+	struct omap_drm_private *priv = dev->dev_private;
+
+	switch (flags & OMAP_BO_CACHE_MASK) {
+	case OMAP_BO_CACHED:
+	case OMAP_BO_WC:
+	case OMAP_BO_CACHE_MASK:
+		break;
+
+	default:
+		return false;
+	}
+
+	if ((flags & OMAP_BO_MEM_CONTIG) && (flags & OMAP_BO_MEM_DMM))
+		return false;
+
+	if ((flags & OMAP_BO_MEM_DMM) && !priv->usergart)
+		return false;
+
+	if (flags & OMAP_BO_TILED_MASK) {
+		if (!priv->usergart)
+			return false;
+
+		if (flags & OMAP_BO_MEM_CONTIG)
+			return false;
+
+		switch (flags & OMAP_BO_TILED_MASK) {
+		case OMAP_BO_TILED_8:
+		case OMAP_BO_TILED_16:
+		case OMAP_BO_TILED_32:
+			break;
+
+		default:
+			return false;
+		}
+	}
+
+	return true;
+}
+
 /* GEM buffer object constructor */
 struct drm_gem_object *omap_gem_new(struct drm_device *dev,
 		union omap_gem_size gsize, u32 flags)
@@ -1140,18 +1197,15 @@
 	size_t size;
 	int ret;
 
-	/* Validate the flags and compute the memory and cache flags. */
-	if (flags & OMAP_BO_TILED) {
-		if (!priv->usergart) {
-			dev_err(dev->dev, "Tiled buffers require DMM\n");
-			return NULL;
-		}
+	if (!omap_gem_validate_flags(dev, flags))
+		return NULL;
 
+	/* Validate the flags and compute the memory and cache flags. */
+	if (flags & OMAP_BO_TILED_MASK) {
 		/*
 		 * Tiled buffers are always shmem paged backed. When they are
 		 * scanned out, they are remapped into DMM/TILER.
 		 */
-		flags &= ~OMAP_BO_SCANOUT;
 		flags |= OMAP_BO_MEM_SHMEM;
 
 		/*
@@ -1160,11 +1214,11 @@
 		 */
 		flags &= ~(OMAP_BO_CACHED|OMAP_BO_WC|OMAP_BO_UNCACHED);
 		flags |= tiler_get_cpu_cache_flags();
-	} else if ((flags & OMAP_BO_SCANOUT) && !priv->has_dmm) {
+	} else if ((flags & OMAP_BO_MEM_CONTIG) ||
+		((flags & OMAP_BO_SCANOUT) && !priv->has_dmm)) {
 		/*
-		 * OMAP_BO_SCANOUT hints that the buffer doesn't need to be
-		 * tiled. However, to lower the pressure on memory allocation,
-		 * use contiguous memory only if no TILER is available.
+		 * If we don't have DMM, we must allocate scanout buffers
+		 * from contiguous DMA memory.
 		 */
 		flags |= OMAP_BO_MEM_DMA_API;
 	} else if (!(flags & OMAP_BO_MEM_DMABUF)) {
@@ -1183,7 +1237,7 @@
 	omap_obj->flags = flags;
 	mutex_init(&omap_obj->lock);
 
-	if (flags & OMAP_BO_TILED) {
+	if (flags & OMAP_BO_TILED_MASK) {
 		/*
 		 * For tiled buffers align dimensions to slot boundaries and
 		 * calculate size based on aligned dimensions.
@@ -1221,12 +1275,22 @@
 			goto err_release;
 	}
 
+	if (flags & OMAP_BO_MEM_PIN) {
+		ret = omap_gem_pin(obj, NULL);
+		if (ret)
+			goto err_free_dma;
+	}
+
 	mutex_lock(&priv->list_lock);
 	list_add(&omap_obj->mm_list, &priv->obj_list);
 	mutex_unlock(&priv->list_lock);
 
 	return obj;
 
+err_free_dma:
+	if (flags & OMAP_BO_MEM_DMA_API)
+		dma_free_writecombine(dev->dev, size,
+				omap_obj->vaddr, omap_obj->dma_addr);
 err_release:
 	drm_gem_object_release(obj);
 err_free:
@@ -1312,7 +1376,7 @@
 	}
 
 	/* drop reference from allocate - handle holds it now */
-	drm_gem_object_unreference_unlocked(obj);
+	drm_gem_object_put_unlocked(obj);
 
 	return 0;
 }
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/omap_gem_dmabuf.c b/drivers/gpu/drm/omapdrm/omap_gem_dmabuf.c
--- a/drivers/gpu/drm/omapdrm/omap_gem_dmabuf.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/omap_gem_dmabuf.c	2020-06-19 21:00:31.412227715 -0400
@@ -75,7 +75,7 @@
 {
 	struct drm_gem_object *obj = buffer->priv;
 	struct page **pages;
-	if (omap_gem_flags(obj) & OMAP_BO_TILED) {
+	if (omap_gem_flags(obj) & OMAP_BO_TILED_MASK) {
 		/* TODO we would need to pin at least part of the buffer to
 		 * get de-tiled view.  For now just reject it.
 		 */
@@ -98,8 +98,20 @@
 {
 	struct drm_gem_object *obj = buffer->priv;
 	struct page **pages;
+	dma_addr_t dma_addr;
 	omap_gem_get_pages(obj, &pages, false);
 	omap_gem_cpu_sync_page(obj, page_num);
+
+	/*
+	 * invalidate/flush the cache for this page deliberately.
+	 * XXX: revisit this, to find the proper place for invoking these calls.
+	 */
+	dma_addr = dma_map_page(obj->dev->dev, pages[page_num], 0, PAGE_SIZE,
+				DMA_BIDIRECTIONAL);
+	if (!dma_mapping_error(obj->dev->dev, dma_addr))
+		dma_unmap_page(obj->dev->dev, dma_addr, PAGE_SIZE,
+			       DMA_BIDIRECTIONAL);
+
 	return kmap(pages[page_num]);
 }
 
@@ -108,8 +120,19 @@
 {
 	struct drm_gem_object *obj = buffer->priv;
 	struct page **pages;
+	dma_addr_t dma_addr;
 	omap_gem_get_pages(obj, &pages, false);
 	kunmap(pages[page_num]);
+
+	/*
+	 * invalidate/flush the cache for this page deliberately.
+	 * XXX: revisit this, to find the proper place for invoking these calls.
+	 */
+	dma_addr = dma_map_page(obj->dev->dev, pages[page_num], 0, PAGE_SIZE,
+				DMA_BIDIRECTIONAL);
+	if (!dma_mapping_error(obj->dev->dev, dma_addr))
+		dma_unmap_page(obj->dev->dev, dma_addr, PAGE_SIZE,
+			       DMA_BIDIRECTIONAL);
 }
 
 static int omap_gem_dmabuf_mmap(struct dma_buf *buffer,
@@ -168,7 +191,7 @@
 			 * Importing dmabuf exported from out own gem increases
 			 * refcount on gem itself instead of f_count of dmabuf.
 			 */
-			drm_gem_object_reference(obj);
+			drm_gem_object_get(obj);
 			return obj;
 		}
 	}
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/omap_irq.c b/drivers/gpu/drm/omapdrm/omap_irq.c
--- a/drivers/gpu/drm/omapdrm/omap_irq.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/omap_irq.c	2020-06-19 21:00:31.412227715 -0400
@@ -206,8 +206,8 @@
 
 	VERB("irqs: %08x", irqstatus);
 
-	for (id = 0; id < priv->num_crtcs; id++) {
-		struct drm_crtc *crtc = priv->crtcs[id];
+	for (id = 0; id < priv->num_pipes; id++) {
+		struct drm_crtc *crtc = priv->pipes[id].crtc;
 		enum omap_channel channel = omap_crtc_channel(crtc);
 
 		if (irqstatus & priv->dispc_ops->mgr_get_vsync_irq(priv->dispc, channel)) {
@@ -221,6 +221,7 @@
 
 	omap_irq_ocp_error_handler(dev, irqstatus);
 	omap_irq_fifo_underflow(priv, irqstatus);
+	omap_wb_irq(priv->wb_private, irqstatus);
 
 	spin_lock_irqsave(&priv->wait_lock, flags);
 	list_for_each_entry_safe(wait, n, &priv->wait_list, node) {
@@ -269,6 +270,9 @@
 	for (i = 0; i < num_mgrs; ++i)
 		priv->irq_mask |= priv->dispc_ops->mgr_get_sync_lost_irq(priv->dispc, i);
 
+	if (priv->dispc_ops->has_writeback(priv->dispc))
+		priv->irq_mask |= OMAP_WB_IRQ_MASK;
+
 	priv->dispc_ops->runtime_get(priv->dispc);
 	priv->dispc_ops->clear_irqstatus(priv->dispc, 0xffffffff);
 	priv->dispc_ops->runtime_put(priv->dispc);
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/omap_overlay.c b/drivers/gpu/drm/omapdrm/omap_overlay.c
--- a/drivers/gpu/drm/omapdrm/omap_overlay.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/omap_overlay.c	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,333 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated -  http://www.ti.com/
+ * Author: Benoit Parrot, <bparrot@ti.com>
+ */
+
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_plane_helper.h>
+
+#include "omap_dmm_tiler.h"
+#include "omap_drv.h"
+
+/*
+ * overlay funcs
+ */
+static const char * const overlay_id_to_name[] = {
+	[OMAP_DSS_GFX] = "gfx",
+	[OMAP_DSS_VIDEO1] = "vid1",
+	[OMAP_DSS_VIDEO2] = "vid2",
+	[OMAP_DSS_VIDEO3] = "vid3",
+};
+
+static struct omap_hw_overlay *
+omap_plane_find_free_overlay(struct drm_device *dev,
+			     struct drm_plane *hwoverlay_to_plane[],
+			     u32 caps, u32 fourcc, u32 crtc_mask)
+{
+	struct omap_drm_private *priv = dev->dev_private;
+	const struct dispc_ops *ops = priv->dispc_ops;
+	int i;
+
+	DBG("caps: %x fourcc: %x crtc: %x", caps, fourcc, crtc_mask);
+
+	for (i = 0; i < priv->num_ovls; i++) {
+		struct omap_hw_overlay *cur = priv->overlays[i];
+
+		DBG("%d: id: %d cur->caps: %x cur->crtc: %x",
+		    cur->idx, cur->overlay_id, cur->caps, cur->possible_crtcs);
+
+		/* skip if already in-use */
+		if (hwoverlay_to_plane[cur->idx])
+			continue;
+
+		/* check if allowed on crtc */
+		if (!(cur->possible_crtcs & crtc_mask))
+			continue;
+
+		/* skip if doesn't support some required caps: */
+		if (caps & ~cur->caps)
+			continue;
+
+		/* check supported format */
+		if (!ops->ovl_color_mode_supported(priv->dispc,
+						   cur->overlay_id,
+						   fourcc))
+			continue;
+
+		return cur;
+	}
+
+	DBG("no match");
+	return NULL;
+}
+
+int omap_overlay_assign(struct drm_atomic_state *s, struct drm_plane *plane,
+			u32 caps, u32 fourcc, u32 crtc_mask,
+			struct omap_hw_overlay **overlay,
+			struct omap_hw_overlay **r_overlay)
+{
+	struct omap_drm_private *priv = s->dev->dev_private;
+	struct omap_global_state *new_global_state, *old_global_state;
+	struct drm_plane **overlay_map;
+	struct omap_hw_overlay *ovl, *r_ovl;
+	u32 save_possible_crtcs;
+
+	new_global_state = omap_get_global_state(s);
+	if (IS_ERR(new_global_state))
+		return PTR_ERR(new_global_state);
+
+	/*
+	 * grab old_state after omap_get_global_state(),
+	 * since now we hold lock:
+	 */
+	old_global_state = omap_get_existing_global_state(priv);
+	DBG("new_global_state: %p old_global_state: %p",
+	    new_global_state, old_global_state);
+
+	overlay_map = new_global_state->hwoverlay_to_plane;
+
+	if (!*overlay) {
+		ovl = omap_plane_find_free_overlay(s->dev, overlay_map,
+						   caps, fourcc, crtc_mask);
+		if (!ovl)
+			return -ENOMEM;
+
+		/* in case we need to backtrack */
+		save_possible_crtcs = ovl->possible_crtcs;
+
+		ovl->possible_crtcs = crtc_mask;
+		overlay_map[ovl->idx] = plane;
+		*overlay = ovl;
+
+		if (r_overlay) {
+			r_ovl = omap_plane_find_free_overlay(s->dev,
+							     overlay_map,
+							     caps, fourcc,
+							     crtc_mask);
+			if (!r_ovl) {
+				ovl->possible_crtcs = save_possible_crtcs;
+				overlay_map[ovl->idx] = NULL;
+				*overlay = NULL;
+				return -ENOMEM;
+			}
+
+			r_ovl->possible_crtcs = crtc_mask;
+			overlay_map[r_ovl->idx] = plane;
+			*r_overlay = r_ovl;
+		}
+
+		DBG("%s: assign to plane %s caps %x on crtc %x",
+		    (*overlay)->name, plane->name, caps, crtc_mask);
+
+		if (r_overlay) {
+			DBG("%s: assign to right of plane %s caps %x on crtc %x",
+			    (*r_overlay)->name, plane->name, caps, crtc_mask);
+		}
+	}
+
+	return 0;
+}
+
+void omap_overlay_release(struct drm_atomic_state *s,
+			  struct drm_plane *plane,
+			  struct omap_hw_overlay *overlay)
+{
+	struct omap_global_state *state = omap_get_global_state(s);
+	struct drm_plane **overlay_map = state->hwoverlay_to_plane;
+
+	if (!overlay)
+		return;
+
+	if (WARN_ON(!overlay_map[overlay->idx]))
+		return;
+	/*
+	 * Check that the overlay we are releasing is actually
+	 * assigned to the plane we are trying to release it from.
+	 */
+	if (overlay_map[overlay->idx] == plane) {
+		DBG("%s: release from plane %s", overlay->name, plane->name);
+
+		overlay_map[overlay->idx] = NULL;
+	}
+}
+
+void omap_overlay_disable(struct drm_atomic_state *s,
+			  struct drm_plane *plane,
+			  struct omap_hw_overlay *overlay)
+{
+	struct omap_drm_private *priv = s->dev->dev_private;
+	struct drm_plane **overlay_map;
+	struct omap_global_state *old_state;
+
+	old_state = omap_get_existing_global_state(priv);
+	overlay_map = old_state->hwoverlay_to_plane;
+
+	if (!overlay)
+		return;
+
+	/*
+	 * Check that the overlay we are trying to disable has not
+	 * been re-assigned to another plane already
+	 */
+	if (!overlay_map[overlay->idx]) {
+		DBG("%s: on %s disabled", overlay->name, plane->name);
+
+		/* disable the overlay */
+		priv->dispc_ops->ovl_enable(priv->dispc,
+					    overlay->overlay_id, false);
+
+		/*
+		 * Since we are disabling this overlay in this
+		 * atomic cycle we can reset the available crtcs
+		 * it can be used on
+		 */
+		overlay->possible_crtcs = (1 << priv->num_pipes) - 1;
+	}
+
+	/*
+	 * Otherwise the overlay is still in use so leave it alone
+	 */
+}
+
+int omap_overlay_assign_wb(struct omap_drm_private *priv,
+			   struct drm_plane *plane,
+			   u32 caps, u32 fourcc, u32 crtc_mask,
+			   struct omap_hw_overlay **overlay)
+{
+	struct omap_global_state *old_global_state;
+	struct drm_plane **overlay_map;
+	struct omap_hw_overlay *ovl;
+
+	/*
+	 * As there is no state here we can't really grab the global obj lock.
+	 * This might cause issue!
+	 */
+	old_global_state = omap_get_existing_global_state(priv);
+	DBG("old_global_state: %p", old_global_state);
+
+	overlay_map = old_global_state->hwoverlay_to_plane;
+
+	if (!*overlay) {
+		ovl = omap_plane_find_free_overlay(plane->dev, overlay_map,
+						   caps, fourcc, crtc_mask);
+		if (!ovl)
+			return -ENOMEM;
+
+		overlay_map[ovl->idx] = plane;
+		*overlay = ovl;
+
+		DBG("%s: assign to WB plane %s for caps %x",
+		    (*overlay)->name, plane->name, caps);
+	}
+
+	return 0;
+}
+
+void omap_overlay_release_wb(struct omap_drm_private *priv,
+			     struct drm_plane *plane,
+			     struct omap_hw_overlay *overlay)
+{
+	struct omap_global_state *old_global_state;
+	struct drm_plane **overlay_map;
+
+	if (!overlay)
+		return;
+
+	/*
+	 * As there is no state here we can't really grab the global obj lock.
+	 * This might cause issue!
+	 */
+	old_global_state = omap_get_existing_global_state(priv);
+	DBG("old_global_state: %p", old_global_state);
+
+	overlay_map = old_global_state->hwoverlay_to_plane;
+
+	if (WARN_ON(!overlay_map[overlay->idx]))
+		return;
+	/*
+	 * Check that the overlay we are releasing is actually
+	 * assigned to the plane we are trying to release it from.
+	 */
+	if (overlay_map[overlay->idx] == plane) {
+		DBG("%s: release from WB plane %s", overlay->name, plane->name);
+
+		/*
+		 * As this might get called without having done any other
+		 * actual h/w access make sure the module is enabled before
+		 * trying to access it.
+		 */
+		priv->dispc_ops->runtime_get(priv->dispc);
+		priv->dispc_ops->ovl_enable(priv->dispc, overlay->overlay_id,
+					    false);
+		priv->dispc_ops->runtime_put(priv->dispc);
+		overlay->possible_crtcs = (1 << priv->num_pipes) - 1;
+		overlay_map[overlay->idx] = NULL;
+	}
+}
+
+static void omap_overlay_destroy(struct omap_hw_overlay *overlay)
+{
+	kfree(overlay);
+}
+
+static struct omap_hw_overlay *omap_overlay_init(enum omap_plane_id overlay_id,
+						 enum omap_overlay_caps caps)
+{
+	struct omap_hw_overlay *overlay;
+
+	overlay = kzalloc(sizeof(*overlay), GFP_KERNEL);
+	if (!overlay)
+		return ERR_PTR(-ENOMEM);
+
+	overlay->name = overlay_id_to_name[overlay_id];
+	overlay->overlay_id = overlay_id;
+	overlay->caps = caps;
+	/*
+	 * When this is called priv->num_crtcs is not known yet.
+	 * Use a safe mask value to start with, it will get updated to the
+	 * proper value after the first use.
+	 */
+	overlay->possible_crtcs = 0xff;
+
+	return overlay;
+}
+
+int omap_hwoverlays_init(struct omap_drm_private *priv)
+{
+	static const enum omap_plane_id hw_plane_ids[] = {
+			OMAP_DSS_GFX, OMAP_DSS_VIDEO1,
+			OMAP_DSS_VIDEO2, OMAP_DSS_VIDEO3,
+	};
+	u32 num_overlays = priv->dispc_ops->get_num_ovls(priv->dispc);
+	enum omap_overlay_caps caps;
+	int i, ret;
+
+	for (i = 0; i < num_overlays; i++) {
+		struct omap_hw_overlay *overlay;
+
+		caps = priv->dispc_ops->ovl_get_caps(priv->dispc, hw_plane_ids[i]);
+		overlay = omap_overlay_init(hw_plane_ids[i], caps);
+		if (IS_ERR(overlay)) {
+			ret = PTR_ERR(overlay);
+			dev_err(priv->dev, "failed to construct overlay for %s (%d)\n",
+				overlay_id_to_name[i], ret);
+			return ret;
+		}
+		overlay->idx = priv->num_ovls;
+		priv->overlays[priv->num_ovls++] = overlay;
+	}
+
+	return 0;
+}
+
+void omap_hwoverlays_destroy(struct omap_drm_private *priv)
+{
+	int i;
+
+	for (i = 0; i < priv->num_ovls; i++) {
+		omap_overlay_destroy(priv->overlays[i]);
+		priv->overlays[i] = NULL;
+	}
+}
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/omap_overlay.h b/drivers/gpu/drm/omapdrm/omap_overlay.h
--- a/drivers/gpu/drm/omapdrm/omap_overlay.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/omap_overlay.h	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,49 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated -  http://www.ti.com/
+ * Author: Benoit Parrot, <bparrot@ti.com>
+ */
+
+#ifndef __OMAPDRM_OVERLAY_H__
+#define __OMAPDRM_OVERLAY_H__
+
+#include <linux/types.h>
+
+enum drm_plane_type;
+
+struct drm_device;
+struct drm_mode_object;
+struct drm_plane;
+
+/* Used to associate a HW overlay/plane to a plane */
+struct omap_hw_overlay {
+	int idx;
+
+	const char *name;
+	enum omap_plane_id overlay_id;
+
+	enum omap_overlay_caps caps;
+	u32 possible_crtcs;
+};
+
+int omap_hwoverlays_init(struct omap_drm_private *priv);
+void omap_hwoverlays_destroy(struct omap_drm_private *priv);
+int omap_overlay_assign(struct drm_atomic_state *s, struct drm_plane *plane,
+			u32 caps, u32 fourcc, u32 crtc_mask,
+			struct omap_hw_overlay **overlay,
+			struct omap_hw_overlay **r_overlay);
+void omap_overlay_release(struct drm_atomic_state *s,
+			  struct drm_plane *plane,
+			  struct omap_hw_overlay *overlay);
+void omap_overlay_disable(struct drm_atomic_state *s,
+			  struct drm_plane *plane,
+			  struct omap_hw_overlay *overlay);
+int omap_overlay_assign_wb(struct omap_drm_private *priv,
+			   struct drm_plane *plane,
+			   u32 caps, u32 fourcc, u32 crtc_mask,
+			   struct omap_hw_overlay **overlay);
+void omap_overlay_release_wb(struct omap_drm_private *priv,
+			     struct drm_plane *plane,
+			     struct omap_hw_overlay *overlay);
+
+#endif /* __OMAPDRM_OVERLAY_H__ */
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/omap_plane.c b/drivers/gpu/drm/omapdrm/omap_plane.c
--- a/drivers/gpu/drm/omapdrm/omap_plane.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/omap_plane.c	2020-06-19 21:00:31.412227715 -0400
@@ -26,14 +26,39 @@
  * plane funcs
  */
 
+#define to_omap_plane_state(x) container_of(x, struct omap_plane_state, base)
+
+struct omap_plane_state {
+	struct drm_plane_state base;
+
+	struct omap_hw_overlay *overlay;
+	struct omap_hw_overlay *r_overlay;  /* right overlay */
+
+	unsigned int global_alpha;
+	unsigned int pre_mult_alpha;
+};
+
 #define to_omap_plane(x) container_of(x, struct omap_plane, base)
 
 struct omap_plane {
 	struct drm_plane base;
 	enum omap_plane_id id;
 	const char *name;
+
+	/*
+	 * WB has no notion of atomic state we need to keep
+	 * a reference to the allocated overlay here.
+	 */
+	struct omap_hw_overlay *reserved_wb_overlay;
 };
 
+bool is_omap_plane_dual_overlay(struct drm_plane_state *state)
+{
+	struct omap_plane_state *omap_state = to_omap_plane_state(state);
+
+	return !!omap_state->r_overlay;
+}
+
 static int omap_plane_prepare_fb(struct drm_plane *plane,
 				 struct drm_plane_state *new_state)
 {
@@ -56,85 +81,307 @@
 	struct omap_drm_private *priv = plane->dev->dev_private;
 	struct omap_plane *omap_plane = to_omap_plane(plane);
 	struct drm_plane_state *state = plane->state;
-	struct omap_overlay_info info;
+	struct omap_plane_state *new_omap_state;
+	struct omap_plane_state *old_omap_state;
+	struct omap_overlay_info info, r_info;
+	enum omap_plane_id ovl_id, r_ovl_id;
 	int ret;
+	bool dual_ovl;
+
+	new_omap_state = to_omap_plane_state(state);
+	old_omap_state = to_omap_plane_state(old_state);
+
+	dual_ovl = is_omap_plane_dual_overlay(state);
 
+	/* Cleanup previously held overlay if needed */
+	omap_overlay_disable(old_state->state, plane, old_omap_state->overlay);
+	omap_overlay_disable(old_state->state, plane,
+			     old_omap_state->r_overlay);
+
+	if (!new_omap_state->overlay) {
+		DBG("[PLANE:%d:%s] overlay_id: ??? (%p)", plane->base.id, plane->name,
+		    new_omap_state->overlay);
+		return;
+	}
+
+	ovl_id = new_omap_state->overlay->overlay_id;
+	DBG("[PLANE:%d:%s] overlay_id: %d", plane->base.id, plane->name,
+	    ovl_id);
 	DBG("%s, crtc=%p fb=%p", omap_plane->name, state->crtc, state->fb);
 
 	memset(&info, 0, sizeof(info));
 	info.rotation_type = OMAP_DSS_ROT_NONE;
 	info.rotation = DRM_MODE_ROTATE_0;
-	info.global_alpha = 0xff;
+	info.global_alpha = new_omap_state->global_alpha;
+	info.pre_mult_alpha = new_omap_state->pre_mult_alpha;
 	info.zorder = state->normalized_zpos;
+	info.color_encoding = state->color_encoding;
+	info.color_range = state->color_range;
+
+	r_info = info;
 
 	/* update scanout: */
-	omap_framebuffer_update_scanout(state->fb, state, &info);
+	omap_framebuffer_update_scanout(state->fb, state, &info,
+					dual_ovl ? &r_info : NULL);
 
-	DBG("%dx%d -> %dx%d (%d)", info.width, info.height,
-			info.out_width, info.out_height,
-			info.screen_width);
+	DBG("%s: %dx%d -> %dx%d (%d)",
+	    new_omap_state->overlay->name, info.width, info.height,
+	    info.out_width, info.out_height, info.screen_width);
 	DBG("%d,%d %pad %pad", info.pos_x, info.pos_y,
-			&info.paddr, &info.p_uv_addr);
+	    &info.paddr, &info.p_uv_addr);
+
+	if (dual_ovl) {
+		r_ovl_id = new_omap_state->r_overlay->overlay_id;
+		/*
+		 * If the current plane uses 2 hw planes the very next
+		 * zorder is used by the r_overlay so we just use the
+		 * main overlay zorder + 1
+		 */
+		r_info.zorder = info.zorder + 1;
+
+		DBG("%s: %dx%d -> %dx%d (%d)",
+		    new_omap_state->r_overlay->name,
+		    r_info.width, r_info.height,
+		    r_info.out_width, r_info.out_height, r_info.screen_width);
+		DBG("%d,%d %pad %pad", r_info.pos_x, r_info.pos_y,
+		    &r_info.paddr, &r_info.p_uv_addr);
+	}
 
 	/* and finally, update omapdss: */
-	ret = priv->dispc_ops->ovl_setup(priv->dispc, omap_plane->id, &info,
+	ret = priv->dispc_ops->ovl_setup(priv->dispc, ovl_id, &info,
 			      omap_crtc_timings(state->crtc), false,
 			      omap_crtc_channel(state->crtc));
 	if (ret) {
-		dev_err(plane->dev->dev, "Failed to setup plane %s\n",
+		dev_err(plane->dev->dev, "Failed to setup plane1 %s\n",
 			omap_plane->name);
-		priv->dispc_ops->ovl_enable(priv->dispc, omap_plane->id, false);
+		priv->dispc_ops->ovl_enable(priv->dispc, ovl_id, false);
 		return;
 	}
 
-	priv->dispc_ops->ovl_enable(priv->dispc, omap_plane->id, true);
+	priv->dispc_ops->ovl_enable(priv->dispc, ovl_id, true);
+
+	if (dual_ovl) {
+		ret = priv->dispc_ops->ovl_setup(priv->dispc, r_ovl_id, &r_info,
+				      omap_crtc_timings(state->crtc), false,
+				      omap_crtc_channel(state->crtc));
+		if (ret) {
+			dev_err(plane->dev->dev, "Failed to setup plane2 %s\n",
+				omap_plane->name);
+			priv->dispc_ops->ovl_enable(priv->dispc, r_ovl_id, false);
+			priv->dispc_ops->ovl_enable(priv->dispc, ovl_id, false);
+			return;
+		}
+
+		priv->dispc_ops->ovl_enable(priv->dispc, r_ovl_id, true);
+	}
 }
 
 static void omap_plane_atomic_disable(struct drm_plane *plane,
 				      struct drm_plane_state *old_state)
 {
-	struct omap_drm_private *priv = plane->dev->dev_private;
-	struct omap_plane *omap_plane = to_omap_plane(plane);
+	struct drm_plane_state *state = plane->state;
+	struct omap_plane_state *new_omap_state;
+	struct omap_plane_state *old_omap_state;
+
+	new_omap_state = to_omap_plane_state(state);
+	old_omap_state = to_omap_plane_state(old_state);
+
+	if (!old_omap_state->overlay)
+		return;
 
 	plane->state->rotation = DRM_MODE_ROTATE_0;
 	plane->state->zpos = plane->type == DRM_PLANE_TYPE_PRIMARY
-			   ? 0 : omap_plane->id;
+			   ? 0 : old_omap_state->overlay->overlay_id;
 
-	priv->dispc_ops->ovl_enable(priv->dispc, omap_plane->id, false);
+	omap_overlay_disable(old_state->state, plane, old_omap_state->overlay);
+	new_omap_state->overlay = NULL;
+	if (is_omap_plane_dual_overlay(old_state)) {
+		omap_overlay_disable(old_state->state, plane,
+				     old_omap_state->r_overlay);
+		new_omap_state->r_overlay = NULL;
+	}
 }
 
+#define FRAC_16_16(mult, div)    (((mult) << 16) / (div))
 static int omap_plane_atomic_check(struct drm_plane *plane,
 				   struct drm_plane_state *state)
 {
+	struct omap_drm_private *priv = plane->dev->dev_private;
+	struct omap_plane *omap_plane = to_omap_plane(plane);
+	struct drm_crtc *crtc;
 	struct drm_crtc_state *crtc_state;
+	u16 width, height;
+	u32 width_fp, height_fp;
+	struct drm_plane_state *old_state = plane->state;
+	struct omap_plane_state *omap_state = to_omap_plane_state(state);
+	struct omap_global_state *omap_overlay_global_state;
+	u32 crtc_mask;
+	u32 fourcc;
+	u32 caps = 0;
+	bool new_hw_overlay = false;
+	bool new_r_hw_overlay = false;
+	bool is_fourcc_yuv = false;
+	int min_scale, max_scale;
+	int ret;
 
-	if (!state->fb)
-		return 0;
+	if (omap_plane->reserved_wb_overlay)
+		return -EBUSY;
+
+	omap_overlay_global_state = omap_get_global_state(state->state);
+	if (IS_ERR(omap_overlay_global_state))
+		return PTR_ERR(omap_overlay_global_state);
+	DBG("%s: omap_overlay_global_state: %p", plane->name,
+	    omap_overlay_global_state);
+
+	priv->dispc_ops->ovl_get_max_size(priv->dispc, &width, &height);
+	width_fp = width << 16;
+	height_fp = height << 16;
 
-	/* crtc should only be NULL when disabling (i.e., !state->fb) */
-	if (WARN_ON(!state->crtc))
+	crtc = state->crtc ? state->crtc : plane->state->crtc;
+	if (!crtc)
 		return 0;
 
-	crtc_state = drm_atomic_get_existing_crtc_state(state->state, state->crtc);
+	crtc_state = drm_atomic_get_existing_crtc_state(state->state, crtc);
 	/* we should have a crtc state if the plane is attached to a crtc */
 	if (WARN_ON(!crtc_state))
 		return 0;
 
-	if (!crtc_state->enable)
-		return 0;
-
-	if (state->crtc_x < 0 || state->crtc_y < 0)
+	/* Make sure dimensions are within bounds. */
+	if (state->src_h > height_fp || state->crtc_h > height)
 		return -EINVAL;
 
-	if (state->crtc_x + state->crtc_w > crtc_state->adjusted_mode.hdisplay)
-		return -EINVAL;
+	if (state->fb)
+		is_fourcc_yuv = state->fb->format->is_yuv;
 
-	if (state->crtc_y + state->crtc_h > crtc_state->adjusted_mode.vdisplay)
-		return -EINVAL;
+	if (state->src_w > width_fp || state->crtc_w > width) {
+		/*
+		 * We cannot have dual plane/overlay and trans_key_mode
+		 * enabled concurrently, hence rejecting this configuration
+		 */
+		if (omap_crtc_atomic_get_trans_key_mode(crtc, crtc_state))
+			return -EINVAL;
+
+		if (is_fourcc_yuv &&
+		    (((state->src_w >> 16) / 2 & 1) ||
+		     state->crtc_w / 2 & 1)) {
+			/*
+			 * When calculating the split overlay width
+			 * and it yield an odd value we will need to adjust
+			 * the indivual width +/- 1. So make sure it fits
+			 */
+			if (state->src_w <= ((2 * width - 1) << 16) &&
+			    state->crtc_w <= (2 * width - 1))
+				new_r_hw_overlay = true;
+			else
+				return -EINVAL;
+		} else {
+			if (state->src_w <= (2 * width_fp) &&
+			    state->crtc_w <= (2 * width))
+				new_r_hw_overlay = true;
+			else
+				return -EINVAL;
+		}
+	}
 
-	if (state->rotation != DRM_MODE_ROTATE_0 &&
-	    !omap_framebuffer_supports_rotation(state->fb))
-		return -EINVAL;
+	/*
+	 * Note: these are just sanity checks to filter out totally bad scaling
+	 * factors. The real limits must be calculated case by case, and
+	 * unfortunately we currently do those checks only at the commit
+	 * phase in dispc.
+	 */
+	min_scale = FRAC_16_16(1, 8);
+	max_scale = FRAC_16_16(8, 1);
+
+	ret = drm_atomic_helper_check_plane_state(state, crtc_state,
+						  min_scale, max_scale,
+						  true, true);
+	if (ret)
+		return ret;
+
+	DBG("%s: check (%d -> %d)", plane->name,
+	    old_state->visible, state->visible);
+
+	if (state->visible) {
+		if (state->rotation != DRM_MODE_ROTATE_0 &&
+		    !omap_framebuffer_supports_rotation(state->fb))
+			return -EINVAL;
+
+		if ((state->src_w >> 16) != state->crtc_w ||
+		    (state->src_h >> 16) != state->crtc_h)
+			caps |= OMAP_DSS_OVL_CAP_SCALE;
+
+		fourcc = state->fb->format->format;
+		crtc_mask = drm_crtc_mask(state->crtc);
+
+		/*
+		 * (re)allocate hw overlay if we don't have one or
+		 * there is a caps mismatch
+		 */
+		if (!omap_state->overlay ||
+		    (caps & ~omap_state->overlay->caps)) {
+			new_hw_overlay = true;
+		} else {
+			/* check if allowed on crtc */
+			if (!(omap_state->overlay->possible_crtcs & crtc_mask))
+				new_hw_overlay = true;
+
+			/* check supported format */
+			if (!priv->dispc_ops->ovl_color_mode_supported(priv->dispc,
+						omap_state->overlay->overlay_id,
+						fourcc))
+				new_hw_overlay = true;
+		}
+		/*
+		 * check if we need two overlays and only have 1 or
+		 * if we had 2 overlays but will only need 1
+		 */
+		if ((new_r_hw_overlay && !omap_state->r_overlay) ||
+		    (!new_r_hw_overlay && omap_state->r_overlay))
+			new_hw_overlay = true;
+
+		if (new_hw_overlay) {
+			struct omap_hw_overlay *old_ovl =
+						omap_state->overlay;
+			struct omap_hw_overlay *old_r_ovl =
+						omap_state->r_overlay;
+			struct omap_hw_overlay *new_ovl = NULL;
+			struct omap_hw_overlay *new_r_ovl = NULL;
+
+			omap_overlay_release(state->state, plane, old_ovl);
+			omap_overlay_release(state->state, plane, old_r_ovl);
+
+			ret = omap_overlay_assign(state->state, plane, caps,
+						  fourcc, crtc_mask, &new_ovl,
+						  new_r_hw_overlay ?
+						  &new_r_ovl : NULL);
+			if (ret) {
+				DBG("%s: failed to assign hw_overlay(s)!",
+				    plane->name);
+				omap_state->overlay = NULL;
+				omap_state->r_overlay = NULL;
+				return ret;
+			}
+
+			omap_state->overlay = new_ovl;
+			if (new_r_hw_overlay)
+				omap_state->r_overlay = new_r_ovl;
+			else
+				omap_state->r_overlay = NULL;
+		}
+	} else {
+		omap_overlay_release(state->state, plane, omap_state->overlay);
+		omap_overlay_release(state->state, plane,
+				     omap_state->r_overlay);
+		omap_state->overlay = NULL;
+		omap_state->r_overlay = NULL;
+	}
+
+	if (omap_state->overlay)
+		DBG("plane: %s overlay_id: %d", plane->name,
+		    omap_state->overlay->overlay_id);
+	if (omap_state->r_overlay)
+		DBG("plane: %s r_overlay_id: %d", plane->name,
+		    omap_state->r_overlay->overlay_id);
 
 	return 0;
 }
@@ -182,20 +429,91 @@
 	drm_object_attach_property(obj, priv->zorder_prop, 0);
 }
 
+static void omap_plane_atomic_destroy_state(struct drm_plane *plane,
+					    struct drm_plane_state *state)
+{
+	__drm_atomic_helper_plane_destroy_state(state);
+	kfree(to_omap_plane_state(state));
+}
+
 static void omap_plane_reset(struct drm_plane *plane)
 {
 	struct omap_plane *omap_plane = to_omap_plane(plane);
+	struct omap_plane_state *omap_state;
 
-	drm_atomic_helper_plane_reset(plane);
-	if (!plane->state)
+	if (plane->state)
+		omap_plane_atomic_destroy_state(plane, plane->state);
+
+	omap_state = kzalloc(sizeof(*omap_state), GFP_KERNEL);
+	if (!omap_state)
 		return;
 
+	omap_state->base.plane = plane;
+	plane->state = &omap_state->base;
+	plane->state->plane = plane;
+	plane->state->rotation = DRM_MODE_ROTATE_0;
 	/*
 	 * Set the zpos default depending on whether we are a primary or overlay
 	 * plane.
 	 */
 	plane->state->zpos = plane->type == DRM_PLANE_TYPE_PRIMARY
 			   ? 0 : omap_plane->id;
+	plane->state->color_encoding = DRM_COLOR_YCBCR_BT601;
+	plane->state->color_range = DRM_COLOR_YCBCR_FULL_RANGE;
+
+	omap_state->global_alpha = 0xff;
+	omap_state->pre_mult_alpha = 0;
+}
+
+static struct drm_plane_state *
+omap_plane_atomic_duplicate_state(struct drm_plane *plane)
+{
+	struct omap_plane_state *state;
+	struct omap_plane_state *copy;
+
+	if (WARN_ON(!plane->state))
+		return NULL;
+
+	state = to_omap_plane_state(plane->state);
+	copy = kmemdup(state, sizeof(*state), GFP_KERNEL);
+	if (!copy)
+		return NULL;
+
+	__drm_atomic_helper_plane_duplicate_state(plane, &copy->base);
+
+	copy->global_alpha = state->global_alpha;
+	copy->pre_mult_alpha = state->pre_mult_alpha;
+
+	return &copy->base;
+}
+
+static void omap_plane_atomic_print_state(struct drm_printer *p,
+					  const struct drm_plane_state *state)
+{
+	struct omap_plane_state *omap_state = to_omap_plane_state(state);
+
+	drm_printf(p, "\toverlay=%s\n", omap_state->overlay ?
+					omap_state->overlay->name : "(null)");
+	if (omap_state->overlay) {
+		drm_printf(p, "\t\tidx=%d\n", omap_state->overlay->idx);
+		drm_printf(p, "\t\toverlay_id=%d\n",
+			   omap_state->overlay->overlay_id);
+		drm_printf(p, "\t\tcaps=0x%x\n", omap_state->overlay->caps);
+		drm_printf(p, "\t\tpossible_crtcs=0x%x\n",
+			   omap_state->overlay->possible_crtcs);
+	}
+
+	drm_printf(p, "\tr_overlay=%s\n", omap_state->r_overlay ?
+					  omap_state->r_overlay->name :
+					  "(null)");
+	if (omap_state->r_overlay) {
+		drm_printf(p, "\t\tidx=%d\n", omap_state->r_overlay->idx);
+		drm_printf(p, "\t\toverlay_id=%d\n",
+			   omap_state->r_overlay->overlay_id);
+		drm_printf(p, "\t\tcaps=0x%x\n", omap_state->r_overlay->caps);
+		drm_printf(p, "\t\tpossible_crtcs=0x%x\n",
+			   omap_state->r_overlay->possible_crtcs);
+	}
 }
 
 static int omap_plane_atomic_set_property(struct drm_plane *plane,
@@ -204,9 +522,14 @@
 					  u64 val)
 {
 	struct omap_drm_private *priv = plane->dev->dev_private;
+	struct omap_plane_state *omap_state = to_omap_plane_state(state);
 
 	if (property == priv->zorder_prop)
 		state->zpos = val;
+	else if (property == priv->global_alpha_prop)
+		omap_state->global_alpha = val;
+	else if (property == priv->pre_mult_alpha_prop)
+		omap_state->pre_mult_alpha = val;
 	else
 		return -EINVAL;
 
@@ -219,9 +542,14 @@
 					  u64 *val)
 {
 	struct omap_drm_private *priv = plane->dev->dev_private;
+	const struct omap_plane_state *omap_state = to_omap_plane_state(state);
 
 	if (property == priv->zorder_prop)
 		*val = state->zpos;
+	else if (property == priv->global_alpha_prop)
+		*val = omap_state->global_alpha;
+	else if (property == priv->pre_mult_alpha_prop)
+		*val = omap_state->pre_mult_alpha;
 	else
 		return -EINVAL;
 
@@ -233,12 +561,30 @@
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.reset = omap_plane_reset,
 	.destroy = omap_plane_destroy,
-	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
-	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
+	.atomic_duplicate_state = omap_plane_atomic_duplicate_state,
+	.atomic_destroy_state = omap_plane_atomic_destroy_state,
 	.atomic_set_property = omap_plane_atomic_set_property,
 	.atomic_get_property = omap_plane_atomic_get_property,
+	.atomic_print_state = omap_plane_atomic_print_state,
 };
 
+static bool omap_plane_supports_yuv(struct drm_plane *plane)
+{
+	struct omap_drm_private *priv = plane->dev->dev_private;
+	struct omap_plane *omap_plane = to_omap_plane(plane);
+	const u32 *formats =
+		priv->dispc_ops->ovl_get_color_modes(priv->dispc, omap_plane->id);
+	int i;
+
+	for (i = 0; formats[i]; i++)
+		if (formats[i] == DRM_FORMAT_YUYV ||
+		    formats[i] == DRM_FORMAT_UYVY ||
+		    formats[i] == DRM_FORMAT_NV12)
+			return true;
+
+	return false;
+}
+
 static const char *plane_id_to_name[] = {
 	[OMAP_DSS_GFX] = "gfx",
 	[OMAP_DSS_VIDEO1] = "vid1",
@@ -246,13 +592,6 @@
 	[OMAP_DSS_VIDEO3] = "vid3",
 };
 
-static const enum omap_plane_id plane_idx_to_id[] = {
-	OMAP_DSS_GFX,
-	OMAP_DSS_VIDEO1,
-	OMAP_DSS_VIDEO2,
-	OMAP_DSS_VIDEO3,
-};
-
 /* initialize plane */
 struct drm_plane *omap_plane_init(struct drm_device *dev,
 		int idx, enum drm_plane_type type,
@@ -262,27 +601,28 @@
 	unsigned int num_planes = priv->dispc_ops->get_num_ovls(priv->dispc);
 	struct drm_plane *plane;
 	struct omap_plane *omap_plane;
-	enum omap_plane_id id;
 	int ret;
 	u32 nformats;
 	const u32 *formats;
 
-	if (WARN_ON(idx >= ARRAY_SIZE(plane_idx_to_id)))
+	if (WARN_ON(idx >= num_planes))
 		return ERR_PTR(-EINVAL);
 
-	id = plane_idx_to_id[idx];
-
-	DBG("%s: type=%d", plane_id_to_name[id], type);
-
 	omap_plane = kzalloc(sizeof(*omap_plane), GFP_KERNEL);
 	if (!omap_plane)
 		return ERR_PTR(-ENOMEM);
 
-	formats = priv->dispc_ops->ovl_get_color_modes(priv->dispc, id);
+	omap_plane->id = idx;
+	omap_plane->name = plane_id_to_name[idx];
+
+	DBG("%s: type=%d", omap_plane->name, type);
+	DBG("	omap_plane->id: %d", omap_plane->id);
+	DBG("	crtc_mask: 0x%04x", possible_crtcs);
+
+	formats = priv->dispc_ops->ovl_get_color_modes(priv->dispc,
+						       omap_plane->id);
 	for (nformats = 0; formats[nformats]; ++nformats)
 		;
-	omap_plane->id = id;
-	omap_plane->name = plane_id_to_name[id];
 
 	plane = &omap_plane->base;
 
@@ -297,12 +637,88 @@
 	omap_plane_install_properties(plane, &plane->base);
 	drm_plane_create_zpos_property(plane, 0, 0, num_planes - 1);
 
+	if (omap_plane_supports_yuv(plane))
+		drm_plane_create_color_properties(plane,
+					BIT(DRM_COLOR_YCBCR_BT601) |
+					BIT(DRM_COLOR_YCBCR_BT709),
+					BIT(DRM_COLOR_YCBCR_FULL_RANGE) |
+					BIT(DRM_COLOR_YCBCR_LIMITED_RANGE),
+					DRM_COLOR_YCBCR_BT601,
+					DRM_COLOR_YCBCR_FULL_RANGE);
+
+	drm_object_attach_property(&plane->base, priv->global_alpha_prop, 0);
+	drm_object_attach_property(&plane->base, priv->pre_mult_alpha_prop, 0);
+
 	return plane;
 
 error:
 	dev_err(dev->dev, "%s(): could not create plane: %s\n",
-		__func__, plane_id_to_name[id]);
+		__func__, omap_plane->name);
 
 	kfree(omap_plane);
 	return NULL;
 }
+
+enum omap_plane_id omap_plane_id_wb(struct drm_plane *plane)
+{
+	struct omap_plane *omap_plane = to_omap_plane(plane);
+
+	return omap_plane->reserved_wb_overlay->overlay_id;
+}
+
+struct drm_plane *omap_plane_reserve_wb(struct drm_device *dev)
+{
+	struct omap_drm_private *priv = dev->dev_private;
+	int i, ret;
+
+	/*
+	 * Look from the last plane to the first to lessen chances of the
+	 * display side trying to use the same plane as writeback.
+	 */
+	for (i = priv->num_planes - 1; i >= 0; --i) {
+		struct drm_plane *plane = priv->planes[i];
+		struct omap_plane *omap_plane = to_omap_plane(plane);
+		struct omap_hw_overlay *new_ovl = NULL;
+		u32 crtc_mask = (1 << priv->num_pipes) - 1;
+		u32 fourcc = DRM_FORMAT_YUYV;
+		u32 caps = OMAP_DSS_OVL_CAP_SCALE;
+
+		if (plane->state->crtc || plane->state->fb)
+			continue;
+
+		if (omap_plane->reserved_wb_overlay)
+			continue;
+
+		ret = omap_overlay_assign_wb(priv, plane, caps, fourcc,
+					     crtc_mask, &new_ovl);
+		if (ret) {
+			DBG("%s: failed to assign hw_overlay for wb!",
+			    plane->name);
+			return NULL;
+		}
+
+		omap_plane->reserved_wb_overlay = new_ovl;
+
+		return plane;
+	}
+
+	return NULL;
+}
+
+void omap_plane_release_wb(struct drm_plane *plane)
+{
+	struct omap_drm_private *priv = plane->dev->dev_private;
+	struct omap_plane *omap_plane;
+
+	/*
+	 * This is also called on module unload at which point plane might
+	 * not be set. In that case just return as there is nothing to do.
+	 */
+	if (!plane)
+		return;
+
+	omap_plane = to_omap_plane(plane);
+
+	omap_overlay_release_wb(priv, plane, omap_plane->reserved_wb_overlay);
+	omap_plane->reserved_wb_overlay = NULL;
+}
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/omap_plane.h b/drivers/gpu/drm/omapdrm/omap_plane.h
--- a/drivers/gpu/drm/omapdrm/omap_plane.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/omap_plane.h	2020-06-19 21:00:31.412227715 -0400
@@ -33,5 +33,10 @@
 		u32 possible_crtcs);
 void omap_plane_install_properties(struct drm_plane *plane,
 		struct drm_mode_object *obj);
+bool is_omap_plane_dual_overlay(struct drm_plane_state *state);
+
+enum omap_plane_id omap_plane_id_wb(struct drm_plane *plane);
+struct drm_plane *omap_plane_reserve_wb(struct drm_device *dev);
+void omap_plane_release_wb(struct drm_plane *plane);
 
 #endif /* __OMAPDRM_PLANE_H__ */
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/omap_wb.c b/drivers/gpu/drm/omapdrm/omap_wb.c
--- a/drivers/gpu/drm/omapdrm/omap_wb.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/omap_wb.c	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,179 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2016-2018 Texas Instruments Incorporated -  http://www.ti.com/
+ * Author: Benoit Parrot <bparrot@ti.com>
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+
+#include "omap_wb.h"
+
+unsigned int wbdebug;
+module_param(wbdebug, uint, 0644);
+MODULE_PARM_DESC(wbdebug, "activates debug info");
+
+struct wb_fmt wb_formats[] = {
+	{
+		.fourcc		= V4L2_PIX_FMT_NV12,
+		.coplanar	= 0,
+		.depth		= {8, 4},
+	},
+	{
+		.fourcc		= V4L2_PIX_FMT_NV12M,
+		.coplanar	= 1,
+		.depth		= {8, 4},
+	},
+	{
+		.fourcc		= V4L2_PIX_FMT_YUYV,
+		.coplanar	= 0,
+		.depth		= {16, 0},
+	},
+	{
+		.fourcc		= V4L2_PIX_FMT_UYVY,
+		.coplanar	= 0,
+		.depth		= {16, 0},
+	},
+	{
+		/* "XR24", DRM_FORMAT_XRGB8888 */
+		.fourcc		= V4L2_PIX_FMT_XBGR32,
+		.coplanar	= 0,
+		.depth		= {32, 0},
+	},
+};
+
+unsigned int num_wb_formats = ARRAY_SIZE(wb_formats);
+
+/* find our format description corresponding to the passed v4l2_format */
+struct wb_fmt *find_format(struct v4l2_format *f)
+{
+	struct wb_fmt *fmt;
+	unsigned int k;
+
+	for (k = 0; k < num_wb_formats; k++) {
+		fmt = &wb_formats[k];
+		if (fmt->fourcc == f->fmt.pix_mp.pixelformat)
+			return fmt;
+	}
+
+	return NULL;
+}
+
+int omap_wb_fourcc_v4l2_to_drm(u32 fourcc)
+{
+	switch (fourcc) {
+	case V4L2_PIX_FMT_NV12:
+	case V4L2_PIX_FMT_NV12M:
+		return DRM_FORMAT_NV12;
+	case V4L2_PIX_FMT_YUYV:
+		return DRM_FORMAT_YUYV;
+	case V4L2_PIX_FMT_UYVY:
+		return DRM_FORMAT_UYVY;
+	case V4L2_PIX_FMT_XBGR32:
+		return DRM_FORMAT_XRGB8888;
+	default:
+		WARN(1, "WB: unsupported fourcc\n");
+		return 0;
+	}
+}
+
+void omap_wb_irq(void *priv, u32 irqstatus)
+{
+	struct wb_dev *dev = (struct wb_dev *)priv;
+	const u32 mask = OMAP_WB_IRQ_MASK |
+			 DISPC_IRQ_VSYNC |
+			 DISPC_IRQ_VSYNC2 |
+			 DISPC_IRQ_VSYNC3 |
+			 DISPC_IRQ_EVSYNC_EVEN |
+			 DISPC_IRQ_EVSYNC_ODD;
+
+	if (!dev)
+		return;
+
+	irqstatus &= mask;
+	if (!irqstatus)
+		return;
+
+	if (!atomic_read(&dev->irq_enabled))
+		return;
+
+	switch (dev->mode) {
+	case OMAP_WB_NOT_CONFIGURED:
+		break;
+	case OMAP_WB_MEM2MEM_OVL:
+		wbm2m_irq(dev->m2m, irqstatus);
+		break;
+	case OMAP_WB_MEM2MEM_MGR:
+		/* To be added */
+		break;
+	case OMAP_WB_CAPTURE_MGR:
+		wbcap_irq(dev->cap, irqstatus);
+		break;
+	default:
+		WARN_ONCE(1, "WB: unknown WB mode: 0x%x\n", dev->mode);
+		break;
+	}
+}
+
+/*
+ * The initial setup of this device instance. Note that the initial state of
+ * the driver should be complete. So the initial format, standard, timings
+ * and video input should all be initialized to some reasonable value.
+ */
+int omap_wb_init(struct drm_device *drmdev)
+{
+	struct omap_drm_private *priv = drmdev->dev_private;
+	struct wb_dev *dev;
+	int ret = 0;
+
+	/* Allocate a new instance */
+	dev = devm_kzalloc(drmdev->dev, sizeof(*dev), GFP_KERNEL);
+	if (!dev)
+		return -ENOMEM;
+
+	dev->drm_dev = drmdev;
+
+	/* set pseudo v4l2 device name so we can use v4l2_printk */
+	strlcpy(dev->v4l2_dev.name, WB_MODULE_NAME,
+		sizeof(dev->v4l2_dev.name));
+
+	priv->wb_private = dev;
+
+	mutex_init(&dev->lock);
+
+	atomic_set(&dev->irq_enabled, 0);
+
+	dev->mode = OMAP_WB_NOT_CONFIGURED;
+
+	ret = wbcap_init(dev);
+	if (ret) {
+		log_err(dev, "Failed to initialize wb capture\n");
+		goto error;
+	}
+
+	ret = wbm2m_init(dev);
+	if (ret) {
+		log_err(dev, "Failed to initialize wb m2m\n");
+		goto free_cap;
+	}
+
+	log_dbg(dev, "WB loaded\n");
+	return 0;
+
+free_cap:
+	wbcap_cleanup(dev);
+error:
+	return ret;
+}
+
+void omap_wb_cleanup(struct drm_device *drmdev)
+{
+	struct omap_drm_private *priv = drmdev->dev_private;
+	struct wb_dev *dev = priv->wb_private;
+
+	log_dbg(dev, "Cleanup WB\n");
+
+	wbcap_cleanup(dev);
+	wbm2m_cleanup(dev);
+}
+
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/omap_wb_cap.c b/drivers/gpu/drm/omapdrm/omap_wb_cap.c
--- a/drivers/gpu/drm/omapdrm/omap_wb_cap.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/omap_wb_cap.c	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,1044 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2016-2018 Texas Instruments Incorporated -  http://www.ti.com/
+ * Author: Benoit Parrot <bparrot@ti.com>
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+
+#include "omap_wb.h"
+
+static int omap_channel_to_wb_channel(int oc)
+{
+	switch (oc) {
+	case OMAP_DSS_CHANNEL_LCD:
+		return DSS_WB_LCD1_MGR;
+	case OMAP_DSS_CHANNEL_DIGIT:
+		return DSS_WB_TV_MGR;
+	case OMAP_DSS_CHANNEL_LCD2:
+		return DSS_WB_LCD2_MGR;
+	case OMAP_DSS_CHANNEL_LCD3:
+		return DSS_WB_LCD3_MGR;
+	default:
+		return DSS_WB_LCD1_MGR;
+	}
+}
+
+static char *omap_channel_to_name(int oc)
+{
+	switch (oc) {
+	case OMAP_DSS_CHANNEL_LCD:
+		return "LCD1";
+	case OMAP_DSS_CHANNEL_DIGIT:
+		return "DIGIT/TV";
+	case OMAP_DSS_CHANNEL_LCD2:
+		return "LCD2";
+	case OMAP_DSS_CHANNEL_LCD3:
+		return "LCD3";
+	default:
+		return "LCD1";
+	}
+}
+
+/* driver info for each of the supported input overlay/mgr */
+struct wb_input {
+	char name[64];
+	u32 wb_channel;
+	u32 omap_channel;
+	u32 crtc_index;
+};
+
+static struct wb_input wb_inputs[8];
+static int num_wb_input;
+
+static bool is_input_active(struct wbcap_dev *wbcap)
+{
+	struct omap_drm_private *priv = wbcap->dev->drm_dev->dev_private;
+	u32 oc = wb_inputs[wbcap->input].omap_channel;
+
+	return priv->dispc_ops->mgr_is_enabled(priv->dispc, oc);
+}
+
+static bool is_input_enabled(struct wbcap_dev *wbcap)
+{
+	struct omap_drm_private *priv = wbcap->dev->drm_dev->dev_private;
+	struct drm_crtc *crtc;
+	struct wb_input *input;
+
+	input = &wb_inputs[wbcap->input];
+	crtc = priv->pipes[input->crtc_index].crtc;
+
+	return crtc->enabled;
+}
+
+static void build_input_table(struct wbcap_dev *wbcap)
+{
+	struct omap_drm_private *priv = wbcap->dev->drm_dev->dev_private;
+	struct drm_crtc *crtc;
+	struct wb_input *input;
+	int i;
+
+	for (i = 0; i < priv->num_pipes; i++) {
+		crtc = priv->pipes[i].crtc;
+		input = &wb_inputs[i];
+
+		input->crtc_index = i;
+		input->omap_channel = omap_crtc_channel(crtc);
+		input->wb_channel =
+			omap_channel_to_wb_channel(input->omap_channel);
+		snprintf(input->name, sizeof(input->name), "CRTC#%d - %s",
+			 i, omap_channel_to_name(input->omap_channel));
+
+		log_dbg(wbcap, "Input# %d, name:'%s' omap_channel:%d wb_channel:%d\n",
+			i, input->name, input->omap_channel, input->wb_channel);
+	}
+	num_wb_input = i;
+}
+
+static struct wb_q_data *get_q_data(struct wbcap_dev *dev,
+				    enum v4l2_buf_type type)
+{
+	switch (type) {
+	case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
+	case V4L2_BUF_TYPE_VIDEO_CAPTURE:
+		return &dev->q_data[Q_DATA_DST];
+	default:
+		return NULL;
+	}
+	return NULL;
+}
+
+static bool wb_cap_setup(struct wbcap_dev *dev,
+			 enum dss_writeback_channel wb_channel,
+			 const struct omap_dss_writeback_info *wb_info)
+{
+	struct omap_drm_private *priv = dev->dev->drm_dev->dev_private;
+	struct drm_crtc *crtc;
+	struct videomode *ct;
+	int r;
+
+	crtc = priv->pipes[wb_inputs[dev->input].crtc_index].crtc;
+	ct = omap_crtc_timings(crtc);
+
+	/* configure wb */
+	r = priv->dispc_ops->wb_setup(priv->dispc, wb_info, false, ct, wb_channel);
+	if (r)
+		return false;
+
+	if (is_input_active(dev)) {
+		priv->dispc_ops->ovl_enable(priv->dispc, OMAP_DSS_WB, true);
+		priv->dispc_ops->wb_go(priv->dispc);
+	} else {
+		log_err(dev, "CHANNEL %u not enabled, skip WB GO\n",
+			wb_inputs[dev->input].omap_channel);
+	}
+
+	return true;
+}
+
+static bool is_input_irq_vsync_set(struct wbcap_dev *dev, u32 irqstatus)
+{
+	struct omap_drm_private *priv = dev->dev->drm_dev->dev_private;
+	u32 oc = wb_inputs[dev->input].omap_channel;
+
+	if (irqstatus & priv->dispc_ops->mgr_get_vsync_irq(priv->dispc, oc))
+		return true;
+	return false;
+}
+
+static int wbcap_schedule_next_buffer(struct wbcap_dev *dev)
+{
+	struct wb_buffer *buf;
+	unsigned long addr_y = 0;
+	unsigned long addr_uv = 0;
+	struct wb_q_data *q_data;
+	int num_planes;
+	bool ok;
+	struct omap_dss_writeback_info wb_info = { 0 };
+	struct v4l2_pix_format_mplane *pix;
+	unsigned long flags;
+
+	if (!is_input_active(dev)) {
+		dev->next_frm = NULL;
+		return 0;
+	}
+
+	spin_lock_irqsave(&dev->qlock, flags);
+	if (list_empty(&dev->buf_list)) {
+		dev->next_frm = NULL;
+		spin_unlock_irqrestore(&dev->qlock, flags);
+		return 0;
+	}
+
+	buf = list_entry(dev->buf_list.next, struct wb_buffer, list);
+	dev->next_frm = buf;
+	list_del(&buf->list);
+	spin_unlock_irqrestore(&dev->qlock, flags);
+
+	q_data = get_q_data(dev, buf->vb.vb2_buf.type);
+	if (!q_data)
+		return -EINVAL;
+
+	pix = &q_data->format.fmt.pix_mp;
+	num_planes = pix->num_planes;
+
+	addr_y = vb2_dma_contig_plane_dma_addr(&buf->vb.vb2_buf, 0);
+	if (num_planes == 2)
+		addr_uv = vb2_dma_contig_plane_dma_addr(&buf->vb.vb2_buf, 1);
+	else if (pix->pixelformat == V4L2_PIX_FMT_NV12)
+		addr_uv = addr_y + (pix->plane_fmt[0].bytesperline *
+				    pix->height);
+
+	/* fill WB DSS info */
+	wb_info.paddr = (u32)addr_y;
+	wb_info.p_uv_addr = (u32)addr_uv;
+	wb_info.buf_width = pix->plane_fmt[0].bytesperline /
+			    (q_data->fmt->depth[LUMA_PLANE] / 8);
+
+	wb_info.width = pix->width;
+	wb_info.height = pix->height;
+	wb_info.fourcc = omap_wb_fourcc_v4l2_to_drm(pix->pixelformat);
+	wb_info.pre_mult_alpha = 1;
+
+	wb_info.rotation = DRM_MODE_ROTATE_0;
+	wb_info.rotation_type = OMAP_DSS_ROT_NONE;
+
+	ok = wb_cap_setup(dev,
+			  wb_inputs[dev->input].wb_channel,
+			  &wb_info);
+	if (!ok)
+		return -EINVAL;
+
+	return 0;
+}
+
+static void wbcap_process_buffer_complete(struct wbcap_dev *dev)
+{
+	dev->cur_frm->vb.vb2_buf.timestamp = ktime_get_ns();
+	dev->cur_frm->vb.field = dev->field;
+	dev->cur_frm->vb.sequence = dev->sequence++;
+
+	vb2_buffer_done(&dev->cur_frm->vb.vb2_buf, VB2_BUF_STATE_DONE);
+	dev->cur_frm = dev->next_frm;
+}
+
+static enum hrtimer_restart wbcap_wbgo_timer(struct hrtimer *timer)
+{
+	struct wbcap_dev *dev = container_of(timer,
+					     struct wbcap_dev, wbgo_timer);
+	struct omap_drm_private *priv = dev->dev->drm_dev->dev_private;
+
+	if (priv->dispc_ops->wb_go_busy(priv->dispc))
+		log_err(dev, "WARNING, WB BUSY at hrtimer, state %u\n",
+			dev->state);
+
+	switch (dev->state) {
+	case WB_STATE_NONE:
+		break;
+
+	case WB_STATE_FIRST_FRAME:
+		dev->cur_frm = dev->next_frm;
+		wbcap_schedule_next_buffer(dev);
+		dev->state = WB_STATE_CAPTURING;
+		break;
+
+	case WB_STATE_CAPTURING:
+		if (dev->cur_frm && dev->next_frm) {
+			/*
+			 * We have cur_frm that was just captured, and next_frm
+			 * to which the HW will start capturing.
+			 * This means cur_frm is now released from DSS HW.
+			 */
+			wbcap_process_buffer_complete(dev);
+			dev->next_frm = NULL;
+		} else {
+			/*
+			 * We have cur_frm which has a captured frame,
+			 * but we don't have next_frm.
+			 * This means cur_frm is will still be used by
+			 * DSS for capture
+			 */
+		}
+
+		if (dev->stopping) {
+			/* XXX should we set WB GO? */
+			priv->dispc_ops->ovl_enable(priv->dispc, OMAP_DSS_WB,
+						    false);
+			dev->state = WB_STATE_STOPPING;
+		} else {
+			wbcap_schedule_next_buffer(dev);
+		}
+		break;
+
+	case WB_STATE_STOPPING:
+		if (dev->cur_frm)
+			wbcap_process_buffer_complete(dev);
+
+		dev->state = WB_STATE_STOPPED;
+		atomic_dec(&dev->dev->irq_enabled);
+		dev->stopping = false;
+		wake_up(&dev->event);
+		break;
+
+	case WB_STATE_STOPPED:
+		log_err(dev, "ERROR: timer triggered in the stopped state. This shouldn't happen\n");
+		break;
+	}
+
+	return HRTIMER_NORESTART;
+}
+
+static void wbcap_handle_vsync(struct wbcap_dev *dev)
+{
+	/*
+	 * In writeback capture mode, the GO bit doesn't get reset
+	 * at the manager's VSYNC interrupt. It takes an extra
+	 * 'WBDELAYCOUNTER' time after VSYNC when the writeback
+	 * FIFOs are flushed and the shadow registers are taken in.
+	 * There isn't any DSS interrupt to notify this point in time.
+	 * The correct solution is to set a timer far enough that it
+	 * should cover the period defined by WBDELAYCOUNTER.
+	 * The max value allowed in WBDELAYCOUNTER is 255 which
+	 * correspond to 255 lines. So waiting anywhere from 1/4 to
+	 * 1/2 a frame (i.e. 2ms at 60  to 120 fps) should be safe
+	 * enough.
+	 */
+
+	hrtimer_start_range_ns(&dev->wbgo_timer, ms_to_ktime(3), 1000000,
+			       HRTIMER_MODE_REL);
+}
+
+void wbcap_irq(struct wbcap_dev *dev, u32 irqstatus)
+{
+	if (irqstatus & DISPC_IRQ_FRAMEDONEWB)
+		log_dbg(dev, "WB: FRAMEDONE\n");
+
+	if (irqstatus & DISPC_IRQ_WBBUFFEROVERFLOW)
+		log_err(dev, "WB: UNDERFLOW\n");
+
+	if (irqstatus & DISPC_IRQ_WBUNCOMPLETEERROR)
+		log_err(dev, "WB: WBUNCOMPLETEERROR\n");
+
+	if (is_input_irq_vsync_set(dev, irqstatus)) {
+		if (dev->field != V4L2_FIELD_NONE) {
+			if (irqstatus & DISPC_IRQ_EVSYNC_EVEN)
+				dev->field = V4L2_FIELD_BOTTOM;
+			else if (irqstatus & DISPC_IRQ_EVSYNC_ODD)
+				dev->field = V4L2_FIELD_TOP;
+		}
+		wbcap_handle_vsync(dev);
+	}
+}
+
+/*
+ * Setup the constraints of the queue: besides setting the number of planes
+ * per buffer and the size and allocation context of each plane, it also
+ * checks if sufficient buffers have been allocated. Usually 3 is a good
+ * minimum number: many DMA engines need a minimum of 2 buffers in the
+ * queue and you need to have another available for userspace processing.
+ */
+static int queue_setup(struct vb2_queue *vq,
+		       unsigned int *nbuffers, unsigned int *nplanes,
+		       unsigned int sizes[], struct device *alloc_devs[])
+{
+	int i;
+	struct wbcap_dev *wbcap = vb2_get_drv_priv(vq);
+	struct wb_q_data *q_data;
+
+	q_data = get_q_data(wbcap, vq->type);
+
+	if (!q_data)
+		return -EINVAL;
+
+	if (vq->num_buffers + *nbuffers < 2)
+		*nbuffers = 2 - vq->num_buffers;
+
+	*nplanes = q_data->format.fmt.pix_mp.num_planes;
+
+	for (i = 0; i < *nplanes; i++)
+		sizes[i] = q_data->format.fmt.pix_mp.plane_fmt[i].sizeimage;
+
+	log_dbg(wbcap, "get %d buffer(s) of size %d\n", *nbuffers,
+		sizes[LUMA_PLANE]);
+	if (*nplanes == 2)
+		log_dbg(wbcap, " and %d\n", sizes[CHROMA_PLANE]);
+
+	return 0;
+}
+
+/*
+ * Prepare the buffer for queueing to the DMA engine: check and set the
+ * payload size.
+ */
+static int buffer_prepare(struct vb2_buffer *vb)
+{
+	struct wbcap_dev *wbcap = vb2_get_drv_priv(vb->vb2_queue);
+	struct wb_q_data *q_data;
+	struct v4l2_pix_format_mplane *mp;
+	int i, num_planes;
+
+	q_data = get_q_data(wbcap, vb->vb2_queue->type);
+	if (!q_data)
+		return -EINVAL;
+	num_planes = q_data->format.fmt.pix_mp.num_planes;
+
+	for (i = 0; i < num_planes; i++) {
+		mp = &q_data->format.fmt.pix_mp;
+		if (vb2_plane_size(vb, i) < mp->plane_fmt[i].sizeimage) {
+			log_err(wbcap,
+				"data will not fit into plane (%lu < %lu)\n",
+				vb2_plane_size(vb, i),
+				(long)mp->plane_fmt[i].sizeimage);
+			return -EINVAL;
+		}
+		vb2_set_plane_payload(vb, i, mp->plane_fmt[i].sizeimage);
+	}
+
+	return 0;
+}
+
+/*
+ * Queue this buffer to the DMA engine.
+ */
+static void buffer_queue(struct vb2_buffer *vb)
+{
+	struct wbcap_dev *wbcap = vb2_get_drv_priv(vb->vb2_queue);
+	struct wb_buffer *buf = to_wb_buffer(vb);
+	unsigned long flags;
+
+	spin_lock_irqsave(&wbcap->qlock, flags);
+	list_add_tail(&buf->list, &wbcap->buf_list);
+
+	spin_unlock_irqrestore(&wbcap->qlock, flags);
+}
+
+static void return_all_buffers(struct wbcap_dev *wbcap,
+			       enum vb2_buffer_state state)
+{
+	struct wb_buffer *buf, *node;
+	unsigned long flags;
+
+	spin_lock_irqsave(&wbcap->qlock, flags);
+	list_for_each_entry_safe(buf, node, &wbcap->buf_list, list) {
+		vb2_buffer_done(&buf->vb.vb2_buf, state);
+		list_del(&buf->list);
+	}
+
+	if (wbcap->cur_frm) {
+		vb2_buffer_done(&wbcap->cur_frm->vb.vb2_buf, state);
+		wbcap->cur_frm = NULL;
+	}
+
+	if (wbcap->next_frm) {
+		vb2_buffer_done(&wbcap->next_frm->vb.vb2_buf, state);
+		wbcap->next_frm = NULL;
+	}
+
+	spin_unlock_irqrestore(&wbcap->qlock, flags);
+}
+
+/*
+ * Start streaming. First check if the minimum number of buffers have been
+ * queued. If not, then return -ENOBUFS and the vb2 framework will call
+ * this function again the next time a buffer has been queued until enough
+ * buffers are available to actually start the DMA engine.
+ */
+static int start_streaming(struct vb2_queue *vq, unsigned int count)
+{
+	struct wbcap_dev *wbcap = vb2_get_drv_priv(vq);
+	struct omap_drm_private *priv = wbcap->dev->drm_dev->dev_private;
+	struct drm_crtc *crtc;
+	int ret;
+	struct wb_q_data *q_data;
+
+	priv->dispc_ops->runtime_get(priv->dispc);
+
+	wbcap->sequence = 0;
+	q_data = get_q_data(wbcap, wbcap->queue.type);
+	if (!q_data) {
+		log_err(wbcap, "ERROR: getting q_data failed\n");
+		return_all_buffers(wbcap, VB2_BUF_STATE_QUEUED);
+		priv->dispc_ops->runtime_put(priv->dispc);
+		return -EINVAL;
+	}
+
+	if (q_data->format.fmt.pix_mp.field == V4L2_FIELD_ALTERNATE)
+		wbcap->field = V4L2_FIELD_TOP;
+	else
+		wbcap->field = V4L2_FIELD_NONE;
+
+	log_dbg(wbcap, "Input (%s) is %s : %s\n",
+		wb_inputs[wbcap->input].name,
+		is_input_enabled(wbcap) ? "enabled" : "disabled",
+		is_input_active(wbcap) ? "active" : "inactive");
+
+	if (!is_input_active(wbcap)) {
+		log_err(wbcap, "ERROR: Selected input (%s) is not active, bailing out\n",
+			wb_inputs[wbcap->input].name);
+		return_all_buffers(wbcap, VB2_BUF_STATE_QUEUED);
+		priv->dispc_ops->runtime_put(priv->dispc);
+		return -EINVAL;
+	}
+
+	/* Enable vsync irq on the input crtc */
+	crtc = priv->pipes[wb_inputs[wbcap->input].crtc_index].crtc;
+	ret = drm_crtc_vblank_get(crtc);
+	WARN_ON(ret != 0);
+
+	if (wbcap_schedule_next_buffer(wbcap)) {
+		return_all_buffers(wbcap, VB2_BUF_STATE_QUEUED);
+		priv->dispc_ops->runtime_put(priv->dispc);
+		return -EINVAL;
+	}
+
+	wbcap->state = WB_STATE_FIRST_FRAME;
+	atomic_inc(&wbcap->dev->irq_enabled);
+	return 0;
+}
+
+/*
+ * Stop the DMA engine. Any remaining buffers in the DMA queue are dequeued
+ * and passed on to the vb2 framework marked as STATE_ERROR.
+ */
+static void stop_streaming(struct vb2_queue *vq)
+{
+	struct wbcap_dev *wbcap = vb2_get_drv_priv(vq);
+	struct omap_drm_private *priv = wbcap->dev->drm_dev->dev_private;
+	struct drm_crtc *crtc;
+	int ret;
+
+	log_dbg(wbcap, "Stopping WB\n");
+	log_dbg(wbcap, "current state: %d\n", wbcap->state);
+
+	wbcap->stopping = true;
+	ret = wait_event_timeout(wbcap->event,
+				 !wbcap->stopping,
+				 msecs_to_jiffies(250));
+
+	log_dbg(wbcap, "Returning VB2 buffers\n");
+
+	if (priv->dispc_ops->wb_go_busy(priv->dispc))
+		log_err(wbcap, "WARNING, WB BUSY when stopping\n");
+
+	/* Release all active buffers */
+	return_all_buffers(wbcap, VB2_BUF_STATE_ERROR);
+
+	/* Disable vsync irq on the input crtc */
+	crtc = priv->pipes[wb_inputs[wbcap->input].crtc_index].crtc;
+	drm_crtc_vblank_put(crtc);
+
+	priv->dispc_ops->runtime_put(priv->dispc);
+}
+
+/*
+ * The vb2 queue ops. Note that since q->lock is set we can use the standard
+ * vb2_ops_wait_prepare/finish helper functions. If q->lock would be NULL,
+ * then this driver would have to provide these ops.
+ */
+static struct vb2_ops wbcap_qops = {
+	.queue_setup		= queue_setup,
+	.buf_prepare		= buffer_prepare,
+	.buf_queue		= buffer_queue,
+	.start_streaming	= start_streaming,
+	.stop_streaming		= stop_streaming,
+	.wait_prepare		= vb2_ops_wait_prepare,
+	.wait_finish		= vb2_ops_wait_finish,
+};
+
+/*
+ * Required ioctl querycap. Note that the version field is prefilled with
+ * the version of the kernel.
+ */
+static int wbcap_querycap(struct file *file, void *priv,
+			  struct v4l2_capability *cap)
+{
+	struct wbcap_dev *wbcap = video_drvdata(file);
+
+	strlcpy(cap->driver, WBCAP_MODULE_NAME, sizeof(cap->driver));
+	strlcpy(cap->card, WBCAP_MODULE_NAME, sizeof(cap->card));
+	snprintf(cap->bus_info, sizeof(cap->bus_info), "platform:%s",
+		 wbcap->v4l2_dev.name);
+	cap->device_caps = V4L2_CAP_VIDEO_CAPTURE_MPLANE | V4L2_CAP_READWRITE |
+			   V4L2_CAP_STREAMING;
+	cap->capabilities = cap->device_caps | V4L2_CAP_DEVICE_CAPS;
+	return 0;
+}
+
+/*
+ * Helper function to check and correct struct v4l2_pix_format. It's used
+ * not only in VIDIOC_TRY/S_FMT, but also elsewhere if changes to the SDTV
+ * standard, HDTV timings or the video input would require updating the
+ * current format.
+ */
+static int wbcap_fill_pix_format(struct wbcap_dev *wbcap,
+				 struct v4l2_format *f)
+{
+	struct wb_fmt *fmt = find_format(f);
+	struct v4l2_pix_format_mplane *pix = &f->fmt.pix_mp;
+	struct v4l2_plane_pix_format *plane_fmt;
+	unsigned int w_align;
+	int i, depth, depth_bytes;
+
+	if (!fmt) {
+		log_dbg(wbcap, "Fourcc format (0x%08x) invalid.\n",
+			pix->pixelformat);
+		fmt = &wb_formats[1];
+	}
+
+	/* we only allow V4L2_FIELD_NONE or V4L2_FIELD_ALTERNATE */
+	if (pix->field != V4L2_FIELD_NONE &&
+	    pix->field != V4L2_FIELD_ALTERNATE)
+		pix->field = V4L2_FIELD_NONE;
+
+	depth = fmt->depth[LUMA_PLANE];
+
+	/*
+	 * The line stride needs to be even is even.
+	 * Special case is with YUV422 interleaved format an even number
+	 * of pixels is required also.
+	 */
+	depth_bytes = depth >> 3;
+
+	w_align = 0;
+	if ((depth_bytes == 3) || (depth_bytes == 1))
+		w_align = 1;
+	else if ((depth_bytes == 2) &&
+		 (fmt->fourcc == V4L2_PIX_FMT_YUYV ||
+		  fmt->fourcc == V4L2_PIX_FMT_UYVY))
+		w_align = 1;
+
+	v4l_bound_align_image(&pix->width, MIN_W, MAX_W, w_align,
+			      &pix->height, MIN_H, MAX_H, H_ALIGN,
+			      S_ALIGN);
+	pix->num_planes = fmt->coplanar ? 2 : 1;
+	pix->pixelformat = fmt->fourcc;
+
+	pix->colorspace = V4L2_COLORSPACE_SRGB;
+	pix->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
+	pix->quantization = V4L2_QUANTIZATION_DEFAULT;
+	pix->xfer_func = V4L2_XFER_FUNC_DEFAULT;
+
+	memset(pix->reserved, 0, sizeof(pix->reserved));
+	for (i = 0; i < pix->num_planes; i++) {
+		plane_fmt = &pix->plane_fmt[i];
+		depth = fmt->depth[i];
+
+		if (i == LUMA_PLANE)
+			plane_fmt->bytesperline = pix->width * depth / 8;
+		else
+			plane_fmt->bytesperline = pix->width;
+
+		plane_fmt->sizeimage = (pix->height * pix->width *
+					depth) / 8;
+
+		if (fmt->fourcc == V4L2_PIX_FMT_NV12) {
+			/*
+			 * Since we are using a single plane buffer
+			 * we need to adjust the reported sizeimage
+			 * to include the colocated UV part.
+			 */
+			plane_fmt->sizeimage += (pix->height / 2 *
+				plane_fmt->bytesperline);
+		}
+
+		memset(plane_fmt->reserved, 0, sizeof(plane_fmt->reserved));
+	}
+
+	return 0;
+}
+
+static int wbcap_try_fmt_vid_cap(struct file *file, void *priv,
+				 struct v4l2_format *f)
+{
+	struct wbcap_dev *wbcap = video_drvdata(file);
+	struct omap_drm_private *drmpriv = wbcap->dev->drm_dev->dev_private;
+	struct drm_crtc *crtc;
+	struct videomode *ct;
+
+	log_dbg(wbcap, "requested fourcc:%4.4s size: %dx%d\n",
+		(char *)&f->fmt.pix_mp.pixelformat,
+		f->fmt.pix_mp.width, f->fmt.pix_mp.height);
+
+	/*
+	 * Scaling currently does not work properly for Capture mode.
+	 * So we are temporarily forcing the frame size to be the
+	 * same as the source crtc for now.
+	 */
+	crtc = drmpriv->pipes[wb_inputs[wbcap->input].crtc_index].crtc;
+	ct = omap_crtc_timings(crtc);
+
+	f->fmt.pix.width = ct->hactive;
+	f->fmt.pix.height = ct->vactive;
+
+	if (ct->flags & DISPLAY_FLAGS_INTERLACED) {
+		f->fmt.pix.height /= 2;
+		f->fmt.pix_mp.field = V4L2_FIELD_ALTERNATE;
+	}
+
+	log_dbg(wbcap, "replied fourcc:%4.4s size: %dx%d\n",
+		(char *)&f->fmt.pix_mp.pixelformat,
+		f->fmt.pix_mp.width, f->fmt.pix_mp.height);
+
+	return wbcap_fill_pix_format(wbcap, f);
+}
+
+static int wbcap_s_fmt_vid_cap(struct file *file, void *priv,
+			       struct v4l2_format *f)
+{
+	struct wbcap_dev *wbcap = video_drvdata(file);
+	int ret;
+	struct wb_q_data *q_data;
+
+	log_dbg(wbcap, "type:%d\n", f->type);
+
+	ret = wbcap_try_fmt_vid_cap(file, priv, f);
+	if (ret)
+		return ret;
+
+	q_data = get_q_data(wbcap, f->type);
+	if (!q_data)
+		return -EINVAL;
+
+	/*
+	 * It is not allowed to change the format while buffers for use with
+	 * streaming have already been allocated.
+	 */
+	if (vb2_is_busy(&wbcap->queue))
+		return -EBUSY;
+
+	q_data->format = *f;
+	q_data->fmt = find_format(f);
+
+	log_dbg(wbcap, "Setting format for type %d, %dx%d, fmt: %4.4s bpl_y %d",
+		f->type, f->fmt.pix_mp.width, f->fmt.pix_mp.height,
+		(char *)&f->fmt.pix_mp.pixelformat,
+		f->fmt.pix_mp.plane_fmt[LUMA_PLANE].bytesperline);
+	if (f->fmt.pix_mp.num_planes == 2)
+		log_dbg(wbcap, " bpl_uv %d\n",
+			f->fmt.pix_mp.plane_fmt[CHROMA_PLANE].bytesperline);
+
+	return 0;
+}
+
+static int wbcap_g_fmt_vid_cap(struct file *file, void *priv,
+			       struct v4l2_format *f)
+{
+	struct wbcap_dev *wbcap = video_drvdata(file);
+	struct wb_q_data *q_data;
+
+	log_dbg(wbcap, "type:%d\n", f->type);
+
+	q_data = get_q_data(wbcap, f->type);
+	if (!q_data)
+		return -EINVAL;
+
+	*f = q_data->format;
+	return 0;
+}
+
+static int wbcap_enum_fmt_vid_cap(struct file *file, void *priv,
+				  struct v4l2_fmtdesc *f)
+{
+	if (f->index >= num_wb_formats)
+		return -EINVAL;
+
+	f->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+	f->pixelformat = wb_formats[f->index].fourcc;
+	return 0;
+}
+
+static int wbcap_enum_input(struct file *file, void *priv,
+			    struct v4l2_input *i)
+{
+	if (i->index >= num_wb_input)
+		return -EINVAL;
+
+	i->type = V4L2_INPUT_TYPE_CAMERA;
+	strlcpy(i->name, wb_inputs[i->index].name, sizeof(i->name));
+	return 0;
+}
+
+static int wbcap_s_input(struct file *file, void *priv, unsigned int i)
+{
+	struct wbcap_dev *wbcap = video_drvdata(file);
+	struct wb_q_data *q_data;
+
+	log_dbg(wbcap, "%d\n", i);
+
+	q_data = get_q_data(wbcap, wbcap->queue.type);
+	if (!q_data)
+		return -EINVAL;
+
+	if (i >= num_wb_input)
+		return -EINVAL;
+
+	/*
+	 * Changing the input implies a format change, which is not allowed
+	 * while buffers for use with streaming have already been allocated.
+	 */
+	if (vb2_is_busy(&wbcap->queue))
+		return -EBUSY;
+
+	wbcap->input = i;
+
+	/* Update the internal format to match the selected input */
+	wbcap_try_fmt_vid_cap(file, priv, &q_data->format);
+	return 0;
+}
+
+static int wbcap_g_input(struct file *file, void *priv, unsigned int *i)
+{
+	struct wbcap_dev *wbcap = video_drvdata(file);
+
+	log_dbg(wbcap, "%d\n", wbcap->input);
+
+	*i = wbcap->input;
+	return 0;
+}
+
+/*
+ * File operations
+ */
+static int wbcap_open(struct file *file)
+{
+	struct wbcap_dev *dev = video_drvdata(file);
+	int ret;
+
+	log_dbg(dev, "enter\n");
+
+	if (mutex_lock_interruptible(&dev->dev->lock)) {
+		ret = -ERESTARTSYS;
+		goto unlock;
+	}
+
+	if ((dev->dev->mode != OMAP_WB_NOT_CONFIGURED) &&
+	    (dev->dev->mode != OMAP_WB_CAPTURE_MGR)) {
+		/* WB is already open for other modes */
+		ret = -EBUSY;
+		goto unlock;
+	}
+
+	ret = v4l2_fh_open(file);
+	if (ret) {
+		log_err(dev, "v4l2_fh_open failed\n");
+		goto unlock;
+	}
+
+	if (v4l2_fh_is_singular_file(file))
+		dev->dev->mode = OMAP_WB_CAPTURE_MGR;
+
+unlock:
+	mutex_unlock(&dev->dev->lock);
+	return ret;
+}
+
+static int wbcap_release(struct file *file)
+{
+	struct wbcap_dev *dev = video_drvdata(file);
+	bool fh_singular;
+	int ret;
+
+	log_dbg(dev, "releasing\n");
+
+	mutex_lock(&dev->dev->lock);
+
+	/* Save the singular status before we call the clean-up helper */
+	fh_singular = v4l2_fh_is_singular_file(file);
+
+	/* the release helper will cleanup any on-going streaming */
+	ret = _vb2_fop_release(file, NULL);
+
+	if (fh_singular)
+		dev->dev->mode = OMAP_WB_NOT_CONFIGURED;
+
+	mutex_unlock(&dev->dev->lock);
+
+	return ret;
+}
+
+/*
+ * The set of all supported ioctls. Note that all the streaming ioctls
+ * use the vb2 helper functions that take care of all the locking and
+ * that also do ownership tracking (i.e. only the filehandle that requested
+ * the buffers can call the streaming ioctls, all other filehandles will
+ * receive -EBUSY if they attempt to call the same streaming ioctls).
+ *
+ * The last three ioctls also use standard helper functions: these implement
+ * standard behavior for drivers with controls.
+ */
+static const struct v4l2_ioctl_ops wbcap_ioctl_ops = {
+	.vidioc_querycap = wbcap_querycap,
+	.vidioc_try_fmt_vid_cap_mplane = wbcap_try_fmt_vid_cap,
+	.vidioc_s_fmt_vid_cap_mplane = wbcap_s_fmt_vid_cap,
+	.vidioc_g_fmt_vid_cap_mplane = wbcap_g_fmt_vid_cap,
+	.vidioc_enum_fmt_vid_cap_mplane = wbcap_enum_fmt_vid_cap,
+
+	.vidioc_enum_input = wbcap_enum_input,
+	.vidioc_g_input = wbcap_g_input,
+	.vidioc_s_input = wbcap_s_input,
+
+	.vidioc_reqbufs = vb2_ioctl_reqbufs,
+	.vidioc_create_bufs = vb2_ioctl_create_bufs,
+	.vidioc_prepare_buf = vb2_ioctl_prepare_buf,
+	.vidioc_querybuf = vb2_ioctl_querybuf,
+	.vidioc_qbuf = vb2_ioctl_qbuf,
+	.vidioc_dqbuf = vb2_ioctl_dqbuf,
+	.vidioc_expbuf = vb2_ioctl_expbuf,
+	.vidioc_streamon = vb2_ioctl_streamon,
+	.vidioc_streamoff = vb2_ioctl_streamoff,
+
+	.vidioc_log_status = v4l2_ctrl_log_status,
+	.vidioc_subscribe_event = v4l2_ctrl_subscribe_event,
+	.vidioc_unsubscribe_event = v4l2_event_unsubscribe,
+};
+
+/*
+ * The set of file operations. Note that all these ops are standard core
+ * helper functions.
+ */
+static const struct v4l2_file_operations wbcap_fops = {
+	.owner = THIS_MODULE,
+	.open = wbcap_open,
+	.release = wbcap_release,
+	.unlocked_ioctl = video_ioctl2,
+	.read = vb2_fop_read,
+	.mmap = vb2_fop_mmap,
+	.poll = vb2_fop_poll,
+};
+
+/*
+ * The initial setup of this device instance. Note that the initial state of
+ * the driver should be complete. So the initial format, standard, timings
+ * and video input should all be initialized to some reasonable value.
+ */
+int wbcap_init(struct wb_dev *dev)
+{
+	struct wbcap_dev *wbcap;
+	struct video_device *vdev;
+	struct vb2_queue *q;
+	struct wb_q_data *q_data;
+	int ret;
+
+	if (!dev)
+		return -ENOMEM;
+
+	/* Allocate a new instance */
+	wbcap = devm_kzalloc(dev->drm_dev->dev, sizeof(*wbcap), GFP_KERNEL);
+	if (!wbcap)
+		return -ENOMEM;
+
+	dev->cap = wbcap;
+	wbcap->dev = dev;
+
+	/* Fill in the initial format-related settings */
+	q_data = &wbcap->q_data[Q_DATA_DST];
+	q_data->fmt = &wb_formats[1];
+	q_data->format.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+	q_data->format.fmt.pix_mp.pixelformat = q_data->fmt->fourcc;
+	q_data->format.fmt.pix_mp.width = 1920;
+	q_data->format.fmt.pix_mp.height = 1080;
+	wbcap_fill_pix_format(wbcap, &q_data->format);
+
+	/* Initialize the top-level structure */
+	strlcpy(wbcap->v4l2_dev.name, WBCAP_MODULE_NAME,
+		sizeof(wbcap->v4l2_dev.name));
+	ret = v4l2_device_register(dev->drm_dev->dev, &wbcap->v4l2_dev);
+	if (ret)
+		return ret;
+
+	/*
+	 * This lock is now created by the main level.
+	 * We might need one per sub structure in the future
+	 *
+	 *  mutex_init(&dev->lock);
+	 */
+
+	/* Initialize the vb2 queue */
+	q = &wbcap->queue;
+	q->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+	q->io_modes = VB2_MMAP | VB2_DMABUF | VB2_READ;
+	q->drv_priv = wbcap;
+	q->buf_struct_size = sizeof(struct wb_buffer);
+	q->ops = &wbcap_qops;
+	q->mem_ops = &vb2_dma_contig_memops;
+	q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
+	q->dev = wbcap->v4l2_dev.dev;
+
+	/*
+	 * Assume that this DMA engine needs to have at least two buffers
+	 * available before it can be started. The start_streaming() op
+	 * won't be called until at least this many buffers are queued up.
+	 */
+	q->min_buffers_needed = 2;
+	/*
+	 * The serialization lock for the streaming ioctls. This is the same
+	 * as the main serialization lock, but if some of the non-streaming
+	 * ioctls could take a long time to execute, then you might want to
+	 * have a different lock here to prevent VIDIOC_DQBUF from being
+	 * blocked while waiting for another action to finish. This is
+	 * generally not needed for PCI devices, but USB devices usually do
+	 * want a separate lock here.
+	 */
+	q->lock = &dev->lock;
+	/*
+	 * Since this driver can only do 32-bit DMA we must make sure that
+	 * the vb2 core will allocate the buffers in 32-bit DMA memory.
+	 */
+	q->gfp_flags = GFP_DMA32;
+	ret = vb2_queue_init(q);
+	if (ret)
+		goto free_hdl;
+
+	INIT_LIST_HEAD(&wbcap->buf_list);
+	spin_lock_init(&wbcap->qlock);
+
+	/* Initialize the video_device structure */
+	vdev = &wbcap->vdev;
+	strlcpy(vdev->name, WBCAP_MODULE_NAME, sizeof(vdev->name));
+	/*
+	 * There is nothing to clean up, so release is set to an empty release
+	 * function. The release callback must be non-NULL.
+	 */
+	vdev->release = video_device_release_empty;
+	vdev->fops = &wbcap_fops,
+	vdev->ioctl_ops = &wbcap_ioctl_ops,
+	/*
+	 * The main serialization lock. All ioctls are serialized by this
+	 * lock. Exception: if q->lock is set, then the streaming ioctls
+	 * are serialized by that separate lock.
+	 */
+	vdev->lock = &dev->lock;
+	vdev->queue = q;
+	vdev->v4l2_dev = &wbcap->v4l2_dev;
+	video_set_drvdata(vdev, wbcap);
+
+	hrtimer_init(&wbcap->wbgo_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+	wbcap->wbgo_timer.function = wbcap_wbgo_timer;
+
+	init_waitqueue_head(&wbcap->event);
+	wbcap->stopping = false;
+
+	build_input_table(wbcap);
+
+	ret = video_register_device(vdev, VFL_TYPE_GRABBER, 11);
+	if (ret)
+		goto free_hdl;
+
+	log_dbg(wbcap, "Device registered as %s\n",
+		video_device_node_name(vdev));
+	return 0;
+
+free_hdl:
+	v4l2_device_unregister(&wbcap->v4l2_dev);
+	return ret;
+}
+
+void wbcap_cleanup(struct wb_dev *dev)
+{
+	log_dbg(dev, "Cleanup WB Capture\n");
+
+	video_unregister_device(&dev->cap->vdev);
+	v4l2_device_unregister(&dev->cap->v4l2_dev);
+}
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/omap_wb.h b/drivers/gpu/drm/omapdrm/omap_wb.h
--- a/drivers/gpu/drm/omapdrm/omap_wb.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/omap_wb.h	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,214 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2016-2018 Texas Instruments Incorporated -  http://www.ti.com/
+ * Author: Benoit Parrot <bparrot@ti.com>
+ */
+
+#ifndef __OMAP_WB_H__
+#define __OMAP_WB_H__
+
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/mutex.h>
+#include <linux/wait.h>
+#include <linux/hrtimer.h>
+#include <drm/drm_fourcc.h>
+
+#include <linux/videodev2.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-dev.h>
+#include <media/v4l2-ioctl.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-event.h>
+#include <media/videobuf2-v4l2.h>
+#include <media/videobuf2-dma-contig.h>
+
+#include "dss/omapdss.h"
+#include "omap_drv.h"
+
+#define WB_MODULE_NAME "omapwb"
+#define WBM2M_MODULE_NAME "omapwb-m2m"
+#define WBCAP_MODULE_NAME "omapwb-cap"
+
+extern unsigned int wbdebug;
+
+#define log_dbg(dev, fmt, arg...)	\
+		v4l2_dbg(1, wbdebug, &dev->v4l2_dev, "%s: " fmt, \
+			 __func__, ## arg)
+#define log_err(dev, fmt, arg...)	\
+		v4l2_err(&dev->v4l2_dev, fmt, ## arg)
+#define log_info(dev, fmt, arg...)	\
+		v4l2_info(&dev->v4l2_dev, fmt, ## arg)
+
+/* minimum and maximum frame sizes */
+#define MIN_W		32
+#define MIN_H		32
+#define MAX_W		2048
+#define MAX_H		2048
+
+/* required alignments */
+#define S_ALIGN		0	/* multiple of 1 */
+#define H_ALIGN		0	/* multiple of 2 */
+
+/* used as plane indices */
+#define MAX_PLANES	2
+#define LUMA_PLANE	0
+#define CHROMA_PLANE	1
+
+enum omap_wb_mode {
+	OMAP_WB_NOT_CONFIGURED = 0,
+	/* mem2mem from single ovl to wb */
+	OMAP_WB_MEM2MEM_OVL = 1,
+	/* mem2mem from N overlays via single mgr to wb */
+	OMAP_WB_MEM2MEM_MGR = 2,
+	/* capture from single mgr to wb */
+	OMAP_WB_CAPTURE_MGR = 3
+};
+
+enum wb_state {
+	WB_STATE_NONE = 0,
+	WB_STATE_FIRST_FRAME,
+	WB_STATE_CAPTURING,
+	WB_STATE_STOPPING,
+	WB_STATE_STOPPED,
+};
+
+/* driver info for each of the supported video formats */
+struct wb_fmt {
+	u32	fourcc;			/* standard format identifier */
+	u8	coplanar;		/* set for unpacked Luma and Chroma */
+	u8	depth[MAX_PLANES];	/* Bits per pixel per plane*/
+};
+
+extern struct wb_fmt wb_formats[];
+extern unsigned int num_wb_formats;
+
+struct wb_buffer {
+	struct vb2_v4l2_buffer	vb;
+	struct list_head	list;
+};
+
+/*
+ * per-queue, driver-specific private data.
+ * MEM-2-MEM: Source: V4L2_BUF_TYPE_VIDEO_OUTPUT*
+ *            Destination: V4L2_BUF_TYPE_VIDEO_CAPTURE*
+ * CAPTURE:   Destination: V4L2_BUF_TYPE_VIDEO_CAPTURE* only
+ */
+struct wb_q_data {
+	/* format info */
+	struct v4l2_format	format;
+	/* crop/compose rectangle */
+	struct v4l2_rect	c_rect;
+	/* format info */
+	struct wb_fmt		*fmt;
+};
+
+enum {
+	Q_DATA_SRC = 0,
+	Q_DATA_DST = 1,
+};
+
+/* find our format description corresponding to the passed v4l2_format */
+struct wb_fmt *find_format(struct v4l2_format *f);
+
+struct wb_dev {
+	struct v4l2_device	v4l2_dev;
+	struct drm_device	*drm_dev;
+
+	atomic_t		irq_enabled;
+
+	/* v4l2_ioctl mutex */
+	struct mutex		lock;
+
+	enum omap_wb_mode	mode;
+	struct wbcap_dev	*cap;
+	struct wbm2m_dev	*m2m;
+};
+
+/*
+ * there is one wbcap_dev structure in the driver.
+ */
+struct wbcap_dev {
+	struct v4l2_device	v4l2_dev;
+	struct video_device	vdev;
+	struct v4l2_fh		fh;
+	struct wb_dev		*dev;
+	struct v4l2_ctrl_handler hdl;
+
+	/* dst queue data */
+	struct wb_q_data	q_data[2];
+
+	unsigned int		input;
+
+	struct vb2_queue	queue;
+	struct vb2_alloc_ctx	*alloc_ctx;
+
+	spinlock_t		qlock;
+	struct list_head	buf_list;
+
+	/* Current  v4l2_buffer */
+	struct wb_buffer	*cur_frm;
+	/* Next v4l2_buffer */
+	struct wb_buffer	*next_frm;
+
+	unsigned int		field;
+	unsigned int		sequence;
+
+	bool			stopping;
+	wait_queue_head_t	event;
+
+	enum wb_state state;
+
+	/* timer used to wait for wb go bit to be cleared */
+	struct hrtimer		wbgo_timer;
+};
+
+/*
+ * there is one wbm2m_dev structure in the driver.
+ */
+struct wbm2m_dev {
+	struct v4l2_device	v4l2_dev;
+	struct video_device	vfd;
+	struct v4l2_m2m_dev	*m2m_dev;
+	struct wb_dev		*dev;
+	struct drm_plane	*plane;
+
+	/* v4l2 buffers lock */
+	spinlock_t		lock;
+
+	struct vb2_alloc_ctx	*alloc_ctx;
+};
+
+/*
+ * There is one wbm2m_ctx structure for each m2m context.
+ */
+struct wbm2m_ctx {
+	struct v4l2_fh		fh;
+	struct wbm2m_dev	*dev;
+	struct v4l2_ctrl_handler hdl;
+
+	/* current frame seq */
+	unsigned int		sequence;
+	/* abort after next irq */
+	unsigned int		aborting;
+
+	/* src & dst queue data */
+	struct wb_q_data	q_data[2];
+};
+
+static inline struct wb_buffer *to_wb_buffer(struct vb2_buffer *vb2)
+{
+	return container_of(vb2, struct wb_buffer, vb.vb2_buf);
+}
+
+int omap_wb_fourcc_v4l2_to_drm(u32 fourcc);
+
+void wbm2m_irq(struct wbm2m_dev *dev, uint32_t irqstatus);
+int wbm2m_init(struct wb_dev *dev);
+void wbm2m_cleanup(struct wb_dev *dev);
+
+void wbcap_irq(struct wbcap_dev *dev, u32 irqstatus);
+int wbcap_init(struct wb_dev *dev);
+void wbcap_cleanup(struct wb_dev *dev);
+
+#endif /* __OMAP_WB_H__ */
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/omap_wb_m2m.c b/drivers/gpu/drm/omapdrm/omap_wb_m2m.c
--- a/drivers/gpu/drm/omapdrm/omap_wb_m2m.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/omap_wb_m2m.c	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,1199 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2015-2018 Texas Instruments Incorporated -  http://www.ti.com/
+ * Benoit Parrot <bparrot@ti.com>
+ *
+ * Based on the virtual v4l2-mem2mem example device
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <media/v4l2-mem2mem.h>
+
+#include "omap_wb.h"
+
+MODULE_DESCRIPTION("TI OMAP WB M2M driver");
+MODULE_AUTHOR("Benoit Parrot <bparrot@ti.com>");
+MODULE_LICENSE("GPL v2");
+
+/*
+ * M2M devices get 2 queues.
+ * Return the queue given the type.
+ */
+static struct wb_q_data *get_q_data(struct wbm2m_ctx *ctx,
+				    enum v4l2_buf_type type)
+{
+	switch (type) {
+	case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
+	case V4L2_BUF_TYPE_VIDEO_OUTPUT:
+		return &ctx->q_data[Q_DATA_SRC];
+	case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
+	case V4L2_BUF_TYPE_VIDEO_CAPTURE:
+		return &ctx->q_data[Q_DATA_DST];
+	default:
+		return NULL;
+	}
+	return NULL;
+}
+
+static bool wbm2m_convert(struct wbm2m_dev *dev, enum omap_plane_id src_plane,
+			  const struct omap_overlay_info *src_info,
+			  const struct omap_dss_writeback_info *wb_info)
+{
+	struct omap_drm_private *priv = dev->dev->drm_dev->dev_private;
+	enum dss_writeback_channel wb_channel;
+	struct videomode t = { 0 };
+	int r;
+
+	t.hactive = src_info->out_width;
+	t.vactive = src_info->out_height;
+
+	/* configure input */
+
+	r = priv->dispc_ops->ovl_setup(priv->dispc, src_plane, src_info, &t,
+				       true, OMAP_DSS_CHANNEL_WB);
+	if (r)
+		return false;
+
+	priv->dispc_ops->ovl_enable(priv->dispc, src_plane, true);
+
+	/* configure output */
+
+	switch (src_plane) {
+	case OMAP_DSS_GFX:
+		wb_channel = DSS_WB_OVL0; break;
+	case OMAP_DSS_VIDEO1:
+		wb_channel = DSS_WB_OVL1; break;
+	case OMAP_DSS_VIDEO2:
+		wb_channel = DSS_WB_OVL2; break;
+	case OMAP_DSS_VIDEO3:
+		wb_channel = DSS_WB_OVL3; break;
+	default:
+		/*
+		 * if src_plane is not valid it should have been flagged
+		 * during the ovl_setup() step above. Let's set a default
+		 * at any rate.
+		 */
+		wb_channel = DSS_WB_OVL3; break;
+	}
+
+	r = priv->dispc_ops->wb_setup(priv->dispc, wb_info, true, &t,
+				      wb_channel);
+	if (r) {
+		priv->dispc_ops->ovl_enable(priv->dispc, src_plane, false);
+		return false;
+	}
+
+	priv->dispc_ops->ovl_enable(priv->dispc, OMAP_DSS_WB, true);
+
+	return true;
+}
+
+/*
+ * mem2mem callbacks
+ */
+
+/**
+ * job_ready() - check whether an instance is ready to be scheduled to run
+ */
+static int job_ready(void *priv)
+{
+	struct wbm2m_ctx *ctx = priv;
+
+	/*
+	 * This check is needed as this might be called directly from driver
+	 * When called by m2m framework, this will always satisy, but when
+	 * called from wbm2m_irq, this might fail.
+	 * (src stream with zero buffers)
+	 */
+	if (v4l2_m2m_num_src_bufs_ready(ctx->fh.m2m_ctx) <= 0 ||
+	    v4l2_m2m_num_dst_bufs_ready(ctx->fh.m2m_ctx) <= 0)
+		return 0;
+
+	return 1;
+}
+
+static void job_abort(void *priv)
+{
+	struct wbm2m_ctx *ctx = priv;
+
+	/* Will cancel the transaction in the next interrupt handler */
+	ctx->aborting = 1;
+
+	log_dbg(ctx->dev, "Aborting transaction\n");
+	v4l2_m2m_job_finish(ctx->dev->m2m_dev, ctx->fh.m2m_ctx);
+}
+
+/* device_run() - prepares and starts the device
+ *
+ * This function is only called when both the source and destination
+ * buffers are in place.
+ */
+static void device_run(void *priv)
+{
+	struct wbm2m_ctx *ctx = priv;
+	struct wbm2m_dev *dev = ctx->dev;
+	struct wb_q_data *d_q_data = &ctx->q_data[Q_DATA_DST];
+	struct wb_q_data *s_q_data = &ctx->q_data[Q_DATA_SRC];
+	struct vb2_buffer *s_vb, *d_vb;
+	struct vb2_v4l2_buffer *src_vb, *dst_vb;
+	dma_addr_t src_dma_addr[2] = {0, 0};
+	dma_addr_t dst_dma_addr[2] = {0, 0};
+	struct omap_overlay_info src_info = { 0 };
+	struct omap_dss_writeback_info wb_info = { 0 };
+	struct v4l2_pix_format_mplane *spix, *dpix;
+	struct v4l2_rect *srect, *drect;
+	u32 stride, depth;
+	bool ok;
+
+	src_vb = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx);
+	if (!src_vb) {
+		log_err(dev, "getting next source buffer failed\n");
+		return;
+	}
+
+	s_vb = &src_vb->vb2_buf;
+	if (!s_vb) {
+		log_err(dev, "getting next src vb2_buf addr failed\n");
+		return;
+	}
+
+	dst_vb = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx);
+	if (!dst_vb) {
+		log_err(dev, "getting next dest buffer failed\n");
+		return;
+	}
+
+	d_vb = &dst_vb->vb2_buf;
+	if (!d_vb) {
+		log_err(dev, "getting next dest vb2_buf addr failed\n");
+		return;
+	}
+
+	srect = &s_q_data->c_rect;
+	spix = &s_q_data->format.fmt.pix_mp;
+	src_dma_addr[0] = vb2_dma_contig_plane_dma_addr(s_vb, 0);
+	if (spix->num_planes == 2)
+		src_dma_addr[1] = vb2_dma_contig_plane_dma_addr(s_vb, 1);
+	else if (spix->pixelformat == V4L2_PIX_FMT_NV12)
+		src_dma_addr[1] = src_dma_addr[0] +
+			(spix->plane_fmt[0].bytesperline * spix->height);
+	if (!src_dma_addr[0]) {
+		log_err(dev,
+			"acquiring source buffer(%d) dma_addr failed\n",
+			s_vb->index);
+		return;
+	}
+
+	drect = &d_q_data->c_rect;
+	dpix = &d_q_data->format.fmt.pix_mp;
+	dst_dma_addr[0] = vb2_dma_contig_plane_dma_addr(d_vb, 0);
+	if (dpix->num_planes == 2)
+		dst_dma_addr[1] = vb2_dma_contig_plane_dma_addr(d_vb, 1);
+	else if (dpix->pixelformat == V4L2_PIX_FMT_NV12)
+		dst_dma_addr[1] = dst_dma_addr[0] +
+			(dpix->plane_fmt[0].bytesperline * dpix->height);
+	if (!dst_dma_addr[0]) {
+		log_err(dev,
+			"acquiring destination buffer(%d) dma_addr failed\n",
+			d_vb->index);
+		return;
+	}
+
+	/* fill source DSS info */
+	src_info.paddr = (u32)src_dma_addr[0];
+	src_info.p_uv_addr = (u32)src_dma_addr[1];
+
+	/* update addr based on cropping window */
+	stride = spix->plane_fmt[0].bytesperline;
+	depth = s_q_data->fmt->depth[0];
+	src_info.paddr += srect->top * stride + (srect->left * depth / 8);
+
+	if (src_info.p_uv_addr) {
+		u32 top = srect->top;
+
+		if (spix->pixelformat == V4L2_PIX_FMT_NV12 ||
+		    spix->pixelformat == V4L2_PIX_FMT_NV12M) {
+			top >>= 1;
+			depth = 8;
+		}
+		src_info.p_uv_addr += top * stride + (srect->left * depth / 8);
+	}
+
+	src_info.screen_width = spix->plane_fmt[0].bytesperline /
+				(s_q_data->fmt->depth[0] / 8);
+
+	src_info.width = srect->width;
+	src_info.height = srect->height;
+
+	src_info.pos_x = 0;
+	src_info.pos_y = 0;
+	src_info.out_width = srect->width;
+	src_info.out_height = srect->height;
+
+	src_info.fourcc = omap_wb_fourcc_v4l2_to_drm(spix->pixelformat);
+	src_info.global_alpha = 0xff;
+
+	src_info.rotation = DRM_MODE_ROTATE_0;
+	src_info.rotation_type = OMAP_DSS_ROT_NONE;
+
+	log_dbg(dev, "SRC: ctx %pa buf_index %d %dx%d, sw %d\n",
+		&ctx, s_vb->index,
+		src_info.width, src_info.height, src_info.screen_width);
+
+	/* fill WB DSS info */
+	wb_info.paddr = (u32)dst_dma_addr[0];
+	wb_info.p_uv_addr = (u32)dst_dma_addr[1];
+
+	wb_info.buf_width = dpix->plane_fmt[0].bytesperline /
+			    (d_q_data->fmt->depth[0] / 8);
+
+	/* update addr based on compose window */
+	stride = dpix->plane_fmt[0].bytesperline;
+	depth = d_q_data->fmt->depth[0];
+	wb_info.paddr += drect->top * stride + (drect->left * depth / 8);
+
+	if (wb_info.p_uv_addr) {
+		u32 top = drect->top;
+
+		if (dpix->pixelformat == V4L2_PIX_FMT_NV12 ||
+		    dpix->pixelformat == V4L2_PIX_FMT_NV12M) {
+			top >>= 1;
+			depth = 8;
+		}
+		wb_info.p_uv_addr += top * stride + (drect->left * depth / 8);
+	}
+
+	wb_info.width = drect->width;
+	wb_info.height = drect->height;
+	wb_info.fourcc = omap_wb_fourcc_v4l2_to_drm(dpix->pixelformat);
+	wb_info.pre_mult_alpha = 1;
+
+	wb_info.rotation = DRM_MODE_ROTATE_0;
+	wb_info.rotation_type = OMAP_DSS_ROT_NONE;
+
+	log_dbg(dev, "DST: ctx %pa buf_index %d %dx%d, sw %d\n",
+		&ctx, d_vb->index,
+		wb_info.width, wb_info.height, wb_info.buf_width);
+
+	ok = wbm2m_convert(dev, omap_plane_id_wb(dev->plane), &src_info,
+			   &wb_info);
+	if (!ok) {
+		log_err(dev,
+			"Conversion setup failed, check source and destination parameters\n"
+			);
+		log_err(dev, "\tSRC: %dx%d, fmt: %4.4s sw %d\n",
+			src_info.width, src_info.height,
+			(char *)&spix->pixelformat,
+			src_info.screen_width);
+		log_err(dev, "\tDST: %dx%d, fmt: %4.4s sw %d\n",
+			wb_info.width, wb_info.height,
+			(char *)&dpix->pixelformat,
+			wb_info.buf_width);
+		return;
+	}
+}
+
+void wbm2m_irq(struct wbm2m_dev *wbm2m, u32 irqstatus)
+{
+	struct wbm2m_ctx *ctx;
+	struct vb2_v4l2_buffer *s_vb, *d_vb;
+	unsigned long flags;
+
+	if (irqstatus & DISPC_IRQ_WBBUFFEROVERFLOW)
+		log_err(wbm2m, "WB: UNDERFLOW\n");
+
+	if (irqstatus & DISPC_IRQ_WBUNCOMPLETEERROR)
+		log_err(wbm2m, "WB: DISPC_IRQ_WBUNCOMPLETEERROR\n");
+
+	/* If DISPC_IRQ_FRAMEDONEWB is not set then we are done */
+	if (!(irqstatus & DISPC_IRQ_FRAMEDONEWB))
+		goto handled;
+
+	log_dbg(wbm2m, "WB: FRAMEDONE\n");
+
+	ctx = v4l2_m2m_get_curr_priv(wbm2m->m2m_dev);
+	if (!ctx) {
+		log_err(wbm2m, "instance released before end of transaction\n");
+		goto handled;
+	}
+
+	log_dbg(ctx->dev, "ctx %pa\n", &ctx);
+
+	s_vb = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx);
+	d_vb = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
+	if (!s_vb || !d_vb) {
+		log_err(wbm2m, "source or dest vb pointer is NULL!!");
+		goto handled;
+	}
+
+	d_vb->flags = s_vb->flags;
+
+	d_vb->vb2_buf.timestamp = s_vb->vb2_buf.timestamp;
+	if (s_vb->flags & V4L2_BUF_FLAG_TIMECODE)
+		d_vb->timecode = s_vb->timecode;
+
+	d_vb->sequence = ctx->sequence;
+	s_vb->sequence = ctx->sequence;
+	log_dbg(wbm2m, "ctx %pa sequence %d\n",
+		&ctx, ctx->sequence);
+
+	d_vb->field = V4L2_FIELD_NONE;
+	ctx->sequence++;
+
+	spin_lock_irqsave(&wbm2m->lock, flags);
+
+	v4l2_m2m_buf_done(s_vb, VB2_BUF_STATE_DONE);
+	v4l2_m2m_buf_done(d_vb, VB2_BUF_STATE_DONE);
+
+	spin_unlock_irqrestore(&wbm2m->lock, flags);
+
+	v4l2_m2m_job_finish(wbm2m->m2m_dev, ctx->fh.m2m_ctx);
+handled:
+	return;
+}
+
+/*
+ * video ioctls
+ */
+static int wbm2m_querycap(struct file *file, void *priv,
+			  struct v4l2_capability *cap)
+{
+	struct wbm2m_ctx *ctx = file->private_data;
+
+	strncpy(cap->driver, WBM2M_MODULE_NAME, sizeof(cap->driver) - 1);
+	strncpy(cap->card, WBM2M_MODULE_NAME, sizeof(cap->card) - 1);
+	snprintf(cap->bus_info, sizeof(cap->bus_info), "platform:%s",
+		 ctx->dev->v4l2_dev.name);
+	cap->device_caps  = V4L2_CAP_VIDEO_M2M_MPLANE | V4L2_CAP_STREAMING;
+	cap->capabilities = cap->device_caps | V4L2_CAP_DEVICE_CAPS;
+	return 0;
+}
+
+static int wbm2m_enum_fmt(struct file *file, void *priv,
+			  struct v4l2_fmtdesc *f)
+{
+	if (f->index >= num_wb_formats)
+		return -EINVAL;
+
+	f->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+	f->pixelformat = wb_formats[f->index].fourcc;
+	return 0;
+}
+
+static int wbm2m_g_fmt(struct file *file, void *priv, struct v4l2_format *f)
+{
+	struct wbm2m_ctx *ctx = file->private_data;
+	struct vb2_queue *vq;
+	struct wb_q_data *q_data;
+	struct v4l2_pix_format_mplane *pix = &f->fmt.pix_mp;
+
+	vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx, f->type);
+	if (!vq)
+		return -EINVAL;
+
+	q_data = get_q_data(ctx, f->type);
+	if (!q_data)
+		return -EINVAL;
+
+	*f = q_data->format;
+
+	if (!V4L2_TYPE_IS_OUTPUT(f->type)) {
+		struct wb_q_data *s_q_data;
+
+		/* get colorspace from the source queue */
+		s_q_data = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE);
+
+		f->fmt.pix_mp.colorspace =
+			s_q_data->format.fmt.pix_mp.colorspace;
+	}
+
+	log_dbg(ctx->dev, "ctx %pa type %d, %dx%d, fmt: %4.4s bpl_y %d",
+		&ctx, f->type, pix->width, pix->height,
+		(char *)&pix->pixelformat,
+		pix->plane_fmt[LUMA_PLANE].bytesperline);
+	if (pix->num_planes == 2)
+		log_dbg(ctx->dev, " bpl_uv %d\n",
+			pix->plane_fmt[CHROMA_PLANE].bytesperline);
+
+	return 0;
+}
+
+static int wbm2m_try_fmt(struct file *file, void *priv, struct v4l2_format *f)
+{
+	struct wbm2m_ctx *ctx = file->private_data;
+	struct wb_fmt *fmt = find_format(f);
+	struct v4l2_pix_format_mplane *pix = &f->fmt.pix_mp;
+	struct v4l2_plane_pix_format *plane_fmt;
+	unsigned int w_align;
+	int i, depth, depth_bytes;
+
+	if (!fmt) {
+		log_dbg(ctx->dev, "Fourcc format (0x%08x) invalid.\n",
+			pix->pixelformat);
+		fmt = &wb_formats[1];
+	}
+
+	/* we only allow V4L2_FIELD_NONE */
+	if (pix->field != V4L2_FIELD_NONE)
+		pix->field = V4L2_FIELD_NONE;
+
+	depth = fmt->depth[LUMA_PLANE];
+
+	/*
+	 * The line stride needs to be even is even.
+	 * Special case is with YUV422 interleaved format an even number
+	 * of pixels is required also.
+	 */
+	depth_bytes = depth >> 3;
+
+	w_align = 0;
+	if ((depth_bytes == 3) || (depth_bytes == 1))
+		w_align = 1;
+	else if ((depth_bytes == 2) &&
+		 (fmt->fourcc == V4L2_PIX_FMT_YUYV ||
+		  fmt->fourcc == V4L2_PIX_FMT_UYVY))
+		w_align = 1;
+
+	v4l_bound_align_image(&pix->width, MIN_W, MAX_W, w_align,
+			      &pix->height, MIN_H, MAX_H, H_ALIGN,
+			      S_ALIGN);
+	pix->num_planes = fmt->coplanar ? 2 : 1;
+	pix->pixelformat = fmt->fourcc;
+
+	/* Probably need something better here */
+	if (!pix->colorspace) {
+		if (fmt->fourcc == V4L2_PIX_FMT_RGB24 ||
+		    fmt->fourcc == V4L2_PIX_FMT_BGR24 ||
+		    fmt->fourcc == V4L2_PIX_FMT_RGB32 ||
+		    fmt->fourcc == V4L2_PIX_FMT_BGR32) {
+			pix->colorspace = V4L2_COLORSPACE_SRGB;
+		} else {
+			if (pix->height > 1280)	/* HD */
+				pix->colorspace = V4L2_COLORSPACE_REC709;
+			else			/* SD */
+				pix->colorspace = V4L2_COLORSPACE_SMPTE170M;
+		}
+	}
+
+	memset(pix->reserved, 0, sizeof(pix->reserved));
+	for (i = 0; i < pix->num_planes; i++) {
+		plane_fmt = &pix->plane_fmt[i];
+		depth = fmt->depth[i];
+
+		if (i == LUMA_PLANE)
+			plane_fmt->bytesperline = pix->width * depth / 8;
+		else
+			plane_fmt->bytesperline = pix->width;
+
+		plane_fmt->sizeimage = (pix->height * pix->width *
+					depth) / 8;
+
+		if (fmt->fourcc == V4L2_PIX_FMT_NV12) {
+			/*
+			 * Since we are using a single plane buffer
+			 * we need to adjust the reported sizeimage
+			 * to include the colocated UV part.
+			 */
+			plane_fmt->sizeimage += (pix->height / 2 *
+				plane_fmt->bytesperline);
+		}
+
+		memset(plane_fmt->reserved, 0, sizeof(plane_fmt->reserved));
+	}
+
+	return 0;
+}
+
+static int __wbm2m_s_fmt(struct wbm2m_ctx *ctx, struct v4l2_format *f)
+{
+	struct v4l2_pix_format_mplane *pix = &f->fmt.pix_mp;
+	struct wb_q_data *q_data;
+	struct vb2_queue *vq;
+
+	vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx, f->type);
+	if (!vq)
+		return -EINVAL;
+
+	if (vb2_is_busy(vq)) {
+		log_err(ctx->dev, "queue busy\n");
+		return -EBUSY;
+	}
+
+	q_data = get_q_data(ctx, f->type);
+	if (!q_data)
+		return -EINVAL;
+
+	q_data->fmt = find_format(f);
+	q_data->format = *f;
+
+	q_data->c_rect.left	= 0;
+	q_data->c_rect.top	= 0;
+	q_data->c_rect.width	= pix->width;
+	q_data->c_rect.height	= pix->height;
+
+	log_dbg(ctx->dev, "ctx %pa type %d, %dx%d, fmt: %4.4s bpl_y %d",
+		&ctx, f->type, pix->width, pix->height,
+		(char *)&pix->pixelformat,
+		pix->plane_fmt[LUMA_PLANE].bytesperline);
+	if (pix->num_planes == 2)
+		log_dbg(ctx->dev, " bpl_uv %d\n",
+			pix->plane_fmt[CHROMA_PLANE].bytesperline);
+
+	return 0;
+}
+
+static int wbm2m_s_fmt(struct file *file, void *priv, struct v4l2_format *f)
+{
+	int ret;
+	struct wbm2m_ctx *ctx = file->private_data;
+
+	ret = wbm2m_try_fmt(file, priv, f);
+	if (ret)
+		return ret;
+
+	ret = __wbm2m_s_fmt(ctx, f);
+	if (ret)
+		return ret;
+
+	ctx->sequence = 0;
+
+	return 0;
+}
+
+static int __wbm2m_try_selection(struct wbm2m_ctx *ctx,
+				 struct v4l2_selection *s)
+{
+	struct wb_q_data *q_data;
+	struct v4l2_pix_format_mplane *pix;
+	unsigned int w_align;
+	int depth_bytes;
+
+	if ((s->type != V4L2_BUF_TYPE_VIDEO_CAPTURE) &&
+	    (s->type != V4L2_BUF_TYPE_VIDEO_OUTPUT))
+		return -EINVAL;
+
+	q_data = get_q_data(ctx, s->type);
+	if (!q_data)
+		return -EINVAL;
+
+	pix = &q_data->format.fmt.pix_mp;
+
+	switch (s->target) {
+	case V4L2_SEL_TGT_COMPOSE:
+		/*
+		 * COMPOSE target is only valid for capture buffer type, return
+		 * error for output buffer type
+		 */
+		if (s->type == V4L2_BUF_TYPE_VIDEO_OUTPUT)
+			return -EINVAL;
+		break;
+	case V4L2_SEL_TGT_CROP:
+		/*
+		 * CROP target is only valid for output buffer type, return
+		 * error for capture buffer type
+		 */
+		if (s->type == V4L2_BUF_TYPE_VIDEO_CAPTURE)
+			return -EINVAL;
+		break;
+	/*
+	 * bound and default crop/compose targets are invalid targets to
+	 * try/set
+	 */
+	default:
+		return -EINVAL;
+	}
+
+	if (s->r.top < 0 || s->r.left < 0) {
+		log_err(ctx->dev, "negative values for top and left\n");
+		s->r.top = 0;
+		s->r.left = 0;
+	}
+
+	depth_bytes = q_data->fmt->depth[LUMA_PLANE] >> 3;
+
+	w_align = 0;
+	if ((depth_bytes == 3) || (depth_bytes == 1))
+		w_align = 1;
+	else if ((depth_bytes == 2) &&
+		 (pix->pixelformat == V4L2_PIX_FMT_YUYV ||
+		  pix->pixelformat == V4L2_PIX_FMT_UYVY))
+		w_align = 1;
+
+	v4l_bound_align_image(&s->r.width, MIN_W, pix->width, w_align,
+			      &s->r.height, MIN_H, pix->height,
+			      H_ALIGN, S_ALIGN);
+
+	/* adjust left/top if cropping rectangle is out of bounds */
+	if (s->r.left + s->r.width > pix->width)
+		s->r.left = pix->width - s->r.width;
+	if (s->r.top + s->r.height > pix->height)
+		s->r.top = pix->height - s->r.height;
+
+	return 0;
+}
+
+static int wbm2m_g_selection(struct file *file, void *fh,
+			     struct v4l2_selection *s)
+{
+	struct wbm2m_ctx *ctx = file->private_data;
+	struct wb_q_data *q_data;
+	struct v4l2_pix_format_mplane *pix;
+	bool use_c_rect = false;
+
+	if ((s->type != V4L2_BUF_TYPE_VIDEO_CAPTURE) &&
+	    (s->type != V4L2_BUF_TYPE_VIDEO_OUTPUT))
+		return -EINVAL;
+
+	q_data = get_q_data(ctx, s->type);
+	if (!q_data)
+		return -EINVAL;
+
+	pix = &q_data->format.fmt.pix_mp;
+
+	switch (s->target) {
+	case V4L2_SEL_TGT_COMPOSE_DEFAULT:
+	case V4L2_SEL_TGT_COMPOSE_BOUNDS:
+		if (s->type == V4L2_BUF_TYPE_VIDEO_OUTPUT)
+			return -EINVAL;
+		break;
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+		if (s->type == V4L2_BUF_TYPE_VIDEO_CAPTURE)
+			return -EINVAL;
+		break;
+	case V4L2_SEL_TGT_COMPOSE:
+		if (s->type == V4L2_BUF_TYPE_VIDEO_OUTPUT)
+			return -EINVAL;
+		use_c_rect = true;
+		break;
+	case V4L2_SEL_TGT_CROP:
+		if (s->type == V4L2_BUF_TYPE_VIDEO_CAPTURE)
+			return -EINVAL;
+		use_c_rect = true;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (use_c_rect) {
+		/*
+		 * for CROP/COMPOSE target type, return c_rect params from the
+		 * respective buffer type
+		 */
+		s->r = q_data->c_rect;
+	} else {
+		/*
+		 * for DEFAULT/BOUNDS target type, return width and height from
+		 * S_FMT of the respective buffer type
+		 */
+		s->r.left = 0;
+		s->r.top = 0;
+		s->r.width = pix->width;
+		s->r.height = pix->height;
+	}
+
+	return 0;
+}
+
+static int wbm2m_s_selection(struct file *file, void *fh,
+			     struct v4l2_selection *s)
+{
+	struct wbm2m_ctx *ctx = file->private_data;
+	struct wb_q_data *q_data;
+	struct v4l2_selection sel = *s;
+	int ret;
+
+	ret = __wbm2m_try_selection(ctx, &sel);
+	if (ret)
+		return ret;
+
+	q_data = get_q_data(ctx, sel.type);
+	if (!q_data)
+		return -EINVAL;
+
+	if ((q_data->c_rect.left == sel.r.left) &&
+	    (q_data->c_rect.top == sel.r.top) &&
+	    (q_data->c_rect.width == sel.r.width) &&
+	    (q_data->c_rect.height == sel.r.height)) {
+		log_dbg(ctx->dev,
+			"type: %d, requested crop/compose values are already set\n",
+			sel.type);
+		return 0;
+	}
+
+	q_data->c_rect = sel.r;
+
+	ctx->sequence = 0;
+
+	return 0;
+}
+
+static const struct v4l2_ioctl_ops wbm2m_ioctl_ops = {
+	.vidioc_querycap		= wbm2m_querycap,
+
+	.vidioc_enum_fmt_vid_cap_mplane	= wbm2m_enum_fmt,
+	.vidioc_g_fmt_vid_cap_mplane	= wbm2m_g_fmt,
+	.vidioc_try_fmt_vid_cap_mplane	= wbm2m_try_fmt,
+	.vidioc_s_fmt_vid_cap_mplane	= wbm2m_s_fmt,
+
+	.vidioc_enum_fmt_vid_out_mplane	= wbm2m_enum_fmt,
+	.vidioc_g_fmt_vid_out_mplane	= wbm2m_g_fmt,
+	.vidioc_try_fmt_vid_out_mplane	= wbm2m_try_fmt,
+	.vidioc_s_fmt_vid_out_mplane	= wbm2m_s_fmt,
+
+	.vidioc_g_selection		= wbm2m_g_selection,
+	.vidioc_s_selection		= wbm2m_s_selection,
+
+	.vidioc_reqbufs			= v4l2_m2m_ioctl_reqbufs,
+	.vidioc_create_bufs		= v4l2_m2m_ioctl_create_bufs,
+	.vidioc_prepare_buf		= v4l2_m2m_ioctl_prepare_buf,
+	.vidioc_querybuf		= v4l2_m2m_ioctl_querybuf,
+	.vidioc_qbuf			= v4l2_m2m_ioctl_qbuf,
+	.vidioc_dqbuf			= v4l2_m2m_ioctl_dqbuf,
+	.vidioc_expbuf			= v4l2_m2m_ioctl_expbuf,
+	.vidioc_streamon		= v4l2_m2m_ioctl_streamon,
+	.vidioc_streamoff		= v4l2_m2m_ioctl_streamoff,
+
+	.vidioc_subscribe_event		= v4l2_ctrl_subscribe_event,
+	.vidioc_unsubscribe_event	= v4l2_event_unsubscribe,
+};
+
+/*
+ * Queue operations
+ */
+static int wbm2m_queue_setup(struct vb2_queue *vq,
+			     unsigned int *nbuffers, unsigned int *nplanes,
+			     unsigned int sizes[], struct device *alloc_devs[])
+{
+	int i;
+	struct wbm2m_ctx *ctx = vb2_get_drv_priv(vq);
+	struct wb_q_data *q_data;
+
+	q_data = get_q_data(ctx, vq->type);
+	if (!q_data)
+		return -EINVAL;
+
+	*nplanes = q_data->format.fmt.pix_mp.num_planes;
+
+	for (i = 0; i < *nplanes; i++)
+		sizes[i] = q_data->format.fmt.pix_mp.plane_fmt[i].sizeimage;
+
+	log_dbg(ctx->dev, "get %d buffer(s) of size %d\n", *nbuffers,
+		sizes[LUMA_PLANE]);
+	if (*nplanes == 2)
+		log_dbg(ctx->dev, " and %d\n", sizes[CHROMA_PLANE]);
+
+	return 0;
+}
+
+static int wbm2m_buf_prepare(struct vb2_buffer *vb)
+{
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct wbm2m_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue);
+	struct wb_q_data *q_data;
+	struct v4l2_pix_format_mplane *mp;
+	int i, num_planes;
+
+	log_dbg(ctx->dev, "type: %d\n", vb->vb2_queue->type);
+
+	q_data = get_q_data(ctx, vb->vb2_queue->type);
+	if (!q_data)
+		return -EINVAL;
+
+	if (vb->vb2_queue->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE)
+		vbuf->field = V4L2_FIELD_NONE;
+
+	num_planes = q_data->format.fmt.pix_mp.num_planes;
+
+	for (i = 0; i < num_planes; i++) {
+		mp = &q_data->format.fmt.pix_mp;
+
+		if (vb->vb2_queue->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
+			if (vb2_get_plane_payload(vb, i) <
+			    mp->plane_fmt[i].sizeimage) {
+				log_dbg(ctx->dev,
+					"the payload is too small for plane plane (%lu < %lu)\n",
+					vb2_get_plane_payload(vb, i),
+					(long)mp->plane_fmt[i].sizeimage);
+				return -EINVAL;
+			}
+		} else {
+			if (vb2_plane_size(vb, i) <
+			    mp->plane_fmt[i].sizeimage) {
+				log_dbg(ctx->dev,
+					"data will not fit into plane (%lu < %lu)\n",
+					vb2_plane_size(vb, i),
+					(long)mp->plane_fmt[i].sizeimage);
+				return -EINVAL;
+			}
+			vb2_set_plane_payload(vb, i,
+					      mp->plane_fmt[i].sizeimage);
+		}
+	}
+
+	if (num_planes == 2) {
+		if (vb->planes[0].m.fd ==
+		    vb->planes[1].m.fd) {
+			/*
+			 * So it appears we are in a single memory buffer
+			 * with 2 plane case. Then we need to also set the
+			 * data_offset properly
+			 */
+			vb->planes[1].data_offset =
+				vb2_get_plane_payload(vb, 0);
+		}
+	}
+	return 0;
+}
+
+static void wbm2m_buf_queue(struct vb2_buffer *vb)
+{
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct wbm2m_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue);
+
+	log_dbg(ctx->dev, "queueing buffer: %s index %d\n",
+		V4L2_TYPE_IS_OUTPUT(vb->type) ? "OUTPUT" : "CAPTURE",
+		vb->index);
+
+	v4l2_m2m_buf_queue(ctx->fh.m2m_ctx, vbuf);
+}
+
+static int wbm2m_start_streaming(struct vb2_queue *q, unsigned int count)
+{
+	struct wbm2m_ctx *ctx = vb2_get_drv_priv(q);
+	struct omap_drm_private *priv = ctx->dev->dev->drm_dev->dev_private;
+
+	log_dbg(ctx->dev, "ctx %pa queue: %s\n", &ctx,
+		V4L2_TYPE_IS_OUTPUT(q->type) ? "OUTPUT" : "CAPTURE");
+
+	ctx->sequence = 0;
+
+	priv->dispc_ops->runtime_get(priv->dispc);
+	atomic_inc(&ctx->dev->dev->irq_enabled);
+
+	return 0;
+}
+
+static void wbm2m_stop_streaming(struct vb2_queue *q)
+{
+	struct wbm2m_ctx *ctx = vb2_get_drv_priv(q);
+	struct omap_drm_private *priv = ctx->dev->dev->drm_dev->dev_private;
+	struct vb2_v4l2_buffer *vb;
+	unsigned long flags;
+
+	log_dbg(ctx->dev, "ctx %pa queue: %s\n", &ctx,
+		V4L2_TYPE_IS_OUTPUT(q->type) ? "OUTPUT" : "CAPTURE");
+
+	atomic_dec(&ctx->dev->dev->irq_enabled);
+
+	for (;;) {
+		if (V4L2_TYPE_IS_OUTPUT(q->type))
+			vb = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx);
+		else
+			vb = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
+		if (!vb)
+			break;
+		log_dbg(ctx->dev, "returning from queue: buffer index %d\n",
+			vb->vb2_buf.index);
+		spin_lock_irqsave(&ctx->dev->lock, flags);
+		v4l2_m2m_buf_done(vb, VB2_BUF_STATE_ERROR);
+		spin_unlock_irqrestore(&ctx->dev->lock, flags);
+	}
+
+	/*
+	 * Cleanup the in-transit vb2 buffers that have been
+	 * removed from their respective queue already but for
+	 * which procecessing has not been completed yet.
+	 */
+	if (V4L2_TYPE_IS_OUTPUT(q->type)) {
+		/*
+		 * DRA7xx errata i829 (Reusing Pipe Connected To Writeback
+		 * Pipeline On The Fly To An Active Panel)
+		 */
+		priv->dispc_ops->ovl_enable(priv->dispc,
+					    omap_plane_id_wb(ctx->dev->plane),
+					    false);
+		priv->dispc_ops->ovl_enable(priv->dispc, OMAP_DSS_WB, true);
+		priv->dispc_ops->ovl_enable(priv->dispc, OMAP_DSS_WB, false);
+	}
+
+	priv->dispc_ops->runtime_put(priv->dispc);
+}
+
+static struct vb2_ops wbm2m_qops = {
+	.queue_setup	 = wbm2m_queue_setup,
+	.buf_prepare	 = wbm2m_buf_prepare,
+	.buf_queue	 = wbm2m_buf_queue,
+	.wait_prepare	 = vb2_ops_wait_prepare,
+	.wait_finish	 = vb2_ops_wait_finish,
+	.start_streaming = wbm2m_start_streaming,
+	.stop_streaming  = wbm2m_stop_streaming,
+};
+
+static int queue_init(void *priv, struct vb2_queue *src_vq,
+		      struct vb2_queue *dst_vq)
+{
+	struct wbm2m_ctx *ctx = priv;
+	struct wbm2m_dev *dev = ctx->dev;
+	int ret;
+
+	memset(src_vq, 0, sizeof(*src_vq));
+	src_vq->type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
+	src_vq->io_modes = VB2_MMAP | VB2_DMABUF;
+	src_vq->drv_priv = ctx;
+	src_vq->buf_struct_size = sizeof(struct v4l2_m2m_buffer);
+	src_vq->ops = &wbm2m_qops;
+	src_vq->mem_ops = &vb2_dma_contig_memops;
+	src_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY;
+	src_vq->lock = &dev->dev->lock;
+	src_vq->min_buffers_needed = 1;
+	src_vq->dev = dev->v4l2_dev.dev;
+
+	ret = vb2_queue_init(src_vq);
+	if (ret)
+		return ret;
+
+	memset(dst_vq, 0, sizeof(*dst_vq));
+	dst_vq->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+	dst_vq->io_modes = VB2_MMAP | VB2_DMABUF;
+	dst_vq->drv_priv = ctx;
+	dst_vq->buf_struct_size = sizeof(struct v4l2_m2m_buffer);
+	dst_vq->ops = &wbm2m_qops;
+	dst_vq->mem_ops = &vb2_dma_contig_memops;
+	dst_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY;
+	dst_vq->lock = &dev->dev->lock;
+	dst_vq->min_buffers_needed = 1;
+	dst_vq->dev = dev->v4l2_dev.dev;
+
+	return vb2_queue_init(dst_vq);
+}
+
+/*
+ * File operations
+ */
+static int wbm2m_open(struct file *file)
+{
+	struct wbm2m_dev *dev = video_drvdata(file);
+	struct wb_q_data *s_q_data;
+	struct wbm2m_ctx *ctx;
+	struct v4l2_pix_format_mplane *pix;
+	int ret;
+
+	log_dbg(dev, "enter\n");
+
+	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return -ENOMEM;
+
+	ctx->dev = dev;
+
+	if (mutex_lock_interruptible(&dev->dev->lock)) {
+		ret = -ERESTARTSYS;
+		goto free_ctx;
+	}
+
+	if ((dev->dev->mode != OMAP_WB_NOT_CONFIGURED) &&
+	    (dev->dev->mode != OMAP_WB_MEM2MEM_OVL)) {
+		/* WB is already open for other modes */
+		ret = -EBUSY;
+		goto unlock;
+	}
+
+	v4l2_fh_init(&ctx->fh, video_devdata(file));
+	file->private_data = ctx;
+
+	s_q_data = &ctx->q_data[Q_DATA_SRC];
+	s_q_data->fmt = &wb_formats[1];
+	pix = &s_q_data->format.fmt.pix_mp;
+	pix->pixelformat = s_q_data->fmt->fourcc;
+	s_q_data->format.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
+	pix->width = 1920;
+	pix->height = 1080;
+	pix->plane_fmt[LUMA_PLANE].bytesperline = (pix->width *
+			s_q_data->fmt->depth[LUMA_PLANE]) >> 3;
+	pix->plane_fmt[LUMA_PLANE].sizeimage =
+			pix->plane_fmt[LUMA_PLANE].bytesperline *
+			pix->height;
+	pix->num_planes = s_q_data->fmt->coplanar ? 2 : 1;
+	pix->colorspace = V4L2_COLORSPACE_REC709;
+	pix->field = V4L2_FIELD_NONE;
+	s_q_data->c_rect.left = 0;
+	s_q_data->c_rect.top = 0;
+	s_q_data->c_rect.width = pix->width;
+	s_q_data->c_rect.height = pix->height;
+
+	ctx->q_data[Q_DATA_DST] = *s_q_data;
+	ctx->q_data[Q_DATA_DST].format.type =
+			V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+
+	ctx->sequence = 0;
+
+	ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(dev->m2m_dev, ctx, &queue_init);
+
+	if (IS_ERR(ctx->fh.m2m_ctx)) {
+		ret = PTR_ERR(ctx->fh.m2m_ctx);
+		goto exit_fh;
+	}
+
+	v4l2_fh_add(&ctx->fh);
+
+	if (v4l2_fh_is_singular_file(file)) {
+		log_dbg(dev, "first instance created\n");
+
+		drm_modeset_lock_all(dev->dev->drm_dev);
+		dev->plane = omap_plane_reserve_wb(dev->dev->drm_dev);
+		drm_modeset_unlock_all(dev->dev->drm_dev);
+
+		if (!dev->plane) {
+			log_dbg(dev, "Could not reserve plane!\n");
+			ret = -EBUSY;
+			goto free_fh;
+		}
+
+		dev->dev->mode = OMAP_WB_MEM2MEM_OVL;
+	}
+
+	log_dbg(dev, "created instance %pa, m2m_ctx: %pa\n",
+		&ctx, &ctx->fh.m2m_ctx);
+
+	mutex_unlock(&dev->dev->lock);
+
+	return 0;
+
+free_fh:
+	v4l2_fh_del(&ctx->fh);
+	v4l2_m2m_ctx_release(ctx->fh.m2m_ctx);
+exit_fh:
+	v4l2_fh_exit(&ctx->fh);
+unlock:
+	mutex_unlock(&dev->dev->lock);
+free_ctx:
+	kfree(ctx);
+	return ret;
+}
+
+static int wbm2m_release(struct file *file)
+{
+	struct wbm2m_dev *dev = video_drvdata(file);
+	struct wbm2m_ctx *ctx = file->private_data;
+	bool fh_singular;
+
+	log_dbg(dev, "releasing instance %pa\n", &ctx);
+
+	mutex_lock(&dev->dev->lock);
+
+	/* Save the singular status before we call the clean-up helper */
+	fh_singular = v4l2_fh_is_singular_file(file);
+
+	v4l2_fh_del(&ctx->fh);
+	v4l2_fh_exit(&ctx->fh);
+	v4l2_m2m_ctx_release(ctx->fh.m2m_ctx);
+
+	kfree(ctx);
+
+	if (fh_singular) {
+		log_dbg(dev, "last instance released\n");
+
+		drm_modeset_lock_all(dev->dev->drm_dev);
+		omap_plane_release_wb(dev->plane);
+		drm_modeset_unlock_all(dev->dev->drm_dev);
+		dev->dev->mode = OMAP_WB_NOT_CONFIGURED;
+	}
+
+	mutex_unlock(&dev->dev->lock);
+
+	return 0;
+}
+
+static const struct v4l2_file_operations wbm2m_fops = {
+	.owner		= THIS_MODULE,
+	.open		= wbm2m_open,
+	.release	= wbm2m_release,
+	.poll		= v4l2_m2m_fop_poll,
+	.unlocked_ioctl	= video_ioctl2,
+	.mmap		= v4l2_m2m_fop_mmap,
+};
+
+static struct video_device wbm2m_videodev = {
+	.name		= WBM2M_MODULE_NAME,
+	.fops		= &wbm2m_fops,
+	.ioctl_ops	= &wbm2m_ioctl_ops,
+	.minor		= -1,
+	.release	= video_device_release_empty,
+	.vfl_dir	= VFL_DIR_M2M,
+};
+
+static struct v4l2_m2m_ops m2m_ops = {
+	.device_run	= device_run,
+	.job_ready	= job_ready,
+	.job_abort	= job_abort,
+};
+
+int wbm2m_init(struct wb_dev *dev)
+{
+	struct wbm2m_dev *wbm2m;
+	struct video_device *vfd;
+	int ret;
+
+	if (!dev)
+		return -ENOMEM;
+
+	/* Allocate a new instance */
+	wbm2m = devm_kzalloc(dev->drm_dev->dev, sizeof(*wbm2m), GFP_KERNEL);
+	if (!wbm2m)
+		return -ENOMEM;
+
+	dev->m2m = wbm2m;
+	wbm2m->dev = dev;
+
+	spin_lock_init(&wbm2m->lock);
+
+	snprintf(wbm2m->v4l2_dev.name, sizeof(wbm2m->v4l2_dev.name),
+		 "%s", WBM2M_MODULE_NAME);
+	ret = v4l2_device_register(dev->drm_dev->dev, &wbm2m->v4l2_dev);
+	if (ret)
+		return ret;
+
+	wbm2m->m2m_dev = v4l2_m2m_init(&m2m_ops);
+	if (IS_ERR(wbm2m->m2m_dev)) {
+		log_err(wbm2m, "Failed to init mem2mem device\n");
+		ret = PTR_ERR(wbm2m->m2m_dev);
+		goto v4l2_dev_unreg;
+	}
+
+	vfd = &wbm2m->vfd;
+	*vfd = wbm2m_videodev;
+	vfd->lock = &dev->lock;
+	vfd->v4l2_dev = &wbm2m->v4l2_dev;
+
+	ret = video_register_device(vfd, VFL_TYPE_GRABBER, 10);
+	if (ret) {
+		log_err(wbm2m, "Failed to register video device\n");
+		goto rel_m2m;
+	}
+
+	video_set_drvdata(vfd, wbm2m);
+	snprintf(vfd->name, sizeof(vfd->name), "%s", wbm2m_videodev.name);
+	log_dbg(wbm2m, "Device registered as %s\n",
+		video_device_node_name(vfd));
+
+	return 0;
+
+rel_m2m:
+	v4l2_m2m_release(wbm2m->m2m_dev);
+v4l2_dev_unreg:
+	v4l2_device_unregister(&wbm2m->v4l2_dev);
+
+	return ret;
+}
+
+void wbm2m_cleanup(struct wb_dev *dev)
+{
+	log_dbg(dev->m2m, "Cleanup WB M2M\n");
+
+	v4l2_m2m_release(dev->m2m->m2m_dev);
+	video_unregister_device(&dev->m2m->vfd);
+	v4l2_device_unregister(&dev->m2m->v4l2_dev);
+}
diff -Naur --no-dereference a/drivers/gpu/drm/omapdrm/tcm-sita.h b/drivers/gpu/drm/omapdrm/tcm-sita.h
--- a/drivers/gpu/drm/omapdrm/tcm-sita.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/omapdrm/tcm-sita.h	1969-12-31 19:00:00.000000000 -0500
@@ -1,93 +0,0 @@
-/*
- * SImple Tiler Allocator (SiTA) private structures.
- *
- * Copyright (C) 2009-2011 Texas Instruments Incorporated - http://www.ti.com/
- * Author: Ravi Ramachandra <r.ramachandra@ti.com>
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- *   notice, this list of conditions and the following disclaimer.
- *
- * * Redistributions in binary form must reproduce the above copyright
- *   notice, this list of conditions and the following disclaimer in the
- *   documentation and/or other materials provided with the distribution.
- *
- * * Neither the name of Texas Instruments Incorporated nor the names of
- *   its contributors may be used to endorse or promote products derived
- *   from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
- * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
- * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
- * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef _TCM_SITA_H
-#define _TCM_SITA_H
-
-#include "tcm.h"
-
-/* length between two coordinates */
-#define LEN(a, b) ((a) > (b) ? (a) - (b) + 1 : (b) - (a) + 1)
-
-enum criteria {
-	CR_MAX_NEIGHS		= 0x01,
-	CR_FIRST_FOUND		= 0x10,
-	CR_BIAS_HORIZONTAL	= 0x20,
-	CR_BIAS_VERTICAL	= 0x40,
-	CR_DIAGONAL_BALANCE	= 0x80
-};
-
-/* nearness to the beginning of the search field from 0 to 1000 */
-struct nearness_factor {
-	s32 x;
-	s32 y;
-};
-
-/*
- * Statistics on immediately neighboring slots.  Edge is the number of
- * border segments that are also border segments of the scan field.  Busy
- * refers to the number of neighbors that are occupied.
- */
-struct neighbor_stats {
-	u16 edge;
-	u16 busy;
-};
-
-/* structure to keep the score of a potential allocation */
-struct score {
-	struct nearness_factor	f;
-	struct neighbor_stats	n;
-	struct tcm_area		a;
-	u16    neighs;		/* number of busy neighbors */
-};
-
-struct sita_pvt {
-	spinlock_t lock;	/* spinlock to protect access */
-	struct tcm_pt div_pt;	/* divider point splitting container */
-	struct tcm_area ***map;	/* pointers to the parent area for each slot */
-};
-
-/* assign coordinates to area */
-static inline
-void assign(struct tcm_area *a, u16 x0, u16 y0, u16 x1, u16 y1)
-{
-	a->p0.x = x0;
-	a->p0.y = y0;
-	a->p1.x = x1;
-	a->p1.y = y1;
-}
-
-#endif
diff -Naur --no-dereference a/drivers/gpu/drm/panel/Kconfig b/drivers/gpu/drm/panel/Kconfig
--- a/drivers/gpu/drm/panel/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/panel/Kconfig	2020-06-19 21:00:31.412227715 -0400
@@ -99,6 +99,12 @@
 	  Say Y here if you want to enable support for Orise Technology
 	  otm8009a 480x800 dsi 2dl panel.
 
+config DRM_PANEL_OSD_OSD101T2587_53TS
+	tristate "OSD OSD101T2587-53TS DSI 1920x1200 video mode panel"
+	depends on OF
+	depends on DRM_MIPI_DSI
+	depends on BACKLIGHT_CLASS_DEVICE
+
 config DRM_PANEL_PANASONIC_VVX10F034N00
 	tristate "Panasonic VVX10F034N00 1920x1200 video mode panel"
 	depends on OF
diff -Naur --no-dereference a/drivers/gpu/drm/panel/Makefile b/drivers/gpu/drm/panel/Makefile
--- a/drivers/gpu/drm/panel/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/panel/Makefile	2020-06-19 21:00:31.412227715 -0400
@@ -8,6 +8,7 @@
 obj-$(CONFIG_DRM_PANEL_JDI_LT070ME05000) += panel-jdi-lt070me05000.o
 obj-$(CONFIG_DRM_PANEL_LG_LG4573) += panel-lg-lg4573.o
 obj-$(CONFIG_DRM_PANEL_ORISETECH_OTM8009A) += panel-orisetech-otm8009a.o
+obj-$(CONFIG_DRM_PANEL_OSD_OSD101T2587_53TS) += panel-osd-osd101t2587-53ts.o
 obj-$(CONFIG_DRM_PANEL_PANASONIC_VVX10F034N00) += panel-panasonic-vvx10f034n00.o
 obj-$(CONFIG_DRM_PANEL_RASPBERRYPI_TOUCHSCREEN) += panel-raspberrypi-touchscreen.o
 obj-$(CONFIG_DRM_PANEL_RAYDIUM_RM68200) += panel-raydium-rm68200.o
diff -Naur --no-dereference a/drivers/gpu/drm/panel/panel-arm-versatile.c b/drivers/gpu/drm/panel/panel-arm-versatile.c
--- a/drivers/gpu/drm/panel/panel-arm-versatile.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/panel/panel-arm-versatile.c	2020-06-19 21:00:31.412227715 -0400
@@ -191,7 +191,7 @@
 			.vrefresh = 390,
 			.flags = DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC,
 		},
-		.bus_flags = DRM_BUS_FLAG_PIXDATA_NEGEDGE,
+		.bus_flags = DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE,
 	},
 	/*
 	 * Sanyo ALR252RGT 240x320 portrait display found on the
@@ -215,7 +215,7 @@
 			.vrefresh = 116,
 			.flags = DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC,
 		},
-		.bus_flags = DRM_BUS_FLAG_PIXDATA_NEGEDGE,
+		.bus_flags = DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE,
 		.ib2 = true,
 	},
 };
diff -Naur --no-dereference a/drivers/gpu/drm/panel/panel-ilitek-ili9322.c b/drivers/gpu/drm/panel/panel-ilitek-ili9322.c
--- a/drivers/gpu/drm/panel/panel-ilitek-ili9322.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/panel/panel-ilitek-ili9322.c	2020-06-19 21:00:31.412227715 -0400
@@ -412,11 +412,11 @@
 	if (ili->conf->dclk_active_high) {
 		reg = ILI9322_POL_DCLK;
 		connector->display_info.bus_flags |=
-			DRM_BUS_FLAG_PIXDATA_POSEDGE;
+			DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE;
 	} else {
 		reg = 0;
 		connector->display_info.bus_flags |=
-			DRM_BUS_FLAG_PIXDATA_NEGEDGE;
+			DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE;
 	}
 	if (ili->conf->de_active_high) {
 		reg |= ILI9322_POL_DE;
diff -Naur --no-dereference a/drivers/gpu/drm/panel/panel-osd-osd101t2587-53ts.c b/drivers/gpu/drm/panel/panel-osd-osd101t2587-53ts.c
--- a/drivers/gpu/drm/panel/panel-osd-osd101t2587-53ts.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/panel/panel-osd-osd101t2587-53ts.c	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,252 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
+ *  Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
+ */
+
+#include <linux/backlight.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/regulator/consumer.h>
+
+#include <drm/drm_device.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_panel.h>
+
+#include <video/mipi_display.h>
+
+struct osd101t2587_panel {
+	struct drm_panel base;
+	struct mipi_dsi_device *dsi;
+
+	struct backlight_device *backlight;
+	struct regulator *supply;
+
+	bool prepared;
+	bool enabled;
+
+	const struct drm_display_mode *default_mode;
+};
+
+static inline struct osd101t2587_panel *to_osd101t2587_panel(struct drm_panel *panel)
+{
+	return container_of(panel, struct osd101t2587_panel, base);
+}
+
+static int osd101t2587_panel_disable(struct drm_panel *panel)
+{
+	struct osd101t2587_panel *osd101t2587 = to_osd101t2587_panel(panel);
+	int ret;
+
+	if (!osd101t2587->enabled)
+		return 0;
+
+	backlight_disable(osd101t2587->backlight);
+
+	ret = mipi_dsi_shutdown_peripheral(osd101t2587->dsi);
+
+	osd101t2587->enabled = false;
+
+	return ret;
+}
+
+static int osd101t2587_panel_unprepare(struct drm_panel *panel)
+{
+	struct osd101t2587_panel *osd101t2587 = to_osd101t2587_panel(panel);
+
+	if (!osd101t2587->prepared)
+		return 0;
+
+	regulator_disable(osd101t2587->supply);
+	osd101t2587->prepared = false;
+
+	return 0;
+}
+
+static int osd101t2587_panel_prepare(struct drm_panel *panel)
+{
+	struct osd101t2587_panel *osd101t2587 = to_osd101t2587_panel(panel);
+	int ret;
+
+	if (osd101t2587->prepared)
+		return 0;
+
+	ret = regulator_enable(osd101t2587->supply);
+	if (!ret)
+		osd101t2587->prepared = true;
+
+	return ret;
+}
+
+static int osd101t2587_panel_enable(struct drm_panel *panel)
+{
+	struct osd101t2587_panel *osd101t2587 = to_osd101t2587_panel(panel);
+	int ret;
+
+	if (osd101t2587->enabled)
+		return 0;
+
+	ret = mipi_dsi_turn_on_peripheral(osd101t2587->dsi);
+	if (ret)
+		return ret;
+
+	backlight_enable(osd101t2587->backlight);
+
+	osd101t2587->enabled = true;
+
+	return ret;
+}
+
+static const struct drm_display_mode default_mode_osd101t2587 = {
+	.clock = 164400,
+	.hdisplay = 1920,
+	.hsync_start = 1920 + 152,
+	.hsync_end = 1920 + 152 + 52,
+	.htotal = 1920 + 152 + 52 + 20,
+	.vdisplay = 1200,
+	.vsync_start = 1200 + 24,
+	.vsync_end = 1200 + 24 + 6,
+	.vtotal = 1200 + 24 + 6 + 48,
+	.vrefresh = 60,
+	.flags = DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC,
+};
+
+static int osd101t2587_panel_get_modes(struct drm_panel *panel)
+{
+	struct osd101t2587_panel *osd101t2587 = to_osd101t2587_panel(panel);
+	struct drm_display_mode *mode;
+
+	mode = drm_mode_duplicate(panel->drm, osd101t2587->default_mode);
+	if (!mode) {
+		dev_err(panel->drm->dev, "failed to add mode %ux%ux@%u\n",
+			osd101t2587->default_mode->hdisplay,
+			osd101t2587->default_mode->vdisplay,
+			osd101t2587->default_mode->vrefresh);
+		return -ENOMEM;
+	}
+
+	drm_mode_set_name(mode);
+
+	drm_mode_probed_add(panel->connector, mode);
+
+	panel->connector->display_info.width_mm = 217;
+	panel->connector->display_info.height_mm = 136;
+
+	return 1;
+}
+
+static const struct drm_panel_funcs osd101t2587_panel_funcs = {
+	.disable = osd101t2587_panel_disable,
+	.unprepare = osd101t2587_panel_unprepare,
+	.prepare = osd101t2587_panel_prepare,
+	.enable = osd101t2587_panel_enable,
+	.get_modes = osd101t2587_panel_get_modes,
+};
+
+static const struct of_device_id osd101t2587_of_match[] = {
+	{
+		.compatible = "osd,osd101t2587-53ts",
+		.data = &default_mode_osd101t2587,
+	}, {
+		/* sentinel */
+	}
+};
+MODULE_DEVICE_TABLE(of, osd101t2587_of_match);
+
+static int osd101t2587_panel_add(struct osd101t2587_panel *osd101t2587)
+{
+	struct device *dev = &osd101t2587->dsi->dev;
+
+	osd101t2587->supply = devm_regulator_get(dev, "power");
+	if (IS_ERR(osd101t2587->supply))
+		return PTR_ERR(osd101t2587->supply);
+
+	osd101t2587->backlight = devm_of_find_backlight(dev);
+	if (IS_ERR(osd101t2587->backlight))
+		return PTR_ERR(osd101t2587->backlight);
+
+	drm_panel_init(&osd101t2587->base);
+	osd101t2587->base.funcs = &osd101t2587_panel_funcs;
+	osd101t2587->base.dev = &osd101t2587->dsi->dev;
+
+	return drm_panel_add(&osd101t2587->base);
+}
+
+static int osd101t2587_panel_probe(struct mipi_dsi_device *dsi)
+{
+	struct osd101t2587_panel *osd101t2587;
+	const struct of_device_id *id;
+	int ret;
+
+	id = of_match_node(osd101t2587_of_match, dsi->dev.of_node);
+	if (!id)
+		return -ENODEV;
+
+	dsi->lanes = 4;
+	dsi->format = MIPI_DSI_FMT_RGB888;
+	dsi->mode_flags = MIPI_DSI_MODE_VIDEO |
+			  MIPI_DSI_MODE_VIDEO_BURST |
+			  MIPI_DSI_MODE_VIDEO_SYNC_PULSE |
+			  MIPI_DSI_MODE_EOT_PACKET;
+
+	osd101t2587 = devm_kzalloc(&dsi->dev, sizeof(*osd101t2587), GFP_KERNEL);
+	if (!osd101t2587)
+		return -ENOMEM;
+
+	mipi_dsi_set_drvdata(dsi, osd101t2587);
+
+	osd101t2587->dsi = dsi;
+	osd101t2587->default_mode = id->data;
+
+	ret = osd101t2587_panel_add(osd101t2587);
+	if (ret < 0)
+		return ret;
+
+	ret = mipi_dsi_attach(dsi);
+	if (ret)
+		drm_panel_remove(&osd101t2587->base);
+
+	return ret;
+}
+
+static int osd101t2587_panel_remove(struct mipi_dsi_device *dsi)
+{
+	struct osd101t2587_panel *osd101t2587 = mipi_dsi_get_drvdata(dsi);
+	int ret;
+
+	ret = osd101t2587_panel_disable(&osd101t2587->base);
+	if (ret < 0)
+		dev_err(&dsi->dev, "failed to disable panel: %d\n", ret);
+
+	ret = mipi_dsi_detach(dsi);
+	if (ret < 0)
+		dev_err(&dsi->dev, "failed to detach from DSI host: %d\n", ret);
+
+	drm_panel_remove(&osd101t2587->base);
+
+	return 0;
+}
+
+static void osd101t2587_panel_shutdown(struct mipi_dsi_device *dsi)
+{
+	struct osd101t2587_panel *osd101t2587 = mipi_dsi_get_drvdata(dsi);
+
+	osd101t2587_panel_disable(&osd101t2587->base);
+	osd101t2587_panel_unprepare(&osd101t2587->base);
+}
+
+static struct mipi_dsi_driver osd101t2587_panel_driver = {
+	.driver = {
+		.name = "panel-osd-osd101t2587-53ts",
+		.of_match_table = osd101t2587_of_match,
+	},
+	.probe = osd101t2587_panel_probe,
+	.remove = osd101t2587_panel_remove,
+	.shutdown = osd101t2587_panel_shutdown,
+};
+module_mipi_dsi_driver(osd101t2587_panel_driver);
+
+MODULE_AUTHOR("Peter Ujfalusi <peter.ujfalusi@ti.com>");
+MODULE_DESCRIPTION("OSD101T2587-53TS DSI panel");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/gpu/drm/panel/panel-seiko-43wvf1g.c b/drivers/gpu/drm/panel/panel-seiko-43wvf1g.c
--- a/drivers/gpu/drm/panel/panel-seiko-43wvf1g.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/panel/panel-seiko-43wvf1g.c	2020-06-19 21:00:31.412227715 -0400
@@ -331,7 +331,7 @@
 		.height = 57,
 	},
 	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
-	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_NEGEDGE,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE,
 };
 
 static const struct of_device_id platform_of_match[] = {
diff -Naur --no-dereference a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c
--- a/drivers/gpu/drm/panel/panel-simple.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/panel/panel-simple.c	2020-06-19 21:00:31.412227715 -0400
@@ -634,6 +634,32 @@
 	},
 };
 
+static const struct drm_display_mode auo_g101evn01_0_mode = {
+	.clock = 68930,
+	.hdisplay = 1280,
+	.hsync_start = 1280 + 48,
+	.hsync_end = 1280 + 48 + 32,
+	.htotal = 1280 + 48 + 32 + 48,
+	.vdisplay = 800,
+	.vsync_start = 800 + 4,
+	.vsync_end = 800 + 4 + 4,
+	.vtotal = 800 + 4 + 4 + 8,
+	.vrefresh = 60,
+	.flags = DRM_MODE_FLAG_NVSYNC | DRM_MODE_FLAG_NHSYNC,
+};
+
+static const struct panel_desc auo_g101evn01_0 = {
+	.modes = &auo_g101evn01_0_mode,
+	.num_modes = 1,
+	.bpc = 8,
+	.size = {
+		.width = 217,
+		.height = 136,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB666_1X7X3_SPWG,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE,
+};
+
 static const struct drm_display_mode auo_g104sn02_mode = {
 	.clock = 40000,
 	.hdisplay = 800,
@@ -955,7 +981,7 @@
 		.height = 91,
 	},
 	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
-	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_POSEDGE,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE,
 };
 
 static const struct display_timing dlc_dlc0700yzg_1_timing = {
@@ -1010,7 +1036,7 @@
 		.height = 86,
 	},
 	.bus_format = MEDIA_BUS_FMT_RGB666_1X18,
-	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_NEGEDGE,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE,
 };
 
 static const struct drm_display_mode edt_etm0700g0dh6_mode = {
@@ -1036,7 +1062,7 @@
 		.height = 91,
 	},
 	.bus_format = MEDIA_BUS_FMT_RGB666_1X18,
-	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_NEGEDGE,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE,
 };
 
 static const struct panel_desc edt_etm0700g0bdh6 = {
@@ -1048,7 +1074,7 @@
 		.height = 91,
 	},
 	.bus_format = MEDIA_BUS_FMT_RGB666_1X18,
-	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_POSEDGE,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE,
 };
 
 static const struct drm_display_mode foxlink_fl500wvr00_a0t_mode = {
@@ -1202,7 +1228,7 @@
 		.height = 54,
 	},
 	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
-	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_POSEDGE,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE,
 };
 
 static const struct drm_display_mode innolux_at070tn92_mode = {
@@ -1684,7 +1710,7 @@
 		.height = 54,
 	},
 	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
-	.bus_flags = DRM_BUS_FLAG_PIXDATA_POSEDGE,
+	.bus_flags = DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE,
 };
 
 static const struct drm_display_mode netron_dy_e231732_mode = {
@@ -1733,8 +1759,8 @@
 		.height = 54,
 	},
 	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
-	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_POSEDGE |
-		     DRM_BUS_FLAG_SYNC_POSEDGE,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE |
+		     DRM_BUS_FLAG_SYNC_DRIVE_POSEDGE,
 };
 
 static const struct display_timing nlt_nl192108ac18_02d_timing = {
@@ -1895,7 +1921,33 @@
 		.height = 93,
 	},
 	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
-	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_POSEDGE,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE,
+};
+
+static const struct drm_display_mode osddisplays_osd070t1718_19ts_mode  = {
+	.clock = 33000,
+	.hdisplay = 800,
+	.hsync_start = 800 + 210,
+	.hsync_end = 800 + 210 + 30,
+	.htotal = 800 + 210 + 30 + 16,
+	.vdisplay = 480,
+	.vsync_start = 480 + 22,
+	.vsync_end = 480 + 22 + 13,
+	.vtotal = 480 + 22 + 13 + 10,
+	.vrefresh = 60,
+	.flags = DRM_MODE_FLAG_NVSYNC | DRM_MODE_FLAG_NHSYNC,
+};
+
+static const struct panel_desc osddisplays_osd070t1718_19ts = {
+	.modes = &osddisplays_osd070t1718_19ts_mode,
+	.num_modes = 1,
+	.bpc = 8,
+	.size = {
+		.width = 152,
+		.height = 91,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE,
 };
 
 static const struct drm_display_mode qd43003c0_40_mode = {
@@ -1952,6 +2004,34 @@
 	.bus_format = MEDIA_BUS_FMT_RGB666_1X18,
 };
 
+static const struct drm_display_mode rocktech_rk101ii01d_ct_mode = {
+	.clock = 71100,
+	.hdisplay = 1280,
+	.hsync_start = 1280 + 48,
+	.hsync_end = 1280 + 48 + 32,
+	.htotal = 1280 + 48 + 32 + 80,
+	.vdisplay = 800,
+	.vsync_start = 800 + 2,
+	.vsync_end = 800 + 2 + 5,
+	.vtotal = 800 + 2 + 5 + 16,
+	.vrefresh = 60,
+};
+
+static const struct panel_desc rocktech_rk101ii01d_ct = {
+	.modes = &rocktech_rk101ii01d_ct_mode,
+	.num_modes = 1,
+	.size = {
+		.width = 217,
+		.height = 136,
+	},
+	.delay = {
+		.prepare = 50,
+		.disable = 50,
+	},
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH,
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
+};
+
 static const struct drm_display_mode samsung_lsn122dl01_c01_mode = {
 	.clock = 271560,
 	.hdisplay = 2560,
@@ -2240,7 +2320,7 @@
 		.height = 116,
 	},
 	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
-	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_POSEDGE,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE,
 };
 
 static const struct drm_display_mode tpk_f07a_0102_mode = {
@@ -2263,7 +2343,7 @@
 		.width = 152,
 		.height = 91,
 	},
-	.bus_flags = DRM_BUS_FLAG_PIXDATA_POSEDGE,
+	.bus_flags = DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE,
 };
 
 static const struct drm_display_mode tpk_f10a_0102_mode = {
@@ -2381,6 +2461,9 @@
 		.compatible = "auo,g070vvn01",
 		.data = &auo_g070vvn01,
 	}, {
+		.compatible = "auo,g101evn01.0",
+		.data = &auo_g101evn01_0,
+	}, {
 		.compatible = "auo,g104sn02",
 		.data = &auo_g104sn02,
 	}, {
@@ -2534,12 +2617,18 @@
 		.compatible = "ortustech,com43h4m85ulc",
 		.data = &ortustech_com43h4m85ulc,
 	}, {
+		.compatible = "osddisplays,osd070t1718-19ts",
+		.data = &osddisplays_osd070t1718_19ts,
+	}, {
 		.compatible = "qiaodian,qd43003c0-40",
 		.data = &qd43003c0_40,
 	}, {
 		.compatible = "rocktech,rk070er9427",
 		.data = &rocktech_rk070er9427,
 	}, {
+		.compatible = "rockteck,rk101ii01d-ct",
+		.data = &rocktech_rk101ii01d_ct,
+	}, {
 		.compatible = "samsung,lsn122dl01-c01",
 		.data = &samsung_lsn122dl01_c01,
 	}, {
@@ -2790,6 +2879,37 @@
 	.lanes = 4,
 };
 
+static const struct drm_display_mode osd101t2045_53ts_mode = {
+	.clock = 154500,
+	.hdisplay = 1920,
+	.hsync_start = 1920 + 112,
+	.hsync_end = 1920 + 112 + 16,
+	.htotal = 1920 + 112 + 16 + 32,
+	.vdisplay = 1200,
+	.vsync_start = 1200 + 16,
+	.vsync_end = 1200 + 16 + 2,
+	.vtotal = 1200 + 16 + 2 + 16,
+	.vrefresh = 60,
+	.flags = DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC,
+};
+
+static const struct panel_desc_dsi osd101t2045_53ts = {
+	.desc = {
+		.modes = &osd101t2045_53ts_mode,
+		.num_modes = 1,
+		.bpc = 8,
+		.size = {
+			.width = 217,
+			.height = 136,
+		},
+	},
+	.flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST |
+		 MIPI_DSI_MODE_VIDEO_SYNC_PULSE |
+		 MIPI_DSI_MODE_EOT_PACKET,
+	.format = MIPI_DSI_FMT_RGB888,
+	.lanes = 4,
+};
+
 static const struct of_device_id dsi_of_match[] = {
 	{
 		.compatible = "auo,b080uan01",
@@ -2807,6 +2927,9 @@
 		.compatible = "panasonic,vvx10f004b00",
 		.data = &panasonic_vvx10f004b00
 	}, {
+		.compatible = "osd,osd101t2045-53ts",
+		.data = &osd101t2045_53ts
+	}, {
 		/* sentinel */
 	}
 };
diff -Naur --no-dereference a/drivers/gpu/drm/pl111/pl111_display.c b/drivers/gpu/drm/pl111/pl111_display.c
--- a/drivers/gpu/drm/pl111/pl111_display.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/pl111/pl111_display.c	2020-06-19 21:00:31.412227715 -0400
@@ -188,7 +188,7 @@
 			tim2 |= TIM2_IOE;
 
 		if (connector->display_info.bus_flags &
-		    DRM_BUS_FLAG_PIXDATA_NEGEDGE)
+		    DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE)
 			tim2 |= TIM2_IPC;
 	}
 
diff -Naur --no-dereference a/drivers/gpu/drm/tegra/hub.c b/drivers/gpu/drm/tegra/hub.c
--- a/drivers/gpu/drm/tegra/hub.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/tegra/hub.c	2020-06-19 21:00:31.412227715 -0400
@@ -718,7 +718,7 @@
 	if (!state)
 		return -ENOMEM;
 
-	drm_atomic_private_obj_init(&hub->base, &state->base,
+	drm_atomic_private_obj_init(drm, &hub->base, &state->base,
 				    &tegra_display_hub_state_funcs);
 
 	tegra->hub = hub;
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/Kconfig b/drivers/gpu/drm/tidss/Kconfig
--- a/drivers/gpu/drm/tidss/Kconfig	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/Kconfig	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,41 @@
+config DRM_TIDSS
+	tristate "DRM Support for TI Keystone"
+	depends on DRM && OF
+	depends on ARM || ARM64 || COMPILE_TEST
+	select DRM_KMS_HELPER
+	select DRM_KMS_CMA_HELPER
+	select DRM_GEM_CMA_HELPER
+	select VIDEOMODE_HELPERS
+	help
+	  The TI Keystone family SoCs introduced a new generation of
+	  Display SubSystem. They are called DSS6 and DSS7 and at the
+	  time of writing this these DSSs are found on 66AK2Gx and
+	  AM6x SoCs. Set this to Y or M to add display support for
+	  TI Keystone family platforms.
+
+if DRM_TIDSS
+
+config DRM_TIDSS_DSS6
+	bool "DSS6 support"
+	default y
+	help
+	  Support for DSS6 IP on K2G SoC
+
+config DRM_TIDSS_DSS7
+	bool "DSS7 support"
+	default y
+	help
+	  Support for DSS7 IP on AM6 SoC
+
+config DRM_TIDSS_WB
+	bool "Enable writeback support for TIDSS DRM driver"
+	depends on DRM_TIDSS
+	depends on (VIDEO_V4L2 = y) || (VIDEO_V4L2 = m && DRM_TIDSS = m)
+	depends on VIDEO_DEV && HAS_DMA
+	select VIDEOBUF2_DMA_CONTIG
+	select V4L2_MEM2MEM_DEV
+	default y
+	help
+		Select this to enable memory-to-memory/capture writeback support.
+
+endif
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/Makefile b/drivers/gpu/drm/tidss/Makefile
--- a/drivers/gpu/drm/tidss/Makefile	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/Makefile	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: GPL-2.0
+
+tidss-y := tidss_crtc.o \
+	tidss_drv.o \
+	tidss_encoder.o \
+	tidss_irq.o \
+	tidss_kms.o \
+	tidss_plane.o \
+	tidss_scale_coefs.o \
+	tidss_v_connector.o \
+	tidss_v_encoder.o \
+	tidss_v_plane.o \
+	tidss_v_crtc.o
+
+tidss-$(CONFIG_DRM_TIDSS_DSS6) += tidss_dispc6.o
+tidss-$(CONFIG_DRM_TIDSS_DSS7) += tidss_dispc7.o
+
+tidss-$(CONFIG_DRM_TIDSS_WB) += tidss_wb.o tidss_wb_m2m.o
+
+obj-$(CONFIG_DRM_TIDSS) += tidss.o
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/tidss_crtc.c b/drivers/gpu/drm/tidss/tidss_crtc.c
--- a/drivers/gpu/drm/tidss/tidss_crtc.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/tidss_crtc.c	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,408 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ */
+
+#include <drm/drmP.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_plane_helper.h>
+
+#include "tidss_crtc.h"
+#include "tidss_dispc.h"
+#include "tidss_drv.h"
+#include "tidss_irq.h"
+
+/* -----------------------------------------------------------------------------
+ * Page Flip
+ */
+
+static void tidss_crtc_finish_page_flip(struct tidss_crtc *tcrtc)
+{
+	struct drm_device *ddev = tcrtc->crtc.dev;
+	struct tidss_device *tidss = ddev->dev_private;
+	struct drm_pending_vblank_event *event;
+	unsigned long flags;
+	bool busy;
+
+	spin_lock_irqsave(&ddev->event_lock, flags);
+
+	/*
+	 * New settings are taken into use at VFP, and GO bit is cleared at
+	 * the same time. This happens before the vertical blank interrupt.
+	 * So there is a small change that the driver sets GO bit after VFP, but
+	 * before vblank, and we have to check for that case here.
+	 */
+	busy = tidss->dispc_ops->vp_go_busy(tidss->dispc, tcrtc->hw_videoport);
+	if (busy) {
+		spin_unlock_irqrestore(&ddev->event_lock, flags);
+		return;
+	}
+
+	event = tcrtc->event;
+	tcrtc->event = NULL;
+
+	if (!event) {
+		spin_unlock_irqrestore(&ddev->event_lock, flags);
+		return;
+	}
+
+	drm_crtc_send_vblank_event(&tcrtc->crtc, event);
+
+	spin_unlock_irqrestore(&ddev->event_lock, flags);
+
+	drm_crtc_vblank_put(&tcrtc->crtc);
+}
+
+void tidss_crtc_vblank_irq(struct drm_crtc *crtc)
+{
+	struct tidss_crtc *tcrtc = to_tidss_crtc(crtc);
+
+	drm_crtc_handle_vblank(crtc);
+
+	tidss_crtc_finish_page_flip(tcrtc);
+}
+
+void tidss_crtc_framedone_irq(struct drm_crtc *crtc)
+{
+	struct tidss_crtc *tcrtc = to_tidss_crtc(crtc);
+
+	complete(&tcrtc->framedone_completion);
+}
+
+void tidss_crtc_error_irq(struct drm_crtc *crtc, u64 irqstatus)
+{
+	struct tidss_crtc *tcrtc = to_tidss_crtc(crtc);
+
+	dev_err_ratelimited(crtc->dev->dev, "CRTC%u SYNC LOST: (irq %llx)\n",
+			    tcrtc->hw_videoport, irqstatus);
+}
+
+/* -----------------------------------------------------------------------------
+ * CRTC Functions
+ */
+
+static int tidss_crtc_atomic_check(struct drm_crtc *crtc,
+				   struct drm_crtc_state *state)
+{
+	struct tidss_crtc *tcrtc = to_tidss_crtc(crtc);
+	struct drm_device *ddev = crtc->dev;
+	struct tidss_device *tidss = ddev->dev_private;
+	int r;
+
+	dev_dbg(ddev->dev, "%s\n", __func__);
+
+	if (!state->enable)
+		return 0;
+
+	r = tidss->dispc_ops->vp_check(tidss->dispc, tcrtc->hw_videoport,
+				       state);
+
+	return r;
+}
+
+static void tidss_crtc_atomic_flush(struct drm_crtc *crtc,
+				    struct drm_crtc_state *old_crtc_state)
+{
+	struct tidss_crtc *tcrtc = to_tidss_crtc(crtc);
+	struct drm_device *ddev = crtc->dev;
+	struct tidss_device *tidss = ddev->dev_private;
+	unsigned long flags;
+
+	dev_dbg(ddev->dev, "%s, crtc enabled %d, event %p\n",
+		__func__, tcrtc->enabled, crtc->state->event);
+
+	/* Only flush the CRTC if it is currently enabled. */
+	if (!tcrtc->enabled)
+		return;
+
+	/* If the GO bit is stuck we better quit here. */
+	if (WARN_ON(tidss->dispc_ops->vp_go_busy(tidss->dispc,
+						 tcrtc->hw_videoport)))
+		return;
+
+	/* We should have event if CRTC is enabled through out this commit. */
+	if (WARN_ON(!crtc->state->event))
+		return;
+
+	tidss->dispc_ops->vp_setup(tidss->dispc,
+				   tcrtc->hw_videoport,
+				   crtc->state);
+
+	WARN_ON(drm_crtc_vblank_get(crtc) != 0);
+
+	spin_lock_irqsave(&ddev->event_lock, flags);
+	tidss->dispc_ops->vp_go(tidss->dispc, tcrtc->hw_videoport);
+
+	WARN_ON(tcrtc->event);
+
+	tcrtc->event = crtc->state->event;
+	crtc->state->event = NULL;
+
+	spin_unlock_irqrestore(&ddev->event_lock, flags);
+}
+
+static void tidss_crtc_atomic_enable(struct drm_crtc *crtc,
+				     struct drm_crtc_state *old_state)
+{
+	struct tidss_crtc *tcrtc = to_tidss_crtc(crtc);
+	struct drm_device *ddev = crtc->dev;
+	struct tidss_device *tidss = ddev->dev_private;
+	const struct drm_display_mode *mode = &crtc->state->adjusted_mode;
+	int r;
+	unsigned long flags;
+
+	dev_dbg(ddev->dev, "%s, event %p\n", __func__, crtc->state->event);
+
+	tidss->dispc_ops->runtime_get(tidss->dispc);
+
+	r = tidss->dispc_ops->vp_set_clk_rate(tidss->dispc, tcrtc->hw_videoport,
+					      mode->clock * 1000);
+	if (r != 0)
+		return;
+
+	r = tidss->dispc_ops->vp_enable_clk(tidss->dispc, tcrtc->hw_videoport);
+	if (r != 0)
+		return;
+
+	tidss->dispc_ops->vp_setup(tidss->dispc, tcrtc->hw_videoport,
+				   crtc->state);
+
+	/* Turn vertical blanking interrupt reporting on. */
+	drm_crtc_vblank_on(crtc);
+
+	if (tidss->dispc_ops->vp_prepare)
+		tidss->dispc_ops->vp_prepare(tidss->dispc, tcrtc->hw_videoport,
+					     crtc->state);
+
+	tcrtc->enabled = true;
+
+	tidss->dispc_ops->vp_enable(tidss->dispc, tcrtc->hw_videoport,
+				    crtc->state);
+
+	spin_lock_irqsave(&ddev->event_lock, flags);
+
+	if (crtc->state->event) {
+		drm_crtc_send_vblank_event(crtc, crtc->state->event);
+		crtc->state->event = NULL;
+	}
+
+	spin_unlock_irqrestore(&ddev->event_lock, flags);
+}
+
+static void tidss_crtc_atomic_disable(struct drm_crtc *crtc,
+				      struct drm_crtc_state *old_state)
+{
+	struct tidss_crtc *tcrtc = to_tidss_crtc(crtc);
+	struct drm_device *ddev = crtc->dev;
+	struct tidss_device *tidss = ddev->dev_private;
+	unsigned long flags;
+
+	dev_dbg(ddev->dev, "%s, event %p\n", __func__, crtc->state->event);
+
+	reinit_completion(&tcrtc->framedone_completion);
+
+	tidss->dispc_ops->vp_disable(tidss->dispc, tcrtc->hw_videoport);
+
+	if (!wait_for_completion_timeout(&tcrtc->framedone_completion,
+					 msecs_to_jiffies(500)))
+		dev_err(tidss->dev, "Timeout waiting for framedone on crtc %d",
+			tcrtc->hw_videoport);
+
+	if (tidss->dispc_ops->vp_unprepare)
+		tidss->dispc_ops->vp_unprepare(tidss->dispc,
+					       tcrtc->hw_videoport);
+
+	spin_lock_irqsave(&ddev->event_lock, flags);
+	if (crtc->state->event) {
+		drm_crtc_send_vblank_event(crtc, crtc->state->event);
+		crtc->state->event = NULL;
+	}
+	spin_unlock_irqrestore(&ddev->event_lock, flags);
+
+	tcrtc->enabled = false;
+
+	drm_crtc_vblank_off(crtc);
+
+	tidss->dispc_ops->vp_disable_clk(tidss->dispc, tcrtc->hw_videoport);
+
+	tidss->dispc_ops->runtime_put(tidss->dispc);
+}
+
+static
+enum drm_mode_status tidss_crtc_mode_valid(struct drm_crtc *crtc,
+					   const struct drm_display_mode *mode)
+{
+	struct tidss_crtc *tcrtc = to_tidss_crtc(crtc);
+	struct drm_device *ddev = crtc->dev;
+	struct tidss_device *tidss = ddev->dev_private;
+
+	return tidss->dispc_ops->vp_mode_valid(tidss->dispc,
+					       tcrtc->hw_videoport,
+					       mode);
+}
+
+static const struct drm_crtc_helper_funcs crtc_helper_funcs = {
+	.atomic_check = tidss_crtc_atomic_check,
+	.atomic_flush = tidss_crtc_atomic_flush,
+	.atomic_enable = tidss_crtc_atomic_enable,
+	.atomic_disable = tidss_crtc_atomic_disable,
+
+	.mode_valid = tidss_crtc_mode_valid,
+};
+
+static void tidss_crtc_reset(struct drm_crtc *crtc)
+{
+	struct tidss_crtc_state *tcrtc;
+
+	if (crtc->state)
+		__drm_atomic_helper_crtc_destroy_state(crtc->state);
+
+	kfree(crtc->state);
+
+	tcrtc = kzalloc(sizeof(*tcrtc), GFP_KERNEL);
+	if (!tcrtc) {
+		crtc->state = NULL;
+		return;
+	}
+
+	crtc->state = &tcrtc->base;
+	crtc->state->crtc = crtc;
+}
+
+static struct drm_crtc_state *
+tidss_crtc_duplicate_state(struct drm_crtc *crtc)
+{
+	struct tidss_crtc_state *state, *current_state;
+
+	if (WARN_ON(!crtc->state))
+		return NULL;
+
+	current_state = to_tidss_crtc_state(crtc->state);
+
+	state = kmalloc(sizeof(*state), GFP_KERNEL);
+	if (!state)
+		return NULL;
+
+	__drm_atomic_helper_crtc_duplicate_state(crtc, &state->base);
+
+	state->bus_format = current_state->bus_format;
+	state->bus_flags = current_state->bus_flags;
+
+	return &state->base;
+}
+
+
+static int tidss_crtc_atomic_set_property(struct drm_crtc *crtc,
+					  struct drm_crtc_state *state,
+					  struct drm_property *property,
+					  uint64_t val)
+{
+	return -EINVAL;
+}
+
+static int tidss_crtc_atomic_get_property(struct drm_crtc *crtc,
+					  const struct drm_crtc_state *state,
+					  struct drm_property *property,
+					  uint64_t *val)
+{
+	return -EINVAL;
+}
+
+static int tidss_crtc_enable_vblank(struct drm_crtc *crtc)
+{
+	struct drm_device *ddev = crtc->dev;
+	struct tidss_device *tidss = ddev->dev_private;
+
+	dev_dbg(ddev->dev, "%s\n", __func__);
+
+	tidss->dispc_ops->runtime_get(tidss->dispc);
+
+	tidss_irq_enable_vblank(crtc);
+
+	return 0;
+}
+
+static void tidss_crtc_disable_vblank(struct drm_crtc *crtc)
+{
+	struct drm_device *ddev = crtc->dev;
+	struct tidss_device *tidss = ddev->dev_private;
+
+	dev_dbg(ddev->dev, "%s\n", __func__);
+
+	tidss_irq_disable_vblank(crtc);
+
+	tidss->dispc_ops->runtime_put(tidss->dispc);
+}
+
+static const struct drm_crtc_funcs crtc_funcs = {
+	.reset = tidss_crtc_reset,
+	.destroy = drm_crtc_cleanup,
+	.set_config = drm_atomic_helper_set_config,
+	.page_flip = drm_atomic_helper_page_flip,
+	.atomic_duplicate_state = tidss_crtc_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
+	.atomic_set_property = tidss_crtc_atomic_set_property,
+	.atomic_get_property = tidss_crtc_atomic_get_property,
+	.enable_vblank = tidss_crtc_enable_vblank,
+	.disable_vblank = tidss_crtc_disable_vblank,
+};
+
+static void tidss_crtc_install_properties(struct tidss_device *tidss,
+					  const struct tidss_vp_feat *vp_feat,
+					  struct drm_crtc *crtc)
+{
+}
+
+struct tidss_crtc *tidss_crtc_create(struct tidss_device *tidss, u32 hw_videoport,
+				     struct drm_plane *primary)
+{
+	struct tidss_crtc *tcrtc;
+	struct drm_crtc *crtc;
+	const struct tidss_vp_feat *vp_feat;
+	unsigned int gamma_lut_size = 0;
+	int ret;
+
+	vp_feat = tidss->dispc_ops->vp_feat(tidss->dispc, hw_videoport);
+
+	tcrtc = devm_kzalloc(tidss->dev, sizeof(*tcrtc), GFP_KERNEL);
+	if (!tcrtc)
+		return ERR_PTR(-ENOMEM);
+
+	tcrtc->hw_videoport = hw_videoport;
+	init_completion(&tcrtc->framedone_completion);
+
+	crtc =  &tcrtc->crtc;
+
+	ret = drm_crtc_init_with_planes(tidss->ddev, crtc, primary,
+					NULL, &crtc_funcs, NULL);
+	if (ret < 0)
+		return ERR_PTR(ret);
+
+	drm_crtc_helper_add(crtc, &crtc_helper_funcs);
+
+	/*
+	 * The dispc API adapts to what ever size we ask from it no
+	 * matter what HW supports. X-server assumes 256 element gamma
+	 * tables so lets use that. Size of HW gamma table size is
+	 * found from struct tidss_vp_feat that is extracted with
+	 * dispc_vp_feats(). If gamma_size is 0 gamma table is not
+	 * supported.
+	 */
+	if (vp_feat->color.gamma_size)
+		gamma_lut_size = 256;
+
+	drm_crtc_enable_color_mgmt(crtc, 0, vp_feat->color.has_ctm,
+				   gamma_lut_size);
+	if (gamma_lut_size)
+		drm_mode_crtc_set_gamma_size(crtc, gamma_lut_size);
+
+	tidss_crtc_install_properties(tidss, vp_feat, crtc);
+
+	return tcrtc;
+}
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/tidss_crtc.h b/drivers/gpu/drm/tidss/tidss_crtc.h
--- a/drivers/gpu/drm/tidss/tidss_crtc.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/tidss_crtc.h	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,49 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ */
+
+#ifndef __TIDSS_CRTC_H__
+#define __TIDSS_CRTC_H__
+
+#include <drm/drm_crtc.h>
+#include <linux/completion.h>
+#include <linux/wait.h>
+
+#include "tidss_dispc.h"
+
+#define to_tidss_crtc(c) container_of((c), struct tidss_crtc, crtc)
+
+struct tidss_crtc {
+	struct drm_crtc crtc;
+
+	u32 hw_videoport;
+
+	struct drm_pending_vblank_event *event;
+
+	/* has crtc_atomic_enable been called? */
+	bool enabled;
+
+	struct completion framedone_completion;
+};
+
+#define to_tidss_crtc_state(x) container_of(x, struct tidss_crtc_state, base)
+
+struct tidss_crtc_state {
+	/* Must be first. */
+	struct drm_crtc_state base;
+
+	u32 bus_format;
+	u32 bus_flags;
+};
+
+struct tidss_crtc *tidss_crtc_create(struct tidss_device *tidss, u32 hw_videoport,
+				     struct drm_plane *primary);
+
+
+void tidss_crtc_vblank_irq(struct drm_crtc *crtc);
+void tidss_crtc_framedone_irq(struct drm_crtc *crtc);
+void tidss_crtc_error_irq(struct drm_crtc *crtc, u64 irqstatus);
+
+#endif
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/tidss_dispc6.c b/drivers/gpu/drm/tidss/tidss_dispc6.c
--- a/drivers/gpu/drm/tidss/tidss_dispc6.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/tidss_dispc6.c	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,1505 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/component.h>
+#include <linux/delay.h>
+#include <linux/export.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/of.h>
+#include <linux/of_graph.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+
+#include <drm/drm_fourcc.h>
+#include <drm/drm_of.h>
+#include <drm/drm_panel.h>
+
+#include "tidss_crtc.h"
+#include "tidss_drv.h"
+#include "tidss_encoder.h"
+#include "tidss_plane.h"
+
+#include "tidss_dispc6.h"
+
+static const struct {
+	u32 fmt;
+	u32 port_width;
+} dispc6_bus_formats[] = {
+	{ MEDIA_BUS_FMT_RGB444_1X12, 12 },
+	{ MEDIA_BUS_FMT_RGB565_1X16, 16 },
+	{ MEDIA_BUS_FMT_RGB666_1X18, 18 },
+	{ MEDIA_BUS_FMT_RGB888_1X24, 24 },
+};
+
+/*
+ * TRM gives bitfields as start:end, where start is the higher bit
+ * number. For example 7:0
+ */
+
+#define FLD_MASK(start, end)	(((1 << ((start) - (end) + 1)) - 1) << (end))
+#define FLD_VAL(val, start, end) (((val) << (end)) & FLD_MASK(start, end))
+#define FLD_GET(val, start, end) (((val) & FLD_MASK(start, end)) >> (end))
+#define FLD_MOD(orig, val, start, end) \
+	(((orig) & ~FLD_MASK(start, end)) | FLD_VAL(val, start, end))
+
+
+#define REG_GET(dispc, idx, start, end)	\
+	FLD_GET(dispc6_read(dispc, idx), start, end)
+
+#define REG_FLD_MOD(dispc, idx, val, start, end) \
+	dispc6_write(dispc, idx, FLD_MOD(dispc6_read(dispc, idx), val, start, end))
+
+#define VID_REG_GET(dispc, plane, idx, start, end) \
+	FLD_GET(dispc6_vid_read(dispc, plane, idx), start, end)
+
+#define VID_REG_FLD_MOD(dispc, plane, idx, val, start, end) \
+	dispc6_vid_write(dispc, plane, idx, FLD_MOD(dispc6_vid_read(dispc, plane, idx), val, start, end))
+
+
+#define VP_REG_GET(dispc, vp, idx, start, end) \
+	FLD_GET(dispc6_vp_read(dispc, vp, idx), start, end)
+
+#define VP_REG_FLD_MOD(dispc, vp, idx, val, start, end)	\
+	dispc6_vp_write(dispc, vp, idx, FLD_MOD(dispc6_vp_read(dispc, vp, idx), val, start, end))
+
+#define DISPC6_GAMMA_TABLE_SIZE 256
+
+struct dispc_features {
+	/* XXX should these come from the .dts? Min pclk is not feature of DSS IP */
+	unsigned long min_pclk;
+	unsigned long max_pclk;
+};
+
+/* Note: 9MHz is a special allowed case, and is handled separately in the code */
+static const struct dispc_features k2g_dispc_feats = {
+	.min_pclk = 43750000,
+	.max_pclk = 150000000,
+};
+
+static const struct of_device_id dispc6_of_match[] = {
+	{ .compatible = "ti,k2g-dss", .data = &k2g_dispc_feats, },
+	{},
+};
+
+struct dispc_device {
+	struct device *dev;
+
+	void __iomem *base_cfg;
+	void __iomem *base_common;
+	void __iomem *base_vid1;
+	void __iomem *base_ovr1;
+	void __iomem *base_vp1;
+
+	const struct dispc_features *feat;
+
+	struct clk *fclk;
+	struct clk *vp_clk;
+
+	u32 memory_bandwidth_limit;
+
+	bool is_enabled;
+
+	u32 gamma_table[DISPC6_GAMMA_TABLE_SIZE];
+
+	struct tidss_device *tidss;
+};
+
+static void dispc6_write(struct dispc_device *dispc, u16 reg, u32 val)
+{
+	iowrite32(val, dispc->base_common + reg);
+}
+
+static u32 dispc6_read(struct dispc_device *dispc, u16 reg)
+{
+	return ioread32(dispc->base_common + reg);
+}
+
+static void dispc6_vid_write(struct dispc_device *dispc,
+			     u32 hw_plane, u16 reg, u32 val)
+{
+	void __iomem *base = dispc->base_vid1;
+
+	iowrite32(val, base + reg);
+}
+
+static u32 dispc6_vid_read(struct dispc_device *dispc,
+			   u32 hw_plane, u16 reg)
+{
+	void __iomem *base = dispc->base_vid1;
+
+	return ioread32(base + reg);
+}
+
+static void dispc6_ovr_write(struct dispc_device *dispc,
+			     u32 hw_videoport, u16 reg, u32 val)
+{
+	void __iomem *base = dispc->base_ovr1;
+
+	iowrite32(val, base + reg);
+}
+
+__maybe_unused
+static u32 dispc6_ovr_read(struct dispc_device *dispc,
+			   u32 hw_videoport, u16 reg)
+{
+	void __iomem *base = dispc->base_ovr1;
+
+	return ioread32(base + reg);
+}
+
+static void dispc6_vp_write(struct dispc_device *dispc,
+			    u32 hw_videoport, u16 reg, u32 val)
+{
+	void __iomem *base = dispc->base_vp1;
+
+	iowrite32(val, base + reg);
+}
+
+static u32 dispc6_vp_read(struct dispc_device *dispc,
+			  u32 hw_videoport, u16 reg)
+{
+	void __iomem *base = dispc->base_vp1;
+
+	return ioread32(base + reg);
+}
+
+static int dispc6_runtime_get(struct dispc_device *dispc)
+{
+	int r;
+
+	dev_dbg(dispc->dev, "dispc_runtime_get\n");
+
+	r = pm_runtime_get_sync(dispc->dev);
+	WARN_ON(r < 0);
+	return r < 0 ? r : 0;
+}
+
+static void dispc6_runtime_put(struct dispc_device *dispc)
+{
+	int r;
+
+	dev_dbg(dispc->dev, "dispc_runtime_put\n");
+
+	r = pm_runtime_put_sync(dispc->dev);
+	WARN_ON(r < 0);
+}
+
+static u64 dispc6_vp_irq_from_raw(u32 stat)
+{
+	u32 vp = 0;
+	u64 vp_stat = 0;
+
+	if (stat & BIT(0))
+		vp_stat |= DSS_IRQ_VP_FRAME_DONE(vp);
+	if (stat & BIT(1))
+		vp_stat |= DSS_IRQ_VP_VSYNC_EVEN(vp);
+	if (stat & BIT(2))
+		vp_stat |= DSS_IRQ_VP_VSYNC_ODD(vp);
+	if (stat & BIT(4))
+		vp_stat |= DSS_IRQ_VP_SYNC_LOST(vp);
+
+	return vp_stat;
+}
+
+static u32 dispc6_vp_irq_to_raw(u64 vpstat)
+{
+	u32 vp = 0;
+	u32 stat = 0;
+
+	if (vpstat & DSS_IRQ_VP_FRAME_DONE(vp))
+		stat |= BIT(0);
+	if (vpstat & DSS_IRQ_VP_VSYNC_EVEN(vp))
+		stat |= BIT(1);
+	if (vpstat & DSS_IRQ_VP_VSYNC_ODD(vp))
+		stat |= BIT(2);
+	if (vpstat & DSS_IRQ_VP_SYNC_LOST(vp))
+		stat |= BIT(4);
+
+	return stat;
+}
+
+static u64 dispc6_vid_irq_from_raw(u32 stat)
+{
+	u32 plane = 0;
+	u64 vid_stat = 0;
+
+	if (stat & BIT(0))
+		vid_stat |= DSS_IRQ_PLANE_FIFO_UNDERFLOW(plane);
+
+	return vid_stat;
+}
+
+static u32 dispc6_vid_irq_to_raw(u64 vidstat)
+{
+	u32 plane = 0;
+	u32 stat = 0;
+
+	if (vidstat & DSS_IRQ_PLANE_FIFO_UNDERFLOW(plane))
+		stat |= BIT(0);
+
+	return stat;
+}
+
+
+static u64 dispc6_vp_read_irqstatus(struct dispc_device *dispc,
+				    u32 hw_videoport)
+{
+	u32 stat = dispc6_vp_read(dispc, hw_videoport, DISPC_VP_IRQSTATUS);
+
+	return dispc6_vp_irq_from_raw(stat);
+}
+
+static void dispc6_vp_write_irqstatus(struct dispc_device *dispc,
+				      u32 hw_videoport,
+				      u64 vpstat)
+{
+	u32 stat = dispc6_vp_irq_to_raw(vpstat);
+
+	dispc6_vp_write(dispc, hw_videoport, DISPC_VP_IRQSTATUS, stat);
+}
+
+static u64 dispc6_vid_read_irqstatus(struct dispc_device *dispc,
+				     u32 hw_plane)
+{
+	u32 stat = dispc6_vid_read(dispc, hw_plane, DISPC_VID_IRQSTATUS);
+
+	return dispc6_vid_irq_from_raw(stat);
+}
+
+static void dispc6_vid_write_irqstatus(struct dispc_device *dispc,
+				       u32 hw_plane,
+				       u64 vidstat)
+{
+	u32 stat = dispc6_vid_irq_to_raw(vidstat);
+
+	dispc6_vid_write(dispc, hw_plane, DISPC_VID_IRQSTATUS, stat);
+}
+
+
+static u64 dispc6_vp_read_irqenable(struct dispc_device *dispc,
+				    u32 hw_videoport)
+{
+	u32 stat = dispc6_vp_read(dispc, hw_videoport, DISPC_VP_IRQENABLE);
+
+	return dispc6_vp_irq_from_raw(stat);
+}
+
+static void dispc6_vp_write_irqenable(struct dispc_device *dispc,
+				      u32 hw_videoport,
+				      u64 vpstat)
+{
+	u32 stat = dispc6_vp_irq_to_raw(vpstat);
+
+	dispc6_vp_write(dispc, hw_videoport, DISPC_VP_IRQENABLE, stat);
+}
+
+static u64 dispc6_vid_read_irqenable(struct dispc_device *dispc,
+				     u32 hw_plane)
+{
+	u32 stat = dispc6_vid_read(dispc, hw_plane, DISPC_VID_IRQENABLE);
+
+	return dispc6_vid_irq_from_raw(stat);
+}
+
+static void dispc6_vid_write_irqenable(struct dispc_device *dispc,
+				       u32 hw_plane,
+				       u64 vidstat)
+{
+	u32 stat = dispc6_vid_irq_to_raw(vidstat);
+
+	dispc6_vid_write(dispc, hw_plane, DISPC_VID_IRQENABLE, stat);
+}
+
+
+static void dispc6_clear_irqstatus(struct dispc_device *dispc, u64 mask)
+{
+	dispc6_vp_write_irqstatus(dispc, 0, mask);
+	dispc6_vid_write_irqstatus(dispc, 0, mask);
+}
+
+static u64 dispc6_read_and_clear_irqstatus(struct dispc_device *dispc)
+{
+	u64 stat = 0;
+
+	/* always clear the top level irqstatus */
+	dispc6_write(dispc, DISPC_IRQSTATUS,
+		     dispc6_read(dispc, DISPC_IRQSTATUS));
+
+	stat |= dispc6_vp_read_irqstatus(dispc, 0);
+	stat |= dispc6_vid_read_irqstatus(dispc, 0);
+
+	dispc6_clear_irqstatus(dispc, stat);
+
+	return stat;
+}
+
+static u64 dispc6_read_irqenable(struct dispc_device *dispc)
+{
+	u64 stat = 0;
+
+	stat |= dispc6_vp_read_irqenable(dispc, 0);
+	stat |= dispc6_vid_read_irqenable(dispc, 0);
+
+	return stat;
+}
+
+static void dispc6_write_irqenable(struct dispc_device *dispc, u64 mask)
+{
+	u64 old_mask = dispc6_read_irqenable(dispc);
+
+	/* clear the irqstatus for newly enabled irqs */
+	dispc6_clear_irqstatus(dispc, (mask ^ old_mask) & mask);
+
+	dispc6_vp_write_irqenable(dispc, 0, mask);
+	dispc6_vid_write_irqenable(dispc, 0, mask);
+
+	dispc6_write(dispc, DISPC_IRQENABLE_SET, (1 << 0) | (1 << 7));
+
+	/* flush posted write */
+	dispc6_read_irqenable(dispc);
+}
+
+static void dispc6_set_num_datalines(struct dispc_device *dispc,
+				     u32 hw_videoport, int num_lines)
+{
+	int v;
+
+	switch (num_lines) {
+	case 12:
+		v = 0; break;
+	case 16:
+		v = 1; break;
+	case 18:
+		v = 2; break;
+	case 24:
+		v = 3; break;
+	case 30:
+		v = 4; break;
+	case 36:
+		v = 5; break;
+	default:
+		WARN_ON(1);
+		v = 3;
+		break;
+	}
+
+	VP_REG_FLD_MOD(dispc, hw_videoport, DISPC_VP_CONTROL, v, 10, 8);
+}
+
+static void dispc6_vp_enable(struct dispc_device *dispc, u32 hw_videoport,
+			     const struct drm_crtc_state *state)
+{
+	const struct drm_display_mode *mode = &state->adjusted_mode;
+	const struct tidss_crtc_state *tstate = to_tidss_crtc_state(state);
+	bool align, onoff, rf, ieo, ipc, ihs, ivs;
+	unsigned int i;
+	u32 port_width;
+	u32 hsw, hfp, hbp, vsw, vfp, vbp;
+
+	for (i = 0; i < ARRAY_SIZE(dispc6_bus_formats); ++i) {
+		if (dispc6_bus_formats[i].fmt != tstate->bus_format)
+			continue;
+
+		port_width = dispc6_bus_formats[i].port_width;
+		break;
+	}
+
+	if (WARN_ON(i == ARRAY_SIZE(dispc6_bus_formats)))
+		return;
+
+	dispc6_set_num_datalines(dispc, hw_videoport, port_width);
+
+	hfp = mode->hsync_start - mode->hdisplay;
+	hsw = mode->hsync_end - mode->hsync_start;
+	hbp = mode->htotal - mode->hsync_end;
+
+	vfp = mode->vsync_start - mode->vdisplay;
+	vsw = mode->vsync_end - mode->vsync_start;
+	vbp = mode->vtotal - mode->vsync_end;
+
+	dispc6_vp_write(dispc, hw_videoport, DISPC_VP_TIMING_H,
+			FLD_VAL(hsw - 1, 7, 0) |
+			FLD_VAL(hfp - 1, 19, 8) |
+			FLD_VAL(hbp - 1, 31, 20));
+
+	dispc6_vp_write(dispc, hw_videoport, DISPC_VP_TIMING_V,
+			FLD_VAL(vsw - 1, 7, 0) |
+			FLD_VAL(vfp, 19, 8) |
+			FLD_VAL(vbp, 31, 20));
+
+	if (mode->flags & DRM_MODE_FLAG_NVSYNC)
+		ivs = true;
+	else
+		ivs = false;
+
+	if (mode->flags & DRM_MODE_FLAG_NHSYNC)
+		ihs = true;
+	else
+		ihs = false;
+
+	if (tstate->bus_flags & DRM_BUS_FLAG_DE_LOW)
+		ieo = true;
+	else
+		ieo = false;
+
+	if (tstate->bus_flags & DRM_BUS_FLAG_PIXDATA_NEGEDGE)
+		ipc = true;
+	else
+		ipc = false;
+
+	/* always use the 'rf' setting */
+	onoff = true;
+
+	if (tstate->bus_flags & DRM_BUS_FLAG_SYNC_NEGEDGE)
+		rf = false;
+	else
+		rf = true;
+
+	/* always use aligned syncs */
+	align = true;
+
+	dispc6_vp_write(dispc, hw_videoport, DISPC_VP_POL_FREQ,
+			FLD_VAL(align, 18, 18) |
+			FLD_VAL(onoff, 17, 17) |
+			FLD_VAL(rf, 16, 16) |
+			FLD_VAL(ieo, 15, 15) |
+			FLD_VAL(ipc, 14, 14) |
+			FLD_VAL(ihs, 13, 13) |
+			FLD_VAL(ivs, 12, 12));
+
+	dispc6_vp_write(dispc, hw_videoport, DISPC_VP_SIZE_SCREEN,
+			FLD_VAL(mode->hdisplay - 1, 11, 0) |
+			FLD_VAL(mode->vdisplay - 1, 27, 16));
+
+	VP_REG_FLD_MOD(dispc, hw_videoport, DISPC_VP_CONTROL, 1, 0, 0);
+}
+
+static void dispc6_vp_disable(struct dispc_device *dispc, u32 hw_videoport)
+{
+	VP_REG_FLD_MOD(dispc, hw_videoport, DISPC_VP_CONTROL, 0, 0, 0);
+}
+
+static bool dispc6_vp_go_busy(struct dispc_device *dispc,
+			      u32 hw_videoport)
+{
+	return VP_REG_GET(dispc, hw_videoport, DISPC_VP_CONTROL, 5, 5);
+}
+
+static void dispc6_vp_go(struct dispc_device *dispc,
+			 u32 hw_videoport)
+{
+	VP_REG_FLD_MOD(dispc, hw_videoport, DISPC_VP_CONTROL, 1, 5, 5);
+}
+
+static u16 c8_to_c12(u8 c8)
+{
+	u16 c12;
+
+	c12 = c8 << 4;
+
+	/* Replication logic: Copy c8 4 MSB to 4 LSB for full scale c12 */
+	c12 = c8 >> 4;
+
+	return c12;
+}
+
+static u64 argb8888_to_argb12121212(u32 argb8888)
+{
+	u8 a, r, g, b;
+	u64 v;
+
+	a = (argb8888 >> 24) & 0xff;
+	r = (argb8888 >> 16) & 0xff;
+	g = (argb8888 >> 8) & 0xff;
+	b = (argb8888 >> 0) & 0xff;
+
+	v = ((u64)c8_to_c12(a) << 36) | ((u64)c8_to_c12(r) << 24) |
+	    ((u64)c8_to_c12(g) << 12) | (u64)c8_to_c12(b);
+
+	return v;
+}
+
+static void dispc6_vp_set_default_color(struct dispc_device *dispc,
+					u32 hw_videoport, u32 default_color)
+{
+	u64 v;
+
+	v = argb8888_to_argb12121212(default_color);
+
+	dispc6_ovr_write(dispc, 0, DISPC_OVR_DEFAULT_COLOR, v & 0xffffffff);
+	dispc6_ovr_write(dispc, 0, DISPC_OVR_DEFAULT_COLOR2,
+			 (v >> 32) & 0xffff);
+}
+
+static enum drm_mode_status dispc6_vp_mode_valid(struct dispc_device *dispc,
+						 u32 hw_videoport,
+						 const struct drm_display_mode *mode)
+{
+	u32 hsw, hfp, hbp, vsw, vfp, vbp;
+
+	/* special case for 9MHz */
+	if (mode->clock * 1000 < dispc->feat->min_pclk && mode->clock != 9000)
+		return MODE_CLOCK_LOW;
+
+	if (mode->clock * 1000 > dispc->feat->max_pclk)
+		return MODE_CLOCK_HIGH;
+
+	if (mode->hdisplay > 4096)
+		return MODE_BAD;
+
+	if (mode->vdisplay > 4096)
+		return MODE_BAD;
+
+	/* TODO: add interlace support */
+	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
+		return MODE_NO_INTERLACE;
+
+	hfp = mode->hsync_start - mode->hdisplay;
+	hsw = mode->hsync_end - mode->hsync_start;
+	hbp = mode->htotal - mode->hsync_end;
+
+	vfp = mode->vsync_start - mode->vdisplay;
+	vsw = mode->vsync_end - mode->vsync_start;
+	vbp = mode->vtotal - mode->vsync_end;
+
+	if (hsw < 1 || hsw > 256 ||
+	    hfp < 1 || hfp > 4096 ||
+	    hbp < 1 || hbp > 4096)
+		return MODE_BAD_HVALUE;
+
+	if (vsw < 1 || vsw > 256 ||
+	    vfp > 4095 || vbp > 4095)
+		return MODE_BAD_VVALUE;
+
+	if (dispc->memory_bandwidth_limit) {
+		const unsigned int bpp = 4;
+		u64 bandwidth;
+
+		bandwidth = 1000 * mode->clock;
+		bandwidth = bandwidth * mode->hdisplay * mode->vdisplay * bpp;
+		bandwidth = div_u64(bandwidth, mode->htotal * mode->vtotal);
+
+		if (dispc->memory_bandwidth_limit < bandwidth)
+			return MODE_BAD;
+	}
+
+	return MODE_OK;
+}
+
+static int dispc6_vp_check(struct dispc_device *dispc, u32 hw_videoport,
+			   const struct drm_crtc_state *state)
+{
+	const struct drm_display_mode *mode = &state->adjusted_mode;
+	const struct tidss_crtc_state *tstate = to_tidss_crtc_state(state);
+	enum drm_mode_status ok;
+	unsigned int i;
+
+	ok = dispc6_vp_mode_valid(dispc, hw_videoport, mode);
+	if (ok != MODE_OK) {
+		dev_dbg(dispc->dev, "%s: bad mode: %ux%u pclk %u kHz\n",
+			__func__, mode->hdisplay, mode->vdisplay, mode->clock);
+		return -EINVAL;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(dispc6_bus_formats); ++i) {
+		if (dispc6_bus_formats[i].fmt == tstate->bus_format)
+			break;
+	}
+
+	if (i == ARRAY_SIZE(dispc6_bus_formats)) {
+		dev_dbg(dispc->dev, "%s: Unsupported bus format: %u\n",
+			__func__, tstate->bus_format);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int dispc6_vp_enable_clk(struct dispc_device *dispc, u32 hw_videoport)
+{
+	int ret = clk_prepare_enable(dispc->vp_clk);
+
+	if (ret)
+		dev_err(dispc->dev, "%s: enabling clk failed: %d\n", __func__,
+			ret);
+
+	return ret;
+}
+
+static void dispc6_vp_disable_clk(struct dispc_device *dispc,
+				  u32 hw_videoport)
+{
+	clk_disable_unprepare(dispc->vp_clk);
+}
+
+static int dispc6_vp_set_clk_rate(struct dispc_device *dispc,
+				  u32 hw_videoport, unsigned long rate)
+{
+	int r;
+	unsigned long new_rate;
+
+	r = clk_set_rate(dispc->vp_clk, rate);
+	if (r) {
+		dev_err(dispc->dev, "Failed to set vp clk rate to %lu\n",
+			rate);
+		return r;
+	}
+
+	new_rate = clk_get_rate(dispc->vp_clk);
+
+	if (rate != new_rate)
+		dev_warn(dispc->dev,
+			 "Failed to get exact pix clock %lu != %lu\n",
+			 rate, new_rate);
+
+	dev_dbg(dispc->dev, "New VP rate %lu Hz (requested %lu Hz)\n",
+		clk_get_rate(dispc->vp_clk), rate);
+
+	return 0;
+}
+
+/* CSC */
+
+struct color_conv_coef {
+	int ry, rcb, rcr;
+	int gy, gcb, gcr;
+	int by, bcb, bcr;
+	int roffset, goffset, boffset;
+	bool full_range;
+};
+
+static void dispc6_vid_write_color_conv_coefs(struct dispc_device *dispc,
+					      u32 hw_plane,
+					      const struct color_conv_coef *ct)
+{
+#define CVAL(x, y) (FLD_VAL(x, 26, 16) | FLD_VAL(y, 10, 0))
+
+	dispc6_vid_write(dispc, hw_plane,
+			 DISPC_VID_CONV_COEF(0), CVAL(ct->rcr, ct->ry));
+	dispc6_vid_write(dispc, hw_plane,
+			 DISPC_VID_CONV_COEF(1), CVAL(ct->gy,  ct->rcb));
+	dispc6_vid_write(dispc, hw_plane,
+			 DISPC_VID_CONV_COEF(2), CVAL(ct->gcb, ct->gcr));
+	dispc6_vid_write(dispc, hw_plane,
+			 DISPC_VID_CONV_COEF(3), CVAL(ct->bcr, ct->by));
+	dispc6_vid_write(dispc, hw_plane,
+			 DISPC_VID_CONV_COEF(4), CVAL(0, ct->bcb));
+
+	dispc6_vid_write(dispc, hw_plane, DISPC_VID_CONV_COEF(5),
+			 FLD_VAL(ct->roffset, 15, 3) |
+			 FLD_VAL(ct->goffset, 31, 19));
+	dispc6_vid_write(dispc, hw_plane, DISPC_VID_CONV_COEF(6),
+			 FLD_VAL(ct->boffset, 15, 3));
+
+	VID_REG_FLD_MOD(dispc, hw_plane, DISPC_VID_ATTRIBUTES,
+			ct->full_range, 11, 11);
+
+#undef CVAL
+}
+
+static void dispc6_vid_csc_setup(struct dispc_device *dispc)
+{
+	/* YUV -> RGB, ITU-R BT.601, full range */
+	const struct color_conv_coef coefs_yuv2rgb_bt601_full = {
+		256,   0,  358,
+		256, -88, -182,
+		256, 452,    0,
+		0, -2048, -2048,
+		true,
+	};
+
+	dispc6_vid_write_color_conv_coefs(dispc, 0, &coefs_yuv2rgb_bt601_full);
+}
+
+static void dispc6_vid_csc_enable(struct dispc_device *dispc,
+				  u32 hw_plane, bool enable)
+{
+	VID_REG_FLD_MOD(dispc, hw_plane, DISPC_VID_ATTRIBUTES, !!enable, 9, 9);
+}
+
+/* SCALER */
+
+static u32 dispc6_calc_fir_inc(u32 in, u32 out)
+{
+	return (u32)div_u64(0x200000ull * in, out);
+}
+
+struct dispc6_vid_fir_coefs {
+	s16 c2[16];
+	s16 c1[16];
+	u16 c0[9];
+};
+
+static const struct dispc6_vid_fir_coefs dispc6_fir_coefs_null = {
+	.c2 = {	0 },
+	.c1 = { 0 },
+	.c0 = { 512, 512, 512, 512, 512, 512, 512, 512, 256,  },
+};
+
+enum dispc6_vid_fir_coef_set {
+	DISPC6_VID_FIR_COEF_HORIZ,
+	DISPC6_VID_FIR_COEF_HORIZ_UV,
+	DISPC6_VID_FIR_COEF_VERT,
+	DISPC6_VID_FIR_COEF_VERT_UV,
+};
+
+static void dispc6_vid_write_fir_coefs(struct dispc_device *dispc,
+				       u32 hw_plane,
+				       enum dispc6_vid_fir_coef_set coef_set,
+				       const struct dispc6_vid_fir_coefs *coefs)
+{
+	static const u16 c0_regs[] = {
+		[DISPC6_VID_FIR_COEF_HORIZ] = DISPC_VID_FIR_COEFS_H0,
+		[DISPC6_VID_FIR_COEF_HORIZ_UV] = DISPC_VID_FIR_COEFS_H0_C,
+		[DISPC6_VID_FIR_COEF_VERT] = DISPC_VID_FIR_COEFS_V0,
+		[DISPC6_VID_FIR_COEF_VERT_UV] = DISPC_VID_FIR_COEFS_V0_C,
+	};
+
+	static const u16 c12_regs[] = {
+		[DISPC6_VID_FIR_COEF_HORIZ] = DISPC_VID_FIR_COEFS_H12,
+		[DISPC6_VID_FIR_COEF_HORIZ_UV] = DISPC_VID_FIR_COEFS_H12_C,
+		[DISPC6_VID_FIR_COEF_VERT] = DISPC_VID_FIR_COEFS_V12,
+		[DISPC6_VID_FIR_COEF_VERT_UV] = DISPC_VID_FIR_COEFS_V12_C,
+	};
+
+	const u16 c0_base = c0_regs[coef_set];
+	const u16 c12_base = c12_regs[coef_set];
+	int phase;
+
+	for (phase = 0; phase <= 8; ++phase) {
+		u16 reg = c0_base + phase * 4;
+		u16 c0 = coefs->c0[phase];
+
+		dispc6_vid_write(dispc, hw_plane, reg, c0);
+	}
+
+	for (phase = 0; phase <= 15; ++phase) {
+		u16 reg = c12_base + phase * 4;
+		s16 c1, c2;
+		u32 c12;
+
+		c1 = coefs->c1[phase];
+		c2 = coefs->c2[phase];
+		c12 = FLD_VAL(c1, 19, 10) | FLD_VAL(c2, 29, 20);
+
+		dispc6_vid_write(dispc, hw_plane, reg, c12);
+	}
+}
+
+static void dispc6_vid_write_scale_coefs(struct dispc_device *dispc,
+					 u32 hw_plane)
+{
+	dispc6_vid_write_fir_coefs(dispc, hw_plane, DISPC6_VID_FIR_COEF_HORIZ,
+				   &dispc6_fir_coefs_null);
+	dispc6_vid_write_fir_coefs(dispc, hw_plane, DISPC6_VID_FIR_COEF_HORIZ_UV,
+				   &dispc6_fir_coefs_null);
+	dispc6_vid_write_fir_coefs(dispc, hw_plane, DISPC6_VID_FIR_COEF_VERT,
+				   &dispc6_fir_coefs_null);
+	dispc6_vid_write_fir_coefs(dispc, hw_plane, DISPC6_VID_FIR_COEF_VERT_UV,
+				   &dispc6_fir_coefs_null);
+}
+
+static void dispc6_vid_set_scaling(struct dispc_device *dispc,
+				   u32 hw_plane,
+				   u32 orig_width, u32 orig_height,
+				   u32 out_width, u32 out_height,
+				   u32 fourcc)
+{
+	u32 in_w, in_h, in_w_uv, in_h_uv;
+	u32 fir_hinc, fir_vinc, fir_hinc_uv, fir_vinc_uv;
+	bool scale_x, scale_y;
+	bool five_taps = false;		/* XXX always 3-tap for now */
+
+	in_w = in_w_uv = orig_width;
+	in_h = in_h_uv = orig_height;
+
+	switch (fourcc) {
+	case DRM_FORMAT_NV12:
+		/* UV is subsampled by 2 horizontally and vertically */
+		in_h_uv >>= 1;
+		in_w_uv >>= 1;
+		break;
+
+	case DRM_FORMAT_YUYV:
+	case DRM_FORMAT_UYVY:
+		/* UV is subsampled by 2 horizontally */
+		in_w_uv >>= 1;
+		break;
+
+	default:
+		break;
+	}
+
+	scale_x = in_w != out_width || in_w_uv != out_width;
+	scale_y = in_h != out_height || in_h_uv != out_height;
+
+	/* HORIZONTAL RESIZE ENABLE */
+	VID_REG_FLD_MOD(dispc, hw_plane, DISPC_VID_ATTRIBUTES, scale_x, 7, 7);
+
+	/* VERTICAL RESIZE ENABLE */
+	VID_REG_FLD_MOD(dispc, hw_plane, DISPC_VID_ATTRIBUTES, scale_y, 8, 8);
+
+	/* Skip the rest if no scaling is used */
+	if (!scale_x && !scale_y)
+		return;
+
+	/* VERTICAL 5-TAPS  */
+	VID_REG_FLD_MOD(dispc, hw_plane, DISPC_VID_ATTRIBUTES, five_taps, 21, 21);
+
+	/* FIR INC */
+
+	fir_hinc = dispc6_calc_fir_inc(in_w, out_width);
+	fir_vinc = dispc6_calc_fir_inc(in_h, out_height);
+	fir_hinc_uv = dispc6_calc_fir_inc(in_w_uv, out_width);
+	fir_vinc_uv = dispc6_calc_fir_inc(in_h_uv, out_height);
+
+	dispc6_vid_write(dispc, hw_plane, DISPC_VID_FIRH, fir_hinc);
+	dispc6_vid_write(dispc, hw_plane, DISPC_VID_FIRV, fir_vinc);
+	dispc6_vid_write(dispc, hw_plane, DISPC_VID_FIRH2, fir_hinc_uv);
+	dispc6_vid_write(dispc, hw_plane, DISPC_VID_FIRV2, fir_vinc_uv);
+
+	dispc6_vid_write_scale_coefs(dispc, hw_plane);
+}
+
+/* OTHER */
+
+static const struct {
+	u32 fourcc;
+	u8 dss_code;
+} dispc6_color_formats[] = {
+	{ DRM_FORMAT_ARGB4444, 0x0, },
+	{ DRM_FORMAT_ABGR4444, 0x1, },
+	{ DRM_FORMAT_RGBA4444, 0x2, },
+
+	{ DRM_FORMAT_RGB565, 0x3, },
+	{ DRM_FORMAT_BGR565, 0x4, },
+
+	{ DRM_FORMAT_ARGB1555, 0x5, },
+	{ DRM_FORMAT_ABGR1555, 0x6, },
+
+	{ DRM_FORMAT_ARGB8888, 0x7, },
+	{ DRM_FORMAT_ABGR8888, 0x8, },
+	{ DRM_FORMAT_RGBA8888, 0x9, },
+	{ DRM_FORMAT_BGRA8888, 0xa, },
+
+	{ DRM_FORMAT_XRGB8888, 0x27, },
+	{ DRM_FORMAT_XBGR8888, 0x28, },
+	{ DRM_FORMAT_RGBX8888, 0x29, },
+	{ DRM_FORMAT_BGRX8888, 0x2a, },
+
+	{ DRM_FORMAT_YUYV, 0x3e, },
+	{ DRM_FORMAT_UYVY, 0x3f, },
+
+	{ DRM_FORMAT_NV12, 0x3d, },
+};
+
+static bool dispc6_fourcc_is_yuv(u32 fourcc)
+{
+	switch (fourcc) {
+	case DRM_FORMAT_YUYV:
+	case DRM_FORMAT_UYVY:
+	case DRM_FORMAT_NV12:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static void dispc6_plane_set_pixel_format(struct dispc_device *dispc,
+					  u32 hw_plane, u32 fourcc)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(dispc6_color_formats); ++i) {
+		if (dispc6_color_formats[i].fourcc == fourcc) {
+			VID_REG_FLD_MOD(dispc, hw_plane, DISPC_VID_ATTRIBUTES,
+					dispc6_color_formats[i].dss_code,
+					6, 1);
+			return;
+		}
+	}
+
+	WARN_ON(1);
+}
+
+static s32 pixinc(int pixels, u8 ps)
+{
+	if (pixels == 1)
+		return 1;
+	else if (pixels > 1)
+		return 1 + (pixels - 1) * ps;
+	else if (pixels < 0)
+		return 1 - (-pixels + 1) * ps;
+
+	WARN_ON(1);
+	return 0;
+}
+
+static
+const struct tidss_plane_feat *dispc6_plane_feat(struct dispc_device *dispc,
+						 u32 hw_plane)
+{
+	static const struct tidss_plane_feat pfeat = {
+		.color = {
+			.encodings = BIT(DRM_COLOR_YCBCR_BT601),
+			.ranges = BIT(DRM_COLOR_YCBCR_FULL_RANGE),
+			.default_encoding = DRM_COLOR_YCBCR_BT601,
+			.default_range = DRM_COLOR_YCBCR_FULL_RANGE,
+		},
+		.blend = {
+			.global_alpha = false,
+		},
+	};
+
+	return &pfeat;
+}
+
+static int dispc6_plane_check(struct dispc_device *dispc, u32 hw_plane,
+			      const struct drm_plane_state *state,
+			      u32 hw_videoport)
+{
+	return 0; /* XXX: Dummy check function to be implemented later */
+}
+
+static int dispc6_plane_setup(struct dispc_device *dispc, u32 hw_plane,
+			      const struct drm_plane_state *state,
+			      u32 hw_videoport)
+{
+	u32 fourcc = state->fb->format->format;
+	dma_addr_t paddr = dispc7_plane_state_paddr(state);
+	u16 cpp = state->fb->format->cpp[0];
+	u32 fb_width = state->fb->pitches[0] / cpp;
+	u32 src_w = state->src_w >> 16;
+	u32 src_h = state->src_h >> 16;
+
+	dispc6_plane_set_pixel_format(dispc, hw_plane, fourcc);
+
+	dispc6_vid_write(dispc, hw_plane, DISPC_VID_BA_0, paddr);
+	dispc6_vid_write(dispc, hw_plane, DISPC_VID_BA_1, paddr);
+
+	if (state->fb->format->num_planes == 2) {
+		dma_addr_t p_uv_addr = dispc7_plane_state_p_uv_addr(state);
+
+		dispc6_vid_write(dispc, hw_plane, DISPC_VID_BA_UV_0, p_uv_addr);
+		dispc6_vid_write(dispc, hw_plane, DISPC_VID_BA_UV_1, p_uv_addr);
+	}
+
+	dispc6_vid_write(dispc, hw_plane, DISPC_VID_PICTURE_SIZE,
+			 (src_w - 1) | ((src_h - 1) << 16));
+
+	dispc6_vid_write(dispc, hw_plane, DISPC_VID_PIXEL_INC,
+			 pixinc(1, cpp));
+	dispc6_vid_write(dispc, hw_plane, DISPC_VID_ROW_INC,
+			 pixinc(1 + fb_width - src_w, cpp));
+
+	dispc6_vid_write(dispc, hw_plane, DISPC_VID_POSITION,
+			 state->crtc_x | (state->crtc_y << 16));
+
+	dispc6_vid_write(dispc, hw_plane, DISPC_VID_SIZE,
+			 (state->crtc_w - 1) | ((state->crtc_h - 1) << 16));
+
+	dispc6_vid_set_scaling(dispc, hw_plane, src_w, src_h,
+			       state->crtc_w, state->crtc_h,
+			       fourcc);
+
+	/* enable YUV->RGB color conversion */
+	if (dispc6_fourcc_is_yuv(fourcc))
+		dispc6_vid_csc_enable(dispc, hw_plane, true);
+	else
+		dispc6_vid_csc_enable(dispc, hw_plane, false);
+
+	/* hw_videoport */
+	VID_REG_FLD_MOD(dispc, hw_plane, DISPC_VID_ATTRIBUTES, 0, 16, 14);
+
+	return 0;
+}
+
+static int dispc6_plane_enable(struct dispc_device *dispc,
+			       u32 hw_plane, bool enable)
+{
+	VID_REG_FLD_MOD(dispc, hw_plane, DISPC_VID_ATTRIBUTES, !!enable, 0, 0);
+	return 0;
+}
+
+static bool dispc6_has_writeback(struct dispc_device *dispc)
+{
+	return false;
+}
+
+static u32 dispc6_vid_get_fifo_size(struct dispc_device *dispc,
+				    u32 hw_plane)
+{
+	const u32 unit_size = 16;	/* 128-bits */
+
+	return VID_REG_GET(dispc, hw_plane, DISPC_VID_BUF_SIZE_STATUS, 15, 0) *
+	       unit_size;
+}
+
+static void dispc6_vid_set_mflag_threshold(struct dispc_device *dispc,
+					   u32 hw_plane,
+					   u32 low, u32 high)
+{
+	dispc6_vid_write(dispc, hw_plane, DISPC_VID_MFLAG_THRESHOLD,
+			 FLD_VAL(high, 31, 16) | FLD_VAL(low, 15, 0));
+}
+
+static void dispc6_mflag_setup(struct dispc_device *dispc)
+{
+	u32 hw_plane = 0;
+	const u32 unit_size = 16;	/* 128-bits */
+	u32 size = dispc6_vid_get_fifo_size(dispc, hw_plane);
+	u32 low, high;
+
+	/* MFLAG_CTRL */
+	REG_FLD_MOD(dispc, DISPC_GLOBAL_MFLAG_ATTRIBUTE, 1, 1, 0);
+	/* MFLAG_START */
+	REG_FLD_MOD(dispc, DISPC_GLOBAL_MFLAG_ATTRIBUTE, 0, 2, 2);
+
+	/*
+	 * Simulation team suggests below thesholds:
+	 * HT = fifosize * 5 / 8;
+	 * LT = fifosize * 4 / 8;
+	 */
+
+	low = size * 4 / 8 / unit_size;
+	high = size * 5 / 8 / unit_size;
+
+	dispc6_vid_set_mflag_threshold(dispc, hw_plane, low, high);
+}
+
+static void dispc6_initial_config(struct dispc_device *dispc)
+{
+	dispc6_vid_csc_setup(dispc);
+	dispc6_mflag_setup(dispc);
+
+	/* Enable the gamma Shadow bit-field */
+	VP_REG_FLD_MOD(dispc, 0, DISPC_VP_CONFIG, 1, 2, 2);
+}
+
+static int dispc6_init_features(struct dispc_device *dispc)
+{
+	const struct of_device_id *match;
+
+	match = of_match_node(dispc6_of_match, dispc->dev->of_node);
+	if (!match) {
+		dev_err(dispc->dev, "Unsupported DISPC version\n");
+		return -ENODEV;
+	}
+
+	dispc->feat = match->data;
+
+	return 0;
+}
+
+static int dispc6_get_num_planes(struct dispc_device *dispc)
+{
+	return 1;
+}
+
+static int dispc6_get_num_vps(struct dispc_device *dispc)
+{
+	return 1;
+}
+
+static const struct tidss_vp_feat *dispc6_vp_feat(struct dispc_device *dispc,
+						  u32 hw_videoport)
+{
+	static const struct tidss_vp_feat vp_feat = {
+		.color = {
+			.gamma_size = DISPC6_GAMMA_TABLE_SIZE,
+			.gamma_type = TIDSS_GAMMA_8BIT,
+			.has_ctm = false, /* Driver implementation missing */
+		},
+	};
+
+	return &vp_feat;
+}
+
+static void dispc6_vp_write_gamma_table(struct dispc_device *dispc,
+					u32 hw_videoport)
+{
+	u32 *table = dispc->gamma_table;
+	unsigned int hwlen = ARRAY_SIZE(dispc->gamma_table);
+	unsigned int i;
+
+	dev_dbg(dispc->dev, "%s: hw_videoport %d\n", __func__, hw_videoport);
+
+	for (i = 0; i < hwlen; ++i) {
+		u32 v = table[i];
+
+		v |= i << 24;
+
+		dispc6_vp_write(dispc, hw_videoport, DISPC_VP_GAMMA_TABLE, v);
+	}
+}
+
+static void dispc6_restore_gamma_tables(struct dispc_device *dispc)
+{
+	dev_dbg(dispc->dev, "%s()\n", __func__);
+
+	dispc6_vp_write_gamma_table(dispc, 0);
+}
+
+static const struct drm_color_lut dispc6_vp_gamma_default_lut[] = {
+	{ .red = 0, .green = 0, .blue = 0, },
+	{ .red = U16_MAX, .green = U16_MAX, .blue = U16_MAX, },
+};
+
+static void dispc6_vp_set_gamma(struct dispc_device *dispc,
+				u32 hw_videoport,
+				const struct drm_color_lut *lut,
+				unsigned int length)
+{
+	u32 *table = dispc->gamma_table;
+	unsigned int hwlen = ARRAY_SIZE(dispc->gamma_table);
+	static const unsigned int hwbits = 8;
+	unsigned int i;
+
+	dev_dbg(dispc->dev, "%s: hw_videoport %d, lut len %u, hw len %u\n",
+		__func__, hw_videoport, length, hwlen);
+
+	if (lut == NULL || length < 2) {
+		lut = dispc6_vp_gamma_default_lut;
+		length = ARRAY_SIZE(dispc6_vp_gamma_default_lut);
+	}
+
+	for (i = 0; i < length - 1; ++i) {
+		unsigned int first = i * (hwlen - 1) / (length - 1);
+		unsigned int last = (i + 1) * (hwlen - 1) / (length - 1);
+		unsigned int w = last - first;
+		u16 r, g, b;
+		unsigned int j;
+
+		if (w == 0)
+			continue;
+
+		for (j = 0; j <= w; j++) {
+			r = (lut[i].red * (w - j) + lut[i + 1].red * j) / w;
+			g = (lut[i].green * (w - j) + lut[i + 1].green * j) / w;
+			b = (lut[i].blue * (w - j) + lut[i + 1].blue * j) / w;
+
+			r >>= 16 - hwbits;
+			g >>= 16 - hwbits;
+			b >>= 16 - hwbits;
+
+			table[first + j] = (r << (hwbits * 2)) |
+					   (g << hwbits) | b;
+		}
+	}
+
+	if (dispc->is_enabled)
+		dispc6_vp_write_gamma_table(dispc, hw_videoport);
+}
+
+static void dispc6_vp_set_color_mgmt(struct dispc_device *dispc,
+				     u32 hw_videoport,
+				     const struct drm_crtc_state *state)
+{
+	struct drm_color_lut *lut = NULL;
+	unsigned int length = 0;
+
+	if (!state->color_mgmt_changed)
+		return;
+
+	if (state->gamma_lut) {
+		lut = (struct drm_color_lut *) state->gamma_lut->data;
+		length = state->gamma_lut->length / sizeof(*lut);
+	}
+
+	dispc6_vp_set_gamma(dispc, hw_videoport, lut, length);
+}
+
+static void dispc6_vp_setup(struct dispc_device *dispc, u32 hw_videoport,
+			    const struct drm_crtc_state *state)
+{
+	dispc6_vp_set_default_color(dispc, hw_videoport, 0);
+	dispc6_vp_set_color_mgmt(dispc, hw_videoport, state);
+}
+
+static int dispc6_init_gamma_tables(struct dispc_device *dispc)
+{
+	dispc6_vp_set_gamma(dispc, 0, NULL, 0);
+
+	return 0;
+}
+
+static const char *dispc6_plane_name(struct dispc_device *dispc,
+				     u32 hw_plane)
+{
+	return "vid1";
+}
+
+static const char *dispc6_vp_name(struct dispc_device *dispc,
+				  u32 hw_videoport)
+{
+	return "vp1";
+}
+
+static int dispc6_runtime_suspend(struct dispc_device *dispc)
+{
+	struct device *dev = dispc->dev;
+
+	dev_dbg(dev, "suspend\n");
+
+	dispc->is_enabled = false;
+
+	clk_disable_unprepare(dispc->fclk);
+
+	return 0;
+}
+
+static int dispc6_runtime_resume(struct dispc_device *dispc)
+{
+	struct device *dev = dispc->dev;
+
+	dev_dbg(dev, "resume\n");
+
+	clk_prepare_enable(dispc->fclk);
+
+	if (REG_GET(dispc, DISPC_SYSSTATUS, 0, 0) == 0)
+		dev_warn(dev, "DISPC FUNC RESET not done!\n");
+	if (REG_GET(dispc, DISPC_SYSSTATUS, 1, 1) == 0)
+		dev_warn(dev, "DISPC VP RESET not done!\n");
+
+	dispc6_initial_config(dispc);
+
+	dispc6_restore_gamma_tables(dispc);
+
+	dispc->is_enabled = true;
+
+	return 0;
+}
+
+static int dispc6_modeset_init(struct dispc_device *dispc)
+{
+	struct tidss_device *tidss = dispc->tidss;
+	struct device *dev = tidss->dev;
+	const u32 hw_videoport = 0;
+	const u32 crtc_mask = 1;
+	const u32 hw_plane_id = 0;
+	struct drm_panel *panel;
+	struct drm_bridge *bridge;
+	u32 enc_type;
+	int ret;
+	struct tidss_plane *tplane;
+	struct tidss_crtc *tcrtc;
+	struct drm_encoder *enc;
+	u32 fourccs[ARRAY_SIZE(dispc6_color_formats)];
+	unsigned int i;
+
+	/* first find if there is a connected panel/bridge */
+
+	ret = drm_of_find_panel_or_bridge(dev->of_node, hw_videoport, 0, &panel, &bridge);
+	if (ret) {
+		dev_dbg(dev, "no panel or bridge found\n");
+		return ret;
+	}
+
+	if (panel) {
+		dev_dbg(dev, "Setting up panel\n");
+
+		enc_type = DRM_MODE_ENCODER_DPI;
+
+		bridge = devm_drm_panel_bridge_add(dev, panel, DRM_MODE_CONNECTOR_DPI);
+		if (IS_ERR(bridge)) {
+			dev_err(dev, "failed to set up panel bridge\n");
+			return PTR_ERR(bridge);
+		}
+	} else {
+		enc_type = DRM_MODE_ENCODER_NONE;
+	}
+
+	/* then create a plane, a crtc and an encoder for the panel/bridge */
+
+	for (i = 0; i < ARRAY_SIZE(dispc6_color_formats); ++i)
+		fourccs[i] = dispc6_color_formats[i].fourcc;
+
+	tplane = tidss_plane_create(tidss, hw_plane_id, DRM_PLANE_TYPE_PRIMARY,
+				    crtc_mask, fourccs, ARRAY_SIZE(fourccs));
+	if (IS_ERR(tplane)) {
+		dev_err(tidss->dev, "plane create failed\n");
+		return PTR_ERR(tplane);
+	}
+
+	tidss->planes[tidss->num_planes++] = &tplane->plane;
+
+	tcrtc = tidss_crtc_create(tidss, hw_videoport, &tplane->plane);
+	if (IS_ERR(tcrtc)) {
+		dev_err(tidss->dev, "crtc create failed\n");
+		return PTR_ERR(tcrtc);
+	}
+
+	tidss->crtcs[tidss->num_crtcs++] = &tcrtc->crtc;
+
+	enc = tidss_encoder_create(tidss, enc_type, 1 << tcrtc->crtc.index);
+	if (IS_ERR(enc)) {
+		dev_err(tidss->dev, "encoder create failed\n");
+		return PTR_ERR(enc);
+	}
+
+	ret = drm_bridge_attach(enc, bridge, NULL);
+	if (ret) {
+		dev_err(tidss->dev, "bridge attach failed: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int dispc6_get_irq(struct dispc_device *dispc)
+{
+	return platform_get_irq(to_platform_device(dispc->tidss->dev), 0);
+}
+
+static void dispc6_remove(struct dispc_device *dispc);
+
+static const struct tidss_dispc_ops dispc6_ops = {
+	.read_and_clear_irqstatus = dispc6_read_and_clear_irqstatus,
+	.write_irqenable = dispc6_write_irqenable,
+
+	.get_num_vps = dispc6_get_num_vps,
+	.vp_name = dispc6_vp_name,
+	.vp_feat = dispc6_vp_feat,
+	.vp_enable = dispc6_vp_enable,
+	.vp_disable = dispc6_vp_disable,
+	.vp_go_busy = dispc6_vp_go_busy,
+	.vp_go = dispc6_vp_go,
+	.vp_mode_valid = dispc6_vp_mode_valid,
+	.vp_check = dispc6_vp_check,
+	.vp_setup = dispc6_vp_setup,
+
+	.vp_set_clk_rate = dispc6_vp_set_clk_rate,
+	.vp_enable_clk = dispc6_vp_enable_clk,
+	.vp_disable_clk = dispc6_vp_disable_clk,
+
+	.get_num_planes = dispc6_get_num_planes,
+	.plane_name = dispc6_plane_name,
+	.plane_feat = dispc6_plane_feat,
+	.plane_enable = dispc6_plane_enable,
+	.plane_check = dispc6_plane_check,
+	.plane_setup = dispc6_plane_setup,
+
+	.runtime_get = dispc6_runtime_get,
+	.runtime_put = dispc6_runtime_put,
+
+	.runtime_suspend = dispc6_runtime_suspend,
+	.runtime_resume = dispc6_runtime_resume,
+
+	.remove = dispc6_remove,
+
+	.modeset_init = dispc6_modeset_init,
+
+	.get_irq = dispc6_get_irq,
+
+	.has_writeback = dispc6_has_writeback,
+};
+
+static int dispc6_iomap_resource(struct platform_device *pdev, const char *name,
+				 void __iomem **base)
+{
+	struct resource *res;
+	void __iomem *b;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, name);
+	b = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(b)) {
+		dev_err(&pdev->dev, "cannot ioremap resource '%s'\n", name);
+		return PTR_ERR(b);
+	}
+
+	*base = b;
+
+	return 0;
+}
+
+int dispc6_init(struct tidss_device *tidss)
+{
+	struct device *dev = tidss->dev;
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dispc_device *dispc;
+	int r;
+
+	dev_dbg(dev, "%s\n", __func__);
+
+	dispc = devm_kzalloc(dev, sizeof(*dispc), GFP_KERNEL);
+	if (!dispc)
+		return -ENOMEM;
+
+	dispc->tidss = tidss;
+	dispc->dev = dev;
+
+	r = dispc6_init_features(dispc);
+	if (r)
+		goto err_free;
+
+	r = dispc6_iomap_resource(pdev, "cfg", &dispc->base_cfg);
+	if (r)
+		goto err_free;
+
+	r = dispc6_iomap_resource(pdev, "common", &dispc->base_common);
+	if (r)
+		goto err_free;
+
+	r = dispc6_iomap_resource(pdev, "vid1", &dispc->base_vid1);
+	if (r)
+		goto err_free;
+
+	r = dispc6_iomap_resource(pdev, "ovr1", &dispc->base_ovr1);
+	if (r)
+		goto err_free;
+
+	r = dispc6_iomap_resource(pdev, "vp1", &dispc->base_vp1);
+	if (r)
+		goto err_free;
+
+	dev_dbg(dev, "dispc6_bind: iores ok\n");
+
+	dispc->fclk = devm_clk_get(dev, "fck");
+	if (IS_ERR(dispc->fclk)) {
+		dev_err(dev, "Failed to get fclk\n");
+		r = PTR_ERR(dispc->fclk);
+		goto err_free;
+	}
+
+	dispc->vp_clk = devm_clk_get(dev, "vp1");
+	if (IS_ERR(dispc->vp_clk)) {
+		dev_err(dev, "Failed to get vp1 clk\n");
+		r = PTR_ERR(dispc->vp_clk);
+		goto err_free;
+	}
+
+	of_property_read_u32(dispc->dev->of_node, "max-memory-bandwidth",
+			     &dispc->memory_bandwidth_limit);
+
+	r = dispc6_init_gamma_tables(dispc);
+	if (r)
+		goto err_free;
+
+	tidss->dispc_ops = &dispc6_ops;
+	tidss->dispc = dispc;
+
+	dev_dbg(dev, "%s done\n", __func__);
+
+	return 0;
+err_free:
+	dev_err(dev, "%s failed: %d\n", __func__, r);
+	return r;
+}
+
+static void dispc6_remove(struct dispc_device *dispc)
+{
+	struct device *dev = dispc->dev;
+
+	dev_dbg(dev, "dispc6_unbind\n");
+
+	dispc->tidss->dispc_ops = NULL;
+	dispc->tidss->dispc = NULL;
+
+	dev_dbg(dev, "dispc6_unbind done\n");
+}
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/tidss_dispc6.h b/drivers/gpu/drm/tidss/tidss_dispc6.h
--- a/drivers/gpu/drm/tidss/tidss_dispc6.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/tidss_dispc6.h	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,109 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ */
+
+#ifndef __TIDSS_DISPC6_H__
+#define __TIDSS_DISPC6_H__
+
+/* COMMON */
+
+#define DISPC_REVISION			0x000
+#define DISPC_SYSCONFIG			0x004
+#define DISPC_SYSSTATUS			0x008
+
+#define DISPC_IRQ_EOI			0x020
+#define DISPC_IRQSTATUS_RAW		0x024
+#define DISPC_IRQSTATUS			0x028
+#define DISPC_IRQENABLE_SET		0x02c
+#define DISPC_IRQENABLE_CLR		0x030
+#define DISPC_IRQWAKEEN			0x034
+
+#define DISPC_GLOBAL_MFLAG_ATTRIBUTE	0x040
+#define DISPC_GLOBAL_BUFFER		0x044
+#define DISPC_BA0_FLIPIMMEDIATE_EN	0x048
+
+#define DISPC_DBG_CONTROL		0x04c
+#define DISPC_DBG_STATUS		0x050
+
+#define DISPC_CLKGATING_DISABLE		0x054
+
+/* VID */
+
+#define DISPC_VID_ACCUH_0		0x0
+#define DISPC_VID_ACCUH_1		0x4
+#define DISPC_VID_ACCUH2_0		0x8
+#define DISPC_VID_ACCUH2_1		0xc
+
+#define DISPC_VID_ACCUV_0		0x10
+#define DISPC_VID_ACCUV_1		0x14
+#define DISPC_VID_ACCUV2_0		0x18
+#define DISPC_VID_ACCUV2_1		0x1c
+
+#define DISPC_VID_ATTRIBUTES		0x20
+#define DISPC_VID_ATTRIBUTES2		0x24
+
+#define DISPC_VID_BA_0			0x28
+#define DISPC_VID_BA_1			0x2c
+#define DISPC_VID_BA_UV_0		0x30
+#define DISPC_VID_BA_UV_1		0x34
+#define DISPC_VID_BUF_SIZE_STATUS	0x38
+#define DISPC_VID_BUF_THRESHOLD		0x3c
+
+#define DISPC_VID_CONV_COEF(n)		(0x40 + (n) * 4)
+
+#define DISPC_VID_FIRH			0x5c
+#define DISPC_VID_FIRH2			0x60
+#define DISPC_VID_FIRV			0x64
+#define DISPC_VID_FIRV2			0x68
+
+#define DISPC_VID_FIR_COEFS_H0		0x6c
+#define DISPC_VID_FIR_COEF_H0(phase)	(0x6c + (phase) * 4)
+#define DISPC_VID_FIR_COEFS_H0_C	0x90
+#define DISPC_VID_FIR_COEF_H0_C(phase)	(0x90 + (phase) * 4)
+
+#define DISPC_VID_FIR_COEFS_H12		0xb4
+#define DISPC_VID_FIR_COEF_H12(phase)	(0xb4 + (phase) * 4)
+#define DISPC_VID_FIR_COEFS_H12_C	0xf4
+#define DISPC_VID_FIR_COEF_H12_C(phase)	(0xf4 + (phase) * 4)
+
+#define DISPC_VID_FIR_COEFS_V0		0x134
+#define DISPC_VID_FIR_COEF_V0(phase)	(0x134 + (phase) * 4)
+#define DISPC_VID_FIR_COEFS_V0_C	0x158
+#define DISPC_VID_FIR_COEF_V0_C(phase)	(0x158 + (phase) * 4)
+
+#define DISPC_VID_FIR_COEFS_V12		0x17c
+#define DISPC_VID_FIR_COEF_V12(phase)	(0x17c + (phase) * 4)
+#define DISPC_VID_FIR_COEFS_V12_C	0x1bc
+#define DISPC_VID_FIR_COEF_V12_C(phase)	(0x1bc + (phase) * 4)
+
+#define DISPC_VID_IRQENABLE		0x200
+#define DISPC_VID_IRQSTATUS		0x204
+
+#define DISPC_VID_MFLAG_THRESHOLD	0x208
+#define DISPC_VID_PICTURE_SIZE		0x20c
+#define DISPC_VID_PIXEL_INC		0x210
+#define DISPC_VID_POSITION		0x214
+#define DISPC_VID_PRELOAD		0x218
+#define DISPC_VID_ROW_INC		0x21c
+#define DISPC_VID_SIZE			0x220
+
+/* OVR */
+
+#define DISPC_OVR_DEFAULT_COLOR		0x08
+#define DISPC_OVR_DEFAULT_COLOR2	0x0c
+
+/* VP */
+
+#define DISPC_VP_CONFIG			0x00
+#define DISPC_VP_CONTROL		0x04
+#define DISPC_VP_GAMMA_TABLE		0x20
+#define DISPC_VP_IRQENABLE		0x3c
+#define DISPC_VP_IRQSTATUS		0x40
+#define DISPC_VP_POL_FREQ		0x4c
+#define DISPC_VP_SIZE_SCREEN		0x50
+#define DISPC_VP_TIMING_H		0x54
+#define DISPC_VP_TIMING_V		0x58
+
+#endif
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/tidss_dispc7.c b/drivers/gpu/drm/tidss/tidss_dispc7.c
--- a/drivers/gpu/drm/tidss/tidss_dispc7.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/tidss_dispc7.c	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,3581 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2016-2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Jyri Sarha <jsarha@ti.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/err.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mfd/syscon.h>
+#include <linux/of.h>
+#include <linux/of_graph.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/regmap.h>
+
+#include <drm/drm_fourcc.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_of.h>
+
+#include "tidss_drv.h"
+#include "tidss_crtc.h"
+#include "tidss_plane.h"
+#include "tidss_encoder.h"
+
+#include "tidss_dispc.h"
+#include "tidss_scale_coefs.h"
+#include "tidss_dispc7.h"
+
+static const char *dispc7_plane_name(struct dispc_device *dispc, u32 hw_plane);
+
+static const struct dispc7_features dispc7_am6_feats = {
+	.max_pclk_kHz = {
+		[DISPC7_VP_DPI] = 165000,
+		[DISPC7_VP_OLDI] = 165000,
+	},
+
+	.num_commons = 1,
+	.common_name = { "common" },
+	.common_cfg = { true },
+
+	.scaling = {
+		.in_width_max_5tap_rgb = 1280,
+		.in_width_max_3tap_rgb = 2560,
+		.in_width_max_5tap_yuv = 2560,
+		.in_width_max_3tap_yuv = 4096,
+		.upscale_limit = 16,
+		.downscale_limit_5tap = 4,
+		.downscale_limit_3tap = 2,
+		/*
+		 * The max supported pixel inc value is 255. The value
+		 * of pixel inc is calculated like this: 1+(xinc-1)*bpp.
+		 * The maximum bpp of all formats supported by the HW
+		 * is 8. So the maximum supported xinc value is 32,
+		 * because 1+(32-1)*8 < 255 < 1+(33-1)*4.
+		 */
+		.xinc_max = 32,
+	},
+
+	.subrev = DSS7_AM6,
+
+	.num_vps = 2,
+	.vp_name = { "vp1", "vp2" },
+	.ovr_name = { "ovr1", "ovr2" },
+	.vpclk_name =  { "vp1", "vp2" },
+	.vp_bus_type = { DISPC7_VP_OLDI, DISPC7_VP_DPI },
+
+	.vp_feat = { .color = {
+			.has_ctm = true,
+			.gamma_size = 256,
+			.gamma_type = TIDSS_GAMMA_8BIT,
+		},
+	},
+
+	.num_planes = 2,
+	/* note: vid is plane_id 0 and vidl1 is plane_id 1 */
+	.vid_name = { "vid", "vidl1" },
+	.vid_lite = { false, true, },
+	.vid_order = { 1, 0 },
+
+	.errata = {
+		.i2000 = true,
+	},
+};
+
+static const struct dispc7_features dispc7_j721e_feats = {
+	.max_pclk_kHz = {
+		[DISPC7_VP_DPI] = 170000,
+		[DISPC7_VP_INTERNAL] = 600000,
+	},
+
+	.num_commons = 4,
+	.common_name = { "common_m", "common_s0", "common_s1", "common_s2" },
+	.common_cfg = { true, false, false, false },
+
+	.scaling = {
+		.in_width_max_5tap_rgb = 2048,
+		.in_width_max_3tap_rgb = 4096,
+		.in_width_max_5tap_yuv = 4096,
+		.in_width_max_3tap_yuv = 4096,
+		.upscale_limit = 16,
+		.downscale_limit_5tap = 4,
+		.downscale_limit_3tap = 2,
+		/*
+		 * The max supported pixel inc value is 255. The value
+		 * of pixel inc is calculated like this: 1+(xinc-1)*bpp.
+		 * The maximum bpp of all formats supported by the HW
+		 * is 8. So the maximum supported xinc value is 32,
+		 * because 1+(32-1)*8 < 255 < 1+(33-1)*4.
+		 */
+		.xinc_max = 32,
+	},
+
+	.subrev = DSS7_J721E,
+
+	.num_vps = 4,
+	.vp_name = { "vp1", "vp2", "vp3", "vp4" },
+	.ovr_name = { "ovr1", "ovr2", "ovr3", "ovr4" },
+	.vpclk_name = { "vp1", "vp2", "vp3", "vp4" },
+	/* Currently hard coded VP routing (see dispc7_initial_config()) */
+	.vp_bus_type =	{ DISPC7_VP_INTERNAL, DISPC7_VP_DPI,
+			  DISPC7_VP_INTERNAL, DISPC7_VP_DPI, },
+	.vp_feat = { .color = {
+			.has_ctm = true,
+			.gamma_size = 1024,
+			.gamma_type = TIDSS_GAMMA_10BIT,
+		},
+	},
+	.num_planes = 4,
+	.vid_name = { "vid1", "vidl1", "vid2", "vidl2" },
+	.vid_lite = { 0, 1, 0, 1, },
+	.vid_order = { 1, 3, 0, 2 },
+	.has_writeback = true,
+	.wb_name = "wb",
+};
+
+static const struct of_device_id dispc7_of_table[] = {
+	{ .compatible = "ti,am6-dss", .data = &dispc7_am6_feats },
+	{ .compatible = "ti,j721e-dss", .data = &dispc7_j721e_feats },
+	{ }
+};
+
+/*
+ * TRM gives bitfields as start:end, where start is the higher bit
+ * number. For example 7:0
+ */
+
+#define FLD_MASK(start, end)	(((1 << ((start) - (end) + 1)) - 1) << (end))
+#define FLD_VAL(val, start, end) (((val) << (end)) & FLD_MASK(start, end))
+#define FLD_GET(val, start, end) (((val) & FLD_MASK(start, end)) >> (end))
+#define FLD_MOD(orig, val, start, end) \
+	(((orig) & ~FLD_MASK(start, end)) | FLD_VAL(val, start, end))
+
+#define CFG_REG_GET(dispc, idx, start, end) \
+	FLD_GET(dispc7_cfg_read(dispc, idx), start, end)
+
+#define CFG_REG_FLD_MOD(dispc, idx, val, start, end) \
+	dispc7_cfg_write(dispc, idx, FLD_MOD(dispc7_cfg_read(dispc, idx), val, start, end))
+
+#define VID_REG_GET(dispc, hw_plane, idx, start, end) \
+	FLD_GET(dispc7_vid_read(dispc, hw_plane, idx), start, end)
+
+#define VID_REG_FLD_MOD(dispc, hw_plane, idx, val, start, end) \
+	dispc7_vid_write(dispc, hw_plane, idx, FLD_MOD(dispc7_vid_read(dispc, hw_plane, idx), val, start, end))
+
+#define WB_REG_GET(dispc, idx, start, end) \
+	FLD_GET(dispc7_wb_read(dispc, idx), start, end)
+
+#define WB_REG_FLD_MOD(dispc, idx, val, start, end) \
+	dispc7_wb_write(dispc, idx, FLD_MOD(dispc7_wb_read(dispc, idx), val, start, end))
+
+#define VP_REG_GET(dispc, vp, idx, start, end) \
+	FLD_GET(dispc7_vp_read(dispc, vp, idx), start, end)
+
+#define VP_REG_FLD_MOD(dispc, vp, idx, val, start, end) \
+	dispc7_vp_write(dispc, vp, idx, FLD_MOD(dispc7_vp_read(dispc, vp, idx), val, start, end))
+
+#define OVR_REG_GET(dispc, ovr, idx, start, end) \
+	FLD_GET(dispc7_ovr_read(dispc, ovr, idx), start, end)
+
+#define OVR_REG_FLD_MOD(dispc, ovr, idx, val, start, end) \
+	dispc7_ovr_write(dispc, ovr, idx, FLD_MOD(dispc7_ovr_read(dispc, ovr, idx), val, start, end))
+
+static const u16 tidss_am6_common_regs[DSS7_COMMON_REG_TABLE_LEN] = {
+	[DSS_REVISION_OFF] =			0x4,
+	[DSS_SYSCONFIG_OFF] =			0x8,
+	[DSS_SYSSTATUS_OFF] =			0x20,
+	[DISPC_IRQ_EOI_OFF] =			0x24,
+	[DISPC_IRQSTATUS_RAW_OFF] =		0x28,
+	[DISPC_IRQSTATUS_OFF] =			0x2c,
+	[DISPC_IRQENABLE_SET_OFF] =		0x30,
+	[DISPC_IRQENABLE_CLR_OFF] =		0x40,
+	[DISPC_VID_IRQENABLE_OFF] =		0x44,
+	[DISPC_VID_IRQSTATUS_OFF] =		0x58,
+	[DISPC_VP_IRQENABLE_OFF] =		0x70,
+	[DISPC_VP_IRQSTATUS_OFF] =		0x7c,
+
+	[WB_IRQENABLE_OFF] =			0x88,
+	[WB_IRQSTATUS_OFF] =			0x8c,
+
+	[DISPC_GLOBAL_MFLAG_ATTRIBUTE_OFF] =	0x90,
+	[DISPC_GLOBAL_OUTPUT_ENABLE_OFF] =	0x94,
+	[DISPC_GLOBAL_BUFFER_OFF] =		0x98,
+	[DSS_CBA_CFG_OFF] =			0x9c,
+	[DISPC_DBG_CONTROL_OFF] =		0xa0,
+	[DISPC_DBG_STATUS_OFF] =		0xa4,
+	[DISPC_CLKGATING_DISABLE_OFF] =		0xa8,
+	[DISPC_SECURE_DISABLE_OFF] =		0xac,
+};
+
+static const u16 tidss_j721e_common_regs[DSS7_COMMON_REG_TABLE_LEN] = {
+	[DSS_REVISION_OFF] =			0x4,
+	[DSS_SYSCONFIG_OFF] =			0x8,
+	[DSS_SYSSTATUS_OFF] =			0x20,
+	[DISPC_IRQ_EOI_OFF] =			0x80,
+	[DISPC_IRQSTATUS_RAW_OFF] =		0x28,
+	[DISPC_IRQSTATUS_OFF] =			0x2c,
+	[DISPC_IRQENABLE_SET_OFF] =		0x30,
+	[DISPC_IRQENABLE_CLR_OFF] =		0x34,
+	[DISPC_VID_IRQENABLE_OFF] =		0x38,
+	[DISPC_VID_IRQSTATUS_OFF] =		0x48,
+	[DISPC_VP_IRQENABLE_OFF] =		0x58,
+	[DISPC_VP_IRQSTATUS_OFF] =		0x68,
+
+	[WB_IRQENABLE_OFF] =			0x78,
+	[WB_IRQSTATUS_OFF] =			0x7c,
+
+	[DISPC_GLOBAL_MFLAG_ATTRIBUTE_OFF] =	0x98,
+	[DISPC_GLOBAL_OUTPUT_ENABLE_OFF] =	0x9c,
+	[DISPC_GLOBAL_BUFFER_OFF] =		0xa0,
+	[DSS_CBA_CFG_OFF] =			0xa4,
+	[DISPC_DBG_CONTROL_OFF] =		0xa8,
+	[DISPC_DBG_STATUS_OFF] =		0xac,
+	[DISPC_CLKGATING_DISABLE_OFF] =		0xb0,
+	[DISPC_SECURE_DISABLE_OFF] =		0x90,
+
+	[FBDC_REVISION_1_OFF] =			0xb8,
+	[FBDC_REVISION_2_OFF] =			0xbc,
+	[FBDC_REVISION_3_OFF] =			0xc0,
+	[FBDC_REVISION_4_OFF] =			0xc4,
+	[FBDC_REVISION_5_OFF] =			0xc8,
+	[FBDC_REVISION_6_OFF] =			0xcc,
+	[FBDC_COMMON_CONTROL_OFF] =		0xd0,
+	[FBDC_CONSTANT_COLOR_0_OFF] =		0xd4,
+	[FBDC_CONSTANT_COLOR_1_OFF] =		0xd8,
+	[DISPC_CONNECTIONS_OFF] =		0xe4,
+	[DISPC_MSS_VP1_OFF] =			0xe8,
+	[DISPC_MSS_VP3_OFF] =			0xec,
+};
+
+static const u16 *dispc7_common_regmap;
+
+struct dss_vp_data {
+	u32 *gamma_table;
+};
+
+struct dss_plane_data {
+	u32 zorder;
+	u32 hw_videoport;
+};
+
+struct dispc_device {
+	struct tidss_device *tidss;
+	struct device *dev;
+
+	void __iomem *base_common_cfg;
+	void __iomem *base_common_intr;
+	void __iomem *base_vid[DISPC7_MAX_PLANES];
+	void __iomem *base_ovr[DISPC7_MAX_PORTS];
+	void __iomem *base_vp[DISPC7_MAX_PORTS];
+	void __iomem *base_wb;
+
+	int irq;
+
+	bool has_cfg_common;
+
+	struct regmap *syscon;
+
+	struct clk *vp_clk[DISPC7_MAX_PORTS];
+
+	const struct dispc7_features *feat;
+
+	bool vp_managed[DISPC7_MAX_PORTS];
+	bool plane_managed[DISPC7_MAX_PLANES];
+	bool wb_managed;
+	u32 wb_reserved_ovr;
+
+	struct clk *fclk;
+
+	bool is_enabled;
+
+	struct dss_vp_data vp_data[DISPC7_MAX_PORTS];
+
+	struct dss_plane_data plane_data[DISPC7_MAX_PLANES];
+};
+
+static bool dispc7_has_writeback(struct dispc_device *dispc);
+
+#define dispc_for_each_managed_vp(dispc, hw_videoport) \
+	for ((hw_videoport) = 0; (hw_videoport) < (dispc)->feat->num_vps; (hw_videoport)++) \
+		if ((dispc)->vp_managed[(hw_videoport)])
+
+#define dispc_for_each_managed_plane(dispc, hw_plane) \
+	for ((hw_plane) = 0; (hw_plane) < (dispc)->feat->num_planes; (hw_plane)++) \
+		if ((dispc)->plane_managed[(hw_plane)])
+
+static u32 dispc7_get_next_managed_plane(struct dispc_device *dispc,
+					 u32 *plane_idx)
+{
+	while (*plane_idx < dispc->feat->num_planes) {
+		u32 plane_id = dispc->feat->vid_order[(*plane_idx)++];
+
+		if (dispc->plane_managed[plane_id])
+			return plane_id;
+	}
+
+	return dispc->feat->num_planes;
+}
+
+static void dispc7_intr_write(struct dispc_device *dispc, u16 reg, u32 val)
+{
+	iowrite32(val, dispc->base_common_intr + reg);
+}
+
+static u32 dispc7_intr_read(struct dispc_device *dispc, u16 reg)
+{
+	return ioread32(dispc->base_common_intr + reg);
+}
+
+static void dispc7_cfg_write(struct dispc_device *dispc, u16 reg, u32 val)
+{
+	iowrite32(val, dispc->base_common_cfg + reg);
+}
+
+static u32 dispc7_cfg_read(struct dispc_device *dispc, u16 reg)
+{
+	return ioread32(dispc->base_common_cfg + reg);
+}
+
+static void dispc7_vid_write(struct dispc_device *dispc, u32 hw_plane, u16 reg, u32 val)
+{
+	void __iomem *base = dispc->base_vid[hw_plane];
+
+	iowrite32(val, base + reg);
+}
+
+static u32 dispc7_vid_read(struct dispc_device *dispc, u32 hw_plane, u16 reg)
+{
+	void __iomem *base = dispc->base_vid[hw_plane];
+
+	return ioread32(base + reg);
+}
+
+static void dispc7_wb_write(struct dispc_device *dispc, u16 reg, u32 val)
+{
+	void __iomem *base = dispc->base_wb;
+
+	iowrite32(val, base + reg);
+}
+
+static u32 dispc7_wb_read(struct dispc_device *dispc, u16 reg)
+{
+	void __iomem *base = dispc->base_wb;
+
+	return ioread32(base + reg);
+}
+
+static void dispc7_ovr_write(struct dispc_device *dispc, u32 hw_videoport, u16 reg, u32 val)
+{
+	void __iomem *base = dispc->base_ovr[hw_videoport];
+
+	iowrite32(val, base + reg);
+}
+
+static u32 dispc7_ovr_read(struct dispc_device *dispc, u32 hw_videoport, u16 reg)
+{
+	void __iomem *base = dispc->base_ovr[hw_videoport];
+
+	return ioread32(base + reg);
+}
+
+static void dispc7_vp_write(struct dispc_device *dispc, u32 hw_videoport, u16 reg, u32 val)
+{
+	void __iomem *base = dispc->base_vp[hw_videoport];
+
+	iowrite32(val, base + reg);
+}
+
+static u32 dispc7_vp_read(struct dispc_device *dispc, u32 hw_videoport, u16 reg)
+{
+	void __iomem *base = dispc->base_vp[hw_videoport];
+
+	return ioread32(base + reg);
+}
+
+
+static int dispc7_runtime_get(struct dispc_device *dispc)
+{
+	int r;
+
+	dev_dbg(dispc->dev, "%s\n", __func__);
+
+	r = pm_runtime_get_sync(dispc->dev);
+	WARN_ON(r < 0);
+	return r < 0 ? r : 0;
+}
+
+static void dispc7_runtime_put(struct dispc_device *dispc)
+{
+	int r;
+
+	dev_dbg(dispc->dev, "%s\n", __func__);
+
+	r = pm_runtime_put_sync(dispc->dev);
+	WARN_ON(r < 0);
+}
+
+static u64 dispc7_vp_irq_from_raw(u32 stat, u32 hw_videoport)
+{
+	u64 vp_stat = 0;
+
+	if (stat & BIT(0))
+		vp_stat |= DSS_IRQ_VP_FRAME_DONE(hw_videoport);
+	if (stat & BIT(1))
+		vp_stat |= DSS_IRQ_VP_VSYNC_EVEN(hw_videoport);
+	if (stat & BIT(2))
+		vp_stat |= DSS_IRQ_VP_VSYNC_ODD(hw_videoport);
+	if (stat & BIT(4))
+		vp_stat |= DSS_IRQ_VP_SYNC_LOST(hw_videoport);
+
+	return vp_stat;
+}
+
+static u32 dispc7_vp_irq_to_raw(u64 vpstat, u32 hw_videoport)
+{
+	u32 stat = 0;
+
+	if (vpstat & DSS_IRQ_VP_FRAME_DONE(hw_videoport))
+		stat |= BIT(0);
+	if (vpstat & DSS_IRQ_VP_VSYNC_EVEN(hw_videoport))
+		stat |= BIT(1);
+	if (vpstat & DSS_IRQ_VP_VSYNC_ODD(hw_videoport))
+		stat |= BIT(2);
+	if (vpstat & DSS_IRQ_VP_SYNC_LOST(hw_videoport))
+		stat |= BIT(4);
+
+	return stat;
+}
+
+static u64 dispc7_vid_irq_from_raw(u32 stat, u32 hw_plane)
+{
+	u64 vid_stat = 0;
+
+	if (stat & BIT(0))
+		vid_stat |= DSS_IRQ_PLANE_FIFO_UNDERFLOW(hw_plane);
+
+	return vid_stat;
+}
+
+static u32 dispc7_vid_irq_to_raw(u64 vidstat, u32 hw_plane)
+{
+	u32 stat = 0;
+
+	if (vidstat & DSS_IRQ_PLANE_FIFO_UNDERFLOW(hw_plane))
+		stat |= BIT(0);
+
+	return stat;
+}
+
+static u64 dispc7_wb_irq_from_raw(u32 stat)
+{
+	u64 vp_stat = 0;
+
+	if (stat & BIT(0))
+		vp_stat |= DSS_IRQ_DEVICE_WBBUFFEROVERFLOW;
+	if (stat & BIT(1))
+		vp_stat |= DSS_IRQ_DEVICE_WBUNCOMPLETEERROR;
+	if (stat & BIT(2))
+		vp_stat |= DSS_IRQ_DEVICE_FRAMEDONEWB;
+	if (stat & BIT(3))
+		vp_stat |= DSS_IRQ_DEVICE_WBSECURITYVIOLATION;
+	if (stat & BIT(4))
+		vp_stat |= DSS_IRQ_DEVICE_WBSYNC;
+
+	return vp_stat;
+}
+
+static u32 dispc7_wb_irq_to_raw(u64 vpstat)
+{
+	u32 stat = 0;
+
+	if (vpstat & DSS_IRQ_DEVICE_WBBUFFEROVERFLOW)
+		stat |= BIT(0);
+	if (vpstat & DSS_IRQ_DEVICE_WBUNCOMPLETEERROR)
+		stat |= BIT(1);
+	if (vpstat & DSS_IRQ_DEVICE_FRAMEDONEWB)
+		stat |= BIT(2);
+	if (vpstat & DSS_IRQ_DEVICE_WBSECURITYVIOLATION)
+		stat |= BIT(3);
+	if (vpstat & DSS_IRQ_DEVICE_WBSYNC)
+		stat |= BIT(4);
+
+	return stat;
+}
+
+static u64 dispc7_vp_read_irqstatus(struct dispc_device *dispc,
+				    u32 hw_videoport)
+{
+	u32 stat = dispc7_intr_read(dispc, DISPC_VP_IRQSTATUS(hw_videoport));
+
+	return dispc7_vp_irq_from_raw(stat, hw_videoport);
+}
+
+static void dispc7_vp_write_irqstatus(struct dispc_device *dispc,
+				      u32 hw_videoport, u64 vpstat)
+{
+	u32 stat = dispc7_vp_irq_to_raw(vpstat, hw_videoport);
+
+	dispc7_intr_write(dispc, DISPC_VP_IRQSTATUS(hw_videoport), stat);
+}
+
+static u64 dispc7_vid_read_irqstatus(struct dispc_device *dispc,
+				     u32 hw_plane)
+{
+	u32 stat = dispc7_intr_read(dispc, DISPC_VID_IRQSTATUS(hw_plane));
+
+	return dispc7_vid_irq_from_raw(stat, hw_plane);
+}
+
+static void dispc7_vid_write_irqstatus(struct dispc_device *dispc,
+				       u32 hw_plane, u64 vidstat)
+{
+	u32 stat = dispc7_vid_irq_to_raw(vidstat, hw_plane);
+
+	dispc7_intr_write(dispc, DISPC_VID_IRQSTATUS(hw_plane), stat);
+}
+
+static u64 dispc7_wb_read_irqstatus(struct dispc_device *dispc)
+{
+	u32 stat = dispc7_intr_read(dispc, WB_IRQSTATUS);
+
+	return dispc7_wb_irq_from_raw(stat);
+}
+
+static void dispc7_wb_write_irqstatus(struct dispc_device *dispc,
+				      u64 vpstat)
+{
+	u32 stat = dispc7_wb_irq_to_raw(vpstat);
+
+	dispc7_intr_write(dispc, WB_IRQSTATUS, stat);
+}
+
+static u64 dispc7_vp_read_irqenable(struct dispc_device *dispc,
+				    u32 hw_videoport)
+{
+	u32 stat = dispc7_intr_read(dispc, DISPC_VP_IRQENABLE(hw_videoport));
+
+	return dispc7_vp_irq_from_raw(stat, hw_videoport);
+}
+
+static void dispc7_vp_write_irqenable(struct dispc_device *dispc,
+				      u32 hw_videoport, u64 vpstat)
+{
+	u32 stat = dispc7_vp_irq_to_raw(vpstat, hw_videoport);
+
+	dispc7_intr_write(dispc, DISPC_VP_IRQENABLE(hw_videoport), stat);
+}
+
+
+static u64 dispc7_vid_read_irqenable(struct dispc_device *dispc,
+				     u32 hw_plane)
+{
+	u32 stat = dispc7_intr_read(dispc, DISPC_VID_IRQENABLE(hw_plane));
+
+	return dispc7_vid_irq_from_raw(stat, hw_plane);
+}
+
+static void dispc7_vid_write_irqenable(struct dispc_device *dispc,
+				       u32 hw_plane, u64 vidstat)
+{
+	u32 stat = dispc7_vid_irq_to_raw(vidstat, hw_plane);
+
+	dispc7_intr_write(dispc, DISPC_VID_IRQENABLE(hw_plane), stat);
+}
+
+static u64 dispc7_wb_read_irqenable(struct dispc_device *dispc)
+{
+	u32 stat = dispc7_intr_read(dispc, WB_IRQENABLE);
+
+	return dispc7_wb_irq_from_raw(stat);
+}
+
+static void dispc7_wb_write_irqenable(struct dispc_device *dispc,
+				      u64 vpstat)
+{
+	u32 stat = dispc7_wb_irq_to_raw(vpstat);
+
+	dispc7_intr_write(dispc, WB_IRQENABLE, stat);
+}
+
+static void dispc7_clear_irqstatus(struct dispc_device *dispc, u64 clearmask)
+{
+	unsigned int i;
+	u32 top_clear = 0;
+
+	dispc_for_each_managed_vp(dispc, i) {
+		if (clearmask & DSS_IRQ_VP_MASK(i)) {
+			dispc7_vp_write_irqstatus(dispc, i, clearmask);
+			top_clear |= BIT(i);
+		}
+	}
+	dispc_for_each_managed_plane(dispc, i) {
+		if (clearmask & DSS_IRQ_PLANE_MASK(i)) {
+			dispc7_vid_write_irqstatus(dispc, i, clearmask);
+			top_clear |= BIT(4 + i);
+		}
+	}
+	if (dispc7_has_writeback(dispc)) {
+		if (clearmask & DSS_IRQ_DEVICE_WB_MASK) {
+			dispc7_wb_write_irqstatus(dispc, clearmask);
+			top_clear |= BIT(14);
+		}
+	}
+	dispc7_intr_write(dispc, DISPC_IRQSTATUS, top_clear);
+
+	/* Flush posted writes */
+	dispc7_intr_read(dispc, DISPC_IRQSTATUS);
+}
+
+static u64 dispc7_read_and_clear_irqstatus(struct dispc_device *dispc)
+{
+	u64 status = 0;
+	unsigned int i;
+
+	dispc_for_each_managed_vp(dispc, i)
+		status |= dispc7_vp_read_irqstatus(dispc, i);
+
+	dispc_for_each_managed_plane(dispc, i)
+		status |= dispc7_vid_read_irqstatus(dispc, i);
+
+	if (dispc7_has_writeback(dispc))
+		status |= dispc7_wb_read_irqstatus(dispc);
+
+	dispc7_clear_irqstatus(dispc, status);
+
+	return status;
+}
+
+static u64 dispc7_read_irqenable(struct dispc_device *dispc)
+{
+	u64 enable = 0;
+	unsigned int i;
+
+	dispc_for_each_managed_vp(dispc, i)
+		enable |= dispc7_vp_read_irqenable(dispc, i);
+
+	dispc_for_each_managed_plane(dispc, i)
+		enable |= dispc7_vid_read_irqenable(dispc, i);
+
+	if (dispc7_has_writeback(dispc))
+		enable |= dispc7_wb_read_irqenable(dispc);
+
+	return enable;
+}
+
+static void dispc7_write_irqenable(struct dispc_device *dispc, u64 mask)
+{
+	unsigned int i;
+	u32 main_enable = 0, main_disable = 0;
+	u64 old_mask;
+
+	old_mask = dispc7_read_irqenable(dispc);
+
+	/* clear the irqstatus for newly enabled irqs */
+	dispc7_clear_irqstatus(dispc, (old_mask ^ mask) & mask);
+
+	dispc_for_each_managed_vp(dispc, i) {
+		dispc7_vp_write_irqenable(dispc, i, mask);
+		if (mask & DSS_IRQ_VP_MASK(i))
+			main_enable |= BIT(i);		/* VP IRQ */
+		else
+			main_disable |= BIT(i);		/* VP IRQ */
+	}
+
+	dispc_for_each_managed_plane(dispc, i) {
+		dispc7_vid_write_irqenable(dispc, i, mask);
+		if (mask & DSS_IRQ_PLANE_MASK(i))
+			main_enable |= BIT(i + 4);	/* VID IRQ */
+		else
+			main_disable |= BIT(i + 4);	/* VID IRQ */
+	}
+
+	if (dispc7_has_writeback(dispc)) {
+		dispc7_wb_write_irqenable(dispc, mask);
+		if (mask & DSS_IRQ_DEVICE_WB_MASK)
+			main_enable |= BIT(14);		/* WB_IRQ */
+		else
+			main_disable |= BIT(14);	/* WB_IRQ */
+	}
+
+	if (main_enable)
+		dispc7_intr_write(dispc, DISPC_IRQENABLE_SET, main_enable);
+
+	if (main_disable)
+		dispc7_intr_write(dispc, DISPC_IRQENABLE_CLR, main_disable);
+
+	/* Flush posted writes */
+	dispc7_intr_read(dispc, DISPC_IRQENABLE_SET);
+}
+
+enum dispc7_oldi_mode_reg_val { SPWG_18 = 0, JEIDA_24 = 1, SPWG_24 = 2 };
+
+struct dispc7_bus_format {
+	u32 bus_fmt;
+	u32 data_width;
+	bool is_oldi_fmt;
+	enum dispc7_oldi_mode_reg_val oldi_mode_reg_val;
+};
+
+static const struct dispc7_bus_format dispc7_bus_formats[] = {
+	{ MEDIA_BUS_FMT_RGB444_1X12,		12, false, 0 },
+	{ MEDIA_BUS_FMT_RGB565_1X16,		16, false, 0 },
+	{ MEDIA_BUS_FMT_RGB666_1X18,		18, false, 0 },
+	{ MEDIA_BUS_FMT_RGB888_1X24,		24, false, 0 },
+	{ MEDIA_BUS_FMT_RGB101010_1X30,		30, false, 0 },
+	{ MEDIA_BUS_FMT_RGB121212_1X36,		36, false, 0 },
+	{ MEDIA_BUS_FMT_RGB666_1X7X3_SPWG,	18, true, SPWG_18 },
+	{ MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,	24, true, SPWG_24 },
+	{ MEDIA_BUS_FMT_RGB888_1X7X4_JEIDA,	24, true, JEIDA_24 },
+};
+
+static const
+struct dispc7_bus_format *dispc7_vp_find_bus_fmt(struct dispc_device *dispc,
+						 u32 hw_videoport,
+						 u32 bus_fmt, u32 bus_flags)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(dispc7_bus_formats); ++i) {
+		if (dispc7_bus_formats[i].bus_fmt == bus_fmt)
+			return &dispc7_bus_formats[i];
+	}
+
+	return NULL;
+}
+
+static void dispc7_oldi_tx_power(struct dispc_device *dispc, bool power)
+{
+	u32 val = power ? 0 : CTRLMMR0P1_OLDI_PWRDN_TX;
+
+	if (WARN_ON(!dispc->syscon))
+		return;
+
+	regmap_update_bits(dispc->syscon, CTRLMMR0P1_OLDI_DAT0_IO_CTRL,
+			   CTRLMMR0P1_OLDI_PWRDN_TX, val);
+	regmap_update_bits(dispc->syscon, CTRLMMR0P1_OLDI_DAT1_IO_CTRL,
+			   CTRLMMR0P1_OLDI_PWRDN_TX, val);
+	regmap_update_bits(dispc->syscon, CTRLMMR0P1_OLDI_DAT2_IO_CTRL,
+			   CTRLMMR0P1_OLDI_PWRDN_TX, val);
+	regmap_update_bits(dispc->syscon, CTRLMMR0P1_OLDI_DAT3_IO_CTRL,
+			   CTRLMMR0P1_OLDI_PWRDN_TX, val);
+	regmap_update_bits(dispc->syscon, CTRLMMR0P1_OLDI_CLK_IO_CTRL,
+			   CTRLMMR0P1_OLDI_PWRDN_TX, val);
+}
+
+static void dispc7_set_num_datalines(struct dispc_device *dispc,
+				     u32 hw_videoport, int num_lines)
+{
+	int v;
+
+	switch (num_lines) {
+	case 12:
+		v = 0; break;
+	case 16:
+		v = 1; break;
+	case 18:
+		v = 2; break;
+	case 24:
+		v = 3; break;
+	case 30:
+		v = 4; break;
+	case 36:
+		v = 5; break;
+	default:
+		WARN_ON(1);
+		v = 3;
+	}
+
+	VP_REG_FLD_MOD(dispc, hw_videoport, DISPC_VP_CONTROL, v, 10, 8);
+}
+
+static void dispc7_enable_oldi(struct dispc_device *dispc, u32 hw_videoport,
+			       const struct dispc7_bus_format *fmt)
+{
+	u32 oldi_cfg = 0;
+	u32 oldi_reset_bit = BIT(5 + hw_videoport);
+	int count = 0;
+
+	/*
+	 * On am6 DUALMODESYNC, MASTERSLAVE, MODE, and SRC are set
+	 * statically to 0.
+	 */
+
+	if (fmt->data_width == 24)
+		oldi_cfg |= BIT(8); /* MSB */
+	else if (fmt->data_width != 18)
+		dev_warn(dispc->dev, "%s: %d port width not supported\n",
+			 __func__, fmt->data_width);
+
+	oldi_cfg |= BIT(7); /* DEPOL */
+
+	oldi_cfg = FLD_MOD(oldi_cfg, fmt->oldi_mode_reg_val, 3, 1);
+
+	oldi_cfg |= BIT(12); /* SOFTRST */
+
+	oldi_cfg |= BIT(0); /* ENABLE */
+
+	dispc7_vp_write(dispc, hw_videoport, DISPC_VP_DSS_OLDI_CFG, oldi_cfg);
+
+	while (!(oldi_reset_bit & dispc7_cfg_read(dispc, DSS_SYSSTATUS)) &&
+	       count < 10000)
+		count++;
+
+	if (!(oldi_reset_bit & dispc7_cfg_read(dispc, DSS_SYSSTATUS)))
+		dev_warn(dispc->dev, "%s: timeout waiting OLDI reset done\n",
+			 __func__);
+}
+
+static void dispc7_vp_prepare(struct dispc_device *dispc, u32 hw_videoport,
+			      const struct drm_crtc_state *state)
+{
+	const struct tidss_crtc_state *tstate = to_tidss_crtc_state(state);
+	const struct dispc7_bus_format *fmt;
+
+	fmt = dispc7_vp_find_bus_fmt(dispc, hw_videoport, tstate->bus_format,
+				     tstate->bus_flags);
+
+	if (WARN_ON(!fmt))
+		return;
+
+	if (dispc->feat->vp_bus_type[hw_videoport] == DISPC7_VP_OLDI) {
+		dispc7_oldi_tx_power(dispc, true);
+
+		if (dispc->has_cfg_common)
+			dispc7_enable_oldi(dispc, hw_videoport, fmt);
+	}
+}
+
+static void dispc7_vp_enable(struct dispc_device *dispc, u32 hw_videoport,
+			      const struct drm_crtc_state *state)
+{
+	const struct drm_display_mode *mode = &state->adjusted_mode;
+	const struct tidss_crtc_state *tstate = to_tidss_crtc_state(state);
+	bool align, onoff, rf, ieo, ipc, ihs, ivs;
+	const struct dispc7_bus_format *fmt;
+	u32 hsw, hfp, hbp, vsw, vfp, vbp;
+
+	fmt = dispc7_vp_find_bus_fmt(dispc, hw_videoport, tstate->bus_format,
+				     tstate->bus_flags);
+
+	if (WARN_ON(!fmt))
+		return;
+
+	dispc7_set_num_datalines(dispc, hw_videoport, fmt->data_width);
+
+	hfp = mode->hsync_start - mode->hdisplay;
+	hsw = mode->hsync_end - mode->hsync_start;
+	hbp = mode->htotal - mode->hsync_end;
+
+	vfp = mode->vsync_start - mode->vdisplay;
+	vsw = mode->vsync_end - mode->vsync_start;
+	vbp = mode->vtotal - mode->vsync_end;
+
+	dispc7_vp_write(dispc, hw_videoport, DISPC_VP_TIMING_H,
+			FLD_VAL(hsw - 1, 7, 0) |
+			FLD_VAL(hfp - 1, 19, 8) |
+			FLD_VAL(hbp - 1, 31, 20));
+
+	dispc7_vp_write(dispc, hw_videoport, DISPC_VP_TIMING_V,
+			FLD_VAL(vsw - 1, 7, 0) |
+			FLD_VAL(vfp, 19, 8) |
+			FLD_VAL(vbp, 31, 20));
+
+	ivs = !!(mode->flags & DRM_MODE_FLAG_NVSYNC);
+
+	ihs = !!(mode->flags & DRM_MODE_FLAG_NHSYNC);
+
+	ieo = !!(tstate->bus_flags & DRM_BUS_FLAG_DE_LOW);
+
+	ipc = !!(tstate->bus_flags & DRM_BUS_FLAG_PIXDATA_NEGEDGE);
+
+	/* always use the 'rf' setting */
+	onoff = true;
+
+	rf = !!(tstate->bus_flags & DRM_BUS_FLAG_SYNC_POSEDGE);
+
+	/* always use aligned syncs */
+	align = true;
+
+	/* always use DE_HIGH for OLDI */
+	if (dispc->feat->vp_bus_type[hw_videoport] == DISPC7_VP_OLDI)
+		ieo = false;
+
+	dispc7_vp_write(dispc, hw_videoport, DISPC_VP_POL_FREQ,
+			FLD_VAL(align, 18, 18) |
+			FLD_VAL(onoff, 17, 17) |
+			FLD_VAL(rf, 16, 16) |
+			FLD_VAL(ieo, 15, 15) |
+			FLD_VAL(ipc, 14, 14) |
+			FLD_VAL(ihs, 13, 13) |
+			FLD_VAL(ivs, 12, 12));
+
+	dispc7_vp_write(dispc, hw_videoport, DISPC_VP_SIZE_SCREEN,
+			FLD_VAL(mode->hdisplay - 1, 11, 0) |
+			FLD_VAL(mode->vdisplay - 1, 27, 16));
+
+	VP_REG_FLD_MOD(dispc, hw_videoport, DISPC_VP_CONTROL, 1, 0, 0);
+}
+
+static void dispc7_vp_disable(struct dispc_device *dispc, u32 hw_videoport)
+{
+	VP_REG_FLD_MOD(dispc, hw_videoport, DISPC_VP_CONTROL, 0, 0, 0);
+}
+
+static void dispc7_vp_unprepare(struct dispc_device *dispc, u32 hw_videoport)
+{
+	if (dispc->feat->vp_bus_type[hw_videoport] == DISPC7_VP_OLDI) {
+		dispc7_vp_write(dispc, hw_videoport, DISPC_VP_DSS_OLDI_CFG, 0);
+
+		dispc7_oldi_tx_power(dispc, false);
+	}
+}
+
+static bool dispc7_vp_go_busy(struct dispc_device *dispc,
+			      u32 hw_videoport)
+{
+	return VP_REG_GET(dispc, hw_videoport, DISPC_VP_CONTROL, 5, 5);
+}
+
+static void dispc7_vp_go(struct dispc_device *dispc, u32 hw_videoport)
+{
+	WARN_ON(VP_REG_GET(dispc, hw_videoport, DISPC_VP_CONTROL, 5, 5));
+	VP_REG_FLD_MOD(dispc, hw_videoport, DISPC_VP_CONTROL, 1, 5, 5);
+}
+
+enum c8_to_c12_mode { C8_TO_C12_REPLICATE, C8_TO_C12_MAX, C8_TO_C12_MIN };
+
+static u16 c8_to_c12(u8 c8, enum c8_to_c12_mode mode)
+{
+	u16 c12;
+
+	c12 = c8 << 4;
+
+	switch (mode) {
+	case C8_TO_C12_REPLICATE:
+		/* Copy c8 4 MSB to 4 LSB for full scale c12 */
+		c12 |= c8 >> 4;
+		break;
+	case C8_TO_C12_MAX:
+		c12 |= 0xF;
+		break;
+	default:
+	case C8_TO_C12_MIN:
+		break;
+	}
+
+	return c12;
+}
+
+static u64 argb8888_to_argb12121212(u32 argb8888, enum c8_to_c12_mode m)
+{
+	u8 a, r, g, b;
+	u64 v;
+
+	a = (argb8888 >> 24) & 0xff;
+	r = (argb8888 >> 16) & 0xff;
+	g = (argb8888 >> 8) & 0xff;
+	b = (argb8888 >> 0) & 0xff;
+
+	v = ((u64)c8_to_c12(a, m) << 36) | ((u64)c8_to_c12(r, m) << 24) |
+		((u64)c8_to_c12(g, m) << 12) | (u64)c8_to_c12(b, m);
+
+	return v;
+}
+
+static void dispc7_vp_set_default_color(struct dispc_device *dispc,
+					u32 hw_videoport, u32 default_color)
+{
+	u64 v;
+
+	v = argb8888_to_argb12121212(default_color, C8_TO_C12_REPLICATE);
+
+	dispc7_ovr_write(dispc, hw_videoport,
+			 DISPC_OVR_DEFAULT_COLOR, v & 0xffffffff);
+	dispc7_ovr_write(dispc, hw_videoport,
+			 DISPC_OVR_DEFAULT_COLOR2, (v >> 32) & 0xffff);
+}
+
+static enum drm_mode_status dispc7_vp_mode_valid(struct dispc_device *dispc,
+						 u32 hw_videoport,
+						 const struct drm_display_mode *mode)
+{
+	u32 hsw, hfp, hbp, vsw, vfp, vbp;
+	enum dispc7_vp_bus_type bus_type;
+	int max_pclk;
+
+	bus_type = dispc->feat->vp_bus_type[hw_videoport];
+
+	max_pclk = dispc->feat->max_pclk_kHz[bus_type];
+
+	if (WARN_ON(max_pclk == 0))
+		return MODE_BAD;
+
+	if (mode->clock > max_pclk)
+		return MODE_CLOCK_HIGH;
+
+	if (mode->hdisplay > 4096)
+		return MODE_BAD;
+
+	if (mode->vdisplay > 4096)
+		return MODE_BAD;
+
+	/* TODO: add interlace support */
+	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
+		return MODE_NO_INTERLACE;
+
+	/*
+	 * Enforce the output width is divisible by 2. Actually this
+	 * is only needed in following cases:
+	 * - YUV output selected (BT656, BT1120)
+	 * - Dithering enabled
+	 * - TDM with TDMCycleFormat == 3
+	 * But for simplicity we enforce that always.
+	 */
+	if ((mode->hdisplay % 2) != 0)
+		return MODE_BAD_HVALUE;
+
+	hfp = mode->hsync_start - mode->hdisplay;
+	hsw = mode->hsync_end - mode->hsync_start;
+	hbp = mode->htotal - mode->hsync_end;
+
+	vfp = mode->vsync_start - mode->vdisplay;
+	vsw = mode->vsync_end - mode->vsync_start;
+	vbp = mode->vtotal - mode->vsync_end;
+
+	if (hsw < 1 || hsw > 256 ||
+	    hfp < 1 || hfp > 4096 ||
+	    hbp < 1 || hbp > 4096)
+		return MODE_BAD_HVALUE;
+
+	if (vsw < 1 || vsw > 256 ||
+	    vfp > 4095 || vbp > 4095)
+		return MODE_BAD_VVALUE;
+
+	return MODE_OK;
+}
+
+static int dispc7_vp_check(struct dispc_device *dispc, u32 hw_videoport,
+			   const struct drm_crtc_state *state)
+{
+	const struct drm_display_mode *mode = &state->adjusted_mode;
+	const struct tidss_crtc_state *tstate = to_tidss_crtc_state(state);
+	const struct dispc7_bus_format *fmt;
+	enum drm_mode_status ok;
+
+	ok = dispc7_vp_mode_valid(dispc, hw_videoport, mode);
+	if (ok != MODE_OK) {
+		dev_dbg(dispc->dev, "%s: bad mode: %ux%u pclk %u kHz\n",
+			__func__, mode->hdisplay, mode->vdisplay, mode->clock);
+		return -EINVAL;
+	}
+
+	fmt = dispc7_vp_find_bus_fmt(dispc, hw_videoport, tstate->bus_format,
+				     tstate->bus_flags);
+	if (!fmt) {
+		dev_dbg(dispc->dev, "%s: Unsupported bus format: %u\n",
+			__func__, tstate->bus_format);
+		return -EINVAL;
+	}
+
+	if (dispc->feat->vp_bus_type[hw_videoport] != DISPC7_VP_OLDI &&
+	    fmt->is_oldi_fmt) {
+		dev_dbg(dispc->dev, "%s: %s is not OLDI-port\n",
+			__func__, dispc->feat->vp_name[hw_videoport]);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int dispc7_vp_enable_clk(struct dispc_device *dispc, u32 hw_videoport)
+{
+	int ret = clk_prepare_enable(dispc->vp_clk[hw_videoport]);
+
+	if (ret)
+		dev_err(dispc->dev, "%s: enabling clk failed: %d\n", __func__,
+			ret);
+
+	return ret;
+}
+
+static void dispc7_vp_disable_clk(struct dispc_device *dispc, u32 hw_videoport)
+{
+	clk_disable_unprepare(dispc->vp_clk[hw_videoport]);
+}
+
+/*
+ * Calculate the percentage difference between the requested pixel clock rate
+ * and the effective rate resulting from calculating the clock divider value.
+ */
+static unsigned int dispc7_pclk_diff(unsigned long rate,
+				     unsigned long real_rate)
+{
+	int r = rate / 100, rr = real_rate / 100;
+
+	return (unsigned int)(abs(((rr - r) * 100) / r));
+}
+
+static int dispc7_vp_set_clk_rate(struct dispc_device *dispc, u32 hw_videoport,
+				  unsigned long rate)
+{
+	int r;
+	unsigned long new_rate;
+
+	r = clk_set_rate(dispc->vp_clk[hw_videoport], rate);
+	if (r) {
+		dev_err(dispc->dev, "vp%d: failed to set clk rate to %lu\n",
+			hw_videoport, rate);
+		return r;
+	}
+
+	new_rate = clk_get_rate(dispc->vp_clk[hw_videoport]);
+
+	if (dispc7_pclk_diff(rate, new_rate) > 5)
+		dev_warn(dispc->dev,
+			 "vp%d: Clock rate %lu differs over 5%% from requsted %lu\n",
+			 hw_videoport, new_rate, rate);
+
+	dev_dbg(dispc->dev, "vp%d: new rate %lu Hz (requested %lu Hz)\n",
+		hw_videoport, clk_get_rate(dispc->vp_clk[hw_videoport]), rate);
+
+	return 0;
+}
+
+/* OVR */
+static void dispc7_am6_ovr_set_plane(struct dispc_device *dispc,
+				     u32 hw_plane, u32 hw_videoport,
+				     u32 x, u32 y, u32 zpos)
+{
+	OVR_REG_FLD_MOD(dispc, hw_videoport, DISPC_OVR_ATTRIBUTES(zpos),
+			hw_plane, 4, 1);
+	OVR_REG_FLD_MOD(dispc, hw_videoport, DISPC_OVR_ATTRIBUTES(zpos),
+			x, 17, 6);
+	OVR_REG_FLD_MOD(dispc, hw_videoport, DISPC_OVR_ATTRIBUTES(zpos),
+			y, 30, 19);
+}
+
+static void dispc7_j721e_ovr_set_plane(struct dispc_device *dispc,
+				      u32 hw_plane, u32 hw_videoport,
+				      u32 x, u32 y, u32 zpos)
+{
+	OVR_REG_FLD_MOD(dispc, hw_videoport, DISPC_OVR_ATTRIBUTES(zpos),
+			hw_plane, 4, 1);
+	OVR_REG_FLD_MOD(dispc, hw_videoport, DISPC_OVR_ATTRIBUTES2(zpos),
+			x, 13, 0);
+	OVR_REG_FLD_MOD(dispc, hw_videoport, DISPC_OVR_ATTRIBUTES2(zpos),
+			y, 29, 16);
+}
+
+static void dispc7_ovr_set_plane(struct dispc_device *dispc,
+				 u32 hw_plane, u32 hw_videoport,
+				 u32 x, u32 y, u32 zpos)
+{
+	switch (dispc->feat->subrev) {
+	case DSS7_AM6:
+		dispc7_am6_ovr_set_plane(dispc, hw_plane, hw_videoport,
+					 x, y, zpos);
+		break;
+	case DSS7_J721E:
+		dispc7_j721e_ovr_set_plane(dispc, hw_plane, hw_videoport,
+					  x, y, zpos);
+		break;
+	default:
+		WARN_ON(1);
+		break;
+	}
+}
+
+static void dispc7_ovr_enable_plane(struct dispc_device *dispc,
+				    u32 hw_videoport, u32 zpos, bool enable)
+{
+	OVR_REG_FLD_MOD(dispc, hw_videoport, DISPC_OVR_ATTRIBUTES(zpos),
+			!!enable, 0, 0);
+}
+
+/* CSC */
+enum csc_ctm {
+	CSC_RR, CSC_RG, CSC_RB,
+	CSC_GR, CSC_GG, CSC_GB,
+	CSC_BR, CSC_BG, CSC_BB,
+};
+
+enum csc_yuv2rgb {
+	CSC_RY, CSC_RCb, CSC_RCr,
+	CSC_GY, CSC_GCb, CSC_GCr,
+	CSC_BY, CSC_BCb, CSC_BCr,
+};
+
+enum csc_rgb2yuv {
+	CSC_YR,  CSC_YG,  CSC_YB,
+	CSC_CbR, CSC_CbG, CSC_CbB,
+	CSC_CrR, CSC_CrG, CSC_CrB,
+};
+
+struct dispc7_csc_coef {
+	void (*to_regval)(const struct dispc7_csc_coef *csc, u32 *regval);
+	int m[9];
+	int preoffset[3];
+	int postoffset[3];
+	enum { CLIP_LIMITED_RANGE = 0, CLIP_FULL_RANGE = 1, } cliping;
+	const char *name;
+};
+
+#define DISPC7_CSC_REGVAL_LEN 8
+
+static void dispc7_csc_offset_regval(const struct dispc7_csc_coef *csc,
+				     u32 *regval)
+{
+#define OVAL(x, y) (FLD_VAL(x, 15, 3) | FLD_VAL(y, 31, 19))
+	regval[5] = OVAL(csc->preoffset[0], csc->preoffset[1]);
+	regval[6] = OVAL(csc->preoffset[2], csc->postoffset[0]);
+	regval[7] = OVAL(csc->postoffset[1], csc->postoffset[2]);
+#undef OVAL
+}
+
+#define CVAL(x, y) (FLD_VAL(x, 10, 0) | FLD_VAL(y, 26, 16))
+static void dispc7_csc_yuv2rgb_regval(const struct dispc7_csc_coef *csc,
+				      u32 *regval)
+{
+	regval[0] = CVAL(csc->m[CSC_RY], csc->m[CSC_RCr]);
+	regval[1] = CVAL(csc->m[CSC_RCb], csc->m[CSC_GY]);
+	regval[2] = CVAL(csc->m[CSC_GCr], csc->m[CSC_GCb]);
+	regval[3] = CVAL(csc->m[CSC_BY], csc->m[CSC_BCr]);
+	regval[4] = CVAL(csc->m[CSC_BCb], 0);
+
+	dispc7_csc_offset_regval(csc, regval);
+}
+
+static void __maybe_unused dispc7_csc_rgb2yuv_regval(const struct dispc7_csc_coef *csc,
+				      u32 *regval)
+{
+	regval[0] = CVAL(csc->m[CSC_YR], csc->m[CSC_YG]);
+	regval[1] = CVAL(csc->m[CSC_YB], csc->m[CSC_CrR]);
+	regval[2] = CVAL(csc->m[CSC_CrG], csc->m[CSC_CrB]);
+	regval[3] = CVAL(csc->m[CSC_CbR], csc->m[CSC_CbG]);
+	regval[4] = CVAL(csc->m[CSC_CbB], 0);
+
+	dispc7_csc_offset_regval(csc, regval);
+}
+
+static void dispc7_csc_cpr_regval(const struct dispc7_csc_coef *csc,
+				  u32 *regval)
+{
+	regval[0] = CVAL(csc->m[CSC_RR], csc->m[CSC_RG]);
+	regval[1] = CVAL(csc->m[CSC_RB], csc->m[CSC_GR]);
+	regval[2] = CVAL(csc->m[CSC_GG], csc->m[CSC_GB]);
+	regval[3] = CVAL(csc->m[CSC_BR], csc->m[CSC_BG]);
+	regval[4] = CVAL(csc->m[CSC_BB], 0);
+
+	dispc7_csc_offset_regval(csc, regval);
+}
+#undef CVAL
+
+
+static void dispc7_vid_write_csc(struct dispc_device *dispc, u32 hw_plane,
+				 const struct dispc7_csc_coef *csc)
+{
+	static const u16 dispc_vid_csc_coef_reg[DISPC7_CSC_REGVAL_LEN] = {
+		DISPC_VID_CSC_COEF(0), DISPC_VID_CSC_COEF(1),
+		DISPC_VID_CSC_COEF(2), DISPC_VID_CSC_COEF(3),
+		DISPC_VID_CSC_COEF(4), DISPC_VID_CSC_COEF(5),
+		DISPC_VID_CSC_COEF(6), DISPC_VID_CSC_COEF7,
+	};
+	u32 regval[DISPC7_CSC_REGVAL_LEN];
+	unsigned int i;
+
+	csc->to_regval(csc, regval);
+
+	for (i = 0; i < ARRAY_SIZE(dispc_vid_csc_coef_reg); i++)
+		dispc7_vid_write(dispc, hw_plane, dispc_vid_csc_coef_reg[i],
+				regval[i]);
+}
+
+static void dispc7_wb_write_csc(struct dispc_device *dispc,
+				 const struct dispc7_csc_coef *csc)
+{
+	static const u16 dispc_wb_csc_coef_reg[DISPC7_CSC_REGVAL_LEN] = {
+		DISPC_WB_CSC_COEF(0), DISPC_WB_CSC_COEF(1),
+		DISPC_WB_CSC_COEF(2), DISPC_WB_CSC_COEF(3),
+		DISPC_WB_CSC_COEF(4), DISPC_WB_CSC_COEF(5),
+		DISPC_WB_CSC_COEF(6), DISPC_WB_CSC_COEF7,
+	};
+	u32 regval[DISPC7_CSC_REGVAL_LEN];
+	unsigned int i;
+
+	csc->to_regval(csc, regval);
+
+	for (i = 0; i < ARRAY_SIZE(dispc_wb_csc_coef_reg); i++)
+		dispc7_wb_write(dispc, dispc_wb_csc_coef_reg[i],
+				regval[i]);
+}
+
+/* YUV -> RGB, ITU-R BT.601, full range */
+static const struct dispc7_csc_coef csc_yuv2rgb_bt601_full = {
+	dispc7_csc_yuv2rgb_regval,
+	{ 256,   0,  358,	/* ry, rcb, rcr |1.000  0.000  1.402|*/
+	  256, -88, -182,	/* gy, gcb, gcr |1.000 -0.344 -0.714|*/
+	  256, 452,    0, },	/* by, bcb, bcr |1.000  1.772  0.000|*/
+	{    0, -2048, -2048, },	/* full range */
+	{    0,     0,     0, },
+	CLIP_FULL_RANGE,
+	"BT.601 Full",
+};
+
+/* YUV -> RGB, ITU-R BT.601, limited range */
+static const struct dispc7_csc_coef csc_yuv2rgb_bt601_lim = {
+	dispc7_csc_yuv2rgb_regval,
+	{ 298,    0,  409,	/* ry, rcb, rcr |1.164  0.000  1.596|*/
+	  298, -100, -208,	/* gy, gcb, gcr |1.164 -0.392 -0.813|*/
+	  298,  516,    0, },	/* by, bcb, bcr |1.164  2.017  0.000|*/
+	{ -256, -2048, -2048, },	/* limited range */
+	{    0,     0,     0, },
+	CLIP_FULL_RANGE,
+	"BT.601 Limited",
+};
+
+/* YUV -> RGB, ITU-R BT.709, full range */
+static const struct dispc7_csc_coef csc_yuv2rgb_bt709_full = {
+	dispc7_csc_yuv2rgb_regval,
+	{ 256,	  0,  402,	/* ry, rcb, rcr |1.000	0.000  1.570|*/
+	  256,  -48, -120,	/* gy, gcb, gcr |1.000 -0.187 -0.467|*/
+	  256,  475,    0, },	/* by, bcb, bcr |1.000	1.856  0.000|*/
+	{    0, -2048, -2048, },	/* full range */
+	{    0,     0,     0, },
+	CLIP_FULL_RANGE,
+	"BT.709 Full",
+};
+
+/* YUV -> RGB, ITU-R BT.709, limited range */
+static const struct dispc7_csc_coef csc_yuv2rgb_bt709_lim = {
+	dispc7_csc_yuv2rgb_regval,
+	{ 298,    0,  459,	/* ry, rcb, rcr |1.164  0.000  1.793|*/
+	  298,  -55, -136,	/* gy, gcb, gcr |1.164 -0.213 -0.533|*/
+	  298,  541,    0, },	/* by, bcb, bcr |1.164  2.112  0.000|*/
+	{ -256, -2048, -2048, },	/* limited range */
+	{    0,     0,     0, },
+	CLIP_FULL_RANGE,
+	"BT.709 Limited",
+};
+
+/* RGB -> YUV, ITU-R BT.601, full range */
+static const struct dispc7_csc_coef csc_rgb2yuv_bt601_full = {
+	dispc7_csc_rgb2yuv_regval,
+	{ 77,  150,  29,	/* yr,   yg,  yb | 0.299  0.587  0.114|*/
+	 -43,  -85, 128,	/* cbr, cbg, cbb |-0.173 -0.339  0.511|*/
+	 128, -107, -21 },	/* crr, crg, crb | 0.511 -0.428 -0.083|*/
+	{    0,     0,     0, },
+	{    0,  2048,  2048, },	/* full range */
+	CLIP_FULL_RANGE,
+	"BT.601 Full",
+};
+
+/* RGB -> YUV, ITU-R BT.601, limited range */
+static const struct dispc7_csc_coef csc_rgb2yuv_bt601_lim = {
+	dispc7_csc_rgb2yuv_regval,
+	{ 66,  129,  25,	/* yr,   yg,  yb | 0.257  0.504  0.098|*/
+	 -38,  -74, 112,	/* cbr, cbg, cbb |-0.148 -0.291  0.439|*/
+	 112,  -94, -18 },	/* crr, crg, crb | 0.439 -0.368 -0.071|*/
+	{    0,     0,     0, },
+	{  256,  2048,  2048, },	/* limited range */
+	CLIP_FULL_RANGE,
+	"BT.601 Limited",
+};
+
+/* RGB -> YUV, ITU-R BT.709, full range */
+static const struct dispc7_csc_coef csc_rgb2yuv_bt709_full = {
+	dispc7_csc_rgb2yuv_regval,
+	{ 54,  183,  18,	/* yr,   yg,  yb | 0.1826  0.6142  0.0620|*/
+	 -30, -101, 131,	/* cbr, cbg, cbb |-0.1006 -0.3386  0.4392|*/
+	 131, -119, -12, },	/* crr, crg, crb | 0.4392 -0.3989 -0.0403|*/
+	{    0,     0,     0, },
+	{    0,  2048,  2048, },	/* full range */
+	CLIP_FULL_RANGE,
+	"BT.709 Full",
+};
+
+/* RGB -> YUV, ITU-R BT.709, limited range */
+static const struct dispc7_csc_coef csc_rgb2yuv_bt709_lim = {
+	dispc7_csc_rgb2yuv_regval,
+	{ 47,  157,   16,	/* yr,   yg,  yb | 0.1826  0.6142  0.0620|*/
+	 -26,  -87,  112,	/* cbr, cbg, cbb |-0.1006 -0.3386  0.4392|*/
+	 112, -102,  -10, },	/* crr, crg, crb | 0.4392 -0.3989 -0.0403|*/
+	{    0,     0,     0, },
+	{  256,  2048,  2048, },	/* limited range */
+	CLIP_FULL_RANGE,
+	"BT.709 Limited",
+};
+
+struct dispc7_csc_entry {
+	enum drm_color_encoding encoding;
+	enum drm_color_range range;
+	const struct dispc7_csc_coef *csc;
+};
+
+static const struct dispc7_csc_entry dispc7_yuv2rgb_table[] = {
+	{ DRM_COLOR_YCBCR_BT601, DRM_COLOR_YCBCR_FULL_RANGE,
+	  &csc_yuv2rgb_bt601_full, },
+	{ DRM_COLOR_YCBCR_BT601, DRM_COLOR_YCBCR_LIMITED_RANGE,
+	  &csc_yuv2rgb_bt601_lim, },
+	{ DRM_COLOR_YCBCR_BT709, DRM_COLOR_YCBCR_FULL_RANGE,
+	  &csc_yuv2rgb_bt709_full, },
+	{ DRM_COLOR_YCBCR_BT709, DRM_COLOR_YCBCR_LIMITED_RANGE,
+	  &csc_yuv2rgb_bt709_lim, },
+};
+
+static const struct dispc7_csc_entry dispc7_rgb2yuv_table[] = {
+	{ DRM_COLOR_YCBCR_BT601, DRM_COLOR_YCBCR_FULL_RANGE,
+	  &csc_rgb2yuv_bt601_full, },
+	{ DRM_COLOR_YCBCR_BT601, DRM_COLOR_YCBCR_LIMITED_RANGE,
+	  &csc_rgb2yuv_bt601_lim, },
+	{ DRM_COLOR_YCBCR_BT709, DRM_COLOR_YCBCR_FULL_RANGE,
+	  &csc_rgb2yuv_bt709_full, },
+	{ DRM_COLOR_YCBCR_BT709, DRM_COLOR_YCBCR_LIMITED_RANGE,
+	  &csc_rgb2yuv_bt709_lim, },
+};
+
+enum dispc7_csc_direction {
+	DISPC7_YUV2RGB,
+	DISPC7_RGB2YUV,
+};
+
+static const
+struct dispc7_csc_coef *dispc7_find_csc(enum dispc7_csc_direction direction,
+					enum drm_color_encoding encoding,
+					enum drm_color_range range)
+{
+	unsigned int i;
+	const struct dispc7_csc_entry *csc_table;
+
+	if (direction == DISPC7_YUV2RGB)
+		csc_table = dispc7_yuv2rgb_table;
+	else
+		csc_table = dispc7_rgb2yuv_table;
+
+	for (i = 0; i < ARRAY_SIZE(dispc7_yuv2rgb_table); i++) {
+		if (csc_table[i].encoding == encoding &&
+		    csc_table[i].range == range) {
+			return csc_table[i].csc;
+		}
+	}
+	return NULL;
+}
+
+static void dispc7_vid_csc_setup(struct dispc_device *dispc, u32 hw_plane,
+				 const struct drm_plane_state *state)
+{
+	static const struct dispc7_csc_coef *coef;
+
+	coef = dispc7_find_csc(DISPC7_YUV2RGB, state->color_encoding,
+			       state->color_range);
+	if (!coef) {
+		dev_err(dispc->dev, "%s: YUV2RGB CSC (%u,%u) not found\n",
+			__func__, state->color_encoding, state->color_range);
+		return;
+	}
+
+	dispc7_vid_write_csc(dispc, hw_plane, coef);
+}
+
+static void dispc7_vid_csc_enable(struct dispc_device *dispc, u32 hw_plane,
+				  bool enable)
+{
+	VID_REG_FLD_MOD(dispc, hw_plane, DISPC_VID_ATTRIBUTES, !!enable, 9, 9);
+}
+
+static void dispc7_wb_csc_setup(struct dispc_device *dispc,
+				const struct drm_plane_state *state)
+{
+	static const struct dispc7_csc_coef *coef;
+
+	coef = dispc7_find_csc(DISPC7_RGB2YUV, state->color_encoding,
+			       state->color_range);
+	if (!coef) {
+		dev_err(dispc->dev, "%s: RGB2YUV CSC (%u,%u) not found\n",
+			__func__, state->color_encoding, state->color_range);
+		return;
+	}
+	dispc7_wb_write_csc(dispc, coef);
+}
+
+static void dispc7_wb_csc_enable(struct dispc_device *dispc, bool enable)
+{
+	WB_REG_FLD_MOD(dispc, DISPC_WB_ATTRIBUTES, !!enable, 11, 11);
+}
+
+/* SCALER */
+
+static u32 dispc7_calc_fir_inc(u32 in, u32 out)
+{
+	return (u32)div_u64(0x200000ull * in, out);
+}
+
+enum dispc7_vid_fir_coef_set {
+	DISPC7_VID_FIR_COEF_HORIZ,
+	DISPC7_VID_FIR_COEF_HORIZ_UV,
+	DISPC7_VID_FIR_COEF_VERT,
+	DISPC7_VID_FIR_COEF_VERT_UV,
+};
+
+static void dispc7_vid_write_fir_coefs(struct dispc_device *dispc,
+				       u32 hw_plane,
+				       enum dispc7_vid_fir_coef_set coef_set,
+				       const struct tidss_scale_coefs *coefs)
+{
+	static const u16 c0_regs[] = {
+		[DISPC7_VID_FIR_COEF_HORIZ] = DISPC_VID_FIR_COEFS_H0,
+		[DISPC7_VID_FIR_COEF_HORIZ_UV] = DISPC_VID_FIR_COEFS_H0_C,
+		[DISPC7_VID_FIR_COEF_VERT] = DISPC_VID_FIR_COEFS_V0,
+		[DISPC7_VID_FIR_COEF_VERT_UV] = DISPC_VID_FIR_COEFS_V0_C,
+	};
+
+	static const u16 c12_regs[] = {
+		[DISPC7_VID_FIR_COEF_HORIZ] = DISPC_VID_FIR_COEFS_H12,
+		[DISPC7_VID_FIR_COEF_HORIZ_UV] = DISPC_VID_FIR_COEFS_H12_C,
+		[DISPC7_VID_FIR_COEF_VERT] = DISPC_VID_FIR_COEFS_V12,
+		[DISPC7_VID_FIR_COEF_VERT_UV] = DISPC_VID_FIR_COEFS_V12_C,
+	};
+
+	const u16 c0_base = c0_regs[coef_set];
+	const u16 c12_base = c12_regs[coef_set];
+	int phase;
+
+	if (!coefs) {
+		dev_err(dispc->dev, "%s: No coefficients given.\n", __func__);
+		return;
+	}
+
+	for (phase = 0; phase <= 8; ++phase) {
+		u16 reg = c0_base + phase * 4;
+		u16 c0 = coefs->c0[phase];
+
+		dispc7_vid_write(dispc, hw_plane, reg, c0);
+	}
+
+	for (phase = 0; phase <= 15; ++phase) {
+		u16 reg = c12_base + phase * 4;
+		s16 c1, c2;
+		u32 c12;
+
+		c1 = coefs->c1[phase];
+		c2 = coefs->c2[phase];
+		c12 = FLD_VAL(c1, 19, 10) | FLD_VAL(c2, 29, 20);
+
+		dispc7_vid_write(dispc, hw_plane, reg, c12);
+	}
+}
+
+enum dispc7_wb_fir_coef_set {
+	DISPC7_WB_FIR_COEF_HORIZ,
+	DISPC7_WB_FIR_COEF_HORIZ_UV,
+	DISPC7_WB_FIR_COEF_VERT,
+	DISPC7_WB_FIR_COEF_VERT_UV,
+};
+
+static void dispc7_wb_write_fir_coefs(struct dispc_device *dispc,
+				      enum dispc7_wb_fir_coef_set coef_set,
+				      const struct tidss_scale_coefs *coefs)
+{
+	static const u16 c0_regs[] = {
+		[DISPC7_WB_FIR_COEF_HORIZ] = DISPC_WB_FIR_COEFS_H0,
+		[DISPC7_WB_FIR_COEF_HORIZ_UV] = DISPC_WB_FIR_COEFS_H0_C,
+		[DISPC7_WB_FIR_COEF_VERT] = DISPC_WB_FIR_COEFS_V0,
+		[DISPC7_WB_FIR_COEF_VERT_UV] = DISPC_WB_FIR_COEFS_V0_C,
+	};
+
+	static const u16 c12_regs[] = {
+		[DISPC7_WB_FIR_COEF_HORIZ] = DISPC_WB_FIR_COEFS_H12,
+		[DISPC7_WB_FIR_COEF_HORIZ_UV] = DISPC_WB_FIR_COEFS_H12_C,
+		[DISPC7_WB_FIR_COEF_VERT] = DISPC_WB_FIR_COEFS_V12,
+		[DISPC7_WB_FIR_COEF_VERT_UV] = DISPC_WB_FIR_COEFS_V12_C,
+	};
+
+	const u16 c0_base = c0_regs[coef_set];
+	const u16 c12_base = c12_regs[coef_set];
+	int phase;
+
+	if (!coefs) {
+		dev_err(dispc->dev, "%s: No coefficients given.\n", __func__);
+		return;
+	}
+
+	for (phase = 0; phase <= 8; ++phase) {
+		u16 reg = c0_base + phase * 4;
+		u16 c0 = coefs->c0[phase];
+
+		dispc7_wb_write(dispc, reg, c0);
+	}
+
+	for (phase = 0; phase <= 15; ++phase) {
+		u16 reg = c12_base + phase * 4;
+		s16 c1, c2;
+		u32 c12;
+
+		c1 = coefs->c1[phase];
+		c2 = coefs->c2[phase];
+		c12 = FLD_VAL(c1, 19, 10) | FLD_VAL(c2, 29, 20);
+
+		dispc7_wb_write(dispc, reg, c12);
+	}
+}
+
+static bool dispc7_fourcc_is_yuv(u32 fourcc)
+{
+	switch (fourcc) {
+	case DRM_FORMAT_YUYV:
+	case DRM_FORMAT_UYVY:
+	case DRM_FORMAT_NV12:
+		return true;
+	default:
+		return false;
+	}
+}
+
+struct dispc7_scaling_params {
+	int xinc, yinc;
+	u32 in_w, in_h, in_w_uv, in_h_uv;
+	u32 out_w, out_h, out_w_uv, out_h_uv;
+	u32 fir_xinc, fir_yinc, fir_xinc_uv, fir_yinc_uv;
+	bool scale_x, scale_y;
+	const struct tidss_scale_coefs *xcoef, *ycoef, *xcoef_uv, *ycoef_uv;
+	bool five_taps;
+};
+
+static int dispc7_vid_calc_scaling(struct dispc_device *dispc,
+				   const struct drm_plane_state *state,
+				   struct dispc7_scaling_params *sp,
+				   bool lite_plane)
+{
+	const struct dispc7_features_scaling *f = &dispc->feat->scaling;
+	u32 fourcc = state->fb->format->format;
+	u32 in_width_max_5tap = f->in_width_max_5tap_rgb;
+	u32 in_width_max_3tap = f->in_width_max_3tap_rgb;
+	u32 downscale_limit;
+	u32 in_width_max;
+
+	memset(sp, 0, sizeof(*sp));
+	sp->xinc = sp->yinc = 1;
+	sp->in_w = sp->in_w_uv = state->src_w >> 16;
+	sp->in_h = sp->in_h_uv = state->src_h >> 16;
+
+	sp->scale_x = sp->in_w != state->crtc_w;
+	sp->scale_y = sp->in_h != state->crtc_h;
+
+	if (dispc7_fourcc_is_yuv(fourcc)) {
+		in_width_max_5tap = f->in_width_max_5tap_yuv;
+		in_width_max_3tap = f->in_width_max_3tap_yuv;
+
+		sp->in_w_uv >>= 1;
+		sp->scale_x = true;
+
+		if (fourcc == DRM_FORMAT_NV12) {
+			sp->in_h_uv >>= 1;
+			sp->scale_y = true;
+		}
+	}
+
+	/* Skip the rest if no scaling is used */
+	if ((!sp->scale_x && !sp->scale_y) || lite_plane)
+		return 0;
+
+	if (sp->in_w > in_width_max_5tap) {
+		sp->five_taps = false;
+		in_width_max = in_width_max_3tap;
+		downscale_limit = f->downscale_limit_3tap;
+	} else {
+		sp->five_taps = true;
+		in_width_max = in_width_max_5tap;
+		downscale_limit = f->downscale_limit_5tap;
+	}
+
+	if (sp->scale_x) {
+		sp->fir_xinc = dispc7_calc_fir_inc(sp->in_w, state->crtc_w);
+
+		if (sp->fir_xinc < dispc7_calc_fir_inc(1, f->upscale_limit)) {
+			dev_dbg(dispc->dev,
+				"%s: X-scaling factor %u/%u > %u\n",
+				__func__, state->crtc_w, state->src_w >> 16,
+				f->upscale_limit);
+			return -EINVAL;
+		}
+
+		if (sp->fir_xinc >= dispc7_calc_fir_inc(downscale_limit, 1)) {
+			sp->xinc = DIV_ROUND_UP(DIV_ROUND_UP(sp->in_w,
+							     state->crtc_w),
+						downscale_limit);
+
+			if (sp->xinc > f->xinc_max) {
+				dev_dbg(dispc->dev,
+					"%s: X-scaling factor %u/%u < 1/%u\n",
+					__func__, state->crtc_w,
+					state->src_w >> 16,
+					downscale_limit * f->xinc_max);
+				return -EINVAL;
+			}
+
+			sp->in_w = (state->src_w >> 16) / sp->xinc;
+		}
+
+		while (sp->in_w > in_width_max) {
+			sp->xinc++;
+			sp->in_w = (state->src_w >> 16) / sp->xinc;
+		}
+
+		if (sp->xinc > f->xinc_max) {
+			dev_dbg(dispc->dev,
+				"%s: Too wide input bufer %u > %u\n", __func__,
+				state->src_w >> 16, in_width_max * f->xinc_max);
+			return -EINVAL;
+		}
+
+		/*
+		 * We need even line length for YUV formats. Decimation
+		 * can lead to odd length, so we need to make it even
+		 * again.
+		 */
+		if (dispc7_fourcc_is_yuv(fourcc))
+			sp->in_w &= ~1;
+
+		sp->fir_xinc = dispc7_calc_fir_inc(sp->in_w, state->crtc_w);
+	}
+
+	if (sp->scale_y) {
+		sp->fir_yinc = dispc7_calc_fir_inc(sp->in_h, state->crtc_h);
+
+		if (sp->fir_yinc < dispc7_calc_fir_inc(1, f->upscale_limit)) {
+			dev_dbg(dispc->dev,
+				"%s: Y-scaling factor %u/%u > %u\n",
+				__func__, state->crtc_h, state->src_h >> 16,
+				f->upscale_limit);
+			return -EINVAL;
+		}
+
+		if (sp->fir_yinc >= dispc7_calc_fir_inc(downscale_limit, 1)) {
+			sp->yinc = DIV_ROUND_UP(DIV_ROUND_UP(sp->in_h,
+							     state->crtc_h),
+						downscale_limit);
+
+			sp->in_h /= sp->yinc;
+			sp->fir_yinc = dispc7_calc_fir_inc(sp->in_h,
+							   state->crtc_h);
+		}
+	}
+
+	dev_dbg(dispc->dev,
+		"%s: %ux%u decim %ux%u -> %ux%u firinc %u.%03ux%u.%03u taps %u -> %ux%u\n",
+		__func__, state->src_w >> 16, state->src_h >> 16,
+		sp->xinc, sp->yinc, sp->in_w, sp->in_h,
+		sp->fir_xinc / 0x200000u,
+		((sp->fir_xinc & 0x1FFFFFu) * 999u) / 0x1FFFFFu,
+		sp->fir_yinc / 0x200000u,
+		((sp->fir_yinc & 0x1FFFFFu) * 999u) / 0x1FFFFFu,
+		sp->five_taps ? 5 : 3,
+		state->crtc_w, state->crtc_h);
+
+	if (dispc7_fourcc_is_yuv(fourcc)) {
+		if (sp->scale_x) {
+			sp->in_w_uv /= sp->xinc;
+			sp->fir_xinc_uv = dispc7_calc_fir_inc(sp->in_w_uv,
+							      state->crtc_w);
+			sp->xcoef_uv = tidss_get_scale_coefs(dispc->dev,
+							     sp->fir_xinc_uv,
+							     true);
+		}
+		if (sp->scale_y) {
+			sp->in_h_uv /= sp->yinc;
+			sp->fir_yinc_uv = dispc7_calc_fir_inc(sp->in_h_uv,
+							      state->crtc_h);
+			sp->ycoef_uv = tidss_get_scale_coefs(dispc->dev,
+							     sp->fir_yinc_uv,
+							     sp->five_taps);
+		}
+	}
+
+	if (sp->scale_x)
+		sp->xcoef = tidss_get_scale_coefs(dispc->dev, sp->fir_xinc,
+						  true);
+
+	if (sp->scale_y)
+		sp->ycoef = tidss_get_scale_coefs(dispc->dev, sp->fir_yinc,
+						  sp->five_taps);
+
+	return 0;
+}
+
+static int dispc7_wb_calc_scaling(struct dispc_device *dispc,
+				  const struct drm_plane_state *state,
+				  struct dispc7_scaling_params *sp)
+{
+	const struct dispc7_features_scaling *f = &dispc->feat->scaling;
+	u32 fourcc = state->fb->format->format;
+	u32 in_width_max_5tap = f->in_width_max_5tap_rgb;
+	u32 in_width_max_3tap = f->in_width_max_3tap_rgb;
+	u32 downscale_limit;
+	u32 in_width_max;
+
+	memset(sp, 0, sizeof(*sp));
+	sp->xinc = sp->yinc = 1;
+	sp->in_w = sp->in_w_uv = state->src_w >> 16;
+	sp->in_h = sp->in_h_uv = state->src_h >> 16;
+	sp->out_w = sp->out_w_uv = state->crtc_w;
+	sp->out_h = sp->out_h_uv = state->crtc_h;
+
+	sp->scale_x = sp->in_w != sp->out_w;
+	sp->scale_y = sp->in_h != sp->out_h;
+
+	if (dispc7_fourcc_is_yuv(fourcc)) {
+		in_width_max_5tap = f->in_width_max_5tap_yuv;
+		in_width_max_3tap = f->in_width_max_3tap_yuv;
+
+		sp->out_w_uv >>= 1;
+		sp->scale_x = true;
+
+		if (fourcc == DRM_FORMAT_NV12) {
+			sp->out_h_uv >>= 1;
+			sp->scale_y = true;
+		}
+	}
+
+	/* Skip the rest if no scaling is used */
+	if (!sp->scale_x && !sp->scale_y)
+		return 0;
+
+	if (sp->in_w > in_width_max_5tap) {
+		sp->five_taps = false;
+		in_width_max = in_width_max_3tap;
+		downscale_limit = f->downscale_limit_3tap;
+	} else {
+		sp->five_taps = true;
+		in_width_max = in_width_max_5tap;
+		downscale_limit = f->downscale_limit_5tap;
+	}
+
+	if (sp->scale_x) {
+		sp->fir_xinc = dispc7_calc_fir_inc(sp->in_w, sp->out_w);
+
+		if (sp->fir_xinc < dispc7_calc_fir_inc(1, f->upscale_limit)) {
+			dev_dbg(dispc->dev,
+				"%s: X-scaling factor %u/%u > %u\n",
+				__func__, state->crtc_w, state->src_w >> 16,
+				f->upscale_limit);
+			return -EINVAL;
+		}
+
+		if (sp->fir_xinc >= dispc7_calc_fir_inc(downscale_limit, 1)) {
+			sp->xinc = DIV_ROUND_UP(DIV_ROUND_UP(sp->in_w,
+							     sp->out_w),
+						downscale_limit);
+
+			if (sp->xinc > f->xinc_max) {
+				dev_dbg(dispc->dev,
+					"%s: X-scaling factor %u/%u < 1/%u\n",
+					__func__, state->crtc_w,
+					state->src_w >> 16,
+					downscale_limit * f->xinc_max);
+				return -EINVAL;
+			}
+
+			sp->in_w = (state->src_w >> 16) / sp->xinc;
+		}
+
+		while (sp->in_w > in_width_max) {
+			sp->xinc++;
+			sp->in_w = (state->src_w >> 16) / sp->xinc;
+		}
+
+		if (sp->xinc > f->xinc_max) {
+			dev_dbg(dispc->dev,
+				"%s: Too wide input bufer %u > %u\n", __func__,
+				state->src_w >> 16, in_width_max * f->xinc_max);
+			return -EINVAL;
+		}
+
+		/*
+		 * We need even line length for YUV formats. Decimation
+		 * can lead to odd length, so we need to make it even
+		 * again.
+		 */
+		if (dispc7_fourcc_is_yuv(fourcc))
+			sp->in_w &= ~1;
+
+		sp->fir_xinc = dispc7_calc_fir_inc(sp->in_w, sp->out_w);
+	}
+
+	if (sp->scale_y) {
+		sp->fir_yinc = dispc7_calc_fir_inc(sp->in_h, sp->out_h);
+
+		if (sp->fir_yinc < dispc7_calc_fir_inc(1, f->upscale_limit)) {
+			dev_dbg(dispc->dev,
+				"%s: Y-scaling factor %u/%u > %u\n",
+				__func__, state->crtc_h, state->src_h >> 16,
+				f->upscale_limit);
+			return -EINVAL;
+		}
+
+		if (sp->fir_yinc >= dispc7_calc_fir_inc(downscale_limit, 1)) {
+			sp->yinc = DIV_ROUND_UP(DIV_ROUND_UP(sp->in_h,
+							     sp->out_h),
+						downscale_limit);
+
+			sp->in_h /= sp->yinc;
+			sp->fir_yinc = dispc7_calc_fir_inc(sp->in_h,
+							   sp->out_h);
+		}
+	}
+
+	dev_dbg(dispc->dev,
+		"%s: %ux%u decim %ux%u -> %ux%u firinc %u.%03ux%u.%03u taps %u -> %ux%u\n",
+		__func__, state->src_w >> 16, state->src_h >> 16,
+		sp->xinc, sp->yinc, sp->in_w, sp->in_h,
+		sp->fir_xinc / 0x200000u,
+		((sp->fir_xinc & 0x1FFFFFu) * 999u) / 0x1FFFFFu,
+		sp->fir_yinc / 0x200000u,
+		((sp->fir_yinc & 0x1FFFFFu) * 999u) / 0x1FFFFFu,
+		sp->five_taps ? 5 : 3,
+		state->crtc_w, state->crtc_h);
+
+	if (dispc7_fourcc_is_yuv(fourcc)) {
+		if (sp->scale_x) {
+			sp->in_w_uv /= sp->xinc;
+			sp->fir_xinc_uv = dispc7_calc_fir_inc(sp->in_w_uv,
+							      sp->out_w_uv);
+			sp->xcoef_uv = tidss_get_scale_coefs(dispc->dev,
+							     sp->fir_xinc_uv,
+							     true);
+		}
+		if (sp->scale_y) {
+			sp->in_h_uv /= sp->yinc;
+			sp->fir_yinc_uv = dispc7_calc_fir_inc(sp->in_h_uv,
+							      sp->out_h_uv);
+			sp->ycoef_uv = tidss_get_scale_coefs(dispc->dev,
+							     sp->fir_yinc_uv,
+							     sp->five_taps);
+		}
+	}
+
+	if (sp->scale_x)
+		sp->xcoef = tidss_get_scale_coefs(dispc->dev, sp->fir_xinc,
+						  true);
+
+	if (sp->scale_y)
+		sp->ycoef = tidss_get_scale_coefs(dispc->dev, sp->fir_yinc,
+						  sp->five_taps);
+
+	return 0;
+}
+
+static void dispc7_vid_set_scaling(struct dispc_device *dispc,
+				   u32 hw_plane,
+				   struct dispc7_scaling_params *sp,
+				   u32 fourcc)
+{
+	/* HORIZONTAL RESIZE ENABLE */
+	VID_REG_FLD_MOD(dispc, hw_plane, DISPC_VID_ATTRIBUTES,
+			sp->scale_x, 7, 7);
+
+	/* VERTICAL RESIZE ENABLE */
+	VID_REG_FLD_MOD(dispc, hw_plane, DISPC_VID_ATTRIBUTES,
+			sp->scale_y, 8, 8);
+
+	/* Skip the rest if no scaling is used */
+	if (!sp->scale_x && !sp->scale_y)
+		return;
+
+	/* VERTICAL 5-TAPS  */
+	VID_REG_FLD_MOD(dispc, hw_plane, DISPC_VID_ATTRIBUTES,
+			sp->five_taps, 21, 21);
+
+	if (dispc7_fourcc_is_yuv(fourcc)) {
+		if (sp->scale_x) {
+			dispc7_vid_write(dispc, hw_plane, DISPC_VID_FIRH2,
+					 sp->fir_xinc_uv);
+			dispc7_vid_write_fir_coefs(dispc, hw_plane,
+						   DISPC7_VID_FIR_COEF_HORIZ_UV,
+						   sp->xcoef_uv);
+		}
+		if (sp->scale_y) {
+			dispc7_vid_write(dispc, hw_plane, DISPC_VID_FIRV2,
+					 sp->fir_yinc_uv);
+			dispc7_vid_write_fir_coefs(dispc, hw_plane,
+						   DISPC7_VID_FIR_COEF_VERT_UV,
+						   sp->ycoef_uv);
+		}
+	}
+
+	if (sp->scale_x) {
+		dispc7_vid_write(dispc, hw_plane, DISPC_VID_FIRH, sp->fir_xinc);
+		dispc7_vid_write_fir_coefs(dispc, hw_plane,
+					   DISPC7_VID_FIR_COEF_HORIZ,
+					   sp->xcoef);
+	}
+
+	if (sp->scale_y) {
+		dispc7_vid_write(dispc, hw_plane, DISPC_VID_FIRV, sp->fir_yinc);
+		dispc7_vid_write_fir_coefs(dispc, hw_plane,
+					   DISPC7_VID_FIR_COEF_VERT, sp->ycoef);
+	}
+}
+
+static void dispc7_wb_set_scaling(struct dispc_device *dispc,
+				  struct dispc7_scaling_params *sp,
+				  u32 fourcc)
+{
+	/* HORIZONTAL RESIZE ENABLE */
+	WB_REG_FLD_MOD(dispc, DISPC_WB_ATTRIBUTES, sp->scale_x, 7, 7);
+
+	/* VERTICAL RESIZE ENABLE */
+	WB_REG_FLD_MOD(dispc, DISPC_WB_ATTRIBUTES, sp->scale_y, 8, 8);
+
+	/* Skip the rest if no scaling is used */
+	if (!sp->scale_x && !sp->scale_y)
+		return;
+
+	/* VERTICAL 5-TAPS  */
+	WB_REG_FLD_MOD(dispc, DISPC_WB_ATTRIBUTES, sp->five_taps, 21, 21);
+
+	if (dispc7_fourcc_is_yuv(fourcc)) {
+		if (sp->scale_x) {
+			dispc7_wb_write(dispc, DISPC_WB_FIRH2, sp->fir_xinc_uv);
+			dispc7_wb_write_fir_coefs(dispc,
+						  DISPC7_WB_FIR_COEF_HORIZ_UV,
+						  sp->xcoef_uv);
+		}
+		if (sp->scale_y) {
+			dispc7_wb_write(dispc, DISPC_WB_FIRV2, sp->fir_yinc_uv);
+			dispc7_wb_write_fir_coefs(dispc,
+						  DISPC7_WB_FIR_COEF_VERT_UV,
+						  sp->ycoef_uv);
+		}
+	}
+
+	if (sp->scale_x) {
+		dispc7_wb_write(dispc, DISPC_WB_FIRH, sp->fir_xinc);
+		dispc7_wb_write_fir_coefs(dispc, DISPC7_WB_FIR_COEF_HORIZ,
+					  sp->xcoef);
+	}
+
+	if (sp->scale_y) {
+		dispc7_wb_write(dispc, DISPC_WB_FIRV, sp->fir_yinc);
+		dispc7_wb_write_fir_coefs(dispc, DISPC7_WB_FIR_COEF_VERT,
+					  sp->ycoef);
+	}
+}
+
+/* OTHER */
+
+static const struct {
+	u32 fourcc;
+	u8 dss_code;
+} dispc7_color_formats[] = {
+	{ DRM_FORMAT_ARGB4444, 0x0, },
+	{ DRM_FORMAT_ABGR4444, 0x1, },
+	{ DRM_FORMAT_RGBA4444, 0x2, },
+
+	{ DRM_FORMAT_RGB565, 0x3, },
+	{ DRM_FORMAT_BGR565, 0x4, },
+
+	{ DRM_FORMAT_ARGB1555, 0x5, },
+	{ DRM_FORMAT_ABGR1555, 0x6, },
+
+	{ DRM_FORMAT_ARGB8888, 0x7, },
+	{ DRM_FORMAT_ABGR8888, 0x8, },
+	{ DRM_FORMAT_RGBA8888, 0x9, },
+	{ DRM_FORMAT_BGRA8888, 0xa, },
+
+	{ DRM_FORMAT_RGB888, 0xb, },
+	{ DRM_FORMAT_BGR888, 0xc, },
+
+	{ DRM_FORMAT_ARGB2101010, 0xe, },
+	{ DRM_FORMAT_ABGR2101010, 0xf, },
+
+	{ DRM_FORMAT_XRGB4444, 0x20, },
+	{ DRM_FORMAT_XBGR4444, 0x21, },
+	{ DRM_FORMAT_RGBX4444, 0x22, },
+
+	{ DRM_FORMAT_ARGB1555, 0x25, },
+	{ DRM_FORMAT_ABGR1555, 0x26, },
+
+	{ DRM_FORMAT_XRGB8888, 0x27, },
+	{ DRM_FORMAT_XBGR8888, 0x28, },
+	{ DRM_FORMAT_RGBX8888, 0x29, },
+	{ DRM_FORMAT_BGRX8888, 0x2a, },
+
+	{ DRM_FORMAT_XRGB2101010, 0x2e, },
+	{ DRM_FORMAT_XBGR2101010, 0x2f, },
+
+	{ DRM_FORMAT_YUYV, 0x3e, },
+	{ DRM_FORMAT_UYVY, 0x3f, },
+
+	{ DRM_FORMAT_NV12, 0x3d, },
+};
+
+static void dispc7_plane_set_pixel_format(struct dispc_device *dispc,
+					  u32 hw_plane, u32 fourcc)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(dispc7_color_formats); ++i) {
+		if (dispc7_color_formats[i].fourcc == fourcc) {
+			VID_REG_FLD_MOD(dispc, hw_plane, DISPC_VID_ATTRIBUTES,
+					dispc7_color_formats[i].dss_code,
+					6, 1);
+			return;
+		}
+	}
+
+	WARN_ON(1);
+}
+
+static void dispc7_wb_set_pixel_format(struct dispc_device *dispc,
+				       u32 fourcc)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(dispc7_color_formats); ++i) {
+		if (dispc7_color_formats[i].fourcc == fourcc) {
+			WB_REG_FLD_MOD(dispc, DISPC_WB_ATTRIBUTES,
+				       dispc7_color_formats[i].dss_code, 6, 1);
+			return;
+		}
+	}
+
+	WARN_ON(1);
+}
+
+static s32 pixinc(int pixels, u8 ps)
+{
+	if (pixels == 1)
+		return 1;
+	else if (pixels > 1)
+		return 1 + (pixels - 1) * ps;
+	else if (pixels < 0)
+		return 1 - (-pixels + 1) * ps;
+
+	WARN_ON(1);
+	return 0;
+}
+
+static
+const struct tidss_plane_feat *dispc7_plane_feat(struct dispc_device *dispc,
+						 u32 hw_plane)
+{
+	static const struct tidss_plane_feat pfeat = {
+		.color = {
+			.encodings = (BIT(DRM_COLOR_YCBCR_BT601) |
+				      BIT(DRM_COLOR_YCBCR_BT709)),
+			.ranges = (BIT(DRM_COLOR_YCBCR_FULL_RANGE) |
+				   BIT(DRM_COLOR_YCBCR_LIMITED_RANGE)),
+			.default_encoding = DRM_COLOR_YCBCR_BT601,
+			.default_range = DRM_COLOR_YCBCR_FULL_RANGE,
+		},
+		.blend = {
+			.global_alpha = true,
+		},
+	};
+
+	return &pfeat;
+}
+
+static int dispc7_plane_check(struct dispc_device *dispc, u32 hw_plane,
+			      const struct drm_plane_state *state,
+			      u32 hw_videoport)
+{
+	bool lite = dispc->feat->vid_lite[hw_plane];
+	u32 fourcc = state->fb->format->format;
+	bool need_scaling = state->src_w >> 16 != state->crtc_w ||
+		state->src_h >> 16 != state->crtc_h;
+	struct dispc7_scaling_params scaling;
+	int ret;
+
+	if (dispc7_fourcc_is_yuv(fourcc)) {
+		if (!dispc7_find_csc(DISPC7_YUV2RGB, state->color_encoding,
+				     state->color_range)) {
+			dev_dbg(dispc->dev,
+				"%s: Unsupported CSC (%u,%u) for HW plane %u\n",
+				__func__, state->color_encoding,
+				state->color_range, hw_plane);
+			return -EINVAL;
+		}
+	}
+
+	if (need_scaling) {
+		if (lite) {
+			dev_dbg(dispc->dev,
+				"%s: Lite plane %u can't scale %ux%u!=%ux%u\n",
+				__func__, hw_plane,
+				state->src_w >> 16, state->src_h >> 16,
+				state->crtc_w, state->crtc_h);
+			return -EINVAL;
+		}
+		ret = dispc7_vid_calc_scaling(dispc, state, &scaling, false);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int dispc7_plane_setup(struct dispc_device *dispc, u32 hw_plane,
+			      const struct drm_plane_state *state,
+			      u32 hw_videoport)
+{
+	bool lite = dispc->feat->vid_lite[hw_plane];
+	u32 fourcc = state->fb->format->format;
+	u16 cpp = state->fb->format->cpp[0];
+	u32 fb_width = state->fb->pitches[0] / cpp;
+	dma_addr_t paddr = dispc7_plane_state_paddr(state);
+	struct dispc7_scaling_params scale;
+
+	dispc7_vid_calc_scaling(dispc, state, &scale, lite);
+
+	dispc7_plane_set_pixel_format(dispc, hw_plane, fourcc);
+
+	dispc7_vid_write(dispc, hw_plane, DISPC_VID_BA_0, paddr & 0xffffffff);
+	dispc7_vid_write(dispc, hw_plane, DISPC_VID_BA_EXT_0, (u64)paddr >> 32);
+	dispc7_vid_write(dispc, hw_plane, DISPC_VID_BA_1, paddr & 0xffffffff);
+	dispc7_vid_write(dispc, hw_plane, DISPC_VID_BA_EXT_1, (u64)paddr >> 32);
+
+	dispc7_vid_write(dispc, hw_plane, DISPC_VID_PICTURE_SIZE,
+			 (scale.in_w - 1) | ((scale.in_h - 1) << 16));
+
+	/* For YUV422 format we use the macropixel size for pixel inc */
+	if (fourcc == DRM_FORMAT_YUYV || fourcc == DRM_FORMAT_UYVY)
+		dispc7_vid_write(dispc, hw_plane, DISPC_VID_PIXEL_INC,
+				 pixinc(scale.xinc, cpp * 2));
+	else
+		dispc7_vid_write(dispc, hw_plane, DISPC_VID_PIXEL_INC,
+				 pixinc(scale.xinc, cpp));
+
+	dispc7_vid_write(dispc, hw_plane, DISPC_VID_ROW_INC,
+			 pixinc(1 + (scale.yinc * fb_width -
+				     scale.xinc * scale.in_w),
+				cpp));
+
+	if (state->fb->format->num_planes == 2) {
+		u16 cpp_uv = state->fb->format->cpp[1];
+		u32 fb_width_uv = state->fb->pitches[1] / cpp_uv;
+		dma_addr_t p_uv_addr = dispc7_plane_state_p_uv_addr(state);
+
+		dispc7_vid_write(dispc, hw_plane,
+				 DISPC_VID_BA_UV_0, p_uv_addr & 0xffffffff);
+		dispc7_vid_write(dispc, hw_plane,
+				 DISPC_VID_BA_UV_EXT_0, (u64)p_uv_addr >> 32);
+		dispc7_vid_write(dispc, hw_plane,
+				 DISPC_VID_BA_UV_1, p_uv_addr & 0xffffffff);
+		dispc7_vid_write(dispc, hw_plane,
+				 DISPC_VID_BA_UV_EXT_1, (u64)p_uv_addr >> 32);
+
+		dispc7_vid_write(dispc, hw_plane, DISPC_VID_ROW_INC_UV,
+				 pixinc(1 + (scale.yinc * fb_width_uv -
+					     scale.xinc * scale.in_w_uv),
+					cpp_uv));
+	}
+
+	if (!lite) {
+		dispc7_vid_write(dispc, hw_plane, DISPC_VID_SIZE,
+				 (state->crtc_w - 1) |
+				 ((state->crtc_h - 1) << 16));
+
+		dispc7_vid_set_scaling(dispc, hw_plane, &scale, fourcc);
+	}
+
+	/* enable YUV->RGB color conversion */
+	if (dispc7_fourcc_is_yuv(fourcc)) {
+		dispc7_vid_csc_setup(dispc, hw_plane, state);
+		dispc7_vid_csc_enable(dispc, hw_plane, true);
+	} else {
+		dispc7_vid_csc_enable(dispc, hw_plane, false);
+	}
+
+	dispc7_vid_write(dispc, hw_plane, DISPC_VID_GLOBAL_ALPHA,
+			 0xFF & (state->alpha >> 8));
+
+	/* Set pre-multiplied alpha as default. */
+	VID_REG_FLD_MOD(dispc, hw_plane, DISPC_VID_ATTRIBUTES, 1, 28, 28);
+
+	dispc7_ovr_set_plane(dispc, hw_plane, hw_videoport,
+			     state->crtc_x, state->crtc_y,
+			     state->normalized_zpos);
+
+	dispc->plane_data[hw_plane].zorder = state->normalized_zpos;
+	dispc->plane_data[hw_plane].hw_videoport = hw_videoport;
+
+	return 0;
+}
+
+static int dispc7_plane_enable(struct dispc_device *dispc,
+			       u32 hw_plane, bool enable)
+{
+	dispc7_ovr_enable_plane(dispc, dispc->plane_data[hw_plane].hw_videoport,
+				dispc->plane_data[hw_plane].zorder, enable);
+
+	VID_REG_FLD_MOD(dispc, hw_plane, DISPC_VID_ATTRIBUTES, !!enable, 0, 0);
+
+	return 0;
+}
+
+static int dispc7_wb_setup(struct dispc_device *dispc,
+			   const struct drm_plane_state *state,
+			   bool mem_to_mem, u32 src_hw_plane,
+			   u32 src_hw_videoport)
+{
+	u32 fourcc = state->fb->format->format;
+	u16 cpp = state->fb->format->cpp[0];
+	u32 fb_width = state->fb->pitches[0] / cpp;
+	dma_addr_t paddr = dispc7_plane_state_paddr(state);
+	struct dispc7_scaling_params scale;
+	enum dss7_writeback_connections wb_conn;
+
+	switch (src_hw_videoport) {
+	case 0: /* OVR_1 */
+		wb_conn = DSS7_WB_OVR1;
+		break;
+	case 1: /* OVR_2 */
+		wb_conn = DSS7_WB_OVR2;
+		break;
+	case 2: /* OVR_3 */
+		wb_conn = DSS7_WB_OVR3;
+		break;
+	case 3: /* OVR_4 */
+		wb_conn = DSS7_WB_OVR4;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	dispc7_wb_calc_scaling(dispc, state, &scale);
+
+	dispc7_wb_set_pixel_format(dispc, fourcc);
+
+	dispc7_wb_write(dispc, DISPC_WB_BA_0, paddr & 0xffffffff);
+	dispc7_wb_write(dispc, DISPC_WB_BA_EXT_0, (u64)paddr >> 32);
+	dispc7_wb_write(dispc, DISPC_WB_BA_1, paddr & 0xffffffff);
+	dispc7_wb_write(dispc, DISPC_WB_BA_EXT_1, (u64)paddr >> 32);
+
+	dispc7_wb_write(dispc, DISPC_WB_SIZE,
+			 (scale.in_w - 1) | ((scale.in_h - 1) << 16));
+
+	dispc7_wb_write(dispc, DISPC_WB_ROW_INC,
+			 pixinc(1 + (scale.yinc * fb_width -
+				     scale.xinc * scale.out_w),
+				cpp));
+
+	if (state->fb->format->num_planes == 2) {
+		u16 cpp_uv = state->fb->format->cpp[1];
+		u32 fb_width_uv = state->fb->pitches[1] / cpp_uv;
+		dma_addr_t p_uv_addr = dispc7_plane_state_p_uv_addr(state);
+
+		dispc7_wb_write(dispc, DISPC_WB_BA_UV_0,
+				p_uv_addr & 0xffffffff);
+		dispc7_wb_write(dispc, DISPC_WB_BA_UV_EXT_0,
+				(u64)p_uv_addr >> 32);
+		dispc7_wb_write(dispc, DISPC_WB_BA_UV_1,
+				p_uv_addr & 0xffffffff);
+		dispc7_wb_write(dispc, DISPC_WB_BA_UV_EXT_1,
+				(u64)p_uv_addr >> 32);
+
+		dispc7_wb_write(dispc, DISPC_WB_ROW_INC_UV,
+				pixinc(1 + (scale.yinc * fb_width_uv -
+					    scale.xinc * scale.out_w_uv),
+				       cpp_uv));
+	}
+
+	dispc7_wb_write(dispc, DISPC_WB_PICTURE_SIZE,
+			(state->crtc_w - 1) | ((state->crtc_h - 1) << 16));
+
+	dispc7_wb_set_scaling(dispc, &scale, fourcc);
+
+	/* enable YUV->RGB color conversion */
+	if (dispc7_fourcc_is_yuv(fourcc)) {
+		dispc7_wb_csc_setup(dispc, state);
+		dispc7_wb_csc_enable(dispc, true);
+	} else {
+		dispc7_wb_csc_enable(dispc, false);
+	}
+
+	/* Set writeback mode */
+	WB_REG_FLD_MOD(dispc, DISPC_WB_ATTRIBUTES, mem_to_mem, 19, 19);
+
+	CFG_REG_FLD_MOD(dispc, DISPC_CONNECTIONS, wb_conn, 20, 16);
+
+	return 0;
+}
+
+static int dispc7_wb_enable(struct dispc_device *dispc, bool enable)
+{
+	WB_REG_FLD_MOD(dispc, DISPC_WB_ATTRIBUTES, !!enable, 0, 0);
+
+	return 0;
+}
+
+static bool dispc7_has_writeback(struct dispc_device *dispc)
+{
+	return dispc->wb_managed;
+}
+
+static u32 dispc7_vid_get_fifo_size(struct dispc_device *dispc, u32 hw_plane)
+{
+	return VID_REG_GET(dispc, hw_plane, DISPC_VID_BUF_SIZE_STATUS, 15, 0);
+}
+
+static void dispc7_vid_set_mflag_threshold(struct dispc_device *dispc,
+					   u32 hw_plane, u32 low, u32 high)
+{
+	dispc7_vid_write(dispc, hw_plane, DISPC_VID_MFLAG_THRESHOLD,
+			 FLD_VAL(high, 31, 16) | FLD_VAL(low, 15, 0));
+}
+
+static void dispc7_vid_set_buf_threshold(struct dispc_device *dispc,
+					 u32 hw_plane, u32 low, u32 high)
+{
+	dispc7_vid_write(dispc, hw_plane, DISPC_VID_BUF_THRESHOLD,
+			 FLD_VAL(high, 31, 16) | FLD_VAL(low, 15, 0));
+}
+
+static u32 dispc7_wb_get_fifo_size(struct dispc_device *dispc)
+{
+	return WB_REG_GET(dispc, DISPC_VID_BUF_SIZE_STATUS, 15, 0);
+}
+
+static void dispc7_wb_set_mflag_threshold(struct dispc_device *dispc,
+					  u32 low, u32 high)
+{
+	dispc7_wb_write(dispc, DISPC_WB_MFLAG_THRESHOLD,
+			FLD_VAL(high, 31, 16) | FLD_VAL(low, 15, 0));
+}
+
+static void dispc7_wb_set_buf_threshold(struct dispc_device *dispc,
+					 u32 low, u32 high)
+{
+	dispc7_wb_write(dispc, DISPC_WB_BUF_THRESHOLD,
+			FLD_VAL(high, 31, 16) | FLD_VAL(low, 15, 0));
+}
+
+static void dispc7_plane_init(struct dispc_device *dispc)
+{
+	unsigned int hw_plane;
+
+	dev_dbg(dispc->dev, "%s()\n", __func__);
+
+	if (dispc->has_cfg_common) {
+		u32 cba_lo_pri = 1;
+		u32 cba_hi_pri = 0;
+
+		CFG_REG_FLD_MOD(dispc, DSS_CBA_CFG, cba_lo_pri, 2, 0);
+		CFG_REG_FLD_MOD(dispc, DSS_CBA_CFG, cba_hi_pri, 5, 3);
+
+		/* MFLAG_CTRL = ENABLED */
+		CFG_REG_FLD_MOD(dispc, DISPC_GLOBAL_MFLAG_ATTRIBUTE, 2, 1, 0);
+		/* MFLAG_START = MFLAGNORMALSTARTMODE */
+		CFG_REG_FLD_MOD(dispc, DISPC_GLOBAL_MFLAG_ATTRIBUTE, 0, 6, 6);
+	}
+
+	dispc_for_each_managed_plane(dispc, hw_plane) {
+		u32 size = dispc7_vid_get_fifo_size(dispc, hw_plane);
+		u32 thr_low, thr_high;
+		u32 mflag_low, mflag_high;
+		u32 preload;
+
+		thr_high = size - 1;
+		thr_low = size / 2;
+
+		mflag_high = size * 2 / 3;
+		mflag_low = size / 3;
+
+		preload = thr_low;
+
+		dev_dbg(dispc->dev,
+			"%s: bufsize %u, buf_threshold %u/%u, mflag threshold %u/%u preload %u\n",
+			dispc7_plane_name(dispc, hw_plane),
+			size,
+			thr_high, thr_low,
+			mflag_high, mflag_low,
+			preload);
+
+		dispc7_vid_set_buf_threshold(dispc, hw_plane,
+					     thr_low, thr_high);
+		dispc7_vid_set_mflag_threshold(dispc, hw_plane,
+					       mflag_low, mflag_high);
+
+		dispc7_vid_write(dispc, hw_plane, DISPC_VID_PRELOAD, preload);
+
+		/*
+		 * Prefech up to fifo high-threshold value to minimize the
+		 * possibility of underflows. Note that this means the PRELOAD
+		 * register is ignored.
+		 */
+		VID_REG_FLD_MOD(dispc, hw_plane, DISPC_VID_ATTRIBUTES, 1, 19, 19);
+	}
+
+	if (dispc7_has_writeback(dispc)) {
+		u32 size = dispc7_wb_get_fifo_size(dispc);
+		u32 thr_low, thr_high;
+		u32 mflag_low, mflag_high;
+
+		thr_high = size - 1;
+		thr_low = size / 2;
+
+		mflag_high = size * 2 / 3;
+		mflag_low = size / 3;
+
+		dev_dbg(dispc->dev,
+			"wb: bufsize %u, buf_threshold %u/%u, mflag threshold %u/%u\n",
+			size,
+			thr_high, thr_low,
+			mflag_high, mflag_low);
+
+		dispc7_wb_set_buf_threshold(dispc, thr_low, thr_high);
+		dispc7_wb_set_mflag_threshold(dispc, mflag_low, mflag_high);
+	}
+}
+
+static void dispc7_vp_init(struct dispc_device *dispc)
+{
+	unsigned int i;
+
+	dev_dbg(dispc->dev, "%s()\n", __func__);
+
+	/* Enable the gamma Shadow bit-field for all VPs*/
+	dispc_for_each_managed_vp(dispc, i)
+		VP_REG_FLD_MOD(dispc, i, DISPC_VP_CONFIG, 1, 2, 2);
+}
+
+static void dispc7_initial_config(struct dispc_device *dispc)
+{
+	dispc7_plane_init(dispc);
+	dispc7_vp_init(dispc);
+
+	/* Note: Harcdoded DPI routing on J721E for now */
+	if (dispc->feat->subrev == DSS7_J721E && dispc->has_cfg_common) {
+		dispc7_cfg_write(dispc, DISPC_CONNECTIONS,
+			     FLD_VAL(2, 3, 0) |		/* VP1 to DPI0 */
+			     FLD_VAL(8, 7, 4)		/* VP3 to DPI1 */
+			    );
+	}
+}
+
+static int dispc7_get_num_planes(struct dispc_device *dispc)
+{
+	return dispc->feat->num_planes;
+}
+
+static int dispc7_get_num_vps(struct dispc_device *dispc)
+{
+	return dispc->feat->num_vps;
+}
+
+static const struct tidss_vp_feat *dispc7_vp_feat(struct dispc_device *dispc,
+						  u32 hw_videoport)
+{
+	return &dispc->feat->vp_feat;
+}
+
+static void dispc7_vp_write_gamma_table(struct dispc_device *dispc,
+					u32 hw_videoport)
+{
+	u32 *table = dispc->vp_data[hw_videoport].gamma_table;
+	u32 hwlen = dispc->feat->vp_feat.color.gamma_size;
+	unsigned int i;
+
+	dev_dbg(dispc->dev, "%s: hw_videoport %d\n", __func__, hw_videoport);
+
+	for (i = 0; i < hwlen; ++i) {
+		u32 v = table[i];
+
+		switch (dispc->feat->vp_feat.color.gamma_type) {
+		case TIDSS_GAMMA_8BIT:
+			v |= i << 24;
+			break;
+		case TIDSS_GAMMA_10BIT:
+			if (i == 0)
+				v |= 1 << 31;
+			break;
+		default:
+			WARN_ON(1);
+			return;
+		}
+
+		dispc7_vp_write(dispc, hw_videoport, DISPC_VP_GAMMA_TABLE, v);
+	}
+}
+
+static void dispc7_restore_gamma_tables(struct dispc_device *dispc)
+{
+	unsigned int i;
+
+	dev_dbg(dispc->dev, "%s()\n", __func__);
+
+	dispc_for_each_managed_vp(dispc, i)
+		dispc7_vp_write_gamma_table(dispc, i);
+}
+
+static const struct drm_color_lut dispc7_vp_gamma_default_lut[] = {
+	{ .red = 0, .green = 0, .blue = 0, },
+	{ .red = U16_MAX, .green = U16_MAX, .blue = U16_MAX, },
+};
+
+static void dispc7_vp_set_gamma(struct dispc_device *dispc,
+				u32 hw_videoport,
+				const struct drm_color_lut *lut,
+				unsigned int length)
+{
+	u32 *table = dispc->vp_data[hw_videoport].gamma_table;
+	u32 hwlen = dispc->feat->vp_feat.color.gamma_size;
+	u32 hwbits;
+	unsigned int i;
+
+	dev_dbg(dispc->dev, "%s: hw_videoport %d, lut len %u, hw len %u\n",
+		__func__, hw_videoport, length, hwlen);
+
+	if (dispc->feat->vp_feat.color.gamma_type == TIDSS_GAMMA_10BIT)
+		hwbits = 10;
+	else
+		hwbits = 8;
+
+	if (lut == NULL || length < 2) {
+		lut = dispc7_vp_gamma_default_lut;
+		length = ARRAY_SIZE(dispc7_vp_gamma_default_lut);
+	}
+
+	for (i = 0; i < length - 1; ++i) {
+		unsigned int first = i * (hwlen - 1) / (length - 1);
+		unsigned int last = (i + 1) * (hwlen - 1) / (length - 1);
+		unsigned int w = last - first;
+		u16 r, g, b;
+		unsigned int j;
+
+		if (w == 0)
+			continue;
+
+		for (j = 0; j <= w; j++) {
+			r = (lut[i].red * (w - j) + lut[i + 1].red * j) / w;
+			g = (lut[i].green * (w - j) + lut[i + 1].green * j) / w;
+			b = (lut[i].blue * (w - j) + lut[i + 1].blue * j) / w;
+
+			r >>= 16 - hwbits;
+			g >>= 16 - hwbits;
+			b >>= 16 - hwbits;
+
+			table[first + j] = (r << (hwbits * 2)) |
+					   (g << hwbits) | b;
+		}
+	}
+
+	if (dispc->is_enabled)
+		dispc7_vp_write_gamma_table(dispc, hw_videoport);
+}
+
+static s16 dispc7_S31_32_to_s2_8(s64 coef)
+{
+	u64 sign_bit = 1ULL << 63;
+	u64 cbits = (u64) coef;
+	s16 ret = clamp_val(((cbits & ~sign_bit) >> 24), 0, 0x1FF);
+
+	if (cbits & sign_bit)
+		ret = -ret;
+
+	return ret;
+}
+
+static void dispc7_cpr_csc_from_ctm(const struct drm_color_ctm *ctm,
+				    struct dispc7_csc_coef *cpr)
+{
+	memset(cpr, 0, sizeof(*cpr));
+
+	cpr->to_regval = dispc7_csc_cpr_regval;
+	cpr->m[CSC_RR] = dispc7_S31_32_to_s2_8(ctm->matrix[0]);
+	cpr->m[CSC_RG] = dispc7_S31_32_to_s2_8(ctm->matrix[1]);
+	cpr->m[CSC_RB] = dispc7_S31_32_to_s2_8(ctm->matrix[2]);
+	cpr->m[CSC_GR] = dispc7_S31_32_to_s2_8(ctm->matrix[3]);
+	cpr->m[CSC_GG] = dispc7_S31_32_to_s2_8(ctm->matrix[4]);
+	cpr->m[CSC_GB] = dispc7_S31_32_to_s2_8(ctm->matrix[5]);
+	cpr->m[CSC_BR] = dispc7_S31_32_to_s2_8(ctm->matrix[6]);
+	cpr->m[CSC_BG] = dispc7_S31_32_to_s2_8(ctm->matrix[7]);
+	cpr->m[CSC_BB] = dispc7_S31_32_to_s2_8(ctm->matrix[8]);
+}
+
+static void dispc7_vp_write_csc(struct dispc_device *dispc, u32 hw_videoport,
+				const struct dispc7_csc_coef *csc)
+{
+	static const u16 dispc_vp_csc_coef_reg[DISPC7_CSC_REGVAL_LEN] = {
+		DISPC_VP_CSC_COEF0, DISPC_VP_CSC_COEF1, DISPC_VP_CSC_COEF2,
+		DISPC_VP_CSC_COEF3, DISPC_VP_CSC_COEF4, DISPC_VP_CSC_COEF5,
+		DISPC_VP_CSC_COEF6, DISPC_VP_CSC_COEF7,
+	};
+	u32 regval[DISPC7_CSC_REGVAL_LEN];
+	unsigned int i;
+
+	csc->to_regval(csc, regval);
+
+	for (i = 0; i < ARRAY_SIZE(regval); i++)
+		dispc7_vp_write(dispc, hw_videoport, dispc_vp_csc_coef_reg[i],
+				regval[i]);
+}
+
+static void dispc7_vp_set_color_mgmt(struct dispc_device *dispc,
+				     u32 hw_videoport,
+				     const struct drm_crtc_state *state)
+{
+	struct drm_color_lut *lut = NULL;
+	unsigned int length = 0;
+	bool colorconvenable = false;
+
+	if (!state->color_mgmt_changed)
+		return;
+
+	if (state->gamma_lut) {
+		lut = (struct drm_color_lut *) state->gamma_lut->data;
+		length = state->gamma_lut->length / sizeof(*lut);
+	}
+
+	if (state->ctm) {
+		struct drm_color_ctm *ctm =
+			(struct drm_color_ctm *) state->ctm->data;
+		struct dispc7_csc_coef cpr;
+
+		dispc7_cpr_csc_from_ctm(ctm, &cpr);
+		dispc7_vp_write_csc(dispc, hw_videoport, &cpr);
+
+		colorconvenable = true;
+	}
+
+	dispc7_vp_set_gamma(dispc, hw_videoport, lut, length);
+	VP_REG_FLD_MOD(dispc, hw_videoport, DISPC_VP_CONFIG,
+		       colorconvenable, 24, 24);
+}
+
+static void dispc7_vp_setup(struct dispc_device *dispc, u32 hw_videoport,
+			    const struct drm_crtc_state *state)
+{
+	dispc7_vp_set_default_color(dispc, hw_videoport, 0);
+	dispc7_vp_set_color_mgmt(dispc, hw_videoport, state);
+}
+
+static int dispc7_init_gamma_tables(struct dispc_device *dispc)
+{
+	unsigned int i;
+
+	dev_dbg(dispc->dev, "%s()\n", __func__);
+
+	dispc_for_each_managed_vp(dispc, i)
+		dispc7_vp_set_gamma(dispc, i, NULL, 0);
+
+	return 0;
+}
+
+static const char *dispc7_plane_name(struct dispc_device *dispc, u32 hw_plane)
+{
+	if (WARN_ON(hw_plane >= dispc->feat->num_planes))
+		return "ERROR";
+
+	return dispc->feat->vid_name[hw_plane];
+}
+
+static const char *dispc7_vp_name(struct dispc_device *dispc, u32 hw_videoport)
+{
+	if (WARN_ON(hw_videoport >= dispc->feat->num_vps))
+		return "ERROR";
+
+	return dispc->feat->vp_name[hw_videoport];
+}
+
+static int dispc7_runtime_suspend(struct dispc_device *dispc)
+{
+	dev_dbg(dispc->dev, "suspend\n");
+
+	dispc->is_enabled = false;
+
+	clk_disable_unprepare(dispc->fclk);
+
+	return 0;
+}
+
+static int dispc7_runtime_resume(struct dispc_device *dispc)
+{
+	dev_dbg(dispc->dev, "resume\n");
+
+	clk_prepare_enable(dispc->fclk);
+
+	if (!dispc->has_cfg_common)
+		goto no_cfg;
+
+	if (CFG_REG_GET(dispc, DSS_SYSSTATUS, 0, 0) == 0)
+		dev_warn(dispc->dev, "DSS FUNC RESET not done!\n");
+
+	dev_dbg(dispc->dev, "OMAP DSS7 rev 0x%x\n",
+		dispc7_cfg_read(dispc, DSS_REVISION));
+
+	dev_dbg(dispc->dev, "VP RESETDONE %d,%d,%d\n",
+		CFG_REG_GET(dispc, DSS_SYSSTATUS, 1, 1),
+		CFG_REG_GET(dispc, DSS_SYSSTATUS, 2, 2),
+		CFG_REG_GET(dispc, DSS_SYSSTATUS, 3, 3));
+
+	dev_dbg(dispc->dev, "OLDI RESETDONE %d,%d,%d\n",
+		CFG_REG_GET(dispc, DSS_SYSSTATUS, 5, 5),
+		CFG_REG_GET(dispc, DSS_SYSSTATUS, 6, 6),
+		CFG_REG_GET(dispc, DSS_SYSSTATUS, 7, 7));
+
+	dev_dbg(dispc->dev, "DISPC IDLE %d\n",
+		CFG_REG_GET(dispc, DSS_SYSSTATUS, 9, 9));
+
+no_cfg:
+	dispc7_initial_config(dispc);
+
+	dispc7_restore_gamma_tables(dispc);
+
+	dispc->is_enabled = true;
+
+	return 0;
+}
+
+static void dispc7_wb_find_free_ovr(struct dispc_device *dispc)
+{
+	struct tidss_device *tidss = dispc->tidss;
+	int i, j;
+	bool found;
+	u32 ovr_id = 0xff;
+
+	dispc_for_each_managed_vp(dispc, i) {
+		found = false;
+		for (j = 0; j < tidss->num_crtcs; j++) {
+			struct drm_crtc *crtc = tidss->crtcs[j];
+			struct tidss_crtc *tcrtc = to_tidss_crtc(crtc);
+
+			if (tcrtc->hw_videoport == i) {
+				found = true;
+				break;
+			}
+		}
+		if (!found) {
+			/* this means this ovr is available */
+			ovr_id = i;
+			break;
+		}
+	}
+
+	if (ovr_id != 0xff)
+		dispc->wb_reserved_ovr = ovr_id;
+	else
+		dispc->wb_managed = false;
+}
+
+static u32 dispc7_wb_get_reserved_ovr(struct dispc_device *dispc)
+{
+	return dispc->wb_reserved_ovr;
+}
+
+static int dispc7_modeset_init(struct dispc_device *dispc)
+{
+	struct tidss_device *tidss = dispc->tidss;
+	struct device *dev = tidss->dev;
+	u32 fourccs[ARRAY_SIZE(dispc7_color_formats)];
+	unsigned int i, num_fourccs;
+
+	struct pipe {
+		u32 hw_videoport;
+		struct drm_bridge *bridge;
+		u32 enc_type;
+	};
+
+	struct pipe pipes[DISPC7_MAX_PORTS];
+	u32 num_pipes = 0;
+	u32 plane_idx = 0;
+	u32 crtc_mask;
+
+	num_fourccs = 0;
+	for (i = 0; i < ARRAY_SIZE(fourccs); ++i) {
+		if (dispc->feat->errata.i2000 &&
+		    dispc7_fourcc_is_yuv(dispc7_color_formats[i].fourcc))
+			continue;
+
+		fourccs[num_fourccs++] = dispc7_color_formats[i].fourcc;
+	}
+
+	/* first find all the connected panels & bridges */
+	/* exclude the VPs that are not managed.         */
+
+	dispc_for_each_managed_vp(dispc, i) {
+		struct drm_panel *panel;
+		struct drm_bridge *bridge;
+		u32 enc_type = DRM_MODE_ENCODER_NONE;
+		int ret;
+
+		ret = drm_of_find_panel_or_bridge(dev->of_node, i, 0,
+						  &panel, &bridge);
+		if (ret == -ENODEV) {
+			dev_dbg(dev, "no panel/bridge for port %d\n", i);
+			continue;
+		} else if (ret) {
+			dev_dbg(dev, "port %d probe returned %d\n", i, ret);
+			return ret;
+		}
+
+		if (panel) {
+			u32 conn_type;
+
+			dev_dbg(dev, "Setting up panel for port %d\n", i);
+
+			switch (dispc->feat->vp_bus_type[i]) {
+			case DISPC7_VP_OLDI:
+				enc_type = DRM_MODE_ENCODER_LVDS;
+				conn_type = DRM_MODE_CONNECTOR_LVDS;
+				break;
+			case DISPC7_VP_DPI:
+				enc_type = DRM_MODE_ENCODER_DPI;
+				conn_type = DRM_MODE_CONNECTOR_DPI;
+				break;
+			default:
+				dev_warn(dev, "%s: Bad vp bus type: %d\n",
+					 __func__, dispc->feat->vp_bus_type[i]);
+				conn_type = DRM_MODE_CONNECTOR_Unknown;
+				break;
+			}
+
+			bridge = devm_drm_panel_bridge_add(dev, panel,
+							   conn_type);
+			if (IS_ERR(bridge)) {
+				dev_err(dev, "failed to set up panel bridge for port %d\n", i);
+				return PTR_ERR(bridge);
+			}
+		}
+
+		pipes[num_pipes].hw_videoport = i;
+		pipes[num_pipes].bridge = bridge;
+		pipes[num_pipes].enc_type = enc_type;
+		num_pipes++;
+	}
+
+	/* all planes can be on any crtc */
+	crtc_mask = (1 << num_pipes) - 1;
+
+	/* then create a plane, a crtc and an encoder for each panel/bridge */
+
+	for (i = 0; i < num_pipes; ++i) {
+		struct tidss_plane *tplane;
+		struct tidss_crtc *tcrtc;
+		struct drm_encoder *enc;
+		u32 hw_plane_id;
+		int ret;
+
+		hw_plane_id = dispc7_get_next_managed_plane(dispc, &plane_idx);
+		if (hw_plane_id == dispc->feat->num_planes) {
+			dev_err(tidss->dev, "no managed HW plane found for CRTC\n");
+			return -EINVAL;
+		}
+
+		tplane = tidss_plane_create(tidss, hw_plane_id,
+					    DRM_PLANE_TYPE_PRIMARY, crtc_mask,
+					    fourccs, num_fourccs);
+		if (IS_ERR(tplane)) {
+			dev_err(tidss->dev, "plane create failed\n");
+			return PTR_ERR(tplane);
+		}
+
+		hw_plane_id++;
+
+		tidss->planes[tidss->num_planes++] = &tplane->plane;
+
+		tcrtc = tidss_crtc_create(tidss, pipes[i].hw_videoport,
+					  &tplane->plane);
+		if (IS_ERR(tcrtc)) {
+			dev_err(tidss->dev, "crtc create failed\n");
+			return PTR_ERR(tcrtc);
+		}
+
+		tidss->crtcs[tidss->num_crtcs++] = &tcrtc->crtc;
+
+		enc = tidss_encoder_create(tidss, pipes[i].enc_type,
+					   1 << tcrtc->crtc.index);
+		if (IS_ERR(enc)) {
+			dev_err(tidss->dev, "encoder create failed\n");
+			return PTR_ERR(enc);
+		}
+
+		ret = drm_bridge_attach(enc, pipes[i].bridge, NULL);
+		if (ret) {
+			dev_err(tidss->dev, "bridge attach failed: %d\n", ret);
+			return ret;
+		}
+	}
+
+	/* Try to find an available OVR to use for WB */
+	if (dispc7_has_writeback(dispc))
+		dispc7_wb_find_free_ovr(dispc);
+
+	/* create overlay planes of the leftover planes */
+
+	while (tidss->num_planes < dispc->feat->num_planes) {
+		struct tidss_plane *tplane;
+		u32 hw_plane_id;
+
+		hw_plane_id = dispc7_get_next_managed_plane(dispc, &plane_idx);
+		if (hw_plane_id == dispc->feat->num_planes)
+			break;
+
+		tplane = tidss_plane_create(tidss, hw_plane_id,
+					    DRM_PLANE_TYPE_OVERLAY, crtc_mask,
+					    fourccs, num_fourccs);
+
+		if (IS_ERR(tplane)) {
+			dev_err(tidss->dev, "plane create failed\n");
+			return PTR_ERR(tplane);
+		}
+
+		hw_plane_id++;
+
+		tidss->planes[tidss->num_planes++] = &tplane->plane;
+	}
+
+	return 0;
+}
+
+static int dispc7_get_irq(struct dispc_device *dispc)
+{
+	return dispc->irq;
+}
+
+static void dispc7_remove(struct dispc_device *dispc)
+{
+	struct device *dev = dispc->dev;
+
+	dev_dbg(dev, "%s\n", __func__);
+
+	dispc->tidss->dispc_ops = NULL;
+	dispc->tidss->dispc = NULL;
+}
+
+static const struct tidss_dispc_ops dispc7_ops = {
+	.read_and_clear_irqstatus = dispc7_read_and_clear_irqstatus,
+	.write_irqenable = dispc7_write_irqenable,
+
+	.get_num_vps = dispc7_get_num_vps,
+	.vp_name = dispc7_vp_name,
+	.vp_feat = dispc7_vp_feat,
+	.vp_mode_valid = dispc7_vp_mode_valid,
+	.vp_check = dispc7_vp_check,
+	.vp_setup = dispc7_vp_setup,
+	.vp_prepare = dispc7_vp_prepare,
+	.vp_enable = dispc7_vp_enable,
+	.vp_disable = dispc7_vp_disable,
+	.vp_unprepare = dispc7_vp_unprepare,
+	.vp_go_busy = dispc7_vp_go_busy,
+	.vp_go = dispc7_vp_go,
+
+	.vp_set_clk_rate = dispc7_vp_set_clk_rate,
+	.vp_enable_clk = dispc7_vp_enable_clk,
+	.vp_disable_clk = dispc7_vp_disable_clk,
+
+	.get_num_planes = dispc7_get_num_planes,
+	.plane_name = dispc7_plane_name,
+	.plane_feat = dispc7_plane_feat,
+	.plane_enable = dispc7_plane_enable,
+	.plane_check = dispc7_plane_check,
+	.plane_setup = dispc7_plane_setup,
+
+	.runtime_get = dispc7_runtime_get,
+	.runtime_put = dispc7_runtime_put,
+
+	.runtime_suspend = dispc7_runtime_suspend,
+	.runtime_resume = dispc7_runtime_resume,
+
+	.remove = dispc7_remove,
+
+	.modeset_init = dispc7_modeset_init,
+
+	.get_irq = dispc7_get_irq,
+
+	.has_writeback = dispc7_has_writeback,
+	.wb_setup = dispc7_wb_setup,
+	.wb_enable = dispc7_wb_enable,
+	.wb_get_reserved_ovr = dispc7_wb_get_reserved_ovr,
+};
+
+static int dispc7_iomap_resource(struct platform_device *pdev, const char *name,
+				 void __iomem **base)
+{
+	struct resource *res;
+	void __iomem *b;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, name);
+	if (!res) {
+		dev_err(&pdev->dev, "cannot get mem resource '%s'\n", name);
+		return -EINVAL;
+	}
+
+	b = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(b)) {
+		dev_err(&pdev->dev, "cannot ioremap resource '%s'\n", name);
+		return PTR_ERR(b);
+	}
+
+	*base = b;
+
+	return 0;
+}
+
+static int dispc_j721e_get_managed_common_intr(struct dispc_device *dispc,
+		u32 *intr)
+{
+	int ret;
+	struct tidss_device *tidss = dispc->tidss;
+	struct device *dev = tidss->dev;
+	struct device_node *dss_commons_node;
+	struct device_node *intr_node;
+	u32 value;
+
+	dss_commons_node = of_get_child_by_name(dev->of_node, "dss-commons");
+	if (!dss_commons_node) {
+		*intr = 0;
+		return 0;
+	}
+
+	intr_node = of_get_child_by_name(dss_commons_node, "interrupt-common");
+	if (!intr_node) {
+		ret = -ENOENT;
+		goto out;
+	}
+
+	ret = of_property_read_u32(intr_node, "reg", &value);
+	if (ret)
+		goto out2;
+
+	*intr = value;
+
+out2:
+	of_node_put(intr_node);
+out:
+	of_node_put(dss_commons_node);
+	return ret;
+}
+
+static int dispc_j721e_get_managed_common_cfg(struct dispc_device *dispc,
+		u32 *cfg)
+{
+	int ret;
+	struct tidss_device *tidss = dispc->tidss;
+	struct device *dev = tidss->dev;
+	struct device_node *dss_commons_node;
+	struct device_node *cfg_node;
+	u32 value;
+
+	dss_commons_node = of_get_child_by_name(dev->of_node, "dss-commons");
+	if (!dss_commons_node) {
+		*cfg = 0;
+		return 0;
+	}
+
+	cfg_node = of_get_child_by_name(dss_commons_node, "config-common");
+	if (!cfg_node) {
+		ret = -ENOENT;
+		goto out;
+	}
+
+	if (!of_device_is_available(cfg_node)) {
+		ret = -ENOENT;
+		goto out;
+	}
+
+	ret = of_property_read_u32(cfg_node, "reg", &value);
+	if (ret)
+		goto out2;
+
+	*cfg = value;
+
+out2:
+	of_node_put(cfg_node);
+out:
+	of_node_put(dss_commons_node);
+	return ret;
+}
+
+/*
+ * The logic for J721E is simple:
+ * 1. Must find an interrupt common, the driver cannot work
+ *    without one.
+ *
+ *    If the dss device-tree node does not have a subnode
+ *    "dss-commons", assume tidss is the only module handling
+ *    DSS and therefore use intr_common = COMMON_M.
+ *
+ *    If "dss-commons" subnode is present, then it must have
+ *    a child node "interrupt-common", or else fail. And then,
+ *    interrupt-common must have a "reg" property that
+ *    indicates which common area to use for interrupts. Must
+ *    be 0 to feat->num_commons
+ *
+ * 2. Optionally, find a configuration region, or make certain
+ *    assumptions and proceed
+ *
+ *    If the dss has a remote device, assume tidss is a slave
+ *    and do not search for a config_common
+ *
+ *    If the dss device-tree node does not have a
+ *    subnode called "dss-commons", assume tidss is the only
+ *    module handling DSS and therefore use config_common =
+ *    COMMON_M.
+ *
+ *    If "dss-commons" is present, search for a child
+ *    "config-common". If no such child is present, tidss
+ *    assumes that DSS is early-configured and does not
+ *    search for a config-common.
+ *
+ *    If "config-common" child is present, then it must have a
+ *    "reg" property that indicates which common area to use for
+ *    configuration. Must be COMMON_M
+ */
+static int dispc7_j721e_setup_commons(struct dispc_device *dispc)
+{
+	int r;
+	struct tidss_device *tidss = dispc->tidss;
+	struct device *dev = tidss->dev;
+	struct platform_device *pdev = to_platform_device(dev);
+	u32 common_intr_id, common_cfg_id;
+
+	r = dispc_j721e_get_managed_common_intr(dispc, &common_intr_id);
+	if (r || common_intr_id >= dispc->feat->num_commons)
+		return -EINVAL;
+
+	r = dispc7_iomap_resource(pdev, dispc->feat->common_name[common_intr_id],
+			&dispc->base_common_intr);
+	if (r)
+		return r;
+
+	dispc->irq = platform_get_irq(pdev, common_intr_id);
+	if (dispc->irq < 0)
+		return dispc->irq;
+
+	if (tidss->rdev) {
+		dev_dbg(dev, "%s: continuing with remote device\n", __func__);
+		dispc->has_cfg_common = false;
+		return 0;
+	}
+
+	r = dispc_j721e_get_managed_common_cfg(dispc, &common_cfg_id);
+	if (r) {
+		dev_dbg(dev, "%s: continuing without configuration common\n", __func__);
+		dispc->has_cfg_common = false;
+		return 0;
+	}
+
+	if (common_cfg_id >= dispc->feat->num_commons ||
+			!dispc->feat->common_cfg[common_cfg_id])
+		return -EINVAL;
+
+	if (common_intr_id == common_cfg_id)
+		dispc->base_common_cfg = dispc->base_common_intr;
+	else {
+		r = dispc7_iomap_resource(pdev, dispc->feat->common_name[common_cfg_id],
+				&dispc->base_common_cfg);
+		if (r)
+			return r;
+	}
+
+	dispc->has_cfg_common = true;
+
+	return 0;
+}
+
+static int dispc7_am6_setup_commons(struct dispc_device *dispc)
+{
+	int r;
+	struct tidss_device *tidss = dispc->tidss;
+	struct platform_device *pdev = to_platform_device(tidss->dev);
+
+	r = dispc7_iomap_resource(pdev, "common", &dispc->base_common_cfg);
+	if (r)
+		return r;
+
+	dispc->base_common_intr = dispc->base_common_cfg;
+
+	dispc->irq = platform_get_irq(pdev, 0);
+	if (dispc->irq < 0)
+		return dispc->irq;
+
+	dispc->has_cfg_common = true;
+
+	return 0;
+}
+
+static int dispc7_setup_commons(struct dispc_device *dispc)
+{
+	switch (dispc->feat->subrev) {
+	case DSS7_AM6:
+		return dispc7_am6_setup_commons(dispc);
+	case DSS7_J721E:
+		return dispc7_j721e_setup_commons(dispc);
+	default:
+		WARN_ON(1);
+		return -EINVAL;
+	}
+}
+
+static struct device_node *dispc7_of_dss_plane_for_id(struct device_node *parent, u32 id)
+{
+	struct device_node *dss_planes_node, *plane;
+
+	dss_planes_node = of_get_child_by_name(parent, "dss-planes");
+	if (!dss_planes_node)
+		return NULL;
+
+	for_each_child_of_node(dss_planes_node, plane) {
+		u32 plane_id = 0;
+
+		if (of_node_cmp(plane->name, "plane") != 0)
+			continue;
+		of_property_read_u32(plane, "reg", &plane_id);
+		if (id == plane_id)
+			break;
+	}
+
+	of_node_put(dss_planes_node);
+
+	return plane;
+}
+
+static struct device_node *dispc7_of_dss_vp_for_id(struct device_node *parent, u32 id)
+{
+	struct device_node *dss_vps_node, *vp;
+
+	dss_vps_node = of_get_child_by_name(parent, "dss-vps");
+	if (!dss_vps_node)
+		return NULL;
+
+	for_each_child_of_node(dss_vps_node, vp) {
+		u32 vp_id = 0;
+
+		if (of_node_cmp(vp->name, "vp") != 0)
+			continue;
+		of_property_read_u32(vp, "reg", &vp_id);
+		if (id == vp_id)
+			break;
+	}
+
+	of_node_put(dss_vps_node);
+
+	return vp;
+}
+
+static struct device_node *dispc7_of_dss_wb(struct device_node *parent)
+{
+	struct device_node *dss_wbs_node, *wb;
+
+	dss_wbs_node = of_get_child_by_name(parent, "dss-wbs");
+	if (!dss_wbs_node)
+		return NULL;
+
+	for_each_child_of_node(dss_wbs_node, wb) {
+		if (of_node_cmp(wb->name, "wb") == 0)
+			break;
+	}
+
+	of_node_put(dss_wbs_node);
+
+	return wb;
+}
+
+static bool dispc7_is_plane_managed(struct tidss_device *tidss, u32 plane_id)
+{
+	struct device *dev = tidss->dev;
+	struct device_node *plane;
+	u32 managed;
+	bool ret;
+
+	plane = dispc7_of_dss_plane_for_id(dev->of_node, plane_id);
+	if (!plane)
+		return true;
+
+	ret = true;
+
+	if (of_property_read_u32(plane, "managed", &managed))
+		goto out;
+
+	if (!managed)
+		ret = false;
+
+out:
+	of_node_put(plane);
+	return ret;
+}
+
+static bool dispc7_is_vp_managed(struct tidss_device *tidss, u32 vp_id)
+{
+	struct device *dev = tidss->dev;
+	struct device_node *vp;
+	u32 managed;
+	bool ret;
+
+	vp = dispc7_of_dss_vp_for_id(dev->of_node, vp_id);
+	if (!vp)
+		return true;
+
+	ret = true;
+
+	if (of_property_read_u32(vp, "managed", &managed))
+		goto out;
+
+	if (!managed)
+		ret = false;
+
+out:
+	of_node_put(vp);
+	return ret;
+}
+
+static bool dispc7_is_wb_managed(struct tidss_device *tidss)
+{
+	struct device *dev = tidss->dev;
+	struct device_node *wb;
+	u32 managed;
+	bool ret;
+
+	wb = dispc7_of_dss_wb(dev->of_node);
+	if (!wb)
+		return true;
+
+	ret = true;
+
+	if (of_property_read_u32(wb, "managed", &managed))
+		goto out;
+
+	if (!managed)
+		ret = false;
+
+out:
+	of_node_put(wb);
+	return ret;
+}
+
+int dispc7_init(struct tidss_device *tidss)
+{
+	struct device *dev = tidss->dev;
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dispc_device *dispc;
+	const struct dispc7_features *feat;
+	unsigned int i;
+	int r = 0;
+
+	dev_dbg(dev, "%s\n", __func__);
+
+	r = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
+	if (r)
+		dev_warn(dev, "cannot set DMA masks to 48-bit\n");
+
+	feat = of_match_device(dispc7_of_table, dev)->data;
+
+	dispc = devm_kzalloc(dev, sizeof(*dispc), GFP_KERNEL);
+	if (!dispc)
+		return -ENOMEM;
+
+	dispc->tidss = tidss;
+	dispc->dev = dev;
+	dispc->feat = feat;
+
+	switch (feat->subrev) {
+	case DSS7_AM6:
+		dispc7_common_regmap = tidss_am6_common_regs;
+		break;
+	case DSS7_J721E:
+		dispc7_common_regmap = tidss_j721e_common_regs;
+		break;
+	default:
+		WARN_ON(1);
+		return -EINVAL;
+	}
+
+	r = dispc7_setup_commons(dispc);
+	if (r) {
+		dev_err(dev, "%s: could not setup common regions\n", __func__);
+		return r;
+	}
+
+	for (i = 0; i < dispc->feat->num_vps; i++)
+		dispc->vp_managed[i] = dispc7_is_vp_managed(tidss, i);
+
+	for (i = 0; i < dispc->feat->num_planes; i++)
+		dispc->plane_managed[i] = dispc7_is_plane_managed(tidss, i);
+
+	if (dispc->feat->has_writeback)
+		dispc->wb_managed = dispc7_is_wb_managed(tidss);
+
+	dispc_for_each_managed_plane(dispc, i) {
+		r = dispc7_iomap_resource(pdev, dispc->feat->vid_name[i],
+					  &dispc->base_vid[i]);
+		dev_dbg(dev, "%s: %u %s %d\n", __func__,
+			i, dispc->feat->vid_name[i], r);
+		if (r)
+			return r;
+	}
+
+	if (dispc7_has_writeback(dispc)) {
+		r = dispc7_iomap_resource(pdev, dispc->feat->wb_name,
+					  &dispc->base_wb);
+		dev_dbg(dev, "%s: %s %d\n", __func__,
+			dispc->feat->wb_name, r);
+		if (r)
+			return r;
+	}
+
+	dispc_for_each_managed_vp(dispc, i) {
+		struct clk *clk;
+
+		r = dispc7_iomap_resource(pdev, dispc->feat->ovr_name[i],
+					  &dispc->base_ovr[i]);
+		dev_dbg(dev, "%s: %u %s %d\n", __func__,
+			i, dispc->feat->ovr_name[i], r);
+		if (r)
+			return r;
+
+		r = dispc7_iomap_resource(pdev, dispc->feat->vp_name[i],
+					  &dispc->base_vp[i]);
+		dev_dbg(dev, "%s: %u %s %d\n", __func__,
+			i, dispc->feat->vp_name[i], r);
+		if (r)
+			return r;
+
+		clk = devm_clk_get(dev, dispc->feat->vpclk_name[i]);
+		if (IS_ERR(clk)) {
+			dev_err(dev, "%s: Failed to get clk %s:%ld\n", __func__,
+				dispc->feat->vpclk_name[i], PTR_ERR(clk));
+			return PTR_ERR(clk);
+		}
+		dispc->vp_clk[i] = clk;
+
+		dispc->vp_data[i].gamma_table = devm_kmalloc_array(
+			dev, dispc->feat->vp_feat.color.gamma_size,
+			sizeof(*dispc->vp_data[i].gamma_table), GFP_KERNEL);
+		if (!dispc->vp_data[i].gamma_table)
+			return -ENOMEM;
+	}
+
+	if (feat->subrev == DSS7_AM6) {
+		dispc->syscon = syscon_regmap_lookup_by_phandle(dev->of_node,
+								"syscon");
+		if (IS_ERR(dispc->syscon)) {
+			dev_err(dev, "%s: syscon_regmap_lookup_by_phandle failed %ld\n",
+				__func__, PTR_ERR(dispc->syscon));
+			return PTR_ERR(dispc->syscon);
+		}
+	}
+
+	dispc->fclk = devm_clk_get(dev, "fck");
+	if (IS_ERR(dispc->fclk)) {
+		dev_err(dev, "Failed to get fclk\n");
+		return PTR_ERR(dispc->fclk);
+	}
+	dev_dbg(dev, "DSS fclk %lu Hz\n", clk_get_rate(dispc->fclk));
+
+	r = dispc7_init_gamma_tables(dispc);
+	if (r)
+		return r;
+
+	tidss->dispc_ops = &dispc7_ops;
+	tidss->dispc = dispc;
+
+	return 0;
+}
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/tidss_dispc7.h b/drivers/gpu/drm/tidss/tidss_dispc7.h
--- a/drivers/gpu/drm/tidss/tidss_dispc7.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/tidss_dispc7.h	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,360 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2016-2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Jyri Sarha <jsarha@ti.com>
+ */
+
+#ifndef __TIDSS_DISPC7_H
+#define __TIDSS_DISPC7_H
+
+#define DISPC7_MAX_COMMONS	4
+#define DISPC7_MAX_PORTS	4
+#define DISPC7_MAX_PLANES	4
+
+struct dispc7_features_scaling {
+	u32 in_width_max_5tap_rgb;
+	u32 in_width_max_3tap_rgb;
+	u32 in_width_max_5tap_yuv;
+	u32 in_width_max_3tap_yuv;
+	u32 upscale_limit;
+	u32 downscale_limit_5tap;
+	u32 downscale_limit_3tap;
+	u32 xinc_max;
+};
+
+struct dispc7_errata {
+	bool i2000; /* DSS Does Not Support YUV Pixel Data Formats */
+};
+
+enum dispc7_vp_bus_type {
+	DISPC7_VP_DPI,		/* DPI output */
+	DISPC7_VP_OLDI,		/* OLDI (LVDS) output */
+	DISPC7_VP_INTERNAL,	/* SoC internal routing */
+	DISPC7_VP_MAX_BUS_TYPE,
+};
+
+enum dispc7_dss_subrevision {
+	DSS7_AM6,
+	DSS7_J721E,
+};
+
+struct dispc7_features {
+	int max_pclk_kHz[DISPC7_VP_MAX_BUS_TYPE];
+
+	u32 num_commons;
+	const char *common_name[DISPC7_MAX_COMMONS];
+	bool common_cfg[DISPC7_MAX_COMMONS];
+
+	struct dispc7_features_scaling scaling;
+
+	enum dispc7_dss_subrevision subrev;
+
+	u32 num_vps;
+	const char *vp_name[DISPC7_MAX_PORTS]; /* Should match dt reg names */
+	const char *ovr_name[DISPC7_MAX_PORTS]; /* Should match dt reg names */
+	const char *vpclk_name[DISPC7_MAX_PORTS]; /* Should match dt clk names */
+	const enum dispc7_vp_bus_type vp_bus_type[DISPC7_MAX_PORTS];
+	struct tidss_vp_feat vp_feat;
+	u32 num_planes;
+	const char *vid_name[DISPC7_MAX_PLANES]; /* Should match dt reg names */
+	bool vid_lite[DISPC7_MAX_PLANES];
+	u32 vid_order[DISPC7_MAX_PLANES];
+
+	bool has_writeback;
+	const char *wb_name; /* Should match dt reg names */
+
+	struct dispc7_errata errata;
+};
+
+enum dss7_common_regs {
+	NOT_APPLICABLE_OFF = 0,
+	DSS_REVISION_OFF,
+	DSS_SYSCONFIG_OFF,
+	DSS_SYSSTATUS_OFF,
+	DISPC_IRQ_EOI_OFF,
+	DISPC_IRQSTATUS_RAW_OFF,
+	DISPC_IRQSTATUS_OFF,
+	DISPC_IRQENABLE_SET_OFF,
+	DISPC_IRQENABLE_CLR_OFF,
+	DISPC_VID_IRQENABLE_OFF,
+	DISPC_VID_IRQSTATUS_OFF,
+	DISPC_VP_IRQENABLE_OFF,
+	DISPC_VP_IRQSTATUS_OFF,
+	WB_IRQENABLE_OFF,
+	WB_IRQSTATUS_OFF,
+	DISPC_GLOBAL_MFLAG_ATTRIBUTE_OFF,
+	DISPC_GLOBAL_OUTPUT_ENABLE_OFF,
+	DISPC_GLOBAL_BUFFER_OFF,
+	DSS_CBA_CFG_OFF,
+	DISPC_DBG_CONTROL_OFF,
+	DISPC_DBG_STATUS_OFF,
+	DISPC_CLKGATING_DISABLE_OFF,
+	DISPC_SECURE_DISABLE_OFF,
+	FBDC_REVISION_1_OFF,
+	FBDC_REVISION_2_OFF,
+	FBDC_REVISION_3_OFF,
+	FBDC_REVISION_4_OFF,
+	FBDC_REVISION_5_OFF,
+	FBDC_REVISION_6_OFF,
+	FBDC_COMMON_CONTROL_OFF,
+	FBDC_CONSTANT_COLOR_0_OFF,
+	FBDC_CONSTANT_COLOR_1_OFF,
+	DISPC_CONNECTIONS_OFF,
+	DISPC_MSS_VP1_OFF,
+	DISPC_MSS_VP3_OFF,
+	DSS7_COMMON_REG_TABLE_LEN,
+};
+
+enum dss7_writeback_connections {
+	DSS7_WB_BYPASS = 0x0,
+	DSS7_WB_VIDL2 = 0x1,
+	DSS7_WB_OVR1 = 0x2,
+	DSS7_WB_OVR2 = 0x4,
+	DSS7_WB_OVR3 = 0x8,
+	DSS7_WB_OVR4 = 0x10,
+};
+
+/*
+ * dispc7_common_regmap should be defined as const u16 * and pointing
+ * to a valid dss common register map for the platform, before the
+ * macros bellow can be used.
+ */
+
+#define REG(r) (dispc7_common_regmap[r ## _OFF])
+
+#define DSS_REVISION			REG(DSS_REVISION)
+#define DSS_SYSCONFIG			REG(DSS_SYSCONFIG)
+#define DSS_SYSSTATUS			REG(DSS_SYSSTATUS)
+#define DISPC_IRQ_EOI			REG(DISPC_IRQ_EOI)
+#define DISPC_IRQSTATUS_RAW		REG(DISPC_IRQSTATUS_RAW)
+#define DISPC_IRQSTATUS			REG(DISPC_IRQSTATUS)
+#define DISPC_IRQENABLE_SET		REG(DISPC_IRQENABLE_SET)
+#define DISPC_IRQENABLE_CLR		REG(DISPC_IRQENABLE_CLR)
+#define DISPC_VID_IRQENABLE(n)		(REG(DISPC_VID_IRQENABLE) + (n) * 4)
+#define DISPC_VID_IRQSTATUS(n)		(REG(DISPC_VID_IRQSTATUS) + (n) * 4)
+#define DISPC_VP_IRQENABLE(n)		(REG(DISPC_VP_IRQENABLE) + (n) * 4)
+#define DISPC_VP_IRQSTATUS(n)		(REG(DISPC_VP_IRQSTATUS) + (n) * 4)
+#define WB_IRQENABLE			REG(WB_IRQENABLE)
+#define WB_IRQSTATUS			REG(WB_IRQSTATUS)
+
+#define DISPC_GLOBAL_MFLAG_ATTRIBUTE	REG(DISPC_GLOBAL_MFLAG_ATTRIBUTE)
+#define DISPC_GLOBAL_OUTPUT_ENABLE	REG(DISPC_GLOBAL_OUTPUT_ENABLE)
+#define DISPC_GLOBAL_BUFFER		REG(DISPC_GLOBAL_BUFFER)
+#define DSS_CBA_CFG			REG(DSS_CBA_CFG)
+#define DISPC_DBG_CONTROL		REG(DISPC_DBG_CONTROL)
+#define DISPC_DBG_STATUS		REG(DISPC_DBG_STATUS)
+#define DISPC_CLKGATING_DISABLE		REG(DISPC_CLKGATING_DISABLE)
+#define DISPC_SECURE_DISABLE		REG(DISPC_SECURE_DISABLE)
+
+#define FBDC_REVISION_1			REG(FBDC_REVISION_1)
+#define FBDC_REVISION_2			REG(FBDC_REVISION_2)
+#define FBDC_REVISION_3			REG(FBDC_REVISION_3)
+#define FBDC_REVISION_4			REG(FBDC_REVISION_4)
+#define FBDC_REVISION_5			REG(FBDC_REVISION_5)
+#define FBDC_REVISION_6			REG(FBDC_REVISION_6)
+#define FBDC_COMMON_CONTROL		REG(FBDC_COMMON_CONTROL)
+#define FBDC_CONSTANT_COLOR_0		REG(FBDC_CONSTANT_COLOR_0)
+#define FBDC_CONSTANT_COLOR_1		REG(FBDC_CONSTANT_COLOR_1)
+#define DISPC_CONNECTIONS		REG(DISPC_CONNECTIONS)
+#define DISPC_MSS_VP1			REG(DISPC_MSS_VP1)
+#define DISPC_MSS_VP3			REG(DISPC_MSS_VP3)
+
+/* COMMON1 not implemented */
+
+/* VID */
+
+#define DISPC_VID_ACCUH_0		0x0
+#define DISPC_VID_ACCUH_1		0x4
+#define DISPC_VID_ACCUH2_0		0x8
+#define DISPC_VID_ACCUH2_1		0xc
+#define DISPC_VID_ACCUV_0		0x10
+#define DISPC_VID_ACCUV_1		0x14
+#define DISPC_VID_ACCUV2_0		0x18
+#define DISPC_VID_ACCUV2_1		0x1c
+#define DISPC_VID_ATTRIBUTES		0x20
+#define DISPC_VID_ATTRIBUTES2		0x24
+#define DISPC_VID_BA_0			0x28
+#define DISPC_VID_BA_1			0x2c
+#define DISPC_VID_BA_UV_0		0x30
+#define DISPC_VID_BA_UV_1		0x34
+#define DISPC_VID_BUF_SIZE_STATUS	0x38
+#define DISPC_VID_BUF_THRESHOLD		0x3c
+#define DISPC_VID_CSC_COEF(n)		(0x40 + (n) * 4)
+
+#define DISPC_VID_FIRH			0x5c
+#define DISPC_VID_FIRH2			0x60
+#define DISPC_VID_FIRV			0x64
+#define DISPC_VID_FIRV2			0x68
+
+#define DISPC_VID_FIR_COEFS_H0		0x6c
+#define DISPC_VID_FIR_COEF_H0(phase)	(0x6c + (phase) * 4)
+#define DISPC_VID_FIR_COEFS_H0_C	0x90
+#define DISPC_VID_FIR_COEF_H0_C(phase)	(0x90 + (phase) * 4)
+
+#define DISPC_VID_FIR_COEFS_H12		0xb4
+#define DISPC_VID_FIR_COEF_H12(phase)	(0xb4 + (phase) * 4)
+#define DISPC_VID_FIR_COEFS_H12_C	0xf4
+#define DISPC_VID_FIR_COEF_H12_C(phase)	(0xf4 + (phase) * 4)
+
+#define DISPC_VID_FIR_COEFS_V0		0x134
+#define DISPC_VID_FIR_COEF_V0(phase)	(0x134 + (phase) * 4)
+#define DISPC_VID_FIR_COEFS_V0_C	0x158
+#define DISPC_VID_FIR_COEF_V0_C(phase)	(0x158 + (phase) * 4)
+
+#define DISPC_VID_FIR_COEFS_V12		0x17c
+#define DISPC_VID_FIR_COEF_V12(phase)	(0x17c + (phase) * 4)
+#define DISPC_VID_FIR_COEFS_V12_C	0x1bc
+#define DISPC_VID_FIR_COEF_V12_C(phase)	(0x1bc + (phase) * 4)
+
+#define DISPC_VID_GLOBAL_ALPHA		0x1fc
+#define DISPC_VID_MFLAG_THRESHOLD	0x208
+#define DISPC_VID_PICTURE_SIZE		0x20c
+#define DISPC_VID_PIXEL_INC		0x210
+#define DISPC_VID_PRELOAD		0x218
+#define DISPC_VID_ROW_INC		0x21c
+#define DISPC_VID_SIZE			0x220
+#define DISPC_VID_BA_EXT_0		0x22c
+#define DISPC_VID_BA_EXT_1		0x230
+#define DISPC_VID_BA_UV_EXT_0		0x234
+#define DISPC_VID_BA_UV_EXT_1		0x238
+#define DISPC_VID_CSC_COEF7		0x23c
+#define DISPC_VID_ROW_INC_UV		0x248
+#define DISPC_VID_CLUT			0x260
+#define DISPC_VID_SAFETY_ATTRIBUTES	0x2a0
+#define DISPC_VID_SAFETY_CAPT_SIGNATURE	0x2a4
+#define DISPC_VID_SAFETY_POSITION	0x2a8
+#define DISPC_VID_SAFETY_REF_SIGNATURE	0x2ac
+#define DISPC_VID_SAFETY_SIZE		0x2b0
+#define DISPC_VID_SAFETY_LFSR_SEED	0x2b4
+#define DISPC_VID_LUMAKEY		0x2b8
+#define DISPC_VID_DMA_BUFSIZE		0x2bc /* J721E */
+
+/* WB */
+
+#define DISPC_WB_ACCUH_0		0x0
+#define DISPC_WB_ACCUH_1		0x4
+#define DISPC_WB_ACCUH2_0		0x8
+#define DISPC_WB_ACCUH2_1		0xc
+#define DISPC_WB_ACCUV_0		0x10
+#define DISPC_WB_ACCUV_1		0x14
+#define DISPC_WB_ACCUV2_0		0x18
+#define DISPC_WB_ACCUV2_1		0x1c
+#define DISPC_WB_ATTRIBUTES		0x20
+#define DISPC_WB_ATTRIBUTES2		0x24
+#define DISPC_WB_BA_0			0x28
+#define DISPC_WB_BA_1			0x2c
+#define DISPC_WB_BA_UV_0		0x30
+#define DISPC_WB_BA_UV_1		0x34
+#define DISPC_WB_BUF_SIZE_STATUS	0x38
+#define DISPC_WB_BUF_THRESHOLD		0x3c
+#define DISPC_WB_CSC_COEF(n)		(0x40 + (n) * 4)
+
+#define DISPC_WB_FIRH			0x5c
+#define DISPC_WB_FIRH2			0x60
+#define DISPC_WB_FIRV			0x64
+#define DISPC_WB_FIRV2			0x68
+
+#define DISPC_WB_FIR_COEFS_H0		0x6c
+#define DISPC_WB_FIR_COEF_H0(phase)	(0x6c + (phase) * 4)
+#define DISPC_WB_FIR_COEFS_H0_C	0x90
+#define DISPC_WB_FIR_COEF_H0_C(phase)	(0x90 + (phase) * 4)
+
+#define DISPC_WB_FIR_COEFS_H12		0xb4
+#define DISPC_WB_FIR_COEF_H12(phase)	(0xb4 + (phase) * 4)
+#define DISPC_WB_FIR_COEFS_H12_C	0xf4
+#define DISPC_WB_FIR_COEF_H12_C(phase)	(0xf4 + (phase) * 4)
+
+#define DISPC_WB_FIR_COEFS_V0		0x134
+#define DISPC_WB_FIR_COEF_V0(phase)	(0x134 + (phase) * 4)
+#define DISPC_WB_FIR_COEFS_V0_C	0x158
+#define DISPC_WB_FIR_COEF_V0_C(phase)	(0x158 + (phase) * 4)
+
+#define DISPC_WB_FIR_COEFS_V12		0x17c
+#define DISPC_WB_FIR_COEF_V12(phase)	(0x17c + (phase) * 4)
+#define DISPC_WB_FIR_COEFS_V12_C	0x1bc
+#define DISPC_WB_FIR_COEF_V12_C(phase)	(0x1bc + (phase) * 4)
+
+#define DISPC_WB_MFLAG_THRESHOLD	0x204
+#define DISPC_WB_PICTURE_SIZE		0x208
+#define DISPC_WB_SIZE			0x210
+#define DISPC_WB_POSITION		0x214
+#define DISPC_WB_CSC_COEF7		0x21c
+#define DISPC_WB_ROW_INC		0x224
+#define DISPC_WB_ROW_INC_UV		0x228
+#define DISPC_WB_BA_EXT_0		0x22c
+#define DISPC_WB_BA_EXT_1		0x230
+#define DISPC_WB_BA_UV_EXT_0		0x234
+#define DISPC_WB_BA_UV_EXT_1		0x238
+#define DISPC_WB_SECURE			0x248
+
+/* OVR */
+
+#define DISPC_OVR_CONFIG		0x0
+#define DISPC_OVR_VIRTVP		0x4 /* J721E */
+#define DISPC_OVR_DEFAULT_COLOR		0x8
+#define DISPC_OVR_DEFAULT_COLOR2	0xc
+#define DISPC_OVR_TRANS_COLOR_MAX	0x10
+#define DISPC_OVR_TRANS_COLOR_MAX2	0x14
+#define DISPC_OVR_TRANS_COLOR_MIN	0x18
+#define DISPC_OVR_TRANS_COLOR_MIN2	0x1c
+#define DISPC_OVR_ATTRIBUTES(n)		(0x20 + (n) * 4)
+#define DISPC_OVR_ATTRIBUTES2(n)	(0x34 + (n) * 4) /* J721E */
+/* VP */
+
+#define DISPC_VP_CONFIG				0x0
+#define DISPC_VP_CONTROL			0x4
+#define DISPC_VP_CSC_COEF0			0x8
+#define DISPC_VP_CSC_COEF1			0xc
+#define DISPC_VP_CSC_COEF2			0x10
+#define DISPC_VP_DATA_CYCLE_0			0x14
+#define DISPC_VP_DATA_CYCLE_1			0x18
+#define DISPC_VP_DATA_CYCLE_2			0x1c
+#define DISPC_VP_LINE_NUMBER			0x44
+#define DISPC_VP_POL_FREQ			0x4c
+#define DISPC_VP_SIZE_SCREEN			0x50
+#define DISPC_VP_TIMING_H			0x54
+#define DISPC_VP_TIMING_V			0x58
+#define DISPC_VP_CSC_COEF3			0x5c
+#define DISPC_VP_CSC_COEF4			0x60
+#define DISPC_VP_CSC_COEF5			0x64
+#define DISPC_VP_CSC_COEF6			0x68
+#define DISPC_VP_CSC_COEF7			0x6c
+#define DISPC_VP_SAFETY_ATTRIBUTES_0		0x70
+#define DISPC_VP_SAFETY_ATTRIBUTES_1		0x74
+#define DISPC_VP_SAFETY_ATTRIBUTES_2		0x78
+#define DISPC_VP_SAFETY_ATTRIBUTES_3		0x7c
+#define DISPC_VP_SAFETY_CAPT_SIGNATURE_0	0x90
+#define DISPC_VP_SAFETY_CAPT_SIGNATURE_1	0x94
+#define DISPC_VP_SAFETY_CAPT_SIGNATURE_2	0x98
+#define DISPC_VP_SAFETY_CAPT_SIGNATURE_3	0x9c
+#define DISPC_VP_SAFETY_POSITION_0		0xb0
+#define DISPC_VP_SAFETY_POSITION_1		0xb4
+#define DISPC_VP_SAFETY_POSITION_2		0xb8
+#define DISPC_VP_SAFETY_POSITION_3		0xbc
+#define DISPC_VP_SAFETY_REF_SIGNATURE_0		0xd0
+#define DISPC_VP_SAFETY_REF_SIGNATURE_1		0xd4
+#define DISPC_VP_SAFETY_REF_SIGNATURE_2		0xd8
+#define DISPC_VP_SAFETY_REF_SIGNATURE_3		0xdc
+#define DISPC_VP_SAFETY_SIZE_0			0xf0
+#define DISPC_VP_SAFETY_SIZE_1			0xf4
+#define DISPC_VP_SAFETY_SIZE_2			0xf8
+#define DISPC_VP_SAFETY_SIZE_3			0xfc
+#define DISPC_VP_SAFETY_LFSR_SEED		0x110
+#define DISPC_VP_GAMMA_TABLE			0x120
+#define DISPC_VP_DSS_OLDI_CFG			0x160
+#define DISPC_VP_DSS_OLDI_STATUS		0x164
+#define DISPC_VP_DSS_OLDI_LB			0x168
+#define DISPC_VP_DSS_MERGE_SPLIT		0x16c /* J721E */
+#define DISPC_VP_DSS_DMA_THREADSIZE		0x170 /* J721E */
+#define DISPC_VP_DSS_DMA_THREADSIZE_STATUS	0x174 /* J721E */
+
+/* CTRL_MMR0 access trough syscon */
+#define CTRLMMR0P1_OLDI_DAT0_IO_CTRL		0x41E0
+#define CTRLMMR0P1_OLDI_DAT1_IO_CTRL		0x41E4
+#define CTRLMMR0P1_OLDI_DAT2_IO_CTRL		0x41E8
+#define CTRLMMR0P1_OLDI_DAT3_IO_CTRL		0x41EC
+#define CTRLMMR0P1_OLDI_CLK_IO_CTRL		0x41F0
+
+#define CTRLMMR0P1_OLDI_PWRDN_TX		BIT(8)
+
+#endif /* __TIDSS_DISPC7_H */
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/tidss_dispc.h b/drivers/gpu/drm/tidss/tidss_dispc.h
--- a/drivers/gpu/drm/tidss/tidss_dispc.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/tidss_dispc.h	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,154 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ */
+
+#ifndef __TIDSS_DISPC_H__
+#define __TIDSS_DISPC_H__
+
+struct dispc_device;
+struct tidss_device;
+
+struct drm_crtc_state;
+
+#define DSS_MAX_CHANNELS 8
+#define DSS_MAX_PLANES 8
+
+/*
+ * Based on the above 2 defines the bellow defines describe following
+ * u64 IRQ bits:
+ *
+ * bit group |dev |mrg0|mrg1|mrg2|mrg3|mrg4|mrg5|mrg6|mrg7|plane 0-7|<unused> |
+ * bit use   |Dfou|FEOL|FEOL|FEOL|FEOL|FEOL|FEOL|FEOL|FEOL|UUUU|UUUU| | | | | |
+ * bit number|0-3 |4-7 |8-11|            12-35            |  36-43  |  44-63  |
+ *
+ * device bits:	D = OCP error
+ * WB bits:	f = frame done wb, o = wb buffer overflow,
+ *		u = wb buffer uncomplete
+ * vp bits:	F = frame done, E = vsync even, O = vsync odd, L = sync lost
+ * plane bits:	U = fifo underflow
+ */
+
+#define DSS_IRQ_DEVICE_OCP_ERR			BIT_ULL(0)
+
+#define DSS_IRQ_DEVICE_FRAMEDONEWB		BIT_ULL(1)
+#define DSS_IRQ_DEVICE_WBBUFFEROVERFLOW		BIT_ULL(2)
+#define DSS_IRQ_DEVICE_WBUNCOMPLETEERROR	BIT_ULL(3)
+#define DSS_IRQ_DEVICE_WBSECURITYVIOLATION	BIT_ULL(44)
+#define DSS_IRQ_DEVICE_WBSYNC			BIT_ULL(45)
+#define DSS_IRQ_DEVICE_WB_MASK			(GENMASK_ULL(3, 1) | BIT_ULL(44) | BIT_ULL(45))
+
+#define DSS_IRQ_VP_BIT_N(ch, bit)	(4 + 4 * (ch) + (bit))
+#define DSS_IRQ_PLANE_BIT_N(plane, bit) \
+	(DSS_IRQ_VP_BIT_N(DSS_MAX_CHANNELS, 0) + 1 * (plane) + (bit))
+
+#define DSS_IRQ_VP_BIT(ch, bit)	BIT_ULL(DSS_IRQ_VP_BIT_N((ch), (bit)))
+#define DSS_IRQ_PLANE_BIT(plane, bit) \
+	BIT_ULL(DSS_IRQ_PLANE_BIT_N((plane), (bit)))
+
+#define DSS_IRQ_VP_MASK(ch) \
+	GENMASK_ULL(DSS_IRQ_VP_BIT_N((ch), 3), DSS_IRQ_VP_BIT_N((ch), 0))
+#define DSS_IRQ_PLANE_MASK(plane) \
+	GENMASK_ULL(DSS_IRQ_PLANE_BIT_N((plane), 0), \
+		    DSS_IRQ_PLANE_BIT_N((plane), 0))
+
+#define DSS_IRQ_VP_FRAME_DONE(ch)	DSS_IRQ_VP_BIT((ch), 0)
+#define DSS_IRQ_VP_VSYNC_EVEN(ch)	DSS_IRQ_VP_BIT((ch), 1)
+#define DSS_IRQ_VP_VSYNC_ODD(ch)	DSS_IRQ_VP_BIT((ch), 2)
+#define DSS_IRQ_VP_SYNC_LOST(ch)	DSS_IRQ_VP_BIT((ch), 3)
+
+#define DSS_IRQ_PLANE_FIFO_UNDERFLOW(plane)	DSS_IRQ_PLANE_BIT((plane), 0)
+
+enum tidss_gamma_type { TIDSS_GAMMA_8BIT, TIDSS_GAMMA_10BIT };
+
+struct tidss_vp_feat {
+	struct tidss_vp_color_feat {
+		u32 gamma_size;
+		enum tidss_gamma_type gamma_type;
+		bool has_ctm;
+	} color;
+};
+
+struct tidss_plane_feat {
+	struct tidss_plane_color_feat {
+		u32 encodings;
+		u32 ranges;
+		enum drm_color_encoding default_encoding;
+		enum drm_color_range default_range;
+	} color;
+	struct tidss_plane_blend_feat {
+		bool global_alpha;
+	} blend;
+};
+
+struct tidss_dispc_ops {
+	u64 (*read_and_clear_irqstatus)(struct dispc_device *dispc);
+	void (*write_irqenable)(struct dispc_device *dispc, u64 enable);
+
+	int (*get_num_vps)(struct dispc_device *dispc);
+	const char *(*vp_name)(struct dispc_device *dispc,
+			       u32 hw_videoport);
+	const struct tidss_vp_feat *(*vp_feat)(struct dispc_device *dispc,
+					       u32 hw_videoport);
+	void (*vp_prepare)(struct dispc_device *dispc, u32 hw_videoport,
+			   const struct drm_crtc_state *state);
+	void (*vp_enable)(struct dispc_device *dispc, u32 hw_videoport,
+			  const struct drm_crtc_state *state);
+	void (*vp_disable)(struct dispc_device *dispc, u32 hw_videoport);
+	void (*vp_unprepare)(struct dispc_device *dispc, u32 hw_videoport);
+	bool (*vp_go_busy)(struct dispc_device *dispc,
+			   u32 hw_videoport);
+	void (*vp_go)(struct dispc_device *dispc, u32 hw_videoport);
+	enum drm_mode_status (*vp_mode_valid)(struct dispc_device *dispc,
+					u32 hw_videoport,
+					const struct drm_display_mode *mode);
+	int (*vp_check)(struct dispc_device *dispc, u32 hw_videoport,
+			const struct drm_crtc_state *state);
+	void (*vp_setup)(struct dispc_device *dispc, u32 hw_videoport,
+			 const struct drm_crtc_state *state);
+
+	int (*vp_set_clk_rate)(struct dispc_device *dispc,
+			       u32 hw_videoport, unsigned long rate);
+	int (*vp_enable_clk)(struct dispc_device *dispc, u32 hw_videoport);
+	void (*vp_disable_clk)(struct dispc_device *dispc, u32 hw_videoport);
+
+	int (*get_num_planes)(struct dispc_device *dispc);
+	const char *(*plane_name)(struct dispc_device *dispc,
+				  u32 hw_plane);
+	const struct tidss_plane_feat *(*plane_feat)(struct dispc_device *dispc,
+						     u32 hw_plane);
+	int (*plane_enable)(struct dispc_device *dispc, u32 hw_plane,
+			    bool enable);
+	int (*plane_check)(struct dispc_device *dispc, u32 hw_plane,
+			   const struct drm_plane_state *state,
+			   u32 hw_videoport);
+	int (*plane_setup)(struct dispc_device *dispc, u32 hw_plane,
+			   const struct drm_plane_state *state,
+			   u32 hw_videoport);
+
+	int (*runtime_get)(struct dispc_device *dispc);
+	void (*runtime_put)(struct dispc_device *dispc);
+
+	int (*runtime_suspend)(struct dispc_device *dispc);
+	int (*runtime_resume)(struct dispc_device *dispc);
+
+	void (*remove)(struct dispc_device *dispc);
+
+	int (*modeset_init)(struct dispc_device *dispc);
+
+	int (*get_irq)(struct dispc_device *dispc);
+
+	bool (*has_writeback)(struct dispc_device *dispc);
+	int (*wb_setup)(struct dispc_device *dispc,
+			const struct drm_plane_state *state,
+			bool mem_to_mem, u32 src_hw_plane,
+			u32 src_hw_videoport);
+	int (*wb_enable)(struct dispc_device *dispc, bool enable);
+	u32 (*wb_get_reserved_ovr)(struct dispc_device *dispc);
+};
+
+int dispc6_init(struct tidss_device *tidss);
+int dispc7_init(struct tidss_device *tidss);
+
+#endif
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/tidss_drv.c b/drivers/gpu/drm/tidss/tidss_drv.c
--- a/drivers/gpu/drm/tidss/tidss_drv.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/tidss_drv.c	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,380 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ */
+
+#include <linux/console.h>
+#include <linux/of_device.h>
+#include <linux/pm_runtime.h>
+
+#include <drm/drmP.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_gem_cma_helper.h>
+
+#include "tidss_dispc.h"
+#include "tidss_drv.h"
+#include "tidss_irq.h"
+#include "tidss_kms.h"
+
+/* -----------------------------------------------------------------------------
+ * Device Information
+ */
+
+DEFINE_DRM_GEM_CMA_FOPS(tidss_fops);
+
+static struct drm_driver tidss_driver = {
+	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_PRIME
+				| DRIVER_ATOMIC | DRIVER_HAVE_IRQ,
+	.gem_free_object_unlocked = drm_gem_cma_free_object,
+	.gem_vm_ops		= &drm_gem_cma_vm_ops,
+	.prime_handle_to_fd	= drm_gem_prime_handle_to_fd,
+	.prime_fd_to_handle	= drm_gem_prime_fd_to_handle,
+	.gem_prime_import	= drm_gem_prime_import,
+	.gem_prime_export	= drm_gem_prime_export,
+	.gem_prime_get_sg_table	= drm_gem_cma_prime_get_sg_table,
+	.gem_prime_import_sg_table = drm_gem_cma_prime_import_sg_table,
+	.gem_prime_vmap		= drm_gem_cma_prime_vmap,
+	.gem_prime_vunmap	= drm_gem_cma_prime_vunmap,
+	.gem_prime_mmap		= drm_gem_cma_prime_mmap,
+	.dumb_create		= drm_gem_cma_dumb_create,
+	.fops			= &tidss_fops,
+	.name			= "tidss",
+	.desc			= "TI Keystone DSS",
+	.date			= "20180215",
+	.major			= 1,
+	.minor			= 0,
+
+	.irq_preinstall		= tidss_irq_preinstall,
+	.irq_postinstall	= tidss_irq_postinstall,
+	.irq_handler		= tidss_irq_handler,
+	.irq_uninstall		= tidss_irq_uninstall,
+};
+
+#ifdef CONFIG_PM
+/* -----------------------------------------------------------------------------
+ * Power management
+ */
+
+static int tidss_pm_runtime_suspend(struct device *dev)
+{
+	struct tidss_device *tidss = dev_get_drvdata(dev);
+
+	dev_dbg(dev, "%s\n", __func__);
+
+	return tidss->dispc_ops->runtime_suspend(tidss->dispc);
+}
+
+static int tidss_pm_runtime_resume(struct device *dev)
+{
+	struct tidss_device *tidss = dev_get_drvdata(dev);
+	int r;
+
+	dev_dbg(dev, "%s\n", __func__);
+
+	r = tidss->dispc_ops->runtime_resume(tidss->dispc);
+	if (r)
+		return r;
+
+	tidss_irq_resume(tidss->ddev);
+
+	return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int tidss_suspend(struct device *dev)
+{
+	struct tidss_device *tidss = dev_get_drvdata(dev);
+
+	dev_dbg(dev, "%s\n", __func__);
+
+	return drm_mode_config_helper_suspend(tidss->ddev);
+}
+
+static int tidss_resume(struct device *dev)
+{
+	struct tidss_device *tidss = dev_get_drvdata(dev);
+
+	dev_dbg(dev, "%s\n", __func__);
+
+	return drm_mode_config_helper_resume(tidss->ddev);
+}
+#endif /* CONFIG_PM_SLEEP */
+
+static const struct dev_pm_ops tidss_pm_ops = {
+	.runtime_suspend = tidss_pm_runtime_suspend,
+	.runtime_resume = tidss_pm_runtime_resume,
+	SET_SYSTEM_SLEEP_PM_OPS(tidss_suspend, tidss_resume)
+};
+
+#endif /* CONFIG_PM */
+
+/* -----------------------------------------------------------------------------
+ * Platform driver
+ */
+
+static int tidss_init_remote_device(struct tidss_device *tidss)
+{
+	int ret;
+	struct device *dev = tidss->dev;
+	struct device_node *dss_remote_dev_node;
+	const char *name;
+
+	dss_remote_dev_node = of_get_child_by_name(dev->of_node, "dss-remote");
+	if (!dss_remote_dev_node)
+		return 0;
+
+	if (!of_find_property(dss_remote_dev_node, "remote-name", NULL)) {
+		ret = 0;
+		goto out;
+	}
+
+	ret = of_property_read_string(dss_remote_dev_node, "remote-name", &name);
+	if (ret) {
+		dev_err(dev, "%s: could not read remote-name property\n", __func__);
+		goto out;
+	}
+
+	tidss->rdev = rpmsg_remotedev_get_named_device(name);
+	if (!tidss->rdev) {
+		ret = -EPROBE_DEFER;
+		goto out;
+	} else if (IS_ERR(tidss->rdev)) {
+		ret = PTR_ERR(tidss->rdev);
+		goto out;
+	}
+
+	tidss->rdev->cb_data = tidss;
+
+	if (tidss->rdev->device.display.ops->ready == NULL ||
+			tidss->rdev->device.display.ops->get_res_info == NULL ||
+			tidss->rdev->device.display.ops->commit == NULL) {
+		dev_err(dev, "%s: rpmsg remotedev ops not complete\n", __func__);
+		ret = -EINVAL;
+		goto disconnect;
+	}
+
+	/* Cant really do much if the remotedev is not ready yet */
+	if (!tidss->rdev->device.display.ops->ready(tidss->rdev)) {
+		ret = -EPROBE_DEFER;
+		goto disconnect;
+	}
+
+	goto out;
+
+disconnect:
+	rpmsg_remotedev_put_device(tidss->rdev);
+	tidss->rdev->cb_data = NULL;
+	tidss->rdev = NULL;
+out:
+	of_node_put(dss_remote_dev_node);
+	return ret;
+}
+
+static int tidss_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct tidss_device *tidss;
+	struct drm_device *ddev;
+	int ret;
+	int irq;
+
+	dev_dbg(dev, "%s\n", __func__);
+
+	tidss = devm_kzalloc(dev, sizeof(*tidss), GFP_KERNEL);
+	if (tidss == NULL)
+		return -ENOMEM;
+
+	tidss->dev = dev;
+	tidss->features = of_device_get_match_data(dev);
+
+	ret = tidss_init_remote_device(tidss);
+	if (ret)
+		return ret;
+
+	platform_set_drvdata(pdev, tidss);
+
+	ddev = drm_dev_alloc(&tidss_driver, dev);
+	if (IS_ERR(ddev)) {
+		ret = PTR_ERR(ddev);
+		goto err_fini_rdev;
+	}
+
+	tidss->ddev = ddev;
+	ddev->dev_private = tidss;
+
+	pm_runtime_enable(dev);
+
+	ret = tidss->features->dispc_init(tidss);
+	if (ret) {
+		dev_err(dev, "failed to initialize dispc: %d\n", ret);
+		goto err_disable_pm;
+	}
+
+#ifndef CONFIG_PM
+	/* If we don't have PM, we need to call resume manually */
+	tidss->dispc_ops->runtime_resume(tidss->dispc);
+#endif
+
+	ret = tidss_modeset_init(tidss);
+	if (ret < 0) {
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "failed to init DRM/KMS (%d)\n", ret);
+		goto err_runtime_suspend;
+	}
+
+	irq = tidss->dispc_ops->get_irq(tidss->dispc);
+	if (irq < 0) {
+		ret = irq;
+		dev_err(dev, "failed to get dispc irq: %d\n", ret);
+		goto err_modeset_cleanup;
+	}
+
+	ret = drm_irq_install(ddev, irq);
+	if (ret) {
+		dev_err(dev, "drm_irq_install failed: %d\n", ret);
+		goto err_modeset_cleanup;
+	}
+
+	drm_kms_helper_poll_init(ddev);
+
+	if (tidss->dispc_ops->has_writeback(tidss->dispc)) {
+		ret = tidss_wb_init(ddev);
+		if (ret)
+			dev_warn(dev, "failed to initialize writeback\n");
+		else
+			tidss->wb_initialized = true;
+	}
+
+	ret = drm_dev_register(ddev, 0);
+	if (ret) {
+		dev_err(dev, "failed to register DRM device\n");
+		goto err_poll_fini;
+	}
+
+	drm_fbdev_generic_setup(ddev, 32);
+
+	dev_dbg(dev, "%s done\n", __func__);
+
+	return 0;
+
+err_poll_fini:
+	if (tidss->wb_initialized)
+		tidss_wb_cleanup(ddev);
+
+	drm_kms_helper_poll_fini(ddev);
+
+	drm_atomic_helper_shutdown(ddev);
+
+	drm_irq_uninstall(ddev);
+
+err_modeset_cleanup:
+	tidss_modeset_cleanup(tidss);
+
+err_runtime_suspend:
+#ifndef CONFIG_PM
+	/* If we don't have PM, we need to call suspend manually */
+	tidss->dispc_ops->runtime_suspend(tidss->dispc);
+#endif
+
+	tidss->dispc_ops->remove(tidss->dispc);
+
+err_disable_pm:
+	pm_runtime_disable(dev);
+
+err_fini_rdev:
+	if (tidss->rdev)
+		rpmsg_remotedev_put_device(tidss->rdev);
+
+	drm_dev_put(ddev);
+
+	return ret;
+}
+
+static int tidss_remove(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct tidss_device *tidss = platform_get_drvdata(pdev);
+	struct drm_device *ddev = tidss->ddev;
+
+	dev_dbg(dev, "%s\n", __func__);
+
+	drm_dev_unregister(ddev);
+
+	if (tidss->wb_initialized)
+		tidss_wb_cleanup(ddev);
+
+	drm_kms_helper_poll_fini(ddev);
+
+	drm_atomic_helper_shutdown(ddev);
+
+	drm_irq_uninstall(ddev);
+
+	tidss_modeset_cleanup(tidss);
+
+#ifndef CONFIG_PM
+	/* If we don't have PM, we need to call suspend manually */
+	tidss->dispc_ops->runtime_suspend(tidss->dispc);
+#endif
+
+	tidss->dispc_ops->remove(tidss->dispc);
+
+	pm_runtime_disable(dev);
+
+	drm_dev_put(ddev);
+
+	dev_dbg(dev, "%s done\n", __func__);
+
+	return 0;
+}
+
+#ifdef CONFIG_DRM_TIDSS_DSS6
+static const struct tidss_features tidss_k2g_features = {
+	.dispc_init = dispc6_init,
+};
+#endif
+
+#ifdef CONFIG_DRM_TIDSS_DSS7
+static const struct tidss_features tidss_am6_features = {
+	.dispc_init = dispc7_init,
+};
+
+static const struct tidss_features tidss_j721e_features = {
+	.dispc_init = dispc7_init,
+};
+#endif
+static const struct of_device_id tidss_of_table[] = {
+#ifdef CONFIG_DRM_TIDSS_DSS6
+	{ .compatible = "ti,k2g-dss", .data = &tidss_k2g_features },
+#endif
+#ifdef CONFIG_DRM_TIDSS_DSS7
+	{ .compatible = "ti,am6-dss", .data = &tidss_am6_features },
+	{ .compatible = "ti,j721e-dss", .data = &tidss_j721e_features },
+#endif
+	{ }
+};
+
+MODULE_DEVICE_TABLE(of, tidss_of_table);
+
+static struct platform_driver tidss_platform_driver = {
+	.probe		= tidss_probe,
+	.remove		= tidss_remove,
+	.driver		= {
+		.name	= "tidss",
+#ifdef CONFIG_PM
+		.pm	= &tidss_pm_ops,
+#endif
+		.of_match_table = tidss_of_table,
+		.suppress_bind_attrs = true,
+	},
+};
+
+module_platform_driver(tidss_platform_driver);
+
+MODULE_AUTHOR("Tomi Valkeinen <tomi.valkeinen@ti.com>");
+MODULE_DESCRIPTION("TI Keystone DSS Driver");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/tidss_drv.h b/drivers/gpu/drm/tidss/tidss_drv.h
--- a/drivers/gpu/drm/tidss/tidss_drv.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/tidss_drv.h	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,64 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ */
+
+#ifndef __TIDSS_DRV_H__
+#define __TIDSS_DRV_H__
+
+#include <linux/spinlock.h>
+#include <linux/rpmsg-remotedev/rpmsg-remotedev.h>
+
+struct wb_dev;
+
+struct tidss_device {
+	struct device *dev;		/* Underlying DSS device */
+	struct drm_device *ddev;	/* DRM device for DSS */
+
+	struct rpmsg_remotedev *rdev;
+	struct rpmsg_remotedev_display_resinfo rres;
+
+	struct drm_fbdev_cma *fbdev;
+
+	struct dispc_device *dispc;
+	const struct tidss_dispc_ops *dispc_ops;
+
+	const struct tidss_features *features;
+
+	unsigned int num_crtcs;
+	unsigned int num_v_crtcs;
+	struct drm_crtc *crtcs[8];
+	struct drm_crtc *v_crtcs[8];
+
+	unsigned int num_planes;
+	struct drm_plane *planes[8];
+
+	spinlock_t wait_lock;	/* protects the irq masks */
+	u64 irq_mask;		/* enabled irqs in addition to wait_list */
+
+	struct drm_atomic_state *saved_state;
+
+	struct wb_dev *wdev;	/* Write-back private data */
+	bool wb_initialized;
+};
+
+struct tidss_features {
+	int (*dispc_init)(struct tidss_device *tidss);
+};
+
+#if IS_ENABLED(CONFIG_DRM_TIDSS_WB)
+
+int tidss_wb_init(struct drm_device *drmdev);
+void tidss_wb_cleanup(struct drm_device *drmdev);
+void tidss_wb_irq(struct wb_dev *wdev, u64 irqstatus);
+
+#else
+
+static inline int tidss_wb_init(struct drm_device *drmdev) { return 0; }
+static inline void tidss_wb_cleanup(struct drm_device *drmdev) { }
+static inline void tidss_wb_irq(struct wb_dev *wdev, u64 irqstatus) { }
+
+#endif
+
+#endif
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/tidss_encoder.c b/drivers/gpu/drm/tidss/tidss_encoder.c
--- a/drivers/gpu/drm/tidss/tidss_encoder.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/tidss_encoder.c	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,93 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ */
+
+#include <linux/export.h>
+
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_of.h>
+
+#include "tidss_drv.h"
+#include "tidss_encoder.h"
+#include "tidss_crtc.h"
+
+/* -----------------------------------------------------------------------------
+ * Encoder
+ */
+
+static int tidss_encoder_atomic_check(struct drm_encoder *encoder,
+				      struct drm_crtc_state *crtc_state,
+				      struct drm_connector_state *conn_state)
+{
+	struct drm_device *ddev = encoder->dev;
+	struct tidss_crtc_state *tcrtc_state = to_tidss_crtc_state(crtc_state);
+	struct drm_display_info *di = &conn_state->connector->display_info;
+	struct drm_bridge *bridge;
+	bool bus_flags_set = false;
+
+	dev_dbg(ddev->dev, "%s\n", __func__);
+
+	/*
+	 * Take the bus_flags from the first bridge that defines
+	 * bridge timings, or from the connector's display_info if no
+	 * bridge defines the timings.
+	 */
+	for (bridge = encoder->bridge; bridge; bridge = bridge->next) {
+		if (!bridge->timings)
+			continue;
+
+		tcrtc_state->bus_flags = bridge->timings->input_bus_flags;
+		bus_flags_set = true;
+		break;
+	}
+
+	if (!di->bus_formats || di->num_bus_formats == 0)  {
+		dev_err(ddev->dev, "%s: No bus_formats in connected display\n",
+			__func__);
+		return -EINVAL;
+	}
+
+	// XXX any cleaner way to set bus format and flags?
+	tcrtc_state->bus_format = di->bus_formats[0];
+	if (!bus_flags_set)
+		tcrtc_state->bus_flags = di->bus_flags;
+
+	return 0;
+}
+
+static const struct drm_encoder_helper_funcs encoder_helper_funcs = {
+	.atomic_check = tidss_encoder_atomic_check,
+};
+
+static const struct drm_encoder_funcs encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
+struct drm_encoder *tidss_encoder_create(struct tidss_device *tidss,
+					 u32 encoder_type, u32 possible_crtcs)
+{
+	struct drm_encoder *enc;
+	int ret;
+
+	enc = devm_kzalloc(tidss->dev, sizeof(*enc), GFP_KERNEL);
+	if (!enc)
+		return ERR_PTR(-ENOMEM);
+
+	enc->possible_crtcs = possible_crtcs;
+
+	ret = drm_encoder_init(tidss->ddev, enc, &encoder_funcs,
+			       encoder_type, NULL);
+	if (ret < 0)
+		return ERR_PTR(ret);
+
+	drm_encoder_helper_add(enc, &encoder_helper_funcs);
+
+	dev_dbg(tidss->dev, "Encoder create done\n");
+
+	return enc;
+}
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/tidss_encoder.h b/drivers/gpu/drm/tidss/tidss_encoder.h
--- a/drivers/gpu/drm/tidss/tidss_encoder.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/tidss_encoder.h	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ */
+
+#ifndef __TIDSS_ENCODER_H__
+#define __TIDSS_ENCODER_H__
+
+#include <drm/drm_encoder.h>
+
+struct tidss_device;
+
+struct drm_encoder *tidss_encoder_create(struct tidss_device *tidss,
+					 u32 encoder_type, u32 possible_crtcs);
+
+#endif
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/tidss_irq.c b/drivers/gpu/drm/tidss/tidss_irq.c
--- a/drivers/gpu/drm/tidss/tidss_irq.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/tidss_irq.c	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,159 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ */
+
+#include <drm/drmP.h>
+
+#include "tidss_irq.h"
+#include "tidss_drv.h"
+#include "tidss_dispc.h"
+#include "tidss_crtc.h"
+#include "tidss_plane.h"
+
+/* call with wait_lock and dispc runtime held */
+static void tidss_irq_update(struct drm_device *ddev)
+{
+	struct tidss_device *tidss = ddev->dev_private;
+
+	assert_spin_locked(&tidss->wait_lock);
+
+	tidss->dispc_ops->write_irqenable(tidss->dispc, tidss->irq_mask);
+}
+
+void tidss_irq_enable_vblank(struct drm_crtc *crtc)
+{
+	struct drm_device *ddev = crtc->dev;
+	struct tidss_device *tidss = ddev->dev_private;
+	struct tidss_crtc *tcrtc = to_tidss_crtc(crtc);
+	u32 hw_videoport = tcrtc->hw_videoport;
+	unsigned long flags;
+
+	spin_lock_irqsave(&tidss->wait_lock, flags);
+	tidss->irq_mask |= DSS_IRQ_VP_VSYNC_EVEN(hw_videoport) |
+			   DSS_IRQ_VP_VSYNC_ODD(hw_videoport);
+	tidss_irq_update(ddev);
+	spin_unlock_irqrestore(&tidss->wait_lock, flags);
+}
+
+void tidss_irq_disable_vblank(struct drm_crtc *crtc)
+{
+	struct drm_device *ddev = crtc->dev;
+	struct tidss_device *tidss = ddev->dev_private;
+	struct tidss_crtc *tcrtc = to_tidss_crtc(crtc);
+	u32 hw_videoport = tcrtc->hw_videoport;
+	unsigned long flags;
+
+	spin_lock_irqsave(&tidss->wait_lock, flags);
+	tidss->irq_mask &= ~(DSS_IRQ_VP_VSYNC_EVEN(hw_videoport) |
+			     DSS_IRQ_VP_VSYNC_ODD(hw_videoport));
+	tidss_irq_update(ddev);
+	spin_unlock_irqrestore(&tidss->wait_lock, flags);
+}
+
+static void tidss_irq_ocp_error_handler(struct drm_device *ddev,
+					u64 irqstatus)
+{
+	if (irqstatus & DSS_IRQ_DEVICE_OCP_ERR)
+		dev_err_ratelimited(ddev->dev, "OCP error\n");
+}
+
+irqreturn_t tidss_irq_handler(int irq, void *arg)
+{
+	struct drm_device *ddev = (struct drm_device *) arg;
+	struct tidss_device *tidss = ddev->dev_private;
+	unsigned int id;
+	u64 irqstatus;
+
+	if (WARN_ON(!ddev->irq_enabled))
+		return IRQ_NONE;
+
+	irqstatus = tidss->dispc_ops->read_and_clear_irqstatus(tidss->dispc);
+
+	for (id = 0; id < tidss->num_crtcs; id++) {
+		struct drm_crtc *crtc = tidss->crtcs[id];
+		struct tidss_crtc *tcrtc = to_tidss_crtc(crtc);
+		u32 hw_videoport = tcrtc->hw_videoport;
+
+		if (irqstatus & (DSS_IRQ_VP_VSYNC_EVEN(hw_videoport) |
+				 DSS_IRQ_VP_VSYNC_ODD(hw_videoport)))
+			tidss_crtc_vblank_irq(crtc);
+
+		if (irqstatus & (DSS_IRQ_VP_FRAME_DONE(hw_videoport)))
+			tidss_crtc_framedone_irq(crtc);
+
+		if (irqstatus & DSS_IRQ_VP_SYNC_LOST(hw_videoport))
+			tidss_crtc_error_irq(crtc, irqstatus);
+	}
+
+	tidss_irq_ocp_error_handler(ddev, irqstatus);
+	tidss_wb_irq(tidss->wdev, irqstatus);
+
+	return IRQ_HANDLED;
+}
+
+void tidss_irq_preinstall(struct drm_device *ddev)
+{
+	struct tidss_device *tidss = ddev->dev_private;
+
+	spin_lock_init(&tidss->wait_lock);
+
+	tidss->dispc_ops->runtime_get(tidss->dispc);
+
+	tidss->dispc_ops->write_irqenable(tidss->dispc, 0);
+	tidss->dispc_ops->read_and_clear_irqstatus(tidss->dispc);
+
+	tidss->dispc_ops->runtime_put(tidss->dispc);
+}
+
+int tidss_irq_postinstall(struct drm_device *ddev)
+{
+	struct tidss_device *tidss = ddev->dev_private;
+	unsigned int i;
+	unsigned long flags;
+
+	tidss->dispc_ops->runtime_get(tidss->dispc);
+
+	spin_lock_irqsave(&tidss->wait_lock, flags);
+
+	tidss->irq_mask = DSS_IRQ_DEVICE_OCP_ERR;
+
+	for (i = 0; i < tidss->num_crtcs; ++i) {
+		struct tidss_crtc *tcrtc = to_tidss_crtc(tidss->crtcs[i]);
+
+		tidss->irq_mask |= DSS_IRQ_VP_SYNC_LOST(tcrtc->hw_videoport);
+
+		tidss->irq_mask |= DSS_IRQ_VP_FRAME_DONE(tcrtc->hw_videoport);
+	}
+
+	if (tidss->dispc_ops->has_writeback(tidss->dispc))
+		tidss->irq_mask |= DSS_IRQ_DEVICE_WB_MASK;
+
+	tidss_irq_update(ddev);
+
+	spin_unlock_irqrestore(&tidss->wait_lock, flags);
+
+	tidss->dispc_ops->runtime_put(tidss->dispc);
+
+	return 0;
+}
+
+void tidss_irq_uninstall(struct drm_device *ddev)
+{
+	struct tidss_device *tidss = ddev->dev_private;
+
+	tidss->dispc_ops->runtime_get(tidss->dispc);
+	tidss->dispc_ops->write_irqenable(tidss->dispc, 0);
+	tidss->dispc_ops->runtime_put(tidss->dispc);
+}
+
+void tidss_irq_resume(struct drm_device *ddev)
+{
+	struct tidss_device *tidss = ddev->dev_private;
+	unsigned long flags;
+
+	spin_lock_irqsave(&tidss->wait_lock, flags);
+	tidss_irq_update(ddev);
+	spin_unlock_irqrestore(&tidss->wait_lock, flags);
+}
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/tidss_irq.h b/drivers/gpu/drm/tidss/tidss_irq.h
--- a/drivers/gpu/drm/tidss/tidss_irq.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/tidss_irq.h	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ */
+
+#ifndef __TIDSS_IRQ_H__
+#define __TIDSS_IRQ_H__
+
+#include <linux/types.h>
+
+struct drm_crtc;
+struct drm_device;
+
+void tidss_irq_enable_vblank(struct drm_crtc *crtc);
+void tidss_irq_disable_vblank(struct drm_crtc *crtc);
+
+void tidss_irq_preinstall(struct drm_device *ddev);
+int tidss_irq_postinstall(struct drm_device *ddev);
+void tidss_irq_uninstall(struct drm_device *ddev);
+irqreturn_t tidss_irq_handler(int irq, void *arg);
+
+void tidss_irq_resume(struct drm_device *ddev);
+
+#endif
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/tidss_kms.c b/drivers/gpu/drm/tidss/tidss_kms.c
--- a/drivers/gpu/drm/tidss/tidss_kms.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/tidss_kms.c	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,195 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ */
+
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_gem_framebuffer_helper.h>
+
+#include "tidss_crtc.h"
+#include "tidss_drv.h"
+#include "tidss_encoder.h"
+#include "tidss_kms.h"
+#include "tidss_plane.h"
+#include "tidss_v_display.h"
+
+static bool tidss_is_v_crtc(struct drm_crtc *crtc)
+{
+	int i;
+	struct drm_device *dev = crtc->dev;
+	struct tidss_device *tidss = dev->dev_private;
+
+	for (i = 0; i < tidss->num_v_crtcs; i++)
+		if (crtc == tidss->v_crtcs[i])
+			return true;
+	return false;
+}
+
+static void tidss_atomic_commit_tail(struct drm_atomic_state *old_state)
+{
+	struct drm_device *ddev = old_state->dev;
+	struct tidss_device *tidss = ddev->dev_private;
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
+	int i;
+
+	dev_dbg(ddev->dev, "%s\n", __func__);
+
+	for_each_oldnew_crtc_in_state(old_state, crtc, old_crtc_state, new_crtc_state, i)
+		if (!tidss_is_v_crtc(crtc))
+			tidss->dispc_ops->runtime_get(tidss->dispc);
+
+	drm_atomic_helper_commit_modeset_disables(ddev, old_state);
+	drm_atomic_helper_commit_planes(ddev, old_state, 0);
+	drm_atomic_helper_commit_modeset_enables(ddev, old_state);
+
+	drm_atomic_helper_commit_hw_done(old_state);
+	drm_atomic_helper_wait_for_flip_done(ddev, old_state);
+
+	drm_atomic_helper_cleanup_planes(ddev, old_state);
+
+	for_each_oldnew_crtc_in_state(old_state, crtc, old_crtc_state, new_crtc_state, i)
+		if (!tidss_is_v_crtc(crtc))
+			tidss->dispc_ops->runtime_put(tidss->dispc);
+}
+
+static const struct drm_mode_config_helper_funcs mode_config_helper_funcs = {
+	.atomic_commit_tail = tidss_atomic_commit_tail,
+};
+
+static const struct drm_mode_config_funcs mode_config_funcs = {
+	.fb_create = drm_gem_fb_create,
+	.atomic_check = drm_atomic_helper_check,
+	.atomic_commit = drm_atomic_helper_commit,
+};
+
+static struct drm_crtc *tidss_v_modeset_init_v_crtc(struct tidss_device *tidss, struct rpmsg_remotedev_display_disp *vp)
+{
+	struct drm_device *dev = tidss->ddev;
+	struct drm_encoder *encoder;
+	struct drm_connector *connector;
+	struct drm_plane *planes[8];
+	struct drm_crtc *crtc;
+	int last_crtc = tidss->num_crtcs + tidss->num_v_crtcs;
+	int num_planes = 0;
+	int p;
+
+	encoder = v_encoder_init(tidss, last_crtc, 1 << last_crtc, vp);
+	if (!encoder) {
+		dev_err(dev->dev, "could not create encoder: %u\n", last_crtc);
+		return NULL;
+	}
+
+	connector = v_connector_init(tidss, last_crtc, encoder, vp);
+	if (!connector) {
+		dev_err(dev->dev, "could not create connector: %u\n", last_crtc);
+		goto connector_fail;
+	}
+
+	for (p = 0; p < vp->num_pipes; p++) {
+		int plane_id = p << 8 | last_crtc;
+
+		planes[p] = v_plane_init(tidss, plane_id, (1 << last_crtc),
+				p == 0 ? DRM_PLANE_TYPE_PRIMARY : DRM_PLANE_TYPE_OVERLAY,
+				vp, p);
+		if (!planes[p]) {
+			dev_err(dev->dev, "could not create plane: %u\n", plane_id);
+			goto plane_fail;
+		}
+		num_planes++;
+	}
+
+	crtc = v_crtc_init(tidss, last_crtc, planes[0], vp);
+	if (!crtc) {
+		dev_err(dev->dev, "could not create crtc: %d\n", last_crtc);
+		goto plane_fail;
+	}
+
+	tidss->num_v_crtcs++;
+
+	return crtc;
+
+plane_fail:
+	for (p = 0; p < num_planes; p++)
+		v_plane_fini(tidss, planes[p]);
+	v_connector_fini(tidss, connector);
+connector_fail:
+	v_encoder_fini(tidss, encoder);
+	return NULL;
+}
+
+static const struct rpmsg_remotedev_display_cb tidss_rdev_cb  = {
+	.commit_done = v_crtc_commit_done,
+	.buffer_done = v_crtc_buffer_done,
+};
+
+int tidss_modeset_init(struct tidss_device *tidss)
+{
+	struct drm_device *ddev = tidss->ddev;
+	unsigned int i;
+	int ret;
+
+	dev_dbg(tidss->dev, "%s\n", __func__);
+
+	drm_mode_config_init(ddev);
+
+	ddev->mode_config.min_width = 8;
+	ddev->mode_config.min_height = 8;
+	ddev->mode_config.max_width = 8096;
+	ddev->mode_config.max_height = 8096;
+	ddev->mode_config.normalize_zpos = true;
+	ddev->mode_config.funcs = &mode_config_funcs;
+	ddev->mode_config.helper_private = &mode_config_helper_funcs;
+
+	ret = tidss->dispc_ops->modeset_init(tidss->dispc);
+	if (ret)
+		goto err_mode_config_cleanup;
+
+	if (tidss->rdev) {
+		tidss->rdev->device.display.ops->get_res_info(tidss->rdev, &tidss->rres);
+
+		if (tidss->rres.num_disps)
+			tidss->rdev->device.display.cb_ops = &tidss_rdev_cb;
+
+		for (i = 0; i < tidss->rres.num_disps; i++) {
+			tidss->v_crtcs[i] = tidss_v_modeset_init_v_crtc(tidss, &tidss->rres.disps[i]);
+			if (!tidss->v_crtcs[i]) {
+				ret = -ENOMEM;
+				goto err_mode_config_cleanup;
+			}
+		}
+	}
+
+	ret = drm_vblank_init(ddev, tidss->num_crtcs + tidss->num_v_crtcs);
+	if (ret)
+		goto err_mode_config_cleanup;
+
+	/* Start with vertical blanking interrupt reporting disabled. */
+	for (i = 0; i < tidss->num_crtcs; ++i)
+		drm_crtc_vblank_reset(tidss->crtcs[i]);
+
+	for (i = 0; i < tidss->num_v_crtcs; ++i)
+		drm_crtc_vblank_reset(tidss->v_crtcs[i]);
+
+	drm_mode_config_reset(ddev);
+
+	dev_dbg(tidss->dev, "%s done\n", __func__);
+
+	return 0;
+
+err_mode_config_cleanup:
+	drm_mode_config_cleanup(ddev);
+	return ret;
+}
+
+void tidss_modeset_cleanup(struct tidss_device *tidss)
+{
+	struct drm_device *ddev = tidss->ddev;
+
+	drm_mode_config_cleanup(ddev);
+}
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/tidss_kms.h b/drivers/gpu/drm/tidss/tidss_kms.h
--- a/drivers/gpu/drm/tidss/tidss_kms.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/tidss_kms.h	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ */
+
+#ifndef __TIDSS_KMS_H__
+#define __TIDSS_KMS_H__
+
+struct tidss_device;
+
+int tidss_modeset_init(struct tidss_device *tidss);
+void tidss_modeset_cleanup(struct tidss_device *tidss);
+
+#endif
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/tidss_plane.c b/drivers/gpu/drm/tidss/tidss_plane.c
--- a/drivers/gpu/drm/tidss/tidss_plane.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/tidss_plane.c	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,285 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ */
+
+#include <drm/drmP.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_plane_helper.h>
+
+#include "tidss_crtc.h"
+#include "tidss_drv.h"
+#include "tidss_plane.h"
+
+dma_addr_t dispc7_plane_state_paddr(const struct drm_plane_state *state)
+{
+	struct drm_framebuffer *fb = state->fb;
+	struct drm_gem_cma_object *gem;
+	u32 x = state->src_x >> 16;
+	u32 y = state->src_y >> 16;
+
+	gem = drm_fb_cma_get_gem_obj(state->fb, 0);
+
+	return gem->paddr + fb->offsets[0] + x * fb->format->cpp[0] +
+		y * fb->pitches[0];
+}
+
+dma_addr_t dispc7_plane_state_p_uv_addr(const struct drm_plane_state *state)
+{
+	struct drm_framebuffer *fb = state->fb;
+	struct drm_gem_cma_object *gem;
+	u32 x = state->src_x >> 16;
+	u32 y = state->src_y >> 16;
+
+	if (WARN_ON(state->fb->format->num_planes != 2))
+		return 0;
+
+	gem = drm_fb_cma_get_gem_obj(fb, 1);
+
+	return gem->paddr + fb->offsets[1] +
+		(x * fb->format->cpp[1] / fb->format->hsub) +
+		(y * fb->pitches[1] / fb->format->vsub);
+}
+
+static int tidss_plane_atomic_check(struct drm_plane *plane,
+				    struct drm_plane_state *state)
+{
+	struct drm_device *ddev = plane->dev;
+	struct tidss_device *tidss = ddev->dev_private;
+	struct drm_crtc_state *crtc_state;
+	struct tidss_plane *tplane = to_tidss_plane(plane);
+	const struct drm_format_info *finfo;
+	u32 hw_videoport;
+	int ret;
+
+	dev_dbg(ddev->dev, "%s\n", __func__);
+
+	if (tplane->reserved_wb)
+		return -EBUSY;
+
+	if (!state->crtc) {
+		/*
+		 * The visible field is not reset by the DRM core but only
+		 * updated by drm_plane_helper_check_state(), set it manually.
+		 */
+		state->visible = false;
+		return 0;
+	}
+
+	crtc_state = drm_atomic_get_crtc_state(state->state, state->crtc);
+	if (IS_ERR(crtc_state))
+		return PTR_ERR(crtc_state);
+
+	ret = drm_atomic_helper_check_plane_state(state, crtc_state,
+						  0,
+						  INT_MAX,
+						  true, true);
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * The HW is only able to start drawing at subpixel boundary
+	 * (the two first checks bellow). At the end of a row the HW
+	 * can only jump integer number of subpixels forward to
+	 * beginning of next row. So we can only show picture with
+	 * integer subpixel width (the third check). However, after
+	 * reaching the end of the drawn picture the drawing starts
+	 * again at the absolute memory address where top left corner
+	 * position of the drawn picture is (so there is no need to
+	 * check for odd height).
+	 */
+
+	finfo = drm_format_info(state->fb->format->format);
+
+	if ((state->src_x >> 16) % finfo->hsub != 0) {
+		dev_dbg(ddev->dev,
+			"%s: x-position %u not divisible subpixel size %u\n",
+			__func__, (state->src_x >> 16), finfo->hsub);
+		return -EINVAL;
+	}
+
+	if ((state->src_y >> 16) % finfo->vsub != 0) {
+		dev_dbg(ddev->dev,
+			"%s: y-position %u not divisible subpixel size %u\n",
+			__func__, (state->src_y >> 16), finfo->vsub);
+		return -EINVAL;
+	}
+
+	if ((state->src_w >> 16) % finfo->hsub != 0) {
+		dev_dbg(ddev->dev,
+			"%s: src width %u not divisible by subpixel size %u\n",
+			 __func__, (state->src_w >> 16), finfo->hsub);
+		return -EINVAL;
+	}
+
+	if (!state->visible)
+		return 0;
+
+	hw_videoport = to_tidss_crtc(state->crtc)->hw_videoport;
+
+	return tidss->dispc_ops->plane_check(tidss->dispc,
+					     tplane->hw_plane_id,
+					     state, hw_videoport);
+}
+
+static void tidss_plane_atomic_update(struct drm_plane *plane,
+				      struct drm_plane_state *old_state)
+{
+	struct drm_device *ddev = plane->dev;
+	struct tidss_device *tidss = ddev->dev_private;
+	struct tidss_plane *tplane = to_tidss_plane(plane);
+	struct drm_plane_state *state = plane->state;
+	u32 hw_videoport;
+	int ret;
+
+	dev_dbg(ddev->dev, "%s\n", __func__);
+
+	if (!state->visible) {
+		tidss->dispc_ops->plane_enable(tidss->dispc, tplane->hw_plane_id,
+					       false);
+		return;
+	}
+
+	hw_videoport = to_tidss_crtc(state->crtc)->hw_videoport;
+
+	ret = tidss->dispc_ops->plane_setup(tidss->dispc, tplane->hw_plane_id,
+					    state, hw_videoport);
+
+	if (ret) {
+		dev_err(plane->dev->dev, "Failed to setup plane %d\n",
+			tplane->hw_plane_id);
+		tidss->dispc_ops->plane_enable(tidss->dispc, tplane->hw_plane_id,
+					       false);
+		return;
+	}
+
+	tidss->dispc_ops->plane_enable(tidss->dispc, tplane->hw_plane_id, true);
+}
+
+static void tidss_plane_atomic_disable(struct drm_plane *plane,
+				       struct drm_plane_state *old_state)
+{
+	struct drm_device *ddev = plane->dev;
+	struct tidss_device *tidss = ddev->dev_private;
+	struct tidss_plane *tplane = to_tidss_plane(plane);
+
+	dev_dbg(ddev->dev, "%s\n", __func__);
+
+	tidss->dispc_ops->plane_enable(tidss->dispc, tplane->hw_plane_id, false);
+}
+
+static const struct drm_plane_helper_funcs tidss_plane_helper_funcs = {
+	.atomic_check = tidss_plane_atomic_check,
+	.atomic_update = tidss_plane_atomic_update,
+	.atomic_disable = tidss_plane_atomic_disable,
+};
+
+static const struct drm_plane_funcs tidss_plane_funcs = {
+	.update_plane = drm_atomic_helper_update_plane,
+	.disable_plane = drm_atomic_helper_disable_plane,
+	.reset = drm_atomic_helper_plane_reset,
+	.destroy = drm_plane_cleanup,
+	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
+};
+
+struct tidss_plane *tidss_plane_create(struct tidss_device *tidss,
+				       u32 hw_plane_id,	u32 plane_type,
+				       u32 crtc_mask, const u32 *formats,
+				       u32 num_formats)
+{
+	const struct tidss_plane_feat *pfeat;
+	struct tidss_plane *tplane;
+	enum drm_plane_type type;
+	u32 possible_crtcs;
+	u32 num_planes = tidss->dispc_ops->get_num_planes(tidss->dispc);
+	int ret;
+
+	pfeat = tidss->dispc_ops->plane_feat(tidss->dispc, hw_plane_id);
+
+	tplane = devm_kzalloc(tidss->dev, sizeof(*tplane), GFP_KERNEL);
+	if (!tplane)
+		return ERR_PTR(-ENOMEM);
+
+	tplane->hw_plane_id = hw_plane_id;
+
+	possible_crtcs = crtc_mask;
+	type = plane_type;
+
+	ret = drm_universal_plane_init(tidss->ddev, &tplane->plane,
+				       possible_crtcs,
+				       &tidss_plane_funcs,
+				       formats, num_formats,
+				       NULL, type, NULL);
+	if (ret < 0)
+		return ERR_PTR(ret);
+
+	drm_plane_helper_add(&tplane->plane, &tidss_plane_helper_funcs);
+	if (num_planes > 1)
+		drm_plane_create_zpos_property(&tplane->plane, hw_plane_id, 0,
+					       num_planes - 1);
+
+	ret = drm_plane_create_color_properties(&tplane->plane,
+						pfeat->color.encodings,
+						pfeat->color.ranges,
+						pfeat->color.default_encoding,
+						pfeat->color.default_range);
+	if (ret)
+		return ERR_PTR(ret);
+
+	if (pfeat->blend.global_alpha) {
+		ret = drm_plane_create_alpha_property(&tplane->plane);
+		if (ret)
+			return ERR_PTR(ret);
+	}
+
+	return tplane;
+}
+
+struct drm_plane *tidss_plane_reserve_wb(struct drm_device *dev)
+{
+	struct tidss_device *tidss = dev->dev_private;
+	int i;
+	u32 ovr_id = tidss->dispc_ops->wb_get_reserved_ovr(tidss->dispc);
+
+	for (i = tidss->num_planes - 1; i >= 0; --i) {
+		struct drm_plane *plane = tidss->planes[i];
+		struct tidss_plane *tplane = to_tidss_plane(plane);
+
+		if (plane->state->crtc || plane->state->fb)
+			continue;
+
+		if (tplane->reserved_wb)
+			continue;
+
+		/*
+		 * We found an available plane so just mark the
+		 * associated video port as the one found in the last step
+		 */
+		tplane->reserved_wb = true;
+
+		dev_dbg(dev->dev, "%s: found plane %s (%d) on %s (%d)\n",
+			__func__,
+			tidss->dispc_ops->plane_name(tidss->dispc, tplane->hw_plane_id),
+			tplane->hw_plane_id,
+			tidss->dispc_ops->vp_name(tidss->dispc, ovr_id), ovr_id);
+
+		return plane;
+	}
+	return NULL;
+}
+
+void tidss_plane_release_wb(struct drm_plane *plane)
+{
+	struct tidss_plane *tplane = to_tidss_plane(plane);
+
+	WARN_ON(!tplane->reserved_wb);
+
+	tplane->reserved_wb = false;
+}
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/tidss_plane.h b/drivers/gpu/drm/tidss/tidss_plane.h
--- a/drivers/gpu/drm/tidss/tidss_plane.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/tidss_plane.h	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ */
+
+#ifndef __TIDSS_PLANE_H__
+#define __TIDSS_PLANE_H__
+
+#include "tidss_dispc.h"
+
+#define to_tidss_plane(p) container_of((p), struct tidss_plane, plane)
+
+struct tidss_plane {
+	struct drm_plane plane;
+
+	u32 hw_plane_id;
+
+	bool reserved_wb;
+
+};
+
+struct tidss_plane *tidss_plane_create(struct tidss_device *tidss,
+				       u32 hw_plane_id,	u32 plane_type,
+				       u32 crtc_mask, const u32 *formats,
+				       u32 num_formats);
+
+dma_addr_t dispc7_plane_state_paddr(const struct drm_plane_state *state);
+dma_addr_t dispc7_plane_state_p_uv_addr(const struct drm_plane_state *state);
+struct drm_plane *tidss_plane_reserve_wb(struct drm_device *dev);
+void tidss_plane_release_wb(struct drm_plane *plane);
+
+#endif
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/tidss_scale_coefs.c b/drivers/gpu/drm/tidss/tidss_scale_coefs.c
--- a/drivers/gpu/drm/tidss/tidss_scale_coefs.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/tidss_scale_coefs.c	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,178 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Jyri Sarha <jsarha@ti.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/device.h>
+#include "tidss_scale_coefs.h"
+
+/*
+ * These are interpolated with a custom python script from DSS5
+ * (drivers/gpu/drm/omapdrm/dss/dispc_coef.c) coefficients.
+ */
+static const struct tidss_scale_coefs coef5_M32 = {
+	.c2 = { 28, 34, 40, 46, 52, 58, 64, 70, 0, 2, 4, 8, 12, 16, 20, 24, },
+	.c1 = { 132, 138, 144, 150, 156, 162, 168, 174, 76, 84, 92, 98, 104, 110, 116, 124, },
+	.c0 = { 192, 192, 192, 190, 188, 186, 184, 182, 180, },
+};
+static const struct tidss_scale_coefs coef5_M26 = {
+	.c2 = { 24, 28, 32, 38, 44, 50, 56, 64, 0, 2, 4, 6, 8, 12, 16, 20, },
+	.c1 = { 132, 138, 144, 152, 160, 166, 172, 178, 72, 80, 88, 94, 100, 108, 116, 124, },
+	.c0 = { 200, 202, 204, 202, 200, 196, 192, 188, 184, },
+};
+static const struct tidss_scale_coefs coef5_M22 = {
+	.c2 = { 16, 20, 24, 30, 36, 42, 48, 56, 0, 0, 0, 2, 4, 8, 12, 14, },
+	.c1 = { 132, 140, 148, 156, 164, 172, 180, 186, 64, 72, 80, 88, 96, 104, 112, 122, },
+	.c0 = { 216, 216, 216, 214, 212, 208, 204, 198, 192, },
+};
+static const struct tidss_scale_coefs coef5_M19 = {
+	.c2 = { 12, 14, 16, 22, 28, 34, 40, 48, 0, 0, 0, 2, 4, 4, 4, 8, },
+	.c1 = { 128, 140, 152, 160, 168, 176, 184, 192, 56, 64, 72, 82, 92, 100, 108, 118, },
+	.c0 = { 232, 232, 232, 226, 220, 218, 216, 208, 200, },
+};
+static const struct tidss_scale_coefs coef5_M16 = {
+	.c2 = { 0, 2, 4, 8, 12, 18, 24, 32, 0, 0, 0, -2, -4, -4, -4, -2, },
+	.c1 = { 124, 138, 152, 164, 176, 186, 196, 206, 40, 48, 56, 68, 80, 90, 100, 112, },
+	.c0 = { 264, 262, 260, 254, 248, 242, 236, 226, 216, },
+};
+static const struct tidss_scale_coefs coef5_M14 = {
+	.c2 = { -8, -6, -4, -2, 0, 6, 12, 18, 0, -2, -4, -6, -8, -8, -8, -8, },
+	.c1 = { 120, 134, 148, 164, 180, 194, 208, 220, 24, 32, 40, 52, 64, 78, 92, 106, },
+	.c0 = { 288, 286, 284, 280, 276, 266, 256, 244, 232, },
+};
+static const struct tidss_scale_coefs coef5_M13 = {
+	.c2 = { -12, -12, -12, -10, -8, -4, 0, 6, 0, -2, -4, -6, -8, -10, -12, -12, },
+	.c1 = { 112, 130, 148, 164, 180, 196, 212, 228, 12, 22, 32, 44, 56, 70, 84, 98, },
+	.c0 = { 312, 308, 304, 298, 292, 282, 272, 258, 244, },
+};
+static const struct tidss_scale_coefs coef5_M12 = {
+	.c2 = { -16, -18, -20, -18, -16, -14, -12, -6, 0, -2, -4, -6, -8, -10, -12, -14, },
+	.c1 = { 104, 124, 144, 164, 184, 202, 220, 238, 0, 10, 20, 30, 40, 56, 72, 88, },
+	.c0 = { 336, 332, 328, 320, 312, 300, 288, 272, 256, },
+};
+static const struct tidss_scale_coefs coef5_M11 = {
+	.c2 = { -20, -22, -24, -24, -24, -24, -24, -20, 0, -2, -4, -6, -8, -10, -12, -16, },
+	.c1 = { 92, 114, 136, 158, 180, 204, 228, 250, -16, -8, 0, 12, 24, 38, 52, 72, },
+	.c0 = { 368, 364, 360, 350, 340, 326, 312, 292, 272, },
+};
+static const struct tidss_scale_coefs coef5_M10 = {
+	.c2 = { -16, -20, -24, -28, -32, -34, -36, -34, 0, 0, 0, -2, -4, -8, -12, -14, },
+	.c1 = { 72, 96, 120, 148, 176, 204, 232, 260, -32, -26, -20, -10, 0, 16, 32, 52, },
+	.c0 = { 400, 398, 396, 384, 372, 354, 336, 312, 288, },
+};
+static const struct tidss_scale_coefs coef5_M9 = {
+	.c2 = { -12, -18, -24, -28, -32, -38, -44, -46, 0, 2, 4, 2, 0, -2, -4, -8, },
+	.c1 = { 40, 68, 96, 128, 160, 196, 232, 268, -48, -46, -44, -36, -28, -14, 0, 20, },
+	.c0 = { 456, 450, 444, 428, 412, 388, 364, 334, 304, },
+};
+static const struct tidss_scale_coefs coef5_M8 = {
+	.c2 = { 0, -4, -8, -16, -24, -32, -40, -48, 0, 2, 4, 6, 8, 6, 4, 2, },
+	.c1 = { 0, 28, 56, 94, 132, 176, 220, 266, -56, -60, -64, -62, -60, -50, -40, -20, },
+	.c0 = { 512, 506, 500, 478, 456, 424, 392, 352, 312, },
+};
+static const struct tidss_scale_coefs coef3_M32 = {
+	.c1 = { 108, 92, 76, 62, 48, 36, 24, 140, 256, 236, 216, 198, 180, 162, 144, 126, },
+	.c0 = { 296, 294, 292, 288, 284, 278, 272, 136, 256, },
+};
+static const struct tidss_scale_coefs coef3_M26 = {
+	.c1 = { 104, 90, 76, 60, 44, 32, 20, 138, 256, 236, 216, 198, 180, 160, 140, 122, },
+	.c0 = { 304, 300, 296, 292, 288, 282, 276, 138, 256, },
+};
+static const struct tidss_scale_coefs coef3_M22 = {
+	.c1 = { 100, 84, 68, 54, 40, 30, 20, 138, 256, 236, 216, 196, 176, 156, 136, 118, },
+	.c0 = { 312, 310, 308, 302, 296, 286, 276, 138, 256, },
+};
+static const struct tidss_scale_coefs coef3_M19 = {
+	.c1 = { 96, 80, 64, 50, 36, 26, 16, 136, 256, 236, 216, 194, 172, 152, 132, 114, },
+	.c0 = { 320, 318, 316, 310, 304, 292, 280, 140, 256, },
+};
+static const struct tidss_scale_coefs coef3_M16 = {
+	.c1 = { 88, 72, 56, 44, 32, 22, 12, 134, 256, 234, 212, 190, 168, 148, 128, 108, },
+	.c0 = { 336, 332, 328, 320, 312, 300, 288, 144, 256, },
+};
+static const struct tidss_scale_coefs coef3_M14 = {
+	.c1 = { 80, 64, 48, 36, 24, 16, 8, 132, 256, 232, 208, 186, 164, 142, 120, 100, },
+	.c0 = { 352, 348, 344, 334, 324, 310, 296, 148, 256, },
+};
+static const struct tidss_scale_coefs coef3_M13 = {
+	.c1 = { 72, 56, 40, 30, 20, 12, 4, 130, 256, 232, 208, 184, 160, 136, 112, 92, },
+	.c0 = { 368, 364, 360, 346, 332, 316, 300, 150, 256, },
+};
+static const struct tidss_scale_coefs coef3_M12 = {
+	.c1 = { 64, 50, 36, 26, 16, 10, 4, 130, 256, 230, 204, 178, 152, 128, 104, 84, },
+	.c0 = { 384, 378, 372, 358, 344, 324, 304, 152, 256, },
+};
+static const struct tidss_scale_coefs coef3_M11 = {
+	.c1 = { 56, 40, 24, 16, 8, 4, 0, 128, 256, 228, 200, 172, 144, 120, 96, 76, },
+	.c0 = { 400, 396, 392, 376, 360, 336, 312, 156, 256, },
+};
+static const struct tidss_scale_coefs coef3_M10 = {
+	.c1 = { 40, 26, 12, 6, 0, -2, -4, 126, 256, 226, 196, 166, 136, 110, 84, 62, },
+	.c0 = { 432, 424, 416, 396, 376, 348, 320, 160, 256, },
+};
+static const struct tidss_scale_coefs coef3_M9 = {
+	.c1 = { 24, 12, 0, -4, -8, -8, -8, 124, 256, 222, 188, 154, 120, 92, 64, 44, },
+	.c0 = { 464, 456, 448, 424, 400, 366, 332, 166, 256, },
+};
+static const struct tidss_scale_coefs coef3_M8 = {
+	.c1 = { 0, -8, -16, -16, -16, -12, -8, 124, 256, 214, 172, 134, 96, 66, 36, 18, },
+	.c0 = { 512, 502, 492, 462, 432, 390, 348, 174, 256, },
+};
+
+const struct tidss_scale_coefs *tidss_get_scale_coefs(struct device *dev,
+						      u32 firinc,
+						      bool five_taps)
+{
+	int i;
+	int inc;
+	static const struct {
+		int Mmin;
+		int Mmax;
+		const struct tidss_scale_coefs *coef3;
+		const struct tidss_scale_coefs *coef5;
+		const char *name;
+	} coefs[] = {
+		{ 27, 32, &coef3_M32, &coef5_M32, "M32" },
+		{ 23, 26, &coef3_M26, &coef5_M26, "M26" },
+		{ 20, 22, &coef3_M22, &coef5_M22, "M22" },
+		{ 17, 19, &coef3_M19, &coef5_M19, "M19" },
+		{ 15, 16, &coef3_M16, &coef5_M16, "M16" },
+		{ 14, 14, &coef3_M14, &coef5_M14, "M14" },
+		{ 13, 13, &coef3_M13, &coef5_M13, "M13" },
+		{ 12, 12, &coef3_M12, &coef5_M12, "M12" },
+		{ 11, 11, &coef3_M11, &coef5_M11, "M11" },
+		{ 10, 10, &coef3_M10, &coef5_M10, "M10" },
+		{  9,  9, &coef3_M9, &coef5_M9, "M9" },
+		{  4,  8, &coef3_M8, &coef5_M8, "M8" },
+		/*
+		 * When upscaling more than two times, blockiness and outlines
+		 * around the image are observed when M8 tables are used. M11,
+		 * M16 and M19 tables are used to prevent this.
+		 */
+		{  3,  3, &coef3_M11, &coef5_M11, "M11" },
+		{  2,  2, &coef3_M16, &coef5_M16, "M16" },
+		{  0,  1, &coef3_M19, &coef5_M19, "M19" },
+	};
+
+	/*
+	 * inc is result of 0x200000 * in_size / out_size. This dividing
+	 * by 0x40000 scales it down to 8 * in_size / out_size. After
+	 * division the actual scaling factor is 8/inc.
+	 */
+	inc = firinc / 0x40000;
+	for (i = 0; i < ARRAY_SIZE(coefs); ++i) {
+		if (inc >= coefs[i].Mmin && inc <= coefs[i].Mmax) {
+			if (five_taps)
+				return coefs[i].coef5;
+			else
+				return coefs[i].coef3;
+		}
+	}
+
+	dev_err(dev, "%s: Coefficients not found for firinc 0x%08x, inc %d\n",
+		__func__, firinc, inc);
+
+	return NULL;
+}
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/tidss_scale_coefs.h b/drivers/gpu/drm/tidss/tidss_scale_coefs.h
--- a/drivers/gpu/drm/tidss/tidss_scale_coefs.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/tidss_scale_coefs.h	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Jyri Sarha <jsarha@ti.com>
+ */
+
+#ifndef __TIDSS_DISPC_COEF_H__
+#define __TIDSS_DISPC_COEF_H__
+
+#include <linux/types.h>
+
+struct tidss_scale_coefs {
+	s16 c2[16];
+	s16 c1[16];
+	u16 c0[9];
+};
+
+const struct tidss_scale_coefs *tidss_get_scale_coefs(struct device *dev,
+						      u32 firinc,
+						      bool five_taps);
+
+#endif
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/tidss_v_connector.c b/drivers/gpu/drm/tidss/tidss_v_connector.c
--- a/drivers/gpu/drm/tidss/tidss_v_connector.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/tidss_v_connector.c	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,144 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Subhajit Paul <subhajit_paul@ti.com>
+ */
+
+/*
+ * Virtual connectors : reuse connector_init logic from
+ * drivers/gpu/drm/bridge/panel.c
+ */
+
+#include <drm/drmP.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include "tidss_v_display.h"
+
+struct v_connector {
+	struct drm_connector base;
+	struct tidss_device *tidss;
+	int id;
+	struct drm_encoder *encoder;
+	struct drm_display_mode mode;
+};
+
+#define to_v_connector(x) container_of(x, struct v_connector, base)
+
+static int v_connector_get_modes(struct drm_connector *connector)
+{
+	struct v_connector *v_connector = to_v_connector(connector);
+	struct drm_device *dev = connector->dev;
+	struct drm_display_mode *mode;
+	const struct drm_display_mode *m = &v_connector->mode;
+
+	dev_dbg(connector->dev->dev, "%s\n", __func__);
+
+	mode = drm_mode_duplicate(dev, m);
+	if (!mode) {
+		dev_err(dev->dev, "failed to add mode %ux%u@%u\n",
+				m->hdisplay, m->vdisplay, m->vrefresh);
+		return 0;
+	}
+
+	mode->type |= (DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED);
+
+	drm_mode_set_name(mode);
+
+	drm_mode_probed_add(connector, mode);
+
+	return 1;
+}
+
+static const struct drm_connector_helper_funcs v_connector_helper_funcs = {
+	.get_modes = v_connector_get_modes,
+};
+
+static void v_connector_destroy(struct drm_connector *connector)
+{
+	struct v_connector *v_connector = to_v_connector(connector);
+
+	dev_dbg(connector->dev->dev, "%s\n", __func__);
+
+	drm_connector_cleanup(connector);
+	kfree(v_connector);
+}
+
+static const struct drm_connector_funcs v_connector_funcs = {
+	.reset = drm_atomic_helper_connector_reset,
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.destroy = v_connector_destroy,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+};
+
+struct drm_connector *v_connector_init(struct tidss_device *tidss, int id,
+		struct drm_encoder *encoder, struct rpmsg_remotedev_display_disp *vp)
+{
+	struct drm_device *dev = tidss->ddev;
+	struct drm_connector *connector = NULL;
+	struct v_connector *v_connector;
+	unsigned int hfp, hbp, hsw, vfp, vbp, vsw;
+
+	v_connector = kzalloc(sizeof(struct v_connector), GFP_KERNEL);
+	if (!v_connector)
+		return NULL;
+
+	hfp = vp->width / 100;
+	if (!hfp)
+		hfp = 1;
+
+	hbp = hfp / 2;
+	if (!hbp)
+		hbp = 1;
+
+	hsw = hbp / 2;
+	if (!hsw)
+		hsw = 1;
+
+	vfp = vp->height / 100;
+	if (!vfp)
+		vfp = 1;
+
+	vbp = vfp / 2;
+	if (!vbp)
+		vbp = 1;
+
+	vsw = vbp / 2;
+	if (!vsw)
+		vsw = 1;
+
+	v_connector->tidss = tidss;
+
+	v_connector->mode.hdisplay = vp->width;
+	v_connector->mode.hsync_start = v_connector->mode.hdisplay + hfp;
+	v_connector->mode.hsync_end = v_connector->mode.hsync_start + hsw;
+	v_connector->mode.htotal = v_connector->mode.hsync_end + hbp;
+
+	v_connector->mode.vdisplay = vp->height;
+	v_connector->mode.vsync_start = v_connector->mode.vdisplay + vfp;
+	v_connector->mode.vsync_end = v_connector->mode.vsync_start + vsw;
+	v_connector->mode.vtotal = v_connector->mode.vsync_end + vbp;
+
+	v_connector->mode.vrefresh = vp->refresh;
+
+	v_connector->mode.clock = (v_connector->mode.vtotal * v_connector->mode.htotal * v_connector->mode.vrefresh) / 1000;
+
+	v_connector->encoder = encoder;
+	v_connector->id = id;
+
+	connector = &v_connector->base;
+
+	drm_connector_init(dev, connector, &v_connector_funcs,
+				DRM_MODE_CONNECTOR_VIRTUAL);
+	drm_connector_helper_add(connector, &v_connector_helper_funcs);
+
+	drm_connector_attach_encoder(connector, encoder);
+
+	return connector;
+}
+
+void v_connector_fini(struct tidss_device *tidss, struct drm_connector *connector)
+{
+	v_connector_destroy(connector);
+}
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/tidss_v_crtc.c b/drivers/gpu/drm/tidss/tidss_v_crtc.c
--- a/drivers/gpu/drm/tidss/tidss_v_crtc.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/tidss_v_crtc.c	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,488 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Subhajit Paul <subhajit_paul@ti.com>
+ */
+
+/*
+ * Virtual crtcs: reuse drivers/gpu/drm/tidss/tidss_crtc.c
+ */
+
+#include <drm/drmP.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_plane.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_gem_cma_helper.h>
+#include "tidss_v_display.h"
+
+static DEFINE_MUTEX(out_buffers_lock);
+static LIST_HEAD(out_buffers_list);
+
+struct v_crtc {
+	struct drm_crtc base;
+	struct tidss_device *tidss;
+	int id;
+	unsigned int remote_id;
+
+	bool disable_in_progress;
+	bool enabled;
+	struct drm_pending_vblank_event *event;
+
+	struct mutex vsync_lock;
+
+	struct completion framedone_completion;
+	struct work_struct fake_vsync;
+
+};
+
+struct v_crtc_buffer_ref {
+	struct drm_framebuffer *fb;
+	atomic_t refcount;
+	struct list_head node;
+};
+
+#define to_v_crtc(x) container_of(x, struct v_crtc, base)
+
+static void v_crtc_finish_page_flip(struct drm_crtc *crtc)
+{
+	struct v_crtc *v_crtc = to_v_crtc(crtc);
+	struct drm_pending_vblank_event *event;
+	unsigned long flags;
+
+	spin_lock_irqsave(&crtc->dev->event_lock, flags);
+
+	event = v_crtc->event;
+	v_crtc->event = NULL;
+
+	if (!event) {
+		spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
+		return;
+	}
+
+	drm_crtc_send_vblank_event(crtc, event);
+
+	spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
+
+	drm_crtc_vblank_put(crtc);
+}
+
+static void v_crtc_vblank_irq(struct drm_crtc *crtc)
+{
+	struct v_crtc *v_crtc = to_v_crtc(crtc);
+
+	drm_crtc_handle_vblank(crtc);
+
+	v_crtc_finish_page_flip(crtc);
+
+	mutex_lock(&v_crtc->vsync_lock);
+	if (v_crtc->disable_in_progress) {
+		v_crtc->disable_in_progress = false;
+		complete(&v_crtc->framedone_completion);
+	}
+	mutex_unlock(&v_crtc->vsync_lock);
+
+}
+
+void v_crtc_commit_done(struct rpmsg_remotedev_display_commit *commit, void *data)
+{
+	struct drm_crtc *crtc = commit->priv;
+
+	v_crtc_vblank_irq(crtc);
+
+	kfree(commit);
+}
+
+void v_crtc_buffer_done(struct rpmsg_remotedev_display_buffer *buffer, void *data)
+{
+	struct v_crtc_buffer_ref *ref = buffer->priv;
+
+	mutex_lock(&out_buffers_lock);
+	if (atomic_dec_and_test(&ref->refcount)) {
+		list_del(&ref->node);
+		kfree(ref);
+		// TODO release the fence here
+	}
+	mutex_unlock(&out_buffers_lock);
+
+	kfree(buffer);
+}
+
+static void fake_vsync_fn(struct work_struct *work)
+{
+	struct v_crtc *v_crtc = container_of(work, struct v_crtc, fake_vsync);
+	struct drm_crtc *crtc = &v_crtc->base;
+
+	v_crtc_vblank_irq(crtc);
+}
+
+
+static u32 v_crtc_update_plane_mask(struct drm_crtc *crtc, struct drm_crtc_state *old, struct drm_crtc_state *new)
+{
+	struct drm_plane *plane;
+	u32 update_plane_mask = 0;
+	u32 cont_u_planes = 0;
+	u32 new_planes = ~old->plane_mask & new->plane_mask;
+	u32 old_planes = ~new->plane_mask & old->plane_mask;
+	u32 cont_planes = old->plane_mask & new->plane_mask;
+
+	drm_for_each_plane_mask(plane, crtc->dev, cont_planes)
+		if (v_plane_update_needed(plane))
+			cont_u_planes |= (1 << drm_plane_index(plane));
+
+
+	update_plane_mask = new_planes | old_planes | cont_u_planes;
+
+	return update_plane_mask;
+}
+
+static struct rpmsg_remotedev_display_buffer *v_crtc_buffer_alloc_for_req(struct drm_crtc *crtc, struct drm_framebuffer *fb,
+		struct drm_plane_state *pstate)
+{
+	struct v_crtc *v_crtc = to_v_crtc(crtc);
+	struct rpmsg_remotedev_display_buffer *buffer;
+	struct v_crtc_buffer_ref *ref, *buffer_ref = NULL;
+	int cnt;
+
+	buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
+	if (!buffer)
+		return NULL;
+
+
+	mutex_lock(&out_buffers_lock);
+	list_for_each_entry(ref, &out_buffers_list, node) {
+		if (ref->fb == fb) {
+			atomic_inc(&ref->refcount);
+			buffer_ref = ref;
+			break;
+		}
+	}
+	if (buffer_ref)
+		goto found;
+
+	buffer_ref = kzalloc(sizeof(*buffer_ref), GFP_KERNEL);
+	if (!buffer_ref) {
+		mutex_unlock(&out_buffers_lock);
+		return NULL;
+	}
+	buffer_ref->fb = fb;
+	atomic_set(&buffer_ref->refcount, 1);
+	// TODO acquire fence here
+	list_add(&buffer_ref->node, &out_buffers_list);
+
+found:
+	mutex_unlock(&out_buffers_lock);
+
+	buffer->width = pstate->src_w >> 16;
+	buffer->height = pstate->src_h >> 16;
+	buffer->format = fb->format->format;
+	buffer->num_planes = min_t(u32, fb->format->num_planes, RPMSG_REMOTEDEV_DISPLAY_MAX_PLANES);
+	WARN_ON(buffer->num_planes != fb->format->num_planes);
+
+	for (cnt = 0; cnt < buffer->num_planes; cnt++) {
+		struct drm_gem_cma_object *gem;
+
+		gem = drm_fb_cma_get_gem_obj(fb, cnt);
+		buffer->planes[cnt] = gem->paddr + fb->offsets[cnt] +
+			(pstate->src_x >> 16) * fb->format->cpp[cnt] +
+			(pstate->src_y >> 16) * fb->pitches[cnt];
+		buffer->pitches[cnt] = fb->pitches[cnt];
+	}
+
+	buffer->rdev = v_crtc->tidss->rdev;
+	buffer->priv = buffer_ref;
+
+	return buffer;
+}
+
+static bool v_crtc_plane_state_to_vid_update_info(struct drm_crtc *crtc, struct drm_plane *plane,
+		struct rpmsg_remotedev_display_pipe_update *vid)
+{
+	struct drm_plane_state *pstate;
+
+	if (WARN_ON(!plane || !plane->state))
+		return false;
+
+	pstate = plane->state;
+
+	vid->pipe_id = v_plane_get_remote_id(plane);
+
+	if (!pstate->fb) {
+		vid->enabled = false;
+		dev_dbg(plane->dev->dev, "%s: disabling plane 0x%x\n", __func__, v_plane_get_remote_id(plane));
+	} else {
+		struct drm_framebuffer *fb = pstate->fb;
+		struct rpmsg_remotedev_display_buffer *buffer;
+
+		buffer = v_crtc_buffer_alloc_for_req(crtc, fb, pstate);
+		if (!buffer) {
+			dev_err(plane->dev->dev, "%s: could not allocate buffer\n", __func__);
+			return false;
+		}
+
+		if (WARN_ON(!fb->format))
+			return false;
+
+		vid->enabled = true;
+		vid->dst_w = pstate->crtc_w;
+		vid->dst_h = pstate->crtc_h;
+		vid->dst_x = pstate->crtc_x;
+		vid->dst_y = pstate->crtc_y;
+
+		vid->buffer = buffer;
+		dev_dbg(plane->dev->dev, "%s: updating plane 0x%x\n", __func__, v_plane_get_remote_id(plane));
+
+	}
+
+	return true;
+}
+
+
+static void v_crtc_flush_to_remote(struct drm_crtc *crtc, u32 planes, char *stage)
+{
+	struct v_crtc *v_crtc = to_v_crtc(crtc);
+	struct drm_plane *plane;
+	int i = 0;
+	struct rpmsg_remotedev_display_commit *commit;
+
+	if (!planes)
+		return;
+
+	commit = kzalloc(sizeof(*commit), GFP_KERNEL);
+	if (!commit)
+		return;
+
+	dev_dbg(crtc->dev->dev, "%s: flushing out vp in %s = 0x%x\n", __func__, stage, v_crtc->remote_id);
+	drm_for_each_plane_mask(plane, crtc->dev, planes) {
+		bool ret;
+
+		if (WARN_ON(i >= RPMSG_REMOTEDEV_DISPLAY_MAX_PIPES))
+			continue;
+		ret = v_crtc_plane_state_to_vid_update_info(crtc, plane, &commit->pipes[i]);
+		if (!ret) {
+			dev_err(crtc->dev->dev, "%s: error creating vid commit req [%d]\n", __func__, i);
+			return;
+		}
+		i++;
+	}
+
+	commit->disp_id = v_crtc->remote_id;
+	commit->num_pipe_updates = i;
+	commit->priv = crtc;
+	commit->rdev = v_crtc->tidss->rdev;
+
+	v_crtc->tidss->rdev->device.display.ops->commit(v_crtc->tidss->rdev, commit);
+}
+
+static int v_crtc_atomic_check(struct drm_crtc *crtc, struct drm_crtc_state *state)
+{
+	dev_dbg(crtc->dev->dev, "%s\n", __func__);
+
+	return 0;
+}
+
+static void v_crtc_atomic_begin(struct drm_crtc *crtc, struct drm_crtc_state *old_crtc_state)
+{
+	dev_dbg(crtc->dev->dev, "%s\n", __func__);
+}
+
+static enum drm_mode_status v_crtc_mode_valid(struct drm_crtc *crtc,
+					   const struct drm_display_mode *mode)
+{
+	dev_dbg(crtc->dev->dev, "%s\n", __func__);
+
+	return MODE_OK;
+}
+
+static void v_crtc_atomic_enable(struct drm_crtc *crtc,
+				     struct drm_crtc_state *old_state)
+{
+	struct v_crtc *v_crtc = to_v_crtc(crtc);
+	u32 upd_planes;
+
+	dev_dbg(crtc->dev->dev, "%s\n", __func__);
+
+	WARN_ON(!crtc->state->event);
+
+	/* Turn vertical blanking interrupt reporting on. */
+	drm_crtc_vblank_on(crtc);
+	WARN_ON(drm_crtc_vblank_get(crtc) != 0);
+	v_crtc->enabled = true;
+
+	spin_lock_irq(&crtc->dev->event_lock);
+	if (crtc->state->event) {
+		v_crtc->event = crtc->state->event;
+		crtc->state->event = NULL;
+	}
+	spin_unlock_irq(&crtc->dev->event_lock);
+
+	upd_planes = v_crtc_update_plane_mask(crtc, old_state, crtc->state);
+	if (upd_planes)
+		v_crtc_flush_to_remote(crtc, upd_planes, "enable");
+	else
+		schedule_work(&v_crtc->fake_vsync);
+}
+
+static void v_crtc_atomic_disable(struct drm_crtc *crtc,
+				      struct drm_crtc_state *old_state)
+{
+	struct v_crtc *v_crtc = to_v_crtc(crtc);
+	u32 upd_planes;
+
+	dev_dbg(crtc->dev->dev, "%s\n", __func__);
+
+	reinit_completion(&v_crtc->framedone_completion);
+
+	WARN_ON(!crtc->state->event);
+
+	WARN_ON(drm_crtc_vblank_get(crtc) != 0);
+
+	mutex_lock(&v_crtc->vsync_lock);
+	v_crtc->disable_in_progress = true;
+	mutex_unlock(&v_crtc->vsync_lock);
+
+	spin_lock_irq(&crtc->dev->event_lock);
+	if (crtc->state->event) {
+		v_crtc->event = crtc->state->event;
+		crtc->state->event = NULL;
+	}
+	spin_unlock_irq(&crtc->dev->event_lock);
+
+	upd_planes = v_crtc_update_plane_mask(crtc, old_state, crtc->state);
+	if (upd_planes)
+		v_crtc_flush_to_remote(crtc, upd_planes, "disable");
+	else
+		schedule_work(&v_crtc->fake_vsync);
+
+	if (!wait_for_completion_timeout(&v_crtc->framedone_completion,
+					 msecs_to_jiffies(500)))
+		dev_err(crtc->dev->dev, "Timeout waiting for disable complete on v_crtc %d",
+			v_crtc->remote_id);
+
+	v_crtc->enabled = false;
+	drm_crtc_vblank_off(crtc);
+}
+
+
+static void v_crtc_atomic_flush(struct drm_crtc *crtc,
+				    struct drm_crtc_state *old_crtc_state)
+{
+	struct v_crtc *v_crtc = to_v_crtc(crtc);
+	u32 upd_planes;
+
+	dev_dbg(crtc->dev->dev, "%s\n", __func__);
+
+	/* Only flush the CRTC if it is currently enabled. */
+	if (!v_crtc->enabled)
+		return;
+
+	/* TODO : check if the old frame is still there */
+
+	// I think we always need the event to signal flip done
+	WARN_ON(!crtc->state->event);
+
+	WARN_ON(drm_crtc_vblank_get(crtc) != 0);
+
+	spin_lock_irq(&crtc->dev->event_lock);
+
+	if (crtc->state->event) {
+		v_crtc->event = crtc->state->event;
+		crtc->state->event = NULL;
+	}
+
+	spin_unlock_irq(&crtc->dev->event_lock);
+
+	upd_planes = v_crtc_update_plane_mask(crtc, old_crtc_state, crtc->state);
+	if (upd_planes)
+		v_crtc_flush_to_remote(crtc, upd_planes, "flush");
+	else
+		schedule_work(&v_crtc->fake_vsync);
+}
+
+
+static const struct drm_crtc_helper_funcs v_crtc_helper_funcs = {
+	.atomic_check = v_crtc_atomic_check,
+	.atomic_begin = v_crtc_atomic_begin,
+	.atomic_flush = v_crtc_atomic_flush,
+	.atomic_enable = v_crtc_atomic_enable,
+	.atomic_disable = v_crtc_atomic_disable,
+
+	.mode_valid = v_crtc_mode_valid,
+};
+
+static void v_crtc_destroy(struct drm_crtc *crtc)
+{
+	struct v_crtc *v_crtc = to_v_crtc(crtc);
+
+	dev_dbg(crtc->dev->dev, "%s\n", __func__);
+
+	drm_crtc_cleanup(crtc);
+	kfree(v_crtc);
+}
+
+static int v_crtc_enable_vblank(struct drm_crtc *crtc)
+{
+	dev_dbg(crtc->dev->dev, "%s\n", __func__);
+
+	return 0;
+}
+
+static void v_crtc_disable_vblank(struct drm_crtc *crtc)
+{
+	dev_dbg(crtc->dev->dev, "%s\n", __func__);
+}
+
+static const struct drm_crtc_funcs v_crtc_funcs = {
+	.reset = drm_atomic_helper_crtc_reset,
+	.destroy = v_crtc_destroy,
+	.set_config = drm_atomic_helper_set_config,
+	.page_flip = drm_atomic_helper_page_flip,
+	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
+	.enable_vblank = v_crtc_enable_vblank,
+	.disable_vblank = v_crtc_disable_vblank,
+};
+
+struct drm_crtc *v_crtc_init(struct tidss_device *tidss, int id,
+		struct drm_plane *plane, struct rpmsg_remotedev_display_disp *vp)
+{
+	struct drm_device *dev = tidss->ddev;
+	struct drm_crtc *crtc = NULL;
+	struct v_crtc *v_crtc;
+	int ret;
+
+	v_crtc = kzalloc(sizeof(*v_crtc), GFP_KERNEL);
+	if (!v_crtc)
+		return NULL;
+
+	v_crtc->tidss = tidss;
+	v_crtc->id = id;
+	v_crtc->remote_id = vp->disp_id;
+	init_completion(&v_crtc->framedone_completion);
+	INIT_WORK(&v_crtc->fake_vsync, fake_vsync_fn);
+
+	mutex_init(&v_crtc->vsync_lock);
+
+	crtc = &v_crtc->base;
+
+	ret = drm_crtc_init_with_planes(dev, crtc, plane, NULL,
+					&v_crtc_funcs, "crtc-0x%x", id);
+	if (ret < 0)
+		goto err;
+
+	drm_crtc_helper_add(crtc, &v_crtc_helper_funcs);
+
+	return crtc;
+
+err:
+	kfree(v_crtc);
+	return NULL;
+}
+
+void v_crtc_fini(struct tidss_device *tidss, struct drm_crtc *crtc)
+{
+	v_crtc_destroy(crtc);
+}
+
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/tidss_v_display.h b/drivers/gpu/drm/tidss/tidss_v_display.h
--- a/drivers/gpu/drm/tidss/tidss_v_display.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/tidss_v_display.h	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Subhajit Paul <subhajit_paul@ti.com>
+ */
+
+#ifndef __TIDSS_V_DISPLAY_H__
+#define __TIDSS_V_DISPLAY_H__
+
+#include "tidss_drv.h"
+
+struct drm_encoder *v_encoder_init(struct tidss_device *tidss, int id,
+		int crtc_mask, struct rpmsg_remotedev_display_disp *vp);
+struct drm_connector *v_connector_init(struct tidss_device *tidss, int id,
+		struct drm_encoder *encoder, struct rpmsg_remotedev_display_disp *vp);
+struct drm_plane *v_plane_init(struct tidss_device *tidss, int id,
+		int crtc_bitmask, enum drm_plane_type type, struct rpmsg_remotedev_display_disp *vp, int index);
+struct drm_crtc *v_crtc_init(struct tidss_device *tidss, int id,
+		struct drm_plane *plane, struct rpmsg_remotedev_display_disp *vp);
+
+void v_crtc_fini(struct tidss_device *tidss, struct drm_crtc *crtc);
+void v_plane_fini(struct tidss_device *tidss, struct drm_plane *plane);
+void v_connector_fini(struct tidss_device *tidss, struct drm_connector *connector);
+void v_encoder_fini(struct tidss_device *tidss, struct drm_encoder *encoder);
+
+unsigned int v_plane_get_remote_id(struct drm_plane *plane);
+bool v_plane_update_needed(struct drm_plane *plane);
+
+void v_crtc_commit_done(struct rpmsg_remotedev_display_commit *commit, void *data);
+void v_crtc_buffer_done(struct rpmsg_remotedev_display_buffer *buffer, void *data);
+#endif
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/tidss_v_encoder.c b/drivers/gpu/drm/tidss/tidss_v_encoder.c
--- a/drivers/gpu/drm/tidss/tidss_v_encoder.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/tidss_v_encoder.c	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,101 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Subhajit Paul <subhajit_paul@ti.com>
+ */
+
+/*
+ * Virtual encoders : reuse drivers/gpu/drm/tidss/tidss_encoder.c
+ */
+
+#include <drm/drmP.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include "tidss_v_display.h"
+
+struct v_encoder {
+	struct drm_encoder base;
+	struct tidss_device *tidss;
+	int id;
+	int possible_crtcs;
+};
+
+#define to_v_encoder(x) container_of(x, struct v_encoder, base)
+
+static void v_encoder_disable(struct drm_encoder *encoder)
+{
+	dev_dbg(encoder->dev->dev, "%s\n", __func__);
+}
+
+static void v_encoder_enable(struct drm_encoder *encoder)
+{
+	dev_dbg(encoder->dev->dev, "%s\n", __func__);
+}
+
+static int v_encoder_atomic_check(struct drm_encoder *encoder,
+				      struct drm_crtc_state *crtc_state,
+				      struct drm_connector_state *conn_state)
+{
+	dev_dbg(encoder->dev->dev, "%s\n", __func__);
+
+	return 0;
+}
+
+static void v_encoder_atomic_mode_set(struct drm_encoder *encoder,
+				   struct drm_crtc_state *crtc_state,
+				   struct drm_connector_state *conn_state)
+{
+	dev_dbg(encoder->dev->dev, "%s\n", __func__);
+}
+
+static const struct drm_encoder_helper_funcs v_encoder_helper_funcs = {
+	.atomic_mode_set = v_encoder_atomic_mode_set,
+	.disable = v_encoder_disable,
+	.enable = v_encoder_enable,
+	.atomic_check = v_encoder_atomic_check,
+};
+
+static void v_encoder_destroy(struct drm_encoder *encoder)
+{
+	struct v_encoder *v_encoder = to_v_encoder(encoder);
+
+	dev_dbg(encoder->dev->dev, "%s\n", __func__);
+
+	drm_encoder_cleanup(encoder);
+	kfree(v_encoder);
+}
+
+static const struct drm_encoder_funcs v_encoder_funcs = {
+	.destroy = v_encoder_destroy,
+};
+
+struct drm_encoder *v_encoder_init(struct tidss_device *tidss, int id, int crtc_mask,
+		struct rpmsg_remotedev_display_disp *vp)
+{
+	struct drm_device *dev = tidss->ddev;
+	struct drm_encoder *encoder = NULL;
+	struct v_encoder *v_encoder;
+
+	v_encoder = kzalloc(sizeof(*v_encoder), GFP_KERNEL);
+	if (!v_encoder)
+		return NULL;
+
+	v_encoder->tidss = tidss;
+	v_encoder->id = id;
+	v_encoder->possible_crtcs = crtc_mask;
+
+	encoder = &v_encoder->base;
+	encoder->possible_crtcs = crtc_mask;
+
+	drm_encoder_init(dev, encoder, &v_encoder_funcs,
+			 DRM_MODE_ENCODER_VIRTUAL, "encoder-0x%x", id);
+	drm_encoder_helper_add(encoder, &v_encoder_helper_funcs);
+
+	return encoder;
+}
+
+void v_encoder_fini(struct tidss_device *tidss, struct drm_encoder *encoder)
+{
+	v_encoder_destroy(encoder);
+}
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/tidss_v_plane.c b/drivers/gpu/drm/tidss/tidss_v_plane.c
--- a/drivers/gpu/drm/tidss/tidss_v_plane.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/tidss_v_plane.c	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,276 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Subhajit Paul <subhajit_paul@ti.com>
+ */
+
+/*
+ * Virtual Planes: reuse drivers/gpu/drm/tidss/tidss_plane.c
+ */
+#include <drm/drmP.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include "tidss_v_display.h"
+
+#define V_PLANE_MAX_FORMATS (32)
+
+struct v_plane_state {
+	struct drm_plane_state state;
+	bool need_update;
+};
+
+struct v_plane {
+	struct drm_plane base;
+	struct tidss_device *tidss;
+	unsigned int remote_id;
+	int possible_crtcs;
+	int id;
+
+	int nformats;
+	u32 formats[V_PLANE_MAX_FORMATS];
+
+	bool can_scale;
+	bool can_mod_win;
+	int fx;
+	int fy;
+	int fw;
+	int fh;
+};
+
+#define to_v_plane(x) container_of(x, struct v_plane, base)
+#define to_v_plane_state(x) container_of(x, struct v_plane_state, state)
+
+unsigned int v_plane_get_remote_id(struct drm_plane *plane)
+{
+	struct v_plane *v_plane = to_v_plane(plane);
+
+	return v_plane->remote_id;
+}
+
+bool v_plane_update_needed(struct drm_plane *plane)
+{
+	struct v_plane_state *v_state = to_v_plane_state(plane->state);
+
+	return v_state->need_update;
+}
+
+static bool v_plane_state_changed(struct drm_plane_state *old, struct drm_plane_state *new)
+{
+	if (old->crtc != new->crtc)
+		return true;
+
+	if (old->fb != new->fb)
+		return true;
+
+	if (old->crtc_x != new->crtc_x)
+		return true;
+
+	if (old->crtc_y != new->crtc_y)
+		return true;
+
+	if (old->crtc_w != new->crtc_w)
+		return true;
+
+	if (old->crtc_h != new->crtc_h)
+		return true;
+
+	if (old->src_x != new->src_x)
+		return true;
+
+	if (old->src_y != new->src_y)
+		return true;
+
+	if (old->src_w != new->src_w)
+		return true;
+
+	if (old->src_h != new->src_h)
+		return true;
+
+	return false;
+}
+
+static int v_plane_atomic_check(struct drm_plane *plane,
+				    struct drm_plane_state *state)
+{
+	struct v_plane *v_plane = to_v_plane(plane);
+	struct drm_crtc_state *crtc_state;
+	int ret;
+
+	dev_dbg(plane->dev->dev, "%s\n", __func__);
+
+	if (!state->crtc) {
+		/*
+		 * The visible field is not reset by the DRM core but only
+		 * updated by drm_plane_helper_check_state(), set it manually.
+		 */
+		state->visible = false;
+		return 0;
+	}
+
+	crtc_state = drm_atomic_get_crtc_state(state->state, state->crtc);
+	if (IS_ERR(crtc_state))
+		return PTR_ERR(crtc_state);
+
+	ret = drm_atomic_helper_check_plane_state(state, crtc_state,
+						  0,
+						  INT_MAX,
+						  true, true);
+	if (ret < 0)
+		return ret;
+
+	if (v_plane->can_scale == false && (
+				((state->src_w >> 16) != state->crtc_w) ||
+				((state->src_h >> 16) != state->crtc_h)))
+		return -EINVAL;
+
+	if (v_plane->can_mod_win == false && (
+				state->crtc_x != v_plane->fx ||
+				state->crtc_y != v_plane->fy ||
+				state->crtc_w != v_plane->fw ||
+				state->crtc_h != v_plane->fh))
+		return -EINVAL;
+
+	return 0;
+}
+
+static void v_plane_atomic_update(struct drm_plane *plane,
+				      struct drm_plane_state *old_state)
+{
+	struct drm_plane_state *plane_state = plane->state;
+	struct v_plane_state *v_state = to_v_plane_state(plane_state);
+
+	dev_dbg(plane->dev->dev, "%s\n", __func__);
+
+	v_state->need_update = v_plane_state_changed(old_state, plane_state);
+}
+
+static void v_plane_atomic_disable(struct drm_plane *plane, struct drm_plane_state *old_state)
+{
+	struct drm_plane_state *plane_state = plane->state;
+	struct v_plane_state *v_state = to_v_plane_state(plane_state);
+
+	dev_dbg(plane->dev->dev, "%s\n", __func__);
+
+	v_state->need_update = v_plane_state_changed(old_state, plane_state);
+}
+
+static const struct drm_plane_helper_funcs v_plane_helper_funcs = {
+	.atomic_check = v_plane_atomic_check,
+	.atomic_update = v_plane_atomic_update,
+	.atomic_disable = v_plane_atomic_disable,
+};
+
+static void v_plane_destroy(struct drm_plane *plane)
+{
+	struct v_plane *v_plane = to_v_plane(plane);
+
+	dev_dbg(plane->dev->dev, "%s\n", __func__);
+
+	drm_plane_cleanup(plane);
+	kfree(v_plane);
+}
+
+static struct drm_plane_state *
+v_plane_atomic_duplicate_state(struct drm_plane *plane)
+{
+	struct v_plane_state *v_state;
+	struct drm_plane_state *state;
+
+	if (WARN_ON(!plane->state))
+		return NULL;
+
+	v_state = kmalloc(sizeof(*v_state), GFP_KERNEL);
+	if (WARN_ON(!v_state))
+		return NULL;
+
+	state = &v_state->state;
+	__drm_atomic_helper_plane_duplicate_state(plane, &v_state->state);
+
+	v_state->need_update = false;
+
+	return state;
+}
+
+static void v_plane_atomic_destroy_state(struct drm_plane *plane,
+					   struct drm_plane_state *state)
+{
+	struct v_plane_state *v_state = to_v_plane_state(state);
+
+	if (WARN_ON(!state))
+		return;
+
+	__drm_atomic_helper_plane_destroy_state(state);
+
+	kfree(v_state);
+}
+
+static const struct drm_plane_funcs v_plane_funcs = {
+	.update_plane = drm_atomic_helper_update_plane,
+	.disable_plane = drm_atomic_helper_disable_plane,
+	.reset = drm_atomic_helper_plane_reset,
+	.destroy = v_plane_destroy,
+	.atomic_duplicate_state = v_plane_atomic_duplicate_state,
+	.atomic_destroy_state = v_plane_atomic_destroy_state,
+};
+
+struct drm_plane *v_plane_init(struct tidss_device *tidss,
+		int id, int crtc_bitmask,
+		enum drm_plane_type type,
+		struct rpmsg_remotedev_display_disp *vp, int index)
+{
+	struct drm_device *dev = tidss->ddev;
+	struct drm_plane *plane;
+	struct v_plane *v_plane;
+	int ret;
+	int fmt;
+	struct rpmsg_remotedev_display_pipe *vid = &vp->pipes[index];
+
+	if (vid->num_formats > V_PLANE_MAX_FORMATS)
+		return NULL;
+
+	v_plane = kzalloc(sizeof(*v_plane), GFP_KERNEL);
+	if (!v_plane)
+		return NULL;
+
+	v_plane->tidss = tidss;
+	v_plane->id = id;
+	v_plane->remote_id = vid->pipe_id;
+	v_plane->can_scale = vid->can_scale;
+	v_plane->can_mod_win = vid->can_mod_win;
+	if (!v_plane->can_mod_win) {
+		v_plane->fx = vid->fixed_win_x;
+		v_plane->fy = vid->fixed_win_y;
+		v_plane->fw = vid->fixed_win_w;
+		v_plane->fh = vid->fixed_win_h;
+	}
+	v_plane->possible_crtcs = crtc_bitmask;
+	v_plane->nformats = vid->num_formats;
+
+	for (fmt = 0; fmt < vid->num_formats; fmt++)
+		v_plane->formats[fmt] = vid->formats[fmt];
+
+	plane = &v_plane->base;
+
+	ret = drm_universal_plane_init(dev, plane, crtc_bitmask,
+				       &v_plane_funcs, v_plane->formats,
+				       v_plane->nformats, NULL, type, "plane-0x%x", id);
+	if (ret < 0)
+		goto error;
+
+	drm_plane_helper_add(plane, &v_plane_helper_funcs);
+
+	drm_plane_create_zpos_immutable_property(plane, vid->initial_zorder);
+
+	return plane;
+
+error:
+	kfree(v_plane);
+	return NULL;
+}
+
+void v_plane_fini(struct tidss_device *tidss, struct drm_plane *plane)
+{
+	v_plane_destroy(plane);
+}
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/tidss_wb.c b/drivers/gpu/drm/tidss/tidss_wb.c
--- a/drivers/gpu/drm/tidss/tidss_wb.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/tidss_wb.c	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,166 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 Texas Instruments Incorporated -  http://www.ti.com/
+ * Author: Benoit Parrot <bparrot@ti.com>
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+
+#include "tidss_wb.h"
+
+unsigned int tidss_wbdebug;
+module_param(tidss_wbdebug, uint, 0644);
+MODULE_PARM_DESC(wbdebug, "activates debug info");
+
+const struct wb_fmt tidss_wb_formats[] = {
+	{
+		.fourcc		= V4L2_PIX_FMT_NV12,
+		.coplanar	= 0,
+		.depth		= {8, 4},
+	},
+	{
+		.fourcc		= V4L2_PIX_FMT_NV12M,
+		.coplanar	= 1,
+		.depth		= {8, 4},
+	},
+	{
+		.fourcc		= V4L2_PIX_FMT_YUYV,
+		.coplanar	= 0,
+		.depth		= {16, 0},
+	},
+	{
+		.fourcc		= V4L2_PIX_FMT_UYVY,
+		.coplanar	= 0,
+		.depth		= {16, 0},
+	},
+	{
+		/* "XR24", DRM_FORMAT_XRGB8888 */
+		.fourcc		= V4L2_PIX_FMT_XBGR32,
+		.coplanar	= 0,
+		.depth		= {32, 0},
+	},
+};
+
+const unsigned int tidss_num_wb_formats = ARRAY_SIZE(tidss_wb_formats);
+
+/* find our format description corresponding to the passed v4l2_format */
+const struct wb_fmt *__tidss_wb_find_format(u32 fourcc)
+{
+	const struct wb_fmt *fmt;
+	unsigned int k;
+
+	for (k = 0; k < tidss_num_wb_formats; k++) {
+		fmt = &tidss_wb_formats[k];
+		if (fmt->fourcc == fourcc)
+			return fmt;
+	}
+
+	return NULL;
+}
+
+const struct wb_fmt *tidss_wb_find_format(struct v4l2_format *f)
+{
+	return __tidss_wb_find_format(f->fmt.pix_mp.pixelformat);
+}
+
+int tidss_wb_fourcc_v4l2_to_drm(u32 fourcc)
+{
+	switch (fourcc) {
+	case V4L2_PIX_FMT_NV12:
+	case V4L2_PIX_FMT_NV12M:
+		return DRM_FORMAT_NV12;
+	case V4L2_PIX_FMT_YUYV:
+		return DRM_FORMAT_YUYV;
+	case V4L2_PIX_FMT_UYVY:
+		return DRM_FORMAT_UYVY;
+	case V4L2_PIX_FMT_XBGR32:
+		return DRM_FORMAT_XRGB8888;
+	default:
+		WARN(1, "WB: unsupported fourcc\n");
+		return 0;
+	}
+}
+
+void tidss_wb_irq(struct wb_dev *wdev, u64 irqstatus)
+{
+	const u64 mask = DSS_IRQ_DEVICE_WB_MASK;
+
+	if (!wdev)
+		return;
+
+	irqstatus &= mask;
+	if (!irqstatus)
+		return;
+
+	if (!atomic_read(&wdev->irq_enabled))
+		return;
+
+	switch (wdev->mode) {
+	case TIDSS_WB_NOT_CONFIGURED:
+		break;
+	case TIDSS_WB_MEM2MEM_OVL:
+		tidss_wbm2m_irq(wdev->m2m, irqstatus);
+		break;
+	case TIDSS_WB_MEM2MEM_MGR:
+		/* To be added */
+		break;
+	case TIDSS_WB_CAPTURE_MGR:
+		/* To be added */
+		break;
+	default:
+		WARN_ONCE(1, "WB: unknown WB mode: 0x%x\n", wdev->mode);
+		break;
+	}
+}
+
+/*
+ * The initial setup of this device instance. Note that the initial state of
+ * the driver should be complete. So the initial format, standard, timings
+ * and video input should all be initialized to some reasonable value.
+ */
+int tidss_wb_init(struct drm_device *drmdev)
+{
+	struct tidss_device *tidss = drmdev->dev_private;
+	struct wb_dev *wdev;
+	int ret = 0;
+
+	/* Allocate a new instance */
+	wdev = devm_kzalloc(drmdev->dev, sizeof(*wdev), GFP_KERNEL);
+	if (!wdev)
+		return -ENOMEM;
+
+	wdev->drm_dev = drmdev;
+
+	/* set pseudo v4l2 device name so we can use v4l2_printk */
+	strlcpy(wdev->v4l2_dev.name, WB_MODULE_NAME,
+		sizeof(wdev->v4l2_dev.name));
+
+	tidss->wdev = wdev;
+
+	mutex_init(&wdev->lock);
+
+	atomic_set(&wdev->irq_enabled, 0);
+
+	wdev->mode = TIDSS_WB_NOT_CONFIGURED;
+
+	ret = tidss_wbm2m_init(wdev);
+	if (ret) {
+		log_err(wdev, "Failed to initialize wb m2m\n");
+		return ret;
+	}
+
+	log_dbg(wdev, "WB loaded\n");
+	return 0;
+}
+
+void tidss_wb_cleanup(struct drm_device *drmdev)
+{
+	struct tidss_device *tidss = drmdev->dev_private;
+	struct wb_dev *wdev = tidss->wdev;
+
+	log_dbg(wdev, "Cleanup WB\n");
+
+	tidss_wbm2m_cleanup(wdev);
+}
+
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/tidss_wb.h b/drivers/gpu/drm/tidss/tidss_wb.h
--- a/drivers/gpu/drm/tidss/tidss_wb.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/tidss_wb.h	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,190 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2019 Texas Instruments Incorporated -  http://www.ti.com/
+ * Author: Benoit Parrot <bparrot@ti.com>
+ */
+
+#ifndef __TIDSS_WB_H__
+#define __TIDSS_WB_H__
+
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/mutex.h>
+#include <linux/wait.h>
+#include <linux/hrtimer.h>
+#include <drm/drmP.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fourcc.h>
+#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_plane_helper.h>
+
+#include <linux/videodev2.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-dev.h>
+#include <media/v4l2-ioctl.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-event.h>
+#include <media/videobuf2-v4l2.h>
+#include <media/videobuf2-dma-contig.h>
+
+#include "tidss_dispc.h"
+#include "tidss_dispc7.h"
+#include "tidss_drv.h"
+#include "tidss_plane.h"
+
+#define WB_MODULE_NAME "tidsswb"
+#define WBM2M_MODULE_NAME "tidss-m2m"
+
+extern unsigned int tidss_wbdebug;
+
+#define log_dbg(dev, fmt, arg...)	\
+		v4l2_dbg(1, tidss_wbdebug, &dev->v4l2_dev, "%s: " fmt, \
+			 __func__, ## arg)
+#define log_err(dev, fmt, arg...)	\
+		v4l2_err(&dev->v4l2_dev, fmt, ## arg)
+#define log_info(dev, fmt, arg...)	\
+		v4l2_info(&dev->v4l2_dev, fmt, ## arg)
+
+/* minimum and maximum frame sizes */
+#define MIN_W		2
+#define MIN_H		1
+#define MAX_W		4096
+#define MAX_H		4096
+
+/* required alignments */
+#define S_ALIGN		0	/* multiple of 1 */
+#define H_ALIGN		0	/* multiple of 2 */
+
+/* used as plane indices */
+#define MAX_PLANES	2
+#define LUMA_PLANE	0
+#define CHROMA_PLANE	1
+
+enum tidss_wb_mode {
+	TIDSS_WB_NOT_CONFIGURED = 0,
+	/* mem2mem from single ovl to wb */
+	TIDSS_WB_MEM2MEM_OVL = 1,
+	/* mem2mem from N overlays via single mgr to wb */
+	TIDSS_WB_MEM2MEM_MGR = 2,
+	/* capture from single mgr to wb */
+	TIDSS_WB_CAPTURE_MGR = 3
+};
+
+enum tidss_wb_state {
+	WB_STATE_NONE = 0,
+	WB_STATE_FIRST_FRAME,
+	WB_STATE_CAPTURING,
+	WB_STATE_STOPPING,
+	WB_STATE_STOPPED,
+};
+
+/* driver info for each of the supported video formats */
+struct wb_fmt {
+	u32	fourcc;			/* standard format identifier */
+	u8	coplanar;		/* set for unpacked Luma and Chroma */
+	u8	depth[MAX_PLANES];	/* Bits per pixel per plane*/
+};
+
+extern const struct wb_fmt tidss_wb_formats[];
+extern const unsigned int tidss_num_wb_formats;
+
+struct wb_buffer {
+	struct vb2_v4l2_buffer	vb;
+	struct list_head	list;
+};
+
+/*
+ * per-queue, driver-specific private data.
+ * MEM-2-MEM: Source: V4L2_BUF_TYPE_VIDEO_OUTPUT*
+ *            Destination: V4L2_BUF_TYPE_VIDEO_CAPTURE*
+ * CAPTURE:   Destination: V4L2_BUF_TYPE_VIDEO_CAPTURE* only
+ */
+struct wb_q_data {
+	/* format info */
+	struct v4l2_format	format;
+	/* crop/compose rectangle */
+	struct v4l2_rect	c_rect;
+	/* format info */
+	const struct wb_fmt	*fmt;
+};
+
+enum {
+	Q_DATA_SRC = 0,
+	Q_DATA_DST = 1,
+};
+
+/* find our format description corresponding to the passed v4l2_format */
+const struct wb_fmt *tidss_wb_find_format(struct v4l2_format *f);
+const struct wb_fmt *__tidss_wb_find_format(u32 fourcc);
+
+struct wb_dev {
+	struct v4l2_device	v4l2_dev;
+	struct drm_device	*drm_dev;
+
+	atomic_t		irq_enabled;
+
+	/* v4l2_ioctl mutex */
+	struct mutex		lock;
+
+	enum tidss_wb_mode	mode;
+	struct wbm2m_dev	*m2m;
+};
+
+/*
+ * there is one wbm2m_dev structure in the driver.
+ */
+struct wbm2m_dev {
+	struct v4l2_device	v4l2_dev;
+	struct video_device	vfd;
+	struct v4l2_m2m_dev	*m2m_dev;
+	struct wb_dev		*dev;
+	struct drm_plane	*plane;
+
+	/* v4l2 buffers lock */
+	spinlock_t		lock;
+
+	struct vb2_alloc_ctx	*alloc_ctx;
+};
+
+/*
+ * There is one wbm2m_ctx structure for each m2m context.
+ */
+struct wbm2m_ctx {
+	struct v4l2_fh		fh;
+	struct wbm2m_dev	*dev;
+	struct v4l2_ctrl_handler hdl;
+
+	/* current frame seq */
+	unsigned int		sequence;
+	/* abort after next irq */
+	unsigned int		aborting;
+
+	/* src & dst queue data */
+	struct wb_q_data	q_data[2];
+
+	/* src & dst state data */
+	struct drm_plane_state s_state;
+	struct drm_framebuffer s_fb;
+	struct drm_gem_cma_object s_cma_gem_obj[2];
+
+	struct drm_plane_state d_state;
+	struct drm_framebuffer d_fb;
+	struct drm_gem_cma_object d_cma_gem_obj[2];
+};
+
+static inline struct wb_buffer *to_wb_buffer(struct vb2_buffer *vb2)
+{
+	return container_of(vb2, struct wb_buffer, vb.vb2_buf);
+}
+
+int tidss_wb_fourcc_v4l2_to_drm(u32 fourcc);
+
+void tidss_wbm2m_irq(struct wbm2m_dev *dev, u64 irqstatus);
+int tidss_wbm2m_init(struct wb_dev *dev);
+void tidss_wbm2m_cleanup(struct wb_dev *dev);
+
+#endif /* __TIDSS_WB_H__ */
diff -Naur --no-dereference a/drivers/gpu/drm/tidss/tidss_wb_m2m.c b/drivers/gpu/drm/tidss/tidss_wb_m2m.c
--- a/drivers/gpu/drm/tidss/tidss_wb_m2m.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/gpu/drm/tidss/tidss_wb_m2m.c	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,1164 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 Texas Instruments Incorporated -  http://www.ti.com/
+ * Benoit Parrot <bparrot@ti.com>
+ *
+ * Based on the virtual v4l2-mem2mem example device
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <media/v4l2-mem2mem.h>
+
+#include "tidss_wb.h"
+
+MODULE_DESCRIPTION("TI TIDSS WB M2M driver");
+MODULE_AUTHOR("Benoit Parrot <bparrot@ti.com>");
+MODULE_LICENSE("GPL v2");
+
+/*
+ * M2M devices get 2 queues.
+ * Return the queue given the type.
+ */
+static struct wb_q_data *get_q_data(struct wbm2m_ctx *ctx,
+				    enum v4l2_buf_type type)
+{
+	switch (type) {
+	case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
+	case V4L2_BUF_TYPE_VIDEO_OUTPUT:
+		return &ctx->q_data[Q_DATA_SRC];
+	case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
+	case V4L2_BUF_TYPE_VIDEO_CAPTURE:
+		return &ctx->q_data[Q_DATA_DST];
+	default:
+		return NULL;
+	}
+	return NULL;
+}
+
+static bool wbm2m_convert(struct wbm2m_dev *dev, struct drm_plane *plane,
+			  const struct drm_plane_state *in_state,
+			  const struct drm_plane_state *out_state)
+{
+	struct tidss_device *tidss = dev->dev->drm_dev->dev_private;
+	struct tidss_plane *tplane = to_tidss_plane(plane);
+	u32 hw_plane = tplane->hw_plane_id;
+	u32 hw_videoport = tidss->dispc_ops->wb_get_reserved_ovr(tidss->dispc);
+	int r;
+
+	/* configure input */
+
+	r = tidss->dispc_ops->plane_setup(tidss->dispc, hw_plane,
+					  in_state, hw_videoport);
+
+	if (r) {
+		log_err(dev, "Failed to setup plane %d\n", hw_plane);
+		tidss->dispc_ops->plane_enable(tidss->dispc, hw_plane,
+					       false);
+		return false;
+	}
+
+	tidss->dispc_ops->plane_enable(tidss->dispc, hw_plane, true);
+
+	/* configure output */
+
+	r = tidss->dispc_ops->wb_setup(tidss->dispc, out_state,
+				       true, hw_plane, hw_videoport);
+	if (r) {
+		log_err(dev, "Failed to setup wb\n");
+		tidss->dispc_ops->plane_enable(tidss->dispc, hw_plane,
+					       false);
+		return false;
+	}
+
+	tidss->dispc_ops->wb_enable(tidss->dispc, true);
+
+	return true;
+}
+
+/*
+ * mem2mem callbacks
+ */
+
+/**
+ * job_ready() - check whether an instance is ready to be scheduled to run
+ */
+static int job_ready(void *priv)
+{
+	struct wbm2m_ctx *ctx = priv;
+
+	/*
+	 * This check is needed as this might be called directly from driver
+	 * When called by m2m framework, this will always satisfy, but when
+	 * called from tidss_wbm2m_irq, this might fail.
+	 * (src stream with zero buffers)
+	 */
+	if (v4l2_m2m_num_src_bufs_ready(ctx->fh.m2m_ctx) <= 0 ||
+	    v4l2_m2m_num_dst_bufs_ready(ctx->fh.m2m_ctx) <= 0)
+		return 0;
+
+	return 1;
+}
+
+static void job_abort(void *priv)
+{
+	struct wbm2m_ctx *ctx = priv;
+
+	/* Will cancel the transaction in the next interrupt handler */
+	ctx->aborting = 1;
+
+	log_dbg(ctx->dev, "Aborting transaction\n");
+	v4l2_m2m_job_finish(ctx->dev->m2m_dev, ctx->fh.m2m_ctx);
+}
+
+static void prepare_plane_state(struct drm_plane_state *state,
+				struct drm_framebuffer *fb,
+				int v4l2_pixelformat,
+				struct drm_gem_cma_object gem_obj[])
+{
+	int drm_fourcc = tidss_wb_fourcc_v4l2_to_drm(v4l2_pixelformat);
+
+	memset(state, 0, sizeof(*state));
+	memset(fb, 0, sizeof(*fb));
+	memset(&gem_obj[0], 0, sizeof(gem_obj[0]));
+	memset(&gem_obj[1], 0, sizeof(gem_obj[1]));
+
+	state->fb = fb;
+	state->fb->format = drm_format_info(drm_fourcc);
+	state->fb->obj[0] = &gem_obj[0].base;
+	state->fb->obj[1] = &gem_obj[1].base;
+}
+
+/* device_run() - prepares and starts the device
+ *
+ * This function is only called when both the source and destination
+ * buffers are in place.
+ */
+static void device_run(void *priv)
+{
+	struct wbm2m_ctx *ctx = priv;
+	struct wbm2m_dev *dev = ctx->dev;
+	struct wb_q_data *d_q_data = &ctx->q_data[Q_DATA_DST];
+	struct wb_q_data *s_q_data = &ctx->q_data[Q_DATA_SRC];
+	struct vb2_v4l2_buffer *s_vb, *d_vb;
+	struct v4l2_pix_format_mplane *spix, *dpix;
+	struct v4l2_rect *srect, *drect;
+	bool ok;
+
+	s_vb = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx);
+	d_vb = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx);
+
+	/* fill source info */
+	srect = &s_q_data->c_rect;
+	spix = &s_q_data->format.fmt.pix_mp;
+	prepare_plane_state(&ctx->s_state, &ctx->s_fb,
+			    spix->pixelformat, ctx->s_cma_gem_obj);
+
+	ctx->s_state.src_w = srect->width << 16;
+	ctx->s_state.src_h = srect->height << 16;
+	ctx->s_state.src_x = srect->left << 16;
+	ctx->s_state.src_y = srect->top << 16;
+	ctx->s_state.crtc_w = srect->width;
+	ctx->s_state.crtc_h = srect->height;
+	ctx->s_state.fb->pitches[0] = spix->plane_fmt[0].bytesperline;
+	ctx->s_state.alpha = DRM_BLEND_ALPHA_OPAQUE;
+	ctx->s_state.color_encoding = DRM_COLOR_YCBCR_BT601;
+	ctx->s_state.color_range = DRM_COLOR_YCBCR_FULL_RANGE;
+
+	ctx->s_cma_gem_obj[0].paddr =
+			vb2_dma_contig_plane_dma_addr(&s_vb->vb2_buf, 0);
+	if (spix->num_planes == 2) {
+		ctx->s_cma_gem_obj[1].paddr =
+			vb2_dma_contig_plane_dma_addr(&s_vb->vb2_buf, 1);
+		ctx->s_state.fb->pitches[1] = spix->plane_fmt[1].bytesperline;
+	} else if (spix->pixelformat == V4L2_PIX_FMT_NV12) {
+		ctx->s_cma_gem_obj[1].paddr = ctx->s_cma_gem_obj[0].paddr +
+			(spix->plane_fmt[0].bytesperline * spix->height);
+		ctx->s_state.fb->pitches[1] = spix->plane_fmt[0].bytesperline;
+	}
+	if (!ctx->s_cma_gem_obj[0].paddr) {
+		log_err(dev,
+			"acquiring source buffer(%d) dma_addr failed\n",
+			(&s_vb->vb2_buf)->index);
+		return;
+	}
+
+	log_dbg(dev, "SRC: ctx %pa buf_index %d %dx%d, pitches: %d, cpp: %d, sw %d\n",
+		&ctx, (&s_vb->vb2_buf)->index,
+		ctx->s_state.crtc_w, ctx->s_state.crtc_h,
+		ctx->s_state.fb->pitches[0], ctx->s_state.fb->format->cpp[0],
+		ctx->s_state.fb->pitches[0] / ctx->s_state.fb->format->cpp[0]);
+	if (spix->num_planes == 2 || spix->pixelformat == V4L2_PIX_FMT_NV12) {
+		log_dbg(dev, "SRC: pitches_uv: %d, cpp_uv: %d, sw_uv %d\n",
+			ctx->s_state.fb->pitches[1],
+			ctx->s_state.fb->format->cpp[1],
+			ctx->s_state.fb->pitches[1] /
+				ctx->s_state.fb->format->cpp[1]);
+	}
+
+	/* fill WB info */
+	drect = &d_q_data->c_rect;
+	dpix = &d_q_data->format.fmt.pix_mp;
+	prepare_plane_state(&ctx->d_state, &ctx->d_fb,
+			    dpix->pixelformat, ctx->d_cma_gem_obj);
+
+	ctx->d_state.src_w = ctx->s_state.crtc_w << 16;
+	ctx->d_state.src_h = ctx->s_state.crtc_h << 16;
+	ctx->d_state.src_x = drect->left << 16;
+	ctx->d_state.src_y = drect->top << 16;
+	ctx->d_state.crtc_w = drect->width;
+	ctx->d_state.crtc_h = drect->height;
+	ctx->d_state.fb->pitches[0] = dpix->plane_fmt[0].bytesperline;
+	ctx->d_state.alpha = DRM_BLEND_ALPHA_OPAQUE;
+	ctx->d_state.color_encoding = DRM_COLOR_YCBCR_BT601;
+	ctx->d_state.color_range = DRM_COLOR_YCBCR_FULL_RANGE;
+
+	ctx->d_cma_gem_obj[0].paddr =
+			vb2_dma_contig_plane_dma_addr(&d_vb->vb2_buf, 0);
+	if (dpix->num_planes == 2) {
+		ctx->d_cma_gem_obj[1].paddr =
+			vb2_dma_contig_plane_dma_addr(&d_vb->vb2_buf, 1);
+		ctx->d_state.fb->pitches[1] = dpix->plane_fmt[1].bytesperline;
+	} else if (dpix->pixelformat == V4L2_PIX_FMT_NV12) {
+		ctx->d_cma_gem_obj[1].paddr = ctx->d_cma_gem_obj[0].paddr +
+			(dpix->plane_fmt[0].bytesperline * dpix->height);
+		ctx->d_state.fb->pitches[1] = dpix->plane_fmt[0].bytesperline;
+	}
+	if (!ctx->d_cma_gem_obj[0].paddr) {
+		log_err(dev,
+			"acquiring destination buffer(%d) dma_addr failed\n",
+			(&d_vb->vb2_buf)->index);
+		return;
+	}
+
+	log_dbg(dev, "DST: ctx %pa buf_index %d %dx%d, pitches: %d, cpp: %d, sw %d\n",
+		&ctx, (&d_vb->vb2_buf)->index,
+		ctx->d_state.crtc_w, ctx->d_state.crtc_h,
+		ctx->d_state.fb->pitches[0], ctx->d_state.fb->format->cpp[0],
+		ctx->d_state.fb->pitches[0] / ctx->d_state.fb->format->cpp[0]);
+	if (dpix->num_planes == 2 || dpix->pixelformat == V4L2_PIX_FMT_NV12) {
+		log_dbg(dev, "DST: pitches_uv: %d, cpp_uv: %d, sw_uv %d\n",
+			ctx->d_state.fb->pitches[1],
+			ctx->d_state.fb->format->cpp[1],
+			ctx->d_state.fb->pitches[1] /
+				ctx->d_state.fb->format->cpp[1]);
+	}
+
+	ok = wbm2m_convert(dev, ctx->dev->plane,
+			   (const struct drm_plane_state *)&ctx->s_state,
+			   (const struct drm_plane_state *)&ctx->d_state);
+	if (!ok) {
+		log_err(dev,
+			"Conversion setup failed, check source and destination parameters\n"
+			);
+		log_err(dev, "\tSRC: %dx%d, fmt: %4.4s sw %d\n",
+			ctx->s_state.crtc_w, ctx->s_state.crtc_h,
+			(char *)&spix->pixelformat,
+			ctx->s_state.fb->pitches[0] /
+				ctx->s_state.fb->format->cpp[0]);
+		log_err(dev, "\tDST: %dx%d, fmt: %4.4s sw %d\n",
+			ctx->d_state.crtc_w, ctx->d_state.crtc_h,
+			(char *)&dpix->pixelformat,
+			ctx->d_state.fb->pitches[0] /
+				ctx->d_state.fb->format->cpp[0]);
+		return;
+	}
+}
+
+void tidss_wbm2m_irq(struct wbm2m_dev *wbm2m, u64 irqstatus)
+{
+	struct wbm2m_ctx *ctx;
+	struct vb2_v4l2_buffer *s_vb, *d_vb;
+	unsigned long flags;
+
+	if (irqstatus & DSS_IRQ_DEVICE_WBBUFFEROVERFLOW)
+		log_err(wbm2m, "WB: UNDERFLOW\n");
+
+	if (irqstatus & DSS_IRQ_DEVICE_WBUNCOMPLETEERROR)
+		log_err(wbm2m, "WB: UNCOMPLETEERROR\n");
+
+	if (irqstatus & DSS_IRQ_DEVICE_WBSECURITYVIOLATION)
+		log_err(wbm2m, "WB: SECURITYVIOLATION\n");
+
+	if (irqstatus & DSS_IRQ_DEVICE_WBSYNC)
+		log_dbg(wbm2m, "WB: SYNC\n");
+
+	/* If DISPC_IRQ_FRAMEDONEWB is not set then we are done */
+	if (!(irqstatus & DSS_IRQ_DEVICE_FRAMEDONEWB))
+		goto handled;
+
+	log_dbg(wbm2m, "WB: FRAMEDONE\n");
+
+	ctx = v4l2_m2m_get_curr_priv(wbm2m->m2m_dev);
+	if (!ctx) {
+		log_err(wbm2m, "instance released before end of transaction\n");
+		goto handled;
+	}
+
+	log_dbg(ctx->dev, "ctx %pa\n", &ctx);
+
+	s_vb = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx);
+	d_vb = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
+	if (!s_vb || !d_vb) {
+		log_err(wbm2m, "source or dest vb pointer is NULL!!");
+		goto handled;
+	}
+
+	d_vb->flags = s_vb->flags;
+
+	d_vb->vb2_buf.timestamp = s_vb->vb2_buf.timestamp;
+	if (s_vb->flags & V4L2_BUF_FLAG_TIMECODE)
+		d_vb->timecode = s_vb->timecode;
+
+	d_vb->sequence = ctx->sequence;
+	s_vb->sequence = ctx->sequence;
+	log_dbg(wbm2m, "ctx %pa sequence %d\n",
+		&ctx, ctx->sequence);
+
+	d_vb->field = V4L2_FIELD_NONE;
+	ctx->sequence++;
+
+	spin_lock_irqsave(&wbm2m->lock, flags);
+
+	v4l2_m2m_buf_done(s_vb, VB2_BUF_STATE_DONE);
+	v4l2_m2m_buf_done(d_vb, VB2_BUF_STATE_DONE);
+
+	spin_unlock_irqrestore(&wbm2m->lock, flags);
+
+	v4l2_m2m_job_finish(wbm2m->m2m_dev, ctx->fh.m2m_ctx);
+handled:
+	return;
+}
+
+/*
+ * video ioctls
+ */
+static int wbm2m_querycap(struct file *file, void *priv,
+			  struct v4l2_capability *cap)
+{
+	struct wbm2m_ctx *ctx = file->private_data;
+
+	strscpy(cap->driver, WBM2M_MODULE_NAME, sizeof(cap->driver));
+	strscpy(cap->card, WBM2M_MODULE_NAME, sizeof(cap->card));
+	snprintf(cap->bus_info, sizeof(cap->bus_info), "platform:%s",
+		 ctx->dev->v4l2_dev.name);
+
+	return 0;
+}
+
+static int wbm2m_enum_fmt(struct file *file, void *priv,
+			  struct v4l2_fmtdesc *f)
+{
+	if (f->index >= tidss_num_wb_formats)
+		return -EINVAL;
+
+	f->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+	f->pixelformat = tidss_wb_formats[f->index].fourcc;
+	return 0;
+}
+
+static int wbm2m_g_fmt(struct file *file, void *priv, struct v4l2_format *f)
+{
+	struct wbm2m_ctx *ctx = file->private_data;
+	struct vb2_queue *vq;
+	struct wb_q_data *q_data;
+	struct v4l2_pix_format_mplane *pix = &f->fmt.pix_mp;
+
+	vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx, f->type);
+	if (!vq)
+		return -EINVAL;
+
+	q_data = get_q_data(ctx, f->type);
+	if (!q_data)
+		return -EINVAL;
+
+	*f = q_data->format;
+
+	if (!V4L2_TYPE_IS_OUTPUT(f->type)) {
+		struct wb_q_data *s_q_data;
+
+		/* get colorspace from the source queue */
+		s_q_data = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE);
+
+		f->fmt.pix_mp.colorspace =
+			s_q_data->format.fmt.pix_mp.colorspace;
+	}
+
+	log_dbg(ctx->dev, "ctx %pa type %d, %dx%d, fmt: %4.4s bpl_y %d",
+		&ctx, f->type, pix->width, pix->height,
+		(char *)&pix->pixelformat,
+		pix->plane_fmt[LUMA_PLANE].bytesperline);
+	if (pix->num_planes == 2)
+		log_dbg(ctx->dev, " bpl_uv %d\n",
+			pix->plane_fmt[CHROMA_PLANE].bytesperline);
+
+	return 0;
+}
+
+static int wbm2m_try_fmt(struct file *file, void *priv, struct v4l2_format *f)
+{
+	struct wbm2m_ctx *ctx = file->private_data;
+	const struct wb_fmt *fmt = tidss_wb_find_format(f);
+	struct v4l2_pix_format_mplane *pix = &f->fmt.pix_mp;
+	struct v4l2_plane_pix_format *plane_fmt;
+	unsigned int w_align;
+	int i, depth, depth_bytes;
+
+	if (!fmt) {
+		log_dbg(ctx->dev, "Fourcc format (0x%08x) invalid.\n",
+			pix->pixelformat);
+		fmt = __tidss_wb_find_format(V4L2_PIX_FMT_YUYV);
+	}
+
+	/* we only allow V4L2_FIELD_NONE */
+	if (pix->field != V4L2_FIELD_NONE)
+		pix->field = V4L2_FIELD_NONE;
+
+	depth = fmt->depth[LUMA_PLANE];
+
+	/*
+	 * The line stride needs to be even is even.
+	 * Special case is with YUV422 interleaved format an even number
+	 * of pixels is required also.
+	 */
+	depth_bytes = depth >> 3;
+
+	w_align = 0;
+	if ((depth_bytes == 3) || (depth_bytes == 1))
+		w_align = 1;
+	else if ((depth_bytes == 2) &&
+		 (fmt->fourcc == V4L2_PIX_FMT_YUYV ||
+		  fmt->fourcc == V4L2_PIX_FMT_UYVY))
+		w_align = 1;
+
+	v4l_bound_align_image(&pix->width, MIN_W, MAX_W, w_align,
+			      &pix->height, MIN_H, MAX_H, H_ALIGN,
+			      S_ALIGN);
+	pix->num_planes = fmt->coplanar ? 2 : 1;
+	pix->pixelformat = fmt->fourcc;
+
+	/* Probably need something better here */
+	if (!pix->colorspace) {
+		if (fmt->fourcc == V4L2_PIX_FMT_RGB24 ||
+		    fmt->fourcc == V4L2_PIX_FMT_BGR24 ||
+		    fmt->fourcc == V4L2_PIX_FMT_RGB32 ||
+		    fmt->fourcc == V4L2_PIX_FMT_BGR32) {
+			pix->colorspace = V4L2_COLORSPACE_SRGB;
+		} else {
+			if (pix->height > 1280)	/* HD */
+				pix->colorspace = V4L2_COLORSPACE_REC709;
+			else			/* SD */
+				pix->colorspace = V4L2_COLORSPACE_SMPTE170M;
+		}
+	}
+
+	memset(pix->reserved, 0, sizeof(pix->reserved));
+	for (i = 0; i < pix->num_planes; i++) {
+		plane_fmt = &pix->plane_fmt[i];
+		depth = fmt->depth[i];
+
+		if (i == LUMA_PLANE)
+			plane_fmt->bytesperline = pix->width * depth / 8;
+		else
+			plane_fmt->bytesperline = pix->width;
+
+		plane_fmt->sizeimage = (pix->height * pix->width *
+					depth) / 8;
+
+		if (fmt->fourcc == V4L2_PIX_FMT_NV12) {
+			/*
+			 * Since we are using a single plane buffer
+			 * we need to adjust the reported sizeimage
+			 * to include the colocated UV part.
+			 */
+			plane_fmt->sizeimage += (pix->height / 2 *
+				plane_fmt->bytesperline);
+		}
+
+		memset(plane_fmt->reserved, 0, sizeof(plane_fmt->reserved));
+	}
+
+	return 0;
+}
+
+static int __wbm2m_s_fmt(struct wbm2m_ctx *ctx, struct v4l2_format *f)
+{
+	struct v4l2_pix_format_mplane *pix = &f->fmt.pix_mp;
+	struct wb_q_data *q_data;
+	struct vb2_queue *vq;
+
+	vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx, f->type);
+	if (!vq)
+		return -EINVAL;
+
+	if (vb2_is_busy(vq)) {
+		log_err(ctx->dev, "queue busy\n");
+		return -EBUSY;
+	}
+
+	q_data = get_q_data(ctx, f->type);
+	if (!q_data)
+		return -EINVAL;
+
+	q_data->fmt = tidss_wb_find_format(f);
+	q_data->format = *f;
+
+	q_data->c_rect.left	= 0;
+	q_data->c_rect.top	= 0;
+	q_data->c_rect.width	= pix->width;
+	q_data->c_rect.height	= pix->height;
+
+	log_dbg(ctx->dev, "ctx %pa type %d, %dx%d, fmt: %4.4s bpl_y %d",
+		&ctx, f->type, pix->width, pix->height,
+		(char *)&pix->pixelformat,
+		pix->plane_fmt[LUMA_PLANE].bytesperline);
+	if (pix->num_planes == 2)
+		log_dbg(ctx->dev, " bpl_uv %d\n",
+			pix->plane_fmt[CHROMA_PLANE].bytesperline);
+
+	return 0;
+}
+
+static int wbm2m_s_fmt(struct file *file, void *priv, struct v4l2_format *f)
+{
+	int ret;
+	struct wbm2m_ctx *ctx = file->private_data;
+
+	ret = wbm2m_try_fmt(file, priv, f);
+	if (ret)
+		return ret;
+
+	ret = __wbm2m_s_fmt(ctx, f);
+	if (ret)
+		return ret;
+
+	ctx->sequence = 0;
+
+	return 0;
+}
+
+static int __wbm2m_try_selection(struct wbm2m_ctx *ctx,
+				 struct v4l2_selection *s)
+{
+	struct wb_q_data *q_data;
+	struct v4l2_pix_format_mplane *pix;
+	unsigned int w_align;
+	int depth_bytes;
+
+	if ((s->type != V4L2_BUF_TYPE_VIDEO_CAPTURE) &&
+	    (s->type != V4L2_BUF_TYPE_VIDEO_OUTPUT))
+		return -EINVAL;
+
+	q_data = get_q_data(ctx, s->type);
+	if (!q_data)
+		return -EINVAL;
+
+	pix = &q_data->format.fmt.pix_mp;
+
+	switch (s->target) {
+	case V4L2_SEL_TGT_COMPOSE:
+		/*
+		 * COMPOSE target is only valid for capture buffer type, return
+		 * error for output buffer type
+		 */
+		if (s->type == V4L2_BUF_TYPE_VIDEO_OUTPUT)
+			return -EINVAL;
+		break;
+	case V4L2_SEL_TGT_CROP:
+		/*
+		 * CROP target is only valid for output buffer type, return
+		 * error for capture buffer type
+		 */
+		if (s->type == V4L2_BUF_TYPE_VIDEO_CAPTURE)
+			return -EINVAL;
+		break;
+	/*
+	 * bound and default crop/compose targets are invalid targets to
+	 * try/set
+	 */
+	default:
+		return -EINVAL;
+	}
+
+	if (s->r.top < 0 || s->r.left < 0) {
+		log_err(ctx->dev, "negative values for top and left\n");
+		s->r.top = 0;
+		s->r.left = 0;
+	}
+
+	depth_bytes = q_data->fmt->depth[LUMA_PLANE] >> 3;
+
+	w_align = 0;
+	if ((depth_bytes == 3) || (depth_bytes == 1))
+		w_align = 1;
+	else if ((depth_bytes == 2) &&
+		 (pix->pixelformat == V4L2_PIX_FMT_YUYV ||
+		  pix->pixelformat == V4L2_PIX_FMT_UYVY))
+		w_align = 1;
+
+	v4l_bound_align_image(&s->r.width, MIN_W, pix->width, w_align,
+			      &s->r.height, MIN_H, pix->height,
+			      H_ALIGN, S_ALIGN);
+
+	/* adjust left/top if cropping rectangle is out of bounds */
+	if (s->r.left + s->r.width > pix->width)
+		s->r.left = pix->width - s->r.width;
+	if (s->r.top + s->r.height > pix->height)
+		s->r.top = pix->height - s->r.height;
+
+	return 0;
+}
+
+static int wbm2m_g_selection(struct file *file, void *fh,
+			     struct v4l2_selection *s)
+{
+	struct wbm2m_ctx *ctx = file->private_data;
+	struct wb_q_data *q_data;
+	struct v4l2_pix_format_mplane *pix;
+	bool use_c_rect = false;
+
+	if ((s->type != V4L2_BUF_TYPE_VIDEO_CAPTURE) &&
+	    (s->type != V4L2_BUF_TYPE_VIDEO_OUTPUT))
+		return -EINVAL;
+
+	q_data = get_q_data(ctx, s->type);
+	if (!q_data)
+		return -EINVAL;
+
+	pix = &q_data->format.fmt.pix_mp;
+
+	switch (s->target) {
+	case V4L2_SEL_TGT_COMPOSE_DEFAULT:
+	case V4L2_SEL_TGT_COMPOSE_BOUNDS:
+		if (s->type == V4L2_BUF_TYPE_VIDEO_OUTPUT)
+			return -EINVAL;
+		break;
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+		if (s->type == V4L2_BUF_TYPE_VIDEO_CAPTURE)
+			return -EINVAL;
+		break;
+	case V4L2_SEL_TGT_COMPOSE:
+		if (s->type == V4L2_BUF_TYPE_VIDEO_OUTPUT)
+			return -EINVAL;
+		use_c_rect = true;
+		break;
+	case V4L2_SEL_TGT_CROP:
+		if (s->type == V4L2_BUF_TYPE_VIDEO_CAPTURE)
+			return -EINVAL;
+		use_c_rect = true;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (use_c_rect) {
+		/*
+		 * for CROP/COMPOSE target type, return c_rect params from the
+		 * respective buffer type
+		 */
+		s->r = q_data->c_rect;
+	} else {
+		/*
+		 * for DEFAULT/BOUNDS target type, return width and height from
+		 * S_FMT of the respective buffer type
+		 */
+		s->r.left = 0;
+		s->r.top = 0;
+		s->r.width = pix->width;
+		s->r.height = pix->height;
+	}
+
+	return 0;
+}
+
+static int wbm2m_s_selection(struct file *file, void *fh,
+			     struct v4l2_selection *s)
+{
+	struct wbm2m_ctx *ctx = file->private_data;
+	struct wb_q_data *q_data;
+	struct v4l2_selection sel = *s;
+	int ret;
+
+	ret = __wbm2m_try_selection(ctx, &sel);
+	if (ret)
+		return ret;
+
+	q_data = get_q_data(ctx, sel.type);
+	if (!q_data)
+		return -EINVAL;
+
+	if ((q_data->c_rect.left == sel.r.left) &&
+	    (q_data->c_rect.top == sel.r.top) &&
+	    (q_data->c_rect.width == sel.r.width) &&
+	    (q_data->c_rect.height == sel.r.height)) {
+		log_dbg(ctx->dev,
+			"type: %d, requested crop/compose values are already set\n",
+			sel.type);
+		return 0;
+	}
+
+	q_data->c_rect = sel.r;
+
+	ctx->sequence = 0;
+
+	return 0;
+}
+
+static const struct v4l2_ioctl_ops wbm2m_ioctl_ops = {
+	.vidioc_querycap		= wbm2m_querycap,
+
+	.vidioc_enum_fmt_vid_cap_mplane	= wbm2m_enum_fmt,
+	.vidioc_g_fmt_vid_cap_mplane	= wbm2m_g_fmt,
+	.vidioc_try_fmt_vid_cap_mplane	= wbm2m_try_fmt,
+	.vidioc_s_fmt_vid_cap_mplane	= wbm2m_s_fmt,
+
+	.vidioc_enum_fmt_vid_out_mplane	= wbm2m_enum_fmt,
+	.vidioc_g_fmt_vid_out_mplane	= wbm2m_g_fmt,
+	.vidioc_try_fmt_vid_out_mplane	= wbm2m_try_fmt,
+	.vidioc_s_fmt_vid_out_mplane	= wbm2m_s_fmt,
+
+	.vidioc_g_selection		= wbm2m_g_selection,
+	.vidioc_s_selection		= wbm2m_s_selection,
+
+	.vidioc_reqbufs			= v4l2_m2m_ioctl_reqbufs,
+	.vidioc_create_bufs		= v4l2_m2m_ioctl_create_bufs,
+	.vidioc_prepare_buf		= v4l2_m2m_ioctl_prepare_buf,
+	.vidioc_querybuf		= v4l2_m2m_ioctl_querybuf,
+	.vidioc_qbuf			= v4l2_m2m_ioctl_qbuf,
+	.vidioc_dqbuf			= v4l2_m2m_ioctl_dqbuf,
+	.vidioc_expbuf			= v4l2_m2m_ioctl_expbuf,
+	.vidioc_streamon		= v4l2_m2m_ioctl_streamon,
+	.vidioc_streamoff		= v4l2_m2m_ioctl_streamoff,
+
+	.vidioc_subscribe_event		= v4l2_ctrl_subscribe_event,
+	.vidioc_unsubscribe_event	= v4l2_event_unsubscribe,
+};
+
+/*
+ * Queue operations
+ */
+static int wbm2m_queue_setup(struct vb2_queue *vq,
+			     unsigned int *nbuffers, unsigned int *nplanes,
+			     unsigned int sizes[], struct device *alloc_devs[])
+{
+	int i;
+	struct wbm2m_ctx *ctx = vb2_get_drv_priv(vq);
+	struct wb_q_data *q_data;
+
+	q_data = get_q_data(ctx, vq->type);
+	if (!q_data)
+		return -EINVAL;
+
+	*nplanes = q_data->format.fmt.pix_mp.num_planes;
+
+	for (i = 0; i < *nplanes; i++)
+		sizes[i] = q_data->format.fmt.pix_mp.plane_fmt[i].sizeimage;
+
+	log_dbg(ctx->dev, "get %d buffer(s) of size %d\n", *nbuffers,
+		sizes[LUMA_PLANE]);
+	if (*nplanes == 2)
+		log_dbg(ctx->dev, " and %d\n", sizes[CHROMA_PLANE]);
+
+	return 0;
+}
+
+static int wbm2m_buf_prepare(struct vb2_buffer *vb)
+{
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct wbm2m_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue);
+	struct wb_q_data *q_data;
+	struct v4l2_pix_format_mplane *mp;
+	int i, num_planes;
+
+	log_dbg(ctx->dev, "type: %d\n", vb->vb2_queue->type);
+
+	q_data = get_q_data(ctx, vb->vb2_queue->type);
+	if (!q_data)
+		return -EINVAL;
+
+	if (vb->vb2_queue->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE)
+		vbuf->field = V4L2_FIELD_NONE;
+
+	num_planes = q_data->format.fmt.pix_mp.num_planes;
+
+	for (i = 0; i < num_planes; i++) {
+		mp = &q_data->format.fmt.pix_mp;
+
+		if (vb->vb2_queue->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
+			if (vb2_get_plane_payload(vb, i) <
+			    mp->plane_fmt[i].sizeimage) {
+				log_dbg(ctx->dev,
+					"the payload is too small for plane plane (%lu < %lu)\n",
+					vb2_get_plane_payload(vb, i),
+					(long)mp->plane_fmt[i].sizeimage);
+				return -EINVAL;
+			}
+		} else {
+			if (vb2_plane_size(vb, i) <
+			    mp->plane_fmt[i].sizeimage) {
+				log_dbg(ctx->dev,
+					"data will not fit into plane (%lu < %lu)\n",
+					vb2_plane_size(vb, i),
+					(long)mp->plane_fmt[i].sizeimage);
+				return -EINVAL;
+			}
+			vb2_set_plane_payload(vb, i,
+					      mp->plane_fmt[i].sizeimage);
+		}
+	}
+
+	return 0;
+}
+
+static void wbm2m_buf_queue(struct vb2_buffer *vb)
+{
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct wbm2m_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue);
+
+	log_dbg(ctx->dev, "queueing buffer: %s index %d\n",
+		V4L2_TYPE_IS_OUTPUT(vb->type) ? "OUTPUT" : "CAPTURE",
+		vb->index);
+
+	v4l2_m2m_buf_queue(ctx->fh.m2m_ctx, vbuf);
+}
+
+static int wbm2m_start_streaming(struct vb2_queue *q, unsigned int count)
+{
+	struct wbm2m_ctx *ctx = vb2_get_drv_priv(q);
+	struct tidss_device *tidss = ctx->dev->dev->drm_dev->dev_private;
+
+	log_dbg(ctx->dev, "ctx %pa queue: %s\n", &ctx,
+		V4L2_TYPE_IS_OUTPUT(q->type) ? "OUTPUT" : "CAPTURE");
+
+	ctx->sequence = 0;
+
+	tidss->dispc_ops->runtime_get(tidss->dispc);
+	atomic_inc(&ctx->dev->dev->irq_enabled);
+
+	return 0;
+}
+
+static void wbm2m_stop_streaming(struct vb2_queue *q)
+{
+	struct wbm2m_ctx *ctx = vb2_get_drv_priv(q);
+	struct tidss_device *tidss = ctx->dev->dev->drm_dev->dev_private;
+	struct vb2_v4l2_buffer *vb;
+	unsigned long flags;
+
+	log_dbg(ctx->dev, "ctx %pa queue: %s\n", &ctx,
+		V4L2_TYPE_IS_OUTPUT(q->type) ? "OUTPUT" : "CAPTURE");
+
+	atomic_dec(&ctx->dev->dev->irq_enabled);
+
+	for (;;) {
+		if (V4L2_TYPE_IS_OUTPUT(q->type))
+			vb = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx);
+		else
+			vb = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
+		if (!vb)
+			break;
+		log_dbg(ctx->dev, "returning from queue: buffer index %d\n",
+			vb->vb2_buf.index);
+		spin_lock_irqsave(&ctx->dev->lock, flags);
+		v4l2_m2m_buf_done(vb, VB2_BUF_STATE_ERROR);
+		spin_unlock_irqrestore(&ctx->dev->lock, flags);
+	}
+
+	/*
+	 * Cleanup the in-transit vb2 buffers that have been
+	 * removed from their respective queue already but for
+	 * which processing has not been completed yet.
+	 */
+	if (V4L2_TYPE_IS_OUTPUT(q->type)) {
+		struct tidss_plane *tplane = to_tidss_plane(ctx->dev->plane);
+
+		tidss->dispc_ops->plane_enable(tidss->dispc,
+					       tplane->hw_plane_id, false);
+		tidss->dispc_ops->wb_enable(tidss->dispc, false);
+	}
+
+	tidss->dispc_ops->runtime_put(tidss->dispc);
+}
+
+static struct vb2_ops wbm2m_qops = {
+	.queue_setup	 = wbm2m_queue_setup,
+	.buf_prepare	 = wbm2m_buf_prepare,
+	.buf_queue	 = wbm2m_buf_queue,
+	.wait_prepare	 = vb2_ops_wait_prepare,
+	.wait_finish	 = vb2_ops_wait_finish,
+	.start_streaming = wbm2m_start_streaming,
+	.stop_streaming  = wbm2m_stop_streaming,
+};
+
+static int queue_init(void *priv, struct vb2_queue *src_vq,
+		      struct vb2_queue *dst_vq)
+{
+	struct wbm2m_ctx *ctx = priv;
+	struct wbm2m_dev *dev = ctx->dev;
+	int ret;
+
+	memset(src_vq, 0, sizeof(*src_vq));
+	src_vq->type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
+	src_vq->io_modes = VB2_MMAP | VB2_DMABUF;
+	src_vq->drv_priv = ctx;
+	src_vq->buf_struct_size = sizeof(struct v4l2_m2m_buffer);
+	src_vq->ops = &wbm2m_qops;
+	src_vq->mem_ops = &vb2_dma_contig_memops;
+	src_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY;
+	src_vq->lock = &dev->dev->lock;
+	src_vq->min_buffers_needed = 1;
+	src_vq->dev = dev->v4l2_dev.dev;
+
+	ret = vb2_queue_init(src_vq);
+	if (ret)
+		return ret;
+
+	memset(dst_vq, 0, sizeof(*dst_vq));
+	dst_vq->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+	dst_vq->io_modes = VB2_MMAP | VB2_DMABUF;
+	dst_vq->drv_priv = ctx;
+	dst_vq->buf_struct_size = sizeof(struct v4l2_m2m_buffer);
+	dst_vq->ops = &wbm2m_qops;
+	dst_vq->mem_ops = &vb2_dma_contig_memops;
+	dst_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY;
+	dst_vq->lock = &dev->dev->lock;
+	dst_vq->min_buffers_needed = 1;
+	dst_vq->dev = dev->v4l2_dev.dev;
+
+	return vb2_queue_init(dst_vq);
+}
+
+/*
+ * File operations
+ */
+static int wbm2m_open(struct file *file)
+{
+	struct wbm2m_dev *dev = video_drvdata(file);
+	struct wb_q_data *s_q_data;
+	struct wbm2m_ctx *ctx;
+	struct v4l2_pix_format_mplane *pix;
+	int ret;
+
+	log_dbg(dev, "enter\n");
+
+	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return -ENOMEM;
+
+	ctx->dev = dev;
+
+	if (mutex_lock_interruptible(&dev->dev->lock)) {
+		ret = -ERESTARTSYS;
+		goto free_ctx;
+	}
+
+	if ((dev->dev->mode != TIDSS_WB_NOT_CONFIGURED) &&
+	    (dev->dev->mode != TIDSS_WB_MEM2MEM_OVL)) {
+		/* WB is already open for other modes */
+		ret = -EBUSY;
+		goto unlock;
+	}
+
+	v4l2_fh_init(&ctx->fh, video_devdata(file));
+	file->private_data = ctx;
+
+	s_q_data = &ctx->q_data[Q_DATA_SRC];
+	s_q_data->fmt = __tidss_wb_find_format(V4L2_PIX_FMT_YUYV);
+	pix = &s_q_data->format.fmt.pix_mp;
+	pix->pixelformat = s_q_data->fmt->fourcc;
+	s_q_data->format.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
+	pix->width = 1920;
+	pix->height = 1080;
+	pix->plane_fmt[LUMA_PLANE].bytesperline = (pix->width *
+			s_q_data->fmt->depth[LUMA_PLANE]) >> 3;
+	pix->plane_fmt[LUMA_PLANE].sizeimage =
+			pix->plane_fmt[LUMA_PLANE].bytesperline *
+			pix->height;
+	pix->num_planes = s_q_data->fmt->coplanar ? 2 : 1;
+	pix->colorspace = V4L2_COLORSPACE_REC709;
+	pix->field = V4L2_FIELD_NONE;
+	s_q_data->c_rect.left = 0;
+	s_q_data->c_rect.top = 0;
+	s_q_data->c_rect.width = pix->width;
+	s_q_data->c_rect.height = pix->height;
+
+	ctx->q_data[Q_DATA_DST] = *s_q_data;
+	ctx->q_data[Q_DATA_DST].format.type =
+			V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+
+	ctx->sequence = 0;
+
+	ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(dev->m2m_dev, ctx, &queue_init);
+
+	if (IS_ERR(ctx->fh.m2m_ctx)) {
+		ret = PTR_ERR(ctx->fh.m2m_ctx);
+		goto exit_fh;
+	}
+
+	v4l2_fh_add(&ctx->fh);
+
+	if (v4l2_fh_is_singular_file(file)) {
+		log_dbg(dev, "first instance created\n");
+
+		drm_modeset_lock_all(dev->dev->drm_dev);
+		dev->plane = tidss_plane_reserve_wb(dev->dev->drm_dev);
+		drm_modeset_unlock_all(dev->dev->drm_dev);
+
+		if (!dev->plane) {
+			log_dbg(dev, "Could not reserve plane!\n");
+			ret = -EBUSY;
+			goto free_fh;
+		}
+
+		dev->dev->mode = TIDSS_WB_MEM2MEM_OVL;
+	}
+
+	log_dbg(dev, "created instance %pa, m2m_ctx: %pa\n",
+		&ctx, &ctx->fh.m2m_ctx);
+
+	mutex_unlock(&dev->dev->lock);
+
+	return 0;
+
+free_fh:
+	v4l2_fh_del(&ctx->fh);
+	v4l2_m2m_ctx_release(ctx->fh.m2m_ctx);
+exit_fh:
+	v4l2_fh_exit(&ctx->fh);
+unlock:
+	mutex_unlock(&dev->dev->lock);
+free_ctx:
+	kfree(ctx);
+	return ret;
+}
+
+static int wbm2m_release(struct file *file)
+{
+	struct wbm2m_dev *dev = video_drvdata(file);
+	struct wbm2m_ctx *ctx = file->private_data;
+	bool fh_singular;
+
+	log_dbg(dev, "releasing instance %pa\n", &ctx);
+
+	mutex_lock(&dev->dev->lock);
+
+	/* Save the singular status before we call the clean-up helper */
+	fh_singular = v4l2_fh_is_singular_file(file);
+
+	v4l2_fh_del(&ctx->fh);
+	v4l2_fh_exit(&ctx->fh);
+	v4l2_m2m_ctx_release(ctx->fh.m2m_ctx);
+
+	kfree(ctx);
+
+	if (fh_singular) {
+		log_dbg(dev, "last instance released\n");
+
+		drm_modeset_lock_all(dev->dev->drm_dev);
+		tidss_plane_release_wb(dev->plane);
+		drm_modeset_unlock_all(dev->dev->drm_dev);
+		dev->dev->mode = TIDSS_WB_NOT_CONFIGURED;
+	}
+
+	mutex_unlock(&dev->dev->lock);
+
+	return 0;
+}
+
+static const struct v4l2_file_operations wbm2m_fops = {
+	.owner		= THIS_MODULE,
+	.open		= wbm2m_open,
+	.release	= wbm2m_release,
+	.poll		= v4l2_m2m_fop_poll,
+	.unlocked_ioctl	= video_ioctl2,
+	.mmap		= v4l2_m2m_fop_mmap,
+};
+
+static struct video_device wbm2m_videodev = {
+	.name		= WBM2M_MODULE_NAME,
+	.fops		= &wbm2m_fops,
+	.ioctl_ops	= &wbm2m_ioctl_ops,
+	.minor		= -1,
+	.release	= video_device_release_empty,
+	.vfl_dir	= VFL_DIR_M2M,
+	.device_caps	= V4L2_CAP_VIDEO_M2M_MPLANE | V4L2_CAP_STREAMING,
+};
+
+static struct v4l2_m2m_ops m2m_ops = {
+	.device_run	= device_run,
+	.job_ready	= job_ready,
+	.job_abort	= job_abort,
+};
+
+int tidss_wbm2m_init(struct wb_dev *dev)
+{
+	struct wbm2m_dev *wbm2m;
+	struct video_device *vfd;
+	int ret;
+
+	if (!dev)
+		return -ENOMEM;
+
+	/* Allocate a new instance */
+	wbm2m = devm_kzalloc(dev->drm_dev->dev, sizeof(*wbm2m), GFP_KERNEL);
+	if (!wbm2m)
+		return -ENOMEM;
+
+	dev->m2m = wbm2m;
+	wbm2m->dev = dev;
+
+	spin_lock_init(&wbm2m->lock);
+
+	snprintf(wbm2m->v4l2_dev.name, sizeof(wbm2m->v4l2_dev.name),
+		 "%s", WBM2M_MODULE_NAME);
+	ret = v4l2_device_register(dev->drm_dev->dev, &wbm2m->v4l2_dev);
+	if (ret)
+		return ret;
+
+	wbm2m->m2m_dev = v4l2_m2m_init(&m2m_ops);
+	if (IS_ERR(wbm2m->m2m_dev)) {
+		log_err(wbm2m, "Failed to init mem2mem device\n");
+		ret = PTR_ERR(wbm2m->m2m_dev);
+		goto v4l2_dev_unreg;
+	}
+
+	vb2_dma_contig_set_max_seg_size(dev->drm_dev->dev, DMA_BIT_MASK(32));
+
+	vfd = &wbm2m->vfd;
+	*vfd = wbm2m_videodev;
+	vfd->lock = &dev->lock;
+	vfd->v4l2_dev = &wbm2m->v4l2_dev;
+
+	ret = video_register_device(vfd, VFL_TYPE_GRABBER, 10);
+	if (ret) {
+		log_err(wbm2m, "Failed to register video device\n");
+		goto rel_m2m;
+	}
+
+	video_set_drvdata(vfd, wbm2m);
+	snprintf(vfd->name, sizeof(vfd->name), "%s", wbm2m_videodev.name);
+	log_dbg(wbm2m, "Device registered as %s\n",
+		video_device_node_name(vfd));
+
+	return 0;
+
+rel_m2m:
+	vb2_dma_contig_clear_max_seg_size(dev->drm_dev->dev);
+	v4l2_m2m_release(wbm2m->m2m_dev);
+v4l2_dev_unreg:
+	v4l2_device_unregister(&wbm2m->v4l2_dev);
+
+	return ret;
+}
+
+void tidss_wbm2m_cleanup(struct wb_dev *dev)
+{
+	log_dbg(dev->m2m, "Cleanup WB M2M\n");
+
+	vb2_dma_contig_clear_max_seg_size(dev->drm_dev->dev);
+	v4l2_m2m_release(dev->m2m->m2m_dev);
+	video_unregister_device(&dev->m2m->vfd);
+	v4l2_device_unregister(&dev->m2m->v4l2_dev);
+}
diff -Naur --no-dereference a/drivers/gpu/drm/tilcdc/tilcdc_tfp410.c b/drivers/gpu/drm/tilcdc/tilcdc_tfp410.c
--- a/drivers/gpu/drm/tilcdc/tilcdc_tfp410.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/tilcdc/tilcdc_tfp410.c	2020-06-19 21:00:31.412227715 -0400
@@ -379,7 +379,7 @@
 	.remove = tfp410_remove,
 	.driver = {
 		.owner = THIS_MODULE,
-		.name = "tfp410",
+		.name = "tilcdc-tfp410",
 		.of_match_table = tfp410_of_match,
 	},
 };
diff -Naur --no-dereference a/drivers/gpu/drm/tve200/tve200_display.c b/drivers/gpu/drm/tve200/tve200_display.c
--- a/drivers/gpu/drm/tve200/tve200_display.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/tve200/tve200_display.c	2020-06-19 21:00:31.412227715 -0400
@@ -149,7 +149,8 @@
 	/* Vsync IRQ at start of Vsync at first */
 	ctrl1 |= TVE200_VSTSTYPE_VSYNC;
 
-	if (connector->display_info.bus_flags & DRM_BUS_FLAG_PIXDATA_NEGEDGE)
+	if (connector->display_info.bus_flags &
+	    DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE)
 		ctrl1 |= TVE200_CTRL_TVCLKP;
 
 	if ((mode->hdisplay == 352 && mode->vdisplay == 240) || /* SIF(525) */
diff -Naur --no-dereference a/drivers/gpu/drm/vc4/vc4_kms.c b/drivers/gpu/drm/vc4/vc4_kms.c
--- a/drivers/gpu/drm/vc4/vc4_kms.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/gpu/drm/vc4/vc4_kms.c	2020-06-19 21:00:31.412227715 -0400
@@ -435,7 +435,8 @@
 	ctm_state = kzalloc(sizeof(*ctm_state), GFP_KERNEL);
 	if (!ctm_state)
 		return -ENOMEM;
-	drm_atomic_private_obj_init(&vc4->ctm_manager, &ctm_state->base,
+
+	drm_atomic_private_obj_init(dev, &vc4->ctm_manager, &ctm_state->base,
 				    &vc4_ctm_state_funcs);
 
 	drm_mode_config_reset(dev);
diff -Naur --no-dereference a/drivers/hwspinlock/Kconfig b/drivers/hwspinlock/Kconfig
--- a/drivers/hwspinlock/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/hwspinlock/Kconfig	2020-06-19 21:00:31.412227715 -0400
@@ -9,7 +9,7 @@
 config HWSPINLOCK_OMAP
 	tristate "OMAP Hardware Spinlock device"
 	depends on HWSPINLOCK
-	depends on ARCH_OMAP4 || SOC_OMAP5 || SOC_DRA7XX || SOC_AM33XX || SOC_AM43XX
+	depends on ARCH_OMAP4 || SOC_OMAP5 || SOC_DRA7XX || SOC_AM33XX || SOC_AM43XX || ARCH_K3
 	help
 	  Say y here to support the OMAP Hardware Spinlock device (firstly
 	  introduced in OMAP4).
diff -Naur --no-dereference a/drivers/hwspinlock/omap_hwspinlock.c b/drivers/hwspinlock/omap_hwspinlock.c
--- a/drivers/hwspinlock/omap_hwspinlock.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/hwspinlock/omap_hwspinlock.c	2020-06-19 21:00:31.412227715 -0400
@@ -140,6 +140,9 @@
 	if (ret)
 		goto reg_fail;
 
+	dev_dbg(&pdev->dev, "Registered %d locks with HwSpinlock core\n",
+		num_locks);
+
 	return 0;
 
 reg_fail:
@@ -171,6 +174,7 @@
 
 static const struct of_device_id omap_hwspinlock_of_match[] = {
 	{ .compatible = "ti,omap4-hwspinlock", },
+	{ .compatible = "ti,am654-hwspinlock", },
 	{ /* end */ },
 };
 MODULE_DEVICE_TABLE(of, omap_hwspinlock_of_match);
diff -Naur --no-dereference a/drivers/i2c/algos/i2c-algo-bit.c b/drivers/i2c/algos/i2c-algo-bit.c
--- a/drivers/i2c/algos/i2c-algo-bit.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/i2c/algos/i2c-algo-bit.c	2020-06-19 21:00:31.412227715 -0400
@@ -612,6 +612,23 @@
 	return ret;
 }
 
+/*
+ * We print a warning when we are not flagged to support atomic transfers but
+ * will try anyhow. That's what the I2C core would do as well. Sadly, we can't
+ * modify the algorithm struct at probe time because this struct is exported
+ * 'const'.
+ */
+static int bit_xfer_atomic(struct i2c_adapter *i2c_adap, struct i2c_msg msgs[],
+			   int num)
+{
+	struct i2c_algo_bit_data *adap = i2c_adap->algo_data;
+
+	if (!adap->can_do_atomic)
+		dev_warn(&i2c_adap->dev, "not flagged for atomic transfers\n");
+
+	return bit_xfer(i2c_adap, msgs, num);
+}
+
 static u32 bit_func(struct i2c_adapter *adap)
 {
 	return I2C_FUNC_I2C | I2C_FUNC_NOSTART | I2C_FUNC_SMBUS_EMUL |
@@ -624,8 +641,9 @@
 /* -----exported algorithm data: -------------------------------------	*/
 
 const struct i2c_algorithm i2c_bit_algo = {
-	.master_xfer	= bit_xfer,
-	.functionality	= bit_func,
+	.master_xfer = bit_xfer,
+	.master_xfer_atomic = bit_xfer_atomic,
+	.functionality = bit_func,
 };
 EXPORT_SYMBOL(i2c_bit_algo);
 
diff -Naur --no-dereference a/drivers/i2c/busses/i2c-gpio.c b/drivers/i2c/busses/i2c-gpio.c
--- a/drivers/i2c/busses/i2c-gpio.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/i2c/busses/i2c-gpio.c	2020-06-19 21:00:31.412227715 -0400
@@ -317,6 +317,8 @@
 
 	if (gpiod_cansleep(priv->sda) || gpiod_cansleep(priv->scl))
 		dev_warn(dev, "Slow GPIO pins might wreak havoc into I2C/SMBus bus timing");
+	else
+		bit_data->can_do_atomic = true;
 
 	bit_data->setsda = i2c_gpio_setsda_val;
 	bit_data->setscl = i2c_gpio_setscl_val;
diff -Naur --no-dereference a/drivers/i2c/busses/i2c-omap.c b/drivers/i2c/busses/i2c-omap.c
--- a/drivers/i2c/busses/i2c-omap.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/i2c/busses/i2c-omap.c	2020-06-19 21:00:31.412227715 -0400
@@ -269,6 +269,8 @@
 	[OMAP_I2C_IP_V2_IRQENABLE_CLR] = 0x30,
 };
 
+static int omap_i2c_xfer_data(struct omap_i2c_dev *omap);
+
 static inline void omap_i2c_write_reg(struct omap_i2c_dev *omap,
 				      int reg, u16 val)
 {
@@ -648,15 +650,28 @@
 			(1000 * omap->speed / 8);
 }
 
+static void omap_i2c_wait(struct omap_i2c_dev *omap)
+{
+	u16 stat;
+	u16 mask = omap_i2c_read_reg(omap, OMAP_I2C_IE_REG);
+	int count = 0;
+
+	do {
+		stat = omap_i2c_read_reg(omap, OMAP_I2C_STAT_REG);
+		count++;
+	} while (!(stat & mask) && count < 5);
+}
+
 /*
  * Low level master read/write transaction.
  */
 static int omap_i2c_xfer_msg(struct i2c_adapter *adap,
-			     struct i2c_msg *msg, int stop)
+			     struct i2c_msg *msg, int stop, bool polling)
 {
 	struct omap_i2c_dev *omap = i2c_get_adapdata(adap);
 	unsigned long timeout;
 	u16 w;
+	int ret;
 
 	dev_dbg(omap->dev, "addr: 0x%04x, len: %d, flags: 0x%x, stop: %d\n",
 		msg->addr, msg->len, msg->flags, stop);
@@ -680,7 +695,8 @@
 	w |= OMAP_I2C_BUF_RXFIF_CLR | OMAP_I2C_BUF_TXFIF_CLR;
 	omap_i2c_write_reg(omap, OMAP_I2C_BUF_REG, w);
 
-	reinit_completion(&omap->cmd_complete);
+	if (!polling)
+		reinit_completion(&omap->cmd_complete);
 	omap->cmd_err = 0;
 
 	w = OMAP_I2C_CON_EN | OMAP_I2C_CON_MST | OMAP_I2C_CON_STT;
@@ -732,8 +748,18 @@
 	 * REVISIT: We should abort the transfer on signals, but the bus goes
 	 * into arbitration and we're currently unable to recover from it.
 	 */
-	timeout = wait_for_completion_timeout(&omap->cmd_complete,
-						OMAP_I2C_TIMEOUT);
+	if (!polling) {
+		timeout = wait_for_completion_timeout(&omap->cmd_complete,
+						      OMAP_I2C_TIMEOUT);
+	} else {
+		do {
+			omap_i2c_wait(omap);
+			ret = omap_i2c_xfer_data(omap);
+		} while (ret == -EAGAIN);
+
+		timeout = !ret;
+	}
+
 	if (timeout == 0) {
 		dev_err(omap->dev, "controller timed out\n");
 		omap_i2c_reset(omap);
@@ -772,7 +798,8 @@
  * to do the work during IRQ processing.
  */
 static int
-omap_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num)
+omap_i2c_xfer_common(struct i2c_adapter *adap, struct i2c_msg msgs[], int num,
+		     bool polling)
 {
 	struct omap_i2c_dev *omap = i2c_get_adapdata(adap);
 	int i;
@@ -794,7 +821,8 @@
 		omap->set_mpu_wkup_lat(omap->dev, omap->latency);
 
 	for (i = 0; i < num; i++) {
-		r = omap_i2c_xfer_msg(adap, &msgs[i], (i == (num - 1)));
+		r = omap_i2c_xfer_msg(adap, &msgs[i], (i == (num - 1)),
+				      polling);
 		if (r != 0)
 			break;
 	}
@@ -813,6 +841,18 @@
 	return r;
 }
 
+static int
+omap_i2c_xfer_irq(struct i2c_adapter *adap, struct i2c_msg msgs[], int num)
+{
+	return omap_i2c_xfer_common(adap, msgs, num, false);
+}
+
+static int
+omap_i2c_xfer_polling(struct i2c_adapter *adap, struct i2c_msg msgs[], int num)
+{
+	return omap_i2c_xfer_common(adap, msgs, num, true);
+}
+
 static u32
 omap_i2c_func(struct i2c_adapter *adap)
 {
@@ -1035,10 +1075,8 @@
 	return ret;
 }
 
-static irqreturn_t
-omap_i2c_isr_thread(int this_irq, void *dev_id)
+static int omap_i2c_xfer_data(struct omap_i2c_dev *omap)
 {
-	struct omap_i2c_dev *omap = dev_id;
 	u16 bits;
 	u16 stat;
 	int err = 0, count = 0;
@@ -1056,7 +1094,8 @@
 
 		if (!stat) {
 			/* my work here is done */
-			goto out;
+			err = -EAGAIN;
+			break;
 		}
 
 		dev_dbg(omap->dev, "IRQ (ISR = 0x%04x)\n", stat);
@@ -1165,14 +1204,25 @@
 		}
 	} while (stat);
 
-	omap_i2c_complete_cmd(omap, err);
+	return err;
+}
+
+static irqreturn_t
+omap_i2c_isr_thread(int this_irq, void *dev_id)
+{
+	int ret;
+	struct omap_i2c_dev *omap = dev_id;
+
+	ret = omap_i2c_xfer_data(omap);
+	if (ret != -EAGAIN)
+		omap_i2c_complete_cmd(omap, ret);
 
-out:
 	return IRQ_HANDLED;
 }
 
 static const struct i2c_algorithm omap_i2c_algo = {
-	.master_xfer	= omap_i2c_xfer,
+	.master_xfer	= omap_i2c_xfer_irq,
+	.master_xfer_atomic	= omap_i2c_xfer_polling,
 	.functionality	= omap_i2c_func,
 };
 
diff -Naur --no-dereference a/drivers/i2c/busses/i2c-stu300.c b/drivers/i2c/busses/i2c-stu300.c
--- a/drivers/i2c/busses/i2c-stu300.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/i2c/busses/i2c-stu300.c	2020-06-19 21:00:31.412227715 -0400
@@ -328,12 +328,6 @@
 {
 	int ret;
 
-	if (unlikely(irqs_disabled())) {
-		/* TODO: implement polling for this case if need be. */
-		WARN(1, "irqs are disabled, cannot poll for event\n");
-		return -EIO;
-	}
-
 	/* Lock command issue, fill in an event we wait for */
 	spin_lock_irq(&dev->cmd_issue_lock);
 	init_completion(&dev->cmd_complete);
@@ -380,13 +374,6 @@
 {
 	int ret;
 
-	if (unlikely(irqs_disabled())) {
-		/* TODO: implement polling for this case if need be. */
-		dev_err(&dev->pdev->dev, "irqs are disabled on this "
-			"system!\n");
-		return -EIO;
-	}
-
 	/* Is it already here? */
 	spin_lock_irq(&dev->cmd_issue_lock);
 	dev->cmd_err = STU300_ERROR_NONE;
@@ -846,6 +833,13 @@
 	return num;
 }
 
+static int stu300_xfer_todo(struct i2c_adapter *adap, struct i2c_msg *msgs, int num)
+{
+	/* TODO: implement polling for this case if need be. */
+	WARN(1, "%s: atomic transfers not implemented\n", dev_name(&adap->dev));
+	return -EOPNOTSUPP;
+}
+
 static u32 stu300_func(struct i2c_adapter *adap)
 {
 	/* This is the simplest thing you can think of... */
@@ -853,8 +847,9 @@
 }
 
 static const struct i2c_algorithm stu300_algo = {
-	.master_xfer	= stu300_xfer,
-	.functionality	= stu300_func,
+	.master_xfer = stu300_xfer,
+	.master_xfer_atomic = stu300_xfer_todo,
+	.functionality = stu300_func,
 };
 
 static const struct i2c_adapter_quirks stu300_quirks = {
diff -Naur --no-dereference a/drivers/i2c/busses/i2c-tegra-bpmp.c b/drivers/i2c/busses/i2c-tegra-bpmp.c
--- a/drivers/i2c/busses/i2c-tegra-bpmp.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/i2c/busses/i2c-tegra-bpmp.c	2020-06-19 21:00:31.412227715 -0400
@@ -207,7 +207,8 @@
 
 static int tegra_bpmp_i2c_msg_xfer(struct tegra_bpmp_i2c *i2c,
 				   struct mrq_i2c_request *request,
-				   struct mrq_i2c_response *response)
+				   struct mrq_i2c_response *response,
+				   bool atomic)
 {
 	struct tegra_bpmp_message msg;
 	int err;
@@ -222,7 +223,7 @@
 	msg.rx.data = response;
 	msg.rx.size = sizeof(*response);
 
-	if (irqs_disabled())
+	if (atomic)
 		err = tegra_bpmp_transfer_atomic(i2c->bpmp, &msg);
 	else
 		err = tegra_bpmp_transfer(i2c->bpmp, &msg);
@@ -230,8 +231,9 @@
 	return err;
 }
 
-static int tegra_bpmp_i2c_xfer(struct i2c_adapter *adapter,
-			       struct i2c_msg *msgs, int num)
+static int tegra_bpmp_i2c_xfer_common(struct i2c_adapter *adapter,
+				      struct i2c_msg *msgs, int num,
+				      bool atomic)
 {
 	struct tegra_bpmp_i2c *i2c = i2c_get_adapdata(adapter);
 	struct mrq_i2c_response response;
@@ -253,7 +255,7 @@
 		return err;
 	}
 
-	err = tegra_bpmp_i2c_msg_xfer(i2c, &request, &response);
+	err = tegra_bpmp_i2c_msg_xfer(i2c, &request, &response, atomic);
 	if (err < 0) {
 		dev_err(i2c->dev, "failed to transfer message: %d\n", err);
 		return err;
@@ -268,6 +270,18 @@
 	return num;
 }
 
+static int tegra_bpmp_i2c_xfer(struct i2c_adapter *adapter,
+			       struct i2c_msg *msgs, int num)
+{
+	return tegra_bpmp_i2c_xfer_common(adapter, msgs, num, false);
+}
+
+static int tegra_bpmp_i2c_xfer_atomic(struct i2c_adapter *adapter,
+				      struct i2c_msg *msgs, int num)
+{
+	return tegra_bpmp_i2c_xfer_common(adapter, msgs, num, true);
+}
+
 static u32 tegra_bpmp_i2c_func(struct i2c_adapter *adapter)
 {
 	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL | I2C_FUNC_10BIT_ADDR |
@@ -276,6 +290,7 @@
 
 static const struct i2c_algorithm tegra_bpmp_i2c_algo = {
 	.master_xfer = tegra_bpmp_i2c_xfer,
+	.master_xfer_atomic = tegra_bpmp_i2c_xfer_atomic,
 	.functionality = tegra_bpmp_i2c_func,
 };
 
diff -Naur --no-dereference a/drivers/i2c/i2c-core-base.c b/drivers/i2c/i2c-core-base.c
--- a/drivers/i2c/i2c-core-base.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/i2c/i2c-core-base.c	2020-06-19 21:00:31.412227715 -0400
@@ -1891,7 +1891,11 @@
 	/* Retry automatically on arbitration loss */
 	orig_jiffies = jiffies;
 	for (ret = 0, try = 0; try <= adap->retries; try++) {
-		ret = adap->algo->master_xfer(adap, msgs, num);
+		if (i2c_in_atomic_xfer_mode() && adap->algo->master_xfer_atomic)
+			ret = adap->algo->master_xfer_atomic(adap, msgs, num);
+		else
+			ret = adap->algo->master_xfer(adap, msgs, num);
+
 		if (ret != -EAGAIN)
 			break;
 		if (time_after(jiffies, orig_jiffies + adap->timeout))
@@ -1953,15 +1957,9 @@
 				(msgs[ret].flags & I2C_M_RECV_LEN) ? "+" : "");
 		}
 #endif
-
-		if (in_atomic() || irqs_disabled()) {
-			ret = i2c_trylock_bus(adap, I2C_LOCK_SEGMENT);
-			if (!ret)
-				/* I2C activity is ongoing. */
-				return -EAGAIN;
-		} else {
-			i2c_lock_bus(adap, I2C_LOCK_SEGMENT);
-		}
+		ret = __i2c_lock_bus_helper(adap);
+		if (ret)
+			return ret;
 
 		ret = __i2c_transfer(adap, msgs, num);
 		i2c_unlock_bus(adap, I2C_LOCK_SEGMENT);
diff -Naur --no-dereference a/drivers/i2c/i2c-core.h b/drivers/i2c/i2c-core.h
--- a/drivers/i2c/i2c-core.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/i2c/i2c-core.h	2020-06-19 21:00:31.412227715 -0400
@@ -29,6 +29,31 @@
 
 int i2c_check_7bit_addr_validity_strict(unsigned short addr);
 
+/*
+ * We only allow atomic transfers for very late communication, e.g. to send
+ * the powerdown command to a PMIC. Atomic transfers are a corner case and not
+ * for generic use!
+ */
+static inline bool i2c_in_atomic_xfer_mode(void)
+{
+	return system_state > SYSTEM_RUNNING && irqs_disabled();
+}
+
+static inline int __i2c_lock_bus_helper(struct i2c_adapter *adap)
+{
+	int ret = 0;
+
+	if (i2c_in_atomic_xfer_mode()) {
+		WARN(!adap->algo->master_xfer_atomic && !adap->algo->smbus_xfer_atomic,
+		     "No atomic I2C transfer handler for '%s'\n", dev_name(&adap->dev));
+		ret = i2c_trylock_bus(adap, I2C_LOCK_SEGMENT) ? 0 : -EAGAIN;
+	} else {
+		i2c_lock_bus(adap, I2C_LOCK_SEGMENT);
+	}
+
+	return ret;
+}
+
 #ifdef CONFIG_ACPI
 const struct acpi_device_id *
 i2c_acpi_match_device(const struct acpi_device_id *matches,
diff -Naur --no-dereference a/drivers/i2c/i2c-core-smbus.c b/drivers/i2c/i2c-core-smbus.c
--- a/drivers/i2c/i2c-core-smbus.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/i2c/i2c-core-smbus.c	2020-06-19 21:00:31.412227715 -0400
@@ -23,6 +23,8 @@
 #define CREATE_TRACE_POINTS
 #include <trace/events/smbus.h>
 
+#include "i2c-core.h"
+
 
 /* The SMBus parts */
 
@@ -530,7 +532,10 @@
 {
 	s32 res;
 
-	i2c_lock_bus(adapter, I2C_LOCK_SEGMENT);
+	res = __i2c_lock_bus_helper(adapter);
+	if (res)
+		return res;
+
 	res = __i2c_smbus_xfer(adapter, addr, flags, read_write,
 			       command, protocol, data);
 	i2c_unlock_bus(adapter, I2C_LOCK_SEGMENT);
@@ -543,6 +548,9 @@
 		     unsigned short flags, char read_write,
 		     u8 command, int protocol, union i2c_smbus_data *data)
 {
+	int (*xfer_func)(struct i2c_adapter *adap, u16 addr,
+			 unsigned short flags, char read_write,
+			 u8 command, int size, union i2c_smbus_data *data);
 	unsigned long orig_jiffies;
 	int try;
 	s32 res;
@@ -557,13 +565,20 @@
 
 	flags &= I2C_M_TEN | I2C_CLIENT_PEC | I2C_CLIENT_SCCB;
 
-	if (adapter->algo->smbus_xfer) {
+	xfer_func = adapter->algo->smbus_xfer;
+	if (i2c_in_atomic_xfer_mode()) {
+		if (adapter->algo->smbus_xfer_atomic)
+			xfer_func = adapter->algo->smbus_xfer_atomic;
+		else if (adapter->algo->master_xfer_atomic)
+			xfer_func = NULL; /* fallback to I2C emulation */
+	}
+
+	if (xfer_func) {
 		/* Retry automatically on arbitration loss */
 		orig_jiffies = jiffies;
 		for (res = 0, try = 0; try <= adapter->retries; try++) {
-			res = adapter->algo->smbus_xfer(adapter, addr, flags,
-							read_write, command,
-							protocol, data);
+			res = xfer_func(adapter, addr, flags, read_write,
+					command, protocol, data);
 			if (res != -EAGAIN)
 				break;
 			if (time_after(jiffies,
diff -Naur --no-dereference a/drivers/i2c/i2c-mux.c b/drivers/i2c/i2c-mux.c
--- a/drivers/i2c/i2c-mux.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/i2c/i2c-mux.c	2020-06-19 21:00:31.412227715 -0400
@@ -310,12 +310,18 @@
 		else
 			priv->algo.master_xfer = __i2c_mux_master_xfer;
 	}
+	if (parent->algo->master_xfer_atomic)
+		priv->algo.master_xfer_atomic = priv->algo.master_xfer;
+
 	if (parent->algo->smbus_xfer) {
 		if (muxc->mux_locked)
 			priv->algo.smbus_xfer = i2c_mux_smbus_xfer;
 		else
 			priv->algo.smbus_xfer = __i2c_mux_smbus_xfer;
 	}
+	if (parent->algo->smbus_xfer_atomic)
+		priv->algo.smbus_xfer_atomic = priv->algo.smbus_xfer;
+
 	priv->algo.functionality = i2c_mux_functionality;
 
 	/* Now fill out new adapter structure */
diff -Naur --no-dereference a/drivers/i2c/muxes/i2c-demux-pinctrl.c b/drivers/i2c/muxes/i2c-demux-pinctrl.c
--- a/drivers/i2c/muxes/i2c-demux-pinctrl.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/i2c/muxes/i2c-demux-pinctrl.c	2020-06-19 21:00:31.412227715 -0400
@@ -99,6 +99,8 @@
 
 	/* Now fill out current adapter structure. cur_chan must be up to date */
 	priv->algo.master_xfer = i2c_demux_master_xfer;
+	if (adap->algo->master_xfer_atomic)
+		priv->algo.master_xfer_atomic = i2c_demux_master_xfer;
 	priv->algo.functionality = i2c_demux_functionality;
 
 	snprintf(priv->cur_adap.name, sizeof(priv->cur_adap.name),
diff -Naur --no-dereference a/drivers/iio/adc/Kconfig b/drivers/iio/adc/Kconfig
--- a/drivers/iio/adc/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/iio/adc/Kconfig	2020-06-19 21:00:31.412227715 -0400
@@ -705,6 +705,13 @@
 	  This driver can also be built as a module.  If so, the module
 	  will be called stm32-dfsdm-adc.
 
+config STMPE_ADC
+	tristate "STMicroelectronics STMPE ADC driver"
+	depends on OF && MFD_STMPE
+	help
+	  Say yes here to build support for ST Microelectronics STMPE
+	  built-in ADC block (stmpe811).
+
 config STX104
 	tristate "Apex Embedded Systems STX104 driver"
 	depends on PC104 && X86
diff -Naur --no-dereference a/drivers/iio/adc/Makefile b/drivers/iio/adc/Makefile
--- a/drivers/iio/adc/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/iio/adc/Makefile	2020-06-19 21:00:31.412227715 -0400
@@ -67,6 +67,7 @@
 obj-$(CONFIG_STM32_ADC) += stm32-adc.o
 obj-$(CONFIG_STM32_DFSDM_CORE) += stm32-dfsdm-core.o
 obj-$(CONFIG_STM32_DFSDM_ADC) += stm32-dfsdm-adc.o
+obj-$(CONFIG_STMPE_ADC) += stmpe-adc.o
 obj-$(CONFIG_TI_ADC081C) += ti-adc081c.o
 obj-$(CONFIG_TI_ADC0832) += ti-adc0832.o
 obj-$(CONFIG_TI_ADC084S021) += ti-adc084s021.o
diff -Naur --no-dereference a/drivers/iio/adc/stmpe-adc.c b/drivers/iio/adc/stmpe-adc.c
--- a/drivers/iio/adc/stmpe-adc.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/iio/adc/stmpe-adc.c	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,366 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  STMicroelectronics STMPE811 IIO ADC Driver
+ *
+ *  4 channel, 10/12-bit ADC
+ *
+ *  Copyright (C) 2013-2018 Toradex AG <stefan.agner@toradex.com>
+ */
+
+#include <linux/completion.h>
+#include <linux/err.h>
+#include <linux/iio/iio.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/mfd/stmpe.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/device.h>
+
+#define STMPE_REG_INT_STA		0x0B
+#define STMPE_REG_ADC_INT_EN		0x0E
+#define STMPE_REG_ADC_INT_STA		0x0F
+
+#define STMPE_REG_ADC_CTRL1		0x20
+#define STMPE_REG_ADC_CTRL2		0x21
+#define STMPE_REG_ADC_CAPT		0x22
+#define STMPE_REG_ADC_DATA_CH(channel)	(0x30 + 2 * (channel))
+
+#define STMPE_REG_TEMP_CTRL		0x60
+#define STMPE_TEMP_CTRL_ENABLE		BIT(0)
+#define STMPE_TEMP_CTRL_ACQ		BIT(1)
+#define STMPE_TEMP_CTRL_THRES_EN	BIT(3)
+#define STMPE_START_ONE_TEMP_CONV	(STMPE_TEMP_CTRL_ENABLE | \
+					STMPE_TEMP_CTRL_ACQ | \
+					STMPE_TEMP_CTRL_THRES_EN)
+#define STMPE_REG_TEMP_DATA		0x61
+#define STMPE_REG_TEMP_TH		0x63
+#define STMPE_ADC_LAST_NR		7
+#define STMPE_TEMP_CHANNEL		(STMPE_ADC_LAST_NR + 1)
+
+#define STMPE_ADC_CH(channel)		((1 << (channel)) & 0xff)
+
+#define STMPE_ADC_TIMEOUT		msecs_to_jiffies(1000)
+
+struct stmpe_adc {
+	struct stmpe *stmpe;
+	struct clk *clk;
+	struct device *dev;
+	struct mutex lock;
+
+	/* We are allocating plus one for the temperature channel */
+	struct iio_chan_spec stmpe_adc_iio_channels[STMPE_ADC_LAST_NR + 2];
+
+	struct completion completion;
+
+	u8 channel;
+	u32 value;
+};
+
+static int stmpe_read_voltage(struct stmpe_adc *info,
+		struct iio_chan_spec const *chan, int *val)
+{
+	long ret;
+
+	mutex_lock(&info->lock);
+
+	reinit_completion(&info->completion);
+
+	info->channel = (u8)chan->channel;
+
+	if (info->channel > STMPE_ADC_LAST_NR) {
+		mutex_unlock(&info->lock);
+		return -EINVAL;
+	}
+
+	stmpe_reg_write(info->stmpe, STMPE_REG_ADC_CAPT,
+			STMPE_ADC_CH(info->channel));
+
+	ret = wait_for_completion_timeout(&info->completion, STMPE_ADC_TIMEOUT);
+
+	if (ret <= 0) {
+		stmpe_reg_write(info->stmpe, STMPE_REG_ADC_INT_STA,
+				STMPE_ADC_CH(info->channel));
+		mutex_unlock(&info->lock);
+		return -ETIMEDOUT;
+	}
+
+	*val = info->value;
+
+	mutex_unlock(&info->lock);
+
+	return 0;
+}
+
+static int stmpe_read_temp(struct stmpe_adc *info,
+		struct iio_chan_spec const *chan, int *val)
+{
+	long ret;
+
+	mutex_lock(&info->lock);
+
+	reinit_completion(&info->completion);
+
+	info->channel = (u8)chan->channel;
+
+	if (info->channel != STMPE_TEMP_CHANNEL) {
+		mutex_unlock(&info->lock);
+		return -EINVAL;
+	}
+
+	stmpe_reg_write(info->stmpe, STMPE_REG_TEMP_CTRL,
+			STMPE_START_ONE_TEMP_CONV);
+
+	ret = wait_for_completion_timeout(&info->completion, STMPE_ADC_TIMEOUT);
+
+	if (ret <= 0) {
+		mutex_unlock(&info->lock);
+		return -ETIMEDOUT;
+	}
+
+	/*
+	 * absolute temp = +V3.3 * value /7.51 [K]
+	 * scale to [milli °C]
+	 */
+	*val = ((449960l * info->value) / 1024l) - 273150;
+
+	mutex_unlock(&info->lock);
+
+	return 0;
+}
+
+static int stmpe_read_raw(struct iio_dev *indio_dev,
+			  struct iio_chan_spec const *chan,
+			  int *val,
+			  int *val2,
+			  long mask)
+{
+	struct stmpe_adc *info = iio_priv(indio_dev);
+	long ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+	case IIO_CHAN_INFO_PROCESSED:
+
+		switch (chan->type) {
+		case IIO_VOLTAGE:
+			ret = stmpe_read_voltage(info, chan, val);
+			break;
+
+		case IIO_TEMP:
+			ret = stmpe_read_temp(info, chan, val);
+			break;
+		default:
+			return -EINVAL;
+		}
+
+		if (ret < 0)
+			return ret;
+
+		return IIO_VAL_INT;
+
+	case IIO_CHAN_INFO_SCALE:
+		*val = 3300;
+		*val2 = info->stmpe->mod_12b ? 12 : 10;
+		return IIO_VAL_FRACTIONAL_LOG2;
+
+	default:
+		break;
+	}
+
+	return -EINVAL;
+}
+
+static irqreturn_t stmpe_adc_isr(int irq, void *dev_id)
+{
+	struct stmpe_adc *info = (struct stmpe_adc *)dev_id;
+	u16 data;
+
+	if (info->channel <= STMPE_ADC_LAST_NR) {
+		int int_sta;
+
+		int_sta = stmpe_reg_read(info->stmpe, STMPE_REG_ADC_INT_STA);
+
+		/* Is the interrupt relevant */
+		if (!(int_sta & STMPE_ADC_CH(info->channel)))
+			return IRQ_NONE;
+
+		/* Read value */
+		stmpe_block_read(info->stmpe,
+			STMPE_REG_ADC_DATA_CH(info->channel), 2, (u8 *) &data);
+
+		stmpe_reg_write(info->stmpe, STMPE_REG_ADC_INT_STA, int_sta);
+	} else if (info->channel == STMPE_TEMP_CHANNEL) {
+		/* Read value */
+		stmpe_block_read(info->stmpe, STMPE_REG_TEMP_DATA, 2,
+				(u8 *) &data);
+	} else {
+		return IRQ_NONE;
+	}
+
+	info->value = (u32) be16_to_cpu(data);
+	complete(&info->completion);
+
+	return IRQ_HANDLED;
+}
+
+static const struct iio_info stmpe_adc_iio_info = {
+	.read_raw = &stmpe_read_raw,
+};
+
+static void stmpe_adc_voltage_chan(struct iio_chan_spec *ics, int chan)
+{
+	ics->type = IIO_VOLTAGE;
+	ics->info_mask_separate = BIT(IIO_CHAN_INFO_RAW);
+	ics->info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE);
+	ics->indexed = 1;
+	ics->channel = chan;
+}
+
+static void stmpe_adc_temp_chan(struct iio_chan_spec *ics, int chan)
+{
+	ics->type = IIO_TEMP;
+	ics->info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED);
+	ics->indexed = 1;
+	ics->channel = chan;
+}
+
+static int stmpe_adc_init_hw(struct stmpe_adc *adc)
+{
+	int ret;
+	struct stmpe *stmpe = adc->stmpe;
+
+	ret = stmpe_enable(stmpe, STMPE_BLOCK_ADC);
+	if (ret) {
+		dev_err(stmpe->dev, "Could not enable clock for ADC\n");
+		return ret;
+	}
+
+	ret = stmpe811_adc_common_init(stmpe);
+	if (ret) {
+		stmpe_disable(stmpe, STMPE_BLOCK_ADC);
+		return ret;
+	}
+
+	/* use temp irq for each conversion completion */
+	stmpe_reg_write(stmpe, STMPE_REG_TEMP_TH, 0);
+	stmpe_reg_write(stmpe, STMPE_REG_TEMP_TH + 1, 0);
+
+	return 0;
+}
+
+static int stmpe_adc_probe(struct platform_device *pdev)
+{
+	struct iio_dev *indio_dev;
+	struct stmpe_adc *info;
+	struct device_node *np;
+	u32 norequest_mask = 0;
+	int irq_temp, irq_adc;
+	int num_chan = 0;
+	int i = 0;
+	int ret;
+
+	irq_adc = platform_get_irq_byname(pdev, "STMPE_ADC");
+	if (irq_adc < 0)
+		return irq_adc;
+
+	indio_dev = devm_iio_device_alloc(&pdev->dev, sizeof(struct stmpe_adc));
+	if (!indio_dev) {
+		dev_err(&pdev->dev, "failed allocating iio device\n");
+		return -ENOMEM;
+	}
+
+	info = iio_priv(indio_dev);
+	mutex_init(&info->lock);
+
+	init_completion(&info->completion);
+	ret = devm_request_threaded_irq(&pdev->dev, irq_adc, NULL,
+					stmpe_adc_isr, IRQF_ONESHOT,
+					"stmpe-adc", info);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "failed requesting irq, irq = %d\n",
+				irq_adc);
+		return ret;
+	}
+
+	irq_temp = platform_get_irq_byname(pdev, "STMPE_TEMP_SENS");
+	if (irq_temp >= 0) {
+		ret = devm_request_threaded_irq(&pdev->dev, irq_temp, NULL,
+						stmpe_adc_isr, IRQF_ONESHOT,
+						"stmpe-adc", info);
+		if (ret < 0)
+			dev_warn(&pdev->dev, "failed requesting irq for"
+				 " temp sensor, irq = %d\n", irq_temp);
+	}
+
+	platform_set_drvdata(pdev, indio_dev);
+
+	indio_dev->name		= dev_name(&pdev->dev);
+	indio_dev->dev.parent	= &pdev->dev;
+	indio_dev->info		= &stmpe_adc_iio_info;
+	indio_dev->modes	= INDIO_DIRECT_MODE;
+
+	info->stmpe = dev_get_drvdata(pdev->dev.parent);
+
+	np = pdev->dev.of_node;
+
+	if (!np)
+		dev_err(&pdev->dev, "no device tree node found\n");
+
+	of_property_read_u32(np, "st,norequest-mask", &norequest_mask);
+
+	for_each_clear_bit(i, (unsigned long *) &norequest_mask,
+			   (STMPE_ADC_LAST_NR + 1)) {
+		stmpe_adc_voltage_chan(&info->stmpe_adc_iio_channels[num_chan], i);
+		num_chan++;
+	}
+	stmpe_adc_temp_chan(&info->stmpe_adc_iio_channels[num_chan], i);
+	num_chan++;
+	indio_dev->channels = info->stmpe_adc_iio_channels;
+	indio_dev->num_channels = num_chan;
+
+	ret = stmpe_adc_init_hw(info);
+	if (ret)
+		return ret;
+
+	stmpe_reg_write(info->stmpe, STMPE_REG_ADC_INT_EN,
+			~(norequest_mask & 0xFF));
+
+	stmpe_reg_write(info->stmpe, STMPE_REG_ADC_INT_STA,
+			~(norequest_mask & 0xFF));
+
+	return devm_iio_device_register(&pdev->dev, indio_dev);
+}
+
+static int __maybe_unused stmpe_adc_resume(struct device *dev)
+{
+	struct iio_dev *indio_dev = dev_get_drvdata(dev);
+	struct stmpe_adc *info = iio_priv(indio_dev);
+
+	stmpe_adc_init_hw(info);
+
+	return 0;
+}
+
+static SIMPLE_DEV_PM_OPS(stmpe_adc_pm_ops, NULL, stmpe_adc_resume);
+
+static struct platform_driver stmpe_adc_driver = {
+	.probe		= stmpe_adc_probe,
+	.driver		= {
+		.name	= "stmpe-adc",
+		.pm	= &stmpe_adc_pm_ops,
+	},
+};
+module_platform_driver(stmpe_adc_driver);
+
+static const struct of_device_id stmpe_adc_ids[] = {
+	{ .compatible = "st,stmpe-adc", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, stmpe_adc_ids);
+
+MODULE_AUTHOR("Stefan Agner <stefan.agner@toradex.com>");
+MODULE_DESCRIPTION("STMPEXXX ADC driver");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:stmpe-adc");
diff -Naur --no-dereference a/drivers/iio/adc/ti_am335x_adc.c b/drivers/iio/adc/ti_am335x_adc.c
--- a/drivers/iio/adc/ti_am335x_adc.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/iio/adc/ti_am335x_adc.c	2020-06-19 21:00:31.412227715 -0400
@@ -142,7 +142,10 @@
 			stepconfig |= STEPCONFIG_MODE_SWCNT;
 
 		tiadc_writel(adc_dev, REG_STEPCONFIG(steps),
-				stepconfig | STEPCONFIG_INP(chan));
+				stepconfig | STEPCONFIG_INP(chan) |
+				STEPCONFIG_INM_ADCREFM |
+				STEPCONFIG_RFP_VREFP |
+				STEPCONFIG_RFM_VREFN);
 
 		if (adc_dev->open_delay[i] > STEPDELAY_OPEN_MASK) {
 			dev_warn(dev, "chan %d open delay truncating to 0x3FFFF\n",
@@ -693,16 +696,12 @@
 {
 	struct iio_dev *indio_dev = dev_get_drvdata(dev);
 	struct tiadc_device *adc_dev = iio_priv(indio_dev);
-	struct ti_tscadc_dev *tscadc_dev;
 	unsigned int idle;
 
-	tscadc_dev = ti_tscadc_dev_get(to_platform_device(dev));
-	if (!device_may_wakeup(tscadc_dev->dev)) {
-		idle = tiadc_readl(adc_dev, REG_CTRL);
-		idle &= ~(CNTRLREG_TSCSSENB);
-		tiadc_writel(adc_dev, REG_CTRL, (idle |
-				CNTRLREG_POWERDOWN));
-	}
+	idle = tiadc_readl(adc_dev, REG_CTRL);
+	idle &= ~(CNTRLREG_TSCSSENB);
+	tiadc_writel(adc_dev, REG_CTRL, (idle |
+			CNTRLREG_POWERDOWN));
 
 	return 0;
 }
diff -Naur --no-dereference a/drivers/iio/proximity/Kconfig b/drivers/iio/proximity/Kconfig
--- a/drivers/iio/proximity/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/iio/proximity/Kconfig	2020-06-19 21:00:31.412227715 -0400
@@ -92,4 +92,15 @@
 	  To compile this driver as a module, choose M here: the
 	  module will be called srf08.
 
+config VL53L0X_I2C
+	tristate "STMicroelectronics VL53L0X ToF ranger sensor (I2C)"
+	depends on I2C
+	help
+	  Say Y here to build a driver for STMicroelectronics VL53L0X
+	  ToF ranger sensors with i2c interface.
+	  This driver can be used to measure the distance of objects.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called vl53l0x-i2c.
+
 endmenu
diff -Naur --no-dereference a/drivers/iio/proximity/Makefile b/drivers/iio/proximity/Makefile
--- a/drivers/iio/proximity/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/iio/proximity/Makefile	2020-06-19 21:00:31.412227715 -0400
@@ -11,3 +11,5 @@
 obj-$(CONFIG_SRF04)		+= srf04.o
 obj-$(CONFIG_SRF08)		+= srf08.o
 obj-$(CONFIG_SX9500)		+= sx9500.o
+obj-$(CONFIG_VL53L0X_I2C)	+= vl53l0x-i2c.o
+
diff -Naur --no-dereference a/drivers/iio/proximity/vl53l0x-i2c.c b/drivers/iio/proximity/vl53l0x-i2c.c
--- a/drivers/iio/proximity/vl53l0x-i2c.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/iio/proximity/vl53l0x-i2c.c	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,164 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Support for ST VL53L0X FlightSense ToF Ranging Sensor on a i2c bus.
+ *
+ * Copyright (C) 2016 STMicroelectronics Imaging Division.
+ * Copyright (C) 2018 Song Qiang <songqiang1304521@gmail.com>
+ *
+ * Datasheet available at
+ * <https://www.st.com/resource/en/datasheet/vl53l0x.pdf>
+ *
+ * Default 7-bit i2c slave address 0x29.
+ *
+ * TODO: FIFO buffer, continuous mode, interrupts, range selection,
+ * sensor ID check.
+ */
+
+#include <linux/delay.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+
+#include <linux/iio/iio.h>
+
+#define VL_REG_SYSRANGE_START				0x00
+
+#define VL_REG_SYSRANGE_MODE_MASK			GENMASK(3, 0)
+#define VL_REG_SYSRANGE_MODE_SINGLESHOT			0x00
+#define VL_REG_SYSRANGE_MODE_START_STOP			BIT(0)
+#define VL_REG_SYSRANGE_MODE_BACKTOBACK			BIT(1)
+#define VL_REG_SYSRANGE_MODE_TIMED			BIT(2)
+#define VL_REG_SYSRANGE_MODE_HISTOGRAM			BIT(3)
+
+#define VL_REG_RESULT_INT_STATUS			0x13
+#define VL_REG_RESULT_RANGE_STATUS			0x14
+#define VL_REG_RESULT_RANGE_STATUS_COMPLETE		BIT(0)
+
+struct vl53l0x_data {
+	struct i2c_client *client;
+};
+
+static int vl53l0x_read_proximity(struct vl53l0x_data *data,
+				  const struct iio_chan_spec *chan,
+				  int *val)
+{
+	struct i2c_client *client = data->client;
+	u16 tries = 20;
+	u8 buffer[12];
+	int ret;
+
+	ret = i2c_smbus_write_byte_data(client, VL_REG_SYSRANGE_START, 1);
+	if (ret < 0)
+		return ret;
+
+	do {
+		ret = i2c_smbus_read_byte_data(client,
+					       VL_REG_RESULT_RANGE_STATUS);
+		if (ret < 0)
+			return ret;
+
+		if (ret & VL_REG_RESULT_RANGE_STATUS_COMPLETE)
+			break;
+
+		usleep_range(1000, 5000);
+	} while (--tries);
+	if (!tries)
+		return -ETIMEDOUT;
+
+	ret = i2c_smbus_read_i2c_block_data(client, VL_REG_RESULT_RANGE_STATUS,
+					    12, buffer);
+	if (ret < 0)
+		return ret;
+	else if (ret != 12)
+		return -EREMOTEIO;
+
+	/* Values should be between 30~1200 in millimeters. */
+	*val = (buffer[10] << 8) + buffer[11];
+
+	return 0;
+}
+
+static const struct iio_chan_spec vl53l0x_channels[] = {
+	{
+		.type = IIO_DISTANCE,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				      BIT(IIO_CHAN_INFO_SCALE),
+	},
+};
+
+static int vl53l0x_read_raw(struct iio_dev *indio_dev,
+			    const struct iio_chan_spec *chan,
+			    int *val, int *val2, long mask)
+{
+	struct vl53l0x_data *data = iio_priv(indio_dev);
+	int ret;
+
+	if (chan->type != IIO_DISTANCE)
+		return -EINVAL;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		ret = vl53l0x_read_proximity(data, chan, val);
+		if (ret < 0)
+			return ret;
+
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SCALE:
+		*val = 0;
+		*val2 = 1000;
+
+		return IIO_VAL_INT_PLUS_MICRO;
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct iio_info vl53l0x_info = {
+	.read_raw = vl53l0x_read_raw,
+};
+
+static int vl53l0x_probe(struct i2c_client *client)
+{
+	struct vl53l0x_data *data;
+	struct iio_dev *indio_dev;
+
+	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	data = iio_priv(indio_dev);
+	data->client = client;
+	i2c_set_clientdata(client, indio_dev);
+
+	if (!i2c_check_functionality(client->adapter,
+				     I2C_FUNC_SMBUS_READ_I2C_BLOCK |
+				     I2C_FUNC_SMBUS_BYTE_DATA))
+		return -EOPNOTSUPP;
+
+	indio_dev->dev.parent = &client->dev;
+	indio_dev->name = "vl53l0x";
+	indio_dev->info = &vl53l0x_info;
+	indio_dev->channels = vl53l0x_channels;
+	indio_dev->num_channels = ARRAY_SIZE(vl53l0x_channels);
+	indio_dev->modes = INDIO_DIRECT_MODE;
+
+	return devm_iio_device_register(&client->dev, indio_dev);
+}
+
+static const struct of_device_id st_vl53l0x_dt_match[] = {
+	{ .compatible = "st,vl53l0x", },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, st_vl53l0x_dt_match);
+
+static struct i2c_driver vl53l0x_driver = {
+	.driver = {
+		.name = "vl53l0x-i2c",
+		.of_match_table = st_vl53l0x_dt_match,
+	},
+	.probe_new = vl53l0x_probe,
+};
+module_i2c_driver(vl53l0x_driver);
+
+MODULE_AUTHOR("Song Qiang <songqiang1304521@gmail.com>");
+MODULE_DESCRIPTION("ST vl53l0x ToF ranging sensor driver");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/input/misc/tps65218-pwrbutton.c b/drivers/input/misc/tps65218-pwrbutton.c
--- a/drivers/input/misc/tps65218-pwrbutton.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/input/misc/tps65218-pwrbutton.c	2020-06-19 21:00:31.412227715 -0400
@@ -36,7 +36,7 @@
 static const struct tps6521x_data tps65217_data = {
 	.reg_status = TPS65217_REG_STATUS,
 	.pb_mask = TPS65217_STATUS_PB,
-	.name = "tps65217_pwrbutton",
+	.name = "tps65217_pwr_but",
 };
 
 static const struct tps6521x_data tps65218_data = {
diff -Naur --no-dereference a/drivers/input/touchscreen/ar1021_i2c.c b/drivers/input/touchscreen/ar1021_i2c.c
--- a/drivers/input/touchscreen/ar1021_i2c.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/input/touchscreen/ar1021_i2c.c	2020-06-19 21:00:31.412227715 -0400
@@ -18,6 +18,11 @@
 
 #define AR1021_MAX_X	4095
 #define AR1021_MAX_Y	4095
+#define AR1021_MAX_PRESSURE 255
+
+#define AR1021_CMD	0x55
+
+#define AR1021_CMD_ENABLE_TOUCH		0x12
 
 #define AR1021_CMD	0x55
 
@@ -27,8 +32,29 @@
 	struct i2c_client *client;
 	struct input_dev *input;
 	u8 data[AR1021_TOCUH_PKG_SIZE];
+	bool invert_x;
+	bool invert_y;
+	bool swap_xy;
 };
 
+static bool ar1021_get_prop_u32(struct device *dev,
+				     const char *property,
+				     unsigned int default_value,
+				     unsigned int *value)
+{
+	u32 val;
+	int error;
+
+	error = device_property_read_u32(dev, property, &val);
+	if (error) {
+		*value = default_value;
+		return false;
+	}
+
+	*value = val;
+	return true;
+}
+
 static irqreturn_t ar1021_i2c_irq(int irq, void *dev_id)
 {
 	struct ar1021_i2c *ar1021 = dev_id;
@@ -50,9 +76,22 @@
 	x = ((data[2] & 0x1f) << 7) | (data[1] & 0x7f);
 	y = ((data[4] & 0x1f) << 7) | (data[3] & 0x7f);
 
-	input_report_abs(input, ABS_X, x);
-	input_report_abs(input, ABS_Y, y);
+	if (ar1021->invert_x)
+		x = AR1021_MAX_X - x;
+
+	if (ar1021->invert_y)
+		y = AR1021_MAX_Y - y;
+
+	if (ar1021->swap_xy) {
+		input_report_abs(input, ABS_X, y);
+		input_report_abs(input, ABS_Y, x);
+	} else {
+		input_report_abs(input, ABS_X, x);
+		input_report_abs(input, ABS_Y, y);
+	}
+
 	input_report_key(input, BTN_TOUCH, button);
+	input_report_abs(input, ABS_PRESSURE, AR1021_MAX_PRESSURE);
 	input_sync(input);
 
 out:
@@ -94,6 +133,8 @@
 	struct ar1021_i2c *ar1021;
 	struct input_dev *input;
 	int error;
+	unsigned int offset_x, offset_y;
+	bool data_present;
 
 	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) {
 		dev_err(&client->dev, "i2c_check_functionality error\n");
@@ -117,10 +158,44 @@
 	input->open = ar1021_i2c_open;
 	input->close = ar1021_i2c_close;
 
+	ar1021->invert_x = device_property_read_bool(&client->dev, "touchscreen-inverted-x");
+	ar1021->invert_y = device_property_read_bool(&client->dev, "touchscreen-inverted-y");
+	ar1021->swap_xy = device_property_read_bool(&client->dev, "touchscreen-swapped-x-y");
+
+	data_present = ar1021_get_prop_u32(&client->dev,
+						"touchscreen-offset-x",
+						0,
+						&offset_x);
+
+	if (data_present)
+		dev_info(&client->dev, "touchscreen-offset-x: %d\n", offset_x);
+
+	data_present = ar1021_get_prop_u32(&client->dev,
+						"touchscreen-offset-y",
+						0,
+						&offset_y);
+
+	if (data_present)
+		dev_info(&client->dev, "touchscreen-offset-y: %d\n", offset_y);
+
 	__set_bit(INPUT_PROP_DIRECT, input->propbit);
-	input_set_capability(input, EV_KEY, BTN_TOUCH);
-	input_set_abs_params(input, ABS_X, 0, AR1021_MAX_X, 0, 0);
-	input_set_abs_params(input, ABS_Y, 0, AR1021_MAX_Y, 0, 0);
+	//input_set_capability(input, EV_KEY, BTN_TOUCH);
+
+	input->evbit[0] = BIT_MASK(EV_KEY) | BIT_MASK(EV_ABS);
+	input->keybit[BIT_WORD(BTN_TOUCH)] = BIT_MASK(BTN_TOUCH);
+
+	if(ar1021->swap_xy)
+	{
+		input_set_abs_params(input, ABS_X, 0, AR1021_MAX_Y, 0, 0);
+		input_set_abs_params(input, ABS_Y, 0, AR1021_MAX_X, 0, 0);
+	}
+	else
+	{
+		input_set_abs_params(input, ABS_X, offset_x, AR1021_MAX_X-offset_x, 0, 0);
+		input_set_abs_params(input, ABS_Y, offset_y, AR1021_MAX_Y-offset_y, 0, 0);
+	}
+
+	input_set_abs_params(input, ABS_PRESSURE, 0, AR1021_MAX_PRESSURE, 0, 0);
 
 	input_set_drvdata(input, ar1021);
 
diff -Naur --no-dereference a/drivers/input/touchscreen/edt-ft5x06.c b/drivers/input/touchscreen/edt-ft5x06.c
--- a/drivers/input/touchscreen/edt-ft5x06.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/input/touchscreen/edt-ft5x06.c	2020-06-19 21:00:31.412227715 -0400
@@ -1121,8 +1121,10 @@
 {
 	struct i2c_client *client = to_i2c_client(dev);
 
-	if (device_may_wakeup(dev))
+	if (device_may_wakeup(dev)) {
 		enable_irq_wake(client->irq);
+		disable_irq(client->irq);
+	}
 
 	return 0;
 }
@@ -1131,8 +1133,10 @@
 {
 	struct i2c_client *client = to_i2c_client(dev);
 
-	if (device_may_wakeup(dev))
+	if (device_may_wakeup(dev)) {
 		disable_irq_wake(client->irq);
+		enable_irq(client->irq);
+	}
 
 	return 0;
 }
diff -Naur --no-dereference a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c
--- a/drivers/input/touchscreen/goodix.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/input/touchscreen/goodix.c	2020-06-19 21:00:31.412227715 -0400
@@ -493,31 +493,17 @@
 {
 	int error;
 
-	/* begin select I2C slave addr */
-	error = gpiod_direction_output(ts->gpiod_rst, 0);
-	if (error)
-		return error;
-
 	msleep(20);				/* T2: > 10ms */
 
 	/* HIGH: 0x28/0x29, LOW: 0xBA/0xBB */
-	error = gpiod_direction_output(ts->gpiod_int, ts->client->addr == 0x14);
-	if (error)
-		return error;
+	gpiod_set_value_cansleep(ts->gpiod_int, ts->client->addr == 0x14);
 
 	usleep_range(100, 2000);		/* T3: > 100us */
 
-	error = gpiod_direction_output(ts->gpiod_rst, 1);
-	if (error)
-		return error;
+	gpiod_set_value_cansleep(ts->gpiod_rst, 1);
 
 	usleep_range(6000, 10000);		/* T4: > 5ms */
 
-	/* end select I2C slave addr */
-	error = gpiod_direction_input(ts->gpiod_rst);
-	if (error)
-		return error;
-
 	error = goodix_int_sync(ts);
 	if (error)
 		return error;
@@ -541,7 +527,8 @@
 	dev = &ts->client->dev;
 
 	/* Get the interrupt GPIO pin number */
-	gpiod = devm_gpiod_get_optional(dev, GOODIX_GPIO_INT_NAME, GPIOD_IN);
+	gpiod = devm_gpiod_get_optional(dev, GOODIX_GPIO_INT_NAME,
+					GPIOD_OUT_LOW);
 	if (IS_ERR(gpiod)) {
 		error = PTR_ERR(gpiod);
 		if (error != -EPROBE_DEFER)
@@ -553,7 +540,8 @@
 	ts->gpiod_int = gpiod;
 
 	/* Get the reset line GPIO pin number */
-	gpiod = devm_gpiod_get_optional(dev, GOODIX_GPIO_RST_NAME, GPIOD_IN);
+	gpiod = devm_gpiod_get_optional(dev, GOODIX_GPIO_RST_NAME,
+					GPIOD_OUT_LOW);
 	if (IS_ERR(gpiod)) {
 		error = PTR_ERR(gpiod);
 		if (error != -EPROBE_DEFER)
@@ -935,6 +923,13 @@
 
 static const struct i2c_device_id goodix_ts_id[] = {
 	{ "GDIX1001:00", 0 },
+	{ "gt911", 0 },
+	{ "gt9110", 0 },
+	{ "gt912", 0 },
+	{ "gt927", 0 },
+	{ "gt9271", 0 },
+	{ "gt928", 0 },
+	{ "gt967", 0 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, goodix_ts_id);
diff -Naur --no-dereference a/drivers/input/touchscreen/ti_am335x_tsc.c b/drivers/input/touchscreen/ti_am335x_tsc.c
--- a/drivers/input/touchscreen/ti_am335x_tsc.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/input/touchscreen/ti_am335x_tsc.c	2020-06-19 21:00:31.412227715 -0400
@@ -27,12 +27,14 @@
 #include <linux/of.h>
 #include <linux/of_device.h>
 #include <linux/sort.h>
+#include <linux/pm_wakeirq.h>
 
 #include <linux/mfd/ti_am335x_tscadc.h>
 
 #define ADCFSM_STEPID		0x10
 #define SEQ_SETTLE		275
 #define MAX_12BIT		((1 << 12) - 1)
+#define PRESSURE_MAX		1000
 
 #define TSC_IRQENB_MASK		(IRQENB_FIFO0THRES | IRQENB_EOS | IRQENB_HW_PEN)
 
@@ -46,6 +48,7 @@
 struct titsc {
 	struct input_dev	*input;
 	struct ti_tscadc_dev	*mfd_tscadc;
+	struct device		*dev;
 	unsigned int		irq;
 	unsigned int		wires;
 	unsigned int		x_plate_resistance;
@@ -232,6 +235,7 @@
 	for (i = 0; i < creads; i++) {
 		xvals[i] = titsc_readl(ts_dev, REG_FIFO0);
 		xvals[i] &= 0xfff;
+		pr_debug("i %d xval %d yval %d z1 %d z2 %d\n", i, xvals[i], yvals[i], *z1, *z2);
 	}
 
 	/*
@@ -276,7 +280,7 @@
 	if (status & IRQENB_HW_PEN) {
 		ts_dev->pen_down = true;
 		irqclr |= IRQENB_HW_PEN;
-		pm_stay_awake(ts_dev->mfd_tscadc->dev);
+		pm_stay_awake(ts_dev->dev);
 	}
 
 	if (status & IRQENB_PENUP) {
@@ -286,7 +290,7 @@
 			input_report_key(input_dev, BTN_TOUCH, 0);
 			input_report_abs(input_dev, ABS_PRESSURE, 0);
 			input_sync(input_dev);
-			pm_relax(ts_dev->mfd_tscadc->dev);
+			pm_relax(ts_dev->dev);
 		} else {
 			ts_dev->pen_down = true;
 		}
@@ -310,13 +314,13 @@
 			 * Resistance(touch) = x plate resistance *
 			 * x postion/4096 * ((z2 / z1) - 1)
 			 */
-			z = z1 - z2;
+			z = z2 - z1;
 			z *= x;
 			z *= ts_dev->x_plate_resistance;
-			z /= z2;
+			z /= z1;
 			z = (z + 2047) >> 12;
-
-			if (z <= MAX_12BIT) {
+			pr_debug("x %d y %d z1 %d z2 %d z %d\n", x, y, z1, z2, z);
+			if (z <= PRESSURE_MAX) {
 				input_report_abs(input_dev, ABS_X, x);
 				input_report_abs(input_dev, ABS_Y, y);
 				input_report_abs(input_dev, ABS_PRESSURE, z);
@@ -422,6 +426,7 @@
 	ts_dev->mfd_tscadc = tscadc_dev;
 	ts_dev->input = input_dev;
 	ts_dev->irq = tscadc_dev->irq;
+	ts_dev->dev = &pdev->dev;
 
 	err = titsc_parse_dt(pdev, ts_dev);
 	if (err) {
@@ -436,6 +441,11 @@
 		goto err_free_mem;
 	}
 
+	device_init_wakeup(&pdev->dev, true);
+	err = dev_pm_set_wake_irq(&pdev->dev, ts_dev->irq);
+	if (err)
+		dev_err(&pdev->dev, "irq wake enable failed.\n");
+
 	titsc_writel(ts_dev, REG_IRQSTATUS, TSC_IRQENB_MASK);
 	titsc_writel(ts_dev, REG_IRQENABLE, IRQENB_FIFO0THRES);
 	titsc_writel(ts_dev, REG_IRQENABLE, IRQENB_EOS);
@@ -451,6 +461,7 @@
 	input_dev->name = "ti-tsc";
 	input_dev->dev.parent = &pdev->dev;
 
+	__set_bit(INPUT_PROP_DIRECT, input_dev->propbit);
 	input_dev->evbit[0] = BIT_MASK(EV_KEY) | BIT_MASK(EV_ABS);
 	input_dev->keybit[BIT_WORD(BTN_TOUCH)] = BIT_MASK(BTN_TOUCH);
 
@@ -467,6 +478,8 @@
 	return 0;
 
 err_free_irq:
+	dev_pm_clear_wake_irq(&pdev->dev);
+	device_init_wakeup(&pdev->dev, false);
 	free_irq(ts_dev->irq, ts_dev);
 err_free_mem:
 	input_free_device(input_dev);
@@ -479,6 +492,8 @@
 	struct titsc *ts_dev = platform_get_drvdata(pdev);
 	u32 steps;
 
+	dev_pm_clear_wake_irq(&pdev->dev);
+	device_init_wakeup(&pdev->dev, false);
 	free_irq(ts_dev->irq, ts_dev);
 
 	/* total steps followed by the enable mask */
@@ -499,7 +514,7 @@
 	unsigned int idle;
 
 	tscadc_dev = ti_tscadc_dev_get(to_platform_device(dev));
-	if (device_may_wakeup(tscadc_dev->dev)) {
+	if (device_may_wakeup(dev)) {
 		titsc_writel(ts_dev, REG_IRQSTATUS, TSC_IRQENB_MASK);
 		idle = titsc_readl(ts_dev, REG_IRQENABLE);
 		titsc_writel(ts_dev, REG_IRQENABLE,
@@ -515,11 +530,11 @@
 	struct ti_tscadc_dev *tscadc_dev;
 
 	tscadc_dev = ti_tscadc_dev_get(to_platform_device(dev));
-	if (device_may_wakeup(tscadc_dev->dev)) {
+	if (device_may_wakeup(dev)) {
 		titsc_writel(ts_dev, REG_IRQWAKEUP,
 				0x00);
 		titsc_writel(ts_dev, REG_IRQCLR, IRQENB_HW_PEN);
-		pm_relax(ts_dev->mfd_tscadc->dev);
+		pm_relax(dev);
 	}
 	titsc_step_config(ts_dev);
 	titsc_writel(ts_dev, REG_FIFO0THR,
diff -Naur --no-dereference a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c
--- a/drivers/iommu/omap-iommu.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/iommu/omap-iommu.c	2020-06-19 21:00:31.412227715 -0400
@@ -44,6 +44,12 @@
 /* bitmap of the page sizes currently supported */
 #define OMAP_IOMMU_PGSIZES	(SZ_4K | SZ_64K | SZ_1M | SZ_16M)
 
+/*
+ * total size of L1 and L2 page tables reserved/used by bootloader per rproc
+ * for early boot usecases, must match the value used in bootloader
+ */
+#define EARLY_PAGE_TABLES_SIZE	SZ_256K
+
 #define MMU_LOCK_BASE_SHIFT	10
 #define MMU_LOCK_BASE_MASK	(0x1f << MMU_LOCK_BASE_SHIFT)
 #define MMU_LOCK_BASE(x)	\
@@ -69,6 +75,9 @@
 /**
  * omap_iommu_save_ctx - Save registers for pm off-mode support
  * @dev:	client device
+ *
+ * This should be treated as an deprecated API. It is preserved only
+ * to maintain existing functionality for OMAP3 ISP driver.
  **/
 void omap_iommu_save_ctx(struct device *dev)
 {
@@ -96,6 +105,9 @@
 /**
  * omap_iommu_restore_ctx - Restore registers for pm off-mode support
  * @dev:	client device
+ *
+ * This should be treated as an deprecated API. It is preserved only
+ * to maintain existing functionality for OMAP3 ISP driver.
  **/
 void omap_iommu_restore_ctx(struct device *dev)
 {
@@ -157,7 +169,7 @@
 	if (!obj->iopgd || !IS_ALIGNED((u32)obj->iopgd,  SZ_16K))
 		return -EINVAL;
 
-	pa = virt_to_phys(obj->iopgd);
+	pa = obj->iopgd_pa;
 	if (!IS_ALIGNED(pa, SZ_16K))
 		return -EINVAL;
 
@@ -190,36 +202,27 @@
 
 static int iommu_enable(struct omap_iommu *obj)
 {
-	int err;
-	struct platform_device *pdev = to_platform_device(obj->dev);
-	struct iommu_platform_data *pdata = dev_get_platdata(&pdev->dev);
-
-	if (pdata && pdata->deassert_reset) {
-		err = pdata->deassert_reset(pdev, pdata->reset_name);
-		if (err) {
-			dev_err(obj->dev, "deassert_reset failed: %d\n", err);
-			return err;
-		}
-	}
+	int ret;
 
-	pm_runtime_get_sync(obj->dev);
+	/*
+	 * now that the threat of idling has passed, decrement the
+	 * device usage count to balance the increment done in probe,
+	 * the pm runtime device usage count will be managed normally
+	 * from here on
+	 */
+	if (obj->late_attach)
+		pm_runtime_put_noidle(obj->dev);
 
-	err = omap2_iommu_enable(obj);
+	ret = pm_runtime_get_sync(obj->dev);
+	if (ret < 0)
+		pm_runtime_put_noidle(obj->dev);
 
-	return err;
+	return ret < 0 ? ret : 0;
 }
 
 static void iommu_disable(struct omap_iommu *obj)
 {
-	struct platform_device *pdev = to_platform_device(obj->dev);
-	struct iommu_platform_data *pdata = dev_get_platdata(&pdev->dev);
-
-	omap2_iommu_disable(obj);
-
 	pm_runtime_put_sync(obj->dev);
-
-	if (pdata && pdata->assert_reset)
-		pdata->assert_reset(pdev, pdata->reset_name);
 }
 
 /*
@@ -549,7 +552,7 @@
 	}
 
 pte_ready:
-	iopte = iopte_offset(iopgd, da);
+	iopte = iopte_get(obj, iopgd, da);
 	*pt_dma = iopgd_page_paddr(iopgd);
 	dev_vdbg(obj->dev,
 		 "%s: da:%08x pgd:%p *pgd:%08x pte:%p *pte:%08x\n",
@@ -708,7 +711,7 @@
 		goto out;
 
 	if (iopgd_is_table(*iopgd))
-		iopte = iopte_offset(iopgd, da);
+		iopte = iopte_get(obj, iopgd, da);
 out:
 	*ppgd = iopgd;
 	*ppte = iopte;
@@ -728,13 +731,13 @@
 
 	if (iopgd_is_table(*iopgd)) {
 		int i;
-		u32 *iopte = iopte_offset(iopgd, da);
+		u32 *iopte = iopte_get(obj, iopgd, da);
 
 		bytes = IOPTE_SIZE;
 		if (*iopte & IOPTE_LARGE) {
 			nent *= 16;
 			/* rewind to the 1st entry */
-			iopte = iopte_offset(iopgd, (da & IOLARGE_MASK));
+			iopte = iopte_get(obj, iopgd, (da & IOLARGE_MASK));
 		}
 		bytes *= nent;
 		memset(iopte, 0, nent * sizeof(*iopte));
@@ -744,7 +747,8 @@
 		/*
 		 * do table walk to check if this table is necessary or not
 		 */
-		iopte = iopte_offset(iopgd, 0);
+		iopte = iopte_get(obj, iopgd, 0);
+
 		for (i = 0; i < PTRS_PER_IOPTE; i++)
 			if (iopte[i])
 				goto out;
@@ -803,8 +807,15 @@
 		if (!*iopgd)
 			continue;
 
-		if (iopgd_is_table(*iopgd))
-			iopte_free(obj, iopte_offset(iopgd, 0), true);
+		if (iopgd_is_table(*iopgd)) {
+			if (obj->late_attach)
+				iopte_free(obj, iopte_offset_lateattach(obj,
+									iopgd,
+									0),
+					   true);
+			else
+				iopte_free(obj, iopte_offset(iopgd, 0), true);
+		}
 
 		*iopgd = 0;
 		flush_iopte_range(obj->dev, obj->pd_dma, offset, 1);
@@ -847,7 +858,7 @@
 		return IRQ_NONE;
 	}
 
-	iopte = iopte_offset(iopgd, da);
+	iopte = iopte_get(obj, iopgd, da);
 
 	dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:0x%08x pte:0x%p *pte:0x%08x\n",
 		obj->name, errs, da, iopgd, *iopgd, iopte, *iopte);
@@ -863,6 +874,16 @@
 static int omap_iommu_attach(struct omap_iommu *obj, u32 *iopgd)
 {
 	int err;
+	u32 iopgd_pa;
+
+	if (obj->late_attach) {
+		iopgd_pa = iommu_read_reg(obj, MMU_TTB);
+		iopgd = ioremap(iopgd_pa, EARLY_PAGE_TABLES_SIZE);
+		if (!iopgd)
+			return -ENOMEM;
+	} else {
+		iopgd_pa = virt_to_phys(iopgd);
+	}
 
 	spin_lock(&obj->iommu_lock);
 
@@ -874,11 +895,14 @@
 		goto out_err;
 	}
 
+	obj->iopgd_pa = iopgd_pa;
 	obj->iopgd = iopgd;
 	err = iommu_enable(obj);
 	if (err)
 		goto out_err;
-	flush_iotlb_all(obj);
+
+	if (!obj->late_attach)
+		flush_iotlb_all(obj);
 
 	spin_unlock(&obj->iommu_lock);
 
@@ -901,19 +925,231 @@
 	if (!obj || IS_ERR(obj))
 		return;
 
+	if (obj->late_attach && obj->iopgd)
+		iounmap(obj->iopgd);
+
 	spin_lock(&obj->iommu_lock);
 
 	dma_unmap_single(obj->dev, obj->pd_dma, IOPGD_TABLE_SIZE,
 			 DMA_TO_DEVICE);
-	iommu_disable(obj);
 	obj->pd_dma = 0;
+
+	obj->iopgd_pa = 0;
 	obj->iopgd = NULL;
+	iommu_disable(obj);
+	obj->late_attach = 0;
 
 	spin_unlock(&obj->iommu_lock);
 
 	dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name);
 }
 
+static void omap_iommu_save_tlb_entries(struct omap_iommu *obj)
+{
+	struct iotlb_lock lock;
+	struct cr_regs cr;
+	struct cr_regs *tmp;
+	int i;
+
+	/* check if there are any locked tlbs to save */
+	iotlb_lock_get(obj, &lock);
+	obj->num_cr_ctx = lock.base;
+	if (!obj->num_cr_ctx)
+		return;
+
+	tmp = obj->cr_ctx;
+	for_each_iotlb_cr(obj, obj->num_cr_ctx, i, cr)
+		*tmp++ = cr;
+}
+
+static void omap_iommu_restore_tlb_entries(struct omap_iommu *obj)
+{
+	struct iotlb_lock l;
+	struct cr_regs *tmp;
+	int i;
+
+	/* no locked tlbs to restore */
+	if (!obj->num_cr_ctx)
+		return;
+
+	l.base = 0;
+	tmp = obj->cr_ctx;
+	for (i = 0; i < obj->num_cr_ctx; i++, tmp++) {
+		l.vict = i;
+		iotlb_lock_set(obj, &l);
+		iotlb_load_cr(obj, tmp);
+	}
+	l.base = obj->num_cr_ctx;
+	l.vict = i;
+	iotlb_lock_set(obj, &l);
+}
+
+/**
+ * omap_iommu_domain_deactivate - deactivate attached iommu devices
+ * @domain: iommu domain attached to the target iommu device
+ *
+ * This API allows the client devices of IOMMU devices to suspend
+ * the IOMMUs they control at runtime, after they are idled and
+ * suspended all activity. System Suspend will leverage the PM
+ * driver late callbacks.
+ **/
+int omap_iommu_domain_deactivate(struct iommu_domain *domain)
+{
+	struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
+	struct omap_iommu_device *iommu;
+	struct omap_iommu *oiommu;
+	int i;
+
+	if (!omap_domain->dev)
+		return 0;
+
+	iommu = omap_domain->iommus;
+	iommu += (omap_domain->num_iommus - 1);
+	for (i = 0; i < omap_domain->num_iommus; i++, iommu--) {
+		oiommu = iommu->iommu_dev;
+		pm_runtime_put_sync(oiommu->dev);
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(omap_iommu_domain_deactivate);
+
+/**
+ * omap_iommu_domain_activate - activate attached iommu devices
+ * @domain: iommu domain attached to the target iommu device
+ *
+ * This API allows the client devices of IOMMU devices to resume the
+ * IOMMUs they control at runtime, before they can resume operations.
+ * System Resume will leverage the PM driver late callbacks.
+ **/
+int omap_iommu_domain_activate(struct iommu_domain *domain)
+{
+	struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
+	struct omap_iommu_device *iommu;
+	struct omap_iommu *oiommu;
+	int i;
+
+	if (!omap_domain->dev)
+		return 0;
+
+	iommu = omap_domain->iommus;
+	for (i = 0; i < omap_domain->num_iommus; i++, iommu++) {
+		oiommu = iommu->iommu_dev;
+		pm_runtime_get_sync(oiommu->dev);
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(omap_iommu_domain_activate);
+
+/**
+ * omap_iommu_runtime_suspend - disable an iommu device
+ * @dev:	iommu device
+ *
+ * This function performs all that is necessary to disable an
+ * IOMMU device, either during final detachment from a client
+ * device, or during system/runtime suspend of the device. This
+ * includes programming all the appropriate IOMMU registers, and
+ * managing the associated omap_hwmod's state and the device's
+ * reset line. This function also saves the context of any
+ * locked TLBs if suspending.
+ **/
+static int omap_iommu_runtime_suspend(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct iommu_platform_data *pdata = dev_get_platdata(dev);
+	struct omap_iommu *obj = to_iommu(dev);
+	int ret;
+
+	/* save the TLBs only during suspend, and not for power down */
+	if (obj->domain && obj->iopgd)
+		omap_iommu_save_tlb_entries(obj);
+
+	omap2_iommu_disable(obj);
+
+	if (pdata && pdata->device_idle)
+		pdata->device_idle(pdev);
+
+	if (pdata && pdata->assert_reset)
+		pdata->assert_reset(pdev, pdata->reset_name);
+
+	if (pdata && pdata->set_pwrdm_constraint) {
+		ret = pdata->set_pwrdm_constraint(pdev, false, &obj->pwrst);
+		if (ret) {
+			dev_warn(obj->dev, "pwrdm_constraint failed to be reset, status = %d\n",
+				 ret);
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * omap_iommu_runtime_resume - enable an iommu device
+ * @dev:	iommu device
+ *
+ * This function performs all that is necessary to enable an
+ * IOMMU device, either during initial attachment to a client
+ * device, or during system/runtime resume of the device. This
+ * includes programming all the appropriate IOMMU registers, and
+ * managing the associated omap_hwmod's state and the device's
+ * reset line. The function also restores any locked TLBs if
+ * resuming after a suspend.
+ **/
+static int omap_iommu_runtime_resume(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct iommu_platform_data *pdata = dev_get_platdata(dev);
+	struct omap_iommu *obj = to_iommu(dev);
+	int ret = 0;
+
+	if (pdata && pdata->set_pwrdm_constraint) {
+		ret = pdata->set_pwrdm_constraint(pdev, true, &obj->pwrst);
+		if (ret) {
+			dev_warn(obj->dev, "pwrdm_constraint failed to be set, status = %d\n",
+				 ret);
+		}
+	}
+
+	/* do not deassert reset only during initial boot for late attach */
+	if ((!obj->late_attach || obj->domain) &&
+	    pdata && pdata->deassert_reset) {
+		ret = pdata->deassert_reset(pdev, pdata->reset_name);
+		if (ret) {
+			dev_err(dev, "deassert_reset failed: %d\n", ret);
+			return ret;
+		}
+	}
+
+	if (pdata && pdata->device_enable)
+		pdata->device_enable(pdev);
+
+	/* restore the TLBs only during resume, and not for power up */
+	if (obj->domain)
+		omap_iommu_restore_tlb_entries(obj);
+
+	ret = omap2_iommu_enable(obj);
+
+	return ret;
+}
+
+/**
+ * omap_iommu_suspend_prepare - prepare() dev_pm_ops implementation
+ * @dev:	iommu device
+ *
+ * This function performs the necessary checks to determine if the IOMMU
+ * device needs suspending or not. The function checks if the runtime_pm
+ * status of the device is suspended, and returns 1 in that case. This
+ * results in the PM core to skip invoking any of the Sleep PM callbacks
+ * (suspend, suspend_late, resume, resume_early etc).
+ */
+static int omap_iommu_prepare(struct device *dev)
+{
+	if (pm_runtime_status_suspended(dev))
+		return 1;
+	return 0;
+}
+
 static bool omap_iommu_can_register(struct platform_device *pdev)
 {
 	struct device_node *np = pdev->dev.of_node;
@@ -978,6 +1214,7 @@
 	struct omap_iommu *obj;
 	struct resource *res;
 	struct device_node *of = pdev->dev.of_node;
+	struct iommu_platform_data *pdata = dev_get_platdata(&pdev->dev);
 
 	if (!of) {
 		pr_err("%s: only DT-based devices are supported\n", __func__);
@@ -988,9 +1225,19 @@
 	if (!obj)
 		return -ENOMEM;
 
+	/*
+	 * self-manage the ordering dependencies between omap_device_enable/idle
+	 * and omap_device_assert/deassert_hardreset API
+	 */
+	if (pdev->dev.pm_domain) {
+		dev_dbg(&pdev->dev, "device pm_domain is being reset\n");
+		pdev->dev.pm_domain = NULL;
+	}
+
 	obj->name = dev_name(&pdev->dev);
 	obj->nr_tlb_entries = 32;
 	err = of_property_read_u32(of, "ti,#tlb-entries", &obj->nr_tlb_entries);
+
 	if (err && err != -EINVAL)
 		return err;
 	if (obj->nr_tlb_entries != 32 && obj->nr_tlb_entries != 8)
@@ -998,8 +1245,17 @@
 	if (of_find_property(of, "ti,iommu-bus-err-back", NULL))
 		obj->has_bus_err_back = MMU_GP_REG_BUS_ERR_BACK_EN;
 
+	if (pdata && pdata->device_is_enabled &&
+	    pdata->device_is_enabled(pdev))
+		obj->late_attach = 1;
+
 	obj->dev = &pdev->dev;
 	obj->ctx = (void *)obj + sizeof(*obj);
+	obj->cr_ctx = devm_kzalloc(&pdev->dev,
+				   sizeof(*obj->cr_ctx) * obj->nr_tlb_entries,
+				   GFP_KERNEL);
+	if (!obj->cr_ctx)
+		return -ENOMEM;
 
 	spin_lock_init(&obj->iommu_lock);
 	spin_lock_init(&obj->page_table_lock);
@@ -1041,6 +1297,15 @@
 	}
 
 	pm_runtime_irq_safe(obj->dev);
+
+	/*
+	 * increment the device usage count so that runtime_suspend is not
+	 * invoked immediately after the probe (due to the ti,no-idle-on-init)
+	 * and before any remoteproc has attached to the iommu
+	 */
+	if (obj->late_attach)
+		pm_runtime_get_noresume(obj->dev);
+
 	pm_runtime_enable(obj->dev);
 
 	omap_iommu_debugfs_add(obj);
@@ -1076,6 +1341,14 @@
 	return 0;
 }
 
+static const struct dev_pm_ops omap_iommu_pm_ops = {
+	.prepare = omap_iommu_prepare,
+	SET_LATE_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				     pm_runtime_force_resume)
+	SET_RUNTIME_PM_OPS(omap_iommu_runtime_suspend,
+			   omap_iommu_runtime_resume, NULL)
+};
+
 static const struct of_device_id omap_iommu_of_match[] = {
 	{ .compatible = "ti,omap2-iommu" },
 	{ .compatible = "ti,omap4-iommu" },
@@ -1089,6 +1362,7 @@
 	.remove	= omap_iommu_remove,
 	.driver	= {
 		.name	= "omap-iommu",
+		.pm	= &omap_iommu_pm_ops,
 		.of_match_table = of_match_ptr(omap_iommu_of_match),
 	},
 };
@@ -1213,6 +1487,11 @@
 
 	iommu = odomain->iommus;
 	for (i = 0; i < odomain->num_iommus; i++, iommu++) {
+		/*
+		 * not necessary for late attach, the page table would be setup
+		 * by the boot loader. Leaving the below code in place, it does
+		 * not have any side effects during late attach.
+		 */
 		iommu->pgtable = kzalloc(IOPGD_TABLE_SIZE, GFP_ATOMIC);
 		if (!iommu->pgtable)
 			return -ENOMEM;
@@ -1334,7 +1613,8 @@
 	arch_data += (omap_domain->num_iommus - 1);
 	for (i = 0; i < omap_domain->num_iommus; i++, iommu--, arch_data--) {
 		oiommu = iommu->iommu_dev;
-		iopgtable_clear_entry_all(oiommu);
+		if (!oiommu->late_attach)
+			iopgtable_clear_entry_all(oiommu);
 
 		omap_iommu_detach(oiommu);
 		iommu->iommu_dev = NULL;
@@ -1558,7 +1838,7 @@
 static int __init omap_iommu_init(void)
 {
 	struct kmem_cache *p;
-	const unsigned long flags = SLAB_HWCACHE_ALIGN;
+	const slab_flags_t flags = SLAB_HWCACHE_ALIGN;
 	size_t align = 1 << 10; /* L2 pagetable alignement */
 	struct device_node *np;
 	int ret;
diff -Naur --no-dereference a/drivers/iommu/omap-iommu-debug.c b/drivers/iommu/omap-iommu-debug.c
--- a/drivers/iommu/omap-iommu-debug.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/iommu/omap-iommu-debug.c	2020-06-19 21:00:31.412227715 -0400
@@ -159,7 +159,7 @@
 	return 0;
 }
 
-static int debug_read_tlb(struct seq_file *s, void *data)
+static int tlb_show(struct seq_file *s, void *data)
 {
 	struct omap_iommu *obj = s->private;
 
@@ -197,7 +197,7 @@
 			continue;
 		}
 
-		iopte = iopte_offset(iopgd, 0);
+		iopte = iopte_get(obj, iopgd, 0);
 		for (j = 0; j < PTRS_PER_IOPTE; j++, iopte++) {
 			if (!*iopte)
 				continue;
@@ -210,7 +210,7 @@
 	spin_unlock(&obj->page_table_lock);
 }
 
-static int debug_read_pagetable(struct seq_file *s, void *data)
+static int pagetable_show(struct seq_file *s, void *data)
 {
 	struct omap_iommu *obj = s->private;
 
@@ -228,35 +228,22 @@
 	return 0;
 }
 
-#define DEBUG_SEQ_FOPS_RO(name)						       \
-	static int debug_open_##name(struct inode *inode, struct file *file)   \
-	{								       \
-		return single_open(file, debug_read_##name, inode->i_private); \
-	}								       \
-									       \
-	static const struct file_operations debug_##name##_fops = {	       \
-		.open		= debug_open_##name,			       \
-		.read		= seq_read,				       \
-		.llseek		= seq_lseek,				       \
-		.release	= single_release,			       \
-	}
-
 #define DEBUG_FOPS_RO(name)						\
-	static const struct file_operations debug_##name##_fops = {	\
+	static const struct file_operations name##_fops = {	        \
 		.open = simple_open,					\
 		.read = debug_read_##name,				\
 		.llseek = generic_file_llseek,				\
 	}
 
 DEBUG_FOPS_RO(regs);
-DEBUG_SEQ_FOPS_RO(tlb);
-DEBUG_SEQ_FOPS_RO(pagetable);
+DEFINE_SHOW_ATTRIBUTE(tlb);
+DEFINE_SHOW_ATTRIBUTE(pagetable);
 
 #define __DEBUG_ADD_FILE(attr, mode)					\
 	{								\
 		struct dentry *dent;					\
 		dent = debugfs_create_file(#attr, mode, obj->debug_dir,	\
-					   obj, &debug_##attr##_fops);	\
+					   obj, &attr##_fops);	        \
 		if (!dent)						\
 			goto err;					\
 	}
diff -Naur --no-dereference a/drivers/iommu/omap-iommu.h b/drivers/iommu/omap-iommu.h
--- a/drivers/iommu/omap-iommu.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/iommu/omap-iommu.h	2020-06-19 21:00:31.412227715 -0400
@@ -69,6 +69,8 @@
 	 * but share it globally for each iommu.
 	 */
 	u32		*iopgd;
+	u32		iopgd_pa;
+	u32		late_attach;
 	spinlock_t	page_table_lock; /* protect iopgd */
 	dma_addr_t	pd_dma;
 
@@ -76,11 +78,16 @@
 
 	void *ctx; /* iommu context: registres saved area */
 
+	struct cr_regs *cr_ctx;
+	u32 num_cr_ctx;
+
 	int has_bus_err_back;
 	u32 id;
 
 	struct iommu_device iommu;
 	struct iommu_group *group;
+
+	u8 pwrst;
 };
 
 /**
@@ -267,4 +274,12 @@
 	return cr->cam & MMU_CAM_V;
 }
 
+static inline u32 *iopte_get(struct omap_iommu *obj, u32 *iopgd, u32 da)
+{
+	if (obj->late_attach)
+		return iopte_offset_lateattach(obj, iopgd, da);
+	else
+		return iopte_offset(iopgd, da);
+}
+
 #endif /* _OMAP_IOMMU_H */
diff -Naur --no-dereference a/drivers/iommu/omap-iopgtable.h b/drivers/iommu/omap-iopgtable.h
--- a/drivers/iommu/omap-iopgtable.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/iommu/omap-iopgtable.h	2020-06-19 21:00:31.412227715 -0400
@@ -99,4 +99,16 @@
 #define iopte_index(da)		(((da) >> IOPTE_SHIFT) & (PTRS_PER_IOPTE - 1))
 #define iopte_offset(iopgd, da)	(iopgd_page_vaddr(iopgd) + iopte_index(da))
 
+/*
+ * compute vaddr for second-level page table relative to page table directory
+ * for late-attach mode
+ */
+#define iopgd_page_vaddr_lateattach(obj, pgd)				\
+	((u32 *)((u32 *)((obj)->iopgd)) +				\
+	((u32 *)iopgd_page_paddr((pgd)) - (u32 *)((obj)->iopgd_pa)))
+
+/* to find an entry in the second-level page table for late-attach mode */
+#define iopte_offset_lateattach(obj, iopgd, da)				\
+	(iopgd_page_vaddr_lateattach(obj, iopgd) + iopte_index(da))
+
 #endif /* _OMAP_IOPGTABLE_H */
diff -Naur --no-dereference a/drivers/irqchip/irq-pruss-intc.c b/drivers/irqchip/irq-pruss-intc.c
--- a/drivers/irqchip/irq-pruss-intc.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/irqchip/irq-pruss-intc.c	2020-06-19 21:00:31.412227715 -0400
@@ -0,0 +1,797 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * PRU-ICSS INTC IRQChip driver for various TI SoCs
+ *
+ * Copyright (C) 2016-2019 Texas Instruments Incorporated - http://www.ti.com/
+ *	Andrew F. Davis <afd@ti.com>
+ *	Suman Anna <s-anna@ti.com>
+ */
+
+#include <linux/bitmap.h>
+#include <linux/irq.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/irqdomain.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/pruss_driver.h>
+
+/*
+ * Number of host interrupts reaching the main MPU sub-system. Note that this
+ * is not the same as the total number of host interrupts supported by the PRUSS
+ * INTC instance
+ */
+#define MAX_HOST_NUM_IRQS	8
+
+/* PRU_ICSS_INTC registers */
+#define PRU_INTC_REVID		0x0000
+#define PRU_INTC_CR		0x0004
+#define PRU_INTC_GER		0x0010
+#define PRU_INTC_GNLR		0x001C
+#define PRU_INTC_SISR		0x0020
+#define PRU_INTC_SICR		0x0024
+#define PRU_INTC_EISR		0x0028
+#define PRU_INTC_EICR		0x002C
+#define PRU_INTC_HIEISR		0x0034
+#define PRU_INTC_HIDISR		0x0038
+#define PRU_INTC_GPIR		0x0080
+#define PRU_INTC_SRSR(x)	(0x0200 + (x) * 4)
+#define PRU_INTC_SRSR0		0x0200
+#define PRU_INTC_SRSR1		0x0204
+#define PRU_INTC_SECR(x)	(0x0280 + (x) * 4)
+#define PRU_INTC_SECR0		0x0280
+#define PRU_INTC_SECR1		0x0284
+#define PRU_INTC_ESR(x)		(0x0300 + (x) * 4)
+#define PRU_INTC_ESR0		0x0300
+#define PRU_INTC_ESR1		0x0304
+#define PRU_INTC_ECR(x)		(0x0380 + (x) * 4)
+#define PRU_INTC_ECR0		0x0380
+#define PRU_INTC_ECR1		0x0384
+#define PRU_INTC_CMR(x)		(0x0400 + (x) * 4)
+#define PRU_INTC_HMR(x)		(0x0800 + (x) * 4)
+#define PRU_INTC_HIPIR(x)	(0x0900 + (x) * 4)
+#define PRU_INTC_SIPR(x)	(0x0D00 + (x) * 4)
+#define PRU_INTC_SITR(x)	(0x0D80 + (x) * 4)
+#define PRU_INTC_HINLR(x)	(0x1100 + (x) * 4)
+#define PRU_INTC_HIER		0x1500
+
+/* CMR register bit-field macros */
+#define CMR_EVT_MAP_MASK	0xf
+#define CMR_EVT_MAP_BITS	8
+#define CMR_EVT_PER_REG		4
+
+/* HMR register bit-field macros */
+#define HMR_CH_MAP_MASK		0xf
+#define HMR_CH_MAP_BITS		8
+#define HMR_CH_PER_REG		4
+
+/* HIPIR register bit-fields */
+#define INTC_HIPIR_NONE_HINT	0x80000000
+
+static const char * const irq_names[] = {
+	"host2", "host3", "host4", "host5", "host6", "host7", "host8", "host9",
+};
+
+/**
+ * struct pruss_intc_match_data - match data to handle SoC variations
+ * @num_system_events: number of input system events handled by the PRUSS INTC
+ * @num_host_intrs: number of host interrupts supported by the PRUSS INTC
+ * @no_host7_intr: flag denoting the absence of host7 interrupt into MPU
+ */
+struct pruss_intc_match_data {
+	u8 num_system_events;
+	u8 num_host_intrs;
+	bool no_host7_intr;
+};
+
+/**
+ * struct pruss_intc - PRUSS interrupt controller structure
+ * @pruss: back-reference to parent PRUSS structure
+ * @irqs: kernel irq numbers corresponding to PRUSS host interrupts
+ * @base: base virtual address of INTC register space
+ * @irqchip: irq chip for this interrupt controller
+ * @domain: irq domain for this interrupt controller
+ * @data: cached PRUSS INTC IP configuration data
+ * @config_map: stored INTC configuration mapping data
+ * @lock: mutex to serialize access to INTC
+ * @host_mask: indicate which HOST IRQs are enabled
+ */
+struct pruss_intc {
+	struct pruss *pruss;
+	unsigned int irqs[MAX_HOST_NUM_IRQS];
+	void __iomem *base;
+	struct irq_chip *irqchip;
+	struct irq_domain *domain;
+	const struct pruss_intc_match_data *data;
+	struct pruss_intc_config config_map;
+	struct mutex lock; /* PRUSS INTC lock */
+	u32 host_mask;
+};
+
+static inline u32 pruss_intc_read_reg(struct pruss_intc *intc, unsigned int reg)
+{
+	return readl_relaxed(intc->base + reg);
+}
+
+static inline void pruss_intc_write_reg(struct pruss_intc *intc,
+					unsigned int reg, u32 val)
+{
+	writel_relaxed(val, intc->base + reg);
+}
+
+static int pruss_intc_check_write(struct pruss_intc *intc, unsigned int reg,
+				  unsigned int sysevent)
+{
+	if (!intc)
+		return -EINVAL;
+
+	if (sysevent >= intc->data->num_system_events)
+		return -EINVAL;
+
+	pruss_intc_write_reg(intc, reg, sysevent);
+
+	return 0;
+}
+
+static struct pruss_intc *to_pruss_intc(struct pruss *pruss)
+{
+	struct device_node *parent = pruss->dev->of_node;
+	struct device_node *np;
+	struct platform_device *pdev;
+	struct pruss_intc *intc = NULL;
+
+	np = of_get_child_by_name(parent, "interrupt-controller");
+	if (!np) {
+		dev_err(pruss->dev, "pruss does not have an interrupt-controller node\n");
+		return NULL;
+	}
+
+	pdev = of_find_device_by_node(np);
+	if (!pdev) {
+		dev_err(pruss->dev, "no associated platform device\n");
+		goto out;
+	}
+
+	intc = platform_get_drvdata(pdev);
+out:
+	of_node_put(np);
+	return intc;
+}
+
+/**
+ * pruss_intc_configure() - configure the PRUSS INTC
+ * @pruss: the pruss instance
+ * @intc_config: PRU core-specific INTC configuration
+ *
+ * Configures the PRUSS INTC with the provided configuration from
+ * a PRU core. Any existing event to channel mappings or channel to
+ * host interrupt mappings are checked to make sure there are no
+ * conflicting configuration between both the PRU cores. The function
+ * is intended to be used only by the PRU remoteproc driver.
+ *
+ * Returns 0 on success, or a suitable error code otherwise
+ */
+int pruss_intc_configure(struct pruss *pruss,
+			 struct pruss_intc_config *intc_config)
+{
+	struct device *dev = pruss->dev;
+	struct pruss_intc *intc;
+	int i, idx;
+	s8 ch, host;
+	u32 num_events, num_intrs, num_regs;
+	u32 ch_mask = 0;
+	u32 host_mask = 0;
+	int ret = 0;
+	u32 val;
+	unsigned long *sysevt_bitmap;
+	u32 *sysevts;
+
+	intc = to_pruss_intc(pruss);
+	if (!intc)
+		return -EINVAL;
+
+	num_events = intc->data->num_system_events;
+	num_intrs = intc->data->num_host_intrs;
+	num_regs = DIV_ROUND_UP(num_events, 32);
+
+	sysevt_bitmap = bitmap_zalloc(num_events, GFP_KERNEL);
+	if (!sysevt_bitmap)
+		return -ENOMEM;
+	sysevts = (u32 *)sysevt_bitmap;
+
+	mutex_lock(&intc->lock);
+
+	/*
+	 * configure channel map registers - each register holds map info
+	 * for 4 events, with each event occupying the lower nibble in
+	 * a register byte address in little-endian fashion
+	 */
+	for (i = 0; i < num_events; i++) {
+		ch = intc_config->sysev_to_ch[i];
+		if (ch < 0)
+			continue;
+
+		/* check if sysevent already assigned */
+		if (intc->config_map.sysev_to_ch[i] != -1) {
+			dev_err(dev, "event %d (req. channel %d) already assigned to channel %d\n",
+				i, ch, intc->config_map.sysev_to_ch[i]);
+			ret = -EEXIST;
+			goto unlock;
+		}
+
+		intc->config_map.sysev_to_ch[i] = ch;
+
+		idx = i / CMR_EVT_PER_REG;
+		val = pruss_intc_read_reg(intc, PRU_INTC_CMR(idx));
+		val &= ~(CMR_EVT_MAP_MASK <<
+			 ((i % CMR_EVT_PER_REG) * CMR_EVT_MAP_BITS));
+		val |= ch << ((i % CMR_EVT_PER_REG) * CMR_EVT_MAP_BITS);
+		pruss_intc_write_reg(intc, PRU_INTC_CMR(idx), val);
+		bitmap_set(sysevt_bitmap, i, 1);
+		ch_mask |= BIT(ch);
+
+		dev_dbg(dev, "SYSEV%d -> CH%d (CMR%d 0x%08x)\n", i, ch, idx,
+			pruss_intc_read_reg(intc, PRU_INTC_CMR(idx)));
+	}
+
+	/*
+	 * set host map registers - each register holds map info for
+	 * 4 channels, with each channel occupying the lower nibble in
+	 * a register byte address in little-endian fashion
+	 */
+	for (i = 0; i < num_intrs; i++) {
+		host = intc_config->ch_to_host[i];
+		if (host < 0)
+			continue;
+
+		/* check if channel already assigned */
+		if (intc->config_map.ch_to_host[i] != -1) {
+			dev_err(dev, "channel %d (req. intr_no %d) already assigned to intr_no %d\n",
+				i, host, intc->config_map.ch_to_host[i]);
+			ret = -EEXIST;
+			goto unlock;
+		}
+
+		/* check if host intr is already in use by other PRU */
+		if (intc->host_mask & (1U << host)) {
+			dev_err(dev, "%s: host intr %d already in use\n",
+				__func__, host);
+			ret = -EEXIST;
+			goto unlock;
+		}
+
+		intc->config_map.ch_to_host[i] = host;
+
+		idx = i / HMR_CH_PER_REG;
+
+		val = pruss_intc_read_reg(intc, PRU_INTC_HMR(idx));
+		val &= ~(HMR_CH_MAP_MASK <<
+			 ((i % HMR_CH_PER_REG) * HMR_CH_MAP_BITS));
+		val |= host << ((i % HMR_CH_PER_REG) * HMR_CH_MAP_BITS);
+		pruss_intc_write_reg(intc, PRU_INTC_HMR(idx), val);
+
+		ch_mask |= BIT(i);
+		host_mask |= BIT(host);
+
+		dev_dbg(dev, "CH%d -> HOST%d (HMR%d 0x%08x)\n", i, host, idx,
+			pruss_intc_read_reg(intc, PRU_INTC_HMR(idx)));
+	}
+
+	dev_info(dev, "configured system_events[%d-0] = %*pb\n",
+		 num_events - 1, num_events, sysevt_bitmap);
+	dev_info(dev, "configured intr_channels = 0x%08x host_intr = 0x%08x\n",
+		 ch_mask, host_mask);
+
+	/* enable system events, writing 0 has no-effect */
+	for (i = 0; i < num_regs; i++) {
+		pruss_intc_write_reg(intc, PRU_INTC_ESR(i), sysevts[i]);
+		pruss_intc_write_reg(intc, PRU_INTC_SECR(i), sysevts[i]);
+	}
+
+	/* enable host interrupts */
+	for (i = 0; i < num_intrs; i++) {
+		if (host_mask & BIT(i))
+			pruss_intc_write_reg(intc, PRU_INTC_HIEISR, i);
+	}
+
+	/* global interrupt enable */
+	pruss_intc_write_reg(intc, PRU_INTC_GER, 1);
+
+	intc->host_mask |= host_mask;
+
+unlock:
+	mutex_unlock(&intc->lock);
+	bitmap_free(sysevt_bitmap);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(pruss_intc_configure);
+
+/**
+ * pruss_intc_unconfigure() - unconfigure the PRUSS INTC
+ * @pruss: the pruss instance
+ * @intc_config: PRU core specific INTC configuration
+ *
+ * Undo whatever was done in pruss_intc_configure() for a PRU core.
+ * It should be sufficient to just mark the resources free in the
+ * global map and disable the host interrupts and sysevents.
+ */
+int pruss_intc_unconfigure(struct pruss *pruss,
+			   struct pruss_intc_config *intc_config)
+{
+	struct device *dev = pruss->dev;
+	struct pruss_intc *intc;
+	int i;
+	s8 ch, host;
+	u32 num_events, num_intrs, num_regs;
+	u32 host_mask = 0;
+	unsigned long *sysevt_bitmap;
+	u32 *sysevts;
+
+	intc = to_pruss_intc(pruss);
+	if (!intc)
+		return -EINVAL;
+
+	num_events = intc->data->num_system_events;
+	num_intrs = intc->data->num_host_intrs;
+	num_regs = DIV_ROUND_UP(num_events, 32);
+
+	sysevt_bitmap = bitmap_zalloc(num_events, GFP_KERNEL);
+	if (!sysevt_bitmap)
+		return -ENOMEM;
+	sysevts = (u32 *)sysevt_bitmap;
+
+	mutex_lock(&intc->lock);
+
+	for (i = 0; i < num_events; i++) {
+		ch = intc_config->sysev_to_ch[i];
+		if (ch < 0)
+			continue;
+
+		/* mark sysevent free in global map */
+		intc->config_map.sysev_to_ch[i] = -1;
+		bitmap_set(sysevt_bitmap, i, 1);
+	}
+
+	for (i = 0; i < num_intrs; i++) {
+		host = intc_config->ch_to_host[i];
+		if (host < 0)
+			continue;
+
+		/* mark channel free in global map */
+		intc->config_map.ch_to_host[i] = -1;
+		host_mask |= BIT(host);
+	}
+
+	dev_info(dev, "unconfigured system_events[%d-0] = %*pb\n",
+		 num_events - 1, num_events, sysevt_bitmap);
+	dev_info(dev, "unconfigured host_intr = 0x%08x\n", host_mask);
+
+	for (i = 0; i < num_regs; i++) {
+		/* disable system events, writing 0 has no-effect */
+		pruss_intc_write_reg(intc, PRU_INTC_ECR(i), sysevts[i]);
+		/* clear any pending status */
+		pruss_intc_write_reg(intc, PRU_INTC_SECR(i), sysevts[i]);
+	}
+
+	/* disable host interrupts */
+	for (i = 0; i < num_intrs; i++) {
+		if (host_mask & BIT(i))
+			pruss_intc_write_reg(intc, PRU_INTC_HIDISR, i);
+	}
+
+	intc->host_mask &= ~host_mask;
+	mutex_unlock(&intc->lock);
+	bitmap_free(sysevt_bitmap);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(pruss_intc_unconfigure);
+
+static void pruss_intc_init(struct pruss_intc *intc)
+{
+	int i;
+	int num_chnl_map_regs = DIV_ROUND_UP(intc->data->num_system_events,
+					     CMR_EVT_PER_REG);
+	int num_host_intr_regs = DIV_ROUND_UP(intc->data->num_host_intrs,
+					      HMR_CH_PER_REG);
+	int num_event_type_regs =
+			DIV_ROUND_UP(intc->data->num_system_events, 32);
+
+	/*
+	 * configure polarity (SIPR register) to active high and
+	 * type (SITR register) to pulse interrupt for all system events
+	 */
+	for (i = 0; i < num_event_type_regs; i++) {
+		pruss_intc_write_reg(intc, PRU_INTC_SIPR(i), 0xffffffff);
+		pruss_intc_write_reg(intc, PRU_INTC_SITR(i), 0);
+	}
+
+	/* clear all interrupt channel map registers, 4 events per register */
+	for (i = 0; i < num_chnl_map_regs; i++)
+		pruss_intc_write_reg(intc, PRU_INTC_CMR(i), 0);
+
+	/* clear all host interrupt map registers, 4 channels per register */
+	for (i = 0; i < num_host_intr_regs; i++)
+		pruss_intc_write_reg(intc, PRU_INTC_HMR(i), 0);
+}
+
+static void pruss_intc_irq_ack(struct irq_data *data)
+{
+	struct pruss_intc *intc = irq_data_get_irq_chip_data(data);
+	unsigned int hwirq = data->hwirq;
+
+	pruss_intc_check_write(intc, PRU_INTC_SICR, hwirq);
+}
+
+static void pruss_intc_irq_mask(struct irq_data *data)
+{
+	struct pruss_intc *intc = irq_data_get_irq_chip_data(data);
+	unsigned int hwirq = data->hwirq;
+
+	pruss_intc_check_write(intc, PRU_INTC_EICR, hwirq);
+}
+
+static void pruss_intc_irq_unmask(struct irq_data *data)
+{
+	struct pruss_intc *intc = irq_data_get_irq_chip_data(data);
+	unsigned int hwirq = data->hwirq;
+
+	pruss_intc_check_write(intc, PRU_INTC_EISR, hwirq);
+}
+
+static int pruss_intc_irq_retrigger(struct irq_data *data)
+{
+	struct pruss_intc *intc = irq_data_get_irq_chip_data(data);
+	unsigned int hwirq = data->hwirq;
+
+	return pruss_intc_check_write(intc, PRU_INTC_SISR, hwirq);
+}
+
+static int pruss_intc_irq_reqres(struct irq_data *data)
+{
+	if (!try_module_get(THIS_MODULE))
+		return -ENODEV;
+
+	return 0;
+}
+
+static void pruss_intc_irq_relres(struct irq_data *data)
+{
+	module_put(THIS_MODULE);
+}
+
+#ifdef CONFIG_SMP
+static int pruss_intc_irq_set_affinity(struct irq_data *data,
+				       const struct cpumask *mask_val,
+				       bool force)
+{
+	struct pruss_intc *intc = irq_data_get_irq_chip_data(data);
+	u32 ch, host;
+	s8 sch, shost;
+	unsigned int pirq;
+	struct irq_chip *pchip;
+	struct irq_data *pdata;
+	struct cpumask *eff_mask;
+	int ret;
+
+	/* check for stored channel & host config for this event */
+	sch = intc->config_map.sysev_to_ch[data->hwirq];
+	shost = sch != -1 ? intc->config_map.ch_to_host[sch] : -1;
+	if (sch == -1 || shost == -1) {
+		pr_err("%s: event %lu not configured: ch = %d, host = %d\n",
+		       __func__, data->hwirq, sch, shost);
+		return -EINVAL;
+	}
+
+	/* find programmed channel */
+	ch = pruss_intc_read_reg(intc,
+				 PRU_INTC_CMR(data->hwirq / CMR_EVT_PER_REG));
+	ch >>= (data->hwirq % CMR_EVT_PER_REG) * CMR_EVT_MAP_BITS;
+	ch &= CMR_EVT_MAP_MASK;
+
+	/* find programmed host interrupt */
+	host = pruss_intc_read_reg(intc, PRU_INTC_HMR(ch / HMR_CH_PER_REG));
+	host >>= (ch % HMR_CH_PER_REG) * HMR_CH_MAP_BITS;
+	host &= HMR_CH_MAP_MASK;
+
+	/* check programmed configuration for sanity */
+	if (ch != sch || host != shost) {
+		pr_err("%s: event %lu has mismatched configuration, ch = %d, host = %d\n",
+		       __func__, data->hwirq, sch, shost);
+		return -EINVAL;
+	}
+
+	/* program affinity using parent GIC irqchip and irqdata */
+	pirq = intc->irqs[host - MIN_PRU_HOST_INT];
+	pchip = irq_get_chip(pirq);
+	pdata = irq_get_irq_data(pirq);
+
+	if (pchip && pchip->irq_set_affinity) {
+		ret = pchip->irq_set_affinity(pdata, mask_val, force);
+		if (ret >= 0) {
+			eff_mask = irq_data_get_effective_affinity_mask(pdata);
+			irq_data_update_effective_affinity(data, eff_mask);
+		}
+
+		return ret;
+	}
+
+	return -EINVAL;
+}
+#endif
+
+/**
+ * pruss_intc_trigger() - trigger a PRU system event
+ * @irq: linux IRQ number associated with a PRU system event
+ *
+ * Trigger an interrupt by signalling a specific PRU system event.
+ * This can be used by PRUSS client users to raise/send an event to
+ * a PRU or any other core that is listening on the host interrupt
+ * mapped to that specific PRU system event. The @irq variable is the
+ * Linux IRQ number associated with a specific PRU system event that
+ * a client user/application uses. The interrupt mappings for this is
+ * provided by the PRUSS INTC irqchip instance.
+ *
+ * Returns 0 on success, or an error value upon failure.
+ */
+int pruss_intc_trigger(unsigned int irq)
+{
+	struct irq_desc *desc;
+
+	if (irq <= 0)
+		return -EINVAL;
+
+	desc = irq_to_desc(irq);
+	if (!desc)
+		return -EINVAL;
+
+	pruss_intc_irq_retrigger(&desc->irq_data);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(pruss_intc_trigger);
+
+static int pruss_intc_irq_domain_map(struct irq_domain *d, unsigned int virq,
+				     irq_hw_number_t hw)
+{
+	struct pruss_intc *intc = d->host_data;
+
+	irq_set_chip_data(virq, intc);
+	irq_set_chip_and_handler(virq, intc->irqchip, handle_level_irq);
+
+	return 0;
+}
+
+static void pruss_intc_irq_domain_unmap(struct irq_domain *d, unsigned int virq)
+{
+	irq_set_chip_and_handler(virq, NULL, NULL);
+	irq_set_chip_data(virq, NULL);
+}
+
+static const struct irq_domain_ops pruss_intc_irq_domain_ops = {
+	.xlate	= irq_domain_xlate_onecell,
+	.map	= pruss_intc_irq_domain_map,
+	.unmap	= pruss_intc_irq_domain_unmap,
+};
+
+static void pruss_intc_irq_handler(struct irq_desc *desc)
+{
+	unsigned int irq = irq_desc_get_irq(desc);
+	struct irq_chip *chip = irq_desc_get_chip(desc);
+	struct pruss_intc *intc = irq_get_handler_data(irq);
+	u32 hipir;
+	unsigned int virq;
+	int i, hwirq;
+
+	chained_irq_enter(chip, desc);
+
+	/* find our host irq number */
+	for (i = 0; i < MAX_HOST_NUM_IRQS; i++)
+		if (intc->irqs[i] == irq)
+			break;
+	if (i == MAX_HOST_NUM_IRQS)
+		goto err;
+
+	i += MIN_PRU_HOST_INT;
+
+	/* get highest priority pending PRUSS system event */
+	hipir = pruss_intc_read_reg(intc, PRU_INTC_HIPIR(i));
+	while (!(hipir & BIT(31))) {
+		hwirq = hipir & GENMASK(9, 0);
+		virq = irq_linear_revmap(intc->domain, hwirq);
+
+		/*
+		 * XXX: manually ACK any system events that do not have a
+		 * handler mapped yet
+		 */
+		if (unlikely(!virq))
+			pruss_intc_check_write(intc, PRU_INTC_SICR, hwirq);
+		else
+			generic_handle_irq(virq);
+
+		/* get next system event */
+		hipir = pruss_intc_read_reg(intc, PRU_INTC_HIPIR(i));
+	}
+err:
+	chained_irq_exit(chip, desc);
+}
+
+static int pruss_intc_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct platform_device *ppdev = to_platform_device(dev->parent);
+	struct pruss_intc *intc;
+	struct resource *res;
+	struct irq_chip *irqchip;
+	int i, irq;
+	const struct pruss_intc_match_data *data;
+	bool skip_host7;
+	u8 max_system_events;
+
+	data = of_device_get_match_data(dev);
+	if (!data)
+		return -ENODEV;
+
+	skip_host7 = data->no_host7_intr;
+	max_system_events = data->num_system_events;
+
+	intc = devm_kzalloc(dev, sizeof(*intc), GFP_KERNEL);
+	if (!intc)
+		return -ENOMEM;
+	intc->data = data;
+	platform_set_drvdata(pdev, intc);
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	intc->base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(intc->base)) {
+		dev_err(dev, "failed to parse and map intc memory resource\n");
+		return PTR_ERR(intc->base);
+	}
+
+	dev_dbg(dev, "intc memory: pa %pa size 0x%zx va %pK\n", &res->start,
+		(size_t)resource_size(res), intc->base);
+
+	mutex_init(&intc->lock);
+
+	for (i = 0; i < ARRAY_SIZE(intc->config_map.sysev_to_ch); i++)
+		intc->config_map.sysev_to_ch[i] = -1;
+
+	for (i = 0; i < ARRAY_SIZE(intc->config_map.ch_to_host); i++)
+		intc->config_map.ch_to_host[i] = -1;
+
+	intc->pruss = platform_get_drvdata(ppdev);
+	pruss_intc_init(intc);
+
+	irqchip = devm_kzalloc(dev, sizeof(*irqchip), GFP_KERNEL);
+	if (!irqchip)
+		return -ENOMEM;
+
+	irqchip->irq_ack = pruss_intc_irq_ack;
+	irqchip->irq_mask = pruss_intc_irq_mask;
+	irqchip->irq_unmask = pruss_intc_irq_unmask;
+	irqchip->irq_retrigger = pruss_intc_irq_retrigger;
+	irqchip->irq_request_resources = pruss_intc_irq_reqres;
+	irqchip->irq_release_resources = pruss_intc_irq_relres;
+#ifdef CONFIG_SMP
+	irqchip->irq_set_affinity = pruss_intc_irq_set_affinity;
+#endif
+	irqchip->name = dev_name(dev);
+	intc->irqchip = irqchip;
+
+	intc->domain = irq_domain_add_linear(dev->of_node, max_system_events,
+					     &pruss_intc_irq_domain_ops, intc);
+	if (!intc->domain)
+		return -ENOMEM;
+
+	for (i = 0; i < MAX_HOST_NUM_IRQS; i++) {
+		irq = platform_get_irq_byname(ppdev, irq_names[i]);
+		if (irq < 0) {
+			if (!strcmp(irq_names[i], "host7") && !!skip_host7)
+				continue;
+
+			dev_err(dev->parent, "platform_get_irq_byname failed for %s : %d\n",
+				irq_names[i], irq);
+			goto fail_irq;
+		}
+
+		intc->irqs[i] = irq;
+		irq_set_handler_data(irq, intc);
+		irq_set_chained_handler(irq, pruss_intc_irq_handler);
+	}
+
+	return 0;
+
+fail_irq:
+	while (--i >= 0) {
+		if (intc->irqs[i])
+			irq_set_chained_handler_and_data(intc->irqs[i], NULL,
+							 NULL);
+	}
+	irq_domain_remove(intc->domain);
+	return irq;
+}
+
+static int pruss_intc_remove(struct platform_device *pdev)
+{
+	struct pruss_intc *intc = platform_get_drvdata(pdev);
+	u8 max_system_events = intc->data->num_system_events;
+	unsigned int hwirq;
+	int i;
+
+	for (i = 0; i < MAX_HOST_NUM_IRQS; i++) {
+		if (intc->irqs[i])
+			irq_set_chained_handler_and_data(intc->irqs[i], NULL,
+							 NULL);
+	}
+
+	if (intc->domain) {
+		for (hwirq = 0; hwirq < max_system_events; hwirq++)
+			irq_dispose_mapping(irq_find_mapping(intc->domain,
+							     hwirq));
+		irq_domain_remove(intc->domain);
+	}
+
+	return 0;
+}
+
+static const struct pruss_intc_match_data am335x_am57xx_pruss_intc_data = {
+	.num_system_events = 64,
+	.num_host_intrs = 10,
+	.no_host7_intr = false,
+};
+
+static const struct pruss_intc_match_data am437x_k2g_pruss_intc_data = {
+	.num_system_events = 64,
+	.num_host_intrs = 10,
+	.no_host7_intr = true,
+};
+
+static const struct pruss_intc_match_data am65x_j721e_icssg_intc_data = {
+	.num_system_events = 160,
+	.num_host_intrs = 20,
+	.no_host7_intr = false,
+};
+
+static const struct of_device_id pruss_intc_of_match[] = {
+	{
+		.compatible = "ti,am3356-pruss-intc",
+		.data = &am335x_am57xx_pruss_intc_data,
+	},
+	{
+		.compatible = "ti,am4376-pruss-intc",
+		.data = &am437x_k2g_pruss_intc_data,
+	},
+	{
+		.compatible = "ti,am5728-pruss-intc",
+		.data = &am335x_am57xx_pruss_intc_data,
+	},
+	{
+		.compatible = "ti,k2g-pruss-intc",
+		.data = &am437x_k2g_pruss_intc_data,
+	},
+	{
+		.compatible = "ti,am654-icssg-intc",
+		.data = &am65x_j721e_icssg_intc_data,
+	},
+	{
+		.compatible = "ti,j721e-icssg-intc",
+		.data = &am65x_j721e_icssg_intc_data,
+	},
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, pruss_intc_of_match);
+
+static struct platform_driver pruss_intc_driver = {
+	.driver = {
+		.name = "pruss-intc",
+		.of_match_table = pruss_intc_of_match,
+	},
+	.probe  = pruss_intc_probe,
+	.remove = pruss_intc_remove,
+};
+module_platform_driver(pruss_intc_driver);
+
+MODULE_AUTHOR("Andrew F. Davis <afd@ti.com>");
+MODULE_AUTHOR("Suman Anna <s-anna@ti.com>");
+MODULE_DESCRIPTION("PRU-ICSS INTC Driver");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/irqchip/irq-ti-sci-inta.c b/drivers/irqchip/irq-ti-sci-inta.c
--- a/drivers/irqchip/irq-ti-sci-inta.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/irqchip/irq-ti-sci-inta.c	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,613 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Texas Instruments' K3 Interrupt Aggregator irqchip driver
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ *	Lokesh Vutla <lokeshvutla@ti.com>
+ */
+
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/irqchip.h>
+#include <linux/of_platform.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/irqdomain.h>
+#include <linux/soc/ti/ti_sci_protocol.h>
+
+#define MAX_EVENTS_PER_VINT	64
+#define TI_SCI_EVENT_IRQ	BIT(31)
+
+#define VINT_ENABLE_CLR_OFFSET	0x18
+
+/**
+ * struct ti_sci_inta_irq_domain - Structure representing a TISCI based
+ *				   Interrupt Aggregator IRQ domain.
+ * @sci:	Pointer to TISCI handle
+ * @vint:	TISCI resource pointer representing IA inerrupts.
+ * @global_event:TISCI resource pointer representing global events.
+ * @base:	Base address of the memory mapped IO registers
+ * @ia_id:	TISCI device ID of this Interrupt Aggregator.
+ * @dst_id:	TISCI device ID of the destination irq controller.
+ */
+struct ti_sci_inta_irq_domain {
+	const struct ti_sci_handle *sci;
+	struct ti_sci_resource *vint;
+	struct ti_sci_resource *global_event;
+	void __iomem *base;
+	u16 ia_id;
+	u16 dst_id;
+};
+
+/**
+ * struct ti_sci_inta_event_desc - Description of an event coming to
+ *				   Interrupt Aggregator.
+ * @global_event:	Global event number corresponding to this event
+ * @src_id:		TISCI device ID of the event source
+ * @src_index:		Event source index within the device.
+ */
+struct ti_sci_inta_event_desc {
+	u16 global_event;
+	u16 src_id;
+	u16 src_index;
+};
+
+/**
+ * struct ti_sci_inta_vint_desc - Description of a virtual interrupt coming out
+ *				  of Interrupt Aggregator.
+ * @lock:		lock to guard the event map
+ * @event_map:		Bitmap to manage the allocation of events to vint.
+ * @events:		Array of event descriptors assigned to this vint.
+ * @ack_needed:		Event needs to be acked via INTA. This is used when
+ *			HW generating events cannot clear the events by itself.
+ *			Assuming that only events from the same hw block are
+ *			grouped. So all the events attached to vint needs
+ *			an ack or none needs an ack.
+ */
+struct ti_sci_inta_vint_desc {
+	raw_spinlock_t lock;
+	unsigned long *event_map;
+	struct ti_sci_inta_event_desc events[MAX_EVENTS_PER_VINT];
+	bool ack_needed;
+};
+
+static void ti_sci_inta_irq_eoi(struct irq_data *data)
+{
+	struct ti_sci_inta_irq_domain *inta = data->domain->host_data;
+	struct ti_sci_inta_vint_desc *vint_desc;
+	u64 val;
+	int bit;
+
+	vint_desc = irq_data_get_irq_chip_data(data);
+	if (!vint_desc->ack_needed)
+		goto out;
+
+	for_each_set_bit(bit, vint_desc->event_map, MAX_EVENTS_PER_VINT) {
+		val = 1 << bit;
+		__raw_writeq(val, inta->base + data->hwirq * 0x1000 +
+			     VINT_ENABLE_CLR_OFFSET);
+	}
+
+out:
+	irq_chip_eoi_parent(data);
+}
+
+static struct irq_chip ti_sci_inta_irq_chip = {
+	.name			= "INTA",
+	.irq_eoi		= ti_sci_inta_irq_eoi,
+	.irq_mask		= irq_chip_mask_parent,
+	.irq_unmask		= irq_chip_unmask_parent,
+	.irq_retrigger		= irq_chip_retrigger_hierarchy,
+	.irq_set_type		= irq_chip_set_type_parent,
+	.irq_set_affinity	= irq_chip_set_affinity_parent,
+};
+
+/**
+ * ti_sci_inta_irq_domain_translate() - Retrieve hwirq and type from
+ *					IRQ firmware specific handler.
+ * @domain:	Pointer to IRQ domain
+ * @fwspec:	Pointer to IRQ specific firmware structure
+ * @hwirq:	IRQ number identified by hardware
+ * @type:	IRQ type
+ *
+ * Return 0 if all went ok else appropriate error.
+ */
+static int ti_sci_inta_irq_domain_translate(struct irq_domain *domain,
+					    struct irq_fwspec *fwspec,
+					    unsigned long *hwirq,
+					    unsigned int *type)
+{
+	if (is_of_node(fwspec->fwnode)) {
+		if (fwspec->param_count != 4)
+			return -EINVAL;
+
+		*hwirq = fwspec->param[2];
+		*type = fwspec->param[3] & IRQ_TYPE_SENSE_MASK;
+
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+/**
+ * ti_sci_free_event_irq() - Free an event from vint
+ * @inta:	Pointer to Interrupt Aggregator IRQ domain
+ * @vint_desc:	Virtual interrupt descriptor containing the event.
+ * @event_index:Event Index within the vint.
+ * @dst_irq:	Destination host irq
+ * @vint:	Interrupt number within interrupt aggregator.
+ */
+static void ti_sci_free_event_irq(struct ti_sci_inta_irq_domain *inta,
+				  struct ti_sci_inta_vint_desc *vint_desc,
+				  u32 event_index, u16 dst_irq, u16 vint)
+{
+	struct ti_sci_inta_event_desc *event;
+	unsigned long flags;
+
+	if (event_index >= MAX_EVENTS_PER_VINT)
+		return;
+
+	event = &vint_desc->events[event_index];
+	inta->sci->ops.rm_irq_ops.free_event_irq(inta->sci,
+						 event->src_id,
+						 event->src_index,
+						 inta->dst_id,
+						 dst_irq,
+						 inta->ia_id, vint,
+						 event->global_event,
+						 event_index);
+
+	raw_spin_lock_irqsave(&vint_desc->lock, flags);
+	clear_bit(event_index, vint_desc->event_map);
+	raw_spin_unlock_irqrestore(&vint_desc->lock, flags);
+
+	ti_sci_release_resource(inta->global_event, event->global_event);
+}
+
+/**
+ * ti_sci_inta_irq_domain_free() - Free an IRQ from the IRQ domain
+ * @domain:	Domain to which the irqs belong
+ * @virq:	base linux virtual IRQ to be freed.
+ * @nr_irqs:	Number of continuous irqs to be freed
+ */
+static void ti_sci_inta_irq_domain_free(struct irq_domain *domain,
+					unsigned int virq, unsigned int nr_irqs)
+{
+	struct ti_sci_inta_irq_domain *inta = domain->host_data;
+	struct ti_sci_inta_vint_desc *vint_desc;
+	struct irq_data *data, *gic_data;
+	int event_index;
+
+	data = irq_domain_get_irq_data(domain, virq);
+	gic_data = irq_domain_get_irq_data(domain->parent->parent, virq);
+
+	vint_desc = irq_data_get_irq_chip_data(data);
+
+	/* This is the last event in the vint */
+	event_index = find_first_bit(vint_desc->event_map, MAX_EVENTS_PER_VINT);
+	ti_sci_free_event_irq(inta, vint_desc, event_index,
+			      gic_data->hwirq, data->hwirq);
+	irq_domain_free_irqs_parent(domain, virq, 1);
+	ti_sci_release_resource(inta->vint, data->hwirq);
+	irq_domain_reset_irq_data(data);
+	kfree(vint_desc->event_map);
+	kfree(vint_desc);
+}
+
+/**
+ * ti_sci_allocate_event_irq() - Allocate an event to a IA vint.
+ * @inta:	Pointer to Interrupt Aggregator IRQ domain
+ * @vint_desc:	Virtual interrupt descriptor to which the event gets attached.
+ * @src_id:	TISCI device id of the event source
+ * @src_index:	Event index with in the device.
+ * @dst_irq:	Destination host irq
+ * @vint:	Interrupt number within interrupt aggregator.
+ *
+ * Return 0 if all went ok else appropriate error value.
+ */
+static int ti_sci_allocate_event_irq(struct ti_sci_inta_irq_domain *inta,
+				     struct ti_sci_inta_vint_desc *vint_desc,
+				     u16 src_id, u16 src_index, u16 dst_irq,
+				     u16 vint)
+{
+	struct ti_sci_inta_event_desc *event;
+	unsigned long flags;
+	u32 free_bit;
+	int err;
+
+	raw_spin_lock_irqsave(&vint_desc->lock, flags);
+	free_bit = find_first_zero_bit(vint_desc->event_map,
+				       MAX_EVENTS_PER_VINT);
+	if (free_bit != MAX_EVENTS_PER_VINT)
+		set_bit(free_bit, vint_desc->event_map);
+	raw_spin_unlock_irqrestore(&vint_desc->lock, flags);
+
+	if (free_bit >= MAX_EVENTS_PER_VINT)
+		return -ENODEV;
+
+	event = &vint_desc->events[free_bit];
+
+	event->src_id = src_id;
+	event->src_index = src_index;
+	event->global_event = ti_sci_get_free_resource(inta->global_event);
+
+	err = inta->sci->ops.rm_irq_ops.set_event_irq(inta->sci,
+						      src_id, src_index,
+						      inta->dst_id,
+						      dst_irq,
+						      inta->ia_id,
+						      vint,
+						      event->global_event,
+						      free_bit);
+	if (err) {
+		pr_err("%s: Event allocation failed from src = %d, index = %d, to dst = %d,irq = %d,via ia_id = %d, vint = %d,global event = %d, status_bit = %d\n",
+		       __func__, src_id, src_index, inta->dst_id, dst_irq,
+		       inta->ia_id, vint, event->global_event, free_bit);
+		return err;
+	}
+
+	return 0;
+}
+
+/**
+ * alloc_parent_irq() - Allocate parent irq to Interrupt aggregator
+ * @domain:	IRQ domain corresponding to Interrupt Aggregator
+ * @virq:	Linux virtual IRQ number
+ * @src_id:	TISCI device id of the event source
+ * @src_index:	Event index with in the device.
+ * @vint:	Virtual interrupt number within IA
+ * @flags:	Corresponding IRQ flags
+ *
+ * Return pointer to vint descriptor if all went well else corresponding
+ * error pointer.
+ */
+static struct ti_sci_inta_vint_desc *alloc_parent_irq(struct irq_domain *domain,
+						      unsigned int virq,
+						      u32 src_id, u32 src_index,
+						      u32 vint, u32 flags)
+{
+	struct ti_sci_inta_irq_domain *inta = domain->host_data;
+	struct ti_sci_inta_vint_desc *vint_desc;
+	struct irq_data *gic_data;
+	struct irq_fwspec fwspec;
+	int err;
+
+	if (!irq_domain_get_of_node(domain->parent))
+		return ERR_PTR(-EINVAL);
+
+	vint_desc = kzalloc(sizeof(*vint_desc), GFP_KERNEL);
+	if (!vint_desc)
+		return ERR_PTR(-ENOMEM);
+
+	vint_desc->event_map = kcalloc(BITS_TO_LONGS(MAX_EVENTS_PER_VINT),
+				       sizeof(*vint_desc->event_map),
+				       GFP_KERNEL);
+	if (!vint_desc->event_map) {
+		kfree(vint_desc);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	fwspec.fwnode = domain->parent->fwnode;
+	fwspec.param_count = 3;
+	/* Interrupt parent is Interrupt Router */
+	fwspec.param[0] = inta->ia_id;
+	fwspec.param[1] = vint;
+	fwspec.param[2] = flags | TI_SCI_EVENT_IRQ;
+
+	err = irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec);
+	if (err)
+		goto err_irqs;
+
+	gic_data = irq_domain_get_irq_data(domain->parent->parent, virq);
+
+	raw_spin_lock_init(&vint_desc->lock);
+
+	err = ti_sci_allocate_event_irq(inta, vint_desc, src_id, src_index,
+					gic_data->hwirq, vint);
+	if (err)
+		goto err_events;
+
+	return vint_desc;
+
+err_events:
+	irq_domain_free_irqs_parent(domain, virq, 1);
+err_irqs:
+	ti_sci_release_resource(inta->vint, vint);
+	kfree(vint_desc);
+	return ERR_PTR(err);
+}
+
+/**
+ * ti_sci_inta_irq_domain_alloc() - Allocate Interrupt aggregator IRQs
+ * @domain:	Point to the interrupt aggregator IRQ domain
+ * @virq:	Corresponding Linux virtual IRQ number
+ * @nr_irqs:	Continuous irqs to be allocated
+ * @data:	Pointer to firmware specifier
+ *
+ * Return 0 if all went well else appropriate error value.
+ */
+static int ti_sci_inta_irq_domain_alloc(struct irq_domain *domain,
+					unsigned int virq, unsigned int nr_irqs,
+					void *data)
+{
+	struct ti_sci_inta_vint_desc *vint_desc;
+	struct irq_fwspec *fwspec = data;
+	int err;
+
+	vint_desc = alloc_parent_irq(domain, virq, fwspec->param[0],
+				     fwspec->param[1], fwspec->param[2],
+				     fwspec->param[3]);
+	if (IS_ERR(vint_desc))
+		return PTR_ERR(vint_desc);
+
+	err = irq_domain_set_hwirq_and_chip(domain, virq, fwspec->param[2],
+					    &ti_sci_inta_irq_chip, vint_desc);
+
+	return err;
+}
+
+static const struct irq_domain_ops ti_sci_inta_irq_domain_ops = {
+	.alloc		= ti_sci_inta_irq_domain_alloc,
+	.free		= ti_sci_inta_irq_domain_free,
+	.translate	= ti_sci_inta_irq_domain_translate,
+};
+
+static int ti_sci_inta_irq_domain_probe(struct platform_device *pdev)
+{
+	struct irq_domain *parent_domain, *domain;
+	struct ti_sci_inta_irq_domain *inta;
+	struct device_node *parent_node;
+	struct device *dev = &pdev->dev;
+	struct resource *res;
+	int ret;
+
+	parent_node = of_irq_find_parent(dev_of_node(dev));
+	if (!parent_node) {
+		dev_err(dev, "Failed to get IRQ parent node\n");
+		return -ENODEV;
+	}
+
+	parent_domain = irq_find_host(parent_node);
+	if (!parent_domain)
+		return -EPROBE_DEFER;
+
+	inta = devm_kzalloc(dev, sizeof(*inta), GFP_KERNEL);
+	if (!inta)
+		return -ENOMEM;
+
+	inta->sci = devm_ti_sci_get_by_phandle(dev, "ti,sci");
+	if (IS_ERR(inta->sci)) {
+		ret = PTR_ERR(inta->sci);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "ti,sci read fail %d\n", ret);
+		inta->sci = NULL;
+		return ret;
+	}
+
+	ret = of_property_read_u32(dev->of_node, "ti,sci-dev-id",
+				   (u32 *)&inta->ia_id);
+	if (ret) {
+		dev_err(dev, "missing 'ti,sci-dev-id' property\n");
+		return -EINVAL;
+	}
+
+	inta->vint = devm_ti_sci_get_of_resource(inta->sci, dev,
+						 inta->ia_id,
+						 "ti,sci-rm-range-vint");
+	if (IS_ERR(inta->vint)) {
+		dev_err(dev, "VINT resource allocation failed\n");
+		return PTR_ERR(inta->vint);
+	}
+
+	inta->global_event =
+		devm_ti_sci_get_of_resource(inta->sci, dev,
+					    inta->ia_id,
+					    "ti,sci-rm-range-global-event");
+	if (IS_ERR(inta->global_event)) {
+		dev_err(dev, "Global event resource allocation failed\n");
+		return PTR_ERR(inta->global_event);
+	}
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	inta->base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(inta->base))
+		return -ENODEV;
+
+	ret = of_property_read_u32(parent_node, "ti,sci-dst-id",
+				   (u32 *)&inta->dst_id);
+
+	domain = irq_domain_add_hierarchy(parent_domain, 0, 0, dev_of_node(dev),
+					  &ti_sci_inta_irq_domain_ops, inta);
+	if (!domain) {
+		dev_err(dev, "Failed to allocate IRQ domain\n");
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+/**
+ * ti_sci_inta_register_event() - Register a event to an interrupt aggregator
+ * @dev:	Device pointer to source generating the event
+ * @src_id:	TISCI device ID of the event source
+ * @src_index:	Event source index within the device.
+ * @virq:	Linux Virtual IRQ number
+ * @ack_needed:	If explicit clearing of event is required.
+ * @flags:	Corresponding IRQ flags
+ *
+ * Creates a new irq and attaches to IA domain if virq is not specified
+ * else attaches the event to vint corresponding to virq.
+ * When using TISCI within the client drivers, source indexes are always
+ * generated dynamically and cannot be represented in DT. So client
+ * drivers should call this API instead of platform_get_irq().
+ *
+ * Return virq if all went well else appropriate error value.
+ */
+int ti_sci_inta_register_event(struct device *dev, u16 src_id, u16 src_index,
+			       unsigned int virq, bool ack_needed, u32 flags)
+{
+	struct ti_sci_inta_vint_desc *vint_desc;
+	struct ti_sci_inta_irq_domain *inta;
+	struct irq_data *data, *gic_data;
+	struct device_node *parent_node;
+	struct irq_domain *domain;
+	struct irq_fwspec fwspec;
+	int err;
+
+	parent_node = of_irq_find_parent(dev->of_node);
+	if (!parent_node)
+		return -ENODEV;
+
+	domain = irq_find_host(parent_node);
+	if (!domain)
+		return -EPROBE_DEFER;
+
+	inta = domain->host_data;
+
+	if (virq > 0) {
+		/* If Group already available */
+		data = irq_domain_get_irq_data(domain, virq);
+		gic_data = irq_domain_get_irq_data(domain->parent->parent,
+						   virq);
+
+		vint_desc = irq_data_get_irq_chip_data(data);
+
+		err = ti_sci_allocate_event_irq(inta, vint_desc, src_id,
+						src_index, gic_data->hwirq,
+						data->hwirq);
+		if (err)
+			return err;
+	} else {
+		fwspec.param_count = 4;
+		fwspec.fwnode = domain->fwnode;
+		fwspec.param[0] = src_id;
+		fwspec.param[1] = src_index;
+		fwspec.param[2] = ti_sci_get_free_resource(inta->vint);
+		fwspec.param[3] = flags;
+
+		virq = irq_create_fwspec_mapping(&fwspec);
+		if (virq <= 0)
+			ti_sci_release_resource(inta->vint, fwspec.param[2]);
+
+		data = irq_domain_get_irq_data(domain, virq);
+		vint_desc = irq_data_get_irq_chip_data(data);
+		vint_desc->ack_needed = ack_needed;
+	}
+
+	return virq;
+}
+EXPORT_SYMBOL_GPL(ti_sci_inta_register_event);
+
+/**
+ * get_event_index() - Helper api to get the event index with the vint.
+ * @vint_desc:	Pointer to the IA irq descriptor
+ * @src_id:	TISCI device ID of the event source
+ * @src_index:	Event source index within the device.
+ *
+ * Return the index if all went well else MAX_EVENTS_PER_VINT.
+ */
+static u8 get_event_index(struct ti_sci_inta_vint_desc *vint_desc, u16 src_id,
+			  u16 src_index)
+{
+	int i;
+
+	for (i = 0; i < MAX_EVENTS_PER_VINT; i++) {
+		if (vint_desc->events[i].src_id == src_id &&
+		    vint_desc->events[i].src_index == src_index)
+			return i;
+	}
+
+	return MAX_EVENTS_PER_VINT;
+}
+
+/**
+ * is_event_last() -  Helper api to determine if the event is the last
+ *		      event within the vint
+ * @vint_desc:	Pointer to the IA irq descriptor
+ * @event_index:	Event index within the vint of IA.
+ *
+ * Return true if specified index is the last event else false.
+ */
+static bool is_event_last(struct ti_sci_inta_vint_desc *vint_desc,
+			  u8 event_index)
+{
+	unsigned long flags;
+	int next_event;
+	bool status;
+
+	raw_spin_lock_irqsave(&vint_desc->lock, flags);
+	next_event = find_next_bit(vint_desc->event_map, MAX_EVENTS_PER_VINT,
+				   event_index + 1);
+	status = (next_event != MAX_EVENTS_PER_VINT) ? false : true;
+	if (event_index !=
+		find_first_bit(vint_desc->event_map, MAX_EVENTS_PER_VINT))
+		status = false;
+	raw_spin_unlock_irqrestore(&vint_desc->lock, flags);
+
+	return status;
+}
+
+/**
+ * ti_sci_inta_unregister_event() - Unregister a event from an IA.
+ * @dev:	Device pointer to source generating the event
+ * @src_id:	TISCI device ID of the event source
+ * @src_index:	Event source index within the device.
+ * @virq:	Linux Virtual IRQ number
+ *
+ * Deletes the event from IA interrupt descriptor. If this event is the
+ * last event then delete the vint as well.
+ */
+void ti_sci_inta_unregister_event(struct device *dev, u16 src_id,
+				  u16 src_index, unsigned int virq)
+{
+	struct ti_sci_inta_vint_desc *vint_desc;
+	struct irq_data *data, *gic_data;
+	struct device_node *parent_node;
+	struct irq_domain *domain;
+	u8 event_index;
+
+	parent_node = of_irq_find_parent(dev->of_node);
+	if (!parent_node)
+		return;
+
+	domain = irq_find_host(parent_node);
+
+	data = irq_domain_get_irq_data(domain, virq);
+	gic_data = irq_domain_get_irq_data(domain->parent->parent, virq);
+
+	vint_desc = irq_data_get_irq_chip_data(data);
+
+	event_index = get_event_index(vint_desc, src_id, src_index);
+	if (event_index == MAX_EVENTS_PER_VINT)
+		return;
+
+	if (is_event_last(vint_desc, event_index))
+		irq_dispose_mapping(virq);
+	else
+		ti_sci_free_event_irq(domain->host_data, vint_desc,
+				      event_index, gic_data->hwirq,
+				      data->hwirq);
+}
+EXPORT_SYMBOL_GPL(ti_sci_inta_unregister_event);
+
+static const struct of_device_id ti_sci_inta_irq_domain_of_match[] = {
+	{ .compatible = "ti,sci-inta", },
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, ti_sci_inta_irq_domain_of_match);
+
+static struct platform_driver ti_sci_inta_irq_domain_driver = {
+	.probe = ti_sci_inta_irq_domain_probe,
+	.driver = {
+		.name = "ti-sci-inta",
+		.of_match_table = ti_sci_inta_irq_domain_of_match,
+	},
+};
+module_platform_driver(ti_sci_inta_irq_domain_driver);
+
+MODULE_AUTHOR("Lokesh Vutla <lokeshvutla@ticom>");
+MODULE_DESCRIPTION("K3 Interrupt Aggregator driver over TI SCI protocol");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/irqchip/irq-ti-sci-intr.c b/drivers/irqchip/irq-ti-sci-intr.c
--- a/drivers/irqchip/irq-ti-sci-intr.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/irqchip/irq-ti-sci-intr.c	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,302 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Texas Instruments' K3 Interrupt Router irqchip driver
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ *	Lokesh Vutla <lokeshvutla@ti.com>
+ */
+
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/irqchip.h>
+#include <linux/of_platform.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/irqdomain.h>
+#include <linux/soc/ti/ti_sci_protocol.h>
+
+#define TI_SCI_DEV_ID_MASK	0xffff
+#define TI_SCI_DEV_ID_SHIFT	16
+#define TI_SCI_IRQ_ID_MASK	0xffff
+#define TI_SCI_IRQ_ID_SHIFT	0
+#define TI_SCI_IS_EVENT_IRQ	BIT(31)
+
+#define HWIRQ_TO_DEVID(hwirq)	(((hwirq) >> (TI_SCI_DEV_ID_SHIFT)) & \
+				 (TI_SCI_DEV_ID_MASK))
+#define HWIRQ_TO_IRQID(hwirq)	((hwirq) & (TI_SCI_IRQ_ID_MASK))
+
+/**
+ * struct ti_sci_intr_irq_domain - Structure representing a TISCI based
+ *				   Interrupt Router IRQ domain.
+ * @sci:	Pointer to TISCI handle
+ * @dst_irq:	TISCI resource pointer representing destination irq controller.
+ * @dst_id:	TISCI device ID of the destination irq controller.
+ */
+struct ti_sci_intr_irq_domain {
+	const struct ti_sci_handle *sci;
+	struct ti_sci_resource *dst_irq;
+	u16 dst_id;
+};
+
+static struct irq_chip ti_sci_intr_irq_chip = {
+	.name			= "INTR",
+	.irq_eoi		= irq_chip_eoi_parent,
+	.irq_mask		= irq_chip_mask_parent,
+	.irq_unmask		= irq_chip_unmask_parent,
+	.irq_retrigger		= irq_chip_retrigger_hierarchy,
+	.irq_set_type		= irq_chip_set_type_parent,
+	.irq_set_affinity	= irq_chip_set_affinity_parent,
+};
+
+/**
+ * ti_sci_intr_irq_domain_translate() - Retrieve hwirq and type from
+ *					IRQ firmware specific handler.
+ * @domain:	Pointer to IRQ domain
+ * @fwspec:	Pointer to IRQ specific firmware structure
+ * @hwirq:	IRQ number identified by hardware
+ * @type:	IRQ type
+ *
+ * Return 0 if all went ok else appropriate error.
+ */
+static int ti_sci_intr_irq_domain_translate(struct irq_domain *domain,
+					    struct irq_fwspec *fwspec,
+					    unsigned long *hwirq,
+					    unsigned int *type)
+{
+	if (is_of_node(fwspec->fwnode)) {
+		if (fwspec->param_count != 3)
+			return -EINVAL;
+
+		*hwirq = ((fwspec->param[0] & TI_SCI_DEV_ID_MASK) <<
+			  TI_SCI_DEV_ID_SHIFT) |
+			 (fwspec->param[1] & TI_SCI_IRQ_ID_MASK);
+		*type = fwspec->param[2];
+
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+static inline void ti_sci_intr_delete_desc(struct ti_sci_intr_irq_domain *intr,
+					   u16 src_id, u16 src_index,
+					   u16 dst_irq)
+{
+	intr->sci->ops.rm_irq_ops.free_direct_irq(intr->sci, src_id, src_index,
+						  intr->dst_id, dst_irq);
+}
+
+/**
+ * ti_sci_intr_irq_domain_free() - Free the specified IRQs from the domain.
+ * @domain:	Domain to which the irqs belong
+ * @virq:	Linux virtual IRQ to be freed.
+ * @nr_irqs:	Number of continuous irqs to be freed
+ */
+static void ti_sci_intr_irq_domain_free(struct irq_domain *domain,
+					unsigned int virq, unsigned int nr_irqs)
+{
+	struct ti_sci_intr_irq_domain *intr = domain->host_data;
+	struct irq_data *data, *parent_data;
+	u32 flags;
+	int i;
+
+	intr = domain->host_data;
+
+	for (i = 0; i < nr_irqs; i++) {
+		data = irq_domain_get_irq_data(domain, virq + i);
+		flags = (u32)(u64)irq_data_get_irq_chip_data(data);
+		parent_data = irq_domain_get_irq_data(domain->parent, virq + i);
+
+		if (!(flags & TI_SCI_IS_EVENT_IRQ))
+			ti_sci_intr_delete_desc(intr,
+						HWIRQ_TO_DEVID(data->hwirq),
+						HWIRQ_TO_IRQID(data->hwirq),
+						parent_data->hwirq);
+		ti_sci_release_resource(intr->dst_irq, parent_data->hwirq);
+		irq_domain_free_irqs_parent(domain, virq + i, 1);
+		irq_domain_reset_irq_data(data);
+	}
+}
+
+/**
+ * allocate_gic_irq() - Allocate GIC specific IRQ
+ * @domain:	Point to the interrupt router IRQ domain
+ * @dev:	TISCI device IRQ generating the IRQ
+ * @irq:	IRQ offset within the device
+ * @flags:	Corresponding flags to the IRQ
+ *
+ * Returns 0 if all went well else appropriate error pointer.
+ */
+static int allocate_gic_irq(struct irq_domain *domain, unsigned int virq,
+			    u16 dev, u16 irq, u32 flags)
+{
+	struct ti_sci_intr_irq_domain *intr = domain->host_data;
+	struct irq_fwspec fwspec;
+	u16 dst_irq;
+	int err;
+
+	if (!irq_domain_get_of_node(domain->parent))
+		return -EINVAL;
+
+	dst_irq = ti_sci_get_free_resource(intr->dst_irq);
+
+	fwspec.fwnode = domain->parent->fwnode;
+	fwspec.param_count = 3;
+	fwspec.param[0] = 0;	/* SPI */
+	fwspec.param[1] = dst_irq - 32; /* SPI offset */
+	fwspec.param[2] = flags & IRQ_TYPE_SENSE_MASK;
+
+	err = irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec);
+	if (err)
+		goto err_irqs;
+
+	/* If event is requested then return */
+	if (flags & TI_SCI_IS_EVENT_IRQ)
+		return 0;
+
+	err = intr->sci->ops.rm_irq_ops.set_direct_irq(intr->sci, dev, irq,
+						       intr->dst_id, dst_irq);
+	if (err) {
+		pr_err("%s: IRQ allocation failed from src = %d, src_index = %d to dst_id = %d, dst_irq = %d",
+		       __func__, dev, irq, intr->dst_id, dst_irq);
+		goto err_msg;
+	}
+
+	return 0;
+
+err_msg:
+	irq_domain_free_irqs_parent(domain, virq, 1);
+err_irqs:
+	ti_sci_release_resource(intr->dst_irq, dst_irq);
+	return err;
+}
+
+/**
+ * ti_sci_intr_irq_domain_alloc() - Allocate Interrupt router IRQs
+ * @domain:	Point to the interrupt router IRQ domain
+ * @virq:	Corresponding Linux virtual IRQ number
+ * @nr_irqs:	Continuous irqs to be allocated
+ * @data:	Pointer to firmware specifier
+ *
+ * Return 0 if all went well else appropriate error value.
+ */
+static int ti_sci_intr_irq_domain_alloc(struct irq_domain *domain,
+					unsigned int virq, unsigned int nr_irqs,
+					void *data)
+{
+	struct irq_fwspec *fwspec = data;
+	u16 src_id, src_index;
+	unsigned long hwirq;
+	int i, err;
+	u32 type;
+
+	err = ti_sci_intr_irq_domain_translate(domain, fwspec, &hwirq, &type);
+	if (err)
+		return err;
+
+	src_id = HWIRQ_TO_DEVID(hwirq);
+	src_index = HWIRQ_TO_IRQID(hwirq);
+
+	for (i = 0; i < nr_irqs; i++) {
+		err = allocate_gic_irq(domain, virq + i, src_id, src_index + i,
+				       type);
+		if (err)
+			goto err_irq;
+
+		err = irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i,
+						    &ti_sci_intr_irq_chip,
+						    (void *)(u64)type);
+		if (err)
+			goto err_irq;
+	}
+
+	return 0;
+err_irq:
+	ti_sci_intr_irq_domain_free(domain, virq, i);
+	return err;
+}
+
+static const struct irq_domain_ops ti_sci_intr_irq_domain_ops = {
+	.alloc		= ti_sci_intr_irq_domain_alloc,
+	.free		= ti_sci_intr_irq_domain_free,
+	.translate	= ti_sci_intr_irq_domain_translate,
+};
+
+static int ti_sci_intr_irq_domain_probe(struct platform_device *pdev)
+{
+	struct irq_domain *parent_domain, *domain;
+	struct ti_sci_intr_irq_domain *intr;
+	struct device_node *parent_node;
+	struct device *dev = &pdev->dev;
+	int ret;
+
+	parent_node = of_irq_find_parent(dev_of_node(dev));
+	if (!parent_node) {
+		dev_err(dev, "Failed to get IRQ parent node\n");
+		return -ENODEV;
+	}
+
+	parent_domain = irq_find_host(parent_node);
+	if (!parent_domain) {
+		dev_err(dev, "Failed to find IRQ parent domain\n");
+		return -ENODEV;
+	}
+
+	intr = devm_kzalloc(dev, sizeof(*intr), GFP_KERNEL);
+	if (!intr)
+		return -ENOMEM;
+
+	intr->sci = devm_ti_sci_get_by_phandle(dev, "ti,sci");
+	if (IS_ERR(intr->sci)) {
+		ret = PTR_ERR(intr->sci);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "ti,sci read fail %d\n", ret);
+		intr->sci = NULL;
+		return ret;
+	}
+
+	ret = of_property_read_u32(dev_of_node(dev), "ti,sci-dst-id",
+				   (u32 *)&intr->dst_id);
+	if (ret) {
+		dev_err(dev, "missing 'ti,sci-dst-id' property\n");
+		return -EINVAL;
+	}
+
+	intr->dst_irq = devm_ti_sci_get_of_resource(intr->sci, dev,
+						    intr->dst_id,
+						    "ti,sci-rm-range-girq");
+	if (IS_ERR(intr->dst_irq)) {
+		dev_err(dev, "Destination irq resource allocation failed\n");
+		return PTR_ERR(intr->dst_irq);
+	}
+
+	domain = irq_domain_add_hierarchy(parent_domain, 0, 0, dev_of_node(dev),
+					  &ti_sci_intr_irq_domain_ops, intr);
+	if (!domain) {
+		dev_err(dev, "Failed to allocate IRQ domain\n");
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static const struct of_device_id ti_sci_intr_irq_domain_of_match[] = {
+	{ .compatible = "ti,sci-intr", },
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, ti_sci_intr_irq_domain_of_match);
+
+static struct platform_driver ti_sci_intr_irq_domain_driver = {
+	.probe = ti_sci_intr_irq_domain_probe,
+	.driver = {
+		.name = "ti-sci-intr",
+		.of_match_table = ti_sci_intr_irq_domain_of_match,
+	},
+};
+module_platform_driver(ti_sci_intr_irq_domain_driver);
+
+MODULE_AUTHOR("Lokesh Vutla <lokeshvutla@ticom>");
+MODULE_DESCRIPTION("K3 Interrupt Router driver over TI SCI protocol");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig
--- a/drivers/irqchip/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/irqchip/Kconfig	2020-06-19 21:00:31.412227715 -0400
@@ -371,6 +371,28 @@
 	  Power Domain Controller driver to manage and configure wakeup
 	  IRQs for Qualcomm Technologies Inc (QTI) mobile chips.
 
+config TI_SCI_INTR_IRQCHIP
+	bool
+	depends on TI_SCI_PROTOCOL && ARCH_K3
+	select IRQ_DOMAIN
+	select IRQ_DOMAIN_HIERARCHY
+	help
+	  This enables the irqchip driver support for K3 Interrupt router
+	  over TI System Control Interface available on some new TI's SoCs.
+	  If you wish to use interrupt router irq resources managed by the
+	  TI System Controller, say Y here. Otherwise, say N.
+
+config TI_SCI_INTA_IRQCHIP
+	bool
+	depends on TI_SCI_PROTOCOL && ARCH_K3
+	select IRQ_DOMAIN
+	select IRQ_DOMAIN_HIERARCHY
+	help
+	  This enables the irqchip driver support for K3 Interrupt aggregator
+	  over TI System Control Interface available on some new TI's SoCs.
+	  If you wish to use interrupt aggregator irq resources managed by the
+	  TI System Controller, say Y here. Otherwise, say N.
+
 endmenu
 
 config SIFIVE_PLIC
diff -Naur --no-dereference a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile
--- a/drivers/irqchip/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/irqchip/Makefile	2020-06-19 21:00:31.412227715 -0400
@@ -88,3 +88,6 @@
 obj-$(CONFIG_NDS32)			+= irq-ativic32.o
 obj-$(CONFIG_QCOM_PDC)			+= qcom-pdc.o
 obj-$(CONFIG_SIFIVE_PLIC)		+= irq-sifive-plic.o
+obj-$(CONFIG_TI_PRUSS)			+= irq-pruss-intc.o
+obj-$(CONFIG_TI_SCI_INTR_IRQCHIP)	+= irq-ti-sci-intr.o
+obj-$(CONFIG_TI_SCI_INTA_IRQCHIP)	+= irq-ti-sci-inta.o
diff -Naur --no-dereference a/drivers/Kconfig b/drivers/Kconfig
--- a/drivers/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/Kconfig	2020-06-19 21:00:31.408227715 -0400
@@ -155,6 +155,8 @@
 
 source "drivers/rpmsg/Kconfig"
 
+source "drivers/rpmsg-kdrv/Kconfig"
+
 source "drivers/soundwire/Kconfig"
 
 source "drivers/soc/Kconfig"
diff -Naur --no-dereference a/drivers/leds/led-class.c b/drivers/leds/led-class.c
--- a/drivers/leds/led-class.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/leds/led-class.c	2020-06-19 21:00:31.416227715 -0400
@@ -21,6 +21,7 @@
 #include <linux/spinlock.h>
 #include <linux/timer.h>
 #include <uapi/linux/uleds.h>
+#include <linux/of.h>
 #include "leds.h"
 
 static struct class *leds_class;
@@ -215,6 +216,80 @@
 
 static SIMPLE_DEV_PM_OPS(leds_class_dev_pm_ops, led_suspend, led_resume);
 
+/* find OF node for the given led_cdev */
+static struct device_node *find_led_of_node(struct led_classdev *led_cdev)
+{
+	struct device *led_dev = led_cdev->dev;
+	struct device_node *child;
+
+	for_each_child_of_node(led_dev->parent->of_node, child) {
+		if (of_property_match_string(child, "label", led_cdev->name) == 0)
+			return child;
+	}
+
+	return NULL;
+}
+
+static int led_match_led_node(struct device *led_dev, const void *data)
+{
+	struct led_classdev *led_cdev = dev_get_drvdata(led_dev);
+	const struct device_node *target_node = data;
+	struct device_node *led_node;
+
+	led_node = find_led_of_node(led_cdev);
+	if (!led_node)
+		return 0;
+
+	of_node_put(led_node);
+
+	return led_node == target_node ? 1 : 0;
+}
+
+/**
+ * of_led_get() - request a LED device via the LED framework
+ * @np: device node to get the LED device from
+ *
+ * Returns the LED device parsed from the phandle specified in the "leds"
+ * property of a device tree node or a negative error-code on failure.
+ */
+struct led_classdev *of_led_get(struct device_node *np)
+{
+	struct device *led_dev;
+	struct led_classdev *led_cdev;
+	struct device_node *led_node;
+
+	led_node = of_parse_phandle(np, "leds", 0);
+	if (!led_node)
+		return ERR_PTR(-ENODEV);
+
+	led_dev = class_find_device(leds_class, NULL, led_node,
+		led_match_led_node);
+	if (!led_dev) {
+		of_node_put(led_node);
+		return ERR_PTR(-EPROBE_DEFER);
+	}
+
+	of_node_put(led_node);
+
+	led_cdev = dev_get_drvdata(led_dev);
+
+	if (!try_module_get(led_cdev->dev->parent->driver->owner))
+		return ERR_PTR(-ENODEV);
+
+	return led_cdev;
+}
+EXPORT_SYMBOL_GPL(of_led_get);
+
+/**
+ * led_put() - release a LED device
+ * @led_cdev: LED device
+ */
+void led_put(struct led_classdev *led_cdev)
+{
+	module_put(led_cdev->dev->parent->driver->owner);
+}
+EXPORT_SYMBOL_GPL(led_put);
+
 static int match_name(struct device *dev, const void *data)
 {
 	if (!dev_name(dev))
diff -Naur --no-dereference a/drivers/mailbox/Kconfig b/drivers/mailbox/Kconfig
--- a/drivers/mailbox/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/mailbox/Kconfig	2020-06-19 21:00:31.416227715 -0400
@@ -43,7 +43,7 @@
 
 config OMAP2PLUS_MBOX
 	tristate "OMAP2+ Mailbox framework support"
-	depends on ARCH_OMAP2PLUS
+	depends on ARCH_OMAP2PLUS || ARCH_K3
 	help
 	  Mailbox implementation for OMAP family chips with hardware for
 	  interprocessor communication involving DSP, IVA1.0 and IVA2 in
@@ -105,12 +105,12 @@
 
 config TI_MESSAGE_MANAGER
 	tristate "Texas Instruments Message Manager Driver"
-	depends on ARCH_KEYSTONE
+	depends on ARCH_KEYSTONE || ARCH_K3
 	help
 	  An implementation of Message Manager slave driver for Keystone
-	  architecture SoCs from Texas Instruments. Message Manager is a
-	  communication entity found on few of Texas Instrument's keystone
-	  architecture SoCs. These may be used for communication between
+	  and K3 architecture SoCs from Texas Instruments. Message Manager
+	  is a communication entity found on few of Texas Instrument's keystone
+	  and K3 architecture SoCs. These may be used for communication between
 	  multiple processors within the SoC. Select this driver if your
 	  platform has support for the hardware block.
 
diff -Naur --no-dereference a/drivers/mailbox/omap-mailbox.c b/drivers/mailbox/omap-mailbox.c
--- a/drivers/mailbox/omap-mailbox.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/mailbox/omap-mailbox.c	2020-06-19 21:00:31.416227715 -0400
@@ -3,7 +3,7 @@
  * OMAP mailbox driver
  *
  * Copyright (C) 2006-2009 Nokia Corporation. All rights reserved.
- * Copyright (C) 2013-2016 Texas Instruments Incorporated - http://www.ti.com
+ * Copyright (C) 2013-2018 Texas Instruments Incorporated - http://www.ti.com
  *
  * Contact: Hiroshi DOYU <Hiroshi.DOYU@nokia.com>
  *          Suman Anna <s-anna@ti.com>
@@ -141,14 +141,14 @@
 }
 
 /* Mailbox FIFO handle functions */
-static mbox_msg_t mbox_fifo_read(struct omap_mbox *mbox)
+static u32 mbox_fifo_read(struct omap_mbox *mbox)
 {
 	struct omap_mbox_fifo *fifo = &mbox->rx_fifo;
 
-	return (mbox_msg_t)mbox_read_reg(mbox->parent, fifo->msg);
+	return mbox_read_reg(mbox->parent, fifo->msg);
 }
 
-static void mbox_fifo_write(struct omap_mbox *mbox, mbox_msg_t msg)
+static void mbox_fifo_write(struct omap_mbox *mbox, u32 msg)
 {
 	struct omap_mbox_fifo *fifo = &mbox->tx_fifo;
 
@@ -256,14 +256,16 @@
 {
 	struct omap_mbox_queue *mq =
 			container_of(work, struct omap_mbox_queue, work);
-	mbox_msg_t msg;
+	mbox_msg_t data;
+	u32 msg;
 	int len;
 
 	while (kfifo_len(&mq->fifo) >= sizeof(msg)) {
 		len = kfifo_out(&mq->fifo, (unsigned char *)&msg, sizeof(msg));
 		WARN_ON(len != sizeof(msg));
+		data = msg;
 
-		mbox_chan_received_data(mq->mbox->chan, (void *)msg);
+		mbox_chan_received_data(mq->mbox->chan, (void *)data);
 		spin_lock_irq(&mq->lock);
 		if (mq->full) {
 			mq->full = false;
@@ -286,7 +288,7 @@
 static void __mbox_rx_interrupt(struct omap_mbox *mbox)
 {
 	struct omap_mbox_queue *mq = mbox->rxq;
-	mbox_msg_t msg;
+	u32 msg;
 	int len;
 
 	while (!mbox_fifo_empty(mbox)) {
@@ -542,13 +544,13 @@
 	mutex_unlock(&mdev->cfg_lock);
 }
 
-static int omap_mbox_chan_send_noirq(struct omap_mbox *mbox, void *data)
+static int omap_mbox_chan_send_noirq(struct omap_mbox *mbox, u32 msg)
 {
 	int ret = -EBUSY;
 
 	if (!mbox_fifo_full(mbox)) {
 		_omap_mbox_enable_irq(mbox, IRQ_RX);
-		mbox_fifo_write(mbox, (mbox_msg_t)data);
+		mbox_fifo_write(mbox, msg);
 		ret = 0;
 		_omap_mbox_disable_irq(mbox, IRQ_RX);
 
@@ -560,12 +562,12 @@
 	return ret;
 }
 
-static int omap_mbox_chan_send(struct omap_mbox *mbox, void *data)
+static int omap_mbox_chan_send(struct omap_mbox *mbox, u32 msg)
 {
 	int ret = -EBUSY;
 
 	if (!mbox_fifo_full(mbox)) {
-		mbox_fifo_write(mbox, (mbox_msg_t)data);
+		mbox_fifo_write(mbox, msg);
 		ret = 0;
 	}
 
@@ -578,14 +580,15 @@
 {
 	struct omap_mbox *mbox = mbox_chan_to_omap_mbox(chan);
 	int ret;
+	u32 msg = to_omap_mbox_msg(data);
 
 	if (!mbox)
 		return -EINVAL;
 
 	if (mbox->send_no_irq)
-		ret = omap_mbox_chan_send_noirq(mbox, data);
+		ret = omap_mbox_chan_send_noirq(mbox, msg);
 	else
-		ret = omap_mbox_chan_send(mbox, data);
+		ret = omap_mbox_chan_send(mbox, msg);
 
 	return ret;
 }
@@ -659,6 +662,10 @@
 		.data		= &omap4_data,
 	},
 	{
+		.compatible	= "ti,am654-mailbox",
+		.data		= &omap4_data,
+	},
+	{
 		/* end */
 	},
 };
@@ -832,7 +839,10 @@
 	mdev->intr_type = intr_type;
 	mdev->mboxes = list;
 
-	/* OMAP does not have a Tx-Done IRQ, but rather a Tx-Ready IRQ */
+	/*
+	 * OMAP/K3 Mailbox IP does not have a Tx-Done IRQ, but rather a Tx-Ready
+	 * IRQ and is needed to run the Tx state machine
+	 */
 	mdev->controller.txdone_irq = true;
 	mdev->controller.dev = mdev->dev;
 	mdev->controller.ops = &omap_mbox_chan_ops;
@@ -901,9 +911,8 @@
 		return err;
 
 	/* kfifo size sanity check: alignment and minimal size */
-	mbox_kfifo_size = ALIGN(mbox_kfifo_size, sizeof(mbox_msg_t));
-	mbox_kfifo_size = max_t(unsigned int, mbox_kfifo_size,
-							sizeof(mbox_msg_t));
+	mbox_kfifo_size = ALIGN(mbox_kfifo_size, sizeof(u32));
+	mbox_kfifo_size = max_t(unsigned int, mbox_kfifo_size, sizeof(u32));
 
 	err = platform_driver_register(&omap_mbox_driver);
 	if (err)
diff -Naur --no-dereference a/drivers/Makefile b/drivers/Makefile
--- a/drivers/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/Makefile	2020-06-19 21:00:31.408227715 -0400
@@ -55,6 +55,9 @@
 obj-y				+= tty/
 obj-y				+= char/
 
+# put mmc early as many morden devices use emm/sd card as rootfs storage
+obj-y				+= mmc/
+
 # iommu/ comes before gpu as gpu are using iommu controllers
 obj-$(CONFIG_IOMMU_SUPPORT)	+= iommu/
 
@@ -128,7 +131,6 @@
 obj-$(CONFIG_PM_OPP)		+= opp/
 obj-$(CONFIG_CPU_FREQ)		+= cpufreq/
 obj-$(CONFIG_CPU_IDLE)		+= cpuidle/
-obj-y				+= mmc/
 obj-$(CONFIG_MEMSTICK)		+= memstick/
 obj-$(CONFIG_NEW_LEDS)		+= leds/
 obj-$(CONFIG_INFINIBAND)	+= infiniband/
@@ -155,6 +157,7 @@
 obj-$(CONFIG_HWSPINLOCK)	+= hwspinlock/
 obj-$(CONFIG_REMOTEPROC)	+= remoteproc/
 obj-$(CONFIG_RPMSG)		+= rpmsg/
+obj-$(CONFIG_RPMSG_KDRV)	+= rpmsg-kdrv/
 obj-$(CONFIG_SOUNDWIRE)		+= soundwire/
 
 # Virtualization drivers
diff -Naur --no-dereference a/drivers/media/i2c/Kconfig b/drivers/media/i2c/Kconfig
--- a/drivers/media/i2c/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/media/i2c/Kconfig	2020-06-19 21:00:31.416227715 -0400
@@ -631,7 +631,7 @@
 config VIDEO_OV2659
 	tristate "OmniVision OV2659 sensor support"
 	depends on VIDEO_V4L2 && I2C
-	depends on MEDIA_CAMERA_SUPPORT
+	depends on MEDIA_CAMERA_SUPPORT && GPIOLIB
 	select V4L2_FWNODE
 	---help---
 	  This is a Video4Linux2 sensor driver for the OmniVision
@@ -802,6 +802,29 @@
 	  This is a Video4Linux2 sensor driver for the OmniVision
 	  OV13858 camera.
 
+config VIDEO_OV1063X
+	tristate "OmniVision OV1063X sensor support"
+	depends on I2C && VIDEO_V4L2
+	depends on MEDIA_CAMERA_SUPPORT && GPIOLIB
+	help
+	  This is a Video4Linux2 sensor-level driver for the OmniVision
+	  OV1063X Sensor.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called ov1063x.
+
+config VIDEO_OV490
+	tristate "OmniVision OV490 sensor support"
+	depends on I2C && VIDEO_V4L2 && GPIOLIB
+	depends on MEDIA_CAMERA_SUPPORT
+	select V4L2_FWNODE
+	help
+	  This is a Video4Linux2 sensor-level driver for the OmniVision
+	  OV490 Sensor.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called ov490.
+
 config VIDEO_VS6624
 	tristate "ST VS6624 sensor support"
 	depends on VIDEO_V4L2 && I2C
@@ -887,6 +910,16 @@
 	  To compile this driver as a module, choose M here: the
 	  module will be called mt9v111.
 
+config VIDEO_MT9T11X
+	tristate "Aptina MT9T11x sensor support"
+	depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API && GPIOLIB
+	help
+	  This is a V4L2 sensor-level driver for the Aptina
+	  MT9T11x camera sensors.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called mt9t11x.
+
 config VIDEO_SR030PC30
 	tristate "Siliconfile SR030PC30 sensor support"
 	depends on I2C && VIDEO_V4L2
diff -Naur --no-dereference a/drivers/media/i2c/Makefile b/drivers/media/i2c/Makefile
--- a/drivers/media/i2c/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/media/i2c/Makefile	2020-06-19 21:00:31.416227715 -0400
@@ -80,6 +80,8 @@
 obj-$(CONFIG_VIDEO_OV7740) += ov7740.o
 obj-$(CONFIG_VIDEO_OV9650) += ov9650.o
 obj-$(CONFIG_VIDEO_OV13858) += ov13858.o
+obj-$(CONFIG_VIDEO_OV1063X) += ov1063x.o
+obj-$(CONFIG_VIDEO_OV490) += ov490.o
 obj-$(CONFIG_VIDEO_MT9M032) += mt9m032.o
 obj-$(CONFIG_VIDEO_MT9M111) += mt9m111.o
 obj-$(CONFIG_VIDEO_MT9P031) += mt9p031.o
@@ -88,6 +90,7 @@
 obj-$(CONFIG_VIDEO_MT9V011) += mt9v011.o
 obj-$(CONFIG_VIDEO_MT9V032) += mt9v032.o
 obj-$(CONFIG_VIDEO_MT9V111) += mt9v111.o
+obj-$(CONFIG_VIDEO_MT9T11X)	+= mt9t11x.o
 obj-$(CONFIG_VIDEO_SR030PC30)	+= sr030pc30.o
 obj-$(CONFIG_VIDEO_NOON010PC30)	+= noon010pc30.o
 obj-$(CONFIG_VIDEO_RJ54N1)	+= rj54n1cb0c.o
diff -Naur --no-dereference a/drivers/media/i2c/mt9t11x.c b/drivers/media/i2c/mt9t11x.c
--- a/drivers/media/i2c/mt9t11x.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/media/i2c/mt9t11x.c	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,2292 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Aptina mt9t11x CMOS Image Sensor driver
+ *
+ * This an adaptation of the existing soc_camera/mt9t112.c
+ * to use the newer v4l2 subdevice framework.
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Benoit Parrot <bparrot@ti.com>
+ *
+ * Register definitions and initial settings based on
+ * soc_camera/mt9t112.c driver written by Kuninori Morimoto.
+ * Copyright (C) 2009 Renesas Solutions Corp.
+ * Kuninori Morimoto <morimoto.kuninori@renesas.com>
+ */
+
+#include <linux/delay.h>
+#include <linux/gpio.h>
+#include <linux/i2c.h>
+#include <linux/kernel.h>
+#include <linux/media.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/gpio/consumer.h>
+#include <linux/ratelimit.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/videodev2.h>
+
+#include <media/media-entity.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-event.h>
+#include <media/v4l2-image-sizes.h>
+#include <media/v4l2-subdev.h>
+#include <media/v4l2-mediabus.h>
+#include <media/v4l2-common.h>
+#include <linux/of_graph.h>
+#include <media/v4l2-fwnode.h>
+
+static int debug;
+module_param(debug, int, 0644);
+MODULE_PARM_DESC(debug, "Debug level (0-2)");
+
+#define DRIVER_NAME "mt9t11x"
+
+#define MT9T111_ID	0
+#define MT9T112_ID	1
+
+/* you can check PLL/clock info */
+#define EXT_CLOCK 32000000
+#define MT9T11x_DEF_PIXEL_CLOCK 96000000
+#define MT9T11x_DEF_EXT_CLOCK 32000000
+
+#define MT9T11x_MAX_EXT_CLK 54000000
+#define MT9T11x_MAX_VCO_CLK 768000000
+#define MT9T11x_MAX_PIXEL_CLK 96000000
+#define MT9T11x_MAX_MIPI_CLK 768000000
+#define MT9T11x_MAX_MCU_CLK 96000000
+#define MT9T11x_MAX_SOC_CLK 54000000
+#define MT9T11x_MAX_SENSOR_CLK 70000000
+#define MT9T11x_MAX_PFD_CLK 24000000
+
+#define MT9T11x_FLAG_VFLIP	BIT(2)
+#define MT9T11x_FLAG_HFLIP	BIT(3)
+
+struct mt9t11x_pll_divider {
+	u8 m, n;
+	u8 p1, p2, p3, p4, p5, p6, p7;
+};
+
+/*
+ * mt9t11x camera info
+ */
+struct mt9t11x_camera_info {
+	u32 flags;
+	struct mt9t11x_pll_divider divider;
+	unsigned int mclk;
+	unsigned int pclk;
+	struct v4l2_fwnode_endpoint endpoint;
+};
+
+/************************************************************************
+ *			macro
+ ***********************************************************************/
+/*
+ * frame size
+ */
+#define MIN_WIDTH   32
+#define MIN_HEIGHT  32
+#define MAX_WIDTH   2048
+#define MAX_HEIGHT  1536
+
+#define VGA_WIDTH   640
+#define VGA_HEIGHT  480
+
+/*
+ * Logical address
+ */
+#define _VAR(id, offset, base) ((base) | ((id) & 0x1f) << 10 | \
+				((offset) & 0x3ff))
+#define VAR(id, offset)  _VAR((id), (offset), 0x0000)
+#define VAR8(id, offset) _VAR((id), (offset), 0x8000)
+
+/************************************************************************
+ *			struct
+ ***********************************************************************/
+struct mt9t11x_format {
+	u32 code;
+	u32 colorspace;
+	u16 fmt;
+	u16 order;
+};
+
+struct mt9t11x_framesize {
+	u16 width;
+	u16 height;
+};
+
+struct mt9t11x_resolution_param {
+	u16 col_strt;
+	u16 row_end;
+	u16 col_end;
+	u16 read_mode;
+	u16 fine_cor;
+	u16 fine_min;
+	u16 fine_max;
+	u16 base_lines;
+	u16 min_lin_len;
+	u16 line_len;
+	u16 con_width;
+	u16 con_height;
+	u16 s_f1_50;
+	u16 s_f2_50;
+	u16 s_f1_60;
+	u16 s_f2_60;
+	u16 per_50;
+	u16 per_50_M;
+	u16 per_60;
+	u16 fd_w_height;
+	u16 tx_water;
+	u16 max_fd_50;
+	u16 max_fd_60;
+	u16 targ_fd;
+};
+
+struct mt9t11x_priv {
+	struct v4l2_subdev		subdev;
+#if defined(CONFIG_MEDIA_CONTROLLER)
+	struct media_pad		pad;
+#endif
+	struct v4l2_ctrl_handler	handler;
+
+	struct mt9t11x_camera_info	*info;
+	struct i2c_client		*client;
+	struct v4l2_rect		frame;
+	struct v4l2_clk			*clk;
+	const struct mt9t11x_format	*format;
+	int				num_formats;
+	u32				flags;
+
+	/* GPIOs */
+	struct gpio_desc		*reset_gpio;
+	struct gpio_desc		*powerdown_gpio;
+	struct gpio_desc		*oscen_gpio;
+	struct gpio_desc		*bufen_gpio;
+	struct gpio_desc		*camen_gpio;
+
+	/* V4L2 Ctrl handle */
+	struct v4l2_ctrl		*hflip;
+	struct v4l2_ctrl		*vflip;
+
+	/* Protects the struct fields below */
+	struct mutex			lock;
+	int				streaming;
+	int				power;
+
+	struct mt9t11x_resolution_param	resolution;
+};
+
+/************************************************************************
+ *			supported frame sizes
+ ***********************************************************************/
+
+static const struct mt9t11x_framesize mt9t11x_framesizes[] = {
+	{
+		.width		= 2048,
+		.height		= 1536,
+	}, {
+		.width		= 1920,
+		.height		= 1200,
+	}, {
+		.width		= 1920,
+		.height		= 1080,
+	}, {
+		.width		= 1600,
+		.height		= 1200,
+	}, {
+		.width		= 1440,
+		.height		= 900,
+	}, {
+		.width		= 1280,
+		.height		= 800,
+	}, {
+		.width		= 1280,
+		.height		= 720,
+	}, {
+		.width		= 800,
+		.height		= 600,
+	}, {
+		.width		= 800,
+		.height		= 480,
+	}, {
+		.width		= 640,
+		.height		= 480,
+	}, {
+		.width		= 600,
+		.height		= 400,
+	}, {
+		.width		= 352,
+		.height		= 288,
+	}, {
+		.width		= 320,
+		.height		= 240,
+	}, {
+		.width		= 160,
+		.height		= 120,
+	},
+};
+
+/************************************************************************
+ *			supported format
+ ***********************************************************************/
+
+static const struct mt9t11x_format mt9t11x_cfmts[] = {
+	{
+		.code		= MEDIA_BUS_FMT_UYVY8_2X8,
+		.colorspace	= V4L2_COLORSPACE_JPEG,
+		.fmt		= 1,
+		.order		= 0,
+	}, {
+		.code		= MEDIA_BUS_FMT_VYUY8_2X8,
+		.colorspace	= V4L2_COLORSPACE_JPEG,
+		.fmt		= 1,
+		.order		= 1,
+	}, {
+		.code		= MEDIA_BUS_FMT_YUYV8_2X8,
+		.colorspace	= V4L2_COLORSPACE_JPEG,
+		.fmt		= 1,
+		.order		= 2,
+	}, {
+		.code		= MEDIA_BUS_FMT_YVYU8_2X8,
+		.colorspace	= V4L2_COLORSPACE_JPEG,
+		.fmt		= 1,
+		.order		= 3,
+	}, {
+		.code		= MEDIA_BUS_FMT_RGB555_2X8_PADHI_LE,
+		.colorspace	= V4L2_COLORSPACE_SRGB,
+		.fmt		= 8,
+		.order		= 2,
+	}, {
+		.code		= MEDIA_BUS_FMT_RGB565_2X8_LE,
+		.colorspace	= V4L2_COLORSPACE_SRGB,
+		.fmt		= 4,
+		.order		= 2,
+	},
+};
+
+/************************************************************************
+ *			general function
+ ***********************************************************************/
+static inline struct mt9t11x_priv *sd_to_mt9t11x(struct v4l2_subdev *sd)
+{
+	return container_of(sd, struct mt9t11x_priv, subdev);
+}
+
+static inline struct mt9t11x_priv *to_mt9t11x(const struct i2c_client *client)
+{
+	return sd_to_mt9t11x(i2c_get_clientdata(client));
+}
+
+static inline struct mt9t11x_priv *ctrl_to_mt9t11x(struct v4l2_ctrl *ctrl)
+{
+	return container_of(ctrl->handler, struct mt9t11x_priv, handler);
+}
+
+static int mt9t11x_reg_read(const struct i2c_client *client, u16 command)
+{
+	struct i2c_msg msg[2];
+	u8 buf[2];
+	int ret;
+
+	command = swab16(command);
+
+	msg[0].addr  = client->addr;
+	msg[0].flags = 0;
+	msg[0].len   = 2;
+	msg[0].buf   = (u8 *)&command;
+
+	msg[1].addr  = client->addr;
+	msg[1].flags = I2C_M_RD;
+	msg[1].len   = 2;
+	msg[1].buf   = buf;
+
+	/*
+	 * if return value of this function is < 0,
+	 * it mean error.
+	 * else, under 16bit is valid data.
+	 */
+	ret = i2c_transfer(client->adapter, msg, 2);
+	if (ret < 0)
+		return ret;
+
+	memcpy(&ret, buf, 2);
+	return swab16(ret);
+}
+
+static int mt9t11x_reg_write(const struct i2c_client *client,
+			     u16 command, u16 data)
+{
+	struct i2c_msg msg;
+	u8 buf[4];
+	int ret;
+
+	command = swab16(command);
+	data = swab16(data);
+
+	memcpy(buf + 0, &command, 2);
+	memcpy(buf + 2, &data,    2);
+
+	msg.addr  = client->addr;
+	msg.flags = 0;
+	msg.len   = 4;
+	msg.buf   = buf;
+
+	/*
+	 * i2c_transfer return message length,
+	 * but this function should return 0 if correct case
+	 */
+	ret = i2c_transfer(client->adapter, &msg, 1);
+	if (ret >= 0)
+		ret = 0;
+
+	return ret;
+}
+
+static int mt9t11x_reg_mask_set(const struct i2c_client *client,
+				u16  command,
+				u16  mask,
+				u16  set)
+{
+	int val = mt9t11x_reg_read(client, command);
+
+	if (val < 0)
+		return val;
+
+	val &= ~mask;
+	val |= set & mask;
+
+	return mt9t11x_reg_write(client, command, val);
+}
+
+/* mcu access */
+static int mt9t11x_mcu_read(const struct i2c_client *client, u16 command)
+{
+	int ret;
+
+	ret = mt9t11x_reg_write(client, 0x098E, command);
+	if (ret < 0)
+		return ret;
+
+	return mt9t11x_reg_read(client, 0x0990);
+}
+
+static int mt9t11x_mcu_write(const struct i2c_client *client,
+			     u16 command, u16 data)
+{
+	int ret;
+
+	ret = mt9t11x_reg_write(client, 0x098E, command);
+	if (ret < 0)
+		return ret;
+
+	return mt9t11x_reg_write(client, 0x0990, data);
+}
+
+static int mt9t11x_mcu_mask_set(const struct i2c_client *client,
+				u16  command,
+				u16  mask,
+				u16  set)
+{
+	int val = mt9t11x_mcu_read(client, command);
+
+	if (val < 0)
+		return val;
+
+	val &= ~mask;
+	val |= set & mask;
+
+	return mt9t11x_mcu_write(client, command, val);
+}
+
+static int mt9t11x_reset(const struct i2c_client *client)
+{
+	int ret;
+
+	dev_dbg(&client->dev, "%s:\n", __func__);
+
+	ret = mt9t11x_reg_mask_set(client, 0x001a, 0x0001, 0x0001);
+	if (ret < 0)
+		return ret;
+	usleep_range(1000, 2000);
+	ret = mt9t11x_reg_mask_set(client, 0x001a, 0x0001, 0x0000);
+	if (ret < 0)
+		return ret;
+
+	return ret;
+}
+
+static int mt9t11x_streaming(struct mt9t11x_priv *priv, int on)
+{
+	struct i2c_client *client = priv->client;
+	int ret, tmp;
+
+	dev_dbg(&client->dev, "%s: on: %d\n", __func__, on);
+	on = (on) ? 1 : 0;
+
+	ret = mt9t11x_reg_mask_set(client, 0x001a, 0x0200, on << 9);
+	if (ret < 0)
+		return ret;
+
+	tmp = mt9t11x_reg_read(client, 0x001A);
+	dev_dbg(&client->dev, "reset_and_misc_control: 0x%04x\n", tmp);
+
+	return 0;
+}
+
+#define CLOCK_INFO(a, b)				\
+	do {						\
+		if (debug > 1)				\
+			mt9t11x_clock_info(a, b);	\
+	} while (0)
+static int mt9t11x_clock_info(const struct i2c_client *client, u32 ext)
+{
+	int m, n, p1, p2, p3, p4, p5, p6, p7;
+	u32 vco, clk;
+	char *enable;
+
+	ext /= 1000; /* kbyte order */
+
+	n = mt9t11x_reg_read(client, 0x0012);
+	p1 = n & 0x000f;
+	n = n >> 4;
+	p2 = n & 0x000f;
+	n = n >> 4;
+	p3 = n & 0x000f;
+
+	n = mt9t11x_reg_read(client, 0x002a);
+	p4 = n & 0x000f;
+	n = n >> 4;
+	p5 = n & 0x000f;
+	n = n >> 4;
+	p6 = n & 0x000f;
+
+	n = mt9t11x_reg_read(client, 0x002c);
+	p7 = n & 0x000f;
+
+	n = mt9t11x_reg_read(client, 0x0010);
+	m = n & 0x00ff;
+	n = (n >> 8) & 0x003f;
+
+	enable = ((ext < 6000) || (ext > 54000)) ? "X" : "";
+	dev_dbg(&client->dev, "EXTCLK          : %10u K %s\n", ext, enable);
+
+	vco = 2 * m * ext / (n + 1);
+	enable = ((vco < 384000) || (vco > 768000)) ? "X" : "";
+	dev_dbg(&client->dev, "VCO             : %10u K %s\n", vco, enable);
+
+	clk = vco / (p1 + 1) / (p2 + 1);
+	enable = (clk > 96000) ? "X" : "";
+	dev_dbg(&client->dev, "PIXCLK          : %10u K %s\n", clk, enable);
+
+	clk = vco / (p3 + 1);
+	enable = (clk > 768000) ? "X" : "";
+	dev_dbg(&client->dev, "MIPICLK         : %10u K %s\n", clk, enable);
+
+	clk = vco / (p6 + 1);
+	enable = (clk > 96000) ? "X" : "";
+	dev_dbg(&client->dev, "MCU CLK         : %10u K %s\n", clk, enable);
+
+	clk = vco / (p5 + 1);
+	enable = (clk > 54000) ? "X" : "";
+	dev_dbg(&client->dev, "SOC CLK         : %10u K %s\n", clk, enable);
+
+	clk = vco / (p4 + 1);
+	enable = (clk > 70000) ? "X" : "";
+	dev_dbg(&client->dev, "Sensor CLK      : %10u K %s\n", clk, enable);
+
+	clk = vco / (p7 + 1);
+	dev_dbg(&client->dev, "External sensor : %10u K\n", clk);
+
+	clk = ext / (n + 1);
+	enable = ((clk < 2000) || (clk > 24000)) ? "X" : "";
+	dev_dbg(&client->dev, "PFD             : %10u K %s\n", clk, enable);
+
+	return 0;
+}
+
+static void mt9t11x_frame_check(u32 *width, u32 *height, u32 *left, u32 *top)
+{
+	v4l_bound_align_image(width, MIN_WIDTH, MAX_WIDTH, 1,
+			      height, MIN_HEIGHT, MAX_HEIGHT, 1, 0);
+	*left = 0;
+	*top = 0;
+}
+
+static int mt9t11x_set_a_frame_size(const struct i2c_client *client,
+				    u16 width,
+				    u16 height)
+{
+	int ret;
+	u16 wstart = (MAX_WIDTH - width) / 2;
+	u16 hstart = (MAX_HEIGHT - height) / 2;
+
+	dev_dbg(&client->dev, "%s:\n", __func__);
+
+	/* (Context A) Image Width/Height */
+	ret = mt9t11x_mcu_write(client, VAR(26, 0), width);
+	if (ret < 0)
+		return ret;
+	ret = mt9t11x_mcu_write(client, VAR(26, 2), height);
+	if (ret < 0)
+		return ret;
+
+	/* (Context A) Output Width/Height */
+	ret = mt9t11x_mcu_write(client, VAR(18, 43), 8 + width);
+	if (ret < 0)
+		return ret;
+	ret = mt9t11x_mcu_write(client, VAR(18, 45), 8 + height);
+	if (ret < 0)
+		return ret;
+
+	/* (Context A) Start Row/Column */
+	ret = mt9t11x_mcu_write(client, VAR(18, 2), 4 + hstart);
+	if (ret < 0)
+		return ret;
+	ret = mt9t11x_mcu_write(client, VAR(18, 4), 4 + wstart);
+	if (ret < 0)
+		return ret;
+
+	/* (Context A) End Row/Column */
+	ret = mt9t11x_mcu_write(client, VAR(18, 6), 11 + height + hstart);
+	if (ret < 0)
+		return ret;
+	ret = mt9t11x_mcu_write(client, VAR(18, 8), 11 + width  + wstart);
+	if (ret < 0)
+		return ret;
+
+	ret = mt9t11x_mcu_write(client, VAR8(1, 0), 0x06);
+	if (ret < 0)
+		return ret;
+
+	return ret;
+}
+
+static int mt9t11x_set_pll_dividers(const struct i2c_client *client,
+				    u8 m, u8 n,
+				    u8 p1, u8 p2, u8 p3,
+				    u8 p4, u8 p5, u8 p6,
+				    u8 p7)
+{
+	int ret;
+	u16 val;
+
+	dev_dbg(&client->dev, "%s:\n", __func__);
+
+	/* N/M */
+	val = (n << 8) |
+	      (m << 0);
+	ret = mt9t11x_reg_mask_set(client, 0x0010, 0x3fff, val);
+	if (ret < 0)
+		return ret;
+
+	/* P1/P2/P3 */
+	val = ((p3 & 0x0F) << 8) |
+	      ((p2 & 0x0F) << 4) |
+	      ((p1 & 0x0F) << 0);
+	ret = mt9t11x_reg_mask_set(client, 0x0012, 0x0fff, val);
+	if (ret < 0)
+		return ret;
+
+	/* P4/P5/P6 */
+	val = (0x7         << 12) |
+	      ((p6 & 0x0F) <<  8) |
+	      ((p5 & 0x0F) <<  4) |
+	      ((p4 & 0x0F) <<  0);
+	ret = mt9t11x_reg_mask_set(client, 0x002A, 0x7fff, val);
+	if (ret < 0)
+		return ret;
+
+	/* P7 */
+	val = (0x1         << 12) |
+	      ((p7 & 0x0F) <<  0);
+	ret = mt9t11x_reg_mask_set(client, 0x002C, 0x100f, val);
+	if (ret < 0)
+		return ret;
+
+	return ret;
+}
+
+static int mt9t11x_set_resolution_params(const struct i2c_client *client)
+{
+	int ret = 1;
+	struct mt9t11x_priv *priv = to_mt9t11x(client);
+	struct mt9t11x_resolution_param *resolution = &priv->resolution;
+
+	if (priv->frame.width == 1280 && priv->frame.height == 720) {
+		resolution->col_strt    = 0x0004;
+		resolution->row_end     = 0x05AD;
+		resolution->col_end     = 0x050B;
+		resolution->read_mode   = 0x002C;
+		resolution->fine_cor    = 0x008C;
+		resolution->fine_min    = 0x01F1;
+		resolution->fine_max    = 0x00FF;
+		resolution->base_lines  = 0x032D;
+		resolution->min_lin_len = 0x0378;
+		resolution->line_len    = 0x091C;
+		resolution->con_width   = 0x0508;
+		resolution->con_height  = 0x02D8;
+		resolution->s_f1_50     = 0x23;
+		resolution->s_f2_50     = 0x25;
+		resolution->s_f1_60     = 0x2B;
+		resolution->s_f2_60     = 0x2D;
+		resolution->per_50      = 0xDC;
+		resolution->per_50_M    = 0x00;
+		resolution->per_60      = 0xB7;
+		resolution->fd_w_height = 0x05;
+		resolution->tx_water    = 0x0210;
+		resolution->max_fd_50   = 0x0004;
+		resolution->max_fd_60   = 0x0004;
+		resolution->targ_fd     = 0x0004;
+	} else if (priv->frame.width <= 1024 &&
+		   priv->frame.height <= 768 &&
+		   priv->frame.width != priv->frame.height) {
+		resolution->col_strt    = 0x000;
+		resolution->row_end     = 0x60D;
+		resolution->col_end     = 0x80D;
+		resolution->read_mode   = 0x046C;
+		resolution->fine_cor    = 0x00CC;
+		resolution->fine_min    = 0x0381;
+		resolution->fine_max    = 0x024F;
+		resolution->base_lines  = 0x0364;
+		resolution->min_lin_len = 0x05D0;
+		resolution->line_len    = 0x07AC;
+		resolution->con_width   = 0x0408;
+		resolution->con_height  = 0x0308;
+		resolution->s_f1_50     = 0x23;
+		resolution->s_f2_50     = 0x25;
+		resolution->s_f1_60     = 0x2A;
+		resolution->s_f2_60     = 0x2C;
+		resolution->per_50      = 0x05;
+		resolution->per_50_M    = 0x01;
+		resolution->per_60      = 0xD9;
+		resolution->fd_w_height = 0x06;
+		resolution->max_fd_50   = 0x0003;
+		resolution->max_fd_60   = 0x0004;
+		resolution->targ_fd     = 0x0003;
+		if (priv->frame.width == 1024 &&
+		    priv->frame.height == 768) {
+			resolution->tx_water = 0x0218;
+		} else if (priv->frame.width == 800 &&
+			   priv->frame.height == 480) {
+			resolution->tx_water = 0x02DA;
+		} else {
+			/*
+			 * 640 x 480 but use it with everything else until
+			 * we figure out how to calc it
+			 */
+			resolution->tx_water = 0x0352;
+		}
+	} else {
+		ret = 0;
+	}
+
+	return ret;
+}
+
+static int mt9t11x_pll_setup_pll(const struct i2c_client *client)
+{
+	struct mt9t11x_priv *priv = to_mt9t11x(client);
+	int data, i, ret;
+
+	/* Bypass PLL */
+	ret = mt9t11x_reg_mask_set(client, 0x14, 1, 1);
+	if (ret < 0)
+		return ret;
+	/* Power-down PLL */
+	ret = mt9t11x_reg_mask_set(client, 0X14, 2, 0);
+	if (ret < 0)
+		return ret;
+	/* PLL control: BYPASS PLL = 8517 */
+	ret = mt9t11x_reg_write(client, 0x0014, 0x2145);
+	if (ret < 0)
+		return ret;
+
+	/* Replace these registers when new timing parameters are generated */
+	ret = mt9t11x_set_pll_dividers(client,
+				       priv->info->divider.m,
+				       priv->info->divider.n,
+				       priv->info->divider.p1,
+				       priv->info->divider.p2,
+				       priv->info->divider.p3,
+				       priv->info->divider.p4,
+				       priv->info->divider.p5,
+				       priv->info->divider.p6,
+				       priv->info->divider.p7);
+	if (ret < 0)
+		return ret;
+
+	/* Reset Misc. Control = 536 */
+	/* make sure parallel interface is not enable at first */
+	ret = mt9t11x_reg_write(client, 0x001A, 0x018);
+	if (ret < 0)
+		return ret;
+	/* PLL control: TEST_BYPASS on = 9541 */
+	ret = mt9t11x_reg_write(client, 0x0014, 0x2545);
+	if (ret < 0)
+		return ret;
+	/* PLL control: PLL_ENABLE on = 9543 */
+	ret = mt9t11x_reg_write(client, 0x0014, 0x2547);
+	if (ret < 0)
+		return ret;
+	/* PLL control: SEL_LOCK_DET on = 9287 */
+	ret = mt9t11x_reg_write(client, 0x0014, 0x2447);
+	if (ret < 0)
+		return ret;
+	/* PLL control: TEST_BYPASS off = 8263 */
+	ret = mt9t11x_reg_write(client, 0x0014, 0x2047);
+	if (ret < 0)
+		return ret;
+
+	/*  Wait for the PLL to lock */
+	for (i = 0; i < 1000; i++) {
+		data = mt9t11x_reg_read(client, 0x0014);
+		if (0x8000 & data)
+			break;
+
+		usleep_range(10000, 11000);
+	}
+
+	/* PLL control: PLL_BYPASS off = 8262 */
+	ret = mt9t11x_reg_write(client, 0x0014, 0x2046);
+	if (ret < 0)
+		return ret;
+	/* Reference clock count for 20 us = 640 */
+	ret = mt9t11x_reg_write(client, 0x0022, 0x0280);
+	if (ret < 0)
+		return ret;
+	/* Pad Slew Rate = 1911 */
+	ret = mt9t11x_reg_write(client, 0x001E, 0x0777);
+	if (ret < 0)
+		return ret;
+	/* JPEG Clock = 1024 */
+	ret = mt9t11x_reg_write(client, 0x0016, 0x0400);
+
+	return ret;
+}
+
+inline u32 calc_vco(u32 desired, u32 ext, u8 *_m, u8 *_n)
+{
+	u32 m, n;
+	u64 actual;
+	s64 delta, bestdelta = -1;
+
+	n = *_n + 1;
+	if (n == 0)
+		n = 1;
+
+	for (; n <= 64; n++)
+		for (m = 1; m <= 256; m++) {
+			actual = ext * 2;
+			actual *= m;
+			actual = div_s64(actual, n);
+			delta = (desired - actual);
+			if (delta < 0)
+				continue;
+			if (delta < bestdelta || bestdelta == -1) {
+				bestdelta = delta;
+				*_m = (u8)(m);
+				*_n = (u8)(n - 1);
+			}
+		}
+	actual = ext * 2 * (*_m);
+	actual = div_s64(actual, *_n + 1);
+
+	return actual;
+}
+
+static inline u32 calc_pixclk(u32 desired, u32 vco, u8 *_p1, u8 *_p2)
+{
+	u32 p1, p2;
+	u32 actual;
+	s32 delta, bestdelta = -1;
+
+	for (p1 = 1; p1 <= 16; p1++)
+		for (p2 = 1; p2 <= 16; p2++) {
+			actual = vco;
+			actual /= p1;
+			actual /= p2;
+			delta = (desired - actual);
+			if (delta < 0)
+				continue;
+			if (delta < bestdelta || bestdelta == -1) {
+				bestdelta = delta;
+				*_p1 = (u8)(p1 - 1);
+				*_p2 = (u8)(p2 - 1);
+			}
+		}
+	actual = vco / (*_p1 + 1);
+	actual /= (*_p2 + 1);
+
+	return actual;
+}
+
+static inline u32 calc_div(u32 desired, u32 src, u8 *_div)
+{
+	u32 div;
+
+	if (src > desired) {
+		div = src / desired;
+		if ((src % desired) > 0)
+			div++;
+	} else {
+		div = 1;
+	}
+
+	*_div = (u8)(div - 1);
+
+	return 0;
+}
+
+static unsigned int mt9t11x_pll_calc_params(struct mt9t11x_priv *priv)
+{
+	struct i2c_client *client = priv->client;
+	struct mt9t11x_camera_info *info = priv->info;
+	u32 vco;
+
+	dev_dbg(&client->dev, "%s:\n", __func__);
+
+	calc_div(MT9T11x_MAX_PFD_CLK, info->mclk, &info->divider.n);
+	vco = calc_vco(MT9T11x_MAX_VCO_CLK, info->mclk, &info->divider.m,
+		       &info->divider.n);
+	calc_pixclk(info->pclk, vco, &info->divider.p1, &info->divider.p2);
+	calc_div(MT9T11x_MAX_MIPI_CLK, vco, &info->divider.p3);
+	calc_div(MT9T11x_MAX_MCU_CLK, vco, &info->divider.p6);
+	calc_div(MT9T11x_MAX_SOC_CLK, vco, &info->divider.p5);
+	calc_div(MT9T11x_MAX_SENSOR_CLK, vco, &info->divider.p4);
+
+	return 0;
+}
+
+static int mt9t11x_sysctl_startup(const struct i2c_client *client)
+{
+	int ret;
+
+	/* reset */
+	ret = mt9t11x_reset(client);
+	if (ret < 0)
+		return ret;
+
+	/* Setup PLL */
+	ret = mt9t11x_pll_setup_pll(client);
+
+	return ret;
+}
+
+static int mt9t11x_high_speed_overrides(const struct i2c_client *client)
+{
+	int ret;
+
+	/*
+	 * Use this section to apply settings that are specific to this
+	 * revision of SOC or for any other specialized settings
+	 * clear the "Output Buffer Enable Adaptive Clock" bit to enable
+	 * the SYSCTL slew rate settings, change this in the variables
+	 * and register
+	 */
+
+	/* PRI_A_CONFIG_JPEG_OB_TX_CONTROL_VAR */
+	ret = mt9t11x_mcu_write(client, VAR(26, 160), 0x082E);
+	if (ret < 0)
+		return ret;
+	/* PRI_B_CONFIG_JPEG_OB_TX_CONTROL_VAR */
+	ret = mt9t11x_mcu_write(client, VAR(27, 160), 0x082E);
+	if (ret < 0)
+		return ret;
+	/* SEC_A_CONFIG_JPEG_OB_TX_CONTROL_VAR */
+	ret = mt9t11x_mcu_write(client, VAR(28, 160), 0x082E);
+	if (ret < 0)
+		return ret;
+	/* SEC_B_CONFIG_JPEG_OB_TX_CONTROL_VAR */
+	ret = mt9t11x_mcu_write(client, VAR(29, 160), 0x082E);
+	if (ret < 0)
+		return ret;
+	ret = mt9t11x_reg_mask_set(client, 0x3C52, 0x0040, 0);
+	if (ret < 0)
+		return ret;
+
+	/* Set correct values for Context B FIFO control */
+	/* CAM1_CTX_B_RX_FIFO_TRIGGER_MARK */
+	ret = mt9t11x_mcu_write(client, VAR(18, 142), 32);
+	if (ret < 0)
+		return ret;
+	/* PRI_B_CONFIG_IO_OB_MANUAL_FLAG */
+	ret = mt9t11x_mcu_write(client, VAR(27, 172), 0);
+
+	return ret;
+}
+
+static int mt9t11x_go(const struct i2c_client *client)
+{
+	int data, i, ret;
+
+	/* release MCU from standby */
+	ret = mt9t11x_reg_mask_set(client, 0x0018, 0x0001, 0);
+	if (ret < 0)
+		return ret;
+
+	/* wait for K26A to come out of standby */
+	for (i = 0; i < 100; i++) {
+		data = mt9t11x_reg_read(client, 0x0018);
+		if (!(0x4000 & data))
+			break;
+
+		usleep_range(10000, 11000);
+	}
+
+	return ret;
+}
+
+static int mt9t11x_continue(const struct i2c_client *client)
+{
+	int data, i, ret;
+
+	/* clear powerup stop bit */
+	ret = mt9t11x_reg_mask_set(client, 0x0018, 0x0004, 0);
+	if (ret < 0)
+		return ret;
+
+	/* wait for sequencer to enter preview state */
+	for (i = 0; i < 100; i++) {
+		data = mt9t11x_mcu_read(client, VAR8(1, 1));
+		if (data == 3)
+			break;
+
+		usleep_range(10000, 11000);
+	}
+
+	return ret;
+}
+
+static int mt9t11x_mcu_powerup_stop_enable(const struct i2c_client *client)
+{
+	int ret;
+
+	/* set powerup stop bit */
+	ret = mt9t11x_reg_mask_set(client, 0x0018, 0x0004, 0x0004);
+
+	return ret;
+}
+
+static int mt9t11x_custom_setup(const struct i2c_client *client)
+{
+	struct mt9t11x_priv *priv = to_mt9t11x(client);
+	struct mt9t11x_resolution_param *resolution = &priv->resolution;
+	int ret;
+
+	/* I2C Master Clock Divider */
+	ret = mt9t11x_mcu_write(client, VAR(24, 6), 0x0100);
+	if (ret < 0)
+		return ret;
+	/* Output Width (A) */
+	ret = mt9t11x_mcu_write(client, VAR(26, 0), priv->frame.width);
+	if (ret < 0)
+		return ret;
+	/* Output Height (A) */
+	ret = mt9t11x_mcu_write(client, VAR(26, 2), priv->frame.height);
+	if (ret < 0)
+		return ret;
+	/* JPEG (A) */
+	ret = mt9t11x_mcu_write(client, VAR8(26, 142), 0x00);
+	if (ret < 0)
+		return ret;
+	/* Adaptive Output Clock (A) */
+	ret = mt9t11x_mcu_mask_set(client, VAR(26, 160), 0x0040, 0x0000);
+	if (ret < 0)
+		return ret;
+	/* Row Start (A) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 2), 0x000);
+	if (ret < 0)
+		return ret;
+	/* Column Start (A) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 4), resolution->col_strt);
+	if (ret < 0)
+		return ret;
+	/* Row End (A) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 6), resolution->row_end);
+	if (ret < 0)
+		return ret;
+	/* Column End (A) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 8), resolution->col_end);
+	if (ret < 0)
+		return ret;
+	/* Row Speed (A) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 10), 0x0111);
+	if (ret < 0)
+		return ret;
+	/* Read Mode (A) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 12), resolution->read_mode);
+	if (ret < 0)
+		return ret;
+	/* Fine Correction (A) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 15), resolution->fine_cor);
+	if (ret < 0)
+		return ret;
+	/* Fine IT Min (A) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 17), resolution->fine_min);
+	if (ret < 0)
+		return ret;
+	/* Fine IT Max Margin (A) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 19), resolution->fine_max);
+	if (ret < 0)
+		return ret;
+	/* Base Frame Lines (A) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 29), resolution->base_lines);
+	if (ret < 0)
+		return ret;
+	/* Min Line Length (A) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 31), resolution->min_lin_len);
+	if (ret < 0)
+		return ret;
+	/* Line Length (A) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 37), resolution->line_len);
+	if (ret < 0)
+		return ret;
+	/* Contex Width (A) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 43), resolution->con_width);
+	if (ret < 0)
+		return ret;
+	/* Context Height (A) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 45), resolution->con_height);
+	if (ret < 0)
+		return ret;
+	/* Output Width (B) */
+	ret = mt9t11x_mcu_write(client, VAR(27, 0), 0x0800);
+	if (ret < 0)
+		return ret;
+	/* Output Height (B) */
+	ret = mt9t11x_mcu_write(client, VAR(27, 2), 0x0600);
+	if (ret < 0)
+		return ret;
+	/* JPEG (B) */
+	ret = mt9t11x_mcu_write(client, VAR8(27, 142), 0x01);
+	if (ret < 0)
+		return ret;
+	/* Adaptive Output Clock (B) */
+	ret = mt9t11x_mcu_mask_set(client, VAR(27, 160), 0x0040, 0x0000);
+	if (ret < 0)
+		return ret;
+	/* Row Start (B) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 74), 0x004);
+	if (ret < 0)
+		return ret;
+	/* Column Start (B) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 76), 0x004);
+	if (ret < 0)
+		return ret;
+	/* Row End (B) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 78), 0x60B);
+	if (ret < 0)
+		return ret;
+	/* Column End (B) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 80), 0x80B);
+	if (ret < 0)
+		return ret;
+	/* Row Speed (B) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 82), 0x0111);
+	if (ret < 0)
+		return ret;
+	/* Read Mode (B) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 84), 0x0024);
+	if (ret < 0)
+		return ret;
+	/* Fine Correction (B) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 87), 0x008C);
+	if (ret < 0)
+		return ret;
+	/* Fine IT Min (B) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 89), 0x01F1);
+	if (ret < 0)
+		return ret;
+	/* Fine IT Max Margin (B) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 91), 0x00FF);
+	if (ret < 0)
+		return ret;
+	/* Base Frame Lines (B) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 101), 0x06AE);
+	if (ret < 0)
+		return ret;
+	/* Min Line Length (B) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 103), 0x0378);
+	if (ret < 0)
+		return ret;
+	/* Line Length (B) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 109), 0x0A3A);
+	if (ret < 0)
+		return ret;
+	/* Contex Width (B) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 115), 0x0808);
+	if (ret < 0)
+		return ret;
+	/* Context Height (B) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 117), 0x0608);
+	if (ret < 0)
+		return ret;
+	/* search_f1_50 */
+	ret = mt9t11x_mcu_write(client, VAR8(18, 165), resolution->s_f1_50);
+	if (ret < 0)
+		return ret;
+	/* search_f2_50 */
+	ret = mt9t11x_mcu_write(client, VAR8(18, 166), resolution->s_f2_50);
+	if (ret < 0)
+		return ret;
+	/* search_f1_60 */
+	ret = mt9t11x_mcu_write(client, VAR8(18, 167), resolution->s_f1_60);
+	if (ret < 0)
+		return ret;
+	/* search_f2_60 */
+	ret = mt9t11x_mcu_write(client, VAR8(18, 168), resolution->s_f2_60);
+	if (ret < 0)
+		return ret;
+	/* period_50Hz (A) */
+	ret = mt9t11x_mcu_write(client, VAR8(18, 68), resolution->per_50);
+	if (ret < 0)
+		return ret;
+	/* period_50Hz (A MSB) */
+	ret = mt9t11x_mcu_write(client, VAR8(18, 303), resolution->per_50_M);
+	if (ret < 0)
+		return ret;
+	/* period_60Hz (A) */
+	ret = mt9t11x_mcu_write(client, VAR8(18, 69), resolution->per_60);
+	if (ret < 0)
+		return ret;
+	/* period_60Hz (A MSB) */
+	ret = mt9t11x_mcu_write(client, VAR8(18, 301), 0x00);
+	if (ret < 0)
+		return ret;
+	/* period_50Hz (B) */
+	ret = mt9t11x_mcu_write(client, VAR8(18, 140), 0xD2);
+	if (ret < 0)
+		return ret;
+	/* period_50Hz (B) MSB */
+	ret = mt9t11x_mcu_write(client, VAR8(18, 304), 0x00);
+	if (ret < 0)
+		return ret;
+	/* period_60Hz (B) */
+	ret = mt9t11x_mcu_write(client, VAR8(18, 141), 0xAF);
+	if (ret < 0)
+		return ret;
+	/* period_60Hz (B) MSB */
+	ret = mt9t11x_mcu_write(client, VAR8(18, 302), 0x00);
+	if (ret < 0)
+		return ret;
+	/* FD Window Height */
+	ret = mt9t11x_mcu_write(client, VAR8(14, 37), resolution->fd_w_height);
+	if (ret < 0)
+		return ret;
+	/* Stat_min */
+	ret = mt9t11x_mcu_write(client, VAR8(8, 9), 0x02);
+	if (ret < 0)
+		return ret;
+	/* Stat_max */
+	ret = mt9t11x_mcu_write(client, VAR8(8, 10), 0x03);
+	if (ret < 0)
+		return ret;
+	/* Min_amplitude */
+	ret = mt9t11x_mcu_write(client, VAR8(8, 12), 0x0A);
+	if (ret < 0)
+		return ret;
+	/* RX FIFO Watermark (A) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 70), 0x0080);
+	if (ret < 0)
+		return ret;
+	/* TX FIFO Watermark (A) */
+	ret = mt9t11x_mcu_write(client, VAR(26, 170), resolution->tx_water);
+	if (ret < 0)
+		return ret;
+	/* Max FD Zone 50 Hz */
+	ret = mt9t11x_mcu_write(client, VAR(26, 21), resolution->max_fd_50);
+	if (ret < 0)
+		return ret;
+	/* Max FD Zone 60 Hz */
+	ret = mt9t11x_mcu_write(client, VAR(26, 23), resolution->max_fd_60);
+	if (ret < 0)
+		return ret;
+	/* AE Target FD Zone */
+	ret = mt9t11x_mcu_write(client, VAR(26, 45), resolution->targ_fd);
+	if (ret < 0)
+		return ret;
+	/* RX FIFO Watermark (B) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 142), 0x0080);
+	if (ret < 0)
+		return ret;
+	/* TX FIFO Watermark (B) */
+	ret = mt9t11x_mcu_write(client, VAR(27, 170), 0x01D0);
+	if (ret < 0)
+		return ret;
+	/* Refresh Sequencer Mode */
+	ret = mt9t11x_mcu_write(client, VAR8(1, 0), 0x06);
+	if (ret < 0)
+		return ret;
+	/* Refresh Sequencer */
+	ret = mt9t11x_mcu_write(client, VAR8(1, 0), 0x05);
+	if (ret < 0)
+		return ret;
+
+#ifdef TEST_PATTERN
+	ret = mt9t11x_mcu_write(client, VAR(24, 3), 0x100);
+	if (ret < 0)
+		return ret;
+	ret = mt9t11x_mcu_write(client, VAR(24, 37), 0x0B);
+#endif
+
+	return ret;
+}
+
+static int mt9t11x_optimal_power_consumption(const struct i2c_client *client)
+{
+	int ret;
+
+	/* Analog setting B */
+	ret = mt9t11x_reg_write(client, 0x3084, 0x2409);
+	if (ret < 0)
+		return ret;
+	ret = mt9t11x_reg_write(client, 0x3092, 0x0A49);
+	if (ret < 0)
+		return ret;
+	ret = mt9t11x_reg_write(client, 0x3094, 0x4949);
+	if (ret < 0)
+		return ret;
+	ret = mt9t11x_reg_write(client, 0x3096, 0x4950);
+
+	return ret;
+}
+
+static int mt9t11x_blooming_row_pattern(const struct i2c_client *client)
+{
+	int ret;
+
+	/* Improve high light image quality */
+
+	/* [CAM1_CTX_A_COARSE_ITMIN] */
+	ret = mt9t11x_mcu_write(client, VAR(18, 21), 0x0004);
+	if (ret < 0)
+		return ret;
+	/* [CAM1_CTX_B_COARSE_ITMIN] */
+	ret = mt9t11x_mcu_write(client, VAR(18, 93), 0x0004);
+
+	return ret;
+}
+
+static int mt9t11x_set_orientation(const struct i2c_client *client,
+				   u32 mask, u32 flip)
+{
+	int ret;
+
+	flip &= mask;
+
+	dev_dbg(&client->dev, "%s:\n", __func__);
+
+	/* [CAM1_CTX_A_READ_MODE] */
+	ret = mt9t11x_mcu_mask_set(client, VAR(18, 12), mask, flip);
+	if (ret < 0)
+		return ret;
+	/* [CAM1_CTX_A_PIXEL_ORDER] */
+	ret = mt9t11x_mcu_mask_set(client, VAR8(18, 14), mask, flip);
+	if (ret < 0)
+		return ret;
+
+	/* [CAM1_CTX_B_READ_MODE] */
+	ret = mt9t11x_mcu_mask_set(client, VAR(18, 84), mask, flip);
+	if (ret < 0)
+		return ret;
+	/* [CAM1_CTX_B_PIXEL_ORDER] */
+	ret = mt9t11x_mcu_mask_set(client, VAR8(18, 86), mask, flip);
+	if (ret < 0)
+		return ret;
+
+	/* [SEQ_CMD] */
+	ret = mt9t11x_mcu_write(client, VAR8(1, 0), 0x06);
+
+	return ret;
+}
+
+static int mt9t11x_init_camera_optimized(const struct i2c_client *client)
+{
+	int ret;
+
+	dev_dbg(&client->dev, "%s:\n", __func__);
+
+	ret = mt9t11x_sysctl_startup(client);
+	if (ret < 0)
+		return ret;
+	ret = mt9t11x_mcu_powerup_stop_enable(client);
+	if (ret < 0)
+		return ret;
+	ret = mt9t11x_go(client);
+	if (ret < 0)
+		return ret;
+
+	ret = mt9t11x_custom_setup(client);
+	if (ret < 0)
+		return ret;
+
+	ret = mt9t11x_high_speed_overrides(client);
+	if (ret < 0)
+		return ret;
+
+	ret = mt9t11x_optimal_power_consumption(client);
+	if (ret < 0)
+		return ret;
+
+	ret = mt9t11x_blooming_row_pattern(client);
+	if (ret < 0)
+		return ret;
+
+	ret = mt9t11x_continue(client);
+
+	return ret;
+}
+
+static int mt9t11x_init_setting(const struct i2c_client *client)
+{
+	struct mt9t11x_priv *priv = to_mt9t11x(client);
+	int ret;
+
+	dev_dbg(&client->dev, "%s:\n", __func__);
+
+	/* Adaptive Output Clock (A) */
+	ret = mt9t11x_mcu_mask_set(client, VAR(26, 160), 0x0040, 0x0000);
+	if (ret < 0)
+		return ret;
+	/* Read Mode (A) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 12), 0x0024);
+	if (ret < 0)
+		return ret;
+	/* Fine Correction (A) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 15), 0x00CC);
+	if (ret < 0)
+		return ret;
+	/* Fine IT Min (A) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 17), 0x01f1);
+	if (ret < 0)
+		return ret;
+	/* Fine IT Max Margin (A) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 19), 0x00fF);
+	if (ret < 0)
+		return ret;
+	/* Base Frame Lines (A) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 29), 0x032D);
+	if (ret < 0)
+		return ret;
+	/* Min Line Length (A) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 31), 0x073a);
+	if (ret < 0)
+		return ret;
+	/* Line Length (A) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 37), 0x07d0);
+	if (ret < 0)
+		return ret;
+	/* Adaptive Output Clock (B) */
+	ret = mt9t11x_mcu_mask_set(client, VAR(27, 160), 0x0040, 0x0000);
+	if (ret < 0)
+		return ret;
+	/* Row Start (B) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 74), 0x004);
+	if (ret < 0)
+		return ret;
+	/* Column Start (B) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 76), 0x004);
+	if (ret < 0)
+		return ret;
+	/* Row End (B) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 78), 0x60B);
+	if (ret < 0)
+		return ret;
+	/* Column End (B) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 80), 0x80B);
+	if (ret < 0)
+		return ret;
+	/* Fine Correction (B) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 87), 0x008C);
+	if (ret < 0)
+		return ret;
+	/* Fine IT Min (B) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 89), 0x01F1);
+	if (ret < 0)
+		return ret;
+	/* Fine IT Max Margin (B) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 91), 0x00FF);
+	if (ret < 0)
+		return ret;
+	/* Base Frame Lines (B) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 101), 0x0668);
+	if (ret < 0)
+		return ret;
+	/* Min Line Length (B) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 103), 0x0AF0);
+	if (ret < 0)
+		return ret;
+	/* Line Length (B) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 109), 0x0AF0);
+	if (ret < 0)
+		return ret;
+	/*
+	 * Flicker Dectection registers
+	 * This section should be replaced whenever new Timing file is
+	 * generated.
+	 * All the following registers need to be replaced
+	 * Following registers are generated from Register Wizard but user can
+	 * modify them. For detail see auto flicker detection tuning
+	 */
+	/* FD_FDPERIOD_SELECT */
+	ret = mt9t11x_mcu_write(client, VAR8(8, 5), 0x01);
+	if (ret < 0)
+		return ret;
+	/* PRI_B_CONFIG_FD_ALGO_RUN */
+	ret = mt9t11x_mcu_write(client, VAR(27, 17), 0x0003);
+	if (ret < 0)
+		return ret;
+	/* PRI_A_CONFIG_FD_ALGO_RUN */
+	ret = mt9t11x_mcu_write(client, VAR(26, 17), 0x0003);
+	if (ret < 0)
+		return ret;
+	/*
+	 * AFD range detection tuning registers
+	 */
+	/* search_f1_50 */
+	ret = mt9t11x_mcu_write(client, VAR8(18, 165), 0x25);
+	if (ret < 0)
+		return ret;
+	/* search_f2_50 */
+	ret = mt9t11x_mcu_write(client, VAR8(18, 166), 0x28);
+	if (ret < 0)
+		return ret;
+	/* search_f1_60 */
+	ret = mt9t11x_mcu_write(client, VAR8(18, 167), 0x2C);
+	if (ret < 0)
+		return ret;
+	/* search_f2_60 */
+	ret = mt9t11x_mcu_write(client, VAR8(18, 168), 0x2F);
+	if (ret < 0)
+		return ret;
+	/* period_50Hz (A) */
+	ret = mt9t11x_mcu_write(client, VAR8(18, 68), 0xBA);
+	if (ret < 0)
+		return ret;
+	/* period_50Hz (A MSB) */
+	ret = mt9t11x_mcu_write(client, VAR8(18, 303), 0x00);
+	if (ret < 0)
+		return ret;
+	/* period_60Hz (A) */
+	ret = mt9t11x_mcu_write(client, VAR8(18, 69), 0x9B);
+	if (ret < 0)
+		return ret;
+	/* period_60Hz (A MSB) */
+	ret = mt9t11x_mcu_write(client, VAR8(18, 301), 0x00);
+	if (ret < 0)
+		return ret;
+	/* period_50Hz (B) */
+	ret = mt9t11x_mcu_write(client, VAR8(18, 140), 0x82);
+	if (ret < 0)
+		return ret;
+	/* period_50Hz (B) MSB */
+	ret = mt9t11x_mcu_write(client, VAR8(18, 304), 0x00);
+	if (ret < 0)
+		return ret;
+	/* period_60Hz (B) */
+	ret = mt9t11x_mcu_write(client, VAR8(18, 141), 0x6D);
+	if (ret < 0)
+		return ret;
+	/* period_60Hz (B) MSB */
+	ret = mt9t11x_mcu_write(client, VAR8(18, 302), 0x00);
+	if (ret < 0)
+		return ret;
+	/* FD Mode */
+	ret = mt9t11x_mcu_write(client, VAR8(8, 2), 0x10);
+	if (ret < 0)
+		return ret;
+	/* Stat_min */
+	ret = mt9t11x_mcu_write(client, VAR8(8, 9), 0x02);
+	if (ret < 0)
+		return ret;
+	/* Stat_max */
+	ret = mt9t11x_mcu_write(client, VAR8(8, 10), 0x03);
+	if (ret < 0)
+		return ret;
+	/* Min_amplitude */
+	ret = mt9t11x_mcu_write(client, VAR8(8, 12), 0x0A);
+	if (ret < 0)
+		return ret;
+	/* RX FIFO Watermark (A) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 70), 0x0014);
+	if (ret < 0)
+		return ret;
+	/* RX FIFO Watermark (B) */
+	ret = mt9t11x_mcu_write(client, VAR(18, 142), 0x0014);
+	if (ret < 0)
+		return ret;
+
+	ret = mt9t11x_set_a_frame_size(client,
+				       priv->frame.width,
+				       priv->frame.height);
+
+	return ret;
+}
+
+static int mt9t11x_init_camera(const struct i2c_client *client)
+{
+	int ret;
+
+	dev_dbg(&client->dev, "%s:\n", __func__);
+
+	ret = mt9t11x_sysctl_startup(client);
+	if (ret < 0)
+		return ret;
+	ret = mt9t11x_mcu_powerup_stop_enable(client);
+	if (ret < 0)
+		return ret;
+	ret = mt9t11x_go(client);
+	if (ret < 0)
+		return ret;
+
+	ret = mt9t11x_init_setting(client);
+	if (ret < 0)
+		return ret;
+
+	ret = mt9t11x_high_speed_overrides(client);
+	if (ret < 0)
+		return ret;
+
+	ret = mt9t11x_optimal_power_consumption(client);
+	if (ret < 0)
+		return ret;
+
+	ret = mt9t11x_blooming_row_pattern(client);
+	if (ret < 0)
+		return ret;
+
+	ret = mt9t11x_continue(client);
+	if (ret < 0)
+		return ret;
+
+	return ret;
+}
+
+/************************************************************************
+ *			v4l2_subdev_core_ops
+ ***********************************************************************/
+
+#ifdef CONFIG_VIDEO_ADV_DEBUG
+static int mt9t11x_g_register(struct v4l2_subdev *sd,
+			      struct v4l2_dbg_register *reg)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(sd);
+	int                ret;
+
+	reg->size = 2;
+	ret = mt9t11x_reg_read(client, reg->reg);
+
+	reg->val = (__u64)ret;
+
+	return 0;
+}
+
+static int mt9t11x_s_register(struct v4l2_subdev *sd,
+			      const struct v4l2_dbg_register *reg)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(sd);
+	int ret;
+
+	ret = mt9t11x_reg_write(client, reg->reg, reg->val);
+
+	return ret;
+}
+#endif
+
+static void __mt9t11x_set_power(struct mt9t11x_priv *priv, int on)
+{
+	struct i2c_client *client = priv->client;
+
+	dev_dbg(&client->dev, "%s: on: %d\n", __func__, on);
+	on = (on) ? 1 : 0;
+
+	if (priv->power == on)
+		return;
+
+	if (on) {
+		if (priv->powerdown_gpio)
+			gpiod_set_value_cansleep(priv->powerdown_gpio, 0);
+		if (priv->reset_gpio)
+			gpiod_set_value_cansleep(priv->reset_gpio, 0);
+		usleep_range(25000, 26000);
+	} else {
+		if (priv->powerdown_gpio)
+			gpiod_set_value_cansleep(priv->powerdown_gpio, 1);
+		if (priv->reset_gpio)
+			gpiod_set_value_cansleep(priv->reset_gpio, 1);
+	}
+
+	priv->power = on;
+}
+
+static int mt9t11x_s_power(struct v4l2_subdev *sd, int on)
+{
+	struct mt9t11x_priv *priv = sd_to_mt9t11x(sd);
+	struct i2c_client *client = priv->client;
+
+	dev_dbg(&client->dev, "%s: on: %d\n", __func__, on);
+
+	mutex_lock(&priv->lock);
+	__mt9t11x_set_power(priv, on);
+	mutex_unlock(&priv->lock);
+	return 0;
+}
+
+static struct v4l2_subdev_core_ops mt9t11x_subdev_core_ops = {
+	.log_status = v4l2_ctrl_subdev_log_status,
+	.subscribe_event = v4l2_ctrl_subdev_subscribe_event,
+	.unsubscribe_event = v4l2_event_subdev_unsubscribe,
+	.s_power	= mt9t11x_s_power,
+#ifdef CONFIG_VIDEO_ADV_DEBUG
+	.g_register	= mt9t11x_g_register,
+	.s_register	= mt9t11x_s_register,
+#endif
+};
+
+/*
+ * V4L2 controls
+ */
+static int mt9t11x_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct mt9t11x_priv *priv = ctrl_to_mt9t11x(ctrl);
+	struct i2c_client *client = priv->client;
+	int ret = -EINVAL;
+
+	dev_dbg(&client->dev, "%s: ctrl_id:0x%x (%s), value: %d\n",
+		__func__, ctrl->id, ctrl->name, ctrl->val);
+
+	mutex_lock(&priv->lock);
+	/*
+	 * If the device is not powered up now, postpone applying control's
+	 * value to the hardware, until it is ready to accept commands.
+	 */
+	if (priv->power == 0) {
+		mutex_unlock(&priv->lock);
+		return 0;
+	}
+
+	switch (ctrl->id) {
+	case V4L2_CID_VFLIP:
+		mt9t11x_set_orientation(client, 0x2, (ctrl->val) ? 2 : 0);
+		ret = 0;
+		break;
+	case V4L2_CID_HFLIP:
+		mt9t11x_set_orientation(client, 0x1, (ctrl->val) ? 1 : 0);
+		ret = 0;
+		break;
+	}
+
+	mutex_unlock(&priv->lock);
+	return ret;
+}
+
+static const struct v4l2_ctrl_ops mt9t11x_ctrl_ops = {
+	.s_ctrl	= mt9t11x_s_ctrl,
+};
+
+static int mt9t11x_initialize_controls(struct mt9t11x_priv *priv)
+{
+	const struct v4l2_ctrl_ops *ops = &mt9t11x_ctrl_ops;
+	struct i2c_client *client = priv->client;
+	struct v4l2_subdev *sd = &priv->subdev;
+	struct v4l2_ctrl_handler *hdl = &priv->handler;
+	int ret;
+
+	dev_dbg(&client->dev, "%s:\n", __func__);
+
+	ret = v4l2_ctrl_handler_init(hdl, 2);
+	if (ret < 0)
+		return ret;
+
+	priv->vflip = v4l2_ctrl_new_std(hdl, ops, V4L2_CID_VFLIP, 0, 1, 1, 0);
+	priv->hflip = v4l2_ctrl_new_std(hdl, ops, V4L2_CID_HFLIP, 0, 1, 1, 0);
+	sd->ctrl_handler = hdl;
+	if (priv->handler.error)
+		return priv->handler.error;
+
+	return 0;
+}
+
+/************************************************************************
+ *			v4l2_subdev_video_ops
+ ***********************************************************************/
+static int mt9t11x_set_params(struct mt9t11x_priv *priv,
+			      const struct v4l2_rect *rect,
+			      u32 code)
+{
+	int i;
+
+	/*
+	 * get color format
+	 */
+	for (i = 0; i < priv->num_formats; i++)
+		if (mt9t11x_cfmts[i].code == code)
+			break;
+
+	if (i == priv->num_formats)
+		return -EINVAL;
+
+	priv->frame  = *rect;
+
+	/*
+	 * frame size check
+	 */
+	mt9t11x_frame_check(&priv->frame.width, &priv->frame.height,
+			    &priv->frame.left, &priv->frame.top);
+
+	priv->format = mt9t11x_cfmts + i;
+
+	return 0;
+}
+
+static int mt9t11x_s_stream(struct v4l2_subdev *sd, int enable)
+{
+	struct mt9t11x_priv *priv = sd_to_mt9t11x(sd);
+	struct i2c_client *client = priv->client;
+	int ret = 0;
+	int optimize = 0;
+	struct v4l2_rect rect = {
+		.width = priv->frame.width,
+		.height = priv->frame.height,
+		.left = priv->frame.left,
+		.top = priv->frame.top,
+	};
+	u16 param;
+
+	dev_dbg(&client->dev, "%s: enable: %d\n", __func__, enable);
+
+	mutex_lock(&priv->lock);
+
+	if (priv->streaming == enable)
+		goto unlock;
+
+	if (!enable) {
+		/* Stop Streaming Sequence */
+		mt9t11x_streaming(priv, false);
+		__mt9t11x_set_power(priv, 0);
+		priv->streaming = enable;
+		goto unlock;
+	}
+
+	mt9t11x_set_params(priv, &rect, priv->format->code);
+
+	__mt9t11x_set_power(priv, 1);
+
+	/* fill the structure with new resolution parameters */
+	optimize = mt9t11x_set_resolution_params(client);
+
+	if (optimize) {
+		ret = mt9t11x_init_camera_optimized(client);
+		if (ret < 0)
+			goto unlock;
+	} else {
+		ret = mt9t11x_init_camera(client);
+		if (ret < 0)
+			goto unlock;
+	}
+	/*
+	 * By default data is sampled on falling edge of pixclk.
+	 * Change the default to be rising edge. i.e. Invert PCLK
+	 */
+	param = (priv->info->flags & V4L2_MBUS_PCLK_SAMPLE_RISING) ?
+		0x0001 : 0x0000;
+	ret = mt9t11x_reg_write(client, 0x3C20, param);
+	if (ret < 0)
+		goto unlock;
+	usleep_range(5000, 6000);
+
+	ret = mt9t11x_mcu_write(client, VAR(26, 7), priv->format->fmt);
+	if (ret < 0)
+		goto unlock;
+	ret = mt9t11x_mcu_write(client, VAR(26, 9), priv->format->order);
+	if (ret < 0)
+		goto unlock;
+
+	ret = mt9t11x_mcu_write(client, VAR8(1, 0), 0x06);
+	if (ret < 0)
+		goto unlock;
+
+	if (priv->flags & MT9T11x_FLAG_VFLIP)
+		v4l2_ctrl_s_ctrl(priv->vflip, 1);
+
+	/* Make sure H/W is consistent with current control settings */
+	ret = mt9t11x_set_orientation(client, 0x3,
+				      (v4l2_ctrl_g_ctrl(priv->vflip) << 1 |
+				       v4l2_ctrl_g_ctrl(priv->hflip)));
+	if (ret < 0)
+		goto unlock;
+
+	dev_dbg(&client->dev, "format : %04x\n", priv->format->code);
+	dev_dbg(&client->dev, "size   : %d x %d\n",
+		priv->frame.width,
+		priv->frame.height);
+
+	priv->streaming = enable;
+	mt9t11x_streaming(priv, true);
+
+	CLOCK_INFO(client, EXT_CLOCK);
+
+unlock:
+	mutex_unlock(&priv->lock);
+	return ret;
+}
+
+static void mt9t11x_get_default_format(struct mt9t11x_priv *priv,
+				       struct v4l2_mbus_framefmt *mf)
+{
+	struct v4l2_rect rect = {
+		.width = VGA_WIDTH,
+		.height = VGA_HEIGHT,
+		.left = (MAX_WIDTH - VGA_WIDTH) / 2,
+		.top = (MAX_HEIGHT - VGA_HEIGHT) / 2,
+	};
+
+	mt9t11x_set_params(priv, &rect, MEDIA_BUS_FMT_UYVY8_2X8);
+
+	/* Need fixing */
+	mf->width = rect.width;
+	mf->height = rect.height;
+	mf->colorspace = mt9t11x_cfmts[0].colorspace;
+	mf->code = mt9t11x_cfmts[0].code;
+
+	mf->field = V4L2_FIELD_NONE;
+}
+
+static int mt9t11x_get_fmt(struct v4l2_subdev *sd,
+			   struct v4l2_subdev_pad_config *cfg,
+			   struct v4l2_subdev_format *fmt)
+{
+	struct mt9t11x_priv *priv = sd_to_mt9t11x(sd);
+	struct v4l2_mbus_framefmt *mf;
+
+	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) {
+		mf = v4l2_subdev_get_try_format(sd, cfg, 0);
+		mutex_lock(&priv->lock);
+		fmt->format = *mf;
+		mutex_unlock(&priv->lock);
+		return 0;
+	}
+
+	mutex_lock(&priv->lock);
+	mf = &fmt->format;
+	mf->width	= priv->frame.width;
+	mf->height	= priv->frame.height;
+	mf->colorspace	= priv->format->colorspace;
+	mf->code	= priv->format->code;
+	mf->field	= V4L2_FIELD_NONE;
+	mutex_unlock(&priv->lock);
+
+	return 0;
+}
+
+static void __mt9t11x_try_frame_size(struct v4l2_mbus_framefmt *mf)
+{
+	const struct mt9t11x_framesize *fsize = &mt9t11x_framesizes[0];
+	const struct mt9t11x_framesize *match = NULL;
+	int i = ARRAY_SIZE(mt9t11x_framesizes);
+	unsigned int min_err = UINT_MAX;
+
+	while (i--) {
+		int err = abs(fsize->width - mf->width)
+				+ abs(fsize->height - mf->height);
+		if (err < min_err) {
+			min_err = err;
+			match = fsize;
+		}
+		fsize++;
+	}
+
+	if (!match)
+		match = &mt9t11x_framesizes[0];
+
+	mf->width  = match->width;
+	mf->height = match->height;
+}
+
+static int mt9t11x_set_fmt(struct v4l2_subdev *sd,
+			   struct v4l2_subdev_pad_config *cfg,
+			   struct v4l2_subdev_format *fmt)
+{
+	struct mt9t11x_priv *priv = sd_to_mt9t11x(sd);
+	int index = priv->num_formats;
+	struct v4l2_mbus_framefmt *mf = &fmt->format;
+	int ret = 0;
+	struct v4l2_rect rect;
+
+	__mt9t11x_try_frame_size(mf);
+	rect.width = mf->width;
+	rect.height = mf->height;
+	rect.left = priv->frame.left;
+	rect.top = priv->frame.top;
+
+	while (--index >= 0)
+		if (mt9t11x_cfmts[index].code == mf->code)
+			break;
+
+	if (index < 0)
+		return -EINVAL;
+
+	mf->colorspace = mt9t11x_cfmts[index].colorspace;
+	mf->code = mt9t11x_cfmts[index].code;
+	mf->field = V4L2_FIELD_NONE;
+
+	mutex_lock(&priv->lock);
+
+	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) {
+		mf = v4l2_subdev_get_try_format(sd, cfg, fmt->pad);
+		*mf = fmt->format;
+	} else {
+		ret = mt9t11x_set_params(priv, &rect, mf->code);
+	}
+
+	mutex_unlock(&priv->lock);
+
+	return ret;
+}
+
+static int mt9t11x_enum_mbus_code(struct v4l2_subdev *sd,
+				  struct v4l2_subdev_pad_config *cfg,
+				  struct v4l2_subdev_mbus_code_enum *code)
+{
+	struct mt9t11x_priv *priv = sd_to_mt9t11x(sd);
+
+	if (code->index >= priv->num_formats)
+		return -EINVAL;
+
+	code->code = mt9t11x_cfmts[code->index].code;
+
+	return 0;
+}
+
+static int mt9t11x_enum_frame_sizes(struct v4l2_subdev *sd,
+				    struct v4l2_subdev_pad_config *cfg,
+				    struct v4l2_subdev_frame_size_enum *fse)
+{
+	struct mt9t11x_priv *priv = sd_to_mt9t11x(sd);
+	int i = priv->num_formats;
+
+	if (fse->index >= ARRAY_SIZE(mt9t11x_framesizes))
+		return -EINVAL;
+
+	while (--i)
+		if (mt9t11x_cfmts[i].code == fse->code)
+			break;
+
+	fse->code = mt9t11x_cfmts[i].code;
+
+	fse->min_width  = mt9t11x_framesizes[fse->index].width;
+	fse->max_width  = fse->min_width;
+	fse->max_height = mt9t11x_framesizes[fse->index].height;
+	fse->min_height = fse->max_height;
+
+	return 0;
+}
+
+static const struct v4l2_subdev_video_ops mt9t11x_subdev_video_ops = {
+	.s_stream	= mt9t11x_s_stream,
+};
+
+static const struct v4l2_subdev_pad_ops mt9t11x_subdev_pad_ops = {
+	.enum_mbus_code		= mt9t11x_enum_mbus_code,
+	.enum_frame_size	= mt9t11x_enum_frame_sizes,
+	.get_fmt		= mt9t11x_get_fmt,
+	.set_fmt		= mt9t11x_set_fmt,
+};
+
+/************************************************************************
+ *			i2c driver
+ ***********************************************************************/
+static struct v4l2_subdev_ops mt9t11x_subdev_ops = {
+	.core	= &mt9t11x_subdev_core_ops,
+	.video	= &mt9t11x_subdev_video_ops,
+	.pad   = &mt9t11x_subdev_pad_ops,
+};
+
+/*
+ * V4L2 subdev internal operations
+ */
+static int mt9t11x_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
+{
+	struct mt9t11x_priv *priv = sd_to_mt9t11x(sd);
+	struct i2c_client *client = priv->client;
+	struct v4l2_mbus_framefmt *mf;
+
+	dev_dbg(&client->dev, "%s:\n", __func__);
+
+	mf = v4l2_subdev_get_try_format(sd, fh->pad, 0);
+	mt9t11x_get_default_format(priv, mf);
+
+	return 0;
+}
+
+static const struct v4l2_subdev_internal_ops mt9t11x_subdev_internal_ops = {
+	.open = mt9t11x_open,
+};
+
+static int mt9t11x_camera_probe(struct i2c_client *client)
+{
+	struct mt9t11x_priv *priv = to_mt9t11x(client);
+	const char          *devname;
+	int                  chipid;
+	int                  custom_rev;
+	int		     ret = 0;
+
+	__mt9t11x_set_power(priv, 1);
+
+	/*
+	 * check and show chip ID
+	 */
+	chipid = mt9t11x_reg_read(client, 0x0000);
+
+	switch (chipid) {
+	case 0x2680:
+		devname = "mt9t111";
+		/*
+		 * Looks like only uyvy is supported
+		 * so limiting available formats.
+		 */
+		priv->num_formats = 1;
+		break;
+	case 0x2682:
+		devname = "mt9t112";
+		priv->num_formats = ARRAY_SIZE(mt9t11x_cfmts);
+		break;
+	default:
+		dev_err(&client->dev, "Product ID error %04x\n", chipid);
+		ret = -ENODEV;
+		goto done;
+	}
+
+	custom_rev = mt9t11x_reg_read(client, 0x31FE);
+
+	dev_info(&client->dev, "%s chip ID %04x rev %04x\n",
+		 devname, chipid, custom_rev);
+
+done:
+	__mt9t11x_set_power(priv, 0);
+	return ret;
+}
+
+static struct mt9t11x_camera_info *
+mt9t11x_get_pdata(struct i2c_client *client)
+{
+	struct mt9t11x_camera_info *pdata;
+	struct device_node *endpoint;
+	struct v4l2_fwnode_endpoint *v4l2_endpoint;
+
+	dev_dbg(&client->dev, "_get_pdata invoked\n");
+
+	if (!IS_ENABLED(CONFIG_OF) || !client->dev.of_node)
+		return client->dev.platform_data;
+
+	dev_dbg(&client->dev, "_get_pdata: DT Node found\n");
+
+	endpoint = of_graph_get_next_endpoint(client->dev.of_node, NULL);
+	if (!endpoint)
+		return NULL;
+
+	dev_dbg(&client->dev, "_get_pdata: endpoint found\n");
+
+	pdata = devm_kzalloc(&client->dev, sizeof(*pdata), GFP_KERNEL);
+	if (!pdata)
+		goto done;
+
+	v4l2_endpoint = &pdata->endpoint;
+	v4l2_fwnode_endpoint_parse(of_fwnode_handle(endpoint), v4l2_endpoint);
+
+	if (of_property_read_u32(endpoint, "input-clock-freq", &pdata->mclk)) {
+		dev_err(&client->dev, "input-clock-freq property not found\n");
+		goto err_out;
+	} else if (pdata->mclk > MT9T11x_MAX_EXT_CLK) {
+		dev_err(&client->dev, "input-clock-freq property exceed max value\n");
+		goto err_out;
+	}
+	dev_info(&client->dev, "input-clock-freq: %d\n", pdata->mclk);
+
+	if (of_property_read_u32(endpoint, "pixel-clock-freq", &pdata->pclk)) {
+		dev_err(&client->dev, "pixel-clock-freq property not found\n");
+		goto err_out;
+	} else if (pdata->pclk > MT9T11x_MAX_PIXEL_CLK) {
+		dev_err(&client->dev, "pixel-clock-freq property exceed max value\n");
+		goto err_out;
+	}
+	dev_info(&client->dev, "pixel-clock-freq: %d\n", pdata->pclk);
+
+	/* Just copy them for now */
+	if (pdata->endpoint.bus_type == V4L2_MBUS_PARALLEL)
+		pdata->flags = pdata->endpoint.bus.parallel.flags;
+	else
+		pdata->flags = 0;
+done:
+	of_node_put(endpoint);
+	return pdata;
+err_out:
+	of_node_put(endpoint);
+	return NULL;
+}
+
+static int mt9t11x_probe(struct i2c_client *client,
+			 const struct i2c_device_id *did)
+{
+	struct mt9t11x_priv *priv;
+	struct v4l2_subdev *sd;
+	struct mt9t11x_camera_info *info = mt9t11x_get_pdata(client);
+	struct v4l2_mbus_framefmt mf;
+
+	int ret;
+	struct gpio_desc *gpio;
+
+	if (!info) {
+		dev_err(&client->dev, "mt9t11x: missing platform data!\n");
+		return -EINVAL;
+	}
+
+	priv = devm_kzalloc(&client->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->info = info;
+
+	mutex_init(&priv->lock);
+	priv->client = client;
+
+	gpio = devm_gpiod_get(&client->dev, "reset", GPIOD_OUT_LOW);
+	if (IS_ERR(gpio)) {
+		if (PTR_ERR(gpio) != -ENOENT)
+			return PTR_ERR(gpio);
+		gpio = NULL;
+	}
+	priv->reset_gpio = gpio;
+
+	gpio = devm_gpiod_get(&client->dev, "powerdown", GPIOD_OUT_LOW);
+	if (IS_ERR(gpio)) {
+		if (PTR_ERR(gpio) != -ENOENT)
+			return PTR_ERR(gpio);
+		gpio = NULL;
+	}
+	priv->powerdown_gpio = gpio;
+
+	gpio = devm_gpiod_get(&client->dev, "oscen", GPIOD_OUT_LOW);
+	if (IS_ERR(gpio)) {
+		if (PTR_ERR(gpio) != -ENOENT)
+			return PTR_ERR(gpio);
+		gpio = NULL;
+	}
+	priv->oscen_gpio = gpio;
+
+	gpio = devm_gpiod_get(&client->dev, "bufen", GPIOD_OUT_LOW);
+	if (IS_ERR(gpio)) {
+		if (PTR_ERR(gpio) != -ENOENT)
+			return PTR_ERR(gpio);
+		gpio = NULL;
+	}
+	priv->bufen_gpio = gpio;
+
+	gpio = devm_gpiod_get(&client->dev, "camen", GPIOD_OUT_LOW);
+	if (IS_ERR(gpio)) {
+		if (PTR_ERR(gpio) != -ENOENT)
+			return PTR_ERR(gpio);
+		gpio = NULL;
+	}
+	priv->camen_gpio = gpio;
+
+	/* Do these here for now */
+	if (priv->bufen_gpio)
+		gpiod_set_value_cansleep(priv->bufen_gpio, 1);
+
+	if (priv->oscen_gpio)
+		gpiod_set_value_cansleep(priv->oscen_gpio, 1);
+
+	if (priv->camen_gpio)
+		gpiod_set_value_cansleep(priv->camen_gpio, 1);
+
+	sd = &priv->subdev;
+	v4l2_i2c_subdev_init(sd, client, &mt9t11x_subdev_ops);
+	strlcpy(sd->name, DRIVER_NAME, sizeof(sd->name));
+
+	sd->internal_ops = &mt9t11x_subdev_internal_ops;
+	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE |
+		     V4L2_SUBDEV_FL_HAS_EVENTS;
+
+#if defined(CONFIG_MEDIA_CONTROLLER)
+	priv->pad.flags = MEDIA_PAD_FL_SOURCE;
+	sd->entity.function = MEDIA_ENT_F_CAM_SENSOR;
+	ret = media_entity_pads_init(&sd->entity, 1, &priv->pad);
+	if (ret < 0)
+		return ret;
+#endif
+	ret = mt9t11x_initialize_controls(priv);
+	if (ret < 0)
+		goto err_me;
+
+	ret = mt9t11x_camera_probe(client);
+	if (ret < 0)
+		goto err_ctrls;
+
+	mt9t11x_get_default_format(priv, &mf);
+
+	/* Calculate the PLL register value needed */
+	mt9t11x_pll_calc_params(priv);
+
+	ret = v4l2_async_register_subdev(sd);
+	if (ret)
+		goto err_ctrls;
+
+	dev_info(&client->dev, "%s sensor driver registered !!\n", sd->name);
+
+	return 0;
+err_ctrls:
+	v4l2_ctrl_handler_free(sd->ctrl_handler);
+err_me:
+#if defined(CONFIG_MEDIA_CONTROLLER)
+	media_entity_cleanup(&sd->entity);
+#endif
+	return ret;
+}
+
+static int mt9t11x_remove(struct i2c_client *client)
+{
+	struct mt9t11x_priv *priv = to_mt9t11x(client);
+
+	v4l2_async_unregister_subdev(&priv->subdev);
+	v4l2_device_unregister_subdev(&priv->subdev);
+#if defined(CONFIG_MEDIA_CONTROLLER)
+	media_entity_cleanup(&priv->subdev.entity);
+#endif
+	v4l2_ctrl_handler_free(priv->subdev.ctrl_handler);
+	return 0;
+}
+
+static const struct i2c_device_id mt9t11x_id[] = {
+	{ "mt9t111", 0 },
+	{ "mt9t112", 0 },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(i2c, mt9t11x_id);
+
+#if IS_ENABLED(CONFIG_OF)
+static const struct of_device_id mt9t11x_of_match[] = {
+	{ .compatible = "aptina,mt9t111", .data = (void *)MT9T111_ID},
+	{ .compatible = "aptina,mt9t112", .data = (void *)MT9T112_ID},
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, mt9t11x_of_match);
+#endif
+
+static struct i2c_driver mt9t11x_i2c_driver = {
+	.driver = {
+		.name	= DRIVER_NAME,
+		.of_match_table = of_match_ptr(mt9t11x_of_match),
+	},
+	.probe		= mt9t11x_probe,
+	.remove		= mt9t11x_remove,
+	.id_table	= mt9t11x_id,
+};
+
+module_i2c_driver(mt9t11x_i2c_driver);
+
+MODULE_AUTHOR("Benoit Parrot <bparrot@ti.com>");
+MODULE_DESCRIPTION("MT9T11x CMOS Image Sensor driver");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/media/i2c/ov1063x.c b/drivers/media/i2c/ov1063x.c
--- a/drivers/media/i2c/ov1063x.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/media/i2c/ov1063x.c	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,1070 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * OmniVision OV1063X Camera Driver
+ *
+ * Based on the original driver written by Phil Edworthy.
+ * Copyright (C) 2013 Phil Edworthy
+ * Copyright (C) 2013 Renesas Electronics
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This driver has been tested at QVGA, VGA and 720p, and 1280x800 at up to
+ * 30fps and it should work at any resolution in between and any frame rate
+ * up to 30fps.
+ *
+ * FIXME:
+ *  Horizontal flip (mirroring) does not work correctly. The image is flipped,
+ *  but the colors are wrong.
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/i2c.h>
+#include <linux/gpio.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/regulator/consumer.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+#include <linux/videodev2.h>
+#include <linux/of.h>
+#include <linux/of_gpio.h>
+#include <linux/gpio/consumer.h>
+#include <linux/of_device.h>
+
+#include <linux/v4l2-mediabus.h>
+#include <media/media-entity.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-event.h>
+#include <media/v4l2-image-sizes.h>
+#include <media/v4l2-subdev.h>
+#include <media/v4l2-mediabus.h>
+#include <media/v4l2-common.h>
+#include <linux/of_graph.h>
+#include <media/v4l2-fwnode.h>
+
+#include "ov1063x_regs.h"
+
+/* Register definitions */
+#define	OV1063X_VFLIP			0x381c
+#define	 OV1063X_VFLIP_ON		GENMASK(7, 6)
+#define	 OV1063X_VFLIP_SUBSAMPLE	BIT_MASK(0)
+#define	OV1063X_HMIRROR			0x381d
+#define	 OV1063X_HMIRROR_ON		GENMASK(1, 0)
+#define OV1063X_PID			0x300a
+#define OV1063X_VER			0x300b
+
+#define OV1063X_FORMAT_CTRL00		0x4300
+#define   OV1063X_FORMAT_YUYV		0x38
+#define   OV1063X_FORMAT_YYYU		0x39
+#define   OV1063X_FORMAT_UYVY		0x3A
+#define   OV1063X_FORMAT_VYUY		0x3B
+
+/* IDs */
+#define OV10633_VERSION_REG		0xa630
+#define OV10635_VERSION_REG		0xa635
+#define OV1063X_VERSION(pid, ver)	(((pid) << 8) | ((ver) & 0xff))
+
+enum ov1063x_model {
+	SENSOR_OV10633,
+	SENSOR_OV10635,
+};
+
+#define OV1063X_SENSOR_WIDTH		1312
+#define OV1063X_SENSOR_HEIGHT		814
+
+#define OV1063X_MAX_WIDTH		1280
+#define OV1063X_MAX_HEIGHT		800
+
+#define MAX_NUM_GPIOS			20
+
+struct ov1063x_color_format {
+	u32 code;
+	u32 colorspace;
+};
+
+struct ov1063x_framesize {
+	u16 width;
+	u16 height;
+};
+
+struct ov1063x_priv {
+	struct v4l2_subdev		subdev;
+	struct v4l2_async_subdev	asd;
+	struct v4l2_ctrl_handler	hdl;
+	int				model;
+	int				revision;
+	int				xvclk_rate;
+	/* Protects the struct fields below */
+	struct mutex lock;
+
+	int				fps_numerator;
+	int				fps_denominator;
+	struct v4l2_mbus_framefmt	format;
+	int				width;
+	int				height;
+
+	struct gpio			mux_gpios[MAX_NUM_GPIOS];
+	int				num_gpios;
+
+	struct regmap			*regmap;
+
+	/* Sensor reference clock */
+	struct clk			*xvclk;
+
+	bool				power;
+
+	/* GPIOs */
+	struct gpio_desc		*reset_gpio;
+	struct gpio_desc		*powerdown_gpio;
+
+	struct v4l2_ctrl		*colorbar;
+};
+
+static int ov1063x_init_gpios(struct i2c_client *client);
+
+static const struct ov1063x_framesize ov1063x_framesizes[] = {
+	{
+		.width		= 1280,
+		.height		= 800,
+	}, {
+		.width		= 1280,
+		.height		= 720,
+	}, {
+		.width		= 752,
+		.height		= 480,
+	}, {
+		.width		= 640,
+		.height		= 480,
+	}, {
+		.width		= 600,
+		.height		= 400,
+	}, {
+		.width		= 352,
+		.height		= 288,
+	}, {
+		.width		= 320,
+		.height		= 240,
+	},
+};
+
+/*
+ * supported color format list
+ */
+static const struct ov1063x_color_format ov1063x_cfmts[] = {
+	{
+		.code		= MEDIA_BUS_FMT_YUYV8_2X8,
+		.colorspace	= V4L2_COLORSPACE_SMPTE170M,
+	},
+	{
+		.code		= MEDIA_BUS_FMT_UYVY8_2X8,
+		.colorspace	= V4L2_COLORSPACE_SMPTE170M,
+	},
+	{
+		.code		= MEDIA_BUS_FMT_VYUY8_2X8,
+		.colorspace	= V4L2_COLORSPACE_SMPTE170M,
+	},
+	{
+		.code		= MEDIA_BUS_FMT_YVYU8_2X8,
+		.colorspace	= V4L2_COLORSPACE_SMPTE170M,
+	},
+	{
+		.code		= MEDIA_BUS_FMT_YUYV10_2X10,
+		.colorspace	= V4L2_COLORSPACE_SMPTE170M,
+	},
+};
+
+static struct ov1063x_priv *to_ov1063x(const struct i2c_client *client)
+{
+	return container_of(i2c_get_clientdata(client), struct ov1063x_priv,
+			subdev);
+}
+
+/* Helper function to write consecutive 8 bit registers */
+static int ov1063x_regmap_write16(struct regmap *map, u16 reg, u16 val)
+{
+	int ret;
+
+	ret = regmap_write(map, reg, val >> 8);
+	if (ret)
+		return ret;
+
+	return regmap_write(map, reg + 1, val & 0xff);
+}
+
+/* Start/Stop streaming from the device */
+static int ov1063x_s_stream(struct v4l2_subdev *sd, int enable)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(sd);
+	struct ov1063x_priv *priv = to_ov1063x(client);
+	struct regmap *map = priv->regmap;
+	int ret;
+
+	ret = ov1063x_init_gpios(client);
+	if (ret) {
+		dev_err(&client->dev, "Failed to request gpios");
+		return ret;
+	}
+
+	ret = regmap_write(map, 0x0100, enable);
+	if (ret)
+		return ret;
+
+	return regmap_write(map, 0x301c, enable ? 0xf0 : 0x70);
+}
+
+static int ov1063x_set_regs(struct i2c_client *client,
+			    const struct ov1063x_reg *regs, int nr_regs);
+
+/* Set status of additional camera capabilities */
+static int ov1063x_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct ov1063x_priv *priv = container_of(ctrl->handler,
+					struct ov1063x_priv, hdl);
+	struct i2c_client *client = v4l2_get_subdevdata(&priv->subdev);
+	struct regmap *map = priv->regmap;
+	const struct ov1063x_reg *regs;
+	int n_regs;
+
+	switch (ctrl->id) {
+	case V4L2_CID_VFLIP:
+		return regmap_update_bits(map, OV1063X_VFLIP,
+					  OV1063X_VFLIP_ON,
+					  ctrl->val ? OV1063X_VFLIP_ON : 0);
+	case V4L2_CID_HFLIP:
+		return regmap_update_bits(map, OV1063X_HMIRROR,
+					  OV1063X_HMIRROR_ON,
+					  ctrl->val ? OV1063X_HMIRROR_ON : 0);
+	case V4L2_CID_TEST_PATTERN:
+		if (ctrl->val) {
+			n_regs = ARRAY_SIZE(ov1063x_regs_colorbar_enable);
+			regs = ov1063x_regs_colorbar_enable;
+		} else {
+			n_regs = ARRAY_SIZE(ov1063x_regs_colorbar_disable);
+			regs = ov1063x_regs_colorbar_disable;
+		}
+		return ov1063x_set_regs(client, regs, n_regs);
+	}
+
+	return -EINVAL;
+}
+
+/*
+ * Get the best pixel clock (pclk) that meets minimum hts/vts requirements.
+ * xvclk_rate => pre-divider => clk1 => multiplier => clk2 => post-divider
+ * => pclk
+ * We try all valid combinations of settings for the 3 blocks to get the pixel
+ * clock, and from that calculate the actual hts/vts to use. The vts is
+ * extended so as to achieve the required frame rate. The function also returns
+ * the PLL register contents needed to set the pixel clock.
+ */
+static int ov1063x_get_pclk(int xvclk_rate, int *htsmin, int *vtsmin,
+			    int fps_numerator, int fps_denominator,
+			    u8 *r3003, u8 *r3004)
+{
+	int pre_divs[] = { 2, 3, 4, 6, 8, 10, 12, 14 };
+	int pclk;
+	int best_pclk = INT_MAX;
+	int best_hts = 0;
+	int i, j, k;
+	int best_i = 0, best_j = 0, best_k = 0;
+	int clk1, clk2;
+	int hts;
+
+	/* Pre-div, reg 0x3004, bits 6:4 */
+	for (i = 0; i < ARRAY_SIZE(pre_divs); i++) {
+		clk1 = (xvclk_rate / pre_divs[i]) * 2;
+
+		if (clk1 < 3000000 || clk1 > 27000000)
+			continue;
+
+		/* Mult = reg 0x3003, bits 5:0 */
+		for (j = 1; j < 32; j++) {
+			clk2 = (clk1 * j);
+
+			if (clk2 < 200000000 || clk2 > 500000000)
+				continue;
+
+			/* Post-div, reg 0x3004, bits 2:0 */
+			for (k = 0; k < 8; k++) {
+				pclk = clk2 / (2 * (k + 1));
+
+				if (pclk > 96000000)
+					continue;
+
+				hts = *htsmin + 200 + pclk / 300000;
+
+				/* 2 clock cycles for every YUV422 pixel */
+				if (pclk < (((hts * *vtsmin) / fps_denominator)
+					* fps_numerator * 2))
+					continue;
+
+				if (pclk < best_pclk) {
+					best_pclk = pclk;
+					best_hts = hts;
+					best_i = i;
+					best_j = j;
+					best_k = k;
+				}
+			}
+		}
+	}
+
+	/* register contents */
+	*r3003 = (u8)best_j;
+	*r3004 = ((u8)best_i << 4) | (u8)best_k;
+
+	/* Did we get a valid PCLK? */
+	if (best_pclk == INT_MAX)
+		return -1;
+
+	*htsmin = best_hts;
+
+	/* Adjust vts to get as close to the desired frame rate as we can */
+	*vtsmin = best_pclk / ((best_hts / fps_denominator) *
+		  fps_numerator * 2);
+
+	return best_pclk;
+}
+
+static int ov1063x_set_regs(struct i2c_client *client,
+			    const struct ov1063x_reg *regs, int nr_regs)
+{
+	struct ov1063x_priv *priv = to_ov1063x(client);
+	struct regmap *map = priv->regmap;
+	int i, ret;
+	u8 val;
+
+	for (i = 0; i < nr_regs; i++) {
+		if (regs[i].reg == 0x300c) {
+			val = ((client->addr * 2) | 0x1);
+
+			ret = regmap_write(map, regs[i].reg, val);
+			if (ret)
+				return ret;
+		} else {
+			ret = regmap_write(map, regs[i].reg, regs[i].val);
+			if (ret)
+				return ret;
+		}
+	}
+
+	return 0;
+}
+
+/* Setup registers according to resolution and color encoding */
+static int ov1063x_set_params(struct i2c_client *client, u32 width, u32 height)
+{
+	struct ov1063x_priv *priv = to_ov1063x(client);
+	struct regmap *map = priv->regmap;
+	int ret = -EINVAL;
+	int pclk;
+	int hts, vts;
+	u8 r3003, r3004, r4300;
+	int tmp;
+	u32 height_pre_subsample;
+	u32 width_pre_subsample;
+	u8 horiz_crop_mode;
+	int nr_isp_pixels;
+	int vert_sub_sample = 0;
+	int horiz_sub_sample = 0;
+	int sensor_width;
+	int n_regs;
+
+	if (width > OV1063X_MAX_WIDTH || height > OV1063X_MAX_HEIGHT)
+		return ret;
+
+	priv->width = width;
+	priv->height = height;
+
+	/* Vertical sub-sampling? */
+	height_pre_subsample = priv->height;
+	if (priv->height <= 400) {
+		vert_sub_sample = 1;
+		height_pre_subsample <<= 1;
+	}
+
+	/* Horizontal sub-sampling? */
+	width_pre_subsample = priv->width;
+	if (priv->width <= 640) {
+		horiz_sub_sample = 1;
+		width_pre_subsample <<= 1;
+	}
+
+	/* Horizontal cropping */
+	if (width_pre_subsample > 768) {
+		sensor_width = OV1063X_SENSOR_WIDTH;
+		horiz_crop_mode = 0x63;
+	} else if (width_pre_subsample > 656) {
+		sensor_width = 768;
+		horiz_crop_mode = 0x6b;
+	} else {
+		sensor_width = 656;
+		horiz_crop_mode = 0x73;
+	}
+
+	/* minimum values for hts and vts */
+	hts = sensor_width;
+	vts = height_pre_subsample + 50;
+	dev_dbg(&client->dev, "fps=(%d/%d), hts=%d, vts=%d\n",
+		priv->fps_numerator, priv->fps_denominator, hts, vts);
+
+	/* Get the best PCLK & adjust hts,vts accordingly */
+	pclk = ov1063x_get_pclk(priv->xvclk_rate, &hts, &vts,
+				priv->fps_numerator, priv->fps_denominator,
+				&r3003, &r3004);
+	if (pclk < 0)
+		return ret;
+	dev_dbg(&client->dev, "pclk=%d, hts=%d, vts=%d\n", pclk, hts, vts);
+	dev_dbg(&client->dev, "r3003=0x%X r3004=0x%X\n", r3003, r3004);
+
+	/* Disable ISP & program all registers that we might modify */
+	ret = ov1063x_set_regs(client, ov1063x_regs_change_mode,
+			       ARRAY_SIZE(ov1063x_regs_change_mode));
+	if (ret)
+		return ret;
+
+	/* Set to 1280x720 */
+	ret = regmap_write(map, 0x380f, 0x80);
+	if (ret)
+		return ret;
+
+	/* Set PLL */
+	ret = regmap_write(map, 0x3003, r3003);
+	if (ret)
+		return ret;
+	ret = regmap_write(map, 0x3004, r3004);
+	if (ret)
+		return ret;
+
+	/* Set HSYNC */
+	ret = regmap_write(map, 0x4700, 0x00);
+	if (ret)
+		return ret;
+
+	switch (priv->format.code) {
+	case MEDIA_BUS_FMT_UYVY8_2X8:
+		r4300 = OV1063X_FORMAT_UYVY;
+		break;
+	case MEDIA_BUS_FMT_VYUY8_2X8:
+		r4300 = OV1063X_FORMAT_VYUY;
+		break;
+	case MEDIA_BUS_FMT_YUYV8_2X8:
+		r4300 = OV1063X_FORMAT_YUYV;
+		break;
+	case MEDIA_BUS_FMT_YVYU8_2X8:
+		r4300 = OV1063X_FORMAT_YYYU;
+		break;
+	default:
+		r4300 = OV1063X_FORMAT_UYVY;
+		break;
+	}
+
+	/* Set format to UYVY */
+	ret = regmap_write(map, OV1063X_FORMAT_CTRL00, r4300);
+	if (ret)
+		return ret;
+
+	dev_dbg(&client->dev, "r4300=0x%X\n", r4300);
+
+	/* Set output to 8-bit yuv */
+	ret = regmap_write(map, 0x4605, 0x08);
+	if (ret)
+		return ret;
+
+	/* Horizontal cropping */
+	ret = regmap_write(map, 0x3621, horiz_crop_mode);
+	if (ret)
+		return ret;
+
+	ret = regmap_write(map, 0x3702, (pclk + 1500000) / 3000000);
+	if (ret)
+		return ret;
+	ret = regmap_write(map, 0x3703, (pclk + 666666) / 1333333);
+	if (ret)
+		return ret;
+	ret = regmap_write(map, 0x3704, (pclk + 961500) / 1923000);
+	if (ret)
+		return ret;
+
+	/* Vertical cropping */
+	tmp = ((OV1063X_SENSOR_HEIGHT - height_pre_subsample) / 2) & ~0x1;
+	ret = ov1063x_regmap_write16(map, 0x3802, tmp);
+	if (ret)
+		return ret;
+	tmp = tmp + height_pre_subsample + 3;
+	ret = ov1063x_regmap_write16(map, 0x3806, tmp);
+	if (ret)
+		return ret;
+
+	dev_dbg(&client->dev, "width x height = %x x %x\n",
+		priv->width, priv->height);
+	/* Output size */
+	ret = ov1063x_regmap_write16(map, 0x3808, priv->width);
+	if (ret)
+		return ret;
+	ret = ov1063x_regmap_write16(map, 0x380a, priv->height);
+	if (ret)
+		return ret;
+
+	dev_dbg(&client->dev, "hts x vts = %x x %x\n", hts, vts);
+
+	ret = ov1063x_regmap_write16(map, 0x380c, hts);
+	if (ret)
+		return ret;
+
+	ret = ov1063x_regmap_write16(map, 0x380e, vts);
+	if (ret)
+		return ret;
+
+	if (vert_sub_sample) {
+		ret = regmap_update_bits(map, OV1063X_VFLIP,
+					 OV1063X_VFLIP_SUBSAMPLE,
+					 OV1063X_VFLIP_SUBSAMPLE);
+		if (ret)
+			return ret;
+		n_regs = ARRAY_SIZE(ov1063x_regs_vert_sub_sample);
+		ret = ov1063x_set_regs(client, ov1063x_regs_vert_sub_sample,
+				       n_regs);
+		if (ret)
+			return ret;
+	}
+
+	ret = ov1063x_regmap_write16(map, 0x4606, 2 * hts);
+	if (ret)
+		return ret;
+	ret = ov1063x_regmap_write16(map, 0x460a,
+				     2 * (hts - width_pre_subsample));
+	if (ret)
+		return ret;
+
+	tmp = (vts - 8) * 16;
+	ret = ov1063x_regmap_write16(map, 0xc488, tmp);
+	if (ret)
+		return ret;
+	ret = ov1063x_regmap_write16(map, 0xc48a, tmp);
+	if (ret)
+		return ret;
+
+	nr_isp_pixels = sensor_width * (priv->height + 4);
+	ret = ov1063x_regmap_write16(map, 0xc4cc, nr_isp_pixels / 256);
+	if (ret)
+		return ret;
+	ret = ov1063x_regmap_write16(map, 0xc4ce, nr_isp_pixels / 256);
+	if (ret)
+		return ret;
+	ret = ov1063x_regmap_write16(map, 0xc512, nr_isp_pixels / 16);
+	if (ret)
+		return ret;
+
+	/* Horizontal sub-sampling */
+	if (horiz_sub_sample) {
+		ret = regmap_write(map, 0x5005, 0x9);
+		if (ret)
+			return ret;
+
+		ret = regmap_write(map, 0x3007, 0x2);
+		if (ret)
+			return ret;
+	}
+
+	ret = ov1063x_regmap_write16(map, 0xc518, vts);
+	if (ret)
+		return ret;
+	ret = ov1063x_regmap_write16(map, 0xc51a, hts);
+	if (ret)
+		return ret;
+
+	/* Enable ISP blocks */
+	ret = ov1063x_set_regs(client, ov1063x_regs_enable,
+			       ARRAY_SIZE(ov1063x_regs_enable));
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+/*
+ * V4L2 subdev video and pad level operations
+ */
+
+static void ov1063x_get_default_format(struct v4l2_mbus_framefmt *mf)
+{
+	mf->width = ov1063x_framesizes[0].width;
+	mf->height = ov1063x_framesizes[0].height;
+	mf->colorspace = ov1063x_cfmts[0].colorspace;
+	mf->code = ov1063x_cfmts[0].code;
+
+	mf->field = V4L2_FIELD_NONE;
+}
+
+static int ov1063x_get_fmt(struct v4l2_subdev *sd,
+			   struct v4l2_subdev_pad_config *cfg,
+			   struct v4l2_subdev_format *fmt)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(sd);
+	struct ov1063x_priv *priv = to_ov1063x(client);
+	struct v4l2_mbus_framefmt *mf;
+
+	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) {
+		mf = v4l2_subdev_get_try_format(sd, cfg, 0);
+		mutex_lock(&priv->lock);
+		fmt->format = *mf;
+		mutex_unlock(&priv->lock);
+		return 0;
+	}
+
+	mutex_lock(&priv->lock);
+	fmt->format = priv->format;
+	mutex_unlock(&priv->lock);
+
+	return 0;
+}
+
+static void __ov1063x_try_frame_size(struct v4l2_mbus_framefmt *mf)
+{
+	const struct ov1063x_framesize *fsize = &ov1063x_framesizes[0];
+	const struct ov1063x_framesize *match = NULL;
+	int i = ARRAY_SIZE(ov1063x_framesizes);
+	unsigned int min_err = UINT_MAX;
+
+	while (i--) {
+		int err = abs(fsize->width - mf->width)
+				+ abs(fsize->height - mf->height);
+		if (err < min_err) {
+			min_err = err;
+			match = fsize;
+		}
+		fsize++;
+	}
+
+	if (!match)
+		match = &ov1063x_framesizes[0];
+
+	mf->width  = match->width;
+	mf->height = match->height;
+}
+
+static int ov1063x_set_fmt(struct v4l2_subdev *sd,
+			   struct v4l2_subdev_pad_config *cfg,
+			   struct v4l2_subdev_format *fmt)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(sd);
+	int index = ARRAY_SIZE(ov1063x_cfmts);
+	struct ov1063x_priv *priv = to_ov1063x(client);
+	struct v4l2_mbus_framefmt *mf = &fmt->format;
+	int ret = 0;
+
+	__ov1063x_try_frame_size(mf);
+
+	while (--index >= 0)
+		if (ov1063x_cfmts[index].code == mf->code)
+			break;
+
+	if (index < 0)
+		return -EINVAL;
+
+	mf->colorspace = ov1063x_cfmts[index].colorspace;
+	mf->code = ov1063x_cfmts[index].code;
+	mf->field = V4L2_FIELD_NONE;
+
+	mutex_lock(&priv->lock);
+
+	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) {
+		mf = v4l2_subdev_get_try_format(sd, cfg, fmt->pad);
+		*mf = fmt->format;
+	} else {
+		priv->format = fmt->format;
+		ret = ov1063x_set_params(client, mf->width, mf->height);
+	}
+
+	mutex_unlock(&priv->lock);
+
+	return ret;
+}
+
+static int ov1063x_enum_mbus_code(struct v4l2_subdev *sd,
+				  struct v4l2_subdev_pad_config *cfg,
+				  struct v4l2_subdev_mbus_code_enum *code)
+{
+	if (code->index >= ARRAY_SIZE(ov1063x_cfmts))
+		return -EINVAL;
+
+	code->code = ov1063x_cfmts[code->index].code;
+
+	return 0;
+}
+
+static int ov1063x_enum_frame_sizes(struct v4l2_subdev *sd,
+				    struct v4l2_subdev_pad_config *cfg,
+				    struct v4l2_subdev_frame_size_enum *fse)
+{
+	int i = ARRAY_SIZE(ov1063x_cfmts);
+
+	if (fse->index >= ARRAY_SIZE(ov1063x_framesizes))
+		return -EINVAL;
+
+	while (--i)
+		if (ov1063x_cfmts[i].code == fse->code)
+			break;
+
+	fse->code = ov1063x_cfmts[i].code;
+
+	fse->min_width  = ov1063x_framesizes[fse->index].width;
+	fse->max_width  = fse->min_width;
+	fse->max_height = ov1063x_framesizes[fse->index].height;
+	fse->min_height = fse->max_height;
+
+	return 0;
+}
+
+static int ov1063x_init_gpios(struct i2c_client *client)
+{
+	struct ov1063x_priv *priv = to_ov1063x(client);
+	int ret = 0;
+
+	ret = gpio_request_array(priv->mux_gpios, priv->num_gpios);
+	if (ret)
+		goto done;
+	gpio_free_array(priv->mux_gpios, priv->num_gpios);
+
+done:
+	return ret;
+}
+
+static int ov1063x_init_cam_gpios(struct i2c_client *client)
+{
+	struct ov1063x_priv *priv = to_ov1063x(client);
+	struct gpio_desc *gpio;
+
+	gpio = devm_gpiod_get_optional(&client->dev, "reset",
+				       GPIOD_OUT_LOW);
+	if (IS_ERR(gpio))
+		return PTR_ERR(gpio);
+	priv->reset_gpio = gpio;
+
+	gpio = devm_gpiod_get_optional(&client->dev, "powerdown",
+				       GPIOD_OUT_LOW);
+	if (IS_ERR(gpio))
+		return PTR_ERR(gpio);
+	priv->powerdown_gpio = gpio;
+
+	return 0;
+}
+
+static void ov1063x_set_power(struct i2c_client *client, bool on)
+{
+	struct ov1063x_priv *priv = to_ov1063x(client);
+
+	dev_dbg(&client->dev, "%s: on: %d\n", __func__, on);
+
+	if (priv->power == on)
+		return;
+
+	if (on) {
+		if (priv->powerdown_gpio) {
+			gpiod_set_value_cansleep(priv->powerdown_gpio, 1);
+			usleep_range(1000, 1200);
+		}
+		if (priv->reset_gpio) {
+			gpiod_set_value_cansleep(priv->reset_gpio, 1);
+			usleep_range(250000, 260000);
+		}
+	} else {
+		if (priv->powerdown_gpio)
+			gpiod_set_value_cansleep(priv->powerdown_gpio, 0);
+		if (priv->reset_gpio)
+			gpiod_set_value_cansleep(priv->reset_gpio, 0);
+	}
+
+	priv->power = on;
+}
+
+static int ov1063x_video_probe(struct i2c_client *client)
+{
+	struct ov1063x_priv *priv = to_ov1063x(client);
+	struct regmap *map = priv->regmap;
+	u32 pid, ver;
+	int ret;
+
+	ov1063x_set_power(client, true);
+
+	ret = ov1063x_set_regs(client, ov1063x_regs_default,
+			       ARRAY_SIZE(ov1063x_regs_default));
+	if (ret)
+		return ret;
+
+	usleep_range(500, 510);
+
+	/* check and show product ID and manufacturer ID */
+	ret = regmap_read(map, OV1063X_PID, &pid);
+	if (ret)
+		return ret;
+
+	ret = regmap_read(map, OV1063X_VER, &ver);
+	if (ret)
+		return ret;
+
+	if (OV1063X_VERSION(pid, ver) == OV10635_VERSION_REG) {
+		priv->model = SENSOR_OV10635;
+		priv->revision = 1;
+	} else if (OV1063X_VERSION(pid, ver) == OV10633_VERSION_REG) {
+		priv->model = SENSOR_OV10633;
+		priv->revision = 1;
+	} else {
+		dev_err(&client->dev, "Product ID error %x:%x\n", pid, ver);
+		return -ENODEV;
+	}
+
+	dev_info(&client->dev, "ov1063x Product ID %x Manufacturer ID %x\n",
+		 pid, ver);
+
+	/* Program all the 'standard' registers */
+
+	return v4l2_ctrl_handler_setup(&priv->hdl);
+}
+
+/*
+ * V4L2 subdev internal operations
+ */
+static int ov1063x_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(sd);
+	struct v4l2_mbus_framefmt *mf;
+
+	dev_dbg(&client->dev, "%s:\n", __func__);
+
+	mf = v4l2_subdev_get_try_format(sd, fh->pad, 0);
+	ov1063x_get_default_format(mf);
+	return 0;
+}
+
+static const struct v4l2_ctrl_ops ov1063x_ctrl_ops = {
+	.s_ctrl = ov1063x_s_ctrl,
+};
+
+static const char * const ov1063x_test_pattern_menu[] = {
+	"Disabled",
+	"Vertical Color Bars",
+};
+
+static const struct v4l2_subdev_video_ops ov1063x_subdev_video_ops = {
+	.s_stream	= ov1063x_s_stream,
+};
+
+static const struct v4l2_subdev_internal_ops ov1063x_sd_internal_ops = {
+	.open		= ov1063x_open,
+};
+
+static const struct v4l2_subdev_core_ops ov1063x_subdev_core_ops = {
+	.log_status		= v4l2_ctrl_subdev_log_status,
+	.subscribe_event	= v4l2_ctrl_subdev_subscribe_event,
+	.unsubscribe_event	= v4l2_event_subdev_unsubscribe,
+};
+
+static const struct v4l2_subdev_pad_ops ov1063x_subdev_pad_ops = {
+	.enum_mbus_code		= ov1063x_enum_mbus_code,
+	.enum_frame_size	= ov1063x_enum_frame_sizes,
+	.get_fmt		= ov1063x_get_fmt,
+	.set_fmt		= ov1063x_set_fmt,
+};
+
+static struct v4l2_subdev_ops ov1063x_subdev_ops = {
+	.core	= &ov1063x_subdev_core_ops,
+	.video	= &ov1063x_subdev_video_ops,
+	.pad	= &ov1063x_subdev_pad_ops,
+};
+
+static const struct regmap_config ov1063x_regmap_config = {
+	.reg_bits = 16,
+	.val_bits = 8,
+};
+
+/*
+ * i2c_driver function
+ */
+
+static int ov1063x_of_probe(struct i2c_client *client,
+			    struct device_node *node)
+{
+	struct ov1063x_priv *priv = to_ov1063x(client);
+	struct gpio *gpios = &priv->mux_gpios[0];
+	unsigned int flags;
+	int i, gpio;
+
+	/*
+	 * Iterate over all the gpios in the device tree
+	 * ENOENT is returned when trying to access last + 1 gpio
+	 */
+	for (i = 0; i < MAX_NUM_GPIOS; i++) {
+		gpio = of_get_named_gpio_flags(node, "mux-gpios", i, &flags);
+		if (gpio_is_valid(gpio)) {
+			gpios[i].gpio	= gpio;
+			gpios[i].flags	= (flags & OF_GPIO_ACTIVE_LOW) ?
+				GPIOF_OUT_INIT_LOW : GPIOF_OUT_INIT_HIGH;
+			gpios[i].label	= client->name;
+		} else {
+			if (gpio == -ENOENT)
+				break;
+
+			return gpio;
+		}
+	}
+	priv->num_gpios = i;
+	return 0;
+}
+
+static int ov1063x_probe(struct i2c_client *client,
+			 const struct i2c_device_id *did)
+{
+	struct device_node *node = client->dev.of_node;
+	struct ov1063x_priv *priv;
+	struct v4l2_subdev *sd;
+	struct clk *clk;
+	unsigned int menu_size;
+	int ret = 0;
+
+	priv = devm_kzalloc(&client->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	i2c_set_clientdata(client, priv);
+
+	priv->regmap = devm_regmap_init_i2c(client, &ov1063x_regmap_config);
+	if (IS_ERR(priv->regmap))
+		return PTR_ERR(priv->regmap);
+
+	clk = devm_clk_get(&client->dev, "xvclk");
+	if (IS_ERR(clk)) {
+		dev_err(&client->dev, "xvclk reference is missing!\n");
+		ret = PTR_ERR(clk);
+		goto err;
+	}
+	priv->xvclk = clk;
+
+	priv->xvclk_rate = clk_get_rate(clk);
+	dev_dbg(&client->dev, "xvclk_rate: %d (Hz)\n", priv->xvclk_rate);
+
+	if (priv->xvclk_rate < 6000000 ||
+	    priv->xvclk_rate > 27000000) {
+		ret = -EINVAL;
+		goto err;
+	}
+
+	ret = clk_prepare_enable(priv->xvclk);
+	if (ret < 0)
+		goto err;
+
+	ret = ov1063x_of_probe(client, node);
+	if (ret)
+		goto err;
+
+	/* Default framerate */
+	priv->fps_numerator = 30;
+	priv->fps_denominator = 1;
+	ov1063x_get_default_format(&priv->format);
+	priv->width = priv->format.width;
+	priv->height = priv->format.height;
+
+	sd = &priv->subdev;
+	v4l2_i2c_subdev_init(sd, client, &ov1063x_subdev_ops);
+
+	sd->internal_ops = &ov1063x_sd_internal_ops;
+	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE |
+		     V4L2_SUBDEV_FL_HAS_EVENTS;
+
+	v4l2_ctrl_handler_init(&priv->hdl, 3);
+	v4l2_ctrl_new_std(&priv->hdl, &ov1063x_ctrl_ops,
+			  V4L2_CID_VFLIP, 0, 1, 1, 0);
+	v4l2_ctrl_new_std(&priv->hdl, &ov1063x_ctrl_ops,
+			  V4L2_CID_HFLIP, 0, 1, 1, 0);
+	menu_size = ARRAY_SIZE(ov1063x_test_pattern_menu) - 1;
+	priv->colorbar =
+		v4l2_ctrl_new_std_menu_items(&priv->hdl, &ov1063x_ctrl_ops,
+					     V4L2_CID_TEST_PATTERN, menu_size,
+					     0, 0, ov1063x_test_pattern_menu);
+	priv->subdev.ctrl_handler = &priv->hdl;
+	if (priv->hdl.error) {
+		ret = priv->hdl.error;
+		goto err;
+	}
+
+	mutex_init(&priv->lock);
+
+	ret = ov1063x_init_cam_gpios(client);
+	if (ret) {
+		dev_err(&client->dev, "Failed to request cam gpios");
+		goto err;
+	}
+
+	ret = ov1063x_init_gpios(client);
+	if (ret) {
+		dev_err(&client->dev, "Failed to request mux gpios");
+		goto err;
+	}
+
+	ret = ov1063x_video_probe(client);
+	if (ret) {
+		v4l2_ctrl_handler_free(&priv->hdl);
+		goto err;
+	}
+
+	sd->dev = &client->dev;
+	ret = v4l2_async_register_subdev(sd);
+
+	dev_info(&client->dev, "%s sensor driver registered !!\n", sd->name);
+
+	return 0;
+err:
+	clk_disable_unprepare(priv->xvclk);
+	return ret;
+}
+
+static int ov1063x_remove(struct i2c_client *client)
+{
+	struct ov1063x_priv *priv = i2c_get_clientdata(client);
+
+	v4l2_device_unregister_subdev(&priv->subdev);
+	v4l2_ctrl_handler_free(&priv->hdl);
+	ov1063x_set_power(client, false);
+	clk_disable_unprepare(priv->xvclk);
+
+	return 0;
+}
+
+static const struct i2c_device_id ov1063x_id[] = {
+	{ "ov10635", 0 },
+	{ "ov10633", 0 },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, ov1063x_id);
+
+#if IS_ENABLED(CONFIG_OF)
+static const struct of_device_id ov1063x_dt_id[] = {
+	{
+		.compatible = "ovti,ov10635", .data = "ov10635"
+	},
+	{
+		.compatible = "ovti,ov10633", .data = "ov10633"
+	},
+	{
+	}
+};
+MODULE_DEVICE_TABLE(of, ov1063x_dt_id);
+#endif
+
+static struct i2c_driver ov1063x_i2c_driver = {
+	.driver = {
+		.name	= "ov1063x",
+		.of_match_table = of_match_ptr(ov1063x_dt_id),
+	},
+	.probe = ov1063x_probe,
+	.remove = ov1063x_remove,
+	.id_table = ov1063x_id,
+};
+
+module_i2c_driver(ov1063x_i2c_driver);
+
+MODULE_DESCRIPTION("SoC Camera driver for OmniVision OV1063X");
+MODULE_AUTHOR("Phil Edworthy <phil.edworthy@renesas.com>");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/media/i2c/ov1063x_regs.h b/drivers/media/i2c/ov1063x_regs.h
--- a/drivers/media/i2c/ov1063x_regs.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/media/i2c/ov1063x_regs.h	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,699 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * OmniVision OV1063X Camera Driver
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated -  http://www.ti.com/
+ */
+
+struct ov1063x_reg {
+	u16	reg;
+	u8	val;
+};
+
+static const struct ov1063x_reg ov1063x_regs_default[] = {
+	/* Register configuration for full resolution : 1280x720 */
+		{0x103, 0x1},	 /** Software Reset */
+		{0x301b, 0xff},  /** System Control Clock Reset #1 */
+		{0x301c, 0xff},  /** System Control Clock Reset #2 */
+		{0x301a, 0xff},  /** System Control Clock Reset #0 */
+		{0x300c, 0x61},  /** Serial Camera Control Bus ID */
+		{0x300c, 0x61},
+		{0x300c, 0x61},
+		{0x300c, 0x61},
+		{0x300c, 0x61},
+		{0x300c, 0x61},
+		{0x300c, 0x61},
+		{0x300c, 0x61},
+		{0x300c, 0x61},
+		{0x300c, 0x61},
+		{0x300c, 0x61},
+		{0x300c, 0x61},
+		{0x300c, 0x61},
+		{0x300c, 0x61},
+		{0x300c, 0x61},
+		{0x300c, 0x61},
+		{0x300c, 0x61},
+		{0x300c, 0x61},
+		{0x300c, 0x61},
+		{0x300c, 0x61},
+		{0x300c, 0x61},
+		{0x300c, 0x61},
+		{0x300c, 0x61},
+		{0x3021, 0x3},  /** System Control Misc */
+		{0x3011, 0x2},
+		{0x6900, 0xc},
+		{0x6901, 0x1},
+		{0x3033, 0x8},  /** System clock/4 */
+		{0x3503, 0x10}, /** AEC Delay enabled */
+		{0x302d, 0x2f}, /** Power Down Control */
+		{0x3025, 0x3},  /** Debug Control enabled */
+		/*
+		 * FPS is computed as:
+		 * XVCLK	   = 9MHz
+		 * preDivide   = {/1,/1.5,/2,/3,/4,/5,/6,/7}
+		 * decided based on (0x3004[6:4].
+		 * numerator   = (0x3003[5:0] * XVCLK)/preDivide)
+		 * denominator = (2 * (1+0x3004[2:0]))
+		 * FPS = numerator/denominator.
+		 */
+		/* {0x3003, 0x1B}, */ /* fps = 30fps. */
+		/* {0x3004, 0x03}, */
+		{0x3003, 0x20}, /* fps = 15fps. */
+		{0x3004, 0x3},
+		{0x3005, 0x20},
+		{0x3006, 0x91},
+		{0x3600, 0x74},
+		{0x3601, 0x2b},
+		{0x3612, 0x0},
+		{0x3611, 0x67},
+		{0x3633, 0xba},
+		{0x3602, 0x2f},
+		{0x3603, 0x0},
+		{0x3630, 0xa8},
+		{0x3631, 0x16},
+		{0x3714, 0x10},
+		{0x371d, 0x1},
+		{0x4300, 0x3A}, /* UYVY mode */
+		{0x3007, 0x1},
+		/*
+		 * RAW mode and Pixel CLK selct.
+		 * 0x3024[0] = 1 :: System CLK (0x3003,0x3004)
+		 * 0x3024[0] = 0 :: secondary CLK (0x3005,0x3006)
+		 */
+		{0x3024, 0x1},
+		{0x3020, 0xb},
+		{0x3702, 0xd},
+		{0x3703, 0x20},
+		{0x3704, 0x15},
+		{0x3709, 0x28},
+		{0x370d, 0x0},
+		{0x3712, 0x0},
+		{0x3713, 0x20},
+		{0x3715, 0x4},
+		{0x381d, 0x40},
+		{0x381c, 0x0},
+		{0x3824, 0x10},
+		{0x3815, 0x8c},
+		{0x3804, 0x5},
+		{0x3805, 0x1f},
+		{0x3800, 0x0},
+		{0x3801, 0x0},
+		{0x3806, 0x3},
+		{0x3807, 0x1},
+		{0x3802, 0x0},
+		{0x3803, 0x2e},
+		{0x3808, 0x5},
+		{0x3809, 0x0},
+		{0x380a, 0x2},
+		{0x380b, 0xd0},
+		{0x380c, 0x6},
+		{0x380d, 0xf6}, /* 1280x720 */
+		{0x380e, 0x2},
+		{0x380f, 0xec}, /* 1280x720 */
+		{0x3811, 0x8},
+		{0x381f, 0xc},
+		{0x3621, 0x63},
+		{0x5005, 0x8},
+		{0x56d5, 0x0},
+		{0x56d6, 0x80},
+		{0x56d7, 0x0},
+		{0x56d8, 0x0},
+		{0x56d9, 0x0},
+		{0x56da, 0x80},
+		{0x56db, 0x0},
+		{0x56dc, 0x0},
+		{0x56e8, 0x0},
+		{0x56e9, 0x7f},
+		{0x56ea, 0x0},
+		{0x56eb, 0x7f},
+		{0x5100, 0x0},
+		{0x5101, 0x80},
+		{0x5102, 0x0},
+		{0x5103, 0x80},
+		{0x5104, 0x0},
+		{0x5105, 0x80},
+		{0x5106, 0x0},
+		{0x5107, 0x80},
+		{0x5108, 0x0},
+		{0x5109, 0x0},
+		{0x510a, 0x0},
+		{0x510b, 0x0},
+		{0x510c, 0x0},
+		{0x510d, 0x0},
+		{0x510e, 0x0},
+		{0x510f, 0x0},
+		{0x5110, 0x0},
+		{0x5111, 0x80},
+		{0x5112, 0x0},
+		{0x5113, 0x80},
+		{0x5114, 0x0},
+		{0x5115, 0x80},
+		{0x5116, 0x0},
+		{0x5117, 0x80},
+		{0x5118, 0x0},
+		{0x5119, 0x0},
+		{0x511a, 0x0},
+		{0x511b, 0x0},
+		{0x511c, 0x0},
+		{0x511d, 0x0},
+		{0x511e, 0x0},
+		{0x511f, 0x0},
+		{0x56d0, 0x0},
+		{0x5006, 0x24},
+		{0x5608, 0x0},
+		{0x52d7, 0x6},
+		{0x528d, 0x8},
+		{0x5293, 0x12},
+		{0x52d3, 0x12},
+		{0x5288, 0x6},
+		{0x5289, 0x20},
+		{0x52c8, 0x6},
+		{0x52c9, 0x20},
+		{0x52cd, 0x4},
+		{0x5381, 0x0},
+		{0x5382, 0xff},
+		{0x5589, 0x76},
+		{0x558a, 0x47},
+		{0x558b, 0xef},
+		{0x558c, 0xc9},
+		{0x558d, 0x49},
+		{0x558e, 0x30},
+		{0x558f, 0x67},
+		{0x5590, 0x3f},
+		{0x5591, 0xf0},
+		{0x5592, 0x10},
+		{0x55a2, 0x6d},
+		{0x55a3, 0x55},
+		{0x55a4, 0xc3},
+		{0x55a5, 0xb5},
+		{0x55a6, 0x43},
+		{0x55a7, 0x38},
+		{0x55a8, 0x5f},
+		{0x55a9, 0x4b},
+		{0x55aa, 0xf0},
+		{0x55ab, 0x10},
+		{0x5581, 0x52},
+		{0x5300, 0x1},
+		{0x5301, 0x0},
+		{0x5302, 0x0},
+		{0x5303, 0xe},
+		{0x5304, 0x0},
+		{0x5305, 0xe},
+		{0x5306, 0x0},
+		{0x5307, 0x36},
+		{0x5308, 0x0},
+		{0x5309, 0xd9},
+		{0x530a, 0x0},
+		{0x530b, 0xf},
+		{0x530c, 0x0},
+		{0x530d, 0x2c},
+		{0x530e, 0x0},
+		{0x530f, 0x59},
+		{0x5310, 0x0},
+		{0x5311, 0x7b},
+		{0x5312, 0x0},
+		{0x5313, 0x22},
+		{0x5314, 0x0},
+		{0x5315, 0xd5},
+		{0x5316, 0x0},
+		{0x5317, 0x13},
+		{0x5318, 0x0},
+		{0x5319, 0x18},
+		{0x531a, 0x0},
+		{0x531b, 0x26},
+		{0x531c, 0x0},
+		{0x531d, 0xdc},
+		{0x531e, 0x0},
+		{0x531f, 0x2},
+		{0x5320, 0x0},
+		{0x5321, 0x24},
+		{0x5322, 0x0},
+		{0x5323, 0x56},
+		{0x5324, 0x0},
+		{0x5325, 0x85},
+		{0x5326, 0x0},
+		{0x5327, 0x20},
+		{0x5609, 0x1},
+		{0x560a, 0x40},
+		{0x560b, 0x1},
+		{0x560c, 0x40},
+		{0x560d, 0x0},
+		{0x560e, 0xfa},
+		{0x560f, 0x0},
+		{0x5610, 0xfa},
+		{0x5611, 0x2},
+		{0x5612, 0x80},
+		{0x5613, 0x2},
+		{0x5614, 0x80},
+		{0x5615, 0x1},
+		{0x5616, 0x2c},
+		{0x5617, 0x1},
+		{0x5618, 0x2c},
+		{0x563b, 0x1},
+		{0x563c, 0x1},
+		{0x563d, 0x1},
+		{0x563e, 0x1},
+		{0x563f, 0x3},
+		{0x5640, 0x3},
+		{0x5641, 0x3},
+		{0x5642, 0x5},
+		{0x5643, 0x9},
+		{0x5644, 0x5},
+		{0x5645, 0x5},
+		{0x5646, 0x5},
+		{0x5647, 0x5},
+		{0x5651, 0x0},
+		{0x5652, 0x80},
+		{0x521a, 0x1},
+		{0x521b, 0x3},
+		{0x521c, 0x6},
+		{0x521d, 0xa},
+		{0x521e, 0xe},
+		{0x521f, 0x12},
+		{0x5220, 0x16},
+		{0x5223, 0x2},
+		{0x5225, 0x4},
+		{0x5227, 0x8},
+		{0x5229, 0xc},
+		{0x522b, 0x12},
+		{0x522d, 0x18},
+		{0x522f, 0x1e},
+		{0x5241, 0x4},
+		{0x5242, 0x1},
+		{0x5243, 0x3},
+		{0x5244, 0x6},
+		{0x5245, 0xa},
+		{0x5246, 0xe},
+		{0x5247, 0x12},
+		{0x5248, 0x16},
+		{0x524a, 0x3},
+		{0x524c, 0x4},
+		{0x524e, 0x8},
+		{0x5250, 0xc},
+		{0x5252, 0x12},
+		{0x5254, 0x18},
+		{0x5256, 0x1e},
+		{0x4605, 0x00}, /* 8-bit YUV mode. */
+		{0x4606, 0x7},
+		{0x4607, 0x71},
+		{0x460a, 0x2},
+		{0x460b, 0x70},
+		{0x460c, 0x0},
+		{0x4620, 0xe},
+		{0x4700, 0x4},
+		{0x4701, 0x01},
+		/* {0x4702, 0x1}, */
+		{0x4702, 0x00},  /* 01 */
+		{0x4703, 0x00},
+		{0x4704, 0x00}, /* 01 */
+		/*
+		 * Non-overlapping HSYNC-VSYNC.
+		 * Therefore do not set the VSYNC delay registers.
+		 */
+		{0x4705, 0x00}, /* Vsync delay high byte */
+		{0x4706, 0x00}, /* Vsync delay middle byte */
+		{0x4707, 0x00}, /* Vsync delay low byte{0x4708, 0x1}, */
+		/* {0x4709, 0x50}, */
+		{0x4004, 0x8},
+		{0x4005, 0x18},
+		{0x4001, 0x4},
+		{0x4050, 0x20},
+		{0x4051, 0x22},
+		{0x4057, 0x9c},
+		{0x405a, 0x0},
+		{0x4202, 0x2},
+		{0x3023, 0x10},
+		{0x100, 0x1},
+		{0x100, 0x1},
+		{0x6f0e, 0x0},
+		{0x6f0f, 0x0},
+		{0x460e, 0x8},
+		{0x460f, 0x1},
+		{0x4610, 0x0},
+		{0x4611, 0x1},
+		{0x4612, 0x0},
+		{0x4613, 0x1},
+		{0x4605, 0x00},
+		{0x4608, 0x0},
+		{0x4609, 0x8},
+		{0x6804, 0x0},
+		{0x6805, 0x6},
+		{0x6806, 0x0},
+		{0x5120, 0x0},
+		{0x3510, 0x0},
+		{0x3504, 0x0},
+		{0x6800, 0x0},
+		{0x6f0d, 0x0},
+		{0x5000, 0xff},
+		{0x5001, 0xbf},
+		{0x5002, 0xfe},
+		{0x503d, 0x0},
+		/* {0x503e, 0x00}, */
+		{0xc450, 0x1},
+		{0xc452, 0x4},
+		{0xc453, 0x0},
+		{0xc454, 0x0},
+		{0xc455, 0x0},
+		{0xc456, 0x0},
+		{0xc457, 0x0},
+		{0xc458, 0x0},
+		{0xc459, 0x0},
+		{0xc45b, 0x0},
+		{0xc45c, 0x0},
+		{0xc45d, 0x0},
+		{0xc45e, 0x0},
+		{0xc45f, 0x0},
+		{0xc460, 0x0},
+		{0xc461, 0x1},
+		{0xc462, 0x1},
+		{0xc464, 0x88},
+		{0xc465, 0x0},
+		{0xc466, 0x8a},
+		{0xc467, 0x0},
+		{0xc468, 0x86},
+		{0xc469, 0x0},
+		{0xc46a, 0x40},
+		{0xc46b, 0x50},
+		{0xc46c, 0x30},
+		{0xc46d, 0x28},
+		{0xc46e, 0x60},
+		{0xc46f, 0x40},
+		{0xc47c, 0x1},
+		{0xc47d, 0x38},
+		{0xc47e, 0x0},
+		{0xc47f, 0x0},
+		{0xc480, 0x0},
+		{0xc481, 0xff},
+		{0xc482, 0x0},
+		{0xc483, 0x40},
+		{0xc484, 0x0},
+		{0xc485, 0x18},
+		{0xc486, 0x0},
+		{0xc487, 0x18},
+		{0xc488, 0x2e},
+		{0xc489, 0x80},
+		{0xc48a, 0x2e},
+		{0xc48b, 0x80},
+		{0xc48c, 0x0},
+		{0xc48d, 0x4},
+		{0xc48e, 0x0},
+		{0xc48f, 0x4},
+		{0xc490, 0x7},
+		{0xc492, 0x20},
+		{0xc493, 0x8},
+		{0xc498, 0x2},
+		{0xc499, 0x0},
+		{0xc49a, 0x2},
+		{0xc49b, 0x0},
+		{0xc49c, 0x2},
+		{0xc49d, 0x0},
+		{0xc49e, 0x2},
+		{0xc49f, 0x60},
+		{0xc4a0, 0x4},
+		{0xc4a1, 0x0},
+		{0xc4a2, 0x6},
+		{0xc4a3, 0x0},
+		{0xc4a4, 0x0},
+		{0xc4a5, 0x10},
+		{0xc4a6, 0x0},
+		{0xc4a7, 0x40},
+		{0xc4a8, 0x0},
+		{0xc4a9, 0x80},
+		{0xc4aa, 0xd},
+		{0xc4ab, 0x0},
+		{0xc4ac, 0xf},
+		{0xc4ad, 0xc0},
+		{0xc4b4, 0x1},
+		{0xc4b5, 0x1},
+		{0xc4b6, 0x0},
+		{0xc4b7, 0x1},
+		{0xc4b8, 0x0},
+		{0xc4b9, 0x1},
+		{0xc4ba, 0x1},
+		{0xc4bb, 0x0},
+		{0xc4be, 0x2},
+		{0xc4bf, 0x33},
+		{0xc4c8, 0x3},
+		{0xc4c9, 0xd0},
+		{0xc4ca, 0xe},
+		{0xc4cb, 0x0},
+		{0xc4cc, 0xe},
+		{0xc4cd, 0x51},
+		{0xc4ce, 0xe},
+		{0xc4cf, 0x51},
+		{0xc4d0, 0x4},
+		{0xc4d1, 0x80},
+		{0xc4e0, 0x4},
+		{0xc4e1, 0x2},
+		{0xc4e2, 0x1},
+		{0xc4e4, 0x10},
+		{0xc4e5, 0x20},
+		{0xc4e6, 0x30},
+		{0xc4e7, 0x40},
+		{0xc4e8, 0x50},
+		{0xc4e9, 0x60},
+		{0xc4ea, 0x70},
+		{0xc4eb, 0x80},
+		{0xc4ec, 0x90},
+		{0xc4ed, 0xa0},
+		{0xc4ee, 0xb0},
+		{0xc4ef, 0xc0},
+		{0xc4f0, 0xd0},
+		{0xc4f1, 0xe0},
+		{0xc4f2, 0xf0},
+		{0xc4f3, 0x80},
+		{0xc4f4, 0x0},
+		{0xc4f5, 0x20},
+		{0xc4f6, 0x2},
+		{0xc4f7, 0x0},
+		{0xc4f8, 0x4},
+		{0xc4f9, 0xb},
+		{0xc4fa, 0x0},
+		{0xc4fb, 0x1},
+		{0xc4fc, 0x1},
+		{0xc4fd, 0x1},
+		{0xc4fe, 0x4},
+		{0xc4ff, 0x2},
+		{0xc500, 0x68},
+		{0xc501, 0x74},
+		{0xc502, 0x70},
+		{0xc503, 0x80},
+		{0xc504, 0x5},
+		{0xc505, 0x80},
+		{0xc506, 0x3},
+		{0xc507, 0x80},
+		{0xc508, 0x1},
+		{0xc509, 0xc0},
+		{0xc50a, 0x1},
+		{0xc50b, 0xa0},
+		{0xc50c, 0x1},
+		{0xc50d, 0x2c},
+		{0xc50e, 0x1},
+		{0xc50f, 0xa},
+		{0xc510, 0x0},
+		{0xc511, 0x0},
+		{0xc512, 0xe5},
+		{0xc513, 0x14},
+		{0xc514, 0x4},
+		{0xc515, 0x0},
+		{0xc518, 0x3},
+		{0xc519, 0x48},
+		{0xc51a, 0x7},
+		{0xc51b, 0x70},
+		{0xc2e0, 0x0},
+		{0xc2e1, 0x51},
+		{0xc2e2, 0x0},
+		{0xc2e3, 0xd6},
+		{0xc2e4, 0x1},
+		{0xc2e5, 0x5e},
+		{0xc2e9, 0x1},
+		{0xc2ea, 0x7a},
+		{0xc2eb, 0x90},
+		{0xc2ed, 0x1},
+		{0xc2ee, 0x7a},
+		{0xc2ef, 0x64},
+		{0xc308, 0x0},
+		{0xc309, 0x0},
+		{0xc30a, 0x0},
+		{0xc30c, 0x0},
+		{0xc30d, 0x1},
+		{0xc30e, 0x0},
+		{0xc30f, 0x0},
+		{0xc310, 0x1},
+		{0xc311, 0x60},
+		{0xc312, 0xff},
+		{0xc313, 0x8},
+		{0xc314, 0x1},
+		{0xc315, 0x7f},
+		{0xc316, 0xff},
+		{0xc317, 0xb},
+		{0xc318, 0x0},
+		{0xc319, 0xc},
+		{0xc31a, 0x0},
+		{0xc31b, 0xe0},
+		{0xc31c, 0x0},
+		{0xc31d, 0x14},
+		{0xc31e, 0x0},
+		{0xc31f, 0xc5},
+		{0xc320, 0xff},
+		{0xc321, 0x4b},
+		{0xc322, 0xff},
+		{0xc323, 0xf0},
+		{0xc324, 0xff},
+		{0xc325, 0xe8},
+		{0xc326, 0x0},
+		{0xc327, 0x46},
+		{0xc328, 0xff},
+		{0xc329, 0xd2},
+		{0xc32a, 0xff},
+		{0xc32b, 0xe4},
+		{0xc32c, 0xff},
+		{0xc32d, 0xbb},
+		{0xc32e, 0x0},
+		{0xc32f, 0x61},
+		{0xc330, 0xff},
+		{0xc331, 0xf9},
+		{0xc332, 0x0},
+		{0xc333, 0xd9},
+		{0xc334, 0x0},
+		{0xc335, 0x2e},
+		{0xc336, 0x0},
+		{0xc337, 0xb1},
+		{0xc338, 0xff},
+		{0xc339, 0x64},
+		{0xc33a, 0xff},
+		{0xc33b, 0xeb},
+		{0xc33c, 0xff},
+		{0xc33d, 0xe8},
+		{0xc33e, 0x0},
+		{0xc33f, 0x48},
+		{0xc340, 0xff},
+		{0xc341, 0xd0},
+		{0xc342, 0xff},
+		{0xc343, 0xed},
+		{0xc344, 0xff},
+		{0xc345, 0xad},
+		{0xc346, 0x0},
+		{0xc347, 0x66},
+		{0xc348, 0x1},
+		{0xc349, 0x0},
+		{0x6700, 0x4},
+		{0x6701, 0x7b},
+		{0x6702, 0xfd},
+		{0x6703, 0xf9},
+		{0x6704, 0x3d},
+		{0x6705, 0x71},
+		/*
+		 * 0x6706[3:0] :: XVCLK
+		 * 0x6706[3:0] :: 0 = 6MHz
+		 * 0x6706[3:0] :: 1 = 9MHz
+		 * 0x6706[3:0] :: 8 = 24MHz
+		 * 0x6706[3:0] :: 9 = 27MHz
+		 */
+		{0x6706, 0x71},
+		{0x6708, 0x5},
+		{0x3822, 0x50},
+		{0x6f06, 0x6f},
+		{0x6f07, 0x0},
+		{0x6f0a, 0x6f},
+		{0x6f0b, 0x0},
+		{0x6f00, 0x3},
+		{0x3042, 0xf0},
+		{0x3042, 0xf0},
+		{0x3042, 0xf0},
+		{0x3042, 0xf0},
+		{0x3042, 0xf0},
+		{0x3042, 0xf0},
+		{0x3042, 0xf0},
+		{0x3042, 0xf0},
+		{0x3042, 0xf0},
+		{0x3042, 0xf0},
+		{0x3042, 0xf0},
+		{0x3042, 0xf0},
+		{0x3042, 0xf0},
+		{0x3042, 0xf0},
+		{0x3042, 0xf0},
+		{0x3042, 0xf0},
+		{0x3042, 0xf0},
+		{0x3042, 0xf0},
+		{0x3042, 0xf0},
+		{0x3042, 0xf0},
+		{0x3042, 0xf0},
+		{0x3042, 0xf0},
+		{0x3042, 0xf0},
+		{0x3042, 0xf0},
+		{0x3042, 0xf0},
+		{0x3042, 0xf0},
+		{0x301b, 0xf0},
+		{0x301c, 0xf0},
+		{0x301a, 0xf0},
+};
+
+static const struct ov1063x_reg ov1063x_regs_change_mode[] = {
+	{ 0x301b, 0xff }, { 0x301c, 0xff }, { 0x301a, 0xff }, { 0x5005, 0x08 },
+	{ 0x3007, 0x01 }, { 0x381c, 0x00 }, { 0x381f, 0x0C }, { 0x4001, 0x04 },
+	{ 0x4004, 0x08 }, { 0x4050, 0x20 }, { 0x4051, 0x22 }, { 0x6e47, 0x0C },
+	{ 0x4610, 0x05 }, { 0x4613, 0x10 },
+};
+
+static const struct ov1063x_reg ov1063x_regs_bt656[] = {
+	{ 0x4700, 0x02 }, { 0x4302, 0x03 }, { 0x4303, 0xf8 }, { 0x4304, 0x00 },
+	{ 0x4305, 0x08 }, { 0x4306, 0x03 }, { 0x4307, 0xf8 }, { 0x4308, 0x00 },
+	{ 0x4309, 0x08 },
+};
+
+static const struct ov1063x_reg ov1063x_regs_bt656_10bit[] = {
+	{ 0x4700, 0x02 }, { 0x4302, 0x03 }, { 0x4303, 0xfe }, { 0x4304, 0x00 },
+	{ 0x4305, 0x02 }, { 0x4306, 0x03 }, { 0x4307, 0xfe }, { 0x4308, 0x00 },
+	{ 0x4309, 0x02 },
+};
+
+static const struct ov1063x_reg ov1063x_regs_vert_sub_sample[] = {
+	{ 0x381f, 0x06 }, { 0x4001, 0x02 }, { 0x4004, 0x02 }, { 0x4050, 0x10 },
+	{ 0x4051, 0x11 }, { 0x6e47, 0x06 }, { 0x4610, 0x03 }, { 0x4613, 0x0a },
+};
+
+static const struct ov1063x_reg ov1063x_regs_enable[] = {
+	{ 0x3042, 0xf0 }, { 0x3042, 0xf0 }, { 0x3042, 0xf0 }, { 0x3042, 0xf0 },
+	{ 0x3042, 0xf0 }, { 0x3042, 0xf0 }, { 0x3042, 0xf0 }, { 0x3042, 0xf0 },
+	{ 0x3042, 0xf0 }, { 0x3042, 0xf0 }, { 0x3042, 0xf0 }, { 0x3042, 0xf0 },
+	{ 0x3042, 0xf0 }, { 0x3042, 0xf0 }, { 0x3042, 0xf0 }, { 0x3042, 0xf0 },
+	{ 0x3042, 0xf0 }, { 0x3042, 0xf0 }, { 0x3042, 0xf0 }, { 0x3042, 0xf0 },
+	{ 0x3042, 0xf0 }, { 0x3042, 0xf0 }, { 0x3042, 0xf0 }, { 0x3042, 0xf0 },
+	{ 0x3042, 0xf0 }, { 0x3042, 0xf0 }, { 0x301b, 0xf0 }, { 0x301c, 0xf0 },
+	{ 0x301a, 0xf0 },
+};
+
+/*
+ * Datasheet highlight the following sequence to enable and disable
+ * Test Pattern mode i.e. colobar
+ */
+static const struct ov1063x_reg ov1063x_regs_colorbar_enable[] = {
+	{0x5120, 0x01}, {0x56d0, 0x01}, {0x5300, 0x01}, {0x5301, 0x00},
+	{0x5302, 0x00},	{0x5303, 0x00},	{0x5304, 0x00},	{0x5305, 0x00},
+	{0x5306, 0x00},	{0x5307, 0x00},	{0x5308, 0x01},	{0x5309, 0x00},
+	{0x530a, 0x00},	{0x530b, 0x00},	{0x530c, 0x00},	{0x530d, 0x00},
+	{0x530e, 0x00},	{0x530f, 0x00},	{0x5310, 0x01},	{0x5311, 0x00},
+	{0x5312, 0x00},	{0x5313, 0x00},	{0x5314, 0x01},	{0x5315, 0x00},
+	{0x5316, 0x00},	{0x5317, 0x00},	{0x5318, 0x00},	{0x5319, 0x00},
+	{0x531a, 0x00},	{0x531b, 0x00},	{0x531c, 0x01},	{0x531d, 0x00},
+	{0x531e, 0x00},	{0x531f, 0x00},	{0x5320, 0x00},	{0x5321, 0x00},
+	{0x5322, 0x00},	{0x5323, 0x00},	{0x5324, 0x01},	{0x5325, 0x00},
+	{0x5326, 0x00},	{0x5327, 0x00},	{0xc2ea, 0x80},	{0xc2eb, 0x80},
+	{0x5000, 0x79},	{0x503d, 0x80},
+};
+
+static const struct ov1063x_reg ov1063x_regs_colorbar_disable[] = {
+	{0x503d, 0x00},	{0x5120, 0x00},	{0x56d0, 0x00},	{0x5300, 0x01},
+	{0x5301, 0x00},	{0x5302, 0x00},	{0x5303, 0x0e},	{0x5304, 0x00},
+	{0x5305, 0x0e},	{0x5306, 0x00},	{0x5307, 0x36},	{0x5308, 0x00},
+	{0x5309, 0xd9},	{0x530a, 0x00},	{0x530b, 0x0f},	{0x530c, 0x00},
+	{0x530d, 0x2c},	{0x530e, 0x00},	{0x530f, 0x59},	{0x5310, 0x00},
+	{0x5311, 0x7b},	{0x5312, 0x00},	{0x5313, 0x22},	{0x5314, 0x00},
+	{0x5315, 0xd5},	{0x5316, 0x00},	{0x5317, 0x13},	{0x5318, 0x00},
+	{0x5319, 0x18},	{0x531a, 0x00},	{0x531b, 0x26},	{0x531c, 0x00},
+	{0x531d, 0xdc},	{0x531e, 0x00},	{0x531f, 0x02},	{0x5320, 0x00},
+	{0x5321, 0x24},	{0x5322, 0x00},	{0x5323, 0x56},	{0x5324, 0x00},
+	{0x5325, 0x85},	{0x5326, 0x00},	{0x5327, 0x20},	{0xc2ea, 0x7a},
+	{0xc2eb, 0x90},	{0x5000, 0xff},
+};
diff -Naur --no-dereference a/drivers/media/i2c/ov2659.c b/drivers/media/i2c/ov2659.c
--- a/drivers/media/i2c/ov2659.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/media/i2c/ov2659.c	2020-06-19 21:00:31.416227715 -0400
@@ -32,6 +32,8 @@
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_graph.h>
+#include <linux/of_gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/slab.h>
 #include <linux/uaccess.h>
 #include <linux/videodev2.h>
@@ -232,6 +234,8 @@
 	struct sensor_register *format_ctrl_regs;
 	struct ov2659_pll_ctrl pll;
 	int streaming;
+	/* used to control the sensor powerdownN pin */
+	struct gpio_desc *pwrdn_gpio;
 };
 
 static const struct sensor_register ov2659_init_regs[] = {
@@ -665,7 +669,7 @@
 	{ REG_TIMING_HORIZ_FORMAT, 0x01 },
 	{ 0x370a, 0x52 },
 	{ REG_VFIFO_READ_START_H, 0x00 },
-	{ REG_VFIFO_READ_START_L, 0x80 },
+	{ REG_VFIFO_READ_START_L, 0xa0 },
 	{ REG_ISP_CTRL02, 0x10 },
 	{ REG_NULL, 0x00 },
 };
@@ -713,7 +717,7 @@
 	{ REG_TIMING_HORIZ_FORMAT, 0x01 },
 	{ 0x370a, 0x52 },
 	{ REG_VFIFO_READ_START_H, 0x00 },
-	{ REG_VFIFO_READ_START_L, 0x80 },
+	{ REG_VFIFO_READ_START_L, 0xa0 },
 	{ REG_ISP_CTRL02, 0x10 },
 	{ REG_NULL, 0x00 },
 };
@@ -1340,11 +1344,12 @@
 		unsigned short id;
 
 		id = OV265X_ID(pid, ver);
-		if (id != OV2659_ID)
+		if (id != OV2659_ID) {
 			dev_err(&client->dev,
 				"Sensor detection failed (%04X, %d)\n",
 				id, ret);
-		else {
+			ret = -ENODEV;
+		} else {
 			dev_info(&client->dev, "Found OV%04X sensor\n", id);
 			ret = ov2659_init(sd, 0);
 		}
@@ -1399,6 +1404,7 @@
 	struct v4l2_subdev *sd;
 	struct ov2659 *ov2659;
 	struct clk *clk;
+	struct gpio_desc *gpio;
 	int ret;
 
 	if (!pdata) {
@@ -1422,6 +1428,13 @@
 	    ov2659->xvclk_frequency > 27000000)
 		return -EINVAL;
 
+	/* Optional gpio don't fail if not present */
+	gpio = devm_gpiod_get_optional(&client->dev, "pwrdn", GPIOD_OUT_HIGH);
+	if (IS_ERR(gpio))
+		return PTR_ERR(gpio);
+
+	ov2659->pwrdn_gpio = gpio;
+
 	v4l2_ctrl_handler_init(&ov2659->ctrls, 2);
 	ov2659->link_frequency =
 			v4l2_ctrl_new_std(&ov2659->ctrls, &ov2659_ctrl_ops,
diff -Naur --no-dereference a/drivers/media/i2c/ov490.c b/drivers/media/i2c/ov490.c
--- a/drivers/media/i2c/ov490.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/media/i2c/ov490.c	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,676 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * OmniVision OV490 Camera Driver
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated -  http://www.ti.com/
+ * Author: Nikhil Devshatwar <nikhil.nd@ti.com>
+ */
+
+#include <linux/delay.h>
+#include <linux/i2c.h>
+#include <linux/gpio.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+#include <linux/v4l2-mediabus.h>
+#include <linux/videodev2.h>
+#include <linux/mutex.h>
+#include <linux/delay.h>
+
+#include <media/soc_camera.h>
+#include <media/v4l2-async.h>
+#include <media/v4l2-common.h>
+
+#include <linux/gpio/consumer.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/of_graph.h>
+#include <media/v4l2-fwnode.h>
+#include <media/v4l2-event.h>
+#include <media/v4l2-subdev.h>
+
+/* Register definitions */
+#define OV490_PID			0x300a
+#define OV490_VER			0x300b
+#define OV490_BANK_HIGH			0xfffd
+#define OV490_BANK_LOW			0xfffe
+
+#define OV490_MIPI_TX_LANE_CTRL2	0x8029202D
+#define OV490_MIPI_TX_LANE_CTRL0	0x80292015
+
+#define OV490_SC_RESET1			0x80800011
+#define OV490_FW_VER_HIGH		0x80800102
+#define OV490_FW_VER_LOW		0x80800103
+
+#define OV490_IMAGE0_CTRL		0x8082000a
+#define OV490_IMAGE0_BYTE_INVERT	BIT(7)
+#define OV490_IMAGE0_BYTE_SEQUENCE	GEN_MASK(6, 4)
+#define OV490_IMAGE0_BYTE_SEQUENCE_1	BIT(4)
+#define OV490_IMAGE0_FORMAT_SELECT	GEN_MASK(3, 0)
+#define OV490_IMAGE0_FORMAT_PURE_RAW12		0x0
+#define OV490_IMAGE0_FORMAT_3X12_RAW		0x2
+#define OV490_IMAGE0_FORMAT_2X12_RAW		0x3
+#define OV490_IMAGE0_FORMAT_20_COMBINED_RAW	0x4
+#define OV490_IMAGE0_FORMAT_16_COMPRESSED_RAW	0x5
+#define OV490_IMAGE0_FORMAT_12_COMPRESSED_RAW	0x6
+#define OV490_IMAGE0_FORMAT_2X12_COMPRESSED_RAW	0x7
+#define OV490_IMAGE0_FORMAT_12_SELECTED_RAW	0x8
+#define OV490_IMAGE0_FORMAT_12_YUV422		0xf
+
+/* IDs */
+#define OV490_VERSION_REG		0x0490
+#define OV490_VERSION(pid, ver)	(((pid) << 8) | ((ver) & 0xff))
+
+#define OV490_MAX_WIDTH			1280
+#define OV490_MAX_HEIGHT		800
+
+#define MAX_NUM_GPIOS			10
+
+/* Host Command Flow */
+#define OV490_STATUS_ADDR		(0x80195ffc)
+#define OV490_HOST_CMD_PARA_ADDR	(0x80195000)
+#define OV490_HOST_CMD_RESULT_ADDR	(0x80195000)
+#define OV490_HOST_INT_ADDR		(0x808000c0)
+#define OV490_STATUS_FINISH		(0x99)
+#define OV490_STATUS_ERROR		(0x55)
+
+/* Host Command List */
+#define OV490_CMD_BRIGHTNESS_SET	(0xf1)
+#define OV490_CMD_SATURATION_SET	(0xf3)
+#define OV490_CMD_HUE_SET		(0xf5)
+#define OV490_CMD_FRAMERATE_SET		(0xf7)
+#define OV490_CMD_GAMMA_SET		(0xf9)
+#define OV490_CMD_SHARPNESS_SET		(0xfb)
+#define OV490_CMD_CONTRAST_SET		(0xfd)
+#define OV490_CMD_GROUPWRITE_SET	(0xe1)
+#define OV490_CMD_STREAMING_CTRL	(0xe2)
+#define OV490_CMD_CONTEXT_SWITCH_CONFIG	(0xe3)
+#define OV490_CMD_CONTEXT_SWITCH_CTRL	(0xe4)
+#define OV490_CMD_MULT_CMD		(0xe5)
+#define OV490_CMD_GPIO_SET		(0xe6)
+#define OV490_CMD_GPIO_GET		(0xe7)
+#define OV490_CMD_FORMAT_SET		(0xe8)
+#define OV490_CMD_TEMP_GET		(0xe9)
+#define OV490_CMD_EXPOSURE_GAIN_SET	(0xea)
+#define OV490_CMD_AWBGAIN_SET		(0xeb)
+#define OV490_CMD_DENOISE_SET		(0xec)
+#define OV490_CMD_TONECURVE_SET		(0xed)
+#define OV490_CMD_COMB_WEIGHT_SET	(0xee)
+#define OV490_CMD_AEC_WEIGHT_SET	(0xd2)
+#define OV490_CMD_AWB_ROI_SET		(0xd3)
+#define OV490_CMD_TONEMAPPING_ROI_SET	(0xd4)
+#define OV490_CMD_STAT_ROI_SET		(0xd5)
+#define OV490_CMD_TESTPATTERN_SET	(0xd6)
+#define OV490_CMD_MTF_SET		(0xd7)
+#define OV490_CMD_LENC_SET		(0xd8)
+#define OV490_CMD_BLC_SET		(0xd9)
+#define OV490_CMD_GROUPWRITE_LAUNCH	(0xda)
+#define OV490_CMD_EMBLINE_CTRL		(0xdb)
+#define OV490_CMD_MIRRFLIP_CTRL		(0xdc)
+#define OV490_CMD_EXTRA_VTS_SET		(0xde)
+#define OV490_CMD_SNR_REG_ACCESS	(0xc1)
+#define OV490_CMD_POSTAWBGAIN_SET	(0xc2)
+#define OV490_CMD_CROP_SET		(0xc3)
+#define OV490_CMD_FRAMESYNC		(0xc4)
+#define OV490_CMD_BANDING_SET		(0xc5)
+#define OV490_CMD_TOPEMB_SET		(0xc7)
+#define OV490_CMD_FWREG_ACCESS		(0x35)
+#define OV490_CMD_FADE_CTRL		(0x37)
+#define OV490_CMD_INIT_CTRL		(0x39)
+#define OV490_CMD_RESET_CTRL		(0x3a)
+
+/*
+ * = fvco / pixel_width * num_lanes
+ * = 804,000,000 / 16 bits * 4 lanes
+ */
+#define OV490_PIXEL_RATE_PER_LANE	50250000
+
+static u8 ov490_init_param[] = {
+	0x31,	/* [3:0]input port, [7:4]  input format */
+	0x01,	/* [3:0]output port, [7:4]  output format */
+	0x05,	/* in width: 1288 */
+	0x08,
+	0x04,	/* in height:1080 */
+	0x40,
+	0x05,	/* out width: 1288 */
+	0x08,
+	0x04,	/* out height:1080 */
+	0x40,
+	0x00,
+};
+
+struct ov490_color_format {
+	u32 code;
+	u32 colorspace;
+};
+
+struct ov490_priv {
+	struct v4l2_subdev		subdev;
+	struct v4l2_async_subdev	asd;
+	const struct ov490_color_format	*cfmt;
+	int				width;
+	int				height;
+	int				num_lanes;
+
+	struct regmap			*regmap;
+
+	struct gpio_descs		*mux_gpios;
+
+	struct v4l2_ctrl_handler	handler;
+	struct v4l2_ctrl		*pixel_rate;
+};
+
+/* Main access control */
+DEFINE_MUTEX(ov490_lock);
+static int ov490_init_gpios(struct i2c_client *client);
+
+/*
+ * supported color format list
+ */
+static const struct ov490_color_format ov490_cfmts[] = {
+	{
+		.code		= MEDIA_BUS_FMT_YUYV8_2X8,
+		.colorspace	= V4L2_COLORSPACE_SMPTE170M,
+	},
+};
+
+static struct ov490_priv *to_ov490(const struct i2c_client *client)
+{
+	return container_of(i2c_get_clientdata(client), struct ov490_priv,
+			    subdev);
+}
+
+static struct v4l2_subdev *ctrl_to_sd(struct v4l2_ctrl *ctrl)
+{
+	return &container_of(ctrl->handler, struct ov490_priv, handler)->subdev;
+}
+
+static int ov490_reg_write32(struct regmap *map, u32 reg, u8 val)
+{
+	u8 bank_high = (reg >> 24) & 0xff;
+	u8 bank_low  = (reg >> 16) & 0xff;
+	u16 reg_addr = reg & 0xffff;
+	int ret = 0;
+
+	/* For writing a register with 32 bit address, First set the bank
+	 * address by writing to two BANK address registers. Then access
+	 * the register using 16LSB bits.
+	 */
+	ret = regmap_write(map, OV490_BANK_HIGH, bank_high);
+	if (!ret)
+		ret = regmap_write(map, OV490_BANK_LOW, bank_low);
+	if (!ret)
+		ret = regmap_write(map, reg_addr, val);
+	return ret;
+}
+
+static int ov490_reg_read32(struct regmap *map, u32 reg, u8 *val)
+{
+	u8 bank_high = (reg >> 24) & 0xff;
+	u8 bank_low  = (reg >> 16) & 0xff;
+	u16 reg_addr = reg & 0xffff;
+	int ret = 0;
+	u32 tval = 0;
+
+	/*
+	 * For reading a register with 32 bit address, First set the bank
+	 * address by writing to two BANK address registers. Then access
+	 * the register using 16LSB bits.
+	 */
+	ret = regmap_write(map, OV490_BANK_HIGH, bank_high);
+	if (!ret)
+		ret = regmap_write(map, OV490_BANK_LOW, bank_low);
+	if (!ret)
+		ret = regmap_read(map, reg_addr, &tval);
+
+	*val = (u8)tval;
+	return ret;
+}
+
+static int ov490_host_control_set(struct regmap *map, u8 host_cmd,
+				  u8 *param, u16 number)
+{
+	int i, ret;
+	u8 status = 0;
+
+	/* Host reset OV490_status_register */
+	ret = ov490_reg_write32(map, OV490_STATUS_ADDR, 0);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < number; i++) {
+		ret = ov490_reg_write32(map, OV490_HOST_CMD_PARA_ADDR + i,
+					*(param + i));
+		if (ret)
+			return ret;
+	}
+
+	ret = ov490_reg_write32(map, OV490_HOST_INT_ADDR, host_cmd);
+	if (ret)
+		return ret;
+
+	for (i = 500; i && status != OV490_STATUS_FINISH; i--) {
+		ret = ov490_reg_read32(map, OV490_STATUS_ADDR, &status);
+		if (ret)
+			return ret;
+		usleep_range(500, 1000);
+	}
+
+	if (!i)
+		return -ETIMEDOUT;
+
+	return 0;
+}
+
+static int __maybe_unused ov490_host_control_get(struct regmap *map,
+						 u8 host_cmd, u8 *param,
+						 u16 number)
+{
+	int i, ret;
+	u8 status = 0;
+
+	/* Host reset OV490_status_register */
+	ret = ov490_reg_write32(map, OV490_STATUS_ADDR, 0);
+	if (ret)
+		return ret;
+
+	ret = ov490_reg_write32(map, OV490_HOST_INT_ADDR, host_cmd);
+	if (ret)
+		return ret;
+
+	for (i = 500; i && status != OV490_STATUS_FINISH; i--) {
+		ret = ov490_reg_read32(map, OV490_STATUS_ADDR, &status);
+		if (ret)
+			return ret;
+		usleep_range(500, 1000);
+	}
+
+	if (!i)
+		return -ETIMEDOUT;
+
+	for (i = 0; i < number; i++) {
+		ret = ov490_reg_read32(map, OV490_HOST_CMD_PARA_ADDR + i,
+				       (param + i));
+		if (ret)
+			return ret;
+	}
+	return 0;
+}
+
+static int ov490_get_fw_version(struct regmap *map, u16 *id)
+{
+	u8 hi, lo;
+	int ret;
+
+	ret = ov490_reg_read32(map, OV490_FW_VER_HIGH, &hi);
+	if (ret)
+		return ret;
+
+	ret = ov490_reg_read32(map, OV490_FW_VER_LOW, &lo);
+	if (ret)
+		return ret;
+
+	*id = hi << 8 | lo;
+
+	return 0;
+}
+
+/* Start/Stop streaming from the device */
+static int ov490_s_stream(struct v4l2_subdev *sd, int enable)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(sd);
+	struct ov490_priv *priv = to_ov490(client);
+	struct regmap *map = priv->regmap;
+	int ret, val;
+	u8 streaming;
+
+	mutex_lock(&ov490_lock);
+	ret = ov490_init_gpios(client);
+	if (ret) {
+		dev_err(&client->dev, "Failed to request gpios");
+		goto unlock;
+	}
+
+	if (!enable) {
+		streaming = 0;
+		/* Stop Streaming */
+		ret = ov490_host_control_set(map, OV490_CMD_STREAMING_CTRL,
+					     &streaming, 1);
+		if (ret)
+			goto unlock;
+		ret = ov490_reg_write32(map, OV490_MIPI_TX_LANE_CTRL0, 0xa0);
+		if (ret)
+			goto unlock;
+		/* Put MIPI_TX in reset */
+		ret = ov490_reg_write32(map, OV490_SC_RESET1, 0x80);
+		goto unlock;
+	}
+
+	/* Take MIPI_TX out of reset */
+	ret = ov490_reg_write32(map, OV490_SC_RESET1, 0x00);
+	if (ret)
+		goto unlock;
+	ret = ov490_reg_write32(map, OV490_MIPI_TX_LANE_CTRL0, 0x80);
+	if (ret)
+		goto unlock;
+
+	/* Initialize slave */
+	ret = ov490_host_control_set(map, OV490_CMD_INIT_CTRL, ov490_init_param,
+				     ARRAY_SIZE(ov490_init_param));
+	if (ret)
+		goto unlock;
+
+	ret = ov490_reg_write32(map, OV490_IMAGE0_CTRL,
+				OV490_IMAGE0_BYTE_INVERT |
+				OV490_IMAGE0_BYTE_SEQUENCE_1 |
+				OV490_IMAGE0_FORMAT_3X12_RAW);
+	if (ret)
+		goto unlock;
+
+	/* Set number of data lane to use */
+	val = priv->num_lanes == 2 ? 0x03 : priv->num_lanes == 4 ? 0x0F : 0x0F;
+	ret = ov490_reg_write32(map, OV490_MIPI_TX_LANE_CTRL2, val);
+	if (ret)
+		goto unlock;
+
+	/* Start Streaming */
+	streaming = 0x1;
+	ret = ov490_host_control_set(map, OV490_CMD_STREAMING_CTRL,
+				     &streaming, 1);
+
+unlock:
+	mutex_unlock(&ov490_lock);
+	return ret;
+}
+
+static int ov490_get_fmt(struct v4l2_subdev *sd,
+			 struct v4l2_subdev_pad_config *cfg,
+			 struct v4l2_subdev_format *fmt)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(sd);
+	struct ov490_priv *priv = to_ov490(client);
+	struct v4l2_mbus_framefmt *mf = &fmt->format;
+
+	mf->width	= priv->width;
+	mf->height	= priv->height;
+	mf->code	= priv->cfmt->code;
+	mf->colorspace	= priv->cfmt->colorspace;
+	mf->field	= V4L2_FIELD_NONE;
+
+	return 0;
+}
+
+/* Fixed format - no configurability */
+static int ov490_set_fmt(struct v4l2_subdev *sd,
+			 struct v4l2_subdev_pad_config *cfg,
+			 struct v4l2_subdev_format *fmt)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(sd);
+	struct ov490_priv *priv = to_ov490(client);
+	struct v4l2_mbus_framefmt *mf = &fmt->format;
+
+	mf->width	= priv->width;
+	mf->height	= priv->height;
+	mf->code	= priv->cfmt->code;
+	mf->colorspace	= priv->cfmt->colorspace;
+	mf->field	= V4L2_FIELD_NONE;
+
+	return 0;
+}
+
+static int ov490_enum_code(struct v4l2_subdev *sd,
+			   struct v4l2_subdev_pad_config *cfg,
+			   struct v4l2_subdev_mbus_code_enum *code)
+{
+	if (code->index >= ARRAY_SIZE(ov490_cfmts))
+		return -EINVAL;
+
+	code->code = ov490_cfmts[code->index].code;
+
+	return 0;
+}
+
+static int ov490_enum_size(struct v4l2_subdev *sd,
+			   struct v4l2_subdev_pad_config *cfg,
+			   struct v4l2_subdev_frame_size_enum *fse)
+{
+	int cam_width[] =	{ OV490_MAX_WIDTH };
+	int cam_height[] =	{ OV490_MAX_HEIGHT };
+
+	if (fse->index >= ARRAY_SIZE(cam_width))
+		return -EINVAL;
+
+	fse->min_width  = cam_width[fse->index];
+	fse->max_width  = fse->min_width;
+	fse->min_height = cam_height[fse->index];
+	fse->max_height = fse->min_height;
+	return 0;
+}
+
+static int ov490_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct v4l2_subdev *sd = ctrl_to_sd(ctrl);
+	struct i2c_client *client = v4l2_get_subdevdata(sd);
+
+	/* If the board has programmable pixel clock, set it here */
+	if (ctrl->id == V4L2_CID_PIXEL_RATE)
+		dev_info(&client->dev, "Pixel rate set to %d\n",
+			 ctrl->val);
+	return 0;
+}
+
+static int ov490_init_gpios(struct i2c_client *client)
+{
+	struct ov490_priv *priv = to_ov490(client);
+	int ret = 0;
+
+	/* Request the gpio lines and set the values
+	 * then release them so that other drivers can use them
+	 * This allows changing common board muxes which are
+	 * controlled by multiple drivers
+	 */
+	priv->mux_gpios = gpiod_get_array(&client->dev, "mux", GPIOD_OUT_HIGH);
+	if (IS_ERR(priv->mux_gpios))
+		goto done;
+	gpiod_put_array(priv->mux_gpios);
+done:
+	return ret;
+}
+
+static int ov490_video_probe(struct i2c_client *client)
+{
+	struct ov490_priv *priv = i2c_get_clientdata(client);
+	u32 pid, ver;
+	int ret;
+	u16 version;
+
+	/* check and show product ID and manufacturer ID */
+	ret = regmap_read(priv->regmap, OV490_PID, &pid);
+	if (ret)
+		return ret;
+
+	ret = regmap_read(priv->regmap, OV490_VER, &ver);
+	if (ret)
+		return ret;
+
+	if (OV490_VERSION(pid, ver) != OV490_VERSION_REG) {
+		dev_err(&client->dev, "Product ID error %02x:%02x\n", pid, ver);
+		return -ENODEV;
+	}
+
+	dev_info(&client->dev, "ov490 Product ID %02x Manufacturer ID %02x\n",
+		 pid, ver);
+
+	ret = ov490_get_fw_version(priv->regmap, &version);
+	if (ret)
+		return ret;
+
+	dev_info(&client->dev, "ov490 Firmware ID 0x%04x\n", version);
+
+	return 0;
+}
+
+static const struct v4l2_subdev_video_ops ov490_video_ops = {
+	.s_stream		= ov490_s_stream,
+};
+
+static const struct v4l2_subdev_core_ops ov490_core_ops = {
+	.log_status		= v4l2_ctrl_subdev_log_status,
+	.subscribe_event	= v4l2_ctrl_subdev_subscribe_event,
+	.unsubscribe_event	= v4l2_event_subdev_unsubscribe,
+
+};
+
+static const struct v4l2_subdev_pad_ops ov490_pad_ops = {
+	.enum_mbus_code		= ov490_enum_code,
+	.enum_frame_size	= ov490_enum_size,
+	.get_fmt		= ov490_get_fmt,
+	.set_fmt		= ov490_set_fmt,
+};
+
+static const struct v4l2_subdev_ops ov490_subdev_ops = {
+	.video	= &ov490_video_ops,
+	.core	= &ov490_core_ops,
+	.pad	= &ov490_pad_ops,
+};
+
+static const struct v4l2_ctrl_ops ov490_ctrl_ops = {
+	.s_ctrl = ov490_s_ctrl,
+};
+
+static const struct regmap_config ov490_regmap_config = {
+	.reg_bits = 16,
+	.val_bits = 8,
+};
+
+/*
+ * i2c_driver function
+ */
+
+static int ov490_of_probe(struct i2c_client *client,
+			  struct device_node *node)
+{
+	struct ov490_priv *priv = to_ov490(client);
+	struct v4l2_fwnode_endpoint endpoint = {};
+	struct device_node *ep;
+	int num_lanes = 0;
+
+	ep = of_graph_get_next_endpoint(node, NULL);
+	if (ep) {
+		v4l2_fwnode_endpoint_parse(of_fwnode_handle(ep), &endpoint);
+		if (endpoint.bus_type == V4L2_MBUS_CSI2) {
+			num_lanes = endpoint.bus.mipi_csi2.num_data_lanes;
+			if (num_lanes == 2 || num_lanes == 4)
+				priv->num_lanes = num_lanes;
+		} else {
+			dev_err(&client->dev, "Endpoint bus is not CSI bus!");
+		}
+	}
+
+	dev_info(&client->dev, "Using %d data lanes\n", priv->num_lanes);
+
+	return 0;
+}
+
+static int ov490_probe(struct i2c_client *client,
+		       const struct i2c_device_id *did)
+{
+	struct device_node *node = client->dev.of_node;
+	struct v4l2_ctrl_handler *hdl;
+	struct ov490_priv *priv;
+	struct v4l2_subdev *sd;
+	int ret = 0;
+
+	priv = devm_kzalloc(&client->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	i2c_set_clientdata(client, priv);
+	priv->num_lanes = 4;
+	priv->cfmt = &ov490_cfmts[0];
+	priv->width = OV490_MAX_WIDTH;
+	priv->height = OV490_MAX_HEIGHT;
+
+	priv->regmap = devm_regmap_init_i2c(client, &ov490_regmap_config);
+	if (IS_ERR(priv->regmap))
+		return PTR_ERR(priv->regmap);
+
+	ret = ov490_video_probe(client);
+	if (ret)
+		goto err;
+
+	ret = ov490_of_probe(client, node);
+	if (ret)
+		goto err;
+
+	sd = &priv->subdev;
+	v4l2_i2c_subdev_init(sd, client, &ov490_subdev_ops);
+
+	hdl = &priv->handler;
+	sd->ctrl_handler = hdl;
+	v4l2_ctrl_handler_init(hdl, 1);
+	priv->pixel_rate =
+		v4l2_ctrl_new_std(hdl, &ov490_ctrl_ops,
+				  V4L2_CID_PIXEL_RATE, 1, INT_MAX, 1,
+				  OV490_PIXEL_RATE_PER_LANE * priv->num_lanes);
+
+	if (hdl->error) {
+		dev_err(&client->dev, "Failed to add controls");
+		ret = hdl->error;
+		goto err;
+	}
+
+	ret = ov490_init_gpios(client);
+	if (ret) {
+		dev_err(&client->dev, "Failed to request gpios");
+		goto err;
+	}
+
+	sd->dev = &client->dev;
+	ret = v4l2_async_register_subdev(sd);
+
+err:
+	return ret;
+}
+
+static int ov490_remove(struct i2c_client *client)
+{
+	struct ov490_priv *priv = i2c_get_clientdata(client);
+
+	v4l2_device_unregister_subdev(&priv->subdev);
+	v4l2_ctrl_handler_free(&priv->handler);
+
+	return 0;
+}
+
+static const struct i2c_device_id ov490_id[] = {
+	{ "ov490", 0 },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, ov490_id);
+
+static const struct of_device_id ov490_dt_id[] = {
+	{
+		.compatible = "ovti,ov490",
+	},
+	{
+	}
+};
+MODULE_DEVICE_TABLE(of, ov490_dt_id);
+
+static struct i2c_driver ov490_i2c_driver = {
+	.driver = {
+		.name	= "ov490",
+		.of_match_table = ov490_dt_id,
+	},
+	.probe = ov490_probe,
+	.remove = ov490_remove,
+	.id_table = ov490_id,
+};
+
+module_i2c_driver(ov490_i2c_driver);
+
+MODULE_DESCRIPTION("SoC Camera driver for OmniVision OV490");
+MODULE_AUTHOR("Nikhil Devshatwar <nikhil.nd@ti.com>");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/media/i2c/ov5640.c b/drivers/media/i2c/ov5640.c
--- a/drivers/media/i2c/ov5640.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/media/i2c/ov5640.c	2020-06-19 21:00:31.416227715 -0400
@@ -25,6 +25,7 @@
 #include <media/v4l2-async.h>
 #include <media/v4l2-ctrls.h>
 #include <media/v4l2-device.h>
+#include <media/v4l2-event.h>
 #include <media/v4l2-fwnode.h>
 #include <media/v4l2-subdev.h>
 
@@ -94,9 +95,6 @@
 #define OV5640_REG_SDE_CTRL5		0x5585
 #define OV5640_REG_AVG_READOUT		0x56a1
 
-#define OV5640_SCLK2X_ROOT_DIVIDER_DEFAULT	1
-#define OV5640_SCLK_ROOT_DIVIDER_DEFAULT	2
-
 enum ov5640_mode_id {
 	OV5640_MODE_QCIF_176_144 = 0,
 	OV5640_MODE_QVGA_320_240,
@@ -113,6 +111,7 @@
 enum ov5640_frame_rate {
 	OV5640_15_FPS = 0,
 	OV5640_30_FPS,
+	OV5640_60_FPS,
 	OV5640_NUM_FRAMERATES,
 };
 
@@ -141,6 +140,7 @@
 static const int ov5640_framerates[] = {
 	[OV5640_15_FPS] = 15,
 	[OV5640_30_FPS] = 30,
+	[OV5640_60_FPS] = 60,
 };
 
 /* regulator supplies */
@@ -182,6 +182,9 @@
 struct ov5640_ctrls {
 	struct v4l2_ctrl_handler handler;
 	struct {
+		struct v4l2_ctrl *pixel_rate;
+	};
+	struct {
 		struct v4l2_ctrl *auto_exp;
 		struct v4l2_ctrl *exposure;
 	};
@@ -229,6 +232,7 @@
 	const struct ov5640_mode_info *last_mode;
 	enum ov5640_frame_rate current_fr;
 	struct v4l2_fract frame_interval;
+	u64 pixel_rate;
 
 	struct ov5640_ctrls ctrls;
 
@@ -261,8 +265,7 @@
 static const struct reg_value ov5640_init_setting_30fps_VGA[] = {
 	{0x3103, 0x11, 0, 0}, {0x3008, 0x82, 0, 5}, {0x3008, 0x42, 0, 0},
 	{0x3103, 0x03, 0, 0}, {0x3017, 0x00, 0, 0}, {0x3018, 0x00, 0, 0},
-	{0x3034, 0x18, 0, 0}, {0x3035, 0x14, 0, 0}, {0x3036, 0x38, 0, 0},
-	{0x3037, 0x13, 0, 0}, {0x3630, 0x36, 0, 0},
+	{0x3630, 0x36, 0, 0},
 	{0x3631, 0x0e, 0, 0}, {0x3632, 0xe2, 0, 0}, {0x3633, 0x12, 0, 0},
 	{0x3621, 0xe0, 0, 0}, {0x3704, 0xa0, 0, 0}, {0x3703, 0x5a, 0, 0},
 	{0x3715, 0x78, 0, 0}, {0x3717, 0x01, 0, 0}, {0x370b, 0x60, 0, 0},
@@ -344,85 +347,8 @@
 	{0x3a1f, 0x14, 0, 0}, {0x3008, 0x02, 0, 0}, {0x3c00, 0x04, 0, 300},
 };
 
-static const struct reg_value ov5640_setting_30fps_VGA_640_480[] = {
-	{0x3035, 0x14, 0, 0}, {0x3036, 0x38, 0, 0}, {0x3c07, 0x08, 0, 0},
-	{0x3c09, 0x1c, 0, 0}, {0x3c0a, 0x9c, 0, 0}, {0x3c0b, 0x40, 0, 0},
-	{0x3814, 0x31, 0, 0},
-	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
-	{0x3802, 0x00, 0, 0}, {0x3803, 0x04, 0, 0}, {0x3804, 0x0a, 0, 0},
-	{0x3805, 0x3f, 0, 0}, {0x3806, 0x07, 0, 0}, {0x3807, 0x9b, 0, 0},
-	{0x3810, 0x00, 0, 0},
-	{0x3811, 0x10, 0, 0}, {0x3812, 0x00, 0, 0}, {0x3813, 0x06, 0, 0},
-	{0x3618, 0x00, 0, 0}, {0x3612, 0x29, 0, 0}, {0x3708, 0x64, 0, 0},
-	{0x3709, 0x52, 0, 0}, {0x370c, 0x03, 0, 0}, {0x3a02, 0x03, 0, 0},
-	{0x3a03, 0xd8, 0, 0}, {0x3a08, 0x01, 0, 0}, {0x3a09, 0x0e, 0, 0},
-	{0x3a0a, 0x00, 0, 0}, {0x3a0b, 0xf6, 0, 0}, {0x3a0e, 0x03, 0, 0},
-	{0x3a0d, 0x04, 0, 0}, {0x3a14, 0x03, 0, 0}, {0x3a15, 0xd8, 0, 0},
-	{0x4001, 0x02, 0, 0}, {0x4004, 0x02, 0, 0}, {0x4713, 0x03, 0, 0},
-	{0x4407, 0x04, 0, 0}, {0x460b, 0x35, 0, 0}, {0x460c, 0x22, 0, 0},
-	{0x3824, 0x02, 0, 0}, {0x5001, 0xa3, 0, 0}, {0x3503, 0x00, 0, 0},
-};
-
-static const struct reg_value ov5640_setting_15fps_VGA_640_480[] = {
-	{0x3035, 0x22, 0, 0}, {0x3036, 0x38, 0, 0}, {0x3c07, 0x08, 0, 0},
-	{0x3c09, 0x1c, 0, 0}, {0x3c0a, 0x9c, 0, 0}, {0x3c0b, 0x40, 0, 0},
-	{0x3814, 0x31, 0, 0},
-	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
-	{0x3802, 0x00, 0, 0}, {0x3803, 0x04, 0, 0}, {0x3804, 0x0a, 0, 0},
-	{0x3805, 0x3f, 0, 0}, {0x3806, 0x07, 0, 0}, {0x3807, 0x9b, 0, 0},
-	{0x3810, 0x00, 0, 0},
-	{0x3811, 0x10, 0, 0}, {0x3812, 0x00, 0, 0}, {0x3813, 0x06, 0, 0},
-	{0x3618, 0x00, 0, 0}, {0x3612, 0x29, 0, 0}, {0x3708, 0x64, 0, 0},
-	{0x3709, 0x52, 0, 0}, {0x370c, 0x03, 0, 0}, {0x3a02, 0x03, 0, 0},
-	{0x3a03, 0xd8, 0, 0}, {0x3a08, 0x01, 0, 0}, {0x3a09, 0x27, 0, 0},
-	{0x3a0a, 0x00, 0, 0}, {0x3a0b, 0xf6, 0, 0}, {0x3a0e, 0x03, 0, 0},
-	{0x3a0d, 0x04, 0, 0}, {0x3a14, 0x03, 0, 0}, {0x3a15, 0xd8, 0, 0},
-	{0x4001, 0x02, 0, 0}, {0x4004, 0x02, 0, 0}, {0x4713, 0x03, 0, 0},
-	{0x4407, 0x04, 0, 0}, {0x460b, 0x35, 0, 0}, {0x460c, 0x22, 0, 0},
-	{0x3824, 0x02, 0, 0}, {0x5001, 0xa3, 0, 0},
-};
-
-static const struct reg_value ov5640_setting_30fps_XGA_1024_768[] = {
-	{0x3035, 0x14, 0, 0}, {0x3036, 0x38, 0, 0}, {0x3c07, 0x08, 0, 0},
-	{0x3c09, 0x1c, 0, 0}, {0x3c0a, 0x9c, 0, 0}, {0x3c0b, 0x40, 0, 0},
-	{0x3814, 0x31, 0, 0},
-	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
-	{0x3802, 0x00, 0, 0}, {0x3803, 0x04, 0, 0}, {0x3804, 0x0a, 0, 0},
-	{0x3805, 0x3f, 0, 0}, {0x3806, 0x07, 0, 0}, {0x3807, 0x9b, 0, 0},
-	{0x3810, 0x00, 0, 0},
-	{0x3811, 0x10, 0, 0}, {0x3812, 0x00, 0, 0}, {0x3813, 0x06, 0, 0},
-	{0x3618, 0x00, 0, 0}, {0x3612, 0x29, 0, 0}, {0x3708, 0x64, 0, 0},
-	{0x3709, 0x52, 0, 0}, {0x370c, 0x03, 0, 0}, {0x3a02, 0x03, 0, 0},
-	{0x3a03, 0xd8, 0, 0}, {0x3a08, 0x01, 0, 0}, {0x3a09, 0x0e, 0, 0},
-	{0x3a0a, 0x00, 0, 0}, {0x3a0b, 0xf6, 0, 0}, {0x3a0e, 0x03, 0, 0},
-	{0x3a0d, 0x04, 0, 0}, {0x3a14, 0x03, 0, 0}, {0x3a15, 0xd8, 0, 0},
-	{0x4001, 0x02, 0, 0}, {0x4004, 0x02, 0, 0}, {0x4713, 0x03, 0, 0},
-	{0x4407, 0x04, 0, 0}, {0x460b, 0x35, 0, 0}, {0x460c, 0x22, 0, 0},
-	{0x3824, 0x02, 0, 0}, {0x5001, 0xa3, 0, 0}, {0x3503, 0x00, 0, 0},
-	{0x3035, 0x12, 0, 0},
-};
-
-static const struct reg_value ov5640_setting_15fps_XGA_1024_768[] = {
-	{0x3035, 0x22, 0, 0}, {0x3036, 0x38, 0, 0}, {0x3c07, 0x08, 0, 0},
-	{0x3c09, 0x1c, 0, 0}, {0x3c0a, 0x9c, 0, 0}, {0x3c0b, 0x40, 0, 0},
-	{0x3814, 0x31, 0, 0},
-	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
-	{0x3802, 0x00, 0, 0}, {0x3803, 0x04, 0, 0}, {0x3804, 0x0a, 0, 0},
-	{0x3805, 0x3f, 0, 0}, {0x3806, 0x07, 0, 0}, {0x3807, 0x9b, 0, 0},
-	{0x3810, 0x00, 0, 0},
-	{0x3811, 0x10, 0, 0}, {0x3812, 0x00, 0, 0}, {0x3813, 0x06, 0, 0},
-	{0x3618, 0x00, 0, 0}, {0x3612, 0x29, 0, 0}, {0x3708, 0x64, 0, 0},
-	{0x3709, 0x52, 0, 0}, {0x370c, 0x03, 0, 0}, {0x3a02, 0x03, 0, 0},
-	{0x3a03, 0xd8, 0, 0}, {0x3a08, 0x01, 0, 0}, {0x3a09, 0x27, 0, 0},
-	{0x3a0a, 0x00, 0, 0}, {0x3a0b, 0xf6, 0, 0}, {0x3a0e, 0x03, 0, 0},
-	{0x3a0d, 0x04, 0, 0}, {0x3a14, 0x03, 0, 0}, {0x3a15, 0xd8, 0, 0},
-	{0x4001, 0x02, 0, 0}, {0x4004, 0x02, 0, 0}, {0x4713, 0x03, 0, 0},
-	{0x4407, 0x04, 0, 0}, {0x460b, 0x35, 0, 0}, {0x460c, 0x22, 0, 0},
-	{0x3824, 0x02, 0, 0}, {0x5001, 0xa3, 0, 0},
-};
-
-static const struct reg_value ov5640_setting_30fps_QVGA_320_240[] = {
-	{0x3035, 0x14, 0, 0}, {0x3036, 0x38, 0, 0}, {0x3c07, 0x08, 0, 0},
+static const struct reg_value ov5640_setting_VGA_640_480[] = {
+	{0x3c07, 0x08, 0, 0},
 	{0x3c09, 0x1c, 0, 0}, {0x3c0a, 0x9c, 0, 0}, {0x3c0b, 0x40, 0, 0},
 	{0x3814, 0x31, 0, 0},
 	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
@@ -440,8 +366,8 @@
 	{0x3824, 0x02, 0, 0}, {0x5001, 0xa3, 0, 0},
 };
 
-static const struct reg_value ov5640_setting_15fps_QVGA_320_240[] = {
-	{0x3035, 0x22, 0, 0}, {0x3036, 0x38, 0, 0}, {0x3c07, 0x08, 0, 0},
+static const struct reg_value ov5640_setting_XGA_1024_768[] = {
+	{0x3c07, 0x08, 0, 0},
 	{0x3c09, 0x1c, 0, 0}, {0x3c0a, 0x9c, 0, 0}, {0x3c0b, 0x40, 0, 0},
 	{0x3814, 0x31, 0, 0},
 	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
@@ -459,8 +385,8 @@
 	{0x3824, 0x02, 0, 0}, {0x5001, 0xa3, 0, 0},
 };
 
-static const struct reg_value ov5640_setting_30fps_QCIF_176_144[] = {
-	{0x3035, 0x14, 0, 0}, {0x3036, 0x38, 0, 0}, {0x3c07, 0x08, 0, 0},
+static const struct reg_value ov5640_setting_QVGA_320_240[] = {
+	{0x3c07, 0x08, 0, 0},
 	{0x3c09, 0x1c, 0, 0}, {0x3c0a, 0x9c, 0, 0}, {0x3c0b, 0x40, 0, 0},
 	{0x3814, 0x31, 0, 0},
 	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
@@ -478,8 +404,8 @@
 	{0x3824, 0x02, 0, 0}, {0x5001, 0xa3, 0, 0},
 };
 
-static const struct reg_value ov5640_setting_15fps_QCIF_176_144[] = {
-	{0x3035, 0x22, 0, 0}, {0x3036, 0x38, 0, 0}, {0x3c07, 0x08, 0, 0},
+static const struct reg_value ov5640_setting_QCIF_176_144[] = {
+	{0x3c07, 0x08, 0, 0},
 	{0x3c09, 0x1c, 0, 0}, {0x3c0a, 0x9c, 0, 0}, {0x3c0b, 0x40, 0, 0},
 	{0x3814, 0x31, 0, 0},
 	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
@@ -497,8 +423,8 @@
 	{0x3824, 0x02, 0, 0}, {0x5001, 0xa3, 0, 0},
 };
 
-static const struct reg_value ov5640_setting_30fps_NTSC_720_480[] = {
-	{0x3035, 0x12, 0, 0}, {0x3036, 0x38, 0, 0}, {0x3c07, 0x08, 0, 0},
+static const struct reg_value ov5640_setting_NTSC_720_480[] = {
+	{0x3c07, 0x08, 0, 0},
 	{0x3c09, 0x1c, 0, 0}, {0x3c0a, 0x9c, 0, 0}, {0x3c0b, 0x40, 0, 0},
 	{0x3814, 0x31, 0, 0},
 	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
@@ -516,46 +442,8 @@
 	{0x3824, 0x02, 0, 0}, {0x5001, 0xa3, 0, 0},
 };
 
-static const struct reg_value ov5640_setting_15fps_NTSC_720_480[] = {
-	{0x3035, 0x22, 0, 0}, {0x3036, 0x38, 0, 0}, {0x3c07, 0x08, 0, 0},
-	{0x3c09, 0x1c, 0, 0}, {0x3c0a, 0x9c, 0, 0}, {0x3c0b, 0x40, 0, 0},
-	{0x3814, 0x31, 0, 0},
-	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
-	{0x3802, 0x00, 0, 0}, {0x3803, 0x04, 0, 0}, {0x3804, 0x0a, 0, 0},
-	{0x3805, 0x3f, 0, 0}, {0x3806, 0x07, 0, 0}, {0x3807, 0x9b, 0, 0},
-	{0x3810, 0x00, 0, 0},
-	{0x3811, 0x10, 0, 0}, {0x3812, 0x00, 0, 0}, {0x3813, 0x3c, 0, 0},
-	{0x3618, 0x00, 0, 0}, {0x3612, 0x29, 0, 0}, {0x3708, 0x64, 0, 0},
-	{0x3709, 0x52, 0, 0}, {0x370c, 0x03, 0, 0}, {0x3a02, 0x03, 0, 0},
-	{0x3a03, 0xd8, 0, 0}, {0x3a08, 0x01, 0, 0}, {0x3a09, 0x27, 0, 0},
-	{0x3a0a, 0x00, 0, 0}, {0x3a0b, 0xf6, 0, 0}, {0x3a0e, 0x03, 0, 0},
-	{0x3a0d, 0x04, 0, 0}, {0x3a14, 0x03, 0, 0}, {0x3a15, 0xd8, 0, 0},
-	{0x4001, 0x02, 0, 0}, {0x4004, 0x02, 0, 0}, {0x4713, 0x03, 0, 0},
-	{0x4407, 0x04, 0, 0}, {0x460b, 0x35, 0, 0}, {0x460c, 0x22, 0, 0},
-	{0x3824, 0x02, 0, 0}, {0x5001, 0xa3, 0, 0},
-};
-
-static const struct reg_value ov5640_setting_30fps_PAL_720_576[] = {
-	{0x3035, 0x12, 0, 0}, {0x3036, 0x38, 0, 0}, {0x3c07, 0x08, 0, 0},
-	{0x3c09, 0x1c, 0, 0}, {0x3c0a, 0x9c, 0, 0}, {0x3c0b, 0x40, 0, 0},
-	{0x3814, 0x31, 0, 0},
-	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
-	{0x3802, 0x00, 0, 0}, {0x3803, 0x04, 0, 0}, {0x3804, 0x0a, 0, 0},
-	{0x3805, 0x3f, 0, 0}, {0x3806, 0x07, 0, 0}, {0x3807, 0x9b, 0, 0},
-	{0x3810, 0x00, 0, 0},
-	{0x3811, 0x38, 0, 0}, {0x3812, 0x00, 0, 0}, {0x3813, 0x06, 0, 0},
-	{0x3618, 0x00, 0, 0}, {0x3612, 0x29, 0, 0}, {0x3708, 0x64, 0, 0},
-	{0x3709, 0x52, 0, 0}, {0x370c, 0x03, 0, 0}, {0x3a02, 0x03, 0, 0},
-	{0x3a03, 0xd8, 0, 0}, {0x3a08, 0x01, 0, 0}, {0x3a09, 0x27, 0, 0},
-	{0x3a0a, 0x00, 0, 0}, {0x3a0b, 0xf6, 0, 0}, {0x3a0e, 0x03, 0, 0},
-	{0x3a0d, 0x04, 0, 0}, {0x3a14, 0x03, 0, 0}, {0x3a15, 0xd8, 0, 0},
-	{0x4001, 0x02, 0, 0}, {0x4004, 0x02, 0, 0}, {0x4713, 0x03, 0, 0},
-	{0x4407, 0x04, 0, 0}, {0x460b, 0x35, 0, 0}, {0x460c, 0x22, 0, 0},
-	{0x3824, 0x02, 0, 0}, {0x5001, 0xa3, 0, 0},
-};
-
-static const struct reg_value ov5640_setting_15fps_PAL_720_576[] = {
-	{0x3035, 0x22, 0, 0}, {0x3036, 0x38, 0, 0}, {0x3c07, 0x08, 0, 0},
+static const struct reg_value ov5640_setting_PAL_720_576[] = {
+	{0x3c07, 0x08, 0, 0},
 	{0x3c09, 0x1c, 0, 0}, {0x3c0a, 0x9c, 0, 0}, {0x3c0b, 0x40, 0, 0},
 	{0x3814, 0x31, 0, 0},
 	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
@@ -573,9 +461,8 @@
 	{0x3824, 0x02, 0, 0}, {0x5001, 0xa3, 0, 0},
 };
 
-static const struct reg_value ov5640_setting_30fps_720P_1280_720[] = {
-	{0x3008, 0x42, 0, 0},
-	{0x3035, 0x21, 0, 0}, {0x3036, 0x54, 0, 0}, {0x3c07, 0x07, 0, 0},
+static const struct reg_value ov5640_setting_720P_1280_720[] = {
+	{0x3c07, 0x07, 0, 0},
 	{0x3c09, 0x1c, 0, 0}, {0x3c0a, 0x9c, 0, 0}, {0x3c0b, 0x40, 0, 0},
 	{0x3814, 0x31, 0, 0},
 	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
@@ -588,67 +475,13 @@
 	{0x3a03, 0xe4, 0, 0}, {0x3a08, 0x01, 0, 0}, {0x3a09, 0xbc, 0, 0},
 	{0x3a0a, 0x01, 0, 0}, {0x3a0b, 0x72, 0, 0}, {0x3a0e, 0x01, 0, 0},
 	{0x3a0d, 0x02, 0, 0}, {0x3a14, 0x02, 0, 0}, {0x3a15, 0xe4, 0, 0},
-	{0x4001, 0x02, 0, 0}, {0x4004, 0x02, 0, 0}, {0x4713, 0x02, 0, 0},
-	{0x4407, 0x04, 0, 0}, {0x460b, 0x37, 0, 0}, {0x460c, 0x20, 0, 0},
-	{0x3824, 0x04, 0, 0}, {0x5001, 0x83, 0, 0}, {0x4005, 0x1a, 0, 0},
-	{0x3008, 0x02, 0, 0}, {0x3503, 0,    0, 0},
-};
-
-static const struct reg_value ov5640_setting_15fps_720P_1280_720[] = {
-	{0x3035, 0x41, 0, 0}, {0x3036, 0x54, 0, 0}, {0x3c07, 0x07, 0, 0},
-	{0x3c09, 0x1c, 0, 0}, {0x3c0a, 0x9c, 0, 0}, {0x3c0b, 0x40, 0, 0},
-	{0x3814, 0x31, 0, 0},
-	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
-	{0x3802, 0x00, 0, 0}, {0x3803, 0xfa, 0, 0}, {0x3804, 0x0a, 0, 0},
-	{0x3805, 0x3f, 0, 0}, {0x3806, 0x06, 0, 0}, {0x3807, 0xa9, 0, 0},
-	{0x3810, 0x00, 0, 0},
-	{0x3811, 0x10, 0, 0}, {0x3812, 0x00, 0, 0}, {0x3813, 0x04, 0, 0},
-	{0x3618, 0x00, 0, 0}, {0x3612, 0x29, 0, 0}, {0x3708, 0x64, 0, 0},
-	{0x3709, 0x52, 0, 0}, {0x370c, 0x03, 0, 0}, {0x3a02, 0x02, 0, 0},
-	{0x3a03, 0xe4, 0, 0}, {0x3a08, 0x01, 0, 0}, {0x3a09, 0xbc, 0, 0},
-	{0x3a0a, 0x01, 0, 0}, {0x3a0b, 0x72, 0, 0}, {0x3a0e, 0x01, 0, 0},
-	{0x3a0d, 0x02, 0, 0}, {0x3a14, 0x02, 0, 0}, {0x3a15, 0xe4, 0, 0},
-	{0x4001, 0x02, 0, 0}, {0x4004, 0x02, 0, 0}, {0x4713, 0x02, 0, 0},
+	{0x4001, 0x02, 0, 0}, {0x4004, 0x02, 0, 0}, {0x4713, 0x03, 0, 0},
 	{0x4407, 0x04, 0, 0}, {0x460b, 0x37, 0, 0}, {0x460c, 0x20, 0, 0},
 	{0x3824, 0x04, 0, 0}, {0x5001, 0x83, 0, 0},
 };
 
-static const struct reg_value ov5640_setting_30fps_1080P_1920_1080[] = {
-	{0x3008, 0x42, 0, 0},
-	{0x3035, 0x21, 0, 0}, {0x3036, 0x54, 0, 0}, {0x3c07, 0x08, 0, 0},
-	{0x3c09, 0x1c, 0, 0}, {0x3c0a, 0x9c, 0, 0}, {0x3c0b, 0x40, 0, 0},
-	{0x3814, 0x11, 0, 0},
-	{0x3815, 0x11, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
-	{0x3802, 0x00, 0, 0}, {0x3803, 0x00, 0, 0}, {0x3804, 0x0a, 0, 0},
-	{0x3805, 0x3f, 0, 0}, {0x3806, 0x07, 0, 0}, {0x3807, 0x9f, 0, 0},
-	{0x3810, 0x00, 0, 0},
-	{0x3811, 0x10, 0, 0}, {0x3812, 0x00, 0, 0}, {0x3813, 0x04, 0, 0},
-	{0x3618, 0x04, 0, 0}, {0x3612, 0x29, 0, 0}, {0x3708, 0x21, 0, 0},
-	{0x3709, 0x12, 0, 0}, {0x370c, 0x00, 0, 0}, {0x3a02, 0x03, 0, 0},
-	{0x3a03, 0xd8, 0, 0}, {0x3a08, 0x01, 0, 0}, {0x3a09, 0x27, 0, 0},
-	{0x3a0a, 0x00, 0, 0}, {0x3a0b, 0xf6, 0, 0}, {0x3a0e, 0x03, 0, 0},
-	{0x3a0d, 0x04, 0, 0}, {0x3a14, 0x03, 0, 0}, {0x3a15, 0xd8, 0, 0},
-	{0x4001, 0x02, 0, 0}, {0x4004, 0x06, 0, 0}, {0x4713, 0x03, 0, 0},
-	{0x4407, 0x04, 0, 0}, {0x460b, 0x35, 0, 0}, {0x460c, 0x22, 0, 0},
-	{0x3824, 0x02, 0, 0}, {0x5001, 0x83, 0, 0}, {0x3035, 0x11, 0, 0},
-	{0x3036, 0x54, 0, 0}, {0x3c07, 0x07, 0, 0}, {0x3c08, 0x00, 0, 0},
-	{0x3c09, 0x1c, 0, 0}, {0x3c0a, 0x9c, 0, 0}, {0x3c0b, 0x40, 0, 0},
-	{0x3800, 0x01, 0, 0}, {0x3801, 0x50, 0, 0}, {0x3802, 0x01, 0, 0},
-	{0x3803, 0xb2, 0, 0}, {0x3804, 0x08, 0, 0}, {0x3805, 0xef, 0, 0},
-	{0x3806, 0x05, 0, 0}, {0x3807, 0xf1, 0, 0},
-	{0x3612, 0x2b, 0, 0}, {0x3708, 0x64, 0, 0},
-	{0x3a02, 0x04, 0, 0}, {0x3a03, 0x60, 0, 0}, {0x3a08, 0x01, 0, 0},
-	{0x3a09, 0x50, 0, 0}, {0x3a0a, 0x01, 0, 0}, {0x3a0b, 0x18, 0, 0},
-	{0x3a0e, 0x03, 0, 0}, {0x3a0d, 0x04, 0, 0}, {0x3a14, 0x04, 0, 0},
-	{0x3a15, 0x60, 0, 0}, {0x4713, 0x02, 0, 0}, {0x4407, 0x04, 0, 0},
-	{0x460b, 0x37, 0, 0}, {0x460c, 0x20, 0, 0}, {0x3824, 0x04, 0, 0},
-	{0x4005, 0x1a, 0, 0}, {0x3008, 0x02, 0, 0},
-	{0x3503, 0, 0, 0},
-};
-
-static const struct reg_value ov5640_setting_15fps_1080P_1920_1080[] = {
-	{0x3008, 0x42, 0, 0},
-	{0x3035, 0x21, 0, 0}, {0x3036, 0x54, 0, 0}, {0x3c07, 0x08, 0, 0},
+static const struct reg_value ov5640_setting_1080P_1920_1080[] = {
+	{0x3c07, 0x08, 0, 0},
 	{0x3c09, 0x1c, 0, 0}, {0x3c0a, 0x9c, 0, 0}, {0x3c0b, 0x40, 0, 0},
 	{0x3814, 0x11, 0, 0},
 	{0x3815, 0x11, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
@@ -663,8 +496,8 @@
 	{0x3a0d, 0x04, 0, 0}, {0x3a14, 0x03, 0, 0}, {0x3a15, 0xd8, 0, 0},
 	{0x4001, 0x02, 0, 0}, {0x4004, 0x06, 0, 0}, {0x4713, 0x03, 0, 0},
 	{0x4407, 0x04, 0, 0}, {0x460b, 0x35, 0, 0}, {0x460c, 0x22, 0, 0},
-	{0x3824, 0x02, 0, 0}, {0x5001, 0x83, 0, 0}, {0x3035, 0x21, 0, 0},
-	{0x3036, 0x54, 0, 1}, {0x3c07, 0x07, 0, 0}, {0x3c08, 0x00, 0, 0},
+	{0x3824, 0x02, 0, 0}, {0x5001, 0x83, 0, 0},
+	{0x3c07, 0x07, 0, 0}, {0x3c08, 0x00, 0, 0},
 	{0x3c09, 0x1c, 0, 0}, {0x3c0a, 0x9c, 0, 0}, {0x3c0b, 0x40, 0, 0},
 	{0x3800, 0x01, 0, 0}, {0x3801, 0x50, 0, 0}, {0x3802, 0x01, 0, 0},
 	{0x3803, 0xb2, 0, 0}, {0x3804, 0x08, 0, 0}, {0x3805, 0xef, 0, 0},
@@ -675,11 +508,11 @@
 	{0x3a0e, 0x03, 0, 0}, {0x3a0d, 0x04, 0, 0}, {0x3a14, 0x04, 0, 0},
 	{0x3a15, 0x60, 0, 0}, {0x4713, 0x02, 0, 0}, {0x4407, 0x04, 0, 0},
 	{0x460b, 0x37, 0, 0}, {0x460c, 0x20, 0, 0}, {0x3824, 0x04, 0, 0},
-	{0x4005, 0x1a, 0, 0}, {0x3008, 0x02, 0, 0}, {0x3503, 0, 0, 0},
+	{0x4005, 0x1a, 0, 0},
 };
 
-static const struct reg_value ov5640_setting_15fps_QSXGA_2592_1944[] = {
-	{0x3035, 0x21, 0, 0}, {0x3036, 0x54, 0, 0}, {0x3c07, 0x08, 0, 0},
+static const struct reg_value ov5640_setting_QSXGA_2592_1944[] = {
+	{0x3c07, 0x08, 0, 0},
 	{0x3c09, 0x1c, 0, 0}, {0x3c0a, 0x9c, 0, 0}, {0x3c0b, 0x40, 0, 0},
 	{0x3814, 0x11, 0, 0},
 	{0x3815, 0x11, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
@@ -705,79 +538,43 @@
 };
 
 static const struct ov5640_mode_info
-ov5640_mode_data[OV5640_NUM_FRAMERATES][OV5640_NUM_MODES] = {
-	{
-		{OV5640_MODE_QCIF_176_144, SUBSAMPLING,
-		 176, 1896, 144, 984,
-		 ov5640_setting_15fps_QCIF_176_144,
-		 ARRAY_SIZE(ov5640_setting_15fps_QCIF_176_144)},
-		{OV5640_MODE_QVGA_320_240, SUBSAMPLING,
-		 320, 1896, 240, 984,
-		 ov5640_setting_15fps_QVGA_320_240,
-		 ARRAY_SIZE(ov5640_setting_15fps_QVGA_320_240)},
-		{OV5640_MODE_VGA_640_480, SUBSAMPLING,
-		 640, 1896, 480, 1080,
-		 ov5640_setting_15fps_VGA_640_480,
-		 ARRAY_SIZE(ov5640_setting_15fps_VGA_640_480)},
-		{OV5640_MODE_NTSC_720_480, SUBSAMPLING,
-		 720, 1896, 480, 984,
-		 ov5640_setting_15fps_NTSC_720_480,
-		 ARRAY_SIZE(ov5640_setting_15fps_NTSC_720_480)},
-		{OV5640_MODE_PAL_720_576, SUBSAMPLING,
-		 720, 1896, 576, 984,
-		 ov5640_setting_15fps_PAL_720_576,
-		 ARRAY_SIZE(ov5640_setting_15fps_PAL_720_576)},
-		{OV5640_MODE_XGA_1024_768, SUBSAMPLING,
-		 1024, 1896, 768, 1080,
-		 ov5640_setting_15fps_XGA_1024_768,
-		 ARRAY_SIZE(ov5640_setting_15fps_XGA_1024_768)},
-		{OV5640_MODE_720P_1280_720, SUBSAMPLING,
-		 1280, 1892, 720, 740,
-		 ov5640_setting_15fps_720P_1280_720,
-		 ARRAY_SIZE(ov5640_setting_15fps_720P_1280_720)},
-		{OV5640_MODE_1080P_1920_1080, SCALING,
-		 1920, 2500, 1080, 1120,
-		 ov5640_setting_15fps_1080P_1920_1080,
-		 ARRAY_SIZE(ov5640_setting_15fps_1080P_1920_1080)},
-		{OV5640_MODE_QSXGA_2592_1944, SCALING,
-		 2592, 2844, 1944, 1968,
-		 ov5640_setting_15fps_QSXGA_2592_1944,
-		 ARRAY_SIZE(ov5640_setting_15fps_QSXGA_2592_1944)},
-	}, {
-		{OV5640_MODE_QCIF_176_144, SUBSAMPLING,
-		 176, 1896, 144, 984,
-		 ov5640_setting_30fps_QCIF_176_144,
-		 ARRAY_SIZE(ov5640_setting_30fps_QCIF_176_144)},
-		{OV5640_MODE_QVGA_320_240, SUBSAMPLING,
-		 320, 1896, 240, 984,
-		 ov5640_setting_30fps_QVGA_320_240,
-		 ARRAY_SIZE(ov5640_setting_30fps_QVGA_320_240)},
-		{OV5640_MODE_VGA_640_480, SUBSAMPLING,
-		 640, 1896, 480, 1080,
-		 ov5640_setting_30fps_VGA_640_480,
-		 ARRAY_SIZE(ov5640_setting_30fps_VGA_640_480)},
-		{OV5640_MODE_NTSC_720_480, SUBSAMPLING,
-		 720, 1896, 480, 984,
-		 ov5640_setting_30fps_NTSC_720_480,
-		 ARRAY_SIZE(ov5640_setting_30fps_NTSC_720_480)},
-		{OV5640_MODE_PAL_720_576, SUBSAMPLING,
-		 720, 1896, 576, 984,
-		 ov5640_setting_30fps_PAL_720_576,
-		 ARRAY_SIZE(ov5640_setting_30fps_PAL_720_576)},
-		{OV5640_MODE_XGA_1024_768, SUBSAMPLING,
-		 1024, 1896, 768, 1080,
-		 ov5640_setting_30fps_XGA_1024_768,
-		 ARRAY_SIZE(ov5640_setting_30fps_XGA_1024_768)},
-		{OV5640_MODE_720P_1280_720, SUBSAMPLING,
-		 1280, 1892, 720, 740,
-		 ov5640_setting_30fps_720P_1280_720,
-		 ARRAY_SIZE(ov5640_setting_30fps_720P_1280_720)},
-		{OV5640_MODE_1080P_1920_1080, SCALING,
-		 1920, 2500, 1080, 1120,
-		 ov5640_setting_30fps_1080P_1920_1080,
-		 ARRAY_SIZE(ov5640_setting_30fps_1080P_1920_1080)},
-		{OV5640_MODE_QSXGA_2592_1944, -1, 0, 0, 0, 0, NULL, 0},
-	},
+ov5640_mode_data[OV5640_NUM_MODES] = {
+	{OV5640_MODE_QCIF_176_144, SUBSAMPLING,
+	 176, 1896, 144, 984,
+	 ov5640_setting_QCIF_176_144,
+	 ARRAY_SIZE(ov5640_setting_QCIF_176_144)},
+	{OV5640_MODE_QVGA_320_240, SUBSAMPLING,
+	 320, 1896, 240, 984,
+	 ov5640_setting_QVGA_320_240,
+	 ARRAY_SIZE(ov5640_setting_QVGA_320_240)},
+	{OV5640_MODE_VGA_640_480, SUBSAMPLING,
+	 640, 1896, 480, 1080,
+	 ov5640_setting_VGA_640_480,
+	 ARRAY_SIZE(ov5640_setting_VGA_640_480)},
+	{OV5640_MODE_NTSC_720_480, SUBSAMPLING,
+	 720, 1896, 480, 984,
+	 ov5640_setting_NTSC_720_480,
+	 ARRAY_SIZE(ov5640_setting_NTSC_720_480)},
+	{OV5640_MODE_PAL_720_576, SUBSAMPLING,
+	 720, 1896, 576, 984,
+	 ov5640_setting_PAL_720_576,
+	 ARRAY_SIZE(ov5640_setting_PAL_720_576)},
+	{OV5640_MODE_XGA_1024_768, SUBSAMPLING,
+	 1024, 1896, 768, 1080,
+	 ov5640_setting_XGA_1024_768,
+	 ARRAY_SIZE(ov5640_setting_XGA_1024_768)},
+	{OV5640_MODE_720P_1280_720, SUBSAMPLING,
+	 1280, 1892, 720, 740,
+	 ov5640_setting_720P_1280_720,
+	 ARRAY_SIZE(ov5640_setting_720P_1280_720)},
+	{OV5640_MODE_1080P_1920_1080, SCALING,
+	 1920, 2500, 1080, 1120,
+	 ov5640_setting_1080P_1920_1080,
+	 ARRAY_SIZE(ov5640_setting_1080P_1920_1080)},
+	{OV5640_MODE_QSXGA_2592_1944, SCALING,
+	 2592, 2844, 1944, 1968,
+	 ov5640_setting_QSXGA_2592_1944,
+	 ARRAY_SIZE(ov5640_setting_QSXGA_2592_1944)},
 };
 
 static int ov5640_init_slave_id(struct ov5640_dev *sensor)
@@ -909,6 +706,333 @@
 	return ov5640_write_reg(sensor, reg, val);
 }
 
+/*
+ * After trying the various combinations, reading various
+ * documentations spreaded around the net, and from the various
+ * feedback, the clock tree is probably as follows:
+ *
+ *   +--------------+
+ *   |  Ext. Clock  |
+ *   +-+------------+
+ *     |  +----------+
+ *     +->|   PLL1   | - reg 0x3036, for the multiplier
+ *        +-+--------+ - reg 0x3037, bits 0-3 for the pre-divider
+ *          |  +--------------+
+ *          +->| System Clock |  - reg 0x3035, bits 4-7
+ *             +-+------------+
+ *               |  +--------------+
+ *               +->| MIPI Divider | - reg 0x3035, bits 0-3
+ *               |  +-+------------+
+ *               |    +----------------> MIPI SCLK
+ *               |    +  +-----+
+ *               |    +->| / 2 |-------> MIPI BIT CLK
+ *               |       +-----+
+ *               |  +--------------+
+ *               +->| PLL Root Div | - reg 0x3037, bit 4
+ *                  +-+------------+
+ *                    |  +---------+
+ *                    +->| Bit Div | - reg 0x3035, bits 0-3
+ *                       +-+-------+
+ *                         |  +-------------+
+ *                         +->| SCLK Div    | - reg 0x3108, bits 0-1
+ *                         |  +-+-----------+
+ *                         |    +---------------> SCLK
+ *                         |  +-------------+
+ *                         +->| SCLK 2X Div | - reg 0x3108, bits 2-3
+ *                         |  +-+-----------+
+ *                         |    +---------------> SCLK 2X
+ *                         |  +-------------+
+ *                         +->| PCLK Div    | - reg 0x3108, bits 4-5
+ *                            ++------------+
+ *                             +  +-----------+
+ *                             +->|   P_DIV   | - reg 0x3035, bits 0-3
+ *                                +-----+-----+
+ *                                       +------------> PCLK
+ *
+ * This is deviating from the datasheet at least for the register
+ * 0x3108, since it's said here that the PCLK would be clocked from
+ * the PLL.
+ *
+ * There seems to be also (unverified) constraints:
+ *  - the PLL pre-divider output rate should be in the 4-27MHz range
+ *  - the PLL multiplier output rate should be in the 500-1000MHz range
+ *  - PCLK >= SCLK * 2 in YUV, >= SCLK in Raw or JPEG
+ *
+ * In the two latter cases, these constraints are met since our
+ * factors are hardcoded. If we were to change that, we would need to
+ * take this into account. The only varying parts are the PLL
+ * multiplier and the system clock divider, which are shared between
+ * all these clocks so won't cause any issue.
+ */
+
+/*
+ * This is supposed to be ranging from 1 to 8, but the value is always
+ * set to 3 in the vendor kernels.
+ */
+#define OV5640_PLL_PREDIV	3
+
+#define OV5640_PLL_MULT_MIN	4
+#define OV5640_PLL_MULT_MAX	252
+
+/*
+ * This is supposed to be ranging from 1 to 16, but the value is
+ * always set to either 1 or 2 in the vendor kernels.
+ */
+#define OV5640_SYSDIV_MIN	1
+#define OV5640_SYSDIV_MAX	16
+
+/*
+ * Hardcode these values for scaler and non-scaler modes.
+ * FIXME: to be re-calcualted for 1 data lanes setups
+ */
+#define OV5640_MIPI_DIV_PCLK	2
+#define OV5640_MIPI_DIV_SCLK	1
+
+/*
+ * This is supposed to be ranging from 1 to 2, but the value is always
+ * set to 2 in the vendor kernels.
+ */
+#define OV5640_PLL_ROOT_DIV			2
+#define OV5640_PLL_CTRL3_PLL_ROOT_DIV_2		BIT(4)
+
+/*
+ * We only supports 8-bit formats at the moment
+ */
+#define OV5640_BIT_DIV				2
+#define OV5640_PLL_CTRL0_MIPI_MODE_8BIT		0x08
+
+/*
+ * This is supposed to be ranging from 1 to 8, but the value is always
+ * set to 2 in the vendor kernels.
+ */
+#define OV5640_SCLK_ROOT_DIV	2
+
+/*
+ * This is hardcoded so that the consistency is maintained between SCLK and
+ * SCLK 2x.
+ */
+#define OV5640_SCLK2X_ROOT_DIV (OV5640_SCLK_ROOT_DIV / 2)
+
+/*
+ * This is supposed to be ranging from 1 to 8, but the value is always
+ * set to 1 in the vendor kernels.
+ */
+#define OV5640_PCLK_ROOT_DIV			1
+#define OV5640_PLL_SYS_ROOT_DIVIDER_BYPASS	0x00
+
+static unsigned long ov5640_compute_sys_clk(struct ov5640_dev *sensor,
+					    u8 pll_prediv, u8 pll_mult,
+					    u8 sysdiv)
+{
+	unsigned long sysclk = sensor->xclk_freq / pll_prediv * pll_mult;
+
+	/* PLL1 output cannot exceed 1GHz. */
+	if (sysclk / 1000000 > 1000)
+		return 0;
+
+	return sysclk / sysdiv;
+}
+
+static unsigned long ov5640_calc_sys_clk(struct ov5640_dev *sensor,
+					 unsigned long rate,
+					 u8 *pll_prediv, u8 *pll_mult,
+					 u8 *sysdiv)
+{
+	unsigned long best = ~0;
+	u8 best_sysdiv = 1, best_mult = 1;
+	u8 _sysdiv, _pll_mult;
+
+	for (_sysdiv = OV5640_SYSDIV_MIN;
+	     _sysdiv <= OV5640_SYSDIV_MAX;
+	     _sysdiv++) {
+		for (_pll_mult = OV5640_PLL_MULT_MIN;
+		     _pll_mult <= OV5640_PLL_MULT_MAX;
+		     _pll_mult++) {
+			unsigned long _rate;
+
+			/*
+			 * The PLL multiplier cannot be odd if above
+			 * 127.
+			 */
+			if (_pll_mult > 127 && (_pll_mult % 2))
+				continue;
+
+			_rate = ov5640_compute_sys_clk(sensor,
+						       OV5640_PLL_PREDIV,
+						       _pll_mult, _sysdiv);
+
+			/*
+			 * We have reached the maximum allowed PLL1 output,
+			 * increase sysdiv.
+			 */
+			if (!rate)
+				break;
+
+			/*
+			 * Prefer rates above the expected clock rate than
+			 * below, even if that means being less precise.
+			 */
+			if (_rate < rate)
+				continue;
+
+			if (abs(rate - _rate) < abs(rate - best)) {
+				best = _rate;
+				best_sysdiv = _sysdiv;
+				best_mult = _pll_mult;
+			}
+
+			if (_rate == rate)
+				goto out;
+		}
+	}
+
+out:
+	*sysdiv = best_sysdiv;
+	*pll_prediv = OV5640_PLL_PREDIV;
+	*pll_mult = best_mult;
+
+	return best;
+}
+
+/*
+ * ov5640_set_mipi_pclk() - Calculate the clock tree configuration values
+ *			    for the MIPI CSI-2 output.
+ *
+ * @rate: The requested bandwidth per lane in bytes per second.
+ *	  'Bandwidth Per Lane' is calculated as:
+ *	  bpl = HTOT * VTOT * FPS * bpp / num_lanes;
+ *
+ * This function use the requested bandwidth to calculate:
+ * - sample_rate = bpl / (bpp / num_lanes);
+ *	         = bpl / (PLL_RDIV * BIT_DIV * PCLK_DIV * MIPI_DIV / num_lanes);
+ *
+ * - mipi_sclk   = bpl / MIPI_DIV / 2; ( / 2 is for CSI-2 DDR)
+ *
+ * with these fixed parameters:
+ *	PLL_RDIV	= 2;
+ *	BIT_DIVIDER	= 2; (MIPI_BIT_MODE == 8 ? 2 : 2,5);
+ *	PCLK_DIV	= 1;
+ *
+ * The MIPI clock generation differs for modes that use the scaler and modes
+ * that do not. In case the scaler is in use, the MIPI_SCLK generates the MIPI
+ * BIT CLk, and thus:
+ *
+ * - mipi_sclk = bpl / MIPI_DIV / 2;
+ *   MIPI_DIV = 1;
+ *
+ * For modes that do not go through the scaler, the MIPI BIT CLOCK is generated
+ * from the pixel clock, and thus:
+ *
+ * - sample_rate = bpl / (bpp / num_lanes);
+ *	         = bpl / (2 * 2 * 1 * MIPI_DIV / num_lanes);
+ *		 = bpl / (4 * MIPI_DIV / num_lanes);
+ * - MIPI_DIV	 = bpp / (4 * num_lanes);
+ *
+ * FIXME: this have been tested with 16bpp and 2 lanes setup only.
+ * MIPI_DIV is fixed to value 2, but it -might- be changed according to the
+ * above formula for setups with 1 lane or image formats with different bpp.
+ *
+ * FIXME: this deviates from the sensor manual documentation which is quite
+ * thin on the MIPI clock tree generation part.
+ */
+static int ov5640_set_mipi_pclk(struct ov5640_dev *sensor,
+				unsigned long rate)
+{
+	const struct ov5640_mode_info *mode = sensor->current_mode;
+	u8 prediv, mult, sysdiv;
+	u8 mipi_div;
+	int ret;
+
+	/*
+	 * 1280x720 is reported to use 'SUBSAMPLING' only,
+	 * but according to the sensor manual it goes through the
+	 * scaler before subsampling.
+	 */
+	if (mode->dn_mode == SCALING ||
+	   (mode->id == OV5640_MODE_720P_1280_720))
+		mipi_div = OV5640_MIPI_DIV_SCLK;
+	else
+		mipi_div = OV5640_MIPI_DIV_PCLK;
+
+	ov5640_calc_sys_clk(sensor, rate, &prediv, &mult, &sysdiv);
+
+	ret = ov5640_mod_reg(sensor, OV5640_REG_SC_PLL_CTRL0,
+			     0x0f, OV5640_PLL_CTRL0_MIPI_MODE_8BIT);
+
+	ret = ov5640_mod_reg(sensor, OV5640_REG_SC_PLL_CTRL1,
+			     0xff, sysdiv << 4 | mipi_div);
+	if (ret)
+		return ret;
+
+	ret = ov5640_mod_reg(sensor, OV5640_REG_SC_PLL_CTRL2, 0xff, mult);
+	if (ret)
+		return ret;
+
+	ret = ov5640_mod_reg(sensor, OV5640_REG_SC_PLL_CTRL3,
+			     0x1f, OV5640_PLL_CTRL3_PLL_ROOT_DIV_2 | prediv);
+	if (ret)
+		return ret;
+
+	return ov5640_mod_reg(sensor, OV5640_REG_SYS_ROOT_DIVIDER,
+			      0x30, OV5640_PLL_SYS_ROOT_DIVIDER_BYPASS);
+}
+
+static unsigned long ov5640_calc_pclk(struct ov5640_dev *sensor,
+				      unsigned long rate,
+				      u8 *pll_prediv, u8 *pll_mult, u8 *sysdiv,
+				      u8 *pll_rdiv, u8 *bit_div, u8 *pclk_div)
+{
+	unsigned long _rate = rate * OV5640_PLL_ROOT_DIV * OV5640_BIT_DIV *
+				OV5640_PCLK_ROOT_DIV;
+
+	_rate = ov5640_calc_sys_clk(sensor, _rate, pll_prediv, pll_mult,
+				    sysdiv);
+	*pll_rdiv = OV5640_PLL_ROOT_DIV;
+	*bit_div = OV5640_BIT_DIV;
+	*pclk_div = OV5640_PCLK_ROOT_DIV;
+
+	return _rate / *pll_rdiv / *bit_div / *pclk_div;
+}
+
+static int ov5640_set_dvp_pclk(struct ov5640_dev *sensor, unsigned long rate)
+{
+	u8 prediv, mult, sysdiv, pll_rdiv, bit_div, pclk_div;
+	int ret;
+
+	ov5640_calc_pclk(sensor, rate, &prediv, &mult, &sysdiv, &pll_rdiv,
+			 &bit_div, &pclk_div);
+
+	if (bit_div == 2)
+		bit_div = 8;
+
+	ret = ov5640_mod_reg(sensor, OV5640_REG_SC_PLL_CTRL0,
+			     0x0f, bit_div);
+	if (ret)
+		return ret;
+
+	/*
+	 * We need to set sysdiv according to the clock, and to clear
+	 * the MIPI divider.
+	 */
+	ret = ov5640_mod_reg(sensor, OV5640_REG_SC_PLL_CTRL1,
+			     0xff, sysdiv << 4);
+	if (ret)
+		return ret;
+
+	ret = ov5640_mod_reg(sensor, OV5640_REG_SC_PLL_CTRL2,
+			     0xff, mult);
+	if (ret)
+		return ret;
+
+	ret = ov5640_mod_reg(sensor, OV5640_REG_SC_PLL_CTRL3,
+			     0x1f, prediv | ((pll_rdiv - 1) << 4));
+	if (ret)
+		return ret;
+
+	return ov5640_mod_reg(sensor, OV5640_REG_SYS_ROOT_DIVIDER, 0x30,
+			      (ilog2(pclk_div) << 4));
+}
+
 /* download ov5640 settings to sensor through i2c */
 static int ov5640_set_timings(struct ov5640_dev *sensor,
 			      const struct ov5640_mode_info *mode)
@@ -1062,16 +1186,6 @@
 
 	if (on) {
 		/*
-		 * reset MIPI PCLK/SERCLK divider
-		 *
-		 * SC PLL CONTRL1 0
-		 * - [3..0]:	MIPI PCLK/SERCLK divider
-		 */
-		ret = ov5640_mod_reg(sensor, OV5640_REG_SC_PLL_CTRL1, 0x0f, 0);
-		if (ret)
-			return ret;
-
-		/*
 		 * configure parallel port control lines polarity
 		 *
 		 * POLARITY CTRL0
@@ -1444,8 +1558,8 @@
 {
 	const struct ov5640_mode_info *mode;
 
-	mode = v4l2_find_nearest_size(ov5640_mode_data[fr],
-				      ARRAY_SIZE(ov5640_mode_data[fr]),
+	mode = v4l2_find_nearest_size(ov5640_mode_data,
+				      ARRAY_SIZE(ov5640_mode_data),
 				      hact, vact,
 				      width, height);
 
@@ -1453,6 +1567,16 @@
 	    (!nearest && (mode->hact != width || mode->vact != height)))
 		return NULL;
 
+	/* Only 640x480 can operate at 60fps (for now) */
+	if (fr == OV5640_60_FPS &&
+	    !(mode->hact == 640 && mode->vact == 480))
+		return NULL;
+
+	/* 2592x1944 only works at 15fps */
+	if (fr != OV5640_15_FPS &&
+	    (mode->hact == 2592 && mode->vact == 1944))
+		return NULL;
+
 	return mode;
 }
 
@@ -1637,6 +1761,7 @@
 	enum ov5640_downsize_mode dn_mode, orig_dn_mode;
 	bool auto_gain = sensor->ctrls.auto_gain->val == 1;
 	bool auto_exp =  sensor->ctrls.auto_exp->val == V4L2_EXPOSURE_AUTO;
+	unsigned long rate;
 	int ret;
 
 	dn_mode = mode->dn_mode;
@@ -1655,6 +1780,23 @@
 			goto restore_auto_gain;
 	}
 
+	/*
+	 * All the formats we support have 16 bits per pixel, seems to require
+	 * the same rate than YUV, so we can just use 16 bpp all the time.
+	 */
+	rate = mode->vtot * mode->htot * 16;
+	rate *= ov5640_framerates[sensor->current_fr];
+	if (sensor->ep.bus_type == V4L2_MBUS_CSI2) {
+		rate = rate / sensor->ep.bus.mipi_csi2.num_data_lanes;
+		ret = ov5640_set_mipi_pclk(sensor, rate);
+	} else {
+		rate = rate / sensor->ep.bus.parallel.bus_width;
+		ret = ov5640_set_dvp_pclk(sensor, rate);
+	}
+
+	if (ret < 0)
+		return 0;
+
 	if ((dn_mode == SUBSAMPLING && orig_dn_mode == SCALING) ||
 	    (dn_mode == SCALING && orig_dn_mode == SUBSAMPLING)) {
 		/*
@@ -1724,8 +1866,8 @@
 	sensor->last_mode = &ov5640_mode_init_data;
 
 	ret = ov5640_mod_reg(sensor, OV5640_REG_SYS_ROOT_DIVIDER, 0x3f,
-			     (ilog2(OV5640_SCLK2X_ROOT_DIVIDER_DEFAULT) << 2) |
-			     ilog2(OV5640_SCLK_ROOT_DIVIDER_DEFAULT));
+			     (ilog2(OV5640_SCLK2X_ROOT_DIV) << 2) |
+			     ilog2(OV5640_SCLK_ROOT_DIV));
 	if (ret)
 		return ret;
 
@@ -1925,34 +2067,39 @@
 				     u32 width, u32 height)
 {
 	const struct ov5640_mode_info *mode;
-	u32 minfps, maxfps, fps;
-	int ret;
+	enum ov5640_frame_rate rate = OV5640_30_FPS;
+	int minfps, maxfps, best_fps, fps;
+	int i;
 
 	minfps = ov5640_framerates[OV5640_15_FPS];
-	maxfps = ov5640_framerates[OV5640_30_FPS];
+	maxfps = ov5640_framerates[OV5640_60_FPS];
 
 	if (fi->numerator == 0) {
 		fi->denominator = maxfps;
 		fi->numerator = 1;
-		return OV5640_30_FPS;
+		rate = OV5640_60_FPS;
+		goto find_mode;
 	}
 
-	fps = DIV_ROUND_CLOSEST(fi->denominator, fi->numerator);
+	fps = clamp_val(DIV_ROUND_CLOSEST(fi->denominator, fi->numerator),
+			minfps, maxfps);
 
-	fi->numerator = 1;
-	if (fps > maxfps)
-		fi->denominator = maxfps;
-	else if (fps < minfps)
-		fi->denominator = minfps;
-	else if (2 * fps >= 2 * minfps + (maxfps - minfps))
-		fi->denominator = maxfps;
-	else
-		fi->denominator = minfps;
+	best_fps = minfps;
+	for (i = 0; i < ARRAY_SIZE(ov5640_framerates); i++) {
+		int curr_fps = ov5640_framerates[i];
+
+		if (abs(curr_fps - fps) < abs(best_fps - fps)) {
+			best_fps = curr_fps;
+			rate = i;
+		}
+	}
 
-	ret = (fi->denominator == minfps) ? OV5640_15_FPS : OV5640_30_FPS;
+	fi->numerator = 1;
+	fi->denominator = best_fps;
 
-	mode = ov5640_find_mode(sensor, ret, width, height, false);
-	return mode ? ret : -EINVAL;
+find_mode:
+	mode = ov5640_find_mode(sensor, rate, width, height, false);
+	return mode ? rate : -EINVAL;
 }
 
 static int ov5640_get_fmt(struct v4l2_subdev *sd,
@@ -2021,6 +2168,7 @@
 	const struct ov5640_mode_info *new_mode;
 	struct v4l2_mbus_framefmt *mbus_fmt = &format->format;
 	struct v4l2_mbus_framefmt *fmt;
+	u64 rate;
 	int ret;
 
 	if (format->pad != 0)
@@ -2052,6 +2200,12 @@
 	if (mbus_fmt->code != sensor->fmt.code)
 		sensor->pending_fmt_change = true;
 
+	rate = sensor->current_mode->vtot * sensor->current_mode->htot;
+	rate *= ov5640_framerates[sensor->current_fr];
+	sensor->pixel_rate = rate;
+
+	__v4l2_ctrl_s_ctrl_int64(sensor->ctrls.pixel_rate,
+				 sensor->pixel_rate);
 out:
 	mutex_unlock(&sensor->lock);
 	return ret;
@@ -2429,6 +2583,12 @@
 	/* we can use our own mutex for the ctrl lock */
 	hdl->lock = &sensor->lock;
 
+	/* Clock related controls */
+	ctrls->pixel_rate =
+		v4l2_ctrl_new_std(hdl, ops,
+				  V4L2_CID_PIXEL_RATE, 0, INT_MAX, 1,
+				  55969920);
+
 	/* Auto/manual white balance */
 	ctrls->auto_wb = v4l2_ctrl_new_std(hdl, ops,
 					   V4L2_CID_AUTO_WHITE_BALANCE,
@@ -2476,6 +2636,7 @@
 		goto free_ctrls;
 	}
 
+	ctrls->pixel_rate->flags |= V4L2_CTRL_FLAG_READ_ONLY;
 	ctrls->gain->flags |= V4L2_CTRL_FLAG_VOLATILE;
 	ctrls->exposure->flags |= V4L2_CTRL_FLAG_VOLATILE;
 
@@ -2501,10 +2662,10 @@
 		return -EINVAL;
 
 	fse->min_width =
-		ov5640_mode_data[0][fse->index].hact;
+		ov5640_mode_data[fse->index].hact;
 	fse->max_width = fse->min_width;
 	fse->min_height =
-		ov5640_mode_data[0][fse->index].vact;
+		ov5640_mode_data[fse->index].vact;
 	fse->max_height = fse->min_height;
 
 	return 0;
@@ -2554,6 +2715,7 @@
 	struct ov5640_dev *sensor = to_ov5640_dev(sd);
 	const struct ov5640_mode_info *mode;
 	int frame_rate, ret = 0;
+	u64 rate;
 
 	if (fi->pad != 0)
 		return -EINVAL;
@@ -2569,8 +2731,11 @@
 
 	frame_rate = ov5640_try_frame_interval(sensor, &fi->interval,
 					       mode->hact, mode->vact);
-	if (frame_rate < 0)
-		frame_rate = OV5640_15_FPS;
+	if (frame_rate < 0) {
+		/* Always return a valid frame interval value */
+		fi->interval = sensor->frame_interval;
+		goto out;
+	}
 
 	mode = ov5640_find_mode(sensor, frame_rate, mode->hact,
 				mode->vact, true);
@@ -2585,6 +2750,12 @@
 		sensor->frame_interval = fi->interval;
 		sensor->current_mode = mode;
 		sensor->pending_mode_change = true;
+
+		rate = sensor->current_mode->vtot * sensor->current_mode->htot;
+		rate *= ov5640_framerates[sensor->current_fr];
+		sensor->pixel_rate = rate;
+		__v4l2_ctrl_s_ctrl_int64(sensor->ctrls.pixel_rate,
+					 sensor->pixel_rate);
 	}
 out:
 	mutex_unlock(&sensor->lock);
@@ -2640,6 +2811,9 @@
 
 static const struct v4l2_subdev_core_ops ov5640_core_ops = {
 	.s_power = ov5640_s_power,
+	.log_status = v4l2_ctrl_subdev_log_status,
+	.subscribe_event = v4l2_ctrl_subdev_subscribe_event,
+	.unsubscribe_event = v4l2_event_subdev_unsubscribe,
 };
 
 static const struct v4l2_subdev_video_ops ov5640_video_ops = {
@@ -2735,7 +2909,7 @@
 	sensor->frame_interval.denominator = ov5640_framerates[OV5640_30_FPS];
 	sensor->current_fr = OV5640_30_FPS;
 	sensor->current_mode =
-		&ov5640_mode_data[OV5640_30_FPS][OV5640_MODE_VGA_640_480];
+		&ov5640_mode_data[OV5640_MODE_VGA_640_480];
 	sensor->last_mode = sensor->current_mode;
 
 	sensor->ae_target = 52;
@@ -2799,7 +2973,8 @@
 
 	v4l2_i2c_subdev_init(&sensor->sd, client, &ov5640_subdev_ops);
 
-	sensor->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
+	sensor->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE |
+			    V4L2_SUBDEV_FL_HAS_EVENTS;
 	sensor->pad.flags = MEDIA_PAD_FL_SOURCE;
 	sensor->sd.entity.function = MEDIA_ENT_F_CAM_SENSOR;
 	ret = media_entity_pads_init(&sensor->sd.entity, 1, &sensor->pad);
diff -Naur --no-dereference a/drivers/media/platform/am437x/am437x-vpfe.c b/drivers/media/platform/am437x/am437x-vpfe.c
--- a/drivers/media/platform/am437x/am437x-vpfe.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/media/platform/am437x/am437x-vpfe.c	2020-06-19 21:00:31.416227715 -0400
@@ -69,137 +69,64 @@
 	{V4L2_STD_625_50, 720, 576, {54, 59}, 1},
 };
 
-struct bus_format {
-	unsigned int width;
-	unsigned int bpp;
-};
-
-/*
- * struct vpfe_fmt - VPFE media bus format information
- * @name: V4L2 format description
- * @code: V4L2 media bus format code
- * @shifted: V4L2 media bus format code for the same pixel layout but
- *	shifted to be 8 bits per pixel. =0 if format is not shiftable.
- * @pixelformat: V4L2 pixel format FCC identifier
- * @width: Bits per pixel (when transferred over a bus)
- * @bpp: Bytes per pixel (when stored in memory)
- * @supported: Indicates format supported by subdev
- */
-struct vpfe_fmt {
-	const char *name;
-	u32 fourcc;
-	u32 code;
-	struct bus_format l;
-	struct bus_format s;
-	bool supported;
-	u32 index;
-};
-
 static struct vpfe_fmt formats[] = {
 	{
-		.name		= "YUV 4:2:2 packed, YCbYCr",
 		.fourcc		= V4L2_PIX_FMT_YUYV,
 		.code		= MEDIA_BUS_FMT_YUYV8_2X8,
-		.l.width	= 10,
-		.l.bpp		= 4,
-		.s.width	= 8,
-		.s.bpp		= 2,
-		.supported	= false,
+		.bitsperpixel	= 16,
 	}, {
-		.name		= "YUV 4:2:2 packed, CbYCrY",
 		.fourcc		= V4L2_PIX_FMT_UYVY,
 		.code		= MEDIA_BUS_FMT_UYVY8_2X8,
-		.l.width	= 10,
-		.l.bpp		= 4,
-		.s.width	= 8,
-		.s.bpp		= 2,
-		.supported	= false,
+		.bitsperpixel	= 16,
 	}, {
-		.name		= "YUV 4:2:2 packed, YCrYCb",
 		.fourcc		= V4L2_PIX_FMT_YVYU,
 		.code		= MEDIA_BUS_FMT_YVYU8_2X8,
-		.l.width	= 10,
-		.l.bpp		= 4,
-		.s.width	= 8,
-		.s.bpp		= 2,
-		.supported	= false,
+		.bitsperpixel	= 16,
 	}, {
-		.name		= "YUV 4:2:2 packed, CrYCbY",
 		.fourcc		= V4L2_PIX_FMT_VYUY,
 		.code		= MEDIA_BUS_FMT_VYUY8_2X8,
-		.l.width	= 10,
-		.l.bpp		= 4,
-		.s.width	= 8,
-		.s.bpp		= 2,
-		.supported	= false,
+		.bitsperpixel	= 16,
 	}, {
-		.name		= "RAW8 BGGR",
 		.fourcc		= V4L2_PIX_FMT_SBGGR8,
 		.code		= MEDIA_BUS_FMT_SBGGR8_1X8,
-		.l.width	= 10,
-		.l.bpp		= 2,
-		.s.width	= 8,
-		.s.bpp		= 1,
-		.supported	= false,
+		.bitsperpixel	= 8,
 	}, {
-		.name		= "RAW8 GBRG",
 		.fourcc		= V4L2_PIX_FMT_SGBRG8,
 		.code		= MEDIA_BUS_FMT_SGBRG8_1X8,
-		.l.width	= 10,
-		.l.bpp		= 2,
-		.s.width	= 8,
-		.s.bpp		= 1,
-		.supported	= false,
+		.bitsperpixel	= 8,
 	}, {
-		.name		= "RAW8 GRBG",
 		.fourcc		= V4L2_PIX_FMT_SGRBG8,
 		.code		= MEDIA_BUS_FMT_SGRBG8_1X8,
-		.l.width	= 10,
-		.l.bpp		= 2,
-		.s.width	= 8,
-		.s.bpp		= 1,
-		.supported	= false,
+		.bitsperpixel	= 8,
 	}, {
-		.name		= "RAW8 RGGB",
 		.fourcc		= V4L2_PIX_FMT_SRGGB8,
 		.code		= MEDIA_BUS_FMT_SRGGB8_1X8,
-		.l.width	= 10,
-		.l.bpp		= 2,
-		.s.width	= 8,
-		.s.bpp		= 1,
-		.supported	= false,
+		.bitsperpixel	= 8,
 	}, {
-		.name		= "RGB565 (LE)",
 		.fourcc		= V4L2_PIX_FMT_RGB565,
 		.code		= MEDIA_BUS_FMT_RGB565_2X8_LE,
-		.l.width	= 10,
-		.l.bpp		= 4,
-		.s.width	= 8,
-		.s.bpp		= 2,
-		.supported	= false,
+		.bitsperpixel	= 16,
 	}, {
-		.name		= "RGB565 (BE)",
 		.fourcc		= V4L2_PIX_FMT_RGB565X,
 		.code		= MEDIA_BUS_FMT_RGB565_2X8_BE,
-		.l.width	= 10,
-		.l.bpp		= 4,
-		.s.width	= 8,
-		.s.bpp		= 2,
-		.supported	= false,
+		.bitsperpixel	= 16,
 	},
 };
 
-static int
-__vpfe_get_format(struct vpfe_device *vpfe,
-		  struct v4l2_format *format, unsigned int *bpp);
+static int __subdev_get_format(struct vpfe_device *vpfe,
+			       struct v4l2_mbus_framefmt *fmt);
+static int vpfe_calc_format_size(struct vpfe_device *vpfe,
+				 const struct vpfe_fmt *fmt,
+				 struct v4l2_format *f);
 
-static struct vpfe_fmt *find_format_by_code(unsigned int code)
+static const struct vpfe_fmt *find_format_by_code(struct vpfe_device *vpfe,
+						  unsigned int code)
 {
 	struct vpfe_fmt *fmt;
 	unsigned int k;
 
-	for (k = 0; k < ARRAY_SIZE(formats); k++) {
-		fmt = &formats[k];
+	for (k = 0; k < vpfe->num_active_fmt; k++) {
+		fmt = vpfe->active_fmt[k];
 		if (fmt->code == code)
 			return fmt;
 	}
@@ -207,13 +134,14 @@
 	return NULL;
 }
 
-static struct vpfe_fmt *find_format_by_pix(unsigned int pixelformat)
+static const struct vpfe_fmt *find_format_by_pix(struct vpfe_device *vpfe,
+						 unsigned int pixelformat)
 {
 	struct vpfe_fmt *fmt;
 	unsigned int k;
 
-	for (k = 0; k < ARRAY_SIZE(formats); k++) {
-		fmt = &formats[k];
+	for (k = 0; k < vpfe->num_active_fmt; k++) {
+		fmt = vpfe->active_fmt[k];
 		if (fmt->fourcc == pixelformat)
 			return fmt;
 	}
@@ -221,76 +149,18 @@
 	return NULL;
 }
 
-static void
-mbus_to_pix(struct vpfe_device *vpfe,
-	    const struct v4l2_mbus_framefmt *mbus,
-	    struct v4l2_pix_format *pix, unsigned int *bpp)
+static unsigned int __get_bytesperpixel(struct vpfe_device *vpfe,
+					const struct vpfe_fmt *fmt)
 {
 	struct vpfe_subdev_info *sdinfo = vpfe->current_subdev;
 	unsigned int bus_width = sdinfo->vpfe_param.bus_width;
-	struct vpfe_fmt *fmt;
-
-	fmt = find_format_by_code(mbus->code);
-	if (WARN_ON(fmt == NULL)) {
-		pr_err("Invalid mbus code set\n");
-		*bpp = 1;
-		return;
-	}
-
-	memset(pix, 0, sizeof(*pix));
-	v4l2_fill_pix_format(pix, mbus);
-	pix->pixelformat = fmt->fourcc;
-	*bpp = (bus_width == 10) ?  fmt->l.bpp : fmt->s.bpp;
-
-	/* pitch should be 32 bytes aligned */
-	pix->bytesperline = ALIGN(pix->width * *bpp, 32);
-	pix->sizeimage = pix->bytesperline * pix->height;
-}
-
-static void pix_to_mbus(struct vpfe_device *vpfe,
-			struct v4l2_pix_format *pix_fmt,
-			struct v4l2_mbus_framefmt *mbus_fmt)
-{
-	struct vpfe_fmt *fmt;
+	u32 bpp, bus_width_bytes, clocksperpixel;
 
-	fmt = find_format_by_pix(pix_fmt->pixelformat);
-	if (!fmt) {
-		/* default to first entry */
-		vpfe_dbg(3, vpfe, "Invalid pixel code: %x, default used instead\n",
-			pix_fmt->pixelformat);
-		fmt = &formats[0];
-	}
+	bus_width_bytes = ALIGN(bus_width, 8) >> 3;
+	clocksperpixel = DIV_ROUND_UP(fmt->bitsperpixel, bus_width);
+	bpp = clocksperpixel * bus_width_bytes;
 
-	memset(mbus_fmt, 0, sizeof(*mbus_fmt));
-	v4l2_fill_mbus_format(mbus_fmt, pix_fmt, fmt->code);
-}
-
-/*  Print Four-character-code (FOURCC) */
-static char *print_fourcc(u32 fmt)
-{
-	static char code[5];
-
-	code[0] = (unsigned char)(fmt & 0xff);
-	code[1] = (unsigned char)((fmt >> 8) & 0xff);
-	code[2] = (unsigned char)((fmt >> 16) & 0xff);
-	code[3] = (unsigned char)((fmt >> 24) & 0xff);
-	code[4] = '\0';
-
-	return code;
-}
-
-static int
-cmp_v4l2_format(const struct v4l2_format *lhs, const struct v4l2_format *rhs)
-{
-	return lhs->type == rhs->type &&
-		lhs->fmt.pix.width == rhs->fmt.pix.width &&
-		lhs->fmt.pix.height == rhs->fmt.pix.height &&
-		lhs->fmt.pix.pixelformat == rhs->fmt.pix.pixelformat &&
-		lhs->fmt.pix.field == rhs->fmt.pix.field &&
-		lhs->fmt.pix.colorspace == rhs->fmt.pix.colorspace &&
-		lhs->fmt.pix.ycbcr_enc == rhs->fmt.pix.ycbcr_enc &&
-		lhs->fmt.pix.quantization == rhs->fmt.pix.quantization &&
-		lhs->fmt.pix.xfer_func == rhs->fmt.pix.xfer_func;
+	return bpp;
 }
 
 static inline u32 vpfe_reg_read(struct vpfe_ccdc *ccdc, u32 offset)
@@ -357,13 +227,9 @@
 	if (frm_fmt == CCDC_FRMFMT_INTERLACED) {
 		vert_nr_lines = (image_win->height >> 1) - 1;
 		vert_start >>= 1;
-		/* Since first line doesn't have any data */
-		vert_start += 1;
 		/* configure VDINT0 */
 		val = (vert_start << VPFE_VDINT_VDINT0_SHIFT);
 	} else {
-		/* Since first line doesn't have any data */
-		vert_start += 1;
 		vert_nr_lines = image_win->height - 1;
 		/*
 		 * configure VDINT0 and VDINT1. VDINT1 will be at half
@@ -457,40 +323,25 @@
 
 static int vpfe_ccdc_close(struct vpfe_ccdc *ccdc, struct device *dev)
 {
-	int dma_cntl, i, pcr;
-
-	/* If the CCDC module is still busy wait for it to be done */
-	for (i = 0; i < 10; i++) {
-		usleep_range(5000, 6000);
-		pcr = vpfe_reg_read(ccdc, VPFE_PCR);
-		if (!pcr)
-			break;
+	struct vpfe_device *vpfe = container_of(ccdc, struct vpfe_device, ccdc);
+	u32 dma_cntl, pcr;
 
-		/* make sure it it is disabled */
-		vpfe_pcr_enable(ccdc, 0);
-	}
+	pcr = vpfe_reg_read(ccdc, VPFE_PCR);
+	if (pcr)
+		vpfe_dbg(1, vpfe, "VPFE_PCR is still set (%x)", pcr);
+
+	dma_cntl = vpfe_reg_read(ccdc, VPFE_DMA_CNTL);
+	if ((dma_cntl & VPFE_DMA_CNTL_OVERFLOW))
+		vpfe_dbg(1, vpfe, "VPFE_DMA_CNTL_OVERFLOW is still set (%x)",
+			 dma_cntl);
 
 	/* Disable CCDC by resetting all register to default POR values */
 	vpfe_ccdc_restore_defaults(ccdc);
 
-	/* if DMA_CNTL overflow bit is set. Clear it
-	 *  It appears to take a while for this to become quiescent ~20ms
-	 */
-	for (i = 0; i < 10; i++) {
-		dma_cntl = vpfe_reg_read(ccdc, VPFE_DMA_CNTL);
-		if (!(dma_cntl & VPFE_DMA_CNTL_OVERFLOW))
-			break;
-
-		/* Clear the overflow bit */
-		vpfe_reg_write(ccdc, dma_cntl, VPFE_DMA_CNTL);
-		usleep_range(5000, 6000);
-	}
-
 	/* Disabled the module at the CONFIG level */
 	vpfe_config_enable(ccdc, 0);
 
 	pm_runtime_put_sync(dev);
-
 	return 0;
 }
 
@@ -763,8 +614,8 @@
 {
 	struct vpfe_device *vpfe = container_of(ccdc, struct vpfe_device, ccdc);
 
-	vpfe_dbg(1, vpfe, "vpfe_ccdc_set_pixel_format: if_type: %d, pixfmt:%s\n",
-		 ccdc->ccdc_cfg.if_type, print_fourcc(pixfmt));
+	vpfe_dbg(1, vpfe, "vpfe_ccdc_set_pixel_format: if_type: %d, pixfmt:%4.4s\n",
+		 ccdc->ccdc_cfg.if_type, (char *)&pixfmt);
 
 	if (ccdc->ccdc_cfg.if_type == VPFE_RAW_BAYER) {
 		ccdc->ccdc_cfg.bayer.pix_fmt = CCDC_PIXFMT_RAW;
@@ -1048,23 +899,25 @@
 static int vpfe_config_ccdc_image_format(struct vpfe_device *vpfe)
 {
 	enum ccdc_frmfmt frm_fmt = CCDC_FRMFMT_INTERLACED;
+	u32 bpp;
 	int ret = 0;
 
 	vpfe_dbg(2, vpfe, "vpfe_config_ccdc_image_format\n");
 
-	vpfe_dbg(1, vpfe, "pixelformat: %s\n",
-		print_fourcc(vpfe->fmt.fmt.pix.pixelformat));
+	vpfe_dbg(1, vpfe, "pixelformat: %4.4s\n",
+		 (char *)&vpfe->v_fmt.fmt.pix.pixelformat);
 
 	if (vpfe_ccdc_set_pixel_format(&vpfe->ccdc,
-			vpfe->fmt.fmt.pix.pixelformat) < 0) {
+			vpfe->v_fmt.fmt.pix.pixelformat) < 0) {
 		vpfe_err(vpfe, "couldn't set pix format in ccdc\n");
 		return -EINVAL;
 	}
 
 	/* configure the image window */
-	vpfe_ccdc_set_image_window(&vpfe->ccdc, &vpfe->crop, vpfe->bpp);
+	bpp = __get_bytesperpixel(vpfe, vpfe->fmt);
+	vpfe_ccdc_set_image_window(&vpfe->ccdc, &vpfe->crop, bpp);
 
-	switch (vpfe->fmt.fmt.pix.field) {
+	switch (vpfe->v_fmt.fmt.pix.field) {
 	case V4L2_FIELD_INTERLACED:
 		/* do nothing, since it is default */
 		ret = vpfe_ccdc_set_buftype(
@@ -1106,7 +959,8 @@
 static int vpfe_config_image_format(struct vpfe_device *vpfe,
 				    v4l2_std_id std_id)
 {
-	struct v4l2_pix_format *pix = &vpfe->fmt.fmt.pix;
+	const struct vpfe_fmt *fmt;
+	struct v4l2_mbus_framefmt mbus_fmt;
 	int i, ret;
 
 	for (i = 0; i < ARRAY_SIZE(vpfe_standards); i++) {
@@ -1128,26 +982,29 @@
 		return -EINVAL;
 	}
 
-	vpfe->crop.top = vpfe->crop.left = 0;
-	vpfe->crop.width = vpfe->std_info.active_pixels;
-	vpfe->crop.height = vpfe->std_info.active_lines;
-	pix->width = vpfe->crop.width;
-	pix->height = vpfe->crop.height;
-	pix->pixelformat = V4L2_PIX_FMT_YUYV;
-
-	/* first field and frame format based on standard frame format */
-	if (vpfe->std_info.frame_format)
-		pix->field = V4L2_FIELD_INTERLACED;
-	else
-		pix->field = V4L2_FIELD_NONE;
-
-	ret = __vpfe_get_format(vpfe, &vpfe->fmt, &vpfe->bpp);
+	ret = __subdev_get_format(vpfe, &mbus_fmt);
 	if (ret)
 		return ret;
 
+	fmt = find_format_by_code(vpfe, mbus_fmt.code);
+	if (!fmt) {
+		vpfe_dbg(3, vpfe, "mbus code format (0x%08x) not found.\n",
+			 mbus_fmt.code);
+		return -EINVAL;
+	}
+
+	/* Save current subdev format */
+	v4l2_fill_pix_format(&vpfe->v_fmt.fmt.pix, &mbus_fmt);
+	vpfe->v_fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+	vpfe->v_fmt.fmt.pix.pixelformat = fmt->fourcc;
+	vpfe_calc_format_size(vpfe, fmt, &vpfe->v_fmt);
+	vpfe->fmt = fmt;
+
 	/* Update the crop window based on found values */
-	vpfe->crop.width = pix->width;
-	vpfe->crop.height = pix->height;
+	vpfe->crop.top = 0;
+	vpfe->crop.left = 0;
+	vpfe->crop.width = mbus_fmt.width;
+	vpfe->crop.height = mbus_fmt.height;
 
 	return vpfe_config_ccdc_image_format(vpfe);
 }
@@ -1249,22 +1106,29 @@
  * This function will get next buffer from the dma queue and
  * set the buffer address in the vpfe register for capture.
  * the buffer is marked active
- *
- * Assumes caller is holding vpfe->dma_queue_lock already
  */
-static inline void vpfe_schedule_next_buffer(struct vpfe_device *vpfe)
+static void vpfe_schedule_next_buffer(struct vpfe_device *vpfe)
 {
+	dma_addr_t addr;
+
+	spin_lock(&vpfe->dma_queue_lock);
+	if (list_empty(&vpfe->dma_queue)) {
+		spin_unlock(&vpfe->dma_queue_lock);
+		return;
+	}
+
 	vpfe->next_frm = list_entry(vpfe->dma_queue.next,
 				    struct vpfe_cap_buffer, list);
 	list_del(&vpfe->next_frm->list);
+	spin_unlock(&vpfe->dma_queue_lock);
 
-	vpfe_set_sdr_addr(&vpfe->ccdc,
-	       vb2_dma_contig_plane_dma_addr(&vpfe->next_frm->vb.vb2_buf, 0));
+	addr = vb2_dma_contig_plane_dma_addr(&vpfe->next_frm->vb.vb2_buf, 0);
+	vpfe_set_sdr_addr(&vpfe->ccdc, addr);
 }
 
 static inline void vpfe_schedule_bottom_field(struct vpfe_device *vpfe)
 {
-	unsigned long addr;
+	dma_addr_t addr;
 
 	addr = vb2_dma_contig_plane_dma_addr(&vpfe->next_frm->vb.vb2_buf, 0) +
 					vpfe->field_off;
@@ -1283,12 +1147,64 @@
 static inline void vpfe_process_buffer_complete(struct vpfe_device *vpfe)
 {
 	vpfe->cur_frm->vb.vb2_buf.timestamp = ktime_get_ns();
-	vpfe->cur_frm->vb.field = vpfe->fmt.fmt.pix.field;
+	vpfe->cur_frm->vb.field = vpfe->v_fmt.fmt.pix.field;
 	vpfe->cur_frm->vb.sequence = vpfe->sequence++;
 	vb2_buffer_done(&vpfe->cur_frm->vb.vb2_buf, VB2_BUF_STATE_DONE);
 	vpfe->cur_frm = vpfe->next_frm;
 }
 
+static void vpfe_handle_interlaced_irq(struct vpfe_device *vpfe,
+				       enum v4l2_field field)
+{
+	int fid;
+
+	/* interlaced or TB capture check which field
+	 * we are in hardware
+	 */
+	fid = vpfe_ccdc_getfid(&vpfe->ccdc);
+
+	/* switch the software maintained field id */
+	vpfe->field ^= 1;
+	if (fid == vpfe->field) {
+		/* we are in-sync here,continue */
+		if (fid == 0) {
+			/*
+			 * One frame is just being captured. If the
+			 * next frame is available, release the
+			 * current frame and move on
+			 */
+			if (vpfe->cur_frm != vpfe->next_frm)
+				vpfe_process_buffer_complete(vpfe);
+
+			if (vpfe->stopping)
+				return;
+
+			/*
+			 * based on whether the two fields are stored
+			 * interleave or separately in memory,
+			 * reconfigure the CCDC memory address
+			 */
+			if (field == V4L2_FIELD_SEQ_TB)
+				vpfe_schedule_bottom_field(vpfe);
+		} else {
+			/*
+			 * if one field is just being captured configure
+			 * the next frame get the next frame from the empty
+			 * queue if no frame is available hold on to the
+			 * current buffer
+			 */
+			if (vpfe->cur_frm == vpfe->next_frm)
+				vpfe_schedule_next_buffer(vpfe);
+		}
+	} else if (fid == 0) {
+		/*
+		 * out of sync. Recover from any hardware out-of-sync.
+		 * May loose one frame
+		 */
+		vpfe->field = fid;
+	}
+}
+
 /*
  * vpfe_isr : ISR handler for vpfe capture (VINT0)
  * @irq: irq number
@@ -1300,76 +1216,28 @@
 static irqreturn_t vpfe_isr(int irq, void *dev)
 {
 	struct vpfe_device *vpfe = (struct vpfe_device *)dev;
-	enum v4l2_field field;
-	int intr_status;
-	int fid;
+	enum v4l2_field field = vpfe->v_fmt.fmt.pix.field;
+	int intr_status, stopping = vpfe->stopping;
 
 	intr_status = vpfe_reg_read(&vpfe->ccdc, VPFE_IRQ_STS);
 
 	if (intr_status & VPFE_VDINT0) {
-		field = vpfe->fmt.fmt.pix.field;
-
 		if (field == V4L2_FIELD_NONE) {
-			/* handle progressive frame capture */
 			if (vpfe->cur_frm != vpfe->next_frm)
 				vpfe_process_buffer_complete(vpfe);
-			goto next_intr;
+		} else {
+			vpfe_handle_interlaced_irq(vpfe, field);
 		}
-
-		/* interlaced or TB capture check which field
-		   we are in hardware */
-		fid = vpfe_ccdc_getfid(&vpfe->ccdc);
-
-		/* switch the software maintained field id */
-		vpfe->field ^= 1;
-		if (fid == vpfe->field) {
-			/* we are in-sync here,continue */
-			if (fid == 0) {
-				/*
-				 * One frame is just being captured. If the
-				 * next frame is available, release the
-				 * current frame and move on
-				 */
-				if (vpfe->cur_frm != vpfe->next_frm)
-					vpfe_process_buffer_complete(vpfe);
-				/*
-				 * based on whether the two fields are stored
-				 * interleave or separately in memory,
-				 * reconfigure the CCDC memory address
-				 */
-				if (field == V4L2_FIELD_SEQ_TB)
-					vpfe_schedule_bottom_field(vpfe);
-
-				goto next_intr;
-			}
-			/*
-			 * if one field is just being captured configure
-			 * the next frame get the next frame from the empty
-			 * queue if no frame is available hold on to the
-			 * current buffer
-			 */
-			spin_lock(&vpfe->dma_queue_lock);
-			if (!list_empty(&vpfe->dma_queue) &&
-			    vpfe->cur_frm == vpfe->next_frm)
-				vpfe_schedule_next_buffer(vpfe);
-			spin_unlock(&vpfe->dma_queue_lock);
-		} else if (fid == 0) {
-			/*
-			 * out of sync. Recover from any hardware out-of-sync.
-			 * May loose one frame
-			 */
-			vpfe->field = fid;
+		if (stopping) {
+			vpfe->stopping = false;
+			complete(&vpfe->capture_stop);
 		}
 	}
 
-next_intr:
-	if (intr_status & VPFE_VDINT1) {
-		spin_lock(&vpfe->dma_queue_lock);
-		if (vpfe->fmt.fmt.pix.field == V4L2_FIELD_NONE &&
-		    !list_empty(&vpfe->dma_queue) &&
+	if (intr_status & VPFE_VDINT1 && !stopping) {
+		if (field == V4L2_FIELD_NONE &&
 		    vpfe->cur_frm == vpfe->next_frm)
 			vpfe_schedule_next_buffer(vpfe);
-		spin_unlock(&vpfe->dma_queue_lock);
 	}
 
 	vpfe_clear_intr(&vpfe->ccdc, intr_status);
@@ -1420,83 +1288,75 @@
 }
 
 /* get the format set at output pad of the adjacent subdev */
-static int __vpfe_get_format(struct vpfe_device *vpfe,
-			     struct v4l2_format *format, unsigned int *bpp)
+static int __subdev_get_format(struct vpfe_device *vpfe,
+			       struct v4l2_mbus_framefmt *fmt)
 {
-	struct v4l2_mbus_framefmt mbus_fmt;
-	struct vpfe_subdev_info *sdinfo;
-	struct v4l2_subdev_format fmt;
+	struct v4l2_subdev *sd = vpfe->current_subdev->sd;
+	struct v4l2_subdev_format sd_fmt;
+	struct v4l2_mbus_framefmt *mbus_fmt = &sd_fmt.format;
 	int ret;
 
-	sdinfo = vpfe->current_subdev;
-	if (!sdinfo->sd)
-		return -EINVAL;
-
-	fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
-	fmt.pad = 0;
+	sd_fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+	sd_fmt.pad = 0;
 
-	ret = v4l2_subdev_call(sdinfo->sd, pad, get_fmt, NULL, &fmt);
-	if (ret && ret != -ENOIOCTLCMD && ret != -ENODEV)
+	ret = v4l2_subdev_call(sd, pad, get_fmt, NULL, &sd_fmt);
+	if (ret)
 		return ret;
 
-	if (!ret) {
-		v4l2_fill_pix_format(&format->fmt.pix, &fmt.format);
-		mbus_to_pix(vpfe, &fmt.format, &format->fmt.pix, bpp);
-	} else {
-		ret = v4l2_device_call_until_err(&vpfe->v4l2_dev,
-						 sdinfo->grp_id,
-						 pad, get_fmt,
-						 NULL, &fmt);
-		if (ret && ret != -ENOIOCTLCMD && ret != -ENODEV)
-			return ret;
-		v4l2_fill_pix_format(&format->fmt.pix, &mbus_fmt);
-		mbus_to_pix(vpfe, &mbus_fmt, &format->fmt.pix, bpp);
-	}
-
-	format->type = vpfe->fmt.type;
+	*fmt = *mbus_fmt;
 
-	vpfe_dbg(1, vpfe,
-		 "%s size %dx%d (%s) bytesperline = %d, size = %d, bpp = %d\n",
-		 __func__, format->fmt.pix.width, format->fmt.pix.height,
-		 print_fourcc(format->fmt.pix.pixelformat),
-		 format->fmt.pix.bytesperline, format->fmt.pix.sizeimage, *bpp);
+	vpfe_dbg(1, vpfe, "%s %dx%d code:%04X\n", __func__,
+		 fmt->width, fmt->height, fmt->code);
 
 	return 0;
 }
 
 /* set the format at output pad of the adjacent subdev */
-static int __vpfe_set_format(struct vpfe_device *vpfe,
-			     struct v4l2_format *format, unsigned int *bpp)
+static int __subdev_set_format(struct vpfe_device *vpfe,
+			       struct v4l2_mbus_framefmt *fmt)
 {
-	struct vpfe_subdev_info *sdinfo;
-	struct v4l2_subdev_format fmt;
+	struct v4l2_subdev *sd = vpfe->current_subdev->sd;
+	struct v4l2_subdev_format sd_fmt;
+	struct v4l2_mbus_framefmt *mbus_fmt = &sd_fmt.format;
 	int ret;
 
-	vpfe_dbg(2, vpfe, "__vpfe_set_format\n");
+	sd_fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+	sd_fmt.pad = 0;
+	*mbus_fmt = *fmt;
 
-	sdinfo = vpfe->current_subdev;
-	if (!sdinfo->sd)
-		return -EINVAL;
+	ret = v4l2_subdev_call(sd, pad, set_fmt, NULL, &sd_fmt);
+	if (ret)
+		return ret;
 
-	fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
-	fmt.pad = 0;
+	vpfe_dbg(1, vpfe, "%s %dx%d code:%04X\n", __func__,
+		 fmt->width, fmt->height, fmt->code);
 
-	pix_to_mbus(vpfe, &format->fmt.pix, &fmt.format);
+	return 0;
+}
 
-	ret = v4l2_subdev_call(sdinfo->sd, pad, set_fmt, NULL, &fmt);
-	if (ret)
-		return ret;
+/* Needs more work */
+static int vpfe_calc_format_size(struct vpfe_device *vpfe,
+				 const struct vpfe_fmt *fmt,
+				 struct v4l2_format *f)
+{
+	u32 bpp;
 
-	v4l2_fill_pix_format(&format->fmt.pix, &fmt.format);
-	mbus_to_pix(vpfe, &fmt.format, &format->fmt.pix, bpp);
+	if (!fmt) {
+		vpfe_dbg(3, vpfe, "No vpfe_fmt provided!\n");
+		return -EINVAL;
+	}
 
-	format->type = vpfe->fmt.type;
+	bpp = __get_bytesperpixel(vpfe, fmt);
 
-	vpfe_dbg(1, vpfe,
-		 "%s size %dx%d (%s) bytesperline = %d, size = %d, bpp = %d\n",
-		 __func__,  format->fmt.pix.width, format->fmt.pix.height,
-		 print_fourcc(format->fmt.pix.pixelformat),
-		 format->fmt.pix.bytesperline, format->fmt.pix.sizeimage, *bpp);
+	/* pitch should be 32 bytes aligned */
+	f->fmt.pix.bytesperline = ALIGN(f->fmt.pix.width * bpp, 32);
+	f->fmt.pix.sizeimage = f->fmt.pix.bytesperline *
+			       f->fmt.pix.height;
+
+	vpfe_dbg(3, vpfe, "%s: fourcc: %4.4s size: %dx%d bpl:%d img_size:%d\n",
+		 __func__, (char *)&f->fmt.pix.pixelformat,
+		 f->fmt.pix.width, f->fmt.pix.height,
+		 f->fmt.pix.bytesperline, f->fmt.pix.sizeimage);
 
 	return 0;
 }
@@ -1508,7 +1368,7 @@
 
 	vpfe_dbg(2, vpfe, "vpfe_g_fmt\n");
 
-	*fmt = vpfe->fmt;
+	*fmt = vpfe->v_fmt;
 
 	return 0;
 }
@@ -1518,8 +1378,7 @@
 {
 	struct vpfe_device *vpfe = video_drvdata(file);
 	struct vpfe_subdev_info *sdinfo;
-	struct vpfe_fmt *fmt = NULL;
-	unsigned int k;
+	struct vpfe_fmt *fmt;
 
 	vpfe_dbg(2, vpfe, "vpfe_enum_format index:%d\n",
 		f->index);
@@ -1528,78 +1387,122 @@
 	if (!sdinfo->sd)
 		return -EINVAL;
 
-	if (f->index > ARRAY_SIZE(formats))
+	if (f->index >= vpfe->num_active_fmt)
 		return -EINVAL;
 
-	for (k = 0; k < ARRAY_SIZE(formats); k++) {
-		if (formats[k].index == f->index) {
-			fmt = &formats[k];
-			break;
-		}
-	}
-	if (!fmt)
-		return -EINVAL;
+	fmt = vpfe->active_fmt[f->index];
 
-	strncpy(f->description, fmt->name, sizeof(f->description) - 1);
 	f->pixelformat = fmt->fourcc;
-	f->type = vpfe->fmt.type;
-
-	vpfe_dbg(1, vpfe, "vpfe_enum_format: mbus index: %d code: %x pixelformat: %s [%s]\n",
-		f->index, fmt->code, print_fourcc(fmt->fourcc), fmt->name);
+	f->type = vpfe->v_fmt.type;
+	vpfe_dbg(1, vpfe, "vpfe_enum_format: mbus index: %d code: %x pixelformat: %4.4s\n",
+		 f->index, fmt->code, (char *)&fmt->fourcc);
 
 	return 0;
 }
 
 static int vpfe_try_fmt(struct file *file, void *priv,
-			struct v4l2_format *fmt)
+			struct v4l2_format *f)
 {
 	struct vpfe_device *vpfe = video_drvdata(file);
-	unsigned int bpp;
+	struct v4l2_subdev *sd = vpfe->current_subdev->sd;
+	const struct vpfe_fmt *fmt;
+	struct v4l2_subdev_frame_size_enum fse;
+	int ret, found;
 
-	vpfe_dbg(2, vpfe, "vpfe_try_fmt\n");
+	fmt = find_format_by_pix(vpfe, f->fmt.pix.pixelformat);
+	if (!fmt) {
+		/* default to first entry */
+		vpfe_dbg(3, vpfe, "Invalid pixel code: %x, default used instead\n",
+			 f->fmt.pix.pixelformat);
+		fmt = vpfe->active_fmt[0];
+		f->fmt.pix.pixelformat = fmt->fourcc;
+	}
+
+	f->fmt.pix.field = vpfe->v_fmt.fmt.pix.field;
 
-	return __vpfe_get_format(vpfe, fmt, &bpp);
+	/* check for/find a valid width/height */
+	ret = 0;
+	found = false;
+	fse.pad = 0;
+	fse.code = fmt->code;
+	fse.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+	for (fse.index = 0; ; fse.index++) {
+		ret = v4l2_subdev_call(sd, pad, enum_frame_size,
+				       NULL, &fse);
+		if (ret)
+			break;
+
+		if (f->fmt.pix.width == fse.max_width &&
+		    f->fmt.pix.height == fse.max_height) {
+			found = true;
+			break;
+		} else if (f->fmt.pix.width >= fse.min_width &&
+			   f->fmt.pix.width <= fse.max_width &&
+			   f->fmt.pix.height >= fse.min_height &&
+			   f->fmt.pix.height <= fse.max_height) {
+			found = true;
+			break;
+		}
+	}
+
+	if (!found) {
+		/* use existing values as default */
+		f->fmt.pix.width = vpfe->v_fmt.fmt.pix.width;
+		f->fmt.pix.height =  vpfe->v_fmt.fmt.pix.height;
+	}
+
+	/*
+	 * Use current colorspace for now, it will get
+	 * updated properly during s_fmt
+	 */
+	f->fmt.pix.colorspace = vpfe->v_fmt.fmt.pix.colorspace;
+	return vpfe_calc_format_size(vpfe, fmt, f);
 }
 
 static int vpfe_s_fmt(struct file *file, void *priv,
-		      struct v4l2_format *fmt)
+		      struct v4l2_format *f)
 {
 	struct vpfe_device *vpfe = video_drvdata(file);
-	struct v4l2_format format;
-	unsigned int bpp;
+	const struct vpfe_fmt *fmt;
+	struct v4l2_mbus_framefmt mbus_fmt;
 	int ret;
 
-	vpfe_dbg(2, vpfe, "vpfe_s_fmt\n");
-
 	/* If streaming is started, return error */
 	if (vb2_is_busy(&vpfe->buffer_queue)) {
 		vpfe_err(vpfe, "%s device busy\n", __func__);
 		return -EBUSY;
 	}
 
-	ret = __vpfe_get_format(vpfe, &format, &bpp);
-	if (ret)
+	ret = vpfe_try_fmt(file, priv, f);
+	if (ret < 0)
 		return ret;
 
+	fmt = find_format_by_pix(vpfe, f->fmt.pix.pixelformat);
 
-	if (!cmp_v4l2_format(fmt, &format)) {
-		/* Sensor format is different from the requested format
-		 * so we need to change it
-		 */
-		ret = __vpfe_set_format(vpfe, fmt, &bpp);
-		if (ret)
-			return ret;
-	} else /* Just make sure all of the fields are consistent */
-		*fmt = format;
+	v4l2_fill_mbus_format(&mbus_fmt, &f->fmt.pix, fmt->code);
 
-	/* First detach any IRQ if currently attached */
-	vpfe_detach_irq(vpfe);
-	vpfe->fmt = *fmt;
-	vpfe->bpp = bpp;
+	ret = __subdev_set_format(vpfe, &mbus_fmt);
+	if (ret)
+		return ret;
+
+	/* Just double check nothing has gone wrong */
+	if (mbus_fmt.code != fmt->code) {
+		vpfe_dbg(3, vpfe,
+			 "%s subdev changed format on us, this should not happen\n",
+			 __func__);
+		return -EINVAL;
+	}
+
+	v4l2_fill_pix_format(&vpfe->v_fmt.fmt.pix, &mbus_fmt);
+	vpfe->v_fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+	vpfe->v_fmt.fmt.pix.pixelformat  = fmt->fourcc;
+	vpfe_calc_format_size(vpfe, fmt, &vpfe->v_fmt);
+	*f = vpfe->v_fmt;
+	vpfe->fmt = fmt;
 
 	/* Update the crop window based on found values */
-	vpfe->crop.width = fmt->fmt.pix.width;
-	vpfe->crop.height = fmt->fmt.pix.height;
+	vpfe->crop.width = f->fmt.pix.width;
+	vpfe->crop.height = f->fmt.pix.height;
 
 	/* set image capture parameters in the ccdc */
 	return vpfe_config_ccdc_image_format(vpfe);
@@ -1610,45 +1513,30 @@
 {
 	struct vpfe_device *vpfe = video_drvdata(file);
 	struct v4l2_subdev_frame_size_enum fse;
-	struct vpfe_subdev_info *sdinfo;
-	struct v4l2_mbus_framefmt mbus;
-	struct v4l2_pix_format pix;
-	struct vpfe_fmt *fmt;
+	struct v4l2_subdev *sd = vpfe->current_subdev->sd;
+	const struct vpfe_fmt *fmt;
 	int ret;
 
 	vpfe_dbg(2, vpfe, "vpfe_enum_size\n");
 
 	/* check for valid format */
-	fmt = find_format_by_pix(fsize->pixel_format);
+	fmt = find_format_by_pix(vpfe, fsize->pixel_format);
 	if (!fmt) {
-		vpfe_dbg(3, vpfe, "Invalid pixel code: %x, default used instead\n",
-			fsize->pixel_format);
+		vpfe_dbg(3, vpfe, "Invalid pixel code: %x\n",
+			 fsize->pixel_format);
 		return -EINVAL;
 	}
 
 	memset(fsize->reserved, 0x0, sizeof(fsize->reserved));
 
-	sdinfo = vpfe->current_subdev;
-	if (!sdinfo->sd)
-		return -EINVAL;
-
-	memset(&pix, 0x0, sizeof(pix));
-	/* Construct pix from parameter and use default for the rest */
-	pix.pixelformat = fsize->pixel_format;
-	pix.width = 640;
-	pix.height = 480;
-	pix.colorspace = V4L2_COLORSPACE_SRGB;
-	pix.field = V4L2_FIELD_NONE;
-	pix_to_mbus(vpfe, &pix, &mbus);
-
 	memset(&fse, 0x0, sizeof(fse));
 	fse.index = fsize->index;
 	fse.pad = 0;
-	fse.code = mbus.code;
+	fse.code = fmt->code;
 	fse.which = V4L2_SUBDEV_FORMAT_ACTIVE;
-	ret = v4l2_subdev_call(sdinfo->sd, pad, enum_frame_size, NULL, &fse);
+	ret = v4l2_subdev_call(sd, pad, enum_frame_size, NULL, &fse);
 	if (ret)
-		return -EINVAL;
+		return ret;
 
 	vpfe_dbg(1, vpfe, "vpfe_enum_size: index: %d code: %x W:[%d,%d] H:[%d,%d]\n",
 		fse.index, fse.code, fse.min_width, fse.max_width,
@@ -1658,9 +1546,9 @@
 	fsize->discrete.width = fse.max_width;
 	fsize->discrete.height = fse.max_height;
 
-	vpfe_dbg(1, vpfe, "vpfe_enum_size: index: %d pixformat: %s size: %dx%d\n",
-		fsize->index, print_fourcc(fsize->pixel_format),
-		fsize->discrete.width, fsize->discrete.height);
+	vpfe_dbg(1, vpfe, "vpfe_enum_size: index: %d pixformat: %4.4s size: %dx%d\n",
+		 fsize->index, (char *)&fsize->pixel_format,
+		 fsize->discrete.width, fsize->discrete.height);
 
 	return 0;
 }
@@ -1916,7 +1804,7 @@
 			    unsigned int sizes[], struct device *alloc_devs[])
 {
 	struct vpfe_device *vpfe = vb2_get_drv_priv(vq);
-	unsigned size = vpfe->fmt.fmt.pix.sizeimage;
+	unsigned int size = vpfe->v_fmt.fmt.pix.sizeimage;
 
 	if (vq->num_buffers + *nbuffers < 3)
 		*nbuffers = 3 - vq->num_buffers;
@@ -1952,12 +1840,12 @@
 	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
 	struct vpfe_device *vpfe = vb2_get_drv_priv(vb->vb2_queue);
 
-	vb2_set_plane_payload(vb, 0, vpfe->fmt.fmt.pix.sizeimage);
+	vb2_set_plane_payload(vb, 0, vpfe->v_fmt.fmt.pix.sizeimage);
 
 	if (vb2_get_plane_payload(vb, 0) > vb2_plane_size(vb, 0))
 		return -EINVAL;
 
-	vbuf->field = vpfe->fmt.fmt.pix.field;
+	vbuf->field = vpfe->v_fmt.fmt.pix.field;
 
 	return 0;
 }
@@ -1979,6 +1867,29 @@
 	spin_unlock_irqrestore(&vpfe->dma_queue_lock, flags);
 }
 
+static void vpfe_return_all_buffers(struct vpfe_device *vpfe,
+				    enum vb2_buffer_state state)
+{
+	struct vpfe_cap_buffer *buf, *node;
+	unsigned long flags;
+
+	spin_lock_irqsave(&vpfe->dma_queue_lock, flags);
+	list_for_each_entry_safe(buf, node, &vpfe->dma_queue, list) {
+		vb2_buffer_done(&buf->vb.vb2_buf, state);
+		list_del(&buf->list);
+	}
+
+	if (vpfe->cur_frm)
+		vb2_buffer_done(&vpfe->cur_frm->vb.vb2_buf, state);
+
+	if (vpfe->next_frm && vpfe->next_frm != vpfe->cur_frm)
+		vb2_buffer_done(&vpfe->next_frm->vb.vb2_buf, state);
+
+	vpfe->cur_frm = NULL;
+	vpfe->next_frm = NULL;
+	spin_unlock_irqrestore(&vpfe->dma_queue_lock, flags);
+}
+
 /*
  * vpfe_start_streaming : Starts the DMA engine for streaming
  * @vb: ptr to vb2_buffer
@@ -1987,7 +1898,6 @@
 static int vpfe_start_streaming(struct vb2_queue *vq, unsigned int count)
 {
 	struct vpfe_device *vpfe = vb2_get_drv_priv(vq);
-	struct vpfe_cap_buffer *buf, *tmp;
 	struct vpfe_subdev_info *sdinfo;
 	unsigned long flags;
 	unsigned long addr;
@@ -2002,6 +1912,9 @@
 
 	vpfe_attach_irq(vpfe);
 
+	vpfe->stopping = false;
+	init_completion(&vpfe->capture_stop);
+
 	if (vpfe->ccdc.ccdc_cfg.if_type == VPFE_RAW_BAYER)
 		vpfe_ccdc_config_raw(&vpfe->ccdc);
 	else
@@ -2030,11 +1943,8 @@
 	return 0;
 
 err:
-	list_for_each_entry_safe(buf, tmp, &vpfe->dma_queue, list) {
-		list_del(&buf->list);
-		vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_QUEUED);
-	}
-
+	vpfe_return_all_buffers(vpfe, VB2_BUF_STATE_QUEUED);
+	vpfe_pcr_enable(&vpfe->ccdc, 0);
 	return ret;
 }
 
@@ -2049,11 +1959,15 @@
 {
 	struct vpfe_device *vpfe = vb2_get_drv_priv(vq);
 	struct vpfe_subdev_info *sdinfo;
-	unsigned long flags;
 	int ret;
 
 	vpfe_pcr_enable(&vpfe->ccdc, 0);
 
+	/* Wait for the last frame to be captured */
+	vpfe->stopping = true;
+	wait_for_completion_timeout(&vpfe->capture_stop,
+				    msecs_to_jiffies(250));
+
 	vpfe_detach_irq(vpfe);
 
 	sdinfo = vpfe->current_subdev;
@@ -2062,27 +1976,7 @@
 		vpfe_dbg(1, vpfe, "stream off failed in subdev\n");
 
 	/* release all active buffers */
-	spin_lock_irqsave(&vpfe->dma_queue_lock, flags);
-	if (vpfe->cur_frm == vpfe->next_frm) {
-		vb2_buffer_done(&vpfe->cur_frm->vb.vb2_buf,
-				VB2_BUF_STATE_ERROR);
-	} else {
-		if (vpfe->cur_frm != NULL)
-			vb2_buffer_done(&vpfe->cur_frm->vb.vb2_buf,
-					VB2_BUF_STATE_ERROR);
-		if (vpfe->next_frm != NULL)
-			vb2_buffer_done(&vpfe->next_frm->vb.vb2_buf,
-					VB2_BUF_STATE_ERROR);
-	}
-
-	while (!list_empty(&vpfe->dma_queue)) {
-		vpfe->next_frm = list_entry(vpfe->dma_queue.next,
-						struct vpfe_cap_buffer, list);
-		list_del(&vpfe->next_frm->list);
-		vb2_buffer_done(&vpfe->next_frm->vb.vb2_buf,
-				VB2_BUF_STATE_ERROR);
-	}
-	spin_unlock_irqrestore(&vpfe->dma_queue_lock, flags);
+	vpfe_return_all_buffers(vpfe, VB2_BUF_STATE_ERROR);
 }
 
 static int vpfe_cropcap(struct file *file, void *priv,
@@ -2092,6 +1986,9 @@
 
 	vpfe_dbg(2, vpfe, "vpfe_cropcap\n");
 
+	if (crop->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+		return -EINVAL;
+
 	if (vpfe->std_index >= ARRAY_SIZE(vpfe_standards))
 		return -EINVAL;
 
@@ -2151,6 +2048,7 @@
 	struct vpfe_device *vpfe = video_drvdata(file);
 	struct v4l2_rect cr = vpfe->crop;
 	struct v4l2_rect r = s->r;
+	u32 bpp;
 
 	/* If streaming is started, return error */
 	if (vb2_is_busy(&vpfe->buffer_queue)) {
@@ -2176,12 +2074,14 @@
 
 	s->r = vpfe->crop = r;
 
-	vpfe_ccdc_set_image_window(&vpfe->ccdc, &r, vpfe->bpp);
-	vpfe->fmt.fmt.pix.width = r.width;
-	vpfe->fmt.fmt.pix.height = r.height;
-	vpfe->fmt.fmt.pix.bytesperline = vpfe_ccdc_get_line_length(&vpfe->ccdc);
-	vpfe->fmt.fmt.pix.sizeimage = vpfe->fmt.fmt.pix.bytesperline *
-						vpfe->fmt.fmt.pix.height;
+	bpp = __get_bytesperpixel(vpfe, vpfe->fmt);
+	vpfe_ccdc_set_image_window(&vpfe->ccdc, &r, bpp);
+	vpfe->v_fmt.fmt.pix.width = r.width;
+	vpfe->v_fmt.fmt.pix.height = r.height;
+	vpfe->v_fmt.fmt.pix.bytesperline =
+		vpfe_ccdc_get_line_length(&vpfe->ccdc);
+	vpfe->v_fmt.fmt.pix.sizeimage = vpfe->v_fmt.fmt.pix.bytesperline *
+						vpfe->v_fmt.fmt.pix.height;
 
 	vpfe_dbg(1, vpfe, "cropped (%d,%d)/%dx%d of %dx%d\n",
 		 r.left, r.top, r.width, r.height, cr.width, cr.height);
@@ -2217,7 +2117,7 @@
 			return ret;
 		}
 		ret = vpfe_get_ccdc_image_format(vpfe,
-						 &vpfe->fmt);
+						 &vpfe->v_fmt);
 		if (ret < 0) {
 			vpfe_dbg(2, vpfe,
 				"Invalid image format at CCDC\n");
@@ -2302,8 +2202,10 @@
 					       struct vpfe_device, v4l2_dev);
 	struct v4l2_subdev_mbus_code_enum mbus_code;
 	struct vpfe_subdev_info *sdinfo;
+	struct vpfe_fmt *fmt;
+	int ret = 0;
 	bool found = false;
-	int i, j;
+	int i, j, k;
 
 	vpfe_dbg(1, vpfe, "vpfe_async_bound\n");
 
@@ -2325,27 +2227,37 @@
 
 	vpfe->video_dev.tvnorms |= sdinfo->inputs[0].std;
 
-	/* setup the supported formats & indexes */
-	for (j = 0, i = 0; ; ++j) {
-		struct vpfe_fmt *fmt;
-		int ret;
-
+	vpfe->num_active_fmt = 0;
+	for (j = 0, i = 0; (ret != -EINVAL); ++j) {
 		memset(&mbus_code, 0, sizeof(mbus_code));
 		mbus_code.index = j;
 		mbus_code.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 		ret = v4l2_subdev_call(subdev, pad, enum_mbus_code,
-			       NULL, &mbus_code);
+				       NULL, &mbus_code);
 		if (ret)
-			break;
-
-		fmt = find_format_by_code(mbus_code.code);
-		if (!fmt)
 			continue;
 
-		fmt->supported = true;
-		fmt->index = i++;
+		vpfe_dbg(3, vpfe,
+			 "subdev %s: code: %04x idx: %d\n",
+			 subdev->name, mbus_code.code, j);
+
+		for (k = 0; k < ARRAY_SIZE(formats); k++) {
+			fmt = &formats[k];
+			if (mbus_code.code != fmt->code)
+				continue;
+			vpfe->active_fmt[i] = fmt;
+			vpfe_dbg(3, vpfe,
+				 "matched fourcc: %4.4s code: %04x idx: %d\n",
+				 (char *)&fmt->fourcc, mbus_code.code, i);
+			vpfe->num_active_fmt = ++i;
+		}
 	}
 
+	if (!i) {
+		vpfe_err(vpfe, "No suitable format reported by subdev %s\n",
+			 subdev->name);
+		return -EINVAL;
+	}
 	return 0;
 }
 
@@ -2358,7 +2270,7 @@
 	spin_lock_init(&vpfe->dma_queue_lock);
 	mutex_init(&vpfe->lock);
 
-	vpfe->fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+	vpfe->v_fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
 
 	/* set first sub device as current one */
 	vpfe->current_subdev = &vpfe->cfg->sub_devs[0];
@@ -2671,22 +2583,22 @@
 	struct vpfe_device *vpfe = dev_get_drvdata(dev);
 	struct vpfe_ccdc *ccdc = &vpfe->ccdc;
 
-	/* if streaming has not started we don't care */
-	if (!vb2_start_streaming_called(&vpfe->buffer_queue))
-		return 0;
+	/* only do full suspend if streaming has started */
+	if (vb2_start_streaming_called(&vpfe->buffer_queue)) {
 
-	pm_runtime_get_sync(dev);
-	vpfe_config_enable(ccdc, 1);
+		pm_runtime_get_sync(dev);
+		vpfe_config_enable(ccdc, 1);
 
-	/* Save VPFE context */
-	vpfe_save_context(ccdc);
+		/* Save VPFE context */
+		vpfe_save_context(ccdc);
 
-	/* Disable CCDC */
-	vpfe_pcr_enable(ccdc, 0);
-	vpfe_config_enable(ccdc, 0);
+		/* Disable CCDC */
+		vpfe_pcr_enable(ccdc, 0);
+		vpfe_config_enable(ccdc, 0);
 
-	/* Disable both master and slave clock */
-	pm_runtime_put_sync(dev);
+		/* Disable both master and slave clock */
+		pm_runtime_put_sync(dev);
+	}
 
 	/* Select sleep pin state */
 	pinctrl_pm_select_sleep_state(dev);
@@ -2728,19 +2640,19 @@
 	struct vpfe_device *vpfe = dev_get_drvdata(dev);
 	struct vpfe_ccdc *ccdc = &vpfe->ccdc;
 
-	/* if streaming has not started we don't care */
-	if (!vb2_start_streaming_called(&vpfe->buffer_queue))
-		return 0;
+	/* only do full resume if streaming has started */
+	if (vb2_start_streaming_called(&vpfe->buffer_queue)) {
 
-	/* Enable both master and slave clock */
-	pm_runtime_get_sync(dev);
-	vpfe_config_enable(ccdc, 1);
+		/* Enable both master and slave clock */
+		pm_runtime_get_sync(dev);
+		vpfe_config_enable(ccdc, 1);
 
-	/* Restore VPFE context */
-	vpfe_restore_context(ccdc);
+		/* Restore VPFE context */
+		vpfe_restore_context(ccdc);
 
-	vpfe_config_enable(ccdc, 0);
-	pm_runtime_put_sync(dev);
+		vpfe_config_enable(ccdc, 0);
+		pm_runtime_put_sync(dev);
+	}
 
 	/* Select default pin state */
 	pinctrl_pm_select_default_state(dev);
diff -Naur --no-dereference a/drivers/media/platform/am437x/am437x-vpfe.h b/drivers/media/platform/am437x/am437x-vpfe.h
--- a/drivers/media/platform/am437x/am437x-vpfe.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/media/platform/am437x/am437x-vpfe.h	2020-06-19 21:00:31.416227715 -0400
@@ -23,6 +23,7 @@
 
 #include <linux/am437x-vpfe.h>
 #include <linux/clk.h>
+#include <linux/completion.h>
 #include <linux/device.h>
 #include <linux/io.h>
 #include <linux/i2c.h>
@@ -220,6 +221,25 @@
 	u32 ccdc_ctx[VPFE_REG_END / sizeof(u32)];
 };
 
+/*
+ * struct vpfe_fmt - VPFE media bus format information
+ * @fourcc: V4L2 pixel format code
+ * @code: V4L2 media bus format code
+ * @bitsperpixel: Bits per pixel over the bus
+ */
+struct vpfe_fmt {
+	u32 fourcc;
+	u32 code;
+	u32 bitsperpixel;
+};
+
+/*
+ * This value needs to be at least as large as the number of entry in
+ * formats[].
+ * When formats[] is modified make sure to adjust this value also.
+ */
+#define VPFE_MAX_ACTIVE_FMT	10
+
 struct vpfe_device {
 	/* V4l2 specific parameters */
 	/* Identifies video device for this channel */
@@ -254,9 +274,11 @@
 	/* Pointer pointing to next v4l2_buffer */
 	struct vpfe_cap_buffer *next_frm;
 	/* Used to store pixel format */
-	struct v4l2_format fmt;
-	/* Used to store current bytes per pixel based on current format */
-	unsigned int bpp;
+	const struct vpfe_fmt *fmt;
+	struct v4l2_format v_fmt;
+	struct vpfe_fmt	*active_fmt[VPFE_MAX_ACTIVE_FMT];
+	unsigned int num_active_fmt;
+
 	/*
 	 * used when IMP is chained to store the crop window which
 	 * is different from the image window
@@ -276,6 +298,8 @@
 	 */
 	u32 field_off;
 	struct vpfe_ccdc ccdc;
+	int stopping;
+	struct completion capture_stop;
 };
 
 #endif	/* AM437X_VPFE_H */
diff -Naur --no-dereference a/drivers/media/platform/Kconfig b/drivers/media/platform/Kconfig
--- a/drivers/media/platform/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/media/platform/Kconfig	2020-06-19 21:00:31.416227715 -0400
@@ -141,7 +141,7 @@
 config VIDEO_TI_CAL
 	tristate "TI CAL (Camera Adaptation Layer) driver"
 	depends on VIDEO_DEV && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
-	depends on SOC_DRA7XX || COMPILE_TEST
+	depends on SOC_DRA7XX || ARCH_K3 || COMPILE_TEST
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_FWNODE
 	default n
@@ -151,6 +151,19 @@
 	  In TI Technical Reference Manual this module is referred as
 	  Camera Interface Subsystem (CAMSS).
 
+config VIDEO_TI_VIP
+	tristate "TI Video Input Port"
+	default n
+	depends on VIDEO_DEV && VIDEO_V4L2 && SOC_DRA7XX
+	depends on HAS_DMA
+	select VIDEOBUF2_DMA_CONTIG
+	select VIDEO_TI_VPDMA
+	select VIDEO_TI_SC
+	select VIDEO_TI_CSC
+	help
+	Driver support for VIP module on certain TI SoC's
+	VIP = Video Input Port.
+
 endif # V4L_PLATFORM_DRIVERS
 
 menuconfig V4L_MEM2MEM_DRIVERS
@@ -468,6 +481,32 @@
 	---help---
 	  Enable debug messages on VPE driver.
 
+config VIDEO_IMG_VXD_DEC
+	tristate "IMG VXD DEC (Video Decoder) driver"
+	depends on VIDEO_DEV && VIDEO_V4L2
+	depends on ARCH_K3_J721E_SOC || COMPILE_TEST
+	select VIDEOBUF2_CORE
+	select VIDEOBUF2_DMA_CONTIG
+	select VIDEOBUF2_DMA_SG
+	select V4L2_MEM2MEM_DEV
+	---help---
+	This is an IMG VXD DEC V4L2 driver that adds support for the
+	Imagination D5520 (Video Decoder) hardware.
+	The module name when built is vxd-dec.
+
+config VIDEO_IMG_VXE_ENC
+	tristate "IMG VXE ENC (Video Encoder) driver"
+	depends on VIDEO_DEV && VIDEO_V4L2
+	depends on ARCH_K3_J721E_SOC || COMPILE_TEST
+	select VIDEOBUF2_CORE
+	select VIDEOBUF2_DMA_CONTIG
+	select VIDEOBUF2_DMA_SG
+	select V4L2_MEM2MEM_DEV
+	---help---
+	This is an IMG VXE ENC V4L2 driver that adds support for the
+	Imagination VXE384 (Video Encoder) hardware.
+	The module name when built is vxe-enc.
+
 config VIDEO_QCOM_VENUS
 	tristate "Qualcomm Venus V4L2 encoder/decoder driver"
 	depends on VIDEO_DEV && VIDEO_V4L2
diff -Naur --no-dereference a/drivers/media/platform/Makefile b/drivers/media/platform/Makefile
--- a/drivers/media/platform/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/media/platform/Makefile	2020-06-19 21:00:31.416227715 -0400
@@ -18,9 +18,7 @@
 obj-$(CONFIG_VIDEO_VIM2M)		+= vim2m.o
 obj-$(CONFIG_VIDEO_VICODEC)		+= vicodec/
 
-obj-$(CONFIG_VIDEO_TI_VPE)		+= ti-vpe/
-
-obj-$(CONFIG_VIDEO_TI_CAL)		+= ti-vpe/
+obj-y	+= ti-vpe/
 
 obj-$(CONFIG_VIDEO_MX2_EMMAPRP)		+= mx2_emmaprp.o
 obj-$(CONFIG_VIDEO_CODA)		+= coda/
diff -Naur --no-dereference a/drivers/media/platform/ti-vpe/cal.c b/drivers/media/platform/ti-vpe/cal.c
--- a/drivers/media/platform/ti-vpe/cal.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/media/platform/ti-vpe/cal.c	2020-06-19 21:00:31.416227715 -0400
@@ -17,6 +17,8 @@
 #include <linux/delay.h>
 #include <linux/pm_runtime.h>
 #include <linux/slab.h>
+#include <linux/mfd/syscon.h>
+#include <linux/regmap.h>
 #include <linux/videodev2.h>
 #include <linux/of_device.h>
 #include <linux/of_graph.h>
@@ -35,8 +37,8 @@
 
 #define CAL_MODULE_NAME "cal"
 
-#define MAX_WIDTH 1920
-#define MAX_HEIGHT 1200
+#define MAX_WIDTH_BYTES (8192 * 8)
+#define MAX_HEIGHT_LINES 16383
 
 #define CAL_VERSION "0.1.0"
 
@@ -74,8 +76,6 @@
 #define CAL_NUM_INPUT 1
 #define CAL_NUM_CONTEXT 2
 
-#define bytes_per_line(pixel, bpp) (ALIGN(pixel * bpp, 16))
-
 #define reg_read(dev, offset) ioread32(dev->base + offset)
 #define reg_write(dev, offset, val) iowrite32(val, dev->base + offset)
 
@@ -94,102 +94,103 @@
 struct cal_fmt {
 	u32	fourcc;
 	u32	code;
-	u8	depth;
+	/* Bits per pixel */
+	u8	bpp;
 };
 
 static struct cal_fmt cal_formats[] = {
 	{
 		.fourcc		= V4L2_PIX_FMT_YUYV,
 		.code		= MEDIA_BUS_FMT_YUYV8_2X8,
-		.depth		= 16,
+		.bpp		= 16,
 	}, {
 		.fourcc		= V4L2_PIX_FMT_UYVY,
 		.code		= MEDIA_BUS_FMT_UYVY8_2X8,
-		.depth		= 16,
+		.bpp		= 16,
 	}, {
 		.fourcc		= V4L2_PIX_FMT_YVYU,
 		.code		= MEDIA_BUS_FMT_YVYU8_2X8,
-		.depth		= 16,
+		.bpp		= 16,
 	}, {
 		.fourcc		= V4L2_PIX_FMT_VYUY,
 		.code		= MEDIA_BUS_FMT_VYUY8_2X8,
-		.depth		= 16,
+		.bpp		= 16,
 	}, {
 		.fourcc		= V4L2_PIX_FMT_RGB565, /* gggbbbbb rrrrrggg */
 		.code		= MEDIA_BUS_FMT_RGB565_2X8_LE,
-		.depth		= 16,
+		.bpp		= 16,
 	}, {
 		.fourcc		= V4L2_PIX_FMT_RGB565X, /* rrrrrggg gggbbbbb */
 		.code		= MEDIA_BUS_FMT_RGB565_2X8_BE,
-		.depth		= 16,
+		.bpp		= 16,
 	}, {
 		.fourcc		= V4L2_PIX_FMT_RGB555, /* gggbbbbb arrrrrgg */
 		.code		= MEDIA_BUS_FMT_RGB555_2X8_PADHI_LE,
-		.depth		= 16,
+		.bpp		= 16,
 	}, {
 		.fourcc		= V4L2_PIX_FMT_RGB555X, /* arrrrrgg gggbbbbb */
 		.code		= MEDIA_BUS_FMT_RGB555_2X8_PADHI_BE,
-		.depth		= 16,
+		.bpp		= 16,
 	}, {
 		.fourcc		= V4L2_PIX_FMT_RGB24, /* rgb */
 		.code		= MEDIA_BUS_FMT_RGB888_2X12_LE,
-		.depth		= 24,
+		.bpp		= 24,
 	}, {
 		.fourcc		= V4L2_PIX_FMT_BGR24, /* bgr */
 		.code		= MEDIA_BUS_FMT_RGB888_2X12_BE,
-		.depth		= 24,
+		.bpp		= 24,
 	}, {
 		.fourcc		= V4L2_PIX_FMT_RGB32, /* argb */
 		.code		= MEDIA_BUS_FMT_ARGB8888_1X32,
-		.depth		= 32,
+		.bpp		= 32,
 	}, {
 		.fourcc		= V4L2_PIX_FMT_SBGGR8,
 		.code		= MEDIA_BUS_FMT_SBGGR8_1X8,
-		.depth		= 8,
+		.bpp		= 8,
 	}, {
 		.fourcc		= V4L2_PIX_FMT_SGBRG8,
 		.code		= MEDIA_BUS_FMT_SGBRG8_1X8,
-		.depth		= 8,
+		.bpp		= 8,
 	}, {
 		.fourcc		= V4L2_PIX_FMT_SGRBG8,
 		.code		= MEDIA_BUS_FMT_SGRBG8_1X8,
-		.depth		= 8,
+		.bpp		= 8,
 	}, {
 		.fourcc		= V4L2_PIX_FMT_SRGGB8,
 		.code		= MEDIA_BUS_FMT_SRGGB8_1X8,
-		.depth		= 8,
+		.bpp		= 8,
 	}, {
 		.fourcc		= V4L2_PIX_FMT_SBGGR10,
 		.code		= MEDIA_BUS_FMT_SBGGR10_1X10,
-		.depth		= 16,
+		.bpp		= 10,
 	}, {
 		.fourcc		= V4L2_PIX_FMT_SGBRG10,
 		.code		= MEDIA_BUS_FMT_SGBRG10_1X10,
-		.depth		= 16,
+		.bpp		= 10,
 	}, {
 		.fourcc		= V4L2_PIX_FMT_SGRBG10,
 		.code		= MEDIA_BUS_FMT_SGRBG10_1X10,
-		.depth		= 16,
+		.bpp		= 10,
 	}, {
 		.fourcc		= V4L2_PIX_FMT_SRGGB10,
 		.code		= MEDIA_BUS_FMT_SRGGB10_1X10,
-		.depth		= 16,
+		.bpp		= 10,
 	}, {
 		.fourcc		= V4L2_PIX_FMT_SBGGR12,
 		.code		= MEDIA_BUS_FMT_SBGGR12_1X12,
-		.depth		= 16,
+		.bpp		= 12,
 	}, {
 		.fourcc		= V4L2_PIX_FMT_SGBRG12,
 		.code		= MEDIA_BUS_FMT_SGBRG12_1X12,
-		.depth		= 16,
+		.bpp		= 12,
 	}, {
 		.fourcc		= V4L2_PIX_FMT_SGRBG12,
 		.code		= MEDIA_BUS_FMT_SGRBG12_1X12,
-		.depth		= 16,
+		.bpp		= 12,
 	}, {
 		.fourcc		= V4L2_PIX_FMT_SRGGB12,
 		.code		= MEDIA_BUS_FMT_SRGGB12_1X12,
-		.depth		= 16,
+		.bpp		= 12,
 	},
 };
 
@@ -223,20 +224,125 @@
 	int			ini_jiffies;
 };
 
-struct cm_data {
+struct cc_data {
 	void __iomem		*base;
 	struct resource		*res;
 
-	unsigned int		camerrx_control;
-
 	struct platform_device *pdev;
 };
 
-struct cc_data {
-	void __iomem		*base;
-	struct resource		*res;
+/* CTRL_CORE_CAMERRX_CONTROL register field id */
+enum cal_camerarx_field {
+	F_CTRLCLKEN,
+	F_CAMMODE,
+	F_LANEENABLE,
+	F_CSI_MODE,
 
-	struct platform_device *pdev;
+	F_MAX_FIELDS,
+};
+
+struct cal_csi2_phy {
+	struct regmap_field *fields[F_MAX_FIELDS];
+	struct reg_field *base_fields;
+	const int num_lanes;
+};
+
+struct cal_data {
+	const int num_csi2_phy;
+	struct cal_csi2_phy *csi2_phy_core;
+
+	const unsigned int flags;
+};
+
+static struct reg_field dra72x_ctrl_core_csi0_reg_fields[F_MAX_FIELDS] = {
+	[F_CTRLCLKEN] = REG_FIELD(0, 10, 10),
+	[F_CAMMODE] = REG_FIELD(0, 11, 12),
+	[F_LANEENABLE] = REG_FIELD(0, 13, 16),
+	[F_CSI_MODE] = REG_FIELD(0, 17, 17),
+};
+
+static struct reg_field dra72x_ctrl_core_csi1_reg_fields[F_MAX_FIELDS] = {
+	[F_CTRLCLKEN] = REG_FIELD(0, 0, 0),
+	[F_CAMMODE] = REG_FIELD(0, 1, 2),
+	[F_LANEENABLE] = REG_FIELD(0, 3, 4),
+	[F_CSI_MODE] = REG_FIELD(0, 5, 5),
+};
+
+static struct cal_csi2_phy dra72x_cal_csi_phy[] = {
+	{
+		.base_fields = dra72x_ctrl_core_csi0_reg_fields,
+		.num_lanes = 4,
+	},
+	{
+		.base_fields = dra72x_ctrl_core_csi1_reg_fields,
+		.num_lanes = 2,
+	},
+};
+
+static struct cal_data dra72x_cal_data = {
+	.csi2_phy_core = dra72x_cal_csi_phy,
+	.num_csi2_phy = ARRAY_SIZE(dra72x_cal_csi_phy),
+
+	.flags = 0,
+};
+
+static struct cal_data dra72x_es1_cal_data = {
+	.csi2_phy_core = dra72x_cal_csi_phy,
+	.num_csi2_phy = ARRAY_SIZE(dra72x_cal_csi_phy),
+
+	.flags = DRA72_CAL_PRE_ES2_LDO_DISABLE,
+};
+
+static struct reg_field dra76x_ctrl_core_csi0_reg_fields[F_MAX_FIELDS] = {
+	[F_CTRLCLKEN] = REG_FIELD(0, 8, 8),
+	[F_CAMMODE] = REG_FIELD(0, 9, 10),
+	[F_CSI_MODE] = REG_FIELD(0, 11, 11),
+	[F_LANEENABLE] = REG_FIELD(0, 27, 31),
+};
+
+static struct reg_field dra76x_ctrl_core_csi1_reg_fields[F_MAX_FIELDS] = {
+	[F_CTRLCLKEN] = REG_FIELD(0, 0, 0),
+	[F_CAMMODE] = REG_FIELD(0, 1, 2),
+	[F_CSI_MODE] = REG_FIELD(0, 3, 3),
+	[F_LANEENABLE] = REG_FIELD(0, 24, 26),
+};
+
+static struct cal_csi2_phy dra76x_cal_csi_phy[] = {
+	{
+		.base_fields = dra76x_ctrl_core_csi0_reg_fields,
+		.num_lanes = 5,
+	},
+	{
+		.base_fields = dra76x_ctrl_core_csi1_reg_fields,
+		.num_lanes = 3,
+	},
+};
+
+static struct cal_data dra76x_cal_data = {
+	.csi2_phy_core = dra76x_cal_csi_phy,
+	.num_csi2_phy = ARRAY_SIZE(dra76x_cal_csi_phy),
+
+	.flags = 0,
+};
+
+static struct reg_field am654_ctrl_core_csi0_reg_fields[F_MAX_FIELDS] = {
+	[F_CTRLCLKEN] = REG_FIELD(0, 15, 15),
+	[F_CAMMODE] = REG_FIELD(0, 24, 25),
+	[F_LANEENABLE] = REG_FIELD(0, 0, 4),
+};
+
+static struct cal_csi2_phy am654_cal_csi_phy[] = {
+	{
+		.base_fields = am654_ctrl_core_csi0_reg_fields,
+		.num_lanes = 5,
+	},
+};
+
+static struct cal_data am654_cal_data = {
+	.csi2_phy_core = am654_cal_csi_phy,
+	.num_csi2_phy = ARRAY_SIZE(am654_cal_csi_phy),
+
+	.flags = 0,
 };
 
 /*
@@ -250,8 +356,15 @@
 	struct platform_device	*pdev;
 	struct v4l2_device	v4l2_dev;
 
+	/* Controller flags for special cases */
+	unsigned int		flags;
+
+	struct cal_data		*data;
+
 	/* Control Module handle */
-	struct cm_data		*cm;
+	struct regmap		*syscon_camerrx;
+	u32			syscon_camerrx_offset;
+
 	/* Camera Core Module handle */
 	struct cc_data		*cc[CAL_NUM_CSI2_PORTS];
 
@@ -363,73 +476,115 @@
 	*valp = val;
 }
 
-/*
- * Control Module block access
- */
-static struct cm_data *cm_create(struct cal_dev *dev)
+static u32 cal_data_get_phy_max_lanes(struct cal_ctx *ctx)
 {
-	struct platform_device *pdev = dev->pdev;
-	struct cm_data *cm;
+	struct cal_dev *dev = ctx->dev;
+	u32 phy_id = ctx->csi2_port - 1;
 
-	cm = devm_kzalloc(&pdev->dev, sizeof(*cm), GFP_KERNEL);
-	if (!cm)
-		return ERR_PTR(-ENOMEM);
+	return dev->data->csi2_phy_core[phy_id].num_lanes;
+}
+
+static u32 cal_data_get_num_csi2_phy(struct cal_dev *dev)
+{
+	return dev->data->num_csi2_phy;
+}
+
+static int cal_camerarx_regmap_init(struct cal_dev *dev)
+{
+	struct reg_field *field;
+	struct cal_csi2_phy *phy;
+	int i, j;
 
-	cm->res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
-						"camerrx_control");
-	cm->base = devm_ioremap_resource(&pdev->dev, cm->res);
-	if (IS_ERR(cm->base)) {
+	if (!dev->data)
+		return -EINVAL;
+
+	for (i = 0; i < cal_data_get_num_csi2_phy(dev); i++) {
+		phy = &dev->data->csi2_phy_core[i];
+		for (j = 0; j < F_MAX_FIELDS; j++) {
+			field = &phy->base_fields[j];
+			/*
+			 * Here we update the reg offset with the
+			 * value found in DT
+			 */
+			field->reg = dev->syscon_camerrx_offset;
+			phy->fields[j] =
+				devm_regmap_field_alloc(&dev->pdev->dev,
+							dev->syscon_camerrx,
+							*field);
+			if (IS_ERR(phy->fields[j])) {
+				cal_err(dev, "Unable to allocate regmap fields\n");
+				return PTR_ERR(phy->fields[j]);
+			}
+		}
+	}
+	return 0;
+}
+
+static const struct regmap_config cal_regmap_config = {
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = 4,
+};
+
+static struct regmap *cal_get_camerarx_regmap(struct cal_dev *dev)
+{
+	struct platform_device *pdev = dev->pdev;
+	struct regmap *regmap;
+	void __iomem *base;
+	u32 reg_io_width;
+	struct regmap_config r_config = cal_regmap_config;
+	struct resource *res;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+					   "camerrx_control");
+	base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(base)) {
 		cal_err(dev, "failed to ioremap\n");
-		return ERR_CAST(cm->base);
+		return ERR_CAST(base);
 	}
 
 	cal_dbg(1, dev, "ioresource %s at %pa - %pa\n",
-		cm->res->name, &cm->res->start, &cm->res->end);
+		res->name, &res->start, &res->end);
+
+	reg_io_width = 4;
+	r_config.reg_stride = reg_io_width;
+	r_config.val_bits = reg_io_width * 8;
+	r_config.max_register = resource_size(res) - reg_io_width;
+
+	regmap = regmap_init_mmio(NULL, base, &r_config);
+	if (IS_ERR(regmap))
+		pr_err("regmap init failed\n");
 
-	return cm;
+	return regmap;
 }
 
+/*
+ * Control Module CAMERARX block access
+ */
 static void camerarx_phy_enable(struct cal_ctx *ctx)
 {
-	u32 val;
-
-	if (!ctx->dev->cm->base) {
-		ctx_err(ctx, "cm not mapped\n");
-		return;
-	}
-
-	val = reg_read(ctx->dev->cm, CM_CTRL_CORE_CAMERRX_CONTROL);
-	if (ctx->csi2_port == 1) {
-		set_field(&val, 1, CM_CAMERRX_CTRL_CSI0_CTRLCLKEN_MASK);
-		set_field(&val, 0, CM_CAMERRX_CTRL_CSI0_CAMMODE_MASK);
-		/* enable all lanes by default */
-		set_field(&val, 0xf, CM_CAMERRX_CTRL_CSI0_LANEENABLE_MASK);
-		set_field(&val, 1, CM_CAMERRX_CTRL_CSI0_MODE_MASK);
-	} else if (ctx->csi2_port == 2) {
-		set_field(&val, 1, CM_CAMERRX_CTRL_CSI1_CTRLCLKEN_MASK);
-		set_field(&val, 0, CM_CAMERRX_CTRL_CSI1_CAMMODE_MASK);
-		/* enable all lanes by default */
-		set_field(&val, 0x3, CM_CAMERRX_CTRL_CSI1_LANEENABLE_MASK);
-		set_field(&val, 1, CM_CAMERRX_CTRL_CSI1_MODE_MASK);
-	}
-	reg_write(ctx->dev->cm, CM_CTRL_CORE_CAMERRX_CONTROL, val);
+	struct cal_csi2_phy *phy;
+	u32 phy_id = ctx->csi2_port - 1;
+	u32 max_lanes;
+
+	phy = &ctx->dev->data->csi2_phy_core[phy_id];
+	regmap_field_write(phy->fields[F_CAMMODE], 0);
+	/* Always enable all lanes at the phy control level */
+	max_lanes = (1 << cal_data_get_phy_max_lanes(ctx)) - 1;
+	regmap_field_write(phy->fields[F_LANEENABLE], max_lanes);
+	/* F_CSI_MODE is not present on every architecture */
+	if (phy->fields[F_CSI_MODE])
+		regmap_field_write(phy->fields[F_CSI_MODE], 1);
+	regmap_field_write(phy->fields[F_CTRLCLKEN], 1);
 }
 
 static void camerarx_phy_disable(struct cal_ctx *ctx)
 {
-	u32 val;
-
-	if (!ctx->dev->cm->base) {
-		ctx_err(ctx, "cm not mapped\n");
-		return;
-	}
+	struct cal_csi2_phy *phy;
+	u32 phy_id = ctx->csi2_port - 1;
 
-	val = reg_read(ctx->dev->cm, CM_CTRL_CORE_CAMERRX_CONTROL);
-	if (ctx->csi2_port == 1)
-		set_field(&val, 0x0, CM_CAMERRX_CTRL_CSI0_CTRLCLKEN_MASK);
-	else if (ctx->csi2_port == 2)
-		set_field(&val, 0x0, CM_CAMERRX_CTRL_CSI1_CTRLCLKEN_MASK);
-	reg_write(ctx->dev->cm, CM_CTRL_CORE_CAMERRX_CONTROL, val);
+	phy = &ctx->dev->data->csi2_phy_core[phy_id];
+	regmap_field_write(phy->fields[F_CTRLCLKEN], 0);
 }
 
 /*
@@ -478,9 +633,52 @@
 		hwinfo);
 }
 
+/*
+ *   Errata i913: CSI2 LDO Needs to be disabled when module is powered on
+ *
+ *   Enabling CSI2 LDO shorts it to core supply. It is crucial the 2 CSI2
+ *   LDOs on the device are disabled if CSI-2 module is powered on
+ *   (0x4845 B304 | 0x4845 B384 [28:27] = 0x1) or in ULPS (0x4845 B304
+ *   | 0x4845 B384 [28:27] = 0x2) mode. Common concerns include: high
+ *   current draw on the module supply in active mode.
+ *
+ *   Errata does not apply when CSI-2 module is powered off
+ *   (0x4845 B304 | 0x4845 B384 [28:27] = 0x0).
+ *
+ * SW Workaround:
+ *	Set the following register bits to disable the LDO,
+ *	which is essentially CSI2 REG10 bit 6:
+ *
+ *		Core 0:  0x4845 B828 = 0x0000 0040
+ *		Core 1:  0x4845 B928 = 0x0000 0040
+ */
+static void i913_errata(struct cal_dev *dev, unsigned int port)
+{
+	u32 reg10 = reg_read(dev->cc[port], CAL_CSI2_PHY_REG10);
+
+	set_field(&reg10, CAL_CSI2_PHY_REG0_HSCLOCKCONFIG_DISABLE,
+		  CAL_CSI2_PHY_REG10_I933_LDO_DISABLE_MASK);
+
+	cal_dbg(1, dev, "CSI2_%d_REG10 = 0x%08x\n", port, reg10);
+	reg_write(dev->cc[port], CAL_CSI2_PHY_REG10, reg10);
+}
+
 static inline int cal_runtime_get(struct cal_dev *dev)
 {
-	return pm_runtime_get_sync(&dev->pdev->dev);
+	int r;
+
+	r = pm_runtime_get_sync(&dev->pdev->dev);
+
+	if (dev->flags & DRA72_CAL_PRE_ES2_LDO_DISABLE) {
+		/*
+		 * Apply errata on both paort eveytime we (re-)enable
+		 * the clock
+		 */
+		i913_errata(dev, 0);
+		i913_errata(dev, 1);
+	}
+
+	return r;
 }
 
 static inline void cal_runtime_put(struct cal_dev *dev)
@@ -512,12 +710,6 @@
 			       resource_size(dev->ctx[1]->cc->res),
 			       false);
 	}
-
-	cal_info(dev, "CAMERRX_Control Registers @ %pa:\n",
-		 &dev->cm->res->start);
-	print_hex_dump(KERN_INFO, "", DUMP_PREFIX_OFFSET, 16, 4,
-		       (__force const void *)dev->cm->base,
-		       resource_size(dev->cm->res), false);
 }
 
 /*
@@ -555,29 +747,76 @@
 	reg_write(ctx->dev, CAL_CSI2_VC_IRQENABLE(1), 0);
 }
 
-static void csi2_init(struct cal_ctx *ctx)
+static void csi2_phy_config(struct cal_ctx *ctx);
+
+static void csi2_phy_init(struct cal_ctx *ctx)
 {
 	int i;
 	u32 val;
 
+	/* Steps
+	 *  1. Configure D-PHY mode and enable required lanes
+	 *  2. Reset complex IO - Wait for completion of reset
+	 *          Note if the external sensor is not sending byte clock,
+	 *          the reset will timeout
+	 *  3 Program Stop States
+	 *      A. Program THS_TERM, THS_SETTLE, etc... Timings parameters
+	 *              in terms of DDR clock periods
+	 *      B. Enable stop state transition timeouts
+	 *  4.Force FORCERXMODE
+	 *      D. Enable pull down using pad control
+	 *      E. Power up PHY
+	 *      F. Wait for power up completion
+	 *      G. Wait for all enabled lane to reach stop state
+	 *      H. Disable pull down using pad control
+	 */
+
+	/* 1. Configure D-PHY mode and enable required lanes */
+	camerarx_phy_enable(ctx);
+
+	/* 2. Reset complex IO - Do not wait for reset completion */
+	val = reg_read(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port));
+	set_field(&val, CAL_CSI2_COMPLEXIO_CFG_RESET_CTRL_OPERATIONAL,
+		  CAL_CSI2_COMPLEXIO_CFG_RESET_CTRL_MASK);
+	reg_write(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port), val);
+	ctx_dbg(3, ctx, "CAL_CSI2_COMPLEXIO_CFG(%d) = 0x%08x De-assert Complex IO Reset\n",
+		ctx->csi2_port,
+		reg_read(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port)));
+
+	/* Dummy read to allow SCP to complete */
+	val = reg_read(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port));
+
+	/* 3.A. Program Phy Timing Parameters */
+	csi2_phy_config(ctx);
+
+	/* 3.B. Program Stop States */
 	val = reg_read(ctx->dev, CAL_CSI2_TIMING(ctx->csi2_port));
 	set_field(&val, CAL_GEN_ENABLE,
-		  CAL_CSI2_TIMING_FORCE_RX_MODE_IO1_MASK);
-	set_field(&val, CAL_GEN_ENABLE,
 		  CAL_CSI2_TIMING_STOP_STATE_X16_IO1_MASK);
 	set_field(&val, CAL_GEN_DISABLE,
 		  CAL_CSI2_TIMING_STOP_STATE_X4_IO1_MASK);
 	set_field(&val, 407, CAL_CSI2_TIMING_STOP_STATE_COUNTER_IO1_MASK);
 	reg_write(ctx->dev, CAL_CSI2_TIMING(ctx->csi2_port), val);
-	ctx_dbg(3, ctx, "CAL_CSI2_TIMING(%d) = 0x%08x\n", ctx->csi2_port,
+	ctx_dbg(3, ctx, "CAL_CSI2_TIMING(%d) = 0x%08x Stop States\n",
+		ctx->csi2_port,
 		reg_read(ctx->dev, CAL_CSI2_TIMING(ctx->csi2_port)));
 
+	/* 4. Force FORCERXMODE */
+	val = reg_read(ctx->dev, CAL_CSI2_TIMING(ctx->csi2_port));
+	set_field(&val, CAL_GEN_ENABLE,
+		  CAL_CSI2_TIMING_FORCE_RX_MODE_IO1_MASK);
+	reg_write(ctx->dev, CAL_CSI2_TIMING(ctx->csi2_port), val);
+	ctx_dbg(3, ctx, "CAL_CSI2_TIMING(%d) = 0x%08x Force RXMODE\n",
+		ctx->csi2_port,
+		reg_read(ctx->dev, CAL_CSI2_TIMING(ctx->csi2_port)));
+
+	/* E. Power up the PHY using the complex IO */
 	val = reg_read(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port));
-	set_field(&val, CAL_CSI2_COMPLEXIO_CFG_RESET_CTRL_OPERATIONAL,
-		  CAL_CSI2_COMPLEXIO_CFG_RESET_CTRL_MASK);
 	set_field(&val, CAL_CSI2_COMPLEXIO_CFG_PWR_CMD_STATE_ON,
 		  CAL_CSI2_COMPLEXIO_CFG_PWR_CMD_MASK);
 	reg_write(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port), val);
+
+	/* F. Wait for power up completion */
 	for (i = 0; i < 10; i++) {
 		if (reg_read_field(ctx->dev,
 				   CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port),
@@ -586,18 +825,104 @@
 			break;
 		usleep_range(1000, 1100);
 	}
-	ctx_dbg(3, ctx, "CAL_CSI2_COMPLEXIO_CFG(%d) = 0x%08x\n", ctx->csi2_port,
-		reg_read(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port)));
+	ctx_dbg(3, ctx, "CAL_CSI2_COMPLEXIO_CFG(%d) = 0x%08x Powered UP %s\n",
+		ctx->csi2_port,
+		reg_read(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port)),
+		(i >= 10) ? "(timeout)" : "");
+}
 
-	val = reg_read(ctx->dev, CAL_CTRL);
-	set_field(&val, CAL_CTRL_BURSTSIZE_BURST128, CAL_CTRL_BURSTSIZE_MASK);
-	set_field(&val, 0xF, CAL_CTRL_TAGCNT_MASK);
-	set_field(&val, CAL_CTRL_POSTED_WRITES_NONPOSTED,
-		  CAL_CTRL_POSTED_WRITES_MASK);
-	set_field(&val, 0xFF, CAL_CTRL_MFLAGL_MASK);
-	set_field(&val, 0xFF, CAL_CTRL_MFLAGH_MASK);
-	reg_write(ctx->dev, CAL_CTRL, val);
-	ctx_dbg(3, ctx, "CAL_CTRL = 0x%08x\n", reg_read(ctx->dev, CAL_CTRL));
+static void csi2_wait_for_phy(struct cal_ctx *ctx)
+{
+	int i;
+
+	/* Steps
+	 *  2. Wait for completion of reset
+	 *          Note if the external sensor is not sending byte clock,
+	 *          the reset will timeout
+	 *  4.Force FORCERXMODE
+	 *      G. Wait for all enabled lane to reach stop state
+	 *      H. Disable pull down using pad control
+	 */
+
+	/* 2. Wait for reset completion */
+	for (i = 0; i < 250; i++) {
+		if (reg_read_field(ctx->dev,
+				   CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port),
+				   CAL_CSI2_COMPLEXIO_CFG_RESET_DONE_MASK) ==
+		    CAL_CSI2_COMPLEXIO_CFG_RESET_DONE_RESETCOMPLETED)
+			break;
+		usleep_range(1000, 1100);
+	}
+	ctx_dbg(3, ctx, "CAL_CSI2_COMPLEXIO_CFG(%d) = 0x%08x Complex IO Reset Done (%d) %s\n",
+		ctx->csi2_port,
+		reg_read(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port)), i,
+		(i >= 250) ? "(timeout)" : "");
+
+	/* 4. G. Wait for all enabled lane to reach stop state */
+	for (i = 0; i < 10; i++) {
+		if (reg_read_field(ctx->dev,
+				   CAL_CSI2_TIMING(ctx->csi2_port),
+				   CAL_CSI2_TIMING_FORCE_RX_MODE_IO1_MASK) ==
+		    CAL_GEN_DISABLE)
+			break;
+		usleep_range(1000, 1100);
+	}
+	ctx_dbg(3, ctx, "CAL_CSI2_TIMING(%d) = 0x%08x Stop State Reached %s\n",
+		ctx->csi2_port,
+		reg_read(ctx->dev, CAL_CSI2_TIMING(ctx->csi2_port)),
+		(i >= 10) ? "(timeout)" : "");
+
+	ctx_dbg(1, ctx, "CSI2_%d_REG1 = 0x%08x (Bit(31,28) should be set!)\n",
+		(ctx->csi2_port - 1), reg_read(ctx->cc, CAL_CSI2_PHY_REG1));
+}
+
+static void csi2_phy_deinit(struct cal_ctx *ctx)
+{
+	int i;
+	u32 val;
+
+	/* Power down the PHY using the complex IO */
+	val = reg_read(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port));
+	set_field(&val, CAL_CSI2_COMPLEXIO_CFG_PWR_CMD_STATE_OFF,
+		  CAL_CSI2_COMPLEXIO_CFG_PWR_CMD_MASK);
+	reg_write(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port), val);
+
+	/* Wait for power down completion */
+	for (i = 0; i < 10; i++) {
+		if (reg_read_field(ctx->dev,
+				   CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port),
+				   CAL_CSI2_COMPLEXIO_CFG_PWR_STATUS_MASK) ==
+		    CAL_CSI2_COMPLEXIO_CFG_PWR_STATUS_STATE_OFF)
+			break;
+		usleep_range(1000, 1100);
+	}
+	ctx_dbg(3, ctx, "CAL_CSI2_COMPLEXIO_CFG(%d) = 0x%08x Powered Down %s\n",
+		ctx->csi2_port,
+		reg_read(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port)),
+		(i >= 10) ? "(timeout)" : "");
+
+	/* Assert Comple IO Reset */
+	val = reg_read(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port));
+	set_field(&val, CAL_CSI2_COMPLEXIO_CFG_RESET_CTRL,
+		  CAL_CSI2_COMPLEXIO_CFG_RESET_CTRL_MASK);
+	reg_write(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port), val);
+
+	/* Wait for power down completion */
+	for (i = 0; i < 10; i++) {
+		if (reg_read_field(ctx->dev,
+				   CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port),
+				   CAL_CSI2_COMPLEXIO_CFG_RESET_DONE_MASK) ==
+		    CAL_CSI2_COMPLEXIO_CFG_RESET_DONE_RESETONGOING)
+			break;
+		usleep_range(1000, 1100);
+	}
+	ctx_dbg(3, ctx, "CAL_CSI2_COMPLEXIO_CFG(%d) = 0x%08x Complex IO in Reset (%d) %s\n",
+		ctx->csi2_port,
+		reg_read(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port)), i,
+		(i >= 10) ? "(timeout)" : "");
+
+	/* Disable the phy */
+	camerarx_phy_disable(ctx);
 }
 
 static void csi2_lane_config(struct cal_ctx *ctx)
@@ -669,13 +994,45 @@
 
 static void pix_proc_config(struct cal_ctx *ctx)
 {
-	u32 val;
+	u32 val, extract, pack;
+
+	switch (ctx->fmt->bpp) {
+	default:
+		/*
+		 * If you see this warning then it means that you added
+		 * some new entry in the cal_formats[] array with a different
+		 * bit per pixel values then the one supported below.
+		 * Either add support for the new bpp value below or adjust
+		 * the new entry to use one of the value below.
+		 *
+		 * Instead of failing here just use 8 bpp as a default.
+		 */
+		dev_warn_once(&ctx->dev->pdev->dev,
+			      "%s:%d:%s: bpp:%d unsupported! Overwritten with 8.\n",
+			      __FILE__, __LINE__, __func__, ctx->fmt->bpp);
+	case 8:
+		extract = CAL_PIX_PROC_EXTRACT_B8;
+		pack = CAL_PIX_PROC_PACK_B8;
+		break;
+	case 10:
+		extract = CAL_PIX_PROC_EXTRACT_B10_MIPI;
+		pack = CAL_PIX_PROC_PACK_B16;
+		break;
+	case 12:
+		extract = CAL_PIX_PROC_EXTRACT_B12_MIPI;
+		pack = CAL_PIX_PROC_PACK_B16;
+		break;
+	case 16:
+		extract = CAL_PIX_PROC_EXTRACT_B16_LE;
+		pack = CAL_PIX_PROC_PACK_B16;
+		break;
+	}
 
 	val = reg_read(ctx->dev, CAL_PIX_PROC(ctx->csi2_port));
-	set_field(&val, CAL_PIX_PROC_EXTRACT_B8, CAL_PIX_PROC_EXTRACT_MASK);
+	set_field(&val, extract, CAL_PIX_PROC_EXTRACT_MASK);
 	set_field(&val, CAL_PIX_PROC_DPCMD_BYPASS, CAL_PIX_PROC_DPCMD_MASK);
 	set_field(&val, CAL_PIX_PROC_DPCME_BYPASS, CAL_PIX_PROC_DPCME_MASK);
-	set_field(&val, CAL_PIX_PROC_PACK_B8, CAL_PIX_PROC_PACK_MASK);
+	set_field(&val, pack, CAL_PIX_PROC_PACK_MASK);
 	set_field(&val, ctx->csi2_port, CAL_PIX_PROC_CPORT_MASK);
 	set_field(&val, CAL_GEN_ENABLE, CAL_PIX_PROC_EN_MASK);
 	reg_write(ctx->dev, CAL_PIX_PROC(ctx->csi2_port), val);
@@ -684,12 +1041,13 @@
 }
 
 static void cal_wr_dma_config(struct cal_ctx *ctx,
-			      unsigned int width)
+			      unsigned int width, unsigned int height)
 {
 	u32 val;
 
 	val = reg_read(ctx->dev, CAL_WR_DMA_CTRL(ctx->csi2_port));
 	set_field(&val, ctx->csi2_port, CAL_WR_DMA_CTRL_CPORT_MASK);
+	set_field(&val, height, CAL_WR_DMA_CTRL_YSIZE_MASK);
 	set_field(&val, CAL_WR_DMA_CTRL_DTAG_PIX_DAT,
 		  CAL_WR_DMA_CTRL_DTAG_MASK);
 	set_field(&val, CAL_WR_DMA_CTRL_MODE_CONST,
@@ -724,6 +1082,16 @@
 	reg_write(ctx->dev, CAL_WR_DMA_XSIZE(ctx->csi2_port), val);
 	ctx_dbg(3, ctx, "CAL_WR_DMA_XSIZE(%d) = 0x%08x\n", ctx->csi2_port,
 		reg_read(ctx->dev, CAL_WR_DMA_XSIZE(ctx->csi2_port)));
+
+	val = reg_read(ctx->dev, CAL_CTRL);
+	set_field(&val, CAL_CTRL_BURSTSIZE_BURST128, CAL_CTRL_BURSTSIZE_MASK);
+	set_field(&val, 0xF, CAL_CTRL_TAGCNT_MASK);
+	set_field(&val, CAL_CTRL_POSTED_WRITES_NONPOSTED,
+		  CAL_CTRL_POSTED_WRITES_MASK);
+	set_field(&val, 0xFF, CAL_CTRL_MFLAGL_MASK);
+	set_field(&val, 0xFF, CAL_CTRL_MFLAGH_MASK);
+	reg_write(ctx->dev, CAL_CTRL, val);
+	ctx_dbg(3, ctx, "CAL_CTRL = 0x%08x\n", reg_read(ctx->dev, CAL_CTRL));
 }
 
 static void cal_wr_dma_addr(struct cal_ctx *ctx, unsigned int dmaaddr)
@@ -737,41 +1105,28 @@
 #define TCLK_TERM	0
 #define TCLK_MISS	1
 #define TCLK_SETTLE	14
-#define THS_SETTLE	15
 
 static void csi2_phy_config(struct cal_ctx *ctx)
 {
 	unsigned int reg0, reg1;
 	unsigned int ths_term, ths_settle;
-	unsigned int ddrclkperiod_us;
+	unsigned int csi2_ddrclk_khz;
+	struct v4l2_fwnode_bus_mipi_csi2 *mipi_csi2 =
+			&ctx->endpoint.bus.mipi_csi2;
+	u32 num_lanes = mipi_csi2->num_data_lanes;
 
-	/*
-	 * THS_TERM: Programmed value = floor(20 ns/DDRClk period) - 2.
-	 */
-	ddrclkperiod_us = ctx->external_rate / 2000000;
-	ddrclkperiod_us = 1000000 / ddrclkperiod_us;
-	ctx_dbg(1, ctx, "ddrclkperiod_us: %d\n", ddrclkperiod_us);
+	/* DPHY timing configuration */
+	/* CSI-2 is DDR and we only count used lanes. */
+	csi2_ddrclk_khz = ctx->external_rate / 1000
+		/ (2 * num_lanes) * ctx->fmt->bpp;
+	ctx_dbg(1, ctx, "csi2_ddrclk_khz: %d\n", csi2_ddrclk_khz);
 
-	ths_term = 20000 / ddrclkperiod_us;
-	ths_term = (ths_term >= 2) ? ths_term - 2 : ths_term;
+	/* THS_TERM: Programmed value = floor(20 ns/DDRClk period) */
+	ths_term = 20 * csi2_ddrclk_khz / 1000000;
 	ctx_dbg(1, ctx, "ths_term: %d (0x%02x)\n", ths_term, ths_term);
 
-	/*
-	 * THS_SETTLE: Programmed value = floor(176.3 ns/CtrlClk period) - 1.
-	 *	Since CtrlClk is fixed at 96Mhz then we get
-	 *	ths_settle = floor(176.3 / 10.416) - 1 = 15
-	 * If we ever switch to a dynamic clock then this code might be useful
-	 *
-	 * unsigned int ctrlclkperiod_us;
-	 * ctrlclkperiod_us = 96000000 / 1000000;
-	 * ctrlclkperiod_us = 1000000 / ctrlclkperiod_us;
-	 * ctx_dbg(1, ctx, "ctrlclkperiod_us: %d\n", ctrlclkperiod_us);
-
-	 * ths_settle = 176300  / ctrlclkperiod_us;
-	 * ths_settle = (ths_settle > 1) ? ths_settle - 1 : ths_settle;
-	 */
-
-	ths_settle = THS_SETTLE;
+	/* THS_SETTLE: Programmed value = floor(105 ns/DDRClk period) + 4 */
+	ths_settle = (105 * csi2_ddrclk_khz / 1000000) + 4;
 	ctx_dbg(1, ctx, "ths_settle: %d (0x%02x)\n", ths_settle, ths_settle);
 
 	reg0 = reg_read(ctx->cc, CAL_CSI2_PHY_REG0);
@@ -986,15 +1341,25 @@
 				const struct cal_fmt *fmt,
 				struct v4l2_format *f)
 {
+	u32 bpl, max_width;
+
 	if (!fmt) {
 		ctx_dbg(3, ctx, "No cal_fmt provided!\n");
 		return -EINVAL;
 	}
 
-	v4l_bound_align_image(&f->fmt.pix.width, 48, MAX_WIDTH, 2,
-			      &f->fmt.pix.height, 32, MAX_HEIGHT, 0, 0);
-	f->fmt.pix.bytesperline = bytes_per_line(f->fmt.pix.width,
-						 fmt->depth >> 3);
+	/*
+	 * Maximum width is bound by the DMA max width in bytes.
+	 * We need to recalculate the actual maxi width depending on the
+	 * number of bytes per pixels required.
+	 */
+	max_width = MAX_WIDTH_BYTES / (ALIGN(fmt->bpp, 8) >> 3);
+	v4l_bound_align_image(&f->fmt.pix.width, 48, max_width, 2,
+			      &f->fmt.pix.height, 32, MAX_HEIGHT_LINES, 0, 0);
+
+	bpl = (f->fmt.pix.width * ALIGN(fmt->bpp, 8)) >> 3;
+	f->fmt.pix.bytesperline = ALIGN(bpl, 16);
+
 	f->fmt.pix.sizeimage = f->fmt.pix.height *
 			       f->fmt.pix.bytesperline;
 
@@ -1306,36 +1671,50 @@
 	if (ret < 0)
 		goto err;
 
+	ret = v4l2_subdev_call(ctx->sensor, core, s_power, 1);
+	if (ret < 0 && ret != -ENOIOCTLCMD && ret != -ENODEV) {
+		ctx_err(ctx, "power on failed in subdev\n");
+		goto err;
+	}
+
 	cal_runtime_get(ctx->dev);
 
-	enable_irqs(ctx);
-	camerarx_phy_enable(ctx);
-	csi2_init(ctx);
-	csi2_phy_config(ctx);
-	csi2_lane_config(ctx);
 	csi2_ctx_config(ctx);
 	pix_proc_config(ctx);
-	cal_wr_dma_config(ctx, ctx->v_fmt.fmt.pix.bytesperline);
-	cal_wr_dma_addr(ctx, addr);
-	csi2_ppi_enable(ctx);
+	cal_wr_dma_config(ctx, ctx->v_fmt.fmt.pix.bytesperline,
+			  ctx->v_fmt.fmt.pix.height);
+	csi2_lane_config(ctx);
+
+	enable_irqs(ctx);
+	csi2_phy_init(ctx);
 
 	ret = v4l2_subdev_call(ctx->sensor, video, s_stream, 1);
 	if (ret) {
+		v4l2_subdev_call(ctx->sensor, core, s_power, 0);
 		ctx_err(ctx, "stream on failed in subdev\n");
 		cal_runtime_put(ctx->dev);
 		goto err;
 	}
 
+	csi2_wait_for_phy(ctx);
+	cal_wr_dma_addr(ctx, addr);
+	csi2_ppi_enable(ctx);
+
 	if (debug >= 4)
 		cal_quickdump_regs(ctx->dev);
 
 	return 0;
 
 err:
+	spin_lock_irqsave(&ctx->slock, flags);
+	vb2_buffer_done(&ctx->cur_frm->vb.vb2_buf, VB2_BUF_STATE_QUEUED);
+	ctx->cur_frm = NULL;
+	ctx->next_frm = NULL;
 	list_for_each_entry_safe(buf, tmp, &dma_q->active, list) {
 		list_del(&buf->list);
 		vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_QUEUED);
 	}
+	spin_unlock_irqrestore(&ctx->slock, flags);
 	return ret;
 }
 
@@ -1345,12 +1724,18 @@
 	struct cal_dmaqueue *dma_q = &ctx->vidq;
 	struct cal_buffer *buf, *tmp;
 	unsigned long flags;
+	int ret;
+
+	csi2_ppi_disable(ctx);
+	disable_irqs(ctx);
+	csi2_phy_deinit(ctx);
 
 	if (v4l2_subdev_call(ctx->sensor, video, s_stream, 0))
 		ctx_err(ctx, "stream off failed in subdev\n");
 
-	csi2_ppi_disable(ctx);
-	disable_irqs(ctx);
+	ret = v4l2_subdev_call(ctx->sensor, core, s_power, 0);
+	if (ret < 0 && ret != -ENOIOCTLCMD && ret != -ENODEV)
+		ctx_err(ctx, "power off failed in subdev\n");
 
 	/* Release all active buffers */
 	spin_lock_irqsave(&ctx->slock, flags);
@@ -1406,6 +1791,7 @@
 	.vidioc_querybuf      = vb2_ioctl_querybuf,
 	.vidioc_qbuf          = vb2_ioctl_qbuf,
 	.vidioc_dqbuf         = vb2_ioctl_dqbuf,
+	.vidioc_expbuf        = vb2_ioctl_expbuf,
 	.vidioc_enum_input    = cal_enum_input,
 	.vidioc_g_input       = cal_g_input,
 	.vidioc_s_input       = cal_s_input,
@@ -1807,9 +2193,15 @@
 	return NULL;
 }
 
+static const struct of_device_id cal_of_match[];
+
 static int cal_probe(struct platform_device *pdev)
 {
 	struct cal_dev *dev;
+	const struct of_device_id *match;
+	struct device_node *parent = pdev->dev.of_node;
+	struct regmap *syscon_camerrx = NULL;
+	u32 syscon_camerrx_offset = 0;
 	int ret;
 	int irq;
 
@@ -1817,6 +2209,18 @@
 	if (!dev)
 		return -ENOMEM;
 
+	match = of_match_device(of_match_ptr(cal_of_match), &pdev->dev);
+	if (!match)
+		return -ENODEV;
+
+	if (match->data) {
+		dev->data = (struct cal_data *)match->data;
+		dev->flags = dev->data->flags;
+	} else {
+		dev_err(&pdev->dev, "Could not get feature data based on compatible version\n");
+		return -ENODEV;
+	}
+
 	/* set pseudo v4l2 device name so we can use v4l2_printk */
 	strlcpy(dev->v4l2_dev.name, CAL_MODULE_NAME,
 		sizeof(dev->v4l2_dev.name));
@@ -1824,6 +2228,43 @@
 	/* save pdev pointer */
 	dev->pdev = pdev;
 
+	if (parent && of_property_read_bool(parent, "syscon-camerrx")) {
+		syscon_camerrx =
+			syscon_regmap_lookup_by_phandle(parent,
+							"syscon-camerrx");
+		if (IS_ERR(syscon_camerrx)) {
+			dev_err(&pdev->dev, "failed to get syscon-camerrx regmap\n");
+			return PTR_ERR(syscon_camerrx);
+		}
+
+		if (of_property_read_u32_index(parent, "syscon-camerrx", 1,
+					       &syscon_camerrx_offset)) {
+			dev_err(&pdev->dev, "failed to get syscon-camerrx offset\n");
+			return -EINVAL;
+		}
+	} else {
+		/*
+		 * Backward DTS compatibility.
+		 * If syscon entry is not present then check if the
+		 * camerrx_control resource is present.
+		 */
+		syscon_camerrx = cal_get_camerarx_regmap(dev);
+		if (IS_ERR(syscon_camerrx)) {
+			dev_err(&pdev->dev, "failed to get camerrx_control regmap\n");
+			return PTR_ERR(syscon_camerrx);
+		}
+		/* In this case the base already point to the direct
+		 * CM register so no need for an offset
+		 */
+		syscon_camerrx_offset = 0;
+	}
+
+	dev->syscon_camerrx = syscon_camerrx;
+	dev->syscon_camerrx_offset = syscon_camerrx_offset;
+	ret = cal_camerarx_regmap_init(dev);
+	if (ret)
+		return ret;
+
 	dev->res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
 						"cal_top");
 	dev->base = devm_ioremap_resource(&pdev->dev, dev->res);
@@ -1842,23 +2283,25 @@
 
 	platform_set_drvdata(pdev, dev);
 
-	dev->cm = cm_create(dev);
-	if (IS_ERR(dev->cm))
-		return PTR_ERR(dev->cm);
-
 	dev->cc[0] = cc_create(dev, 0);
 	if (IS_ERR(dev->cc[0]))
 		return PTR_ERR(dev->cc[0]);
 
-	dev->cc[1] = cc_create(dev, 1);
-	if (IS_ERR(dev->cc[1]))
-		return PTR_ERR(dev->cc[1]);
+	if (cal_data_get_num_csi2_phy(dev) > 1) {
+		dev->cc[1] = cc_create(dev, 1);
+		if (IS_ERR(dev->cc[1]))
+			return PTR_ERR(dev->cc[1]);
+	} else {
+		dev->cc[1] = NULL;
+		cal_info(dev, "One phy/port only.\n");
+	}
 
 	dev->ctx[0] = NULL;
 	dev->ctx[1] = NULL;
 
 	dev->ctx[0] = cal_create_instance(dev, 0);
-	dev->ctx[1] = cal_create_instance(dev, 1);
+	if (cal_data_get_num_csi2_phy(dev) > 1)
+		dev->ctx[1] = cal_create_instance(dev, 1);
 	if (!dev->ctx[0] && !dev->ctx[1]) {
 		cal_err(dev, "Neither port is configured, no point in staying up\n");
 		return -ENODEV;
@@ -1914,7 +2357,22 @@
 
 #if defined(CONFIG_OF)
 static const struct of_device_id cal_of_match[] = {
-	{ .compatible = "ti,dra72-cal", },
+	{
+		.compatible = "ti,dra72-cal",
+		.data = (void *)&dra72x_cal_data,
+	},
+	{
+		.compatible = "ti,dra72-pre-es2-cal",
+		.data = (void *)&dra72x_es1_cal_data,
+	},
+	{
+		.compatible = "ti,dra76-cal",
+		.data = (void *)&dra76x_cal_data,
+	},
+	{
+		.compatible = "ti,am654-cal",
+		.data = (void *)&am654_cal_data,
+	},
 	{},
 };
 MODULE_DEVICE_TABLE(of, cal_of_match);
diff -Naur --no-dereference a/drivers/media/platform/ti-vpe/cal_regs.h b/drivers/media/platform/ti-vpe/cal_regs.h
--- a/drivers/media/platform/ti-vpe/cal_regs.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/media/platform/ti-vpe/cal_regs.h	2020-06-19 21:00:31.416227715 -0400
@@ -13,6 +13,30 @@
 #ifndef __TI_CAL_REGS_H
 #define __TI_CAL_REGS_H
 
+/*
+ * struct cal_dev.flags possibilities
+ *
+ * DRA72_CAL_PRE_ES2_LDO_DISABLE:
+ *   Errata i913: CSI2 LDO Needs to be disabled when module is powered on
+ *
+ *   Enabling CSI2 LDO shorts it to core supply. It is crucial the 2 CSI2
+ *   LDOs on the device are disabled if CSI-2 module is powered on
+ *   (0x4845 B304 | 0x4845 B384 [28:27] = 0x1) or in ULPS (0x4845 B304
+ *   | 0x4845 B384 [28:27] = 0x2) mode. Common concerns include: high
+ *   current draw on the module supply in active mode.
+ *
+ *   Errata does not apply when CSI-2 module is powered off
+ *   (0x4845 B304 | 0x4845 B384 [28:27] = 0x0).
+ *
+ * SW Workaround:
+ *	Set the following register bits to disable the LDO,
+ *	which is essentially CSI2 REG10 bit 6:
+ *
+ *		Core 0:  0x4845 B828 = 0x0000 0040
+ *		Core 1:  0x4845 B928 = 0x0000 0040
+ */
+#define DRA72_CAL_PRE_ES2_LDO_DISABLE BIT(0)
+
 #define CAL_NUM_CSI2_PORTS		2
 
 /* CAL register offsets */
@@ -74,6 +98,7 @@
 #define CAL_CSI2_PHY_REG0		0x000
 #define CAL_CSI2_PHY_REG1		0x004
 #define CAL_CSI2_PHY_REG2		0x008
+#define CAL_CSI2_PHY_REG10		0x028
 
 /* CAL Control Module Core Camerrx Control register offsets */
 #define CM_CTRL_CORE_CAMERRX_CONTROL	0x000
@@ -461,6 +486,8 @@
 #define CAL_CSI2_PHY_REG1_CLOCK_MISS_DETECTOR_STATUS_SUCCESS		0
 #define CAL_CSI2_PHY_REG1_RESET_DONE_STATUS_MASK		GENMASK(29, 28)
 
+#define CAL_CSI2_PHY_REG10_I933_LDO_DISABLE_MASK		BIT_MASK(6)
+
 #define CAL_CSI2_PHY_REG2_CCP2_SYNC_PATTERN_MASK		GENMASK(23, 0)
 #define CAL_CSI2_PHY_REG2_TRIGGER_CMD_RXTRIGESC3_MASK		GENMASK(25, 24)
 #define CAL_CSI2_PHY_REG2_TRIGGER_CMD_RXTRIGESC2_MASK		GENMASK(27, 26)
diff -Naur --no-dereference a/drivers/media/platform/ti-vpe/Makefile b/drivers/media/platform/ti-vpe/Makefile
--- a/drivers/media/platform/ti-vpe/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/media/platform/ti-vpe/Makefile	2020-06-19 21:00:31.416227715 -0400
@@ -3,11 +3,13 @@
 obj-$(CONFIG_VIDEO_TI_VPDMA) += ti-vpdma.o
 obj-$(CONFIG_VIDEO_TI_SC) += ti-sc.o
 obj-$(CONFIG_VIDEO_TI_CSC) += ti-csc.o
+obj-$(CONFIG_VIDEO_TI_VIP) += ti-vip.o
 
 ti-vpe-y := vpe.o
 ti-vpdma-y := vpdma.o
 ti-sc-y := sc.o
 ti-csc-y := csc.o
+ti-vip-y := vip.o
 
 ccflags-$(CONFIG_VIDEO_TI_VPE_DEBUG) += -DDEBUG
 
diff -Naur --no-dereference a/drivers/media/platform/ti-vpe/vip.c b/drivers/media/platform/ti-vpe/vip.c
--- a/drivers/media/platform/ti-vpe/vip.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/media/platform/ti-vpe/vip.c	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,3968 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * TI VIP capture driver
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated -  http://www.ti.com/
+ * David Griego, <dagriego@biglakesoftware.com>
+ * Dale Farnsworth, <dale@farnsworth.org>
+ * Nikhil Devshatwar, <nikhil.nd@ti.com>
+ * Benoit Parrot, <bparrot@ti.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/err.h>
+#include <linux/fs.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/ioctl.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/workqueue.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/mfd/syscon.h>
+#include <linux/regmap.h>
+
+#include <linux/pinctrl/consumer.h>
+#include <linux/of_device.h>
+#include <linux/of_graph.h>
+
+#include <media/v4l2-fwnode.h>
+#include <media/v4l2-async.h>
+
+#include "vip.h"
+
+#define VIP_MODULE_NAME "vip"
+#define VIP_INPUT_NAME "Vin0"
+
+static int debug;
+module_param(debug, int, 0644);
+MODULE_PARM_DESC(debug, "debug level (0-8)");
+
+/*
+ * Minimum and maximum frame sizes
+ */
+#define MIN_W		128
+#define MIN_H		128
+#define MAX_W		2048
+#define MAX_H		1536
+
+/*
+ * Required alignments
+ */
+#define S_ALIGN		0 /* multiple of 1 */
+#define H_ALIGN		1 /* multiple of 2 */
+#define W_ALIGN		1 /* multiple of 2 */
+#define L_ALIGN		7 /* multiple of 128, line stride, 16 bytes */
+
+/*
+ * Need a descriptor entry for each of up to 15 outputs,
+ * and up to 2 control transfers.
+ */
+#define VIP_DESC_LIST_SIZE	(17 * sizeof(struct vpdma_dtd))
+
+#define vip_dbg(level, dev, fmt, arg...)	\
+		v4l2_dbg(level, debug, dev, fmt, ##arg)
+#define vip_err(dev, fmt, arg...)	\
+		v4l2_err(dev, fmt, ##arg)
+#define vip_info(dev, fmt, arg...)	\
+		v4l2_info(dev, fmt, ##arg)
+
+#define CTRL_CORE_SMA_SW_1      0x534
+/*
+ * The srce_info structure contains per-srce data.
+ */
+struct vip_srce_info {
+	u8	base_channel;	/* the VPDMA channel nummber */
+	u8	vb_index;	/* input frame f, f-1, f-2 index */
+	u8	vb_part;	/* identifies section of co-planar formats */
+};
+
+#define VIP_VPDMA_FIFO_SIZE	2
+#define VIP_DROPQ_SIZE		3
+
+/*
+ * Define indices into the srce_info tables
+ */
+
+#define VIP_SRCE_MULT_PORT		0
+#define VIP_SRCE_MULT_ANC		1
+#define VIP_SRCE_LUMA		2
+#define VIP_SRCE_CHROMA		3
+#define VIP_SRCE_RGB		4
+
+static struct vip_srce_info srce_info[5] = {
+	[VIP_SRCE_MULT_PORT] = {
+		.base_channel	= VIP1_CHAN_NUM_MULT_PORT_A_SRC0,
+		.vb_index	= 0,
+		.vb_part	= VIP_CHROMA,
+	},
+	[VIP_SRCE_MULT_ANC] = {
+		.base_channel	= VIP1_CHAN_NUM_MULT_ANC_A_SRC0,
+		.vb_index	= 0,
+		.vb_part	= VIP_LUMA,
+	},
+	[VIP_SRCE_LUMA] = {
+		.base_channel	= VIP1_CHAN_NUM_PORT_A_LUMA,
+		.vb_index	= 1,
+		.vb_part	= VIP_LUMA,
+	},
+	[VIP_SRCE_CHROMA] = {
+		.base_channel	= VIP1_CHAN_NUM_PORT_A_CHROMA,
+		.vb_index	= 1,
+		.vb_part	= VIP_CHROMA,
+	},
+	[VIP_SRCE_RGB] = {
+		.base_channel	= VIP1_CHAN_NUM_PORT_A_RGB,
+		.vb_part	= VIP_LUMA,
+	},
+};
+
+static struct vip_fmt vip_formats[VIP_MAX_ACTIVE_FMT] = {
+	{
+		.fourcc		= V4L2_PIX_FMT_NV12,
+		.code		= MEDIA_BUS_FMT_UYVY8_2X8,
+		.colorspace	= V4L2_COLORSPACE_SMPTE170M,
+		.coplanar	= 1,
+		.vpdma_fmt	= { &vpdma_yuv_fmts[VPDMA_DATA_FMT_Y420],
+				    &vpdma_yuv_fmts[VPDMA_DATA_FMT_C420],
+				  },
+	},
+	{
+		.fourcc		= V4L2_PIX_FMT_UYVY,
+		.code		= MEDIA_BUS_FMT_UYVY8_2X8,
+		.colorspace	= V4L2_COLORSPACE_SMPTE170M,
+		.coplanar	= 0,
+		.vpdma_fmt	= { &vpdma_yuv_fmts[VPDMA_DATA_FMT_CBY422],
+				  },
+	},
+	{
+		.fourcc		= V4L2_PIX_FMT_YUYV,
+		.code		= MEDIA_BUS_FMT_UYVY8_2X8,
+		.colorspace	= V4L2_COLORSPACE_SMPTE170M,
+		.coplanar	= 0,
+		.vpdma_fmt	= { &vpdma_yuv_fmts[VPDMA_DATA_FMT_YCB422],
+				  },
+	},
+	{
+		.fourcc		= V4L2_PIX_FMT_VYUY,
+		.code		= MEDIA_BUS_FMT_UYVY8_2X8,
+		.colorspace	= V4L2_COLORSPACE_SMPTE170M,
+		.coplanar	= 0,
+		.vpdma_fmt	= { &vpdma_yuv_fmts[VPDMA_DATA_FMT_CRY422],
+				  },
+	},
+	{
+		.fourcc		= V4L2_PIX_FMT_YVYU,
+		.code		= MEDIA_BUS_FMT_UYVY8_2X8,
+		.colorspace	= V4L2_COLORSPACE_SMPTE170M,
+		.coplanar	= 0,
+		.vpdma_fmt	= { &vpdma_yuv_fmts[VPDMA_DATA_FMT_YCR422],
+				  },
+	},
+	{
+		.fourcc		= V4L2_PIX_FMT_RGB24,
+		.code		= MEDIA_BUS_FMT_UYVY8_2X8,
+		.colorspace	= V4L2_COLORSPACE_SRGB,
+		.coplanar	= 0,
+		.vpdma_fmt	= { &vpdma_rgb_fmts[VPDMA_DATA_FMT_RGB24],
+				  },
+	},
+	{
+		.fourcc		= V4L2_PIX_FMT_RGB32,
+		.code		= MEDIA_BUS_FMT_UYVY8_2X8,
+		.colorspace	= V4L2_COLORSPACE_SRGB,
+		.coplanar	= 0,
+		.vpdma_fmt	= { &vpdma_rgb_fmts[VPDMA_DATA_FMT_ARGB32],
+				  },
+	},
+	{
+		.fourcc		= V4L2_PIX_FMT_BGR24,
+		.code		= MEDIA_BUS_FMT_UYVY8_2X8,
+		.colorspace	= V4L2_COLORSPACE_SRGB,
+		.coplanar	= 0,
+		.vpdma_fmt	= { &vpdma_rgb_fmts[VPDMA_DATA_FMT_BGR24],
+				  },
+	},
+	{
+		.fourcc		= V4L2_PIX_FMT_BGR32,
+		.code		= MEDIA_BUS_FMT_UYVY8_2X8,
+		.colorspace	= V4L2_COLORSPACE_SRGB,
+		.coplanar	= 0,
+		.vpdma_fmt	= { &vpdma_rgb_fmts[VPDMA_DATA_FMT_ABGR32],
+				  },
+	},
+	{
+		.fourcc		= V4L2_PIX_FMT_RGB24,
+		.code		= MEDIA_BUS_FMT_RGB888_1X24,
+		.colorspace	= V4L2_COLORSPACE_SRGB,
+		.coplanar	= 0,
+		.vpdma_fmt	= { &vpdma_rgb_fmts[VPDMA_DATA_FMT_RGB24],
+				  },
+	},
+	{
+		.fourcc		= V4L2_PIX_FMT_RGB32,
+		.code		= MEDIA_BUS_FMT_ARGB8888_1X32,
+		.colorspace	= V4L2_COLORSPACE_SRGB,
+		.coplanar	= 0,
+		.vpdma_fmt	= { &vpdma_rgb_fmts[VPDMA_DATA_FMT_ARGB32],
+				  },
+	},
+	{
+		.fourcc		= V4L2_PIX_FMT_SBGGR8,
+		.code		= MEDIA_BUS_FMT_SBGGR8_1X8,
+		.colorspace	= V4L2_COLORSPACE_SMPTE170M,
+		.coplanar	= 0,
+		.vpdma_fmt	= { &vpdma_raw_fmts[VPDMA_DATA_FMT_RAW8],
+				  },
+	},
+	{
+		.fourcc		= V4L2_PIX_FMT_SGBRG8,
+		.code		= MEDIA_BUS_FMT_SGBRG8_1X8,
+		.colorspace	= V4L2_COLORSPACE_SMPTE170M,
+		.coplanar	= 0,
+		.vpdma_fmt	= { &vpdma_raw_fmts[VPDMA_DATA_FMT_RAW8],
+				  },
+	},
+	{
+		.fourcc		= V4L2_PIX_FMT_SGRBG8,
+		.code		= MEDIA_BUS_FMT_SGRBG8_1X8,
+		.colorspace	= V4L2_COLORSPACE_SMPTE170M,
+		.coplanar	= 0,
+		.vpdma_fmt	= { &vpdma_raw_fmts[VPDMA_DATA_FMT_RAW8],
+				  },
+	},
+	{
+		.fourcc		= V4L2_PIX_FMT_SRGGB8,
+		.code		= MEDIA_BUS_FMT_SRGGB8_1X8,
+		.colorspace	= V4L2_COLORSPACE_SMPTE170M,
+		.coplanar	= 0,
+		.vpdma_fmt	= { &vpdma_raw_fmts[VPDMA_DATA_FMT_RAW8],
+				  },
+	},
+	{
+		/* V4L2 currently only defines one 16 bit variant */
+		.fourcc		= V4L2_PIX_FMT_SBGGR16,
+		.code		= MEDIA_BUS_FMT_SBGGR16_1X16,
+		.colorspace	= V4L2_COLORSPACE_SMPTE170M,
+		.coplanar	= 0,
+		.vpdma_fmt	= { &vpdma_raw_fmts[VPDMA_DATA_FMT_RAW16],
+				  },
+	},
+};
+
+/*  Print Four-character-code (FOURCC) */
+static char *fourcc_to_str(u32 fmt)
+{
+	static char code[5];
+
+	code[0] = (unsigned char)(fmt & 0xff);
+	code[1] = (unsigned char)((fmt >> 8) & 0xff);
+	code[2] = (unsigned char)((fmt >> 16) & 0xff);
+	code[3] = (unsigned char)((fmt >> 24) & 0xff);
+	code[4] = '\0';
+
+	return code;
+}
+
+/*
+ * Find our format description corresponding to the passed v4l2_format
+ */
+
+static struct vip_fmt *find_port_format_by_pix(struct vip_port *port,
+					       u32 pixelformat)
+{
+	struct vip_fmt *fmt;
+	unsigned int k;
+
+	for (k = 0; k < port->num_active_fmt; k++) {
+		fmt = port->active_fmt[k];
+		if (fmt->fourcc == pixelformat)
+			return fmt;
+	}
+
+	return NULL;
+}
+
+static struct vip_fmt *find_port_format_by_code(struct vip_port *port,
+						u32 code)
+{
+	struct vip_fmt *fmt;
+	unsigned int k;
+
+	for (k = 0; k < port->num_active_fmt; k++) {
+		fmt = port->active_fmt[k];
+		if (fmt->code == code)
+			return fmt;
+	}
+
+	return NULL;
+}
+
+inline struct vip_port *notifier_to_vip_port(struct v4l2_async_notifier *n)
+{
+	return container_of(n, struct vip_port, notifier);
+}
+
+static bool vip_is_fmt_yuv(u32 fourcc)
+{
+	if (fourcc == V4L2_PIX_FMT_NV12 ||
+	    fourcc == V4L2_PIX_FMT_UYVY ||
+	    fourcc == V4L2_PIX_FMT_YUYV ||
+	    fourcc == V4L2_PIX_FMT_VYUY ||
+	    fourcc == V4L2_PIX_FMT_YVYU)
+		return true;
+
+	return false;
+}
+
+static bool vip_is_fmt_rgb(u32 fourcc)
+{
+	if (fourcc == V4L2_PIX_FMT_RGB24 ||
+	    fourcc == V4L2_PIX_FMT_BGR24 ||
+	    fourcc == V4L2_PIX_FMT_RGB32 ||
+	    fourcc == V4L2_PIX_FMT_BGR32)
+		return true;
+
+	return false;
+}
+
+static bool vip_is_mbuscode_yuv(u32 code)
+{
+	return ((code & 0xFF00) == 0x2000);
+}
+
+static bool vip_is_mbuscode_rgb(u32 code)
+{
+	return ((code & 0xFF00) == 0x1000);
+}
+
+static bool vip_is_mbuscode_raw(u32 code)
+{
+	return ((code & 0xFF00) == 0x3000);
+}
+
+static enum  v4l2_colorspace vip_fourcc_to_colorspace(u32 fourcc)
+{
+	if (vip_is_fmt_rgb(fourcc))
+		return V4L2_COLORSPACE_SRGB;
+
+	return V4L2_COLORSPACE_SMPTE170M;
+}
+
+static enum  v4l2_colorspace vip_code_to_colorspace(u32 code)
+{
+	if (vip_is_mbuscode_rgb(code))
+		return V4L2_COLORSPACE_SRGB;
+
+	return V4L2_COLORSPACE_SMPTE170M;
+}
+
+static enum vip_csc_state vip_csc_direction(u32 src_code, u32 dst_fourcc)
+{
+	if (vip_is_mbuscode_yuv(src_code) && vip_is_fmt_rgb(dst_fourcc))
+		return VIP_CSC_Y2R;
+	else if (vip_is_mbuscode_rgb(src_code) && vip_is_fmt_yuv(dst_fourcc))
+		return VIP_CSC_R2Y;
+	else
+		return VIP_CSC_NA;
+}
+
+/*
+ * port flag bits
+ */
+#define FLAG_FRAME_1D		BIT(0)
+#define FLAG_EVEN_LINE_SKIP	BIT(1)
+#define FLAG_ODD_LINE_SKIP	BIT(2)
+#define FLAG_MODE_TILED		BIT(3)
+#define FLAG_INTERLACED		BIT(4)
+#define FLAG_MULTIPLEXED	BIT(5)
+#define FLAG_MULT_PORT		BIT(6)
+#define FLAG_MULT_ANC		BIT(7)
+
+/*
+ * Function prototype declarations
+ */
+static int alloc_port(struct vip_dev *, int);
+static void free_port(struct vip_port *);
+static int vip_setup_parser(struct vip_port *port);
+static int vip_setup_scaler(struct vip_stream *stream);
+static void vip_enable_parser(struct vip_port *port, bool on);
+static void vip_reset_parser(struct vip_port *port, bool on);
+static void vip_parser_stop_imm(struct vip_port *port, bool on);
+static void stop_dma(struct vip_stream *stream, bool clear_list);
+static int vip_load_vpdma_list_fifo(struct vip_stream *stream);
+static inline bool is_scaler_available(struct vip_port *port);
+static inline bool allocate_scaler(struct vip_port *port);
+static inline void free_scaler(struct vip_port *port);
+static bool is_csc_available(struct vip_port *port);
+static bool allocate_csc(struct vip_port *port,
+				enum vip_csc_state csc_direction);
+static void free_csc(struct vip_port *port);
+
+#define reg_read(dev, offset) ioread32(dev->base + offset)
+#define reg_write(dev, offset, val) iowrite32(val, dev->base + offset)
+
+/*
+ * Insert a masked field into a 32-bit field
+ */
+static void insert_field(u32 *valp, u32 field, u32 mask, int shift)
+{
+	u32 val = *valp;
+
+	val &= ~(mask << shift);
+	val |= (field & mask) << shift;
+	*valp = val;
+}
+
+/*
+ * DMA address/data block for the shadow registers
+ */
+struct vip_mmr_adb {
+	struct vpdma_adb_hdr	sc_hdr0;
+	u32			sc_regs0[7];
+	u32			sc_pad0[1];
+	struct vpdma_adb_hdr	sc_hdr8;
+	u32			sc_regs8[6];
+	u32			sc_pad8[2];
+	struct vpdma_adb_hdr	sc_hdr17;
+	u32			sc_regs17[9];
+	u32			sc_pad17[3];
+	struct vpdma_adb_hdr	csc_hdr;
+	u32			csc_regs[6];
+	u32			csc_pad[2];
+};
+
+#define GET_OFFSET_TOP(port, obj, reg)	\
+	((obj)->res->start - port->dev->res->start + reg)
+
+#define VIP_SET_MMR_ADB_HDR(port, hdr, regs, offset_a)	\
+	VPDMA_SET_MMR_ADB_HDR(port->mmr_adb, vip_mmr_adb, hdr, regs, offset_a)
+
+/*
+ * Set the headers for all of the address/data block structures.
+ */
+static void init_adb_hdrs(struct vip_port *port)
+{
+	VIP_SET_MMR_ADB_HDR(port, sc_hdr0, sc_regs0,
+			    GET_OFFSET_TOP(port, port->dev->sc, CFG_SC0));
+	VIP_SET_MMR_ADB_HDR(port, sc_hdr8, sc_regs8,
+			    GET_OFFSET_TOP(port, port->dev->sc, CFG_SC8));
+	VIP_SET_MMR_ADB_HDR(port, sc_hdr17, sc_regs17,
+			    GET_OFFSET_TOP(port, port->dev->sc, CFG_SC17));
+	VIP_SET_MMR_ADB_HDR(port, csc_hdr, csc_regs,
+			    GET_OFFSET_TOP(port, port->dev->csc, CSC_CSC00));
+
+};
+
+/*
+ * These represent the module resets bit for slice 1
+ * Upon detecting slice2 we simply left shift by 1
+ */
+#define VIP_DP_RST	BIT(16)
+#define VIP_PARSER_RST	BIT(18)
+#define VIP_CSC_RST	BIT(20)
+#define VIP_SC_RST	BIT(22)
+#define VIP_DS0_RST	BIT(25)
+#define VIP_DS1_RST	BIT(27)
+
+static void vip_module_reset(struct vip_dev *dev, uint32_t module, bool on)
+{
+	u32 val = 0;
+
+	val = reg_read(dev, VIP_CLK_RESET);
+
+	if (dev->slice_id == VIP_SLICE2)
+		module <<= 1;
+
+	if (on)
+		val |= module;
+	else
+		val &= ~module;
+
+	reg_write(dev, VIP_CLK_RESET, val);
+}
+
+/*
+ * Enable or disable the VIP clocks
+ */
+static void vip_set_clock_enable(struct vip_dev *dev, bool on)
+{
+	u32 val = 0;
+
+	val = reg_read(dev, VIP_CLK_ENABLE);
+	if (on) {
+		val |= VIP_VPDMA_CLK_ENABLE;
+		if (dev->slice_id == VIP_SLICE1)
+			val |= VIP_VIP1_DATA_PATH_CLK_ENABLE;
+		else
+			val |= VIP_VIP2_DATA_PATH_CLK_ENABLE;
+	} else {
+		if (dev->slice_id == VIP_SLICE1)
+			val &= ~VIP_VIP1_DATA_PATH_CLK_ENABLE;
+		else
+			val &= ~VIP_VIP2_DATA_PATH_CLK_ENABLE;
+
+		/* Both VIP are disabled then shutdown VPDMA also */
+		if (!(val & (VIP_VIP1_DATA_PATH_CLK_ENABLE |
+			     VIP_VIP2_DATA_PATH_CLK_ENABLE)))
+			val = 0;
+	}
+
+	reg_write(dev, VIP_CLK_ENABLE, val);
+}
+
+/* This helper function is used to enable the clock early on to
+ * enable vpdma firmware loading before the slice device are created
+ */
+static void vip_shared_set_clock_enable(struct vip_shared *shared, bool on)
+{
+	u32 val = 0;
+
+	if (on)
+		val = VIP_VIP1_DATA_PATH_CLK_ENABLE | VIP_VPDMA_CLK_ENABLE;
+
+	reg_write(shared, VIP_CLK_ENABLE, val);
+}
+
+static void vip_top_reset(struct vip_dev *dev)
+{
+	u32 val = 0;
+
+	val = reg_read(dev, VIP_CLK_RESET);
+
+	if (dev->slice_id == VIP_SLICE1)
+		insert_field(&val, 1, VIP_DATA_PATH_CLK_RESET_MASK,
+			     VIP_VIP1_DATA_PATH_RESET_SHIFT);
+	else
+		insert_field(&val, 1, VIP_DATA_PATH_CLK_RESET_MASK,
+			     VIP_VIP2_DATA_PATH_RESET_SHIFT);
+
+	reg_write(dev, VIP_CLK_RESET, val);
+
+	usleep_range(200, 250);
+
+	val = reg_read(dev, VIP_CLK_RESET);
+
+	if (dev->slice_id == VIP_SLICE1)
+		insert_field(&val, 0, VIP_DATA_PATH_CLK_RESET_MASK,
+			     VIP_VIP1_DATA_PATH_RESET_SHIFT);
+	else
+		insert_field(&val, 0, VIP_DATA_PATH_CLK_RESET_MASK,
+			     VIP_VIP2_DATA_PATH_RESET_SHIFT);
+	reg_write(dev, VIP_CLK_RESET, val);
+}
+
+static void vip_top_vpdma_reset(struct vip_shared *shared)
+{
+	u32 val;
+
+	val = reg_read(shared, VIP_CLK_RESET);
+	insert_field(&val, 1, VIP_VPDMA_CLK_RESET_MASK,
+		     VIP_VPDMA_CLK_RESET_SHIFT);
+	reg_write(shared, VIP_CLK_RESET, val);
+
+	usleep_range(200, 250);
+
+	val = reg_read(shared, VIP_CLK_RESET);
+	insert_field(&val, 0, VIP_VPDMA_CLK_RESET_MASK,
+		     VIP_VPDMA_CLK_RESET_SHIFT);
+	reg_write(shared, VIP_CLK_RESET, val);
+}
+
+static void vip_set_pclk_invert(struct vip_port *port)
+{
+	u32 offset;
+	/*
+	 * When the VIP parser is configured to so that the pixel clock
+	 * is to be sampled at falling edge, the pixel clock needs to be
+	 * inverted before it is given to the VIP module. This is done
+	 * by setting a bit in the CTRL_CORE_SMA_SW1 register.
+	 */
+
+	if (port->dev->instance_id == VIP_INSTANCE1) {
+		offset = 0 + 2 * port->port_id + port->dev->slice_id;
+	} else if (port->dev->instance_id == VIP_INSTANCE2) {
+		offset = 4 + 2 * port->port_id + port->dev->slice_id;
+	} else if (port->dev->instance_id == VIP_INSTANCE3) {
+		offset = 10 - port->dev->slice_id;
+	} else {
+		vip_err(port, "%s: VIP instance id out of range...\n",
+			__func__);
+		return;
+	}
+
+	if (port->dev->syscon_pol)
+		regmap_update_bits(port->dev->syscon_pol,
+				   port->dev->syscon_pol_offset,
+				   1 << offset, 1 << offset);
+}
+
+#define VIP_PARSER_PORT(p)	(VIP_PARSER_PORTA_0 + (p * 0x8U))
+#define VIP_PARSER_EXTRA_PORT(p)	(VIP_PARSER_PORTA_1 + (p * 0x8U))
+#define VIP_PARSER_CROP_H_PORT(p)	(VIP_PARSER_PORTA_EXTRA4 + (p * 0x10U))
+#define VIP_PARSER_CROP_V_PORT(p)	(VIP_PARSER_PORTA_EXTRA5 + (p * 0x10U))
+#define VIP_PARSER_STOP_IMM_PORT(p)	(VIP_PARSER_PORTA_EXTRA6 + (p * 0x4U))
+
+static void vip_set_data_interface(struct vip_port *port,
+				   enum data_interface_modes mode)
+{
+	u32 val = 0;
+
+	insert_field(&val, mode, VIP_DATA_INTERFACE_MODE_MASK,
+		     VIP_DATA_INTERFACE_MODE_SHFT);
+
+	reg_write(port->dev->parser, VIP_PARSER_MAIN_CFG, val);
+}
+
+static void vip_set_slice_path(struct vip_dev *dev,
+			       enum data_path_select data_path, u32 path_val)
+{
+	u32 val = 0;
+	int data_path_reg;
+
+	vip_dbg(3, dev, "%s:\n", __func__);
+
+	data_path_reg = VIP_VIP1_DATA_PATH_SELECT + 4 * dev->slice_id;
+
+	switch (data_path) {
+	case ALL_FIELDS_DATA_SELECT:
+		val |= path_val;
+		break;
+	case VIP_CSC_SRC_DATA_SELECT:
+		insert_field(&val, path_val, VIP_CSC_SRC_SELECT_MASK,
+			     VIP_CSC_SRC_SELECT_SHFT);
+		break;
+	case VIP_SC_SRC_DATA_SELECT:
+		insert_field(&val, path_val, VIP_SC_SRC_SELECT_MASK,
+			     VIP_SC_SRC_SELECT_SHFT);
+		break;
+	case VIP_RGB_SRC_DATA_SELECT:
+		val |= (path_val) ? VIP_RGB_SRC_SELECT : 0;
+		break;
+	case VIP_RGB_OUT_LO_DATA_SELECT:
+		val |= (path_val) ? VIP_RGB_OUT_LO_SRC_SELECT : 0;
+		break;
+	case VIP_RGB_OUT_HI_DATA_SELECT:
+		val |= (path_val) ? VIP_RGB_OUT_HI_SRC_SELECT : 0;
+		break;
+	case VIP_CHR_DS_1_SRC_DATA_SELECT:
+		insert_field(&val, path_val, VIP_DS1_SRC_SELECT_MASK,
+			     VIP_DS1_SRC_SELECT_SHFT);
+		break;
+	case VIP_CHR_DS_2_SRC_DATA_SELECT:
+		insert_field(&val, path_val, VIP_DS2_SRC_SELECT_MASK,
+			     VIP_DS2_SRC_SELECT_SHFT);
+		break;
+	case VIP_MULTI_CHANNEL_DATA_SELECT:
+		val |= (path_val) ? VIP_MULTI_CHANNEL_SELECT : 0;
+		break;
+	case VIP_CHR_DS_1_DATA_BYPASS:
+		val |= (path_val) ? VIP_DS1_BYPASS : 0;
+		break;
+	case VIP_CHR_DS_2_DATA_BYPASS:
+		val |= (path_val) ? VIP_DS2_BYPASS : 0;
+		break;
+	default:
+		vip_err(dev, "%s: data_path 0x%x is not valid\n",
+			__func__, data_path);
+		return;
+	}
+	insert_field(&val, data_path, VIP_DATAPATH_SELECT_MASK,
+		     VIP_DATAPATH_SELECT_SHFT);
+	reg_write(dev, data_path_reg, val);
+	vip_dbg(3, dev, "%s: DATA_PATH_SELECT(%08X): %08X\n", __func__,
+		data_path_reg, reg_read(dev, data_path_reg));
+}
+
+/*
+ * Return the vip_stream structure for a given struct file
+ */
+static inline struct vip_stream *file2stream(struct file *file)
+{
+	return video_drvdata(file);
+}
+
+/*
+ * Append a destination descriptor to the current descriptor list,
+ * setting up dma to the given srce.
+ */
+static int add_out_dtd(struct vip_stream *stream, int srce_type)
+{
+	struct vip_port *port = stream->port;
+	struct vip_dev *dev = port->dev;
+	struct vip_srce_info *sinfo = &srce_info[srce_type];
+	struct v4l2_rect *c_rect = &port->c_rect;
+	struct vip_fmt *fmt = port->fmt;
+	int channel, plane = 0;
+	int max_width, max_height;
+	dma_addr_t dma_addr;
+	u32 flags;
+	u32 width = stream->width;
+
+	channel = sinfo->base_channel;
+
+	switch (srce_type) {
+	case VIP_SRCE_MULT_PORT:
+	case VIP_SRCE_MULT_ANC:
+		if (port->port_id == VIP_PORTB)
+			channel += VIP_CHAN_MULT_PORTB_OFFSET;
+		channel += stream->stream_id;
+		flags = 0;
+		break;
+	case VIP_SRCE_CHROMA:
+		plane = 1;
+	case VIP_SRCE_LUMA:
+		if (port->port_id == VIP_PORTB) {
+			if (port->scaler && !port->fmt->coplanar)
+				/*
+				 * In this case Port A Chroma channel
+				 * is used to carry Port B scaled YUV422
+				 */
+				channel += 1;
+			else
+				channel += VIP_CHAN_YUV_PORTB_OFFSET;
+		}
+		flags = port->flags;
+		break;
+	case VIP_SRCE_RGB:
+		if ((port->port_id == VIP_PORTB) ||
+		    ((port->port_id == VIP_PORTA) &&
+		     (port->csc == VIP_CSC_NA) &&
+		     vip_is_fmt_rgb(port->fmt->fourcc)))
+			/*
+			 * RGB sensor only connect to Y_LO
+			 * channel i.e. port B channel.
+			 */
+			channel += VIP_CHAN_RGB_PORTB_OFFSET;
+		flags = port->flags;
+		break;
+	default:
+		vip_err(stream, "%s: srce_type 0x%x is not valid\n",
+			__func__, srce_type);
+		return -1;
+	}
+
+	if (dev->slice_id == VIP_SLICE2)
+		channel += VIP_CHAN_VIP2_OFFSET;
+
+	/* This is just for initialization purposes.
+	 * The actual dma_addr will be configured in vpdma_update_dma_addr
+	 */
+	dma_addr = 0;
+
+	if (port->fmt->vpdma_fmt[0] == &vpdma_raw_fmts[VPDMA_DATA_FMT_RAW8]) {
+		/*
+		 * Special case since we are faking a YUV422 16bit format
+		 * to have the vpdma perform the needed byte swap
+		 * we need to adjust the pixel width accordingly
+		 * otherwise the parser will attempt to collect more pixels
+		 * then available and the vpdma transfer will exceed the
+		 * allocated frame buffer.
+		 */
+		width >>= 1;
+		vip_dbg(1, stream, "%s: 8 bit raw detected, adjusting width to %d\n",
+			__func__, width);
+	}
+
+	/*
+	 * Use VPDMA_MAX_SIZE1 or VPDMA_MAX_SIZE2 register for slice0/1
+	 */
+
+	if (dev->slice_id == VIP_SLICE1) {
+		vpdma_set_max_size(dev->shared->vpdma, VPDMA_MAX_SIZE1,
+				   width, stream->height);
+
+		max_width = MAX_OUT_WIDTH_REG1;
+		max_height = MAX_OUT_HEIGHT_REG1;
+	} else {
+		vpdma_set_max_size(dev->shared->vpdma, VPDMA_MAX_SIZE2,
+				   width, stream->height);
+
+		max_width = MAX_OUT_WIDTH_REG2;
+		max_height = MAX_OUT_HEIGHT_REG2;
+	}
+
+	/*
+	 * Mark this channel to be cleared while cleaning up resources
+	 * This will make sure that an abort descriptor for this channel
+	 * would be submitted to VPDMA causing any ongoing  transaction to be
+	 * aborted and cleanup the VPDMA FSM for this channel
+	 */
+	stream->vpdma_channels[channel] = 1;
+
+	vpdma_rawchan_add_out_dtd(&stream->desc_list, c_rect->width,
+				  stream->bytesperline, c_rect,
+				  fmt->vpdma_fmt[plane], dma_addr,
+				  max_width, max_height, channel, flags);
+
+	return 0;
+}
+
+/*
+ * add_stream_dtds - prepares and starts DMA for pending transfers
+ */
+static void add_stream_dtds(struct vip_stream *stream)
+{
+	struct vip_port *port = stream->port;
+	int srce_type;
+
+	if (port->flags & FLAG_MULT_PORT)
+		srce_type = VIP_SRCE_MULT_PORT;
+	else if (port->flags & FLAG_MULT_ANC)
+		srce_type = VIP_SRCE_MULT_ANC;
+	else if (vip_is_fmt_rgb(port->fmt->fourcc))
+		srce_type = VIP_SRCE_RGB;
+	else
+		srce_type = VIP_SRCE_LUMA;
+
+	add_out_dtd(stream, srce_type);
+
+	if (srce_type == VIP_SRCE_LUMA && port->fmt->coplanar)
+		add_out_dtd(stream, VIP_SRCE_CHROMA);
+}
+
+#define PARSER_IRQ_MASK (VIP_PORTA_OUTPUT_FIFO_YUV | \
+			 VIP_PORTB_OUTPUT_FIFO_YUV)
+
+static void enable_irqs(struct vip_dev *dev, int irq_num, int list_num)
+{
+	struct vip_parser_data *parser = dev->parser;
+	u32 reg_addr = VIP_INT0_ENABLE0_SET +
+			VIP_INTC_INTX_OFFSET * irq_num;
+	u32 irq_val = (1 << (list_num * 2)) |
+		      (VIP_VIP1_PARSER_INT << (irq_num * 1));
+
+	/* Enable Parser Interrupt */
+	reg_write(parser, VIP_PARSER_FIQ_MASK, ~PARSER_IRQ_MASK);
+
+	reg_write(dev->shared, reg_addr, irq_val);
+
+	vpdma_enable_list_complete_irq(dev->shared->vpdma,
+				       irq_num, list_num, true);
+}
+
+static void disable_irqs(struct vip_dev *dev, int irq_num, int list_num)
+{
+	struct vip_parser_data *parser = dev->parser;
+	u32 reg_addr = VIP_INT0_ENABLE0_CLR +
+			VIP_INTC_INTX_OFFSET * irq_num;
+	u32 irq_val = (1 << (list_num * 2)) |
+		      (VIP_VIP1_PARSER_INT << (irq_num * 1));
+
+	/* Disable all Parser Interrupt */
+	reg_write(parser, VIP_PARSER_FIQ_MASK, 0xffffffff);
+
+	reg_write(dev->shared, reg_addr, irq_val);
+
+	vpdma_enable_list_complete_irq(dev->shared->vpdma,
+				       irq_num, list_num, false);
+}
+
+static void clear_irqs(struct vip_dev *dev, int irq_num, int list_num)
+{
+	struct vip_parser_data *parser = dev->parser;
+	u32 reg_addr = VIP_INT0_STATUS0_CLR +
+			VIP_INTC_INTX_OFFSET * irq_num;
+	u32 irq_val = (1 << (list_num * 2)) |
+		      (VIP_VIP1_PARSER_INT << (irq_num * 1));
+
+	/* Clear all Parser Interrupt */
+	reg_write(parser, VIP_PARSER_FIQ_CLR, 0xffffffff);
+	reg_write(parser, VIP_PARSER_FIQ_CLR, 0x0);
+
+	reg_write(dev->shared, reg_addr, irq_val);
+
+	vpdma_clear_list_stat(dev->shared->vpdma, irq_num, dev->slice_id);
+}
+
+static void populate_desc_list(struct vip_stream *stream)
+{
+	struct vip_port *port = stream->port;
+	struct vip_dev *dev = port->dev;
+	unsigned int list_length;
+
+	stream->desc_next = stream->desc_list.buf.addr;
+	add_stream_dtds(stream);
+
+	list_length = stream->desc_next - stream->desc_list.buf.addr;
+	vpdma_map_desc_buf(dev->shared->vpdma, &stream->desc_list.buf);
+}
+
+/*
+ * start_dma - adds descriptors to the dma list and submits them.
+ * Should be called after a new vb is queued and on a vpdma list
+ * completion interrupt.
+ */
+static void start_dma(struct vip_stream *stream, struct vip_buffer *buf)
+{
+	struct vip_dev *dev = stream->port->dev;
+	struct vpdma_data *vpdma = dev->shared->vpdma;
+	int list_num = stream->list_num;
+	dma_addr_t dma_addr;
+	int drop_data;
+
+	if (vpdma_list_busy(vpdma, list_num)) {
+		vip_err(stream, "vpdma list busy, cannot post\n");
+		return;				/* nothing to do */
+	}
+
+	if (buf) {
+		dma_addr = vb2_dma_contig_plane_dma_addr(&buf->vb.vb2_buf, 0);
+		drop_data = 0;
+		vip_dbg(4, stream, "%s: vb2 buf idx:%d, dma_addr:%pad\n",
+			__func__, buf->vb.vb2_buf.index, &dma_addr);
+	} else {
+		dma_addr = 0;
+		drop_data = 1;
+		vip_dbg(4, stream, "%s: dropped\n", __func__);
+	}
+
+	vpdma_update_dma_addr(dev->shared->vpdma, &stream->desc_list,
+			      dma_addr, stream->write_desc, drop_data, 0);
+
+	if (stream->port->fmt->coplanar) {
+		dma_addr += stream->bytesperline * stream->height;
+		vpdma_update_dma_addr(dev->shared->vpdma, &stream->desc_list,
+				      dma_addr, stream->write_desc + 1,
+				      drop_data, 1);
+	}
+
+	vpdma_submit_descs(dev->shared->vpdma,
+			   &stream->desc_list, stream->list_num);
+}
+
+static void vip_schedule_next_buffer(struct vip_stream *stream)
+{
+	struct vip_dev *dev = stream->port->dev;
+	struct vip_buffer *buf;
+	unsigned long flags;
+
+	spin_lock_irqsave(&dev->slock, flags);
+	if (list_empty(&stream->vidq)) {
+		vip_dbg(4, stream, "Dropping frame\n");
+		if (list_empty(&stream->dropq)) {
+			vip_err(stream, "No dropq buffer left!");
+			spin_unlock_irqrestore(&dev->slock, flags);
+			return;
+		}
+		buf = list_entry(stream->dropq.next,
+				 struct vip_buffer, list);
+
+		buf->drop = true;
+		list_move_tail(&buf->list, &stream->post_bufs);
+		buf = NULL;
+	} else {
+		buf = list_entry(stream->vidq.next,
+				 struct vip_buffer, list);
+		buf->drop = false;
+		list_move_tail(&buf->list, &stream->post_bufs);
+		vip_dbg(4, stream, "added next buffer\n");
+	}
+
+	spin_unlock_irqrestore(&dev->slock, flags);
+	start_dma(stream, buf);
+}
+
+static void vip_process_buffer_complete(struct vip_stream *stream)
+{
+	struct vip_dev *dev = stream->port->dev;
+	struct vip_buffer *buf;
+	struct vb2_v4l2_buffer *vb = NULL;
+	unsigned long flags, fld;
+
+	buf = list_first_entry(&stream->post_bufs, struct vip_buffer, list);
+
+	if (stream->port->flags & FLAG_INTERLACED) {
+		vpdma_unmap_desc_buf(dev->shared->vpdma,
+				     &stream->desc_list.buf);
+
+		fld = dtd_get_field(stream->write_desc);
+		stream->field = fld ? V4L2_FIELD_BOTTOM : V4L2_FIELD_TOP;
+
+		vpdma_map_desc_buf(dev->shared->vpdma, &stream->desc_list.buf);
+	}
+
+	if (buf) {
+		vip_dbg(4, stream, "vip buffer complete 0x%x, 0x%x\n",
+			(unsigned int)buf, buf->drop);
+
+		vb = &buf->vb;
+		vb->field = stream->field;
+		vb->sequence = stream->sequence;
+		vb->vb2_buf.timestamp = ktime_get_ns();
+
+		if (buf->drop) {
+			spin_lock_irqsave(&dev->slock, flags);
+			list_move_tail(&buf->list, &stream->dropq);
+			spin_unlock_irqrestore(&dev->slock, flags);
+		} else {
+			spin_lock_irqsave(&dev->slock, flags);
+			list_del(&buf->list);
+			spin_unlock_irqrestore(&dev->slock, flags);
+			vb2_buffer_done(&vb->vb2_buf, VB2_BUF_STATE_DONE);
+		}
+	} else {
+		vip_err(stream, "%s: buf is null!!!\n", __func__);
+		return;
+	}
+
+	stream->sequence++;
+}
+
+static int vip_reset_vpdma(struct vip_stream *stream)
+{
+	struct vip_port *port = stream->port;
+	struct vip_dev *dev = port->dev;
+	struct vip_buffer *buf;
+	unsigned long flags;
+
+	stop_dma(stream, false);
+
+	spin_lock_irqsave(&dev->slock, flags);
+	/* requeue all active buffers in the opposite order */
+	while (!list_empty(&stream->post_bufs)) {
+		buf = list_last_entry(&stream->post_bufs,
+				      struct vip_buffer, list);
+		list_del(&buf->list);
+		if (buf->drop == 1) {
+			list_add_tail(&buf->list, &stream->dropq);
+			vip_dbg(4, stream, "requeueing drop buffer on dropq\n");
+		} else {
+			list_add(&buf->list, &stream->vidq);
+			vip_dbg(4, stream, "requeueing vb2 buf idx:%d on vidq\n",
+				buf->vb.vb2_buf.index);
+		}
+	}
+	spin_unlock_irqrestore(&dev->slock, flags);
+
+	/* Make sure the desc_list is unmapped */
+	vpdma_unmap_desc_buf(dev->shared->vpdma, &stream->desc_list.buf);
+
+	return 0;
+}
+
+static void vip_overflow_recovery_work(struct work_struct *work)
+{
+	struct vip_stream *stream = container_of(work, struct vip_stream,
+						 recovery_work);
+	struct vip_port *port = stream->port;
+	struct vip_dev *dev = port->dev;
+
+	vip_err(stream, "%s: Port %c\n", __func__,
+		port->port_id == VIP_PORTA ? 'A' : 'B');
+
+	disable_irqs(dev, dev->slice_id, stream->list_num);
+	clear_irqs(dev, dev->slice_id, stream->list_num);
+
+	/* 1.	Set VIP_XTRA6_PORT_A[31:16] YUV_SRCNUM_STOP_IMMEDIATELY */
+	/* 2.	Set VIP_XTRA6_PORT_A[15:0] ANC_SRCNUM_STOP_IMMEDIATELY */
+	vip_parser_stop_imm(port, 1);
+
+	/* 3.	Clear VIP_PORT_A[8] ENABLE */
+	/*
+	 * 4.	Set VIP_PORT_A[7] CLR_ASYNC_FIFO_RD
+	 *      Set VIP_PORT_A[6] CLR_ASYNC_FIFO_WR
+	 */
+	vip_enable_parser(port, false);
+
+	/* 5.	Set VIP_PORT_A[23] SW_RESET */
+	vip_reset_parser(port, 1);
+
+	/*
+	 * 6.	Reset other VIP modules
+	 *	For each module used downstream of VIP_PARSER, write 1 to the
+	 *      bit location of the VIP_CLKC_RST register which is connected
+	 *      to VIP_PARSER
+	 */
+	vip_module_reset(dev, VIP_DP_RST, true);
+
+	usleep_range(200, 250);
+
+	/*
+	 * 7.	Abort VPDMA channels
+	 *	Write to list attribute to stop list 0
+	 *	Write to list address register location of abort list
+	 *	Write to list attribute register list 0 and size of abort list
+	 */
+	vip_reset_vpdma(stream);
+
+	/* 8.	Clear VIP_PORT_A[23] SW_RESET */
+	vip_reset_parser(port, 0);
+
+	/*
+	 * 9.	Un-reset other VIP modules
+	 *	For each module used downstream of VIP_PARSER, write 0 to
+	 *	the bit location of the VIP_CLKC_RST register which is
+	 *	connected to VIP_PARSER
+	 */
+	vip_module_reset(dev, VIP_DP_RST, false);
+
+	/* 10.	(Delay) */
+	/* 11.	SC coeff downloaded (if VIP_SCALER is being used) */
+	vip_setup_scaler(stream);
+
+	/* 12.	(Delay) */
+		/* the above are not needed here yet */
+
+	populate_desc_list(stream);
+	stream->num_recovery++;
+	if (stream->num_recovery < 5) {
+		/* Reload the vpdma */
+		vip_load_vpdma_list_fifo(stream);
+
+		enable_irqs(dev, dev->slice_id, stream->list_num);
+		vip_schedule_next_buffer(stream);
+
+		/* 13.	Clear VIP_XTRA6_PORT_A[31:16] YUV_SRCNUM_STOP_IMM */
+		/* 14.	Clear VIP_XTRA6_PORT_A[15:0] ANC_SRCNUM_STOP_IMM */
+
+		vip_parser_stop_imm(port, 0);
+
+		/* 15.	Set VIP_PORT_A[8] ENABLE */
+		/*
+		 * 16.	Clear VIP_PORT_A[7] CLR_ASYNC_FIFO_RD
+		 *	Clear VIP_PORT_A[6] CLR_ASYNC_FIFO_WR
+		 */
+		vip_enable_parser(port, true);
+	} else {
+		vip_err(stream, "%s: num_recovery limit exceeded leaving disabled\n",
+			__func__);
+	}
+}
+
+static void handle_parser_irqs(struct vip_dev *dev)
+{
+	struct vip_parser_data *parser = dev->parser;
+	struct vip_port *porta = dev->ports[VIP_PORTA];
+	struct vip_port *portb = dev->ports[VIP_PORTB];
+	struct vip_stream *stream = NULL;
+	u32 irq_stat = reg_read(parser, VIP_PARSER_FIQ_STATUS);
+	int i;
+
+	vip_dbg(3, dev, "%s: FIQ_STATUS: 0x%08x\n", __func__, irq_stat);
+
+	/* Clear all Parser Interrupt */
+	reg_write(parser, VIP_PARSER_FIQ_CLR, irq_stat);
+	reg_write(parser, VIP_PARSER_FIQ_CLR, 0x0);
+
+	if (irq_stat & VIP_PORTA_VDET)
+		vip_dbg(3, dev, "VIP_PORTA_VDET\n");
+	if (irq_stat & VIP_PORTB_VDET)
+		vip_dbg(3, dev, "VIP_PORTB_VDET\n");
+	if (irq_stat & VIP_PORTA_ASYNC_FIFO_OF)
+		vip_err(dev, "VIP_PORTA_ASYNC_FIFO_OF\n");
+	if (irq_stat & VIP_PORTB_ASYNC_FIFO_OF)
+		vip_err(dev, "VIP_PORTB_ASYNC_FIFO_OF\n");
+	if (irq_stat & VIP_PORTA_OUTPUT_FIFO_YUV)
+		vip_err(dev, "VIP_PORTA_OUTPUT_FIFO_YUV\n");
+	if (irq_stat & VIP_PORTA_OUTPUT_FIFO_ANC)
+		vip_err(dev, "VIP_PORTA_OUTPUT_FIFO_ANC\n");
+	if (irq_stat & VIP_PORTB_OUTPUT_FIFO_YUV)
+		vip_err(dev, "VIP_PORTB_OUTPUT_FIFO_YUV\n");
+	if (irq_stat & VIP_PORTB_OUTPUT_FIFO_ANC)
+		vip_err(dev, "VIP_PORTB_OUTPUT_FIFO_ANC\n");
+	if (irq_stat & VIP_PORTA_CONN)
+		vip_dbg(3, dev, "VIP_PORTA_CONN\n");
+	if (irq_stat & VIP_PORTA_DISCONN)
+		vip_dbg(3, dev, "VIP_PORTA_DISCONN\n");
+	if (irq_stat & VIP_PORTB_CONN)
+		vip_dbg(3, dev, "VIP_PORTB_CONN\n");
+	if (irq_stat & VIP_PORTB_DISCONN)
+		vip_dbg(3, dev, "VIP_PORTB_DISCONN\n");
+	if (irq_stat & VIP_PORTA_SRC0_SIZE)
+		vip_dbg(3, dev, "VIP_PORTA_SRC0_SIZE\n");
+	if (irq_stat & VIP_PORTB_SRC0_SIZE)
+		vip_dbg(3, dev, "VIP_PORTB_SRC0_SIZE\n");
+	if (irq_stat & VIP_PORTA_YUV_PROTO_VIOLATION)
+		vip_dbg(3, dev, "VIP_PORTA_YUV_PROTO_VIOLATION\n");
+	if (irq_stat & VIP_PORTA_ANC_PROTO_VIOLATION)
+		vip_dbg(3, dev, "VIP_PORTA_ANC_PROTO_VIOLATION\n");
+	if (irq_stat & VIP_PORTB_YUV_PROTO_VIOLATION)
+		vip_dbg(3, dev, "VIP_PORTB_YUV_PROTO_VIOLATION\n");
+	if (irq_stat & VIP_PORTB_ANC_PROTO_VIOLATION)
+		vip_dbg(3, dev, "VIP_PORTB_ANC_PROTO_VIOLATION\n");
+	if (irq_stat & VIP_PORTA_CFG_DISABLE_COMPLETE)
+		vip_dbg(3, dev, "VIP_PORTA_CFG_DISABLE_COMPLETE\n");
+	if (irq_stat & VIP_PORTB_CFG_DISABLE_COMPLETE)
+		vip_dbg(3, dev, "VIP_PORTB_CFG_DISABLE_COMPLETE\n");
+
+	if (irq_stat & (VIP_PORTA_ASYNC_FIFO_OF |
+			VIP_PORTA_OUTPUT_FIFO_YUV |
+			VIP_PORTA_OUTPUT_FIFO_ANC)) {
+		for (i = 0; i < VIP_CAP_STREAMS_PER_PORT; i++) {
+			if (porta->cap_streams[i] &&
+			    porta->cap_streams[i]->port->port_id ==
+			    porta->port_id) {
+				stream = porta->cap_streams[i];
+				break;
+			}
+		}
+		if (stream) {
+			disable_irqs(dev, dev->slice_id,
+				     stream->list_num);
+			schedule_work(&stream->recovery_work);
+			return;
+		}
+	}
+	if (irq_stat & (VIP_PORTB_ASYNC_FIFO_OF |
+			VIP_PORTB_OUTPUT_FIFO_YUV |
+			VIP_PORTB_OUTPUT_FIFO_ANC)) {
+		for (i = 0; i < VIP_CAP_STREAMS_PER_PORT; i++) {
+			if (portb->cap_streams[i] &&
+			    portb->cap_streams[i]->port->port_id ==
+			    portb->port_id) {
+				stream = portb->cap_streams[i];
+				break;
+			}
+		}
+		if (stream) {
+			disable_irqs(dev, dev->slice_id,
+				     stream->list_num);
+			schedule_work(&stream->recovery_work);
+			return;
+		}
+	}
+}
+
+static irqreturn_t vip_irq(int irq_vip, void *data)
+{
+	struct vip_dev *dev = (struct vip_dev *)data;
+	struct vpdma_data *vpdma;
+	struct vip_stream *stream;
+	int list_num;
+	int irq_num = dev->slice_id;
+	u32 irqst, irqst_saved, reg_addr;
+
+	if (!dev->shared)
+		return IRQ_HANDLED;
+
+	vpdma = dev->shared->vpdma;
+	reg_addr = VIP_INT0_STATUS0 +
+			VIP_INTC_INTX_OFFSET * irq_num;
+	irqst_saved = reg_read(dev->shared, reg_addr);
+	irqst = irqst_saved;
+
+	vip_dbg(8, dev, "IRQ %d VIP_INT%d_STATUS0 0x%x\n",
+		irq_vip, irq_num, irqst);
+	if (irqst) {
+		if (irqst & (VIP_VIP1_PARSER_INT << (irq_num * 1))) {
+			irqst &= ~(VIP_VIP1_PARSER_INT << (irq_num * 1));
+			handle_parser_irqs(dev);
+		}
+
+		for (list_num = 0; irqst && (list_num < 8);  list_num++) {
+			/* Check for LIST_COMPLETE IRQ */
+			if (!(irqst & (1 << list_num * 2)))
+				continue;
+
+			vip_dbg(8, dev, "IRQ %d: handling LIST%d_COMPLETE\n",
+				irq_num, list_num);
+
+			stream = vpdma_hwlist_get_priv(vpdma, list_num);
+			if (!stream || stream->list_num != list_num) {
+				vip_err(dev, "IRQ occurred for unused list");
+				continue;
+			}
+
+			vpdma_clear_list_stat(vpdma, irq_num, list_num);
+
+			vip_process_buffer_complete(stream);
+
+			vip_schedule_next_buffer(stream);
+
+			irqst &= ~((1 << list_num * 2));
+		}
+	}
+
+	/* Acknowledge that we are done with all interrupts */
+	reg_write(dev->shared, VIP_INTC_E0I, 1 << irq_num);
+
+	/* Clear handled events from status register */
+	reg_addr = VIP_INT0_STATUS0_CLR +
+		   VIP_INTC_INTX_OFFSET * irq_num;
+	reg_write(dev->shared, reg_addr, irqst_saved);
+
+	return IRQ_HANDLED;
+}
+
+/*
+ * video ioctls
+ */
+static int vip_querycap(struct file *file, void *priv,
+			struct v4l2_capability *cap)
+{
+	struct vip_stream *stream = file2stream(file);
+	struct vip_port *port = stream->port;
+	struct vip_dev *dev = port->dev;
+	u32 vin_id = 1 + ((dev->instance_id - 1) * 2) + dev->slice_id;
+
+	strlcpy(cap->driver, VIP_MODULE_NAME, sizeof(cap->driver));
+	strlcpy(cap->card, VIP_MODULE_NAME, sizeof(cap->card));
+	snprintf(cap->bus_info, sizeof(cap->bus_info),
+		 "platform:vip%1d:vin%1d%c:stream%1d", dev->instance_id, vin_id,
+		 port->port_id == VIP_PORTA ? 'a' : 'b', stream->stream_id);
+	cap->device_caps  = V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_CAPTURE |
+			    V4L2_CAP_READWRITE;
+	cap->capabilities = cap->device_caps | V4L2_CAP_DEVICE_CAPS;
+	return 0;
+}
+
+static int vip_enuminput(struct file *file, void *priv,
+			 struct v4l2_input *inp)
+{
+	struct vip_stream *stream = file2stream(file);
+
+	if (inp->index)
+		return -EINVAL;
+
+	inp->type = V4L2_INPUT_TYPE_CAMERA;
+	inp->std = stream->vfd->tvnorms;
+	sprintf(inp->name, "camera %u", stream->vfd->num);
+
+	return 0;
+}
+
+static int vip_g_input(struct file *file, void *priv, unsigned int *i)
+{
+	*i = 0;
+	return 0;
+}
+
+static int vip_s_input(struct file *file, void *priv, unsigned int i)
+{
+	if (i != 0)
+		return -EINVAL;
+	return 0;
+}
+
+static int vip_querystd(struct file *file, void *fh, v4l2_std_id *std)
+{
+	struct vip_stream *stream = file2stream(file);
+	struct vip_port *port = stream->port;
+
+	*std = stream->vfd->tvnorms;
+	v4l2_subdev_call(port->subdev, video, querystd, std);
+	vip_dbg(1, stream, "querystd: 0x%lx\n", (unsigned long)*std);
+	return 0;
+}
+
+static int vip_g_std(struct file *file, void *fh, v4l2_std_id *std)
+{
+	struct vip_stream *stream = file2stream(file);
+	struct vip_port *port = stream->port;
+
+	*std = stream->vfd->tvnorms;
+	v4l2_subdev_call(port->subdev, video, g_std_output, std);
+	vip_dbg(1, stream, "g_std: 0x%lx\n", (unsigned long)*std);
+
+	return 0;
+}
+
+static int vip_s_std(struct file *file, void *fh, v4l2_std_id std)
+{
+	struct vip_stream *stream = file2stream(file);
+	struct vip_port *port = stream->port;
+
+	vip_dbg(1, stream, "s_std: 0x%lx\n", (unsigned long)std);
+
+	if (!(std & stream->vfd->tvnorms)) {
+		vip_dbg(1, stream, "s_std after check: 0x%lx\n",
+			(unsigned long)std);
+		return -EINVAL;
+	}
+
+	v4l2_subdev_call(port->subdev, video, s_std_output, std);
+	return 0;
+}
+
+static int vip_enum_fmt_vid_cap(struct file *file, void *priv,
+				struct v4l2_fmtdesc *f)
+{
+	struct vip_stream *stream = file2stream(file);
+	struct vip_port *port = stream->port;
+	struct vip_fmt *fmt;
+
+	vip_dbg(3, stream, "enum_fmt index:%d\n", f->index);
+	if (f->index >= port->num_active_fmt)
+		return -EINVAL;
+
+	fmt = port->active_fmt[f->index];
+
+	f->pixelformat = fmt->fourcc;
+	f->type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+	vip_dbg(3, stream, "enum_fmt fourcc:%s\n",
+		fourcc_to_str(f->pixelformat));
+
+	return 0;
+}
+
+static int vip_enum_framesizes(struct file *file, void *priv,
+			       struct v4l2_frmsizeenum *f)
+{
+	struct vip_stream *stream = file2stream(file);
+	struct vip_port *port = stream->port;
+	struct vip_fmt *fmt;
+	struct v4l2_subdev_frame_size_enum fse;
+	int ret;
+
+	fmt = find_port_format_by_pix(port, f->pixel_format);
+	if (!fmt)
+		return -EINVAL;
+
+	fse.index = f->index;
+	fse.pad = 0;
+	fse.code = fmt->code;
+
+	ret = v4l2_subdev_call(port->subdev, pad, enum_frame_size, NULL, &fse);
+	if (ret)
+		return -EINVAL;
+
+	vip_dbg(1, stream, "%s: index: %d code: %x W:[%d,%d] H:[%d,%d]\n",
+		__func__, fse.index, fse.code, fse.min_width, fse.max_width,
+		fse.min_height, fse.max_height);
+
+	f->type = V4L2_FRMSIZE_TYPE_DISCRETE;
+	f->discrete.width = fse.max_width;
+	f->discrete.height = fse.max_height;
+
+	return 0;
+}
+
+static int vip_enum_frameintervals(struct file *file, void *priv,
+				   struct v4l2_frmivalenum *f)
+{
+	struct vip_stream *stream = file2stream(file);
+	struct vip_port *port = stream->port;
+	struct vip_fmt *fmt;
+	struct v4l2_subdev_frame_interval_enum fie = {
+		.index = f->index,
+		.width = f->width,
+		.height = f->height,
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
+	int ret;
+
+	fmt = find_port_format_by_pix(port, f->pixel_format);
+	if (!fmt)
+		return -EINVAL;
+
+	fie.code = fmt->code;
+	ret = v4l2_subdev_call(port->subdev, pad, enum_frame_interval,
+			       NULL, &fie);
+	if (ret)
+		return ret;
+	f->type = V4L2_FRMIVAL_TYPE_DISCRETE;
+	f->discrete = fie.interval;
+
+	return 0;
+}
+
+static int vip_g_parm(struct file *file, void *priv,
+		      struct v4l2_streamparm *parm)
+{
+	if (parm->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+		return -EINVAL;
+
+	parm->parm.capture.capability   = V4L2_CAP_TIMEPERFRAME;
+	parm->parm.capture.timeperframe.numerator = 1;
+	parm->parm.capture.timeperframe.denominator = 30;
+	parm->parm.capture.readbuffers  = 4;
+	return 0;
+}
+
+static int vip_s_parm(struct file *file, void *priv,
+		      struct v4l2_streamparm *parm)
+{
+	if (parm->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+		return -EINVAL;
+
+	parm->parm.capture.timeperframe.numerator = 1;
+	parm->parm.capture.timeperframe.denominator = 30;
+	parm->parm.capture.readbuffers  = 4;
+
+	return 0;
+}
+
+static int vip_calc_format_size(struct vip_port *port,
+				struct vip_fmt *fmt,
+				struct v4l2_format *f)
+{
+	enum v4l2_field *field;
+	unsigned int stride;
+
+	if (!fmt) {
+		vip_dbg(2, port,
+			"no vip_fmt format provided!\n");
+		return -EINVAL;
+	}
+
+	field = &f->fmt.pix.field;
+	if (*field == V4L2_FIELD_ANY)
+		*field = V4L2_FIELD_NONE;
+	else if (V4L2_FIELD_NONE != *field && V4L2_FIELD_ALTERNATE != *field)
+		return -EINVAL;
+
+	v4l_bound_align_image(&f->fmt.pix.width, MIN_W, MAX_W, W_ALIGN,
+			      &f->fmt.pix.height, MIN_H, MAX_H, H_ALIGN,
+			      S_ALIGN);
+
+	stride = f->fmt.pix.width * (fmt->vpdma_fmt[0]->depth >> 3);
+	if (stride > f->fmt.pix.bytesperline)
+		f->fmt.pix.bytesperline = stride;
+	f->fmt.pix.bytesperline = ALIGN(f->fmt.pix.bytesperline,
+					VPDMA_STRIDE_ALIGN);
+
+	f->fmt.pix.sizeimage = f->fmt.pix.height * f->fmt.pix.bytesperline;
+	if (fmt->coplanar) {
+		f->fmt.pix.sizeimage += f->fmt.pix.height *
+					f->fmt.pix.bytesperline *
+					fmt->vpdma_fmt[VIP_CHROMA]->depth >> 3;
+	}
+
+	f->fmt.pix.colorspace = fmt->colorspace;
+	f->fmt.pix.priv = 0;
+
+	vip_dbg(3, port, "calc_format_size: fourcc:%s size: %dx%d bpl:%d img_size:%d\n",
+		fourcc_to_str(f->fmt.pix.pixelformat),
+		f->fmt.pix.width, f->fmt.pix.height,
+		f->fmt.pix.bytesperline, f->fmt.pix.sizeimage);
+
+	return 0;
+}
+
+static inline bool vip_is_size_dma_aligned(u32 bpp, u32 width)
+{
+	return ((width * bpp) == ALIGN(width * bpp, VPDMA_STRIDE_ALIGN));
+}
+
+static int vip_try_fmt_vid_cap(struct file *file, void *priv,
+			       struct v4l2_format *f)
+{
+	struct vip_stream *stream = file2stream(file);
+	struct vip_port *port = stream->port;
+	struct v4l2_subdev_frame_size_enum fse;
+	struct vip_fmt *fmt;
+	u32 best_width, best_height, largest_width, largest_height;
+	int ret, found;
+	enum vip_csc_state csc_direction;
+
+	vip_dbg(3, stream, "try_fmt fourcc:%s size: %dx%d\n",
+		fourcc_to_str(f->fmt.pix.pixelformat),
+		f->fmt.pix.width, f->fmt.pix.height);
+
+	fmt = find_port_format_by_pix(port, f->fmt.pix.pixelformat);
+	if (!fmt) {
+		vip_dbg(2, stream,
+			"Fourcc format (0x%08x) not found.\n",
+			f->fmt.pix.pixelformat);
+
+		/* Just get the first one enumerated */
+		fmt = port->active_fmt[0];
+		f->fmt.pix.pixelformat = fmt->fourcc;
+	}
+
+	csc_direction =  vip_csc_direction(fmt->code, fmt->fourcc);
+	if (csc_direction != VIP_CSC_NA) {
+		if (!is_csc_available(port)) {
+			vip_dbg(2, stream,
+				"CSC not available for Fourcc format (0x%08x).\n",
+				f->fmt.pix.pixelformat);
+
+			/* Just get the first one enumerated */
+			fmt = port->active_fmt[0];
+			f->fmt.pix.pixelformat = fmt->fourcc;
+			/* re-evaluate the csc_direction here */
+			csc_direction =  vip_csc_direction(fmt->code,
+							   fmt->fourcc);
+		} else {
+			vip_dbg(3, stream, "CSC active on Port %c: going %s\n",
+				port->port_id == VIP_PORTA ? 'A' : 'B',
+				(csc_direction == VIP_CSC_Y2R) ? "Y2R" : "R2Y");
+		}
+	}
+
+	/*
+	 * Given that sensors might support multiple mbus code we need
+	 * to use the one that matches the requested pixel format
+	 */
+	port->try_mbus_framefmt = port->mbus_framefmt;
+	port->try_mbus_framefmt.code = fmt->code;
+
+	/* check for/find a valid width/height */
+	ret = 0;
+	found = false;
+	best_width = 0;
+	best_height = 0;
+	largest_width = 0;
+	largest_height = 0;
+	fse.pad = 0;
+	fse.code = fmt->code;
+	fse.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+	for (fse.index = 0; ; fse.index++) {
+		u32 bpp = fmt->vpdma_fmt[0]->depth >> 3;
+
+		ret = v4l2_subdev_call(port->subdev, pad,
+				       enum_frame_size, NULL, &fse);
+		if (ret)
+			break;
+
+		vip_dbg(3, stream, "try_fmt loop:%d fourcc:%s size: %dx%d\n",
+			fse.index, fourcc_to_str(f->fmt.pix.pixelformat),
+			fse.max_width, fse.max_height);
+
+		if (!vip_is_size_dma_aligned(bpp, fse.max_width))
+			continue;
+
+		if ((fse.max_width >= largest_width) &&
+		    (fse.max_height >= largest_height)) {
+			vip_dbg(3, stream, "try_fmt loop:%d found new larger: %dx%d\n",
+				fse.index, fse.max_width, fse.max_height);
+			largest_width = fse.max_width;
+			largest_height = fse.max_height;
+		}
+
+		if ((fse.max_width >= f->fmt.pix.width) &&
+		    (fse.max_height >= f->fmt.pix.height)) {
+			vip_dbg(3, stream, "try_fmt loop:%d found at least larger: %dx%d\n",
+				fse.index, fse.max_width, fse.max_height);
+
+			if (!best_width ||
+			    ((abs(best_width - f->fmt.pix.width) >=
+			      abs(fse.max_width - f->fmt.pix.width)) &&
+			     (abs(best_height - f->fmt.pix.height) >=
+			      abs(fse.max_height - f->fmt.pix.height)))) {
+				best_width = fse.max_width;
+				best_height = fse.max_height;
+				vip_dbg(3, stream, "try_fmt loop:%d found new best: %dx%d\n",
+					fse.index, fse.max_width,
+					fse.max_height);
+			}
+		}
+
+		if ((f->fmt.pix.width == fse.max_width) &&
+		    (f->fmt.pix.height == fse.max_height)) {
+			found = true;
+			vip_dbg(3, stream, "try_fmt loop:%d found direct match: %dx%d\n",
+				fse.index, fse.max_width,
+				fse.max_height);
+			break;
+		}
+
+		if ((f->fmt.pix.width >= fse.min_width) &&
+		    (f->fmt.pix.width <= fse.max_width) &&
+		    (f->fmt.pix.height >= fse.min_height) &&
+		    (f->fmt.pix.height <= fse.max_height)) {
+			found = true;
+			vip_dbg(3, stream, "try_fmt loop:%d found direct range match: %dx%d\n",
+				fse.index, fse.max_width,
+				fse.max_height);
+			break;
+		}
+	}
+
+	if (found) {
+		port->try_mbus_framefmt.width = f->fmt.pix.width;
+		port->try_mbus_framefmt.height = f->fmt.pix.height;
+		/* No need to check for scaling */
+		goto calc_size;
+	} else if (f->fmt.pix.width > largest_width) {
+		port->try_mbus_framefmt.width = largest_width;
+		port->try_mbus_framefmt.height = largest_height;
+	} else if (best_width) {
+		port->try_mbus_framefmt.width = best_width;
+		port->try_mbus_framefmt.height = best_height;
+	} else {
+		/* use existing values as default */
+	}
+
+	vip_dbg(3, stream, "try_fmt best subdev size: %dx%d\n",
+		port->try_mbus_framefmt.width,
+		port->try_mbus_framefmt.height);
+
+	if (is_scaler_available(port) &&
+	    csc_direction != VIP_CSC_Y2R &&
+	    !vip_is_mbuscode_raw(fmt->code) &&
+	    f->fmt.pix.height <= port->try_mbus_framefmt.height &&
+	    port->try_mbus_framefmt.height <= SC_MAX_PIXEL_HEIGHT &&
+	    port->try_mbus_framefmt.width <= SC_MAX_PIXEL_WIDTH) {
+		/*
+		 * Scaler is only accessible if the dst colorspace is YUV.
+		 * As the input to the scaler must be in YUV mode only.
+		 *
+		 * Scaling up is allowed only horizontally.
+		 */
+		unsigned int hratio, vratio, width_align, height_align;
+		u32 bpp = fmt->vpdma_fmt[0]->depth >> 3;
+
+		vip_dbg(3, stream, "Scaler active on Port %c: requesting %dx%d\n",
+			port->port_id == VIP_PORTA ? 'A' : 'B',
+			f->fmt.pix.width, f->fmt.pix.height);
+
+		/* Just make sure everything is properly aligned */
+		width_align = ALIGN(f->fmt.pix.width * bpp, VPDMA_STRIDE_ALIGN);
+		width_align /= bpp;
+		height_align = ALIGN(f->fmt.pix.height, 2);
+
+		f->fmt.pix.width = width_align;
+		f->fmt.pix.height = height_align;
+
+		hratio = f->fmt.pix.width * 1000 /
+			 port->try_mbus_framefmt.width;
+		vratio = f->fmt.pix.height * 1000 /
+			 port->try_mbus_framefmt.height;
+		if (hratio < 125) {
+			f->fmt.pix.width = port->try_mbus_framefmt.width / 8;
+			vip_dbg(3, stream, "Horizontal scaling ratio out of range adjusting -> %d\n",
+				f->fmt.pix.width);
+		}
+
+		if (vratio < 188) {
+			f->fmt.pix.height = port->try_mbus_framefmt.height / 4;
+			vip_dbg(3, stream, "Vertical scaling ratio out of range adjusting -> %d\n",
+				f->fmt.pix.height);
+		}
+		vip_dbg(3, stream, "Scaler: got %dx%d\n",
+			f->fmt.pix.width, f->fmt.pix.height);
+	} else {
+		/* use existing values as default */
+		f->fmt.pix.width = port->try_mbus_framefmt.width;
+		f->fmt.pix.height = port->try_mbus_framefmt.height;
+	}
+
+calc_size:
+	/* That we have a fmt calculate imagesize and bytesperline */
+	return vip_calc_format_size(port, fmt, f);
+}
+
+static int vip_g_fmt_vid_cap(struct file *file, void *priv,
+			     struct v4l2_format *f)
+{
+	struct vip_stream *stream = file2stream(file);
+	struct vip_port *port = stream->port;
+	struct vip_fmt *fmt = port->fmt;
+
+	/* Use last known values or defaults */
+	f->fmt.pix.width	= stream->width;
+	f->fmt.pix.height	= stream->height;
+	f->fmt.pix.pixelformat	= port->fmt->fourcc;
+	f->fmt.pix.field	= stream->sup_field;
+	f->fmt.pix.colorspace	= port->fmt->colorspace;
+	f->fmt.pix.bytesperline	= stream->bytesperline;
+	f->fmt.pix.sizeimage	= stream->sizeimage;
+
+	vip_dbg(3, stream,
+		"g_fmt fourcc:%s code: %04x size: %dx%d bpl:%d img_size:%d\n",
+		fourcc_to_str(f->fmt.pix.pixelformat),
+		fmt->code,
+		f->fmt.pix.width, f->fmt.pix.height,
+		f->fmt.pix.bytesperline, f->fmt.pix.sizeimage);
+	vip_dbg(3, stream, "g_fmt vpdma data type: 0x%02X\n",
+		port->fmt->vpdma_fmt[0]->data_type);
+
+	return 0;
+}
+
+static int vip_s_fmt_vid_cap(struct file *file, void *priv,
+			     struct v4l2_format *f)
+{
+	struct vip_stream *stream = file2stream(file);
+	struct vip_port *port = stream->port;
+	struct v4l2_subdev_format sfmt;
+	struct v4l2_mbus_framefmt *mf;
+	enum vip_csc_state csc_direction;
+	int ret;
+
+	vip_dbg(3, stream, "s_fmt input fourcc:%s size: %dx%d bpl:%d img_size:%d\n",
+		fourcc_to_str(f->fmt.pix.pixelformat),
+		f->fmt.pix.width, f->fmt.pix.height,
+		f->fmt.pix.bytesperline, f->fmt.pix.sizeimage);
+
+	ret = vip_try_fmt_vid_cap(file, priv, f);
+	if (ret)
+		return ret;
+
+	vip_dbg(3, stream, "s_fmt try_fmt fourcc:%s size: %dx%d bpl:%d img_size:%d\n",
+		fourcc_to_str(f->fmt.pix.pixelformat),
+		f->fmt.pix.width, f->fmt.pix.height,
+		f->fmt.pix.bytesperline, f->fmt.pix.sizeimage);
+
+	if (vb2_is_busy(&stream->vb_vidq)) {
+		vip_err(stream, "%s queue busy\n", __func__);
+		return -EBUSY;
+	}
+
+	/*
+	 * Check if we need the scaler or not
+	 *
+	 * Since on previous S_FMT call the scaler might have been
+	 * allocated if it is not needed in this instance we will
+	 * attempt to free it just in case.
+	 *
+	 * free_scaler() is harmless unless the current port
+	 * allocated it.
+	 */
+	if (f->fmt.pix.width == port->try_mbus_framefmt.width &&
+	    f->fmt.pix.height == port->try_mbus_framefmt.height)
+		free_scaler(port);
+	else
+		allocate_scaler(port);
+
+	port->fmt = find_port_format_by_pix(port,
+					    f->fmt.pix.pixelformat);
+	stream->width		= f->fmt.pix.width;
+	stream->height		= f->fmt.pix.height;
+	stream->bytesperline	= f->fmt.pix.bytesperline;
+	stream->sizeimage	= f->fmt.pix.sizeimage;
+	stream->sup_field	= f->fmt.pix.field;
+
+	port->c_rect.left	= 0;
+	port->c_rect.top	= 0;
+	port->c_rect.width	= stream->width;
+	port->c_rect.height	= stream->height;
+
+	/*
+	 * Check if we need the csc unit or not
+	 *
+	 * Since on previous S_FMT call, the csc might have been
+	 * allocated if it is not needed in this instance we will
+	 * attempt to free it just in case.
+	 *
+	 * free_csc() is harmless unless the current port
+	 * allocated it.
+	 */
+	csc_direction =  vip_csc_direction(port->fmt->code, port->fmt->fourcc);
+	if (csc_direction == VIP_CSC_NA)
+		free_csc(port);
+	else
+		allocate_csc(port, csc_direction);
+
+	if (stream->sup_field == V4L2_FIELD_ALTERNATE)
+		port->flags |= FLAG_INTERLACED;
+	else
+		port->flags &= ~FLAG_INTERLACED;
+
+	vip_dbg(3, stream, "s_fmt fourcc:%s size: %dx%d bpl:%d img_size:%d\n",
+		fourcc_to_str(f->fmt.pix.pixelformat),
+		f->fmt.pix.width, f->fmt.pix.height,
+		f->fmt.pix.bytesperline, f->fmt.pix.sizeimage);
+
+	mf = &sfmt.format;
+	v4l2_fill_mbus_format(mf, &f->fmt.pix, port->fmt->code);
+	/* Make sure to use the subdev size found in the try_fmt */
+	mf->width = port->try_mbus_framefmt.width;
+	mf->height = port->try_mbus_framefmt.height;
+
+	vip_dbg(3, stream, "s_fmt pix_to_mbus mbus_code: %04X size: %dx%d\n",
+		mf->code,
+		mf->width, mf->height);
+
+	sfmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+	sfmt.pad = 0;
+	ret = v4l2_subdev_call(port->subdev, pad, set_fmt, NULL, &sfmt);
+	if (ret) {
+		vip_dbg(1, stream, "set_fmt failed in subdev\n");
+		return ret;
+	}
+
+	/* Save it */
+	port->mbus_framefmt = *mf;
+
+	vip_dbg(3, stream, "s_fmt subdev fmt mbus_code: %04X size: %dx%d\n",
+		port->mbus_framefmt.code,
+		port->mbus_framefmt.width, port->mbus_framefmt.height);
+	vip_dbg(3, stream, "s_fmt vpdma data type: 0x%02X\n",
+		port->fmt->vpdma_fmt[0]->data_type);
+
+	return 0;
+}
+
+/*
+ * Does the exact opposite of set_fmt_params
+ * It makes sure the DataPath register is sane after tear down
+ */
+static void unset_fmt_params(struct vip_stream *stream)
+{
+	struct vip_dev *dev = stream->port->dev;
+	struct vip_port *port = stream->port;
+
+	stream->sequence = 0;
+	stream->field = V4L2_FIELD_TOP;
+
+	if (port->csc == VIP_CSC_Y2R) {
+		if (port->port_id == VIP_PORTA) {
+			vip_set_slice_path(dev, VIP_CSC_SRC_DATA_SELECT, 0);
+			vip_set_slice_path(dev,
+					   VIP_MULTI_CHANNEL_DATA_SELECT, 0);
+			vip_set_slice_path(dev, VIP_RGB_OUT_HI_DATA_SELECT, 0);
+			vip_set_slice_path(dev, VIP_RGB_SRC_DATA_SELECT, 0);
+		} else {
+			vip_set_slice_path(dev, VIP_CSC_SRC_DATA_SELECT, 0);
+			vip_set_slice_path(dev,
+					   VIP_MULTI_CHANNEL_DATA_SELECT, 0);
+			vip_set_slice_path(dev, VIP_RGB_OUT_LO_DATA_SELECT, 0);
+		}
+		/* We are done */
+		return;
+	} else if (port->csc == VIP_CSC_R2Y) {
+		if (port->scaler && port->fmt->coplanar) {
+			if (port->port_id == VIP_PORTA) {
+				vip_set_slice_path(dev,
+						   VIP_CSC_SRC_DATA_SELECT, 0);
+				vip_set_slice_path(dev,
+						   VIP_SC_SRC_DATA_SELECT, 0);
+				vip_set_slice_path(dev,
+						   VIP_CHR_DS_1_SRC_DATA_SELECT,
+						   0);
+				vip_set_slice_path(dev,
+						   VIP_CHR_DS_1_DATA_BYPASS, 0);
+				vip_set_slice_path(dev,
+						   VIP_RGB_OUT_HI_DATA_SELECT,
+						   0);
+			}
+		} else if (port->scaler) {
+			if (port->port_id == VIP_PORTA) {
+				vip_set_slice_path(dev,
+						   VIP_CSC_SRC_DATA_SELECT, 0);
+				vip_set_slice_path(dev,
+						   VIP_SC_SRC_DATA_SELECT, 0);
+				vip_set_slice_path(dev,
+						   VIP_CHR_DS_1_SRC_DATA_SELECT,
+						   0);
+				vip_set_slice_path(dev,
+						   VIP_CHR_DS_1_DATA_BYPASS, 0);
+				vip_set_slice_path(dev,
+						   VIP_RGB_OUT_HI_DATA_SELECT,
+						   0);
+			}
+		} else if (port->fmt->coplanar) {
+			if (port->port_id == VIP_PORTA) {
+				vip_set_slice_path(dev,
+						   VIP_CSC_SRC_DATA_SELECT, 0);
+				vip_set_slice_path(dev,
+						   VIP_CHR_DS_1_SRC_DATA_SELECT,
+						   0);
+				vip_set_slice_path(dev,
+						   VIP_CHR_DS_1_DATA_BYPASS, 0);
+				vip_set_slice_path(dev,
+						   VIP_RGB_OUT_HI_DATA_SELECT,
+						   0);
+			}
+		} else {
+			if (port->port_id == VIP_PORTA) {
+				vip_set_slice_path(dev,
+						   VIP_CSC_SRC_DATA_SELECT, 0);
+				vip_set_slice_path(dev,
+						   VIP_CHR_DS_1_SRC_DATA_SELECT,
+						   0);
+				vip_set_slice_path(dev,
+						   VIP_CHR_DS_1_DATA_BYPASS, 0);
+				vip_set_slice_path(dev,
+						   VIP_RGB_OUT_HI_DATA_SELECT,
+						   0);
+			}
+		}
+		/* We are done */
+		return;
+	} else if (vip_is_fmt_rgb(port->fmt->fourcc)) {
+		if (port->port_id == VIP_PORTA) {
+			vip_set_slice_path(dev,
+					   VIP_MULTI_CHANNEL_DATA_SELECT, 0);
+			vip_set_slice_path(dev, VIP_RGB_OUT_LO_DATA_SELECT, 0);
+		}
+		/* We are done */
+		return;
+	}
+
+	if (port->scaler && port->fmt->coplanar) {
+		if (port->port_id == VIP_PORTA) {
+			vip_set_slice_path(dev, VIP_SC_SRC_DATA_SELECT, 0);
+			vip_set_slice_path(dev,
+					   VIP_CHR_DS_1_SRC_DATA_SELECT, 0);
+			vip_set_slice_path(dev, VIP_CHR_DS_1_DATA_BYPASS, 0);
+			vip_set_slice_path(dev, VIP_RGB_OUT_HI_DATA_SELECT, 0);
+		} else {
+			vip_set_slice_path(dev, VIP_SC_SRC_DATA_SELECT, 0);
+			vip_set_slice_path(dev,
+					   VIP_CHR_DS_2_SRC_DATA_SELECT, 0);
+			vip_set_slice_path(dev, VIP_CHR_DS_1_DATA_BYPASS, 0);
+			vip_set_slice_path(dev, VIP_RGB_OUT_LO_DATA_SELECT, 0);
+			vip_set_slice_path(dev,
+					   VIP_MULTI_CHANNEL_DATA_SELECT, 0);
+		}
+	} else if (port->scaler) {
+		if (port->port_id == VIP_PORTA) {
+			vip_set_slice_path(dev, VIP_SC_SRC_DATA_SELECT, 0);
+			vip_set_slice_path(dev,
+					   VIP_CHR_DS_1_SRC_DATA_SELECT, 0);
+			vip_set_slice_path(dev, VIP_CHR_DS_1_DATA_BYPASS, 0);
+			vip_set_slice_path(dev, VIP_RGB_OUT_HI_DATA_SELECT, 0);
+		} else {
+			vip_set_slice_path(dev, VIP_SC_SRC_DATA_SELECT, 0);
+			vip_set_slice_path(dev,
+					   VIP_CHR_DS_2_SRC_DATA_SELECT, 0);
+			vip_set_slice_path(dev, VIP_CHR_DS_1_DATA_BYPASS, 0);
+			vip_set_slice_path(dev, VIP_CHR_DS_2_DATA_BYPASS, 0);
+			vip_set_slice_path(dev, VIP_RGB_OUT_HI_DATA_SELECT, 0);
+		}
+	} else if (port->fmt->coplanar) {
+		if (port->port_id == VIP_PORTA) {
+			vip_set_slice_path(dev,
+					   VIP_CHR_DS_1_SRC_DATA_SELECT, 0);
+			vip_set_slice_path(dev, VIP_CHR_DS_1_DATA_BYPASS, 0);
+			vip_set_slice_path(dev, VIP_RGB_OUT_HI_DATA_SELECT, 0);
+		} else {
+			vip_set_slice_path(dev,
+					   VIP_CHR_DS_2_SRC_DATA_SELECT, 0);
+			vip_set_slice_path(dev, VIP_CHR_DS_2_DATA_BYPASS, 0);
+			vip_set_slice_path(dev,
+					   VIP_MULTI_CHANNEL_DATA_SELECT, 0);
+			vip_set_slice_path(dev, VIP_RGB_OUT_LO_DATA_SELECT, 0);
+		}
+	} else {
+		/*
+		 * We undo all data path setting except for the multi
+		 * stream case.
+		 * Because we cannot disrupt other on-going capture if only
+		 * one stream is terminated the other might still be going
+		 */
+		vip_set_slice_path(dev, VIP_MULTI_CHANNEL_DATA_SELECT, 1);
+		vip_set_slice_path(dev, VIP_RGB_OUT_LO_DATA_SELECT, 0);
+	}
+}
+
+/*
+ * Set the registers that are modified when the video format changes.
+ */
+static void set_fmt_params(struct vip_stream *stream)
+{
+	struct vip_dev *dev = stream->port->dev;
+	struct vip_port *port = stream->port;
+
+	stream->sequence = 0;
+	stream->field = V4L2_FIELD_TOP;
+
+	if (port->csc == VIP_CSC_Y2R) {
+		port->flags &= ~FLAG_MULT_PORT;
+		/* Set alpha component in background color */
+		vpdma_set_bg_color(dev->shared->vpdma,
+				   (struct vpdma_data_format *)
+				   port->fmt->vpdma_fmt[0],
+				   0xff);
+		if (port->port_id == VIP_PORTA) {
+			/*
+			 * Input A: YUV422
+			 * Output: Y_UP/UV_UP: RGB
+			 * CSC_SRC_SELECT       = 1
+			 * RGB_OUT_HI_SELECT    = 1
+			 * RGB_SRC_SELECT       = 1
+			 * MULTI_CHANNEL_SELECT = 0
+			 */
+			vip_set_slice_path(dev, VIP_CSC_SRC_DATA_SELECT, 1);
+			vip_set_slice_path(dev,
+					   VIP_MULTI_CHANNEL_DATA_SELECT, 0);
+			vip_set_slice_path(dev, VIP_RGB_OUT_HI_DATA_SELECT, 1);
+			vip_set_slice_path(dev, VIP_RGB_SRC_DATA_SELECT, 1);
+		} else {
+			/*
+			 * Input B: YUV422
+			 * Output: Y_UP/UV_UP: RGB
+			 * CSC_SRC_SELECT       = 2
+			 * RGB_OUT_LO_SELECT    = 1
+			 * MULTI_CHANNEL_SELECT = 0
+			 */
+			vip_set_slice_path(dev, VIP_CSC_SRC_DATA_SELECT, 2);
+			vip_set_slice_path(dev,
+					   VIP_MULTI_CHANNEL_DATA_SELECT, 0);
+			vip_set_slice_path(dev, VIP_RGB_OUT_LO_DATA_SELECT, 1);
+		}
+		/* We are done */
+		return;
+	} else if (port->csc == VIP_CSC_R2Y) {
+		port->flags &= ~FLAG_MULT_PORT;
+		if (port->scaler && port->fmt->coplanar) {
+			if (port->port_id == VIP_PORTA) {
+				/*
+				 * Input A: RGB
+				 * Output: Y_UP/UV_UP: Scaled YUV420
+				 * CSC_SRC_SELECT       = 4
+				 * SC_SRC_SELECT        = 1
+				 * CHR_DS_1_SRC_SELECT  = 1
+				 * CHR_DS_1_BYPASS      = 0
+				 * RGB_OUT_HI_SELECT    = 0
+				 */
+				vip_set_slice_path(dev,
+						   VIP_CSC_SRC_DATA_SELECT, 4);
+				vip_set_slice_path(dev,
+						   VIP_SC_SRC_DATA_SELECT, 1);
+				vip_set_slice_path(dev,
+						   VIP_CHR_DS_1_SRC_DATA_SELECT,
+						   1);
+				vip_set_slice_path(dev,
+						   VIP_CHR_DS_1_DATA_BYPASS, 0);
+				vip_set_slice_path(dev,
+						   VIP_RGB_OUT_HI_DATA_SELECT,
+						   0);
+			} else {
+				vip_err(stream, "RGB sensor can only be on Port A\n");
+			}
+		} else if (port->scaler) {
+			if (port->port_id == VIP_PORTA) {
+				/*
+				 * Input A: RGB
+				 * Output: Y_UP: Scaled YUV422
+				 * CSC_SRC_SELECT       = 4
+				 * SC_SRC_SELECT        = 1
+				 * CHR_DS_1_SRC_SELECT  = 1
+				 * CHR_DS_1_BYPASS      = 1
+				 * RGB_OUT_HI_SELECT    = 0
+				 */
+				vip_set_slice_path(dev,
+						   VIP_CSC_SRC_DATA_SELECT, 4);
+				vip_set_slice_path(dev,
+						   VIP_SC_SRC_DATA_SELECT, 1);
+				vip_set_slice_path(dev,
+						   VIP_CHR_DS_1_SRC_DATA_SELECT,
+						   1);
+				vip_set_slice_path(dev,
+						   VIP_CHR_DS_1_DATA_BYPASS, 1);
+				vip_set_slice_path(dev,
+						   VIP_RGB_OUT_HI_DATA_SELECT,
+						   0);
+			} else {
+				vip_err(stream, "RGB sensor can only be on Port A\n");
+			}
+		} else if (port->fmt->coplanar) {
+			if (port->port_id == VIP_PORTA) {
+				/*
+				 * Input A: RGB
+				 * Output: Y_UP/UV_UP: YUV420
+				 * CSC_SRC_SELECT       = 4
+				 * CHR_DS_1_SRC_SELECT  = 2
+				 * CHR_DS_1_BYPASS      = 0
+				 * RGB_OUT_HI_SELECT    = 0
+				 */
+				vip_set_slice_path(dev,
+						   VIP_CSC_SRC_DATA_SELECT, 4);
+				vip_set_slice_path(dev,
+						   VIP_CHR_DS_1_SRC_DATA_SELECT,
+						   2);
+				vip_set_slice_path(dev,
+						   VIP_CHR_DS_1_DATA_BYPASS, 0);
+				vip_set_slice_path(dev,
+						   VIP_RGB_OUT_HI_DATA_SELECT,
+						   0);
+			} else {
+				vip_err(stream, "RGB sensor can only be on Port A\n");
+			}
+		} else {
+			if (port->port_id == VIP_PORTA) {
+				/*
+				 * Input A: RGB
+				 * Output: Y_UP/UV_UP: YUV420
+				 * CSC_SRC_SELECT       = 4
+				 * CHR_DS_1_SRC_SELECT  = 2
+				 * CHR_DS_1_BYPASS      = 1
+				 * RGB_OUT_HI_SELECT    = 0
+				 */
+				vip_set_slice_path(dev,
+						   VIP_CSC_SRC_DATA_SELECT, 4);
+				vip_set_slice_path(dev,
+						   VIP_CHR_DS_1_SRC_DATA_SELECT,
+						   2);
+				vip_set_slice_path(dev,
+						   VIP_CHR_DS_1_DATA_BYPASS, 1);
+				vip_set_slice_path(dev,
+						   VIP_RGB_OUT_HI_DATA_SELECT,
+						   0);
+			} else {
+				vip_err(stream, "RGB sensor can only be on Port A\n");
+			}
+		}
+		/* We are done */
+		return;
+	} else if (vip_is_fmt_rgb(port->fmt->fourcc)) {
+		port->flags &= ~FLAG_MULT_PORT;
+		/* Set alpha component in background color */
+		vpdma_set_bg_color(dev->shared->vpdma,
+				   (struct vpdma_data_format *)
+				   port->fmt->vpdma_fmt[0],
+				   0xff);
+		if (port->port_id == VIP_PORTA) {
+			/*
+			 * Input A: RGB
+			 * Output: Y_LO/UV_LO: RGB
+			 * RGB_OUT_LO_SELECT    = 1
+			 * MULTI_CHANNEL_SELECT = 1
+			 */
+			vip_set_slice_path(dev,
+					   VIP_MULTI_CHANNEL_DATA_SELECT, 1);
+			vip_set_slice_path(dev, VIP_RGB_OUT_LO_DATA_SELECT, 1);
+		} else {
+			vip_err(stream, "RGB sensor can only be on Port A\n");
+		}
+		/* We are done */
+		return;
+	}
+
+	if (port->scaler && port->fmt->coplanar) {
+		port->flags &= ~FLAG_MULT_PORT;
+		if (port->port_id == VIP_PORTA) {
+			/*
+			 * Input A: YUV422
+			 * Output: Y_UP/UV_UP: Scaled YUV420
+			 * SC_SRC_SELECT        = 2
+			 * CHR_DS_1_SRC_SELECT  = 1
+			 * CHR_DS_1_BYPASS      = 0
+			 * RGB_OUT_HI_SELECT    = 0
+			 */
+			vip_set_slice_path(dev, VIP_SC_SRC_DATA_SELECT, 2);
+			vip_set_slice_path(dev,
+					   VIP_CHR_DS_1_SRC_DATA_SELECT, 1);
+			vip_set_slice_path(dev, VIP_CHR_DS_1_DATA_BYPASS, 0);
+			vip_set_slice_path(dev, VIP_RGB_OUT_HI_DATA_SELECT, 0);
+		} else {
+			/*
+			 * Input B: YUV422
+			 * Output: Y_LO/UV_LO: Scaled YUV420
+			 * SC_SRC_SELECT        = 3
+			 * CHR_DS_2_SRC_SELECT  = 1
+			 * RGB_OUT_LO_SELECT    = 0
+			 * MULTI_CHANNEL_SELECT = 0
+			 */
+			vip_set_slice_path(dev, VIP_SC_SRC_DATA_SELECT, 3);
+			vip_set_slice_path(dev,
+					   VIP_CHR_DS_2_SRC_DATA_SELECT, 1);
+			vip_set_slice_path(dev, VIP_CHR_DS_1_DATA_BYPASS, 0);
+			vip_set_slice_path(dev, VIP_RGB_OUT_LO_DATA_SELECT, 0);
+			vip_set_slice_path(dev,
+					   VIP_MULTI_CHANNEL_DATA_SELECT, 0);
+		}
+	} else if (port->scaler) {
+		port->flags &= ~FLAG_MULT_PORT;
+		if (port->port_id == VIP_PORTA) {
+			/*
+			 * Input A: YUV422
+			 * Output: Y_UP: Scaled YUV422
+			 * SC_SRC_SELECT        = 2
+			 * CHR_DS_1_SRC_SELECT  = 1
+			 * CHR_DS_1_BYPASS      = 1
+			 * RGB_OUT_HI_SELECT    = 0
+			 */
+			vip_set_slice_path(dev, VIP_SC_SRC_DATA_SELECT, 2);
+			vip_set_slice_path(dev,
+					   VIP_CHR_DS_1_SRC_DATA_SELECT, 1);
+			vip_set_slice_path(dev, VIP_CHR_DS_1_DATA_BYPASS, 1);
+			vip_set_slice_path(dev, VIP_RGB_OUT_HI_DATA_SELECT, 0);
+		} else {
+			/*
+			 * Input B: YUV422
+			 * Output: UV_UP: Scaled YUV422
+			 * SC_SRC_SELECT        = 3
+			 * CHR_DS_2_SRC_SELECT  = 1
+			 * CHR_DS_1_BYPASS      = 1
+			 * CHR_DS_2_BYPASS      = 1
+			 * RGB_OUT_HI_SELECT    = 0
+			 */
+			vip_set_slice_path(dev, VIP_SC_SRC_DATA_SELECT, 3);
+			vip_set_slice_path(dev,
+					   VIP_CHR_DS_2_SRC_DATA_SELECT, 1);
+			vip_set_slice_path(dev, VIP_CHR_DS_1_DATA_BYPASS, 1);
+			vip_set_slice_path(dev, VIP_CHR_DS_2_DATA_BYPASS, 1);
+			vip_set_slice_path(dev, VIP_RGB_OUT_HI_DATA_SELECT, 0);
+		}
+	} else if (port->fmt->coplanar) {
+		port->flags &= ~FLAG_MULT_PORT;
+		if (port->port_id == VIP_PORTA) {
+			/*
+			 * Input A: YUV422
+			 * Output: Y_UP/UV_UP: YUV420
+			 * CHR_DS_1_SRC_SELECT  = 3
+			 * CHR_DS_1_BYPASS      = 0
+			 * RGB_OUT_HI_SELECT    = 0
+			 */
+			vip_set_slice_path(dev,
+					   VIP_CHR_DS_1_SRC_DATA_SELECT, 3);
+			vip_set_slice_path(dev, VIP_CHR_DS_1_DATA_BYPASS, 0);
+			vip_set_slice_path(dev, VIP_RGB_OUT_HI_DATA_SELECT, 0);
+		} else {
+			/*
+			 * Input B: YUV422
+			 * Output: Y_LO/UV_LO: YUV420
+			 * CHR_DS_2_SRC_SELECT  = 4
+			 * CHR_DS_2_BYPASS      = 0
+			 * RGB_OUT_LO_SELECT    = 0
+			 * MULTI_CHANNEL_SELECT = 0
+			 */
+			vip_set_slice_path(dev,
+					   VIP_CHR_DS_2_SRC_DATA_SELECT, 4);
+			vip_set_slice_path(dev, VIP_CHR_DS_2_DATA_BYPASS, 0);
+			vip_set_slice_path(dev,
+					   VIP_MULTI_CHANNEL_DATA_SELECT, 0);
+			vip_set_slice_path(dev, VIP_RGB_OUT_LO_DATA_SELECT, 0);
+		}
+	} else {
+		port->flags |= FLAG_MULT_PORT;
+		/*
+		 * Input A/B: YUV422
+		 * Output: Y_LO: YUV422 - UV_LO: YUV422
+		 * MULTI_CHANNEL_SELECT = 1
+		 * RGB_OUT_LO_SELECT    = 0
+		 */
+		vip_set_slice_path(dev, VIP_MULTI_CHANNEL_DATA_SELECT, 1);
+		vip_set_slice_path(dev, VIP_RGB_OUT_LO_DATA_SELECT, 0);
+	}
+}
+
+static int vip_g_selection(struct file *file, void *fh,
+			   struct v4l2_selection *s)
+{
+	struct vip_stream *stream = file2stream(file);
+
+	switch (s->target) {
+	case V4L2_SEL_TGT_COMPOSE_DEFAULT:
+	case V4L2_SEL_TGT_COMPOSE_BOUNDS:
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+		s->r.left = 0;
+		s->r.top = 0;
+		s->r.width = stream->width;
+		s->r.height = stream->height;
+		return 0;
+
+	case V4L2_SEL_TGT_COMPOSE:
+	case V4L2_SEL_TGT_CROP:
+		s->r = stream->port->c_rect;
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+static int enclosed_rectangle(struct v4l2_rect *a, struct v4l2_rect *b)
+{
+	if (a->left < b->left || a->top < b->top)
+		return 0;
+	if (a->left + a->width > b->left + b->width)
+		return 0;
+	if (a->top + a->height > b->top + b->height)
+		return 0;
+
+	return 1;
+}
+
+static int vip_s_selection(struct file *file, void *fh,
+			   struct v4l2_selection *s)
+{
+	struct vip_stream *stream = file2stream(file);
+	struct v4l2_rect r = s->r;
+
+	v4l_bound_align_image(&r.width, 0, stream->width, 0,
+			      &r.height, 0, stream->height, 0, 0);
+
+	r.left = clamp_t(unsigned int, r.left, 0, stream->width - r.width);
+	r.top  = clamp_t(unsigned int, r.top, 0, stream->height - r.height);
+
+	if (s->flags & V4L2_SEL_FLAG_LE && !enclosed_rectangle(&r, &s->r))
+		return -ERANGE;
+
+	if (s->flags & V4L2_SEL_FLAG_GE && !enclosed_rectangle(&s->r, &r))
+		return -ERANGE;
+
+	s->r = r;
+	stream->port->c_rect = r;
+
+	vip_dbg(1, stream, "cropped (%d,%d)/%dx%d of %dx%d\n",
+		r.left, r.top, r.width, r.height,
+		stream->width, stream->height);
+
+	return 0;
+}
+
+static long vip_ioctl_default(struct file *file, void *fh, bool valid_prio,
+			      unsigned int cmd, void *arg)
+{
+	struct vip_stream *stream = file2stream(file);
+
+	if (!valid_prio) {
+		vip_err(stream, "%s device busy\n", __func__);
+		return -EBUSY;
+	}
+
+	switch (cmd) {
+	default:
+		return -ENOTTY;
+	}
+}
+
+static const struct v4l2_ioctl_ops vip_ioctl_ops = {
+	.vidioc_querycap	= vip_querycap,
+	.vidioc_enum_input	= vip_enuminput,
+	.vidioc_g_input		= vip_g_input,
+	.vidioc_s_input		= vip_s_input,
+
+	.vidioc_querystd	= vip_querystd,
+	.vidioc_g_std		= vip_g_std,
+	.vidioc_s_std		= vip_s_std,
+
+	.vidioc_enum_fmt_vid_cap = vip_enum_fmt_vid_cap,
+	.vidioc_g_fmt_vid_cap	= vip_g_fmt_vid_cap,
+	.vidioc_try_fmt_vid_cap	= vip_try_fmt_vid_cap,
+	.vidioc_s_fmt_vid_cap	= vip_s_fmt_vid_cap,
+
+	.vidioc_enum_frameintervals	= vip_enum_frameintervals,
+	.vidioc_enum_framesizes		= vip_enum_framesizes,
+	.vidioc_s_parm			= vip_s_parm,
+	.vidioc_g_parm			= vip_g_parm,
+	.vidioc_g_selection	= vip_g_selection,
+	.vidioc_s_selection	= vip_s_selection,
+	.vidioc_reqbufs		= vb2_ioctl_reqbufs,
+	.vidioc_create_bufs	= vb2_ioctl_create_bufs,
+	.vidioc_prepare_buf	= vb2_ioctl_prepare_buf,
+	.vidioc_querybuf	= vb2_ioctl_querybuf,
+	.vidioc_qbuf		= vb2_ioctl_qbuf,
+	.vidioc_dqbuf		= vb2_ioctl_dqbuf,
+	.vidioc_expbuf		= vb2_ioctl_expbuf,
+
+	.vidioc_streamon	= vb2_ioctl_streamon,
+	.vidioc_streamoff	= vb2_ioctl_streamoff,
+	.vidioc_log_status	= v4l2_ctrl_log_status,
+	.vidioc_subscribe_event = v4l2_ctrl_subscribe_event,
+	.vidioc_unsubscribe_event = v4l2_event_unsubscribe,
+	.vidioc_default		= vip_ioctl_default,
+};
+
+/*
+ * Videobuf operations
+ */
+static int vip_queue_setup(struct vb2_queue *vq,
+			   unsigned int *nbuffers, unsigned int *nplanes,
+			   unsigned int sizes[], struct device *alloc_devs[])
+{
+	struct vip_stream *stream = vb2_get_drv_priv(vq);
+	unsigned int size = stream->sizeimage;
+
+	if (vq->num_buffers + *nbuffers < 3)
+		*nbuffers = 3 - vq->num_buffers;
+
+	if (*nplanes) {
+		if (sizes[0] < size)
+			return -EINVAL;
+		size = sizes[0];
+	}
+
+	*nplanes = 1;
+	sizes[0] = size;
+
+	vip_dbg(1, stream, "get %d buffer(s) of size %d each.\n",
+		*nbuffers, sizes[0]);
+
+	return 0;
+}
+
+static int vip_buf_prepare(struct vb2_buffer *vb)
+{
+	struct vip_stream *stream = vb2_get_drv_priv(vb->vb2_queue);
+
+	if (vb2_plane_size(vb, 0) < stream->sizeimage) {
+		vip_dbg(1, stream,
+			"%s data will not fit into plane (%lu < %lu)\n",
+			__func__, vb2_plane_size(vb, 0),
+			(long)stream->sizeimage);
+		return -EINVAL;
+	}
+
+	vb2_set_plane_payload(vb, 0, stream->sizeimage);
+
+	return 0;
+}
+
+static void vip_buf_queue(struct vb2_buffer *vb)
+{
+	struct vip_stream *stream = vb2_get_drv_priv(vb->vb2_queue);
+	struct vip_dev *dev = stream->port->dev;
+	struct vip_buffer *buf = container_of(vb, struct vip_buffer,
+					      vb.vb2_buf);
+	unsigned long flags;
+
+	spin_lock_irqsave(&dev->slock, flags);
+	list_add_tail(&buf->list, &stream->vidq);
+	spin_unlock_irqrestore(&dev->slock, flags);
+}
+
+static int vip_setup_scaler(struct vip_stream *stream)
+{
+	struct vip_port *port = stream->port;
+	struct vip_dev *dev = port->dev;
+	struct sc_data *sc = dev->sc;
+	struct csc_data *csc = dev->csc;
+	struct vpdma_data *vpdma = dev->shared->vpdma;
+	struct vip_mmr_adb *mmr_adb = port->mmr_adb.addr;
+	int list_num = stream->list_num;
+	int timeout = 500;
+
+	/* if scaler not associated with this port then skip */
+	if (port->scaler) {
+		sc_set_hs_coeffs(sc, port->sc_coeff_h.addr,
+				 port->mbus_framefmt.width,
+				 port->c_rect.width);
+		sc_set_vs_coeffs(sc, port->sc_coeff_v.addr,
+				 port->mbus_framefmt.height,
+				 port->c_rect.height);
+		sc_config_scaler(sc, &mmr_adb->sc_regs0[0],
+				 &mmr_adb->sc_regs8[0], &mmr_adb->sc_regs17[0],
+				 port->mbus_framefmt.width,
+				 port->mbus_framefmt.height,
+				 port->c_rect.width,
+				 port->c_rect.height);
+		port->load_mmrs = true;
+	}
+
+	/* if csc not associated with this port then skip */
+	if (port->csc) {
+		csc_set_coeff(csc, &mmr_adb->csc_regs[0],
+			      vip_code_to_colorspace(port->fmt->code),
+			      vip_fourcc_to_colorspace(port->fmt->fourcc));
+		port->load_mmrs = true;
+	}
+
+	/* If coeff are already loaded then skip */
+	if (!sc->load_coeff_v && !sc->load_coeff_h && !port->load_mmrs)
+		return 0;
+
+	if (vpdma_list_busy(vpdma, list_num)) {
+		vip_dbg(3, stream, "%s: List %d is busy\n",
+			__func__, list_num);
+	}
+
+	/* Make sure we start with a clean list */
+	vpdma_reset_desc_list(&stream->desc_list);
+
+	/* config descriptors */
+	if (port->load_mmrs) {
+		vpdma_map_desc_buf(vpdma, &port->mmr_adb);
+		vpdma_add_cfd_adb(&stream->desc_list, CFD_MMR_CLIENT,
+				  &port->mmr_adb);
+
+		port->load_mmrs = false;
+		vip_dbg(3, stream, "Added mmr_adb config desc\n");
+	}
+
+	if (sc->loaded_coeff_h != port->sc_coeff_h.dma_addr ||
+	    sc->load_coeff_h) {
+		vpdma_map_desc_buf(vpdma, &port->sc_coeff_h);
+		vpdma_add_cfd_block(&stream->desc_list,
+				    VIP_SLICE1_CFD_SC_CLIENT + dev->slice_id,
+				    &port->sc_coeff_h, 0);
+
+		sc->loaded_coeff_h = port->sc_coeff_h.dma_addr;
+		sc->load_coeff_h = false;
+		vip_dbg(3, stream, "Added sc_coeff_h config desc\n");
+	}
+
+	if (sc->loaded_coeff_v != port->sc_coeff_v.dma_addr ||
+	    sc->load_coeff_v) {
+		vpdma_map_desc_buf(vpdma, &port->sc_coeff_v);
+		vpdma_add_cfd_block(&stream->desc_list,
+				    VIP_SLICE1_CFD_SC_CLIENT + dev->slice_id,
+				    &port->sc_coeff_v, SC_COEF_SRAM_SIZE >> 4);
+
+		sc->loaded_coeff_v = port->sc_coeff_v.dma_addr;
+		sc->load_coeff_v = false;
+		vip_dbg(3, stream, "Added sc_coeff_v config desc\n");
+	}
+	vip_dbg(3, stream, "CFD_SC_CLIENT %d slice_id: %d\n",
+		VIP_SLICE1_CFD_SC_CLIENT + dev->slice_id, dev->slice_id);
+
+	vpdma_map_desc_buf(vpdma, &stream->desc_list.buf);
+	vip_dbg(3, stream, "Submitting desc on list# %d\n", list_num);
+	vpdma_submit_descs(vpdma, &stream->desc_list, list_num);
+
+	while (vpdma_list_busy(vpdma, list_num) && timeout--)
+		usleep_range(1000, 1100);
+
+	vpdma_unmap_desc_buf(dev->shared->vpdma, &port->mmr_adb);
+	vpdma_unmap_desc_buf(dev->shared->vpdma, &port->sc_coeff_h);
+	vpdma_unmap_desc_buf(dev->shared->vpdma, &port->sc_coeff_v);
+	vpdma_unmap_desc_buf(dev->shared->vpdma, &stream->desc_list.buf);
+
+	vpdma_reset_desc_list(&stream->desc_list);
+
+	if (timeout <= 0) {
+		vip_err(stream, "Timed out setting up scaler through VPDMA list\n");
+		return -EBUSY;
+	}
+
+	return 0;
+}
+
+static int vip_load_vpdma_list_fifo(struct vip_stream *stream)
+{
+	struct vip_port *port = stream->port;
+	struct vip_dev *dev = port->dev;
+	struct vpdma_data *vpdma = dev->shared->vpdma;
+	int list_num = stream->list_num;
+	struct vip_buffer *buf;
+	unsigned long flags;
+	int timeout, i;
+
+	if (vpdma_list_busy(dev->shared->vpdma, stream->list_num))
+		return -EBUSY;
+
+	for (i = 0; i < VIP_VPDMA_FIFO_SIZE; i++) {
+		spin_lock_irqsave(&dev->slock, flags);
+		if (list_empty(&stream->vidq)) {
+			vip_err(stream, "No buffer left!");
+			spin_unlock_irqrestore(&dev->slock, flags);
+			return -EINVAL;
+		}
+
+		buf = list_entry(stream->vidq.next,
+				 struct vip_buffer, list);
+		buf->drop = false;
+
+		list_move_tail(&buf->list, &stream->post_bufs);
+		spin_unlock_irqrestore(&dev->slock, flags);
+
+		vip_dbg(2, stream, "%s: start_dma vb2 buf idx:%d\n",
+			__func__, buf->vb.vb2_buf.index);
+		start_dma(stream, buf);
+
+		timeout = 500;
+		while (vpdma_list_busy(vpdma, list_num) && timeout--)
+			usleep_range(1000, 1100);
+
+		if (timeout <= 0) {
+			vip_err(stream, "Timed out loading VPDMA list fifo\n");
+			return -EBUSY;
+		}
+	}
+	return 0;
+}
+
+static int vip_start_streaming(struct vb2_queue *vq, unsigned int count)
+{
+	struct vip_stream *stream = vb2_get_drv_priv(vq);
+	struct vip_port *port = stream->port;
+	struct vip_dev *dev = port->dev;
+	int ret;
+
+	vip_setup_scaler(stream);
+
+	/*
+	 * Make sure the scaler is configured before the datapath is
+	 * enabled. The scaler can only load the coefficient
+	 * parameters when it is idle. If the scaler path is enabled
+	 * and video data is being received then the VPDMA transfer will
+	 * stall indefinetely.
+	 */
+	set_fmt_params(stream);
+	vip_setup_parser(port);
+
+	if (port->subdev) {
+		ret = v4l2_subdev_call(port->subdev, video, s_stream, 1);
+		if (ret) {
+			vip_dbg(1, stream, "stream on failed in subdev\n");
+			return ret;
+		}
+	}
+
+	stream->sequence = 0;
+	stream->field = V4L2_FIELD_TOP;
+	populate_desc_list(stream);
+
+	ret = vip_load_vpdma_list_fifo(stream);
+	if (ret)
+		return ret;
+
+	stream->num_recovery = 0;
+
+	clear_irqs(dev, dev->slice_id, stream->list_num);
+	enable_irqs(dev, dev->slice_id, stream->list_num);
+	vip_schedule_next_buffer(stream);
+	vip_parser_stop_imm(port, false);
+	vip_enable_parser(port, true);
+
+	return 0;
+}
+
+/*
+ * Abort streaming and wait for last buffer
+ */
+static void vip_stop_streaming(struct vb2_queue *vq)
+{
+	struct vip_stream *stream = vb2_get_drv_priv(vq);
+	struct vip_port *port = stream->port;
+	struct vip_dev *dev = port->dev;
+	struct vip_buffer *buf;
+	int ret;
+
+	vip_dbg(2, stream, "%s:\n", __func__);
+
+	vip_parser_stop_imm(port, true);
+	vip_enable_parser(port, false);
+	unset_fmt_params(stream);
+
+	disable_irqs(dev, dev->slice_id, stream->list_num);
+	clear_irqs(dev, dev->slice_id, stream->list_num);
+
+	if (port->subdev) {
+		ret = v4l2_subdev_call(port->subdev, video, s_stream, 0);
+		if (ret)
+			vip_dbg(1, stream, "stream on failed in subdev\n");
+	}
+
+	stop_dma(stream, true);
+
+	/* release all active buffers */
+	while (!list_empty(&stream->post_bufs)) {
+		buf = list_entry(stream->post_bufs.next,
+				 struct vip_buffer, list);
+		list_del(&buf->list);
+		if (buf->drop == 1)
+			list_add_tail(&buf->list, &stream->dropq);
+		else
+			vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR);
+	}
+	while (!list_empty(&stream->vidq)) {
+		buf = list_entry(stream->vidq.next, struct vip_buffer, list);
+		list_del(&buf->list);
+		vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR);
+	}
+
+	if (!vb2_is_streaming(vq))
+		return;
+
+	vpdma_unmap_desc_buf(dev->shared->vpdma, &stream->desc_list.buf);
+	vpdma_reset_desc_list(&stream->desc_list);
+}
+
+static const struct vb2_ops vip_video_qops = {
+	.queue_setup		= vip_queue_setup,
+	.buf_prepare		= vip_buf_prepare,
+	.buf_queue		= vip_buf_queue,
+	.start_streaming	= vip_start_streaming,
+	.stop_streaming		= vip_stop_streaming,
+	.wait_prepare		= vb2_ops_wait_prepare,
+	.wait_finish		= vb2_ops_wait_finish,
+};
+
+/*
+ * File operations
+ */
+
+static int vip_init_dev(struct vip_dev *dev)
+{
+	if (dev->num_ports != 0)
+		goto done;
+
+	vip_set_clock_enable(dev, 1);
+	vip_module_reset(dev, VIP_SC_RST, false);
+	vip_module_reset(dev, VIP_CSC_RST, false);
+done:
+	dev->num_ports++;
+
+	return 0;
+}
+
+static inline bool is_scaler_available(struct vip_port *port)
+{
+	if (port->endpoint->bus_type == V4L2_MBUS_PARALLEL)
+		if (port->dev->sc_assigned == VIP_NOT_ASSIGNED ||
+		    port->dev->sc_assigned == port->port_id)
+			return true;
+	return false;
+}
+
+static inline bool allocate_scaler(struct vip_port *port)
+{
+	if (is_scaler_available(port)) {
+		if (port->dev->sc_assigned == VIP_NOT_ASSIGNED ||
+		    port->dev->sc_assigned == port->port_id) {
+			port->dev->sc_assigned = port->port_id;
+			port->scaler = true;
+			return true;
+		}
+	}
+	return false;
+}
+
+static inline void free_scaler(struct vip_port *port)
+{
+	if (port->dev->sc_assigned == port->port_id) {
+		port->dev->sc_assigned = VIP_NOT_ASSIGNED;
+		port->scaler = false;
+	}
+}
+
+static bool is_csc_available(struct vip_port *port)
+{
+	if (port->endpoint->bus_type == V4L2_MBUS_PARALLEL)
+		if (port->dev->csc_assigned == VIP_NOT_ASSIGNED ||
+		    port->dev->csc_assigned == port->port_id)
+			return true;
+	return false;
+}
+
+static bool allocate_csc(struct vip_port *port,
+				enum vip_csc_state csc_direction)
+{
+	/* Is CSC needed? */
+	if (csc_direction != VIP_CSC_NA) {
+		if (is_csc_available(port)) {
+			port->dev->csc_assigned = port->port_id;
+			port->csc = csc_direction;
+			vip_dbg(1, port, "%s: csc allocated: dir: %d\n",
+				__func__, csc_direction);
+			return true;
+		}
+	}
+	return false;
+}
+
+static void free_csc(struct vip_port *port)
+{
+	if (port->dev->csc_assigned == port->port_id) {
+		port->dev->csc_assigned = VIP_NOT_ASSIGNED;
+		port->csc = VIP_CSC_NA;
+		vip_dbg(1, port, "%s: csc freed\n",
+			__func__);
+	}
+}
+
+static int vip_init_port(struct vip_port *port)
+{
+	int ret;
+	struct vip_fmt *fmt;
+	struct v4l2_subdev_format sd_fmt;
+	struct v4l2_mbus_framefmt *mbus_fmt = &sd_fmt.format;
+
+	if (port->num_streams != 0)
+		goto done;
+
+	ret = vip_init_dev(port->dev);
+	if (ret)
+		goto done;
+
+	/* Get subdevice current frame format */
+	sd_fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+	sd_fmt.pad = 0;
+	ret = v4l2_subdev_call(port->subdev, pad, get_fmt, NULL, &sd_fmt);
+	if (ret)
+		vip_dbg(1, port, "init_port get_fmt failed in subdev: (%d)\n",
+			ret);
+
+	/* try to find one that matches */
+	fmt = find_port_format_by_code(port, mbus_fmt->code);
+	if (!fmt) {
+		vip_dbg(1, port, "subdev default mbus_fmt %04x is not matched.\n",
+			mbus_fmt->code);
+		/* if all else fails just pick the first one */
+		fmt = port->active_fmt[0];
+
+		mbus_fmt->code = fmt->code;
+		sd_fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+		sd_fmt.pad = 0;
+		ret = v4l2_subdev_call(port->subdev, pad, set_fmt,
+				       NULL, &sd_fmt);
+		if (ret)
+			vip_dbg(1, port, "init_port set_fmt failed in subdev: (%d)\n",
+				ret);
+	}
+
+	/* Assign current format */
+	port->fmt = fmt;
+	port->mbus_framefmt = *mbus_fmt;
+
+	vip_dbg(3, port, "%s: g_mbus_fmt subdev mbus_code: %04X fourcc:%s size: %dx%d\n",
+		__func__, fmt->code,
+		fourcc_to_str(fmt->fourcc),
+		mbus_fmt->width, mbus_fmt->height);
+
+	if (mbus_fmt->field == V4L2_FIELD_ALTERNATE)
+		port->flags |= FLAG_INTERLACED;
+	else
+		port->flags &= ~FLAG_INTERLACED;
+
+	port->c_rect.left	= 0;
+	port->c_rect.top	= 0;
+	port->c_rect.width	= mbus_fmt->width;
+	port->c_rect.height	= mbus_fmt->height;
+
+	ret = vpdma_alloc_desc_buf(&port->sc_coeff_h, SC_COEF_SRAM_SIZE);
+	if (ret != 0)
+		return ret;
+
+	ret = vpdma_alloc_desc_buf(&port->sc_coeff_v, SC_COEF_SRAM_SIZE);
+	if (ret != 0)
+		goto free_sc_h;
+
+	ret = vpdma_alloc_desc_buf(&port->mmr_adb, sizeof(struct vip_mmr_adb));
+	if (ret != 0)
+		goto free_sc_v;
+
+	init_adb_hdrs(port);
+
+	vip_enable_parser(port, false);
+done:
+	port->num_streams++;
+	return 0;
+
+free_sc_v:
+	vpdma_free_desc_buf(&port->sc_coeff_v);
+free_sc_h:
+	vpdma_free_desc_buf(&port->sc_coeff_h);
+	return ret;
+}
+
+static int vip_init_stream(struct vip_stream *stream)
+{
+	struct vip_port *port = stream->port;
+	struct vip_fmt *fmt;
+	struct v4l2_mbus_framefmt *mbus_fmt;
+	struct v4l2_format f;
+	int ret;
+
+	ret = vip_init_port(port);
+	if (ret != 0)
+		return ret;
+
+	fmt = port->fmt;
+	mbus_fmt = &port->mbus_framefmt;
+
+	memset(&f, 0, sizeof(f));
+
+	/* Properly calculate the sizeimage and bytesperline values. */
+	v4l2_fill_pix_format(&f.fmt.pix, mbus_fmt);
+	f.fmt.pix.pixelformat = fmt->fourcc;
+	ret = vip_calc_format_size(port, fmt, &f);
+	if (ret)
+		return ret;
+
+	stream->width = f.fmt.pix.width;
+	stream->height = f.fmt.pix.height;
+	stream->sup_field = f.fmt.pix.field;
+	stream->bytesperline = f.fmt.pix.bytesperline;
+	stream->sizeimage = f.fmt.pix.sizeimage;
+
+	vip_dbg(3, stream, "init_stream fourcc:%s size: %dx%d bpl:%d img_size:%d\n",
+		fourcc_to_str(f.fmt.pix.pixelformat),
+		f.fmt.pix.width, f.fmt.pix.height,
+		f.fmt.pix.bytesperline, f.fmt.pix.sizeimage);
+	vip_dbg(3, stream, "init_stream vpdma data type: 0x%02X\n",
+		port->fmt->vpdma_fmt[0]->data_type);
+
+	ret = vpdma_create_desc_list(&stream->desc_list, VIP_DESC_LIST_SIZE,
+				     VPDMA_LIST_TYPE_NORMAL);
+
+	if (ret != 0)
+		return ret;
+
+	stream->write_desc = (struct vpdma_dtd *)stream->desc_list.buf.addr
+				+ 15;
+
+	vip_dbg(1, stream, "%s: stream instance %pa\n",
+		__func__, &stream);
+
+	return 0;
+}
+
+static void vip_release_dev(struct vip_dev *dev)
+{
+	/*
+	 * On last close, disable clocks to conserve power
+	 */
+
+	if (--dev->num_ports == 0) {
+		/* reset the scaler module */
+		vip_module_reset(dev, VIP_SC_RST, true);
+		vip_module_reset(dev, VIP_CSC_RST, true);
+		vip_set_clock_enable(dev, 0);
+	}
+}
+
+static int vip_set_crop_parser(struct vip_port *port)
+{
+	struct vip_dev *dev = port->dev;
+	struct vip_parser_data *parser = dev->parser;
+	u32 hcrop = 0, vcrop = 0;
+	u32 width = port->mbus_framefmt.width;
+
+	if (port->fmt->vpdma_fmt[0] == &vpdma_raw_fmts[VPDMA_DATA_FMT_RAW8]) {
+		/*
+		 * Special case since we are faking a YUV422 16bit format
+		 * to have the vpdma perform the needed byte swap
+		 * we need to adjust the pixel width accordingly
+		 * otherwise the parser will attempt to collect more pixels
+		 * then available and the vpdma transfer will exceed the
+		 * allocated frame buffer.
+		 */
+		width >>= 1;
+		vip_dbg(1, port, "%s: 8 bit raw detected, adjusting width to %d\n",
+			__func__, width);
+	}
+
+	/*
+	 * Set Parser Crop parameters to source size otherwise
+	 * scaler and colorspace converter will yield garbage.
+	 */
+	hcrop = VIP_ACT_BYPASS;
+	insert_field(&hcrop, 0, VIP_ACT_SKIP_NUMPIX_MASK,
+		     VIP_ACT_SKIP_NUMPIX_SHFT);
+	insert_field(&hcrop, width,
+		     VIP_ACT_USE_NUMPIX_MASK, VIP_ACT_USE_NUMPIX_SHFT);
+	reg_write(parser, VIP_PARSER_CROP_H_PORT(port->port_id), hcrop);
+
+	insert_field(&vcrop, 0, VIP_ACT_SKIP_NUMLINES_MASK,
+		     VIP_ACT_SKIP_NUMLINES_SHFT);
+	insert_field(&vcrop, port->mbus_framefmt.height,
+		     VIP_ACT_USE_NUMLINES_MASK, VIP_ACT_USE_NUMLINES_SHFT);
+	reg_write(parser, VIP_PARSER_CROP_V_PORT(port->port_id), vcrop);
+
+	return 0;
+}
+
+static int vip_setup_parser(struct vip_port *port)
+{
+	struct vip_dev *dev = port->dev;
+	struct vip_parser_data *parser = dev->parser;
+	struct v4l2_fwnode_endpoint *endpoint = port->endpoint;
+	int iface, sync_type;
+	u32 flags = 0, config0;
+
+	/* Reset the port */
+	vip_reset_parser(port, true);
+	usleep_range(200, 250);
+	vip_reset_parser(port, false);
+
+	config0 = reg_read(parser, VIP_PARSER_PORT(port->port_id));
+
+	if (endpoint->bus_type == V4L2_MBUS_BT656) {
+		flags = endpoint->bus.parallel.flags;
+		iface = DUAL_8B_INTERFACE;
+
+		/*
+		 * Ideally, this should come from subdev
+		 * port->fmt can be anything once CSC is enabled
+		 */
+		if (vip_is_mbuscode_rgb(port->fmt->code)) {
+			sync_type = EMBEDDED_SYNC_SINGLE_RGB_OR_YUV444;
+		} else {
+			switch (endpoint->bus.parallel.num_channels) {
+			case 4:
+				sync_type = EMBEDDED_SYNC_4X_MULTIPLEXED_YUV422;
+				break;
+			case 2:
+				sync_type = EMBEDDED_SYNC_2X_MULTIPLEXED_YUV422;
+				break;
+			case 1:
+				sync_type = EMBEDDED_SYNC_SINGLE_YUV422;
+				break;
+			default:
+				sync_type =
+				EMBEDDED_SYNC_LINE_MULTIPLEXED_YUV422;
+			}
+			if (endpoint->bus.parallel.pixmux == 0)
+				sync_type =
+				EMBEDDED_SYNC_LINE_MULTIPLEXED_YUV422;
+		}
+
+	} else if (endpoint->bus_type == V4L2_MBUS_PARALLEL) {
+		flags = endpoint->bus.parallel.flags;
+
+		switch (endpoint->bus.parallel.bus_width) {
+		case 24:
+			iface = SINGLE_24B_INTERFACE;
+		break;
+		case 16:
+			iface = SINGLE_16B_INTERFACE;
+		break;
+		case 8:
+		default:
+			iface = DUAL_8B_INTERFACE;
+		}
+
+		if (vip_is_mbuscode_rgb(port->fmt->code))
+			sync_type = DISCRETE_SYNC_SINGLE_RGB_24B;
+		else
+			sync_type = DISCRETE_SYNC_SINGLE_YUV422;
+
+		if (flags & V4L2_MBUS_HSYNC_ACTIVE_HIGH)
+			config0 |= VIP_HSYNC_POLARITY;
+		else if (flags & V4L2_MBUS_HSYNC_ACTIVE_LOW)
+			config0 &= ~VIP_HSYNC_POLARITY;
+
+		if (flags & V4L2_MBUS_VSYNC_ACTIVE_HIGH)
+			config0 |= VIP_VSYNC_POLARITY;
+		else if (flags & V4L2_MBUS_VSYNC_ACTIVE_LOW)
+			config0 &= ~VIP_VSYNC_POLARITY;
+
+		config0 &= ~VIP_USE_ACTVID_HSYNC_ONLY;
+		config0 |= VIP_ACTVID_POLARITY;
+		config0 |= VIP_DISCRETE_BASIC_MODE;
+
+	} else {
+		vip_err(port, "Device doesn't support CSI2");
+		return -EINVAL;
+	}
+
+	if (flags & V4L2_MBUS_PCLK_SAMPLE_FALLING) {
+		vip_set_pclk_invert(port);
+		config0 |= VIP_PIXCLK_EDGE_POLARITY;
+	} else {
+		config0 &= ~VIP_PIXCLK_EDGE_POLARITY;
+	}
+
+	config0 |= ((sync_type & VIP_SYNC_TYPE_MASK) << VIP_SYNC_TYPE_SHFT);
+
+	reg_write(parser, VIP_PARSER_PORT(port->port_id), config0);
+
+	vip_set_data_interface(port, iface);
+	vip_set_crop_parser(port);
+
+	return 0;
+}
+
+static void vip_enable_parser(struct vip_port *port, bool on)
+{
+	u32 config0;
+	struct vip_dev *dev = port->dev;
+	struct vip_parser_data *parser = dev->parser;
+
+	config0 = reg_read(parser, VIP_PARSER_PORT(port->port_id));
+
+	if (on) {
+		config0 |= VIP_PORT_ENABLE;
+		config0 &= ~(VIP_ASYNC_FIFO_RD | VIP_ASYNC_FIFO_WR);
+	} else {
+		config0 &= ~VIP_PORT_ENABLE;
+		config0 |= (VIP_ASYNC_FIFO_RD | VIP_ASYNC_FIFO_WR);
+	}
+	reg_write(parser, VIP_PARSER_PORT(port->port_id), config0);
+}
+
+static void vip_reset_parser(struct vip_port *port, bool on)
+{
+	u32 config0;
+	struct vip_dev *dev = port->dev;
+	struct vip_parser_data *parser = dev->parser;
+
+	config0 = reg_read(parser, VIP_PARSER_PORT(port->port_id));
+
+	if (on)
+		config0 |= VIP_SW_RESET;
+	else
+		config0 &= ~VIP_SW_RESET;
+
+	reg_write(parser, VIP_PARSER_PORT(port->port_id), config0);
+}
+
+static void vip_parser_stop_imm(struct vip_port *port, bool on)
+{
+	u32 config0;
+	struct vip_dev *dev = port->dev;
+	struct vip_parser_data *parser = dev->parser;
+
+	config0 = reg_read(parser, VIP_PARSER_STOP_IMM_PORT(port->port_id));
+
+	if (on)
+		config0 = 0xffffffff;
+	else
+		config0 = 0;
+
+	reg_write(parser, VIP_PARSER_STOP_IMM_PORT(port->port_id), config0);
+}
+
+static void vip_release_stream(struct vip_stream *stream)
+{
+	struct vip_dev *dev = stream->port->dev;
+
+	vip_dbg(1, stream, "%s: stream instance %pa\n",
+		__func__, &stream);
+
+	vpdma_unmap_desc_buf(dev->shared->vpdma, &stream->desc_list.buf);
+	vpdma_free_desc_buf(&stream->desc_list.buf);
+	vpdma_free_desc_list(&stream->desc_list);
+}
+
+static void vip_release_port(struct vip_port *port)
+{
+	vip_dbg(1, port, "%s: port instance %pa\n",
+		__func__, &port);
+
+	vpdma_free_desc_buf(&port->mmr_adb);
+	vpdma_free_desc_buf(&port->sc_coeff_h);
+	vpdma_free_desc_buf(&port->sc_coeff_v);
+}
+
+static void stop_dma(struct vip_stream *stream, bool clear_list)
+{
+	struct vip_dev *dev = stream->port->dev;
+	int ch, size = 0;
+
+	/* Create a list of channels to be cleared */
+	for (ch = 0; ch < VPDMA_MAX_CHANNELS; ch++) {
+		if (stream->vpdma_channels[ch] == 1) {
+			stream->vpdma_channels_to_abort[size++] = ch;
+			vip_dbg(2, stream, "Clear channel no: %d\n", ch);
+		}
+	}
+
+	/* Clear all the used channels for the list */
+	vpdma_list_cleanup(dev->shared->vpdma, stream->list_num,
+			   stream->vpdma_channels_to_abort, size);
+
+	if (clear_list)
+		for (ch = 0; ch < VPDMA_MAX_CHANNELS; ch++)
+			stream->vpdma_channels[ch] = 0;
+}
+
+static int vip_open(struct file *file)
+{
+	struct vip_stream *stream = video_drvdata(file);
+	struct vip_port *port = stream->port;
+	struct vip_dev *dev = port->dev;
+	int ret = 0;
+
+	vip_dbg(2, stream, "%s\n", __func__);
+
+	mutex_lock(&dev->mutex);
+
+	ret = v4l2_fh_open(file);
+	if (ret) {
+		vip_err(stream, "v4l2_fh_open failed\n");
+		goto unlock;
+	}
+
+	/*
+	 * If this is the first open file.
+	 * Then initialize hw module.
+	 */
+	if (!v4l2_fh_is_singular_file(file))
+		goto unlock;
+
+	if (vip_init_stream(stream))
+		ret = -ENODEV;
+unlock:
+	mutex_unlock(&dev->mutex);
+	return ret;
+}
+
+static int vip_release(struct file *file)
+{
+	struct vip_stream *stream = video_drvdata(file);
+	struct vip_port *port = stream->port;
+	struct vip_dev *dev = port->dev;
+	bool fh_singular;
+	int ret;
+
+	vip_dbg(2, stream, "%s\n", __func__);
+
+	mutex_lock(&dev->mutex);
+
+	/* Save the singular status before we call the clean-up helper */
+	fh_singular = v4l2_fh_is_singular_file(file);
+
+	/* the release helper will cleanup any on-going streaming */
+	ret = _vb2_fop_release(file, NULL);
+
+	free_csc(port);
+	free_scaler(port);
+
+	/*
+	 * If this is the last open file.
+	 * Then de-initialize hw module.
+	 */
+	if (fh_singular) {
+		vip_release_stream(stream);
+
+		if (--port->num_streams == 0) {
+			vip_release_port(port);
+			vip_release_dev(port->dev);
+		}
+	}
+
+	mutex_unlock(&dev->mutex);
+
+	return ret;
+}
+
+static const struct v4l2_file_operations vip_fops = {
+	.owner		= THIS_MODULE,
+	.open		= vip_open,
+	.release	= vip_release,
+	.read		= vb2_fop_read,
+	.poll		= vb2_fop_poll,
+	.unlocked_ioctl	= video_ioctl2,
+	.mmap		= vb2_fop_mmap,
+};
+
+static struct video_device vip_videodev = {
+	.name		= VIP_MODULE_NAME,
+	.fops		= &vip_fops,
+	.ioctl_ops	= &vip_ioctl_ops,
+	.minor		= -1,
+	.release	= video_device_release,
+	.tvnorms	= V4L2_STD_NTSC | V4L2_STD_PAL | V4L2_STD_SECAM,
+};
+
+static int alloc_stream(struct vip_port *port, int stream_id, int vfl_type)
+{
+	struct vip_stream *stream;
+	struct vip_dev *dev = port->dev;
+	struct vb2_queue *q;
+	struct video_device *vfd;
+	struct vip_buffer *buf;
+	struct list_head *pos, *tmp;
+	int ret, i;
+	u32 vin_id;
+
+	stream = kzalloc(sizeof(*stream), GFP_KERNEL);
+	if (!stream)
+		return -ENOMEM;
+
+	stream->port = port;
+	stream->stream_id = stream_id;
+	stream->vfl_type = vfl_type;
+
+	vin_id = 1 + ((dev->instance_id - 1) * 2) + dev->slice_id;
+	snprintf(stream->name, sizeof(stream->name), "vin%d%c-%d",
+		 vin_id, (port->port_id == VIP_PORTA) ? 'a' : 'b', stream_id);
+
+	stream->list_num = vpdma_hwlist_alloc(dev->shared->vpdma, stream);
+	if (stream->list_num < 0) {
+		vip_err(stream, "Could not get VPDMA hwlist");
+		ret = -ENODEV;
+		goto do_free_stream;
+	}
+
+	INIT_LIST_HEAD(&stream->post_bufs);
+
+	if (vfl_type == VFL_TYPE_GRABBER)
+		port->cap_streams[stream_id] = stream;
+	else
+		port->vbi_streams[stream_id] = stream;
+
+	/*
+	 * Initialize queue
+	 */
+	q = &stream->vb_vidq;
+	q->type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+	q->io_modes = VB2_MMAP | VB2_DMABUF | VB2_READ;
+	q->drv_priv = stream;
+	q->buf_struct_size = sizeof(struct vip_buffer);
+	q->ops = &vip_video_qops;
+	q->mem_ops = &vb2_dma_contig_memops;
+	q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
+	q->lock = &dev->mutex;
+	q->min_buffers_needed = 3;
+	q->dev = dev->v4l2_dev->dev;
+
+	ret = vb2_queue_init(q);
+	if (ret)
+		goto do_free_hwlist;
+
+	INIT_WORK(&stream->recovery_work, vip_overflow_recovery_work);
+
+	INIT_LIST_HEAD(&stream->vidq);
+
+	/* Allocate/populate Drop queue entries */
+	INIT_LIST_HEAD(&stream->dropq);
+	for (i = 0; i < VIP_DROPQ_SIZE; i++) {
+		buf = kzalloc(sizeof(*buf), GFP_ATOMIC);
+		if (!buf) {
+			ret = -ENOMEM;
+			goto do_free_dropq;
+		}
+		buf->drop = true;
+		list_add(&buf->list, &stream->dropq);
+	}
+
+	vfd = video_device_alloc();
+	if (!vfd)
+		goto do_free_dropq;
+	*vfd = vip_videodev;
+	vfd->v4l2_dev = dev->v4l2_dev;
+	vfd->queue = q;
+
+	vfd->lock = &dev->mutex;
+	video_set_drvdata(vfd, stream);
+
+	ret = video_register_device(vfd, vfl_type, -1);
+	if (ret) {
+		vip_err(stream, "Failed to register video device\n");
+		goto do_free_vfd;
+	}
+
+	stream->vfd = vfd;
+
+	vip_info(stream, "device registered as %s\n",
+		 video_device_node_name(vfd));
+	return 0;
+
+do_free_vfd:
+	video_device_release(vfd);
+do_free_dropq:
+	list_for_each_safe(pos, tmp, &stream->dropq) {
+		buf = list_entry(pos,
+				 struct vip_buffer, list);
+		vip_dbg(1, dev, "dropq buffer\n");
+		list_del(pos);
+		kfree(buf);
+	}
+do_free_hwlist:
+	vpdma_hwlist_release(dev->shared->vpdma, stream->list_num);
+do_free_stream:
+	kfree(stream);
+	return ret;
+}
+
+static void free_stream(struct vip_stream *stream)
+{
+	struct vip_dev *dev;
+	struct vip_buffer *buf;
+	struct list_head *pos, *q;
+
+	if (!stream)
+		return;
+
+	dev = stream->port->dev;
+	/* Free up the Drop queue */
+	list_for_each_safe(pos, q, &stream->dropq) {
+		buf = list_entry(pos,
+				 struct vip_buffer, list);
+		vip_dbg(1, stream, "dropq buffer\n");
+		list_del(pos);
+		kfree(buf);
+	}
+
+	video_unregister_device(stream->vfd);
+	video_device_release(stream->vfd);
+	vpdma_hwlist_release(dev->shared->vpdma, stream->list_num);
+	stream->port->cap_streams[stream->stream_id] = NULL;
+	kfree(stream);
+}
+
+static int get_subdev_active_format(struct vip_port *port,
+				    struct v4l2_subdev *subdev)
+{
+	struct vip_fmt *fmt;
+	struct v4l2_subdev_mbus_code_enum mbus_code;
+	int ret = 0;
+	unsigned int k, i, j;
+	enum vip_csc_state csc;
+
+	/* Enumerate sub device formats and enable all matching local formats */
+	port->num_active_fmt = 0;
+	for (k = 0, i = 0; (ret != -EINVAL); k++) {
+		memset(&mbus_code, 0, sizeof(mbus_code));
+		mbus_code.index = k;
+		ret = v4l2_subdev_call(subdev, pad, enum_mbus_code,
+				       NULL, &mbus_code);
+		if (ret)
+			continue;
+
+		vip_dbg(2, port,
+			"subdev %s: code: %04x idx: %d\n",
+			subdev->name, mbus_code.code, k);
+
+		for (j = 0; j < ARRAY_SIZE(vip_formats); j++) {
+			fmt = &vip_formats[j];
+			if (mbus_code.code != fmt->code)
+				continue;
+
+			/*
+			 * When the port is configured for BT656
+			 * then none of the downstream unit can be used.
+			 * So here we need to skip all format requiring
+			 * either CSC or CHR_DS
+			 */
+			csc = vip_csc_direction(fmt->code, fmt->fourcc);
+			if (port->endpoint->bus_type == V4L2_MBUS_BT656 &&
+			    (csc != VIP_CSC_NA || fmt->coplanar))
+				continue;
+
+			port->active_fmt[i] = fmt;
+			vip_dbg(2, port,
+				"matched fourcc: %s: code: %04x idx: %d\n",
+				fourcc_to_str(fmt->fourcc), fmt->code, i);
+			port->num_active_fmt = ++i;
+		}
+	}
+
+	if (i == 0) {
+		vip_err(port, "No suitable format reported by subdev %s\n",
+			subdev->name);
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static int alloc_port(struct vip_dev *dev, int id)
+{
+	struct vip_port *port;
+	u32 vin_id;
+
+	if (dev->ports[id])
+		return -EINVAL;
+
+	port = devm_kzalloc(&dev->pdev->dev, sizeof(*port), GFP_KERNEL);
+	if (!port)
+		return -ENOMEM;
+
+	dev->ports[id] = port;
+	port->dev = dev;
+	port->port_id = id;
+	vin_id = 1 + ((dev->instance_id - 1) * 2) + dev->slice_id;
+	snprintf(port->name, sizeof(port->name),
+		 "vin%d%c", vin_id, (id == VIP_PORTA) ? 'a' : 'b');
+	port->num_streams = 0;
+	return 0;
+}
+
+static void free_port(struct vip_port *port)
+{
+	if (!port)
+		return;
+
+	v4l2_async_notifier_unregister(&port->notifier);
+	free_stream(port->cap_streams[0]);
+}
+
+static int get_field(u32 value, u32 mask, int shift)
+{
+	return (value & (mask << shift)) >> shift;
+}
+
+static int vip_of_probe(struct platform_device *pdev);
+static void vip_vpdma_fw_cb(struct platform_device *pdev)
+{
+	dev_info(&pdev->dev, "VPDMA firmware loaded\n");
+
+	if (pdev->dev.of_node)
+		vip_of_probe(pdev);
+}
+
+static int vip_create_streams(struct vip_port *port,
+			      struct v4l2_subdev *subdev)
+{
+	struct v4l2_fwnode_bus_parallel *bus;
+	int i;
+
+	for (i = 0; i < VIP_CAP_STREAMS_PER_PORT; i++)
+		free_stream(port->cap_streams[i]);
+
+	if (get_subdev_active_format(port, subdev))
+		return -ENODEV;
+
+	port->subdev = subdev;
+
+	if (port->endpoint->bus_type == V4L2_MBUS_PARALLEL) {
+		port->flags |= FLAG_MULT_PORT;
+		port->num_streams_configured = 1;
+		alloc_stream(port, 0, VFL_TYPE_GRABBER);
+	} else if (port->endpoint->bus_type == V4L2_MBUS_BT656) {
+		port->flags |= FLAG_MULT_PORT;
+		bus = &port->endpoint->bus.parallel;
+		port->num_streams_configured = bus->num_channels;
+		for (i = 0; i < bus->num_channels; i++) {
+			if (bus->channels[i] >= 16)
+				continue;
+			alloc_stream(port, bus->channels[i], VFL_TYPE_GRABBER);
+		}
+	}
+	return 0;
+}
+
+static int vip_async_bound(struct v4l2_async_notifier *notifier,
+			   struct v4l2_subdev *subdev,
+			   struct v4l2_async_subdev *asd)
+{
+	struct vip_port *port = notifier_to_vip_port(notifier);
+	struct vip_async_config *config = &port->config;
+	unsigned int idx = asd - &config->asd[0];
+	int ret;
+
+	vip_dbg(1, port, "%s\n", __func__);
+	if (idx > config->asd_sizes)
+		return -EINVAL;
+
+	if (port->subdev) {
+		if (asd < port->subdev->asd)
+			/* Notified of a subdev earlier in the array */
+			vip_info(port, "Switching to subdev %s (High priority)",
+				 subdev->name);
+
+		else {
+			vip_info(port, "Rejecting subdev %s (Low priority)",
+				 subdev->name);
+			return 0;
+		}
+	}
+
+	port->endpoint = &config->endpoints[idx];
+	vip_info(port, "Port %c: Using subdev %s for capture\n",
+		 port->port_id == VIP_PORTA ? 'A' : 'B', subdev->name);
+
+	ret = vip_create_streams(port, subdev);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int vip_async_complete(struct v4l2_async_notifier *notifier)
+{
+	struct vip_port *port = notifier_to_vip_port(notifier);
+
+	vip_dbg(1, port, "%s\n", __func__);
+	return 0;
+}
+
+static const struct v4l2_async_notifier_operations vip_async_ops = {
+	.bound = vip_async_bound,
+	.complete = vip_async_complete,
+};
+
+static struct fwnode_handle *
+fwnode_graph_get_next_endpoint_by_regs(const struct fwnode_handle *fwnode,
+				       int port_reg, int reg)
+{
+	return of_fwnode_handle(of_graph_get_endpoint_by_regs(to_of_node(fwnode),
+							      port_reg, reg));
+}
+
+static int vip_register_subdev_notif(struct vip_port *port,
+				     struct fwnode_handle *ep)
+{
+	struct vip_async_config *config = &port->config;
+	struct v4l2_async_notifier *notifier = &port->notifier;
+	struct vip_dev *dev = port->dev;
+	struct fwnode_handle *subdev, *remote_ep;
+	int ret;
+
+	subdev = fwnode_graph_get_remote_port_parent(ep);
+	if (!subdev) {
+		vip_dbg(3, port, "can't get remote parent\n");
+		return -EINVAL;
+	}
+
+	remote_ep = fwnode_graph_get_remote_endpoint(ep);
+	if (!remote_ep) {
+		vip_dbg(3, port, "can't get remote-endpoint\n");
+		fwnode_handle_put(subdev);
+		return -EINVAL;
+	}
+
+	ret = v4l2_fwnode_endpoint_parse(remote_ep, &config->endpoints[0]);
+	if (ret) {
+		vip_dbg(3, port, "Failed to parse endpoint:\n");
+		fwnode_handle_put(subdev);
+		fwnode_handle_put(remote_ep);
+		return -EINVAL;
+	}
+
+	fwnode_handle_put(subdev);
+	fwnode_handle_put(remote_ep);
+
+	config->asd[0].match_type = V4L2_ASYNC_MATCH_FWNODE;
+	config->asd[0].match.fwnode = subdev;
+	config->asd_list[0] = &config->asd[0];
+	config->asd_sizes = 1;
+	notifier->ops = &vip_async_ops;
+	notifier->subdevs = config->asd_list;
+	notifier->num_subdevs = config->asd_sizes;
+
+	ret = v4l2_async_notifier_register(dev->v4l2_dev, notifier);
+	if (ret) {
+		vip_dbg(1, port, "Error registering async notifier\n");
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
+static int vip_of_probe(struct platform_device *pdev)
+{
+	struct vip_shared *shared = platform_get_drvdata(pdev);
+	struct regmap *syscon_pol = NULL;
+	u32 syscon_pol_offset = 0;
+	struct vip_port *port;
+	struct vip_dev *dev;
+	struct device_node *parent = pdev->dev.of_node;
+	struct fwnode_handle *ep = NULL;
+	int ret, slice_id, port_id, p;
+
+	if (parent && of_property_read_bool(parent, "syscon-pol")) {
+		syscon_pol = syscon_regmap_lookup_by_phandle(parent,
+							     "syscon-pol");
+		if (IS_ERR(syscon_pol)) {
+			dev_err(&pdev->dev, "failed to get syscon-pol regmap\n");
+			return PTR_ERR(syscon_pol);
+		}
+
+		if (of_property_read_u32_index(parent, "syscon-pol", 1,
+					       &syscon_pol_offset)) {
+			dev_err(&pdev->dev, "failed to get syscon-pol offset\n");
+			return -EINVAL;
+		}
+	}
+
+	for (p = 0; p < (VIP_NUM_PORTS * VIP_NUM_SLICES); p++) {
+		ep = fwnode_graph_get_next_endpoint_by_regs(of_fwnode_handle(parent),
+							    p, 0);
+		if (!ep)
+			continue;
+
+		switch (p) {
+		case 0:
+			slice_id = VIP_SLICE1;	port_id = VIP_PORTA;
+			break;
+		case 1:
+			slice_id = VIP_SLICE2;	port_id = VIP_PORTA;
+			break;
+		case 2:
+			slice_id = VIP_SLICE1;	port_id = VIP_PORTB;
+			break;
+		case 3:
+			slice_id = VIP_SLICE2;	port_id = VIP_PORTB;
+			break;
+		default:
+			dev_err(&pdev->dev, "Unknown port reg=<%d>\n", p);
+			continue;
+		}
+
+		ret = alloc_port(shared->devs[slice_id], port_id);
+		if (ret < 0)
+			continue;
+
+		dev = shared->devs[slice_id];
+		dev->syscon_pol = syscon_pol;
+		dev->syscon_pol_offset = syscon_pol_offset;
+		port = dev->ports[port_id];
+
+		vip_register_subdev_notif(port, ep);
+	}
+	return 0;
+}
+
+static const struct of_device_id vip_of_match[];
+static int vip_probe(struct platform_device *pdev)
+{
+	struct vip_dev *dev;
+	struct vip_shared *shared;
+	struct vip_parser_data *parser;
+	const struct of_device_id *of_dev_id;
+	struct pinctrl *pinctrl;
+	int ret, slice = VIP_SLICE1;
+	u32 tmp, pid;
+	struct v4l2_ctrl_handler *hdl;
+
+	pm_runtime_enable(&pdev->dev);
+
+	ret = pm_runtime_get_sync(&pdev->dev);
+	if (ret)
+		goto err_runtime_get;
+
+	of_dev_id = of_match_device(vip_of_match, &pdev->dev);
+	if (!of_dev_id) {
+		dev_err(&pdev->dev, "%s: Unable to match device\n", __func__);
+		return -ENODEV;
+	}
+
+	ret = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
+	if (ret) {
+		dev_err(&pdev->dev,
+			"32-bit consistent DMA enable failed\n");
+		return ret;
+	}
+
+	shared = devm_kzalloc(&pdev->dev, sizeof(*shared), GFP_KERNEL);
+	if (!shared)
+		return -ENOMEM;
+
+	shared->res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "vip");
+	shared->base = devm_ioremap_resource(&pdev->dev, shared->res);
+	if (IS_ERR(shared->base)) {
+		dev_err(&pdev->dev, "failed to ioremap\n");
+		ret = PTR_ERR(shared->base);
+		goto err_runtime_get;
+	}
+
+	pinctrl = devm_pinctrl_get_select_default(&pdev->dev);
+
+	/* Make sure H/W module has the right functionality */
+	pid = reg_read(shared, VIP_PID);
+	tmp = get_field(pid, VIP_PID_FUNC_MASK, VIP_PID_FUNC_SHIFT);
+
+	if (tmp != VIP_PID_FUNC) {
+		dev_info(&pdev->dev, "vip: unexpected PID function: 0x%x\n",
+			 tmp);
+		ret = -ENODEV;
+		goto err_runtime_get;
+	}
+
+	ret = v4l2_device_register(&pdev->dev, &shared->v4l2_dev);
+	if (ret)
+		goto err_runtime_get;
+
+	/* enable clocks, so the firmware will load properly */
+	vip_shared_set_clock_enable(shared, 1);
+	vip_top_vpdma_reset(shared);
+
+	platform_set_drvdata(pdev, shared);
+
+	for (slice = VIP_SLICE1; slice < VIP_NUM_SLICES; slice++) {
+		u32 vin_id;
+
+		dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL);
+		if (!dev) {
+			ret = -ENOMEM;
+			goto err_runtime_get;
+		}
+		dev->instance_id = (int)of_dev_id->data;
+		vin_id = 1 + ((dev->instance_id - 1) * 2) + slice;
+		snprintf(dev->name, sizeof(dev->name),
+			 "vin%d", vin_id);
+
+		dev->irq = platform_get_irq(pdev, slice);
+		if (!dev->irq) {
+			dev_err(&pdev->dev, "Could not get IRQ");
+			goto err_runtime_get;
+		}
+
+		if (devm_request_irq(&pdev->dev, dev->irq, vip_irq,
+				     0, dev->name, dev) < 0) {
+			ret = -ENOMEM;
+			goto dev_unreg;
+		}
+
+		spin_lock_init(&dev->slock);
+		spin_lock_init(&dev->lock);
+
+		mutex_init(&dev->mutex);
+
+		hdl = &dev->ctrl_handler;
+		v4l2_ctrl_handler_init(hdl, 11);
+		shared->v4l2_dev.ctrl_handler = hdl;
+
+		dev->slice_id = slice;
+		dev->pdev = pdev;
+		dev->res = shared->res;
+		dev->base = shared->base;
+		dev->v4l2_dev = &shared->v4l2_dev;
+
+		dev->shared = shared;
+		shared->devs[slice] = dev;
+
+		vip_top_reset(dev);
+		vip_set_slice_path(dev, VIP_MULTI_CHANNEL_DATA_SELECT, 1);
+
+		parser = devm_kzalloc(&pdev->dev, sizeof(*dev->parser),
+				      GFP_KERNEL);
+		if (!parser)
+			return PTR_ERR(parser);
+
+		parser->res = platform_get_resource_byname(pdev,
+							   IORESOURCE_MEM,
+							   (slice == 0) ?
+							   "parser0" :
+							   "parser1");
+		parser->base = devm_ioremap_resource(&pdev->dev, parser->res);
+		if (IS_ERR(parser->base)) {
+			ret = PTR_ERR(parser->base);
+			goto dev_unreg;
+		}
+		parser->pdev = pdev;
+		dev->parser = parser;
+
+		dev->sc_assigned = VIP_NOT_ASSIGNED;
+		dev->sc = sc_create(pdev, (slice == 0) ? "sc0" : "sc1");
+		if (IS_ERR(dev->sc)) {
+			ret = PTR_ERR(dev->sc);
+			goto dev_unreg;
+		}
+
+		dev->csc_assigned = VIP_NOT_ASSIGNED;
+		dev->csc = csc_create(pdev, (slice == 0) ? "csc0" : "csc1");
+		if (IS_ERR(dev->sc)) {
+			ret = PTR_ERR(dev->sc);
+			goto dev_unreg;
+		}
+	}
+
+	shared->vpdma = &shared->vpdma_data;
+	ret = vpdma_create(pdev, shared->vpdma, vip_vpdma_fw_cb);
+	if (ret) {
+		dev_err(&pdev->dev, "Creating VPDMA failed");
+		goto dev_unreg;
+	}
+
+	return 0;
+dev_unreg:
+	v4l2_device_unregister(&shared->v4l2_dev);
+err_runtime_get:
+	if (slice == VIP_SLICE1) {
+		pm_runtime_disable(&pdev->dev);
+		return ret;
+	} else {
+		return 0;
+	}
+}
+
+static int vip_remove(struct platform_device *pdev)
+{
+	struct vip_shared *shared = platform_get_drvdata(pdev);
+	struct vip_dev *dev;
+	int slice;
+
+	for (slice = 0; slice < VIP_NUM_SLICES; slice++) {
+		dev = shared->devs[slice];
+		if (!dev)
+			continue;
+
+		free_port(dev->ports[VIP_PORTA]);
+		free_port(dev->ports[VIP_PORTB]);
+	}
+	pm_runtime_put_sync(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+
+	return 0;
+}
+
+#if defined(CONFIG_OF)
+static const struct of_device_id vip_of_match[] = {
+	{
+		.compatible = "ti,vip1", .data = (void *)VIP_INSTANCE1,
+	},
+
+	{
+		.compatible = "ti,vip2", .data = (void *)VIP_INSTANCE2,
+	},
+
+	{
+		.compatible = "ti,vip3", .data = (void *)VIP_INSTANCE3,
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(of, vip_of_match);
+#endif
+
+static struct platform_driver vip_pdrv = {
+	.probe		= vip_probe,
+	.remove		= vip_remove,
+	.driver		= {
+		.name	= VIP_MODULE_NAME,
+		.of_match_table = of_match_ptr(vip_of_match),
+	},
+};
+
+module_platform_driver(vip_pdrv);
+
+MODULE_DESCRIPTION("TI VIP driver");
+MODULE_AUTHOR("Texas Instruments");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/media/platform/ti-vpe/vip.h b/drivers/media/platform/ti-vpe/vip.h
--- a/drivers/media/platform/ti-vpe/vip.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/media/platform/ti-vpe/vip.h	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,727 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * TI VIP capture driver
+ *
+ * Copyright (C) 2018 Texas Instruments Incorpated - http://www.ti.com/
+ * David Griego, <dagriego@biglakesoftware.com>
+ * Dale Farnsworth, <dale@farnsworth.org>
+ * Nikhil Devshatwar, <nikhil.nd@ti.com>
+ * Benoit Parrot, <bparrot@ti.com>
+ */
+
+#ifndef __TI_VIP_H
+#define __TI_VIP_H
+
+#include <linux/videodev2.h>
+#include <media/v4l2-async.h>
+#include <media/v4l2-common.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-event.h>
+#include <media/v4l2-ioctl.h>
+#include <media/v4l2-mem2mem.h>
+#include <media/videobuf2-core.h>
+#include <media/videobuf2-dma-contig.h>
+#include <media/videobuf2-memops.h>
+#include <media/v4l2-fwnode.h>
+
+#include "vpdma.h"
+#include "vpdma_priv.h"
+#include "sc.h"
+#include "csc.h"
+
+#define VIP_INSTANCE1	1
+#define VIP_INSTANCE2	2
+#define VIP_INSTANCE3	3
+
+#define VIP_SLICE1	0
+#define VIP_SLICE2	1
+#define VIP_NUM_SLICES	2
+
+/*
+ * Additionnal client identifiers used for VPDMA configuration descriptors
+ */
+#define VIP_SLICE1_CFD_SC_CLIENT	7
+#define VIP_SLICE2_CFD_SC_CLIENT	8
+
+#define VIP_PORTA	0
+#define VIP_PORTB	1
+#define VIP_NUM_PORTS	2
+
+#define VIP_MAX_PLANES	2
+#define	VIP_LUMA	0
+#define VIP_CHROMA	1
+
+#define VIP_CAP_STREAMS_PER_PORT	16
+#define VIP_VBI_STREAMS_PER_PORT	16
+
+#define VIP_MAX_SUBDEV			5
+/*
+ * This value needs to be at least as large as the number of entry in
+ * vip_formats[].
+ * When vip_formats[] is modified make sure to adjust this value also.
+ */
+#define VIP_MAX_ACTIVE_FMT		16
+/*
+ * Colorspace conversion unit can be in one of 3 modes:
+ * NA  - Not Available on this port
+ * Y2R - Needed for YUV to RGB on this port
+ * R2Y - Needed for RGB to YUV on this port
+ */
+enum vip_csc_state {
+	VIP_CSC_NA = 0,
+	VIP_CSC_Y2R,
+	VIP_CSC_R2Y,
+};
+
+/* buffer for one video frame */
+struct vip_buffer {
+	/* common v4l buffer stuff */
+	struct vb2_v4l2_buffer	vb;
+	struct list_head	list;
+	bool			drop;
+};
+
+/*
+ * struct vip_fmt - VIP media bus format information
+ * @fourcc: V4L2 pixel format FCC identifier
+ * @code: V4L2 media bus format code
+ * @colorspace: V4L2 colorspace identifier
+ * @coplanar: 1 if unpacked Luma and Chroma, 0 otherwise (packed/interleaved)
+ * @vpdma_fmt: VPDMA data format per plane.
+ */
+struct vip_fmt {
+	u32	fourcc;
+	u32	code;
+	u32	colorspace;
+	u8	coplanar;
+	const struct vpdma_data_format *vpdma_fmt[VIP_MAX_PLANES];
+};
+
+/*
+ * The vip_parser_data structures contains the memory mapped
+ * info to access the parser registers.
+ */
+struct vip_parser_data {
+	void __iomem		*base;
+	struct resource		*res;
+
+	int			slice_id;
+
+	struct platform_device *pdev;
+};
+
+/*
+ * The vip_shared structure contains data that is shared by both
+ * the VIP1 and VIP2 slices.
+ */
+struct vip_shared {
+	struct list_head	list;
+	struct resource		*res;
+	void __iomem		*base;
+	struct vpdma_data	vpdma_data;
+	struct vpdma_data	*vpdma;
+	struct v4l2_device	v4l2_dev;
+	struct vip_dev		*devs[VIP_NUM_SLICES];
+};
+
+/*
+ * There are two vip_dev structure, one for each vip slice: VIP1 & VIP2.
+ */
+
+struct vip_async_config {
+	struct v4l2_async_subdev *asd_list[VIP_MAX_SUBDEV];
+	struct v4l2_async_subdev asd[VIP_MAX_SUBDEV];
+	struct v4l2_fwnode_endpoint endpoints[VIP_MAX_SUBDEV];
+	int asd_sizes;
+};
+
+
+struct vip_dev {
+	struct v4l2_device	*v4l2_dev;
+	struct v4l2_ctrl_handler ctrl_handler;
+	struct platform_device *pdev;
+	struct vip_shared	*shared;
+	struct resource		*res;
+	struct regmap		*syscon_pol;
+	u32			syscon_pol_offset;
+	int			instance_id;
+	int			slice_id;
+	int			num_ports;	/* count of open ports */
+	struct mutex		mutex;
+	spinlock_t		slock;
+	spinlock_t		lock; /* used in videobuf2 callback */
+
+	int			irq;
+	void __iomem		*base;
+
+	struct vip_port		*ports[VIP_NUM_PORTS];
+
+	char			name[16];
+	/* parser data handle */
+	struct vip_parser_data	*parser;
+	/* scaler data handle */
+	struct sc_data		*sc;
+	/* scaler port assignation */
+	int			sc_assigned;
+	/* csc data handle */
+	struct csc_data		*csc;
+	/* csc port assignation */
+	int			csc_assigned;
+};
+
+/*
+ * There are two vip_port structures for each vip_dev, one for port A
+ * and one for port B.
+ */
+struct vip_port {
+	struct vip_dev		*dev;
+	int			port_id;
+
+	enum v4l2_colorspace	src_colorspace;
+	unsigned int		flags;
+	unsigned int		src_width;
+	unsigned int		src_height;
+	struct v4l2_rect	c_rect;		/* crop rectangle */
+	struct v4l2_mbus_framefmt mbus_framefmt;
+	struct v4l2_mbus_framefmt try_mbus_framefmt;
+
+	char			name[16];
+	struct vip_fmt		*fmt;		/* current format info */
+	/* Number of channels/streams configured */
+	int			num_streams_configured;
+	int			num_streams;	/* count of open streams */
+	struct vip_stream	*cap_streams[VIP_CAP_STREAMS_PER_PORT];
+	struct vip_stream	*vbi_streams[VIP_VBI_STREAMS_PER_PORT];
+
+	struct vip_async_config	config;
+	struct v4l2_async_notifier notifier;
+	struct v4l2_subdev	*subdev;
+	struct v4l2_fwnode_endpoint *endpoint;
+	struct vip_fmt		*active_fmt[VIP_MAX_ACTIVE_FMT];
+	int			num_active_fmt;
+	/* have new shadow reg values */
+	bool			load_mmrs;
+	/* shadow reg addr/data block */
+	struct vpdma_buf	mmr_adb;
+	/* h coeff buffer */
+	struct vpdma_buf	sc_coeff_h;
+	/* v coeff buffer */
+	struct vpdma_buf	sc_coeff_v;
+	/* Show if scaler resource is available on this port */
+	bool			scaler;
+	/* Show the csc resource state on this port */
+	enum vip_csc_state	csc;
+};
+
+/*
+ * When handling multiplexed video, there can be multiple streams for each
+ * port.  The vip_stream structure holds per-stream data.
+ */
+struct vip_stream {
+	struct video_device	*vfd;
+	struct vip_port		*port;
+	int			stream_id;
+	int			list_num;
+	int			vfl_type;
+	char			name[16];
+	struct work_struct	recovery_work;
+	int			num_recovery;
+	enum v4l2_field		field;		/* current field */
+	unsigned int		sequence;	/* current frame/field seq */
+	enum v4l2_field		sup_field;	/* supported field value */
+	unsigned int		width;		/* frame width */
+	unsigned int		height;		/* frame height */
+	unsigned int		bytesperline;	/* bytes per line in memory */
+	unsigned int		sizeimage;	/* image size in memory */
+	struct list_head	vidq;		/* incoming vip_bufs queue */
+	struct list_head	dropq;		/* drop vip_bufs queue */
+	struct list_head	post_bufs;	/* vip_bufs to be DMAed */
+	/* Maintain a list of used channels - Needed for VPDMA cleanup */
+	int			vpdma_channels[VPDMA_MAX_CHANNELS];
+	int			vpdma_channels_to_abort[VPDMA_MAX_CHANNELS];
+	struct vpdma_desc_list	desc_list;	/* DMA descriptor list */
+	struct vpdma_dtd	*write_desc;
+	/* next unused desc_list addr */
+	void			*desc_next;
+	struct vb2_queue	vb_vidq;
+};
+
+/*
+ * VIP Enumerations
+ */
+enum data_path_select {
+	ALL_FIELDS_DATA_SELECT = 0,
+	VIP_CSC_SRC_DATA_SELECT,
+	VIP_SC_SRC_DATA_SELECT,
+	VIP_RGB_SRC_DATA_SELECT,
+	VIP_RGB_OUT_LO_DATA_SELECT,
+	VIP_RGB_OUT_HI_DATA_SELECT,
+	VIP_CHR_DS_1_SRC_DATA_SELECT,
+	VIP_CHR_DS_2_SRC_DATA_SELECT,
+	VIP_MULTI_CHANNEL_DATA_SELECT,
+	VIP_CHR_DS_1_DATA_BYPASS,
+	VIP_CHR_DS_2_DATA_BYPASS,
+};
+
+
+enum data_interface_modes {
+	SINGLE_24B_INTERFACE = 0,
+	SINGLE_16B_INTERFACE = 1,
+	DUAL_8B_INTERFACE = 2,
+};
+
+enum sync_types {
+	EMBEDDED_SYNC_SINGLE_YUV422 = 0,
+	EMBEDDED_SYNC_2X_MULTIPLEXED_YUV422 = 1,
+	EMBEDDED_SYNC_4X_MULTIPLEXED_YUV422 = 2,
+	EMBEDDED_SYNC_LINE_MULTIPLEXED_YUV422 = 3,
+	DISCRETE_SYNC_SINGLE_YUV422 = 4,
+	EMBEDDED_SYNC_SINGLE_RGB_OR_YUV444 = 5,
+	DISCRETE_SYNC_SINGLE_RGB_24B = 10,
+};
+
+#define VIP_NOT_ASSIGNED	-1
+
+/*
+ * Register offsets and field selectors
+ */
+#define VIP_PID_FUNC			0xf02
+
+#define VIP_PID				0x0000
+#define VIP_PID_MINOR_MASK              0x3f
+#define VIP_PID_MINOR_SHIFT             0
+#define VIP_PID_CUSTOM_MASK             0x03
+#define VIP_PID_CUSTOM_SHIFT            6
+#define VIP_PID_MAJOR_MASK              0x07
+#define VIP_PID_MAJOR_SHIFT             8
+#define VIP_PID_RTL_MASK                0x1f
+#define VIP_PID_RTL_SHIFT               11
+#define VIP_PID_FUNC_MASK               0xfff
+#define VIP_PID_FUNC_SHIFT              16
+#define VIP_PID_SCHEME_MASK             0x03
+#define VIP_PID_SCHEME_SHIFT            30
+
+#define VIP_SYSCONFIG			0x0010
+#define VIP_SYSCONFIG_IDLE_MASK         0x03
+#define VIP_SYSCONFIG_IDLE_SHIFT        2
+#define VIP_SYSCONFIG_STANDBY_MASK      0x03
+#define VIP_SYSCONFIG_STANDBY_SHIFT     4
+#define VIP_FORCE_IDLE_MODE             0
+#define VIP_NO_IDLE_MODE                1
+#define VIP_SMART_IDLE_MODE             2
+#define VIP_SMART_IDLE_WAKEUP_MODE      3
+#define VIP_FORCE_STANDBY_MODE          0
+#define VIP_NO_STANDBY_MODE             1
+#define VIP_SMART_STANDBY_MODE          2
+#define VIP_SMART_STANDBY_WAKEUP_MODE   3
+
+#define VIP_INTC_INTX_OFFSET		0x0020
+
+#define VIP_INT0_STATUS0_RAW_SET	0x0020
+#define VIP_INT0_STATUS0_RAW		VIP_INT0_STATUS0_RAW_SET
+#define VIP_INT0_STATUS0_CLR		0x0028
+#define VIP_INT0_STATUS0		VIP_INT0_STATUS0_CLR
+#define VIP_INT0_ENABLE0_SET		0x0030
+#define VIP_INT0_ENABLE0		VIP_INT0_ENABLE0_SET
+#define VIP_INT0_ENABLE0_CLR		0x0038
+#define VIP_INT0_LIST0_COMPLETE         (1 << 0)
+#define VIP_INT0_LIST0_NOTIFY           (1 << 1)
+#define VIP_INT0_LIST1_COMPLETE         (1 << 2)
+#define VIP_INT0_LIST1_NOTIFY           (1 << 3)
+#define VIP_INT0_LIST2_COMPLETE         (1 << 4)
+#define VIP_INT0_LIST2_NOTIFY           (1 << 5)
+#define VIP_INT0_LIST3_COMPLETE         (1 << 6)
+#define VIP_INT0_LIST3_NOTIFY           (1 << 7)
+#define VIP_INT0_LIST4_COMPLETE         (1 << 8)
+#define VIP_INT0_LIST4_NOTIFY           (1 << 9)
+#define VIP_INT0_LIST5_COMPLETE         (1 << 10)
+#define VIP_INT0_LIST5_NOTIFY           (1 << 11)
+#define VIP_INT0_LIST6_COMPLETE         (1 << 12)
+#define VIP_INT0_LIST6_NOTIFY           (1 << 13)
+#define VIP_INT0_LIST7_COMPLETE         (1 << 14)
+#define VIP_INT0_LIST7_NOTIFY           (1 << 15)
+#define VIP_INT0_DESCRIPTOR             (1 << 16)
+#define VIP_VIP1_PARSER_INT		(1 << 20)
+#define VIP_VIP2_PARSER_INT		(1 << 21)
+
+#define VIP_INT0_STATUS1_RAW_SET        0x0024
+#define VIP_INT0_STATUS1_RAW            VIP_INT0_STATUS0_RAW_SET
+#define VIP_INT0_STATUS1_CLR            0x002c
+#define VIP_INT0_STATUS1                VIP_INT0_STATUS0_CLR
+#define VIP_INT0_ENABLE1_SET            0x0034
+#define VIP_INT0_ENABLE1                VIP_INT0_ENABLE0_SET
+#define VIP_INT0_ENABLE1_CLR            0x003c
+#define VIP_INT0_ENABLE1_STAT		0x004c
+#define VIP_INT0_CHANNEL_GROUP0		(1 << 0)
+#define VIP_INT0_CHANNEL_GROUP1		(1 << 1)
+#define VIP_INT0_CHANNEL_GROUP2		(1 << 2)
+#define VIP_INT0_CHANNEL_GROUP3		(1 << 3)
+#define VIP_INT0_CHANNEL_GROUP4		(1 << 4)
+#define VIP_INT0_CHANNEL_GROUP5		(1 << 5)
+#define VIP_INT0_CLIENT			(1 << 7)
+#define VIP_VIP1_DS1_UV_ERROR_INT	(1 << 22)
+#define VIP_VIP1_DS2_UV_ERROR_INT	(1 << 23)
+#define VIP_VIP2_DS1_UV_ERROR_INT	(1 << 24)
+#define VIP_VIP2_DS2_UV_ERROR_INT	(1 << 25)
+
+#define VIP_INTC_E0I			0x00a0
+
+#define VIP_CLK_ENABLE			0x0100
+#define VIP_VPDMA_CLK_ENABLE		(1 << 0)
+#define VIP_VIP1_DATA_PATH_CLK_ENABLE	(1 << 16)
+#define VIP_VIP2_DATA_PATH_CLK_ENABLE	(1 << 17)
+
+#define VIP_CLK_RESET			0x0104
+#define VIP_VPDMA_RESET			(1 << 0)
+#define VIP_VPDMA_CLK_RESET_MASK	0x1
+#define VIP_VPDMA_CLK_RESET_SHIFT	0
+#define VIP_DATA_PATH_CLK_RESET_MASK	0x1
+#define VIP_VIP1_DATA_PATH_RESET_SHIFT	16
+#define VIP_VIP2_DATA_PATH_RESET_SHIFT	17
+#define VIP_VIP1_DATA_PATH_RESET	(1 << 16)
+#define VIP_VIP2_DATA_PATH_RESET	(1 << 17)
+#define VIP_VIP1_PARSER_RESET		(1 << 18)
+#define VIP_VIP2_PARSER_RESET		(1 << 19)
+#define VIP_VIP1_CSC_RESET		(1 << 20)
+#define VIP_VIP2_CSC_RESET		(1 << 21)
+#define VIP_VIP1_SC_RESET		(1 << 22)
+#define VIP_VIP2_SC_RESET		(1 << 23)
+#define VIP_VIP1_DS1_RESET		(1 << 25)
+#define VIP_VIP2_DS1_RESET		(1 << 26)
+#define VIP_VIP1_DS2_RESET		(1 << 27)
+#define VIP_VIP2_DS2_RESET		(1 << 28)
+#define VIP_MAIN_RESET			(1 << 31)
+
+#define VIP_VIP1_DATA_PATH_SELECT	0x010c
+#define VIP_VIP2_DATA_PATH_SELECT	0x0110
+#define VIP_CSC_SRC_SELECT_MASK		0x07
+#define VIP_CSC_SRC_SELECT_SHFT		0
+#define VIP_SC_SRC_SELECT_MASK		0x07
+#define VIP_SC_SRC_SELECT_SHFT		3
+#define VIP_RGB_SRC_SELECT		(1 << 6)
+#define VIP_RGB_OUT_LO_SRC_SELECT	(1 << 7)
+#define VIP_RGB_OUT_HI_SRC_SELECT	(1 << 8)
+#define VIP_DS1_SRC_SELECT_MASK		0x07
+#define VIP_DS1_SRC_SELECT_SHFT		9
+#define VIP_DS2_SRC_SELECT_MASK		0x07
+#define VIP_DS2_SRC_SELECT_SHFT		12
+#define VIP_MULTI_CHANNEL_SELECT	(1 << 15)
+#define VIP_DS1_BYPASS			(1 << 16)
+#define VIP_DS2_BYPASS			(1 << 17)
+#define VIP_TESTPORT_B_SELECT		(1 << 26)
+#define VIP_TESTPORT_A_SELECT		(1 << 27)
+#define VIP_DATAPATH_SELECT_MASK	0x0f
+#define VIP_DATAPATH_SELECT_SHFT	28
+
+#define VIP1_PARSER_REG_OFFSET		0x5500
+#define VIP2_PARSER_REG_OFFSET		0x5a00
+
+#define VIP_PARSER_MAIN_CFG		0x0000
+#define VIP_DATA_INTERFACE_MODE_MASK	0x03
+#define VIP_DATA_INTERFACE_MODE_SHFT	0
+#define VIP_CLIP_BLANK			(1 << 4)
+#define VIP_CLIP_ACTIVE			(1 << 5)
+
+#define VIP_PARSER_PORTA_0		0x0004
+#define VIP_PARSER_PORTB_0		0x000c
+#define VIP_SYNC_TYPE_MASK		0x0f
+#define VIP_SYNC_TYPE_SHFT		0
+#define VIP_CTRL_CHANNEL_SEL_MASK	0x03
+#define VIP_CTRL_CHANNEL_SEL_SHFT	4
+#define VIP_ASYNC_FIFO_WR		(1 << 6)
+#define VIP_ASYNC_FIFO_RD		(1 << 7)
+#define VIP_PORT_ENABLE			(1 << 8)
+#define VIP_FID_POLARITY		(1 << 9)
+#define VIP_PIXCLK_EDGE_POLARITY	(1 << 10)
+#define VIP_HSYNC_POLARITY		(1 << 11)
+#define VIP_VSYNC_POLARITY		(1 << 12)
+#define VIP_ACTVID_POLARITY		(1 << 13)
+#define VIP_FID_DETECT_MODE		(1 << 14)
+#define VIP_USE_ACTVID_HSYNC_ONLY	(1 << 15)
+#define VIP_FID_SKEW_PRECOUNT_MASK	0x3f
+#define VIP_FID_SKEW_PRECOUNT_SHFT	16
+#define VIP_DISCRETE_BASIC_MODE		(1 << 22)
+#define VIP_SW_RESET			(1 << 23)
+#define VIP_FID_SKEW_POSTCOUNT_MASK	0x3f
+#define VIP_FID_SKEW_POSTCOUNT_SHFT	24
+#define VIP_ANALYZER_2X4X_SRCNUM_POS	(1 << 30)
+#define VIP_ANALYZER_FVH_ERR_COR_EN	(1 << 31)
+
+#define VIP_PARSER_PORTA_1		0x0008
+#define VIP_PARSER_PORTB_1		0x0010
+#define VIP_SRC0_NUMLINES_MASK		0x0fff
+#define VIP_SRC0_NUMLINES_SHFT		0
+#define VIP_ANC_CHAN_SEL_8B_MASK	0x03
+#define VIP_ANC_CHAN_SEL_8B_SHFT	13
+#define VIP_SRC0_NUMPIX_MASK		0x0fff
+#define VIP_SRC0_NUMPIX_SHFT		16
+#define VIP_REPACK_SEL_MASK		0x07
+#define VIP_REPACK_SEL_SHFT		28
+
+#define VIP_PARSER_FIQ_MASK		0x0014
+#define VIP_PARSER_FIQ_CLR		0x0018
+#define VIP_PARSER_FIQ_STATUS		0x001c
+#define VIP_PORTA_VDET			(1 << 0)
+#define VIP_PORTB_VDET			(1 << 1)
+#define VIP_PORTA_ASYNC_FIFO_OF		(1 << 2)
+#define VIP_PORTB_ASYNC_FIFO_OF		(1 << 3)
+#define VIP_PORTA_OUTPUT_FIFO_YUV	(1 << 4)
+#define VIP_PORTA_OUTPUT_FIFO_ANC	(1 << 6)
+#define VIP_PORTB_OUTPUT_FIFO_YUV	(1 << 7)
+#define VIP_PORTB_OUTPUT_FIFO_ANC	(1 << 9)
+#define VIP_PORTA_CONN			(1 << 10)
+#define VIP_PORTA_DISCONN		(1 << 11)
+#define VIP_PORTB_CONN			(1 << 12)
+#define VIP_PORTB_DISCONN		(1 << 13)
+#define VIP_PORTA_SRC0_SIZE		(1 << 14)
+#define VIP_PORTB_SRC0_SIZE		(1 << 15)
+#define VIP_PORTA_YUV_PROTO_VIOLATION	(1 << 16)
+#define VIP_PORTA_ANC_PROTO_VIOLATION	(1 << 17)
+#define VIP_PORTB_YUV_PROTO_VIOLATION	(1 << 18)
+#define VIP_PORTB_ANC_PROTO_VIOLATION	(1 << 19)
+#define VIP_PORTA_CFG_DISABLE_COMPLETE	(1 << 20)
+#define VIP_PORTB_CFG_DISABLE_COMPLETE	(1 << 21)
+
+#define VIP_PARSER_PORTA_SOURCE_FID	0x0020
+#define VIP_PARSER_PORTA_ENCODER_FID	0x0024
+#define VIP_PARSER_PORTB_SOURCE_FID	0x0028
+#define VIP_PARSER_PORTB_ENCODER_FID	0x002c
+
+#define VIP_PARSER_PORTA_SRC0_SIZE	0x0030
+#define VIP_PARSER_PORTB_SRC0_SIZE	0x0070
+#define VIP_SOURCE_HEIGHT_MASK		0x0fff
+#define VIP_SOURCE_HEIGHT_SHFT		0
+#define VIP_SOURCE_WIDTH_MASK		0x0fff
+#define VIP_SOURCE_WIDTH_SHFT		16
+
+#define VIP_PARSER_PORTA_VDET_VEC	0x00b0
+#define VIP_PARSER_PORTB_VDET_VEC	0x00b4
+
+#define VIP_PARSER_PORTA_EXTRA2		0x00b8
+#define VIP_PARSER_PORTB_EXTRA2		0x00c8
+#define VIP_ANC_SKIP_NUMPIX_MASK	0x0fff
+#define VIP_ANC_SKIP_NUMPIX_SHFT	0
+#define VIP_ANC_BYPASS			(1 << 15)
+#define VIP_ANC_USE_NUMPIX_MASK		0x0fff
+#define VIP_ANC_USE_NUMPIX_SHFT		16
+#define VIP_ANC_TARGET_SRCNUM_MASK	0x0f
+#define VIP_ANC_TARGET_SRCNUM_SHFT	28
+
+#define VIP_PARSER_PORTA_EXTRA3		0x00bc
+#define VIP_PARSER_PORTB_EXTRA3		0x00cc
+#define VIP_ANC_SKIP_NUMLINES_MASK	0x0fff
+#define VIP_ANC_SKIP_NUMLINES_SHFT	0
+#define VIP_ANC_USE_NUMLINES_MASK	0x0fff
+#define VIP_ANC_USE_NUMLINES_SHFT	16
+
+#define VIP_PARSER_PORTA_EXTRA4		0x00c0
+#define VIP_PARSER_PORTB_EXTRA4		0x00d0
+#define VIP_ACT_SKIP_NUMPIX_MASK	0x0fff
+#define VIP_ACT_SKIP_NUMPIX_SHFT	0
+#define VIP_ACT_BYPASS			(1 << 15)
+#define VIP_ACT_USE_NUMPIX_MASK		0x0fff
+#define VIP_ACT_USE_NUMPIX_SHFT		16
+#define VIP_ACT_TARGET_SRCNUM_MASK	0x0f
+#define VIP_ACT_TARGET_SRCNUM_SHFT	28
+
+#define VIP_PARSER_PORTA_EXTRA5		0x00c4
+#define VIP_PARSER_PORTB_EXTRA5		0x00d4
+#define VIP_ACT_SKIP_NUMLINES_MASK	0x0fff
+#define VIP_ACT_SKIP_NUMLINES_SHFT	0
+#define VIP_ACT_USE_NUMLINES_MASK	0x0fff
+#define VIP_ACT_USE_NUMLINES_SHFT	16
+
+#define VIP_PARSER_PORTA_EXTRA6		0x00d8
+#define VIP_PARSER_PORTB_EXTRA6		0x00dc
+#define VIP_ANC_SRCNUM_STOP_IMM_SHFT	0
+#define VIP_YUV_SRCNUM_STOP_IMM_SHFT	16
+
+#define VIP_CSC_CSC00			0x0200
+#define VIP_CSC_A0_MASK			0x1fff
+#define VIP_CSC_A0_SHFT			0
+#define VIP_CSC_B0_MASK			0x1fff
+#define VIP_CSC_B0_SHFT			16
+
+#define VIP_CSC_CSC01			0x0204
+#define VIP_CSC_C0_MASK			0x1fff
+#define VIP_CSC_C0_SHFT			0
+#define VIP_CSC_A1_MASK			0x1fff
+#define VIP_CSC_A1_SHFT			16
+
+#define VIP_CSC_CSC02			0x0208
+#define VIP_CSC_B1_MASK			0x1fff
+#define VIP_CSC_B1_SHFT			0
+#define VIP_CSC_C1_MASK			0x1fff
+#define VIP_CSC_C1_SHFT			16
+
+#define VIP_CSC_CSC03			0x020c
+#define VIP_CSC_A2_MASK			0x1fff
+#define VIP_CSC_A2_SHFT			0
+#define VIP_CSC_B2_MASK			0x1fff
+#define VIP_CSC_B2_SHFT			16
+
+#define VIP_CSC_CSC04			0x0210
+#define VIP_CSC_C2_MASK			0x1fff
+#define VIP_CSC_C2_SHFT			0
+#define VIP_CSC_D0_MASK			0x0fff
+#define VIP_CSC_D0_SHFT			16
+
+#define VIP_CSC_CSC05			0x0214
+#define VIP_CSC_D1_MASK			0x0fff
+#define VIP_CSC_D1_SHFT			0
+#define VIP_CSC_D2_MASK			0x0fff
+#define VIP_CSC_D2_SHFT			16
+#define VIP_CSC_BYPASS			(1 << 28)
+
+#define VIP_SC_MP_SC0			0x0300
+#define VIP_INTERLACE_O			(1 << 0)
+#define VIP_LINEAR			(1 << 1)
+#define VIP_SC_BYPASS			(1 << 2)
+#define VIP_INVT_FID			(1 << 3)
+#define VIP_USE_RAV			(1 << 4)
+#define VIP_ENABLE_EV			(1 << 5)
+#define VIP_AUTH_HS			(1 << 6)
+#define VIP_DCM_2X			(1 << 7)
+#define VIP_DCM_4X			(1 << 8)
+#define VIP_HP_BYPASS			(1 << 9)
+#define VIP_INTERLACE_I			(1 << 10)
+#define VIP_ENABLE_SIN2_VER_INTP	(1 << 11)
+#define VIP_Y_PK_EN			(1 << 14)
+#define VIP_TRIM			(1 << 15)
+#define VIP_SELFGEN_FID			(1 << 16)
+
+#define VIP_SC_MP_SC1			0x0304
+#define VIP_ROW_ACC_INC_MASK		0x07ffffff
+#define VIP_ROW_ACC_INC_SHFT		0
+
+#define VIP_SC_MP_SC2			0x0308
+#define VIP_ROW_ACC_OFFSET_MASK		0x0fffffff
+#define VIP_ROW_ACC_OFFSET_SHFT		0
+
+#define VIP_SC_MP_SC3			0x030c
+#define VIP_ROW_ACC_OFFSET_B_MASK	0x0fffffff
+#define VIP_ROW_ACC_OFFSET_B_SHFT	0
+
+#define VIP_SC_MP_SC4			0x0310
+#define VIP_TAR_H_MASK			0x07ff
+#define VIP_TAR_H_SHFT			0
+#define VIP_TAR_W_MASK			0x07ff
+#define VIP_TAR_W_SHFT			12
+#define VIP_LIN_ACC_INC_U_MASK		0x07
+#define VIP_LIN_ACC_INC_U_SHFT		24
+#define VIP_NLIN_ACC_INIT_U_MASK	0x07
+#define VIP_NLIN_ACC_INIT_U_SHFT	28
+
+#define VIP_SC_MP_SC5			0x0314
+#define VIP_SRC_H_MASK			0x03ff
+#define VIP_SRC_H_SHFT			0
+#define VIP_SRC_W_MASK			0x07ff
+#define VIP_SRC_W_SHFT			12
+#define VIP_NLIN_ACC_INC_U_MASK		0x07
+#define VIP_NLIN_ACC_INC_U_SHFT		24
+
+#define VIP_SC_MP_SC6			0x0318
+#define VIP_ROW_ACC_INIT_RAV_MASK	0x03ff
+#define VIP_ROW_ACC_INIT_RAV_SHFT	0
+#define VIP_ROW_ACC_INIT_RAV_B_MASK	0x03ff
+#define VIP_ROW_ACC_INIT_RAV_B_SHFT	10
+
+#define VIP_SC_MP_SC8			0x0320
+#define VIP_NLIN_LEFT_MASK		0x07ff
+#define VIP_NLIN_LEFT_SHFT		0
+#define VIP_NLIN_RIGHT_MASK		0x07ff
+#define VIP_NLIN_RIGHT_SHFT		12
+
+#define VIP_SC_MP_SC9			0x0324
+#define VIP_LIN_ACC_INC			VIP_SC_MP_SC9
+
+#define VIP_SC_MP_SC10			0x0328
+#define VIP_NLIN_ACC_INIT		VIP_SC_MP_SC10
+
+#define VIP_SC_MP_SC11			0x032c
+#define VIP_NLIN_ACC_INC		VIP_SC_MP_SC11
+
+#define VIP_SC_MP_SC12			0x0330
+#define VIP_COL_ACC_OFFSET_MASK		0x01ffffff
+#define VIP_COL_ACC_OFFSET_SHFT		0
+
+#define VIP_SC_MP_SC13			0x0334
+#define VIP_SC_FACTOR_RAV_MASK		0x03ff
+#define VIP_SC_FACTOR_RAV_SHFT		0
+#define VIP_CHROMA_INTP_THR_MASK	0x03ff
+#define VIP_CHROMA_INTP_THR_SHFT	12
+#define VIP_DELTA_CHROMA_THR_MASK	0x0f
+#define VIP_DELTA_CHROMA_THR_SHFT	24
+
+#define VIP_SC_MP_SC17			0x0344
+#define VIP_EV_THR_MASK			0x03ff
+#define VIP_EV_THR_SHFT			12
+#define VIP_DELTA_LUMA_THR_MASK		0x0f
+#define VIP_DELTA_LUMA_THR_SHFT		24
+#define VIP_DELTA_EV_THR_MASK		0x0f
+#define VIP_DELTA_EV_THR_SHFT		28
+
+#define VIP_SC_MP_SC18			0x0348
+#define VIP_HS_FACTOR_MASK		0x03ff
+#define VIP_HS_FACTOR_SHFT		0
+#define VIP_CONF_DEFAULT_MASK		0x01ff
+#define VIP_CONF_DEFAULT_SHFT		16
+
+#define VIP_SC_MP_SC19			0x034c
+#define VIP_HPF_COEFF0_MASK		0xff
+#define VIP_HPF_COEFF0_SHFT		0
+#define VIP_HPF_COEFF1_MASK		0xff
+#define VIP_HPF_COEFF1_SHFT		8
+#define VIP_HPF_COEFF2_MASK		0xff
+#define VIP_HPF_COEFF2_SHFT		16
+#define VIP_HPF_COEFF3_MASK		0xff
+#define VIP_HPF_COEFF3_SHFT		23
+
+#define VIP_SC_MP_SC20			0x0350
+#define VIP_HPF_COEFF4_MASK		0xff
+#define VIP_HPF_COEFF4_SHFT		0
+#define VIP_HPF_COEFF5_MASK		0xff
+#define VIP_HPF_COEFF5_SHFT		8
+#define VIP_HPF_NORM_SHFT_MASK		0x07
+#define VIP_HPF_NORM_SHFT_SHFT		16
+#define VIP_NL_LIMIT_MASK		0x1ff
+#define VIP_NL_LIMIT_SHFT		20
+
+#define VIP_SC_MP_SC21			0x0354
+#define VIP_NL_LO_THR_MASK		0x01ff
+#define VIP_NL_LO_THR_SHFT		0
+#define VIP_NL_LO_SLOPE_MASK		0xff
+#define VIP_NL_LO_SLOPE_SHFT		16
+
+#define VIP_SC_MP_SC22			0x0358
+#define VIP_NL_HI_THR_MASK		0x01ff
+#define VIP_NL_HI_THR_SHFT		0
+#define VIP_NL_HI_SLOPE_SH_MASK		0x07
+#define VIP_NL_HI_SLOPE_SH_SHFT		16
+
+#define VIP_SC_MP_SC23			0x035c
+#define VIP_GRADIENT_THR_MASK		0x07ff
+#define VIP_GRADIENT_THR_SHFT		0
+#define VIP_GRADIENT_THR_RANGE_MASK	0x0f
+#define VIP_GRADIENT_THR_RANGE_SHFT	12
+#define VIP_MIN_GY_THR_MASK		0xff
+#define VIP_MIN_GY_THR_SHFT		16
+#define VIP_MIN_GY_THR_RANGE_MASK	0x0f
+#define VIP_MIN_GY_THR_RANGE_SHFT	28
+
+#define VIP_SC_MP_SC24			0x0360
+#define VIP_ORG_H_MASK			0x07ff
+#define VIP_ORG_H_SHFT			0
+#define VIP_ORG_W_MASK			0x07ff
+#define VIP_ORG_W_SHFT			16
+
+#define VIP_SC_MP_SC25			0x0364
+#define VIP_OFF_H_MASK			0x07ff
+#define VIP_OFF_H_SHFT			0
+#define VIP_OFF_W_MASK			0x07ff
+#define VIP_OFF_W_SHFT			16
+
+#define VIP_VPDMA_REG_OFFSET		0xd000
+
+#endif
diff -Naur --no-dereference a/drivers/media/platform/ti-vpe/vpdma.c b/drivers/media/platform/ti-vpe/vpdma.c
--- a/drivers/media/platform/ti-vpe/vpdma.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/media/platform/ti-vpe/vpdma.c	2020-06-19 21:00:31.416227715 -0400
@@ -59,6 +59,11 @@
 		.data_type	= DATA_TYPE_C420,
 		.depth		= 4,
 	},
+	[VPDMA_DATA_FMT_CB420] = {
+		.type		= VPDMA_DATA_FMT_TYPE_YUV,
+		.data_type	= DATA_TYPE_CB420,
+		.depth		= 4,
+	},
 	[VPDMA_DATA_FMT_YCR422] = {
 		.type		= VPDMA_DATA_FMT_TYPE_YUV,
 		.data_type	= DATA_TYPE_YCR422,
@@ -826,7 +831,8 @@
 	channel = next_chan = raw_vpdma_chan;
 
 	if (fmt->type == VPDMA_DATA_FMT_TYPE_YUV &&
-			fmt->data_type == DATA_TYPE_C420) {
+	    (fmt->data_type == DATA_TYPE_C420 ||
+	     fmt->data_type == DATA_TYPE_CB420)) {
 		rect.height >>= 1;
 		rect.top >>= 1;
 		depth = 8;
@@ -894,7 +900,8 @@
 	channel = next_chan = chan_info[chan].num;
 
 	if (fmt->type == VPDMA_DATA_FMT_TYPE_YUV &&
-			fmt->data_type == DATA_TYPE_C420) {
+	    (fmt->data_type == DATA_TYPE_C420 ||
+	     fmt->data_type == DATA_TYPE_CB420)) {
 		rect.height >>= 1;
 		rect.top >>= 1;
 		depth = 8;
diff -Naur --no-dereference a/drivers/media/platform/ti-vpe/vpdma.h b/drivers/media/platform/ti-vpe/vpdma.h
--- a/drivers/media/platform/ti-vpe/vpdma.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/media/platform/ti-vpe/vpdma.h	2020-06-19 21:00:31.416227715 -0400
@@ -75,6 +75,7 @@
 	VPDMA_DATA_FMT_C444,
 	VPDMA_DATA_FMT_C422,
 	VPDMA_DATA_FMT_C420,
+	VPDMA_DATA_FMT_CB420,
 	VPDMA_DATA_FMT_YCR422,
 	VPDMA_DATA_FMT_YC444,
 	VPDMA_DATA_FMT_CRY422,
diff -Naur --no-dereference a/drivers/media/platform/ti-vpe/vpdma_priv.h b/drivers/media/platform/ti-vpe/vpdma_priv.h
--- a/drivers/media/platform/ti-vpe/vpdma_priv.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/media/platform/ti-vpe/vpdma_priv.h	2020-06-19 21:00:31.416227715 -0400
@@ -95,6 +95,7 @@
 #define DATA_TYPE_C444				0x4
 #define DATA_TYPE_C422				0x5
 #define DATA_TYPE_C420				0x6
+#define DATA_TYPE_CB420				0x16
 #define DATA_TYPE_YC444				0x8
 #define DATA_TYPE_YCB422			0x7
 #define DATA_TYPE_YCR422			0x17
@@ -168,11 +169,11 @@
 		u32		xfer_length_height;
 		u32		w1;
 	};
-	dma_addr_t		start_addr;
+	u32			start_addr;
 	u32			pkt_ctl;
 	union {
 		u32		frame_width_height;	/* inbound */
-		dma_addr_t	desc_write_addr;	/* outbound */
+		u32		desc_write_addr;	/* outbound */
 	};
 	union {
 		u32		start_h_v;		/* inbound */
diff -Naur --no-dereference a/drivers/media/platform/ti-vpe/vpe.c b/drivers/media/platform/ti-vpe/vpe.c
--- a/drivers/media/platform/ti-vpe/vpe.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/media/platform/ti-vpe/vpe.c	2020-06-19 21:00:31.416227715 -0400
@@ -55,7 +55,7 @@
 #define MIN_W		32
 #define MIN_H		32
 #define MAX_W		2048
-#define MAX_H		1184
+#define MAX_H		2048
 
 /* required alignments */
 #define S_ALIGN		0	/* multiple of 1 */
@@ -255,6 +255,15 @@
 				  },
 	},
 	{
+		.name		= "NV21 YUV 420 co-planar",
+		.fourcc		= V4L2_PIX_FMT_NV21,
+		.types		= VPE_FMT_TYPE_CAPTURE | VPE_FMT_TYPE_OUTPUT,
+		.coplanar	= 1,
+		.vpdma_fmt	= { &vpdma_yuv_fmts[VPDMA_DATA_FMT_Y420],
+				    &vpdma_yuv_fmts[VPDMA_DATA_FMT_CB420],
+				  },
+	},
+	{
 		.name		= "YUYV 422 packed",
 		.fourcc		= V4L2_PIX_FMT_YUYV,
 		.types		= VPE_FMT_TYPE_CAPTURE | VPE_FMT_TYPE_OUTPUT,
@@ -342,9 +351,14 @@
 #define	Q_DATA_MODE_TILED		BIT(1)
 #define	Q_DATA_INTERLACED_ALTERNATE	BIT(2)
 #define	Q_DATA_INTERLACED_SEQ_TB	BIT(3)
+#define	Q_DATA_INTERLACED_SEQ_BT	BIT(4)
+
+#define Q_IS_SEQ_XX		(Q_DATA_INTERLACED_SEQ_TB | \
+				Q_DATA_INTERLACED_SEQ_BT)
 
 #define Q_IS_INTERLACED		(Q_DATA_INTERLACED_ALTERNATE | \
-				Q_DATA_INTERLACED_SEQ_TB)
+				Q_DATA_INTERLACED_SEQ_TB | \
+				Q_DATA_INTERLACED_SEQ_BT)
 
 enum {
 	Q_DATA_SRC = 0,
@@ -700,7 +714,8 @@
 	 * Cfg Mode 1: YUV422 source, disable upsampler, DEI is de-interlacing.
 	 */
 
-	if (fmt->fourcc == V4L2_PIX_FMT_NV12)
+	if (fmt->fourcc == V4L2_PIX_FMT_NV12 ||
+	    fmt->fourcc == V4L2_PIX_FMT_NV21)
 		cfg_mode = 0;
 
 	write_field(us1_reg0, cfg_mode, VPE_US_MODE_MASK, VPE_US_MODE_SHIFT);
@@ -715,7 +730,8 @@
 	struct vpe_fmt *fmt = ctx->q_data[Q_DATA_SRC].fmt;
 	int line_mode = 1;
 
-	if (fmt->fourcc == V4L2_PIX_FMT_NV12)
+	if (fmt->fourcc == V4L2_PIX_FMT_NV12 ||
+	    fmt->fourcc == V4L2_PIX_FMT_NV21)
 		line_mode = 0;		/* double lines to line buffer */
 
 	/* regs for now */
@@ -777,7 +793,8 @@
 	 */
 	val |= VPE_DS_SRC_DEI_SCALER | VPE_CSC_SRC_DEI_SCALER;
 
-	if (fmt->fourcc != V4L2_PIX_FMT_NV12)
+	if (fmt->fourcc != V4L2_PIX_FMT_NV12 &&
+	    fmt->fourcc != V4L2_PIX_FMT_NV21)
 		val |= VPE_DS_BYPASS;
 
 	mmr_adb->out_fmt_reg[0] = val;
@@ -920,14 +937,6 @@
 }
 
 /*
- * Return the vpe_ctx structure for a given struct file
- */
-static struct vpe_ctx *file2ctx(struct file *file)
-{
-	return container_of(file->private_data, struct vpe_ctx, fh);
-}
-
-/*
  * mem2mem callbacks
  */
 
@@ -1132,24 +1141,36 @@
 		dma_addr += offset;
 		stride = q_data->bytesperline[VPE_LUMA];
 
-		if (q_data->flags & Q_DATA_INTERLACED_SEQ_TB) {
-			/*
-			 * Use top or bottom field from same vb alternately
-			 * f,f-1,f-2 = TBT when seq is even
-			 * f,f-1,f-2 = BTB when seq is odd
-			 */
-			field = (p_data->vb_index + (ctx->sequence % 2)) % 2;
+		/*
+		 * field used in VPDMA desc  = 0 (top) / 1 (bottom)
+		 * Use top or bottom field from same vb alternately
+		 * For each de-interlacing operation, f,f-1,f-2 should be one
+		 * of TBT or BTB
+		 */
+		if (q_data->flags & Q_DATA_INTERLACED_SEQ_TB ||
+		    q_data->flags & Q_DATA_INTERLACED_SEQ_BT) {
+			/* Select initial value based on format */
+			if (q_data->flags & Q_DATA_INTERLACED_SEQ_BT)
+				field = 1;
+			else
+				field = 0;
+
+			/* Toggle for each vb_index and each operation */
+			field = (field + p_data->vb_index + ctx->sequence) % 2;
 
 			if (field) {
-				/*
-				 * bottom field of a SEQ_TB buffer
-				 * Skip the top field data by
-				 */
 				int height = q_data->height / 2;
-				int bpp = fmt->fourcc == V4L2_PIX_FMT_NV12 ?
-						1 : (vpdma_fmt->depth >> 3);
+				int bpp;
+
+				if (fmt->fourcc == V4L2_PIX_FMT_NV12 ||
+				    fmt->fourcc == V4L2_PIX_FMT_NV21)
+					bpp = 1;
+				else
+					bpp = vpdma_fmt->depth >> 3;
+
 				if (plane)
 					height /= 2;
+
 				dma_addr += q_data->width * height * bpp;
 			}
 		}
@@ -1163,7 +1184,8 @@
 	frame_width = q_data->c_rect.width;
 	frame_height = q_data->c_rect.height;
 
-	if (p_data->vb_part && fmt->fourcc == V4L2_PIX_FMT_NV12)
+	if (p_data->vb_part && (fmt->fourcc == V4L2_PIX_FMT_NV12 ||
+				fmt->fourcc == V4L2_PIX_FMT_NV21))
 		frame_height /= 2;
 
 	vpdma_add_in_dtd(&ctx->desc_list, q_data->width, stride,
@@ -1204,12 +1226,14 @@
 	struct vpe_q_data *d_q_data = &ctx->q_data[Q_DATA_DST];
 	struct vpe_q_data *s_q_data = &ctx->q_data[Q_DATA_SRC];
 
-	if (ctx->deinterlacing && s_q_data->flags & Q_DATA_INTERLACED_SEQ_TB &&
-		ctx->sequence % 2 == 0) {
-		/* When using SEQ_TB buffers, When using it first time,
-		 * No need to remove the buffer as the next field is present
-		 * in the same buffer. (so that job_ready won't fail)
-		 * It will be removed when using bottom field
+	if (ctx->deinterlacing && s_q_data->flags & Q_IS_SEQ_XX &&
+	    ctx->sequence % 2 == 0) {
+		/* When using SEQ_XX type buffers, each buffer has two fields
+		 * each buffer has two fields (top & bottom)
+		 * Removing one buffer is actually getting two fields
+		 * Alternate between two operations:-
+		 * Even : consume one field but DO NOT REMOVE from queue
+		 * Odd : consume other field and REMOVE from queue
 		 */
 		ctx->src_vbs[0] = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx);
 		WARN_ON(ctx->src_vbs[0] == NULL);
@@ -1550,7 +1574,7 @@
 static int vpe_g_fmt(struct file *file, void *priv, struct v4l2_format *f)
 {
 	struct v4l2_pix_format_mplane *pix = &f->fmt.pix_mp;
-	struct vpe_ctx *ctx = file2ctx(file);
+	struct vpe_ctx *ctx = file->private_data;
 	struct vb2_queue *vq;
 	struct vpe_q_data *q_data;
 	int i;
@@ -1560,6 +1584,8 @@
 		return -EINVAL;
 
 	q_data = get_q_data(ctx, f->type);
+	if (!q_data)
+		return -EINVAL;
 
 	pix->width = q_data->width;
 	pix->height = q_data->height;
@@ -1602,8 +1628,10 @@
 		fmt = __find_format(V4L2_PIX_FMT_YUYV);
 	}
 
-	if (pix->field != V4L2_FIELD_NONE && pix->field != V4L2_FIELD_ALTERNATE
-			&& pix->field != V4L2_FIELD_SEQ_TB)
+	if (pix->field != V4L2_FIELD_NONE &&
+	    pix->field != V4L2_FIELD_ALTERNATE &&
+	    pix->field != V4L2_FIELD_SEQ_TB &&
+	    pix->field != V4L2_FIELD_SEQ_BT)
 		pix->field = V4L2_FIELD_NONE;
 
 	depth = fmt->vpdma_fmt[VPE_LUMA]->depth;
@@ -1655,9 +1683,9 @@
 
 	/*
 	 * For the actual image parameters, we need to consider the field
-	 * height of the image for SEQ_TB buffers.
+	 * height of the image for SEQ_XX buffers.
 	 */
-	if (pix->field == V4L2_FIELD_SEQ_TB)
+	if (pix->field == V4L2_FIELD_SEQ_TB || pix->field == V4L2_FIELD_SEQ_BT)
 		height = pix->height / 2;
 	else
 		height = pix->height;
@@ -1714,7 +1742,7 @@
 
 static int vpe_try_fmt(struct file *file, void *priv, struct v4l2_format *f)
 {
-	struct vpe_ctx *ctx = file2ctx(file);
+	struct vpe_ctx *ctx = file->private_data;
 	struct vpe_fmt *fmt = find_format(f);
 
 	if (V4L2_TYPE_IS_OUTPUT(f->type))
@@ -1767,11 +1795,13 @@
 		q_data->flags |= Q_DATA_INTERLACED_ALTERNATE;
 	else if (q_data->field == V4L2_FIELD_SEQ_TB)
 		q_data->flags |= Q_DATA_INTERLACED_SEQ_TB;
+	else if (q_data->field == V4L2_FIELD_SEQ_BT)
+		q_data->flags |= Q_DATA_INTERLACED_SEQ_BT;
 	else
 		q_data->flags &= ~Q_IS_INTERLACED;
 
-	/* the crop height is halved for the case of SEQ_TB buffers */
-	if (q_data->flags & Q_DATA_INTERLACED_SEQ_TB)
+	/* the crop height is halved for the case of SEQ_XX buffers */
+	if (q_data->flags & Q_IS_SEQ_XX)
 		q_data->c_rect.height /= 2;
 
 	vpe_dbg(ctx->dev, "Setting format for type %d, wxh: %dx%d, fmt: %d bpl_y %d",
@@ -1787,7 +1817,7 @@
 static int vpe_s_fmt(struct file *file, void *priv, struct v4l2_format *f)
 {
 	int ret;
-	struct vpe_ctx *ctx = file2ctx(file);
+	struct vpe_ctx *ctx = file->private_data;
 
 	ret = vpe_try_fmt(file, priv, f);
 	if (ret)
@@ -1844,10 +1874,10 @@
 	}
 
 	/*
-	 * For SEQ_TB buffers, crop height should be less than the height of
+	 * For SEQ_XX buffers, crop height should be less than the height of
 	 * the field height, not the buffer height
 	 */
-	if (q_data->flags & Q_DATA_INTERLACED_SEQ_TB)
+	if (q_data->flags & Q_IS_SEQ_XX)
 		height = q_data->height / 2;
 	else
 		height = q_data->height;
@@ -1872,7 +1902,7 @@
 static int vpe_g_selection(struct file *file, void *fh,
 		struct v4l2_selection *s)
 {
-	struct vpe_ctx *ctx = file2ctx(file);
+	struct vpe_ctx *ctx = file->private_data;
 	struct vpe_q_data *q_data;
 	bool use_c_rect = false;
 
@@ -1933,7 +1963,7 @@
 static int vpe_s_selection(struct file *file, void *fh,
 		struct v4l2_selection *s)
 {
-	struct vpe_ctx *ctx = file2ctx(file);
+	struct vpe_ctx *ctx = file->private_data;
 	struct vpe_q_data *q_data;
 	struct v4l2_selection sel = *s;
 	int ret;
@@ -2028,6 +2058,8 @@
 	struct vpe_q_data *q_data;
 
 	q_data = get_q_data(ctx, vq->type);
+	if (!q_data)
+		return -EINVAL;
 
 	*nplanes = q_data->nplanes;
 
@@ -2052,6 +2084,8 @@
 	vpe_dbg(ctx->dev, "type: %d\n", vb->vb2_queue->type);
 
 	q_data = get_q_data(ctx, vb->vb2_queue->type);
+	if (!q_data)
+		return -EINVAL;
 	num_planes = q_data->nplanes;
 
 	if (vb->vb2_queue->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
@@ -2060,7 +2094,8 @@
 		} else {
 			if (vbuf->field != V4L2_FIELD_TOP &&
 			    vbuf->field != V4L2_FIELD_BOTTOM &&
-			    vbuf->field != V4L2_FIELD_SEQ_TB)
+			    vbuf->field != V4L2_FIELD_SEQ_TB &&
+			    vbuf->field != V4L2_FIELD_SEQ_BT)
 				return -EINVAL;
 		}
 	}
@@ -2296,7 +2331,7 @@
 	init_adb_hdrs(ctx);
 
 	v4l2_fh_init(&ctx->fh, video_devdata(file));
-	file->private_data = &ctx->fh;
+	file->private_data = ctx;
 
 	hdl = &ctx->hdl;
 	v4l2_ctrl_handler_init(hdl, 1);
@@ -2381,7 +2416,7 @@
 static int vpe_release(struct file *file)
 {
 	struct vpe_dev *dev = video_drvdata(file);
-	struct vpe_ctx *ctx = file2ctx(file);
+	struct vpe_ctx *ctx = file->private_data;
 
 	vpe_dbg(dev, "releasing instance %p\n", ctx);
 
@@ -2499,6 +2534,13 @@
 	struct vpe_dev *dev;
 	int ret, irq, func;
 
+	ret = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
+	if (ret) {
+		dev_err(&pdev->dev,
+			"32-bit consistent DMA enable failed\n");
+		return ret;
+	}
+
 	dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL);
 	if (!dev)
 		return -ENOMEM;
@@ -2513,7 +2555,12 @@
 	mutex_init(&dev->dev_mutex);
 
 	dev->res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
-			"vpe_top");
+						"vpe_top");
+	if (!dev->res) {
+		dev_err(&pdev->dev, "missing 'vpe_top' resources data\n");
+		return -ENODEV;
+	}
+
 	/*
 	 * HACK: we get resource info from device tree in the form of a list of
 	 * VPE sub blocks, the driver currently uses only the base of vpe_top
diff -Naur --no-dereference a/drivers/media/v4l2-core/v4l2-fwnode.c b/drivers/media/v4l2-core/v4l2-fwnode.c
--- a/drivers/media/v4l2-core/v4l2-fwnode.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/media/v4l2-core/v4l2-fwnode.c	2020-06-19 21:00:31.416227715 -0400
@@ -114,6 +114,7 @@
 	struct v4l2_fwnode_bus_parallel *bus = &vep->bus.parallel;
 	unsigned int flags = 0;
 	u32 v;
+	int rval;
 
 	if (!fwnode_property_read_u32(fwnode, "hsync-active", &v))
 		flags |= v ? V4L2_MBUS_HSYNC_ACTIVE_HIGH :
@@ -158,6 +159,26 @@
 		flags |= v ? V4L2_MBUS_DATA_ENABLE_HIGH :
 			V4L2_MBUS_DATA_ENABLE_LOW;
 
+	if (vep->bus_type == V4L2_MBUS_BT656) {
+		if (fwnode_property_present(fwnode, "pixel-mux"))
+			bus->pixmux = 1;
+		else
+			bus->pixmux = 0;
+
+		bus->num_channels = 0;
+		rval = fwnode_property_read_u8_array(fwnode, "channels",
+						     NULL, 0);
+		if (rval > 0) {
+			bus->num_channels = min_t(int,
+						  ARRAY_SIZE(bus->channels),
+						  rval);
+
+			fwnode_property_read_u8_array(fwnode, "channels",
+						      bus->channels,
+						      bus->num_channels);
+		}
+	}
+
 	bus->flags = flags;
 
 }
diff -Naur --no-dereference a/drivers/media/v4l2-core/v4l2-ioctl.c b/drivers/media/v4l2-core/v4l2-ioctl.c
--- a/drivers/media/v4l2-core/v4l2-ioctl.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/media/v4l2-core/v4l2-ioctl.c	2020-06-19 21:00:31.416227715 -0400
@@ -1295,6 +1295,8 @@
 	case V4L2_META_FMT_VSP1_HGO:	descr = "R-Car VSP1 1-D Histogram"; break;
 	case V4L2_META_FMT_VSP1_HGT:	descr = "R-Car VSP1 2-D Histogram"; break;
 	case V4L2_META_FMT_UVC:		descr = "UVC payload header metadata"; break;
+	case V4L2_PIX_FMT_TI1210:       descr = "10-bit YUV 4:2:0 (NV12)"; break;
+	case V4L2_PIX_FMT_TI1610:	descr = "10-bit YUV 4:2:2 (NV16)"; break;
 
 	default:
 		/* Compressed formats */
diff -Naur --no-dereference a/drivers/memory/emif.h b/drivers/memory/emif.h
--- a/drivers/memory/emif.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/memory/emif.h	2020-06-19 21:00:31.416227715 -0400
@@ -537,6 +537,9 @@
 #define MCONNID_SHIFT					0
 #define MCONNID_MASK					(0xff << 0)
 
+/* READ_WRITE_LEVELING_CONTROL */
+#define RDWRLVLFULL_START				0x80000000
+
 /* DDR_PHY_CTRL_1 - EMIF4D */
 #define DLL_SLAVE_DLY_CTRL_SHIFT_4D			4
 #define DLL_SLAVE_DLY_CTRL_MASK_4D			(0xFF << 4)
@@ -598,6 +601,7 @@
 
 void ti_emif_save_context(void);
 void ti_emif_restore_context(void);
+void ti_emif_run_hw_leveling(void);
 void ti_emif_enter_sr(void);
 void ti_emif_exit_sr(void);
 void ti_emif_abort_sr(void);
diff -Naur --no-dereference a/drivers/memory/Makefile.asm-offsets b/drivers/memory/Makefile.asm-offsets
--- a/drivers/memory/Makefile.asm-offsets	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/memory/Makefile.asm-offsets	2020-06-19 21:00:31.416227715 -0400
@@ -1,4 +1,6 @@
-drivers/memory/emif-asm-offsets.s: drivers/memory/emif-asm-offsets.c
+targets += emif-asm-offsets.s
+
+drivers/memory/emif-asm-offsets.s: drivers/memory/emif-asm-offsets.c FORCE
 	$(call if_changed_dep,cc_s_c)
 
 include/generated/ti-emif-asm-offsets.h: drivers/memory/emif-asm-offsets.s FORCE
diff -Naur --no-dereference a/drivers/memory/ti-emif-pm.c b/drivers/memory/ti-emif-pm.c
--- a/drivers/memory/ti-emif-pm.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/memory/ti-emif-pm.c	2020-06-19 21:00:31.416227715 -0400
@@ -138,6 +138,9 @@
 	emif_data->pm_functions.exit_sr =
 		sram_resume_address(emif_data,
 				    (unsigned long)ti_emif_exit_sr);
+	emif_data->pm_functions.run_hw_leveling =
+		sram_resume_address(emif_data,
+				    (unsigned long)ti_emif_run_hw_leveling);
 
 	emif_data->pm_data.regs_virt =
 		(struct emif_regs_amx3 *)emif_data->ti_emif_sram_data_virt;
diff -Naur --no-dereference a/drivers/memory/ti-emif-sram-pm.S b/drivers/memory/ti-emif-sram-pm.S
--- a/drivers/memory/ti-emif-sram-pm.S	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/memory/ti-emif-sram-pm.S	2020-06-19 21:00:31.416227715 -0400
@@ -27,6 +27,7 @@
 #define EMIF_POWER_MGMT_SELF_REFRESH_MODE_MASK		0x0700
 
 #define EMIF_SDCFG_TYPE_DDR2				0x2 << SDRAM_TYPE_SHIFT
+#define EMIF_SDCFG_TYPE_DDR3				0x3 << SDRAM_TYPE_SHIFT
 #define EMIF_STATUS_READY				0x4
 
 #define AM43XX_EMIF_PHY_CTRL_REG_COUNT                  0x120
@@ -245,6 +246,46 @@
 ENDPROC(ti_emif_restore_context)
 
 /*
+ * void ti_emif_run_hw_leveling(void)
+ *
+ * Used during resume to run hardware leveling again and restore the
+ * configuration of the EMIF PHY, only for DDR3.
+ */
+ENTRY(ti_emif_run_hw_leveling)
+	adr	r4, ti_emif_pm_sram_data
+	ldr	r0, [r4, #EMIF_PM_BASE_ADDR_PHYS_OFFSET]
+
+	ldr	r3, [r0, #EMIF_READ_WRITE_LEVELING_CONTROL]
+	orr	r3, r3, #RDWRLVLFULL_START
+	ldr	r2, [r0, #EMIF_SDRAM_CONFIG]
+	and	r2, r2, #SDRAM_TYPE_MASK
+	cmp	r2, #EMIF_SDCFG_TYPE_DDR3
+	bne	skip_hwlvl
+
+	str	r3, [r0, #EMIF_READ_WRITE_LEVELING_CONTROL]
+
+	/*
+	 * If EMIF registers are touched during initial stage of HW
+	 * leveling sequence there will be an L3 NOC timeout error issued
+	 * as the EMIF will not respond, which is not fatal, but it is
+	 * avoidable. This small wait loop is enough time for this condition
+	 * to clear, even at worst case of CPU running at max speed of 1Ghz.
+	 */
+	mov	r2, #0x2000
+1:
+	subs	r2, r2, #0x1
+	bne	1b
+
+	/* Bit clears when operation is complete */
+2:	ldr     r1, [r0, #EMIF_READ_WRITE_LEVELING_CONTROL]
+	tst     r1, #RDWRLVLFULL_START
+	bne     2b
+
+skip_hwlvl:
+	mov	pc, lr
+ENDPROC(ti_emif_run_hw_leveling)
+
+/*
  * void ti_emif_enter_sr(void)
  *
  * Programs the EMIF to tell the SDRAM to enter into self-refresh
diff -Naur --no-dereference a/drivers/mfd/stmpe.c b/drivers/mfd/stmpe.c
--- a/drivers/mfd/stmpe.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/mfd/stmpe.c	2020-06-19 21:00:31.416227715 -0400
@@ -1,9 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0-only
 /*
  * ST Microelectronics MFD: stmpe's driver
  *
  * Copyright (C) ST-Ericsson SA 2010
  *
- * License Terms: GNU General Public License, version 2
  * Author: Rabin Vincent <rabin.vincent@stericsson.com> for ST-Ericsson
  */
 
@@ -464,6 +464,28 @@
 };
 
 /*
+ * ADC (STMPE811)
+ */
+
+static struct resource stmpe_adc_resources[] = {
+	{
+		.name	= "STMPE_TEMP_SENS",
+		.flags	= IORESOURCE_IRQ,
+	},
+	{
+		.name	= "STMPE_ADC",
+		.flags	= IORESOURCE_IRQ,
+	},
+};
+
+static const struct mfd_cell stmpe_adc_cell = {
+	.name		= "stmpe-adc",
+	.of_compatible	= "st,stmpe-adc",
+	.resources	= stmpe_adc_resources,
+	.num_resources	= ARRAY_SIZE(stmpe_adc_resources),
+};
+
+/*
  * STMPE811 or STMPE610
  */
 
@@ -497,6 +519,11 @@
 		.irq	= STMPE811_IRQ_TOUCH_DET,
 		.block	= STMPE_BLOCK_TOUCHSCREEN,
 	},
+	{
+		.cell	= &stmpe_adc_cell,
+		.irq	= STMPE811_IRQ_TEMP_SENS,
+		.block	= STMPE_BLOCK_ADC,
+	},
 };
 
 static int stmpe811_enable(struct stmpe *stmpe, unsigned int blocks,
@@ -517,6 +544,35 @@
 				enable ? 0 : mask);
 }
 
+int stmpe811_adc_common_init(struct stmpe *stmpe)
+{
+	int ret;
+	u8 adc_ctrl1, adc_ctrl1_mask;
+
+	adc_ctrl1 = STMPE_SAMPLE_TIME(stmpe->sample_time) |
+		    STMPE_MOD_12B(stmpe->mod_12b) |
+		    STMPE_REF_SEL(stmpe->ref_sel);
+	adc_ctrl1_mask = STMPE_SAMPLE_TIME(0xff) | STMPE_MOD_12B(0xff) |
+			 STMPE_REF_SEL(0xff);
+
+	ret = stmpe_set_bits(stmpe, STMPE811_REG_ADC_CTRL1,
+			adc_ctrl1_mask, adc_ctrl1);
+	if (ret) {
+		dev_err(stmpe->dev, "Could not setup ADC\n");
+		return ret;
+	}
+
+	ret = stmpe_set_bits(stmpe, STMPE811_REG_ADC_CTRL2,
+			STMPE_ADC_FREQ(0xff), STMPE_ADC_FREQ(stmpe->adc_freq));
+	if (ret) {
+		dev_err(stmpe->dev, "Could not setup ADC\n");
+		return ret;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(stmpe811_adc_common_init);
+
 static int stmpe811_get_altfunc(struct stmpe *stmpe, enum stmpe_block block)
 {
 	/* 0 for touchscreen, 1 for GPIO */
@@ -1302,17 +1358,17 @@
 	pdata->autosleep = (pdata->autosleep_timeout) ? true : false;
 
 	for_each_child_of_node(np, child) {
-		if (!strcmp(child->name, "stmpe_gpio")) {
+		if (of_node_name_eq(child, "stmpe_gpio")) {
 			pdata->blocks |= STMPE_BLOCK_GPIO;
-		} else if (!strcmp(child->name, "stmpe_keypad")) {
+		} else if (of_node_name_eq(child, "stmpe_keypad")) {
 			pdata->blocks |= STMPE_BLOCK_KEYPAD;
-		} else if (!strcmp(child->name, "stmpe_touchscreen")) {
+		} else if (of_node_name_eq(child, "stmpe_touchscreen")) {
 			pdata->blocks |= STMPE_BLOCK_TOUCHSCREEN;
-		} else if (!strcmp(child->name, "stmpe_adc")) {
+		} else if (of_node_name_eq(child, "stmpe_adc")) {
 			pdata->blocks |= STMPE_BLOCK_ADC;
-		} else if (!strcmp(child->name, "stmpe_pwm")) {
+		} else if (of_node_name_eq(child, "stmpe_pwm")) {
 			pdata->blocks |= STMPE_BLOCK_PWM;
-		} else if (!strcmp(child->name, "stmpe_rotator")) {
+		} else if (of_node_name_eq(child, "stmpe_rotator")) {
 			pdata->blocks |= STMPE_BLOCK_ROTATOR;
 		}
 	}
@@ -1325,6 +1381,7 @@
 	struct device_node *np = ci->dev->of_node;
 	struct stmpe *stmpe;
 	int ret;
+	u32 val;
 
 	pdata = devm_kzalloc(ci->dev, sizeof(*pdata), GFP_KERNEL);
 	if (!pdata)
@@ -1342,6 +1399,15 @@
 	mutex_init(&stmpe->irq_lock);
 	mutex_init(&stmpe->lock);
 
+	if (!of_property_read_u32(np, "st,sample-time", &val))
+		stmpe->sample_time = val;
+	if (!of_property_read_u32(np, "st,mod-12b", &val))
+		stmpe->mod_12b = val;
+	if (!of_property_read_u32(np, "st,ref-sel", &val))
+		stmpe->ref_sel = val;
+	if (!of_property_read_u32(np, "st,adc-freq", &val))
+		stmpe->adc_freq = val;
+
 	stmpe->dev = ci->dev;
 	stmpe->client = ci->client;
 	stmpe->pdata = pdata;
@@ -1433,6 +1499,8 @@
 	if (!IS_ERR(stmpe->vcc))
 		regulator_disable(stmpe->vcc);
 
+	__stmpe_disable(stmpe, STMPE_BLOCK_ADC);
+
 	mfd_remove_devices(stmpe->dev);
 
 	return 0;
diff -Naur --no-dereference a/drivers/mfd/ti_am335x_tscadc.c b/drivers/mfd/ti_am335x_tscadc.c
--- a/drivers/mfd/ti_am335x_tscadc.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/mfd/ti_am335x_tscadc.c	2020-06-19 21:00:31.416227715 -0400
@@ -182,11 +182,11 @@
 		tscadc->irq = err;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	tscadc->tscadc_phys_base = res->start;
 	tscadc->tscadc_base = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(tscadc->tscadc_base))
 		return PTR_ERR(tscadc->tscadc_base);
 
+	tscadc->tscadc_phys_base = res->start;
 	tscadc->regmap = devm_regmap_init_mmio(&pdev->dev,
 			tscadc->tscadc_base, &tscadc_regmap_config);
 	if (IS_ERR(tscadc->regmap)) {
@@ -270,7 +270,6 @@
 	if (err < 0)
 		goto err_disable_clk;
 
-	device_init_wakeup(&pdev->dev, true);
 	platform_set_drvdata(pdev, tscadc);
 	return 0;
 
diff -Naur --no-dereference a/drivers/misc/cape/beaglebone/bone-pinmux-helper.c b/drivers/misc/cape/beaglebone/bone-pinmux-helper.c
--- a/drivers/misc/cape/beaglebone/bone-pinmux-helper.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/misc/cape/beaglebone/bone-pinmux-helper.c	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,242 @@
+/*
+ * Pinmux helper driver
+ *
+ * Copyright (C) 2013 Pantelis Antoniou <panto@antoniou-consulting.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/init.h>
+#include <linux/err.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/of_gpio.h>
+#include <linux/slab.h>
+#include <linux/pinctrl/pinctrl.h>
+#include <linux/pinctrl/pinmux.h>
+#include <linux/pinctrl/consumer.h>
+
+static const struct of_device_id bone_pinmux_helper_of_match[] = {
+	{
+		.compatible = "bone-pinmux-helper",
+	},
+	{ },
+};
+MODULE_DEVICE_TABLE(of, bone_pinmux_helper_of_match);
+
+struct pinmux_helper_data {
+	struct pinctrl *pinctrl;
+	char *selected_state_name;
+};
+
+static ssize_t pinmux_helper_show_state(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct pinmux_helper_data *data = platform_get_drvdata(pdev);
+	const char *name;
+
+	name = data->selected_state_name;
+	if (name == NULL || strlen(name) == 0)
+		name = "none";
+	return sprintf(buf, "%s\n", name);
+}
+
+static ssize_t pinmux_helper_store_state(struct device *dev,
+		struct device_attribute *attr, const char *buf, size_t count)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct pinmux_helper_data *data = platform_get_drvdata(pdev);
+	struct pinctrl_state *state;
+	char *state_name;
+	char *s;
+	int err;
+
+	/* duplicate (as a null terminated string) */
+	state_name = kmalloc(count + 1, GFP_KERNEL);
+	if (state_name == NULL)
+		return -ENOMEM;
+	memcpy(state_name, buf, count);
+	state_name[count] = '\0';
+
+	/* and chop off newline */
+	s = strchr(state_name, '\n');
+	if (s != NULL)
+		*s = '\0';
+
+	/* try to select default state at first (if it exists) */
+	state = pinctrl_lookup_state(data->pinctrl, state_name);
+	if (!IS_ERR(state)) {
+		err = pinctrl_select_state(data->pinctrl, state);
+		if (err != 0)
+			dev_err(dev, "Failed to select state %s\n",
+					state_name);
+	} else {
+		dev_err(dev, "Failed to find state %s\n", state_name);
+		err = PTR_RET(state);
+	}
+
+	if (err == 0) {
+		kfree(data->selected_state_name);
+		data->selected_state_name = state_name;
+	}
+
+	return err ? err : count;
+}
+
+static DEVICE_ATTR(state, S_IWUSR | S_IRUGO,
+		   pinmux_helper_show_state, pinmux_helper_store_state);
+
+static struct attribute *pinmux_helper_attributes[] = {
+	&dev_attr_state.attr,
+	NULL
+};
+
+static const struct attribute_group pinmux_helper_attr_group = {
+	.attrs = pinmux_helper_attributes,
+};
+
+static int bone_pinmux_helper_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct pinmux_helper_data *data;
+	struct pinctrl_state *state;
+	char *state_name;
+	const char *mode_name;
+	int mode_len;
+	int err;
+
+	data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL);
+	if (data == NULL) {
+		dev_err(dev, "Failed to allocate data\n");
+		err = -ENOMEM;
+		goto err_no_mem;
+	}
+
+	state_name = kmalloc(strlen(PINCTRL_STATE_DEFAULT) + 1,
+			GFP_KERNEL);
+	if (state_name == NULL) {
+		dev_err(dev, "Failed to allocate state name\n");
+		err = -ENOMEM;
+		goto err_no_state_mem;
+	}
+	data->selected_state_name = state_name;
+	strcpy(data->selected_state_name, PINCTRL_STATE_DEFAULT);
+
+	platform_set_drvdata(pdev, data);
+
+	data->pinctrl = devm_pinctrl_get(dev);
+	if (IS_ERR(data->pinctrl)) {
+		dev_err(dev, "Failed to get pinctrl\n");
+		err = PTR_RET(data->pinctrl);
+		goto err_no_pinctrl;
+	}
+
+	/* See if an initial mode is specified in the device tree */
+	mode_name = of_get_property(dev->of_node, "mode", &mode_len);
+
+	err = -1;
+	if (mode_name != NULL ) {
+		state_name = kmalloc(mode_len + 1, GFP_KERNEL);
+		if (state_name == NULL) {
+			dev_err(dev, "Failed to allocate state name\n");
+			err = -ENOMEM;
+			goto err_no_mode_mem;
+		}
+		strncpy(state_name, mode_name, mode_len);
+
+		/* try to select requested mode */
+		state = pinctrl_lookup_state(data->pinctrl, state_name);
+		if (!IS_ERR(state)) {
+			err = pinctrl_select_state(data->pinctrl, state);
+			if (err != 0) {
+				dev_warn(dev, "Unable to select requested mode %s\n", state_name);
+				kfree(state_name);
+			} else {
+				kfree(data->selected_state_name);
+				data->selected_state_name = state_name;
+				dev_notice(dev, "Set initial pinmux mode to %s\n", state_name);
+			}
+		}
+	}
+
+	/* try to select default state if mode_name failed */
+	if ( err != 0) {
+		state = pinctrl_lookup_state(data->pinctrl,
+				data->selected_state_name);
+		if (!IS_ERR(state)) {
+			err = pinctrl_select_state(data->pinctrl, state);
+			if (err != 0) {
+				dev_err(dev, "Failed to select default state\n");
+				goto err_no_state;
+			}
+		} else {
+			data->selected_state_name = '\0';
+		}
+	}
+
+	/* Register sysfs hooks */
+	err = sysfs_create_group(&dev->kobj, &pinmux_helper_attr_group);
+	if (err) {
+		dev_err(dev, "Failed to create sysfs group\n");
+		goto err_no_sysfs;
+	}
+
+	return 0;
+
+err_no_sysfs:
+err_no_state:
+err_no_mode_mem:
+	devm_pinctrl_put(data->pinctrl);
+err_no_pinctrl:
+	devm_kfree(dev, data->selected_state_name);
+err_no_state_mem:
+	devm_kfree(dev, data);
+err_no_mem:
+	return err;
+}
+
+static int bone_pinmux_helper_remove(struct platform_device *pdev)
+{
+	struct pinmux_helper_data *data = platform_get_drvdata(pdev);
+	struct device *dev = &pdev->dev;
+
+	sysfs_remove_group(&dev->kobj, &pinmux_helper_attr_group);
+	kfree(data->selected_state_name);
+	devm_pinctrl_put(data->pinctrl);
+	devm_kfree(dev, data);
+
+	return 0;
+}
+
+struct platform_driver bone_pinmux_helper_driver = {
+	.probe		= bone_pinmux_helper_probe,
+	.remove		= bone_pinmux_helper_remove,
+	.driver = {
+		.name		= "bone-pinmux-helper",
+		.owner		= THIS_MODULE,
+		.of_match_table	= bone_pinmux_helper_of_match,
+	},
+};
+
+module_platform_driver(bone_pinmux_helper_driver);
+
+MODULE_AUTHOR("Pantelis Antoniou");
+MODULE_DESCRIPTION("Beaglebone pinmux helper driver");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:bone-pinmux-helper");
diff -Naur --no-dereference a/drivers/misc/cape/beaglebone/Kconfig b/drivers/misc/cape/beaglebone/Kconfig
--- a/drivers/misc/cape/beaglebone/Kconfig	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/misc/cape/beaglebone/Kconfig	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,10 @@
+#
+# Beaglebone capes
+#
+
+config BEAGLEBONE_PINMUX_HELPER
+	tristate "Beaglebone Pinmux Helper"
+	depends on ARCH_OMAP2PLUS && OF
+	default n
+	help
+	  Say Y here to include support for the pinmux helper
diff -Naur --no-dereference a/drivers/misc/cape/beaglebone/Makefile b/drivers/misc/cape/beaglebone/Makefile
--- a/drivers/misc/cape/beaglebone/Makefile	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/misc/cape/beaglebone/Makefile	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,5 @@
+#
+# Makefile for beaglebone capes
+#
+
+obj-$(CONFIG_BEAGLEBONE_PINMUX_HELPER)	+= bone-pinmux-helper.o
diff -Naur --no-dereference a/drivers/misc/cape/Kconfig b/drivers/misc/cape/Kconfig
--- a/drivers/misc/cape/Kconfig	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/misc/cape/Kconfig	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,5 @@
+#
+# Capes
+#
+
+source "drivers/misc/cape/beaglebone/Kconfig"
diff -Naur --no-dereference a/drivers/misc/cape/Makefile b/drivers/misc/cape/Makefile
--- a/drivers/misc/cape/Makefile	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/misc/cape/Makefile	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,5 @@
+#
+# Makefile for cape like devices
+#
+
+obj-y				+= beaglebone/
diff -Naur --no-dereference a/drivers/misc/dma-buf-phys.c b/drivers/misc/dma-buf-phys.c
--- a/drivers/misc/dma-buf-phys.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/misc/dma-buf-phys.c	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,217 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * DMA-BUF contiguous buffer physical address user-space exporter
+ *
+ * Copyright (C) 2018-2019 Texas Instruments Incorporated - http://www.ti.com/
+ *	Andrew F. Davis <afd@ti.com>
+ */
+
+#include <linux/device.h>
+#include <linux/dma-buf.h>
+#include <linux/err.h>
+#include <linux/kernel.h>
+#include <linux/miscdevice.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+
+#include <uapi/linux/dma_buf_phys.h>
+
+#define DEVICE_NAME "dma-buf-phys"
+
+struct dma_buf_phys_priv {
+	struct miscdevice miscdev;
+};
+
+struct dma_buf_phys_file {
+	struct device *dev;
+	struct dma_buf *dma_buf;
+	struct dma_buf_attachment *attachment;
+	struct sg_table *sgt;
+};
+
+static int dma_buf_phys_open(struct inode *inode, struct file *file)
+{
+	struct miscdevice *miscdev = file->private_data;
+	struct device *dev = miscdev->this_device;
+	struct dma_buf_phys_file *priv;
+
+	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+	priv->dev = dev;
+	file->private_data = (void *)priv;
+
+	return 0;
+}
+
+static int dma_buf_phys_release(struct inode *inode, struct file *file)
+{
+	struct dma_buf_phys_file *priv = file->private_data;
+
+	if (priv->attachment && priv->sgt)
+		dma_buf_unmap_attachment(priv->attachment, priv->sgt, DMA_BIDIRECTIONAL);
+	if (priv->dma_buf && priv->attachment)
+		dma_buf_detach(priv->dma_buf, priv->attachment);
+	if (priv->dma_buf)
+		dma_buf_put(priv->dma_buf);
+
+	kfree(priv);
+
+	return 0;
+}
+
+static int dma_buf_phys_convert(struct dma_buf_phys_file *priv, int fd, u64 *phys)
+{
+	struct device *dev = priv->dev;
+	struct dma_buf *dma_buf;
+	struct dma_buf_attachment *attachment;
+	struct sg_table *sgt;
+	dma_addr_t dma_addr;
+	int ret;
+
+	dma_buf = dma_buf_get(fd);
+	if (IS_ERR(dma_buf))
+		return PTR_ERR(dma_buf);
+
+	/* Attach as the parent device as it will have the correct DMA ops set */
+	attachment = dma_buf_attach(dma_buf, dev->parent);
+	if (IS_ERR(attachment)) {
+		ret = PTR_ERR(attachment);
+		goto fail_put;
+	}
+
+	sgt = dma_buf_map_attachment(attachment, DMA_BIDIRECTIONAL);
+	if (IS_ERR(sgt)) {
+		ret = PTR_ERR(sgt);
+		goto fail_detach;
+	}
+
+	/* Without PAT only physically contiguous buffers can be supported */
+	if (sgt->orig_nents != 1) {
+		dev_err(dev, "DMA-BUF not contiguous\n");
+		ret = -EINVAL;
+		goto fail_unmap;
+	}
+
+	dma_addr = sg_dma_address(sgt->sgl);
+
+	*phys = dma_addr;
+
+	priv->dma_buf = dma_buf;
+	priv->attachment = attachment;
+	priv->sgt = sgt;
+
+	return 0;
+
+fail_unmap:
+	dma_buf_unmap_attachment(attachment, sgt, DMA_BIDIRECTIONAL);
+fail_detach:
+	dma_buf_detach(dma_buf, attachment);
+fail_put:
+	dma_buf_put(dma_buf);
+
+	return ret;
+}
+
+static long dma_buf_phys_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	struct dma_buf_phys_file *priv = file->private_data;
+
+	switch (cmd) {
+	case DMA_BUF_PHYS_IOC_CONVERT:
+	{
+		struct dma_buf_phys_data data;
+		int ret;
+
+		/*
+		 * TODO: this should likely be properly serialized, but I
+		 * see no reason this file would ever need to be shared.
+		 */
+		/* one attachment per file */
+		if (priv->dma_buf)
+			return -EFAULT;
+
+		if (copy_from_user(&data, (void __user *)arg, _IOC_SIZE(cmd)))
+			return -EFAULT;
+
+		ret = dma_buf_phys_convert(priv, data.fd, &data.phys);
+		if (ret)
+			return ret;
+
+		if (copy_to_user((void __user *)arg, &data, _IOC_SIZE(cmd)))
+			return -EFAULT;
+
+		break;
+	}
+	default:
+		return -ENOTTY;
+	}
+
+	return 0;
+}
+
+static const struct file_operations dma_buf_phys_fops = {
+	.owner = THIS_MODULE,
+	.open = dma_buf_phys_open,
+	.release = dma_buf_phys_release,
+	.unlocked_ioctl = dma_buf_phys_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl	= dma_buf_phys_ioctl,
+#endif
+};
+
+static int dma_buf_phys_probe(struct platform_device *pdev)
+{
+	struct dma_buf_phys_priv *priv;
+	struct device *dev = &pdev->dev;
+	int err;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+	dev_set_drvdata(dev, priv);
+
+	priv->miscdev.minor = MISC_DYNAMIC_MINOR;
+	priv->miscdev.name = devm_kasprintf(dev, GFP_KERNEL, "%s", DEVICE_NAME);
+	priv->miscdev.fops = &dma_buf_phys_fops;
+	priv->miscdev.parent = dev;
+	err = misc_register(&priv->miscdev);
+	if (err) {
+		dev_err(dev, "unable to register DMA-BUF to Phys misc device\n");
+		return err;
+	}
+
+	return 0;
+}
+
+static int dma_buf_phys_remove(struct platform_device *pdev)
+{
+	struct dma_buf_phys_priv *priv = dev_get_drvdata(&pdev->dev);
+
+	misc_deregister(&priv->miscdev);
+
+	return 0;
+}
+
+static const struct of_device_id dma_buf_phys_of_match[] = {
+	{ .compatible = "ti,dma_buf_phys", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, dma_buf_phys_of_match);
+
+static struct platform_driver dma_buf_phys_driver = {
+	.probe = dma_buf_phys_probe,
+	.remove = dma_buf_phys_remove,
+	.driver = {
+		.name = "dma_buf_phys",
+		.of_match_table = dma_buf_phys_of_match,
+	}
+};
+module_platform_driver(dma_buf_phys_driver);
+
+MODULE_AUTHOR("Andrew F. Davis <afd@ti.com>");
+MODULE_DESCRIPTION("DMA-BUF contiguous buffer physical address user-space exporter");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/misc/Kconfig b/drivers/misc/Kconfig
--- a/drivers/misc/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/misc/Kconfig	2020-06-19 21:00:31.416227715 -0400
@@ -513,6 +513,22 @@
 	tristate
 	default MISC_RTSX_PCI || MISC_RTSX_USB
 
+config DMA_BUF_PHYS
+	tristate "DMA-BUF physical address user-space exporter"
+	help
+	  Exports CPU physical address of DMA-BUF to user-space.
+
+config TIEQEP
+	tristate "EQEP Hardware quadrature encoder controller"
+	depends on SOC_AM33XX
+	select PWM_TIPWMSS
+	help
+	  Driver support for the EQEP quadrature encoder controller AM33XX
+	  TI SOC
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called tieqep.
+
 source "drivers/misc/c2port/Kconfig"
 source "drivers/misc/eeprom/Kconfig"
 source "drivers/misc/cb710/Kconfig"
@@ -523,6 +539,7 @@
 source "drivers/misc/vmw_vmci/Kconfig"
 source "drivers/misc/mic/Kconfig"
 source "drivers/misc/genwqe/Kconfig"
+source "drivers/misc/cape/Kconfig"
 source "drivers/misc/echo/Kconfig"
 source "drivers/misc/cxl/Kconfig"
 source "drivers/misc/ocxl/Kconfig"
diff -Naur --no-dereference a/drivers/misc/Makefile b/drivers/misc/Makefile
--- a/drivers/misc/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/misc/Makefile	2020-06-19 21:00:31.416227715 -0400
@@ -43,6 +43,7 @@
 obj-y				+= lis3lv02d/
 obj-$(CONFIG_USB_SWITCH_FSA9480) += fsa9480.o
 obj-$(CONFIG_ALTERA_STAPL)	+=altera-stapl/
+obj-$(CONFIG_TIEQEP)		+= tieqep.o
 obj-$(CONFIG_INTEL_MEI)		+= mei/
 obj-$(CONFIG_VMWARE_VMCI)	+= vmw_vmci/
 obj-$(CONFIG_LATTICE_ECP3_CONFIG)	+= lattice-ecp3-config.o
@@ -50,6 +51,7 @@
 obj-$(CONFIG_SRAM_EXEC)		+= sram-exec.o
 obj-y				+= mic/
 obj-$(CONFIG_GENWQE)		+= genwqe/
+obj-y				+= cape/
 obj-$(CONFIG_ECHO)		+= echo/
 obj-$(CONFIG_VEXPRESS_SYSCFG)	+= vexpress-syscfg.o
 obj-$(CONFIG_CXL_BASE)		+= cxl/
@@ -58,3 +60,4 @@
 obj-$(CONFIG_PCI_ENDPOINT_TEST)	+= pci_endpoint_test.o
 obj-$(CONFIG_OCXL)		+= ocxl/
 obj-$(CONFIG_MISC_RTSX)		+= cardreader/
+obj-$(CONFIG_DMA_BUF_PHYS)	+= dma-buf-phys.o
diff -Naur --no-dereference a/drivers/misc/pci_endpoint_test.c b/drivers/misc/pci_endpoint_test.c
--- a/drivers/misc/pci_endpoint_test.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/misc/pci_endpoint_test.c	2020-06-19 21:00:31.416227715 -0400
@@ -75,6 +75,21 @@
 #define PCI_ENDPOINT_TEST_IRQ_TYPE		0x24
 #define PCI_ENDPOINT_TEST_IRQ_NUMBER		0x28
 
+#define PCI_DEVICE_ID_TI_J721E			0xb00d
+#define PCI_DEVICE_ID_TI_AM654			0xb00c
+#define PCI_DEVICE_ID_TI_K2G			0xb00b
+
+#define is_am654_pci_dev(pdev)		\
+		((pdev)->device == PCI_DEVICE_ID_TI_AM654)
+
+#define is_j721e_pci_dev(pdev)		\
+		((pdev)->device == PCI_DEVICE_ID_TI_J721E)
+
+#define K2G_IB_START_L0(n)		(0x304 + (0x10 * (n)))
+#define K2G_IB_START_HI(n)		(0x308 + (0x10 * (n)))
+
+#define is_k2g_pci_dev(pdev)		((pdev)->device == PCI_DEVICE_ID_TI_K2G)
+
 static DEFINE_IDA(pci_endpoint_test_ida);
 
 #define to_endpoint_test(priv) container_of((priv), struct pci_endpoint_test, \
@@ -104,11 +119,13 @@
 	struct completion irq_raised;
 	int		last_irq;
 	int		num_irqs;
+	int		irq_type;
 	/* mutex to protect the ioctls */
 	struct mutex	mutex;
 	struct miscdevice miscdev;
 	enum pci_barno test_reg_bar;
 	size_t alignment;
+	const char *name;
 };
 
 struct pci_endpoint_test_data {
@@ -163,6 +180,7 @@
 	struct pci_dev *pdev = test->pdev;
 
 	pci_free_irq_vectors(pdev);
+	test->irq_type = IRQ_TYPE_UNDEFINED;
 }
 
 static bool pci_endpoint_test_alloc_irq_vectors(struct pci_endpoint_test *test,
@@ -197,6 +215,8 @@
 		irq = 0;
 		res = false;
 	}
+
+	test->irq_type = type;
 	test->num_irqs = irq;
 
 	return res;
@@ -224,7 +244,7 @@
 	for (i = 0; i < test->num_irqs; i++) {
 		err = devm_request_irq(dev, pci_irq_vector(pdev, i),
 				       pci_endpoint_test_irqhandler,
-				       IRQF_SHARED, DRV_MODULE_NAME, test);
+				       IRQF_SHARED, test->name, test);
 		if (err)
 			goto fail;
 	}
@@ -336,6 +356,7 @@
 	dma_addr_t orig_dst_phys_addr;
 	size_t offset;
 	size_t alignment = test->alignment;
+	int irq_type = test->irq_type;
 	u32 src_crc32;
 	u32 dst_crc32;
 
@@ -347,14 +368,22 @@
 		goto err;
 	}
 
-	orig_src_addr = dma_alloc_coherent(dev, size + alignment,
-					   &orig_src_phys_addr, GFP_KERNEL);
+	orig_src_addr = kzalloc(size + alignment, GFP_KERNEL);
 	if (!orig_src_addr) {
 		dev_err(dev, "Failed to allocate source buffer\n");
 		ret = false;
 		goto err;
 	}
 
+	get_random_bytes(orig_src_addr, size + alignment);
+	orig_src_phys_addr = dma_map_single(dev, orig_src_addr,
+					    size + alignment, DMA_TO_DEVICE);
+	if (dma_mapping_error(dev, orig_src_phys_addr)) {
+		dev_err(dev, "failed to map source buffer address\n");
+		ret = false;
+		goto err_src_phys_addr;
+	}
+
 	if (alignment && !IS_ALIGNED(orig_src_phys_addr, alignment)) {
 		src_phys_addr = PTR_ALIGN(orig_src_phys_addr, alignment);
 		offset = src_phys_addr - orig_src_phys_addr;
@@ -370,15 +399,21 @@
 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_UPPER_SRC_ADDR,
 				 upper_32_bits(src_phys_addr));
 
-	get_random_bytes(src_addr, size);
 	src_crc32 = crc32_le(~0, src_addr, size);
 
-	orig_dst_addr = dma_alloc_coherent(dev, size + alignment,
-					   &orig_dst_phys_addr, GFP_KERNEL);
+	orig_dst_addr = kzalloc(size + alignment, GFP_KERNEL);
 	if (!orig_dst_addr) {
 		dev_err(dev, "Failed to allocate destination address\n");
 		ret = false;
-		goto err_orig_src_addr;
+		goto err_dst_addr;
+	}
+
+	orig_dst_phys_addr = dma_map_single(dev, orig_dst_addr,
+					    size + alignment, DMA_FROM_DEVICE);
+	if (dma_mapping_error(dev, orig_dst_phys_addr)) {
+		dev_err(dev, "failed to map destination buffer address\n");
+		ret = false;
+		goto err_dst_phys_addr;
 	}
 
 	if (alignment && !IS_ALIGNED(orig_dst_phys_addr, alignment)) {
@@ -405,16 +440,22 @@
 
 	wait_for_completion(&test->irq_raised);
 
+	dma_unmap_single(dev, orig_dst_phys_addr, size + alignment,
+			 DMA_FROM_DEVICE);
+
 	dst_crc32 = crc32_le(~0, dst_addr, size);
 	if (dst_crc32 == src_crc32)
 		ret = true;
 
-	dma_free_coherent(dev, size + alignment, orig_dst_addr,
-			  orig_dst_phys_addr);
+err_dst_phys_addr:
+	kfree(orig_dst_addr);
+
+err_dst_addr:
+	dma_unmap_single(dev, orig_src_phys_addr, size + alignment,
+			 DMA_TO_DEVICE);
 
-err_orig_src_addr:
-	dma_free_coherent(dev, size + alignment, orig_src_addr,
-			  orig_src_phys_addr);
+err_src_phys_addr:
+	kfree(orig_src_addr);
 
 err:
 	return ret;
@@ -432,6 +473,7 @@
 	dma_addr_t orig_phys_addr;
 	size_t offset;
 	size_t alignment = test->alignment;
+	int irq_type = test->irq_type;
 	u32 crc32;
 
 	if (size > SIZE_MAX - alignment)
@@ -442,14 +484,23 @@
 		goto err;
 	}
 
-	orig_addr = dma_alloc_coherent(dev, size + alignment, &orig_phys_addr,
-				       GFP_KERNEL);
+	orig_addr = kzalloc(size + alignment, GFP_KERNEL);
 	if (!orig_addr) {
 		dev_err(dev, "Failed to allocate address\n");
 		ret = false;
 		goto err;
 	}
 
+	get_random_bytes(orig_addr, size + alignment);
+
+	orig_phys_addr = dma_map_single(dev, orig_addr, size + alignment,
+					DMA_TO_DEVICE);
+	if (dma_mapping_error(dev, orig_phys_addr)) {
+		dev_err(dev, "failed to map source buffer address\n");
+		ret = false;
+		goto err_phys_addr;
+	}
+
 	if (alignment && !IS_ALIGNED(orig_phys_addr, alignment)) {
 		phys_addr =  PTR_ALIGN(orig_phys_addr, alignment);
 		offset = phys_addr - orig_phys_addr;
@@ -459,8 +510,6 @@
 		addr = orig_addr;
 	}
 
-	get_random_bytes(addr, size);
-
 	crc32 = crc32_le(~0, addr, size);
 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_CHECKSUM,
 				 crc32);
@@ -483,7 +532,11 @@
 	if (reg & STATUS_READ_SUCCESS)
 		ret = true;
 
-	dma_free_coherent(dev, size + alignment, orig_addr, orig_phys_addr);
+	dma_unmap_single(dev, orig_phys_addr, size + alignment,
+			 DMA_TO_DEVICE);
+
+err_phys_addr:
+	kfree(orig_addr);
 
 err:
 	return ret;
@@ -500,6 +553,7 @@
 	dma_addr_t orig_phys_addr;
 	size_t offset;
 	size_t alignment = test->alignment;
+	int irq_type = test->irq_type;
 	u32 crc32;
 
 	if (size > SIZE_MAX - alignment)
@@ -510,14 +564,21 @@
 		goto err;
 	}
 
-	orig_addr = dma_alloc_coherent(dev, size + alignment, &orig_phys_addr,
-				       GFP_KERNEL);
+	orig_addr = kzalloc(size + alignment, GFP_KERNEL);
 	if (!orig_addr) {
 		dev_err(dev, "Failed to allocate destination address\n");
 		ret = false;
 		goto err;
 	}
 
+	orig_phys_addr = dma_map_single(dev, orig_addr, size + alignment,
+					DMA_FROM_DEVICE);
+	if (dma_mapping_error(dev, orig_phys_addr)) {
+		dev_err(dev, "failed to map source buffer address\n");
+		ret = false;
+		goto err_phys_addr;
+	}
+
 	if (alignment && !IS_ALIGNED(orig_phys_addr, alignment)) {
 		phys_addr = PTR_ALIGN(orig_phys_addr, alignment);
 		offset = phys_addr - orig_phys_addr;
@@ -541,15 +602,26 @@
 
 	wait_for_completion(&test->irq_raised);
 
+	dma_unmap_single(dev, orig_phys_addr, size + alignment,
+			 DMA_FROM_DEVICE);
+
 	crc32 = crc32_le(~0, addr, size);
 	if (crc32 == pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_CHECKSUM))
 		ret = true;
 
-	dma_free_coherent(dev, size + alignment, orig_addr, orig_phys_addr);
+err_phys_addr:
+	kfree(orig_addr);
 err:
 	return ret;
 }
 
+static bool pci_endpoint_test_clear_irq(struct pci_endpoint_test *test)
+{
+	pci_endpoint_test_release_irq(test);
+	pci_endpoint_test_free_irq_vectors(test);
+	return true;
+}
+
 static bool pci_endpoint_test_set_irq(struct pci_endpoint_test *test,
 				      int req_irq_type)
 {
@@ -561,7 +633,7 @@
 		return false;
 	}
 
-	if (irq_type == req_irq_type)
+	if (test->irq_type == req_irq_type)
 		return true;
 
 	pci_endpoint_test_release_irq(test);
@@ -573,12 +645,10 @@
 	if (!pci_endpoint_test_request_irq(test))
 		goto err;
 
-	irq_type = req_irq_type;
 	return true;
 
 err:
 	pci_endpoint_test_free_irq_vectors(test);
-	irq_type = IRQ_TYPE_UNDEFINED;
 	return false;
 }
 
@@ -588,6 +658,7 @@
 	int ret = -EINVAL;
 	enum pci_barno bar;
 	struct pci_endpoint_test *test = to_endpoint_test(file->private_data);
+	struct pci_dev *pdev = test->pdev;
 
 	mutex_lock(&test->mutex);
 	switch (cmd) {
@@ -595,6 +666,9 @@
 		bar = arg;
 		if (bar < 0 || bar > 5)
 			goto ret;
+		if ((is_am654_pci_dev(pdev) || is_k2g_pci_dev(pdev)) &&
+		    bar == BAR_0)
+			goto ret;
 		ret = pci_endpoint_test_bar(test, bar);
 		break;
 	case PCITEST_LEGACY_IRQ:
@@ -619,6 +693,9 @@
 	case PCITEST_GET_IRQTYPE:
 		ret = irq_type;
 		break;
+	case PCITEST_CLEAR_IRQ:
+		ret = pci_endpoint_test_clear_irq(test);
+		break;
 	}
 
 ret:
@@ -631,12 +708,34 @@
 	.unlocked_ioctl = pci_endpoint_test_ioctl,
 };
 
+static int pci_endpoint_test_k2g_init(struct pci_endpoint_test *test)
+{
+	struct pci_dev *pdev = test->pdev;
+	enum pci_barno bar;
+	resource_size_t start;
+
+	if (!test->bar[0])
+		return -EINVAL;
+
+	for (bar = BAR_1; bar <= BAR_5; bar++) {
+		start = pci_resource_start(pdev, bar);
+		pci_endpoint_test_bar_writel(test, BAR_0,
+					     K2G_IB_START_L0(bar - 1),
+					     lower_32_bits(start));
+		pci_endpoint_test_bar_writel(test, BAR_0,
+					     K2G_IB_START_HI(bar - 1),
+					     upper_32_bits(start));
+	}
+
+	return 0;
+}
+
 static int pci_endpoint_test_probe(struct pci_dev *pdev,
 				   const struct pci_device_id *ent)
 {
 	int err;
 	int id;
-	char name[20];
+	char name[24];
 	enum pci_barno bar;
 	void __iomem *base;
 	struct device *dev = &pdev->dev;
@@ -670,6 +769,12 @@
 	init_completion(&test->irq_raised);
 	mutex_init(&test->mutex);
 
+	if ((dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)) != 0) &&
+	    dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
+		dev_err(dev, "Cannot set DMA mask\n");
+		return -EINVAL;
+	}
+
 	err = pci_enable_device(pdev);
 	if (err) {
 		dev_err(dev, "Cannot enable PCI device\n");
@@ -683,12 +788,15 @@
 	}
 
 	pci_set_master(pdev);
+	pci_intx(pdev, true);
 
-	if (!pci_endpoint_test_alloc_irq_vectors(test, irq_type))
-		goto err_disable_irq;
+	if (!(is_am654_pci_dev(pdev) || is_j721e_pci_dev(pdev))) {
+		if (!pci_endpoint_test_alloc_irq_vectors(test, irq_type))
+			goto err_disable_irq;
 
-	if (!pci_endpoint_test_request_irq(test))
-		goto err_disable_irq;
+		if (!pci_endpoint_test_request_irq(test))
+			goto err_disable_irq;
+	}
 
 	for (bar = BAR_0; bar <= BAR_5; bar++) {
 		if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) {
@@ -709,6 +817,12 @@
 		goto err_iounmap;
 	}
 
+	if (is_k2g_pci_dev(pdev)) {
+		err = pci_endpoint_test_k2g_init(test);
+		if (err)
+			goto err_iounmap;
+	}
+
 	pci_set_drvdata(pdev, test);
 
 	id = ida_simple_get(&pci_endpoint_test_ida, 0, 0, GFP_KERNEL);
@@ -733,6 +847,7 @@
 		dev_err(dev, "Failed to register device\n");
 		goto err_kfree_name;
 	}
+	test->name = kstrdup(name, GFP_KERNEL);
 
 	return 0;
 
@@ -773,6 +888,7 @@
 
 	misc_deregister(&test->miscdev);
 	kfree(misc_device->name);
+	kfree(test->name);
 	ida_simple_remove(&pci_endpoint_test_ida, id);
 	for (bar = BAR_0; bar <= BAR_5; bar++) {
 		if (test->bar[bar])
@@ -786,10 +902,43 @@
 	pci_disable_device(pdev);
 }
 
+static const struct pci_endpoint_test_data default_data = {
+	.test_reg_bar = BAR_0,
+	.alignment = SZ_4K,
+};
+
+static const struct pci_endpoint_test_data am654_data = {
+	.test_reg_bar = BAR_2,
+	.alignment = SZ_64K,
+};
+
+static const struct pci_endpoint_test_data k2g_data = {
+	.test_reg_bar = BAR_1,
+	.alignment = SZ_1M,
+};
+
+static const struct pci_endpoint_test_data j721e_data = {
+	.alignment = 256,
+	.irq_type = IRQ_TYPE_MSI,
+};
+
 static const struct pci_device_id pci_endpoint_test_tbl[] = {
-	{ PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_DRA74x) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_DRA72x) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_SYNOPSYS, 0xedda) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_DRA74x),
+	  .driver_data = (kernel_ulong_t)&default_data,
+	},
+	{ PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_DRA72x),
+	  .driver_data = (kernel_ulong_t)&default_data,
+	},
+	{ PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_AM654),
+	  .driver_data = (kernel_ulong_t)&am654_data
+	},
+	{ PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_K2G),
+	  .driver_data = (kernel_ulong_t)&k2g_data
+	},
+	{ PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_J721E),
+	  .driver_data = (kernel_ulong_t)&j721e_data,
+	},
 	{ }
 };
 MODULE_DEVICE_TABLE(pci, pci_endpoint_test_tbl);
@@ -799,6 +948,7 @@
 	.id_table	= pci_endpoint_test_tbl,
 	.probe		= pci_endpoint_test_probe,
 	.remove		= pci_endpoint_test_remove,
+	.sriov_configure = pci_sriov_configure_simple,
 };
 module_pci_driver(pci_endpoint_test_driver);
 
diff -Naur --no-dereference a/drivers/misc/tieqep.c b/drivers/misc/tieqep.c
--- a/drivers/misc/tieqep.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/misc/tieqep.c	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,754 @@
+/*
+ * TI eQEP driver for AM33xx devices
+ *
+ * Copyright (C) 2013 Nathaniel R. Lewis - http://teknoman117.wordpress.com/
+ * Copyright (C) 2015 SoftPLC Corporation, Dick Hollenbeck <dick@softplc.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *
+ * sysfs entries
+ *	 - position = absolute - current position; relative - last latched value
+ *	 - mode => 0 - absolute; 1 - relative
+ *	 - period => sampling period for the hardware
+ *	 - enable => 0 - eQEP disabled, 1 - eQEP enabled
+ */
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/err.h>
+#include <linux/clk.h>
+#include <linux/pm_runtime.h>
+#include <linux/of_device.h>
+#include <linux/pinctrl/consumer.h>
+#include <linux/input.h>
+
+/* eQEP register offsets from its base IO address */
+#define QPOSCNT		0x0000
+#define QPOSINIT	0x0004
+#define QPOSMAX		0x0008
+#define QPOSCMP		0x000C
+#define QPOSILAT	0x0010
+#define QPOSSLAT	0x0014
+#define QPOSLAT		0x0018
+#define QUTMR		0x001C
+#define QUPRD		0x0020
+#define QWDTMR		0x0024
+#define QWDPRD		0x0026
+#define QDECCTL		0x0028
+#define QEPCTL		0x002A
+#define QCAPCTL		0x002C
+#define QPOSCTL		0x002E
+#define QEINT		0x0030
+#define QFLG		0x0032
+#define QCLR		0x0034
+#define QFRC		0x0036
+#define QEPSTS		0x0038
+#define QCTMR		0x003A
+#define QCPRD		0x003C
+#define QCTMRLAT	0x003E
+#define QCPRDLAT	0x0040
+#define QREVID		0x005C
+
+#if 0	/* if you wanted another way to modify IP registers... */
+typedef volatile u32	REG32;
+typedef volatile u16	REG16;
+struct EQEP_REGS {
+	REG32	q_poscnt;		/*	0x00	position counter */
+	REG32	q_posinit;		/*	0x04	position counter initialization */
+	REG32	q_posmax;		/*	0x08	maximum position count */
+	REG32	q_poscmp;		/*	0x0C	position compare */
+	REG32	q_posilat;		/*	0x10	index position latch */
+	REG32	q_posslat;		/*	0x14	strobe position latch */
+	REG32	q_poslat;		/*	0x18	position counter latch */
+	REG32	q_utmr;			/*	0x1C	unit timer */
+	REG32	q_uprd;			/*	0x20	unit period */
+	REG16	q_wdtmr;		/*	0x24	watchdog timer */
+	REG16	q_wdprd;		/*	0x26	watchdog period */
+	REG16	q_decctl;		/*	0x28	decoder control */
+	REG16	q_epctl;		/*	0x2A	control register */
+	REG16	q_capctl;		/*	0x2C	capture control */
+	REG16	q_posctl;		/*	0x2E	position compare control */
+	REG16	q_eint;			/*	0x30	interrupt enable */
+	REG16	q_flg;			/*	0x32	interrupt flag */
+	REG16	q_clr;			/*	0x34	interrupt clear */
+	REG16	q_frc;			/*	0x36	interrupt force */
+	REG16	q_epsts;		/*	0x38	status */
+	REG16	q_ctmr;			/*	0x3A	capture timer */
+	REG16	q_cprd;			/*	0x3C	capture period */
+	REG16	q_ctmrlat;		/*	0x3E	capture timer latch */
+	REG16	q_prdlat;		/*	0x40	capture period latch */
+	char	q_fill1[0x5c-0x40];
+	REG32	q_revid;		/*	0x5C	revision id */
+};
+#endif
+
+
+/* Bits for the QDECTL register */
+#define QSRC1		(1 << 15)
+#define QSRC0		(1 << 14)
+#define SOEN		(1 << 13)
+#define SPSEL		(1 << 12)
+#define XCR		(1 << 11)
+#define SWAP		(1 << 10)
+#define IGATE		(1 << 9)
+#define QAP		(1 << 8)
+#define QBP		(1 << 7)
+#define QIP		(1 << 6)
+#define QSP		(1 << 5)
+
+/* Bits for the QEPCTL register */
+#define FREESOFT1	(1 << 15)
+#define FREESOFT0	(1 << 14)
+#define PCRM1		(1 << 13)
+#define PCRM0		(1 << 12)
+#define SEI1		(1 << 11)
+#define SEI0		(1 << 10)
+#define IEI1		(1 << 9)
+#define IEI0		(1 << 8)
+#define SWI		(1 << 7)
+#define SEL		(1 << 6)
+#define IEL1		(1 << 5)
+#define IEL0		(1 << 4)
+#define PHEN		(1 << 3)
+#define QCLM		(1 << 2)
+#define UTE		(1 << 1)
+#define WDE		(1 << 0)
+
+/* Bits for the QCAPCTL register */
+#define CEN		(1 << 15)
+#define CCPS2		(1 << 6)
+#define CCPS0		(1 << 5)
+#define CCPS1		(1 << 4)
+#define UPPS3		(1 << 3)
+#define UPPS2		(1 << 2)
+#define UPPS1		(1 << 1)
+#define UPPS0		(1 << 0)
+
+/* Bits for the QPOSCTL register */
+#define PCSHDW		(1 << 15)
+#define PCLOAD		(1 << 14)
+#define PCPOL		(1 << 13)
+#define PCE		(1 << 12)
+#define PCSPW11		(1 << 11)
+#define PCSPW10		(1 << 10)
+#define PCSPW9		(1 << 9)
+#define PCSPW8		(1 << 8)
+#define PCSPW7		(1 << 7)
+#define PCSPW6		(1 << 6)
+#define PCSPW5		(1 << 5)
+#define PCSPW4		(1 << 4)
+#define PCSPW3		(1 << 3)
+#define PCSPW2		(1 << 2)
+#define PCSPW1		(1 << 1)
+#define PCSPW0		(1 << 0)
+
+/* Bits for the interrupt registers */
+#define EQEP_INTERRUPT_MASK	0x0FFF
+#define UTOF			(1 << 11)
+
+/* Bits to control the clock in the PWMSS subsystem */
+#define PWMSS_EQEPCLK_EN	BIT(4)
+#define PWMSS_EQEPCLK_STOP_REQ	BIT(5)
+#define PWMSS_EQEPCLK_EN_ACK	BIT(4)
+
+/*
+ * Modes for the eQEP unit
+ *	Absolute - the position entry represents the current position of the encoder.
+ *		   Poll this value and it will be notified every period nanoseconds
+ *	Relative - the position entry represents the last latched position of the encoder
+ *		   This value is latched every period nanoseconds and the internal counter
+ *		   is subsequenty reset
+ */
+#define TIEQEP_MODE_ABSOLUTE	0
+#define TIEQEP_MODE_RELATIVE	1
+
+/* Structure defining the characteristics of the eQEP unit */
+struct eqep_chip
+{
+	/* Platform device for this eQEP unit */
+	struct platform_device *pdev;
+
+	/* Pointer to the base of the memory of the eQEP unit */
+	void __iomem	*mmio_base;
+
+	/* SYSCLKOUT to the eQEP unit */
+	u32		clk_rate;
+
+	/* IRQ for the eQEP unit */
+	u16		irq;
+
+	/* Mode of the eQEP unit */
+	u8		op_mode;
+
+	/* work stuct for the notify userspace work */
+	struct work_struct notify_work;
+
+	/* Backup for driver suspension */
+	u16		prior_qepctl;
+	u16		prior_qeint;
+};
+
+/* Notify userspace work */
+static void notify_handler(struct work_struct *work)
+{
+	/* Get a reference to the eQEP driver */
+	struct eqep_chip *eqep = container_of(work, struct eqep_chip, notify_work);
+
+	/* Notify the userspace */
+	sysfs_notify(&eqep->pdev->dev.kobj, NULL, "position");
+}
+
+/* eQEP Interrupt handler */
+static irqreturn_t eqep_irq_handler(int irq, void *dev_id)
+{
+	/* Get the instance information */
+	struct platform_device	*pdev = dev_id;
+	struct eqep_chip	*eqep = platform_get_drvdata(pdev);
+
+	/* Get the interrupt flags */
+	u16 iflags = readw(eqep->mmio_base + QFLG) & EQEP_INTERRUPT_MASK;
+
+	/* Check the interrupt source(s) */
+	if (iflags & UTOF) {
+		/* Handle the unit timer overflow interrupt by notifying any potential pollers */
+		schedule_work(&eqep->notify_work);
+	}
+
+	/* Clear interrupt flags (write back triggered flags to the clear register) */
+	writew(iflags, eqep->mmio_base + QCLR);
+
+	/* Return that the IRQ was handled successfully */
+	return IRQ_HANDLED;
+}
+
+/* Function to read whether the eQEP unit is enabled or disabled */
+static ssize_t eqep_get_enabled(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	/* Get the instance structure */
+	struct eqep_chip *eqep = dev_get_drvdata(dev);
+	u16 enabled = 0;
+
+	/* Increment the device usage count and run pm_runtime_resume() */
+	pm_runtime_get_sync(dev);
+
+	/* Read the qep control register and mask all but the enabled bit */
+	enabled = readw(eqep->mmio_base + QEPCTL) & PHEN;
+
+	/* Return the target in string format */
+	return sprintf(buf, "%u\n", (enabled) ? 1 : 0);
+}
+
+/* Function to set if the eQEP is enabled */
+static ssize_t eqep_set_enabled(struct device *dev, struct device_attribute *attr, const char *buf, size_t count)
+{
+	/* Get the instance structure */
+	int	rc;
+	u16	val;
+	u8	enabled;
+	struct eqep_chip *eqep = dev_get_drvdata(dev);
+
+	/* Convert the input string to an 8 bit uint */
+	if ((rc = kstrtou8(buf, 0, &enabled)))
+		return rc;
+
+	/* Increment the device usage count and run pm_runtime_resume() */
+	pm_runtime_get_sync(dev);
+	/* Get the existing state of QEPCTL */
+	val = readw(eqep->mmio_base + QEPCTL);
+
+	/* If we passed a number that is not 0, enable the eQEP */
+	if (enabled)
+		/* Enable the eQEP (Set PHEN in QEPCTL) */
+		val |= PHEN;
+	else
+		/* Disable the eQEP (Clear PHEN in QEPCTL) */
+		val &= ~PHEN;
+
+	/* Write flags back to control register */
+	writew(val, eqep->mmio_base + QEPCTL);
+
+	/* Return buffer length consumed (all) */
+	return count;
+}
+
+/* Function to read the current position of the eQEP */
+static ssize_t eqep_get_position(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct eqep_chip *eqep = dev_get_drvdata(dev);
+
+	s32 position = 0;
+	/* Increment the device usage count and run pm_runtime_resume() */
+	pm_runtime_get_sync(dev);
+
+	if (eqep->op_mode == TIEQEP_MODE_ABSOLUTE) {
+		position = readl(eqep->mmio_base + QPOSCNT);
+	} else if (eqep->op_mode == TIEQEP_MODE_RELATIVE) {
+		/* in relative mode, use the last latched value of the eQEP hardware */
+		position = readl(eqep->mmio_base + QPOSLAT);
+		dev_dbg(dev, "get_position:0x%08x\n", position);
+	}
+
+	return sprintf(buf, "%d\n", position);
+}
+
+/* Function to set the position of the eQEP hardware */
+static ssize_t eqep_set_position(struct device *dev, struct device_attribute *attr, const char *buf, size_t count)
+{
+	int rc;
+	s32 position;
+	struct eqep_chip *eqep = dev_get_drvdata(dev);
+
+	if ((rc = kstrtos32(buf, 0, &position)))
+		return rc;
+
+	/* Increment the device usage count and run pm_runtime_resume() */
+	pm_runtime_get_sync(dev);
+	/*
+	 * If we are in absolute mode, set the position of the encoder,
+	 * discard relative mode because thats pointless
+	 */
+	if (eqep->op_mode == TIEQEP_MODE_ABSOLUTE) {
+		/* If absolute mode, set the current value of the eQEP hardware */
+		writel(position, eqep->mmio_base + QPOSCNT);
+	}
+
+	/* Return buffer length consumed (all) */
+	return count;
+}
+
+/* Function to read the period of the unit time event timer */
+static ssize_t eqep_get_timer_period(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct eqep_chip *eqep = dev_get_drvdata(dev);
+	u64 period;
+
+	/* Increment the device usage count and run pm_runtime_resume() */
+	pm_runtime_get_sync(dev);
+	/* Convert from counts per interrupt back into period_ns */
+	period = readl(eqep->mmio_base + QUPRD);
+	period = period * NSEC_PER_SEC;
+	do_div(period, eqep->clk_rate);
+
+	/* Otherwise write out the data */
+	return sprintf(buf, "%llu\n", period);
+}
+
+/* Function to set the unit timer period.  0 = off, greater than zero sets the period */
+static ssize_t eqep_set_timer_period(struct device *dev, struct device_attribute *attr, const char *buf, size_t count)
+{
+	int	rc;
+	u16	tmp;
+	u64	period;
+
+	struct eqep_chip *eqep = dev_get_drvdata(dev);
+
+	if ((rc = kstrtou64(buf, 0, &period)))
+		return rc;
+
+	/* Increment the device usage count and run pm_runtime_resume() */
+	pm_runtime_get_sync(dev);
+	/* Disable the unit timer before modifying its period register */
+	tmp = readw(eqep->mmio_base + QEPCTL);
+	tmp &= ~(UTE | QCLM);
+	writew(tmp, eqep->mmio_base + QEPCTL);
+
+	/* Zero the unit timer counter register */
+	writel(0, eqep->mmio_base + QUTMR);
+
+	/* If the timer is enabled (a non-zero period has been passed) */
+	if (period) {
+		/* update the period */
+		period = period * eqep->clk_rate;
+		do_div(period, NSEC_PER_SEC);
+
+		dev_dbg(dev, "eqep_set_timer_period:%llu\n", period);
+
+		writel(period, eqep->mmio_base + QUPRD);
+
+		/* Enable unit timer, and latch QPOSLAT to QPOSCNT on timer expiration */
+		tmp |= UTE | QCLM;
+		writew(tmp, eqep->mmio_base + QEPCTL);
+	}
+
+	return count;
+}
+
+/* Function to read the mode of the eQEP hardware */
+static ssize_t eqep_get_mode(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct eqep_chip *eqep = dev_get_drvdata(dev);
+
+	return sprintf(buf, "%u\n", eqep->op_mode);
+}
+
+/* Function to set the mode of the eQEP hardware */
+static ssize_t eqep_set_mode(struct device *dev, struct device_attribute *attr, const char *buf, size_t count)
+{
+	int 	rc;
+	u16 	val;
+	u8	tmp_mode;
+	struct eqep_chip *eqep = dev_get_drvdata(dev);
+
+	if ((rc = kstrtou8(buf, 0, &tmp_mode)))
+		return rc;
+
+	dev_dbg(dev, "eqep_set_mode:%d\n", tmp_mode);
+
+	/* Increment the device usage count and run pm_runtime_resume() */
+	pm_runtime_get_sync(dev);
+	val = readw(eqep->mmio_base + QEPCTL);
+
+	if (tmp_mode == TIEQEP_MODE_ABSOLUTE) {
+		/*
+		 * In absolute mode, don't reset the hardware based on time,
+		 * so disable the unit timer position reset (Set PCRM[1:0] = 0)
+		 */
+		val &= ~(PCRM1 | PCRM0);
+
+		eqep->op_mode = TIEQEP_MODE_ABSOLUTE;
+	} else if (tmp_mode == TIEQEP_MODE_RELATIVE) {
+		/*
+		 * In relative mode, latch the value of the eQEP hardware on the
+		 * overflow of the unit timer.	So enable the unit timer position reset
+		 * (Set PCRM[1:0] = 3)
+		 */
+		val |= PCRM1 | PCRM0;
+
+		eqep->op_mode = TIEQEP_MODE_RELATIVE;
+	}
+
+	writew(val, eqep->mmio_base + QEPCTL);
+
+	return count;
+}
+
+/* Bind read/write functions to sysfs entries */
+static DEVICE_ATTR(enabled,	0644, eqep_get_enabled,		eqep_set_enabled);
+static DEVICE_ATTR(position,	0644, eqep_get_position,	eqep_set_position);
+static DEVICE_ATTR(period,	0644, eqep_get_timer_period,	eqep_set_timer_period);
+static DEVICE_ATTR(mode,	0644, eqep_get_mode,		eqep_set_mode);
+
+/* Array holding all of the sysfs entries */
+static const struct attribute *eqep_attrs[] = {
+	&dev_attr_enabled.attr,
+	&dev_attr_position.attr,
+	&dev_attr_period.attr,
+	&dev_attr_mode.attr,
+	NULL,
+};
+
+/* Driver function group */
+static const struct attribute_group eqep_device_attr_group = {
+	.attrs = (struct attribute **) eqep_attrs,
+};
+
+/* Driver compatibility list */
+static struct of_device_id eqep_of_match[] =
+{
+	{ .compatible = "ti,am33xx-eqep" },
+	{ }
+};
+
+/* Register our compatibilities for device trees */
+MODULE_DEVICE_TABLE(of, eqep_of_match);
+
+/* Create an instance of the eQEP driver */
+static int eqep_probe(struct platform_device *pdev)
+{
+	struct resource	 *r;
+	struct clk	 *clk;
+	struct eqep_chip *eqep;
+	struct pinctrl	 *pinctrl;
+
+	u64	period;
+	u16	status;
+	u32	value;
+
+	dev_info(&pdev->dev, "ver. 1.0\n");
+
+	/* Select pins provided through the device tree */
+	pinctrl = devm_pinctrl_get_select_default(&pdev->dev);
+	if (IS_ERR(pinctrl))
+	{
+		dev_warn(&pdev->dev, "unable to select pin group\n");
+	}
+
+	/* Allocate a eqep_driver object */
+	eqep = devm_kzalloc(&pdev->dev, sizeof(struct eqep_chip), GFP_KERNEL);
+	if (!eqep) {
+		dev_err(&pdev->dev, "failed to allocate memory\n");
+		return -ENOMEM;
+	}
+
+	/* Get a handle to the system clock object */
+	clk = devm_clk_get(pdev->dev.parent, "fck");
+	if (IS_ERR(clk)) {
+		dev_err(&pdev->dev, "failed to get clock\n");
+		return PTR_ERR(clk);
+	}
+
+	/* Get the frequency of the system clock */
+	eqep->clk_rate = clk_get_rate(clk);
+	if (!eqep->clk_rate) {
+		dev_err(&pdev->dev, "failed to get clock rate\n");
+		return -EINVAL;
+	}
+
+	/* Get a resource containing the IRQ for this eQEP controller */
+	r = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
+	if (unlikely(!r)) {
+		dev_err(&pdev->dev, "Invalid IRQ resource\n");
+		return -ENODEV;
+	}
+
+	/* Store the irq */
+	eqep->irq = r->start;
+
+	/* Get a resource containing the requested (from DT) memory address and range of eQEP controller */
+	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!r) {
+		dev_err(&pdev->dev, "no memory resource defined\n");
+		return -ENODEV;
+	}
+
+	/* Remap the eQEP controller memory into our own memory space */
+	eqep->mmio_base = devm_ioremap_resource(&pdev->dev, r);
+	if (IS_ERR(eqep->mmio_base))
+		return PTR_ERR(eqep->mmio_base);
+
+	/* Store the platform device in our eQEP data structure for later usage */
+	eqep->pdev = pdev;
+
+	/* Subscribe to the eQEP interrupt */
+	if (request_irq(eqep->irq, eqep_irq_handler, IRQF_IRQPOLL, "eqep_interrupt", pdev))
+	{
+		dev_err(&pdev->dev, "unable to request irq for eQEP\n");
+		return -ENODEV;
+	}
+
+	/* Register controls to sysfs */
+	if (sysfs_create_group(&pdev->dev.kobj, &eqep_device_attr_group))
+	{
+		dev_err(&pdev->dev, "sysfs creation failed\n");
+		return -EINVAL;
+	}
+
+	/* set QDECCTL */
+	status = 0;	/* default to Quadrature count mode, QSRC1 & QSRC0 = 0 */
+
+	/* set QSRC1 & QSRC0 bits, one of 4 count_modes. */
+	if (!of_property_read_u32(pdev->dev.of_node, "count_mode", &value) && value <= 3) {
+		status |= value << 14;
+
+		/*
+		 * in count up or count down mode, count on rising edge only
+		 * not on both edges.
+		 */
+		if (value >= 2)
+			status |= XCR;
+	}
+	dev_info(&pdev->dev, "count_mode:%d\n", value);
+
+	/* Should we invert the qa input */
+	if (!of_property_read_u32(pdev->dev.of_node, "invert_qa", &value))
+		status = value ? status | QAP : status & ~QAP;
+	dev_info(&pdev->dev, "invert_qa:%d\n", value);
+
+	/* Should we invert the qb input */
+	if (!of_property_read_u32(pdev->dev.of_node, "invert_qb", &value))
+		status = value ? status | QBP : status & ~QBP;
+	dev_info(&pdev->dev, "invert_qb:%d\n", value);
+
+	/* Should we invert the index input */
+	if (!of_property_read_u32(pdev->dev.of_node, "invert_qi", &value))
+		status = value ? status | QIP : status & ~QIP;
+	dev_info(&pdev->dev, "invert_qi:%d\n", value);
+
+	/* Should we invert the strobe input */
+	if (!of_property_read_u32(pdev->dev.of_node, "invert_qs", &value))
+		status = value ? status | QSP : status & ~QSP;
+	dev_info(&pdev->dev, "invert_qs:%d\n", value);
+
+	/* Should we swap the cha and chb inputs */
+	if (!of_property_read_u32(pdev->dev.of_node, "swap_inputs", &value))
+		status = value ? status | SWAP : status & ~SWAP;
+	dev_info(&pdev->dev, "swap_inputs:%d\n", value);
+
+	dev_info(&pdev->dev, "QDECCTL:0x%04x\n", status);
+
+	/* Write the decoder control settings back to the control register */
+	writew(status, eqep->mmio_base + QDECCTL);
+
+	writel( 0, eqep->mmio_base + QPOSINIT);
+	writel(~0, eqep->mmio_base + QPOSMAX);
+	writel( 0, eqep->mmio_base + QPOSCNT);
+
+	dev_info(&pdev->dev, "QPOSINIT:0x%08x\n", readl(eqep->mmio_base + QPOSINIT));
+	dev_info(&pdev->dev, "QPOSMAX:0x%08x\n", readl(eqep->mmio_base + QPOSMAX));
+	dev_info(&pdev->dev, "QPOSCNT:0x%08x\n", readl(eqep->mmio_base + QPOSCNT));
+
+	status = UTOF;		/* Enable Unit Time Period interrupt. */
+	if (!of_property_read_u32(pdev->dev.of_node, "omit_interrupt", &value) && value) {
+		status = 0;	/* no interrupt */
+	}
+	writew(status, eqep->mmio_base + QEINT);
+	dev_info(&pdev->dev, "omit_interrupt:%d\n", value);
+	dev_info(&pdev->dev, "QEINT:0x%04x\n", status);
+
+	/* Calculate the timer ticks per second */
+	period = 1000000000;
+	period = period * eqep->clk_rate;
+	do_div(period, NSEC_PER_SEC);
+
+	/* Set this period into the unit timer period register */
+	writel(period, eqep->mmio_base + QUPRD);
+	dev_info(&pdev->dev, "QUPRD:0x%08x\n", (u32) period);
+
+	/*
+	 * Enable the eQEP with basic position counting turned on
+	 * PHEN - Quadrature position counter enable bit
+	 * UTE	- unit timer enable
+	 * QCLM - latch QPOSLAT to QPOSCNT upon unit timer overflow
+	 * IEL0 - Latch QPOSILAT on index signal.  Rising or falling, IEL[1:0] = 0 is reserved
+	 * SWI	- Software initialization of position count register, i.e. set QPOSCNT <= QPOSINIT,
+	 *  but this bit was not being reset by hardware as advertised in TRM,
+	 *  (so omit & clear QPOSCNT manually elsewhere?)
+	 */
+	status = PHEN | UTE | QCLM | IEL0 | SWI;
+	writew(status, eqep->mmio_base + QEPCTL);
+	dev_info(&pdev->dev, "QEPCTL:0x%04x write\n", status);
+	dev_info(&pdev->dev, "QEPCTL:0x%04x read\n", readw(eqep->mmio_base + QEPCTL));
+
+	/* We default to absolute mode */
+	eqep->op_mode = TIEQEP_MODE_ABSOLUTE;
+
+	/* Enable the power management runtime */
+	pm_runtime_enable(&pdev->dev);
+
+	/* Increment the device usage count and run pm_runtime_resume() */
+	pm_runtime_get_sync(&pdev->dev);
+
+	/* Initialize the notify work struture */
+	INIT_WORK(&eqep->notify_work, notify_handler);
+
+	/* Decrement the device usage count (twice) and run pm_runtime_idle() if zero */
+	pm_runtime_put_sync(&pdev->dev);
+
+	/* Set the platform driver data to the data object we've been creating for the eQEP unit */
+	platform_set_drvdata(pdev, eqep);
+
+	/* Success! */
+	dev_info(&pdev->dev, "irq:%d, clk_rate:%u\n", eqep->irq, eqep->clk_rate);
+	return 0;
+}
+
+/* Remove an instance of the eQEP driver */
+static int eqep_remove(struct platform_device *pdev)
+{
+	/* Get the eQEP driver data from the platform device structure */
+	struct eqep_chip *eqep = platform_get_drvdata(pdev);
+
+	/* Cancel work */
+	cancel_work_sync(&eqep->notify_work);
+
+	/* Unmap from sysfs */
+	sysfs_remove_group(&pdev->dev.kobj, &eqep_device_attr_group);
+
+	/* Release important assets */
+	free_irq(eqep->irq, pdev);
+
+	/* Increment the device usage count and run pm_runtime_resume() */
+	pm_runtime_get_sync(&pdev->dev);
+
+	/* Decrement the device usage count (twice) and run pm_runtime_idle() if zero */
+	pm_runtime_put_sync(&pdev->dev);
+	pm_runtime_put_sync(&pdev->dev);
+
+	/* Disable the runtime power management of this device */
+	pm_runtime_disable(&pdev->dev);
+
+	/* Return success */
+	return 0;
+}
+
+/* Power management suspend device */
+static int eqep_suspend(struct device *dev)
+{
+	/* Get the eqep driver information */
+	struct eqep_chip   *eqep = dev_get_drvdata(dev);
+	u16					tmp;
+
+	/* Shut down interrupts */
+	eqep->prior_qeint = readw(eqep->mmio_base + QEINT);
+	tmp = eqep->prior_qeint & ~UTOF;
+	writew(tmp, eqep->mmio_base + QEINT);
+
+	/* Get the existing state of QEPCTL */
+	eqep->prior_qepctl = readw(eqep->mmio_base + QEPCTL);
+
+	/* Disable eQEP controller */
+	writew(eqep->prior_qepctl & ~PHEN, eqep->mmio_base + QEPCTL);
+
+	/* Decrement the device usage count and run pm_runtime_idle() if zero */
+	pm_runtime_put_sync(dev);
+
+	/* Return success */
+	return 0;
+}
+
+/* Power management wake device back up */
+static int eqep_resume(struct device *dev)
+{
+	/* Get the eqep driver information */
+	struct eqep_chip *eqep = dev_get_drvdata(dev);
+
+	/* Restore interrupt enabled register */
+	writew(eqep->prior_qeint, eqep->mmio_base + QEINT);
+
+	/* Restore prior qep control register */
+	writew(eqep->prior_qepctl, eqep->mmio_base + QEPCTL);
+
+	/* Increment the device usage count and run pm_runtime_resume() */
+	pm_runtime_get_sync(dev);
+
+	/* Success */
+	return 0;
+}
+
+/* create pm functions object */
+static SIMPLE_DEV_PM_OPS(eqep_pm_ops, eqep_suspend, eqep_resume);
+
+/* Platform driver information */
+static struct platform_driver eqep_driver = {
+	.driver = {
+		.name	= "eqep",
+		.owner	= THIS_MODULE,
+		.pm	= &eqep_pm_ops,
+		.of_match_table = eqep_of_match,
+	},
+	.probe = eqep_probe,
+	.remove = eqep_remove,
+};
+
+/* Register this platform driver */
+module_platform_driver(eqep_driver);
+
+/* Module information */
+MODULE_DESCRIPTION("TI eQEP driver");
+MODULE_AUTHOR("Nathaniel R. Lewis");
+MODULE_LICENSE("GPL");
diff -Naur --no-dereference a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig
--- a/drivers/mmc/host/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/mmc/host/Kconfig	2020-06-19 21:00:31.416227715 -0400
@@ -935,11 +935,27 @@
 config MMC_SDHCI_OMAP
 	tristate "TI SDHCI Controller Support"
 	depends on MMC_SDHCI_PLTFM && OF
+	select THERMAL
+	imply TI_SOC_THERMAL
 	help
 	  This selects the Secure Digital Host Controller Interface (SDHCI)
 	  support present in TI's DRA7 SOCs. The controller supports
 	  SD/MMC/SDIO devices.
 
 	  If you have a controller with this interface, say Y or M here.
+
+	  If unsure, say N.
+
+config MMC_SDHCI_AM654
+	tristate "Support for the SDHCI Controller in TI's AM654 SOCs"
+	depends on MMC_SDHCI_PLTFM && OF
+	select MMC_SDHCI_IO_ACCESSORS
+	select MMC_CQHCI
+	help
+	  This selects the Secure Digital Host Controller Interface (SDHCI)
+	  support present in TI's AM654 SOCs. The controller supports
+	  SD/MMC/SDIO devices.
+
+	  If you have a controller with this interface, say Y or M here.
 
 	  If unsure, say N.
diff -Naur --no-dereference a/drivers/mmc/host/Makefile b/drivers/mmc/host/Makefile
--- a/drivers/mmc/host/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/mmc/host/Makefile	2020-06-19 21:00:31.416227715 -0400
@@ -21,6 +21,7 @@
 obj-$(CONFIG_MMC_SDHCI_SIRF)   	+= sdhci-sirf.o
 obj-$(CONFIG_MMC_SDHCI_F_SDH30)	+= sdhci_f_sdh30.o
 obj-$(CONFIG_MMC_SDHCI_SPEAR)	+= sdhci-spear.o
+obj-$(CONFIG_MMC_SDHCI_AM654)	+= sdhci_am654.o
 obj-$(CONFIG_MMC_WBSD)		+= wbsd.o
 obj-$(CONFIG_MMC_AU1X)		+= au1xmmc.o
 obj-$(CONFIG_MMC_MTK)		+= mtk-sd.o
diff -Naur --no-dereference a/drivers/mmc/host/sdhci_am654.c b/drivers/mmc/host/sdhci_am654.c
--- a/drivers/mmc/host/sdhci_am654.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/mmc/host/sdhci_am654.c	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,705 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * sdhci_am654.c - SDHCI driver for TI's AM654 SOCs
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
+ *
+ */
+#include <linux/clk.h>
+#include <linux/dma-mapping.h>
+#include <linux/of.h>
+#include <linux/module.h>
+#include <linux/pm_runtime.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+
+#include "cqhci.h"
+#include "sdhci-pltfm.h"
+
+/* CTL_CFG Registers */
+#define CTL_CFG_2		0x14
+
+#define SLOTTYPE_MASK		GENMASK(31, 30)
+#define SLOTTYPE_EMBEDDED	BIT(30)
+
+/* PHY Registers */
+#define PHY_CTRL1	0x100
+#define PHY_CTRL2	0x104
+#define PHY_CTRL3	0x108
+#define PHY_CTRL4	0x10C
+#define PHY_CTRL5	0x110
+#define PHY_CTRL6	0x114
+#define PHY_STAT1	0x130
+#define PHY_STAT2	0x134
+
+#define IOMUX_ENABLE_SHIFT	31
+#define IOMUX_ENABLE_MASK	BIT(IOMUX_ENABLE_SHIFT)
+#define OTAPDLYENA_SHIFT	20
+#define OTAPDLYENA_MASK		BIT(OTAPDLYENA_SHIFT)
+#define OTAPDLYSEL_SHIFT	12
+#define OTAPDLYSEL_MASK		GENMASK(15, 12)
+#define STRBSEL_SHIFT		24
+#define STRBSEL_4BIT_MASK	GENMASK(27, 24)
+#define STRBSEL_8BIT_MASK	GENMASK(31, 24)
+#define SEL50_SHIFT		8
+#define SEL50_MASK		BIT(SEL50_SHIFT)
+#define SEL100_SHIFT		9
+#define SEL100_MASK		BIT(SEL100_SHIFT)
+#define FREQSEL_SHIFT		8
+#define FREQSEL_MASK		GENMASK(10, 8)
+#define CLKBUFSEL_SHIFT		0
+#define CLKBUFSEL_MASK		GENMASK(2, 0)
+#define DLL_TRIM_ICP_SHIFT	4
+#define DLL_TRIM_ICP_MASK	GENMASK(7, 4)
+#define DR_TY_SHIFT		20
+#define DR_TY_MASK		GENMASK(22, 20)
+#define ENDLL_SHIFT		1
+#define ENDLL_MASK		BIT(ENDLL_SHIFT)
+#define DLLRDY_SHIFT		0
+#define DLLRDY_MASK		BIT(DLLRDY_SHIFT)
+#define PDB_SHIFT		0
+#define PDB_MASK		BIT(PDB_SHIFT)
+#define CALDONE_SHIFT		1
+#define CALDONE_MASK		BIT(CALDONE_SHIFT)
+#define RETRIM_SHIFT		17
+#define RETRIM_MASK		BIT(RETRIM_SHIFT)
+
+#define DRIVER_STRENGTH_50_OHM	0x0
+#define DRIVER_STRENGTH_33_OHM	0x1
+#define DRIVER_STRENGTH_66_OHM	0x2
+#define DRIVER_STRENGTH_100_OHM	0x3
+#define DRIVER_STRENGTH_40_OHM	0x4
+
+#define CLOCK_TOO_SLOW_HZ	400000
+
+/* Command Queue Host Controller Interface Base address */
+#define SDHCI_AM654_CQE_BASE_ADDR 0x200
+
+static struct regmap_config sdhci_am654_regmap_config = {
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = 4,
+	.fast_io = true,
+};
+
+struct sdhci_am654_data {
+	struct regmap *base;
+	bool legacy_otapdly;
+	int otap_del_sel[11];
+	int clkbuf_sel;
+	int trm_icp;
+	int drv_strength;
+	bool dll_on;
+	int strb_sel;
+	u32 flags;
+};
+
+struct sdhci_am654_driver_data {
+	const struct sdhci_pltfm_data *pdata;
+	u32 flags;
+#define IOMUX_PRESENT	(1 << 0)
+#define FREQSEL_2_BIT	(1 << 1)
+#define STRBSEL_4_BIT	(1 << 2)
+#define DLL_PRESENT	(1 << 3)
+};
+
+struct timing_data {
+	const char *binding;
+	u32 capability;
+};
+
+static const struct timing_data td[] = {
+	[MMC_TIMING_LEGACY] = {"ti,otap-del-sel-legacy", 0},
+	[MMC_TIMING_MMC_HS] = {"ti,otap-del-sel-mmc-hs", MMC_CAP_MMC_HIGHSPEED},
+	[MMC_TIMING_SD_HS]  = {"ti,otap-del-sel-sd-hs", MMC_CAP_SD_HIGHSPEED},
+	[MMC_TIMING_UHS_SDR12] = {"ti,otap-del-sel-sdr12", MMC_CAP_UHS_SDR12},
+	[MMC_TIMING_UHS_SDR25] = {"ti,otap-del-sel-sdr25", MMC_CAP_UHS_SDR25},
+	[MMC_TIMING_UHS_SDR50] = {"ti,otap-del-sel-sdr50", MMC_CAP_UHS_SDR50},
+	[MMC_TIMING_UHS_SDR104] = {"ti,otap-del-sel-sdr104",
+				   MMC_CAP_UHS_SDR104},
+	[MMC_TIMING_UHS_DDR50] = {"ti,otap-del-sel-ddr50", MMC_CAP_UHS_DDR50},
+	[MMC_TIMING_MMC_DDR52] = {"ti,otap-del-sel-ddr52", MMC_CAP_DDR},
+	[MMC_TIMING_MMC_HS200] = {"ti,otap-del-sel-hs200", MMC_CAP2_HS200},
+	[MMC_TIMING_MMC_HS400] = {"ti,otap-del-sel-hs400", MMC_CAP2_HS400},
+};
+
+static void sdhci_am654_setup_dll(struct sdhci_host *host, unsigned int clock)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_am654_data *sdhci_am654 = sdhci_pltfm_priv(pltfm_host);
+	int sel50, sel100, freqsel;
+	u32 mask, val;
+	int ret;
+
+	if (sdhci_am654->flags & FREQSEL_2_BIT) {
+		switch (clock) {
+		case 200000000:
+			sel50 = 0;
+			sel100 = 0;
+			break;
+		case 100000000:
+			sel50 = 0;
+			sel100 = 1;
+			break;
+		default:
+			sel50 = 1;
+			sel100 = 0;
+		}
+
+		/* Configure PHY DLL frequency */
+		mask = SEL50_MASK | SEL100_MASK;
+		val = (sel50 << SEL50_SHIFT) | (sel100 << SEL100_SHIFT);
+		regmap_update_bits(sdhci_am654->base, PHY_CTRL5, mask, val);
+
+	} else {
+		switch (clock) {
+		case 200000000:
+			freqsel = 0x0;
+			break;
+		default:
+			freqsel = 0x4;
+		}
+
+		regmap_update_bits(sdhci_am654->base, PHY_CTRL5, FREQSEL_MASK,
+				   freqsel << FREQSEL_SHIFT);
+	}
+
+	/* Enable DLL */
+	regmap_update_bits(sdhci_am654->base, PHY_CTRL1, ENDLL_MASK,
+			   0x1 << ENDLL_SHIFT);
+	/*
+	 * Poll for DLL ready. Use a one second timeout.
+	 * Works in all experiments done so far
+	 */
+	ret = regmap_read_poll_timeout(sdhci_am654->base, PHY_STAT1, val,
+				       val & DLLRDY_MASK, 1000, 1000000);
+	if (ret) {
+		dev_err(mmc_dev(host->mmc), "DLL failed to relock\n");
+		return;
+	}
+
+	sdhci_am654->dll_on = true;
+}
+
+static void sdhci_am654_set_clock(struct sdhci_host *host, unsigned int clock)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_am654_data *sdhci_am654 = sdhci_pltfm_priv(pltfm_host);
+	unsigned char timing = host->mmc->ios.timing;
+	u32 otap_del_sel;
+	u32 otap_del_ena;
+	u32 mask, val;
+
+	if (sdhci_am654->dll_on) {
+		regmap_update_bits(sdhci_am654->base, PHY_CTRL1, ENDLL_MASK, 0);
+
+		sdhci_am654->dll_on = false;
+	}
+
+	sdhci_set_clock(host, clock);
+
+	if (clock > CLOCK_TOO_SLOW_HZ) {
+		/* Setup DLL Output TAP delay */
+		if (sdhci_am654->legacy_otapdly)
+			otap_del_sel = sdhci_am654->otap_del_sel[0];
+		else
+			otap_del_sel = sdhci_am654->otap_del_sel[timing];
+
+		otap_del_ena = (timing > MMC_TIMING_UHS_SDR25) ? 1 : 0;
+
+		mask = OTAPDLYENA_MASK | OTAPDLYSEL_MASK;
+		val = (otap_del_ena << OTAPDLYENA_SHIFT) |
+		      (otap_del_sel << OTAPDLYSEL_SHIFT);
+
+		/* Write to STRBSEL for HS400 speed mode */
+		if (timing == MMC_TIMING_MMC_HS400) {
+			if (sdhci_am654->flags & STRBSEL_4_BIT)
+				mask |= STRBSEL_4BIT_MASK;
+			else
+				mask |= STRBSEL_8BIT_MASK;
+
+			val |= sdhci_am654->strb_sel << STRBSEL_SHIFT;
+		}
+
+		regmap_update_bits(sdhci_am654->base, PHY_CTRL4, mask, val);
+
+		if (timing > MMC_TIMING_UHS_SDR25)
+			sdhci_am654_setup_dll(host, clock);
+	}
+}
+
+void sdhci_j721e_4bit_set_clock(struct sdhci_host *host, unsigned int clock)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_am654_data *sdhci_am654 = sdhci_pltfm_priv(pltfm_host);
+	unsigned char timing = host->mmc->ios.timing;
+	u32 otap_del_sel;
+	u32 mask, val;
+
+	/* Setup DLL Output TAP delay */
+	if (sdhci_am654->legacy_otapdly)
+		otap_del_sel = sdhci_am654->otap_del_sel[0];
+	else
+		otap_del_sel = sdhci_am654->otap_del_sel[timing];
+
+	mask = OTAPDLYENA_MASK | OTAPDLYSEL_MASK;
+	val = (0x1 << OTAPDLYENA_SHIFT) |
+	      (otap_del_sel << OTAPDLYSEL_SHIFT);
+	regmap_update_bits(sdhci_am654->base, PHY_CTRL4, mask, val);
+
+	regmap_update_bits(sdhci_am654->base, PHY_CTRL5, CLKBUFSEL_MASK,
+			   sdhci_am654->clkbuf_sel);
+
+	sdhci_set_clock(host, clock);
+}
+
+static void sdhci_am654_set_power(struct sdhci_host *host, unsigned char mode,
+				  unsigned short vdd)
+{
+	if (!IS_ERR(host->mmc->supply.vmmc)) {
+		struct mmc_host *mmc = host->mmc;
+
+		mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, vdd);
+	}
+	sdhci_set_power_noreg(host, mode, vdd);
+}
+
+static void sdhci_am654_write_b(struct sdhci_host *host, u8 val, int reg)
+{
+	unsigned char timing = host->mmc->ios.timing;
+
+	if (reg == SDHCI_HOST_CONTROL) {
+		switch (timing) {
+		/*
+		 * According to the data manual, HISPD bit
+		 * should not be set in these speed modes.
+		 */
+		case MMC_TIMING_SD_HS:
+		case MMC_TIMING_MMC_HS:
+		case MMC_TIMING_UHS_SDR12:
+		case MMC_TIMING_UHS_SDR25:
+			val &= ~SDHCI_CTRL_HISPD;
+		}
+	}
+
+	writeb(val, host->ioaddr + reg);
+}
+
+struct sdhci_ops sdhci_am654_ops = {
+	.get_max_clock = sdhci_pltfm_clk_get_max_clock,
+	.get_timeout_clock = sdhci_pltfm_clk_get_max_clock,
+	.set_uhs_signaling = sdhci_set_uhs_signaling,
+	.set_bus_width = sdhci_set_bus_width,
+	.set_power = sdhci_am654_set_power,
+	.set_clock = sdhci_am654_set_clock,
+	.write_b = sdhci_am654_write_b,
+	.reset = sdhci_reset,
+};
+
+static const struct sdhci_pltfm_data sdhci_am654_pdata = {
+	.ops = &sdhci_am654_ops,
+	.quirks = SDHCI_QUIRK_MULTIBLOCK_READ_ACMD12,
+	.quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
+};
+
+static const struct sdhci_am654_driver_data sdhci_am654_drvdata = {
+	.pdata = &sdhci_am654_pdata,
+	.flags = IOMUX_PRESENT | FREQSEL_2_BIT | STRBSEL_4_BIT | DLL_PRESENT,
+};
+
+static u32 sdhci_am654_cqhci_irq(struct sdhci_host *host, u32 intmask)
+{
+	int cmd_error = 0;
+	int data_error = 0;
+
+	if (!sdhci_cqe_irq(host, intmask, &cmd_error, &data_error))
+		return intmask;
+
+	cqhci_irq(host->mmc, intmask, cmd_error, data_error);
+
+	return 0;
+}
+
+struct sdhci_ops sdhci_j721e_8bit_ops = {
+	.get_max_clock = sdhci_pltfm_clk_get_max_clock,
+	.get_timeout_clock = sdhci_pltfm_clk_get_max_clock,
+	.set_uhs_signaling = sdhci_set_uhs_signaling,
+	.set_bus_width = sdhci_set_bus_width,
+	.set_power = sdhci_am654_set_power,
+	.set_clock = sdhci_am654_set_clock,
+	.write_b = sdhci_am654_write_b,
+	.irq = sdhci_am654_cqhci_irq,
+	.reset = sdhci_reset,
+};
+
+static const struct sdhci_pltfm_data sdhci_j721e_8bit_pdata = {
+	.ops = &sdhci_j721e_8bit_ops,
+	.quirks = SDHCI_QUIRK_MULTIBLOCK_READ_ACMD12,
+	.quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
+};
+
+static const struct sdhci_am654_driver_data sdhci_j721e_8bit_drvdata = {
+	.pdata = &sdhci_j721e_8bit_pdata,
+	.flags = DLL_PRESENT,
+};
+
+struct sdhci_ops sdhci_j721e_4bit_ops = {
+	.get_max_clock = sdhci_pltfm_clk_get_max_clock,
+	.get_timeout_clock = sdhci_pltfm_clk_get_max_clock,
+	.set_uhs_signaling = sdhci_set_uhs_signaling,
+	.set_bus_width = sdhci_set_bus_width,
+	.set_power = sdhci_am654_set_power,
+	.set_clock = sdhci_j721e_4bit_set_clock,
+	.write_b = sdhci_am654_write_b,
+	.irq = sdhci_am654_cqhci_irq,
+	.reset = sdhci_reset,
+};
+
+static const struct sdhci_pltfm_data sdhci_j721e_4bit_pdata = {
+	.ops = &sdhci_j721e_4bit_ops,
+	.quirks = SDHCI_QUIRK_MULTIBLOCK_READ_ACMD12,
+	.quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
+};
+
+static const struct sdhci_am654_driver_data sdhci_j721e_4bit_drvdata = {
+	.pdata = &sdhci_j721e_4bit_pdata,
+	.flags = IOMUX_PRESENT,
+};
+
+static int sdhci_am654_get_otap_delay(struct sdhci_host *host,
+				      struct sdhci_am654_data *sdhci_am654)
+{
+	struct device *dev = mmc_dev(host->mmc);
+	int i;
+	int ret;
+
+	ret = device_property_read_u32(dev, td[MMC_TIMING_LEGACY].binding,
+				 &sdhci_am654->otap_del_sel[MMC_TIMING_LEGACY]);
+	if (ret) {
+		/*
+		 * ti,otap-del-sel-legacy is mandatory, look for old binding
+		 * if not found.
+		 */
+		ret = device_property_read_u32(dev, "ti,otap-del-sel",
+					       &sdhci_am654->otap_del_sel[0]);
+		if (ret) {
+			dev_err(dev, "Couldn't find otap-del-sel\n");
+
+			return ret;
+		}
+
+		dev_info(dev, "Using legacy binding ti,otap-del-sel\n");
+		sdhci_am654->legacy_otapdly = true;
+
+		return 0;
+	}
+
+	for (i = MMC_TIMING_MMC_HS; i <= MMC_TIMING_MMC_HS400; i++) {
+
+		ret = device_property_read_u32(dev, td[i].binding,
+					       &sdhci_am654->otap_del_sel[i]);
+		if (ret) {
+			dev_dbg(dev, "Couldn't find %s\n",
+				td[i].binding);
+			/*
+			 * Remove the corresponding capability
+			 * if an otap-del-sel value is not found
+			 */
+			if (i <= MMC_TIMING_MMC_DDR52)
+				host->mmc->caps &= ~td[i].capability;
+			else
+				host->mmc->caps2 &= ~td[i].capability;
+		}
+	}
+
+	return 0;
+}
+
+static void sdhci_am654_dumpregs(struct mmc_host *mmc)
+{
+	sdhci_dumpregs(mmc_priv(mmc));
+}
+
+static const struct cqhci_host_ops sdhci_am654_cqhci_ops = {
+	.enable		= sdhci_cqe_enable,
+	.disable	= sdhci_cqe_disable,
+	.dumpregs	= sdhci_am654_dumpregs,
+};
+
+static int sdhci_am654_cqe_add_host(struct sdhci_host *host)
+{
+	struct cqhci_host *cq_host;
+	int ret;
+
+	cq_host = devm_kzalloc(host->mmc->parent, sizeof(struct cqhci_host),
+			       GFP_KERNEL);
+	if (!cq_host)
+		return -ENOMEM;
+
+	cq_host->mmio = host->ioaddr + SDHCI_AM654_CQE_BASE_ADDR;
+	cq_host->quirks |= CQHCI_QUIRK_SHORT_TXFR_DESC_SZ;
+	cq_host->caps |= CQHCI_TASK_DESC_SZ_128;
+	cq_host->ops = &sdhci_am654_cqhci_ops;
+
+	host->mmc->caps2 |= MMC_CAP2_CQE;
+
+	ret = cqhci_init(cq_host, host->mmc, 1);
+
+	return ret;
+}
+
+static int sdhci_am654_init(struct sdhci_host *host)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_am654_data *sdhci_am654 = sdhci_pltfm_priv(pltfm_host);
+	struct device *dev = mmc_dev(host->mmc);
+	u32 ctl_cfg_2 = 0;
+	u32 mask;
+	u32 val;
+	int ret;
+
+	/* Reset OTAP to default value */
+	mask = OTAPDLYENA_MASK | OTAPDLYSEL_MASK;
+	regmap_update_bits(sdhci_am654->base, PHY_CTRL4, mask, 0x0);
+
+	if (sdhci_am654->flags & DLL_PRESENT) {
+		/* Configure DLL TRIM */
+		mask = DLL_TRIM_ICP_MASK;
+		val = sdhci_am654->trm_icp << DLL_TRIM_ICP_SHIFT;
+
+		/* Configure DLL driver strength */
+		mask |= DR_TY_MASK;
+		val |= sdhci_am654->drv_strength << DR_TY_SHIFT;
+		regmap_update_bits(sdhci_am654->base, PHY_CTRL1, mask, val);
+
+		regmap_read(sdhci_am654->base, PHY_STAT1, &val);
+		if (~val & CALDONE_MASK) {
+			/* Calibrate IO lines */
+			regmap_update_bits(sdhci_am654->base, PHY_CTRL1,
+					   PDB_MASK, PDB_MASK);
+			ret = regmap_read_poll_timeout(sdhci_am654->base,
+						       PHY_STAT1, val,
+						       val & CALDONE_MASK,
+						       1, 20);
+			if (ret)
+				return ret;
+		}
+	}
+
+	/* Enable pins by setting IO mux to 0 */
+	if (sdhci_am654->flags & IOMUX_PRESENT)
+		regmap_update_bits(sdhci_am654->base, PHY_CTRL1,
+				   IOMUX_ENABLE_MASK, 0);
+
+	/* Set slot type based on SD or eMMC */
+	if (host->mmc->caps & MMC_CAP_NONREMOVABLE)
+		ctl_cfg_2 = SLOTTYPE_EMBEDDED;
+
+	regmap_update_bits(sdhci_am654->base, CTL_CFG_2, SLOTTYPE_MASK,
+			   ctl_cfg_2);
+
+	ret = sdhci_setup_host(host);
+	if (ret)
+		return ret;
+
+	ret = sdhci_am654_get_otap_delay(host, sdhci_am654);
+	if (ret)
+		goto err_cleanup_host;
+	if (!of_device_is_compatible(dev->of_node, "ti,am654-sdhci-5.1")) {
+		ret = sdhci_am654_cqe_add_host(host);
+		if (ret)
+			goto err_cleanup_host;
+	}
+
+	ret = __sdhci_add_host(host);
+	if (ret)
+		goto err_cleanup_host;
+
+	return 0;
+
+err_cleanup_host:
+	sdhci_cleanup_host(host);
+	return ret;
+}
+
+static int sdhci_am654_get_of_property(struct platform_device *pdev,
+					struct sdhci_am654_data *sdhci_am654)
+{
+	struct device *dev = &pdev->dev;
+	int drv_strength;
+	int ret;
+
+	if (sdhci_am654->flags & DLL_PRESENT) {
+		ret = device_property_read_u32(dev, "ti,trm-icp",
+					       &sdhci_am654->trm_icp);
+		if (ret)
+			return ret;
+
+		ret = device_property_read_u32(dev, "ti,driver-strength-ohm",
+					       &drv_strength);
+		if (ret)
+			return ret;
+
+		switch (drv_strength) {
+		case 50:
+			sdhci_am654->drv_strength = DRIVER_STRENGTH_50_OHM;
+			break;
+		case 33:
+			sdhci_am654->drv_strength = DRIVER_STRENGTH_33_OHM;
+			break;
+		case 66:
+			sdhci_am654->drv_strength = DRIVER_STRENGTH_66_OHM;
+			break;
+		case 100:
+			sdhci_am654->drv_strength = DRIVER_STRENGTH_100_OHM;
+			break;
+		case 40:
+			sdhci_am654->drv_strength = DRIVER_STRENGTH_40_OHM;
+			break;
+		default:
+			dev_err(dev, "Invalid driver strength\n");
+			return -EINVAL;
+		}
+	}
+
+	device_property_read_u32(dev, "ti,strobe-sel", &sdhci_am654->strb_sel);
+	device_property_read_u32(dev, "ti,clkbuf-sel",
+				 &sdhci_am654->clkbuf_sel);
+
+	sdhci_get_of_property(pdev);
+
+	return 0;
+}
+
+static const struct of_device_id sdhci_am654_of_match[] = {
+	{
+		.compatible = "ti,am654-sdhci-5.1",
+		.data = &sdhci_am654_drvdata,
+	},
+	{
+		.compatible = "ti,j721e-sdhci-8bit",
+		.data = &sdhci_j721e_8bit_drvdata,
+	},
+	{
+		.compatible = "ti,j721e-sdhci-4bit",
+		.data = &sdhci_j721e_4bit_drvdata,
+	},
+	{ /* sentinel */ }
+};
+
+static int sdhci_am654_probe(struct platform_device *pdev)
+{
+	const struct sdhci_am654_driver_data *drvdata;
+	struct sdhci_pltfm_host *pltfm_host;
+	struct sdhci_am654_data *sdhci_am654;
+	const struct of_device_id *match;
+	struct sdhci_host *host;
+	struct resource *res;
+	struct clk *clk_xin;
+	struct device *dev = &pdev->dev;
+	void __iomem *base;
+	int ret;
+
+	match = of_match_node(sdhci_am654_of_match, pdev->dev.of_node);
+	drvdata = match->data;
+	host = sdhci_pltfm_init(pdev, drvdata->pdata, sizeof(*sdhci_am654));
+	if (IS_ERR(host))
+		return PTR_ERR(host);
+
+	pltfm_host = sdhci_priv(host);
+	sdhci_am654 = sdhci_pltfm_priv(pltfm_host);
+	sdhci_am654->flags = drvdata->flags;
+
+	clk_xin = devm_clk_get(dev, "clk_xin");
+	if (IS_ERR(clk_xin)) {
+		dev_err(dev, "clk_xin clock not found.\n");
+		ret = PTR_ERR(clk_xin);
+		goto err_pltfm_free;
+	}
+
+	pltfm_host->clk = clk_xin;
+
+	/* Clocks are enabled using pm_runtime */
+	pm_runtime_enable(dev);
+	ret = pm_runtime_get_sync(dev);
+	if (ret < 0) {
+		pm_runtime_put_noidle(dev);
+		goto pm_runtime_disable;
+	}
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+	base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(base)) {
+		ret = PTR_ERR(base);
+		goto pm_runtime_put;
+	}
+
+	sdhci_am654->base = devm_regmap_init_mmio(dev, base,
+						  &sdhci_am654_regmap_config);
+	if (IS_ERR(sdhci_am654->base)) {
+		dev_err(dev, "Failed to initialize regmap\n");
+		ret = PTR_ERR(sdhci_am654->base);
+		goto pm_runtime_put;
+	}
+
+	ret = sdhci_am654_get_of_property(pdev, sdhci_am654);
+	if (ret)
+		goto pm_runtime_put;
+
+	ret = mmc_of_parse(host->mmc);
+	if (ret) {
+		dev_err(dev, "parsing dt failed (%d)\n", ret);
+		goto pm_runtime_put;
+	}
+
+	ret = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(48));
+	if (ret) {
+		dev_err(dev, "error setting dma mask: %d\n", ret);
+		goto pm_runtime_put;
+	}
+
+	ret = sdhci_am654_init(host);
+	if (ret)
+		goto pm_runtime_put;
+
+	return 0;
+
+pm_runtime_put:
+	pm_runtime_put_sync(dev);
+pm_runtime_disable:
+	pm_runtime_disable(dev);
+err_pltfm_free:
+	sdhci_pltfm_free(pdev);
+	return ret;
+}
+
+static int sdhci_am654_remove(struct platform_device *pdev)
+{
+	struct sdhci_host *host = platform_get_drvdata(pdev);
+	int ret;
+
+	sdhci_remove_host(host, true);
+	ret = pm_runtime_put_sync(&pdev->dev);
+	if (ret < 0)
+		return ret;
+
+	pm_runtime_disable(&pdev->dev);
+	sdhci_pltfm_free(pdev);
+
+	return 0;
+}
+
+static struct platform_driver sdhci_am654_driver = {
+	.driver = {
+		.name = "sdhci-am654",
+		.of_match_table = sdhci_am654_of_match,
+	},
+	.probe = sdhci_am654_probe,
+	.remove = sdhci_am654_remove,
+};
+
+module_platform_driver(sdhci_am654_driver);
+
+MODULE_DESCRIPTION("Driver for SDHCI Controller on TI's AM654 devices");
+MODULE_AUTHOR("Faiz Abbas <faiz_abbas@ti.com>");
+MODULE_LICENSE("GPL");
diff -Naur --no-dereference a/drivers/mmc/host/sdhci-omap.c b/drivers/mmc/host/sdhci-omap.c
--- a/drivers/mmc/host/sdhci-omap.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/mmc/host/sdhci-omap.c	2020-06-19 21:00:31.416227715 -0400
@@ -27,6 +27,7 @@
 #include <linux/regulator/consumer.h>
 #include <linux/pinctrl/consumer.h>
 #include <linux/sys_soc.h>
+#include <linux/thermal.h>
 
 #include "sdhci-pltfm.h"
 
@@ -115,6 +116,7 @@
 
 	struct pinctrl		*pinctrl;
 	struct pinctrl_state	**pinctrl_state;
+	bool			is_tuning;
 };
 
 static void sdhci_omap_start_clock(struct sdhci_omap_host *omap_host);
@@ -289,19 +291,19 @@
 	struct sdhci_host *host = mmc_priv(mmc);
 	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 	struct sdhci_omap_host *omap_host = sdhci_pltfm_priv(pltfm_host);
+	struct thermal_zone_device *thermal_dev;
 	struct device *dev = omap_host->dev;
 	struct mmc_ios *ios = &mmc->ios;
 	u32 start_window = 0, max_window = 0;
+	bool single_point_failure = false;
 	bool dcrc_was_enabled = false;
 	u8 cur_match, prev_match = 0;
 	u32 length = 0, max_len = 0;
 	u32 phase_delay = 0;
+	int temperature;
 	int ret = 0;
 	u32 reg;
-
-	pltfm_host = sdhci_priv(host);
-	omap_host = sdhci_pltfm_priv(pltfm_host);
-	dev = omap_host->dev;
+	int i;
 
 	/* clock tuning is not needed for upto 52MHz */
 	if (ios->clock <= 52000000)
@@ -311,6 +313,16 @@
 	if (ios->timing == MMC_TIMING_UHS_SDR50 && !(reg & CAPA2_TSDR50))
 		return 0;
 
+	thermal_dev = thermal_zone_get_zone_by_name("cpu_thermal");
+	if (IS_ERR(thermal_dev)) {
+		dev_err(dev, "Unable to get thermal zone for tuning\n");
+		return PTR_ERR(thermal_dev);
+	}
+
+	ret = thermal_zone_get_temp(thermal_dev, &temperature);
+	if (ret)
+		return ret;
+
 	reg = sdhci_omap_readl(omap_host, SDHCI_OMAP_DLL);
 	reg |= DLL_SWT;
 	sdhci_omap_writel(omap_host, SDHCI_OMAP_DLL, reg);
@@ -326,6 +338,13 @@
 		dcrc_was_enabled = true;
 	}
 
+	omap_host->is_tuning = true;
+
+	/*
+	 * Stage 1: Search for a maximum pass window ignoring any
+	 * any single point failures. If the tuning value ends up
+	 * near it, move away from it in stage 2 below
+	 */
 	while (phase_delay <= MAX_PHASE_DELAY) {
 		sdhci_omap_set_dll(omap_host, phase_delay);
 
@@ -333,10 +352,15 @@
 		if (cur_match) {
 			if (prev_match) {
 				length++;
+			} else if (single_point_failure) {
+				/* ignore single point failure */
+				length++;
 			} else {
 				start_window = phase_delay;
 				length = 1;
 			}
+		} else {
+			single_point_failure = prev_match;
 		}
 
 		if (length > max_len) {
@@ -354,18 +378,84 @@
 		goto tuning_error;
 	}
 
+	/*
+	 * Assign tuning value as a ratio of maximum pass window based
+	 * on temperature
+	 */
+	if (temperature < -20000)
+		phase_delay = min((max_window + 4 * (max_len - 1)) - 24,
+				  max_window +
+				  DIV_ROUND_UP(13 * max_len, 16) * 4);
+	else if (temperature < 20000)
+		phase_delay = max_window + DIV_ROUND_UP(9 * max_len, 16) * 4;
+	else if (temperature < 40000)
+		phase_delay = max_window + DIV_ROUND_UP(8 * max_len, 16) * 4;
+	else if (temperature < 70000)
+		phase_delay = max_window + DIV_ROUND_UP(7 * max_len, 16) * 4;
+	else if (temperature < 90000)
+		phase_delay = max_window + DIV_ROUND_UP(5 * max_len, 16) * 4;
+	else if (temperature < 120000)
+		phase_delay = max_window + DIV_ROUND_UP(4 * max_len, 16) * 4;
+	else
+		phase_delay = max_window + DIV_ROUND_UP(3 * max_len, 16) * 4;
+
+	/*
+	 * Stage 2: Search for a single point failure near the chosen tuning
+	 * value in two steps. First in the +3 to +10 range and then in the
+	 * +2 to -10 range. If found, move away from it in the appropriate
+	 * direction by the appropriate amount depending on the temperature.
+	 */
+	for (i = 3; i <= 10; i++) {
+		sdhci_omap_set_dll(omap_host, phase_delay + i);
+
+		if (mmc_send_tuning(mmc, opcode, NULL)) {
+			if (temperature < 10000)
+				phase_delay += i + 6;
+			else if (temperature < 20000)
+				phase_delay += i - 12;
+			else if (temperature < 70000)
+				phase_delay += i - 8;
+			else
+				phase_delay += i - 6;
+
+			goto single_failure_found;
+		}
+	}
+
+	for (i = 2; i >= -10; i--) {
+		sdhci_omap_set_dll(omap_host, phase_delay + i);
+
+		if (mmc_send_tuning(mmc, opcode, NULL)) {
+			if (temperature < 10000)
+				phase_delay += i + 12;
+			else if (temperature < 20000)
+				phase_delay += i + 8;
+			else if (temperature < 70000)
+				phase_delay += i + 8;
+			else if (temperature < 90000)
+				phase_delay += i + 10;
+			else
+				phase_delay += i + 12;
+
+			goto single_failure_found;
+		}
+	}
+
+single_failure_found:
 	reg = sdhci_omap_readl(omap_host, SDHCI_OMAP_AC12);
 	if (!(reg & AC12_SCLK_SEL)) {
 		ret = -EIO;
 		goto tuning_error;
 	}
 
-	phase_delay = max_window + 4 * (max_len >> 1);
 	sdhci_omap_set_dll(omap_host, phase_delay);
 
+	omap_host->is_tuning = false;
+
 	goto ret;
 
 tuning_error:
+	omap_host->is_tuning = false;
 	dev_err(dev, "Tuning failed\n");
 	sdhci_omap_disable_tuning(omap_host);
 
@@ -695,6 +785,55 @@
 	sdhci_omap_start_clock(omap_host);
 }
 
+void sdhci_omap_reset(struct sdhci_host *host, u8 mask)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_omap_host *omap_host = sdhci_pltfm_priv(pltfm_host);
+
+	/* Don't reset data lines during tuning operation */
+	if (omap_host->is_tuning)
+		mask &= ~SDHCI_RESET_DATA;
+
+	sdhci_reset(host, mask);
+}
+
+#define CMD_ERR_MASK (SDHCI_INT_CRC | SDHCI_INT_END_BIT | SDHCI_INT_INDEX |\
+		      SDHCI_INT_TIMEOUT)
+#define CMD_MASK (CMD_ERR_MASK | SDHCI_INT_RESPONSE)
+
+static irqreturn_t sdhci_omap_irq(struct sdhci_host *host, u32 intmask)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_omap_host *omap_host = sdhci_pltfm_priv(pltfm_host);
+
+	if (omap_host->is_tuning && host->cmd && !host->data_early &&
+	    (intmask & CMD_ERR_MASK)) {
+
+		/*
+		 * Since we are not resetting data lines during tuning
+		 * operation, data error or data complete interrupts
+		 * might still arrive. Mark this request as a failure
+		 * but still wait for the data interrupt
+		 */
+		if (intmask & SDHCI_INT_TIMEOUT)
+			host->cmd->error = -ETIMEDOUT;
+		else
+			host->cmd->error = -EILSEQ;
+
+		host->cmd = NULL;
+
+		/*
+		 * Sometimes command error interrupts and command complete
+		 * interrupt will arrive together. Clear all command related
+		 * interrupts here.
+		 */
+		sdhci_writel(host, intmask & CMD_MASK, SDHCI_INT_STATUS);
+		intmask &= ~CMD_MASK;
+	}
+
+	return intmask;
+}
+
 static struct sdhci_ops sdhci_omap_ops = {
 	.set_clock = sdhci_omap_set_clock,
 	.set_power = sdhci_omap_set_power,
@@ -703,8 +842,9 @@
 	.get_min_clock = sdhci_omap_get_min_clock,
 	.set_bus_width = sdhci_omap_set_bus_width,
 	.platform_send_init_74_clocks = sdhci_omap_init_74_clocks,
-	.reset = sdhci_reset,
+	.reset = sdhci_omap_reset,
 	.set_uhs_signaling = sdhci_omap_set_uhs_signaling,
+	.irq = sdhci_omap_irq,
 };
 
 static int sdhci_omap_set_capabilities(struct sdhci_omap_host *omap_host)
diff -Naur --no-dereference a/drivers/mtd/chips/cfi_cmdset_0002.c b/drivers/mtd/chips/cfi_cmdset_0002.c
--- a/drivers/mtd/chips/cfi_cmdset_0002.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/mtd/chips/cfi_cmdset_0002.c	2020-06-19 21:00:31.416227715 -0400
@@ -49,6 +49,16 @@
 #define SST49LF008A		0x005a
 #define AT49BV6416		0x00d6
 
+/*
+ * Status Register bit description. Used by flash devices that don't
+ * support DQ polling (e.g. HyperFlash)
+ */
+#define CFI_SR_DRB		BIT(7)
+#define CFI_SR_ESB		BIT(5)
+#define CFI_SR_PSB		BIT(4)
+#define CFI_SR_WBASB		BIT(3)
+#define CFI_SR_SLSB		BIT(1)
+
 static int cfi_amdstd_read (struct mtd_info *, loff_t, size_t, size_t *, u_char *);
 static int cfi_amdstd_write_words(struct mtd_info *, loff_t, size_t, size_t *, const u_char *);
 static int cfi_amdstd_write_buffers(struct mtd_info *, loff_t, size_t, size_t *, const u_char *);
@@ -97,6 +107,48 @@
 	.module		= THIS_MODULE
 };
 
+/*
+ * Use status register to poll for Erase/write completion when DQ is not
+ * supported. This is indicated by Bit[1:0] of SoftwareFeatures field in
+ * CFI Primary Vendor-Specific Extended Query table 1.5
+ */
+static int cfi_use_status_reg(struct cfi_private *cfi)
+{
+	struct cfi_pri_amdstd *extp = cfi->cmdset_priv;
+
+	return extp->MinorVersion >= '5' &&
+		(extp->SoftwareFeatures & 0x3) == 0x1;
+}
+
+static void cfi_check_err_status(struct map_info *map, unsigned long adr)
+{
+	struct cfi_private *cfi = map->fldrv_priv;
+	map_word status;
+
+	if (!cfi_use_status_reg(cfi))
+		return;
+
+	cfi_send_gen_cmd(0x70, cfi->addr_unlock1, 0, map, cfi,
+			 cfi->device_type, NULL);
+	status = map_read(map, adr);
+
+	if (map_word_bitsset(map, status, CMD(0x3a))) {
+		unsigned long chipstatus = MERGESTATUS(status);
+
+		if (chipstatus & CFI_SR_ESB)
+			pr_err("%s erase operation failed, status %lx\n",
+			       map->name, chipstatus);
+		if (chipstatus & CFI_SR_PSB)
+			pr_err("%s program operation failed, status %lx\n",
+			       map->name, chipstatus);
+		if (chipstatus & CFI_SR_WBASB)
+			pr_err("%s buffer program command aborted, status %lx\n",
+			       map->name, chipstatus);
+		if (chipstatus & CFI_SR_SLSB)
+			pr_err("%s sector write protected, status %lx\n",
+			       map->name, chipstatus);
+	}
+}
 
 /* #define DEBUG_CFI_FEATURES */
 
@@ -744,8 +796,22 @@
  */
 static int __xipram chip_ready(struct map_info *map, unsigned long addr)
 {
+	struct cfi_private *cfi = map->fldrv_priv;
 	map_word d, t;
 
+	if (cfi_use_status_reg(cfi)) {
+		map_word ready = CMD(CFI_SR_DRB);
+		/*
+		 * For chips that support status register, check device
+		 * ready bit
+		 */
+		cfi_send_gen_cmd(0x70, cfi->addr_unlock1, 0, map, cfi,
+				 cfi->device_type, NULL);
+		d = map_read(map, addr);
+
+		return map_word_andequal(map, d, ready, ready);
+	}
+
 	d = map_read(map, addr);
 	t = map_read(map, addr);
 
@@ -769,8 +835,27 @@
  */
 static int __xipram chip_good(struct map_info *map, unsigned long addr, map_word expected)
 {
+	struct cfi_private *cfi = map->fldrv_priv;
 	map_word oldd, curd;
 
+	if (cfi_use_status_reg(cfi)) {
+		map_word ready = CMD(CFI_SR_DRB);
+		map_word err = CMD(CFI_SR_PSB | CFI_SR_ESB);
+		/*
+		 * For chips that support status register, check device
+		 * ready bit and Erase/Program status bit to know if
+		 * operation succeeded.
+		 */
+		cfi_send_gen_cmd(0x70, cfi->addr_unlock1, 0, map, cfi,
+				 cfi->device_type, NULL);
+		curd = map_read(map, addr);
+
+		if (map_word_andequal(map, curd, ready, ready))
+			return !map_word_bitsset(map, curd, err);
+
+		return 0;
+	}
+
 	oldd = map_read(map, addr);
 	curd = map_read(map, addr);
 
@@ -1649,6 +1734,7 @@
 	/* Did we succeed? */
 	if (ret) {
 		/* reset on all failures. */
+		cfi_check_err_status(map, adr);
 		map_write(map, CMD(0xF0), chip->start);
 		/* FIXME - should have reset delay before continuing */
 
@@ -1902,6 +1988,7 @@
 	 * See e.g.
 	 * http://www.spansion.com/Support/Application%20Notes/MirrorBit_Write_Buffer_Prog_Page_Buffer_Read_AN.pdf
 	 */
+	cfi_check_err_status(map, adr);
 	cfi_send_gen_cmd(0xAA, cfi->addr_unlock1, chip->start, map, cfi,
 			 cfi->device_type, NULL);
 	cfi_send_gen_cmd(0x55, cfi->addr_unlock2, chip->start, map, cfi,
@@ -2108,6 +2195,7 @@
 
 	if (!chip_good(map, adr, datum)) {
 		/* reset on all failures. */
+		cfi_check_err_status(map, adr);
 		map_write(map, CMD(0xF0), chip->start);
 		/* FIXME - should have reset delay before continuing */
 
@@ -2317,6 +2405,7 @@
 	/* Did we succeed? */
 	if (ret) {
 		/* reset on all failures. */
+		cfi_check_err_status(map, adr);
 		map_write(map, CMD(0xF0), chip->start);
 		/* FIXME - should have reset delay before continuing */
 
@@ -2413,6 +2502,7 @@
 	/* Did we succeed? */
 	if (ret) {
 		/* reset on all failures. */
+		cfi_check_err_status(map, adr);
 		map_write(map, CMD(0xF0), chip->start);
 		/* FIXME - should have reset delay before continuing */
 
diff -Naur --no-dereference a/drivers/mtd/hyperbus/hbmc-am654.c b/drivers/mtd/hyperbus/hbmc-am654.c
--- a/drivers/mtd/hyperbus/hbmc-am654.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/mtd/hyperbus/hbmc-am654.c	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,141 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+// Author: Vignesh Raghavendra <vigneshr@ti.com>
+
+#include <linux/err.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mtd/cfi.h>
+#include <linux/mtd/hyperbus.h>
+#include <linux/mtd/mtd.h>
+#include <linux/mux/consumer.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/types.h>
+
+#define AM654_HBMC_CALIB_COUNT 25
+
+struct am654_hbmc_priv {
+	struct hyperbus_ctlr ctlr;
+	struct hyperbus_device hbdev;
+};
+
+static int am654_hbmc_calibrate(struct hyperbus_device *hbdev)
+{
+	struct map_info *map = &hbdev->map;
+	struct cfi_private cfi;
+	int count = AM654_HBMC_CALIB_COUNT;
+	int pass_count = 0;
+	int ret;
+
+	cfi.interleave = 1;
+	cfi.device_type = CFI_DEVICETYPE_X16;
+	cfi_send_gen_cmd(0xF0, 0, 0, map, &cfi, cfi.device_type, NULL);
+	cfi_send_gen_cmd(0x98, 0x55, 0, map, &cfi, cfi.device_type, NULL);
+
+	while (count--) {
+		ret = cfi_qry_present(map, 0, &cfi);
+		if (ret)
+			pass_count++;
+		else
+			pass_count = 0;
+		if (pass_count == 5)
+			break;
+	}
+
+	cfi_qry_mode_off(0, map, &cfi);
+
+	return ret;
+}
+
+static const struct hyperbus_ops am654_hbmc_ops = {
+	.calibrate = am654_hbmc_calibrate,
+};
+
+static int am654_hbmc_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct am654_hbmc_priv *priv;
+	int ret;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, priv);
+
+	if (of_property_read_bool(dev->of_node, "mux-controls")) {
+		struct mux_control *control = devm_mux_control_get(dev, NULL);
+
+		if (IS_ERR(control))
+			return PTR_ERR(control);
+
+		ret = mux_control_select(control, 1);
+		if (ret) {
+			dev_err(dev, "Failed to select HBMC mux\n");
+			return ret;
+		}
+	}
+
+	pm_runtime_enable(dev);
+	ret = pm_runtime_get_sync(dev);
+	if (ret < 0) {
+		pm_runtime_put_noidle(dev);
+		goto disable_pm;
+	}
+
+	priv->ctlr.dev = dev;
+	priv->ctlr.ops = &am654_hbmc_ops;
+	priv->hbdev.ctlr = &priv->ctlr;
+	priv->hbdev.np = of_get_next_child(dev->of_node, NULL);
+	ret = hyperbus_register_device(&priv->hbdev);
+	if (ret) {
+		dev_err(dev, "failed to register controller\n");
+		pm_runtime_put_sync(&pdev->dev);
+		goto disable_pm;
+	}
+
+	return 0;
+disable_pm:
+	pm_runtime_disable(dev);
+	return ret;
+}
+
+static int am654_hbmc_remove(struct platform_device *pdev)
+{
+	struct am654_hbmc_priv *priv = platform_get_drvdata(pdev);
+	int ret;
+
+	ret = hyperbus_unregister_device(&priv->hbdev);
+	pm_runtime_put_sync(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+
+	return ret;
+}
+
+static const struct of_device_id am654_hbmc_dt_ids[] = {
+	{
+		.compatible = "ti,am654-hbmc",
+	},
+	{ /* end of table */ }
+};
+
+MODULE_DEVICE_TABLE(of, am654_hbmc_dt_ids);
+
+static struct platform_driver am654_hbmc_platform_driver = {
+	.probe = am654_hbmc_probe,
+	.remove = am654_hbmc_remove,
+	.driver = {
+		.name = "hbmc-am654",
+		.of_match_table = am654_hbmc_dt_ids,
+	},
+};
+
+module_platform_driver(am654_hbmc_platform_driver);
+
+MODULE_DESCRIPTION("HBMC driver for AM654 SoC");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:hbmc-am654");
+MODULE_AUTHOR("Vignesh Raghavendra <vigneshr@ti.com>");
diff -Naur --no-dereference a/drivers/mtd/hyperbus/hyperbus-core.c b/drivers/mtd/hyperbus/hyperbus-core.c
--- a/drivers/mtd/hyperbus/hyperbus-core.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/mtd/hyperbus/hyperbus-core.c	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,154 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+// Author: Vignesh Raghavendra <vigneshr@ti.com>
+
+#include <linux/err.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mtd/hyperbus.h>
+#include <linux/mtd/map.h>
+#include <linux/mtd/mtd.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/types.h>
+
+static struct hyperbus_device *map_to_hbdev(struct map_info *map)
+{
+	return container_of(map, struct hyperbus_device, map);
+}
+
+static map_word hyperbus_read16(struct map_info *map, unsigned long addr)
+{
+	struct hyperbus_device *hbdev = map_to_hbdev(map);
+	struct hyperbus_ctlr *ctlr = hbdev->ctlr;
+	map_word read_data;
+
+	read_data.x[0] = ctlr->ops->read16(hbdev, addr);
+
+	return read_data;
+}
+
+static void hyperbus_write16(struct map_info *map, map_word d,
+			     unsigned long addr)
+{
+	struct hyperbus_device *hbdev = map_to_hbdev(map);
+	struct hyperbus_ctlr *ctlr = hbdev->ctlr;
+
+	ctlr->ops->write16(hbdev, addr, d.x[0]);
+}
+
+static void hyperbus_copy_from(struct map_info *map, void *to,
+			       unsigned long from, ssize_t len)
+{
+	struct hyperbus_device *hbdev = map_to_hbdev(map);
+	struct hyperbus_ctlr *ctlr = hbdev->ctlr;
+
+	ctlr->ops->copy_from(hbdev, to, from, len);
+}
+
+static void hyperbus_copy_to(struct map_info *map, unsigned long to,
+			     const void *from, ssize_t len)
+{
+	struct hyperbus_device *hbdev = map_to_hbdev(map);
+	struct hyperbus_ctlr *ctlr = hbdev->ctlr;
+
+	ctlr->ops->copy_to(hbdev, to, from, len);
+}
+
+int hyperbus_register_device(struct hyperbus_device *hbdev)
+{
+	const struct hyperbus_ops *ops;
+	struct hyperbus_ctlr *ctlr;
+	struct device_node *np;
+	struct map_info *map;
+	struct resource res;
+	struct device *dev;
+	int ret;
+
+	if (!hbdev || !hbdev->np || !hbdev->ctlr || !hbdev->ctlr->dev) {
+		pr_err("hyperbus: please fill all the necessary fields!\n");
+		return -EINVAL;
+	}
+
+	np = hbdev->np;
+	ctlr = hbdev->ctlr;
+	if (!of_device_is_compatible(np, "cypress,hyperflash"))
+		return -ENODEV;
+
+	hbdev->memtype = HYPERFLASH;
+
+	ret = of_address_to_resource(np, 0, &res);
+	if (ret)
+		return ret;
+
+	dev = ctlr->dev;
+	map = &hbdev->map;
+	map->size = resource_size(&res);
+	map->virt = devm_ioremap_resource(dev, &res);
+	if (IS_ERR(map->virt))
+		return PTR_ERR(map->virt);
+
+	map->name = dev_name(dev);
+	map->bankwidth = 2;
+	map->device_node = np;
+
+	simple_map_init(map);
+	ops = ctlr->ops;
+	if (ops) {
+		if (ops->read16)
+			map->read = hyperbus_read16;
+		if (ops->write16)
+			map->write = hyperbus_write16;
+		if (ops->copy_to)
+			map->copy_to = hyperbus_copy_to;
+		if (ops->copy_from)
+			map->copy_from = hyperbus_copy_from;
+
+		if (ops->calibrate && !ctlr->calibrated) {
+			ret = ops->calibrate(hbdev);
+			if (!ret) {
+				dev_err(dev, "Calibration failed\n");
+				return -ENODEV;
+			}
+			ctlr->calibrated = true;
+		}
+	}
+
+	hbdev->mtd = do_map_probe("cfi_probe", map);
+	if (!hbdev->mtd) {
+		dev_err(dev, "probing of hyperbus device failed\n");
+		return -ENODEV;
+	}
+
+	hbdev->mtd->dev.parent = dev;
+	mtd_set_of_node(hbdev->mtd, np);
+
+	ret = mtd_device_register(hbdev->mtd, NULL, 0);
+	if (ret) {
+		dev_err(dev, "failed to register mtd device\n");
+		map_destroy(hbdev->mtd);
+		return ret;
+	}
+	hbdev->registered = true;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(hyperbus_register_device);
+
+int hyperbus_unregister_device(struct hyperbus_device *hbdev)
+{
+	int ret = 0;
+
+	if (hbdev && hbdev->mtd && hbdev->registered) {
+		ret = mtd_device_unregister(hbdev->mtd);
+		map_destroy(hbdev->mtd);
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(hyperbus_unregister_device);
+
+MODULE_DESCRIPTION("HyperBus Framework");
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Vignesh Raghavendra <vigneshr@ti.com>");
diff -Naur --no-dereference a/drivers/mtd/hyperbus/Kconfig b/drivers/mtd/hyperbus/Kconfig
--- a/drivers/mtd/hyperbus/Kconfig	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/mtd/hyperbus/Kconfig	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,23 @@
+menuconfig MTD_HYPERBUS
+	tristate "HyperBus support"
+	select MTD_CFI
+	select MTD_MAP_BANK_WIDTH_2
+	select MTD_CFI_AMDSTD
+	select MTD_COMPLEX_MAPPINGS
+	help
+	  This is the framework for the HyperBus which can be used by
+	  the HyperBus Controller driver to communicate with
+	  HyperFlash. See Cypress HyperBus specification for more
+	  details
+
+if MTD_HYPERBUS
+
+config HBMC_AM654
+	tristate "HyperBus controller driver for AM65x SoC"
+	select MULTIPLEXER
+	select MUX_MMIO
+	help
+	 This is the driver for HyperBus controller on TI's AM65x and
+	 other SoCs
+
+endif # MTD_HYPERBUS
diff -Naur --no-dereference a/drivers/mtd/hyperbus/Makefile b/drivers/mtd/hyperbus/Makefile
--- a/drivers/mtd/hyperbus/Makefile	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/mtd/hyperbus/Makefile	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
+
+obj-$(CONFIG_MTD_HYPERBUS)	+= hyperbus-core.o
+obj-$(CONFIG_HBMC_AM654)	+= hbmc-am654.o
diff -Naur --no-dereference a/drivers/mtd/Kconfig b/drivers/mtd/Kconfig
--- a/drivers/mtd/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/mtd/Kconfig	2020-06-19 21:00:31.416227715 -0400
@@ -339,4 +339,6 @@
 
 source "drivers/mtd/ubi/Kconfig"
 
+source "drivers/mtd/hyperbus/Kconfig"
+
 endif # MTD
diff -Naur --no-dereference a/drivers/mtd/Makefile b/drivers/mtd/Makefile
--- a/drivers/mtd/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/mtd/Makefile	2020-06-19 21:00:31.416227715 -0400
@@ -36,3 +36,4 @@
 
 obj-$(CONFIG_MTD_SPI_NOR)	+= spi-nor/
 obj-$(CONFIG_MTD_UBI)		+= ubi/
+obj-$(CONFIG_MTD_HYPERBUS)	+= hyperbus/
diff -Naur --no-dereference a/drivers/mtd/spi-nor/cadence-quadspi.c b/drivers/mtd/spi-nor/cadence-quadspi.c
--- a/drivers/mtd/spi-nor/cadence-quadspi.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/mtd/spi-nor/cadence-quadspi.c	2020-06-19 21:00:31.416227715 -0400
@@ -40,10 +40,17 @@
 
 #define CQSPI_NAME			"cadence-qspi"
 #define CQSPI_MAX_CHIPSELECT		16
+#define CQSPI_MAX_RX_DLL_DELAY		0x7F
 
 /* Quirks */
 #define CQSPI_NEEDS_WR_DELAY		BIT(0)
 
+/* Capabilities mask */
+#define CQSPI_BASE_HWCAPS_MASK					\
+	(SNOR_HWCAPS_READ | SNOR_HWCAPS_READ_FAST |		\
+	SNOR_HWCAPS_READ_1_1_2 | SNOR_HWCAPS_READ_1_1_4 |	\
+	SNOR_HWCAPS_PP)
+
 struct cqspi_st;
 
 struct cqspi_flash_pdata {
@@ -61,6 +68,9 @@
 	u8		cs;
 	bool		registered;
 	bool		use_direct_mode;
+	bool		phy_mode;
+	bool		phy_calibrated;
+	bool		dqs_en;
 };
 
 struct cqspi_st {
@@ -78,6 +88,7 @@
 	struct dma_chan		*rx_chan;
 	struct completion	rx_dma_complete;
 	dma_addr_t		mmap_phys_base;
+	void			*dma_bb_rx;
 
 	int			current_cs;
 	int			current_page_size;
@@ -90,9 +101,15 @@
 	bool			rclk_en;
 	u32			trigger_address;
 	u32			wr_delay;
+	u32			delayelem_ps;
 	struct cqspi_flash_pdata f_pdata[CQSPI_MAX_CHIPSELECT];
 };
 
+struct cqspi_driver_platdata {
+	u32 hwcaps_mask;
+	u8 quirks;
+};
+
 /* Operation timeout value */
 #define CQSPI_TIMEOUT_MS			500
 #define CQSPI_READ_TIMEOUT_MS			10
@@ -101,6 +118,7 @@
 #define CQSPI_INST_TYPE_SINGLE			0
 #define CQSPI_INST_TYPE_DUAL			1
 #define CQSPI_INST_TYPE_QUAD			2
+#define CQSPI_INST_TYPE_OCTAL			3
 
 #define CQSPI_DUMMY_CLKS_PER_BYTE		8
 #define CQSPI_DUMMY_BYTES_MAX			4
@@ -111,17 +129,21 @@
 /* Register map */
 #define CQSPI_REG_CONFIG			0x00
 #define CQSPI_REG_CONFIG_ENABLE_MASK		BIT(0)
+#define CQSPI_REG_CONFIG_PHY			BIT(3)
 #define CQSPI_REG_CONFIG_ENB_DIR_ACC_CTRL	BIT(7)
 #define CQSPI_REG_CONFIG_DECODE_MASK		BIT(9)
 #define CQSPI_REG_CONFIG_CHIPSELECT_LSB		10
 #define CQSPI_REG_CONFIG_DMA_MASK		BIT(15)
 #define CQSPI_REG_CONFIG_BAUD_LSB		19
+#define CQSPI_REG_CONFIG_DTR_PROTO		BIT(24)
+#define CQSPI_REG_CONFIG_PHY_PIPELINE		BIT(25)
 #define CQSPI_REG_CONFIG_IDLE_LSB		31
 #define CQSPI_REG_CONFIG_CHIPSELECT_MASK	0xF
 #define CQSPI_REG_CONFIG_BAUD_MASK		0xF
 
 #define CQSPI_REG_RD_INSTR			0x04
 #define CQSPI_REG_RD_INSTR_OPCODE_LSB		0
+#define CQSPI_REG_RD_DDR_EN			BIT(10)
 #define CQSPI_REG_RD_INSTR_TYPE_INSTR_LSB	8
 #define CQSPI_REG_RD_INSTR_TYPE_ADDR_LSB	12
 #define CQSPI_REG_RD_INSTR_TYPE_DATA_LSB	16
@@ -150,6 +172,7 @@
 #define CQSPI_REG_READCAPTURE			0x10
 #define CQSPI_REG_READCAPTURE_BYPASS_LSB	0
 #define CQSPI_REG_READCAPTURE_DELAY_LSB		1
+#define CQSPI_REG_READCAPTURE_DQS		BIT(8)
 #define CQSPI_REG_READCAPTURE_DELAY_MASK	0xF
 
 #define CQSPI_REG_SIZE				0x14
@@ -219,6 +242,20 @@
 #define CQSPI_REG_CMDWRITEDATALOWER		0xA8
 #define CQSPI_REG_CMDWRITEDATAUPPER		0xAC
 
+#define CQSPI_REG_PHY_CONFIGURATION			0xB4
+#define CQSPI_REG_PHY_CONFIGURATION_DLL_RESET		BIT(30)
+#define CQSPI_REG_PHY_CONFIGURATION_DLL_RESYNC		BIT(31)
+#define CQSPI_REG_PHY_CONFIGURATION_RX_DLL_MASK		GENMASK(6, 0)
+#define CQSPI_REG_PHY_CONFIGURATION_TX_DLL_MASK		GENMASK(22, 16)
+#define CQSPI_REG_PHY_CONFIGURATION_TX_DLL_SHIFT	16
+
+#define CQSPI_REG_PHY_MASTER_CONTROL			0xB8
+#define CQSPI_REG_PHY_MASTER_BYPASS_MODE		BIT(23)
+#define CQSPI_REG_PHY_MASTER_CONTROL_INIT_DELAY_MASK	GENMASK(6, 0)
+
+#define CQSPI_REG_DLL_OBS_LOWER				0xBC
+#define CQSPI_REG_DLL_OBS_LOWER_LOCK			BIT(15)
+
 /* Interrupt status bits */
 #define CQSPI_REG_IRQ_MODE_ERR			BIT(0)
 #define CQSPI_REG_IRQ_UNDERFLOW			BIT(1)
@@ -239,6 +276,8 @@
 
 #define CQSPI_IRQ_STATUS_MASK		0x1FFFF
 
+#define CQSPI_DMA_BUFFER_SIZE		SZ_1M
+
 static int cqspi_wait_for_bit(void __iomem *reg, const u32 mask, bool clear)
 {
 	unsigned long end = jiffies + msecs_to_jiffies(CQSPI_TIMEOUT_MS);
@@ -475,19 +514,9 @@
 	if (dummy_clk > CQSPI_DUMMY_CLKS_MAX)
 		dummy_clk = CQSPI_DUMMY_CLKS_MAX;
 
-	if (dummy_clk / 8) {
-		reg |= (1 << CQSPI_REG_RD_INSTR_MODE_EN_LSB);
-		/* Set mode bits high to ensure chip doesn't enter XIP */
-		writel(0xFF, reg_base + CQSPI_REG_MODE_BIT);
-
-		/* Need to subtract the mode byte (8 clocks). */
-		if (f_pdata->inst_width != CQSPI_INST_TYPE_QUAD)
-			dummy_clk -= 8;
-
-		if (dummy_clk)
-			reg |= (dummy_clk & CQSPI_REG_RD_INSTR_DUMMY_MASK)
-			       << CQSPI_REG_RD_INSTR_DUMMY_LSB;
-	}
+	if (dummy_clk / 8)
+		reg |= (dummy_clk & CQSPI_REG_RD_INSTR_DUMMY_MASK)
+		       << CQSPI_REG_RD_INSTR_DUMMY_LSB;
 
 	writel(reg, reg_base + CQSPI_REG_RD_INSTR);
 
@@ -600,14 +629,20 @@
 
 	/* Set opcode. */
 	reg = nor->program_opcode << CQSPI_REG_WR_INSTR_OPCODE_LSB;
+	reg |= f_pdata->data_width << CQSPI_REG_WR_INSTR_TYPE_DATA_LSB;
+	reg |= f_pdata->addr_width << CQSPI_REG_WR_INSTR_TYPE_ADDR_LSB;
 	writel(reg, reg_base + CQSPI_REG_WR_INSTR);
-	reg = cqspi_calc_rdreg(nor, nor->program_opcode);
+	reg = readl(reg_base + CQSPI_REG_RD_INSTR);
+	reg &= ~(CQSPI_REG_RD_INSTR_TYPE_INSTR_MASK <<
+			CQSPI_REG_RD_INSTR_TYPE_INSTR_LSB);
+	reg |= f_pdata->inst_width << CQSPI_REG_RD_INSTR_TYPE_INSTR_LSB;
 	writel(reg, reg_base + CQSPI_REG_RD_INSTR);
 
 	reg = readl(reg_base + CQSPI_REG_SIZE);
 	reg &= ~CQSPI_REG_SIZE_ADDRESS_MASK;
 	reg |= (nor->addr_width - 1);
 	writel(reg, reg_base + CQSPI_REG_SIZE);
+
 	return 0;
 }
 
@@ -821,7 +856,8 @@
 
 static void cqspi_readdata_capture(struct cqspi_st *cqspi,
 				   const bool bypass,
-				   const unsigned int delay)
+				   const unsigned int delay,
+				   const bool dqs_en)
 {
 	void __iomem *reg_base = cqspi->iobase;
 	unsigned int reg;
@@ -833,6 +869,11 @@
 	else
 		reg &= ~(1 << CQSPI_REG_READCAPTURE_BYPASS_LSB);
 
+	if (dqs_en)
+		reg |= CQSPI_REG_READCAPTURE_DQS;
+	else
+		reg &= ~CQSPI_REG_READCAPTURE_DQS;
+
 	reg &= ~(CQSPI_REG_READCAPTURE_DELAY_MASK
 		 << CQSPI_REG_READCAPTURE_DELAY_LSB);
 
@@ -857,6 +898,151 @@
 	writel(reg, reg_base + CQSPI_REG_CONFIG);
 }
 
+static void cqspi_phy_dtr_enable(struct spi_nor *nor, bool enable)
+{
+	struct cqspi_flash_pdata *f_pdata = nor->priv;
+	struct cqspi_st *cqspi = f_pdata->cqspi;
+	void __iomem *reg_base = cqspi->iobase;
+	unsigned int reg;
+
+	reg = readl(reg_base + CQSPI_REG_CONFIG);
+
+	if (enable)
+		reg |= (CQSPI_REG_CONFIG_PHY |
+			CQSPI_REG_CONFIG_PHY_PIPELINE |
+			CQSPI_REG_CONFIG_DTR_PROTO);
+	else
+		reg &= ~(CQSPI_REG_CONFIG_PHY |
+			 CQSPI_REG_CONFIG_PHY_PIPELINE |
+			 CQSPI_REG_CONFIG_DTR_PROTO);
+
+	writel(reg, reg_base + CQSPI_REG_CONFIG);
+
+	if (f_pdata->dqs_en && enable)
+		cqspi_readdata_capture(cqspi, !cqspi->rclk_en,
+				       0, true);
+	else
+		cqspi_readdata_capture(cqspi, !cqspi->rclk_en,
+				       f_pdata->read_delay, false);
+
+	cqspi_wait_idle(cqspi);
+}
+
+static void cqspi_phy_dll_config(struct cqspi_st *cqspi, u8 tx_dly, u8 rx_dly)
+{
+	void __iomem *reg_base = cqspi->iobase;
+	u32 reg;
+
+	reg = readl(reg_base + CQSPI_REG_PHY_CONFIGURATION);
+	reg &= ~CQSPI_REG_PHY_CONFIGURATION_TX_DLL_MASK;
+	reg |= tx_dly << CQSPI_REG_PHY_CONFIGURATION_TX_DLL_SHIFT;
+
+	reg &= ~CQSPI_REG_PHY_CONFIGURATION_RX_DLL_MASK;
+	reg |= rx_dly;
+	writel(reg, reg_base + CQSPI_REG_PHY_CONFIGURATION);
+
+	reg &= ~CQSPI_REG_PHY_CONFIGURATION_DLL_RESYNC;
+	writel(reg, reg_base + CQSPI_REG_PHY_CONFIGURATION);
+	reg |= CQSPI_REG_PHY_CONFIGURATION_DLL_RESYNC;
+	writel(reg, reg_base + CQSPI_REG_PHY_CONFIGURATION);
+	/* Make sure 0->1 transition of DLL resync bit is triggered */
+	wmb();
+	/* satisfy wait for 20 reference clock cycles @166MHz clock */
+	udelay(1);
+}
+
+/* Configure OSPI PHY in Master Bypass mode */
+static int cqspi_calibrate_phy(struct spi_nor *nor, void *calib_data,
+			       size_t len)
+{
+	struct cqspi_flash_pdata *f_pdata = nor->priv;
+	u8 rx_dly_start[CQSPI_MAX_RX_DLL_DELAY + 1];
+	struct cqspi_st *cqspi = f_pdata->cqspi;
+	u8 tx_dly_start = 0xff, tx_dly_end = 0;
+	void __iomem *reg_base = cqspi->iobase;
+	u8 tx_dly = 0, rx_dly = 0;
+	char *read_data;
+	int ret = 0;
+	u32 reg;
+
+	read_data = kmalloc(len, GFP_KERNEL);
+	if (!read_data)
+		return -ENOMEM;
+
+	memset(rx_dly_start, 0xff, sizeof(rx_dly_start));
+
+	reg = readl(reg_base + CQSPI_REG_CONFIG);
+	/* Reset PHY */
+	reg &= ~(CQSPI_REG_CONFIG_PHY | CQSPI_REG_CONFIG_PHY_PIPELINE);
+	writel(reg, reg_base + CQSPI_REG_CONFIG);
+	cqspi_phy_dtr_enable(nor, true);
+
+	/* Find range of TX DLL values which have at least one passing RX
+	 * DLL value
+	 */
+	while (tx_dly <= CQSPI_MAX_RX_DLL_DELAY) {
+		rx_dly = 0;
+
+		while (rx_dly <= CQSPI_MAX_RX_DLL_DELAY) {
+			cqspi_phy_dll_config(cqspi, tx_dly, rx_dly);
+
+			nor->read(nor, 0, len, read_data);
+
+			if (!memcmp(read_data, calib_data, len)) {
+				if (tx_dly_start == 0xff)
+					tx_dly_start = tx_dly;
+				rx_dly_start[tx_dly] = rx_dly;
+				break;
+			}
+			rx_dly++;
+		}
+		if (tx_dly_start != 0xff && rx_dly_start[tx_dly] == 0xff) {
+			tx_dly_end = tx_dly - 1;
+			break;
+		}
+		tx_dly++;
+	}
+
+	if (tx_dly_start == 0xff) {
+		ret = -EINVAL;
+		goto disable_phy;
+	}
+	/* Choose mid value from range of passing TX DLL values */
+	tx_dly = (tx_dly_start + tx_dly_end) / 2;
+	dev_dbg(nor->dev, "TX DLL valid range %x...%x\n",
+		tx_dly_start, tx_dly_end);
+	rx_dly = rx_dly_start[tx_dly];
+
+	/* For the chosen TX DLL value, find range of passing RX DLL
+	 * values
+	 */
+	while (rx_dly <= CQSPI_MAX_RX_DLL_DELAY) {
+		cqspi_phy_dll_config(cqspi, tx_dly, rx_dly);
+
+		nor->read(nor, 0, len, read_data);
+
+		if (memcmp(read_data, calib_data, len))
+			break;
+		rx_dly++;
+	}
+
+	dev_dbg(nor->dev, "RX DLL valid range %x...%x\n",
+		rx_dly_start[tx_dly], rx_dly - 1);
+	/* Choose mid value from range of passing RX DLL values */
+	rx_dly = (rx_dly_start[tx_dly] + rx_dly - 1) / 2;
+
+	dev_dbg(nor->dev, "Calibrating DLLs to (%x, %x)\n", tx_dly, rx_dly);
+	/* Set the chosen TX and RX DLL value pair */
+	cqspi_phy_dll_config(cqspi, tx_dly, rx_dly);
+
+	f_pdata->phy_calibrated = true;
+disable_phy:
+	cqspi_phy_dtr_enable(nor, false);
+	kfree(read_data);
+
+	return ret;
+}
+
 static void cqspi_configure(struct spi_nor *nor)
 {
 	struct cqspi_flash_pdata *f_pdata = nor->priv;
@@ -885,7 +1071,7 @@
 		cqspi_config_baudrate_div(cqspi);
 		cqspi_delay(nor);
 		cqspi_readdata_capture(cqspi, !cqspi->rclk_en,
-				       f_pdata->read_delay);
+				       f_pdata->read_delay, false);
 	}
 
 	if (switch_cs || switch_ck)
@@ -911,6 +1097,27 @@
 		case SNOR_PROTO_1_1_4:
 			f_pdata->data_width = CQSPI_INST_TYPE_QUAD;
 			break;
+		case SNOR_PROTO_1_1_8:
+			f_pdata->data_width = CQSPI_INST_TYPE_OCTAL;
+			break;
+		case SNOR_PROTO_8_8_8_DTR:
+			f_pdata->inst_width = CQSPI_INST_TYPE_OCTAL;
+			f_pdata->data_width = CQSPI_INST_TYPE_OCTAL;
+			f_pdata->addr_width = CQSPI_INST_TYPE_OCTAL;
+			break;
+		default:
+			return -EINVAL;
+		}
+	} else {
+		switch (nor->write_proto) {
+		case SNOR_PROTO_1_1_1:
+			f_pdata->data_width = CQSPI_INST_TYPE_SINGLE;
+			break;
+		case SNOR_PROTO_8_8_8_DTR:
+			f_pdata->data_width = CQSPI_INST_TYPE_OCTAL;
+			f_pdata->addr_width = CQSPI_INST_TYPE_OCTAL;
+			f_pdata->inst_width = CQSPI_INST_TYPE_OCTAL;
+			break;
 		default:
 			return -EINVAL;
 		}
@@ -937,8 +1144,14 @@
 		return ret;
 
 	if (f_pdata->use_direct_mode) {
+		if (nor->mode == SPI_NOR_MODE_OPI_DTR)
+			cqspi_phy_dtr_enable(nor, true);
+
 		memcpy_toio(cqspi->ahb_base + to, buf, len);
 		ret = cqspi_wait_idle(cqspi);
+
+		if (nor->mode == SPI_NOR_MODE_OPI_DTR)
+			cqspi_phy_dtr_enable(nor, false);
 	} else {
 		ret = cqspi_indirect_write_execute(nor, to, buf, len);
 	}
@@ -966,14 +1179,19 @@
 	struct dma_async_tx_descriptor *tx;
 	dma_cookie_t cookie;
 	dma_addr_t dma_dst;
+	struct device *ddev;
 
-	if (!cqspi->rx_chan || !virt_addr_valid(buf)) {
+	cqspi_wait_idle(cqspi);
+
+	if (!cqspi->rx_chan || !virt_addr_valid(buf) || len <= 16) {
 		memcpy_fromio(buf, cqspi->ahb_base + from, len);
+		cqspi_wait_idle(cqspi);
 		return 0;
 	}
+	ddev = cqspi->rx_chan->device->dev;
 
-	dma_dst = dma_map_single(nor->dev, buf, len, DMA_FROM_DEVICE);
-	if (dma_mapping_error(nor->dev, dma_dst)) {
+	dma_dst = dma_map_single(ddev, buf, len, DMA_FROM_DEVICE);
+	if (dma_mapping_error(ddev, dma_dst)) {
 		dev_err(nor->dev, "dma mapping failed\n");
 		return -ENOMEM;
 	}
@@ -1005,9 +1223,79 @@
 		ret = -ETIMEDOUT;
 		goto err_unmap;
 	}
-
 err_unmap:
-	dma_unmap_single(nor->dev, dma_dst, len, DMA_FROM_DEVICE);
+	dma_unmap_single(ddev, dma_dst, len, DMA_FROM_DEVICE);
+	cqspi_wait_idle(cqspi);
+
+	return ret;
+}
+
+static int cqspi_direct_read_execute_phy(struct spi_nor *nor, u_char *buf,
+					 loff_t from, size_t len)
+{
+	struct cqspi_flash_pdata *f_pdata = nor->priv;
+	struct cqspi_st *cqspi = f_pdata->cqspi;
+	size_t readsize, copy_len, xfer_len;
+	loff_t align_from;
+	u_char *to = buf;
+	int ret = 0;
+	u_char *dst;
+	u32 off_delta, len_delta;
+
+	/* Enable DDR mode, PHY and PHY pipeline mode */
+	if (f_pdata->phy_calibrated)
+		cqspi_phy_dtr_enable(nor, true);
+	/*
+	 * Cadence OSPI IP requires 16 byte aligned accesses when PHY
+	 * pipeline mode is enabled in Octal DDR mode. So, driver
+	 * will have to align read addresses to 16 bytes.
+	 * We convert read start address and length to 16 byte
+	 * aligned start address and length. Then use bounce buffer for
+	 * intermediate copy and finally copy only the requested range
+	 * of data to destination buffer.
+	 */
+	if (IS_ALIGNED(from, 16) && IS_ALIGNED(len, 16) &&
+	    virt_addr_valid(buf)) {
+		ret = cqspi_direct_read_execute(nor, buf, from, len);
+		goto disable_phy;
+	}
+
+	dst = cqspi->dma_bb_rx;
+
+	align_from = ALIGN_DOWN(from, 16);
+	off_delta = from - align_from;
+	readsize = ALIGN(len + off_delta, 16);
+	len_delta = readsize - (len + off_delta);
+
+	xfer_len = min_t(size_t, CQSPI_DMA_BUFFER_SIZE,
+			 readsize);
+
+	ret = cqspi_direct_read_execute(nor, dst, align_from, xfer_len);
+	if (ret)
+		goto disable_phy;
+	copy_len = min_t(size_t, xfer_len, len);
+	memcpy(to, dst + off_delta, copy_len);
+	readsize -= xfer_len;
+	align_from += xfer_len;
+	to += xfer_len;
+
+	while (readsize != 0) {
+		xfer_len = min_t(size_t, CQSPI_DMA_BUFFER_SIZE,
+				 readsize);
+
+		ret = cqspi_direct_read_execute(nor, dst, align_from, xfer_len);
+		if (ret)
+			goto disable_phy;
+		copy_len = min_t(size_t, xfer_len, readsize - len_delta);
+		memcpy(to, dst, copy_len);
+		readsize -= xfer_len;
+		align_from += xfer_len;
+		to += xfer_len;
+	}
+
+disable_phy:
+	if (f_pdata->phy_calibrated)
+		cqspi_phy_dtr_enable(nor, false);
 
 	return ret;
 }
@@ -1026,8 +1314,10 @@
 	if (ret)
 		return ret;
 
-	if (f_pdata->use_direct_mode)
-		ret = cqspi_direct_read_execute(nor, buf, from, len);
+	if (f_pdata->use_direct_mode && f_pdata->phy_calibrated)
+		ret = cqspi_direct_read_execute_phy(nor, buf, from, len);
+	else if (f_pdata->use_direct_mode)
+		ret = cqspi_direct_read_execute(nor, buf, from,	len);
 	else
 		ret = cqspi_indirect_read_execute(nor, buf, from, len);
 	if (ret)
@@ -1080,6 +1370,13 @@
 	int ret;
 
 	ret = cqspi_set_protocol(nor, 0);
+	if (ret)
+		return ret;
+
+	ret = cqspi_read_setup(nor);
+	if (ret)
+		return ret;
+
 	if (!ret)
 		ret = cqspi_command_read(nor, &opcode, 1, buf, len);
 
@@ -1091,9 +1388,22 @@
 	int ret;
 
 	ret = cqspi_set_protocol(nor, 0);
+	if (ret)
+		return ret;
+
+	ret = cqspi_write_setup(nor);
+	if (ret)
+		return ret;
+
+	if (nor->mode == SPI_NOR_MODE_OPI_DTR)
+		cqspi_phy_dtr_enable(nor, true);
+
 	if (!ret)
 		ret = cqspi_command_write(nor, opcode, buf, len);
 
+	if (nor->mode == SPI_NOR_MODE_OPI_DTR)
+		cqspi_phy_dtr_enable(nor, false);
+
 	return ret;
 }
 
@@ -1126,6 +1436,9 @@
 		return -ENXIO;
 	}
 
+	f_pdata->phy_mode = of_property_read_bool(np, "cdns,phy-mode");
+	f_pdata->dqs_en = of_property_read_bool(np, "spi-dqs");
+
 	if (of_property_read_u32(np, "spi-max-frequency", &f_pdata->clk_rate)) {
 		dev_err(&pdev->dev, "couldn't determine spi-max-frequency\n");
 		return -ENXIO;
@@ -1196,38 +1509,71 @@
 	cqspi_controller_enable(cqspi, 1);
 }
 
-static void cqspi_request_mmap_dma(struct cqspi_st *cqspi)
+static int cqspi_request_mmap_dma(struct cqspi_st *cqspi)
 {
+	struct dma_chan *rx_chan;
+
 	dma_cap_mask_t mask;
 
 	dma_cap_zero(mask);
 	dma_cap_set(DMA_MEMCPY, mask);
 
-	cqspi->rx_chan = dma_request_chan_by_mask(&mask);
-	if (IS_ERR(cqspi->rx_chan)) {
+	rx_chan = dma_request_chan_by_mask(&mask);
+	if (PTR_ERR(rx_chan) == -EPROBE_DEFER)
+		return -EPROBE_DEFER;
+	if (IS_ERR(rx_chan)) {
 		dev_err(&cqspi->pdev->dev, "No Rx DMA available\n");
-		cqspi->rx_chan = NULL;
+		return 0;
 	}
+	cqspi->rx_chan = rx_chan;
 	init_completion(&cqspi->rx_dma_complete);
+
+	return 0;
+}
+
+static int  cqspi_config_direct_mode(struct spi_nor *nor)
+{
+	struct cqspi_flash_pdata *f_pdata = nor->priv;
+	struct cqspi_st *cqspi = f_pdata->cqspi;
+	int ret = 0;
+
+	f_pdata->use_direct_mode = true;
+
+	if (!cqspi->rx_chan) {
+		ret = cqspi_request_mmap_dma(cqspi);
+		if (ret)
+			return ret;
+	}
+
+	if (!cqspi->dma_bb_rx) {
+		cqspi->dma_bb_rx = devm_kmalloc(nor->dev,
+						CQSPI_DMA_BUFFER_SIZE,
+						GFP_KERNEL | GFP_DMA);
+		if (!cqspi->dma_bb_rx)
+			return -ENOMEM;
+	}
+	return 0;
 }
 
 static int cqspi_setup_flash(struct cqspi_st *cqspi, struct device_node *np)
 {
-	const struct spi_nor_hwcaps hwcaps = {
-		.mask = SNOR_HWCAPS_READ |
-			SNOR_HWCAPS_READ_FAST |
-			SNOR_HWCAPS_READ_1_1_2 |
-			SNOR_HWCAPS_READ_1_1_4 |
-			SNOR_HWCAPS_PP,
-	};
 	struct platform_device *pdev = cqspi->pdev;
 	struct device *dev = &pdev->dev;
+	const struct cqspi_driver_platdata *ddata;
+	struct spi_nor_hwcaps hwcaps;
 	struct cqspi_flash_pdata *f_pdata;
 	struct spi_nor *nor;
 	struct mtd_info *mtd;
 	unsigned int cs;
 	int i, ret;
 
+	ddata = of_device_get_match_data(dev);
+	if (!ddata) {
+		dev_err(dev, "Couldnt't find driver data\n");
+		return -EINVAL;
+	}
+	hwcaps.mask = ddata->hwcaps_mask;
+
 	/* Get flash device data */
 	for_each_available_child_of_node(dev->of_node, np) {
 		ret = of_property_read_u32(np, "reg", &cs);
@@ -1267,6 +1613,11 @@
 		nor->prepare = cqspi_prep;
 		nor->unprepare = cqspi_unprep;
 
+		if (f_pdata->phy_mode) {
+			nor->calibrate = cqspi_calibrate_phy;
+			f_pdata->use_direct_mode = true;
+		}
+
 		mtd->name = devm_kasprintf(dev, GFP_KERNEL, "%s.%d",
 					   dev_name(dev), cs);
 		if (!mtd->name) {
@@ -1285,12 +1636,11 @@
 		f_pdata->registered = true;
 
 		if (mtd->size <= cqspi->ahb_size) {
-			f_pdata->use_direct_mode = true;
 			dev_dbg(nor->dev, "using direct mode for %s\n",
 				mtd->name);
-
-			if (!cqspi->rx_chan)
-				cqspi_request_mmap_dma(cqspi);
+			ret = cqspi_config_direct_mode(nor);
+			if (ret)
+				goto err;
 		}
 	}
 
@@ -1310,7 +1660,7 @@
 	struct cqspi_st *cqspi;
 	struct resource *res;
 	struct resource *res_ahb;
-	unsigned long data;
+	const struct cqspi_driver_platdata *ddata;
 	int ret;
 	int irq;
 
@@ -1377,8 +1727,8 @@
 	}
 
 	cqspi->master_ref_clk_hz = clk_get_rate(cqspi->clk);
-	data  = (unsigned long)of_device_get_match_data(dev);
-	if (data & CQSPI_NEEDS_WR_DELAY)
+	ddata  = of_device_get_match_data(dev);
+	if (ddata && (ddata->quirks & CQSPI_NEEDS_WR_DELAY))
 		cqspi->wr_delay = 5 * DIV_ROUND_UP(NSEC_PER_SEC,
 						   cqspi->master_ref_clk_hz);
 
@@ -1460,14 +1810,32 @@
 #define CQSPI_DEV_PM_OPS	NULL
 #endif
 
+static const struct cqspi_driver_platdata cdns_qspi = {
+	.hwcaps_mask = CQSPI_BASE_HWCAPS_MASK,
+};
+
+static const struct cqspi_driver_platdata k2g_qspi = {
+	.hwcaps_mask = CQSPI_BASE_HWCAPS_MASK,
+	.quirks = CQSPI_NEEDS_WR_DELAY,
+};
+
+static const struct cqspi_driver_platdata am654_ospi = {
+	.hwcaps_mask = CQSPI_BASE_HWCAPS_MASK | SNOR_HWCAPS_READ_8_8_8,
+	.quirks = CQSPI_NEEDS_WR_DELAY,
+};
+
 static const struct of_device_id cqspi_dt_ids[] = {
 	{
 		.compatible = "cdns,qspi-nor",
-		.data = (void *)0,
+		.data = &cdns_qspi,
 	},
 	{
 		.compatible = "ti,k2g-qspi",
-		.data = (void *)CQSPI_NEEDS_WR_DELAY,
+		.data = &k2g_qspi,
+	},
+	{
+		.compatible = "ti,am654-ospi",
+		.data = &am654_ospi,
 	},
 	{ /* end of table */ }
 };
diff -Naur --no-dereference a/drivers/mtd/spi-nor/spi-nor.c b/drivers/mtd/spi-nor/spi-nor.c
--- a/drivers/mtd/spi-nor/spi-nor.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/mtd/spi-nor/spi-nor.c	2020-06-19 21:00:31.416227715 -0400
@@ -89,8 +89,11 @@
 #define NO_CHIP_ERASE		BIT(12) /* Chip does not support chip erase */
 #define SPI_NOR_SKIP_SFDP	BIT(13)	/* Skip parsing of SFDP tables */
 #define USE_CLSR		BIT(14)	/* use CLSR command */
+#define SPI_NOR_OPI_DTR		BIT(15)	/* Flash supports Octal Read */
 
 	int	(*quad_enable)(struct spi_nor *nor);
+	int	(*change_mode)(struct spi_nor *nor, enum spi_nor_mode mode);
+	void	(*adjust_op)(struct spi_nor *nor, enum spi_nor_mode mode);
 };
 
 #define JEDEC_MFR(info)	((info)->id[0])
@@ -208,10 +211,13 @@
 		{ SPINOR_OP_READ_1_2_2,	SPINOR_OP_READ_1_2_2_4B },
 		{ SPINOR_OP_READ_1_1_4,	SPINOR_OP_READ_1_1_4_4B },
 		{ SPINOR_OP_READ_1_4_4,	SPINOR_OP_READ_1_4_4_4B },
+		{ SPINOR_OP_READ_1_1_8,	SPINOR_OP_READ_1_1_8_4B },
+		{ SPINOR_OP_READ_1_8_8,	SPINOR_OP_READ_1_8_8_4B },
 
 		{ SPINOR_OP_READ_1_1_1_DTR,	SPINOR_OP_READ_1_1_1_DTR_4B },
 		{ SPINOR_OP_READ_1_2_2_DTR,	SPINOR_OP_READ_1_2_2_DTR_4B },
 		{ SPINOR_OP_READ_1_4_4_DTR,	SPINOR_OP_READ_1_4_4_DTR_4B },
+		{ SPINOR_OP_READ_8_8_8_DTR,	SPINOR_OP_READ_8_8_8_DTR_4B },
 	};
 
 	return spi_nor_convert_opcode(opcode, spi_nor_3to4_read,
@@ -224,6 +230,8 @@
 		{ SPINOR_OP_PP,		SPINOR_OP_PP_4B },
 		{ SPINOR_OP_PP_1_1_4,	SPINOR_OP_PP_1_1_4_4B },
 		{ SPINOR_OP_PP_1_4_4,	SPINOR_OP_PP_1_4_4_4B },
+		{ SPINOR_OP_PP_1_1_8,	SPINOR_OP_PP_1_1_8_4B },
+		{ SPINOR_OP_PP_1_8_8,	SPINOR_OP_PP_1_8_8_4B },
 	};
 
 	return spi_nor_convert_opcode(opcode, spi_nor_3to4_program,
@@ -271,6 +279,7 @@
 	u8 cmd;
 
 	switch (JEDEC_MFR(info)) {
+	case SNOR_MFR_ST:
 	case SNOR_MFR_MICRON:
 		/* Some Micron need WREN command; all will accept it */
 		need_wren = true;
@@ -908,6 +917,62 @@
 	return ret;
 }
 
+static void spi_nor_micron_adjust_op(struct spi_nor *nor,
+				     enum spi_nor_mode mode)
+{
+	if (mode == SPI_NOR_MODE_OPI_DTR) {
+		nor->reg_proto = SNOR_PROTO_8_8_8_DTR;
+		nor->read_proto = SNOR_PROTO_8_8_8_DTR;
+		nor->write_proto = SNOR_PROTO_8_8_8_DTR;
+		nor->read_dummy = 16;
+		nor->addr_width = 4;
+		nor->read_opcode = SPINOR_OP_READ_8_8_8_DTR_4B;
+	} else {
+		nor->reg_proto = SNOR_PROTO_1_1_1;
+		nor->read_proto = SNOR_PROTO_1_1_1;
+		nor->write_proto = SNOR_PROTO_1_1_1;
+		nor->read_dummy = 8;
+		nor->addr_width = 4;
+		nor->read_opcode = SPINOR_OP_READ_FAST_4B;
+	}
+}
+
+static int spi_nor_micron_set_octal_ddr_mode(struct spi_nor *nor,
+					     enum spi_nor_mode mode)
+{
+	u8 addr_width, program_opcode, vcr_val;
+	enum spi_nor_protocol write_proto;
+	int ret;
+
+	program_opcode = nor->program_opcode;
+	write_proto  = nor->write_proto;
+	addr_width = nor->addr_width;
+
+	/* Enable Octal DTR mode */
+	nor->program_opcode = SPINOR_OP_WR_VCR;
+
+	if (mode == SPI_NOR_MODE_OPI_DTR) {
+		nor->addr_width = 3;
+		vcr_val = VCR_OCTAL_DDR_EN_MICRON;
+		ret = nor->write(nor, 0x0, sizeof(vcr_val), &vcr_val);
+		if (ret < 0)
+			return ret;
+	} else {
+		nor->write_proto = SNOR_PROTO_8_8_8_DTR;
+		/* Soft reset the flash to return to SPI_NOR_MODE_SPI */
+		nor->write_reg(nor, SPINOR_OP_RESET_EN, NULL, 0);
+		udelay(1);
+		nor->write_reg(nor, SPINOR_OP_RESET_MEM, NULL, 0);
+		udelay(1);
+	}
+
+	nor->program_opcode = program_opcode;
+	nor->write_proto = write_proto;
+	nor->addr_width = addr_width;
+
+	return 0;
+}
+
 static int macronix_quad_enable(struct spi_nor *nor);
 
 /* Used when the "_ext_id" is two bytes at most */
@@ -1096,7 +1161,7 @@
 	{ "mx66l1g45g",  INFO(0xc2201b, 0, 64 * 1024, 2048, SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) },
 	{ "mx66l1g55g",  INFO(0xc2261b, 0, 64 * 1024, 2048, SPI_NOR_QUAD_READ) },
 
-	/* Micron */
+	/* Micron <--> ST Micro */
 	{ "n25q016a",	 INFO(0x20bb15, 0, 64 * 1024,   32, SECT_4K | SPI_NOR_QUAD_READ) },
 	{ "n25q032",	 INFO(0x20ba16, 0, 64 * 1024,   64, SPI_NOR_QUAD_READ) },
 	{ "n25q032a",	 INFO(0x20bb16, 0, 64 * 1024,   64, SPI_NOR_QUAD_READ) },
@@ -1112,6 +1177,15 @@
 	{ "n25q00a",     INFO(0x20bb21, 0, 64 * 1024, 2048, SECT_4K | USE_FSR | SPI_NOR_QUAD_READ | NO_CHIP_ERASE) },
 	{ "mt25qu02g",   INFO(0x20bb22, 0, 64 * 1024, 4096, SECT_4K | USE_FSR | SPI_NOR_QUAD_READ | NO_CHIP_ERASE) },
 
+	/* Micron */
+	{
+		"mt35xu512aba", INFO(0x2c5b1a, 0, 128 * 1024, 512,
+			SECT_4K | USE_FSR | SPI_NOR_OPI_DTR |
+			SPI_NOR_4B_OPCODES)
+			.change_mode = spi_nor_micron_set_octal_ddr_mode,
+			.adjust_op = spi_nor_micron_adjust_op
+	},
+
 	/* PMC */
 	{ "pm25lv512",   INFO(0,        0, 32 * 1024,    2, SECT_4K_PMC) },
 	{ "pm25lv010",   INFO(0,        0, 32 * 1024,    4, SECT_4K_PMC) },
@@ -1290,6 +1364,28 @@
 	return ERR_PTR(-ENODEV);
 }
 
+static int spi_nor_select_mode(struct spi_nor *nor, enum spi_nor_mode mode)
+{
+	int ret;
+
+	if (nor->mode == mode)
+		return 0;
+
+	if (!nor->change_mode)
+		return -ENOTSUPP;
+
+	ret = nor->change_mode(nor, mode);
+	if (ret)
+		return ret;
+
+	if (nor->adjust_op)
+		nor->adjust_op(nor, mode);
+
+	nor->mode = mode;
+
+	return ret;
+}
+
 static int spi_nor_read(struct mtd_info *mtd, loff_t from, size_t len,
 			size_t *retlen, u_char *buf)
 {
@@ -1302,6 +1398,8 @@
 	if (ret)
 		return ret;
 
+	spi_nor_select_mode(nor, nor->preferred_mode);
+
 	while (len) {
 		loff_t addr = from;
 
@@ -1326,6 +1424,8 @@
 	ret = 0;
 
 read_err:
+	spi_nor_select_mode(nor, SPI_NOR_MODE_SPI);
+
 	spi_nor_unlock_and_unprep(nor, SPI_NOR_OPS_READ);
 	return ret;
 }
@@ -1806,6 +1906,7 @@
 	SNOR_CMD_READ_1_8_8,
 	SNOR_CMD_READ_8_8_8,
 	SNOR_CMD_READ_1_8_8_DTR,
+	SNOR_CMD_READ_8_8_8_DTR,
 
 	SNOR_CMD_READ_MAX
 };
@@ -1890,6 +1991,8 @@
 	nor->addr_width = 3;
 	nor->read_dummy = 8;
 
+	nor->read_proto = SNOR_PROTO_1_1_1;
+
 	while (len) {
 		ret = nor->read(nor, addr, len, (u8 *)buf);
 		if (!ret || ret > len) {
@@ -2489,6 +2592,13 @@
 					  SNOR_PROTO_1_1_4);
 	}
 
+	if (info->flags & SPI_NOR_OPI_DTR) {
+		params->hwcaps.mask |= SNOR_HWCAPS_READ_8_8_8_DTR;
+		spi_nor_set_read_settings(&params->reads[SNOR_CMD_READ_8_8_8_DTR],
+					  0, 16, SPINOR_OP_READ_8_8_8_DTR,
+					  SNOR_PROTO_8_8_8_DTR);
+	}
+
 	/* Page Program settings. */
 	params->hwcaps.mask |= SNOR_HWCAPS_PP;
 	spi_nor_set_pp_settings(&params->page_programs[SNOR_CMD_PP],
@@ -2502,6 +2612,7 @@
 			params->quad_enable = macronix_quad_enable;
 			break;
 
+		case SNOR_MFR_ST:
 		case SNOR_MFR_MICRON:
 			break;
 
@@ -2569,6 +2680,7 @@
 		{ SNOR_HWCAPS_READ_1_8_8,	SNOR_CMD_READ_1_8_8 },
 		{ SNOR_HWCAPS_READ_8_8_8,	SNOR_CMD_READ_8_8_8 },
 		{ SNOR_HWCAPS_READ_1_8_8_DTR,	SNOR_CMD_READ_1_8_8_DTR },
+		{ SNOR_HWCAPS_READ_8_8_8_DTR,	SNOR_CMD_READ_8_8_8_DTR },
 	};
 
 	return spi_nor_hwcaps2cmd(hwcaps, hwcaps_read2cmd,
@@ -2669,6 +2781,43 @@
 	return 0;
 }
 
+/* Calibrate controller for given mode */
+static int spi_nor_calibrate(struct spi_nor *nor, enum spi_nor_mode mode)
+{
+	/* Use first 16 bytes of SFDP Header as calibration data */
+	int size = sizeof(struct sfdp_parameter_header) << 1;
+	u8 addr_width = nor->addr_width;
+	u8 *calibration_data;
+	int ret;
+
+	if (!nor->calibrate)
+		return 0;
+
+	calibration_data = kmalloc(size, GFP_KERNEL);
+	if (!calibration_data)
+		return -ENOMEM;
+
+	ret = spi_nor_read_sfdp(nor, 0, size, calibration_data);
+	if (ret)
+		goto free_mem;
+
+	nor->preferred_mode = mode;
+	spi_nor_select_mode(nor, nor->preferred_mode);
+
+	nor->addr_width = 4;
+	nor->read_opcode = SPINOR_OP_RDSFDP;
+	nor->read_dummy = 8;
+
+	ret = nor->calibrate(nor, calibration_data, size);
+
+	spi_nor_select_mode(nor, SPI_NOR_MODE_SPI);
+	nor->addr_width = addr_width;
+free_mem:
+	kfree(calibration_data);
+
+	return ret;
+}
+
 static int spi_nor_setup(struct spi_nor *nor, const struct flash_info *info,
 			 const struct spi_nor_flash_parameter *params,
 			 const struct spi_nor_hwcaps *hwcaps)
@@ -2686,7 +2835,6 @@
 	/* SPI n-n-n protocols are not supported yet. */
 	ignored_mask = (SNOR_HWCAPS_READ_2_2_2 |
 			SNOR_HWCAPS_READ_4_4_4 |
-			SNOR_HWCAPS_READ_8_8_8 |
 			SNOR_HWCAPS_PP_4_4_4 |
 			SNOR_HWCAPS_PP_8_8_8);
 	if (shared_mask & ignored_mask) {
@@ -2695,6 +2843,9 @@
 		shared_mask &= ~ignored_mask;
 	}
 
+	nor->change_mode = info->change_mode;
+	nor->adjust_op = info->adjust_op;
+
 	/* Select the (Fast) Read command. */
 	err = spi_nor_select_read(nor, params, shared_mask);
 	if (err) {
@@ -2719,6 +2870,14 @@
 		return err;
 	}
 
+	if (info->flags & SPI_NOR_OPI_DTR) {
+		err = spi_nor_calibrate(nor, SPI_NOR_MODE_OPI_DTR);
+		if (err) {
+			dev_err(nor->dev, "Controller calibration failed\n");
+			return err;
+		}
+	}
+
 	/* Enable Quad I/O if needed. */
 	enable_quad_io = (spi_nor_get_protocol_width(nor->read_proto) == 4 ||
 			  spi_nor_get_protocol_width(nor->write_proto) == 4);
@@ -2876,8 +3035,9 @@
 	mtd->_resume = spi_nor_resume;
 
 	/* NOR protection support for STmicro/Micron chips and similar */
-	if (JEDEC_MFR(info) == SNOR_MFR_MICRON ||
-			info->flags & SPI_NOR_HAS_LOCK) {
+	if (JEDEC_MFR(info) == SNOR_MFR_ST ||
+	    JEDEC_MFR(info) == SNOR_MFR_MICRON ||
+	    info->flags & SPI_NOR_HAS_LOCK) {
 		nor->flash_lock = stm_lock;
 		nor->flash_unlock = stm_unlock;
 		nor->flash_is_locked = stm_is_locked;
diff -Naur --no-dereference a/drivers/net/can/m_can/m_can.c b/drivers/net/can/m_can/m_can.c
--- a/drivers/net/can/m_can/m_can.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/can/m_can/m_can.c	2020-06-19 21:00:31.416227715 -0400
@@ -15,6 +15,7 @@
 
 #include <linux/clk.h>
 #include <linux/delay.h>
+#include <linux/gpio/consumer.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/kernel.h>
@@ -1601,6 +1602,7 @@
 	struct clk *hclk, *cclk;
 	int irq, ret;
 	struct device_node *np;
+	struct gpio_desc *stb;
 	u32 mram_config_vals[MRAM_CFG_LEN];
 	u32 tx_fifo_size;
 
@@ -1681,6 +1683,15 @@
 	if (ret)
 		goto clk_disable;
 
+	stb = devm_gpiod_get_optional(&pdev->dev, "stb", GPIOD_OUT_HIGH);
+	if (IS_ERR(stb)) {
+		ret = PTR_ERR(stb);
+		if (ret != -EPROBE_DEFER)
+			netdev_err(dev, "gpio request failed, ret %d\n", ret);
+
+		goto clk_disable;
+	}
+
 	ret = register_m_can_dev(dev);
 	if (ret) {
 		dev_err(&pdev->dev, "registering %s failed (err=%d)\n",
diff -Naur --no-dereference a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
--- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c	2020-06-19 21:00:31.416227715 -0400
@@ -1142,28 +1142,13 @@
  */
 static int mvpp22_comphy_init(struct mvpp2_port *port)
 {
-	enum phy_mode mode;
 	int ret;
 
 	if (!port->comphy)
 		return 0;
 
-	switch (port->phy_interface) {
-	case PHY_INTERFACE_MODE_SGMII:
-	case PHY_INTERFACE_MODE_1000BASEX:
-		mode = PHY_MODE_SGMII;
-		break;
-	case PHY_INTERFACE_MODE_2500BASEX:
-		mode = PHY_MODE_2500SGMII;
-		break;
-	case PHY_INTERFACE_MODE_10GKR:
-		mode = PHY_MODE_10GKR;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	ret = phy_set_mode(port->comphy, mode);
+	ret = phy_set_mode_ext(port->comphy, PHY_MODE_ETHERNET,
+			       port->phy_interface);
 	if (ret)
 		return ret;
 
diff -Naur --no-dereference a/drivers/net/ethernet/ti/am65-cpsw-ethtool.c b/drivers/net/ethernet/ti/am65-cpsw-ethtool.c
--- a/drivers/net/ethernet/ti/am65-cpsw-ethtool.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/net/ethernet/ti/am65-cpsw-ethtool.c	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,772 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Texas Instruments K3 AM65 Ethernet Switch SubSystem Driver ethtool ops
+ *
+ * Copyright (C) 2016-2018 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ */
+
+#include <linux/net_tstamp.h>
+#include <linux/phy.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+
+#include "am65-cpsw-nuss.h"
+#include "cpsw_ale.h"
+#include "am65-cpts.h"
+
+#define AM65_CPSW_DRV_VER "0.1"
+
+#define AM65_CPSW_REGDUMP_VER 0x1
+
+enum {
+	AM65_CPSW_REGDUMP_MOD_NUSS = 1,
+	AM65_CPSW_REGDUMP_MOD_RGMII_STATUS = 2,
+	AM65_CPSW_REGDUMP_MOD_MDIO = 3,
+	AM65_CPSW_REGDUMP_MOD_CPSW = 4,
+	AM65_CPSW_REGDUMP_MOD_CPSW_P0 = 5,
+	AM65_CPSW_REGDUMP_MOD_CPSW_P1 = 6,
+	AM65_CPSW_REGDUMP_MOD_CPSW_CPTS = 7,
+	AM65_CPSW_REGDUMP_MOD_CPSW_ALE = 8,
+	AM65_CPSW_REGDUMP_MOD_CPSW_ALE_TBL = 9,
+	AM65_CPSW_REGDUMP_MOD_LAST,
+};
+
+struct am65_cpsw_regdump_hdr {
+	u32 module_id;
+	u32 len;
+} __packed;
+
+struct am65_cpsw_regdump_item {
+	struct am65_cpsw_regdump_hdr hdr;
+	u32 start_ofs;
+	u32 end_ofs;
+};
+
+#define AM65_CPSW_REGDUMP_REC(mod, start, end) { \
+	.hdr.module_id = (mod), \
+	.hdr.len = (((u32 *)(end)) - ((u32 *)(start)) + 1) * sizeof(u32) * 2 + \
+		   sizeof(struct am65_cpsw_regdump_hdr), \
+	.start_ofs = start, \
+	.end_ofs = end, \
+}
+
+static const struct am65_cpsw_regdump_item am65_cpsw_regdump[] = {
+	AM65_CPSW_REGDUMP_REC(AM65_CPSW_REGDUMP_MOD_NUSS, 0x0, 0x1c),
+	AM65_CPSW_REGDUMP_REC(AM65_CPSW_REGDUMP_MOD_RGMII_STATUS, 0x30, 0x4c),
+	AM65_CPSW_REGDUMP_REC(AM65_CPSW_REGDUMP_MOD_MDIO, 0xf00, 0xffc),
+	AM65_CPSW_REGDUMP_REC(AM65_CPSW_REGDUMP_MOD_CPSW, 0x20000, 0x2011c),
+	AM65_CPSW_REGDUMP_REC(AM65_CPSW_REGDUMP_MOD_CPSW_P0, 0x21000, 0x21320),
+	AM65_CPSW_REGDUMP_REC(AM65_CPSW_REGDUMP_MOD_CPSW_P1, 0x22000, 0x223a4),
+	AM65_CPSW_REGDUMP_REC(AM65_CPSW_REGDUMP_MOD_CPSW_CPTS, 0x3d000, 0x3d048),
+	AM65_CPSW_REGDUMP_REC(AM65_CPSW_REGDUMP_MOD_CPSW_ALE, 0x3e000, 0x3e13c),
+	AM65_CPSW_REGDUMP_REC(AM65_CPSW_REGDUMP_MOD_CPSW_ALE_TBL, 0, 0),
+};
+
+struct am65_cpsw_stats_regs {
+	u32	rx_good_frames;
+	u32	rx_broadcast_frames;
+	u32	rx_multicast_frames;
+	u32	rx_pause_frames;		/* slave */
+	u32	rx_crc_errors;
+	u32	rx_align_code_errors;		/* slave */
+	u32	rx_oversized_frames;
+	u32	rx_jabber_frames;		/* slave */
+	u32	rx_undersized_frames;
+	u32	rx_fragments;			/* slave */
+	u32	ale_drop;
+	u32	ale_overrun_drop;
+	u32	rx_octets;
+	u32	tx_good_frames;
+	u32	tx_broadcast_frames;
+	u32	tx_multicast_frames;
+	u32	tx_pause_frames;		/* slave */
+	u32	tx_deferred_frames;		/* slave */
+	u32	tx_collision_frames;		/* slave */
+	u32	tx_single_coll_frames;		/* slave */
+	u32	tx_mult_coll_frames;		/* slave */
+	u32	tx_excessive_collisions;	/* slave */
+	u32	tx_late_collisions;		/* slave */
+	u32	rx_ipg_error;			/* slave 10G only */
+	u32	tx_carrier_sense_errors;	/* slave */
+	u32	tx_octets;
+	u32	tx_64B_frames;
+	u32	tx_65_to_127B_frames;
+	u32	tx_128_to_255B_frames;
+	u32	tx_256_to_511B_frames;
+	u32	tx_512_to_1023B_frames;
+	u32	tx_1024B_frames;
+	u32	net_octets;
+	u32	rx_bottom_fifo_drop;
+	u32	rx_port_mask_drop;
+	u32	rx_top_fifo_drop;
+	u32	ale_rate_limit_drop;
+	u32	ale_vid_ingress_drop;
+	u32	ale_da_eq_sa_drop;
+	u32	ale_block_drop;			/* K3 */
+	u32	ale_secure_drop;		/* K3 */
+	u32	ale_auth_drop;			/* K3 */
+	u32	ale_unknown_ucast;
+	u32	ale_unknown_ucast_bytes;
+	u32	ale_unknown_mcast;
+	u32	ale_unknown_mcast_bytes;
+	u32	ale_unknown_bcast;
+	u32	ale_unknown_bcast_bytes;
+	u32	ale_pol_match;
+	u32	ale_pol_match_red;
+	u32	ale_pol_match_yellow;
+	u32	ale_mcast_sa_drop;		/* K3 */
+	u32	ale_dual_vlan_drop;		/* K3 */
+	u32	ale_len_err_drop;		/* K3 */
+	u32	ale_ip_next_hdr_drop;		/* K3 */
+	u32	ale_ipv4_frag_drop;		/* K3 */
+	u32	__rsvd_1[24];
+	u32	iet_rx_assembly_err;		/* K3 slave */
+	u32	iet_rx_assembly_ok;		/* K3 slave */
+	u32	iet_rx_smd_err;			/* K3 slave */
+	u32	iet_rx_frag;			/* K3 slave */
+	u32	iet_tx_hold;			/* K3 slave */
+	u32	iet_tx_frag;			/* K3 slave */
+	u32	__rsvd_2[9];
+	u32	tx_mem_protect_err;
+	/* following NU only */
+	u32	tx_pri0;
+	u32	tx_pri1;
+	u32	tx_pri2;
+	u32	tx_pri3;
+	u32	tx_pri4;
+	u32	tx_pri5;
+	u32	tx_pri6;
+	u32	tx_pri7;
+	u32	tx_pri0_bcnt;
+	u32	tx_pri1_bcnt;
+	u32	tx_pri2_bcnt;
+	u32	tx_pri3_bcnt;
+	u32	tx_pri4_bcnt;
+	u32	tx_pri5_bcnt;
+	u32	tx_pri6_bcnt;
+	u32	tx_pri7_bcnt;
+	u32	tx_pri0_drop;
+	u32	tx_pri1_drop;
+	u32	tx_pri2_drop;
+	u32	tx_pri3_drop;
+	u32	tx_pri4_drop;
+	u32	tx_pri5_drop;
+	u32	tx_pri6_drop;
+	u32	tx_pri7_drop;
+	u32	tx_pri0_drop_bcnt;
+	u32	tx_pri1_drop_bcnt;
+	u32	tx_pri2_drop_bcnt;
+	u32	tx_pri3_drop_bcnt;
+	u32	tx_pri4_drop_bcnt;
+	u32	tx_pri5_drop_bcnt;
+	u32	tx_pri6_drop_bcnt;
+	u32	tx_pri7_drop_bcnt;
+};
+
+struct am65_cpsw_ethtool_stat {
+	char desc[ETH_GSTRING_LEN];
+	int offset;
+};
+
+#define AM65_CPSW_STATS(prefix, field)			\
+{							\
+	#prefix#field,					\
+	offsetof(struct am65_cpsw_stats_regs, field)	\
+}
+
+static const struct am65_cpsw_ethtool_stat am65_host_stats[] = {
+	AM65_CPSW_STATS(p0_, rx_good_frames),
+	AM65_CPSW_STATS(p0_, rx_broadcast_frames),
+	AM65_CPSW_STATS(p0_, rx_multicast_frames),
+	AM65_CPSW_STATS(p0_, rx_crc_errors),
+	AM65_CPSW_STATS(p0_, rx_oversized_frames),
+	AM65_CPSW_STATS(p0_, rx_undersized_frames),
+	AM65_CPSW_STATS(p0_, ale_drop),
+	AM65_CPSW_STATS(p0_, ale_overrun_drop),
+	AM65_CPSW_STATS(p0_, rx_octets),
+	AM65_CPSW_STATS(p0_, tx_good_frames),
+	AM65_CPSW_STATS(p0_, tx_broadcast_frames),
+	AM65_CPSW_STATS(p0_, tx_multicast_frames),
+	AM65_CPSW_STATS(p0_, tx_octets),
+	AM65_CPSW_STATS(p0_, tx_64B_frames),
+	AM65_CPSW_STATS(p0_, tx_65_to_127B_frames),
+	AM65_CPSW_STATS(p0_, tx_128_to_255B_frames),
+	AM65_CPSW_STATS(p0_, tx_256_to_511B_frames),
+	AM65_CPSW_STATS(p0_, tx_512_to_1023B_frames),
+	AM65_CPSW_STATS(p0_, tx_1024B_frames),
+	AM65_CPSW_STATS(p0_, net_octets),
+	AM65_CPSW_STATS(p0_, rx_bottom_fifo_drop),
+	AM65_CPSW_STATS(p0_, rx_port_mask_drop),
+	AM65_CPSW_STATS(p0_, rx_top_fifo_drop),
+	AM65_CPSW_STATS(p0_, ale_rate_limit_drop),
+	AM65_CPSW_STATS(p0_, ale_vid_ingress_drop),
+	AM65_CPSW_STATS(p0_, ale_da_eq_sa_drop),
+	AM65_CPSW_STATS(p0_, ale_block_drop),
+	AM65_CPSW_STATS(p0_, ale_secure_drop),
+	AM65_CPSW_STATS(p0_, ale_auth_drop),
+	AM65_CPSW_STATS(p0_, ale_unknown_ucast),
+	AM65_CPSW_STATS(p0_, ale_unknown_ucast_bytes),
+	AM65_CPSW_STATS(p0_, ale_unknown_mcast),
+	AM65_CPSW_STATS(p0_, ale_unknown_mcast_bytes),
+	AM65_CPSW_STATS(p0_, ale_unknown_bcast),
+	AM65_CPSW_STATS(p0_, ale_unknown_bcast_bytes),
+	AM65_CPSW_STATS(p0_, ale_pol_match),
+	AM65_CPSW_STATS(p0_, ale_pol_match_red),
+	AM65_CPSW_STATS(p0_, ale_pol_match_yellow),
+	AM65_CPSW_STATS(p0_, ale_mcast_sa_drop),
+	AM65_CPSW_STATS(p0_, ale_dual_vlan_drop),
+	AM65_CPSW_STATS(p0_, ale_len_err_drop),
+	AM65_CPSW_STATS(p0_, ale_ip_next_hdr_drop),
+	AM65_CPSW_STATS(p0_, ale_ipv4_frag_drop),
+	AM65_CPSW_STATS(p0_, tx_mem_protect_err),
+	AM65_CPSW_STATS(p0_, tx_pri0),
+	AM65_CPSW_STATS(p0_, tx_pri1),
+	AM65_CPSW_STATS(p0_, tx_pri2),
+	AM65_CPSW_STATS(p0_, tx_pri3),
+	AM65_CPSW_STATS(p0_, tx_pri4),
+	AM65_CPSW_STATS(p0_, tx_pri5),
+	AM65_CPSW_STATS(p0_, tx_pri6),
+	AM65_CPSW_STATS(p0_, tx_pri7),
+	AM65_CPSW_STATS(p0_, tx_pri0_bcnt),
+	AM65_CPSW_STATS(p0_, tx_pri1_bcnt),
+	AM65_CPSW_STATS(p0_, tx_pri2_bcnt),
+	AM65_CPSW_STATS(p0_, tx_pri3_bcnt),
+	AM65_CPSW_STATS(p0_, tx_pri4_bcnt),
+	AM65_CPSW_STATS(p0_, tx_pri5_bcnt),
+	AM65_CPSW_STATS(p0_, tx_pri6_bcnt),
+	AM65_CPSW_STATS(p0_, tx_pri7_bcnt),
+	AM65_CPSW_STATS(p0_, tx_pri0_drop),
+	AM65_CPSW_STATS(p0_, tx_pri1_drop),
+	AM65_CPSW_STATS(p0_, tx_pri2_drop),
+	AM65_CPSW_STATS(p0_, tx_pri3_drop),
+	AM65_CPSW_STATS(p0_, tx_pri4_drop),
+	AM65_CPSW_STATS(p0_, tx_pri5_drop),
+	AM65_CPSW_STATS(p0_, tx_pri6_drop),
+	AM65_CPSW_STATS(p0_, tx_pri7_drop),
+	AM65_CPSW_STATS(p0_, tx_pri0_drop_bcnt),
+	AM65_CPSW_STATS(p0_, tx_pri1_drop_bcnt),
+	AM65_CPSW_STATS(p0_, tx_pri2_drop_bcnt),
+	AM65_CPSW_STATS(p0_, tx_pri3_drop_bcnt),
+	AM65_CPSW_STATS(p0_, tx_pri4_drop_bcnt),
+	AM65_CPSW_STATS(p0_, tx_pri5_drop_bcnt),
+	AM65_CPSW_STATS(p0_, tx_pri6_drop_bcnt),
+	AM65_CPSW_STATS(p0_, tx_pri7_drop_bcnt),
+};
+
+static const struct am65_cpsw_ethtool_stat am65_slave_stats[] = {
+	AM65_CPSW_STATS(, rx_good_frames),
+	AM65_CPSW_STATS(, rx_broadcast_frames),
+	AM65_CPSW_STATS(, rx_multicast_frames),
+	AM65_CPSW_STATS(, rx_pause_frames),
+	AM65_CPSW_STATS(, rx_crc_errors),
+	AM65_CPSW_STATS(, rx_align_code_errors),
+	AM65_CPSW_STATS(, rx_oversized_frames),
+	AM65_CPSW_STATS(, rx_jabber_frames),
+	AM65_CPSW_STATS(, rx_undersized_frames),
+	AM65_CPSW_STATS(, rx_fragments),
+	AM65_CPSW_STATS(, ale_drop),
+	AM65_CPSW_STATS(, ale_overrun_drop),
+	AM65_CPSW_STATS(, rx_octets),
+	AM65_CPSW_STATS(, tx_good_frames),
+	AM65_CPSW_STATS(, tx_broadcast_frames),
+	AM65_CPSW_STATS(, tx_multicast_frames),
+	AM65_CPSW_STATS(, tx_pause_frames),
+	AM65_CPSW_STATS(, tx_deferred_frames),
+	AM65_CPSW_STATS(, tx_collision_frames),
+	AM65_CPSW_STATS(, tx_single_coll_frames),
+	AM65_CPSW_STATS(, tx_mult_coll_frames),
+	AM65_CPSW_STATS(, tx_excessive_collisions),
+	AM65_CPSW_STATS(, tx_late_collisions),
+	AM65_CPSW_STATS(, rx_ipg_error),
+	AM65_CPSW_STATS(, tx_carrier_sense_errors),
+	AM65_CPSW_STATS(, tx_octets),
+	AM65_CPSW_STATS(, tx_64B_frames),
+	AM65_CPSW_STATS(, tx_65_to_127B_frames),
+	AM65_CPSW_STATS(, tx_128_to_255B_frames),
+	AM65_CPSW_STATS(, tx_256_to_511B_frames),
+	AM65_CPSW_STATS(, tx_512_to_1023B_frames),
+	AM65_CPSW_STATS(, tx_1024B_frames),
+	AM65_CPSW_STATS(, net_octets),
+	AM65_CPSW_STATS(, rx_bottom_fifo_drop),
+	AM65_CPSW_STATS(, rx_port_mask_drop),
+	AM65_CPSW_STATS(, rx_top_fifo_drop),
+	AM65_CPSW_STATS(, ale_rate_limit_drop),
+	AM65_CPSW_STATS(, ale_vid_ingress_drop),
+	AM65_CPSW_STATS(, ale_da_eq_sa_drop),
+	AM65_CPSW_STATS(, ale_block_drop),
+	AM65_CPSW_STATS(, ale_secure_drop),
+	AM65_CPSW_STATS(, ale_auth_drop),
+	AM65_CPSW_STATS(, ale_unknown_ucast),
+	AM65_CPSW_STATS(, ale_unknown_ucast_bytes),
+	AM65_CPSW_STATS(, ale_unknown_mcast),
+	AM65_CPSW_STATS(, ale_unknown_mcast_bytes),
+	AM65_CPSW_STATS(, ale_unknown_bcast),
+	AM65_CPSW_STATS(, ale_unknown_bcast_bytes),
+	AM65_CPSW_STATS(, ale_pol_match),
+	AM65_CPSW_STATS(, ale_pol_match_red),
+	AM65_CPSW_STATS(, ale_pol_match_yellow),
+	AM65_CPSW_STATS(, ale_mcast_sa_drop),
+	AM65_CPSW_STATS(, ale_dual_vlan_drop),
+	AM65_CPSW_STATS(, ale_len_err_drop),
+	AM65_CPSW_STATS(, ale_ip_next_hdr_drop),
+	AM65_CPSW_STATS(, ale_ipv4_frag_drop),
+	AM65_CPSW_STATS(, iet_rx_assembly_err),
+	AM65_CPSW_STATS(, iet_rx_assembly_ok),
+	AM65_CPSW_STATS(, iet_rx_smd_err),
+	AM65_CPSW_STATS(, iet_rx_frag),
+	AM65_CPSW_STATS(, iet_tx_hold),
+	AM65_CPSW_STATS(, iet_tx_frag),
+	AM65_CPSW_STATS(, tx_mem_protect_err),
+	AM65_CPSW_STATS(, tx_pri0),
+	AM65_CPSW_STATS(, tx_pri1),
+	AM65_CPSW_STATS(, tx_pri2),
+	AM65_CPSW_STATS(, tx_pri3),
+	AM65_CPSW_STATS(, tx_pri4),
+	AM65_CPSW_STATS(, tx_pri5),
+	AM65_CPSW_STATS(, tx_pri6),
+	AM65_CPSW_STATS(, tx_pri7),
+	AM65_CPSW_STATS(, tx_pri0_bcnt),
+	AM65_CPSW_STATS(, tx_pri1_bcnt),
+	AM65_CPSW_STATS(, tx_pri2_bcnt),
+	AM65_CPSW_STATS(, tx_pri3_bcnt),
+	AM65_CPSW_STATS(, tx_pri4_bcnt),
+	AM65_CPSW_STATS(, tx_pri5_bcnt),
+	AM65_CPSW_STATS(, tx_pri6_bcnt),
+	AM65_CPSW_STATS(, tx_pri7_bcnt),
+	AM65_CPSW_STATS(, tx_pri0_drop),
+	AM65_CPSW_STATS(, tx_pri1_drop),
+	AM65_CPSW_STATS(, tx_pri2_drop),
+	AM65_CPSW_STATS(, tx_pri3_drop),
+	AM65_CPSW_STATS(, tx_pri4_drop),
+	AM65_CPSW_STATS(, tx_pri5_drop),
+	AM65_CPSW_STATS(, tx_pri6_drop),
+	AM65_CPSW_STATS(, tx_pri7_drop),
+	AM65_CPSW_STATS(, tx_pri0_drop_bcnt),
+	AM65_CPSW_STATS(, tx_pri1_drop_bcnt),
+	AM65_CPSW_STATS(, tx_pri2_drop_bcnt),
+	AM65_CPSW_STATS(, tx_pri3_drop_bcnt),
+	AM65_CPSW_STATS(, tx_pri4_drop_bcnt),
+	AM65_CPSW_STATS(, tx_pri5_drop_bcnt),
+	AM65_CPSW_STATS(, tx_pri6_drop_bcnt),
+	AM65_CPSW_STATS(, tx_pri7_drop_bcnt),
+};
+
+/* Ethtool priv_flags */
+static const char am65_cpsw_ethtool_priv_flags[][ETH_GSTRING_LEN] = {
+#define	AM65_CPSW_PRIV_P0_RX_PTYPE_RROBIN	BIT(0)
+	"p0-rx-ptype-rrobin",
+};
+
+static int am65_cpsw_ethtool_op_begin(struct net_device *ndev)
+{
+	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
+	int ret;
+
+	ret = pm_runtime_get_sync(common->dev);
+	if (ret < 0) {
+		dev_err(common->dev, "ethtool begin failed %d\n", ret);
+		pm_runtime_put_noidle(common->dev);
+	}
+
+	return ret;
+}
+
+static void am65_cpsw_ethtool_op_complete(struct net_device *ndev)
+{
+	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
+	int ret;
+
+	ret = pm_runtime_put(common->dev);
+	if (ret < 0)
+		dev_err(common->dev, "ethtool complete failed %d\n", ret);
+}
+
+static void am65_cpsw_get_drvinfo(struct net_device *ndev,
+				  struct ethtool_drvinfo *info)
+{
+	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
+
+	strlcpy(info->driver, dev_driver_string(common->dev),
+		sizeof(info->driver));
+	strlcpy(info->version, AM65_CPSW_DRV_VER, sizeof(info->version));
+	strlcpy(info->bus_info, dev_name(common->dev), sizeof(info->bus_info));
+}
+
+static u32 am65_cpsw_get_msglevel(struct net_device *ndev)
+{
+	struct am65_cpsw_ndev_priv *priv = am65_ndev_to_priv(ndev);
+
+	return priv->msg_enable;
+}
+
+static void am65_cpsw_set_msglevel(struct net_device *ndev, u32 value)
+{
+	struct am65_cpsw_ndev_priv *priv = am65_ndev_to_priv(ndev);
+
+	priv->msg_enable = value;
+}
+
+static void am65_cpsw_get_channels(struct net_device *ndev,
+				   struct ethtool_channels *ch)
+{
+	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
+
+	ch->max_combined = 0;
+	ch->max_rx = AM65_CPSW_MAX_RX_QUEUES;
+	ch->max_tx = AM65_CPSW_MAX_TX_QUEUES;
+	ch->max_other = 0;
+	ch->other_count = 0;
+	ch->rx_count = common->rx_ch_num;
+	ch->tx_count = common->tx_ch_num;
+	ch->combined_count = 0;
+}
+
+static void am65_cpsw_get_ringparam(struct net_device *ndev,
+				    struct ethtool_ringparam *ering)
+{
+	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
+
+	/* not supported */
+	ering->tx_max_pending = 0;
+	ering->tx_pending = common->tx_chns[0].descs_num;
+	ering->rx_max_pending = 0;
+	ering->rx_pending = common->rx_chns.descs_num;
+}
+
+static void am65_cpsw_get_pauseparam(struct net_device *ndev,
+				     struct ethtool_pauseparam *pause)
+{
+	struct am65_cpsw_slave_data *salve = am65_ndev_to_slave(ndev);
+
+	pause->autoneg = AUTONEG_DISABLE;
+	pause->rx_pause = salve->rx_pause ? true : false;
+	pause->tx_pause = salve->tx_pause ? true : false;
+}
+
+static int am65_cpsw_set_pauseparam(struct net_device *ndev,
+				    struct ethtool_pauseparam *pause)
+{
+	struct am65_cpsw_slave_data *salve = am65_ndev_to_slave(ndev);
+
+	salve->rx_pause = pause->rx_pause ? true : false;
+	salve->tx_pause = pause->tx_pause ? true : false;
+
+	am65_cpsw_nuss_adjust_link(ndev);
+	return 0;
+}
+
+static void am65_cpsw_get_wol(struct net_device *ndev,
+			      struct ethtool_wolinfo *wol)
+{
+	struct am65_cpsw_slave_data *salve = am65_ndev_to_slave(ndev);
+
+	wol->supported = 0;
+	wol->wolopts = 0;
+
+	if (salve->phy)
+		phy_ethtool_get_wol(salve->phy, wol);
+}
+
+static int am65_cpsw_set_wol(struct net_device *ndev,
+			     struct ethtool_wolinfo *wol)
+{
+	struct am65_cpsw_slave_data *salve = am65_ndev_to_slave(ndev);
+
+	if (!salve->phy)
+		return -EOPNOTSUPP;
+
+	return phy_ethtool_set_wol(salve->phy, wol);
+}
+
+static int am65_cpsw_get_link_ksettings(struct net_device *ndev,
+					struct ethtool_link_ksettings *ecmd)
+{
+	struct am65_cpsw_slave_data *salve = am65_ndev_to_slave(ndev);
+
+	if (!salve->phy)
+		return -EOPNOTSUPP;
+
+	phy_ethtool_ksettings_get(salve->phy, ecmd);
+	return 0;
+}
+
+static int am65_cpsw_set_link_ksettings(
+		struct net_device *ndev,
+		const struct ethtool_link_ksettings *ecmd)
+{
+	struct am65_cpsw_slave_data *salve = am65_ndev_to_slave(ndev);
+
+	if (!salve->phy || phy_is_pseudo_fixed_link(salve->phy))
+		return -EOPNOTSUPP;
+
+	return phy_ethtool_ksettings_set(salve->phy, ecmd);
+}
+
+static int am65_cpsw_get_eee(struct net_device *ndev, struct ethtool_eee *edata)
+{
+	struct am65_cpsw_slave_data *salve = am65_ndev_to_slave(ndev);
+
+	if (!salve->phy || phy_is_pseudo_fixed_link(salve->phy))
+		return -EOPNOTSUPP;
+
+	return phy_ethtool_get_eee(salve->phy, edata);
+}
+
+static int am65_cpsw_set_eee(struct net_device *ndev, struct ethtool_eee *edata)
+{
+	struct am65_cpsw_slave_data *salve = am65_ndev_to_slave(ndev);
+
+	if (!salve->phy || phy_is_pseudo_fixed_link(salve->phy))
+		return -EOPNOTSUPP;
+
+	return phy_ethtool_set_eee(salve->phy, edata);
+}
+
+static int am65_cpsw_nway_reset(struct net_device *ndev)
+{
+	struct am65_cpsw_slave_data *salve = am65_ndev_to_slave(ndev);
+
+	if (!salve->phy || phy_is_pseudo_fixed_link(salve->phy))
+		return -EOPNOTSUPP;
+
+	return genphy_restart_aneg(salve->phy);
+}
+
+static int am65_cpsw_get_regs_len(struct net_device *ndev)
+{
+	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
+	u32 i, regdump_len = 0;
+
+	for (i = 0; i < ARRAY_SIZE(am65_cpsw_regdump); i++) {
+		if (am65_cpsw_regdump[i].hdr.module_id ==
+		    AM65_CPSW_REGDUMP_MOD_CPSW_ALE_TBL) {
+			regdump_len += sizeof(struct am65_cpsw_regdump_hdr);
+			regdump_len += common->ale->params.ale_entries *
+				       ALE_ENTRY_WORDS * sizeof(u32);
+			continue;
+		}
+		regdump_len += am65_cpsw_regdump[i].hdr.len;
+	}
+
+	return regdump_len;
+}
+
+static void am65_cpsw_get_regs(struct net_device *ndev,
+			       struct ethtool_regs *regs, void *p)
+{
+	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
+	u32 i, j, pos, *reg = p;
+
+	/* update CPSW IP version */
+	regs->version = AM65_CPSW_REGDUMP_VER;
+
+	pos = 0;
+	for (i = 0; i < ARRAY_SIZE(am65_cpsw_regdump); i++) {
+		reg[pos++] = am65_cpsw_regdump[i].hdr.module_id;
+
+		if (am65_cpsw_regdump[i].hdr.module_id ==
+		    AM65_CPSW_REGDUMP_MOD_CPSW_ALE_TBL) {
+			u32 ale_tbl_len = common->ale->params.ale_entries *
+					  ALE_ENTRY_WORDS * sizeof(u32) +
+					  sizeof(struct am65_cpsw_regdump_hdr);
+			reg[pos++] = ale_tbl_len;
+			cpsw_ale_dump(common->ale, &reg[pos]);
+			pos += ale_tbl_len;
+			continue;
+		}
+
+		reg[pos++] = am65_cpsw_regdump[i].hdr.len;
+
+		j = am65_cpsw_regdump[i].start_ofs;
+		do {
+			reg[pos++] = j;
+			reg[pos++] = readl_relaxed(common->ss_base + j);
+			j += sizeof(u32);
+		} while (j <= am65_cpsw_regdump[i].end_ofs);
+	}
+}
+
+static int am65_cpsw_get_sset_count(struct net_device *ndev, int sset)
+{
+	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
+	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
+
+	switch (sset) {
+	case ETH_SS_STATS:
+		if (AM65_CPSW_IS_CPSW2G(common))
+			return ARRAY_SIZE(am65_host_stats) +
+			       ARRAY_SIZE(am65_slave_stats);
+
+		if (!port->port_id)
+			return ARRAY_SIZE(am65_host_stats);
+		else
+			return ARRAY_SIZE(am65_slave_stats);
+	case ETH_SS_PRIV_FLAGS:
+		return ARRAY_SIZE(am65_cpsw_ethtool_priv_flags);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static void am65_cpsw_get_strings(struct net_device *ndev,
+				  u32 stringset, u8 *data)
+{
+	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
+	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
+	const struct am65_cpsw_ethtool_stat *hw_stats;
+	u8 *p = data;
+	u32 i, num_stats;
+
+	switch (stringset) {
+	case ETH_SS_STATS:
+		if (AM65_CPSW_IS_CPSW2G(common)) {
+			num_stats = ARRAY_SIZE(am65_host_stats);
+			hw_stats = am65_host_stats;
+			for (i = 0; i < num_stats; i++) {
+				memcpy(p, hw_stats[i].desc, ETH_GSTRING_LEN);
+				p += ETH_GSTRING_LEN;
+			}
+
+			num_stats = ARRAY_SIZE(am65_slave_stats);
+			hw_stats = am65_slave_stats;
+			for (i = 0; i < num_stats; i++) {
+				memcpy(p, hw_stats[i].desc, ETH_GSTRING_LEN);
+				p += ETH_GSTRING_LEN;
+			}
+		} else {
+			if (!port->port_id) {
+				num_stats = ARRAY_SIZE(am65_host_stats);
+				hw_stats = am65_host_stats;
+			} else {
+				num_stats = ARRAY_SIZE(am65_slave_stats);
+				hw_stats = am65_slave_stats;
+			}
+
+			for (i = 0; i < num_stats; i++) {
+				memcpy(p, hw_stats[i].desc, ETH_GSTRING_LEN);
+				p += ETH_GSTRING_LEN;
+			}
+		}
+		break;
+	case ETH_SS_PRIV_FLAGS:
+		num_stats = ARRAY_SIZE(am65_cpsw_ethtool_priv_flags);
+
+		for (i = 0; i < num_stats; i++) {
+			memcpy(p, am65_cpsw_ethtool_priv_flags[i],
+			       ETH_GSTRING_LEN);
+			p += ETH_GSTRING_LEN;
+		}
+		break;
+	}
+}
+
+static void am65_cpsw_get_ethtool_stats(struct net_device *ndev,
+					struct ethtool_stats *stats, u64 *data)
+{
+	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
+	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
+	struct am65_cpsw_port *host_port = am65_common_get_host_port(common);
+	const struct am65_cpsw_ethtool_stat *hw_stats;
+	u32 i, num_stats;
+
+	if (AM65_CPSW_IS_CPSW2G(common)) {
+		num_stats = ARRAY_SIZE(am65_host_stats);
+		hw_stats = am65_host_stats;
+		for (i = 0; i < num_stats; i++)
+			*data++ = readl_relaxed(host_port->stat_base +
+						hw_stats[i].offset);
+
+		num_stats = ARRAY_SIZE(am65_slave_stats);
+		hw_stats = am65_slave_stats;
+		for (i = 0; i < num_stats; i++)
+			*data++ = readl_relaxed(port->stat_base +
+						hw_stats[i].offset);
+	} else {
+		if (!port->port_id) {
+			num_stats = ARRAY_SIZE(am65_host_stats);
+			hw_stats = am65_host_stats;
+		} else {
+			num_stats = ARRAY_SIZE(am65_slave_stats);
+			hw_stats = am65_slave_stats;
+		}
+
+		for (i = 0; i < num_stats; i++)
+			data[i] = readl_relaxed(port->stat_base +
+						hw_stats[i].offset);
+	}
+}
+
+#if IS_ENABLED(CONFIG_TI_AM65_CPTS)
+static int am65_cpsw_get_ethtool_ts_info(struct net_device *ndev,
+					 struct ethtool_ts_info *info)
+{
+	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
+
+	info->so_timestamping =
+		SOF_TIMESTAMPING_TX_HARDWARE |
+		SOF_TIMESTAMPING_TX_SOFTWARE |
+		SOF_TIMESTAMPING_RX_HARDWARE |
+		SOF_TIMESTAMPING_RX_SOFTWARE |
+		SOF_TIMESTAMPING_SOFTWARE |
+		SOF_TIMESTAMPING_RAW_HARDWARE;
+	info->phc_index = am65_cpts_phc_index(common->cpts);
+	info->tx_types = BIT(HWTSTAMP_TX_OFF) | BIT(HWTSTAMP_TX_ON);
+	info->rx_filters = BIT(HWTSTAMP_FILTER_NONE) | BIT(HWTSTAMP_FILTER_ALL);
+	return 0;
+}
+#else
+#define am65_cpsw_get_ethtool_ts_info ethtool_op_get_ts_info
+#endif
+
+static u32 am65_cpsw_get_ethtool_priv_flags(struct net_device *ndev)
+{
+	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
+	u32 priv_flags = 0;
+
+	if (common->pf_p0_rx_ptype_rrobin)
+		priv_flags |= AM65_CPSW_PRIV_P0_RX_PTYPE_RROBIN;
+
+	return priv_flags;
+}
+
+static int am65_cpsw_set_ethtool_priv_flags(struct net_device *ndev, u32 flags)
+{
+	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
+
+	common->pf_p0_rx_ptype_rrobin =
+			!!(flags & AM65_CPSW_PRIV_P0_RX_PTYPE_RROBIN);
+	am65_cpsw_nuss_set_p0_ptype(common);
+
+	return 0;
+}
+
+const struct ethtool_ops am65_cpsw_ethtool_ops_slave = {
+	.begin			= am65_cpsw_ethtool_op_begin,
+	.complete		= am65_cpsw_ethtool_op_complete,
+	.get_drvinfo		= am65_cpsw_get_drvinfo,
+	.get_msglevel		= am65_cpsw_get_msglevel,
+	.set_msglevel		= am65_cpsw_set_msglevel,
+	.get_channels		= am65_cpsw_get_channels,
+	.get_ringparam		= am65_cpsw_get_ringparam,
+	.get_regs_len		= am65_cpsw_get_regs_len,
+	.get_regs		= am65_cpsw_get_regs,
+	.get_sset_count		= am65_cpsw_get_sset_count,
+	.get_strings		= am65_cpsw_get_strings,
+	.get_ethtool_stats	= am65_cpsw_get_ethtool_stats,
+	.get_ts_info		= am65_cpsw_get_ethtool_ts_info,
+	.get_priv_flags		= am65_cpsw_get_ethtool_priv_flags,
+	.set_priv_flags		= am65_cpsw_set_ethtool_priv_flags,
+
+	.get_link		= ethtool_op_get_link,
+	.get_link_ksettings	= am65_cpsw_get_link_ksettings,
+	.set_link_ksettings	= am65_cpsw_set_link_ksettings,
+	.get_pauseparam		= am65_cpsw_get_pauseparam,
+	.set_pauseparam		= am65_cpsw_set_pauseparam,
+	.get_wol		= am65_cpsw_get_wol,
+	.set_wol		= am65_cpsw_set_wol,
+	.get_eee		= am65_cpsw_get_eee,
+	.set_eee		= am65_cpsw_set_eee,
+	.nway_reset		= am65_cpsw_nway_reset,
+};
+EXPORT_SYMBOL_GPL(am65_cpsw_ethtool_ops_slave);
diff -Naur --no-dereference a/drivers/net/ethernet/ti/am65-cpsw-nuss.c b/drivers/net/ethernet/ti/am65-cpsw-nuss.c
--- a/drivers/net/ethernet/ti/am65-cpsw-nuss.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.c	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,2223 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Texas Instruments K3 AM65 Ethernet Switch SubSystem Driver
+ *
+ * Copyright (C) 2017-2018 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ */
+
+#include <linux/clk.h>
+#include <linux/etherdevice.h>
+#include <linux/if_vlan.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/kmemleak.h>
+#include <linux/module.h>
+#include <linux/netdevice.h>
+#include <linux/net_tstamp.h>
+#include <linux/of.h>
+#include <linux/of_mdio.h>
+#include <linux/of_net.h>
+#include <linux/of_device.h>
+#include <linux/phy.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/regmap.h>
+#include <linux/mfd/syscon.h>
+#include <linux/soc/ti/k3-navss-desc-pool.h>
+#include <linux/dma/ti-cppi5.h>
+#include <linux/dma/k3-navss-udma.h>
+#include <linux/net_switch_config.h>
+
+#include "cpsw_ale.h"
+#include "cpsw_sl.h"
+#include "cpsw.h"
+#include "am65-cpsw-nuss.h"
+#include "davinci_mdio_int.h"
+#include "am65-cpts.h"
+
+#define AM65_CPSW_SS_BASE	0x0
+#define AM65_CPSW_SGMII_BASE	0x100
+#define AM65_CPSW_MDIO_BASE	0xf00
+#define AM65_CPSW_XGMII_BASE	0x2100
+#define AM65_CPSW_CPSW_NU_BASE	0x20000
+#define AM65_CPSW_NU_PORTS_BASE	0x1000
+#define AM65_CPSW_NU_STATS_BASE	0x1a000
+#define AM65_CPSW_NU_ALE_BASE	0x1e000
+#define AM65_CPSW_NU_CPTS_BASE	0x1d000
+
+#define AM65_CPSW_NU_PORTS_OFFSET	0x1000
+#define AM65_CPSW_NU_STATS_PORT_OFFSET	0x200
+
+#define AM65_CPSW_MAX_PORTS	9
+
+#define AM65_CPSW_MIN_PACKET_SIZE	VLAN_ETH_ZLEN
+#define AM65_CPSW_MAX_PACKET_SIZE	(VLAN_ETH_FRAME_LEN + ETH_FCS_LEN)
+
+#define AM65_CPSW_REG_CTL		0x004
+#define AM65_CPSW_REG_STAT_PORT_EN	0x014
+#define AM65_CPSW_REG_PTYPE		0x018
+
+#define AM65_CPSW_P0_REG_CTL			0x004
+#define AM65_CPSW_PORT0_REG_FLOW_ID_OFFSET	0x008
+
+#define AM65_CPSW_PORT_REG_PRI_CTL		0x01c
+#define AM65_CPSW_PORT_REG_RX_PRI_MAP		0x020
+#define AM65_CPSW_PORT_REG_RX_MAXLEN		0x024
+
+#define AM65_CPSW_PORTN_REG_SA_L		0x308
+#define AM65_CPSW_PORTN_REG_SA_H		0x30c
+#define AM65_CPSW_PORTN_REG_TS_CTL              0x310
+#define AM65_CPSW_PORTN_REG_TS_SEQ_LTYPE_REG	0x314
+#define AM65_CPSW_PORTN_REG_TS_VLAN_LTYPE_REG	0x318
+#define AM65_CPSW_PORTN_REG_TS_CTL_LTYPE2       0x31C
+
+#define AM65_CPSW_CTL_VLAN_AWARE		BIT(1)
+#define AM65_CPSW_CTL_P0_ENABLE			BIT(2)
+#define AM65_CPSW_CTL_P0_TX_CRC_REMOVE		BIT(13)
+#define AM65_CPSW_CTL_P0_RX_PAD			BIT(14)
+
+/* AM65_CPSW_P0_REG_CTL */
+#define AM65_CPSW_P0_REG_CTL_RX_CHECKSUM_EN	BIT(0)
+
+/* AM65_CPSW_PORT_REG_PRI_CTL */
+#define AM65_CPSW_PORT_REG_PRI_CTL_RX_PTYPE_RROBIN	BIT(8)
+
+/* AM65_CPSW_PN_TS_CTL register fields */
+#define AM65_CPSW_PN_TS_CTL_TX_ANX_F_EN		BIT(4)
+#define AM65_CPSW_PN_TS_CTL_TX_VLAN_LT1_EN	BIT(5)
+#define AM65_CPSW_PN_TS_CTL_TX_VLAN_LT2_EN	BIT(6)
+#define AM65_CPSW_PN_TS_CTL_TX_ANX_D_EN		BIT(7)
+#define AM65_CPSW_PN_TS_CTL_TX_ANX_E_EN		BIT(10)
+#define AM65_CPSW_PN_TS_CTL_TX_HOST_TS_EN	BIT(11)
+#define AM65_CPSW_PN_TS_CTL_MSG_TYPE_EN_SHIFT	16
+
+/* AM65_CPSW_PORTN_REG_TS_SEQ_LTYPE_REG register fields */
+#define AM65_CPSW_PN_TS_SEQ_ID_OFFSET_SHIFT	16
+
+/* AM65_CPSW_PORTN_REG_TS_CTL_LTYPE2 */
+#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_107	BIT(16)
+#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_129	BIT(17)
+#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_130	BIT(18)
+#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_131	BIT(19)
+#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_132	BIT(20)
+#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_319	BIT(21)
+#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_320	BIT(22)
+#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_TTL_NONZERO BIT(23)
+
+/* The PTP event messages - Sync, Delay_Req, Pdelay_Req, and Pdelay_Resp. */
+#define AM65_CPSW_TS_EVENT_MSG_TYPE_BITS (BIT(0) | BIT(1) | BIT(2) | BIT(3))
+
+#define AM65_CPSW_TS_SEQ_ID_OFFSET (0x1e)
+
+#define AM65_CPSW_TS_TX_ANX_ALL_EN		\
+	(AM65_CPSW_PN_TS_CTL_TX_ANX_D_EN |	\
+	 AM65_CPSW_PN_TS_CTL_TX_ANX_E_EN |	\
+	 AM65_CPSW_PN_TS_CTL_TX_ANX_F_EN)
+
+#define AM65_CPSW_ALE_AGEOUT_DEFAULT	30
+/* Number of TX/RX descriptors */
+#define AM65_CPSW_MAX_TX_DESC	500
+#define AM65_CPSW_MAX_RX_DESC	500
+
+#define mac_hi(mac)	(((mac)[0] << 0) | ((mac)[1] << 8) |	\
+			 ((mac)[2] << 16) | ((mac)[3] << 24))
+#define mac_lo(mac)	(((mac)[4] << 0) | ((mac)[5] << 8))
+
+#define AM65_CPSW_NAV_PS_DATA_SIZE 16
+#define AM65_CPSW_NAV_SW_DATA_SIZE 16
+
+static int am65_cpsw_debug_level;
+module_param(am65_cpsw_debug_level, int, 0);
+MODULE_PARM_DESC(am65_cpsw_debug_level,
+		 "TI AM65x cpsw debug level (NETIF_MSG bits)");
+
+static void am65_cpsw_port_set_sl_mac(struct am65_cpsw_port *slave,
+				      const u8 *dev_addr)
+{
+	writel(mac_hi(dev_addr), slave->port_base + AM65_CPSW_PORTN_REG_SA_H);
+	writel(mac_lo(dev_addr), slave->port_base + AM65_CPSW_PORTN_REG_SA_L);
+}
+
+static void am65_cpsw_sl_ctl_reset(struct am65_cpsw_port *port)
+{
+	cpsw_sl_reset(port->slave.mac_sl, 100);
+	/* Max length register has to be restored after MAC SL reset */
+	writel(AM65_CPSW_MAX_PACKET_SIZE,
+	       port->port_base + AM65_CPSW_PORT_REG_RX_MAXLEN);
+}
+
+static void am65_cpsw_nuss_get_ver(struct am65_cpsw_common *common)
+{
+	common->nuss_ver = readl(common->ss_base);
+	common->cpsw_ver = readl(common->cpsw_base);
+	dev_info(common->dev,
+		 "initializing am65 cpsw nuss version 0x%08X, cpsw version 0x%08X Ports: %u\n",
+		common->nuss_ver,
+		common->cpsw_ver,
+		common->port_num);
+}
+
+void am65_cpsw_nuss_adjust_link(struct net_device *ndev)
+{
+	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
+	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
+	struct phy_device *phy = port->slave.phy;
+	u32 mac_control = 0;
+
+	if (!phy)
+		return;
+
+	if (phy->link) {
+		mac_control = CPSW_SL_CTL_GMII_EN;
+
+		if (phy->speed == 1000)
+			mac_control |= CPSW_SL_CTL_GIG;
+		if (phy->speed == 10 && phy_interface_is_rgmii(phy))
+			/* Can be used with in band mode only */
+			mac_control |= CPSW_SL_CTL_EXT_EN;
+		if (phy->duplex)
+			mac_control |= CPSW_SL_CTL_FULLDUPLEX;
+
+		/* RGMII speed is 100M if !CPSW_SL_CTL_GIG*/
+
+		/* rx_pause/tx_pause */
+		if (port->slave.rx_pause)
+			mac_control |= CPSW_SL_CTL_RX_FLOW_EN;
+
+		if (port->slave.tx_pause)
+			mac_control |= CPSW_SL_CTL_TX_FLOW_EN;
+
+		cpsw_sl_ctl_set(port->slave.mac_sl, mac_control);
+
+		/* enable forwarding */
+		cpsw_ale_control_set(common->ale, port->port_id,
+				     ALE_PORT_STATE, ALE_PORT_STATE_FORWARD);
+
+		netif_tx_wake_all_queues(ndev);
+		netif_carrier_on(ndev);
+	} else {
+		int tmo;
+		/* disable forwarding */
+		cpsw_ale_control_set(common->ale, port->port_id,
+				     ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
+
+		cpsw_sl_ctl_set(port->slave.mac_sl, CPSW_SL_CTL_CMD_IDLE);
+
+		tmo = cpsw_sl_wait_for_idle(port->slave.mac_sl);
+		dev_dbg(common->dev, "donw msc_sl %08x tmo %d\n",
+			cpsw_sl_reg_read(port->slave.mac_sl, CPSW_SL_MACSTATUS),
+			tmo);
+
+		cpsw_sl_ctl_reset(port->slave.mac_sl);
+
+		netif_carrier_off(ndev);
+		netif_tx_stop_all_queues(ndev);
+	}
+
+	phy_print_status(phy);
+}
+EXPORT_SYMBOL_GPL(am65_cpsw_nuss_adjust_link);
+
+static int am65_cpsw_nuss_ndo_slave_add_vid(struct net_device *ndev,
+					    __be16 proto, u16 vid)
+{
+	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
+	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
+	u32 port_mask, unrag_mask = 0;
+	int ret;
+
+	ret = pm_runtime_get_sync(common->dev);
+	if (ret < 0) {
+		pm_runtime_put_noidle(common->dev);
+		return ret;
+	}
+
+	port_mask = BIT(port->port_id) | ALE_PORT_HOST;
+	if (!vid)
+		unrag_mask = port_mask;
+	dev_info(common->dev, "Adding vlan %d to vlan filter\n", vid);
+	ret = cpsw_ale_add_vlan(common->ale, vid, port_mask,
+				unrag_mask, port_mask, 0);
+
+	pm_runtime_put(common->dev);
+	return ret;
+}
+
+static int am65_cpsw_nuss_ndo_slave_kill_vid(struct net_device *ndev,
+					     __be16 proto, u16 vid)
+{
+	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
+	int ret;
+
+	ret = pm_runtime_get_sync(common->dev);
+	if (ret < 0) {
+		pm_runtime_put_noidle(common->dev);
+		return ret;
+	}
+
+	dev_info(common->dev, "Removing vlan %d from vlan filter\n", vid);
+	ret = cpsw_ale_del_vlan(common->ale, vid, 0);
+
+	pm_runtime_put(common->dev);
+	return ret;
+}
+
+static void am65_cpsw_slave_set_promisc_2g(struct am65_cpsw_port *port,
+					   bool promisc)
+{
+	struct am65_cpsw_common *common = port->common;
+
+	cpsw_ale_control_set(common->ale, ALE_PORT_HOST,
+			     ALE_BYPASS, promisc);
+
+	dev_dbg(common->dev, "promisc disabled %s\n",
+		promisc ? "enabled" : "disabled");
+}
+
+static void am65_cpsw_slave_set_promisc_slave(struct am65_cpsw_port *port,
+					      bool promisc)
+{
+	struct am65_cpsw_common *common = port->common;
+
+	if (promisc) {
+		/* Enable promiscuous mode */
+		cpsw_ale_control_set(common->ale, port->port_id,
+				     ALE_PORT_NOLEARN, 1);
+		cpsw_ale_control_set(common->ale, port->port_id,
+				     ALE_PORT_NO_SA_UPDATE, 1);
+		cpsw_ale_control_set(common->ale, port->port_id,
+				     ALE_PORT_MACONLY_CAF, 1);
+		dev_dbg(common->dev, "promisc enabled\n");
+	} else {
+		/* Disable promiscuous mode */
+		cpsw_ale_control_set(common->ale, port->port_id,
+				     ALE_PORT_NOLEARN, 0);
+		cpsw_ale_control_set(common->ale, port->port_id,
+				     ALE_PORT_NO_SA_UPDATE, 0);
+		cpsw_ale_control_set(common->ale, port->port_id,
+				     ALE_PORT_MACONLY_CAF, 0);
+		dev_dbg(common->dev, "promisc disabled\n");
+	}
+}
+
+static void am65_cpsw_nuss_ndo_slave_set_rx_mode(struct net_device *ndev)
+{
+	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
+	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
+	u32 port_mask;
+	bool promisc;
+
+	promisc = !!(ndev->flags & IFF_PROMISC);
+	if (AM65_CPSW_IS_CPSW2G(common))
+		am65_cpsw_slave_set_promisc_2g(port, promisc);
+	else
+		am65_cpsw_slave_set_promisc_slave(port, promisc);
+
+	if (promisc)
+		return;
+
+	/* Restore allmulti on vlans if necessary */
+	cpsw_ale_set_allmulti(common->ale, ndev->flags & IFF_ALLMULTI);
+
+	port_mask = BIT(port->port_id) | ALE_PORT_HOST;
+	/* Clear all mcast from ALE */
+	cpsw_ale_flush_multicast(common->ale, port_mask, -1);
+
+	if (!netdev_mc_empty(ndev)) {
+		struct netdev_hw_addr *ha;
+
+		/* program multicast address list into ALE register */
+		netdev_for_each_mc_addr(ha, ndev) {
+			cpsw_ale_add_mcast(common->ale, ha->addr,
+					   port_mask, 0, 0, 0);
+		}
+	}
+}
+
+static void am65_cpsw_nuss_ndo_host_tx_timeout(struct net_device *ndev)
+{
+	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
+	int ch;
+
+	/* process every txq*/
+	for (ch = 0; ch < common->tx_ch_num; ch++) {
+		struct am65_cpsw_tx_chn *tx_chn;
+		struct netdev_queue *netif_txq;
+		unsigned long trans_start;
+
+		netif_txq = netdev_get_tx_queue(ndev, ch);
+		trans_start = netif_txq->trans_start;
+		if (netif_xmit_stopped(netif_txq) &&
+		    time_after(jiffies, (trans_start + ndev->watchdog_timeo))) {
+			tx_chn = &common->tx_chns[ch];
+			netdev_err(ndev, "txq:%d DRV_XOFF:%d tmo:%u dql_avail:%d free_desc:%zu\n",
+				   ch,
+				   netif_tx_queue_stopped(netif_txq),
+				   jiffies_to_msecs(jiffies - trans_start),
+				   dql_avail(&netif_txq->dql),
+				   k3_knav_pool_avail(tx_chn->desc_pool));
+
+			if (netif_tx_queue_stopped(netif_txq)) {
+				/* try recover if stopped by us */
+				txq_trans_update(netif_txq);
+				netif_tx_wake_queue(netif_txq);
+			}
+		}
+	}
+}
+
+static int am65_cpsw_nuss_rx_push(struct am65_cpsw_common *common,
+				  struct sk_buff *skb)
+{
+	struct cppi5_host_desc_t *desc_rx;
+	struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
+	struct device *dev = common->dev;
+	dma_addr_t desc_dma;
+	dma_addr_t buf_dma;
+	u32 pkt_len = skb_tailroom(skb);
+	void *swdata;
+
+	desc_rx = k3_knav_pool_alloc(rx_chn->desc_pool);
+	if (!desc_rx) {
+		dev_err(dev, "Failed to allocate RXFDQ descriptor\n");
+		return -ENOMEM;
+	}
+	desc_dma = k3_knav_pool_virt2dma(rx_chn->desc_pool, desc_rx);
+
+	buf_dma = dma_map_single(dev, skb->data, pkt_len, DMA_FROM_DEVICE);
+	if (unlikely(dma_mapping_error(dev, buf_dma))) {
+		k3_knav_pool_free(rx_chn->desc_pool, desc_rx);
+		dev_err(dev, "Failed to map rx skb buffer\n");
+		return -EINVAL;
+	}
+
+	cppi5_hdesc_init(desc_rx, CPPI5_INFO0_HDESC_EPIB_PRESENT,
+			 AM65_CPSW_NAV_PS_DATA_SIZE);
+	cppi5_hdesc_attach_buf(desc_rx, 0, 0, buf_dma, skb_tailroom(skb));
+	swdata = cppi5_hdesc_get_swdata(desc_rx);
+	*((void **)swdata) = skb;
+
+	return k3_nav_udmax_push_rx_chn(rx_chn->rx_chn, 0, desc_rx, desc_dma);
+}
+
+void am65_cpsw_nuss_set_p0_ptype(struct am65_cpsw_common *common)
+{
+	struct am65_cpsw_port *host_p = am65_common_get_host_port(common);
+	u32 val, pri_map;
+
+	/* P0 set Receive Priority Type */
+	val = readl(host_p->port_base + AM65_CPSW_PORT_REG_PRI_CTL);
+
+	if (common->pf_p0_rx_ptype_rrobin) {
+		val |= AM65_CPSW_PORT_REG_PRI_CTL_RX_PTYPE_RROBIN;
+		/* Enet Ports fifos works in fixed priority mode only, so
+		 * reset P0_Rx_Pri_Map so all packet will go in Enet fifo 0
+		 */
+		pri_map = 0x0;
+	} else {
+		val &= ~AM65_CPSW_PORT_REG_PRI_CTL_RX_PTYPE_RROBIN;
+		/* restore P0_Rx_Pri_Map */
+		pri_map = 0x76543210;
+	}
+
+	writel(pri_map, host_p->port_base + AM65_CPSW_PORT_REG_RX_PRI_MAP);
+	writel(val, host_p->port_base + AM65_CPSW_PORT_REG_PRI_CTL);
+}
+
+static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common,
+				      netdev_features_t features)
+{
+	struct am65_cpsw_port *host_p = am65_common_get_host_port(common);
+	int port_idx, i, ret;
+	struct sk_buff *skb;
+	u32 val, port_mask;
+
+	if (common->usage_count)
+		return 0;
+
+	/* Control register */
+	writel(AM65_CPSW_CTL_P0_ENABLE | AM65_CPSW_CTL_P0_TX_CRC_REMOVE |
+	       AM65_CPSW_CTL_VLAN_AWARE | AM65_CPSW_CTL_P0_RX_PAD,
+	       common->cpsw_base + AM65_CPSW_REG_CTL);
+	/* Max length register */
+	writel(AM65_CPSW_MAX_PACKET_SIZE,
+	       host_p->port_base + AM65_CPSW_PORT_REG_RX_MAXLEN);
+	/* set base flow_id */
+	writel(common->rx_flow_id_base,
+	       host_p->port_base + AM65_CPSW_PORT0_REG_FLOW_ID_OFFSET);
+	/* en tx crc offload */
+	if (features & NETIF_F_HW_CSUM)
+		writel(AM65_CPSW_P0_REG_CTL_RX_CHECKSUM_EN,
+		       host_p->port_base + AM65_CPSW_P0_REG_CTL);
+
+	am65_cpsw_nuss_set_p0_ptype(common);
+
+	/* enable statistic */
+	val = 0;
+	for (port_idx = 0; port_idx < common->port_num; port_idx++) {
+		if (!common->ports[port_idx].disabled)
+			val |=  BIT(common->ports[port_idx].port_id);
+	}
+	writel(val, common->cpsw_base + AM65_CPSW_REG_STAT_PORT_EN);
+
+	/* disable priority elevation */
+	writel(0, common->cpsw_base + AM65_CPSW_REG_PTYPE);
+
+	cpsw_ale_start(common->ale);
+
+	/* switch to vlan unaware mode */
+	cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_VLAN_AWARE, 1);
+	cpsw_ale_control_set(common->ale, HOST_PORT_NUM,
+			     ALE_PORT_STATE, ALE_PORT_STATE_FORWARD);
+
+	/* default vlan cfg */
+	/* create mask based on enabled ports */
+	port_mask = GENMASK(common->port_num - 1, 0) &
+		    ~common->disabled_ports_mask;
+
+	cpsw_ale_add_vlan(common->ale, 0, port_mask,
+			  port_mask, port_mask,
+			  port_mask & ~ALE_PORT_HOST);
+
+	for (i = 0; i < common->rx_chns.descs_num; i++) {
+		skb = __netdev_alloc_skb_ip_align(NULL,
+						  AM65_CPSW_MAX_PACKET_SIZE,
+						  GFP_KERNEL);
+		if (!skb) {
+			dev_err(common->dev, "cannot allocate skb\n");
+			return -ENOMEM;
+		}
+
+		ret = am65_cpsw_nuss_rx_push(common, skb);
+		if (ret < 0) {
+			dev_err(common->dev,
+				"cannot submit skb to channel rx, error %d\n",
+				ret);
+			kfree_skb(skb);
+			return ret;
+		}
+		kmemleak_not_leak(skb);
+	}
+	k3_nav_udmax_enable_rx_chn(common->rx_chns.rx_chn);
+
+	for (i = 0; i < AM65_CPSW_MAX_TX_QUEUES; i++) {
+		ret = k3_nav_udmax_enable_tx_chn(common->tx_chns[i].tx_chn);
+		if (ret)
+			return ret;
+	}
+
+	napi_enable(&common->napi_tx);
+	napi_enable(&common->napi_rx);
+
+	dev_dbg(common->dev, "cpsw_nuss started\n");
+	return 0;
+}
+
+static void am65_cpsw_nuss_tx_cleanup(void *data, dma_addr_t desc_dma);
+static void am65_cpsw_nuss_rx_cleanup(void *data, dma_addr_t desc_dma);
+
+static int am65_cpsw_nuss_common_stop(struct am65_cpsw_common *common)
+{
+	int i;
+
+	if (common->usage_count != 1)
+		return 0;
+
+	cpsw_ale_control_set(common->ale, HOST_PORT_NUM,
+			     ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
+
+	/* shutdown tx channels */
+	atomic_set(&common->tdown_cnt, AM65_CPSW_MAX_TX_QUEUES);
+	/* ensure new tdown_cnt value is visible */
+	smp_mb__after_atomic();
+	reinit_completion(&common->tdown_complete);
+
+	for (i = 0; i < AM65_CPSW_MAX_TX_QUEUES; i++)
+		k3_nav_udmax_tdown_tx_chn(common->tx_chns[i].tx_chn, false);
+
+	i = wait_for_completion_timeout(&common->tdown_complete,
+					msecs_to_jiffies(1000));
+	if (!i)
+		dev_err(common->dev, "tx timeout\n");
+	napi_disable(&common->napi_tx);
+
+	for (i = 0; i < AM65_CPSW_MAX_TX_QUEUES; i++) {
+		k3_nav_udmax_reset_tx_chn(common->tx_chns[i].tx_chn,
+					  &common->tx_chns[i],
+					  am65_cpsw_nuss_tx_cleanup);
+		k3_nav_udmax_disable_tx_chn(common->tx_chns[i].tx_chn);
+	}
+
+	k3_nav_udmax_tdown_rx_chn(common->rx_chns.rx_chn, true);
+	napi_disable(&common->napi_rx);
+
+	for (i = 0; i < AM65_CPSW_MAX_RX_FLOWS; i++)
+		k3_nav_udmax_reset_rx_chn(common->rx_chns.rx_chn, i,
+					  &common->rx_chns,
+					  am65_cpsw_nuss_rx_cleanup, !!i);
+
+	k3_nav_udmax_disable_rx_chn(common->rx_chns.rx_chn);
+
+	cpsw_ale_stop(common->ale);
+
+	writel(0, common->cpsw_base + AM65_CPSW_REG_CTL);
+	writel(0, common->cpsw_base + AM65_CPSW_REG_STAT_PORT_EN);
+
+	dev_dbg(common->dev, "cpsw_nuss stopped\n");
+	return 0;
+}
+
+static int am65_cpsw_nuss_ndo_slave_stop(struct net_device *ndev)
+{
+	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
+	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
+	int ret;
+
+	if (port->slave.phy)
+		phy_stop(port->slave.phy);
+
+	netif_tx_stop_all_queues(ndev);
+
+	if (port->slave.phy) {
+		phy_disconnect(port->slave.phy);
+		port->slave.phy = NULL;
+	}
+
+	ret = am65_cpsw_nuss_common_stop(common);
+	if (ret)
+		return ret;
+
+	common->usage_count--;
+	pm_runtime_put(common->dev);
+	return 0;
+}
+
+static int am65_cpsw_nuss_ndo_slave_open(struct net_device *ndev)
+{
+	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
+	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
+	u32 port_mask;
+	int ret, i;
+
+	ret = pm_runtime_get_sync(common->dev);
+	if (ret < 0) {
+		pm_runtime_put_noidle(common->dev);
+		return ret;
+	}
+
+	/* Notify the stack of the actual queue counts. */
+	ret = netif_set_real_num_tx_queues(ndev, common->tx_ch_num);
+	if (ret) {
+		dev_err(common->dev, "cannot set real number of tx queues\n");
+		return ret;
+	}
+
+	ret = netif_set_real_num_rx_queues(ndev, common->rx_ch_num);
+	if (ret) {
+		dev_err(common->dev, "cannot set real number of rx queues\n");
+		return ret;
+	}
+
+	for (i = 0; i < AM65_CPSW_MAX_TX_QUEUES; i++)
+		netdev_tx_reset_queue(netdev_get_tx_queue(ndev, i));
+
+	ret = am65_cpsw_nuss_common_open(common, ndev->features);
+	if (ret)
+		return ret;
+
+	common->usage_count++;
+
+	am65_cpsw_port_set_sl_mac(port, ndev->dev_addr);
+
+	if (port->slave.mac_only)
+		/* enable mac-only mode on port */
+		cpsw_ale_control_set(common->ale, port->port_id,
+				     ALE_PORT_MACONLY, 1);
+
+	port_mask = BIT(port->port_id) | ALE_PORT_HOST;
+	cpsw_ale_add_ucast(common->ale, ndev->dev_addr,
+			   HOST_PORT_NUM, ALE_SECURE, 0);
+	cpsw_ale_add_mcast(common->ale, ndev->broadcast,
+			   port_mask, 0, 0, ALE_MCAST_FWD_2);
+
+	/* mac_sl should be configured via phy-link interface */
+	am65_cpsw_sl_ctl_reset(port);
+
+	cpsw_phy_sel(common->dev, port->slave.phy_if, port->port_id);
+
+	if (port->slave.phy_node) {
+		port->slave.phy = of_phy_connect(ndev,
+						 port->slave.phy_node,
+						 &am65_cpsw_nuss_adjust_link,
+						 0, port->slave.phy_if);
+		if (!port->slave.phy) {
+			dev_err(common->dev, "phy %pOF not found on slave %d\n",
+				port->slave.phy_node,
+				port->port_id);
+			ret = -ENODEV;
+			goto error_cleanup;
+		}
+	}
+
+	phy_attached_info(port->slave.phy);
+	phy_start(port->slave.phy);
+
+	return 0;
+
+error_cleanup:
+	am65_cpsw_nuss_ndo_slave_stop(ndev);
+	return ret;
+}
+
+static void am65_cpsw_nuss_rx_cleanup(void *data, dma_addr_t desc_dma)
+{
+	struct am65_cpsw_rx_chn *rx_chn = data;
+	struct cppi5_host_desc_t *desc_rx;
+	struct sk_buff *skb;
+	dma_addr_t buf_dma;
+	u32 buf_dma_len;
+	void **swdata;
+
+	desc_rx = k3_knav_pool_dma2virt(rx_chn->desc_pool, desc_dma);
+	swdata = cppi5_hdesc_get_swdata(desc_rx);
+	skb = *swdata;
+	cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);
+
+	dma_unmap_single(rx_chn->dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE);
+	k3_knav_pool_free(rx_chn->desc_pool, desc_rx);
+
+	dev_kfree_skb_any(skb);
+}
+
+static void am65_cpsw_nuss_rx_ts(struct sk_buff *skb, u32 *psdata)
+{
+	struct skb_shared_hwtstamps *ssh;
+	u64 ns;
+
+	ns = ((u64)psdata[1] << 32) | psdata[0];
+
+	ssh = skb_hwtstamps(skb);
+	memset(ssh, 0, sizeof(*ssh));
+	ssh->hwtstamp = ns_to_ktime(ns);
+}
+
+/* RX psdata[2] word format - checksum information */
+#define AM65_CPSW_RX_PSD_CSUM_ADD	GENMASK(15, 0)
+#define AM65_CPSW_RX_PSD_CSUM_ERR	BIT(16)
+#define AM65_CPSW_RX_PSD_IS_FRAGMENT	BIT(17)
+#define AM65_CPSW_RX_PSD_IS_TCP		BIT(18)
+#define AM65_CPSW_RX_PSD_IPV6_VALID	BIT(19)
+#define AM65_CPSW_RX_PSD_IPV4_VALID	BIT(20)
+
+static void am65_cpsw_nuss_rx_csum(struct sk_buff *skb, u32 csum_info)
+{
+	/* HW can verify IPv4/IPv6 TCP/UDP packets checksum
+	 * csum information provides in psdata[2] word:
+	 * AM65_CPSW_RX_PSD_CSUM_ERR bit - indicates csum error
+	 * AM65_CPSW_RX_PSD_IPV6_VALID and AM65_CPSW_RX_PSD_IPV4_VALID
+	 * bits - indicates IPv4/IPv6 packet
+	 * AM65_CPSW_RX_PSD_IS_FRAGMENT bit - indicates fragmented packet
+	 * AM65_CPSW_RX_PSD_CSUM_ADD has value 0xFFFF for non fragmented packets
+	 * or csum value for fragmented packets if !AM65_CPSW_RX_PSD_CSUM_ERR
+	 */
+	skb_checksum_none_assert(skb);
+
+	if (unlikely(!(skb->dev->features & NETIF_F_RXCSUM)))
+		return;
+
+	if ((csum_info & (AM65_CPSW_RX_PSD_IPV6_VALID |
+			  AM65_CPSW_RX_PSD_IPV4_VALID)) &&
+			  !(csum_info & AM65_CPSW_RX_PSD_CSUM_ERR)) {
+		if (csum_info & AM65_CPSW_RX_PSD_IS_FRAGMENT) {
+			skb->ip_summed = CHECKSUM_COMPLETE;
+			skb->csum = csum_unfold(csum_info &
+						AM65_CPSW_RX_PSD_CSUM_ADD);
+		} else {
+			skb->ip_summed = CHECKSUM_UNNECESSARY;
+		}
+	}
+}
+
+static int am65_cpsw_nuss_rx_packets(struct am65_cpsw_common *common,
+				     u32 flow_idx)
+{
+	struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
+	struct device *dev = common->dev;
+	struct am65_cpsw_ndev_priv *ndev_priv;
+	struct am65_cpsw_ndev_stats *stats;
+	struct net_device *ndev;
+	struct cppi5_host_desc_t *desc_rx;
+	struct sk_buff *skb, *new_skb;
+	dma_addr_t desc_dma, buf_dma;
+	u32 buf_dma_len, pkt_len, port_id = 0, csum_info;
+	int ret = 0;
+	void **swdata;
+	u32 *psdata;
+
+	ret = k3_nav_udmax_pop_rx_chn(rx_chn->rx_chn, flow_idx, &desc_dma);
+	if (ret) {
+		if (ret != -ENODATA)
+			dev_err(dev, "RX: pop chn fail %d\n", ret);
+		return ret;
+	}
+
+	if (desc_dma & 0x1) {
+		dev_dbg(dev, "%s RX tdown flow: %u\n", __func__, flow_idx);
+		return 0;
+	}
+
+	desc_rx = k3_knav_pool_dma2virt(rx_chn->desc_pool, desc_dma);
+	dev_dbg(dev, "%s flow_idx: %u desc %pad\n",
+		__func__, flow_idx, &desc_dma);
+
+	swdata = cppi5_hdesc_get_swdata(desc_rx);
+	skb = *swdata;
+	cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);
+	pkt_len = cppi5_hdesc_get_pktlen(desc_rx);
+	cppi5_desc_get_tags_ids(&desc_rx->hdr, &port_id, NULL);
+	if (port_id)
+		port_id = (port_id % common->port_num);
+	dev_dbg(dev, "%s rx port_id:%d\n", __func__, port_id);
+	ndev = common->ports[port_id].ndev;
+	skb->dev = ndev;
+
+	psdata = cppi5_hdesc_get_psdata32(desc_rx);
+	/* add RX timestamp */
+	if (common->ports[port_id].rx_ts_enabled)
+		am65_cpsw_nuss_rx_ts(skb, psdata);
+	csum_info = psdata[2];
+	dev_dbg(dev, "%s rx csum_info:%#x\n", __func__, csum_info);
+
+	dma_unmap_single(dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE);
+
+	k3_knav_pool_free(rx_chn->desc_pool, desc_rx);
+
+	if (unlikely(!netif_running(skb->dev))) {
+		dev_kfree_skb_any(skb);
+		return 0;
+	}
+
+	new_skb = netdev_alloc_skb_ip_align(ndev, AM65_CPSW_MAX_PACKET_SIZE);
+	if (new_skb) {
+		skb_put(skb, pkt_len);
+		skb->protocol = eth_type_trans(skb, ndev);
+		am65_cpsw_nuss_rx_csum(skb, csum_info);
+		napi_gro_receive(&common->napi_rx, skb);
+
+		ndev_priv = netdev_priv(ndev);
+		stats = this_cpu_ptr(ndev_priv->stats);
+
+		u64_stats_update_begin(&stats->syncp);
+		stats->rx_packets++;
+		stats->rx_bytes += pkt_len;
+		u64_stats_update_end(&stats->syncp);
+		kmemleak_not_leak(new_skb);
+	} else {
+		ndev->stats.rx_dropped++;
+		new_skb = skb;
+	}
+
+	if (netif_dormant(ndev)) {
+		dev_kfree_skb_any(new_skb);
+		ndev->stats.rx_dropped++;
+		return 0;
+	}
+
+	ret = am65_cpsw_nuss_rx_push(common, new_skb);
+	if (WARN_ON(ret < 0)) {
+		dev_kfree_skb_any(new_skb);
+		ndev->stats.rx_errors++;
+		ndev->stats.rx_dropped++;
+	}
+
+	return ret;
+}
+
+static int am65_cpsw_nuss_rx_poll(struct napi_struct *napi_rx, int budget)
+{
+	struct am65_cpsw_common *common = am65_cpsw_napi_to_common(napi_rx);
+	int num_rx = 0;
+	int cur_budget, flow = AM65_CPSW_MAX_RX_FLOWS;
+	int ret;
+
+	/* process every flow */
+	while (flow--) {
+		cur_budget = budget - num_rx;
+
+		while (cur_budget--) {
+			ret = am65_cpsw_nuss_rx_packets(common, flow);
+			if (ret)
+				break;
+			num_rx++;
+		}
+
+		if (num_rx >= budget)
+			break;
+	}
+
+	dev_dbg(common->dev, "%s num_rx:%d %d\n", __func__, num_rx, budget);
+
+	if (num_rx < budget && napi_complete_done(napi_rx, num_rx))
+		enable_irq(common->rx_chns.irq);
+
+	return num_rx;
+}
+
+static void am65_cpsw_nuss_xmit_free(struct am65_cpsw_tx_chn *tx_chn,
+				     struct device *dev,
+				     struct cppi5_host_desc_t *desc)
+{
+	struct cppi5_host_desc_t *first_desc, *next_desc;
+	dma_addr_t buf_dma, next_desc_dma;
+	u32 buf_dma_len;
+
+	first_desc = desc;
+	next_desc = first_desc;
+
+	cppi5_hdesc_get_obuf(first_desc, &buf_dma, &buf_dma_len);
+
+	dma_unmap_single(dev, buf_dma, buf_dma_len,
+			 DMA_TO_DEVICE);
+
+	next_desc_dma = cppi5_hdesc_get_next_hbdesc(first_desc);
+	while (next_desc_dma) {
+		next_desc = k3_knav_pool_dma2virt(tx_chn->desc_pool,
+						  next_desc_dma);
+		cppi5_hdesc_get_obuf(next_desc, &buf_dma, &buf_dma_len);
+
+		dma_unmap_page(dev, buf_dma, buf_dma_len,
+			       DMA_TO_DEVICE);
+
+		next_desc_dma = cppi5_hdesc_get_next_hbdesc(next_desc);
+
+		k3_knav_pool_free(tx_chn->desc_pool, next_desc);
+	}
+
+	k3_knav_pool_free(tx_chn->desc_pool, first_desc);
+}
+
+static void am65_cpsw_nuss_tx_cleanup(void *data, dma_addr_t desc_dma)
+{
+	struct am65_cpsw_tx_chn *tx_chn = data;
+	struct cppi5_host_desc_t *desc_tx;
+	struct sk_buff *skb;
+	void **swdata;
+
+	desc_tx = k3_knav_pool_dma2virt(tx_chn->desc_pool, desc_dma);
+	swdata = cppi5_hdesc_get_swdata(desc_tx);
+	skb = *(swdata);
+	am65_cpsw_nuss_xmit_free(tx_chn, tx_chn->dev, desc_tx);
+
+	dev_kfree_skb_any(skb);
+}
+
+static int am65_cpsw_nuss_tx_compl_packets(struct am65_cpsw_common *common,
+					   int chn, unsigned int budget)
+{
+	struct cppi5_host_desc_t *desc_tx;
+	struct device *dev = common->dev;
+	struct netdev_queue *netif_txq;
+	struct am65_cpsw_tx_chn *tx_chn;
+	struct net_device *ndev;
+	unsigned int total_bytes = 0;
+	struct sk_buff *skb;
+	dma_addr_t desc_dma;
+	int res, num_tx = 0;
+	void **swdata;
+
+	tx_chn = &common->tx_chns[chn - 1];
+
+	while (budget--) {
+		struct am65_cpsw_ndev_priv *ndev_priv;
+		struct am65_cpsw_ndev_stats *stats;
+
+		res = k3_nav_udmax_pop_tx_chn(tx_chn->tx_chn, &desc_dma);
+		if (res == -ENODATA)
+			break;
+
+		if (desc_dma & 0x1) {
+			if (atomic_dec_and_test(&common->tdown_cnt))
+				complete(&common->tdown_complete);
+			break;
+		}
+
+		desc_tx = k3_knav_pool_dma2virt(tx_chn->desc_pool, desc_dma);
+		swdata = cppi5_hdesc_get_swdata(desc_tx);
+		skb = *(swdata);
+		am65_cpsw_nuss_xmit_free(tx_chn, dev, desc_tx);
+
+		ndev = skb->dev;
+
+		am65_cpts_tx_timestamp(common->cpts, skb);
+
+		ndev_priv = netdev_priv(ndev);
+		stats = this_cpu_ptr(ndev_priv->stats);
+		u64_stats_update_begin(&stats->syncp);
+		stats->tx_packets++;
+		stats->tx_bytes += skb->len;
+		u64_stats_update_end(&stats->syncp);
+
+		total_bytes += skb->len;
+		napi_consume_skb(skb, budget);
+		num_tx++;
+	}
+
+	if (!num_tx)
+		return 0;
+
+	netif_txq = netdev_get_tx_queue(ndev, chn - 1);
+
+	netdev_tx_completed_queue(netif_txq, num_tx, total_bytes);
+
+	if (netif_tx_queue_stopped(netif_txq)) {
+		/* Check whether the queue is stopped due to stalled tx dma,
+		 * if the queue is stopped then wake the queue as
+		 * we have free desc for tx
+		 */
+		__netif_tx_lock(netif_txq, smp_processor_id());
+		if (netif_running(ndev) &&
+		    (k3_knav_pool_avail(tx_chn->desc_pool) >= MAX_SKB_FRAGS))
+			netif_tx_wake_queue(netif_txq);
+
+		__netif_tx_unlock(netif_txq);
+	}
+	dev_dbg(dev, "%s:%u pkt:%d\n", __func__, chn, num_tx);
+
+	return num_tx;
+}
+
+static int am65_cpsw_nuss_tx_poll(struct napi_struct *napi_tx, int budget)
+{
+	struct am65_cpsw_common *common = am65_cpsw_napi_to_common(napi_tx);
+	int cur_budget, ch, num_tx = 0;
+
+	/* process every unprocessed channel */
+	for (ch = common->tx_ch_num; ch; ch--) {
+		u32 cur_ch = ch;
+
+		cur_budget = budget - num_tx;
+
+		if (common->pf_p0_rx_ptype_rrobin) {
+			common->cur_txq = (common->cur_txq + 1) %
+					   common->tx_ch_num;
+			cur_ch = common->cur_txq + 1;
+		}
+		num_tx += am65_cpsw_nuss_tx_compl_packets(common, cur_ch,
+							  cur_budget);
+		if (num_tx >= budget)
+			break;
+	}
+
+	if (num_tx < budget) {
+		napi_complete(napi_tx);
+		enable_irq(common->tx_chns[0].irq);
+	}
+
+	return num_tx;
+}
+
+static irqreturn_t am65_cpsw_nuss_rx_irq(int irq, void *dev_id)
+{
+	struct am65_cpsw_common *common = dev_id;
+
+	disable_irq_nosync(irq);
+	napi_schedule(&common->napi_rx);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t am65_cpsw_nuss_tx_irq(int irq, void *dev_id)
+{
+	struct am65_cpsw_common *common = dev_id;
+
+	disable_irq_nosync(irq);
+	napi_schedule(&common->napi_tx);
+
+	return IRQ_HANDLED;
+}
+
+static netdev_tx_t am65_cpsw_nuss_ndo_slave_xmit(struct sk_buff *skb,
+						 struct net_device *ndev)
+{
+	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
+	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
+	struct device *dev = common->dev;
+	struct cppi5_host_desc_t *first_desc, *next_desc, *cur_desc;
+	struct am65_cpsw_tx_chn *tx_chn;
+	struct netdev_queue *netif_txq;
+	dma_addr_t desc_dma, buf_dma;
+	int ret, q_idx, i;
+	u32 pkt_len;
+	void **swdata;
+	u32 *psdata;
+
+	/* frag list based linkage is not supported for now. */
+	if (skb_shinfo(skb)->frag_list) {
+		dev_err_ratelimited(dev, "NETIF_F_FRAGLIST not supported\n");
+		ret = -EINVAL;
+		goto drop_free_skb;
+	}
+
+	/* padding enabled in hw */
+	pkt_len = skb_headlen(skb);
+
+	/* SKB TX timestamp */
+	if (port->tx_ts_enabled)
+		am65_cpts_ask_tx_timestamp(common->cpts, skb);
+
+	q_idx = skb_get_queue_mapping(skb);
+	dev_dbg(dev, "%s skb_queue:%d\n", __func__, q_idx);
+	q_idx = q_idx % common->tx_ch_num;
+
+	tx_chn = &common->tx_chns[q_idx];
+	netif_txq = netdev_get_tx_queue(ndev, q_idx);
+
+	/* Map the linear buffer */
+	buf_dma = dma_map_single(dev, skb->data, pkt_len,
+				 DMA_TO_DEVICE);
+	if (unlikely(dma_mapping_error(dev, buf_dma))) {
+		dev_err(dev, "Failed to map tx skb buffer\n");
+		ret = -EINVAL;
+		ndev->stats.tx_errors++;
+		goto drop_stop_q;
+	}
+
+	first_desc = k3_knav_pool_alloc(tx_chn->desc_pool);
+	if (!first_desc) {
+		dev_dbg(dev, "Failed to allocate descriptor\n");
+		dma_unmap_single(dev, buf_dma, pkt_len, DMA_TO_DEVICE);
+		goto drop_stop_q_busy;
+	}
+
+	cppi5_hdesc_init(first_desc, CPPI5_INFO0_HDESC_EPIB_PRESENT,
+			 AM65_CPSW_NAV_PS_DATA_SIZE);
+	cppi5_desc_set_pktids(&first_desc->hdr, 0, 0x3FFF);
+	cppi5_hdesc_set_pkttype(first_desc, 0x7);
+	cppi5_desc_set_tags_ids(&first_desc->hdr, 0, port->port_id);
+
+	cppi5_hdesc_attach_buf(first_desc, buf_dma, pkt_len, buf_dma, pkt_len);
+	swdata = cppi5_hdesc_get_swdata(first_desc);
+	*(swdata) = skb;
+	psdata = cppi5_hdesc_get_psdata32(first_desc);
+
+	/* HW csum offload if enabled */
+	psdata[2] = 0;
+	if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
+		unsigned int cs_start, cs_offset;
+
+		cs_start = skb_transport_offset(skb);
+		cs_offset = cs_start + skb->csum_offset;
+		/* HW numerates bytes starting from 1 */
+		psdata[2] = ((cs_offset + 1) << 24) |
+			    ((cs_start + 1) << 16) | (skb->len - cs_start);
+		dev_dbg(dev, "%s tx psdata:%#x\n", __func__, psdata[2]);
+	}
+
+	if (!skb_is_nonlinear(skb))
+		goto done_tx;
+
+	dev_dbg(dev, "fragmented SKB\n");
+
+	/* Handle the case where skb is fragmented in pages */
+	cur_desc = first_desc;
+	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
+		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
+		u32 frag_size = skb_frag_size(frag);
+
+		next_desc = k3_knav_pool_alloc(tx_chn->desc_pool);
+
+		if (!next_desc) {
+			dev_err(dev, "Failed to allocate descriptor\n");
+			ret = -ENOMEM;
+			goto drop_free_descs;
+		}
+
+		buf_dma = skb_frag_dma_map(dev, frag, 0, frag_size,
+					   DMA_TO_DEVICE);
+		if (unlikely(dma_mapping_error(dev, buf_dma))) {
+			dev_err(dev, "Failed to map tx skb page\n");
+			k3_knav_pool_free(tx_chn->desc_pool, next_desc);
+			ret = -EINVAL;
+			ndev->stats.tx_errors++;
+			goto drop_free_descs;
+		}
+
+		cppi5_hdesc_reset_hbdesc(next_desc);
+		cppi5_hdesc_attach_buf(next_desc,
+				       buf_dma, frag_size, buf_dma, frag_size);
+
+		desc_dma = k3_knav_pool_virt2dma(tx_chn->desc_pool, next_desc);
+		cppi5_hdesc_link_hbdesc(cur_desc, desc_dma);
+
+		pkt_len += frag_size;
+		cur_desc = next_desc;
+	}
+	WARN_ON(pkt_len != skb->len);
+
+done_tx:
+	skb_tx_timestamp(skb);
+
+	/* report bql before sending packet */
+	netdev_tx_sent_queue(netif_txq, pkt_len);
+
+	cppi5_hdesc_set_pktlen(first_desc, pkt_len);
+	desc_dma = k3_knav_pool_virt2dma(tx_chn->desc_pool, first_desc);
+	ret = k3_nav_udmax_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma);
+	if (ret) {
+		dev_err(dev, "can't push desc %d\n", ret);
+		ndev->stats.tx_errors++;
+		goto drop_free_descs;
+	}
+
+	if (k3_knav_pool_avail(tx_chn->desc_pool) < MAX_SKB_FRAGS) {
+		netif_tx_stop_queue(netif_txq);
+		/* Barrier, so that stop_queue visible to other cpus */
+		smp_mb__after_atomic();
+		dev_err(dev, "netif_tx_stop_queue %d\n", q_idx);
+
+		/* re-check for smp */
+		if (k3_knav_pool_avail(tx_chn->desc_pool) >= MAX_SKB_FRAGS) {
+			netif_tx_wake_queue(netif_txq);
+			dev_err(dev, "netif_tx_wake_queue %d\n", q_idx);
+		}
+	}
+
+	return NETDEV_TX_OK;
+
+drop_free_descs:
+	am65_cpsw_nuss_xmit_free(tx_chn, dev, first_desc);
+drop_stop_q:
+	netif_tx_stop_queue(netif_txq);
+drop_free_skb:
+	ndev->stats.tx_dropped++;
+	dev_kfree_skb_any(skb);
+	return ret;
+
+drop_stop_q_busy:
+	netif_tx_stop_queue(netif_txq);
+	return NETDEV_TX_BUSY;
+}
+
+static int am65_cpsw_nuss_ndo_slave_set_mac_address(struct net_device *ndev,
+						    void *addr)
+{
+	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
+	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
+	struct sockaddr *sockaddr = (struct sockaddr *)addr;
+	int ret;
+
+	ret = eth_prepare_mac_addr_change(ndev, addr);
+	if (ret < 0)
+		return ret;
+
+	ret = pm_runtime_get_sync(common->dev);
+	if (ret < 0) {
+		pm_runtime_put_noidle(common->dev);
+		return ret;
+	}
+
+	cpsw_ale_del_ucast(common->ale, ndev->dev_addr,
+			   HOST_PORT_NUM, 0, 0);
+	cpsw_ale_add_ucast(common->ale, sockaddr->sa_data,
+			   HOST_PORT_NUM, ALE_SECURE, 0);
+
+	am65_cpsw_port_set_sl_mac(port, addr);
+	eth_commit_mac_addr_change(ndev, sockaddr);
+
+	pm_runtime_put(common->dev);
+
+	return 0;
+}
+
+#if IS_ENABLED(CONFIG_TI_AM65_CPTS)
+static int am65_cpsw_nuss_hwtstamp_set(struct net_device *ndev,
+				       struct ifreq *ifr)
+{
+	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
+	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
+	struct hwtstamp_config cfg;
+	u32 ts_ctrl, seq_id, ts_ctrl_ltype2, ts_vlan_ltype;
+
+	if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
+		return -EFAULT;
+
+	if (cfg.tx_type != HWTSTAMP_TX_OFF && cfg.tx_type != HWTSTAMP_TX_ON)
+		return -ERANGE;
+
+	switch (cfg.rx_filter) {
+	case HWTSTAMP_FILTER_NONE:
+		port->rx_ts_enabled = false;
+		break;
+	case HWTSTAMP_FILTER_ALL:
+	case HWTSTAMP_FILTER_SOME:
+	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
+	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
+	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
+	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
+	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
+	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
+	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
+	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
+	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
+	case HWTSTAMP_FILTER_PTP_V2_EVENT:
+	case HWTSTAMP_FILTER_PTP_V2_SYNC:
+	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
+	case HWTSTAMP_FILTER_NTP_ALL:
+		port->rx_ts_enabled = true;
+		cfg.rx_filter = HWTSTAMP_FILTER_ALL;
+		break;
+	default:
+		return -ERANGE;
+	}
+
+	port->tx_ts_enabled = (cfg.tx_type == HWTSTAMP_TX_ON);
+
+	/* cfg TX timestamp */
+	seq_id = (AM65_CPSW_TS_SEQ_ID_OFFSET <<
+		  AM65_CPSW_PN_TS_SEQ_ID_OFFSET_SHIFT) | ETH_P_1588;
+
+	ts_vlan_ltype = ETH_P_8021Q;
+
+	ts_ctrl_ltype2 = ETH_P_1588 |
+			 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_107 |
+			 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_129 |
+			 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_130 |
+			 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_131 |
+			 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_132 |
+			 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_319 |
+			 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_320 |
+			 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_TTL_NONZERO;
+
+	ts_ctrl = AM65_CPSW_TS_EVENT_MSG_TYPE_BITS <<
+		  AM65_CPSW_PN_TS_CTL_MSG_TYPE_EN_SHIFT;
+
+	if (port->tx_ts_enabled)
+		ts_ctrl |= AM65_CPSW_TS_TX_ANX_ALL_EN |
+			   AM65_CPSW_PN_TS_CTL_TX_VLAN_LT1_EN;
+
+	writel(seq_id, port->port_base + AM65_CPSW_PORTN_REG_TS_SEQ_LTYPE_REG);
+	writel(ts_vlan_ltype, port->port_base +
+	       AM65_CPSW_PORTN_REG_TS_VLAN_LTYPE_REG);
+	writel(ts_ctrl_ltype2, port->port_base +
+	       AM65_CPSW_PORTN_REG_TS_CTL_LTYPE2);
+	writel(ts_ctrl, port->port_base + AM65_CPSW_PORTN_REG_TS_CTL);
+
+	/* en/dis RX timestamp */
+	am65_cpts_rx_enable(common->cpts, port->rx_ts_enabled);
+
+	return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
+}
+
+static int am65_cpsw_nuss_hwtstamp_get(struct net_device *ndev,
+				       struct ifreq *ifr)
+{
+	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
+	struct hwtstamp_config cfg;
+
+	cfg.flags = 0;
+	cfg.tx_type = port->tx_ts_enabled ?
+		      HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF;
+	cfg.rx_filter = port->rx_ts_enabled ?
+			HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
+
+	return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
+}
+#else
+static int am65_cpsw_nuss_hwtstamp_get(struct net_device *ndev,
+				       struct ifreq *ifr)
+{
+	return -EOPNOTSUPP;
+}
+
+static int am65_cpsw_nuss_hwtstamp_set(struct net_device *ndev,
+				       struct ifreq *ifr)
+{
+	return -EOPNOTSUPP;
+}
+#endif /* CONFIG_TI_AM65_CPTS */
+
+static int am65_cpsw_switch_config_ioctl(struct net_device *ndev,
+					 struct ifreq *ifrq, int cmd)
+{
+	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
+	struct net_switch_config config;
+	int ret = -EINVAL;
+
+	/* Only SIOCSWITCHCONFIG is used as cmd argument and hence, there is no
+	 * switch statement required.
+	 * Function calls are based on switch_config.cmd
+	 */
+
+	if (copy_from_user(&config, ifrq->ifr_data, sizeof(config)))
+		return -EFAULT;
+
+	switch (config.cmd) {
+	case CONFIG_SWITCH_RATELIMIT:
+	{
+		if (config.port > 1) {
+			dev_err(common->dev, "Invalid Port number\n");
+			break;
+		}
+
+		ret = cpsw_ale_set_ratelimit(common->ale,
+					     common->bus_freq_mhz * 1000000,
+					     config.port,
+					     config.bcast_rate_limit,
+					     config.mcast_rate_limit,
+					     !!config.direction);
+		if (ret)
+			dev_err(common->dev, "CPSW_ALE set ratelimit failed");
+		break;
+	}
+
+	default:
+		ret = -EOPNOTSUPP;
+	}
+
+	return ret;
+}
+
+static int am65_cpsw_nuss_ndo_slave_ioctl(struct net_device *ndev,
+					  struct ifreq *req, int cmd)
+{
+	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
+
+	if (!netif_running(ndev))
+		return -EINVAL;
+
+	switch (cmd) {
+	case SIOCSHWTSTAMP:
+		return am65_cpsw_nuss_hwtstamp_set(ndev, req);
+	case SIOCGHWTSTAMP:
+		return am65_cpsw_nuss_hwtstamp_get(ndev, req);
+	case SIOCSWITCHCONFIG:
+		return am65_cpsw_switch_config_ioctl(ndev, req, cmd);
+	}
+
+	if (!port->slave.phy)
+		return -EOPNOTSUPP;
+
+	return phy_mii_ioctl(port->slave.phy, req, cmd);
+}
+
+static void am65_cpsw_nuss_ndo_get_stats(struct net_device *dev,
+					 struct rtnl_link_stats64 *stats)
+{
+	struct am65_cpsw_ndev_priv *ndev_priv = netdev_priv(dev);
+	unsigned int start;
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+		struct am65_cpsw_ndev_stats *cpu_stats;
+		u64 rx_packets;
+		u64 rx_bytes;
+		u64 tx_packets;
+		u64 tx_bytes;
+
+		cpu_stats = per_cpu_ptr(ndev_priv->stats, cpu);
+		do {
+			start = u64_stats_fetch_begin_irq(&cpu_stats->syncp);
+			rx_packets = cpu_stats->rx_packets;
+			rx_bytes   = cpu_stats->rx_bytes;
+			tx_packets = cpu_stats->tx_packets;
+			tx_bytes   = cpu_stats->tx_bytes;
+		} while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start));
+
+		stats->rx_packets += rx_packets;
+		stats->rx_bytes   += rx_bytes;
+		stats->tx_packets += tx_packets;
+		stats->tx_bytes   += tx_bytes;
+	}
+
+	stats->rx_errors	= dev->stats.rx_errors;
+	stats->rx_dropped	= dev->stats.rx_dropped;
+	stats->tx_dropped	= dev->stats.tx_dropped;
+}
+
+static int am65_cpsw_nuss_ndo_slave_set_features(struct net_device *ndev,
+						 netdev_features_t features)
+{
+	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
+	struct am65_cpsw_port *host_p = am65_common_get_host_port(common);
+	netdev_features_t changes = features ^ ndev->features;
+
+	if (changes & NETIF_F_HW_CSUM) {
+		bool enable = !!(features & NETIF_F_HW_CSUM);
+
+		dev_err(common->dev, "Turn %s tx-checksum-ip-generic\n",
+			enable ? "ON" : "OFF");
+		if (enable)
+			writel(AM65_CPSW_P0_REG_CTL_RX_CHECKSUM_EN,
+			       host_p->port_base + AM65_CPSW_P0_REG_CTL);
+		else
+			writel(0,
+			       host_p->port_base + AM65_CPSW_P0_REG_CTL);
+	}
+
+	return 0;
+}
+
+static const struct net_device_ops am65_cpsw_nuss_netdev_ops_2g = {
+	.ndo_open		= am65_cpsw_nuss_ndo_slave_open,
+	.ndo_stop		= am65_cpsw_nuss_ndo_slave_stop,
+	.ndo_start_xmit		= am65_cpsw_nuss_ndo_slave_xmit,
+	.ndo_set_rx_mode	= am65_cpsw_nuss_ndo_slave_set_rx_mode,
+	.ndo_get_stats64        = am65_cpsw_nuss_ndo_get_stats,
+	.ndo_validate_addr	= eth_validate_addr,
+	.ndo_set_mac_address	= am65_cpsw_nuss_ndo_slave_set_mac_address,
+	.ndo_tx_timeout		= am65_cpsw_nuss_ndo_host_tx_timeout,
+	.ndo_vlan_rx_add_vid	= am65_cpsw_nuss_ndo_slave_add_vid,
+	.ndo_vlan_rx_kill_vid	= am65_cpsw_nuss_ndo_slave_kill_vid,
+	.ndo_do_ioctl		= am65_cpsw_nuss_ndo_slave_ioctl,
+	.ndo_set_features	= am65_cpsw_nuss_ndo_slave_set_features,
+};
+
+static void am65_cpsw_nuss_slave_disable_unused(struct am65_cpsw_port *port)
+{
+	struct am65_cpsw_common *common = port->common;
+
+	if (!port->disabled)
+		return;
+
+	common->disabled_ports_mask |= BIT(port->port_id);
+	cpsw_ale_control_set(common->ale, port->port_id,
+			     ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
+
+	cpsw_sl_reset(port->slave.mac_sl, 100);
+	cpsw_sl_ctl_reset(port->slave.mac_sl);
+}
+
+static void am65_cpsw_nuss_free_tx_chns(void *data)
+{
+	struct am65_cpsw_common *common = data;
+	int i;
+
+	for (i = 0; i < AM65_CPSW_MAX_TX_QUEUES; i++) {
+		struct am65_cpsw_tx_chn	*tx_chn = &common->tx_chns[i];
+
+		if (tx_chn->irq > 0)
+			k3_nav_udmax_tx_put_irq(tx_chn->tx_chn);
+
+		if (!IS_ERR_OR_NULL(tx_chn->tx_chn))
+			k3_nav_udmax_release_tx_chn(tx_chn->tx_chn);
+
+		if (!IS_ERR_OR_NULL(tx_chn->desc_pool))
+			k3_knav_pool_destroy(tx_chn->desc_pool);
+
+		memset(tx_chn, 0, sizeof(*tx_chn));
+	};
+}
+
+static int am65_cpsw_nuss_init_tx_chns(struct am65_cpsw_common *common)
+{
+	struct device *dev = common->dev;
+	u32  max_desc_num = ALIGN(AM65_CPSW_MAX_TX_DESC, MAX_SKB_FRAGS);
+	struct k3_nav_udmax_tx_channel_cfg tx_cfg = {0};
+	struct k3_ring_cfg ring_cfg = {
+		.elm_size = K3_RINGACC_RING_ELSIZE_8,
+		.mode = K3_RINGACC_RING_MODE_RING,
+		.flags = 0
+	};
+	u32 hdesc_size;
+	int i, ret;
+
+	init_completion(&common->tdown_complete);
+
+	hdesc_size = cppi5_hdesc_calc_size(true, AM65_CPSW_NAV_PS_DATA_SIZE,
+					   AM65_CPSW_NAV_SW_DATA_SIZE);
+
+	tx_cfg.swdata_size = AM65_CPSW_NAV_SW_DATA_SIZE;
+	tx_cfg.tx_cfg = ring_cfg;
+	tx_cfg.txcq_cfg = ring_cfg;
+	tx_cfg.tx_cfg.size = max_desc_num;
+	tx_cfg.txcq_cfg.size = max_desc_num;
+
+	for (i = 0; i < AM65_CPSW_MAX_TX_QUEUES; i++) {
+		struct am65_cpsw_tx_chn	*tx_chn = &common->tx_chns[i];
+		char	tx_chn_name[IFNAMSIZ];
+
+		snprintf(tx_chn_name, sizeof(tx_chn_name), "tx%d", i);
+
+		tx_chn->dev = dev;
+		tx_chn->id = i;
+		tx_chn->descs_num = max_desc_num;
+		tx_chn->desc_pool = k3_knav_pool_create_name(dev,
+							     tx_chn->descs_num,
+							     hdesc_size,
+							     tx_chn_name);
+		if (IS_ERR(tx_chn->desc_pool)) {
+			ret = PTR_ERR(tx_chn->desc_pool);
+			dev_err(dev, "Failed to create poll %d\n", ret);
+			goto err;
+		}
+
+		tx_chn->tx_chn =
+			k3_nav_udmax_request_tx_chn(dev, tx_chn_name, &tx_cfg);
+		if (IS_ERR(tx_chn->tx_chn)) {
+			ret = PTR_ERR(tx_chn->tx_chn);
+			dev_err(dev, "Failed to request tx dma channel %d\n",
+				ret);
+			goto err;
+		}
+		ret = k3_nav_udmax_tx_get_irq(
+				tx_chn->tx_chn, &tx_chn->irq,
+				IRQF_TRIGGER_HIGH, true,
+				i ? common->tx_chns[0].tx_chn : NULL);
+		if (ret) {
+			dev_err(dev, "Failed to get tx dma irq %d\n", ret);
+			goto err;
+		}
+	}
+
+err:
+	i = devm_add_action(dev, am65_cpsw_nuss_free_tx_chns, common);
+	if (i) {
+		dev_err(dev, "failed to add free_tx_chns action %d", i);
+		return i;
+	}
+
+	return ret;
+}
+
+static void am65_cpsw_nuss_free_rx_chns(void *data)
+{
+	struct am65_cpsw_common *common = data;
+	struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
+	int i;
+
+	if (!IS_ERR_OR_NULL(rx_chn->rx_chn)) {
+		for (i = 0; i < AM65_CPSW_MAX_RX_FLOWS; i++)
+			k3_nav_udmax_rx_put_irq(rx_chn->rx_chn, i);
+
+		k3_nav_udmax_release_rx_chn(rx_chn->rx_chn);
+	}
+
+	if (!IS_ERR_OR_NULL(rx_chn->desc_pool))
+		k3_knav_pool_destroy(rx_chn->desc_pool);
+}
+
+static int am65_cpsw_nuss_init_rx_chns(struct am65_cpsw_common *common)
+{
+	struct device *dev = common->dev;
+	u32  max_desc_num = AM65_CPSW_MAX_RX_DESC;
+	struct k3_nav_udmax_rx_channel_cfg rx_cfg = {0};
+	u32 hdesc_size;
+	u32 fdqring_id;
+	int i, ret = 0;
+	struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
+
+	/* TODO: we need to calc total number of flows required
+	 * N = n_mac_only*8 [+ 8 if switch ports] rest can be used for class
+	 */
+
+	hdesc_size = cppi5_hdesc_calc_size(true, AM65_CPSW_NAV_PS_DATA_SIZE,
+					   AM65_CPSW_NAV_SW_DATA_SIZE);
+
+	rx_cfg.swdata_size = AM65_CPSW_NAV_SW_DATA_SIZE;
+	rx_cfg.flow_id_num = AM65_CPSW_MAX_RX_FLOWS;
+	rx_cfg.flow_id_base = common->rx_flow_id_base;
+
+	/* init all flows */
+	rx_chn->dev = dev;
+	rx_chn->descs_num = max_desc_num;
+	rx_chn->desc_pool = k3_knav_pool_create_name(dev, rx_chn->descs_num,
+						     hdesc_size, "rx");
+	if (IS_ERR(rx_chn->desc_pool)) {
+		ret = PTR_ERR(rx_chn->desc_pool);
+		dev_err(dev, "Failed to create rx poll %d\n", ret);
+		goto err;
+	}
+
+	rx_chn->rx_chn = k3_nav_udmax_request_rx_chn(dev, "rx", &rx_cfg);
+	if (IS_ERR(rx_chn->rx_chn)) {
+		ret = PTR_ERR(rx_chn->rx_chn);
+		dev_err(dev, "Failed to request rx dma channel %d\n", ret);
+		goto err;
+	}
+
+	common->rx_flow_id_base =
+			k3_nav_udmax_rx_get_flow_id_base(rx_chn->rx_chn);
+	dev_info(dev, "set new flow-id-base %u\n", common->rx_flow_id_base);
+
+	fdqring_id = K3_RINGACC_RING_ID_ANY;
+	for (i = 0; i < rx_cfg.flow_id_num; i++) {
+		struct k3_ring_cfg rxring_cfg = {
+			.elm_size = K3_RINGACC_RING_ELSIZE_8,
+			.mode = K3_RINGACC_RING_MODE_RING,
+			.flags = 0,
+		};
+		struct k3_ring_cfg fdqring_cfg = {
+			.elm_size = K3_RINGACC_RING_ELSIZE_8,
+			.mode = K3_RINGACC_RING_MODE_MESSAGE,
+			.flags = K3_RINGACC_RING_SHARED,
+		};
+		struct k3_nav_udmax_rx_flow_cfg rx_flow_cfg = {
+			.rx_cfg = rxring_cfg,
+			.rxfdq_cfg = fdqring_cfg,
+			.ring_rxq_id = K3_RINGACC_RING_ID_ANY,
+			.src_tag_lo_sel =
+				K3_NAV_UDMAX_SRC_TAG_LO_USE_REMOTE_SRC_TAG,
+		};
+
+		rx_flow_cfg.ring_rxfdq0_id = fdqring_id;
+		rx_flow_cfg.rx_cfg.size = max_desc_num;
+		rx_flow_cfg.rxfdq_cfg.size = max_desc_num;
+
+		ret = k3_nav_udmax_rx_flow_init(rx_chn->rx_chn,
+						i, &rx_flow_cfg);
+		if (ret) {
+			dev_err(dev, "Failed to init rx flow%d %d\n", i, ret);
+			goto err;
+		}
+		if (!i)
+			fdqring_id = k3_nav_udmax_rx_flow_get_fdq_id(
+					rx_chn->rx_chn, i);
+
+		ret = k3_nav_udmax_rx_get_irq(rx_chn->rx_chn, i, &rx_chn->irq,
+					      IRQF_TRIGGER_HIGH,
+					      true, i ? 0 : -1);
+		if (ret) {
+			dev_err(dev, "Failed to get rx dma irq %d\n", ret);
+			goto err;
+		}
+	}
+
+err:
+	i = devm_add_action(dev, am65_cpsw_nuss_free_rx_chns, common);
+	if (i) {
+		dev_err(dev, "failed to add free_rx_chns action %d", i);
+		return i;
+	}
+
+	return ret;
+}
+
+static int am65_cpsw_nuss_init_host_p(struct am65_cpsw_common *common)
+{
+	struct am65_cpsw_port *host_p = am65_common_get_host_port(common);
+	struct device *dev = common->dev;
+	struct device_node *node, *host_np;
+
+	host_p->common = common;
+
+	node = of_get_child_by_name(dev->of_node, "ports");
+	if (!node)
+		return -ENOENT;
+	host_np = of_get_child_by_name(node, "host");
+	if (!host_np)
+		return -ENOENT;
+
+	host_p->name = of_get_property(host_np, "ti,label", NULL);
+
+	of_node_put(host_np);
+	of_node_put(node);
+
+	host_p->port_base = common->cpsw_base + AM65_CPSW_NU_PORTS_BASE;
+	host_p->stat_base = common->cpsw_base + AM65_CPSW_NU_STATS_BASE;
+
+	return 0;
+}
+
+static int am65_cpsw_init_mdio(struct am65_cpsw_common *common)
+{
+	struct device *dev = common->dev;
+	struct device_node *node;
+	struct davinci_mdio_data *mdio;
+	void __iomem *reg_base;
+
+	node = of_get_child_by_name(dev->of_node, "mdio");
+	if (!node)
+		return -ENOENT;
+
+	reg_base = common->ss_base + AM65_CPSW_MDIO_BASE;
+	mdio = davinci_mdio_create(dev, node, reg_base, "fck");
+	if (IS_ERR(mdio)) {
+		of_node_put(node);
+		dev_err(dev, "MDIO init err %ld\n", PTR_ERR(mdio));
+		return PTR_ERR(mdio);
+	}
+	common->mdio = mdio;
+
+	return 0;
+}
+
+static int am65_cpsw_am654_get_efuse_macid(struct device_node *of_node,
+					   int slave, u8 *mac_addr)
+{
+	struct regmap *syscon;
+	u32 mac_lo, mac_hi, offset;
+	int ret;
+
+	syscon = syscon_regmap_lookup_by_phandle(of_node, "ti,syscon-efuse");
+	if (IS_ERR(syscon)) {
+		if (PTR_ERR(syscon) == -ENODEV)
+			return 0;
+		return PTR_ERR(syscon);
+	}
+
+	ret = of_property_read_u32_index(of_node, "ti,syscon-efuse", 1,
+					 &offset);
+	if (ret)
+		return ret;
+
+	regmap_read(syscon, offset, &mac_lo);
+	regmap_read(syscon, offset + 4, &mac_hi);
+
+	mac_addr[0] = (mac_hi >> 8) & 0xff;
+	mac_addr[1] = mac_hi & 0xff;
+	mac_addr[2] = (mac_lo >> 24) & 0xff;
+	mac_addr[3] = (mac_lo >> 16) & 0xff;
+	mac_addr[4] = (mac_lo >> 8) & 0xff;
+	mac_addr[5] = mac_lo & 0xff;
+
+	return 0;
+}
+
+static int am65_cpsw_init_cpts(struct am65_cpsw_common *common)
+{
+	struct device *dev = common->dev;
+	struct device_node *node;
+	struct am65_cpts *cpts;
+	void __iomem *reg_base;
+
+	node = of_get_child_by_name(dev->of_node, "cpts");
+	if (!node) {
+		dev_err(dev, "%s cpts not found\n", __func__);
+		return -ENOENT;
+	}
+
+	reg_base = common->cpsw_base + AM65_CPSW_NU_CPTS_BASE;
+	cpts = am65_cpts_create(dev, reg_base, node);
+	if (IS_ERR(cpts)) {
+		int ret = PTR_ERR(cpts);
+
+		if (ret == -EOPNOTSUPP) {
+			dev_info(dev, "cpts disabled\n");
+			return 0;
+		}
+
+		dev_err(dev, "cpts create err %d\n", ret);
+		return ret;
+	}
+	common->cpts = cpts;
+
+	return 0;
+}
+
+static int am65_cpsw_nuss_init_slave_ports(struct am65_cpsw_common *common)
+{
+	struct device *dev = common->dev;
+	struct device_node *node, *port_np;
+	int ret;
+
+	node = of_get_child_by_name(dev->of_node, "ports");
+	if (!node)
+		return -ENOENT;
+
+	for_each_child_of_node(node, port_np) {
+		struct am65_cpsw_port *port;
+		const void *mac_addr;
+		u32 port_id;
+
+		/* it is not a slave port node, continue */
+		if (strcmp(port_np->name, "port"))
+			continue;
+
+		ret = of_property_read_u32(port_np, "reg", &port_id);
+		if (ret < 0) {
+			dev_err(dev, "%pOF error reading port_id %d\n",
+				port_np, ret);
+			return ret;
+		}
+
+		if (!port_id || port_id >= common->port_num) {
+			dev_err(dev, "%pOF has invalid port_id %u %s\n",
+				port_np, port_id, port_np->name);
+			return -EINVAL;
+		}
+
+		port = &common->ports[port_id];
+		port->port_id = port_id;
+		port->common = common;
+		port->port_base = common->cpsw_base + AM65_CPSW_NU_PORTS_BASE +
+				  AM65_CPSW_NU_PORTS_OFFSET * (port_id);
+		port->stat_base = common->cpsw_base + AM65_CPSW_NU_STATS_BASE +
+				  (AM65_CPSW_NU_STATS_PORT_OFFSET * port_id);
+		port->name = of_get_property(port_np, "ti,label", NULL);
+
+		port->disabled = !of_device_is_available(port_np);
+		if (port->disabled)
+			continue;
+
+		port->slave.mac_only =
+				of_property_read_bool(port_np, "ti,mac-only");
+
+		/* get phy/link info */
+		if (of_phy_is_fixed_link(port_np)) {
+			ret = of_phy_register_fixed_link(port_np);
+			if (ret) {
+				if (ret != -EPROBE_DEFER)
+					dev_err(dev, "%pOF failed to register fixed-link phy: %d\n",
+						port_np, ret);
+				return ret;
+			}
+			port->slave.phy_node = of_node_get(port_np);
+		} else {
+			port->slave.phy_node =
+				of_parse_phandle(port_np, "phy-handle", 0);
+		}
+
+		if (!port->slave.phy_node) {
+			dev_err(dev,
+				"slave[%d] no phy found\n", port_id);
+			return -ENODEV;
+		}
+
+		port->slave.phy_if = of_get_phy_mode(port_np);
+		if (port->slave.phy_if < 0) {
+			dev_err(dev, "%pOF read phy-mode err %d\n",
+				port_np, port->slave.phy_if);
+			return port->slave.phy_if;
+		}
+
+		port->slave.mac_sl = cpsw_sl_get("am65", dev, port->port_base);
+		if (IS_ERR(port->slave.mac_sl))
+			return PTR_ERR(port->slave.mac_sl);
+
+		mac_addr = of_get_mac_address(port_np);
+		if (mac_addr && is_valid_ether_addr(mac_addr)) {
+			ether_addr_copy(port->slave.mac_addr, mac_addr);
+		} else if (am65_cpsw_am654_get_efuse_macid(
+				port_np, port->port_id, port->slave.mac_addr) ||
+			   !is_valid_ether_addr(port->slave.mac_addr)) {
+			random_ether_addr(port->slave.mac_addr);
+			dev_err(dev, "Use rundom MAC address\n");
+		}
+
+		/* alloc netdev */
+		port->ndev = devm_alloc_etherdev_mqs(
+				common->dev,
+				sizeof(struct am65_cpsw_ndev_priv),
+				common->tx_ch_num, common->rx_ch_num);
+		if (!port->ndev) {
+			dev_err(dev, "error allocating slave net_device %u\n",
+				port_id);
+			return -ENOMEM;
+		}
+	}
+	of_node_put(node);
+
+	return 0;
+}
+
+static int am65_cpsw_nuss_init_ndev_2g(struct am65_cpsw_common *common)
+{
+	struct device *dev = common->dev;
+	struct am65_cpsw_ndev_priv *ndev_priv;
+	struct am65_cpsw_port *port;
+	int ret;
+
+	port = &common->ports[1];
+
+	/* alloc netdev */
+	port->ndev = devm_alloc_etherdev_mqs(common->dev,
+					     sizeof(struct am65_cpsw_ndev_priv),
+					     common->tx_ch_num,
+					     common->rx_ch_num);
+	if (!port->ndev) {
+		dev_err(dev, "error allocating slave net_device %u\n",
+			port->port_id);
+		return -ENOMEM;
+	}
+
+	ndev_priv = netdev_priv(port->ndev);
+	ndev_priv->port = port;
+	ndev_priv->msg_enable = netif_msg_init(am65_cpsw_debug_level,
+					       AM65_CPSW_DEBUG);
+	SET_NETDEV_DEV(port->ndev, dev);
+
+	ether_addr_copy(port->ndev->dev_addr, port->slave.mac_addr);
+
+	port->ndev->min_mtu = AM65_CPSW_MIN_PACKET_SIZE;
+	port->ndev->max_mtu = AM65_CPSW_MAX_PACKET_SIZE;
+	port->ndev->hw_features = NETIF_F_SG |
+				  NETIF_F_RXCSUM |
+				  NETIF_F_HW_CSUM;
+	port->ndev->features = port->ndev->hw_features |
+			       NETIF_F_HW_VLAN_CTAG_FILTER;
+	port->ndev->vlan_features |=  NETIF_F_SG;
+	port->ndev->netdev_ops = &am65_cpsw_nuss_netdev_ops_2g;
+	port->ndev->ethtool_ops = &am65_cpsw_ethtool_ops_slave;
+
+	/* Disable TX checksum offload by default due to HW bug */
+	if (common->pdata->quirks & AM65_CPSW_QUIRK_I2027_NO_TX_CSUM)
+		port->ndev->features &= ~NETIF_F_HW_CSUM;
+
+	ndev_priv->stats = netdev_alloc_pcpu_stats(struct am65_cpsw_ndev_stats);
+	if (!ndev_priv->stats)
+		return -ENOMEM;
+
+	ret = devm_add_action_or_reset(dev, (void(*)(void *))free_percpu,
+				       ndev_priv->stats);
+	if (ret) {
+		dev_err(dev, "failed to add percpu stat free action %d", ret);
+		return ret;
+	}
+
+	netif_tx_napi_add(port->ndev, &common->napi_tx,
+			  am65_cpsw_nuss_tx_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(port->ndev, &common->napi_rx,
+		       am65_cpsw_nuss_rx_poll, NAPI_POLL_WEIGHT);
+
+	common->pf_p0_rx_ptype_rrobin = true;
+	ret = register_netdev(port->ndev);
+	if (ret)
+		dev_err(dev, "error registering slave net device %d\n", ret);
+
+	/* can't auto unregister ndev using devm_add_action() due to broken
+	 * devres release sequence in DD core
+	 */
+
+	return ret;
+}
+
+static void am65_cpsw_nuss_cleanup_ndev(struct am65_cpsw_common *common)
+{
+	struct am65_cpsw_port *port;
+	int i;
+
+	for (i = 0; i < common->port_num; i++) {
+		port = &common->ports[i];
+		if (port->ndev)
+			unregister_netdev(port->ndev);
+	}
+}
+
+static const struct am65_cpsw_pdata am65x_sr1_0 = {
+	.quirks = AM65_CPSW_QUIRK_I2027_NO_TX_CSUM,
+};
+
+static const struct am65_cpsw_pdata j721e_sr1_0 = {
+	.quirks = 0,
+};
+
+static const struct of_device_id am65_cpsw_nuss_of_mtable[] = {
+	{ .compatible = "ti,am654-cpsw-nuss", .data = &am65x_sr1_0 },
+	{ .compatible = "ti,j721e-cpsw-nuss", .data = &j721e_sr1_0 },
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, am65_cpsw_nuss_of_mtable);
+
+static int am65_cpsw_nuss_probe(struct platform_device *pdev)
+{
+	const struct of_device_id *of_id;
+	struct cpsw_ale_params ale_params;
+	struct device *dev = &pdev->dev;
+	struct am65_cpsw_common *common;
+	struct device_node *node;
+	struct resource *res;
+	struct clk *clk;
+	int ret, i;
+
+	common = devm_kzalloc(dev, sizeof(struct am65_cpsw_common), GFP_KERNEL);
+	if (!common)
+		return -ENOMEM;
+	common->dev = dev;
+
+	of_id = of_match_device(am65_cpsw_nuss_of_mtable, dev);
+	if (!of_id)
+		return -EINVAL;
+	common->pdata = of_id->data;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cpsw_nuss");
+	common->ss_base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(common->ss_base))
+		return PTR_ERR(common->ss_base);
+	common->cpsw_base = common->ss_base + AM65_CPSW_CPSW_NU_BASE;
+
+	node = of_get_child_by_name(dev->of_node, "ports");
+	if (!node)
+		return -ENOENT;
+	common->port_num = of_get_child_count(node);
+	if (common->port_num < 2 || common->port_num > AM65_CPSW_MAX_PORTS)
+		return -ENOENT;
+	of_node_put(node);
+
+	common->rx_flow_id_base = -1;
+	ret = of_property_read_u32(dev->of_node, "ti,rx-flow-id-base",
+				   &common->rx_flow_id_base);
+	if (ret < 0)
+		dev_info(dev, "rx-flow-id-base is not set %d\n", ret);
+
+	clk = devm_clk_get(dev, "fck");
+	if (IS_ERR(clk)) {
+		ret = PTR_ERR(clk);
+
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "error getting fck clock %d\n", ret);
+		return ret;
+	}
+	common->bus_freq_mhz = clk_get_rate(clk) / 1000000;
+
+	pm_runtime_enable(dev);
+	ret = devm_add_action_or_reset(dev, (void(*)(void *))pm_runtime_disable,
+				       dev);
+	if (ret) {
+		dev_err(dev, "failed to add pm reset action %d", ret);
+		return ret;
+	}
+
+	ret = pm_runtime_get_sync(dev);
+	if (ret < 0) {
+		pm_runtime_put_noidle(dev);
+		return ret;
+	}
+
+	/* register devres action here, so dev will be disabled
+	 * at right moment. It's required to enable dev in .remove() callback
+	 * unconditionally.
+	 */
+	ret = devm_add_action_or_reset(dev,
+				       (void(*)(void *))pm_runtime_put_sync,
+				       dev);
+	if (ret) {
+		dev_err(dev, "failed to add pm put reset action %d", ret);
+		return ret;
+	}
+
+	ret = of_platform_populate(dev->of_node, NULL, NULL, dev);
+	/* We do not want to force this, as in some cases may not have child */
+	if (ret)
+		dev_warn(dev, "Doesn't have any child node\n");
+
+	common->tx_ch_num = AM65_CPSW_MAX_TX_QUEUES;
+	common->rx_ch_num = AM65_CPSW_MAX_RX_QUEUES;
+
+	am65_cpsw_nuss_get_ver(common);
+
+	common->ports = devm_kcalloc(dev, common->port_num,
+				     sizeof(struct am65_cpsw_port),
+				     GFP_KERNEL);
+	if (!common->ports)
+		return -ENOMEM;
+
+	/* init tx channels */
+	ret = am65_cpsw_nuss_init_tx_chns(common);
+	if (ret)
+		return ret;
+	ret = am65_cpsw_nuss_init_rx_chns(common);
+	if (ret)
+		return ret;
+
+	if (common->tx_chns[0].irq == 0 || common->rx_chns.irq == 0)
+		return -ENXIO;
+
+	ret = am65_cpsw_nuss_init_host_p(common);
+	if (ret)
+		return ret;
+
+	ret = am65_cpsw_nuss_init_slave_ports(common);
+	if (ret)
+		return ret;
+
+	ret = am65_cpsw_init_cpts(common);
+	if (ret)
+		return ret;
+
+	/* init common data */
+	ale_params.dev = dev;
+	ale_params.ale_ageout = AM65_CPSW_ALE_AGEOUT_DEFAULT;
+	ale_params.ale_entries = 0;
+	ale_params.ale_ports = common->port_num;
+	ale_params.ale_regs = common->cpsw_base + AM65_CPSW_NU_ALE_BASE;
+	ale_params.nu_switch_ale = true;
+
+	common->ale = cpsw_ale_create(&ale_params);
+	if (!common->ale) {
+		dev_err(dev, "error initializing ale engine\n");
+		return -ENODEV;
+	}
+
+	ret = am65_cpsw_init_mdio(common);
+	if (ret)
+		return ret;
+
+	/* init ports */
+	for (i = 0; i < common->port_num; i++)
+		am65_cpsw_nuss_slave_disable_unused(&common->ports[i]);
+
+	if (common->port_num != 2)
+		return -EOPNOTSUPP;
+
+	dev_set_drvdata(dev, common);
+
+	ret = am65_cpsw_nuss_init_ndev_2g(common);
+	if (ret)
+		return ret;
+
+	ret = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(48));
+	if (ret) {
+		dev_err(dev, "error setting dma mask: %d\n", ret);
+		goto unreg_ndev;
+	}
+
+	ret = devm_request_irq(dev, common->tx_chns[0].irq,
+			       am65_cpsw_nuss_tx_irq,
+			       0, dev_name(dev), common);
+	if (ret) {
+		dev_err(dev, "failure requesting tx irq %u, %d\n",
+			common->tx_chns[0].irq, ret);
+		goto unreg_ndev;
+	}
+
+	ret = devm_request_irq(dev, common->rx_chns.irq,
+			       am65_cpsw_nuss_rx_irq,
+			       0, dev_name(dev), common);
+	if (ret) {
+		dev_err(dev, "failure requesting rx irq %u, %d\n",
+			common->rx_chns.irq, ret);
+		goto unreg_ndev;
+	}
+
+	pm_runtime_put(dev);
+	return 0;
+
+unreg_ndev:
+	am65_cpsw_nuss_cleanup_ndev(common);
+	return ret;
+}
+
+static int am65_cpsw_nuss_remove(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct am65_cpsw_common *common = dev_get_drvdata(dev);
+	int ret;
+
+	/* enable dev here it will be released later by devres action */
+	ret = pm_runtime_get_sync(&pdev->dev);
+	if (ret < 0)
+		pm_runtime_put_noidle(&pdev->dev);
+
+	/* must unregister ndevs here because DD release_driver routine calls
+	 * dma_deconfigure(dev) before devres_release_all(dev)
+	 */
+	am65_cpsw_nuss_cleanup_ndev(common);
+
+	return 0;
+}
+
+static struct platform_driver am65_cpsw_nuss_driver = {
+	.driver = {
+		.name	 = AM65_CPSW_DRV_NAME,
+		.of_match_table = am65_cpsw_nuss_of_mtable,
+	},
+	.probe = am65_cpsw_nuss_probe,
+	.remove = am65_cpsw_nuss_remove,
+};
+
+module_platform_driver(am65_cpsw_nuss_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Grygorii Strashko <grygorii.strashko@ti.com>");
+MODULE_DESCRIPTION("TI AM65 CPSW Ethernet driver");
diff -Naur --no-dereference a/drivers/net/ethernet/ti/am65-cpsw-nuss.h b/drivers/net/ethernet/ti/am65-cpsw-nuss.h
--- a/drivers/net/ethernet/ti/am65-cpsw-nuss.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.h	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,153 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2016-2017 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ */
+
+#ifndef AM65_CPSW_NUSS_H_
+#define AM65_CPSW_NUSS_H_
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/netdevice.h>
+
+#define AM65_CPSW_DEBUG	(NETIF_MSG_HW		| NETIF_MSG_WOL		| \
+			 NETIF_MSG_DRV		| NETIF_MSG_LINK	| \
+			 NETIF_MSG_IFUP		| NETIF_MSG_INTR	| \
+			 NETIF_MSG_PROBE	| NETIF_MSG_TIMER	| \
+			 NETIF_MSG_IFDOWN	| NETIF_MSG_RX_ERR	| \
+			 NETIF_MSG_TX_ERR	| NETIF_MSG_TX_DONE	| \
+			 NETIF_MSG_PKTDATA	| NETIF_MSG_TX_QUEUED	| \
+			 NETIF_MSG_RX_STATUS)
+
+#define	am65_nav_dbg(dev, arg...) dev_err(dev, arg)
+
+#define HOST_PORT_NUM		0
+
+#define AM65_CPSW_MAX_TX_QUEUES	8
+#define AM65_CPSW_MAX_RX_QUEUES	1
+#define AM65_CPSW_MAX_RX_FLOWS	8
+
+struct am65_cpsw_slave_data {
+	bool				mac_only;
+	struct cpsw_sl			*mac_sl;
+	struct device_node		*phy_node;
+	struct phy_device		*phy;
+	int				phy_if;
+	bool				rx_pause;
+	bool				tx_pause;
+	u8				mac_addr[ETH_ALEN];
+};
+
+struct am65_cpsw_port {
+	struct am65_cpsw_common		*common;
+	struct net_device		*ndev;
+	const char			*name;
+	u32				port_id;
+	void __iomem			*port_base;
+	void __iomem			*stat_base;
+	bool				disabled;
+	struct am65_cpsw_slave_data	slave;
+	bool				tx_ts_enabled;
+	bool				rx_ts_enabled;
+};
+
+struct am65_cpsw_tx_chn {
+	struct device *dev;
+	struct k3_knav_desc_pool *desc_pool;
+	struct k3_nav_udmax_tx_channel *tx_chn;
+	u32 descs_num;
+	unsigned int irq;
+	u32 id;
+};
+
+struct am65_cpsw_rx_chn {
+	struct device *dev;
+	struct k3_knav_desc_pool *desc_pool;
+	struct k3_nav_udmax_rx_channel *rx_chn;
+	u32 descs_num;
+	unsigned int irq;
+};
+
+#define AM65_CPSW_QUIRK_I2027_NO_TX_CSUM BIT(0)
+
+struct am65_cpsw_pdata {
+	u32	quirks;
+};
+
+struct am65_cpsw_common {
+	struct device		*dev;
+	const struct am65_cpsw_pdata *pdata;
+
+	void __iomem		*ss_base;
+	void __iomem		*cpsw_base;
+
+	u32			port_num;
+	struct am65_cpsw_port	*ports;
+	u32			disabled_ports_mask;
+
+	int			usage_count; /* number of opened ports */
+	struct cpsw_ale		*ale;
+	int			rx_ch_num;
+	int			tx_ch_num;
+	u32			rx_flow_id_base;
+
+	struct am65_cpsw_tx_chn	tx_chns[AM65_CPSW_MAX_TX_QUEUES];
+	struct napi_struct	napi_tx;
+	struct completion	tdown_complete;
+	atomic_t		tdown_cnt;
+
+	struct am65_cpsw_rx_chn	rx_chns;
+	struct napi_struct	napi_rx;
+
+	u32			nuss_ver;
+	u32			cpsw_ver;
+	u32			bus_freq_mhz;
+	struct davinci_mdio_data *mdio;
+	struct am65_cpts *cpts;
+
+	bool			pf_p0_rx_ptype_rrobin;
+	u32			cur_txq;
+};
+
+struct am65_cpsw_ndev_stats {
+	u64 tx_packets;
+	u64 tx_bytes;
+	u64 rx_packets;
+	u64 rx_bytes;
+	struct u64_stats_sync syncp;
+};
+
+struct am65_cpsw_ndev_priv {
+	u32			msg_enable;
+	struct am65_cpsw_port	*port;
+	struct am65_cpsw_ndev_stats __percpu *stats;
+};
+
+#define am65_ndev_to_priv(ndev) \
+	((struct am65_cpsw_ndev_priv *)netdev_priv(ndev))
+#define am65_ndev_to_port(ndev) (am65_ndev_to_priv(ndev)->port)
+#define am65_ndev_to_common(ndev) (am65_ndev_to_port(ndev)->common)
+#define am65_ndev_to_slave(ndev) (&am65_ndev_to_port(ndev)->slave)
+
+#define am65_common_get_host_port(common) (&(common)->ports[HOST_PORT_NUM])
+
+#define am65_cpsw_napi_to_common(napi) \
+	container_of(napi, struct am65_cpsw_common, napi)
+
+#define AM65_CPSW_DRV_NAME "am65-cpsw-nuss"
+
+#define AM65_CPSW_IS_CPSW2G(common) (common->port_num == 2)
+
+extern const struct ethtool_ops am65_cpsw_ethtool_ops_slave;
+
+void am65_cpsw_nuss_adjust_link(struct net_device *ndev);
+void am65_cpsw_nuss_set_p0_ptype(struct am65_cpsw_common *common);
+
+#define	am65_ndev_dbg(ndev, arg...) \
+	do {                                                          \
+		struct device *dev = am65_ndev_to_common(ndev)->dev;  \
+		dev_err(dev, arg); \
+	} while (0)
+
+#endif /* AM65_CPSW_NUSS_H_ */
diff -Naur --no-dereference a/drivers/net/ethernet/ti/am65-cpts.c b/drivers/net/ethernet/ti/am65-cpts.c
--- a/drivers/net/ethernet/ti/am65-cpts.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/net/ethernet/ti/am65-cpts.c	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,1098 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * TI K3 AM65 Common Platform Time Sync
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
+ *
+ */
+
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/err.h>
+#include <linux/if_vlan.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/netdevice.h>
+#include <linux/net_tstamp.h>
+#include <linux/of.h>
+#include <linux/of_irq.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/ptp_classify.h>
+#include <linux/ptp_clock_kernel.h>
+
+struct am65_genf_regs {
+	u32 comp_lo;	/* Comparison Low Value 0:31 */
+	u32 comp_hi;	/* Comparison High Value 32:63 */
+	u32 control;	/* control */
+	u32 length;	/* Length */
+	u32 ppm_low;	/* PPM Load Low Value 0:31 */
+	u32 ppm_hi;	/* PPM Load High Value 32:63 */
+	u32 ts_nudge;	/* Nudge value */
+} __aligned(32) __packed;
+
+#define AM65_CPTS_GENF_MAX_NUM 9
+
+struct am65_cpts_regs {
+	u32 idver;		/* Identification and version */
+	u32 control;		/* Time sync control */
+	u32 rftclk_sel;		/* Reference Clock Select Register */
+	u32 ts_push;		/* Time stamp event push */
+	u32 ts_load_val_lo;	/* Time Stamp Load Low Value 0:31 */
+	u32 ts_load_en;		/* Time stamp load enable */
+	u32 ts_comp_lo;		/* Time Stamp Comparison Low Value 0:31 */
+	u32 ts_comp_length;	/* Time Stamp Comparison Length */
+	u32 intstat_raw;	/* Time sync interrupt status raw */
+	u32 intstat_masked;	/* Time sync interrupt status masked */
+	u32 int_enable;		/* Time sync interrupt enable */
+	u32 ts_comp_nudge;	/* Time Stamp Comparison Nudge Value */
+	u32 event_pop;		/* Event interrupt pop */
+	u32 event_0;		/* Event Time Stamp lo 0:31 */
+	u32 event_1;		/* Event Type Fields */
+	u32 event_2;		/* Event Type Fields domain */
+	u32 event_3;		/* Event Time Stamp hi 32:63 */
+	u32 ts_load_val_hi;	/* Time Stamp Load High Value 32:63 */
+	u32 ts_comp_hi;		/* Time Stamp Comparison High Value 32:63 */
+	u32 ts_add_val;		/* Time Stamp Add value */
+	u32 ts_ppm_low;		/* Time Stamp PPM Load Low Value 0:31 */
+	u32 ts_ppm_hi;		/* Time Stamp PPM Load High Value 32:63 */
+	u32 ts_nudge;		/* Time Stamp Nudge value */
+	u32 reserv[33];
+	struct am65_genf_regs genf[AM65_CPTS_GENF_MAX_NUM];
+};
+
+/* CONTROL_REG */
+#define AM65_CPTS_CONTROL_EN			BIT(0)
+#define AM65_CPTS_CONTROL_INT_TEST		BIT(1)
+#define AM65_CPTS_CONTROL_TS_COMP_POLARITY	BIT(2)
+#define AM65_CPTS_CONTROL_TSTAMP_EN		BIT(3)
+#define AM65_CPTS_CONTROL_SEQUENCE_EN		BIT(4)
+#define AM65_CPTS_CONTROL_64MODE		BIT(5)
+#define AM65_CPTS_CONTROL_TS_COMP_TOG		BIT(6)
+#define AM65_CPTS_CONTROL_TS_PPM_DIR		BIT(7)
+#define AM65_CPTS_CONTROL_HW1_TS_PUSH_EN	BIT(8)
+#define AM65_CPTS_CONTROL_HW2_TS_PUSH_EN	BIT(9)
+#define AM65_CPTS_CONTROL_HW3_TS_PUSH_EN	BIT(10)
+#define AM65_CPTS_CONTROL_HW4_TS_PUSH_EN	BIT(11)
+#define AM65_CPTS_CONTROL_HW5_TS_PUSH_EN	BIT(12)
+#define AM65_CPTS_CONTROL_HW6_TS_PUSH_EN	BIT(13)
+#define AM65_CPTS_CONTROL_HW7_TS_PUSH_EN	BIT(14)
+#define AM65_CPTS_CONTROL_HW8_TS_PUSH_EN	BIT(15)
+#define AM65_CPTS_CONTROL_HW1_TS_PUSH_OFFSET	(8)
+
+#define AM65_CPTS_CONTROL_TS_SYNC_SEL_MASK	(0xF)
+#define AM65_CPTS_CONTROL_TS_SYNC_SEL_SHIFT	(28)
+
+/* RFTCLK_SEL_REG */
+#define AM65_CPTS_RFTCLK_SEL_MASK		(0x1F)
+
+/* TS_PUSH_REG */
+#define AM65_CPTS_TS_PUSH			BIT(0)
+
+/* TS_LOAD_EN_REG */
+#define AM65_CPTS_TS_LOAD_EN			BIT(0)
+
+/* INTSTAT_RAW_REG */
+#define AM65_CPTS_INTSTAT_RAW_TS_PEND		BIT(0)
+
+/* INTSTAT_MASKED_REG */
+#define AM65_CPTS_INTSTAT_MASKED_TS_PEND	BIT(0)
+
+/* INT_ENABLE_REG */
+#define AM65_CPTS_INT_ENABLE_TS_PEND_EN		BIT(0)
+
+/* TS_COMP_NUDGE_REG */
+#define AM65_CPTS_TS_COMP_NUDGE_MASK		(0xFF)
+
+/* EVENT_POP_REG */
+#define AM65_CPTS_EVENT_POP			BIT(0)
+
+/* EVENT_1_REG */
+#define AM65_CPTS_EVENT_1_SEQUENCE_ID_MASK	(0xFFFF)
+#define AM65_CPTS_EVENT_1_SEQUENCE_ID_SHIFT	(0)
+
+#define AM65_CPTS_EVENT_1_MESSAGE_TYPE_MASK	(0xF)
+#define AM65_CPTS_EVENT_1_MESSAGE_TYPE_SHIFT	(16)
+
+#define AM65_CPTS_EVENT_1_EVENT_TYPE_MASK	(0xF)
+#define AM65_CPTS_EVENT_1_EVENT_TYPE_SHIFT	(20)
+
+#define AM65_CPTS_EVENT_1_PORT_NUMBER_MASK	(0x1F)
+#define AM65_CPTS_EVENT_1_PORT_NUMBER_SHIFT	(24)
+
+/* EVENT_2_REG */
+#define AM65_CPTS_EVENT_2_REG_DOMAIN_MASK	(0xFF)
+#define AM65_CPTS_EVENT_2_REG_DOMAIN_SHIFT	(0)
+
+enum {
+	AM65_CPTS_EV_PUSH,	/* Time Stamp Push Event */
+	AM65_CPTS_EV_ROLL,	/* Time Stamp Rollover Event */
+	AM65_CPTS_EV_HALF,	/* Time Stamp Half Rollover Event */
+	AM65_CPTS_EV_HW,		/* Hardware Time Stamp Push Event */
+	AM65_CPTS_EV_RX,		/* Ethernet Receive Event */
+	AM65_CPTS_EV_TX,		/* Ethernet Transmit Event */
+	AM65_CPTS_EV_TS_COMP,	/* Time Stamp Compare Event */
+	AM65_CPTS_EV_HOST,	/* Host Transmit Event */
+};
+
+struct am65_cpts_event {
+	struct list_head list;
+	unsigned long tmo;
+	u32 event1;
+	u32 event2;
+	u64 timestamp;
+};
+
+#define AM65_CPTS_FIFO_DEPTH		(16)
+#define AM65_CPTS_MAX_EVENTS		(32)
+#define AM65_CPTS_EVENT_RX_TX_TIMEOUT	(20) /* ms */
+#define AM65_CPTS_SKB_TX_WORK_TIMEOUT	1 /* jiffies */
+#define AM65_CPTS_MIN_PPM		0x400
+
+struct am65_cpts {
+	struct device *dev;
+	struct am65_cpts_regs __iomem *reg;
+	struct ptp_clock_info ptp_info;
+	struct ptp_clock *ptp_clock;
+	int phc_index;
+	struct clk *clk_mux;
+	struct device_node *clk_mux_np;
+	struct clk *refclk;
+	u32 refclk_freq;
+	struct list_head events;
+	struct list_head pool;
+	struct am65_cpts_event pool_data[AM65_CPTS_MAX_EVENTS];
+	spinlock_t lock; /* protects events lists*/
+	u32 ext_ts_inputs;
+	u32 genf_num;
+	u32 ts_add_val;
+	int irq;
+	spinlock_t ptp_clk_lock; /* PHC access sync */
+	u64 timestamp;
+	u32 genf_enable;
+	u32 hw_ts_enable;
+	struct sk_buff_head txq;
+	bool pps_enabled;
+	bool pps_present;
+	u32 pps_hw_ts_idx;
+	u32 pps_genf_idx;
+};
+
+struct am65_cpts_skb_cb_data {
+	unsigned long tmo;
+	unsigned int ptp_class;
+};
+
+#define am65_cpts_write32(c, v, r) writel(v, &c->reg->r)
+#define am65_cpts_read32(c, r) readl(&c->reg->r)
+
+static void am65_cpts_settime(struct am65_cpts *cpts, u64 start_tstamp)
+{
+	u32 val;
+
+	val = upper_32_bits(start_tstamp);
+	am65_cpts_write32(cpts, val, ts_load_val_hi);
+	val = lower_32_bits(start_tstamp);
+	am65_cpts_write32(cpts, val, ts_load_val_lo);
+
+	am65_cpts_write32(cpts, AM65_CPTS_TS_LOAD_EN, ts_load_en);
+}
+
+static void am65_cpts_set_add_val(struct am65_cpts *cpts)
+{
+	/* select coefficient according to the rate */
+	cpts->ts_add_val = (NSEC_PER_SEC / cpts->refclk_freq - 1) & 0x7;
+
+	am65_cpts_write32(cpts, cpts->ts_add_val, ts_add_val);
+}
+
+static void am65_cpts_disable(struct am65_cpts *cpts)
+{
+	am65_cpts_write32(cpts, 0, control);
+	am65_cpts_write32(cpts, 0, int_enable);
+}
+
+static int am65_cpts_event_get_port(struct am65_cpts_event *event)
+{
+	return (event->event1 >> AM65_CPTS_EVENT_1_PORT_NUMBER_SHIFT) &
+	       AM65_CPTS_EVENT_1_PORT_NUMBER_MASK;
+}
+
+static int am65_cpts_event_get_type(struct am65_cpts_event *event)
+{
+	return (event->event1 >> AM65_CPTS_EVENT_1_EVENT_TYPE_SHIFT) &
+	       AM65_CPTS_EVENT_1_EVENT_TYPE_MASK;
+}
+
+static int am65_cpts_event_get_msg_type(struct am65_cpts_event *event)
+{
+	return (event->event1 >> AM65_CPTS_EVENT_1_MESSAGE_TYPE_SHIFT) &
+	       AM65_CPTS_EVENT_1_MESSAGE_TYPE_MASK;
+}
+
+static int am65_cpts_event_get_seqid(struct am65_cpts_event *event)
+{
+	return (event->event1 >> AM65_CPTS_EVENT_1_SEQUENCE_ID_SHIFT) &
+	       AM65_CPTS_EVENT_1_SEQUENCE_ID_MASK;
+}
+
+static int am65_cpts_cpts_purge_events(struct am65_cpts *cpts)
+{
+	struct list_head *this, *next;
+	struct am65_cpts_event *event;
+	int removed = 0;
+
+	list_for_each_safe(this, next, &cpts->events) {
+		event = list_entry(this, struct am65_cpts_event, list);
+		if (time_after(jiffies, event->tmo)) {
+			list_del_init(&event->list);
+			list_add(&event->list, &cpts->pool);
+			++removed;
+		}
+	}
+
+	if (removed)
+		dev_dbg(cpts->dev, "event pool cleaned up %d\n", removed);
+	return removed ? 0 : -1;
+}
+
+static bool am65_cpts_fifo_pop_event(struct am65_cpts *cpts,
+				     struct am65_cpts_event *event)
+{
+	u32 r = am65_cpts_read32(cpts, intstat_raw);
+
+	if (r & AM65_CPTS_INTSTAT_RAW_TS_PEND) {
+		event->timestamp = am65_cpts_read32(cpts, event_0);
+		event->event1 = am65_cpts_read32(cpts, event_1);
+		event->event2 = am65_cpts_read32(cpts, event_2);
+		event->timestamp |= (u64)am65_cpts_read32(cpts, event_3) << 32;
+		am65_cpts_write32(cpts, AM65_CPTS_EVENT_POP, event_pop);
+		return false;
+	}
+	return true;
+}
+
+static int am65_cpts_fifo_read(struct am65_cpts *cpts)
+{
+	struct ptp_clock_event pevent;
+	struct am65_cpts_event *event;
+	unsigned long flags;
+	int i, type, ret = 0;
+	bool schedule = false;
+
+	spin_lock_irqsave(&cpts->lock, flags);
+	for (i = 0; i < AM65_CPTS_FIFO_DEPTH; i++) {
+		event = list_first_entry_or_null(&cpts->pool,
+						 struct am65_cpts_event, list);
+
+		if (!event) {
+			if (am65_cpts_cpts_purge_events(cpts)) {
+				dev_err(cpts->dev, "cpts: event pool empty\n");
+				ret = -1;
+				goto out;
+			}
+			continue;
+		}
+
+		if (am65_cpts_fifo_pop_event(cpts, event))
+			break;
+
+		type = am65_cpts_event_get_type(event);
+		switch (type) {
+		case AM65_CPTS_EV_PUSH:
+			cpts->timestamp = event->timestamp;
+			dev_dbg(cpts->dev, "AM65_CPTS_EV_PUSH t:%llu\n",
+				cpts->timestamp);
+			break;
+		case AM65_CPTS_EV_RX:
+		case AM65_CPTS_EV_TX:
+			event->tmo = jiffies +
+				msecs_to_jiffies(AM65_CPTS_EVENT_RX_TX_TIMEOUT);
+
+			list_del_init(&event->list);
+			list_add_tail(&event->list, &cpts->events);
+
+			dev_dbg(cpts->dev,
+				"AM65_CPTS_EV_TX e1:%08x e2:%08x t:%lld\n",
+				event->event1, event->event2,
+				event->timestamp);
+			schedule = true;
+			break;
+		case AM65_CPTS_EV_HW:
+			pevent.index = am65_cpts_event_get_port(event) - 1;
+			pevent.timestamp = event->timestamp;
+			if (cpts->pps_enabled &&
+			    pevent.index == cpts->pps_hw_ts_idx) {
+				pevent.type = PTP_CLOCK_PPSUSR;
+				pevent.pps_times.ts_real =
+					ns_to_timespec64(pevent.timestamp);
+			} else {
+				pevent.type = PTP_CLOCK_EXTTS;
+			}
+			dev_dbg(cpts->dev, "AM65_CPTS_EV_HW:%s p:%d t:%llu\n",
+				pevent.type == PTP_CLOCK_EXTTS ?
+				"extts" : "pps",
+				pevent.index, event->timestamp);
+
+			ptp_clock_event(cpts->ptp_clock, &pevent);
+			break;
+		case AM65_CPTS_EV_HOST:
+			event->tmo = jiffies +
+				msecs_to_jiffies(AM65_CPTS_EVENT_RX_TX_TIMEOUT);
+
+			list_del_init(&event->list);
+			list_add_tail(&event->list, &cpts->events);
+
+			dev_dbg(cpts->dev,
+				"AM65_CPTS_EV_HOST e1:%08x e2:%08x t:%lld\n",
+				event->event1, event->event2,
+				event->timestamp);
+			schedule = true;
+			break;
+		case AM65_CPTS_EV_ROLL:
+		case AM65_CPTS_EV_HALF:
+		case AM65_CPTS_EV_TS_COMP:
+			break;
+		default:
+			dev_err(cpts->dev, "cpts: unknown event type\n");
+			ret = -1;
+			goto out;
+		}
+	}
+
+out:
+	spin_unlock_irqrestore(&cpts->lock, flags);
+
+	if (schedule)
+		ptp_schedule_worker(cpts->ptp_clock, 0);
+
+	return ret;
+}
+
+static u64 am65_cpts_gettime(struct am65_cpts *cpts)
+{
+	u64 val = 0;
+
+	/* temporarily disable cpts interrupt to avoid intentional
+	 * doubled read. Interrupt can be in-flight - it's Ok.
+	 */
+	am65_cpts_write32(cpts, 0, int_enable);
+
+	am65_cpts_write32(cpts, AM65_CPTS_TS_PUSH, ts_push);
+	am65_cpts_fifo_read(cpts);
+
+	am65_cpts_write32(cpts, AM65_CPTS_INT_ENABLE_TS_PEND_EN, int_enable);
+
+	val = cpts->timestamp;
+
+	return val;
+}
+
+static irqreturn_t am65_cpts_interrupt(int irq, void *dev_id)
+{
+	struct am65_cpts *cpts = dev_id;
+
+	if (am65_cpts_fifo_read(cpts))
+		dev_dbg(cpts->dev, "cpts: unable to obtain a time stamp\n");
+
+	return IRQ_HANDLED;
+}
+
+/* PTP clock operations */
+static int am65_cpts_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb)
+{
+	struct am65_cpts *cpts = container_of(ptp, struct am65_cpts, ptp_info);
+	int neg_adj = 0;
+	u64 adj_period;
+	u32 val;
+
+	if (ppb < 0) {
+		neg_adj = 1;
+		ppb = -ppb;
+	}
+
+	/* base freq = 1GHz = 1 000 000 000
+	 * ppb_norm = ppb * base_freq / clock_freq;
+	 * ppm_norm = ppb_norm / 1000
+	 * adj_period = 1 000 000 / ppm_norm
+	 * adj_period = 1 000 000 000 / ppb_norm
+	 * adj_period = 1 000 000 000 / (ppb * base_freq / clock_freq)
+	 * adj_period = (1 000 000 000 * clock_freq) / (ppb * base_freq)
+	 * adj_period = clock_freq / ppb
+	 */
+	adj_period = div_u64(cpts->refclk_freq, ppb);
+
+	spin_lock(&cpts->ptp_clk_lock);
+
+	val = am65_cpts_read32(cpts, control);
+	if (neg_adj)
+		val |= AM65_CPTS_CONTROL_TS_PPM_DIR;
+	else
+		val &= ~AM65_CPTS_CONTROL_TS_PPM_DIR;
+	am65_cpts_write32(cpts, val, control);
+
+	val = upper_32_bits(adj_period) & 0x3FF;
+	am65_cpts_write32(cpts, val, ts_ppm_hi);
+	val = lower_32_bits(adj_period);
+	am65_cpts_write32(cpts, val, ts_ppm_low);
+
+	spin_unlock(&cpts->ptp_clk_lock);
+
+	return 0;
+}
+
+static int am65_cpts_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
+{
+	struct am65_cpts *cpts = container_of(ptp, struct am65_cpts, ptp_info);
+	s64 ns;
+
+	spin_lock(&cpts->ptp_clk_lock);
+	ns = am65_cpts_gettime(cpts);
+	ns += delta;
+	am65_cpts_settime(cpts, ns);
+	spin_unlock(&cpts->ptp_clk_lock);
+
+	return 0;
+}
+
+static int am65_cpts_ptp_gettime(struct ptp_clock_info *ptp,
+				 struct timespec64 *ts)
+{
+	struct am65_cpts *cpts = container_of(ptp, struct am65_cpts, ptp_info);
+	u64 ns;
+
+	spin_lock(&cpts->ptp_clk_lock);
+	ns = am65_cpts_gettime(cpts);
+	spin_unlock(&cpts->ptp_clk_lock);
+	*ts = ns_to_timespec64(ns);
+
+	return 0;
+}
+
+static int am65_cpts_ptp_settime(struct ptp_clock_info *ptp,
+				 const struct timespec64 *ts)
+{
+	struct am65_cpts *cpts = container_of(ptp, struct am65_cpts, ptp_info);
+	u64 ns;
+
+	ns = timespec64_to_ns(ts);
+	spin_lock(&cpts->ptp_clk_lock);
+	am65_cpts_settime(cpts, ns);
+	spin_unlock(&cpts->ptp_clk_lock);
+
+	return 0;
+}
+
+/**
+ * Enable HW TimeStamp inputs
+ */
+static void am65_cpts_extts_enable_hw(struct am65_cpts *cpts, u32 index, int on)
+{
+	u32 v;
+
+	v = am65_cpts_read32(cpts, control);
+	if (on) {
+		v |= BIT(AM65_CPTS_CONTROL_HW1_TS_PUSH_OFFSET + index);
+		cpts->hw_ts_enable |= BIT(index);
+	} else {
+		v &= ~BIT(AM65_CPTS_CONTROL_HW1_TS_PUSH_OFFSET + index);
+		cpts->hw_ts_enable &= ~BIT(index);
+	}
+	am65_cpts_write32(cpts, v, control);
+}
+
+static int am65_cpts_extts_enable(struct am65_cpts *cpts, u32 index, int on)
+{
+	if (index >= cpts->ptp_info.n_ext_ts)
+		return -ENXIO;
+
+	if (cpts->pps_present && index == cpts->pps_hw_ts_idx)
+		return -EINVAL;
+
+	if (((cpts->hw_ts_enable & BIT(index)) >> index) == on)
+		return 0;
+
+	spin_lock(&cpts->ptp_clk_lock);
+	am65_cpts_extts_enable_hw(cpts, index, on);
+	spin_unlock(&cpts->ptp_clk_lock);
+
+	dev_dbg(cpts->dev, "%s: ExtTS:%u %s\n",
+		__func__, index, on ? "enabled" : "disabled");
+
+	return 0;
+}
+
+/**
+ * Enable GENf periodic output
+ */
+static void am65_cpts_perout_enable_hw(struct am65_cpts *cpts,
+				       struct ptp_perout_request *req, int on)
+{
+	struct timespec64 ts;
+	u64 ns_period, ns_start;
+	u64 cycles;
+	u32 val;
+
+	if (on) {
+		ts.tv_sec = req->period.sec;
+		ts.tv_nsec = req->period.nsec;
+		ns_period = timespec64_to_ns(&ts);
+
+		cycles = (ns_period * cpts->refclk_freq) / NSEC_PER_SEC;
+
+		ts.tv_sec = req->start.sec;
+		ts.tv_nsec = req->start.nsec;
+		ns_start = timespec64_to_ns(&ts);
+
+		val = upper_32_bits(ns_start);
+		am65_cpts_write32(cpts, val, genf[req->index].comp_hi);
+		val = lower_32_bits(ns_start);
+		am65_cpts_write32(cpts, val, genf[req->index].comp_lo);
+		val = lower_32_bits(cycles);
+		am65_cpts_write32(cpts, val, genf[req->index].length);
+
+		cpts->genf_enable |= BIT(req->index);
+	} else {
+		am65_cpts_write32(cpts, 0, genf[req->index].length);
+
+		cpts->genf_enable &= ~BIT(req->index);
+	}
+}
+
+static int am65_cpts_perout_enable(struct am65_cpts *cpts,
+				   struct ptp_perout_request *req, int on)
+{
+	if (req->index >= cpts->ptp_info.n_per_out)
+		return -ENXIO;
+
+	if (cpts->pps_present && req->index == cpts->pps_genf_idx)
+		return -EINVAL;
+
+	if (!!(cpts->genf_enable & BIT(req->index)) == !!on)
+		return 0;
+
+	spin_lock(&cpts->ptp_clk_lock);
+	am65_cpts_perout_enable_hw(cpts, req, on);
+	spin_unlock(&cpts->ptp_clk_lock);
+
+	dev_dbg(cpts->dev, "%s: GenF:%u %s\n",
+		__func__, req->index, on ? "enabled" : "disabled");
+
+	return 0;
+}
+
+static int am65_cpts_pps_enable(struct am65_cpts *cpts, int on)
+{
+	int ret = 0;
+	struct timespec64 ts;
+	struct ptp_clock_request rq;
+	u64 ns;
+
+	if (!cpts->pps_present)
+		return -EINVAL;
+
+	if (cpts->pps_enabled == !!on)
+		return 0;
+
+	spin_lock(&cpts->ptp_clk_lock);
+
+	if (on) {
+		am65_cpts_extts_enable_hw(cpts, cpts->pps_hw_ts_idx, on);
+
+		ns = am65_cpts_gettime(cpts);
+		ts = ns_to_timespec64(ns);
+		rq.perout.period.sec = 1;
+		rq.perout.period.nsec = 0;
+		rq.perout.start.sec = ts.tv_sec + 2;
+		rq.perout.start.nsec = 0;
+		rq.perout.index = cpts->pps_genf_idx;
+
+		am65_cpts_perout_enable_hw(cpts, &rq.perout, on);
+		cpts->pps_enabled = true;
+	} else {
+		rq.perout.index = cpts->pps_genf_idx;
+		am65_cpts_perout_enable_hw(cpts, &rq.perout, on);
+		am65_cpts_extts_enable_hw(cpts, cpts->pps_hw_ts_idx, on);
+		cpts->pps_enabled = false;
+	}
+
+	spin_unlock(&cpts->ptp_clk_lock);
+
+	dev_dbg(cpts->dev, "%s: pps: %s\n",
+		__func__, on ? "enabled" : "disabled");
+	return ret;
+}
+
+static int am65_cpts_ptp_enable(struct ptp_clock_info *ptp,
+				struct ptp_clock_request *rq, int on)
+{
+	struct am65_cpts *cpts = container_of(ptp, struct am65_cpts, ptp_info);
+
+	switch (rq->type) {
+	case PTP_CLK_REQ_EXTTS:
+		return am65_cpts_extts_enable(cpts, rq->extts.index, on);
+	case PTP_CLK_REQ_PEROUT:
+		return am65_cpts_perout_enable(cpts, &rq->perout, on);
+	case PTP_CLK_REQ_PPS:
+		return am65_cpts_pps_enable(cpts, on);
+	default:
+		break;
+	}
+
+	return -EOPNOTSUPP;
+}
+
+static long am65_cpts_ts_work(struct ptp_clock_info *ptp);
+
+static struct ptp_clock_info am65_ptp_info = {
+	.owner		= THIS_MODULE,
+	.name		= "CTPS timer",
+	.n_ext_ts	= 0,
+	.n_per_out	= 0,
+	.n_pins		= 0,
+	.pps		= 0,
+	.adjfreq	= am65_cpts_ptp_adjfreq,
+	.adjtime	= am65_cpts_ptp_adjtime,
+	.gettime64	= am65_cpts_ptp_gettime,
+	.settime64	= am65_cpts_ptp_settime,
+	.enable		= am65_cpts_ptp_enable,
+	.do_aux_work	= am65_cpts_ts_work,
+};
+
+static bool am65_cpts_match(struct sk_buff *skb, unsigned int ptp_class,
+			    u16 ts_seqid, u8 ts_msgtype)
+{
+	u16 *seqid;
+	unsigned int offset = 0;
+	u8 *msgtype, *data = skb->data;
+
+	if (ptp_class & PTP_CLASS_VLAN)
+		offset += VLAN_HLEN;
+
+	switch (ptp_class & PTP_CLASS_PMASK) {
+	case PTP_CLASS_IPV4:
+		offset += ETH_HLEN + IPV4_HLEN(data + offset) + UDP_HLEN;
+		break;
+	case PTP_CLASS_IPV6:
+		offset += ETH_HLEN + IP6_HLEN + UDP_HLEN;
+		break;
+	case PTP_CLASS_L2:
+		offset += ETH_HLEN;
+		break;
+	default:
+		return 0;
+	}
+
+	if (skb->len + ETH_HLEN < offset + OFF_PTP_SEQUENCE_ID + sizeof(*seqid))
+		return 0;
+
+	if (unlikely(ptp_class & PTP_CLASS_V1))
+		msgtype = data + offset + OFF_PTP_CONTROL;
+	else
+		msgtype = data + offset;
+
+	seqid = (u16 *)(data + offset + OFF_PTP_SEQUENCE_ID);
+
+	return (ts_msgtype == (*msgtype & 0xf) && ts_seqid == ntohs(*seqid));
+}
+
+static bool am65_cpts_match_tx_ts(struct am65_cpts *cpts,
+				  struct am65_cpts_event *event)
+{
+	struct sk_buff *skb, *tmp;
+	u16 seqid;
+	u8 mtype;
+	bool found = false;
+	unsigned long flags;
+	struct sk_buff_head txq_list;
+
+	mtype = am65_cpts_event_get_msg_type(event);
+	seqid = am65_cpts_event_get_seqid(event);
+
+	__skb_queue_head_init(&txq_list);
+
+	spin_lock_irqsave(&cpts->txq.lock, flags);
+	skb_queue_splice_init(&cpts->txq, &txq_list);
+	spin_unlock_irqrestore(&cpts->txq.lock, flags);
+
+	/* no need to grab txq.lock as access is always done under cpts->lock */
+	skb_queue_walk_safe(&txq_list, skb, tmp) {
+		struct skb_shared_hwtstamps ssh;
+		struct am65_cpts_skb_cb_data *skb_cb =
+					(struct am65_cpts_skb_cb_data *)skb->cb;
+
+		if (am65_cpts_match(skb, skb_cb->ptp_class, seqid, mtype)) {
+			u64 ns = event->timestamp;
+
+			memset(&ssh, 0, sizeof(ssh));
+			ssh.hwtstamp = ns_to_ktime(ns);
+			skb_tstamp_tx(skb, &ssh);
+			found = true;
+			__skb_unlink(skb, &txq_list);
+			dev_consume_skb_any(skb);
+			dev_dbg(cpts->dev, "match tx timestamp mtype %u seqid %04x\n",
+				mtype, seqid);
+		} else if (time_after(jiffies, skb_cb->tmo)) {
+			/* timeout any expired skbs over 100 ms */
+			dev_dbg(cpts->dev,
+				"expiring tx timestamp mtype %u seqid %04x\n",
+				mtype, seqid);
+			__skb_unlink(skb, &txq_list);
+			dev_consume_skb_any(skb);
+		}
+	}
+
+	spin_lock_irqsave(&cpts->txq.lock, flags);
+	skb_queue_splice(&txq_list, &cpts->txq);
+	spin_unlock_irqrestore(&cpts->txq.lock, flags);
+
+	return found;
+}
+
+static void am65_cpts_find_ts(struct am65_cpts *cpts)
+{
+	struct am65_cpts_event *event;
+	struct list_head *this, *next;
+	unsigned long flags;
+	LIST_HEAD(events);
+	LIST_HEAD(events_free);
+
+	spin_lock_irqsave(&cpts->lock, flags);
+	list_splice_init(&cpts->events, &events);
+	spin_unlock_irqrestore(&cpts->lock, flags);
+
+	list_for_each_safe(this, next, &events) {
+		event = list_entry(this, struct am65_cpts_event, list);
+		if (am65_cpts_match_tx_ts(cpts, event) ||
+		    time_after(jiffies, event->tmo)) {
+			list_del_init(&event->list);
+			list_add(&event->list, &events_free);
+		}
+	}
+
+	spin_lock_irqsave(&cpts->lock, flags);
+	list_splice_tail(&events, &cpts->events);
+	list_splice_tail(&events_free, &cpts->pool);
+	spin_unlock_irqrestore(&cpts->lock, flags);
+}
+
+static long am65_cpts_ts_work(struct ptp_clock_info *ptp)
+{
+	struct am65_cpts *cpts = container_of(ptp, struct am65_cpts, ptp_info);
+	long delay = -1;
+	unsigned long flags;
+
+	am65_cpts_find_ts(cpts);
+
+	spin_lock_irqsave(&cpts->txq.lock, flags);
+	if (!skb_queue_empty(&cpts->txq))
+		delay = AM65_CPTS_SKB_TX_WORK_TIMEOUT;
+	spin_unlock_irqrestore(&cpts->txq.lock, flags);
+
+	return delay;
+}
+
+void am65_cpts_rx_enable(struct am65_cpts *cpts, bool en)
+{
+	u32 val;
+
+	spin_lock(&cpts->ptp_clk_lock);
+	val = am65_cpts_read32(cpts, control);
+	if (en)
+		val |= AM65_CPTS_CONTROL_TSTAMP_EN;
+	else
+		val &= ~AM65_CPTS_CONTROL_TSTAMP_EN;
+	am65_cpts_write32(cpts, val, control);
+	spin_unlock(&cpts->ptp_clk_lock);
+}
+EXPORT_SYMBOL_GPL(am65_cpts_rx_enable);
+
+void am65_cpts_tx_timestamp(struct am65_cpts *cpts, struct sk_buff *skb)
+{
+	struct am65_cpts_skb_cb_data *skb_cb = (void *)skb->cb;
+
+	if (!(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))
+		return;
+
+	/* add frame to queue for processing later.
+	 * The periodic FIFO check will handle this.
+	 */
+	skb_get(skb);
+	/* get the timestamp for timeouts */
+	skb_cb->tmo = jiffies + msecs_to_jiffies(100);
+	skb_queue_tail(&cpts->txq, skb);
+	ptp_schedule_worker(cpts->ptp_clock, 0);
+}
+EXPORT_SYMBOL_GPL(am65_cpts_tx_timestamp);
+
+void am65_cpts_ask_tx_timestamp(struct am65_cpts *cpts, struct sk_buff *skb)
+{
+	struct am65_cpts_skb_cb_data *skb_cb = (void *)skb->cb;
+	unsigned int ptp_class;
+
+	if (!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))
+		return;
+
+	ptp_class = ptp_classify_raw(skb);
+	if (ptp_class == PTP_CLASS_NONE)
+		return;
+
+	skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
+	skb_cb->ptp_class = ptp_class;
+}
+EXPORT_SYMBOL_GPL(am65_cpts_ask_tx_timestamp);
+
+int am65_cpts_phc_index(struct am65_cpts *cpts)
+{
+	return cpts->phc_index;
+}
+EXPORT_SYMBOL_GPL(am65_cpts_phc_index);
+
+static void cpts_of_free_clk_provider(void *data)
+{
+	struct am65_cpts *cpts = data;
+
+	of_clk_del_provider(cpts->clk_mux_np);
+	of_node_put(cpts->clk_mux_np);
+}
+
+static int cpts_of_mux_clk_setup(struct am65_cpts *cpts,
+				 struct device_node *node)
+{
+	unsigned int num_parents;
+	const char **parent_names;
+	void __iomem *reg;
+	int ret;
+	char *clk_mux_name = NULL;
+
+	cpts->clk_mux_np = of_get_child_by_name(node, "cpts_refclk_mux");
+	if (!cpts->clk_mux_np)
+		return 0;
+
+	num_parents = of_clk_get_parent_count(cpts->clk_mux_np);
+	if (num_parents < 1) {
+		dev_err(cpts->dev, "mux-clock %pOF must have parents\n",
+			cpts->clk_mux_np);
+		return -EINVAL;
+	}
+
+	parent_names = devm_kcalloc(cpts->dev, sizeof(char *), num_parents,
+				    GFP_KERNEL);
+	if (!parent_names)
+		return -ENOMEM;
+
+	of_clk_parent_fill(cpts->clk_mux_np, parent_names, num_parents);
+
+	clk_mux_name = devm_kasprintf(cpts->dev, GFP_KERNEL, "%s.%pOFn",
+				      dev_name(cpts->dev), cpts->clk_mux_np);
+	if (!clk_mux_name)
+		return -ENOMEM;
+
+	reg = &cpts->reg->rftclk_sel;
+	/* WARN: dev must be NULL to avoid recursive incrementing
+	 * of module refcnt
+	 */
+	cpts->clk_mux = clk_register_mux(NULL, clk_mux_name,
+					 parent_names, num_parents,
+					 0, reg, 0, 5, 0, NULL);
+	if (IS_ERR(cpts->clk_mux))
+		return PTR_ERR(cpts->clk_mux);
+
+	ret = devm_add_action_or_reset(cpts->dev,
+				       (void(*)(void *))clk_unregister_mux,
+				       cpts->clk_mux);
+	if (ret) {
+		dev_err(cpts->dev, "failed to add clkmux reset action %d", ret);
+		return ret;
+	}
+
+	ret = of_clk_add_provider(cpts->clk_mux_np, of_clk_src_simple_get,
+				  cpts->clk_mux);
+	if (ret)
+		return ret;
+
+	ret = devm_add_action_or_reset(cpts->dev,
+				       cpts_of_free_clk_provider,
+				       cpts->clk_mux_np);
+	if (ret)
+		dev_err(cpts->dev, "failed to add clkmux reset action %d", ret);
+
+	return ret;
+}
+
+static int am65_cpts_of_parse(struct am65_cpts *cpts, struct device_node *node)
+{
+	u32 prop[2];
+
+	if (!of_property_read_u32(node, "ti,cpts-ext-ts-inputs", &prop[0]))
+		cpts->ext_ts_inputs = prop[0];
+
+	if (!of_property_read_u32(node, "ti,cpts-periodic-outputs", &prop[0]))
+		cpts->genf_num = prop[0];
+
+	if (!of_property_read_u32_array(node, "ti,pps", prop, 2)) {
+		cpts->pps_present = true;
+		cpts->pps_hw_ts_idx = prop[0];
+		cpts->pps_genf_idx = prop[1];
+	}
+
+	return cpts_of_mux_clk_setup(cpts, node);
+}
+
+struct am65_cpts *am65_cpts_create(struct device *dev, void __iomem *regs,
+				   struct device_node *node)
+{
+	struct am65_cpts *cpts;
+	int ret, i;
+
+	cpts = devm_kzalloc(dev, sizeof(*cpts), GFP_KERNEL);
+	if (!cpts)
+		return ERR_PTR(-ENOMEM);
+
+	cpts->dev = dev;
+	cpts->reg = (struct am65_cpts_regs __iomem *)regs;
+
+	cpts->irq = of_irq_get_byname(node, "cpts");
+	if (cpts->irq <= 0) {
+		ret = cpts->irq ?: -ENXIO;
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "Failed to get IRQ number (err = %d)\n",
+				ret);
+		return ERR_PTR(ret);
+	}
+
+	ret = am65_cpts_of_parse(cpts, node);
+	if (ret)
+		return ERR_PTR(ret);
+
+	spin_lock_init(&cpts->ptp_clk_lock);
+	INIT_LIST_HEAD(&cpts->events);
+	INIT_LIST_HEAD(&cpts->pool);
+	spin_lock_init(&cpts->lock);
+	skb_queue_head_init(&cpts->txq);
+
+	for (i = 0; i < AM65_CPTS_MAX_EVENTS; i++)
+		list_add(&cpts->pool_data[i].list, &cpts->pool);
+
+	cpts->refclk = devm_get_clk_from_child(dev, node, "cpts");
+	if (IS_ERR(cpts->refclk)) {
+		ret = PTR_ERR(cpts->refclk);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "Failed to get refclk %d\n", ret);
+		return ERR_PTR(ret);
+	}
+
+	ret = clk_prepare_enable(cpts->refclk);
+	if (ret) {
+		dev_err(dev, "Failed to enable refclk %d\n", ret);
+		return ERR_PTR(ret);
+	}
+
+	ret = devm_add_action_or_reset(dev,
+				       (void(*)(void *))clk_disable_unprepare,
+				       cpts->refclk);
+	if (ret) {
+		dev_err(dev, "failed to add clk reset action %d", ret);
+		return ERR_PTR(ret);
+	}
+
+	cpts->refclk_freq = clk_get_rate(cpts->refclk);
+
+	am65_ptp_info.max_adj = cpts->refclk_freq / AM65_CPTS_MIN_PPM;
+	cpts->ptp_info = am65_ptp_info;
+
+	if (cpts->ext_ts_inputs)
+		cpts->ptp_info.n_ext_ts = cpts->ext_ts_inputs;
+	if (cpts->genf_num)
+		cpts->ptp_info.n_per_out = cpts->genf_num;
+	if (cpts->pps_present)
+		cpts->ptp_info.pps = 1;
+
+	am65_cpts_set_add_val(cpts);
+
+	am65_cpts_write32(cpts, AM65_CPTS_CONTROL_EN | AM65_CPTS_CONTROL_64MODE,
+			  control);
+	am65_cpts_write32(cpts, AM65_CPTS_INT_ENABLE_TS_PEND_EN, int_enable);
+
+	/* set time to the current system time */
+	am65_cpts_settime(cpts, ktime_to_ns(ktime_get_real()));
+
+	ret = devm_add_action_or_reset(dev,
+				       (void(*)(void *))am65_cpts_disable,
+				       cpts);
+	if (ret) {
+		dev_err(dev, "failed to add cpts reset action %d", ret);
+		return ERR_PTR(ret);
+	}
+
+	cpts->ptp_clock = ptp_clock_register(&cpts->ptp_info, cpts->dev);
+	if (IS_ERR(cpts->ptp_clock)) {
+		dev_err(dev, "Failed to register ptp clk %ld\n",
+			PTR_ERR(cpts->ptp_clock));
+		return ERR_CAST(cpts->ptp_clock);
+	}
+	cpts->phc_index = ptp_clock_index(cpts->ptp_clock);
+
+	ret = devm_add_action_or_reset(dev,
+				       (void(*)(void *))ptp_clock_unregister,
+				       cpts->ptp_clock);
+	if (ret) {
+		dev_err(dev, "failed to add ptpclk reset action %d", ret);
+		return ERR_PTR(ret);
+	}
+
+	ret = devm_request_threaded_irq(dev, cpts->irq, NULL,
+					am65_cpts_interrupt,
+					IRQF_ONESHOT, dev_name(dev), cpts);
+	if (ret < 0) {
+		dev_err(cpts->dev, "error attaching irq %d\n", ret);
+		return ERR_PTR(ret);
+	}
+
+	dev_info(dev, "CPTS ver 0x%08x, freq:%u, add_val:%u pps:%d\n",
+		 am65_cpts_read32(cpts, idver),
+		 cpts->refclk_freq, cpts->ts_add_val, cpts->pps_present);
+
+	return cpts;
+}
+EXPORT_SYMBOL_GPL(am65_cpts_create);
+
+static int am65_cpts_probe(struct platform_device *pdev)
+{
+	struct device_node *node = pdev->dev.of_node;
+	struct device *dev = &pdev->dev;
+	struct am65_cpts *cpts;
+	struct resource *res;
+	void __iomem *base;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cpts");
+	base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	cpts = am65_cpts_create(dev, base, node);
+	if (IS_ERR(cpts))
+		return PTR_ERR(cpts);
+
+	return 0;
+}
+
+static const struct of_device_id am65_cpts_of_match[] = {
+	{ .compatible = "ti,am65-cpts", },
+	{ .compatible = "ti,j721e-cpts", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, am65_cpts_of_match);
+
+static struct platform_driver am65_cpts_driver = {
+	.probe		= am65_cpts_probe,
+	.driver		= {
+		.name	= "am65-cpts",
+		.of_match_table = am65_cpts_of_match,
+	},
+};
+module_platform_driver(am65_cpts_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Grygorii Strashko <grygorii.strashko@ti.com>");
+MODULE_DESCRIPTION("TI K3 AM65 CPTS driver");
diff -Naur --no-dereference a/drivers/net/ethernet/ti/am65-cpts.h b/drivers/net/ethernet/ti/am65-cpts.h
--- a/drivers/net/ethernet/ti/am65-cpts.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/net/ethernet/ti/am65-cpts.h	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * TI K3 AM65 CPTS driver interface
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
+ */
+
+#ifndef K3_CPTS_H_
+#define K3_CPTS_H_
+
+#include <linux/device.h>
+#include <linux/of.h>
+
+struct am65_cpts;
+
+#if IS_ENABLED(CONFIG_TI_AM65_CPTS)
+struct am65_cpts *am65_cpts_create(struct device *dev, void __iomem *regs,
+				   struct device_node *node);
+int am65_cpts_phc_index(struct am65_cpts *cpts);
+void am65_cpts_tx_timestamp(struct am65_cpts *cpts, struct sk_buff *skb);
+void am65_cpts_ask_tx_timestamp(struct am65_cpts *cpts, struct sk_buff *skb);
+void am65_cpts_rx_enable(struct am65_cpts *cpts, bool en);
+#else
+static inline struct am65_cpts *am65_cpts_create(struct device *dev,
+						 void __iomem *regs,
+						 struct device_node *node)
+{
+	return ERR_PTR(-EOPNOTSUPP);
+}
+
+static inline int am65_cpts_phc_index(struct am65_cpts *cpts)
+{
+	return -1;
+}
+
+static inline void am65_cpts_tx_timestamp(struct am65_cpts *cpts,
+					  struct sk_buff *skb)
+{
+}
+
+static inline void am65_cpts_ask_tx_timestamp(struct am65_cpts *cpts,
+					      struct sk_buff *skb)
+{
+}
+
+static inline void am65_cpts_rx_enable(struct am65_cpts *cpts, bool en)
+{
+}
+#endif
+
+#endif /* K3_CPTS_H_ */
diff -Naur --no-dereference a/drivers/net/ethernet/ti/cpsw_ale.c b/drivers/net/ethernet/ti/cpsw_ale.c
--- a/drivers/net/ethernet/ti/cpsw_ale.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/ethernet/ti/cpsw_ale.c	2020-06-19 21:00:31.416227715 -0400
@@ -26,11 +26,14 @@
 #include "cpsw_ale.h"
 
 #define BITMASK(bits)		(BIT(bits) - 1)
+#define ALE_ENTRY_BITS		68
+#define ALE_ENTRY_WORDS	DIV_ROUND_UP(ALE_ENTRY_BITS, 32)
 
 #define ALE_VERSION_MAJOR(rev, mask) (((rev) >> 8) & (mask))
 #define ALE_VERSION_MINOR(rev)	(rev & 0xff)
 #define ALE_VERSION_1R3		0x0103
 #define ALE_VERSION_1R4		0x0104
+#define ALE_VERSION_9R3		0x0903
 
 /* ALE Registers */
 #define ALE_IDVER		0x00
@@ -64,6 +67,39 @@
 #define ALE_TABLE_SIZE_MULTIPLIER	1024
 #define ALE_STATUS_SIZE_MASK		0x1f
 #define ALE_TABLE_SIZE_DEFAULT		64
+#define ALE_TBL_ENTRY_SHOW_LEN		160
+#define ALE_RAW_TBL_ENTRY_SHOW_LEN	32
+
+/* ALE Table store VLAN command param indices */
+enum {
+	ALE_VP_VID,
+	ALE_VP_FORCE_UT_EGR,
+	ALE_VP_REG_FLD,
+	ALE_VP_UNREG_FLD,
+	ALE_VP_M_LIST,
+	ALE_VP_NUM,
+};
+
+/* ALE Table store UCAST command param indices */
+enum {
+	ALE_UP_PORT,
+	ALE_UP_BLOCK,
+	ALE_UP_SECURE,
+	ALE_UP_AGEABLE,
+	ALE_UP_ADDR,
+	ALE_UP_VID,
+	ALE_UP_NUM,
+};
+
+/* ALE Table store MCAST command param indices */
+enum {
+	ALE_MP_PORT_MASK,
+	ALE_MP_SUPER,
+	ALE_MP_FW_ST,
+	ALE_MP_ADDR,
+	ALE_MP_VID,
+	ALE_MP_NUM
+};
 
 static inline int cpsw_ale_get_field(u32 *ale_entry, u32 start, u32 bits)
 {
@@ -127,6 +163,8 @@
 DEFINE_ALE_FIELD(vlan_unreg_mcast_idx,	20,	3)
 DEFINE_ALE_FIELD(vlan_reg_mcast_idx,	44,	3)
 
+#define NU_VLAN_UNREG_MCAST_IDX	1
+
 /* The MAC address field in the ALE entry cannot be macroized as above */
 static inline void cpsw_ale_get_addr(u32 *ale_entry, u8 *addr)
 {
@@ -136,7 +174,7 @@
 		addr[i] = cpsw_ale_get_field(ale_entry, 40 - 8*i, 8);
 }
 
-static inline void cpsw_ale_set_addr(u32 *ale_entry, u8 *addr)
+static inline void cpsw_ale_set_addr(u32 *ale_entry, const u8 *addr)
 {
 	int i;
 
@@ -175,7 +213,7 @@
 	return idx;
 }
 
-static int cpsw_ale_match_addr(struct cpsw_ale *ale, u8 *addr, u16 vid)
+static int cpsw_ale_match_addr(struct cpsw_ale *ale, const u8 *addr, u16 vid)
 {
 	u32 ale_entry[ALE_ENTRY_WORDS];
 	int type, idx;
@@ -309,7 +347,7 @@
 	}
 }
 
-int cpsw_ale_add_ucast(struct cpsw_ale *ale, u8 *addr, int port,
+int cpsw_ale_add_ucast(struct cpsw_ale *ale, const u8 *addr, int port,
 		       int flags, u16 vid)
 {
 	u32 ale_entry[ALE_ENTRY_WORDS] = {0, 0, 0};
@@ -336,7 +374,7 @@
 }
 EXPORT_SYMBOL_GPL(cpsw_ale_add_ucast);
 
-int cpsw_ale_del_ucast(struct cpsw_ale *ale, u8 *addr, int port,
+int cpsw_ale_del_ucast(struct cpsw_ale *ale, const u8 *addr, int port,
 		       int flags, u16 vid)
 {
 	u32 ale_entry[ALE_ENTRY_WORDS] = {0, 0, 0};
@@ -352,7 +390,7 @@
 }
 EXPORT_SYMBOL_GPL(cpsw_ale_del_ucast);
 
-int cpsw_ale_add_mcast(struct cpsw_ale *ale, u8 *addr, int port_mask,
+int cpsw_ale_add_mcast(struct cpsw_ale *ale, const u8 *addr, int port_mask,
 		       int flags, u16 vid, int mcast_state)
 {
 	u32 ale_entry[ALE_ENTRY_WORDS] = {0, 0, 0};
@@ -386,7 +424,7 @@
 }
 EXPORT_SYMBOL_GPL(cpsw_ale_add_mcast);
 
-int cpsw_ale_del_mcast(struct cpsw_ale *ale, u8 *addr, int port_mask,
+int cpsw_ale_del_mcast(struct cpsw_ale *ale, const u8 *addr, int port_mask,
 		       int flags, u16 vid)
 {
 	u32 ale_entry[ALE_ENTRY_WORDS] = {0, 0, 0};
@@ -424,6 +462,22 @@
 	writel(unreg_mcast, ale->params.ale_regs + ALE_VLAN_MASK_MUX(idx));
 }
 
+static void cpsw_ale_get_vlan_mcast(struct cpsw_ale *ale, u32 *ale_entry,
+				    int *reg_mcast, int *unreg_mcast)
+{
+	int idx;
+
+	/* Get VLAN registered multicast flood mask */
+	idx = cpsw_ale_get_vlan_reg_mcast_idx(ale_entry);
+	*reg_mcast = __raw_readl(ale->params.ale_regs +
+				 ALE_VLAN_MASK_MUX(idx));
+
+	/* Get VLAN unregistered multicast flood mask */
+	idx = cpsw_ale_get_vlan_unreg_mcast_idx(ale_entry);
+	*unreg_mcast = __raw_readl(ale->params.ale_regs +
+				   ALE_VLAN_MASK_MUX(idx));
+}
+
 int cpsw_ale_add_vlan(struct cpsw_ale *ale, u16 vid, int port, int untag,
 		      int reg_mcast, int unreg_mcast)
 {
@@ -444,6 +498,8 @@
 		cpsw_ale_set_vlan_unreg_mcast(ale_entry, unreg_mcast,
 					      ale->vlan_field_bits);
 	} else {
+		cpsw_ale_set_vlan_unreg_mcast_idx(ale_entry,
+						  NU_VLAN_UNREG_MCAST_IDX);
 		cpsw_ale_set_vlan_mcast(ale, ale_entry, reg_mcast, unreg_mcast);
 	}
 	cpsw_ale_set_vlan_member_list(ale_entry, port, ale->vlan_field_bits);
@@ -523,6 +579,14 @@
 };
 
 static struct ale_control_info ale_controls[ALE_NUM_CONTROLS] = {
+	[ALE_VERSION]		= {
+		.name		= "version",
+		.offset		= ALE_IDVER,
+		.port_offset	= 0,
+		.shift		= 0,
+		.port_shift	= 0,
+		.bits		= 32,
+	},
 	[ALE_ENABLE]		= {
 		.name		= "enable",
 		.offset		= ALE_CONTROL,
@@ -659,6 +723,22 @@
 		.port_shift	= 0,
 		.bits		= 1,
 	},
+	[ALE_PORT_MACONLY]	= {
+		.name		= "mac_only_port_mode",
+		.offset		= ALE_PORTCTL,
+		.port_offset	= 4,
+		.shift		= 11,
+		.port_shift	= 0,
+		.bits		= 1,
+	},
+	[ALE_PORT_MACONLY_CAF]	= {
+		.name		= "mac_only_port_caf",
+		.offset		= ALE_PORTCTL,
+		.port_offset	= 4,
+		.shift		= 13,
+		.port_shift	= 0,
+		.bits		= 1,
+	},
 	[ALE_PORT_MCAST_LIMIT]	= {
 		.name		= "mcast_limit",
 		.offset		= ALE_PORTCTL,
@@ -675,6 +755,7 @@
 		.port_shift	= 0,
 		.bits		= 8,
 	},
+	/* Fields below has individual registers on NetCP NU switch */
 	[ALE_PORT_UNKNOWN_VLAN_MEMBER] = {
 		.name		= "unknown_vlan_member",
 		.offset		= ALE_UNKNOWNVLAN,
@@ -765,6 +846,806 @@
 }
 EXPORT_SYMBOL_GPL(cpsw_ale_control_get);
 
+int cpsw_ale_set_ratelimit(struct cpsw_ale *ale, unsigned long freq, int port,
+			   unsigned int bcast_rate_limit,
+			   unsigned int mcast_rate_limit,
+			   bool direction)
+
+{
+	unsigned int rate_limit;
+	unsigned long ale_prescale;
+	int val;
+
+	if (!bcast_rate_limit && !mcast_rate_limit) {
+		/* disable rate limit */
+		cpsw_ale_control_set(ale, 0, ALE_RATE_LIMIT, 0);
+		cpsw_ale_control_set(ale, port, ALE_PORT_BCAST_LIMIT, 0);
+		cpsw_ale_control_set(ale, port, ALE_PORT_MCAST_LIMIT, 0);
+		writel(0, ale->params.ale_regs + ALE_PRESCALE);
+		return 0;
+	}
+
+	/* configure Broadcast and Multicast Rate Limit
+	 * number_of_packets = (Fclk / ALE_PRESCALE) * port.BCASTMCAST/_LIMIT
+	 * ALE_PRESCALE width is 19bit and min value 0x10
+	 * with Fclk = 125MHz and port.BCASTMCAST/_LIMIT = 1
+	 *
+	 * max number_of_packets = (125MHz / 0x10) * 1 = 7812500
+	 * min number_of_packets = (125MHz / 0xFFFFF) * 1 = 119
+	 *
+	 * above values are more than enough (with higher Fclk they will be
+	 * just better), so port.BCASTMCAST/_LIMIT can be selected to be 1
+	 * while ALE_PRESCALE is calculated as:
+	 *  ALE_PRESCALE = Fclk / number_of_packets
+	 */
+	rate_limit = max_t(unsigned int, bcast_rate_limit, mcast_rate_limit);
+	ale_prescale = freq / rate_limit;
+	if (ale_prescale & (~0xfffff))
+		return -EINVAL;
+
+	cpsw_ale_control_set(ale, 0, ALE_RATE_LIMIT_TX, direction);
+	val = bcast_rate_limit ? 1 : 0;
+	cpsw_ale_control_set(ale, port, ALE_PORT_BCAST_LIMIT, val);
+	val = mcast_rate_limit ? 1 : 0;
+	cpsw_ale_control_set(ale, port, ALE_PORT_MCAST_LIMIT, val);
+	writel((u32)ale_prescale, ale->params.ale_regs + ALE_PRESCALE);
+	cpsw_ale_control_set(ale, 0, ALE_RATE_LIMIT, 1);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(cpsw_ale_set_ratelimit);
+
+static int cpsw_ale_dump_mcast(struct cpsw_ale *ale, u32 *ale_entry, char *buf,
+			       int len)
+{
+	static const char * const str_mcast_state[] = {"f", "blf", "lf", "f"};
+	int mcast_state = cpsw_ale_get_mcast_state(ale_entry);
+	int port_mask   = cpsw_ale_get_port_mask(ale_entry,
+						 ale->port_mask_bits);
+	int super       = cpsw_ale_get_super(ale_entry);
+	int outlen = 0;
+
+	outlen += snprintf(buf + outlen, len - outlen,
+			   "mcstate: %s(%d), ", str_mcast_state[mcast_state],
+			   mcast_state);
+	outlen += snprintf(buf + outlen, len - outlen,
+			   "port mask: %x, %ssuper\n", port_mask,
+			   super ? "" : "no ");
+	return outlen;
+}
+
+static int cpsw_ale_dump_ucast(struct cpsw_ale *ale,
+			       u32 *ale_entry, char *buf, int len)
+{
+	int outlen = 0;
+	static const char * const str_ucast_type[] = {"persistent", "untouched",
+							"oui", "touched"};
+	int ucast_type  = cpsw_ale_get_ucast_type(ale_entry);
+	int port_num    = cpsw_ale_get_port_num(ale_entry,
+						ale->port_num_bits);
+	int secure      = cpsw_ale_get_secure(ale_entry);
+	int blocked     = cpsw_ale_get_blocked(ale_entry);
+
+	outlen += snprintf(buf + outlen, len - outlen,
+			   "uctype: %s(%d)", str_ucast_type[ucast_type],
+			   ucast_type);
+	if (ucast_type == ALE_UCAST_OUI)
+		outlen += snprintf(buf + outlen, len - outlen, "\n");
+	else
+		outlen += snprintf(buf + outlen, len - outlen,
+				", port: %d%s%s\n", port_num,
+				secure ? ", Secure" : "",
+				blocked ? ", Blocked" : "");
+	return outlen;
+}
+
+static int cpsw_ale_dump_vlan(struct cpsw_ale *ale, u32 *ale_entry,
+			      char *buf, int len)
+{
+	int outlen = 0, reg_mc_fld, unreg_mc_fld;
+	int force_utag_egress	=
+		cpsw_ale_get_vlan_untag_force(ale_entry,
+					      ale->vlan_field_bits);
+	int mem_list	=
+		cpsw_ale_get_vlan_member_list(ale_entry, ale->vlan_field_bits);
+
+	if (!ale->params.nu_switch_ale) {
+		reg_mc_fld =
+			cpsw_ale_get_vlan_reg_mcast(ale_entry,
+						    ale->vlan_field_bits);
+		unreg_mc_fld =
+			cpsw_ale_get_vlan_unreg_mcast(ale_entry,
+						      ale->vlan_field_bits);
+	} else {
+		cpsw_ale_get_vlan_mcast(ale, ale_entry, &reg_mc_fld,
+					&unreg_mc_fld);
+	}
+
+	outlen += snprintf(buf + outlen, len - outlen,
+			   "force_untag_egress: %02x, ", force_utag_egress);
+	outlen += snprintf(buf + outlen, len - outlen,
+			   "reg_fld: %02x, ", reg_mc_fld);
+	outlen += snprintf(buf + outlen, len - outlen,
+			   "unreg_fld: %02x, ", unreg_mc_fld);
+	outlen += snprintf(buf + outlen, len - outlen,
+			   "mem_list: %02x\n", mem_list);
+	return outlen;
+}
+
+static int cpsw_ale_dump_entry(struct cpsw_ale *ale, int idx, u32 *ale_entry,
+			       char *buf, int len)
+{
+	static const char * const str_type[] = {"free", "addr",
+						"vlan", "vlan+addr"};
+	int type, outlen = 0;
+	u8 addr[6];
+
+	type = cpsw_ale_get_entry_type(ale_entry);
+	if (type == ALE_TYPE_FREE)
+		return outlen;
+
+	if (len < ALE_TBL_ENTRY_SHOW_LEN)
+		return outlen;
+
+	if (idx >= 0) {
+		outlen += snprintf(buf + outlen, len - outlen,
+				   "index %d, ", idx);
+	}
+
+	outlen += snprintf(buf + outlen, len - outlen, "raw: %08x %08x %08x, ",
+			   ale_entry[0], ale_entry[1], ale_entry[2]);
+
+	outlen += snprintf(buf + outlen, len - outlen,
+			   "type: %s(%d), ", str_type[type], type);
+
+	if (type != ALE_TYPE_VLAN) {
+		cpsw_ale_get_addr(ale_entry, addr);
+		outlen += snprintf(buf + outlen, len - outlen,
+			   "addr: %02x:%02x:%02x:%02x:%02x:%02x, ",
+			   (addr)[0], (addr)[1], (addr)[2],
+			   (addr)[3], (addr)[4], (addr)[5]);
+	}
+
+	if (type == ALE_TYPE_VLAN || type == ALE_TYPE_VLAN_ADDR) {
+		outlen += snprintf(buf + outlen, len - outlen, "vlan: %d, ",
+				   cpsw_ale_get_vlan_id(ale_entry));
+	}
+
+	if (type == ALE_TYPE_VLAN)
+		outlen += cpsw_ale_dump_vlan(ale, ale_entry,
+				buf + outlen, len - outlen);
+	else
+		outlen += cpsw_ale_get_mcast(ale_entry) ?
+		  cpsw_ale_dump_mcast(ale, ale_entry, buf + outlen,
+				      len - outlen) :
+		  cpsw_ale_dump_ucast(ale, ale_entry, buf + outlen,
+				      len - outlen);
+
+	return outlen;
+}
+
+static ssize_t ale_control_show(struct device *dev,
+				struct device_attribute *attr,
+				char *buf)
+{
+	int i, port, len = 0, max_control = ARRAY_SIZE(ale_controls);
+	struct cpsw_ale *ale = control_attr_to_ale(attr);
+	struct cpsw_ale_params *params = &ale->params;
+	const struct ale_control_info *info;
+	const char *fmt = "%s=%d\n";
+	u32 reg;
+
+	for (i = 0, info = ale_controls; i < max_control; i++, info++) {
+		if (i == ALE_VERSION) {
+			reg = cpsw_ale_control_get(ale, 0, i);
+			len +=
+			snprintf(buf + len, SZ_4K - len,
+				 "%s=(ALE_ID=0x%04x) Rev %d.%d\n",
+				 info->name,
+				 (reg & 0xffff0000) >> 16,
+				 ALE_VERSION_MAJOR(reg,
+						   params->major_ver_mask),
+						   ALE_VERSION_MINOR(reg));
+			continue;
+		}
+
+		/* global controls */
+		if (info->port_shift == 0 &&  info->port_offset == 0) {
+			if (i >= ALE_PORT_UNKNOWN_VLAN_MEMBER &&
+			    i <= ALE_PORT_UNTAGGED_EGRESS)
+				fmt = "%s=0x%x\n";
+
+			len += snprintf(buf + len, SZ_4K - len,
+					fmt, info->name,
+					cpsw_ale_control_get(ale, 0, i));
+			continue;
+		}
+
+		/* port specific controls */
+		for (port = 0; port < ale->params.ale_ports; port++) {
+			len += snprintf(buf + len, SZ_4K - len,
+					"%s.%d=%d\n", info->name, port,
+					cpsw_ale_control_get(ale, port, i));
+		}
+	}
+
+	return len;
+}
+
+static ssize_t ale_control_store(struct device *dev,
+				 struct device_attribute *attr,
+				 const char *buf, size_t count)
+{
+	int port = 0, value, len, ret, control, max_control = ALE_NUM_CONTROLS;
+	struct cpsw_ale *ale = control_attr_to_ale(attr);
+	char ctrl_str[33], tmp_str[9];
+	unsigned long end;
+
+	len = strcspn(buf, ".=");
+	if (len >= 32)
+		return -ENOMEM;
+
+	strncpy(ctrl_str, buf, len);
+	ctrl_str[len] = '\0';
+	buf += len;
+
+	if (*buf == '.') {
+		++buf;
+		len = strcspn(buf, "=");
+		if (len >= 8)
+			return -ENOMEM;
+		strncpy(tmp_str, buf, len);
+		tmp_str[len] = '\0';
+		if (kstrtoul(tmp_str, 0, &end))
+			return -EINVAL;
+		port = (int)end;
+		buf += len;
+	}
+
+	if (*buf != '=')
+		return -EINVAL;
+
+	if (kstrtoul(buf + 1, 0, &end))
+		return -EINVAL;
+
+	value = (int)end;
+
+	for (control = 0; control < max_control; control++)
+		if (strcmp(ctrl_str, ale_controls[control].name) == 0)
+			break;
+
+	if (control >= max_control)
+		return -ENOENT;
+
+	dev_dbg(ale->params.dev, "processing command %s.%d=%d\n",
+		ale_controls[control].name, port, value);
+
+	ret = cpsw_ale_control_set(ale, port, control, value);
+	if (ret < 0)
+		return ret;
+
+	return count;
+}
+DEVICE_ATTR_RW(ale_control);
+
+static ssize_t ale_table_show(struct device *dev,
+			      struct device_attribute *attr,
+			      char *buf)
+{
+	int not_shown = 0, total_outlen = 0, type, shown = 0;
+	struct cpsw_ale *ale = table_attr_to_ale(attr);
+	int len = SZ_4K, outlen = 0, idx, start;
+	u32 ale_entry[ALE_ENTRY_WORDS];
+
+	start = ale->show_next;
+
+	for (idx = start; (idx < ale->params.ale_entries) &&
+	     (len > total_outlen); idx++) {
+		cpsw_ale_read(ale, idx, ale_entry);
+		outlen = cpsw_ale_dump_entry(ale, idx, ale_entry,
+					     buf + total_outlen,
+					     len - total_outlen);
+		if (outlen == 0) {
+			type = cpsw_ale_get_entry_type(ale_entry);
+			if (type != ALE_TYPE_FREE) {
+				++not_shown;
+				break;
+			}
+		} else {
+			total_outlen += outlen;
+			++shown;
+		}
+	}
+
+	/* update next show index */
+	if (idx >= ale->params.ale_entries)
+		ale->show_next = 0;
+	else
+		ale->show_next = idx;
+
+	if (len > total_outlen + 32)
+		total_outlen += snprintf(buf + total_outlen, len - total_outlen,
+				"[%d..%d]: %d entries%s\n", start, idx - 1,
+				shown, not_shown ? ", +" : "");
+
+	return total_outlen;
+}
+
+struct ale_table_param {
+	const char *name;
+	union	{
+		int	val;
+		u8	addr[6];
+	};
+};
+
+struct ale_table_cmd {
+	const char *name;
+	int (*process)(struct cpsw_ale *ale,
+		       const u8 *params_str, size_t str_len);
+};
+
+static struct ale_table_param vlan_params[] = {
+	[ALE_VP_VID]		= { .name = "vid", },
+	[ALE_VP_FORCE_UT_EGR]	= { .name = "force_untag_egress", },
+	[ALE_VP_REG_FLD]	= { .name = "reg_fld_mask", },
+	[ALE_VP_UNREG_FLD]	= { .name = "unreg_fld_mask", },
+	[ALE_VP_M_LIST]		= { .name = "mem_list", },
+};
+
+static struct ale_table_param vlan_ucast_params[] = {
+	[ALE_UP_PORT]		= { .name = "port", },
+	[ALE_UP_BLOCK]		= { .name = "block", },
+	[ALE_UP_SECURE]		= { .name = "secure", },
+	[ALE_UP_AGEABLE]	= { .name = "ageable", },
+	[ALE_UP_ADDR]		= { .name = "addr", },
+	[ALE_UP_VID]		= { .name = "vid", },
+};
+
+static struct ale_table_param vlan_mcast_params[] = {
+	[ALE_MP_PORT_MASK]	= { .name = "port_mask", },
+	[ALE_MP_SUPER]		= { .name = "supervisory", },
+	[ALE_MP_FW_ST]		= { .name = "mc_fw_st", },
+	[ALE_MP_ADDR]		= { .name = "addr", },
+	[ALE_MP_VID]		= { .name = "vid", },
+};
+
+static struct ale_table_param oui_params[] = {
+	{ .name	= "addr", },
+};
+
+static void cpsw_ale_table_store_init_params(struct ale_table_param *params,
+					     int param_num)
+{
+	int i;
+
+	for (i = 0; i < param_num; i++)
+		memset(params[i].addr, 0, 6);
+}
+
+static int cpsw_ale_table_store_get_params(struct cpsw_ale *ale,
+					   struct ale_table_param *params,
+					   int param_num, const u8 *params_str,
+					   size_t str_len)
+{
+	char param_name[33], val_str[33];
+	size_t tmp_len = str_len;
+	int len, i, n, addr_len;
+	unsigned int iaddr[6];
+	unsigned long end;
+
+	while (tmp_len > 0) {
+		len = strcspn(params_str, "=");
+		if (len >= 32)
+			return -ENOMEM;
+
+		strncpy(param_name, params_str, len);
+		param_name[len] = '\0';
+		params_str += len;
+		tmp_len -= len;
+
+		if (*params_str != '=')
+			return -EINVAL;
+
+		++params_str;
+		--tmp_len;
+
+		len = strcspn(params_str, ".");
+		if (len >= 32)
+			return -ENOMEM;
+
+		strncpy(val_str, params_str, len);
+		val_str[len] = '\0';
+		params_str += len;
+		tmp_len -= len;
+
+		if (*params_str == '.') {
+			++params_str;
+			--tmp_len;
+		}
+
+		for (n = 0; n < param_num; n++) {
+			if (strcmp(param_name, params[n].name) != 0)
+				continue;
+
+			if (strcmp(param_name, "addr") == 0) {
+				addr_len =
+					sscanf(val_str,
+					       "%02x:%02x:%02x:%02x:%02x:%02x",
+					       &iaddr[0], &iaddr[1], &iaddr[2],
+					       &iaddr[3], &iaddr[4], &iaddr[5]);
+				if (addr_len != 6 && addr_len != 3)
+					return -EINVAL;
+
+				for (i = 0; i < addr_len; i++)
+					params[n].addr[i] = iaddr[i];
+
+				break;
+			}
+
+			if (kstrtoul(val_str, 0, &end))
+				return -EINVAL;
+
+			params[n].val = (int)end;
+			break;
+		}
+
+		if (n >= param_num)
+			return -EINVAL;
+	}
+
+	return str_len;
+}
+
+static int cpsw_ale_table_store_vlan(struct cpsw_ale *ale, const u8 *params_str,
+				     size_t str_len)
+{
+	int ret;
+
+	cpsw_ale_table_store_init_params(vlan_params, ALE_VP_NUM);
+	vlan_params[ALE_VP_VID].val = -1;
+
+	ret = cpsw_ale_table_store_get_params(ale, vlan_params, ALE_VP_NUM,
+					      params_str, str_len);
+	if (ret < 0)
+		return ret;
+
+	ret = cpsw_ale_add_vlan(ale, vlan_params[ALE_VP_VID].val,
+				vlan_params[ALE_VP_M_LIST].val,
+				vlan_params[ALE_VP_FORCE_UT_EGR].val,
+				vlan_params[ALE_VP_REG_FLD].val,
+				vlan_params[ALE_VP_UNREG_FLD].val);
+	if (ret < 0)
+		return ret;
+	return str_len;
+}
+
+static int
+cpsw_ale_table_store_vlan_ucast(struct cpsw_ale *ale, const u8 *params_str,
+				size_t str_len, int has_vid)
+{
+	int ret, flags = 0;
+
+	cpsw_ale_table_store_init_params(vlan_ucast_params, ALE_UP_NUM);
+	vlan_ucast_params[ALE_UP_VID].val = -1;
+
+	ret = cpsw_ale_table_store_get_params(ale, vlan_ucast_params,
+					      ALE_UP_NUM, params_str, str_len);
+
+	if (ret < 0)
+		return ret;
+
+	if (!has_vid && vlan_ucast_params[ALE_UP_VID].val >= 0)
+		return -EINVAL;
+
+	if (vlan_ucast_params[ALE_UP_BLOCK].val)
+		flags |= ALE_BLOCKED;
+
+	if (vlan_ucast_params[ALE_UP_SECURE].val)
+		flags |= ALE_SECURE;
+
+	ret = cpsw_ale_add_ucast(ale, vlan_ucast_params[ALE_UP_ADDR].addr,
+				 vlan_ucast_params[ALE_UP_PORT].val, flags,
+				 vlan_ucast_params[ALE_UP_VID].val);
+	if (ret < 0)
+		return ret;
+
+	return str_len;
+}
+
+static int cpsw_ale_table_store_u_proc(struct cpsw_ale *ale,
+				       const u8 *params_str, size_t str_len)
+{
+	return  cpsw_ale_table_store_vlan_ucast(ale, params_str, str_len, 0);
+}
+
+static int cpsw_ale_table_store_vu_proc(struct cpsw_ale *ale,
+					const u8 *params_str, size_t str_len)
+{
+	return  cpsw_ale_table_store_vlan_ucast(ale, params_str, str_len, 1);
+}
+
+static int cpsw_ale_table_store_vlan_mcast(struct cpsw_ale *ale,
+					   const u8 *params_str,
+					   size_t str_len, int has_vid)
+{
+	int ret;
+
+	cpsw_ale_table_store_init_params(vlan_mcast_params, ALE_MP_NUM);
+	vlan_mcast_params[ALE_MP_VID].val = -1;
+
+	ret = cpsw_ale_table_store_get_params(ale, vlan_mcast_params,
+					      ALE_MP_NUM, params_str, str_len);
+	if (ret < 0)
+		return ret;
+
+	if (!has_vid && vlan_mcast_params[ALE_MP_VID].val >= 0)
+		return -EINVAL;
+
+	ret = cpsw_ale_add_mcast(ale, vlan_mcast_params[ALE_MP_ADDR].addr,
+				 vlan_mcast_params[ALE_MP_PORT_MASK].val,
+				 vlan_mcast_params[ALE_MP_SUPER].val,
+				 vlan_mcast_params[ALE_MP_FW_ST].val,
+				 vlan_mcast_params[ALE_MP_VID].val);
+	if (ret < 0)
+		return ret;
+
+	return str_len;
+}
+
+static int cpsw_ale_table_store_m_proc(struct cpsw_ale *ale,
+				       const u8 *params_str, size_t str_len)
+{
+	return  cpsw_ale_table_store_vlan_mcast(ale, params_str, str_len, 0);
+}
+
+static int cpsw_ale_table_store_vm_proc(struct cpsw_ale *ale,
+					const u8 *params_str,
+					size_t str_len)
+{
+	return  cpsw_ale_table_store_vlan_mcast(ale, params_str, str_len, 1);
+}
+
+static int cpsw_ale_add_oui(struct cpsw_ale *ale, u8 *addr)
+{
+	u32 ale_entry[ALE_ENTRY_WORDS] = {0, 0, 0};
+	int idx;
+
+	cpsw_ale_set_entry_type(ale_entry, ALE_TYPE_ADDR);
+
+	cpsw_ale_set_addr(ale_entry, addr);
+	cpsw_ale_set_ucast_type(ale_entry, ALE_UCAST_OUI);
+
+	idx = cpsw_ale_match_addr(ale, addr, -1);
+	if (idx < 0)
+		idx = cpsw_ale_match_free(ale);
+	if (idx < 0)
+		idx = cpsw_ale_find_ageable(ale);
+	if (idx < 0)
+		return -ENOMEM;
+
+	cpsw_ale_write(ale, idx, ale_entry);
+	return 0;
+}
+
+static int cpsw_ale_table_store_oui(struct cpsw_ale *ale, const u8 *params_str,
+				    size_t str_len)
+{
+	int ret;
+
+	cpsw_ale_table_store_init_params(oui_params, 1);
+
+	ret = cpsw_ale_table_store_get_params(ale, oui_params, 1, params_str,
+					      str_len);
+	if (ret < 0)
+		return ret;
+
+	/* Clear out the don't cares */
+	oui_params[0].addr[3] = 0;
+	oui_params[0].addr[4] = 0;
+	oui_params[0].addr[5] = 0;
+
+	ret = cpsw_ale_add_oui(ale, oui_params[0].addr);
+	if (ret < 0)
+		return ret;
+
+	return str_len;
+}
+
+static int cpsw_ale_table_store_del(struct cpsw_ale *ale, int idx)
+{
+	u32 ale_entry[ALE_ENTRY_WORDS];
+	int type;
+
+	dev_dbg(ale->params.dev, "deleting entry[%d] ...\n", idx);
+
+	if (idx >= ale->params.ale_entries)
+		return -EINVAL;
+
+	cpsw_ale_read(ale, idx, ale_entry);
+
+	type = cpsw_ale_get_entry_type(ale_entry);
+	if (type == ALE_TYPE_FREE)
+		return -EINVAL;
+
+	cpsw_ale_set_entry_type(ale_entry, ALE_TYPE_FREE);
+	cpsw_ale_write(ale, idx, ale_entry);
+	return 0;
+}
+
+static struct ale_table_cmd ale_table_cmds[] = {
+	{
+		.name		= "v",
+		.process	= cpsw_ale_table_store_vlan,
+	},
+	{
+		.name		= "m",
+		.process	= cpsw_ale_table_store_m_proc,
+	},
+	{
+		.name		= "vm",
+		.process	= cpsw_ale_table_store_vm_proc,
+	},
+	{
+		.name		= "u",
+		.process	= cpsw_ale_table_store_u_proc,
+	},
+	{
+		.name		= "vu",
+		.process	= cpsw_ale_table_store_vu_proc,
+	},
+	{
+		.name		= "o",
+		.process	= cpsw_ale_table_store_oui,
+	},
+};
+
+static ssize_t cpsw_ale_table_store_proc(struct cpsw_ale *ale,
+					 const char *buf, size_t count)
+{
+	int len, i, tmp_count = count, ret = -EINVAL;
+	char ctrl_str[33];
+	unsigned long end;
+
+	len = strcspn(buf, ".:");
+	if (len >= 5)
+		return -ENOMEM;
+
+	strncpy(ctrl_str, buf, len);
+	ctrl_str[len] = '\0';
+
+	/* skip to param beginning */
+	buf += len;
+	tmp_count -= len;
+
+	if (*buf == ':') {
+		/* delete cmd */
+		if (kstrtoul(ctrl_str, 0, &end))
+			return -EINVAL;
+		ret = cpsw_ale_table_store_del(ale, end);
+		if (ret != 0)
+			return ret;
+		else
+			return count;
+	}
+
+	if (len >= 3)
+		return -ENOMEM;
+
+	if (*buf != '.')
+		return -EINVAL;
+
+	++buf;
+	--tmp_count;
+
+	for (i = 0; i < ARRAY_SIZE(ale_table_cmds); i++) {
+		if (strcmp(ale_table_cmds[i].name, ctrl_str) == 0) {
+			ret = ale_table_cmds[i].process(ale, buf, tmp_count);
+			break;
+		}
+	}
+
+	if (ret < 0)
+		return ret;
+	else
+		return count;
+}
+
+static ssize_t ale_table_store(struct device *dev,
+			       struct device_attribute *attr,
+			       const char *buf, size_t count)
+{
+	struct cpsw_ale *ale = table_attr_to_ale(attr);
+
+	return cpsw_ale_table_store_proc(ale, buf, count);
+}
+DEVICE_ATTR_RW(ale_table);
+
+static int cpsw_ale_dump_entry_raw(int idx, u32 *ale_entry, char *buf, int len)
+{
+	int type, outlen = 0;
+
+	type = cpsw_ale_get_entry_type(ale_entry);
+	if (type == ALE_TYPE_FREE)
+		return outlen;
+
+	if (len < ALE_RAW_TBL_ENTRY_SHOW_LEN)
+		return outlen;
+
+	if (idx >= 0)
+		outlen += snprintf(buf + outlen, len - outlen,
+				   "%d: ", idx);
+
+	outlen += snprintf(buf + outlen, len - outlen, "%02x %08x %08x\n",
+			   ale_entry[0], ale_entry[1], ale_entry[2]);
+
+	return outlen;
+}
+
+static ssize_t ale_table_raw_show(struct device *dev,
+				  struct device_attribute *attr, char *buf)
+{
+	struct cpsw_ale *ale = table_raw_attr_to_ale(attr);
+	int not_shown = 0, total_outlen = 0, shown = 0;
+	int outlen = 0, idx, start, type;
+	u32 ale_entry[ALE_ENTRY_WORDS];
+
+	start = ale->raw_show_next;
+
+	for (idx = start; (idx < ale->params.ale_entries) &&
+	     (total_outlen < PAGE_SIZE); idx++) {
+		cpsw_ale_read(ale, idx, ale_entry);
+		outlen = cpsw_ale_dump_entry_raw(idx, ale_entry,
+						 buf + total_outlen,
+						 PAGE_SIZE - total_outlen);
+		if (outlen == 0) {
+			type = cpsw_ale_get_entry_type(ale_entry);
+			if (type != ALE_TYPE_FREE) {
+				++not_shown;
+				break;
+			}
+		} else {
+			total_outlen += outlen;
+			++shown;
+		}
+	}
+
+	/* update next show index */
+	if (idx >= ale->params.ale_entries)
+		ale->raw_show_next = 0;
+	else
+		ale->raw_show_next = idx;
+
+	if ((total_outlen + 32) < PAGE_SIZE)
+		total_outlen += snprintf(buf + total_outlen,
+					 PAGE_SIZE - total_outlen,
+					 "[%d..%d]: %d entries%s\n",
+					 start, idx - 1, shown, not_shown ?
+					 ", +" : "");
+
+	return total_outlen;
+}
+
+static ssize_t ale_table_raw_store(struct device *dev,
+				   struct device_attribute *attr,
+				   const char *buf, size_t count)
+{
+	struct cpsw_ale *ale = table_raw_attr_to_ale(attr);
+	unsigned long end;
+
+	if (kstrtoul(buf, 0, &end) == 0) {
+		/* set start-show-index command */
+		ale->raw_show_next = (int)end;
+		if (ale->raw_show_next >= ale->params.ale_entries)
+			ale->raw_show_next = 0;
+		return count;
+	}
+
+	/* add or delete command */
+	return cpsw_ale_table_store_proc(ale, buf, count);
+}
+DEVICE_ATTR_RW(ale_table_raw);
+
 static void cpsw_ale_timer(struct timer_list *t)
 {
 	struct cpsw_ale *ale = from_timer(ale, t, timer);
@@ -779,6 +1660,33 @@
 
 void cpsw_ale_start(struct cpsw_ale *ale)
 {
+	int ret, i;
+
+	if (ale->version == ALE_VERSION_1R3 ||
+	    ale->params.nu_switch_ale ||
+	    ale->version == ALE_VERSION_9R3) {
+		/* disable forwarding on all ports */
+		for (i = 0; i < ale->params.ale_ports; ++i)
+			cpsw_ale_control_set(ale, i, ALE_PORT_STATE,
+					     ALE_PORT_STATE_DISABLE);
+
+		ale->ale_control_attr = dev_attr_ale_control;
+		sysfs_attr_init(&ale->ale_control_attr.attr);
+		ret = device_create_file(ale->params.dev,
+					 &ale->ale_control_attr);
+		WARN_ON(ret < 0);
+
+		ale->ale_table_attr = dev_attr_ale_table;
+		sysfs_attr_init(&ale->ale_table_attr.attr);
+		ret = device_create_file(ale->params.dev, &ale->ale_table_attr);
+		WARN_ON(ret < 0);
+
+		ale->ale_table_raw_attr = dev_attr_ale_table_raw;
+		sysfs_attr_init(&ale->ale_table_raw_attr.attr);
+		ret = device_create_file(ale->params.dev,
+					 &ale->ale_table_raw_attr);
+		WARN_ON(ret < 0);
+	}
 	cpsw_ale_control_set(ale, 0, ALE_ENABLE, 1);
 	cpsw_ale_control_set(ale, 0, ALE_CLEAR, 1);
 
@@ -787,6 +1695,7 @@
 		ale->timer.expires = jiffies + ale->ageout;
 		add_timer(&ale->timer);
 	}
+
 }
 EXPORT_SYMBOL_GPL(cpsw_ale_start);
 
@@ -795,6 +1704,13 @@
 	del_timer_sync(&ale->timer);
 	cpsw_ale_control_set(ale, 0, ALE_CLEAR, 1);
 	cpsw_ale_control_set(ale, 0, ALE_ENABLE, 0);
+	if (ale->version == ALE_VERSION_1R3 ||
+	    ale->params.nu_switch_ale ||
+	    ale->version == ALE_VERSION_9R3) {
+		device_remove_file(ale->params.dev, &ale->ale_table_attr);
+		device_remove_file(ale->params.dev, &ale->ale_control_attr);
+		device_remove_file(ale->params.dev, &ale->ale_table_raw_attr);
+	}
 }
 EXPORT_SYMBOL_GPL(cpsw_ale_stop);
 
diff -Naur --no-dereference a/drivers/net/ethernet/ti/cpsw_ale.h b/drivers/net/ethernet/ti/cpsw_ale.h
--- a/drivers/net/ethernet/ti/cpsw_ale.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/ethernet/ti/cpsw_ale.h	2020-06-19 21:00:31.416227715 -0400
@@ -39,6 +39,17 @@
 	unsigned long		ageout;
 	int			allmulti;
 	u32			version;
+	struct device_attribute ale_control_attr;
+#define control_attr_to_ale(attr)	\
+	container_of(attr, struct cpsw_ale, ale_control_attr)
+	struct device_attribute ale_table_attr;
+#define table_attr_to_ale(attr)		\
+	container_of(attr, struct cpsw_ale, ale_table_attr)
+	struct device_attribute ale_table_raw_attr;
+#define table_raw_attr_to_ale(attr)		\
+	container_of(attr, struct cpsw_ale, ale_table_raw_attr)
+	int			show_next;
+	int			raw_show_next;
 	/* These bits are different on NetCP NU Switch ALE */
 	u32			port_mask_bits;
 	u32			port_num_bits;
@@ -47,6 +58,7 @@
 
 enum cpsw_ale_control {
 	/* global */
+	ALE_VERSION,
 	ALE_ENABLE,
 	ALE_CLEAR,
 	ALE_AGEOUT,
@@ -69,6 +81,8 @@
 	ALE_PORT_UNKNOWN_MCAST_FLOOD,
 	ALE_PORT_UNKNOWN_REG_MCAST_FLOOD,
 	ALE_PORT_UNTAGGED_EGRESS,
+	ALE_PORT_MACONLY,
+	ALE_PORT_MACONLY_CAF,
 	ALE_PORT_BCAST_LIMIT,
 	ALE_PORT_MCAST_LIMIT,
 	ALE_NUM_CONTROLS,
@@ -105,18 +119,22 @@
 void cpsw_ale_stop(struct cpsw_ale *ale);
 
 int cpsw_ale_flush_multicast(struct cpsw_ale *ale, int port_mask, int vid);
-int cpsw_ale_add_ucast(struct cpsw_ale *ale, u8 *addr, int port,
+int cpsw_ale_add_ucast(struct cpsw_ale *ale, const u8 *addr, int port,
 		       int flags, u16 vid);
-int cpsw_ale_del_ucast(struct cpsw_ale *ale, u8 *addr, int port,
+int cpsw_ale_del_ucast(struct cpsw_ale *ale, const u8 *addr, int port,
 		       int flags, u16 vid);
-int cpsw_ale_add_mcast(struct cpsw_ale *ale, u8 *addr, int port_mask,
+int cpsw_ale_add_mcast(struct cpsw_ale *ale, const u8 *addr, int port_mask,
 		       int flags, u16 vid, int mcast_state);
-int cpsw_ale_del_mcast(struct cpsw_ale *ale, u8 *addr, int port_mask,
+int cpsw_ale_del_mcast(struct cpsw_ale *ale, const u8 *addr, int port_mask,
 		       int flags, u16 vid);
 int cpsw_ale_add_vlan(struct cpsw_ale *ale, u16 vid, int port, int untag,
 			int reg_mcast, int unreg_mcast);
 int cpsw_ale_del_vlan(struct cpsw_ale *ale, u16 vid, int port);
 void cpsw_ale_set_allmulti(struct cpsw_ale *ale, int allmulti);
+int cpsw_ale_set_ratelimit(struct cpsw_ale *ale, unsigned long freq, int port,
+			   unsigned int bcast_rate_limit,
+			   unsigned int mcast_rate_limit,
+			   bool direction);
 
 int cpsw_ale_control_get(struct cpsw_ale *ale, int port, int control);
 int cpsw_ale_control_set(struct cpsw_ale *ale, int port,
diff -Naur --no-dereference a/drivers/net/ethernet/ti/cpsw.c b/drivers/net/ethernet/ti/cpsw.c
--- a/drivers/net/ethernet/ti/cpsw.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/ethernet/ti/cpsw.c	2020-06-19 21:00:31.416227715 -0400
@@ -37,6 +37,7 @@
 #include <linux/if_vlan.h>
 #include <linux/kmemleak.h>
 #include <linux/sys_soc.h>
+#include <linux/net_switch_config.h>
 
 #include <linux/pinctrl/consumer.h>
 #include <net/pkt_cls.h>
@@ -187,6 +188,10 @@
 module_param(rx_packet_max, int, 0);
 MODULE_PARM_DESC(rx_packet_max, "maximum receive packet size (bytes)");
 
+static int tx_packet_min = CPSW_MIN_PACKET_SIZE;
+module_param(tx_packet_min, int, 0444);
+MODULE_PARM_DESC(tx_packet_min, "minimum tx packet size (bytes)");
+
 static int descs_pool_size = CPSW_CPDMA_DESCS_POOL_SIZE_DEFAULT;
 module_param(descs_pool_size, int, 0444);
 MODULE_PARM_DESC(descs_pool_size, "Number of CPDMA CPPI descriptors in pool");
@@ -283,7 +288,7 @@
 
 #define CTRL_V2_TS_BITS \
 	(TS_320 | TS_319 | TS_132 | TS_131 | TS_130 | TS_129 |\
-	 TS_TTL_NONZERO  | TS_ANNEX_D_EN | TS_LTYPE1_EN)
+	 TS_TTL_NONZERO  | TS_ANNEX_D_EN | TS_LTYPE1_EN | VLAN_LTYPE1_EN)
 
 #define CTRL_V2_ALL_TS_MASK (CTRL_V2_TS_BITS | TS_TX_EN | TS_RX_EN)
 #define CTRL_V2_TX_TS_BITS  (CTRL_V2_TS_BITS | TS_TX_EN)
@@ -293,7 +298,7 @@
 #define CTRL_V3_TS_BITS \
 	(TS_107 | TS_320 | TS_319 | TS_132 | TS_131 | TS_130 | TS_129 |\
 	 TS_TTL_NONZERO | TS_ANNEX_F_EN | TS_ANNEX_D_EN |\
-	 TS_LTYPE1_EN)
+	 TS_LTYPE1_EN | VLAN_LTYPE1_EN)
 
 #define CTRL_V3_ALL_TS_MASK (CTRL_V3_TS_BITS | TS_TX_EN | TS_RX_EN)
 #define CTRL_V3_TX_TS_BITS  (CTRL_V3_TS_BITS | TS_TX_EN)
@@ -466,8 +471,11 @@
 	bool				mqprio_hw;
 	int				fifo_bw[CPSW_TC_NUM];
 	int				shp_cfg_speed;
+	int				tx_ts_enabled;
+	int				rx_ts_enabled;
 	u32 emac_port;
 	struct cpsw_common *cpsw;
+	u8				port_state[3];
 };
 
 struct cpsw_stats {
@@ -484,13 +492,13 @@
 };
 
 #define CPSW_STAT(m)		CPSW_STATS,				\
-				sizeof(((struct cpsw_hw_stats *)0)->m), \
+				FIELD_SIZEOF(struct cpsw_hw_stats, m), \
 				offsetof(struct cpsw_hw_stats, m)
 #define CPDMA_RX_STAT(m)	CPDMA_RX_STATS,				   \
-				sizeof(((struct cpdma_chan_stats *)0)->m), \
+				FIELD_SIZEOF(struct cpdma_chan_stats, m), \
 				offsetof(struct cpdma_chan_stats, m)
 #define CPDMA_TX_STAT(m)	CPDMA_TX_STATS,				   \
-				sizeof(((struct cpdma_chan_stats *)0)->m), \
+				FIELD_SIZEOF(struct cpdma_chan_stats, m), \
 				offsetof(struct cpdma_chan_stats, m)
 
 static const struct cpsw_stats cpsw_gstrings_stats[] = {
@@ -570,16 +578,14 @@
 	return slave_num + 1;
 }
 
-static void cpsw_add_mcast(struct cpsw_priv *priv, u8 *addr)
+static void cpsw_add_mcast(struct cpsw_priv *priv, const u8 *addr)
 {
 	struct cpsw_common *cpsw = priv->cpsw;
 
 	if (cpsw->data.dual_emac) {
 		struct cpsw_slave *slave = cpsw->slaves + priv->emac_port;
-		int slave_port = cpsw_get_slave_port(slave->slave_num);
 
-		cpsw_ale_add_mcast(cpsw->ale, addr,
-				   1 << slave_port | ALE_PORT_HOST,
+		cpsw_ale_add_mcast(cpsw->ale, addr, ALE_PORT_HOST,
 				   ALE_VLAN, slave->port_vlan, 0);
 		return;
 	}
@@ -663,16 +669,35 @@
 	}
 }
 
-static void cpsw_ndo_set_rx_mode(struct net_device *ndev)
+static int cpsw_add_mc_addr(struct net_device *ndev, const u8 *addr)
+{
+	struct cpsw_priv *priv = netdev_priv(ndev);
+
+	cpsw_add_mcast(priv, addr);
+	return 0;
+}
+
+static int cpsw_del_mc_addr(struct net_device *ndev, const u8 *addr)
 {
 	struct cpsw_priv *priv = netdev_priv(ndev);
 	struct cpsw_common *cpsw = priv->cpsw;
-	int vid;
+	int vid, flags;
 
-	if (cpsw->data.dual_emac)
+	if (cpsw->data.dual_emac) {
 		vid = cpsw->slaves[priv->emac_port].port_vlan;
-	else
-		vid = cpsw->data.default_vlan;
+		flags = ALE_VLAN;
+	} else {
+		vid = 0;
+		flags = 0;
+	}
+
+	cpsw_ale_del_mcast(cpsw->ale, addr, 0, flags, vid);
+	return 0;
+}
+
+static void cpsw_ndo_set_rx_mode(struct net_device *ndev)
+{
+	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
 
 	if (ndev->flags & IFF_PROMISC) {
 		/* Enable promiscuous mode */
@@ -685,19 +710,9 @@
 	}
 
 	/* Restore allmulti on vlans if necessary */
-	cpsw_ale_set_allmulti(cpsw->ale, priv->ndev->flags & IFF_ALLMULTI);
-
-	/* Clear all mcast from ALE */
-	cpsw_ale_flush_multicast(cpsw->ale, ALE_ALL_PORTS, vid);
+	cpsw_ale_set_allmulti(cpsw->ale, ndev->flags & IFF_ALLMULTI);
 
-	if (!netdev_mc_empty(ndev)) {
-		struct netdev_hw_addr *ha;
-
-		/* program multicast address list into ALE register */
-		netdev_for_each_mc_addr(ha, ndev) {
-			cpsw_add_mcast(priv, ha->addr);
-		}
-	}
+	__dev_mc_sync(ndev, cpsw_add_mc_addr, cpsw_del_mc_addr);
 }
 
 static void cpsw_intr_enable(struct cpsw_common *cpsw)
@@ -789,6 +804,7 @@
 	struct net_device	*ndev = skb->dev;
 	int			ret = 0, port;
 	struct cpsw_common	*cpsw = ndev_to_cpsw(ndev);
+	struct cpsw_priv	*priv;
 
 	if (cpsw->data.dual_emac) {
 		port = CPDMA_RX_SOURCE_PORT(status);
@@ -823,7 +839,9 @@
 		skb_put(skb, len);
 		if (status & CPDMA_RX_VLAN_ENCAP)
 			cpsw_rx_vlan_encap(skb);
-		cpts_rx_timestamp(cpsw->cpts, skb);
+		priv = netdev_priv(ndev);
+		if (priv->rx_ts_enabled)
+			cpts_rx_timestamp(cpsw->cpts, skb);
 		skb->protocol = eth_type_trans(skb, ndev);
 		netif_receive_skb(skb);
 		ndev->stats.rx_bytes += len;
@@ -1135,7 +1153,8 @@
 
 		/* enable forwarding */
 		cpsw_ale_control_set(cpsw->ale, slave_port,
-				     ALE_PORT_STATE, ALE_PORT_STATE_FORWARD);
+				     ALE_PORT_STATE,
+				     priv->port_state[slave_port]);
 
 		if (phy->speed == 1000)
 			mac_control |= BIT(7);	/* GIGABITEN	*/
@@ -1411,7 +1430,7 @@
 	cpsw_ale_add_vlan(cpsw->ale, slave->port_vlan, port_mask,
 			  port_mask, port_mask, 0);
 	cpsw_ale_add_mcast(cpsw->ale, priv->ndev->broadcast,
-			   port_mask, ALE_VLAN, slave->port_vlan, 0);
+			   ALE_PORT_HOST, ALE_VLAN, slave->port_vlan, 0);
 	cpsw_ale_add_ucast(cpsw->ale, priv->mac_addr,
 			   HOST_PORT_NUM, ALE_VLAN |
 			   ALE_SECURE, slave->port_vlan);
@@ -1468,6 +1487,7 @@
 	slave->mac_control = 0;	/* no link yet */
 
 	slave_port = cpsw_get_slave_port(slave->slave_num);
+	priv->port_state[slave_port] = ALE_PORT_STATE_FORWARD;
 
 	if (cpsw->data.dual_emac)
 		cpsw_add_dual_emac_def_ale_entries(priv, slave, slave_port);
@@ -1957,6 +1977,7 @@
 	struct cpsw_common *cpsw = priv->cpsw;
 
 	cpsw_info(priv, ifdown, "shutting down cpsw device\n");
+	__dev_mc_unsync(priv->ndev, cpsw_del_mc_addr);
 	netif_tx_stop_all_queues(priv->ndev);
 	netif_carrier_off(priv->ndev);
 
@@ -1988,14 +2009,14 @@
 	struct cpdma_chan *txch;
 	int ret, q_idx;
 
-	if (skb_padto(skb, CPSW_MIN_PACKET_SIZE)) {
+	if (skb_padto(skb, tx_packet_min)) {
 		cpsw_err(priv, tx_err, "packet pad failed\n");
 		ndev->stats.tx_dropped++;
 		return NET_XMIT_DROP;
 	}
 
 	if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP &&
-	    cpts_is_tx_enabled(cpts) && cpts_can_timestamp(cpts, skb))
+	    priv->tx_ts_enabled && cpts_can_timestamp(cpts, skb))
 		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
 
 	q_idx = skb_get_queue_mapping(skb);
@@ -2039,13 +2060,13 @@
 
 #if IS_ENABLED(CONFIG_TI_CPTS)
 
-static void cpsw_hwtstamp_v1(struct cpsw_common *cpsw)
+static void cpsw_hwtstamp_v1(struct cpsw_priv *priv)
 {
+	struct cpsw_common *cpsw = priv->cpsw;
 	struct cpsw_slave *slave = &cpsw->slaves[cpsw->data.active_slave];
 	u32 ts_en, seq_id;
 
-	if (!cpts_is_tx_enabled(cpsw->cpts) &&
-	    !cpts_is_rx_enabled(cpsw->cpts)) {
+	if (!priv->tx_ts_enabled && !priv->rx_ts_enabled) {
 		slave_write(slave, 0, CPSW1_TS_CTL);
 		return;
 	}
@@ -2053,10 +2074,10 @@
 	seq_id = (30 << CPSW_V1_SEQ_ID_OFS_SHIFT) | ETH_P_1588;
 	ts_en = EVENT_MSG_BITS << CPSW_V1_MSG_TYPE_OFS;
 
-	if (cpts_is_tx_enabled(cpsw->cpts))
+	if (priv->tx_ts_enabled)
 		ts_en |= CPSW_V1_TS_TX_EN;
 
-	if (cpts_is_rx_enabled(cpsw->cpts))
+	if (priv->rx_ts_enabled)
 		ts_en |= CPSW_V1_TS_RX_EN;
 
 	slave_write(slave, ts_en, CPSW1_TS_CTL);
@@ -2076,20 +2097,20 @@
 	case CPSW_VERSION_2:
 		ctrl &= ~CTRL_V2_ALL_TS_MASK;
 
-		if (cpts_is_tx_enabled(cpsw->cpts))
+		if (priv->tx_ts_enabled)
 			ctrl |= CTRL_V2_TX_TS_BITS;
 
-		if (cpts_is_rx_enabled(cpsw->cpts))
+		if (priv->rx_ts_enabled)
 			ctrl |= CTRL_V2_RX_TS_BITS;
 		break;
 	case CPSW_VERSION_3:
 	default:
 		ctrl &= ~CTRL_V3_ALL_TS_MASK;
 
-		if (cpts_is_tx_enabled(cpsw->cpts))
+		if (priv->tx_ts_enabled)
 			ctrl |= CTRL_V3_TX_TS_BITS;
 
-		if (cpts_is_rx_enabled(cpsw->cpts))
+		if (priv->rx_ts_enabled)
 			ctrl |= CTRL_V3_RX_TS_BITS;
 		break;
 	}
@@ -2099,6 +2120,7 @@
 	slave_write(slave, mtype, CPSW2_TS_SEQ_MTYPE);
 	slave_write(slave, ctrl, CPSW2_CONTROL);
 	writel_relaxed(ETH_P_1588, &cpsw->regs->ts_ltype);
+	writel_relaxed(ETH_P_8021Q, &cpsw->regs->vlan_ltype);
 }
 
 static int cpsw_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
@@ -2106,7 +2128,6 @@
 	struct cpsw_priv *priv = netdev_priv(dev);
 	struct hwtstamp_config cfg;
 	struct cpsw_common *cpsw = priv->cpsw;
-	struct cpts *cpts = cpsw->cpts;
 
 	if (cpsw->version != CPSW_VERSION_1 &&
 	    cpsw->version != CPSW_VERSION_2 &&
@@ -2125,7 +2146,7 @@
 
 	switch (cfg.rx_filter) {
 	case HWTSTAMP_FILTER_NONE:
-		cpts_rx_enable(cpts, 0);
+		priv->rx_ts_enabled = 0;
 		break;
 	case HWTSTAMP_FILTER_ALL:
 	case HWTSTAMP_FILTER_NTP_ALL:
@@ -2133,7 +2154,7 @@
 	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
 	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
 	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
-		cpts_rx_enable(cpts, HWTSTAMP_FILTER_PTP_V1_L4_EVENT);
+		priv->rx_ts_enabled = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
 		cfg.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
 		break;
 	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
@@ -2145,18 +2166,18 @@
 	case HWTSTAMP_FILTER_PTP_V2_EVENT:
 	case HWTSTAMP_FILTER_PTP_V2_SYNC:
 	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
-		cpts_rx_enable(cpts, HWTSTAMP_FILTER_PTP_V2_EVENT);
+		priv->rx_ts_enabled = HWTSTAMP_FILTER_PTP_V2_EVENT;
 		cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
 		break;
 	default:
 		return -ERANGE;
 	}
 
-	cpts_tx_enable(cpts, cfg.tx_type == HWTSTAMP_TX_ON);
+	priv->tx_ts_enabled = cfg.tx_type == HWTSTAMP_TX_ON;
 
 	switch (cpsw->version) {
 	case CPSW_VERSION_1:
-		cpsw_hwtstamp_v1(cpsw);
+		cpsw_hwtstamp_v1(priv);
 		break;
 	case CPSW_VERSION_2:
 	case CPSW_VERSION_3:
@@ -2172,7 +2193,7 @@
 static int cpsw_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
 {
 	struct cpsw_common *cpsw = ndev_to_cpsw(dev);
-	struct cpts *cpts = cpsw->cpts;
+	struct cpsw_priv *priv = netdev_priv(dev);
 	struct hwtstamp_config cfg;
 
 	if (cpsw->version != CPSW_VERSION_1 &&
@@ -2181,10 +2202,8 @@
 		return -EOPNOTSUPP;
 
 	cfg.flags = 0;
-	cfg.tx_type = cpts_is_tx_enabled(cpts) ?
-		      HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF;
-	cfg.rx_filter = (cpts_is_rx_enabled(cpts) ?
-			 cpts->rx_enable : HWTSTAMP_FILTER_NONE);
+	cfg.tx_type = priv->tx_ts_enabled ? HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF;
+	cfg.rx_filter = priv->rx_ts_enabled;
 
 	return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
 }
@@ -2200,6 +2219,272 @@
 }
 #endif /*CONFIG_TI_CPTS*/
 
+static int cpsw_set_port_state(struct cpsw_priv *priv, int port,
+			       int port_state)
+{
+	switch (port_state) {
+	case PORT_STATE_DISABLED:
+		priv->port_state[port] = ALE_PORT_STATE_DISABLE;
+		break;
+	case PORT_STATE_BLOCKED:
+		priv->port_state[port] = ALE_PORT_STATE_BLOCK;
+		break;
+	case PORT_STATE_LEARN:
+		priv->port_state[port] = ALE_PORT_STATE_LEARN;
+		break;
+	case PORT_STATE_FORWARD:
+		priv->port_state[port] = ALE_PORT_STATE_FORWARD;
+		break;
+	default:
+		dev_err(priv->dev, "Switch config: Invalid port state\n");
+		return -EINVAL;
+	}
+	return cpsw_ale_control_set(priv->cpsw->ale, port, ALE_PORT_STATE,
+			priv->port_state[port]);
+}
+
+static int cpsw_switch_config_ioctl(struct net_device *ndev,
+				    struct ifreq *ifrq, int cmd)
+{
+	struct cpsw_priv *priv = netdev_priv(ndev);
+	struct cpsw_common *cpsw = priv->cpsw;
+	struct net_switch_config config;
+	int ret = -EINVAL;
+
+	if (cpsw->data.dual_emac) {
+		dev_err(priv->dev, "CPSW not in switch mode\n");
+		return -ENOTSUPP;
+	}
+
+	/* Only SIOCSWITCHCONFIG is used as cmd argument and hence, there is no
+	 * switch statement required.
+	 * Function calls are based on switch_config.cmd
+	 */
+
+	if (copy_from_user(&config, (ifrq->ifr_data), sizeof(config)))
+		return -EFAULT;
+
+	if (config.vid > 4095) {
+		dev_err(priv->dev, "Invalid VLAN id Arguments for cmd %d\n",
+			config.cmd);
+		return ret;
+	}
+
+	switch (config.cmd) {
+	case CONFIG_SWITCH_ADD_MULTICAST:
+		if ((config.port > 0) && (config.port <= 7) &&
+		    is_multicast_ether_addr(config.addr)) {
+			ret = cpsw_ale_add_mcast(cpsw->ale, config.addr,
+						 config.port, ALE_VLAN,
+						 config.vid, 0);
+		} else {
+			dev_err(priv->dev, "Invalid Arguments for cmd %d\n",
+				config.cmd);
+		}
+		break;
+	case CONFIG_SWITCH_DEL_MULTICAST:
+		if (is_multicast_ether_addr(config.addr)) {
+			ret = cpsw_ale_del_mcast(cpsw->ale, config.addr,
+						 0, ALE_VLAN, config.vid);
+		} else {
+			dev_err(priv->dev, "Invalid Arguments for cmd %d\n",
+				config.cmd);
+		}
+		break;
+	case CONFIG_SWITCH_ADD_VLAN:
+		if ((config.port > 0) && (config.port <= 7)) {
+			ret = cpsw_ale_add_vlan(cpsw->ale, config.vid,
+						config.port,
+						config.untag_port,
+						config.reg_multi,
+						config.unreg_multi);
+		} else {
+			dev_err(priv->dev, "Invalid Arguments for cmd %d\n",
+				config.cmd);
+		}
+		break;
+	case CONFIG_SWITCH_DEL_VLAN:
+		ret = cpsw_ale_del_vlan(cpsw->ale, config.vid, 0);
+		break;
+	case CONFIG_SWITCH_SET_PORT_CONFIG:
+	{
+		struct phy_device *phy = NULL;
+		struct ethtool_link_ksettings cmd;
+
+		if ((config.port == 1) || (config.port == 2))
+			phy = cpsw->slaves[config.port - 1].phy;
+
+		if (!phy) {
+			dev_err(priv->dev, "Phy not Found\n");
+			break;
+		}
+
+		convert_legacy_settings_to_link_ksettings(&cmd, &config.ecmd);
+		cmd.base.phy_address = phy->mdio.addr;
+		ret = phy_ethtool_ksettings_set(phy, &cmd);
+		break;
+	}
+	case CONFIG_SWITCH_GET_PORT_CONFIG:
+	{
+		struct phy_device *phy = NULL;
+		struct ethtool_link_ksettings cmd;
+
+		if ((config.port == 1) || (config.port == 2))
+			phy = cpsw->slaves[config.port - 1].phy;
+
+		if (!phy) {
+			dev_err(priv->dev, "Phy not Found\n");
+			break;
+		}
+
+		cmd.base.phy_address = phy->mdio.addr;
+		phy_ethtool_ksettings_get(phy, &cmd);
+		convert_link_ksettings_to_legacy_settings(&config.ecmd, &cmd);
+
+		ret = copy_to_user(ifrq->ifr_data, &config, sizeof(config));
+		break;
+	}
+	case CONFIG_SWITCH_ADD_UNKNOWN_VLAN_INFO:
+		if ((config.unknown_vlan_member <= 7) &&
+		    (config.unknown_vlan_untag <= 7) &&
+		    (config.unknown_vlan_unreg_multi <= 7) &&
+		    (config.unknown_vlan_reg_multi <= 7)) {
+			cpsw_ale_control_set(cpsw->ale, 0,
+					     ALE_PORT_UNTAGGED_EGRESS,
+					     config.unknown_vlan_untag);
+			cpsw_ale_control_set(cpsw->ale, 0,
+					     ALE_PORT_UNKNOWN_REG_MCAST_FLOOD,
+					     config.unknown_vlan_reg_multi);
+			cpsw_ale_control_set(cpsw->ale, 0,
+					     ALE_PORT_UNKNOWN_MCAST_FLOOD,
+					     config.unknown_vlan_unreg_multi);
+			cpsw_ale_control_set(cpsw->ale, 0,
+					     ALE_PORT_UNKNOWN_VLAN_MEMBER,
+					     config.unknown_vlan_member);
+			ret = 0;
+		} else {
+			dev_err(priv->dev, "Invalid Unknown VLAN Arguments\n");
+		}
+		break;
+	case CONFIG_SWITCH_GET_PORT_STATE:
+		if (config.port == 1 || config.port == 2) {
+			config.port_state = priv->port_state[config.port];
+			ret = copy_to_user(ifrq->ifr_data, &config,
+					   sizeof(config));
+		} else {
+			dev_err(priv->dev, "Invalid Port number\n");
+		}
+		break;
+	case CONFIG_SWITCH_SET_PORT_STATE:
+		if (config.port == 1 || config.port == 2) {
+			ret = cpsw_set_port_state(priv, config.port,
+						  config.port_state);
+		} else {
+			dev_err(priv->dev, "Invalid Port number\n");
+		}
+		break;
+	case CONFIG_SWITCH_GET_PORT_VLAN_CONFIG:
+	{
+		u32 __iomem *port_vlan_reg;
+		u32 port_vlan;
+
+		switch (config.port) {
+		case 0:
+			port_vlan_reg = &cpsw->host_port_regs->port_vlan;
+			port_vlan = readl(port_vlan_reg);
+			ret = 0;
+
+			break;
+		case 1:
+		case 2:
+		{
+			int slave = config.port - 1;
+			int reg = CPSW2_PORT_VLAN;
+
+			if (cpsw->version == CPSW_VERSION_1)
+				reg = CPSW1_PORT_VLAN;
+
+			port_vlan = slave_read(cpsw->slaves + slave, reg);
+			ret = 0;
+
+			break;
+		}
+		default:
+			dev_err(priv->dev, "Invalid Port number\n");
+			break;
+		}
+
+		if (!ret) {
+			config.vid = port_vlan & 0xfff;
+			config.vlan_cfi = port_vlan & BIT(12) ? true : false;
+			config.prio = (port_vlan >> 13) & 0x7;
+			ret = copy_to_user(ifrq->ifr_data, &config,
+					   sizeof(config));
+		}
+		break;
+	}
+	case CONFIG_SWITCH_SET_PORT_VLAN_CONFIG:
+	{
+		void __iomem *port_vlan_reg;
+		u32 port_vlan;
+
+		port_vlan = config.vid;
+		port_vlan |= config.vlan_cfi ? BIT(12) : 0;
+		port_vlan |= (config.prio & 0x7) << 13;
+
+		switch (config.port) {
+		case 0:
+			port_vlan_reg = &cpsw->host_port_regs->port_vlan;
+			writel(port_vlan, port_vlan_reg);
+			ret = 0;
+
+			break;
+		case 1:
+		case 2:
+		{
+			int slave = config.port - 1;
+			int reg = CPSW2_PORT_VLAN;
+
+			if (cpsw->version == CPSW_VERSION_1)
+				reg = CPSW1_PORT_VLAN;
+
+			slave_write(cpsw->slaves + slave, port_vlan, reg);
+			ret = 0;
+
+			break;
+		}
+		default:
+			dev_err(priv->dev, "Invalid Port number\n");
+			break;
+		}
+
+		break;
+	}
+	case CONFIG_SWITCH_RATELIMIT:
+	{
+		if (config.port > 2) {
+			dev_err(priv->dev, "Invalid Port number\n");
+			break;
+		}
+
+		ret = cpsw_ale_set_ratelimit(cpsw->ale,
+					     cpsw->bus_freq_mhz * 1000000,
+					     config.port,
+					     config.bcast_rate_limit,
+					     config.mcast_rate_limit,
+					     !!config.direction);
+		if (ret)
+			dev_err(priv->dev, "CPSW_ALE set ratelimit failed");
+		break;
+	}
+
+	default:
+		ret = -EOPNOTSUPP;
+	}
+
+	return ret;
+}
+
 static int cpsw_ndo_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
 {
 	struct cpsw_priv *priv = netdev_priv(dev);
@@ -2214,6 +2499,8 @@
 		return cpsw_hwtstamp_set(dev, req);
 	case SIOCGHWTSTAMP:
 		return cpsw_hwtstamp_get(dev, req);
+	case SIOCSWITCHCONFIG:
+		return cpsw_switch_config_ioctl(dev, req, cmd);
 	}
 
 	if (!cpsw->slaves[slave_no].phy)
@@ -2294,16 +2581,19 @@
 {
 	int ret;
 	int unreg_mcast_mask = 0;
+	int mcast_mask;
 	u32 port_mask;
 	struct cpsw_common *cpsw = priv->cpsw;
 
 	if (cpsw->data.dual_emac) {
 		port_mask = (1 << (priv->emac_port + 1)) | ALE_PORT_HOST;
 
+		mcast_mask = ALE_PORT_HOST;
 		if (priv->ndev->flags & IFF_ALLMULTI)
-			unreg_mcast_mask = port_mask;
+			unreg_mcast_mask = mcast_mask;
 	} else {
 		port_mask = ALE_ALL_PORTS;
+		mcast_mask = port_mask;
 
 		if (priv->ndev->flags & IFF_ALLMULTI)
 			unreg_mcast_mask = ALE_ALL_PORTS;
@@ -2322,7 +2612,7 @@
 		goto clean_vid;
 
 	ret = cpsw_ale_add_mcast(cpsw->ale, priv->ndev->broadcast,
-				 port_mask, ALE_VLAN, vid, 0);
+				 mcast_mask, ALE_VLAN, vid, 0);
 	if (ret != 0)
 		goto clean_vlan_ucast;
 	return 0;
@@ -3481,7 +3771,7 @@
 
 	dma_params.num_chan		= data->channels;
 	dma_params.has_soft_reset	= true;
-	dma_params.min_packet_size	= CPSW_MIN_PACKET_SIZE;
+	dma_params.min_packet_size	= tx_packet_min;
 	dma_params.desc_mem_size	= data->bd_ram_size;
 	dma_params.desc_align		= 16;
 	dma_params.has_ext_regs		= true;
@@ -3659,8 +3949,7 @@
 #ifdef CONFIG_PM_SLEEP
 static int cpsw_suspend(struct device *dev)
 {
-	struct platform_device	*pdev = to_platform_device(dev);
-	struct net_device	*ndev = platform_get_drvdata(pdev);
+	struct net_device	*ndev = dev_get_drvdata(dev);
 	struct cpsw_common	*cpsw = ndev_to_cpsw(ndev);
 
 	if (cpsw->data.dual_emac) {
@@ -3683,8 +3972,7 @@
 
 static int cpsw_resume(struct device *dev)
 {
-	struct platform_device	*pdev = to_platform_device(dev);
-	struct net_device	*ndev = platform_get_drvdata(pdev);
+	struct net_device	*ndev = dev_get_drvdata(dev);
 	struct cpsw_common	*cpsw = ndev_to_cpsw(ndev);
 
 	/* Select default pin state */
diff -Naur --no-dereference a/drivers/net/ethernet/ti/cpsw-phy-sel.c b/drivers/net/ethernet/ti/cpsw-phy-sel.c
--- a/drivers/net/ethernet/ti/cpsw-phy-sel.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/ethernet/ti/cpsw-phy-sel.c	2020-06-19 21:00:31.416227715 -0400
@@ -33,6 +33,8 @@
 #define AM33XX_GMII_SEL_RGMII2_IDMODE	BIT(5)
 #define AM33XX_GMII_SEL_RGMII1_IDMODE	BIT(4)
 
+#define AM65_GMII_SEL_RGMII_IDMODE	BIT(4)
+
 #define GMII_SEL_MODE_MASK		0x3
 
 struct cpsw_phy_sel_priv {
@@ -157,6 +159,56 @@
 	writel(reg, priv->gmii_sel);
 }
 
+static void cpsw_gmii_sel_am654(struct cpsw_phy_sel_priv *priv,
+				phy_interface_t phy_mode, int slave)
+{
+	u32 reg;
+	u32 mode = 0;
+	bool rgmii_id = false;
+
+	reg = readl(priv->gmii_sel + slave - 1);
+
+	dev_dbg(priv->dev, "old gmii_sel: %08x\n", reg);
+
+	switch (phy_mode) {
+	case PHY_INTERFACE_MODE_RMII:
+		mode = AM33XX_GMII_SEL_MODE_RMII;
+		break;
+
+	case PHY_INTERFACE_MODE_RGMII:
+	case PHY_INTERFACE_MODE_RGMII_RXID:
+		mode = AM33XX_GMII_SEL_MODE_RGMII;
+		break;
+
+	case PHY_INTERFACE_MODE_RGMII_ID:
+	case PHY_INTERFACE_MODE_RGMII_TXID:
+		mode = AM33XX_GMII_SEL_MODE_RGMII;
+		rgmii_id = true;
+		break;
+
+	case PHY_INTERFACE_MODE_SGMII:
+		mode = AM33XX_GMII_SEL_MODE_RGMII;
+		break;
+
+	default:
+		dev_warn(priv->dev,
+			 "Unsupported PHY mode: \"%s\". Defaulting to MII.\n",
+			phy_modes(phy_mode));
+		/* fallthrough */
+	case PHY_INTERFACE_MODE_MII:
+		mode = AM33XX_GMII_SEL_MODE_MII;
+		break;
+	};
+
+	if (rgmii_id)
+		mode |= AM65_GMII_SEL_RGMII_IDMODE;
+
+	reg = mode;
+	dev_dbg(priv->dev, "gmii_sel PHY mode: %s, new gmii_sel: %08x\n",
+		phy_modes(phy_mode), reg);
+	writel(reg, priv->gmii_sel + slave - 1);
+}
+
 static struct platform_driver cpsw_phy_sel_driver;
 static int match(struct device *dev, void *data)
 {
@@ -208,6 +260,10 @@
 		.compatible	= "ti,am43xx-cpsw-phy-sel",
 		.data		= &cpsw_gmii_sel_am3352,
 	},
+	{
+		.compatible	= "ti,am654-cpsw-phy-sel",
+		.data		= &cpsw_gmii_sel_am654,
+	},
 	{}
 };
 
diff -Naur --no-dereference a/drivers/net/ethernet/ti/cpsw_sl.c b/drivers/net/ethernet/ti/cpsw_sl.c
--- a/drivers/net/ethernet/ti/cpsw_sl.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/net/ethernet/ti/cpsw_sl.c	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,314 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Texas Instruments Ethernet Switch media-access-controller (MAC) submodule/
+ * Ethernet MAC Sliver (CPGMAC_SL)
+ *
+ * Copyright (C) 2018 Texas Instruments
+ *
+ */
+
+#include <linux/io.h>
+#include <linux/kernel.h>
+
+#include "cpsw_sl.h"
+
+#define CPSW_SL_REG_NOTUSED U16_MAX
+
+static const u16 cpsw_sl_reg_map_cpsw[] = {
+	[CPSW_SL_IDVER] = 0x00,
+	[CPSW_SL_MACCONTROL] = 0x04,
+	[CPSW_SL_MACSTATUS] = 0x08,
+	[CPSW_SL_SOFT_RESET] = 0x0c,
+	[CPSW_SL_RX_MAXLEN] = 0x10,
+	[CPSW_SL_BOFFTEST] = 0x14,
+	[CPSW_SL_RX_PAUSE] = 0x18,
+	[CPSW_SL_TX_PAUSE] = 0x1c,
+	[CPSW_SL_EMCONTROL] = 0x20,
+	[CPSW_SL_RX_PRI_MAP] = 0x24,
+	[CPSW_SL_TX_GAP] = 0x28,
+};
+
+static const u16 cpsw_sl_reg_map_66ak2hk[] = {
+	[CPSW_SL_IDVER] = 0x00,
+	[CPSW_SL_MACCONTROL] = 0x04,
+	[CPSW_SL_MACSTATUS] = 0x08,
+	[CPSW_SL_SOFT_RESET] = 0x0c,
+	[CPSW_SL_RX_MAXLEN] = 0x10,
+	[CPSW_SL_BOFFTEST] = CPSW_SL_REG_NOTUSED,
+	[CPSW_SL_RX_PAUSE] = 0x18,
+	[CPSW_SL_TX_PAUSE] = 0x1c,
+	[CPSW_SL_EMCONTROL] = 0x20,
+	[CPSW_SL_RX_PRI_MAP] = 0x24,
+	[CPSW_SL_TX_GAP] = CPSW_SL_REG_NOTUSED,
+};
+
+static const u16 cpsw_sl_reg_map_66ak2x_xgbe[] = {
+	[CPSW_SL_IDVER] = 0x00,
+	[CPSW_SL_MACCONTROL] = 0x04,
+	[CPSW_SL_MACSTATUS] = 0x08,
+	[CPSW_SL_SOFT_RESET] = 0x0c,
+	[CPSW_SL_RX_MAXLEN] = 0x10,
+	[CPSW_SL_BOFFTEST] = CPSW_SL_REG_NOTUSED,
+	[CPSW_SL_RX_PAUSE] = 0x18,
+	[CPSW_SL_TX_PAUSE] = 0x1c,
+	[CPSW_SL_EMCONTROL] = 0x20,
+	[CPSW_SL_RX_PRI_MAP] = CPSW_SL_REG_NOTUSED,
+	[CPSW_SL_TX_GAP] = 0x28,
+};
+
+static const u16 cpsw_sl_reg_map_66ak2elg_am65[] = {
+	[CPSW_SL_IDVER] = CPSW_SL_REG_NOTUSED,
+	[CPSW_SL_MACCONTROL] = 0x00,
+	[CPSW_SL_MACSTATUS] = 0x04,
+	[CPSW_SL_SOFT_RESET] = 0x08,
+	[CPSW_SL_RX_MAXLEN] = CPSW_SL_REG_NOTUSED,
+	[CPSW_SL_BOFFTEST] = 0x0c,
+	[CPSW_SL_RX_PAUSE] = 0x10,
+	[CPSW_SL_TX_PAUSE] = 0x40,
+	[CPSW_SL_EMCONTROL] = 0x70,
+	[CPSW_SL_RX_PRI_MAP] = CPSW_SL_REG_NOTUSED,
+	[CPSW_SL_TX_GAP] = 0x74,
+};
+
+#define CPSW_SL_SOFT_RESET_BIT		BIT(0)
+
+#define CPSW_SL_STATUS_PN_IDLE		BIT(31)
+#define CPSW_SL_AM65_STATUS_PN_E_IDLE	BIT(30)
+#define CPSW_SL_AM65_STATUS_PN_P_IDLE	BIT(29)
+#define CPSW_SL_AM65_STATUS_PN_TX_IDLE	BIT(28)
+
+#define CPSW_SL_STATUS_IDLE_MASK_BASE (CPSW_SL_STATUS_PN_IDLE)
+
+#define CPSW_SL_STATUS_IDLE_MASK_K3 \
+	(CPSW_SL_STATUS_IDLE_MASK_BASE | CPSW_SL_AM65_STATUS_PN_E_IDLE | \
+	 CPSW_SL_AM65_STATUS_PN_P_IDLE | CPSW_SL_AM65_STATUS_PN_TX_IDLE)
+
+#define CPSW_SL_CTL_FUNC_BASE \
+	(CPSW_SL_CTL_FULLDUPLEX |\
+	CPSW_SL_CTL_LOOPBACK |\
+	CPSW_SL_CTL_RX_FLOW_EN |\
+	CPSW_SL_CTL_TX_FLOW_EN |\
+	CPSW_SL_CTL_GMII_EN |\
+	CPSW_SL_CTL_TX_PACE |\
+	CPSW_SL_CTL_GIG |\
+	CPSW_SL_CTL_CMD_IDLE |\
+	CPSW_SL_CTL_IFCTL_A |\
+	CPSW_SL_CTL_IFCTL_B |\
+	CPSW_SL_CTL_GIG_FORCE |\
+	CPSW_SL_CTL_EXT_EN |\
+	CPSW_SL_CTL_RX_CEF_EN |\
+	CPSW_SL_CTL_RX_CSF_EN |\
+	CPSW_SL_CTL_RX_CMF_EN)
+
+struct cpsw_sl {
+	struct device *dev;
+	void __iomem *sl_base;
+	const u16 *regs;
+	u32 control_features;
+};
+
+struct cpsw_sl_dev_id {
+	const char *device_id;
+	const u16 *regs;
+	const u32 control_features;
+	const u32 regs_offset;
+	const u32 idle_mask;
+};
+
+static const struct cpsw_sl_dev_id cpsw_sl_id_match[] = {
+	{
+		.device_id = "cpsw",
+		.regs = cpsw_sl_reg_map_cpsw,
+		.control_features = CPSW_SL_CTL_FUNC_BASE |
+				    CPSW_SL_CTL_MTEST |
+				    CPSW_SL_CTL_TX_SHORT_GAP_EN |
+				    CPSW_SL_CTL_TX_SG_LIM_EN,
+		.idle_mask = CPSW_SL_STATUS_IDLE_MASK_BASE,
+	},
+	{
+		.device_id = "66ak2hk",
+		.regs = cpsw_sl_reg_map_66ak2hk,
+		.control_features = CPSW_SL_CTL_FUNC_BASE |
+				    CPSW_SL_CTL_TX_SHORT_GAP_EN,
+		.idle_mask = CPSW_SL_STATUS_IDLE_MASK_BASE,
+	},
+	{
+		.device_id = "66ak2x_xgbe",
+		.regs = cpsw_sl_reg_map_66ak2x_xgbe,
+		.control_features = CPSW_SL_CTL_FUNC_BASE |
+				    CPSW_SL_CTL_XGIG |
+				    CPSW_SL_CTL_TX_SHORT_GAP_EN |
+				    CPSW_SL_CTL_CRC_TYPE |
+				    CPSW_SL_CTL_XGMII_EN,
+		.idle_mask = CPSW_SL_STATUS_IDLE_MASK_BASE,
+	},
+	{
+		.device_id = "66ak2el",
+		.regs = cpsw_sl_reg_map_66ak2elg_am65,
+		.regs_offset = 0x330,
+		.control_features = CPSW_SL_CTL_FUNC_BASE |
+				    CPSW_SL_CTL_MTEST |
+				    CPSW_SL_CTL_TX_SHORT_GAP_EN |
+				    CPSW_SL_CTL_CRC_TYPE |
+				    CPSW_SL_CTL_EXT_EN_RX_FLO |
+				    CPSW_SL_CTL_EXT_EN_TX_FLO |
+				    CPSW_SL_CTL_TX_SG_LIM_EN,
+		.idle_mask = CPSW_SL_STATUS_IDLE_MASK_BASE,
+	},
+	{
+		.device_id = "66ak2g",
+		.regs = cpsw_sl_reg_map_66ak2elg_am65,
+		.regs_offset = 0x330,
+		.control_features = CPSW_SL_CTL_FUNC_BASE |
+				    CPSW_SL_CTL_MTEST |
+				    CPSW_SL_CTL_CRC_TYPE |
+				    CPSW_SL_CTL_EXT_EN_RX_FLO |
+				    CPSW_SL_CTL_EXT_EN_TX_FLO,
+	},
+	{
+		.device_id = "am65",
+		.regs = cpsw_sl_reg_map_66ak2elg_am65,
+		.regs_offset = 0x330,
+		.control_features = CPSW_SL_CTL_FUNC_BASE |
+				    CPSW_SL_CTL_MTEST |
+				    CPSW_SL_CTL_XGIG |
+				    CPSW_SL_CTL_TX_SHORT_GAP_EN |
+				    CPSW_SL_CTL_CRC_TYPE |
+				    CPSW_SL_CTL_XGMII_EN |
+				    CPSW_SL_CTL_EXT_EN_RX_FLO |
+				    CPSW_SL_CTL_EXT_EN_TX_FLO |
+				    CPSW_SL_CTL_TX_SG_LIM_EN |
+				    CPSW_SL_CTL_EXT_EN_XGIG,
+		.idle_mask = CPSW_SL_STATUS_IDLE_MASK_K3,
+	},
+	{ },
+};
+
+u32 cpsw_sl_reg_read(struct cpsw_sl *sl, enum cpsw_sl_regs reg)
+{
+	int val;
+
+	val = readl(sl->sl_base + sl->regs[reg]);
+	dev_dbg(sl->dev, "cpsw_sl: reg: %04x r 0x%08X\n", sl->regs[reg], val);
+	return val;
+}
+EXPORT_SYMBOL_GPL(cpsw_sl_reg_read);
+
+void cpsw_sl_reg_write(struct cpsw_sl *sl, enum cpsw_sl_regs reg, u32 val)
+{
+	dev_dbg(sl->dev, "cpsw_sl: reg: %04x w 0x%08X\n", sl->regs[reg], val);
+	writel(val, sl->sl_base + sl->regs[reg]);
+}
+EXPORT_SYMBOL_GPL(cpsw_sl_reg_write);
+
+static const struct cpsw_sl_dev_id *cpsw_sl_match_id(
+		const struct cpsw_sl_dev_id *id,
+		const char *device_id)
+{
+	if (!(id && device_id))
+		return NULL;
+
+	while (id->device_id[0]) {
+		if (strcmp(device_id, id->device_id) == 0)
+			return id;
+		id++;
+	}
+	return NULL;
+}
+
+struct cpsw_sl *cpsw_sl_get(const char *device_id, struct device *dev,
+			    void __iomem *sl_base)
+{
+	const struct cpsw_sl_dev_id *sl_dev_id;
+	struct cpsw_sl *sl;
+
+	sl = devm_kzalloc(dev, sizeof(struct cpsw_sl), GFP_KERNEL);
+	if (!sl)
+		return ERR_PTR(-ENOMEM);
+	sl->dev = dev;
+	sl->sl_base = sl_base;
+
+	sl_dev_id = cpsw_sl_match_id(cpsw_sl_id_match, device_id);
+	if (!sl_dev_id)
+		return ERR_PTR(-EINVAL);
+	sl->regs = sl_dev_id->regs;
+	sl->control_features = sl_dev_id->control_features;
+	sl->sl_base += sl_dev_id->regs_offset;
+
+	return sl;
+}
+EXPORT_SYMBOL_GPL(cpsw_sl_get);
+
+void cpsw_sl_reset(struct cpsw_sl *sl, unsigned long tmo)
+{
+	unsigned long timeout = jiffies + msecs_to_jiffies(tmo);
+
+	/* Set the soft reset bit */
+	cpsw_sl_reg_write(sl, CPSW_SL_SOFT_RESET, CPSW_SL_SOFT_RESET_BIT);
+
+	/* Wait for the bit to clear */
+	do {
+		cpu_relax();
+	} while ((cpsw_sl_reg_read(sl, CPSW_SL_SOFT_RESET) &
+		  CPSW_SL_SOFT_RESET_BIT) &&
+		  time_after(timeout, jiffies));
+
+	WARN(cpsw_sl_reg_read(sl, CPSW_SL_SOFT_RESET) & CPSW_SL_SOFT_RESET_BIT,
+	     "cpsw_sl failed to soft-reset.\n");
+}
+EXPORT_SYMBOL_GPL(cpsw_sl_reset);
+
+u32 cpsw_sl_ctl_set(struct cpsw_sl *sl, u32 ctl_funcs)
+{
+	u32 val;
+
+	if (ctl_funcs & ~sl->control_features) {
+		dev_err(sl->dev, "cpsw_sl: unsupported func 0x%08X\n",
+			ctl_funcs & (~sl->control_features));
+		return -EINVAL;
+	}
+
+	val = cpsw_sl_reg_read(sl, CPSW_SL_MACCONTROL);
+	val |= ctl_funcs;
+	cpsw_sl_reg_write(sl, CPSW_SL_MACCONTROL, val);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(cpsw_sl_ctl_set);
+
+u32 cpsw_sl_ctl_clr(struct cpsw_sl *sl, u32 ctl_funcs)
+{
+	u32 val;
+
+	if (ctl_funcs & ~sl->control_features) {
+		dev_err(sl->dev, "cpsw_sl: unsupported func 0x%08X\n",
+			ctl_funcs & (~sl->control_features));
+		return -EINVAL;
+	}
+
+	val = cpsw_sl_reg_read(sl, CPSW_SL_MACCONTROL);
+	val &= ~ctl_funcs;
+	cpsw_sl_reg_write(sl, CPSW_SL_MACCONTROL, val);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(cpsw_sl_ctl_clr);
+
+void cpsw_sl_ctl_reset(struct cpsw_sl *sl)
+{
+	cpsw_sl_reg_write(sl, CPSW_SL_MACCONTROL, 0);
+}
+EXPORT_SYMBOL_GPL(cpsw_sl_ctl_reset);
+
+int cpsw_sl_wait_for_idle(struct cpsw_sl *sl)
+{
+	u32 i = 100;
+
+	while (((cpsw_sl_reg_read(sl, CPSW_SL_MACSTATUS) &
+	       CPSW_SL_STATUS_IDLE_MASK_K3) != CPSW_SL_STATUS_IDLE_MASK_K3) &&
+	       i--)
+		cpu_relax();
+
+	return i;
+}
+EXPORT_SYMBOL_GPL(cpsw_sl_wait_for_idle);
diff -Naur --no-dereference a/drivers/net/ethernet/ti/cpsw_sl.h b/drivers/net/ethernet/ti/cpsw_sl.h
--- a/drivers/net/ethernet/ti/cpsw_sl.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/net/ethernet/ti/cpsw_sl.h	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,73 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Texas Instruments Ethernet Switch media-access-controller (MAC) submodule/
+ * Ethernet MAC Sliver (CPGMAC_SL) APIs
+ *
+ * Copyright (C) 2018 Texas Instruments
+ *
+ */
+
+#ifndef __TI_CPSW_SL_H__
+#define __TI_CPSW_SL_H__
+
+#include <linux/device.h>
+
+enum cpsw_sl_regs {
+	CPSW_SL_IDVER,
+	CPSW_SL_MACCONTROL,
+	CPSW_SL_MACSTATUS,
+	CPSW_SL_SOFT_RESET,
+	CPSW_SL_RX_MAXLEN,
+	CPSW_SL_BOFFTEST,
+	CPSW_SL_RX_PAUSE,
+	CPSW_SL_TX_PAUSE,
+	CPSW_SL_EMCONTROL,
+	CPSW_SL_RX_PRI_MAP,
+	CPSW_SL_TX_GAP,
+};
+
+enum {
+	CPSW_SL_CTL_FULLDUPLEX = BIT(0), /* Full Duplex mode */
+	CPSW_SL_CTL_LOOPBACK = BIT(1), /* Loop Back Mode */
+	CPSW_SL_CTL_MTEST = BIT(2), /* Manufacturing Test mode */
+	CPSW_SL_CTL_RX_FLOW_EN = BIT(3), /* Receive Flow Control Enable */
+	CPSW_SL_CTL_TX_FLOW_EN = BIT(4), /* Transmit Flow Control Enable */
+	CPSW_SL_CTL_GMII_EN = BIT(5), /* GMII Enable */
+	CPSW_SL_CTL_TX_PACE = BIT(6), /* Transmit Pacing Enable */
+	CPSW_SL_CTL_GIG = BIT(7), /* Gigabit Mode */
+	CPSW_SL_CTL_XGIG = BIT(8), /* 10 Gigabit Mode */
+	CPSW_SL_CTL_TX_SHORT_GAP_EN = BIT(10), /* Transmit Short Gap Enable */
+	CPSW_SL_CTL_CMD_IDLE = BIT(11), /* Command Idle */
+	CPSW_SL_CTL_CRC_TYPE = BIT(12), /* Port CRC Type */
+	CPSW_SL_CTL_XGMII_EN = BIT(13), /* XGMII Enable */
+	CPSW_SL_CTL_IFCTL_A = BIT(15), /* Interface Control A */
+	CPSW_SL_CTL_IFCTL_B = BIT(16), /* Interface Control B */
+	CPSW_SL_CTL_GIG_FORCE = BIT(17), /* Gigabit Mode Force */
+	CPSW_SL_CTL_EXT_EN = BIT(18), /* External Control Enable */
+	CPSW_SL_CTL_EXT_EN_RX_FLO = BIT(19), /* Ext RX Flow Control Enable */
+	CPSW_SL_CTL_EXT_EN_TX_FLO = BIT(20), /* Ext TX Flow Control Enable */
+	CPSW_SL_CTL_TX_SG_LIM_EN = BIT(21), /* TXt Short Gap Limit Enable */
+	CPSW_SL_CTL_RX_CEF_EN = BIT(22), /* RX Copy Error Frames Enable */
+	CPSW_SL_CTL_RX_CSF_EN = BIT(23), /* RX Copy Short Frames Enable */
+	CPSW_SL_CTL_RX_CMF_EN = BIT(24), /* RX Copy MAC Control Frames Enable */
+	CPSW_SL_CTL_EXT_EN_XGIG = BIT(25),  /* Ext XGIG Control En, k3 only */
+
+	CPSW_SL_CTL_FUNCS_COUNT
+};
+
+struct cpsw_sl;
+
+struct cpsw_sl *cpsw_sl_get(const char *device_id, struct device *dev,
+			    void __iomem *sl_base);
+
+void cpsw_sl_reset(struct cpsw_sl *sl, unsigned long tmo);
+
+u32 cpsw_sl_ctl_set(struct cpsw_sl *sl, u32 ctl_funcs);
+u32 cpsw_sl_ctl_clr(struct cpsw_sl *sl, u32 ctl_funcs);
+void cpsw_sl_ctl_reset(struct cpsw_sl *sl);
+int cpsw_sl_wait_for_idle(struct cpsw_sl *sl);
+
+u32 cpsw_sl_reg_read(struct cpsw_sl *sl, enum cpsw_sl_regs reg);
+void cpsw_sl_reg_write(struct cpsw_sl *sl, enum cpsw_sl_regs reg, u32 val);
+
+#endif /* __TI_CPSW_SL_H__ */
diff -Naur --no-dereference a/drivers/net/ethernet/ti/cpts.c b/drivers/net/ethernet/ti/cpts.c
--- a/drivers/net/ethernet/ti/cpts.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/ethernet/ti/cpts.c	2020-06-19 21:00:31.416227715 -0400
@@ -40,6 +40,11 @@
 #define cpts_read32(c, r)	readl_relaxed(&c->reg->r)
 #define cpts_write32(c, v, r)	writel_relaxed(v, &c->reg->r)
 
+static int cpts_event_port(struct cpts_event *event)
+{
+	return (event->high >> PORT_NUMBER_SHIFT) & PORT_NUMBER_MASK;
+}
+
 static int cpts_match(struct sk_buff *skb, unsigned int ptp_class,
 		      u16 ts_seqid, u8 ts_msgtype);
 
@@ -86,6 +91,25 @@
 	return removed ? 0 : -1;
 }
 
+static void cpts_purge_txq(struct cpts *cpts)
+{
+	struct cpts_skb_cb_data *skb_cb;
+	struct sk_buff *skb, *tmp;
+	int removed = 0;
+
+	skb_queue_walk_safe(&cpts->txq, skb, tmp) {
+		skb_cb = (struct cpts_skb_cb_data *)skb->cb;
+		if (time_after(jiffies, skb_cb->tmo)) {
+			__skb_unlink(skb, &cpts->txq);
+			dev_consume_skb_any(skb);
+			++removed;
+		}
+	}
+
+	if (removed)
+		dev_dbg(cpts->dev, "txq cleaned up %d\n", removed);
+}
+
 static bool cpts_match_tx_ts(struct cpts *cpts, struct cpts_event *event)
 {
 	struct sk_buff *skb, *tmp;
@@ -147,11 +171,18 @@
 		}
 
 		event = list_first_entry(&cpts->pool, struct cpts_event, list);
-		event->tmo = jiffies + 2;
+		event->tmo = jiffies +
+			     msecs_to_jiffies(CPTS_EVENT_RX_TX_TIMEOUT);
 		event->high = hi;
 		event->low = lo;
 		type = event_type(event);
 		switch (type) {
+		case CPTS_EV_HW:
+			event->tmo +=
+				msecs_to_jiffies(CPTS_EVENT_HWSTAMP_TIMEOUT);
+			list_del_init(&event->list);
+			list_add_tail(&event->list, &cpts->events);
+			break;
 		case CPTS_EV_TX:
 			if (cpts_match_tx_ts(cpts, event)) {
 				/* if the new event matches an existing skb,
@@ -167,7 +198,6 @@
 			break;
 		case CPTS_EV_ROLL:
 		case CPTS_EV_HALF:
-		case CPTS_EV_HW:
 			break;
 		default:
 			pr_err("cpts: unknown event type\n");
@@ -276,9 +306,82 @@
 	return 0;
 }
 
+static int cpts_report_ts_events(struct cpts *cpts)
+{
+	struct list_head *this, *next;
+	struct ptp_clock_event pevent;
+	struct cpts_event *event;
+	int reported = 0, ev;
+
+	list_for_each_safe(this, next, &cpts->events) {
+		event = list_entry(this, struct cpts_event, list);
+		ev = event_type(event);
+		if (ev == CPTS_EV_HW) {
+			list_del_init(&event->list);
+			list_add(&event->list, &cpts->pool);
+			/* report the event */
+			pevent.timestamp =
+				timecounter_cyc2time(&cpts->tc, event->low);
+			pevent.type = PTP_CLOCK_EXTTS;
+			pevent.index = cpts_event_port(event) - 1;
+			ptp_clock_event(cpts->clock, &pevent);
+			++reported;
+			continue;
+		}
+	}
+	return reported;
+}
+
+/* HW TS */
+static int cpts_extts_enable(struct cpts *cpts, u32 index, int on)
+{
+	unsigned long flags;
+	u32 v;
+
+	if (index >= cpts->info.n_ext_ts)
+		return -ENXIO;
+
+	if (((cpts->hw_ts_enable & BIT(index)) >> index) == on)
+		return 0;
+
+	spin_lock_irqsave(&cpts->lock, flags);
+
+	v = cpts_read32(cpts, control);
+	if (on) {
+		v |= BIT(8 + index);
+		cpts->hw_ts_enable |= BIT(index);
+	} else {
+		v &= ~BIT(8 + index);
+		cpts->hw_ts_enable &= ~BIT(index);
+	}
+	cpts_write32(cpts, v, control);
+
+	spin_unlock_irqrestore(&cpts->lock, flags);
+
+	if (cpts->hw_ts_enable)
+		/* poll for events faster - evry 200 ms */
+		cpts->ov_check_period =
+			msecs_to_jiffies(CPTS_EVENT_HWSTAMP_TIMEOUT);
+	else
+		cpts->ov_check_period = cpts->ov_check_period_slow;
+
+	ptp_schedule_worker(cpts->clock, cpts->ov_check_period);
+
+	return 0;
+}
+
 static int cpts_ptp_enable(struct ptp_clock_info *ptp,
 			   struct ptp_clock_request *rq, int on)
 {
+	struct cpts *cpts = container_of(ptp, struct cpts, info);
+
+	switch (rq->type) {
+	case PTP_CLK_REQ_EXTTS:
+		return cpts_extts_enable(cpts, rq->extts.index, on);
+	default:
+		break;
+	}
+
 	return -EOPNOTSUPP;
 }
 
@@ -292,8 +395,13 @@
 	spin_lock_irqsave(&cpts->lock, flags);
 	ts = ns_to_timespec64(timecounter_read(&cpts->tc));
 
-	if (!skb_queue_empty(&cpts->txq))
-		delay = CPTS_SKB_TX_WORK_TIMEOUT;
+	if (cpts->hw_ts_enable)
+		cpts_report_ts_events(cpts);
+	if (!skb_queue_empty(&cpts->txq)) {
+		cpts_purge_txq(cpts);
+		if (!skb_queue_empty(&cpts->txq))
+			delay = CPTS_SKB_TX_WORK_TIMEOUT;
+	}
 	spin_unlock_irqrestore(&cpts->lock, flags);
 
 	pr_debug("cpts overflow check at %lld.%09ld\n",
@@ -403,35 +511,37 @@
 	return ns;
 }
 
-void cpts_rx_timestamp(struct cpts *cpts, struct sk_buff *skb)
+int cpts_rx_timestamp(struct cpts *cpts, struct sk_buff *skb)
 {
 	u64 ns;
 	struct skb_shared_hwtstamps *ssh;
 
-	if (!cpts->rx_enable)
-		return;
 	ns = cpts_find_ts(cpts, skb, CPTS_EV_RX);
 	if (!ns)
-		return;
+		return -ENOENT;
 	ssh = skb_hwtstamps(skb);
 	memset(ssh, 0, sizeof(*ssh));
 	ssh->hwtstamp = ns_to_ktime(ns);
+
+	return 0;
 }
 EXPORT_SYMBOL_GPL(cpts_rx_timestamp);
 
-void cpts_tx_timestamp(struct cpts *cpts, struct sk_buff *skb)
+int cpts_tx_timestamp(struct cpts *cpts, struct sk_buff *skb)
 {
 	u64 ns;
 	struct skb_shared_hwtstamps ssh;
 
 	if (!(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))
-		return;
+		return -EPERM;
 	ns = cpts_find_ts(cpts, skb, CPTS_EV_TX);
 	if (!ns)
-		return;
+		return -ENOENT;
 	memset(&ssh, 0, sizeof(ssh));
 	ssh.hwtstamp = ns_to_ktime(ns);
 	skb_tstamp_tx(skb, &ssh);
+
+	return 0;
 }
 EXPORT_SYMBOL_GPL(cpts_tx_timestamp);
 
@@ -507,6 +617,8 @@
 
 	/* Calc overflow check period (maxsec / 2) */
 	cpts->ov_check_period = (HZ * maxsec) / 2;
+	cpts->ov_check_period_slow = cpts->ov_check_period;
+
 	dev_info(cpts->dev, "cpts: overflow check period %lu (jiffies)\n",
 		 cpts->ov_check_period);
 
@@ -539,6 +651,18 @@
 	    (!cpts->cc.mult && cpts->cc.shift))
 		goto of_error;
 
+	if (!of_property_read_u32(node, "cpts-rftclk-sel", &prop)) {
+		if (prop & ~CPTS_RFTCLK_SEL_MASK) {
+			dev_err(cpts->dev, "cpts: invalid cpts_rftclk_sel.\n");
+			goto of_error;
+		}
+		cpts->caps |= CPTS_CAP_RFTCLK_SEL;
+		cpts->rftclk_sel = prop & CPTS_RFTCLK_SEL_MASK;
+	}
+
+	if (!of_property_read_u32(node, "cpts-ext-ts-inputs", &prop))
+		cpts->ext_ts_inputs = prop;
+
 	return 0;
 
 of_error:
@@ -574,10 +698,16 @@
 	if (ret)
 		return ERR_PTR(ret);
 
+	if (cpts->caps & CPTS_CAP_RFTCLK_SEL)
+		cpts_write32(cpts, cpts->rftclk_sel, rftclk_sel);
+
 	cpts->cc.read = cpts_systim_read;
 	cpts->cc.mask = CLOCKSOURCE_MASK(32);
 	cpts->info = cpts_info;
 
+	if (cpts->ext_ts_inputs)
+		cpts->info.n_ext_ts = cpts->ext_ts_inputs;
+
 	cpts_calc_mult_shift(cpts);
 	/* save cc.mult original value as it can be modified
 	 * by cpts_ptp_adjfreq().
diff -Naur --no-dereference a/drivers/net/ethernet/ti/cpts.h b/drivers/net/ethernet/ti/cpts.h
--- a/drivers/net/ethernet/ti/cpts.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/ethernet/ti/cpts.h	2020-06-19 21:00:31.416227715 -0400
@@ -36,7 +36,7 @@
 struct cpsw_cpts {
 	u32 idver;                /* Identification and version */
 	u32 control;              /* Time sync control */
-	u32 res1;
+	u32 rftclk_sel;		  /* Reference Clock Select Register */
 	u32 ts_push;              /* Time stamp event push */
 	u32 ts_load_val;          /* Time stamp load value */
 	u32 ts_load_en;           /* Time stamp load enable */
@@ -68,6 +68,8 @@
 #define INT_TEST             (1<<1)  /* Interrupt Test */
 #define CPTS_EN              (1<<0)  /* Time Sync Enable */
 
+#define CPTS_RFTCLK_SEL_MASK 0x1f
+
 /*
  * Definitions for the single bit resisters:
  * TS_PUSH TS_LOAD_EN  INTSTAT_RAW INTSTAT_MASKED INT_ENABLE EVENT_POP
@@ -101,6 +103,9 @@
 #define CPTS_FIFO_DEPTH 16
 #define CPTS_MAX_EVENTS 32
 
+#define CPTS_EVENT_RX_TX_TIMEOUT 20 /* ms */
+#define CPTS_EVENT_HWSTAMP_TIMEOUT 200 /* ms */
+
 struct cpts_event {
 	struct list_head list;
 	unsigned long tmo;
@@ -108,6 +113,8 @@
 	u32 low;
 };
 
+#define CPTS_CAP_RFTCLK_SEL BIT(0)
+
 struct cpts {
 	struct device *dev;
 	struct cpsw_cpts __iomem *reg;
@@ -126,36 +133,21 @@
 	struct cpts_event pool_data[CPTS_MAX_EVENTS];
 	unsigned long ov_check_period;
 	struct sk_buff_head txq;
+	unsigned long ov_check_period_slow;
+	u32 rftclk_sel;
+	u32 ext_ts_inputs;
+	u32 hw_ts_enable;
+	u32 caps;
 };
 
-void cpts_rx_timestamp(struct cpts *cpts, struct sk_buff *skb);
-void cpts_tx_timestamp(struct cpts *cpts, struct sk_buff *skb);
+int cpts_rx_timestamp(struct cpts *cpts, struct sk_buff *skb);
+int cpts_tx_timestamp(struct cpts *cpts, struct sk_buff *skb);
 int cpts_register(struct cpts *cpts);
 void cpts_unregister(struct cpts *cpts);
 struct cpts *cpts_create(struct device *dev, void __iomem *regs,
 			 struct device_node *node);
 void cpts_release(struct cpts *cpts);
 
-static inline void cpts_rx_enable(struct cpts *cpts, int enable)
-{
-	cpts->rx_enable = enable;
-}
-
-static inline bool cpts_is_rx_enabled(struct cpts *cpts)
-{
-	return !!cpts->rx_enable;
-}
-
-static inline void cpts_tx_enable(struct cpts *cpts, int enable)
-{
-	cpts->tx_enable = enable;
-}
-
-static inline bool cpts_is_tx_enabled(struct cpts *cpts)
-{
-	return !!cpts->tx_enable;
-}
-
 static inline bool cpts_can_timestamp(struct cpts *cpts, struct sk_buff *skb)
 {
 	unsigned int class = ptp_classify_raw(skb);
@@ -169,11 +161,14 @@
 #else
 struct cpts;
 
-static inline void cpts_rx_timestamp(struct cpts *cpts, struct sk_buff *skb)
+static inline int cpts_rx_timestamp(struct cpts *cpts, struct sk_buff *skb)
 {
+	return -EOPNOTSUPP;
 }
-static inline void cpts_tx_timestamp(struct cpts *cpts, struct sk_buff *skb)
+
+static inline int cpts_tx_timestamp(struct cpts *cpts, struct sk_buff *skb)
 {
+	return -EOPNOTSUPP;
 }
 
 static inline
@@ -197,24 +192,6 @@
 {
 }
 
-static inline void cpts_rx_enable(struct cpts *cpts, int enable)
-{
-}
-
-static inline bool cpts_is_rx_enabled(struct cpts *cpts)
-{
-	return false;
-}
-
-static inline void cpts_tx_enable(struct cpts *cpts, int enable)
-{
-}
-
-static inline bool cpts_is_tx_enabled(struct cpts *cpts)
-{
-	return false;
-}
-
 static inline bool cpts_can_timestamp(struct cpts *cpts, struct sk_buff *skb)
 {
 	return false;
diff -Naur --no-dereference a/drivers/net/ethernet/ti/davinci_emac.c b/drivers/net/ethernet/ti/davinci_emac.c
--- a/drivers/net/ethernet/ti/davinci_emac.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/ethernet/ti/davinci_emac.c	2020-06-19 21:00:31.416227715 -0400
@@ -2002,8 +2002,7 @@
 
 static int davinci_emac_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct net_device *ndev = platform_get_drvdata(pdev);
+	struct net_device *ndev = dev_get_drvdata(dev);
 
 	if (netif_running(ndev))
 		emac_dev_stop(ndev);
@@ -2013,8 +2012,7 @@
 
 static int davinci_emac_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct net_device *ndev = platform_get_drvdata(pdev);
+	struct net_device *ndev = dev_get_drvdata(dev);
 
 	if (netif_running(ndev))
 		emac_dev_open(ndev);
diff -Naur --no-dereference a/drivers/net/ethernet/ti/davinci_mdio.c b/drivers/net/ethernet/ti/davinci_mdio.c
--- a/drivers/net/ethernet/ti/davinci_mdio.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/ethernet/ti/davinci_mdio.c	2020-06-19 21:00:31.416227715 -0400
@@ -24,6 +24,7 @@
  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  * ---------------------------------------------------------------------------
  */
+#define pr_fmt(fmt) "MDIO: " fmt
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/platform_device.h>
@@ -107,6 +108,10 @@
 	u32		clk_div;
 };
 
+#if IS_ENABLED(CONFIG_OF)
+static void davinci_mdio_update_dt_from_phymask(u32 phy_mask);
+#endif
+
 static void davinci_mdio_init_clk(struct davinci_mdio_data *data)
 {
 	u32 mdio_in, div, mdio_out_khz, access_time;
@@ -174,6 +179,12 @@
 		/* restrict mdio bus to live phys only */
 		dev_info(data->dev, "detected phy mask %x\n", ~phy_mask);
 		phy_mask = ~phy_mask;
+
+		#if IS_ENABLED(CONFIG_OF)
+		if (of_machine_is_compatible("ti,am335x-bone"))
+			davinci_mdio_update_dt_from_phymask(phy_mask);
+		#endif
+
 	} else {
 		/* desperately scan all phys */
 		dev_warn(data->dev, "no live phy, scanning all\n");
@@ -487,6 +498,93 @@
 	davinci_mdio_enable(data);
 	return 0;
 }
+static void davinci_mdio_update_dt_from_phymask(u32 phy_mask)
+{
+	int i, len, skip;
+	u32 addr;
+	__be32 *old_phy_p, *phy_id_p;
+	struct property *phy_id_property = NULL;
+	struct device_node *node_p, *slave_p;
+
+	addr = 0;
+
+	for (i = 0; i < PHY_MAX_ADDR; i++) {
+		if ((phy_mask & (1 << i)) == 0) {
+			addr = (u32) i;
+		break;
+		}
+	}
+
+	for_each_compatible_node(node_p, NULL, "ti,cpsw") {
+		for_each_node_by_name(slave_p, "slave") {
+
+#if IS_ENABLED(CONFIG_OF_OVERLAY)
+			skip = 1;
+			// Hack, the overlay fixup "slave" doesn't have phy-mode...
+			old_phy_p = (__be32 *) of_get_property(slave_p, "phy-mode", &len);
+
+			if (len != (sizeof(__be32 *) * 1))
+			{
+				skip = 0;
+			}
+
+			if (skip) {
+#endif
+
+			old_phy_p = (__be32 *) of_get_property(slave_p, "phy_id", &len);
+
+			if (len != (sizeof(__be32 *) * 2))
+				goto err_out;
+
+			if (old_phy_p) {
+
+				phy_id_property = kzalloc(sizeof(*phy_id_property), GFP_KERNEL);
+
+				if (! phy_id_property)
+					goto err_out;
+
+				phy_id_property->length = len;
+				phy_id_property->name = kstrdup("phy_id", GFP_KERNEL);
+				phy_id_property->value = kzalloc(len, GFP_KERNEL);
+
+				if (! phy_id_property->name)
+					goto err_out;
+
+				if (! phy_id_property->value)
+					goto err_out;
+
+				memcpy(phy_id_property->value, old_phy_p, len);
+
+				phy_id_p = (__be32 *) phy_id_property->value + 1;
+
+				*phy_id_p = cpu_to_be32(addr);
+
+				of_update_property(slave_p, phy_id_property);
+				pr_info("davinci_mdio: dt: updated phy_id[%d] from phy_mask[%x]\n", addr, phy_mask);
+
+				++addr;
+			}
+#if IS_ENABLED(CONFIG_OF_OVERLAY)
+		}
+#endif
+		}
+	}
+
+	return;
+
+err_out:
+
+	if (phy_id_property) {
+		if (phy_id_property->name)
+			kfree(phy_id_property->name);
+
+	if (phy_id_property->value)
+		kfree(phy_id_property->value);
+
+	if (phy_id_property)
+		kfree(phy_id_property);
+	}
+}
 #endif
 
 #ifdef CONFIG_PM_SLEEP
@@ -521,6 +619,69 @@
 }
 #endif
 
+struct davinci_mdio_data *davinci_mdio_create(struct device *dev,
+					      struct device_node *node,
+					      void __iomem *reg_base,
+					      const char *clk_name)
+{
+	struct davinci_mdio_data *data;
+	int ret;
+	u32 prop;
+
+	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return ERR_PTR(-ENOMEM);
+
+	data->dev = dev;
+	data->regs = reg_base;
+	data->bus = devm_mdiobus_alloc(dev);
+	if (!data->bus)
+		return ERR_PTR(-ENOMEM);
+
+	if (of_property_read_u32(node, "bus_freq", &prop)) {
+		dev_err(dev, "Missing bus_freq property in the DT.\n");
+		return ERR_PTR(-EINVAL);
+	}
+	data->pdata.bus_freq = prop;
+
+	snprintf(data->bus->id, MII_BUS_ID_SIZE, "k3-cpsw-mdio");
+
+	data->bus->name		= dev_name(dev);
+	data->bus->read		= davinci_mdio_read,
+	data->bus->write	= davinci_mdio_write,
+	data->bus->reset	= davinci_mdio_reset,
+	data->bus->parent	= dev;
+	data->bus->priv		= data;
+
+	data->clk = devm_clk_get(dev, clk_name);
+	if (IS_ERR(data->clk)) {
+		dev_err(dev, "failed to get device clock\n");
+		return ERR_CAST(data->clk);
+	}
+
+	davinci_mdio_init_clk(data);
+
+	data->skip_scan = true;
+	ret = of_mdiobus_register(data->bus, node);
+	if (ret) {
+		dev_err(dev, "mdio register err %d.\n", ret);
+		goto bail_out;
+	}
+
+	ret = devm_add_action_or_reset(dev, (void(*)(void *))mdiobus_unregister,
+				       data->bus);
+	if (ret) {
+		dev_err(dev, "failed to add percpu stat free action %d", ret);
+		return ERR_PTR(ret);
+	}
+
+	return data;
+
+bail_out:
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(davinci_mdio_create);
+
 static const struct dev_pm_ops davinci_mdio_pm_ops = {
 	SET_RUNTIME_PM_OPS(davinci_mdio_runtime_suspend,
 			   davinci_mdio_runtime_resume, NULL)
diff -Naur --no-dereference a/drivers/net/ethernet/ti/davinci_mdio_int.h b/drivers/net/ethernet/ti/davinci_mdio_int.h
--- a/drivers/net/ethernet/ti/davinci_mdio_int.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/net/ethernet/ti/davinci_mdio_int.h	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Davinci MDIO internal definitions
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+#ifndef DAVINCI_MDIO_INT_H_
+#define DAVINCI_MDIO_INT_H_
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+
+struct davinci_mdio_data;
+struct davinci_mdio_data *davinci_mdio_create(struct device *dev,
+					      struct device_node *node,
+					      void __iomem *reg_base,
+					      const char *clk_name);
+void davinci_mdio_release(struct davinci_mdio_data *mdio);
+
+#endif /* DAVINCI_MDIO_INT_H_ */
diff -Naur --no-dereference a/drivers/net/ethernet/ti/icssg_classifier.c b/drivers/net/ethernet/ti/icssg_classifier.c
--- a/drivers/net/ethernet/ti/icssg_classifier.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/net/ethernet/ti/icssg_classifier.c	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,443 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Texas Instruments ICSSG Ethernet Driver
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ */
+
+#include <linux/etherdevice.h>
+#include <linux/types.h>
+#include <linux/regmap.h>
+
+#define ICSSG_NUM_CLASSIFIERS	16
+#define ICSSG_NUM_FT1_SLOTS	8
+#define ICSSG_NUM_FT3_SLOTS	16
+
+#define ICSSG_NUM_CLASSIFIERS_IN_USE	5
+
+/* Filter 1 - FT1 */
+#define FT1_NUM_SLOTS	8
+#define FT1_SLOT_SIZE	0x10	/* bytes */
+
+/* offsets from FT1 slot base i.e. slot 1 start */
+#define FT1_DA0		0x0
+#define FT1_DA1		0x4
+#define FT1_DA0_MASK	0x8
+#define FT1_DA1_MASK	0xc
+
+#define FT1_N_REG(slize, n, reg)	(offs[slice].ft1_slot_base + FT1_SLOT_SIZE * (n) + (reg))
+
+#define FT1_LEN_MASK	GENMASK(19, 16)
+#define FT1_LEN_SHIFT	16
+#define FT1_LEN(len)	(((len) << FT1_LEN_SHIFT) & FT1_LEN_MASK)
+
+#define FT1_START_MASK	GENMASK(14, 0)
+#define FT1_START(start)	((start) & FT1_START_MASK)
+
+#define FT1_MATCH_SLOT(n)	(GENMASK(23, 16) & (BIT(n) << 16))
+
+enum ft1_cfg_type {
+	FT1_CFG_TYPE_DISABLED = 0,
+	FT1_CFG_TYPE_EQ,
+	FT1_CFG_TYPE_GT,
+	FT1_CFG_TYPE_LT,
+};
+
+#define FT1_CFG_SHIFT(n)	(2 * (n))
+#define FT1_CFG_MASK(n)	(0x3 << FT1_CFG_SHIFT((n)))
+
+/* Filter 3 -  FT3 */
+#define FT3_NUM_SLOTS	16
+#define FT3_SLOT_SIZE	0x20	/* bytes */
+
+/* offsets from FT3 slot n's base */
+#define FT3_START	0
+#define FT3_START_AUTO	0x4
+#define FT3_START_OFFSET	0x8
+#define FT3_JUMP_OFFSET	0xc
+#define FT3_LEN		0x10
+#define FT3_CFG		0x14
+#define FT3_T		0x18
+#define FT3_T_MASK	0x1c
+
+#define FT3_N_REG(slize, n, reg)	(offs[slice].ft3_slot_base + FT3_SLOT_SIZE * (n) + (reg))
+
+/* offsets from rx_class n's base */
+#define RX_CLASS_AND_EN	0
+#define RX_CLASS_OR_EN	0x4
+
+#define RX_CLASS_NUM_SLOTS	16
+#define RX_CLASS_EN_SIZE	0x8	/* bytes */
+
+#define RX_CLASS_N_REG(slice, n, reg)	(offs[slice].rx_class_base + RX_CLASS_EN_SIZE * (n) + (reg))
+
+/* RX Class Gates */
+#define RX_CLASS_GATES_SIZE	0x4	/* bytes */
+
+#define RX_CLASS_GATES_N_REG(slice, n)	(offs[slice].rx_class_gates_base + RX_CLASS_GATES_SIZE * (n))
+
+#define RX_CLASS_GATES_ALLOW_MASK	BIT(6)
+#define RX_CLASS_GATES_RAW_MASK		BIT(5)
+#define RX_CLASS_GATES_PHASE_MASK	BIT(4)
+
+/* RX Class traffic data matching bits */
+#define RX_CLASS_FT_UC		BIT(31)
+#define RX_CLASS_FT_MC		BIT(30)
+#define RX_CLASS_FT_BC		BIT(29)
+#define RX_CLASS_FT_FW		BIT(28)
+#define RX_CLASS_FT_RCV		BIT(27)
+#define RX_CLASS_FT_VLAN	BIT(26)
+#define RX_CLASS_FT_DA_P	BIT(25)
+#define RX_CLASS_FT_DA_I	BIT(24)
+#define RX_CLASS_FT_FT1_MATCH_MASK	GENMASK(23, 16)
+#define RX_CLASS_FT_FT1_MATCH_SHIFT	16
+#define RX_CLASS_FT_FT3_MATCH_MASK	GENMASK(15, 0)
+#define RX_CLASS_FT_FT3_MATCH_SHIFT	0
+
+#define RX_CLASS_FT_FT1_MATCH(slot)	((BIT(slot) << RX_CLASS_FT_FT1_MATCH_SHIFT) & RX_CLASS_FT_FT1_MATCH_MASK)
+
+enum rx_class_sel_type {
+	RX_CLASS_SEL_TYPE_OR = 0,
+	RX_CLASS_SEL_TYPE_AND = 1,
+	RX_CLASS_SEL_TYPE_OR_AND_AND = 2,
+	RX_CLASS_SEL_TYPE_OR_OR_AND = 3,
+};
+
+#define FT1_CFG_SHIFT(n)	(2 * (n))
+#define FT1_CFG_MASK(n)		(0x3 << FT1_CFG_SHIFT((n)))
+
+#define RX_CLASS_SEL_SHIFT(n)	(2 * (n))
+#define RX_CLASS_SEL_MASK(n)	(0x3 << RX_CLASS_SEL_SHIFT((n)))
+
+#define ICSSG_CFG_OFFSET	0
+
+#define ICSSG_CFG_RX_L2_G_EN	BIT(2)
+
+/* these are register offsets per PRU */
+struct miig_rt_offsets {
+	u32 mac0;
+	u32 mac1;
+	u32 ft1_start_len;
+	u32 ft1_cfg;
+	u32 ft1_slot_base;
+	u32 ft3_slot_base;
+	u32 ft3_p_base;
+	u32 ft_rx_ptr;
+	u32 rx_class_base;
+	u32 rx_class_cfg1;
+	u32 rx_class_cfg2;
+	u32 rx_class_gates_base;
+	u32 rx_green;
+	u32 rx_rate_cfg_base;
+	u32 rx_rate_src_sel0;
+	u32 rx_rate_src_sel1;
+	u32 tx_rate_cfg_base;
+	u32 stat_base;
+	u32 tx_hsr_tag;
+	u32 tx_hsr_seq;
+	u32 tx_vlan_type;
+	u32 tx_vlan_ins;
+};
+
+static struct miig_rt_offsets offs[] = {
+	/* PRU0 */
+	{
+		0x8,
+		0xc,
+		0x80,
+		0x84,
+		0x88,
+		0x108,
+		0x308,
+		0x408,
+		0x40c,
+		0x48c,
+		0x490,
+		0x494,
+		0x4d4,
+		0x4e4,
+		0x504,
+		0x508,
+		0x50c,
+		0x54c,
+		0x63c,
+		0x640,
+		0x644,
+		0x648,
+	},
+	/* PRU1 */
+	{
+		0x10,
+		0x14,
+		0x64c,
+		0x650,
+		0x654,
+		0x6d4,
+		0x8d4,
+		0x9d4,
+		0x9d8,
+		0xa58,
+		0xa5c,
+		0xa60,
+		0xaa0,
+		0xab0,
+		0xad0,
+		0xad4,
+		0xad8,
+		0xb18,
+		0xc08,
+		0xc0c,
+		0xc10,
+		0xc14,
+	},
+};
+
+static inline u32 addr_to_da0(const u8 *addr)
+{
+	return (u32)(addr[0] | addr[1] << 8 |
+		addr[2] << 16 | addr[3] << 24);
+};
+
+static inline u32 addr_to_da1(const u8 *addr)
+{
+	return (u32)(addr[4] | addr[5] << 8);
+};
+
+static void rx_class_ft1_set_start_len(struct regmap *miig_rt, int slice,
+				       u16 start, u8 len)
+{
+	u32 offset, val;
+
+	offset = offs[slice].ft1_start_len;
+	val = FT1_LEN(len) | FT1_START(start);
+	regmap_write(miig_rt, offset, val);
+}
+
+static void rx_class_ft1_set_da(struct regmap *miig_rt, int slice,
+				int n, const u8 *addr)
+{
+	u32 offset;
+
+	offset = FT1_N_REG(slice, n, FT1_DA0);
+	regmap_write(miig_rt, offset, addr_to_da0(addr));
+	offset = FT1_N_REG(slice, n, FT1_DA1);
+	regmap_write(miig_rt, offset, addr_to_da1(addr));
+}
+
+static void rx_class_ft1_set_da_mask(struct regmap *miig_rt, int slice,
+				     int n, const u8 *addr)
+{
+	u32 offset;
+
+	offset = FT1_N_REG(slice, n, FT1_DA0_MASK);
+	regmap_write(miig_rt, offset, addr_to_da0(addr));
+	offset = FT1_N_REG(slice, n, FT1_DA1_MASK);
+	regmap_write(miig_rt, offset, addr_to_da1(addr));
+}
+
+static void rx_class_ft1_cfg_set_type(struct regmap *miig_rt, int slice, int n,
+				      enum ft1_cfg_type type)
+{
+	u32 offset;
+
+	offset = offs[slice].ft1_cfg;
+	regmap_update_bits(miig_rt, offset, FT1_CFG_MASK(n),
+			   type << FT1_CFG_SHIFT(n));
+}
+
+static void rx_class_sel_set_type(struct regmap *miig_rt, int slice, int n,
+				  enum rx_class_sel_type type)
+{
+	u32 offset;
+
+	offset = offs[slice].rx_class_cfg1;
+	regmap_update_bits(miig_rt, offset, RX_CLASS_SEL_MASK(n),
+			   type << RX_CLASS_SEL_SHIFT(n));
+}
+
+static void rx_class_set_and(struct regmap *miig_rt, int slice, int n,
+			     u32 data)
+{
+	u32 offset;
+
+	offset = RX_CLASS_N_REG(slice, n, RX_CLASS_AND_EN);
+	regmap_write(miig_rt, offset, data);
+}
+
+static void rx_class_set_or(struct regmap *miig_rt, int slice, int n,
+			    u32 data)
+{
+	u32 offset;
+
+	offset = RX_CLASS_N_REG(slice, n, RX_CLASS_OR_EN);
+	regmap_write(miig_rt, offset, data);
+}
+
+static u32 rx_class_get_or(struct regmap *miig_rt, int slice, int n)
+{
+	u32 offset, val;
+
+	offset = RX_CLASS_N_REG(slice, n, RX_CLASS_OR_EN);
+	regmap_read(miig_rt, offset, &val);
+
+	return val;
+}
+
+void icssg_class_set_mac_addr(struct regmap *miig_rt, int slice, u8 *mac)
+{
+	regmap_write(miig_rt, offs[slice].mac0, addr_to_da0(mac));
+	regmap_write(miig_rt, offs[slice].mac1, addr_to_da1(mac));
+}
+
+static void icssg_class_ft1_add_mcast(struct regmap *miig_rt, int slice,
+				      int slot, const u8 *addr, const u8 *mask)
+{
+	int i;
+	u32 val;
+
+	WARN(slot >= FT1_NUM_SLOTS, "invalid slot: %d\n", slot);
+
+	rx_class_ft1_set_da(miig_rt, slice, slot, addr);
+	rx_class_ft1_set_da_mask(miig_rt, slice, slot, mask);
+	rx_class_ft1_cfg_set_type(miig_rt, slice, slot, FT1_CFG_TYPE_EQ);
+
+	/* Enable the FT1 slot in OR enable for all classifiers */
+	for (i = 0; i < ICSSG_NUM_CLASSIFIERS_IN_USE; i++) {
+		val = rx_class_get_or(miig_rt, slice, i);
+		val |= RX_CLASS_FT_FT1_MATCH(slot);
+		rx_class_set_or(miig_rt, slice, i, val);
+	}
+}
+
+/* disable all RX traffic */
+void icssg_class_disable(struct regmap *miig_rt, int slice)
+{
+	u32 data, offset;
+	int n;
+
+	/* Enable RX_L2_G */
+	regmap_update_bits(miig_rt, ICSSG_CFG_OFFSET, ICSSG_CFG_RX_L2_G_EN,
+			   ICSSG_CFG_RX_L2_G_EN);
+
+	for (n = 0; n < ICSSG_NUM_CLASSIFIERS; n++) {
+		/* AND_EN = 0 */
+		rx_class_set_and(miig_rt, slice, n, 0);
+		/* OR_EN = 0 */
+		rx_class_set_or(miig_rt, slice, n, 0);
+
+		/* set CFG1 to OR */
+		rx_class_sel_set_type(miig_rt, slice, n, RX_CLASS_SEL_TYPE_OR);
+
+		/* configure gate */
+		offset = RX_CLASS_GATES_N_REG(slice, n);
+		regmap_read(miig_rt, offset, &data);
+		/* clear class_raw so we go through filters */
+		data &= ~RX_CLASS_GATES_RAW_MASK;
+		/* set allow and phase mask */
+		data |= RX_CLASS_GATES_ALLOW_MASK | RX_CLASS_GATES_PHASE_MASK;
+		regmap_write(miig_rt, offset, data);
+	}
+
+	/* FT1 Disabled */
+	for (n = 0; n < ICSSG_NUM_FT1_SLOTS; n++) {
+		u8 addr[] = { 0, 0, 0, 0, 0, 0, };
+
+		rx_class_ft1_cfg_set_type(miig_rt, slice, n,
+					  FT1_CFG_TYPE_DISABLED);
+		rx_class_ft1_set_da(miig_rt, slice, n, addr);
+		rx_class_ft1_set_da_mask(miig_rt, slice, n, addr);
+	}
+
+	/* clear CFG2 */
+	regmap_write(miig_rt, offs[slice].rx_class_cfg2, 0);
+}
+
+void icssg_class_default(struct regmap *miig_rt, int slice, bool allmulti)
+{
+	u32 data;
+	int n;
+
+	/* defaults */
+	icssg_class_disable(miig_rt, slice);
+
+	/* Setup Classifier */
+	for (n = 0; n < ICSSG_NUM_CLASSIFIERS_IN_USE; n++) {
+		/* match on Broadcast or MAC_PRU address */
+		data = RX_CLASS_FT_BC | RX_CLASS_FT_DA_P;
+
+		/* multicast? */
+		if (allmulti)
+			data |= RX_CLASS_FT_MC;
+
+		rx_class_set_or(miig_rt, slice, n, data);
+
+		/* set CFG1 for OR_OR_AND for classifier */
+		rx_class_sel_set_type(miig_rt, slice, n,
+				      RX_CLASS_SEL_TYPE_OR_OR_AND);
+	}
+
+	/* clear CFG2 */
+	regmap_write(miig_rt, offs[slice].rx_class_cfg2, 0);
+}
+
+void icssg_class_promiscuous(struct regmap *miig_rt, int slice)
+{
+	u32 data;
+	u32 offset;
+	int n;
+
+	/* defaults */
+	icssg_class_disable(miig_rt, slice);
+
+	/* Setup Classifier */
+	for (n = 0; n < ICSSG_NUM_CLASSIFIERS_IN_USE; n++) {
+		/* set RAW_MASK to bypass filters */
+		offset = RX_CLASS_GATES_N_REG(slice, n);
+		regmap_read(miig_rt, offset, &data);
+		data |= RX_CLASS_GATES_RAW_MASK;
+		regmap_write(miig_rt, offset, data);
+	}
+}
+
+void icssg_class_add_mcast(struct regmap *miig_rt, int slice,
+			   struct net_device *ndev)
+{
+	int slot;
+	struct netdev_hw_addr *ha;
+	u8 sr_addr[] = { 0x01, 0x80, 0xC2, 0, 0, 0, };
+	u8 cb_addr[] = { 0x01, 0x00, 0x5e, 0, 0, 0, };
+	u8 mask_addr[] = { 0, 0, 0, 0, 0, 0, };
+
+	rx_class_ft1_set_start_len(miig_rt, slice, 0, 6);
+	/* reserve first 2 slots for
+	 *	1) 01-80-C2-00-00-XX Known Service Ethernet Multicast addresses
+	 *	2) 01-00-5e-00-00-XX Local Network Control Block
+	 *			      (224.0.0.0 - 224.0.0.255  (224.0.0/24))
+	 */
+	mask_addr[5] = 0xff;
+	icssg_class_ft1_add_mcast(miig_rt, slice, 0, sr_addr, mask_addr);
+	icssg_class_ft1_add_mcast(miig_rt, slice, 1, cb_addr, mask_addr);
+	mask_addr[5] = 0;
+	slot = 2;
+	netdev_for_each_mc_addr(ha, ndev) {
+		/* skip addresses matching reserved slots */
+		if (!memcmp(sr_addr, ha->addr, 5) ||
+		    !memcmp(cb_addr, ha->addr, 5)) {
+			netdev_dbg(ndev, "mcast skip %pM\n", ha->addr);
+			continue;
+		}
+
+		if (slot >= FT1_NUM_SLOTS) {
+			netdev_dbg(ndev,
+				   "can't add more than %d MC addresses, enabling allmulti\n",
+				   FT1_NUM_SLOTS);
+			icssg_class_default(miig_rt, slice, 1);
+			break;
+		}
+
+		netdev_dbg(ndev, "mcast add %pM\n", ha->addr);
+		icssg_class_ft1_add_mcast(miig_rt, slice, slot,
+					  ha->addr, mask_addr);
+		slot++;
+	}
+}
diff -Naur --no-dereference a/drivers/net/ethernet/ti/icssg_config.h b/drivers/net/ethernet/ti/icssg_config.h
--- a/drivers/net/ethernet/ti/icssg_config.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/net/ethernet/ti/icssg_config.h	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,53 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Texas Instruments ICSSG Ethernet driver
+ *
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ */
+
+#ifndef __NET_TI_ICSSG_CONFIG_H
+#define __NET_TI_ICSSG_CONFIG_H
+
+/* Config area lies in shared RAM */
+#define ICSSG_CONFIG_OFFSET_SLICE0   0
+#define ICSSG_CONFIG_OFFSET_SLICE1   0x8000
+
+/* Load time Fiwmware Configuration */
+struct icssg_config {
+	__le32 status;	/* Firmware status */
+	__le32 addr_lo;	/* MSMC Buffer pool base address low. */
+	__le32 addr_hi;	/* MSMC Buffer pool base address high. Must be 0 */
+	__le32 tx_buf_sz[16];	/* Array of buffer pool sizes */
+	__le32 num_tx_threads;	/* Number of active egress threads, 1 to 4 */
+	__le32 tx_rate_lim_en;	/* Bitmask: Egress rate limit en per thread */
+	__le32 rx_flow_id;	/* RX flow id for first rx ring */
+	__le32 rx_mgr_flow_id;	/* RX flow id for the first management ring */
+	__le32 flags;		/* TBD */
+	__le32 n_burst;		/* for debug */
+	__le32 rtu_status;	/* RTU status */
+	__le32 info;		/* reserved */
+} __packed;
+
+/* Shutdown command to stop processing at firmware.
+ * Command format : 0x8101ss00. ss - sequence number. Currently not used
+ * by driver.
+ */
+#define ICSSG_SHUTDOWN_CMD		0x81010000
+
+/* pstate speed/duplex command to set speed and duplex settings
+ * in firmware.
+ * Command format : 0x8102ssPN. ss - sequence number: currently not
+ * used by driver, P - port number: For switch, N - Speed/Duplex state
+ * - Possible values of N:
+ * 0x0 - 10Mbps/Half duplex ;
+ * 0x8 - 10Mbps/Full duplex ;
+ * 0x2 - 100Mbps/Half duplex;
+ * 0xa - 100Mbps/Full duplex;
+ * 0xc - 1Gbps/Full duplex;
+ * NOTE: The above are same as bits [3..1](slice 0) or bits [8..6](slice 1) of
+ * RGMII CFG register. So suggested to read the register to populate the command
+ * bits.
+ */
+#define ICSSG_PSTATE_SPEED_DUPLEX_CMD	0x81020000
+
+#endif /* __NET_TI_ICSSG_CONFIG_H */
diff -Naur --no-dereference a/drivers/net/ethernet/ti/icssg_ethtool.c b/drivers/net/ethernet/ti/icssg_ethtool.c
--- a/drivers/net/ethernet/ti/icssg_ethtool.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/net/ethernet/ti/icssg_ethtool.c	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,316 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Texas Instruments ICSSG Ethernet driver
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ */
+
+#include "icssg_prueth.h"
+#include <linux/regmap.h>
+
+static u32 stats_base[] = {	0x54c,	/* Slice 0 stats start */
+				0xb18,	/* Slice 1 stats start */
+};
+
+struct miig_stats_regs {
+	/* Rx */
+	u32 rx_good_frames;
+	u32 rx_broadcast_frames;
+	u32 rx_multicast_frames;
+	u32 rx_crc_error_frames;
+	u32 rx_mii_error_frames;
+	u32 rx_odd_nibble_frames;
+	u32 rx_frame_max_size;
+	u32 rx_max_size_error_frames;
+	u32 rx_frame_min_size;
+	u32 rx_min_size_error_frames;
+	u32 rx_overrun_frames;
+	u32 rx_class0_hits;
+	u32 rx_class1_hits;
+	u32 rx_class2_hits;
+	u32 rx_class3_hits;
+	u32 rx_class4_hits;
+	u32 rx_class5_hits;
+	u32 rx_class6_hits;
+	u32 rx_class7_hits;
+	u32 rx_class8_hits;
+	u32 rx_class9_hits;
+	u32 rx_class10_hits;
+	u32 rx_class11_hits;
+	u32 rx_class12_hits;
+	u32 rx_class13_hits;
+	u32 rx_class14_hits;
+	u32 rx_class15_hits;
+	u32 rx_smd_frags;
+	u32 rx_bucket1_size;
+	u32 rx_bucket2_size;
+	u32 rx_bucket3_size;
+	u32 rx_bucket4_size;
+	u32 rx_64B_frames;
+	u32 rx_bucket1_frames;
+	u32 rx_bucket2_frames;
+	u32 rx_bucket3_frames;
+	u32 rx_bucket4_frames;
+	u32 rx_bucket5_frames;
+	u32 rx_total_bytes;
+	u32 rx_tx_total_bytes;
+	/* Tx */
+	u32 tx_good_frames;
+	u32 tx_broadcast_frames;
+	u32 tx_multicast_frames;
+	u32 tx_odd_nibble_frames;
+	u32 tx_underflow_errors;
+	u32 tx_frame_max_size;
+	u32 tx_max_size_error_frames;
+	u32 tx_frame_min_size;
+	u32 tx_min_size_error_frames;
+	u32 tx_bucket1_size;
+	u32 tx_bucket2_size;
+	u32 tx_bucket3_size;
+	u32 tx_bucket4_size;
+	u32 tx_64B_frames;
+	u32 tx_bucket1_frames;
+	u32 tx_bucket2_frames;
+	u32 tx_bucket3_frames;
+	u32 tx_bucket4_frames;
+	u32 tx_bucket5_frames;
+	u32 tx_total_bytes;
+};
+
+#define ICSSG_STATS(field)				\
+{							\
+	#field,						\
+	offsetof(struct miig_stats_regs, field),	\
+}
+
+struct icssg_stats {
+	char name[ETH_GSTRING_LEN];
+	u32 offset;
+};
+
+static const struct icssg_stats icssg_ethtool_stats[] = {
+	/* Rx */
+	ICSSG_STATS(rx_good_frames),
+	ICSSG_STATS(rx_broadcast_frames),
+	ICSSG_STATS(rx_multicast_frames),
+	ICSSG_STATS(rx_crc_error_frames),
+	ICSSG_STATS(rx_mii_error_frames),
+	ICSSG_STATS(rx_odd_nibble_frames),
+	ICSSG_STATS(rx_frame_max_size),
+	ICSSG_STATS(rx_max_size_error_frames),
+	ICSSG_STATS(rx_frame_min_size),
+	ICSSG_STATS(rx_min_size_error_frames),
+	ICSSG_STATS(rx_overrun_frames),
+	ICSSG_STATS(rx_class0_hits),
+	ICSSG_STATS(rx_class1_hits),
+	ICSSG_STATS(rx_class2_hits),
+	ICSSG_STATS(rx_class3_hits),
+	ICSSG_STATS(rx_class4_hits),
+	ICSSG_STATS(rx_class5_hits),
+	ICSSG_STATS(rx_class6_hits),
+	ICSSG_STATS(rx_class7_hits),
+	ICSSG_STATS(rx_class8_hits),
+	ICSSG_STATS(rx_class9_hits),
+	ICSSG_STATS(rx_class10_hits),
+	ICSSG_STATS(rx_class11_hits),
+	ICSSG_STATS(rx_class12_hits),
+	ICSSG_STATS(rx_class13_hits),
+	ICSSG_STATS(rx_class14_hits),
+	ICSSG_STATS(rx_class15_hits),
+	ICSSG_STATS(rx_smd_frags),
+	ICSSG_STATS(rx_bucket1_size),
+	ICSSG_STATS(rx_bucket2_size),
+	ICSSG_STATS(rx_bucket3_size),
+	ICSSG_STATS(rx_bucket4_size),
+	ICSSG_STATS(rx_64B_frames),
+	ICSSG_STATS(rx_bucket1_frames),
+	ICSSG_STATS(rx_bucket2_frames),
+	ICSSG_STATS(rx_bucket3_frames),
+	ICSSG_STATS(rx_bucket4_frames),
+	ICSSG_STATS(rx_bucket5_frames),
+	ICSSG_STATS(rx_total_bytes),
+	ICSSG_STATS(rx_tx_total_bytes),
+	/* Tx */
+	ICSSG_STATS(tx_good_frames),
+	ICSSG_STATS(tx_broadcast_frames),
+	ICSSG_STATS(tx_multicast_frames),
+	ICSSG_STATS(tx_odd_nibble_frames),
+	ICSSG_STATS(tx_underflow_errors),
+	ICSSG_STATS(tx_frame_max_size),
+	ICSSG_STATS(tx_max_size_error_frames),
+	ICSSG_STATS(tx_frame_min_size),
+	ICSSG_STATS(tx_min_size_error_frames),
+	ICSSG_STATS(tx_bucket1_size),
+	ICSSG_STATS(tx_bucket2_size),
+	ICSSG_STATS(tx_bucket3_size),
+	ICSSG_STATS(tx_bucket4_size),
+	ICSSG_STATS(tx_64B_frames),
+	ICSSG_STATS(tx_bucket1_frames),
+	ICSSG_STATS(tx_bucket2_frames),
+	ICSSG_STATS(tx_bucket3_frames),
+	ICSSG_STATS(tx_bucket4_frames),
+	ICSSG_STATS(tx_bucket5_frames),
+	ICSSG_STATS(tx_total_bytes),
+};
+
+static void emac_get_drvinfo(struct net_device *ndev,
+			     struct ethtool_drvinfo *info)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+	struct prueth *prueth = emac->prueth;
+
+	strlcpy(info->driver, dev_driver_string(prueth->dev),
+		sizeof(info->driver));
+	/* TODO: info->fw_version */
+	strlcpy(info->bus_info, dev_name(prueth->dev), sizeof(info->bus_info));
+}
+
+static u32 emac_get_msglevel(struct net_device *ndev)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+
+	return emac->msg_enable;
+}
+
+static void emac_set_msglevel(struct net_device *ndev, u32 value)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+
+	emac->msg_enable = value;
+}
+
+static int emac_get_link_ksettings(struct net_device *ndev,
+				   struct ethtool_link_ksettings *ecmd)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+
+	if (!emac->phydev)
+		return -EOPNOTSUPP;
+
+	phy_ethtool_ksettings_get(emac->phydev, ecmd);
+	return 0;
+}
+
+static int emac_set_link_ksettings(struct net_device *ndev,
+				   const struct ethtool_link_ksettings *ecmd)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+
+	if (!emac->phydev || phy_is_pseudo_fixed_link(emac->phydev))
+		return -EOPNOTSUPP;
+
+	return phy_ethtool_ksettings_set(emac->phydev, ecmd);
+}
+
+static int emac_get_eee(struct net_device *ndev, struct ethtool_eee *edata)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+
+	if (!emac->phydev || phy_is_pseudo_fixed_link(emac->phydev))
+		return -EOPNOTSUPP;
+
+	return phy_ethtool_get_eee(emac->phydev, edata);
+}
+
+static int emac_set_eee(struct net_device *ndev, struct ethtool_eee *edata)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+
+	if (!emac->phydev || phy_is_pseudo_fixed_link(emac->phydev))
+		return -EOPNOTSUPP;
+
+	return phy_ethtool_set_eee(emac->phydev, edata);
+}
+
+static int emac_nway_reset(struct net_device *ndev)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+
+	if (!emac->phydev || phy_is_pseudo_fixed_link(emac->phydev))
+		return -EOPNOTSUPP;
+
+	return genphy_restart_aneg(emac->phydev);
+}
+
+static int emac_get_sset_count(struct net_device *ndev, int stringset)
+{
+	switch (stringset) {
+	case ETH_SS_STATS:
+		return ARRAY_SIZE(icssg_ethtool_stats);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static void emac_get_strings(struct net_device *ndev, u32 stringset, u8 *data)
+{
+	u8 *p = data;
+	int i;
+
+	switch (stringset) {
+	case ETH_SS_STATS:
+		for (i = 0; i < ARRAY_SIZE(icssg_ethtool_stats); i++) {
+			memcpy(p, icssg_ethtool_stats[i].name,
+			       ETH_GSTRING_LEN);
+			p += ETH_GSTRING_LEN;
+		}
+		break;
+	default:
+		break;
+	}
+}
+
+static void emac_get_ethtool_stats(struct net_device *ndev,
+				   struct ethtool_stats *stats, u64 *data)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+	struct prueth *prueth = emac->prueth;
+	int i;
+	int slice = prueth_emac_slice(emac);
+	u32 base = stats_base[slice];
+	u32 val;
+
+	for (i = 0; i < ARRAY_SIZE(icssg_ethtool_stats); i++) {
+		regmap_read(prueth->miig_rt,
+			    base + icssg_ethtool_stats[i].offset,
+			    &val);
+		data[i] = val;
+	}
+}
+
+static int emac_get_ts_info(struct net_device *ndev,
+			    struct ethtool_ts_info *info)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+
+	info->so_timestamping =
+		SOF_TIMESTAMPING_TX_HARDWARE |
+		SOF_TIMESTAMPING_TX_SOFTWARE |
+		SOF_TIMESTAMPING_RX_HARDWARE |
+		SOF_TIMESTAMPING_RX_SOFTWARE |
+		SOF_TIMESTAMPING_SOFTWARE |
+		SOF_TIMESTAMPING_RAW_HARDWARE;
+
+	info->phc_index = ptp_clock_index(emac->iep.ptp_clock);
+	info->tx_types = BIT(HWTSTAMP_TX_OFF) | BIT(HWTSTAMP_TX_ON);
+	info->rx_filters = BIT(HWTSTAMP_FILTER_NONE) | BIT(HWTSTAMP_FILTER_ALL);
+
+	return 0;
+}
+
+const struct ethtool_ops icssg_ethtool_ops = {
+	.get_drvinfo = emac_get_drvinfo,
+	.get_msglevel = emac_get_msglevel,
+	.set_msglevel = emac_set_msglevel,
+	.get_sset_count = emac_get_sset_count,
+	.get_strings = emac_get_strings,
+	.get_ethtool_stats = emac_get_ethtool_stats,
+	.get_ts_info = emac_get_ts_info,
+
+	.get_link_ksettings = emac_get_link_ksettings,
+	.set_link_ksettings = emac_set_link_ksettings,
+	.get_link = ethtool_op_get_link,
+	.get_eee = emac_get_eee,
+	.set_eee = emac_set_eee,
+	.nway_reset = emac_nway_reset,
+};
diff -Naur --no-dereference a/drivers/net/ethernet/ti/icssg_iep.c b/drivers/net/ethernet/ti/icssg_iep.c
--- a/drivers/net/ethernet/ti/icssg_iep.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/net/ethernet/ti/icssg_iep.c	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,273 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Texas Instruments ICSSG Industrial Ethernet Peripheral (IEP) Driver
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
+ *
+ */
+
+#include <linux/bitops.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/timekeeping.h>
+
+#include "icssg_iep.h"
+
+#define IEP_MAX_DEF_INC		0xf
+#define IEP_MAX_COMPEN_INC		0xfff
+#define IEP_MAX_COMPEN_COUNT	0xffffff
+
+#define IEP_GLOBAL_CFG_REG	0x0
+#define IEP_GLOBAL_STATUS_REG	0x4
+#define IEP_COMPEN_REG		0x8
+#define IEP_SLOW_COMPEN_REG	0xc
+#define IEP_COUNT_LOW_REG	0x10
+#define IEP_COUNT_HIGH_REG	0x14
+#define IEP_CAP_CFG_REG		0x18
+#define IEP_CAP_STATUS_REG	0x1c
+
+#define IEP_GLOBAL_CFG_CNT_ENABLE	BIT(0)
+#define IEP_GLOBAL_CFG_DEFAULT_INC_MASK		GENMASK(7, 4)
+#define IEP_GLOBAL_CFG_DEFAULT_INC_SHIFT	4
+#define IEP_GLOBAL_CFG_COMPEN_INC_MASK		GENMASK(19, 8)
+#define IEP_GLOBAL_CFG_COMPEN_INC_SHIFT		8
+
+static void iep_settime(struct icssg_iep *iep, u64 tstamp)
+{
+	u32 val;
+
+	val = upper_32_bits(tstamp);
+	regmap_write(iep->map, IEP_COUNT_HIGH_REG, val);
+	val = lower_32_bits(tstamp);
+	regmap_write(iep->map, IEP_COUNT_LOW_REG, val);
+}
+
+static u64 iep_gettime(struct icssg_iep *iep)
+{
+	u64 val;
+	u32 tmp;
+
+	regmap_read(iep->map, IEP_COUNT_LOW_REG, &tmp);
+	val = tmp;
+	regmap_read(iep->map, IEP_COUNT_HIGH_REG, &tmp);
+	val |= (u64)tmp << 32;
+
+	return val;
+}
+
+static void iep_enable(struct icssg_iep *iep)
+{
+	regmap_update_bits(iep->map, IEP_GLOBAL_CFG_REG,
+			   IEP_GLOBAL_CFG_CNT_ENABLE,
+			   IEP_GLOBAL_CFG_CNT_ENABLE);
+}
+
+static void iep_disable(struct icssg_iep *iep)
+{
+	regmap_update_bits(iep->map, IEP_GLOBAL_CFG_REG,
+			   IEP_GLOBAL_CFG_CNT_ENABLE,
+			   0);
+}
+
+static void iep_set_default_inc(struct icssg_iep *iep, u8 def_inc)
+{
+	regmap_update_bits(iep->map, IEP_GLOBAL_CFG_REG,
+			   IEP_GLOBAL_CFG_DEFAULT_INC_MASK,
+			   def_inc << IEP_GLOBAL_CFG_DEFAULT_INC_SHIFT);
+}
+
+static void iep_set_compensation_inc(struct icssg_iep *iep, u16 compen_inc)
+{
+	struct device *dev = regmap_get_device(iep->map);
+
+	if (compen_inc > IEP_MAX_COMPEN_INC) {
+		dev_err(dev, "%s: too high compensation inc %d\n",
+			__func__, compen_inc);
+		compen_inc = IEP_MAX_COMPEN_INC;
+	}
+
+	regmap_update_bits(iep->map, IEP_GLOBAL_CFG_REG,
+			   IEP_GLOBAL_CFG_COMPEN_INC_MASK,
+			   compen_inc << IEP_GLOBAL_CFG_COMPEN_INC_SHIFT);
+}
+
+static void iep_set_compensation_count(struct icssg_iep *iep, u32 compen_count)
+{
+	struct device *dev = regmap_get_device(iep->map);
+
+	if (compen_count > IEP_MAX_COMPEN_COUNT) {
+		dev_err(dev, "%s: too high compensation count %d\n",
+			__func__, compen_count);
+		compen_count = IEP_MAX_COMPEN_COUNT;
+	}
+
+	regmap_write(iep->map, IEP_COMPEN_REG, compen_count);
+}
+
+static void iep_set_slow_compensation_count(struct icssg_iep *iep,
+					    u32 compen_count)
+{
+	regmap_write(iep->map, IEP_SLOW_COMPEN_REG, compen_count);
+}
+
+/* PTP PHC operations */
+static int iep_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb)
+{
+	struct icssg_iep *iep = container_of(ptp, struct icssg_iep, ptp_info);
+	u32 cyc_count;
+	u16 cmp_inc;
+
+	mutex_lock(&iep->ptp_clk_mutex);
+
+	/* ppb is amount of frequency we want to adjust in 1GHz (billion)
+	 * e.g. 100ppb means we need to speed up clock by 100Hz
+	 * i.e. at end of 1 second (1 billion ns) clock time, we should be
+	 * counting 100 more ns.
+	 * We use IEP slow compensation to achieve continuous freq. adjustment.
+	 * There are 2 parts. Cycle time and adjustment per cycle.
+	 * Simplest case would be 1 sec Cycle time. Then adjustment
+	 * pre cycle would be (def_inc + ppb) value.
+	 * Cycle time will have to be chosen based on how worse the ppb is.
+	 * e.g. smaller the ppb, cycle time has to be large.
+	 * The minimum adjustment we can do is +-1ns per cycle so let's
+	 * reduce the cycle time to get 1ns per cycle adjustment.
+	 *	1ppb = 1sec cycle time & 1ns adjust
+	 *	1000ppb = 1/1000 cycle time & 1ns adjust per cycle
+	 */
+
+	iep->slow_cmp_inc = 1;	/* 1ns adjust per cycle */
+	if (ppb < 0) {
+		iep->slow_cmp_inc = -iep->slow_cmp_inc;
+		ppb = -ppb;
+	}
+
+	cyc_count = NSEC_PER_SEC;		/* 1s cycle time @1GHz */
+	cyc_count /= ppb;		/* cycle time per ppb */
+
+	/* slow_cmp_count is decremented every clock cycle, e.g. @250MHz */
+	cyc_count /= iep->clk_tick_time;
+	iep->slow_cmp_count = cyc_count;
+
+	/* iep->clk_tick_time is def_inc */
+	cmp_inc = iep->clk_tick_time + iep->slow_cmp_inc;
+	iep_set_compensation_inc(iep, cmp_inc);
+	iep_set_slow_compensation_count(iep, iep->slow_cmp_count);
+	iep->slow_cmp_active = 1;
+
+	mutex_unlock(&iep->ptp_clk_mutex);
+
+	return 0;
+}
+
+static int iep_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
+{
+	struct icssg_iep *iep = container_of(ptp, struct icssg_iep, ptp_info);
+	s64 ns;
+
+	mutex_lock(&iep->ptp_clk_mutex);
+	ns = iep_gettime(iep);
+	ns += delta;
+	iep_settime(iep, ns);
+	mutex_unlock(&iep->ptp_clk_mutex);
+
+	return 0;
+}
+
+static int iep_ptp_gettime(struct ptp_clock_info *ptp,
+			   struct timespec64 *ts)
+{
+	struct icssg_iep *iep = container_of(ptp, struct icssg_iep, ptp_info);
+	u64 ns;
+
+	mutex_lock(&iep->ptp_clk_mutex);
+	ns = iep_gettime(iep);
+	*ts = ns_to_timespec64(ns);
+	mutex_unlock(&iep->ptp_clk_mutex);
+
+	return 0;
+}
+
+static int iep_ptp_settime(struct ptp_clock_info *ptp,
+			   const struct timespec64 *ts)
+{
+	struct icssg_iep *iep = container_of(ptp, struct icssg_iep, ptp_info);
+	u64 ns;
+
+	mutex_lock(&iep->ptp_clk_mutex);
+	ns = timespec64_to_ns(ts);
+	iep_settime(iep, ns);
+	mutex_unlock(&iep->ptp_clk_mutex);
+
+	return 0;
+}
+
+static int iep_ptp_enable(struct ptp_clock_info *ptp,
+			  struct ptp_clock_request *rq, int on)
+{
+	return -EOPNOTSUPP;
+}
+
+static struct ptp_clock_info iep_ptp_info = {
+	.owner		= THIS_MODULE,
+	.name		= "ICSS IEP timer",
+	.max_adj	= 10000000,
+	.adjfreq	= iep_ptp_adjfreq,
+	.adjtime	= iep_ptp_adjtime,
+	.gettime64	= iep_ptp_gettime,
+	.settime64	= iep_ptp_settime,
+	.enable		= iep_ptp_enable,
+};
+
+int icssg_iep_init(struct icssg_iep *iep, struct device *parent_dev,
+		   struct regmap *iep_map, u32 refclk_freq)
+{
+	int ret;
+	u32 def_inc;
+	u64 tstamp;
+
+	iep->map = iep_map;
+	iep->refclk_freq = refclk_freq;
+	mutex_init(&iep->ptp_clk_mutex);
+
+	def_inc = NSEC_PER_SEC / iep->refclk_freq;	/* ns per clock tick */
+	if (def_inc > IEP_MAX_DEF_INC)
+		/* iep_core_clk too slow to be supported */
+		return -EINVAL;
+
+	iep_set_default_inc(iep, def_inc);
+	iep_set_compensation_inc(iep, def_inc);
+	iep_set_compensation_count(iep, 0);
+	iep_set_slow_compensation_count(iep, 0);
+	iep_enable(iep);
+
+	tstamp = ktime_to_ns(ktime_get_real());
+	iep_settime(iep, tstamp);
+
+	iep->clk_tick_time = def_inc;
+	iep->ptp_info = iep_ptp_info;
+	iep->ptp_clock = ptp_clock_register(&iep->ptp_info, parent_dev);
+	if (IS_ERR(iep->ptp_clock)) {
+		ret = PTR_ERR(iep->ptp_clock);
+		iep->ptp_clock = NULL;
+		dev_err(parent_dev, "Failed to register ptp clk %d\n", ret);
+		goto err_disable;
+	}
+	iep->ptp_index = ptp_clock_index(iep->ptp_clock);
+
+	return 0;
+
+err_disable:
+	iep_disable(iep);
+
+	return ret;
+}
+
+int icssg_iep_exit(struct icssg_iep *iep)
+{
+	ptp_clock_unregister(iep->ptp_clock);
+	iep_disable(iep);
+
+	return 0;
+}
diff -Naur --no-dereference a/drivers/net/ethernet/ti/icssg_iep.h b/drivers/net/ethernet/ti/icssg_iep.h
--- a/drivers/net/ethernet/ti/icssg_iep.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/net/ethernet/ti/icssg_iep.h	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Texas Instruments ICSSG Industrial Ethernet Peripheral (IEP) Driver
+ *
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ */
+
+#ifndef __NET_TI_ICSSG_IEP_H
+#define __NET_TI_ICSSG_IEP_H
+
+#include <linux/mutex.h>
+#include <linux/ptp_clock_kernel.h>
+#include <linux/regmap.h>
+
+struct icssg_iep {
+	struct regmap *map;
+	u32 refclk_freq;
+	int clk_tick_time;	/* one refclk tick time in ns */
+	struct ptp_clock_info ptp_info;
+	struct ptp_clock *ptp_clock;
+	int ptp_index;
+	struct mutex ptp_clk_mutex;	/* PHC access serializer */
+	unsigned int slow_cmp_active;
+	s16 slow_cmp_inc;
+	u32 slow_cmp_count;
+};
+
+int icssg_iep_init(struct icssg_iep *iep, struct device *parent_dev,
+		   struct regmap *iep_map, u32 refclk_freq);
+int icssg_iep_exit(struct icssg_iep *iep);
+
+#endif /* __NET_TI_ICSSG_IEP_H */
diff -Naur --no-dereference a/drivers/net/ethernet/ti/icssg_prueth.c b/drivers/net/ethernet/ti/icssg_prueth.c
--- a/drivers/net/ethernet/ti/icssg_prueth.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/net/ethernet/ti/icssg_prueth.c	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,2149 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Texas Instruments ICSSG Ethernet Driver
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ */
+
+#include <linux/bitops.h>
+#include <linux/clk.h>
+#include <linux/etherdevice.h>
+#include <linux/dma-mapping.h>
+#include <linux/genalloc.h>
+#include <linux/if_vlan.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_irq.h>
+#include <linux/of_mdio.h>
+#include <linux/of_net.h>
+#include <linux/of_platform.h>
+#include <linux/phy.h>
+#include <linux/pruss.h>
+#include <linux/regmap.h>
+#include <linux/remoteproc.h>
+#include <linux/dma/ti-cppi5.h>
+#include <linux/soc/ti/k3-navss-desc-pool.h>
+
+#include "icssg_prueth.h"
+#include "icss_mii_rt.h"
+
+#define PRUETH_MODULE_VERSION "0.1"
+#define PRUETH_MODULE_DESCRIPTION "PRUSS ICSSG Ethernet driver"
+
+/* Port queue size in MSMC from firmware
+ * PORTQSZ_HP .set (0x1800)
+ * PORTQSZ_HP2 .set (PORTQSZ_HP+128) ;include barrier area
+ * 0x1880 x 8 bytes per slice  (port)
+ */
+
+#define MSMC_RAM_SIZE	(SZ_64K + SZ_32K + SZ_2K)	/* 0x1880 x 8 x 2 */
+
+#define PRUETH_PKT_TYPE_CMD	0x10
+#define PRUETH_NAV_PS_DATA_SIZE	16	/* Protocol specific data size */
+#define PRUETH_NAV_SW_DATA_SIZE	16	/* SW related data size */
+#define PRUETH_MAX_TX_DESC	512
+#define PRUETH_MAX_RX_DESC	512
+#define PRUETH_MAX_RX_MGM_DESC	8
+#define PRUETH_MAX_RX_FLOWS	4	/* excluding default flow */
+#define PRUETH_MAX_RX_MGM_FLOWS	3	/* excluding default flow */
+#define PRUETH_RX_MGM_FLOW_RESPONSE	0
+#define PRUETH_RX_MGM_FLOW_TIMESTAMP	1
+#define PRUETH_RX_MGM_FLOW_OTHER	2
+
+#define PRUETH_NUM_BUF_POOLS		16
+#define PRUETH_EMAC_BUF_POOL_START	8
+#define PRUETH_EMAC_BUF_POOL_SIZE	0x1800
+
+#define PRUETH_MIN_PKT_SIZE	(VLAN_ETH_ZLEN)
+#define PRUETH_MAX_PKT_SIZE	(VLAN_ETH_FRAME_LEN + ETH_FCS_LEN)
+
+/* Netif debug messages possible */
+#define PRUETH_EMAC_DEBUG	(NETIF_MSG_DRV | \
+				 NETIF_MSG_PROBE | \
+				 NETIF_MSG_LINK | \
+				 NETIF_MSG_TIMER | \
+				 NETIF_MSG_IFDOWN | \
+				 NETIF_MSG_IFUP | \
+				 NETIF_MSG_RX_ERR | \
+				 NETIF_MSG_TX_ERR | \
+				 NETIF_MSG_TX_QUEUED | \
+				 NETIF_MSG_INTR | \
+				 NETIF_MSG_TX_DONE | \
+				 NETIF_MSG_RX_STATUS | \
+				 NETIF_MSG_PKTDATA | \
+				 NETIF_MSG_HW | \
+				 NETIF_MSG_WOL)
+
+#define prueth_napi_to_emac(napi) container_of(napi, struct prueth_emac, napi)
+
+/* CTRLMMR_ICSSG_RGMII_CTRL register bits */
+#define ICSSG_CTRL_RGMII_ID_MODE		BIT(24)
+
+static int debug_level = -1;
+module_param(debug_level, int, 0644);
+MODULE_PARM_DESC(debug_level, "PRUETH debug level (NETIF_MSG bits)");
+
+static void prueth_cleanup_rx_chns(struct prueth_emac *emac,
+				   struct prueth_rx_chn *rx_chn,
+				   int max_rflows)
+{
+	int i;
+
+	if (rx_chn->rx_chn) {
+		for (i = 0; i < max_rflows; i++)
+			k3_nav_udmax_rx_put_irq(rx_chn->rx_chn, i);
+
+		k3_nav_udmax_release_rx_chn(rx_chn->rx_chn);
+	}
+
+	if (rx_chn->desc_pool)
+		k3_knav_pool_destroy(rx_chn->desc_pool);
+}
+
+static void prueth_cleanup_tx_chns(struct prueth_emac *emac)
+{
+	struct prueth_tx_chn *tx_chn = &emac->tx_chns;
+
+	if (tx_chn->irq)
+		k3_nav_udmax_tx_put_irq(tx_chn->tx_chn);
+
+	if (tx_chn->tx_chn)
+		k3_nav_udmax_release_tx_chn(tx_chn->tx_chn);
+
+	if (tx_chn->desc_pool)
+		k3_knav_pool_destroy(tx_chn->desc_pool);
+}
+
+static int prueth_init_tx_chns(struct prueth_emac *emac)
+{
+	struct net_device *ndev = emac->ndev;
+	struct device *dev = emac->prueth->dev;
+	struct k3_nav_udmax_tx_channel_cfg tx_cfg;
+	static const struct k3_ring_cfg ring_cfg = {
+		.elm_size = K3_RINGACC_RING_ELSIZE_8,
+		.mode = K3_RINGACC_RING_MODE_RING,
+		.flags = 0,
+		.size = PRUETH_MAX_TX_DESC,
+	};
+	u32 hdesc_size;
+	int ret, slice;
+	struct prueth_tx_chn *tx_chn = &emac->tx_chns;
+	char tx_chn_name[16];
+
+	slice = prueth_emac_slice(emac);
+	if (slice < 0)
+		return slice;
+
+	init_completion(&emac->tdown_complete);
+
+	hdesc_size = cppi5_hdesc_calc_size(true, PRUETH_NAV_PS_DATA_SIZE,
+					   PRUETH_NAV_SW_DATA_SIZE);
+	memset(&tx_cfg, 0, sizeof(tx_cfg));
+	tx_cfg.swdata_size = PRUETH_NAV_SW_DATA_SIZE;
+	tx_cfg.tx_cfg = ring_cfg;
+	tx_cfg.txcq_cfg = ring_cfg;
+
+	/* To differentiate channels for SLICE0 vs SLICE1 */
+	snprintf(tx_chn_name, sizeof(tx_chn_name), "tx%d-0", slice);
+
+	tx_chn->descs_num = PRUETH_MAX_TX_DESC;
+	spin_lock_init(&tx_chn->lock);
+	tx_chn->desc_pool = k3_knav_pool_create_name(dev, tx_chn->descs_num,
+						     hdesc_size, tx_chn_name);
+	if (IS_ERR(tx_chn->desc_pool)) {
+		ret = PTR_ERR(tx_chn->desc_pool);
+		tx_chn->desc_pool = NULL;
+		netdev_err(ndev, "Failed to create tx pool: %d\n", ret);
+		goto fail;
+	}
+
+	tx_chn->tx_chn = k3_nav_udmax_request_tx_chn(dev, tx_chn_name, &tx_cfg);
+	if (IS_ERR(tx_chn->tx_chn)) {
+		ret = PTR_ERR(tx_chn->tx_chn);
+		tx_chn->tx_chn = NULL;
+		netdev_err(ndev, "Failed to request tx dma ch: %d\n", ret);
+		goto fail;
+	}
+
+	ret = k3_nav_udmax_tx_get_irq(tx_chn->tx_chn, &tx_chn->irq,
+				      IRQF_TRIGGER_HIGH, false, NULL);
+	if (ret) {
+		tx_chn->irq = 0;
+		netdev_err(ndev, "failed to get tx irq\n");
+		goto fail;
+	}
+
+	return 0;
+
+fail:
+	prueth_cleanup_tx_chns(emac);
+	return ret;
+}
+
+static int prueth_init_rx_chns(struct prueth_emac *emac,
+			       struct prueth_rx_chn *rx_chn,
+			       char *name, u32 max_rflows,
+			       u32 max_desc_num)
+{
+	struct net_device *ndev = emac->ndev;
+	struct device *dev = emac->prueth->dev;
+	struct k3_nav_udmax_rx_channel_cfg rx_cfg;
+	u32 fdqring_id;
+	u32 hdesc_size;
+	int i, ret = 0, slice;
+	char rx_chn_name[16];
+
+	slice = prueth_emac_slice(emac);
+	if (slice < 0)
+		return slice;
+
+	/* To differentiate channels for SLICE0 vs SLICE1 */
+	snprintf(rx_chn_name, sizeof(rx_chn_name), "%s%d", name, slice);
+
+	hdesc_size = cppi5_hdesc_calc_size(true, PRUETH_NAV_PS_DATA_SIZE,
+					   PRUETH_NAV_SW_DATA_SIZE);
+	memset(&rx_cfg, 0, sizeof(rx_cfg));
+	rx_cfg.swdata_size = PRUETH_NAV_SW_DATA_SIZE;
+	rx_cfg.flow_id_num = max_rflows;
+	rx_cfg.flow_id_base = -1; /* udmax will auto select flow id base */
+
+	/* init all flows */
+	rx_chn->dev = dev;
+	rx_chn->descs_num = max_desc_num;
+	spin_lock_init(&rx_chn->lock);
+	rx_chn->desc_pool = k3_knav_pool_create_name(dev, rx_chn->descs_num,
+						     hdesc_size, rx_chn_name);
+	if (IS_ERR(rx_chn->desc_pool)) {
+		ret = PTR_ERR(rx_chn->desc_pool);
+		rx_chn->desc_pool = NULL;
+		netdev_err(ndev, "Failed to create rx pool: %d\n", ret);
+		goto fail;
+	}
+
+	rx_chn->rx_chn = k3_nav_udmax_request_rx_chn(dev, rx_chn_name, &rx_cfg);
+	if (IS_ERR(rx_chn->rx_chn)) {
+		ret = PTR_ERR(rx_chn->rx_chn);
+		rx_chn->rx_chn = NULL;
+		netdev_err(ndev, "Failed to request rx dma ch: %d\n", ret);
+		goto fail;
+	}
+
+	if (!strncmp(name, "rxmgm", 5)) {
+		emac->rx_mgm_flow_id_base = k3_nav_udmax_rx_get_flow_id_base(rx_chn->rx_chn);
+		netdev_dbg(ndev, "mgm flow id base = %d\n",
+			   emac->rx_mgm_flow_id_base);
+	} else {
+		emac->rx_flow_id_base = k3_nav_udmax_rx_get_flow_id_base(rx_chn->rx_chn);
+		netdev_dbg(ndev, "flow id base = %d\n",
+			   emac->rx_flow_id_base);
+	}
+
+	fdqring_id = K3_RINGACC_RING_ID_ANY;
+	for (i = 0; i < rx_cfg.flow_id_num; i++) {
+		struct k3_ring_cfg rxring_cfg = {
+			.elm_size = K3_RINGACC_RING_ELSIZE_8,
+			.mode = K3_RINGACC_RING_MODE_MESSAGE,
+			.flags = 0,
+		};
+		struct k3_ring_cfg fdqring_cfg = {
+			.elm_size = K3_RINGACC_RING_ELSIZE_8,
+			.mode = K3_RINGACC_RING_MODE_MESSAGE,
+			.flags = K3_RINGACC_RING_SHARED,
+		};
+		struct k3_nav_udmax_rx_flow_cfg rx_flow_cfg = {
+			.rx_cfg = rxring_cfg,
+			.rxfdq_cfg = fdqring_cfg,
+			.ring_rxq_id = K3_RINGACC_RING_ID_ANY,
+			.src_tag_lo_sel =
+				K3_NAV_UDMAX_SRC_TAG_LO_USE_REMOTE_SRC_TAG,
+		};
+
+		rx_flow_cfg.ring_rxfdq0_id = fdqring_id;
+		rx_flow_cfg.rx_cfg.size = max_desc_num;
+		rx_flow_cfg.rxfdq_cfg.size = max_desc_num;
+
+		ret = k3_nav_udmax_rx_flow_init(rx_chn->rx_chn,
+						i, &rx_flow_cfg);
+		if (ret) {
+			dev_err(dev, "Failed to init rx flow%d %d\n", i, ret);
+			goto fail;
+		}
+		if (!i)
+			fdqring_id = k3_nav_udmax_rx_flow_get_fdq_id(rx_chn->rx_chn,
+								     i);
+		ret = k3_nav_udmax_rx_get_irq(rx_chn->rx_chn, i, &rx_chn->irq,
+					      IRQF_TRIGGER_HIGH,
+					      true, i ? 0 : -1);
+		if (ret) {
+			dev_err(dev, "Failed to get rx dma irq %d\n", ret);
+			goto fail;
+		}
+	}
+
+	return 0;
+
+fail:
+	prueth_cleanup_rx_chns(emac, rx_chn, max_rflows);
+	return ret;
+}
+
+static int prueth_dma_rx_push(struct prueth_emac *emac,
+			      struct sk_buff *skb,
+			      struct prueth_rx_chn *rx_chn)
+{
+	struct cppi5_host_desc_t *desc_rx;
+	struct device *dev = emac->prueth->dev;
+	struct net_device *ndev = emac->ndev;
+	dma_addr_t desc_dma;
+	dma_addr_t buf_dma;
+	u32 pkt_len = skb_tailroom(skb);
+	void **swdata;
+
+	desc_rx = k3_knav_pool_alloc(rx_chn->desc_pool);
+	if (!desc_rx) {
+		netdev_err(ndev, "rx push: failed to allocate descriptor\n");
+		return -ENOMEM;
+	}
+	desc_dma = k3_knav_pool_virt2dma(rx_chn->desc_pool, desc_rx);
+
+	buf_dma = dma_map_single(dev, skb->data, pkt_len, DMA_FROM_DEVICE);
+	if (unlikely(dma_mapping_error(dev, buf_dma))) {
+		k3_knav_pool_free(rx_chn->desc_pool, desc_rx);
+		netdev_err(ndev, "rx push: failed to map rx pkt buffer\n");
+		return -EINVAL;
+	}
+
+	cppi5_hdesc_init(desc_rx, CPPI5_INFO0_HDESC_EPIB_PRESENT,
+			 PRUETH_NAV_PS_DATA_SIZE);
+	cppi5_hdesc_attach_buf(desc_rx, 0, 0, buf_dma, skb_tailroom(skb));
+
+	swdata = cppi5_hdesc_get_swdata(desc_rx);
+	*swdata = skb;
+
+	return k3_nav_udmax_push_rx_chn(rx_chn->rx_chn, 0,
+					desc_rx, desc_dma);
+}
+
+static void emac_rx_timestamp(struct sk_buff *skb, u32 *psdata)
+{
+	struct skb_shared_hwtstamps *ssh;
+	u64 ns;
+
+	ns = (u64)psdata[1] << 32 | psdata[0];
+
+	ssh = skb_hwtstamps(skb);
+	memset(ssh, 0, sizeof(*ssh));
+	ssh->hwtstamp = ns_to_ktime(ns);
+}
+
+/**
+ * emac_rx_packet - Get one packet from RX ring and push to netdev.
+ * Returns 0 on success, else error code.
+ */
+static int emac_rx_packet(struct prueth_emac *emac, u32 flow_id)
+{
+	struct prueth_rx_chn *rx_chn = &emac->rx_chns;
+	struct device *dev = emac->prueth->dev;
+	struct net_device *ndev = emac->ndev;
+	struct cppi5_host_desc_t *desc_rx;
+	dma_addr_t desc_dma, buf_dma;
+	u32 buf_dma_len, pkt_len, port_id = 0;
+	int ret;
+	void **swdata;
+	struct sk_buff *skb, *new_skb;
+	u32 *psdata;
+
+	ret = k3_nav_udmax_pop_rx_chn(rx_chn->rx_chn, flow_id, &desc_dma);
+	if (ret) {
+		if (ret != -ENODATA)
+			netdev_err(ndev, "rx pop: failed: %d\n", ret);
+		return ret;
+	}
+
+	if (desc_dma & 0x1) /* Teardown ? */
+		return 0;
+
+	desc_rx = k3_knav_pool_dma2virt(rx_chn->desc_pool, desc_dma);
+
+	swdata = cppi5_hdesc_get_swdata(desc_rx);
+	skb = *swdata;
+
+	psdata = cppi5_hdesc_get_psdata32(desc_rx);
+	/* RX HW timestamp */
+	if (emac->rx_ts_enabled)
+		emac_rx_timestamp(skb, psdata);
+
+	cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);
+	pkt_len = cppi5_hdesc_get_pktlen(desc_rx);
+	/* firmware adds 4 CRC bytes, strip them */
+	pkt_len -= 4;
+	cppi5_desc_get_tags_ids(&desc_rx->hdr, &port_id, NULL);
+
+	dma_unmap_single(dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE);
+	k3_knav_pool_free(rx_chn->desc_pool, desc_rx);
+
+	skb->dev = ndev;
+	if (!netif_running(skb->dev)) {
+		dev_kfree_skb_any(skb);
+		return 0;
+	}
+
+	new_skb = netdev_alloc_skb_ip_align(ndev, PRUETH_MAX_PKT_SIZE);
+	/* if allocation fails we drop the packet but push the
+	 * descriptor back to the ring with old skb to prevent a stall
+	 */
+	if (!new_skb) {
+		ndev->stats.rx_dropped++;
+		new_skb = skb;
+	} else {
+		/* send the filled skb up the n/w stack */
+		skb_put(skb, pkt_len);
+		skb->protocol = eth_type_trans(skb, ndev);
+		netif_receive_skb(skb);
+		ndev->stats.rx_bytes += pkt_len;
+		ndev->stats.rx_packets++;
+	}
+
+	/* queue another RX DMA */
+	ret = prueth_dma_rx_push(emac, new_skb, &emac->rx_chns);
+	if (WARN_ON(ret < 0)) {
+		dev_kfree_skb_any(new_skb);
+		ndev->stats.rx_errors++;
+		ndev->stats.rx_dropped++;
+	}
+
+	return ret;
+}
+
+static void prueth_rx_cleanup(void *data, dma_addr_t desc_dma)
+{
+	struct prueth_rx_chn *rx_chn = data;
+	struct cppi5_host_desc_t *desc_rx;
+	struct sk_buff *skb;
+	dma_addr_t buf_dma;
+	u32 buf_dma_len;
+	void **swdata;
+
+	desc_rx = k3_knav_pool_dma2virt(rx_chn->desc_pool, desc_dma);
+	swdata = cppi5_hdesc_get_swdata(desc_rx);
+	skb = *swdata;
+	cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);
+
+	dma_unmap_single(rx_chn->dev, buf_dma, buf_dma_len,
+			 DMA_FROM_DEVICE);
+	k3_knav_pool_free(rx_chn->desc_pool, desc_rx);
+
+	dev_kfree_skb_any(skb);
+}
+
+static void prueth_xmit_free(struct prueth_tx_chn *tx_chn,
+			     struct device *dev,
+			     struct cppi5_host_desc_t *desc)
+{
+	struct cppi5_host_desc_t *first_desc, *next_desc;
+	dma_addr_t buf_dma, next_desc_dma;
+	u32 buf_dma_len;
+
+	first_desc = desc;
+	next_desc = first_desc;
+
+	cppi5_hdesc_get_obuf(first_desc, &buf_dma, &buf_dma_len);
+
+	dma_unmap_single(dev, buf_dma, buf_dma_len,
+			 DMA_TO_DEVICE);
+
+	next_desc_dma = cppi5_hdesc_get_next_hbdesc(first_desc);
+	while (next_desc_dma) {
+		next_desc = k3_knav_pool_dma2virt(tx_chn->desc_pool,
+						  next_desc_dma);
+		cppi5_hdesc_get_obuf(next_desc, &buf_dma, &buf_dma_len);
+
+		dma_unmap_page(dev, buf_dma, buf_dma_len,
+			       DMA_TO_DEVICE);
+
+		next_desc_dma = cppi5_hdesc_get_next_hbdesc(next_desc);
+
+		k3_knav_pool_free(tx_chn->desc_pool, next_desc);
+	}
+
+	k3_knav_pool_free(tx_chn->desc_pool, first_desc);
+}
+
+/* TODO: Convert this to use worker/workqueue mechanism to serialize the
+ * request to firmware
+ */
+static int emac_send_command(struct prueth_emac *emac, u32 cmd)
+{
+	struct device *dev = emac->prueth->dev;
+	dma_addr_t desc_dma, buf_dma;
+	struct prueth_tx_chn *tx_chn;
+	struct cppi5_host_desc_t *first_desc;
+	int ret = 0;
+	u32 *epib;
+	u32 *data = emac->cmd_data;
+	u32 pkt_len = sizeof(emac->cmd_data);
+	void **swdata;
+
+	netdev_dbg(emac->ndev, "Sending cmd %x\n", cmd);
+
+	/* only one command at a time allowed to firmware */
+	mutex_lock(&emac->cmd_lock);
+	data[0] = cpu_to_le32(cmd);
+
+	/* Map the linear buffer */
+	buf_dma = dma_map_single(dev, data, pkt_len, DMA_TO_DEVICE);
+	if (dma_mapping_error(dev, buf_dma)) {
+		netdev_err(emac->ndev, "cmd %x: failed to map cmd buffer\n",
+			   cmd);
+		ret = -EINVAL;
+		goto err_unlock;
+	}
+
+	tx_chn = &emac->tx_chns;
+
+	first_desc = k3_knav_pool_alloc(tx_chn->desc_pool);
+	if (!first_desc) {
+		netdev_err(emac->ndev,
+			   "cmd %x: failed to allocate descriptor\n", cmd);
+		dma_unmap_single(dev, buf_dma, pkt_len, DMA_TO_DEVICE);
+		ret = -ENOMEM;
+		goto err_unlock;
+	}
+
+	cppi5_hdesc_init(first_desc, CPPI5_INFO0_HDESC_EPIB_PRESENT,
+			 PRUETH_NAV_PS_DATA_SIZE);
+	cppi5_hdesc_set_pkttype(first_desc, PRUETH_PKT_TYPE_CMD);
+	epib = first_desc->epib;
+	epib[0] = 0;
+	epib[1] = 0;
+
+	cppi5_hdesc_attach_buf(first_desc, buf_dma, pkt_len, buf_dma, pkt_len);
+	swdata = cppi5_hdesc_get_swdata(first_desc);
+	*swdata = data;
+
+	cppi5_hdesc_set_pktlen(first_desc, pkt_len);
+	desc_dma = k3_knav_pool_virt2dma(tx_chn->desc_pool, first_desc);
+
+	/* send command */
+	reinit_completion(&emac->cmd_complete);
+	ret = k3_nav_udmax_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma);
+	if (ret) {
+		netdev_err(emac->ndev, "cmd %x: push failed: %d\n", cmd, ret);
+		goto free_desc;
+	}
+	ret = wait_for_completion_timeout(&emac->cmd_complete,
+					  msecs_to_jiffies(100));
+	if (!ret)
+		netdev_err(emac->ndev, "cmd %x: completion timeout\n", cmd);
+
+	mutex_unlock(&emac->cmd_lock);
+
+	return ret;
+free_desc:
+	prueth_xmit_free(tx_chn, dev, first_desc);
+err_unlock:
+	mutex_unlock(&emac->cmd_lock);
+
+	return ret;
+}
+
+static void emac_change_port_speed_duplex(struct prueth_emac *emac,
+					  bool full_duplex, int speed)
+{
+	u32 cmd = ICSSG_PSTATE_SPEED_DUPLEX_CMD, val;
+	struct prueth *prueth = emac->prueth;
+	int slice = prueth_emac_slice(emac);
+
+	/* only 100M and 1G and full duplex supported for now */
+	if (!(full_duplex && (speed == SPEED_1000 || speed == SPEED_100)))
+		return;
+
+	val = icssg_rgmii_get_speed(prueth->miig_rt, slice);
+	/* firmware expects full duplex settings in bit 2-1 */
+	val <<= 1;
+	cmd |= val;
+
+	val = icssg_rgmii_get_fullduplex(prueth->miig_rt, slice);
+	/* firmware expects full duplex settings in bit 3 */
+	val <<= 3;
+	cmd |= val;
+	emac_send_command(emac, cmd);
+}
+
+static int emac_shutdown(struct net_device *ndev)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+
+	return emac_send_command(emac, ICSSG_SHUTDOWN_CMD);
+}
+
+/**
+ * emac_ndo_start_xmit - EMAC Transmit function
+ * @skb: SKB pointer
+ * @ndev: EMAC network adapter
+ *
+ * Called by the system to transmit a packet  - we queue the packet in
+ * EMAC hardware transmit queue
+ * Doesn't wait for completion we'll check for TX completion in
+ * emac_tx_complete_packets().
+ *
+ * Returns success(NETDEV_TX_OK) or error code (typically out of descs)
+ */
+static int emac_ndo_start_xmit(struct sk_buff *skb, struct net_device *ndev)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+	int ret = 0;
+	struct device *dev = emac->prueth->dev;
+	struct cppi5_host_desc_t *first_desc, *next_desc, *cur_desc;
+	struct prueth_tx_chn *tx_chn;
+	dma_addr_t desc_dma, buf_dma;
+	u32 pkt_len;
+	int i;
+	void **swdata;
+	u32 *epib;
+	bool in_tx_ts = 0;
+
+	/* frag list based linkage is not supported for now. */
+	if (skb_shinfo(skb)->frag_list) {
+		dev_err_ratelimited(dev, "NETIF_F_FRAGLIST not supported\n");
+		ret = -EINVAL;
+		goto drop_free_skb;
+	}
+
+	pkt_len = skb_headlen(skb);
+	tx_chn = &emac->tx_chns;
+
+	/* Map the linear buffer */
+	buf_dma = dma_map_single(dev, skb->data, pkt_len, DMA_TO_DEVICE);
+	if (dma_mapping_error(dev, buf_dma)) {
+		netdev_err(ndev, "tx: failed to map skb buffer\n");
+		ret = -EINVAL;
+		goto drop_stop_q;
+	}
+
+	first_desc = k3_knav_pool_alloc(tx_chn->desc_pool);
+	if (!first_desc) {
+		netdev_dbg(ndev, "tx: failed to allocate descriptor\n");
+		dma_unmap_single(dev, buf_dma, pkt_len, DMA_TO_DEVICE);
+		ret = -ENOMEM;
+		goto drop_stop_q_busy;
+	}
+
+	cppi5_hdesc_init(first_desc, CPPI5_INFO0_HDESC_EPIB_PRESENT,
+			 PRUETH_NAV_PS_DATA_SIZE);
+	cppi5_hdesc_set_pkttype(first_desc, 0);
+	epib = first_desc->epib;
+	epib[0] = 0;
+	epib[1] = 0;
+	if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP &&
+	    emac->tx_ts_enabled) {
+		/* We currently support only one TX HW timestamp at a time */
+		if (!test_and_set_bit_lock(__STATE_TX_TS_IN_PROGRESS,
+					   &emac->state)) {
+			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
+			/* Request TX timestamp */
+			epib[0] = emac->tx_ts_cookie;
+			epib[1] = 0x80000000;	/* TX TS request */
+			emac->tx_ts_skb = skb_get(skb);
+			in_tx_ts = 1;
+			/* TODO: note time and check for timeout if HW
+			 * doesn't come back with TS
+			 */
+		}
+	}
+
+	cppi5_hdesc_attach_buf(first_desc, buf_dma, pkt_len, buf_dma, pkt_len);
+	swdata = cppi5_hdesc_get_swdata(first_desc);
+	*swdata = skb;
+
+	if (!skb_is_nonlinear(skb))
+		goto tx_push;
+
+	/* Handle the case where skb is fragmented in pages */
+	cur_desc = first_desc;
+	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
+		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
+		u32 frag_size = skb_frag_size(frag);
+
+		next_desc = k3_knav_pool_alloc(tx_chn->desc_pool);
+		if (!next_desc) {
+			netdev_err(ndev,
+				   "tx: failed to allocate frag. descriptor\n");
+			ret = -ENOMEM;
+			goto cleanup_tx_ts;
+		}
+
+		buf_dma = skb_frag_dma_map(dev, frag, 0, frag_size,
+					   DMA_TO_DEVICE);
+		if (dma_mapping_error(dev, buf_dma)) {
+			netdev_err(ndev, "tx: Failed to map skb page\n");
+			k3_knav_pool_free(tx_chn->desc_pool, next_desc);
+			ret = -EINVAL;
+			goto cleanup_tx_ts;
+		}
+
+		cppi5_hdesc_reset_hbdesc(next_desc);
+		cppi5_hdesc_attach_buf(next_desc,
+				       buf_dma, frag_size, buf_dma, frag_size);
+
+		desc_dma = k3_knav_pool_virt2dma(tx_chn->desc_pool, next_desc);
+		cppi5_hdesc_link_hbdesc(cur_desc, desc_dma);
+
+		pkt_len += frag_size;
+		cur_desc = next_desc;
+	}
+	WARN_ON(pkt_len != skb->len);
+
+tx_push:
+	/* report bql before sending packet */
+	netdev_sent_queue(ndev, pkt_len);
+
+	cppi5_hdesc_set_pktlen(first_desc, pkt_len);
+	desc_dma = k3_knav_pool_virt2dma(tx_chn->desc_pool, first_desc);
+	/* cppi5_desc_dump(first_desc, 64); */
+
+	skb_tx_timestamp(skb);	/* SW timestamp if SKBTX_IN_PROGRESS not set */
+	ret = k3_nav_udmax_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma);
+	if (ret) {
+		netdev_err(ndev, "tx: push failed: %d\n", ret);
+		goto drop_free_descs;
+	}
+
+	if (k3_knav_pool_avail(tx_chn->desc_pool) < MAX_SKB_FRAGS)
+		netif_stop_queue(ndev);
+
+	return NETDEV_TX_OK;
+
+cleanup_tx_ts:
+	if (in_tx_ts) {
+		dev_kfree_skb_any(emac->tx_ts_skb);
+		emac->tx_ts_skb = NULL;
+		clear_bit_unlock(__STATE_TX_TS_IN_PROGRESS, &emac->state);
+	}
+
+drop_free_descs:
+	prueth_xmit_free(tx_chn, dev, first_desc);
+drop_stop_q:
+	netif_stop_queue(ndev);
+drop_free_skb:
+	dev_kfree_skb_any(skb);
+
+	/* error */
+	ndev->stats.tx_dropped++;
+	netdev_err(ndev, "tx: error: %d\n", ret);
+
+	return ret;
+
+drop_stop_q_busy:
+	netif_stop_queue(ndev);
+	return NETDEV_TX_BUSY;
+}
+
+/**
+ * emac_tx_complete_packets - Check if TX completed packets upto budget.
+ * Returns number of completed TX packets.
+ */
+static int emac_tx_complete_packets(struct prueth_emac *emac, int budget)
+{
+	struct net_device *ndev = emac->ndev;
+	struct cppi5_host_desc_t *desc_tx;
+	struct device *dev = emac->prueth->dev;
+	struct prueth_tx_chn *tx_chn;
+	unsigned int total_bytes = 0;
+	struct sk_buff *skb;
+	dma_addr_t desc_dma;
+	int res, num_tx = 0;
+	void **swdata;
+
+	tx_chn = &emac->tx_chns;
+
+	while (budget--) {
+		res = k3_nav_udmax_pop_tx_chn(tx_chn->tx_chn, &desc_dma);
+		if (res == -ENODATA)
+			break;
+
+		/* teardown completion */
+		if (desc_dma & 0x1) {
+			complete(&emac->tdown_complete);
+			break;
+		}
+
+		desc_tx = k3_knav_pool_dma2virt(tx_chn->desc_pool, desc_dma);
+		swdata = cppi5_hdesc_get_swdata(desc_tx);
+
+		/* was this command's TX complete? */
+		if (*(swdata) == emac->cmd_data) {
+			prueth_xmit_free(tx_chn, dev, desc_tx);
+			budget++;	/* not a data packet */
+			continue;
+		}
+
+		skb = *(swdata);
+		prueth_xmit_free(tx_chn, dev, desc_tx);
+
+		ndev = skb->dev;
+		ndev->stats.tx_packets++;
+		ndev->stats.tx_bytes += skb->len;
+		total_bytes += skb->len;
+		napi_consume_skb(skb, budget);
+		num_tx++;
+	}
+
+	if (!num_tx)
+		return 0;
+
+	netdev_completed_queue(ndev, num_tx, total_bytes);
+
+	if (netif_queue_stopped(ndev)) {
+		/* If the the TX queue was stopped, wake it now
+		 * if we have enough room.
+		 */
+		netif_tx_lock(ndev);
+		if (netif_running(ndev) &&
+		    (k3_knav_pool_avail(tx_chn->desc_pool) >= MAX_SKB_FRAGS))
+			netif_wake_queue(ndev);
+		netif_tx_unlock(ndev);
+	}
+
+	return num_tx;
+}
+
+static void prueth_tx_cleanup(void *data, dma_addr_t desc_dma)
+{
+	struct prueth_emac *emac = data;
+	struct prueth_tx_chn *tx_chn = &emac->tx_chns;
+	struct cppi5_host_desc_t *desc_tx;
+	struct sk_buff *skb;
+	void **swdata;
+
+	desc_tx = k3_knav_pool_dma2virt(tx_chn->desc_pool, desc_dma);
+	swdata = cppi5_hdesc_get_swdata(desc_tx);
+	skb = *(swdata);
+	prueth_xmit_free(tx_chn, emac->prueth->dev, desc_tx);
+
+	dev_kfree_skb_any(skb);
+}
+
+/* get one packet from requested flow_id
+ *
+ * Returns skb pointer if packet found else NULL
+ * Caller must free the returned skb.
+ */
+static struct sk_buff *prueth_process_rx_mgm(struct prueth_emac *emac,
+					     u32 flow_id)
+{
+	struct prueth_rx_chn *rx_chn = &emac->rx_mgm_chn;
+	struct device *dev = emac->prueth->dev;
+	struct net_device *ndev = emac->ndev;
+	struct cppi5_host_desc_t *desc_rx;
+	dma_addr_t desc_dma, buf_dma;
+	u32 buf_dma_len, pkt_len;
+	int ret;
+	void **swdata;
+	struct sk_buff *skb, *new_skb;
+
+	ret = k3_nav_udmax_pop_rx_chn(rx_chn->rx_chn, flow_id, &desc_dma);
+	if (ret) {
+		if (ret != -ENODATA)
+			netdev_err(ndev, "rx mgm pop: failed: %d\n", ret);
+		return NULL;
+	}
+
+	if (desc_dma & 0x1) /* Teardown ? */
+		return NULL;
+
+	desc_rx = k3_knav_pool_dma2virt(rx_chn->desc_pool, desc_dma);
+
+	/* Fix FW bug about incorrect PSDATA size */
+	if (cppi5_hdesc_get_psdata_size(desc_rx) != PRUETH_NAV_PS_DATA_SIZE) {
+		cppi5_hdesc_update_psdata_size(desc_rx,
+					       PRUETH_NAV_PS_DATA_SIZE);
+	}
+
+	swdata = cppi5_hdesc_get_swdata(desc_rx);
+	skb = *swdata;
+	cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);
+	pkt_len = cppi5_hdesc_get_pktlen(desc_rx);
+
+	dma_unmap_single(dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE);
+	k3_knav_pool_free(rx_chn->desc_pool, desc_rx);
+
+	new_skb = netdev_alloc_skb_ip_align(ndev, PRUETH_MAX_PKT_SIZE);
+	/* if allocation fails we drop the packet but push the
+	 * descriptor back to the ring with old skb to prevent a stall
+	 */
+	if (!new_skb) {
+		netdev_err(ndev,
+			   "skb alloc failed, dropped mgm pkt from flow %d\n",
+			   flow_id);
+		new_skb = skb;
+		skb = NULL;	/* return NULL */
+	} else {
+		/* return the filled skb */
+		skb_put(skb, pkt_len);
+	}
+
+	/* queue another DMA */
+	ret = prueth_dma_rx_push(emac, new_skb, &emac->rx_mgm_chn);
+	if (WARN_ON(ret < 0))
+		dev_kfree_skb_any(new_skb);
+
+	return skb;
+}
+
+static void prueth_tx_ts(struct prueth_emac *emac,
+			 struct emac_tx_ts_response *tsr)
+{
+	u64 ns;
+	struct skb_shared_hwtstamps ssh;
+	struct sk_buff *skb;
+
+	ns = (u64)tsr->hi_ts << 32 | tsr->lo_ts;
+
+	if (!test_bit(__STATE_TX_TS_IN_PROGRESS, &emac->state)) {
+		netdev_err(emac->ndev, "unexpected TS response\n");
+		return;
+	}
+
+	skb = emac->tx_ts_skb;
+	if (tsr->cookie != emac->tx_ts_cookie) {
+		netdev_err(emac->ndev, "TX TS cookie mismatch 0x%x:0x%x\n",
+			   tsr->cookie, emac->tx_ts_cookie);
+		goto error;
+	}
+
+	emac->tx_ts_cookie++;
+	memset(&ssh, 0, sizeof(ssh));
+	ssh.hwtstamp = ns_to_ktime(ns);
+	clear_bit_unlock(__STATE_TX_TS_IN_PROGRESS, &emac->state);
+
+	skb_tstamp_tx(skb, &ssh);
+	dev_consume_skb_any(skb);
+
+	return;
+
+error:
+	dev_kfree_skb_any(skb);
+	emac->tx_ts_skb = NULL;
+	clear_bit_unlock(__STATE_TX_TS_IN_PROGRESS, &emac->state);
+}
+
+static irqreturn_t prueth_rx_mgm_irq_thread(int irq, void *dev_id)
+{
+	struct prueth_emac *emac = dev_id;
+	struct sk_buff *skb;
+	int flow = PRUETH_MAX_RX_MGM_FLOWS - 1;
+	u32 rsp;
+
+	while (flow--) {
+		skb = prueth_process_rx_mgm(emac, flow);
+		if (!skb)
+			continue;
+
+		switch (flow) {
+		case PRUETH_RX_MGM_FLOW_RESPONSE:
+			/* Process command response */
+			rsp = le32_to_cpu(*(u32 *)skb->data);
+			if ((rsp & 0xffff0000) == ICSSG_SHUTDOWN_CMD) {
+				netdev_dbg(emac->ndev,
+					   "f/w Shutdown cmd resp %x\n", rsp);
+				complete(&emac->cmd_complete);
+			} else if ((rsp & 0xffff0000) ==
+				ICSSG_PSTATE_SPEED_DUPLEX_CMD) {
+				netdev_dbg(emac->ndev,
+					   "f/w Speed/Duplex cmd rsp %x\n",
+					    rsp);
+				complete(&emac->cmd_complete);
+			} else {
+				netdev_err(emac->ndev, "Unknown f/w cmd rsp %x\n",
+					   rsp);
+			}
+			break;
+		case PRUETH_RX_MGM_FLOW_TIMESTAMP:
+			prueth_tx_ts(emac, (void *)skb->data);
+			break;
+		default:
+			continue;
+		}
+
+		dev_kfree_skb_any(skb);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t prueth_rx_irq(int irq, void *dev_id)
+{
+	struct prueth_emac *emac = dev_id;
+
+	disable_irq_nosync(irq);
+	napi_schedule(&emac->napi_rx);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t prueth_tx_irq(int irq, void *dev_id)
+{
+	struct prueth_emac *emac = dev_id;
+
+	disable_irq_nosync(irq);
+	napi_schedule(&emac->napi_tx);
+
+	return IRQ_HANDLED;
+}
+
+static void icssg_config_set(struct prueth *prueth, int slice)
+{
+	void __iomem *va;
+
+	va = prueth->shram.va + slice * ICSSG_CONFIG_OFFSET_SLICE1;
+	memcpy_toio(va, &prueth->config[slice], sizeof(prueth->config[slice]));
+}
+
+static int prueth_emac_start(struct prueth *prueth, struct prueth_emac *emac)
+{
+	struct device *dev = prueth->dev;
+	int slice, ret;
+	struct icssg_config *config;
+	int i;
+
+	slice = prueth_emac_slice(emac);
+	if (slice < 0) {
+		netdev_err(emac->ndev, "invalid port\n");
+		return -EINVAL;
+	}
+
+	/* Set Load time configuration */
+	config = &prueth->config[slice];
+	memset(config, 0, sizeof(*config));
+	config->addr_lo = cpu_to_le32(lower_32_bits(prueth->msmcram.pa));
+	config->addr_hi = cpu_to_le32(upper_32_bits(prueth->msmcram.pa));
+	config->num_tx_threads = 0;
+	config->rx_flow_id = emac->rx_flow_id_base; /* flow id for host port */
+	config->rx_mgr_flow_id = emac->rx_mgm_flow_id_base; /* for mgm ch */
+
+	/* set buffer sizes for the pools. 0-7 are not used for dual-emac */
+	for (i = PRUETH_EMAC_BUF_POOL_START;
+	     i < PRUETH_NUM_BUF_POOLS; i++)
+		config->tx_buf_sz[i] = cpu_to_le32(PRUETH_EMAC_BUF_POOL_SIZE);
+
+	icssg_config_set(prueth, slice);
+
+	ret = rproc_boot(prueth->pru[slice]);
+	if (ret) {
+		dev_err(dev, "failed to boot PRU%d: %d\n", slice, ret);
+		return -EINVAL;
+	}
+
+	ret = rproc_boot(prueth->rtu[slice]);
+	if (ret) {
+		dev_err(dev, "failed to boot RTU%d: %d\n", slice, ret);
+		goto halt_pru;
+	}
+
+	return 0;
+
+halt_pru:
+	rproc_shutdown(prueth->pru[slice]);
+
+	return ret;
+}
+
+static void prueth_emac_stop(struct prueth_emac *emac)
+{
+	struct prueth *prueth = emac->prueth;
+	int slice;
+
+	switch (emac->port_id) {
+	case PRUETH_PORT_MII0:
+		slice = ICSS_SLICE0;
+		break;
+	case PRUETH_PORT_MII1:
+		slice = ICSS_SLICE1;
+		break;
+	default:
+		netdev_err(emac->ndev, "invalid port\n");
+		return;
+	}
+
+	rproc_shutdown(prueth->rtu[slice]);
+	rproc_shutdown(prueth->pru[slice]);
+}
+
+/* called back by PHY layer if there is change in link state of hw port*/
+static void emac_adjust_link(struct net_device *ndev)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+	struct phy_device *phydev = emac->phydev;
+	bool gig_en = false, full_duplex = false;
+	struct prueth *prueth = emac->prueth;
+	int slice = prueth_emac_slice(emac);
+	bool new_state = false;
+	unsigned long flags;
+
+	if (phydev->link) {
+		/* check the mode of operation - full/half duplex */
+		if (phydev->duplex != emac->duplex) {
+			new_state = true;
+			emac->duplex = phydev->duplex;
+		}
+		if (phydev->speed != emac->speed) {
+			new_state = true;
+			emac->speed = phydev->speed;
+		}
+		if (!emac->link) {
+			new_state = true;
+			emac->link = 1;
+		}
+	} else if (emac->link) {
+		new_state = true;
+		emac->link = 0;
+		/* defaults for no link */
+
+		/* f/w should support 100 & 1000 */
+		emac->speed = SPEED_1000;
+
+		/* half duplex may not be supported by f/w */
+		emac->duplex = DUPLEX_FULL;
+	}
+
+	if (new_state) {
+		phy_print_status(phydev);
+
+		/* update RGMII and MII configuration based on PHY negotiated
+		 * values
+		 */
+		spin_lock_irqsave(&emac->lock, flags);
+		if (emac->link) {
+			if (phydev->speed == SPEED_1000)
+				gig_en = true;
+
+			if (phydev->duplex == DUPLEX_FULL)
+				full_duplex = true;
+
+			/* Set the RGMII cfg for gig en and full duplex */
+			icssg_update_rgmii_cfg(prueth->miig_rt, gig_en,
+					       full_duplex, slice);
+			/* update the Tx IPG based on 100M/1G speed */
+			icssg_update_mii_rt_cfg(prueth->mii_rt, emac->speed,
+						slice);
+		} else {
+			icssg_update_rgmii_cfg(prueth->miig_rt, true, true,
+					       slice);
+			icssg_update_mii_rt_cfg(prueth->mii_rt, emac->speed,
+						slice);
+		}
+		spin_unlock_irqrestore(&emac->lock, flags);
+
+		/* send command to firmware to change speed and duplex
+		 * setting when link is up.
+		 */
+		if (emac->link)
+			emac_change_port_speed_duplex(emac, full_duplex,
+						      emac->speed);
+	}
+
+	if (emac->link) {
+		/* link ON */
+		netif_carrier_on(ndev);
+		/* reactivate the transmit queue */
+		netif_tx_wake_all_queues(ndev);
+	} else {
+		/* link OFF */
+		netif_carrier_off(ndev);
+		netif_tx_stop_all_queues(ndev);
+	}
+}
+
+static int emac_napi_rx_poll(struct napi_struct *napi_rx, int budget)
+{
+	struct prueth_emac *emac = prueth_napi_to_emac(napi_rx);
+	int num_rx = 0;
+	int flow = PRUETH_MAX_RX_FLOWS;
+	int cur_budget;
+	int ret;
+
+	while (flow--) {
+		cur_budget = budget - num_rx;
+
+		while (cur_budget--) {
+			ret = emac_rx_packet(emac, flow);
+			if (ret)
+				break;
+			num_rx++;
+		}
+
+		if (num_rx >= budget)
+			break;
+	}
+
+	if (num_rx < budget) {
+		napi_complete(napi_rx);
+		enable_irq(emac->rx_chns.irq);
+	}
+
+	return num_rx;
+}
+
+static int emac_napi_tx_poll(struct napi_struct *napi_tx, int budget)
+{
+	struct prueth_emac *emac = prueth_napi_to_emac(napi_tx);
+	int num_tx_packets;
+
+	num_tx_packets = emac_tx_complete_packets(emac, budget);
+
+	if (num_tx_packets < budget) {
+		napi_complete(napi_tx);
+		enable_irq(emac->tx_chns.irq);
+	}
+
+	return num_tx_packets;
+}
+
+/**
+ * emac_ndo_open - EMAC device open
+ * @ndev: network adapter device
+ *
+ * Called when system wants to start the interface.
+ *
+ * Returns 0 for a successful open, or appropriate error code
+ */
+static int emac_ndo_open(struct net_device *ndev)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+	struct prueth *prueth = emac->prueth;
+	struct device *dev = prueth->dev;
+	int ret, i;
+	struct sk_buff *skb;
+	int slice = prueth_emac_slice(emac);
+
+	/* clear SMEM of this slice */
+	memset_io(prueth->shram.va + slice * ICSSG_CONFIG_OFFSET_SLICE1,
+		  0, ICSSG_CONFIG_OFFSET_SLICE1);
+	/* set h/w MAC as user might have re-configured */
+	ether_addr_copy(emac->mac_addr, ndev->dev_addr);
+
+	icssg_class_set_mac_addr(prueth->miig_rt, slice, emac->mac_addr);
+	icssg_class_default(prueth->miig_rt, slice, 0);
+
+	netif_carrier_off(ndev);
+
+	init_completion(&emac->cmd_complete);
+	ret = prueth_init_tx_chns(emac);
+	if (ret) {
+		dev_err(dev, "failed to init tx channel: %d\n", ret);
+		return ret;
+	}
+
+	ret = prueth_init_rx_chns(emac, &emac->rx_chns, "rx",
+				  PRUETH_MAX_RX_FLOWS, PRUETH_MAX_RX_DESC);
+	if (ret) {
+		dev_err(dev, "failed to init rx channel: %d\n", ret);
+		goto cleanup_tx;
+	}
+
+	ret = prueth_init_rx_chns(emac, &emac->rx_mgm_chn, "rxmgm",
+				  PRUETH_MAX_RX_MGM_FLOWS,
+				  PRUETH_MAX_RX_MGM_DESC);
+	if (ret) {
+		dev_err(dev, "failed to init rx management channel: %d\n", ret);
+		goto cleanup_rx;
+	}
+
+	ret = request_irq(emac->tx_chns.irq, prueth_tx_irq, 0,
+			  dev_name(dev), emac);
+	if (ret) {
+		dev_err(dev, "unable to request TX IRQ\n");
+		goto cleanup_rx_mgm;
+	}
+
+	ret = request_irq(emac->rx_chns.irq, prueth_rx_irq, 0,
+			  dev_name(dev), emac);
+	if (ret) {
+		dev_err(dev, "unable to request RX IRQ\n");
+		goto free_tx_irq;
+	}
+
+	ret = request_threaded_irq(emac->rx_mgm_chn.irq, NULL,
+				   prueth_rx_mgm_irq_thread, IRQF_ONESHOT,
+				   dev_name(dev), emac);
+	if (ret) {
+		dev_err(dev, "unable to request RX Management IRQ\n");
+		goto free_rx_irq;
+	}
+
+	/* reset and start PRU firmware */
+	ret = prueth_emac_start(prueth, emac);
+	if (ret)
+		goto free_rx_mgm_irq;
+
+	/* Get attached phy details */
+	phy_attached_info(emac->phydev);
+
+	/* start PHY */
+	phy_start(emac->phydev);
+
+	/* prepare RX & TX */
+	for (i = 0; i < emac->rx_chns.descs_num; i++) {
+		skb = __netdev_alloc_skb_ip_align(NULL,
+						  PRUETH_MAX_PKT_SIZE,
+						  GFP_KERNEL);
+		if (!skb) {
+			netdev_err(ndev, "cannot allocate skb\n");
+			ret = -ENOMEM;
+			goto err;
+		}
+
+		ret = prueth_dma_rx_push(emac, skb, &emac->rx_chns);
+		if (ret < 0) {
+			netdev_err(ndev, "cannot submit skb for rx: %d\n",
+				   ret);
+			kfree_skb(skb);
+			goto err;
+		}
+	}
+
+	for (i = 0; i < emac->rx_mgm_chn.descs_num; i++) {
+		skb = __netdev_alloc_skb_ip_align(NULL,
+						  64,
+						  GFP_KERNEL);
+		if (!skb) {
+			netdev_err(ndev, "cannot allocate skb\n");
+			ret = -ENOMEM;
+			goto err;
+		}
+
+		ret = prueth_dma_rx_push(emac, skb, &emac->rx_mgm_chn);
+		if (ret < 0) {
+			netdev_err(ndev, "cannot submit skb for rx_mgm: %d\n",
+				   ret);
+			kfree_skb(skb);
+			goto err;
+		}
+	}
+
+	k3_nav_udmax_enable_rx_chn(emac->rx_mgm_chn.rx_chn);
+	k3_nav_udmax_enable_rx_chn(emac->rx_chns.rx_chn);
+	k3_nav_udmax_enable_tx_chn(emac->tx_chns.tx_chn);
+
+	napi_enable(&emac->napi_tx);
+	napi_enable(&emac->napi_rx);
+
+	if (netif_msg_drv(emac))
+		dev_notice(&ndev->dev, "started\n");
+
+	return 0;
+
+err:
+	prueth_emac_stop(emac);
+free_rx_mgm_irq:
+	free_irq(emac->rx_mgm_chn.irq, emac);
+free_rx_irq:
+	free_irq(emac->rx_chns.irq, emac);
+free_tx_irq:
+	free_irq(emac->tx_chns.irq, emac);
+cleanup_rx_mgm:
+	prueth_cleanup_rx_chns(emac, &emac->rx_mgm_chn,
+			       PRUETH_MAX_RX_MGM_FLOWS);
+cleanup_rx:
+	prueth_cleanup_rx_chns(emac, &emac->rx_chns, PRUETH_MAX_RX_FLOWS);
+cleanup_tx:
+	prueth_cleanup_tx_chns(emac);
+
+	return ret;
+}
+
+/**
+ * emac_ndo_stop - EMAC device stop
+ * @ndev: network adapter device
+ *
+ * Called when system wants to stop or down the interface.
+ */
+static int emac_ndo_stop(struct net_device *ndev)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+	struct prueth *prueth = emac->prueth;
+	int ret, i;
+
+	/* inform the upper layers. */
+	netif_stop_queue(ndev);
+
+	/* block packets from wire */
+	phy_stop(emac->phydev);
+	icssg_class_disable(prueth->miig_rt, prueth_emac_slice(emac));
+
+	/* send shutdown command */
+	emac_shutdown(ndev);
+
+	/* tear down and disable UDMA channels */
+	reinit_completion(&emac->tdown_complete);
+	k3_nav_udmax_tdown_tx_chn(emac->tx_chns.tx_chn, false);
+	ret = wait_for_completion_timeout(&emac->tdown_complete,
+			msecs_to_jiffies(1000));
+	if (!ret)
+		netdev_err(ndev, "tx teardown timeout\n");
+
+	k3_nav_udmax_reset_tx_chn(emac->tx_chns.tx_chn,
+				  emac,
+				  prueth_tx_cleanup);
+	k3_nav_udmax_disable_tx_chn(emac->tx_chns.tx_chn);
+
+	k3_nav_udmax_tdown_rx_chn(emac->rx_chns.rx_chn, true);
+	for (i = 0; i < PRUETH_MAX_RX_FLOWS; i++)
+		k3_nav_udmax_reset_rx_chn(emac->rx_chns.rx_chn, i,
+					  &emac->rx_chns,
+					  prueth_rx_cleanup, !!i);
+
+	k3_nav_udmax_disable_rx_chn(emac->rx_chns.rx_chn);
+
+	/* Teardown RX MGM channel */
+	k3_nav_udmax_tdown_rx_chn(emac->rx_mgm_chn.rx_chn, true);
+	for (i = 0; i < PRUETH_MAX_RX_MGM_FLOWS; i++)
+		k3_nav_udmax_reset_rx_chn(emac->rx_mgm_chn.rx_chn, i,
+					  &emac->rx_mgm_chn,
+					  prueth_rx_cleanup, !!i);
+
+	k3_nav_udmax_disable_rx_chn(emac->rx_mgm_chn.rx_chn);
+
+	napi_disable(&emac->napi_tx);
+	napi_disable(&emac->napi_rx);
+
+	/* stop PRUs */
+	prueth_emac_stop(emac);
+
+	free_irq(emac->rx_mgm_chn.irq, emac);
+	free_irq(emac->rx_chns.irq, emac);
+	free_irq(emac->tx_chns.irq, emac);
+
+	prueth_cleanup_rx_chns(emac, &emac->rx_mgm_chn,
+			       PRUETH_MAX_RX_MGM_FLOWS);
+	prueth_cleanup_rx_chns(emac, &emac->rx_chns, PRUETH_MAX_RX_FLOWS);
+	prueth_cleanup_tx_chns(emac);
+
+	if (netif_msg_drv(emac))
+		dev_notice(&ndev->dev, "stopped\n");
+
+	return 0;
+}
+
+/**
+ * emac_ndo_tx_timeout - EMAC Transmit timeout function
+ * @ndev: The EMAC network adapter
+ *
+ * Called when system detects that a skb timeout period has expired
+ * potentially due to a fault in the adapter in not being able to send
+ * it out on the wire.
+ */
+static void emac_ndo_tx_timeout(struct net_device *ndev)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+
+	if (netif_msg_tx_err(emac))
+		netdev_err(ndev, "xmit timeout");
+
+	ndev->stats.tx_errors++;
+
+	/* TODO: can we recover or need to reboot firmware? */
+}
+
+/**
+ * emac_ndo_set_rx_mode - EMAC set receive mode function
+ * @ndev: The EMAC network adapter
+ *
+ * Called when system wants to set the receive mode of the device.
+ *
+ */
+static void emac_ndo_set_rx_mode(struct net_device *ndev)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+	struct prueth *prueth = emac->prueth;
+	int slice = prueth_emac_slice(emac);
+	bool promisc = ndev->flags & IFF_PROMISC;
+	bool allmulti = ndev->flags & IFF_ALLMULTI;
+
+	if (promisc) {
+		icssg_class_promiscuous(prueth->miig_rt, slice);
+		return;
+	}
+
+	if (allmulti) {
+		icssg_class_default(prueth->miig_rt, slice, 1);
+		return;
+	}
+
+	icssg_class_default(prueth->miig_rt, slice, 0);
+	if (!netdev_mc_empty(ndev)) {
+		/* program multicast address list into Classifier */
+		icssg_class_add_mcast(prueth->miig_rt, slice, ndev);
+		return;
+	}
+}
+
+static int emac_set_timestamp_mode(struct prueth_emac *emac,
+				   struct hwtstamp_config *config)
+{
+	/* reserved for future extensions */
+	if (config->flags)
+		return -EINVAL;
+
+	switch (config->tx_type) {
+	case HWTSTAMP_TX_OFF:
+		emac->tx_ts_enabled = 0;
+		break;
+	case HWTSTAMP_TX_ON:
+		emac->tx_ts_enabled = 1;
+		break;
+	default:
+		return -ERANGE;
+	}
+
+	switch (config->rx_filter) {
+	case HWTSTAMP_FILTER_NONE:
+		emac->rx_ts_enabled = 0;
+		break;
+	case HWTSTAMP_FILTER_ALL:
+		emac->rx_ts_enabled = 1;
+		break;
+	default:
+		emac->rx_ts_enabled = 1;
+	}
+
+	return 0;
+}
+
+static int emac_set_ts_config(struct net_device *ndev, struct ifreq *ifr)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+	struct hwtstamp_config config;
+	int ret;
+
+	if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
+		return -EFAULT;
+
+	ret = emac_set_timestamp_mode(emac, &config);
+	if (ret)
+		return ret;
+
+	/* save these settings for future reference */
+	memcpy(&emac->tstamp_config, &config,
+	       sizeof(emac->tstamp_config));
+
+	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
+		-EFAULT : 0;
+}
+
+static int emac_get_ts_config(struct net_device *ndev, struct ifreq *ifr)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+	struct hwtstamp_config *config = &emac->tstamp_config;
+
+	return copy_to_user(ifr->ifr_data, config, sizeof(*config)) ?
+			    -EFAULT : 0;
+}
+
+static int emac_ndo_ioctl(struct net_device *ndev, struct ifreq *ifr, int cmd)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+
+	switch (cmd) {
+	case SIOCGHWTSTAMP:
+		return emac_get_ts_config(ndev, ifr);
+	case SIOCSHWTSTAMP:
+		return emac_set_ts_config(ndev, ifr);
+	default:
+		break;
+	}
+
+	return phy_mii_ioctl(emac->phydev, ifr, cmd);
+}
+
+static const struct net_device_ops emac_netdev_ops = {
+	.ndo_open = emac_ndo_open,
+	.ndo_stop = emac_ndo_stop,
+	.ndo_start_xmit = emac_ndo_start_xmit,
+	.ndo_set_mac_address = eth_mac_addr,
+	.ndo_validate_addr = eth_validate_addr,
+	.ndo_change_mtu	= eth_change_mtu,
+	.ndo_tx_timeout = emac_ndo_tx_timeout,
+	.ndo_set_rx_mode = emac_ndo_set_rx_mode,
+	.ndo_do_ioctl = emac_ndo_ioctl,
+};
+
+/* get emac_port corresponding to eth_node name */
+static int prueth_node_port(struct device_node *eth_node)
+{
+	if (!strcmp(eth_node->name, "ethernet-mii0"))
+		return PRUETH_PORT_MII0;
+	else if (!strcmp(eth_node->name, "ethernet-mii1"))
+		return PRUETH_PORT_MII1;
+	else
+		return -EINVAL;
+}
+
+/* get MAC instance corresponding to eth_node name */
+static int prueth_node_mac(struct device_node *eth_node)
+{
+	if (!strcmp(eth_node->name, "ethernet-mii0"))
+		return PRUETH_MAC0;
+	else if (!strcmp(eth_node->name, "ethernet-mii1"))
+		return PRUETH_MAC1;
+	else
+		return -EINVAL;
+}
+
+extern const struct ethtool_ops icssg_ethtool_ops;
+
+static int prueth_netdev_init(struct prueth *prueth,
+			      struct device_node *eth_node)
+{
+	enum prueth_port port;
+	enum prueth_mac mac;
+	struct net_device *ndev;
+	struct prueth_emac *emac;
+	const u8 *mac_addr;
+	int ret;
+	u32 refclk_freq;
+	struct regmap *iep_map;
+
+	port = prueth_node_port(eth_node);
+	if (port < 0)
+		return -EINVAL;
+
+	mac = prueth_node_mac(eth_node);
+	if (mac < 0)
+		return -EINVAL;
+
+	ndev = alloc_etherdev(sizeof(*emac));
+	if (!ndev)
+		return -ENOMEM;
+
+	emac = netdev_priv(ndev);
+	iep_map = syscon_regmap_lookup_by_phandle(eth_node, "iep");
+	if (IS_ERR(iep_map)) {
+		ret = PTR_ERR(iep_map);
+		if (ret != -EPROBE_DEFER)
+			dev_err(prueth->dev, "couldn't get iep regmap\n");
+		goto free;
+	}
+
+	/* Firmware sets IEP clock to Vbus clk (250MHz) using internal mux.
+	 * see AM65 TRM "Figure 6-113. PRU_ICSSG CORE Clock Diagram"
+	 */
+	refclk_freq = 250e6;
+
+	SET_NETDEV_DEV(ndev, prueth->dev);
+	prueth->emac[mac] = emac;
+	emac->prueth = prueth;
+	emac->ndev = ndev;
+	emac->port_id = port;
+	emac->msg_enable = netif_msg_init(debug_level, PRUETH_EMAC_DEBUG);
+	spin_lock_init(&emac->lock);
+	mutex_init(&emac->cmd_lock);
+
+	emac->phy_node = of_parse_phandle(eth_node, "phy-handle", 0);
+	if (!emac->phy_node) {
+		dev_err(prueth->dev, "couldn't find phy-handle\n");
+		ret = -ENODEV;
+		goto free;
+	}
+
+	if (of_phy_is_fixed_link(emac->phy_node)) {
+		ret = of_phy_register_fixed_link(emac->phy_node);
+		if (ret) {
+			if (ret != -EPROBE_DEFER) {
+				dev_err(prueth->dev,
+					"failed to register fixed-link phy: %d\n",
+					ret);
+			}
+
+			goto free;
+		}
+	}
+
+	emac->phy_if = of_get_phy_mode(eth_node);
+	if (emac->phy_if < 0) {
+		dev_err(prueth->dev, "could not get phy-mode property\n");
+		ret = emac->phy_if;
+		goto free;
+	}
+
+	/* connect PHY */
+	emac->phydev = of_phy_connect(ndev, emac->phy_node,
+				      &emac_adjust_link, 0, emac->phy_if);
+	if (!emac->phydev) {
+		dev_dbg(prueth->dev, "couldn't connect to phy %s\n",
+			emac->phy_node->full_name);
+		ret = -EPROBE_DEFER;
+		goto free;
+	}
+
+	/* remove unsupported modes */
+	emac->phydev->supported &= ~(PHY_10BT_FEATURES |
+				     SUPPORTED_100baseT_Half |
+				     SUPPORTED_1000baseT_Half |
+				     SUPPORTED_Pause |
+				     SUPPORTED_Asym_Pause);
+	emac->phydev->advertising = emac->phydev->supported;
+
+	/* get mac address from DT and set private and netdev addr */
+	mac_addr = of_get_mac_address(eth_node);
+	if (mac_addr)
+		ether_addr_copy(ndev->dev_addr, mac_addr);
+	if (!is_valid_ether_addr(ndev->dev_addr)) {
+		eth_hw_addr_random(ndev);
+		dev_warn(prueth->dev, "port %d: using random MAC addr: %pM\n",
+			 port, ndev->dev_addr);
+	}
+	ether_addr_copy(emac->mac_addr, ndev->dev_addr);
+
+	ndev->netdev_ops = &emac_netdev_ops;
+	ndev->ethtool_ops = &icssg_ethtool_ops;
+
+	ret = icssg_iep_init(&emac->iep, prueth->dev, iep_map, refclk_freq);
+	if (ret)
+		goto free;
+
+	netif_tx_napi_add(ndev, &emac->napi_tx,
+			  emac_napi_tx_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(ndev, &emac->napi_rx,
+		       emac_napi_rx_poll, NAPI_POLL_WEIGHT);
+
+	return 0;
+
+free:
+	free_netdev(ndev);
+	prueth->emac[mac] = NULL;
+
+	return ret;
+}
+
+static void prueth_netdev_exit(struct prueth *prueth,
+			       struct device_node *eth_node)
+{
+	struct prueth_emac *emac;
+	enum prueth_mac mac;
+
+	mac = prueth_node_mac(eth_node);
+	if (mac < 0)
+		return;
+
+	emac = prueth->emac[mac];
+	if (!emac)
+		return;
+
+	phy_disconnect(emac->phydev);
+
+	if (of_phy_is_fixed_link(emac->phy_node))
+		of_phy_deregister_fixed_link(emac->phy_node);
+
+	netif_napi_del(&emac->napi_rx);
+	netif_napi_del(&emac->napi_tx);
+	icssg_iep_exit(&emac->iep);
+	free_netdev(emac->ndev);
+	prueth->emac[mac] = NULL;
+}
+
+static int prueth_get_cores(struct prueth *prueth, int slice)
+{
+	struct device *dev = prueth->dev;
+	struct device_node *np = dev->of_node;
+	int pru, rtu, ret;
+
+	switch (slice) {
+	case ICSS_SLICE0:
+		pru = 0;
+		rtu = 1;
+		break;
+	case ICSS_SLICE1:
+		pru = 2;
+		rtu = 3;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	prueth->pru[slice] = pru_rproc_get(np, pru);
+	if (IS_ERR(prueth->pru[slice])) {
+		ret = PTR_ERR(prueth->pru[slice]);
+		prueth->pru[slice] = NULL;
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "unable to get PRU%d: %d\n", slice, ret);
+		return ret;
+	}
+
+	prueth->rtu[slice] = pru_rproc_get(np, rtu);
+	if (IS_ERR(prueth->rtu[slice])) {
+		ret = PTR_ERR(prueth->rtu[slice]);
+		prueth->rtu[slice] = NULL;
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "unable to get RTU%d: %d\n", slice, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void prueth_put_cores(struct prueth *prueth, int slice)
+{
+	if (prueth->rtu[slice])
+		pru_rproc_put(prueth->rtu[slice]);
+
+	if (prueth->pru[slice])
+		pru_rproc_put(prueth->pru[slice]);
+}
+
+static int prueth_config_rgmiidelay(struct prueth *prueth,
+				    struct device_node *eth_np)
+{
+	struct device *dev = prueth->dev;
+	struct regmap *ctrl_mmr;
+	u32 icssgctrl;
+	struct device_node *np = dev->of_node;
+
+	if (!of_device_is_compatible(np, "ti,am654-icssg-prueth"))
+		return 0;
+
+	ctrl_mmr = syscon_regmap_lookup_by_phandle(eth_np, "syscon-rgmii-delay");
+	if (IS_ERR(ctrl_mmr)) {
+		dev_err(dev, "couldn't get syscon-rgmii-delay\n");
+		return -ENODEV;
+	}
+
+	if (of_property_read_u32_index(eth_np, "syscon-rgmii-delay", 1,
+				       &icssgctrl)) {
+		dev_err(dev, "couldn't get rgmii-delay reg. offset\n");
+		return -ENODEV;
+	}
+
+	regmap_update_bits(ctrl_mmr, icssgctrl, ICSSG_CTRL_RGMII_ID_MODE, 0);
+
+	return 0;
+}
+
+static const struct of_device_id prueth_dt_match[];
+
+static int prueth_probe(struct platform_device *pdev)
+{
+	struct prueth *prueth;
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct device_node *eth0_node, *eth1_node;
+	const struct of_device_id *match;
+	struct pruss *pruss;
+	int i, ret;
+
+	if (!np)
+		return -ENODEV;	/* we don't support non DT */
+
+	match = of_match_device(prueth_dt_match, dev);
+	if (!match)
+		return -ENODEV;
+
+	prueth = devm_kzalloc(dev, sizeof(*prueth), GFP_KERNEL);
+	if (!prueth)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, prueth);
+
+	prueth->dev = dev;
+	eth0_node = of_get_child_by_name(np, "ethernet-mii0");
+	if (!of_device_is_available(eth0_node)) {
+		of_node_put(eth0_node);
+		eth0_node = NULL;
+	}
+
+	eth1_node = of_get_child_by_name(np, "ethernet-mii1");
+	if (!of_device_is_available(eth1_node)) {
+		of_node_put(eth1_node);
+		eth1_node = NULL;
+	}
+
+	/* At least one node must be present and available else we fail */
+	if (!eth0_node && !eth1_node) {
+		dev_err(dev, "neither ethernet-mii0 nor ethernet-mii1 node available\n");
+		return -ENODEV;
+	}
+
+	prueth->eth_node[PRUETH_MAC0] = eth0_node;
+	prueth->eth_node[PRUETH_MAC1] = eth1_node;
+
+	prueth->miig_rt = syscon_regmap_lookup_by_phandle(np, "mii-g-rt");
+	if (IS_ERR(prueth->miig_rt)) {
+		dev_err(dev, "couldn't get mii-g-rt syscon regmap\n");
+		return -ENODEV;
+	}
+
+	prueth->mii_rt = syscon_regmap_lookup_by_phandle(np, "mii-rt");
+	if (IS_ERR(prueth->mii_rt)) {
+		dev_err(dev, "couldn't get mii-rt syscon regmap\n");
+		return -ENODEV;
+	}
+
+	if (eth0_node) {
+		ret = prueth_config_rgmiidelay(prueth, eth0_node);
+		if (ret)
+			goto put_cores;
+
+		ret = prueth_get_cores(prueth, ICSS_SLICE0);
+		if (ret)
+			goto put_cores;
+	}
+
+	if (eth1_node) {
+		ret = prueth_config_rgmiidelay(prueth, eth1_node);
+		if (ret)
+			goto put_cores;
+
+		ret = prueth_get_cores(prueth, ICSS_SLICE1);
+		if (ret)
+			goto put_cores;
+	}
+
+	pruss = pruss_get(eth0_node ?
+			  prueth->pru[ICSS_SLICE0] : prueth->pru[ICSS_SLICE1]);
+	if (IS_ERR(pruss)) {
+		ret = PTR_ERR(pruss);
+		dev_err(dev, "unable to get pruss handle\n");
+		goto put_cores;
+	}
+
+	prueth->pruss = pruss;
+
+	ret = pruss_request_mem_region(pruss, PRUSS_MEM_SHRD_RAM2,
+				       &prueth->shram);
+	if (ret) {
+		dev_err(dev, "unable to get PRUSS SHRD RAM2: %d\n", ret);
+		goto put_mem;
+	}
+
+	prueth->sram_pool = of_gen_pool_get(np, "sram", 0);
+	if (!prueth->sram_pool) {
+		dev_err(dev, "unable to get SRAM pool\n");
+		ret = -ENODEV;
+
+		goto put_mem;
+	}
+	prueth->msmcram.va =
+			(void __iomem *)gen_pool_alloc(prueth->sram_pool,
+						       MSMC_RAM_SIZE);
+	if (!prueth->msmcram.va) {
+		ret = -ENOMEM;
+		dev_err(dev, "unable to allocate MSMC resource\n");
+		goto put_mem;
+	}
+	prueth->msmcram.pa = gen_pool_virt_to_phys(prueth->sram_pool,
+						   (unsigned long)prueth->msmcram.va);
+	prueth->msmcram.size = MSMC_RAM_SIZE;
+	dev_dbg(dev, "sram: pa %llx va %p size %zx\n", prueth->msmcram.pa,
+		prueth->msmcram.va, prueth->msmcram.size);
+
+	/* setup netdev interfaces */
+	if (eth0_node) {
+		ret = prueth_netdev_init(prueth, eth0_node);
+		if (ret) {
+			if (ret != -EPROBE_DEFER) {
+				dev_err(dev, "netdev init %s failed: %d\n",
+					eth0_node->name, ret);
+			}
+			goto free_pool;
+		}
+	}
+
+	if (eth1_node) {
+		ret = prueth_netdev_init(prueth, eth1_node);
+		if (ret) {
+			if (ret != -EPROBE_DEFER) {
+				dev_err(dev, "netdev init %s failed: %d\n",
+					eth1_node->name, ret);
+			}
+			goto netdev_exit;
+		}
+	}
+
+	/* register the network devices */
+	if (eth0_node) {
+		ret = register_netdev(prueth->emac[PRUETH_MAC0]->ndev);
+		if (ret) {
+			dev_err(dev, "can't register netdev for port MII0");
+			goto netdev_exit;
+		}
+
+		prueth->registered_netdevs[PRUETH_MAC0] = prueth->emac[PRUETH_MAC0]->ndev;
+	}
+
+	if (eth1_node) {
+		ret = register_netdev(prueth->emac[PRUETH_MAC1]->ndev);
+		if (ret) {
+			dev_err(dev, "can't register netdev for port MII1");
+			goto netdev_unregister;
+		}
+
+		prueth->registered_netdevs[PRUETH_MAC1] = prueth->emac[PRUETH_MAC1]->ndev;
+	}
+
+	dev_info(dev, "TI PRU ethernet driver initialized: %s EMAC mode\n",
+		 (!eth0_node || !eth1_node) ? "single" : "dual");
+
+	if (eth1_node)
+		of_node_put(eth1_node);
+	if (eth0_node)
+		of_node_put(eth0_node);
+
+	return 0;
+
+netdev_unregister:
+	for (i = 0; i < PRUETH_NUM_MACS; i++) {
+		if (!prueth->registered_netdevs[i])
+			continue;
+		unregister_netdev(prueth->registered_netdevs[i]);
+	}
+
+netdev_exit:
+	for (i = 0; i < PRUETH_NUM_MACS; i++) {
+		struct device_node *eth_node;
+
+		eth_node = prueth->eth_node[i];
+		if (!eth_node)
+			continue;
+
+		prueth_netdev_exit(prueth, eth_node);
+	}
+
+free_pool:
+	gen_pool_free(prueth->sram_pool,
+		      (unsigned long)prueth->msmcram.va, MSMC_RAM_SIZE);
+
+put_mem:
+	pruss_release_mem_region(prueth->pruss, &prueth->shram);
+	pruss_put(prueth->pruss);
+
+put_cores:
+	if (eth1_node) {
+		prueth_put_cores(prueth, ICSS_SLICE1);
+		of_node_put(eth1_node);
+	}
+
+	if (eth0_node) {
+		prueth_put_cores(prueth, ICSS_SLICE0);
+		of_node_put(eth0_node);
+	}
+
+	return ret;
+}
+
+static int prueth_remove(struct platform_device *pdev)
+{
+	struct device_node *eth_node;
+	struct prueth *prueth = platform_get_drvdata(pdev);
+	int i;
+
+	for (i = 0; i < PRUETH_NUM_MACS; i++) {
+		if (!prueth->registered_netdevs[i])
+			continue;
+		unregister_netdev(prueth->registered_netdevs[i]);
+	}
+
+	for (i = 0; i < PRUETH_NUM_MACS; i++) {
+		eth_node = prueth->eth_node[i];
+		if (!eth_node)
+			continue;
+
+		prueth_netdev_exit(prueth, eth_node);
+	}
+
+	gen_pool_free(prueth->sram_pool,
+		      (unsigned long)prueth->msmcram.va,
+		      MSMC_RAM_SIZE);
+
+	pruss_release_mem_region(prueth->pruss, &prueth->shram);
+
+	pruss_put(prueth->pruss);
+
+	if (prueth->eth_node[PRUETH_MAC1])
+		prueth_put_cores(prueth, ICSS_SLICE1);
+
+	if (prueth->eth_node[PRUETH_MAC0])
+		prueth_put_cores(prueth, ICSS_SLICE0);
+
+	return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int prueth_suspend(struct device *dev)
+{
+	struct prueth *prueth = dev_get_drvdata(dev);
+	struct net_device *ndev;
+	int i, ret;
+
+	for (i = 0; i < PRUETH_NUM_MACS; i++) {
+		ndev = prueth->registered_netdevs[i];
+
+		if (!ndev)
+			continue;
+
+		if (netif_running(ndev)) {
+			netif_device_detach(ndev);
+			ret = emac_ndo_stop(ndev);
+			if (ret < 0) {
+				netdev_err(ndev, "failed to stop: %d", ret);
+				return ret;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static int prueth_resume(struct device *dev)
+{
+	struct prueth *prueth = dev_get_drvdata(dev);
+	struct net_device *ndev;
+	int i, ret;
+
+	for (i = 0; i < PRUETH_NUM_MACS; i++) {
+		ndev = prueth->registered_netdevs[i];
+
+		if (!ndev)
+			continue;
+
+		if (netif_running(ndev)) {
+			ret = emac_ndo_open(ndev);
+			if (ret < 0) {
+				netdev_err(ndev, "failed to start: %d", ret);
+				return ret;
+			}
+			netif_device_attach(ndev);
+		}
+	}
+
+	return 0;
+}
+#endif /* CONFIG_PM_SLEEP */
+
+static const struct dev_pm_ops prueth_dev_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(prueth_suspend, prueth_resume)
+};
+
+static const struct of_device_id prueth_dt_match[] = {
+	{ .compatible = "ti,am654-icssg-prueth", },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, prueth_dt_match);
+
+static struct platform_driver prueth_driver = {
+	.probe = prueth_probe,
+	.remove = prueth_remove,
+	.driver = {
+		.name = "icssg-prueth",
+		.of_match_table = prueth_dt_match,
+		.pm = &prueth_dev_pm_ops,
+	},
+};
+module_platform_driver(prueth_driver);
+
+MODULE_AUTHOR("Roger Quadros <rogerq@ti.com>");
+MODULE_DESCRIPTION("PRUSS ICSSG Ethernet Driver");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/net/ethernet/ti/icssg_prueth.h b/drivers/net/ethernet/ti/icssg_prueth.h
--- a/drivers/net/ethernet/ti/icssg_prueth.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/net/ethernet/ti/icssg_prueth.h	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,214 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Texas Instruments ICSSG Ethernet driver
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ */
+
+#ifndef __NET_TI_ICSSG_PRUETH_H
+#define __NET_TI_ICSSG_PRUETH_H
+
+#include <linux/etherdevice.h>
+#include <linux/genalloc.h>
+#include <linux/if_vlan.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_irq.h>
+#include <linux/of_mdio.h>
+#include <linux/of_net.h>
+#include <linux/of_platform.h>
+#include <linux/mfd/syscon.h>
+#include <linux/mutex.h>
+#include <linux/net_tstamp.h>
+#include <linux/phy.h>
+#include <linux/pruss.h>
+#include <linux/ptp_clock_kernel.h>
+#include <linux/remoteproc.h>
+
+#include <linux/dma-mapping.h>
+#include <linux/dma/ti-cppi5.h>
+#include <linux/dma/k3-navss-udma.h>
+
+#include "icssg_config.h"
+#include "icssg_iep.h"
+
+#define ICSS_SLICE0	0
+#define ICSS_SLICE1	1
+
+#define ICSS_FW_PRU	0
+#define ICSS_FW_RTU	1
+
+/* Firmware status codes */
+#define ICSS_HS_FW_READY 0x55555555
+#define ICSS_HS_FW_DEAD 0xDEAD0000	/* lower 16 bits contain error code */
+
+/* Firmware command codes */
+#define ICSS_HS_CMD_BUSY 0x40000000
+#define ICSS_HS_CMD_DONE 0x80000000
+#define ICSS_HS_CMD_CANCEL 0x10000000
+
+/* Firmware commands */
+#define ICSS_CMD_SPAD 0x20
+#define ICSS_CMD_RXTX 0x10
+#define ICSS_CMD_ADD_FDB 0x1
+#define ICSS_CMD_SET_RUN 0x4
+#define ICSS_CMD_ENABLE_VLAN 0x5
+#define ICSS_CMD_DISABLE_VLAN 0x6
+#define ICSS_CMD_ADD_FILTER 0x7
+#define ICSS_CMD_ADD_MAC 0x8
+
+/* Firmware flags */
+#define ICSS_SET_RUN_FLAG_VLAN_ENABLE		BIT(0)	/* switch only */
+#define ICSS_SET_RUN_FLAG_FLOOD_UNICAST		BIT(1)	/* switch only */
+#define ICSS_SET_RUN_FLAG_PROMISC		BIT(2)	/* MAC only */
+#define ICSS_SET_RUN_FLAG_MULTICAST_PROMISC	BIT(3)	/* MAC only */
+
+/* In switch mode there are 3 real ports i.e. 3 mac addrs.
+ * however Linux sees only the host side port. The other 2 ports
+ * are the switch ports.
+ * In emac mode there are 2 real ports i.e. 2 mac addrs.
+ * Linux sees both the ports.
+ */
+enum prueth_port {
+	PRUETH_PORT_HOST = 0,	/* host side port */
+	PRUETH_PORT_MII0,	/* physical port RG/SG MII 0 */
+	PRUETH_PORT_MII1,	/* physical port RG/SG MII 1 */
+};
+
+enum prueth_mac {
+	PRUETH_MAC0 = 0,
+	PRUETH_MAC1,
+	PRUETH_NUM_MACS,
+};
+
+struct prueth_tx_chn {
+	struct k3_knav_desc_pool *desc_pool;
+	struct k3_nav_udmax_tx_channel *tx_chn;
+	u32 descs_num;
+	spinlock_t lock;	/* to serialize */
+	unsigned int irq;
+};
+
+struct prueth_rx_chn {
+	struct device *dev;
+	struct k3_knav_desc_pool *desc_pool;
+	struct k3_nav_udmax_rx_channel *rx_chn;
+	u32 descs_num;
+	spinlock_t lock;	/* to serialize */
+	unsigned int irq;
+};
+
+enum prueth_state_flags {
+	__STATE_TX_TS_IN_PROGRESS,
+};
+
+/* data for each emac port */
+struct prueth_emac {
+	struct prueth *prueth;
+	struct net_device *ndev;
+	u8 mac_addr[6];
+	struct napi_struct napi_tx;
+	struct napi_struct napi_rx;
+	u32 msg_enable;
+
+	int link;
+	int speed;
+	int duplex;
+
+	const char *phy_id;
+	struct device_node *phy_node;
+	int phy_if;
+	struct phy_device *phydev;
+	enum prueth_port port_id;
+	struct icssg_iep iep;
+	struct hwtstamp_config tstamp_config;
+	unsigned int rx_ts_enabled : 1;
+	unsigned int tx_ts_enabled : 1;
+
+	/* DMA related */
+	struct prueth_tx_chn tx_chns;
+	struct completion tdown_complete;
+	struct prueth_rx_chn rx_chns;
+	int rx_flow_id_base;
+	struct prueth_rx_chn rx_mgm_chn;
+	int rx_mgm_flow_id_base;
+
+	spinlock_t lock;	/* serialize access */
+
+	/* TX HW Timestamping */
+	u32 tx_ts_cookie;
+	struct sk_buff *tx_ts_skb;
+	unsigned long state;
+
+	/* shutdown related */
+	u32 cmd_data[4];
+	struct completion cmd_complete;
+	/* Mutex to serialize access to firmware command interface */
+	struct mutex cmd_lock;
+};
+
+/**
+ * struct prueth - PRUeth structure
+ * @dev: device
+ * @pruss: pruss handle
+ * @pru0: rproc instance of PRUs
+ * @rtu0: rproc instance of RTUs
+ * @shram: PRUSS shared RAM region
+ * @sram_pool: MSMC RAM pool for buffers
+ * @msmcram: MSMC RAM region
+ * @eth_node: DT node for the port
+ * @emac: private EMAC data structure
+ * @registered_netdevs: list of registered netdevs
+ * @fw_data: firmware names to be used with PRU remoteprocs
+ * @config: firmware load time configuration per slice
+ * @miig_rt: regmap to mii_g_rt block
+ */
+struct prueth {
+	struct device *dev;
+	struct pruss *pruss;
+	struct rproc *pru[PRUSS_NUM_PRUS];
+	struct rproc *rtu[PRUSS_NUM_PRUS];
+	struct pruss_mem_region shram;
+	struct gen_pool *sram_pool;
+	struct pruss_mem_region msmcram;
+
+	struct device_node *eth_node[PRUETH_NUM_MACS];
+	struct prueth_emac *emac[PRUETH_NUM_MACS];
+	struct net_device *registered_netdevs[PRUETH_NUM_MACS];
+	const struct prueth_private_data *fw_data;
+	struct icssg_config config[PRUSS_NUM_PRUS];
+	struct regmap *miig_rt;
+	struct regmap *mii_rt;
+};
+
+struct emac_tx_ts_response {
+	u32 lo_ts;
+	u32 hi_ts;
+	u32 reserved;
+	u32 cookie;
+};
+
+/* Classifier helpers */
+void icssg_class_set_mac_addr(struct regmap *miig_rt, int slice, u8 *mac);
+void icssg_class_disable(struct regmap *miig_rt, int slice);
+void icssg_class_default(struct regmap *miig_rt, int slice, bool allmulti);
+void icssg_class_promiscuous(struct regmap *miig_rt, int slice);
+void icssg_class_add_mcast(struct regmap *miig_rt, int slice,
+			   struct net_device *ndev);
+
+
+/* get PRUSS SLICE number from prueth_emac */
+static inline int prueth_emac_slice(struct prueth_emac *emac)
+{
+	switch (emac->port_id) {
+	case PRUETH_PORT_MII0:
+		return ICSS_SLICE0;
+	case PRUETH_PORT_MII1:
+		return ICSS_SLICE1;
+	default:
+		return -EINVAL;
+	}
+}
+#endif /* __NET_TI_ICSSG_PRUETH_H */
diff -Naur --no-dereference a/drivers/net/ethernet/ti/icss_mii_rt.h b/drivers/net/ethernet/ti/icss_mii_rt.h
--- a/drivers/net/ethernet/ti/icss_mii_rt.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/net/ethernet/ti/icss_mii_rt.h	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,190 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+/* PRU-ICSS MII_RT register definitions
+ *
+ * Copyright (C) 2015-2018 Texas Instruments Incorporated - http://www.ti.com
+ */
+
+#ifndef __NET_PRUSS_MII_RT_H__
+#define __NET_PRUSS_MII_RT_H__
+
+/* PRUSS_MII_RT Registers */
+#define PRUSS_MII_RT_RXCFG0		0x0
+#define PRUSS_MII_RT_RXCFG1		0x4
+#define PRUSS_MII_RT_TXCFG0		0x10
+#define PRUSS_MII_RT_TXCFG1		0x14
+#define PRUSS_MII_RT_TX_CRC0		0x20
+#define PRUSS_MII_RT_TX_CRC1		0x24
+#define PRUSS_MII_RT_TX_IPG0		0x30
+#define PRUSS_MII_RT_TX_IPG1		0x34
+#define PRUSS_MII_RT_PRS0		0x38
+#define PRUSS_MII_RT_PRS1		0x3c
+#define PRUSS_MII_RT_RX_FRMS0		0x40
+#define PRUSS_MII_RT_RX_FRMS1		0x44
+#define PRUSS_MII_RT_RX_PCNT0		0x48
+#define PRUSS_MII_RT_RX_PCNT1		0x4c
+#define PRUSS_MII_RT_RX_ERR0		0x50
+#define PRUSS_MII_RT_RX_ERR1		0x54
+
+/* PRUSS_MII_RT_RXCFG0/1 bits */
+#define PRUSS_MII_RT_RXCFG_RX_ENABLE		BIT(0)
+#define PRUSS_MII_RT_RXCFG_RX_DATA_RDY_MODE_DIS	BIT(1)
+#define PRUSS_MII_RT_RXCFG_RX_CUT_PREAMBLE	BIT(2)
+#define PRUSS_MII_RT_RXCFG_RX_MUX_SEL		BIT(3)
+#define PRUSS_MII_RT_RXCFG_RX_L2_EN		BIT(4)
+#define PRUSS_MII_RT_RXCFG_RX_BYTE_SWAP		BIT(5)
+#define PRUSS_MII_RT_RXCFG_RX_AUTO_FWD_PRE	BIT(6)
+#define PRUSS_MII_RT_RXCFG_RX_L2_EOF_SCLR_DIS	BIT(9)
+
+/* PRUSS_MII_RT_TXCFG0/1 bits */
+#define PRUSS_MII_RT_TXCFG_TX_ENABLE		BIT(0)
+#define PRUSS_MII_RT_TXCFG_TX_AUTO_PREAMBLE	BIT(1)
+#define PRUSS_MII_RT_TXCFG_TX_EN_MODE		BIT(2)
+#define PRUSS_MII_RT_TXCFG_TX_BYTE_SWAP		BIT(3)
+#define PRUSS_MII_RT_TXCFG_TX_MUX_SEL		BIT(8)
+#define PRUSS_MII_RT_TXCFG_PRE_TX_AUTO_SEQUENCE	BIT(9)
+#define PRUSS_MII_RT_TXCFG_PRE_TX_AUTO_ESC_ERR	BIT(10)
+#define PRUSS_MII_RT_TXCFG_TX_32_MODE_EN	BIT(11)
+
+#define PRUSS_MII_RT_TXCFG_TX_START_DELAY_SHIFT	16
+#define PRUSS_MII_RT_TXCFG_TX_START_DELAY_MASK	GENMASK(25, 16)
+
+#define PRUSS_MII_RT_TXCFG_TX_CLK_DELAY_SHIFT	28
+#define PRUSS_MII_RT_TXCFG_TX_CLK_DELAY_MASK	GENMASK(30, 28)
+
+/* PRUSS_MII_RT_TX_IPG0/1 bits */
+#define PRUSS_MII_RT_TX_IPG_IPG_SHIFT	0
+#define PRUSS_MII_RT_TX_IPG_IPG_MASK	GENMASK(9, 0)
+
+/* PRUSS_MII_RT_PRS0/1 bits */
+#define PRUSS_MII_RT_PRS_COL	BIT(0)
+#define PRUSS_MII_RT_PRS_CRS	BIT(1)
+
+/* PRUSS_MII_RT_RX_FRMS0/1 bits */
+#define PRUSS_MII_RT_RX_FRMS_MIN_FRM_SHIFT	0
+#define PRUSS_MII_RT_RX_FRMS_MIN_FRM_MASK	GENMASK(15, 0)
+
+#define PRUSS_MII_RT_RX_FRMS_MAX_FRM_SHIFT	16
+#define PRUSS_MII_RT_RX_FRMS_MAX_FRM_MASK	GENMASK(31, 16)
+
+/* PRUSS_MII_RT_RX_PCNT0/1 bits */
+#define PRUSS_MII_RT_RX_PCNT_MIN_PCNT_SHIFT	0
+#define PRUSS_MII_RT_RX_PCNT_MIN_PCNT_MASK	GENMASK(3, 0)
+
+#define PRUSS_MII_RT_RX_PCNT_MAX_PCNT_SHIFT	4
+#define PRUSS_MII_RT_RX_PCNT_MAX_PCNT_MASK	GENMASK(7, 4)
+
+/* PRUSS_MII_RT_RX_ERR0/1 bits */
+#define PRUSS_MII_RT_RX_ERR_MIN_PCNT_ERR	BIT(0)
+#define PRUSS_MII_RT_RX_ERR_MAX_PCNT_ERR	BIT(1)
+#define PRUSS_MII_RT_RX_ERR_MIN_FRM_ERR		BIT(2)
+#define PRUSS_MII_RT_RX_ERR_MAX_FRM_ERR		BIT(3)
+
+/* TX IPG Values to be set for 100M and 1G link speeds.  These values are
+ * in ocp_clk cycles. So need change if ocp_clk is changed for a specific
+ * h/w design.
+ */
+#define MII_RT_TX_IPG_100M	0x166
+#define MII_RT_TX_IPG_1G	0x18
+
+#define RGMII_CFG_OFFSET	4
+
+/* Constant to choose between MII0 and MII1 */
+#define ICSS_MII0	0
+#define ICSS_MII1	1
+
+/* RGMII CFG Register bits */
+#define RGMII_CFG_GIG_EN_MII0	BIT(17)
+#define RGMII_CFG_GIG_EN_MII1	BIT(21)
+#define RGMII_CFG_FULL_DUPLEX_MII0	BIT(18)
+#define RGMII_CFG_FULL_DUPLEX_MII1	BIT(22)
+#define RGMII_CFG_SPEED_MII0	GENMASK(2, 1)
+#define RGMII_CFG_SPEED_MII1	GENMASK(6, 5)
+#define RGMII_CFG_SPEED_MII0_SHIFT	1
+#define RGMII_CFG_SPEED_MII1_SHIFT	5
+#define RGMII_CFG_FULLDUPLEX_MII0	BIT(3)
+#define RGMII_CFG_FULLDUPLEX_MII1	BIT(7)
+#define RGMII_CFG_FULLDUPLEX_MII0_SHIFT	3
+#define RGMII_CFG_FULLDUPLEX_MII1_SHIFT	7
+#define RGMII_CFG_SPEED_10M	0
+#define RGMII_CFG_SPEED_100M	1
+#define RGMII_CFG_SPEED_1G	2
+
+static inline void icssg_update_rgmii_cfg(struct regmap *miig_rt, bool gig_en,
+					  bool full_duplex, int mii)
+{
+	u32 gig_en_mask, gig_val = 0, full_duplex_mask, full_duplex_val = 0;
+
+	gig_en_mask = (mii == ICSS_MII0) ? RGMII_CFG_GIG_EN_MII0 :
+					RGMII_CFG_GIG_EN_MII1;
+	if (gig_en)
+		gig_val = gig_en_mask;
+	regmap_update_bits(miig_rt, RGMII_CFG_OFFSET, gig_en_mask, gig_val);
+
+	full_duplex_mask = (mii == ICSS_MII0) ? RGMII_CFG_FULL_DUPLEX_MII0 :
+					   RGMII_CFG_FULL_DUPLEX_MII1;
+	if (full_duplex)
+		full_duplex_val = full_duplex_mask;
+	regmap_update_bits(miig_rt, RGMII_CFG_OFFSET, full_duplex_mask,
+			   full_duplex_val);
+}
+
+static inline u32 icssg_rgmii_cfg_get_bitfield(struct regmap *miig_rt,
+					       u32 mask, u32 shift)
+{
+	u32 val;
+
+	regmap_read(miig_rt, RGMII_CFG_OFFSET, &val);
+	val &= mask;
+	val >>= shift;
+
+	return val;
+}
+
+static inline u32 icssg_rgmii_get_speed(struct regmap *miig_rt, int mii)
+{
+	u32 shift = RGMII_CFG_SPEED_MII0_SHIFT, mask = RGMII_CFG_SPEED_MII0;
+
+	if (mii == ICSS_MII1) {
+		shift = RGMII_CFG_SPEED_MII1_SHIFT;
+		mask = RGMII_CFG_SPEED_MII1;
+	}
+
+	return icssg_rgmii_cfg_get_bitfield(miig_rt, mask, shift);
+}
+
+static inline u32 icssg_rgmii_get_fullduplex(struct regmap *miig_rt, int mii)
+{
+	u32 shift = RGMII_CFG_FULLDUPLEX_MII0_SHIFT;
+	u32 mask = RGMII_CFG_FULLDUPLEX_MII0;
+
+	if (mii == ICSS_MII1) {
+		shift = RGMII_CFG_FULLDUPLEX_MII1_SHIFT;
+		mask = RGMII_CFG_FULLDUPLEX_MII1;
+	}
+
+	return icssg_rgmii_cfg_get_bitfield(miig_rt, mask, shift);
+}
+
+static inline void icssg_update_mii_rt_cfg(struct regmap *mii_rt, int speed,
+					   int mii)
+{
+	u32 ipg_reg, val;
+
+	ipg_reg = (mii == ICSS_MII0) ? PRUSS_MII_RT_TX_IPG0 :
+				       PRUSS_MII_RT_TX_IPG1;
+	switch (speed) {
+	case SPEED_1000:
+		val = MII_RT_TX_IPG_1G;
+		break;
+	case SPEED_100:
+		val = MII_RT_TX_IPG_100M;
+		break;
+	default:
+		/* Other links speeds not supported */
+		pr_err("Unsupported link speed\n");
+		return;
+	}
+	regmap_write(mii_rt, ipg_reg, val);
+}
+#endif /* __NET_PRUSS_MII_RT_H__ */
diff -Naur --no-dereference a/drivers/net/ethernet/ti/icss_switch.h b/drivers/net/ethernet/ti/icss_switch.h
--- a/drivers/net/ethernet/ti/icss_switch.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/net/ethernet/ti/icss_switch.h	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,204 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+/* Copyright (C) 2015-2018 Texas Instruments Incorporated - http://www.ti.com
+ */
+
+#ifndef __ICSS_SWITCH_H
+#define __ICSS_SWITCH_H
+
+/* Basic Switch Parameters
+ * Used to auto compute offset addresses on L3 OCMC RAM. Do not modify these
+ * without changing firmware accordingly
+ */
+#define SWITCH_BUFFER_SIZE	(64 * 1024)	/* L3 buffer */
+#define ICSS_BLOCK_SIZE		32		/* data bytes per BD */
+#define BD_SIZE			4		/* byte buffer descriptor */
+
+#define PORT_LINK_MASK		0x1
+#define PORT_IS_HD_MASK		0x2
+
+/* Physical Port queue size (number of BDs). Same for both ports */
+#define QUEUE_1_SIZE		97	/* Network Management high */
+#define QUEUE_2_SIZE		97	/* Network Management low */
+#define QUEUE_3_SIZE		97	/* Protocol specific */
+#define QUEUE_4_SIZE		97	/* NRT (IP,ARP, ICMP) */
+
+/* Host queue size (number of BDs). Each BD points to data buffer of 32 bytes.
+ * HOST PORT QUEUES can buffer up to 4 full sized frames per queue
+ */
+#define	HOST_QUEUE_1_SIZE	194	/* Protocol and VLAN priority 7 & 6 */
+#define HOST_QUEUE_2_SIZE	194	/* Protocol mid */
+#define HOST_QUEUE_3_SIZE	194	/* Protocol low */
+#define HOST_QUEUE_4_SIZE	194	/* NRT (IP, ARP, ICMP) */
+
+/* NRT Buffer descriptor definition
+ * Each buffer descriptor points to a max 32 byte block and has 32 bit in size
+ * to have atomic operation.
+ * PRU can address bytewise into memory.
+ * Definition of 32 bit descriptor is as follows
+ *
+ * Bits		Name			Meaning
+ * =============================================================================
+ * 0..7		Index		points to index in buffer queue, max 256 x 32
+ *				byte blocks can be addressed
+ * 8..12	Block_length	number of valid bytes in this specific block.
+ *				Will be <=32 bytes on last block of packet
+ * 13		More		"More" bit indicating that there are more blocks
+ * 14		Shadow		indicates that "index" is pointing into shadow
+ *				buffer
+ * 15		TimeStamp	indicates that this packet has time stamp in
+ *				separate buffer - only needed of PTCP runs on
+ *				host
+ * 16..17	Port		different meaning for ingress and egress,
+ *				Ingress: Port = 0 indicates phy port 1 and
+ *				Port = 1 indicates phy port 2.
+ *				Egress: 0 sends on phy port 1 and 1 sends on
+ *				phy port 2. Port = 2 goes over MAC table
+ *				look-up
+ * 18..28	Length		11 bit of total packet length which is put into
+ *				first BD only so that host access only one BD
+ * 29		VlanTag		indicates that packet has Length/Type field of
+ *				0x08100 with VLAN tag in following byte
+ * 30		Broadcast	indicates that packet goes out on both physical
+ *				ports,  there will be two bd but only one buffer
+ * 31		Error		indicates there was an error in the packet
+ */
+#define PRUETH_BD_SHADOW_MASK		BIT(14)
+
+#define PRUETH_BD_PORT_MASK		GENMASK(17, 16)
+#define PRUETH_BD_PORT_SHIFT		16
+
+#define PRUETH_BD_LENGTH_MASK		GENMASK(28, 18)
+#define PRUETH_BD_LENGTH_SHIFT		18
+
+#define PRUETH_BD_BROADCAST_MASK	BIT(30)
+#define PRUETH_BD_BROADCAST_SHIFT	30
+
+#define PRUETH_BD_ERROR_MASK		BIT(31)
+#define PRUETH_BD_ERROR_SHIFT		31
+
+/* The following offsets indicate which sections of the memory are used
+ * for EMAC internal tasks
+ */
+#define DRAM_START_OFFSET		0x1e98
+#define SRAM_START_OFFSET		0x400
+
+/* General Purpose Statistics
+ * These are present on both PRU0 and PRU1 DRAM
+ */
+/* base statistics offset */
+#define STATISTICS_OFFSET	0x1f00
+#define STAT_SIZE		0x90
+
+/* Offset for storing
+ * 1. Storm Prevention Params
+ * 2. PHY Speed Offset
+ * 3. Port Status Offset
+ * These are present on both PRU0 and PRU1
+ */
+/* 4 bytes */
+#define STORM_PREVENTION_OFFSET		(STATISTICS_OFFSET + STAT_SIZE)
+/* 4 bytes */
+#define PHY_SPEED_OFFSET		(STATISTICS_OFFSET + STAT_SIZE + 4)
+/* 1 byte */
+#define PORT_STATUS_OFFSET		(STATISTICS_OFFSET + STAT_SIZE + 8)
+/* 1 byte */
+#define COLLISION_COUNTER		(STATISTICS_OFFSET + STAT_SIZE + 9)
+/* 4 bytes */
+#define RX_PKT_SIZE_OFFSET		(STATISTICS_OFFSET + STAT_SIZE + 10)
+/* 4 bytes */
+#define PORT_CONTROL_ADDR		(STATISTICS_OFFSET + STAT_SIZE + 14)
+/* 6 bytes */
+#define PORT_MAC_ADDR			(STATISTICS_OFFSET + STAT_SIZE + 18)
+/* 1 byte */
+#define RX_INT_STATUS_OFFSET		(STATISTICS_OFFSET + STAT_SIZE + 24)
+
+/* DRAM Offsets for EMAC
+ * Present on Both DRAM0 and DRAM1
+ */
+
+/* 4 queue descriptors for port tx = 32 bytes */
+#define TX_CONTEXT_Q1_OFFSET_ADDR	(PORT_QUEUE_DESC_OFFSET + 32)
+#define PORT_QUEUE_DESC_OFFSET	(ICSS_EMAC_TTS_CYC_TX_SOF + 8)
+
+/* EMAC Time Triggered Send Offsets */
+#define ICSS_EMAC_TTS_CYC_TX_SOF	(ICSS_EMAC_TTS_PREV_TX_SOF + 8)
+#define ICSS_EMAC_TTS_PREV_TX_SOF	(ICSS_EMAC_TTS_MISSED_CYCLE_CNT_OFFSET + 4)
+#define ICSS_EMAC_TTS_MISSED_CYCLE_CNT_OFFSET	(ICSS_EMAC_TTS_STATUS_OFFSET + 4)
+#define ICSS_EMAC_TTS_STATUS_OFFSET	(ICSS_EMAC_TTS_CFG_TIME_OFFSET + 4)
+#define ICSS_EMAC_TTS_CFG_TIME_OFFSET	(ICSS_EMAC_TTS_CYCLE_PERIOD_OFFSET + 4)
+#define ICSS_EMAC_TTS_CYCLE_PERIOD_OFFSET	(ICSS_EMAC_TTS_CYCLE_START_OFFSET + 8)
+#define ICSS_EMAC_TTS_CYCLE_START_OFFSET	ICSS_EMAC_TTS_BASE_OFFSET
+#define ICSS_EMAC_TTS_BASE_OFFSET	DRAM_START_OFFSET
+
+/* Shared RAM offsets for EMAC */
+
+/* Queue Descriptors */
+
+/* 4 queue descriptors for port 0 (host receive). 32 bytes */
+#define HOST_QUEUE_DESC_OFFSET		(HOST_QUEUE_SIZE_ADDR + 16)
+
+/* table offset for queue size:
+ * 3 ports * 4 Queues * 1 byte offset = 12 bytes
+ */
+#define HOST_QUEUE_SIZE_ADDR		(HOST_QUEUE_OFFSET_ADDR + 8)
+/* table offset for queue:
+ * 4 Queues * 2 byte offset = 8 bytes
+ */
+#define HOST_QUEUE_OFFSET_ADDR		(HOST_QUEUE_DESCRIPTOR_OFFSET_ADDR + 8)
+/* table offset for Host queue descriptors:
+ * 1 ports * 4 Queues * 2 byte offset = 8 bytes
+ */
+#define HOST_QUEUE_DESCRIPTOR_OFFSET_ADDR	(HOST_Q4_RX_CONTEXT_OFFSET + 8)
+
+/* Host Port Rx Context */
+#define HOST_Q4_RX_CONTEXT_OFFSET	(HOST_Q3_RX_CONTEXT_OFFSET + 8)
+#define HOST_Q3_RX_CONTEXT_OFFSET	(HOST_Q2_RX_CONTEXT_OFFSET + 8)
+#define HOST_Q2_RX_CONTEXT_OFFSET	(HOST_Q1_RX_CONTEXT_OFFSET + 8)
+#define HOST_Q1_RX_CONTEXT_OFFSET	(EMAC_PROMISCUOUS_MODE_OFFSET + 4)
+
+/* Promiscuous mode control */
+#define EMAC_P1_PROMISCUOUS_BIT		BIT(0)
+#define EMAC_P2_PROMISCUOUS_BIT		BIT(1)
+#define EMAC_PROMISCUOUS_MODE_OFFSET	(EMAC_RESERVED + 4)
+#define EMAC_RESERVED			EOF_48K_BUFFER_BD
+
+/* allow for max 48k buffer which spans the descriptors up to 0x1800 6kB */
+#define EOF_48K_BUFFER_BD	(P0_BUFFER_DESC_OFFSET + HOST_BD_SIZE + PORT_BD_SIZE)
+
+#define HOST_BD_SIZE		((HOST_QUEUE_1_SIZE + HOST_QUEUE_2_SIZE + HOST_QUEUE_3_SIZE + HOST_QUEUE_4_SIZE) * BD_SIZE)
+#define PORT_BD_SIZE		((QUEUE_1_SIZE + QUEUE_2_SIZE + QUEUE_3_SIZE + QUEUE_4_SIZE) * 2 * BD_SIZE)
+
+#define END_OF_BD_POOL		(P2_Q4_BD_OFFSET + QUEUE_4_SIZE * BD_SIZE)
+#define P2_Q4_BD_OFFSET		(P2_Q3_BD_OFFSET + QUEUE_3_SIZE * BD_SIZE)
+#define P2_Q3_BD_OFFSET		(P2_Q2_BD_OFFSET + QUEUE_2_SIZE * BD_SIZE)
+#define P2_Q2_BD_OFFSET		(P2_Q1_BD_OFFSET + QUEUE_1_SIZE * BD_SIZE)
+#define P2_Q1_BD_OFFSET		(P1_Q4_BD_OFFSET + QUEUE_4_SIZE * BD_SIZE)
+#define P1_Q4_BD_OFFSET		(P1_Q3_BD_OFFSET + QUEUE_3_SIZE * BD_SIZE)
+#define P1_Q3_BD_OFFSET		(P1_Q2_BD_OFFSET + QUEUE_2_SIZE * BD_SIZE)
+#define P1_Q2_BD_OFFSET		(P1_Q1_BD_OFFSET + QUEUE_1_SIZE * BD_SIZE)
+#define P1_Q1_BD_OFFSET		(P0_Q4_BD_OFFSET + HOST_QUEUE_4_SIZE * BD_SIZE)
+#define P0_Q4_BD_OFFSET		(P0_Q3_BD_OFFSET + HOST_QUEUE_3_SIZE * BD_SIZE)
+#define P0_Q3_BD_OFFSET		(P0_Q2_BD_OFFSET + HOST_QUEUE_2_SIZE * BD_SIZE)
+#define P0_Q2_BD_OFFSET		(P0_Q1_BD_OFFSET + HOST_QUEUE_1_SIZE * BD_SIZE)
+#define P0_Q1_BD_OFFSET		P0_BUFFER_DESC_OFFSET
+#define P0_BUFFER_DESC_OFFSET	SRAM_START_OFFSET
+
+/* Memory Usage of L3 OCMC RAM */
+
+/* L3 64KB Memory - mainly buffer Pool */
+#define END_OF_BUFFER_POOL	(P2_Q4_BUFFER_OFFSET + QUEUE_4_SIZE * ICSS_BLOCK_SIZE)
+#define P2_Q4_BUFFER_OFFSET	(P2_Q3_BUFFER_OFFSET + QUEUE_3_SIZE * ICSS_BLOCK_SIZE)
+#define P2_Q3_BUFFER_OFFSET	(P2_Q2_BUFFER_OFFSET + QUEUE_2_SIZE * ICSS_BLOCK_SIZE)
+#define P2_Q2_BUFFER_OFFSET	(P2_Q1_BUFFER_OFFSET + QUEUE_1_SIZE * ICSS_BLOCK_SIZE)
+#define P2_Q1_BUFFER_OFFSET	(P1_Q4_BUFFER_OFFSET + QUEUE_4_SIZE * ICSS_BLOCK_SIZE)
+#define P1_Q4_BUFFER_OFFSET	(P1_Q3_BUFFER_OFFSET + QUEUE_3_SIZE * ICSS_BLOCK_SIZE)
+#define P1_Q3_BUFFER_OFFSET	(P1_Q2_BUFFER_OFFSET + QUEUE_2_SIZE * ICSS_BLOCK_SIZE)
+#define P1_Q2_BUFFER_OFFSET	(P1_Q1_BUFFER_OFFSET + QUEUE_1_SIZE * ICSS_BLOCK_SIZE)
+#define P1_Q1_BUFFER_OFFSET	(P0_Q4_BUFFER_OFFSET + HOST_QUEUE_4_SIZE * ICSS_BLOCK_SIZE)
+#define P0_Q4_BUFFER_OFFSET	(P0_Q3_BUFFER_OFFSET + HOST_QUEUE_3_SIZE * ICSS_BLOCK_SIZE)
+#define P0_Q3_BUFFER_OFFSET	(P0_Q2_BUFFER_OFFSET + HOST_QUEUE_2_SIZE * ICSS_BLOCK_SIZE)
+#define P0_Q2_BUFFER_OFFSET	(P0_Q1_BUFFER_OFFSET + HOST_QUEUE_1_SIZE * ICSS_BLOCK_SIZE)
+#define P0_Q1_BUFFER_OFFSET	0x0000
+
+#endif /* __ICSS_SWITCH_H */
diff -Naur --no-dereference a/drivers/net/ethernet/ti/icss_vlan_mcast_filter_mmap.h b/drivers/net/ethernet/ti/icss_vlan_mcast_filter_mmap.h
--- a/drivers/net/ethernet/ti/icss_vlan_mcast_filter_mmap.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/net/ethernet/ti/icss_vlan_mcast_filter_mmap.h	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,80 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+/* Copyright (C) 2015-2019 Texas Instruments Incorporated - http://www.ti.com
+ *
+ * This file contains VLAN/Multicast filtering feature memory map
+ *
+ */
+
+#ifndef ICSS_VLAN_MULTICAST_FILTER_MM_H
+#define ICSS_VLAN_MULTICAST_FILTER_MM_H
+
+/*  VLAN/Multicast filter defines & offsets, present on both PRU0 and PRU1 DRAM */
+
+/* Feature enable/disable values for multicast filtering */
+#define ICSS_EMAC_FW_MULTICAST_FILTER_CTRL_DISABLED             0x00
+#define ICSS_EMAC_FW_MULTICAST_FILTER_CTRL_ENABLED              0x01
+
+/* Feature enable/disable values  for VLAN filtering */
+#define ICSS_EMAC_FW_VLAN_FILTER_CTRL_DISABLED                  0x00
+#define ICSS_EMAC_FW_VLAN_FILTER_CTRL_ENABLED                   0x01
+
+/* Add/remove multicast mac id for filtering bin */
+#define ICSS_EMAC_FW_MULTICAST_FILTER_HOST_RCV_ALLOWED          0x01
+#define ICSS_EMAC_FW_MULTICAST_FILTER_HOST_RCV_NOT_ALLOWED      0x00
+
+/* Default HASH value for the multicast filtering Mask */
+#define ICSS_EMAC_FW_MULTICAST_FILTER_INIT_VAL                  0xFF
+
+/* Size requirements for Multicast filtering feature */
+#define ICSS_EMAC_FW_MULTICAST_TABLE_SIZE_BYTES                        256
+#define ICSS_EMAC_FW_MULTICAST_FILTER_MASK_SIZE_BYTES                    6
+#define ICSS_EMAC_FW_MULTICAST_FILTER_CTRL_SIZE_BYTES                    1
+#define ICSS_EMAC_FW_MULTICAST_FILTER_MASK_OVERRIDE_STATUS_SIZE_BYTES    1
+#define ICSS_EMAC_FW_MULTICAST_FILTER_DROP_CNT_SIZE_BYTES                4
+
+/* Size requirements for VLAN filtering feature : 4096 bits = 512 bytes */
+#define ICSS_EMAC_FW_VLAN_FILTER_TABLE_SIZE_BYTES                      512
+#define ICSS_EMAC_FW_VLAN_FILTER_CTRL_SIZE_BYTES                         1
+#define ICSS_EMAC_FW_VLAN_FILTER_DROP_CNT_SIZE_BYTES                     4
+
+/* Mask override set status */
+#define ICSS_EMAC_FW_MULTICAST_FILTER_MASK_OVERRIDE_SET                  1
+/* Mask override not set status */
+#define ICSS_EMAC_FW_MULTICAST_FILTER_MASK_OVERRIDE_NOT_SET              0
+/* 6 bytes HASH Mask for the MAC */
+#define ICSS_EMAC_FW_MULTICAST_FILTER_MASK_OFFSET         0xF4
+/* 0 -> multicast filtering disabled | 1 -> multicast filtering enabled */
+#define ICSS_EMAC_FW_MULTICAST_FILTER_CTRL_OFFSET        (ICSS_EMAC_FW_MULTICAST_FILTER_MASK_OFFSET + ICSS_EMAC_FW_MULTICAST_FILTER_MASK_SIZE_BYTES)
+/* Status indicating if the HASH override is done or not: 0: no, 1: yes */
+#define ICSS_EMAC_FW_MULTICAST_FILTER_OVERRIDE_STATUS    (ICSS_EMAC_FW_MULTICAST_FILTER_CTRL_OFFSET + ICSS_EMAC_FW_MULTICAST_FILTER_CTRL_SIZE_BYTES)
+/* Multicast drop statistics */
+#define ICSS_EMAC_FW_MULTICAST_FILTER_DROP_CNT_OFFSET    (ICSS_EMAC_FW_MULTICAST_FILTER_OVERRIDE_STATUS + ICSS_EMAC_FW_MULTICAST_FILTER_MASK_OVERRIDE_STATUS_SIZE_BYTES)
+/* Multicast table */
+#define ICSS_EMAC_FW_MULTICAST_FILTER_TABLE              (ICSS_EMAC_FW_MULTICAST_FILTER_DROP_CNT_OFFSET + ICSS_EMAC_FW_MULTICAST_FILTER_DROP_CNT_SIZE_BYTES)
+
+/* VLAN table Offsets */
+#define ICSS_EMAC_FW_VLAN_FLTR_TBL_BASE_ADDR             0x200
+#define ICSS_EMAC_FW_VLAN_FILTER_CTRL_BITMAP_OFFSET      0xEF
+#define ICSS_EMAC_FW_VLAN_FILTER_DROP_CNT_OFFSET         (ICSS_EMAC_FW_VLAN_FILTER_CTRL_BITMAP_OFFSET + ICSS_EMAC_FW_VLAN_FILTER_CTRL_SIZE_BYTES)
+
+/* VLAN filter Control Bit maps */
+/* one bit field, bit 0: | 0 : VLAN filter disabled (default), 1: VLAN filter enabled */
+#define ICSS_EMAC_FW_VLAN_FILTER_CTRL_ENABLE_BIT                       0
+/* one bit field, bit 1: | 0 : untagged host rcv allowed (default), 1: untagged host rcv not allowed */
+#define ICSS_EMAC_FW_VLAN_FILTER_UNTAG_HOST_RCV_ALLOW_CTRL_BIT         1
+/* one bit field, bit 1: | 0 : priotag host rcv allowed (default), 1: priotag host rcv not allowed */
+#define ICSS_EMAC_FW_VLAN_FILTER_PRIOTAG_HOST_RCV_ALLOW_CTRL_BIT       2
+/* one bit field, bit 1: | 0 : skip sv vlan flow   :1 : take sv vlan flow  (not applicable for dual emac */
+#define ICSS_EMAC_FW_VLAN_FILTER_SV_VLAN_FLOW_HOST_RCV_ALLOW_CTRL_BIT  3
+
+/* VLAN IDs */
+#define ICSS_EMAC_FW_VLAN_FILTER_PRIOTAG_VID                           0
+#define ICSS_EMAC_FW_VLAN_FILTER_VID_MIN                               0x0000
+#define ICSS_EMAC_FW_VLAN_FILTER_VID_MAX                               0x0FFF
+
+/* VLAN Filtering Commands */
+#define ICSS_EMAC_FW_VLAN_FILTER_ADD_VLAN_VID_CMD                      0x00
+#define ICSS_EMAC_FW_VLAN_FILTER_REMOVE_VLAN_VID_CMD                   0x01
+
+#endif /* ICSS_MULTICAST_FILTER_MM_H */
diff -Naur --no-dereference a/drivers/net/ethernet/ti/j721e-cpsw-virt-mac.c b/drivers/net/ethernet/ti/j721e-cpsw-virt-mac.c
--- a/drivers/net/ethernet/ti/j721e-cpsw-virt-mac.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/net/ethernet/ti/j721e-cpsw-virt-mac.c	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,1446 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Texas Instruments K3 J721 Virt Ethernet Switch MAC Driver
+ *
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ */
+
+#include <linux/etherdevice.h>
+#include <linux/if_vlan.h>
+#include <linux/interrupt.h>
+#include <linux/inetdevice.h>
+#include <linux/kernel.h>
+#include <linux/kmemleak.h>
+#include <linux/module.h>
+#include <linux/netdevice.h>
+#include <linux/of.h>
+#include <linux/of_net.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/dma/ti-cppi5.h>
+#include <linux/dma/k3-navss-udma.h>
+#include <linux/rpmsg-remotedev/rpmsg-remotedev.h>
+#include <linux/soc/ti/k3-navss-desc-pool.h>
+
+#define VIRT_CPSW_DRV_VER "0.1"
+
+#define VIRT_CPSW_MAX_TX_QUEUES	1
+#define VIRT_CPSW_MAX_RX_QUEUES	1
+#define VIRT_CPSW_MAX_RX_FLOWS	1
+
+#define VIRT_CPSW_MIN_PACKET_SIZE	ETH_ZLEN
+#define VIRT_CPSW_MAX_PACKET_SIZE	(VLAN_ETH_FRAME_LEN + ETH_FCS_LEN)
+
+/* Number of TX/RX descriptors */
+#define VIRT_CPSW_MAX_TX_DESC	256
+#define VIRT_CPSW_MAX_RX_DESC	256
+
+#define VIRT_CPSW_NAV_PS_DATA_SIZE 16
+#define VIRT_CPSW_NAV_SW_DATA_SIZE 16
+
+#define VIRT_CPSW_DRV_NAME "j721e-cpsw-virt-mac"
+
+struct virt_cpsw_tx_chn {
+	struct device *dev;
+	struct k3_knav_desc_pool *desc_pool;
+	struct k3_nav_udmax_tx_channel *tx_chn;
+	u32 descs_num;
+	unsigned int irq;
+	u32 id;
+};
+
+struct virt_cpsw_rx_chn {
+	struct device *dev;
+	struct k3_knav_desc_pool *desc_pool;
+	struct k3_nav_udmax_rx_channel *rx_chn;
+	u32 descs_num;
+	unsigned int irq;
+};
+
+struct virt_cpsw_port {
+	struct virt_cpsw_common *common;
+	struct net_device *ndev;
+	const char *name;
+	u8 local_mac_addr[ETH_ALEN];
+};
+
+struct virt_cpsw_common {
+	struct device *dev;
+	struct virt_cpsw_port ports;
+
+	struct virt_cpsw_tx_chn tx_chns;
+	struct napi_struct napi_tx;
+	struct completion tdown_complete;
+	atomic_t tdown_cnt;
+	struct virt_cpsw_rx_chn rx_chns;
+	struct napi_struct napi_rx;
+
+	const char *rdev_name;
+	struct rpmsg_remotedev *rdev;
+	struct rpmsg_remotedev_eth_switch_ops *rdev_switch_ops;
+	u32 rdev_features;
+	u32 rdev_mtu;
+	u8 rdev_mac_addr[ETH_ALEN];
+	u32 rdev_tx_psil_dst_id;
+	u32 tx_psil_id_base;
+	u32 rdev_rx_flow_id;
+};
+
+struct virt_cpsw_ndev_stats {
+	u64 tx_packets;
+	u64 tx_bytes;
+	u64 rx_packets;
+	u64 rx_bytes;
+	struct u64_stats_sync syncp;
+};
+
+struct virt_cpsw_ndev_priv {
+	struct virt_cpsw_ndev_stats __percpu *stats;
+	struct virt_cpsw_port	*port;
+};
+
+#define virt_ndev_to_priv(ndev) \
+	((struct virt_cpsw_ndev_priv *)netdev_priv(ndev))
+#define virt_ndev_to_port(ndev) (virt_ndev_to_priv(ndev)->port)
+#define virt_ndev_to_common(ndev) (virt_ndev_to_port(ndev)->common)
+
+static void virt_cpsw_nuss_ndo_host_tx_timeout(struct net_device *ndev)
+{
+	struct virt_cpsw_common *common = virt_ndev_to_common(ndev);
+	struct virt_cpsw_tx_chn *tx_chn = &common->tx_chns;
+	struct netdev_queue *netif_txq;
+	unsigned long trans_start;
+
+	/* process every txq*/
+	netif_txq = netdev_get_tx_queue(ndev, 0);
+	trans_start = netif_txq->trans_start;
+	if (netif_xmit_stopped(netif_txq) &&
+	    time_after(jiffies, (trans_start + ndev->watchdog_timeo))) {
+		netdev_err(ndev, "txq:%d DRV_XOFF:%d tmo:%u dql_avail:%d free_desc:%zu\n",
+			   0,
+			   netif_tx_queue_stopped(netif_txq),
+			   jiffies_to_msecs(jiffies - trans_start),
+			   dql_avail(&netif_txq->dql),
+			   k3_knav_pool_avail(tx_chn->desc_pool));
+
+		if (netif_tx_queue_stopped(netif_txq)) {
+			/* try recover if stopped by us */
+			txq_trans_update(netif_txq);
+			netif_tx_wake_queue(netif_txq);
+		}
+	}
+}
+
+static int virt_cpsw_nuss_rx_push(struct virt_cpsw_common *common,
+				  struct sk_buff *skb)
+{
+	struct cppi5_host_desc_t *desc_rx;
+	struct virt_cpsw_rx_chn *rx_chn = &common->rx_chns;
+	struct device *dev = common->dev;
+	dma_addr_t desc_dma;
+	dma_addr_t buf_dma;
+	u32 pkt_len = skb_tailroom(skb);
+	void *swdata;
+
+	desc_rx = k3_knav_pool_alloc(rx_chn->desc_pool);
+	if (!desc_rx) {
+		dev_err(dev, "Failed to allocate RXFDQ descriptor\n");
+		return -ENOMEM;
+	}
+	desc_dma = k3_knav_pool_virt2dma(rx_chn->desc_pool, desc_rx);
+
+	buf_dma = dma_map_single(dev, skb->data, pkt_len, DMA_FROM_DEVICE);
+	if (unlikely(dma_mapping_error(dev, buf_dma))) {
+		k3_knav_pool_free(rx_chn->desc_pool, desc_rx);
+		dev_err(dev, "Failed to map rx skb buffer\n");
+		return -EINVAL;
+	}
+
+	cppi5_hdesc_init(desc_rx, CPPI5_INFO0_HDESC_EPIB_PRESENT,
+			 VIRT_CPSW_NAV_PS_DATA_SIZE);
+	cppi5_hdesc_attach_buf(desc_rx, 0, 0, buf_dma, skb_tailroom(skb));
+	swdata = cppi5_hdesc_get_swdata(desc_rx);
+	*((void **)swdata) = skb;
+
+	return k3_nav_udmax_push_rx_chn(rx_chn->rx_chn, 0, desc_rx, desc_dma);
+}
+
+static int virt_cpsw_nuss_common_open(struct virt_cpsw_common *common,
+				      netdev_features_t features)
+{
+	struct sk_buff *skb;
+	int i, ret;
+
+	for (i = 0; i < common->rx_chns.descs_num; i++) {
+		skb = __netdev_alloc_skb_ip_align(NULL,
+						  VIRT_CPSW_MAX_PACKET_SIZE,
+						  GFP_KERNEL);
+		if (!skb) {
+			dev_err(common->dev, "cannot allocate skb\n");
+			return -ENOMEM;
+		}
+
+		ret = virt_cpsw_nuss_rx_push(common, skb);
+		if (ret < 0) {
+			dev_err(common->dev,
+				"cannot submit skb to channel rx, error %d\n",
+				ret);
+			kfree_skb(skb);
+			return ret;
+		}
+		kmemleak_not_leak(skb);
+	}
+	ret = k3_nav_udmax_rx_flow_enable(common->rx_chns.rx_chn, 0);
+	if (ret)
+		return ret;
+
+	ret = k3_nav_udmax_enable_tx_chn(common->tx_chns.tx_chn);
+	if (ret)
+		return ret;
+
+	napi_enable(&common->napi_tx);
+	napi_enable(&common->napi_rx);
+
+	return 0;
+}
+
+static void virt_cpsw_nuss_tx_cleanup(void *data, dma_addr_t desc_dma);
+static void virt_cpsw_nuss_rx_cleanup(void *data, dma_addr_t desc_dma);
+
+static void virt_cpsw_nuss_common_stop(struct virt_cpsw_common *common)
+{
+	int i;
+
+	/* shutdown tx channels */
+	atomic_set(&common->tdown_cnt, VIRT_CPSW_MAX_TX_QUEUES);
+	/* ensure new tdown_cnt value is visible */
+	smp_mb__after_atomic();
+	reinit_completion(&common->tdown_complete);
+
+	k3_nav_udmax_tdown_tx_chn(common->tx_chns.tx_chn, false);
+
+	i = wait_for_completion_timeout(&common->tdown_complete,
+					msecs_to_jiffies(1000));
+	if (!i)
+		dev_err(common->dev, "tx teardown timeout\n");
+
+	k3_nav_udmax_reset_tx_chn(common->tx_chns.tx_chn,
+				  &common->tx_chns,
+				  virt_cpsw_nuss_tx_cleanup);
+	k3_nav_udmax_disable_tx_chn(common->tx_chns.tx_chn);
+	napi_disable(&common->napi_tx);
+
+	k3_nav_udmax_rx_flow_disable(common->rx_chns.rx_chn, 0);
+	/* Need some delay to process RX ring before reset */
+	msleep(100);
+	k3_nav_udmax_reset_rx_chn(common->rx_chns.rx_chn, 0,
+				  &common->rx_chns,
+				  virt_cpsw_nuss_rx_cleanup, false);
+	napi_disable(&common->napi_rx);
+}
+
+static int virt_cpsw_nuss_ndo_stop(struct net_device *ndev)
+{
+	struct virt_cpsw_common *common = virt_ndev_to_common(ndev);
+	struct rpmsg_remotedev_eth_switch_ops *rdev_ops;
+	struct device *dev = common->dev;
+	int ret;
+
+	rdev_ops = common->rdev_switch_ops;
+	netif_tx_stop_all_queues(ndev);
+	netif_carrier_off(ndev);
+
+	ret = rdev_ops->unregister_mac(common->rdev, ndev->dev_addr,
+				       common->rdev_rx_flow_id);
+	if (ret)
+		dev_err(dev, "unregister_mac rpmsg - fail %d\n", ret);
+
+	virt_cpsw_nuss_common_stop(common);
+
+	dev_info(common->dev, "virt_cpsw_nuss mac stopped\n");
+	return 0;
+}
+
+static int virt_cpsw_nuss_ndo_open(struct net_device *ndev)
+{
+	struct virt_cpsw_common *common = virt_ndev_to_common(ndev);
+	struct rpmsg_remotedev_eth_switch_ops *rdev_ops;
+	struct device *dev = common->dev;
+	int ret;
+
+	rdev_ops = common->rdev_switch_ops;
+	netdev_tx_reset_queue(netdev_get_tx_queue(ndev, 0));
+
+	ret = virt_cpsw_nuss_common_open(common, ndev->features);
+	if (ret)
+		return ret;
+
+	ret = rdev_ops->register_mac(common->rdev,
+				     ndev->dev_addr,
+				     common->rdev_rx_flow_id);
+	if (ret) {
+		dev_err(dev, "register_mac rpmsg - fail %d\n", ret);
+		virt_cpsw_nuss_common_stop(common);
+		return ret;
+	}
+
+	netif_tx_wake_all_queues(ndev);
+	netif_carrier_on(ndev);
+
+	dev_info(common->dev, "virt_cpsw_nuss mac started\n");
+	return 0;
+}
+
+static void virt_cpsw_nuss_rx_cleanup(void *data, dma_addr_t desc_dma)
+{
+	struct virt_cpsw_rx_chn *rx_chn = data;
+	struct cppi5_host_desc_t *desc_rx;
+	struct sk_buff *skb;
+	dma_addr_t buf_dma;
+	u32 buf_dma_len;
+	void **swdata;
+
+	desc_rx = k3_knav_pool_dma2virt(rx_chn->desc_pool, desc_dma);
+	swdata = cppi5_hdesc_get_swdata(desc_rx);
+	skb = *swdata;
+	cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);
+
+	dma_unmap_single(rx_chn->dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE);
+	k3_knav_pool_free(rx_chn->desc_pool, desc_rx);
+
+	dev_kfree_skb_any(skb);
+}
+
+/* RX psdata[2] word format - checksum information */
+#define AM65_CPSW_RX_PSD_CSUM_ADD	GENMASK(15, 0)
+#define AM65_CPSW_RX_PSD_CSUM_ERR	BIT(16)
+#define AM65_CPSW_RX_PSD_IS_FRAGMENT	BIT(17)
+#define AM65_CPSW_RX_PSD_IS_TCP		BIT(18)
+#define AM65_CPSW_RX_PSD_IPV6_VALID	BIT(19)
+#define AM65_CPSW_RX_PSD_IPV4_VALID	BIT(20)
+
+static void virt_cpsw_nuss_rx_csum(struct sk_buff *skb, u32 csum_info)
+{
+	/* HW can verify IPv4/IPv6 TCP/UDP packets checksum
+	 * csum information provides in psdata[2] word:
+	 * AM65_CPSW_RX_PSD_CSUM_ERR bit - indicates csum error
+	 * AM65_CPSW_RX_PSD_IPV6_VALID and AM65_CPSW_RX_PSD_IPV4_VALID
+	 * bits - indicates IPv4/IPv6 packet
+	 * AM65_CPSW_RX_PSD_IS_FRAGMENT bit - indicates fragmented packet
+	 * AM65_CPSW_RX_PSD_CSUM_ADD has value 0xFFFF for non fragmented packets
+	 * or csum value for fragmented packets if !AM65_CPSW_RX_PSD_CSUM_ERR
+	 */
+	skb_checksum_none_assert(skb);
+
+	if (unlikely(!(skb->dev->features & NETIF_F_RXCSUM)))
+		return;
+
+	if ((csum_info & (AM65_CPSW_RX_PSD_IPV6_VALID |
+			  AM65_CPSW_RX_PSD_IPV4_VALID)) &&
+			  !(csum_info & AM65_CPSW_RX_PSD_CSUM_ERR)) {
+		if (csum_info & AM65_CPSW_RX_PSD_IS_FRAGMENT) {
+			skb->ip_summed = CHECKSUM_COMPLETE;
+			skb->csum = csum_unfold(csum_info &
+						AM65_CPSW_RX_PSD_CSUM_ADD);
+		} else {
+			skb->ip_summed = CHECKSUM_UNNECESSARY;
+		}
+	}
+}
+
+static int virt_cpsw_nuss_rx_packets(struct virt_cpsw_common *common,
+				     u32 flow_idx)
+{
+	struct virt_cpsw_rx_chn *rx_chn = &common->rx_chns;
+	struct device *dev = common->dev;
+	struct virt_cpsw_ndev_priv *ndev_priv;
+	struct virt_cpsw_ndev_stats *stats;
+	struct net_device *ndev;
+	struct cppi5_host_desc_t *desc_rx;
+	struct sk_buff *skb, *new_skb;
+	dma_addr_t desc_dma, buf_dma;
+	u32 buf_dma_len, pkt_len, port_id = 0, csum_info;
+	int ret = 0;
+	void **swdata;
+	u32 *psdata;
+
+	ret = k3_nav_udmax_pop_rx_chn(rx_chn->rx_chn, flow_idx, &desc_dma);
+	if (ret) {
+		if (ret != -ENODATA)
+			dev_err(dev, "RX: pop chn fail %d\n", ret);
+		return ret;
+	}
+
+	if (desc_dma & 0x1) {
+		dev_dbg(dev, "%s RX tdown flow: %u\n", __func__, flow_idx);
+		return 0;
+	}
+
+	desc_rx = k3_knav_pool_dma2virt(rx_chn->desc_pool, desc_dma);
+	dev_dbg(dev, "%s flow_idx: %u desc %pad\n",
+		__func__, flow_idx, &desc_dma);
+
+	swdata = cppi5_hdesc_get_swdata(desc_rx);
+	skb = *swdata;
+	cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);
+	pkt_len = cppi5_hdesc_get_pktlen(desc_rx);
+	cppi5_desc_get_tags_ids(&desc_rx->hdr, &port_id, NULL);
+	/* read port for dbg */
+	dev_dbg(dev, "%s rx port_id:%d\n", __func__, port_id);
+	ndev = common->ports.ndev;
+	skb->dev = ndev;
+
+	psdata = cppi5_hdesc_get_psdata32(desc_rx);
+	csum_info = psdata[2];
+	dev_dbg(dev, "%s rx csum_info:%#x\n", __func__, csum_info);
+
+	dma_unmap_single(dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE);
+
+	k3_knav_pool_free(rx_chn->desc_pool, desc_rx);
+
+	if (unlikely(!netif_running(skb->dev))) {
+		dev_kfree_skb_any(skb);
+		return -ENODEV;
+	}
+
+	new_skb = netdev_alloc_skb_ip_align(ndev, VIRT_CPSW_MAX_PACKET_SIZE);
+	if (new_skb) {
+		skb_put(skb, pkt_len);
+		skb->protocol = eth_type_trans(skb, ndev);
+		virt_cpsw_nuss_rx_csum(skb, csum_info);
+		napi_gro_receive(&common->napi_rx, skb);
+
+		ndev_priv = netdev_priv(ndev);
+		stats = this_cpu_ptr(ndev_priv->stats);
+
+		u64_stats_update_begin(&stats->syncp);
+		stats->rx_packets++;
+		stats->rx_bytes += pkt_len;
+		u64_stats_update_end(&stats->syncp);
+		kmemleak_not_leak(new_skb);
+	} else {
+		ndev->stats.rx_dropped++;
+		new_skb = skb;
+	}
+
+	if (netif_dormant(ndev)) {
+		dev_kfree_skb_any(new_skb);
+		ndev->stats.rx_dropped++;
+		return -ENODEV;
+	}
+
+	ret = virt_cpsw_nuss_rx_push(common, new_skb);
+	if (WARN_ON(ret < 0)) {
+		dev_kfree_skb_any(new_skb);
+		ndev->stats.rx_errors++;
+		ndev->stats.rx_dropped++;
+	}
+
+	return ret;
+}
+
+static int virt_cpsw_nuss_rx_poll(struct napi_struct *napi_rx, int budget)
+{
+	struct virt_cpsw_common *common =
+			container_of(napi_rx, struct virt_cpsw_common, napi_rx);
+	int num_rx = 0;
+	int cur_budget;
+	int ret;
+
+	/* process every flow */
+	cur_budget = budget;
+
+	while (cur_budget--) {
+		ret = virt_cpsw_nuss_rx_packets(common, 0);
+		if (ret)
+			break;
+		num_rx++;
+	}
+
+	dev_dbg(common->dev, "%s num_rx:%d %d\n", __func__, num_rx, budget);
+
+	if (num_rx < budget && napi_complete_done(napi_rx, num_rx))
+		enable_irq(common->rx_chns.irq);
+
+	return num_rx;
+}
+
+static void virt_cpsw_nuss_xmit_free(struct virt_cpsw_tx_chn *tx_chn,
+				     struct device *dev,
+				     struct cppi5_host_desc_t *desc)
+{
+	struct cppi5_host_desc_t *first_desc, *next_desc;
+	dma_addr_t buf_dma, next_desc_dma;
+	u32 buf_dma_len;
+
+	first_desc = desc;
+	next_desc = first_desc;
+
+	cppi5_hdesc_get_obuf(first_desc, &buf_dma, &buf_dma_len);
+
+	dma_unmap_single(dev, buf_dma, buf_dma_len,
+			 DMA_TO_DEVICE);
+
+	next_desc_dma = cppi5_hdesc_get_next_hbdesc(first_desc);
+	while (next_desc_dma) {
+		next_desc = k3_knav_pool_dma2virt(tx_chn->desc_pool,
+						  next_desc_dma);
+		cppi5_hdesc_get_obuf(next_desc, &buf_dma, &buf_dma_len);
+
+		dma_unmap_page(dev, buf_dma, buf_dma_len,
+			       DMA_TO_DEVICE);
+
+		next_desc_dma = cppi5_hdesc_get_next_hbdesc(next_desc);
+
+		k3_knav_pool_free(tx_chn->desc_pool, next_desc);
+	}
+
+	k3_knav_pool_free(tx_chn->desc_pool, first_desc);
+}
+
+static void virt_cpsw_nuss_tx_cleanup(void *data, dma_addr_t desc_dma)
+{
+	struct virt_cpsw_tx_chn *tx_chn = data;
+	struct cppi5_host_desc_t *desc_tx;
+	struct sk_buff *skb;
+	void **swdata;
+
+	desc_tx = k3_knav_pool_dma2virt(tx_chn->desc_pool, desc_dma);
+	swdata = cppi5_hdesc_get_swdata(desc_tx);
+	skb = *(swdata);
+	virt_cpsw_nuss_xmit_free(tx_chn, tx_chn->dev, desc_tx);
+
+	dev_kfree_skb_any(skb);
+}
+
+static int virt_cpsw_nuss_tx_compl_packets(struct virt_cpsw_common *common,
+					   int chn, unsigned int budget)
+{
+	struct cppi5_host_desc_t *desc_tx;
+	struct device *dev = common->dev;
+	struct netdev_queue *netif_txq;
+	struct virt_cpsw_tx_chn *tx_chn;
+	struct net_device *ndev;
+	unsigned int total_bytes = 0;
+	struct sk_buff *skb;
+	dma_addr_t desc_dma;
+	int res, num_tx = 0;
+	void **swdata;
+
+	tx_chn = &common->tx_chns;
+
+	while (budget--) {
+		struct virt_cpsw_ndev_priv *ndev_priv;
+		struct virt_cpsw_ndev_stats *stats;
+
+		res = k3_nav_udmax_pop_tx_chn(tx_chn->tx_chn, &desc_dma);
+		if (res == -ENODATA)
+			break;
+
+		if (desc_dma & 0x1) {
+			if (atomic_dec_and_test(&common->tdown_cnt))
+				complete(&common->tdown_complete);
+			break;
+		}
+
+		desc_tx = k3_knav_pool_dma2virt(tx_chn->desc_pool, desc_dma);
+		swdata = cppi5_hdesc_get_swdata(desc_tx);
+		skb = *(swdata);
+		virt_cpsw_nuss_xmit_free(tx_chn, dev, desc_tx);
+
+		ndev = skb->dev;
+
+		ndev_priv = netdev_priv(ndev);
+		stats = this_cpu_ptr(ndev_priv->stats);
+		u64_stats_update_begin(&stats->syncp);
+		stats->tx_packets++;
+		stats->tx_bytes += skb->len;
+		u64_stats_update_end(&stats->syncp);
+
+		total_bytes += skb->len;
+		napi_consume_skb(skb, budget);
+		num_tx++;
+	}
+
+	if (!num_tx)
+		return 0;
+
+	netif_txq = netdev_get_tx_queue(ndev, 0);
+
+	netdev_tx_completed_queue(netif_txq, num_tx, total_bytes);
+	dev_dbg(dev, "compl 0 %d Bytes\n", total_bytes);
+
+	if (netif_tx_queue_stopped(netif_txq)) {
+		/* Check whether the queue is stopped due to stalled tx dma,
+		 * if the queue is stopped then wake the queue as
+		 * we have free desc for tx
+		 */
+		__netif_tx_lock(netif_txq, smp_processor_id());
+		if (netif_running(ndev) &&
+		    (k3_knav_pool_avail(tx_chn->desc_pool) >= MAX_SKB_FRAGS))
+			netif_tx_wake_queue(netif_txq);
+
+		__netif_tx_unlock(netif_txq);
+	}
+	dev_dbg(dev, "%s:%u pkt:%d\n", __func__, chn, num_tx);
+
+	return num_tx;
+}
+
+static int virt_cpsw_nuss_tx_poll(struct napi_struct *napi_tx, int budget)
+{
+	struct virt_cpsw_common *common =
+			container_of(napi_tx, struct virt_cpsw_common, napi_tx);
+	int num_tx;
+
+	/* process every unprocessed channel */
+	num_tx = virt_cpsw_nuss_tx_compl_packets(common, 0, budget);
+
+	if (num_tx < budget) {
+		napi_complete(napi_tx);
+		enable_irq(common->tx_chns.irq);
+	}
+
+	return num_tx;
+}
+
+static irqreturn_t virt_cpsw_nuss_rx_irq(int irq, void *dev_id)
+{
+	struct virt_cpsw_common *common = dev_id;
+
+	disable_irq_nosync(irq);
+	napi_schedule(&common->napi_rx);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t virt_cpsw_nuss_tx_irq(int irq, void *dev_id)
+{
+	struct virt_cpsw_common *common = dev_id;
+
+	disable_irq_nosync(irq);
+	napi_schedule(&common->napi_tx);
+
+	return IRQ_HANDLED;
+}
+
+static netdev_tx_t virt_cpsw_nuss_ndo_xmit(struct sk_buff *skb,
+					   struct net_device *ndev)
+{
+	struct virt_cpsw_common *common = virt_ndev_to_common(ndev);
+	struct device *dev = common->dev;
+	struct cppi5_host_desc_t *first_desc, *next_desc, *cur_desc;
+	struct virt_cpsw_tx_chn *tx_chn;
+	struct netdev_queue *netif_txq;
+	dma_addr_t desc_dma, buf_dma;
+	int ret, i;
+	u32 pkt_len;
+	void **swdata;
+	u32 *psdata;
+
+	/* frag list based linkage is not supported for now. */
+	if (skb_shinfo(skb)->frag_list) {
+		dev_err_ratelimited(dev, "NETIF_F_FRAGLIST not supported\n");
+		ret = -EINVAL;
+		goto drop_free_skb;
+	}
+
+	/* padding enabled in hw */
+	pkt_len = skb_headlen(skb);
+
+	tx_chn = &common->tx_chns;
+	netif_txq = netdev_get_tx_queue(ndev, 0);
+
+	/* Map the linear buffer */
+	buf_dma = dma_map_single(dev, skb->data, pkt_len,
+				 DMA_TO_DEVICE);
+	if (unlikely(dma_mapping_error(dev, buf_dma))) {
+		dev_err(dev, "Failed to map tx skb buffer\n");
+		ret = -EINVAL;
+		ndev->stats.tx_errors++;
+		goto drop_stop_q;
+	}
+
+	first_desc = k3_knav_pool_alloc(tx_chn->desc_pool);
+	if (!first_desc) {
+		dev_dbg(dev, "Failed to allocate descriptor\n");
+		dma_unmap_single(dev, buf_dma, pkt_len, DMA_TO_DEVICE);
+		goto drop_stop_q_busy;
+	}
+
+	cppi5_hdesc_init(first_desc, CPPI5_INFO0_HDESC_EPIB_PRESENT,
+			 VIRT_CPSW_NAV_PS_DATA_SIZE);
+	cppi5_desc_set_pktids(&first_desc->hdr, 0, 0x3FFF);
+	cppi5_hdesc_set_pkttype(first_desc, 0x7);
+	/* target port has to be 0 */
+	cppi5_desc_set_tags_ids(&first_desc->hdr, 0, 0);
+
+	cppi5_hdesc_attach_buf(first_desc, buf_dma, pkt_len, buf_dma, pkt_len);
+	swdata = cppi5_hdesc_get_swdata(first_desc);
+	*(swdata) = skb;
+	psdata = cppi5_hdesc_get_psdata32(first_desc);
+
+	/* HW csum offload if enabled */
+	psdata[2] = 0;
+	if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
+		unsigned int cs_start, cs_offset;
+
+		cs_start = skb_transport_offset(skb);
+		cs_offset = cs_start + skb->csum_offset;
+		/* HW numerates bytes starting from 1 */
+		psdata[2] = ((cs_offset + 1) << 24) |
+			    ((cs_start + 1) << 16) | (skb->len - cs_start);
+		dev_dbg(dev, "%s tx psdata:%#x\n", __func__, psdata[2]);
+	}
+
+	if (!skb_is_nonlinear(skb))
+		goto done_tx;
+
+	dev_dbg(dev, "fragmented SKB\n");
+
+	/* Handle the case where skb is fragmented in pages */
+	cur_desc = first_desc;
+	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
+		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
+		u32 frag_size = skb_frag_size(frag);
+
+		next_desc = k3_knav_pool_alloc(tx_chn->desc_pool);
+
+		if (!next_desc) {
+			dev_err(dev, "Failed to allocate descriptor\n");
+			ret = -ENOMEM;
+			goto drop_free_descs;
+		}
+
+		buf_dma = skb_frag_dma_map(dev, frag, 0, frag_size,
+					   DMA_TO_DEVICE);
+		if (unlikely(dma_mapping_error(dev, buf_dma))) {
+			dev_err(dev, "Failed to map tx skb page\n");
+			k3_knav_pool_free(tx_chn->desc_pool, next_desc);
+			ret = -EINVAL;
+			ndev->stats.tx_errors++;
+			goto drop_free_descs;
+		}
+
+		cppi5_hdesc_reset_hbdesc(next_desc);
+		cppi5_hdesc_attach_buf(next_desc,
+				       buf_dma, frag_size, buf_dma, frag_size);
+
+		desc_dma = k3_knav_pool_virt2dma(tx_chn->desc_pool, next_desc);
+		cppi5_hdesc_link_hbdesc(cur_desc, desc_dma);
+
+		pkt_len += frag_size;
+		cur_desc = next_desc;
+	}
+	WARN_ON(pkt_len != skb->len);
+
+done_tx:
+	skb_tx_timestamp(skb);
+
+	/* report bql before sending packet */
+	dev_dbg(dev, "push 0 %d Bytes\n", pkt_len);
+
+	netdev_tx_sent_queue(netif_txq, pkt_len);
+
+	cppi5_hdesc_set_pktlen(first_desc, pkt_len);
+	desc_dma = k3_knav_pool_virt2dma(tx_chn->desc_pool, first_desc);
+	ret = k3_nav_udmax_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma);
+	if (ret) {
+		dev_err(dev, "can't push desc %d\n", ret);
+		ndev->stats.tx_errors++;
+		goto drop_free_descs;
+	}
+
+	if (k3_knav_pool_avail(tx_chn->desc_pool) < MAX_SKB_FRAGS) {
+		netif_tx_stop_queue(netif_txq);
+		/* Barrier, so that stop_queue visible to other cpus */
+		smp_mb__after_atomic();
+		dev_err(dev, "netif_tx_stop_queue %d\n", 0);
+
+		/* re-check for smp */
+		if (k3_knav_pool_avail(tx_chn->desc_pool) >= MAX_SKB_FRAGS) {
+			netif_tx_wake_queue(netif_txq);
+			dev_err(dev, "netif_tx_wake_queue %d\n", 0);
+		}
+	}
+
+	return NETDEV_TX_OK;
+
+drop_free_descs:
+	virt_cpsw_nuss_xmit_free(tx_chn, dev, first_desc);
+drop_stop_q:
+	netif_tx_stop_queue(netif_txq);
+drop_free_skb:
+	ndev->stats.tx_dropped++;
+	dev_kfree_skb_any(skb);
+	return ret;
+
+drop_stop_q_busy:
+	netif_tx_stop_queue(netif_txq);
+	return NETDEV_TX_BUSY;
+}
+
+static void virt_cpsw_nuss_ndo_get_stats(struct net_device *dev,
+					 struct rtnl_link_stats64 *stats)
+{
+	struct virt_cpsw_ndev_priv *ndev_priv = netdev_priv(dev);
+	unsigned int start;
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+		struct virt_cpsw_ndev_stats *cpu_stats;
+		u64 rx_packets;
+		u64 rx_bytes;
+		u64 tx_packets;
+		u64 tx_bytes;
+
+		cpu_stats = per_cpu_ptr(ndev_priv->stats, cpu);
+		do {
+			start = u64_stats_fetch_begin_irq(&cpu_stats->syncp);
+			rx_packets = cpu_stats->rx_packets;
+			rx_bytes   = cpu_stats->rx_bytes;
+			tx_packets = cpu_stats->tx_packets;
+			tx_bytes   = cpu_stats->tx_bytes;
+		} while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start));
+
+		stats->rx_packets += rx_packets;
+		stats->rx_bytes   += rx_bytes;
+		stats->tx_packets += tx_packets;
+		stats->tx_bytes   += tx_bytes;
+	}
+
+	stats->rx_errors	= dev->stats.rx_errors;
+	stats->rx_dropped	= dev->stats.rx_dropped;
+	stats->tx_dropped	= dev->stats.tx_dropped;
+}
+
+static const struct net_device_ops virt_cpsw_nuss_netdev_ops = {
+	.ndo_open		= virt_cpsw_nuss_ndo_open,
+	.ndo_stop		= virt_cpsw_nuss_ndo_stop,
+	.ndo_start_xmit		= virt_cpsw_nuss_ndo_xmit,
+	.ndo_get_stats64        = virt_cpsw_nuss_ndo_get_stats,
+	.ndo_validate_addr	= eth_validate_addr,
+	.ndo_set_mac_address	= eth_mac_addr,
+	.ndo_tx_timeout		= virt_cpsw_nuss_ndo_host_tx_timeout,
+};
+
+static void virt_cpsw_nuss_get_drvinfo(struct net_device *ndev,
+				       struct ethtool_drvinfo *info)
+{
+	struct virt_cpsw_common *common = virt_ndev_to_common(ndev);
+	struct rpmsg_remotedev_eth_switch_ops *rdev_ops;
+	char fw_version[ETHTOOL_FWVERS_LEN];
+
+	rdev_ops = common->rdev_switch_ops;
+
+	strlcpy(info->driver, dev_driver_string(common->dev),
+		sizeof(info->driver));
+	strlcpy(info->version, VIRT_CPSW_DRV_VER, sizeof(info->version));
+	strlcpy(info->bus_info, dev_name(common->dev), sizeof(info->bus_info));
+
+	rdev_ops->get_fw_ver(common->rdev, fw_version, ETHTOOL_FWVERS_LEN);
+	strlcpy(info->fw_version, fw_version, ETHTOOL_FWVERS_LEN);
+}
+
+static const char virt_cpsw_nuss_ethtool_priv_flags[][ETH_GSTRING_LEN] = {
+	"RPMSG Ping test",
+	"RPMSG Read reg",
+	"RPMSG Dump stat",
+};
+
+static int
+virt_cpsw_nuss_get_sset_count(struct net_device __always_unused *ndev, int sset)
+{
+	switch (sset) {
+	case ETH_SS_TEST:
+		return ARRAY_SIZE(virt_cpsw_nuss_ethtool_priv_flags);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static void
+virt_cpsw_nuss_get_strings(struct net_device __always_unused *ndev,
+			   u32 stringset, u8 *data)
+{
+	switch (stringset) {
+	case ETH_SS_TEST:
+		memcpy(data, virt_cpsw_nuss_ethtool_priv_flags,
+		       sizeof(virt_cpsw_nuss_ethtool_priv_flags));
+		break;
+	}
+}
+
+static void virt_cpsw_nuss_self_test(struct net_device *ndev,
+				     struct ethtool_test *eth_test, u64 *data)
+{
+	struct virt_cpsw_common *common = virt_ndev_to_common(ndev);
+	struct device *dev = common->dev;
+	const char ping_data[] = "0123456789";
+	u32 reg_val;
+	int ret;
+
+	data[0] = 0;
+	ret = common->rdev_switch_ops->ping(common->rdev,
+					    ping_data, strlen(ping_data));
+	if (ret) {
+		dev_err(dev, "rpmsg ping fail %d\n", ret);
+		eth_test->flags |= ETH_TEST_FL_FAILED;
+		data[0] = 1;
+	}
+
+	data[1] = 0;
+	ret = common->rdev_switch_ops->read_reg(common->rdev,
+						0x0C000000, &reg_val);
+	if (ret) {
+		dev_err(dev, "rpmsg read_reg fail %d\n", ret);
+		eth_test->flags |= ETH_TEST_FL_FAILED;
+		data[1] = 1;
+	}
+	dev_dbg(dev, "read_reg rpmsg cpsw_nuss_ver - 0x0C000000:%08X\n",
+		reg_val);
+
+	ret = common->rdev_switch_ops->read_reg(common->rdev,
+						0x0C020000, &reg_val);
+	if (ret) {
+		dev_err(dev, "rpmsg read_reg fail %d\n", ret);
+		eth_test->flags |= ETH_TEST_FL_FAILED;
+		data[1] = 1;
+	}
+	dev_dbg(dev, "read_reg rpmsg cpsw_ver - 0x0C020000:%08X\n",
+		reg_val);
+
+	ret = 0;
+	data[2] = 0;
+	if (common->rdev_features & RPMSG_KDRV_ETHSWITCH_FEATURE_DUMP_STATS)
+		ret = common->rdev_switch_ops->dbg_dump_stats(common->rdev);
+	if (ret) {
+		dev_err(dev, "rpmsg dump_stats fail %d\n", ret);
+		eth_test->flags |= ETH_TEST_FL_FAILED;
+		data[2] = 1;
+	}
+}
+
+const struct ethtool_ops virt_cpsw_nuss_ethtool_ops = {
+	.get_drvinfo		= virt_cpsw_nuss_get_drvinfo,
+	.get_sset_count		= virt_cpsw_nuss_get_sset_count,
+	.get_strings		= virt_cpsw_nuss_get_strings,
+	.self_test		= virt_cpsw_nuss_self_test,
+	.get_link		= ethtool_op_get_link,
+};
+
+static void virt_cpsw_nuss_free_tx_chns(void *data)
+{
+	struct virt_cpsw_common *common = data;
+	struct virt_cpsw_tx_chn	*tx_chn = &common->tx_chns;
+
+	if (tx_chn->irq > 0)
+		k3_nav_udmax_tx_put_irq(tx_chn->tx_chn);
+
+	if (!IS_ERR_OR_NULL(tx_chn->tx_chn))
+		k3_nav_udmax_release_tx_chn(tx_chn->tx_chn);
+
+	if (!IS_ERR_OR_NULL(tx_chn->desc_pool))
+		k3_knav_pool_destroy(tx_chn->desc_pool);
+
+	memset(tx_chn, 0, sizeof(*tx_chn));
+}
+
+static int virt_cpsw_nuss_init_tx_chns(struct virt_cpsw_common *common)
+{
+	u32 max_desc_num = ALIGN(VIRT_CPSW_MAX_TX_DESC, MAX_SKB_FRAGS);
+	struct virt_cpsw_tx_chn	*tx_chn = &common->tx_chns;
+	struct k3_nav_udmax_tx_channel_cfg tx_cfg = { 0 };
+	struct device *dev = common->dev;
+	struct k3_ring_cfg ring_cfg = {
+		.elm_size = K3_RINGACC_RING_ELSIZE_8,
+		.mode = K3_RINGACC_RING_MODE_RING,
+		.flags = 0
+	};
+	char tx_chn_name[IFNAMSIZ];
+	u32 hdesc_size, tx_chn_num;
+	int ret, ret1;
+
+	/* convert to tx chn offset */
+	tx_chn_num = common->rdev_tx_psil_dst_id - common->tx_psil_id_base;
+	snprintf(tx_chn_name, sizeof(tx_chn_name), "tx%d", tx_chn_num);
+
+	init_completion(&common->tdown_complete);
+
+	hdesc_size = cppi5_hdesc_calc_size(true, VIRT_CPSW_NAV_PS_DATA_SIZE,
+					   VIRT_CPSW_NAV_SW_DATA_SIZE);
+
+	tx_cfg.swdata_size = VIRT_CPSW_NAV_SW_DATA_SIZE;
+	tx_cfg.tx_cfg = ring_cfg;
+	tx_cfg.txcq_cfg = ring_cfg;
+	tx_cfg.tx_cfg.size = max_desc_num;
+	tx_cfg.txcq_cfg.size = max_desc_num;
+
+	tx_chn->dev = dev;
+	tx_chn->id = 0;
+	tx_chn->descs_num = max_desc_num;
+	tx_chn->desc_pool = k3_knav_pool_create_name(dev,
+						     tx_chn->descs_num,
+						     hdesc_size,
+						     tx_chn_name);
+	if (IS_ERR(tx_chn->desc_pool)) {
+		ret = PTR_ERR(tx_chn->desc_pool);
+		dev_err(dev, "Failed to create poll %d\n", ret);
+		goto err;
+	}
+
+	tx_chn->tx_chn = k3_nav_udmax_request_tx_chn(dev, tx_chn_name, &tx_cfg);
+	if (IS_ERR(tx_chn->tx_chn)) {
+		ret = PTR_ERR(tx_chn->tx_chn);
+		dev_err(dev, "Failed to request tx dma channel %d\n",
+			ret);
+		goto err;
+	}
+
+	ret = k3_nav_udmax_tx_get_irq(tx_chn->tx_chn, &tx_chn->irq,
+				      IRQF_TRIGGER_HIGH, false, NULL);
+	if (ret)
+		dev_err(dev, "Failed to get tx dma irq %d\n", ret);
+
+err:
+	ret1 = devm_add_action(dev, virt_cpsw_nuss_free_tx_chns, common);
+	if (ret1) {
+		dev_err(dev, "failed to add free_tx_chns action %d", ret1);
+		return ret1;
+	}
+
+	return ret;
+}
+
+static void virt_cpsw_nuss_free_rx_chns(void *data)
+{
+	struct virt_cpsw_common *common = data;
+	struct virt_cpsw_rx_chn *rx_chn = &common->rx_chns;
+
+	if (!IS_ERR_OR_NULL(rx_chn->rx_chn)) {
+		k3_nav_udmax_rx_put_irq(rx_chn->rx_chn, 0);
+
+		k3_nav_udmax_release_rx_chn(rx_chn->rx_chn);
+	}
+
+	if (!IS_ERR_OR_NULL(rx_chn->desc_pool))
+		k3_knav_pool_destroy(rx_chn->desc_pool);
+}
+
+static int virt_cpsw_nuss_init_rx_chns(struct virt_cpsw_common *common)
+{
+	struct virt_cpsw_rx_chn *rx_chn = &common->rx_chns;
+	struct k3_nav_udmax_rx_channel_cfg rx_cfg = {0};
+	u32  max_desc_num = VIRT_CPSW_MAX_RX_DESC;
+	struct device *dev = common->dev;
+	u32 hdesc_size;
+	int ret = 0, ret1;
+	struct k3_ring_cfg rxring_cfg = {
+		.elm_size = K3_RINGACC_RING_ELSIZE_8,
+		.mode = K3_RINGACC_RING_MODE_MESSAGE,
+		.flags = 0,
+	};
+	struct k3_ring_cfg fdqring_cfg = {
+		.elm_size = K3_RINGACC_RING_ELSIZE_8,
+		.mode = K3_RINGACC_RING_MODE_MESSAGE,
+		.flags = 0,
+	};
+	struct k3_nav_udmax_rx_flow_cfg rx_flow_cfg = {
+		.rx_cfg = rxring_cfg,
+		.rxfdq_cfg = fdqring_cfg,
+		.ring_rxq_id = K3_RINGACC_RING_ID_ANY,
+		.ring_rxfdq0_id = K3_RINGACC_RING_ID_ANY,
+		.src_tag_lo_sel = K3_NAV_UDMAX_SRC_TAG_LO_USE_REMOTE_SRC_TAG,
+	};
+
+	hdesc_size = cppi5_hdesc_calc_size(true, VIRT_CPSW_NAV_PS_DATA_SIZE,
+					   VIRT_CPSW_NAV_SW_DATA_SIZE);
+
+	rx_cfg.swdata_size = VIRT_CPSW_NAV_SW_DATA_SIZE;
+	rx_cfg.flow_id_num = VIRT_CPSW_MAX_RX_FLOWS;
+	rx_cfg.flow_id_base = common->rdev_rx_flow_id;
+	rx_cfg.remote = true;
+
+	/* init all flows */
+	rx_chn->dev = dev;
+	rx_chn->descs_num = max_desc_num;
+	rx_chn->desc_pool = k3_knav_pool_create_name(dev, rx_chn->descs_num,
+						     hdesc_size, "rx");
+	if (IS_ERR(rx_chn->desc_pool)) {
+		ret = PTR_ERR(rx_chn->desc_pool);
+		dev_err(dev, "Failed to create rx poll %d\n", ret);
+		goto err;
+	}
+
+	rx_chn->rx_chn = k3_nav_udmax_request_rx_chn(dev, "rx", &rx_cfg);
+	if (IS_ERR(rx_chn->rx_chn)) {
+		ret = PTR_ERR(rx_chn->rx_chn);
+		dev_err(dev, "Failed to request rx dma channel %d\n", ret);
+		goto err;
+	}
+
+	common->rdev_rx_flow_id =
+			k3_nav_udmax_rx_get_flow_id_base(rx_chn->rx_chn);
+	dev_dbg(dev, "used flow-id-base %u\n", common->rdev_rx_flow_id);
+
+	rx_flow_cfg.rx_cfg.size = max_desc_num;
+	rx_flow_cfg.rxfdq_cfg.size = max_desc_num;
+	ret = k3_nav_udmax_rx_flow_init(rx_chn->rx_chn,
+					0, &rx_flow_cfg);
+	if (ret) {
+		dev_err(dev, "Failed to init rx flow%d %d\n", 0, ret);
+		goto err;
+	}
+
+	ret = k3_nav_udmax_rx_get_irq(rx_chn->rx_chn, 0, &rx_chn->irq,
+				      IRQF_TRIGGER_HIGH,
+				      false, -1);
+	if (ret) {
+		dev_err(dev, "Failed to get rx dma irq %d\n", ret);
+		goto err;
+	}
+
+err:
+	ret1 = devm_add_action(dev, virt_cpsw_nuss_free_rx_chns, common);
+	if (ret1) {
+		dev_err(dev, "failed to add free_rx_chns action %d", ret1);
+		return ret1;
+	}
+
+	return ret;
+}
+
+static int virt_cpsw_nuss_of(struct virt_cpsw_common *common)
+{
+	struct device *dev = common->dev;
+	struct device_node *port_np;
+	struct virt_cpsw_port *port;
+	const void *mac_addr;
+	int ret;
+
+	common->rdev_rx_flow_id = -1;
+	ret = of_property_read_u32(dev->of_node, "ti,rx-flow-id-base",
+				   &common->rdev_rx_flow_id);
+	if (!ret) {
+		dev_info(dev, "use rx-flow-id-base from DT %d\n",
+			 common->rdev_rx_flow_id);
+	}
+
+	ret = of_property_read_u32(dev->of_node, "ti,psil-base",
+				   &common->tx_psil_id_base);
+	if (ret) {
+		dev_err(dev, "ti,psil-base read fail %d\n", ret);
+		return ret;
+	}
+
+	port_np = of_get_child_by_name(dev->of_node, "virt_emac_port");
+	if (!port_np)
+		return -ENOENT;
+
+	port = &common->ports;
+	port->common = common;
+	port->name = of_get_property(port_np, "ti,label", NULL);
+
+	mac_addr = of_get_mac_address(port_np);
+	if (mac_addr && is_valid_ether_addr(mac_addr))
+		ether_addr_copy(port->local_mac_addr, mac_addr);
+
+	of_node_put(port_np);
+	return 0;
+}
+
+static int virt_cpsw_nuss_rdev_init(struct virt_cpsw_common *common)
+{
+	struct rpmsg_rdev_eth_switch_attach_ext_info attach_info = {0};
+	struct device *dev = common->dev;
+	int ret;
+
+	ret = common->rdev_switch_ops->attach_ext(common->rdev, &attach_info);
+	if (ret) {
+		dev_err(dev, "rpmsg attach - fail %d\n", ret);
+		return ret;
+	}
+	dev_dbg(dev, "rpmsg attach_ext - rx_mtu:%d features:%08X tx_mtu[0]:%d flow_idx:%d tx_cpsw_psil_dst_id:%d mac_addr:%pM\n",
+		attach_info.rx_mtu, attach_info.features,
+		attach_info.tx_mtu[0],
+		attach_info.flow_idx,
+		attach_info.tx_cpsw_psil_dst_id,
+		attach_info.mac_addr);
+	common->rdev_features = attach_info.features;
+	common->rdev_mtu = VIRT_CPSW_MAX_PACKET_SIZE;
+	common->rdev_tx_psil_dst_id = attach_info.tx_cpsw_psil_dst_id &
+				     (~0x8000);
+	if (common->rdev_rx_flow_id == -1)
+		common->rdev_rx_flow_id = attach_info.flow_idx;
+	ether_addr_copy(common->rdev_mac_addr, attach_info.mac_addr);
+
+	return 0;
+}
+
+static int virt_cpsw_nuss_init_ndev(struct virt_cpsw_common *common)
+{
+	struct virt_cpsw_ndev_priv *ndev_priv;
+	struct device *dev = common->dev;
+	struct virt_cpsw_port *port;
+	int ret;
+
+	port = &common->ports;
+
+	/* alloc netdev */
+	port->ndev = devm_alloc_etherdev_mqs(common->dev,
+					     sizeof(struct virt_cpsw_ndev_priv),
+					     1, 1);
+	if (!port->ndev) {
+		dev_err(dev, "error allocating net_device\n");
+		return -ENOMEM;
+	}
+
+	ndev_priv = netdev_priv(port->ndev);
+	ndev_priv->port = port;
+	SET_NETDEV_DEV(port->ndev, dev);
+
+	if (is_valid_ether_addr(port->local_mac_addr))
+		ether_addr_copy(port->ndev->dev_addr, port->local_mac_addr);
+	else if (is_valid_ether_addr(common->rdev_mac_addr))
+		ether_addr_copy(port->ndev->dev_addr, common->rdev_mac_addr);
+
+	port->ndev->min_mtu = VIRT_CPSW_MIN_PACKET_SIZE;
+	port->ndev->max_mtu = VIRT_CPSW_MAX_PACKET_SIZE;
+	port->ndev->hw_features = NETIF_F_SG |
+				  NETIF_F_RXCSUM;
+	port->ndev->features = port->ndev->hw_features;
+	port->ndev->vlan_features |=  NETIF_F_SG;
+	port->ndev->netdev_ops = &virt_cpsw_nuss_netdev_ops;
+	port->ndev->ethtool_ops = &virt_cpsw_nuss_ethtool_ops;
+
+	/* TX checksum offload if supported */
+	if (common->rdev_features & RPMSG_KDRV_ETHSWITCH_FEATURE_TXCSUM)
+		port->ndev->features |= NETIF_F_HW_CSUM;
+
+	ndev_priv->stats = netdev_alloc_pcpu_stats(struct virt_cpsw_ndev_stats);
+	if (!ndev_priv->stats)
+		return -ENOMEM;
+
+	ret = devm_add_action_or_reset(dev, (void(*)(void *))free_percpu,
+				       ndev_priv->stats);
+	if (ret) {
+		dev_err(dev, "failed to add percpu stat free action %d", ret);
+		return ret;
+	}
+
+	netif_tx_napi_add(port->ndev, &common->napi_tx,
+			  virt_cpsw_nuss_tx_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(port->ndev, &common->napi_rx,
+		       virt_cpsw_nuss_rx_poll, NAPI_POLL_WEIGHT);
+
+	ret = register_netdev(port->ndev);
+	if (ret)
+		dev_err(dev, "error registering slave net device %d\n", ret);
+
+	/* can't auto unregister ndev using devm_add_action() due to broken
+	 * devres release sequence in DD core
+	 */
+
+	return ret;
+}
+
+static void virt_cpsw_nuss_cleanup_ndev(struct virt_cpsw_common *common)
+{
+	if (common->ports.ndev)
+		unregister_netdev(common->ports.ndev);
+}
+
+static bool virt_cpsw_dev_check(const struct net_device *ndev)
+{
+	return ndev->netdev_ops == &virt_cpsw_nuss_netdev_ops;
+}
+
+static int virt_cpsw_inetaddr_event(struct notifier_block *unused,
+				    unsigned long event, void *ptr)
+{
+	struct in_ifaddr *ifa = (struct in_ifaddr *)ptr;
+	struct rpmsg_remotedev_eth_switch_ops *rdev_ops;
+	struct net_device *ndev = ifa->ifa_dev->dev;
+	struct virt_cpsw_common *common;
+	int ret = 0;
+
+	if (!virt_cpsw_dev_check(ndev))
+		goto out;
+
+	common = virt_ndev_to_common(ndev);
+	rdev_ops = common->rdev_switch_ops;
+	switch (event) {
+	case NETDEV_UP:
+		ret = rdev_ops->register_ipv4(common->rdev,
+					      ndev->dev_addr,
+					      ifa->ifa_address);
+		if (ret)
+			dev_err(common->dev, "register_ipv4 rpmsg - fail %d\n",
+				ret);
+		dev_dbg(common->dev, "NETDEV_UP %pI4 %s\n",
+			&ifa->ifa_address, ifa->ifa_label);
+		break;
+
+	case NETDEV_DOWN:
+		ret = rdev_ops->unregister_ipv4(common->rdev,
+						ifa->ifa_address);
+		if (ret)
+			dev_err(common->dev, "unregister_ipv4 rpmsg - fail %d\n",
+				ret);
+		dev_dbg(common->dev, "NETDEV_DOWN %pI4\n", &ifa->ifa_address);
+		break;
+	}
+
+out:
+	return notifier_from_errno(ret);
+}
+
+static struct notifier_block virt_cpsw_inetaddr_nb __read_mostly = {
+	.notifier_call = virt_cpsw_inetaddr_event,
+};
+
+static const struct of_device_id virt_cpsw_virt_of_mtable[] = {
+	{ .compatible = "ti,j721e-cpsw-virt-mac", },
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, virt_cpsw_virt_of_mtable);
+
+static int virt_cpsw_nuss_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct virt_cpsw_common *common;
+	int ret;
+
+	common = devm_kzalloc(dev, sizeof(struct virt_cpsw_common), GFP_KERNEL);
+	if (!common)
+		return -ENOMEM;
+	common->dev = dev;
+
+	ret = of_property_read_string(dev->of_node, "ti,remote-name",
+				      &common->rdev_name);
+	if (ret < 0) {
+		dev_info(dev, "remote-name is not set %d\n", ret);
+		return ret;
+	}
+
+	common->rdev = rpmsg_remotedev_get_named_device(common->rdev_name);
+	if (!common->rdev)
+		return -EPROBE_DEFER;
+	if (IS_ERR(common->rdev)) {
+		ret = PTR_ERR(common->rdev);
+		return ret;
+	}
+
+	common->rdev_switch_ops = common->rdev->device.eth_switch.ops;
+	ret =
+	devm_add_action_or_reset(dev,
+				 (void(*)(void *))rpmsg_remotedev_put_device,
+				 common->rdev);
+	if (ret) {
+		dev_err(dev, "add remotedev put device action fail:%d", ret);
+		return ret;
+	}
+
+	ret = virt_cpsw_nuss_of(common);
+	if (ret)
+		return ret;
+
+	ret = virt_cpsw_nuss_rdev_init(common);
+	if (ret)
+		return ret;
+	/* init tx channels */
+	ret = virt_cpsw_nuss_init_tx_chns(common);
+	if (ret)
+		return ret;
+	ret = virt_cpsw_nuss_init_rx_chns(common);
+	if (ret)
+		return ret;
+
+	if (common->tx_chns.irq == 0 || common->rx_chns.irq == 0)
+		return -ENXIO;
+
+	dev_set_drvdata(dev, common);
+
+	ret = virt_cpsw_nuss_init_ndev(common);
+	if (ret)
+		return ret;
+
+	ret = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(48));
+	if (ret) {
+		dev_err(dev, "error setting dma mask: %d\n", ret);
+		goto unreg_ndev;
+	}
+
+	ret = devm_request_irq(dev, common->tx_chns.irq,
+			       virt_cpsw_nuss_tx_irq,
+
+			       0, dev_name(dev), common);
+	if (ret) {
+		dev_err(dev, "failure requesting tx irq %u, %d\n",
+			common->tx_chns.irq, ret);
+		goto unreg_ndev;
+	}
+
+	ret = devm_request_irq(dev, common->rx_chns.irq,
+			       virt_cpsw_nuss_rx_irq,
+			       0, dev_name(dev), common);
+	if (ret) {
+		dev_err(dev, "failure requesting rx irq %u, %d\n",
+			common->rx_chns.irq, ret);
+		goto unreg_ndev;
+	}
+
+	register_inetaddr_notifier(&virt_cpsw_inetaddr_nb);
+
+	dev_info(common->dev, "virt_cpsw_nuss mac loaded\n");
+	dev_info(dev, "rdev_features:%08X rdev_mtu:%d flow_id:%d tx_psil_dst_id:%04X\n",
+		 common->rdev_features,
+		 common->rdev_mtu,
+		 common->rdev_rx_flow_id,
+		 common->rdev_tx_psil_dst_id);
+	dev_info(dev, "local_mac_addr:%pM rdev_mac_addr:%pM\n",
+		 common->ports.local_mac_addr,
+		 common->rdev_mac_addr);
+
+	return 0;
+
+unreg_ndev:
+	virt_cpsw_nuss_cleanup_ndev(common);
+	return ret;
+}
+
+static int virt_cpsw_nuss_remove(struct platform_device *pdev)
+{
+	struct virt_cpsw_common *common = platform_get_drvdata(pdev);
+	struct device *dev = common->dev;
+	int ret;
+
+	unregister_inetaddr_notifier(&virt_cpsw_inetaddr_nb);
+
+	/* must unregister ndevs here because DD release_driver routine calls
+	 * dma_deconfigure(dev) before devres_release_all(dev)
+	 */
+	virt_cpsw_nuss_cleanup_ndev(common);
+
+	ret = common->rdev_switch_ops->detach(common->rdev);
+	if (ret)
+		dev_err(dev, "rpmsg  detach - fail %d\n", ret);
+
+	return 0;
+}
+
+static struct platform_driver virt_cpsw_nuss_driver = {
+	.driver = {
+		.name = VIRT_CPSW_DRV_NAME,
+		.of_match_table = virt_cpsw_virt_of_mtable,
+	},
+	.probe = virt_cpsw_nuss_probe,
+	.remove = virt_cpsw_nuss_remove,
+};
+
+module_platform_driver(virt_cpsw_nuss_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Grygorii Strashko <grygorii.strashko@ti.com>");
+MODULE_DESCRIPTION("TI J721E VIRT CPSW Ethernet mac driver");
diff -Naur --no-dereference a/drivers/net/ethernet/ti/Kconfig b/drivers/net/ethernet/ti/Kconfig
--- a/drivers/net/ethernet/ti/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/ethernet/ti/Kconfig	2020-06-19 21:00:31.416227715 -0400
@@ -5,7 +5,7 @@
 config NET_VENDOR_TI
 	bool "Texas Instruments (TI) devices"
 	default y
-	depends on PCI || EISA || AR7 || ARCH_DAVINCI || ARCH_OMAP2PLUS || ARCH_KEYSTONE
+	depends on PCI || EISA || AR7 || ARCH_DAVINCI || ARCH_OMAP2PLUS || ARCH_KEYSTONE || ARCH_K3
 	---help---
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
@@ -30,7 +30,7 @@
 
 config TI_DAVINCI_MDIO
 	tristate "TI DaVinci MDIO Support"
-	depends on ARCH_DAVINCI || ARCH_OMAP2PLUS || ARCH_KEYSTONE || COMPILE_TEST
+	depends on ARCH_DAVINCI || ARCH_OMAP2PLUS || ARCH_KEYSTONE || ARCH_K3 || COMPILE_TEST
 	select PHYLIB
 	---help---
 	  This driver supports TI's DaVinci MDIO module.
@@ -92,6 +92,28 @@
 	imply PTP_1588_CLOCK
 	default m
 
+config TI_AM65_CPSW_NUSS
+	tristate "TI K3 CPSW Ethernet driver"
+	select TI_CPSW_ALE
+	select TI_DAVINCI_MDIO
+	select TI_CPSW_PHY_SEL
+	depends on ARCH_K3 && OF && TI_K3_UDMA_GLUE_LAYER
+	default n
+	---help---
+	  This driver supports K3 AM654 Ethernet Switch SubSystem Driver
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called ti-am65-cpsw-nuss.
+
+config TI_AM65_CPTS
+	tristate "TI K3 AM65x CPTS"
+	depends on ARCH_K3
+	default y if TI_AM65_CPSW_NUSS=y
+	select NET_PTP_CLASSIFY
+	imply PTP_1588_CLOCK
+	help
+	  Say y here to support the TI K3 AM65x CPTS with 1588 features
+
 config TI_KEYSTONE_NETCP
 	tristate "TI Keystone NETCP Core Support"
 	select TI_CPSW_ALE
@@ -135,4 +157,29 @@
 	---help---
 	  TI AR7 CPMAC Ethernet support
 
+config TI_PRUETH
+	tristate "TI PRU Ethernet EMAC driver"
+	depends on PRU_REMOTEPROC
+	---help---
+	  Support single or dual EMAC over PRUSS.
+
+config TI_ICSSG_PRUETH
+	tristate "TI Gigabit PRU Ethernet driver"
+	select TI_DAVINCI_MDIO
+	select NET_PTP_CLASSIFY
+	imply PTP_1588_CLOCK
+	depends on PRU_REMOTEPROC
+	depends on ARCH_K3 && OF && TI_K3_UDMA_GLUE_LAYER
+	---help---
+	  Support dual Gigabit Ethernet ports over the ICSSG PRU Subsystem
+	  This subsystem is available starting with the AM65 platform.
+
+config TI_RDEV_ETH_SWITCH_VIRT_EMAC
+	tristate "TI Virtual Eth MAC driver"
+	depends on ARCH_K3 && OF && TI_K3_UDMA_GLUE_LAYER
+	help
+	  Support for 1 Port Virtual Eth MAC driver over remotedev
+	  R5F Eth Switch FW RPMSG protocol.
+	  This is available starting with the J721E platform.
+
 endif # NET_VENDOR_TI
diff -Naur --no-dereference a/drivers/net/ethernet/ti/Makefile b/drivers/net/ethernet/ti/Makefile
--- a/drivers/net/ethernet/ti/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/ethernet/ti/Makefile	2020-06-19 21:00:31.416227715 -0400
@@ -20,4 +20,14 @@
 obj-$(CONFIG_TI_KEYSTONE_NETCP) += keystone_netcp.o
 keystone_netcp-y := netcp_core.o
 obj-$(CONFIG_TI_KEYSTONE_NETCP_ETHSS) += keystone_netcp_ethss.o
-keystone_netcp_ethss-y := netcp_ethss.o netcp_sgmii.o netcp_xgbepcsr.o
+keystone_netcp_ethss-y := netcp_ethss.o netcp_ethss_sysfs.o netcp_sgmii.o
+
+obj-$(CONFIG_TI_AM65_CPSW_NUSS) += ti-am65-cpsw-nuss.o
+ti-am65-cpsw-nuss-y := am65-cpsw-nuss.o cpsw_sl.o am65-cpsw-ethtool.o
+obj-$(CONFIG_TI_AM65_CPTS) += am65-cpts.o
+
+obj-$(CONFIG_TI_PRUETH) += prueth.o
+obj-$(CONFIG_TI_ICSSG_PRUETH) += icssg-prueth.o
+icssg-prueth-y := icssg_prueth.o icssg_classifier.o icssg_ethtool.o icssg_iep.o
+
+obj-$(CONFIG_TI_RDEV_ETH_SWITCH_VIRT_EMAC) += j721e-cpsw-virt-mac.o
diff -Naur --no-dereference a/drivers/net/ethernet/ti/netcp_ethss.c b/drivers/net/ethernet/ti/netcp_ethss.c
--- a/drivers/net/ethernet/ti/netcp_ethss.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/ethernet/ti/netcp_ethss.c	2020-06-19 21:00:31.416227715 -0400
@@ -19,35 +19,38 @@
  */
 
 #include <linux/io.h>
+#include <linux/mfd/syscon.h>
 #include <linux/module.h>
 #include <linux/of_mdio.h>
 #include <linux/of_net.h>
 #include <linux/of_address.h>
+#include <linux/regmap.h>
 #include <linux/if_vlan.h>
 #include <linux/ptp_classify.h>
 #include <linux/net_tstamp.h>
 #include <linux/ethtool.h>
+#include <linux/phy/phy.h>
 
 #include "cpsw.h"
-#include "cpsw_ale.h"
-#include "netcp.h"
 #include "cpts.h"
+#include "netcp_ethss.h"
 
 #define NETCP_DRIVER_NAME		"TI KeyStone Ethernet Driver"
 #define NETCP_DRIVER_VERSION		"v1.0"
 
-#define GBE_IDENT(reg)			((reg >> 16) & 0xffff)
-#define GBE_MAJOR_VERSION(reg)		(reg >> 8 & 0x7)
-#define GBE_MINOR_VERSION(reg)		(reg & 0xff)
-#define GBE_RTL_VERSION(reg)		((reg >> 11) & 0x1f)
-
 /* 1G Ethernet SS defines */
 #define GBE_MODULE_NAME			"netcp-gbe"
-#define GBE_SS_VERSION_14		0x4ed2
 
+/* for devicetree backward compatible only */
 #define GBE_SS_REG_INDEX		0
+
+#define GBE_SGMII_REG_INDEX		0
 #define GBE_SGMII34_REG_INDEX		1
 #define GBE_SM_REG_INDEX		2
+
+/* For 2U, index 0 points to Switch module register base */
+#define GBE_2U_SM_REG_INDEX		0
+
 /* offset relative to base of GBE_SS_REG_INDEX */
 #define GBE13_SGMII_MODULE_OFFSET	0x100
 /* offset relative to base of GBE_SM_REG_INDEX */
@@ -60,27 +63,21 @@
 #define GBE13_ALE_OFFSET		0x600
 #define GBE13_HOST_PORT_NUM		0
 #define GBE13_NUM_ALE_ENTRIES		1024
+/* offset relative to PCSR regmap */
+#define XGBE10_PCSR_OFFSET(x)		((x) * 0x80)
+#define XGBE10_PCSR_RX_STATUS(x)	(XGBE10_PCSR_OFFSET(x) + 0xc)
+
+#define XGBE10_PCSR_BLOCK_LOCK_MASK	BIT(30)
+#define XGBE10_PCSR_BLOCK_LOCK_SHIFT	30
 
 /* 1G Ethernet NU SS defines */
 #define GBENU_MODULE_NAME		"netcp-gbenu"
-#define GBE_SS_ID_NU			0x4ee6
-#define GBE_SS_ID_2U			0x4ee8
-
-#define IS_SS_ID_MU(d) \
-	((GBE_IDENT((d)->ss_version) == GBE_SS_ID_NU) || \
-	 (GBE_IDENT((d)->ss_version) == GBE_SS_ID_2U))
-
-#define IS_SS_ID_NU(d) \
-	(GBE_IDENT((d)->ss_version) == GBE_SS_ID_NU)
-
-#define IS_SS_ID_VER_14(d) \
-	(GBE_IDENT((d)->ss_version) == GBE_SS_VERSION_14)
-#define IS_SS_ID_2U(d) \
-	(GBE_IDENT((d)->ss_version) == GBE_SS_ID_2U)
 
-#define GBENU_SS_REG_INDEX		0
+#define GBENU_SGMII_REG_INDEX		0
 #define GBENU_SM_REG_INDEX		1
+/* offset relative to base of GBE_SS_REG_INDEX */
 #define GBENU_SGMII_MODULE_OFFSET	0x100
+/* offset relative to base of GBENU_SM_REG_INDEX */
 #define GBENU_HOST_PORT_OFFSET		0x1000
 #define GBENU_SLAVE_PORT_OFFSET		0x2000
 #define GBENU_EMAC_OFFSET		0x2330
@@ -88,13 +85,11 @@
 #define GBENU_CPTS_OFFSET		0x1d000
 #define GBENU_ALE_OFFSET		0x1e000
 #define GBENU_HOST_PORT_NUM		0
-#define GBENU_SGMII_MODULE_SIZE		0x100
 
 /* 10G Ethernet SS defines */
 #define XGBE_MODULE_NAME		"netcp-xgbe"
-#define XGBE_SS_VERSION_10		0x4ee4
 
-#define XGBE_SS_REG_INDEX		0
+#define XGBE_SGMII_REG_INDEX		0
 #define XGBE_SM_REG_INDEX		1
 #define XGBE_SERDES_REG_INDEX		2
 
@@ -144,25 +139,6 @@
 		(MACSL_XGIG_MODE | MACSL_XGMII_ENABLE |		\
 		 MACSL_ENABLE_EXT_CTL |	MACSL_RX_ENABLE_CSF)
 
-#define GBE_STATSA_MODULE			0
-#define GBE_STATSB_MODULE			1
-#define GBE_STATSC_MODULE			2
-#define GBE_STATSD_MODULE			3
-
-#define GBENU_STATS0_MODULE			0
-#define GBENU_STATS1_MODULE			1
-#define GBENU_STATS2_MODULE			2
-#define GBENU_STATS3_MODULE			3
-#define GBENU_STATS4_MODULE			4
-#define GBENU_STATS5_MODULE			5
-#define GBENU_STATS6_MODULE			6
-#define GBENU_STATS7_MODULE			7
-#define GBENU_STATS8_MODULE			8
-
-#define XGBE_STATS0_MODULE			0
-#define XGBE_STATS1_MODULE			1
-#define XGBE_STATS2_MODULE			2
-
 /* s: 0-based slave_port */
 #define SGMII_BASE(d, s) \
 	(((s) < 2) ? (d)->sgmii_port_regs : (d)->sgmii_port34_regs)
@@ -185,9 +161,10 @@
 		offsetof(struct gbenu##_##rb, rn)
 #define XGBE_SET_REG_OFS(p, rb, rn) p->rb##_ofs.rn = \
 		offsetof(struct xgbe##_##rb, rn)
-#define GBE_REG_ADDR(p, rb, rn) (p->rb + p->rb##_ofs.rn)
+#define GBE_REG_OFS(p, rb, rn) ((p)->rb##_ofs.rn)
 
 #define HOST_TX_PRI_MAP_DEFAULT			0x00000000
+#define SGMII_MODULE_SIZE			0x100
 
 #if IS_ENABLED(CONFIG_TI_CPTS)
 /* Px_TS_CTL register fields */
@@ -239,6 +216,7 @@
 #define EVENT_MSG_BITS (BIT(0) | BIT(1) | BIT(2) | BIT(3))
 #endif /* CONFIG_TI_CPTS */
 
+#define SGMII_MODULE_SIZE			0x100
 struct xgbe_ss_regs {
 	u32	id_ver;
 	u32	synce_count;
@@ -557,12 +535,6 @@
 	u32	synce_mux;
 };
 
-struct gbe_ss_regs_ofs {
-	u16	id_ver;
-	u16	control;
-	u16	rgmii_status; /* 2U */
-};
-
 struct gbe_switch_regs {
 	u32	id_ver;
 	u32	control;
@@ -576,16 +548,6 @@
 	u32	flow_control;
 };
 
-struct gbe_switch_regs_ofs {
-	u16	id_ver;
-	u16	control;
-	u16	soft_reset;
-	u16	emcontrol;
-	u16	stat_port_en;
-	u16	ptype;
-	u16	flow_control;
-};
-
 struct gbe_port_regs {
 	u32	max_blks;
 	u32	blk_cnt;
@@ -600,20 +562,6 @@
 	u32	ts_ctl2;
 };
 
-struct gbe_port_regs_ofs {
-	u16	port_vlan;
-	u16	tx_pri_map;
-	u16     rx_pri_map;
-	u16	sa_lo;
-	u16	sa_hi;
-	u16	ts_ctl;
-	u16	ts_seq_ltype;
-	u16	ts_vlan;
-	u16	ts_ctl_ltype2;
-	u16	ts_ctl2;
-	u16	rx_maxlen;	/* 2U, NU */
-};
-
 struct gbe_host_port_regs {
 	u32	src_id;
 	u32	port_vlan;
@@ -621,12 +569,6 @@
 	u32	rx_maxlen;
 };
 
-struct gbe_host_port_regs_ofs {
-	u16	port_vlan;
-	u16	tx_pri_map;
-	u16	rx_maxlen;
-};
-
 struct gbe_emac_regs {
 	u32	id_ver;
 	u32	mac_control;
@@ -641,12 +583,6 @@
 	u32	rsvd[6];
 };
 
-struct gbe_emac_regs_ofs {
-	u16	mac_control;
-	u16	soft_reset;
-	u16	rx_maxlen;
-};
-
 struct gbe_hw_stats {
 	u32	rx_good_frames;
 	u32	rx_broadcast_frames;
@@ -685,96 +621,8 @@
 	u32	rx_dma_overruns;
 };
 
-#define GBE_MAX_HW_STAT_MODS			9
 #define GBE_HW_STATS_REG_MAP_SZ			0x100
 
-struct ts_ctl {
-	int     uni;
-	u8      dst_port_map;
-	u8      maddr_map;
-	u8      ts_mcast_type;
-};
-
-struct gbe_slave {
-	void __iomem			*port_regs;
-	void __iomem			*emac_regs;
-	struct gbe_port_regs_ofs	port_regs_ofs;
-	struct gbe_emac_regs_ofs	emac_regs_ofs;
-	int				slave_num; /* 0 based logical number */
-	int				port_num;  /* actual port number */
-	atomic_t			link_state;
-	bool				open;
-	struct phy_device		*phy;
-	u32				link_interface;
-	u32				mac_control;
-	u8				phy_port_t;
-	struct device_node		*node;
-	struct device_node		*phy_node;
-	struct ts_ctl                   ts_ctl;
-	struct list_head		slave_list;
-};
-
-struct gbe_priv {
-	struct device			*dev;
-	struct netcp_device		*netcp_device;
-	struct timer_list		timer;
-	u32				num_slaves;
-	u32				ale_entries;
-	u32				ale_ports;
-	bool				enable_ale;
-	u8				max_num_slaves;
-	u8				max_num_ports; /* max_num_slaves + 1 */
-	u8				num_stats_mods;
-	struct netcp_tx_pipe		tx_pipe;
-
-	int				host_port;
-	u32				rx_packet_max;
-	u32				ss_version;
-	u32				stats_en_mask;
-
-	void __iomem			*ss_regs;
-	void __iomem			*switch_regs;
-	void __iomem			*host_port_regs;
-	void __iomem			*ale_reg;
-	void __iomem                    *cpts_reg;
-	void __iomem			*sgmii_port_regs;
-	void __iomem			*sgmii_port34_regs;
-	void __iomem			*xgbe_serdes_regs;
-	void __iomem			*hw_stats_regs[GBE_MAX_HW_STAT_MODS];
-
-	struct gbe_ss_regs_ofs		ss_regs_ofs;
-	struct gbe_switch_regs_ofs	switch_regs_ofs;
-	struct gbe_host_port_regs_ofs	host_port_regs_ofs;
-
-	struct cpsw_ale			*ale;
-	unsigned int			tx_queue_id;
-	const char			*dma_chan_name;
-
-	struct list_head		gbe_intf_head;
-	struct list_head		secondary_slaves;
-	struct net_device		*dummy_ndev;
-
-	u64				*hw_stats;
-	u32				*hw_stats_prev;
-	const struct netcp_ethtool_stat *et_stats;
-	int				num_et_stats;
-	/*  Lock for updating the hwstats */
-	spinlock_t			hw_stats_lock;
-
-	int                             cpts_registered;
-	struct cpts                     *cpts;
-};
-
-struct gbe_intf {
-	struct net_device	*ndev;
-	struct device		*dev;
-	struct gbe_priv		*gbe_dev;
-	struct netcp_tx_pipe	tx_pipe;
-	struct gbe_slave	*slave;
-	struct list_head	gbe_intf_list;
-	unsigned long		active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
-};
-
 static struct netcp_module gbe_module;
 static struct netcp_module xgbe_module;
 
@@ -1731,9 +1579,6 @@
 	XGBE_STATS2_INFO(rx_dma_overruns),
 };
 
-#define for_each_intf(i, priv) \
-	list_for_each_entry((i), &(priv)->gbe_intf_head, gbe_intf_list)
-
 #define for_each_sec_slave(slave, priv) \
 	list_for_each_entry((slave), &(priv)->secondary_slaves, slave_list)
 
@@ -1820,7 +1665,7 @@
 	}
 }
 
-static void gbe_reset_mod_stats(struct gbe_priv *gbe_dev, int stats_mod)
+void gbe_reset_mod_stats(struct gbe_priv *gbe_dev, int stats_mod)
 {
 	void __iomem *base = gbe_dev->hw_stats_regs[stats_mod];
 	u32  __iomem *p_stats_entry;
@@ -1889,7 +1734,7 @@
 	writel(val, GBE_REG_ADDR(gbe_dev, switch_regs, stat_port_en));
 }
 
-static void gbe_reset_mod_stats_ver14(struct gbe_priv *gbe_dev, int stats_mod)
+void gbe_reset_mod_stats_ver14(struct gbe_priv *gbe_dev, int stats_mod)
 {
 	gbe_stats_mod_visible_ver14(gbe_dev, stats_mod);
 	gbe_reset_mod_stats(gbe_dev, stats_mod);
@@ -2079,6 +1924,24 @@
 	return slave_num;
 }
 
+/* Number of GBE_TIMER_INTERVAL */
+#define LINK_RECOVER_THRESHOLD	6
+
+static void gbe_slave_link_recover(struct work_struct *work)
+{
+	struct gbe_slave *slave = container_of(work, struct gbe_slave,
+					       link_recover_work.work);
+	struct device *dev = slave->gbe_dev->dev;
+	int lane = slave->slave_num;
+	int ret;
+
+	dev_dbg(dev, "recovering serdes lane %d ...\n", lane);
+
+	ret = phy_reset(slave->serdes_phy);
+	if (!ret)
+		dev_dbg(dev, "Serdes Lane %u rx recovered\n", lane);
+}
+
 static void netcp_ethss_link_state_action(struct gbe_priv *gbe_dev,
 					  struct net_device *ndev,
 					  struct gbe_slave *slave,
@@ -2109,6 +1972,11 @@
 		    (slave->link_interface != RGMII_LINK_MAC_PHY) &&
 		    (slave->link_interface != XGMII_LINK_MAC_PHY)))
 			netif_carrier_on(ndev);
+
+		if (phy)
+			phy_print_status(phy);
+		else
+			netdev_printk(KERN_INFO, ndev, "Link is Up\n");
 	} else {
 		writel(mac_control, GBE_REG_ADDR(slave, emac_regs,
 						 mac_control));
@@ -2120,10 +1988,30 @@
 		    (slave->link_interface != RGMII_LINK_MAC_PHY) &&
 		    (slave->link_interface != XGMII_LINK_MAC_PHY)))
 			netif_carrier_off(ndev);
+
+		netdev_printk(KERN_INFO, ndev, "Link is Down\n");
 	}
 
-	if (phy)
-		phy_print_status(phy);
+	if (slave->link_interface == XGMII_LINK_MAC_MAC_FORCED ||
+	    (slave->link_interface == SGMII_LINK_MAC_MAC_FORCED && ndev)) {
+		if (up) {
+			if (slave->link_recover_thresh ||
+			    slave->link_recovering) {
+				slave->link_recover_thresh = 0;
+				slave->link_recovering = 0;
+				dev_info(gbe_dev->dev,
+					 "link_recover process cancelled: %s slave %d\n",
+					 netdev_name(ndev), slave->slave_num);
+			}
+		} else {
+			/* from up to down */
+			slave->link_recover_thresh = LINK_RECOVER_THRESHOLD;
+			slave->link_recovering = 1;
+			dev_info(gbe_dev->dev,
+				 "link_recover process initiated: %s slave %d\n",
+				 netdev_name(ndev), slave->slave_num);
+		}
+	}
 }
 
 static bool gbe_phy_link_status(struct gbe_slave *slave)
@@ -2133,12 +2021,21 @@
 
 #define RGMII_REG_STATUS_LINK	BIT(0)
 
-static void netcp_2u_rgmii_get_port_link(struct gbe_priv *gbe_dev, bool *status)
+static int netcp_2u_rgmii_get_port_link(struct gbe_priv *gbe_dev, bool *status)
 {
 	u32 val = 0;
+	int ret;
 
-	val = readl(GBE_REG_ADDR(gbe_dev, ss_regs, rgmii_status));
+	ret = regmap_read(gbe_dev->ss_regmap,
+			  GBE_REG_OFS(gbe_dev, ss_regs, rgmii_status), &val);
+	if (ret) {
+		dev_err(gbe_dev->dev,
+			"%s: error in reading rgmii status\n", __func__);
+		return ret;
+	}
 	*status = !!(val & RGMII_REG_STATUS_LINK);
+
+	return ret;
 }
 
 static void netcp_ethss_update_link_state(struct gbe_priv *gbe_dev,
@@ -2146,24 +2043,50 @@
 					  struct net_device *ndev)
 {
 	bool sw_link_state = true, phy_link_state;
-	int sp = slave->slave_num, link_state;
+	int sp = slave->slave_num, link_state, ret;
+	u32 pcsr_rx_stat;
 
 	if (!slave->open)
 		return;
 
-	if (SLAVE_LINK_IS_RGMII(slave))
-		netcp_2u_rgmii_get_port_link(gbe_dev,
-					     &sw_link_state);
+	if (SLAVE_LINK_IS_RGMII(slave)) {
+		ret = netcp_2u_rgmii_get_port_link(gbe_dev,
+						   &sw_link_state);
+		if (ret)
+			return;
+	}
 	if (SLAVE_LINK_IS_SGMII(slave))
 		sw_link_state =
 		netcp_sgmii_get_port_link(SGMII_BASE(gbe_dev, sp), sp);
 
+	if (SLAVE_LINK_IS_XGMII(slave) &&
+	    slave->link_interface == XGMII_LINK_MAC_MAC_FORCED) {
+		/* read status from pcsr status reg */
+		ret = regmap_read(gbe_dev->pcsr_regmap,
+				  XGBE10_PCSR_RX_STATUS(sp), &pcsr_rx_stat);
+		if (ret)
+			return;
+
+		sw_link_state = (pcsr_rx_stat & XGBE10_PCSR_BLOCK_LOCK_MASK) >>
+				 XGBE10_PCSR_BLOCK_LOCK_SHIFT;
+	}
+
 	phy_link_state = gbe_phy_link_status(slave);
 	link_state = phy_link_state & sw_link_state;
 
-	if (atomic_xchg(&slave->link_state, link_state) != link_state)
+	if (atomic_xchg(&slave->link_state, link_state) != link_state) {
 		netcp_ethss_link_state_action(gbe_dev, ndev, slave,
 					      link_state);
+	} else {
+		if (slave->link_recover_thresh) {
+			if (++slave->link_recovering >=
+					slave->link_recover_thresh) {
+				schedule_delayed_work(&slave->link_recover_work,
+						      0);
+				slave->link_recovering = 1;
+			}
+		}
+	}
 }
 
 static void xgbe_adjust_link(struct net_device *ndev)
@@ -2227,7 +2150,7 @@
 			    int max_rx_len)
 {
 	void __iomem *rx_maxlen_reg;
-	u32 xgmii_mode;
+	int ret;
 
 	if (max_rx_len > NETCP_MAX_FRAME_SIZE)
 		max_rx_len = NETCP_MAX_FRAME_SIZE;
@@ -2235,9 +2158,16 @@
 	/* Enable correct MII mode at SS level */
 	if (IS_SS_ID_XGBE(gbe_dev) &&
 	    (slave->link_interface >= XGMII_LINK_MAC_PHY)) {
-		xgmii_mode = readl(GBE_REG_ADDR(gbe_dev, ss_regs, control));
-		xgmii_mode |= (1 << slave->slave_num);
-		writel(xgmii_mode, GBE_REG_ADDR(gbe_dev, ss_regs, control));
+		ret = regmap_update_bits(gbe_dev->ss_regmap,
+					 GBE_REG_OFS(gbe_dev, ss_regs, control),
+					 BIT(slave->slave_num),
+					 BIT(slave->slave_num));
+
+		if (ret) {
+			dev_err(gbe_dev->dev,
+				"regmap update xgmii mode bit Failed\n");
+			return;
+		}
 	}
 
 	if (IS_SS_ID_MU(gbe_dev))
@@ -2564,7 +2494,7 @@
 	struct gbe_priv *gbe_dev = gbe_intf->gbe_dev;
 
 	if (!(skb_shinfo(p_info->skb)->tx_flags & SKBTX_HW_TSTAMP) ||
-	    !cpts_is_tx_enabled(gbe_dev->cpts))
+	    !gbe_dev->tx_ts_enabled)
 		return 0;
 
 	/* If phy has the txtstamp api, assume it will do it.
@@ -2598,8 +2528,9 @@
 		return 0;
 	}
 
-	cpts_rx_timestamp(gbe_dev->cpts, p_info->skb);
-	p_info->rxtstamp_complete = true;
+	if (gbe_dev->rx_ts_enabled &&
+	    !cpts_rx_timestamp(gbe_dev->cpts, p_info->skb))
+		p_info->rxtstamp_complete = true;
 
 	return 0;
 }
@@ -2614,10 +2545,8 @@
 		return -EOPNOTSUPP;
 
 	cfg.flags = 0;
-	cfg.tx_type = cpts_is_tx_enabled(cpts) ?
-		      HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF;
-	cfg.rx_filter = (cpts_is_rx_enabled(cpts) ?
-			 cpts->rx_enable : HWTSTAMP_FILTER_NONE);
+	cfg.tx_type = gbe_dev->tx_ts_enabled ? HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF;
+	cfg.rx_filter = gbe_dev->rx_ts_enabled;
 
 	return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
 }
@@ -2628,8 +2557,8 @@
 	struct gbe_slave *slave = gbe_intf->slave;
 	u32 ts_en, seq_id, ctl;
 
-	if (!cpts_is_rx_enabled(gbe_dev->cpts) &&
-	    !cpts_is_tx_enabled(gbe_dev->cpts)) {
+	if (!gbe_dev->rx_ts_enabled &&
+	    !gbe_dev->tx_ts_enabled) {
 		writel(0, GBE_REG_ADDR(slave, port_regs, ts_ctl));
 		return;
 	}
@@ -2641,10 +2570,10 @@
 		(slave->ts_ctl.uni ?  TS_UNI_EN :
 			slave->ts_ctl.maddr_map << TS_CTL_MADDR_SHIFT);
 
-	if (cpts_is_tx_enabled(gbe_dev->cpts))
+	if (gbe_dev->tx_ts_enabled)
 		ts_en |= (TS_TX_ANX_ALL_EN | TS_TX_VLAN_LT1_EN);
 
-	if (cpts_is_rx_enabled(gbe_dev->cpts))
+	if (gbe_dev->rx_ts_enabled)
 		ts_en |= (TS_RX_ANX_ALL_EN | TS_RX_VLAN_LT1_EN);
 
 	writel(ts_en,  GBE_REG_ADDR(slave, port_regs, ts_ctl));
@@ -2670,10 +2599,10 @@
 
 	switch (cfg.tx_type) {
 	case HWTSTAMP_TX_OFF:
-		cpts_tx_enable(cpts, 0);
+		gbe_dev->tx_ts_enabled = 0;
 		break;
 	case HWTSTAMP_TX_ON:
-		cpts_tx_enable(cpts, 1);
+		gbe_dev->tx_ts_enabled = 1;
 		break;
 	default:
 		return -ERANGE;
@@ -2681,12 +2610,12 @@
 
 	switch (cfg.rx_filter) {
 	case HWTSTAMP_FILTER_NONE:
-		cpts_rx_enable(cpts, 0);
+		gbe_dev->rx_ts_enabled = HWTSTAMP_FILTER_NONE;
 		break;
 	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
 	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
 	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
-		cpts_rx_enable(cpts, HWTSTAMP_FILTER_PTP_V1_L4_EVENT);
+		gbe_dev->rx_ts_enabled = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
 		cfg.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
 		break;
 	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
@@ -2698,7 +2627,7 @@
 	case HWTSTAMP_FILTER_PTP_V2_EVENT:
 	case HWTSTAMP_FILTER_PTP_V2_SYNC:
 	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
-		cpts_rx_enable(cpts, HWTSTAMP_FILTER_PTP_V2_EVENT);
+		gbe_dev->rx_ts_enabled = HWTSTAMP_FILTER_PTP_V2_EVENT;
 		cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
 		break;
 	default:
@@ -2775,6 +2704,27 @@
 }
 #endif /* CONFIG_TI_CPTS */
 
+static int init_serdes_phys(struct gbe_priv *gbe_dev, struct gbe_slave *slave,
+			    struct device_node *node, bool do_phy_init)
+{
+	struct device *dev = gbe_dev->dev;
+	struct phy *phy;
+
+	phy = devm_of_phy_get_by_index(dev, node, 0);
+	if (IS_ERR(phy)) {
+		/* this one may be disabled, quietly skip */
+		dev_dbg(dev, "%s sl-%d: No serdes phy found: %ld\n",
+			node->name, slave->slave_num, PTR_ERR(phy));
+		return 0;
+	}
+
+	slave->serdes_phy = phy;
+	if (!do_phy_init)
+		return 0;
+
+	return phy_init(phy);
+}
+
 static int gbe_set_rx_mode(void *intf_priv, bool promisc)
 {
 	struct gbe_intf *gbe_intf = intf_priv;
@@ -3020,6 +2970,7 @@
 	}
 
 	slave->node = node;
+	slave->gbe_dev = gbe_dev;
 	slave->open = false;
 	if ((slave->link_interface == SGMII_LINK_MAC_PHY) ||
 	    (slave->link_interface == RGMII_LINK_MAC_PHY) ||
@@ -3082,6 +3033,9 @@
 		GBE_SET_REG_OFS(slave, emac_regs, soft_reset);
 		GBE_SET_REG_OFS(slave, emac_regs, rx_maxlen);
 
+		if (slave->link_interface == SGMII_LINK_MAC_MAC_FORCED)
+			INIT_DELAYED_WORK(&slave->link_recover_work,
+					  gbe_slave_link_recover);
 	} else if (IS_SS_ID_MU(gbe_dev)) {
 		/* Initialize  slave port register offsets */
 		GBENU_SET_REG_OFS(slave, port_regs, port_vlan);
@@ -3100,6 +3054,10 @@
 		GBENU_SET_REG_OFS(slave, emac_regs, mac_control);
 		GBENU_SET_REG_OFS(slave, emac_regs, soft_reset);
 
+		if (slave->link_interface == SGMII_LINK_MAC_MAC_FORCED)
+			INIT_DELAYED_WORK(&slave->link_recover_work,
+					  gbe_slave_link_recover);
+
 	} else if (IS_SS_ID_XGBE(gbe_dev)) {
 		/* Initialize  slave port register offsets */
 		XGBE_SET_REG_OFS(slave, port_regs, port_vlan);
@@ -3116,6 +3074,8 @@
 		XGBE_SET_REG_OFS(slave, emac_regs, mac_control);
 		XGBE_SET_REG_OFS(slave, emac_regs, soft_reset);
 		XGBE_SET_REG_OFS(slave, emac_regs, rx_maxlen);
+		INIT_DELAYED_WORK(&slave->link_recover_work,
+				  gbe_slave_link_recover);
 	}
 
 	atomic_set(&slave->link_state, NETCP_LINK_STATE_INVALID);
@@ -3133,6 +3093,7 @@
 	struct device_node *port;
 	struct gbe_slave *slave;
 	bool mac_phy_link = false;
+	int ret;
 
 	for_each_child_of_node(node, port) {
 		slave = devm_kzalloc(dev, sizeof(*slave), GFP_KERNEL);
@@ -3150,7 +3111,16 @@
 			continue;
 		}
 
-		if (!IS_SS_ID_2U(gbe_dev))
+		if (!IS_SS_ID_2U(gbe_dev)) {
+			ret = init_serdes_phys(gbe_dev, slave, port, true);
+			if (ret && (ret != -ENODEV)) {
+				dev_err(dev, "serdes phy init failed\n");
+				devm_kfree(dev, slave);
+				continue;
+			}
+		}
+
+		if (IS_SS_ID_VER_14(gbe_dev) || IS_SS_ID_NU(gbe_dev))
 			gbe_sgmii_config(gbe_dev, slave);
 		gbe_port_reset(slave);
 		gbe_port_config(gbe_dev, slave, gbe_dev->rx_packet_max);
@@ -3221,6 +3191,8 @@
 	while (!list_empty(&gbe_dev->secondary_slaves)) {
 		slave = first_sec_slave(gbe_dev);
 
+		phy_exit(slave->serdes_phy);
+
 		if (slave->phy)
 			phy_disconnect(slave->phy);
 		list_del(&slave->slave_list);
@@ -3236,20 +3208,25 @@
 	void __iomem *regs;
 	int ret, i;
 
-	ret = of_address_to_resource(node, XGBE_SS_REG_INDEX, &res);
-	if (ret) {
+	gbe_dev->ss_regmap = syscon_regmap_lookup_by_phandle(node,
+							     "syscon-subsys");
+
+	if (IS_ERR(gbe_dev->ss_regmap)) {
 		dev_err(gbe_dev->dev,
-			"Can't xlate xgbe of node(%s) ss address at %d\n",
-			node->name, XGBE_SS_REG_INDEX);
-		return ret;
+			"subsys regmap lookup failed: %ld\n",
+			PTR_ERR(gbe_dev->ss_regmap));
+		return PTR_ERR(gbe_dev->ss_regmap);
 	}
 
-	regs = devm_ioremap_resource(gbe_dev->dev, &res);
-	if (IS_ERR(regs)) {
-		dev_err(gbe_dev->dev, "Failed to map xgbe ss register base\n");
-		return PTR_ERR(regs);
+	gbe_dev->pcsr_regmap = syscon_regmap_lookup_by_phandle(node,
+							       "syscon-pcsr");
+
+	if (IS_ERR(gbe_dev->pcsr_regmap)) {
+		dev_err(gbe_dev->dev,
+			"pcsr regmap lookup failed: %ld\n",
+			PTR_ERR(gbe_dev->pcsr_regmap));
+		return PTR_ERR(gbe_dev->pcsr_regmap);
 	}
-	gbe_dev->ss_regs = regs;
 
 	ret = of_address_to_resource(node, XGBE_SM_REG_INDEX, &res);
 	if (ret) {
@@ -3266,20 +3243,23 @@
 	}
 	gbe_dev->switch_regs = regs;
 
-	ret = of_address_to_resource(node, XGBE_SERDES_REG_INDEX, &res);
+	ret = of_address_to_resource(node, XGBE_SGMII_REG_INDEX, &res);
 	if (ret) {
 		dev_err(gbe_dev->dev,
-			"Can't xlate xgbe serdes of node(%s) address at %d\n",
-			node->name, XGBE_SERDES_REG_INDEX);
+			"Can't xlate xgbe of node(%s) sgmii address at %d\n",
+			node->name, XGBE_SGMII_REG_INDEX);
 		return ret;
 	}
 
 	regs = devm_ioremap_resource(gbe_dev->dev, &res);
 	if (IS_ERR(regs)) {
-		dev_err(gbe_dev->dev, "Failed to map xgbe serdes register base\n");
+		dev_err(gbe_dev->dev,
+			"Failed to map xgbe sgmii register base\n");
 		return PTR_ERR(regs);
 	}
-	gbe_dev->xgbe_serdes_regs = regs;
+	gbe_dev->sgmii_port_regs = regs;
+	gbe_dev->sgmii_port34_regs = gbe_dev->sgmii_port_regs +
+				     (2 * SGMII_MODULE_SIZE);
 
 	gbe_dev->num_stats_mods = gbe_dev->max_num_ports;
 	gbe_dev->et_stats = xgbe10_et_stats;
@@ -3304,9 +3284,9 @@
 	}
 
 	gbe_dev->ss_version = XGBE_SS_VERSION_10;
-	gbe_dev->sgmii_port_regs = gbe_dev->ss_regs +
-					XGBE10_SGMII_MODULE_OFFSET;
-	gbe_dev->host_port_regs = gbe_dev->ss_regs + XGBE10_HOST_PORT_OFFSET;
+
+	gbe_dev->host_port_regs = gbe_dev->switch_regs +
+					XGBE10_HOST_PORT_OFFSET;
 
 	for (i = 0; i < gbe_dev->max_num_ports; i++)
 		gbe_dev->hw_stats_regs[i] = gbe_dev->switch_regs +
@@ -3337,8 +3317,8 @@
 	return 0;
 }
 
-static int get_gbe_resource_version(struct gbe_priv *gbe_dev,
-				    struct device_node *node)
+static int get_gbe_resource_version_ss_regs(struct gbe_priv *gbe_dev,
+					    struct device_node *node)
 {
 	struct resource res;
 	void __iomem *regs;
@@ -3357,8 +3337,27 @@
 		dev_err(gbe_dev->dev, "Failed to map gbe register base\n");
 		return PTR_ERR(regs);
 	}
+
 	gbe_dev->ss_regs = regs;
 	gbe_dev->ss_version = readl(gbe_dev->ss_regs);
+	gbe_dev->ss_regmap = NULL;
+	return 0;
+}
+
+static int get_gbe_resource_version(struct gbe_priv *gbe_dev,
+				    struct device_node *node)
+{
+	gbe_dev->ss_regmap = syscon_regmap_lookup_by_phandle(node,
+							     "syscon-subsys");
+	if (IS_ERR(gbe_dev->ss_regmap)) {
+		dev_dbg(gbe_dev->dev,
+			"subsys regmap lookup failed: %ld. try reg property\n",
+			PTR_ERR(gbe_dev->ss_regmap));
+		return get_gbe_resource_version_ss_regs(gbe_dev, node);
+	}
+
+	regmap_read(gbe_dev->ss_regmap, 0, &gbe_dev->ss_version);
+	gbe_dev->ss_regs = NULL;
 	return 0;
 }
 
@@ -3369,6 +3368,27 @@
 	void __iomem *regs;
 	int i, ret;
 
+	if (gbe_dev->ss_regs) {
+		gbe_dev->sgmii_port_regs = gbe_dev->ss_regs +
+					   GBE13_SGMII_MODULE_OFFSET;
+	} else {
+		ret = of_address_to_resource(node, GBE_SGMII_REG_INDEX, &res);
+		if (ret) {
+			dev_err(gbe_dev->dev,
+				"Can't translate of gbe node(%s) address at index %d\n",
+				node->name, GBE_SGMII_REG_INDEX);
+			return ret;
+		}
+
+		regs = devm_ioremap_resource(gbe_dev->dev, &res);
+		if (IS_ERR(regs)) {
+			dev_err(gbe_dev->dev,
+				"Failed to map gbe sgmii port register base\n");
+			return PTR_ERR(regs);
+		}
+		gbe_dev->sgmii_port_regs = regs;
+	}
+
 	ret = of_address_to_resource(node, GBE_SGMII34_REG_INDEX, &res);
 	if (ret) {
 		dev_err(gbe_dev->dev,
@@ -3423,7 +3443,6 @@
 		return -ENOMEM;
 	}
 
-	gbe_dev->sgmii_port_regs = gbe_dev->ss_regs + GBE13_SGMII_MODULE_OFFSET;
 	gbe_dev->host_port_regs = gbe_dev->switch_regs + GBE13_HOST_PORT_OFFSET;
 
 	/* K2HK has only 2 hw stats modules visible at a time, so
@@ -3463,9 +3482,9 @@
 static int set_gbenu_ethss_priv(struct gbe_priv *gbe_dev,
 				struct device_node *node)
 {
+	int i, ret, sm_index = GBENU_SM_REG_INDEX;
 	struct resource res;
 	void __iomem *regs;
-	int i, ret;
 
 	gbe_dev->num_stats_mods = gbe_dev->max_num_ports;
 	gbe_dev->et_stats = gbenu_et_stats;
@@ -3495,11 +3514,14 @@
 		return -ENOMEM;
 	}
 
-	ret = of_address_to_resource(node, GBENU_SM_REG_INDEX, &res);
+	if (IS_SS_ID_2U(gbe_dev))
+		sm_index = GBE_2U_SM_REG_INDEX;
+
+	ret = of_address_to_resource(node, sm_index, &res);
 	if (ret) {
 		dev_err(gbe_dev->dev,
 			"Can't translate of gbenu node(%s) addr at index %d\n",
-			node->name, GBENU_SM_REG_INDEX);
+			node->name, sm_index);
 		return ret;
 	}
 
@@ -3511,16 +3533,37 @@
 	}
 	gbe_dev->switch_regs = regs;
 
-	if (!IS_SS_ID_2U(gbe_dev))
-		gbe_dev->sgmii_port_regs =
-		       gbe_dev->ss_regs + GBENU_SGMII_MODULE_OFFSET;
+	if (!IS_SS_ID_2U(gbe_dev)) {
+		if (gbe_dev->ss_regs) {
+			gbe_dev->sgmii_port_regs =
+				gbe_dev->ss_regs + GBENU_SGMII_MODULE_OFFSET;
+		} else {
+			ret = of_address_to_resource(node,
+						     GBENU_SGMII_REG_INDEX,
+						     &res);
+			if (ret) {
+				dev_err(gbe_dev->dev,
+					"Can't xslate of node(%s) addr at %d\n",
+					node->name, GBENU_SGMII_REG_INDEX);
+				return ret;
+			}
+
+			regs = devm_ioremap_resource(gbe_dev->dev, &res);
+			if (IS_ERR(regs)) {
+				dev_err(gbe_dev->dev,
+					"Failed to map gbenu sgmii reg base\n");
+				return PTR_ERR(regs);
+			}
+			gbe_dev->sgmii_port_regs = regs;
+		}
 
-	/* Although sgmii modules are mem mapped to one contiguous
-	 * region on GBENU devices, setting sgmii_port34_regs allows
-	 * consistent code when accessing sgmii api
-	 */
-	gbe_dev->sgmii_port34_regs = gbe_dev->sgmii_port_regs +
-				     (2 * GBENU_SGMII_MODULE_SIZE);
+		/* Although sgmii modules are mem mapped to one contiguous
+		 * region on GBENU devices, setting sgmii_port34_regs allows
+		 * consistent code when accessing sgmii api
+		 */
+		gbe_dev->sgmii_port34_regs = gbe_dev->sgmii_port_regs +
+				     (2 * SGMII_MODULE_SIZE);
+	}
 
 	gbe_dev->host_port_regs = gbe_dev->switch_regs + GBENU_HOST_PORT_OFFSET;
 
@@ -3564,6 +3607,7 @@
 	struct device_node *secondary_ports;
 	struct cpsw_ale_params ale_params;
 	struct gbe_priv *gbe_dev;
+	struct phy *phy;
 	u32 slave_num;
 	int i, ret = 0;
 
@@ -3637,10 +3681,6 @@
 
 	} else if (!strcmp(node->name, "xgbe")) {
 		ret = set_xgbe_ethss10_priv(gbe_dev, node);
-		if (ret)
-			return ret;
-		ret = netcp_xgbe_serdes_init(gbe_dev->xgbe_serdes_regs,
-					     gbe_dev->ss_regs);
 	} else {
 		dev_err(dev, "unknown GBE node(%s)\n", node->name);
 		ret = -ENODEV;
@@ -3669,12 +3709,19 @@
 	/* Create network interfaces */
 	INIT_LIST_HEAD(&gbe_dev->gbe_intf_head);
 	for_each_child_of_node(interfaces, interface) {
+		if (!IS_SS_ID_2U(gbe_dev)) {
+			phy = devm_of_phy_get_by_index(dev, interface, 0);
+			if (!IS_ERR(phy))
+				phy_init(phy);
+		}
+
 		ret = of_property_read_u32(interface, "slave-port", &slave_num);
 		if (ret) {
 			dev_err(dev, "missing slave-port parameter, skipping interface configuration for %s\n",
 				interface->name);
 			continue;
 		}
+
 		gbe_dev->num_slaves++;
 		if (gbe_dev->num_slaves >= gbe_dev->max_num_slaves) {
 			of_node_put(interface);
@@ -3737,10 +3784,14 @@
 	}
 	spin_unlock_bh(&gbe_dev->hw_stats_lock);
 
+	ret = gbe_create_sysfs_entries(gbe_dev);
+	if (ret)
+		goto free_sec_ports;
 	timer_setup(&gbe_dev->timer, netcp_ethss_timer, 0);
 	gbe_dev->timer.expires	 = jiffies + GBE_TIMER_INTERVAL;
 	add_timer(&gbe_dev->timer);
 	*inst_priv = gbe_dev;
+	dev_dbg(dev, "probed");
 	return 0;
 
 free_sec_ports:
@@ -3781,6 +3832,14 @@
 		goto fail;
 	}
 
+	if (!IS_SS_ID_2U(gbe_dev)) {
+		ret = init_serdes_phys(gbe_dev, gbe_intf->slave, node, false);
+		if (ret && (ret != -ENODEV)) {
+			dev_err(gbe_dev->dev, "serdes phy init failed\n");
+			goto fail;
+		}
+	}
+
 	gbe_intf->tx_pipe = gbe_dev->tx_pipe;
 	ndev->ethtool_ops = &keystone_ethtool_ops;
 	list_add_tail(&gbe_intf->gbe_intf_list, &gbe_dev->gbe_intf_head);
@@ -3799,6 +3858,7 @@
 {
 	struct gbe_intf *gbe_intf = intf_priv;
 
+	phy_exit(gbe_intf->slave->serdes_phy);
 	gbe_intf->ndev->ethtool_ops = NULL;
 	list_del(&gbe_intf->gbe_intf_list);
 	devm_kfree(gbe_intf->dev, gbe_intf->slave);
@@ -3814,6 +3874,7 @@
 	cpts_release(gbe_dev->cpts);
 	cpsw_ale_stop(gbe_dev->ale);
 	netcp_txpipe_close(&gbe_dev->tx_pipe);
+	gbe_remove_sysfs_entries(gbe_dev);
 	free_secondary_ports(gbe_dev);
 
 	if (!list_empty(&gbe_dev->gbe_intf_head))
diff -Naur --no-dereference a/drivers/net/ethernet/ti/netcp_ethss.h b/drivers/net/ethernet/ti/netcp_ethss.h
--- a/drivers/net/ethernet/ti/netcp_ethss.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/net/ethernet/ti/netcp_ethss.h	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,240 @@
+/*
+ * NetCP ethss header file
+ *
+ * Copyright (C) 2014 - 2019 Texas Instruments Incorporated
+ * Authors:	Sandeep Nair <sandeep_n@ti.com>
+ *		Sandeep Paulraj <s-paulraj@ti.com>
+ *		Cyril Chemparathy <cyril@ti.com>
+ *		Santosh Shilimkar <santosh.shilimkar@ti.com>
+ *		Wingman Kwok <w-kwok2@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation version 2.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef __NETCP_ETHSS_H__
+#define __NETCP_ETHSS_H__
+
+#include <linux/device.h>
+#include <linux/netdevice.h>
+#include <linux/if_vlan.h>
+#include <linux/io.h>
+#include <linux/kobject.h>
+#include <linux/list.h>
+#include <linux/phy/phy.h>
+#include <linux/spinlock.h>
+#include <linux/regmap.h>
+#include <linux/timer.h>
+#include <linux/ethtool.h>
+
+#include "cpsw_ale.h"
+#include "netcp.h"
+
+#define MAX_SLAVES				8
+
+struct gbe_ss_regs_ofs {
+	u16	id_ver;
+	u16	control;
+	u16	rgmii_status; /* 2U */
+};
+
+struct gbe_switch_regs_ofs {
+	u16	id_ver;
+	u16	control;
+	u16	soft_reset;
+	u16	emcontrol;
+	u16	stat_port_en;
+	u16	ptype;
+	u16	flow_control;
+};
+
+struct gbe_port_regs_ofs {
+	u16	port_vlan;
+	u16	tx_pri_map;
+	u16	rx_pri_map;
+	u16	sa_lo;
+	u16	sa_hi;
+	u16	ts_ctl;
+	u16	ts_seq_ltype;
+	u16	ts_vlan;
+	u16	ts_ctl_ltype2;
+	u16	ts_ctl2;
+	u16	rx_maxlen;	/* 2U, NU */
+};
+
+struct gbe_host_port_regs_ofs {
+	u16	port_vlan;
+	u16	tx_pri_map;
+	u16	rx_maxlen;
+};
+
+struct gbe_emac_regs_ofs {
+	u16	mac_control;
+	u16	mac_status;
+	u16	soft_reset;
+	u16	rx_maxlen;
+};
+
+#define GBE_MAX_HW_STAT_MODS			9
+
+struct cpts;
+
+struct gbe_priv {
+	struct device			*dev;
+	struct netcp_device		*netcp_device;
+	struct timer_list		timer;
+	u32				num_slaves;
+	u32				ale_entries;
+	u32				ale_ports;
+	int				enable_ale;
+	u8				max_num_slaves;
+	u8				max_num_ports; /* max_num_slaves + 1 */
+	u8				num_stats_mods;
+	struct netcp_tx_pipe		tx_pipe;
+
+	int				host_port;
+	u32				rx_packet_max;
+	u32				ss_version;
+	u32				stats_en_mask;
+
+	struct regmap			*ss_regmap;
+	struct regmap			*pcsr_regmap;
+	void __iomem                    *ss_regs;
+	void __iomem			*switch_regs;
+	void __iomem			*host_port_regs;
+	void __iomem			*ale_reg;
+	void __iomem                    *cpts_reg;
+	void __iomem			*sgmii_port_regs;
+	void __iomem			*sgmii_port34_regs;
+	void __iomem			*hw_stats_regs[GBE_MAX_HW_STAT_MODS];
+
+	struct gbe_ss_regs_ofs		ss_regs_ofs;
+	struct gbe_switch_regs_ofs	switch_regs_ofs;
+	struct gbe_host_port_regs_ofs	host_port_regs_ofs;
+
+	struct cpsw_ale			*ale;
+	unsigned int			tx_queue_id;
+	const char			*dma_chan_name;
+
+	struct list_head		gbe_intf_head;
+	struct list_head		secondary_slaves;
+	struct net_device		*dummy_ndev;
+
+	u64				*hw_stats;
+	u32				*hw_stats_prev;
+	const struct netcp_ethtool_stat *et_stats;
+	int				num_et_stats;
+	/*  Lock for updating the hwstats */
+	spinlock_t			hw_stats_lock;
+	int                             cpts_registered;
+	struct cpts                     *cpts;
+	int				rx_ts_enabled;
+	int				tx_ts_enabled;
+
+	struct kobject			kobj;
+	struct kobject			tx_pri_kobj;
+	struct kobject			pvlan_kobj;
+	struct kobject			port_ts_kobj[MAX_SLAVES];
+	struct kobject			stats_kobj;
+};
+
+struct ts_ctl {
+	int     uni;
+	u8      dst_port_map;
+	u8      maddr_map;
+	u8      ts_mcast_type;
+};
+
+struct gbe_slave {
+	struct gbe_priv			*gbe_dev;
+	void __iomem			*port_regs;
+	void __iomem			*emac_regs;
+	struct gbe_port_regs_ofs	port_regs_ofs;
+	struct gbe_emac_regs_ofs	emac_regs_ofs;
+	int				slave_num; /* 0 based logical number */
+	int				port_num;  /* actual port number */
+	atomic_t			link_state;
+	int				open;
+	struct phy_device		*phy;
+	u32				link_interface;
+	u32				mac_control;
+	u8				phy_port_t;
+					/* work trigger threshold
+					 *   0: triger disabled
+					 * > 1: trigger enabled
+					 */
+	u32				link_recover_thresh;
+					/* 0:NOT, > 0:recovering */
+	u32				link_recovering;
+	struct delayed_work		link_recover_work;
+	struct device_node		*node;
+	struct device_node		*phy_node;
+	struct ts_ctl                   ts_ctl;
+	struct list_head		slave_list;
+	struct phy			*serdes_phy;
+};
+
+struct gbe_intf {
+	struct net_device	*ndev;
+	struct device		*dev;
+	struct gbe_priv		*gbe_dev;
+	struct netcp_tx_pipe	tx_pipe;
+	struct gbe_slave	*slave;
+	struct list_head	gbe_intf_list;
+	unsigned long		active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
+};
+
+int gbe_create_sysfs_entries(struct gbe_priv *gbe_dev);
+void gbe_remove_sysfs_entries(struct gbe_priv *gbe_dev);
+void gbe_reset_mod_stats(struct gbe_priv *gbe_dev, int stats_mod);
+void gbe_reset_mod_stats_ver14(struct gbe_priv *gbe_dev, int stats_mod);
+
+#define for_each_intf(i, priv) \
+	list_for_each_entry((i), &(priv)->gbe_intf_head, gbe_intf_list)
+
+#define GBE_REG_ADDR(p, rb, rn) ((p)->rb + (p)->rb##_ofs.rn)
+#define GBE_MAJOR_VERSION(reg)		((reg) >> 8 & 0x7)
+#define GBE_MINOR_VERSION(reg)		((reg) & 0xff)
+#define GBE_RTL_VERSION(reg)		(((reg) >> 11) & 0x1f)
+#define GBE_IDENT(reg)			(((reg) >> 16) & 0xffff)
+#define GBE_SS_ID_NU			0x4ee6
+#define GBE_SS_ID_2U			0x4ee8
+#define GBE_SS_VERSION_14		0x4ed2
+#define XGBE_SS_VERSION_10		0x4ee4
+
+#define IS_SS_ID_MU(d) \
+	((GBE_IDENT((d)->ss_version) == GBE_SS_ID_NU) || \
+	 (GBE_IDENT((d)->ss_version) == GBE_SS_ID_2U))
+#define IS_SS_ID_NU(d) \
+	(GBE_IDENT((d)->ss_version) == GBE_SS_ID_NU)
+#define IS_SS_ID_VER_14(d) \
+	(GBE_IDENT((d)->ss_version) == GBE_SS_VERSION_14)
+#define IS_SS_ID_2U(d) \
+	(GBE_IDENT((d)->ss_version) == GBE_SS_ID_2U)
+
+#define GBE_STATSA_MODULE			0
+#define GBE_STATSB_MODULE			1
+#define GBE_STATSC_MODULE			2
+#define GBE_STATSD_MODULE			3
+
+#define GBENU_STATS0_MODULE			0
+#define GBENU_STATS1_MODULE			1
+#define GBENU_STATS2_MODULE			2
+#define GBENU_STATS3_MODULE			3
+#define GBENU_STATS4_MODULE			4
+#define GBENU_STATS5_MODULE			5
+#define GBENU_STATS6_MODULE			6
+#define GBENU_STATS7_MODULE			7
+#define GBENU_STATS8_MODULE			8
+
+#define XGBE_STATS0_MODULE			0
+#define XGBE_STATS1_MODULE			1
+#define XGBE_STATS2_MODULE			2
+
+#endif /* __NETCP_ETHSS_H */
diff -Naur --no-dereference a/drivers/net/ethernet/ti/netcp_ethss_sysfs.c b/drivers/net/ethernet/ti/netcp_ethss_sysfs.c
--- a/drivers/net/ethernet/ti/netcp_ethss_sysfs.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/net/ethernet/ti/netcp_ethss_sysfs.c	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,1403 @@
+/*
+ * Keystone GBE and XGBE sysfs driver code
+ *
+ * Copyright (C) 2016-2019 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * Authors:	Sandeep Nair <sandeep_n@ti.com>
+ *		Sandeep Paulraj <s-paulraj@ti.com>
+ *		Cyril Chemparathy <cyril@ti.com>
+ *		Santosh Shilimkar <santosh.shilimkar@ti.com>
+ *		Wingman Kwok <w-kwok2@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation version 2.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include "netcp_ethss.h"
+
+#define to_gbe_dev(obj) container_of(obj, struct gbe_priv, kobj)
+#define tx_pri_to_gbe_dev(obj) container_of(obj, struct gbe_priv, tx_pri_kobj)
+#define pvlan_to_gbe_dev(obj) container_of(obj, struct gbe_priv, pvlan_kobj)
+#define stats_to_gbe_dev(obj) container_of(obj, struct gbe_priv, stats_kobj)
+#define gbe_sw_mod_info_field_val(r, i) \
+	(((r) & BITMASK((i)->bits, (i)->shift)) >> (i)->shift)
+
+#define __GBE_SW_ATTR_FULL(_name, _mode, _show, _store, _info,	\
+				_info_size, _ctxt)		\
+	{ \
+		.attr = {.name = __stringify(_name), .mode = _mode },	\
+		.show	= _show,		\
+		.store	= _store,		\
+		.info	= _info,		\
+		.info_size = _info_size,	\
+		.context = (_ctxt),		\
+	}
+
+#define __GBE_SW_ATTR(_name, _mode, _show, _store, _info) \
+		__GBE_SW_ATTR_FULL(_name, _mode, _show, _store, _info, \
+					(ARRAY_SIZE(_info)), NULL)
+
+#define __GBE_SW_CTXT_ATTR(_name, _mode, _show, _store, _info, _ctxt) \
+		__GBE_SW_ATTR_FULL(_name, _mode, _show, _store, _info, \
+					(ARRAY_SIZE(_info)), _ctxt)
+
+#define BITS(x)			(BIT(x) - 1)
+#define BITMASK(n, s)		(BITS(n) << (s))
+
+enum gbe_sysfs_sw_entry {
+	GBE_SYSFS_SW_CONTROL,
+	GBE_SYSFS_SW_TX_PRIO,
+	GBE_SYSFS_SW_VLAN,
+	GBE_SYSFS_SW_STATS,
+	GBE_SYSFS_SW_MAX
+};
+
+struct gbe_sw_mod_info {
+	const char	*name;
+	int		shift;
+	int		bits;
+};
+
+struct gbe_sw_parse_result {
+	int control;
+	int port;
+	u32 value;
+};
+
+struct gbe_attribute {
+	struct attribute attr;
+	ssize_t (*show)(struct gbe_priv *gbe_dev,
+			struct gbe_attribute *attr, char *buf);
+	ssize_t	(*store)(struct gbe_priv *gbe_dev,
+			 struct gbe_attribute *attr,
+			 const char *buf, size_t size);
+	const struct gbe_sw_mod_info *info;
+	ssize_t info_size;
+	void *context;
+};
+
+#define to_gbe_attr(_attr) container_of(_attr, struct gbe_attribute, attr)
+
+static struct gbe_slave *gbe_port_num_get_slave(struct gbe_priv *gbe_dev,
+						int port)
+{
+	struct gbe_intf *gbe_intf;
+
+	for_each_intf(gbe_intf, gbe_dev) {
+		if (gbe_intf->slave->port_num == port)
+			return gbe_intf->slave;
+	}
+	return NULL;
+}
+
+static ssize_t gbe_sw_version_show(struct gbe_priv *gbe_dev,
+				   struct gbe_attribute *attr, char *buf)
+{
+	u32 reg;
+
+	reg = readl(GBE_REG_ADDR(gbe_dev, switch_regs, id_ver));
+
+	return snprintf(buf, PAGE_SIZE,
+		"\nGBE Switch Version %d.%d (%d) Identification value 0x%x\n",
+		 GBE_MAJOR_VERSION(reg), GBE_MINOR_VERSION(reg),
+		 GBE_RTL_VERSION(reg), GBE_IDENT(reg));
+}
+
+static struct gbe_attribute gbe_sw_version_attribute =
+	      __ATTR(version, 0444, gbe_sw_version_show, NULL);
+
+static const struct gbe_sw_mod_info gbe_sw_ver14_controls[] = {
+	{
+		.name		= "fifo_loopback",
+		.shift		= 0,
+		.bits		= 1,
+	},
+	{
+		.name		= "vlan_aware",
+		.shift		= 1,
+		.bits		= 1,
+	},
+	{
+		.name		= "p0_enable",
+		.shift		= 2,
+		.bits		= 1,
+	},
+	{
+		.name		= "p0_pass_pri_tagged",
+		.shift		= 3,
+		.bits		= 1,
+	},
+	{
+		.name		= "p1_pass_pri_tagged",
+		.shift		= 4,
+		.bits		= 1,
+	},
+	{
+		.name		= "p2_pass_pri_tagged",
+		.shift		= 5,
+		.bits		= 1,
+	},
+	{
+		.name		= "p3_pass_pri_tagged",
+		.shift		= 7,
+		.bits		= 1,
+	},
+	{
+		.name		= "p4_pass_pri_tagged",
+		.shift		= 8,
+		.bits		= 1,
+	},
+};
+
+static const struct gbe_sw_mod_info gbe_sw_xge_controls[] = {
+	{
+		.name		= "fifo_loopback",
+		.shift		= 0,
+		.bits		= 1,
+	},
+	{
+		.name		= "vlan_aware",
+		.shift		= 1,
+		.bits		= 1,
+	},
+	{
+		.name		= "p0_enable",
+		.shift		= 2,
+		.bits		= 1,
+	},
+	{
+		.name		= "p0_pass_pri_tagged",
+		.shift		= 3,
+		.bits		= 1,
+	},
+	{
+		.name		= "p1_pass_pri_tagged",
+		.shift		= 4,
+		.bits		= 1,
+	},
+	{
+		.name		= "p2_pass_pri_tagged",
+		.shift		= 5,
+		.bits		= 1,
+	},
+	{
+		.name		= "p0_tx_crc_type",
+		.shift		= 12,
+		.bits		= 1,
+	},
+};
+
+static const struct gbe_sw_mod_info gbe_sw_nu_controls[] = {
+	{
+		.name		= "vlan_aware",
+		.shift		= 1,
+		.bits		= 1,
+	},
+	{
+		.name		= "p0_enable",
+		.shift		= 2,
+		.bits		= 1,
+	},
+	{
+		.name		= "p0_pass_pri_tagged",
+		.shift		= 3,
+		.bits		= 1,
+	},
+	{
+		.name		= "p1_pass_pri_tagged",
+		.shift		= 4,
+		.bits		= 1,
+	},
+	{
+		.name		= "p2_pass_pri_tagged",
+		.shift		= 5,
+		.bits		= 1,
+	},
+	{
+		.name		= "p3_pass_pri_tagged",
+		.shift		= 6,
+		.bits		= 1,
+	},
+	{
+		.name		= "p4_pass_pri_tagged",
+		.shift		= 7,
+		.bits		= 1,
+	},
+	{
+		.name		= "p5_pass_pri_tagged",
+		.shift		= 8,
+		.bits		= 1,
+	},
+	{
+		.name		= "p6_pass_pri_tagged",
+		.shift		= 9,
+		.bits		= 1,
+	},
+	{
+		.name		= "p7_pass_pri_tagged",
+		.shift		= 10,
+		.bits		= 1,
+	},
+	{
+		.name		= "p8_pass_pri_tagged",
+		.shift		= 11,
+		.bits		= 1,
+	},
+	{
+		.name		= "p0_tx_crc_type",
+		.shift		= 12,
+		.bits		= 1,
+	},
+	{
+		.name		= "p0_rx_pad",
+		.shift		= 14,
+		.bits		= 1,
+	},
+	{
+		.name		= "p0_rx_pass_crc_err",
+		.shift		= 15,
+		.bits		= 1,
+	},
+};
+
+static inline void
+gbe_sw_info_set_reg_field(void __iomem *reg, const struct gbe_sw_mod_info *info,
+			  int val)
+{
+	u32 rv;
+
+	rv = readl(reg);
+	rv = ((rv & ~BITMASK(info->bits, info->shift)) | (val << info->shift));
+	writel(rv, reg);
+}
+
+static ssize_t
+gbe_sw_attr_parse_set_command(struct gbe_priv *gbe_dev,
+			      struct gbe_attribute *attr,
+			      const char *buf, size_t count,
+			      struct gbe_sw_parse_result *res)
+{
+	char ctrl_str[33], tmp_str[9];
+	int port = -1, value, len, control;
+	unsigned long end;
+	const struct gbe_sw_mod_info *info = attr->info;
+
+	len = strcspn(buf, ".=");
+	if (len >= 32)
+		return -ENOMEM;
+
+	strncpy(ctrl_str, buf, len);
+	ctrl_str[len] = '\0';
+	buf += len;
+
+	if (*buf == '.') {
+		++buf;
+		len = strcspn(buf, "=");
+		if (len >= 8)
+			return -ENOMEM;
+		strncpy(tmp_str, buf, len);
+		tmp_str[len] = '\0';
+		if (kstrtoul(tmp_str, 0, &end))
+			return -EINVAL;
+		port = (int)end;
+		buf += len;
+	}
+
+	if (*buf != '=')
+		return -EINVAL;
+
+	if (kstrtoul(buf + 1, 0, &end))
+		return -EINVAL;
+
+	value = (int)end;
+
+	for (control = 0; control < attr->info_size; control++)
+		if (strcmp(ctrl_str, info[control].name) == 0)
+			break;
+
+	if (control >= attr->info_size)
+		return -ENOENT;
+
+	res->control = control;
+	res->port = port;
+	res->value = value;
+
+	dev_info(gbe_dev->dev, "parsed command %s.%d=%d\n",
+		 attr->info[control].name, port, value);
+
+	return 0;
+}
+
+static ssize_t
+gbe_sw_attr_info_show(const struct gbe_sw_mod_info *info, int info_size,
+		      u32 reg_val, char *buf)
+{
+	int i, len = 0;
+
+	for (i = 0; i < info_size; i++, info++) {
+		len += snprintf(buf + len, PAGE_SIZE - len,
+			"%s=%d\n", info->name,
+			(int)gbe_sw_mod_info_field_val(reg_val, info));
+	}
+
+	return len;
+}
+
+static ssize_t gbe_sw_control_show(struct gbe_priv *gbe_dev,
+				   struct gbe_attribute *attr, char *buf)
+{
+	u32 reg_val = readl(GBE_REG_ADDR(gbe_dev, switch_regs, control));
+
+	return gbe_sw_attr_info_show(attr->info, attr->info_size, reg_val, buf);
+}
+
+static ssize_t
+gbe_sw_control_store(struct gbe_priv *gbe_dev, struct gbe_attribute *attr,
+		     const char *buf, size_t count)
+{
+	const struct gbe_sw_mod_info *info;
+	struct gbe_sw_parse_result res;
+	int ret;
+
+	ret = gbe_sw_attr_parse_set_command(gbe_dev, attr, buf, count, &res);
+	if (ret)
+		return ret;
+
+	info = &attr->info[res.control];
+
+	gbe_sw_info_set_reg_field(GBE_REG_ADDR(gbe_dev, switch_regs, control),
+				  info, res.value);
+	return count;
+}
+
+static struct gbe_attribute gbe_sw_ver14_control_attribute =
+	      __GBE_SW_ATTR(control, 0644, gbe_sw_control_show,
+			    gbe_sw_control_store, gbe_sw_ver14_controls);
+
+static struct gbe_attribute gbe_sw_xge_control_attribute =
+	      __GBE_SW_ATTR(control, 0644, gbe_sw_control_show,
+			    gbe_sw_control_store, gbe_sw_xge_controls);
+
+static struct gbe_attribute gbe_sw_nu_control_attribute =
+	      __GBE_SW_ATTR(control, 0644, gbe_sw_control_show,
+			    gbe_sw_control_store, gbe_sw_nu_controls);
+
+static const struct gbe_sw_mod_info gbe_sw_ver14_ptypes[] = {
+	{
+		.name		= "escalate_pri_load_val",
+		.shift		= 0,
+		.bits		= 5,
+	},
+	{
+		.name		= "port0_pri_type_escalate",
+		.shift		= 8,
+		.bits		= 1,
+	},
+	{
+		.name		= "port1_pri_type_escalate",
+		.shift		= 9,
+		.bits		= 1,
+	},
+	{
+		.name		= "port2_pri_type_escalate",
+		.shift		= 10,
+		.bits		= 1,
+	},
+	{
+		.name		= "port3_pri_type_escalate",
+		.shift		= 11,
+		.bits		= 1,
+	},
+	{
+		.name		= "port4_pri_type_escalate",
+		.shift		= 12,
+		.bits		= 1,
+	},
+};
+
+static const struct gbe_sw_mod_info gbe_sw_xge_ptypes[] = {
+	{
+		.name		= "escalate_pri_load_val",
+		.shift		= 0,
+		.bits		= 5,
+	},
+	{
+		.name		= "port0_pri_type_escalate",
+		.shift		= 8,
+		.bits		= 1,
+	},
+	{
+		.name		= "port1_pri_type_escalate",
+		.shift		= 9,
+		.bits		= 1,
+	},
+	{
+		.name		= "port2_pri_type_escalate",
+		.shift		= 10,
+		.bits		= 1,
+	},
+};
+
+static const struct gbe_sw_mod_info gbe_sw_nu_ptypes[] = {
+	{
+		.name		= "escalate_pri_load_val",
+		.shift		= 0,
+		.bits		= 5,
+	},
+	{
+		.name		= "port0_pri_type_escalate",
+		.shift		= 8,
+		.bits		= 1,
+	},
+	{
+		.name		= "port1_pri_type_escalate",
+		.shift		= 9,
+		.bits		= 1,
+	},
+	{
+		.name		= "port2_pri_type_escalate",
+		.shift		= 10,
+		.bits		= 1,
+	},
+	{
+		.name		= "port3_pri_type_escalate",
+		.shift		= 11,
+		.bits		= 1,
+	},
+	{
+		.name		= "port4_pri_type_escalate",
+		.shift		= 12,
+		.bits		= 1,
+	},
+	{
+		.name		= "port5_pri_type_escalate",
+		.shift		= 13,
+		.bits		= 1,
+	},
+	{
+		.name		= "port6_pri_type_escalate",
+		.shift		= 14,
+		.bits		= 1,
+	},
+	{
+		.name		= "port7_pri_type_escalate",
+		.shift		= 15,
+		.bits		= 1,
+	},
+	{
+		.name		= "port8_pri_type_escalate",
+		.shift		= 16,
+		.bits		= 1,
+	},
+};
+
+static ssize_t gbe_sw_pri_type_show(struct gbe_priv *gbe_dev,
+				    struct gbe_attribute *attr, char *buf)
+{
+	u32 reg_val = readl(GBE_REG_ADDR(gbe_dev, switch_regs, ptype));
+
+	return gbe_sw_attr_info_show(attr->info, attr->info_size, reg_val, buf);
+}
+
+static ssize_t gbe_sw_pri_type_store(struct gbe_priv *gbe_dev,
+				     struct gbe_attribute *attr,
+				     const char *buf, size_t count)
+{
+	const struct gbe_sw_mod_info *info;
+	struct gbe_sw_parse_result res;
+	int ret;
+
+	ret = gbe_sw_attr_parse_set_command(gbe_dev, attr, buf, count, &res);
+	if (ret)
+		return ret;
+
+	info = &attr->info[res.control];
+
+	gbe_sw_info_set_reg_field(GBE_REG_ADDR(gbe_dev, switch_regs, ptype),
+				  info, res.value);
+	return count;
+}
+
+static struct gbe_attribute gbe_sw_ver14_pri_type_attribute =
+			__GBE_SW_ATTR(priority_type, 0644,
+				      gbe_sw_pri_type_show,
+				      gbe_sw_pri_type_store,
+				      gbe_sw_ver14_ptypes);
+
+static struct gbe_attribute gbe_sw_xge_pri_type_attribute =
+			__GBE_SW_ATTR(priority_type, 0644,
+				      gbe_sw_pri_type_show,
+				      gbe_sw_pri_type_store,
+				      gbe_sw_xge_ptypes);
+
+static struct gbe_attribute gbe_sw_nu_pri_type_attribute =
+			__GBE_SW_ATTR(priority_type, 0644,
+				      gbe_sw_pri_type_show,
+				      gbe_sw_pri_type_store,
+				      gbe_sw_nu_ptypes);
+
+static const struct gbe_sw_mod_info gbe_sw_ver14_flow_controls[] = {
+	{
+		.name		= "port0_flow_control_en",
+		.shift		= 0,
+		.bits		= 1,
+	},
+	{
+		.name		= "port1_flow_control_en",
+		.shift		= 1,
+		.bits		= 1,
+	},
+	{
+		.name		= "port2_flow_control_en",
+		.shift		= 2,
+		.bits		= 1,
+	},
+	{
+		.name		= "port3_flow_control_en",
+		.shift		= 3,
+		.bits		= 1,
+	},
+	{
+		.name		= "port4_flow_control_en",
+		.shift		= 4,
+		.bits		= 1,
+	},
+};
+
+static const struct gbe_sw_mod_info gbe_sw_xge_flow_controls[] = {
+	{
+		.name		= "port0_flow_control_en",
+		.shift		= 0,
+		.bits		= 1,
+	},
+	{
+		.name		= "port1_flow_control_en",
+		.shift		= 1,
+		.bits		= 1,
+	},
+	{
+		.name		= "port2_flow_control_en",
+		.shift		= 2,
+		.bits		= 1,
+	},
+};
+
+static ssize_t gbe_sw_flow_control_show(struct gbe_priv *gbe_dev,
+					struct gbe_attribute *attr, char *buf)
+{
+	u32 reg_val = readl(GBE_REG_ADDR(gbe_dev, switch_regs, flow_control));
+
+	return gbe_sw_attr_info_show(attr->info, attr->info_size, reg_val, buf);
+}
+
+static ssize_t gbe_sw_flow_control_store(struct gbe_priv *gbe_dev,
+					 struct gbe_attribute *attr,
+					 const char *buf, size_t count)
+{
+	const struct gbe_sw_mod_info *info;
+	struct gbe_sw_parse_result res;
+	int ret;
+
+	ret = gbe_sw_attr_parse_set_command(gbe_dev, attr, buf, count, &res);
+	if (ret)
+		return ret;
+
+	info = &attr->info[res.control];
+
+	gbe_sw_info_set_reg_field(GBE_REG_ADDR(gbe_dev, switch_regs,
+					       flow_control),
+					       info, res.value);
+	return count;
+}
+
+static struct gbe_attribute gbe_sw_ver14_flow_control_attribute =
+			__GBE_SW_ATTR(flow_control, 0644,
+				      gbe_sw_flow_control_show,
+				      gbe_sw_flow_control_store,
+				      gbe_sw_ver14_flow_controls);
+
+static struct gbe_attribute gbe_sw_xge_flow_control_attribute =
+			__GBE_SW_ATTR(flow_control, 0644,
+				      gbe_sw_flow_control_show,
+				      gbe_sw_flow_control_store,
+				      gbe_sw_xge_flow_controls);
+
+static struct attribute *gbe_sw_ver14_default_attrs[] = {
+	&gbe_sw_version_attribute.attr,
+	&gbe_sw_ver14_control_attribute.attr,
+	&gbe_sw_ver14_pri_type_attribute.attr,
+	&gbe_sw_ver14_flow_control_attribute.attr,
+	NULL
+};
+
+static struct attribute *gbe_sw_xge_default_attrs[] = {
+	&gbe_sw_version_attribute.attr,
+	&gbe_sw_xge_control_attribute.attr,
+	&gbe_sw_xge_pri_type_attribute.attr,
+	&gbe_sw_xge_flow_control_attribute.attr,
+	NULL
+};
+
+static struct attribute *gbe_sw_nu_default_attrs[] = {
+	&gbe_sw_version_attribute.attr,
+	&gbe_sw_nu_control_attribute.attr,
+	&gbe_sw_nu_pri_type_attribute.attr,
+	NULL
+};
+
+static const struct gbe_sw_mod_info gbe_sw_port_tx_pri_maps[] = {
+	{
+		.name		= "port_tx_pri_0",
+		.shift		= 0,
+		.bits		= 3,
+	},
+	{
+		.name		= "port_tx_pri_1",
+		.shift		= 4,
+		.bits		= 3,
+	},
+	{
+		.name		= "port_tx_pri_2",
+		.shift		= 8,
+		.bits		= 3,
+	},
+	{
+		.name		= "port_tx_pri_3",
+		.shift		= 12,
+		.bits		= 3,
+	},
+	{
+		.name		= "port_tx_pri_4",
+		.shift		= 16,
+		.bits		= 3,
+	},
+	{
+		.name		= "port_tx_pri_5",
+		.shift		= 20,
+		.bits		= 3,
+	},
+	{
+		.name		= "port_tx_pri_6",
+		.shift		= 24,
+		.bits		= 3,
+	},
+	{
+		.name		= "port_tx_pri_7",
+		.shift		= 28,
+		.bits		= 3,
+	},
+};
+
+static ssize_t gbe_sw_port_tx_pri_map_show(struct gbe_priv *gbe_dev,
+					   struct gbe_attribute *attr,
+					   char *buf)
+{
+	int len = 0, total_len = 0, port;
+	struct gbe_slave *slave;
+	u32 reg_val;
+
+	port = (int)(attr->context);
+
+	slave = gbe_port_num_get_slave(gbe_dev, port);
+	if (!slave)
+		return 0;
+
+	reg_val = readl(GBE_REG_ADDR(slave, port_regs, tx_pri_map));
+	len = gbe_sw_attr_info_show(attr->info, attr->info_size, reg_val, buf);
+
+	return (total_len += len);
+}
+
+static ssize_t gbe_sw_port_tx_pri_map_store(struct gbe_priv *gbe_dev,
+					    struct gbe_attribute *attr,
+					    const char *buf, size_t count)
+{
+	const struct gbe_sw_mod_info *info;
+	struct gbe_sw_parse_result res;
+	void __iomem *reg = NULL;
+	struct gbe_slave *slave;
+	int ret, port;
+
+	port = (int)(attr->context);
+
+	slave = gbe_port_num_get_slave(gbe_dev, port);
+	if (!slave)
+		return -EINVAL;
+
+	ret = gbe_sw_attr_parse_set_command(gbe_dev, attr, buf, count, &res);
+	if (ret)
+		return ret;
+
+	info = &attr->info[res.control];
+
+	reg = GBE_REG_ADDR(slave, port_regs, tx_pri_map);
+	if (!reg)
+		return  -ENOENT;
+
+	gbe_sw_info_set_reg_field(reg, info, res.value);
+
+	return count;
+}
+
+static struct gbe_attribute gbe_sw_tx_pri_0_attribute =
+			__GBE_SW_CTXT_ATTR(0, 0644,
+					   gbe_sw_port_tx_pri_map_show,
+					   gbe_sw_port_tx_pri_map_store,
+					   gbe_sw_port_tx_pri_maps, (void *)0);
+
+static struct gbe_attribute gbe_sw_tx_pri_1_attribute =
+			__GBE_SW_CTXT_ATTR(1, 0644,
+					   gbe_sw_port_tx_pri_map_show,
+					   gbe_sw_port_tx_pri_map_store,
+					   gbe_sw_port_tx_pri_maps, (void *)1);
+
+static struct gbe_attribute gbe_sw_tx_pri_2_attribute =
+			__GBE_SW_CTXT_ATTR(2, 0644,
+					   gbe_sw_port_tx_pri_map_show,
+					   gbe_sw_port_tx_pri_map_store,
+					   gbe_sw_port_tx_pri_maps, (void *)2);
+
+static struct gbe_attribute gbe_sw_tx_pri_3_attribute =
+			__GBE_SW_CTXT_ATTR(3, 0644,
+					   gbe_sw_port_tx_pri_map_show,
+					   gbe_sw_port_tx_pri_map_store,
+					   gbe_sw_port_tx_pri_maps, (void *)3);
+
+static struct gbe_attribute gbe_sw_tx_pri_4_attribute =
+			__GBE_SW_CTXT_ATTR(4, 0644,
+					   gbe_sw_port_tx_pri_map_show,
+					   gbe_sw_port_tx_pri_map_store,
+					   gbe_sw_port_tx_pri_maps, (void *)4);
+
+static struct gbe_attribute gbe_sw_tx_pri_5_attribute =
+			__GBE_SW_CTXT_ATTR(5, 0644,
+					   gbe_sw_port_tx_pri_map_show,
+					   gbe_sw_port_tx_pri_map_store,
+					   gbe_sw_port_tx_pri_maps, (void *)5);
+
+static struct gbe_attribute gbe_sw_tx_pri_6_attribute =
+			__GBE_SW_CTXT_ATTR(6, 0644,
+					   gbe_sw_port_tx_pri_map_show,
+					   gbe_sw_port_tx_pri_map_store,
+					   gbe_sw_port_tx_pri_maps, (void *)6);
+
+static struct gbe_attribute gbe_sw_tx_pri_7_attribute =
+			__GBE_SW_CTXT_ATTR(7, 0644,
+					   gbe_sw_port_tx_pri_map_show,
+					   gbe_sw_port_tx_pri_map_store,
+					   gbe_sw_port_tx_pri_maps, (void *)7);
+
+static struct gbe_attribute gbe_sw_tx_pri_8_attribute =
+			__GBE_SW_CTXT_ATTR(8, 0644,
+					   gbe_sw_port_tx_pri_map_show,
+					   gbe_sw_port_tx_pri_map_store,
+					   gbe_sw_port_tx_pri_maps, (void *)8);
+
+static struct attribute *gbe_sw_ver14_tx_pri_default_attrs[] = {
+	&gbe_sw_tx_pri_1_attribute.attr,
+	&gbe_sw_tx_pri_2_attribute.attr,
+	&gbe_sw_tx_pri_3_attribute.attr,
+	&gbe_sw_tx_pri_4_attribute.attr,
+	NULL
+};
+
+static struct attribute *gbe_sw_xge_tx_pri_default_attrs[] = {
+	&gbe_sw_tx_pri_0_attribute.attr,
+	&gbe_sw_tx_pri_1_attribute.attr,
+	&gbe_sw_tx_pri_2_attribute.attr,
+	NULL
+};
+
+static struct attribute *gbe_sw_nu_tx_pri_default_attrs[] = {
+	&gbe_sw_tx_pri_1_attribute.attr,
+	&gbe_sw_tx_pri_2_attribute.attr,
+	&gbe_sw_tx_pri_3_attribute.attr,
+	&gbe_sw_tx_pri_4_attribute.attr,
+	&gbe_sw_tx_pri_5_attribute.attr,
+	&gbe_sw_tx_pri_6_attribute.attr,
+	&gbe_sw_tx_pri_7_attribute.attr,
+	&gbe_sw_tx_pri_8_attribute.attr,
+	NULL
+};
+
+static ssize_t gbe_sw_tx_pri_attr_show(struct kobject *kobj,
+				       struct attribute *attr, char *buf)
+{
+	struct gbe_attribute *attribute = to_gbe_attr(attr);
+	struct gbe_priv *gbe_dev = tx_pri_to_gbe_dev(kobj);
+
+	if (!attribute->show)
+		return -EIO;
+
+	return attribute->show(gbe_dev, attribute, buf);
+}
+
+static ssize_t gbe_sw_tx_pri_attr_store(struct kobject *kobj,
+					struct attribute *attr,
+					const char *buf, size_t count)
+{
+	struct gbe_attribute *attribute = to_gbe_attr(attr);
+	struct gbe_priv *gbe_dev = tx_pri_to_gbe_dev(kobj);
+
+	if (!attribute->store)
+		return -EIO;
+
+	return attribute->store(gbe_dev, attribute, buf, count);
+}
+
+static const struct sysfs_ops gbe_sw_tx_pri_sysfs_ops = {
+	.show = gbe_sw_tx_pri_attr_show,
+	.store = gbe_sw_tx_pri_attr_store,
+};
+
+static struct kobj_type gbe_sw_ver14_tx_pri_ktype = {
+	.sysfs_ops = &gbe_sw_tx_pri_sysfs_ops,
+	.default_attrs = gbe_sw_ver14_tx_pri_default_attrs,
+};
+
+static struct kobj_type gbe_sw_xge_tx_pri_ktype = {
+	.sysfs_ops = &gbe_sw_tx_pri_sysfs_ops,
+	.default_attrs = gbe_sw_xge_tx_pri_default_attrs,
+};
+
+static struct kobj_type gbe_sw_nu_tx_pri_ktype = {
+	.sysfs_ops = &gbe_sw_tx_pri_sysfs_ops,
+	.default_attrs = gbe_sw_nu_tx_pri_default_attrs,
+};
+
+static const struct gbe_sw_mod_info gbe_sw_port_vlans[] = {
+	{
+		.name		= "port_vlan_id",
+		.shift		= 0,
+		.bits		= 12,
+	},
+	{
+		.name		= "port_cfi",
+		.shift		= 12,
+		.bits		= 1,
+	},
+	{
+		.name		= "port_vlan_pri",
+		.shift		= 13,
+		.bits		= 3,
+	},
+};
+
+static ssize_t gbe_sw_port_vlan_show(struct gbe_priv *gbe_dev,
+				     struct gbe_attribute *attr,
+				     char *buf)
+{
+	int len = 0, total_len = 0, port;
+	struct gbe_slave *slave;
+	u32 reg_val;
+
+	port = (int)(attr->context);
+
+	if (port == gbe_dev->host_port) {
+		/* Host port */
+		reg_val = readl(GBE_REG_ADDR(gbe_dev, host_port_regs,
+					     port_vlan));
+		len = gbe_sw_attr_info_show(attr->info, attr->info_size,
+					    reg_val, buf);
+		return len;
+	}
+
+	slave = gbe_port_num_get_slave(gbe_dev, port);
+	if (!slave)
+		return 0;
+
+	reg_val = readl(GBE_REG_ADDR(slave, port_regs, port_vlan));
+	len = gbe_sw_attr_info_show(attr->info, attr->info_size, reg_val, buf);
+
+	return (total_len += len);
+}
+
+static ssize_t gbe_sw_port_vlan_store(struct gbe_priv *gbe_dev,
+				      struct gbe_attribute *attr,
+				      const char *buf, size_t count)
+{
+	const struct gbe_sw_mod_info *info;
+	struct gbe_sw_parse_result res;
+	struct gbe_slave *slave;
+	void __iomem *reg = NULL;
+	int ret, port;
+
+	port = (int)(attr->context);
+
+	ret = gbe_sw_attr_parse_set_command(gbe_dev, attr, buf, count, &res);
+	if (ret)
+		return ret;
+
+	info = &attr->info[res.control];
+
+	/* Host port */
+	if (port == gbe_dev->host_port) {
+		reg = GBE_REG_ADDR(gbe_dev, host_port_regs, port_vlan);
+		goto set;
+	}
+
+	slave = gbe_port_num_get_slave(gbe_dev, port);
+	if (!slave)
+		return -EINVAL;
+
+	/* Slave port */
+	reg = GBE_REG_ADDR(slave, port_regs, port_vlan);
+	if (!reg)
+		return  -ENOENT;
+
+set:
+	gbe_sw_info_set_reg_field(reg, info, res.value);
+
+	return count;
+}
+
+static struct gbe_attribute gbe_sw_pvlan_0_attribute =
+	__GBE_SW_CTXT_ATTR(0, 0644,
+			   gbe_sw_port_vlan_show,
+			   gbe_sw_port_vlan_store,
+			   gbe_sw_port_vlans, (void *)0);
+
+static struct gbe_attribute gbe_sw_pvlan_1_attribute =
+	__GBE_SW_CTXT_ATTR(1, 0644,
+			   gbe_sw_port_vlan_show,
+			   gbe_sw_port_vlan_store,
+			   gbe_sw_port_vlans, (void *)1);
+
+static struct gbe_attribute gbe_sw_pvlan_2_attribute =
+	__GBE_SW_CTXT_ATTR(2, 0644,
+			   gbe_sw_port_vlan_show,
+			   gbe_sw_port_vlan_store,
+			   gbe_sw_port_vlans, (void *)2);
+
+static struct gbe_attribute gbe_sw_pvlan_3_attribute =
+	__GBE_SW_CTXT_ATTR(3, 0644,
+			   gbe_sw_port_vlan_show,
+			   gbe_sw_port_vlan_store,
+			   gbe_sw_port_vlans, (void *)3);
+
+static struct gbe_attribute gbe_sw_pvlan_4_attribute =
+	__GBE_SW_CTXT_ATTR(4, 0644,
+			   gbe_sw_port_vlan_show,
+			   gbe_sw_port_vlan_store,
+			   gbe_sw_port_vlans, (void *)4);
+
+static struct gbe_attribute gbe_sw_pvlan_5_attribute =
+	__GBE_SW_CTXT_ATTR(5, 0644,
+			   gbe_sw_port_vlan_show,
+			   gbe_sw_port_vlan_store,
+			   gbe_sw_port_vlans, (void *)5);
+
+static struct gbe_attribute gbe_sw_pvlan_6_attribute =
+	__GBE_SW_CTXT_ATTR(6, 0644,
+			   gbe_sw_port_vlan_show,
+			   gbe_sw_port_vlan_store,
+			   gbe_sw_port_vlans, (void *)6);
+
+static struct gbe_attribute gbe_sw_pvlan_7_attribute =
+	__GBE_SW_CTXT_ATTR(7, 0644,
+			   gbe_sw_port_vlan_show,
+			   gbe_sw_port_vlan_store,
+			   gbe_sw_port_vlans, (void *)7);
+
+static struct gbe_attribute gbe_sw_pvlan_8_attribute =
+	__GBE_SW_CTXT_ATTR(8, 0644,
+			   gbe_sw_port_vlan_show,
+			   gbe_sw_port_vlan_store,
+			   gbe_sw_port_vlans, (void *)8);
+
+static struct attribute *gbe_sw_ver14_pvlan_default_attrs[] = {
+	&gbe_sw_pvlan_0_attribute.attr,
+	&gbe_sw_pvlan_1_attribute.attr,
+	&gbe_sw_pvlan_2_attribute.attr,
+	&gbe_sw_pvlan_3_attribute.attr,
+	&gbe_sw_pvlan_4_attribute.attr,
+	NULL
+};
+
+static struct attribute *gbe_sw_xge_pvlan_default_attrs[] = {
+	&gbe_sw_pvlan_0_attribute.attr,
+	&gbe_sw_pvlan_1_attribute.attr,
+	&gbe_sw_pvlan_2_attribute.attr,
+	NULL
+};
+
+static struct attribute *gbe_sw_nu_pvlan_default_attrs[] = {
+	&gbe_sw_pvlan_0_attribute.attr,
+	&gbe_sw_pvlan_1_attribute.attr,
+	&gbe_sw_pvlan_2_attribute.attr,
+	&gbe_sw_pvlan_3_attribute.attr,
+	&gbe_sw_pvlan_4_attribute.attr,
+	&gbe_sw_pvlan_5_attribute.attr,
+	&gbe_sw_pvlan_6_attribute.attr,
+	&gbe_sw_pvlan_7_attribute.attr,
+	&gbe_sw_pvlan_8_attribute.attr,
+	NULL
+};
+
+static ssize_t gbe_sw_pvlan_attr_show(struct kobject *kobj,
+				      struct attribute *attr, char *buf)
+{
+	struct gbe_attribute *attribute = to_gbe_attr(attr);
+	struct gbe_priv *gbe_dev = pvlan_to_gbe_dev(kobj);
+
+	if (!attribute->show)
+		return -EIO;
+
+	return attribute->show(gbe_dev, attribute, buf);
+}
+
+static ssize_t gbe_sw_pvlan_attr_store(struct kobject *kobj,
+				       struct attribute *attr,
+				       const char *buf, size_t count)
+{
+	struct gbe_attribute *attribute = to_gbe_attr(attr);
+	struct gbe_priv *gbe_dev = pvlan_to_gbe_dev(kobj);
+
+	if (!attribute->store)
+		return -EIO;
+
+	return attribute->store(gbe_dev, attribute, buf, count);
+}
+
+static const struct sysfs_ops gbe_sw_pvlan_sysfs_ops = {
+	.show = gbe_sw_pvlan_attr_show,
+	.store = gbe_sw_pvlan_attr_store,
+};
+
+static struct kobj_type gbe_sw_ver14_pvlan_ktype = {
+	.sysfs_ops = &gbe_sw_pvlan_sysfs_ops,
+	.default_attrs = gbe_sw_ver14_pvlan_default_attrs,
+};
+
+static struct kobj_type gbe_sw_xge_pvlan_ktype = {
+	.sysfs_ops = &gbe_sw_pvlan_sysfs_ops,
+	.default_attrs = gbe_sw_xge_pvlan_default_attrs,
+};
+
+static struct kobj_type gbe_sw_nu_pvlan_ktype = {
+	.sysfs_ops = &gbe_sw_pvlan_sysfs_ops,
+	.default_attrs = gbe_sw_nu_pvlan_default_attrs,
+};
+
+static ssize_t gbe_sw_stats_attr_store(struct kobject *kobj,
+				       struct attribute *attr,
+				       const char *buf, size_t count)
+{
+	struct gbe_attribute *attribute = to_gbe_attr(attr);
+	struct gbe_priv *gbe_dev = stats_to_gbe_dev(kobj);
+
+	if (!attribute->store)
+		return -EIO;
+
+	return attribute->store(gbe_dev, attribute, buf, count);
+}
+
+static const struct sysfs_ops gbe_sw_stats_sysfs_ops = {
+	.store = gbe_sw_stats_attr_store,
+};
+
+static ssize_t gbe_sw_stats_mod_store(struct gbe_priv *gbe_dev,
+				      struct gbe_attribute *attr,
+				      const char *buf, size_t count)
+{
+	int stat_mod, max_ports;
+	unsigned long end;
+
+	if (kstrtoul(buf, 0, &end) != 0 || end != 0)
+		return -EINVAL;
+
+	stat_mod = (int)(attr->context);
+
+	/* We have stats blocks for only slave ports on GBE_SS_VERSION_14
+	 * but also for host port for other variations. So check this
+	 * value accordingly
+	 */
+	max_ports = (gbe_dev->ss_version == GBE_SS_VERSION_14) ?
+		     gbe_dev->max_num_slaves : gbe_dev->max_num_ports;
+
+	if (stat_mod >= max_ports)
+		return -EINVAL;
+
+	spin_lock_bh(&gbe_dev->hw_stats_lock);
+	if (gbe_dev->ss_version == GBE_SS_VERSION_14)
+		gbe_reset_mod_stats_ver14(gbe_dev, stat_mod);
+	else
+		gbe_reset_mod_stats(gbe_dev, stat_mod);
+	spin_unlock_bh(&gbe_dev->hw_stats_lock);
+	return count;
+}
+
+static struct gbe_attribute gbe_sw_stats_a_attribute =
+			__GBE_SW_ATTR_FULL(A, 0200, NULL,
+					   gbe_sw_stats_mod_store,
+					   NULL, 0, (void *)GBE_STATSA_MODULE);
+
+static struct gbe_attribute gbe_sw_stats_b_attribute =
+			__GBE_SW_ATTR_FULL(B, 0200, NULL,
+					   gbe_sw_stats_mod_store,
+					   NULL, 0, (void *)GBE_STATSB_MODULE);
+
+static struct gbe_attribute gbe_sw_stats_c_attribute =
+			__GBE_SW_ATTR_FULL(C, 0200, NULL,
+					   gbe_sw_stats_mod_store,
+					   NULL, 0, (void *)GBE_STATSC_MODULE);
+
+static struct gbe_attribute gbe_sw_stats_d_attribute =
+			__GBE_SW_ATTR_FULL(D, 0200, NULL,
+					   gbe_sw_stats_mod_store,
+					   NULL, 0, (void *)GBE_STATSD_MODULE);
+
+static struct attribute *gbe_sw_ver14_stats_default_attrs[] = {
+	&gbe_sw_stats_a_attribute.attr,
+	&gbe_sw_stats_b_attribute.attr,
+	&gbe_sw_stats_c_attribute.attr,
+	&gbe_sw_stats_d_attribute.attr,
+	NULL
+};
+
+static struct kobj_type gbe_sw_ver14_stats_ktype = {
+	.sysfs_ops = &gbe_sw_stats_sysfs_ops,
+	.default_attrs = gbe_sw_ver14_stats_default_attrs,
+};
+
+static struct gbe_attribute gbe_sw_xge_stats_0_attribute =
+			__GBE_SW_ATTR_FULL(0, 0200, NULL,
+					   gbe_sw_stats_mod_store,
+					   NULL, 0, (void *)XGBE_STATS0_MODULE);
+
+static struct gbe_attribute gbe_sw_xge_stats_1_attribute =
+			__GBE_SW_ATTR_FULL(1, 0200, NULL,
+					   gbe_sw_stats_mod_store,
+					   NULL, 0, (void *)XGBE_STATS1_MODULE);
+
+static struct gbe_attribute gbe_sw_xge_stats_2_attribute =
+			__GBE_SW_ATTR_FULL(2, 0200, NULL,
+					   gbe_sw_stats_mod_store,
+					   NULL, 0, (void *)XGBE_STATS2_MODULE);
+
+static struct attribute *gbe_sw_xge_stats_default_attrs[] = {
+	&gbe_sw_xge_stats_0_attribute.attr,
+	&gbe_sw_xge_stats_1_attribute.attr,
+	&gbe_sw_xge_stats_2_attribute.attr,
+	NULL
+};
+
+static struct kobj_type gbe_sw_xge_stats_ktype = {
+	.sysfs_ops = &gbe_sw_stats_sysfs_ops,
+	.default_attrs = gbe_sw_xge_stats_default_attrs,
+};
+
+static struct gbe_attribute gbe_sw_stats_0_attribute =
+			__GBE_SW_ATTR_FULL(0, 0200, NULL,
+					   gbe_sw_stats_mod_store,
+					   NULL, 0,
+					   (void *)GBENU_STATS0_MODULE);
+
+static struct gbe_attribute gbe_sw_stats_1_attribute =
+			__GBE_SW_ATTR_FULL(1, 0200, NULL,
+					   gbe_sw_stats_mod_store,
+					   NULL, 0,
+					   (void *)GBENU_STATS1_MODULE);
+
+static struct gbe_attribute gbe_sw_stats_2_attribute =
+			__GBE_SW_ATTR_FULL(2, 0200, NULL,
+					   gbe_sw_stats_mod_store,
+					   NULL, 0,
+					   (void *)GBENU_STATS2_MODULE);
+
+static struct gbe_attribute gbe_sw_stats_3_attribute =
+			__GBE_SW_ATTR_FULL(3, 0200, NULL,
+					   gbe_sw_stats_mod_store,
+					   NULL, 0,
+					   (void *)GBENU_STATS3_MODULE);
+
+static struct gbe_attribute gbe_sw_stats_4_attribute =
+			__GBE_SW_ATTR_FULL(4, 0200, NULL,
+					   gbe_sw_stats_mod_store,
+					   NULL, 0,
+					   (void *)GBENU_STATS4_MODULE);
+
+static struct gbe_attribute gbe_sw_stats_5_attribute =
+			__GBE_SW_ATTR_FULL(5, 0200, NULL,
+					   gbe_sw_stats_mod_store,
+					   NULL, 0,
+					   (void *)GBENU_STATS5_MODULE);
+
+static struct gbe_attribute gbe_sw_stats_6_attribute =
+			__GBE_SW_ATTR_FULL(6, 0200, NULL,
+					   gbe_sw_stats_mod_store,
+					   NULL, 0,
+					   (void *)GBENU_STATS6_MODULE);
+
+static struct gbe_attribute gbe_sw_stats_7_attribute =
+			__GBE_SW_ATTR_FULL(7, 0200, NULL,
+					   gbe_sw_stats_mod_store,
+					   NULL, 0,
+					   (void *)GBENU_STATS7_MODULE);
+
+static struct gbe_attribute gbe_sw_stats_8_attribute =
+			__GBE_SW_ATTR_FULL(8, 0200, NULL,
+					   gbe_sw_stats_mod_store,
+					   NULL, 0,
+					   (void *)GBENU_STATS8_MODULE);
+
+static struct attribute *gbe_sw_nu_stats_default_attrs[] = {
+	&gbe_sw_stats_0_attribute.attr,
+	&gbe_sw_stats_1_attribute.attr,
+	&gbe_sw_stats_2_attribute.attr,
+	&gbe_sw_stats_3_attribute.attr,
+	&gbe_sw_stats_4_attribute.attr,
+	&gbe_sw_stats_5_attribute.attr,
+	&gbe_sw_stats_6_attribute.attr,
+	&gbe_sw_stats_7_attribute.attr,
+	&gbe_sw_stats_8_attribute.attr,
+	NULL
+};
+
+static struct kobj_type gbe_sw_nu_stats_ktype = {
+	.sysfs_ops = &gbe_sw_stats_sysfs_ops,
+	.default_attrs = gbe_sw_nu_stats_default_attrs,
+};
+
+static ssize_t gbe_sw_attr_show(struct kobject *kobj,
+				struct attribute *attr, char *buf)
+{
+	struct gbe_attribute *attribute = to_gbe_attr(attr);
+	struct gbe_priv *gbe_dev = to_gbe_dev(kobj);
+
+	if (!attribute->show)
+		return -EIO;
+
+	return attribute->show(gbe_dev, attribute, buf);
+}
+
+static ssize_t gbe_sw_attr_store(struct kobject *kobj,
+				 struct attribute *attr, const char *buf,
+				 size_t count)
+{
+	struct gbe_attribute *attribute = to_gbe_attr(attr);
+	struct gbe_priv *gbe_dev = to_gbe_dev(kobj);
+
+	if (!attribute->store)
+		return -EIO;
+
+	return attribute->store(gbe_dev, attribute, buf, count);
+}
+
+static const struct sysfs_ops gbe_sw_sysfs_ops = {
+	.show = gbe_sw_attr_show,
+	.store = gbe_sw_attr_store,
+};
+
+static struct kobj_type gbe_sw_ver14_ktype = {
+	.sysfs_ops = &gbe_sw_sysfs_ops,
+	.default_attrs = gbe_sw_ver14_default_attrs,
+};
+
+static struct kobj_type gbe_sw_xge_ktype = {
+	.sysfs_ops = &gbe_sw_sysfs_ops,
+	.default_attrs = gbe_sw_xge_default_attrs,
+};
+
+static struct kobj_type gbe_sw_nu_ktype = {
+	.sysfs_ops = &gbe_sw_sysfs_ops,
+	.default_attrs = gbe_sw_nu_default_attrs,
+};
+
+/* for ver14 switch */
+static struct kobj_type *gbe_sw_ver14_kobjs[GBE_SYSFS_SW_MAX] = {
+	&gbe_sw_ver14_ktype,
+	&gbe_sw_ver14_tx_pri_ktype,
+	&gbe_sw_ver14_pvlan_ktype,
+	&gbe_sw_ver14_stats_ktype,
+};
+
+/* for xge switch */
+static struct kobj_type *gbe_sw_xge_kobjs[GBE_SYSFS_SW_MAX] = {
+	&gbe_sw_xge_ktype,
+	&gbe_sw_xge_tx_pri_ktype,
+	&gbe_sw_xge_pvlan_ktype,
+	&gbe_sw_xge_stats_ktype,
+};
+
+/* for NU switch */
+static struct kobj_type *gbe_sw_nu_kobjs[GBE_SYSFS_SW_MAX] = {
+	&gbe_sw_nu_ktype,
+	&gbe_sw_nu_tx_pri_ktype,
+	&gbe_sw_nu_pvlan_ktype,
+	&gbe_sw_nu_stats_ktype,
+};
+
+int gbe_create_sysfs_entries(struct gbe_priv *gbe_dev)
+{
+	struct device *dev = gbe_dev->dev;
+	static struct kobj_type **kobjs;
+	int ret;
+
+	switch (gbe_dev->ss_version) {
+	case XGBE_SS_VERSION_10:
+		kobjs = &gbe_sw_xge_kobjs[0];
+		break;
+	case GBE_SS_VERSION_14:
+		kobjs = &gbe_sw_ver14_kobjs[0];
+		break;
+	default:
+		kobjs = &gbe_sw_nu_kobjs[0];
+	}
+
+	ret = kobject_init_and_add(&gbe_dev->kobj, kobjs[GBE_SYSFS_SW_CONTROL],
+				   &dev->kobj, "gbe_sw");
+	if (ret) {
+		dev_err(dev, "failed to create gbe sw sysfs entry\n");
+		return ret;
+	}
+
+	ret = kobject_init_and_add(&gbe_dev->tx_pri_kobj,
+				   kobjs[GBE_SYSFS_SW_TX_PRIO],
+				   &gbe_dev->kobj,
+				   "port_tx_pri_map");
+	if (ret) {
+		dev_err(dev, "failed to create sysfs port_tx_pri_map entry\n");
+		goto clean_tx_pri_kobj;
+	}
+
+	ret = kobject_init_and_add(&gbe_dev->pvlan_kobj,
+				   kobjs[GBE_SYSFS_SW_VLAN],
+				   &gbe_dev->kobj, "port_vlan");
+	if (ret) {
+		dev_err(dev, "failed to create sysfs port_vlan entry\n");
+		goto clean_pvlan_kobj;
+	}
+
+	ret = kobject_init_and_add(&gbe_dev->stats_kobj,
+				   kobjs[GBE_SYSFS_SW_STATS],
+				   &gbe_dev->kobj, "stats");
+	if (ret) {
+		dev_err(dev, "failed to create sysfs stats entry\n");
+		goto clean_stats_kobj;
+	}
+
+	return ret;
+
+clean_stats_kobj:
+	kobject_put(&gbe_dev->pvlan_kobj);
+clean_pvlan_kobj:
+	kobject_put(&gbe_dev->tx_pri_kobj);
+clean_tx_pri_kobj:
+	kobject_put(&gbe_dev->kobj);
+	return ret;
+}
+
+void gbe_remove_sysfs_entries(struct gbe_priv *gbe_dev)
+{
+	kobject_put(&gbe_dev->stats_kobj);
+	kobject_put(&gbe_dev->pvlan_kobj);
+	kobject_put(&gbe_dev->tx_pri_kobj);
+	kobject_put(&gbe_dev->kobj);
+}
diff -Naur --no-dereference a/drivers/net/ethernet/ti/netcp.h b/drivers/net/ethernet/ti/netcp.h
--- a/drivers/net/ethernet/ti/netcp.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/ethernet/ti/netcp.h	2020-06-19 21:00:31.416227715 -0400
@@ -247,7 +247,7 @@
 /* SGMII functions */
 int netcp_sgmii_reset(void __iomem *sgmii_ofs, int port);
 bool netcp_sgmii_rtreset(void __iomem *sgmii_ofs, int port, bool set);
-int netcp_sgmii_get_port_link(void __iomem *sgmii_ofs, int port);
+bool netcp_sgmii_get_port_link(void __iomem *sgmii_ofs, int port);
 int netcp_sgmii_config(void __iomem *sgmii_ofs, int port, u32 interface);
 
 /* XGBE SERDES init functions */
diff -Naur --no-dereference a/drivers/net/ethernet/ti/netcp_sgmii.c b/drivers/net/ethernet/ti/netcp_sgmii.c
--- a/drivers/net/ethernet/ti/netcp_sgmii.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/ethernet/ti/netcp_sgmii.c	2020-06-19 21:00:31.416227715 -0400
@@ -83,13 +83,14 @@
 	return oldval;
 }
 
-int netcp_sgmii_get_port_link(void __iomem *sgmii_ofs, int port)
+bool netcp_sgmii_get_port_link(void __iomem *sgmii_ofs, int port)
 {
-	u32 status = 0, link = 0;
+	u32 status = 0;
+	bool link = false;
 
 	status = sgmii_read_reg(sgmii_ofs, SGMII_STATUS_REG(port));
 	if ((status & SGMII_REG_STATUS_LINK) != 0)
-		link = 1;
+		link = true;
 	return link;
 }
 
diff -Naur --no-dereference a/drivers/net/ethernet/ti/netcp_xgbepcsr.c b/drivers/net/ethernet/ti/netcp_xgbepcsr.c
--- a/drivers/net/ethernet/ti/netcp_xgbepcsr.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/ethernet/ti/netcp_xgbepcsr.c	1969-12-31 19:00:00.000000000 -0500
@@ -1,501 +0,0 @@
-/*
- * XGE PCSR module initialisation
- *
- * Copyright (C) 2014 Texas Instruments Incorporated
- * Authors:	Sandeep Nair <sandeep_n@ti.com>
- *		WingMan Kwok <w-kwok2@ti.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation version 2.
- *
- * This program is distributed "as is" WITHOUT ANY WARRANTY of any
- * kind, whether express or implied; without even the implied warranty
- * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
-#include "netcp.h"
-
-/* XGBE registers */
-#define XGBE_CTRL_OFFSET		0x0c
-#define XGBE_SGMII_1_OFFSET		0x0114
-#define XGBE_SGMII_2_OFFSET		0x0214
-
-/* PCS-R registers */
-#define PCSR_CPU_CTRL_OFFSET		0x1fd0
-#define POR_EN				BIT(29)
-
-#define reg_rmw(addr, value, mask) \
-	writel(((readl(addr) & (~(mask))) | \
-			(value & (mask))), (addr))
-
-/* bit mask of width w at offset s */
-#define MASK_WID_SH(w, s)		(((1 << w) - 1) << s)
-
-/* shift value v to offset s */
-#define VAL_SH(v, s)			(v << s)
-
-#define PHY_A(serdes)			0
-
-struct serdes_cfg {
-	u32 ofs;
-	u32 val;
-	u32 mask;
-};
-
-static struct serdes_cfg cfg_phyb_1p25g_156p25mhz_cmu0[] = {
-	{0x0000, 0x00800002, 0x00ff00ff},
-	{0x0014, 0x00003838, 0x0000ffff},
-	{0x0060, 0x1c44e438, 0xffffffff},
-	{0x0064, 0x00c18400, 0x00ffffff},
-	{0x0068, 0x17078200, 0xffffff00},
-	{0x006c, 0x00000014, 0x000000ff},
-	{0x0078, 0x0000c000, 0x0000ff00},
-	{0x0000, 0x00000003, 0x000000ff},
-};
-
-static struct serdes_cfg cfg_phyb_10p3125g_156p25mhz_cmu1[] = {
-	{0x0c00, 0x00030002, 0x00ff00ff},
-	{0x0c14, 0x00005252, 0x0000ffff},
-	{0x0c28, 0x80000000, 0xff000000},
-	{0x0c2c, 0x000000f6, 0x000000ff},
-	{0x0c3c, 0x04000405, 0xff00ffff},
-	{0x0c40, 0xc0800000, 0xffff0000},
-	{0x0c44, 0x5a202062, 0xffffffff},
-	{0x0c48, 0x40040424, 0xffffffff},
-	{0x0c4c, 0x00004002, 0x0000ffff},
-	{0x0c50, 0x19001c00, 0xff00ff00},
-	{0x0c54, 0x00002100, 0x0000ff00},
-	{0x0c58, 0x00000060, 0x000000ff},
-	{0x0c60, 0x80131e7c, 0xffffffff},
-	{0x0c64, 0x8400cb02, 0xff00ffff},
-	{0x0c68, 0x17078200, 0xffffff00},
-	{0x0c6c, 0x00000016, 0x000000ff},
-	{0x0c74, 0x00000400, 0x0000ff00},
-	{0x0c78, 0x0000c000, 0x0000ff00},
-	{0x0c00, 0x00000003, 0x000000ff},
-};
-
-static struct serdes_cfg cfg_phyb_10p3125g_16bit_lane[] = {
-	{0x0204, 0x00000080, 0x000000ff},
-	{0x0208, 0x0000920d, 0x0000ffff},
-	{0x0204, 0xfc000000, 0xff000000},
-	{0x0208, 0x00009104, 0x0000ffff},
-	{0x0210, 0x1a000000, 0xff000000},
-	{0x0214, 0x00006b58, 0x00ffffff},
-	{0x0218, 0x75800084, 0xffff00ff},
-	{0x022c, 0x00300000, 0x00ff0000},
-	{0x0230, 0x00003800, 0x0000ff00},
-	{0x024c, 0x008f0000, 0x00ff0000},
-	{0x0250, 0x30000000, 0xff000000},
-	{0x0260, 0x00000002, 0x000000ff},
-	{0x0264, 0x00000057, 0x000000ff},
-	{0x0268, 0x00575700, 0x00ffff00},
-	{0x0278, 0xff000000, 0xff000000},
-	{0x0280, 0x00500050, 0x00ff00ff},
-	{0x0284, 0x00001f15, 0x0000ffff},
-	{0x028c, 0x00006f00, 0x0000ff00},
-	{0x0294, 0x00000000, 0xffffff00},
-	{0x0298, 0x00002640, 0xff00ffff},
-	{0x029c, 0x00000003, 0x000000ff},
-	{0x02a4, 0x00000f13, 0x0000ffff},
-	{0x02a8, 0x0001b600, 0x00ffff00},
-	{0x0380, 0x00000030, 0x000000ff},
-	{0x03c0, 0x00000200, 0x0000ff00},
-	{0x03cc, 0x00000018, 0x000000ff},
-	{0x03cc, 0x00000000, 0x000000ff},
-};
-
-static struct serdes_cfg cfg_phyb_10p3125g_comlane[] = {
-	{0x0a00, 0x00000800, 0x0000ff00},
-	{0x0a84, 0x00000000, 0x000000ff},
-	{0x0a8c, 0x00130000, 0x00ff0000},
-	{0x0a90, 0x77a00000, 0xffff0000},
-	{0x0a94, 0x00007777, 0x0000ffff},
-	{0x0b08, 0x000f0000, 0xffff0000},
-	{0x0b0c, 0x000f0000, 0x00ffffff},
-	{0x0b10, 0xbe000000, 0xff000000},
-	{0x0b14, 0x000000ff, 0x000000ff},
-	{0x0b18, 0x00000014, 0x000000ff},
-	{0x0b5c, 0x981b0000, 0xffff0000},
-	{0x0b64, 0x00001100, 0x0000ff00},
-	{0x0b78, 0x00000c00, 0x0000ff00},
-	{0x0abc, 0xff000000, 0xff000000},
-	{0x0ac0, 0x0000008b, 0x000000ff},
-};
-
-static struct serdes_cfg cfg_cm_c1_c2[] = {
-	{0x0208, 0x00000000, 0x00000f00},
-	{0x0208, 0x00000000, 0x0000001f},
-	{0x0204, 0x00000000, 0x00040000},
-	{0x0208, 0x000000a0, 0x000000e0},
-};
-
-static void netcp_xgbe_serdes_cmu_init(void __iomem *serdes_regs)
-{
-	int i;
-
-	/* cmu0 setup */
-	for (i = 0; i < ARRAY_SIZE(cfg_phyb_1p25g_156p25mhz_cmu0); i++) {
-		reg_rmw(serdes_regs + cfg_phyb_1p25g_156p25mhz_cmu0[i].ofs,
-			cfg_phyb_1p25g_156p25mhz_cmu0[i].val,
-			cfg_phyb_1p25g_156p25mhz_cmu0[i].mask);
-	}
-
-	/* cmu1 setup */
-	for (i = 0; i < ARRAY_SIZE(cfg_phyb_10p3125g_156p25mhz_cmu1); i++) {
-		reg_rmw(serdes_regs + cfg_phyb_10p3125g_156p25mhz_cmu1[i].ofs,
-			cfg_phyb_10p3125g_156p25mhz_cmu1[i].val,
-			cfg_phyb_10p3125g_156p25mhz_cmu1[i].mask);
-	}
-}
-
-/* lane is 0 based */
-static void netcp_xgbe_serdes_lane_config(
-			void __iomem *serdes_regs, int lane)
-{
-	int i;
-
-	/* lane setup */
-	for (i = 0; i < ARRAY_SIZE(cfg_phyb_10p3125g_16bit_lane); i++) {
-		reg_rmw(serdes_regs +
-				cfg_phyb_10p3125g_16bit_lane[i].ofs +
-				(0x200 * lane),
-			cfg_phyb_10p3125g_16bit_lane[i].val,
-			cfg_phyb_10p3125g_16bit_lane[i].mask);
-	}
-
-	/* disable auto negotiation*/
-	reg_rmw(serdes_regs + (0x200 * lane) + 0x0380,
-		0x00000000, 0x00000010);
-
-	/* disable link training */
-	reg_rmw(serdes_regs + (0x200 * lane) + 0x03c0,
-		0x00000000, 0x00000200);
-}
-
-static void netcp_xgbe_serdes_com_enable(void __iomem *serdes_regs)
-{
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(cfg_phyb_10p3125g_comlane); i++) {
-		reg_rmw(serdes_regs + cfg_phyb_10p3125g_comlane[i].ofs,
-			cfg_phyb_10p3125g_comlane[i].val,
-			cfg_phyb_10p3125g_comlane[i].mask);
-	}
-}
-
-static void netcp_xgbe_serdes_lane_enable(
-			void __iomem *serdes_regs, int lane)
-{
-	/* Set Lane Control Rate */
-	writel(0xe0e9e038, serdes_regs + 0x1fe0 + (4 * lane));
-}
-
-static void netcp_xgbe_serdes_phyb_rst_clr(void __iomem *serdes_regs)
-{
-	reg_rmw(serdes_regs + 0x0a00, 0x0000001f, 0x000000ff);
-}
-
-static void netcp_xgbe_serdes_pll_disable(void __iomem *serdes_regs)
-{
-	writel(0x88000000, serdes_regs + 0x1ff4);
-}
-
-static void netcp_xgbe_serdes_pll_enable(void __iomem *serdes_regs)
-{
-	netcp_xgbe_serdes_phyb_rst_clr(serdes_regs);
-	writel(0xee000000, serdes_regs + 0x1ff4);
-}
-
-static int netcp_xgbe_wait_pll_locked(void __iomem *sw_regs)
-{
-	unsigned long timeout;
-	int ret = 0;
-	u32 val_1, val_0;
-
-	timeout = jiffies + msecs_to_jiffies(500);
-	do {
-		val_0 = (readl(sw_regs + XGBE_SGMII_1_OFFSET) & BIT(4));
-		val_1 = (readl(sw_regs + XGBE_SGMII_2_OFFSET) & BIT(4));
-
-		if (val_1 && val_0)
-			return 0;
-
-		if (time_after(jiffies, timeout)) {
-			ret = -ETIMEDOUT;
-			break;
-		}
-
-		cpu_relax();
-	} while (true);
-
-	pr_err("XGBE serdes not locked: time out.\n");
-	return ret;
-}
-
-static void netcp_xgbe_serdes_enable_xgmii_port(void __iomem *sw_regs)
-{
-	writel(0x03, sw_regs + XGBE_CTRL_OFFSET);
-}
-
-static u32 netcp_xgbe_serdes_read_tbus_val(void __iomem *serdes_regs)
-{
-	u32 tmp;
-
-	if (PHY_A(serdes_regs)) {
-		tmp  = (readl(serdes_regs + 0x0ec) >> 24) & 0x0ff;
-		tmp |= ((readl(serdes_regs + 0x0fc) >> 16) & 0x00f00);
-	} else {
-		tmp  = (readl(serdes_regs + 0x0f8) >> 16) & 0x0fff;
-	}
-
-	return tmp;
-}
-
-static void netcp_xgbe_serdes_write_tbus_addr(void __iomem *serdes_regs,
-					      int select, int ofs)
-{
-	if (PHY_A(serdes_regs)) {
-		reg_rmw(serdes_regs + 0x0008, ((select << 5) + ofs) << 24,
-			~0x00ffffff);
-		return;
-	}
-
-	/* For 2 lane Phy-B, lane0 is actually lane1 */
-	switch (select) {
-	case 1:
-		select = 2;
-		break;
-	case 2:
-		select = 3;
-		break;
-	default:
-		return;
-	}
-
-	reg_rmw(serdes_regs + 0x00fc, ((select << 8) + ofs) << 16, ~0xf800ffff);
-}
-
-static u32 netcp_xgbe_serdes_read_select_tbus(void __iomem *serdes_regs,
-					      int select, int ofs)
-{
-	/* Set tbus address */
-	netcp_xgbe_serdes_write_tbus_addr(serdes_regs, select, ofs);
-	/* Get TBUS Value */
-	return netcp_xgbe_serdes_read_tbus_val(serdes_regs);
-}
-
-static void netcp_xgbe_serdes_reset_cdr(void __iomem *serdes_regs,
-					void __iomem *sig_detect_reg, int lane)
-{
-	u32 tmp, dlpf, tbus;
-
-	/*Get the DLPF values */
-	tmp = netcp_xgbe_serdes_read_select_tbus(
-			serdes_regs, lane + 1, 5);
-
-	dlpf = tmp >> 2;
-
-	if (dlpf < 400 || dlpf > 700) {
-		reg_rmw(sig_detect_reg, VAL_SH(2, 1), MASK_WID_SH(2, 1));
-		mdelay(1);
-		reg_rmw(sig_detect_reg, VAL_SH(0, 1), MASK_WID_SH(2, 1));
-	} else {
-		tbus = netcp_xgbe_serdes_read_select_tbus(serdes_regs, lane +
-							  1, 0xe);
-
-		pr_debug("XGBE: CDR centered, DLPF: %4d,%d,%d.\n",
-			 tmp >> 2, tmp & 3, (tbus >> 2) & 3);
-	}
-}
-
-/* Call every 100 ms */
-static int netcp_xgbe_check_link_status(void __iomem *serdes_regs,
-					void __iomem *sw_regs, u32 lanes,
-					u32 *current_state, u32 *lane_down)
-{
-	void __iomem *pcsr_base = sw_regs + 0x0600;
-	void __iomem *sig_detect_reg;
-	u32 pcsr_rx_stat, blk_lock, blk_errs;
-	int loss, i, status = 1;
-
-	for (i = 0; i < lanes; i++) {
-		/* Get the Loss bit */
-		loss = readl(serdes_regs + 0x1fc0 + 0x20 + (i * 0x04)) & 0x1;
-
-		/* Get Block Errors and Block Lock bits */
-		pcsr_rx_stat = readl(pcsr_base + 0x0c + (i * 0x80));
-		blk_lock = (pcsr_rx_stat >> 30) & 0x1;
-		blk_errs = (pcsr_rx_stat >> 16) & 0x0ff;
-
-		/* Get Signal Detect Overlay Address */
-		sig_detect_reg = serdes_regs + (i * 0x200) + 0x200 + 0x04;
-
-		/* If Block errors maxed out, attempt recovery! */
-		if (blk_errs == 0x0ff)
-			blk_lock = 0;
-
-		switch (current_state[i]) {
-		case 0:
-			/* if good link lock the signal detect ON! */
-			if (!loss && blk_lock) {
-				pr_debug("XGBE PCSR Linked Lane: %d\n", i);
-				reg_rmw(sig_detect_reg, VAL_SH(3, 1),
-					MASK_WID_SH(2, 1));
-				current_state[i] = 1;
-			} else if (!blk_lock) {
-				/* if no lock, then reset CDR */
-				pr_debug("XGBE PCSR Recover Lane: %d\n", i);
-				netcp_xgbe_serdes_reset_cdr(serdes_regs,
-							    sig_detect_reg, i);
-			}
-			break;
-
-		case 1:
-			if (!blk_lock) {
-				/* Link Lost? */
-				lane_down[i] = 1;
-				current_state[i] = 2;
-			}
-			break;
-
-		case 2:
-			if (blk_lock)
-				/* Nope just noise */
-				current_state[i] = 1;
-			else {
-				/* Lost the block lock, reset CDR if it is
-				 * not centered and go back to sync state
-				 */
-				netcp_xgbe_serdes_reset_cdr(serdes_regs,
-							    sig_detect_reg, i);
-				current_state[i] = 0;
-			}
-			break;
-
-		default:
-			pr_err("XGBE: unknown current_state[%d] %d\n",
-			       i, current_state[i]);
-			break;
-		}
-
-		if (blk_errs > 0) {
-			/* Reset the Error counts! */
-			reg_rmw(pcsr_base + 0x08 + (i * 0x80), VAL_SH(0x19, 0),
-				MASK_WID_SH(8, 0));
-
-			reg_rmw(pcsr_base + 0x08 + (i * 0x80), VAL_SH(0x00, 0),
-				MASK_WID_SH(8, 0));
-		}
-
-		status &= (current_state[i] == 1);
-	}
-
-	return status;
-}
-
-static int netcp_xgbe_serdes_check_lane(void __iomem *serdes_regs,
-					void __iomem *sw_regs)
-{
-	u32 current_state[2] = {0, 0};
-	int retries = 0, link_up;
-	u32 lane_down[2];
-
-	do {
-		lane_down[0] = 0;
-		lane_down[1] = 0;
-
-		link_up = netcp_xgbe_check_link_status(serdes_regs, sw_regs, 2,
-						       current_state,
-						       lane_down);
-
-		/* if we did not get link up then wait 100ms before calling
-		 * it again
-		 */
-		if (link_up)
-			break;
-
-		if (lane_down[0])
-			pr_debug("XGBE: detected link down on lane 0\n");
-
-		if (lane_down[1])
-			pr_debug("XGBE: detected link down on lane 1\n");
-
-		if (++retries > 1) {
-			pr_debug("XGBE: timeout waiting for serdes link up\n");
-			return -ETIMEDOUT;
-		}
-		mdelay(100);
-	} while (!link_up);
-
-	pr_debug("XGBE: PCSR link is up\n");
-	return 0;
-}
-
-static void netcp_xgbe_serdes_setup_cm_c1_c2(void __iomem *serdes_regs,
-					     int lane, int cm, int c1, int c2)
-{
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(cfg_cm_c1_c2); i++) {
-		reg_rmw(serdes_regs + cfg_cm_c1_c2[i].ofs + (0x200 * lane),
-			cfg_cm_c1_c2[i].val,
-			cfg_cm_c1_c2[i].mask);
-	}
-}
-
-static void netcp_xgbe_reset_serdes(void __iomem *serdes_regs)
-{
-	/* Toggle the POR_EN bit in CONFIG.CPU_CTRL */
-	/* enable POR_EN bit */
-	reg_rmw(serdes_regs + PCSR_CPU_CTRL_OFFSET, POR_EN, POR_EN);
-	usleep_range(10, 100);
-
-	/* disable POR_EN bit */
-	reg_rmw(serdes_regs + PCSR_CPU_CTRL_OFFSET, 0, POR_EN);
-	usleep_range(10, 100);
-}
-
-static int netcp_xgbe_serdes_config(void __iomem *serdes_regs,
-				    void __iomem *sw_regs)
-{
-	u32 ret, i;
-
-	netcp_xgbe_serdes_pll_disable(serdes_regs);
-	netcp_xgbe_serdes_cmu_init(serdes_regs);
-
-	for (i = 0; i < 2; i++)
-		netcp_xgbe_serdes_lane_config(serdes_regs, i);
-
-	netcp_xgbe_serdes_com_enable(serdes_regs);
-	/* This is EVM + RTM-BOC specific */
-	for (i = 0; i < 2; i++)
-		netcp_xgbe_serdes_setup_cm_c1_c2(serdes_regs, i, 0, 0, 5);
-
-	netcp_xgbe_serdes_pll_enable(serdes_regs);
-	for (i = 0; i < 2; i++)
-		netcp_xgbe_serdes_lane_enable(serdes_regs, i);
-
-	/* SB PLL Status Poll */
-	ret = netcp_xgbe_wait_pll_locked(sw_regs);
-	if (ret)
-		return ret;
-
-	netcp_xgbe_serdes_enable_xgmii_port(sw_regs);
-	netcp_xgbe_serdes_check_lane(serdes_regs, sw_regs);
-	return ret;
-}
-
-int netcp_xgbe_serdes_init(void __iomem *serdes_regs, void __iomem *xgbe_regs)
-{
-	u32 val;
-
-	/* read COMLANE bits 4:0 */
-	val = readl(serdes_regs + 0xa00);
-	if (val & 0x1f) {
-		pr_debug("XGBE: serdes already in operation - reset\n");
-		netcp_xgbe_reset_serdes(serdes_regs);
-	}
-	return netcp_xgbe_serdes_config(serdes_regs, xgbe_regs);
-}
diff -Naur --no-dereference a/drivers/net/ethernet/ti/prueth.c b/drivers/net/ethernet/ti/prueth.c
--- a/drivers/net/ethernet/ti/prueth.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/net/ethernet/ti/prueth.c	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,2046 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+/* PRU ICSS Ethernet Driver
+ *
+ * Copyright (C) 2015-2018 Texas Instruments Incorporated - http://www.ti.com
+ *	Roger Quadros <rogerq@ti.com>
+ *	Andrew F. Davis <afd@ti.com>
+ */
+
+#include <linux/etherdevice.h>
+#include <linux/genalloc.h>
+#include <linux/if_vlan.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_irq.h>
+#include <linux/of_mdio.h>
+#include <linux/of_net.h>
+#include <linux/of_platform.h>
+#include <linux/phy.h>
+#include <linux/pruss.h>
+#include <linux/regmap.h>
+#include <linux/remoteproc.h>
+
+#include "prueth.h"
+#include "icss_mii_rt.h"
+#include "icss_switch.h"
+#include "icss_vlan_mcast_filter_mmap.h"
+
+#define PRUETH_MODULE_VERSION "0.2"
+#define PRUETH_MODULE_DESCRIPTION "PRUSS Ethernet driver"
+
+#define OCMC_RAM_SIZE		(SZ_64K - SZ_8K)
+
+/* TX Minimum Inter packet gap */
+#define TX_MIN_IPG		0xb8
+
+#define TX_START_DELAY		0x40
+#define TX_CLK_DELAY_100M	0x6
+#define TX_CLK_DELAY_10M	0
+
+/* PRUSS_IEP_GLOBAL_CFG register definitions */
+#define PRUSS_IEP_GLOBAL_CFG	0
+
+#define PRUSS_IEP_GLOBAL_CFG_CNT_ENABLE		BIT(0)
+
+/* PRUSS local memory map */
+#define ICSS_LOCAL_SHARED_RAM   0x00010000
+
+/* Netif debug messages possible */
+#define PRUETH_EMAC_DEBUG	(NETIF_MSG_DRV | \
+				 NETIF_MSG_PROBE | \
+				 NETIF_MSG_LINK | \
+				 NETIF_MSG_TIMER | \
+				 NETIF_MSG_IFDOWN | \
+				 NETIF_MSG_IFUP | \
+				 NETIF_MSG_RX_ERR | \
+				 NETIF_MSG_TX_ERR | \
+				 NETIF_MSG_TX_QUEUED | \
+				 NETIF_MSG_INTR | \
+				 NETIF_MSG_TX_DONE | \
+				 NETIF_MSG_RX_STATUS | \
+				 NETIF_MSG_PKTDATA | \
+				 NETIF_MSG_HW | \
+				 NETIF_MSG_WOL)
+
+static int debug_level = -1;
+module_param(debug_level, int, 0444);
+MODULE_PARM_DESC(debug_level, "PRUETH debug level (NETIF_MSG bits)");
+
+#define EMAC_POLL_WEIGHT	(64) /* Default NAPI poll weight */
+#define EMAC_MAX_PKTLEN		(ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
+#define EMAC_MIN_PKTLEN		(60)
+
+/* In switch mode there are 3 real ports i.e. 3 mac addrs.
+ * however Linux sees only the host side port. The other 2 ports
+ * are the switch ports.
+ * In emac mode there are 2 real ports i.e. 2 mac addrs.
+ * Linux sees both the ports.
+ */
+enum prueth_port {
+	PRUETH_PORT_HOST = 0,	/* host side port */
+	PRUETH_PORT_MII0,	/* physical port MII 0 */
+	PRUETH_PORT_MII1,	/* physical port MII 1 */
+};
+
+enum prueth_mac {
+	PRUETH_MAC0 = 0,
+	PRUETH_MAC1,
+	PRUETH_NUM_MACS,
+};
+
+/* In both switch & emac modes there are 3 port queues
+ * EMAC mode:
+ *	RX packets for both MII0 & MII1 ports come on
+ *	QUEUE_HOST.
+ *	TX packets for MII0 go on QUEUE_MII0, TX packets
+ *	for MII1 go on QUEUE_MII1.
+ * Switch mode:
+ *	Host port RX packets come on QUEUE_HOST
+ *	TX packets might have to go on MII0 or MII1 or both.
+ *	MII0 TX queue is QUEUE_MII0 and MII1 TX queue is
+ *	QUEUE_MII1.
+ */
+enum prueth_port_queue_id {
+	PRUETH_PORT_QUEUE_HOST = 0,
+	PRUETH_PORT_QUEUE_MII0,
+	PRUETH_PORT_QUEUE_MII1,
+	PRUETH_PORT_QUEUE_MAX,
+};
+
+/* Each port queue has 4 queues and 1 collision queue */
+enum prueth_queue_id {
+	PRUETH_QUEUE1 = 0,
+	PRUETH_QUEUE2,
+	PRUETH_QUEUE3,
+	PRUETH_QUEUE4,
+	PRUETH_COLQUEUE,	/* collision queue */
+};
+
+/* PRUeth memory range identifiers */
+enum prueth_mem {
+	PRUETH_MEM_DRAM0 = 0,
+	PRUETH_MEM_DRAM1,
+	PRUETH_MEM_SHARED_RAM,
+	PRUETH_MEM_OCMC,
+	PRUETH_MEM_MAX,
+};
+
+/* ensure that order of PRUSS mem regions is same as above */
+static enum pruss_mem pruss_mem_ids[] = { PRUSS_MEM_DRAM0, PRUSS_MEM_DRAM1,
+					  PRUSS_MEM_SHRD_RAM2 };
+
+/**
+ * struct prueth_private_data - PRU Ethernet private data
+ * @fw_names: firmware names to be used for PRUSS ethernet usecases
+ */
+struct prueth_private_data {
+	const char *fw_names[PRUSS_NUM_PRUS];
+};
+
+/* data for each emac port */
+struct prueth_emac {
+	struct prueth *prueth;
+	struct net_device *ndev;
+	u8 mac_addr[6];
+	struct napi_struct napi;
+	u32 msg_enable;
+
+	int link;
+	int speed;
+	int duplex;
+
+	const char *phy_id;
+	struct device_node *phy_node;
+	int phy_if;
+	struct phy_device *phydev;
+	struct rproc *pru;
+
+	enum prueth_port port_id;
+	enum prueth_port_queue_id tx_port_queue;
+
+	enum prueth_queue_id rx_queue_start;
+	enum prueth_queue_id rx_queue_end;
+
+	enum prueth_mem dram;
+
+	int rx_irq;
+	int tx_irq;
+
+	struct prueth_queue_desc __iomem *rx_queue_descs;
+	struct prueth_queue_desc __iomem *tx_queue_descs;
+
+	struct port_statistics stats; /* stats holder when i/f is down */
+	unsigned char mc_filter_mask[ETH_ALEN];	/* for multicast filtering */
+
+	spinlock_t lock;	/* serialize access */
+};
+
+/**
+ * struct prueth - PRUeth structure
+ * @dev: device
+ * @pruss: pruss handle
+ * @pru0: rproc instance to PRU0
+ * @pru1: rproc instance to PRU1
+ * @mem: PRUSS memory resources we need to access
+ * @sram_pool: OCMC ram pool for buffers
+ * @mii_rt: regmap to mii_rt block
+ * @iep: regmap to IEP block
+ *
+ * @eth_node: node for each emac node
+ * @emac: emac data for three ports, one host and two physical
+ * @registered_netdevs: net device for each registered emac
+ */
+struct prueth {
+	struct device *dev;
+	struct pruss *pruss;
+	struct rproc *pru0, *pru1;
+	struct pruss_mem_region mem[PRUETH_MEM_MAX];
+	struct gen_pool *sram_pool;
+	struct regmap *mii_rt;
+	struct regmap *iep;
+
+	struct device_node *eth_node[PRUETH_NUM_MACS];
+	struct prueth_emac *emac[PRUETH_NUM_MACS];
+	struct net_device *registered_netdevs[PRUETH_NUM_MACS];
+};
+
+static inline u32 prueth_read_reg(struct prueth *prueth,
+				  enum prueth_mem region,
+				  unsigned int reg)
+{
+	return readl_relaxed(prueth->mem[region].va + reg);
+}
+
+static inline void prueth_write_reg(struct prueth *prueth,
+				    enum prueth_mem region,
+				    unsigned int reg, u32 val)
+{
+	writel_relaxed(val, prueth->mem[region].va + reg);
+}
+
+static inline
+void prueth_set_reg(struct prueth *prueth, enum prueth_mem region,
+		    unsigned int reg, u32 mask, u32 set)
+{
+	u32 val;
+
+	val = prueth_read_reg(prueth, region, reg);
+	val &= ~mask;
+	val |= (set & mask);
+	prueth_write_reg(prueth, region, reg, val);
+}
+
+static const struct prueth_queue_info queue_infos[][4] = {
+	[PRUETH_PORT_QUEUE_HOST] = {
+		[PRUETH_QUEUE1] = {
+			P0_Q1_BUFFER_OFFSET,
+			HOST_QUEUE_DESC_OFFSET,
+			P0_Q1_BD_OFFSET,
+			P0_Q1_BD_OFFSET + ((HOST_QUEUE_1_SIZE - 1) * BD_SIZE),
+		},
+		[PRUETH_QUEUE2] = {
+			P0_Q2_BUFFER_OFFSET,
+			HOST_QUEUE_DESC_OFFSET + 8,
+			P0_Q2_BD_OFFSET,
+			P0_Q2_BD_OFFSET + ((HOST_QUEUE_2_SIZE - 1) * BD_SIZE),
+		},
+		[PRUETH_QUEUE3] = {
+			P0_Q3_BUFFER_OFFSET,
+			HOST_QUEUE_DESC_OFFSET + 16,
+			P0_Q3_BD_OFFSET,
+			P0_Q3_BD_OFFSET + ((HOST_QUEUE_3_SIZE - 1) * BD_SIZE),
+		},
+		[PRUETH_QUEUE4] = {
+			P0_Q4_BUFFER_OFFSET,
+			HOST_QUEUE_DESC_OFFSET + 24,
+			P0_Q4_BD_OFFSET,
+			P0_Q4_BD_OFFSET + ((HOST_QUEUE_4_SIZE - 1) * BD_SIZE),
+		},
+	},
+	[PRUETH_PORT_QUEUE_MII0] = {
+		[PRUETH_QUEUE1] = {
+			P1_Q1_BUFFER_OFFSET,
+			P1_Q1_BUFFER_OFFSET + ((QUEUE_1_SIZE - 1) * ICSS_BLOCK_SIZE),
+			P1_Q1_BD_OFFSET,
+			P1_Q1_BD_OFFSET + ((QUEUE_1_SIZE - 1) * BD_SIZE),
+		},
+		[PRUETH_QUEUE2] = {
+			P1_Q2_BUFFER_OFFSET,
+			P1_Q2_BUFFER_OFFSET + ((QUEUE_2_SIZE - 1) * ICSS_BLOCK_SIZE),
+			P1_Q2_BD_OFFSET,
+			P1_Q2_BD_OFFSET + ((QUEUE_2_SIZE - 1) * BD_SIZE),
+		},
+		[PRUETH_QUEUE3] = {
+			P1_Q3_BUFFER_OFFSET,
+			P1_Q3_BUFFER_OFFSET + ((QUEUE_3_SIZE - 1) * ICSS_BLOCK_SIZE),
+			P1_Q3_BD_OFFSET,
+			P1_Q3_BD_OFFSET + ((QUEUE_3_SIZE - 1) * BD_SIZE),
+		},
+		[PRUETH_QUEUE4] = {
+			P1_Q4_BUFFER_OFFSET,
+			P1_Q4_BUFFER_OFFSET + ((QUEUE_4_SIZE - 1) * ICSS_BLOCK_SIZE),
+			P1_Q4_BD_OFFSET,
+			P1_Q4_BD_OFFSET + ((QUEUE_4_SIZE - 1) * BD_SIZE),
+		},
+	},
+	[PRUETH_PORT_QUEUE_MII1] = {
+		[PRUETH_QUEUE1] = {
+			P2_Q1_BUFFER_OFFSET,
+			P2_Q1_BUFFER_OFFSET + ((QUEUE_1_SIZE - 1) * ICSS_BLOCK_SIZE),
+			P2_Q1_BD_OFFSET,
+			P2_Q1_BD_OFFSET + ((QUEUE_1_SIZE - 1) * BD_SIZE),
+		},
+		[PRUETH_QUEUE2] = {
+			P2_Q2_BUFFER_OFFSET,
+			P2_Q2_BUFFER_OFFSET + ((QUEUE_2_SIZE - 1) * ICSS_BLOCK_SIZE),
+			P2_Q2_BD_OFFSET,
+			P2_Q2_BD_OFFSET + ((QUEUE_2_SIZE - 1) * BD_SIZE),
+		},
+		[PRUETH_QUEUE3] = {
+			P2_Q3_BUFFER_OFFSET,
+			P2_Q3_BUFFER_OFFSET + ((QUEUE_3_SIZE - 1) * ICSS_BLOCK_SIZE),
+			P2_Q3_BD_OFFSET,
+			P2_Q3_BD_OFFSET + ((QUEUE_3_SIZE - 1) * BD_SIZE),
+		},
+		[PRUETH_QUEUE4] = {
+			P2_Q4_BUFFER_OFFSET,
+			P2_Q4_BUFFER_OFFSET + ((QUEUE_4_SIZE - 1) * ICSS_BLOCK_SIZE),
+			P2_Q4_BD_OFFSET,
+			P2_Q4_BD_OFFSET + ((QUEUE_4_SIZE - 1) * BD_SIZE),
+		},
+	},
+};
+
+static const struct prueth_queue_desc queue_descs[][4] = {
+	[PRUETH_PORT_QUEUE_HOST] = {
+		{ .rd_ptr = P0_Q1_BD_OFFSET, .wr_ptr = P0_Q1_BD_OFFSET, },
+		{ .rd_ptr = P0_Q2_BD_OFFSET, .wr_ptr = P0_Q2_BD_OFFSET, },
+		{ .rd_ptr = P0_Q3_BD_OFFSET, .wr_ptr = P0_Q3_BD_OFFSET, },
+		{ .rd_ptr = P0_Q4_BD_OFFSET, .wr_ptr = P0_Q4_BD_OFFSET, },
+	},
+	[PRUETH_PORT_QUEUE_MII0] = {
+		{ .rd_ptr = P1_Q1_BD_OFFSET, .wr_ptr = P1_Q1_BD_OFFSET, },
+		{ .rd_ptr = P1_Q2_BD_OFFSET, .wr_ptr = P1_Q2_BD_OFFSET, },
+		{ .rd_ptr = P1_Q3_BD_OFFSET, .wr_ptr = P1_Q3_BD_OFFSET, },
+		{ .rd_ptr = P1_Q4_BD_OFFSET, .wr_ptr = P1_Q4_BD_OFFSET, },
+	},
+	[PRUETH_PORT_QUEUE_MII1] = {
+		{ .rd_ptr = P2_Q1_BD_OFFSET, .wr_ptr = P2_Q1_BD_OFFSET, },
+		{ .rd_ptr = P2_Q2_BD_OFFSET, .wr_ptr = P2_Q2_BD_OFFSET, },
+		{ .rd_ptr = P2_Q3_BD_OFFSET, .wr_ptr = P2_Q3_BD_OFFSET, },
+		{ .rd_ptr = P2_Q4_BD_OFFSET, .wr_ptr = P2_Q4_BD_OFFSET, },
+	}
+};
+
+static int prueth_hostconfig(struct prueth *prueth)
+{
+	void __iomem *sram_base = prueth->mem[PRUETH_MEM_SHARED_RAM].va;
+	void __iomem *sram;
+
+	/* queue size lookup table */
+	sram = sram_base + HOST_QUEUE_SIZE_ADDR;
+	writew(HOST_QUEUE_1_SIZE, sram);
+	writew(HOST_QUEUE_2_SIZE, sram + 2);
+	writew(HOST_QUEUE_3_SIZE, sram + 4);
+	writew(HOST_QUEUE_4_SIZE, sram + 6);
+
+	/* queue information table */
+	sram = sram_base + HOST_Q1_RX_CONTEXT_OFFSET;
+	memcpy_toio(sram, queue_infos[PRUETH_PORT_QUEUE_HOST],
+		    sizeof(queue_infos[PRUETH_PORT_QUEUE_HOST]));
+
+	/* buffer offset table */
+	sram = sram_base + HOST_QUEUE_OFFSET_ADDR;
+	writew(P0_Q1_BUFFER_OFFSET, sram);
+	writew(P0_Q2_BUFFER_OFFSET, sram + 2);
+	writew(P0_Q3_BUFFER_OFFSET, sram + 4);
+	writew(P0_Q4_BUFFER_OFFSET, sram + 6);
+
+	/* buffer descriptor offset table*/
+	sram = sram_base + HOST_QUEUE_DESCRIPTOR_OFFSET_ADDR;
+	writew(P0_Q1_BD_OFFSET, sram);
+	writew(P0_Q2_BD_OFFSET, sram + 2);
+	writew(P0_Q3_BD_OFFSET, sram + 4);
+	writew(P0_Q4_BD_OFFSET, sram + 6);
+
+	/* queue table */
+	sram = sram_base + HOST_QUEUE_DESC_OFFSET;
+	memcpy_toio(sram, queue_descs[PRUETH_PORT_QUEUE_HOST],
+		    sizeof(queue_descs[PRUETH_PORT_QUEUE_HOST]));
+
+	return 0;
+}
+
+#define prueth_mii_set(dir, port, mask, set) \
+	regmap_update_bits(prueth->mii_rt, PRUSS_MII_RT_##dir##CFG##port, \
+			   PRUSS_MII_RT_##dir##CFG_##dir##_##mask, set)
+
+static void prueth_mii_init(struct prueth *prueth)
+{
+	/* Configuration of Port 0 Rx */
+	prueth_mii_set(RX, 0, ENABLE, PRUSS_MII_RT_RXCFG_RX_ENABLE);
+	prueth_mii_set(RX, 0, DATA_RDY_MODE_DIS,
+		       PRUSS_MII_RT_RXCFG_RX_DATA_RDY_MODE_DIS);
+	prueth_mii_set(RX, 0, MUX_SEL, 0x0);
+	prueth_mii_set(RX, 0, L2_EN, PRUSS_MII_RT_RXCFG_RX_L2_EN);
+	prueth_mii_set(RX, 0, CUT_PREAMBLE, PRUSS_MII_RT_RXCFG_RX_CUT_PREAMBLE);
+	prueth_mii_set(RX, 0, L2_EOF_SCLR_DIS,
+		       PRUSS_MII_RT_RXCFG_RX_L2_EOF_SCLR_DIS);
+
+	/* Configuration of Port 0 Tx */
+	prueth_mii_set(TX, 0, ENABLE, PRUSS_MII_RT_TXCFG_TX_ENABLE);
+	prueth_mii_set(TX, 0, AUTO_PREAMBLE,
+		       PRUSS_MII_RT_TXCFG_TX_AUTO_PREAMBLE);
+	prueth_mii_set(TX, 0, 32_MODE_EN, PRUSS_MII_RT_TXCFG_TX_32_MODE_EN);
+	prueth_mii_set(TX, 0, MUX_SEL, 0x0);
+	prueth_mii_set(TX, 0, START_DELAY_MASK,
+		       TX_START_DELAY << PRUSS_MII_RT_TXCFG_TX_START_DELAY_SHIFT);
+	prueth_mii_set(TX, 0, CLK_DELAY_MASK,
+		       TX_CLK_DELAY_100M << PRUSS_MII_RT_TXCFG_TX_CLK_DELAY_SHIFT);
+
+	/* Configuration of Port 1 Rx */
+	prueth_mii_set(RX, 1, ENABLE, PRUSS_MII_RT_RXCFG_RX_ENABLE);
+	prueth_mii_set(RX, 1,
+		       DATA_RDY_MODE_DIS, PRUSS_MII_RT_RXCFG_RX_DATA_RDY_MODE_DIS);
+	prueth_mii_set(RX, 1, MUX_SEL, PRUSS_MII_RT_RXCFG_RX_MUX_SEL);
+	prueth_mii_set(RX, 1, L2_EN, PRUSS_MII_RT_RXCFG_RX_L2_EN);
+	prueth_mii_set(RX, 1, CUT_PREAMBLE, PRUSS_MII_RT_RXCFG_RX_CUT_PREAMBLE);
+	prueth_mii_set(RX, 1, L2_EOF_SCLR_DIS,
+		       PRUSS_MII_RT_RXCFG_RX_L2_EOF_SCLR_DIS);
+
+	/* Configuration of Port 1 Tx */
+	prueth_mii_set(TX, 1, ENABLE, PRUSS_MII_RT_TXCFG_TX_ENABLE);
+	prueth_mii_set(TX, 1, AUTO_PREAMBLE,
+		       PRUSS_MII_RT_TXCFG_TX_AUTO_PREAMBLE);
+	prueth_mii_set(TX, 1, 32_MODE_EN, PRUSS_MII_RT_TXCFG_TX_32_MODE_EN);
+	prueth_mii_set(TX, 1, MUX_SEL, PRUSS_MII_RT_TXCFG_TX_MUX_SEL);
+	prueth_mii_set(TX, 1, START_DELAY_MASK,
+		       TX_START_DELAY << PRUSS_MII_RT_TXCFG_TX_START_DELAY_SHIFT);
+	prueth_mii_set(TX, 1, CLK_DELAY_MASK,
+		       TX_CLK_DELAY_100M << PRUSS_MII_RT_TXCFG_TX_CLK_DELAY_SHIFT);
+}
+
+static void prueth_clearmem(struct prueth *prueth, enum prueth_mem region)
+{
+	memset_io(prueth->mem[region].va, 0, prueth->mem[region].size);
+}
+
+static int prueth_hostinit(struct prueth *prueth)
+{
+	/* Clear shared RAM */
+	prueth_clearmem(prueth, PRUETH_MEM_SHARED_RAM);
+
+	/* Clear OCMC RAM */
+	prueth_clearmem(prueth, PRUETH_MEM_OCMC);
+
+	/* Clear data RAMs */
+	if (prueth->eth_node[PRUETH_MAC0])
+		prueth_clearmem(prueth, PRUETH_MEM_DRAM0);
+	if (prueth->eth_node[PRUETH_MAC1])
+		prueth_clearmem(prueth, PRUETH_MEM_DRAM1);
+
+	/* Initialize host queues in shared RAM */
+	prueth_hostconfig(prueth);
+
+	/* Configure MII_RT */
+	prueth_mii_init(prueth);
+
+	/* Enable IEP Counter */
+	regmap_update_bits(prueth->iep, PRUSS_IEP_GLOBAL_CFG,
+			   PRUSS_IEP_GLOBAL_CFG_CNT_ENABLE,
+			   PRUSS_IEP_GLOBAL_CFG_CNT_ENABLE);
+
+	return 0;
+}
+
+static int prueth_port_enable(struct prueth_emac *emac, bool enable)
+{
+	void __iomem *port_ctrl;
+	struct prueth *prueth = emac->prueth;
+
+	port_ctrl = prueth->mem[emac->dram].va + PORT_CONTROL_ADDR;
+
+	writeb(!!enable, port_ctrl);
+
+	return 0;
+}
+
+static int prueth_emac_config(struct prueth_emac *emac)
+{
+	struct prueth *prueth = emac->prueth;
+
+	/* PRU needs local shared RAM address for C28 */
+	u32 sharedramaddr = ICSS_LOCAL_SHARED_RAM;
+
+	/* PRU needs real global OCMC address for C30*/
+	u32 ocmcaddr = (u32)prueth->mem[PRUETH_MEM_OCMC].pa;
+	void __iomem *sram = prueth->mem[PRUETH_MEM_SHARED_RAM].va;
+	void __iomem *dram_base;
+	void __iomem *mac_addr;
+	void __iomem *dram;
+
+	/* Clear data RAM */
+	prueth_clearmem(prueth, emac->dram);
+
+	dram_base = prueth->mem[emac->dram].va;
+
+	/* setup mac address */
+	mac_addr = dram_base + PORT_MAC_ADDR;
+	memcpy_toio(mac_addr, emac->mac_addr, 6);
+
+	/* queue information table */
+	dram = dram_base + TX_CONTEXT_Q1_OFFSET_ADDR;
+	memcpy_toio(dram, queue_infos[emac->port_id],
+		    sizeof(queue_infos[emac->port_id]));
+
+	/* queue table */
+	dram = dram_base + PORT_QUEUE_DESC_OFFSET;
+	memcpy_toio(dram, queue_descs[emac->port_id],
+		    sizeof(queue_descs[emac->port_id]));
+
+	emac->rx_queue_descs = sram + HOST_QUEUE_DESC_OFFSET;
+	emac->tx_queue_descs = dram;
+
+	/* Set in constant table C28 of PRU0 to ICSS Shared memory */
+	pru_rproc_set_ctable(emac->pru, PRU_C28, sharedramaddr);
+
+	/* Set in constant table C30 of PRU0 to OCMC memory */
+	pru_rproc_set_ctable(emac->pru, PRU_C30, ocmcaddr);
+
+	return 0;
+}
+
+/* update phy/port status information for firmware */
+static void emac_update_phystatus(struct prueth_emac *emac)
+{
+	struct prueth *prueth = emac->prueth;
+	enum prueth_mem region;
+	u32 phy_speed, port_status = 0;
+	u8 delay;
+
+	region = emac->dram;
+	phy_speed = emac->speed;
+	prueth_write_reg(prueth, region, PHY_SPEED_OFFSET, phy_speed);
+
+	if (phy_speed == SPEED_10)
+		delay = TX_CLK_DELAY_10M;
+	else
+		delay = TX_CLK_DELAY_100M;
+
+	if (emac->port_id) {
+		prueth_mii_set(TX, 1, CLK_DELAY_MASK,
+			       delay << PRUSS_MII_RT_TXCFG_TX_CLK_DELAY_SHIFT);
+	} else {
+		prueth_mii_set(TX, 0, CLK_DELAY_MASK,
+			       delay << PRUSS_MII_RT_TXCFG_TX_CLK_DELAY_SHIFT);
+	}
+
+	if (emac->duplex == DUPLEX_HALF)
+		port_status |= PORT_IS_HD_MASK;
+	if (emac->link)
+		port_status |= PORT_LINK_MASK;
+	writeb(port_status, prueth->mem[region].va + PORT_STATUS_OFFSET);
+}
+
+/* called back by PHY layer if there is change in link state of hw port*/
+static void emac_adjust_link(struct net_device *ndev)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+	struct phy_device *phydev = emac->phydev;
+	unsigned long flags;
+	bool new_state = false;
+
+	spin_lock_irqsave(&emac->lock, flags);
+
+	if (phydev->link) {
+		/* check the mode of operation - full/half duplex */
+		if (phydev->duplex != emac->duplex) {
+			new_state = true;
+			emac->duplex = phydev->duplex;
+		}
+		if (phydev->speed != emac->speed) {
+			new_state = true;
+			emac->speed = phydev->speed;
+		}
+		if (!emac->link) {
+			new_state = true;
+			emac->link = 1;
+		}
+	} else if (emac->link) {
+		new_state = true;
+		emac->link = 0;
+		/* defaults for no link */
+
+		/* f/w only support 10 or 100 */
+		emac->speed = SPEED_100;
+
+		/* half duplex may not be supported by f/w */
+		emac->duplex = DUPLEX_FULL;
+	}
+
+	emac_update_phystatus(emac);
+
+	if (new_state)
+		phy_print_status(phydev);
+
+	if (emac->link) {
+		/* link ON */
+		netif_carrier_on(ndev);
+
+		/* reactivate the transmit queue if it is stopped */
+		if (netif_running(ndev) && netif_queue_stopped(ndev))
+			netif_wake_queue(ndev);
+	} else {
+		/* link OFF */
+		netif_carrier_off(ndev);
+		if (!netif_queue_stopped(ndev))
+			netif_stop_queue(ndev);
+	}
+
+	spin_unlock_irqrestore(&emac->lock, flags);
+}
+
+/**
+ * emac_tx_hardirq - EMAC Tx interrupt handler
+ * @irq: interrupt number
+ * @dev_id: pointer to net_device
+ *
+ * This is called whenever a packet has finished being transmitted, this clears
+ * up hardware buffer space, our only task is to re-enable the transmit queue
+ * if it was previously disabled due to hardware queue being full
+ *
+ * Returns interrupt handled condition
+ */
+static irqreturn_t emac_tx_hardirq(int irq, void *dev_id)
+{
+	struct net_device *ndev = (struct net_device *)dev_id;
+
+	if (unlikely(netif_queue_stopped(ndev)))
+		netif_wake_queue(ndev);
+
+	return IRQ_HANDLED;
+}
+
+/**
+ * emac_rx_hardirq - EMAC Rx interrupt handler
+ * @irq: interrupt number
+ * @dev_id: pointer to net_device
+ *
+ * EMAC Interrupt handler - we only schedule NAPI and not process any packets
+ * here.
+ *
+ * Returns interrupt handled condition
+ */
+static irqreturn_t emac_rx_hardirq(int irq, void *dev_id)
+{
+	struct net_device *ndev = (struct net_device *)dev_id;
+	struct prueth_emac *emac = netdev_priv(ndev);
+
+	if (likely(netif_running(ndev))) {
+		/* disable Rx system event */
+		disable_irq_nosync(emac->rx_irq);
+		napi_schedule(&emac->napi);
+	}
+
+	return IRQ_HANDLED;
+}
+
+/**
+ * prueth_tx_enqueue - queue a packet to firmware for transmission
+ *
+ * @emac: EMAC data structure
+ * @skb: packet data buffer
+ * @queue_id: priority queue id
+ */
+static int prueth_tx_enqueue(struct prueth_emac *emac, struct sk_buff *skb,
+			     enum prueth_queue_id queue_id)
+{
+	struct net_device *ndev = emac->ndev;
+	int pktlen;
+	struct prueth_queue_desc __iomem *queue_desc;
+	const struct prueth_queue_info *txqueue;
+	u16 bd_rd_ptr, bd_wr_ptr, update_wr_ptr;
+	int write_block, read_block, free_blocks, update_block, pkt_block_size;
+	unsigned int buffer_desc_count;
+	bool buffer_wrapped = false;
+	void *src_addr;
+	void *dst_addr;
+
+	/* OCMC RAM is not cached and write order is not important */
+	void *ocmc_ram = (__force void *)emac->prueth->mem[PRUETH_MEM_OCMC].va;
+	void __iomem *dram;
+	u32 wr_buf_desc;
+	int ret;
+	int txport = emac->tx_port_queue; /* which port to tx: MII0 or MII1 */
+
+	dram = emac->prueth->mem[emac->dram].va;
+	ret = skb_padto(skb, EMAC_MIN_PKTLEN);
+	if (ret) {
+		if (netif_msg_tx_err(emac) && net_ratelimit())
+			netdev_err(ndev, "packet pad failed");
+		return ret;
+	}
+	src_addr = skb->data;
+
+	/* pad packet if needed */
+	pktlen = skb->len;
+	if (pktlen < EMAC_MIN_PKTLEN)
+		pktlen = EMAC_MIN_PKTLEN;
+
+	/* Get the tx queue */
+	queue_desc = emac->tx_queue_descs + queue_id;
+	txqueue = &queue_infos[txport][queue_id];
+	buffer_desc_count = txqueue->buffer_desc_end -
+			    txqueue->buffer_desc_offset;
+	buffer_desc_count /= BD_SIZE;
+	buffer_desc_count++;
+
+	bd_rd_ptr = readw(&queue_desc->rd_ptr);
+	bd_wr_ptr = readw(&queue_desc->wr_ptr);
+
+	/* the PRU firmware deals mostly in pointers already
+	 * offset into ram, we would like to deal in indexes
+	 * within the queue we are working with for code
+	 * simplicity, calculate this here
+	 */
+	write_block = (bd_wr_ptr - txqueue->buffer_desc_offset) / BD_SIZE;
+	read_block = (bd_rd_ptr - txqueue->buffer_desc_offset) / BD_SIZE;
+	if (write_block > read_block) {
+		free_blocks = buffer_desc_count - write_block;
+		free_blocks += read_block;
+	} else if (write_block < read_block) {
+		free_blocks = read_block - write_block;
+	} else { /* they are all free */
+		free_blocks = buffer_desc_count;
+	}
+	pkt_block_size = DIV_ROUND_UP(pktlen, ICSS_BLOCK_SIZE);
+	if (pkt_block_size > free_blocks) /* out of queue space */
+		return -ENOBUFS;
+
+	/* calculate end BD address post write */
+	update_block = write_block + pkt_block_size;
+
+	/* Check for wrap around */
+	if (update_block >= buffer_desc_count) {
+		update_block %= buffer_desc_count;
+		buffer_wrapped = true;
+	}
+
+	dst_addr = ocmc_ram + txqueue->buffer_offset +
+		   (write_block * ICSS_BLOCK_SIZE);
+
+	/* Copy the data from socket buffer(DRAM) to PRU buffers(OCMC) */
+	if (buffer_wrapped) { /* wrapped around buffer */
+		int bytes = (buffer_desc_count - write_block) * ICSS_BLOCK_SIZE;
+		int remaining;
+
+		/* bytes is integral multiple of ICSS_BLOCK_SIZE but
+		 * entire packet may have fit within the last BD
+		 * if pkt_info.length is not integral multiple of
+		 * ICSS_BLOCK_SIZE
+		 */
+		if (pktlen < bytes)
+			bytes = pktlen;
+
+		/* copy non-wrapped part */
+		memcpy(dst_addr, src_addr, bytes);
+
+		/* copy wrapped part */
+		src_addr += bytes;
+		remaining = pktlen - bytes;
+		dst_addr = ocmc_ram + txqueue->buffer_offset;
+		memcpy(dst_addr, src_addr, remaining);
+	} else {
+		memcpy(dst_addr, src_addr, pktlen);
+	}
+
+	/* update first buffer descriptor */
+	wr_buf_desc = (pktlen << PRUETH_BD_LENGTH_SHIFT) & PRUETH_BD_LENGTH_MASK;
+	writel(wr_buf_desc, dram + bd_wr_ptr);
+
+	/* update the write pointer in this queue descriptor, the firmware
+	 * polls for this change so this will signal the start of transmission
+	 */
+	update_wr_ptr = txqueue->buffer_desc_offset + (update_block * BD_SIZE);
+	writew(update_wr_ptr, &queue_desc->wr_ptr);
+
+	return 0;
+}
+
+static void parse_packet_info(u32 buffer_descriptor,
+			      struct prueth_packet_info *pkt_info)
+{
+	pkt_info->shadow = !!(buffer_descriptor & PRUETH_BD_SHADOW_MASK);
+	pkt_info->port = (buffer_descriptor & PRUETH_BD_PORT_MASK) >>
+			 PRUETH_BD_PORT_SHIFT;
+	pkt_info->length = (buffer_descriptor & PRUETH_BD_LENGTH_MASK) >>
+			   PRUETH_BD_LENGTH_SHIFT;
+	pkt_info->broadcast = !!(buffer_descriptor & PRUETH_BD_BROADCAST_MASK);
+	pkt_info->error = !!(buffer_descriptor & PRUETH_BD_ERROR_MASK);
+}
+
+/* get packet from queue
+ * negative for error
+ */
+static int emac_rx_packet(struct prueth_emac *emac, u16 *bd_rd_ptr,
+			  struct prueth_packet_info pkt_info,
+			  const struct prueth_queue_info *rxqueue)
+{
+	struct net_device *ndev = emac->ndev;
+	int read_block, update_block, pkt_block_size;
+	unsigned int buffer_desc_count;
+	bool buffer_wrapped = false;
+	struct sk_buff *skb;
+	void *src_addr;
+	void *dst_addr;
+
+	/* OCMC RAM is not cached and read order is not important */
+	void *ocmc_ram = (__force void *)emac->prueth->mem[PRUETH_MEM_OCMC].va;
+
+	/* the PRU firmware deals mostly in pointers already
+	 * offset into ram, we would like to deal in indexes
+	 * within the queue we are working with for code
+	 * simplicity, calculate this here
+	 */
+	buffer_desc_count = rxqueue->buffer_desc_end -
+			    rxqueue->buffer_desc_offset;
+	buffer_desc_count /= BD_SIZE;
+	buffer_desc_count++;
+	read_block = (*bd_rd_ptr - rxqueue->buffer_desc_offset) / BD_SIZE;
+	pkt_block_size = DIV_ROUND_UP(pkt_info.length, ICSS_BLOCK_SIZE);
+
+	/* calculate end BD address post read */
+	update_block = read_block + pkt_block_size;
+
+	/* Check for wrap around */
+	if (update_block >= buffer_desc_count) {
+		update_block %= buffer_desc_count;
+		buffer_wrapped = true;
+	}
+
+	/* calculate new pointer in ram */
+	*bd_rd_ptr = rxqueue->buffer_desc_offset + (update_block * BD_SIZE);
+
+	/* Allocate a socket buffer for this packet */
+	skb = netdev_alloc_skb_ip_align(ndev, pkt_info.length);
+	if (!skb) {
+		if (netif_msg_rx_err(emac) && net_ratelimit())
+			netdev_err(ndev, "failed rx buffer alloc\n");
+		return -ENOMEM;
+	}
+	dst_addr = skb->data;
+
+	/* Get the start address of the first buffer from
+	 * the read buffer description
+	 */
+	src_addr = ocmc_ram + rxqueue->buffer_offset + (read_block * ICSS_BLOCK_SIZE);
+
+	/* Copy the data from PRU buffers(OCMC) to socket buffer(DRAM) */
+	if (buffer_wrapped) { /* wrapped around buffer */
+		int bytes = (buffer_desc_count - read_block) * ICSS_BLOCK_SIZE;
+		int remaining;
+
+		/* bytes is integral multiple of ICSS_BLOCK_SIZE but
+		 * entire packet may have fit within the last BD
+		 * if pkt_info.length is not integral multiple of
+		 * ICSS_BLOCK_SIZE
+		 */
+		if (pkt_info.length < bytes)
+			bytes = pkt_info.length;
+
+		/* copy non-wrapped part */
+		memcpy(dst_addr, src_addr, bytes);
+
+		/* copy wrapped part */
+		dst_addr += bytes;
+		remaining = pkt_info.length - bytes;
+		src_addr = ocmc_ram + rxqueue->buffer_offset;
+		memcpy(dst_addr, src_addr, remaining);
+	} else {
+		memcpy(dst_addr, src_addr, pkt_info.length);
+	}
+
+	/* send packet up the stack */
+	skb_put(skb, pkt_info.length);
+	skb->protocol = eth_type_trans(skb, ndev);
+	netif_receive_skb(skb);
+
+	/* update stats */
+	ndev->stats.rx_bytes += pkt_info.length;
+	ndev->stats.rx_packets++;
+
+	return 0;
+}
+
+/* get upto quota number of packets */
+static int emac_rx_packets(struct prueth_emac *emac, int quota)
+{
+	int start_queue, end_queue;
+	struct prueth_queue_desc __iomem *queue_desc;
+	const struct prueth_queue_info *rxqueue;
+	u8 overflow_cnt;
+	u16 bd_rd_ptr, bd_wr_ptr, update_rd_ptr;
+	u32 rd_buf_desc;
+	void __iomem *shared_ram = emac->prueth->mem[PRUETH_MEM_SHARED_RAM].va;
+	struct prueth_packet_info pkt_info;
+	struct net_device_stats *ndevstats = &emac->ndev->stats;
+	int i, ret, used = 0;
+
+	start_queue = emac->rx_queue_start;
+	end_queue = emac->rx_queue_end;
+
+	/* search host queues for packets */
+	for (i = start_queue; i <= end_queue; i++) {
+		queue_desc = emac->rx_queue_descs + i;
+		rxqueue = &queue_infos[PRUETH_PORT_HOST][i];
+
+		overflow_cnt = readb(&queue_desc->overflow_cnt);
+		if (overflow_cnt > 0) {
+			emac->ndev->stats.rx_over_errors += overflow_cnt;
+
+			/* reset to zero */
+			writeb(0, &queue_desc->overflow_cnt);
+		}
+
+		bd_rd_ptr = readw(&queue_desc->rd_ptr);
+		bd_wr_ptr = readw(&queue_desc->wr_ptr);
+
+		/* while packets are available in this queue */
+		while (bd_rd_ptr != bd_wr_ptr) {
+			/* get packet info from the read buffer descriptor */
+			rd_buf_desc = readl(shared_ram + bd_rd_ptr);
+			parse_packet_info(rd_buf_desc, &pkt_info);
+
+			if (pkt_info.length <= 0) {
+				/* a packet length of zero will cause us to
+				 * never move the read pointer ahead, locking
+				 * the driver, so we manually have to move it
+				 * to the write pointer, discarding all
+				 * remaining packets in this queue. This should
+				 * never happen.
+				 */
+				update_rd_ptr = bd_wr_ptr;
+				ndevstats->rx_length_errors++;
+			} else if (pkt_info.length > EMAC_MAX_PKTLEN) {
+				/* if the packet is too large we skip it but we
+				 * still need to move the read pointer ahead
+				 * and assume something is wrong with the read
+				 * pointer as the firmware should be filtering
+				 * these packets
+				 */
+				update_rd_ptr = bd_wr_ptr;
+				ndevstats->rx_length_errors++;
+			} else {
+				update_rd_ptr = bd_rd_ptr;
+				ret = emac_rx_packet(emac, &update_rd_ptr,
+						     pkt_info, rxqueue);
+				if (ret)
+					return ret;
+
+				used++;
+			}
+
+			/* after reading the buffer descriptor we clear it
+			 * to prevent improperly moved read pointer errors
+			 * from simply looking like old packets.
+			 */
+			writel(0, shared_ram + bd_rd_ptr);
+
+			/* update read pointer in queue descriptor */
+			writew(update_rd_ptr, &queue_desc->rd_ptr);
+			bd_rd_ptr = update_rd_ptr;
+
+			/* all we have room for? */
+			if (used >= quota)
+				return used;
+		}
+	}
+
+	return used;
+}
+
+/* get statistics maintained by the PRU firmware into @pstats */
+static void emac_get_stats(struct prueth_emac *emac,
+			   struct port_statistics *pstats)
+{
+	void __iomem *dram;
+
+	dram = emac->prueth->mem[emac->dram].va;
+	memcpy_fromio(pstats, dram + STATISTICS_OFFSET, sizeof(*pstats));
+}
+
+/* set PRU firmware statistics */
+static void emac_set_stats(struct prueth_emac *emac,
+			   struct port_statistics *pstats)
+{
+	void __iomem *dram;
+
+	dram = emac->prueth->mem[emac->dram].va;
+	memcpy_fromio(dram + STATISTICS_OFFSET, pstats, sizeof(*pstats));
+}
+
+/**
+ * emac_napi_poll - EMAC NAPI Poll function
+ * @ndev: EMAC network adapter
+ * @budget: Number of receive packets to process (as told by NAPI layer)
+ *
+ * NAPI Poll function implemented to process packets as per budget. We check
+ * the type of interrupt on the device and accordingly call the TX or RX
+ * packet processing functions. We follow the budget for RX processing and
+ * also put a cap on number of TX pkts processed through config param. The
+ * NAPI schedule function is called if more packets pending.
+ *
+ * Returns number of packets received (in most cases; else TX pkts - rarely)
+ */
+static int emac_napi_poll(struct napi_struct *napi, int budget)
+{
+	struct prueth_emac *emac = container_of(napi, struct prueth_emac, napi);
+	int num_rx_packets;
+
+	num_rx_packets = emac_rx_packets(emac, budget);
+	if (num_rx_packets < budget) {
+		napi_complete(napi);
+
+		enable_irq(emac->rx_irq);
+	}
+
+	return num_rx_packets;
+}
+
+/**
+ * emac_ndo_open - EMAC device open
+ * @ndev: network adapter device
+ *
+ * Called when system wants to start the interface.
+ *
+ * Returns 0 for a successful open, or appropriate error code
+ */
+static int emac_ndo_open(struct net_device *ndev)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+	int ret;
+
+	/* set h/w MAC as user might have re-configured */
+	ether_addr_copy(emac->mac_addr, ndev->dev_addr);
+
+	netif_carrier_off(ndev);
+
+	/* reset and start PRU firmware */
+	prueth_emac_config(emac);
+
+	/* restore stats */
+	emac_set_stats(emac, &emac->stats);
+
+	/* boot the PRU */
+	ret = rproc_boot(emac->pru);
+	if (ret) {
+		netdev_err(ndev, "failed to boot PRU: %d\n", ret);
+		return ret;
+	}
+
+	ret = request_irq(emac->rx_irq, emac_rx_hardirq,
+			  IRQF_TRIGGER_HIGH | IRQF_ONESHOT,
+			  ndev->name, ndev);
+	if (ret) {
+		netdev_err(ndev, "unable to request RX IRQ\n");
+		goto rproc_shutdown;
+	}
+	ret = request_irq(emac->tx_irq, emac_tx_hardirq,
+			  IRQF_TRIGGER_HIGH | IRQF_ONESHOT,
+			  ndev->name, ndev);
+	if (ret) {
+		netdev_err(ndev, "unable to request TX IRQ\n");
+		goto free_rx_irq;
+	}
+
+	/* start PHY */
+	phy_start(emac->phydev);
+	napi_enable(&emac->napi);
+
+	/* enable the port */
+	prueth_port_enable(emac, true);
+
+	if (netif_msg_drv(emac))
+		dev_notice(&ndev->dev, "started\n");
+
+	return 0;
+
+free_rx_irq:
+	free_irq(emac->rx_irq, ndev);
+rproc_shutdown:
+	rproc_shutdown(emac->pru);
+
+	return ret;
+}
+
+/**
+ * emac_ndo_stop - EMAC device stop
+ * @ndev: network adapter device
+ *
+ * Called when system wants to stop or down the interface.
+ */
+static int emac_ndo_stop(struct net_device *ndev)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+
+	/* disable the mac port */
+	prueth_port_enable(emac, 0);
+
+	/* stop PHY */
+	phy_stop(emac->phydev);
+
+	/* inform the upper layers. */
+	netif_stop_queue(ndev);
+	napi_disable(&emac->napi);
+	netif_carrier_off(ndev);
+
+	/* stop the PRU */
+	rproc_shutdown(emac->pru);
+
+	/* save stats */
+	emac_get_stats(emac, &emac->stats);
+
+	/* free rx and tx interrupts */
+	free_irq(emac->tx_irq, ndev);
+	free_irq(emac->rx_irq, ndev);
+
+	if (netif_msg_drv(emac))
+		dev_notice(&ndev->dev, "stopped\n");
+
+	return 0;
+}
+
+/**
+ * emac_ndo_start_xmit - EMAC Transmit function
+ * @skb: SKB pointer
+ * @ndev: EMAC network adapter
+ *
+ * Called by the system to transmit a packet  - we queue the packet in
+ * EMAC hardware transmit queue
+ *
+ * Returns success(NETDEV_TX_OK) or error code (typically out of desc's)
+ */
+static int emac_ndo_start_xmit(struct sk_buff *skb, struct net_device *ndev)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+	int ret = 0;
+
+	if (unlikely(!emac->link)) {
+		if (netif_msg_tx_err(emac) && net_ratelimit())
+			netdev_err(ndev, "No link to transmit");
+		goto fail_tx;
+	}
+
+	/* we don't yet support different TX priority queues */
+	ret = prueth_tx_enqueue(emac, skb, PRUETH_QUEUE4);
+	if (ret) {
+		if (ret != -ENOBUFS && netif_msg_tx_err(emac) && net_ratelimit())
+			netdev_err(ndev, "packet queue failed: %d\n", ret);
+		goto fail_tx;
+	}
+
+	ndev->stats.tx_packets++;
+	ndev->stats.tx_bytes += skb->len;
+	dev_kfree_skb_any(skb);
+
+	return NETDEV_TX_OK;
+
+fail_tx:
+	if (ret == -ENOBUFS) {
+		/* no free TX queue */
+		netif_stop_queue(ndev);
+		ret = NETDEV_TX_BUSY;
+	} else {
+		/* error */
+		ndev->stats.tx_dropped++;
+		ret = NET_XMIT_DROP;
+	}
+
+	return ret;
+}
+
+/**
+ * emac_ndo_tx_timeout - EMAC Transmit timeout function
+ * @ndev: The EMAC network adapter
+ *
+ * Called when system detects that a skb timeout period has expired
+ * potentially due to a fault in the adapter in not being able to send
+ * it out on the wire.
+ */
+static void emac_ndo_tx_timeout(struct net_device *ndev)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+
+	if (netif_msg_tx_err(emac))
+		netdev_err(ndev, "xmit timeout");
+
+	ndev->stats.tx_errors++;
+
+	/* TODO: can we recover or need to reboot firmware? */
+
+	netif_wake_queue(ndev);
+}
+
+/**
+ * emac_ndo_getstats - EMAC get statistics function
+ * @ndev: The EMAC network adapter
+ *
+ * Called when system wants to get statistics from the device.
+ *
+ * We return the statistics in net_device_stats structure pulled from emac
+ */
+static struct net_device_stats *emac_ndo_get_stats(struct net_device *ndev)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+	struct port_statistics pstats;
+	struct net_device_stats *stats = &ndev->stats;
+
+	emac_get_stats(emac, &pstats);
+	stats->collisions = pstats.late_coll + pstats.single_coll +
+			    pstats.multi_coll + pstats.excess_coll;
+	stats->multicast = pstats.rx_mcast;
+
+	return stats;
+}
+
+/* enable/disable MC filter */
+static void emac_mc_filter_ctrl(struct prueth_emac *emac, bool enable)
+{
+	struct prueth *prueth = emac->prueth;
+	void __iomem *mc_filter_ctrl;
+	u32 reg;
+
+	mc_filter_ctrl = prueth->mem[emac->dram].va +
+			 ICSS_EMAC_FW_MULTICAST_FILTER_CTRL_OFFSET;
+
+	if (enable)
+		reg = ICSS_EMAC_FW_MULTICAST_FILTER_CTRL_ENABLED;
+	else
+		reg = ICSS_EMAC_FW_MULTICAST_FILTER_CTRL_DISABLED;
+
+	writeb(reg, mc_filter_ctrl);
+}
+
+/* reset MC filter bins */
+static void emac_mc_filter_reset(struct prueth_emac *emac)
+{
+	struct prueth *prueth = emac->prueth;
+	void __iomem *mc_filter_tbl;
+
+	mc_filter_tbl = prueth->mem[emac->dram].va +
+			 ICSS_EMAC_FW_MULTICAST_FILTER_TABLE;
+	memset_io(mc_filter_tbl, 0, ICSS_EMAC_FW_MULTICAST_TABLE_SIZE_BYTES);
+}
+
+/* set MC filter hashmask */
+static void emac_mc_filter_hashmask(struct prueth_emac *emac,
+				    u8 mask[ICSS_EMAC_FW_MULTICAST_FILTER_MASK_SIZE_BYTES])
+{
+	struct prueth *prueth = emac->prueth;
+	void __iomem *mc_filter_mask;
+
+	mc_filter_mask = prueth->mem[emac->dram].va +
+			 ICSS_EMAC_FW_MULTICAST_FILTER_MASK_OFFSET;
+	memcpy_toio(mc_filter_mask, mask,
+		    ICSS_EMAC_FW_MULTICAST_FILTER_MASK_SIZE_BYTES);
+}
+
+static void emac_mc_filter_bin_allow(struct prueth_emac *emac, u8 hash)
+{
+	struct prueth *prueth = emac->prueth;
+	void __iomem *mc_filter_tbl;
+
+	mc_filter_tbl = prueth->mem[emac->dram].va +
+			 ICSS_EMAC_FW_MULTICAST_FILTER_TABLE;
+	writeb(ICSS_EMAC_FW_MULTICAST_FILTER_HOST_RCV_ALLOWED,
+	       mc_filter_tbl + hash);
+}
+
+static u8 emac_get_mc_hash(u8 *mac, u8 *mask)
+{
+	int j;
+	u8 hash;
+
+	for (j = 0, hash = 0; j < ETH_ALEN; j++)
+		hash ^= (mac[j] & mask[j]);
+
+	return hash;
+}
+
+/**
+ * emac_ndo_set_rx_mode - EMAC set receive mode function
+ * @ndev: The EMAC network adapter
+ *
+ * Called when system wants to set the receive mode of the device.
+ *
+ */
+static void emac_ndo_set_rx_mode(struct net_device *ndev)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+	struct prueth *prueth = emac->prueth;
+	void __iomem *sram = prueth->mem[PRUETH_MEM_SHARED_RAM].va;
+	u32 reg = readl(sram + EMAC_PROMISCUOUS_MODE_OFFSET);
+	u32 mask;
+	bool promisc = ndev->flags & IFF_PROMISC;
+	struct netdev_hw_addr *ha;
+	u8 hash;
+
+	switch (emac->port_id) {
+	case PRUETH_PORT_MII0:
+		mask = EMAC_P1_PROMISCUOUS_BIT;
+		break;
+	case PRUETH_PORT_MII1:
+		mask = EMAC_P2_PROMISCUOUS_BIT;
+		break;
+	default:
+		netdev_err(ndev, "%s: invalid port\n", __func__);
+		return;
+	}
+
+	/* Disable and reset multicast filter, allows allmulti */
+	emac_mc_filter_ctrl(emac, false);
+	emac_mc_filter_reset(emac);
+	emac_mc_filter_hashmask(emac, emac->mc_filter_mask);
+
+	if (promisc) {
+		/* Enable promiscuous mode */
+		reg |= mask;
+	} else {
+		/* Disable promiscuous mode */
+		reg &= ~mask;
+	}
+
+	writel(reg, sram + EMAC_PROMISCUOUS_MODE_OFFSET);
+
+	if (promisc)
+		return;
+
+	if (ndev->flags & IFF_ALLMULTI)
+		return;
+
+	emac_mc_filter_ctrl(emac, true);	/* all multicast blocked */
+
+	if (netdev_mc_empty(ndev))
+		return;
+
+	netdev_for_each_mc_addr(ha, ndev) {
+		hash = emac_get_mc_hash(ha->addr, emac->mc_filter_mask);
+		emac_mc_filter_bin_allow(emac, hash);
+	}
+}
+
+static int emac_ndo_ioctl(struct net_device *ndev, struct ifreq *ifr, int cmd)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+
+	return phy_mii_ioctl(emac->phydev, ifr, cmd);
+}
+
+static const struct net_device_ops emac_netdev_ops = {
+	.ndo_open = emac_ndo_open,
+	.ndo_stop = emac_ndo_stop,
+	.ndo_start_xmit = emac_ndo_start_xmit,
+	.ndo_set_mac_address = eth_mac_addr,
+	.ndo_validate_addr = eth_validate_addr,
+	.ndo_change_mtu	= eth_change_mtu,
+	.ndo_tx_timeout = emac_ndo_tx_timeout,
+	.ndo_get_stats = emac_ndo_get_stats,
+	.ndo_set_rx_mode = emac_ndo_set_rx_mode,
+	.ndo_do_ioctl = emac_ndo_ioctl,
+};
+
+/**
+ * emac_get_drvinfo - Get EMAC driver information
+ * @ndev: The network adapter
+ * @info: ethtool info structure containing name and version
+ *
+ * Returns EMAC driver information (name and version)
+ */
+static void emac_get_drvinfo(struct net_device *ndev,
+			     struct ethtool_drvinfo *info)
+{
+	strlcpy(info->driver, PRUETH_MODULE_DESCRIPTION, sizeof(info->driver));
+	strlcpy(info->version, PRUETH_MODULE_VERSION, sizeof(info->version));
+}
+
+/**
+ * emac_get_link_ksettings - Get EMAC settings
+ * @ndev: The network adapter
+ * @ecmd: ethtool command
+ *
+ * Executes ethool get command
+ */
+static int emac_get_link_ksettings(struct net_device *ndev,
+				   struct ethtool_link_ksettings *ecmd)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+
+	if (!emac->phydev)
+		return -EOPNOTSUPP;
+
+	phy_ethtool_ksettings_get(emac->phydev, ecmd);
+	return 0;
+}
+
+/**
+ * emac_set_link_ksettings - Set EMAC settings
+ * @ndev: The EMAC network adapter
+ * @ecmd: ethtool command
+ *
+ * Executes ethool set command
+ */
+static int emac_set_link_ksettings(struct net_device *ndev,
+				   const struct ethtool_link_ksettings *ecmd)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+
+	if (!emac->phydev)
+		return -EOPNOTSUPP;
+
+	return phy_ethtool_ksettings_set(emac->phydev, ecmd);
+}
+
+#define PRUETH_STAT_OFFSET(m) offsetof(struct port_statistics, m)
+
+static const struct {
+	char string[ETH_GSTRING_LEN];
+	u32 offset;
+} prueth_ethtool_stats[] = {
+	{"txBcast", PRUETH_STAT_OFFSET(tx_bcast)},
+	{"txMcast", PRUETH_STAT_OFFSET(tx_mcast)},
+	{"txUcast", PRUETH_STAT_OFFSET(tx_ucast)},
+	{"txOctets", PRUETH_STAT_OFFSET(tx_octets)},
+	{"rxBcast", PRUETH_STAT_OFFSET(rx_bcast)},
+	{"rxMcast", PRUETH_STAT_OFFSET(rx_mcast)},
+	{"rxUcast", PRUETH_STAT_OFFSET(rx_ucast)},
+	{"rxOctets", PRUETH_STAT_OFFSET(rx_octets)},
+
+	{"tx64byte", PRUETH_STAT_OFFSET(tx64byte)},
+	{"tx65_127byte", PRUETH_STAT_OFFSET(tx65_127byte)},
+	{"tx128_255byte", PRUETH_STAT_OFFSET(tx128_255byte)},
+	{"tx256_511byte", PRUETH_STAT_OFFSET(tx256_511byte)},
+	{"tx512_1023byte", PRUETH_STAT_OFFSET(tx512_1023byte)},
+	{"tx1024byte", PRUETH_STAT_OFFSET(tx1024byte)},
+	{"rx64byte", PRUETH_STAT_OFFSET(rx64byte)},
+	{"rx65_127byte", PRUETH_STAT_OFFSET(rx65_127byte)},
+	{"rx128_255byte", PRUETH_STAT_OFFSET(rx128_255byte)},
+	{"rx256_511byte", PRUETH_STAT_OFFSET(rx256_511byte)},
+	{"rx512_1023byte", PRUETH_STAT_OFFSET(rx512_1023byte)},
+	{"rx1024byte", PRUETH_STAT_OFFSET(rx1024byte)},
+
+	{"lateColl", PRUETH_STAT_OFFSET(late_coll)},
+	{"singleColl", PRUETH_STAT_OFFSET(single_coll)},
+	{"multiColl", PRUETH_STAT_OFFSET(multi_coll)},
+	{"excessColl", PRUETH_STAT_OFFSET(excess_coll)},
+
+	{"rxMisAlignmentFrames", PRUETH_STAT_OFFSET(rx_misalignment_frames)},
+	{"stormPrevCounter", PRUETH_STAT_OFFSET(stormprev_counter)},
+	{"macRxError", PRUETH_STAT_OFFSET(mac_rxerror)},
+	{"SFDError", PRUETH_STAT_OFFSET(sfd_error)},
+	{"defTx", PRUETH_STAT_OFFSET(def_tx)},
+	{"macTxError", PRUETH_STAT_OFFSET(mac_txerror)},
+	{"rxOverSizedFrames", PRUETH_STAT_OFFSET(rx_oversized_frames)},
+	{"rxUnderSizedFrames", PRUETH_STAT_OFFSET(rx_undersized_frames)},
+	{"rxCRCFrames", PRUETH_STAT_OFFSET(rx_crc_frames)},
+	{"droppedPackets", PRUETH_STAT_OFFSET(dropped_packets)},
+
+	{"txHWQOverFlow", PRUETH_STAT_OFFSET(tx_hwq_overflow)},
+	{"txHWQUnderFlow", PRUETH_STAT_OFFSET(tx_hwq_underflow)},
+};
+
+static int emac_get_sset_count(struct net_device *ndev, int stringset)
+{
+	switch (stringset) {
+	case ETH_SS_STATS:
+		return ARRAY_SIZE(prueth_ethtool_stats);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static void emac_get_strings(struct net_device *ndev, u32 stringset, u8 *data)
+{
+	u8 *p = data;
+	int i;
+
+	switch (stringset) {
+	case ETH_SS_STATS:
+		for (i = 0; i < ARRAY_SIZE(prueth_ethtool_stats); i++) {
+			memcpy(p, prueth_ethtool_stats[i].string,
+			       ETH_GSTRING_LEN);
+			p += ETH_GSTRING_LEN;
+		}
+		break;
+	default:
+		break;
+	}
+}
+
+static void emac_get_ethtool_stats(struct net_device *ndev,
+				   struct ethtool_stats *stats, u64 *data)
+{
+	struct prueth_emac *emac = netdev_priv(ndev);
+	struct port_statistics pstats;
+	u32 val;
+	int i;
+	void *ptr;
+
+	emac_get_stats(emac, &pstats);
+
+	for (i = 0; i < ARRAY_SIZE(prueth_ethtool_stats); i++) {
+		ptr = &pstats;
+		ptr += prueth_ethtool_stats[i].offset;
+		val = *(u32 *)ptr;
+		data[i] = val;
+	}
+}
+
+/* Ethtool support for EMAC adapter */
+static const struct ethtool_ops emac_ethtool_ops = {
+	.get_drvinfo = emac_get_drvinfo,
+	.get_link_ksettings = emac_get_link_ksettings,
+	.set_link_ksettings = emac_set_link_ksettings,
+	.get_link = ethtool_op_get_link,
+	.get_ts_info = ethtool_op_get_ts_info,
+	.get_sset_count = emac_get_sset_count,
+	.get_strings = emac_get_strings,
+	.get_ethtool_stats = emac_get_ethtool_stats,
+};
+
+/* get emac_port corresponding to eth_node name */
+static int prueth_node_port(struct device_node *eth_node)
+{
+	if (!strcmp(eth_node->name, "ethernet-mii0"))
+		return PRUETH_PORT_MII0;
+	else if (!strcmp(eth_node->name, "ethernet-mii1"))
+		return PRUETH_PORT_MII1;
+	else
+		return -EINVAL;
+}
+
+/* get MAC instance corresponding to eth_node name */
+static int prueth_node_mac(struct device_node *eth_node)
+{
+	if (!strcmp(eth_node->name, "ethernet-mii0"))
+		return PRUETH_MAC0;
+	else if (!strcmp(eth_node->name, "ethernet-mii1"))
+		return PRUETH_MAC1;
+	else
+		return -EINVAL;
+}
+
+static int prueth_netdev_init(struct prueth *prueth,
+			      struct device_node *eth_node)
+{
+	enum prueth_port port;
+	enum prueth_mac mac;
+	struct net_device *ndev;
+	struct prueth_emac *emac;
+	const u8 *mac_addr;
+	int ret;
+
+	port = prueth_node_port(eth_node);
+	if (port < 0)
+		return -EINVAL;
+
+	mac = prueth_node_mac(eth_node);
+	if (mac < 0)
+		return -EINVAL;
+
+	ndev = devm_alloc_etherdev(prueth->dev, sizeof(*emac));
+	if (!ndev)
+		return -ENOMEM;
+
+	SET_NETDEV_DEV(ndev, prueth->dev);
+	emac = netdev_priv(ndev);
+	prueth->emac[mac] = emac;
+	emac->prueth = prueth;
+	emac->ndev = ndev;
+	emac->port_id = port;
+	memset(&emac->mc_filter_mask[0], 0xff, ETH_ALEN); /* default mask */
+
+	switch (port) {
+	case PRUETH_PORT_MII0:
+		emac->tx_port_queue = PRUETH_PORT_QUEUE_MII0;
+
+		/* packets from MII0 are on queues 1 through 2 */
+		emac->rx_queue_start = PRUETH_QUEUE1;
+		emac->rx_queue_end = PRUETH_QUEUE2;
+
+		emac->dram = PRUETH_MEM_DRAM0;
+		emac->pru = prueth->pru0;
+		break;
+	case PRUETH_PORT_MII1:
+		emac->tx_port_queue = PRUETH_PORT_QUEUE_MII1;
+
+		/* packets from MII1 are on queues 3 through 4 */
+		emac->rx_queue_start = PRUETH_QUEUE3;
+		emac->rx_queue_end = PRUETH_QUEUE4;
+
+		emac->dram = PRUETH_MEM_DRAM1;
+		emac->pru = prueth->pru1;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	emac->rx_irq = of_irq_get_byname(eth_node, "rx");
+	if (emac->rx_irq < 0) {
+		ret = emac->rx_irq;
+		if (ret != -EPROBE_DEFER)
+			dev_err(prueth->dev, "could not get rx irq\n");
+		goto free;
+	}
+	emac->tx_irq = of_irq_get_byname(eth_node, "tx");
+	if (emac->tx_irq < 0) {
+		ret = emac->tx_irq;
+		if (ret != -EPROBE_DEFER)
+			dev_err(prueth->dev, "could not get tx irq\n");
+		goto free;
+	}
+
+	emac->msg_enable = netif_msg_init(debug_level, PRUETH_EMAC_DEBUG);
+	spin_lock_init(&emac->lock);
+
+	/* get mac address from DT and set private and netdev addr */
+	mac_addr = of_get_mac_address(eth_node);
+	if (mac_addr)
+		ether_addr_copy(ndev->dev_addr, mac_addr);
+	if (!is_valid_ether_addr(ndev->dev_addr)) {
+		eth_hw_addr_random(ndev);
+		dev_warn(prueth->dev, "port %d: using random MAC addr: %pM\n",
+			 port, ndev->dev_addr);
+	}
+	ether_addr_copy(emac->mac_addr, ndev->dev_addr);
+
+	emac->phy_node = of_parse_phandle(eth_node, "phy-handle", 0);
+	if (!emac->phy_node) {
+		dev_err(prueth->dev, "couldn't find phy-handle\n");
+		ret = -ENODEV;
+		goto free;
+	}
+
+	emac->phy_if = of_get_phy_mode(eth_node);
+	if (emac->phy_if < 0) {
+		dev_err(prueth->dev, "could not get phy-mode property\n");
+		ret = emac->phy_if;
+		goto free;
+	}
+
+	/* connect PHY */
+	emac->phydev = of_phy_connect(ndev, emac->phy_node,
+				      &emac_adjust_link, 0, emac->phy_if);
+	if (!emac->phydev) {
+		dev_dbg(prueth->dev, "couldn't connect to phy %s\n",
+			emac->phy_node->full_name);
+		ret = -EPROBE_DEFER;
+		goto free;
+	}
+
+	/* remove unsupported modes */
+	emac->phydev->supported &= ~(PHY_10BT_FEATURES |
+				     SUPPORTED_100baseT_Half |
+				     PHY_1000BT_FEATURES |
+				     SUPPORTED_Pause |
+				     SUPPORTED_Asym_Pause);
+	emac->phydev->advertising = emac->phydev->supported;
+
+	ndev->netdev_ops = &emac_netdev_ops;
+	ndev->ethtool_ops = &emac_ethtool_ops;
+
+	netif_napi_add(ndev, &emac->napi, emac_napi_poll, EMAC_POLL_WEIGHT);
+
+	return 0;
+
+free:
+	prueth->emac[mac] = NULL;
+
+	return ret;
+}
+
+static void prueth_netdev_exit(struct prueth *prueth,
+			       struct device_node *eth_node)
+{
+	struct prueth_emac *emac;
+	enum prueth_mac mac;
+
+	mac = prueth_node_mac(eth_node);
+	if (mac < 0)
+		return;
+
+	emac = prueth->emac[mac];
+	if (!emac)
+		return;
+
+	dev_info(prueth->dev, "freeing port %d\n", mac);
+
+	phy_disconnect(emac->phydev);
+
+	netif_napi_del(&emac->napi);
+	prueth->emac[mac] = NULL;
+}
+
+static const struct of_device_id prueth_dt_match[];
+
+static int prueth_probe(struct platform_device *pdev)
+{
+	struct prueth *prueth;
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct device_node *eth0_node, *eth1_node;
+	const struct of_device_id *match;
+	struct pruss *pruss;
+	int i, ret;
+
+	if (!np)
+		return -ENODEV;	/* we don't support non DT */
+
+	match = of_match_device(prueth_dt_match, dev);
+	if (!match)
+		return -ENODEV;
+
+	prueth = devm_kzalloc(dev, sizeof(*prueth), GFP_KERNEL);
+	if (!prueth)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, prueth);
+
+	prueth->dev = dev;
+
+	eth0_node = of_get_child_by_name(np, "ethernet-mii0");
+	if (!of_device_is_available(eth0_node)) {
+		of_node_put(eth0_node);
+		eth0_node = NULL;
+	}
+
+	eth1_node = of_get_child_by_name(np, "ethernet-mii1");
+	if (!of_device_is_available(eth1_node)) {
+		of_node_put(eth1_node);
+		eth1_node = NULL;
+	}
+
+	/* At least one node must be present and available else we fail */
+	if (!eth0_node && !eth1_node) {
+		dev_err(dev, "neither ethernet-mii0 nor ethernet-mii1 node available\n");
+		ret = -ENODEV;
+		goto put_node;
+	}
+
+	prueth->eth_node[PRUETH_MAC0] = eth0_node;
+	prueth->eth_node[PRUETH_MAC1] = eth1_node;
+
+	prueth->mii_rt = syscon_regmap_lookup_by_phandle(np, "mii-rt");
+	if (IS_ERR(prueth->mii_rt)) {
+		dev_err(dev, "couldn't get mii-rt syscon regmap\n");
+		return -ENODEV;
+	}
+
+	prueth->iep = syscon_regmap_lookup_by_phandle(np, "iep");
+	if (IS_ERR(prueth->iep)) {
+		dev_err(dev, "couldn't get iep syscon regmap\n");
+		return -ENODEV;
+	}
+
+	if (eth0_node) {
+		prueth->pru0 = pru_rproc_get(np, 0);
+		if (IS_ERR(prueth->pru0)) {
+			ret = PTR_ERR(prueth->pru0);
+			if (ret != -EPROBE_DEFER)
+				dev_err(dev, "unable to get PRU0: %d\n", ret);
+			goto put_node;
+		}
+	}
+
+	if (eth1_node) {
+		prueth->pru1 = pru_rproc_get(np, 1);
+		if (IS_ERR(prueth->pru1)) {
+			ret = PTR_ERR(prueth->pru1);
+			if (ret != -EPROBE_DEFER)
+				dev_err(dev, "unable to get PRU1: %d\n", ret);
+			goto put_pru0;
+		}
+	}
+
+	pruss = pruss_get(prueth->pru0 ? prueth->pru0 : prueth->pru1);
+	if (IS_ERR(pruss)) {
+		ret = PTR_ERR(pruss);
+		dev_err(dev, "unable to get pruss handle\n");
+		goto put_pru1;
+	}
+	prueth->pruss = pruss;
+
+	/* Configure PRUSS */
+	if (eth0_node)
+		pruss_cfg_gpimode(pruss, prueth->pru0, PRUSS_GPI_MODE_MII);
+	if (eth1_node)
+		pruss_cfg_gpimode(pruss, prueth->pru1, PRUSS_GPI_MODE_MII);
+	pruss_cfg_miirt_enable(pruss, true);
+	pruss_cfg_xfr_enable(pruss, true);
+
+	/* Get PRUSS mem resources */
+	/* OCMC is system resource which we get separately */
+	for (i = 0; i < ARRAY_SIZE(pruss_mem_ids); i++) {
+		/* skip appropriate DRAM if not required */
+		if (!eth0_node && i == PRUETH_MEM_DRAM0)
+			continue;
+
+		if (!eth1_node && i == PRUETH_MEM_DRAM1)
+			continue;
+
+		ret = pruss_request_mem_region(pruss, pruss_mem_ids[i],
+					       &prueth->mem[i]);
+		if (ret) {
+			dev_err(dev, "unable to get PRUSS resource %d: %d\n",
+				i, ret);
+			goto put_mem;
+		}
+	}
+
+	prueth->sram_pool = of_gen_pool_get(np, "sram", 0);
+	if (!prueth->sram_pool) {
+		dev_err(dev, "unable to get SRAM pool\n");
+		ret = -ENODEV;
+
+		goto put_mem;
+	}
+	prueth->mem[PRUETH_MEM_OCMC].va =
+			(void __iomem *)gen_pool_alloc(prueth->sram_pool,
+						       OCMC_RAM_SIZE);
+	if (!prueth->mem[PRUETH_MEM_OCMC].va) {
+		dev_err(dev, "unable to allocate OCMC resource\n");
+		ret = -ENOMEM;
+		goto put_mem;
+	}
+	prueth->mem[PRUETH_MEM_OCMC].pa =
+			gen_pool_virt_to_phys(prueth->sram_pool,
+					      (unsigned long)prueth->mem[PRUETH_MEM_OCMC].va);
+	prueth->mem[PRUETH_MEM_OCMC].size = OCMC_RAM_SIZE;
+	dev_dbg(dev, "ocmc: pa %pa va %p size %#zx\n",
+		&prueth->mem[PRUETH_MEM_OCMC].pa,
+		prueth->mem[PRUETH_MEM_OCMC].va,
+		prueth->mem[PRUETH_MEM_OCMC].size);
+
+	/* setup netdev interfaces */
+	if (eth0_node) {
+		ret = prueth_netdev_init(prueth, eth0_node);
+		if (ret) {
+			if (ret != -EPROBE_DEFER) {
+				dev_err(dev, "netdev init %s failed: %d\n",
+					eth0_node->name, ret);
+			}
+			goto free_pool;
+		}
+	}
+
+	if (eth1_node) {
+		ret = prueth_netdev_init(prueth, eth1_node);
+		if (ret) {
+			if (ret != -EPROBE_DEFER) {
+				dev_err(dev, "netdev init %s failed: %d\n",
+					eth1_node->name, ret);
+			}
+			goto netdev_exit;
+		}
+	}
+
+	ret = prueth_hostinit(prueth);
+	if (ret) {
+		dev_info(dev, "hostinit failed: %d\n", ret);
+		goto netdev_exit;
+	}
+
+	/* register the network devices */
+	if (eth0_node) {
+		ret = register_netdev(prueth->emac[PRUETH_MAC0]->ndev);
+		if (ret) {
+			dev_err(dev, "can't register netdev for port MII0");
+			goto netdev_exit;
+		}
+
+		prueth->registered_netdevs[PRUETH_MAC0] = prueth->emac[PRUETH_MAC0]->ndev;
+	}
+
+	if (eth1_node) {
+		ret = register_netdev(prueth->emac[PRUETH_MAC1]->ndev);
+		if (ret) {
+			dev_err(dev, "can't register netdev for port MII1");
+			goto netdev_unregister;
+		}
+
+		prueth->registered_netdevs[PRUETH_MAC1] = prueth->emac[PRUETH_MAC1]->ndev;
+	}
+
+	dev_info(dev, "TI PRU ethernet driver initialized: %s EMAC mode\n",
+		 (!eth0_node || !eth1_node) ? "single" : "dual");
+
+	return 0;
+
+netdev_unregister:
+	for (i = 0; i < PRUETH_NUM_MACS; i++) {
+		if (!prueth->registered_netdevs[i])
+			continue;
+		unregister_netdev(prueth->registered_netdevs[i]);
+	}
+
+netdev_exit:
+	for (i = 0; i < PRUETH_NUM_MACS; i++) {
+		struct device_node *eth_node;
+
+		eth_node = prueth->eth_node[i];
+		if (!eth_node)
+			continue;
+
+		prueth_netdev_exit(prueth, eth_node);
+	}
+
+free_pool:
+	gen_pool_free(prueth->sram_pool,
+		      (unsigned long)prueth->mem[PRUETH_MEM_OCMC].va, OCMC_RAM_SIZE);
+
+put_mem:
+	for (i = PRUETH_MEM_DRAM0; i < PRUETH_MEM_OCMC; i++) {
+		if (prueth->mem[i].va)
+			pruss_release_mem_region(pruss, &prueth->mem[i]);
+	}
+
+	pruss_put(prueth->pruss);
+
+put_pru1:
+	if (eth1_node)
+		pru_rproc_put(prueth->pru1);
+put_pru0:
+	if (eth0_node)
+		pru_rproc_put(prueth->pru0);
+
+put_node:
+	of_node_put(eth1_node);
+	of_node_put(eth0_node);
+
+	return ret;
+}
+
+static int prueth_remove(struct platform_device *pdev)
+{
+	struct device_node *eth_node;
+	struct prueth *prueth = platform_get_drvdata(pdev);
+	int i;
+
+	for (i = 0; i < PRUETH_NUM_MACS; i++) {
+		if (!prueth->registered_netdevs[i])
+			continue;
+		unregister_netdev(prueth->registered_netdevs[i]);
+	}
+
+	for (i = 0; i < PRUETH_NUM_MACS; i++) {
+		eth_node = prueth->eth_node[i];
+		if (!eth_node)
+			continue;
+
+		prueth_netdev_exit(prueth, eth_node);
+		of_node_put(eth_node);
+	}
+
+	gen_pool_free(prueth->sram_pool,
+		      (unsigned long)prueth->mem[PRUETH_MEM_OCMC].va,
+		      OCMC_RAM_SIZE);
+
+	for (i = PRUETH_MEM_DRAM0; i < PRUETH_MEM_OCMC; i++) {
+		if (prueth->mem[i].va)
+			pruss_release_mem_region(prueth->pruss, &prueth->mem[i]);
+	}
+
+	pruss_put(prueth->pruss);
+
+	if (prueth->eth_node[PRUETH_MAC0])
+		pru_rproc_put(prueth->pru1);
+	if (prueth->eth_node[PRUETH_MAC1])
+		pru_rproc_put(prueth->pru0);
+
+	return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int prueth_suspend(struct device *dev)
+{
+	struct prueth *prueth = dev_get_drvdata(dev);
+	struct net_device *ndev;
+	int i, ret;
+
+	for (i = 0; i < PRUETH_NUM_MACS; i++) {
+		ndev = prueth->registered_netdevs[i];
+
+		if (!ndev)
+			continue;
+
+		if (netif_running(ndev)) {
+			netif_device_detach(ndev);
+			ret = emac_ndo_stop(ndev);
+			if (ret < 0) {
+				netdev_err(ndev, "failed to stop: %d", ret);
+				return ret;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static int prueth_resume(struct device *dev)
+{
+	struct prueth *prueth = dev_get_drvdata(dev);
+	struct net_device *ndev;
+	int i, ret;
+
+	for (i = 0; i < PRUETH_NUM_MACS; i++) {
+		ndev = prueth->registered_netdevs[i];
+
+		if (!ndev)
+			continue;
+
+		if (netif_running(ndev)) {
+			ret = emac_ndo_open(ndev);
+			if (ret < 0) {
+				netdev_err(ndev, "failed to start: %d", ret);
+				return ret;
+			}
+			netif_device_attach(ndev);
+		}
+	}
+
+	return 0;
+}
+
+#endif /* CONFIG_PM_SLEEP */
+
+static const struct dev_pm_ops prueth_dev_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(prueth_suspend, prueth_resume)
+};
+
+static const struct of_device_id prueth_dt_match[] = {
+	{ .compatible = "ti,am57-prueth", },
+	{ .compatible = "ti,am4376-prueth", },
+	{ .compatible = "ti,am3359-prueth", },
+	{ .compatible = "ti,k2g-prueth", },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, prueth_dt_match);
+
+static struct platform_driver prueth_driver = {
+	.probe = prueth_probe,
+	.remove = prueth_remove,
+	.driver = {
+		.name = "prueth",
+		.of_match_table = prueth_dt_match,
+		.pm = &prueth_dev_pm_ops,
+	},
+};
+module_platform_driver(prueth_driver);
+
+MODULE_AUTHOR("Roger Quadros <rogerq@ti.com>");
+MODULE_AUTHOR("Andrew F. Davis <afd@ti.com>");
+MODULE_DESCRIPTION("PRU Ethernet Driver");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/net/ethernet/ti/prueth.h b/drivers/net/ethernet/ti/prueth.h
--- a/drivers/net/ethernet/ti/prueth.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/net/ethernet/ti/prueth.h	2020-06-19 21:00:31.416227715 -0400
@@ -0,0 +1,175 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+/* PRU ICSS Ethernet driver
+ *
+ * Copyright (C) 2015-2018 Texas Instruments Incorporated - http://www.ti.com
+ */
+
+#ifndef __NET_TI_PRUETH_H
+#define __NET_TI_PRUETH_H
+
+#define PRUETH_NUMQUEUES	5
+
+/**
+ * struct prueth_queue_desc - Queue descriptor
+ * @rd_ptr:	Read pointer, points to a buffer descriptor in Shared PRU RAM.
+ * @wr_ptr:	Write pointer, points to a buffer descriptor in Shared PRU RAM.
+ * @busy_s:	Slave queue busy flag, set by slave(us) to request access from
+ *		master(PRU).
+ * @status:	Bit field status register, Bits:
+ *			0: Master queue busy flag.
+ *			1: Packet has been placed in collision queue.
+ *			2: Packet has been discarded due to overflow.
+ * @max_fill_level:	Maximum queue usage seen.
+ * @overflow_cnt:	Count of queue overflows.
+ *
+ * Each port has up to 4 queues with variable length. The queue is processed
+ * as ring buffer with read and write pointers. Both pointers are address
+ * pointers and increment by 4 for each buffer descriptor position. Queue has
+ * a length defined in constants and a status.
+ */
+struct prueth_queue_desc {
+	u16 rd_ptr;
+	u16 wr_ptr;
+	u8 busy_s;
+	u8 status;
+	u8 max_fill_level;
+	u8 overflow_cnt;
+} __packed;
+
+/**
+ * struct prueth_queue - Information about a queue in memory
+ * @buffer_offset: buffer offset in OCMC RAM
+ * @queue_desc_offset: queue descriptor offset in Shared RAM
+ * @buffer_desc_offset: buffer descriptors offset in Shared RAM
+ * @buffer_desc_end: end address of buffer descriptors in Shared RAM
+ */
+struct prueth_queue_info {
+	u16 buffer_offset;
+	u16 queue_desc_offset;
+	u16 buffer_desc_offset;
+	u16 buffer_desc_end;
+} __packed;
+
+/**
+ * struct prueth_packet_info - Info about a packet in buffer
+ * @shadow: this packet is stored in the collision queue
+ * @port: port packet is on
+ * @length: length of packet
+ * @broadcast: this packet is a broadcast packet
+ * @error: this packet has an error
+ */
+struct prueth_packet_info {
+	bool shadow;
+	unsigned int port;
+	unsigned int length;
+	bool broadcast;
+	bool error;
+};
+
+/**
+ * struct port_statistics - Statistics structure for capturing statistics
+ *			    on PRUs
+ * @tx_bcast: Number of broadcast packets sent
+ * @tx_mcast:Number of multicast packets sent
+ * @tx_ucast:Number of unicast packets sent
+ *
+ * @tx_octets:Number of undersized frames rcvd
+ *
+ * @rx_bcast:Number of broadcast packets rcvd
+ * @rx_mcast:Number of multicast packets rcvd
+ * @rx_ucast:Number of unicast packets rcvd
+ *
+ * @rx_octets:Number of Rx packets
+ *
+ * @tx64byte:Number of 64 byte packets sent
+ * @tx65_127byte:Number of 65-127 byte packets sent
+ * @tx128_255byte:Number of 128-255 byte packets sent
+ * @tx256_511byte:Number of 256-511 byte packets sent
+ * @tx512_1023byte:Number of 512-1023 byte packets sent
+ * @tx1024byte:Number of 1024 and larger size packets sent
+ *
+ * @rx64byte:Number of 64 byte packets rcvd
+ * @rx65_127byte:Number of 65-127 byte packets rcvd
+ * @rx128_255byte:Number of 128-255 byte packets rcvd
+ * @rx256_511byte:Number of 256-511 byte packets rcvd
+ * @rx512_1023byte:Number of 512-1023 byte packets rcvd
+ * @rx1024byte:Number of 1024 and larger size packets rcvd
+ *
+ * @late_coll:Number of late collisions(Half Duplex)
+ * @single_coll:Number of single collisions (Half Duplex)
+ * @multi_coll:Number of multiple collisions (Half Duplex)
+ * @excess_coll:Number of excess collisions(Half Duplex)
+ *
+ * @rx_misalignment_frames:Number of non multiple of 8 byte frames rcvd
+ * @stormprev_counter:Number of packets dropped because of Storm Prevention
+ * @mac_rxerror:Number of MAC receive errors
+ * @sfd_error:Number of invalid SFD
+ * @def_tx:Number of transmissions deferred
+ * @mac_txerror:Number of MAC transmit errors
+ * @rx_oversized_frames:Number of oversized frames rcvd
+ * @rx_undersized_frames:Number of undersized frames rcvd
+ * @rx_crc_frames:Number of CRC error frames rcvd
+ * @dropped_packets:Number of packets dropped due to link down on opposite port
+ *
+ * @tx_hwq_overflow:Hardware Tx Queue (on PRU) over flow count
+ * @tx_hwq_underflow:Hardware Tx Queue (on PRU) under flow count
+ *
+ * @u32 cs_error: Number of carrier sense errors
+ * @sqe_test_error: Number of MAC receive errors
+ *
+ * The fields here are aligned here so that it's consistent
+ * with the memory layout in PRU DRAM, this is to facilitate easy
+ * memcpy. Don't change the order of the fields.
+ */
+struct port_statistics {
+	u32 tx_bcast;
+	u32 tx_mcast;
+	u32 tx_ucast;
+
+	u32 tx_octets;
+
+	u32 rx_bcast;
+	u32 rx_mcast;
+	u32 rx_ucast;
+
+	u32 rx_octets;
+
+	u32 tx64byte;
+	u32 tx65_127byte;
+	u32 tx128_255byte;
+	u32 tx256_511byte;
+	u32 tx512_1023byte;
+	u32 tx1024byte;
+
+	u32 rx64byte;
+	u32 rx65_127byte;
+	u32 rx128_255byte;
+	u32 rx256_511byte;
+	u32 rx512_1023byte;
+	u32 rx1024byte;
+
+	u32 late_coll;
+	u32 single_coll;
+	u32 multi_coll;
+	u32 excess_coll;
+
+	u32 rx_misalignment_frames;
+	u32 stormprev_counter;
+	u32 mac_rxerror;
+	u32 sfd_error;
+	u32 def_tx;
+	u32 mac_txerror;
+	u32 rx_oversized_frames;
+	u32 rx_undersized_frames;
+	u32 rx_crc_frames;
+	u32 dropped_packets;
+
+	u32 tx_hwq_overflow;
+	u32 tx_hwq_underflow;
+
+	u32 cs_error;
+	u32 sqe_test_error;
+} __packed;
+
+#endif /* __NET_TI_PRUETH_H */
diff -Naur --no-dereference a/drivers/net/ethernet/wiznet/w5100-spi.c b/drivers/net/ethernet/wiznet/w5100-spi.c
--- a/drivers/net/ethernet/wiznet/w5100-spi.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/ethernet/wiznet/w5100-spi.c	2020-06-19 21:00:31.416227715 -0400
@@ -442,6 +442,21 @@
 	return w5100_remove(&spi->dev);
 }
 
+#ifdef CONFIG_OF
+static const struct of_device_id w5100_spi_of_match[] = {
+	{ .compatible = "wiznet,w5500" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, w5100_spi_of_match);
+#endif
+
+#ifdef CONFIG_OF
+static const struct spi_device_id w5100_spi_ids[] = {
+	{ "w5500", W5500 },
+	{}
+};
+MODULE_DEVICE_TABLE(spi, w5100_spi_ids);
+#else
 static const struct spi_device_id w5100_spi_ids[] = {
 	{ "w5100", W5100 },
 	{ "w5200", W5200 },
@@ -449,11 +464,13 @@
 	{}
 };
 MODULE_DEVICE_TABLE(spi, w5100_spi_ids);
+#endif
 
 static struct spi_driver w5100_spi_driver = {
 	.driver		= {
 		.name	= "w5100",
 		.pm	= &w5100_pm_ops,
+		.of_match_table = of_match_ptr(w5100_spi_of_match),
 	},
 	.probe		= w5100_spi_probe,
 	.remove		= w5100_spi_remove,
diff -Naur --no-dereference a/drivers/net/phy/dp83867.c b/drivers/net/phy/dp83867.c
--- a/drivers/net/phy/dp83867.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/phy/dp83867.c	2020-06-19 21:00:31.416227715 -0400
@@ -19,6 +19,7 @@
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/phy.h>
+#include <linux/delay.h>
 
 #include <dt-bindings/net/ti-dp83867.h>
 
@@ -32,6 +33,7 @@
 #define DP83867_CFG3		0x1e
 
 /* Extended Registers */
+#define DP83867_FLD_THR_CFG	0x002e
 #define DP83867_CFG4            0x0031
 #define DP83867_CFG4_SGMII_ANEG_MASK (BIT(5) | BIT(6))
 #define DP83867_CFG4_SGMII_ANEG_TIMER_11MS   (3 << 5)
@@ -41,7 +43,9 @@
 
 #define DP83867_RGMIICTL	0x0032
 #define DP83867_STRAP_STS1	0x006E
+#define DP83867_STRAP_STS2	0x006f
 #define DP83867_RGMIIDCTL	0x0086
+#define DP83867_DSP_FFE_CFG	0x012C
 #define DP83867_IO_MUX_CFG	0x0170
 #define DP83867_10M_SGMII_CFG   0x016F
 #define DP83867_10M_SGMII_RATE_ADAPT_MASK BIT(7)
@@ -70,13 +74,26 @@
 /* STRAP_STS1 bits */
 #define DP83867_STRAP_STS1_RESERVED		BIT(11)
 
+/* STRAP_STS2 bits */
+#define DP83867_STRAP_STS2_CLK_SKEW_TX_MASK	GENMASK(6, 4)
+#define DP83867_STRAP_STS2_CLK_SKEW_TX_SHIFT	4
+#define DP83867_STRAP_STS2_CLK_SKEW_RX_MASK	GENMASK(2, 0)
+#define DP83867_STRAP_STS2_CLK_SKEW_RX_SHIFT	0
+#define DP83867_STRAP_STS2_CLK_SKEW_NONE	BIT(2)
+#define DP83867_STRAP_STS2_STRAP_FLD		BIT(10)
+
 /* PHY CTRL bits */
 #define DP83867_PHYCR_FIFO_DEPTH_SHIFT		14
 #define DP83867_PHYCR_FIFO_DEPTH_MASK		(3 << 14)
 #define DP83867_PHYCR_RESERVED_MASK		BIT(11)
 
 /* RGMIIDCTL bits */
+#define DP83867_RGMII_TX_CLK_DELAY_MAX		0xf
 #define DP83867_RGMII_TX_CLK_DELAY_SHIFT	4
+#define DP83867_RGMII_TX_CLK_DELAY_INV	(DP83867_RGMII_TX_CLK_DELAY_MAX + 1)
+#define DP83867_RGMII_RX_CLK_DELAY_MAX		0xf
+#define DP83867_RGMII_RX_CLK_DELAY_SHIFT	0
+#define DP83867_RGMII_RX_CLK_DELAY_INV	(DP83867_RGMII_RX_CLK_DELAY_MAX + 1)
 
 /* IO_MUX_CFG bits */
 #define DP83867_IO_MUX_CFG_IO_IMPEDANCE_CTRL	0x1f
@@ -93,6 +110,9 @@
 /* CFG4 bits */
 #define DP83867_CFG4_PORT_MIRROR_EN              BIT(0)
 
+/* FLD_THR_CFG */
+#define DP83867_FLD_THR_CFG_ENERGY_LOST_THR_MASK	0x7
+
 enum {
 	DP83867_PORT_MIRROING_KEEP,
 	DP83867_PORT_MIRROING_EN,
@@ -161,6 +181,48 @@
 	return 0;
 }
 
+static int dp83867_verify_rgmii_cfg(struct phy_device *phydev)
+{
+	struct dp83867_private *dp83867 = phydev->priv;
+
+	/* Existing behavior was to use default pin strapping delay in rgmii
+	 * mode, but rgmii should have meant no delay.  Warn existing users.
+	 */
+	if (phydev->interface == PHY_INTERFACE_MODE_RGMII) {
+		const u16 val = phy_read_mmd(phydev, DP83867_DEVADDR,
+					     DP83867_STRAP_STS2);
+		const u16 txskew = (val & DP83867_STRAP_STS2_CLK_SKEW_TX_MASK) >>
+				   DP83867_STRAP_STS2_CLK_SKEW_TX_SHIFT;
+		const u16 rxskew = (val & DP83867_STRAP_STS2_CLK_SKEW_RX_MASK) >>
+				   DP83867_STRAP_STS2_CLK_SKEW_RX_SHIFT;
+
+		if (txskew != DP83867_STRAP_STS2_CLK_SKEW_NONE ||
+		    rxskew != DP83867_STRAP_STS2_CLK_SKEW_NONE)
+			phydev_err(phydev,
+				   "PHY has delays via pin strapping, but phy-mode = 'rgmii'\n"
+				   "Should be 'rgmii-id' to use internal delays txskew:%x rxskew:%x\n",
+				   txskew, rxskew);
+	}
+
+	/* RX delay *must* be specified if internal delay of RX is used. */
+	if ((phydev->interface == PHY_INTERFACE_MODE_RGMII_ID ||
+	     phydev->interface == PHY_INTERFACE_MODE_RGMII_RXID) &&
+	     dp83867->rx_id_delay == DP83867_RGMII_RX_CLK_DELAY_INV) {
+		phydev_err(phydev, "ti,rx-internal-delay must be specified\n");
+		return -EINVAL;
+	}
+
+	/* TX delay *must* be specified if internal delay of TX is used. */
+	if ((phydev->interface == PHY_INTERFACE_MODE_RGMII_ID ||
+	     phydev->interface == PHY_INTERFACE_MODE_RGMII_TXID) &&
+	     dp83867->tx_id_delay == DP83867_RGMII_TX_CLK_DELAY_INV) {
+		phydev_err(phydev, "ti,tx-internal-delay must be specified\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 #ifdef CONFIG_OF_MDIO
 static int dp83867_of_init(struct phy_device *phydev)
 {
@@ -191,19 +253,25 @@
 	dp83867->rxctrl_strap_quirk = of_property_read_bool(of_node,
 					"ti,dp83867-rxctrl-strap-quirk");
 
+	dp83867->rx_id_delay = DP83867_RGMII_RX_CLK_DELAY_INV;
 	ret = of_property_read_u32(of_node, "ti,rx-internal-delay",
 				   &dp83867->rx_id_delay);
-	if (ret &&
-	    (phydev->interface == PHY_INTERFACE_MODE_RGMII_ID ||
-	     phydev->interface == PHY_INTERFACE_MODE_RGMII_RXID))
-		return ret;
+	if (!ret && dp83867->rx_id_delay > DP83867_RGMII_RX_CLK_DELAY_MAX) {
+		phydev_err(phydev,
+			   "ti,rx-internal-delay value of %u out of range\n",
+			   dp83867->rx_id_delay);
+		return -EINVAL;
+	}
 
+	dp83867->tx_id_delay = DP83867_RGMII_TX_CLK_DELAY_INV;
 	ret = of_property_read_u32(of_node, "ti,tx-internal-delay",
 				   &dp83867->tx_id_delay);
-	if (ret &&
-	    (phydev->interface == PHY_INTERFACE_MODE_RGMII_ID ||
-	     phydev->interface == PHY_INTERFACE_MODE_RGMII_TXID))
-		return ret;
+	if (!ret && dp83867->tx_id_delay > DP83867_RGMII_TX_CLK_DELAY_MAX) {
+		phydev_err(phydev,
+			   "ti,tx-internal-delay value of %u out of range\n",
+			   dp83867->tx_id_delay);
+		return -EINVAL;
+	}
 
 	if (of_property_read_bool(of_node, "enet-phy-lane-swap"))
 		dp83867->port_mirroring = DP83867_PORT_MIRROING_EN;
@@ -221,25 +289,29 @@
 }
 #endif /* CONFIG_OF_MDIO */
 
-static int dp83867_config_init(struct phy_device *phydev)
+static int dp83867_probe(struct phy_device *phydev)
 {
 	struct dp83867_private *dp83867;
+
+	dp83867 = devm_kzalloc(&phydev->mdio.dev, sizeof(*dp83867),
+			       GFP_KERNEL);
+	if (!dp83867)
+		return -ENOMEM;
+
+	phydev->priv = dp83867;
+
+	return dp83867_of_init(phydev);
+}
+
+static int dp83867_config_init(struct phy_device *phydev)
+{
+	struct dp83867_private *dp83867 = phydev->priv;
 	int ret, val, bs;
 	u16 delay;
 
-	if (!phydev->priv) {
-		dp83867 = devm_kzalloc(&phydev->mdio.dev, sizeof(*dp83867),
-				       GFP_KERNEL);
-		if (!dp83867)
-			return -ENOMEM;
-
-		phydev->priv = dp83867;
-		ret = dp83867_of_init(phydev);
-		if (ret)
-			return ret;
-	} else {
-		dp83867 = (struct dp83867_private *)phydev->priv;
-	}
+	ret = dp83867_verify_rgmii_cfg(phydev);
+	if (ret)
+		return ret;
 
 	/* RX_DV/RX_CTRL strapped in mode 1 or mode 2 workaround */
 	if (dp83867->rxctrl_strap_quirk) {
@@ -248,6 +320,22 @@
 		phy_write_mmd(phydev, DP83867_DEVADDR, DP83867_CFG4, val);
 	}
 
+	bs = phy_read_mmd(phydev, DP83867_DEVADDR, DP83867_STRAP_STS2);
+	if (bs & DP83867_STRAP_STS2_STRAP_FLD) {
+		/* When using strap to enable FLD, the ENERGY_LOST_FLD_THR will
+		 * be set to 0x2. This may causes the PHY link to be unstable -
+		 * the default value 0x1 need to be restored.
+		 */
+		val = phy_read_mmd(phydev, DP83867_DEVADDR,
+				   DP83867_FLD_THR_CFG);
+		if ((val & DP83867_FLD_THR_CFG_ENERGY_LOST_THR_MASK) == 0x2) {
+			val &= ~DP83867_FLD_THR_CFG_ENERGY_LOST_THR_MASK;
+			val |= 0x1;
+			phy_write_mmd(phydev, DP83867_DEVADDR,
+				      DP83867_FLD_THR_CFG, val);
+		}
+	}
+
 	if (phy_interface_is_rgmii(phydev)) {
 		val = phy_read(phydev, MII_DP83867_PHYCTRL);
 		if (val < 0)
@@ -276,6 +364,9 @@
 		/* Set up RGMII delays */
 		val = phy_read_mmd(phydev, DP83867_DEVADDR, DP83867_RGMIICTL);
 
+		val &= ~(DP83867_RGMII_TX_CLK_DELAY_EN |
+			 DP83867_RGMII_RX_CLK_DELAY_EN);
+
 		if (phydev->interface == PHY_INTERFACE_MODE_RGMII_ID)
 			val |= (DP83867_RGMII_TX_CLK_DELAY_EN | DP83867_RGMII_RX_CLK_DELAY_EN);
 
@@ -287,8 +378,12 @@
 
 		phy_write_mmd(phydev, DP83867_DEVADDR, DP83867_RGMIICTL, val);
 
-		delay = (dp83867->rx_id_delay |
-			(dp83867->tx_id_delay << DP83867_RGMII_TX_CLK_DELAY_SHIFT));
+		delay = 0;
+		if (dp83867->rx_id_delay != DP83867_RGMII_RX_CLK_DELAY_INV)
+			delay |= dp83867->rx_id_delay;
+		if (dp83867->tx_id_delay != DP83867_RGMII_TX_CLK_DELAY_INV)
+			delay |= dp83867->tx_id_delay <<
+				 DP83867_RGMII_TX_CLK_DELAY_SHIFT;
 
 		phy_write_mmd(phydev, DP83867_DEVADDR, DP83867_RGMIIDCTL,
 			      delay);
@@ -341,6 +436,7 @@
 		val |= DP83867_CFG3_INT_OE;
 
 	val |= DP83867_CFG3_ROBUST_AUTO_MDIX;
+
 	phy_write(phydev, DP83867_CFG3, val);
 
 	if (dp83867->port_mirroring != DP83867_PORT_MIRROING_KEEP)
@@ -359,13 +455,24 @@
 
 static int dp83867_phy_reset(struct phy_device *phydev)
 {
+	struct dp83867_private *dp83867 = phydev->priv;
 	int err;
 
 	err = phy_write(phydev, DP83867_CTRL, DP83867_SW_RESET);
 	if (err < 0)
 		return err;
 
-	return dp83867_config_init(phydev);
+	usleep_range(10, 20);
+
+	phy_write_mmd(phydev, DP83867_DEVADDR, DP83867_DSP_FFE_CFG, 0x0E81);
+
+	err = phy_write(phydev, DP83867_CTRL, DP83867_SW_RESTART);
+	if (err < 0)
+		return err;
+
+	usleep_range(10, 20);
+
+	return 0;
 }
 
 static struct phy_driver dp83867_driver[] = {
@@ -376,6 +483,7 @@
 		.features	= PHY_GBIT_FEATURES,
 		.flags		= PHY_HAS_INTERRUPT,
 
+		.probe          = dp83867_probe,
 		.config_init	= dp83867_config_init,
 		.soft_reset	= dp83867_phy_reset,
 
diff -Naur --no-dereference a/drivers/net/phy/mdio_bus.c b/drivers/net/phy/mdio_bus.c
--- a/drivers/net/phy/mdio_bus.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/phy/mdio_bus.c	2020-06-19 21:00:31.416227715 -0400
@@ -386,20 +386,26 @@
 	mutex_init(&bus->mdio_lock);
 
 	/* de-assert bus level PHY GPIO reset */
-	gpiod = devm_gpiod_get_optional(&bus->dev, "reset", GPIOD_OUT_LOW);
-	if (IS_ERR(gpiod)) {
-		dev_err(&bus->dev, "mii_bus %s couldn't get reset GPIO\n",
-			bus->id);
-		device_del(&bus->dev);
-		return PTR_ERR(gpiod);
-	} else	if (gpiod) {
-		bus->reset_gpiod = gpiod;
-
-		gpiod_set_value_cansleep(gpiod, 1);
-		udelay(bus->reset_delay_us);
-		gpiod_set_value_cansleep(gpiod, 0);
+	for (i = 0; i < PHY_MAX_ADDR; i++) {
+		gpiod = devm_gpiod_get_index_optional(&bus->dev, "reset",
+						      i, GPIOD_OUT_LOW);
+		if (IS_ERR(gpiod)) {
+			dev_err(&bus->dev, "mii_bus %s couldn't get reset GPIO\n",
+				bus->id);
+			return PTR_ERR(gpiod);
+		} else	if (gpiod) {
+			bus->reset_gpiod[i] = gpiod;
+
+			gpiod_set_value_cansleep(gpiod, 1);
+			udelay(bus->reset_delay_us);
+			gpiod_set_value_cansleep(gpiod, 0);
+		} else {
+			break;
+		}
 	}
 
+	bus->num_resets = i;
+
 	if (bus->reset)
 		bus->reset(bus);
 
@@ -432,8 +438,10 @@
 	}
 
 	/* Put PHYs in RESET to save power */
-	if (bus->reset_gpiod)
-		gpiod_set_value_cansleep(bus->reset_gpiod, 1);
+	for (i = 0; i < bus->num_resets; i++) {
+		if (bus->reset_gpiod[i])
+			gpiod_set_value_cansleep(bus->reset_gpiod[i], 1);
+	}
 
 	device_del(&bus->dev);
 	return err;
@@ -461,8 +469,10 @@
 	}
 
 	/* Put PHYs in RESET to save power */
-	if (bus->reset_gpiod)
-		gpiod_set_value_cansleep(bus->reset_gpiod, 1);
+	for (i = 0; i < bus->num_resets; i++) {
+		if (bus->reset_gpiod[i])
+			gpiod_set_value_cansleep(bus->reset_gpiod[i], 1);
+	}
 
 	device_del(&bus->dev);
 }
diff -Naur --no-dereference a/drivers/net/phy/phy.c b/drivers/net/phy/phy.c
--- a/drivers/net/phy/phy.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/phy/phy.c	2020-06-19 21:00:31.416227715 -0400
@@ -861,6 +861,9 @@
 out_unlock:
 	mutex_unlock(&phydev->lock);
 
+	phy_state_machine(&phydev->state_queue.work);
+	phy_stop_machine(phydev);
+
 	/* Cannot call flush_scheduled_work() here as desired because
 	 * of rtnl_lock(), but PHY_HALTED shall guarantee phy_change()
 	 * will not reenable interrupts.
diff -Naur --no-dereference a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c
--- a/drivers/net/phy/phy_device.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/phy/phy_device.c	2020-06-19 21:00:31.416227715 -0400
@@ -825,8 +825,6 @@
 	if (phydev->irq > 0)
 		phy_stop_interrupts(phydev);
 
-	phy_stop_machine(phydev);
-
 	phydev->adjust_link = NULL;
 
 	phy_detach(phydev);
@@ -1614,8 +1612,9 @@
 				phydev->duplex = DUPLEX_FULL;
 
 		if (phydev->duplex == DUPLEX_FULL) {
-			phydev->pause = lpa & LPA_PAUSE_CAP ? 1 : 0;
-			phydev->asym_pause = lpa & LPA_PAUSE_ASYM ? 1 : 0;
+			phydev->pause = common_adv & LPA_PAUSE_CAP ? 1 : 0;
+			phydev->asym_pause = common_adv & LPA_PAUSE_ASYM ?
+					     1 : 0;
 		}
 	} else {
 		int bmcr = phy_read(phydev, MII_BMCR);
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c	2020-06-19 21:00:31.416227715 -0400
@@ -98,6 +98,8 @@
 					 * plus any space that might be needed
 					 * for bus alignment padding.
 					 */
+#define ROUND_UP_MARGIN 2048
+
 struct brcmf_bcdc {
 	u16 reqid;
 	u8 bus_header[BUS_HEADER_LEN];
@@ -165,7 +167,7 @@
 
 static int
 brcmf_proto_bcdc_query_dcmd(struct brcmf_pub *drvr, int ifidx, uint cmd,
-			    void *buf, uint len, int *fwerr)
+			    void *buf, uint len)
 {
 	struct brcmf_bcdc *bcdc = (struct brcmf_bcdc *)drvr->proto->pd;
 	struct brcmf_proto_bcdc_dcmd *msg = &bcdc->msg;
@@ -175,7 +177,6 @@
 
 	brcmf_dbg(BCDC, "Enter, cmd %d len %d\n", cmd, len);
 
-	*fwerr = 0;
 	ret = brcmf_proto_bcdc_msg(drvr, ifidx, cmd, buf, len, false);
 	if (ret < 0) {
 		brcmf_err("brcmf_proto_bcdc_msg failed w/status %d\n",
@@ -212,27 +213,25 @@
 		memcpy(buf, info, len);
 	}
 
-	ret = 0;
-
 	/* Check the ERROR flag */
 	if (flags & BCDC_DCMD_ERROR)
-		*fwerr = le32_to_cpu(msg->status);
+		ret = le32_to_cpu(msg->status);
+
 done:
 	return ret;
 }
 
 static int
 brcmf_proto_bcdc_set_dcmd(struct brcmf_pub *drvr, int ifidx, uint cmd,
-			  void *buf, uint len, int *fwerr)
+			  void *buf, uint len)
 {
 	struct brcmf_bcdc *bcdc = (struct brcmf_bcdc *)drvr->proto->pd;
 	struct brcmf_proto_bcdc_dcmd *msg = &bcdc->msg;
-	int ret;
+	int ret = 0;
 	u32 flags, id;
 
 	brcmf_dbg(BCDC, "Enter, cmd %d len %d\n", cmd, len);
 
-	*fwerr = 0;
 	ret = brcmf_proto_bcdc_msg(drvr, ifidx, cmd, buf, len, true);
 	if (ret < 0)
 		goto done;
@@ -252,11 +251,9 @@
 		goto done;
 	}
 
-	ret = 0;
-
 	/* Check the ERROR flag */
 	if (flags & BCDC_DCMD_ERROR)
-		*fwerr = le32_to_cpu(msg->status);
+		ret = le32_to_cpu(msg->status);
 
 done:
 	return ret;
@@ -445,11 +442,6 @@
 	return 0;
 }
 
-static void brcmf_proto_bcdc_debugfs_create(struct brcmf_pub *drvr)
-{
-	brcmf_fws_debugfs_create(drvr);
-}
-
 int brcmf_proto_bcdc_attach(struct brcmf_pub *drvr)
 {
 	struct brcmf_bcdc *bcdc;
@@ -477,12 +469,11 @@
 	drvr->proto->del_if = brcmf_proto_bcdc_del_if;
 	drvr->proto->reset_if = brcmf_proto_bcdc_reset_if;
 	drvr->proto->init_done = brcmf_proto_bcdc_init_done;
-	drvr->proto->debugfs_create = brcmf_proto_bcdc_debugfs_create;
 	drvr->proto->pd = bcdc;
 
 	drvr->hdrlen += BCDC_HEADER_LEN + BRCMF_PROT_FW_SIGNAL_MAX_TXBYTES;
 	drvr->bus_if->maxctl = BRCMF_DCMD_MAXLEN +
-			sizeof(struct brcmf_proto_bcdc_dcmd);
+			sizeof(struct brcmf_proto_bcdc_dcmd) + ROUND_UP_MARGIN;
 	return 0;
 
 fail:
@@ -490,18 +481,11 @@
 	return -ENOMEM;
 }
 
-void brcmf_proto_bcdc_detach_pre_delif(struct brcmf_pub *drvr)
-{
-	struct brcmf_bcdc *bcdc = drvr->proto->pd;
-
-	brcmf_fws_detach_pre_delif(bcdc->fws);
-}
-
-void brcmf_proto_bcdc_detach_post_delif(struct brcmf_pub *drvr)
+void brcmf_proto_bcdc_detach(struct brcmf_pub *drvr)
 {
 	struct brcmf_bcdc *bcdc = drvr->proto->pd;
 
 	drvr->proto->pd = NULL;
-	brcmf_fws_detach_post_delif(bcdc->fws);
+	brcmf_fws_detach(bcdc->fws);
 	kfree(bcdc);
 }
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.h
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.h	2020-06-19 21:00:31.416227715 -0400
@@ -18,16 +18,14 @@
 
 #ifdef CONFIG_BRCMFMAC_PROTO_BCDC
 int brcmf_proto_bcdc_attach(struct brcmf_pub *drvr);
-void brcmf_proto_bcdc_detach_pre_delif(struct brcmf_pub *drvr);
-void brcmf_proto_bcdc_detach_post_delif(struct brcmf_pub *drvr);
+void brcmf_proto_bcdc_detach(struct brcmf_pub *drvr);
 void brcmf_proto_bcdc_txflowblock(struct device *dev, bool state);
 void brcmf_proto_bcdc_txcomplete(struct device *dev, struct sk_buff *txp,
 				 bool success);
 struct brcmf_fws_info *drvr_to_fws(struct brcmf_pub *drvr);
 #else
 static inline int brcmf_proto_bcdc_attach(struct brcmf_pub *drvr) { return 0; }
-static void brcmf_proto_bcdc_detach_pre_delif(struct brcmf_pub *drvr) {};
-static inline void brcmf_proto_bcdc_detach_post_delif(struct brcmf_pub *drvr) {}
+static inline void brcmf_proto_bcdc_detach(struct brcmf_pub *drvr) {}
 #endif
 
 #endif /* BRCMFMAC_BCDC_H */
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c	2020-06-19 21:00:31.416227715 -0400
@@ -47,6 +47,7 @@
 #include "sdio.h"
 #include "core.h"
 #include "common.h"
+#include "cfg80211.h"
 
 #define SDIOH_API_ACCESS_RETRY_LIMIT	2
 
@@ -54,6 +55,8 @@
 
 #define SDIO_FUNC1_BLOCKSIZE		64
 #define SDIO_FUNC2_BLOCKSIZE		512
+#define SDIO_4373_FUNC2_BLOCKSIZE	256
+#define SDIO_435X_FUNC2_BLOCKSIZE	256
 /* Maximum milliseconds to wait for F2 to come up */
 #define SDIO_WAIT_F2RDY	3000
 
@@ -118,7 +121,7 @@
 
 		ret = request_irq(pdata->oob_irq_nr, brcmf_sdiod_oob_irqhandler,
 				  pdata->oob_irq_flags, "brcmf_oob_intr",
-				  &sdiodev->func1->dev);
+				  &sdiodev->func[1]->dev);
 		if (ret != 0) {
 			brcmf_err("request_irq failed %d\n", ret);
 			return ret;
@@ -132,40 +135,39 @@
 		}
 		sdiodev->irq_wake = true;
 
-		sdio_claim_host(sdiodev->func1);
+		sdio_claim_host(sdiodev->func[1]);
 
 		if (sdiodev->bus_if->chip == BRCM_CC_43362_CHIP_ID) {
 			/* assign GPIO to SDIO core */
 			addr = CORE_CC_REG(SI_ENUM_BASE, gpiocontrol);
-			gpiocontrol = brcmf_sdiod_readl(sdiodev, addr, &ret);
+			gpiocontrol = brcmf_sdiod_regrl(sdiodev, addr, &ret);
 			gpiocontrol |= 0x2;
-			brcmf_sdiod_writel(sdiodev, addr, gpiocontrol, &ret);
+			brcmf_sdiod_regwl(sdiodev, addr, gpiocontrol, &ret);
 
-			brcmf_sdiod_writeb(sdiodev, SBSDIO_GPIO_SELECT,
-					   0xf, &ret);
-			brcmf_sdiod_writeb(sdiodev, SBSDIO_GPIO_OUT, 0, &ret);
-			brcmf_sdiod_writeb(sdiodev, SBSDIO_GPIO_EN, 0x2, &ret);
+			brcmf_sdiod_regwb(sdiodev, SBSDIO_GPIO_SELECT, 0xf,
+					  &ret);
+			brcmf_sdiod_regwb(sdiodev, SBSDIO_GPIO_OUT, 0, &ret);
+			brcmf_sdiod_regwb(sdiodev, SBSDIO_GPIO_EN, 0x2, &ret);
 		}
 
 		/* must configure SDIO_CCCR_IENx to enable irq */
-		data = brcmf_sdiod_func0_rb(sdiodev, SDIO_CCCR_IENx, &ret);
-		data |= SDIO_CCCR_IEN_FUNC1 | SDIO_CCCR_IEN_FUNC2 |
-			SDIO_CCCR_IEN_FUNC0;
-		brcmf_sdiod_func0_wb(sdiodev, SDIO_CCCR_IENx, data, &ret);
+		data = brcmf_sdiod_regrb(sdiodev, SDIO_CCCR_IENx, &ret);
+		data |= 1 << SDIO_FUNC_1 | 1 << SDIO_FUNC_2 | 1;
+		brcmf_sdiod_regwb(sdiodev, SDIO_CCCR_IENx, data, &ret);
 
 		/* redirect, configure and enable io for interrupt signal */
-		data = SDIO_CCCR_BRCM_SEPINT_MASK | SDIO_CCCR_BRCM_SEPINT_OE;
+		data = SDIO_SEPINT_MASK | SDIO_SEPINT_OE;
 		if (pdata->oob_irq_flags & IRQF_TRIGGER_HIGH)
-			data |= SDIO_CCCR_BRCM_SEPINT_ACT_HI;
-		brcmf_sdiod_func0_wb(sdiodev, SDIO_CCCR_BRCM_SEPINT,
-				     data, &ret);
-		sdio_release_host(sdiodev->func1);
+			data |= SDIO_SEPINT_ACT_HI;
+		brcmf_sdiod_regwb(sdiodev, SDIO_CCCR_BRCM_SEPINT, data, &ret);
+
+		sdio_release_host(sdiodev->func[1]);
 	} else {
 		brcmf_dbg(SDIO, "Entering\n");
-		sdio_claim_host(sdiodev->func1);
-		sdio_claim_irq(sdiodev->func1, brcmf_sdiod_ib_irqhandler);
-		sdio_claim_irq(sdiodev->func2, brcmf_sdiod_dummy_irqhandler);
-		sdio_release_host(sdiodev->func1);
+		sdio_claim_host(sdiodev->func[1]);
+		sdio_claim_irq(sdiodev->func[1], brcmf_sdiod_ib_irqhandler);
+		sdio_claim_irq(sdiodev->func[2], brcmf_sdiod_dummy_irqhandler);
+		sdio_release_host(sdiodev->func[1]);
 		sdiodev->sd_irq_requested = true;
 	}
 
@@ -183,26 +185,26 @@
 		struct brcmfmac_sdio_pd *pdata;
 
 		pdata = &sdiodev->settings->bus.sdio;
-		sdio_claim_host(sdiodev->func1);
-		brcmf_sdiod_func0_wb(sdiodev, SDIO_CCCR_BRCM_SEPINT, 0, NULL);
-		brcmf_sdiod_func0_wb(sdiodev, SDIO_CCCR_IENx, 0, NULL);
-		sdio_release_host(sdiodev->func1);
+		sdio_claim_host(sdiodev->func[1]);
+		brcmf_sdiod_regwb(sdiodev, SDIO_CCCR_BRCM_SEPINT, 0, NULL);
+		brcmf_sdiod_regwb(sdiodev, SDIO_CCCR_IENx, 0, NULL);
+		sdio_release_host(sdiodev->func[1]);
 
 		sdiodev->oob_irq_requested = false;
 		if (sdiodev->irq_wake) {
 			disable_irq_wake(pdata->oob_irq_nr);
 			sdiodev->irq_wake = false;
 		}
-		free_irq(pdata->oob_irq_nr, &sdiodev->func1->dev);
+		free_irq(pdata->oob_irq_nr, &sdiodev->func[1]->dev);
 		sdiodev->irq_en = false;
 		sdiodev->oob_irq_requested = false;
 	}
 
 	if (sdiodev->sd_irq_requested) {
-		sdio_claim_host(sdiodev->func1);
-		sdio_release_irq(sdiodev->func2);
-		sdio_release_irq(sdiodev->func1);
-		sdio_release_host(sdiodev->func1);
+		sdio_claim_host(sdiodev->func[1]);
+		sdio_release_irq(sdiodev->func[2]);
+		sdio_release_irq(sdiodev->func[1]);
+		sdio_release_host(sdiodev->func[1]);
 		sdiodev->sd_irq_requested = false;
 	}
 }
@@ -231,121 +233,271 @@
 	sdiodev->state = state;
 }
 
-static int brcmf_sdiod_set_backplane_window(struct brcmf_sdio_dev *sdiodev,
-					    u32 addr)
+static inline int brcmf_sdiod_f0_writeb(struct sdio_func *func,
+					uint regaddr, u8 byte)
 {
-	u32 v, bar0 = addr & SBSDIO_SBWINDOW_MASK;
-	int err = 0, i;
+	int err_ret;
 
-	if (bar0 == sdiodev->sbwad)
-		return 0;
+	/*
+	 * Can only directly write to some F0 registers.
+	 * Handle CCCR_IENx and CCCR_ABORT command
+	 * as a special case.
+	 */
+	if ((regaddr == SDIO_CCCR_ABORT) ||
+	    (regaddr == SDIO_CCCR_IENx))
+		sdio_writeb(func, byte, regaddr, &err_ret);
+	else
+		sdio_f0_writeb(func, byte, regaddr, &err_ret);
 
-	v = bar0 >> 8;
+	return err_ret;
+}
 
-	for (i = 0 ; i < 3 && !err ; i++, v >>= 8)
-		brcmf_sdiod_writeb(sdiodev, SBSDIO_FUNC1_SBADDRLOW + i,
-				   v & 0xff, &err);
+static int brcmf_sdiod_request_data(struct brcmf_sdio_dev *sdiodev, u8 fn,
+				    u32 addr, u8 regsz, void *data, bool write)
+{
+	struct sdio_func *func;
+	int ret = -EINVAL;
 
-	if (!err)
-		sdiodev->sbwad = bar0;
+	brcmf_dbg(SDIO, "rw=%d, func=%d, addr=0x%05x, nbytes=%d\n",
+		  write, fn, addr, regsz);
+
+	/* only allow byte access on F0 */
+	if (WARN_ON(regsz > 1 && !fn))
+		return -EINVAL;
+	func = sdiodev->func[fn];
+
+	switch (regsz) {
+	case sizeof(u8):
+		if (write) {
+			if (fn)
+				sdio_writeb(func, *(u8 *)data, addr, &ret);
+			else
+				ret = brcmf_sdiod_f0_writeb(func, addr,
+							    *(u8 *)data);
+		} else {
+			if (fn)
+				*(u8 *)data = sdio_readb(func, addr, &ret);
+			else
+				*(u8 *)data = sdio_f0_readb(func, addr, &ret);
+		}
+		break;
+	case sizeof(u16):
+		if (write)
+			sdio_writew(func, *(u16 *)data, addr, &ret);
+		else
+			*(u16 *)data = sdio_readw(func, addr, &ret);
+		break;
+	case sizeof(u32):
+		if (write)
+			sdio_writel(func, *(u32 *)data, addr, &ret);
+		else
+			*(u32 *)data = sdio_readl(func, addr, &ret);
+		break;
+	default:
+		brcmf_err("invalid size: %d\n", regsz);
+		break;
+	}
+
+	if (ret)
+		brcmf_dbg(SDIO, "failed to %s data F%d@0x%05x, err: %d\n",
+			  write ? "write" : "read", fn, addr, ret);
+
+	return ret;
+}
+
+static int brcmf_sdiod_regrw_helper(struct brcmf_sdio_dev *sdiodev, u32 addr,
+				   u8 regsz, void *data, bool write)
+{
+	u8 func;
+	s32 retry = 0;
+	int ret;
+
+	if (sdiodev->state == BRCMF_SDIOD_NOMEDIUM)
+		return -ENOMEDIUM;
+
+	/*
+	 * figure out how to read the register based on address range
+	 * 0x00 ~ 0x7FF: function 0 CCCR and FBR
+	 * 0x10000 ~ 0x1FFFF: function 1 miscellaneous registers
+	 * The rest: function 1 silicon backplane core registers
+	 */
+	if ((addr & ~REG_F0_REG_MASK) == 0)
+		func = SDIO_FUNC_0;
+	else
+		func = SDIO_FUNC_1;
+
+	do {
+		if (!write)
+			memset(data, 0, regsz);
+		/* for retry wait for 1 ms till bus get settled down */
+		if (retry)
+			usleep_range(1000, 2000);
+		ret = brcmf_sdiod_request_data(sdiodev, func, addr, regsz,
+					       data, write);
+	} while (ret != 0 && ret != -ENOMEDIUM &&
+		 retry++ < SDIOH_API_ACCESS_RETRY_LIMIT);
+
+	if (ret == -ENOMEDIUM)
+		brcmf_sdiod_change_state(sdiodev, BRCMF_SDIOD_NOMEDIUM);
+	else if (ret != 0) {
+		/*
+		 * SleepCSR register access can fail when
+		 * waking up the device so reduce this noise
+		 * in the logs.
+		 */
+		if (addr != SBSDIO_FUNC1_SLEEPCSR)
+			brcmf_err("failed to %s data F%d@0x%05x, err: %d\n",
+				  write ? "write" : "read", func, addr, ret);
+		else
+			brcmf_dbg(SDIO, "failed to %s data F%d@0x%05x, err: %d\n",
+				  write ? "write" : "read", func, addr, ret);
+	}
+	return ret;
+}
+
+static int
+brcmf_sdiod_set_sbaddr_window(struct brcmf_sdio_dev *sdiodev, u32 address)
+{
+	int err = 0, i;
+	u8 addr[3];
+
+	if (sdiodev->state == BRCMF_SDIOD_NOMEDIUM)
+		return -ENOMEDIUM;
+
+	addr[0] = (address >> 8) & SBSDIO_SBADDRLOW_MASK;
+	addr[1] = (address >> 16) & SBSDIO_SBADDRMID_MASK;
+	addr[2] = (address >> 24) & SBSDIO_SBADDRHIGH_MASK;
+
+	for (i = 0; i < 3; i++) {
+		err = brcmf_sdiod_regrw_helper(sdiodev,
+					       SBSDIO_FUNC1_SBADDRLOW + i,
+					       sizeof(u8), &addr[i], true);
+		if (err) {
+			brcmf_err("failed at addr: 0x%0x\n",
+				  SBSDIO_FUNC1_SBADDRLOW + i);
+			break;
+		}
+	}
 
 	return err;
 }
 
-u32 brcmf_sdiod_readl(struct brcmf_sdio_dev *sdiodev, u32 addr, int *ret)
+static int
+brcmf_sdiod_addrprep(struct brcmf_sdio_dev *sdiodev, uint width, u32 *addr)
 {
-	u32 data = 0;
-	int retval;
+	uint bar0 = *addr & ~SBSDIO_SB_OFT_ADDR_MASK;
+	int err = 0;
 
-	retval = brcmf_sdiod_set_backplane_window(sdiodev, addr);
-	if (retval)
-		goto out;
+	if (bar0 != sdiodev->sbwad) {
+		err = brcmf_sdiod_set_sbaddr_window(sdiodev, bar0);
+		if (err)
+			return err;
 
-	addr &= SBSDIO_SB_OFT_ADDR_MASK;
-	addr |= SBSDIO_SB_ACCESS_2_4B_FLAG;
+		sdiodev->sbwad = bar0;
+	}
 
-	data = sdio_readl(sdiodev->func1, addr, &retval);
+	*addr &= SBSDIO_SB_OFT_ADDR_MASK;
+
+	if (width == 4)
+		*addr |= SBSDIO_SB_ACCESS_2_4B_FLAG;
+
+	return 0;
+}
+
+u8 brcmf_sdiod_regrb(struct brcmf_sdio_dev *sdiodev, u32 addr, int *ret)
+{
+	u8 data;
+	int retval;
+
+	brcmf_dbg(SDIO, "addr:0x%08x\n", addr);
+	retval = brcmf_sdiod_regrw_helper(sdiodev, addr, sizeof(data), &data,
+					  false);
+	brcmf_dbg(SDIO, "data:0x%02x\n", data);
 
-out:
 	if (ret)
 		*ret = retval;
 
 	return data;
 }
 
-void brcmf_sdiod_writel(struct brcmf_sdio_dev *sdiodev, u32 addr,
-			u32 data, int *ret)
+u32 brcmf_sdiod_regrl(struct brcmf_sdio_dev *sdiodev, u32 addr, int *ret)
 {
+	u32 data = 0;
 	int retval;
 
-	retval = brcmf_sdiod_set_backplane_window(sdiodev, addr);
+	brcmf_dbg(SDIO, "addr:0x%08x\n", addr);
+	retval = brcmf_sdiod_addrprep(sdiodev, sizeof(data), &addr);
 	if (retval)
-		goto out;
-
-	addr &= SBSDIO_SB_OFT_ADDR_MASK;
-	addr |= SBSDIO_SB_ACCESS_2_4B_FLAG;
-
-	sdio_writel(sdiodev->func1, data, addr, &retval);
+		goto done;
+	retval = brcmf_sdiod_regrw_helper(sdiodev, addr, sizeof(data), &data,
+					  false);
+	brcmf_dbg(SDIO, "data:0x%08x\n", data);
 
-out:
+done:
 	if (ret)
 		*ret = retval;
+
+	return data;
 }
 
-static int brcmf_sdiod_skbuff_read(struct brcmf_sdio_dev *sdiodev,
-				   struct sdio_func *func, u32 addr,
-				   struct sk_buff *skb)
+void brcmf_sdiod_regwb(struct brcmf_sdio_dev *sdiodev, u32 addr,
+		      u8 data, int *ret)
 {
-	unsigned int req_sz;
-	int err;
+	int retval;
 
-	/* Single skb use the standard mmc interface */
-	req_sz = skb->len + 3;
-	req_sz &= (uint)~3;
+	brcmf_dbg(SDIO, "addr:0x%08x, data:0x%02x\n", addr, data);
+	retval = brcmf_sdiod_regrw_helper(sdiodev, addr, sizeof(data), &data,
+					  true);
+	if (ret)
+		*ret = retval;
+}
 
-	switch (func->num) {
-	case 1:
-		err = sdio_memcpy_fromio(func, ((u8 *)(skb->data)), addr,
-					 req_sz);
-		break;
-	case 2:
-		err = sdio_readsb(func, ((u8 *)(skb->data)), addr, req_sz);
-		break;
-	default:
-		/* bail out as things are really fishy here */
-		WARN(1, "invalid sdio function number: %d\n", func->num);
-		err = -ENOMEDIUM;
-	};
+void brcmf_sdiod_regwl(struct brcmf_sdio_dev *sdiodev, u32 addr,
+		      u32 data, int *ret)
+{
+	int retval;
 
-	if (err == -ENOMEDIUM)
-		brcmf_sdiod_change_state(sdiodev, BRCMF_SDIOD_NOMEDIUM);
+	brcmf_dbg(SDIO, "addr:0x%08x, data:0x%08x\n", addr, data);
+	retval = brcmf_sdiod_addrprep(sdiodev, sizeof(data), &addr);
+	if (retval)
+		goto done;
+	retval = brcmf_sdiod_regrw_helper(sdiodev, addr, sizeof(data), &data,
+					  true);
 
-	return err;
+done:
+	if (ret)
+		*ret = retval;
 }
 
-static int brcmf_sdiod_skbuff_write(struct brcmf_sdio_dev *sdiodev,
-				    struct sdio_func *func, u32 addr,
-				    struct sk_buff *skb)
+static int brcmf_sdiod_buffrw(struct brcmf_sdio_dev *sdiodev, uint fn,
+			     bool write, u32 addr, struct sk_buff *pkt)
 {
 	unsigned int req_sz;
 	int err;
 
 	/* Single skb use the standard mmc interface */
-	req_sz = skb->len + 3;
+	req_sz = pkt->len + 3;
 	req_sz &= (uint)~3;
 
-	err = sdio_memcpy_toio(func, addr, ((u8 *)(skb->data)), req_sz);
-
+	if (write)
+		err = sdio_memcpy_toio(sdiodev->func[fn], addr,
+				       ((u8 *)(pkt->data)), req_sz);
+	else if (fn == 1)
+		err = sdio_memcpy_fromio(sdiodev->func[fn], ((u8 *)(pkt->data)),
+					 addr, req_sz);
+	else
+		/* function 2 read is FIFO operation */
+		err = sdio_readsb(sdiodev->func[fn], ((u8 *)(pkt->data)), addr,
+				  req_sz);
 	if (err == -ENOMEDIUM)
 		brcmf_sdiod_change_state(sdiodev, BRCMF_SDIOD_NOMEDIUM);
-
 	return err;
 }
 
 /**
  * brcmf_sdiod_sglist_rw - SDIO interface function for block data access
  * @sdiodev: brcmfmac sdio device
- * @func: SDIO function
+ * @fn: SDIO function number
  * @write: direction flag
  * @addr: dongle memory address as source/destination
  * @pkt: skb pointer
@@ -354,8 +506,7 @@
  * stack for block data access. It assumes that the skb passed down by the
  * caller has already been padded and aligned.
  */
-static int brcmf_sdiod_sglist_rw(struct brcmf_sdio_dev *sdiodev,
-				 struct sdio_func *func,
+static int brcmf_sdiod_sglist_rw(struct brcmf_sdio_dev *sdiodev, uint fn,
 				 bool write, u32 addr,
 				 struct sk_buff_head *pktlist)
 {
@@ -381,7 +532,7 @@
 		req_sz = 0;
 		skb_queue_walk(pktlist, pkt_next)
 			req_sz += pkt_next->len;
-		req_sz = ALIGN(req_sz, func->cur_blksize);
+		req_sz = ALIGN(req_sz, sdiodev->func[fn]->cur_blksize);
 		while (req_sz > PAGE_SIZE) {
 			pkt_next = brcmu_pkt_buf_get_skb(PAGE_SIZE);
 			if (pkt_next == NULL) {
@@ -400,7 +551,7 @@
 		target_list = &local_list;
 	}
 
-	func_blk_sz = func->cur_blksize;
+	func_blk_sz = sdiodev->func[fn]->cur_blksize;
 	max_req_sz = sdiodev->max_request_size;
 	max_seg_cnt = min_t(unsigned short, sdiodev->max_segment_count,
 			    target_list->qlen);
@@ -417,10 +568,10 @@
 	mmc_dat.flags = write ? MMC_DATA_WRITE : MMC_DATA_READ;
 	mmc_cmd.opcode = SD_IO_RW_EXTENDED;
 	mmc_cmd.arg = write ? 1<<31 : 0;	/* write flag  */
-	mmc_cmd.arg |= (func->num & 0x7) << 28;	/* SDIO func num */
-	mmc_cmd.arg |= 1 << 27;			/* block mode */
+	mmc_cmd.arg |= (fn & 0x7) << 28;	/* SDIO func num */
+	mmc_cmd.arg |= 1<<27;			/* block mode */
 	/* for function 1 the addr will be incremented */
-	mmc_cmd.arg |= (func->num == 1) ? 1 << 26 : 0;
+	mmc_cmd.arg |= (fn == 1) ? 1<<26 : 0;
 	mmc_cmd.flags = MMC_RSP_SPI_R5 | MMC_RSP_R5 | MMC_CMD_ADTC;
 	mmc_req.cmd = &mmc_cmd;
 	mmc_req.data = &mmc_dat;
@@ -466,11 +617,11 @@
 		mmc_cmd.arg |= (addr & 0x1FFFF) << 9;	/* address */
 		mmc_cmd.arg |= mmc_dat.blocks & 0x1FF;	/* block count */
 		/* incrementing addr for function 1 */
-		if (func->num == 1)
+		if (fn == 1)
 			addr += req_sz;
 
-		mmc_set_data_timeout(&mmc_dat, func->card);
-		mmc_wait_for_req(func->card->host, &mmc_req);
+		mmc_set_data_timeout(&mmc_dat, sdiodev->func[fn]->card);
+		mmc_wait_for_req(sdiodev->func[fn]->card->host, &mmc_req);
 
 		ret = mmc_cmd.error ? mmc_cmd.error : mmc_dat.error;
 		if (ret == -ENOMEDIUM) {
@@ -538,19 +689,16 @@
 
 int brcmf_sdiod_recv_pkt(struct brcmf_sdio_dev *sdiodev, struct sk_buff *pkt)
 {
-	u32 addr = sdiodev->cc_core->base;
+	u32 addr = sdiodev->sbwad;
 	int err = 0;
 
 	brcmf_dbg(SDIO, "addr = 0x%x, size = %d\n", addr, pkt->len);
 
-	err = brcmf_sdiod_set_backplane_window(sdiodev, addr);
+	err = brcmf_sdiod_addrprep(sdiodev, 4, &addr);
 	if (err)
 		goto done;
 
-	addr &= SBSDIO_SB_OFT_ADDR_MASK;
-	addr |= SBSDIO_SB_ACCESS_2_4B_FLAG;
-
-	err = brcmf_sdiod_skbuff_read(sdiodev, sdiodev->func2, addr, pkt);
+	err = brcmf_sdiod_buffrw(sdiodev, SDIO_FUNC_2, false, addr, pkt);
 
 done:
 	return err;
@@ -561,28 +709,25 @@
 {
 	struct sk_buff *glom_skb = NULL;
 	struct sk_buff *skb;
-	u32 addr = sdiodev->cc_core->base;
+	u32 addr = sdiodev->sbwad;
 	int err = 0;
 
 	brcmf_dbg(SDIO, "addr = 0x%x, size = %d\n",
 		  addr, pktq->qlen);
 
-	err = brcmf_sdiod_set_backplane_window(sdiodev, addr);
+	err = brcmf_sdiod_addrprep(sdiodev, 4, &addr);
 	if (err)
 		goto done;
 
-	addr &= SBSDIO_SB_OFT_ADDR_MASK;
-	addr |= SBSDIO_SB_ACCESS_2_4B_FLAG;
-
 	if (pktq->qlen == 1)
-		err = brcmf_sdiod_skbuff_read(sdiodev, sdiodev->func2, addr,
-					      pktq->next);
+		err = brcmf_sdiod_buffrw(sdiodev, SDIO_FUNC_2, false, addr,
+					 pktq->next);
 	else if (!sdiodev->sg_support) {
 		glom_skb = brcmu_pkt_buf_get_skb(totlen);
 		if (!glom_skb)
 			return -ENOMEM;
-		err = brcmf_sdiod_skbuff_read(sdiodev, sdiodev->func2, addr,
-					      glom_skb);
+		err = brcmf_sdiod_buffrw(sdiodev, SDIO_FUNC_2, false, addr,
+					 glom_skb);
 		if (err)
 			goto done;
 
@@ -591,8 +736,8 @@
 			skb_pull(glom_skb, skb->len);
 		}
 	} else
-		err = brcmf_sdiod_sglist_rw(sdiodev, sdiodev->func2, false,
-					    addr, pktq);
+		err = brcmf_sdiod_sglist_rw(sdiodev, SDIO_FUNC_2, false, addr,
+					    pktq);
 
 done:
 	brcmu_pkt_buf_free_skb(glom_skb);
@@ -602,11 +747,10 @@
 int brcmf_sdiod_send_buf(struct brcmf_sdio_dev *sdiodev, u8 *buf, uint nbytes)
 {
 	struct sk_buff *mypkt;
-	u32 addr = sdiodev->cc_core->base;
+	u32 addr = sdiodev->sbwad;
 	int err;
 
 	mypkt = brcmu_pkt_buf_get_skb(nbytes);
-
 	if (!mypkt) {
 		brcmf_err("brcmu_pkt_buf_get_skb failed: len %d\n",
 			  nbytes);
@@ -615,49 +759,40 @@
 
 	memcpy(mypkt->data, buf, nbytes);
 
-	err = brcmf_sdiod_set_backplane_window(sdiodev, addr);
-	if (err)
-		return err;
-
-	addr &= SBSDIO_SB_OFT_ADDR_MASK;
-	addr |= SBSDIO_SB_ACCESS_2_4B_FLAG;
+	err = brcmf_sdiod_addrprep(sdiodev, 4, &addr);
 
 	if (!err)
-		err = brcmf_sdiod_skbuff_write(sdiodev, sdiodev->func2, addr,
-					       mypkt);
+		err = brcmf_sdiod_buffrw(sdiodev, SDIO_FUNC_2, true, addr,
+					 mypkt);
 
 	brcmu_pkt_buf_free_skb(mypkt);
-
 	return err;
+
 }
 
 int brcmf_sdiod_send_pkt(struct brcmf_sdio_dev *sdiodev,
 			 struct sk_buff_head *pktq)
 {
 	struct sk_buff *skb;
-	u32 addr = sdiodev->cc_core->base;
+	u32 addr = sdiodev->sbwad;
 	int err;
 
 	brcmf_dbg(SDIO, "addr = 0x%x, size = %d\n", addr, pktq->qlen);
 
-	err = brcmf_sdiod_set_backplane_window(sdiodev, addr);
+	err = brcmf_sdiod_addrprep(sdiodev, 4, &addr);
 	if (err)
 		return err;
 
-	addr &= SBSDIO_SB_OFT_ADDR_MASK;
-	addr |= SBSDIO_SB_ACCESS_2_4B_FLAG;
-
-	if (pktq->qlen == 1 || !sdiodev->sg_support) {
+	if (pktq->qlen == 1 || !sdiodev->sg_support)
 		skb_queue_walk(pktq, skb) {
-			err = brcmf_sdiod_skbuff_write(sdiodev, sdiodev->func2,
-						       addr, skb);
+			err = brcmf_sdiod_buffrw(sdiodev, SDIO_FUNC_2, true,
+						 addr, skb);
 			if (err)
 				break;
 		}
-	} else {
-		err = brcmf_sdiod_sglist_rw(sdiodev, sdiodev->func2, true,
-					    addr, pktq);
-	}
+	else
+		err = brcmf_sdiod_sglist_rw(sdiodev, SDIO_FUNC_2, true, addr,
+					    pktq);
 
 	return err;
 }
@@ -666,7 +801,7 @@
 brcmf_sdiod_ramrw(struct brcmf_sdio_dev *sdiodev, bool write, u32 address,
 		  u8 *data, uint size)
 {
-	int err = 0;
+	int bcmerror = 0;
 	struct sk_buff *pkt;
 	u32 sdaddr;
 	uint dsize;
@@ -686,13 +821,13 @@
 	else
 		dsize = size;
 
-	sdio_claim_host(sdiodev->func1);
+	sdio_claim_host(sdiodev->func[1]);
 
 	/* Do the transfer(s) */
 	while (size) {
 		/* Set the backplane window to include the start address */
-		err = brcmf_sdiod_set_backplane_window(sdiodev, address);
-		if (err)
+		bcmerror = brcmf_sdiod_set_sbaddr_window(sdiodev, address);
+		if (bcmerror)
 			break;
 
 		brcmf_dbg(SDIO, "%s %d bytes at offset 0x%08x in window 0x%08x\n",
@@ -703,17 +838,11 @@
 		sdaddr |= SBSDIO_SB_ACCESS_2_4B_FLAG;
 
 		skb_put(pkt, dsize);
-
-		if (write) {
+		if (write)
 			memcpy(pkt->data, data, dsize);
-			err = brcmf_sdiod_skbuff_write(sdiodev, sdiodev->func1,
-						       sdaddr, pkt);
-		} else {
-			err = brcmf_sdiod_skbuff_read(sdiodev, sdiodev->func1,
-						      sdaddr, pkt);
-		}
-
-		if (err) {
+		bcmerror = brcmf_sdiod_buffrw(sdiodev, SDIO_FUNC_1, write,
+					      sdaddr, pkt);
+		if (bcmerror) {
 			brcmf_err("membytes transfer failed\n");
 			break;
 		}
@@ -733,17 +862,24 @@
 
 	dev_kfree_skb(pkt);
 
-	sdio_release_host(sdiodev->func1);
+	/* Return the window to backplane enumeration space for core access */
+	if (brcmf_sdiod_set_sbaddr_window(sdiodev, sdiodev->sbwad))
+		brcmf_err("FAILED to set window back to 0x%x\n",
+			  sdiodev->sbwad);
 
-	return err;
+	sdio_release_host(sdiodev->func[1]);
+
+	return bcmerror;
 }
 
-int brcmf_sdiod_abort(struct brcmf_sdio_dev *sdiodev, struct sdio_func *func)
+int brcmf_sdiod_abort(struct brcmf_sdio_dev *sdiodev, uint fn)
 {
+	char t_func = (char)fn;
 	brcmf_dbg(SDIO, "Enter\n");
 
-	/* Issue abort cmd52 command through F0 */
-	brcmf_sdiod_func0_wb(sdiodev, SDIO_CCCR_ABORT, func->num, NULL);
+	/* issue abort cmd52 command through F0 */
+	brcmf_sdiod_request_data(sdiodev, SDIO_FUNC_0, SDIO_CCCR_ABORT,
+				 sizeof(t_func), &t_func, true);
 
 	brcmf_dbg(SDIO, "Exit\n");
 	return 0;
@@ -757,7 +893,7 @@
 	uint nents;
 	int err;
 
-	func = sdiodev->func2;
+	func = sdiodev->func[2];
 	host = func->card->host;
 	sdiodev->sg_support = host->max_segs > 1;
 	max_blocks = min_t(uint, host->max_blk_count, 511u);
@@ -818,17 +954,17 @@
 	brcmf_sdio_trigger_dpc(sdiodev->bus);
 	wait_event(sdiodev->freezer->thread_freeze,
 		   atomic_read(expect) == sdiodev->freezer->frozen_count);
-	sdio_claim_host(sdiodev->func1);
+	sdio_claim_host(sdiodev->func[1]);
 	res = brcmf_sdio_sleep(sdiodev->bus, true);
-	sdio_release_host(sdiodev->func1);
+	sdio_release_host(sdiodev->func[1]);
 	return res;
 }
 
 static void brcmf_sdiod_freezer_off(struct brcmf_sdio_dev *sdiodev)
 {
-	sdio_claim_host(sdiodev->func1);
+	sdio_claim_host(sdiodev->func[1]);
 	brcmf_sdio_sleep(sdiodev->bus, false);
-	sdio_release_host(sdiodev->func1);
+	sdio_release_host(sdiodev->func[1]);
 	atomic_set(&sdiodev->freezer->freezing, 0);
 	complete_all(&sdiodev->freezer->resumed);
 }
@@ -867,7 +1003,7 @@
 }
 #endif /* CONFIG_PM_SLEEP */
 
-static int brcmf_sdiod_remove(struct brcmf_sdio_dev *sdiodev)
+int brcmf_sdiod_remove(struct brcmf_sdio_dev *sdiodev)
 {
 	sdiodev->state = BRCMF_SDIOD_DOWN;
 	if (sdiodev->bus) {
@@ -878,19 +1014,19 @@
 	brcmf_sdiod_freezer_detach(sdiodev);
 
 	/* Disable Function 2 */
-	sdio_claim_host(sdiodev->func2);
-	sdio_disable_func(sdiodev->func2);
-	sdio_release_host(sdiodev->func2);
+	sdio_claim_host(sdiodev->func[2]);
+	sdio_disable_func(sdiodev->func[2]);
+	sdio_release_host(sdiodev->func[2]);
 
 	/* Disable Function 1 */
-	sdio_claim_host(sdiodev->func1);
-	sdio_disable_func(sdiodev->func1);
-	sdio_release_host(sdiodev->func1);
+	sdio_claim_host(sdiodev->func[1]);
+	sdio_disable_func(sdiodev->func[1]);
+	sdio_release_host(sdiodev->func[1]);
 
 	sg_free_table(&sdiodev->sgtable);
 	sdiodev->sbwad = 0;
 
-	pm_runtime_allow(sdiodev->func1->card->host->parent);
+	pm_runtime_allow(sdiodev->func[1]->card->host->parent);
 	return 0;
 }
 
@@ -902,31 +1038,51 @@
 	host->caps |= MMC_CAP_NONREMOVABLE;
 }
 
-static int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev)
+int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev)
 {
 	int ret = 0;
+	unsigned int f2_blksz = SDIO_FUNC2_BLOCKSIZE;
 
-	sdio_claim_host(sdiodev->func1);
+	sdiodev->num_funcs = 2;
 
-	ret = sdio_set_block_size(sdiodev->func1, SDIO_FUNC1_BLOCKSIZE);
+	sdio_claim_host(sdiodev->func[1]);
+
+	ret = sdio_set_block_size(sdiodev->func[1], SDIO_FUNC1_BLOCKSIZE);
 	if (ret) {
 		brcmf_err("Failed to set F1 blocksize\n");
-		sdio_release_host(sdiodev->func1);
+		sdio_release_host(sdiodev->func[1]);
 		goto out;
 	}
-	ret = sdio_set_block_size(sdiodev->func2, SDIO_FUNC2_BLOCKSIZE);
+
+	switch (sdiodev->func[0]->device) {
+	case SDIO_DEVICE_ID_CYPRESS_4373:
+		f2_blksz = SDIO_4373_FUNC2_BLOCKSIZE;
+		break;
+	case SDIO_DEVICE_ID_BROADCOM_4359:
+	case SDIO_DEVICE_ID_CYPRESS_89359:
+	case SDIO_DEVICE_ID_BROADCOM_4354:
+	case SDIO_DEVICE_ID_BROADCOM_4356:
+		f2_blksz = SDIO_435X_FUNC2_BLOCKSIZE;
+		break;
+	default:
+		break;
+	}
+
+	ret = sdio_set_block_size(sdiodev->func[2], f2_blksz);
 	if (ret) {
 		brcmf_err("Failed to set F2 blocksize\n");
-		sdio_release_host(sdiodev->func1);
+		sdio_release_host(sdiodev->func[1]);
 		goto out;
+	} else {
+		brcmf_dbg(SDIO, "set F2 blocksize to %d\n", f2_blksz);
 	}
 
 	/* increase F2 timeout */
-	sdiodev->func2->enable_timeout = SDIO_WAIT_F2RDY;
+	sdiodev->func[2]->enable_timeout = SDIO_WAIT_F2RDY;
 
 	/* Enable Function 1 */
-	ret = sdio_enable_func(sdiodev->func1);
-	sdio_release_host(sdiodev->func1);
+	ret = sdio_enable_func(sdiodev->func[1]);
+	sdio_release_host(sdiodev->func[1]);
 	if (ret) {
 		brcmf_err("Failed to enable F1: err=%d\n", ret);
 		goto out;
@@ -942,7 +1098,7 @@
 		ret = -ENODEV;
 		goto out;
 	}
-	brcmf_sdiod_host_fixup(sdiodev->func2->card->host);
+	brcmf_sdiod_host_fixup(sdiodev->func[2]->card->host);
 out:
 	if (ret)
 		brcmf_sdiod_remove(sdiodev);
@@ -971,7 +1127,10 @@
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_43455),
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4354),
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4356),
+	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4359),
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_CYPRESS_4373),
+	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_CYPRESS_43012),
+	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_CYPRESS_89359),
 	{ /* end: all zeroes */ }
 };
 MODULE_DEVICE_TABLE(sdio, brcmf_sdmmc_ids);
@@ -1004,10 +1163,6 @@
 	brcmf_dbg(SDIO, "Function#: %d\n", func->num);
 
 	dev = &func->dev;
-
-	/* Set MMC_QUIRK_LENIENT_FN0 for this card */
-	func->card->quirks |= MMC_QUIRK_LENIENT_FN0;
-
 	/* prohibit ACPI power management for this device */
 	brcmf_sdiod_acpi_set_power_manageable(dev, 0);
 
@@ -1031,15 +1186,18 @@
 	/* store refs to functions used. mmc_card does
 	 * not hold the F0 function pointer.
 	 */
-	sdiodev->func1 = func->card->sdio_func[0];
-	sdiodev->func2 = func;
+	sdiodev->func[0] = kmemdup(func, sizeof(*func), GFP_KERNEL);
+	sdiodev->func[0]->num = 0;
+	sdiodev->func[1] = func->card->sdio_func[0];
+	sdiodev->func[2] = func;
 
 	sdiodev->bus_if = bus_if;
 	bus_if->bus_priv.sdio = sdiodev;
 	bus_if->proto_type = BRCMF_PROTO_BCDC;
 	dev_set_drvdata(&func->dev, bus_if);
-	dev_set_drvdata(&sdiodev->func1->dev, bus_if);
-	sdiodev->dev = &sdiodev->func1->dev;
+	dev_set_drvdata(&sdiodev->func[1]->dev, bus_if);
+	sdiodev->dev = &sdiodev->func[1]->dev;
+	dev_set_drvdata(&sdiodev->func[2]->dev, bus_if);
 
 	brcmf_sdiod_change_state(sdiodev, BRCMF_SDIOD_DOWN);
 
@@ -1055,7 +1213,9 @@
 
 fail:
 	dev_set_drvdata(&func->dev, NULL);
-	dev_set_drvdata(&sdiodev->func1->dev, NULL);
+	dev_set_drvdata(&sdiodev->func[1]->dev, NULL);
+	dev_set_drvdata(&sdiodev->func[2]->dev, NULL);
+	kfree(sdiodev->func[0]);
 	kfree(sdiodev);
 	kfree(bus_if);
 	return err;
@@ -1084,10 +1244,11 @@
 		/* only proceed with rest of cleanup if func 1 */
 		brcmf_sdiod_remove(sdiodev);
 
-		dev_set_drvdata(&sdiodev->func1->dev, NULL);
-		dev_set_drvdata(&sdiodev->func2->dev, NULL);
+		dev_set_drvdata(&sdiodev->func[1]->dev, NULL);
+		dev_set_drvdata(&sdiodev->func[2]->dev, NULL);
 
 		kfree(bus_if);
+		kfree(sdiodev->func[0]);
 		kfree(sdiodev);
 	}
 
@@ -1110,14 +1271,26 @@
 	struct brcmf_bus *bus_if;
 	struct brcmf_sdio_dev *sdiodev;
 	mmc_pm_flag_t sdio_flags;
+	struct brcmf_cfg80211_info *config;
+	int retry = BRCMF_PM_WAIT_MAXRETRY;
 
 	func = container_of(dev, struct sdio_func, dev);
+	bus_if = dev_get_drvdata(dev);
+	config = bus_if->drvr->config;
+
 	brcmf_dbg(SDIO, "Enter: F%d\n", func->num);
-	if (func->num != 1)
-		return 0;
 
+	while (retry &&
+	       config->pm_state == BRCMF_CFG80211_PM_STATE_SUSPENDING) {
+		usleep_range(10000, 20000);
+		retry--;
+	}
+	if (!retry && config->pm_state == BRCMF_CFG80211_PM_STATE_SUSPENDING)
+		brcmf_err("timed out wait for cfg80211 suspended\n");
+
+	if (func->num != SDIO_FUNC_1)
+		return 0;
 
-	bus_if = dev_get_drvdata(dev);
 	sdiodev = bus_if->bus_priv.sdio;
 
 	brcmf_sdiod_freezer_on(sdiodev);
@@ -1130,7 +1303,7 @@
 		else
 			sdio_flags |= MMC_PM_WAKE_SDIO_IRQ;
 	}
-	if (sdio_set_host_pm_flags(sdiodev->func1, sdio_flags))
+	if (sdio_set_host_pm_flags(sdiodev->func[1], sdio_flags))
 		brcmf_err("Failed to set pm_flags %x\n", sdio_flags);
 	return 0;
 }
@@ -1142,7 +1315,7 @@
 	struct sdio_func *func = container_of(dev, struct sdio_func, dev);
 
 	brcmf_dbg(SDIO, "Enter: F%d\n", func->num);
-	if (func->num != 2)
+	if (func->num != SDIO_FUNC_2)
 		return 0;
 
 	brcmf_sdiod_freezer_off(sdiodev);
@@ -1165,7 +1338,6 @@
 #ifdef CONFIG_PM_SLEEP
 		.pm = &brcmf_sdio_pm_ops,
 #endif	/* CONFIG_PM_SLEEP */
-		.coredump = brcmf_dev_coredump,
 	},
 };
 
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/btcoex.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/btcoex.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/btcoex.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/btcoex.c	2020-06-19 21:00:31.416227715 -0400
@@ -462,7 +462,7 @@
 int brcmf_btcoex_set_mode(struct brcmf_cfg80211_vif *vif,
 			  enum brcmf_btcoex_mode mode, u16 duration)
 {
-	struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(vif->wdev.wiphy);
+	struct brcmf_cfg80211_info *cfg = wiphy_priv(vif->wdev.wiphy);
 	struct brcmf_btcoex_info *btci = cfg->btcoex;
 	struct brcmf_if *ifp = brcmf_get_ifp(cfg->pub, 0);
 
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h	2020-06-19 21:00:31.416227715 -0400
@@ -88,8 +88,9 @@
 	void (*wowl_config)(struct device *dev, bool enabled);
 	size_t (*get_ramsize)(struct device *dev);
 	int (*get_memdump)(struct device *dev, void *data, size_t len);
-	int (*get_fwname)(struct device *dev, const char *ext,
+	int (*get_fwname)(struct device *dev, uint chip, uint chiprev,
 			  unsigned char *fw_name);
+	int (*reset)(struct device *dev);
 };
 
 
@@ -140,7 +141,6 @@
  * @always_use_fws_queue: bus wants use queue also when fwsignal is inactive.
  * @wowl_supported: is wowl supported by bus driver.
  * @chiprev: revision of the dongle chip.
- * @msgbuf: msgbuf protocol parameters provided by bus layer.
  */
 struct brcmf_bus {
 	union {
@@ -161,6 +161,7 @@
 
 	const struct brcmf_bus_ops *ops;
 	struct brcmf_bus_msgbuf *msgbuf;
+	bool allow_skborphan;
 };
 
 /*
@@ -229,10 +230,19 @@
 }
 
 static inline
-int brcmf_bus_get_fwname(struct brcmf_bus *bus, const char *ext,
+int brcmf_bus_get_fwname(struct brcmf_bus *bus, uint chip, uint chiprev,
 			 unsigned char *fw_name)
 {
-	return bus->ops->get_fwname(bus->dev, ext, fw_name);
+	return bus->ops->get_fwname(bus->dev, chip, chiprev, fw_name);
+}
+
+static inline
+int brcmf_bus_reset(struct brcmf_bus *bus)
+{
+	if (!bus->ops->reset)
+		return -EOPNOTSUPP;
+
+	return bus->ops->reset(bus->dev);
 }
 
 /*
@@ -252,12 +262,16 @@
 void brcmf_dev_reset(struct device *dev);
 /* Request from bus module to initiate a coredump */
 void brcmf_dev_coredump(struct device *dev);
+/* Indication that firmware has halted or crashed */
+void brcmf_fw_crashed(struct device *dev);
 
 /* Configure the "global" bus state used by upper layers */
 void brcmf_bus_change_state(struct brcmf_bus *bus, enum brcmf_bus_state state);
 
+int brcmf_bus_started(struct device *dev);
 s32 brcmf_iovar_data_set(struct device *dev, char *name, void *data, u32 len);
 void brcmf_bus_add_txhdrlen(struct device *dev, uint len);
+int brcmf_fwlog_attach(struct device *dev);
 
 #ifdef CONFIG_BRCMFMAC_SDIO
 void brcmf_sdio_exit(void);
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c	2020-06-19 21:00:31.416227715 -0400
@@ -33,6 +33,7 @@
 #include "p2p.h"
 #include "btcoex.h"
 #include "pno.h"
+#include "fwsignal.h"
 #include "cfg80211.h"
 #include "feature.h"
 #include "fwil.h"
@@ -65,6 +66,7 @@
 #define RSN_AKM_PSK			2	/* Pre-shared Key */
 #define RSN_AKM_SHA256_1X		5	/* SHA256, 802.1X */
 #define RSN_AKM_SHA256_PSK		6	/* SHA256, Pre-shared Key */
+#define RSN_AKM_SAE			8	/* SAE */
 #define RSN_CAP_LEN			2	/* Length of RSN capabilities */
 #define RSN_CAP_PTK_REPLAY_CNTR_MASK	(BIT(2) | BIT(3))
 #define RSN_CAP_MFPR_MASK		BIT(6)
@@ -239,6 +241,9 @@
 	struct parsed_vndr_ie_info ie_info[VNDR_IE_PARSE_LIMIT];
 };
 
+#define WLC_E_IF_ROLE_STA		0	/* Infra STA */
+#define WLC_E_IF_ROLE_AP		1	/* Access Point */
+
 static u8 nl80211_band_to_fwil(enum nl80211_band band)
 {
 	switch (band) {
@@ -472,6 +477,47 @@
 	return err;
 }
 
+static s32
+brcmf_configure_arp_nd_offload(struct brcmf_if *ifp, bool enable)
+{
+	s32 err;
+	u32 mode;
+
+	if (enable)
+		mode = BRCMF_ARP_OL_AGENT | BRCMF_ARP_OL_PEER_AUTO_REPLY;
+	else
+		mode = 0;
+
+	/* Try to set and enable ARP offload feature, this may fail, then it  */
+	/* is simply not supported and err 0 will be returned                 */
+	err = brcmf_fil_iovar_int_set(ifp, "arp_ol", mode);
+	if (err) {
+		brcmf_dbg(TRACE, "failed to set ARP offload mode to 0x%x, err = %d\n",
+			  mode, err);
+		err = 0;
+	} else {
+		err = brcmf_fil_iovar_int_set(ifp, "arpoe", enable);
+		if (err) {
+			brcmf_dbg(TRACE, "failed to configure (%d) ARP offload err = %d\n",
+				  enable, err);
+			err = 0;
+		} else
+			brcmf_dbg(TRACE, "successfully configured (%d) ARP offload to 0x%x\n",
+				  enable, mode);
+	}
+
+	err = brcmf_fil_iovar_int_set(ifp, "ndoe", enable);
+	if (err) {
+		brcmf_dbg(TRACE, "failed to configure (%d) ND offload err = %d\n",
+			  enable, err);
+		err = 0;
+	} else
+		brcmf_dbg(TRACE, "successfully configured (%d) ND offload to 0x%x\n",
+			  enable, mode);
+
+	return err;
+}
+
 static void
 brcmf_cfg80211_update_proto_addr_mode(struct wireless_dev *wdev)
 {
@@ -753,7 +799,7 @@
 static int brcmf_cfg80211_del_ap_iface(struct wiphy *wiphy,
 				       struct wireless_dev *wdev)
 {
-	struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy);
+	struct brcmf_cfg80211_info *cfg = wiphy_priv(wiphy);
 	struct net_device *ndev = wdev->netdev;
 	struct brcmf_if *ifp = netdev_priv(ndev);
 	int ret;
@@ -786,7 +832,7 @@
 static
 int brcmf_cfg80211_del_iface(struct wiphy *wiphy, struct wireless_dev *wdev)
 {
-	struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy);
+	struct brcmf_cfg80211_info *cfg = wiphy_priv(wiphy);
 	struct net_device *ndev = wdev->netdev;
 
 	if (ndev && ndev == cfg_to_ndev(cfg))
@@ -831,7 +877,7 @@
 			 enum nl80211_iftype type,
 			 struct vif_params *params)
 {
-	struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy);
+	struct brcmf_cfg80211_info *cfg = wiphy_priv(wiphy);
 	struct brcmf_if *ifp = netdev_priv(ndev);
 	struct brcmf_cfg80211_vif *vif = ifp->vif;
 	s32 infra = 0;
@@ -1043,6 +1089,7 @@
 {
 	struct brcmf_cfg80211_info *cfg = ifp->drvr->config;
 	s32 err;
+	u32 passive_scan;
 	struct brcmf_scan_results *results;
 	struct escan_info *escan = &cfg->escan_info;
 
@@ -1050,7 +1097,13 @@
 	escan->ifp = ifp;
 	escan->wiphy = cfg->wiphy;
 	escan->escan_state = WL_ESCAN_STATE_SCANNING;
-
+	passive_scan = cfg->active_scan ? 0 : 1;
+	err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_PASSIVE_SCAN,
+				    passive_scan);
+	if (err) {
+		brcmf_err("error (%d)\n", err);
+		return err;
+	}
 	brcmf_scan_config_mpc(ifp, 0);
 	results = (struct brcmf_scan_results *)cfg->escan_info.escan_buf;
 	results->version = 0;
@@ -1064,16 +1117,21 @@
 }
 
 static s32
-brcmf_cfg80211_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request)
+brcmf_cfg80211_escan(struct wiphy *wiphy, struct brcmf_cfg80211_vif *vif,
+		     struct cfg80211_scan_request *request,
+		     struct cfg80211_ssid *this_ssid)
 {
+	struct brcmf_if *ifp = vif->ifp;
 	struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy);
-	struct brcmf_cfg80211_vif *vif;
-	s32 err = 0;
+	struct cfg80211_ssid *ssids;
+	u32 passive_scan;
+	bool escan_req;
+	bool spec_scan;
+	s32 err;
+	struct brcmf_ssid_le ssid_le;
+	u32 SSID_len;
 
-	brcmf_dbg(TRACE, "Enter\n");
-	vif = container_of(request->wdev, struct brcmf_cfg80211_vif, wdev);
-	if (!check_vif_up(vif))
-		return -EIO;
+	brcmf_dbg(SCAN, "START ESCAN\n");
 
 	if (test_bit(BRCMF_SCAN_STATUS_BUSY, &cfg->scan_status)) {
 		brcmf_err("Scanning already: status (%lu)\n", cfg->scan_status);
@@ -1089,8 +1147,8 @@
 			  cfg->scan_status);
 		return -EAGAIN;
 	}
-	if (test_bit(BRCMF_VIF_STATUS_CONNECTING, &vif->sme_state)) {
-		brcmf_err("Connecting: status (%lu)\n", vif->sme_state);
+	if (test_bit(BRCMF_VIF_STATUS_CONNECTING, &ifp->vif->sme_state)) {
+		brcmf_err("Connecting: status (%lu)\n", ifp->vif->sme_state);
 		return -EAGAIN;
 	}
 
@@ -1098,38 +1156,96 @@
 	if (vif == cfg->p2p.bss_idx[P2PAPI_BSSCFG_DEVICE].vif)
 		vif = cfg->p2p.bss_idx[P2PAPI_BSSCFG_PRIMARY].vif;
 
-	brcmf_dbg(SCAN, "START ESCAN\n");
+	escan_req = false;
+	if (request) {
+		/* scan bss */
+		ssids = request->ssids;
+		escan_req = true;
+	} else {
+		/* scan in ibss */
+		/* we don't do escan in ibss */
+		ssids = this_ssid;
+	}
 
 	cfg->scan_request = request;
 	set_bit(BRCMF_SCAN_STATUS_BUSY, &cfg->scan_status);
+	if (escan_req) {
+		cfg->escan_info.run = brcmf_run_escan;
+		err = brcmf_p2p_scan_prep(wiphy, request, vif);
+		if (err)
+			goto scan_out;
 
-	cfg->escan_info.run = brcmf_run_escan;
-	err = brcmf_p2p_scan_prep(wiphy, request, vif);
-	if (err)
-		goto scan_out;
+		err = brcmf_do_escan(vif->ifp, request);
+		if (err)
+			goto scan_out;
+	} else {
+		brcmf_dbg(SCAN, "ssid \"%s\", ssid_len (%d)\n",
+			  ssids->ssid, ssids->ssid_len);
+		memset(&ssid_le, 0, sizeof(ssid_le));
+		SSID_len = min_t(u8, sizeof(ssid_le.SSID), ssids->ssid_len);
+		ssid_le.SSID_len = cpu_to_le32(0);
+		spec_scan = false;
+		if (SSID_len) {
+			memcpy(ssid_le.SSID, ssids->ssid, SSID_len);
+			ssid_le.SSID_len = cpu_to_le32(SSID_len);
+			spec_scan = true;
+		} else
+			brcmf_dbg(SCAN, "Broadcast scan\n");
 
-	err = brcmf_vif_set_mgmt_ie(vif, BRCMF_VNDR_IE_PRBREQ_FLAG,
-				    request->ie, request->ie_len);
-	if (err)
-		goto scan_out;
+		passive_scan = cfg->active_scan ? 0 : 1;
+		err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_PASSIVE_SCAN,
+					    passive_scan);
+		if (err) {
+			brcmf_err("WLC_SET_PASSIVE_SCAN error (%d)\n", err);
+			goto scan_out;
+		}
+		brcmf_scan_config_mpc(ifp, 0);
+		err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SCAN, &ssid_le,
+					     sizeof(ssid_le));
+		if (err) {
+			if (err == -EBUSY)
+				brcmf_dbg(INFO, "BUSY: scan for \"%s\" canceled\n",
+					  ssid_le.SSID);
+			else
+				brcmf_err("WLC_SCAN error (%d)\n", err);
 
-	err = brcmf_do_escan(vif->ifp, request);
-	if (err)
-		goto scan_out;
+			brcmf_scan_config_mpc(ifp, 1);
+			goto scan_out;
+		}
+	}
 
 	/* Arm scan timeout timer */
-	mod_timer(&cfg->escan_timeout,
-		  jiffies + msecs_to_jiffies(BRCMF_ESCAN_TIMER_INTERVAL_MS));
+	mod_timer(&cfg->escan_timeout, jiffies +
+			BRCMF_ESCAN_TIMER_INTERVAL_MS * HZ / 1000);
 
 	return 0;
 
 scan_out:
-	brcmf_err("scan error (%d)\n", err);
 	clear_bit(BRCMF_SCAN_STATUS_BUSY, &cfg->scan_status);
 	cfg->scan_request = NULL;
 	return err;
 }
 
+static s32
+brcmf_cfg80211_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request)
+{
+	struct brcmf_cfg80211_vif *vif;
+	s32 err = 0;
+
+	brcmf_dbg(TRACE, "Enter\n");
+	vif = container_of(request->wdev, struct brcmf_cfg80211_vif, wdev);
+	if (!check_vif_up(vif))
+		return -EIO;
+
+	err = brcmf_cfg80211_escan(wiphy, vif, request, NULL);
+
+	if (err)
+		brcmf_err("scan error (%d)\n", err);
+
+	brcmf_dbg(TRACE, "Exit\n");
+	return err;
+}
+
 static s32 brcmf_set_rts(struct net_device *ndev, u32 rts_threshold)
 {
 	s32 err = 0;
@@ -1256,7 +1372,32 @@
 	return err;
 }
 
-static void brcmf_link_down(struct brcmf_cfg80211_vif *vif, u16 reason)
+static int brcmf_set_sae_password(struct brcmf_if *ifp, const u8 *pwd_data,
+				  u16 pwd_len)
+{
+	struct brcmf_wsec_sae_pwd_le sae_pwd;
+	int err;
+
+	if (pwd_len > BRCMF_WSEC_MAX_SAE_PASSWORD_LEN) {
+		brcmf_err("sae_password must be less than %d\n",
+			  BRCMF_WSEC_MAX_SAE_PASSWORD_LEN);
+		return -EINVAL;
+	}
+
+	sae_pwd.key_len = cpu_to_le16(pwd_len);
+	memcpy(sae_pwd.key, pwd_data, pwd_len);
+
+	err = brcmf_fil_iovar_data_set(ifp, "sae_password", &sae_pwd,
+				       sizeof(sae_pwd));
+	if (err < 0)
+		brcmf_err("failed to set SAE password in firmware (len=%u)\n",
+			  pwd_len);
+
+	return err;
+}
+
+static void brcmf_link_down(struct brcmf_cfg80211_vif *vif, u16 reason,
+			    bool locally_generated)
 {
 	struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(vif->wdev.wiphy);
 	s32 err = 0;
@@ -1264,7 +1405,7 @@
 	brcmf_dbg(TRACE, "Enter\n");
 
 	if (test_and_clear_bit(BRCMF_VIF_STATUS_CONNECTED, &vif->sme_state)) {
-		brcmf_dbg(INFO, "Call WLC_DISASSOC to stop excess roaming\n");
+		brcmf_dbg(INFO, "Call WLC_DISASSOC to stop excess roaming\n ");
 		err = brcmf_fil_cmd_data_set(vif->ifp,
 					     BRCMF_C_DISASSOC, NULL, 0);
 		if (err) {
@@ -1273,7 +1414,7 @@
 		if ((vif->wdev.iftype == NL80211_IFTYPE_STATION) ||
 		    (vif->wdev.iftype == NL80211_IFTYPE_P2P_CLIENT))
 			cfg80211_disconnected(vif->wdev.netdev, reason, NULL, 0,
-					      true, GFP_KERNEL);
+					      locally_generated, GFP_KERNEL);
 	}
 	clear_bit(BRCMF_VIF_STATUS_CONNECTING, &vif->sme_state);
 	clear_bit(BRCMF_SCAN_STATUS_SUPPRESS, &cfg->scan_status);
@@ -1450,7 +1591,7 @@
 		return 0;
 	}
 
-	brcmf_link_down(ifp->vif, WLAN_REASON_DEAUTH_LEAVING);
+	brcmf_link_down(ifp->vif, WLAN_REASON_DEAUTH_LEAVING, true);
 	brcmf_net_setcarrier(ifp, false);
 
 	brcmf_dbg(TRACE, "Exit\n");
@@ -1470,6 +1611,8 @@
 		val = WPA_AUTH_PSK | WPA_AUTH_UNSPECIFIED;
 	else if (sme->crypto.wpa_versions & NL80211_WPA_VERSION_2)
 		val = WPA2_AUTH_PSK | WPA2_AUTH_UNSPECIFIED;
+	else if (sme->crypto.wpa_versions & NL80211_WPA_VERSION_3)
+		val = WPA3_AUTH_SAE_PSK;
 	else
 		val = WPA_AUTH_DISABLED;
 	brcmf_dbg(CONN, "setting wpa_auth to 0x%0x\n", val);
@@ -1500,6 +1643,10 @@
 		val = 1;
 		brcmf_dbg(CONN, "shared key\n");
 		break;
+	case NL80211_AUTHTYPE_SAE:
+		val = 3;
+		brcmf_dbg(CONN, "SAE authentication\n");
+		break;
 	default:
 		val = 2;
 		brcmf_dbg(CONN, "automatic, auth type (%d)\n", sme->auth_type);
@@ -1598,6 +1745,7 @@
 	struct brcmf_cfg80211_profile *profile = &ifp->vif->profile;
 	s32 val;
 	s32 err;
+	s32 okc_enable;
 	const struct brcmf_tlv *rsn_ie;
 	const u8 *ie;
 	u32 ie_len;
@@ -1607,6 +1755,8 @@
 	u16 count;
 
 	profile->use_fwsup = BRCMF_PROFILE_FWSUP_NONE;
+	profile->is_ft = false;
+	profile->is_okc = false;
 
 	if (!sme->crypto.n_akm_suites)
 		return 0;
@@ -1649,6 +1799,26 @@
 		case WLAN_AKM_SUITE_PSK:
 			val = WPA2_AUTH_PSK;
 			break;
+		case WLAN_AKM_SUITE_FT_8021X:
+			val = WPA2_AUTH_UNSPECIFIED | WPA2_AUTH_FT;
+			profile->is_ft = true;
+			if (sme->want_1x)
+				profile->use_fwsup = BRCMF_PROFILE_FWSUP_1X;
+			break;
+		case WLAN_AKM_SUITE_FT_PSK:
+			val = WPA2_AUTH_PSK | WPA2_AUTH_FT;
+			profile->is_ft = true;
+			break;
+		default:
+			brcmf_err("invalid cipher group (%d)\n",
+				  sme->crypto.cipher_group);
+			return -EINVAL;
+		}
+	} else if (val & WPA3_AUTH_SAE_PSK) {
+		switch (sme->crypto.akm_suites[0]) {
+		case WLAN_AKM_SUITE_SAE:
+			val = WPA3_AUTH_SAE_PSK;
+			break;
 		default:
 			brcmf_err("invalid cipher group (%d)\n",
 				  sme->crypto.cipher_group);
@@ -1656,8 +1826,17 @@
 		}
 	}
 
-	if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X)
+	if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X) {
 		brcmf_dbg(INFO, "using 1X offload\n");
+		err = brcmf_fil_bsscfg_int_get(netdev_priv(ndev), "okc_enable",
+					       &okc_enable);
+		if (err) {
+			brcmf_err("get okc_enable failed (%d)\n", err);
+		} else {
+			brcmf_dbg(INFO, "get okc_enable (%d)\n", okc_enable);
+			profile->is_okc = okc_enable;
+		}
+	}
 
 	if (!brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MFP))
 		goto skip_mfp_config;
@@ -1723,7 +1902,8 @@
 	brcmf_dbg(CONN, "wpa_versions 0x%x cipher_pairwise 0x%x\n",
 		  sec->wpa_versions, sec->cipher_pairwise);
 
-	if (sec->wpa_versions & (NL80211_WPA_VERSION_1 | NL80211_WPA_VERSION_2))
+	if (sec->wpa_versions & (NL80211_WPA_VERSION_1 | NL80211_WPA_VERSION_2 |
+				 NL80211_WPA_VERSION_3))
 		return 0;
 
 	if (!(sec->cipher_pairwise &
@@ -1928,7 +2108,13 @@
 		goto done;
 	}
 
-	if (sme->crypto.psk) {
+	if (sme->crypto.sae_pwd) {
+		brcmf_dbg(INFO, "using SAE offload\n");
+		profile->use_fwsup = BRCMF_PROFILE_FWSUP_SAE;
+	}
+
+	if (sme->crypto.psk &&
+	    profile->use_fwsup != BRCMF_PROFILE_FWSUP_SAE) {
 		if (WARN_ON(profile->use_fwsup != BRCMF_PROFILE_FWSUP_NONE)) {
 			err = -EINVAL;
 			goto done;
@@ -1946,12 +2132,23 @@
 		}
 	}
 
-	if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_PSK) {
+	if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_PSK)
 		err = brcmf_set_pmk(ifp, sme->crypto.psk,
 				    BRCMF_WSEC_MAX_PSK_LEN);
-		if (err)
+	else if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_SAE) {
+		/* clean up user-space RSNE */
+		if (brcmf_fil_iovar_data_set(ifp, "wpaie", NULL, 0)) {
+			brcmf_err("failed to clean up user-space RSNE\n");
 			goto done;
+		}
+		err = brcmf_set_sae_password(ifp, sme->crypto.sae_pwd,
+					     sme->crypto.sae_pwd_len);
+		if (!err && sme->crypto.psk)
+			err = brcmf_set_pmk(ifp, sme->crypto.psk,
+					    BRCMF_WSEC_MAX_PSK_LEN);
 	}
+	if (err)
+		goto done;
 
 	/* Join with specific BSSID and cached SSID
 	 * If SSID is zero join based on BSSID only
@@ -2127,15 +2324,17 @@
 brcmf_cfg80211_get_tx_power(struct wiphy *wiphy, struct wireless_dev *wdev,
 			    s32 *dbm)
 {
-	struct brcmf_cfg80211_vif *vif = wdev_to_vif(wdev);
+	struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy);
+	struct net_device *ndev = cfg_to_ndev(cfg);
+	struct brcmf_if *ifp = netdev_priv(ndev);
 	s32 qdbm = 0;
 	s32 err;
 
 	brcmf_dbg(TRACE, "Enter\n");
-	if (!check_vif_up(vif))
+	if (!check_vif_up(ifp->vif))
 		return -EIO;
 
-	err = brcmf_fil_iovar_int_get(vif->ifp, "qtxpower", &qdbm);
+	err = brcmf_fil_iovar_int_get(ifp, "qtxpower", &qdbm);
 	if (err) {
 		brcmf_err("error (%d)\n", err);
 		goto done;
@@ -2267,6 +2466,17 @@
 	if (!ext_key)
 		key->flags = BRCMF_PRIMARY_KEY;
 
+	if (params->seq && params->seq_len == 6) {
+		/* rx iv */
+		u8 *ivptr;
+
+		ivptr = (u8 *)params->seq;
+		key->rxiv.hi = (ivptr[5] << 24) | (ivptr[4] << 16) |
+			(ivptr[3] << 8) | ivptr[2];
+		key->rxiv.lo = (ivptr[1] << 8) | ivptr[0];
+		key->iv_initialized = true;
+	}
+
 	switch (params->cipher) {
 	case WLAN_CIPHER_SUITE_WEP40:
 		key->algo = CRYPTO_ALGO_WEP1;
@@ -2434,7 +2644,7 @@
 	struct nl80211_sta_flag_update *sfu;
 
 	brcmf_dbg(TRACE, "flags %08x\n", fw_sta_flags);
-	si->filled |= BIT_ULL(NL80211_STA_INFO_STA_FLAGS);
+	si->filled |= BIT(NL80211_STA_INFO_STA_FLAGS);
 	sfu = &si->sta_flags;
 	sfu->mask = BIT(NL80211_STA_FLAG_WME) |
 		    BIT(NL80211_STA_FLAG_AUTHENTICATED) |
@@ -2470,7 +2680,7 @@
 		brcmf_err("Failed to get bss info (%d)\n", err);
 		goto out_kfree;
 	}
-	si->filled |= BIT_ULL(NL80211_STA_INFO_BSS_PARAM);
+	si->filled |= BIT(NL80211_STA_INFO_BSS_PARAM);
 	si->bss_param.beacon_interval = le16_to_cpu(buf->bss_le.beacon_period);
 	si->bss_param.dtim_period = buf->bss_le.dtim_period;
 	capability = le16_to_cpu(buf->bss_le.capability);
@@ -2501,7 +2711,7 @@
 		brcmf_err("BRCMF_C_GET_RATE error (%d)\n", err);
 		return err;
 	}
-	sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE);
+	sinfo->filled |= BIT(NL80211_STA_INFO_TX_BITRATE);
 	sinfo->txrate.legacy = rate * 5;
 
 	memset(&scbval, 0, sizeof(scbval));
@@ -2512,7 +2722,7 @@
 		return err;
 	}
 	rssi = le32_to_cpu(scbval.val);
-	sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL);
+	sinfo->filled |= BIT(NL80211_STA_INFO_SIGNAL);
 	sinfo->signal = rssi;
 
 	err = brcmf_fil_cmd_data_get(ifp, BRCMF_C_GET_GET_PKTCNTS, &pktcnt,
@@ -2521,10 +2731,10 @@
 		brcmf_err("BRCMF_C_GET_GET_PKTCNTS error (%d)\n", err);
 		return err;
 	}
-	sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_PACKETS) |
-			 BIT_ULL(NL80211_STA_INFO_RX_DROP_MISC) |
-			 BIT_ULL(NL80211_STA_INFO_TX_PACKETS) |
-			 BIT_ULL(NL80211_STA_INFO_TX_FAILED);
+	sinfo->filled |= BIT(NL80211_STA_INFO_RX_PACKETS) |
+			 BIT(NL80211_STA_INFO_RX_DROP_MISC) |
+			 BIT(NL80211_STA_INFO_TX_PACKETS) |
+			 BIT(NL80211_STA_INFO_TX_FAILED);
 	sinfo->rx_packets = le32_to_cpu(pktcnt.rx_good_pkt);
 	sinfo->rx_dropped_misc = le32_to_cpu(pktcnt.rx_bad_pkt);
 	sinfo->tx_packets = le32_to_cpu(pktcnt.tx_good_pkt);
@@ -2571,7 +2781,7 @@
 		}
 	}
 	brcmf_dbg(TRACE, "version %d\n", le16_to_cpu(sta_info_le.ver));
-	sinfo->filled = BIT_ULL(NL80211_STA_INFO_INACTIVE_TIME);
+	sinfo->filled = BIT(NL80211_STA_INFO_INACTIVE_TIME);
 	sinfo->inactive_time = le32_to_cpu(sta_info_le.idle) * 1000;
 	sta_flags = le32_to_cpu(sta_info_le.flags);
 	brcmf_convert_sta_flags(sta_flags, sinfo);
@@ -2581,33 +2791,33 @@
 	else
 		sinfo->sta_flags.set &= ~BIT(NL80211_STA_FLAG_TDLS_PEER);
 	if (sta_flags & BRCMF_STA_ASSOC) {
-		sinfo->filled |= BIT_ULL(NL80211_STA_INFO_CONNECTED_TIME);
+		sinfo->filled |= BIT(NL80211_STA_INFO_CONNECTED_TIME);
 		sinfo->connected_time = le32_to_cpu(sta_info_le.in);
 		brcmf_fill_bss_param(ifp, sinfo);
 	}
 	if (sta_flags & BRCMF_STA_SCBSTATS) {
-		sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_FAILED);
+		sinfo->filled |= BIT(NL80211_STA_INFO_TX_FAILED);
 		sinfo->tx_failed = le32_to_cpu(sta_info_le.tx_failures);
-		sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_PACKETS);
+		sinfo->filled |= BIT(NL80211_STA_INFO_TX_PACKETS);
 		sinfo->tx_packets = le32_to_cpu(sta_info_le.tx_pkts);
 		sinfo->tx_packets += le32_to_cpu(sta_info_le.tx_mcast_pkts);
-		sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_PACKETS);
+		sinfo->filled |= BIT(NL80211_STA_INFO_RX_PACKETS);
 		sinfo->rx_packets = le32_to_cpu(sta_info_le.rx_ucast_pkts);
 		sinfo->rx_packets += le32_to_cpu(sta_info_le.rx_mcast_pkts);
 		if (sinfo->tx_packets) {
-			sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE);
+			sinfo->filled |= BIT(NL80211_STA_INFO_TX_BITRATE);
 			sinfo->txrate.legacy =
 				le32_to_cpu(sta_info_le.tx_rate) / 100;
 		}
 		if (sinfo->rx_packets) {
-			sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_BITRATE);
+			sinfo->filled |= BIT(NL80211_STA_INFO_RX_BITRATE);
 			sinfo->rxrate.legacy =
 				le32_to_cpu(sta_info_le.rx_rate) / 100;
 		}
 		if (le16_to_cpu(sta_info_le.ver) >= 4) {
-			sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BYTES);
+			sinfo->filled |= BIT(NL80211_STA_INFO_TX_BYTES);
 			sinfo->tx_bytes = le64_to_cpu(sta_info_le.tx_tot_bytes);
-			sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_BYTES);
+			sinfo->filled |= BIT(NL80211_STA_INFO_RX_BYTES);
 			sinfo->rx_bytes = le64_to_cpu(sta_info_le.rx_tot_bytes);
 		}
 		total_rssi = 0;
@@ -2623,10 +2833,10 @@
 			}
 		}
 		if (count_rssi) {
-			sinfo->filled |= BIT_ULL(NL80211_STA_INFO_CHAIN_SIGNAL);
+			sinfo->filled |= BIT(NL80211_STA_INFO_CHAIN_SIGNAL);
 			sinfo->chains = count_rssi;
 
-			sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL);
+			sinfo->filled |= BIT(NL80211_STA_INFO_SIGNAL);
 			total_rssi /= count_rssi;
 			sinfo->signal = total_rssi;
 		} else if (test_bit(BRCMF_VIF_STATUS_CONNECTED,
@@ -2639,7 +2849,7 @@
 				goto done;
 			} else {
 				rssi = le32_to_cpu(scb_val.val);
-				sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL);
+				sinfo->filled |= BIT(NL80211_STA_INFO_SIGNAL);
 				sinfo->signal = rssi;
 				brcmf_dbg(CONN, "RSSI %d dBm\n", rssi);
 			}
@@ -2704,7 +2914,7 @@
 		goto done;
 	}
 
-	pm = enabled ? PM_FAST : PM_OFF;
+	pm = enabled ? ifp->drvr->settings->default_pm : PM_OFF;
 	/* Do not enable the power save after assoc if it is a p2p interface */
 	if (ifp->vif->wdev.iftype == NL80211_IFTYPE_P2P_CLIENT) {
 		brcmf_dbg(INFO, "Do not enable power save for P2P clients\n");
@@ -2728,8 +2938,9 @@
 				   struct brcmf_bss_info_le *bi)
 {
 	struct wiphy *wiphy = cfg_to_wiphy(cfg);
+	struct ieee80211_channel *notify_channel;
 	struct cfg80211_bss *bss;
-	enum nl80211_band band;
+	struct ieee80211_supported_band *band;
 	struct brcmu_chan ch;
 	u16 channel;
 	u32 freq;
@@ -2737,11 +2948,11 @@
 	u16 notify_interval;
 	u8 *notify_ie;
 	size_t notify_ielen;
-	struct cfg80211_inform_bss bss_data = {};
+	s32 notify_signal;
 
 	if (le32_to_cpu(bi->length) > WL_BSS_INFO_MAX) {
 		brcmf_err("Bss info is larger than buffer. Discarding\n");
-		return 0;
+		return -EINVAL;
 	}
 
 	if (!bi->ctl_ch) {
@@ -2752,33 +2963,32 @@
 	channel = bi->ctl_ch;
 
 	if (channel <= CH_MAX_2G_CHANNEL)
-		band = NL80211_BAND_2GHZ;
+		band = wiphy->bands[NL80211_BAND_2GHZ];
 	else
-		band = NL80211_BAND_5GHZ;
+		band = wiphy->bands[NL80211_BAND_5GHZ];
 
-	freq = ieee80211_channel_to_frequency(channel, band);
-	bss_data.chan = ieee80211_get_channel(wiphy, freq);
-	bss_data.scan_width = NL80211_BSS_CHAN_WIDTH_20;
-	bss_data.boottime_ns = ktime_to_ns(ktime_get_boottime());
+	freq = ieee80211_channel_to_frequency(channel, band->band);
+	notify_channel = ieee80211_get_channel(wiphy, freq);
 
 	notify_capability = le16_to_cpu(bi->capability);
 	notify_interval = le16_to_cpu(bi->beacon_period);
 	notify_ie = (u8 *)bi + le16_to_cpu(bi->ie_offset);
 	notify_ielen = le32_to_cpu(bi->ie_length);
-	bss_data.signal = (s16)le16_to_cpu(bi->RSSI) * 100;
+	notify_signal = (s16)le16_to_cpu(bi->RSSI) * 100;
 
 	brcmf_dbg(CONN, "bssid: %pM\n", bi->BSSID);
 	brcmf_dbg(CONN, "Channel: %d(%d)\n", channel, freq);
 	brcmf_dbg(CONN, "Capability: %X\n", notify_capability);
 	brcmf_dbg(CONN, "Beacon interval: %d\n", notify_interval);
-	brcmf_dbg(CONN, "Signal: %d\n", bss_data.signal);
+	brcmf_dbg(CONN, "Signal: %d\n", notify_signal);
 
-	bss = cfg80211_inform_bss_data(wiphy, &bss_data,
-				       CFG80211_BSS_FTYPE_UNKNOWN,
-				       (const u8 *)bi->BSSID,
-				       0, notify_capability,
-				       notify_interval, notify_ie,
-				       notify_ielen, GFP_KERNEL);
+	bss = cfg80211_inform_bss(wiphy, notify_channel,
+				  CFG80211_BSS_FTYPE_UNKNOWN,
+				  (const u8 *)bi->BSSID,
+				  0, notify_capability,
+				  notify_interval, notify_ie,
+				  notify_ielen, notify_signal,
+				  GFP_KERNEL);
 
 	if (!bss)
 		return -ENOMEM;
@@ -3356,7 +3566,7 @@
 				struct cfg80211_sched_scan_request *req)
 {
 	struct brcmf_if *ifp = netdev_priv(ndev);
-	struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy);
+	struct brcmf_cfg80211_info *cfg = wiphy_priv(wiphy);
 
 	brcmf_dbg(SCAN, "Enter: n_match_sets=%d n_ssids=%d\n",
 		  req->n_match_sets, req->n_ssids);
@@ -3568,10 +3778,24 @@
 	struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy);
 	struct net_device *ndev = cfg_to_ndev(cfg);
 	struct brcmf_if *ifp = netdev_priv(ndev);
+	struct brcmf_pub *drvr = ifp->drvr;
+	struct brcmf_bus *bus_if = drvr->bus_if;
+	struct brcmf_cfg80211_info *config = drvr->config;
+	int retry = BRCMF_PM_WAIT_MAXRETRY;
 
 	brcmf_dbg(TRACE, "Enter\n");
 
+	config->pm_state = BRCMF_CFG80211_PM_STATE_RESUMING;
+
 	if (cfg->wowl.active) {
+		/* wait for bus resumed */
+		while (retry && bus_if->state != BRCMF_BUS_UP) {
+			usleep_range(10000, 20000);
+			retry--;
+		}
+		if (!retry && bus_if->state != BRCMF_BUS_UP)
+			brcmf_err("timed out wait for bus resume\n");
+
 		brcmf_report_wowl_wakeind(wiphy, ifp);
 		brcmf_fil_iovar_int_set(ifp, "wowl_clear", 0);
 		brcmf_config_wowl_pattern(ifp, "clr", NULL, 0, NULL, 0);
@@ -3587,7 +3811,12 @@
 					    brcmf_notify_sched_scan_results);
 			cfg->wowl.nd_enabled = false;
 		}
+
+		/* disable packet filters */
+		brcmf_pktfilter_enable(ifp->ndev, false);
+
 	}
+	config->pm_state = BRCMF_CFG80211_PM_STATE_RESUMED;
 	return 0;
 }
 
@@ -3645,6 +3874,9 @@
 	brcmf_fil_iovar_int_set(ifp, "wowl_activate", 1);
 	brcmf_bus_wowl_config(cfg->pub->bus_if, true);
 	cfg->wowl.active = true;
+
+	/* enable packet filters */
+	brcmf_pktfilter_enable(ifp->ndev, true);
 }
 
 static s32 brcmf_cfg80211_suspend(struct wiphy *wiphy,
@@ -3654,9 +3886,12 @@
 	struct net_device *ndev = cfg_to_ndev(cfg);
 	struct brcmf_if *ifp = netdev_priv(ndev);
 	struct brcmf_cfg80211_vif *vif;
+	struct brcmf_cfg80211_info *config = ifp->drvr->config;
 
 	brcmf_dbg(TRACE, "Enter\n");
 
+	config->pm_state = BRCMF_CFG80211_PM_STATE_SUSPENDING;
+
 	/* if the primary net_device is not READY there is nothing
 	 * we can do but pray resume goes smoothly.
 	 */
@@ -3671,7 +3906,8 @@
 	if (test_bit(BRCMF_SCAN_STATUS_BUSY, &cfg->scan_status))
 		brcmf_abort_scanning(cfg);
 
-	if (wowl == NULL) {
+	if (!wowl || !test_bit(BRCMF_VIF_STATUS_CONNECTED,
+			       &ifp->vif->sme_state)) {
 		brcmf_bus_wowl_config(cfg->pub->bus_if, false);
 		list_for_each_entry(vif, &cfg->vif_list, list) {
 			if (!test_bit(BRCMF_VIF_STATUS_READY, &vif->sme_state))
@@ -3680,7 +3916,7 @@
 			 * disassociate from AP to save power while system is
 			 * in suspended state
 			 */
-			brcmf_link_down(vif, WLAN_REASON_UNSPECIFIED);
+			brcmf_link_down(vif, WLAN_REASON_UNSPECIFIED, true);
 			/* Make sure WPA_Supplicant receives all the event
 			 * generated due to DISASSOC call to the fw to keep
 			 * the state fw and WPA_Supplicant state consistent
@@ -3691,14 +3927,19 @@
 		brcmf_set_mpc(ifp, 1);
 
 	} else {
-		/* Configure WOWL paramaters */
-		brcmf_configure_wowl(cfg, ifp, wowl);
+		if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_WOWL))
+			/* Configure WOWL parameters */
+			brcmf_configure_wowl(cfg, ifp, wowl);
 	}
 
 exit:
-	brcmf_dbg(TRACE, "Exit\n");
+	/* set cfg80211 pm state to cfg80211 suspended state */
+	config->pm_state = BRCMF_CFG80211_PM_STATE_SUSPENDED;
+
 	/* clear any scanning activity */
 	cfg->scan_status = 0;
+
+	brcmf_dbg(TRACE, "Exit\n");
 	return 0;
 }
 
@@ -4005,6 +4246,10 @@
 			brcmf_dbg(TRACE, "RSN_AKM_MFP_1X\n");
 			wpa_auth |= WPA2_AUTH_1X_SHA256;
 			break;
+		case RSN_AKM_SAE:
+			brcmf_dbg(TRACE, "RSN_AKM_SAE\n");
+			wpa_auth |= WPA3_AUTH_SAE_PSK;
+			break;
 		default:
 			brcmf_err("Invalid key mgmt info\n");
 		}
@@ -4022,11 +4267,12 @@
 				brcmf_dbg(TRACE, "MFP Required\n");
 				mfp = BRCMF_MFP_REQUIRED;
 				/* Firmware only supports mfp required in
-				 * combination with WPA2_AUTH_PSK_SHA256 or
-				 * WPA2_AUTH_1X_SHA256.
+				 * combination with WPA2_AUTH_PSK_SHA256,
+				 * WPA2_AUTH_1X_SHA256, or WPA3_AUTH_SAE_PSK.
 				 */
 				if (!(wpa_auth & (WPA2_AUTH_PSK_SHA256 |
-						  WPA2_AUTH_1X_SHA256))) {
+						  WPA2_AUTH_1X_SHA256 |
+						  WPA3_AUTH_SAE_PSK))) {
 					err = -EINVAL;
 					goto exit;
 				}
@@ -4231,6 +4477,11 @@
 		mgmt_ie_len = &saved_ie->assoc_req_ie_len;
 		mgmt_ie_buf_len = sizeof(saved_ie->assoc_req_ie);
 		break;
+	case BRCMF_VNDR_IE_ASSOCRSP_FLAG:
+		mgmt_ie_buf = saved_ie->assoc_res_ie;
+		mgmt_ie_len = &saved_ie->assoc_res_ie_len;
+		mgmt_ie_buf_len = sizeof(saved_ie->assoc_res_ie);
+		break;
 	default:
 		err = -EPERM;
 		brcmf_err("not suitable type\n");
@@ -4380,6 +4631,57 @@
 	else
 		brcmf_dbg(TRACE, "Applied Vndr IEs for Probe Resp\n");
 
+	/* Set Assoc Response IEs to FW */
+	err = brcmf_vif_set_mgmt_ie(vif, BRCMF_VNDR_IE_ASSOCRSP_FLAG,
+				    beacon->assocresp_ies,
+				    beacon->assocresp_ies_len);
+	if (err)
+		brcmf_err("Set Assoc Resp IE Failed\n");
+	else
+		brcmf_dbg(TRACE, "Applied Vndr IEs for Assoc Resp\n");
+
+	return err;
+}
+
+static s32
+brcmf_parse_configure_security(struct brcmf_if *ifp,
+			       struct cfg80211_ap_settings *settings,
+			       enum nl80211_iftype dev_role)
+{
+	const struct brcmf_tlv *rsn_ie;
+	const struct brcmf_vs_tlv *wpa_ie;
+	s32 err = 0;
+
+	/* find the RSN_IE */
+	rsn_ie = brcmf_parse_tlvs((u8 *)settings->beacon.tail,
+				  settings->beacon.tail_len, WLAN_EID_RSN);
+
+	/* find the WPA_IE */
+	wpa_ie = brcmf_find_wpaie((u8 *)settings->beacon.tail,
+				  settings->beacon.tail_len);
+
+	if (wpa_ie || rsn_ie) {
+		brcmf_dbg(TRACE, "WPA(2) IE is found\n");
+		if (wpa_ie) {
+			/* WPA IE */
+			err = brcmf_configure_wpaie(ifp, wpa_ie, false);
+			if (err < 0)
+				return err;
+		} else {
+			struct brcmf_vs_tlv *tmp_ie;
+
+			tmp_ie = (struct brcmf_vs_tlv *)rsn_ie;
+
+			/* RSN IE */
+			err = brcmf_configure_wpaie(ifp, tmp_ie, true);
+			if (err < 0)
+				return err;
+		}
+	} else {
+		brcmf_dbg(TRACE, "No WPA(2) IEs found\n");
+		brcmf_configure_opensecurity(ifp);
+	}
+
 	return err;
 }
 
@@ -4390,12 +4692,12 @@
 	s32 ie_offset;
 	struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy);
 	struct brcmf_if *ifp = netdev_priv(ndev);
+	struct brcmf_cfg80211_profile *profile = &ifp->vif->profile;
+	struct cfg80211_crypto_settings *crypto = &settings->crypto;
 	const struct brcmf_tlv *ssid_ie;
 	const struct brcmf_tlv *country_ie;
 	struct brcmf_ssid_le ssid_le;
 	s32 err = -EPERM;
-	const struct brcmf_tlv *rsn_ie;
-	const struct brcmf_vs_tlv *wpa_ie;
 	struct brcmf_join_params join_params;
 	enum nl80211_iftype dev_role;
 	struct brcmf_fil_bss_enable_le bss_enable;
@@ -4413,6 +4715,7 @@
 		  settings->inactivity_timeout);
 	dev_role = ifp->vif->wdev.iftype;
 	mbss = ifp->vif->mbss;
+	brcmf_dbg(TRACE, "mbss %s\n", mbss ? "enabled" : "disabled");
 
 	/* store current 11d setting */
 	if (brcmf_fil_cmd_int_get(ifp, BRCMF_C_GET_REGULATORY,
@@ -4449,36 +4752,6 @@
 		brcmf_configure_arp_nd_offload(ifp, false);
 	}
 
-	/* find the RSN_IE */
-	rsn_ie = brcmf_parse_tlvs((u8 *)settings->beacon.tail,
-				  settings->beacon.tail_len, WLAN_EID_RSN);
-
-	/* find the WPA_IE */
-	wpa_ie = brcmf_find_wpaie((u8 *)settings->beacon.tail,
-				  settings->beacon.tail_len);
-
-	if ((wpa_ie != NULL || rsn_ie != NULL)) {
-		brcmf_dbg(TRACE, "WPA(2) IE is found\n");
-		if (wpa_ie != NULL) {
-			/* WPA IE */
-			err = brcmf_configure_wpaie(ifp, wpa_ie, false);
-			if (err < 0)
-				goto exit;
-		} else {
-			struct brcmf_vs_tlv *tmp_ie;
-
-			tmp_ie = (struct brcmf_vs_tlv *)rsn_ie;
-
-			/* RSN IE */
-			err = brcmf_configure_wpaie(ifp, tmp_ie, true);
-			if (err < 0)
-				goto exit;
-		}
-	} else {
-		brcmf_dbg(TRACE, "No WPA(2) IEs found\n");
-		brcmf_configure_opensecurity(ifp);
-	}
-
 	/* Parameters shared by all radio interfaces */
 	if (!mbss) {
 		if ((supports_11d) && (is_11d != ifp->vif->is_11d)) {
@@ -4507,9 +4780,7 @@
 			}
 		}
 
-		if ((dev_role == NL80211_IFTYPE_AP) &&
-		    ((ifp->ifidx == 0) ||
-		     !brcmf_feat_is_enabled(ifp, BRCMF_FEAT_RSDB))) {
+		if ((dev_role == NL80211_IFTYPE_AP) && (ifp->ifidx == 0)) {
 			err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_DOWN, 1);
 			if (err < 0) {
 				brcmf_err("BRCMF_C_DOWN error %d\n", err);
@@ -4555,6 +4826,33 @@
 			brcmf_err("BRCMF_C_UP error (%d)\n", err);
 			goto exit;
 		}
+
+		if (crypto->psk) {
+			brcmf_dbg(INFO, "using PSK offload\n");
+			profile->use_fwauth |= BIT(BRCMF_PROFILE_FWAUTH_PSK);
+			err = brcmf_set_pmk(ifp, crypto->psk,
+					    BRCMF_WSEC_MAX_PSK_LEN);
+			if (err < 0)
+				goto exit;
+		}
+		if (crypto->sae_pwd) {
+			brcmf_dbg(INFO, "using SAE offload\n");
+			profile->use_fwauth |= BIT(BRCMF_PROFILE_FWAUTH_SAE);
+			err = brcmf_set_sae_password(ifp, crypto->sae_pwd,
+						     crypto->sae_pwd_len);
+			if (err < 0)
+				goto exit;
+		}
+		if (profile->use_fwauth == 0)
+			profile->use_fwauth = BIT(BRCMF_PROFILE_FWAUTH_NONE);
+
+		err = brcmf_parse_configure_security(ifp, settings,
+						     NL80211_IFTYPE_AP);
+		if (err < 0) {
+			brcmf_err("brcmf_parse_configure_security error\n");
+			goto exit;
+		}
+
 		/* On DOWN the firmware removes the WEP keys, reconfigure
 		 * them if they were set.
 		 */
@@ -4571,12 +4869,14 @@
 			goto exit;
 		}
 
-		if (settings->hidden_ssid) {
-			err = brcmf_fil_iovar_int_set(ifp, "closednet", 1);
-			if (err) {
-				brcmf_err("closednet error (%d)\n", err);
-				goto exit;
-			}
+		err = brcmf_fil_iovar_int_set(ifp, "closednet",
+					      settings->hidden_ssid);
+		if (err) {
+			brcmf_err("%s closednet error (%d)\n",
+				  settings->hidden_ssid ?
+				  "enabled" : "disabled",
+				  err);
+			goto exit;
 		}
 
 		brcmf_dbg(TRACE, "AP mode configuration complete\n");
@@ -4587,6 +4887,14 @@
 				  chanspec, err);
 			goto exit;
 		}
+
+		err = brcmf_parse_configure_security(ifp, settings,
+						     NL80211_IFTYPE_P2P_GO);
+		if (err < 0) {
+			brcmf_err("brcmf_parse_configure_security error\n");
+			goto exit;
+		}
+
 		err = brcmf_fil_bsscfg_data_set(ifp, "ssid", &ssid_le,
 						sizeof(ssid_le));
 		if (err < 0) {
@@ -4615,6 +4923,9 @@
 	if ((err) && (!mbss)) {
 		brcmf_set_mpc(ifp, 1);
 		brcmf_configure_arp_nd_offload(ifp, true);
+	} else {
+		cfg->num_softap++;
+		brcmf_dbg(TRACE, "Num of SoftAP %u\n", cfg->num_softap);
 	}
 	return err;
 }
@@ -4622,9 +4933,12 @@
 static int brcmf_cfg80211_stop_ap(struct wiphy *wiphy, struct net_device *ndev)
 {
 	struct brcmf_if *ifp = netdev_priv(ndev);
+	struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy);
+	struct brcmf_cfg80211_profile *profile = &ifp->vif->profile;
 	s32 err;
 	struct brcmf_fil_bss_enable_le bss_enable;
 	struct brcmf_join_params join_params;
+	s32 apsta = 0;
 
 	brcmf_dbg(TRACE, "Enter\n");
 
@@ -4633,6 +4947,35 @@
 		/* first to make sure they get processed by fw. */
 		msleep(400);
 
+		cfg->num_softap--;
+
+		if (profile->use_fwauth != BIT(BRCMF_PROFILE_FWAUTH_NONE)) {
+			if (profile->use_fwauth & BIT(BRCMF_PROFILE_FWAUTH_PSK))
+				brcmf_set_pmk(ifp, NULL, 0);
+			if (profile->use_fwauth & BIT(BRCMF_PROFILE_FWAUTH_SAE))
+				brcmf_set_sae_password(ifp, NULL, 0);
+			profile->use_fwauth = BIT(BRCMF_PROFILE_FWAUTH_NONE);
+		}
+
+		/* Clear bss configuration and SSID */
+		bss_enable.bsscfgidx = cpu_to_le32(ifp->bsscfgidx);
+		bss_enable.enable = cpu_to_le32(0);
+		err = brcmf_fil_iovar_data_set(ifp, "bss", &bss_enable,
+					       sizeof(bss_enable));
+		if (err < 0)
+			brcmf_err("bss_enable config failed %d\n", err);
+
+		memset(&join_params, 0, sizeof(join_params));
+		err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SET_SSID,
+					     &join_params, sizeof(join_params));
+		if (err < 0)
+			brcmf_err("SET SSID error (%d)\n", err);
+
+		if (cfg->num_softap) {
+			brcmf_dbg(TRACE, "Num of SoftAP %u\n", cfg->num_softap);
+			return 0;
+		}
+
 		if (ifp->vif->mbss) {
 			err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_DOWN, 1);
 			return err;
@@ -4642,17 +4985,18 @@
 		if (ifp->bsscfgidx == 0)
 			brcmf_fil_iovar_int_set(ifp, "closednet", 0);
 
-		memset(&join_params, 0, sizeof(join_params));
-		err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SET_SSID,
-					     &join_params, sizeof(join_params));
+		err = brcmf_fil_iovar_int_get(ifp, "apsta", &apsta);
 		if (err < 0)
-			brcmf_err("SET SSID error (%d)\n", err);
-		err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_DOWN, 1);
-		if (err < 0)
-			brcmf_err("BRCMF_C_DOWN error %d\n", err);
-		err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_AP, 0);
-		if (err < 0)
-			brcmf_err("setting AP mode failed %d\n", err);
+			brcmf_err("wl apsta failed (%d)\n", err);
+
+		if (!apsta) {
+			err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_DOWN, 1);
+			if (err < 0)
+				brcmf_err("BRCMF_C_DOWN error %d\n", err);
+			err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_AP, 0);
+			if (err < 0)
+				brcmf_err("setting AP mode failed %d\n", err);
+		}
 		if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MBSS))
 			brcmf_fil_iovar_int_set(ifp, "mbss", 0);
 		brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_REGULATORY,
@@ -5116,6 +5460,7 @@
 				  const struct cfg80211_pmk_conf *conf)
 {
 	struct brcmf_if *ifp;
+	int ret;
 
 	brcmf_dbg(TRACE, "enter\n");
 
@@ -5124,8 +5469,12 @@
 	if (WARN_ON(ifp->vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_1X))
 		return -EINVAL;
 
-	if (conf->pmk_len > BRCMF_WSEC_MAX_PSK_LEN)
-		return -ERANGE;
+	if (ifp->vif->profile.is_okc) {
+		ret = brcmf_fil_iovar_data_set(ifp, "okc_info_pmk", conf->pmk,
+					       conf->pmk_len);
+		if (ret < 0)
+			brcmf_err("okc_info_pmk iovar failed: ret=%d\n", ret);
+	}
 
 	return brcmf_set_pmk(ifp, conf->pmk, conf->pmk_len);
 }
@@ -5143,6 +5492,26 @@
 	return brcmf_set_pmk(ifp, NULL, 0);
 }
 
+static int
+brcmf_cfg80211_change_bss(struct wiphy *wiphy, struct net_device *dev,
+			  struct bss_parameters *params)
+{
+	struct brcmf_if *ifp;
+	int ret = 0;
+	u32 ap_isolate;
+
+	brcmf_dbg(TRACE, "Enter\n");
+	ifp = netdev_priv(dev);
+	if (params->ap_isolate >= 0) {
+		ap_isolate = (u32)params->ap_isolate;
+		ret = brcmf_fil_iovar_int_set(ifp, "ap_isolate", ap_isolate);
+		if (ret < 0)
+			brcmf_err("ap_isolate iovar failed: ret=%d\n", ret);
+	}
+
+	return ret;
+}
+
 static struct cfg80211_ops brcmf_cfg80211_ops = {
 	.add_virtual_intf = brcmf_cfg80211_add_iface,
 	.del_virtual_intf = brcmf_cfg80211_del_iface,
@@ -5188,27 +5557,16 @@
 	.update_connect_params = brcmf_cfg80211_update_conn_params,
 	.set_pmk = brcmf_cfg80211_set_pmk,
 	.del_pmk = brcmf_cfg80211_del_pmk,
+	.change_bss = brcmf_cfg80211_change_bss,
 };
 
-struct cfg80211_ops *brcmf_cfg80211_get_ops(struct brcmf_mp_device *settings)
-{
-	struct cfg80211_ops *ops;
-
-	ops = kmemdup(&brcmf_cfg80211_ops, sizeof(brcmf_cfg80211_ops),
-		       GFP_KERNEL);
-
-	if (ops && settings->roamoff)
-		ops->update_connect_params = NULL;
-
-	return ops;
-}
-
 struct brcmf_cfg80211_vif *brcmf_alloc_vif(struct brcmf_cfg80211_info *cfg,
 					   enum nl80211_iftype type)
 {
 	struct brcmf_cfg80211_vif *vif_walk;
 	struct brcmf_cfg80211_vif *vif;
 	bool mbss;
+	struct brcmf_if *ifp = brcmf_get_ifp(cfg->pub, 0);
 
 	brcmf_dbg(TRACE, "allocating virtual interface (size=%zu)\n",
 		  sizeof(*vif));
@@ -5221,7 +5579,8 @@
 
 	brcmf_init_prof(&vif->profile);
 
-	if (type == NL80211_IFTYPE_AP) {
+	if (type == NL80211_IFTYPE_AP &&
+	    brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MBSS)) {
 		mbss = false;
 		list_for_each_entry(vif_walk, &cfg->vif_list, list) {
 			if (vif_walk->wdev.iftype == NL80211_IFTYPE_AP) {
@@ -5250,8 +5609,10 @@
 	ifp = netdev_priv(ndev);
 	vif = ifp->vif;
 
-	if (vif)
+	if (vif) {
 		brcmf_free_vif(vif);
+		ifp->vif = NULL;
+	}
 }
 
 static bool brcmf_is_linkup(struct brcmf_cfg80211_vif *vif,
@@ -5260,25 +5621,28 @@
 	u32 event = e->event_code;
 	u32 status = e->status;
 
-	if (vif->profile.use_fwsup == BRCMF_PROFILE_FWSUP_PSK &&
-	    event == BRCMF_E_PSK_SUP &&
-	    status == BRCMF_E_STATUS_FWSUP_COMPLETED)
+	if (event == BRCMF_E_PSK_SUP &&
+	    status == BRCMF_E_STATUS_FWSUP_COMPLETED) {
 		set_bit(BRCMF_VIF_STATUS_EAP_SUCCESS, &vif->sme_state);
+		if (vif->profile.use_fwsup == BRCMF_PROFILE_FWSUP_1X)
+			return true;
+	}
+
 	if (event == BRCMF_E_SET_SSID && status == BRCMF_E_STATUS_SUCCESS) {
 		brcmf_dbg(CONN, "Processing set ssid\n");
 		memcpy(vif->profile.bssid, e->addr, ETH_ALEN);
-		if (vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_PSK)
+		if (vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_PSK &&
+		    vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_SAE)
 			return true;
 
 		set_bit(BRCMF_VIF_STATUS_ASSOC_SUCCESS, &vif->sme_state);
 	}
 
 	if (test_bit(BRCMF_VIF_STATUS_EAP_SUCCESS, &vif->sme_state) &&
-	    test_bit(BRCMF_VIF_STATUS_ASSOC_SUCCESS, &vif->sme_state)) {
-		clear_bit(BRCMF_VIF_STATUS_EAP_SUCCESS, &vif->sme_state);
-		clear_bit(BRCMF_VIF_STATUS_ASSOC_SUCCESS, &vif->sme_state);
+	    test_and_clear_bit(BRCMF_VIF_STATUS_ASSOC_SUCCESS,
+			       &vif->sme_state))
 		return true;
-	}
+
 	return false;
 }
 
@@ -5335,11 +5699,151 @@
 	conn_info->resp_ie_len = 0;
 }
 
+u8 brcmf_map_prio_to_prec(void *config, u8 prio)
+{
+	struct brcmf_cfg80211_info *cfg = (struct brcmf_cfg80211_info *)config;
+
+	if (!cfg)
+		return (prio == PRIO_8021D_NONE || prio == PRIO_8021D_BE) ?
+		       (prio ^ 2) : prio;
+
+	/* For those AC(s) with ACM flag set to 1, convert its 4-level priority
+	 * to an 8-level precedence which is the same as BE's */
+	if (prio > PRIO_8021D_EE &&
+	    cfg->ac_priority[prio] == cfg->ac_priority[PRIO_8021D_BE])
+		return cfg->ac_priority[prio] * 2;
+
+	/* Conversion of 4-level priority to 8-level precedence */
+	if (prio == PRIO_8021D_BE || prio == PRIO_8021D_BK ||
+	    prio == PRIO_8021D_CL || prio == PRIO_8021D_VO)
+		return cfg->ac_priority[prio] * 2;
+	else
+		return cfg->ac_priority[prio] * 2 + 1;
+}
+
+u8 brcmf_map_prio_to_aci(void *config, u8 prio)
+{
+	/* Prio here refers to the 802.1d priority in range of 0 to 7.
+	 * ACI here refers to the WLAN AC Index in range of 0 to 3.
+	 * This function will return ACI corresponding to input prio.
+	 */
+	struct brcmf_cfg80211_info *cfg = (struct brcmf_cfg80211_info *)config;
+
+	if (cfg)
+		return cfg->ac_priority[prio];
+
+	return prio;
+}
+
+static void brcmf_init_wmm_prio(u8 *priority)
+{
+	/* Initialize AC priority array to default
+	 * 802.1d priority as per following table:
+	 * 802.1d prio 0,3 maps to BE
+	 * 802.1d prio 1,2 maps to BK
+	 * 802.1d prio 4,5 maps to VI
+	 * 802.1d prio 6,7 maps to VO
+	 */
+	priority[0] = BRCMF_FWS_FIFO_AC_BE;
+	priority[3] = BRCMF_FWS_FIFO_AC_BE;
+	priority[1] = BRCMF_FWS_FIFO_AC_BK;
+	priority[2] = BRCMF_FWS_FIFO_AC_BK;
+	priority[4] = BRCMF_FWS_FIFO_AC_VI;
+	priority[5] = BRCMF_FWS_FIFO_AC_VI;
+	priority[6] = BRCMF_FWS_FIFO_AC_VO;
+	priority[7] = BRCMF_FWS_FIFO_AC_VO;
+}
+
+static void brcmf_wifi_prioritize_acparams(const
+	struct brcmf_cfg80211_edcf_acparam *acp, u8 *priority)
+{
+	u8 aci;
+	u8 aifsn;
+	u8 ecwmin;
+	u8 ecwmax;
+	u8 acm;
+	u8 ranking_basis[EDCF_AC_COUNT];
+	u8 aci_prio[EDCF_AC_COUNT]; /* AC_BE, AC_BK, AC_VI, AC_VO */
+	u8 index;
+
+	for (aci = 0; aci < EDCF_AC_COUNT; aci++, acp++) {
+		aifsn  = acp->ACI & EDCF_AIFSN_MASK;
+		acm = (acp->ACI & EDCF_ACM_MASK) ? 1 : 0;
+		ecwmin = acp->ECW & EDCF_ECWMIN_MASK;
+		ecwmax = (acp->ECW & EDCF_ECWMAX_MASK) >> EDCF_ECWMAX_SHIFT;
+		brcmf_dbg(CONN, "ACI %d aifsn %d acm %d ecwmin %d ecwmax %d\n",
+			  aci, aifsn, acm, ecwmin, ecwmax);
+		/* Default AC_VO will be the lowest ranking value */
+		ranking_basis[aci] = aifsn + ecwmin + ecwmax;
+		/* Initialise priority starting at 0 (AC_BE) */
+		aci_prio[aci] = 0;
+
+		/* If ACM is set, STA can't use this AC as per 802.11.
+		 * Change the ranking to BE
+		 */
+		if (aci != AC_BE && aci != AC_BK && acm == 1)
+			ranking_basis[aci] = ranking_basis[AC_BE];
+	}
+
+	/* Ranking method which works for AC priority
+	 * swapping when values for cwmin, cwmax and aifsn are varied
+	 * Compare each aci_prio against each other aci_prio
+	 */
+	for (aci = 0; aci < EDCF_AC_COUNT; aci++) {
+		for (index = 0; index < EDCF_AC_COUNT; index++) {
+			if (index != aci) {
+				/* Smaller ranking value has higher priority,
+				 * so increment priority for each ACI which has
+				 * a higher ranking value
+				 */
+				if (ranking_basis[aci] < ranking_basis[index])
+					aci_prio[aci]++;
+			}
+		}
+	}
+
+	/* By now, aci_prio[] will be in range of 0 to 3.
+	 * Use ACI prio to get the new priority value for
+	 * each 802.1d traffic type, in this range.
+	 */
+	if (!(aci_prio[AC_BE] == aci_prio[AC_BK] &&
+	      aci_prio[AC_BK] == aci_prio[AC_VI] &&
+	      aci_prio[AC_VI] == aci_prio[AC_VO])) {
+
+		/* 802.1d 0,3 maps to BE */
+		priority[0] = aci_prio[AC_BE];
+		priority[3] = aci_prio[AC_BE];
+
+		/* 802.1d 1,2 maps to BK */
+		priority[1] = aci_prio[AC_BK];
+		priority[2] = aci_prio[AC_BK];
+
+		/* 802.1d 4,5 maps to VO */
+		priority[4] = aci_prio[AC_VI];
+		priority[5] = aci_prio[AC_VI];
+
+		/* 802.1d 6,7 maps to VO */
+		priority[6] = aci_prio[AC_VO];
+		priority[7] = aci_prio[AC_VO];
+
+	} else {
+		/* Initialize to default priority */
+		brcmf_init_wmm_prio(priority);
+	}
+
+	brcmf_dbg(CONN, "Adj prio BE 0->%d, BK 1->%d, BK 2->%d, BE 3->%d\n",
+		  priority[0], priority[1], priority[2], priority[3]);
+
+	brcmf_dbg(CONN, "Adj prio VI 4->%d, VI 5->%d, VO 6->%d, VO 7->%d\n",
+		  priority[4], priority[5], priority[6], priority[7]);
+}
+
 static s32 brcmf_get_assoc_ies(struct brcmf_cfg80211_info *cfg,
 			       struct brcmf_if *ifp)
 {
 	struct brcmf_cfg80211_assoc_ielen_le *assoc_info;
 	struct brcmf_cfg80211_connect_info *conn_info = cfg_to_conn(cfg);
+	struct brcmf_cfg80211_edcf_acparam edcf_acparam_info[EDCF_AC_COUNT];
 	u32 req_len;
 	u32 resp_len;
 	s32 err = 0;
@@ -5368,8 +5872,6 @@
 		conn_info->req_ie =
 		    kmemdup(cfg->extra_buf, conn_info->req_ie_len,
 			    GFP_KERNEL);
-		if (!conn_info->req_ie)
-			conn_info->req_ie_len = 0;
 	} else {
 		conn_info->req_ie_len = 0;
 		conn_info->req_ie = NULL;
@@ -5386,8 +5888,17 @@
 		conn_info->resp_ie =
 		    kmemdup(cfg->extra_buf, conn_info->resp_ie_len,
 			    GFP_KERNEL);
-		if (!conn_info->resp_ie)
-			conn_info->resp_ie_len = 0;
+
+		err = brcmf_fil_iovar_data_get(ifp, "wme_ac_sta",
+					       edcf_acparam_info,
+					       sizeof(edcf_acparam_info));
+		if (err) {
+			brcmf_err("could not get wme_ac_sta (%d)\n", err);
+			return err;
+		}
+
+		brcmf_wifi_prioritize_acparams(edcf_acparam_info,
+					       cfg->ac_priority);
 	} else {
 		conn_info->resp_ie_len = 0;
 		conn_info->resp_ie = NULL;
@@ -5398,6 +5909,47 @@
 	return err;
 }
 
+static bool
+brcmf_has_pmkid(const u8 *parse, u32 len)
+{
+	const struct brcmf_tlv *rsn_ie;
+	const u8 *ie;
+	u32 ie_len;
+	u32 offset;
+	u16 count;
+
+	rsn_ie = brcmf_parse_tlvs(parse, len, WLAN_EID_RSN);
+	if (!rsn_ie)
+		goto done;
+	ie = (const u8 *)rsn_ie;
+	ie_len = rsn_ie->len + TLV_HDR_LEN;
+	/* Skip group data cipher suite */
+	offset = TLV_HDR_LEN + WPA_IE_VERSION_LEN + WPA_IE_MIN_OUI_LEN;
+	if (offset + WPA_IE_SUITE_COUNT_LEN >= ie_len)
+		goto done;
+	/* Skip pairwise cipher suite(s) */
+	count = ie[offset] + (ie[offset + 1] << 8);
+	offset += WPA_IE_SUITE_COUNT_LEN + (count * WPA_IE_MIN_OUI_LEN);
+	if (offset + WPA_IE_SUITE_COUNT_LEN >= ie_len)
+		goto done;
+	/* Skip auth key management suite(s) */
+	count = ie[offset] + (ie[offset + 1] << 8);
+	offset += WPA_IE_SUITE_COUNT_LEN + (count * WPA_IE_MIN_OUI_LEN);
+	if (offset + RSN_CAP_LEN >= ie_len)
+		goto done;
+	/* Skip rsn capabilities */
+	offset += RSN_CAP_LEN;
+	if (offset + RSN_PMKID_COUNT_LEN > ie_len)
+		goto done;
+	/* Extract PMKID count */
+	count = ie[offset] + (ie[offset + 1] << 8);
+	if (count)
+		return true;
+
+done:
+	return false;
+}
+
 static s32
 brcmf_bss_roaming_done(struct brcmf_cfg80211_info *cfg,
 		       struct net_device *ndev,
@@ -5458,6 +6010,11 @@
 	roam_info.resp_ie = conn_info->resp_ie;
 	roam_info.resp_ie_len = conn_info->resp_ie_len;
 
+	if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X &&
+	    (brcmf_has_pmkid(roam_info.req_ie, roam_info.req_ie_len) ||
+	     profile->is_ft || profile->is_okc))
+		roam_info.authorized = true;
+
 	cfg80211_roamed(ndev, &roam_info, GFP_KERNEL);
 	brcmf_dbg(CONN, "Report roaming result\n");
 
@@ -5495,10 +6052,22 @@
 		conn_params.req_ie_len = conn_info->req_ie_len;
 		conn_params.resp_ie = conn_info->resp_ie;
 		conn_params.resp_ie_len = conn_info->resp_ie_len;
+
+		if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X &&
+		    brcmf_has_pmkid(conn_params.req_ie, conn_params.req_ie_len))
+			conn_params.authorized = true;
+
 		cfg80211_connect_done(ndev, &conn_params, GFP_KERNEL);
 		brcmf_dbg(CONN, "Report connect result - connection %s\n",
 			  completed ? "succeeded" : "failed");
 	}
+
+	if (test_and_clear_bit(BRCMF_VIF_STATUS_EAP_SUCCESS,
+			       &ifp->vif->sme_state) &&
+	    profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X) {
+		cfg80211_port_authorized(ndev, profile->bssid, GFP_KERNEL);
+		brcmf_dbg(CONN, "Report port authorized\n");
+	}
 	brcmf_dbg(TRACE, "Exit\n");
 	return 0;
 }
@@ -5511,7 +6080,7 @@
 	static int generation;
 	u32 event = e->event_code;
 	u32 reason = e->reason;
-	struct station_info *sinfo;
+	struct station_info sinfo;
 
 	brcmf_dbg(CONN, "event %s (%u), reason %d\n",
 		  brcmf_fweh_event_name(event), event, reason);
@@ -5524,22 +6093,16 @@
 
 	if (((event == BRCMF_E_ASSOC_IND) || (event == BRCMF_E_REASSOC_IND)) &&
 	    (reason == BRCMF_E_STATUS_SUCCESS)) {
+		memset(&sinfo, 0, sizeof(sinfo));
 		if (!data) {
 			brcmf_err("No IEs present in ASSOC/REASSOC_IND");
 			return -EINVAL;
 		}
-
-		sinfo = kzalloc(sizeof(*sinfo), GFP_KERNEL);
-		if (!sinfo)
-			return -ENOMEM;
-
-		sinfo->assoc_req_ies = data;
-		sinfo->assoc_req_ies_len = e->datalen;
+		sinfo.assoc_req_ies = data;
+		sinfo.assoc_req_ies_len = e->datalen;
 		generation++;
-		sinfo->generation = generation;
-		cfg80211_new_sta(ndev, e->addr, sinfo, GFP_KERNEL);
-
-		kfree(sinfo);
+		sinfo.generation = generation;
+		cfg80211_new_sta(ndev, e->addr, &sinfo, GFP_KERNEL);
 	} else if ((event == BRCMF_E_DISASSOC_IND) ||
 		   (event == BRCMF_E_DEAUTH_IND) ||
 		   (event == BRCMF_E_DEAUTH)) {
@@ -5586,7 +6149,11 @@
 		if (!brcmf_is_ibssmode(ifp->vif)) {
 			brcmf_bss_connect_done(cfg, ndev, e, false);
 			brcmf_link_down(ifp->vif,
-					brcmf_map_fw_linkdown_reason(e));
+					brcmf_map_fw_linkdown_reason(e),
+					e->event_code &
+					(BRCMF_E_DEAUTH_IND |
+					BRCMF_E_DISASSOC_IND)
+					? false : true);
 			brcmf_init_prof(ndev_to_prof(ndev));
 			if (ndev != cfg_to_ndev(cfg))
 				complete(&cfg->vif_disabled);
@@ -5649,6 +6216,9 @@
 	struct brcmf_if_event *ifevent = (struct brcmf_if_event *)data;
 	struct brcmf_cfg80211_vif_event *event = &cfg->vif_event;
 	struct brcmf_cfg80211_vif *vif;
+	enum nl80211_iftype iftype = NL80211_IFTYPE_UNSPECIFIED;
+	bool vif_pend = false;
+	int err;
 
 	brcmf_dbg(TRACE, "Enter: action %u flags %u ifidx %u bsscfgidx %u\n",
 		  ifevent->action, ifevent->flags, ifevent->ifidx,
@@ -5661,9 +6231,28 @@
 	switch (ifevent->action) {
 	case BRCMF_E_IF_ADD:
 		/* waiting process may have timed out */
-		if (!cfg->vif_event.vif) {
+		if (!vif) {
+			/* handle IF_ADD event from firmware */
 			spin_unlock(&event->vif_event_lock);
-			return -EBADF;
+			vif_pend = true;
+			if (ifevent->role == WLC_E_IF_ROLE_STA)
+				iftype = NL80211_IFTYPE_STATION;
+			else if (ifevent->role == WLC_E_IF_ROLE_AP)
+				iftype = NL80211_IFTYPE_AP;
+			else
+				vif_pend = false;
+
+			if (vif_pend) {
+				vif = brcmf_alloc_vif(cfg, iftype);
+				if (IS_ERR(vif)) {
+					brcmf_err("Role:%d failed to alloc vif\n",
+						  ifevent->role);
+					return PTR_ERR(vif);
+				}
+			} else {
+				brcmf_err("Invalid Role:%d\n", ifevent->role);
+				return -EBADF;
+			}
 		}
 
 		ifp->vif = vif;
@@ -5673,6 +6262,18 @@
 			ifp->ndev->ieee80211_ptr = &vif->wdev;
 			SET_NETDEV_DEV(ifp->ndev, wiphy_dev(cfg->wiphy));
 		}
+
+		if (vif_pend) {
+			err = brcmf_net_attach(ifp, false);
+			if (err) {
+				brcmf_err("netdevice register failed with err:%d\n",
+					  err);
+				brcmf_free_vif(vif);
+				free_netdev(ifp->ndev);
+			}
+			return err;
+		}
+
 		spin_unlock(&event->vif_event_lock);
 		wake_up(&event->vif_wq);
 		return 0;
@@ -5790,6 +6391,7 @@
 
 	cfg->scan_request = NULL;
 	cfg->pwr_save = true;
+	cfg->active_scan = true;	/* we do active scan per default */
 	cfg->dongle_up = false;		/* dongle is not up yet */
 	err = brcmf_init_priv_mem(cfg);
 	if (err)
@@ -5798,6 +6400,7 @@
 	mutex_init(&cfg->usr_sync);
 	brcmf_init_escan(cfg);
 	brcmf_init_conf(cfg->conf);
+	brcmf_init_wmm_prio(cfg->ac_priority);
 	init_completion(&cfg->vif_disabled);
 	return err;
 }
@@ -5920,7 +6523,7 @@
 static int brcmf_construct_chaninfo(struct brcmf_cfg80211_info *cfg,
 				    u32 bw_cap[])
 {
-	struct brcmf_if *ifp = brcmf_get_ifp(cfg->pub, 0);
+	struct brcmf_if *ifp = netdev_priv(cfg_to_ndev(cfg));
 	struct ieee80211_supported_band *band;
 	struct ieee80211_channel *channel;
 	struct wiphy *wiphy;
@@ -6010,8 +6613,7 @@
 			 * for subsequent chanspecs.
 			 */
 			channel->flags = IEEE80211_CHAN_NO_HT40 |
-					 IEEE80211_CHAN_NO_80MHZ |
-					 IEEE80211_CHAN_NO_160MHZ;
+					 IEEE80211_CHAN_NO_80MHZ;
 			ch.bw = BRCMU_CHAN_BW_20;
 			cfg->d11inf.encchspec(&ch);
 			chaninfo = ch.chspec;
@@ -6036,7 +6638,7 @@
 
 static int brcmf_enable_bw40_2g(struct brcmf_cfg80211_info *cfg)
 {
-	struct brcmf_if *ifp = brcmf_get_ifp(cfg->pub, 0);
+	struct brcmf_if *ifp = netdev_priv(cfg_to_ndev(cfg));
 	struct ieee80211_supported_band *band;
 	struct brcmf_fil_bwcap_le band_bwcap;
 	struct brcmf_chanspec_list *list;
@@ -6221,10 +6823,10 @@
 	}
 }
 
-static int brcmf_setup_wiphybands(struct brcmf_cfg80211_info *cfg)
+static int brcmf_setup_wiphybands(struct wiphy *wiphy)
 {
-	struct brcmf_if *ifp = brcmf_get_ifp(cfg->pub, 0);
-	struct wiphy *wiphy;
+	struct brcmf_cfg80211_info *cfg = wiphy_priv(wiphy);
+	struct brcmf_if *ifp = netdev_priv(cfg_to_ndev(cfg));
 	u32 nmode = 0;
 	u32 vhtmode = 0;
 	u32 bw_cap[2] = { WLC_BW_20MHZ_BIT, WLC_BW_20MHZ_BIT };
@@ -6345,6 +6947,9 @@
  *	#STA <= 1, #AP <= 1, channels = 1, 2 total
  *	#AP <= 4, matching BI, channels = 1, 4 total
  *
+ * no p2p and rsdb:
+ *	#STA <= 1, #AP <= 2, channels = 2, 3 total
+ *
  * p2p, no mchan, and mbss:
  *
  *	#STA <= 1, #P2P-DEV <= 1, #{P2P-CL, P2P-GO} <= 1, channels = 1, 3 total
@@ -6356,6 +6961,16 @@
  *	#STA <= 1, #P2P-DEV <= 1, #{P2P-CL, P2P-GO} <= 1, channels = 2, 3 total
  *	#STA <= 1, #P2P-DEV <= 1, #AP <= 1, #P2P-CL <= 1, channels = 1, 4 total
  *	#AP <= 4, matching BI, channels = 1, 4 total
+ *
+ * p2p, rsdb, and no mbss:
+ *	#STA <= 1, #P2P-DEV <= 1, #{P2P-CL, P2P-GO} <= 2, AP <= 2,
+ *	 channels = 2, 4 total
+ *
+ * p2p, rsdb, mbss
+ *	#STA <= 1, #P2P-DEV <= 1, #{P2P-CL, P2P-GO} <= 2, AP <= 2,
+ *	 channels = 2, 4 total
+ *	#AP <= 4, matching BI, channels = 1, 4 total
+ *
  */
 static int brcmf_setup_ifmodes(struct wiphy *wiphy, struct brcmf_if *ifp)
 {
@@ -6363,13 +6978,14 @@
 	struct ieee80211_iface_limit *c0_limits = NULL;
 	struct ieee80211_iface_limit *p2p_limits = NULL;
 	struct ieee80211_iface_limit *mbss_limits = NULL;
-	bool mbss, p2p;
+	bool mbss, p2p, rsdb;
 	int i, c, n_combos;
 
 	mbss = brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MBSS);
 	p2p = brcmf_feat_is_enabled(ifp, BRCMF_FEAT_P2P);
+	rsdb = brcmf_feat_is_enabled(ifp, BRCMF_FEAT_RSDB);
 
-	n_combos = 1 + !!p2p + !!mbss;
+	n_combos = 1 + !!(p2p && !rsdb) + !!mbss;
 	combo = kcalloc(n_combos, sizeof(*combo), GFP_KERNEL);
 	if (!combo)
 		goto err;
@@ -6380,16 +6996,36 @@
 
 	c = 0;
 	i = 0;
-	c0_limits = kcalloc(p2p ? 3 : 2, sizeof(*c0_limits), GFP_KERNEL);
+	if (p2p && rsdb)
+		c0_limits = kcalloc(4, sizeof(*c0_limits), GFP_KERNEL);
+	else if (p2p)
+		c0_limits = kcalloc(3, sizeof(*c0_limits), GFP_KERNEL);
+	else
+		c0_limits = kcalloc(2, sizeof(*c0_limits), GFP_KERNEL);
 	if (!c0_limits)
 		goto err;
-	c0_limits[i].max = 1;
-	c0_limits[i++].types = BIT(NL80211_IFTYPE_STATION);
-	if (p2p) {
+	if (p2p && rsdb) {
+		combo[c].num_different_channels = 2;
+		wiphy->interface_modes |= BIT(NL80211_IFTYPE_P2P_CLIENT) |
+					  BIT(NL80211_IFTYPE_P2P_GO) |
+					  BIT(NL80211_IFTYPE_P2P_DEVICE);
+		c0_limits[i].max = 1;
+		c0_limits[i++].types = BIT(NL80211_IFTYPE_STATION);
+		c0_limits[i].max = 1;
+		c0_limits[i++].types = BIT(NL80211_IFTYPE_P2P_DEVICE);
+		c0_limits[i].max = 2;
+		c0_limits[i++].types = BIT(NL80211_IFTYPE_P2P_CLIENT) |
+				       BIT(NL80211_IFTYPE_P2P_GO);
+		c0_limits[i].max = 2;
+		c0_limits[i++].types = BIT(NL80211_IFTYPE_AP);
+		combo[c].max_interfaces = 4;
+	} else if (p2p) {
 		if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MCHAN))
 			combo[c].num_different_channels = 2;
 		else
 			combo[c].num_different_channels = 1;
+		c0_limits[i].max = 1;
+		c0_limits[i++].types = BIT(NL80211_IFTYPE_STATION);
 		wiphy->interface_modes |= BIT(NL80211_IFTYPE_P2P_CLIENT) |
 					  BIT(NL80211_IFTYPE_P2P_GO) |
 					  BIT(NL80211_IFTYPE_P2P_DEVICE);
@@ -6398,16 +7034,26 @@
 		c0_limits[i].max = 1;
 		c0_limits[i++].types = BIT(NL80211_IFTYPE_P2P_CLIENT) |
 				       BIT(NL80211_IFTYPE_P2P_GO);
+		combo[c].max_interfaces = i;
+	} else if (rsdb) {
+		combo[c].num_different_channels = 2;
+		c0_limits[i].max = 1;
+		c0_limits[i++].types = BIT(NL80211_IFTYPE_STATION);
+		c0_limits[i].max = 2;
+		c0_limits[i++].types = BIT(NL80211_IFTYPE_AP);
+		combo[c].max_interfaces = 3;
 	} else {
 		combo[c].num_different_channels = 1;
 		c0_limits[i].max = 1;
+		c0_limits[i++].types = BIT(NL80211_IFTYPE_STATION);
+		c0_limits[i].max = 1;
 		c0_limits[i++].types = BIT(NL80211_IFTYPE_AP);
+		combo[c].max_interfaces = i;
 	}
-	combo[c].max_interfaces = i;
 	combo[c].n_limits = i;
 	combo[c].limits = c0_limits;
 
-	if (p2p) {
+	if (p2p && !rsdb) {
 		c++;
 		i = 0;
 		p2p_limits = kcalloc(4, sizeof(*p2p_limits), GFP_KERNEL);
@@ -6469,6 +7115,7 @@
 #ifdef CONFIG_PM
 	struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy);
 	struct wiphy_wowlan_support *wowl;
+	struct cfg80211_wowlan *brcmf_wowlan_config = NULL;
 
 	wowl = kmemdup(&brcmf_wowlan_support, sizeof(brcmf_wowlan_support),
 		       GFP_KERNEL);
@@ -6491,12 +7138,34 @@
 	}
 
 	wiphy->wowlan = wowl;
+
+	/* wowlan_config structure report for kernels */
+	brcmf_wowlan_config = kzalloc(sizeof(*brcmf_wowlan_config),
+				      GFP_KERNEL);
+	if (brcmf_wowlan_config) {
+		brcmf_wowlan_config->any = false;
+		brcmf_wowlan_config->disconnect = true;
+		brcmf_wowlan_config->eap_identity_req = true;
+		brcmf_wowlan_config->four_way_handshake = true;
+		brcmf_wowlan_config->rfkill_release = false;
+		brcmf_wowlan_config->patterns = NULL;
+		brcmf_wowlan_config->n_patterns = 0;
+		brcmf_wowlan_config->tcp = NULL;
+		if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_WOWL_GTK))
+			brcmf_wowlan_config->gtk_rekey_failure = true;
+		else
+			brcmf_wowlan_config->gtk_rekey_failure = false;
+	} else {
+		brcmf_err("Can not allocate memory for brcm_wowlan_config\n");
+	}
+	wiphy->wowlan_config = brcmf_wowlan_config;
 #endif
 }
 
 static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp)
 {
 	struct brcmf_pub *drvr = ifp->drvr;
+	struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy);
 	const struct ieee80211_iface_combination *combo;
 	struct ieee80211_supported_band *band;
 	u16 max_interfaces = 0;
@@ -6554,6 +7223,16 @@
 				      NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK);
 		wiphy_ext_feature_set(wiphy,
 				      NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_1X);
+		if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_SAE))
+			wiphy_ext_feature_set(wiphy,
+					      NL80211_EXT_FEATURE_SAE_OFFLOAD);
+	}
+	if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_FWAUTH)) {
+		wiphy_ext_feature_set(wiphy,
+				      NL80211_EXT_FEATURE_4WAY_HANDSHAKE_AP_PSK);
+		if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_SAE))
+			wiphy_ext_feature_set(wiphy,
+					      NL80211_EXT_FEATURE_SAE_OFFLOAD);
 	}
 	wiphy->mgmt_stypes = brcmf_txrx_stypes;
 	wiphy->max_remain_on_channel_duration = 5000;
@@ -6564,9 +7243,12 @@
 	/* vendor commands/events support */
 	wiphy->vendor_commands = brcmf_vendor_cmds;
 	wiphy->n_vendor_commands = BRCMF_VNDR_CMDS_LAST - 1;
+	wiphy->vendor_events = brcmf_vendor_events;
+	wiphy->n_vendor_events = BRCMF_VNDR_EVTS_LAST;
+	brcmf_fweh_register(cfg->pub, BRCMF_E_PHY_TEMP,
+			    brcmf_wiphy_phy_temp_evt_handler);
 
-	if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_WOWL))
-		brcmf_wiphy_wowl_params(wiphy, ifp);
+	brcmf_wiphy_wowl_params(wiphy, ifp);
 	err = brcmf_fil_cmd_data_get(ifp, BRCMF_C_GET_BANDLIST, &bandlist,
 				     sizeof(bandlist));
 	if (err) {
@@ -6623,6 +7305,7 @@
 	struct wireless_dev *wdev;
 	struct brcmf_if *ifp;
 	s32 power_mode;
+	s32 eap_restrict;
 	s32 err = 0;
 
 	if (cfg->dongle_up)
@@ -6637,7 +7320,7 @@
 
 	brcmf_dongle_scantime(ifp);
 
-	power_mode = cfg->pwr_save ? PM_FAST : PM_OFF;
+	power_mode = cfg->pwr_save ? ifp->drvr->settings->default_pm : PM_OFF;
 	err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_PM, power_mode);
 	if (err)
 		goto default_conf_out;
@@ -6647,6 +7330,14 @@
 	err = brcmf_dongle_roam(ifp);
 	if (err)
 		goto default_conf_out;
+
+	eap_restrict = ifp->drvr->settings->eap_restrict;
+	if (eap_restrict) {
+		err = brcmf_fil_iovar_int_set(ifp, "eap_restrict",
+					      eap_restrict);
+		if (err)
+			brcmf_info("eap_restrict error (%d)\n", err);
+	}
 	err = brcmf_cfg80211_change_iface(wdev->wiphy, ndev, wdev->iftype,
 					  NULL);
 	if (err)
@@ -6654,6 +7345,13 @@
 
 	brcmf_configure_arp_nd_offload(ifp, true);
 
+	if (ifp->drvr->settings->frameburst) {
+		err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_FAKEFRAG, 1);
+			if (err)
+				brcmf_info("setting frameburst mode failed\n");
+		brcmf_dbg(INFO, "frameburst mode enabled\n");
+	}
+
 	cfg->dongle_up = true;
 default_conf_out:
 
@@ -6677,7 +7375,7 @@
 	 * from AP to save power
 	 */
 	if (check_vif_up(ifp->vif)) {
-		brcmf_link_down(ifp->vif, WLAN_REASON_UNSPECIFIED);
+		brcmf_link_down(ifp->vif, WLAN_REASON_UNSPECIFIED, true);
 
 		/* Make sure WPA_Supplicant receives all the event
 		   generated due to DISASSOC call to the fw to keep
@@ -6829,8 +7527,8 @@
 static void brcmf_cfg80211_reg_notifier(struct wiphy *wiphy,
 					struct regulatory_request *req)
 {
-	struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy);
-	struct brcmf_if *ifp = brcmf_get_ifp(cfg->pub, 0);
+	struct brcmf_cfg80211_info *cfg = wiphy_priv(wiphy);
+	struct brcmf_if *ifp = netdev_priv(cfg_to_ndev(cfg));
 	struct brcmf_fil_country_le ccreq;
 	s32 err;
 	int i;
@@ -6865,7 +7563,7 @@
 		brcmf_err("Firmware rejected country setting\n");
 		return;
 	}
-	brcmf_setup_wiphybands(cfg);
+	brcmf_setup_wiphybands(wiphy);
 }
 
 static void brcmf_free_wiphy(struct wiphy *wiphy)
@@ -6892,15 +7590,17 @@
 	if (wiphy->wowlan != &brcmf_wowlan_support)
 		kfree(wiphy->wowlan);
 #endif
+	wiphy_free(wiphy);
 }
 
 struct brcmf_cfg80211_info *brcmf_cfg80211_attach(struct brcmf_pub *drvr,
-						  struct cfg80211_ops *ops,
+						  struct device *busdev,
 						  bool p2pdev_forced)
 {
-	struct wiphy *wiphy = drvr->wiphy;
 	struct net_device *ndev = brcmf_get_ifp(drvr, 0)->ndev;
 	struct brcmf_cfg80211_info *cfg;
+	struct wiphy *wiphy;
+	struct cfg80211_ops *ops;
 	struct brcmf_cfg80211_vif *vif;
 	struct brcmf_if *ifp;
 	s32 err = 0;
@@ -6912,14 +7612,29 @@
 		return NULL;
 	}
 
-	cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
-	if (!cfg) {
-		brcmf_err("Could not allocate wiphy device\n");
+	ops = kmemdup(&brcmf_cfg80211_ops, sizeof(*ops), GFP_KERNEL);
+	if (!ops)
 		return NULL;
+
+	ifp = netdev_priv(ndev);
+#ifdef CONFIG_PM
+	if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_WOWL_GTK))
+		ops->set_rekey_data = brcmf_cfg80211_set_rekey_data;
+#endif
+	wiphy = wiphy_new(ops, sizeof(struct brcmf_cfg80211_info));
+	if (!wiphy) {
+		brcmf_err("Could not allocate wiphy device\n");
+		goto ops_out;
 	}
+	memcpy(wiphy->perm_addr, drvr->mac, ETH_ALEN);
+	set_wiphy_dev(wiphy, busdev);
 
+	cfg = wiphy_priv(wiphy);
 	cfg->wiphy = wiphy;
+	cfg->ops = ops;
 	cfg->pub = drvr;
+	cfg->pm_state = BRCMF_CFG80211_PM_STATE_RESUMED;
+	cfg->num_softap = 0;
 	init_vif_event(&cfg->vif_event);
 	INIT_LIST_HEAD(&cfg->vif_list);
 
@@ -6927,7 +7642,6 @@
 	if (IS_ERR(vif))
 		goto wiphy_out;
 
-	ifp = netdev_priv(ndev);
 	vif->ifp = ifp;
 	vif->wdev.netdev = ndev;
 	ndev->ieee80211_ptr = &vif->wdev;
@@ -6950,11 +7664,6 @@
 	cfg->d11inf.io_type = (u8)io_type;
 	brcmu_d11_attach(&cfg->d11inf);
 
-	/* regulatory notifer below needs access to cfg so
-	 * assign it now.
-	 */
-	drvr->config = cfg;
-
 	err = brcmf_setup_wiphy(wiphy, ifp);
 	if (err < 0)
 		goto priv_out;
@@ -6972,17 +7681,13 @@
 		cap = &wiphy->bands[NL80211_BAND_2GHZ]->ht_cap.cap;
 		*cap |= IEEE80211_HT_CAP_SUP_WIDTH_20_40;
 	}
-#ifdef CONFIG_PM
-	if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_WOWL_GTK))
-		ops->set_rekey_data = brcmf_cfg80211_set_rekey_data;
-#endif
 	err = wiphy_register(wiphy);
 	if (err < 0) {
 		brcmf_err("Could not register wiphy device (%d)\n", err);
 		goto priv_out;
 	}
 
-	err = brcmf_setup_wiphybands(cfg);
+	err = brcmf_setup_wiphybands(wiphy);
 	if (err) {
 		brcmf_err("Setting wiphy bands failed (%d)\n", err);
 		goto wiphy_unreg_out;
@@ -6999,7 +7704,12 @@
 		else
 			*cap &= ~IEEE80211_HT_CAP_SUP_WIDTH_20_40;
 	}
-
+	/* p2p might require that "if-events" get processed by fweh. So
+	 * activate the already registered event handlers now and activate
+	 * the rest when initialization has completed. drvr->config needs to
+	 * be assigned before activating events.
+	 */
+	drvr->config = cfg;
 	err = brcmf_fweh_activate_events(ifp);
 	if (err) {
 		brcmf_err("FWEH activation failed (%d)\n", err);
@@ -7067,7 +7777,8 @@
 	ifp->vif = NULL;
 wiphy_out:
 	brcmf_free_wiphy(wiphy);
-	kfree(cfg);
+ops_out:
+	kfree(ops);
 	return NULL;
 }
 
@@ -7082,5 +7793,4 @@
 	kfree(cfg->ops);
 	wl_deinit_priv(cfg);
 	brcmf_free_wiphy(cfg->wiphy);
-	kfree(cfg);
 }
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h	2020-06-19 21:00:31.416227715 -0400
@@ -34,6 +34,23 @@
 #define WL_ROAM_TRIGGER_LEVEL		-75
 #define WL_ROAM_DELTA			20
 
+/* WME Access Category Indices (ACIs) */
+#define AC_BE			0	/* Best Effort */
+#define AC_BK			1	/* Background */
+#define AC_VI			2	/* Video */
+#define AC_VO			3	/* Voice */
+#define EDCF_AC_COUNT		4
+#define MAX_8021D_PRIO		8
+
+#define EDCF_ACI_MASK			0x60
+#define EDCF_ACI_SHIFT			5
+#define EDCF_ACM_MASK                  0x10
+#define EDCF_ECWMIN_MASK		0x0f
+#define EDCF_ECWMAX_SHIFT		4
+#define EDCF_AIFSN_MASK			0x0f
+#define EDCF_AIFSN_MAX			15
+#define EDCF_ECWMAX_MASK		0xf0
+
 /* Keep BRCMF_ESCAN_BUF_SIZE below 64K (65536). Allocing over 64K can be
  * problematic on some systems and should be avoided.
  */
@@ -86,6 +103,15 @@
 
 #define BRCMF_VIF_EVENT_TIMEOUT		msecs_to_jiffies(1500)
 
+#define BRCMF_PM_WAIT_MAXRETRY			100
+
+/* cfg80211 wowlan definitions */
+#define WL_WOWLAN_MAX_PATTERNS			8
+#define WL_WOWLAN_MIN_PATTERN_LEN		1
+#define WL_WOWLAN_MAX_PATTERN_LEN		255
+#define WL_WOWLAN_PKT_FILTER_ID_FIRST	201
+#define WL_WOWLAN_PKT_FILTER_ID_LAST	(WL_WOWLAN_PKT_FILTER_ID_FIRST + \
+					WL_WOWLAN_MAX_PATTERNS - 1)
 /**
  * enum brcmf_scan_status - scan engine status
  *
@@ -118,7 +144,21 @@
 enum brcmf_profile_fwsup {
 	BRCMF_PROFILE_FWSUP_NONE,
 	BRCMF_PROFILE_FWSUP_PSK,
-	BRCMF_PROFILE_FWSUP_1X
+	BRCMF_PROFILE_FWSUP_1X,
+	BRCMF_PROFILE_FWSUP_SAE
+};
+
+/**
+ * enum brcmf_profile_fwauth - firmware authenticator profile
+ *
+ * @BRCMF_PROFILE_FWAUTH_NONE: no firmware authenticator
+ * @BRCMF_PROFILE_FWAUTH_PSK: authenticator for WPA/WPA2-PSK
+ * @BRCMF_PROFILE_FWAUTH_PSK: authenticator for SAE
+ */
+enum brcmf_profile_fwauth {
+	BRCMF_PROFILE_FWAUTH_NONE,
+	BRCMF_PROFILE_FWAUTH_PSK,
+	BRCMF_PROFILE_FWAUTH_SAE
 };
 
 /**
@@ -133,6 +173,9 @@
 	struct brcmf_cfg80211_security sec;
 	struct brcmf_wsec_key key[BRCMF_MAX_DEFAULT_KEYS];
 	enum brcmf_profile_fwsup use_fwsup;
+	u16 use_fwauth;
+	bool is_ft;
+	bool is_okc;
 };
 
 /**
@@ -156,25 +199,36 @@
 	BRCMF_VIF_STATUS_ASSOC_SUCCESS,
 };
 
+enum brcmf_cfg80211_pm_state {
+	BRCMF_CFG80211_PM_STATE_RESUMED,
+	BRCMF_CFG80211_PM_STATE_RESUMING,
+	BRCMF_CFG80211_PM_STATE_SUSPENDED,
+	BRCMF_CFG80211_PM_STATE_SUSPENDING,
+};
+
 /**
  * struct vif_saved_ie - holds saved IEs for a virtual interface.
  *
  * @probe_req_ie: IE info for probe request.
  * @probe_res_ie: IE info for probe response.
  * @beacon_ie: IE info for beacon frame.
+ * @assoc_res_ie: IE info for association response frame.
  * @probe_req_ie_len: IE info length for probe request.
  * @probe_res_ie_len: IE info length for probe response.
  * @beacon_ie_len: IE info length for beacon frame.
+ * @assoc_res_ie_len: IE info length for association response frame.
  */
 struct vif_saved_ie {
 	u8  probe_req_ie[IE_MAX_LEN];
 	u8  probe_res_ie[IE_MAX_LEN];
 	u8  beacon_ie[IE_MAX_LEN];
 	u8  assoc_req_ie[IE_MAX_LEN];
+	u8  assoc_res_ie[IE_MAX_LEN];
 	u32 probe_req_ie_len;
 	u32 probe_res_ie_len;
 	u32 beacon_ie_len;
 	u32 assoc_req_ie_len;
+	u32 assoc_res_ie_len;
 };
 
 /**
@@ -214,6 +268,12 @@
 	__le32 resp_len;
 };
 
+struct brcmf_cfg80211_edcf_acparam {
+	u8 ACI;
+	u8 ECW;
+	u16 TXOP;        /* stored in network order (ls octet first) */
+};
+
 /* dongle escan state */
 enum wl_escan_state {
 	WL_ESCAN_STATE_IDLE,
@@ -283,6 +343,7 @@
  * @scan_status: scan activity on the dongle.
  * @pub: common driver information.
  * @channel: current channel.
+ * @active_scan: current scan mode.
  * @int_escan_map: bucket map for which internal e-scan is done.
  * @ibss_starter: indicates this sta is ibss starter.
  * @pwr_save: indicate whether dongle to support power save mode.
@@ -315,6 +376,7 @@
 	unsigned long scan_status;
 	struct brcmf_pub *pub;
 	u32 channel;
+	bool active_scan;
 	u32 int_escan_map;
 	bool ibss_starter;
 	bool pwr_save;
@@ -333,6 +395,9 @@
 	struct brcmf_assoclist_le assoclist;
 	struct brcmf_cfg80211_wowl wowl;
 	struct brcmf_pno_info *pno;
+	u8 ac_priority[MAX_8021D_PRIO];
+	u8 pm_state;
+	u8 num_softap;
 };
 
 /**
@@ -355,24 +420,20 @@
 
 static inline struct brcmf_cfg80211_info *wiphy_to_cfg(struct wiphy *w)
 {
-	struct brcmf_pub *drvr = wiphy_priv(w);
-	return drvr->config;
+	return (struct brcmf_cfg80211_info *)(wiphy_priv(w));
 }
 
 static inline struct brcmf_cfg80211_info *wdev_to_cfg(struct wireless_dev *wd)
 {
-	return wiphy_to_cfg(wd->wiphy);
-}
-
-static inline struct brcmf_cfg80211_vif *wdev_to_vif(struct wireless_dev *wdev)
-{
-	return container_of(wdev, struct brcmf_cfg80211_vif, wdev);
+	return (struct brcmf_cfg80211_info *)(wdev_priv(wd));
 }
 
 static inline
 struct net_device *cfg_to_ndev(struct brcmf_cfg80211_info *cfg)
 {
-	return brcmf_get_ifp(cfg->pub, 0)->ndev;
+	struct brcmf_cfg80211_vif *vif;
+	vif = list_first_entry(&cfg->vif_list, struct brcmf_cfg80211_vif, list);
+	return vif->wdev.netdev;
 }
 
 static inline struct brcmf_cfg80211_info *ndev_to_cfg(struct net_device *ndev)
@@ -399,12 +460,11 @@
 }
 
 struct brcmf_cfg80211_info *brcmf_cfg80211_attach(struct brcmf_pub *drvr,
-						  struct cfg80211_ops *ops,
+						  struct device *busdev,
 						  bool p2pdev_forced);
 void brcmf_cfg80211_detach(struct brcmf_cfg80211_info *cfg);
 s32 brcmf_cfg80211_up(struct net_device *ndev);
 s32 brcmf_cfg80211_down(struct net_device *ndev);
-struct cfg80211_ops *brcmf_cfg80211_get_ops(struct brcmf_mp_device *settings);
 enum nl80211_iftype brcmf_cfg80211_get_iftype(struct brcmf_if *ifp);
 
 struct brcmf_cfg80211_vif *brcmf_alloc_vif(struct brcmf_cfg80211_info *cfg,
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c	2020-06-19 21:00:31.416227715 -0400
@@ -165,6 +165,7 @@
 #define SRCI_LSS_MASK		0x00f00000
 #define SRCI_LSS_SHIFT		20
 #define	SRCI_SRNB_MASK		0xf0
+#define	SRCI_SRNB_MASK_EXT	0x100
 #define	SRCI_SRNB_SHIFT		4
 #define	SRCI_SRBSZ_MASK		0xf
 #define	SRCI_SRBSZ_SHIFT	0
@@ -223,6 +224,24 @@
 #define	ARMCR4_BSZ_MASK		0x3f
 #define	ARMCR4_BSZ_MULT		8192
 
+/* Minimum PMU resource mask for 43012C0 */
+#define CY_43012_PMU_MIN_RES_MASK       0xF8BFE77
+
+/* PMU STATUS mask for 43012C0 */
+#define CY_43012_PMU_STATUS_MASK        0x1AC
+
+/* PMU CONTROL EXT mask for 43012C0 */
+#define CY_43012_PMU_CONTROL_EXT_MASK   0x11
+
+/* PMU Watchdog Counter Tick value for 43012C0 */
+#define CY_43012_PMU_WATCHDOG_TICK_VAL  0x04
+
+/* PMU Watchdog Counter Tick value for 4373 */
+#define CY_4373_PMU_WATCHDOG_TICK_VAL  0x04
+
+/* Minimum PMU resource mask for 4373 */
+#define CY_4373_PMU_MIN_RES_MASK       0xFCAFF7F
+
 struct brcmf_core_priv {
 	struct brcmf_core pub;
 	u32 wrapbase;
@@ -443,11 +462,25 @@
 {
 	struct brcmf_chip_priv *ci;
 	int count;
+	struct brcmf_core *d11core2 = NULL;
+	struct brcmf_core_priv *d11priv2 = NULL;
 
 	ci = core->chip;
 
+	/* special handle two D11 cores reset */
+	if (core->pub.id == BCMA_CORE_80211) {
+		d11core2 = brcmf_chip_get_d11core(&ci->pub, 1);
+		if (d11core2) {
+			brcmf_dbg(INFO, "found two d11 cores, reset both\n");
+			d11priv2 = container_of(d11core2, struct brcmf_core_priv,
+						 pub);
+		}
+	}
+
 	/* must disable first to work for arbitrary current core state */
 	brcmf_chip_ai_coredisable(core, prereset, reset);
+	if (d11priv2)
+		brcmf_chip_ai_coredisable(d11priv2, prereset, reset);
 
 	count = 0;
 	while (ci->ops->read32(ci->ctx, core->wrapbase + BCMA_RESET_CTL) &
@@ -459,17 +492,38 @@
 		usleep_range(40, 60);
 	}
 
+	if (d11priv2) {
+		count = 0;
+		while (ci->ops->read32(ci->ctx,
+				       d11priv2->wrapbase + BCMA_RESET_CTL) &
+				       BCMA_RESET_CTL_RESET) {
+			ci->ops->write32(ci->ctx,
+					 d11priv2->wrapbase + BCMA_RESET_CTL,
+					 0);
+			count++;
+			if (count > 50)
+				break;
+			usleep_range(40, 60);
+		}
+	}
+
 	ci->ops->write32(ci->ctx, core->wrapbase + BCMA_IOCTL,
 			 postreset | BCMA_IOCTL_CLK);
 	ci->ops->read32(ci->ctx, core->wrapbase + BCMA_IOCTL);
+
+	if (d11priv2) {
+		ci->ops->write32(ci->ctx, d11priv2->wrapbase + BCMA_IOCTL,
+				 postreset | BCMA_IOCTL_CLK);
+		ci->ops->read32(ci->ctx, d11priv2->wrapbase + BCMA_IOCTL);
+	}
 }
 
-char *brcmf_chip_name(u32 id, u32 rev, char *buf, uint len)
+static char *brcmf_chip_name(uint chipid, char *buf, uint len)
 {
 	const char *fmt;
 
-	fmt = ((id > 0xa000) || (id < 0x4000)) ? "BCM%d/%u" : "BCM%x/%u";
-	snprintf(buf, len, fmt, id, rev);
+	fmt = ((chipid > 0xa000) || (chipid < 0x4000)) ? "%d" : "%x";
+	snprintf(buf, len, fmt, chipid);
 	return buf;
 }
 
@@ -592,7 +646,13 @@
 		if (lss != 0)
 			*ramsize += (1 << ((lss - 1) + SR_BSZ_BASE));
 	} else {
-		nb = (coreinfo & SRCI_SRNB_MASK) >> SRCI_SRNB_SHIFT;
+		/* length of SRAM Banks increased for corerev greater than 23 */
+		if (sr->pub.rev >= 23) {
+			nb = (coreinfo & (SRCI_SRNB_MASK | SRCI_SRNB_MASK_EXT))
+				>> SRCI_SRNB_SHIFT;
+		} else {
+			nb = (coreinfo & SRCI_SRNB_MASK) >> SRCI_SRNB_SHIFT;
+		}
 		for (i = 0; i < nb; i++) {
 			retent = brcmf_chip_socram_banksize(sr, i, &banksize);
 			*ramsize += banksize;
@@ -689,10 +749,11 @@
 	case BRCM_CC_43525_CHIP_ID:
 	case BRCM_CC_4365_CHIP_ID:
 	case BRCM_CC_4366_CHIP_ID:
-	case BRCM_CC_43664_CHIP_ID:
 		return 0x200000;
 	case CY_CC_4373_CHIP_ID:
 		return 0x160000;
+	case CY_CC_89459_CHIP_ID:
+		return ((ci->pub.chiprev < 9) ? 0x180000 : 0x160000);
 	default:
 		brcmf_err("unknown chip: %s\n", ci->pub.name);
 		break;
@@ -779,7 +840,7 @@
 				      u32 *regbase, u32 *wrapbase)
 {
 	u8 desc;
-	u32 val;
+	u32 val, szdesc;
 	u8 mpnum = 0;
 	u8 stype, sztype, wraptype;
 
@@ -825,14 +886,15 @@
 
 		/* next size descriptor can be skipped */
 		if (sztype == DMP_SLAVE_SIZE_DESC) {
-			val = brcmf_chip_dmp_get_desc(ci, eromaddr, NULL);
+			szdesc = brcmf_chip_dmp_get_desc(ci, eromaddr, NULL);
 			/* skip upper size descriptor if present */
-			if (val & DMP_DESC_ADDRSIZE_GT32)
+			if (szdesc & DMP_DESC_ADDRSIZE_GT32)
 				brcmf_chip_dmp_get_desc(ci, eromaddr, NULL);
 		}
 
-		/* only look for 4K register regions */
-		if (sztype != DMP_SLAVE_SIZE_4K)
+		/* look for 4K or 8K register regions */
+		if (sztype != DMP_SLAVE_SIZE_4K &&
+		    sztype != DMP_SLAVE_SIZE_8K)
 			continue;
 
 		stype = (val & DMP_SLAVE_TYPE) >> DMP_SLAVE_TYPE_S;
@@ -889,7 +951,8 @@
 
 		/* need core with ports */
 		if (nmw + nsw == 0 &&
-		    id != BCMA_CORE_PMU)
+		    id != BCMA_CORE_PMU &&
+		    id != BCMA_CORE_GCI)
 			continue;
 
 		/* try to obtain register address info */
@@ -925,10 +988,10 @@
 	ci->pub.chiprev = (regdata & CID_REV_MASK) >> CID_REV_SHIFT;
 	socitype = (regdata & CID_TYPE_MASK) >> CID_TYPE_SHIFT;
 
-	brcmf_chip_name(ci->pub.chip, ci->pub.chiprev,
-			ci->pub.name, sizeof(ci->pub.name));
-	brcmf_dbg(INFO, "found %s chip: %s\n",
-		  socitype == SOCI_SB ? "SB" : "AXI", ci->pub.name);
+	brcmf_chip_name(ci->pub.chip, ci->pub.name, sizeof(ci->pub.name));
+	brcmf_dbg(INFO, "found %s chip: BCM%s, rev=%d\n",
+		  socitype == SOCI_SB ? "SB" : "AXI", ci->pub.name,
+		  ci->pub.chiprev);
 
 	if (socitype == SOCI_SB) {
 		if (ci->pub.chip != BRCM_CC_4329_CHIP_ID) {
@@ -1113,6 +1176,21 @@
 	kfree(chip);
 }
 
+struct brcmf_core *brcmf_chip_get_d11core(struct brcmf_chip *pub, u8 unit)
+{
+	struct brcmf_chip_priv *chip;
+	struct brcmf_core_priv *core;
+
+	chip = container_of(pub, struct brcmf_chip_priv, pub);
+	list_for_each_entry(core, &chip->cores, list) {
+		if (core->pub.id == BCMA_CORE_80211) {
+			if (unit-- == 0)
+				return &core->pub;
+		}
+	}
+	return NULL;
+}
+
 struct brcmf_core *brcmf_chip_get_core(struct brcmf_chip *pub, u16 coreid)
 {
 	struct brcmf_chip_priv *chip;
@@ -1155,6 +1233,14 @@
 	return cc;
 }
 
+struct brcmf_core *brcmf_chip_get_gci(struct brcmf_chip *pub)
+{
+	struct brcmf_core *gci;
+
+	gci = brcmf_chip_get_core(pub, BCMA_CORE_GCI);
+	return gci;
+}
+
 bool brcmf_chip_iscoreup(struct brcmf_core *pub)
 {
 	struct brcmf_core_priv *core;
@@ -1323,7 +1409,7 @@
 
 bool brcmf_chip_sr_capable(struct brcmf_chip *pub)
 {
-	u32 base, addr, reg, pmu_cc3_mask = ~0;
+	u32 base, addr, reg, sr_eng_en, pmu_cc3_mask = ~0;
 	struct brcmf_chip_priv *chip;
 	struct brcmf_core *pmu = brcmf_chip_get_pmu(pub);
 
@@ -1356,6 +1442,24 @@
 		addr = CORE_CC_REG(base, sr_control1);
 		reg = chip->ops->read32(chip->ctx, addr);
 		return reg != 0;
+	case CY_CC_4373_CHIP_ID:
+		/* explicitly check SR engine enable bit */
+		sr_eng_en = BIT(0);
+		addr = CORE_CC_REG(base, sr_control0);
+		reg = chip->ops->read32(chip->ctx, addr);
+		return (reg & sr_eng_en) != 0;
+	case BRCM_CC_4359_CHIP_ID:
+	case CY_CC_43012_CHIP_ID:
+		addr = CORE_CC_REG(pmu->base, retention_ctl);
+		reg = chip->ops->read32(chip->ctx, addr);
+		return (reg & (PMU_RCTL_MACPHY_DISABLE_MASK |
+			       PMU_RCTL_LOGIC_DISABLE_MASK)) == 0;
+	case CY_CC_89459_CHIP_ID:
+		/* explicitly check SR engine enable bit */
+		sr_eng_en = BIT(0);
+		addr = CORE_CC_REG(base, sr_control0);
+		reg = chip->ops->read32(chip->ctx, addr);
+		return (reg & sr_eng_en) != 0;
 	default:
 		addr = CORE_CC_REG(pmu->base, pmucapabilities_ext);
 		reg = chip->ops->read32(chip->ctx, addr);
@@ -1368,3 +1472,151 @@
 			       PMU_RCTL_LOGIC_DISABLE_MASK)) == 0;
 	}
 }
+
+void brcmf_chip_reset_pmu_regs(struct brcmf_chip *pub)
+{
+	struct brcmf_chip_priv *chip;
+	u32 addr;
+	u32 base;
+
+	brcmf_dbg(TRACE, "Enter\n");
+
+	chip = container_of(pub, struct brcmf_chip_priv, pub);
+	base = brcmf_chip_get_pmu(pub)->base;
+
+	switch (pub->chip) {
+	case CY_CC_43012_CHIP_ID:
+		/* SW scratch */
+		addr = CORE_CC_REG(base, swscratch);
+		chip->ops->write32(chip->ctx, addr, 0);
+
+		/* PMU status */
+		addr = CORE_CC_REG(base, pmustatus);
+		chip->ops->write32(chip->ctx, addr,
+			CY_43012_PMU_STATUS_MASK);
+
+		/* PMU control ext */
+		addr = CORE_CC_REG(base, pmucontrol_ext);
+		chip->ops->write32(chip->ctx, addr,
+			CY_43012_PMU_CONTROL_EXT_MASK);
+		break;
+
+	default:
+		brcmf_err("Unsupported chip id\n");
+		break;
+	}
+}
+
+void brcmf_chip_set_default_min_res_mask(struct brcmf_chip *pub)
+{
+	struct brcmf_chip_priv *chip;
+	u32 addr;
+	u32 base;
+
+	brcmf_dbg(TRACE, "Enter\n");
+
+	chip = container_of(pub, struct brcmf_chip_priv, pub);
+	base = brcmf_chip_get_pmu(pub)->base;
+	switch (pub->chip) {
+	case CY_CC_43012_CHIP_ID:
+		addr = CORE_CC_REG(base, min_res_mask);
+		chip->ops->write32(chip->ctx, addr,
+			CY_43012_PMU_MIN_RES_MASK);
+		break;
+
+	default:
+		brcmf_err("Unsupported chip id\n");
+		break;
+	}
+}
+
+void brcmf_chip_ulp_reset_lhl_regs(struct brcmf_chip *pub)
+{
+	struct brcmf_chip_priv *chip;
+	u32 base;
+	u32 addr;
+
+	brcmf_dbg(TRACE, "Enter\n");
+
+	chip = container_of(pub, struct brcmf_chip_priv, pub);
+	base = brcmf_chip_get_gci(pub)->base;
+
+	/* LHL Top Level Power Sequence Control */
+	addr = CORE_GCI_REG(base, lhl_top_pwrseq_ctl_adr);
+	chip->ops->write32(chip->ctx, addr, 0);
+
+	/* GPIO Interrupt Enable0 */
+	addr = CORE_GCI_REG(base, gpio_int_en_port_adr[0]);
+	chip->ops->write32(chip->ctx, addr, 0);
+
+	/* GPIO Interrupt Status0 */
+	addr = CORE_GCI_REG(base, gpio_int_st_port_adr[0]);
+	chip->ops->write32(chip->ctx, addr, ~0);
+
+	/* WL ARM Timer0 Interrupt Mask */
+	addr = CORE_GCI_REG(base, lhl_wl_armtim0_intrp_adr);
+	chip->ops->write32(chip->ctx, addr, 0);
+
+	/* WL ARM Timer0 Interrupt Status */
+	addr = CORE_GCI_REG(base, lhl_wl_armtim0_st_adr);
+	chip->ops->write32(chip->ctx, addr, ~0);
+
+	/* WL ARM Timer */
+	addr = CORE_GCI_REG(base, lhl_wl_armtim0_adr);
+	chip->ops->write32(chip->ctx, addr, 0);
+
+	/* WL MAC Timer0 Interrupt Mask */
+	addr = CORE_GCI_REG(base, lhl_wl_mactim0_intrp_adr);
+	chip->ops->write32(chip->ctx, addr, 0);
+
+	/* WL MAC Timer0 Interrupt Status */
+	addr = CORE_GCI_REG(base, lhl_wl_mactim0_st_adr);
+	chip->ops->write32(chip->ctx, addr, ~0);
+
+	/* WL MAC TimerInt0 */
+	addr = CORE_GCI_REG(base, lhl_wl_mactim_int0_adr);
+	chip->ops->write32(chip->ctx, addr, 0x0);
+}
+
+void brcmf_chip_reset_watchdog(struct brcmf_chip *pub)
+{
+	struct brcmf_chip_priv *chip;
+	u32 base;
+	u32 addr;
+
+	brcmf_dbg(TRACE, "Enter\n");
+
+	chip = container_of(pub, struct brcmf_chip_priv, pub);
+	base = brcmf_chip_get_pmu(pub)->base;
+
+	switch (pub->chip) {
+	case CY_CC_43012_CHIP_ID:
+		addr = CORE_CC_REG(base, min_res_mask);
+		chip->ops->write32(chip->ctx, addr,
+			CY_43012_PMU_MIN_RES_MASK);
+		/* Watchdog res mask */
+		addr = CORE_CC_REG(base, watchdog_res_mask);
+		chip->ops->write32(chip->ctx, addr,
+			CY_43012_PMU_MIN_RES_MASK);
+		/* PMU watchdog */
+		addr = CORE_CC_REG(base, pmuwatchdog);
+		chip->ops->write32(chip->ctx, addr,
+			CY_43012_PMU_WATCHDOG_TICK_VAL);
+		break;
+	case CY_CC_4373_CHIP_ID:
+		addr = CORE_CC_REG(base, min_res_mask);
+		chip->ops->write32(chip->ctx, addr,
+			CY_4373_PMU_MIN_RES_MASK);
+		addr = CORE_CC_REG(base, watchdog_res_mask);
+		chip->ops->write32(chip->ctx, addr,
+			CY_4373_PMU_MIN_RES_MASK);
+		addr = CORE_CC_REG(base, pmuwatchdog);
+		chip->ops->write32(chip->ctx, addr,
+			CY_4373_PMU_WATCHDOG_TICK_VAL);
+		mdelay(100);
+		break;
+	default:
+		break;
+	}
+}
+
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h	2020-06-19 21:00:31.420227716 -0400
@@ -21,6 +21,9 @@
 #define CORE_CC_REG(base, field) \
 		(base + offsetof(struct chipcregs, field))
 
+#define CORE_GCI_REG(base, field) \
+		(base + offsetof(struct chipgciregs, field))
+
 /**
  * struct brcmf_chip - chip level information.
  *
@@ -45,7 +48,7 @@
 	u32 rambase;
 	u32 ramsize;
 	u32 srsize;
-	char name[12];
+	char name[8];
 };
 
 /**
@@ -93,6 +96,10 @@
 void brcmf_chip_set_passive(struct brcmf_chip *ci);
 bool brcmf_chip_set_active(struct brcmf_chip *ci, u32 rstvec);
 bool brcmf_chip_sr_capable(struct brcmf_chip *pub);
-char *brcmf_chip_name(u32 chipid, u32 chiprev, char *buf, uint len);
+void brcmf_chip_reset_watchdog(struct brcmf_chip *pub);
+void brcmf_chip_ulp_reset_lhl_regs(struct brcmf_chip *pub);
+void brcmf_chip_reset_pmu_regs(struct brcmf_chip *pub);
+void brcmf_chip_set_default_min_res_mask(struct brcmf_chip *pub);
+struct brcmf_core *brcmf_chip_get_d11core(struct brcmf_chip *pub, u8 unit);
 
 #endif /* BRCMF_AXIDMP_H */
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c	2020-06-19 21:00:31.420227716 -0400
@@ -30,12 +30,16 @@
 #include "common.h"
 #include "of.h"
 #include "firmware.h"
-#include "chip.h"
+#include "fweh.h"
+#include <brcm_hw_ids.h>
+#include "defs.h"
 
 MODULE_AUTHOR("Broadcom Corporation");
 MODULE_DESCRIPTION("Broadcom 802.11 wireless LAN fullmac driver.");
 MODULE_LICENSE("Dual BSD/GPL");
 
+const u8 ALLFFMAC[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
+
 #define BRCMF_DEFAULT_SCAN_CHANNEL_TIME	40
 #define BRCMF_DEFAULT_SCAN_UNASSOC_TIME	40
 
@@ -50,7 +54,7 @@
 
 /* Debug level configuration. See debug.h for bits, sysfs modifiable */
 int brcmf_msg_level;
-module_param_named(debug, brcmf_msg_level, int, 0600);
+module_param_named(debug, brcmf_msg_level, int, S_IRUSR | S_IWUSR);
 MODULE_PARM_DESC(debug, "Level of debug output");
 
 static int brcmf_p2p_enable;
@@ -63,7 +67,7 @@
 
 static char brcmf_firmware_path[BRCMF_FW_ALTPATH_LEN];
 module_param_string(alternative_fw_path, brcmf_firmware_path,
-		    BRCMF_FW_ALTPATH_LEN, 0400);
+		    BRCMF_FW_ALTPATH_LEN, S_IRUSR);
 MODULE_PARM_DESC(alternative_fw_path, "Alternative firmware path");
 
 static int brcmf_fcmode;
@@ -71,12 +75,24 @@
 MODULE_PARM_DESC(fcmode, "Mode of firmware signalled flow control");
 
 static int brcmf_roamoff;
-module_param_named(roamoff, brcmf_roamoff, int, 0400);
+module_param_named(roamoff, brcmf_roamoff, int, S_IRUSR);
 MODULE_PARM_DESC(roamoff, "Do not use internal roaming engine");
 
-static int brcmf_iapp_enable;
-module_param_named(iapp, brcmf_iapp_enable, int, 0);
-MODULE_PARM_DESC(iapp, "Enable partial support for the obsoleted Inter-Access Point Protocol");
+static int brcmf_eap_restrict;
+module_param_named(eap_restrict, brcmf_eap_restrict, int, 0400);
+MODULE_PARM_DESC(eap_restrict, "Block non-802.1X frames until auth finished");
+
+static int brcmf_sdio_wq_highpri;
+module_param_named(sdio_wq_highpri, brcmf_sdio_wq_highpri, int, 0);
+MODULE_PARM_DESC(sdio_wq_highpri, "SDIO workqueue is set to high priority");
+
+static int brcmf_frameburst;
+module_param_named(frameburst, brcmf_frameburst, int, 0);
+MODULE_PARM_DESC(frameburst, "Enable firmware frameburst feature");
+
+static int brcmf_max_pm;
+module_param_named(max_pm, brcmf_max_pm, int, 0);
+MODULE_PARM_DESC(max_pm, "Use max power management mode by default");
 
 #ifdef DEBUG
 /* always succeed brcmf_bus_started() */
@@ -127,9 +143,43 @@
 	return err;
 }
 
-static int brcmf_c_process_clm_blob(struct brcmf_if *ifp)
+static int brcmf_c_get_clm_name(struct brcmf_if *ifp, u8 *clm_name)
 {
 	struct brcmf_bus *bus = ifp->drvr->bus_if;
+	struct brcmf_rev_info *ri = &ifp->drvr->revinfo;
+	u8 fw_name[BRCMF_FW_NAME_LEN];
+	u8 *ptr;
+	size_t len;
+	s32 err;
+
+	memset(fw_name, 0, BRCMF_FW_NAME_LEN);
+	err = brcmf_bus_get_fwname(bus, ri->chipnum, ri->chiprev, fw_name);
+	if (err) {
+		brcmf_err("get firmware name failed (%d)\n", err);
+		goto done;
+	}
+
+	/* generate CLM blob file name */
+	ptr = strrchr(fw_name, '.');
+	if (!ptr) {
+		err = -ENOENT;
+		goto done;
+	}
+
+	len = ptr - fw_name + 1;
+	if (len + strlen(".clm_blob") > BRCMF_FW_NAME_LEN) {
+		err = -E2BIG;
+	} else {
+		strlcpy(clm_name, fw_name, len);
+		strlcat(clm_name, ".clm_blob", BRCMF_FW_NAME_LEN);
+	}
+done:
+	return err;
+}
+
+static int brcmf_c_process_clm_blob(struct brcmf_if *ifp)
+{
+	struct device *dev = ifp->drvr->bus_if->dev;
 	struct brcmf_dload_data_le *chunk_buf;
 	const struct firmware *clm = NULL;
 	u8 clm_name[BRCMF_FW_NAME_LEN];
@@ -142,16 +192,16 @@
 
 	brcmf_dbg(TRACE, "Enter\n");
 
-	memset(clm_name, 0, sizeof(clm_name));
-	err = brcmf_bus_get_fwname(bus, ".clm_blob", clm_name);
+	memset(clm_name, 0, BRCMF_FW_NAME_LEN);
+	err = brcmf_c_get_clm_name(ifp, clm_name);
 	if (err) {
 		brcmf_err("get CLM blob file name failed (%d)\n", err);
 		return err;
 	}
 
-	err = firmware_request_nowarn(&clm, clm_name, bus->dev);
+	err = request_firmware(&clm, clm_name, dev);
 	if (err) {
-		brcmf_info("no clm_blob available (err=%d), device may have limited channels available\n",
+		brcmf_info("no clm_blob available(err=%d), device may have limited channels available\n",
 			   err);
 		return 0;
 	}
@@ -203,35 +253,34 @@
 {
 	s8 eventmask[BRCMF_EVENTING_MASK_LEN];
 	u8 buf[BRCMF_DCMD_SMLEN];
-	struct brcmf_bus *bus;
 	struct brcmf_rev_info_le revinfo;
 	struct brcmf_rev_info *ri;
 	char *clmver;
 	char *ptr;
 	s32 err;
+	struct eventmsgs_ext *eventmask_msg = NULL;
+	u8 msglen;
+	struct brcmf_bus *bus = ifp->drvr->bus_if;
 
-	/* retreive mac address */
+	/* retrieve mac addresses */
 	err = brcmf_fil_iovar_data_get(ifp, "cur_etheraddr", ifp->mac_addr,
 				       sizeof(ifp->mac_addr));
 	if (err < 0) {
 		brcmf_err("Retreiving cur_etheraddr failed, %d\n", err);
 		goto done;
 	}
-	memcpy(ifp->drvr->wiphy->perm_addr, ifp->drvr->mac, ETH_ALEN);
 	memcpy(ifp->drvr->mac, ifp->mac_addr, sizeof(ifp->drvr->mac));
 
-	bus = ifp->drvr->bus_if;
-	ri = &ifp->drvr->revinfo;
-
 	err = brcmf_fil_cmd_data_get(ifp, BRCMF_C_GET_REVINFO,
 				     &revinfo, sizeof(revinfo));
+	ri = &ifp->drvr->revinfo;
 	if (err < 0) {
 		brcmf_err("retrieving revision info failed, %d\n", err);
-		strlcpy(ri->chipname, "UNKNOWN", sizeof(ri->chipname));
 	} else {
 		ri->vendorid = le32_to_cpu(revinfo.vendorid);
 		ri->deviceid = le32_to_cpu(revinfo.deviceid);
 		ri->radiorev = le32_to_cpu(revinfo.radiorev);
+		ri->chiprev = le32_to_cpu(revinfo.chiprev);
 		ri->corerev = le32_to_cpu(revinfo.corerev);
 		ri->boardid = le32_to_cpu(revinfo.boardid);
 		ri->boardvendor = le32_to_cpu(revinfo.boardvendor);
@@ -239,24 +288,15 @@
 		ri->driverrev = le32_to_cpu(revinfo.driverrev);
 		ri->ucoderev = le32_to_cpu(revinfo.ucoderev);
 		ri->bus = le32_to_cpu(revinfo.bus);
+		ri->chipnum = le32_to_cpu(revinfo.chipnum);
 		ri->phytype = le32_to_cpu(revinfo.phytype);
 		ri->phyrev = le32_to_cpu(revinfo.phyrev);
 		ri->anarev = le32_to_cpu(revinfo.anarev);
 		ri->chippkg = le32_to_cpu(revinfo.chippkg);
 		ri->nvramrev = le32_to_cpu(revinfo.nvramrev);
-
-		/* use revinfo if not known yet */
-		if (!bus->chip) {
-			bus->chip = le32_to_cpu(revinfo.chipnum);
-			bus->chiprev = le32_to_cpu(revinfo.chiprev);
-		}
 	}
 	ri->result = err;
 
-	if (bus->chip)
-		brcmf_chip_name(bus->chip, bus->chiprev,
-				ri->chipname, sizeof(ri->chipname));
-
 	/* Do any CLM downloading */
 	err = brcmf_c_process_clm_blob(ifp);
 	if (err < 0) {
@@ -277,7 +317,7 @@
 	strsep(&ptr, "\n");
 
 	/* Print fw version info */
-	brcmf_info("Firmware: %s %s\n", ri->chipname, buf);
+	brcmf_info("Firmware version = %s\n", buf);
 
 	/* locate firmware version number for ethtool */
 	ptr = strrchr(buf, ' ') + 1;
@@ -296,11 +336,18 @@
 		/* Replace all newline/linefeed characters with space
 		 * character
 		 */
-		strreplace(clmver, '\n', ' ');
+		ptr = clmver;
+		while ((ptr = strnchr(ptr, '\n', sizeof(buf))) != NULL)
+			*ptr = ' ';
 
 		brcmf_dbg(INFO, "CLM version = %s\n", clmver);
 	}
 
+	/* set apsta */
+	err = brcmf_fil_iovar_int_set(ifp, "apsta", 1);
+	if (err)
+		brcmf_info("failed setting apsta, %d\n", err);
+
 	/* set mpc */
 	err = brcmf_fil_iovar_int_set(ifp, "mpc", 1);
 	if (err) {
@@ -325,6 +372,41 @@
 		goto done;
 	}
 
+	/* Enable event_msg_ext specific to 43012 chip */
+	if (bus->chip == CY_CC_43012_CHIP_ID) {
+		/* Program event_msg_ext to support event larger than 128 */
+		msglen = (roundup(BRCMF_E_LAST, NBBY) / NBBY) +
+				  EVENTMSGS_EXT_STRUCT_SIZE;
+		/* Allocate buffer for eventmask_msg */
+		eventmask_msg = kzalloc(msglen, GFP_KERNEL);
+		if (!eventmask_msg) {
+			err = -ENOMEM;
+			goto done;
+		}
+
+		/* Read the current programmed event_msgs_ext */
+		eventmask_msg->ver = EVENTMSGS_VER;
+		eventmask_msg->len = roundup(BRCMF_E_LAST, NBBY) / NBBY;
+		err = brcmf_fil_iovar_data_get(ifp, "event_msgs_ext",
+					       eventmask_msg,
+					       msglen);
+
+		/* Enable ULP event */
+		brcmf_dbg(EVENT, "enable event ULP\n");
+		setbit(eventmask_msg->mask, BRCMF_E_ULP);
+
+		/* Write updated Event mask */
+		eventmask_msg->ver = EVENTMSGS_VER;
+		eventmask_msg->command = EVENTMSGS_SET_MASK;
+		eventmask_msg->len = (roundup(BRCMF_E_LAST, NBBY) / NBBY);
+
+		err = brcmf_fil_iovar_data_set(ifp, "event_msgs_ext",
+					       eventmask_msg, msglen);
+		if (err) {
+			brcmf_err("Set event_msgs_ext error (%d)\n", err);
+			goto done;
+		}
+	}
 	/* Setup default scan channel time */
 	err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_SCAN_CHANNEL_TIME,
 				    BRCMF_DEFAULT_SCAN_CHANNEL_TIME);
@@ -345,6 +427,15 @@
 
 	/* Enable tx beamforming, errors can be ignored (not supported) */
 	(void)brcmf_fil_iovar_int_set(ifp, "txbf", 1);
+
+	/* add unicast packet filter */
+	err = brcmf_pktfilter_add_remove(ifp->ndev,
+					 BRCMF_UNICAST_FILTER_NUM, true);
+	if (err)
+		brcmf_info("Add unicast filter error (%d)\n", err);
+
+	/* do bus specific preinit here */
+	err = brcmf_bus_preinit(ifp->drvr->bus_if);
 done:
 	return err;
 }
@@ -413,12 +504,15 @@
 	if (!settings)
 		return NULL;
 
-	/* start by using the module paramaters */
+	/* start by using the module parameters */
 	settings->p2p_enable = !!brcmf_p2p_enable;
 	settings->feature_disable = brcmf_feature_disable;
 	settings->fcmode = brcmf_fcmode;
 	settings->roamoff = !!brcmf_roamoff;
-	settings->iapp = !!brcmf_iapp_enable;
+	settings->eap_restrict = !!brcmf_eap_restrict;
+	settings->sdio_wq_highpri = !!brcmf_sdio_wq_highpri;
+	settings->frameburst = !!brcmf_frameburst;
+	settings->default_pm = !!brcmf_max_pm ? PM_MAX : PM_FAST;
 #ifdef DEBUG
 	settings->ignore_probe_fail = !!brcmf_ignore_probe_fail;
 #endif
@@ -492,17 +586,21 @@
 {
 	int err;
 
+	/* Initialize debug system first */
+	brcmf_debugfs_init();
+
 	/* Get the platform data (if available) for our devices */
 	err = platform_driver_probe(&brcmf_pd, brcmf_common_pd_probe);
 	if (err == -ENODEV)
 		brcmf_dbg(INFO, "No platform data available.\n");
 
-	/* Initialize global module paramaters */
+	/* Initialize global module parameters */
 	brcmf_mp_attach();
 
 	/* Continue the initialization by registering the different busses */
 	err = brcmf_core_init();
 	if (err) {
+		brcmf_debugfs_exit();
 		if (brcmfmac_pdata)
 			platform_driver_unregister(&brcmf_pd);
 	}
@@ -515,6 +613,7 @@
 	brcmf_core_exit();
 	if (brcmfmac_pdata)
 		platform_driver_unregister(&brcmf_pd);
+	brcmf_debugfs_exit();
 }
 
 module_init(brcmfmac_module_init);
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h	2020-06-19 21:00:31.420227716 -0400
@@ -19,6 +19,8 @@
 #include <linux/platform_data/brcmfmac.h>
 #include "fwil_types.h"
 
+extern const u8 ALLFFMAC[ETH_ALEN];
+
 #define BRCMF_FW_ALTPATH_LEN			256
 
 /* Definitions for the module global and device specific settings are defined
@@ -47,6 +49,10 @@
  * @feature_disable: Feature_disable bitmask.
  * @fcmode: FWS flow control.
  * @roamoff: Firmware roaming off?
+ * @eap_restrict: Not allow data tx/rx until 802.1X auth succeeds
+ * @sdio_wq_highpri: Tasks submitted to SDIO workqueue will run immediately.
+ * @frameburst: Firmware frame burst mode.
+ * @default_pm: default power management (PM) mode.
  * @ignore_probe_fail: Ignore probe failure.
  * @country_codes: If available, pointer to struct for translating country codes
  * @bus: Bus specific platform data. Only SDIO at the mmoment.
@@ -56,7 +62,11 @@
 	unsigned int	feature_disable;
 	int		fcmode;
 	bool		roamoff;
-	bool		iapp;
+	bool		eap_restrict;
+	int		sdio_dpc_prio;
+	bool		sdio_wq_highpri;
+	bool		frameburst;
+	int		default_pm;
 	bool		ignore_probe_fail;
 	struct brcmfmac_pd_cc *country_codes;
 	union {
@@ -74,4 +84,8 @@
 /* Sets dongle media info (drv_version, mac address). */
 int brcmf_c_preinit_dcmds(struct brcmf_if *ifp);
 
+u8 brcmf_map_prio_to_prec(void *cfg, u8 prio);
+
+u8 brcmf_map_prio_to_aci(void *cfg, u8 prio);
+
 #endif /* BRCMFMAC_COMMON_H */
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c	2020-06-19 21:00:31.420227716 -0400
@@ -21,7 +21,6 @@
 #include <net/cfg80211.h>
 #include <net/rtnetlink.h>
 #include <net/addrconf.h>
-#include <net/ieee80211_radiotap.h>
 #include <net/ipv6.h>
 #include <brcmu_utils.h>
 #include <brcmu_wifi.h>
@@ -72,43 +71,6 @@
 	return ifp;
 }
 
-void brcmf_configure_arp_nd_offload(struct brcmf_if *ifp, bool enable)
-{
-	s32 err;
-	u32 mode;
-
-	if (enable)
-		mode = BRCMF_ARP_OL_AGENT | BRCMF_ARP_OL_PEER_AUTO_REPLY;
-	else
-		mode = 0;
-
-	/* Try to set and enable ARP offload feature, this may fail, then it  */
-	/* is simply not supported and err 0 will be returned                 */
-	err = brcmf_fil_iovar_int_set(ifp, "arp_ol", mode);
-	if (err) {
-		brcmf_dbg(TRACE, "failed to set ARP offload mode to 0x%x, err = %d\n",
-			  mode, err);
-	} else {
-		err = brcmf_fil_iovar_int_set(ifp, "arpoe", enable);
-		if (err) {
-			brcmf_dbg(TRACE, "failed to configure (%d) ARP offload err = %d\n",
-				  enable, err);
-		} else {
-			brcmf_dbg(TRACE, "successfully configured (%d) ARP offload to 0x%x\n",
-				  enable, mode);
-		}
-	}
-
-	err = brcmf_fil_iovar_int_set(ifp, "ndoe", enable);
-	if (err) {
-		brcmf_dbg(TRACE, "failed to configure (%d) ND offload err = %d\n",
-			  enable, err);
-	} else {
-		brcmf_dbg(TRACE, "successfully configured (%d) ND offload to 0x%x\n",
-			  enable, mode);
-	}
-}
-
 static void _brcmf_set_multicast_list(struct work_struct *work)
 {
 	struct brcmf_if *ifp;
@@ -172,7 +134,6 @@
 	if (err < 0)
 		brcmf_err("Setting BRCMF_C_SET_PROMISC failed, %d\n",
 			  err);
-	brcmf_configure_arp_nd_offload(ifp, !cmd_value);
 }
 
 #if IS_ENABLED(CONFIG_IPV6)
@@ -231,37 +192,6 @@
 	schedule_work(&ifp->multicast_work);
 }
 
-/**
- * brcmf_skb_is_iapp - checks if skb is an IAPP packet
- *
- * @skb: skb to check
- */
-static bool brcmf_skb_is_iapp(struct sk_buff *skb)
-{
-	static const u8 iapp_l2_update_packet[6] __aligned(2) = {
-		0x00, 0x01, 0xaf, 0x81, 0x01, 0x00,
-	};
-	unsigned char *eth_data;
-#if !defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)
-	const u16 *a, *b;
-#endif
-
-	if (skb->len - skb->mac_len != 6 ||
-	    !is_multicast_ether_addr(eth_hdr(skb)->h_dest))
-		return false;
-
-	eth_data = skb_mac_header(skb) + ETH_HLEN;
-#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)
-	return !(((*(const u32 *)eth_data) ^ (*(const u32 *)iapp_l2_update_packet)) |
-		 ((*(const u16 *)(eth_data + 4)) ^ (*(const u16 *)(iapp_l2_update_packet + 4))));
-#else
-	a = (const u16 *)eth_data;
-	b = (const u16 *)iapp_l2_update_packet;
-
-	return !((a[0] ^ b[0]) | (a[1] ^ b[1]) | (a[2] ^ b[2]));
-#endif
-}
-
 static netdev_tx_t brcmf_netdev_start_xmit(struct sk_buff *skb,
 					   struct net_device *ndev)
 {
@@ -282,23 +212,6 @@
 		goto done;
 	}
 
-	/* Some recent Broadcom's firmwares disassociate STA when they receive
-	 * an 802.11f ADD frame. This behavior can lead to a local DoS security
-	 * issue. Attacker may trigger disassociation of any STA by sending a
-	 * proper Ethernet frame to the wireless interface.
-	 *
-	 * Moreover this feature may break AP interfaces in some specific
-	 * setups. This applies e.g. to the bridge with hairpin mode enabled and
-	 * IFLA_BRPORT_MCAST_TO_UCAST set. IAPP packet generated by a firmware
-	 * will get passed back to the wireless interface and cause immediate
-	 * disassociation of a just-connected STA.
-	 */
-	if (!drvr->settings->iapp && brcmf_skb_is_iapp(skb)) {
-		dev_kfree_skb(skb);
-		ret = -EINVAL;
-		goto done;
-	}
-
 	/* Make sure there's enough writeable headroom */
 	if (skb_headroom(skb) < drvr->hdrlen || skb_header_cloned(skb)) {
 		head_delta = max_t(int, drvr->hdrlen - skb_headroom(skb), 0);
@@ -374,15 +287,6 @@
 
 void brcmf_netif_rx(struct brcmf_if *ifp, struct sk_buff *skb)
 {
-	/* Most of Broadcom's firmwares send 802.11f ADD frame every time a new
-	 * STA connects to the AP interface. This is an obsoleted standard most
-	 * users don't use, so don't pass these frames up unless requested.
-	 */
-	if (!ifp->drvr->settings->iapp && brcmf_skb_is_iapp(skb)) {
-		brcmu_pkt_buf_free_skb(skb);
-		return;
-	}
-
 	if (skb->pkt_type == PACKET_MULTICAST)
 		ifp->ndev->stats.multicast++;
 
@@ -405,30 +309,6 @@
 		netif_rx_ni(skb);
 }
 
-void brcmf_netif_mon_rx(struct brcmf_if *ifp, struct sk_buff *skb)
-{
-	if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MONITOR_FMT_RADIOTAP)) {
-		/* Do nothing */
-	} else {
-		struct ieee80211_radiotap_header *radiotap;
-
-		/* TODO: use RX status to fill some radiotap data */
-		radiotap = skb_push(skb, sizeof(*radiotap));
-		memset(radiotap, 0, sizeof(*radiotap));
-		radiotap->it_len = cpu_to_le16(sizeof(*radiotap));
-
-		/* TODO: 4 bytes with receive status? */
-		skb->len -= 4;
-	}
-
-	skb->dev = ifp->ndev;
-	skb_reset_mac_header(skb);
-	skb->pkt_type = PACKET_OTHERHOST;
-	skb->protocol = htons(ETH_P_802_2);
-
-	brcmf_netif_rx(ifp, skb);
-}
-
 static int brcmf_rx_hdrpull(struct brcmf_pub *drvr, struct sk_buff *skb,
 			    struct brcmf_if **ifp)
 {
@@ -491,6 +371,11 @@
 	struct ethhdr *eh;
 	u16 type;
 
+	if (!ifp) {
+		brcmu_pkt_buf_free_skb(txp);
+		return;
+	}
+
 	eh = (struct ethhdr *)(txp->data);
 	type = ntohs(eh->h_proto);
 
@@ -534,8 +419,6 @@
 
 	brcmf_cfg80211_down(ndev);
 
-	brcmf_fil_iovar_data_set(ifp, "arp_hostip_clear", NULL, 0);
-
 	brcmf_net_setcarrier(ifp, false);
 
 	return 0;
@@ -784,17 +667,17 @@
 			 bool rtnl_locked)
 {
 	struct brcmf_if *ifp;
-	int ifidx;
 
 	ifp = drvr->iflist[bsscfgidx];
+	drvr->iflist[bsscfgidx] = NULL;
 	if (!ifp) {
 		brcmf_err("Null interface, bsscfgidx=%d\n", bsscfgidx);
 		return;
 	}
 	brcmf_dbg(TRACE, "Enter, bsscfgidx=%d, ifidx=%d\n", bsscfgidx,
 		  ifp->ifidx);
-	ifidx = ifp->ifidx;
-
+	if (drvr->if2bss[ifp->ifidx] == bsscfgidx)
+		drvr->if2bss[ifp->ifidx] = BRCMF_BSSIDX_INVALID;
 	if (ifp->ndev) {
 		if (bsscfgidx == 0) {
 			if (ifp->ndev->netdev_ops == &brcmf_netdev_ops_pri) {
@@ -822,10 +705,6 @@
 		brcmf_p2p_ifp_removed(ifp, rtnl_locked);
 		kfree(ifp);
 	}
-
-	drvr->iflist[bsscfgidx] = NULL;
-	if (drvr->if2bss[ifidx] == bsscfgidx)
-		drvr->if2bss[ifidx] = BRCMF_BSSIDX_INVALID;
 }
 
 void brcmf_remove_interface(struct brcmf_if *ifp, bool rtnl_locked)
@@ -1001,6 +880,63 @@
 }
 #endif
 
+int brcmf_attach(struct device *dev, struct brcmf_mp_device *settings)
+{
+	struct brcmf_pub *drvr = NULL;
+	int ret = 0;
+	int i;
+
+	brcmf_dbg(TRACE, "Enter\n");
+
+	/* Allocate primary brcmf_info */
+	drvr = kzalloc(sizeof(struct brcmf_pub), GFP_ATOMIC);
+	if (!drvr)
+		return -ENOMEM;
+
+	for (i = 0; i < ARRAY_SIZE(drvr->if2bss); i++)
+		drvr->if2bss[i] = BRCMF_BSSIDX_INVALID;
+
+	mutex_init(&drvr->proto_block);
+
+	/* Link to bus module */
+	drvr->hdrlen = 0;
+	drvr->bus_if = dev_get_drvdata(dev);
+	drvr->bus_if->drvr = drvr;
+	drvr->settings = settings;
+
+	/* attach debug facilities */
+	brcmf_debug_attach(drvr);
+
+	/* Attach and link in the protocol */
+	ret = brcmf_proto_attach(drvr);
+	if (ret != 0) {
+		brcmf_err("brcmf_prot_attach failed\n");
+		goto fail;
+	}
+
+	/* Attach to events important for core code */
+	brcmf_fweh_register(drvr, BRCMF_E_PSM_WATCHDOG,
+			    brcmf_psm_watchdog_notify);
+
+	/* attach firmware event handler */
+	brcmf_fweh_attach(drvr);
+
+	return ret;
+
+fail:
+	brcmf_detach(dev);
+
+	return ret;
+}
+
+int brcmf_fwlog_attach(struct device *dev)
+{
+	struct brcmf_bus *bus_if = dev_get_drvdata(dev);
+	struct brcmf_pub *drvr = bus_if->drvr;
+
+	return brcmf_debug_fwlog_init(drvr);
+}
+
 static int brcmf_revinfo_read(struct seq_file *s, void *data)
 {
 	struct brcmf_bus *bus_if = dev_get_drvdata(s->private);
@@ -1011,7 +947,8 @@
 	seq_printf(s, "vendorid: 0x%04x\n", ri->vendorid);
 	seq_printf(s, "deviceid: 0x%04x\n", ri->deviceid);
 	seq_printf(s, "radiorev: %s\n", brcmu_dotrev_str(ri->radiorev, drev));
-	seq_printf(s, "chip: %s\n", ri->chipname);
+	seq_printf(s, "chipnum: %u (%x)\n", ri->chipnum, ri->chipnum);
+	seq_printf(s, "chiprev: %u\n", ri->chiprev);
 	seq_printf(s, "chippkg: %u\n", ri->chippkg);
 	seq_printf(s, "corerev: %u\n", ri->corerev);
 	seq_printf(s, "boardid: 0x%04x\n", ri->boardid);
@@ -1030,10 +967,42 @@
 	return 0;
 }
 
-static int brcmf_bus_started(struct brcmf_pub *drvr, struct cfg80211_ops *ops)
+static void brcmf_core_bus_reset(struct work_struct *work)
+{
+	struct brcmf_pub *drvr = container_of(work, struct brcmf_pub,
+					      bus_reset);
+
+	brcmf_bus_reset(drvr->bus_if);
+}
+
+static ssize_t bus_reset_write(struct file *file, const char __user *user_buf,
+			       size_t count, loff_t *ppos)
+{
+	struct brcmf_pub *drvr = file->private_data;
+	u8 value;
+
+	if (kstrtou8_from_user(user_buf, count, 0, &value))
+		return -EINVAL;
+
+	if (value != 1)
+		return -EINVAL;
+
+	schedule_work(&drvr->bus_reset);
+
+	return count;
+}
+
+static const struct file_operations bus_reset_fops = {
+	.open	= simple_open,
+	.llseek	= no_llseek,
+	.write	= bus_reset_write,
+};
+
+int brcmf_bus_started(struct device *dev)
 {
 	int ret = -1;
-	struct brcmf_bus *bus_if = drvr->bus_if;
+	struct brcmf_bus *bus_if = dev_get_drvdata(dev);
+	struct brcmf_pub *drvr = bus_if->drvr;
 	struct brcmf_if *ifp;
 	struct brcmf_if *p2p_ifp;
 
@@ -1049,16 +1018,20 @@
 	/* signal bus ready */
 	brcmf_bus_change_state(bus_if, BRCMF_BUS_UP);
 
-	/* do bus specific preinit here */
-	ret = brcmf_bus_preinit(bus_if);
-	if (ret < 0)
-		goto fail;
-
 	/* Bus is ready, do any initialization */
 	ret = brcmf_c_preinit_dcmds(ifp);
 	if (ret < 0)
 		goto fail;
 
+	brcmf_debugfs_add_entry(drvr, "revinfo", brcmf_revinfo_read);
+
+	/* assure we have chipid before feature attach */
+	if (!bus_if->chip) {
+		bus_if->chip = drvr->revinfo.chipnum;
+		bus_if->chiprev = drvr->revinfo.chiprev;
+		brcmf_dbg(INFO, "firmware revinfo: chip %x (%d) rev %d\n",
+			  bus_if->chip, bus_if->chip, bus_if->chiprev);
+	}
 	brcmf_feat_attach(drvr);
 
 	ret = brcmf_proto_init_done(drvr);
@@ -1067,7 +1040,7 @@
 
 	brcmf_proto_add_if(drvr, ifp);
 
-	drvr->config = brcmf_cfg80211_attach(drvr, ops,
+	drvr->config = brcmf_cfg80211_attach(drvr, bus_if->dev,
 					     drvr->settings->p2p_enable);
 	if (drvr->config == NULL) {
 		ret = -ENOMEM;
@@ -1101,10 +1074,11 @@
 #endif
 #endif /* CONFIG_INET */
 
+	INIT_WORK(&drvr->bus_reset, brcmf_core_bus_reset);
+
 	/* populate debugfs */
-	brcmf_debugfs_add_entry(drvr, "revinfo", brcmf_revinfo_read);
-	brcmf_feat_debugfs_create(drvr);
-	brcmf_proto_debugfs_create(drvr);
+	debugfs_create_file("reset", 0600, brcmf_debugfs_get_devdir(drvr), drvr,
+			    &bus_reset_fops);
 
 	return 0;
 
@@ -1125,69 +1099,6 @@
 	return ret;
 }
 
-int brcmf_attach(struct device *dev, struct brcmf_mp_device *settings)
-{
-	struct wiphy *wiphy;
-	struct cfg80211_ops *ops;
-	struct brcmf_pub *drvr = NULL;
-	int ret = 0;
-	int i;
-
-	brcmf_dbg(TRACE, "Enter\n");
-
-	ops = brcmf_cfg80211_get_ops(settings);
-	if (!ops)
-		return -ENOMEM;
-
-	wiphy = wiphy_new(ops, sizeof(*drvr));
-	if (!wiphy)
-		return -ENOMEM;
-
-	set_wiphy_dev(wiphy, dev);
-	drvr = wiphy_priv(wiphy);
-	drvr->wiphy = wiphy;
-
-	for (i = 0; i < ARRAY_SIZE(drvr->if2bss); i++)
-		drvr->if2bss[i] = BRCMF_BSSIDX_INVALID;
-
-	mutex_init(&drvr->proto_block);
-
-	/* Link to bus module */
-	drvr->hdrlen = 0;
-	drvr->bus_if = dev_get_drvdata(dev);
-	drvr->bus_if->drvr = drvr;
-	drvr->settings = settings;
-
-	/* Attach and link in the protocol */
-	ret = brcmf_proto_attach(drvr);
-	if (ret != 0) {
-		brcmf_err("brcmf_prot_attach failed\n");
-		goto fail;
-	}
-
-	/* Attach to events important for core code */
-	brcmf_fweh_register(drvr, BRCMF_E_PSM_WATCHDOG,
-			    brcmf_psm_watchdog_notify);
-
-	/* attach firmware event handler */
-	brcmf_fweh_attach(drvr);
-
-	ret = brcmf_bus_started(drvr, ops);
-	if (ret != 0) {
-		brcmf_err("dongle is not responding: err=%d\n", ret);
-		goto fail;
-	}
-
-	drvr->config->ops = ops;
-	return 0;
-
-fail:
-	brcmf_detach(dev);
-	kfree(ops);
-
-	return ret;
-}
-
 void brcmf_bus_add_txhdrlen(struct device *dev, uint len)
 {
 	struct brcmf_bus *bus_if = dev_get_drvdata(dev);
@@ -1218,6 +1129,18 @@
 		brcmf_dbg(TRACE, "failed to create coredump\n");
 }
 
+void brcmf_fw_crashed(struct device *dev)
+{
+	struct brcmf_bus *bus_if = dev_get_drvdata(dev);
+	struct brcmf_pub *drvr = bus_if->drvr;
+
+	brcmf_err("Firmware has halted or crashed\n");
+
+	brcmf_dev_coredump(dev);
+
+	schedule_work(&drvr->bus_reset);
+}
+
 void brcmf_detach(struct device *dev)
 {
 	s32 i;
@@ -1244,26 +1167,19 @@
 
 	brcmf_bus_change_state(bus_if, BRCMF_BUS_DOWN);
 
-	brcmf_proto_detach_pre_delif(drvr);
-
-	if (drvr->mon_if) {
-		brcmf_net_detach(drvr->mon_if->ndev, false);
-		drvr->mon_if = NULL;
-	}
-
 	/* make sure primary interface removed last */
 	for (i = BRCMF_MAX_IFS-1; i > -1; i--)
 		brcmf_remove_interface(drvr->iflist[i], false);
 
 	brcmf_cfg80211_detach(drvr->config);
-	drvr->config = NULL;
 
 	brcmf_bus_stop(drvr->bus_if);
 
-	brcmf_proto_detach_post_delif(drvr);
+	brcmf_proto_detach(drvr);
 
+	brcmf_debug_detach(drvr);
 	bus_if->drvr = NULL;
-	wiphy_free(drvr->wiphy);
+	kfree(drvr);
 }
 
 s32 brcmf_iovar_data_set(struct device *dev, char *name, void *data, u32 len)
@@ -1287,8 +1203,10 @@
 				 !brcmf_get_pend_8021x_cnt(ifp),
 				 MAX_WAIT_FOR_8021X_TX);
 
-	if (!err)
+	if (!err) {
 		brcmf_err("Timed out waiting for no pending 802.1x packets\n");
+		atomic_set(&ifp->pend_8021x_cnt, 0);
+	}
 
 	return !err;
 }
@@ -1300,12 +1218,6 @@
 	int ifidx;
 
 	brcmf_dbg(TRACE, "%d -> %d\n", bus->state, state);
-
-	if (!drvr) {
-		brcmf_dbg(INFO, "ignoring transition, bus not attached yet\n");
-		return;
-	}
-
 	bus->state = state;
 
 	if (state == BRCMF_BUS_UP) {
@@ -1357,3 +1269,178 @@
 #endif
 }
 
+int
+brcmf_pktfilter_add_remove(struct net_device *ndev, int filter_num, bool add)
+{
+	struct brcmf_if *ifp =  netdev_priv(ndev);
+	struct brcmf_pub *drvr = ifp->drvr;
+	struct brcmf_pkt_filter_le *pkt_filter;
+	int filter_fixed_len = offsetof(struct brcmf_pkt_filter_le, u);
+	int pattern_fixed_len = offsetof(struct brcmf_pkt_filter_pattern_le,
+				  mask_and_pattern);
+	u16 mask_and_pattern[MAX_PKTFILTER_PATTERN_SIZE];
+	int buflen = 0;
+	int ret = 0;
+
+	brcmf_dbg(INFO, "%s packet filter number %d\n",
+		  (add ? "add" : "remove"), filter_num);
+
+	pkt_filter = kzalloc(sizeof(*pkt_filter) +
+			(MAX_PKTFILTER_PATTERN_SIZE * 2), GFP_ATOMIC);
+	if (!pkt_filter)
+		return -ENOMEM;
+
+	switch (filter_num) {
+	case BRCMF_UNICAST_FILTER_NUM:
+		pkt_filter->id = 100;
+		pkt_filter->type = 0;
+		pkt_filter->negate_match = 0;
+		pkt_filter->u.pattern.offset = 0;
+		pkt_filter->u.pattern.size_bytes = 1;
+		mask_and_pattern[0] = 0x0001;
+		break;
+	case BRCMF_BROADCAST_FILTER_NUM:
+		//filter_pattern = "101 0 0 0 0xFFFFFFFFFFFF 0xFFFFFFFFFFFF";
+		pkt_filter->id = 101;
+		pkt_filter->type = 0;
+		pkt_filter->negate_match = 0;
+		pkt_filter->u.pattern.offset = 0;
+		pkt_filter->u.pattern.size_bytes = 6;
+		mask_and_pattern[0] = 0xFFFF;
+		mask_and_pattern[1] = 0xFFFF;
+		mask_and_pattern[2] = 0xFFFF;
+		mask_and_pattern[3] = 0xFFFF;
+		mask_and_pattern[4] = 0xFFFF;
+		mask_and_pattern[5] = 0xFFFF;
+		break;
+	case BRCMF_MULTICAST4_FILTER_NUM:
+		//filter_pattern = "102 0 0 0 0xFFFFFF 0x01005E";
+		pkt_filter->id = 102;
+		pkt_filter->type = 0;
+		pkt_filter->negate_match = 0;
+		pkt_filter->u.pattern.offset = 0;
+		pkt_filter->u.pattern.size_bytes = 3;
+		mask_and_pattern[0] = 0xFFFF;
+		mask_and_pattern[1] = 0x01FF;
+		mask_and_pattern[2] = 0x5E00;
+		break;
+	case BRCMF_MULTICAST6_FILTER_NUM:
+		//filter_pattern = "103 0 0 0 0xFFFF 0x3333";
+		pkt_filter->id = 103;
+		pkt_filter->type = 0;
+		pkt_filter->negate_match = 0;
+		pkt_filter->u.pattern.offset = 0;
+		pkt_filter->u.pattern.size_bytes = 2;
+		mask_and_pattern[0] = 0xFFFF;
+		mask_and_pattern[1] = 0x3333;
+		break;
+	case BRCMF_MDNS_FILTER_NUM:
+		//filter_pattern = "104 0 0 0 0xFFFFFFFFFFFF 0x01005E0000FB";
+		pkt_filter->id = 104;
+		pkt_filter->type = 0;
+		pkt_filter->negate_match = 0;
+		pkt_filter->u.pattern.offset = 0;
+		pkt_filter->u.pattern.size_bytes = 6;
+		mask_and_pattern[0] = 0xFFFF;
+		mask_and_pattern[1] = 0xFFFF;
+		mask_and_pattern[2] = 0xFFFF;
+		mask_and_pattern[3] = 0x0001;
+		mask_and_pattern[4] = 0x005E;
+		mask_and_pattern[5] = 0xFB00;
+		break;
+	case BRCMF_ARP_FILTER_NUM:
+		//filter_pattern = "105 0 0 12 0xFFFF 0x0806";
+		pkt_filter->id = 105;
+		pkt_filter->type = 0;
+		pkt_filter->negate_match = 0;
+		pkt_filter->u.pattern.offset = 12;
+		pkt_filter->u.pattern.size_bytes = 2;
+		mask_and_pattern[0] = 0xFFFF;
+		mask_and_pattern[1] = 0x0608;
+		break;
+	case BRCMF_BROADCAST_ARP_FILTER_NUM:
+		//filter_pattern = "106 0 0 0
+		//0xFFFFFFFFFFFF0000000000000806
+		//0xFFFFFFFFFFFF0000000000000806";
+		pkt_filter->id = 106;
+		pkt_filter->type = 0;
+		pkt_filter->negate_match = 0;
+		pkt_filter->u.pattern.offset = 0;
+		pkt_filter->u.pattern.size_bytes = 14;
+		mask_and_pattern[0] = 0xFFFF;
+		mask_and_pattern[1] = 0xFFFF;
+		mask_and_pattern[2] = 0xFFFF;
+		mask_and_pattern[3] = 0x0000;
+		mask_and_pattern[4] = 0x0000;
+		mask_and_pattern[5] = 0x0000;
+		mask_and_pattern[6] = 0x0608;
+		mask_and_pattern[7] = 0xFFFF;
+		mask_and_pattern[8] = 0xFFFF;
+		mask_and_pattern[9] = 0xFFFF;
+		mask_and_pattern[10] = 0x0000;
+		mask_and_pattern[11] = 0x0000;
+		mask_and_pattern[12] = 0x0000;
+		mask_and_pattern[13] = 0x0608;
+		break;
+	default:
+		ret = -EINVAL;
+		goto failed;
+	}
+	memcpy(pkt_filter->u.pattern.mask_and_pattern, mask_and_pattern,
+	       pkt_filter->u.pattern.size_bytes * 2);
+	buflen = filter_fixed_len + pattern_fixed_len +
+		  pkt_filter->u.pattern.size_bytes * 2;
+
+	if (add) {
+		/* Add filter */
+		ret = brcmf_fil_iovar_data_set(ifp, "pkt_filter_add",
+					       pkt_filter, buflen);
+		if (ret)
+			goto failed;
+		drvr->pkt_filter[filter_num].id = pkt_filter->id;
+		drvr->pkt_filter[filter_num].enable  = 0;
+
+	} else {
+		/* Delete filter */
+		ret = brcmf_fil_iovar_int_set(ifp, "pkt_filter_delete",
+					      pkt_filter->id);
+		if (ret == -ENOENT)
+			ret = 0;
+		if (ret)
+			goto failed;
+
+		drvr->pkt_filter[filter_num].id = 0;
+		drvr->pkt_filter[filter_num].enable  = 0;
+
+	}
+failed:
+	if (ret)
+		brcmf_err("%s packet filter failed, ret=%d\n",
+			  (add ? "add" : "remove"), ret);
+
+	kfree(pkt_filter);
+	return ret;
+}
+
+int brcmf_pktfilter_enable(struct net_device *ndev, bool enable)
+{
+	struct brcmf_if *ifp =  netdev_priv(ndev);
+	struct brcmf_pub *drvr = ifp->drvr;
+	int ret = 0;
+	int idx = 0;
+
+	for (idx = 0; idx < MAX_PKT_FILTER_COUNT; ++idx) {
+		if (drvr->pkt_filter[idx].id != 0) {
+			drvr->pkt_filter[idx].enable = enable;
+			ret = brcmf_fil_iovar_data_set(ifp, "pkt_filter_enable",
+						       &drvr->pkt_filter[idx],
+				sizeof(struct brcmf_pkt_filter_enable_le));
+			if (ret) {
+				brcmf_err("%s packet filter id(%d) failed, ret=%d\n",
+					  (enable ? "enable" : "disable"),
+					  drvr->pkt_filter[idx].id, ret);
+			}
+		}
+	}
+	return ret;
+}
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h	2020-06-19 21:00:31.420227716 -0400
@@ -23,6 +23,7 @@
 
 #include <net/cfg80211.h>
 #include "fweh.h"
+#include "fwil_types.h"
 
 #define TOE_TX_CSUM_OL		0x00000001
 #define TOE_RX_CSUM_OL		0x00000002
@@ -36,7 +37,7 @@
 #define BRCMF_DCMD_MEDLEN	1536
 #define BRCMF_DCMD_MAXLEN	8192
 
-/* IOCTL from host to device are limited in lenght. A device can only handle
+/* IOCTL from host to device are limited in length. A device can only handle
  * ethernet frame size. This limitation is to be applied by protocol layer.
  */
 #define BRCMF_TX_IOCTL_MAX_MSG_SIZE	(ETH_FRAME_LEN+ETH_FCS_LEN)
@@ -87,6 +88,7 @@
 	u32 vendorid;
 	u32 deviceid;
 	u32 radiorev;
+	u32 chiprev;
 	u32 corerev;
 	u32 boardid;
 	u32 boardvendor;
@@ -94,7 +96,7 @@
 	u32 driverrev;
 	u32 ucoderev;
 	u32 bus;
-	char chipname[12];
+	u32 chipnum;
 	u32 phytype;
 	u32 phyrev;
 	u32 anarev;
@@ -107,7 +109,6 @@
 	/* Linkage ponters */
 	struct brcmf_bus *bus_if;
 	struct brcmf_proto *proto;
-	struct wiphy *wiphy;
 	struct brcmf_cfg80211_info *config;
 
 	/* Internal brcmf items */
@@ -121,7 +122,6 @@
 
 	struct brcmf_if *iflist[BRCMF_MAX_IFS];
 	s32 if2bss[BRCMF_MAX_IFS];
-	struct brcmf_if *mon_if;
 
 	struct mutex proto_block;
 	unsigned char proto_buf[BRCMF_DCMD_MAXLEN];
@@ -143,7 +143,11 @@
 	struct notifier_block inet6addr_notifier;
 	struct brcmf_mp_device *settings;
 
+	struct work_struct bus_reset;
+
 	u8 clmver[BRCMF_DCMD_SMLEN];
+	struct brcmf_pkt_filter_enable_le pkt_filter[MAX_PKT_FILTER_COUNT];
+
 };
 
 /* forward declarations */
@@ -182,7 +186,6 @@
  * @netif_stop_lock: spinlock for update netif_stop from multiple sources.
  * @pend_8021x_cnt: tracks outstanding number of 802.1x frames.
  * @pend_8021x_wait: used for signalling change in count.
- * @fwil_fwerr: flag indicating fwil layer should return firmware error codes.
  */
 struct brcmf_if {
 	struct brcmf_pub *drvr;
@@ -200,7 +203,6 @@
 	wait_queue_head_t pend_8021x_wait;
 	struct in6_addr ipv6_addr_tbl[NDOL_MAX_ENTRIES];
 	u8 ipv6addr_idx;
-	bool fwil_fwerr;
 };
 
 int brcmf_netdev_wait_pend8021x(struct brcmf_if *ifp);
@@ -208,7 +210,6 @@
 /* Return pointer to interface name */
 char *brcmf_ifname(struct brcmf_if *ifp);
 struct brcmf_if *brcmf_get_ifp(struct brcmf_pub *drvr, int ifidx);
-void brcmf_configure_arp_nd_offload(struct brcmf_if *ifp, bool enable);
 int brcmf_net_attach(struct brcmf_if *ifp, bool rtnl_locked);
 struct brcmf_if *brcmf_add_if(struct brcmf_pub *drvr, s32 bsscfgidx, s32 ifidx,
 			      bool is_p2pdev, const char *name, u8 *mac_addr);
@@ -217,9 +218,10 @@
 			  enum brcmf_netif_stop_reason reason, bool state);
 void brcmf_txfinalize(struct brcmf_if *ifp, struct sk_buff *txp, bool success);
 void brcmf_netif_rx(struct brcmf_if *ifp, struct sk_buff *skb);
-void brcmf_netif_mon_rx(struct brcmf_if *ifp, struct sk_buff *skb);
 void brcmf_net_setcarrier(struct brcmf_if *ifp, bool on);
 int __init brcmf_core_init(void);
 void __exit brcmf_core_exit(void);
-
+int brcmf_pktfilter_add_remove(struct net_device *ndev, int filter_num,
+			       bool add);
+int brcmf_pktfilter_enable(struct net_device *ndev, bool enable);
 #endif /* BRCMFMAC_CORE_H */
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c	2020-06-19 21:00:31.420227716 -0400
@@ -25,6 +25,84 @@
 #include "fweh.h"
 #include "debug.h"
 
+static struct dentry *root_folder;
+
+static int
+brcmf_debug_msgtrace_seqchk(u32 *prev, u32 cur)
+{
+	if ((cur == 0 && *prev == 0xFFFFFFFF) || ((cur - *prev) == 1)) {
+		goto done;
+	} else if (cur == *prev) {
+		brcmf_dbg(FWCON, "duplicate trace\n");
+		return -1;
+	} else if (cur > *prev) {
+		brcmf_dbg(FWCON, "lost %d packets\n", cur - *prev);
+	} else {
+		brcmf_dbg(FWCON, "seq out of order, host %d, dongle %d\n",
+			  *prev, cur);
+	}
+done:
+	*prev = cur;
+	return 0;
+}
+
+static int
+brcmf_debug_msg_parser(void *event_data)
+{
+	int err = 0;
+	struct msgtrace_hdr *hdr;
+	char *data, *s;
+	static u32 seqnum_prev;
+
+	hdr = (struct msgtrace_hdr *)event_data;
+	data = (char *)event_data + MSGTRACE_HDRLEN;
+
+	/* There are 2 bytes available at the end of data */
+	data[ntohs(hdr->len)] = '\0';
+
+	if (ntohl(hdr->discarded_bytes) || ntohl(hdr->discarded_printf)) {
+		brcmf_dbg(FWCON, "Discarded_bytes %d discarded_printf %d\n",
+			  ntohl(hdr->discarded_bytes),
+				ntohl(hdr->discarded_printf));
+	}
+
+	err = brcmf_debug_msgtrace_seqchk(&seqnum_prev, ntohl(hdr->seqnum));
+	if (err)
+		return err;
+
+	while (*data != '\0' && (s = strstr(data, "\n")) != NULL) {
+		*s = '\0';
+		brcmf_dbg(FWCON, "[FWLOG] %s\n", data);
+		data = s + 1;
+	}
+	if (*data)
+		brcmf_dbg(FWCON, "[FWLOG] %s", data);
+
+	return err;
+}
+
+static int
+brcmf_debug_trace_parser(struct brcmf_if *ifp,
+			 const struct brcmf_event_msg *evtmsg,
+			 void *event_data)
+{
+	int err = 0;
+	struct msgtrace_hdr *hdr;
+
+	hdr = (struct msgtrace_hdr *)event_data;
+	if (hdr->version != MSGTRACE_VERSION) {
+		brcmf_dbg(FWCON, "trace version mismatch host %d dngl %d\n",
+			  MSGTRACE_VERSION, hdr->version);
+		err = -EPROTO;
+		return err;
+	}
+
+	if (hdr->trace_type == MSGTRACE_HDR_TYPE_MSG)
+		err = brcmf_debug_msg_parser(event_data);
+
+	return err;
+}
+
 int brcmf_debug_create_memdump(struct brcmf_bus *bus, const void *data,
 			       size_t len)
 {
@@ -53,9 +131,50 @@
 	return 0;
 }
 
+void brcmf_debugfs_init(void)
+{
+	root_folder = debugfs_create_dir(KBUILD_MODNAME, NULL);
+	if (IS_ERR(root_folder))
+		root_folder = NULL;
+}
+
+void brcmf_debugfs_exit(void)
+{
+	if (!root_folder)
+		return;
+
+	debugfs_remove_recursive(root_folder);
+	root_folder = NULL;
+}
+
+int brcmf_debug_attach(struct brcmf_pub *drvr)
+{
+	struct device *dev = drvr->bus_if->dev;
+
+	if (!root_folder)
+		return -ENODEV;
+
+	drvr->dbgfs_dir = debugfs_create_dir(dev_name(dev), root_folder);
+	return PTR_ERR_OR_ZERO(drvr->dbgfs_dir);
+}
+
+int brcmf_debug_fwlog_init(struct brcmf_pub *drvr)
+{
+	return brcmf_fweh_register(drvr, BRCMF_E_TRACE,
+				brcmf_debug_trace_parser);
+}
+
+void brcmf_debug_detach(struct brcmf_pub *drvr)
+{
+	brcmf_fweh_unregister(drvr, BRCMF_E_PSM_WATCHDOG);
+
+	if (!IS_ERR_OR_NULL(drvr->dbgfs_dir))
+		debugfs_remove_recursive(drvr->dbgfs_dir);
+}
+
 struct dentry *brcmf_debugfs_get_devdir(struct brcmf_pub *drvr)
 {
-	return drvr->wiphy->debugfsdir;
+	return drvr->dbgfs_dir;
 }
 
 int brcmf_debugfs_add_entry(struct brcmf_pub *drvr, const char *fn,
@@ -63,8 +182,7 @@
 {
 	struct dentry *e;
 
-	WARN(!drvr->wiphy->debugfsdir, "wiphy not (yet) registered\n");
 	e = debugfs_create_devm_seqfile(drvr->bus_if->dev, fn,
-					drvr->wiphy->debugfsdir, read_fn);
+					drvr->dbgfs_dir, read_fn);
 	return PTR_ERR_OR_ZERO(e);
 }
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h	2020-06-19 21:00:31.420227716 -0400
@@ -40,6 +40,7 @@
 #define BRCMF_MSGBUF_VAL	0x00040000
 #define BRCMF_PCIE_VAL		0x00080000
 #define BRCMF_FWCON_VAL		0x00100000
+#define BRCMF_ULP_VAL		0x00200000
 
 /* set default print format */
 #undef pr_fmt
@@ -101,6 +102,10 @@
 
 #endif /* defined(DEBUG) || defined(CONFIG_BRCM_TRACING) */
 
+#define MSGTRACE_VERSION 1
+#define MSGTRACE_HDR_TYPE_MSG 0
+#define MSGTRACE_HDR_TYPE_LOG 1
+
 #define brcmf_dbg_hex_dump(test, data, len, fmt, ...)			\
 do {									\
 	trace_brcmf_hexdump((void *)data, len);				\
@@ -113,12 +118,35 @@
 struct brcmf_bus;
 struct brcmf_pub;
 #ifdef DEBUG
+void brcmf_debugfs_init(void);
+void brcmf_debugfs_exit(void);
+int brcmf_debug_attach(struct brcmf_pub *drvr);
+void brcmf_debug_detach(struct brcmf_pub *drvr);
 struct dentry *brcmf_debugfs_get_devdir(struct brcmf_pub *drvr);
 int brcmf_debugfs_add_entry(struct brcmf_pub *drvr, const char *fn,
 			    int (*read_fn)(struct seq_file *seq, void *data));
 int brcmf_debug_create_memdump(struct brcmf_bus *bus, const void *data,
 			       size_t len);
+int brcmf_debug_fwlog_init(struct brcmf_pub *drvr);
 #else
+static inline void brcmf_debugfs_init(void)
+{
+}
+static inline void brcmf_debugfs_exit(void)
+{
+}
+static inline int brcmf_debug_attach(struct brcmf_pub *drvr)
+{
+	return 0;
+}
+static inline void brcmf_debug_detach(struct brcmf_pub *drvr)
+{
+}
+
+static inline struct dentry *brcmf_debugfs_get_devdir(struct brcmf_pub *drvr)
+{
+	return ERR_PTR(-ENOENT);
+}
 static inline
 int brcmf_debugfs_add_entry(struct brcmf_pub *drvr, const char *fn,
 			    int (*read_fn)(struct seq_file *seq, void *data))
@@ -131,6 +159,25 @@
 {
 	return 0;
 }
+
+static inline
+int brcmf_debug_fwlog_init(struct brcmf_pub *drvr)
+{
+	return 0;
+}
 #endif
 
+/* Message trace header */
+struct msgtrace_hdr {
+	u8	version;
+	u8	trace_type;
+	u16	len;    /* Len of the trace */
+	u32	seqnum; /* Sequence number of message */
+	/* Number of discarded bytes because of trace overflow  */
+	u32	discarded_bytes;
+	/* Number of discarded printf because of trace overflow */
+	u32	discarded_printf;
+};
+
+#define MSGTRACE_HDRLEN		sizeof(struct msgtrace_hdr)
 #endif /* BRCMFMAC_DEBUG_H */
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c	2020-06-19 21:00:31.420227716 -0400
@@ -48,8 +48,8 @@
 	{ BRCMF_FEAT_MBSS, "mbss" },
 	{ BRCMF_FEAT_MCHAN, "mchan" },
 	{ BRCMF_FEAT_P2P, "p2p" },
-	{ BRCMF_FEAT_MONITOR, "monitor" },
-	{ BRCMF_FEAT_MONITOR_FMT_RADIOTAP, "rtap" },
+	{ BRCMF_FEAT_SAE, "sae" },
+	{ BRCMF_FEAT_FWAUTH, "idauth" },
 };
 
 #ifdef DEBUG
@@ -93,42 +93,6 @@
 }
 #endif /* DEBUG */
 
-struct brcmf_feat_fwfeat {
-	const char * const fwid;
-	u32 feat_flags;
-};
-
-static const struct brcmf_feat_fwfeat brcmf_feat_fwfeat_map[] = {
-	/* brcmfmac43602-pcie.ap.bin from linux-firmware.git commit ea1178515b88 */
-	{ "01-6cb8e269", BIT(BRCMF_FEAT_MONITOR) },
-	/* brcmfmac4366b-pcie.bin from linux-firmware.git commit 52442afee990 */
-	{ "01-c47a91a4", BIT(BRCMF_FEAT_MONITOR) },
-};
-
-static void brcmf_feat_firmware_overrides(struct brcmf_pub *drv)
-{
-	const struct brcmf_feat_fwfeat *e;
-	u32 feat_flags = 0;
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(brcmf_feat_fwfeat_map); i++) {
-		e = &brcmf_feat_fwfeat_map[i];
-		if (!strcmp(e->fwid, drv->fwver)) {
-			feat_flags = e->feat_flags;
-			break;
-		}
-	}
-
-	if (!feat_flags)
-		return;
-
-	for (i = 0; i < BRCMF_FEAT_LAST; i++)
-		if (feat_flags & BIT(i))
-			brcmf_dbg(INFO, "enabling firmware feature: %s\n",
-				  brcmf_feat_names[i]);
-	drv->feat_flags |= feat_flags;
-}
-
 /**
  * brcmf_feat_iovar_int_get() - determine feature through iovar query.
  *
@@ -142,9 +106,6 @@
 	u32 data;
 	int err;
 
-	/* we need to know firmware error */
-	ifp->fwil_fwerr = true;
-
 	err = brcmf_fil_iovar_int_get(ifp, name, &data);
 	if (err == 0) {
 		brcmf_dbg(INFO, "enabling feature: %s\n", brcmf_feat_names[id]);
@@ -153,8 +114,6 @@
 		brcmf_dbg(TRACE, "%s feature check failed: %d\n",
 			  brcmf_feat_names[id], err);
 	}
-
-	ifp->fwil_fwerr = false;
 }
 
 static void brcmf_feat_iovar_data_set(struct brcmf_if *ifp,
@@ -163,9 +122,6 @@
 {
 	int err;
 
-	/* we need to know firmware error */
-	ifp->fwil_fwerr = true;
-
 	err = brcmf_fil_iovar_data_set(ifp, name, data, len);
 	if (err != -BRCMF_FW_UNSUPPORTED) {
 		brcmf_dbg(INFO, "enabling feature: %s\n", brcmf_feat_names[id]);
@@ -174,14 +130,11 @@
 		brcmf_dbg(TRACE, "%s feature check failed: %d\n",
 			  brcmf_feat_names[id], err);
 	}
-
-	ifp->fwil_fwerr = false;
 }
 
-#define MAX_CAPS_BUFFER_SIZE	768
 static void brcmf_feat_firmware_capabilities(struct brcmf_if *ifp)
 {
-	char caps[MAX_CAPS_BUFFER_SIZE];
+	char caps[512];
 	enum brcmf_feat_id id;
 	int i, err;
 
@@ -203,41 +156,6 @@
 	}
 }
 
-/**
- * brcmf_feat_fwcap_debugfs_read() - expose firmware capabilities to debugfs.
- *
- * @seq: sequence for debugfs entry.
- * @data: raw data pointer.
- */
-static int brcmf_feat_fwcap_debugfs_read(struct seq_file *seq, void *data)
-{
-	struct brcmf_bus *bus_if = dev_get_drvdata(seq->private);
-	struct brcmf_if *ifp = brcmf_get_ifp(bus_if->drvr, 0);
-	char caps[MAX_CAPS_BUFFER_SIZE + 1] = { };
-	char *tmp;
-	int err;
-
-	err = brcmf_fil_iovar_data_get(ifp, "cap", caps, sizeof(caps));
-	if (err) {
-		brcmf_err("could not get firmware cap (%d)\n", err);
-		return err;
-	}
-
-	/* Put every capability in a new line */
-	for (tmp = caps; *tmp; tmp++) {
-		if (*tmp == ' ')
-			*tmp = '\n';
-	}
-
-	/* Usually there is a space at the end of capabilities string */
-	seq_printf(seq, "%s", caps);
-	/* So make sure we don't print two line breaks */
-	if (tmp > caps && *(tmp - 1) != '\n')
-		seq_printf(seq, "\n");
-
-	return 0;
-}
-
 void brcmf_feat_attach(struct brcmf_pub *drvr)
 {
 	struct brcmf_if *ifp = brcmf_get_ifp(drvr, 0);
@@ -289,8 +207,6 @@
 	}
 	brcmf_feat_iovar_int_get(ifp, BRCMF_FEAT_FWSUP, "sup_wpa");
 
-	brcmf_feat_firmware_overrides(drvr);
-
 	/* set chip related quirks */
 	switch (drvr->bus_if->chip) {
 	case BRCM_CC_43236_CHIP_ID:
@@ -303,12 +219,8 @@
 		/* no quirks */
 		break;
 	}
-}
 
-void brcmf_feat_debugfs_create(struct brcmf_pub *drvr)
-{
 	brcmf_debugfs_add_entry(drvr, "features", brcmf_feat_debugfs_read);
-	brcmf_debugfs_add_entry(drvr, "fwcap", brcmf_feat_fwcap_debugfs_read);
 }
 
 bool brcmf_feat_is_enabled(struct brcmf_if *ifp, enum brcmf_feat_id id)
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h	2020-06-19 21:00:31.420227716 -0400
@@ -33,8 +33,8 @@
  * MFP: 802.11w Management Frame Protection.
  * GSCAN: enhanced scan offload feature.
  * FWSUP: Firmware supplicant.
- * MONITOR: firmware can pass monitor packets to host.
- * MONITOR_FMT_RADIOTAP: firmware provides monitor packets with radiotap header
+ * SAE: simultaneous authentication of equals
+ * FWAUTH: Firmware authenticator
  */
 #define BRCMF_FEAT_LIST \
 	BRCMF_FEAT_DEF(MBSS) \
@@ -51,8 +51,8 @@
 	BRCMF_FEAT_DEF(MFP) \
 	BRCMF_FEAT_DEF(GSCAN) \
 	BRCMF_FEAT_DEF(FWSUP) \
-	BRCMF_FEAT_DEF(MONITOR) \
-	BRCMF_FEAT_DEF(MONITOR_FMT_RADIOTAP)
+	BRCMF_FEAT_DEF(SAE) \
+	BRCMF_FEAT_DEF(FWAUTH)
 
 /*
  * Quirks:
@@ -94,13 +94,6 @@
 void brcmf_feat_attach(struct brcmf_pub *drvr);
 
 /**
- * brcmf_feat_debugfs_create() - create debugfs entries.
- *
- * @drvr: driver instance.
- */
-void brcmf_feat_debugfs_create(struct brcmf_pub *drvr);
-
-/**
  * brcmf_feat_is_enabled() - query feature.
  *
  * @ifp: interface instance.
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c	2020-06-19 21:00:31.420227716 -0400
@@ -25,7 +25,6 @@
 #include "firmware.h"
 #include "core.h"
 #include "common.h"
-#include "chip.h"
 
 #define BRCMF_FW_MAX_NVRAM_SIZE			64000
 #define BRCMF_FW_NVRAM_DEVPATH_LEN		19	/* devpath0=pcie/1/4/ */
@@ -438,31 +437,18 @@
 
 struct brcmf_fw {
 	struct device *dev;
-	struct brcmf_fw_request *req;
-	u32 curpos;
-	void (*done)(struct device *dev, int err, struct brcmf_fw_request *req);
+	u16 flags;
+	const struct firmware *code;
+	const char *nvram_name;
+	u16 domain_nr;
+	u16 bus_nr;
+	void (*done)(struct device *dev, int err, const struct firmware *fw,
+		     void *nvram_image, u32 nvram_len);
 };
 
-static void brcmf_fw_request_done(const struct firmware *fw, void *ctx);
-
-static void brcmf_fw_free_request(struct brcmf_fw_request *req)
-{
-	struct brcmf_fw_item *item;
-	int i;
-
-	for (i = 0, item = &req->items[0]; i < req->n_items; i++, item++) {
-		if (item->type == BRCMF_FW_TYPE_BINARY)
-			release_firmware(item->binary);
-		else if (item->type == BRCMF_FW_TYPE_NVRAM)
-			brcmf_fw_nvram_free(item->nv_data.data);
-	}
-	kfree(req);
-}
-
-static int brcmf_fw_request_nvram_done(const struct firmware *fw, void *ctx)
+static void brcmf_fw_request_nvram_done(const struct firmware *fw, void *ctx)
 {
 	struct brcmf_fw *fwctx = ctx;
-	struct brcmf_fw_item *cur;
 	u32 nvram_length = 0;
 	void *nvram = NULL;
 	u8 *data = NULL;
@@ -470,154 +456,83 @@
 	bool raw_nvram;
 
 	brcmf_dbg(TRACE, "enter: dev=%s\n", dev_name(fwctx->dev));
-
-	cur = &fwctx->req->items[fwctx->curpos];
-
 	if (fw && fw->data) {
 		data = (u8 *)fw->data;
 		data_len = fw->size;
 		raw_nvram = false;
 	} else {
 		data = bcm47xx_nvram_get_contents(&data_len);
-		if (!data && !(cur->flags & BRCMF_FW_REQF_OPTIONAL))
+		if (!data && !(fwctx->flags & BRCMF_FW_REQ_NV_OPTIONAL))
 			goto fail;
 		raw_nvram = true;
 	}
 
 	if (data)
 		nvram = brcmf_fw_nvram_strip(data, data_len, &nvram_length,
-					     fwctx->req->domain_nr,
-					     fwctx->req->bus_nr);
+					     fwctx->domain_nr, fwctx->bus_nr);
 
 	if (raw_nvram)
 		bcm47xx_nvram_release_contents(data);
 	release_firmware(fw);
-	if (!nvram && !(cur->flags & BRCMF_FW_REQF_OPTIONAL))
+	if (!nvram && !(fwctx->flags & BRCMF_FW_REQ_NV_OPTIONAL))
 		goto fail;
 
-	brcmf_dbg(TRACE, "nvram %p len %d\n", nvram, nvram_length);
-	cur->nv_data.data = nvram;
-	cur->nv_data.len = nvram_length;
-	return 0;
+	fwctx->done(fwctx->dev, 0, fwctx->code, nvram, nvram_length);
+	kfree(fwctx);
+	return;
 
 fail:
-	return -ENOENT;
-}
-
-static int brcmf_fw_request_next_item(struct brcmf_fw *fwctx, bool async)
-{
-	struct brcmf_fw_item *cur;
-	const struct firmware *fw = NULL;
-	int ret;
-
-	cur = &fwctx->req->items[fwctx->curpos];
-
-	brcmf_dbg(TRACE, "%srequest for %s\n", async ? "async " : "",
-		  cur->path);
-
-	if (async)
-		ret = request_firmware_nowait(THIS_MODULE, true, cur->path,
-					      fwctx->dev, GFP_KERNEL, fwctx,
-					      brcmf_fw_request_done);
-	else
-		ret = request_firmware(&fw, cur->path, fwctx->dev);
-
-	if (ret < 0) {
-		brcmf_fw_request_done(NULL, fwctx);
-	} else if (!async && fw) {
-		brcmf_dbg(TRACE, "firmware %s %sfound\n", cur->path,
-			  fw ? "" : "not ");
-		if (cur->type == BRCMF_FW_TYPE_BINARY)
-			cur->binary = fw;
-		else if (cur->type == BRCMF_FW_TYPE_NVRAM)
-			brcmf_fw_request_nvram_done(fw, fwctx);
-		else
-			release_firmware(fw);
-
-		return -EAGAIN;
-	}
-	return 0;
+	brcmf_dbg(TRACE, "failed: dev=%s\n", dev_name(fwctx->dev));
+	release_firmware(fwctx->code);
+	fwctx->done(fwctx->dev, -ENOENT, NULL, NULL, 0);
+	kfree(fwctx);
 }
 
-static void brcmf_fw_request_done(const struct firmware *fw, void *ctx)
+static void brcmf_fw_request_code_done(const struct firmware *fw, void *ctx)
 {
 	struct brcmf_fw *fwctx = ctx;
-	struct brcmf_fw_item *cur;
 	int ret = 0;
 
-	cur = &fwctx->req->items[fwctx->curpos];
-
-	brcmf_dbg(TRACE, "enter: firmware %s %sfound\n", cur->path,
-		  fw ? "" : "not ");
-
-	if (!fw)
+	brcmf_dbg(TRACE, "enter: dev=%s\n", dev_name(fwctx->dev));
+	if (!fw) {
 		ret = -ENOENT;
-
-	switch (cur->type) {
-	case BRCMF_FW_TYPE_NVRAM:
-		ret = brcmf_fw_request_nvram_done(fw, fwctx);
-		break;
-	case BRCMF_FW_TYPE_BINARY:
-		cur->binary = fw;
-		break;
-	default:
-		/* something fishy here so bail out early */
-		brcmf_err("unknown fw type: %d\n", cur->type);
-		release_firmware(fw);
-		ret = -EINVAL;
 		goto fail;
 	}
-
-	if (ret < 0 && !(cur->flags & BRCMF_FW_REQF_OPTIONAL))
-		goto fail;
-
-	do {
-		if (++fwctx->curpos == fwctx->req->n_items) {
-			ret = 0;
-			goto done;
-		}
-
-		ret = brcmf_fw_request_next_item(fwctx, false);
-	} while (ret == -EAGAIN);
-
+	/* only requested code so done here */
+	if (!(fwctx->flags & BRCMF_FW_REQUEST_NVRAM))
+		goto done;
+
+	fwctx->code = fw;
+	ret = request_firmware_nowait(THIS_MODULE, true, fwctx->nvram_name,
+				      fwctx->dev, GFP_KERNEL, fwctx,
+				      brcmf_fw_request_nvram_done);
+
+	/* pass NULL to nvram callback for bcm47xx fallback */
+	if (ret)
+		brcmf_fw_request_nvram_done(NULL, fwctx);
 	return;
 
 fail:
-	brcmf_dbg(TRACE, "failed err=%d: dev=%s, fw=%s\n", ret,
-		  dev_name(fwctx->dev), cur->path);
-	brcmf_fw_free_request(fwctx->req);
-	fwctx->req = NULL;
+	brcmf_dbg(TRACE, "failed: dev=%s\n", dev_name(fwctx->dev));
 done:
-	fwctx->done(fwctx->dev, ret, fwctx->req);
+	fwctx->done(fwctx->dev, ret, fw, NULL, 0);
 	kfree(fwctx);
 }
 
-static bool brcmf_fw_request_is_valid(struct brcmf_fw_request *req)
-{
-	struct brcmf_fw_item *item;
-	int i;
-
-	if (!req->n_items)
-		return false;
-
-	for (i = 0, item = &req->items[0]; i < req->n_items; i++, item++) {
-		if (!item->path)
-			return false;
-	}
-	return true;
-}
-
-int brcmf_fw_get_firmwares(struct device *dev, struct brcmf_fw_request *req,
-			   void (*fw_cb)(struct device *dev, int err,
-					 struct brcmf_fw_request *req))
+int brcmf_fw_get_firmwares_pcie(struct device *dev, u16 flags,
+				const char *code, const char *nvram,
+				void (*fw_cb)(struct device *dev, int err,
+					      const struct firmware *fw,
+					      void *nvram_image, u32 nvram_len),
+				u16 domain_nr, u16 bus_nr)
 {
 	struct brcmf_fw *fwctx;
 
 	brcmf_dbg(TRACE, "enter: dev=%s\n", dev_name(dev));
-	if (!fw_cb)
+	if (!fw_cb || !code)
 		return -EINVAL;
 
-	if (!brcmf_fw_request_is_valid(req))
+	if ((flags & BRCMF_FW_REQUEST_NVRAM) && !nvram)
 		return -EINVAL;
 
 	fwctx = kzalloc(sizeof(*fwctx), GFP_KERNEL);
@@ -625,26 +540,35 @@
 		return -ENOMEM;
 
 	fwctx->dev = dev;
-	fwctx->req = req;
+	fwctx->flags = flags;
 	fwctx->done = fw_cb;
+	if (flags & BRCMF_FW_REQUEST_NVRAM)
+		fwctx->nvram_name = nvram;
+	fwctx->domain_nr = domain_nr;
+	fwctx->bus_nr = bus_nr;
+
+	return request_firmware_nowait(THIS_MODULE, true, code, dev,
+				       GFP_KERNEL, fwctx,
+				       brcmf_fw_request_code_done);
+}
 
-	brcmf_fw_request_next_item(fwctx, true);
-	return 0;
+int brcmf_fw_get_firmwares(struct device *dev, u16 flags,
+			   const char *code, const char *nvram,
+			   void (*fw_cb)(struct device *dev, int err,
+					 const struct firmware *fw,
+					 void *nvram_image, u32 nvram_len))
+{
+	return brcmf_fw_get_firmwares_pcie(dev, flags, code, nvram, fw_cb, 0,
+					   0);
 }
 
-struct brcmf_fw_request *
-brcmf_fw_alloc_request(u32 chip, u32 chiprev,
-		       const struct brcmf_firmware_mapping mapping_table[],
-		       u32 table_size, struct brcmf_fw_name *fwnames,
-		       u32 n_fwnames)
-{
-	struct brcmf_fw_request *fwreq;
-	char chipname[12];
-	const char *mp_path;
-	size_t mp_path_len;
-	u32 i, j;
-	char end = '\0';
-	size_t reqsz;
+int brcmf_fw_map_chip_to_name(u32 chip, u32 chiprev,
+			      struct brcmf_firmware_mapping mapping_table[],
+			      u32 table_size, char fw_name[BRCMF_FW_NAME_LEN],
+			      char nvram_name[BRCMF_FW_NAME_LEN])
+{
+	u32 i;
+	char end;
 
 	for (i = 0; i < table_size; i++) {
 		if (mapping_table[i].chipid == chip &&
@@ -654,44 +578,35 @@
 
 	if (i == table_size) {
 		brcmf_err("Unknown chipid %d [%d]\n", chip, chiprev);
-		return NULL;
+		return -ENODEV;
 	}
 
-	reqsz = sizeof(*fwreq) + n_fwnames * sizeof(struct brcmf_fw_item);
-	fwreq = kzalloc(reqsz, GFP_KERNEL);
-	if (!fwreq)
-		return NULL;
-
-	brcmf_chip_name(chip, chiprev, chipname, sizeof(chipname));
-
-	brcmf_info("using %s for chip %s\n",
-		   mapping_table[i].fw_base, chipname);
-
-	mp_path = brcmf_mp_global.firmware_path;
-	mp_path_len = strnlen(mp_path, BRCMF_FW_ALTPATH_LEN);
-	if (mp_path_len)
-		end = mp_path[mp_path_len - 1];
-
-	fwreq->n_items = n_fwnames;
-
-	for (j = 0; j < n_fwnames; j++) {
-		fwreq->items[j].path = fwnames[j].path;
-		/* check if firmware path is provided by module parameter */
-		if (brcmf_mp_global.firmware_path[0] != '\0') {
-			strlcpy(fwnames[j].path, mp_path,
+	fw_name[0] = '\0';
+	if (nvram_name)
+		nvram_name[0] = '\0';
+	/* check if firmware path is provided by module parameter */
+	if (brcmf_mp_global.firmware_path[0] != '\0') {
+		strlcpy(fw_name, brcmf_mp_global.firmware_path,
+			BRCMF_FW_NAME_LEN);
+		if ((nvram_name) && (mapping_table[i].nvram))
+			strlcpy(nvram_name, brcmf_mp_global.firmware_path,
 				BRCMF_FW_NAME_LEN);
 
-			if (end != '/') {
-				strlcat(fwnames[j].path, "/",
-					BRCMF_FW_NAME_LEN);
-			}
+		end = brcmf_mp_global.firmware_path[
+				strlen(brcmf_mp_global.firmware_path) - 1];
+		if (end != '/') {
+			strlcat(fw_name, "/", BRCMF_FW_NAME_LEN);
+			if ((nvram_name) && (mapping_table[i].nvram))
+				strlcat(nvram_name, "/", BRCMF_FW_NAME_LEN);
 		}
-		strlcat(fwnames[j].path, mapping_table[i].fw_base,
-			BRCMF_FW_NAME_LEN);
-		strlcat(fwnames[j].path, fwnames[j].extension,
-			BRCMF_FW_NAME_LEN);
-		fwreq->items[j].path = fwnames[j].path;
 	}
+	strlcat(fw_name, mapping_table[i].fw, BRCMF_FW_NAME_LEN);
+	if ((nvram_name) && (mapping_table[i].nvram))
+		strlcat(nvram_name, mapping_table[i].nvram, BRCMF_FW_NAME_LEN);
 
-	return fwreq;
+	brcmf_info("using %s for chip %#08x(%d) rev %#08x\n",
+		   fw_name, chip, chip, chiprev);
+
+	return 0;
 }
+
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.h
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.h	2020-06-19 21:00:31.420227716 -0400
@@ -16,7 +16,10 @@
 #ifndef BRCMFMAC_FIRMWARE_H
 #define BRCMFMAC_FIRMWARE_H
 
-#define BRCMF_FW_REQF_OPTIONAL		0x0001
+#define BRCMF_FW_REQUEST		0x000F
+#define  BRCMF_FW_REQUEST_NVRAM		0x0001
+#define BRCMF_FW_REQ_FLAGS		0x00F0
+#define  BRCMF_FW_REQ_NV_OPTIONAL	0x0010
 
 #define	BRCMF_FW_NAME_LEN		320
 
@@ -35,62 +38,49 @@
 struct brcmf_firmware_mapping {
 	u32 chipid;
 	u32 revmask;
-	const char *fw_base;
+	const char *fw;
+	const char *nvram;
 };
 
-#define BRCMF_FW_DEF(fw_name, fw_base) \
-static const char BRCM_ ## fw_name ## _FIRMWARE_BASENAME[] = \
-	BRCMF_FW_DEFAULT_PATH fw_base; \
-MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH fw_base ".bin")
+#define BRCMF_FW_NVRAM_DEF(fw_nvram_name, fw, nvram) \
+static const char BRCM_ ## fw_nvram_name ## _FIRMWARE_NAME[] = \
+	BRCMF_FW_DEFAULT_PATH fw; \
+static const char BRCM_ ## fw_nvram_name ## _NVRAM_NAME[] = \
+	BRCMF_FW_DEFAULT_PATH nvram; \
+MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH fw);
+
+#define BRCMF_FW_DEF(fw_name, fw) \
+static const char BRCM_ ## fw_name ## _FIRMWARE_NAME[] = \
+	BRCMF_FW_DEFAULT_PATH fw; \
+MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH fw) \
+
+#define BRCMF_FW_NVRAM_ENTRY(chipid, mask, name) \
+	{ chipid, mask, \
+	  BRCM_ ## name ## _FIRMWARE_NAME, BRCM_ ## name ## _NVRAM_NAME }
 
 #define BRCMF_FW_ENTRY(chipid, mask, name) \
-	{ chipid, mask, BRCM_ ## name ## _FIRMWARE_BASENAME }
+	{ chipid, mask, BRCM_ ## name ## _FIRMWARE_NAME, NULL }
 
+int brcmf_fw_map_chip_to_name(u32 chip, u32 chiprev,
+			      struct brcmf_firmware_mapping mapping_table[],
+			      u32 table_size, char fw_name[BRCMF_FW_NAME_LEN],
+			      char nvram_name[BRCMF_FW_NAME_LEN]);
 void brcmf_fw_nvram_free(void *nvram);
-
-enum brcmf_fw_type {
-	BRCMF_FW_TYPE_BINARY,
-	BRCMF_FW_TYPE_NVRAM
-};
-
-struct brcmf_fw_item {
-	const char *path;
-	enum brcmf_fw_type type;
-	u16 flags;
-	union {
-		const struct firmware *binary;
-		struct {
-			void *data;
-			u32 len;
-		} nv_data;
-	};
-};
-
-struct brcmf_fw_request {
-	u16 domain_nr;
-	u16 bus_nr;
-	u32 n_items;
-	struct brcmf_fw_item items[0];
-};
-
-struct brcmf_fw_name {
-	const char *extension;
-	char *path;
-};
-
-struct brcmf_fw_request *
-brcmf_fw_alloc_request(u32 chip, u32 chiprev,
-		       const struct brcmf_firmware_mapping mapping_table[],
-		       u32 table_size, struct brcmf_fw_name *fwnames,
-		       u32 n_fwnames);
-
 /*
  * Request firmware(s) asynchronously. When the asynchronous request
  * fails it will not use the callback, but call device_release_driver()
  * instead which will call the driver .remove() callback.
  */
-int brcmf_fw_get_firmwares(struct device *dev, struct brcmf_fw_request *req,
+int brcmf_fw_get_firmwares_pcie(struct device *dev, u16 flags,
+				const char *code, const char *nvram,
+				void (*fw_cb)(struct device *dev, int err,
+					      const struct firmware *fw,
+					      void *nvram_image, u32 nvram_len),
+				u16 domain_nr, u16 bus_nr);
+int brcmf_fw_get_firmwares(struct device *dev, u16 flags,
+			   const char *code, const char *nvram,
 			   void (*fw_cb)(struct device *dev, int err,
-					 struct brcmf_fw_request *req));
+					 const struct firmware *fw,
+					 void *nvram_image, u32 nvram_len));
 
 #endif /* BRCMFMAC_FIRMWARE_H */
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c	2020-06-19 21:00:31.420227716 -0400
@@ -46,8 +46,6 @@
 	3
 };
 
-static const u8 ALLFFMAC[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
-
 
 static bool
 brcmf_flowring_is_tdls_mac(struct brcmf_flowring *flow, u8 mac[ETH_ALEN])
@@ -429,7 +427,6 @@
 				flowid = flow->hash[i].flowid;
 				if (flow->rings[flowid]->status != RING_OPEN)
 					continue;
-				flow->rings[flowid]->status = RING_CLOSING;
 				brcmf_msgbuf_delete_flowring(drvr, flowid);
 			}
 		}
@@ -468,10 +465,8 @@
 		if ((sta || (memcmp(hash[i].mac, peer, ETH_ALEN) == 0)) &&
 		    (hash[i].ifidx == ifidx)) {
 			flowid = flow->hash[i].flowid;
-			if (flow->rings[flowid]->status == RING_OPEN) {
-				flow->rings[flowid]->status = RING_CLOSING;
+			if (flow->rings[flowid]->status == RING_OPEN)
 				brcmf_msgbuf_delete_flowring(drvr, flowid);
-			}
 		}
 	}
 
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c	2020-06-19 21:00:31.420227716 -0400
@@ -257,6 +257,11 @@
 		brcmf_dbg_hex_dump(BRCMF_EVENT_ON(), event->data,
 				   min_t(u32, emsg.datalen, 64),
 				   "event payload, len=%d\n", emsg.datalen);
+		if (emsg.datalen > event->datalen) {
+			brcmf_err("event invalid length header=%d, msg=%d\n",
+				  event->datalen, emsg.datalen);
+			goto event_free;
+		}
 
 		/* special handling of interface event */
 		if (event->code == BRCMF_E_IF) {
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h	2020-06-19 21:00:31.420227716 -0400
@@ -101,7 +101,9 @@
 	BRCMF_ENUM_DEF(FIFO_CREDIT_MAP, 74) \
 	BRCMF_ENUM_DEF(ACTION_FRAME_RX, 75) \
 	BRCMF_ENUM_DEF(TDLS_PEER_EVENT, 92) \
-	BRCMF_ENUM_DEF(BCMC_CREDIT_SUPPORT, 127)
+	BRCMF_ENUM_DEF(PHY_TEMP, 111) \
+	BRCMF_ENUM_DEF(BCMC_CREDIT_SUPPORT, 127) \
+	BRCMF_ENUM_DEF(ULP, 146)
 
 #define BRCMF_ENUM_DEF(id, val) \
 	BRCMF_E_##id = (val),
@@ -113,7 +115,7 @@
 	 * minimum length check in device firmware so it is
 	 * hard-coded here.
 	 */
-	BRCMF_E_LAST = 139
+	BRCMF_E_LAST = 147
 };
 #undef BRCMF_ENUM_DEF
 
@@ -294,6 +296,28 @@
 	u8 role;
 };
 
+enum event_msgs_ext_command {
+	EVENTMSGS_NONE		=	0,
+	EVENTMSGS_SET_BIT	=	1,
+	EVENTMSGS_RESET_BIT	=	2,
+	EVENTMSGS_SET_MASK	=	3
+};
+
+#define EVENTMSGS_VER 1
+#define EVENTMSGS_EXT_STRUCT_SIZE	offsetof(struct eventmsgs_ext, mask[0])
+
+/* len-	for SET it would be mask size from the application to the firmware */
+/*		for GET it would be actual firmware mask size */
+/* maxgetsize -	is only used for GET. indicate max mask size that the */
+/*				application can read from the firmware */
+struct eventmsgs_ext {
+	u8	ver;
+	u8	command;
+	u8	len;
+	u8	maxgetsize;
+	u8	mask[1];
+};
+
 typedef int (*brcmf_fweh_handler_t)(struct brcmf_if *ifp,
 				    const struct brcmf_event_msg *evtmsg,
 				    void *data);
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.c	2020-06-19 21:00:31.420227716 -0400
@@ -107,7 +107,7 @@
 brcmf_fil_cmd_data(struct brcmf_if *ifp, u32 cmd, void *data, u32 len, bool set)
 {
 	struct brcmf_pub *drvr = ifp->drvr;
-	s32 err, fwerr;
+	s32 err;
 
 	if (drvr->bus_if->state != BRCMF_BUS_UP) {
 		brcmf_err("bus is down. we have nothing to do.\n");
@@ -117,21 +117,15 @@
 	if (data != NULL)
 		len = min_t(uint, len, BRCMF_DCMD_MAXLEN);
 	if (set)
-		err = brcmf_proto_set_dcmd(drvr, ifp->ifidx, cmd,
-					   data, len, &fwerr);
+		err = brcmf_proto_set_dcmd(drvr, ifp->ifidx, cmd, data, len);
 	else
-		err = brcmf_proto_query_dcmd(drvr, ifp->ifidx, cmd,
-					     data, len, &fwerr);
+		err = brcmf_proto_query_dcmd(drvr, ifp->ifidx, cmd, data, len);
 
-	if (err) {
-		brcmf_dbg(FIL, "Failed: error=%d\n", err);
-	} else if (fwerr < 0) {
-		brcmf_dbg(FIL, "Firmware error: %s (%d)\n",
-			  brcmf_fil_get_errstr((u32)(-fwerr)), fwerr);
-		err = -EBADE;
-	}
-	if (ifp->fwil_fwerr)
-		return fwerr;
+	if (err >= 0)
+		return 0;
+
+	brcmf_dbg(FIL, "Failed: %s (%d)\n",
+		  brcmf_fil_get_errstr((u32)(-err)), err);
 
 	return err;
 }
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.h
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.h	2020-06-19 21:00:31.420227716 -0400
@@ -80,6 +80,7 @@
 #define BRCMF_C_SCB_DEAUTHENTICATE_FOR_REASON	201
 #define BRCMF_C_SET_ASSOC_PREFER		205
 #define BRCMF_C_GET_VALID_CHANNELS		217
+#define BRCMF_C_SET_FAKEFRAG			219
 #define BRCMF_C_GET_KEY_PRIMARY			235
 #define BRCMF_C_SET_KEY_PRIMARY			236
 #define BRCMF_C_SET_SCAN_PASSIVE_TIME		258
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h	2020-06-19 21:00:31.420227716 -0400
@@ -30,32 +30,13 @@
 #define BRCMF_ARP_OL_PEER_AUTO_REPLY	0x00000008
 
 #define	BRCMF_BSS_INFO_VERSION	109 /* curr ver of brcmf_bss_info_le struct */
-#define BRCMF_BSS_RSSI_ON_CHANNEL	0x0002
+#define BRCMF_BSS_RSSI_ON_CHANNEL	0x0004
 
-#define BRCMF_STA_BRCM			0x00000001	/* Running a Broadcom driver */
-#define BRCMF_STA_WME			0x00000002	/* WMM association */
-#define BRCMF_STA_NONERP		0x00000004	/* No ERP */
-#define BRCMF_STA_AUTHE			0x00000008	/* Authenticated */
-#define BRCMF_STA_ASSOC			0x00000010	/* Associated */
-#define BRCMF_STA_AUTHO			0x00000020	/* Authorized */
-#define BRCMF_STA_WDS			0x00000040	/* Wireless Distribution System */
-#define BRCMF_STA_WDS_LINKUP		0x00000080	/* WDS traffic/probes flowing properly */
-#define BRCMF_STA_PS			0x00000100	/* STA is in power save mode from AP's viewpoint */
-#define BRCMF_STA_APSD_BE		0x00000200	/* APSD delv/trigger for AC_BE is default enabled */
-#define BRCMF_STA_APSD_BK		0x00000400	/* APSD delv/trigger for AC_BK is default enabled */
-#define BRCMF_STA_APSD_VI		0x00000800	/* APSD delv/trigger for AC_VI is default enabled */
-#define BRCMF_STA_APSD_VO		0x00001000	/* APSD delv/trigger for AC_VO is default enabled */
-#define BRCMF_STA_N_CAP			0x00002000	/* STA 802.11n capable */
-#define BRCMF_STA_SCBSTATS		0x00004000	/* Per STA debug stats */
-#define BRCMF_STA_AMPDU_CAP		0x00008000	/* STA AMPDU capable */
-#define BRCMF_STA_AMSDU_CAP		0x00010000	/* STA AMSDU capable */
-#define BRCMF_STA_MIMO_PS		0x00020000	/* mimo ps mode is enabled */
-#define BRCMF_STA_MIMO_RTS		0x00040000	/* send rts in mimo ps mode */
-#define BRCMF_STA_RIFS_CAP		0x00080000	/* rifs enabled */
-#define BRCMF_STA_VHT_CAP		0x00100000	/* STA VHT(11ac) capable */
-#define BRCMF_STA_WPS			0x00200000	/* WPS state */
-#define BRCMF_STA_DWDS_CAP		0x01000000	/* DWDS CAP */
-#define BRCMF_STA_DWDS			0x02000000	/* DWDS active */
+#define BRCMF_STA_WME              0x00000002      /* WMM association */
+#define BRCMF_STA_AUTHE            0x00000008      /* Authenticated */
+#define BRCMF_STA_ASSOC            0x00000010      /* Associated */
+#define BRCMF_STA_AUTHO            0x00000020      /* Authorized */
+#define BRCMF_STA_SCBSTATS         0x00004000      /* Per STA debug stats */
 
 /* size of brcmf_scan_params not including variable length array */
 #define BRCMF_SCAN_PARAMS_FIXED_SIZE	64
@@ -72,6 +53,8 @@
 #define BRCMF_WSEC_MAX_PSK_LEN		32
 #define	BRCMF_WSEC_PASSPHRASE		BIT(0)
 
+#define BRCMF_WSEC_MAX_SAE_PASSWORD_LEN 128
+
 /* primary (ie tx) key */
 #define BRCMF_PRIMARY_KEY		(1 << 1)
 #define DOT11_BSSTYPE_ANY		2
@@ -144,9 +127,22 @@
 /* Link Down indication in WoWL mode: */
 #define BRCMF_WOWL_LINKDOWN		(1 << 31)
 
-#define BRCMF_WOWL_MAXPATTERNS		8
+#define BRCMF_WOWL_MAXPATTERNS		16
 #define BRCMF_WOWL_MAXPATTERNSIZE	128
 
+enum {
+	BRCMF_UNICAST_FILTER_NUM = 0,
+	BRCMF_BROADCAST_FILTER_NUM,
+	BRCMF_MULTICAST4_FILTER_NUM,
+	BRCMF_MULTICAST6_FILTER_NUM,
+	BRCMF_MDNS_FILTER_NUM,
+	BRCMF_ARP_FILTER_NUM,
+	BRCMF_BROADCAST_ARP_FILTER_NUM,
+	MAX_PKT_FILTER_COUNT
+};
+
+#define MAX_PKTFILTER_PATTERN_SIZE		16
+
 #define BRCMF_COUNTRY_BUF_SZ		4
 #define BRCMF_ANT_MAX			4
 
@@ -174,8 +170,6 @@
 #define BRCMF_MFP_CAPABLE		1
 #define BRCMF_MFP_REQUIRED		2
 
-#define BRCMF_VHT_CAP_MCS_MAP_NSS_MAX	8
-
 /* MAX_CHUNK_LEN is the maximum length for data passing to firmware in each
  * ioctl. It is relatively small because firmware has small maximum size input
  * playload restriction for ioctls.
@@ -527,6 +521,17 @@
 	u8 key[2 * BRCMF_WSEC_MAX_PSK_LEN + 1];
 };
 
+/**
+ * struct brcmf_wsec_sae_pwd_le - firmware SAE password material.
+ *
+ * @key_len: number of octets in key materials.
+ * @key: SAE password material.
+ */
+struct brcmf_wsec_sae_pwd_le {
+	__le16 key_len;
+	u8 key[BRCMF_WSEC_MAX_SAE_PASSWORD_LEN];
+};
+
 /* Used to get specific STA parameters */
 struct brcmf_scb_val_le {
 	__le32 val;
@@ -552,8 +557,6 @@
 						/* w/hi bit set if basic */
 	__le32 in;		/* seconds elapsed since associated */
 	__le32 listen_interval_inms; /* Min Listen interval in ms for STA */
-
-	/* Fields valid for ver >= 3 */
 	__le32 tx_pkts;	/* # of packets transmitted */
 	__le32 tx_failures;	/* # of packets failed */
 	__le32 rx_ucast_pkts;	/* # of unicast packets received */
@@ -562,8 +565,6 @@
 	__le32 rx_rate;	/* Rate of last successful rx frame */
 	__le32 rx_decrypt_succeeds;	/* # of packet decrypted successfully */
 	__le32 rx_decrypt_failures;	/* # of packet decrypted failed */
-
-	/* Fields valid for ver >= 4 */
 	__le32 tx_tot_pkts;    /* # of tx pkts (ucast + mcast) */
 	__le32 rx_tot_pkts;    /* # of data packets recvd (uni + mcast) */
 	__le32 tx_mcast_pkts;  /* # of mcast pkts txed */
@@ -600,14 +601,6 @@
 						*/
 	__le32 rx_pkts_retried;        /* # rx with retry bit set */
 	__le32 tx_rate_fallback;       /* lowest fallback TX rate */
-
-	/* Fields valid for ver >= 5 */
-	struct {
-		__le32 count;					/* # rates in this set */
-		u8 rates[BRCMF_MAXRATES_IN_SET];		/* rates in 500kbps units w/hi bit set if basic */
-		u8 mcs[BRCMF_MCSSET_LEN];			/* supported mcs index bit map */
-		__le16 vht_mcs[BRCMF_VHT_CAP_MCS_MAP_NSS_MAX];	/* supported mcs index bit map per nss */
-	} rateset_adv;
 };
 
 struct brcmf_chanspec_list {
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c	2020-06-19 21:00:31.420227716 -0400
@@ -322,28 +322,6 @@
 /* How long to defer borrowing in jiffies */
 #define BRCMF_FWS_BORROW_DEFER_PERIOD		(HZ / 10)
 
-/**
- * enum brcmf_fws_fifo - fifo indices used by dongle firmware.
- *
- * @BRCMF_FWS_FIFO_FIRST: first fifo, ie. background.
- * @BRCMF_FWS_FIFO_AC_BK: fifo for background traffic.
- * @BRCMF_FWS_FIFO_AC_BE: fifo for best-effort traffic.
- * @BRCMF_FWS_FIFO_AC_VI: fifo for video traffic.
- * @BRCMF_FWS_FIFO_AC_VO: fifo for voice traffic.
- * @BRCMF_FWS_FIFO_BCMC: fifo for broadcast/multicast (AP only).
- * @BRCMF_FWS_FIFO_ATIM: fifo for ATIM (AP only).
- * @BRCMF_FWS_FIFO_COUNT: number of fifos.
- */
-enum brcmf_fws_fifo {
-	BRCMF_FWS_FIFO_FIRST,
-	BRCMF_FWS_FIFO_AC_BK = BRCMF_FWS_FIFO_FIRST,
-	BRCMF_FWS_FIFO_AC_BE,
-	BRCMF_FWS_FIFO_AC_VI,
-	BRCMF_FWS_FIFO_AC_VO,
-	BRCMF_FWS_FIFO_BCMC,
-	BRCMF_FWS_FIFO_ATIM,
-	BRCMF_FWS_FIFO_COUNT
-};
 
 /**
  * enum brcmf_fws_txstatus - txstatus flag values.
@@ -356,6 +334,10 @@
  *	firmware suppress the packet as device is already in PS mode.
  * @BRCMF_FWS_TXSTATUS_FW_TOSSED:
  *	firmware tossed the packet.
+ * @BRCMF_FWS_TXSTATUS_FW_DISCARD_NOACK:
+ *	firmware tossed the packet after retries.
+ * @BRCMF_FWS_TXSTATUS_FW_SUPPRESS_ACKED:
+ *	firmware wrongly reported suppressed previously, now fixing to acked.
  * @BRCMF_FWS_TXSTATUS_HOST_TOSSED:
  *	host tossed the packet.
  */
@@ -364,6 +346,8 @@
 	BRCMF_FWS_TXSTATUS_CORE_SUPPRESS,
 	BRCMF_FWS_TXSTATUS_FW_PS_SUPPRESS,
 	BRCMF_FWS_TXSTATUS_FW_TOSSED,
+	BRCMF_FWS_TXSTATUS_FW_DISCARD_NOACK,
+	BRCMF_FWS_TXSTATUS_FW_SUPPRESS_ACKED,
 	BRCMF_FWS_TXSTATUS_HOST_TOSSED
 };
 
@@ -415,7 +399,8 @@
 	u8 traffic_lastreported_bmp;
 };
 
-#define BRCMF_FWS_HANGER_MAXITEMS	1024
+#define BRCMF_FWS_HANGER_MAXITEMS	3072
+#define BRCMF_BORROW_RATIO			3
 
 /**
  * enum brcmf_fws_hanger_item_state - state of hanger item.
@@ -511,7 +496,9 @@
 	struct work_struct fws_dequeue_work;
 	u32 fifo_enqpkt[BRCMF_FWS_FIFO_COUNT];
 	int fifo_credit[BRCMF_FWS_FIFO_COUNT];
-	int credits_borrowed[BRCMF_FWS_FIFO_AC_VO + 1];
+	int init_fifo_credit[BRCMF_FWS_FIFO_COUNT];
+	int credits_borrowed[BRCMF_FWS_FIFO_AC_VO + 1]
+		[BRCMF_FWS_FIFO_AC_VO + 1];
 	int deq_node_pos[BRCMF_FWS_FIFO_COUNT];
 	u32 fifo_credit_map;
 	u32 fifo_delay_map;
@@ -520,6 +507,7 @@
 	bool creditmap_received;
 	u8 mode;
 	bool avoid_queueing;
+	int fifo_init_credit[BRCMF_FWS_FIFO_COUNT];
 };
 
 /*
@@ -579,6 +567,30 @@
 	return ifidx == *(int *)arg;
 }
 
+static void brcmf_fws_psq_flush(struct brcmf_fws_info *fws, struct pktq *q,
+				int ifidx)
+{
+	struct brcmf_fws_hanger_item *hi;
+	bool (*matchfn)(struct sk_buff *, void *) = NULL;
+	struct sk_buff *skb;
+	int prec;
+	u32 hslot;
+
+	if (ifidx != -1)
+		matchfn = brcmf_fws_ifidx_match;
+	for (prec = 0; prec < q->num_prec; prec++) {
+		skb = brcmu_pktq_pdeq_match(q, prec, matchfn, &ifidx);
+		while (skb) {
+			hslot = brcmf_skb_htod_tag_get_field(skb, HSLOT);
+			hi = &fws->hanger.items[hslot];
+			WARN_ON(skb != hi->pkt);
+			hi->state = BRCMF_FWS_HANGER_ITEM_STATE_FREE;
+			brcmu_pkt_buf_free_skb(skb);
+			skb = brcmu_pktq_pdeq_match(q, prec, matchfn, &ifidx);
+		}
+	}
+}
+
 static void brcmf_fws_hanger_init(struct brcmf_fws_hanger *hanger)
 {
 	int i;
@@ -650,28 +662,6 @@
 	return 0;
 }
 
-static void brcmf_fws_psq_flush(struct brcmf_fws_info *fws, struct pktq *q,
-				int ifidx)
-{
-	bool (*matchfn)(struct sk_buff *, void *) = NULL;
-	struct sk_buff *skb;
-	int prec;
-	u32 hslot;
-
-	if (ifidx != -1)
-		matchfn = brcmf_fws_ifidx_match;
-	for (prec = 0; prec < q->num_prec; prec++) {
-		skb = brcmu_pktq_pdeq_match(q, prec, matchfn, &ifidx);
-		while (skb) {
-			hslot = brcmf_skb_htod_tag_get_field(skb, HSLOT);
-			brcmf_fws_hanger_poppkt(&fws->hanger, hslot, &skb,
-						true);
-			brcmu_pkt_buf_free_skb(skb);
-			skb = brcmu_pktq_pdeq_match(q, prec, matchfn, &ifidx);
-		}
-	}
-}
-
 static int brcmf_fws_hanger_mark_suppressed(struct brcmf_fws_hanger *h,
 					    u32 slot_id)
 {
@@ -1219,11 +1209,11 @@
 
 	fws->fifo_credit_map |= 1 << fifo;
 
-	if ((fifo == BRCMF_FWS_FIFO_AC_BE) &&
-	    (fws->credits_borrowed[0])) {
+	if (fifo > BRCMF_FWS_FIFO_AC_BK &&
+	    fifo <= BRCMF_FWS_FIFO_AC_VO) {
 		for (lender_ac = BRCMF_FWS_FIFO_AC_VO; lender_ac >= 0;
 		     lender_ac--) {
-			borrowed = &fws->credits_borrowed[lender_ac];
+			borrowed = &fws->credits_borrowed[fifo][lender_ac];
 			if (*borrowed) {
 				fws->fifo_credit_map |= (1 << lender_ac);
 				fifo_credit = &fws->fifo_credit[lender_ac];
@@ -1240,7 +1230,13 @@
 		}
 	}
 
-	fws->fifo_credit[fifo] += credits;
+	if (credits) {
+		fws->fifo_credit[fifo] += credits;
+	}
+
+	if (fws->fifo_credit[fifo] > fws->init_fifo_credit[fifo])
+		fws->fifo_credit[fifo] = fws->init_fifo_credit[fifo];
+
 }
 
 static void brcmf_fws_schedule_deq(struct brcmf_fws_info *fws)
@@ -1455,9 +1451,10 @@
 
 static int
 brcmf_fws_txs_process(struct brcmf_fws_info *fws, u8 flags, u32 hslot,
-		      u32 genbit, u16 seq)
+		      u32 genbit, u16 seq, u8 compcnt)
 {
 	u32 fifo;
+	u8 cnt = 0;
 	int ret;
 	bool remove_from_hanger = true;
 	struct sk_buff *skb;
@@ -1468,60 +1465,75 @@
 	brcmf_dbg(DATA, "flags %d\n", flags);
 
 	if (flags == BRCMF_FWS_TXSTATUS_DISCARD)
-		fws->stats.txs_discard++;
+		fws->stats.txs_discard += compcnt;
 	else if (flags == BRCMF_FWS_TXSTATUS_CORE_SUPPRESS) {
-		fws->stats.txs_supp_core++;
+		fws->stats.txs_supp_core += compcnt;
 		remove_from_hanger = false;
 	} else if (flags == BRCMF_FWS_TXSTATUS_FW_PS_SUPPRESS) {
-		fws->stats.txs_supp_ps++;
+		fws->stats.txs_supp_ps += compcnt;
 		remove_from_hanger = false;
 	} else if (flags == BRCMF_FWS_TXSTATUS_FW_TOSSED)
-		fws->stats.txs_tossed++;
+		fws->stats.txs_tossed += compcnt;
+	else if (flags == BRCMF_FWS_TXSTATUS_FW_DISCARD_NOACK)
+		fws->stats.txs_discard += compcnt;
+	else if (flags == BRCMF_FWS_TXSTATUS_FW_SUPPRESS_ACKED)
+		fws->stats.txs_discard += compcnt;
 	else if (flags == BRCMF_FWS_TXSTATUS_HOST_TOSSED)
-		fws->stats.txs_host_tossed++;
+		fws->stats.txs_host_tossed += compcnt;
 	else
 		brcmf_err("unexpected txstatus\n");
 
-	ret = brcmf_fws_hanger_poppkt(&fws->hanger, hslot, &skb,
-				      remove_from_hanger);
-	if (ret != 0) {
-		brcmf_err("no packet in hanger slot: hslot=%d\n", hslot);
-		return ret;
-	}
+	while (cnt < compcnt) {
+		ret = brcmf_fws_hanger_poppkt(&fws->hanger, hslot, &skb,
+					      remove_from_hanger);
+		if (ret != 0) {
+			brcmf_err("no packet in hanger slot: hslot=%d\n",
+				  hslot);
+			goto cont;
+		}
 
-	skcb = brcmf_skbcb(skb);
-	entry = skcb->mac;
-	if (WARN_ON(!entry)) {
-		brcmu_pkt_buf_free_skb(skb);
-		return -EINVAL;
-	}
-	entry->transit_count--;
-	if (entry->suppressed && entry->suppr_transit_count)
-		entry->suppr_transit_count--;
-
-	brcmf_dbg(DATA, "%s flags %d htod %X seq %X\n", entry->name, flags,
-		  skcb->htod, seq);
-
-	/* pick up the implicit credit from this packet */
-	fifo = brcmf_skb_htod_tag_get_field(skb, FIFO);
-	if ((fws->fcmode == BRCMF_FWS_FCMODE_IMPLIED_CREDIT) ||
-	    (brcmf_skb_if_flags_get_field(skb, REQ_CREDIT)) ||
-	    (flags == BRCMF_FWS_TXSTATUS_HOST_TOSSED)) {
-		brcmf_fws_return_credits(fws, fifo, 1);
-		brcmf_fws_schedule_deq(fws);
-	}
-	brcmf_fws_macdesc_return_req_credit(skb);
+		skcb = brcmf_skbcb(skb);
+		entry = skcb->mac;
+		if (WARN_ON(!entry)) {
+			brcmu_pkt_buf_free_skb(skb);
+			goto cont;
+		}
+		entry->transit_count--;
+		if (entry->suppressed && entry->suppr_transit_count)
+			entry->suppr_transit_count--;
 
-	ret = brcmf_proto_hdrpull(fws->drvr, false, skb, &ifp);
-	if (ret) {
-		brcmu_pkt_buf_free_skb(skb);
-		return -EINVAL;
+		brcmf_dbg(DATA, "%s flags %d htod %X seq %X\n", entry->name,
+			  flags, skcb->htod, seq);
+
+		/* pick up the implicit credit from this packet */
+		fifo = brcmf_skb_htod_tag_get_field(skb, FIFO);
+		if ((fws->fcmode == BRCMF_FWS_FCMODE_IMPLIED_CREDIT) ||
+		    (brcmf_skb_if_flags_get_field(skb, REQ_CREDIT)) ||
+		    (flags == BRCMF_FWS_TXSTATUS_HOST_TOSSED)) {
+			brcmf_fws_return_credits(fws, fifo, 1);
+			brcmf_fws_schedule_deq(fws);
+		}
+		brcmf_fws_macdesc_return_req_credit(skb);
+
+		ret = brcmf_proto_hdrpull(fws->drvr, false, skb, &ifp);
+		if (ret) {
+			brcmu_pkt_buf_free_skb(skb);
+			goto cont;
+		}
+		if (!remove_from_hanger)
+			ret = brcmf_fws_txstatus_suppressed(fws, fifo, skb,
+							    genbit, seq);
+		if (remove_from_hanger || ret)
+			brcmf_txfinalize(ifp, skb, true);
+
+cont:
+		hslot = (hslot + 1) & (BRCMF_FWS_TXSTAT_HSLOT_MASK >>
+				       BRCMF_FWS_TXSTAT_HSLOT_SHIFT);
+		if (BRCMF_FWS_MODE_GET_REUSESEQ(fws->mode))
+			seq = (seq + 1) & BRCMF_SKB_HTOD_SEQ_NR_MASK;
+
+		cnt++;
 	}
-	if (!remove_from_hanger)
-		ret = brcmf_fws_txstatus_suppressed(fws, fifo, skb,
-						    genbit, seq);
-	if (remove_from_hanger || ret)
-		brcmf_txfinalize(ifp, skb, true);
 
 	return 0;
 }
@@ -1547,7 +1559,8 @@
 	return BRCMF_FWS_RET_OK_SCHEDULE;
 }
 
-static int brcmf_fws_txstatus_indicate(struct brcmf_fws_info *fws, u8 *data)
+static int brcmf_fws_txstatus_indicate(struct brcmf_fws_info *fws, u8 type,
+				       u8 *data)
 {
 	__le32 status_le;
 	__le16 seq_le;
@@ -1556,23 +1569,31 @@
 	u32 genbit;
 	u8 flags;
 	u16 seq;
+	u8 compcnt;
+	u8 compcnt_offset = BRCMF_FWS_TYPE_TXSTATUS_LEN;
 
-	fws->stats.txs_indicate++;
 	memcpy(&status_le, data, sizeof(status_le));
 	status = le32_to_cpu(status_le);
 	flags = brcmf_txstatus_get_field(status, FLAGS);
 	hslot = brcmf_txstatus_get_field(status, HSLOT);
 	genbit = brcmf_txstatus_get_field(status, GENERATION);
 	if (BRCMF_FWS_MODE_GET_REUSESEQ(fws->mode)) {
-		memcpy(&seq_le, &data[BRCMF_FWS_TYPE_PKTTAG_LEN],
+		memcpy(&seq_le, &data[BRCMF_FWS_TYPE_TXSTATUS_LEN],
 		       sizeof(seq_le));
 		seq = le16_to_cpu(seq_le);
+		compcnt_offset += BRCMF_FWS_TYPE_SEQ_LEN;
 	} else {
 		seq = 0;
 	}
 
+	if (type == BRCMF_FWS_TYPE_COMP_TXSTATUS)
+		compcnt = data[compcnt_offset];
+	else
+		compcnt = 1;
+	fws->stats.txs_indicate += compcnt;
+
 	brcmf_fws_lock(fws);
-	brcmf_fws_txs_process(fws, flags, hslot, genbit, seq);
+	brcmf_fws_txs_process(fws, flags, hslot, genbit, seq, compcnt);
 	brcmf_fws_unlock(fws);
 	return BRCMF_FWS_RET_OK_NOSCHEDULE;
 }
@@ -1599,19 +1620,22 @@
 		brcmf_err("event payload too small (%d)\n", e->datalen);
 		return -EINVAL;
 	}
-	if (fws->creditmap_received)
-		return 0;
 
 	fws->creditmap_received = true;
 
 	brcmf_dbg(TRACE, "enter: credits %pM\n", credits);
 	brcmf_fws_lock(fws);
 	for (i = 0; i < ARRAY_SIZE(fws->fifo_credit); i++) {
-		if (*credits)
+		fws->fifo_credit[i] += credits[i] - fws->init_fifo_credit[i];
+		fws->init_fifo_credit[i] = credits[i];
+		if (fws->fifo_credit[i] > 0)
 			fws->fifo_credit_map |= 1 << i;
 		else
 			fws->fifo_credit_map &= ~(1 << i);
-		fws->fifo_credit[i] = *credits++;
+		if (fws->fifo_credit[i] < 0)
+			brcmf_err("fifo_credit[%d] value is negative(%d)\n",
+				  i, fws->fifo_credit[i]);
+		fws->fifo_init_credit[i] = fws->fifo_credit[i];
 	}
 	brcmf_fws_schedule_deq(fws);
 	brcmf_fws_unlock(fws);
@@ -1845,6 +1869,9 @@
 
 	WARN_ON(siglen > skb->len);
 
+	if (siglen > skb->len)
+		siglen = skb->len;
+
 	if (!siglen)
 		return;
 	/* if flow control disabled, skip to packet data and leave */
@@ -1886,8 +1913,6 @@
 
 		err = BRCMF_FWS_RET_OK_NOSCHEDULE;
 		switch (type) {
-		case BRCMF_FWS_TYPE_COMP_TXSTATUS:
-			break;
 		case BRCMF_FWS_TYPE_HOST_REORDER_RXPKTS:
 			rd = (struct brcmf_skb_reorder_data *)skb->cb;
 			rd->reorder = data;
@@ -1910,7 +1935,8 @@
 			err = brcmf_fws_request_indicate(fws, type, data);
 			break;
 		case BRCMF_FWS_TYPE_TXSTATUS:
-			brcmf_fws_txstatus_indicate(fws, data);
+		case BRCMF_FWS_TYPE_COMP_TXSTATUS:
+			brcmf_fws_txstatus_indicate(fws, type, data);
 			break;
 		case BRCMF_FWS_TYPE_FIFO_CREDITBACK:
 			err = brcmf_fws_fifocreditback_indicate(fws, data);
@@ -1999,7 +2025,7 @@
 		fws->stats.rollback_failed++;
 		hslot = brcmf_skb_htod_tag_get_field(skb, HSLOT);
 		brcmf_fws_txs_process(fws, BRCMF_FWS_TXSTATUS_HOST_TOSSED,
-				      hslot, 0, 0);
+				      hslot, 0, 0, 1);
 	} else {
 		fws->stats.rollback_success++;
 		brcmf_fws_return_credits(fws, fifo, 1);
@@ -2007,27 +2033,31 @@
 	}
 }
 
-static int brcmf_fws_borrow_credit(struct brcmf_fws_info *fws)
+static int brcmf_fws_borrow_credit(struct brcmf_fws_info *fws,
+				   int highest_lender_ac, int borrower_ac,
+				   bool borrow_all)
 {
-	int lender_ac;
+	int lender_ac, borrow_limit = 0;
 
-	if (time_after(fws->borrow_defer_timestamp, jiffies)) {
-		fws->fifo_credit_map &= ~(1 << BRCMF_FWS_FIFO_AC_BE);
-		return -ENAVAIL;
-	}
+	for (lender_ac = 0; lender_ac <= highest_lender_ac; lender_ac++) {
 
-	for (lender_ac = 0; lender_ac <= BRCMF_FWS_FIFO_AC_VO; lender_ac++) {
-		if (fws->fifo_credit[lender_ac]) {
-			fws->credits_borrowed[lender_ac]++;
+		if (!borrow_all)
+			borrow_limit =
+			  fws->init_fifo_credit[lender_ac] / BRCMF_BORROW_RATIO;
+		else
+			borrow_limit = 0;
+
+		if (fws->fifo_credit[lender_ac] > borrow_limit) {
+			fws->credits_borrowed[borrower_ac][lender_ac]++;
 			fws->fifo_credit[lender_ac]--;
 			if (fws->fifo_credit[lender_ac] == 0)
 				fws->fifo_credit_map &= ~(1 << lender_ac);
-			fws->fifo_credit_map |= (1 << BRCMF_FWS_FIFO_AC_BE);
+			fws->fifo_credit_map |= (1 << borrower_ac);
 			brcmf_dbg(DATA, "borrow credit from: %d\n", lender_ac);
 			return 0;
 		}
 	}
-	fws->fifo_credit_map &= ~(1 << BRCMF_FWS_FIFO_AC_BE);
+	fws->fifo_credit_map &= ~(1 << borrower_ac);
 	return -ENAVAIL;
 }
 
@@ -2096,6 +2126,7 @@
 
 int brcmf_fws_process_skb(struct brcmf_if *ifp, struct sk_buff *skb)
 {
+	struct brcmf_pub *drvr = ifp->drvr;
 	struct brcmf_fws_info *fws = drvr_to_fws(ifp->drvr);
 	struct brcmf_skbuff_cb *skcb = brcmf_skbcb(skb);
 	struct ethhdr *eh = (struct ethhdr *)(skb->data);
@@ -2109,8 +2140,10 @@
 	skcb->if_flags = 0;
 	skcb->state = BRCMF_FWS_SKBSTATE_NEW;
 	brcmf_skb_if_flags_set_field(skb, INDEX, ifp->ifidx);
+
+	/* mapping from 802.1d priority to firmware fifo index */
 	if (!multicast)
-		fifo = brcmf_fws_prio2fifo[skb->priority];
+		fifo = brcmf_map_prio_to_aci(drvr->config, skb->priority);
 
 	brcmf_fws_lock(fws);
 	if (fifo != BRCMF_FWS_FIFO_AC_BE && fifo < BRCMF_FWS_FIFO_BCMC)
@@ -2124,8 +2157,7 @@
 		brcmf_fws_enq(fws, BRCMF_FWS_SKBSTATE_DELAYED, fifo, skb);
 		brcmf_fws_schedule_deq(fws);
 	} else {
-		brcmf_err("drop skb: no hanger slot\n");
-		brcmf_txfinalize(ifp, skb, false);
+		brcmf_err("no hanger slot available\n");
 		rc = -ENOMEM;
 	}
 	brcmf_fws_unlock(fws);
@@ -2171,14 +2203,43 @@
 
 	brcmf_fws_lock(fws);
 	ifp->fws_desc = NULL;
+	brcmf_fws_macdesc_cleanup(fws, entry, ifp->ifidx);
 	brcmf_dbg(TRACE, "deleting %s\n", entry->name);
-	brcmf_fws_macdesc_cleanup(fws, &fws->desc.iface[ifp->ifidx],
-				  ifp->ifidx);
 	brcmf_fws_macdesc_deinit(entry);
 	brcmf_fws_cleanup(fws, ifp->ifidx);
 	brcmf_fws_unlock(fws);
 }
 
+static bool brcmf_fws_ismultistream(struct brcmf_fws_info *fws)
+{
+	bool ret = false;
+	u8 credit_usage = 0;
+
+	/* Check only for BE, VI and VO traffic */
+	u32 delay_map = fws->fifo_delay_map &
+		((1 << BRCMF_FWS_FIFO_AC_BE) |
+		 (1 << BRCMF_FWS_FIFO_AC_VI) |
+		 (1 << BRCMF_FWS_FIFO_AC_VO));
+
+	if (hweight_long(delay_map) > 1) {
+		ret = true;
+	} else {
+		if (fws->fifo_credit[BRCMF_FWS_FIFO_AC_BE] <
+			fws->fifo_init_credit[BRCMF_FWS_FIFO_AC_BE])
+			credit_usage++;
+		if (fws->fifo_credit[BRCMF_FWS_FIFO_AC_VI] <
+			fws->fifo_init_credit[BRCMF_FWS_FIFO_AC_VI])
+			credit_usage++;
+		if (fws->fifo_credit[BRCMF_FWS_FIFO_AC_VO] <
+			fws->fifo_init_credit[BRCMF_FWS_FIFO_AC_VO])
+			credit_usage++;
+
+		if (credit_usage > 1)
+			ret = true;
+	}
+	return ret;
+}
+
 static void brcmf_fws_dequeue_worker(struct work_struct *worker)
 {
 	struct brcmf_fws_info *fws;
@@ -2192,6 +2253,11 @@
 	fws = container_of(worker, struct brcmf_fws_info, fws_dequeue_work);
 	drvr = fws->drvr;
 
+	if (brcmf_fws_ismultistream(fws))
+		drvr->bus_if->allow_skborphan = false;
+	else
+		drvr->bus_if->allow_skborphan = true;
+
 	brcmf_fws_lock(fws);
 	for (fifo = BRCMF_FWS_FIFO_BCMC; fifo >= 0 && !fws->bus_flow_blocked;
 	     fifo--) {
@@ -2216,8 +2282,10 @@
 			}
 			continue;
 		}
-		while ((fws->fifo_credit[fifo]) || ((!fws->bcmc_credit_check) &&
-		       (fifo == BRCMF_FWS_FIFO_BCMC))) {
+
+		while ((fws->fifo_credit[fifo]) ||
+		       ((!fws->bcmc_credit_check) &&
+				(fifo == BRCMF_FWS_FIFO_BCMC))) {
 			skb = brcmf_fws_deq(fws, fifo);
 			if (!skb)
 				break;
@@ -2227,10 +2295,14 @@
 			if (fws->bus_flow_blocked)
 				break;
 		}
-		if ((fifo == BRCMF_FWS_FIFO_AC_BE) &&
-		    (fws->fifo_credit[fifo] == 0) &&
-		    (!fws->bus_flow_blocked)) {
-			while (brcmf_fws_borrow_credit(fws) == 0) {
+
+		if (fifo >= BRCMF_FWS_FIFO_AC_BE &&
+		    fifo <= BRCMF_FWS_FIFO_AC_VO &&
+		    fws->fifo_credit[fifo] == 0 &&
+		    !fws->bus_flow_blocked) {
+			while (brcmf_fws_borrow_credit(fws,
+						       fifo - 1, fifo,
+						       true) == 0) {
 				skb = brcmf_fws_deq(fws, fifo);
 				if (!skb) {
 					brcmf_fws_return_credits(fws, fifo, 1);
@@ -2405,30 +2477,26 @@
 	brcmu_pktq_init(&fws->desc.other.psq, BRCMF_FWS_PSQ_PREC_COUNT,
 			BRCMF_FWS_PSQ_LEN);
 
+	/* create debugfs file for statistics */
+	brcmf_debugfs_add_entry(drvr, "fws_stats",
+				brcmf_debugfs_fws_stats_read);
+
 	brcmf_dbg(INFO, "%s bdcv2 tlv signaling [%x]\n",
 		  fws->fw_signals ? "enabled" : "disabled", tlv);
 	return fws;
 
 fail:
-	brcmf_fws_detach_pre_delif(fws);
-	brcmf_fws_detach_post_delif(fws);
+	brcmf_fws_detach(fws);
 	return ERR_PTR(rc);
 }
 
-void brcmf_fws_detach_pre_delif(struct brcmf_fws_info *fws)
+void brcmf_fws_detach(struct brcmf_fws_info *fws)
 {
 	if (!fws)
 		return;
-	if (fws->fws_wq) {
-		destroy_workqueue(fws->fws_wq);
-		fws->fws_wq = NULL;
-	}
-}
 
-void brcmf_fws_detach_post_delif(struct brcmf_fws_info *fws)
-{
-	if (!fws)
-		return;
+	if (fws->fws_wq)
+		destroy_workqueue(fws->fws_wq);
 
 	/* cleanup */
 	brcmf_fws_lock(fws);
@@ -2439,13 +2507,6 @@
 	kfree(fws);
 }
 
-void brcmf_fws_debugfs_create(struct brcmf_pub *drvr)
-{
-	/* create debugfs file for statistics */
-	brcmf_debugfs_add_entry(drvr, "fws_stats",
-				brcmf_debugfs_fws_stats_read);
-}
-
 bool brcmf_fws_queue_skbs(struct brcmf_fws_info *fws)
 {
 	return !fws->avoid_queueing;
@@ -2469,7 +2530,8 @@
 	}
 	brcmf_fws_lock(fws);
 	hslot = brcmf_skb_htod_tag_get_field(skb, HSLOT);
-	brcmf_fws_txs_process(fws, BRCMF_FWS_TXSTATUS_HOST_TOSSED, hslot, 0, 0);
+	brcmf_fws_txs_process(fws, BRCMF_FWS_TXSTATUS_HOST_TOSSED, hslot, 0, 0,
+			      1);
 	brcmf_fws_unlock(fws);
 }
 
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h	2020-06-19 21:00:31.420227716 -0400
@@ -18,10 +18,31 @@
 #ifndef FWSIGNAL_H_
 #define FWSIGNAL_H_
 
+/**
+ * enum brcmf_fws_fifo - fifo indices used by dongle firmware.
+ *
+ * @BRCMF_FWS_FIFO_FIRST: first fifo, ie. background.
+ * @BRCMF_FWS_FIFO_AC_BK: fifo for background traffic.
+ * @BRCMF_FWS_FIFO_AC_BE: fifo for best-effort traffic.
+ * @BRCMF_FWS_FIFO_AC_VI: fifo for video traffic.
+ * @BRCMF_FWS_FIFO_AC_VO: fifo for voice traffic.
+ * @BRCMF_FWS_FIFO_BCMC: fifo for broadcast/multicast (AP only).
+ * @BRCMF_FWS_FIFO_ATIM: fifo for ATIM (AP only).
+ * @BRCMF_FWS_FIFO_COUNT: number of fifos.
+ */
+enum brcmf_fws_fifo {
+	BRCMF_FWS_FIFO_FIRST,
+	BRCMF_FWS_FIFO_AC_BK = BRCMF_FWS_FIFO_FIRST,
+	BRCMF_FWS_FIFO_AC_BE,
+	BRCMF_FWS_FIFO_AC_VI,
+	BRCMF_FWS_FIFO_AC_VO,
+	BRCMF_FWS_FIFO_BCMC,
+	BRCMF_FWS_FIFO_ATIM,
+	BRCMF_FWS_FIFO_COUNT
+};
+
 struct brcmf_fws_info *brcmf_fws_attach(struct brcmf_pub *drvr);
-void brcmf_fws_detach_pre_delif(struct brcmf_fws_info *fws);
-void brcmf_fws_detach_post_delif(struct brcmf_fws_info *fws);
-void brcmf_fws_debugfs_create(struct brcmf_pub *drvr);
+void brcmf_fws_detach(struct brcmf_fws_info *fws);
 bool brcmf_fws_queue_skbs(struct brcmf_fws_info *fws);
 bool brcmf_fws_fc_active(struct brcmf_fws_info *fws);
 void brcmf_fws_hdrpull(struct brcmf_if *ifp, s16 siglen, struct sk_buff *skb);
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c	2020-06-19 21:00:31.420227716 -0400
@@ -63,14 +63,12 @@
 
 #define BRCMF_IOCTL_REQ_PKTID			0xFFFE
 
-#define BRCMF_MSGBUF_MAX_PKT_SIZE		2048
+#define BRCMF_MSGBUF_MAX_PKT_SIZE		8192
 #define BRCMF_MSGBUF_RXBUFPOST_THRESHOLD	32
 #define BRCMF_MSGBUF_MAX_IOCTLRESPBUF_POST	8
 #define BRCMF_MSGBUF_MAX_EVENTBUF_POST		8
 
 #define BRCMF_MSGBUF_PKT_FLAGS_FRAME_802_3	0x01
-#define BRCMF_MSGBUF_PKT_FLAGS_FRAME_802_11	0x02
-#define BRCMF_MSGBUF_PKT_FLAGS_FRAME_MASK	0x07
 #define BRCMF_MSGBUF_PKT_FLAGS_PRIO_SHIFT	5
 
 #define BRCMF_MSGBUF_TX_FLUSH_CNT1		32
@@ -80,6 +78,7 @@
 #define BRCMF_MSGBUF_TRICKLE_TXWORKER_THRS	32
 #define BRCMF_MSGBUF_UPDATE_RX_PTR_THRS		48
 
+#define BRCMF_MAX_TXSTATUS_WAIT_RETRIES		10
 
 struct msgbuf_common_hdr {
 	u8				msgtype;
@@ -332,7 +331,7 @@
 	do {
 		(*idx)++;
 		if (*idx == pktids->array_size)
-			*idx = 0;
+			*idx = 1;
 		if (array[*idx].allocated.counter == 0)
 			if (atomic_cmpxchg(&array[*idx].allocated, 0, 1) == 0)
 				break;
@@ -479,7 +478,7 @@
 
 
 static int brcmf_msgbuf_query_dcmd(struct brcmf_pub *drvr, int ifidx,
-				   uint cmd, void *buf, uint len, int *fwerr)
+				   uint cmd, void *buf, uint len)
 {
 	struct brcmf_msgbuf *msgbuf = (struct brcmf_msgbuf *)drvr->proto->pd;
 	struct sk_buff *skb = NULL;
@@ -487,7 +486,6 @@
 	int err;
 
 	brcmf_dbg(MSGBUF, "ifidx=%d, cmd=%d, len=%d\n", ifidx, cmd, len);
-	*fwerr = 0;
 	msgbuf->ctl_completed = false;
 	err = brcmf_msgbuf_tx_ioctl(drvr, ifidx, cmd, buf, len);
 	if (err)
@@ -511,15 +509,14 @@
 	}
 	brcmu_pkt_buf_free_skb(skb);
 
-	*fwerr = msgbuf->ioctl_resp_status;
-	return 0;
+	return msgbuf->ioctl_resp_status;
 }
 
 
 static int brcmf_msgbuf_set_dcmd(struct brcmf_pub *drvr, int ifidx,
-				 uint cmd, void *buf, uint len, int *fwerr)
+				 uint cmd, void *buf, uint len)
 {
-	return brcmf_msgbuf_query_dcmd(drvr, ifidx, cmd, buf, len, fwerr);
+	return brcmf_msgbuf_query_dcmd(drvr, ifidx, cmd, buf, len);
 }
 
 
@@ -797,8 +794,12 @@
 	flowid = brcmf_flowring_lookup(flow, eh->h_dest, skb->priority, ifidx);
 	if (flowid == BRCMF_FLOWRING_INVALID_ID) {
 		flowid = brcmf_msgbuf_flowring_create(msgbuf, ifidx, skb);
-		if (flowid == BRCMF_FLOWRING_INVALID_ID)
+		if (flowid == BRCMF_FLOWRING_INVALID_ID) {
 			return -ENOMEM;
+		} else {
+			brcmf_flowring_enqueue(flow, flowid, skb);
+			return 0;
+		}
 	}
 	queue_count = brcmf_flowring_enqueue(flow, flowid, skb);
 	force = ((queue_count % BRCMF_MSGBUF_TRICKLE_TXWORKER_THRS) == 0);
@@ -1130,7 +1131,6 @@
 	struct sk_buff *skb;
 	u16 data_offset;
 	u16 buflen;
-	u16 flags;
 	u32 idx;
 	struct brcmf_if *ifp;
 
@@ -1140,7 +1140,6 @@
 	data_offset = le16_to_cpu(rx_complete->data_offset);
 	buflen = le16_to_cpu(rx_complete->data_len);
 	idx = le32_to_cpu(rx_complete->msg.request_id);
-	flags = le16_to_cpu(rx_complete->flags);
 
 	skb = brcmf_msgbuf_get_pktid(msgbuf->drvr->bus_if->dev,
 				     msgbuf->rx_pktids, idx);
@@ -1154,20 +1153,6 @@
 
 	skb_trim(skb, buflen);
 
-	if ((flags & BRCMF_MSGBUF_PKT_FLAGS_FRAME_MASK) ==
-	    BRCMF_MSGBUF_PKT_FLAGS_FRAME_802_11) {
-		ifp = msgbuf->drvr->mon_if;
-
-		if (!ifp) {
-			brcmf_err("Received unexpected monitor pkt\n");
-			brcmu_pkt_buf_free_skb(skb);
-			return;
-		}
-
-		brcmf_netif_mon_rx(ifp, skb);
-		return;
-	}
-
 	ifp = brcmf_get_ifp(msgbuf->drvr, rx_complete->msg.ifidx);
 	if (!ifp || !ifp->ndev) {
 		brcmf_err("Received pkt for invalid ifidx %d\n",
@@ -1344,9 +1329,27 @@
 	struct brcmf_msgbuf *msgbuf = (struct brcmf_msgbuf *)drvr->proto->pd;
 	struct msgbuf_tx_flowring_delete_req *delete;
 	struct brcmf_commonring *commonring;
+	struct brcmf_commonring *commonring_del = msgbuf->flowrings[flowid];
+	struct brcmf_flowring *flow = msgbuf->flow;
 	void *ret_ptr;
 	u8 ifidx;
 	int err;
+	int retry = BRCMF_MAX_TXSTATUS_WAIT_RETRIES;
+
+	/* make sure it is not in txflow */
+	brcmf_commonring_lock(commonring_del);
+	flow->rings[flowid]->status = RING_CLOSING;
+	brcmf_commonring_unlock(commonring_del);
+
+	/* wait for commonring txflow finished */
+	while (retry && atomic_read(&commonring_del->outstanding_tx)) {
+		usleep_range(5000, 10000);
+		retry--;
+	}
+	if (!retry) {
+		brcmf_err("timed out waiting for txstatus\n");
+		atomic_set(&commonring_del->outstanding_tx, 0);
+	}
 
 	commonring = msgbuf->commonrings[BRCMF_H2D_MSGRING_CONTROL_SUBMIT];
 	brcmf_commonring_lock(commonring);
@@ -1436,11 +1439,6 @@
 }
 #endif
 
-static void brcmf_msgbuf_debugfs_create(struct brcmf_pub *drvr)
-{
-	brcmf_debugfs_add_entry(drvr, "msgbuf_stats", brcmf_msgbuf_stats_read);
-}
-
 int brcmf_proto_msgbuf_attach(struct brcmf_pub *drvr)
 {
 	struct brcmf_bus_msgbuf *if_msgbuf;
@@ -1495,7 +1493,6 @@
 	drvr->proto->delete_peer = brcmf_msgbuf_delete_peer;
 	drvr->proto->add_tdls_peer = brcmf_msgbuf_add_tdls_peer;
 	drvr->proto->rxreorder = brcmf_msgbuf_rxreorder;
-	drvr->proto->debugfs_create = brcmf_msgbuf_debugfs_create;
 	drvr->proto->pd = msgbuf;
 
 	init_waitqueue_head(&msgbuf->ioctl_resp_wait);
@@ -1504,9 +1501,8 @@
 		(struct brcmf_commonring **)if_msgbuf->commonrings;
 	msgbuf->flowrings = (struct brcmf_commonring **)if_msgbuf->flowrings;
 	msgbuf->max_flowrings = if_msgbuf->max_flowrings;
-	msgbuf->flowring_dma_handle =
-		kcalloc(msgbuf->max_flowrings,
-			sizeof(*msgbuf->flowring_dma_handle), GFP_KERNEL);
+	msgbuf->flowring_dma_handle = kzalloc(msgbuf->max_flowrings *
+		sizeof(*msgbuf->flowring_dma_handle), GFP_KERNEL);
 	if (!msgbuf->flowring_dma_handle)
 		goto fail;
 
@@ -1550,6 +1546,8 @@
 	spin_lock_init(&msgbuf->flowring_work_lock);
 	INIT_LIST_HEAD(&msgbuf->work_queue);
 
+	brcmf_debugfs_add_entry(drvr, "msgbuf_stats", brcmf_msgbuf_stats_read);
+
 	return 0;
 
 fail:
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c	2020-06-19 21:00:31.420227716 -0400
@@ -30,14 +30,22 @@
 	struct device_node *np = dev->of_node;
 	int irq;
 	u32 irqf;
-	u32 val;
+	u32 val32;
+	u16 val16;
 
 	if (!np || bus_type != BRCMF_BUSTYPE_SDIO ||
 	    !of_device_is_compatible(np, "brcm,bcm4329-fmac"))
 		return;
 
-	if (of_property_read_u32(np, "brcm,drive-strength", &val) == 0)
-		sdio->drive_strength = val;
+	if (of_property_read_u32(np, "brcm,drive-strength", &val32) == 0)
+		sdio->drive_strength = val32;
+
+	sdio->broken_sg_support = of_property_read_bool(np,
+			"brcm,broken_sg_support");
+	if (of_property_read_u16(np, "brcm,sd_head_align", &val16) == 0)
+		sdio->sd_head_align = val16;
+	if (of_property_read_u16(np, "brcm,sd_sgentry_align", &val16) == 0)
+		sdio->sd_sgentry_align = val16;
 
 	/* make sure there are interrupts defined in the node */
 	if (!of_find_property(np, "interrupts", NULL))
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c	2020-06-19 21:00:31.420227716 -0400
@@ -28,6 +28,7 @@
 #include "fwil_types.h"
 #include "p2p.h"
 #include "cfg80211.h"
+#include "feature.h"
 
 /* parameters used for p2p escan */
 #define P2PAPI_SCAN_NPROBES 1
@@ -70,12 +71,13 @@
 #define P2P_AF_MIN_DWELL_TIME		100
 #define P2P_AF_MED_DWELL_TIME		400
 #define P2P_AF_LONG_DWELL_TIME		1000
-#define P2P_AF_TX_MAX_RETRY		1
+#define P2P_AF_TX_MAX_RETRY		5
 #define P2P_AF_MAX_WAIT_TIME		msecs_to_jiffies(2000)
 #define P2P_INVALID_CHANNEL		-1
 #define P2P_CHANNEL_SYNC_RETRY		5
 #define P2P_AF_FRM_SCAN_MAX_WAIT	msecs_to_jiffies(450)
 #define P2P_DEFAULT_SLEEP_TIME_VSDB	200
+#define P2P_AF_RETRY_DELAY_TIME		40
 
 /* WiFi P2P Public Action Frame OUI Subtypes */
 #define P2P_PAF_GON_REQ		0	/* Group Owner Negotiation Req */
@@ -103,6 +105,9 @@
 #define P2PSD_ACTION_ID_GAS_CRESP	0x0d	/* GAS Comback Response AF */
 
 #define BRCMF_P2P_DISABLE_TIMEOUT	msecs_to_jiffies(500)
+
+/* Mask for retry counter of custom dwell time */
+#define CUSTOM_RETRY_MASK 0xff000000
 /**
  * struct brcmf_p2p_disc_st_le - set discovery state in firmware.
  *
@@ -467,10 +472,21 @@
  */
 static void brcmf_p2p_generate_bss_mac(struct brcmf_p2p_info *p2p, u8 *dev_addr)
 {
+	struct brcmf_if *pri_ifp = p2p->bss_idx[P2PAPI_BSSCFG_PRIMARY].vif->ifp;
 	bool random_addr = false;
+	bool local_admin = false;
 
-	if (!dev_addr || is_zero_ether_addr(dev_addr))
-		random_addr = true;
+	if (!dev_addr || is_zero_ether_addr(dev_addr)) {
+		/* If the primary interface address is already locally
+		 * administered, create a new random address.
+		 */
+		if (pri_ifp->mac_addr[0] & 0x02) {
+			random_addr = true;
+		} else {
+			dev_addr = pri_ifp->mac_addr;
+			local_admin = true;
+		}
+	}
 
 	/* Generate the P2P Device Address obtaining a random ethernet
 	 * address with the locally administered bit set.
@@ -480,13 +496,20 @@
 	else
 		memcpy(p2p->dev_addr, dev_addr, ETH_ALEN);
 
+	if (local_admin)
+		p2p->dev_addr[0] |= 0x02;
+
 	/* Generate the P2P Interface Address.  If the discovery and connection
 	 * BSSCFGs need to simultaneously co-exist, then this address must be
 	 * different from the P2P Device Address, but also locally administered.
 	 */
-	memcpy(p2p->int_addr, p2p->dev_addr, ETH_ALEN);
-	p2p->int_addr[0] |= 0x02;
-	p2p->int_addr[4] ^= 0x80;
+	memcpy(p2p->conn_int_addr, p2p->dev_addr, ETH_ALEN);
+	p2p->conn_int_addr[0] |= 0x02;
+	p2p->conn_int_addr[4] ^= 0x80;
+
+	memcpy(p2p->conn2_int_addr, p2p->dev_addr, ETH_ALEN);
+	p2p->conn2_int_addr[0] |= 0x02;
+	p2p->conn2_int_addr[4] ^= 0x90;
 }
 
 /**
@@ -690,7 +713,10 @@
 
 	/* determine the scan engine parameters */
 	sparams->bss_type = DOT11_BSSTYPE_ANY;
-	sparams->scan_type = BRCMF_SCANTYPE_ACTIVE;
+	if (p2p->cfg->active_scan)
+		sparams->scan_type = 0;
+	else
+		sparams->scan_type = 1;
 
 	eth_broadcast_addr(sparams->bssid);
 	sparams->home_time = cpu_to_le32(P2PAPI_SCAN_HOME_TIME_MS);
@@ -879,7 +905,7 @@
 {
 	struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy);
 	struct brcmf_p2p_info *p2p = &cfg->p2p;
-	int err;
+	int err = 0;
 
 	if (brcmf_p2p_scan_is_p2p_request(request)) {
 		/* find my listen channel */
@@ -902,7 +928,9 @@
 		/* override .run_escan() callback. */
 		cfg->escan_info.run = brcmf_p2p_run_escan;
 	}
-	return 0;
+	err = brcmf_vif_set_mgmt_ie(vif, BRCMF_VNDR_IE_PRBREQ_FLAG,
+				    request->ie, request->ie_len);
+	return err;
 }
 
 
@@ -1058,7 +1086,7 @@
 		channel_cnt = AF_PEER_SEARCH_CNT;
 	else
 		channel_cnt = SOCIAL_CHAN_CNT;
-	default_chan_list = kcalloc(channel_cnt, sizeof(*default_chan_list),
+	default_chan_list = kzalloc(channel_cnt * sizeof(*default_chan_list),
 				    GFP_KERNEL);
 	if (default_chan_list == NULL) {
 		brcmf_err("channel list allocation failed\n");
@@ -1249,6 +1277,30 @@
 }
 
 /**
+ * brcmf_p2p_abort_action_frame() - abort action frame.
+ *
+ * @cfg: common configuration struct.
+ *
+ */
+static s32 brcmf_p2p_abort_action_frame(struct brcmf_cfg80211_info *cfg)
+{
+	struct brcmf_p2p_info *p2p = &cfg->p2p;
+	struct brcmf_cfg80211_vif *vif;
+	s32 err;
+	s32 int_val = 1;
+
+	brcmf_dbg(TRACE, "Enter\n");
+
+	vif = p2p->bss_idx[P2PAPI_BSSCFG_DEVICE].vif;
+	err = brcmf_fil_bsscfg_data_set(vif->ifp, "actframe_abort", &int_val,
+					sizeof(s32));
+	if (err)
+		brcmf_err(" aborting action frame has failed (%d)\n", err);
+
+	return err;
+}
+
+/**
  * brcmf_p2p_stop_wait_next_action_frame() - finish scan if af tx complete.
  *
  * @cfg: common configuration struct.
@@ -1259,6 +1311,7 @@
 {
 	struct brcmf_p2p_info *p2p = &cfg->p2p;
 	struct brcmf_if *ifp = p2p->bss_idx[P2PAPI_BSSCFG_PRIMARY].vif->ifp;
+	s32 err;
 
 	if (test_bit(BRCMF_P2P_STATUS_SENDING_ACT_FRAME, &p2p->status) &&
 	    (test_bit(BRCMF_P2P_STATUS_ACTION_TX_COMPLETED, &p2p->status) ||
@@ -1267,8 +1320,13 @@
 		/* if channel is not zero, "actfame" uses off channel scan.
 		 * So abort scan for off channel completion.
 		 */
-		if (p2p->af_sent_channel)
-			brcmf_notify_escan_complete(cfg, ifp, true, true);
+		if (p2p->af_sent_channel) {
+			/* abort actframe using actframe_abort or abort scan */
+			err = brcmf_p2p_abort_action_frame(cfg);
+			if (err)
+				brcmf_notify_escan_complete(
+						cfg, ifp, true, true);
+		}
 	} else if (test_bit(BRCMF_P2P_STATUS_WAITING_NEXT_AF_LISTEN,
 			    &p2p->status)) {
 		brcmf_dbg(TRACE, "*** Wake UP ** abort listen for next af frame\n");
@@ -1493,6 +1551,7 @@
 				     struct brcmf_fil_af_params_le *af_params)
 {
 	struct brcmf_cfg80211_vif *vif;
+	struct brcmf_p2p_action_frame *p2p_act_frame;
 	s32 err = 0;
 	s32 timeout = 0;
 
@@ -1502,7 +1561,13 @@
 	clear_bit(BRCMF_P2P_STATUS_ACTION_TX_COMPLETED, &p2p->status);
 	clear_bit(BRCMF_P2P_STATUS_ACTION_TX_NOACK, &p2p->status);
 
-	vif = p2p->bss_idx[P2PAPI_BSSCFG_DEVICE].vif;
+	/* check if it is a p2p_presence response */
+	p2p_act_frame = (struct brcmf_p2p_action_frame *) af_params->action_frame.data;
+	if (p2p_act_frame->subtype == P2P_AF_PRESENCE_RSP)
+		vif = p2p->bss_idx[P2PAPI_BSSCFG_CONNECTION].vif;
+	else
+		vif = p2p->bss_idx[P2PAPI_BSSCFG_DEVICE].vif;
+
 	err = brcmf_fil_bsscfg_data_set(vif->ifp, "actframe", af_params,
 					sizeof(*af_params));
 	if (err) {
@@ -1641,6 +1706,17 @@
 	return err;
 }
 
+static bool brcmf_p2p_check_dwell_overflow(s32 requested_dwell,
+					   unsigned long dwell_jiffies)
+{
+	if ((requested_dwell & CUSTOM_RETRY_MASK) &&
+	    (jiffies_to_msecs(jiffies - dwell_jiffies) >
+	    (requested_dwell & ~CUSTOM_RETRY_MASK))) {
+		brcmf_err("Action frame TX retry time over dwell time!\n");
+		return true;
+	}
+	return false;
+}
 /**
  * brcmf_p2p_send_action_frame() - send action frame .
  *
@@ -1664,6 +1740,10 @@
 	s32 tx_retry;
 	s32 extra_listen_time;
 	uint delta_ms;
+	unsigned long dwell_jiffies = 0;
+	bool dwell_overflow = false;
+
+	s32 requested_dwell = af_params->dwell_time;
 
 	action_frame = &af_params->action_frame;
 	action_frame_len = le16_to_cpu(action_frame->len);
@@ -1775,12 +1855,21 @@
 		/* update channel */
 		af_params->channel = cpu_to_le32(afx_hdl->peer_chan);
 	}
+	dwell_jiffies = jiffies;
+	dwell_overflow = brcmf_p2p_check_dwell_overflow(requested_dwell,
+							dwell_jiffies);
 
 	tx_retry = 0;
 	while (!p2p->block_gon_req_tx &&
-	       (ack == false) && (tx_retry < P2P_AF_TX_MAX_RETRY)) {
+	       (!ack) && (tx_retry < P2P_AF_TX_MAX_RETRY) &&
+		!dwell_overflow) {
+		if (af_params->channel)
+			msleep(P2P_AF_RETRY_DELAY_TIME);
+
 		ack = !brcmf_p2p_tx_action_frame(p2p, af_params);
 		tx_retry++;
+		dwell_overflow = brcmf_p2p_check_dwell_overflow(requested_dwell,
+								dwell_jiffies);
 	}
 	if (ack == false) {
 		brcmf_err("Failed to send Action Frame(retry %d)\n", tx_retry);
@@ -1858,6 +1947,7 @@
 	struct afx_hdl *afx_hdl = &p2p->afx_hdl;
 	struct brcmf_cfg80211_vif *vif = ifp->vif;
 	struct brcmf_rx_mgmt_data *rxframe = (struct brcmf_rx_mgmt_data *)data;
+	u16 chanspec = be16_to_cpu(rxframe->chanspec);
 	struct brcmu_chan ch;
 	u8 *mgmt_frame;
 	u32 mgmt_frame_len;
@@ -1910,7 +2000,7 @@
 	cfg80211_rx_mgmt(&vif->wdev, freq, 0, mgmt_frame, mgmt_frame_len, 0);
 
 	brcmf_dbg(INFO, "mgmt_frame_len (%d) , e->datalen (%d), chanspec (%04x), freq (%d)\n",
-		  mgmt_frame_len, e->datalen, ch.chspec, freq);
+		  mgmt_frame_len, e->datalen, chanspec, freq);
 
 	return 0;
 }
@@ -1992,7 +2082,7 @@
 
 	if_request.type = cpu_to_le16((u16)if_type);
 	if_request.chspec = cpu_to_le16(chanspec);
-	memcpy(if_request.addr, p2p->int_addr, sizeof(if_request.addr));
+	memcpy(if_request.addr, p2p->conn_int_addr, sizeof(if_request.addr));
 
 	brcmf_cfg80211_arm_vif_event(cfg, vif);
 	err = brcmf_fil_iovar_data_set(vif->ifp, "p2p_ifupd", &if_request,
@@ -2085,13 +2175,6 @@
 	}
 
 	pri_ifp = p2p->bss_idx[P2PAPI_BSSCFG_PRIMARY].vif->ifp;
-
-	/* firmware requires unique mac address for p2pdev interface */
-	if (addr && ether_addr_equal(addr, pri_ifp->mac_addr)) {
-		brcmf_err("discovery vif must be different from primary interface\n");
-		return ERR_PTR(-EINVAL);
-	}
-
 	brcmf_p2p_generate_bss_mac(p2p, addr);
 	brcmf_p2p_set_firmware(pri_ifp, p2p->dev_addr);
 
@@ -2145,6 +2228,27 @@
 	return ERR_PTR(err);
 }
 
+int brcmf_p2p_get_conn_idx(struct brcmf_cfg80211_info *cfg)
+{
+	int i;
+	struct brcmf_if *ifp = netdev_priv(cfg_to_ndev(cfg));
+
+	if (!ifp)
+		return -ENODEV;
+
+	for (i = P2PAPI_BSSCFG_CONNECTION; i < P2PAPI_BSSCFG_MAX; i++) {
+		if (!cfg->p2p.bss_idx[i].vif) {
+			if (i == P2PAPI_BSSCFG_CONNECTION2 &&
+			    !(brcmf_feat_is_enabled(ifp, BRCMF_FEAT_RSDB))) {
+				brcmf_err("Multi p2p not supported");
+				return -EIO;
+			}
+			return i;
+		}
+	}
+	return -EIO;
+}
+
 /**
  * brcmf_p2p_add_vif() - create a new P2P virtual interface.
  *
@@ -2163,7 +2267,9 @@
 	struct brcmf_if *ifp = netdev_priv(cfg_to_ndev(cfg));
 	struct brcmf_cfg80211_vif *vif;
 	enum brcmf_fil_p2p_if_types iftype;
-	int err;
+	int err = 0;
+	int connidx;
+	u8 *p2p_intf_addr;
 
 	if (brcmf_cfg80211_vif_event_armed(cfg))
 		return ERR_PTR(-EBUSY);
@@ -2189,9 +2295,21 @@
 		return (struct wireless_dev *)vif;
 	brcmf_cfg80211_arm_vif_event(cfg, vif);
 
-	err = brcmf_p2p_request_p2p_if(&cfg->p2p, ifp, cfg->p2p.int_addr,
-				       iftype);
+	connidx = brcmf_p2p_get_conn_idx(cfg);
+
+	if (connidx == P2PAPI_BSSCFG_CONNECTION)
+		p2p_intf_addr = cfg->p2p.conn_int_addr;
+	else if (connidx == P2PAPI_BSSCFG_CONNECTION2)
+		p2p_intf_addr = cfg->p2p.conn2_int_addr;
+	else
+		err = -EINVAL;
+
+	if (!err)
+		err =  brcmf_p2p_request_p2p_if(&cfg->p2p, ifp,
+						p2p_intf_addr, iftype);
+
 	if (err) {
+		brcmf_err("request p2p interface failed\n");
 		brcmf_cfg80211_arm_vif_event(cfg, NULL);
 		goto fail;
 	}
@@ -2223,7 +2341,7 @@
 		goto fail;
 	}
 
-	cfg->p2p.bss_idx[P2PAPI_BSSCFG_CONNECTION].vif = vif;
+	cfg->p2p.bss_idx[connidx].vif = vif;
 	/* Disable firmware roaming for P2P interface  */
 	brcmf_fil_iovar_int_set(ifp, "roam_off", 1);
 	if (iftype == BRCMF_FIL_P2P_IF_GO) {
@@ -2246,7 +2364,7 @@
  */
 int brcmf_p2p_del_vif(struct wiphy *wiphy, struct wireless_dev *wdev)
 {
-	struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy);
+	struct brcmf_cfg80211_info *cfg = wiphy_priv(wiphy);
 	struct brcmf_p2p_info *p2p = &cfg->p2p;
 	struct brcmf_cfg80211_vif *vif;
 	enum nl80211_iftype iftype;
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h	2020-06-19 21:00:31.420227716 -0400
@@ -25,13 +25,15 @@
  *
  * @P2PAPI_BSSCFG_PRIMARY: maps to driver's primary bsscfg.
  * @P2PAPI_BSSCFG_DEVICE: maps to driver's P2P device discovery bsscfg.
- * @P2PAPI_BSSCFG_CONNECTION: maps to driver's P2P connection bsscfg.
+ * @P2PAPI_BSSCFG_CONNECTION: maps to driver's 1st P2P connection bsscfg.
+ * @P2PAPI_BSSCFG_CONNECTION2: maps to driver's 2nd P2P connection bsscfg.
  * @P2PAPI_BSSCFG_MAX: used for range checking.
  */
 enum p2p_bss_type {
 	P2PAPI_BSSCFG_PRIMARY, /* maps to driver's primary bsscfg */
 	P2PAPI_BSSCFG_DEVICE, /* maps to driver's P2P device discovery bsscfg */
-	P2PAPI_BSSCFG_CONNECTION, /* maps to driver's P2P connection bsscfg */
+	P2PAPI_BSSCFG_CONNECTION, /* driver's 1st P2P connection bsscfg */
+	P2PAPI_BSSCFG_CONNECTION2, /* driver's 2nd P2P connection bsscfg */
 	P2PAPI_BSSCFG_MAX
 };
 
@@ -130,7 +132,8 @@
 	struct brcmf_cfg80211_info *cfg;
 	unsigned long status;
 	u8 dev_addr[ETH_ALEN];
-	u8 int_addr[ETH_ALEN];
+	u8 conn_int_addr[ETH_ALEN];
+	u8 conn2_int_addr[ETH_ALEN];
 	struct p2p_bss bss_idx[P2PAPI_BSSCFG_MAX];
 	struct timer_list listen_timer;
 	u8 listen_channel;
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c	2020-06-19 21:00:31.420227716 -0400
@@ -39,6 +39,7 @@
 #include "chip.h"
 #include "core.h"
 #include "common.h"
+#include "cfg80211.h"
 
 
 enum brcmf_pcie_state {
@@ -46,40 +47,41 @@
 	BRCMFMAC_PCIE_STATE_UP
 };
 
-BRCMF_FW_DEF(43602, "brcmfmac43602-pcie");
-BRCMF_FW_DEF(4350, "brcmfmac4350-pcie");
-BRCMF_FW_DEF(4350C, "brcmfmac4350c2-pcie");
-BRCMF_FW_DEF(4356, "brcmfmac4356-pcie");
-BRCMF_FW_DEF(43570, "brcmfmac43570-pcie");
-BRCMF_FW_DEF(4358, "brcmfmac4358-pcie");
-BRCMF_FW_DEF(4359, "brcmfmac4359-pcie");
-BRCMF_FW_DEF(4365B, "brcmfmac4365b-pcie");
-BRCMF_FW_DEF(4365C, "brcmfmac4365c-pcie");
-BRCMF_FW_DEF(4366B, "brcmfmac4366b-pcie");
-BRCMF_FW_DEF(4366C, "brcmfmac4366c-pcie");
-BRCMF_FW_DEF(4371, "brcmfmac4371-pcie");
-
-static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = {
-	BRCMF_FW_ENTRY(BRCM_CC_43602_CHIP_ID, 0xFFFFFFFF, 43602),
-	BRCMF_FW_ENTRY(BRCM_CC_43465_CHIP_ID, 0xFFFFFFF0, 4366C),
-	BRCMF_FW_ENTRY(BRCM_CC_4350_CHIP_ID, 0x000000FF, 4350C),
-	BRCMF_FW_ENTRY(BRCM_CC_4350_CHIP_ID, 0xFFFFFF00, 4350),
-	BRCMF_FW_ENTRY(BRCM_CC_43525_CHIP_ID, 0xFFFFFFF0, 4365C),
-	BRCMF_FW_ENTRY(BRCM_CC_4356_CHIP_ID, 0xFFFFFFFF, 4356),
-	BRCMF_FW_ENTRY(BRCM_CC_43567_CHIP_ID, 0xFFFFFFFF, 43570),
-	BRCMF_FW_ENTRY(BRCM_CC_43569_CHIP_ID, 0xFFFFFFFF, 43570),
-	BRCMF_FW_ENTRY(BRCM_CC_43570_CHIP_ID, 0xFFFFFFFF, 43570),
-	BRCMF_FW_ENTRY(BRCM_CC_4358_CHIP_ID, 0xFFFFFFFF, 4358),
-	BRCMF_FW_ENTRY(BRCM_CC_4359_CHIP_ID, 0xFFFFFFFF, 4359),
-	BRCMF_FW_ENTRY(BRCM_CC_4365_CHIP_ID, 0x0000000F, 4365B),
-	BRCMF_FW_ENTRY(BRCM_CC_4365_CHIP_ID, 0xFFFFFFF0, 4365C),
-	BRCMF_FW_ENTRY(BRCM_CC_4366_CHIP_ID, 0x0000000F, 4366B),
-	BRCMF_FW_ENTRY(BRCM_CC_4366_CHIP_ID, 0xFFFFFFF0, 4366C),
-	BRCMF_FW_ENTRY(BRCM_CC_43664_CHIP_ID, 0xFFFFFFF0, 4366C),
-	BRCMF_FW_ENTRY(BRCM_CC_4371_CHIP_ID, 0xFFFFFFFF, 4371),
+BRCMF_FW_NVRAM_DEF(43602, "brcmfmac43602-pcie.bin", "brcmfmac43602-pcie.txt");
+BRCMF_FW_NVRAM_DEF(4350, "brcmfmac4350-pcie.bin", "brcmfmac4350-pcie.txt");
+BRCMF_FW_NVRAM_DEF(4350C, "brcmfmac4350c2-pcie.bin", "brcmfmac4350c2-pcie.txt");
+BRCMF_FW_NVRAM_DEF(4356, "brcmfmac4356-pcie.bin", "brcmfmac4356-pcie.txt");
+BRCMF_FW_NVRAM_DEF(43570, "brcmfmac43570-pcie.bin", "brcmfmac43570-pcie.txt");
+BRCMF_FW_NVRAM_DEF(4358, "brcmfmac4358-pcie.bin", "brcmfmac4358-pcie.txt");
+BRCMF_FW_NVRAM_DEF(4359, "brcmfmac4359-pcie.bin", "brcmfmac4359-pcie.txt");
+BRCMF_FW_NVRAM_DEF(4365B, "brcmfmac4365b-pcie.bin", "brcmfmac4365b-pcie.txt");
+BRCMF_FW_NVRAM_DEF(4365C, "brcmfmac4365c-pcie.bin", "brcmfmac4365c-pcie.txt");
+BRCMF_FW_NVRAM_DEF(4366B, "brcmfmac4366b-pcie.bin", "brcmfmac4366b-pcie.txt");
+BRCMF_FW_NVRAM_DEF(4366C, "brcmfmac4366c-pcie.bin", "brcmfmac4366c-pcie.txt");
+BRCMF_FW_NVRAM_DEF(4371, "brcmfmac4371-pcie.bin", "brcmfmac4371-pcie.txt");
+BRCMF_FW_NVRAM_DEF(4355, "brcmfmac89459-pcie.bin", "brcmfmac89459-pcie.txt");
+
+static struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = {
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43602_CHIP_ID, 0xFFFFFFFF, 43602),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43465_CHIP_ID, 0xFFFFFFF0, 4366C),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4350_CHIP_ID, 0x000000FF, 4350C),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4350_CHIP_ID, 0xFFFFFF00, 4350),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43525_CHIP_ID, 0xFFFFFFF0, 4365C),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4356_CHIP_ID, 0xFFFFFFFF, 4356),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43567_CHIP_ID, 0xFFFFFFFF, 43570),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43569_CHIP_ID, 0xFFFFFFFF, 43570),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43570_CHIP_ID, 0xFFFFFFFF, 43570),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4358_CHIP_ID, 0xFFFFFFFF, 4358),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4359_CHIP_ID, 0xFFFFFFFF, 4359),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4365_CHIP_ID, 0x0000000F, 4365B),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4365_CHIP_ID, 0xFFFFFFF0, 4365C),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4366_CHIP_ID, 0x0000000F, 4366B),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4366_CHIP_ID, 0xFFFFFFF0, 4366C),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4371_CHIP_ID, 0xFFFFFFFF, 4371),
+	BRCMF_FW_NVRAM_ENTRY(CY_CC_89459_CHIP_ID, 0xFFFFFFFF, 4355),
 };
 
-#define BRCMF_PCIE_FW_UP_TIMEOUT		2000 /* msec */
+#define BRCMF_PCIE_FW_UP_TIMEOUT		5000 /* msec */
 
 #define BRCMF_PCIE_REG_MAP_SIZE			(32 * 1024)
 
@@ -105,8 +107,7 @@
 #define BRCMF_PCIE_PCIE2REG_MAILBOXMASK		0x4C
 #define BRCMF_PCIE_PCIE2REG_CONFIGADDR		0x120
 #define BRCMF_PCIE_PCIE2REG_CONFIGDATA		0x124
-#define BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_0	0x140
-#define BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_1	0x144
+#define BRCMF_PCIE_PCIE2REG_H2D_MAILBOX		0x140
 
 #define BRCMF_PCIE2_INTA			0x01
 #define BRCMF_PCIE2_INTB			0x02
@@ -135,14 +136,14 @@
 						 BRCMF_PCIE_MB_INT_D2H2_DB1 | \
 						 BRCMF_PCIE_MB_INT_D2H3_DB0 | \
 						 BRCMF_PCIE_MB_INT_D2H3_DB1)
-
-#define BRCMF_PCIE_SHARED_VERSION_7		7
-#define BRCMF_PCIE_MIN_SHARED_VERSION		5
+#define BRCMF_PCIE_SHARED_VERSION_5             5
+#define BRCMF_PCIE_SHARED_VERSION_6             6
+#define BRCMF_PCIE_SHARED_VERSION_7             7
+#define BRCMF_PCIE_MIN_SHARED_VERSION		BRCMF_PCIE_SHARED_VERSION_5
 #define BRCMF_PCIE_MAX_SHARED_VERSION		BRCMF_PCIE_SHARED_VERSION_7
 #define BRCMF_PCIE_SHARED_VERSION_MASK		0x00FF
 #define BRCMF_PCIE_SHARED_DMA_INDEX		0x10000
 #define BRCMF_PCIE_SHARED_DMA_2B_IDX		0x100000
-#define BRCMF_PCIE_SHARED_HOSTRDY_DB1		0x10000000
 
 #define BRCMF_PCIE_FLAGS_HTOD_SPLIT		0x4000
 #define BRCMF_PCIE_FLAGS_DTOH_SPLIT		0x8000
@@ -270,6 +271,9 @@
 	void (*write_ptr)(struct brcmf_pciedev_info *devinfo, u32 mem_offset,
 			  u16 value);
 	struct brcmf_mp_device *settings;
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+	ulong bar1_size;
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
 };
 
 struct brcmf_pcie_ringbuf {
@@ -336,6 +340,13 @@
 	BRCMF_D2H_MSGRING_RX_COMPLETE_ITEMSIZE
 };
 
+static void brcmf_pcie_setup(struct device *dev, int ret,
+			     const struct firmware *fw,
+			     void *nvram, u32 nvram_len);
+
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+DEFINE_RAW_SPINLOCK(pcie_lock);
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
 
 static u32
 brcmf_pcie_read_reg32(struct brcmf_pciedev_info *devinfo, u32 reg_offset)
@@ -360,8 +371,24 @@
 brcmf_pcie_read_tcm8(struct brcmf_pciedev_info *devinfo, u32 mem_offset)
 {
 	void __iomem *address = devinfo->tcm + mem_offset;
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+	unsigned long flags;
+	u8 value;
+
+	raw_spin_lock_irqsave(&pcie_lock, flags);
+	if ((address - devinfo->tcm) >= devinfo->bar1_size) {
+		pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN,
+				       devinfo->bar1_size);
+		address = address - devinfo->bar1_size;
+	}
+	value = ioread8(address);
+	pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0);
+	raw_spin_unlock_irqrestore(&pcie_lock, flags);
 
+	return value;
+#else
 	return (ioread8(address));
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
 }
 
 
@@ -369,8 +396,24 @@
 brcmf_pcie_read_tcm16(struct brcmf_pciedev_info *devinfo, u32 mem_offset)
 {
 	void __iomem *address = devinfo->tcm + mem_offset;
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+	u16 value;
+	unsigned long flags;
+
+	raw_spin_lock_irqsave(&pcie_lock, flags);
+	if ((address - devinfo->tcm) >= devinfo->bar1_size) {
+		pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN,
+				       devinfo->bar1_size);
+		address = address - devinfo->bar1_size;
+	}
+	value = ioread16(address);
+	pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0);
+	raw_spin_unlock_irqrestore(&pcie_lock, flags);
 
+	return value;
+#else
 	return (ioread16(address));
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
 }
 
 
@@ -379,8 +422,22 @@
 		       u16 value)
 {
 	void __iomem *address = devinfo->tcm + mem_offset;
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+	unsigned long flags;
+
+	raw_spin_lock_irqsave(&pcie_lock, flags);
+	if ((address - devinfo->tcm) >= devinfo->bar1_size) {
+		pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN,
+				       devinfo->bar1_size);
+		address = address - devinfo->bar1_size;
+	}
 
 	iowrite16(value, address);
+	pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0);
+	raw_spin_unlock_irqrestore(&pcie_lock, flags);
+#else
+	iowrite16(value, address);
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
 }
 
 
@@ -407,8 +464,24 @@
 brcmf_pcie_read_tcm32(struct brcmf_pciedev_info *devinfo, u32 mem_offset)
 {
 	void __iomem *address = devinfo->tcm + mem_offset;
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+	u32 value;
+	unsigned long flags;
+
+	raw_spin_lock_irqsave(&pcie_lock, flags);
+	if ((address - devinfo->tcm) >= devinfo->bar1_size) {
+		pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN,
+				       devinfo->bar1_size);
+		address = address - devinfo->bar1_size;
+	}
+	value = ioread32(address);
+	pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0);
+	raw_spin_unlock_irqrestore(&pcie_lock, flags);
 
+	return value;
+#else
 	return (ioread32(address));
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
 }
 
 
@@ -417,17 +490,47 @@
 		       u32 value)
 {
 	void __iomem *address = devinfo->tcm + mem_offset;
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+	unsigned long flags;
 
+	raw_spin_lock_irqsave(&pcie_lock, flags);
+	if ((address - devinfo->tcm) >= devinfo->bar1_size) {
+		pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN,
+				       devinfo->bar1_size);
+		address = address - devinfo->bar1_size;
+	}
 	iowrite32(value, address);
+	pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0);
+	raw_spin_unlock_irqrestore(&pcie_lock, flags);
+#else
+	iowrite32(value, address);
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
 }
 
 
 static u32
 brcmf_pcie_read_ram32(struct brcmf_pciedev_info *devinfo, u32 mem_offset)
 {
-	void __iomem *addr = devinfo->tcm + devinfo->ci->rambase + mem_offset;
+	void __iomem *address = devinfo->tcm + devinfo->ci->rambase
+		+ mem_offset;
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+	u32 value;
+	unsigned long flags;
+
+	raw_spin_lock_irqsave(&pcie_lock, flags);
+	if ((address - devinfo->tcm) >= devinfo->bar1_size) {
+		pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN,
+				       devinfo->bar1_size);
+		address = address - devinfo->bar1_size;
+	}
+	value = ioread32(address);
+	pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0);
+	raw_spin_unlock_irqrestore(&pcie_lock, flags);
 
-	return (ioread32(addr));
+	return value;
+#else
+	return (ioread32(address));
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
 }
 
 
@@ -435,9 +538,23 @@
 brcmf_pcie_write_ram32(struct brcmf_pciedev_info *devinfo, u32 mem_offset,
 		       u32 value)
 {
-	void __iomem *addr = devinfo->tcm + devinfo->ci->rambase + mem_offset;
-
-	iowrite32(value, addr);
+	void __iomem *address = devinfo->tcm + devinfo->ci->rambase
+		+ mem_offset;
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+	unsigned long flags;
+
+	raw_spin_lock_irqsave(&pcie_lock, flags);
+	if ((address - devinfo->tcm) >= devinfo->bar1_size) {
+		pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN,
+				       devinfo->bar1_size);
+		address = address - devinfo->bar1_size;
+	}
+	iowrite32(value, address);
+	pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0);
+	raw_spin_unlock_irqrestore(&pcie_lock, flags);
+#else
+	iowrite32(value, address);
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
 }
 
 
@@ -449,12 +566,30 @@
 	__le32 *src32;
 	__le16 *src16;
 	u8 *src8;
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+	unsigned long flags;
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
 
 	if (((ulong)address & 4) || ((ulong)srcaddr & 4) || (len & 4)) {
 		if (((ulong)address & 2) || ((ulong)srcaddr & 2) || (len & 2)) {
 			src8 = (u8 *)srcaddr;
 			while (len) {
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+				raw_spin_lock_irqsave(&pcie_lock, flags);
+				if ((address - devinfo->tcm) >=
+				    devinfo->bar1_size) {
+					pci_write_config_dword
+						(devinfo->pdev,
+						 BCMA_PCI_BAR1_WIN,
+						 devinfo->bar1_size);
+					address = address -
+						devinfo->bar1_size;
+				}
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
 				iowrite8(*src8, address);
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+				raw_spin_unlock_irqrestore(&pcie_lock, flags);
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
 				address++;
 				src8++;
 				len--;
@@ -463,7 +598,22 @@
 			len = len / 2;
 			src16 = (__le16 *)srcaddr;
 			while (len) {
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+				raw_spin_lock_irqsave(&pcie_lock, flags);
+				if ((address - devinfo->tcm) >=
+					devinfo->bar1_size) {
+					pci_write_config_dword
+						(devinfo->pdev,
+						BCMA_PCI_BAR1_WIN,
+						devinfo->bar1_size);
+					address = address -
+						devinfo->bar1_size;
+				}
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
 				iowrite16(le16_to_cpu(*src16), address);
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+				raw_spin_unlock_irqrestore(&pcie_lock, flags);
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
 				address += 2;
 				src16++;
 				len--;
@@ -473,12 +623,29 @@
 		len = len / 4;
 		src32 = (__le32 *)srcaddr;
 		while (len) {
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+			raw_spin_lock_irqsave(&pcie_lock, flags);
+			if ((address - devinfo->tcm) >=
+			    devinfo->bar1_size) {
+				pci_write_config_dword
+					(devinfo->pdev,
+					 BCMA_PCI_BAR1_WIN,
+					 devinfo->bar1_size);
+				address = address - devinfo->bar1_size;
+			}
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
 			iowrite32(le32_to_cpu(*src32), address);
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+			raw_spin_unlock_irqrestore(&pcie_lock, flags);
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
 			address += 4;
 			src32++;
 			len--;
 		}
 	}
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+	pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0);
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
 }
 
 
@@ -490,12 +657,30 @@
 	__le32 *dst32;
 	__le16 *dst16;
 	u8 *dst8;
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+	unsigned long flags;
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
 
 	if (((ulong)address & 4) || ((ulong)dstaddr & 4) || (len & 4)) {
 		if (((ulong)address & 2) || ((ulong)dstaddr & 2) || (len & 2)) {
 			dst8 = (u8 *)dstaddr;
 			while (len) {
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+				raw_spin_lock_irqsave(&pcie_lock, flags);
+				if ((address - devinfo->tcm) >=
+				    devinfo->bar1_size) {
+					pci_write_config_dword
+						(devinfo->pdev,
+						BCMA_PCI_BAR1_WIN,
+						devinfo->bar1_size);
+					address = address -
+						devinfo->bar1_size;
+				}
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
 				*dst8 = ioread8(address);
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+				raw_spin_unlock_irqrestore(&pcie_lock, flags);
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
 				address++;
 				dst8++;
 				len--;
@@ -504,7 +689,22 @@
 			len = len / 2;
 			dst16 = (__le16 *)dstaddr;
 			while (len) {
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+				raw_spin_lock_irqsave(&pcie_lock, flags);
+				if ((address - devinfo->tcm) >=
+				    devinfo->bar1_size) {
+					pci_write_config_dword
+						(devinfo->pdev,
+						BCMA_PCI_BAR1_WIN,
+						devinfo->bar1_size);
+					address = address -
+						devinfo->bar1_size;
+				}
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
 				*dst16 = cpu_to_le16(ioread16(address));
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+				raw_spin_unlock_irqrestore(&pcie_lock, flags);
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
 				address += 2;
 				dst16++;
 				len--;
@@ -514,12 +714,29 @@
 		len = len / 4;
 		dst32 = (__le32 *)dstaddr;
 		while (len) {
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+			raw_spin_lock_irqsave(&pcie_lock, flags);
+			if ((address - devinfo->tcm) >=
+			    devinfo->bar1_size) {
+				pci_write_config_dword
+					(devinfo->pdev,
+					BCMA_PCI_BAR1_WIN,
+					devinfo->bar1_size);
+				address = address - devinfo->bar1_size;
+			}
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
 			*dst32 = cpu_to_le32(ioread32(address));
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+			raw_spin_unlock_irqrestore(&pcie_lock, flags);
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
 			address += 4;
 			dst32++;
 			len--;
 		}
 	}
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+	pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0);
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
 }
 
 
@@ -562,6 +779,9 @@
 			     BRCMF_PCIE_CFGREG_MSI_ADDR_L,
 			     BRCMF_PCIE_CFGREG_MSI_ADDR_H,
 			     BRCMF_PCIE_CFGREG_MSI_DATA,
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+			     BCMA_PCI_BAR1_WIN,
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
 			     BRCMF_PCIE_CFGREG_LINK_STATUS_CTRL2,
 			     BRCMF_PCIE_CFGREG_RBAR_CTRL,
 			     BRCMF_PCIE_CFGREG_PML1_SUB_CTRL1,
@@ -684,7 +904,6 @@
 
 	brcmf_pcie_write_tcm32(devinfo, addr, htod_mb_data);
 	pci_write_config_dword(devinfo->pdev, BRCMF_PCIE_REG_SBMBX, 1);
-	pci_write_config_dword(devinfo->pdev, BRCMF_PCIE_REG_SBMBX, 1);
 
 	return 0;
 }
@@ -720,7 +939,7 @@
 	}
 	if (dtoh_mb_data & BRCMF_D2H_DEV_FWHALT) {
 		brcmf_dbg(PCIE, "D2H_MB_DATA: FW HALT\n");
-		brcmf_dev_coredump(&devinfo->pdev->dev);
+		brcmf_fw_crashed(&devinfo->pdev->dev);
 	}
 }
 
@@ -798,12 +1017,6 @@
 			       BRCMF_PCIE_MB_INT_FN0_1);
 }
 
-static void brcmf_pcie_hostready(struct brcmf_pciedev_info *devinfo)
-{
-	if (devinfo->shared.flags & BRCMF_PCIE_SHARED_HOSTRDY_DB1)
-		brcmf_pcie_write_reg32(devinfo,
-				       BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_1, 1);
-}
 
 static irqreturn_t brcmf_pcie_quick_check_isr(int irq, void *arg)
 {
@@ -946,7 +1159,7 @@
 
 	brcmf_dbg(PCIE, "RING !\n");
 	/* Any arbitrary value will do, lets use 1 */
-	brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_0, 1);
+	brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_PCIE2REG_H2D_MAILBOX, 1);
 
 	return 0;
 }
@@ -1117,9 +1330,14 @@
 	u16 max_flowrings;
 	u16 max_submissionrings;
 	u16 max_completionrings;
-
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+	brcmf_pcie_copy_dev_tomem(devinfo, devinfo->shared.ring_info_addr,
+				  &ringinfo, sizeof(ringinfo));
+#else
 	memcpy_fromio(&ringinfo, devinfo->tcm + devinfo->shared.ring_info_addr,
 		      sizeof(ringinfo));
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
+
 	if (devinfo->shared.version >= 6) {
 		max_submissionrings = le16_to_cpu(ringinfo.max_submissionrings);
 		max_flowrings = le16_to_cpu(ringinfo.max_flowrings);
@@ -1188,8 +1406,14 @@
 		ringinfo.d2h_r_idx_hostaddr.high_addr =
 			cpu_to_le32(address >> 32);
 
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+		brcmf_pcie_copy_mem_todev(devinfo,
+					  devinfo->shared.ring_info_addr,
+					  &ringinfo, sizeof(ringinfo));
+#else
 		memcpy_toio(devinfo->tcm + devinfo->shared.ring_info_addr,
 			    &ringinfo, sizeof(ringinfo));
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
 		brcmf_dbg(PCIE, "Using host memory indices\n");
 	}
 
@@ -1280,14 +1504,14 @@
 	u64 address;
 	u32 addr;
 
-	devinfo->shared.scratch =
-		dma_zalloc_coherent(&devinfo->pdev->dev,
-					BRCMF_DMA_D2H_SCRATCH_BUF_LEN,
-					&devinfo->shared.scratch_dmahandle,
-					GFP_KERNEL);
+	devinfo->shared.scratch = dma_alloc_coherent(&devinfo->pdev->dev,
+		BRCMF_DMA_D2H_SCRATCH_BUF_LEN,
+		&devinfo->shared.scratch_dmahandle, GFP_KERNEL);
 	if (!devinfo->shared.scratch)
 		goto fail;
 
+	memset(devinfo->shared.scratch, 0, BRCMF_DMA_D2H_SCRATCH_BUF_LEN);
+
 	addr = devinfo->shared.tcm_base_address +
 	       BRCMF_SHARED_DMA_SCRATCH_ADDR_OFFSET;
 	address = (u64)devinfo->shared.scratch_dmahandle;
@@ -1297,14 +1521,14 @@
 	       BRCMF_SHARED_DMA_SCRATCH_LEN_OFFSET;
 	brcmf_pcie_write_tcm32(devinfo, addr, BRCMF_DMA_D2H_SCRATCH_BUF_LEN);
 
-	devinfo->shared.ringupd =
-		dma_zalloc_coherent(&devinfo->pdev->dev,
-					BRCMF_DMA_D2H_RINGUPD_BUF_LEN,
-					&devinfo->shared.ringupd_dmahandle,
-					GFP_KERNEL);
+	devinfo->shared.ringupd = dma_alloc_coherent(&devinfo->pdev->dev,
+		BRCMF_DMA_D2H_RINGUPD_BUF_LEN,
+		&devinfo->shared.ringupd_dmahandle, GFP_KERNEL);
 	if (!devinfo->shared.ringupd)
 		goto fail;
 
+	memset(devinfo->shared.ringupd, 0, BRCMF_DMA_D2H_RINGUPD_BUF_LEN);
+
 	addr = devinfo->shared.tcm_base_address +
 	       BRCMF_SHARED_DMA_RINGUPD_ADDR_OFFSET;
 	address = (u64)devinfo->shared.ringupd_dmahandle;
@@ -1379,24 +1603,63 @@
 	return 0;
 }
 
-static
-int brcmf_pcie_get_fwname(struct device *dev, const char *ext, u8 *fw_name)
+static int brcmf_pcie_get_fwname(struct device *dev, u32 chip, u32 chiprev,
+				 u8 *fw_name)
 {
 	struct brcmf_bus *bus_if = dev_get_drvdata(dev);
-	struct brcmf_fw_request *fwreq;
-	struct brcmf_fw_name fwnames[] = {
-		{ ext, fw_name },
-	};
-
-	fwreq = brcmf_fw_alloc_request(bus_if->chip, bus_if->chiprev,
-				       brcmf_pcie_fwnames,
-				       ARRAY_SIZE(brcmf_pcie_fwnames),
-				       fwnames, ARRAY_SIZE(fwnames));
-	if (!fwreq)
-		return -ENOMEM;
+	struct brcmf_pciedev *buspub = bus_if->bus_priv.pcie;
+	struct brcmf_pciedev_info *devinfo = buspub->devinfo;
+	int ret = 0;
 
-	kfree(fwreq);
-	return 0;
+	if (devinfo->fw_name[0] != '\0')
+		strlcpy(fw_name, devinfo->fw_name, BRCMF_FW_NAME_LEN);
+	else
+		ret = brcmf_fw_map_chip_to_name(chip, chiprev,
+						brcmf_pcie_fwnames,
+						ARRAY_SIZE(brcmf_pcie_fwnames),
+						fw_name, NULL);
+
+	return ret;
+}
+
+static int brcmf_pcie_reset(struct device *dev)
+{
+	struct brcmf_bus *bus;
+	struct brcmf_bus *bus_if = dev_get_drvdata(dev);
+	struct brcmf_pciedev *buspub = bus_if->bus_priv.pcie;
+	struct brcmf_pciedev_info *devinfo = buspub->devinfo;
+	int err;
+	u16 domain_nr;
+	u16 bus_nr;
+
+	domain_nr = pci_domain_nr(devinfo->pdev->bus) + 1;
+	bus_nr = devinfo->pdev->bus->number;
+	brcmf_dbg(PCIE, "Enter %x:%x (%d/%d)\n", devinfo->pdev->vendor,
+		  devinfo->pdev->device, domain_nr, bus_nr);
+	bus = dev_get_drvdata(dev);
+
+	brcmf_detach(dev);
+
+	brcmf_pcie_release_irq(devinfo);
+	brcmf_pcie_release_scratchbuffers(devinfo);
+	brcmf_pcie_release_ringbuffers(devinfo);
+	brcmf_pcie_reset_device(devinfo);
+
+	err = brcmf_fw_map_chip_to_name(devinfo->ci->chip,
+					devinfo->ci->chiprev,
+					brcmf_pcie_fwnames,
+					ARRAY_SIZE(brcmf_pcie_fwnames),
+					devinfo->fw_name, devinfo->nvram_name);
+	if (err)
+		return err;
+
+	err = brcmf_fw_get_firmwares_pcie(bus->dev, BRCMF_FW_REQUEST_NVRAM |
+						    BRCMF_FW_REQ_NV_OPTIONAL,
+					  devinfo->fw_name, devinfo->nvram_name,
+					  brcmf_pcie_setup, domain_nr, bus_nr);
+	if (err == 0)
+		return 0;
+	return err;
 }
 
 static const struct brcmf_bus_ops brcmf_pcie_bus_ops = {
@@ -1408,6 +1671,7 @@
 	.get_ramsize = brcmf_pcie_get_ramsize,
 	.get_memdump = brcmf_pcie_get_memdump,
 	.get_fwname = brcmf_pcie_get_fwname,
+	.reset = brcmf_pcie_reset,
 };
 
 
@@ -1584,6 +1848,9 @@
 
 	devinfo->regs = ioremap_nocache(bar0_addr, BRCMF_PCIE_REG_MAP_SIZE);
 	devinfo->tcm = ioremap_nocache(bar1_addr, bar1_size);
+#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ
+	devinfo->bar1_size = bar1_size;
+#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */
 
 	if (!devinfo->regs || !devinfo->tcm) {
 		brcmf_err("ioremap() failed (%p,%p)\n", devinfo->regs,
@@ -1611,6 +1878,24 @@
 }
 
 
+static int brcmf_pcie_attach_bus(struct brcmf_pciedev_info *devinfo)
+{
+	int ret;
+
+	/* Attach to the common driver interface */
+	ret = brcmf_attach(&devinfo->pdev->dev, devinfo->settings);
+	if (ret) {
+		brcmf_err("brcmf_attach failed\n");
+	} else {
+		ret = brcmf_bus_started(&devinfo->pdev->dev);
+		if (ret)
+			brcmf_err("dongle is not responding\n");
+	}
+
+	return ret;
+}
+
+
 static u32 brcmf_pcie_buscore_prep_addr(const struct pci_dev *pdev, u32 addr)
 {
 	u32 ret_addr;
@@ -1681,19 +1966,15 @@
 	.write32 = brcmf_pcie_buscore_write32,
 };
 
-#define BRCMF_PCIE_FW_CODE	0
-#define BRCMF_PCIE_FW_NVRAM	1
-
 static void brcmf_pcie_setup(struct device *dev, int ret,
-			     struct brcmf_fw_request *fwreq)
+			     const struct firmware *fw,
+			     void *nvram, u32 nvram_len)
 {
-	const struct firmware *fw;
-	void *nvram;
 	struct brcmf_bus *bus;
 	struct brcmf_pciedev *pcie_bus_dev;
 	struct brcmf_pciedev_info *devinfo;
 	struct brcmf_commonring **flowrings;
-	u32 i, nvram_len;
+	u32 i;
 
 	/* check firmware loading result */
 	if (ret)
@@ -1704,11 +1985,6 @@
 	devinfo = pcie_bus_dev->devinfo;
 	brcmf_pcie_attach(devinfo);
 
-	fw = fwreq->items[BRCMF_PCIE_FW_CODE].binary;
-	nvram = fwreq->items[BRCMF_PCIE_FW_NVRAM].nv_data.data;
-	nvram_len = fwreq->items[BRCMF_PCIE_FW_NVRAM].nv_data.len;
-	kfree(fwreq);
-
 	/* Some of the firmwares have the size of the memory of the device
 	 * defined inside the firmware. This is because part of the memory in
 	 * the device is shared and the devision is determined by FW. Parse
@@ -1756,8 +2032,7 @@
 	init_waitqueue_head(&devinfo->mbdata_resp_wait);
 
 	brcmf_pcie_intr_enable(devinfo);
-	brcmf_pcie_hostready(devinfo);
-	if (brcmf_attach(&devinfo->pdev->dev, devinfo->settings) == 0)
+	if (brcmf_pcie_attach_bus(devinfo) == 0)
 		return;
 
 	brcmf_pcie_bus_console_read(devinfo);
@@ -1766,42 +2041,20 @@
 	device_release_driver(dev);
 }
 
-static struct brcmf_fw_request *
-brcmf_pcie_prepare_fw_request(struct brcmf_pciedev_info *devinfo)
-{
-	struct brcmf_fw_request *fwreq;
-	struct brcmf_fw_name fwnames[] = {
-		{ ".bin", devinfo->fw_name },
-		{ ".txt", devinfo->nvram_name },
-	};
-
-	fwreq = brcmf_fw_alloc_request(devinfo->ci->chip, devinfo->ci->chiprev,
-				       brcmf_pcie_fwnames,
-				       ARRAY_SIZE(brcmf_pcie_fwnames),
-				       fwnames, ARRAY_SIZE(fwnames));
-	if (!fwreq)
-		return NULL;
-
-	fwreq->items[BRCMF_PCIE_FW_CODE].type = BRCMF_FW_TYPE_BINARY;
-	fwreq->items[BRCMF_PCIE_FW_NVRAM].type = BRCMF_FW_TYPE_NVRAM;
-	fwreq->items[BRCMF_PCIE_FW_NVRAM].flags = BRCMF_FW_REQF_OPTIONAL;
-	/* NVRAM reserves PCI domain 0 for Broadcom's SDK faked bus */
-	fwreq->domain_nr = pci_domain_nr(devinfo->pdev->bus) + 1;
-	fwreq->bus_nr = devinfo->pdev->bus->number;
-
-	return fwreq;
-}
-
 static int
 brcmf_pcie_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 {
 	int ret;
-	struct brcmf_fw_request *fwreq;
 	struct brcmf_pciedev_info *devinfo;
 	struct brcmf_pciedev *pcie_bus_dev;
 	struct brcmf_bus *bus;
+	u16 domain_nr;
+	u16 bus_nr;
 
-	brcmf_dbg(PCIE, "Enter %x:%x\n", pdev->vendor, pdev->device);
+	domain_nr = pci_domain_nr(pdev->bus) + 1;
+	bus_nr = pdev->bus->number;
+	brcmf_dbg(PCIE, "Enter %x:%x (%d/%d)\n", pdev->vendor, pdev->device,
+		  domain_nr, bus_nr);
 
 	ret = -ENOMEM;
 	devinfo = kzalloc(sizeof(*devinfo), GFP_KERNEL);
@@ -1855,19 +2108,19 @@
 	bus->wowl_supported = pci_pme_capable(pdev, PCI_D3hot);
 	dev_set_drvdata(&pdev->dev, bus);
 
-	fwreq = brcmf_pcie_prepare_fw_request(devinfo);
-	if (!fwreq) {
-		ret = -ENOMEM;
-		goto fail_bus;
-	}
-
-	ret = brcmf_fw_get_firmwares(bus->dev, fwreq, brcmf_pcie_setup);
-	if (ret < 0) {
-		kfree(fwreq);
+	ret = brcmf_fw_map_chip_to_name(devinfo->ci->chip, devinfo->ci->chiprev,
+					brcmf_pcie_fwnames,
+					ARRAY_SIZE(brcmf_pcie_fwnames),
+					devinfo->fw_name, devinfo->nvram_name);
+	if (ret)
 		goto fail_bus;
-	}
-	return 0;
 
+	ret = brcmf_fw_get_firmwares_pcie(bus->dev, BRCMF_FW_REQUEST_NVRAM |
+						    BRCMF_FW_REQ_NV_OPTIONAL,
+					  devinfo->fw_name, devinfo->nvram_name,
+					  brcmf_pcie_setup, domain_nr, bus_nr);
+	if (ret == 0)
+		return 0;
 fail_bus:
 	kfree(bus->msgbuf);
 	kfree(bus);
@@ -1932,11 +2185,22 @@
 {
 	struct brcmf_pciedev_info *devinfo;
 	struct brcmf_bus *bus;
+	struct brcmf_cfg80211_info *config;
+	int retry = BRCMF_PM_WAIT_MAXRETRY;
 
 	brcmf_dbg(PCIE, "Enter\n");
 
 	bus = dev_get_drvdata(dev);
 	devinfo = bus->bus_priv.pcie->devinfo;
+	config = bus->drvr->config;
+
+	while (retry &&
+	       config->pm_state == BRCMF_CFG80211_PM_STATE_SUSPENDING) {
+		usleep_range(10000, 20000);
+		retry--;
+	}
+	if (!retry && config->pm_state == BRCMF_CFG80211_PM_STATE_SUSPENDING)
+		brcmf_err("timed out wait for cfg80211 suspended\n");
 
 	brcmf_bus_change_state(bus, BRCMF_BUS_DOWN);
 
@@ -1980,7 +2244,6 @@
 		brcmf_pcie_select_core(devinfo, BCMA_CORE_PCIE2);
 		brcmf_bus_change_state(bus, BRCMF_BUS_UP);
 		brcmf_pcie_intr_enable(devinfo);
-		brcmf_pcie_hostready(devinfo);
 		return 0;
 	}
 
@@ -2017,9 +2280,12 @@
 
 static const struct pci_device_id brcmf_pcie_devid_table[] = {
 	BRCMF_PCIE_DEVICE(BRCM_PCIE_4350_DEVICE_ID),
+	BRCMF_PCIE_DEVICE(BRCM_PCIE_4354_RAW_DEVICE_ID),
+	BRCMF_PCIE_DEVICE_SUB(BRCM_PCIE_4355_DEVICE_ID,BRCM_PCIE_VENDOR_ID_BROADCOM,BRCM_PCIE_4355_DEVICE_ID),
 	BRCMF_PCIE_DEVICE(BRCM_PCIE_4356_DEVICE_ID),
 	BRCMF_PCIE_DEVICE(BRCM_PCIE_43567_DEVICE_ID),
 	BRCMF_PCIE_DEVICE(BRCM_PCIE_43570_DEVICE_ID),
+	BRCMF_PCIE_DEVICE(BRCM_PCIE_43570_RAW_DEVICE_ID),
 	BRCMF_PCIE_DEVICE(BRCM_PCIE_4358_DEVICE_ID),
 	BRCMF_PCIE_DEVICE(BRCM_PCIE_4359_DEVICE_ID),
 	BRCMF_PCIE_DEVICE(BRCM_PCIE_43602_DEVICE_ID),
@@ -2034,6 +2300,8 @@
 	BRCMF_PCIE_DEVICE(BRCM_PCIE_4366_2G_DEVICE_ID),
 	BRCMF_PCIE_DEVICE(BRCM_PCIE_4366_5G_DEVICE_ID),
 	BRCMF_PCIE_DEVICE(BRCM_PCIE_4371_DEVICE_ID),
+	BRCMF_PCIE_DEVICE(CY_PCIE_89459_DEVICE_ID),
+	BRCMF_PCIE_DEVICE(CY_PCIE_89459_RAW_DEVICE_ID),
 	{ /* end: all zeroes */ }
 };
 
@@ -2050,7 +2318,6 @@
 #ifdef CONFIG_PM
 	.driver.pm = &brcmf_pciedrvr_pm,
 #endif
-	.driver.coredump = brcmf_dev_coredump,
 };
 
 
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c	2020-06-19 21:00:31.420227716 -0400
@@ -496,6 +496,11 @@
 	brcmf_dbg(TRACE, "reqid=%llu\n", reqid);
 
 	pi = ifp_to_pno(ifp);
+
+	/* No PNO reqeuset */
+	if (!pi->n_reqs)
+		return 0;
+
 	err = brcmf_pno_remove_request(pi, reqid);
 	if (err)
 		return err;
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/proto.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/proto.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/proto.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/proto.c	2020-06-19 21:00:31.420227716 -0400
@@ -54,8 +54,7 @@
 	if (!proto->tx_queue_data || (proto->hdrpull == NULL) ||
 	    (proto->query_dcmd == NULL) || (proto->set_dcmd == NULL) ||
 	    (proto->configure_addr_mode == NULL) ||
-	    (proto->delete_peer == NULL) || (proto->add_tdls_peer == NULL) ||
-	    (proto->debugfs_create == NULL)) {
+	    (proto->delete_peer == NULL) || (proto->add_tdls_peer == NULL)) {
 		brcmf_err("Not all proto handlers have been installed\n");
 		goto fail;
 	}
@@ -67,22 +66,16 @@
 	return -ENOMEM;
 }
 
-void brcmf_proto_detach_post_delif(struct brcmf_pub *drvr)
+void brcmf_proto_detach(struct brcmf_pub *drvr)
 {
 	brcmf_dbg(TRACE, "Enter\n");
 
 	if (drvr->proto) {
 		if (drvr->bus_if->proto_type == BRCMF_PROTO_BCDC)
-			brcmf_proto_bcdc_detach_post_delif(drvr);
+			brcmf_proto_bcdc_detach(drvr);
 		else if (drvr->bus_if->proto_type == BRCMF_PROTO_MSGBUF)
 			brcmf_proto_msgbuf_detach(drvr);
 		kfree(drvr->proto);
 		drvr->proto = NULL;
 	}
 }
-
-void brcmf_proto_detach_pre_delif(struct brcmf_pub *drvr)
-{
-	if (drvr->proto && drvr->bus_if->proto_type == BRCMF_PROTO_BCDC)
-		brcmf_proto_bcdc_detach_pre_delif(drvr);
-}
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/proto.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/proto.h
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/proto.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/proto.h	2020-06-19 21:00:31.420227716 -0400
@@ -30,9 +30,9 @@
 	int (*hdrpull)(struct brcmf_pub *drvr, bool do_fws,
 		       struct sk_buff *skb, struct brcmf_if **ifp);
 	int (*query_dcmd)(struct brcmf_pub *drvr, int ifidx, uint cmd,
-			  void *buf, uint len, int *fwerr);
+			  void *buf, uint len);
 	int (*set_dcmd)(struct brcmf_pub *drvr, int ifidx, uint cmd, void *buf,
-			uint len, int *fwerr);
+			uint len);
 	int (*tx_queue_data)(struct brcmf_pub *drvr, int ifidx,
 			     struct sk_buff *skb);
 	int (*txdata)(struct brcmf_pub *drvr, int ifidx, u8 offset,
@@ -48,14 +48,12 @@
 	void (*del_if)(struct brcmf_if *ifp);
 	void (*reset_if)(struct brcmf_if *ifp);
 	int (*init_done)(struct brcmf_pub *drvr);
-	void (*debugfs_create)(struct brcmf_pub *drvr);
 	void *pd;
 };
 
 
 int brcmf_proto_attach(struct brcmf_pub *drvr);
-void brcmf_proto_detach_pre_delif(struct brcmf_pub *drvr);
-void brcmf_proto_detach_post_delif(struct brcmf_pub *drvr);
+void brcmf_proto_detach(struct brcmf_pub *drvr);
 
 static inline int brcmf_proto_hdrpull(struct brcmf_pub *drvr, bool do_fws,
 				      struct sk_buff *skb,
@@ -73,16 +71,14 @@
 	return drvr->proto->hdrpull(drvr, do_fws, skb, ifp);
 }
 static inline int brcmf_proto_query_dcmd(struct brcmf_pub *drvr, int ifidx,
-					 uint cmd, void *buf, uint len,
-					 int *fwerr)
+					 uint cmd, void *buf, uint len)
 {
-	return drvr->proto->query_dcmd(drvr, ifidx, cmd, buf, len,fwerr);
+	return drvr->proto->query_dcmd(drvr, ifidx, cmd, buf, len);
 }
 static inline int brcmf_proto_set_dcmd(struct brcmf_pub *drvr, int ifidx,
-				       uint cmd, void *buf, uint len,
-				       int *fwerr)
+				       uint cmd, void *buf, uint len)
 {
-	return drvr->proto->set_dcmd(drvr, ifidx, cmd, buf, len, fwerr);
+	return drvr->proto->set_dcmd(drvr, ifidx, cmd, buf, len);
 }
 
 static inline int brcmf_proto_tx_queue_data(struct brcmf_pub *drvr, int ifidx,
@@ -158,10 +154,4 @@
 	return drvr->proto->init_done(drvr);
 }
 
-static inline void
-brcmf_proto_debugfs_create(struct brcmf_pub *drvr)
-{
-	drvr->proto->debugfs_create(drvr);
-}
-
 #endif /* BRCMFMAC_PROTO_H */
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c	2020-06-19 21:00:31.420227716 -0400
@@ -27,6 +27,7 @@
 #include <linux/mmc/sdio_ids.h>
 #include <linux/mmc/sdio_func.h>
 #include <linux/mmc/card.h>
+#include <linux/mmc/core.h>
 #include <linux/semaphore.h>
 #include <linux/firmware.h>
 #include <linux/module.h>
@@ -45,14 +46,28 @@
 #include "core.h"
 #include "common.h"
 #include "bcdc.h"
+#include "fwil.h"
 
 #define DCMD_RESP_TIMEOUT	msecs_to_jiffies(2500)
 #define CTL_DONE_TIMEOUT	msecs_to_jiffies(2500)
+#define ULP_HUDI_PROC_DONE_TIME	msecs_to_jiffies(2500)
 
-/* watermark expressed in number of words */
 #define DEFAULT_F2_WATERMARK    0x8
 #define CY_4373_F2_WATERMARK    0x40
-
+#define CY_4373_F1_MESBUSYCTRL  (CY_4373_F2_WATERMARK | SBSDIO_MESBUSYCTRL_ENAB)
+#define CY_43012_F2_WATERMARK    0x60
+#define CY_43012_MES_WATERMARK  0x50
+#define CY_43012_MESBUSYCTRL    (CY_43012_MES_WATERMARK | \
+				 SBSDIO_MESBUSYCTRL_ENAB)
+#define CY_43455_F2_WATERMARK	0x60
+#define CY_43455_MES_WATERMARK	0x50
+#define CY_43455_MESBUSYCTRL	(CY_43455_MES_WATERMARK | \
+				 SBSDIO_MESBUSYCTRL_ENAB)
+#define CY_4339_F2_WATERMARK    48
+#define CY_4339_MES_WATERMARK	80
+#define CY_4339_MESBUSYCTRL	(CY_4339_MES_WATERMARK | SBSDIO_MESBUSYCTRL_ENAB)
+#define CY_435X_F2_WATERMARK	0x40
+#define CY_435X_F1_MESBUSYCTRL	(CY_435X_F2_WATERMARK | SBSDIO_MESBUSYCTRL_ENAB)
 #ifdef DEBUG
 
 #define BRCMF_TRAP_INFO_SIZE	80
@@ -165,8 +180,8 @@
 /* manfid tuple length, include tuple, link bytes */
 #define SBSDIO_CIS_MANFID_TUPLE_LEN	6
 
-#define SD_REG(field) \
-		(offsetof(struct sdpcmd_regs, field))
+#define CORE_BUS_REG(base, field) \
+		(base + offsetof(struct sdpcmd_regs, field))
 
 /* SDIO function 1 register CHIPCLKCSR */
 /* Force ALP request to backplane */
@@ -323,14 +338,13 @@
 #define MAX_KSO_ATTEMPTS (PMU_MAX_TRANSITION_DLY/KSO_WAIT_US)
 #define BRCMF_SDIO_MAX_ACCESS_ERRORS	5
 
-/*
- * Conversion of 802.1D priority to precedence level
- */
-static uint prio2prec(u32 prio)
-{
-	return (prio == PRIO_8021D_NONE || prio == PRIO_8021D_BE) ?
-	       (prio^2) : prio;
-}
+static void brcmf_sdio_firmware_callback(struct device *dev, int err,
+					 const struct firmware *code,
+					 void *nvram, u32 nvram_len);
+static int brcmf_sdio_f2_ready(struct brcmf_sdio *bus);
+static int brcmf_ulp_event_notify(struct brcmf_if *ifp,
+				  const struct brcmf_event_msg *evtmsg,
+				  void *data);
 
 #ifdef DEBUG
 /* Device console log buffer state */
@@ -442,7 +456,6 @@
 struct brcmf_sdio {
 	struct brcmf_sdio_dev *sdiodev;	/* sdio device handler */
 	struct brcmf_chip *ci;	/* Chip info struct */
-	struct brcmf_core *sdio_core; /* sdio core info struct */
 
 	u32 hostintmask;	/* Copy of Host Interrupt Mask */
 	atomic_t intstatus;	/* Intstatus bits (events) pending */
@@ -606,46 +619,55 @@
 	{4,  0x1}
 };
 
-BRCMF_FW_DEF(43143, "brcmfmac43143-sdio");
-BRCMF_FW_DEF(43241B0, "brcmfmac43241b0-sdio");
-BRCMF_FW_DEF(43241B4, "brcmfmac43241b4-sdio");
-BRCMF_FW_DEF(43241B5, "brcmfmac43241b5-sdio");
-BRCMF_FW_DEF(4329, "brcmfmac4329-sdio");
-BRCMF_FW_DEF(4330, "brcmfmac4330-sdio");
-BRCMF_FW_DEF(4334, "brcmfmac4334-sdio");
-BRCMF_FW_DEF(43340, "brcmfmac43340-sdio");
-BRCMF_FW_DEF(4335, "brcmfmac4335-sdio");
-BRCMF_FW_DEF(43362, "brcmfmac43362-sdio");
-BRCMF_FW_DEF(4339, "brcmfmac4339-sdio");
-BRCMF_FW_DEF(43430A0, "brcmfmac43430a0-sdio");
+BRCMF_FW_NVRAM_DEF(43143, "brcmfmac43143-sdio.bin", "brcmfmac43143-sdio.txt");
+BRCMF_FW_NVRAM_DEF(43241B0, "brcmfmac43241b0-sdio.bin",
+		   "brcmfmac43241b0-sdio.txt");
+BRCMF_FW_NVRAM_DEF(43241B4, "brcmfmac43241b4-sdio.bin",
+		   "brcmfmac43241b4-sdio.txt");
+BRCMF_FW_NVRAM_DEF(43241B5, "brcmfmac43241b5-sdio.bin",
+		   "brcmfmac43241b5-sdio.txt");
+BRCMF_FW_NVRAM_DEF(4329, "brcmfmac4329-sdio.bin", "brcmfmac4329-sdio.txt");
+BRCMF_FW_NVRAM_DEF(4330, "brcmfmac4330-sdio.bin", "brcmfmac4330-sdio.txt");
+BRCMF_FW_NVRAM_DEF(4334, "brcmfmac4334-sdio.bin", "brcmfmac4334-sdio.txt");
+BRCMF_FW_NVRAM_DEF(43340, "brcmfmac43340-sdio.bin", "brcmfmac43340-sdio.txt");
+BRCMF_FW_NVRAM_DEF(4335, "brcmfmac4335-sdio.bin", "brcmfmac4335-sdio.txt");
+BRCMF_FW_NVRAM_DEF(43362, "brcmfmac43362-sdio.bin", "brcmfmac43362-sdio.txt");
+BRCMF_FW_NVRAM_DEF(4339, "brcmfmac4339-sdio.bin", "brcmfmac4339-sdio.txt");
+BRCMF_FW_NVRAM_DEF(43430A0, "brcmfmac43430a0-sdio.bin", "brcmfmac43430a0-sdio.txt");
 /* Note the names are not postfixed with a1 for backward compatibility */
-BRCMF_FW_DEF(43430A1, "brcmfmac43430-sdio");
-BRCMF_FW_DEF(43455, "brcmfmac43455-sdio");
-BRCMF_FW_DEF(4354, "brcmfmac4354-sdio");
-BRCMF_FW_DEF(4356, "brcmfmac4356-sdio");
-BRCMF_FW_DEF(4373, "brcmfmac4373-sdio");
-
-static const struct brcmf_firmware_mapping brcmf_sdio_fwnames[] = {
-	BRCMF_FW_ENTRY(BRCM_CC_43143_CHIP_ID, 0xFFFFFFFF, 43143),
-	BRCMF_FW_ENTRY(BRCM_CC_43241_CHIP_ID, 0x0000001F, 43241B0),
-	BRCMF_FW_ENTRY(BRCM_CC_43241_CHIP_ID, 0x00000020, 43241B4),
-	BRCMF_FW_ENTRY(BRCM_CC_43241_CHIP_ID, 0xFFFFFFC0, 43241B5),
-	BRCMF_FW_ENTRY(BRCM_CC_4329_CHIP_ID, 0xFFFFFFFF, 4329),
-	BRCMF_FW_ENTRY(BRCM_CC_4330_CHIP_ID, 0xFFFFFFFF, 4330),
-	BRCMF_FW_ENTRY(BRCM_CC_4334_CHIP_ID, 0xFFFFFFFF, 4334),
-	BRCMF_FW_ENTRY(BRCM_CC_43340_CHIP_ID, 0xFFFFFFFF, 43340),
-	BRCMF_FW_ENTRY(BRCM_CC_43341_CHIP_ID, 0xFFFFFFFF, 43340),
-	BRCMF_FW_ENTRY(BRCM_CC_4335_CHIP_ID, 0xFFFFFFFF, 4335),
-	BRCMF_FW_ENTRY(BRCM_CC_43362_CHIP_ID, 0xFFFFFFFE, 43362),
-	BRCMF_FW_ENTRY(BRCM_CC_4339_CHIP_ID, 0xFFFFFFFF, 4339),
-	BRCMF_FW_ENTRY(BRCM_CC_43430_CHIP_ID, 0x00000001, 43430A0),
-	BRCMF_FW_ENTRY(BRCM_CC_43430_CHIP_ID, 0xFFFFFFFE, 43430A1),
-	BRCMF_FW_ENTRY(BRCM_CC_4345_CHIP_ID, 0xFFFFFFC0, 43455),
-	BRCMF_FW_ENTRY(BRCM_CC_4354_CHIP_ID, 0xFFFFFFFF, 4354),
-	BRCMF_FW_ENTRY(BRCM_CC_4356_CHIP_ID, 0xFFFFFFFF, 4356),
-	BRCMF_FW_ENTRY(CY_CC_4373_CHIP_ID, 0xFFFFFFFF, 4373)
+BRCMF_FW_NVRAM_DEF(43430A1, "brcmfmac43430-sdio.bin", "brcmfmac43430-sdio.txt");
+BRCMF_FW_NVRAM_DEF(43455, "brcmfmac43455-sdio.bin", "brcmfmac43455-sdio.txt");
+BRCMF_FW_NVRAM_DEF(4354, "brcmfmac4354-sdio.bin", "brcmfmac4354-sdio.txt");
+BRCMF_FW_NVRAM_DEF(4356, "brcmfmac4356-sdio.bin", "brcmfmac4356-sdio.txt");
+BRCMF_FW_NVRAM_DEF(4359, "brcmfmac4359-sdio.bin", "brcmfmac4359-sdio.txt");
+BRCMF_FW_NVRAM_DEF(4373, "brcmfmac4373-sdio.bin", "brcmfmac4373-sdio.txt");
+BRCMF_FW_NVRAM_DEF(43012, "brcmfmac43012-sdio.bin", "brcmfmac43012-sdio.txt");
+
+static struct brcmf_firmware_mapping brcmf_sdio_fwnames[] = {
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43143_CHIP_ID, 0xFFFFFFFF, 43143),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43241_CHIP_ID, 0x0000001F, 43241B0),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43241_CHIP_ID, 0x00000020, 43241B4),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43241_CHIP_ID, 0xFFFFFFC0, 43241B5),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4329_CHIP_ID, 0xFFFFFFFF, 4329),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4330_CHIP_ID, 0xFFFFFFFF, 4330),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4334_CHIP_ID, 0xFFFFFFFF, 4334),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43340_CHIP_ID, 0xFFFFFFFF, 43340),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43341_CHIP_ID, 0xFFFFFFFF, 43340),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4335_CHIP_ID, 0xFFFFFFFF, 4335),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43362_CHIP_ID, 0xFFFFFFFE, 43362),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4339_CHIP_ID, 0xFFFFFFFF, 4339),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43430_CHIP_ID, 0x00000001, 43430A0),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43430_CHIP_ID, 0xFFFFFFFE, 43430A1),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4345_CHIP_ID, 0xFFFFFFC0, 43455),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4354_CHIP_ID, 0xFFFFFFFF, 4354),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4356_CHIP_ID, 0xFFFFFFFF, 4356),
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4359_CHIP_ID, 0xFFFFFFFF, 4359),
+	BRCMF_FW_NVRAM_ENTRY(CY_CC_4373_CHIP_ID, 0xFFFFFFFF, 4373),
+	BRCMF_FW_NVRAM_ENTRY(CY_CC_43012_CHIP_ID, 0xFFFFFFFF, 43012)
 };
 
+#define TXCTL_CREDITS	2
+
 static void pkt_align(struct sk_buff *p, int len, int align)
 {
 	uint datalign;
@@ -656,13 +678,48 @@
 	__skb_trim(p, len);
 }
 
-/* To check if there's window offered */
+/* To check if there's window offered
+ * Reserve 3 credits for txctl
+ */
 static bool data_ok(struct brcmf_sdio *bus)
 {
+	return (u8)(bus->tx_max - bus->tx_seq) > TXCTL_CREDITS &&
+	       ((u8)(bus->tx_max - bus->tx_seq) & 0x80) == 0;
+}
+
+/* To check if there's window offered */
+static bool txctl_ok(struct brcmf_sdio *bus)
+{
 	return (u8)(bus->tx_max - bus->tx_seq) != 0 &&
 	       ((u8)(bus->tx_max - bus->tx_seq) & 0x80) == 0;
 }
 
+/*
+ * Reads a register in the SDIO hardware block. This block occupies a series of
+ * adresses on the 32 bit backplane bus.
+ */
+static int r_sdreg32(struct brcmf_sdio *bus, u32 *regvar, u32 offset)
+{
+	struct brcmf_core *core;
+	int ret;
+
+	core = brcmf_chip_get_core(bus->ci, BCMA_CORE_SDIO_DEV);
+	*regvar = brcmf_sdiod_regrl(bus->sdiodev, core->base + offset, &ret);
+
+	return ret;
+}
+
+static int w_sdreg32(struct brcmf_sdio *bus, u32 regval, u32 reg_offset)
+{
+	struct brcmf_core *core;
+	int ret;
+
+	core = brcmf_chip_get_core(bus->ci, BCMA_CORE_SDIO_DEV);
+	brcmf_sdiod_regwl(bus->sdiodev, core->base + reg_offset, regval, &ret);
+
+	return ret;
+}
+
 static int
 brcmf_sdio_kso_control(struct brcmf_sdio *bus, bool on)
 {
@@ -673,15 +730,19 @@
 
 	brcmf_dbg(TRACE, "Enter: on=%d\n", on);
 
-	sdio_retune_crc_disable(bus->sdiodev->func1);
-
-	/* Cannot re-tune if device is asleep; defer till we're awake */
-	if (on)
-		sdio_retune_hold_now(bus->sdiodev->func1);
-
 	wr_val = (on << SBSDIO_FUNC1_SLEEPCSR_KSO_SHIFT);
 	/* 1st KSO write goes to AOS wake up core if device is asleep  */
-	brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR, wr_val, &err);
+	brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR,
+			  wr_val, &err);
+
+	/* In case of 43012 chip, the chip could go down immediately after
+	 * KSO bit is cleared. So the further reads of KSO register could
+	 * fail. Thereby just bailing out immediately after clearing KSO
+	 * bit, to avoid polling of KSO bit.
+	 */
+	if (!on && (bus->ci->chip == CY_CC_43012_CHIP_ID)) {
+		return err;
+	}
 
 	if (on) {
 		/* device WAKEUP through KSO:
@@ -707,7 +768,7 @@
 		 * just one write attempt may fail,
 		 * read it back until it matches written value
 		 */
-		rd_val = brcmf_sdiod_readb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR,
+		rd_val = brcmf_sdiod_regrb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR,
 					   &err);
 		if (!err) {
 			if ((rd_val & bmask) == cmp_val)
@@ -717,11 +778,9 @@
 		/* bail out upon subsequent access errors */
 		if (err && (err_cnt++ > BRCMF_SDIO_MAX_ACCESS_ERRORS))
 			break;
-
 		udelay(KSO_WAIT_US);
-		brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR, wr_val,
-				   &err);
-
+		brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR,
+				  wr_val, &err);
 	} while (try_cnt++ < MAX_KSO_ATTEMPTS);
 
 	if (try_cnt > 2)
@@ -731,11 +790,6 @@
 	if (try_cnt > MAX_KSO_ATTEMPTS)
 		brcmf_err("max tries: rd_val=0x%x err=%d\n", rd_val, err);
 
-	if (on)
-		sdio_retune_release(bus->sdiodev->func1);
-
-	sdio_retune_crc_enable(bus->sdiodev->func1);
-
 	return err;
 }
 
@@ -762,15 +816,15 @@
 		clkreq =
 		    bus->alp_only ? SBSDIO_ALP_AVAIL_REQ : SBSDIO_HT_AVAIL_REQ;
 
-		brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
-				   clkreq, &err);
+		brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
+				  clkreq, &err);
 		if (err) {
 			brcmf_err("HT Avail request error: %d\n", err);
 			return -EBADE;
 		}
 
 		/* Check current status */
-		clkctl = brcmf_sdiod_readb(bus->sdiodev,
+		clkctl = brcmf_sdiod_regrb(bus->sdiodev,
 					   SBSDIO_FUNC1_CHIPCLKCSR, &err);
 		if (err) {
 			brcmf_err("HT Avail read error: %d\n", err);
@@ -780,34 +834,35 @@
 		/* Go to pending and await interrupt if appropriate */
 		if (!SBSDIO_CLKAV(clkctl, bus->alp_only) && pendok) {
 			/* Allow only clock-available interrupt */
-			devctl = brcmf_sdiod_readb(bus->sdiodev,
+			devctl = brcmf_sdiod_regrb(bus->sdiodev,
 						   SBSDIO_DEVICE_CTL, &err);
 			if (err) {
-				brcmf_err("Devctl error setting CA: %d\n", err);
+				brcmf_err("Devctl error setting CA: %d\n",
+					  err);
 				return -EBADE;
 			}
 
 			devctl |= SBSDIO_DEVCTL_CA_INT_ONLY;
-			brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_DEVICE_CTL,
-					   devctl, &err);
+			brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_DEVICE_CTL,
+					  devctl, &err);
 			brcmf_dbg(SDIO, "CLKCTL: set PENDING\n");
 			bus->clkstate = CLK_PENDING;
 
 			return 0;
 		} else if (bus->clkstate == CLK_PENDING) {
 			/* Cancel CA-only interrupt filter */
-			devctl = brcmf_sdiod_readb(bus->sdiodev,
+			devctl = brcmf_sdiod_regrb(bus->sdiodev,
 						   SBSDIO_DEVICE_CTL, &err);
 			devctl &= ~SBSDIO_DEVCTL_CA_INT_ONLY;
-			brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_DEVICE_CTL,
-					   devctl, &err);
+			brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_DEVICE_CTL,
+					  devctl, &err);
 		}
 
 		/* Otherwise, wait here (polling) for HT Avail */
 		timeout = jiffies +
 			  msecs_to_jiffies(PMU_MAX_TRANSITION_DLY/1000);
 		while (!SBSDIO_CLKAV(clkctl, bus->alp_only)) {
-			clkctl = brcmf_sdiod_readb(bus->sdiodev,
+			clkctl = brcmf_sdiod_regrb(bus->sdiodev,
 						   SBSDIO_FUNC1_CHIPCLKCSR,
 						   &err);
 			if (time_after(jiffies, timeout))
@@ -841,16 +896,16 @@
 
 		if (bus->clkstate == CLK_PENDING) {
 			/* Cancel CA-only interrupt filter */
-			devctl = brcmf_sdiod_readb(bus->sdiodev,
+			devctl = brcmf_sdiod_regrb(bus->sdiodev,
 						   SBSDIO_DEVICE_CTL, &err);
 			devctl &= ~SBSDIO_DEVCTL_CA_INT_ONLY;
-			brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_DEVICE_CTL,
-					   devctl, &err);
+			brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_DEVICE_CTL,
+					  devctl, &err);
 		}
 
 		bus->clkstate = CLK_SDONLY;
-		brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
-				   clkreq, &err);
+		brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
+				  clkreq, &err);
 		brcmf_dbg(SDIO, "CLKCTL: turned OFF\n");
 		if (err) {
 			brcmf_err("Failed access turning clock off: %d\n",
@@ -940,14 +995,14 @@
 
 		/* Going to sleep */
 		if (sleep) {
-			clkcsr = brcmf_sdiod_readb(bus->sdiodev,
+			clkcsr = brcmf_sdiod_regrb(bus->sdiodev,
 						   SBSDIO_FUNC1_CHIPCLKCSR,
 						   &err);
 			if ((clkcsr & SBSDIO_CSR_MASK) == 0) {
 				brcmf_dbg(SDIO, "no clock, set ALP\n");
-				brcmf_sdiod_writeb(bus->sdiodev,
-						   SBSDIO_FUNC1_CHIPCLKCSR,
-						   SBSDIO_ALP_AVAIL_REQ, &err);
+				brcmf_sdiod_regwb(bus->sdiodev,
+						  SBSDIO_FUNC1_CHIPCLKCSR,
+						  SBSDIO_ALP_AVAIL_REQ, &err);
 			}
 			err = brcmf_sdio_kso_control(bus, false);
 		} else {
@@ -993,7 +1048,7 @@
 	struct sdpcm_shared_le sh_le;
 	__le32 addr_le;
 
-	sdio_claim_host(bus->sdiodev->func1);
+	sdio_claim_host(bus->sdiodev->func[1]);
 	brcmf_sdio_bus_sleep(bus, false, false);
 
 	/*
@@ -1027,7 +1082,7 @@
 	if (rv < 0)
 		goto fail;
 
-	sdio_release_host(bus->sdiodev->func1);
+	sdio_release_host(bus->sdiodev->func[1]);
 
 	/* Endianness */
 	sh->flags = le32_to_cpu(sh_le.flags);
@@ -1049,7 +1104,7 @@
 fail:
 	brcmf_err("unable to obtain sdpcm_shared info: rv=%d (addr=0x%x)\n",
 		  rv, addr);
-	sdio_release_host(bus->sdiodev->func1);
+	sdio_release_host(bus->sdiodev->func[1]);
 	return rv;
 }
 
@@ -1066,10 +1121,8 @@
 }
 #endif /* DEBUG */
 
-static u32 brcmf_sdio_hostmail(struct brcmf_sdio *bus)
+static u32 brcmf_sdio_hostmail(struct brcmf_sdio *bus, u32 *hmbd)
 {
-	struct brcmf_sdio_dev *sdiod = bus->sdiodev;
-	struct brcmf_core *core = bus->sdio_core;
 	u32 intstatus = 0;
 	u32 hmb_data;
 	u8 fcbits;
@@ -1078,20 +1131,18 @@
 	brcmf_dbg(SDIO, "Enter\n");
 
 	/* Read mailbox data and ack that we did so */
-	hmb_data = brcmf_sdiod_readl(sdiod,
-				     core->base + SD_REG(tohostmailboxdata),
-				     &ret);
-
-	if (!ret)
-		brcmf_sdiod_writel(sdiod, core->base + SD_REG(tosbmailbox),
-				   SMB_INT_ACK, &ret);
+	ret = r_sdreg32(bus, &hmb_data,
+			offsetof(struct sdpcmd_regs, tohostmailboxdata));
 
+	if (ret == 0)
+		w_sdreg32(bus, SMB_INT_ACK,
+			  offsetof(struct sdpcmd_regs, tosbmailbox));
 	bus->sdcnt.f1regdata += 2;
 
 	/* dongle indicates the firmware has halted/crashed */
 	if (hmb_data & HMB_DATA_FWHALT) {
-		brcmf_err("mailbox indicates firmware halted\n");
-		brcmf_dev_coredump(&sdiod->func1->dev);
+		brcmf_dbg(SDIO, "mailbox indicates firmware halted\n");
+		brcmf_fw_crashed(bus->sdiodev->dev);
 	}
 
 	/* Dongle recomposed rx frames, accept them again */
@@ -1155,14 +1206,15 @@
 			 HMB_DATA_FCDATA_MASK | HMB_DATA_VERSION_MASK))
 		brcmf_err("Unknown mailbox data content: 0x%02x\n",
 			  hmb_data);
+	/* Populate hmb_data if argument is passed for DS1 check later */
+	if (hmbd)
+		*hmbd = hmb_data;
 
 	return intstatus;
 }
 
 static void brcmf_sdio_rxfail(struct brcmf_sdio *bus, bool abort, bool rtx)
 {
-	struct brcmf_sdio_dev *sdiod = bus->sdiodev;
-	struct brcmf_core *core = bus->sdio_core;
 	uint retries = 0;
 	u16 lastrbc;
 	u8 hi, lo;
@@ -1173,18 +1225,18 @@
 		  rtx ? ", send NAK" : "");
 
 	if (abort)
-		brcmf_sdiod_abort(bus->sdiodev, bus->sdiodev->func2);
+		brcmf_sdiod_abort(bus->sdiodev, SDIO_FUNC_2);
 
-	brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_FUNC1_FRAMECTRL, SFC_RF_TERM,
-			   &err);
+	brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_FRAMECTRL,
+			  SFC_RF_TERM, &err);
 	bus->sdcnt.f1regdata++;
 
 	/* Wait until the packet has been flushed (device/FIFO stable) */
 	for (lastrbc = retries = 0xffff; retries > 0; retries--) {
-		hi = brcmf_sdiod_readb(bus->sdiodev, SBSDIO_FUNC1_RFRAMEBCHI,
-				       &err);
-		lo = brcmf_sdiod_readb(bus->sdiodev, SBSDIO_FUNC1_RFRAMEBCLO,
-				       &err);
+		hi = brcmf_sdiod_regrb(bus->sdiodev,
+				       SBSDIO_FUNC1_RFRAMEBCHI, &err);
+		lo = brcmf_sdiod_regrb(bus->sdiodev,
+				       SBSDIO_FUNC1_RFRAMEBCLO, &err);
 		bus->sdcnt.f1regdata += 2;
 
 		if ((hi == 0) && (lo == 0))
@@ -1204,8 +1256,8 @@
 
 	if (rtx) {
 		bus->sdcnt.rxrtx++;
-		brcmf_sdiod_writel(sdiod, core->base + SD_REG(tosbmailbox),
-				   SMB_NAK, &err);
+		err = w_sdreg32(bus, SMB_NAK,
+				offsetof(struct sdpcmd_regs, tosbmailbox));
 
 		bus->sdcnt.f1regdata++;
 		if (err == 0)
@@ -1225,13 +1277,13 @@
 	brcmf_err("sdio error, abort command and terminate frame\n");
 	bus->sdcnt.tx_sderrs++;
 
-	brcmf_sdiod_abort(sdiodev, sdiodev->func2);
-	brcmf_sdiod_writeb(sdiodev, SBSDIO_FUNC1_FRAMECTRL, SFC_WF_TERM, NULL);
+	brcmf_sdiod_abort(sdiodev, SDIO_FUNC_2);
+	brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_FRAMECTRL, SFC_WF_TERM, NULL);
 	bus->sdcnt.f1regdata++;
 
 	for (i = 0; i < 3; i++) {
-		hi = brcmf_sdiod_readb(sdiodev, SBSDIO_FUNC1_WFRAMEBCHI, NULL);
-		lo = brcmf_sdiod_readb(sdiodev, SBSDIO_FUNC1_WFRAMEBCLO, NULL);
+		hi = brcmf_sdiod_regrb(sdiodev, SBSDIO_FUNC1_WFRAMEBCHI, NULL);
+		lo = brcmf_sdiod_regrb(sdiodev, SBSDIO_FUNC1_WFRAMEBCLO, NULL);
 		bus->sdcnt.f1regdata += 2;
 		if ((hi == 0) && (lo == 0))
 			break;
@@ -1581,10 +1633,10 @@
 		 * read directly into the chained packet, or allocate a large
 		 * packet and and copy into the chain.
 		 */
-		sdio_claim_host(bus->sdiodev->func1);
+		sdio_claim_host(bus->sdiodev->func[1]);
 		errcode = brcmf_sdiod_recv_chain(bus->sdiodev,
 						 &bus->glom, dlen);
-		sdio_release_host(bus->sdiodev->func1);
+		sdio_release_host(bus->sdiodev->func[1]);
 		bus->sdcnt.f2rxdata++;
 
 		/* On failure, kill the superframe */
@@ -1592,11 +1644,11 @@
 			brcmf_err("glom read of %d bytes failed: %d\n",
 				  dlen, errcode);
 
-			sdio_claim_host(bus->sdiodev->func1);
+			sdio_claim_host(bus->sdiodev->func[1]);
 			brcmf_sdio_rxfail(bus, true, false);
 			bus->sdcnt.rxglomfail++;
 			brcmf_sdio_free_glom(bus);
-			sdio_release_host(bus->sdiodev->func1);
+			sdio_release_host(bus->sdiodev->func[1]);
 			return 0;
 		}
 
@@ -1606,10 +1658,10 @@
 
 		rd_new.seq_num = rxseq;
 		rd_new.len = dlen;
-		sdio_claim_host(bus->sdiodev->func1);
+		sdio_claim_host(bus->sdiodev->func[1]);
 		errcode = brcmf_sdio_hdparse(bus, pfirst->data, &rd_new,
 					     BRCMF_SDIO_FT_SUPER);
-		sdio_release_host(bus->sdiodev->func1);
+		sdio_release_host(bus->sdiodev->func[1]);
 		bus->cur_read.len = rd_new.len_nxtfrm << 4;
 
 		/* Remove superframe header, remember offset */
@@ -1625,10 +1677,10 @@
 
 			rd_new.len = pnext->len;
 			rd_new.seq_num = rxseq++;
-			sdio_claim_host(bus->sdiodev->func1);
+			sdio_claim_host(bus->sdiodev->func[1]);
 			errcode = brcmf_sdio_hdparse(bus, pnext->data, &rd_new,
 						     BRCMF_SDIO_FT_SUB);
-			sdio_release_host(bus->sdiodev->func1);
+			sdio_release_host(bus->sdiodev->func[1]);
 			brcmf_dbg_hex_dump(BRCMF_GLOM_ON(),
 					   pnext->data, 32, "subframe:\n");
 
@@ -1637,11 +1689,11 @@
 
 		if (errcode) {
 			/* Terminate frame on error */
-			sdio_claim_host(bus->sdiodev->func1);
+			sdio_claim_host(bus->sdiodev->func[1]);
 			brcmf_sdio_rxfail(bus, true, false);
 			bus->sdcnt.rxglomfail++;
 			brcmf_sdio_free_glom(bus);
-			sdio_release_host(bus->sdiodev->func1);
+			sdio_release_host(bus->sdiodev->func[1]);
 			bus->cur_read.len = 0;
 			return 0;
 		}
@@ -1722,7 +1774,8 @@
 	u8 *buf = NULL, *rbuf;
 	int sdret;
 
-	brcmf_dbg(SDIO, "Enter\n");
+	brcmf_dbg(TRACE, "Enter\n");
+
 	if (bus->rxblen)
 		buf = vzalloc(bus->rxblen);
 	if (!buf)
@@ -1825,7 +1878,7 @@
 	struct brcmf_sdio_hdrinfo *rd = &bus->cur_read, rd_new;
 	u8 head_read = 0;
 
-	brcmf_dbg(SDIO, "Enter\n");
+	brcmf_dbg(TRACE, "Enter\n");
 
 	/* Not finished unless we encounter no more frames indication */
 	bus->rxpending = true;
@@ -1848,7 +1901,7 @@
 
 		rd->len_left = rd->len;
 		/* read header first for unknow frame length */
-		sdio_claim_host(bus->sdiodev->func1);
+		sdio_claim_host(bus->sdiodev->func[1]);
 		if (!rd->len) {
 			ret = brcmf_sdiod_recv_buf(bus->sdiodev,
 						   bus->rxhdr, BRCMF_FIRSTREAD);
@@ -1858,7 +1911,7 @@
 					  ret);
 				bus->sdcnt.rx_hdrfail++;
 				brcmf_sdio_rxfail(bus, true, true);
-				sdio_release_host(bus->sdiodev->func1);
+				sdio_release_host(bus->sdiodev->func[1]);
 				continue;
 			}
 
@@ -1868,7 +1921,7 @@
 
 			if (brcmf_sdio_hdparse(bus, bus->rxhdr, rd,
 					       BRCMF_SDIO_FT_NORMAL)) {
-				sdio_release_host(bus->sdiodev->func1);
+				sdio_release_host(bus->sdiodev->func[1]);
 				if (!bus->rxpending)
 					break;
 				else
@@ -1884,7 +1937,7 @@
 				rd->len_nxtfrm = 0;
 				/* treat all packet as event if we don't know */
 				rd->channel = SDPCM_EVENT_CHANNEL;
-				sdio_release_host(bus->sdiodev->func1);
+				sdio_release_host(bus->sdiodev->func[1]);
 				continue;
 			}
 			rd->len_left = rd->len > BRCMF_FIRSTREAD ?
@@ -1901,7 +1954,7 @@
 			brcmf_err("brcmu_pkt_buf_get_skb failed\n");
 			brcmf_sdio_rxfail(bus, false,
 					    RETRYCHAN(rd->channel));
-			sdio_release_host(bus->sdiodev->func1);
+			sdio_release_host(bus->sdiodev->func[1]);
 			continue;
 		}
 		skb_pull(pkt, head_read);
@@ -1909,16 +1962,16 @@
 
 		ret = brcmf_sdiod_recv_pkt(bus->sdiodev, pkt);
 		bus->sdcnt.f2rxdata++;
-		sdio_release_host(bus->sdiodev->func1);
+		sdio_release_host(bus->sdiodev->func[1]);
 
 		if (ret < 0) {
 			brcmf_err("read %d bytes from channel %d failed: %d\n",
 				  rd->len, rd->channel, ret);
 			brcmu_pkt_buf_free_skb(pkt);
-			sdio_claim_host(bus->sdiodev->func1);
+			sdio_claim_host(bus->sdiodev->func[1]);
 			brcmf_sdio_rxfail(bus, true,
 					    RETRYCHAN(rd->channel));
-			sdio_release_host(bus->sdiodev->func1);
+			sdio_release_host(bus->sdiodev->func[1]);
 			continue;
 		}
 
@@ -1929,7 +1982,7 @@
 		} else {
 			memcpy(bus->rxhdr, pkt->data, SDPCM_HDRLEN);
 			rd_new.seq_num = rd->seq_num;
-			sdio_claim_host(bus->sdiodev->func1);
+			sdio_claim_host(bus->sdiodev->func[1]);
 			if (brcmf_sdio_hdparse(bus, bus->rxhdr, &rd_new,
 					       BRCMF_SDIO_FT_NORMAL)) {
 				rd->len = 0;
@@ -1942,11 +1995,11 @@
 					  roundup(rd_new.len, 16) >> 4);
 				rd->len = 0;
 				brcmf_sdio_rxfail(bus, true, true);
-				sdio_release_host(bus->sdiodev->func1);
+				sdio_release_host(bus->sdiodev->func[1]);
 				brcmu_pkt_buf_free_skb(pkt);
 				continue;
 			}
-			sdio_release_host(bus->sdiodev->func1);
+			sdio_release_host(bus->sdiodev->func[1]);
 			rd->len_nxtfrm = rd_new.len_nxtfrm;
 			rd->channel = rd_new.channel;
 			rd->dat_offset = rd_new.dat_offset;
@@ -1962,9 +2015,9 @@
 					  rd_new.seq_num);
 				/* Force retry w/normal header read */
 				rd->len = 0;
-				sdio_claim_host(bus->sdiodev->func1);
+				sdio_claim_host(bus->sdiodev->func[1]);
 				brcmf_sdio_rxfail(bus, false, true);
-				sdio_release_host(bus->sdiodev->func1);
+				sdio_release_host(bus->sdiodev->func[1]);
 				brcmu_pkt_buf_free_skb(pkt);
 				continue;
 			}
@@ -1987,9 +2040,9 @@
 			} else {
 				brcmf_err("%s: glom superframe w/o "
 					  "descriptor!\n", __func__);
-				sdio_claim_host(bus->sdiodev->func1);
+				sdio_claim_host(bus->sdiodev->func[1]);
 				brcmf_sdio_rxfail(bus, false, false);
-				sdio_release_host(bus->sdiodev->func1);
+				sdio_release_host(bus->sdiodev->func[1]);
 			}
 			/* prepare the descriptor for the next read */
 			rd->len = rd->len_nxtfrm << 4;
@@ -2087,7 +2140,7 @@
 	int ntail, ret;
 
 	sdiodev = bus->sdiodev;
-	blksize = sdiodev->func2->cur_blksize;
+	blksize = sdiodev->func[SDIO_FUNC_2]->cur_blksize;
 	/* sg entry alignment should be a divisor of block size */
 	WARN_ON(blksize % bus->sgentry_align);
 
@@ -2266,14 +2319,14 @@
 	if (ret)
 		goto done;
 
-	sdio_claim_host(bus->sdiodev->func1);
+	sdio_claim_host(bus->sdiodev->func[1]);
 	ret = brcmf_sdiod_send_pkt(bus->sdiodev, pktq);
 	bus->sdcnt.f2txdata++;
 
 	if (ret < 0)
 		brcmf_sdio_txfail(bus);
 
-	sdio_release_host(bus->sdiodev->func1);
+	sdio_release_host(bus->sdiodev->func[1]);
 
 done:
 	brcmf_sdio_txpkt_postp(bus, pktq);
@@ -2291,7 +2344,6 @@
 {
 	struct sk_buff *pkt;
 	struct sk_buff_head pktq;
-	u32 intstat_addr = bus->sdio_core->base + SD_REG(intstatus);
 	u32 intstatus = 0;
 	int ret = 0, prec_out, i;
 	uint cnt = 0;
@@ -2316,6 +2368,7 @@
 					      &prec_out);
 			if (pkt == NULL)
 				break;
+			skb_orphan(pkt);
 			__skb_queue_tail(&pktq, pkt);
 		}
 		spin_unlock_bh(&bus->txq_lock);
@@ -2329,11 +2382,11 @@
 		/* In poll mode, need to check for other events */
 		if (!bus->intr) {
 			/* Check device status, signal pending interrupt */
-			sdio_claim_host(bus->sdiodev->func1);
-			intstatus = brcmf_sdiod_readl(bus->sdiodev,
-						      intstat_addr, &ret);
-			sdio_release_host(bus->sdiodev->func1);
-
+			sdio_claim_host(bus->sdiodev->func[1]);
+			ret = r_sdreg32(bus, &intstatus,
+					offsetof(struct sdpcmd_regs,
+						 intstatus));
+			sdio_release_host(bus->sdiodev->func[1]);
 			bus->sdcnt.f2txdata++;
 			if (ret != 0)
 				break;
@@ -2360,7 +2413,7 @@
 	struct brcmf_sdio_hdrinfo hd_info = {0};
 	int ret;
 
-	brcmf_dbg(SDIO, "Enter\n");
+	brcmf_dbg(TRACE, "Enter\n");
 
 	/* Back the pointer to make room for bus header */
 	frame -= bus->tx_hdrlen;
@@ -2416,13 +2469,12 @@
 
 static void brcmf_sdio_bus_stop(struct device *dev)
 {
-	struct brcmf_bus *bus_if = dev_get_drvdata(dev);
-	struct brcmf_sdio_dev *sdiodev = bus_if->bus_priv.sdio;
-	struct brcmf_sdio *bus = sdiodev->bus;
-	struct brcmf_core *core = bus->sdio_core;
 	u32 local_hostintmask;
 	u8 saveclk;
 	int err;
+	struct brcmf_bus *bus_if = dev_get_drvdata(dev);
+	struct brcmf_sdio_dev *sdiodev = bus_if->bus_priv.sdio;
+	struct brcmf_sdio *bus = sdiodev->bus;
 
 	brcmf_dbg(TRACE, "Enter\n");
 
@@ -2433,37 +2485,46 @@
 	}
 
 	if (sdiodev->state != BRCMF_SDIOD_NOMEDIUM) {
-		sdio_claim_host(sdiodev->func1);
+		sdio_claim_host(sdiodev->func[1]);
 
 		/* Enable clock for device interrupts */
 		brcmf_sdio_bus_sleep(bus, false, false);
 
 		/* Disable and clear interrupts at the chip level also */
-		brcmf_sdiod_writel(sdiodev, core->base + SD_REG(hostintmask),
-				   0, NULL);
-
+		w_sdreg32(bus, 0, offsetof(struct sdpcmd_regs, hostintmask));
 		local_hostintmask = bus->hostintmask;
 		bus->hostintmask = 0;
 
 		/* Force backplane clocks to assure F2 interrupt propagates */
-		saveclk = brcmf_sdiod_readb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
+		saveclk = brcmf_sdiod_regrb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
 					    &err);
-		if (!err)
-			brcmf_sdiod_writeb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
-					   (saveclk | SBSDIO_FORCE_HT), &err);
+		if (!err) {
+			if (bus->ci->chip == CY_CC_43012_CHIP_ID) {
+				brcmf_sdiod_regwb(sdiodev,
+						  SBSDIO_FUNC1_CHIPCLKCSR,
+						  (saveclk |
+						   SBSDIO_HT_AVAIL_REQ),
+						  &err);
+			} else {
+				brcmf_sdiod_regwb(sdiodev,
+						  SBSDIO_FUNC1_CHIPCLKCSR,
+						  (saveclk | SBSDIO_FORCE_HT),
+						  &err);
+			}
+		}
 		if (err)
 			brcmf_err("Failed to force clock for F2: err %d\n",
 				  err);
 
 		/* Turn off the bus (F2), free any pending packets */
 		brcmf_dbg(INTR, "disable SDIO interrupts\n");
-		sdio_disable_func(sdiodev->func2);
+		sdio_disable_func(sdiodev->func[SDIO_FUNC_2]);
 
 		/* Clear any pending interrupts now that F2 is disabled */
-		brcmf_sdiod_writel(sdiodev, core->base + SD_REG(intstatus),
-				   local_hostintmask, NULL);
+		w_sdreg32(bus, local_hostintmask,
+			  offsetof(struct sdpcmd_regs, intstatus));
 
-		sdio_release_host(sdiodev->func1);
+		sdio_release_host(sdiodev->func[1]);
 	}
 	/* Clear the data packet queues */
 	brcmu_pktq_flush(&bus->txq, true, NULL, NULL);
@@ -2501,14 +2562,15 @@
 
 static int brcmf_sdio_intr_rstatus(struct brcmf_sdio *bus)
 {
-	struct brcmf_core *core = bus->sdio_core;
+	struct brcmf_core *buscore;
 	u32 addr;
 	unsigned long val;
 	int ret;
 
-	addr = core->base + SD_REG(intstatus);
+	buscore = brcmf_chip_get_core(bus->ci, BCMA_CORE_SDIO_DEV);
+	addr = buscore->base + offsetof(struct sdpcmd_regs, intstatus);
 
-	val = brcmf_sdiod_readl(bus->sdiodev, addr, &ret);
+	val = brcmf_sdiod_regrl(bus->sdiodev, addr, &ret);
 	bus->sdcnt.f1regdata++;
 	if (ret != 0)
 		return ret;
@@ -2518,7 +2580,7 @@
 
 	/* Clear interrupts */
 	if (val) {
-		brcmf_sdiod_writel(bus->sdiodev, addr, val, &ret);
+		brcmf_sdiod_regwl(bus->sdiodev, addr, val, &ret);
 		bus->sdcnt.f1regdata++;
 		atomic_or(val, &bus->intstatus);
 	}
@@ -2526,19 +2588,186 @@
 	return ret;
 }
 
+/* This Function is used to retrieve important
+ * details from dongle related to ULP mode Mostly
+ * values/SHM details that will be vary depending
+ * on the firmware branches
+ */
+static void
+brcmf_sdio_ulp_preinit(struct device *dev)
+{
+	struct brcmf_bus *bus_if = dev_get_drvdata(dev);
+	struct brcmf_sdio_dev *sdiodev = bus_if->bus_priv.sdio;
+	struct brcmf_if *ifp = bus_if->drvr->iflist[0];
+
+	brcmf_dbg(ULP, "Enter\n");
+
+	/* Query ulp_sdioctrl iovar to get the ULP related SHM offsets */
+	brcmf_fil_iovar_data_get(ifp, "ulp_sdioctrl",
+				 &sdiodev->fmac_ulp.ulp_shm_offset,
+				 sizeof(sdiodev->fmac_ulp.ulp_shm_offset));
+
+	sdiodev->ulp = false;
+
+	brcmf_dbg(ULP, "m_ulp_ctrl_sdio[%x] m_ulp_wakeevt_ind [%x]\n",
+		  M_DS1_CTRL_SDIO(sdiodev->fmac_ulp),
+		  M_WAKEEVENT_IND(sdiodev->fmac_ulp));
+	brcmf_dbg(ULP, "m_ulp_wakeind [%x]\n",
+		  M_ULP_WAKE_IND(sdiodev->fmac_ulp));
+}
+
+/* Reinitialize ARM because In DS1 mode ARM got off */
+static int
+brcmf_sdio_ulp_reinit_fw(struct brcmf_sdio *bus)
+{
+	struct brcmf_sdio_dev *sdiodev = bus->sdiodev;
+	int err = 0;
+
+	/* After firmware redownload tx/rx seq are reset accordingly
+	 * these values are reset on FMAC side tx_max is initially set to 4,
+	 * which later is updated by FW.
+	 */
+	bus->tx_seq = 0;
+	bus->rx_seq = 0;
+	bus->tx_max = 4;
+
+	err = brcmf_fw_get_firmwares(sdiodev->dev, BRCMF_FW_REQUEST_NVRAM,
+				     sdiodev->fw_name, sdiodev->nvram_name,
+				     brcmf_sdio_firmware_callback);
+	if (err != 0)
+		brcmf_err("async firmware request failed: %d\n", err);
+	return err;
+}
+
+/* Check if device is in DS1 mode and handshake with ULP UCODE */
+static bool
+brcmf_sdio_ulp_pre_redownload_check(struct brcmf_sdio *bus, u32 hmb_data)
+{
+	int err = 0;
+	u32 value = 0;
+	u32 val32, ulp_wake_ind, wowl_wake_ind;
+	int reg_addr;
+	unsigned long timeout;
+	struct brcmf_ulp *fmac_ulp = &bus->sdiodev->fmac_ulp;
+	int i = 0;
+
+	/* If any host mail box data is present, ignore DS1 exit sequence */
+	if (hmb_data)
+		return false;
+	/* Skip if DS1 Exit is already in progress
+	 * This can happen if firmware download is taking more time
+	 */
+	if (fmac_ulp->ulp_state == FMAC_ULP_TRIGGERED)
+		return false;
+
+	value = brcmf_sdiod_regrb(bus->sdiodev, SDIO_CCCR_IOEx, &err);
+
+	if (value == SDIO_FUNC_ENABLE_1) {
+		brcmf_dbg(ULP, "GOT THE INTERRUPT FROM UCODE\n");
+		bus->sdiodev->ulp = true;
+		fmac_ulp->ulp_state = FMAC_ULP_TRIGGERED;
+		ulp_wake_ind = D11SHM_RDW(bus->sdiodev, M_ULP_WAKE_IND(
+				  bus->sdiodev->fmac_ulp), &err);
+		wowl_wake_ind = D11SHM_RDW(bus->sdiodev, M_WAKEEVENT_IND(
+				  bus->sdiodev->fmac_ulp), &err);
+
+		brcmf_dbg(ULP, "wowl_wake_ind: 0x%08x, ulp_wake_ind: 0x%08x state %s\n",
+			  wowl_wake_ind, ulp_wake_ind, (fmac_ulp->ulp_state) ?
+			  "DS1 Exit Triggered" : "IDLE State");
+
+		if (wowl_wake_ind || ulp_wake_ind) {
+			/* RX wake Don't do anything.
+			 * Just bail out and re-download firmware.
+			 */
+			 /* Print out PHY TX error block when bit 9 set */
+			if ((ulp_wake_ind & C_DS1_PHY_TXERR) &&
+			    M_DS1_PHYTX_ERR_BLK(bus->sdiodev->fmac_ulp)) {
+				brcmf_err("Dump PHY TX Error SHM Locations\n");
+				for (i = 0; i < PHYTX_ERR_BLK_SIZE; i++) {
+					pr_err("0x%x", D11SHM_RDW(bus->sdiodev,
+								  (M_DS1_PHYTX_ERR_BLK
+								  (bus->sdiodev->fmac_ulp) +
+								  (i * 2)), &err));
+				}
+				brcmf_err("\n");
+			}
+		} else {
+			/* TX wake negotiate with MAC */
+			brcmf_dbg(ULP, "M_DS1_CTRL_SDIO: 0x%08x\n",
+				  (u32)D11SHM_RDW(bus->sdiodev,
+				  M_DS1_CTRL_SDIO(bus->sdiodev->fmac_ulp),
+				  &err));
+			val32 = D11SHM_RD(bus->sdiodev, M_DS1_CTRL_SDIO(
+				  bus->sdiodev->fmac_ulp), &err);
+			D11SHM_WR(bus->sdiodev, M_DS1_CTRL_SDIO(
+				  bus->sdiodev->fmac_ulp), val32,
+				  (C_DS1_CTRL_SDIO_DS1_EXIT |
+				  C_DS1_CTRL_REQ_VALID),
+				  &err);
+			val32 = D11REG_RD(bus->sdiodev,
+					  D11_MACCONTROL_REG, &err);
+			val32 = val32 | D11_MACCONTROL_REG_WAKE;
+			D11REG_WR(bus->sdiodev,
+				  D11_MACCONTROL_REG, val32, &err);
+
+			/* Poll for PROC_DONE to be set by ucode */
+			value = D11SHM_RDW(bus->sdiodev,
+					   M_DS1_CTRL_SDIO(
+					   bus->sdiodev->fmac_ulp), &err);
+			/* Wait here (polling) for C_DS1_CTRL_PROC_DONE */
+			timeout = jiffies + ULP_HUDI_PROC_DONE_TIME;
+			while (!(value & C_DS1_CTRL_PROC_DONE)) {
+				value = D11SHM_RDW(bus->sdiodev,
+						   M_DS1_CTRL_SDIO
+						   (bus->sdiodev->fmac_ulp),
+						   &err);
+				if (time_after(jiffies, timeout))
+					break;
+				usleep_range(1000, 2000);
+			}
+			brcmf_dbg(ULP, "M_DS1_CTRL_SDIO: 0x%08x\n",
+				  (u32)D11SHM_RDW(bus->sdiodev,
+				  M_DS1_CTRL_SDIO(
+				  bus->sdiodev->fmac_ulp), &err));
+			value = D11SHM_RDW(bus->sdiodev,
+					   M_DS1_CTRL_SDIO(
+					   bus->sdiodev->fmac_ulp), &err);
+			if (!(value & C_DS1_CTRL_PROC_DONE)) {
+				brcmf_err("%s: timeout Failed to enter DS1 Exit state!\n",
+					  __func__);
+				return false;
+			}
+		}
+		ulp_wake_ind = D11SHM_RDW(bus->sdiodev, M_ULP_WAKE_IND(
+				  bus->sdiodev->fmac_ulp), &err);
+		wowl_wake_ind = D11SHM_RDW(bus->sdiodev, M_WAKEEVENT_IND(
+				  bus->sdiodev->fmac_ulp), &err);
+		brcmf_dbg(ULP, "wowl_wake_ind: 0x%08x, ulp_wake_ind: 0x%08x\n",
+			  wowl_wake_ind, ulp_wake_ind);
+		reg_addr = CORE_CC_REG(
+			  brcmf_chip_get_pmu(bus->ci)->base, min_res_mask);
+		brcmf_sdiod_regwl(bus->sdiodev, reg_addr,
+				  DEFAULT_43012_MIN_RES_MASK, &err);
+		if (err)
+			brcmf_err("min_res_mask failed\n");
+
+		return true;
+	}
+
+	return false;
+}
+
 static void brcmf_sdio_dpc(struct brcmf_sdio *bus)
 {
-	struct brcmf_sdio_dev *sdiod = bus->sdiodev;
 	u32 newstatus = 0;
-	u32 intstat_addr = bus->sdio_core->base + SD_REG(intstatus);
 	unsigned long intstatus;
 	uint txlimit = bus->txbound;	/* Tx frames to send before resched */
 	uint framecnt;			/* Temporary counter of tx/rx frames */
 	int err = 0;
 
-	brcmf_dbg(SDIO, "Enter\n");
+	brcmf_dbg(TRACE, "Enter\n");
 
-	sdio_claim_host(bus->sdiodev->func1);
+	sdio_claim_host(bus->sdiodev->func[1]);
 
 	/* If waiting for HTAVAIL, check status */
 	if (!bus->sr_enabled && bus->clkstate == CLK_PENDING) {
@@ -2546,23 +2775,23 @@
 
 #ifdef DEBUG
 		/* Check for inconsistent device control */
-		devctl = brcmf_sdiod_readb(bus->sdiodev, SBSDIO_DEVICE_CTL,
-					   &err);
+		devctl = brcmf_sdiod_regrb(bus->sdiodev,
+					   SBSDIO_DEVICE_CTL, &err);
 #endif				/* DEBUG */
 
 		/* Read CSR, if clock on switch to AVAIL, else ignore */
-		clkctl = brcmf_sdiod_readb(bus->sdiodev,
+		clkctl = brcmf_sdiod_regrb(bus->sdiodev,
 					   SBSDIO_FUNC1_CHIPCLKCSR, &err);
 
 		brcmf_dbg(SDIO, "DPC: PENDING, devctl 0x%02x clkctl 0x%02x\n",
 			  devctl, clkctl);
 
 		if (SBSDIO_HTAV(clkctl)) {
-			devctl = brcmf_sdiod_readb(bus->sdiodev,
+			devctl = brcmf_sdiod_regrb(bus->sdiodev,
 						   SBSDIO_DEVICE_CTL, &err);
 			devctl &= ~SBSDIO_DEVCTL_CA_INT_ONLY;
-			brcmf_sdiod_writeb(bus->sdiodev,
-					   SBSDIO_DEVICE_CTL, devctl, &err);
+			brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_DEVICE_CTL,
+					  devctl, &err);
 			bus->clkstate = CLK_AVAIL;
 		}
 	}
@@ -2585,10 +2814,11 @@
 	 */
 	if (intstatus & I_HMB_FC_CHANGE) {
 		intstatus &= ~I_HMB_FC_CHANGE;
-		brcmf_sdiod_writel(sdiod, intstat_addr, I_HMB_FC_CHANGE, &err);
-
-		newstatus = brcmf_sdiod_readl(sdiod, intstat_addr, &err);
+		err = w_sdreg32(bus, I_HMB_FC_CHANGE,
+				offsetof(struct sdpcmd_regs, intstatus));
 
+		err = r_sdreg32(bus, &newstatus,
+				offsetof(struct sdpcmd_regs, intstatus));
 		bus->sdcnt.f1regdata += 2;
 		atomic_set(&bus->fcstate,
 			   !!(newstatus & (I_HMB_FC_STATE | I_HMB_FC_CHANGE)));
@@ -2597,11 +2827,14 @@
 
 	/* Handle host mailbox indication */
 	if (intstatus & I_HMB_HOST_INT) {
+		u32 hmb_data = 0;
 		intstatus &= ~I_HMB_HOST_INT;
-		intstatus |= brcmf_sdio_hostmail(bus);
+		intstatus |= brcmf_sdio_hostmail(bus, &hmb_data);
+		if (brcmf_sdio_ulp_pre_redownload_check(bus, hmb_data))
+			brcmf_sdio_ulp_reinit_fw(bus);
 	}
 
-	sdio_release_host(bus->sdiodev->func1);
+	sdio_release_host(bus->sdiodev->func[1]);
 
 	/* Generally don't ask for these, can get CRC errors... */
 	if (intstatus & I_WR_OOSYNC) {
@@ -2621,7 +2854,7 @@
 
 	/* Would be active due to wake-wlan in gSPI */
 	if (intstatus & I_CHIPACTIVE) {
-		brcmf_dbg(SDIO, "Dongle reports CHIPACTIVE\n");
+		brcmf_dbg(INFO, "Dongle reports CHIPACTIVE\n");
 		intstatus &= ~I_CHIPACTIVE;
 	}
 
@@ -2643,16 +2876,20 @@
 	brcmf_sdio_clrintr(bus);
 
 	if (bus->ctrl_frame_stat && (bus->clkstate == CLK_AVAIL) &&
-	    data_ok(bus)) {
-		sdio_claim_host(bus->sdiodev->func1);
+	    txctl_ok(bus) && brcmf_sdio_f2_ready(bus)) {
+		sdio_claim_host(bus->sdiodev->func[1]);
 		if (bus->ctrl_frame_stat) {
-			err = brcmf_sdio_tx_ctrlframe(bus,  bus->ctrl_frame_buf,
+			err = brcmf_sdio_tx_ctrlframe(bus,
+						      bus->ctrl_frame_buf,
 						      bus->ctrl_frame_len);
 			bus->ctrl_frame_err = err;
-			wmb();
+			wmb(); /* drain writebuffer */
 			bus->ctrl_frame_stat = false;
+			if (err)
+				brcmf_err("sdio ctrlframe tx failed err=%d\n",
+					  err);
 		}
-		sdio_release_host(bus->sdiodev->func1);
+		sdio_release_host(bus->sdiodev->func[1]);
 		brcmf_sdio_wait_event_wakeup(bus);
 	}
 	/* Send queued frames (limit 1 if rx may still be pending) */
@@ -2668,14 +2905,14 @@
 		brcmf_err("failed backplane access over SDIO, halting operation\n");
 		atomic_set(&bus->intstatus, 0);
 		if (bus->ctrl_frame_stat) {
-			sdio_claim_host(bus->sdiodev->func1);
+			sdio_claim_host(bus->sdiodev->func[1]);
 			if (bus->ctrl_frame_stat) {
 				bus->ctrl_frame_err = -ENODEV;
 				wmb();
 				bus->ctrl_frame_stat = false;
 				brcmf_sdio_wait_event_wakeup(bus);
 			}
-			sdio_release_host(bus->sdiodev->func1);
+			sdio_release_host(bus->sdiodev->func[1]);
 		}
 	} else if (atomic_read(&bus->intstatus) ||
 		   atomic_read(&bus->ipend) > 0 ||
@@ -2753,7 +2990,13 @@
 	skb_push(pkt, bus->tx_hdrlen);
 	/* precondition: IS_ALIGNED((unsigned long)(pkt->data), 2) */
 
-	prec = prio2prec((pkt->priority & PRIOMASK));
+	/* In WLAN, priority is always set by the AP using WMM parameters
+	 * and this need not always follow the standard 802.1d priority.
+	 * Based on AP WMM config, map from 802.1d priority to corresponding
+	 * precedence level.
+	 */
+	prec = brcmf_map_prio_to_prec(bus_if->drvr->config,
+				      (pkt->priority & PRIOMASK));
 
 	/* Check for existing queue, current flow-control,
 			 pending event, or pending clock */
@@ -2890,13 +3133,13 @@
 					 CTL_DONE_TIMEOUT);
 	ret = 0;
 	if (bus->ctrl_frame_stat) {
-		sdio_claim_host(bus->sdiodev->func1);
+		sdio_claim_host(bus->sdiodev->func[1]);
 		if (bus->ctrl_frame_stat) {
 			brcmf_dbg(SDIO, "ctrl_frame timeout\n");
 			bus->ctrl_frame_stat = false;
 			ret = -ETIMEDOUT;
 		}
-		sdio_release_host(bus->sdiodev->func1);
+		sdio_release_host(bus->sdiodev->func[1]);
 	}
 	if (!ret) {
 		brcmf_dbg(SDIO, "ctrl_frame complete, err=%d\n",
@@ -3020,7 +3263,7 @@
 		return 0;
 	}
 
-	sdio_claim_host(bus->sdiodev->func1);
+	sdio_claim_host(bus->sdiodev->func[1]);
 	if (sh->assert_file_addr != 0) {
 		error = brcmf_sdiod_ramrw(bus->sdiodev, false,
 					  sh->assert_file_addr, (u8 *)file, 80);
@@ -3033,7 +3276,7 @@
 		if (error < 0)
 			return error;
 	}
-	sdio_release_host(bus->sdiodev->func1);
+	sdio_release_host(bus->sdiodev->func[1]);
 
 	seq_printf(seq, "dongle assert: %s:%d: assert(%s)\n",
 		   file, sh->assert_line, expr);
@@ -3307,7 +3550,7 @@
 	int bcmerror;
 	u32 rstvec;
 
-	sdio_claim_host(bus->sdiodev->func1);
+	sdio_claim_host(bus->sdiodev->func[1]);
 	brcmf_sdio_clkctl(bus, CLK_AVAIL, false);
 
 	rstvec = get_unaligned_le32(fw->data);
@@ -3336,7 +3579,7 @@
 
 err:
 	brcmf_sdio_clkctl(bus, CLK_SDONLY, false);
-	sdio_release_host(bus->sdiodev->func1);
+	sdio_release_host(bus->sdiodev->func[1]);
 	return bcmerror;
 }
 
@@ -3344,34 +3587,52 @@
 {
 	int err = 0;
 	u8 val;
+	u8 wakeupctrl;
+	u8 cardcap;
+	u8 chipclkcsr;
 
 	brcmf_dbg(TRACE, "Enter\n");
 
-	val = brcmf_sdiod_readb(bus->sdiodev, SBSDIO_FUNC1_WAKEUPCTRL, &err);
+	if (bus->ci->chip == CY_CC_43012_CHIP_ID) {
+		wakeupctrl = SBSDIO_FUNC1_WCTRL_ALPWAIT_SHIFT;
+		chipclkcsr = SBSDIO_HT_AVAIL_REQ;
+	} else {
+		wakeupctrl = SBSDIO_FUNC1_WCTRL_HTWAIT_SHIFT;
+		chipclkcsr = SBSDIO_FORCE_HT;
+	}
+
+	if (bus->ci->chip == CY_CC_43012_CHIP_ID ||
+	    bus->ci->chip == CY_CC_4373_CHIP_ID ||
+	    bus->ci->chip == BRCM_CC_4339_CHIP_ID ||
+	    bus->ci->chip == BRCM_CC_4354_CHIP_ID ||
+	    bus->ci->chip == BRCM_CC_4356_CHIP_ID ||
+	    bus->ci->chip == BRCM_CC_4345_CHIP_ID) {
+		cardcap = SDIO_CCCR_BRCM_CARDCAP_CMD_NODEC;
+	} else {
+		cardcap = (SDIO_CCCR_BRCM_CARDCAP_CMD14_SUPPORT |
+			   SDIO_CCCR_BRCM_CARDCAP_CMD14_EXT);
+	}
+
+	val = brcmf_sdiod_regrb(bus->sdiodev, SBSDIO_FUNC1_WAKEUPCTRL, &err);
 	if (err) {
 		brcmf_err("error reading SBSDIO_FUNC1_WAKEUPCTRL\n");
 		return;
 	}
-
-	val |= 1 << SBSDIO_FUNC1_WCTRL_HTWAIT_SHIFT;
-	brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_FUNC1_WAKEUPCTRL, val, &err);
+	val |= 1 << wakeupctrl;
+	brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_WAKEUPCTRL, val, &err);
 	if (err) {
 		brcmf_err("error writing SBSDIO_FUNC1_WAKEUPCTRL\n");
 		return;
 	}
-
-	/* Add CMD14 Support */
-	brcmf_sdiod_func0_wb(bus->sdiodev, SDIO_CCCR_BRCM_CARDCAP,
-			     (SDIO_CCCR_BRCM_CARDCAP_CMD14_SUPPORT |
-			      SDIO_CCCR_BRCM_CARDCAP_CMD14_EXT),
-			     &err);
+	brcmf_sdiod_regwb(bus->sdiodev, SDIO_CCCR_BRCM_CARDCAP,
+			  cardcap,
+			  &err);
 	if (err) {
 		brcmf_err("error writing SDIO_CCCR_BRCM_CARDCAP\n");
 		return;
 	}
-
-	brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
-			   SBSDIO_FORCE_HT, &err);
+	brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
+		chipclkcsr, &err);
 	if (err) {
 		brcmf_err("error writing SBSDIO_FUNC1_CHIPCLKCSR\n");
 		return;
@@ -3385,17 +3646,16 @@
 /* enable KSO bit */
 static int brcmf_sdio_kso_init(struct brcmf_sdio *bus)
 {
-	struct brcmf_core *core = bus->sdio_core;
 	u8 val;
 	int err = 0;
 
 	brcmf_dbg(TRACE, "Enter\n");
 
 	/* KSO bit added in SDIO core rev 12 */
-	if (core->rev < 12)
+	if (brcmf_chip_get_core(bus->ci, BCMA_CORE_SDIO_DEV)->rev < 12)
 		return 0;
 
-	val = brcmf_sdiod_readb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR, &err);
+	val = brcmf_sdiod_regrb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR, &err);
 	if (err) {
 		brcmf_err("error reading SBSDIO_FUNC1_SLEEPCSR\n");
 		return err;
@@ -3404,8 +3664,8 @@
 	if (!(val & SBSDIO_FUNC1_SLEEPCSR_KSO_MASK)) {
 		val |= (SBSDIO_FUNC1_SLEEPCSR_KSO_EN <<
 			SBSDIO_FUNC1_SLEEPCSR_KSO_SHIFT);
-		brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR,
-				   val, &err);
+		brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR,
+				  val, &err);
 		if (err) {
 			brcmf_err("error writing SBSDIO_FUNC1_SLEEPCSR\n");
 			return err;
@@ -3421,30 +3681,15 @@
 	struct brcmf_bus *bus_if = dev_get_drvdata(dev);
 	struct brcmf_sdio_dev *sdiodev = bus_if->bus_priv.sdio;
 	struct brcmf_sdio *bus = sdiodev->bus;
-	struct brcmf_core *core = bus->sdio_core;
 	uint pad_size;
 	u32 value;
 	int err;
 
-	/* maxctl provided by common layer */
-	if (WARN_ON(!bus_if->maxctl))
-		return -EINVAL;
-
-	/* Allocate control receive buffer */
-	bus_if->maxctl += bus->roundup;
-	value = roundup((bus_if->maxctl + SDPCM_HDRLEN), ALIGNMENT);
-	value += bus->head_align;
-	bus->rxbuf = kmalloc(value, GFP_ATOMIC);
-	if (bus->rxbuf)
-		bus->rxblen = value;
-
-	brcmf_sdio_debugfs_create(bus);
-
 	/* the commands below use the terms tx and rx from
 	 * a device perspective, ie. bus:txglom affects the
 	 * bus transfers from device to host.
 	 */
-	if (core->rev < 12) {
+	if (brcmf_chip_get_core(bus->ci, BCMA_CORE_SDIO_DEV)->rev < 12) {
 		/* for sdio core rev < 12, disable txgloming */
 		value = 0;
 		err = brcmf_iovar_data_set(dev, "bus:txglom", &value,
@@ -3461,11 +3706,15 @@
 	if (err < 0)
 		goto done;
 
+	/* initialize SHM address from firmware for DS1 */
+	if (!bus->sdiodev->ulp)
+		brcmf_sdio_ulp_preinit(dev);
+
 	bus->tx_hdrlen = SDPCM_HWHDR_LEN + SDPCM_SWHDR_LEN;
 	if (sdiodev->sg_support) {
 		bus->txglom = false;
 		value = 1;
-		pad_size = bus->sdiodev->func2->cur_blksize << 1;
+		pad_size = bus->sdiodev->func[2]->cur_blksize << 1;
 		err = brcmf_iovar_data_set(bus->sdiodev->dev, "bus:rxglom",
 					   &value, sizeof(u32));
 		if (err < 0) {
@@ -3507,7 +3756,7 @@
 
 	address = bus->ci->rambase;
 	offset = err = 0;
-	sdio_claim_host(sdiodev->func1);
+	sdio_claim_host(sdiodev->func[1]);
 	while (offset < mem_size) {
 		len = ((offset + MEMBLOCK) < mem_size) ? MEMBLOCK :
 		      mem_size - offset;
@@ -3523,7 +3772,7 @@
 	}
 
 done:
-	sdio_release_host(sdiodev->func1);
+	sdio_release_host(sdiodev->func[1]);
 	return err;
 }
 
@@ -3580,10 +3829,11 @@
 			if (!bus->dpc_triggered) {
 				u8 devpend;
 
-				sdio_claim_host(bus->sdiodev->func1);
-				devpend = brcmf_sdiod_func0_rb(bus->sdiodev,
-						  SDIO_CCCR_INTx, NULL);
-				sdio_release_host(bus->sdiodev->func1);
+				sdio_claim_host(bus->sdiodev->func[1]);
+				devpend = brcmf_sdiod_regrb(bus->sdiodev,
+							    SDIO_CCCR_INTx,
+							    NULL);
+				sdio_release_host(bus->sdiodev->func[1]);
 				intstatus = devpend & (INTR_STATUS_FUNC1 |
 						       INTR_STATUS_FUNC2);
 			}
@@ -3609,13 +3859,13 @@
 		bus->console.count += jiffies_to_msecs(BRCMF_WD_POLL);
 		if (bus->console.count >= bus->console_interval) {
 			bus->console.count -= bus->console_interval;
-			sdio_claim_host(bus->sdiodev->func1);
+			sdio_claim_host(bus->sdiodev->func[1]);
 			/* Make sure backplane clock is on */
 			brcmf_sdio_bus_sleep(bus, false, false);
 			if (brcmf_sdio_readconsole(bus) < 0)
 				/* stop on error */
 				bus->console_interval = 0;
-			sdio_release_host(bus->sdiodev->func1);
+			sdio_release_host(bus->sdiodev->func[1]);
 		}
 	}
 #endif				/* DEBUG */
@@ -3628,11 +3878,15 @@
 			bus->idlecount++;
 			if (bus->idlecount > bus->idletime) {
 				brcmf_dbg(SDIO, "idle\n");
-				sdio_claim_host(bus->sdiodev->func1);
-				brcmf_sdio_wd_timer(bus, false);
+				sdio_claim_host(bus->sdiodev->func[1]);
+#ifdef DEBUG
+				if (!BRCMF_FWCON_ON() ||
+				    bus->console_interval == 0)
+#endif
+					brcmf_sdio_wd_timer(bus, false);
 				bus->idlecount = 0;
 				brcmf_sdio_bus_sleep(bus, true, false);
-				sdio_release_host(bus->sdiodev->func1);
+				sdio_release_host(bus->sdiodev->func[1]);
 			}
 		} else {
 			bus->idlecount = 0;
@@ -3649,7 +3903,7 @@
 
 	bus->dpc_running = true;
 	wmb();
-	while (READ_ONCE(bus->dpc_triggered)) {
+	while (ACCESS_ONCE(bus->dpc_triggered)) {
 		bus->dpc_triggered = false;
 		brcmf_sdio_dpc(bus);
 		bus->idlecount = 0;
@@ -3720,12 +3974,12 @@
 			}
 		}
 		addr = CORE_CC_REG(pmu->base, chipcontrol_addr);
-		brcmf_sdiod_writel(sdiodev, addr, 1, NULL);
-		cc_data_temp = brcmf_sdiod_readl(sdiodev, addr, NULL);
+		brcmf_sdiod_regwl(sdiodev, addr, 1, NULL);
+		cc_data_temp = brcmf_sdiod_regrl(sdiodev, addr, NULL);
 		cc_data_temp &= ~str_mask;
 		drivestrength_sel <<= str_shift;
 		cc_data_temp |= drivestrength_sel;
-		brcmf_sdiod_writel(sdiodev, addr, cc_data_temp, NULL);
+		brcmf_sdiod_regwl(sdiodev, addr, cc_data_temp, NULL);
 
 		brcmf_dbg(INFO, "SDIO: %d mA (req=%d mA) drive strength selected, set to 0x%08x\n",
 			  str_tab[i].strength, drivestrength, cc_data_temp);
@@ -3740,7 +3994,7 @@
 
 	/* Try forcing SDIO core to do ALPAvail request only */
 	clkset = SBSDIO_FORCE_HW_CLKREQ_OFF | SBSDIO_ALP_AVAIL_REQ;
-	brcmf_sdiod_writeb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, clkset, &err);
+	brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, clkset, &err);
 	if (err) {
 		brcmf_err("error writing for HT off\n");
 		return err;
@@ -3748,7 +4002,8 @@
 
 	/* If register supported, wait for ALPAvail and then force ALP */
 	/* This may take up to 15 milliseconds */
-	clkval = brcmf_sdiod_readb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, NULL);
+	clkval = brcmf_sdiod_regrb(sdiodev,
+				   SBSDIO_FUNC1_CHIPCLKCSR, NULL);
 
 	if ((clkval & ~SBSDIO_AVBITS) != clkset) {
 		brcmf_err("ChipClkCSR access: wrote 0x%02x read 0x%02x\n",
@@ -3756,11 +4011,10 @@
 		return -EACCES;
 	}
 
-	SPINWAIT(((clkval = brcmf_sdiod_readb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
-					      NULL)),
-		 !SBSDIO_ALPAV(clkval)),
-		 PMU_MAX_TRANSITION_DLY);
-
+	SPINWAIT(((clkval = brcmf_sdiod_regrb(sdiodev,
+					      SBSDIO_FUNC1_CHIPCLKCSR, NULL)),
+			!SBSDIO_ALPAV(clkval)),
+			PMU_MAX_TRANSITION_DLY);
 	if (!SBSDIO_ALPAV(clkval)) {
 		brcmf_err("timeout on ALPAV wait, clkval 0x%02x\n",
 			  clkval);
@@ -3768,11 +4022,11 @@
 	}
 
 	clkset = SBSDIO_FORCE_HW_CLKREQ_OFF | SBSDIO_FORCE_ALP;
-	brcmf_sdiod_writeb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, clkset, &err);
+	brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, clkset, &err);
 	udelay(65);
 
 	/* Also, disable the extra SDIO pull-ups */
-	brcmf_sdiod_writeb(sdiodev, SBSDIO_FUNC1_SDIOPULLUP, 0, NULL);
+	brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_SDIOPULLUP, 0, NULL);
 
 	return 0;
 }
@@ -3781,12 +4035,13 @@
 					u32 rstvec)
 {
 	struct brcmf_sdio_dev *sdiodev = ctx;
-	struct brcmf_core *core = sdiodev->bus->sdio_core;
+	struct brcmf_core *core;
 	u32 reg_addr;
 
 	/* clear all interrupts */
-	reg_addr = core->base + SD_REG(intstatus);
-	brcmf_sdiod_writel(sdiodev, reg_addr, 0xFFFFFFFF, NULL);
+	core = brcmf_chip_get_core(chip, BCMA_CORE_SDIO_DEV);
+	reg_addr = core->base + offsetof(struct sdpcmd_regs, intstatus);
+	brcmf_sdiod_regwl(sdiodev, reg_addr, 0xFFFFFFFF, NULL);
 
 	if (rstvec)
 		/* Write reset vector to address 0 */
@@ -3799,25 +4054,16 @@
 	struct brcmf_sdio_dev *sdiodev = ctx;
 	u32 val, rev;
 
-	val = brcmf_sdiod_readl(sdiodev, addr, NULL);
-
-	/*
-	 * this is a bit of special handling if reading the chipcommon chipid
-	 * register. The 4339 is a next-gen of the 4335. It uses the same
-	 * SDIO device id as 4335 and the chipid register returns 4335 as well.
-	 * It can be identified as 4339 by looking at the chip revision. It
-	 * is corrected here so the chip.c module has the right info.
-	 */
-	if (addr == CORE_CC_REG(SI_ENUM_BASE, chipid) &&
-	    (sdiodev->func1->device == SDIO_DEVICE_ID_BROADCOM_4339 ||
-	     sdiodev->func1->device == SDIO_DEVICE_ID_BROADCOM_4335_4339)) {
+	val = brcmf_sdiod_regrl(sdiodev, addr, NULL);
+	if ((sdiodev->func[0]->device == SDIO_DEVICE_ID_BROADCOM_4335_4339 ||
+	     sdiodev->func[0]->device == SDIO_DEVICE_ID_BROADCOM_4339) &&
+	    addr == CORE_CC_REG(SI_ENUM_BASE, chipid)) {
 		rev = (val & CID_REV_MASK) >> CID_REV_SHIFT;
 		if (rev >= 2) {
 			val &= ~CID_ID_MASK;
 			val |= BRCM_CC_4339_CHIP_ID;
 		}
 	}
-
 	return val;
 }
 
@@ -3825,7 +4071,7 @@
 {
 	struct brcmf_sdio_dev *sdiodev = ctx;
 
-	brcmf_sdiod_writel(sdiodev, addr, val, NULL);
+	brcmf_sdiod_regwl(sdiodev, addr, val, NULL);
 }
 
 static const struct brcmf_buscore_ops brcmf_sdio_buscore_ops = {
@@ -3846,21 +4092,21 @@
 	u32 drivestrength;
 
 	sdiodev = bus->sdiodev;
-	sdio_claim_host(sdiodev->func1);
+	sdio_claim_host(sdiodev->func[1]);
 
 	pr_debug("F1 signature read @0x18000000=0x%4x\n",
-		 brcmf_sdiod_readl(sdiodev, SI_ENUM_BASE, NULL));
+		 brcmf_sdiod_regrl(sdiodev, SI_ENUM_BASE, NULL));
 
 	/*
 	 * Force PLL off until brcmf_chip_attach()
 	 * programs PLL control regs
 	 */
 
-	brcmf_sdiod_writeb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, BRCMF_INIT_CLKCTL1,
-			   &err);
+	brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
+			  BRCMF_INIT_CLKCTL1, &err);
 	if (!err)
-		clkctl = brcmf_sdiod_readb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
-					   &err);
+		clkctl = brcmf_sdiod_regrb(sdiodev,
+					   SBSDIO_FUNC1_CHIPCLKCSR, &err);
 
 	if (err || ((clkctl & ~SBSDIO_AVBITS) != BRCMF_INIT_CLKCTL1)) {
 		brcmf_err("ChipClkCSR access: err %d wrote 0x%02x read 0x%02x\n",
@@ -3874,17 +4120,6 @@
 		bus->ci = NULL;
 		goto fail;
 	}
-
-	/* Pick up the SDIO core info struct from chip.c */
-	bus->sdio_core   = brcmf_chip_get_core(bus->ci, BCMA_CORE_SDIO_DEV);
-	if (!bus->sdio_core)
-		goto fail;
-
-	/* Pick up the CHIPCOMMON core info struct, for bulk IO in bcmsdh.c */
-	sdiodev->cc_core = brcmf_chip_get_core(bus->ci, BCMA_CORE_CHIPCOMMON);
-	if (!sdiodev->cc_core)
-		goto fail;
-
 	sdiodev->settings = brcmf_get_module_param(sdiodev->dev,
 						   BRCMF_BUSTYPE_SDIO,
 						   bus->ci->chip,
@@ -3913,8 +4148,8 @@
 	/* wowl can be supported when KEEP_POWER is true and (WAKE_SDIO_IRQ
 	 * is true or when platform data OOB irq is true).
 	 */
-	if ((sdio_get_host_pm_caps(sdiodev->func1) & MMC_PM_KEEP_POWER) &&
-	    ((sdio_get_host_pm_caps(sdiodev->func1) & MMC_PM_WAKE_SDIO_IRQ) ||
+	if ((sdio_get_host_pm_caps(sdiodev->func[1]) & MMC_PM_KEEP_POWER) &&
+	    ((sdio_get_host_pm_caps(sdiodev->func[1]) & MMC_PM_WAKE_SDIO_IRQ) ||
 	     (sdiodev->settings->bus.sdio.oob_irq_supported)))
 		sdiodev->bus_if->wowl_supported = true;
 #endif
@@ -3931,29 +4166,29 @@
 	brcmf_sdio_drivestrengthinit(sdiodev, bus->ci, drivestrength);
 
 	/* Set card control so an SDIO card reset does a WLAN backplane reset */
-	reg_val = brcmf_sdiod_func0_rb(sdiodev, SDIO_CCCR_BRCM_CARDCTRL, &err);
+	reg_val = brcmf_sdiod_regrb(sdiodev, SDIO_CCCR_BRCM_CARDCTRL, &err);
 	if (err)
 		goto fail;
 
 	reg_val |= SDIO_CCCR_BRCM_CARDCTRL_WLANRESET;
 
-	brcmf_sdiod_func0_wb(sdiodev, SDIO_CCCR_BRCM_CARDCTRL, reg_val, &err);
+	brcmf_sdiod_regwb(sdiodev, SDIO_CCCR_BRCM_CARDCTRL, reg_val, &err);
 	if (err)
 		goto fail;
 
 	/* set PMUControl so a backplane reset does PMU state reload */
 	reg_addr = CORE_CC_REG(brcmf_chip_get_pmu(bus->ci)->base, pmucontrol);
-	reg_val = brcmf_sdiod_readl(sdiodev, reg_addr, &err);
+	reg_val = brcmf_sdiod_regrl(sdiodev, reg_addr, &err);
 	if (err)
 		goto fail;
 
 	reg_val |= (BCMA_CC_PMU_CTL_RES_RELOAD << BCMA_CC_PMU_CTL_RES_SHIFT);
 
-	brcmf_sdiod_writel(sdiodev, reg_addr, reg_val, &err);
+	brcmf_sdiod_regwl(sdiodev, reg_addr, reg_val, &err);
 	if (err)
 		goto fail;
 
-	sdio_release_host(sdiodev->func1);
+	sdio_release_host(sdiodev->func[1]);
 
 	brcmu_pktq_init(&bus->txq, (PRIOMASK + 1), TXQLEN);
 
@@ -3974,7 +4209,7 @@
 	return true;
 
 fail:
-	sdio_release_host(sdiodev->func1);
+	sdio_release_host(sdiodev->func[1]);
 	return false;
 }
 
@@ -4019,24 +4254,52 @@
 	}
 }
 
-static
-int brcmf_sdio_get_fwname(struct device *dev, const char *ext, u8 *fw_name)
+static int brcmf_sdio_get_fwname(struct device *dev, u32 chip, u32 chiprev,
+				 u8 *fw_name)
 {
 	struct brcmf_bus *bus_if = dev_get_drvdata(dev);
-	struct brcmf_fw_request *fwreq;
-	struct brcmf_fw_name fwnames[] = {
-		{ ext, fw_name },
-	};
-
-	fwreq = brcmf_fw_alloc_request(bus_if->chip, bus_if->chiprev,
-				       brcmf_sdio_fwnames,
-				       ARRAY_SIZE(brcmf_sdio_fwnames),
-				       fwnames, ARRAY_SIZE(fwnames));
-	if (!fwreq)
-		return -ENOMEM;
+	struct brcmf_sdio_dev *sdiodev = bus_if->bus_priv.sdio;
+	int ret = 0;
 
-	kfree(fwreq);
-	return 0;
+	if (sdiodev->fw_name[0] != '\0')
+		strlcpy(fw_name, sdiodev->fw_name, BRCMF_FW_NAME_LEN);
+	else
+		ret = brcmf_fw_map_chip_to_name(chip, chiprev,
+						brcmf_sdio_fwnames,
+						ARRAY_SIZE(brcmf_sdio_fwnames),
+						fw_name, NULL);
+
+	return ret;
+}
+
+static int brcmf_sdio_bus_reset(struct device *dev)
+{
+	int ret = 0;
+	struct brcmf_bus *bus_if = dev_get_drvdata(dev);
+	struct brcmf_sdio_dev *sdiodev = bus_if->bus_priv.sdio;
+
+	brcmf_dbg(SDIO, "Enter\n");
+
+	/* start by unregistering irqs */
+	brcmf_sdiod_intr_unregister(sdiodev);
+
+	brcmf_sdiod_remove(sdiodev);
+
+	/* reset the adapter */
+	sdio_claim_host(sdiodev->func[1]);
+	mmc_hw_reset(sdiodev->func[1]->card->host);
+	sdio_release_host(sdiodev->func[1]);
+
+	brcmf_bus_change_state(sdiodev->bus_if, BRCMF_BUS_DOWN);
+
+	ret = brcmf_sdiod_probe(sdiodev);
+	if (ret) {
+		brcmf_err("Failed to probe after sdio device reset: ret %d\n",
+			  ret);
+		brcmf_sdiod_remove(sdiodev);
+	}
+
+	return ret;
 }
 
 static const struct brcmf_bus_ops brcmf_sdio_bus_ops = {
@@ -4050,33 +4313,29 @@
 	.get_ramsize = brcmf_sdio_bus_get_ramsize,
 	.get_memdump = brcmf_sdio_bus_get_memdump,
 	.get_fwname = brcmf_sdio_get_fwname,
+	.reset = brcmf_sdio_bus_reset,
 };
 
-#define BRCMF_SDIO_FW_CODE	0
-#define BRCMF_SDIO_FW_NVRAM	1
-
 static void brcmf_sdio_firmware_callback(struct device *dev, int err,
-					 struct brcmf_fw_request *fwreq)
+					 const struct firmware *code,
+					 void *nvram, u32 nvram_len)
 {
-	struct brcmf_bus *bus_if = dev_get_drvdata(dev);
-	struct brcmf_sdio_dev *sdiod = bus_if->bus_priv.sdio;
-	struct brcmf_sdio *bus = sdiod->bus;
-	struct brcmf_core *core = bus->sdio_core;
-	const struct firmware *code;
-	void *nvram;
-	u32 nvram_len;
+	struct brcmf_bus *bus_if;
+	struct brcmf_sdio_dev *sdiodev;
+	struct brcmf_sdio *bus;
 	u8 saveclk;
 	u8 devctl;
 
-	brcmf_dbg(TRACE, "Enter: dev=%s, err=%d\n", dev_name(dev), err);
-
+	brcmf_dbg(ULP, "Enter: dev=%s, err=%d\n", dev_name(dev), err);
+	bus_if = dev_get_drvdata(dev);
+	sdiodev = bus_if->bus_priv.sdio;
 	if (err)
 		goto fail;
 
-	code = fwreq->items[BRCMF_SDIO_FW_CODE].binary;
-	nvram = fwreq->items[BRCMF_SDIO_FW_NVRAM].nv_data.data;
-	nvram_len = fwreq->items[BRCMF_SDIO_FW_NVRAM].nv_data.len;
-	kfree(fwreq);
+	if (!bus_if->drvr)
+		return;
+
+	bus = sdiodev->bus;
 
 	/* try to download image and nvram to the dongle */
 	bus->alp_only = true;
@@ -4089,7 +4348,7 @@
 	bus->sdcnt.tickcnt = 0;
 	brcmf_sdio_wd_timer(bus, true);
 
-	sdio_claim_host(sdiod->func1);
+	sdio_claim_host(sdiodev->func[1]);
 
 	/* Make sure backplane clock is on, needed to generate F2 interrupt */
 	brcmf_sdio_clkctl(bus, CLK_AVAIL, false);
@@ -4097,10 +4356,16 @@
 		goto release;
 
 	/* Force clocks on backplane to be sure F2 interrupt propagates */
-	saveclk = brcmf_sdiod_readb(sdiod, SBSDIO_FUNC1_CHIPCLKCSR, &err);
+	saveclk = brcmf_sdiod_regrb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, &err);
 	if (!err) {
-		brcmf_sdiod_writeb(sdiod, SBSDIO_FUNC1_CHIPCLKCSR,
-				   (saveclk | SBSDIO_FORCE_HT), &err);
+		if (bus->ci->chip == CY_CC_43012_CHIP_ID) {
+			brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
+					  (saveclk | SBSDIO_HT_AVAIL_REQ),
+					  &err);
+		} else {
+			brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
+					  (saveclk | SBSDIO_FORCE_HT), &err);
+		}
 	}
 	if (err) {
 		brcmf_err("Failed to force clock for F2: err %d\n", err);
@@ -4108,10 +4373,10 @@
 	}
 
 	/* Enable function 2 (frame transfers) */
-	brcmf_sdiod_writel(sdiod, core->base + SD_REG(tosbmailboxdata),
-			   SDPCM_PROT_VERSION << SMB_DATA_VERSION_SHIFT, NULL);
+	w_sdreg32(bus, SDPCM_PROT_VERSION << SMB_DATA_VERSION_SHIFT,
+		  offsetof(struct sdpcmd_regs, tosbmailboxdata));
+	err = sdio_enable_func(sdiodev->func[SDIO_FUNC_2]);
 
-	err = sdio_enable_func(sdiod->func2);
 
 	brcmf_dbg(INFO, "enable F2: err=%d\n", err);
 
@@ -4119,32 +4384,83 @@
 	if (!err) {
 		/* Set up the interrupt mask and enable interrupts */
 		bus->hostintmask = HOSTINTMASK;
-		brcmf_sdiod_writel(sdiod, core->base + SD_REG(hostintmask),
-				   bus->hostintmask, NULL);
-
-		switch (sdiod->func1->device) {
+		w_sdreg32(bus, bus->hostintmask,
+			  offsetof(struct sdpcmd_regs, hostintmask));
+		switch (sdiodev->func[0]->device) {
 		case SDIO_DEVICE_ID_CYPRESS_4373:
 			brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes\n",
 				  CY_4373_F2_WATERMARK);
-			brcmf_sdiod_writeb(sdiod, SBSDIO_WATERMARK,
-					   CY_4373_F2_WATERMARK, &err);
-			devctl = brcmf_sdiod_readb(sdiod, SBSDIO_DEVICE_CTL,
+			brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK, CY_4373_F2_WATERMARK, &err);
+			devctl = brcmf_sdiod_regrb(sdiodev, SBSDIO_DEVICE_CTL, &err);
+			devctl |= SBSDIO_DEVCTL_F2WM_ENAB;
+			brcmf_sdiod_regwb(sdiodev, SBSDIO_DEVICE_CTL, devctl, &err);
+			brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_MESBUSYCTRL, CY_4373_F1_MESBUSYCTRL, &err);
+			break;
+		case SDIO_DEVICE_ID_CYPRESS_43012:
+			brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes for 43012\n",
+				  CY_43012_F2_WATERMARK);
+			brcmf_sdiod_regwb(sdiodev,
+					  SBSDIO_WATERMARK,
+					  CY_43012_F2_WATERMARK, &err);
+			devctl = brcmf_sdiod_regrb(sdiodev,
+						   SBSDIO_DEVICE_CTL, &err);
+			devctl |= SBSDIO_DEVCTL_F2WM_ENAB;
+			brcmf_sdiod_regwb(sdiodev,
+					  SBSDIO_DEVICE_CTL, devctl, &err);
+			/* Register for ULP events */
+			brcmf_fweh_register(bus_if->drvr, BRCMF_E_ULP,
+					    brcmf_ulp_event_notify);
+			brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_MESBUSYCTRL,
+					  CY_43012_MESBUSYCTRL, &err);
+			break;
+		case SDIO_DEVICE_ID_BROADCOM_43455:
+			brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes for 43455\n",
+				  CY_43455_F2_WATERMARK);
+			brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK,
+					  CY_43455_F2_WATERMARK, &err);
+			devctl = brcmf_sdiod_regrb(sdiodev, SBSDIO_DEVICE_CTL,
 						   &err);
 			devctl |= SBSDIO_DEVCTL_F2WM_ENAB;
-			brcmf_sdiod_writeb(sdiod, SBSDIO_DEVICE_CTL, devctl,
-					   &err);
-			brcmf_sdiod_writeb(sdiod, SBSDIO_FUNC1_MESBUSYCTRL,
-					   CY_4373_F2_WATERMARK |
-					   SBSDIO_MESBUSYCTRL_ENAB, &err);
+			brcmf_sdiod_regwb(sdiodev, SBSDIO_DEVICE_CTL, devctl,
+					  &err);
+			brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_MESBUSYCTRL,
+					  CY_43455_MESBUSYCTRL, &err);
+			break;
+		case SDIO_DEVICE_ID_BROADCOM_4339:
+			brcmf_sdiod_regwb(sdiodev,
+					  SBSDIO_WATERMARK,
+					  CY_4339_F2_WATERMARK, &err);
+			devctl = brcmf_sdiod_regrb(sdiodev,
+						   SBSDIO_DEVICE_CTL, &err);
+			devctl |= SBSDIO_DEVCTL_F2WM_ENAB;
+			brcmf_sdiod_regwb(sdiodev,
+					  SBSDIO_DEVICE_CTL, devctl, &err);
+			brcmf_sdiod_regwb(sdiodev,
+					  SBSDIO_FUNC1_MESBUSYCTRL, CY_4339_MESBUSYCTRL, &err);
+			break;
+		case SDIO_DEVICE_ID_BROADCOM_4359:
+		case SDIO_DEVICE_ID_CYPRESS_89359:
+		case SDIO_DEVICE_ID_BROADCOM_4354:
+		case SDIO_DEVICE_ID_BROADCOM_4356:
+			brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes\n",
+				  CY_435X_F2_WATERMARK);
+			brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK,
+					  CY_435X_F2_WATERMARK, &err);
+			devctl = brcmf_sdiod_regrb(sdiodev, SBSDIO_DEVICE_CTL,
+						   &err);
+			devctl |= SBSDIO_DEVCTL_F2WM_ENAB;
+			brcmf_sdiod_regwb(sdiodev, SBSDIO_DEVICE_CTL, devctl,
+					  &err);
+			brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_MESBUSYCTRL,
+					  CY_435X_F1_MESBUSYCTRL, &err);
 			break;
 		default:
-			brcmf_sdiod_writeb(sdiod, SBSDIO_WATERMARK,
-					   DEFAULT_F2_WATERMARK, &err);
+			brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK, DEFAULT_F2_WATERMARK, &err);
 			break;
 		}
 	} else {
 		/* Disable F2 again */
-		sdio_disable_func(sdiod->func2);
+		sdio_disable_func(sdiodev->func[SDIO_FUNC_2]);
 		goto release;
 	}
 
@@ -4152,15 +4468,15 @@
 		brcmf_sdio_sr_init(bus);
 	} else {
 		/* Restore previous clock setting */
-		brcmf_sdiod_writeb(sdiod, SBSDIO_FUNC1_CHIPCLKCSR,
-				   saveclk, &err);
+		brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
+				  saveclk, &err);
 	}
 
 	if (err == 0) {
 		/* Allow full data communication using DPC from now on. */
 		brcmf_sdiod_change_state(bus->sdiodev, BRCMF_SDIOD_DATA);
 
-		err = brcmf_sdiod_intr_register(sdiod);
+		err = brcmf_sdiod_intr_register(sdiodev);
 		if (err != 0)
 			brcmf_err("intr register failed:%d\n", err);
 	}
@@ -4169,60 +4485,41 @@
 	if (err != 0)
 		brcmf_sdio_clkctl(bus, CLK_NONE, false);
 
-	sdio_release_host(sdiod->func1);
-
-	/* Assign bus interface call back */
-	sdiod->bus_if->dev = sdiod->dev;
-	sdiod->bus_if->ops = &brcmf_sdio_bus_ops;
-	sdiod->bus_if->chip = bus->ci->chip;
-	sdiod->bus_if->chiprev = bus->ci->chiprev;
+	sdio_release_host(sdiodev->func[1]);
 
-	/* Attach to the common layer, reserve hdr space */
-	err = brcmf_attach(sdiod->dev, sdiod->settings);
-	if (err != 0) {
-		brcmf_err("brcmf_attach failed\n");
-		goto fail;
+	/* Waking up from deep sleep don't requirerd to reint the sdio bus
+	 * as all sdiod core registers will get restored by Firmware using
+	 * FCBS engine.
+	 */
+	if (!bus->sdiodev->ulp) {
+		err = brcmf_bus_started(dev);
+		if (err != 0) {
+			brcmf_err("dongle is not responding\n");
+			goto fail;
+		}
+	} else {
+		/* For ULP, after firmware redownload complete
+		 * set ULP state to IDLE
+		 */
+		if (bus->sdiodev->fmac_ulp.ulp_state == FMAC_ULP_TRIGGERED)
+			bus->sdiodev->fmac_ulp.ulp_state = FMAC_ULP_IDLE;
 	}
 
-	/* ready */
 	return;
 
 release:
-	sdio_release_host(sdiod->func1);
+	sdio_release_host(sdiodev->func[1]);
 fail:
 	brcmf_dbg(TRACE, "failed: dev=%s, err=%d\n", dev_name(dev), err);
-	device_release_driver(&sdiod->func2->dev);
+	device_release_driver(&sdiodev->func[2]->dev);
 	device_release_driver(dev);
 }
 
-static struct brcmf_fw_request *
-brcmf_sdio_prepare_fw_request(struct brcmf_sdio *bus)
-{
-	struct brcmf_fw_request *fwreq;
-	struct brcmf_fw_name fwnames[] = {
-		{ ".bin", bus->sdiodev->fw_name },
-		{ ".txt", bus->sdiodev->nvram_name },
-	};
-
-	fwreq = brcmf_fw_alloc_request(bus->ci->chip, bus->ci->chiprev,
-				       brcmf_sdio_fwnames,
-				       ARRAY_SIZE(brcmf_sdio_fwnames),
-				       fwnames, ARRAY_SIZE(fwnames));
-	if (!fwreq)
-		return NULL;
-
-	fwreq->items[BRCMF_SDIO_FW_CODE].type = BRCMF_FW_TYPE_BINARY;
-	fwreq->items[BRCMF_SDIO_FW_NVRAM].type = BRCMF_FW_TYPE_NVRAM;
-
-	return fwreq;
-}
-
 struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev)
 {
 	int ret;
 	struct brcmf_sdio *bus;
 	struct workqueue_struct *wq;
-	struct brcmf_fw_request *fwreq;
 
 	brcmf_dbg(TRACE, "Enter\n");
 
@@ -4239,9 +4536,21 @@
 	bus->txminmax = BRCMF_TXMINMAX;
 	bus->tx_seq = SDPCM_SEQ_WRAP - 1;
 
+	/* attempt to attach to the dongle */
+	if (!(brcmf_sdio_probe_attach(bus))) {
+		brcmf_err("brcmf_sdio_probe_attach failed\n");
+		goto fail;
+	}
+
 	/* single-threaded workqueue */
-	wq = alloc_ordered_workqueue("brcmf_wq/%s", WQ_MEM_RECLAIM,
-				     dev_name(&sdiodev->func1->dev));
+	if (sdiodev->settings->sdio_wq_highpri) {
+		wq = alloc_workqueue("brcmf_wq/%s",
+				     WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_UNBOUND,
+				     1, dev_name(&sdiodev->func[1]->dev));
+	} else {
+		wq = alloc_ordered_workqueue("brcmf_wq/%s", WQ_MEM_RECLAIM,
+					     dev_name(&sdiodev->func[1]->dev));
+	}
 	if (!wq) {
 		brcmf_err("insufficient memory to create txworkqueue\n");
 		goto fail;
@@ -4250,12 +4559,6 @@
 	INIT_WORK(&bus->datawork, brcmf_sdio_dataworker);
 	bus->brcmf_wq = wq;
 
-	/* attempt to attach to the dongle */
-	if (!(brcmf_sdio_probe_attach(bus))) {
-		brcmf_err("brcmf_sdio_probe_attach failed\n");
-		goto fail;
-	}
-
 	spin_lock_init(&bus->rxctl_lock);
 	spin_lock_init(&bus->txq_lock);
 	init_waitqueue_head(&bus->ctrl_wait);
@@ -4267,7 +4570,7 @@
 	init_completion(&bus->watchdog_wait);
 	bus->watchdog_tsk = kthread_run(brcmf_sdio_watchdog_thread,
 					bus, "brcmf_wdog/%s",
-					dev_name(&sdiodev->func1->dev));
+					dev_name(&sdiodev->func[1]->dev));
 	if (IS_ERR(bus->watchdog_tsk)) {
 		pr_warn("brcmf_watchdog thread failed to start\n");
 		bus->watchdog_tsk = NULL;
@@ -4276,24 +4579,50 @@
 	bus->dpc_triggered = false;
 	bus->dpc_running = false;
 
+	/* Assign bus interface call back */
+	bus->sdiodev->bus_if->dev = bus->sdiodev->dev;
+	bus->sdiodev->bus_if->ops = &brcmf_sdio_bus_ops;
+	bus->sdiodev->bus_if->chip = bus->ci->chip;
+	bus->sdiodev->bus_if->chiprev = bus->ci->chiprev;
+
 	/* default sdio bus header length for tx packet */
 	bus->tx_hdrlen = SDPCM_HWHDR_LEN + SDPCM_SWHDR_LEN;
 
+	/* Attach to the common layer, reserve hdr space */
+	ret = brcmf_attach(bus->sdiodev->dev, bus->sdiodev->settings);
+	if (ret != 0) {
+		brcmf_err("brcmf_attach failed\n");
+		goto fail;
+	}
+
 	/* Query the F2 block size, set roundup accordingly */
-	bus->blocksize = bus->sdiodev->func2->cur_blksize;
+	bus->blocksize = bus->sdiodev->func[2]->cur_blksize;
 	bus->roundup = min(max_roundup, bus->blocksize);
 
-	sdio_claim_host(bus->sdiodev->func1);
+	/* Allocate buffers */
+	if (bus->sdiodev->bus_if->maxctl) {
+		bus->sdiodev->bus_if->maxctl += bus->roundup;
+		bus->rxblen =
+		    roundup((bus->sdiodev->bus_if->maxctl + SDPCM_HDRLEN),
+			    ALIGNMENT) + bus->head_align;
+		bus->rxbuf = kmalloc(bus->rxblen, GFP_ATOMIC);
+		if (!(bus->rxbuf)) {
+			brcmf_err("rxbuf allocation failed\n");
+			goto fail;
+		}
+	}
+
+	sdio_claim_host(bus->sdiodev->func[1]);
 
 	/* Disable F2 to clear any intermediate frame state on the dongle */
-	sdio_disable_func(bus->sdiodev->func2);
+	sdio_disable_func(bus->sdiodev->func[SDIO_FUNC_2]);
 
 	bus->rxflow = false;
 
 	/* Done with backplane-dependent accesses, can drop clock... */
-	brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, 0, NULL);
+	brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, 0, NULL);
 
-	sdio_release_host(bus->sdiodev->func1);
+	sdio_release_host(bus->sdiodev->func[1]);
 
 	/* ...and initialize clock/power states */
 	bus->clkstate = CLK_SDONLY;
@@ -4303,19 +4632,21 @@
 	/* SR state */
 	bus->sr_enabled = false;
 
+	brcmf_sdio_debugfs_create(bus);
 	brcmf_dbg(INFO, "completed!!\n");
 
-	fwreq = brcmf_sdio_prepare_fw_request(bus);
-	if (!fwreq) {
-		ret = -ENOMEM;
+	ret = brcmf_fw_map_chip_to_name(bus->ci->chip, bus->ci->chiprev,
+					brcmf_sdio_fwnames,
+					ARRAY_SIZE(brcmf_sdio_fwnames),
+					sdiodev->fw_name, sdiodev->nvram_name);
+	if (ret)
 		goto fail;
-	}
 
-	ret = brcmf_fw_get_firmwares(sdiodev->dev, fwreq,
+	ret = brcmf_fw_get_firmwares(sdiodev->dev, BRCMF_FW_REQUEST_NVRAM,
+				     sdiodev->fw_name, sdiodev->nvram_name,
 				     brcmf_sdio_firmware_callback);
 	if (ret != 0) {
 		brcmf_err("async firmware request failed: %d\n", ret);
-		kfree(fwreq);
 		goto fail;
 	}
 
@@ -4350,7 +4681,7 @@
 
 		if (bus->ci) {
 			if (bus->sdiodev->state != BRCMF_SDIOD_NOMEDIUM) {
-				sdio_claim_host(bus->sdiodev->func1);
+				sdio_claim_host(bus->sdiodev->func[1]);
 				brcmf_sdio_wd_timer(bus, false);
 				brcmf_sdio_clkctl(bus, CLK_AVAIL, false);
 				/* Leave the device in state where it is
@@ -4358,9 +4689,19 @@
 				 * necessary cores.
 				 */
 				msleep(20);
-				brcmf_chip_set_passive(bus->ci);
+				if (bus->sdiodev->fmac_ulp.ulp_state ==
+					FMAC_ULP_ENTRY_RECV) {
+					brcmf_chip_ulp_reset_lhl_regs(bus->ci);
+					brcmf_chip_reset_pmu_regs(bus->ci);
+				} else {
+					brcmf_chip_set_passive(bus->ci);
+				}
+				/* Reset the PMU, backplane and all the
+				 * cores by using the PMUWatchdogCounter.
+				 */
+				brcmf_chip_reset_watchdog(bus->ci);
 				brcmf_sdio_clkctl(bus, CLK_NONE, false);
-				sdio_release_host(bus->sdiodev->func1);
+				sdio_release_host(bus->sdiodev->func[1]);
 			}
 			brcmf_chip_detach(bus->ci);
 		}
@@ -4407,10 +4748,48 @@
 {
 	int ret;
 
-	sdio_claim_host(bus->sdiodev->func1);
+	sdio_claim_host(bus->sdiodev->func[1]);
 	ret = brcmf_sdio_bus_sleep(bus, sleep, false);
-	sdio_release_host(bus->sdiodev->func1);
+	sdio_release_host(bus->sdiodev->func[1]);
 
 	return ret;
 }
 
+/* Check F2 Ready bit before sending data to Firmware */
+static int
+brcmf_sdio_f2_ready(struct brcmf_sdio *bus)
+{
+	int ret = -1;
+	int iordy_status = 0;
+
+	sdio_claim_host(bus->sdiodev->func[1]);
+	/* Read the status of IOR2 */
+	iordy_status = brcmf_sdiod_regrb(bus->sdiodev,
+					 SDIO_CCCR_IORx,
+					 NULL);
+
+	sdio_release_host(bus->sdiodev->func[1]);
+	ret = iordy_status & SDIO_FUNC_ENABLE_2;
+	return ret;
+}
+
+static int brcmf_ulp_event_notify(struct brcmf_if *ifp,
+				  const struct brcmf_event_msg *evtmsg,
+				  void *data)
+{
+	int err = 0;
+	struct brcmf_bus *bus_if = ifp->drvr->bus_if;
+	struct brcmf_sdio_dev *sdiodev;
+	struct brcmf_sdio *bus;
+	struct brcmf_ulp_event *ulp_event = (struct brcmf_ulp_event *)data;
+
+	sdiodev = bus_if->bus_priv.sdio;
+	bus = sdiodev->bus;
+
+	brcmf_dbg(ULP, "Chip went to DS1 state : action %d\n",
+		  ulp_event->ulp_dongle_action);
+	if (ulp_event->ulp_dongle_action == FMAC_ULP_ENTRY)
+		bus->sdiodev->fmac_ulp.ulp_state = FMAC_ULP_ENTRY_RECV;
+
+	return err;
+}
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h	2020-06-19 21:00:31.420227716 -0400
@@ -21,6 +21,10 @@
 #include <linux/firmware.h>
 #include "firmware.h"
 
+#define SDIO_FUNC_0		0
+#define SDIO_FUNC_1		1
+#define SDIO_FUNC_2		2
+
 #define SDIOD_FBR_SIZE		0x100
 
 /* io_en */
@@ -35,29 +39,28 @@
 #define INTR_STATUS_FUNC1	0x2
 #define INTR_STATUS_FUNC2	0x4
 
+/* Maximum number of I/O funcs */
+#define SDIOD_MAX_IOFUNCS	7
+
 /* mask of register map */
 #define REG_F0_REG_MASK		0x7FF
 #define REG_F1_MISC_MASK	0x1FFFF
 
-/* function 0 vendor specific CCCR registers */
+/* as of sdiod rev 0, supports 3 functions */
+#define SBSDIO_NUM_FUNCTION		3
 
+/* function 0 vendor specific CCCR registers */
 #define SDIO_CCCR_BRCM_CARDCAP			0xf0
-#define SDIO_CCCR_BRCM_CARDCAP_CMD14_SUPPORT	BIT(1)
-#define SDIO_CCCR_BRCM_CARDCAP_CMD14_EXT	BIT(2)
-#define SDIO_CCCR_BRCM_CARDCAP_CMD_NODEC	BIT(3)
-
-/* Interrupt enable bits for each function */
-#define SDIO_CCCR_IEN_FUNC0			BIT(0)
-#define SDIO_CCCR_IEN_FUNC1			BIT(1)
-#define SDIO_CCCR_IEN_FUNC2			BIT(2)
-
-#define SDIO_CCCR_BRCM_CARDCTRL			0xf1
-#define SDIO_CCCR_BRCM_CARDCTRL_WLANRESET	BIT(1)
-
+#define SDIO_CCCR_BRCM_CARDCAP_CMD14_SUPPORT	0x02
+#define SDIO_CCCR_BRCM_CARDCAP_CMD14_EXT	0x04
+#define SDIO_CCCR_BRCM_CARDCAP_CMD_NODEC	0x08
+#define SDIO_CCCR_BRCM_CARDCTRL		0xf1
+#define SDIO_CCCR_BRCM_CARDCTRL_WLANRESET	0x02
 #define SDIO_CCCR_BRCM_SEPINT			0xf2
-#define SDIO_CCCR_BRCM_SEPINT_MASK		BIT(0)
-#define SDIO_CCCR_BRCM_SEPINT_OE		BIT(1)
-#define SDIO_CCCR_BRCM_SEPINT_ACT_HI		BIT(2)
+
+#define  SDIO_SEPINT_MASK		0x01
+#define  SDIO_SEPINT_OE			0x02
+#define  SDIO_SEPINT_ACT_HI		0x04
 
 /* function 1 miscellaneous registers */
 
@@ -77,7 +80,7 @@
 #define SBSDIO_GPIO_OUT			0x10006
 /* gpio enable */
 #define SBSDIO_GPIO_EN			0x10007
-/* rev < 7, watermark for sdio device TX path */
+/* rev < 7, watermark for sdio device */
 #define SBSDIO_WATERMARK		0x10008
 /* control busy signal generation */
 #define SBSDIO_DEVICE_CTL		0x10009
@@ -104,12 +107,8 @@
 #define SBSDIO_FUNC1_RFRAMEBCHI		0x1001C
 /* MesBusyCtl (rev 11) */
 #define SBSDIO_FUNC1_MESBUSYCTRL	0x1001D
-/* Watermark for sdio device RX path */
-#define SBSDIO_MESBUSY_RXFIFO_WM_MASK	0x7F
-#define SBSDIO_MESBUSY_RXFIFO_WM_SHIFT	0
 /* Enable busy capability for MES access */
-#define SBSDIO_MESBUSYCTRL_ENAB		0x80
-#define SBSDIO_MESBUSYCTRL_ENAB_SHIFT	7
+#define SBSDIO_MESBUSYCTRL_ENAB         0x80
 
 /* Sdio Core Rev 12 */
 #define SBSDIO_FUNC1_WAKEUPCTRL		0x1001E
@@ -135,6 +134,11 @@
 /* with b15, maps to 32-bit SB access */
 #define SBSDIO_SB_ACCESS_2_4B_FLAG	0x08000
 
+/* valid bits in SBSDIO_FUNC1_SBADDRxxx regs */
+
+#define SBSDIO_SBADDRLOW_MASK		0x80	/* Valid bits in SBADDRLOW */
+#define SBSDIO_SBADDRMID_MASK		0xff	/* Valid bits in SBADDRMID */
+#define SBSDIO_SBADDRHIGH_MASK		0xffU	/* Valid bits in SBADDRHIGH */
 /* Address bits from SBADDR regs */
 #define SBSDIO_SBWINDOW_MASK		0xffff8000
 
@@ -176,11 +180,39 @@
 struct brcmf_sdio;
 struct brcmf_sdiod_freezer;
 
+/* ULP SHM Offsets info */
+struct ulp_shm_info {
+	u32 m_ulp_ctrl_sdio;
+	u32 m_ulp_wakeevt_ind;
+	u32 m_ulp_wakeind;
+	u32 m_ulp_phytxblk;
+};
+
+/* FMAC ULP state machine */
+#define FMAC_ULP_IDLE		(0)
+#define FMAC_ULP_ENTRY_RECV		(1)
+#define FMAC_ULP_TRIGGERED		(2)
+
+/* BRCMF_E_ULP event data */
+#define FMAC_ULP_EVENT_VERSION		1
+#define FMAC_ULP_DISABLE_CONSOLE		1 /* Disable console */
+#define FMAC_ULP_UCODE_DOWNLOAD		2 /* Download ULP ucode file */
+#define FMAC_ULP_ENTRY		3 /* Inform ulp entry to Host */
+
+struct brcmf_ulp {
+	uint ulp_state;
+	struct ulp_shm_info ulp_shm_offset;
+};
+
+struct brcmf_ulp_event {
+	u16 version;
+	u16 ulp_dongle_action;
+};
+
 struct brcmf_sdio_dev {
-	struct sdio_func *func1;
-	struct sdio_func *func2;
+	struct sdio_func *func[SDIO_MAX_FUNCS];
+	u8 num_funcs;			/* Supported funcs on client */
 	u32 sbwad;			/* Save backplane window address */
-	struct brcmf_core *cc_core;	/* chipcommon core info struct */
 	struct brcmf_sdio *bus;
 	struct device *dev;
 	struct brcmf_bus *bus_if;
@@ -201,6 +233,8 @@
 	bool wowl_enabled;
 	enum brcmf_sdiod_state state;
 	struct brcmf_sdiod_freezer *freezer;
+	struct brcmf_ulp fmac_ulp;
+	bool ulp;
 };
 
 /* sdio core registers */
@@ -296,24 +330,13 @@
 int brcmf_sdiod_intr_register(struct brcmf_sdio_dev *sdiodev);
 void brcmf_sdiod_intr_unregister(struct brcmf_sdio_dev *sdiodev);
 
-/* SDIO device register access interface */
-/* Accessors for SDIO Function 0 */
-#define brcmf_sdiod_func0_rb(sdiodev, addr, r) \
-	sdio_f0_readb((sdiodev)->func1, (addr), (r))
-
-#define brcmf_sdiod_func0_wb(sdiodev, addr, v, ret) \
-	sdio_f0_writeb((sdiodev)->func1, (v), (addr), (ret))
-
-/* Accessors for SDIO Function 1 */
-#define brcmf_sdiod_readb(sdiodev, addr, r) \
-	sdio_readb((sdiodev)->func1, (addr), (r))
-
-#define brcmf_sdiod_writeb(sdiodev, addr, v, ret) \
-	sdio_writeb((sdiodev)->func1, (v), (addr), (ret))
-
-u32 brcmf_sdiod_readl(struct brcmf_sdio_dev *sdiodev, u32 addr, int *ret);
-void brcmf_sdiod_writel(struct brcmf_sdio_dev *sdiodev, u32 addr, u32 data,
-			int *ret);
+/* sdio device register access interface */
+u8 brcmf_sdiod_regrb(struct brcmf_sdio_dev *sdiodev, u32 addr, int *ret);
+u32 brcmf_sdiod_regrl(struct brcmf_sdio_dev *sdiodev, u32 addr, int *ret);
+void brcmf_sdiod_regwb(struct brcmf_sdio_dev *sdiodev, u32 addr, u8 data,
+		       int *ret);
+void brcmf_sdiod_regwl(struct brcmf_sdio_dev *sdiodev, u32 addr, u32 data,
+		       int *ret);
 
 /* Buffer transfer to/from device (client) core via cmd53.
  *   fn:       function number
@@ -353,8 +376,7 @@
 		      u8 *data, uint size);
 
 /* Issue an abort to the specified function */
-int brcmf_sdiod_abort(struct brcmf_sdio_dev *sdiodev, struct sdio_func *func);
-
+int brcmf_sdiod_abort(struct brcmf_sdio_dev *sdiodev, uint fn);
 void brcmf_sdiod_sgtable_alloc(struct brcmf_sdio_dev *sdiodev);
 void brcmf_sdiod_change_state(struct brcmf_sdio_dev *sdiodev,
 			      enum brcmf_sdiod_state state);
@@ -379,6 +401,9 @@
 }
 #endif /* CONFIG_PM_SLEEP */
 
+int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev);
+int brcmf_sdiod_remove(struct brcmf_sdio_dev *sdiodev);
+
 struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev);
 void brcmf_sdio_remove(struct brcmf_sdio *bus);
 void brcmf_sdio_isr(struct brcmf_sdio *bus);
@@ -388,4 +413,83 @@
 int brcmf_sdio_sleep(struct brcmf_sdio *bus, bool sleep);
 void brcmf_sdio_trigger_dpc(struct brcmf_sdio *bus);
 
+/* SHM offsets */
+#define M_DS1_CTRL_SDIO(ptr)	((ptr).ulp_shm_offset.m_ulp_ctrl_sdio)
+#define M_WAKEEVENT_IND(ptr)	((ptr).ulp_shm_offset.m_ulp_wakeevt_ind)
+#define M_ULP_WAKE_IND(ptr)		((ptr).ulp_shm_offset.m_ulp_wakeind)
+#define M_DS1_PHYTX_ERR_BLK(ptr)	((ptr).ulp_shm_offset.m_ulp_phytxblk)
+
+#define D11_BASE_ADDR			0x18001000
+#define D11_AXI_BASE_ADDR		0xE8000000
+#define D11_SHM_BASE_ADDR		(D11_AXI_BASE_ADDR + 0x4000)
+
+#define D11REG_ADDR(offset)	(D11_BASE_ADDR + (offset))
+#define D11IHR_ADDR(offset)	(D11_AXI_BASE_ADDR + 0x400 + (2 * (offset)))
+#define D11SHM_ADDR(offset)	(D11_SHM_BASE_ADDR + (offset))
+
+/* MacControl register */
+#define D11_MACCONTROL_REG			D11REG_ADDR(0x120)
+#define D11_MACCONTROL_REG_WAKE		0x4000000
+
+/* HUDI Sequence SHM bits */
+#define	C_DS1_CTRL_SDIO_DS1_SLEEP		0x1
+#define	C_DS1_CTRL_SDIO_MAC_ON			0x2
+#define	C_DS1_CTRL_SDIO_RADIO_PHY_ON	0x4
+#define	C_DS1_CTRL_SDIO_DS1_EXIT		0x8
+#define	C_DS1_CTRL_PROC_DONE			0x100
+#define	C_DS1_CTRL_REQ_VALID			0x200
+
+/* M_ULP_WAKEIND bits */
+#define	C_WATCHDOG_EXPIRY	BIT(0)
+#define	C_FCBS_ERROR		BIT(1)
+#define	C_RETX_FAILURE		BIT(2)
+#define	C_HOST_WAKEUP		BIT(3)
+#define	C_INVALID_FCBS_BLOCK	BIT(4)
+#define	C_HUDI_DS1_EXIT		BIT(5)
+#define	C_LOB_SLEEP		BIT(6)
+#define	C_DS1_PHY_TXERR		BIT(9)
+#define	C_DS1_WAKE_TIMER	BIT(10)
+
+#define PHYTX_ERR_BLK_SIZE		18
+#define D11SHM_FIRST2BYTE_MASK		0xFFFF0000
+#define D11SHM_SECOND2BYTE_MASK		0x0000FFFF
+#define D11SHM_2BYTE_SHIFT		16
+
+#define D11SHM_RD(sdh, offset, ret) \
+	brcmf_sdiod_regrl(sdh, D11SHM_ADDR(offset), ret)
+
+/* SHM Read is motified based on SHM 4 byte alignment as SHM size is 2 bytes and
+ * 2 byte is currently not working on FMAC
+ * If SHM address is not 4 byte aligned, then right shift by 16
+ * otherwise, mask the first two MSB bytes
+ * Suppose data in address 7260 is 0x440002 and it is 4 byte aligned
+ * Correct SHM value is 0x2 for this SHM offset and next SHM value is 0x44
+ */
+#define D11SHM_RDW(sdh, offset, ret) \
+	((offset % 4) ? \
+		(brcmf_sdiod_regrl(sdh, D11SHM_ADDR(offset), ret) \
+		>> D11SHM_2BYTE_SHIFT) : \
+		(brcmf_sdiod_regrl(sdh, D11SHM_ADDR(offset), ret) \
+		& D11SHM_SECOND2BYTE_MASK))
+
+/* SHM is of size 2 bytes, 4 bytes write will overwrite other SHM's
+ * First read 4 bytes and then clear the required two bytes based on
+ * 4 byte alignment, then update the required value and write the
+ * 4 byte value now
+ */
+#define D11SHM_WR(sdh, offset, val, mask, ret) \
+	do { \
+		if ((offset) % 4) \
+			val = (val & D11SHM_SECOND2BYTE_MASK) | \
+				((mask) << D11SHM_2BYTE_SHIFT); \
+		else \
+			val = (mask) | (val & D11SHM_FIRST2BYTE_MASK); \
+		brcmf_sdiod_regwl(sdh, D11SHM_ADDR(offset), val, ret); \
+	} while (0)
+#define D11REG_WR(sdh, addr, val, ret) \
+	brcmf_sdiod_regwl(sdh, addr, val, ret)
+
+#define D11REG_RD(sdh, addr, ret) \
+	brcmf_sdiod_regrl(sdh, addr, ret)
+
 #endif /* BRCMFMAC_SDIO_H */
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c	2020-06-19 21:00:31.420227716 -0400
@@ -30,6 +30,7 @@
 #include "core.h"
 #include "common.h"
 #include "bcdc.h"
+#include "cfg80211.h"
 
 
 #define IOCTL_RESP_TIMEOUT		msecs_to_jiffies(2000)
@@ -46,13 +47,13 @@
 #define BRCMF_USB_CBCTL_READ		1
 #define BRCMF_USB_MAX_PKT_SIZE		1600
 
-BRCMF_FW_DEF(43143, "brcmfmac43143");
-BRCMF_FW_DEF(43236B, "brcmfmac43236b");
-BRCMF_FW_DEF(43242A, "brcmfmac43242a");
-BRCMF_FW_DEF(43569, "brcmfmac43569");
-BRCMF_FW_DEF(4373, "brcmfmac4373");
+BRCMF_FW_DEF(43143, "brcmfmac43143.bin");
+BRCMF_FW_DEF(43236B, "brcmfmac43236b.bin");
+BRCMF_FW_DEF(43242A, "brcmfmac43242a.bin");
+BRCMF_FW_DEF(43569, "brcmfmac43569.bin");
+BRCMF_FW_DEF(4373, "brcmfmac4373.bin");
 
-static const struct brcmf_firmware_mapping brcmf_usb_fwnames[] = {
+static struct brcmf_firmware_mapping brcmf_usb_fwnames[] = {
 	BRCMF_FW_ENTRY(BRCM_CC_43143_CHIP_ID, 0xFFFFFFFF, 43143),
 	BRCMF_FW_ENTRY(BRCM_CC_43235_CHIP_ID, 0x00000008, 43236B),
 	BRCMF_FW_ENTRY(BRCM_CC_43236_CHIP_ID, 0x00000008, 43236B),
@@ -103,7 +104,6 @@
 				 */
 #define DL_IMAGE_TOOBIG	7	/* firmware image too big */
 
-
 struct trx_header_le {
 	__le32 magic;		/* "HDR0" */
 	__le32 len;		/* Length of file including header */
@@ -160,7 +160,7 @@
 
 	struct usb_device *usbdev;
 	struct device *dev;
-	struct completion dev_init_done;
+	struct completion fw_downloaded;
 
 	int ctl_in_pipe, ctl_out_pipe;
 	struct urb *ctl_urb; /* URB for control endpoint */
@@ -175,7 +175,6 @@
 
 	struct urb *bulk_urb; /* used for FW download */
 
-	bool wowl_enabled;
 	struct brcmf_mp_device *settings;
 };
 
@@ -323,27 +322,44 @@
 	int err = 0;
 	int timeout = 0;
 	struct brcmf_usbdev_info *devinfo = brcmf_usb_get_businfo(dev);
+	struct usb_interface *intf = to_usb_interface(dev);
 
 	brcmf_dbg(USB, "Enter\n");
-	if (devinfo->bus_pub.state != BRCMFMAC_USB_STATE_UP)
-		return -EIO;
 
-	if (test_and_set_bit(0, &devinfo->ctl_op))
-		return -EIO;
+	err = usb_autopm_get_interface(intf);
+	if (err)
+		goto out;
+
+	if (devinfo->bus_pub.state != BRCMFMAC_USB_STATE_UP) {
+		err = -EIO;
+		goto fail;
+	}
+
+	if (test_and_set_bit(0, &devinfo->ctl_op)) {
+		err = -EIO;
+		goto fail;
+	}
 
 	devinfo->ctl_completed = false;
 	err = brcmf_usb_send_ctl(devinfo, buf, len);
 	if (err) {
 		brcmf_err("fail %d bytes: %d\n", err, len);
 		clear_bit(0, &devinfo->ctl_op);
-		return err;
+		goto fail;
 	}
 	timeout = brcmf_usb_ioctl_resp_wait(devinfo);
-	clear_bit(0, &devinfo->ctl_op);
 	if (!timeout) {
 		brcmf_err("Txctl wait timed out\n");
+		usb_kill_urb(devinfo->ctl_urb);
+		clear_bit(0, &devinfo->ctl_op);
 		err = -EIO;
+		goto fail;
 	}
+	clear_bit(0, &devinfo->ctl_op);
+
+fail:
+	usb_autopm_put_interface(intf);
+out:
 	return err;
 }
 
@@ -352,32 +368,48 @@
 	int err = 0;
 	int timeout = 0;
 	struct brcmf_usbdev_info *devinfo = brcmf_usb_get_businfo(dev);
+	struct usb_interface *intf = to_usb_interface(dev);
 
 	brcmf_dbg(USB, "Enter\n");
-	if (devinfo->bus_pub.state != BRCMFMAC_USB_STATE_UP)
-		return -EIO;
 
-	if (test_and_set_bit(0, &devinfo->ctl_op))
-		return -EIO;
+	err = usb_autopm_get_interface(intf);
+	if (err)
+		goto out;
+
+	if (devinfo->bus_pub.state != BRCMFMAC_USB_STATE_UP) {
+		err = -EIO;
+		goto fail;
+	}
+
+	if (test_and_set_bit(0, &devinfo->ctl_op)) {
+		err = -EIO;
+		goto fail;
+	}
 
 	devinfo->ctl_completed = false;
 	err = brcmf_usb_recv_ctl(devinfo, buf, len);
 	if (err) {
 		brcmf_err("fail %d bytes: %d\n", err, len);
 		clear_bit(0, &devinfo->ctl_op);
-		return err;
+		goto fail;
 	}
 	timeout = brcmf_usb_ioctl_resp_wait(devinfo);
 	err = devinfo->ctl_urb_status;
-	clear_bit(0, &devinfo->ctl_op);
 	if (!timeout) {
 		brcmf_err("rxctl wait timed out\n");
+		usb_kill_urb(devinfo->ctl_urb);
+		clear_bit(0, &devinfo->ctl_op);
 		err = -EIO;
+		goto fail;
 	}
+	clear_bit(0, &devinfo->ctl_op);
+
+fail:
+	usb_autopm_put_interface(intf);
 	if (!err)
 		return devinfo->ctl_urb_actual_length;
-	else
-		return err;
+out:
+	return err;
 }
 
 static struct brcmf_usbreq *brcmf_usb_deq(struct brcmf_usbdev_info *devinfo,
@@ -515,10 +547,12 @@
 		return;
 	}
 
-	if (devinfo->bus_pub.state == BRCMFMAC_USB_STATE_UP) {
+	if (devinfo->bus_pub.state == BRCMFMAC_USB_STATE_UP ||
+	    devinfo->bus_pub.state == BRCMFMAC_USB_STATE_SLEEP) {
 		skb_put(skb, urb->actual_length);
 		brcmf_rx_frame(devinfo->dev, skb, true);
 		brcmf_usb_rx_refill(devinfo, req);
+		usb_mark_last_busy(urb->dev);
 	} else {
 		brcmu_pkt_buf_free_skb(skb);
 		brcmf_usb_enq(devinfo, &devinfo->rx_freeq, req, NULL);
@@ -604,6 +638,11 @@
 	struct brcmf_usbreq  *req;
 	int ret;
 	unsigned long flags;
+	struct usb_interface *intf = to_usb_interface(dev);
+
+	ret = usb_autopm_get_interface(intf);
+	if (ret)
+		goto out;
 
 	brcmf_dbg(USB, "Enter, skb=%p\n", skb);
 	if (devinfo->bus_pub.state != BRCMFMAC_USB_STATE_UP) {
@@ -619,6 +658,8 @@
 		goto fail;
 	}
 
+	if (devinfo->bus_pub.bus->allow_skborphan)
+		skb_orphan(skb);
 	req->skb = skb;
 	req->devinfo = devinfo;
 	usb_fill_bulk_urb(req->urb, devinfo->usbdev, devinfo->tx_pipe,
@@ -642,9 +683,10 @@
 		devinfo->tx_flowblock = true;
 	}
 	spin_unlock_irqrestore(&devinfo->tx_flowblock_lock, flags);
-	return 0;
 
 fail:
+	usb_autopm_put_interface(intf);
+out:
 	return ret;
 }
 
@@ -684,18 +726,12 @@
 
 static void brcmf_cancel_all_urbs(struct brcmf_usbdev_info *devinfo)
 {
-	int i;
-
 	if (devinfo->ctl_urb)
 		usb_kill_urb(devinfo->ctl_urb);
 	if (devinfo->bulk_urb)
 		usb_kill_urb(devinfo->bulk_urb);
-	if (devinfo->tx_reqs)
-		for (i = 0; i < devinfo->bus_pub.ntxq; i++)
-			usb_kill_urb(devinfo->tx_reqs[i].urb);
-	if (devinfo->rx_reqs)
-		for (i = 0; i < devinfo->bus_pub.nrxq; i++)
-			usb_kill_urb(devinfo->rx_reqs[i].urb);
+	brcmf_usb_free_q(&devinfo->tx_postq, true);
+	brcmf_usb_free_q(&devinfo->rx_postq, true);
 }
 
 static void brcmf_usb_down(struct device *dev)
@@ -1008,20 +1044,32 @@
 brcmf_usb_fw_download(struct brcmf_usbdev_info *devinfo)
 {
 	int err;
+	struct usb_interface *intf;
 
 	brcmf_dbg(USB, "Enter\n");
-	if (devinfo == NULL)
-		return -ENODEV;
+	if (!devinfo) {
+		err = -ENODEV;
+		goto out;
+	}
 
 	if (!devinfo->image) {
 		brcmf_err("No firmware!\n");
-		return -ENOENT;
+		err = -ENOENT;
+		goto out;
 	}
 
+	intf = to_usb_interface(devinfo->dev);
+	err = usb_autopm_get_interface(intf);
+	if (err)
+		goto out;
+
 	err = brcmf_usb_dlstart(devinfo,
 		(u8 *)devinfo->image, devinfo->image_len);
 	if (err == 0)
 		err = brcmf_usb_dlrun(devinfo);
+
+	usb_autopm_put_interface(intf);
+out:
 	return err;
 }
 
@@ -1122,65 +1170,74 @@
 	return NULL;
 }
 
-static void brcmf_usb_wowl_config(struct device *dev, bool enabled)
+static int brcmf_usb_get_fwname(struct device *dev, u32 chip, u32 chiprev,
+				u8 *fw_name)
 {
 	struct brcmf_usbdev_info *devinfo = brcmf_usb_get_businfo(dev);
+	int ret = 0;
 
-	brcmf_dbg(USB, "Configuring WOWL, enabled=%d\n", enabled);
-	devinfo->wowl_enabled = enabled;
-	if (enabled)
-		device_set_wakeup_enable(devinfo->dev, true);
+	if (devinfo->fw_name[0] != '\0')
+		strlcpy(fw_name, devinfo->fw_name, BRCMF_FW_NAME_LEN);
 	else
-		device_set_wakeup_enable(devinfo->dev, false);
-}
-
-static
-int brcmf_usb_get_fwname(struct device *dev, const char *ext, u8 *fw_name)
-{
-	struct brcmf_bus *bus = dev_get_drvdata(dev);
-	struct brcmf_fw_request *fwreq;
-	struct brcmf_fw_name fwnames[] = {
-		{ ext, fw_name },
-	};
-
-	fwreq = brcmf_fw_alloc_request(bus->chip, bus->chiprev,
-				       brcmf_usb_fwnames,
-				       ARRAY_SIZE(brcmf_usb_fwnames),
-				       fwnames, ARRAY_SIZE(fwnames));
-	if (!fwreq)
-		return -ENOMEM;
+		ret = brcmf_fw_map_chip_to_name(chip, chiprev,
+						brcmf_usb_fwnames,
+						ARRAY_SIZE(brcmf_usb_fwnames),
+						fw_name, NULL);
 
-	kfree(fwreq);
-	return 0;
+	return ret;
 }
 
 static const struct brcmf_bus_ops brcmf_usb_bus_ops = {
-	.preinit = brcmf_usb_up,
-	.stop = brcmf_usb_down,
 	.txdata = brcmf_usb_tx,
+	.stop = brcmf_usb_down,
 	.txctl = brcmf_usb_tx_ctlpkt,
 	.rxctl = brcmf_usb_rx_ctlpkt,
-	.wowl_config = brcmf_usb_wowl_config,
 	.get_fwname = brcmf_usb_get_fwname,
 };
 
-#define BRCMF_USB_FW_CODE	0
+static int brcmf_usb_bus_setup(struct brcmf_usbdev_info *devinfo)
+{
+	int ret;
+
+	/* Attach to the common driver interface */
+	ret = brcmf_attach(devinfo->dev, devinfo->settings);
+	if (ret) {
+		brcmf_err("brcmf_attach failed\n");
+		return ret;
+	}
+
+	if (BRCMF_FWCON_ON()) {
+		ret = brcmf_fwlog_attach(devinfo->dev);
+		if (ret)
+			goto fail;
+	}
+
+	ret = brcmf_usb_up(devinfo->dev);
+	if (ret)
+		goto fail;
+
+	ret = brcmf_bus_started(devinfo->dev);
+	if (ret)
+		goto fail;
+
+	return 0;
+fail:
+	brcmf_detach(devinfo->dev);
+	return ret;
+}
 
 static void brcmf_usb_probe_phase2(struct device *dev, int ret,
-				   struct brcmf_fw_request *fwreq)
+				   const struct firmware *fw,
+				   void *nvram, u32 nvlen)
 {
 	struct brcmf_bus *bus = dev_get_drvdata(dev);
 	struct brcmf_usbdev_info *devinfo = bus->bus_priv.usb->devinfo;
-	const struct firmware *fw;
 
 	if (ret)
 		goto error;
 
 	brcmf_dbg(USB, "Start fw downloading\n");
 
-	fw = fwreq->items[BRCMF_USB_FW_CODE].binary;
-	kfree(fwreq);
-
 	ret = check_file(fw->data);
 	if (ret < 0) {
 		brcmf_err("invalid firmware\n");
@@ -1196,46 +1253,23 @@
 	if (ret)
 		goto error;
 
-	/* Attach to the common driver interface */
-	ret = brcmf_attach(devinfo->dev, devinfo->settings);
+	ret = brcmf_usb_bus_setup(devinfo);
 	if (ret)
 		goto error;
 
-	complete(&devinfo->dev_init_done);
+	complete(&devinfo->fw_downloaded);
 	return;
 error:
 	brcmf_dbg(TRACE, "failed: dev=%s, err=%d\n", dev_name(dev), ret);
-	complete(&devinfo->dev_init_done);
+	complete(&devinfo->fw_downloaded);
 	device_release_driver(dev);
 }
 
-static struct brcmf_fw_request *
-brcmf_usb_prepare_fw_request(struct brcmf_usbdev_info *devinfo)
-{
-	struct brcmf_fw_request *fwreq;
-	struct brcmf_fw_name fwnames[] = {
-		{ ".bin", devinfo->fw_name },
-	};
-
-	fwreq = brcmf_fw_alloc_request(devinfo->bus_pub.devid,
-				       devinfo->bus_pub.chiprev,
-				       brcmf_usb_fwnames,
-				       ARRAY_SIZE(brcmf_usb_fwnames),
-				       fwnames, ARRAY_SIZE(fwnames));
-	if (!fwreq)
-		return NULL;
-
-	fwreq->items[BRCMF_USB_FW_CODE].type = BRCMF_FW_TYPE_BINARY;
-
-	return fwreq;
-}
-
 static int brcmf_usb_probe_cb(struct brcmf_usbdev_info *devinfo)
 {
 	struct brcmf_bus *bus = NULL;
 	struct brcmf_usbdev *bus_pub = NULL;
 	struct device *dev = devinfo->dev;
-	struct brcmf_fw_request *fwreq;
 	int ret;
 
 	brcmf_dbg(USB, "Enter\n");
@@ -1256,6 +1290,7 @@
 	bus->ops = &brcmf_usb_bus_ops;
 	bus->proto_type = BRCMF_PROTO_BCDC;
 	bus->always_use_fws_queue = true;
+	bus->allow_skborphan = true;
 #ifdef CONFIG_PM
 	bus->wowl_supported = true;
 #endif
@@ -1269,27 +1304,28 @@
 	}
 
 	if (!brcmf_usb_dlneeded(devinfo)) {
-		ret = brcmf_attach(devinfo->dev, devinfo->settings);
+		ret = brcmf_usb_bus_setup(devinfo);
 		if (ret)
 			goto fail;
 		/* we are done */
-		complete(&devinfo->dev_init_done);
+		complete(&devinfo->fw_downloaded);
 		return 0;
 	}
 	bus->chip = bus_pub->devid;
 	bus->chiprev = bus_pub->chiprev;
 
-	fwreq = brcmf_usb_prepare_fw_request(devinfo);
-	if (!fwreq) {
-		ret = -ENOMEM;
+	ret = brcmf_fw_map_chip_to_name(bus_pub->devid, bus_pub->chiprev,
+					brcmf_usb_fwnames,
+					ARRAY_SIZE(brcmf_usb_fwnames),
+					devinfo->fw_name, NULL);
+	if (ret)
 		goto fail;
-	}
 
 	/* request firmware here */
-	ret = brcmf_fw_get_firmwares(dev, fwreq, brcmf_usb_probe_phase2);
+	ret = brcmf_fw_get_firmwares(dev, 0, devinfo->fw_name, NULL,
+				     brcmf_usb_probe_phase2);
 	if (ret) {
 		brcmf_err("firmware request failed: %d\n", ret);
-		kfree(fwreq);
 		goto fail;
 	}
 
@@ -1333,13 +1369,14 @@
 
 	devinfo->usbdev = usb;
 	devinfo->dev = &usb->dev;
-	/* Init completion, to protect for disconnect while still loading.
-	 * Necessary because of the asynchronous firmware load construction
-	 */
-	init_completion(&devinfo->dev_init_done);
+
+	/* Initialize fw downloaded completion and mark it as not complete */
+	init_completion(&devinfo->fw_downloaded);
 
 	usb_set_intfdata(intf, devinfo);
 
+	intf->needs_remote_wakeup = 1;
+
 	/* Check that the device supports only one configuration */
 	if (usb->descriptor.bNumConfigurations != 1) {
 		brcmf_err("Number of configurations: %d not supported\n",
@@ -1414,7 +1451,7 @@
 	return 0;
 
 fail:
-	complete(&devinfo->dev_init_done);
+	complete(&devinfo->fw_downloaded);
 	kfree(devinfo);
 	usb_set_intfdata(intf, NULL);
 	return ret;
@@ -1429,7 +1466,7 @@
 	devinfo = (struct brcmf_usbdev_info *)usb_get_intfdata(intf);
 
 	if (devinfo) {
-		wait_for_completion(&devinfo->dev_init_done);
+		wait_for_completion(&devinfo->fw_downloaded);
 		/* Make sure that devinfo still exists. Firmware probe routines
 		 * may have released the device and cleared the intfdata.
 		 */
@@ -1450,13 +1487,25 @@
 {
 	struct usb_device *usb = interface_to_usbdev(intf);
 	struct brcmf_usbdev_info *devinfo = brcmf_usb_get_businfo(&usb->dev);
+	struct brcmf_bus *bus;
+	struct brcmf_cfg80211_info *config;
+	int retry = BRCMF_PM_WAIT_MAXRETRY;
 
 	brcmf_dbg(USB, "Enter\n");
+
+	bus = devinfo->bus_pub.bus;
+	config = bus->drvr->config;
+	while (retry &&
+	       config->pm_state == BRCMF_CFG80211_PM_STATE_SUSPENDING) {
+		usleep_range(10000, 20000);
+		retry--;
+	}
+	if (!retry && config->pm_state == BRCMF_CFG80211_PM_STATE_SUSPENDING)
+		brcmf_err("timed out wait for cfg80211 suspended\n");
+
 	devinfo->bus_pub.state = BRCMFMAC_USB_STATE_SLEEP;
-	if (devinfo->wowl_enabled)
-		brcmf_cancel_all_urbs(devinfo);
-	else
-		brcmf_detach(&usb->dev);
+	brcmf_cancel_all_urbs(devinfo);
+	device_set_wakeup_enable(devinfo->dev, true);
 	return 0;
 }
 
@@ -1469,11 +1518,9 @@
 	struct brcmf_usbdev_info *devinfo = brcmf_usb_get_businfo(&usb->dev);
 
 	brcmf_dbg(USB, "Enter\n");
-	if (!devinfo->wowl_enabled)
-		return brcmf_attach(devinfo->dev, devinfo->settings);
-
 	devinfo->bus_pub.state = BRCMFMAC_USB_STATE_UP;
 	brcmf_usb_rx_fill_all(devinfo);
+	device_set_wakeup_enable(devinfo->dev, false);
 	return 0;
 }
 
@@ -1481,20 +1528,11 @@
 {
 	struct usb_device *usb = interface_to_usbdev(intf);
 	struct brcmf_usbdev_info *devinfo = brcmf_usb_get_businfo(&usb->dev);
-	struct brcmf_fw_request *fwreq;
-	int ret;
 
 	brcmf_dbg(USB, "Enter\n");
 
-	fwreq = brcmf_usb_prepare_fw_request(devinfo);
-	if (!fwreq)
-		return -ENOMEM;
-
-	ret = brcmf_fw_get_firmwares(&usb->dev, fwreq, brcmf_usb_probe_phase2);
-	if (ret < 0)
-		kfree(fwreq);
-
-	return ret;
+	return brcmf_fw_get_firmwares(&usb->dev, 0, devinfo->fw_name, NULL,
+				      brcmf_usb_probe_phase2);
 }
 
 #define BRCMF_USB_DEVICE(dev_id)	\
@@ -1530,6 +1568,7 @@
 	.suspend = brcmf_usb_suspend,
 	.resume = brcmf_usb_resume,
 	.reset_resume = brcmf_usb_reset_resume,
+	.supports_autosuspend = true,
 	.disable_hub_initiated_lpm = 1,
 };
 
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c	2020-06-19 21:00:31.420227716 -0400
@@ -35,10 +35,9 @@
 	struct brcmf_if *ifp;
 	const struct brcmf_vndr_dcmd_hdr *cmdhdr = data;
 	struct sk_buff *reply;
-	unsigned int payload, ret_len;
+	int ret, payload, ret_len;
 	void *dcmd_buf = NULL, *wr_pointer;
 	u16 msglen, maxmsglen = PAGE_SIZE - 0x100;
-	int ret;
 
 	if (len < sizeof(*cmdhdr)) {
 		brcmf_err("vendor command too short: %d\n", len);
@@ -66,7 +65,7 @@
 			brcmf_err("oversize return buffer %d\n", ret_len);
 			ret_len = BRCMF_DCMD_MAXLEN;
 		}
-		payload = max_t(unsigned int, ret_len, len) + 1;
+		payload = max(ret_len, len) + 1;
 		dcmd_buf = vzalloc(payload);
 		if (NULL == dcmd_buf)
 			return -ENOMEM;
@@ -75,14 +74,27 @@
 		*(char *)(dcmd_buf + len)  = '\0';
 	}
 
+	if (cmdhdr->cmd == BRCMF_C_SET_AP) {
+		if (*(int *)(dcmd_buf) == 1) {
+			ifp->vif->wdev.iftype = NL80211_IFTYPE_AP;
+			brcmf_net_setcarrier(ifp, true);
+		} else {
+			ifp->vif->wdev.iftype = NL80211_IFTYPE_STATION;
+		}
+	}
+
 	if (cmdhdr->set)
 		ret = brcmf_fil_cmd_data_set(ifp, cmdhdr->cmd, dcmd_buf,
 					     ret_len);
 	else
 		ret = brcmf_fil_cmd_data_get(ifp, cmdhdr->cmd, dcmd_buf,
 					     ret_len);
-	if (ret != 0)
+
+	if (ret != 0) {
+		brcmf_dbg(INFO, "error(%d), return -EPERM\n", ret);
+		ret = -EPERM;
 		goto exit;
+	}
 
 	wr_pointer = dcmd_buf;
 	while (ret_len > 0) {
@@ -115,6 +127,56 @@
 	return ret;
 }
 
+s32
+brcmf_wiphy_phy_temp_evt_handler(struct brcmf_if *ifp,
+				 const struct brcmf_event_msg *e, void *data)
+
+{
+	struct brcmf_cfg80211_info *cfg = ifp->drvr->config;
+	struct wiphy *wiphy = cfg_to_wiphy(cfg);
+	struct sk_buff *skb;
+	struct nlattr *phy_temp_data;
+	u32 version, temp, tempdelta;
+	struct brcmf_phy_temp_evt *phy_temp_evt;
+
+	phy_temp_evt = (struct brcmf_phy_temp_evt *)data;
+
+	version = le32_to_cpu(phy_temp_evt->version);
+	temp = le32_to_cpu(phy_temp_evt->temp);
+	tempdelta = le32_to_cpu(phy_temp_evt->tempdelta);
+
+	skb = cfg80211_vendor_event_alloc(wiphy, NULL,
+					  sizeof(*phy_temp_evt),
+					  BRCMF_VNDR_EVTS_PHY_TEMP,
+					  GFP_KERNEL);
+
+	if (!skb) {
+		brcmf_dbg(EVENT, "NO MEM: can't allocate skb for vendor PHY_TEMP_EVENT\n");
+		return -ENOMEM;
+	}
+
+	phy_temp_data = nla_nest_start(skb, NL80211_ATTR_VENDOR_EVENTS);
+	if (!phy_temp_data) {
+		nla_nest_cancel(skb, phy_temp_data);
+		kfree_skb(skb);
+		brcmf_dbg(EVENT, "skb could not nest vendor attributes\n");
+		return -EMSGSIZE;
+	}
+
+	if (nla_put_u32(skb, BRCMF_NLATTR_VERS, version) ||
+	    nla_put_u32(skb, BRCMF_NLATTR_PHY_TEMP, temp) ||
+	    nla_put_u32(skb, BRCMF_NLATTR_PHY_TEMPDELTA, tempdelta)) {
+		kfree_skb(skb);
+		brcmf_dbg(EVENT, "NO ROOM in skb for vendor PHY_TEMP_EVENT\n");
+		return -EMSGSIZE;
+	}
+
+	nla_nest_end(skb, phy_temp_data);
+
+	cfg80211_vendor_event(skb, GFP_KERNEL);
+	return 0;
+}
+
 const struct wiphy_vendor_command brcmf_vendor_cmds[] = {
 	{
 		{
@@ -126,3 +188,10 @@
 		.doit = brcmf_cfg80211_vndr_cmds_dcmd_handler
 	},
 };
+
+const struct nl80211_vendor_cmd_info brcmf_vendor_events[] = {
+	{
+		.vendor_id = BROADCOM_OUI,
+		.subcmd = BRCMF_VNDR_EVTS_PHY_TEMP,
+	},
+};
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h	2020-06-19 21:00:31.420227716 -0400
@@ -25,6 +25,11 @@
 	BRCMF_VNDR_CMDS_LAST
 };
 
+enum brcmf_vndr_evts {
+	BRCMF_VNDR_EVTS_PHY_TEMP,
+	BRCMF_VNDR_EVTS_LAST
+};
+
 /**
  * enum brcmf_nlattrs - nl80211 message attributes
  *
@@ -36,11 +41,21 @@
 
 	BRCMF_NLATTR_LEN,
 	BRCMF_NLATTR_DATA,
+	BRCMF_NLATTR_VERS,
+	BRCMF_NLATTR_PHY_TEMP,
+	BRCMF_NLATTR_PHY_TEMPDELTA,
 
 	__BRCMF_NLATTR_AFTER_LAST,
 	BRCMF_NLATTR_MAX = __BRCMF_NLATTR_AFTER_LAST - 1
 };
 
+/* structure of event sent up by firmware: is this the right place for it? */
+struct brcmf_phy_temp_evt {
+	__le32 version;
+	__le32 temp;
+	__le32 tempdelta;
+} __packed;
+
 /**
  * struct brcmf_vndr_dcmd_hdr - message header for cfg80211 vendor command dcmd
  *				support
@@ -60,5 +75,9 @@
 };
 
 extern const struct wiphy_vendor_command brcmf_vendor_cmds[];
+extern const struct nl80211_vendor_cmd_info brcmf_vendor_events[];
+s32 brcmf_wiphy_phy_temp_evt_handler(struct brcmf_if *ifp,
+				     const struct brcmf_event_msg *e,
+				     void *data);
 
 #endif /* _vendor_h_ */
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/channel.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/channel.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/channel.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/channel.c	2020-06-19 21:00:31.420227716 -0400
@@ -652,6 +652,7 @@
 		 */
 		if (!(ch->flags & IEEE80211_CHAN_DISABLED))
 			ch->flags |= IEEE80211_CHAN_RADAR |
+				     IEEE80211_CHAN_NO_IR |
 				     IEEE80211_CHAN_NO_IR;
 	}
 }
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/debug.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/debug.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/debug.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/debug.c	2020-06-19 21:00:31.420227716 -0400
@@ -214,7 +214,7 @@
 	entry->read = read_fn;
 	entry->drvr = drvr;
 
-	dentry = debugfs_create_file(fn, 0444, dentry, entry,
+	dentry = debugfs_create_file(fn, S_IRUGO, dentry, entry,
 				     &brcms_debugfs_def_ops);
 
 	return PTR_ERR_OR_ZERO(dentry);
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/mac80211_if.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/mac80211_if.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/mac80211_if.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/mac80211_if.c	2020-06-19 21:00:31.420227716 -0400
@@ -108,7 +108,7 @@
  * flags are specified by the BRCM_DL_* macros in
  * drivers/net/wireless/brcm80211/include/defs.h.
  */
-module_param_named(debug, brcm_msg_level, uint, 0644);
+module_param_named(debug, brcm_msg_level, uint, S_IRUGO | S_IWUSR);
 #endif
 
 static struct ieee80211_channel brcms_2ghz_chantable[] = {
@@ -502,7 +502,6 @@
 	}
 
 	spin_lock_bh(&wl->lock);
-	wl->wlc->vif = vif;
 	wl->mute_tx = false;
 	brcms_c_mute(wl->wlc, false);
 	if (vif->type == NL80211_IFTYPE_STATION)
@@ -520,11 +519,6 @@
 static void
 brcms_ops_remove_interface(struct ieee80211_hw *hw, struct ieee80211_vif *vif)
 {
-	struct brcms_info *wl = hw->priv;
-
-	spin_lock_bh(&wl->lock);
-	wl->wlc->vif = NULL;
-	spin_unlock_bh(&wl->lock);
 }
 
 static int brcms_ops_config(struct ieee80211_hw *hw, u32 changed)
@@ -846,8 +840,8 @@
 		status = brcms_c_aggregatable(wl->wlc, tid);
 		spin_unlock_bh(&wl->lock);
 		if (!status) {
-			brcms_dbg_ht(wl->wlc->hw->d11core,
-				     "START: tid %d is not agg\'able\n", tid);
+			brcms_err(wl->wlc->hw->d11core,
+				  "START: tid %d is not agg\'able\n", tid);
 			return -EINVAL;
 		}
 		ieee80211_start_tx_ba_cb_irqsafe(vif, sta->addr, tid);
@@ -943,25 +937,6 @@
 	spin_unlock_bh(&wl->lock);
 }
 
-static int brcms_ops_beacon_set_tim(struct ieee80211_hw *hw,
-				 struct ieee80211_sta *sta, bool set)
-{
-	struct brcms_info *wl = hw->priv;
-	struct sk_buff *beacon = NULL;
-	u16 tim_offset = 0;
-
-	spin_lock_bh(&wl->lock);
-	if (wl->wlc->vif)
-		beacon = ieee80211_beacon_get_tim(hw, wl->wlc->vif,
-						  &tim_offset, NULL);
-	if (beacon)
-		brcms_c_set_new_beacon(wl->wlc, beacon, tim_offset,
-				       wl->wlc->vif->bss_conf.dtim_period);
-	spin_unlock_bh(&wl->lock);
-
-	return 0;
-}
-
 static const struct ieee80211_ops brcms_ops = {
 	.tx = brcms_ops_tx,
 	.start = brcms_ops_start,
@@ -980,7 +955,6 @@
 	.flush = brcms_ops_flush,
 	.get_tsf = brcms_ops_get_tsf,
 	.set_tsf = brcms_ops_set_tsf,
-	.set_tim = brcms_ops_beacon_set_tim,
 };
 
 void brcms_dpc(unsigned long data)
@@ -1589,7 +1563,7 @@
 }
 
 /*
- * precondition: no locking required
+ * precondition: perimeter lock has been acquired
  */
 int brcms_ucode_init_buf(struct brcms_info *wl, void **pbuf, u32 idx)
 {
@@ -1604,10 +1578,10 @@
 			if (le32_to_cpu(hdr->idx) == idx) {
 				pdata = wl->fw.fw_bin[i]->data +
 					le32_to_cpu(hdr->offset);
-				*pbuf = kvmalloc(len, GFP_KERNEL);
+				*pbuf = kmemdup(pdata, len, GFP_ATOMIC);
 				if (*pbuf == NULL)
 					goto fail;
-				memcpy(*pbuf, pdata, len);
+
 				return 0;
 			}
 		}
@@ -1655,7 +1629,7 @@
  */
 void brcms_ucode_free_buf(void *p)
 {
-	kvfree(p);
+	kfree(p);
 }
 
 /*
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/main.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/main.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/main.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/main.c	2020-06-19 21:00:31.420227716 -0400
@@ -507,7 +507,7 @@
 	wlc->hw->wlc = wlc;
 
 	wlc->hw->bandstate[0] =
-		kcalloc(MAXBANDS, sizeof(struct brcms_hw_band), GFP_ATOMIC);
+		kzalloc(sizeof(struct brcms_hw_band) * MAXBANDS, GFP_ATOMIC);
 	if (wlc->hw->bandstate[0] == NULL) {
 		*err = 1006;
 		goto fail;
@@ -521,8 +521,7 @@
 	}
 
 	wlc->modulecb =
-		kcalloc(BRCMS_MAXMODULES, sizeof(struct modulecb),
-			GFP_ATOMIC);
+		kzalloc(sizeof(struct modulecb) * BRCMS_MAXMODULES, GFP_ATOMIC);
 	if (wlc->modulecb == NULL) {
 		*err = 1009;
 		goto fail;
@@ -554,7 +553,7 @@
 	}
 
 	wlc->bandstate[0] =
-		kcalloc(MAXBANDS, sizeof(struct brcms_band), GFP_ATOMIC);
+		kzalloc(sizeof(struct brcms_band)*MAXBANDS, GFP_ATOMIC);
 	if (wlc->bandstate[0] == NULL) {
 		*err = 1025;
 		goto fail;
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/main.h b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/main.h
--- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/main.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/main.h	2020-06-19 21:00:31.420227716 -0400
@@ -563,7 +563,6 @@
 
 	struct wiphy *wiphy;
 	struct scb pri_scb;
-	struct ieee80211_vif *vif;
 
 	struct sk_buff *beacon;
 	u16 beacon_tim_offset;
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_cmn.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_cmn.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_cmn.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_cmn.c	2020-06-19 21:00:31.420227716 -0400
@@ -159,7 +159,7 @@
 {
 	u16 data;
 
-	if (addr == RADIO_IDCODE)
+	if ((addr == RADIO_IDCODE))
 		return 0xffff;
 
 	switch (pi->pubpi.phy_type) {
@@ -1916,7 +1916,7 @@
 				     pi->hwpwr_txcur);
 
 		for (j = TXP_FIRST_OFDM; j <= TXP_LAST_OFDM; j++) {
-			static const u8 ucode_ofdm_rates[] = {
+			const u8 ucode_ofdm_rates[] = {
 				0x0c, 0x12, 0x18, 0x24, 0x30, 0x48, 0x60, 0x6c
 			};
 			offset = wlapi_bmac_rate_shm_offset(
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_lcn.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_lcn.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_lcn.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_lcn.c	2020-06-19 21:00:31.420227716 -0400
@@ -1387,7 +1387,7 @@
 	s16 *ptr;
 	struct brcms_phy_lcnphy *pi_lcn = pi->u.pi_lcnphy;
 
-	ptr = kmalloc_array(131, sizeof(s16), GFP_ATOMIC);
+	ptr = kmalloc(sizeof(s16) * 131, GFP_ATOMIC);
 	if (NULL == ptr)
 		return false;
 	if (module == 2) {
@@ -2670,7 +2670,7 @@
 	u16 *values_to_save;
 	struct brcms_phy_lcnphy *pi_lcn = pi->u.pi_lcnphy;
 
-	values_to_save = kmalloc_array(20, sizeof(u16), GFP_ATOMIC);
+	values_to_save = kmalloc(sizeof(u16) * 20, GFP_ATOMIC);
 	if (NULL == values_to_save)
 		return;
 
@@ -3388,8 +3388,13 @@
 	u8 phybw40;
 	phybw40 = CHSPEC_IS40(pi->radio_chanspec);
 
-	mod_phy_reg(pi, 0x4b0, (0x1 << 5), (mode) << 5);
-	mod_phy_reg(pi, 0x4b1, (0x1 << 9), 0 << 9);
+	if (LCNREV_LT(pi->pubpi.phy_rev, 2)) {
+		mod_phy_reg(pi, 0x4b0, (0x1 << 5), (mode) << 5);
+		mod_phy_reg(pi, 0x4b1, (0x1 << 9), 0 << 9);
+	} else {
+		mod_phy_reg(pi, 0x4b0, (0x1 << 5), (mode) << 5);
+		mod_phy_reg(pi, 0x4b1, (0x1 << 9), 0 << 9);
+	}
 
 	if (phybw40 == 0) {
 		mod_phy_reg((pi), 0x410,
@@ -3678,11 +3683,11 @@
 	u16 *phy_c32;
 	phy_c21 = 0;
 	phy_c10 = phy_c13 = phy_c14 = phy_c8 = 0;
-	ptr = kmalloc_array(131, sizeof(s16), GFP_ATOMIC);
+	ptr = kmalloc(sizeof(s16) * 131, GFP_ATOMIC);
 	if (NULL == ptr)
 		return;
 
-	phy_c32 = kmalloc_array(20, sizeof(u16), GFP_ATOMIC);
+	phy_c32 = kmalloc(sizeof(u16) * 20, GFP_ATOMIC);
 	if (NULL == phy_c32) {
 		kfree(ptr);
 		return;
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_n.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_n.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_n.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_n.c	2020-06-19 21:00:31.420227716 -0400
@@ -16049,7 +16049,8 @@
 		wlc_phy_set_rfseq_nphy(pi, NPHY_RFSEQ_UPDATEGAINU,
 				       rfseq_updategainu_events,
 				       rfseq_updategainu_dlys,
-				       ARRAY_SIZE(rfseq_updategainu_events));
+				       sizeof(rfseq_updategainu_events) /
+				       sizeof(rfseq_updategainu_events[0]));
 
 		mod_phy_reg(pi, 0x153, (0xff << 8), (90 << 8));
 
@@ -16060,8 +16061,52 @@
 	}
 }
 
-static void wlc_phy_workarounds_nphy_rev7(struct brcms_phy *pi)
+static void wlc_phy_workarounds_nphy(struct brcms_phy *pi)
 {
+	static const u8 rfseq_rx2tx_events[] = {
+		NPHY_RFSEQ_CMD_NOP,
+		NPHY_RFSEQ_CMD_RXG_FBW,
+		NPHY_RFSEQ_CMD_TR_SWITCH,
+		NPHY_RFSEQ_CMD_CLR_HIQ_DIS,
+		NPHY_RFSEQ_CMD_RXPD_TXPD,
+		NPHY_RFSEQ_CMD_TX_GAIN,
+		NPHY_RFSEQ_CMD_EXT_PA
+	};
+	u8 rfseq_rx2tx_dlys[] = { 8, 6, 6, 2, 4, 60, 1 };
+	static const u8 rfseq_tx2rx_events[] = {
+		NPHY_RFSEQ_CMD_NOP,
+		NPHY_RFSEQ_CMD_EXT_PA,
+		NPHY_RFSEQ_CMD_TX_GAIN,
+		NPHY_RFSEQ_CMD_RXPD_TXPD,
+		NPHY_RFSEQ_CMD_TR_SWITCH,
+		NPHY_RFSEQ_CMD_RXG_FBW,
+		NPHY_RFSEQ_CMD_CLR_HIQ_DIS
+	};
+	static const u8 rfseq_tx2rx_dlys[] = { 8, 6, 2, 4, 4, 6, 1 };
+	static const u8 rfseq_tx2rx_events_rev3[] = {
+		NPHY_REV3_RFSEQ_CMD_EXT_PA,
+		NPHY_REV3_RFSEQ_CMD_INT_PA_PU,
+		NPHY_REV3_RFSEQ_CMD_TX_GAIN,
+		NPHY_REV3_RFSEQ_CMD_RXPD_TXPD,
+		NPHY_REV3_RFSEQ_CMD_TR_SWITCH,
+		NPHY_REV3_RFSEQ_CMD_RXG_FBW,
+		NPHY_REV3_RFSEQ_CMD_CLR_HIQ_DIS,
+		NPHY_REV3_RFSEQ_CMD_END
+	};
+	static const u8 rfseq_tx2rx_dlys_rev3[] = { 8, 4, 2, 2, 4, 4, 6, 1 };
+	u8 rfseq_rx2tx_events_rev3[] = {
+		NPHY_REV3_RFSEQ_CMD_NOP,
+		NPHY_REV3_RFSEQ_CMD_RXG_FBW,
+		NPHY_REV3_RFSEQ_CMD_TR_SWITCH,
+		NPHY_REV3_RFSEQ_CMD_CLR_HIQ_DIS,
+		NPHY_REV3_RFSEQ_CMD_RXPD_TXPD,
+		NPHY_REV3_RFSEQ_CMD_TX_GAIN,
+		NPHY_REV3_RFSEQ_CMD_INT_PA_PU,
+		NPHY_REV3_RFSEQ_CMD_EXT_PA,
+		NPHY_REV3_RFSEQ_CMD_END
+	};
+	u8 rfseq_rx2tx_dlys_rev3[] = { 8, 6, 6, 4, 4, 18, 42, 1, 1 };
+
 	static const u8 rfseq_rx2tx_events_rev3_ipa[] = {
 		NPHY_REV3_RFSEQ_CMD_NOP,
 		NPHY_REV3_RFSEQ_CMD_RXG_FBW,
@@ -16073,18 +16118,31 @@
 		NPHY_REV3_RFSEQ_CMD_INT_PA_PU,
 		NPHY_REV3_RFSEQ_CMD_END
 	};
-	static const u8 rfseq_rx2tx_dlys_rev3_ipa[] =
-		{ 8, 6, 6, 4, 4, 16, 43, 1, 1 };
+	static const u8 rfseq_rx2tx_dlys_rev3_ipa[] = { 8, 6, 6, 4, 4, 16, 43, 1, 1 };
 	static const u16 rfseq_rx2tx_dacbufpu_rev7[] = { 0x10f, 0x10f };
-	u32 leg_data_weights;
+
+	s16 alpha0, alpha1, alpha2;
+	s16 beta0, beta1, beta2;
+	u32 leg_data_weights, ht_data_weights, nss1_data_weights,
+	    stbc_data_weights;
 	u8 chan_freq_range = 0;
 	static const u16 dac_control = 0x0002;
 	u16 aux_adc_vmid_rev7_core0[] = { 0x8e, 0x96, 0x96, 0x96 };
 	u16 aux_adc_vmid_rev7_core1[] = { 0x8f, 0x9f, 0x9f, 0x96 };
+	u16 aux_adc_vmid_rev4[] = { 0xa2, 0xb4, 0xb4, 0x89 };
+	u16 aux_adc_vmid_rev3[] = { 0xa2, 0xb4, 0xb4, 0x89 };
+	u16 *aux_adc_vmid;
 	u16 aux_adc_gain_rev7[] = { 0x02, 0x02, 0x02, 0x02 };
+	u16 aux_adc_gain_rev4[] = { 0x02, 0x02, 0x02, 0x00 };
+	u16 aux_adc_gain_rev3[] = { 0x02, 0x02, 0x02, 0x00 };
+	u16 *aux_adc_gain;
+	static const u16 sk_adc_vmid[] = { 0xb4, 0xb4, 0xb4, 0x24 };
+	static const u16 sk_adc_gain[] = { 0x02, 0x02, 0x02, 0x02 };
 	s32 min_nvar_val = 0x18d;
 	s32 min_nvar_offset_6mbps = 20;
 	u8 pdetrange;
+	u8 triso;
+	u16 regval;
 	u16 afectrl_adc_ctrl1_rev7 = 0x20;
 	u16 afectrl_adc_ctrl2_rev7 = 0x0;
 	u16 rfseq_rx2tx_lpf_h_hpc_rev7 = 0x77;
@@ -16113,938 +16171,964 @@
 	u16 freq;
 	int coreNum;
 
+	if (CHSPEC_IS5G(pi->radio_chanspec))
+		wlc_phy_classifier_nphy(pi, NPHY_ClassifierCtrl_cck_en, 0);
+	else
+		wlc_phy_classifier_nphy(pi, NPHY_ClassifierCtrl_cck_en, 1);
 
-	if (NREV_IS(pi->pubpi.phy_rev, 7)) {
-		mod_phy_reg(pi, 0x221, (0x1 << 4), (1 << 4));
+	if (pi->phyhang_avoid)
+		wlc_phy_stay_in_carriersearch_nphy(pi, true);
 
-		mod_phy_reg(pi, 0x160, (0x7f << 0), (32 << 0));
-		mod_phy_reg(pi, 0x160, (0x7f << 8), (39 << 8));
-		mod_phy_reg(pi, 0x161, (0x7f << 0), (46 << 0));
-		mod_phy_reg(pi, 0x161, (0x7f << 8), (51 << 8));
-		mod_phy_reg(pi, 0x162, (0x7f << 0), (55 << 0));
-		mod_phy_reg(pi, 0x162, (0x7f << 8), (58 << 8));
-		mod_phy_reg(pi, 0x163, (0x7f << 0), (60 << 0));
-		mod_phy_reg(pi, 0x163, (0x7f << 8), (62 << 8));
-		mod_phy_reg(pi, 0x164, (0x7f << 0), (62 << 0));
-		mod_phy_reg(pi, 0x164, (0x7f << 8), (63 << 8));
-		mod_phy_reg(pi, 0x165, (0x7f << 0), (63 << 0));
-		mod_phy_reg(pi, 0x165, (0x7f << 8), (64 << 8));
-		mod_phy_reg(pi, 0x166, (0x7f << 0), (64 << 0));
-		mod_phy_reg(pi, 0x166, (0x7f << 8), (64 << 8));
-		mod_phy_reg(pi, 0x167, (0x7f << 0), (64 << 0));
-		mod_phy_reg(pi, 0x167, (0x7f << 8), (64 << 8));
-	}
-
-	if (NREV_LE(pi->pubpi.phy_rev, 8)) {
-		write_phy_reg(pi, 0x23f, 0x1b0);
-		write_phy_reg(pi, 0x240, 0x1b0);
-	}
-
-	if (NREV_GE(pi->pubpi.phy_rev, 8))
-		mod_phy_reg(pi, 0xbd, (0xff << 0), (114 << 0));
-
-	wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1, 0x00, 16,
-				 &dac_control);
-	wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1, 0x10, 16,
-				 &dac_control);
-
-	wlc_phy_table_read_nphy(pi, NPHY_TBL_ID_CMPMETRICDATAWEIGHTTBL,
-				1, 0, 32, &leg_data_weights);
-	leg_data_weights = leg_data_weights & 0xffffff;
-	wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_CMPMETRICDATAWEIGHTTBL,
-				 1, 0, 32, &leg_data_weights);
-
-	wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ,
-				 2, 0x15e, 16, rfseq_rx2tx_dacbufpu_rev7);
-	wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 2, 0x16e, 16,
-				 rfseq_rx2tx_dacbufpu_rev7);
+	or_phy_reg(pi, 0xb1, NPHY_IQFlip_ADC1 | NPHY_IQFlip_ADC2);
 
-	if (PHY_IPA(pi))
-		wlc_phy_set_rfseq_nphy(pi, NPHY_RFSEQ_RX2TX,
-				       rfseq_rx2tx_events_rev3_ipa,
-				       rfseq_rx2tx_dlys_rev3_ipa,
-				       ARRAY_SIZE
-				       (rfseq_rx2tx_events_rev3_ipa));
-
-	mod_phy_reg(pi, 0x299, (0x3 << 14), (0x1 << 14));
-	mod_phy_reg(pi, 0x29d, (0x3 << 14), (0x1 << 14));
-
-	tx_lpf_bw_ofdm_20mhz = wlc_phy_read_lpf_bw_ctl_nphy(pi, 0x154);
-	tx_lpf_bw_ofdm_40mhz = wlc_phy_read_lpf_bw_ctl_nphy(pi, 0x159);
-	tx_lpf_bw_11b = wlc_phy_read_lpf_bw_ctl_nphy(pi, 0x152);
+	if (NREV_GE(pi->pubpi.phy_rev, 7)) {
 
-	if (PHY_IPA(pi)) {
+		if (NREV_IS(pi->pubpi.phy_rev, 7)) {
+			mod_phy_reg(pi, 0x221, (0x1 << 4), (1 << 4));
 
-		if (((pi->pubpi.radiorev == 5)
-		     && (CHSPEC_IS40(pi->radio_chanspec) == 1))
-		    || (pi->pubpi.radiorev == 7)
-		    || (pi->pubpi.radiorev == 8)) {
-
-			rccal_bcap_val =
-			    read_radio_reg(pi, RADIO_2057_RCCAL_BCAP_VAL);
-			rccal_scap_val =
-			    read_radio_reg(pi, RADIO_2057_RCCAL_SCAP_VAL);
-
-			rccal_tx20_11b_bcap = rccal_bcap_val;
-			rccal_tx20_11b_scap = rccal_scap_val;
-
-			if ((pi->pubpi.radiorev == 5) &&
-			    (CHSPEC_IS40(pi->radio_chanspec) == 1)) {
-
-				rccal_tx20_11n_bcap = rccal_bcap_val;
-				rccal_tx20_11n_scap = rccal_scap_val;
-				rccal_tx40_11n_bcap = 0xc;
-				rccal_tx40_11n_scap = 0xc;
+			mod_phy_reg(pi, 0x160, (0x7f << 0), (32 << 0));
+			mod_phy_reg(pi, 0x160, (0x7f << 8), (39 << 8));
+			mod_phy_reg(pi, 0x161, (0x7f << 0), (46 << 0));
+			mod_phy_reg(pi, 0x161, (0x7f << 8), (51 << 8));
+			mod_phy_reg(pi, 0x162, (0x7f << 0), (55 << 0));
+			mod_phy_reg(pi, 0x162, (0x7f << 8), (58 << 8));
+			mod_phy_reg(pi, 0x163, (0x7f << 0), (60 << 0));
+			mod_phy_reg(pi, 0x163, (0x7f << 8), (62 << 8));
+			mod_phy_reg(pi, 0x164, (0x7f << 0), (62 << 0));
+			mod_phy_reg(pi, 0x164, (0x7f << 8), (63 << 8));
+			mod_phy_reg(pi, 0x165, (0x7f << 0), (63 << 0));
+			mod_phy_reg(pi, 0x165, (0x7f << 8), (64 << 8));
+			mod_phy_reg(pi, 0x166, (0x7f << 0), (64 << 0));
+			mod_phy_reg(pi, 0x166, (0x7f << 8), (64 << 8));
+			mod_phy_reg(pi, 0x167, (0x7f << 0), (64 << 0));
+			mod_phy_reg(pi, 0x167, (0x7f << 8), (64 << 8));
+		}
+
+		if (NREV_LE(pi->pubpi.phy_rev, 8)) {
+			write_phy_reg(pi, 0x23f, 0x1b0);
+			write_phy_reg(pi, 0x240, 0x1b0);
+		}
+
+		if (NREV_GE(pi->pubpi.phy_rev, 8))
+			mod_phy_reg(pi, 0xbd, (0xff << 0), (114 << 0));
+
+		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1, 0x00, 16,
+					 &dac_control);
+		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1, 0x10, 16,
+					 &dac_control);
+
+		wlc_phy_table_read_nphy(pi, NPHY_TBL_ID_CMPMETRICDATAWEIGHTTBL,
+					1, 0, 32, &leg_data_weights);
+		leg_data_weights = leg_data_weights & 0xffffff;
+		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_CMPMETRICDATAWEIGHTTBL,
+					 1, 0, 32, &leg_data_weights);
+
+		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ,
+					 2, 0x15e, 16,
+					 rfseq_rx2tx_dacbufpu_rev7);
+		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 2, 0x16e, 16,
+					 rfseq_rx2tx_dacbufpu_rev7);
+
+		if (PHY_IPA(pi))
+			wlc_phy_set_rfseq_nphy(pi, NPHY_RFSEQ_RX2TX,
+					       rfseq_rx2tx_events_rev3_ipa,
+					       rfseq_rx2tx_dlys_rev3_ipa,
+					       ARRAY_SIZE(rfseq_rx2tx_events_rev3_ipa));
+
+		mod_phy_reg(pi, 0x299, (0x3 << 14), (0x1 << 14));
+		mod_phy_reg(pi, 0x29d, (0x3 << 14), (0x1 << 14));
+
+		tx_lpf_bw_ofdm_20mhz = wlc_phy_read_lpf_bw_ctl_nphy(pi, 0x154);
+		tx_lpf_bw_ofdm_40mhz = wlc_phy_read_lpf_bw_ctl_nphy(pi, 0x159);
+		tx_lpf_bw_11b = wlc_phy_read_lpf_bw_ctl_nphy(pi, 0x152);
 
-				rccal_ovrd = true;
+		if (PHY_IPA(pi)) {
 
-			} else if ((pi->pubpi.radiorev == 7)
-				   || (pi->pubpi.radiorev == 8)) {
+			if (((pi->pubpi.radiorev == 5)
+			     && (CHSPEC_IS40(pi->radio_chanspec) == 1))
+			    || (pi->pubpi.radiorev == 7)
+			    || (pi->pubpi.radiorev == 8)) {
 
-				tx_lpf_bw_ofdm_20mhz = 4;
-				tx_lpf_bw_11b = 1;
+				rccal_bcap_val =
+					read_radio_reg(
+						pi,
+						RADIO_2057_RCCAL_BCAP_VAL);
+				rccal_scap_val =
+					read_radio_reg(
+						pi,
+						RADIO_2057_RCCAL_SCAP_VAL);
+
+				rccal_tx20_11b_bcap = rccal_bcap_val;
+				rccal_tx20_11b_scap = rccal_scap_val;
+
+				if ((pi->pubpi.radiorev == 5) &&
+				    (CHSPEC_IS40(pi->radio_chanspec) == 1)) {
+
+					rccal_tx20_11n_bcap = rccal_bcap_val;
+					rccal_tx20_11n_scap = rccal_scap_val;
+					rccal_tx40_11n_bcap = 0xc;
+					rccal_tx40_11n_scap = 0xc;
+
+					rccal_ovrd = true;
+
+				} else if ((pi->pubpi.radiorev == 7)
+					   || (pi->pubpi.radiorev == 8)) {
+
+					tx_lpf_bw_ofdm_20mhz = 4;
+					tx_lpf_bw_11b = 1;
+
+					if (CHSPEC_IS2G(pi->radio_chanspec)) {
+						rccal_tx20_11n_bcap = 0xc;
+						rccal_tx20_11n_scap = 0xc;
+						rccal_tx40_11n_bcap = 0xa;
+						rccal_tx40_11n_scap = 0xa;
+					} else {
+						rccal_tx20_11n_bcap = 0x14;
+						rccal_tx20_11n_scap = 0x14;
+						rccal_tx40_11n_bcap = 0xf;
+						rccal_tx40_11n_scap = 0xf;
+					}
 
-				if (CHSPEC_IS2G(pi->radio_chanspec)) {
-					rccal_tx20_11n_bcap = 0xc;
-					rccal_tx20_11n_scap = 0xc;
-					rccal_tx40_11n_bcap = 0xa;
-					rccal_tx40_11n_scap = 0xa;
-				} else {
-					rccal_tx20_11n_bcap = 0x14;
-					rccal_tx20_11n_scap = 0x14;
-					rccal_tx40_11n_bcap = 0xf;
-					rccal_tx40_11n_scap = 0xf;
+					rccal_ovrd = true;
 				}
-
-				rccal_ovrd = true;
 			}
-		}
 
-	} else {
-
-		if (pi->pubpi.radiorev == 5) {
-
-			tx_lpf_bw_ofdm_20mhz = 1;
-			tx_lpf_bw_ofdm_40mhz = 3;
+		} else {
 
-			rccal_bcap_val =
-			    read_radio_reg(pi, RADIO_2057_RCCAL_BCAP_VAL);
-			rccal_scap_val =
-			    read_radio_reg(pi, RADIO_2057_RCCAL_SCAP_VAL);
+			if (pi->pubpi.radiorev == 5) {
 
-			rccal_tx20_11b_bcap = rccal_bcap_val;
-			rccal_tx20_11b_scap = rccal_scap_val;
+				tx_lpf_bw_ofdm_20mhz = 1;
+				tx_lpf_bw_ofdm_40mhz = 3;
 
-			rccal_tx20_11n_bcap = 0x13;
-			rccal_tx20_11n_scap = 0x11;
-			rccal_tx40_11n_bcap = 0x13;
-			rccal_tx40_11n_scap = 0x11;
+				rccal_bcap_val =
+					read_radio_reg(
+						pi,
+						RADIO_2057_RCCAL_BCAP_VAL);
+				rccal_scap_val =
+					read_radio_reg(
+						pi,
+						RADIO_2057_RCCAL_SCAP_VAL);
+
+				rccal_tx20_11b_bcap = rccal_bcap_val;
+				rccal_tx20_11b_scap = rccal_scap_val;
+
+				rccal_tx20_11n_bcap = 0x13;
+				rccal_tx20_11n_scap = 0x11;
+				rccal_tx40_11n_bcap = 0x13;
+				rccal_tx40_11n_scap = 0x11;
 
-			rccal_ovrd = true;
+				rccal_ovrd = true;
+			}
 		}
-	}
-
-	if (rccal_ovrd) {
 
-		rx2tx_lpf_rc_lut_tx20_11b =
-		    (rccal_tx20_11b_bcap << 8) |
-		    (rccal_tx20_11b_scap << 3) | tx_lpf_bw_11b;
-		rx2tx_lpf_rc_lut_tx20_11n =
-		    (rccal_tx20_11n_bcap << 8) |
-		    (rccal_tx20_11n_scap << 3) | tx_lpf_bw_ofdm_20mhz;
-		rx2tx_lpf_rc_lut_tx40_11n =
-		    (rccal_tx40_11n_bcap << 8) |
-		    (rccal_tx40_11n_scap << 3) | tx_lpf_bw_ofdm_40mhz;
+		if (rccal_ovrd) {
 
-		for (coreNum = 0; coreNum <= 1; coreNum++) {
-			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 1,
-						 0x152 + coreNum * 0x10, 16,
-						 &rx2tx_lpf_rc_lut_tx20_11b);
-			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 1,
-						 0x153 + coreNum * 0x10, 16,
-						 &rx2tx_lpf_rc_lut_tx20_11n);
-			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 1,
-						 0x154 + coreNum * 0x10, 16,
-						 &rx2tx_lpf_rc_lut_tx20_11n);
-			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 1,
-						 0x155 + coreNum * 0x10, 16,
-						 &rx2tx_lpf_rc_lut_tx40_11n);
-			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 1,
-						 0x156 + coreNum * 0x10, 16,
-						 &rx2tx_lpf_rc_lut_tx40_11n);
-			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 1,
-						 0x157 + coreNum * 0x10, 16,
-						 &rx2tx_lpf_rc_lut_tx40_11n);
-			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 1,
-						 0x158 + coreNum * 0x10, 16,
-						 &rx2tx_lpf_rc_lut_tx40_11n);
-			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 1,
-						 0x159 + coreNum * 0x10, 16,
-						 &rx2tx_lpf_rc_lut_tx40_11n);
-		}
+			rx2tx_lpf_rc_lut_tx20_11b =
+				(rccal_tx20_11b_bcap << 8) |
+				(rccal_tx20_11b_scap << 3) |
+				tx_lpf_bw_11b;
+			rx2tx_lpf_rc_lut_tx20_11n =
+				(rccal_tx20_11n_bcap << 8) |
+				(rccal_tx20_11n_scap << 3) |
+				tx_lpf_bw_ofdm_20mhz;
+			rx2tx_lpf_rc_lut_tx40_11n =
+				(rccal_tx40_11n_bcap << 8) |
+				(rccal_tx40_11n_scap << 3) |
+				tx_lpf_bw_ofdm_40mhz;
 
-		wlc_phy_rfctrl_override_nphy_rev7(pi, (0x1 << 4), 1, 0x3, 0,
-						  NPHY_REV7_RFCTRLOVERRIDE_ID2);
-	}
+			for (coreNum = 0; coreNum <= 1; coreNum++) {
+				wlc_phy_table_write_nphy(
+					pi, NPHY_TBL_ID_RFSEQ,
+					1,
+					0x152 + coreNum * 0x10,
+					16,
+					&rx2tx_lpf_rc_lut_tx20_11b);
+				wlc_phy_table_write_nphy(
+					pi, NPHY_TBL_ID_RFSEQ,
+					1,
+					0x153 + coreNum * 0x10,
+					16,
+					&rx2tx_lpf_rc_lut_tx20_11n);
+				wlc_phy_table_write_nphy(
+					pi, NPHY_TBL_ID_RFSEQ,
+					1,
+					0x154 + coreNum * 0x10,
+					16,
+					&rx2tx_lpf_rc_lut_tx20_11n);
+				wlc_phy_table_write_nphy(
+					pi, NPHY_TBL_ID_RFSEQ,
+					1,
+					0x155 + coreNum * 0x10,
+					16,
+					&rx2tx_lpf_rc_lut_tx40_11n);
+				wlc_phy_table_write_nphy(
+					pi, NPHY_TBL_ID_RFSEQ,
+					1,
+					0x156 + coreNum * 0x10,
+					16,
+					&rx2tx_lpf_rc_lut_tx40_11n);
+				wlc_phy_table_write_nphy(
+					pi, NPHY_TBL_ID_RFSEQ,
+					1,
+					0x157 + coreNum * 0x10,
+					16,
+					&rx2tx_lpf_rc_lut_tx40_11n);
+				wlc_phy_table_write_nphy(
+					pi, NPHY_TBL_ID_RFSEQ,
+					1,
+					0x158 + coreNum * 0x10,
+					16,
+					&rx2tx_lpf_rc_lut_tx40_11n);
+				wlc_phy_table_write_nphy(
+					pi, NPHY_TBL_ID_RFSEQ,
+					1,
+					0x159 + coreNum * 0x10,
+					16,
+					&rx2tx_lpf_rc_lut_tx40_11n);
+			}
 
-	write_phy_reg(pi, 0x32f, 0x3);
+			wlc_phy_rfctrl_override_nphy_rev7(
+				pi, (0x1 << 4),
+				1, 0x3, 0,
+				NPHY_REV7_RFCTRLOVERRIDE_ID2);
+		}
 
-	if ((pi->pubpi.radiorev == 4) || (pi->pubpi.radiorev == 6))
-		wlc_phy_rfctrl_override_nphy_rev7(pi, (0x1 << 2), 1, 0x3, 0,
-						  NPHY_REV7_RFCTRLOVERRIDE_ID0);
+		write_phy_reg(pi, 0x32f, 0x3);
 
-	if ((pi->pubpi.radiorev == 3) || (pi->pubpi.radiorev == 4) ||
-	    (pi->pubpi.radiorev == 6)) {
-		if ((pi->sh->sromrev >= 8)
-		    && (pi->sh->boardflags2 & BFL2_IPALVLSHIFT_3P3))
-			ipalvlshift_3p3_war_en = 1;
+		if ((pi->pubpi.radiorev == 4) || (pi->pubpi.radiorev == 6))
+			wlc_phy_rfctrl_override_nphy_rev7(
+				pi, (0x1 << 2),
+				1, 0x3, 0,
+				NPHY_REV7_RFCTRLOVERRIDE_ID0);
 
-		if (ipalvlshift_3p3_war_en) {
-			write_radio_reg(pi, RADIO_2057_GPAIO_CONFIG, 0x5);
-			write_radio_reg(pi, RADIO_2057_GPAIO_SEL1, 0x30);
-			write_radio_reg(pi, RADIO_2057_GPAIO_SEL0, 0x0);
-			or_radio_reg(pi, RADIO_2057_RXTXBIAS_CONFIG_CORE0, 0x1);
-			or_radio_reg(pi, RADIO_2057_RXTXBIAS_CONFIG_CORE1, 0x1);
+		if ((pi->pubpi.radiorev == 3) || (pi->pubpi.radiorev == 4) ||
+		    (pi->pubpi.radiorev == 6)) {
+			if ((pi->sh->sromrev >= 8)
+			    && (pi->sh->boardflags2 & BFL2_IPALVLSHIFT_3P3))
+				ipalvlshift_3p3_war_en = 1;
+
+			if (ipalvlshift_3p3_war_en) {
+				write_radio_reg(pi, RADIO_2057_GPAIO_CONFIG,
+						0x5);
+				write_radio_reg(pi, RADIO_2057_GPAIO_SEL1,
+						0x30);
+				write_radio_reg(pi, RADIO_2057_GPAIO_SEL0, 0x0);
+				or_radio_reg(pi,
+					     RADIO_2057_RXTXBIAS_CONFIG_CORE0,
+					     0x1);
+				or_radio_reg(pi,
+					     RADIO_2057_RXTXBIAS_CONFIG_CORE1,
+					     0x1);
 
-			ipa2g_mainbias = 0x1f;
+				ipa2g_mainbias = 0x1f;
 
-			ipa2g_casconv = 0x6f;
+				ipa2g_casconv = 0x6f;
 
-			ipa2g_biasfilt = 0xaa;
-		} else {
+				ipa2g_biasfilt = 0xaa;
+			} else {
 
-			ipa2g_mainbias = 0x2b;
+				ipa2g_mainbias = 0x2b;
 
-			ipa2g_casconv = 0x7f;
+				ipa2g_casconv = 0x7f;
 
-			ipa2g_biasfilt = 0xee;
-		}
+				ipa2g_biasfilt = 0xee;
+			}
 
-		if (CHSPEC_IS2G(pi->radio_chanspec)) {
-			for (coreNum = 0; coreNum <= 1; coreNum++) {
-				WRITE_RADIO_REG4(pi, RADIO_2057, CORE,
-						 coreNum, IPA2G_IMAIN,
-						 ipa2g_mainbias);
-				WRITE_RADIO_REG4(pi, RADIO_2057, CORE,
-						 coreNum, IPA2G_CASCONV,
-						 ipa2g_casconv);
-				WRITE_RADIO_REG4(pi, RADIO_2057, CORE,
-						 coreNum,
-						 IPA2G_BIAS_FILTER,
-						 ipa2g_biasfilt);
+			if (CHSPEC_IS2G(pi->radio_chanspec)) {
+				for (coreNum = 0; coreNum <= 1; coreNum++) {
+					WRITE_RADIO_REG4(pi, RADIO_2057, CORE,
+							 coreNum, IPA2G_IMAIN,
+							 ipa2g_mainbias);
+					WRITE_RADIO_REG4(pi, RADIO_2057, CORE,
+							 coreNum, IPA2G_CASCONV,
+							 ipa2g_casconv);
+					WRITE_RADIO_REG4(pi, RADIO_2057, CORE,
+							 coreNum,
+							 IPA2G_BIAS_FILTER,
+							 ipa2g_biasfilt);
+				}
 			}
 		}
-	}
 
-	if (PHY_IPA(pi)) {
-		if (CHSPEC_IS2G(pi->radio_chanspec)) {
-			if ((pi->pubpi.radiorev == 3)
-			    || (pi->pubpi.radiorev == 4)
-			    || (pi->pubpi.radiorev == 6))
-				txgm_idac_bleed = 0x7f;
-
-			for (coreNum = 0; coreNum <= 1; coreNum++) {
-				if (txgm_idac_bleed != 0)
-					WRITE_RADIO_REG4(pi, RADIO_2057,
-							 CORE, coreNum,
-							 TXGM_IDAC_BLEED,
-							 txgm_idac_bleed);
-			}
+		if (PHY_IPA(pi)) {
+			if (CHSPEC_IS2G(pi->radio_chanspec)) {
+				if ((pi->pubpi.radiorev == 3)
+				    || (pi->pubpi.radiorev == 4)
+				    || (pi->pubpi.radiorev == 6))
+					txgm_idac_bleed = 0x7f;
 
-			if (pi->pubpi.radiorev == 5) {
 				for (coreNum = 0; coreNum <= 1; coreNum++) {
-					WRITE_RADIO_REG4(pi, RADIO_2057,
-							 CORE, coreNum,
-							 IPA2G_CASCONV,
-							 0x13);
-					WRITE_RADIO_REG4(pi, RADIO_2057,
-							 CORE, coreNum,
-							 IPA2G_IMAIN,
-							 0x1f);
-					WRITE_RADIO_REG4(pi, RADIO_2057,
-							 CORE, coreNum,
-							 IPA2G_BIAS_FILTER,
-							 0xee);
-					WRITE_RADIO_REG4(pi, RADIO_2057,
-							 CORE, coreNum,
-							 PAD2G_IDACS,
-							 0x8a);
-					WRITE_RADIO_REG4(pi, RADIO_2057,
-							 CORE, coreNum,
-							 PAD_BIAS_FILTER_BWS,
-							 0x3e);
+					if (txgm_idac_bleed != 0)
+						WRITE_RADIO_REG4(
+							pi, RADIO_2057,
+							CORE, coreNum,
+							TXGM_IDAC_BLEED,
+							txgm_idac_bleed);
 				}
-			} else if ((pi->pubpi.radiorev == 7) ||
-				   (pi->pubpi.radiorev == 8)) {
 
-				if (CHSPEC_IS40(pi->radio_chanspec) == 0) {
-					WRITE_RADIO_REG4(pi, RADIO_2057, CORE,
-							 0, IPA2G_IMAIN, 0x14);
-					WRITE_RADIO_REG4(pi, RADIO_2057, CORE,
-							 1, IPA2G_IMAIN, 0x12);
-				} else {
+				if (pi->pubpi.radiorev == 5) {
+
+					for (coreNum = 0; coreNum <= 1;
+					     coreNum++) {
+						WRITE_RADIO_REG4(pi, RADIO_2057,
+								 CORE, coreNum,
+								 IPA2G_CASCONV,
+								 0x13);
+						WRITE_RADIO_REG4(pi, RADIO_2057,
+								 CORE, coreNum,
+								 IPA2G_IMAIN,
+								 0x1f);
+						WRITE_RADIO_REG4(
+							pi, RADIO_2057,
+							CORE, coreNum,
+							IPA2G_BIAS_FILTER,
+							0xee);
+						WRITE_RADIO_REG4(pi, RADIO_2057,
+								 CORE, coreNum,
+								 PAD2G_IDACS,
+								 0x8a);
+						WRITE_RADIO_REG4(
+							pi, RADIO_2057,
+							CORE, coreNum,
+							PAD_BIAS_FILTER_BWS,
+							0x3e);
+					}
+
+				} else if ((pi->pubpi.radiorev == 7)
+					   || (pi->pubpi.radiorev == 8)) {
+
+					if (CHSPEC_IS40(pi->radio_chanspec) ==
+					    0) {
+						WRITE_RADIO_REG4(pi, RADIO_2057,
+								 CORE, 0,
+								 IPA2G_IMAIN,
+								 0x14);
+						WRITE_RADIO_REG4(pi, RADIO_2057,
+								 CORE, 1,
+								 IPA2G_IMAIN,
+								 0x12);
+					} else {
+						WRITE_RADIO_REG4(pi, RADIO_2057,
+								 CORE, 0,
+								 IPA2G_IMAIN,
+								 0x16);
+						WRITE_RADIO_REG4(pi, RADIO_2057,
+								 CORE, 1,
+								 IPA2G_IMAIN,
+								 0x16);
+					}
+				}
+
+			} else {
+				freq = CHAN5G_FREQ(CHSPEC_CHANNEL(
+							pi->radio_chanspec));
+				if (((freq >= 5180) && (freq <= 5230))
+				    || ((freq >= 5745) && (freq <= 5805))) {
 					WRITE_RADIO_REG4(pi, RADIO_2057, CORE,
-							 0, IPA2G_IMAIN, 0x16);
+							 0, IPA5G_BIAS_FILTER,
+							 0xff);
 					WRITE_RADIO_REG4(pi, RADIO_2057, CORE,
-							 1, IPA2G_IMAIN, 0x16);
+							 1, IPA5G_BIAS_FILTER,
+							 0xff);
 				}
 			}
-
 		} else {
-			freq =
-			    CHAN5G_FREQ(CHSPEC_CHANNEL
-					(pi->radio_chanspec));
-			if (((freq >= 5180) && (freq <= 5230))
-			    || ((freq >= 5745) && (freq <= 5805))) {
-				WRITE_RADIO_REG4(pi, RADIO_2057, CORE,
-						 0, IPA5G_BIAS_FILTER, 0xff);
-				WRITE_RADIO_REG4(pi, RADIO_2057, CORE,
-						 1, IPA5G_BIAS_FILTER, 0xff);
+
+			if (pi->pubpi.radiorev != 5) {
+				for (coreNum = 0; coreNum <= 1; coreNum++) {
+					WRITE_RADIO_REG4(pi, RADIO_2057, CORE,
+							 coreNum,
+							 TXMIX2G_TUNE_BOOST_PU,
+							 0x61);
+					WRITE_RADIO_REG4(pi, RADIO_2057, CORE,
+							 coreNum,
+							 TXGM_IDAC_BLEED, 0x70);
+				}
 			}
 		}
-	} else {
 
-		if (pi->pubpi.radiorev != 5) {
+		if (pi->pubpi.radiorev == 4) {
+			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1,
+						 0x05, 16,
+						 &afectrl_adc_ctrl1_rev7);
+			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1,
+						 0x15, 16,
+						 &afectrl_adc_ctrl1_rev7);
+
 			for (coreNum = 0; coreNum <= 1; coreNum++) {
 				WRITE_RADIO_REG4(pi, RADIO_2057, CORE, coreNum,
-						 TXMIX2G_TUNE_BOOST_PU, 0x61);
+						 AFE_VCM_CAL_MASTER, 0x0);
+				WRITE_RADIO_REG4(pi, RADIO_2057, CORE, coreNum,
+						 AFE_SET_VCM_I, 0x3f);
 				WRITE_RADIO_REG4(pi, RADIO_2057, CORE, coreNum,
-						 TXGM_IDAC_BLEED, 0x70);
+						 AFE_SET_VCM_Q, 0x3f);
 			}
+		} else {
+			mod_phy_reg(pi, 0xa6, (0x1 << 2), (0x1 << 2));
+			mod_phy_reg(pi, 0x8f, (0x1 << 2), (0x1 << 2));
+			mod_phy_reg(pi, 0xa7, (0x1 << 2), (0x1 << 2));
+			mod_phy_reg(pi, 0xa5, (0x1 << 2), (0x1 << 2));
+
+			mod_phy_reg(pi, 0xa6, (0x1 << 0), 0);
+			mod_phy_reg(pi, 0x8f, (0x1 << 0), (0x1 << 0));
+			mod_phy_reg(pi, 0xa7, (0x1 << 0), 0);
+			mod_phy_reg(pi, 0xa5, (0x1 << 0), (0x1 << 0));
+
+			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1,
+						 0x05, 16,
+						 &afectrl_adc_ctrl2_rev7);
+			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1,
+						 0x15, 16,
+						 &afectrl_adc_ctrl2_rev7);
+
+			mod_phy_reg(pi, 0xa6, (0x1 << 2), 0);
+			mod_phy_reg(pi, 0x8f, (0x1 << 2), 0);
+			mod_phy_reg(pi, 0xa7, (0x1 << 2), 0);
+			mod_phy_reg(pi, 0xa5, (0x1 << 2), 0);
 		}
-	}
-
-	if (pi->pubpi.radiorev == 4) {
-		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1, 0x05, 16,
-					 &afectrl_adc_ctrl1_rev7);
-		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1, 0x15, 16,
-					 &afectrl_adc_ctrl1_rev7);
-
-		for (coreNum = 0; coreNum <= 1; coreNum++) {
-			WRITE_RADIO_REG4(pi, RADIO_2057, CORE, coreNum,
-					 AFE_VCM_CAL_MASTER, 0x0);
-			WRITE_RADIO_REG4(pi, RADIO_2057, CORE, coreNum,
-					 AFE_SET_VCM_I, 0x3f);
-			WRITE_RADIO_REG4(pi, RADIO_2057, CORE, coreNum,
-					 AFE_SET_VCM_Q, 0x3f);
-		}
-	} else {
-		mod_phy_reg(pi, 0xa6, (0x1 << 2), (0x1 << 2));
-		mod_phy_reg(pi, 0x8f, (0x1 << 2), (0x1 << 2));
-		mod_phy_reg(pi, 0xa7, (0x1 << 2), (0x1 << 2));
-		mod_phy_reg(pi, 0xa5, (0x1 << 2), (0x1 << 2));
-
-		mod_phy_reg(pi, 0xa6, (0x1 << 0), 0);
-		mod_phy_reg(pi, 0x8f, (0x1 << 0), (0x1 << 0));
-		mod_phy_reg(pi, 0xa7, (0x1 << 0), 0);
-		mod_phy_reg(pi, 0xa5, (0x1 << 0), (0x1 << 0));
-
-		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1, 0x05, 16,
-					 &afectrl_adc_ctrl2_rev7);
-		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1, 0x15, 16,
-					 &afectrl_adc_ctrl2_rev7);
-
-		mod_phy_reg(pi, 0xa6, (0x1 << 2), 0);
-		mod_phy_reg(pi, 0x8f, (0x1 << 2), 0);
-		mod_phy_reg(pi, 0xa7, (0x1 << 2), 0);
-		mod_phy_reg(pi, 0xa5, (0x1 << 2), 0);
-	}
 
-	write_phy_reg(pi, 0x6a, 0x2);
+		write_phy_reg(pi, 0x6a, 0x2);
 
-	wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1, 256, 32,
-				 &min_nvar_offset_6mbps);
+		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1, 256, 32,
+					 &min_nvar_offset_6mbps);
 
-	wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 2, 0x138, 16,
-				 &rfseq_pktgn_lpf_hpc_rev7);
+		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 2, 0x138, 16,
+					 &rfseq_pktgn_lpf_hpc_rev7);
 
-	wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 1, 0x141, 16,
-				 &rfseq_pktgn_lpf_h_hpc_rev7);
+		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 1, 0x141, 16,
+					 &rfseq_pktgn_lpf_h_hpc_rev7);
 
-	wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 3, 0x133, 16,
-				 &rfseq_htpktgn_lpf_hpc_rev7);
+		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 3, 0x133, 16,
+					 &rfseq_htpktgn_lpf_hpc_rev7);
 
-	wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 2, 0x146, 16,
-				 &rfseq_cckpktgn_lpf_hpc_rev7);
+		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 2, 0x146, 16,
+					 &rfseq_cckpktgn_lpf_hpc_rev7);
 
-	wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 1, 0x123, 16,
-				 &rfseq_tx2rx_lpf_h_hpc_rev7);
+		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 1, 0x123, 16,
+					 &rfseq_tx2rx_lpf_h_hpc_rev7);
 
-	wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 1, 0x12A, 16,
-				 &rfseq_rx2tx_lpf_h_hpc_rev7);
+		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 1, 0x12A, 16,
+					 &rfseq_rx2tx_lpf_h_hpc_rev7);
 
-	if (CHSPEC_IS40(pi->radio_chanspec) == 0) {
-		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1, 3,
-					 32, &min_nvar_val);
-		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1,
-					 127, 32, &min_nvar_val);
-	} else {
-		min_nvar_val = noise_var_tbl_rev7[3];
-		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1, 3,
-					 32, &min_nvar_val);
+		if (CHSPEC_IS40(pi->radio_chanspec) == 0) {
+			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1, 3,
+						 32, &min_nvar_val);
+			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1,
+						 127, 32, &min_nvar_val);
+		} else {
+			min_nvar_val = noise_var_tbl_rev7[3];
+			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1, 3,
+						 32, &min_nvar_val);
 
-		min_nvar_val = noise_var_tbl_rev7[127];
-		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1,
-					 127, 32, &min_nvar_val);
-	}
+			min_nvar_val = noise_var_tbl_rev7[127];
+			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1,
+						 127, 32, &min_nvar_val);
+		}
 
-	wlc_phy_workarounds_nphy_gainctrl(pi);
+		wlc_phy_workarounds_nphy_gainctrl(pi);
 
-	pdetrange = (CHSPEC_IS5G(pi->radio_chanspec)) ?
-		    pi->srom_fem5g.pdetrange : pi->srom_fem2g.pdetrange;
+		pdetrange =
+			(CHSPEC_IS5G(pi->radio_chanspec)) ? pi->srom_fem5g.
+			pdetrange : pi->srom_fem2g.pdetrange;
 
-	if (pdetrange == 0) {
-		chan_freq_range = wlc_phy_get_chan_freq_range_nphy(pi, 0);
-		if (chan_freq_range != WL_CHAN_FREQ_RANGE_2G) {
-			aux_adc_vmid_rev7_core0[3] = 0x70;
-			aux_adc_vmid_rev7_core1[3] = 0x70;
-			aux_adc_gain_rev7[3] = 2;
-		} else {
-			aux_adc_vmid_rev7_core0[3] = 0x80;
-			aux_adc_vmid_rev7_core1[3] = 0x80;
-			aux_adc_gain_rev7[3] = 3;
-		}
-	} else if (pdetrange == 1) {
-		if (chan_freq_range != WL_CHAN_FREQ_RANGE_2G) {
-			aux_adc_vmid_rev7_core0[3] = 0x7c;
-			aux_adc_vmid_rev7_core1[3] = 0x7c;
-			aux_adc_gain_rev7[3] = 2;
-		} else {
-			aux_adc_vmid_rev7_core0[3] = 0x8c;
-			aux_adc_vmid_rev7_core1[3] = 0x8c;
-			aux_adc_gain_rev7[3] = 1;
-		}
-	} else if (pdetrange == 2) {
-		if (pi->pubpi.radioid == BCM2057_ID) {
-			if ((pi->pubpi.radiorev == 5)
-			    || (pi->pubpi.radiorev == 7)
-			    || (pi->pubpi.radiorev == 8)) {
-				if (chan_freq_range ==
-				    WL_CHAN_FREQ_RANGE_2G) {
-					aux_adc_vmid_rev7_core0[3] = 0x8c;
-					aux_adc_vmid_rev7_core1[3] = 0x8c;
-					aux_adc_gain_rev7[3] = 0;
-				} else {
-					aux_adc_vmid_rev7_core0[3] = 0x96;
-					aux_adc_vmid_rev7_core1[3] = 0x96;
-					aux_adc_gain_rev7[3] = 0;
+		if (pdetrange == 0) {
+			chan_freq_range =
+				wlc_phy_get_chan_freq_range_nphy(pi, 0);
+			if (chan_freq_range != WL_CHAN_FREQ_RANGE_2G) {
+				aux_adc_vmid_rev7_core0[3] = 0x70;
+				aux_adc_vmid_rev7_core1[3] = 0x70;
+				aux_adc_gain_rev7[3] = 2;
+			} else {
+				aux_adc_vmid_rev7_core0[3] = 0x80;
+				aux_adc_vmid_rev7_core1[3] = 0x80;
+				aux_adc_gain_rev7[3] = 3;
+			}
+		} else if (pdetrange == 1) {
+			if (chan_freq_range != WL_CHAN_FREQ_RANGE_2G) {
+				aux_adc_vmid_rev7_core0[3] = 0x7c;
+				aux_adc_vmid_rev7_core1[3] = 0x7c;
+				aux_adc_gain_rev7[3] = 2;
+			} else {
+				aux_adc_vmid_rev7_core0[3] = 0x8c;
+				aux_adc_vmid_rev7_core1[3] = 0x8c;
+				aux_adc_gain_rev7[3] = 1;
+			}
+		} else if (pdetrange == 2) {
+			if (pi->pubpi.radioid == BCM2057_ID) {
+				if ((pi->pubpi.radiorev == 5)
+				    || (pi->pubpi.radiorev == 7)
+				    || (pi->pubpi.radiorev == 8)) {
+					if (chan_freq_range ==
+					    WL_CHAN_FREQ_RANGE_2G) {
+						aux_adc_vmid_rev7_core0[3] =
+							0x8c;
+						aux_adc_vmid_rev7_core1[3] =
+							0x8c;
+						aux_adc_gain_rev7[3] = 0;
+					} else {
+						aux_adc_vmid_rev7_core0[3] =
+							0x96;
+						aux_adc_vmid_rev7_core1[3] =
+							0x96;
+						aux_adc_gain_rev7[3] = 0;
+					}
 				}
 			}
-		}
 
-	} else if (pdetrange == 3) {
-		if (chan_freq_range == WL_CHAN_FREQ_RANGE_2G) {
-			aux_adc_vmid_rev7_core0[3] = 0x89;
-			aux_adc_vmid_rev7_core1[3] = 0x89;
-			aux_adc_gain_rev7[3] = 0;
-		}
+		} else if (pdetrange == 3) {
+			if (chan_freq_range == WL_CHAN_FREQ_RANGE_2G) {
+				aux_adc_vmid_rev7_core0[3] = 0x89;
+				aux_adc_vmid_rev7_core1[3] = 0x89;
+				aux_adc_gain_rev7[3] = 0;
+			}
 
-	} else if (pdetrange == 5) {
+		} else if (pdetrange == 5) {
 
-		if (chan_freq_range != WL_CHAN_FREQ_RANGE_2G) {
-			aux_adc_vmid_rev7_core0[3] = 0x80;
-			aux_adc_vmid_rev7_core1[3] = 0x80;
-			aux_adc_gain_rev7[3] = 3;
-		} else {
-			aux_adc_vmid_rev7_core0[3] = 0x70;
-			aux_adc_vmid_rev7_core1[3] = 0x70;
-			aux_adc_gain_rev7[3] = 2;
+			if (chan_freq_range != WL_CHAN_FREQ_RANGE_2G) {
+				aux_adc_vmid_rev7_core0[3] = 0x80;
+				aux_adc_vmid_rev7_core1[3] = 0x80;
+				aux_adc_gain_rev7[3] = 3;
+			} else {
+				aux_adc_vmid_rev7_core0[3] = 0x70;
+				aux_adc_vmid_rev7_core1[3] = 0x70;
+				aux_adc_gain_rev7[3] = 2;
+			}
 		}
-	}
 
-	wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, 0x08, 16,
-				 &aux_adc_vmid_rev7_core0);
-	wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, 0x18, 16,
-				 &aux_adc_vmid_rev7_core1);
-	wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, 0x0c, 16,
-				 &aux_adc_gain_rev7);
-	wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, 0x1c, 16,
-				 &aux_adc_gain_rev7);
-}
+		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, 0x08, 16,
+					 &aux_adc_vmid_rev7_core0);
+		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, 0x18, 16,
+					 &aux_adc_vmid_rev7_core1);
+		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, 0x0c, 16,
+					 &aux_adc_gain_rev7);
+		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, 0x1c, 16,
+					 &aux_adc_gain_rev7);
 
-static void wlc_phy_workarounds_nphy_rev3(struct brcms_phy *pi)
-{
-	static const u8 rfseq_tx2rx_events_rev3[] = {
-		NPHY_REV3_RFSEQ_CMD_EXT_PA,
-		NPHY_REV3_RFSEQ_CMD_INT_PA_PU,
-		NPHY_REV3_RFSEQ_CMD_TX_GAIN,
-		NPHY_REV3_RFSEQ_CMD_RXPD_TXPD,
-		NPHY_REV3_RFSEQ_CMD_TR_SWITCH,
-		NPHY_REV3_RFSEQ_CMD_RXG_FBW,
-		NPHY_REV3_RFSEQ_CMD_CLR_HIQ_DIS,
-		NPHY_REV3_RFSEQ_CMD_END
-	};
-	static const u8 rfseq_tx2rx_dlys_rev3[] = { 8, 4, 2, 2, 4, 4, 6, 1 };
-	u8 rfseq_rx2tx_events_rev3[] = {
-		NPHY_REV3_RFSEQ_CMD_NOP,
-		NPHY_REV3_RFSEQ_CMD_RXG_FBW,
-		NPHY_REV3_RFSEQ_CMD_TR_SWITCH,
-		NPHY_REV3_RFSEQ_CMD_CLR_HIQ_DIS,
-		NPHY_REV3_RFSEQ_CMD_RXPD_TXPD,
-		NPHY_REV3_RFSEQ_CMD_TX_GAIN,
-		NPHY_REV3_RFSEQ_CMD_INT_PA_PU,
-		NPHY_REV3_RFSEQ_CMD_EXT_PA,
-		NPHY_REV3_RFSEQ_CMD_END
-	};
-	u8 rfseq_rx2tx_dlys_rev3[] = { 8, 6, 6, 4, 4, 18, 42, 1, 1 };
-	static const u8 rfseq_rx2tx_events_rev3_ipa[] = {
-		NPHY_REV3_RFSEQ_CMD_NOP,
-		NPHY_REV3_RFSEQ_CMD_RXG_FBW,
-		NPHY_REV3_RFSEQ_CMD_TR_SWITCH,
-		NPHY_REV3_RFSEQ_CMD_CLR_HIQ_DIS,
-		NPHY_REV3_RFSEQ_CMD_RXPD_TXPD,
-		NPHY_REV3_RFSEQ_CMD_TX_GAIN,
-		NPHY_REV3_RFSEQ_CMD_CLR_RXRX_BIAS,
-		NPHY_REV3_RFSEQ_CMD_INT_PA_PU,
-		NPHY_REV3_RFSEQ_CMD_END
-	};
-	static const u8 rfseq_rx2tx_dlys_rev3_ipa[] =
-	    { 8, 6, 6, 4, 4, 16, 43, 1, 1 };
-	s16 alpha0, alpha1, alpha2;
-	s16 beta0, beta1, beta2;
-	u32 leg_data_weights, ht_data_weights, nss1_data_weights,
-	    stbc_data_weights;
-	u8 chan_freq_range = 0;
-	static const u16 dac_control = 0x0002;
-	u16 aux_adc_vmid_rev4[] = { 0xa2, 0xb4, 0xb4, 0x89 };
-	u16 aux_adc_vmid_rev3[] = { 0xa2, 0xb4, 0xb4, 0x89 };
-	u16 *aux_adc_vmid;
-	u16 aux_adc_gain_rev4[] = { 0x02, 0x02, 0x02, 0x00 };
-	u16 aux_adc_gain_rev3[] = { 0x02, 0x02, 0x02, 0x00 };
-	u16 *aux_adc_gain;
-	static const u16 sk_adc_vmid[] = { 0xb4, 0xb4, 0xb4, 0x24 };
-	static const u16 sk_adc_gain[] = { 0x02, 0x02, 0x02, 0x02 };
-	s32 min_nvar_val = 0x18d;
-	u8 pdetrange;
-	u8 triso;
+	} else if (NREV_GE(pi->pubpi.phy_rev, 3)) {
 
-	write_phy_reg(pi, 0x23f, 0x1f8);
-	write_phy_reg(pi, 0x240, 0x1f8);
+		write_phy_reg(pi, 0x23f, 0x1f8);
+		write_phy_reg(pi, 0x240, 0x1f8);
 
-	wlc_phy_table_read_nphy(pi, NPHY_TBL_ID_CMPMETRICDATAWEIGHTTBL,
-				1, 0, 32, &leg_data_weights);
-	leg_data_weights = leg_data_weights & 0xffffff;
-	wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_CMPMETRICDATAWEIGHTTBL,
-				 1, 0, 32, &leg_data_weights);
-
-	alpha0 = 293;
-	alpha1 = 435;
-	alpha2 = 261;
-	beta0 = 366;
-	beta1 = 205;
-	beta2 = 32;
-	write_phy_reg(pi, 0x145, alpha0);
-	write_phy_reg(pi, 0x146, alpha1);
-	write_phy_reg(pi, 0x147, alpha2);
-	write_phy_reg(pi, 0x148, beta0);
-	write_phy_reg(pi, 0x149, beta1);
-	write_phy_reg(pi, 0x14a, beta2);
-
-	write_phy_reg(pi, 0x38, 0xC);
-	write_phy_reg(pi, 0x2ae, 0xC);
-
-	wlc_phy_set_rfseq_nphy(pi, NPHY_RFSEQ_TX2RX,
-			       rfseq_tx2rx_events_rev3,
-			       rfseq_tx2rx_dlys_rev3,
-			       ARRAY_SIZE(rfseq_tx2rx_events_rev3));
+		wlc_phy_table_read_nphy(pi, NPHY_TBL_ID_CMPMETRICDATAWEIGHTTBL,
+					1, 0, 32, &leg_data_weights);
+		leg_data_weights = leg_data_weights & 0xffffff;
+		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_CMPMETRICDATAWEIGHTTBL,
+					 1, 0, 32, &leg_data_weights);
+
+		alpha0 = 293;
+		alpha1 = 435;
+		alpha2 = 261;
+		beta0 = 366;
+		beta1 = 205;
+		beta2 = 32;
+		write_phy_reg(pi, 0x145, alpha0);
+		write_phy_reg(pi, 0x146, alpha1);
+		write_phy_reg(pi, 0x147, alpha2);
+		write_phy_reg(pi, 0x148, beta0);
+		write_phy_reg(pi, 0x149, beta1);
+		write_phy_reg(pi, 0x14a, beta2);
+
+		write_phy_reg(pi, 0x38, 0xC);
+		write_phy_reg(pi, 0x2ae, 0xC);
+
+		wlc_phy_set_rfseq_nphy(pi, NPHY_RFSEQ_TX2RX,
+				       rfseq_tx2rx_events_rev3,
+				       rfseq_tx2rx_dlys_rev3,
+				       ARRAY_SIZE(rfseq_tx2rx_events_rev3));
+
+		if (PHY_IPA(pi))
+			wlc_phy_set_rfseq_nphy(pi, NPHY_RFSEQ_RX2TX,
+					       rfseq_rx2tx_events_rev3_ipa,
+					       rfseq_rx2tx_dlys_rev3_ipa,
+					       ARRAY_SIZE(rfseq_rx2tx_events_rev3_ipa));
 
-	if (PHY_IPA(pi))
-		wlc_phy_set_rfseq_nphy(pi, NPHY_RFSEQ_RX2TX,
-				       rfseq_rx2tx_events_rev3_ipa,
-				       rfseq_rx2tx_dlys_rev3_ipa,
-				       ARRAY_SIZE (rfseq_rx2tx_events_rev3_ipa));
+		if ((pi->sh->hw_phyrxchain != 0x3) &&
+		    (pi->sh->hw_phyrxchain != pi->sh->hw_phytxchain)) {
 
-	if ((pi->sh->hw_phyrxchain != 0x3) &&
-	    (pi->sh->hw_phyrxchain != pi->sh->hw_phytxchain)) {
+			if (PHY_IPA(pi)) {
+				rfseq_rx2tx_dlys_rev3[5] = 59;
+				rfseq_rx2tx_dlys_rev3[6] = 1;
+				rfseq_rx2tx_events_rev3[7] =
+					NPHY_REV3_RFSEQ_CMD_END;
+			}
 
-		if (PHY_IPA(pi)) {
-			rfseq_rx2tx_dlys_rev3[5] = 59;
-			rfseq_rx2tx_dlys_rev3[6] = 1;
-			rfseq_rx2tx_events_rev3[7] = NPHY_REV3_RFSEQ_CMD_END;
+			wlc_phy_set_rfseq_nphy(
+				pi, NPHY_RFSEQ_RX2TX,
+				rfseq_rx2tx_events_rev3,
+				rfseq_rx2tx_dlys_rev3,
+				ARRAY_SIZE(rfseq_rx2tx_events_rev3));
 		}
 
-		wlc_phy_set_rfseq_nphy(pi, NPHY_RFSEQ_RX2TX,
-				       rfseq_rx2tx_events_rev3,
-				       rfseq_rx2tx_dlys_rev3,
-				       ARRAY_SIZE (rfseq_rx2tx_events_rev3));
-	}
+		if (CHSPEC_IS2G(pi->radio_chanspec))
+			write_phy_reg(pi, 0x6a, 0x2);
+		else
+			write_phy_reg(pi, 0x6a, 0x9c40);
 
-	if (CHSPEC_IS2G(pi->radio_chanspec))
-		write_phy_reg(pi, 0x6a, 0x2);
-	else
-		write_phy_reg(pi, 0x6a, 0x9c40);
+		mod_phy_reg(pi, 0x294, (0xf << 8), (7 << 8));
 
-	mod_phy_reg(pi, 0x294, (0xf << 8), (7 << 8));
+		if (CHSPEC_IS40(pi->radio_chanspec) == 0) {
+			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1, 3,
+						 32, &min_nvar_val);
+			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1,
+						 127, 32, &min_nvar_val);
+		} else {
+			min_nvar_val = noise_var_tbl_rev3[3];
+			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1, 3,
+						 32, &min_nvar_val);
 
-	if (CHSPEC_IS40(pi->radio_chanspec) == 0) {
-		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1, 3,
-					 32, &min_nvar_val);
-		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1,
-					 127, 32, &min_nvar_val);
-	} else {
-		min_nvar_val = noise_var_tbl_rev3[3];
-		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1, 3,
-					 32, &min_nvar_val);
+			min_nvar_val = noise_var_tbl_rev3[127];
+			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1,
+						 127, 32, &min_nvar_val);
+		}
 
-		min_nvar_val = noise_var_tbl_rev3[127];
-		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1,
-					 127, 32, &min_nvar_val);
-	}
+		wlc_phy_workarounds_nphy_gainctrl(pi);
 
-	wlc_phy_workarounds_nphy_gainctrl(pi);
+		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1, 0x00, 16,
+					 &dac_control);
+		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1, 0x10, 16,
+					 &dac_control);
 
-	wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1, 0x00, 16,
-				 &dac_control);
-	wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1, 0x10, 16,
-				 &dac_control);
+		pdetrange =
+			(CHSPEC_IS5G(pi->radio_chanspec)) ? pi->srom_fem5g.
+			pdetrange : pi->srom_fem2g.pdetrange;
 
-	pdetrange = (CHSPEC_IS5G(pi->radio_chanspec)) ?
-			pi->srom_fem5g.pdetrange : pi->srom_fem2g.pdetrange;
+		if (pdetrange == 0) {
+			if (NREV_GE(pi->pubpi.phy_rev, 4)) {
+				aux_adc_vmid = aux_adc_vmid_rev4;
+				aux_adc_gain = aux_adc_gain_rev4;
+			} else {
+				aux_adc_vmid = aux_adc_vmid_rev3;
+				aux_adc_gain = aux_adc_gain_rev3;
+			}
+			chan_freq_range =
+				wlc_phy_get_chan_freq_range_nphy(pi, 0);
+			if (chan_freq_range != WL_CHAN_FREQ_RANGE_2G) {
+				switch (chan_freq_range) {
+				case WL_CHAN_FREQ_RANGE_5GL:
+					aux_adc_vmid[3] = 0x89;
+					aux_adc_gain[3] = 0;
+					break;
+				case WL_CHAN_FREQ_RANGE_5GM:
+					aux_adc_vmid[3] = 0x89;
+					aux_adc_gain[3] = 0;
+					break;
+				case WL_CHAN_FREQ_RANGE_5GH:
+					aux_adc_vmid[3] = 0x89;
+					aux_adc_gain[3] = 0;
+					break;
+				default:
+					break;
+				}
+			}
+			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
+						 0x08, 16, aux_adc_vmid);
+			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
+						 0x18, 16, aux_adc_vmid);
+			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
+						 0x0c, 16, aux_adc_gain);
+			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
+						 0x1c, 16, aux_adc_gain);
+		} else if (pdetrange == 1) {
+			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
+						 0x08, 16, sk_adc_vmid);
+			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
+						 0x18, 16, sk_adc_vmid);
+			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
+						 0x0c, 16, sk_adc_gain);
+			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
+						 0x1c, 16, sk_adc_gain);
+		} else if (pdetrange == 2) {
 
-	if (pdetrange == 0) {
-		if (NREV_GE(pi->pubpi.phy_rev, 4)) {
-			aux_adc_vmid = aux_adc_vmid_rev4;
-			aux_adc_gain = aux_adc_gain_rev4;
-		} else {
-			aux_adc_vmid = aux_adc_vmid_rev3;
-			aux_adc_gain = aux_adc_gain_rev3;
-		}
-		chan_freq_range = wlc_phy_get_chan_freq_range_nphy(pi, 0);
-		if (chan_freq_range != WL_CHAN_FREQ_RANGE_2G) {
-			switch (chan_freq_range) {
-			case WL_CHAN_FREQ_RANGE_5GL:
-				aux_adc_vmid[3] = 0x89;
-				aux_adc_gain[3] = 0;
-				break;
-			case WL_CHAN_FREQ_RANGE_5GM:
-				aux_adc_vmid[3] = 0x89;
-				aux_adc_gain[3] = 0;
-				break;
-			case WL_CHAN_FREQ_RANGE_5GH:
-				aux_adc_vmid[3] = 0x89;
-				aux_adc_gain[3] = 0;
-				break;
-			default:
-				break;
+			u16 bcm_adc_vmid[] = { 0xa2, 0xb4, 0xb4, 0x74 };
+			u16 bcm_adc_gain[] = { 0x02, 0x02, 0x02, 0x04 };
+
+			if (NREV_GE(pi->pubpi.phy_rev, 6)) {
+				chan_freq_range =
+					wlc_phy_get_chan_freq_range_nphy(pi, 0);
+				if (chan_freq_range != WL_CHAN_FREQ_RANGE_2G) {
+					bcm_adc_vmid[3] = 0x8e;
+					bcm_adc_gain[3] = 0x03;
+				} else {
+					bcm_adc_vmid[3] = 0x94;
+					bcm_adc_gain[3] = 0x03;
+				}
+			} else if (NREV_IS(pi->pubpi.phy_rev, 5)) {
+				bcm_adc_vmid[3] = 0x84;
+				bcm_adc_gain[3] = 0x02;
 			}
-		}
-		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
-					 0x08, 16, aux_adc_vmid);
-		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
-					 0x18, 16, aux_adc_vmid);
-		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
-					 0x0c, 16, aux_adc_gain);
-		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
-					 0x1c, 16, aux_adc_gain);
-	} else if (pdetrange == 1) {
-		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
-					 0x08, 16, sk_adc_vmid);
-		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
-					 0x18, 16, sk_adc_vmid);
-		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
-					 0x0c, 16, sk_adc_gain);
-		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
-					 0x1c, 16, sk_adc_gain);
-	} else if (pdetrange == 2) {
 
-		u16 bcm_adc_vmid[] = { 0xa2, 0xb4, 0xb4, 0x74 };
-		u16 bcm_adc_gain[] = { 0x02, 0x02, 0x02, 0x04 };
+			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
+						 0x08, 16, bcm_adc_vmid);
+			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
+						 0x18, 16, bcm_adc_vmid);
+			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
+						 0x0c, 16, bcm_adc_gain);
+			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
+						 0x1c, 16, bcm_adc_gain);
+		} else if (pdetrange == 3) {
+			chan_freq_range =
+				wlc_phy_get_chan_freq_range_nphy(pi, 0);
+			if ((NREV_GE(pi->pubpi.phy_rev, 4))
+			    && (chan_freq_range == WL_CHAN_FREQ_RANGE_2G)) {
+
+				u16 auxadc_vmid[] = {
+					0xa2, 0xb4, 0xb4, 0x270
+				};
+				u16 auxadc_gain[] = {
+					0x02, 0x02, 0x02, 0x00
+				};
+
+				wlc_phy_table_write_nphy(pi,
+							 NPHY_TBL_ID_AFECTRL, 4,
+							 0x08, 16, auxadc_vmid);
+				wlc_phy_table_write_nphy(pi,
+							 NPHY_TBL_ID_AFECTRL, 4,
+							 0x18, 16, auxadc_vmid);
+				wlc_phy_table_write_nphy(pi,
+							 NPHY_TBL_ID_AFECTRL, 4,
+							 0x0c, 16, auxadc_gain);
+				wlc_phy_table_write_nphy(pi,
+							 NPHY_TBL_ID_AFECTRL, 4,
+							 0x1c, 16, auxadc_gain);
+			}
+		} else if ((pdetrange == 4) || (pdetrange == 5)) {
+			u16 bcm_adc_vmid[] = { 0xa2, 0xb4, 0xb4, 0x0 };
+			u16 bcm_adc_gain[] = { 0x02, 0x02, 0x02, 0x0 };
+			u16 Vmid[2], Av[2];
 
-		if (NREV_GE(pi->pubpi.phy_rev, 6)) {
 			chan_freq_range =
-			    wlc_phy_get_chan_freq_range_nphy(pi, 0);
+				wlc_phy_get_chan_freq_range_nphy(pi, 0);
 			if (chan_freq_range != WL_CHAN_FREQ_RANGE_2G) {
-				bcm_adc_vmid[3] = 0x8e;
-				bcm_adc_gain[3] = 0x03;
+				Vmid[0] = (pdetrange == 4) ? 0x8e : 0x89;
+				Vmid[1] = (pdetrange == 4) ? 0x96 : 0x89;
+				Av[0] = (pdetrange == 4) ? 2 : 0;
+				Av[1] = (pdetrange == 4) ? 2 : 0;
 			} else {
-				bcm_adc_vmid[3] = 0x94;
-				bcm_adc_gain[3] = 0x03;
+				Vmid[0] = (pdetrange == 4) ? 0x89 : 0x74;
+				Vmid[1] = (pdetrange == 4) ? 0x8b : 0x70;
+				Av[0] = (pdetrange == 4) ? 2 : 0;
+				Av[1] = (pdetrange == 4) ? 2 : 0;
 			}
-		} else if (NREV_IS(pi->pubpi.phy_rev, 5)) {
-			bcm_adc_vmid[3] = 0x84;
-			bcm_adc_gain[3] = 0x02;
-		}
-
-		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
-					 0x08, 16, bcm_adc_vmid);
-		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
-					 0x18, 16, bcm_adc_vmid);
-		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
-					 0x0c, 16, bcm_adc_gain);
-		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
-					 0x1c, 16, bcm_adc_gain);
-	} else if (pdetrange == 3) {
-		chan_freq_range = wlc_phy_get_chan_freq_range_nphy(pi, 0);
-		if ((NREV_GE(pi->pubpi.phy_rev, 4)) &&
-		    (chan_freq_range == WL_CHAN_FREQ_RANGE_2G)) {
-			u16 auxadc_vmid[] = { 0xa2, 0xb4, 0xb4, 0x270 };
-			u16 auxadc_gain[] = { 0x02, 0x02, 0x02, 0x00 };
 
+			bcm_adc_vmid[3] = Vmid[0];
+			bcm_adc_gain[3] = Av[0];
 			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
-						 0x08, 16, auxadc_vmid);
+						 0x08, 16, bcm_adc_vmid);
 			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
-						 0x18, 16, auxadc_vmid);
+						 0x0c, 16, bcm_adc_gain);
+
+			bcm_adc_vmid[3] = Vmid[1];
+			bcm_adc_gain[3] = Av[1];
 			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
-						 0x0c, 16, auxadc_gain);
+						 0x18, 16, bcm_adc_vmid);
 			wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
-						 0x1c, 16, auxadc_gain);
+						 0x1c, 16, bcm_adc_gain);
 		}
-	} else if ((pdetrange == 4) || (pdetrange == 5)) {
-		u16 bcm_adc_vmid[] = { 0xa2, 0xb4, 0xb4, 0x0 };
-		u16 bcm_adc_gain[] = { 0x02, 0x02, 0x02, 0x0 };
-		u16 Vmid[2], Av[2];
 
-		chan_freq_range = wlc_phy_get_chan_freq_range_nphy(pi, 0);
-		if (chan_freq_range != WL_CHAN_FREQ_RANGE_2G) {
-			Vmid[0] = (pdetrange == 4) ? 0x8e : 0x89;
-			Vmid[1] = (pdetrange == 4) ? 0x96 : 0x89;
-			Av[0] = (pdetrange == 4) ? 2 : 0;
-			Av[1] = (pdetrange == 4) ? 2 : 0;
-		} else {
-			Vmid[0] = (pdetrange == 4) ? 0x89 : 0x74;
-			Vmid[1] = (pdetrange == 4) ? 0x8b : 0x70;
-			Av[0] = (pdetrange == 4) ? 2 : 0;
-			Av[1] = (pdetrange == 4) ? 2 : 0;
-		}
-
-		bcm_adc_vmid[3] = Vmid[0];
-		bcm_adc_gain[3] = Av[0];
-		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
-					 0x08, 16, bcm_adc_vmid);
-		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
-					 0x0c, 16, bcm_adc_gain);
-
-		bcm_adc_vmid[3] = Vmid[1];
-		bcm_adc_gain[3] = Av[1];
-		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
-					 0x18, 16, bcm_adc_vmid);
-		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4,
-					 0x1c, 16, bcm_adc_gain);
-	}
-
-	write_radio_reg(pi, (RADIO_2056_RX_MIXA_MAST_BIAS | RADIO_2056_RX0), 0x0);
-	write_radio_reg(pi, (RADIO_2056_RX_MIXA_MAST_BIAS | RADIO_2056_RX1), 0x0);
-
-	write_radio_reg(pi, (RADIO_2056_RX_MIXA_BIAS_MAIN | RADIO_2056_RX0), 0x6);
-	write_radio_reg(pi, (RADIO_2056_RX_MIXA_BIAS_MAIN | RADIO_2056_RX1), 0x6);
-
-	write_radio_reg(pi, (RADIO_2056_RX_MIXA_BIAS_AUX | RADIO_2056_RX0), 0x7);
-	write_radio_reg(pi, (RADIO_2056_RX_MIXA_BIAS_AUX | RADIO_2056_RX1), 0x7);
-
-	write_radio_reg(pi, (RADIO_2056_RX_MIXA_LOB_BIAS | RADIO_2056_RX0), 0x88);
-	write_radio_reg(pi, (RADIO_2056_RX_MIXA_LOB_BIAS | RADIO_2056_RX1), 0x88);
-
-	write_radio_reg(pi, (RADIO_2056_RX_MIXA_CMFB_IDAC | RADIO_2056_RX0), 0x0);
-	write_radio_reg(pi, (RADIO_2056_RX_MIXA_CMFB_IDAC | RADIO_2056_RX1), 0x0);
-
-	write_radio_reg(pi, (RADIO_2056_RX_MIXG_CMFB_IDAC | RADIO_2056_RX0), 0x0);
-	write_radio_reg(pi, (RADIO_2056_RX_MIXG_CMFB_IDAC | RADIO_2056_RX1), 0x0);
-
-	triso = (CHSPEC_IS5G(pi->radio_chanspec)) ?
-		 pi->srom_fem5g.triso : pi->srom_fem2g.triso;
-	if (triso == 7) {
-		wlc_phy_war_force_trsw_to_R_cliplo_nphy(pi, PHY_CORE_0);
-		wlc_phy_war_force_trsw_to_R_cliplo_nphy(pi, PHY_CORE_1);
-	}
-
-	wlc_phy_war_txchain_upd_nphy(pi, pi->sh->hw_phytxchain);
-
-	if (((pi->sh->boardflags2 & BFL2_APLL_WAR) &&
-	     (CHSPEC_IS5G(pi->radio_chanspec))) ||
-	    (((pi->sh->boardflags2 & BFL2_GPLL_WAR) ||
-	      (pi->sh->boardflags2 & BFL2_GPLL_WAR2)) &&
-	     (CHSPEC_IS2G(pi->radio_chanspec)))) {
-		nss1_data_weights = 0x00088888;
-		ht_data_weights = 0x00088888;
-		stbc_data_weights = 0x00088888;
-	} else {
-		nss1_data_weights = 0x88888888;
-		ht_data_weights = 0x88888888;
-		stbc_data_weights = 0x88888888;
-	}
-	wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_CMPMETRICDATAWEIGHTTBL,
-				 1, 1, 32, &nss1_data_weights);
-	wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_CMPMETRICDATAWEIGHTTBL,
-				 1, 2, 32, &ht_data_weights);
-	wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_CMPMETRICDATAWEIGHTTBL,
-				 1, 3, 32, &stbc_data_weights);
+		write_radio_reg(pi,
+				(RADIO_2056_RX_MIXA_MAST_BIAS | RADIO_2056_RX0),
+				0x0);
+		write_radio_reg(pi,
+				(RADIO_2056_RX_MIXA_MAST_BIAS | RADIO_2056_RX1),
+				0x0);
 
-	if (NREV_IS(pi->pubpi.phy_rev, 4)) {
-		if (CHSPEC_IS5G(pi->radio_chanspec)) {
-			write_radio_reg(pi,
-					RADIO_2056_TX_GMBB_IDAC |
-					RADIO_2056_TX0, 0x70);
-			write_radio_reg(pi,
-					RADIO_2056_TX_GMBB_IDAC |
-					RADIO_2056_TX1, 0x70);
-		}
-	}
+		write_radio_reg(pi,
+				(RADIO_2056_RX_MIXA_BIAS_MAIN | RADIO_2056_RX0),
+				0x6);
+		write_radio_reg(pi,
+				(RADIO_2056_RX_MIXA_BIAS_MAIN | RADIO_2056_RX1),
+				0x6);
 
-	if (!pi->edcrs_threshold_lock) {
-		write_phy_reg(pi, 0x224, 0x3eb);
-		write_phy_reg(pi, 0x225, 0x3eb);
-		write_phy_reg(pi, 0x226, 0x341);
-		write_phy_reg(pi, 0x227, 0x341);
-		write_phy_reg(pi, 0x228, 0x42b);
-		write_phy_reg(pi, 0x229, 0x42b);
-		write_phy_reg(pi, 0x22a, 0x381);
-		write_phy_reg(pi, 0x22b, 0x381);
-		write_phy_reg(pi, 0x22c, 0x42b);
-		write_phy_reg(pi, 0x22d, 0x42b);
-		write_phy_reg(pi, 0x22e, 0x381);
-		write_phy_reg(pi, 0x22f, 0x381);
-	}
+		write_radio_reg(pi,
+				(RADIO_2056_RX_MIXA_BIAS_AUX | RADIO_2056_RX0),
+				0x7);
+		write_radio_reg(pi,
+				(RADIO_2056_RX_MIXA_BIAS_AUX | RADIO_2056_RX1),
+				0x7);
 
-	if (NREV_GE(pi->pubpi.phy_rev, 6)) {
+		write_radio_reg(pi,
+				(RADIO_2056_RX_MIXA_LOB_BIAS | RADIO_2056_RX0),
+				0x88);
+		write_radio_reg(pi,
+				(RADIO_2056_RX_MIXA_LOB_BIAS | RADIO_2056_RX1),
+				0x88);
 
-		if (pi->sh->boardflags2 & BFL2_SINGLEANT_CCK)
-			wlapi_bmac_mhf(pi->sh->physhim, MHF4,
-				       MHF4_BPHY_TXCORE0,
-				       MHF4_BPHY_TXCORE0, BRCM_BAND_ALL);
-	}
-}
+		write_radio_reg(pi,
+				(RADIO_2056_RX_MIXA_CMFB_IDAC | RADIO_2056_RX0),
+				0x0);
+		write_radio_reg(pi,
+				(RADIO_2056_RX_MIXA_CMFB_IDAC | RADIO_2056_RX1),
+				0x0);
 
-static void wlc_phy_workarounds_nphy_rev1(struct brcms_phy *pi)
-{
-	static const u8 rfseq_rx2tx_events[] = {
-		NPHY_RFSEQ_CMD_NOP,
-		NPHY_RFSEQ_CMD_RXG_FBW,
-		NPHY_RFSEQ_CMD_TR_SWITCH,
-		NPHY_RFSEQ_CMD_CLR_HIQ_DIS,
-		NPHY_RFSEQ_CMD_RXPD_TXPD,
-		NPHY_RFSEQ_CMD_TX_GAIN,
-		NPHY_RFSEQ_CMD_EXT_PA
-	};
-	u8 rfseq_rx2tx_dlys[] = { 8, 6, 6, 2, 4, 60, 1 };
-	static const u8 rfseq_tx2rx_events[] = {
-		NPHY_RFSEQ_CMD_NOP,
-		NPHY_RFSEQ_CMD_EXT_PA,
-		NPHY_RFSEQ_CMD_TX_GAIN,
-		NPHY_RFSEQ_CMD_RXPD_TXPD,
-		NPHY_RFSEQ_CMD_TR_SWITCH,
-		NPHY_RFSEQ_CMD_RXG_FBW,
-		NPHY_RFSEQ_CMD_CLR_HIQ_DIS
-	};
-	static const u8 rfseq_tx2rx_dlys[] = { 8, 6, 2, 4, 4, 6, 1 };
-	s16 alpha0, alpha1, alpha2;
-	s16 beta0, beta1, beta2;
-	u16 regval;
+		write_radio_reg(pi,
+				(RADIO_2056_RX_MIXG_CMFB_IDAC | RADIO_2056_RX0),
+				0x0);
+		write_radio_reg(pi,
+				(RADIO_2056_RX_MIXG_CMFB_IDAC | RADIO_2056_RX1),
+				0x0);
 
-	if (pi->sh->boardflags2 & BFL2_SKWRKFEM_BRD ||
-	    (pi->sh->boardtype == 0x8b)) {
-		uint i;
-		u8 war_dlys[] = { 1, 6, 6, 2, 4, 20, 1 };
-		for (i = 0; i < ARRAY_SIZE(rfseq_rx2tx_dlys); i++)
-			rfseq_rx2tx_dlys[i] = war_dlys[i];
-	}
-
-	if (CHSPEC_IS5G(pi->radio_chanspec) && pi->phy_5g_pwrgain) {
-		and_radio_reg(pi, RADIO_2055_CORE1_TX_RF_SPARE, 0xf7);
-		and_radio_reg(pi, RADIO_2055_CORE2_TX_RF_SPARE, 0xf7);
-	} else {
-		or_radio_reg(pi, RADIO_2055_CORE1_TX_RF_SPARE, 0x8);
-		or_radio_reg(pi, RADIO_2055_CORE2_TX_RF_SPARE, 0x8);
-	}
-
-	regval = 0x000a;
-	wlc_phy_table_write_nphy(pi, 8, 1, 0, 16, &regval);
-	wlc_phy_table_write_nphy(pi, 8, 1, 0x10, 16, &regval);
-
-	if (NREV_LT(pi->pubpi.phy_rev, 3)) {
-		regval = 0xcdaa;
-		wlc_phy_table_write_nphy(pi, 8, 1, 0x02, 16, &regval);
-		wlc_phy_table_write_nphy(pi, 8, 1, 0x12, 16, &regval);
-	}
+		triso =
+			(CHSPEC_IS5G(pi->radio_chanspec)) ? pi->srom_fem5g.
+			triso : pi->srom_fem2g.triso;
+		if (triso == 7) {
+			wlc_phy_war_force_trsw_to_R_cliplo_nphy(pi, PHY_CORE_0);
+			wlc_phy_war_force_trsw_to_R_cliplo_nphy(pi, PHY_CORE_1);
+		}
+
+		wlc_phy_war_txchain_upd_nphy(pi, pi->sh->hw_phytxchain);
+
+		if (((pi->sh->boardflags2 & BFL2_APLL_WAR) &&
+		     (CHSPEC_IS5G(pi->radio_chanspec))) ||
+		    (((pi->sh->boardflags2 & BFL2_GPLL_WAR) ||
+		      (pi->sh->boardflags2 & BFL2_GPLL_WAR2)) &&
+		     (CHSPEC_IS2G(pi->radio_chanspec)))) {
+			nss1_data_weights = 0x00088888;
+			ht_data_weights = 0x00088888;
+			stbc_data_weights = 0x00088888;
+		} else {
+			nss1_data_weights = 0x88888888;
+			ht_data_weights = 0x88888888;
+			stbc_data_weights = 0x88888888;
+		}
+		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_CMPMETRICDATAWEIGHTTBL,
+					 1, 1, 32, &nss1_data_weights);
+		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_CMPMETRICDATAWEIGHTTBL,
+					 1, 2, 32, &ht_data_weights);
+		wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_CMPMETRICDATAWEIGHTTBL,
+					 1, 3, 32, &stbc_data_weights);
 
-	if (NREV_LT(pi->pubpi.phy_rev, 2)) {
-		regval = 0x0000;
-		wlc_phy_table_write_nphy(pi, 8, 1, 0x08, 16, &regval);
-		wlc_phy_table_write_nphy(pi, 8, 1, 0x18, 16, &regval);
+		if (NREV_IS(pi->pubpi.phy_rev, 4)) {
+			if (CHSPEC_IS5G(pi->radio_chanspec)) {
+				write_radio_reg(pi,
+						RADIO_2056_TX_GMBB_IDAC |
+						RADIO_2056_TX0, 0x70);
+				write_radio_reg(pi,
+						RADIO_2056_TX_GMBB_IDAC |
+						RADIO_2056_TX1, 0x70);
+			}
+		}
+
+		if (!pi->edcrs_threshold_lock) {
+			write_phy_reg(pi, 0x224, 0x3eb);
+			write_phy_reg(pi, 0x225, 0x3eb);
+			write_phy_reg(pi, 0x226, 0x341);
+			write_phy_reg(pi, 0x227, 0x341);
+			write_phy_reg(pi, 0x228, 0x42b);
+			write_phy_reg(pi, 0x229, 0x42b);
+			write_phy_reg(pi, 0x22a, 0x381);
+			write_phy_reg(pi, 0x22b, 0x381);
+			write_phy_reg(pi, 0x22c, 0x42b);
+			write_phy_reg(pi, 0x22d, 0x42b);
+			write_phy_reg(pi, 0x22e, 0x381);
+			write_phy_reg(pi, 0x22f, 0x381);
+		}
 
-		regval = 0x7aab;
-		wlc_phy_table_write_nphy(pi, 8, 1, 0x07, 16, &regval);
-		wlc_phy_table_write_nphy(pi, 8, 1, 0x17, 16, &regval);
+		if (NREV_GE(pi->pubpi.phy_rev, 6)) {
 
-		regval = 0x0800;
-		wlc_phy_table_write_nphy(pi, 8, 1, 0x06, 16, &regval);
-		wlc_phy_table_write_nphy(pi, 8, 1, 0x16, 16, &regval);
-	}
+			if (pi->sh->boardflags2 & BFL2_SINGLEANT_CCK)
+				wlapi_bmac_mhf(pi->sh->physhim, MHF4,
+					      MHF4_BPHY_TXCORE0,
+					      MHF4_BPHY_TXCORE0, BRCM_BAND_ALL);
+		}
+	} else {
 
-	write_phy_reg(pi, 0xf8, 0x02d8);
-	write_phy_reg(pi, 0xf9, 0x0301);
-	write_phy_reg(pi, 0xfa, 0x02d8);
-	write_phy_reg(pi, 0xfb, 0x0301);
+		if (pi->sh->boardflags2 & BFL2_SKWRKFEM_BRD ||
+		    (pi->sh->boardtype == 0x8b)) {
+			uint i;
+			u8 war_dlys[] = { 1, 6, 6, 2, 4, 20, 1 };
+			for (i = 0; i < ARRAY_SIZE(rfseq_rx2tx_dlys); i++)
+				rfseq_rx2tx_dlys[i] = war_dlys[i];
+		}
 
-	wlc_phy_set_rfseq_nphy(pi, NPHY_RFSEQ_RX2TX, rfseq_rx2tx_events,
-			       rfseq_rx2tx_dlys,
-			       ARRAY_SIZE(rfseq_rx2tx_events));
+		if (CHSPEC_IS5G(pi->radio_chanspec) && pi->phy_5g_pwrgain) {
+			and_radio_reg(pi, RADIO_2055_CORE1_TX_RF_SPARE, 0xf7);
+			and_radio_reg(pi, RADIO_2055_CORE2_TX_RF_SPARE, 0xf7);
+		} else {
+			or_radio_reg(pi, RADIO_2055_CORE1_TX_RF_SPARE, 0x8);
+			or_radio_reg(pi, RADIO_2055_CORE2_TX_RF_SPARE, 0x8);
+		}
 
-	wlc_phy_set_rfseq_nphy(pi, NPHY_RFSEQ_TX2RX, rfseq_tx2rx_events,
-			       rfseq_tx2rx_dlys,
-			       ARRAY_SIZE(rfseq_tx2rx_events));
+		regval = 0x000a;
+		wlc_phy_table_write_nphy(pi, 8, 1, 0, 16, &regval);
+		wlc_phy_table_write_nphy(pi, 8, 1, 0x10, 16, &regval);
 
-	wlc_phy_workarounds_nphy_gainctrl(pi);
+		if (NREV_LT(pi->pubpi.phy_rev, 3)) {
+			regval = 0xcdaa;
+			wlc_phy_table_write_nphy(pi, 8, 1, 0x02, 16, &regval);
+			wlc_phy_table_write_nphy(pi, 8, 1, 0x12, 16, &regval);
+		}
 
-	if (NREV_LT(pi->pubpi.phy_rev, 2)) {
+		if (NREV_LT(pi->pubpi.phy_rev, 2)) {
+			regval = 0x0000;
+			wlc_phy_table_write_nphy(pi, 8, 1, 0x08, 16, &regval);
+			wlc_phy_table_write_nphy(pi, 8, 1, 0x18, 16, &regval);
 
-		if (read_phy_reg(pi, 0xa0) & NPHY_MLenable)
-			wlapi_bmac_mhf(pi->sh->physhim, MHF3,
-				       MHF3_NPHY_MLADV_WAR,
-				       MHF3_NPHY_MLADV_WAR, BRCM_BAND_ALL);
-
-	} else if (NREV_IS(pi->pubpi.phy_rev, 2)) {
-		write_phy_reg(pi, 0x1e3, 0x0);
-		write_phy_reg(pi, 0x1e4, 0x0);
-	}
+			regval = 0x7aab;
+			wlc_phy_table_write_nphy(pi, 8, 1, 0x07, 16, &regval);
+			wlc_phy_table_write_nphy(pi, 8, 1, 0x17, 16, &regval);
 
-	if (NREV_LT(pi->pubpi.phy_rev, 2))
-		mod_phy_reg(pi, 0x90, (0x1 << 7), 0);
+			regval = 0x0800;
+			wlc_phy_table_write_nphy(pi, 8, 1, 0x06, 16, &regval);
+			wlc_phy_table_write_nphy(pi, 8, 1, 0x16, 16, &regval);
+		}
 
-	alpha0 = 293;
-	alpha1 = 435;
-	alpha2 = 261;
-	beta0 = 366;
-	beta1 = 205;
-	beta2 = 32;
-	write_phy_reg(pi, 0x145, alpha0);
-	write_phy_reg(pi, 0x146, alpha1);
-	write_phy_reg(pi, 0x147, alpha2);
-	write_phy_reg(pi, 0x148, beta0);
-	write_phy_reg(pi, 0x149, beta1);
-	write_phy_reg(pi, 0x14a, beta2);
-
-	if (NREV_LT(pi->pubpi.phy_rev, 3)) {
-		mod_phy_reg(pi, 0x142, (0xf << 12), 0);
-
-		write_phy_reg(pi, 0x192, 0xb5);
-		write_phy_reg(pi, 0x193, 0xa4);
-		write_phy_reg(pi, 0x194, 0x0);
-	}
+		write_phy_reg(pi, 0xf8, 0x02d8);
+		write_phy_reg(pi, 0xf9, 0x0301);
+		write_phy_reg(pi, 0xfa, 0x02d8);
+		write_phy_reg(pi, 0xfb, 0x0301);
 
-	if (NREV_IS(pi->pubpi.phy_rev, 2))
-		mod_phy_reg(pi, 0x221,
-			    NPHY_FORCESIG_DECODEGATEDCLKS,
-			    NPHY_FORCESIG_DECODEGATEDCLKS);
-}
+		wlc_phy_set_rfseq_nphy(pi, NPHY_RFSEQ_RX2TX, rfseq_rx2tx_events,
+				       rfseq_rx2tx_dlys,
+				       ARRAY_SIZE(rfseq_rx2tx_events));
 
-static void wlc_phy_workarounds_nphy(struct brcms_phy *pi)
-{
-	if (CHSPEC_IS5G(pi->radio_chanspec))
-		wlc_phy_classifier_nphy(pi, NPHY_ClassifierCtrl_cck_en, 0);
-	else
-		wlc_phy_classifier_nphy(pi, NPHY_ClassifierCtrl_cck_en, 1);
+		wlc_phy_set_rfseq_nphy(pi, NPHY_RFSEQ_TX2RX, rfseq_tx2rx_events,
+				       rfseq_tx2rx_dlys,
+				       ARRAY_SIZE(rfseq_tx2rx_events));
 
-	if (pi->phyhang_avoid)
-		wlc_phy_stay_in_carriersearch_nphy(pi, true);
+		wlc_phy_workarounds_nphy_gainctrl(pi);
 
-	or_phy_reg(pi, 0xb1, NPHY_IQFlip_ADC1 | NPHY_IQFlip_ADC2);
+		if (NREV_LT(pi->pubpi.phy_rev, 2)) {
 
-	if (NREV_GE(pi->pubpi.phy_rev, 7))
-		wlc_phy_workarounds_nphy_rev7(pi);
-	else if (NREV_GE(pi->pubpi.phy_rev, 3))
-		wlc_phy_workarounds_nphy_rev3(pi);
-	else
-		wlc_phy_workarounds_nphy_rev1(pi);
+			if (read_phy_reg(pi, 0xa0) & NPHY_MLenable)
+				wlapi_bmac_mhf(pi->sh->physhim, MHF3,
+					       MHF3_NPHY_MLADV_WAR,
+					       MHF3_NPHY_MLADV_WAR,
+					       BRCM_BAND_ALL);
+
+		} else if (NREV_IS(pi->pubpi.phy_rev, 2)) {
+			write_phy_reg(pi, 0x1e3, 0x0);
+			write_phy_reg(pi, 0x1e4, 0x0);
+		}
+
+		if (NREV_LT(pi->pubpi.phy_rev, 2))
+			mod_phy_reg(pi, 0x90, (0x1 << 7), 0);
+
+		alpha0 = 293;
+		alpha1 = 435;
+		alpha2 = 261;
+		beta0 = 366;
+		beta1 = 205;
+		beta2 = 32;
+		write_phy_reg(pi, 0x145, alpha0);
+		write_phy_reg(pi, 0x146, alpha1);
+		write_phy_reg(pi, 0x147, alpha2);
+		write_phy_reg(pi, 0x148, beta0);
+		write_phy_reg(pi, 0x149, beta1);
+		write_phy_reg(pi, 0x14a, beta2);
+
+		if (NREV_LT(pi->pubpi.phy_rev, 3)) {
+			mod_phy_reg(pi, 0x142, (0xf << 12), 0);
+
+			write_phy_reg(pi, 0x192, 0xb5);
+			write_phy_reg(pi, 0x193, 0xa4);
+			write_phy_reg(pi, 0x194, 0x0);
+		}
+
+		if (NREV_IS(pi->pubpi.phy_rev, 2))
+			mod_phy_reg(pi, 0x221,
+				    NPHY_FORCESIG_DECODEGATEDCLKS,
+				    NPHY_FORCESIG_DECODEGATEDCLKS);
+	}
 
 	if (pi->phyhang_avoid)
 		wlc_phy_stay_in_carriersearch_nphy(pi, false);
@@ -23032,7 +23116,7 @@
 	u16 t;
 	u32 *data_buf = NULL;
 
-	data_buf = kmalloc_array(num_samps, sizeof(u32), GFP_ATOMIC);
+	data_buf = kmalloc(sizeof(u32) * num_samps, GFP_ATOMIC);
 	if (data_buf == NULL)
 		return;
 
@@ -23074,8 +23158,7 @@
 		tbl_len = (phy_bw << 1);
 	}
 
-	tone_buf = kmalloc_array(tbl_len, sizeof(struct cordic_iq),
-				 GFP_ATOMIC);
+	tone_buf = kmalloc(sizeof(struct cordic_iq) * tbl_len, GFP_ATOMIC);
 	if (tone_buf == NULL)
 		return 0;
 
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phytbl_n.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phytbl_n.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phytbl_n.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phytbl_n.c	2020-06-19 21:00:31.420227716 -0400
@@ -14,7 +14,6 @@
  * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  */
 
-#include <linux/kernel.h>
 #include <types.h>
 #include "phytbl_n.h"
 
@@ -4438,39 +4437,109 @@
 };
 
 const struct phytbl_info mimophytbl_info_rev0_volatile[] = {
-	{&bdi_tbl_rev0, ARRAY_SIZE(bdi_tbl_rev0), 21, 0, 16},
-	{&pltlut_tbl_rev0, ARRAY_SIZE(pltlut_tbl_rev0), 20, 0, 32},
-	{&gainctrl_lut_core0_rev0, ARRAY_SIZE(gainctrl_lut_core0_rev0), 26, 192, 32},
-	{&gainctrl_lut_core1_rev0, ARRAY_SIZE(gainctrl_lut_core1_rev0), 27, 192, 32},
-	{&est_pwr_lut_core0_rev0, ARRAY_SIZE(est_pwr_lut_core0_rev0), 26, 0, 8},
-	{&est_pwr_lut_core1_rev0, ARRAY_SIZE(est_pwr_lut_core1_rev0), 27, 0, 8},
-	{&adj_pwr_lut_core0_rev0, ARRAY_SIZE(adj_pwr_lut_core0_rev0), 26, 64, 8},
-	{&adj_pwr_lut_core1_rev0, ARRAY_SIZE(adj_pwr_lut_core1_rev0), 27, 64, 8},
-	{&iq_lut_core0_rev0, ARRAY_SIZE(iq_lut_core0_rev0), 26, 320, 32},
-	{&iq_lut_core1_rev0, ARRAY_SIZE(iq_lut_core1_rev0), 27, 320, 32},
-	{&loft_lut_core0_rev0, ARRAY_SIZE(loft_lut_core0_rev0), 26, 448, 16},
-	{&loft_lut_core1_rev0, ARRAY_SIZE(loft_lut_core1_rev0), 27, 448, 16},
+	{&bdi_tbl_rev0, sizeof(bdi_tbl_rev0) / sizeof(bdi_tbl_rev0[0]), 21, 0,
+	 16}
+	,
+	{&pltlut_tbl_rev0, sizeof(pltlut_tbl_rev0) / sizeof(pltlut_tbl_rev0[0]),
+	 20, 0, 32}
+	,
+	{&gainctrl_lut_core0_rev0,
+	 sizeof(gainctrl_lut_core0_rev0) / sizeof(gainctrl_lut_core0_rev0[0]),
+	 26, 192, 32}
+	,
+	{&gainctrl_lut_core1_rev0,
+	 sizeof(gainctrl_lut_core1_rev0) / sizeof(gainctrl_lut_core1_rev0[0]),
+	 27, 192, 32}
+	,
+
+	{&est_pwr_lut_core0_rev0,
+	 sizeof(est_pwr_lut_core0_rev0) / sizeof(est_pwr_lut_core0_rev0[0]), 26,
+	 0, 8}
+	,
+	{&est_pwr_lut_core1_rev0,
+	 sizeof(est_pwr_lut_core1_rev0) / sizeof(est_pwr_lut_core1_rev0[0]), 27,
+	 0, 8}
+	,
+	{&adj_pwr_lut_core0_rev0,
+	 sizeof(adj_pwr_lut_core0_rev0) / sizeof(adj_pwr_lut_core0_rev0[0]), 26,
+	 64, 8}
+	,
+	{&adj_pwr_lut_core1_rev0,
+	 sizeof(adj_pwr_lut_core1_rev0) / sizeof(adj_pwr_lut_core1_rev0[0]), 27,
+	 64, 8}
+	,
+	{&iq_lut_core0_rev0,
+	 sizeof(iq_lut_core0_rev0) / sizeof(iq_lut_core0_rev0[0]), 26, 320, 32}
+	,
+	{&iq_lut_core1_rev0,
+	 sizeof(iq_lut_core1_rev0) / sizeof(iq_lut_core1_rev0[0]), 27, 320, 32}
+	,
+	{&loft_lut_core0_rev0,
+	 sizeof(loft_lut_core0_rev0) / sizeof(loft_lut_core0_rev0[0]), 26, 448,
+	 16}
+	,
+	{&loft_lut_core1_rev0,
+	 sizeof(loft_lut_core1_rev0) / sizeof(loft_lut_core1_rev0[0]), 27, 448,
+	 16}
+	,
 };
 
 const struct phytbl_info mimophytbl_info_rev0[] = {
-	{&frame_struct_rev0, ARRAY_SIZE(frame_struct_rev0), 10, 0, 32},
-	{&frame_lut_rev0, ARRAY_SIZE(frame_lut_rev0), 24, 0, 8},
-	{&tmap_tbl_rev0, ARRAY_SIZE(tmap_tbl_rev0), 12, 0, 32},
-	{&tdtrn_tbl_rev0, ARRAY_SIZE(tdtrn_tbl_rev0), 14, 0, 32},
-	{&intlv_tbl_rev0, ARRAY_SIZE(intlv_tbl_rev0), 13, 0, 32},
-	{&pilot_tbl_rev0, ARRAY_SIZE(pilot_tbl_rev0), 11, 0, 16},
-	{&tdi_tbl20_ant0_rev0, ARRAY_SIZE(tdi_tbl20_ant0_rev0), 19, 128, 32},
-	{&tdi_tbl20_ant1_rev0, ARRAY_SIZE(tdi_tbl20_ant1_rev0), 19, 256, 32},
-	{&tdi_tbl40_ant0_rev0, ARRAY_SIZE(tdi_tbl40_ant0_rev0), 19, 640, 32},
-	{&tdi_tbl40_ant1_rev0, ARRAY_SIZE(tdi_tbl40_ant1_rev0), 19, 768, 32},
-	{&chanest_tbl_rev0, ARRAY_SIZE(chanest_tbl_rev0), 22, 0, 32},
-	{&mcs_tbl_rev0, ARRAY_SIZE(mcs_tbl_rev0), 18, 0, 8},
-	{&noise_var_tbl0_rev0, ARRAY_SIZE(noise_var_tbl0_rev0), 16, 0, 32},
-	{&noise_var_tbl1_rev0, ARRAY_SIZE(noise_var_tbl1_rev0), 16, 128, 32},
-};
-
-const u32 mimophytbl_info_sz_rev0 = ARRAY_SIZE(mimophytbl_info_rev0);
-const u32 mimophytbl_info_sz_rev0_volatile = ARRAY_SIZE(mimophytbl_info_rev0_volatile);
+	{&frame_struct_rev0,
+	 sizeof(frame_struct_rev0) / sizeof(frame_struct_rev0[0]), 10, 0, 32}
+	,
+	{&frame_lut_rev0, sizeof(frame_lut_rev0) / sizeof(frame_lut_rev0[0]),
+	 24, 0, 8}
+	,
+	{&tmap_tbl_rev0, sizeof(tmap_tbl_rev0) / sizeof(tmap_tbl_rev0[0]), 12,
+	 0, 32}
+	,
+	{&tdtrn_tbl_rev0, sizeof(tdtrn_tbl_rev0) / sizeof(tdtrn_tbl_rev0[0]),
+	 14, 0, 32}
+	,
+	{&intlv_tbl_rev0, sizeof(intlv_tbl_rev0) / sizeof(intlv_tbl_rev0[0]),
+	 13, 0, 32}
+	,
+	{&pilot_tbl_rev0, sizeof(pilot_tbl_rev0) / sizeof(pilot_tbl_rev0[0]),
+	 11, 0, 16}
+	,
+	{&tdi_tbl20_ant0_rev0,
+	 sizeof(tdi_tbl20_ant0_rev0) / sizeof(tdi_tbl20_ant0_rev0[0]), 19, 128,
+	 32}
+	,
+	{&tdi_tbl20_ant1_rev0,
+	 sizeof(tdi_tbl20_ant1_rev0) / sizeof(tdi_tbl20_ant1_rev0[0]), 19, 256,
+	 32}
+	,
+	{&tdi_tbl40_ant0_rev0,
+	 sizeof(tdi_tbl40_ant0_rev0) / sizeof(tdi_tbl40_ant0_rev0[0]), 19, 640,
+	 32}
+	,
+	{&tdi_tbl40_ant1_rev0,
+	 sizeof(tdi_tbl40_ant1_rev0) / sizeof(tdi_tbl40_ant1_rev0[0]), 19, 768,
+	 32}
+	,
+	{&chanest_tbl_rev0,
+	 sizeof(chanest_tbl_rev0) / sizeof(chanest_tbl_rev0[0]), 22, 0, 32}
+	,
+	{&mcs_tbl_rev0, sizeof(mcs_tbl_rev0) / sizeof(mcs_tbl_rev0[0]), 18, 0,
+	 8}
+	,
+	{&noise_var_tbl0_rev0,
+	 sizeof(noise_var_tbl0_rev0) / sizeof(noise_var_tbl0_rev0[0]), 16, 0,
+	 32}
+	,
+	{&noise_var_tbl1_rev0,
+	 sizeof(noise_var_tbl1_rev0) / sizeof(noise_var_tbl1_rev0[0]), 16, 128,
+	 32}
+	,
+};
+
+const u32 mimophytbl_info_sz_rev0 =
+	sizeof(mimophytbl_info_rev0) / sizeof(mimophytbl_info_rev0[0]);
+const u32 mimophytbl_info_sz_rev0_volatile =
+	sizeof(mimophytbl_info_rev0_volatile) /
+	sizeof(mimophytbl_info_rev0_volatile[0]);
 
 static const u16 ant_swctrl_tbl_rev3[] = {
 	0x0082,
@@ -9294,53 +9363,132 @@
 };
 
 const struct phytbl_info mimophytbl_info_rev3_volatile[] = {
-	{&ant_swctrl_tbl_rev3, ARRAY_SIZE(ant_swctrl_tbl_rev3), 9, 0, 16},
+	{&ant_swctrl_tbl_rev3,
+	 sizeof(ant_swctrl_tbl_rev3) / sizeof(ant_swctrl_tbl_rev3[0]), 9, 0, 16}
+	,
 };
 
 const struct phytbl_info mimophytbl_info_rev3_volatile1[] = {
-	{&ant_swctrl_tbl_rev3_1, ARRAY_SIZE(ant_swctrl_tbl_rev3_1), 9, 0, 16},
+	{&ant_swctrl_tbl_rev3_1,
+	 sizeof(ant_swctrl_tbl_rev3_1) / sizeof(ant_swctrl_tbl_rev3_1[0]), 9, 0,
+	 16}
+	,
 };
 
 const struct phytbl_info mimophytbl_info_rev3_volatile2[] = {
-	{&ant_swctrl_tbl_rev3_2, ARRAY_SIZE(ant_swctrl_tbl_rev3_2), 9, 0, 16},
+	{&ant_swctrl_tbl_rev3_2,
+	 sizeof(ant_swctrl_tbl_rev3_2) / sizeof(ant_swctrl_tbl_rev3_2[0]), 9, 0,
+	 16}
+	,
 };
 
 const struct phytbl_info mimophytbl_info_rev3_volatile3[] = {
-	{&ant_swctrl_tbl_rev3_3, ARRAY_SIZE(ant_swctrl_tbl_rev3_3), 9, 0, 16},
+	{&ant_swctrl_tbl_rev3_3,
+	 sizeof(ant_swctrl_tbl_rev3_3) / sizeof(ant_swctrl_tbl_rev3_3[0]), 9, 0,
+	 16}
+	,
 };
 
 const struct phytbl_info mimophytbl_info_rev3[] = {
-	{&frame_struct_rev3, ARRAY_SIZE(frame_struct_rev3), 10, 0, 32},
-	{&pilot_tbl_rev3, ARRAY_SIZE(pilot_tbl_rev3), 11, 0, 16},
-	{&tmap_tbl_rev3, ARRAY_SIZE(tmap_tbl_rev3), 12, 0, 32},
-	{&intlv_tbl_rev3, ARRAY_SIZE(intlv_tbl_rev3), 13, 0, 32},
-	{&tdtrn_tbl_rev3, ARRAY_SIZE(tdtrn_tbl_rev3), 14, 0, 32},
-	{&noise_var_tbl_rev3, ARRAY_SIZE(noise_var_tbl_rev3), 16, 0, 32},
-	{&mcs_tbl_rev3, ARRAY_SIZE(mcs_tbl_rev3), 18, 0, 16},
-	{&tdi_tbl20_ant0_rev3, ARRAY_SIZE(tdi_tbl20_ant0_rev3), 19, 128, 32},
-	{&tdi_tbl20_ant1_rev3, ARRAY_SIZE(tdi_tbl20_ant1_rev3), 19, 256, 32},
-	{&tdi_tbl40_ant0_rev3, ARRAY_SIZE(tdi_tbl40_ant0_rev3), 19, 640, 32},
-	{&tdi_tbl40_ant1_rev3, ARRAY_SIZE(tdi_tbl40_ant1_rev3), 19, 768, 32},
-	{&pltlut_tbl_rev3, ARRAY_SIZE(pltlut_tbl_rev3), 20, 0, 32},
-	{&chanest_tbl_rev3, ARRAY_SIZE(chanest_tbl_rev3), 22, 0, 32},
-	{&frame_lut_rev3, ARRAY_SIZE(frame_lut_rev3), 24, 0, 8},
-	{&est_pwr_lut_core0_rev3, ARRAY_SIZE(est_pwr_lut_core0_rev3), 26, 0, 8},
-	{&est_pwr_lut_core1_rev3, ARRAY_SIZE(est_pwr_lut_core1_rev3), 27, 0, 8},
-	{&adj_pwr_lut_core0_rev3, ARRAY_SIZE(adj_pwr_lut_core0_rev3), 26, 64, 8},
-	{&adj_pwr_lut_core1_rev3, ARRAY_SIZE(adj_pwr_lut_core1_rev3), 27, 64, 8},
-	{&gainctrl_lut_core0_rev3, ARRAY_SIZE(gainctrl_lut_core0_rev3), 26, 192, 32},
-	{&gainctrl_lut_core1_rev3, ARRAY_SIZE(gainctrl_lut_core1_rev3), 27, 192, 32},
-	{&iq_lut_core0_rev3, ARRAY_SIZE(iq_lut_core0_rev3), 26, 320, 32},
-	{&iq_lut_core1_rev3, ARRAY_SIZE(iq_lut_core1_rev3), 27, 320, 32},
-	{&loft_lut_core0_rev3, ARRAY_SIZE(loft_lut_core0_rev3), 26, 448, 16},
-	{&loft_lut_core1_rev3, ARRAY_SIZE(loft_lut_core1_rev3), 27, 448, 16}
-};
-
-const u32 mimophytbl_info_sz_rev3 = ARRAY_SIZE(mimophytbl_info_rev3);
-const u32 mimophytbl_info_sz_rev3_volatile = ARRAY_SIZE(mimophytbl_info_rev3_volatile);
-const u32 mimophytbl_info_sz_rev3_volatile1 = ARRAY_SIZE(mimophytbl_info_rev3_volatile1);
-const u32 mimophytbl_info_sz_rev3_volatile2 = ARRAY_SIZE(mimophytbl_info_rev3_volatile2);
-const u32 mimophytbl_info_sz_rev3_volatile3 = ARRAY_SIZE(mimophytbl_info_rev3_volatile3);
+	{&frame_struct_rev3,
+	 sizeof(frame_struct_rev3) / sizeof(frame_struct_rev3[0]), 10, 0, 32}
+	,
+	{&pilot_tbl_rev3, sizeof(pilot_tbl_rev3) / sizeof(pilot_tbl_rev3[0]),
+	 11, 0, 16}
+	,
+	{&tmap_tbl_rev3, sizeof(tmap_tbl_rev3) / sizeof(tmap_tbl_rev3[0]), 12,
+	 0, 32}
+	,
+	{&intlv_tbl_rev3, sizeof(intlv_tbl_rev3) / sizeof(intlv_tbl_rev3[0]),
+	 13, 0, 32}
+	,
+	{&tdtrn_tbl_rev3, sizeof(tdtrn_tbl_rev3) / sizeof(tdtrn_tbl_rev3[0]),
+	 14, 0, 32}
+	,
+	{&noise_var_tbl_rev3,
+	 sizeof(noise_var_tbl_rev3) / sizeof(noise_var_tbl_rev3[0]), 16, 0, 32}
+	,
+	{&mcs_tbl_rev3, sizeof(mcs_tbl_rev3) / sizeof(mcs_tbl_rev3[0]), 18, 0,
+	 16}
+	,
+	{&tdi_tbl20_ant0_rev3,
+	 sizeof(tdi_tbl20_ant0_rev3) / sizeof(tdi_tbl20_ant0_rev3[0]), 19, 128,
+	 32}
+	,
+	{&tdi_tbl20_ant1_rev3,
+	 sizeof(tdi_tbl20_ant1_rev3) / sizeof(tdi_tbl20_ant1_rev3[0]), 19, 256,
+	 32}
+	,
+	{&tdi_tbl40_ant0_rev3,
+	 sizeof(tdi_tbl40_ant0_rev3) / sizeof(tdi_tbl40_ant0_rev3[0]), 19, 640,
+	 32}
+	,
+	{&tdi_tbl40_ant1_rev3,
+	 sizeof(tdi_tbl40_ant1_rev3) / sizeof(tdi_tbl40_ant1_rev3[0]), 19, 768,
+	 32}
+	,
+	{&pltlut_tbl_rev3, sizeof(pltlut_tbl_rev3) / sizeof(pltlut_tbl_rev3[0]),
+	 20, 0, 32}
+	,
+	{&chanest_tbl_rev3,
+	 sizeof(chanest_tbl_rev3) / sizeof(chanest_tbl_rev3[0]), 22, 0, 32}
+	,
+	{&frame_lut_rev3, sizeof(frame_lut_rev3) / sizeof(frame_lut_rev3[0]),
+	 24, 0, 8}
+	,
+	{&est_pwr_lut_core0_rev3,
+	 sizeof(est_pwr_lut_core0_rev3) / sizeof(est_pwr_lut_core0_rev3[0]), 26,
+	 0, 8}
+	,
+	{&est_pwr_lut_core1_rev3,
+	 sizeof(est_pwr_lut_core1_rev3) / sizeof(est_pwr_lut_core1_rev3[0]), 27,
+	 0, 8}
+	,
+	{&adj_pwr_lut_core0_rev3,
+	 sizeof(adj_pwr_lut_core0_rev3) / sizeof(adj_pwr_lut_core0_rev3[0]), 26,
+	 64, 8}
+	,
+	{&adj_pwr_lut_core1_rev3,
+	 sizeof(adj_pwr_lut_core1_rev3) / sizeof(adj_pwr_lut_core1_rev3[0]), 27,
+	 64, 8}
+	,
+	{&gainctrl_lut_core0_rev3,
+	 sizeof(gainctrl_lut_core0_rev3) / sizeof(gainctrl_lut_core0_rev3[0]),
+	 26, 192, 32}
+	,
+	{&gainctrl_lut_core1_rev3,
+	 sizeof(gainctrl_lut_core1_rev3) / sizeof(gainctrl_lut_core1_rev3[0]),
+	 27, 192, 32}
+	,
+	{&iq_lut_core0_rev3,
+	 sizeof(iq_lut_core0_rev3) / sizeof(iq_lut_core0_rev3[0]), 26, 320, 32}
+	,
+	{&iq_lut_core1_rev3,
+	 sizeof(iq_lut_core1_rev3) / sizeof(iq_lut_core1_rev3[0]), 27, 320, 32}
+	,
+	{&loft_lut_core0_rev3,
+	 sizeof(loft_lut_core0_rev3) / sizeof(loft_lut_core0_rev3[0]), 26, 448,
+	 16}
+	,
+	{&loft_lut_core1_rev3,
+	 sizeof(loft_lut_core1_rev3) / sizeof(loft_lut_core1_rev3[0]), 27, 448,
+	 16}
+};
+
+const u32 mimophytbl_info_sz_rev3 =
+	sizeof(mimophytbl_info_rev3) / sizeof(mimophytbl_info_rev3[0]);
+const u32 mimophytbl_info_sz_rev3_volatile =
+	sizeof(mimophytbl_info_rev3_volatile) /
+	sizeof(mimophytbl_info_rev3_volatile[0]);
+const u32 mimophytbl_info_sz_rev3_volatile1 =
+	sizeof(mimophytbl_info_rev3_volatile1) /
+	sizeof(mimophytbl_info_rev3_volatile1[0]);
+const u32 mimophytbl_info_sz_rev3_volatile2 =
+	sizeof(mimophytbl_info_rev3_volatile2) /
+	sizeof(mimophytbl_info_rev3_volatile2[0]);
+const u32 mimophytbl_info_sz_rev3_volatile3 =
+	sizeof(mimophytbl_info_rev3_volatile3) /
+	sizeof(mimophytbl_info_rev3_volatile3[0]);
 
 static const u32 tmap_tbl_rev7[] = {
 	0x8a88aa80,
@@ -10321,58 +10469,162 @@
 };
 
 const struct phytbl_info mimophytbl_info_rev7[] = {
-	{&frame_struct_rev3, ARRAY_SIZE(frame_struct_rev3), 10, 0, 32},
-	{&pilot_tbl_rev3, ARRAY_SIZE(pilot_tbl_rev3), 11, 0, 16},
-	{&tmap_tbl_rev7, ARRAY_SIZE(tmap_tbl_rev7), 12, 0, 32},
-	{&intlv_tbl_rev3, ARRAY_SIZE(intlv_tbl_rev3), 13, 0, 32},
-	{&tdtrn_tbl_rev3, ARRAY_SIZE(tdtrn_tbl_rev3), 14, 0, 32},
-	{&noise_var_tbl_rev7, ARRAY_SIZE(noise_var_tbl_rev7), 16, 0, 32},
-	{&mcs_tbl_rev3, ARRAY_SIZE(mcs_tbl_rev3), 18, 0, 16},
-	{&tdi_tbl20_ant0_rev3, ARRAY_SIZE(tdi_tbl20_ant0_rev3), 19, 128, 32},
-	{&tdi_tbl20_ant1_rev3, ARRAY_SIZE(tdi_tbl20_ant1_rev3), 19, 256, 32},
-	{&tdi_tbl40_ant0_rev3, ARRAY_SIZE(tdi_tbl40_ant0_rev3), 19, 640, 32},
-	{&tdi_tbl40_ant1_rev3, ARRAY_SIZE(tdi_tbl40_ant1_rev3), 19, 768, 32},
-	{&pltlut_tbl_rev3, ARRAY_SIZE(pltlut_tbl_rev3), 20, 0, 32},
-	{&chanest_tbl_rev3, ARRAY_SIZE(chanest_tbl_rev3), 22, 0, 32},
-	{&frame_lut_rev3, ARRAY_SIZE(frame_lut_rev3), 24, 0, 8},
-	{&est_pwr_lut_core0_rev3, ARRAY_SIZE(est_pwr_lut_core0_rev3), 26, 0, 8},
-	{&est_pwr_lut_core1_rev3, ARRAY_SIZE(est_pwr_lut_core1_rev3), 27, 0, 8},
-	{&adj_pwr_lut_core0_rev3, ARRAY_SIZE(adj_pwr_lut_core0_rev3), 26, 64, 8},
-	{&adj_pwr_lut_core1_rev3, ARRAY_SIZE(adj_pwr_lut_core1_rev3), 27, 64, 8},
-	{&gainctrl_lut_core0_rev3, ARRAY_SIZE(gainctrl_lut_core0_rev3), 26, 192, 32},
-	{&gainctrl_lut_core1_rev3, ARRAY_SIZE(gainctrl_lut_core1_rev3), 27, 192, 32},
-	{&iq_lut_core0_rev3, ARRAY_SIZE(iq_lut_core0_rev3), 26, 320, 32},
-	{&iq_lut_core1_rev3, ARRAY_SIZE(iq_lut_core1_rev3), 27, 320, 32},
-	{&loft_lut_core0_rev3, ARRAY_SIZE(loft_lut_core0_rev3), 26, 448, 16},
-	{&loft_lut_core1_rev3, ARRAY_SIZE(loft_lut_core1_rev3), 27, 448, 16},
+	{&frame_struct_rev3,
+	 sizeof(frame_struct_rev3) / sizeof(frame_struct_rev3[0]), 10, 0, 32}
+	,
+	{&pilot_tbl_rev3, sizeof(pilot_tbl_rev3) / sizeof(pilot_tbl_rev3[0]),
+	 11, 0, 16}
+	,
+	{&tmap_tbl_rev7, sizeof(tmap_tbl_rev7) / sizeof(tmap_tbl_rev7[0]), 12,
+	 0, 32}
+	,
+	{&intlv_tbl_rev3, sizeof(intlv_tbl_rev3) / sizeof(intlv_tbl_rev3[0]),
+	 13, 0, 32}
+	,
+	{&tdtrn_tbl_rev3, sizeof(tdtrn_tbl_rev3) / sizeof(tdtrn_tbl_rev3[0]),
+	 14, 0, 32}
+	,
+	{&noise_var_tbl_rev7,
+	 sizeof(noise_var_tbl_rev7) / sizeof(noise_var_tbl_rev7[0]), 16, 0, 32}
+	,
+	{&mcs_tbl_rev3, sizeof(mcs_tbl_rev3) / sizeof(mcs_tbl_rev3[0]), 18, 0,
+	 16}
+	,
+	{&tdi_tbl20_ant0_rev3,
+	 sizeof(tdi_tbl20_ant0_rev3) / sizeof(tdi_tbl20_ant0_rev3[0]), 19, 128,
+	 32}
+	,
+	{&tdi_tbl20_ant1_rev3,
+	 sizeof(tdi_tbl20_ant1_rev3) / sizeof(tdi_tbl20_ant1_rev3[0]), 19, 256,
+	 32}
+	,
+	{&tdi_tbl40_ant0_rev3,
+	 sizeof(tdi_tbl40_ant0_rev3) / sizeof(tdi_tbl40_ant0_rev3[0]), 19, 640,
+	 32}
+	,
+	{&tdi_tbl40_ant1_rev3,
+	 sizeof(tdi_tbl40_ant1_rev3) / sizeof(tdi_tbl40_ant1_rev3[0]), 19, 768,
+	 32}
+	,
+	{&pltlut_tbl_rev3, sizeof(pltlut_tbl_rev3) / sizeof(pltlut_tbl_rev3[0]),
+	 20, 0, 32}
+	,
+	{&chanest_tbl_rev3,
+	 sizeof(chanest_tbl_rev3) / sizeof(chanest_tbl_rev3[0]), 22, 0, 32}
+	,
+	{&frame_lut_rev3, sizeof(frame_lut_rev3) / sizeof(frame_lut_rev3[0]),
+	 24, 0, 8}
+	,
+	{&est_pwr_lut_core0_rev3,
+	 sizeof(est_pwr_lut_core0_rev3) / sizeof(est_pwr_lut_core0_rev3[0]), 26,
+	 0, 8}
+	,
+	{&est_pwr_lut_core1_rev3,
+	 sizeof(est_pwr_lut_core1_rev3) / sizeof(est_pwr_lut_core1_rev3[0]), 27,
+	 0, 8}
+	,
+	{&adj_pwr_lut_core0_rev3,
+	 sizeof(adj_pwr_lut_core0_rev3) / sizeof(adj_pwr_lut_core0_rev3[0]), 26,
+	 64, 8}
+	,
+	{&adj_pwr_lut_core1_rev3,
+	 sizeof(adj_pwr_lut_core1_rev3) / sizeof(adj_pwr_lut_core1_rev3[0]), 27,
+	 64, 8}
+	,
+	{&gainctrl_lut_core0_rev3,
+	 sizeof(gainctrl_lut_core0_rev3) / sizeof(gainctrl_lut_core0_rev3[0]),
+	 26, 192, 32}
+	,
+	{&gainctrl_lut_core1_rev3,
+	 sizeof(gainctrl_lut_core1_rev3) / sizeof(gainctrl_lut_core1_rev3[0]),
+	 27, 192, 32}
+	,
+	{&iq_lut_core0_rev3,
+	 sizeof(iq_lut_core0_rev3) / sizeof(iq_lut_core0_rev3[0]), 26, 320, 32}
+	,
+	{&iq_lut_core1_rev3,
+	 sizeof(iq_lut_core1_rev3) / sizeof(iq_lut_core1_rev3[0]), 27, 320, 32}
+	,
+	{&loft_lut_core0_rev3,
+	 sizeof(loft_lut_core0_rev3) / sizeof(loft_lut_core0_rev3[0]), 26, 448,
+	 16}
+	,
+	{&loft_lut_core1_rev3,
+	 sizeof(loft_lut_core1_rev3) / sizeof(loft_lut_core1_rev3[0]), 27, 448,
+	 16}
+	,
 	{&papd_comp_rfpwr_tbl_core0_rev3,
-	 ARRAY_SIZE(papd_comp_rfpwr_tbl_core0_rev3), 26, 576, 16},
+	 sizeof(papd_comp_rfpwr_tbl_core0_rev3) /
+	 sizeof(papd_comp_rfpwr_tbl_core0_rev3[0]), 26, 576, 16}
+	,
 	{&papd_comp_rfpwr_tbl_core1_rev3,
-	 ARRAY_SIZE(papd_comp_rfpwr_tbl_core1_rev3), 27, 576, 16},
+	 sizeof(papd_comp_rfpwr_tbl_core1_rev3) /
+	 sizeof(papd_comp_rfpwr_tbl_core1_rev3[0]), 27, 576, 16}
+	,
 	{&papd_comp_epsilon_tbl_core0_rev7,
-	 ARRAY_SIZE(papd_comp_epsilon_tbl_core0_rev7), 31, 0, 32},
+	 sizeof(papd_comp_epsilon_tbl_core0_rev7) /
+	 sizeof(papd_comp_epsilon_tbl_core0_rev7[0]), 31, 0, 32}
+	,
 	{&papd_cal_scalars_tbl_core0_rev7,
-	 ARRAY_SIZE(papd_cal_scalars_tbl_core0_rev7), 32, 0, 32},
+	 sizeof(papd_cal_scalars_tbl_core0_rev7) /
+	 sizeof(papd_cal_scalars_tbl_core0_rev7[0]), 32, 0, 32}
+	,
 	{&papd_comp_epsilon_tbl_core1_rev7,
-	 ARRAY_SIZE(papd_comp_epsilon_tbl_core1_rev7), 33, 0, 32},
+	 sizeof(papd_comp_epsilon_tbl_core1_rev7) /
+	 sizeof(papd_comp_epsilon_tbl_core1_rev7[0]), 33, 0, 32}
+	,
 	{&papd_cal_scalars_tbl_core1_rev7,
-	 ARRAY_SIZE(papd_cal_scalars_tbl_core1_rev7), 34, 0, 32},
+	 sizeof(papd_cal_scalars_tbl_core1_rev7) /
+	 sizeof(papd_cal_scalars_tbl_core1_rev7[0]), 34, 0, 32}
+	,
 };
 
-const u32 mimophytbl_info_sz_rev7 = ARRAY_SIZE(mimophytbl_info_rev7);
+const u32 mimophytbl_info_sz_rev7 =
+	sizeof(mimophytbl_info_rev7) / sizeof(mimophytbl_info_rev7[0]);
 
 const struct phytbl_info mimophytbl_info_rev16[] = {
-	{&noise_var_tbl_rev7, ARRAY_SIZE(noise_var_tbl_rev7), 16, 0, 32},
-	{&est_pwr_lut_core0_rev3, ARRAY_SIZE(est_pwr_lut_core0_rev3), 26, 0, 8},
-	{&est_pwr_lut_core1_rev3, ARRAY_SIZE(est_pwr_lut_core1_rev3), 27, 0, 8},
-	{&adj_pwr_lut_core0_rev3, ARRAY_SIZE(adj_pwr_lut_core0_rev3), 26, 64, 8},
-	{&adj_pwr_lut_core1_rev3, ARRAY_SIZE(adj_pwr_lut_core1_rev3), 27, 64, 8},
-	{&gainctrl_lut_core0_rev3, ARRAY_SIZE(gainctrl_lut_core0_rev3), 26, 192, 32},
-	{&gainctrl_lut_core1_rev3, ARRAY_SIZE(gainctrl_lut_core1_rev3), 27, 192, 32},
-	{&iq_lut_core0_rev3, ARRAY_SIZE(iq_lut_core0_rev3), 26, 320, 32},
-	{&iq_lut_core1_rev3, ARRAY_SIZE(iq_lut_core1_rev3), 27, 320, 32},
-	{&loft_lut_core0_rev3, ARRAY_SIZE(loft_lut_core0_rev3), 26, 448, 16},
-	{&loft_lut_core1_rev3, ARRAY_SIZE(loft_lut_core1_rev3), 27, 448, 16},
+	{&noise_var_tbl_rev7,
+	 sizeof(noise_var_tbl_rev7) / sizeof(noise_var_tbl_rev7[0]), 16, 0, 32}
+	,
+	{&est_pwr_lut_core0_rev3,
+	 sizeof(est_pwr_lut_core0_rev3) / sizeof(est_pwr_lut_core0_rev3[0]), 26,
+	 0, 8}
+	,
+	{&est_pwr_lut_core1_rev3,
+	 sizeof(est_pwr_lut_core1_rev3) / sizeof(est_pwr_lut_core1_rev3[0]), 27,
+	 0, 8}
+	,
+	{&adj_pwr_lut_core0_rev3,
+	 sizeof(adj_pwr_lut_core0_rev3) / sizeof(adj_pwr_lut_core0_rev3[0]), 26,
+	 64, 8}
+	,
+	{&adj_pwr_lut_core1_rev3,
+	 sizeof(adj_pwr_lut_core1_rev3) / sizeof(adj_pwr_lut_core1_rev3[0]), 27,
+	 64, 8}
+	,
+	{&gainctrl_lut_core0_rev3,
+	 sizeof(gainctrl_lut_core0_rev3) / sizeof(gainctrl_lut_core0_rev3[0]),
+	 26, 192, 32}
+	,
+	{&gainctrl_lut_core1_rev3,
+	 sizeof(gainctrl_lut_core1_rev3) / sizeof(gainctrl_lut_core1_rev3[0]),
+	 27, 192, 32}
+	,
+	{&iq_lut_core0_rev3,
+	 sizeof(iq_lut_core0_rev3) / sizeof(iq_lut_core0_rev3[0]), 26, 320, 32}
+	,
+	{&iq_lut_core1_rev3,
+	 sizeof(iq_lut_core1_rev3) / sizeof(iq_lut_core1_rev3[0]), 27, 320, 32}
+	,
+	{&loft_lut_core0_rev3,
+	 sizeof(loft_lut_core0_rev3) / sizeof(loft_lut_core0_rev3[0]), 26, 448,
+	 16}
+	,
+	{&loft_lut_core1_rev3,
+	 sizeof(loft_lut_core1_rev3) / sizeof(loft_lut_core1_rev3[0]), 27, 448,
+	 16}
+	,
 };
 
-const u32 mimophytbl_info_sz_rev16 = ARRAY_SIZE(mimophytbl_info_rev16);
+const u32 mimophytbl_info_sz_rev16 =
+	sizeof(mimophytbl_info_rev16) / sizeof(mimophytbl_info_rev16[0]);
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c b/drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c
--- a/drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c	2020-06-19 21:00:31.420227716 -0400
@@ -77,8 +77,6 @@
 		return BRCMU_CHSPEC_D11AC_BW_40;
 	case BRCMU_CHAN_BW_80:
 		return BRCMU_CHSPEC_D11AC_BW_80;
-	case BRCMU_CHAN_BW_160:
-		return BRCMU_CHSPEC_D11AC_BW_160;
 	default:
 		WARN_ON(1);
 	}
@@ -192,41 +190,8 @@
 			break;
 		}
 		break;
-	case BRCMU_CHSPEC_D11AC_BW_160:
-		ch->bw = BRCMU_CHAN_BW_160;
-		ch->sb = brcmu_maskget16(ch->chspec, BRCMU_CHSPEC_D11AC_SB_MASK,
-					 BRCMU_CHSPEC_D11AC_SB_SHIFT);
-		switch (ch->sb) {
-		case BRCMU_CHAN_SB_LLL:
-			ch->control_ch_num -= CH_70MHZ_APART;
-			break;
-		case BRCMU_CHAN_SB_LLU:
-			ch->control_ch_num -= CH_50MHZ_APART;
-			break;
-		case BRCMU_CHAN_SB_LUL:
-			ch->control_ch_num -= CH_30MHZ_APART;
-			break;
-		case BRCMU_CHAN_SB_LUU:
-			ch->control_ch_num -= CH_10MHZ_APART;
-			break;
-		case BRCMU_CHAN_SB_ULL:
-			ch->control_ch_num += CH_10MHZ_APART;
-			break;
-		case BRCMU_CHAN_SB_ULU:
-			ch->control_ch_num += CH_30MHZ_APART;
-			break;
-		case BRCMU_CHAN_SB_UUL:
-			ch->control_ch_num += CH_50MHZ_APART;
-			break;
-		case BRCMU_CHAN_SB_UUU:
-			ch->control_ch_num += CH_70MHZ_APART;
-			break;
-		default:
-			WARN_ON_ONCE(1);
-			break;
-		}
-		break;
 	case BRCMU_CHSPEC_D11AC_BW_8080:
+	case BRCMU_CHSPEC_D11AC_BW_160:
 	default:
 		WARN_ON_ONCE(1);
 		break;
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h
--- a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h	2020-06-19 21:00:31.420227716 -0400
@@ -57,9 +57,10 @@
 #define BRCM_CC_43602_CHIP_ID		43602
 #define BRCM_CC_4365_CHIP_ID		0x4365
 #define BRCM_CC_4366_CHIP_ID		0x4366
-#define BRCM_CC_43664_CHIP_ID		43664
 #define BRCM_CC_4371_CHIP_ID		0x4371
 #define CY_CC_4373_CHIP_ID		0x4373
+#define CY_CC_43012_CHIP_ID		43012
+#define CY_CC_89459_CHIP_ID             0x4355
 
 /* USB Device IDs */
 #define BRCM_USB_43143_DEVICE_ID	0xbd1e
@@ -74,9 +75,12 @@
 /* PCIE Device IDs */
 #define BRCM_PCIE_4350_DEVICE_ID	0x43a3
 #define BRCM_PCIE_4354_DEVICE_ID	0x43df
+#define BRCM_PCIE_4354_RAW_DEVICE_ID	0x4354
+#define BRCM_PCIE_4355_DEVICE_ID        0x4355
 #define BRCM_PCIE_4356_DEVICE_ID	0x43ec
 #define BRCM_PCIE_43567_DEVICE_ID	0x43d3
 #define BRCM_PCIE_43570_DEVICE_ID	0x43d9
+#define BRCM_PCIE_43570_RAW_DEVICE_ID	0xaa31
 #define BRCM_PCIE_4358_DEVICE_ID	0x43e9
 #define BRCM_PCIE_4359_DEVICE_ID	0x43ef
 #define BRCM_PCIE_43602_DEVICE_ID	0x43ba
@@ -90,7 +94,8 @@
 #define BRCM_PCIE_4366_2G_DEVICE_ID	0x43c4
 #define BRCM_PCIE_4366_5G_DEVICE_ID	0x43c5
 #define BRCM_PCIE_4371_DEVICE_ID	0x440d
-
+#define CY_PCIE_89459_DEVICE_ID         0x4415
+#define CY_PCIE_89459_RAW_DEVICE_ID     0x4355
 
 /* brcmsmac IDs */
 #define BCM4313_D11N2G_ID	0x4727	/* 4313 802.11n 2.4G device */
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h
--- a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h	2020-06-19 21:00:31.420227716 -0400
@@ -29,8 +29,6 @@
 #define CH_UPPER_SB			0x01
 #define CH_LOWER_SB			0x02
 #define CH_EWA_VALID			0x04
-#define CH_70MHZ_APART			14
-#define CH_50MHZ_APART			10
 #define CH_30MHZ_APART			6
 #define CH_20MHZ_APART			4
 #define CH_10MHZ_APART			2
@@ -239,8 +237,11 @@
 #define WPA2_AUTH_RESERVED4	0x0400
 #define WPA2_AUTH_RESERVED5	0x0800
 #define WPA2_AUTH_1X_SHA256	0x1000  /* 1X with SHA256 key derivation */
+#define WPA2_AUTH_FT		0x4000	/* Fast BSS Transition */
 #define WPA2_AUTH_PSK_SHA256	0x8000	/* PSK with SHA256 key derivation */
 
+#define WPA3_AUTH_SAE_PSK	0x40000	/* SAE with 4-way handshake */
+
 #define DOT11_DEFAULT_RTS_LEN		2347
 #define DOT11_DEFAULT_FRAG_LEN		2346
 
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h b/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h
--- a/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h	2020-06-19 21:00:31.420227716 -0400
@@ -225,8 +225,197 @@
 	u32 PAD[3];
 	u32 retention_grpidx;       /* 0x680 */
 	u32 retention_grpctl;       /* 0x684 */
-	u32 PAD[94];
-	u16 sromotp[768];
+	u32 mac_res_req_timer;      /* 0x688 */
+	u32 mac_res_req_mask;       /* 0x68c */
+	u32 PAD[18];
+	u32 pmucontrol_ext;         /* 0x6d8 */
+	u32 slowclkperiod;          /* 0x6dc */
+	u32 PAD[8];
+	u32 pmuintmask0;            /* 0x700 */
+	u32 pmuintmask1;            /* 0x704 */
+	u32 PAD[14];
+	u32 pmuintstatus;           /* 0x740 */
+	u32 extwakeupstatus;        /* 0x744 */
+	u32 watchdog_res_mask;      /* 0x748 */
+	u32 swscratch;              /* 0x750 */
+	u32 PAD[3];
+	u32 extwakemask[2];         /* 0x760-0x764 */
+	u32 PAD[2];
+	u32 extwakereqmask[2];      /* 0x770-0x774 */
+	u32 PAD[2];
+	u32 pmuintctrl0;            /* 0x780 */
+	u32 pmuintctrl1;            /* 0x784 */
+	u32 PAD[2];
+	u32 extwakectrl[2];         /* 0x790 */
+};
+
+#define CHIPGCIREGOFFS(field) offsetof(struct chipgciregs, field)
+
+struct chipgciregs {
+	u32 gci_corecaps0;                             /* 0x000 */
+	u32 gci_corecaps1;                             /* 0x004 */
+	u32 gci_corecaps2;                             /* 0x008 */
+	u32 gci_corectrl;                              /* 0x00c */
+	u32 gci_corestat;                              /* 0x010 */
+	u32 gci_intstat;                               /* 0x014 */
+	u32 gci_intmask;                               /* 0x018 */
+	u32 gci_wakemask;                              /* 0x01c */
+	u32 gci_levelintstat;                          /* 0x020 */
+	u32 gci_eventintstat;                          /* 0x024 */
+	u32 gci_wakelevelintstat;                      /* 0x028 */
+	u32 gci_wakeeventintstat;                      /* 0x02c */
+	u32 semaphoreintstatus;                        /* 0x030 */
+	u32 semaphoreintmask;                          /* 0x034 */
+	u32 semaphorerequest;                          /* 0x038 */
+	u32 semaphorereserve;                          /* 0x03c */
+	u32 gci_indirect_addr;                         /* 0x040 */
+	u32 gci_gpioctl;                               /* 0x044 */
+	u32 gci_gpiostatus;                            /* 0x048 */
+	u32 gci_gpiomask;                              /* 0x04c */
+	u32 eventsummary;                              /* 0x050 */
+	u32 gci_miscctl;                               /* 0x054 */
+	u32 gci_gpiointmask;                           /* 0x058 */
+	u32 gci_gpiowakemask;                          /* 0x05c */
+	u32 gci_input[32];                             /* 0x060 */
+	u32 gci_event[32];                             /* 0x0e0 */
+	u32 gci_output[4];                             /* 0x160 */
+	u32 gci_control_0;                             /* 0x170 */
+	u32 gci_control_1;                             /* 0x174 */
+	u32 gci_intpolreg;                             /* 0x178 */
+	u32 gci_levelintmask;                          /* 0x17c */
+	u32 gci_eventintmask;                          /* 0x180 */
+	u32 wakelevelintmask;                          /* 0x184 */
+	u32 wakeeventintmask;                          /* 0x188 */
+	u32 hwmask;                                    /* 0x18c */
+	u32 PAD;
+	u32 gci_inbandeventintmask;                    /* 0x194 */
+	u32 PAD;
+	u32 gci_inbandeventstatus;                     /* 0x19c */
+	u32 gci_seciauxtx;                             /* 0x1a0 */
+	u32 gci_seciauxrx;                             /* 0x1a4 */
+	u32 gci_secitx_datatag;                        /* 0x1a8 */
+	u32 gci_secirx_datatag;                        /* 0x1ac */
+	u32 gci_secitx_datamask;                       /* 0x1b0 */
+	u32 gci_seciusef0tx_reg;                       /* 0x1b4 */
+	u32 gci_secif0tx_offset;                       /* 0x1b8 */
+	u32 gci_secif0rx_offset;                       /* 0x1bc */
+	u32 gci_secif1tx_offset;                       /* 0x1c0 */
+	u32 gci_rxfifo_common_ctrl;                    /* 0x1c4 */
+	u32 gci_rxfifoctrl;                            /* 0x1c8 */
+	u32 gci_hw_sema_status;                        /* 0x1cc */
+	u32 gci_seciuartescval;                        /* 0x1d0 */
+	u32 gic_seciuartautobaudctr;                   /* 0x1d4 */
+	u32 gci_secififolevel;                         /* 0x1d8 */
+	u32 gci_seciuartdata;                          /* 0x1dc */
+	u32 gci_secibauddiv;                           /* 0x1e0 */
+	u32 gci_secifcr;                               /* 0x1e4 */
+	u32 gci_secilcr;                               /* 0x1e8 */
+	u32 gci_secimcr;                               /* 0x1ec */
+	u32 gci_secilsr;                               /* 0x1f0 */
+	u32 gci_secimsr;                               /* 0x1f4 */
+	u32 gci_baudadj;                               /* 0x1f8 */
+	u32 gci_inbandintmask;                         /* 0x1fc */
+	u32 gci_chipctrl;                              /* 0x200 */
+	u32 gci_chipsts;                               /* 0x204 */
+	u32 gci_gpioout;                               /* 0x208 */
+	u32 gci_gpioout_read;                          /* 0x20C */
+	u32 gci_mpwaketx;                              /* 0x210 */
+	u32 gci_mpwakedetect;                          /* 0x214 */
+	u32 gci_seciin_ctrl;                           /* 0x218 */
+	u32 gci_seciout_ctrl;                          /* 0x21C */
+	u32 gci_seciin_auxfifo_en;                     /* 0x220 */
+	u32 gci_seciout_txen_txbr;                     /* 0x224 */
+	u32 gci_seciin_rxbrstatus;                     /* 0x228 */
+	u32 gci_seciin_rxerrstatus;                    /* 0x22C */
+	u32 gci_seciin_fcstatus;                       /* 0x230 */
+	u32 gci_seciout_txstatus;                      /* 0x234 */
+	u32 gci_seciout_txbrstatus;                    /* 0x238 */
+	u32 wlan_mem_info;                             /* 0x23C */
+	u32 wlan_bankxinfo;                            /* 0x240 */
+	u32 bt_smem_select;                            /* 0x244 */
+	u32 bt_smem_stby;                              /* 0x248 */
+	u32 bt_smem_status;                            /* 0x24C */
+	u32 wlan_bankxactivepda;                       /* 0x250 */
+	u32 wlan_bankxsleeppda;                        /* 0x254 */
+	u32 wlan_bankxkill;                            /* 0x258 */
+	u32 PAD[41];
+	u32 gci_chipid;                                /* 0x300 */
+	u32 PAD[3];
+	u32 otpstatus;                                 /* 0x310 */
+	u32 otpcontrol;                                /* 0x314 */
+	u32 otpprog;                                   /* 0x318 */
+	u32 otplayout;                                 /* 0x31c */
+	u32 otplayoutextension;                        /* 0x320 */
+	u32 otpcontrol1;                               /* 0x324 */
+	u32 otpprogdata;                               /* 0x328 */
+	u32 PAD[52];
+	u32 otpECCstatus;                              /* 0x3FC */
+	u32 PAD[512];
+	u32 lhl_core_capab_adr;                        /* 0xC00 */
+	u32 lhl_main_ctl_adr;                          /* 0xC04 */
+	u32 lhl_pmu_ctl_adr;                           /* 0xC08 */
+	u32 lhl_extlpo_ctl_adr;                        /* 0xC0C */
+	u32 lpo_ctl_adr;                               /* 0xC10 */
+	u32 lhl_lpo2_ctl_adr;                          /* 0xC14 */
+	u32 lhl_osc32k_ctl_adr;                        /* 0xC18 */
+	u32 lhl_clk_status_adr;                        /* 0xC1C */
+	u32 lhl_clk_det_ctl_adr;                       /* 0xC20 */
+	u32 lhl_clk_sel_adr;                           /* 0xC24 */
+	u32 hidoff_cnt_adr[2];                         /* 0xC28-0xC2C */
+	u32 lhl_autoclk_ctl_adr;                       /* 0xC30 */
+	u32 PAD;
+	u32 lhl_hibtim_adr;                            /* 0xC38 */
+	u32 lhl_wl_ilp_val_adr;                        /* 0xC3C */
+	u32 lhl_wl_armtim0_intrp_adr;                  /* 0xC40 */
+	u32 lhl_wl_armtim0_st_adr;                     /* 0xC44 */
+	u32 lhl_wl_armtim0_adr;                        /* 0xC48 */
+	u32 PAD[9];
+	u32 lhl_wl_mactim0_intrp_adr;                  /* 0xC70 */
+	u32 lhl_wl_mactim0_st_adr;                     /* 0xC74 */
+	u32 lhl_wl_mactim_int0_adr;                    /* 0xC78 */
+	u32 lhl_wl_mactim_frac0_adr;                   /* 0xC7C */
+	u32 lhl_wl_mactim1_intrp_adr;                  /* 0xC80 */
+	u32 lhl_wl_mactim1_st_adr;                     /* 0xC84 */
+	u32 lhl_wl_mactim_int1_adr;                    /* 0xC88 */
+	u32 lhl_wl_mactim_frac1_adr;                   /* 0xC8C */
+	u32 PAD[8];
+	u32 gpio_int_en_port_adr[4];                   /* 0xCB0-0xCBC */
+	u32 gpio_int_st_port_adr[4];                   /* 0xCC0-0xCCC */
+	u32 gpio_ctrl_iocfg_p_adr[64];                 /* 0xCD0-0xDCC */
+	u32 gpio_gctrl_iocfg_p0_p39_adr;               /* 0xDD0 */
+	u32 gpio_gdsctrl_iocfg_p0_p25_p30_p39_adr;     /* 0xDD4 */
+	u32 gpio_gdsctrl_iocfg_p26_p29_adr;            /* 0xDD8 */
+	u32 PAD[8];
+	u32 lhl_gpio_din0_adr;                         /* 0xDFC */
+	u32 lhl_gpio_din1_adr;                         /* 0xE00 */
+	u32 lhl_wkup_status_adr;                       /* 0xE04 */
+	u32 lhl_ctl_adr;                               /* 0xE08 */
+	u32 lhl_adc_ctl_adr;                           /* 0xE0C */
+	u32 lhl_qdxyz_in_dly_adr;                      /* 0xE10 */
+	u32 lhl_optctl_adr;                            /* 0xE14 */
+	u32 lhl_optct2_adr;                            /* 0xE18 */
+	u32 lhl_scanp_cntr_init_val_adr;               /* 0xE1C */
+	u32 lhl_opt_togg_val_adr[6];                   /* 0xE20-0xE34 */
+	u32 lhl_optx_smp_val_adr;                      /* 0xE38 */
+	u32 lhl_opty_smp_val_adr;                      /* 0xE3C */
+	u32 lhl_optz_smp_val_adr;                      /* 0xE40 */
+	u32 lhl_hidoff_keepstate_adr[3];               /* 0xE44-0xE4C */
+	u32 lhl_bt_slmboot_ctl0_adr[4];                /* 0xE50-0xE5C */
+	u32 lhl_wl_fw_ctl;                             /* 0xE60 */
+	u32 lhl_wl_hw_ctl_adr[2];                      /* 0xE64-0xE68 */
+	u32 lhl_bt_hw_ctl_adr;                         /* 0xE6C */
+	u32 lhl_top_pwrseq_en_adr;                     /* 0xE70 */
+	u32 lhl_top_pwrdn_ctl_adr;                     /* 0xE74 */
+	u32 lhl_top_pwrup_ctl_adr;                     /* 0xE78 */
+	u32 lhl_top_pwrseq_ctl_adr;                    /* 0xE7C */
+	u32 lhl_top_pwrdn2_ctl_adr;                    /* 0xE80 */
+	u32 lhl_top_pwrup2_ctl_adr;                    /* 0xE84 */
+	u32 wpt_regon_intrp_cfg_adr;                   /* 0xE88 */
+	u32 bt_regon_intrp_cfg_adr;                    /* 0xE8C */
+	u32 wl_regon_intrp_cfg_adr;                    /* 0xE90 */
+	u32 regon_intrp_st_adr;                        /* 0xE94 */
+	u32 regon_intrp_en_adr;                        /* 0xE98 */
+
 };
 
 /* chipid */
@@ -298,6 +487,8 @@
 * Maximum delay for the PMU state transition in us.
 * This is an upper bound intended for spinwaits etc.
 */
-#define PMU_MAX_TRANSITION_DLY	15000
+#define PMU_MAX_TRANSITION_DLY 15000
+
+#define DEFAULT_43012_MIN_RES_MASK 0x0f8bfe77
 
 #endif				/* _SBCHIPC_H */
diff -Naur --no-dereference a/drivers/net/wireless/broadcom/brcm80211/Kconfig b/drivers/net/wireless/broadcom/brcm80211/Kconfig
--- a/drivers/net/wireless/broadcom/brcm80211/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/net/wireless/broadcom/brcm80211/Kconfig	2020-06-19 21:00:31.416227715 -0400
@@ -60,6 +60,7 @@
 	bool "PCIE bus interface support for FullMAC driver"
 	depends on BRCMFMAC
 	depends on PCI
+	depends on HAS_DMA
 	select BRCMFMAC_PROTO_MSGBUF
 	select FW_LOADER
 	---help---
@@ -84,3 +85,14 @@
 	select WANT_DEV_COREDUMP
 	---help---
 	  Selecting this enables additional code for debug purposes.
+
+config BRCMFMAC_PCIE_BARWIN_SZ
+	bool "Custom PCIE BAR window size support for FullMAC driver"
+	depends on BRCMFMAC
+	depends on PCI
+	default n
+	---help---
+	  If you say Y here, the FMAC driver will use custom PCIE BAR
+	  window size. Say Y to allow developers to use custom PCIE
+	  BAR window size when HOST PCIE IP can support less then 4MB
+	  BAR window.
diff -Naur --no-dereference a/drivers/ntb/hw/epf/Kconfig b/drivers/ntb/hw/epf/Kconfig
--- a/drivers/ntb/hw/epf/Kconfig	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/ntb/hw/epf/Kconfig	2020-06-19 21:00:31.420227716 -0400
@@ -0,0 +1,5 @@
+config NTB_EPF
+	tristate "Generic EPF Non-Transparent Bridge support"
+	help
+	  This driver supports EPF NTB on configurable endpoint.
+	  If unsure, say N.
diff -Naur --no-dereference a/drivers/ntb/hw/epf/Makefile b/drivers/ntb/hw/epf/Makefile
--- a/drivers/ntb/hw/epf/Makefile	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/ntb/hw/epf/Makefile	2020-06-19 21:00:31.420227716 -0400
@@ -0,0 +1 @@
+obj-$(CONFIG_NTB_EPF) += ntb_hw_epf.o
diff -Naur --no-dereference a/drivers/ntb/hw/epf/ntb_hw_epf.c b/drivers/ntb/hw/epf/ntb_hw_epf.c
--- a/drivers/ntb/hw/epf/ntb_hw_epf.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/ntb/hw/epf/ntb_hw_epf.c	2020-06-19 21:00:31.420227716 -0400
@@ -0,0 +1,654 @@
+// SPDX-License-Identifier: GPL-2.0
+/**
+ * Host side endpoint driver to implement Non-Transparent Bridge functionality
+ *
+ * Copyright (C) 2019 Texas Instruments
+ * Author: Kishon Vijay Abraham I <kishon@ti.com>
+ */
+
+#include <linux/delay.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/slab.h>
+#include <linux/ntb.h>
+
+#define NTB_EPF_COMMAND		0x0
+#define CMD_CONFIGURE_DOORBELL	1
+#define CMD_CONFIGURE_MW	2
+#define CMD_LINK_UP		3
+
+#define NTB_EPF_ARGUMENT	0x4
+
+#define NTB_EPF_STATUS		0x8
+#define COMMAND_STATUS_OK	BIT(0)
+#define COMMAND_STATUS_ERROR	BIT(1)
+#define LINK_STATUS_UP		BIT(2)
+
+#define NTB_EPF_TOPOLOGY	0xc
+#define NTB_EPF_ADDR		0x10
+#define NTB_EPF_SIZE		0x18
+#define NTB_EPF_MW1_OFFSET	0x1c
+#define NTB_EPF_MW_COUNT	0x20
+#define NTB_EPF_SPAD_OFFSET	0x24
+#define NTB_EPF_SPAD_COUNT	0x28
+#define NTB_EPF_DB_ENTRY_SIZE	0x2c
+#define NTB_EPF_DB_DATA(n)	(0x30 + (n) * 4)
+
+#define NTB_MIN_DB_COUNT	2
+#define NTB_MAX_DB_COUNT	32
+#define NTB_MW_OFFSET		2
+
+enum pci_barno {
+	BAR_0,
+	BAR_1,
+	BAR_2,
+	BAR_3,
+	BAR_4,
+	BAR_5,
+};
+
+struct ntb_epf_dev {
+	struct ntb_dev ntb;
+
+	enum pci_barno ctrl_reg_bar;
+	enum pci_barno peer_spad_reg_bar;
+	enum pci_barno db_reg_bar;
+
+	unsigned int mw_count;
+	unsigned int spad_count;
+	unsigned int db_count;
+
+	void __iomem *ctrl_reg;
+	void __iomem *db_reg;
+	void __iomem *peer_spad_reg;
+
+	unsigned int self_spad;
+	unsigned int peer_spad;
+
+	int db_val;
+	u64 db_valid_mask;
+};
+
+#define ntb_ndev(__ntb) container_of(__ntb, struct ntb_epf_dev, ntb)
+
+struct ntb_epf_data {
+	/* BAR that contains both control region and self spad region */
+	enum pci_barno ctrl_reg_bar;
+	/* BAR that contains peer spad region */
+	enum pci_barno peer_spad_reg_bar;
+	/* BAR that contains Doorbell region and Memory window '1' */
+	enum pci_barno db_reg_bar;
+};
+
+static inline u32 ntb_epf_ctrl_readl(struct ntb_epf_dev *ndev, u32 offset)
+{
+	return readl(ndev->ctrl_reg + offset);
+}
+
+static inline void ntb_epf_ctrl_writel(struct ntb_epf_dev *ndev, u32 offset,
+				       u32 val)
+{
+	return writel(val, ndev->ctrl_reg + offset);
+}
+
+static int ntb_epf_send_command(struct ntb_epf_dev *ndev, u32 command,
+				u32 argument)
+{
+	ktime_t timeout;
+	bool timedout;
+	int ret = 0;
+	u32 status;
+
+	ntb_epf_ctrl_writel(ndev, NTB_EPF_ARGUMENT, argument);
+	ntb_epf_ctrl_writel(ndev, NTB_EPF_COMMAND, command);
+
+	/* wait 50ms */
+	timeout = ktime_add_ms(ktime_get(), 50);
+	while (1) {
+		timedout = ktime_after(ktime_get(), timeout);
+		status = ntb_epf_ctrl_readl(ndev, NTB_EPF_STATUS);
+
+		if (status & COMMAND_STATUS_ERROR) {
+			ret = -EINVAL;
+			break;
+		}
+
+		if (status & COMMAND_STATUS_OK)
+			break;
+
+		if (WARN_ON(timedout)) {
+			ret = -ETIMEDOUT;
+			break;
+		}
+
+		usleep_range(5, 10);
+	}
+
+	status &= ~(COMMAND_STATUS_ERROR | COMMAND_STATUS_OK);
+	ntb_epf_ctrl_writel(ndev, NTB_EPF_STATUS, status);
+
+	return ret;
+}
+
+static int ntb_epf_mw_to_bar(struct ntb_epf_dev *ndev, int idx)
+{
+	if (idx < 0 || idx > ndev->mw_count)
+		return -EINVAL;
+
+	return idx + 2;
+}
+
+static int ntb_epf_mw_count(struct ntb_dev *ntb, int pidx)
+{
+	if (pidx != NTB_DEF_PEER_IDX)
+		return -EINVAL;
+
+	return ntb_ndev(ntb)->mw_count;
+}
+
+static int ntb_epf_mw_get_align(struct ntb_dev *ntb, int pidx, int idx,
+				resource_size_t *addr_align,
+				resource_size_t *size_align,
+				resource_size_t *size_max)
+{
+	struct ntb_epf_dev *ndev = ntb_ndev(ntb);
+	int bar;
+
+	if (pidx != NTB_DEF_PEER_IDX)
+		return -EINVAL;
+
+	bar = ntb_epf_mw_to_bar(ndev, idx);
+	if (bar < 0)
+		return bar;
+
+	if (addr_align)
+		*addr_align = SZ_4K;
+
+	if (size_align)
+		*size_align = 1;
+
+	if (size_max)
+		*size_max = pci_resource_len(ndev->ntb.pdev, bar);
+
+	return 0;
+}
+
+static u64 ntb_epf_link_is_up(struct ntb_dev *ntb,
+			      enum ntb_speed *speed,
+			      enum ntb_width *width)
+{
+	struct ntb_epf_dev *ndev = ntb_ndev(ntb);
+	u32 status;
+
+	status = ntb_epf_ctrl_readl(ndev, NTB_EPF_STATUS);
+
+	return !!(status & LINK_STATUS_UP);
+}
+
+static u32 ntb_epf_spad_read(struct ntb_dev *ntb, int idx)
+{
+	struct ntb_epf_dev *ndev = ntb_ndev(ntb);
+	u32 offset;
+
+	if (idx < 0 || idx >= ndev->spad_count)
+		return 0;
+
+	offset = ntb_epf_ctrl_readl(ndev, NTB_EPF_SPAD_OFFSET);
+	offset += (idx << 2);
+
+	return ntb_epf_ctrl_readl(ndev, offset);
+}
+
+static int ntb_epf_spad_write(struct ntb_dev *ntb,
+			      int idx, u32 val)
+{
+	struct ntb_epf_dev *ndev = ntb_ndev(ntb);
+	u32 offset;
+
+	if (idx < 0 || idx >= ndev->spad_count)
+		return -EINVAL;
+
+	offset = ntb_epf_ctrl_readl(ndev, NTB_EPF_SPAD_OFFSET);
+	offset += (idx << 2);
+	ntb_epf_ctrl_writel(ndev, offset, val);
+
+	return 0;
+}
+
+static u32 ntb_epf_peer_spad_read(struct ntb_dev *ntb, int pidx, int idx)
+{
+	struct ntb_epf_dev *ndev = ntb_ndev(ntb);
+	u32 offset;
+
+	if (idx < 0 || idx >= ndev->spad_count)
+		return -EINVAL;
+
+	offset = (idx << 2);
+	return readl(ndev->peer_spad_reg + offset);
+}
+
+static int ntb_epf_peer_spad_write(struct ntb_dev *ntb, int pidx,
+				   int idx, u32 val)
+{
+	struct ntb_epf_dev *ndev = ntb_ndev(ntb);
+	u32 offset;
+
+	if (idx < 0 || idx >= ndev->spad_count)
+		return -EINVAL;
+
+	offset = (idx << 2);
+	writel(val, ndev->peer_spad_reg + offset);
+
+	return 0;
+}
+
+static int ntb_epf_link_enable(struct ntb_dev *ntb,
+			       enum ntb_speed max_speed,
+			       enum ntb_width max_width)
+{
+	struct ntb_epf_dev *ndev = ntb_ndev(ntb);
+	struct device *dev = &ntb->pdev->dev;
+	int ret;
+
+	ret = ntb_epf_send_command(ndev, CMD_LINK_UP, 0);
+	if (ret) {
+		dev_err(dev, "Fail to enable link\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int ntb_epf_link_disable(struct ntb_dev *ntb)
+{
+	return 0;
+}
+
+static irqreturn_t ndev_vec_isr(int irq, void *dev)
+{
+	struct ntb_epf_dev *ndev = dev;
+	int irq_no;
+
+	irq_no = irq - ndev->ntb.pdev->irq;
+	ndev->db_val = irq_no + 1;
+
+	if (irq_no == 0)
+		ntb_link_event(&ndev->ntb);
+	else
+		ntb_db_event(&ndev->ntb, irq_no);
+
+	return IRQ_HANDLED;
+}
+
+static int ntb_epf_init_isr(struct ntb_epf_dev *ndev, int msi_min, int msi_max)
+{
+	struct pci_dev *pdev = ndev->ntb.pdev;
+	struct device *dev = &pdev->dev;
+	int irq;
+	int ret;
+	int i;
+
+	irq = pci_alloc_irq_vectors(pdev, msi_min, msi_max, PCI_IRQ_MSI);
+	if (irq < 0) {
+		dev_err(dev, "Failed to get MSI interrupts\n");
+		return irq;
+	}
+
+	for (i = 0; i < irq; i++) {
+		ret = devm_request_irq(&pdev->dev, pci_irq_vector(pdev, i),
+				       ndev_vec_isr, IRQF_SHARED, "ntb_epf",
+				       ndev);
+		if (ret) {
+			dev_err(dev, "Failed to request irq\n");
+			goto err_request_irq;
+		}
+	}
+
+	ndev->db_count = irq;
+
+	ret = ntb_epf_send_command(ndev, CMD_CONFIGURE_DOORBELL, irq);
+	if (ret) {
+		dev_err(dev, "Failed to configure doorbell\n");
+		goto err_request_irq;
+	}
+
+	return 0;
+
+err_request_irq:
+	pci_free_irq_vectors(pdev);
+
+	return ret;
+}
+
+static int ntb_epf_peer_mw_count(struct ntb_dev *ntb)
+{
+	return ntb_ndev(ntb)->mw_count;
+}
+
+static int ntb_epf_spad_count(struct ntb_dev *ntb)
+{
+	return ntb_ndev(ntb)->spad_count;
+}
+
+static u64 ntb_epf_db_valid_mask(struct ntb_dev *ntb)
+{
+	return ntb_ndev(ntb)->db_valid_mask;
+}
+
+static int ntb_epf_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
+{
+	return 0;
+}
+
+static int ntb_epf_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
+				dma_addr_t addr, resource_size_t size)
+{
+	struct ntb_epf_dev *ndev = ntb_ndev(ntb);
+	struct device *dev = &ntb->pdev->dev;
+	resource_size_t mw_size;
+	int bar;
+
+	if (pidx != NTB_DEF_PEER_IDX)
+		return -EINVAL;
+
+	bar = idx + NTB_MW_OFFSET;
+
+	mw_size = pci_resource_len(ntb->pdev, bar);
+
+	if (size > mw_size) {
+		dev_err(dev, "Size is greater than the MW size\n");
+		return -EINVAL;
+	}
+
+	ntb_epf_ctrl_writel(ndev, NTB_EPF_ADDR, addr);
+	ntb_epf_ctrl_writel(ndev, NTB_EPF_SIZE, size);
+	ntb_epf_send_command(ndev, CMD_CONFIGURE_MW, idx);
+
+	return 0;
+}
+
+static int ntb_epf_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
+				    phys_addr_t *base, resource_size_t *size)
+{
+	struct ntb_epf_dev *ndev = ntb_ndev(ntb);
+	u32 offset = 0;
+	int bar;
+
+	if (idx == 0)
+		offset = ntb_epf_ctrl_readl(ndev, NTB_EPF_MW1_OFFSET);
+
+	bar = idx + NTB_MW_OFFSET;
+
+	if (base)
+		*base = pci_resource_start(ndev->ntb.pdev, bar) + offset;
+
+	if (size)
+		*size = pci_resource_len(ndev->ntb.pdev, bar) - offset;
+
+	return 0;
+}
+
+static int ntb_epf_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
+{
+	struct ntb_epf_dev *ndev = ntb_ndev(ntb);
+	u32 interrupt_num = ffs(db_bits) + 1;
+	u32 db_entry_size;
+	u32 db_data;
+
+	if (interrupt_num > ndev->db_count)
+		return -EINVAL;
+
+	db_entry_size = ntb_epf_ctrl_readl(ndev, NTB_EPF_DB_ENTRY_SIZE);
+
+	db_data = readl(ndev->ctrl_reg + NTB_EPF_DB_DATA(interrupt_num));
+	writel(db_data, ndev->db_reg + (db_entry_size * interrupt_num));
+
+	return 0;
+}
+
+static u64 ntb_epf_db_read(struct ntb_dev *ntb)
+{
+	struct ntb_epf_dev *ndev = ntb_ndev(ntb);
+
+	return ndev->db_val;
+}
+
+static int ntb_epf_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
+{
+	return 0;
+}
+
+static int ntb_epf_db_clear(struct ntb_dev *ntb, u64 db_bits)
+{
+	struct ntb_epf_dev *ndev = ntb_ndev(ntb);
+
+	ndev->db_val = 0;
+
+	return 0;
+}
+
+static const struct ntb_dev_ops ntb_epf_ops = {
+	.mw_count		= ntb_epf_mw_count,
+	.spad_count		= ntb_epf_spad_count,
+	.peer_mw_count		= ntb_epf_peer_mw_count,
+	.db_valid_mask		= ntb_epf_db_valid_mask,
+	.db_set_mask		= ntb_epf_db_set_mask,
+	.mw_set_trans		= ntb_epf_mw_set_trans,
+	.peer_mw_get_addr	= ntb_epf_peer_mw_get_addr,
+	.link_enable		= ntb_epf_link_enable,
+	.spad_read		= ntb_epf_spad_read,
+	.spad_write		= ntb_epf_spad_write,
+	.peer_spad_read		= ntb_epf_peer_spad_read,
+	.peer_spad_write	= ntb_epf_peer_spad_write,
+	.peer_db_set		= ntb_epf_peer_db_set,
+	.db_read		= ntb_epf_db_read,
+	.mw_get_align		= ntb_epf_mw_get_align,
+	.link_is_up		= ntb_epf_link_is_up,
+	.db_clear_mask		= ntb_epf_db_clear_mask,
+	.db_clear		= ntb_epf_db_clear,
+	.link_disable		= ntb_epf_link_disable,
+};
+
+static inline void ntb_epf_init_struct(struct ntb_epf_dev *ndev,
+				       struct pci_dev *pdev)
+{
+	ndev->ntb.pdev = pdev;
+	ndev->ntb.topo = NTB_TOPO_NONE;
+	ndev->ntb.ops = &ntb_epf_ops;
+}
+
+static int ntb_epf_init_dev(struct ntb_epf_dev *ndev)
+{
+	struct pci_dev *pdev = ndev->ntb.pdev;
+	struct device *dev = &pdev->dev;
+	int ret;
+
+	ret = ntb_epf_init_isr(ndev, NTB_MIN_DB_COUNT, NTB_MAX_DB_COUNT);
+	if (ret) {
+		dev_err(dev, "Failed to init ISR\n");
+		return ret;
+	}
+
+	ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
+	ndev->mw_count = ntb_epf_ctrl_readl(ndev, NTB_EPF_MW_COUNT);
+	ndev->spad_count = ntb_epf_ctrl_readl(ndev, NTB_EPF_SPAD_COUNT);
+
+	return 0;
+}
+
+static int ntb_epf_init_pci(struct ntb_epf_dev *ndev,
+			    struct pci_dev *pdev)
+{
+	struct device *dev = &pdev->dev;
+	int ret;
+
+	pci_set_drvdata(pdev, ndev);
+
+	ret = pci_enable_device(pdev);
+	if (ret) {
+		dev_err(dev, "Cannot enable PCI device\n");
+		goto err_pci_enable;
+	}
+
+	ret = pci_request_regions(pdev, "ntb");
+	if (ret) {
+		dev_err(dev, "Cannot obtain PCI resources\n");
+		goto err_pci_regions;
+	}
+
+	pci_set_master(pdev);
+
+	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
+	if (ret) {
+		ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
+		if (ret) {
+			dev_err(dev, "Cannot set DMA mask\n");
+			goto err_dma_mask;
+		}
+		dev_warn(&pdev->dev, "Cannot DMA highmem\n");
+	}
+
+	ndev->ctrl_reg = pci_iomap(pdev, 0, 0);
+	if (!ndev->ctrl_reg) {
+		ret = -EIO;
+		goto err_dma_mask;
+	}
+
+	ndev->peer_spad_reg = pci_iomap(pdev, 1, 0);
+	if (!ndev->peer_spad_reg) {
+		ret = -EIO;
+		goto err_dma_mask;
+	}
+
+	ndev->db_reg = pci_iomap(pdev, 2, 0);
+	if (!ndev->db_reg) {
+		ret = -EIO;
+		goto err_dma_mask;
+	}
+
+	return 0;
+
+err_dma_mask:
+	pci_clear_master(pdev);
+
+err_pci_regions:
+	pci_disable_device(pdev);
+
+err_pci_enable:
+	pci_set_drvdata(pdev, NULL);
+
+	return ret;
+}
+
+static void ntb_epf_deinit_pci(struct ntb_epf_dev *ndev)
+{
+	struct pci_dev *pdev = ndev->ntb.pdev;
+
+	pci_iounmap(pdev, ndev->ctrl_reg);
+	pci_iounmap(pdev, ndev->peer_spad_reg);
+	pci_iounmap(pdev, ndev->db_reg);
+
+	pci_clear_master(pdev);
+	pci_release_regions(pdev);
+	pci_disable_device(pdev);
+	pci_set_drvdata(pdev, NULL);
+}
+
+static int ntb_epf_pci_probe(struct pci_dev *pdev,
+			     const struct pci_device_id *id)
+{
+	enum pci_barno peer_spad_reg_bar = BAR_1;
+	enum pci_barno ctrl_reg_bar = BAR_0;
+	enum pci_barno db_reg_bar = BAR_2;
+	struct device *dev = &pdev->dev;
+	struct ntb_epf_data *data;
+	struct ntb_epf_dev *ndev;
+	int ret;
+
+	if (pci_is_bridge(pdev))
+		return -ENODEV;
+
+	ndev = devm_kzalloc(dev, sizeof(*ndev), GFP_KERNEL);
+	if (!ndev)
+		return -ENOMEM;
+
+	data = (struct ntb_epf_data *)id->driver_data;
+	if (data) {
+		if (data->peer_spad_reg_bar)
+			peer_spad_reg_bar = data->peer_spad_reg_bar;
+		if (data->ctrl_reg_bar)
+			ctrl_reg_bar = data->ctrl_reg_bar;
+		if (data->db_reg_bar)
+			db_reg_bar = data->db_reg_bar;
+	}
+
+	ndev->peer_spad_reg_bar = peer_spad_reg_bar;
+	ndev->ctrl_reg_bar = ctrl_reg_bar;
+	ndev->db_reg_bar = db_reg_bar;
+
+	ntb_epf_init_struct(ndev, pdev);
+
+	ret = ntb_epf_init_pci(ndev, pdev);
+	if (ret) {
+		dev_err(dev, "Failed to init PCI\n");
+		return ret;
+	}
+
+	ret = ntb_epf_init_dev(ndev);
+	if (ret) {
+		dev_err(dev, "Failed to init device\n");
+		goto err_init_dev;
+	}
+
+	ret = ntb_register_device(&ndev->ntb);
+	if (ret) {
+		dev_err(dev, "Failed to register NTB device\n");
+		goto err_register_dev;
+	}
+
+	return 0;
+
+err_register_dev:
+	pci_free_irq_vectors(pdev);
+
+err_init_dev:
+	ntb_epf_deinit_pci(ndev);
+
+	return ret;
+}
+
+static void ntb_epf_pci_remove(struct pci_dev *pdev)
+{
+	struct ntb_epf_dev *ndev = pci_get_drvdata(pdev);
+
+	ntb_unregister_device(&ndev->ntb);
+	pci_free_irq_vectors(pdev);
+	kfree(ndev);
+}
+
+static const struct ntb_epf_data j721e_data = {
+	.ctrl_reg_bar = BAR_0,
+	.peer_spad_reg_bar = BAR_1,
+	.db_reg_bar = BAR_2,
+};
+
+static const struct pci_device_id ntb_epf_pci_tbl[] = {
+	{
+		PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_J721E),
+		.driver_data = (kernel_ulong_t)&j721e_data,
+	},
+	{ },
+};
+MODULE_DEVICE_TABLE(pci, ntb_epf_pci_tbl);
+
+static struct pci_driver ntb_epf_pci_driver = {
+	.name		= KBUILD_MODNAME,
+	.id_table	= ntb_epf_pci_tbl,
+	.probe		= ntb_epf_pci_probe,
+	.remove		= ntb_epf_pci_remove,
+};
+module_pci_driver(ntb_epf_pci_driver);
+
+MODULE_DESCRIPTION("PCI ENDPOINT NTB HOST DRIVER");
+MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/ntb/hw/Kconfig b/drivers/ntb/hw/Kconfig
--- a/drivers/ntb/hw/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/ntb/hw/Kconfig	2020-06-19 21:00:31.420227716 -0400
@@ -1,4 +1,5 @@
 source "drivers/ntb/hw/amd/Kconfig"
 source "drivers/ntb/hw/idt/Kconfig"
 source "drivers/ntb/hw/intel/Kconfig"
+source "drivers/ntb/hw/epf/Kconfig"
 source "drivers/ntb/hw/mscc/Kconfig"
diff -Naur --no-dereference a/drivers/ntb/hw/Makefile b/drivers/ntb/hw/Makefile
--- a/drivers/ntb/hw/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/ntb/hw/Makefile	2020-06-19 21:00:31.420227716 -0400
@@ -1,4 +1,5 @@
 obj-$(CONFIG_NTB_AMD)	+= amd/
 obj-$(CONFIG_NTB_IDT)	+= idt/
 obj-$(CONFIG_NTB_INTEL)	+= intel/
+obj-$(CONFIG_NTB_EPF)	+= epf/
 obj-$(CONFIG_NTB_SWITCHTEC) += mscc/
diff -Naur --no-dereference a/drivers/ntb/test/ntb_perf.c b/drivers/ntb/test/ntb_perf.c
--- a/drivers/ntb/test/ntb_perf.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/ntb/test/ntb_perf.c	2020-06-19 21:00:31.420227716 -0400
@@ -568,6 +568,7 @@
 {
 	resource_size_t xlat_align, size_align, size_max;
 	struct perf_ctx *perf = peer->perf;
+	struct pci_dev *pdev = perf->ntb->pdev;
 	int ret;
 
 	/* Get inbound MW parameters */
@@ -588,7 +589,7 @@
 
 	perf_free_inbuf(peer);
 
-	peer->inbuf = dma_alloc_coherent(&perf->ntb->dev, peer->inbuf_size,
+	peer->inbuf = dma_alloc_coherent(&pdev->dev, peer->inbuf_size,
 					 &peer->inbuf_xlat, GFP_KERNEL);
 	if (!peer->inbuf) {
 		dev_err(&perf->ntb->dev, "Failed to alloc inbuf of %pa\n",
diff -Naur --no-dereference a/drivers/ntb/test/ntb_tool.c b/drivers/ntb/test/ntb_tool.c
--- a/drivers/ntb/test/ntb_tool.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/ntb/test/ntb_tool.c	2020-06-19 21:00:31.420227716 -0400
@@ -576,6 +576,7 @@
 {
 	resource_size_t size, addr_align, size_align;
 	struct tool_mw *inmw = &tc->peers[pidx].inmws[widx];
+	struct pci_dev *pdev = tc->ntb->pdev;
 	char buf[TOOL_BUF_LEN];
 	int ret;
 
@@ -590,7 +591,7 @@
 	inmw->size = min_t(resource_size_t, req_size, size);
 	inmw->size = round_up(inmw->size, addr_align);
 	inmw->size = round_up(inmw->size, size_align);
-	inmw->mm_base = dma_alloc_coherent(&tc->ntb->dev, inmw->size,
+	inmw->mm_base = dma_alloc_coherent(&pdev->dev, inmw->size,
 					   &inmw->dma_base, GFP_KERNEL);
 	if (!inmw->mm_base)
 		return -ENOMEM;
@@ -1638,6 +1639,7 @@
 
 	tool_setup_dbgfs(tc);
 
+	ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
 	return 0;
 
 err_clear_mws:
diff -Naur --no-dereference a/drivers/of/platform.c b/drivers/of/platform.c
--- a/drivers/of/platform.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/of/platform.c	2020-06-19 21:00:31.420227716 -0400
@@ -169,11 +169,9 @@
  * Returns pointer to created platform device, or NULL if a device was not
  * registered.  Unavailable devices will not get registered.
  */
-static struct platform_device *of_platform_device_create_pdata(
-					struct device_node *np,
-					const char *bus_id,
-					void *platform_data,
-					struct device *parent)
+struct platform_device *
+of_platform_device_create_pdata(struct device_node *np, const char *bus_id,
+				void *platform_data, struct device *parent)
 {
 	struct platform_device *dev;
 
@@ -203,6 +201,7 @@
 	of_node_clear_flag(np, OF_POPULATED);
 	return NULL;
 }
+EXPORT_SYMBOL(of_platform_device_create_pdata);
 
 /**
  * of_platform_device_create - Alloc, initialize and register an of_device
diff -Naur --no-dereference a/drivers/opp/of.c b/drivers/opp/of.c
--- a/drivers/opp/of.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/opp/of.c	2020-06-19 21:00:31.420227716 -0400
@@ -374,7 +374,7 @@
 free_opp:
 	_opp_free(new_opp);
 
-	return ret;
+	return 0;
 }
 
 /* Initializes OPP tables based on new bindings */
diff -Naur --no-dereference a/drivers/pci/controller/dwc/Kconfig b/drivers/pci/controller/dwc/Kconfig
--- a/drivers/pci/controller/dwc/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/controller/dwc/Kconfig	2020-06-19 21:00:31.420227716 -0400
@@ -104,15 +104,34 @@
 
 config PCI_KEYSTONE
 	bool "TI Keystone PCIe controller"
-	depends on ARCH_KEYSTONE || (ARM && COMPILE_TEST)
-	depends on PCI_MSI_IRQ_DOMAIN
-	select PCIE_DW_HOST
+	depends on ARCH_KEYSTONE || ARCH_K3 || ((ARM || ARM64) && COMPILE_TEST)
 	help
 	  Say Y here if you want to enable PCI controller support on Keystone
 	  SoCs. The PCI controller on Keystone is based on DesignWare hardware
 	  and therefore the driver re-uses the DesignWare core functions to
 	  implement the driver.
 
+if PCI_KEYSTONE
+
+config PCI_KEYSTONE_HOST
+	bool "PCI Keystone Host Mode"
+	depends on PCI_MSI_IRQ_DOMAIN
+	select PCIE_DW_HOST
+	default y
+	help
+	 Enables support for the PCIe controller in the Keystone SoC to work in
+	 host mode.
+
+config PCI_KEYSTONE_EP
+	bool "PCI Keystone Endpoint Mode"
+	depends on PCI_ENDPOINT
+	select PCIE_DW_EP
+	help
+	 Enables support for the PCIe controller in the Keystone SoC to work in
+	 endpoint mode.
+
+endif
+
 config PCI_LAYERSCAPE
 	bool "Freescale Layerscape PCIe controller"
 	depends on OF && (ARM || ARCH_LAYERSCAPE || COMPILE_TEST)
diff -Naur --no-dereference a/drivers/pci/controller/dwc/Makefile b/drivers/pci/controller/dwc/Makefile
--- a/drivers/pci/controller/dwc/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/controller/dwc/Makefile	2020-06-19 21:00:31.420227716 -0400
@@ -7,7 +7,7 @@
 obj-$(CONFIG_PCI_EXYNOS) += pci-exynos.o
 obj-$(CONFIG_PCI_IMX6) += pci-imx6.o
 obj-$(CONFIG_PCIE_SPEAR13XX) += pcie-spear13xx.o
-obj-$(CONFIG_PCI_KEYSTONE) += pci-keystone-dw.o pci-keystone.o
+obj-$(CONFIG_PCI_KEYSTONE) += pci-keystone.o
 obj-$(CONFIG_PCI_LAYERSCAPE) += pci-layerscape.o
 obj-$(CONFIG_PCIE_QCOM) += pcie-qcom.o
 obj-$(CONFIG_PCIE_ARMADA_8K) += pcie-armada8k.o
diff -Naur --no-dereference a/drivers/pci/controller/dwc/pci-dra7xx.c b/drivers/pci/controller/dwc/pci-dra7xx.c
--- a/drivers/pci/controller/dwc/pci-dra7xx.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/controller/dwc/pci-dra7xx.c	2020-06-19 21:00:31.420227716 -0400
@@ -82,6 +82,10 @@
 #define MSI_REQ_GRANT					BIT(0)
 #define MSI_VECTOR_SHIFT				7
 
+#define PCIE_1LANE_2LANE_SELECTION			BIT(13)
+#define PCIE_B1C0_MODE_SEL				BIT(2)
+#define PCIE_B0_B1_TSYNCEN				BIT(0)
+
 struct dra7xx_pcie {
 	struct dw_pcie		*pci;
 	void __iomem		*base;		/* DT ti_conf */
@@ -94,6 +98,7 @@
 
 struct dra7xx_pcie_of_data {
 	enum dw_pcie_device_mode mode;
+	u32 b1co_mode_sel_mask;
 };
 
 #define to_dra7xx_pcie(x)	dev_get_drvdata((x)->dev)
@@ -258,12 +263,29 @@
 	struct pcie_port *pp = &pci->pp;
 	unsigned long reg;
 	u32 virq, bit;
+	int count = 0;
 
 	reg = dra7xx_pcie_readl(dra7xx, PCIECTRL_DRA7XX_CONF_IRQSTATUS_MSI);
+	dra7xx_pcie_writel(dra7xx, PCIECTRL_DRA7XX_CONF_IRQSTATUS_MSI, reg);
 
 	switch (reg) {
 	case MSI:
-		dw_handle_msi_irq(pp);
+		/*
+		 * Need to make sure no MSI IRQs are pending before
+		 * exiting handler, else the wrapper will not catch new
+		 * IRQs. So loop around till dw_handle_msi_irq() returns
+		 * IRQ_NONE
+		 */
+		while (dw_handle_msi_irq(pp) != IRQ_NONE && count < 1000)
+			count++;
+
+		if (count == 1000) {
+			dev_err(pci->dev, "too much work in msi irq\n");
+			dra7xx_pcie_writel(dra7xx,
+					   PCIECTRL_DRA7XX_CONF_IRQSTATUS_MSI,
+					   reg);
+			return IRQ_HANDLED;
+		}
 		break;
 	case INTA:
 	case INTB:
@@ -277,8 +299,6 @@
 		break;
 	}
 
-	dra7xx_pcie_writel(dra7xx, PCIECTRL_DRA7XX_CONF_IRQSTATUS_MSI, reg);
-
 	return IRQ_HANDLED;
 }
 
@@ -390,9 +410,22 @@
 	return 0;
 }
 
-static struct dw_pcie_ep_ops pcie_ep_ops = {
+static const struct pci_epc_features dra7xx_pcie_epc_features = {
+	.linkup_notifier = true,
+	.msi_capable = true,
+	.msix_capable = false,
+};
+
+static const struct pci_epc_features*
+dra7xx_pcie_get_features(struct dw_pcie_ep *ep)
+{
+	return &dra7xx_pcie_epc_features;
+}
+
+static const struct dw_pcie_ep_ops pcie_ep_ops = {
 	.ep_init = dra7xx_pcie_ep_init,
 	.raise_irq = dra7xx_pcie_raise_irq,
+	.get_features = dra7xx_pcie_get_features,
 };
 
 static int __init dra7xx_add_pcie_ep(struct dra7xx_pcie *dra7xx,
@@ -530,6 +563,26 @@
 	.mode = DW_PCIE_EP_TYPE,
 };
 
+static const struct dra7xx_pcie_of_data dra746_pcie_rc_of_data = {
+	.b1co_mode_sel_mask = BIT(2),
+	.mode = DW_PCIE_RC_TYPE,
+};
+
+static const struct dra7xx_pcie_of_data dra726_pcie_rc_of_data = {
+	.b1co_mode_sel_mask = GENMASK(3, 2),
+	.mode = DW_PCIE_RC_TYPE,
+};
+
+static const struct dra7xx_pcie_of_data dra746_pcie_ep_of_data = {
+	.b1co_mode_sel_mask = BIT(2),
+	.mode = DW_PCIE_EP_TYPE,
+};
+
+static const struct dra7xx_pcie_of_data dra726_pcie_ep_of_data = {
+	.b1co_mode_sel_mask = GENMASK(3, 2),
+	.mode = DW_PCIE_EP_TYPE,
+};
+
 static const struct of_device_id of_dra7xx_pcie_match[] = {
 	{
 		.compatible = "ti,dra7-pcie",
@@ -539,6 +592,22 @@
 		.compatible = "ti,dra7-pcie-ep",
 		.data = &dra7xx_pcie_ep_of_data,
 	},
+	{
+		.compatible = "ti,dra746-pcie-rc",
+		.data = &dra746_pcie_rc_of_data,
+	},
+	{
+		.compatible = "ti,dra726-pcie-rc",
+		.data = &dra726_pcie_rc_of_data,
+	},
+	{
+		.compatible = "ti,dra746-pcie-ep",
+		.data = &dra746_pcie_ep_of_data,
+	},
+	{
+		.compatible = "ti,dra726-pcie-ep",
+		.data = &dra726_pcie_ep_of_data,
+	},
 	{},
 };
 
@@ -584,6 +653,34 @@
 	return ret;
 }
 
+static int dra7xx_pcie_configure_two_lane(struct device *dev,
+					  u32 b1co_mode_sel_mask)
+{
+	struct device_node *np = dev->of_node;
+	struct regmap *pcie_syscon;
+	unsigned int pcie_reg;
+	u32 mask;
+	u32 val;
+
+	pcie_syscon = syscon_regmap_lookup_by_phandle(np, "ti,syscon-lane-sel");
+	if (IS_ERR(pcie_syscon)) {
+		dev_err(dev, "unable to get ti,syscon-lane-sel\n");
+		return -EINVAL;
+	}
+
+	if (of_property_read_u32_index(np, "ti,syscon-lane-sel", 1,
+				       &pcie_reg)) {
+		dev_err(dev, "couldn't get lane selection reg offset\n");
+		return -EINVAL;
+	}
+
+	mask = b1co_mode_sel_mask | PCIE_B0_B1_TSYNCEN;
+	val = PCIE_B1C0_MODE_SEL | PCIE_B0_B1_TSYNCEN;
+	regmap_update_bits(pcie_syscon, pcie_reg, mask, val);
+
+	return 0;
+}
+
 static int __init dra7xx_pcie_probe(struct platform_device *pdev)
 {
 	u32 reg;
@@ -604,6 +701,7 @@
 	const struct of_device_id *match;
 	const struct dra7xx_pcie_of_data *data;
 	enum dw_pcie_device_mode mode;
+	u32 b1co_mode_sel_mask;
 
 	match = of_match_device(of_match_ptr(of_dra7xx_pcie_match), dev);
 	if (!match)
@@ -611,6 +709,7 @@
 
 	data = (struct dra7xx_pcie_of_data *)match->data;
 	mode = (enum dw_pcie_device_mode)data->mode;
+	b1co_mode_sel_mask = data->b1co_mode_sel_mask;
 
 	dra7xx = devm_kzalloc(dev, sizeof(*dra7xx), GFP_KERNEL);
 	if (!dra7xx)
@@ -630,7 +729,7 @@
 	}
 
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ti_conf");
-	base = devm_ioremap_nocache(dev, res->start, resource_size(res));
+	base = devm_ioremap_resource(dev, res);
 	if (!base)
 		return -ENOMEM;
 
@@ -666,6 +765,12 @@
 	dra7xx->pci = pci;
 	dra7xx->phy_count = phy_count;
 
+	if (phy_count == 2) {
+		ret = dra7xx_pcie_configure_two_lane(dev, b1co_mode_sel_mask);
+		if (ret < 0)
+			dra7xx->phy_count = 1; /* Fallback to x1 lane mode */
+	}
+
 	ret = dra7xx_pcie_enable_phy(dra7xx);
 	if (ret) {
 		dev_err(dev, "failed to enable phy\n");
diff -Naur --no-dereference a/drivers/pci/controller/dwc/pcie-artpec6.c b/drivers/pci/controller/dwc/pcie-artpec6.c
--- a/drivers/pci/controller/dwc/pcie-artpec6.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/controller/dwc/pcie-artpec6.c	2020-06-19 21:00:31.420227716 -0400
@@ -444,7 +444,7 @@
 	return 0;
 }
 
-static struct dw_pcie_ep_ops pcie_ep_ops = {
+static const struct dw_pcie_ep_ops pcie_ep_ops = {
 	.ep_init = artpec6_pcie_ep_init,
 	.raise_irq = artpec6_pcie_raise_irq,
 };
diff -Naur --no-dereference a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c
--- a/drivers/pci/controller/dwc/pcie-designware.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/controller/dwc/pcie-designware.c	2020-06-19 21:00:31.420227716 -0400
@@ -89,11 +89,42 @@
 		dev_err(pci->dev, "Write DBI address failed\n");
 }
 
+u32 __dw_pcie_read_dbi2(struct dw_pcie *pci, void __iomem *base, u32 reg,
+			size_t size)
+{
+	int ret;
+	u32 val;
+
+	if (pci->ops->read_dbi2)
+		return pci->ops->read_dbi2(pci, base, reg, size);
+
+	ret = dw_pcie_read(base + reg, size, &val);
+	if (ret)
+		dev_err(pci->dev, "read DBI address failed\n");
+
+	return val;
+}
+
+void __dw_pcie_write_dbi2(struct dw_pcie *pci, void __iomem *base, u32 reg,
+			  size_t size, u32 val)
+{
+	int ret;
+
+	if (pci->ops->write_dbi2) {
+		pci->ops->write_dbi2(pci, base, reg, size, val);
+		return;
+	}
+
+	ret = dw_pcie_write(base + reg, size, val);
+	if (ret)
+		dev_err(pci->dev, "write DBI address failed\n");
+}
+
 static u32 dw_pcie_readl_ob_unroll(struct dw_pcie *pci, u32 index, u32 reg)
 {
 	u32 offset = PCIE_GET_ATU_OUTB_UNR_REG_OFFSET(index);
 
-	return dw_pcie_readl_dbi(pci, offset + reg);
+	return dw_pcie_readl_atu(pci, offset + reg);
 }
 
 static void dw_pcie_writel_ob_unroll(struct dw_pcie *pci, u32 index, u32 reg,
@@ -101,7 +132,7 @@
 {
 	u32 offset = PCIE_GET_ATU_OUTB_UNR_REG_OFFSET(index);
 
-	dw_pcie_writel_dbi(pci, offset + reg, val);
+	dw_pcie_writel_atu(pci, offset + reg, val);
 }
 
 static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index,
@@ -140,8 +171,8 @@
 	dev_err(pci->dev, "Outbound iATU is not being enabled\n");
 }
 
-void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type,
-			       u64 cpu_addr, u64 pci_addr, u32 size)
+int dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type,
+			      u64 cpu_addr, u64 pci_addr, u32 size)
 {
 	u32 retries, val;
 
@@ -151,7 +182,7 @@
 	if (pci->iatu_unroll_enabled) {
 		dw_pcie_prog_outbound_atu_unroll(pci, index, type, cpu_addr,
 						 pci_addr, size);
-		return;
+		return 0;
 	}
 
 	dw_pcie_writel_dbi(pci, PCIE_ATU_VIEWPORT,
@@ -176,18 +207,19 @@
 	for (retries = 0; retries < LINK_WAIT_MAX_IATU_RETRIES; retries++) {
 		val = dw_pcie_readl_dbi(pci, PCIE_ATU_CR2);
 		if (val & PCIE_ATU_ENABLE)
-			return;
+			return 0;
 
 		mdelay(LINK_WAIT_IATU);
 	}
 	dev_err(pci->dev, "Outbound iATU is not being enabled\n");
+	return -EBUSY;
 }
 
 static u32 dw_pcie_readl_ib_unroll(struct dw_pcie *pci, u32 index, u32 reg)
 {
 	u32 offset = PCIE_GET_ATU_INB_UNR_REG_OFFSET(index);
 
-	return dw_pcie_readl_dbi(pci, offset + reg);
+	return dw_pcie_readl_atu(pci, offset + reg);
 }
 
 static void dw_pcie_writel_ib_unroll(struct dw_pcie *pci, u32 index, u32 reg,
@@ -195,7 +227,7 @@
 {
 	u32 offset = PCIE_GET_ATU_INB_UNR_REG_OFFSET(index);
 
-	dw_pcie_writel_dbi(pci, offset + reg, val);
+	dw_pcie_writel_atu(pci, offset + reg, val);
 }
 
 static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, int index,
@@ -339,6 +371,17 @@
 		(!(val & PCIE_PHY_DEBUG_R1_LINK_IN_TRAINING)));
 }
 
+static u8 dw_pcie_iatu_unroll_enabled(struct dw_pcie *pci)
+{
+	u32 val;
+
+	val = dw_pcie_readl_dbi(pci, PCIE_ATU_VIEWPORT);
+	if (val == 0xffffffff)
+		return 1;
+
+	return 0;
+}
+
 void dw_pcie_setup(struct dw_pcie *pci)
 {
 	int ret;
@@ -347,6 +390,16 @@
 	struct device *dev = pci->dev;
 	struct device_node *np = dev->of_node;
 
+	if (pci->version >= 0x480A || (!pci->version &&
+				       dw_pcie_iatu_unroll_enabled(pci))) {
+		pci->iatu_unroll_enabled = true;
+		if (!pci->atu_base)
+			pci->atu_base = pci->dbi_base + DEFAULT_DBI_ATU_OFFSET;
+	}
+	dev_dbg(pci->dev, "iATU unroll: %s\n", pci->iatu_unroll_enabled ?
+		"enabled" : "disabled");
+
+
 	ret = of_property_read_u32(np, "num-lanes", &lanes);
 	if (ret)
 		lanes = 0;
diff -Naur --no-dereference a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c
--- a/drivers/pci/controller/dwc/pcie-designware-ep.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/controller/dwc/pcie-designware-ep.c	2020-06-19 21:00:31.420227716 -0400
@@ -46,16 +46,19 @@
 	u8 cap_id, next_cap_ptr;
 	u16 reg;
 
+	if (!cap_ptr)
+		return 0;
+
 	reg = dw_pcie_readw_dbi(pci, cap_ptr);
-	next_cap_ptr = (reg & 0xff00) >> 8;
 	cap_id = (reg & 0x00ff);
 
-	if (!next_cap_ptr || cap_id > PCI_CAP_ID_MAX)
+	if (cap_id > PCI_CAP_ID_MAX)
 		return 0;
 
 	if (cap_id == cap)
 		return cap_ptr;
 
+	next_cap_ptr = (reg & 0xff00) >> 8;
 	return __dw_pcie_ep_find_next_cap(pci, next_cap_ptr, cap);
 }
 
@@ -67,13 +70,27 @@
 	reg = dw_pcie_readw_dbi(pci, PCI_CAPABILITY_LIST);
 	next_cap_ptr = (reg & 0x00ff);
 
-	if (!next_cap_ptr)
-		return 0;
-
 	return __dw_pcie_ep_find_next_cap(pci, next_cap_ptr, cap);
 }
 
-static int dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no,
+static int dw_pcie_ep_data_transfer(struct pci_epc *epc, struct pci_epf *epf,
+				    dma_addr_t dma_dst, dma_addr_t dma_src,
+				    size_t len)
+{
+	return pci_epf_data_transfer(epf, dma_dst, dma_src, len);
+}
+
+static int dw_pcie_ep_epf_init(struct pci_epc *epc, struct pci_epf *epf)
+{
+	return pci_epf_init_dma_chan(epf);
+}
+
+static void dw_pcie_ep_epf_exit(struct pci_epc *epc, struct pci_epf *epf)
+{
+	pci_epf_clean_dma_chan(epf);
+}
+
+static int dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
 				   struct pci_epf_header *hdr)
 {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
@@ -112,11 +129,17 @@
 		return -EINVAL;
 	}
 
-	ret = dw_pcie_prog_inbound_atu(pci, free_win, bar, cpu_addr,
-				       as_type);
-	if (ret < 0) {
-		dev_err(pci->dev, "Failed to program IB window\n");
-		return ret;
+	if (pci->ops->inbound_atu) {
+		ret = pci->ops->inbound_atu(pci, free_win, bar, cpu_addr);
+		if (ret)
+			return ret;
+	} else {
+		ret = dw_pcie_prog_inbound_atu(pci, free_win, bar, cpu_addr,
+					       as_type);
+		if (ret < 0) {
+			dev_err(pci->dev, "Failed to program IB window\n");
+			return ret;
+		}
 	}
 
 	ep->bar_to_atu[bar] = free_win;
@@ -129,8 +152,14 @@
 				   u64 pci_addr, size_t size)
 {
 	u32 free_win;
+	int ret;
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 
+	if (pci->ops->outbound_atu) {
+		ret = pci->ops->outbound_atu(pci, phys_addr, pci_addr, size);
+		return ret;
+	}
+
 	free_win = find_first_zero_bit(ep->ob_window_map, ep->num_ob_windows);
 	if (free_win >= ep->num_ob_windows) {
 		dev_err(pci->dev, "No free outbound window\n");
@@ -146,7 +175,7 @@
 	return 0;
 }
 
-static void dw_pcie_ep_clear_bar(struct pci_epc *epc, u8 func_no,
+static void dw_pcie_ep_clear_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
 				 struct pci_epf_bar *epf_bar)
 {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
@@ -156,11 +185,16 @@
 
 	__dw_pcie_ep_reset_bar(pci, bar, epf_bar->flags);
 
-	dw_pcie_disable_atu(pci, atu_index, DW_PCIE_REGION_INBOUND);
+	if (pci->ops->disable_atu)
+		pci->ops->disable_atu(pci, 0, atu_index,
+				      DW_PCIE_REGION_INBOUND);
+	else
+		dw_pcie_disable_atu(pci, atu_index, DW_PCIE_REGION_INBOUND);
 	clear_bit(atu_index, ep->ib_window_map);
+	ep->epf_bar[bar] = NULL;
 }
 
-static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no,
+static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
 			      struct pci_epf_bar *epf_bar)
 {
 	int ret;
@@ -191,6 +225,7 @@
 		dw_pcie_writel_dbi(pci, reg + 4, 0);
 	}
 
+	ep->epf_bar[bar] = epf_bar;
 	dw_pcie_dbi_ro_wr_dis(pci);
 
 	return 0;
@@ -211,7 +246,7 @@
 	return -EINVAL;
 }
 
-static void dw_pcie_ep_unmap_addr(struct pci_epc *epc, u8 func_no,
+static void dw_pcie_ep_unmap_addr(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
 				  phys_addr_t addr)
 {
 	int ret;
@@ -219,6 +254,11 @@
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 
+	if (pci->ops->disable_atu) {
+		pci->ops->disable_atu(pci, addr, 0, DW_PCIE_REGION_OUTBOUND);
+		return;
+	}
+
 	ret = dw_pcie_find_index(ep, addr, &atu_index);
 	if (ret < 0)
 		return;
@@ -227,9 +267,8 @@
 	clear_bit(atu_index, ep->ob_window_map);
 }
 
-static int dw_pcie_ep_map_addr(struct pci_epc *epc, u8 func_no,
-			       phys_addr_t addr,
-			       u64 pci_addr, size_t size)
+static int dw_pcie_ep_map_addr(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
+			       phys_addr_t addr, u64 pci_addr, size_t size)
 {
 	int ret;
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
@@ -244,7 +283,7 @@
 	return 0;
 }
 
-static int dw_pcie_ep_get_msi(struct pci_epc *epc, u8 func_no)
+static int dw_pcie_ep_get_msi(struct pci_epc *epc, u8 func_no, u8 vfunc_no)
 {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
@@ -263,7 +302,8 @@
 	return val;
 }
 
-static int dw_pcie_ep_set_msi(struct pci_epc *epc, u8 func_no, u8 interrupts)
+static int dw_pcie_ep_set_msi(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
+			      u8 interrupts)
 {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
@@ -283,7 +323,7 @@
 	return 0;
 }
 
-static int dw_pcie_ep_get_msix(struct pci_epc *epc, u8 func_no)
+static int dw_pcie_ep_get_msix(struct pci_epc *epc, u8 func_no, u8 vfunc_no)
 {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
@@ -302,7 +342,8 @@
 	return val;
 }
 
-static int dw_pcie_ep_set_msix(struct pci_epc *epc, u8 func_no, u16 interrupts)
+static int dw_pcie_ep_set_msix(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
+			       u16 interrupts, enum pci_barno bir, u32 offset)
 {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
@@ -311,18 +352,28 @@
 	if (!ep->msix_cap)
 		return -EINVAL;
 
+	dw_pcie_dbi_ro_wr_en(pci);
+
 	reg = ep->msix_cap + PCI_MSIX_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	val &= ~PCI_MSIX_FLAGS_QSIZE;
 	val |= interrupts;
-	dw_pcie_dbi_ro_wr_en(pci);
 	dw_pcie_writew_dbi(pci, reg, val);
+
+	reg = ep->msix_cap + PCI_MSIX_TABLE;
+	val = offset | bir;
+	dw_pcie_writel_dbi(pci, reg, val);
+
+	reg = ep->msix_cap + PCI_MSIX_PBA;
+	val = (offset + (interrupts * PCI_MSIX_ENTRY_SIZE)) | bir;
+	dw_pcie_writel_dbi(pci, reg, val);
+
 	dw_pcie_dbi_ro_wr_dis(pci);
 
 	return 0;
 }
 
-static int dw_pcie_ep_raise_irq(struct pci_epc *epc, u8 func_no,
+static int dw_pcie_ep_raise_irq(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
 				enum pci_epc_irq_type type, u16 interrupt_num)
 {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
@@ -355,7 +406,21 @@
 	return pci->ops->start_link(pci);
 }
 
+static const struct pci_epc_features*
+dw_pcie_ep_get_features(struct pci_epc *epc, u8 func_no, u8 vfunc_no)
+{
+	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
+
+	if (!ep->ops->get_features)
+		return NULL;
+
+	return ep->ops->get_features(ep);
+}
+
 static const struct pci_epc_ops epc_ops = {
+	.epf_init		= dw_pcie_ep_epf_init,
+	.epf_exit		= dw_pcie_ep_epf_exit,
+	.data_transfer		= dw_pcie_ep_data_transfer,
 	.write_header		= dw_pcie_ep_write_header,
 	.set_bar		= dw_pcie_ep_set_bar,
 	.clear_bar		= dw_pcie_ep_clear_bar,
@@ -368,6 +433,7 @@
 	.raise_irq		= dw_pcie_ep_raise_irq,
 	.start			= dw_pcie_ep_start,
 	.stop			= dw_pcie_ep_stop,
+	.get_features		= dw_pcie_ep_get_features,
 };
 
 int dw_pcie_ep_raise_legacy_irq(struct dw_pcie_ep *ep, u8 func_no)
@@ -414,14 +480,15 @@
 	aligned_offset = msg_addr_lower & (epc->mem->page_size - 1);
 	msg_addr = ((u64)msg_addr_upper) << 32 |
 			(msg_addr_lower & ~aligned_offset);
-	ret = dw_pcie_ep_map_addr(epc, func_no, ep->msi_mem_phys, msg_addr,
+
+	ret = dw_pcie_ep_map_addr(epc, func_no, 0, ep->msi_mem_phys, msg_addr,
 				  epc->mem->page_size);
 	if (ret)
 		return ret;
 
 	writel(msg_data | (interrupt_num - 1), ep->msi_mem + aligned_offset);
 
-	dw_pcie_ep_unmap_addr(epc, func_no, ep->msi_mem_phys);
+	dw_pcie_ep_unmap_addr(epc, func_no, 0, ep->msi_mem_phys);
 
 	return 0;
 }
@@ -430,55 +497,41 @@
 			     u16 interrupt_num)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	struct pci_epf_msix_tbl *msix_tbl;
 	struct pci_epc *epc = ep->epc;
-	u16 tbl_offset, bir;
-	u32 bar_addr_upper, bar_addr_lower;
-	u32 msg_addr_upper, msg_addr_lower;
+	struct pci_epf_bar *epf_bar;
 	u32 reg, msg_data, vec_ctrl;
-	u64 tbl_addr, msg_addr, reg_u64;
-	void __iomem *msix_tbl;
+	unsigned int aligned_offset;
+	u32 tbl_offset;
+	u64 msg_addr;
 	int ret;
+	u8 bir;
 
 	reg = ep->msix_cap + PCI_MSIX_TABLE;
 	tbl_offset = dw_pcie_readl_dbi(pci, reg);
 	bir = (tbl_offset & PCI_MSIX_TABLE_BIR);
 	tbl_offset &= PCI_MSIX_TABLE_OFFSET;
 
-	reg = PCI_BASE_ADDRESS_0 + (4 * bir);
-	bar_addr_upper = 0;
-	bar_addr_lower = dw_pcie_readl_dbi(pci, reg);
-	reg_u64 = (bar_addr_lower & PCI_BASE_ADDRESS_MEM_TYPE_MASK);
-	if (reg_u64 == PCI_BASE_ADDRESS_MEM_TYPE_64)
-		bar_addr_upper = dw_pcie_readl_dbi(pci, reg + 4);
-
-	tbl_addr = ((u64) bar_addr_upper) << 32 | bar_addr_lower;
-	tbl_addr += (tbl_offset + ((interrupt_num - 1) * PCI_MSIX_ENTRY_SIZE));
-	tbl_addr &= PCI_BASE_ADDRESS_MEM_MASK;
-
-	msix_tbl = ioremap_nocache(ep->phys_base + tbl_addr,
-				   PCI_MSIX_ENTRY_SIZE);
-	if (!msix_tbl)
-		return -EINVAL;
-
-	msg_addr_lower = readl(msix_tbl + PCI_MSIX_ENTRY_LOWER_ADDR);
-	msg_addr_upper = readl(msix_tbl + PCI_MSIX_ENTRY_UPPER_ADDR);
-	msg_addr = ((u64) msg_addr_upper) << 32 | msg_addr_lower;
-	msg_data = readl(msix_tbl + PCI_MSIX_ENTRY_DATA);
-	vec_ctrl = readl(msix_tbl + PCI_MSIX_ENTRY_VECTOR_CTRL);
-
-	iounmap(msix_tbl);
+	epf_bar = ep->epf_bar[bir];
+	msix_tbl = epf_bar->addr;
+	msix_tbl = (struct pci_epf_msix_tbl *)((char *)msix_tbl + tbl_offset);
+
+	msg_addr = msix_tbl[(interrupt_num - 1)].msg_addr;
+	msg_data = msix_tbl[(interrupt_num - 1)].msg_data;
+	vec_ctrl = msix_tbl[(interrupt_num - 1)].vector_ctrl;
 
 	if (vec_ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT)
 		return -EPERM;
 
-	ret = dw_pcie_ep_map_addr(epc, func_no, ep->msi_mem_phys, msg_addr,
+	aligned_offset = msg_addr & (epc->mem->page_size - 1);
+	ret = dw_pcie_ep_map_addr(epc, func_no, 0, ep->msi_mem_phys,  msg_addr,
 				  epc->mem->page_size);
 	if (ret)
 		return ret;
 
-	writel(msg_data, ep->msi_mem);
+	writel(msg_data, ep->msi_mem + aligned_offset);
 
-	dw_pcie_ep_unmap_addr(epc, func_no, ep->msi_mem_phys);
+	dw_pcie_ep_unmap_addr(epc, func_no, 0, ep->msi_mem_phys);
 
 	return 0;
 }
@@ -493,10 +546,32 @@
 	pci_epc_mem_exit(epc);
 }
 
+static unsigned int dw_pcie_ep_find_ext_capability(struct dw_pcie *pci, int cap)
+{
+	u32 header;
+	int pos = PCI_CFG_SPACE_SIZE;
+
+	while (pos) {
+		header = dw_pcie_readl_dbi(pci, pos);
+		if (PCI_EXT_CAP_ID(header) == cap)
+			return pos;
+
+		pos = PCI_EXT_CAP_NEXT(header);
+		if (!pos)
+			break;
+	}
+
+	return 0;
+}
+
 int dw_pcie_ep_init(struct dw_pcie_ep *ep)
 {
+	int i;
 	int ret;
+	u32 reg;
 	void *addr;
+	unsigned int nbars;
+	unsigned int offset;
 	struct pci_epc *epc;
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 	struct device *dev = pci->dev;
@@ -580,6 +655,18 @@
 
 	ep->msix_cap = dw_pcie_ep_find_capability(pci, PCI_CAP_ID_MSIX);
 
+	offset = dw_pcie_ep_find_ext_capability(pci, PCI_EXT_CAP_ID_REBAR);
+	if (offset) {
+		reg = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL);
+		nbars = (reg & PCI_REBAR_CTRL_NBAR_MASK) >>
+			PCI_REBAR_CTRL_NBAR_SHIFT;
+
+		dw_pcie_dbi_ro_wr_en(pci);
+		for (i = 0; i < nbars; i++, offset += PCI_REBAR_CTRL)
+			dw_pcie_writel_dbi(pci, offset + PCI_REBAR_CAP, 0x0);
+		dw_pcie_dbi_ro_wr_dis(pci);
+	}
+
 	dw_pcie_setup(pci);
 
 	return 0;
diff -Naur --no-dereference a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h
--- a/drivers/pci/controller/dwc/pcie-designware.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/controller/dwc/pcie-designware.h	2020-06-19 21:00:31.420227716 -0400
@@ -36,6 +36,10 @@
 #define PORT_LINK_MODE_4_LANES		(0x7 << 16)
 #define PORT_LINK_MODE_8_LANES		(0xf << 16)
 
+#define PCIE_PORT_DEBUG0		0x728
+#define PORT_LOGIC_LTSSM_STATE_MASK	0x1f
+#define PORT_LOGIC_LTSSM_STATE_L0	0x11
+
 #define PCIE_LINK_WIDTH_SPEED_CONTROL	0x80C
 #define PORT_LOGIC_SPEED_CHANGE		(0x1 << 17)
 #define PORT_LOGIC_LINK_WIDTH_MASK	(0x1f << 8)
@@ -88,12 +92,20 @@
 #define PCIE_ATU_UNR_LOWER_TARGET	0x14
 #define PCIE_ATU_UNR_UPPER_TARGET	0x18
 
+/*
+ * The default address offset between dbi_base and atu_base. Root controller
+ * drivers are not required to initialize atu_base if the offset matches this
+ * default; the driver core automatically derives atu_base from dbi_base using
+ * this offset, if atu_base not set.
+ */
+#define DEFAULT_DBI_ATU_OFFSET (0x3 << 20)
+
 /* Register address builder */
-#define PCIE_GET_ATU_OUTB_UNR_REG_OFFSET(region)	\
-			((0x3 << 20) | ((region) << 9))
+#define PCIE_GET_ATU_OUTB_UNR_REG_OFFSET(region) \
+		((region) << 9)
 
-#define PCIE_GET_ATU_INB_UNR_REG_OFFSET(region)				\
-			((0x3 << 20) | ((region) << 9) | (0x1 << 8))
+#define PCIE_GET_ATU_INB_UNR_REG_OFFSET(region) \
+		(((region) << 9) | (0x1 << 8))
 
 #define MAX_MSI_IRQS			256
 #define MAX_MSI_IRQS_PER_CTRL		32
@@ -130,14 +142,9 @@
 	int (*wr_other_conf)(struct pcie_port *pp, struct pci_bus *bus,
 			     unsigned int devfn, int where, int size, u32 val);
 	int (*host_init)(struct pcie_port *pp);
-	void (*msi_set_irq)(struct pcie_port *pp, int irq);
-	void (*msi_clear_irq)(struct pcie_port *pp, int irq);
-	phys_addr_t (*get_msi_addr)(struct pcie_port *pp);
-	u32 (*get_msi_data)(struct pcie_port *pp, int pos);
 	void (*scan_bus)(struct pcie_port *pp);
 	void (*set_num_vectors)(struct pcie_port *pp);
 	int (*msi_host_init)(struct pcie_port *pp);
-	void (*msi_irq_ack)(int irq, struct pcie_port *pp);
 };
 
 struct pcie_port {
@@ -164,6 +171,7 @@
 	struct irq_domain	*irq_domain;
 	struct irq_domain	*msi_domain;
 	dma_addr_t		msi_data;
+	struct irq_chip		*msi_irq_chip;
 	struct page		*msi_page;
 	u32			num_vectors;
 	u32			irq_status[MAX_MSI_CTRLS];
@@ -181,11 +189,12 @@
 	void	(*ep_init)(struct dw_pcie_ep *ep);
 	int	(*raise_irq)(struct dw_pcie_ep *ep, u8 func_no,
 			     enum pci_epc_irq_type type, u16 interrupt_num);
+	const struct pci_epc_features* (*get_features)(struct dw_pcie_ep *ep);
 };
 
 struct dw_pcie_ep {
 	struct pci_epc		*epc;
-	struct dw_pcie_ep_ops	*ops;
+	const struct dw_pcie_ep_ops *ops;
 	phys_addr_t		phys_base;
 	size_t			addr_size;
 	size_t			page_size;
@@ -199,6 +208,7 @@
 	phys_addr_t		msi_mem_phys;
 	u8			msi_cap;	/* MSI capability offset */
 	u8			msix_cap;	/* MSI-X capability offset */
+	struct pci_epf_bar	*epf_bar[6];
 };
 
 struct dw_pcie_ops {
@@ -207,20 +217,33 @@
 			    size_t size);
 	void	(*write_dbi)(struct dw_pcie *pcie, void __iomem *base, u32 reg,
 			     size_t size, u32 val);
+	u32     (*read_dbi2)(struct dw_pcie *pcie, void __iomem *base, u32 reg,
+			     size_t size);
+	void    (*write_dbi2)(struct dw_pcie *pcie, void __iomem *base, u32 reg,
+			      size_t size, u32 val);
 	int	(*link_up)(struct dw_pcie *pcie);
 	int	(*start_link)(struct dw_pcie *pcie);
 	void	(*stop_link)(struct dw_pcie *pcie);
+	int	(*inbound_atu)(struct dw_pcie *pci, u32 index,
+			       enum pci_barno bar, dma_addr_t cpu_addr);
+	int	(*outbound_atu)(struct dw_pcie *pci, u64 cpu_addr,
+				u64 pci_addr, size_t size);
+	void	(*disable_atu)(struct dw_pcie *pci, phys_addr_t addr, int index,
+			       enum dw_pcie_region_type type);
 };
 
 struct dw_pcie {
 	struct device		*dev;
 	void __iomem		*dbi_base;
 	void __iomem		*dbi_base2;
+	/* Used when iatu_unroll_enabled is true */
+	void __iomem		*atu_base;
 	u32			num_viewport;
 	u8			iatu_unroll_enabled;
 	struct pcie_port	pp;
 	struct dw_pcie_ep	ep;
 	const struct dw_pcie_ops *ops;
+	unsigned int		version;
 };
 
 #define to_dw_pcie_from_pp(port) container_of((port), struct dw_pcie, pp)
@@ -235,11 +258,15 @@
 		       size_t size);
 void __dw_pcie_write_dbi(struct dw_pcie *pci, void __iomem *base, u32 reg,
 			 size_t size, u32 val);
+u32 __dw_pcie_read_dbi2(struct dw_pcie *pci, void __iomem *base, u32 reg,
+			size_t size);
+void __dw_pcie_write_dbi2(struct dw_pcie *pci, void __iomem *base, u32 reg,
+			  size_t size, u32 val);
 int dw_pcie_link_up(struct dw_pcie *pci);
 int dw_pcie_wait_for_link(struct dw_pcie *pci);
-void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index,
-			       int type, u64 cpu_addr, u64 pci_addr,
-			       u32 size);
+int dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index,
+			      int type, u64 cpu_addr, u64 pci_addr,
+			      u32 size);
 int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int bar,
 			     u64 cpu_addr, enum dw_pcie_as_type as_type);
 void dw_pcie_disable_atu(struct dw_pcie *pci, int index,
@@ -278,12 +305,22 @@
 
 static inline void dw_pcie_writel_dbi2(struct dw_pcie *pci, u32 reg, u32 val)
 {
-	__dw_pcie_write_dbi(pci, pci->dbi_base2, reg, 0x4, val);
+	__dw_pcie_write_dbi2(pci, pci->dbi_base2, reg, 0x4, val);
 }
 
 static inline u32 dw_pcie_readl_dbi2(struct dw_pcie *pci, u32 reg)
 {
-	return __dw_pcie_read_dbi(pci, pci->dbi_base2, reg, 0x4);
+	return __dw_pcie_read_dbi2(pci, pci->dbi_base2, reg, 0x4);
+}
+
+static inline void dw_pcie_writel_atu(struct dw_pcie *pci, u32 reg, u32 val)
+{
+	__dw_pcie_write_dbi(pci, pci->atu_base, reg, 0x4, val);
+}
+
+static inline u32 dw_pcie_readl_atu(struct dw_pcie *pci, u32 reg)
+{
+	return __dw_pcie_read_dbi(pci, pci->atu_base, reg, 0x4);
 }
 
 static inline void dw_pcie_dbi_ro_wr_en(struct dw_pcie *pci)
diff -Naur --no-dereference a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c
--- a/drivers/pci/controller/dwc/pcie-designware-host.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/controller/dwc/pcie-designware-host.c	2020-06-19 21:00:31.420227716 -0400
@@ -126,18 +126,12 @@
 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 	u64 msi_target;
 
-	if (pp->ops->get_msi_addr)
-		msi_target = pp->ops->get_msi_addr(pp);
-	else
-		msi_target = (u64)pp->msi_data;
+	msi_target = (u64)pp->msi_data;
 
 	msg->address_lo = lower_32_bits(msi_target);
 	msg->address_hi = upper_32_bits(msi_target);
 
-	if (pp->ops->get_msi_data)
-		msg->data = pp->ops->get_msi_data(pp, data->hwirq);
-	else
-		msg->data = data->hwirq;
+	msg->data = data->hwirq;
 
 	dev_dbg(pci->dev, "msi#%d address_hi %#x address_lo %#x\n",
 		(int)data->hwirq, msg->address_hi, msg->address_lo);
@@ -157,17 +151,13 @@
 
 	raw_spin_lock_irqsave(&pp->lock, flags);
 
-	if (pp->ops->msi_clear_irq) {
-		pp->ops->msi_clear_irq(pp, data->hwirq);
-	} else {
-		ctrl = data->hwirq / MAX_MSI_IRQS_PER_CTRL;
-		res = ctrl * MSI_REG_CTRL_BLOCK_SIZE;
-		bit = data->hwirq % MAX_MSI_IRQS_PER_CTRL;
-
-		pp->irq_status[ctrl] &= ~(1 << bit);
-		dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_MASK + res, 4,
-				    ~pp->irq_status[ctrl]);
-	}
+	ctrl = data->hwirq / MAX_MSI_IRQS_PER_CTRL;
+	res = ctrl * MSI_REG_CTRL_BLOCK_SIZE;
+	bit = data->hwirq % MAX_MSI_IRQS_PER_CTRL;
+
+	pp->irq_status[ctrl] &= ~(1 << bit);
+	dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_MASK + res, 4,
+			    ~pp->irq_status[ctrl]);
 
 	raw_spin_unlock_irqrestore(&pp->lock, flags);
 }
@@ -180,17 +170,13 @@
 
 	raw_spin_lock_irqsave(&pp->lock, flags);
 
-	if (pp->ops->msi_set_irq) {
-		pp->ops->msi_set_irq(pp, data->hwirq);
-	} else {
-		ctrl = data->hwirq / MAX_MSI_IRQS_PER_CTRL;
-		res = ctrl * MSI_REG_CTRL_BLOCK_SIZE;
-		bit = data->hwirq % MAX_MSI_IRQS_PER_CTRL;
-
-		pp->irq_status[ctrl] |= 1 << bit;
-		dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_MASK + res, 4,
-				    ~pp->irq_status[ctrl]);
-	}
+	ctrl = data->hwirq / MAX_MSI_IRQS_PER_CTRL;
+	res = ctrl * MSI_REG_CTRL_BLOCK_SIZE;
+	bit = data->hwirq % MAX_MSI_IRQS_PER_CTRL;
+
+	pp->irq_status[ctrl] |= 1 << bit;
+	dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_MASK + res, 4,
+			    ~pp->irq_status[ctrl]);
 
 	raw_spin_unlock_irqrestore(&pp->lock, flags);
 }
@@ -199,20 +185,12 @@
 {
 	struct pcie_port *pp  = irq_data_get_irq_chip_data(d);
 	unsigned int res, bit, ctrl;
-	unsigned long flags;
 
 	ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL;
 	res = ctrl * MSI_REG_CTRL_BLOCK_SIZE;
 	bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL;
 
-	raw_spin_lock_irqsave(&pp->lock, flags);
-
 	dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_STATUS + res, 4, 1 << bit);
-
-	if (pp->ops->msi_irq_ack)
-		pp->ops->msi_irq_ack(d->hwirq, pp);
-
-	raw_spin_unlock_irqrestore(&pp->lock, flags);
 }
 
 static struct irq_chip dw_pci_msi_bottom_irq_chip = {
@@ -245,7 +223,7 @@
 
 	for (i = 0; i < nr_irqs; i++)
 		irq_domain_set_info(domain, virq + i, bit + i,
-				    &dw_pci_msi_bottom_irq_chip,
+				    pp->msi_irq_chip,
 				    pp, handle_edge_irq,
 				    NULL, NULL);
 
@@ -256,7 +234,7 @@
 				    unsigned int virq, unsigned int nr_irqs)
 {
 	struct irq_data *data = irq_domain_get_irq_data(domain, virq);
-	struct pcie_port *pp = irq_data_get_irq_chip_data(data);
+	struct pcie_port *pp = domain->host_data;
 	unsigned long flags;
 
 	raw_spin_lock_irqsave(&pp->lock, flags);
@@ -460,6 +438,8 @@
 		}
 
 		if (!pp->ops->msi_host_init) {
+			pp->msi_irq_chip = &dw_pci_msi_bottom_irq_chip;
+
 			ret = dw_pcie_allocate_domains(pp);
 			if (ret)
 				return ret;
@@ -645,17 +625,6 @@
 	.write = dw_pcie_wr_conf,
 };
 
-static u8 dw_pcie_iatu_unroll_enabled(struct dw_pcie *pci)
-{
-	u32 val;
-
-	val = dw_pcie_readl_dbi(pci, PCIE_ATU_VIEWPORT);
-	if (val == 0xffffffff)
-		return 1;
-
-	return 0;
-}
-
 void dw_pcie_setup_rc(struct pcie_port *pp)
 {
 	u32 val, ctrl, num_ctrls;
@@ -663,17 +632,19 @@
 
 	dw_pcie_setup(pci);
 
-	num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL;
+	if (!pp->ops->msi_host_init) {
+		num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL;
 
-	/* Initialize IRQ Status array */
-	for (ctrl = 0; ctrl < num_ctrls; ctrl++) {
-		dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_MASK +
-					(ctrl * MSI_REG_CTRL_BLOCK_SIZE),
-				    4, ~0);
-		dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_ENABLE +
-					(ctrl * MSI_REG_CTRL_BLOCK_SIZE),
-				    4, ~0);
-		pp->irq_status[ctrl] = 0;
+		/* Initialize IRQ Status array */
+		for (ctrl = 0; ctrl < num_ctrls; ctrl++) {
+			dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_MASK +
+					    (ctrl * MSI_REG_CTRL_BLOCK_SIZE),
+					    4, ~0);
+			dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_ENABLE +
+					    (ctrl * MSI_REG_CTRL_BLOCK_SIZE),
+					    4, ~0);
+			pp->irq_status[ctrl] = 0;
+		}
 	}
 
 	/* Setup RC BARs */
@@ -707,11 +678,6 @@
 	 * we should not program the ATU here.
 	 */
 	if (!pp->ops->rd_other_conf) {
-		/* Get iATU unroll support */
-		pci->iatu_unroll_enabled = dw_pcie_iatu_unroll_enabled(pci);
-		dev_dbg(pci->dev, "iATU unroll: %s\n",
-			pci->iatu_unroll_enabled ? "enabled" : "disabled");
-
 		dw_pcie_prog_outbound_atu(pci, PCIE_ATU_REGION_INDEX0,
 					  PCIE_ATU_TYPE_MEM, pp->mem_base,
 					  pp->mem_bus_addr, pp->mem_size);
diff -Naur --no-dereference a/drivers/pci/controller/dwc/pcie-designware-plat.c b/drivers/pci/controller/dwc/pcie-designware-plat.c
--- a/drivers/pci/controller/dwc/pcie-designware-plat.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/controller/dwc/pcie-designware-plat.c	2020-06-19 21:00:31.420227716 -0400
@@ -70,14 +70,10 @@
 static void dw_plat_pcie_ep_init(struct dw_pcie_ep *ep)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
-	struct pci_epc *epc = ep->epc;
 	enum pci_barno bar;
 
 	for (bar = BAR_0; bar <= BAR_5; bar++)
 		dw_pcie_ep_reset_bar(pci, bar);
-
-	epc->features |= EPC_FEATURE_NO_LINKUP_NOTIFIER;
-	epc->features |= EPC_FEATURE_MSIX_AVAILABLE;
 }
 
 static int dw_plat_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
@@ -100,9 +96,22 @@
 	return 0;
 }
 
-static struct dw_pcie_ep_ops pcie_ep_ops = {
+static const struct pci_epc_features dw_plat_pcie_epc_features = {
+	.linkup_notifier = false,
+	.msi_capable = true,
+	.msix_capable = true,
+};
+
+static const struct pci_epc_features*
+dw_plat_pcie_get_features(struct dw_pcie_ep *ep)
+{
+	return &dw_plat_pcie_epc_features;
+}
+
+static const struct dw_pcie_ep_ops pcie_ep_ops = {
 	.ep_init = dw_plat_pcie_ep_init,
 	.raise_irq = dw_plat_pcie_ep_raise_irq,
+	.get_features = dw_plat_pcie_get_features,
 };
 
 static int dw_plat_add_pcie_port(struct dw_plat_pcie *dw_plat_pcie,
diff -Naur --no-dereference a/drivers/pci/controller/dwc/pci-keystone.c b/drivers/pci/controller/dwc/pci-keystone.c
--- a/drivers/pci/controller/dwc/pci-keystone.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/controller/dwc/pci-keystone.c	2020-06-19 21:00:31.420227716 -0400
@@ -9,41 +9,634 @@
  * Implementation based on pci-exynos.c and pcie-designware.c
  */
 
-#include <linux/irqchip/chained_irq.h>
 #include <linux/clk.h>
 #include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/init.h>
 #include <linux/interrupt.h>
+#include <linux/irqchip/chained_irq.h>
 #include <linux/irqdomain.h>
-#include <linux/init.h>
+#include <linux/mfd/syscon.h>
 #include <linux/msi.h>
-#include <linux/of_irq.h>
 #include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/of_irq.h>
 #include <linux/of_pci.h>
-#include <linux/platform_device.h>
 #include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
 #include <linux/resource.h>
 #include <linux/signal.h>
 
+#include "../../pci.h"
 #include "pcie-designware.h"
-#include "pci-keystone.h"
 
-#define DRIVER_NAME	"keystone-pcie"
+#define PCIE_VENDORID_MASK	0xffff
+#define PCIE_DEVICEID_SHIFT	16
 
-/* DEV_STAT_CTRL */
-#define PCIE_CAP_BASE		0x70
+/* Application registers */
+#define CMD_STATUS			0x004
+#define LTSSM_EN_VAL		        BIT(0)
+#define OB_XLAT_EN_VAL		        BIT(1)
+#define IB_XLAT_EN_VAL			BIT(2)
+#define DBI_CS2				BIT(5)
+
+#define CFG_SETUP			0x008
+#define CFG_BUS(x)			(((x) & 0xff) << 16)
+#define CFG_DEVICE(x)			(((x) & 0x1f) << 8)
+#define CFG_FUNC(x)			((x) & 0x7)
+#define CFG_TYPE1			BIT(24)
+
+#define OB_SIZE				0x030
+#define OB_OFFSET_INDEX(n)		(0x200 + (8 * (n)))
+#define OB_OFFSET_HI(n)			(0x204 + (8 * (n)))
+#define OB_ENABLEN			BIT(0)
+#define OB_WIN_SIZE			8	/* 8MB */
+
+#define PCIE_LEGACY_IRQ_ENABLE_SET(n)	(0x188 + (0x10 * ((n) - 1)))
+#define PCIE_LEGACY_IRQ_ENABLE_CLR(n)	(0x18c + (0x10 * ((n) - 1)))
+#define PCIE_EP_IRQ_SET			0x64
+#define PCIE_EP_IRQ_CLR			0x68
+#define INT_ENABLE			BIT(0)
+
+/* IRQ register defines */
+#define IRQ_EOI				0x050
+
+#define MSI_IRQ				0x054
+#define MSI_IRQ_STATUS(n)		(0x104 + ((n) << 4))
+#define MSI_IRQ_ENABLE_SET(n)		(0x108 + ((n) << 4))
+#define MSI_IRQ_ENABLE_CLR(n)		(0x10c + ((n) << 4))
+#define MSI_IRQ_OFFSET			4
+
+#define IRQ_STATUS(n)			(0x184 + ((n) << 4))
+#define IRQ_ENABLE_SET(n)		(0x188 + ((n) << 4))
+#define IRQ_ENABLE_CLR(n)		(0x18C + ((n) << 4))
+#define INTx_EN				BIT(0)
+
+#define ERR_IRQ_STATUS			0x1c4
+#define ERR_IRQ_ENABLE_SET		0x1c8
+#define ERR_AER				BIT(5)	/* ECRC error */
+#define ERR_AXI				BIT(4)	/* AXI tag lookup fatal error */
+#define ERR_CORR			BIT(3)	/* Correctable error */
+#define ERR_NONFATAL			BIT(2)	/* Non-fatal error */
+#define ERR_FATAL			BIT(1)	/* Fatal error */
+#define ERR_SYS				BIT(0)	/* System error */
+#define ERR_IRQ_ALL			(ERR_AER | ERR_AXI | ERR_CORR | \
+					 ERR_NONFATAL | ERR_FATAL | ERR_SYS)
 
 /* PCIE controller device IDs */
-#define PCIE_RC_K2HK		0xb008
-#define PCIE_RC_K2E		0xb009
-#define PCIE_RC_K2L		0xb00a
-#define PCIE_RC_K2G		0xb00b
+#define PCIE_RC_K2HK			0xb008
+#define PCIE_RC_K2E			0xb009
+#define PCIE_RC_K2L			0xb00a
+#define PCIE_RC_K2G			0xb00b
+
+#define KS_PCIE_DEV_TYPE_MASK		(0x3 << 1)
+#define KS_PCIE_DEV_TYPE(mode)		((mode) << 1)
+
+#define EP				0x0
+#define LEG_EP				0x1
+#define RC				0x2
+
+#define KS_PCIE_SYSCLOCKOUTEN		BIT(0)
 
-#define to_keystone_pcie(x)	dev_get_drvdata((x)->dev)
+#define AM654_PCIE_DEV_TYPE_MASK	0x3
+#define AM654_WIN_SIZE			SZ_64K
+
+#define APP_ADDR_SPACE_0		(16 * SZ_1K)
+
+#define EXP_CAP_ID_OFFSET		0x70
+
+#define IB_BAR(n)			(0x300 + (0x10 * (n)))
+#define IB_START_LO(n)			(0x304 + (0x10 * (n)))
+#define IB_START_HI(n)			(0x308 + (0x10 * (n)))
+#define IB_OFFSET(n)			(0x30c + (0x10 * (n)))
+
+#define KS_PCIE_WIN_SIZE		SZ_1M
+#define KS_PCIE_WIN_INDEX_MASK		0x1f
+#define KS_PCIE_WIN_INDEX_SHIFT		20
+
+#define to_keystone_pcie(x)		dev_get_drvdata((x)->dev)
+
+#define PCI_DEVICE_ID_TI_AM654X		0xb00c
+
+struct ks_pcie_of_data {
+	enum dw_pcie_device_mode mode;
+	const struct dw_pcie_host_ops *host_ops;
+	const struct dw_pcie_ep_ops *ep_ops;
+	unsigned int version;
+};
+
+struct ks_pcie_outbound_win {
+	u64	cpu_addr;
+	u8	no_of_regions;
+};
 
-static void quirk_limit_mrrs(struct pci_dev *dev)
+struct keystone_pcie {
+	struct dw_pcie		*pci;
+	/* PCI Device ID */
+	u32			device_id;
+	struct			device_node *legacy_intc_np;
+
+	int			msi_host_irq;
+	struct irq_domain	*legacy_irq_domain;
+	int			num_lanes;
+	u32			num_viewport;
+	struct phy		**phy;
+	struct device_link	**link;
+	struct			device_node *msi_intc_np;
+	struct device_node	*np;
+
+	/* Application register space */
+	void __iomem		*va_app_base;	/* DT 1st resource */
+	struct resource		app;
+	bool			is_am6;
+	unsigned long		ob_window_map;
+	struct ks_pcie_outbound_win *ob_win;
+};
+
+static u32 ks_pcie_app_readl(struct keystone_pcie *ks_pcie, u32 offset)
+{
+	return readl(ks_pcie->va_app_base + offset);
+}
+
+static void ks_pcie_app_writel(struct keystone_pcie *ks_pcie, u32 offset,
+			       u32 val)
+{
+	writel(val, ks_pcie->va_app_base + offset);
+}
+
+static void ks_pcie_msi_irq_ack(struct irq_data *data)
+{
+	struct pcie_port *pp  = irq_data_get_irq_chip_data(data);
+	struct keystone_pcie *ks_pcie;
+	u32 irq = data->hwirq;
+	struct dw_pcie *pci;
+	u32 reg_offset;
+	u32 bit_pos;
+
+	pci = to_dw_pcie_from_pp(pp);
+	ks_pcie = to_keystone_pcie(pci);
+
+	reg_offset = irq % 8;
+	bit_pos = irq >> 3;
+
+	ks_pcie_app_writel(ks_pcie, MSI_IRQ_STATUS(reg_offset),
+			   BIT(bit_pos));
+	ks_pcie_app_writel(ks_pcie, IRQ_EOI, reg_offset + MSI_IRQ_OFFSET);
+}
+
+static void ks_pcie_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
+{
+	struct pcie_port *pp = irq_data_get_irq_chip_data(data);
+	struct keystone_pcie *ks_pcie;
+	struct dw_pcie *pci;
+	u64 msi_target;
+
+	pci = to_dw_pcie_from_pp(pp);
+	ks_pcie = to_keystone_pcie(pci);
+
+	msi_target = ks_pcie->app.start + MSI_IRQ;
+	msg->address_lo = lower_32_bits(msi_target);
+	msg->address_hi = upper_32_bits(msi_target);
+	msg->data = data->hwirq;
+
+	dev_dbg(pci->dev, "msi#%d address_hi %#x address_lo %#x\n",
+		(int)data->hwirq, msg->address_hi, msg->address_lo);
+}
+
+static int ks_pcie_msi_set_affinity(struct irq_data *irq_data,
+				    const struct cpumask *mask, bool force)
+{
+	return -EINVAL;
+}
+
+static void ks_pcie_msi_mask(struct irq_data *data)
+{
+	struct pcie_port *pp = irq_data_get_irq_chip_data(data);
+	struct keystone_pcie *ks_pcie;
+	u32 irq = data->hwirq;
+	struct dw_pcie *pci;
+	unsigned long flags;
+	u32 reg_offset;
+	u32 bit_pos;
+
+	raw_spin_lock_irqsave(&pp->lock, flags);
+
+	pci = to_dw_pcie_from_pp(pp);
+	ks_pcie = to_keystone_pcie(pci);
+
+	reg_offset = irq % 8;
+	bit_pos = irq >> 3;
+
+	ks_pcie_app_writel(ks_pcie, MSI_IRQ_ENABLE_CLR(reg_offset),
+			   BIT(bit_pos));
+
+	raw_spin_unlock_irqrestore(&pp->lock, flags);
+}
+
+static void ks_pcie_msi_unmask(struct irq_data *data)
+{
+	struct pcie_port *pp = irq_data_get_irq_chip_data(data);
+	struct keystone_pcie *ks_pcie;
+	u32 irq = data->hwirq;
+	struct dw_pcie *pci;
+	unsigned long flags;
+	u32 reg_offset;
+	u32 bit_pos;
+
+	raw_spin_lock_irqsave(&pp->lock, flags);
+
+	pci = to_dw_pcie_from_pp(pp);
+	ks_pcie = to_keystone_pcie(pci);
+
+	reg_offset = irq % 8;
+	bit_pos = irq >> 3;
+
+	ks_pcie_app_writel(ks_pcie, MSI_IRQ_ENABLE_SET(reg_offset),
+			   BIT(bit_pos));
+
+	raw_spin_unlock_irqrestore(&pp->lock, flags);
+}
+
+static struct irq_chip ks_pcie_msi_irq_chip = {
+	.name = "KEYSTONE-PCI-MSI",
+	.irq_ack = ks_pcie_msi_irq_ack,
+	.irq_compose_msi_msg = ks_pcie_compose_msi_msg,
+	.irq_set_affinity = ks_pcie_msi_set_affinity,
+	.irq_mask = ks_pcie_msi_mask,
+	.irq_unmask = ks_pcie_msi_unmask,
+};
+
+static int ks_pcie_msi_host_init(struct pcie_port *pp)
+{
+	pp->msi_irq_chip = &ks_pcie_msi_irq_chip;
+	return dw_pcie_allocate_domains(pp);
+}
+
+static int ks_pcie_am654_msi_host_init(struct pcie_port *pp)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
+	struct device *dev = pci->dev;
+
+	dev_vdbg(dev, "dummy function so that DW core doesn't configure MSI\n");
+
+	return 0;
+}
+
+static void ks_pcie_enable_error_irq(struct keystone_pcie *ks_pcie)
+{
+	ks_pcie_app_writel(ks_pcie, ERR_IRQ_ENABLE_SET, ERR_IRQ_ALL);
+}
+
+static irqreturn_t ks_pcie_handle_error_irq(struct keystone_pcie *ks_pcie)
+{
+	u32 reg;
+	struct device *dev = ks_pcie->pci->dev;
+
+	reg = ks_pcie_app_readl(ks_pcie, ERR_IRQ_STATUS);
+	if (!reg)
+		return IRQ_NONE;
+
+	if (reg & ERR_SYS)
+		dev_err(dev, "System Error\n");
+
+	if (reg & ERR_FATAL)
+		dev_err(dev, "Fatal Error\n");
+
+	if (reg & ERR_NONFATAL)
+		dev_dbg(dev, "Non Fatal Error\n");
+
+	if (reg & ERR_CORR)
+		dev_dbg(dev, "Correctable Error\n");
+
+	if (reg & ERR_AXI)
+		dev_err(dev, "AXI tag lookup fatal Error\n");
+
+	if (reg & ERR_AER)
+		dev_err(dev, "ECRC Error\n");
+
+	ks_pcie_app_writel(ks_pcie, ERR_IRQ_STATUS, reg);
+
+	return IRQ_HANDLED;
+}
+
+static void ks_pcie_legacy_irq_handler(struct irq_desc *desc)
+{
+	struct keystone_pcie *ks_pcie = irq_desc_get_handler_data(desc);
+	struct irq_chip *chip = irq_desc_get_chip(desc);
+	int virq, i;
+	u32 reg;
+
+	chained_irq_enter(chip, desc);
+
+	for (i = 0; i < PCI_NUM_INTX; i++) {
+		reg = ks_pcie_app_readl(ks_pcie, IRQ_STATUS(i));
+		if (!(reg & INTx_EN))
+			continue;
+
+		virq = irq_linear_revmap(ks_pcie->legacy_irq_domain, i);
+		generic_handle_irq(virq);
+		ks_pcie_app_writel(ks_pcie, IRQ_STATUS(i), INTx_EN);
+		ks_pcie_app_writel(ks_pcie, IRQ_EOI, i);
+	}
+
+	chained_irq_exit(chip, desc);
+}
+
+void ks_pcie_irq_eoi(struct irq_data *data)
+{
+	struct keystone_pcie *ks_pcie = irq_data_get_irq_chip_data(data);
+	irq_hw_number_t hwirq = data->hwirq;
+
+	ks_pcie_app_writel(ks_pcie, IRQ_EOI, hwirq);
+	irq_chip_eoi_parent(data);
+}
+
+void ks_pcie_irq_enable(struct irq_data *data)
+{
+	struct keystone_pcie *ks_pcie = irq_data_get_irq_chip_data(data);
+	irq_hw_number_t hwirq = data->hwirq;
+
+	ks_pcie_app_writel(ks_pcie, IRQ_ENABLE_SET(hwirq), INTx_EN);
+	irq_chip_enable_parent(data);
+}
+
+void ks_pcie_irq_disable(struct irq_data *data)
+{
+	struct keystone_pcie *ks_pcie = irq_data_get_irq_chip_data(data);
+	irq_hw_number_t hwirq = data->hwirq;
+
+	ks_pcie_app_writel(ks_pcie, IRQ_ENABLE_CLR(hwirq), INTx_EN);
+	irq_chip_disable_parent(data);
+}
+
+static struct irq_chip ks_pcie_legacy_irq_chip = {
+	.name			= "Keystone-PCI-Legacy-IRQ",
+	.irq_enable		= ks_pcie_irq_enable,
+	.irq_disable		= ks_pcie_irq_disable,
+	.irq_eoi		= ks_pcie_irq_eoi,
+	.irq_mask		= irq_chip_mask_parent,
+	.irq_unmask		= irq_chip_unmask_parent,
+	.irq_retrigger		= irq_chip_retrigger_hierarchy,
+	.irq_set_type		= irq_chip_set_type_parent,
+	.irq_set_affinity	= irq_chip_set_affinity_parent,
+};
+
+static int ks_pcie_legacy_irq_domain_alloc(struct irq_domain *domain,
+					   unsigned int virq,
+					   unsigned int nr_irqs, void *data)
+{
+	struct keystone_pcie *ks_pcie = domain->host_data;
+	struct device_node *np = ks_pcie->legacy_intc_np;
+	struct irq_fwspec parent_fwspec, *fwspec = data;
+	struct of_phandle_args out_irq;
+	int ret, i;
+
+	if (nr_irqs != 1)
+		return -EINVAL;
+
+	ret = of_irq_parse_one(np, fwspec->param[0], &out_irq);
+	if (ret < 0) {
+		pr_err("Failed to parse interrupt node\n");
+		return ret;
+	}
+
+	parent_fwspec.fwnode = &out_irq.np->fwnode;
+	parent_fwspec.param_count = out_irq.args_count;
+
+	for (i = 0; i < out_irq.args_count; i++)
+		parent_fwspec.param[i] = out_irq.args[i];
+
+	ret = irq_domain_alloc_irqs_parent(domain, virq, 1, &parent_fwspec);
+	if (ret < 0) {
+		pr_err("Failed to allocate parent irq %u: %d\n",
+		       parent_fwspec.param[0], ret);
+		return ret;
+	}
+
+	ret = irq_domain_set_hwirq_and_chip(domain, virq, fwspec->param[0],
+					    &ks_pcie_legacy_irq_chip, ks_pcie);
+	if (ret < 0) {
+		pr_err("Failed to set hwirq and chip\n");
+		goto err_set_hwirq_and_chip;
+	}
+
+	return 0;
+
+err_set_hwirq_and_chip:
+	irq_domain_free_irqs_parent(domain, virq, 1);
+
+	return ret;
+}
+
+static int ks_pcie_irq_domain_translate(struct irq_domain *domain,
+					struct irq_fwspec *fwspec,
+					unsigned long *hwirq,
+					unsigned int *type)
+{
+	if (is_of_node(fwspec->fwnode)) {
+		if (fwspec->param_count != 2)
+			return -EINVAL;
+
+		if (fwspec->param[0] >= PCI_NUM_INTX)
+			return -EINVAL;
+
+		*hwirq = fwspec->param[0];
+		*type = fwspec->param[1];
+
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+static const struct irq_domain_ops ks_pcie_legacy_irq_domain_ops = {
+	.alloc		= ks_pcie_legacy_irq_domain_alloc,
+	.free		= irq_domain_free_irqs_common,
+	.translate	= ks_pcie_irq_domain_translate,
+};
+
+/**
+ * ks_pcie_set_dbi_mode() - Set DBI mode to access overlaid BAR mask
+ * registers
+ *
+ * Since modification of dbi_cs2 involves different clock domain, read the
+ * status back to ensure the transition is complete.
+ */
+static void ks_pcie_set_dbi_mode(struct keystone_pcie *ks_pcie)
+{
+	u32 val;
+
+	val = ks_pcie_app_readl(ks_pcie, CMD_STATUS);
+	val |= DBI_CS2;
+	ks_pcie_app_writel(ks_pcie, CMD_STATUS, val);
+
+	do {
+		val = ks_pcie_app_readl(ks_pcie, CMD_STATUS);
+	} while (!(val & DBI_CS2));
+}
+
+/**
+ * ks_pcie_clear_dbi_mode() - Disable DBI mode
+ *
+ * Since modification of dbi_cs2 involves different clock domain, read the
+ * status back to ensure the transition is complete.
+ */
+static void ks_pcie_clear_dbi_mode(struct keystone_pcie *ks_pcie)
+{
+	u32 val;
+
+	val = ks_pcie_app_readl(ks_pcie, CMD_STATUS);
+	val &= ~DBI_CS2;
+	ks_pcie_app_writel(ks_pcie, CMD_STATUS, val);
+
+	do {
+		val = ks_pcie_app_readl(ks_pcie, CMD_STATUS);
+	} while (val & DBI_CS2);
+}
+
+static void ks_pcie_setup_rc_app_regs(struct keystone_pcie *ks_pcie)
+{
+	u32 val;
+	u32 num_viewport = ks_pcie->num_viewport;
+	struct dw_pcie *pci = ks_pcie->pci;
+	struct pcie_port *pp = &pci->pp;
+	u64 start = pp->mem->start;
+	u64 end = pp->mem->end;
+	int i;
+
+	/* Disable BARs for inbound access */
+	ks_pcie_set_dbi_mode(ks_pcie);
+	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0);
+	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_1, 0);
+	ks_pcie_clear_dbi_mode(ks_pcie);
+
+	if (ks_pcie->is_am6)
+		return;
+
+	val = ilog2(OB_WIN_SIZE);
+	ks_pcie_app_writel(ks_pcie, OB_SIZE, val);
+
+	/* Using Direct 1:1 mapping of RC <-> PCI memory space */
+	for (i = 0; i < num_viewport && (start < end); i++) {
+		ks_pcie_app_writel(ks_pcie, OB_OFFSET_INDEX(i),
+				   lower_32_bits(start) | OB_ENABLEN);
+		ks_pcie_app_writel(ks_pcie, OB_OFFSET_HI(i),
+				   upper_32_bits(start));
+		start += (OB_WIN_SIZE * SZ_1M);
+	}
+
+	val = ks_pcie_app_readl(ks_pcie, CMD_STATUS);
+	val |= OB_XLAT_EN_VAL;
+	ks_pcie_app_writel(ks_pcie, CMD_STATUS, val);
+}
+
+static int ks_pcie_rd_other_conf(struct pcie_port *pp, struct pci_bus *bus,
+				 unsigned int devfn, int where, int size,
+				 u32 *val)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
+	struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
+	u32 reg;
+
+	reg = CFG_BUS(bus->number) | CFG_DEVICE(PCI_SLOT(devfn)) |
+		CFG_FUNC(PCI_FUNC(devfn));
+	if (bus->parent->number != pp->root_bus_nr)
+		reg |= CFG_TYPE1;
+	ks_pcie_app_writel(ks_pcie, CFG_SETUP, reg);
+
+	return dw_pcie_read(pp->va_cfg0_base + where, size, val);
+}
+
+static int ks_pcie_wr_other_conf(struct pcie_port *pp, struct pci_bus *bus,
+				 unsigned int devfn, int where, int size,
+				 u32 val)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
+	struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
+	u32 reg;
+
+	reg = CFG_BUS(bus->number) | CFG_DEVICE(PCI_SLOT(devfn)) |
+		CFG_FUNC(PCI_FUNC(devfn));
+	if (bus->parent->number != pp->root_bus_nr)
+		reg |= CFG_TYPE1;
+	ks_pcie_app_writel(ks_pcie, CFG_SETUP, reg);
+
+	return dw_pcie_write(pp->va_cfg0_base + where, size, val);
+}
+
+/**
+ * ks_pcie_v3_65_scan_bus() - keystone scan_bus post initialization
+ *
+ * This sets BAR0 to enable inbound access for MSI_IRQ register
+ */
+static void ks_pcie_v3_65_scan_bus(struct pcie_port *pp)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
+	struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
+
+	/* Configure and set up BAR0 */
+	ks_pcie_set_dbi_mode(ks_pcie);
+
+	/* Enable BAR0 */
+	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 1);
+	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, SZ_4K - 1);
+
+	ks_pcie_clear_dbi_mode(ks_pcie);
+
+	 /*
+	  * For BAR0, just setting bus address for inbound writes (MSI) should
+	  * be sufficient.  Use physical address to avoid any conflicts.
+	  */
+	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, ks_pcie->app.start);
+}
+
+/**
+ * ks_pcie_link_up() - Check if link up
+ */
+static int ks_pcie_link_up(struct dw_pcie *pci)
+{
+	u32 val;
+
+	val = dw_pcie_readl_dbi(pci, PCIE_PORT_DEBUG0);
+	val &= PORT_LOGIC_LTSSM_STATE_MASK;
+	return (val == PORT_LOGIC_LTSSM_STATE_L0);
+}
+
+static void ks_pcie_stop_link(struct dw_pcie *pci)
+{
+	struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
+	u32 val;
+
+	/* Disable Link training */
+	val = ks_pcie_app_readl(ks_pcie, CMD_STATUS);
+	val &= ~LTSSM_EN_VAL;
+	ks_pcie_app_writel(ks_pcie, CMD_STATUS, val);
+}
+
+static int ks_pcie_start_link(struct dw_pcie *pci)
+{
+	struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
+	struct device *dev = pci->dev;
+	u32 val;
+
+	if (dw_pcie_link_up(pci)) {
+		dev_dbg(dev, "link is already up\n");
+		return 0;
+	}
+
+	/* Initiate Link Training */
+	val = ks_pcie_app_readl(ks_pcie, CMD_STATUS);
+	ks_pcie_app_writel(ks_pcie, CMD_STATUS, LTSSM_EN_VAL | val);
+
+	return 0;
+}
+
+#ifdef CONFIG_PCI_KEYSTONE_HOST
+static void ks_pcie_quirk(struct pci_dev *dev)
 {
 	struct pci_bus *bus = dev->bus;
-	struct pci_dev *bridge = bus->self;
+	struct pci_dev *bridge;
 	static const struct pci_device_id rc_pci_devids[] = {
 		{ PCI_DEVICE(PCI_VENDOR_ID_TI, PCIE_RC_K2HK),
 		 .class = PCI_CLASS_BRIDGE_PCI << 8, .class_mask = ~0, },
@@ -55,9 +648,14 @@
 		 .class = PCI_CLASS_BRIDGE_PCI << 8, .class_mask = ~0, },
 		{ 0, },
 	};
+	static const struct pci_device_id am6_pci_devids[] = {
+		{ PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_AM654X),
+		 .class = PCI_CLASS_BRIDGE_PCI << 8, .class_mask = ~0, },
+		{ 0, },
+	};
 
 	if (pci_is_root_bus(bus))
-		return;
+		bridge = dev;
 
 	/* look for the host bridge */
 	while (!pci_is_root_bus(bus)) {
@@ -65,56 +663,47 @@
 		bus = bus->parent;
 	}
 
-	if (bridge) {
-		/*
-		 * Keystone PCI controller has a h/w limitation of
-		 * 256 bytes maximum read request size.  It can't handle
-		 * anything higher than this.  So force this limit on
-		 * all downstream devices.
-		 */
-		if (pci_match_id(rc_pci_devids, bridge)) {
-			if (pcie_get_readrq(dev) > 256) {
-				dev_info(&dev->dev, "limiting MRRS to 256\n");
-				pcie_set_readrq(dev, 256);
-			}
-		}
-	}
-}
-DECLARE_PCI_FIXUP_ENABLE(PCI_ANY_ID, PCI_ANY_ID, quirk_limit_mrrs);
-
-static int ks_pcie_establish_link(struct keystone_pcie *ks_pcie)
-{
-	struct dw_pcie *pci = ks_pcie->pci;
-	struct pcie_port *pp = &pci->pp;
-	struct device *dev = pci->dev;
-	unsigned int retries;
-
-	dw_pcie_setup_rc(pp);
+	if (!bridge)
+		return;
 
-	if (dw_pcie_link_up(pci)) {
-		dev_info(dev, "Link already up\n");
-		return 0;
+	/*
+	 * Keystone PCI controller has a h/w limitation of
+	 * 256 bytes maximum read request size.  It can't handle
+	 * anything higher than this.  So force this limit on
+	 * all downstream devices.
+	 */
+	if (pci_match_id(rc_pci_devids, bridge)) {
+		if (pcie_get_readrq(dev) > 256) {
+			dev_info(&dev->dev, "limiting MRRS to 256\n");
+			pcie_set_readrq(dev, 256);
+		}
 	}
 
-	/* check if the link is up or not */
-	for (retries = 0; retries < 5; retries++) {
-		ks_dw_pcie_initiate_link_train(ks_pcie);
-		if (!dw_pcie_wait_for_link(pci))
-			return 0;
+	/*
+	 * Memory transactions fail with PCI controller in AM654 PG1.0
+	 * when MRRS is set to more than 128 Bytes. Force the MRRS to
+	 * 128 Bytes in all downstream devices.
+	 */
+	if (pci_match_id(am6_pci_devids, bridge)) {
+		if (pcie_get_readrq(dev) > 128) {
+			dev_info(&dev->dev, "limiting MRRS to 128\n");
+			pcie_set_readrq(dev, 128);
+		}
 	}
-
-	dev_err(dev, "phy link never came up\n");
-	return -ETIMEDOUT;
 }
+DECLARE_PCI_FIXUP_ENABLE(PCI_ANY_ID, PCI_ANY_ID, ks_pcie_quirk);
+#endif
 
 static void ks_pcie_msi_irq_handler(struct irq_desc *desc)
 {
-	unsigned int irq = irq_desc_get_irq(desc);
+	unsigned int irq = desc->irq_data.hwirq;
 	struct keystone_pcie *ks_pcie = irq_desc_get_handler_data(desc);
-	u32 offset = irq - ks_pcie->msi_host_irqs[0];
+	u32 offset = irq - ks_pcie->msi_host_irq;
 	struct dw_pcie *pci = ks_pcie->pci;
+	struct pcie_port *pp = &pci->pp;
 	struct device *dev = pci->dev;
 	struct irq_chip *chip = irq_desc_get_chip(desc);
+	u32 vector, virq, reg, pos;
 
 	dev_dbg(dev, "%s, irq %d\n", __func__, irq);
 
@@ -124,120 +713,200 @@
 	 * ack operation.
 	 */
 	chained_irq_enter(chip, desc);
-	ks_dw_pcie_handle_msi_irq(ks_pcie, offset);
-	chained_irq_exit(chip, desc);
-}
-
-/**
- * ks_pcie_legacy_irq_handler() - Handle legacy interrupt
- * @irq: IRQ line for legacy interrupts
- * @desc: Pointer to irq descriptor
- *
- * Traverse through pending legacy interrupts and invoke handler for each. Also
- * takes care of interrupt controller level mask/ack operation.
- */
-static void ks_pcie_legacy_irq_handler(struct irq_desc *desc)
-{
-	unsigned int irq = irq_desc_get_irq(desc);
-	struct keystone_pcie *ks_pcie = irq_desc_get_handler_data(desc);
-	struct dw_pcie *pci = ks_pcie->pci;
-	struct device *dev = pci->dev;
-	u32 irq_offset = irq - ks_pcie->legacy_host_irqs[0];
-	struct irq_chip *chip = irq_desc_get_chip(desc);
-
-	dev_dbg(dev, ": Handling legacy irq %d\n", irq);
 
+	reg = ks_pcie_app_readl(ks_pcie, MSI_IRQ_STATUS(offset));
 	/*
-	 * The chained irq handler installation would have replaced normal
-	 * interrupt driver handler so we need to take care of mask/unmask and
-	 * ack operation.
+	 * MSI0 status bit 0-3 shows vectors 0, 8, 16, 24, MSI1 status bit
+	 * shows 1, 9, 17, 25 and so forth
 	 */
-	chained_irq_enter(chip, desc);
-	ks_dw_pcie_handle_legacy_irq(ks_pcie, irq_offset);
+	for (pos = 0; pos < 4; pos++) {
+		if (!(reg & BIT(pos)))
+			continue;
+
+		vector = offset + (pos << 3);
+		virq = irq_linear_revmap(pp->irq_domain, vector);
+		dev_dbg(dev, "irq: bit %d, vector %d, virq %d\n", pos, vector,
+			virq);
+		generic_handle_irq(virq);
+	}
+
 	chained_irq_exit(chip, desc);
 }
 
-static int ks_pcie_get_irq_controller_info(struct keystone_pcie *ks_pcie,
-					   char *controller, int *num_irqs)
+static int ks_pcie_config_msi_irq(struct keystone_pcie *ks_pcie)
 {
-	int temp, max_host_irqs, legacy = 1, *host_irqs;
 	struct device *dev = ks_pcie->pci->dev;
-	struct device_node *np_pcie = dev->of_node, **np_temp;
+	struct device_node *np = ks_pcie->np;
+	struct device_node *intc_np;
+	struct irq_data *irq_data;
+	int irq_count, irq, ret, i;
 
-	if (!strcmp(controller, "msi-interrupt-controller"))
-		legacy = 0;
+	if (!IS_ENABLED(CONFIG_PCI_MSI))
+		return 0;
 
-	if (legacy) {
-		np_temp = &ks_pcie->legacy_intc_np;
-		max_host_irqs = PCI_NUM_INTX;
-		host_irqs = &ks_pcie->legacy_host_irqs[0];
-	} else {
-		np_temp = &ks_pcie->msi_intc_np;
-		max_host_irqs = MAX_MSI_HOST_IRQS;
-		host_irqs =  &ks_pcie->msi_host_irqs[0];
+	intc_np = of_get_child_by_name(np, "msi-interrupt-controller");
+	if (!intc_np) {
+		if (ks_pcie->is_am6)
+			return 0;
+		dev_warn(dev, "msi-interrupt-controller node is absent\n");
+		return -EINVAL;
 	}
 
-	/* interrupt controller is in a child node */
-	*np_temp = of_get_child_by_name(np_pcie, controller);
-	if (!(*np_temp)) {
-		dev_err(dev, "Node for %s is absent\n", controller);
-		return -EINVAL;
+	irq_count = of_irq_count(intc_np);
+	if (!irq_count) {
+		dev_err(dev, "No IRQ entries in msi-interrupt-controller\n");
+		ret = -EINVAL;
+		goto err;
 	}
 
-	temp = of_irq_count(*np_temp);
-	if (!temp) {
-		dev_err(dev, "No IRQ entries in %s\n", controller);
-		of_node_put(*np_temp);
-		return -EINVAL;
+	for (i = 0; i < irq_count; i++) {
+		irq = irq_of_parse_and_map(intc_np, i);
+		if (!irq) {
+			ret = -EINVAL;
+			goto err;
+		}
+
+		if (!ks_pcie->msi_host_irq) {
+			irq_data = irq_get_irq_data(irq);
+			if (!irq_data) {
+				ret = -EINVAL;
+				goto err;
+			}
+			ks_pcie->msi_host_irq = irq_data->hwirq;
+		}
+
+		irq_set_chained_handler_and_data(irq, ks_pcie_msi_irq_handler,
+						 ks_pcie);
 	}
 
-	if (temp > max_host_irqs)
-		dev_warn(dev, "Too many %s interrupts defined %u\n",
-			(legacy ? "legacy" : "MSI"), temp);
+	of_node_put(intc_np);
+	return 0;
 
-	/*
-	 * support upto max_host_irqs. In dt from index 0 to 3 (legacy) or 0 to
-	 * 7 (MSI)
-	 */
-	for (temp = 0; temp < max_host_irqs; temp++) {
-		host_irqs[temp] = irq_of_parse_and_map(*np_temp, temp);
-		if (!host_irqs[temp])
-			break;
+err:
+	of_node_put(intc_np);
+	return ret;
+}
+
+static int ks_pcie_k2_config_legacy_irq(struct keystone_pcie *ks_pcie,
+					struct device_node *intc_np)
+{
+	struct device *dev = ks_pcie->pci->dev;
+	struct irq_domain *legacy_irq_domain;
+	struct irq_domain *parent_domain;
+	struct device_node *parent_node;
+	int ret = 0;
+
+	parent_node = of_irq_find_parent(intc_np);
+	if (!parent_node) {
+		dev_err(dev, "unable to obtain parent node\n");
+		ret = -ENXIO;
+		return ret;
 	}
 
-	of_node_put(*np_temp);
+	parent_domain = irq_find_host(parent_node);
+	if (!parent_domain) {
+		dev_err(dev, "unable to obtain parent domain\n");
+		ret = -ENXIO;
+		goto err;
+	}
 
-	if (temp) {
-		*num_irqs = temp;
-		return 0;
+	legacy_irq_domain =
+		irq_domain_add_hierarchy(parent_domain, 0, PCI_NUM_INTX,
+					 intc_np,
+					 &ks_pcie_legacy_irq_domain_ops,
+					 ks_pcie);
+	if (!legacy_irq_domain) {
+		dev_err(dev, "Failed to add irq domain for legacy irqs\n");
+		ret = -EINVAL;
 	}
 
-	return -EINVAL;
+err:
+	of_node_put(parent_node);
+	return ret;
+}
+
+static int ks_pcie_intx_map(struct irq_domain *domain, unsigned int irq,
+			    irq_hw_number_t hwirq)
+{
+	irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq);
+	irq_set_chip_data(irq, domain->host_data);
+
+	return 0;
 }
 
-static void ks_pcie_setup_interrupts(struct keystone_pcie *ks_pcie)
+static const struct irq_domain_ops ks_pcie_am6_irq_domain_ops = {
+	.map = ks_pcie_intx_map,
+};
+
+static int ks_pcie_am6_config_legacy_irq(struct keystone_pcie *ks_pcie,
+					 struct device_node *intc_np)
 {
+	struct device *dev = ks_pcie->pci->dev;
+	struct irq_domain *legacy_irq_domain;
+	int ret = 0;
+	int irq;
 	int i;
 
-	/* Legacy IRQ */
-	for (i = 0; i < ks_pcie->num_legacy_host_irqs; i++) {
-		irq_set_chained_handler_and_data(ks_pcie->legacy_host_irqs[i],
-						 ks_pcie_legacy_irq_handler,
-						 ks_pcie);
+	irq = irq_of_parse_and_map(intc_np, 0);
+	if (!irq)
+		return -EINVAL;
+
+	irq_set_chained_handler_and_data(irq, ks_pcie_legacy_irq_handler,
+					 ks_pcie);
+
+	legacy_irq_domain = irq_domain_add_linear(intc_np, PCI_NUM_INTX,
+						  &ks_pcie_am6_irq_domain_ops,
+						  ks_pcie);
+	if (!legacy_irq_domain) {
+		dev_err(dev, "Failed to add irq domain for legacy irqs\n");
+		return -EINVAL;
+	}
+	ks_pcie->legacy_irq_domain = legacy_irq_domain;
+
+	for (i = 0; i < PCI_NUM_INTX; i++)
+		ks_pcie_app_writel(ks_pcie, IRQ_ENABLE_SET(i), INTx_EN);
+
+	return ret;
+}
+
+static int ks_pcie_config_legacy_irq(struct keystone_pcie *ks_pcie)
+{
+	struct device *dev = ks_pcie->pci->dev;
+	struct device_node *np = ks_pcie->np;
+	struct device_node *intc_np;
+	int irq_count, ret = 0;
+
+	intc_np = of_get_child_by_name(np, "legacy-interrupt-controller");
+	if (!intc_np) {
+		dev_warn(dev, "legacy-interrupt-controller node is absent\n");
+		return -EINVAL;
 	}
-	ks_dw_pcie_enable_legacy_irqs(ks_pcie);
+	ks_pcie->legacy_intc_np = intc_np;
 
-	/* MSI IRQ */
-	if (IS_ENABLED(CONFIG_PCI_MSI)) {
-		for (i = 0; i < ks_pcie->num_msi_host_irqs; i++) {
-			irq_set_chained_handler_and_data(ks_pcie->msi_host_irqs[i],
-							 ks_pcie_msi_irq_handler,
-							 ks_pcie);
+	irq_count = of_irq_count(intc_np);
+	if (!irq_count) {
+		dev_err(dev, "No IRQ entries in legacy-interrupt-controller\n");
+		ret = -EINVAL;
+		goto err;
+	}
+
+	if (!ks_pcie->is_am6) {
+		ret = ks_pcie_k2_config_legacy_irq(ks_pcie, intc_np);
+		if (ret < 0) {
+			dev_err(dev, "Failed to configure legacy irq\n");
+			goto err;
+		}
+	} else {
+		ret = ks_pcie_am6_config_legacy_irq(ks_pcie, intc_np);
+		if (ret < 0) {
+			dev_err(dev, "Failed to configure legacy irq\n");
+			goto err;
 		}
 	}
 
-	if (ks_pcie->error_irq > 0)
-		ks_dw_pcie_enable_error_irq(ks_pcie);
+err:
+	of_node_put(intc_np);
+	return ret;
 }
 
 #ifdef CONFIG_ARM
@@ -246,8 +915,8 @@
  * bus error instead of returning 0xffffffff. This handler always returns 0
  * for this kind of faults.
  */
-static int keystone_pcie_fault(unsigned long addr, unsigned int fsr,
-				struct pt_regs *regs)
+static int ks_pcie_fault(unsigned long addr, unsigned int fsr,
+			 struct pt_regs *regs)
 {
 	unsigned long instr = *(unsigned long *) instruction_pointer(regs);
 
@@ -262,138 +931,630 @@
 }
 #endif
 
+static int __init ks_pcie_init_id(struct keystone_pcie *ks_pcie)
+{
+	int ret;
+	unsigned int id;
+	struct regmap *devctrl_regs;
+	struct dw_pcie *pci = ks_pcie->pci;
+	struct device *dev = pci->dev;
+	struct device_node *np = dev->of_node;
+
+	devctrl_regs = syscon_regmap_lookup_by_phandle(np, "ti,syscon-pcie-id");
+	if (IS_ERR(devctrl_regs))
+		return PTR_ERR(devctrl_regs);
+
+	ret = regmap_read(devctrl_regs, 0, &id);
+	if (ret)
+		return ret;
+
+	dw_pcie_dbi_ro_wr_en(pci);
+	dw_pcie_writew_dbi(pci, PCI_VENDOR_ID, id & PCIE_VENDORID_MASK);
+	dw_pcie_writew_dbi(pci, PCI_DEVICE_ID, id >> PCIE_DEVICEID_SHIFT);
+	dw_pcie_dbi_ro_wr_dis(pci);
+
+	return 0;
+}
+
 static int __init ks_pcie_host_init(struct pcie_port *pp)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 	struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
-	u32 val;
+	int ret;
+
+	ret = ks_pcie_config_legacy_irq(ks_pcie);
+	if (ret)
+		return ret;
+
+	ret = ks_pcie_config_msi_irq(ks_pcie);
+	if (ret)
+		return ret;
+
+	dw_pcie_setup_rc(pp);
 
-	ks_pcie_establish_link(ks_pcie);
-	ks_dw_pcie_setup_rc_app_regs(ks_pcie);
-	ks_pcie_setup_interrupts(ks_pcie);
+	ks_pcie_stop_link(pci);
+	ks_pcie_setup_rc_app_regs(ks_pcie);
 	writew(PCI_IO_RANGE_TYPE_32 | (PCI_IO_RANGE_TYPE_32 << 8),
 			pci->dbi_base + PCI_IO_BASE);
 
-	/* update the Vendor ID */
-	writew(ks_pcie->device_id, pci->dbi_base + PCI_DEVICE_ID);
-
-	/* update the DEV_STAT_CTRL to publish right mrrs */
-	val = readl(pci->dbi_base + PCIE_CAP_BASE + PCI_EXP_DEVCTL);
-	val &= ~PCI_EXP_DEVCTL_READRQ;
-	/* set the mrrs to 256 bytes */
-	val |= BIT(12);
-	writel(val, pci->dbi_base + PCIE_CAP_BASE + PCI_EXP_DEVCTL);
+	ret = ks_pcie_init_id(ks_pcie);
+	if (ret < 0)
+		return ret;
 
 #ifdef CONFIG_ARM
 	/*
 	 * PCIe access errors that result into OCP errors are caught by ARM as
 	 * "External aborts"
 	 */
-	hook_fault_code(17, keystone_pcie_fault, SIGBUS, 0,
+	hook_fault_code(17, ks_pcie_fault, SIGBUS, 0,
 			"Asynchronous external abort");
 #endif
 
+	ks_pcie_start_link(pci);
+	dw_pcie_wait_for_link(pci);
+
 	return 0;
 }
 
-static const struct dw_pcie_host_ops keystone_pcie_host_ops = {
-	.rd_other_conf = ks_dw_pcie_rd_other_conf,
-	.wr_other_conf = ks_dw_pcie_wr_other_conf,
+static const struct dw_pcie_host_ops ks_pcie_host_ops = {
+	.rd_other_conf = ks_pcie_rd_other_conf,
+	.wr_other_conf = ks_pcie_wr_other_conf,
+	.host_init = ks_pcie_host_init,
+	.msi_host_init = ks_pcie_msi_host_init,
+	.scan_bus = ks_pcie_v3_65_scan_bus,
+};
+
+static const struct dw_pcie_host_ops ks_pcie_am654_host_ops = {
 	.host_init = ks_pcie_host_init,
-	.msi_set_irq = ks_dw_pcie_msi_set_irq,
-	.msi_clear_irq = ks_dw_pcie_msi_clear_irq,
-	.get_msi_addr = ks_dw_pcie_get_msi_addr,
-	.msi_host_init = ks_dw_pcie_msi_host_init,
-	.msi_irq_ack = ks_dw_pcie_msi_irq_ack,
-	.scan_bus = ks_dw_pcie_v3_65_scan_bus,
+	.msi_host_init = ks_pcie_am654_msi_host_init,
 };
 
-static irqreturn_t pcie_err_irq_handler(int irq, void *priv)
+static irqreturn_t ks_pcie_err_irq_handler(int irq, void *priv)
 {
 	struct keystone_pcie *ks_pcie = priv;
 
-	return ks_dw_pcie_handle_error_irq(ks_pcie);
+	return ks_pcie_handle_error_irq(ks_pcie);
 }
 
-static int __init ks_add_pcie_port(struct keystone_pcie *ks_pcie,
-			 struct platform_device *pdev)
+static int __init ks_pcie_add_pcie_port(struct keystone_pcie *ks_pcie,
+					struct platform_device *pdev)
 {
 	struct dw_pcie *pci = ks_pcie->pci;
 	struct pcie_port *pp = &pci->pp;
 	struct device *dev = &pdev->dev;
+	struct resource *res;
 	int ret;
 
-	ret = ks_pcie_get_irq_controller_info(ks_pcie,
-					"legacy-interrupt-controller",
-					&ks_pcie->num_legacy_host_irqs);
-	if (ret)
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config");
+	pp->va_cfg0_base = devm_pci_remap_cfg_resource(dev, res);
+	if (IS_ERR(pp->va_cfg0_base))
+		return PTR_ERR(pp->va_cfg0_base);
+
+	pp->va_cfg1_base = pp->va_cfg0_base;
+
+	ret = dw_pcie_host_init(pp);
+	if (ret) {
+		dev_err(dev, "failed to initialize host\n");
 		return ret;
+	}
+
+	return 0;
+}
+
+static u32 ks_pcie_am654_read_dbi2(struct dw_pcie *pci, void __iomem *base,
+				   u32 reg, size_t size)
+{
+	struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
+	u32 val;
+
+	ks_pcie_set_dbi_mode(ks_pcie);
+	dw_pcie_read(base + reg, size, &val);
+	ks_pcie_clear_dbi_mode(ks_pcie);
+	return val;
+}
+
+static void ks_pcie_am654_write_dbi2(struct dw_pcie *pci, void __iomem *base,
+				     u32 reg, size_t size, u32 val)
+{
+	struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
+
+	ks_pcie_set_dbi_mode(ks_pcie);
+	dw_pcie_write(base + reg, size, val);
+	ks_pcie_clear_dbi_mode(ks_pcie);
+}
+
+static const struct dw_pcie_ops ks_pcie_am654_dw_pcie_ops = {
+	.start_link = ks_pcie_start_link,
+	.stop_link = ks_pcie_stop_link,
+	.link_up = ks_pcie_link_up,
+	.read_dbi2 = ks_pcie_am654_read_dbi2,
+	.write_dbi2 = ks_pcie_am654_write_dbi2,
+};
 
-	if (IS_ENABLED(CONFIG_PCI_MSI)) {
-		ret = ks_pcie_get_irq_controller_info(ks_pcie,
-						"msi-interrupt-controller",
-						&ks_pcie->num_msi_host_irqs);
-		if (ret)
-			return ret;
+static void ks_pcie_am654_ep_init(struct dw_pcie_ep *ep)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	int flags;
+
+	ep->page_size = AM654_WIN_SIZE;
+	flags = PCI_BASE_ADDRESS_SPACE_MEMORY | PCI_BASE_ADDRESS_MEM_TYPE_32;
+	dw_pcie_writel_dbi2(pci, PCI_BASE_ADDRESS_0, APP_ADDR_SPACE_0 - 1);
+	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, flags);
+}
+
+static void ks_pcie_am654_raise_legacy_irq(struct keystone_pcie *ks_pcie)
+{
+	struct dw_pcie *pci = ks_pcie->pci;
+	u8 int_pin;
+
+	int_pin = dw_pcie_readb_dbi(pci, PCI_INTERRUPT_PIN);
+	if (int_pin == 0 || int_pin > 4)
+		return;
+
+	ks_pcie_app_writel(ks_pcie, PCIE_LEGACY_IRQ_ENABLE_SET(int_pin),
+			   INT_ENABLE);
+	ks_pcie_app_writel(ks_pcie, PCIE_EP_IRQ_SET, INT_ENABLE);
+	mdelay(1);
+	ks_pcie_app_writel(ks_pcie, PCIE_EP_IRQ_CLR, INT_ENABLE);
+	ks_pcie_app_writel(ks_pcie, PCIE_LEGACY_IRQ_ENABLE_CLR(int_pin),
+			   INT_ENABLE);
+}
+
+static int ks_pcie_am654_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
+				   enum pci_epc_irq_type type,
+				   u16 interrupt_num)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
+
+	switch (type) {
+	case PCI_EPC_IRQ_LEGACY:
+		ks_pcie_am654_raise_legacy_irq(ks_pcie);
+		break;
+	case PCI_EPC_IRQ_MSI:
+		dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num);
+		break;
+	case PCI_EPC_IRQ_MSIX:
+		dw_pcie_ep_raise_msix_irq(ep, func_no, interrupt_num);
+		break;
+	default:
+		dev_err(pci->dev, "UNKNOWN IRQ type\n");
+		return -EINVAL;
 	}
 
-	/*
-	 * Index 0 is the platform interrupt for error interrupt
-	 * from RC.  This is optional.
-	 */
-	ks_pcie->error_irq = irq_of_parse_and_map(ks_pcie->np, 0);
-	if (ks_pcie->error_irq <= 0)
-		dev_info(dev, "no error IRQ defined\n");
-	else {
-		ret = request_irq(ks_pcie->error_irq, pcie_err_irq_handler,
-				  IRQF_SHARED, "pcie-error-irq", ks_pcie);
+	return 0;
+}
+
+static const struct pci_epc_features ks_pcie_am654_epc_features = {
+	.linkup_notifier = false,
+	.msi_capable = true,
+	.msix_capable = true,
+	.reserved_bar = 1 << BAR_0 | 1 << BAR_1,
+	.bar_fixed_64bit = 1 << BAR_0,
+	.bar_fixed_size[2] = SZ_1M,
+	.bar_fixed_size[3] = SZ_64K,
+	.bar_fixed_size[4] = 256,
+	.bar_fixed_size[5] = SZ_1M,
+	.align = SZ_1M,
+};
+
+static const struct pci_epc_features*
+ks_pcie_am654_get_features(struct dw_pcie_ep *ep)
+{
+	return &ks_pcie_am654_epc_features;
+}
+
+static const struct dw_pcie_ep_ops ks_pcie_am654_ep_ops = {
+	.ep_init = ks_pcie_am654_ep_init,
+	.raise_irq = ks_pcie_am654_raise_irq,
+	.get_features = &ks_pcie_am654_get_features,
+};
+
+static void ks_pcie_disable_outbound_atu(struct keystone_pcie *ks_pcie,
+					 phys_addr_t addr)
+{
+	u8 index;
+	u8 regions;
+	u64 cpu_addr;
+
+	index = (addr >> KS_PCIE_WIN_INDEX_SHIFT) & KS_PCIE_WIN_INDEX_MASK;
+	regions = ks_pcie->ob_win[index].no_of_regions;
+	cpu_addr = ks_pcie->ob_win[index].cpu_addr;
+
+	WARN_ON(cpu_addr != addr);
+
+	while (regions--) {
+		ks_pcie_app_writel(ks_pcie, OB_OFFSET_INDEX(index), 0x0);
+		clear_bit(index++, &ks_pcie->ob_window_map);
+	}
+}
+
+static void ks_pcie_disable_inbound_atu(struct keystone_pcie *ks_pcie,
+					int index)
+{
+	ks_pcie_app_writel(ks_pcie, IB_BAR(index), 0x0);
+	ks_pcie_app_writel(ks_pcie, IB_START_LO(index), 0x0);
+	ks_pcie_app_writel(ks_pcie, IB_START_HI(index), 0x0);
+	ks_pcie_app_writel(ks_pcie, IB_OFFSET(index), 0x0);
+}
+
+static void ks_pcie_disable_atu(struct dw_pcie *pci, phys_addr_t addr,
+				int index, enum dw_pcie_region_type type)
+{
+	struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
+
+	switch (type) {
+	case DW_PCIE_REGION_INBOUND:
+		ks_pcie_disable_inbound_atu(ks_pcie, index);
+		break;
+	case DW_PCIE_REGION_OUTBOUND:
+		ks_pcie_disable_outbound_atu(ks_pcie, addr);
+		break;
+	default:
+		return;
+	}
+}
+
+static int ks_pcie_inbound_atu(struct dw_pcie *pci, u32 index,
+			       enum pci_barno bar, dma_addr_t cpu_addr)
+{
+	struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
+	struct device *dev = pci->dev;
+
+	if (bar == BAR_0) {
+		dev_err(dev, "BAR_0 is reserved\n");
+		return -EINVAL;
+	}
+
+	ks_pcie_app_writel(ks_pcie, IB_BAR(index), bar);
+	ks_pcie_app_writel(ks_pcie, IB_OFFSET(index), lower_32_bits(cpu_addr));
+
+	return 0;
+}
+
+static int ks_pcie_check_free(struct keystone_pcie *ks_pcie, u8 index,
+			      u8 regions)
+{
+	while (regions--) {
+		if (test_bit(index++, &ks_pcie->ob_window_map))
+			return false;
+	}
+
+	return true;
+}
+
+static int ks_pcie_outbound_atu(struct dw_pcie *pci, u64 cpu_addr, u64 pci_addr,
+				size_t size)
+{
+	u8 index;
+	u8 regions;
+	struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
+
+	index = (cpu_addr >> KS_PCIE_WIN_INDEX_SHIFT) & KS_PCIE_WIN_INDEX_MASK;
+	regions = ((size - 1) >> KS_PCIE_WIN_INDEX_SHIFT) + 1;
+
+	if (!ks_pcie_check_free(ks_pcie, index, regions))
+		return -ENOMEM;
+	if (index + regions > ks_pcie->num_viewport)
+		return -ENOMEM;
+
+	ks_pcie->ob_win[index].cpu_addr = cpu_addr;
+	ks_pcie->ob_win[index].no_of_regions = regions;
+
+	while (regions--) {
+		ks_pcie_app_writel(ks_pcie, OB_OFFSET_INDEX(index),
+				   lower_32_bits(pci_addr) | OB_ENABLEN);
+		ks_pcie_app_writel(ks_pcie, OB_OFFSET_HI(index),
+				   upper_32_bits(pci_addr));
+		set_bit(index++, &ks_pcie->ob_window_map);
+		pci_addr += KS_PCIE_WIN_SIZE;
+	}
+
+	return 0;
+}
+
+static const struct dw_pcie_ops ks_pcie_dw_pcie_ops = {
+	.start_link = ks_pcie_start_link,
+	.stop_link = ks_pcie_stop_link,
+	.link_up = ks_pcie_link_up,
+	.read_dbi2 = ks_pcie_am654_read_dbi2,
+	.write_dbi2 = ks_pcie_am654_write_dbi2,
+	.inbound_atu = ks_pcie_inbound_atu,
+	.outbound_atu = ks_pcie_outbound_atu,
+	.disable_atu = ks_pcie_disable_atu,
+};
+
+static void ks_pcie_ep_init(struct dw_pcie_ep *ep)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
+	int flags;
+	u32 val;
+
+	ep->page_size = KS_PCIE_WIN_SIZE;
+	flags = PCI_BASE_ADDRESS_SPACE_MEMORY | PCI_BASE_ADDRESS_MEM_TYPE_32;
+	dw_pcie_writel_dbi2(pci, PCI_BASE_ADDRESS_0, APP_ADDR_SPACE_0 - 1);
+	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, flags);
+
+	ks_pcie_app_writel(ks_pcie, OB_SIZE, 0x0);
+	val = ks_pcie_app_readl(ks_pcie, CMD_STATUS);
+	ks_pcie_app_writel(ks_pcie, CMD_STATUS, val | OB_XLAT_EN_VAL |
+			   IB_XLAT_EN_VAL);
+}
+
+static const struct pci_epc_features ks_pcie_epc_features = {
+	.linkup_notifier = false,
+	.msi_capable = true,
+	.msix_capable = false,
+	.reserved_bar = 1 << BAR_0,
+};
+
+static const struct pci_epc_features*
+ks_pcie_get_features(struct dw_pcie_ep *ep)
+{
+	return &ks_pcie_epc_features;
+}
+
+static const struct dw_pcie_ep_ops ks_pcie_ep_ops = {
+	.ep_init = ks_pcie_ep_init,
+	.raise_irq = ks_pcie_am654_raise_irq,
+	.get_features = &ks_pcie_get_features,
+};
+
+static int __init ks_pcie_add_pcie_ep(struct keystone_pcie *ks_pcie,
+				      struct platform_device *pdev)
+{
+	int ret;
+	struct dw_pcie_ep *ep;
+	struct resource *res;
+	struct device *dev = &pdev->dev;
+	struct dw_pcie *pci = ks_pcie->pci;
+
+	ep = &pci->ep;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "addr_space");
+	if (!res)
+		return -EINVAL;
+
+	ep->phys_base = res->start;
+	ep->addr_size = resource_size(res);
+
+	if (!ks_pcie->is_am6) {
+		ks_pcie->ob_win = devm_kzalloc(dev, sizeof(*ks_pcie->ob_win) *
+					       ks_pcie->num_viewport,
+					       GFP_KERNEL);
+		if (!ks_pcie->ob_win)
+			return -ENOMEM;
+	}
+
+	ret = dw_pcie_ep_init(ep);
+	if (ret) {
+		dev_err(dev, "failed to initialize endpoint\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static void ks_pcie_disable_phy(struct keystone_pcie *ks_pcie)
+{
+	int num_lanes = ks_pcie->num_lanes;
+
+	while (num_lanes--) {
+		phy_power_off(ks_pcie->phy[num_lanes]);
+		phy_exit(ks_pcie->phy[num_lanes]);
+	}
+}
+
+static int ks_pcie_enable_phy(struct keystone_pcie *ks_pcie)
+{
+	int i;
+	int ret;
+	int num_lanes = ks_pcie->num_lanes;
+
+	for (i = 0; i < num_lanes; i++) {
+		ret = phy_reset(ks_pcie->phy[i]);
+		if (ret < 0)
+			goto err_phy;
+
+		ret = phy_init(ks_pcie->phy[i]);
+		if (ret < 0)
+			goto err_phy;
+
+		ret = phy_power_on(ks_pcie->phy[i]);
 		if (ret < 0) {
-			dev_err(dev, "failed to request error IRQ %d\n",
-				ks_pcie->error_irq);
-			return ret;
+			phy_exit(ks_pcie->phy[i]);
+			goto err_phy;
 		}
 	}
 
-	pp->ops = &keystone_pcie_host_ops;
-	ret = ks_dw_pcie_host_init(ks_pcie, ks_pcie->msi_intc_np);
+	return 0;
+
+err_phy:
+	while (--i >= 0) {
+		phy_power_off(ks_pcie->phy[i]);
+		phy_exit(ks_pcie->phy[i]);
+	}
+
+	return ret;
+}
+
+static int ks_pcie_set_mode(struct device *dev, enum dw_pcie_device_mode mode)
+{
+	struct device_node *np = dev->of_node;
+	struct regmap *syscon;
+	u32 val;
+	u32 mask;
+	int ret = 0;
+
+	syscon = syscon_regmap_lookup_by_phandle(np, "ti,syscon-pcie-mode");
+	if (IS_ERR(syscon))
+		return 0;
+
+	mask = KS_PCIE_DEV_TYPE_MASK | KS_PCIE_SYSCLOCKOUTEN;
+
+	switch (mode) {
+	case DW_PCIE_RC_TYPE:
+		val = KS_PCIE_DEV_TYPE(RC) | KS_PCIE_SYSCLOCKOUTEN;
+		break;
+	case DW_PCIE_EP_TYPE:
+		val = KS_PCIE_DEV_TYPE(EP);
+		break;
+	default:
+		dev_err(dev, "INVALID device type %d\n", mode);
+		return -EINVAL;
+	}
+
+	ret = regmap_update_bits(syscon, 0, mask, val);
 	if (ret) {
-		dev_err(dev, "failed to initialize host\n");
+		dev_err(dev, "failed to set pcie mode\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int ks_pcie_am654_set_mode(struct device *dev,
+				  enum dw_pcie_device_mode mode)
+{
+	struct device_node *np = dev->of_node;
+	struct regmap *syscon;
+	u32 val;
+	u32 mask;
+	int ret = 0;
+
+	syscon = syscon_regmap_lookup_by_phandle(np, "ti,syscon-pcie-mode");
+	if (IS_ERR(syscon))
+		return 0;
+
+	mask = AM654_PCIE_DEV_TYPE_MASK;
+
+	switch (mode) {
+	case DW_PCIE_RC_TYPE:
+		val = RC;
+		break;
+	case DW_PCIE_EP_TYPE:
+		val = EP;
+		break;
+	default:
+		dev_err(dev, "INVALID device type %d\n", mode);
+		return -EINVAL;
+	}
+
+	ret = regmap_update_bits(syscon, 0, mask, val);
+	if (ret) {
+		dev_err(dev, "failed to set pcie mode\n");
 		return ret;
 	}
 
 	return 0;
 }
 
+static void ks_pcie_set_link_speed(struct dw_pcie *pci, int link_speed)
+{
+	u32 val;
+
+	dw_pcie_dbi_ro_wr_en(pci);
+
+	val = dw_pcie_readl_dbi(pci, EXP_CAP_ID_OFFSET + PCI_EXP_LNKCAP);
+	if ((val & PCI_EXP_LNKCAP_SLS) != link_speed) {
+		val &= ~((u32)PCI_EXP_LNKCAP_SLS);
+		val |= link_speed;
+		dw_pcie_writel_dbi(pci, EXP_CAP_ID_OFFSET + PCI_EXP_LNKCAP,
+				   val);
+	}
+
+	val = dw_pcie_readl_dbi(pci, EXP_CAP_ID_OFFSET + PCI_EXP_LNKCTL2);
+	if ((val & PCI_EXP_LNKCAP_SLS) != link_speed) {
+		val &= ~((u32)PCI_EXP_LNKCAP_SLS);
+		val |= link_speed;
+		dw_pcie_writel_dbi(pci, EXP_CAP_ID_OFFSET + PCI_EXP_LNKCTL2,
+				   val);
+	}
+
+	dw_pcie_dbi_ro_wr_dis(pci);
+}
+
+static const struct ks_pcie_of_data ks_pcie_rc_of_data = {
+	.host_ops = &ks_pcie_host_ops,
+	.mode = DW_PCIE_RC_TYPE,
+	.version = 0x365A,
+};
+
+static const struct ks_pcie_of_data ks_pcie_ep_of_data = {
+	.ep_ops = &ks_pcie_ep_ops,
+	.mode = DW_PCIE_EP_TYPE,
+	.version = 0x365A,
+};
+
+static const struct ks_pcie_of_data ks_pcie_am654_rc_of_data = {
+	.host_ops = &ks_pcie_am654_host_ops,
+	.mode = DW_PCIE_RC_TYPE,
+	.version = 0x490A,
+};
+
+static const struct ks_pcie_of_data ks_pcie_am654_ep_of_data = {
+	.ep_ops = &ks_pcie_am654_ep_ops,
+	.mode = DW_PCIE_EP_TYPE,
+	.version = 0x490A,
+};
+
 static const struct of_device_id ks_pcie_of_match[] = {
 	{
 		.type = "pci",
+		.data = &ks_pcie_rc_of_data,
 		.compatible = "ti,keystone-pcie",
 	},
+	{
+		.data = &ks_pcie_ep_of_data,
+		.compatible = "ti,keystone-pcie-ep",
+	},
+	{
+		.data = &ks_pcie_am654_rc_of_data,
+		.compatible = "ti,am654-pcie-rc",
+	},
+	{
+		.data = &ks_pcie_am654_ep_of_data,
+		.compatible = "ti,am654-pcie-ep",
+	},
 	{ },
 };
 
-static const struct dw_pcie_ops dw_pcie_ops = {
-	.link_up = ks_dw_pcie_link_up,
-};
-
-static int __exit ks_pcie_remove(struct platform_device *pdev)
-{
-	struct keystone_pcie *ks_pcie = platform_get_drvdata(pdev);
-
-	clk_disable_unprepare(ks_pcie->clk);
-
-	return 0;
-}
-
 static int __init ks_pcie_probe(struct platform_device *pdev)
 {
+	const struct dw_pcie_host_ops *host_ops;
+	const struct dw_pcie_ep_ops *ep_ops;
 	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	const struct ks_pcie_of_data *data;
+	const struct of_device_id *match;
+	enum dw_pcie_device_mode mode;
 	struct dw_pcie *pci;
 	struct keystone_pcie *ks_pcie;
+	struct device_link **link;
+	struct gpio_desc *gpiod;
+	void __iomem *atu_base;
 	struct resource *res;
-	void __iomem *reg_p;
-	struct phy *phy;
+	unsigned int version;
+	void __iomem *base;
+	u32 num_viewport;
+	struct phy **phy;
+	int link_speed;
+	u32 num_lanes;
+	char name[10];
 	int ret;
+	int irq;
+	int i;
+
+	match = of_match_device(of_match_ptr(ks_pcie_of_match), dev);
+	data = (struct ks_pcie_of_data *)match->data;
+	if (!data)
+		return -EINVAL;
+
+	version = data->version;
+	host_ops = data->host_ops;
+	ep_ops = data->ep_ops;
+	mode = data->mode;
 
 	ks_pcie = devm_kzalloc(dev, sizeof(*ks_pcie), GFP_KERNEL);
 	if (!ks_pcie)
@@ -403,55 +1564,221 @@
 	if (!pci)
 		return -ENOMEM;
 
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "app");
+	ks_pcie->va_app_base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(ks_pcie->va_app_base))
+		return PTR_ERR(ks_pcie->va_app_base);
+
+	ks_pcie->app = *res;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbics");
+	base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	if (of_device_is_compatible(np, "ti,am654-pcie-rc") ||
+	    of_device_is_compatible(np, "ti,am654-pcie-ep"))
+		ks_pcie->is_am6 = true;
+
+	pci->dbi_base = base;
+	pci->dbi_base2 = base;
 	pci->dev = dev;
-	pci->ops = &dw_pcie_ops;
+	if (ks_pcie->is_am6)
+		pci->ops = &ks_pcie_am654_dw_pcie_ops;
+	else
+		pci->ops = &ks_pcie_dw_pcie_ops;
+	pci->version = version;
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		dev_err(dev, "missing IRQ resource: %d\n", irq);
+		return irq;
+	}
 
-	ks_pcie->pci = pci;
+	ret = request_irq(irq, ks_pcie_err_irq_handler, IRQF_SHARED,
+			  "ks-pcie-error-irq", ks_pcie);
+	if (ret < 0) {
+		dev_err(dev, "failed to request error IRQ %d\n",
+			irq);
+		return ret;
+	}
+
+	if ((dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)) != 0) &&
+	    dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
+		dev_err(dev, "Cannot set DMA mask\n");
+		return -EINVAL;
+	}
 
-	/* initialize SerDes Phy if present */
-	phy = devm_phy_get(dev, "pcie-phy");
-	if (PTR_ERR_OR_ZERO(phy) == -EPROBE_DEFER)
-		return PTR_ERR(phy);
+	ret = of_property_read_u32(np, "num-lanes", &num_lanes);
+	if (ret)
+		num_lanes = 1;
 
-	if (!IS_ERR_OR_NULL(phy)) {
-		ret = phy_init(phy);
-		if (ret < 0)
-			return ret;
+	phy = devm_kzalloc(dev, sizeof(*phy) * num_lanes, GFP_KERNEL);
+	if (!phy)
+		return -ENOMEM;
+
+	link = devm_kzalloc(dev, sizeof(*link) * num_lanes, GFP_KERNEL);
+	if (!link)
+		return -ENOMEM;
+
+	for (i = 0; i < num_lanes; i++) {
+		snprintf(name, sizeof(name), "pcie-phy%d", i);
+		phy[i] = devm_phy_optional_get(dev, name);
+		if (IS_ERR(phy[i])) {
+			ret = PTR_ERR(phy[i]);
+			goto err_link;
+		}
+
+		if (!phy[i])
+			continue;
+
+		link[i] = device_link_add(dev, &phy[i]->dev, DL_FLAG_STATELESS);
+		if (!link[i]) {
+			ret = -EINVAL;
+			goto err_link;
+		}
 	}
 
-	/* index 2 is to read PCI DEVICE_ID */
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 2);
-	reg_p = devm_ioremap_resource(dev, res);
-	if (IS_ERR(reg_p))
-		return PTR_ERR(reg_p);
-	ks_pcie->device_id = readl(reg_p) >> 16;
-	devm_iounmap(dev, reg_p);
-	devm_release_mem_region(dev, res->start, resource_size(res));
+	ks_pcie->np = np;
+	ks_pcie->pci = pci;
+	ks_pcie->link = link;
+	ks_pcie->num_lanes = num_lanes;
+	ks_pcie->phy = phy;
+
+	gpiod = devm_gpiod_get_optional(dev, "reset",
+					GPIOD_OUT_LOW);
+	if (IS_ERR(gpiod)) {
+		ret = PTR_ERR(gpiod);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "Failed to get reset GPIO\n");
+		goto err_link;
+	}
 
-	ks_pcie->np = dev->of_node;
-	platform_set_drvdata(pdev, ks_pcie);
-	ks_pcie->clk = devm_clk_get(dev, "pcie");
-	if (IS_ERR(ks_pcie->clk)) {
-		dev_err(dev, "Failed to get pcie rc clock\n");
-		return PTR_ERR(ks_pcie->clk);
+	ret = ks_pcie_enable_phy(ks_pcie);
+	if (ret) {
+		dev_err(dev, "failed to enable phy\n");
+		goto err_link;
 	}
-	ret = clk_prepare_enable(ks_pcie->clk);
-	if (ret)
-		return ret;
 
 	platform_set_drvdata(pdev, ks_pcie);
+	pm_runtime_enable(dev);
+	ret = pm_runtime_get_sync(dev);
+	if (ret < 0) {
+		dev_err(dev, "pm_runtime_get_sync failed\n");
+		goto err_get_sync;
+	}
 
-	ret = ks_add_pcie_port(ks_pcie, pdev);
-	if (ret < 0)
-		goto fail_clk;
+	if (pci->version >= 0x480A) {
+		res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "atu");
+		atu_base = devm_ioremap_resource(dev, res);
+		if (IS_ERR(atu_base)) {
+			ret = PTR_ERR(atu_base);
+			goto err_get_sync;
+		}
+
+		pci->atu_base = atu_base;
+
+		ret = ks_pcie_am654_set_mode(dev, mode);
+		if (ret < 0)
+			goto err_get_sync;
+	} else {
+		ret = ks_pcie_set_mode(dev, mode);
+		if (ret < 0)
+			goto err_get_sync;
+	}
+
+	link_speed = of_pci_get_max_link_speed(np);
+	if (link_speed < 0)
+		link_speed = 2;
+
+	ks_pcie_set_link_speed(pci, link_speed);
+
+	switch (mode) {
+	case DW_PCIE_RC_TYPE:
+		if (!IS_ENABLED(CONFIG_PCI_KEYSTONE_HOST)) {
+			ret = -ENODEV;
+			goto err_get_sync;
+		}
+
+		ret = of_property_read_u32(np, "num-viewport", &num_viewport);
+		if (ret < 0) {
+			dev_err(dev, "unable to read *num-viewport* property\n");
+			goto err_get_sync;
+		}
+
+		/*
+		 * "Power Sequencing and Reset Signal Timings" table in
+		 * PCI EXPRESS CARD ELECTROMECHANICAL SPECIFICATION, REV. 2.0
+		 * indicates PERST# should be deasserted after minimum of 100us
+		 * once REFCLK is stable. The REFCLK to the connector in RC
+		 * mode is selected while enabling the PHY. So deassert PERST#
+		 * after 100 us.
+		 */
+		if (gpiod) {
+			usleep_range(100, 200);
+			gpiod_set_value_cansleep(gpiod, 1);
+		}
+
+		ks_pcie->num_viewport = num_viewport;
+		pci->pp.ops = host_ops;
+		ret = ks_pcie_add_pcie_port(ks_pcie, pdev);
+		if (ret < 0)
+			goto err_get_sync;
+		break;
+	case DW_PCIE_EP_TYPE:
+		if (!IS_ENABLED(CONFIG_PCI_KEYSTONE_EP)) {
+			ret = -ENODEV;
+			goto err_get_sync;
+		}
+
+		ret = of_property_read_u32(np, "num-ob-windows", &num_viewport);
+		if (ret < 0) {
+			dev_err(dev, "unable to read *num-viewport* property\n");
+			goto err_get_sync;
+		}
+
+		ks_pcie->num_viewport = num_viewport;
+		pci->ep.ops = ep_ops;
+		ret = ks_pcie_add_pcie_ep(ks_pcie, pdev);
+		if (ret < 0)
+			goto err_get_sync;
+		break;
+	default:
+		dev_err(dev, "INVALID device type %d\n", mode);
+	}
+
+	ks_pcie_enable_error_irq(ks_pcie);
 
 	return 0;
-fail_clk:
-	clk_disable_unprepare(ks_pcie->clk);
+
+err_get_sync:
+	pm_runtime_put(dev);
+	pm_runtime_disable(dev);
+	ks_pcie_disable_phy(ks_pcie);
+
+err_link:
+	while (--i >= 0 && link[i])
+		device_link_del(link[i]);
 
 	return ret;
 }
 
+static int __exit ks_pcie_remove(struct platform_device *pdev)
+{
+	struct keystone_pcie *ks_pcie = platform_get_drvdata(pdev);
+	struct device_link **link = ks_pcie->link;
+	int num_lanes = ks_pcie->num_lanes;
+	struct device *dev = &pdev->dev;
+
+	pm_runtime_put(dev);
+	pm_runtime_disable(dev);
+	ks_pcie_disable_phy(ks_pcie);
+	while (num_lanes--)
+		device_link_del(link[num_lanes]);
+
+	return 0;
+}
+
 static struct platform_driver ks_pcie_driver __refdata = {
 	.probe  = ks_pcie_probe,
 	.remove = __exit_p(ks_pcie_remove),
diff -Naur --no-dereference a/drivers/pci/controller/dwc/pci-keystone-dw.c b/drivers/pci/controller/dwc/pci-keystone-dw.c
--- a/drivers/pci/controller/dwc/pci-keystone-dw.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/controller/dwc/pci-keystone-dw.c	1969-12-31 19:00:00.000000000 -0500
@@ -1,484 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * DesignWare application register space functions for Keystone PCI controller
- *
- * Copyright (C) 2013-2014 Texas Instruments., Ltd.
- *		http://www.ti.com
- *
- * Author: Murali Karicheri <m-karicheri2@ti.com>
- */
-
-#include <linux/irq.h>
-#include <linux/irqdomain.h>
-#include <linux/irqreturn.h>
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/of_pci.h>
-#include <linux/pci.h>
-#include <linux/platform_device.h>
-
-#include "pcie-designware.h"
-#include "pci-keystone.h"
-
-/* Application register defines */
-#define LTSSM_EN_VAL		        1
-#define LTSSM_STATE_MASK		0x1f
-#define LTSSM_STATE_L0			0x11
-#define DBI_CS2_EN_VAL			0x20
-#define OB_XLAT_EN_VAL		        2
-
-/* Application registers */
-#define CMD_STATUS			0x004
-#define CFG_SETUP			0x008
-#define OB_SIZE				0x030
-#define CFG_PCIM_WIN_SZ_IDX		3
-#define CFG_PCIM_WIN_CNT		32
-#define SPACE0_REMOTE_CFG_OFFSET	0x1000
-#define OB_OFFSET_INDEX(n)		(0x200 + (8 * n))
-#define OB_OFFSET_HI(n)			(0x204 + (8 * n))
-
-/* IRQ register defines */
-#define IRQ_EOI				0x050
-#define IRQ_STATUS			0x184
-#define IRQ_ENABLE_SET			0x188
-#define IRQ_ENABLE_CLR			0x18c
-
-#define MSI_IRQ				0x054
-#define MSI0_IRQ_STATUS			0x104
-#define MSI0_IRQ_ENABLE_SET		0x108
-#define MSI0_IRQ_ENABLE_CLR		0x10c
-#define IRQ_STATUS			0x184
-#define MSI_IRQ_OFFSET			4
-
-/* Error IRQ bits */
-#define ERR_AER		BIT(5)	/* ECRC error */
-#define ERR_AXI		BIT(4)	/* AXI tag lookup fatal error */
-#define ERR_CORR	BIT(3)	/* Correctable error */
-#define ERR_NONFATAL	BIT(2)	/* Non-fatal error */
-#define ERR_FATAL	BIT(1)	/* Fatal error */
-#define ERR_SYS		BIT(0)	/* System (fatal, non-fatal, or correctable) */
-#define ERR_IRQ_ALL	(ERR_AER | ERR_AXI | ERR_CORR | \
-			 ERR_NONFATAL | ERR_FATAL | ERR_SYS)
-#define ERR_FATAL_IRQ	(ERR_FATAL | ERR_AXI)
-#define ERR_IRQ_STATUS_RAW		0x1c0
-#define ERR_IRQ_STATUS			0x1c4
-#define ERR_IRQ_ENABLE_SET		0x1c8
-#define ERR_IRQ_ENABLE_CLR		0x1cc
-
-/* Config space registers */
-#define DEBUG0				0x728
-
-#define to_keystone_pcie(x)	dev_get_drvdata((x)->dev)
-
-static inline void update_reg_offset_bit_pos(u32 offset, u32 *reg_offset,
-					     u32 *bit_pos)
-{
-	*reg_offset = offset % 8;
-	*bit_pos = offset >> 3;
-}
-
-phys_addr_t ks_dw_pcie_get_msi_addr(struct pcie_port *pp)
-{
-	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
-	struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
-
-	return ks_pcie->app.start + MSI_IRQ;
-}
-
-static u32 ks_dw_app_readl(struct keystone_pcie *ks_pcie, u32 offset)
-{
-	return readl(ks_pcie->va_app_base + offset);
-}
-
-static void ks_dw_app_writel(struct keystone_pcie *ks_pcie, u32 offset, u32 val)
-{
-	writel(val, ks_pcie->va_app_base + offset);
-}
-
-void ks_dw_pcie_handle_msi_irq(struct keystone_pcie *ks_pcie, int offset)
-{
-	struct dw_pcie *pci = ks_pcie->pci;
-	struct pcie_port *pp = &pci->pp;
-	struct device *dev = pci->dev;
-	u32 pending, vector;
-	int src, virq;
-
-	pending = ks_dw_app_readl(ks_pcie, MSI0_IRQ_STATUS + (offset << 4));
-
-	/*
-	 * MSI0 status bit 0-3 shows vectors 0, 8, 16, 24, MSI1 status bit
-	 * shows 1, 9, 17, 25 and so forth
-	 */
-	for (src = 0; src < 4; src++) {
-		if (BIT(src) & pending) {
-			vector = offset + (src << 3);
-			virq = irq_linear_revmap(pp->irq_domain, vector);
-			dev_dbg(dev, "irq: bit %d, vector %d, virq %d\n",
-				src, vector, virq);
-			generic_handle_irq(virq);
-		}
-	}
-}
-
-void ks_dw_pcie_msi_irq_ack(int irq, struct pcie_port *pp)
-{
-	u32 reg_offset, bit_pos;
-	struct keystone_pcie *ks_pcie;
-	struct dw_pcie *pci;
-
-	pci = to_dw_pcie_from_pp(pp);
-	ks_pcie = to_keystone_pcie(pci);
-	update_reg_offset_bit_pos(irq, &reg_offset, &bit_pos);
-
-	ks_dw_app_writel(ks_pcie, MSI0_IRQ_STATUS + (reg_offset << 4),
-			 BIT(bit_pos));
-	ks_dw_app_writel(ks_pcie, IRQ_EOI, reg_offset + MSI_IRQ_OFFSET);
-}
-
-void ks_dw_pcie_msi_set_irq(struct pcie_port *pp, int irq)
-{
-	u32 reg_offset, bit_pos;
-	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
-	struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
-
-	update_reg_offset_bit_pos(irq, &reg_offset, &bit_pos);
-	ks_dw_app_writel(ks_pcie, MSI0_IRQ_ENABLE_SET + (reg_offset << 4),
-			 BIT(bit_pos));
-}
-
-void ks_dw_pcie_msi_clear_irq(struct pcie_port *pp, int irq)
-{
-	u32 reg_offset, bit_pos;
-	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
-	struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
-
-	update_reg_offset_bit_pos(irq, &reg_offset, &bit_pos);
-	ks_dw_app_writel(ks_pcie, MSI0_IRQ_ENABLE_CLR + (reg_offset << 4),
-			 BIT(bit_pos));
-}
-
-int ks_dw_pcie_msi_host_init(struct pcie_port *pp)
-{
-	return dw_pcie_allocate_domains(pp);
-}
-
-void ks_dw_pcie_enable_legacy_irqs(struct keystone_pcie *ks_pcie)
-{
-	int i;
-
-	for (i = 0; i < PCI_NUM_INTX; i++)
-		ks_dw_app_writel(ks_pcie, IRQ_ENABLE_SET + (i << 4), 0x1);
-}
-
-void ks_dw_pcie_handle_legacy_irq(struct keystone_pcie *ks_pcie, int offset)
-{
-	struct dw_pcie *pci = ks_pcie->pci;
-	struct device *dev = pci->dev;
-	u32 pending;
-	int virq;
-
-	pending = ks_dw_app_readl(ks_pcie, IRQ_STATUS + (offset << 4));
-
-	if (BIT(0) & pending) {
-		virq = irq_linear_revmap(ks_pcie->legacy_irq_domain, offset);
-		dev_dbg(dev, ": irq: irq_offset %d, virq %d\n", offset, virq);
-		generic_handle_irq(virq);
-	}
-
-	/* EOI the INTx interrupt */
-	ks_dw_app_writel(ks_pcie, IRQ_EOI, offset);
-}
-
-void ks_dw_pcie_enable_error_irq(struct keystone_pcie *ks_pcie)
-{
-	ks_dw_app_writel(ks_pcie, ERR_IRQ_ENABLE_SET, ERR_IRQ_ALL);
-}
-
-irqreturn_t ks_dw_pcie_handle_error_irq(struct keystone_pcie *ks_pcie)
-{
-	u32 status;
-
-	status = ks_dw_app_readl(ks_pcie, ERR_IRQ_STATUS_RAW) & ERR_IRQ_ALL;
-	if (!status)
-		return IRQ_NONE;
-
-	if (status & ERR_FATAL_IRQ)
-		dev_err(ks_pcie->pci->dev, "fatal error (status %#010x)\n",
-			status);
-
-	/* Ack the IRQ; status bits are RW1C */
-	ks_dw_app_writel(ks_pcie, ERR_IRQ_STATUS, status);
-	return IRQ_HANDLED;
-}
-
-static void ks_dw_pcie_ack_legacy_irq(struct irq_data *d)
-{
-}
-
-static void ks_dw_pcie_mask_legacy_irq(struct irq_data *d)
-{
-}
-
-static void ks_dw_pcie_unmask_legacy_irq(struct irq_data *d)
-{
-}
-
-static struct irq_chip ks_dw_pcie_legacy_irq_chip = {
-	.name = "Keystone-PCI-Legacy-IRQ",
-	.irq_ack = ks_dw_pcie_ack_legacy_irq,
-	.irq_mask = ks_dw_pcie_mask_legacy_irq,
-	.irq_unmask = ks_dw_pcie_unmask_legacy_irq,
-};
-
-static int ks_dw_pcie_init_legacy_irq_map(struct irq_domain *d,
-				unsigned int irq, irq_hw_number_t hw_irq)
-{
-	irq_set_chip_and_handler(irq, &ks_dw_pcie_legacy_irq_chip,
-				 handle_level_irq);
-	irq_set_chip_data(irq, d->host_data);
-
-	return 0;
-}
-
-static const struct irq_domain_ops ks_dw_pcie_legacy_irq_domain_ops = {
-	.map = ks_dw_pcie_init_legacy_irq_map,
-	.xlate = irq_domain_xlate_onetwocell,
-};
-
-/**
- * ks_dw_pcie_set_dbi_mode() - Set DBI mode to access overlaid BAR mask
- * registers
- *
- * Since modification of dbi_cs2 involves different clock domain, read the
- * status back to ensure the transition is complete.
- */
-static void ks_dw_pcie_set_dbi_mode(struct keystone_pcie *ks_pcie)
-{
-	u32 val;
-
-	val = ks_dw_app_readl(ks_pcie, CMD_STATUS);
-	ks_dw_app_writel(ks_pcie, CMD_STATUS, DBI_CS2_EN_VAL | val);
-
-	do {
-		val = ks_dw_app_readl(ks_pcie, CMD_STATUS);
-	} while (!(val & DBI_CS2_EN_VAL));
-}
-
-/**
- * ks_dw_pcie_clear_dbi_mode() - Disable DBI mode
- *
- * Since modification of dbi_cs2 involves different clock domain, read the
- * status back to ensure the transition is complete.
- */
-static void ks_dw_pcie_clear_dbi_mode(struct keystone_pcie *ks_pcie)
-{
-	u32 val;
-
-	val = ks_dw_app_readl(ks_pcie, CMD_STATUS);
-	ks_dw_app_writel(ks_pcie, CMD_STATUS, ~DBI_CS2_EN_VAL & val);
-
-	do {
-		val = ks_dw_app_readl(ks_pcie, CMD_STATUS);
-	} while (val & DBI_CS2_EN_VAL);
-}
-
-void ks_dw_pcie_setup_rc_app_regs(struct keystone_pcie *ks_pcie)
-{
-	struct dw_pcie *pci = ks_pcie->pci;
-	struct pcie_port *pp = &pci->pp;
-	u32 start = pp->mem->start, end = pp->mem->end;
-	int i, tr_size;
-	u32 val;
-
-	/* Disable BARs for inbound access */
-	ks_dw_pcie_set_dbi_mode(ks_pcie);
-	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0);
-	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_1, 0);
-	ks_dw_pcie_clear_dbi_mode(ks_pcie);
-
-	/* Set outbound translation size per window division */
-	ks_dw_app_writel(ks_pcie, OB_SIZE, CFG_PCIM_WIN_SZ_IDX & 0x7);
-
-	tr_size = (1 << (CFG_PCIM_WIN_SZ_IDX & 0x7)) * SZ_1M;
-
-	/* Using Direct 1:1 mapping of RC <-> PCI memory space */
-	for (i = 0; (i < CFG_PCIM_WIN_CNT) && (start < end); i++) {
-		ks_dw_app_writel(ks_pcie, OB_OFFSET_INDEX(i), start | 1);
-		ks_dw_app_writel(ks_pcie, OB_OFFSET_HI(i), 0);
-		start += tr_size;
-	}
-
-	/* Enable OB translation */
-	val = ks_dw_app_readl(ks_pcie, CMD_STATUS);
-	ks_dw_app_writel(ks_pcie, CMD_STATUS, OB_XLAT_EN_VAL | val);
-}
-
-/**
- * ks_pcie_cfg_setup() - Set up configuration space address for a device
- *
- * @ks_pcie: ptr to keystone_pcie structure
- * @bus: Bus number the device is residing on
- * @devfn: device, function number info
- *
- * Forms and returns the address of configuration space mapped in PCIESS
- * address space 0.  Also configures CFG_SETUP for remote configuration space
- * access.
- *
- * The address space has two regions to access configuration - local and remote.
- * We access local region for bus 0 (as RC is attached on bus 0) and remote
- * region for others with TYPE 1 access when bus > 1.  As for device on bus = 1,
- * we will do TYPE 0 access as it will be on our secondary bus (logical).
- * CFG_SETUP is needed only for remote configuration access.
- */
-static void __iomem *ks_pcie_cfg_setup(struct keystone_pcie *ks_pcie, u8 bus,
-				       unsigned int devfn)
-{
-	u8 device = PCI_SLOT(devfn), function = PCI_FUNC(devfn);
-	struct dw_pcie *pci = ks_pcie->pci;
-	struct pcie_port *pp = &pci->pp;
-	u32 regval;
-
-	if (bus == 0)
-		return pci->dbi_base;
-
-	regval = (bus << 16) | (device << 8) | function;
-
-	/*
-	 * Since Bus#1 will be a virtual bus, we need to have TYPE0
-	 * access only.
-	 * TYPE 1
-	 */
-	if (bus != 1)
-		regval |= BIT(24);
-
-	ks_dw_app_writel(ks_pcie, CFG_SETUP, regval);
-	return pp->va_cfg0_base;
-}
-
-int ks_dw_pcie_rd_other_conf(struct pcie_port *pp, struct pci_bus *bus,
-			     unsigned int devfn, int where, int size, u32 *val)
-{
-	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
-	struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
-	u8 bus_num = bus->number;
-	void __iomem *addr;
-
-	addr = ks_pcie_cfg_setup(ks_pcie, bus_num, devfn);
-
-	return dw_pcie_read(addr + where, size, val);
-}
-
-int ks_dw_pcie_wr_other_conf(struct pcie_port *pp, struct pci_bus *bus,
-			     unsigned int devfn, int where, int size, u32 val)
-{
-	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
-	struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
-	u8 bus_num = bus->number;
-	void __iomem *addr;
-
-	addr = ks_pcie_cfg_setup(ks_pcie, bus_num, devfn);
-
-	return dw_pcie_write(addr + where, size, val);
-}
-
-/**
- * ks_dw_pcie_v3_65_scan_bus() - keystone scan_bus post initialization
- *
- * This sets BAR0 to enable inbound access for MSI_IRQ register
- */
-void ks_dw_pcie_v3_65_scan_bus(struct pcie_port *pp)
-{
-	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
-	struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
-
-	/* Configure and set up BAR0 */
-	ks_dw_pcie_set_dbi_mode(ks_pcie);
-
-	/* Enable BAR0 */
-	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 1);
-	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, SZ_4K - 1);
-
-	ks_dw_pcie_clear_dbi_mode(ks_pcie);
-
-	 /*
-	  * For BAR0, just setting bus address for inbound writes (MSI) should
-	  * be sufficient.  Use physical address to avoid any conflicts.
-	  */
-	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, ks_pcie->app.start);
-}
-
-/**
- * ks_dw_pcie_link_up() - Check if link up
- */
-int ks_dw_pcie_link_up(struct dw_pcie *pci)
-{
-	u32 val;
-
-	val = dw_pcie_readl_dbi(pci, DEBUG0);
-	return (val & LTSSM_STATE_MASK) == LTSSM_STATE_L0;
-}
-
-void ks_dw_pcie_initiate_link_train(struct keystone_pcie *ks_pcie)
-{
-	u32 val;
-
-	/* Disable Link training */
-	val = ks_dw_app_readl(ks_pcie, CMD_STATUS);
-	val &= ~LTSSM_EN_VAL;
-	ks_dw_app_writel(ks_pcie, CMD_STATUS, LTSSM_EN_VAL | val);
-
-	/* Initiate Link Training */
-	val = ks_dw_app_readl(ks_pcie, CMD_STATUS);
-	ks_dw_app_writel(ks_pcie, CMD_STATUS, LTSSM_EN_VAL | val);
-}
-
-/**
- * ks_dw_pcie_host_init() - initialize host for v3_65 dw hardware
- *
- * Ioremap the register resources, initialize legacy irq domain
- * and call dw_pcie_v3_65_host_init() API to initialize the Keystone
- * PCI host controller.
- */
-int __init ks_dw_pcie_host_init(struct keystone_pcie *ks_pcie,
-				struct device_node *msi_intc_np)
-{
-	struct dw_pcie *pci = ks_pcie->pci;
-	struct pcie_port *pp = &pci->pp;
-	struct device *dev = pci->dev;
-	struct platform_device *pdev = to_platform_device(dev);
-	struct resource *res;
-
-	/* Index 0 is the config reg. space address */
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	pci->dbi_base = devm_pci_remap_cfg_resource(dev, res);
-	if (IS_ERR(pci->dbi_base))
-		return PTR_ERR(pci->dbi_base);
-
-	/*
-	 * We set these same and is used in pcie rd/wr_other_conf
-	 * functions
-	 */
-	pp->va_cfg0_base = pci->dbi_base + SPACE0_REMOTE_CFG_OFFSET;
-	pp->va_cfg1_base = pp->va_cfg0_base;
-
-	/* Index 1 is the application reg. space address */
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
-	ks_pcie->va_app_base = devm_ioremap_resource(dev, res);
-	if (IS_ERR(ks_pcie->va_app_base))
-		return PTR_ERR(ks_pcie->va_app_base);
-
-	ks_pcie->app = *res;
-
-	/* Create legacy IRQ domain */
-	ks_pcie->legacy_irq_domain =
-			irq_domain_add_linear(ks_pcie->legacy_intc_np,
-					PCI_NUM_INTX,
-					&ks_dw_pcie_legacy_irq_domain_ops,
-					NULL);
-	if (!ks_pcie->legacy_irq_domain) {
-		dev_err(dev, "Failed to add irq domain for legacy irqs\n");
-		return -EINVAL;
-	}
-
-	return dw_pcie_host_init(pp);
-}
diff -Naur --no-dereference a/drivers/pci/controller/dwc/pci-keystone.h b/drivers/pci/controller/dwc/pci-keystone.h
--- a/drivers/pci/controller/dwc/pci-keystone.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/controller/dwc/pci-keystone.h	1969-12-31 19:00:00.000000000 -0500
@@ -1,57 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Keystone PCI Controller's common includes
- *
- * Copyright (C) 2013-2014 Texas Instruments., Ltd.
- *		http://www.ti.com
- *
- * Author: Murali Karicheri <m-karicheri2@ti.com>
- */
-
-#define MAX_MSI_HOST_IRQS		8
-
-struct keystone_pcie {
-	struct dw_pcie		*pci;
-	struct	clk		*clk;
-	/* PCI Device ID */
-	u32			device_id;
-	int			num_legacy_host_irqs;
-	int			legacy_host_irqs[PCI_NUM_INTX];
-	struct			device_node *legacy_intc_np;
-
-	int			num_msi_host_irqs;
-	int			msi_host_irqs[MAX_MSI_HOST_IRQS];
-	struct			device_node *msi_intc_np;
-	struct irq_domain	*legacy_irq_domain;
-	struct device_node	*np;
-
-	int error_irq;
-
-	/* Application register space */
-	void __iomem		*va_app_base;	/* DT 1st resource */
-	struct resource		app;
-};
-
-/* Keystone DW specific MSI controller APIs/definitions */
-void ks_dw_pcie_handle_msi_irq(struct keystone_pcie *ks_pcie, int offset);
-phys_addr_t ks_dw_pcie_get_msi_addr(struct pcie_port *pp);
-
-/* Keystone specific PCI controller APIs */
-void ks_dw_pcie_enable_legacy_irqs(struct keystone_pcie *ks_pcie);
-void ks_dw_pcie_handle_legacy_irq(struct keystone_pcie *ks_pcie, int offset);
-void ks_dw_pcie_enable_error_irq(struct keystone_pcie *ks_pcie);
-irqreturn_t ks_dw_pcie_handle_error_irq(struct keystone_pcie *ks_pcie);
-int  ks_dw_pcie_host_init(struct keystone_pcie *ks_pcie,
-			struct device_node *msi_intc_np);
-int ks_dw_pcie_wr_other_conf(struct pcie_port *pp, struct pci_bus *bus,
-		unsigned int devfn, int where, int size, u32 val);
-int ks_dw_pcie_rd_other_conf(struct pcie_port *pp, struct pci_bus *bus,
-		unsigned int devfn, int where, int size, u32 *val);
-void ks_dw_pcie_setup_rc_app_regs(struct keystone_pcie *ks_pcie);
-void ks_dw_pcie_initiate_link_train(struct keystone_pcie *ks_pcie);
-void ks_dw_pcie_msi_irq_ack(int i, struct pcie_port *pp);
-void ks_dw_pcie_msi_set_irq(struct pcie_port *pp, int irq);
-void ks_dw_pcie_msi_clear_irq(struct pcie_port *pp, int irq);
-void ks_dw_pcie_v3_65_scan_bus(struct pcie_port *pp);
-int ks_dw_pcie_msi_host_init(struct pcie_port *pp);
-int ks_dw_pcie_link_up(struct dw_pcie *pci);
diff -Naur --no-dereference a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig
--- a/drivers/pci/controller/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/controller/Kconfig	2020-06-19 21:00:31.420227716 -0400
@@ -48,6 +48,15 @@
 
 endmenu
 
+config PCI_J721E
+	bool "J721E PCIe host controller using Cadence PCIe core"
+	depends on PCIE_CADENCE
+	help
+	  Say Y here if you want to support the J721E PCIe controller in host
+	  mode or device mode. This PCIe controller uses the Cadence PCIe core.
+	  This driver also handles legacy interrupts when operating in host
+	  mode.
+
 config PCIE_XILINX_NWL
 	bool "NWL PCIe Core"
 	depends on ARCH_ZYNQMP || COMPILE_TEST
diff -Naur --no-dereference a/drivers/pci/controller/Makefile b/drivers/pci/controller/Makefile
--- a/drivers/pci/controller/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/controller/Makefile	2020-06-19 21:00:31.420227716 -0400
@@ -2,6 +2,7 @@
 obj-$(CONFIG_PCIE_CADENCE) += pcie-cadence.o
 obj-$(CONFIG_PCIE_CADENCE_HOST) += pcie-cadence-host.o
 obj-$(CONFIG_PCIE_CADENCE_EP) += pcie-cadence-ep.o
+obj-$(CONFIG_PCI_J721E) += pci-j721e.o
 obj-$(CONFIG_PCI_FTPCI100) += pci-ftpci100.o
 obj-$(CONFIG_PCI_HYPERV) += pci-hyperv.o
 obj-$(CONFIG_PCI_MVEBU) += pci-mvebu.o
diff -Naur --no-dereference a/drivers/pci/controller/pcie-cadence.c b/drivers/pci/controller/pcie-cadence.c
--- a/drivers/pci/controller/pcie-cadence.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/controller/pcie-cadence.c	2020-06-19 21:00:31.420227716 -0400
@@ -3,10 +3,91 @@
 // Cadence PCIe controller driver.
 // Author: Cyrille Pitchen <cyrille.pitchen@free-electrons.com>
 
+#include <linux/delay.h>
 #include <linux/kernel.h>
 
 #include "pcie-cadence.h"
 
+u32 cdns_pcie_read32(void __iomem *addr, int size)
+{
+	void __iomem *aligned_addr = PTR_ALIGN_DOWN(addr, 0x4);
+	unsigned int offset = (unsigned long)addr & 0x3;
+	u32 val = readl(aligned_addr);
+
+	if (!IS_ALIGNED((uintptr_t)addr, size)) {
+		pr_err("Invalid Address in function:%s\n", __func__);
+		return 0;
+	}
+
+	if (size > 2)
+		return val;
+
+	return (val >> (8 * offset)) & ((1 << (size * 8)) - 1);
+}
+
+void cdns_pcie_write32(void __iomem *addr, int size, u32 value)
+{
+	void __iomem *aligned_addr = PTR_ALIGN_DOWN(addr, 0x4);
+	unsigned int offset = (unsigned long)addr & 0x3;
+	u32 mask;
+	u32 val;
+
+	if (!IS_ALIGNED((uintptr_t)addr, size)) {
+		pr_err("Invalid Address in function:%s\n", __func__);
+		return;
+	}
+
+	if (size > 2) {
+		writel(value, addr);
+		return;
+	}
+
+	mask = ~(((1 << (size * 8)) - 1) << (offset * 8));
+	val = readl(aligned_addr) & mask;
+	val |= value << (offset * 8);
+	writel(val, aligned_addr);
+}
+
+int cdns_pcie_start_link(struct cdns_pcie *pci, bool start)
+{
+	struct cdns_pcie_plat_data *plat_data;
+
+	plat_data = pci->plat_data;
+	if (!plat_data)
+		return 0;
+
+	return plat_data->start_link(plat_data, start);
+}
+
+static bool cdns_pcie_is_link_up(struct cdns_pcie *pci)
+{
+	struct cdns_pcie_plat_data *plat_data;
+
+	plat_data = pci->plat_data;
+	if (!plat_data)
+		return true;
+
+	return plat_data->is_link_up(plat_data);
+}
+
+int cdns_pcie_wait_for_link(struct device *dev, struct cdns_pcie *pci)
+{
+	int retries;
+
+	/* Check if the link is up or not */
+	for (retries = 0; retries < LINK_WAIT_MAX_RETRIES; retries++) {
+		if (cdns_pcie_is_link_up(pci)) {
+			dev_info(dev, "Link up\n");
+			return 0;
+		}
+		usleep_range(LINK_WAIT_USLEEP_MIN, LINK_WAIT_USLEEP_MAX);
+	}
+
+	dev_err(dev, "Phy link never came up\n");
+
+	return -ETIMEDOUT;
+}
+
 void cdns_pcie_set_outbound_region(struct cdns_pcie *pcie, u8 fn,
 				   u32 r, bool is_io,
 				   u64 cpu_addr, u64 pci_addr, size_t size)
@@ -73,7 +154,8 @@
 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC1(r), desc1);
 
 	/* Set the CPU address */
-	cpu_addr -= pcie->mem_res->start;
+	if (pcie->mem_res)
+		cpu_addr -= pcie->mem_res->start;
 	addr0 = CDNS_PCIE_AT_OB_REGION_CPU_ADDR0_NBITS(nbits) |
 		(lower_32_bits(cpu_addr) & GENMASK(31, 8));
 	addr1 = upper_32_bits(cpu_addr);
@@ -100,7 +182,8 @@
 	}
 
 	/* Set the CPU address */
-	cpu_addr -= pcie->mem_res->start;
+	if (pcie->mem_res)
+		cpu_addr -= pcie->mem_res->start;
 	addr0 = CDNS_PCIE_AT_OB_REGION_CPU_ADDR0_NBITS(17) |
 		(lower_32_bits(cpu_addr) & GENMASK(31, 8));
 	addr1 = upper_32_bits(cpu_addr);
diff -Naur --no-dereference a/drivers/pci/controller/pcie-cadence-ep.c b/drivers/pci/controller/pcie-cadence-ep.c
--- a/drivers/pci/controller/pcie-cadence-ep.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/controller/pcie-cadence-ep.c	2020-06-19 21:00:31.420227716 -0400
@@ -6,6 +6,7 @@
 #include <linux/delay.h>
 #include <linux/kernel.h>
 #include <linux/of.h>
+#include <linux/of_device.h>
 #include <linux/pci-epc.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
@@ -16,10 +17,12 @@
 #define CDNS_PCIE_EP_MIN_APERTURE		128	/* 128 bytes */
 #define CDNS_PCIE_EP_IRQ_PCI_ADDR_NONE		0x1
 #define CDNS_PCIE_EP_IRQ_PCI_ADDR_LEGACY	0x3
+#define CDNS_PCIE_EP_MSIX_BAR				0x5
 
 /**
  * struct cdns_pcie_ep - private data for this PCIe endpoint controller driver
  * @pcie: Cadence PCIe controller
+ * @dev: pointer to PCIe EP device
  * @max_regions: maximum number of regions supported by hardware
  * @ob_region_map: bitmask of mapped outbound regions
  * @ob_addr: base addresses in the AXI bus where the outbound regions start
@@ -33,9 +36,11 @@
  * @irq_pci_fn: the latest PCI function that has updated the mapping of
  *		the MSI/legacy IRQ dedicated outbound region.
  * @irq_pending: bitmask of asserted legacy IRQs.
+ * @epf: info about physical functions attached to the endpoint controller
  */
 struct cdns_pcie_ep {
 	struct cdns_pcie		pcie;
+	struct device			*dev;
 	u32				max_regions;
 	unsigned long			ob_region_map;
 	phys_addr_t			*ob_addr;
@@ -44,13 +49,42 @@
 	u64				irq_pci_addr;
 	u8				irq_pci_fn;
 	u8				irq_pending;
+	struct cdns_pcie_epf		*epf;
 };
 
-static int cdns_pcie_ep_write_header(struct pci_epc *epc, u8 fn,
+static int cdns_pcie_ep_epf_init(struct pci_epc *epc, struct pci_epf *epf)
+{
+	return pci_epf_init_dma_chan(epf);
+}
+
+static void cdns_pcie_ep_epf_exit(struct pci_epc *epc, struct pci_epf *epf)
+{
+	pci_epf_clean_dma_chan(epf);
+}
+
+static int cdns_pcie_ep_data_transfer(struct pci_epc *epc, struct pci_epf *epf,
+				      dma_addr_t dma_dst, dma_addr_t dma_src,
+				      size_t len)
+{
+	return pci_epf_data_transfer(epf, dma_dst, dma_src, len);
+}
+
+static int cdns_pcie_ep_write_header(struct pci_epc *epc, u8 fn, u8 vfn,
 				     struct pci_epf_header *hdr)
 {
 	struct cdns_pcie_ep *ep = epc_get_drvdata(epc);
+	u32 cap = CDNS_PCIE_EP_FUNC_SRIOV_CAP_OFFSET;
 	struct cdns_pcie *pcie = &ep->pcie;
+	u32 reg;
+
+	if (vfn > 1) {
+		dev_dbg(&epc->dev, "Only Virtual Function #1 has deviceID\n");
+		return 0;
+	} else if (vfn == 1) {
+		reg = cap + PCI_SRIOV_VF_DID;
+		cdns_pcie_ep_fn_writew(pcie, fn, reg, hdr->deviceid);
+		return 0;
+	}
 
 	cdns_pcie_ep_fn_writew(pcie, fn, PCI_DEVICE_ID, hdr->deviceid);
 	cdns_pcie_ep_fn_writeb(pcie, fn, PCI_REVISION_ID, hdr->revid);
@@ -77,15 +111,18 @@
 	return 0;
 }
 
-static int cdns_pcie_ep_set_bar(struct pci_epc *epc, u8 fn,
+static int cdns_pcie_ep_set_bar(struct pci_epc *epc, u8 fn, u8 vfn,
 				struct pci_epf_bar *epf_bar)
 {
 	struct cdns_pcie_ep *ep = epc_get_drvdata(epc);
+	u32 cap = CDNS_PCIE_EP_FUNC_SRIOV_CAP_OFFSET;
+	struct cdns_pcie_epf *epf = &ep->epf[fn];
 	struct cdns_pcie *pcie = &ep->pcie;
 	dma_addr_t bar_phys = epf_bar->phys_addr;
 	enum pci_barno bar = epf_bar->barno;
 	int flags = epf_bar->flags;
 	u32 addr0, addr1, reg, cfg, b, aperture, ctrl;
+	u32 first_vf_offset, stride;
 	u64 sz;
 
 	/* BAR size is 2^(aperture + 7) */
@@ -121,19 +158,39 @@
 
 	addr0 = lower_32_bits(bar_phys);
 	addr1 = upper_32_bits(bar_phys);
+
+	if (vfn == 1) {
+		if (bar < BAR_4) {
+			reg = CDNS_PCIE_LM_EP_VFUNC_BAR_CFG0(fn);
+			b = bar;
+		} else {
+			reg = CDNS_PCIE_LM_EP_VFUNC_BAR_CFG1(fn);
+			b = bar - BAR_4;
+		}
+	} else {
+		if (bar < BAR_4) {
+			reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG0(fn);
+			b = bar;
+		} else {
+			reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG1(fn);
+			b = bar - BAR_4;
+		}
+	}
+
+	if (vfn > 0) {
+		first_vf_offset = cdns_pcie_ep_fn_readw(pcie, fn, cap +
+							PCI_SRIOV_VF_OFFSET);
+		stride = cdns_pcie_ep_fn_readw(pcie, fn, cap +
+					       PCI_SRIOV_VF_STRIDE);
+		fn = fn + first_vf_offset + ((vfn - 1) * stride);
+		epf = &epf->epf[vfn - 1];
+	}
+
 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar),
 			 addr0);
 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar),
 			 addr1);
 
-	if (bar < BAR_4) {
-		reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG0(fn);
-		b = bar;
-	} else {
-		reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG1(fn);
-		b = bar - BAR_4;
-	}
-
 	cfg = cdns_pcie_readl(pcie, reg);
 	cfg &= ~(CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) |
 		 CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b));
@@ -141,23 +198,47 @@
 		CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl));
 	cdns_pcie_writel(pcie, reg, cfg);
 
+	epf->epf_bar[bar] = epf_bar;
+
 	return 0;
 }
 
-static void cdns_pcie_ep_clear_bar(struct pci_epc *epc, u8 fn,
+static void cdns_pcie_ep_clear_bar(struct pci_epc *epc, u8 fn, u8 vfn,
 				   struct pci_epf_bar *epf_bar)
 {
 	struct cdns_pcie_ep *ep = epc_get_drvdata(epc);
+	u32 cap = CDNS_PCIE_EP_FUNC_SRIOV_CAP_OFFSET;
+	struct cdns_pcie_epf *epf = &ep->epf[fn];
 	struct cdns_pcie *pcie = &ep->pcie;
 	enum pci_barno bar = epf_bar->barno;
+	u32 first_vf_offset, stride;
 	u32 reg, cfg, b, ctrl;
 
-	if (bar < BAR_4) {
-		reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG0(fn);
-		b = bar;
+	if (vfn == 1) {
+		if (bar < BAR_4) {
+			reg = CDNS_PCIE_LM_EP_VFUNC_BAR_CFG0(fn);
+			b = bar;
+		} else {
+			reg = CDNS_PCIE_LM_EP_VFUNC_BAR_CFG1(fn);
+			b = bar - BAR_4;
+		}
 	} else {
-		reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG1(fn);
-		b = bar - BAR_4;
+		if (bar < BAR_4) {
+			reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG0(fn);
+			b = bar;
+		} else {
+			reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG1(fn);
+			b = bar - BAR_4;
+		}
+	}
+
+	if (vfn > 0) {
+		first_vf_offset = cdns_pcie_ep_fn_readw(pcie, fn, cap +
+							PCI_SRIOV_VF_OFFSET);
+		stride = cdns_pcie_ep_fn_readw(pcie, fn, cap +
+					       PCI_SRIOV_VF_STRIDE);
+		fn = fn + first_vf_offset + ((vfn - 1) * stride);
+		epf = &epf->epf[vfn - 1];
 	}
 
 	ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_DISABLED;
@@ -169,15 +250,27 @@
 
 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar), 0);
 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar), 0);
+
+	epf->epf_bar[bar] = NULL;
 }
 
-static int cdns_pcie_ep_map_addr(struct pci_epc *epc, u8 fn, phys_addr_t addr,
-				 u64 pci_addr, size_t size)
+static int cdns_pcie_ep_map_addr(struct pci_epc *epc, u8 fn, u8 vfn,
+				 phys_addr_t addr, u64 pci_addr, size_t size)
 {
 	struct cdns_pcie_ep *ep = epc_get_drvdata(epc);
 	struct cdns_pcie *pcie = &ep->pcie;
+	u32 cap = CDNS_PCIE_EP_FUNC_SRIOV_CAP_OFFSET;
+	u32 first_vf_offset, stride;
 	u32 r;
 
+	if (vfn > 0) {
+		first_vf_offset = cdns_pcie_ep_fn_readw(pcie, fn, cap +
+							PCI_SRIOV_VF_OFFSET);
+		stride = cdns_pcie_ep_fn_readw(pcie, fn, cap +
+					       PCI_SRIOV_VF_STRIDE);
+		fn = fn + first_vf_offset + ((vfn - 1) * stride);
+	}
+
 	r = find_first_zero_bit(&ep->ob_region_map,
 				sizeof(ep->ob_region_map) * BITS_PER_LONG);
 	if (r >= ep->max_regions - 1) {
@@ -193,7 +286,7 @@
 	return 0;
 }
 
-static void cdns_pcie_ep_unmap_addr(struct pci_epc *epc, u8 fn,
+static void cdns_pcie_ep_unmap_addr(struct pci_epc *epc, u8 fn, u8 vfn,
 				    phys_addr_t addr)
 {
 	struct cdns_pcie_ep *ep = epc_get_drvdata(epc);
@@ -213,13 +306,23 @@
 	clear_bit(r, &ep->ob_region_map);
 }
 
-static int cdns_pcie_ep_set_msi(struct pci_epc *epc, u8 fn, u8 mmc)
+static int cdns_pcie_ep_set_msi(struct pci_epc *epc, u8 fn, u8 vfn, u8 mmc)
 {
 	struct cdns_pcie_ep *ep = epc_get_drvdata(epc);
 	struct cdns_pcie *pcie = &ep->pcie;
+	u32 sriov_cap = CDNS_PCIE_EP_FUNC_SRIOV_CAP_OFFSET;
 	u32 cap = CDNS_PCIE_EP_FUNC_MSI_CAP_OFFSET;
+	u32 first_vf_offset, stride;
 	u16 flags;
 
+	if (vfn > 0) {
+		first_vf_offset = cdns_pcie_ep_fn_readw(pcie, fn, sriov_cap +
+							PCI_SRIOV_VF_OFFSET);
+		stride = cdns_pcie_ep_fn_readw(pcie, fn, sriov_cap +
+					       PCI_SRIOV_VF_STRIDE);
+		fn = fn + first_vf_offset + ((vfn - 1) * stride);
+	}
+
 	/*
 	 * Set the Multiple Message Capable bitfield into the Message Control
 	 * register.
@@ -233,13 +336,23 @@
 	return 0;
 }
 
-static int cdns_pcie_ep_get_msi(struct pci_epc *epc, u8 fn)
+static int cdns_pcie_ep_get_msi(struct pci_epc *epc, u8 fn, u8 vfn)
 {
 	struct cdns_pcie_ep *ep = epc_get_drvdata(epc);
 	struct cdns_pcie *pcie = &ep->pcie;
+	u32 sriov_cap = CDNS_PCIE_EP_FUNC_SRIOV_CAP_OFFSET;
 	u32 cap = CDNS_PCIE_EP_FUNC_MSI_CAP_OFFSET;
+	u32 first_vf_offset, stride;
 	u16 flags, mme;
 
+	if (vfn > 0) {
+		first_vf_offset = cdns_pcie_ep_fn_readw(pcie, fn, sriov_cap +
+							PCI_SRIOV_VF_OFFSET);
+		stride = cdns_pcie_ep_fn_readw(pcie, fn, sriov_cap +
+					       PCI_SRIOV_VF_STRIDE);
+		fn = fn + first_vf_offset + ((vfn - 1) * stride);
+	}
+
 	/* Validate that the MSI feature is actually enabled. */
 	flags = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSI_FLAGS);
 	if (!(flags & PCI_MSI_FLAGS_ENABLE))
@@ -254,7 +367,71 @@
 	return mme;
 }
 
-static void cdns_pcie_ep_assert_intx(struct cdns_pcie_ep *ep, u8 fn,
+static int cdns_pcie_ep_get_msix(struct pci_epc *epc, u8 func_no, u8 vfunc_no)
+{
+	u32 sriov_cap = CDNS_PCIE_EP_FUNC_SRIOV_CAP_OFFSET;
+	struct cdns_pcie_ep *ep = epc_get_drvdata(epc);
+	struct cdns_pcie *pcie = &ep->pcie;
+	u32 cap = CDNS_PCIE_EP_FUNC_MSIX_CAP_OFFSET;
+	u32 first_vf_offset, stride;
+	u32 val, reg;
+
+	if (vfunc_no > 0) {
+		first_vf_offset = cdns_pcie_ep_fn_readw(pcie, func_no, sriov_cap
+							+ PCI_SRIOV_VF_OFFSET);
+		stride = cdns_pcie_ep_fn_readw(pcie, func_no, sriov_cap +
+					       PCI_SRIOV_VF_STRIDE);
+		func_no = func_no + first_vf_offset + ((vfunc_no - 1) * stride);
+	}
+
+	reg = cap + PCI_MSIX_FLAGS;
+	val = cdns_pcie_ep_fn_readw(pcie, func_no, reg);
+	if (!(val & PCI_MSIX_FLAGS_ENABLE))
+		return -EINVAL;
+
+	val &= PCI_MSIX_FLAGS_QSIZE;
+
+	return val;
+}
+
+static int cdns_pcie_ep_set_msix(struct pci_epc *epc, u8 fn, u8 vfn,
+				 u16 interrupts, enum pci_barno bir, u32 offset)
+{
+	u32 sriov_cap = CDNS_PCIE_EP_FUNC_SRIOV_CAP_OFFSET;
+	struct cdns_pcie_ep *ep = epc_get_drvdata(epc);
+	struct cdns_pcie *pcie = &ep->pcie;
+	u32 cap = CDNS_PCIE_EP_FUNC_MSIX_CAP_OFFSET;
+	u32 first_vf_offset, stride;
+	u32 val, reg;
+
+	if (vfn > 0) {
+		first_vf_offset = cdns_pcie_ep_fn_readw(pcie, fn, sriov_cap +
+							PCI_SRIOV_VF_OFFSET);
+		stride = cdns_pcie_ep_fn_readw(pcie, fn, sriov_cap +
+					       PCI_SRIOV_VF_STRIDE);
+		fn = fn + first_vf_offset + ((vfn - 1) * stride);
+	}
+
+	reg = cap + PCI_MSIX_FLAGS;
+	val = cdns_pcie_ep_fn_readw(pcie, fn, reg);
+	val &= ~PCI_MSIX_FLAGS_QSIZE;
+	val |= interrupts;
+	cdns_pcie_ep_fn_writew(pcie, fn, reg, val);
+
+	/* Set MSIX BAR and offset */
+	reg = cap + PCI_MSIX_TABLE;
+	val = offset | bir;
+	cdns_pcie_ep_fn_writel(pcie, fn, reg, val);
+
+	/* Set PBA BAR and offset.  BAR must match MSIX BAR */
+	reg = cap + PCI_MSIX_PBA;
+	val = (offset + (interrupts * PCI_MSIX_ENTRY_SIZE)) | bir;
+	cdns_pcie_ep_fn_writel(pcie, fn, reg, val);
+
+	return 0;
+}
+
+static void cdns_pcie_ep_assert_intx(struct cdns_pcie_ep *ep, u8 fn, u8 vfn,
 				     u8 intx, bool is_asserted)
 {
 	struct cdns_pcie *pcie = &ep->pcie;
@@ -294,7 +471,8 @@
 	writel(0, ep->irq_cpu_addr + offset);
 }
 
-static int cdns_pcie_ep_send_legacy_irq(struct cdns_pcie_ep *ep, u8 fn, u8 intx)
+static int cdns_pcie_ep_send_legacy_irq(struct cdns_pcie_ep *ep, u8 fn, u8 vfn,
+					u8 intx)
 {
 	u16 cmd;
 
@@ -302,24 +480,34 @@
 	if (cmd & PCI_COMMAND_INTX_DISABLE)
 		return -EINVAL;
 
-	cdns_pcie_ep_assert_intx(ep, fn, intx, true);
+	cdns_pcie_ep_assert_intx(ep, fn, vfn, intx, true);
 	/*
 	 * The mdelay() value was taken from dra7xx_pcie_raise_legacy_irq()
 	 * from drivers/pci/dwc/pci-dra7xx.c
 	 */
 	mdelay(1);
-	cdns_pcie_ep_assert_intx(ep, fn, intx, false);
+	cdns_pcie_ep_assert_intx(ep, fn, vfn, intx, false);
 	return 0;
 }
 
-static int cdns_pcie_ep_send_msi_irq(struct cdns_pcie_ep *ep, u8 fn,
+static int cdns_pcie_ep_send_msi_irq(struct cdns_pcie_ep *ep, u8 fn, u8 vfn,
 				     u8 interrupt_num)
 {
 	struct cdns_pcie *pcie = &ep->pcie;
+	u32 sriov_cap = CDNS_PCIE_EP_FUNC_SRIOV_CAP_OFFSET;
 	u32 cap = CDNS_PCIE_EP_FUNC_MSI_CAP_OFFSET;
 	u16 flags, mme, data, data_mask;
 	u8 msi_count;
 	u64 pci_addr, pci_addr_mask = 0xff;
+	u32 first_vf_offset, stride;
+
+	if (vfn > 0) {
+		first_vf_offset = cdns_pcie_ep_fn_readw(pcie, fn, sriov_cap +
+							PCI_SRIOV_VF_OFFSET);
+		stride = cdns_pcie_ep_fn_readw(pcie, fn, sriov_cap +
+					       PCI_SRIOV_VF_STRIDE);
+		fn = fn + first_vf_offset + ((vfn - 1) * stride);
+	}
 
 	/* Check whether the MSI feature has been enabled by the PCI host. */
 	flags = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSI_FLAGS);
@@ -332,6 +520,11 @@
 	if (!interrupt_num || interrupt_num > msi_count)
 		return -EINVAL;
 
+	/* Check whether MSI is masked */
+	data = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSI_MASK_64);
+	if (data & (1 << (interrupt_num - 1)))
+		return -EINVAL;
+
 	/* Compute the data value to be written. */
 	data_mask = msi_count - 1;
 	data = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSI_DATA_64);
@@ -360,18 +553,147 @@
 	return 0;
 }
 
-static int cdns_pcie_ep_raise_irq(struct pci_epc *epc, u8 fn,
+static int cdns_pcie_ep_map_msi_irq(struct pci_epc *epc, u8 fn, u8 vfn,
+				    phys_addr_t addr, u8 interrupt_num,
+				    u32 entry_size, u32 *msi_data)
+{
+	u32 sriov_cap = CDNS_PCIE_EP_FUNC_SRIOV_CAP_OFFSET;
+	struct cdns_pcie_ep *ep = epc_get_drvdata(epc);
+	u32 cap = CDNS_PCIE_EP_FUNC_MSI_CAP_OFFSET;
+	struct cdns_pcie *pcie = &ep->pcie;
+	u16 flags, mme, data, data_mask;
+	u32 first_vf_offset, stride;
+	u8 msi_count;
+	u64 pci_addr;
+	int ret;
+	int i;
+
+	if (vfn > 0) {
+		first_vf_offset = cdns_pcie_ep_fn_readw(pcie, fn, sriov_cap +
+							PCI_SRIOV_VF_OFFSET);
+		stride = cdns_pcie_ep_fn_readw(pcie, fn, sriov_cap +
+					       PCI_SRIOV_VF_STRIDE);
+		fn = fn + first_vf_offset + ((vfn - 1) * stride);
+	}
+
+	/* Check whether the MSI feature has been enabled by the PCI host. */
+	flags = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSI_FLAGS);
+	if (!(flags & PCI_MSI_FLAGS_ENABLE))
+		return -EINVAL;
+
+	/* Get the number of enabled MSIs */
+	mme = (flags & PCI_MSI_FLAGS_QSIZE) >> 4;
+	msi_count = 1 << mme;
+	if (!interrupt_num || interrupt_num > msi_count)
+		return -EINVAL;
+
+	/* Compute the data value to be written. */
+	data_mask = msi_count - 1;
+	data = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSI_DATA_64);
+	data = data & ~data_mask;
+
+	/* Get the PCI address where to write the data into. */
+	pci_addr = cdns_pcie_ep_fn_readl(pcie, fn, cap + PCI_MSI_ADDRESS_HI);
+	pci_addr <<= 32;
+	pci_addr |= cdns_pcie_ep_fn_readl(pcie, fn, cap + PCI_MSI_ADDRESS_LO);
+	pci_addr &= GENMASK_ULL(63, 2);
+
+	for (i = 0; i < interrupt_num; i++) {
+		ret = cdns_pcie_ep_map_addr(epc, fn, vfn, addr, pci_addr,
+					    entry_size);
+		if (ret)
+			return ret;
+		addr = addr + entry_size;
+	}
+
+	*msi_data = data;
+
+	return 0;
+}
+
+static int cdns_pcie_ep_send_msix_irq(struct cdns_pcie_ep *ep, u8 fn, u8 vfn,
+				      u16 interrupt_num)
+{
+	u32 sriov_cap = CDNS_PCIE_EP_FUNC_SRIOV_CAP_OFFSET;
+	u32 cap = CDNS_PCIE_EP_FUNC_MSIX_CAP_OFFSET;
+	u32 tbl_offset, msg_data, reg, vec_ctrl;
+	struct cdns_pcie *pcie = &ep->pcie;
+	struct pci_epf_msix_tbl *msix_tbl;
+	struct pci_epf_bar *epf_bar;
+	u32 first_vf_offset, stride;
+	struct cdns_pcie_epf *epf;
+	u64 pci_addr_mask = 0xff;
+	u64 msg_addr;
+	u16 flags;
+	u8 bir;
+
+	epf = &ep->epf[fn];
+
+	if (vfn > 0) {
+		first_vf_offset = cdns_pcie_ep_fn_readw(pcie, fn, sriov_cap +
+							PCI_SRIOV_VF_OFFSET);
+		stride = cdns_pcie_ep_fn_readw(pcie, fn, sriov_cap +
+					       PCI_SRIOV_VF_STRIDE);
+		fn = fn + first_vf_offset + ((vfn - 1) * stride);
+		epf = &epf->epf[vfn - 1];
+	}
+
+	/* Check whether the MSI-X feature has been enabled by the PCI host. */
+	flags = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSIX_FLAGS);
+	if (!(flags & PCI_MSIX_FLAGS_ENABLE))
+		return -EINVAL;
+
+	reg = cap + PCI_MSIX_TABLE;
+	tbl_offset = cdns_pcie_ep_fn_readl(pcie, fn, reg);
+	bir = tbl_offset & PCI_MSIX_TABLE_BIR;
+	tbl_offset &= PCI_MSIX_TABLE_OFFSET;
+
+	epf_bar = epf->epf_bar[bir];
+	msix_tbl = epf_bar->addr;
+	msix_tbl = (struct pci_epf_msix_tbl *)((char *)msix_tbl + tbl_offset);
+
+	msg_addr = msix_tbl[(interrupt_num - 1)].msg_addr;
+	msg_data = msix_tbl[(interrupt_num - 1)].msg_data;
+	vec_ctrl = msix_tbl[(interrupt_num - 1)].vector_ctrl;
+
+	/* Set the outbound region if needed. */
+	if (unlikely(ep->irq_pci_addr != (msg_addr & ~pci_addr_mask) ||
+		     ep->irq_pci_fn != fn)) {
+		/* First region was reserved for IRQ writes. */
+		cdns_pcie_set_outbound_region(pcie, fn, 0,
+					      false,
+					      ep->irq_phys_addr,
+					      msg_addr & ~pci_addr_mask,
+					      pci_addr_mask + 1);
+		ep->irq_pci_addr = (msg_addr & ~pci_addr_mask);
+		ep->irq_pci_fn = fn;
+	}
+	writel(msg_data, ep->irq_cpu_addr + (msg_addr & pci_addr_mask));
+
+	return 0;
+}
+
+static int cdns_pcie_ep_raise_irq(struct pci_epc *epc, u8 fn, u8 vfn,
 				  enum pci_epc_irq_type type,
 				  u16 interrupt_num)
 {
 	struct cdns_pcie_ep *ep = epc_get_drvdata(epc);
+	u32 link_status;
+
+	/* Can't send an IRQ if the link is down. */
+	link_status = cdns_pcie_readl(&ep->pcie, CDNS_PCIE_LM_BASE);
+	if (!(link_status & 0x1))
+		return -EINVAL;
 
 	switch (type) {
 	case PCI_EPC_IRQ_LEGACY:
-		return cdns_pcie_ep_send_legacy_irq(ep, fn, 0);
+		return cdns_pcie_ep_send_legacy_irq(ep, fn, vfn, 0);
 
 	case PCI_EPC_IRQ_MSI:
-		return cdns_pcie_ep_send_msi_irq(ep, fn, interrupt_num);
+		return cdns_pcie_ep_send_msi_irq(ep, fn, vfn, interrupt_num);
+
+	case PCI_EPC_IRQ_MSIX:
+		return cdns_pcie_ep_send_msix_irq(ep, fn, vfn, interrupt_num);
 
 	default:
 		break;
@@ -385,6 +707,7 @@
 	struct cdns_pcie_ep *ep = epc_get_drvdata(epc);
 	struct cdns_pcie *pcie = &ep->pcie;
 	struct pci_epf *epf;
+	int ret = 0;
 	u32 cfg;
 
 	/*
@@ -396,22 +719,40 @@
 		cfg |= BIT(epf->func_no);
 	cdns_pcie_writel(pcie, CDNS_PCIE_LM_EP_FUNC_CFG, cfg);
 
-	/*
-	 * The PCIe links are automatically established by the controller
-	 * once for all at powerup: the software can neither start nor stop
-	 * those links later at runtime.
-	 *
-	 * Then we only have to notify the EP core that our links are already
-	 * established. However we don't call directly pci_epc_linkup() because
-	 * we've already locked the epc->lock.
-	 */
-	list_for_each_entry(epf, &epc->pci_epf, list)
-		pci_epf_linkup(epf);
+	ret = cdns_pcie_start_link(pcie, true);
+	if (ret)
+		dev_err(ep->dev, "Failed to start link\n");
+
+	return ret;
+}
 
-	return 0;
+static const struct pci_epc_features cdns_pcie_epc_features = {
+	.linkup_notifier = false,
+	.msi_capable = true,
+	.msix_capable = true,
+	.align = 256,
+};
+
+static const struct pci_epc_features cdns_pcie_epc_vf_features = {
+	.linkup_notifier = false,
+	.msi_capable = true,
+	.msix_capable = true,
+	.align = 65536,
+};
+
+static const struct pci_epc_features*
+cdns_pcie_ep_get_features(struct pci_epc *epc, u8 func_no, u8 vfunc_no)
+{
+	if (!vfunc_no)
+		return &cdns_pcie_epc_features;
+
+	return &cdns_pcie_epc_vf_features;
 }
 
 static const struct pci_epc_ops cdns_pcie_epc_ops = {
+	.epf_init	= cdns_pcie_ep_epf_init,
+	.epf_exit	= cdns_pcie_ep_epf_exit,
+	.data_transfer	= cdns_pcie_ep_data_transfer,
 	.write_header	= cdns_pcie_ep_write_header,
 	.set_bar	= cdns_pcie_ep_set_bar,
 	.clear_bar	= cdns_pcie_ep_clear_bar,
@@ -419,13 +760,25 @@
 	.unmap_addr	= cdns_pcie_ep_unmap_addr,
 	.set_msi	= cdns_pcie_ep_set_msi,
 	.get_msi	= cdns_pcie_ep_get_msi,
+	.set_msix	= cdns_pcie_ep_set_msix,
+	.get_msix	= cdns_pcie_ep_get_msix,
 	.raise_irq	= cdns_pcie_ep_raise_irq,
+	.map_msi_irq	= cdns_pcie_ep_map_msi_irq,
 	.start		= cdns_pcie_ep_start,
+	.get_features	= cdns_pcie_ep_get_features,
 };
 
-static const struct of_device_id cdns_pcie_ep_of_match[] = {
-	{ .compatible = "cdns,cdns-pcie-ep" },
+static struct cdns_pcie_ep_data cdns_ti_pcie_ep_data = {
+	.read = cdns_pcie_read32,
+	.write = cdns_pcie_write32,
+};
 
+static const struct of_device_id cdns_pcie_ep_of_match[] = {
+	{ .compatible = "cdns,cdns-pcie-ep",
+	},
+	{ .compatible = "ti,j721e-cdns-pcie-ep",
+	  .data = &cdns_ti_pcie_ep_data,
+	},
 	{ },
 };
 
@@ -433,19 +786,39 @@
 {
 	struct device *dev = &pdev->dev;
 	struct device_node *np = dev->of_node;
+	const struct cdns_pcie_ep_data *data;
+	const struct of_device_id *match;
+	struct cdns_pcie_epf *epf;
 	struct cdns_pcie_ep *ep;
 	struct cdns_pcie *pcie;
 	struct pci_epc *epc;
 	struct resource *res;
 	int ret;
 	int phy_count;
+	u8 *max_vfs;
+	int i;
+
+	match = of_match_device(of_match_ptr(cdns_pcie_ep_of_match), dev);
+	if (!match)
+		return -EINVAL;
 
 	ep = devm_kzalloc(dev, sizeof(*ep), GFP_KERNEL);
 	if (!ep)
 		return -ENOMEM;
 
+	ep->dev = dev;
+
 	pcie = &ep->pcie;
 	pcie->is_rc = false;
+	pcie->plat_data = pdev->dev.platform_data;
+
+	data = (struct cdns_pcie_ep_data *)match->data;
+	if (data) {
+		if (data->read)
+			pcie->read = data->read;
+		if (data->write)
+			pcie->write = data->write;
+	}
 
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "reg");
 	pcie->reg_base = devm_ioremap_resource(dev, res);
@@ -455,11 +828,16 @@
 	}
 
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mem");
+	if (!res)
+		dev_dbg(dev, "missing \"mem\"\n");
+	pcie->mem_res = res;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "addr_space");
 	if (!res) {
-		dev_err(dev, "missing \"mem\"\n");
+		dev_err(dev, "missing \"addr_space\"\n");
 		return -EINVAL;
 	}
-	pcie->mem_res = res;
+	pcie->addr_res = res;
 
 	ret = of_property_read_u32(np, "cdns,max-outbound-regions",
 				   &ep->max_regions);
@@ -501,8 +879,31 @@
 	if (of_property_read_u8(np, "max-functions", &epc->max_functions) < 0)
 		epc->max_functions = 1;
 
-	ret = pci_epc_mem_init(epc, pcie->mem_res->start,
-			       resource_size(pcie->mem_res));
+	ep->epf = devm_kcalloc(dev, epc->max_functions, sizeof(*ep->epf),
+			       GFP_KERNEL);
+	if (!ep->epf)
+		return -ENOMEM;
+
+	max_vfs = devm_kcalloc(dev, epc->max_functions, sizeof(*max_vfs),
+			       GFP_KERNEL);
+	if (!max_vfs)
+		return -ENOMEM;
+
+	of_property_read_u8_array(np, "max-virtual-functions", max_vfs,
+				  epc->max_functions);
+
+	for (i = 0; i < epc->max_functions; i++) {
+		epf = &ep->epf[i];
+		if (max_vfs[i] == 0)
+			continue;
+		epf->epf = devm_kcalloc(dev, max_vfs[i], sizeof(*ep->epf),
+					GFP_KERNEL);
+		if (!epf->epf)
+			return -ENOMEM;
+	}
+
+	ret = pci_epc_mem_init(epc, pcie->addr_res->start,
+			       resource_size(pcie->addr_res));
 	if (ret < 0) {
 		dev_err(dev, "failed to initialize the memory space\n");
 		goto err_init;
diff -Naur --no-dereference a/drivers/pci/controller/pcie-cadence.h b/drivers/pci/controller/pcie-cadence.h
--- a/drivers/pci/controller/pcie-cadence.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/controller/pcie-cadence.h	2020-06-19 21:00:31.420227716 -0400
@@ -10,6 +10,11 @@
 #include <linux/pci.h>
 #include <linux/phy/phy.h>
 
+/* Parameters for the waiting for link up routine */
+#define LINK_WAIT_MAX_RETRIES	10
+#define LINK_WAIT_USLEEP_MIN	90000
+#define LINK_WAIT_USLEEP_MAX	100000
+
 /*
  * Local Management Registers
  */
@@ -45,6 +50,10 @@
 	(CDNS_PCIE_LM_BASE + 0x0240 + (fn) * 0x0008)
 #define CDNS_PCIE_LM_EP_FUNC_BAR_CFG1(fn) \
 	(CDNS_PCIE_LM_BASE + 0x0244 + (fn) * 0x0008)
+#define CDNS_PCIE_LM_EP_VFUNC_BAR_CFG0(fn) \
+	(CDNS_PCIE_LM_BASE + 0x0280 + (fn) * 0x0008)
+#define CDNS_PCIE_LM_EP_VFUNC_BAR_CFG1(fn) \
+	(CDNS_PCIE_LM_BASE + 0x0284 + (fn) * 0x0008)
 #define  CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) \
 	(GENMASK(4, 0) << ((b) * 8))
 #define  CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE(b, a) \
@@ -94,6 +103,8 @@
 #define CDNS_PCIE_EP_FUNC_BASE(fn)	(((fn) << 12) & GENMASK(19, 12))
 
 #define CDNS_PCIE_EP_FUNC_MSI_CAP_OFFSET	0x90
+#define CDNS_PCIE_EP_FUNC_SRIOV_CAP_OFFSET	0x200
+#define CDNS_PCIE_EP_FUNC_MSIX_CAP_OFFSET	0xb0
 
 /*
  * Root Port Registers (PCI configuration space for the root port function)
@@ -221,6 +232,21 @@
 	MSG_ROUTING_GATHER,
 };
 
+struct cdns_pcie_plat_data {
+	int (*start_link)(struct cdns_pcie_plat_data *data, bool start);
+	bool (*is_link_up)(struct cdns_pcie_plat_data *data);
+};
+
+/**
+ * struct cdns_pcie_epf - Structure to hold info about endpoint function
+ * @epf: Info about virtual functions attached to the physical function
+ * @epf_bar: reference to the pci_epf_bar for the six Base Address Registers
+ */
+struct cdns_pcie_epf {
+	struct cdns_pcie_epf *epf;
+	struct pci_epf_bar *epf_bar[6];
+};
+
 /**
  * struct cdns_pcie - private data for Cadence PCIe controller drivers
  * @reg_base: IO mapped register base
@@ -231,31 +257,72 @@
 struct cdns_pcie {
 	void __iomem		*reg_base;
 	struct resource		*mem_res;
+	struct resource		*addr_res;
 	bool			is_rc;
 	u8			bus;
 	int			phy_count;
 	struct phy		**phy;
 	struct device_link	**link;
+	struct cdns_pcie_plat_data *plat_data;
+	u32 (*read)(void __iomem *addr, int size);
+	void (*write)(void __iomem *addr, int size, u32 value);
+};
+
+struct cdns_pcie_host_data {
+	struct pci_ops *ops;
+	u32 (*read)(void __iomem *addr, int size);
+	void (*write)(void __iomem *addr, int size, u32 value);
+};
+
+struct cdns_pcie_ep_data {
+	u32 (*read)(void __iomem *addr, int size);
+	void (*write)(void __iomem *addr, int size, u32 value);
 };
 
 /* Register access */
 static inline void cdns_pcie_writeb(struct cdns_pcie *pcie, u32 reg, u8 value)
 {
+	void __iomem *addr = pcie->reg_base + reg;
+
+	if (pcie->write) {
+		pcie->write(addr, 0x1, value);
+		return;
+	}
+
 	writeb(value, pcie->reg_base + reg);
 }
 
 static inline void cdns_pcie_writew(struct cdns_pcie *pcie, u32 reg, u16 value)
 {
+	void __iomem *addr = pcie->reg_base + reg;
+
+	if (pcie->write) {
+		pcie->write(addr, 0x2, value);
+		return;
+	}
+
 	writew(value, pcie->reg_base + reg);
 }
 
 static inline void cdns_pcie_writel(struct cdns_pcie *pcie, u32 reg, u32 value)
 {
+	void __iomem *addr = pcie->reg_base + reg;
+
+	if (pcie->write) {
+		pcie->write(addr, 0x4, value);
+		return;
+	}
+
 	writel(value, pcie->reg_base + reg);
 }
 
 static inline u32 cdns_pcie_readl(struct cdns_pcie *pcie, u32 reg)
 {
+	void __iomem *addr = pcie->reg_base + reg;
+
+	if (pcie->read)
+		return pcie->read(addr, 0x4);
+
 	return readl(pcie->reg_base + reg);
 }
 
@@ -263,47 +330,97 @@
 static inline void cdns_pcie_rp_writeb(struct cdns_pcie *pcie,
 				       u32 reg, u8 value)
 {
-	writeb(value, pcie->reg_base + CDNS_PCIE_RP_BASE + reg);
+	void __iomem *addr = pcie->reg_base + CDNS_PCIE_RP_BASE + reg;
+
+	if (pcie->write) {
+		pcie->write(addr, 0x1, value);
+		return;
+	}
+
+	writeb(value, addr);
 }
 
 static inline void cdns_pcie_rp_writew(struct cdns_pcie *pcie,
 				       u32 reg, u16 value)
 {
-	writew(value, pcie->reg_base + CDNS_PCIE_RP_BASE + reg);
+	void __iomem *addr = pcie->reg_base + CDNS_PCIE_RP_BASE + reg;
+
+	if (pcie->write) {
+		pcie->write(addr, 0x2, value);
+		return;
+	}
+
+	writew(value, addr);
 }
 
 /* Endpoint Function register access */
 static inline void cdns_pcie_ep_fn_writeb(struct cdns_pcie *pcie, u8 fn,
 					  u32 reg, u8 value)
 {
-	writeb(value, pcie->reg_base + CDNS_PCIE_EP_FUNC_BASE(fn) + reg);
+	void __iomem *addr = pcie->reg_base + CDNS_PCIE_EP_FUNC_BASE(fn) + reg;
+
+	if (pcie->write) {
+		pcie->write(addr, 0x1, value);
+		return;
+	}
+
+	writeb(value, addr);
 }
 
 static inline void cdns_pcie_ep_fn_writew(struct cdns_pcie *pcie, u8 fn,
 					  u32 reg, u16 value)
 {
-	writew(value, pcie->reg_base + CDNS_PCIE_EP_FUNC_BASE(fn) + reg);
+	void __iomem *addr = pcie->reg_base + CDNS_PCIE_EP_FUNC_BASE(fn) + reg;
+
+	if (pcie->write) {
+		pcie->write(addr, 0x2, value);
+		return;
+	}
+
+	writew(value, addr);
 }
 
 static inline void cdns_pcie_ep_fn_writel(struct cdns_pcie *pcie, u8 fn,
 					  u32 reg, u32 value)
 {
-	writel(value, pcie->reg_base + CDNS_PCIE_EP_FUNC_BASE(fn) + reg);
+	void __iomem *addr = pcie->reg_base + CDNS_PCIE_EP_FUNC_BASE(fn) + reg;
+
+	if (pcie->write) {
+		pcie->write(addr, 0x4, value);
+		return;
+	}
+
+	writel(value, addr);
 }
 
 static inline u8 cdns_pcie_ep_fn_readb(struct cdns_pcie *pcie, u8 fn, u32 reg)
 {
-	return readb(pcie->reg_base + CDNS_PCIE_EP_FUNC_BASE(fn) + reg);
+	void __iomem *addr = pcie->reg_base + CDNS_PCIE_EP_FUNC_BASE(fn) + reg;
+
+	if (pcie->read)
+		return pcie->read(addr, 0x1);
+
+	return readb(addr);
 }
 
 static inline u16 cdns_pcie_ep_fn_readw(struct cdns_pcie *pcie, u8 fn, u32 reg)
 {
-	return readw(pcie->reg_base + CDNS_PCIE_EP_FUNC_BASE(fn) + reg);
+	void __iomem *addr = pcie->reg_base + CDNS_PCIE_EP_FUNC_BASE(fn) + reg;
+
+	if (pcie->read)
+		return pcie->read(addr, 0x2);
+
+	return readw(addr);
 }
 
 static inline u32 cdns_pcie_ep_fn_readl(struct cdns_pcie *pcie, u8 fn, u32 reg)
 {
-	return readl(pcie->reg_base + CDNS_PCIE_EP_FUNC_BASE(fn) + reg);
+	void __iomem *addr = pcie->reg_base + CDNS_PCIE_EP_FUNC_BASE(fn) + reg;
+
+	if (pcie->read)
+		return pcie->read(addr, 0x4);
+
+	return readl(addr);
 }
 
 void cdns_pcie_set_outbound_region(struct cdns_pcie *pcie, u8 fn,
@@ -317,6 +434,10 @@
 void cdns_pcie_disable_phy(struct cdns_pcie *pcie);
 int cdns_pcie_enable_phy(struct cdns_pcie *pcie);
 int cdns_pcie_init_phy(struct device *dev, struct cdns_pcie *pcie);
+u32 cdns_pcie_read32(void __iomem *addr, int size);
+void cdns_pcie_write32(void __iomem *addr, int size, u32 value);
+int cdns_pcie_start_link(struct cdns_pcie *pci, bool start);
+int cdns_pcie_wait_for_link(struct device *dev, struct cdns_pcie *pci);
 extern const struct dev_pm_ops cdns_pcie_pm_ops;
 
 #endif /* _PCIE_CADENCE_H */
diff -Naur --no-dereference a/drivers/pci/controller/pcie-cadence-host.c b/drivers/pci/controller/pcie-cadence-host.c
--- a/drivers/pci/controller/pcie-cadence-host.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/controller/pcie-cadence-host.c	2020-06-19 21:00:31.420227716 -0400
@@ -3,8 +3,11 @@
 // Cadence PCIe host controller driver.
 // Author: Cyrille Pitchen <cyrille.pitchen@free-electrons.com>
 
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
 #include <linux/kernel.h>
 #include <linux/of_address.h>
+#include <linux/of_device.h>
 #include <linux/of_pci.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
@@ -92,9 +95,52 @@
 	.write		= pci_generic_config_write,
 };
 
-static const struct of_device_id cdns_pcie_host_of_match[] = {
-	{ .compatible = "cdns,cdns-pcie-host" },
+static int cdns_ti_pcie_config_read(struct pci_bus *bus, unsigned int devfn,
+				    int where, int size, u32 *value)
+{
+	struct pci_host_bridge *bridge = pci_find_host_bridge(bus);
+	struct cdns_pcie_rc *rc = pci_host_bridge_priv(bridge);
+	unsigned int busn = bus->number;
+
+	if (busn == rc->bus_range->start)
+		return pci_generic_config_read32(bus, devfn, where, size,
+						 value);
+
+	return pci_generic_config_read(bus, devfn, where, size, value);
+}
+
+static int cdns_ti_pcie_config_write(struct pci_bus *bus, unsigned int devfn,
+				     int where, int size, u32 value)
+{
+	struct pci_host_bridge *bridge = pci_find_host_bridge(bus);
+	struct cdns_pcie_rc *rc = pci_host_bridge_priv(bridge);
+	unsigned int busn = bus->number;
+
+	if (busn == rc->bus_range->start)
+		return pci_generic_config_write32(bus, devfn, where, size,
+						  value);
 
+	return pci_generic_config_write(bus, devfn, where, size, value);
+}
+
+static struct pci_ops cdns_ti_pcie_host_ops = {
+	.map_bus	= cdns_pci_map_bus,
+	.read		= cdns_ti_pcie_config_read,
+	.write		= cdns_ti_pcie_config_write,
+};
+
+static struct cdns_pcie_host_data cdns_ti_pcie_host_data = {
+	.read = cdns_pcie_read32,
+	.write = cdns_pcie_write32,
+	.ops = &cdns_ti_pcie_host_ops,
+};
+
+static const struct of_device_id cdns_pcie_host_of_match[] = {
+	{ .compatible = "cdns,cdns-pcie-host",
+	},
+	{ .compatible = "ti,j721e-cdns-pcie-host",
+	  .data = &cdns_ti_pcie_host_data,
+	},
 	{ },
 };
 
@@ -102,6 +148,7 @@
 {
 	struct cdns_pcie *pcie = &rc->pcie;
 	u32 value, ctrl;
+	u32 id;
 
 	/*
 	 * Set the root complex BAR configuration register:
@@ -121,8 +168,12 @@
 	cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value);
 
 	/* Set root port configuration space */
-	if (rc->vendor_id != 0xffff)
-		cdns_pcie_rp_writew(pcie, PCI_VENDOR_ID, rc->vendor_id);
+	if (rc->vendor_id != 0xffff) {
+		id = CDNS_PCIE_LM_ID_VENDOR(rc->vendor_id) |
+			CDNS_PCIE_LM_ID_SUBSYS(rc->vendor_id);
+		cdns_pcie_writel(pcie, CDNS_PCIE_LM_ID, id);
+	}
+
 	if (rc->device_id != 0xffff)
 		cdns_pcie_rp_writew(pcie, PCI_DEVICE_ID, rc->device_id);
 
@@ -142,9 +193,9 @@
 	struct device *dev = rc->dev;
 	struct device_node *np = dev->of_node;
 	struct of_pci_range_parser parser;
+	u64 cpu_addr = cfg_res->start;
 	struct of_pci_range range;
 	u32 addr0, addr1, desc1;
-	u64 cpu_addr;
 	int r, err;
 
 	/*
@@ -157,7 +208,8 @@
 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR1(0), addr1);
 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC1(0), desc1);
 
-	cpu_addr = cfg_res->start - mem_res->start;
+	if (mem_res)
+		cpu_addr -= mem_res->start;
 	addr0 = CDNS_PCIE_AT_OB_REGION_CPU_ADDR0_NBITS(12) |
 		(lower_32_bits(cpu_addr) & GENMASK(31, 8));
 	addr1 = upper_32_bits(cpu_addr);
@@ -235,17 +287,25 @@
 
 static int cdns_pcie_host_probe(struct platform_device *pdev)
 {
+	struct pci_ops *ops = &cdns_pcie_host_ops;
+	const struct cdns_pcie_host_data *data;
 	const char *type;
 	struct device *dev = &pdev->dev;
 	struct device_node *np = dev->of_node;
+	const struct of_device_id *match;
 	struct pci_host_bridge *bridge;
 	struct list_head resources;
 	struct cdns_pcie_rc *rc;
+	struct gpio_desc *gpiod;
 	struct cdns_pcie *pcie;
 	struct resource *res;
 	int ret;
 	int phy_count;
 
+	match = of_match_device(of_match_ptr(cdns_pcie_host_of_match), dev);
+	if (!match)
+		return -EINVAL;
+
 	bridge = devm_pci_alloc_host_bridge(dev, sizeof(*rc));
 	if (!bridge)
 		return -ENOMEM;
@@ -254,8 +314,19 @@
 	rc->dev = dev;
 
 	pcie = &rc->pcie;
+	pcie->plat_data = pdev->dev.platform_data;
 	pcie->is_rc = true;
 
+	data = (struct cdns_pcie_host_data *)match->data;
+	if (data) {
+		if (data->read)
+			pcie->read = data->read;
+		if (data->write)
+			pcie->write = data->write;
+		if (data->ops)
+			ops = data->ops;
+	}
+
 	rc->max_regions = 32;
 	of_property_read_u32(np, "cdns,max-outbound-regions", &rc->max_regions);
 
@@ -290,17 +361,38 @@
 	rc->cfg_res = res;
 
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mem");
-	if (!res) {
+	if (!res)
 		dev_err(dev, "missing \"mem\"\n");
-		return -EINVAL;
-	}
+
 	pcie->mem_res = res;
 
+	gpiod = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_LOW);
+	if (IS_ERR(gpiod)) {
+		ret = PTR_ERR(gpiod);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "Failed to get reset GPIO\n");
+		return ret;
+	}
+
 	ret = cdns_pcie_init_phy(dev, pcie);
 	if (ret) {
 		dev_err(dev, "failed to init phy\n");
 		return ret;
 	}
+
+	/*
+	 * "Power Sequencing and Reset Signal Timings" table in
+	 * PCI EXPRESS CARD ELECTROMECHANICAL SPECIFICATION, REV. 3.0
+	 * indicates PERST# should be deasserted after minimum of 100us
+	 * once REFCLK is stable. The REFCLK to the connector in RC
+	 * mode is selected while enabling the PHY. So deassert PERST#
+	 * after 100 us.
+	 */
+	if (gpiod) {
+		usleep_range(100, 200);
+		gpiod_set_value_cansleep(gpiod, 1);
+	}
+
 	platform_set_drvdata(pdev, pcie);
 
 	pm_runtime_enable(dev);
@@ -310,6 +402,14 @@
 		goto err_get_sync;
 	}
 
+	ret = cdns_pcie_start_link(pcie, true);
+	if (ret) {
+		dev_err(dev, "Failed to start link\n");
+		goto err_start_link;
+	}
+
+	cdns_pcie_wait_for_link(dev, pcie);
+
 	ret = cdns_pcie_host_init(dev, &resources, rc);
 	if (ret)
 		goto err_init;
@@ -317,7 +417,7 @@
 	list_splice_init(&resources, &bridge->windows);
 	bridge->dev.parent = dev;
 	bridge->busnr = pcie->bus;
-	bridge->ops = &cdns_pcie_host_ops;
+	bridge->ops = ops;
 	bridge->map_irq = of_irq_parse_and_map_pci;
 	bridge->swizzle_irq = pci_common_swizzle;
 
@@ -331,6 +431,9 @@
 	pci_free_resource_list(&resources);
 
  err_init:
+	cdns_pcie_start_link(pcie, false);
+
+ err_start_link:
 	pm_runtime_put_sync(dev);
 
  err_get_sync:
diff -Naur --no-dereference a/drivers/pci/controller/pcie-rockchip-ep.c b/drivers/pci/controller/pcie-rockchip-ep.c
--- a/drivers/pci/controller/pcie-rockchip-ep.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/controller/pcie-rockchip-ep.c	2020-06-19 21:00:31.420227716 -0400
@@ -121,7 +121,7 @@
 			    ROCKCHIP_PCIE_AT_OB_REGION_CPU_ADDR1(r));
 }
 
-static int rockchip_pcie_ep_write_header(struct pci_epc *epc, u8 fn,
+static int rockchip_pcie_ep_write_header(struct pci_epc *epc, u8 fn, u8 vf,
 					 struct pci_epf_header *hdr)
 {
 	struct rockchip_pcie_ep *ep = epc_get_drvdata(epc);
@@ -158,7 +158,7 @@
 	return 0;
 }
 
-static int rockchip_pcie_ep_set_bar(struct pci_epc *epc, u8 fn,
+static int rockchip_pcie_ep_set_bar(struct pci_epc *epc, u8 fn, u8 vf,
 				    struct pci_epf_bar *epf_bar)
 {
 	struct rockchip_pcie_ep *ep = epc_get_drvdata(epc);
@@ -226,7 +226,7 @@
 	return 0;
 }
 
-static void rockchip_pcie_ep_clear_bar(struct pci_epc *epc, u8 fn,
+static void rockchip_pcie_ep_clear_bar(struct pci_epc *epc, u8 fn, u8 vf,
 				       struct pci_epf_bar *epf_bar)
 {
 	struct rockchip_pcie_ep *ep = epc_get_drvdata(epc);
@@ -255,7 +255,7 @@
 			    ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar));
 }
 
-static int rockchip_pcie_ep_map_addr(struct pci_epc *epc, u8 fn,
+static int rockchip_pcie_ep_map_addr(struct pci_epc *epc, u8 fn, u8 vf,
 				     phys_addr_t addr, u64 pci_addr,
 				     size_t size)
 {
@@ -283,7 +283,7 @@
 	return 0;
 }
 
-static void rockchip_pcie_ep_unmap_addr(struct pci_epc *epc, u8 fn,
+static void rockchip_pcie_ep_unmap_addr(struct pci_epc *epc, u8 fn, u8 vf,
 					phys_addr_t addr)
 {
 	struct rockchip_pcie_ep *ep = epc_get_drvdata(epc);
@@ -307,7 +307,7 @@
 	clear_bit(r, &ep->ob_region_map);
 }
 
-static int rockchip_pcie_ep_set_msi(struct pci_epc *epc, u8 fn,
+static int rockchip_pcie_ep_set_msi(struct pci_epc *epc, u8 fn, u8 vf,
 				    u8 multi_msg_cap)
 {
 	struct rockchip_pcie_ep *ep = epc_get_drvdata(epc);
@@ -328,7 +328,7 @@
 	return 0;
 }
 
-static int rockchip_pcie_ep_get_msi(struct pci_epc *epc, u8 fn)
+static int rockchip_pcie_ep_get_msi(struct pci_epc *epc, u8 fn, u8 vf)
 {
 	struct rockchip_pcie_ep *ep = epc_get_drvdata(epc);
 	struct rockchip_pcie *rockchip = &ep->rockchip;
@@ -470,7 +470,7 @@
 	return 0;
 }
 
-static int rockchip_pcie_ep_raise_irq(struct pci_epc *epc, u8 fn,
+static int rockchip_pcie_ep_raise_irq(struct pci_epc *epc, u8 fn, u8 vf,
 				      enum pci_epc_irq_type type,
 				      u16 interrupt_num)
 {
@@ -499,12 +499,21 @@
 
 	rockchip_pcie_write(rockchip, cfg, PCIE_CORE_PHY_FUNC_CFG);
 
-	list_for_each_entry(epf, &epc->pci_epf, list)
-		pci_epf_linkup(epf);
-
 	return 0;
 }
 
+static const struct pci_epc_features rockchip_pcie_epc_features = {
+	.linkup_notifier = false,
+	.msi_capable = true,
+	.msix_capable = false,
+};
+
+static const struct pci_epc_features*
+rockchip_pcie_ep_get_features(struct pci_epc *epc, u8 func_no, u8 vfunc_no)
+{
+	return &rockchip_pcie_epc_features;
+}
+
 static const struct pci_epc_ops rockchip_pcie_epc_ops = {
 	.write_header	= rockchip_pcie_ep_write_header,
 	.set_bar	= rockchip_pcie_ep_set_bar,
@@ -515,6 +524,7 @@
 	.get_msi	= rockchip_pcie_ep_get_msi,
 	.raise_irq	= rockchip_pcie_ep_raise_irq,
 	.start		= rockchip_pcie_ep_start,
+	.get_features	= rockchip_pcie_ep_get_features,
 };
 
 static int rockchip_pcie_parse_ep_dt(struct rockchip_pcie *rockchip,
diff -Naur --no-dereference a/drivers/pci/controller/pci-j721e.c b/drivers/pci/controller/pci-j721e.c
--- a/drivers/pci/controller/pci-j721e.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/pci/controller/pci-j721e.c	2020-06-19 21:00:31.420227716 -0400
@@ -0,0 +1,437 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * pci-j721e - PCIe controller driver for TI's J721E SoCs
+ *
+ * Copyright (C) 2018-2019 Texas Instruments Incorporated - http://www.ti.com
+ *
+ * Author: Kishon Vijay Abraham I <kishon@ti.com>
+ */
+
+#include <dt-bindings/pci/pci.h>
+#include <linux/io.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/irqdomain.h>
+#include <linux/mfd/syscon.h>
+#include <linux/of_device.h>
+#include <linux/of_irq.h>
+#include <linux/pci.h>
+#include <linux/pm_runtime.h>
+#include <linux/regmap.h>
+
+#include "../pci.h"
+#include "pcie-cadence.h"
+
+#define J721E_PCIE_USER_CMD_STATUS	0x4
+#define LINK_TRAINING_ENABLE		BIT(0)
+
+#define J721E_PCIE_USER_LINKSTATUS	0x14
+#define LINK_STATUS			GENMASK(1, 0)
+enum link_status {
+	NO_RECIEVERS_DETECTED,
+	LINK_TRAINING_IN_PROGRESS,
+	LINK_UP_DL_IN_PROGRESS,
+	LINK_UP_DL_COMPLETED,
+};
+
+#define J721E_TRANS_CTRL(a)		((a) * 0xc)
+#define J721E_TRANS_REQ_ID(a)		(((a) * 0xc) + 0x4)
+#define J721E_TRANS_VIRT_ID(a)		(((a) * 0xc) + 0x8)
+
+#define J721E_REQID_MASK		0xffff
+#define J721E_REQID_SHIFT		16
+
+#define J721E_EN			BIT(0)
+#define J721E_ATYPE_SHIFT		16
+
+#define J721E_MODE_RC			BIT(7)
+#define LANE_COUNT_MASK			BIT(8)
+#define LANE_COUNT(n)			((n) << 8)
+
+#define GENERATION_SEL_MASK		GENMASK(1, 0)
+
+#define MAX_LANES			2
+
+enum j721e_atype {
+	PHYS_ADDR,
+	INT_ADDR,
+	VIRT_ADDR,
+	TRANS_ADDR,
+};
+
+#define to_j721e_pcie(x) container_of((x), struct j721e_pcie, plat_data)
+
+struct j721e_pcie {
+	struct device		*dev;
+	struct device_node	*node;
+	u32			mode;
+	u32			num_lanes;
+	struct cdns_pcie_plat_data plat_data;
+	void __iomem		*intd_cfg_base;
+	void __iomem		*user_cfg_base;
+	void __iomem		*vmap_lp_base;
+	u8			vmap_lp_index;
+	bool			enable_smmu;
+};
+
+static inline u32 j721e_pcie_vmap_readl(struct j721e_pcie *pcie, u32 offset)
+{
+	return readl(pcie->vmap_lp_base + offset);
+}
+
+static inline void j721e_pcie_vmap_writel(struct j721e_pcie *pcie, u32 offset,
+					  u32 value)
+{
+	writel(value, pcie->vmap_lp_base + offset);
+}
+
+static inline u32 j721e_pcie_intd_readl(struct j721e_pcie *pcie, u32 offset)
+{
+	return readl(pcie->intd_cfg_base + offset);
+}
+
+static inline void j721e_pcie_intd_writel(struct j721e_pcie *pcie, u32 offset,
+					  u32 value)
+{
+	writel(value, pcie->intd_cfg_base + offset);
+}
+
+static inline u32 j721e_pcie_user_readl(struct j721e_pcie *pcie, u32 offset)
+{
+	return readl(pcie->user_cfg_base + offset);
+}
+
+static inline void j721e_pcie_user_writel(struct j721e_pcie *pcie, u32 offset,
+					  u32 value)
+{
+	writel(value, pcie->user_cfg_base + offset);
+}
+
+static void j721e_pcie_quirk(struct pci_dev *pci_dev)
+{
+	struct pci_bus *root_bus;
+	struct pci_dev *bridge;
+	struct j721e_pcie *pcie;
+	struct pci_bus *bus;
+	struct device *dev;
+	int index;
+	u32 val;
+
+	static const struct pci_device_id rc_pci_devids[] = {
+		{ PCI_DEVICE(0x104c, 0xb00d),
+		.class = PCI_CLASS_BRIDGE_PCI << 8, .class_mask = ~0, },
+		{ 0, },
+	};
+
+	dev = pci_get_host_bridge_device(pci_dev);
+	if (!(dev && dev->parent && dev->parent->parent))
+		return;
+
+	pcie = dev_get_drvdata(dev->parent->parent);
+	if (!pcie)
+		return;
+
+	bus = pci_dev->bus;
+	index = pcie->vmap_lp_index;
+
+	if (!pcie->enable_smmu)
+		return;
+
+	if (index >= 32)
+		return;
+
+	if (pci_is_root_bus(bus))
+		return;
+
+	root_bus = bus;
+	while (!pci_is_root_bus(root_bus)) {
+		bridge = root_bus->self;
+		root_bus = root_bus->parent;
+	}
+
+	if (pci_match_id(rc_pci_devids, bridge)) {
+		val = J721E_REQID_MASK << J721E_REQID_SHIFT |
+			(bus->number << 8 | pci_dev->devfn);
+		j721e_pcie_vmap_writel(pcie, J721E_TRANS_REQ_ID(index), val);
+		val = VIRT_ADDR << J721E_ATYPE_SHIFT;
+		j721e_pcie_vmap_writel(pcie, J721E_TRANS_VIRT_ID(index), val);
+		j721e_pcie_vmap_writel(pcie, J721E_TRANS_CTRL(index), J721E_EN);
+	}
+
+	pcie->vmap_lp_index++;
+}
+DECLARE_PCI_FIXUP_ENABLE(PCI_ANY_ID, PCI_ANY_ID, j721e_pcie_quirk);
+
+static int j721e_pcie_start_link(struct cdns_pcie_plat_data *data, bool start)
+{
+	struct j721e_pcie *pcie = to_j721e_pcie(data);
+	u32 reg;
+
+	reg = j721e_pcie_user_readl(pcie, J721E_PCIE_USER_CMD_STATUS);
+	if (start)
+		reg |= LINK_TRAINING_ENABLE;
+	else
+		reg &= ~LINK_TRAINING_ENABLE;
+	j721e_pcie_user_writel(pcie, J721E_PCIE_USER_CMD_STATUS, reg);
+
+	return 0;
+}
+
+static bool j721e_pcie_is_link_up(struct cdns_pcie_plat_data *data)
+{
+	struct j721e_pcie *pcie = to_j721e_pcie(data);
+	u32 reg;
+
+	reg = j721e_pcie_user_readl(pcie, J721E_PCIE_USER_LINKSTATUS);
+	reg &= LINK_STATUS;
+	if (reg == LINK_UP_DL_COMPLETED)
+		return true;
+
+	return false;
+}
+
+static int j721e_pcie_set_mode(struct j721e_pcie *pcie, struct regmap *syscon)
+{
+	struct device *dev = pcie->dev;
+	u32 mask = J721E_MODE_RC;
+	u32 mode = pcie->mode;
+	u32 val = 0;
+	int ret = 0;
+
+	if (mode == PCI_MODE_RC)
+		val = J721E_MODE_RC;
+
+	ret = regmap_update_bits(syscon, 0, mask, val);
+	if (ret)
+		dev_err(dev, "failed to set pcie mode\n");
+
+	return ret;
+}
+
+static int j721e_pcie_set_link_speed(struct j721e_pcie *pcie,
+				     struct regmap *syscon)
+{
+	struct device *dev = pcie->dev;
+	struct device_node *np = dev->of_node;
+	int link_speed;
+	u32 val = 0;
+	int ret;
+
+	link_speed = of_pci_get_max_link_speed(np);
+	if (link_speed < 2)
+		link_speed = 2;
+
+	val = link_speed - 1;
+	ret = regmap_update_bits(syscon, 0, GENERATION_SEL_MASK, val);
+	if (ret)
+		dev_err(dev, "failed to set link speed\n");
+
+	return ret;
+}
+
+static int j721e_pcie_set_lane_count(struct j721e_pcie *pcie,
+				     struct regmap *syscon)
+{
+	struct device *dev = pcie->dev;
+	u32 lanes = pcie->num_lanes;
+	u32 val = 0;
+	int ret;
+
+	val = LANE_COUNT(lanes - 1);
+	ret = regmap_update_bits(syscon, 0, LANE_COUNT_MASK, val);
+	if (ret)
+		dev_err(dev, "failed to set link count\n");
+
+	return ret;
+}
+
+static int j721e_pcie_ctrl_init(struct j721e_pcie *pcie)
+{
+	struct device *dev = pcie->dev;
+	struct device_node *node = dev->of_node;
+	struct regmap *syscon;
+	int ret;
+
+	syscon = syscon_regmap_lookup_by_phandle(node, "ti,syscon-pcie-ctrl");
+	if (IS_ERR(syscon)) {
+		dev_err(dev, "Unable to get ti,syscon-pcie-ctrl regmap\n");
+		return PTR_ERR(syscon);
+	}
+
+	ret = j721e_pcie_set_mode(pcie, syscon);
+	if (ret < 0) {
+		dev_err(dev, "Failed to set pci mode\n");
+		return ret;
+	}
+
+	ret = j721e_pcie_set_link_speed(pcie, syscon);
+	if (ret < 0) {
+		dev_err(dev, "Failed to set link speed\n");
+		return ret;
+	}
+
+	ret = j721e_pcie_set_lane_count(pcie, syscon);
+	if (ret < 0) {
+		dev_err(dev, "Failed to set num-lanes\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static const struct of_device_id of_j721e_pcie_match[] = {
+	{
+		.compatible = "ti,j721e-pcie",
+	},
+	{},
+};
+
+static int j721e_pcie_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *node = dev->of_node;
+	struct cdns_pcie_plat_data *plat_data;
+	struct platform_device *platform_dev;
+	struct device_node *child_node;
+	struct j721e_pcie *pcie;
+	struct resource *res;
+	void __iomem *base;
+	u32 num_lanes;
+	u32 mode;
+	int ret;
+
+	pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
+	if (!pcie)
+		return -ENOMEM;
+
+	pcie->dev = dev;
+	pcie->node = node;
+	plat_data = &pcie->plat_data;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "intd_cfg");
+	base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+	pcie->intd_cfg_base = base;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "user_cfg");
+	base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+	pcie->user_cfg_base = base;
+
+	plat_data->start_link = j721e_pcie_start_link;
+	plat_data->is_link_up = j721e_pcie_is_link_up;
+
+	ret = of_property_read_u32(node, "pci-mode", &mode);
+	if (ret < 0) {
+		dev_err(dev, "Failed to get pci-mode binding\n");
+		return ret;
+	}
+	pcie->mode = mode;
+
+	ret = of_property_read_u32(node, "num-lanes", &num_lanes);
+	if (ret || num_lanes > MAX_LANES)
+		num_lanes = 1;
+	pcie->num_lanes = num_lanes;
+
+	dev_set_drvdata(dev, pcie);
+	pm_runtime_enable(dev);
+	ret = pm_runtime_get_sync(dev);
+	if (ret < 0) {
+		dev_err(dev, "pm_runtime_get_sync failed\n");
+		goto err_get_sync;
+	}
+
+	ret = j721e_pcie_ctrl_init(pcie);
+	if (ret < 0) {
+		dev_err(dev, "pm_runtime_get_sync failed\n");
+		goto err_get_sync;
+	}
+
+	switch (mode) {
+	case PCI_MODE_RC:
+		if (!IS_ENABLED(CONFIG_PCIE_CADENCE_HOST)) {
+			ret = -ENODEV;
+			goto err_get_sync;
+		}
+
+		res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+						   "vmap");
+		base = devm_ioremap_resource(dev, res);
+		if (IS_ERR(base))
+			goto err_get_sync;
+		pcie->vmap_lp_base = base;
+
+		child_node = of_get_child_by_name(node, "pcie");
+		if (!child_node) {
+			dev_WARN(dev, "pcie-rc node is absent\n");
+			goto err_get_sync;
+		}
+
+		if (of_property_read_bool(child_node, "iommu-map"))
+			pcie->enable_smmu = true;
+
+		platform_dev = of_platform_device_create_pdata(child_node, NULL,
+							       plat_data, dev);
+		if (!platform_dev) {
+			ret = -ENODEV;
+			dev_err(dev, "Failed to create Cadence RC device\n");
+			goto err_get_sync;
+		}
+
+		break;
+	case PCI_MODE_EP:
+		if (!IS_ENABLED(CONFIG_PCIE_CADENCE_EP)) {
+			ret = -ENODEV;
+			goto err_get_sync;
+		}
+
+		child_node = of_get_child_by_name(node, "pcie-ep");
+		if (!child_node) {
+			dev_WARN(dev, "pcie-ep node is absent\n");
+			goto err_get_sync;
+		}
+
+		platform_dev = of_platform_device_create_pdata(child_node, NULL,
+							       plat_data, dev);
+		if (!platform_dev) {
+			ret = -ENODEV;
+			dev_err(dev, "Failed to create Cadence EP device\n");
+			goto err_get_sync;
+		}
+
+		break;
+	default:
+		dev_err(dev, "INVALID device type %d\n", mode);
+	}
+
+	return 0;
+
+err_get_sync:
+	pm_runtime_put(dev);
+	pm_runtime_disable(dev);
+
+	return ret;
+}
+
+static int j721e_pcie_remove(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+
+	pm_runtime_put(dev);
+	pm_runtime_disable(dev);
+	of_platform_depopulate(dev);
+
+	return 0;
+}
+
+static struct platform_driver j721e_pcie_driver = {
+	.probe  = j721e_pcie_probe,
+	.remove = j721e_pcie_remove,
+	.driver = {
+		.name	= "j721e-pcie",
+		.of_match_table = of_j721e_pcie_match,
+		.suppress_bind_attrs = true,
+	},
+};
+builtin_platform_driver(j721e_pcie_driver);
diff -Naur --no-dereference a/drivers/pci/endpoint/functions/Kconfig b/drivers/pci/endpoint/functions/Kconfig
--- a/drivers/pci/endpoint/functions/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/endpoint/functions/Kconfig	2020-06-19 21:00:31.420227716 -0400
@@ -12,3 +12,15 @@
 	   for PCI Endpoint.
 
 	   If in doubt, say "N" to disable Endpoint test driver.
+
+config PCI_EPF_NTB
+	tristate "PCI Endpoint NTB driver"
+	depends on PCI_ENDPOINT
+	help
+	   Select this configuration option to enable the NTB driver
+	   for PCI Endpoint. NTB driver implements NTB controller
+	   functionality using multiple PCIe endpoint instances. It
+	   can support NTB endpoint function devices created using
+	   device tree.
+
+	   If in doubt, say "N" to disable Endpoint NTB driver.
diff -Naur --no-dereference a/drivers/pci/endpoint/functions/Makefile b/drivers/pci/endpoint/functions/Makefile
--- a/drivers/pci/endpoint/functions/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/endpoint/functions/Makefile	2020-06-19 21:00:31.420227716 -0400
@@ -4,3 +4,4 @@
 #
 
 obj-$(CONFIG_PCI_EPF_TEST)		+= pci-epf-test.o
+obj-$(CONFIG_PCI_EPF_NTB)		+= pci-epf-ntb.o
diff -Naur --no-dereference a/drivers/pci/endpoint/functions/pci-epf-ntb.c b/drivers/pci/endpoint/functions/pci-epf-ntb.c
--- a/drivers/pci/endpoint/functions/pci-epf-ntb.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/pci/endpoint/functions/pci-epf-ntb.c	2020-06-19 21:00:31.420227716 -0400
@@ -0,0 +1,1176 @@
+// SPDX-License-Identifier: GPL-2.0
+/**
+ * Endpoint Function Driver to implement Non-Transparent Bridge functionality
+ *
+ * Copyright (C) 2019 Texas Instruments
+ * Author: Kishon Vijay Abraham I <kishon@ti.com>
+ */
+
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+
+#include <linux/pci-epc.h>
+#include <linux/pci-epf.h>
+
+static struct workqueue_struct *kpcintb_workqueue;
+
+#define COMMAND_CONFIGURE_DOORBELL	1
+#define COMMAND_CONFIGURE_MW		2
+#define COMMAND_LINK_UP			3
+
+#define COMMAND_STATUS_OK		BIT(0)
+#define COMMAND_STATUS_ERROR		BIT(1)
+#define LINK_STATUS_UP			BIT(2)
+
+#define SPAD_COUNT			64
+#define DB_COUNT			4
+#define NTB_MW_OFFSET			2
+#define DB_COUNT_MASK			GENMASK(15, 0)
+#define MSIX_ENABLE			BIT(16)
+#define MAX_DB_COUNT			32
+#define MAX_MW				4
+
+enum epf_ntb_bar {
+	BAR_CONFIG,
+	BAR_PEER_SPAD,
+	BAR_DB_MW1,
+	BAR_MW2,
+	BAR_MW3,
+	BAR_MW4,
+};
+
+struct epf_ntb {
+	u32 num_mws;
+	u32 *mws_size;
+	u32 db_count;
+	u32 spad_count;
+	struct pci_epf *epf;
+	struct epf_ntb_epc *epc[2];
+};
+
+struct epf_ntb_epc {
+	u8 func_no;
+	u8 vfunc_no;
+	bool linkup;
+	u32 spad_size;
+	struct pci_epc *epc;
+	struct epf_ntb *epf_ntb;
+	void __iomem *mw_addr[6];
+	struct epf_ntb_ctrl *reg;
+	struct pci_epf_bar *epf_bar;
+	enum pci_barno epf_ntb_bar[6];
+	struct delayed_work cmd_handler;
+	enum pci_epc_interface_type type;
+	const struct pci_epc_features *epc_features;
+};
+
+struct epf_ntb_ctrl {
+	u32	command;
+	u32	argument;
+	u32	status;
+	u32	topology;
+	u64	addr;
+	u32	size;
+	u32	mw1_offset;
+	u32	num_mws;
+	u32	spad_offset;
+	u32	spad_count;
+	u32	db_entry_size;
+	u32	db_data[MAX_DB_COUNT];
+} __packed;
+
+static struct pci_epf_header epf_ntb_header = {
+	.vendorid	= PCI_ANY_ID,
+	.deviceid	= PCI_ANY_ID,
+	.baseclass_code	= PCI_BASE_CLASS_MEMORY,
+	.interrupt_pin	= PCI_INTERRUPT_INTA,
+};
+
+static int epf_ntb_link_up(struct epf_ntb *ntb)
+{
+	enum pci_epc_interface_type type;
+	struct epf_ntb_epc *ntb_epc;
+	struct epf_ntb_ctrl *ctrl;
+	u8 vfunc_no, func_no;
+	int ret;
+
+	for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {
+		ntb_epc = ntb->epc[type];
+		func_no = ntb_epc->func_no;
+		vfunc_no = ntb_epc->vfunc_no;
+		ctrl = ntb_epc->reg;
+		ctrl->status |= LINK_STATUS_UP;
+		ret = pci_epc_raise_irq(ntb_epc->epc, func_no, vfunc_no,
+					PCI_EPC_IRQ_MSI, 1);
+		if (ret < 0) {
+			WARN(1, "%s intf: Failed to raise Link Up IRQ\n",
+			     pci_epc_interface_string(type));
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int
+epf_ntb_configure_mw(struct epf_ntb *ntb, enum pci_epc_interface_type type,
+		     u32 mw)
+{
+	struct epf_ntb_epc *peer_ntb_epc;
+	struct pci_epf_bar *peer_epf_bar;
+	struct epf_ntb_epc *ntb_epc;
+	enum pci_barno peer_barno;
+	struct epf_ntb_ctrl *ctrl;
+	phys_addr_t phys_addr;
+	u8 vfunc_no, func_no;
+	struct pci_epc *epc;
+	u64 addr;
+	u32 size;
+	int ret;
+
+	ntb_epc = ntb->epc[type];
+	epc = ntb_epc->epc;
+
+	peer_ntb_epc = ntb->epc[!type];
+	peer_barno = peer_ntb_epc->epf_ntb_bar[mw + NTB_MW_OFFSET];
+	peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];
+
+	phys_addr = peer_epf_bar->phys_addr;
+	ctrl = ntb_epc->reg;
+	addr = ctrl->addr;
+	size = ctrl->size;
+	if (mw + NTB_MW_OFFSET == BAR_DB_MW1)
+		phys_addr += ctrl->mw1_offset;
+
+	func_no = ntb_epc->func_no;
+	vfunc_no = ntb_epc->vfunc_no;
+
+	ret = pci_epc_map_addr(epc, func_no, vfunc_no, phys_addr, addr, size);
+	WARN(ret < 0, "%s intf: Failed to map memory window %d address\n",
+	     pci_epc_interface_string(type), mw);
+
+	return ret;
+}
+
+static int
+epf_ntb_configure_db(struct epf_ntb *ntb, enum pci_epc_interface_type type,
+		     u16 db_count, bool msix)
+{
+	struct epf_ntb_epc *peer_ntb_epc;
+	struct pci_epf_bar *peer_epf_bar;
+	struct epf_ntb_ctrl *peer_ctrl;
+	struct epf_ntb_epc *ntb_epc;
+	enum pci_barno peer_barno;
+	phys_addr_t phys_addr;
+	u8 vfunc_no, func_no;
+	struct pci_epc *epc;
+	u32 db_entry_size;
+	u32 db_data;
+	int ret, i;
+
+	if (db_count > MAX_DB_COUNT)
+		return -EINVAL;
+
+	ntb_epc = ntb->epc[type];
+	epc = ntb_epc->epc;
+
+	peer_ntb_epc = ntb->epc[!type];
+	peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1];
+	peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];
+	peer_ctrl = peer_ntb_epc->reg;
+	db_entry_size = peer_ctrl->db_entry_size;
+
+	phys_addr = peer_epf_bar->phys_addr;
+	func_no = ntb_epc->func_no;
+	vfunc_no = ntb_epc->vfunc_no;
+
+	ret = pci_epc_map_msi_irq(epc, func_no, vfunc_no, phys_addr, db_count,
+				  db_entry_size, &db_data);
+	if (ret < 0) {
+		WARN(1, "%s intf: Failed to map MSI IRQ\n",
+		     pci_epc_interface_string(type));
+		return ret;
+	}
+
+	for (i = 0; i < db_count; i++)
+		peer_ctrl->db_data[i] = db_data | i;
+
+	return 0;
+}
+
+static void epf_ntb_cmd_handler(struct work_struct *work)
+{
+	enum pci_epc_interface_type type;
+	struct epf_ntb_epc *ntb_epc;
+	struct epf_ntb_ctrl *ctrl;
+	u32 command, argument;
+	struct epf_ntb *ntb;
+	struct device *dev;
+	u16 db_count;
+	bool is_msix;
+	int ret;
+
+	ntb_epc = container_of(work, struct epf_ntb_epc, cmd_handler.work);
+	ctrl = ntb_epc->reg;
+	command = ctrl->command;
+	if (!command)
+		goto reset_handler;
+	argument = ctrl->argument;
+
+	ctrl->command = 0;
+	ctrl->argument = 0;
+
+	ctrl = ntb_epc->reg;
+	type = ntb_epc->type;
+	ntb = ntb_epc->epf_ntb;
+	dev = &ntb->epf->dev;
+
+	switch (command) {
+	case COMMAND_CONFIGURE_DOORBELL:
+		db_count = argument & DB_COUNT_MASK;
+		is_msix = argument & MSIX_ENABLE;
+		ret = epf_ntb_configure_db(ntb, type, db_count, is_msix);
+		if (ret < 0)
+			ctrl->status |= COMMAND_STATUS_ERROR;
+		else
+			ctrl->status |= COMMAND_STATUS_OK;
+		break;
+	case COMMAND_CONFIGURE_MW:
+		ret = epf_ntb_configure_mw(ntb, type, argument);
+		if (ret < 0)
+			ctrl->status |= COMMAND_STATUS_ERROR;
+		else
+			ctrl->status |= COMMAND_STATUS_OK;
+		break;
+	case COMMAND_LINK_UP:
+		ntb_epc->linkup = true;
+		if (ntb->epc[PRIMARY_INTERFACE]->linkup &&
+		    ntb->epc[SECONDARY_INTERFACE]->linkup) {
+			ret = epf_ntb_link_up(ntb);
+			if (ret < 0)
+				ctrl->status |= COMMAND_STATUS_ERROR;
+			else
+				ctrl->status |= COMMAND_STATUS_OK;
+			goto reset_handler;
+		}
+		ctrl->status |= COMMAND_STATUS_OK;
+		break;
+	default:
+		dev_err(dev, "UNKNOWN command: %d\n", command);
+		break;
+	}
+
+reset_handler:
+	queue_delayed_work(kpcintb_workqueue, &ntb_epc->cmd_handler,
+			   msecs_to_jiffies(5));
+}
+
+static void epf_ntb_peer_spad_bar_clear(struct epf_ntb_epc *ntb_epc)
+{
+	struct pci_epf_bar *epf_bar;
+	enum pci_barno barno;
+	u8 vfunc_no, func_no;
+	struct pci_epc *epc;
+
+	epc = ntb_epc->epc;
+	func_no = ntb_epc->func_no;
+	vfunc_no = ntb_epc->vfunc_no;
+	barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD];
+	epf_bar = &ntb_epc->epf_bar[barno];
+	pci_epc_clear_bar(epc, func_no, vfunc_no, epf_bar);
+}
+
+static int
+epf_ntb_peer_spad_bar_set(struct epf_ntb *ntb, enum pci_epc_interface_type type)
+{
+	struct epf_ntb_epc *peer_ntb_epc;
+	struct pci_epf_bar *peer_epf_bar;
+	struct epf_ntb_epc *ntb_epc;
+	struct pci_epf_bar *epf_bar;
+	enum pci_barno peer_barno;
+	u32 peer_spad_offset;
+	enum pci_barno barno;
+	u8 vfunc_no, func_no;
+	struct pci_epc *epc;
+	struct device *dev;
+	int ret;
+
+	dev = &ntb->epf->dev;
+
+	peer_ntb_epc = ntb->epc[!type];
+	peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_CONFIG];
+	peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];
+
+	ntb_epc = ntb->epc[type];
+	barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD];
+	epf_bar = &ntb_epc->epf_bar[barno];
+	func_no = ntb_epc->func_no;
+	vfunc_no = ntb_epc->vfunc_no;
+	epc = ntb_epc->epc;
+
+	peer_spad_offset = peer_ntb_epc->reg->spad_offset;
+	epf_bar->phys_addr = peer_epf_bar->phys_addr + peer_spad_offset;
+	epf_bar->size = peer_ntb_epc->spad_size;
+	epf_bar->barno = barno;
+	epf_bar->flags = PCI_BASE_ADDRESS_MEM_TYPE_32;
+
+	ret = pci_epc_set_bar(ntb_epc->epc, func_no, vfunc_no, epf_bar);
+	if (ret) {
+		dev_err(dev, "%s intf: peer SPAD BAR set failed\n",
+			pci_epc_interface_string(type));
+		return ret;
+	}
+
+	return 0;
+}
+
+static void epf_ntb_config_sspad_bar_clear(struct epf_ntb_epc *ntb_epc)
+{
+	struct pci_epf_bar *epf_bar;
+	u8 vfunc_no, func_no;
+	enum pci_barno barno;
+	struct pci_epc *epc;
+
+	epc = ntb_epc->epc;
+	func_no = ntb_epc->func_no;
+	vfunc_no = ntb_epc->vfunc_no;
+	barno = ntb_epc->epf_ntb_bar[BAR_CONFIG];
+	epf_bar = &ntb_epc->epf_bar[barno];
+	pci_epc_clear_bar(epc, func_no, vfunc_no, epf_bar);
+}
+
+static int epf_ntb_config_sspad_bar_set(struct epf_ntb_epc *ntb_epc)
+{
+	struct pci_epf_bar *epf_bar;
+	enum pci_barno barno;
+	u8 vfunc_no, func_no;
+	struct epf_ntb *ntb;
+	struct pci_epc *epc;
+	struct device *dev;
+	int ret;
+
+	ntb = ntb_epc->epf_ntb;
+	dev = &ntb->epf->dev;
+
+	epc = ntb_epc->epc;
+	func_no = ntb_epc->func_no;
+	vfunc_no = ntb_epc->vfunc_no;
+	barno = ntb_epc->epf_ntb_bar[BAR_CONFIG];
+	epf_bar = &ntb_epc->epf_bar[barno];
+
+	ret = pci_epc_set_bar(epc, func_no, vfunc_no, epf_bar);
+	if (ret) {
+		dev_err(dev, "%s inft: Config/Status/SPAD BAR set failed\n",
+			pci_epc_interface_string(ntb_epc->type));
+		return ret;
+	}
+
+	return 0;
+}
+
+static void epf_ntb_config_spad_bar_free(struct epf_ntb *ntb)
+{
+	enum pci_epc_interface_type type;
+	struct epf_ntb_epc *ntb_epc;
+	enum pci_barno barno;
+	struct pci_epf *epf;
+
+	epf = ntb->epf;
+	for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {
+		ntb_epc = ntb->epc[type];
+		barno = ntb_epc->epf_ntb_bar[BAR_CONFIG];
+		if (ntb_epc->reg)
+			pci_epf_free_space(epf, ntb_epc->reg, barno, type);
+	}
+}
+
+static int
+epf_ntb_config_spad_bar_alloc_interface(struct epf_ntb *ntb,
+					enum pci_epc_interface_type type)
+{
+	const struct pci_epc_features *peer_epc_features;
+	const struct pci_epc_features *epc_features;
+	struct epf_ntb_epc *peer_ntb_epc;
+	struct epf_ntb_epc *ntb_epc;
+	struct epf_ntb_ctrl *ctrl;
+	enum pci_barno peer_barno;
+	struct device_node *node;
+	u32 spad_size, ctrl_size;
+	enum pci_barno barno;
+	u64 size, peer_size;
+	struct pci_epc *epc;
+	struct pci_epf *epf;
+	struct device *dev;
+	u32 spad_count;
+	size_t align;
+	void *base;
+
+	epf = ntb->epf;
+	node = epf->node;
+	dev = &epf->dev;
+	ntb_epc = ntb->epc[type];
+	epc = ntb_epc->epc;
+
+	epc_features = ntb_epc->epc_features;
+	barno = ntb_epc->epf_ntb_bar[BAR_CONFIG];
+	size = epc_features->bar_fixed_size[barno];
+	align = epc_features->align;
+
+	peer_ntb_epc = ntb->epc[!type];
+	peer_epc_features = peer_ntb_epc->epc_features;
+	peer_barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD];
+	peer_size = peer_epc_features->bar_fixed_size[barno];
+
+	/* Check if epc_features is populated incorrectly */
+	if ((!IS_ALIGNED(size, align)))
+		return -EINVAL;
+
+	spad_count = SPAD_COUNT;
+	of_property_read_u32(node, "spad-count", &spad_count);
+
+	ctrl_size = sizeof(struct epf_ntb_ctrl);
+	spad_size = spad_count * 4;
+
+	if (!align) {
+		ctrl_size = roundup_pow_of_two(ctrl_size);
+		spad_size = roundup_pow_of_two(spad_size);
+	} else {
+		ctrl_size = ALIGN(ctrl_size, align);
+		spad_size = ALIGN(spad_size, align);
+	}
+
+	if (peer_size) {
+		if (peer_size < spad_size)
+			spad_count = peer_size / 4;
+		spad_size = peer_size;
+	}
+
+	/*
+	 * In order to make sure SPAD offset is aligned to its size,
+	 * expand control region size to the size of SPAD if SPAD size
+	 * is greater than control region size.
+	 */
+	if (spad_size > ctrl_size)
+		ctrl_size = spad_size;
+
+	if (!size)
+		size = ctrl_size + spad_size;
+	else if (size < ctrl_size + spad_size)
+		return -EINVAL;
+
+	base = pci_epf_alloc_space(epf, size, barno, align, type);
+	if (!base) {
+		dev_err(dev, "%s intf: Config/Status/SPAD alloc region fail\n",
+			pci_epc_interface_string(type));
+		return -ENOMEM;
+	}
+
+	ntb_epc->reg = base;
+
+	ctrl = ntb_epc->reg;
+	ctrl->spad_offset = ctrl_size;
+	ctrl->spad_count = spad_count;
+	ctrl->num_mws = ntb->num_mws;
+	ctrl->db_entry_size = align ? align : 4;
+	ntb_epc->spad_size = spad_size;
+
+	return 0;
+}
+
+static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb)
+{
+	enum pci_epc_interface_type type;
+	struct device *dev;
+	int ret;
+
+	dev = &ntb->epf->dev;
+
+	for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {
+		ret = epf_ntb_config_spad_bar_alloc_interface(ntb, type);
+		if (ret) {
+			dev_err(dev, "%s intf: Config/SPAD BAR alloc failed\n",
+				pci_epc_interface_string(type));
+			goto err_config_spad_bar_alloc;
+		}
+	}
+
+	return 0;
+
+err_config_spad_bar_alloc:
+	epf_ntb_config_spad_bar_free(ntb);
+
+	return ret;
+}
+
+static void epf_ntb_free_peer_mem(struct epf_ntb_epc *ntb_epc)
+{
+	struct pci_epf_bar *epf_bar;
+	void __iomem *mw_addr;
+	phys_addr_t phys_addr;
+	enum epf_ntb_bar bar;
+	enum pci_barno barno;
+	struct pci_epc *epc;
+	size_t size;
+
+	epc = ntb_epc->epc;
+
+	for (bar = BAR_DB_MW1; bar < BAR_MW4; bar++) {
+		barno = ntb_epc->epf_ntb_bar[bar];
+		mw_addr = ntb_epc->mw_addr[barno];
+		epf_bar = &ntb_epc->epf_bar[barno];
+		phys_addr = epf_bar->phys_addr;
+		size = epf_bar->size;
+		if (mw_addr) {
+			pci_epc_mem_free_addr(epc, phys_addr, mw_addr, size);
+			ntb_epc->mw_addr[barno] = NULL;
+		}
+	}
+}
+
+static void epf_ntb_db_mw_bar_clear(struct epf_ntb_epc *ntb_epc)
+{
+	struct pci_epf_bar *epf_bar;
+	enum epf_ntb_bar bar;
+	enum pci_barno barno;
+	u8 vfunc_no, func_no;
+	struct pci_epc *epc;
+
+	epc = ntb_epc->epc;
+
+	func_no = ntb_epc->func_no;
+	vfunc_no = ntb_epc->vfunc_no;
+
+	for (bar = BAR_DB_MW1; bar < BAR_MW4; bar++) {
+		barno = ntb_epc->epf_ntb_bar[bar];
+		epf_bar = &ntb_epc->epf_bar[barno];
+		pci_epc_clear_bar(epc, func_no, vfunc_no, epf_bar);
+	}
+}
+
+static void epf_ntb_db_mw_bar_cleanup(struct epf_ntb_epc *ntb_epc,
+				      struct epf_ntb_epc *peer_ntb_epc)
+{
+	epf_ntb_db_mw_bar_clear(ntb_epc);
+	epf_ntb_free_peer_mem(peer_ntb_epc);
+}
+
+static int
+epf_ntb_alloc_peer_mem(struct device *dev, struct epf_ntb_epc *ntb_epc,
+		       enum epf_ntb_bar bar, struct epf_ntb_epc *peer_ntb_epc,
+		       size_t size)
+{
+	const struct pci_epc_features *epc_features;
+	struct pci_epf_bar *epf_bar;
+	struct pci_epc *peer_epc;
+	phys_addr_t phys_addr;
+	void __iomem *mw_addr;
+	enum pci_barno barno;
+	size_t align;
+
+	epc_features = ntb_epc->epc_features;
+	align = epc_features->align;
+
+	if (size < 128)
+		size = 128;
+
+	if (align)
+		size = ALIGN(size, align);
+	else
+		size = roundup_pow_of_two(size);
+
+	peer_epc = peer_ntb_epc->epc;
+	mw_addr = pci_epc_mem_alloc_addr(peer_epc, &phys_addr, size);
+	if (!mw_addr) {
+		dev_err(dev, "%s intf: Failed to allocate OB address\n",
+			pci_epc_interface_string(peer_ntb_epc->type));
+		return -ENOMEM;
+	}
+
+	barno = ntb_epc->epf_ntb_bar[bar];
+	epf_bar = &ntb_epc->epf_bar[barno];
+	ntb_epc->mw_addr[barno] = mw_addr;
+
+	epf_bar->phys_addr = phys_addr;
+	epf_bar->size = size;
+	epf_bar->barno = barno;
+	epf_bar->flags = PCI_BASE_ADDRESS_MEM_TYPE_32;
+
+	return 0;
+}
+
+static int epf_ntb_configure_interrupt(struct epf_ntb *ntb,
+				       enum pci_epc_interface_type type)
+{
+	const struct pci_epc_features *epc_features;
+	bool msix_capable, msi_capable;
+	struct epf_ntb_epc *ntb_epc;
+	struct device_node *node;
+	u8 vfunc_no, func_no;
+	struct pci_epc *epc;
+	struct device *dev;
+	u32 db_count;
+	int ret;
+
+	ntb_epc = ntb->epc[type];
+	dev = &ntb->epf->dev;
+	node = ntb->epf->node;
+
+	epc_features = ntb_epc->epc_features;
+	msix_capable = epc_features->msix_capable;
+	msi_capable = epc_features->msi_capable;
+
+	if (!(msix_capable || msi_capable)) {
+		dev_err(dev, "MSI or MSI-X is required for doorbell\n");
+		return -EINVAL;
+	}
+
+	func_no = ntb_epc->func_no;
+	vfunc_no = ntb_epc->vfunc_no;
+
+	db_count = DB_COUNT;
+	of_property_read_u32(node, "db-count", &db_count);
+	if (db_count > MAX_DB_COUNT) {
+		dev_err(dev, "DB count cannot be more than %d\n", MAX_DB_COUNT);
+		return -EINVAL;
+	}
+
+	ntb->db_count = db_count;
+	epc = ntb_epc->epc;
+
+	if (msi_capable) {
+		ret = pci_epc_set_msi(epc, func_no, vfunc_no, db_count);
+		if (ret) {
+			dev_err(dev, "%s intf: MSI configuration failed\n",
+				pci_epc_interface_string(type));
+			return ret;
+		}
+	}
+
+	if (msix_capable) {
+		ret = pci_epc_set_msix(epc, func_no, vfunc_no, db_count, 0, 0);
+		if (ret) {
+			dev_err(dev, "MSI configuration failed\n");
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int epf_ntb_db_mw_bar_init(struct epf_ntb *ntb,
+				  enum pci_epc_interface_type type)
+{
+	const struct pci_epc_features *epc_features;
+	struct epf_ntb_epc *peer_ntb_epc;
+	struct epf_ntb_epc *ntb_epc;
+	struct pci_epf_bar *epf_bar;
+	struct epf_ntb_ctrl *ctrl;
+	enum epf_ntb_bar bar;
+	u8 vfunc_no, func_no;
+	enum pci_barno barno;
+	struct pci_epc *epc;
+	struct device *dev;
+	u32 num_mws, size;
+	u32 db_count;
+	size_t align;
+	int ret;
+	int i;
+
+	ntb_epc = ntb->epc[type];
+	peer_ntb_epc = ntb->epc[!type];
+
+	dev = &ntb->epf->dev;
+	epc_features = ntb_epc->epc_features;
+	align = epc_features->align;
+	func_no = ntb_epc->func_no;
+	vfunc_no = ntb_epc->vfunc_no;
+	epc = ntb_epc->epc;
+	num_mws = ntb->num_mws;
+	db_count = ntb->db_count;
+
+	for (bar = BAR_DB_MW1, i = 0; i < num_mws; bar++, i++) {
+		if (bar == BAR_DB_MW1) {
+			align = align ? align : 4;
+			size = db_count * align;
+			size = ALIGN(size, ntb->mws_size[i]);
+			ctrl = ntb_epc->reg;
+			ctrl->mw1_offset = size;
+			size += ntb->mws_size[i];
+		} else {
+			size = ntb->mws_size[i];
+		}
+
+		ret = epf_ntb_alloc_peer_mem(dev, ntb_epc, bar,
+					     peer_ntb_epc, size);
+		if (ret)
+			goto err_alloc_peer_mem;
+
+		barno = ntb_epc->epf_ntb_bar[bar];
+		epf_bar = &ntb_epc->epf_bar[barno];
+
+		ret = pci_epc_set_bar(epc, func_no, vfunc_no, epf_bar);
+		if (ret) {
+			dev_err(dev, "%s intf: DoorBell BAR set failed\n",
+				pci_epc_interface_string(type));
+			goto err_alloc_peer_mem;
+		}
+	}
+
+	return 0;
+
+err_alloc_peer_mem:
+	epf_ntb_db_mw_bar_cleanup(ntb_epc, peer_ntb_epc);
+
+	return ret;
+}
+
+static void epf_ntb_epc_destroy(struct epf_ntb *ntb)
+{
+	enum pci_epc_interface_type type;
+	struct epf_ntb_epc *ntb_epc;
+	struct pci_epc *epc;
+	struct pci_epf *epf;
+
+	epf = ntb->epf;
+	for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {
+		ntb_epc = ntb->epc[type];
+		if (!ntb_epc)
+			return;
+		epc = ntb_epc->epc;
+		pci_epc_remove_epf(epc, epf, type);
+		pci_epc_put(epc);
+	}
+}
+
+static int
+epf_ntb_epc_create_interface(struct epf_ntb *ntb, struct pci_epc *epc,
+			     enum pci_epc_interface_type type)
+{
+	const struct pci_epc_features *epc_features;
+	struct pci_epf_bar *epf_bar;
+	struct epf_ntb_epc *ntb_epc;
+	u8 vfunc_no, func_no;
+	struct pci_epf *epf;
+	struct device *dev;
+
+	dev = &ntb->epf->dev;
+
+	ntb_epc = devm_kzalloc(dev, sizeof(*ntb_epc), GFP_KERNEL);
+	if (!ntb_epc)
+		return -ENOMEM;
+
+	epf = ntb->epf;
+	if (type == PRIMARY_INTERFACE) {
+		func_no = epf->func_no;
+		vfunc_no = epf->vfunc_no;
+		epf_bar = epf->bar;
+	} else {
+		func_no = epf->sec_epc_func_no;
+		vfunc_no = epf->sec_epc_vfunc_no;
+		epf_bar = epf->sec_epc_bar;
+	}
+
+	ntb_epc->linkup = false;
+	ntb_epc->epc = epc;
+	ntb_epc->func_no = func_no;
+	ntb_epc->vfunc_no = vfunc_no;
+	ntb_epc->type = type;
+	ntb_epc->epf_bar = epf_bar;
+	ntb_epc->epf_ntb = ntb;
+
+	epc_features = pci_epc_get_features(epc, func_no, vfunc_no);
+	ntb_epc->epc_features = epc_features;
+
+	ntb->epc[type] = ntb_epc;
+
+	return 0;
+}
+
+static int epf_ntb_epc_create(struct epf_ntb *ntb)
+{
+	enum pci_epc_interface_type type;
+	struct device_node *node;
+	const char *epc_name;
+	struct pci_epc *epc;
+	struct pci_epf *epf;
+	struct device *dev;
+	int ret;
+
+	epf = ntb->epf;
+	node = epf->node;
+	dev = &epf->dev;
+
+	for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {
+		epc_name = pci_epc_interface_string(type);
+
+		epc = of_pci_epc_get_by_name(node, epc_name);
+		if (IS_ERR(epc)) {
+			if (PTR_ERR(epc) != -EPROBE_DEFER)
+				dev_err(dev, "%s intf: Failed to get EPC\n",
+					epc_name);
+			ret = PTR_ERR(epc);
+			goto err_epc_get;
+		}
+
+		ret = pci_epc_add_epf(epc, epf, type);
+		if (ret) {
+			dev_err(dev, "%s intf: Fail to add EPF to EPC\n",
+				epc_name);
+			goto err_epc_get;
+		}
+
+		ret = epf_ntb_epc_create_interface(ntb, epc, type);
+		if (ret) {
+			dev_err(dev, "%s intf: Fail to create NTB EPC\n",
+				epc_name);
+			goto err_epc_get;
+		}
+	}
+
+	return 0;
+
+err_epc_get:
+	epf_ntb_epc_destroy(ntb);
+
+	return ret;
+}
+
+static int epf_ntb_init_epc_bar_interface(struct epf_ntb *ntb,
+					  enum pci_epc_interface_type type)
+{
+	const struct pci_epc_features *epc_features;
+	struct epf_ntb_epc *ntb_epc;
+	enum pci_barno barno;
+	enum epf_ntb_bar bar;
+	struct device *dev;
+	u32 num_mws;
+	int i;
+
+	barno = BAR_0;
+	ntb_epc = ntb->epc[type];
+	num_mws = ntb->num_mws;
+	dev = &ntb->epf->dev;
+	epc_features = ntb_epc->epc_features;
+
+	/* These are required BARs which are mandatory for NTB functionality */
+	for (bar = BAR_CONFIG; bar <= BAR_DB_MW1; bar++, barno++) {
+		barno = pci_epc_get_next_free_bar(epc_features, barno);
+		if (barno < 0) {
+			dev_err(dev, "%s intf: Fail to get NTB function BAR\n",
+				pci_epc_interface_string(type));
+			return barno;
+		}
+		ntb_epc->epf_ntb_bar[bar] = barno;
+	}
+
+	/* These are optional BARs which doesn't impact NTB functionality */
+	for (bar = BAR_MW2, i = 1; i < num_mws; bar++, barno++, i++) {
+		barno = pci_epc_get_next_free_bar(epc_features, barno);
+		if (barno < 0) {
+			ntb->num_mws = i;
+			dev_dbg(dev, "BAR not available for > MW%d\n", i + 1);
+		}
+		ntb_epc->epf_ntb_bar[bar] = barno;
+	}
+
+	return 0;
+}
+
+static int epf_ntb_init_epc_bar(struct epf_ntb *ntb)
+{
+	enum pci_epc_interface_type type;
+	struct device *dev;
+	int ret;
+
+	dev = &ntb->epf->dev;
+	for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {
+		ret = epf_ntb_init_epc_bar_interface(ntb, type);
+		if (ret) {
+			dev_err(dev, "Fail to init EPC bar for %s interface\n",
+				pci_epc_interface_string(type));
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int epf_ntb_epc_init_interface(struct epf_ntb *ntb,
+				      enum pci_epc_interface_type type)
+{
+	struct epf_ntb_epc *ntb_epc;
+	u8 vfunc_no, func_no;
+	struct pci_epc *epc;
+	struct pci_epf *epf;
+	struct device *dev;
+	int ret;
+
+	ntb_epc = ntb->epc[type];
+	epf = ntb->epf;
+	dev = &epf->dev;
+	epc = ntb_epc->epc;
+	func_no = ntb_epc->func_no;
+	vfunc_no = ntb_epc->vfunc_no;
+
+	ret = epf_ntb_config_sspad_bar_set(ntb->epc[type]);
+	if (ret) {
+		dev_err(dev, "%s intf: Config/self SPAD BAR init failed\n",
+			pci_epc_interface_string(type));
+		return ret;
+	}
+
+	ret = epf_ntb_peer_spad_bar_set(ntb, type);
+	if (ret) {
+		dev_err(dev, "%s intf: Peer SPAD BAR init failed\n",
+			pci_epc_interface_string(type));
+		goto err_peer_spad_bar_init;
+	}
+
+	ret = epf_ntb_configure_interrupt(ntb, type);
+	if (ret) {
+		dev_err(dev, "%s intf: Interrupt configuration failed\n",
+			pci_epc_interface_string(type));
+		goto err_peer_spad_bar_init;
+	}
+
+	ret = epf_ntb_db_mw_bar_init(ntb, type);
+	if (ret) {
+		dev_err(dev, "%s intf: DB/MW BAR init failed\n",
+			pci_epc_interface_string(type));
+		goto err_db_mw_bar_init;
+	}
+
+	ret = pci_epc_write_header(epc, func_no, vfunc_no, epf->header);
+	if (ret) {
+		dev_err(dev, "%s intf: Configuration header write failed\n",
+			pci_epc_interface_string(type));
+		goto err_write_header;
+	}
+
+	INIT_DELAYED_WORK(&ntb->epc[type]->cmd_handler, epf_ntb_cmd_handler);
+	queue_work(kpcintb_workqueue, &ntb->epc[type]->cmd_handler.work);
+
+	return 0;
+
+err_write_header:
+	epf_ntb_db_mw_bar_cleanup(ntb->epc[type], ntb->epc[!type]);
+
+err_db_mw_bar_init:
+	epf_ntb_peer_spad_bar_clear(ntb->epc[type]);
+
+err_peer_spad_bar_init:
+	epf_ntb_config_sspad_bar_clear(ntb->epc[type]);
+
+	return ret;
+}
+
+static void epf_ntb_epc_cleanup(struct epf_ntb *ntb)
+{
+	enum pci_epc_interface_type type;
+	struct epf_ntb_epc *peer_ntb_epc;
+	struct epf_ntb_epc *ntb_epc;
+
+	for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {
+		ntb_epc = ntb->epc[type];
+		peer_ntb_epc = ntb->epc[!type];
+		cancel_delayed_work(&ntb_epc->cmd_handler);
+		epf_ntb_db_mw_bar_cleanup(ntb_epc, peer_ntb_epc);
+		epf_ntb_peer_spad_bar_clear(ntb_epc);
+		epf_ntb_config_sspad_bar_clear(ntb_epc);
+	}
+}
+
+static int epf_ntb_epc_init(struct epf_ntb *ntb)
+{
+	enum pci_epc_interface_type type;
+	struct device *dev;
+	int ret;
+
+	dev = &ntb->epf->dev;
+
+	for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {
+		ret = epf_ntb_epc_init_interface(ntb, type);
+		if (ret) {
+			dev_err(dev, "%s intf: Failed to initialize\n",
+				pci_epc_interface_string(type));
+			goto err_init_type;
+		}
+	}
+
+	return 0;
+
+err_init_type:
+	epf_ntb_epc_cleanup(ntb);
+
+	return ret;
+}
+
+static int epf_ntb_of_parse_mw(struct epf_ntb *ntb, struct device_node *node)
+{
+	struct device *dev;
+	u32 *mws_size;
+	u32 num_mws;
+	int ret;
+
+	dev = &ntb->epf->dev;
+	ret = of_property_read_u32(node, "num-mws", &num_mws);
+	if (ret) {
+		dev_err(dev, "Failed to get num-mws dt property\n");
+		return ret;
+	}
+
+	if (num_mws > MAX_MW) {
+		dev_err(dev, "Cannot support more than 4 memory window\n");
+		return ret;
+	}
+
+	mws_size = devm_kzalloc(dev, sizeof(*mws_size) * num_mws, GFP_KERNEL);
+	if (!mws_size)
+		return -ENOMEM;
+
+	ret = of_property_read_u32_array(node, "mws-size", mws_size,
+					 num_mws);
+	if (ret) {
+		dev_err(dev, "Failed to get mws-size dt property\n");
+		return ret;
+	}
+
+	ntb->num_mws = num_mws;
+	ntb->mws_size = mws_size;
+
+	return 0;
+}
+
+static int pci_epf_ntb_of_parse(struct epf_ntb *ntb)
+{
+	struct device_node *node;
+	struct pci_epf *epf;
+	struct device *dev;
+	int ret;
+
+	epf = ntb->epf;
+	node = epf->node;
+	dev = &epf->dev;
+
+	epf->header = &epf_ntb_header;
+	pci_epc_of_parse_header(node, epf->header);
+
+	ret = epf_ntb_of_parse_mw(ntb, node);
+	if (ret) {
+		dev_err(dev, "Invalid memory window configuration in DT\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int pci_epf_ntb_probe(struct pci_epf *epf)
+{
+	struct epf_ntb *ntb;
+	struct device *dev;
+	int ret;
+
+	dev = &epf->dev;
+
+	ntb = devm_kzalloc(dev, sizeof(*ntb), GFP_KERNEL);
+	if (!ntb)
+		return -ENOMEM;
+
+	ntb->epf = epf;
+
+	ret = pci_epf_ntb_of_parse(ntb);
+	if (ret) {
+		dev_err(dev, "Failed to parse NTB DT node\n");
+		return ret;
+	}
+
+	ret = epf_ntb_epc_create(ntb);
+	if (ret) {
+		dev_err(dev, "Failed to create NTB EPC\n");
+		return ret;
+	}
+
+	ret = epf_ntb_init_epc_bar(ntb);
+	if (ret) {
+		dev_err(dev, "Failed to create NTB EPC\n");
+		goto err_bar_init;
+	}
+
+	ret = epf_ntb_config_spad_bar_alloc(ntb);
+	if (ret) {
+		dev_err(dev, "Failed to allocate BAR memory\n");
+		goto err_bar_init;
+	}
+
+	ret = epf_ntb_epc_init(ntb);
+	if (ret) {
+		dev_err(dev, "Failed to initialize EPC\n");
+		goto err_epc_init;
+	}
+
+	epf_set_drvdata(epf, ntb);
+
+	return 0;
+
+err_epc_init:
+	epf_ntb_config_spad_bar_free(ntb);
+
+err_bar_init:
+	epf_ntb_epc_destroy(ntb);
+
+	return ret;
+}
+
+static int pci_epf_ntb_remove(struct pci_epf *epf)
+{
+	struct epf_ntb *ntb = epf_get_drvdata(epf);
+
+	epf_ntb_epc_cleanup(ntb);
+	epf_ntb_config_spad_bar_free(ntb);
+	epf_ntb_epc_destroy(ntb);
+
+	return 0;
+}
+
+static const struct pci_epf_device_id pci_epf_ntb_ids[] = {
+	{
+		.name = "pci-epf-ntb",
+	},
+	{},
+};
+
+static struct pci_epf_driver epf_ntb_driver = {
+	.driver.name	= "pci_epf_ntb",
+	.probe		= pci_epf_ntb_probe,
+	.remove		= pci_epf_ntb_remove,
+	.id_table	= pci_epf_ntb_ids,
+	.owner		= THIS_MODULE,
+};
+
+static int __init pci_epf_ntb_init(void)
+{
+	int ret;
+
+	kpcintb_workqueue = alloc_workqueue("kpcintb", WQ_MEM_RECLAIM |
+					    WQ_HIGHPRI, 0);
+	ret = pci_epf_register_driver(&epf_ntb_driver);
+	if (ret) {
+		pr_err("Failed to register pci epf ntb driver --> %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+module_init(pci_epf_ntb_init);
+
+static void __exit pci_epf_ntb_exit(void)
+{
+	pci_epf_unregister_driver(&epf_ntb_driver);
+}
+module_exit(pci_epf_ntb_exit);
+
+MODULE_DESCRIPTION("PCI EPF NTB DRIVER");
+MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c
--- a/drivers/pci/endpoint/functions/pci-epf-test.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/endpoint/functions/pci-epf-test.c	2020-06-19 21:00:31.420227716 -0400
@@ -8,6 +8,7 @@
 
 #include <linux/crc32.h>
 #include <linux/delay.h>
+#include <linux/dmaengine.h>
 #include <linux/io.h>
 #include <linux/module.h>
 #include <linux/slab.h>
@@ -47,9 +48,9 @@
 	void			*reg[6];
 	struct pci_epf		*epf;
 	enum pci_barno		test_reg_bar;
-	bool			linkup_notifier;
-	bool			msix_available;
+	size_t			msix_table_offset;
 	struct delayed_work	cmd_handler;
+	const struct pci_epc_features *epc_features;
 };
 
 struct pci_epf_test_reg {
@@ -71,20 +72,47 @@
 	.interrupt_pin	= PCI_INTERRUPT_INTA,
 };
 
-struct pci_epf_test_data {
-	enum pci_barno	test_reg_bar;
-	bool		linkup_notifier;
-};
-
 static size_t bar_size[] = { 512, 512, 1024, 16384, 131072, 1048576 };
 
+static void pci_epf_print_rate(const char *ops, u64 size,
+			       struct timespec64 *start, struct timespec64 *end,
+			       bool dma)
+{
+	struct timespec64 ts;
+	u64 rate, ns;
+
+	ts = timespec64_sub(*end, *start);
+
+	/* convert both size (stored in 'rate') and time in terms of 'ns' */
+	ns = timespec64_to_ns(&ts);
+	rate = size * NSEC_PER_SEC;
+
+	/* Divide both size (stored in 'rate') and ns by a common factor */
+	while (ns > UINT_MAX) {
+		rate >>= 1;
+		ns >>= 1;
+	}
+
+	if (!ns)
+		return;
+
+	/* calculate the rate */
+	do_div(rate, (uint32_t)ns);
+
+	pr_info("\n%s => Size: %llu bytes\t DMA: %s\t Time: %llu.%09u seconds\t"
+		"Rate: %llu KB/s\n", ops, size, dma ? "YES" : "NO",
+		(u64)ts.tv_sec, (u32)ts.tv_nsec, rate / 1024);
+}
+
 static int pci_epf_test_copy(struct pci_epf_test *epf_test)
 {
+	int tx;
 	int ret;
 	void __iomem *src_addr;
 	void __iomem *dst_addr;
 	phys_addr_t src_phys_addr;
 	phys_addr_t dst_phys_addr;
+	struct timespec64 start, end;
 	struct pci_epf *epf = epf_test->epf;
 	struct device *dev = &epf->dev;
 	struct pci_epc *epc = epf->epc;
@@ -99,8 +127,8 @@
 		goto err;
 	}
 
-	ret = pci_epc_map_addr(epc, epf->func_no, src_phys_addr, reg->src_addr,
-			       reg->size);
+	ret = pci_epc_map_addr(epc, epf->func_no, epf->vfunc_no, src_phys_addr,
+			       reg->src_addr, reg->size);
 	if (ret) {
 		dev_err(dev, "Failed to map source address\n");
 		reg->status = STATUS_SRC_ADDR_INVALID;
@@ -115,23 +143,31 @@
 		goto err_src_map_addr;
 	}
 
-	ret = pci_epc_map_addr(epc, epf->func_no, dst_phys_addr, reg->dst_addr,
-			       reg->size);
+	ret = pci_epc_map_addr(epc, epf->func_no, epf->vfunc_no, dst_phys_addr,
+			       reg->dst_addr, reg->size);
 	if (ret) {
 		dev_err(dev, "Failed to map destination address\n");
 		reg->status = STATUS_DST_ADDR_INVALID;
 		goto err_dst_addr;
 	}
 
-	memcpy(dst_addr, src_addr, reg->size);
+	ktime_get_ts64(&start);
+	tx = pci_epf_tx(epf, dst_phys_addr, src_phys_addr, reg->size);
+	if (tx) {
+		dev_err(dev, "DMA transfer failed, using memcpy..\n");
+		ktime_get_ts64(&start);
+		memcpy(dst_addr, src_addr, reg->size);
+	}
+	ktime_get_ts64(&end);
+	pci_epf_print_rate("COPY", reg->size, &start, &end, !tx);
 
-	pci_epc_unmap_addr(epc, epf->func_no, dst_phys_addr);
+	pci_epc_unmap_addr(epc, epf->func_no, epf->vfunc_no, dst_phys_addr);
 
 err_dst_addr:
 	pci_epc_mem_free_addr(epc, dst_phys_addr, dst_addr, reg->size);
 
 err_src_map_addr:
-	pci_epc_unmap_addr(epc, epf->func_no, src_phys_addr);
+	pci_epc_unmap_addr(epc, epf->func_no, epf->vfunc_no, src_phys_addr);
 
 err_src_addr:
 	pci_epc_mem_free_addr(epc, src_phys_addr, src_addr, reg->size);
@@ -142,14 +178,18 @@
 
 static int pci_epf_test_read(struct pci_epf_test *epf_test)
 {
+	int tx;
 	int ret;
 	void __iomem *src_addr;
 	void *buf;
 	u32 crc32;
 	phys_addr_t phys_addr;
+	phys_addr_t dst_addr;
+	struct timespec64 start, end;
 	struct pci_epf *epf = epf_test->epf;
 	struct device *dev = &epf->dev;
 	struct pci_epc *epc = epf->epc;
+	struct device *dma_dev = epf->epc->dev.parent;
 	enum pci_barno test_reg_bar = epf_test->test_reg_bar;
 	struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar];
 
@@ -161,8 +201,8 @@
 		goto err;
 	}
 
-	ret = pci_epc_map_addr(epc, epf->func_no, phys_addr, reg->src_addr,
-			       reg->size);
+	ret = pci_epc_map_addr(epc, epf->func_no, epf->vfunc_no, phys_addr,
+			       reg->src_addr, reg->size);
 	if (ret) {
 		dev_err(dev, "Failed to map address\n");
 		reg->status = STATUS_SRC_ADDR_INVALID;
@@ -175,8 +215,33 @@
 		goto err_map_addr;
 	}
 
-	memcpy(buf, src_addr, reg->size);
+	if (epf->dma_chan)
+		dma_dev = epf->dma_chan->device->dev;
 
+	dst_addr = dma_map_single(dma_dev, buf, reg->size, DMA_FROM_DEVICE);
+	if (dma_mapping_error(dma_dev, dst_addr)) {
+		dev_err(dev, "failed to map destination buffer address\n");
+		memcpy_fromio(buf, src_addr, reg->size);
+		goto skip_dma;
+	}
+
+	ktime_get_ts64(&start);
+	tx = pci_epf_tx(epf, dst_addr, phys_addr, reg->size);
+	if (tx) {
+		dev_err(dev, "DMA transfer failed, using memcpy..\n");
+		dma_unmap_single(dma_dev, dst_addr, reg->size, DMA_FROM_DEVICE);
+		ktime_get_ts64(&start);
+		memcpy_fromio(buf, src_addr, reg->size);
+		ktime_get_ts64(&end);
+		pci_epf_print_rate("READ", reg->size, &start, &end, false);
+		goto skip_dma;
+	}
+	ktime_get_ts64(&end);
+	pci_epf_print_rate("READ", reg->size, &start, &end, true);
+
+	dma_unmap_single(dma_dev, dst_addr, reg->size, DMA_FROM_DEVICE);
+
+skip_dma:
 	crc32 = crc32_le(~0, buf, reg->size);
 	if (crc32 != reg->checksum)
 		ret = -EIO;
@@ -184,7 +249,7 @@
 	kfree(buf);
 
 err_map_addr:
-	pci_epc_unmap_addr(epc, epf->func_no, phys_addr);
+	pci_epc_unmap_addr(epc, epf->func_no, epf->vfunc_no, phys_addr);
 
 err_addr:
 	pci_epc_mem_free_addr(epc, phys_addr, src_addr, reg->size);
@@ -195,13 +260,17 @@
 
 static int pci_epf_test_write(struct pci_epf_test *epf_test)
 {
+	int tx;
 	int ret;
 	void __iomem *dst_addr;
 	void *buf;
 	phys_addr_t phys_addr;
+	phys_addr_t src_addr;
+	struct timespec64 start, end;
 	struct pci_epf *epf = epf_test->epf;
 	struct device *dev = &epf->dev;
 	struct pci_epc *epc = epf->epc;
+	struct device *dma_dev = epf->epc->dev.parent;
 	enum pci_barno test_reg_bar = epf_test->test_reg_bar;
 	struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar];
 
@@ -213,8 +282,8 @@
 		goto err;
 	}
 
-	ret = pci_epc_map_addr(epc, epf->func_no, phys_addr, reg->dst_addr,
-			       reg->size);
+	ret = pci_epc_map_addr(epc, epf->func_no, epf->vfunc_no, phys_addr,
+			       reg->dst_addr, reg->size);
 	if (ret) {
 		dev_err(dev, "Failed to map address\n");
 		reg->status = STATUS_DST_ADDR_INVALID;
@@ -230,8 +299,29 @@
 	get_random_bytes(buf, reg->size);
 	reg->checksum = crc32_le(~0, buf, reg->size);
 
-	memcpy(dst_addr, buf, reg->size);
+	if (epf->dma_chan)
+		dma_dev = epf->dma_chan->device->dev;
+
+	src_addr = dma_map_single(dma_dev, buf, reg->size, DMA_TO_DEVICE);
+	if (dma_mapping_error(dma_dev, src_addr)) {
+		dev_err(dev, "failed to map source buffer address\n");
+		memcpy_toio(dst_addr, buf, reg->size);
+		goto skip_dma;
+	}
+
+	ktime_get_ts64(&start);
+	tx = pci_epf_tx(epf, phys_addr, src_addr, reg->size);
+	if (tx) {
+		dev_err(dev, "DMA transfer failed, using memcpy..\n");
+		ktime_get_ts64(&start);
+		memcpy_toio(dst_addr, buf, reg->size);
+	}
+	ktime_get_ts64(&end);
+	pci_epf_print_rate("WRITE", reg->size, &start, &end, !tx);
+
+	dma_unmap_single(dma_dev, src_addr, reg->size, DMA_TO_DEVICE);
 
+skip_dma:
 	/*
 	 * wait 1ms inorder for the write to complete. Without this delay L3
 	 * error in observed in the host system.
@@ -241,7 +331,7 @@
 	kfree(buf);
 
 err_map_addr:
-	pci_epc_unmap_addr(epc, epf->func_no, phys_addr);
+	pci_epc_unmap_addr(epc, epf->func_no, epf->vfunc_no, phys_addr);
 
 err_addr:
 	pci_epc_mem_free_addr(epc, phys_addr, dst_addr, reg->size);
@@ -263,13 +353,16 @@
 
 	switch (irq_type) {
 	case IRQ_TYPE_LEGACY:
-		pci_epc_raise_irq(epc, epf->func_no, PCI_EPC_IRQ_LEGACY, 0);
+		pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no,
+				  PCI_EPC_IRQ_LEGACY, 0);
 		break;
 	case IRQ_TYPE_MSI:
-		pci_epc_raise_irq(epc, epf->func_no, PCI_EPC_IRQ_MSI, irq);
+		pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no,
+				  PCI_EPC_IRQ_MSI, irq);
 		break;
 	case IRQ_TYPE_MSIX:
-		pci_epc_raise_irq(epc, epf->func_no, PCI_EPC_IRQ_MSIX, irq);
+		pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no,
+				  PCI_EPC_IRQ_MSIX, irq);
 		break;
 	default:
 		dev_err(dev, "Failed to raise IRQ, unknown type\n");
@@ -304,7 +397,8 @@
 
 	if (command & COMMAND_RAISE_LEGACY_IRQ) {
 		reg->status = STATUS_IRQ_RAISED;
-		pci_epc_raise_irq(epc, epf->func_no, PCI_EPC_IRQ_LEGACY, 0);
+		pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no,
+				  PCI_EPC_IRQ_LEGACY, 0);
 		goto reset_handler;
 	}
 
@@ -342,22 +436,22 @@
 	}
 
 	if (command & COMMAND_RAISE_MSI_IRQ) {
-		count = pci_epc_get_msi(epc, epf->func_no);
+		count = pci_epc_get_msi(epc, epf->func_no, epf->vfunc_no);
 		if (reg->irq_number > count || count <= 0)
 			goto reset_handler;
 		reg->status = STATUS_IRQ_RAISED;
-		pci_epc_raise_irq(epc, epf->func_no, PCI_EPC_IRQ_MSI,
-				  reg->irq_number);
+		pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no,
+				  PCI_EPC_IRQ_MSI, reg->irq_number);
 		goto reset_handler;
 	}
 
 	if (command & COMMAND_RAISE_MSIX_IRQ) {
-		count = pci_epc_get_msix(epc, epf->func_no);
+		count = pci_epc_get_msix(epc, epf->func_no, epf->vfunc_no);
 		if (reg->irq_number > count || count <= 0)
 			goto reset_handler;
 		reg->status = STATUS_IRQ_RAISED;
-		pci_epc_raise_irq(epc, epf->func_no, PCI_EPC_IRQ_MSIX,
-				  reg->irq_number);
+		pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no,
+				  PCI_EPC_IRQ_MSIX, reg->irq_number);
 		goto reset_handler;
 	}
 
@@ -366,12 +460,16 @@
 			   msecs_to_jiffies(1));
 }
 
-static void pci_epf_test_linkup(struct pci_epf *epf)
+static int pci_epf_test_notifier(struct notifier_block *nb, unsigned long val,
+				 void *data)
 {
+	struct pci_epf *epf = container_of(nb, struct pci_epf, nb);
 	struct pci_epf_test *epf_test = epf_get_drvdata(epf);
 
 	queue_delayed_work(kpcitest_workqueue, &epf_test->cmd_handler,
 			   msecs_to_jiffies(1));
+
+	return NOTIFY_OK;
 }
 
 static void pci_epf_test_unbind(struct pci_epf *epf)
@@ -387,43 +485,49 @@
 		epf_bar = &epf->bar[bar];
 
 		if (epf_test->reg[bar]) {
-			pci_epf_free_space(epf, epf_test->reg[bar], bar);
-			pci_epc_clear_bar(epc, epf->func_no, epf_bar);
+			pci_epc_clear_bar(epc, epf->func_no, epf->vfunc_no,
+					  epf_bar);
+			pci_epf_free_space(epf, epf_test->reg[bar], bar,
+					   PRIMARY_INTERFACE);
 		}
 	}
+	pci_epc_epf_exit(epc, epf);
 }
 
 static int pci_epf_test_set_bar(struct pci_epf *epf)
 {
-	int bar;
+	int bar, add;
 	int ret;
 	struct pci_epf_bar *epf_bar;
 	struct pci_epc *epc = epf->epc;
 	struct device *dev = &epf->dev;
 	struct pci_epf_test *epf_test = epf_get_drvdata(epf);
 	enum pci_barno test_reg_bar = epf_test->test_reg_bar;
+	const struct pci_epc_features *epc_features;
 
-	for (bar = BAR_0; bar <= BAR_5; bar++) {
+	epc_features = epf_test->epc_features;
+
+	for (bar = BAR_0; bar <= BAR_5; bar += add) {
 		epf_bar = &epf->bar[bar];
+		/*
+		 * pci_epc_set_bar() sets PCI_BASE_ADDRESS_MEM_TYPE_64
+		 * if the specific implementation required a 64-bit BAR,
+		 * even if we only requested a 32-bit BAR.
+		 */
+		add = (epf_bar->flags & PCI_BASE_ADDRESS_MEM_TYPE_64) ? 2 : 1;
 
-		epf_bar->flags |= upper_32_bits(epf_bar->size) ?
-			PCI_BASE_ADDRESS_MEM_TYPE_64 :
-			PCI_BASE_ADDRESS_MEM_TYPE_32;
+		if (!!(epc_features->reserved_bar & (1 << bar)))
+			continue;
 
-		ret = pci_epc_set_bar(epc, epf->func_no, epf_bar);
+		ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no,
+				      epf_bar);
 		if (ret) {
-			pci_epf_free_space(epf, epf_test->reg[bar], bar);
+			pci_epf_free_space(epf, epf_test->reg[bar], bar,
+					   PRIMARY_INTERFACE);
 			dev_err(dev, "Failed to set BAR%d\n", bar);
 			if (bar == test_reg_bar)
 				return ret;
 		}
-		/*
-		 * pci_epc_set_bar() sets PCI_BASE_ADDRESS_MEM_TYPE_64
-		 * if the specific implementation required a 64-bit BAR,
-		 * even if we only requested a 32-bit BAR.
-		 */
-		if (epf_bar->flags & PCI_BASE_ADDRESS_MEM_TYPE_64)
-			bar++;
 	}
 
 	return 0;
@@ -433,22 +537,57 @@
 {
 	struct pci_epf_test *epf_test = epf_get_drvdata(epf);
 	struct device *dev = &epf->dev;
+	struct pci_epf_bar *epf_bar;
+	size_t msix_table_size = 0;
+	size_t test_reg_bar_size;
+	size_t pba_size = 0;
+	bool msix_capable;
 	void *base;
-	int bar;
+	int bar, add;
 	enum pci_barno test_reg_bar = epf_test->test_reg_bar;
+	const struct pci_epc_features *epc_features;
+	size_t test_reg_size;
+
+	epc_features = epf_test->epc_features;
 
-	base = pci_epf_alloc_space(epf, sizeof(struct pci_epf_test_reg),
-				   test_reg_bar);
+	test_reg_bar_size = ALIGN(sizeof(struct pci_epf_test_reg), 128);
+
+	msix_capable = epc_features->msix_capable;
+	if (msix_capable) {
+		msix_table_size = PCI_MSIX_ENTRY_SIZE * epf->msix_interrupts;
+		epf_test->msix_table_offset = test_reg_bar_size;
+		/* Align to QWORD or 8 Bytes */
+		pba_size = ALIGN(DIV_ROUND_UP(epf->msix_interrupts, 8), 8);
+	}
+	test_reg_size = test_reg_bar_size + msix_table_size + pba_size;
+
+	if (epc_features->bar_fixed_size[test_reg_bar]) {
+		if (test_reg_size > bar_size[test_reg_bar])
+			return -ENOMEM;
+		test_reg_size = bar_size[test_reg_bar];
+	}
+
+	base = pci_epf_alloc_space(epf, test_reg_size, test_reg_bar,
+				   epc_features->align, PRIMARY_INTERFACE);
 	if (!base) {
 		dev_err(dev, "Failed to allocated register space\n");
 		return -ENOMEM;
 	}
 	epf_test->reg[test_reg_bar] = base;
 
-	for (bar = BAR_0; bar <= BAR_5; bar++) {
+	for (bar = BAR_0; bar <= BAR_5; bar += add) {
+		epf_bar = &epf->bar[bar];
+		add = (epf_bar->flags & PCI_BASE_ADDRESS_MEM_TYPE_64) ? 2 : 1;
+
 		if (bar == test_reg_bar)
 			continue;
-		base = pci_epf_alloc_space(epf, bar_size[bar], bar);
+
+		if (!!(epc_features->reserved_bar & (1 << bar)))
+			continue;
+
+		base = pci_epf_alloc_space(epf, bar_size[bar], bar,
+					   epc_features->align,
+					   PRIMARY_INTERFACE);
 		if (!base)
 			dev_err(dev, "Failed to allocate space for BAR%d\n",
 				bar);
@@ -458,27 +597,58 @@
 	return 0;
 }
 
+static void pci_epf_configure_bar(struct pci_epf *epf,
+				  const struct pci_epc_features *epc_features)
+{
+	struct pci_epf_bar *epf_bar;
+	bool bar_fixed_64bit;
+	int i;
+
+	for (i = BAR_0; i <= BAR_5; i++) {
+		epf_bar = &epf->bar[i];
+		bar_fixed_64bit = !!(epc_features->bar_fixed_64bit & (1 << i));
+		if (bar_fixed_64bit)
+			epf_bar->flags |= PCI_BASE_ADDRESS_MEM_TYPE_64;
+		if (epc_features->bar_fixed_size[i])
+			bar_size[i] = epc_features->bar_fixed_size[i];
+	}
+}
+
 static int pci_epf_test_bind(struct pci_epf *epf)
 {
 	int ret;
 	struct pci_epf_test *epf_test = epf_get_drvdata(epf);
 	struct pci_epf_header *header = epf->header;
+	const struct pci_epc_features *epc_features;
+	enum pci_barno test_reg_bar = BAR_0;
 	struct pci_epc *epc = epf->epc;
 	struct device *dev = &epf->dev;
+	bool linkup_notifier = false;
+	bool msix_capable = false;
+	bool msi_capable = true;
 
 	if (WARN_ON_ONCE(!epc))
 		return -EINVAL;
 
-	if (epc->features & EPC_FEATURE_NO_LINKUP_NOTIFIER)
-		epf_test->linkup_notifier = false;
-	else
-		epf_test->linkup_notifier = true;
+	epc_features = pci_epc_get_features(epc, epf->func_no, epf->vfunc_no);
+	if (epc_features) {
+		linkup_notifier = epc_features->linkup_notifier;
+		msix_capable = epc_features->msix_capable;
+		msi_capable = epc_features->msi_capable;
+		test_reg_bar = pci_epc_get_first_free_bar(epc_features);
+		pci_epf_configure_bar(epf, epc_features);
+	}
 
-	epf_test->msix_available = epc->features & EPC_FEATURE_MSIX_AVAILABLE;
+	epf_test->test_reg_bar = test_reg_bar;
+	epf_test->epc_features = epc_features;
 
-	epf_test->test_reg_bar = EPC_FEATURE_GET_BAR(epc->features);
+	ret = pci_epc_epf_init(epc, epf);
+	if (ret) {
+		dev_err(dev, "Failed to initialize EPF\n");
+		return ret;
+	}
 
-	ret = pci_epc_write_header(epc, epf->func_no, header);
+	ret = pci_epc_write_header(epc, epf->func_no, epf->vfunc_no, header);
 	if (ret) {
 		dev_err(dev, "Configuration header write failed\n");
 		return ret;
@@ -492,22 +662,32 @@
 	if (ret)
 		return ret;
 
-	ret = pci_epc_set_msi(epc, epf->func_no, epf->msi_interrupts);
-	if (ret) {
-		dev_err(dev, "MSI configuration failed\n");
-		return ret;
+	if (msi_capable) {
+		ret = pci_epc_set_msi(epc, epf->func_no, epf->vfunc_no,
+				      epf->msi_interrupts);
+		if (ret) {
+			dev_err(dev, "MSI configuration failed\n");
+			return ret;
+		}
 	}
 
-	if (epf_test->msix_available) {
-		ret = pci_epc_set_msix(epc, epf->func_no, epf->msix_interrupts);
+	if (msix_capable) {
+		ret = pci_epc_set_msix(epc, epf->func_no, epf->vfunc_no,
+				       epf->msix_interrupts,
+				       epf_test->test_reg_bar,
+				       epf_test->msix_table_offset);
 		if (ret) {
 			dev_err(dev, "MSI-X configuration failed\n");
 			return ret;
 		}
 	}
 
-	if (!epf_test->linkup_notifier)
+	if (linkup_notifier) {
+		epf->nb.notifier_call = pci_epf_test_notifier;
+		pci_epc_register_notifier(epc, &epf->nb);
+	} else {
 		queue_work(kpcitest_workqueue, &epf_test->cmd_handler.work);
+	}
 
 	return 0;
 }
@@ -523,17 +703,6 @@
 {
 	struct pci_epf_test *epf_test;
 	struct device *dev = &epf->dev;
-	const struct pci_epf_device_id *match;
-	struct pci_epf_test_data *data;
-	enum pci_barno test_reg_bar = BAR_0;
-	bool linkup_notifier = true;
-
-	match = pci_epf_match_device(pci_epf_test_ids, epf);
-	data = (struct pci_epf_test_data *)match->driver_data;
-	if (data) {
-		test_reg_bar = data->test_reg_bar;
-		linkup_notifier = data->linkup_notifier;
-	}
 
 	epf_test = devm_kzalloc(dev, sizeof(*epf_test), GFP_KERNEL);
 	if (!epf_test)
@@ -541,8 +710,6 @@
 
 	epf->header = &test_header;
 	epf_test->epf = epf;
-	epf_test->test_reg_bar = test_reg_bar;
-	epf_test->linkup_notifier = linkup_notifier;
 
 	INIT_DELAYED_WORK(&epf_test->cmd_handler, pci_epf_test_cmd_handler);
 
@@ -553,7 +720,6 @@
 static struct pci_epf_ops ops = {
 	.unbind	= pci_epf_test_unbind,
 	.bind	= pci_epf_test_bind,
-	.linkup = pci_epf_test_linkup,
 };
 
 static struct pci_epf_driver test_driver = {
diff -Naur --no-dereference a/drivers/pci/endpoint/Makefile b/drivers/pci/endpoint/Makefile
--- a/drivers/pci/endpoint/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/endpoint/Makefile	2020-06-19 21:00:31.420227716 -0400
@@ -5,4 +5,5 @@
 
 obj-$(CONFIG_PCI_ENDPOINT_CONFIGFS)	+= pci-ep-cfs.o
 obj-$(CONFIG_PCI_ENDPOINT)		+= pci-epc-core.o pci-epf-core.o\
-					   pci-epc-mem.o functions/
+					   pci-epc-mem.o pci-epf-bus.o \
+					   functions/
diff -Naur --no-dereference a/drivers/pci/endpoint/pci-epc-core.c b/drivers/pci/endpoint/pci-epc-core.c
--- a/drivers/pci/endpoint/pci-epc-core.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/endpoint/pci-epc-core.c	2020-06-19 21:00:31.420227716 -0400
@@ -32,6 +32,29 @@
 }
 
 /**
+ * pci_epc_of_parse_header() - parse the device tree to get PCI config space
+ *                             header
+ * @node: The device tree node (of endpoint function) which has the PCI config
+ *        space header values
+ * @header: standard configuration space header fields that has to be populated
+ *
+ * Invoke to populate *header* with the PCI configuration space values populated
+ * in device tree.
+ */
+void pci_epc_of_parse_header(struct device_node *node,
+			     struct pci_epf_header *header)
+{
+	of_property_read_u16(node, "vendor-id", &header->vendorid);
+	of_property_read_u16(node, "device-id", &header->deviceid);
+	of_property_read_u8(node, "baseclass-code", &header->baseclass_code);
+	of_property_read_u8(node, "subclass-code", &header->subclass_code);
+	of_property_read_u16(node, "subsys-vendor-id",
+			     &header->subsys_vendor_id);
+	of_property_read_u16(node, "subsys-id", &header->subsys_id);
+}
+EXPORT_SYMBOL_GPL(pci_epc_of_parse_header);
+
+/**
  * pci_epc_put() - release the PCI endpoint controller
  * @epc: epc returned by pci_epc_get()
  *
@@ -84,6 +107,192 @@
 EXPORT_SYMBOL_GPL(pci_epc_get);
 
 /**
+ * of_pci_epc_get() - get PCI endpoint controller from device node and index
+ * @node: device node which contains the phandle to endpoint controller
+ * @index: index of the endpoint controller in "epcs" property
+ *
+ * Returns the EPC corresponding to the _index_ entry in "epcs" property
+ * present in device node, after getting a refcount  to it or -ENODEV if
+ * there is no such EPC or -EPROBE_DEFER if there is a phandle to the phy,
+ * but the device is not yet loaded.
+ */
+struct pci_epc *of_pci_epc_get(struct device_node *node, int index)
+{
+	struct device_node *epc_node;
+	struct class_dev_iter iter;
+	struct pci_epc *epc;
+	struct device *dev;
+
+	epc_node = of_parse_phandle(node, "epcs", index);
+	if (!epc_node)
+		return ERR_PTR(-ENODEV);
+
+	class_dev_iter_init(&iter, pci_epc_class, NULL, NULL);
+	while ((dev = class_dev_iter_next(&iter))) {
+		epc = to_pci_epc(dev);
+		if (epc_node != epc->dev.of_node)
+			continue;
+
+		of_node_put(epc_node);
+		class_dev_iter_exit(&iter);
+		get_device(&epc->dev);
+		return epc;
+	}
+
+	of_node_put(node);
+	class_dev_iter_exit(&iter);
+	return ERR_PTR(-EPROBE_DEFER);
+}
+EXPORT_SYMBOL_GPL(of_pci_epc_get);
+
+/**
+ * of_pci_epc_get_by_name() - get PCI endpoint controller from device node
+ *                            and string
+ * @node: device node which contains the phandle to endpoint controller
+ * @epc_name: name of endpoint controller as present in "epc-names" property
+ *
+ * Returns the EPC corresponding to the epc_name in "epc-names" property
+ * present in device node.
+ */
+struct pci_epc *of_pci_epc_get_by_name(struct device_node *node,
+				       const char *epc_name)
+{
+	int index = 0;
+
+	if (epc_name)
+		index = of_property_match_string(node, "epc-names", epc_name);
+
+	return of_pci_epc_get(node, index);
+}
+EXPORT_SYMBOL_GPL(of_pci_epc_get_by_name);
+
+/**
+ * pci_epc_get_first_free_bar() - helper to get first unreserved BAR
+ * @epc_features: pci_epc_features structure that holds the reserved bar bitmap
+ *
+ * Invoke to get the first unreserved BAR that can be used by the endpoint
+ * function. For any incorrect value in reserved_bar return '0'.
+ */
+int pci_epc_get_first_free_bar(const struct pci_epc_features *epc_features)
+{
+	return pci_epc_get_next_free_bar(epc_features, BAR_0);
+}
+EXPORT_SYMBOL_GPL(pci_epc_get_first_free_bar);
+
+/**
+ * pci_epc_get_next_free_bar() - helper to get unreserved BAR starting from @bar
+ * @epc_features: pci_epc_features structure that holds the reserved bar bitmap
+ * @bar: the starting BAR number from where unreserved BAR should be searched
+ *
+ * Invoke to get the next unreserved BAR starting from @bar that can be used
+ * for endpoint function. For any incorrect value in reserved_bar return '0'.
+ */
+int pci_epc_get_next_free_bar(const struct pci_epc_features
+			      *epc_features, enum pci_barno bar)
+{
+	unsigned long free_bar;
+
+	if (!epc_features)
+		return 0;
+
+	/* If 'bar - 1' is a 64-bit BAR, move to the next BAR */
+	if ((epc_features->bar_fixed_64bit << 1) & 1 << bar)
+		bar++;
+
+	/* Find if the reserved BAR is also a 64-bit BAR */
+	free_bar = epc_features->reserved_bar & epc_features->bar_fixed_64bit;
+
+	/* Set the adjacent bit if the reserved BAR is also a 64-bit BAR */
+	free_bar <<= 1;
+	free_bar |= epc_features->reserved_bar;
+
+	free_bar = find_next_zero_bit(&free_bar, 6, bar);
+	if (free_bar > 5)
+		return -EINVAL;
+
+	return free_bar;
+}
+EXPORT_SYMBOL_GPL(pci_epc_get_next_free_bar);
+
+/**
+ * pci_epc_get_features() - get the features supported by EPC
+ * @epc: the features supported by *this* EPC device will be returned
+ * @func_no: the features supported by the EPC device specific to the
+ *	     endpoint function with func_no will be returned
+ * @vfunc_no: the features supported by the EPC device specific to the
+ *	     virtual endpoint function with vfunc_no will be returned
+ *
+ * Invoke to get the features provided by the EPC which may be
+ * specific to an endpoint function. Returns pci_epc_features on success
+ * and NULL for any failures.
+ */
+const struct pci_epc_features *pci_epc_get_features(struct pci_epc *epc,
+						    u8 func_no, u8 vfunc_no)
+{
+	const struct pci_epc_features *epc_features;
+
+	if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions)
+		return NULL;
+
+	if (!epc->ops->get_features)
+		return NULL;
+
+	mutex_lock(&epc->lock);
+	epc_features = epc->ops->get_features(epc, func_no, vfunc_no);
+	mutex_unlock(&epc->lock);
+
+	return epc_features;
+}
+EXPORT_SYMBOL_GPL(pci_epc_get_features);
+
+/**
+ * pci_epc_epf_init() - EPC specific EPF initialization
+ * @epc: the EPC device that initializes the EPF
+ * @epf: the EPF device that has to be initialized
+ *
+ * Invoke to initialize EPF that is specific to a EPC and varies from
+ * platform to platform
+ */
+int pci_epc_epf_init(struct pci_epc *epc, struct pci_epf *epf)
+{
+	int ret;
+
+	if (IS_ERR_OR_NULL(epc) || IS_ERR_OR_NULL(epf))
+		return -EINVAL;
+
+	if (!epc->ops->epf_init)
+		return 0;
+
+	mutex_lock(&epc->lock);
+	ret = epc->ops->epf_init(epc, epf);
+	mutex_unlock(&epc->lock);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(pci_epc_epf_init);
+
+/**
+ * pci_epc_epf_exit() - Cleanup the EPF initialization
+ * @epc: the EPC device that initialized the EPF
+ * @epf: the EPF device that has to be reset
+ *
+ * Invoke to cleanup the EPF initialization done as part fo pci_epc_epf_init.
+ */
+void pci_epc_epf_exit(struct pci_epc *epc, struct pci_epf *epf)
+{
+	if (IS_ERR_OR_NULL(epc) || IS_ERR_OR_NULL(epf))
+		return;
+
+	if (!epc->ops->epf_exit)
+		return;
+
+	mutex_lock(&epc->lock);
+	epc->ops->epf_exit(epc, epf);
+	mutex_unlock(&epc->lock);
+}
+EXPORT_SYMBOL_GPL(pci_epc_epf_exit);
+
+/**
  * pci_epc_stop() - stop the PCI link
  * @epc: the link of the EPC device that has to be stopped
  *
@@ -91,14 +300,12 @@
  */
 void pci_epc_stop(struct pci_epc *epc)
 {
-	unsigned long flags;
-
 	if (IS_ERR(epc) || !epc->ops->stop)
 		return;
 
-	spin_lock_irqsave(&epc->lock, flags);
+	mutex_lock(&epc->lock);
 	epc->ops->stop(epc);
-	spin_unlock_irqrestore(&epc->lock, flags);
+	mutex_unlock(&epc->lock);
 }
 EXPORT_SYMBOL_GPL(pci_epc_stop);
 
@@ -111,7 +318,6 @@
 int pci_epc_start(struct pci_epc *epc)
 {
 	int ret;
-	unsigned long flags;
 
 	if (IS_ERR(epc))
 		return -EINVAL;
@@ -119,9 +325,9 @@
 	if (!epc->ops->start)
 		return 0;
 
-	spin_lock_irqsave(&epc->lock, flags);
+	mutex_lock(&epc->lock);
 	ret = epc->ops->start(epc);
-	spin_unlock_irqrestore(&epc->lock, flags);
+	mutex_unlock(&epc->lock);
 
 	return ret;
 }
@@ -130,17 +336,17 @@
 /**
  * pci_epc_raise_irq() - interrupt the host system
  * @epc: the EPC device which has to interrupt the host
- * @func_no: the endpoint function number in the EPC device
+ * @func_no: the physical endpoint function number in the EPC device
+ * @vfunc_no: the virtual endpoint function number in the physical function
  * @type: specify the type of interrupt; legacy, MSI or MSI-X
  * @interrupt_num: the MSI or MSI-X interrupt number
  *
  * Invoke to raise an legacy, MSI or MSI-X interrupt
  */
-int pci_epc_raise_irq(struct pci_epc *epc, u8 func_no,
+int pci_epc_raise_irq(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
 		      enum pci_epc_irq_type type, u16 interrupt_num)
 {
 	int ret;
-	unsigned long flags;
 
 	if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions)
 		return -EINVAL;
@@ -148,25 +354,65 @@
 	if (!epc->ops->raise_irq)
 		return 0;
 
-	spin_lock_irqsave(&epc->lock, flags);
-	ret = epc->ops->raise_irq(epc, func_no, type, interrupt_num);
-	spin_unlock_irqrestore(&epc->lock, flags);
+	mutex_lock(&epc->lock);
+	ret = epc->ops->raise_irq(epc, func_no, vfunc_no, type, interrupt_num);
+	mutex_unlock(&epc->lock);
 
 	return ret;
 }
 EXPORT_SYMBOL_GPL(pci_epc_raise_irq);
 
 /**
+ * pci_epc_map_msi_irq() - Map physical address to MSI address and return
+ *                         MSI data
+ * @epc: the EPC device which has the MSI capability
+ * @func_no: the physical endpoint function number in the EPC device
+ * @vfunc_no: the virtual endpoint function number in the physical function
+ * @phys_addr: the physical address of the outbound region
+ * @interrupt_num: the MSI interrupt number
+ * @entry_size: Size of Outbound address region for each interrupt
+ * @msi_data: the data that should be written in order to raise MSI interrupt
+ *            with interrupt number as 'interrupt num'
+ *
+ * Invoke to map physical address to MSI address and return MSI data. The
+ * physical address should be an address in the outbound region. This is
+ * required to implement doorbell functionality of NTB wherein EPC on either
+ * side of the interface (primary and secondary) can directly write to the
+ * physical address (in outbound region) of the other interface to ring
+ * doorbell.
+ */
+int pci_epc_map_msi_irq(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
+			phys_addr_t phys_addr, u8 interrupt_num, u32 entry_size,
+			u32 *msi_data)
+{
+	int ret;
+
+	if (IS_ERR_OR_NULL(epc))
+		return -EINVAL;
+
+	if (!epc->ops->map_msi_irq)
+		return -EINVAL;
+
+	mutex_lock(&epc->lock);
+	ret = epc->ops->map_msi_irq(epc, func_no, vfunc_no, phys_addr,
+				    interrupt_num, entry_size, msi_data);
+	mutex_unlock(&epc->lock);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(pci_epc_map_msi_irq);
+
+/**
  * pci_epc_get_msi() - get the number of MSI interrupt numbers allocated
  * @epc: the EPC device to which MSI interrupts was requested
- * @func_no: the endpoint function number in the EPC device
+ * @func_no: the physical endpoint function number in the EPC device
+ * @vfunc_no: the virtual endpoint function number in the physical function
  *
  * Invoke to get the number of MSI interrupts allocated by the RC
  */
-int pci_epc_get_msi(struct pci_epc *epc, u8 func_no)
+int pci_epc_get_msi(struct pci_epc *epc, u8 func_no, u8 vfunc_no)
 {
 	int interrupt;
-	unsigned long flags;
 
 	if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions)
 		return 0;
@@ -174,9 +420,9 @@
 	if (!epc->ops->get_msi)
 		return 0;
 
-	spin_lock_irqsave(&epc->lock, flags);
-	interrupt = epc->ops->get_msi(epc, func_no);
-	spin_unlock_irqrestore(&epc->lock, flags);
+	mutex_lock(&epc->lock);
+	interrupt = epc->ops->get_msi(epc, func_no, vfunc_no);
+	mutex_unlock(&epc->lock);
 
 	if (interrupt < 0)
 		return 0;
@@ -190,16 +436,16 @@
 /**
  * pci_epc_set_msi() - set the number of MSI interrupt numbers required
  * @epc: the EPC device on which MSI has to be configured
- * @func_no: the endpoint function number in the EPC device
+ * @func_no: the physical endpoint function number in the EPC device
+ * @vfunc_no: the virtual endpoint function number in the physical function
  * @interrupts: number of MSI interrupts required by the EPF
  *
  * Invoke to set the required number of MSI interrupts.
  */
-int pci_epc_set_msi(struct pci_epc *epc, u8 func_no, u8 interrupts)
+int pci_epc_set_msi(struct pci_epc *epc, u8 func_no, u8 vfunc_no, u8 interrupts)
 {
 	int ret;
 	u8 encode_int;
-	unsigned long flags;
 
 	if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions ||
 	    interrupts > 32)
@@ -210,9 +456,9 @@
 
 	encode_int = order_base_2(interrupts);
 
-	spin_lock_irqsave(&epc->lock, flags);
-	ret = epc->ops->set_msi(epc, func_no, encode_int);
-	spin_unlock_irqrestore(&epc->lock, flags);
+	mutex_lock(&epc->lock);
+	ret = epc->ops->set_msi(epc, func_no, vfunc_no, encode_int);
+	mutex_unlock(&epc->lock);
 
 	return ret;
 }
@@ -221,14 +467,14 @@
 /**
  * pci_epc_get_msix() - get the number of MSI-X interrupt numbers allocated
  * @epc: the EPC device to which MSI-X interrupts was requested
- * @func_no: the endpoint function number in the EPC device
+ * @func_no: the physical endpoint function number in the EPC device
+ * @vfunc_no: the virtual endpoint function number in the physical function
  *
  * Invoke to get the number of MSI-X interrupts allocated by the RC
  */
-int pci_epc_get_msix(struct pci_epc *epc, u8 func_no)
+int pci_epc_get_msix(struct pci_epc *epc, u8 func_no, u8 vfunc_no)
 {
 	int interrupt;
-	unsigned long flags;
 
 	if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions)
 		return 0;
@@ -236,9 +482,9 @@
 	if (!epc->ops->get_msix)
 		return 0;
 
-	spin_lock_irqsave(&epc->lock, flags);
-	interrupt = epc->ops->get_msix(epc, func_no);
-	spin_unlock_irqrestore(&epc->lock, flags);
+	mutex_lock(&epc->lock);
+	interrupt = epc->ops->get_msix(epc, func_no, vfunc_no);
+	mutex_unlock(&epc->lock);
 
 	if (interrupt < 0)
 		return 0;
@@ -250,15 +496,18 @@
 /**
  * pci_epc_set_msix() - set the number of MSI-X interrupt numbers required
  * @epc: the EPC device on which MSI-X has to be configured
- * @func_no: the endpoint function number in the EPC device
+ * @func_no: the physical endpoint function number in the EPC device
+ * @vfunc_no: the virtual endpoint function number in the physical function
  * @interrupts: number of MSI-X interrupts required by the EPF
+ * @bir: BAR where the MSI-X table resides
+ * @offset: Offset pointing to the start of MSI-X table
  *
  * Invoke to set the required number of MSI-X interrupts.
  */
-int pci_epc_set_msix(struct pci_epc *epc, u8 func_no, u16 interrupts)
+int pci_epc_set_msix(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
+		     u16 interrupts, enum pci_barno bir, u32 offset)
 {
 	int ret;
-	unsigned long flags;
 
 	if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions ||
 	    interrupts < 1 || interrupts > 2048)
@@ -267,9 +516,10 @@
 	if (!epc->ops->set_msix)
 		return 0;
 
-	spin_lock_irqsave(&epc->lock, flags);
-	ret = epc->ops->set_msix(epc, func_no, interrupts - 1);
-	spin_unlock_irqrestore(&epc->lock, flags);
+	mutex_lock(&epc->lock);
+	ret = epc->ops->set_msix(epc, func_no, vfunc_no, interrupts - 1,
+				 bir, offset);
+	mutex_unlock(&epc->lock);
 
 	return ret;
 }
@@ -278,43 +528,42 @@
 /**
  * pci_epc_unmap_addr() - unmap CPU address from PCI address
  * @epc: the EPC device on which address is allocated
- * @func_no: the endpoint function number in the EPC device
+ * @func_no: the physical endpoint function number in the EPC device
+ * @vfunc_no: the virtual endpoint function number in the physical function
  * @phys_addr: physical address of the local system
  *
  * Invoke to unmap the CPU address from PCI address.
  */
-void pci_epc_unmap_addr(struct pci_epc *epc, u8 func_no,
+void pci_epc_unmap_addr(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
 			phys_addr_t phys_addr)
 {
-	unsigned long flags;
-
 	if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions)
 		return;
 
 	if (!epc->ops->unmap_addr)
 		return;
 
-	spin_lock_irqsave(&epc->lock, flags);
-	epc->ops->unmap_addr(epc, func_no, phys_addr);
-	spin_unlock_irqrestore(&epc->lock, flags);
+	mutex_lock(&epc->lock);
+	epc->ops->unmap_addr(epc, func_no, vfunc_no, phys_addr);
+	mutex_unlock(&epc->lock);
 }
 EXPORT_SYMBOL_GPL(pci_epc_unmap_addr);
 
 /**
  * pci_epc_map_addr() - map CPU address to PCI address
  * @epc: the EPC device on which address is allocated
- * @func_no: the endpoint function number in the EPC device
+ * @func_no: the physical endpoint function number in the EPC device
+ * @vfunc_no: the virtual endpoint function number in the physical function
  * @phys_addr: physical address of the local system
  * @pci_addr: PCI address to which the physical address should be mapped
  * @size: the size of the allocation
  *
  * Invoke to map CPU address with PCI address.
  */
-int pci_epc_map_addr(struct pci_epc *epc, u8 func_no,
+int pci_epc_map_addr(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
 		     phys_addr_t phys_addr, u64 pci_addr, size_t size)
 {
 	int ret;
-	unsigned long flags;
 
 	if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions)
 		return -EINVAL;
@@ -322,9 +571,10 @@
 	if (!epc->ops->map_addr)
 		return 0;
 
-	spin_lock_irqsave(&epc->lock, flags);
-	ret = epc->ops->map_addr(epc, func_no, phys_addr, pci_addr, size);
-	spin_unlock_irqrestore(&epc->lock, flags);
+	mutex_lock(&epc->lock);
+	ret = epc->ops->map_addr(epc, func_no, vfunc_no, phys_addr, pci_addr,
+				 size);
+	mutex_unlock(&epc->lock);
 
 	return ret;
 }
@@ -333,16 +583,15 @@
 /**
  * pci_epc_clear_bar() - reset the BAR
  * @epc: the EPC device for which the BAR has to be cleared
- * @func_no: the endpoint function number in the EPC device
+ * @func_no: the physical endpoint function number in the EPC device
+ * @vfunc_no: the virtual endpoint function number in the physical function
  * @epf_bar: the struct epf_bar that contains the BAR information
  *
  * Invoke to reset the BAR of the endpoint device.
  */
-void pci_epc_clear_bar(struct pci_epc *epc, u8 func_no,
+void pci_epc_clear_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
 		       struct pci_epf_bar *epf_bar)
 {
-	unsigned long flags;
-
 	if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions ||
 	    (epf_bar->barno == BAR_5 &&
 	     epf_bar->flags & PCI_BASE_ADDRESS_MEM_TYPE_64))
@@ -351,25 +600,25 @@
 	if (!epc->ops->clear_bar)
 		return;
 
-	spin_lock_irqsave(&epc->lock, flags);
-	epc->ops->clear_bar(epc, func_no, epf_bar);
-	spin_unlock_irqrestore(&epc->lock, flags);
+	mutex_lock(&epc->lock);
+	epc->ops->clear_bar(epc, func_no, vfunc_no, epf_bar);
+	mutex_unlock(&epc->lock);
 }
 EXPORT_SYMBOL_GPL(pci_epc_clear_bar);
 
 /**
  * pci_epc_set_bar() - configure BAR in order for host to assign PCI addr space
  * @epc: the EPC device on which BAR has to be configured
- * @func_no: the endpoint function number in the EPC device
+ * @func_no: the physical endpoint function number in the EPC device
+ * @vfunc_no: the virtual endpoint function number in the physical function
  * @epf_bar: the struct epf_bar that contains the BAR information
  *
  * Invoke to configure the BAR of the endpoint device.
  */
-int pci_epc_set_bar(struct pci_epc *epc, u8 func_no,
+int pci_epc_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
 		    struct pci_epf_bar *epf_bar)
 {
 	int ret;
-	unsigned long irq_flags;
 	int flags = epf_bar->flags;
 
 	if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions ||
@@ -384,9 +633,9 @@
 	if (!epc->ops->set_bar)
 		return 0;
 
-	spin_lock_irqsave(&epc->lock, irq_flags);
-	ret = epc->ops->set_bar(epc, func_no, epf_bar);
-	spin_unlock_irqrestore(&epc->lock, irq_flags);
+	mutex_lock(&epc->lock);
+	ret = epc->ops->set_bar(epc, func_no, vfunc_no, epf_bar);
+	mutex_unlock(&epc->lock);
 
 	return ret;
 }
@@ -395,7 +644,8 @@
 /**
  * pci_epc_write_header() - write standard configuration header
  * @epc: the EPC device to which the configuration header should be written
- * @func_no: the endpoint function number in the EPC device
+ * @func_no: the physical endpoint function number in the EPC device
+ * @vfunc_no: the virtual endpoint function number in the physical function
  * @header: standard configuration header fields
  *
  * Invoke to write the configuration header to the endpoint controller. Every
@@ -403,11 +653,10 @@
  * configuration header would be written. The callback function should write
  * the header fields to this dedicated location.
  */
-int pci_epc_write_header(struct pci_epc *epc, u8 func_no,
+int pci_epc_write_header(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
 			 struct pci_epf_header *header)
 {
 	int ret;
-	unsigned long flags;
 
 	if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions)
 		return -EINVAL;
@@ -415,9 +664,9 @@
 	if (!epc->ops->write_header)
 		return 0;
 
-	spin_lock_irqsave(&epc->lock, flags);
-	ret = epc->ops->write_header(epc, func_no, header);
-	spin_unlock_irqrestore(&epc->lock, flags);
+	mutex_lock(&epc->lock);
+	ret = epc->ops->write_header(epc, func_no, vfunc_no, header);
+	mutex_unlock(&epc->lock);
 
 	return ret;
 }
@@ -427,17 +676,22 @@
  * pci_epc_add_epf() - bind PCI endpoint function to an endpoint controller
  * @epc: the EPC device to which the endpoint function should be added
  * @epf: the endpoint function to be added
+ * @type: Identifies if the EPC is connected to the primary or secondary
+ *        interface of EPF
  *
  * A PCI endpoint device can have one or more functions. In the case of PCIe,
  * the specification allows up to 8 PCIe endpoint functions. Invoke
  * pci_epc_add_epf() to add a PCI endpoint function to an endpoint controller.
  */
-int pci_epc_add_epf(struct pci_epc *epc, struct pci_epf *epf)
+int pci_epc_add_epf(struct pci_epc *epc, struct pci_epf *epf,
+		    enum pci_epc_interface_type type)
 {
-	unsigned long flags;
+	struct list_head *list;
+	u32 func_no = 0;
+	int ret = 0;
 
-	if (epf->epc)
-		return -EBUSY;
+	if (epf->is_vf)
+		return -EINVAL;
 
 	if (IS_ERR(epc))
 		return -EINVAL;
@@ -445,13 +699,37 @@
 	if (epf->func_no > epc->max_functions - 1)
 		return -EINVAL;
 
-	epf->epc = epc;
+	if (type == PRIMARY_INTERFACE && epf->epc)
+		return -EBUSY;
 
-	spin_lock_irqsave(&epc->lock, flags);
-	list_add_tail(&epf->list, &epc->pci_epf);
-	spin_unlock_irqrestore(&epc->lock, flags);
+	if (type == SECONDARY_INTERFACE && epf->sec_epc)
+		return -EBUSY;
+
+	mutex_lock(&epc->lock);
+	func_no = find_first_zero_bit(&epc->function_num_map,
+				      BITS_PER_LONG);
+	if (func_no >= BITS_PER_LONG) {
+		ret = -EINVAL;
+		goto err;
+	}
+
+	set_bit(func_no, &epc->function_num_map);
+	if (type == PRIMARY_INTERFACE) {
+		epf->func_no = func_no;
+		epf->epc = epc;
+		list = &epf->list;
+	} else {
+		epf->sec_epc_func_no = func_no;
+		epf->sec_epc = epc;
+		list = &epf->sec_epc_list;
+	}
+
+	list_add_tail(list, &epc->pci_epf);
 
-	return 0;
+err:
+	mutex_unlock(&epc->lock);
+
+	return ret;
 }
 EXPORT_SYMBOL_GPL(pci_epc_add_epf);
 
@@ -462,16 +740,28 @@
  *
  * Invoke to remove PCI endpoint function from the endpoint controller.
  */
-void pci_epc_remove_epf(struct pci_epc *epc, struct pci_epf *epf)
+void pci_epc_remove_epf(struct pci_epc *epc, struct pci_epf *epf,
+			enum pci_epc_interface_type type)
 {
-	unsigned long flags;
+	struct list_head *list;
+	u32 func_no = 0;
 
-	if (!epc || IS_ERR(epc))
+	if (!epc || IS_ERR(epc) || !epf)
 		return;
 
-	spin_lock_irqsave(&epc->lock, flags);
-	list_del(&epf->list);
-	spin_unlock_irqrestore(&epc->lock, flags);
+	if (type == PRIMARY_INTERFACE) {
+		func_no = epf->func_no;
+		list = &epf->list;
+	} else {
+		func_no = epf->sec_epc_func_no;
+		list = &epf->sec_epc_list;
+	}
+
+	mutex_lock(&epc->lock);
+	clear_bit(func_no, &epc->function_num_map);
+	list_del(list);
+	epf->epc = NULL;
+	mutex_unlock(&epc->lock);
 }
 EXPORT_SYMBOL_GPL(pci_epc_remove_epf);
 
@@ -485,16 +775,10 @@
  */
 void pci_epc_linkup(struct pci_epc *epc)
 {
-	unsigned long flags;
-	struct pci_epf *epf;
-
 	if (!epc || IS_ERR(epc))
 		return;
 
-	spin_lock_irqsave(&epc->lock, flags);
-	list_for_each_entry(epf, &epc->pci_epf, list)
-		pci_epf_linkup(epf);
-	spin_unlock_irqrestore(&epc->lock, flags);
+	atomic_notifier_call_chain(&epc->notifier, 0, NULL);
 }
 EXPORT_SYMBOL_GPL(pci_epc_linkup);
 
@@ -556,12 +840,14 @@
 		goto err_ret;
 	}
 
-	spin_lock_init(&epc->lock);
+	mutex_init(&epc->lock);
 	INIT_LIST_HEAD(&epc->pci_epf);
+	ATOMIC_INIT_NOTIFIER_HEAD(&epc->notifier);
 
 	device_initialize(&epc->dev);
 	epc->dev.class = pci_epc_class;
 	epc->dev.parent = dev;
+	epc->dev.of_node = dev->of_node;
 	epc->ops = ops;
 
 	ret = dev_set_name(&epc->dev, "%s", dev_name(dev));
diff -Naur --no-dereference a/drivers/pci/endpoint/pci-ep-cfs.c b/drivers/pci/endpoint/pci-ep-cfs.c
--- a/drivers/pci/endpoint/pci-ep-cfs.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/endpoint/pci-ep-cfs.c	2020-06-19 21:00:31.420227716 -0400
@@ -29,7 +29,6 @@
 	struct config_group group;
 	struct pci_epc *epc;
 	bool start;
-	unsigned long function_num_map;
 };
 
 static inline struct pci_epf_group *to_pci_epf_group(struct config_item *item)
@@ -89,37 +88,22 @@
 			    struct config_item *epf_item)
 {
 	int ret;
-	u32 func_no = 0;
 	struct pci_epf_group *epf_group = to_pci_epf_group(epf_item);
 	struct pci_epc_group *epc_group = to_pci_epc_group(epc_item);
 	struct pci_epc *epc = epc_group->epc;
 	struct pci_epf *epf = epf_group->epf;
 
-	func_no = find_first_zero_bit(&epc_group->function_num_map,
-				      BITS_PER_LONG);
-	if (func_no >= BITS_PER_LONG)
-		return -EINVAL;
-
-	set_bit(func_no, &epc_group->function_num_map);
-	epf->func_no = func_no;
-
-	ret = pci_epc_add_epf(epc, epf);
+	ret = pci_epc_add_epf(epc, epf, PRIMARY_INTERFACE);
 	if (ret)
-		goto err_add_epf;
+		return ret;
 
 	ret = pci_epf_bind(epf);
-	if (ret)
-		goto err_epf_bind;
+	if (ret) {
+		pci_epc_remove_epf(epc, epf, PRIMARY_INTERFACE);
+		return ret;
+	}
 
 	return 0;
-
-err_epf_bind:
-	pci_epc_remove_epf(epc, epf);
-
-err_add_epf:
-	clear_bit(func_no, &epc_group->function_num_map);
-
-	return ret;
 }
 
 static void pci_epc_epf_unlink(struct config_item *epc_item,
@@ -134,9 +118,8 @@
 
 	epc = epc_group->epc;
 	epf = epf_group->epf;
-	clear_bit(epf->func_no, &epc_group->function_num_map);
 	pci_epf_unbind(epf);
-	pci_epc_remove_epf(epc, epf);
+	pci_epc_remove_epf(epc, epf, PRIMARY_INTERFACE);
 }
 
 static struct configfs_item_operations pci_epc_item_ops = {
@@ -367,6 +350,28 @@
 	NULL,
 };
 
+static int pci_epf_vepf_link(struct config_item *epf_pf_item,
+			     struct config_item *epf_vf_item)
+{
+	struct pci_epf_group *epf_vf_group = to_pci_epf_group(epf_vf_item);
+	struct pci_epf_group *epf_pf_group = to_pci_epf_group(epf_pf_item);
+	struct pci_epf *epf_pf = epf_pf_group->epf;
+	struct pci_epf *epf_vf = epf_vf_group->epf;
+
+	return pci_epf_add_vepf(epf_pf, epf_vf);
+}
+
+static void pci_epf_vepf_unlink(struct config_item *epf_pf_item,
+				struct config_item *epf_vf_item)
+{
+	struct pci_epf_group *epf_vf_group = to_pci_epf_group(epf_vf_item);
+	struct pci_epf_group *epf_pf_group = to_pci_epf_group(epf_pf_item);
+	struct pci_epf *epf_pf = epf_pf_group->epf;
+	struct pci_epf *epf_vf = epf_vf_group->epf;
+
+	pci_epf_remove_vepf(epf_pf, epf_vf);
+}
+
 static void pci_epf_release(struct config_item *item)
 {
 	struct pci_epf_group *epf_group = to_pci_epf_group(item);
@@ -379,6 +384,8 @@
 }
 
 static struct configfs_item_operations pci_epf_ops = {
+	.allow_link		= pci_epf_vepf_link,
+	.drop_link		= pci_epf_vepf_unlink,
 	.release		= pci_epf_release,
 };
 
diff -Naur --no-dereference a/drivers/pci/endpoint/pci-epc-mem.c b/drivers/pci/endpoint/pci-epc-mem.c
--- a/drivers/pci/endpoint/pci-epc-mem.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/endpoint/pci-epc-mem.c	2020-06-19 21:00:31.420227716 -0400
@@ -79,6 +79,7 @@
 	mem->page_size = page_size;
 	mem->pages = pages;
 	mem->size = size;
+	mutex_init(&mem->lock);
 
 	epc->mem = mem;
 
@@ -122,7 +123,7 @@
 				     phys_addr_t *phys_addr, size_t size)
 {
 	int pageno;
-	void __iomem *virt_addr;
+	void __iomem *virt_addr = NULL;
 	struct pci_epc_mem *mem = epc->mem;
 	unsigned int page_shift = ilog2(mem->page_size);
 	int order;
@@ -130,15 +131,18 @@
 	size = ALIGN(size, mem->page_size);
 	order = pci_epc_mem_get_order(mem, size);
 
+	mutex_lock(&mem->lock);
 	pageno = bitmap_find_free_region(mem->bitmap, mem->pages, order);
 	if (pageno < 0)
-		return NULL;
+		goto ret;
 
 	*phys_addr = mem->phys_base + (pageno << page_shift);
 	virt_addr = ioremap(*phys_addr, size);
 	if (!virt_addr)
 		bitmap_release_region(mem->bitmap, pageno, order);
 
+ret:
+	mutex_unlock(&mem->lock);
 	return virt_addr;
 }
 EXPORT_SYMBOL_GPL(pci_epc_mem_alloc_addr);
@@ -164,7 +168,9 @@
 	pageno = (phys_addr - mem->phys_base) >> page_shift;
 	size = ALIGN(size, mem->page_size);
 	order = pci_epc_mem_get_order(mem, size);
+	mutex_lock(&mem->lock);
 	bitmap_release_region(mem->bitmap, pageno, order);
+	mutex_unlock(&mem->lock);
 }
 EXPORT_SYMBOL_GPL(pci_epc_mem_free_addr);
 
diff -Naur --no-dereference a/drivers/pci/endpoint/pci-epf-bus.c b/drivers/pci/endpoint/pci-epf-bus.c
--- a/drivers/pci/endpoint/pci-epf-bus.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/pci/endpoint/pci-epf-bus.c	2020-06-19 21:00:31.420227716 -0400
@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: GPL-2.0
+/**
+ * PCI Endpoint *Function* Bus Driver
+ *
+ * Copyright (C) 2019 Texas Instruments
+ * Author: Kishon Vijay Abraham I <kishon@ti.com>
+ */
+
+#include <linux/err.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/pci-epf.h>
+#include <linux/platform_device.h>
+
+static int pci_epf_bus_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *node = of_node_get(dev->of_node);
+	struct device_node *child;
+	struct pci_epf *epf;
+
+	for_each_child_of_node(node, child) {
+		epf = devm_pci_epf_of_create(dev, child);
+		if (IS_ERR(epf)) {
+			dev_err(dev, "Failed to create PCI EPF device %s\n",
+				node->full_name);
+			of_node_put(child);
+			break;
+		}
+	}
+	of_node_put(node);
+
+	return 0;
+}
+
+static const struct of_device_id pci_epf_bus_id_table[] = {
+	{ .compatible = "pci-epf-bus" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, pci_epf_bus_id_table);
+
+static struct platform_driver pci_epf_bus_driver = {
+	.probe		= pci_epf_bus_probe,
+	.driver		= {
+		.name	= "pci-epf-bus",
+		.of_match_table = of_match_ptr(pci_epf_bus_id_table),
+	},
+};
+
+module_platform_driver(pci_epf_bus_driver);
+
+MODULE_AUTHOR("Texas Instruments Inc.");
+MODULE_DESCRIPTION("PCI EPF Bus Driver");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/pci/endpoint/pci-epf-core.c b/drivers/pci/endpoint/pci-epf-core.c
--- a/drivers/pci/endpoint/pci-epf-core.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/endpoint/pci-epf-core.c	2020-06-19 21:00:31.420227716 -0400
@@ -8,9 +8,9 @@
 
 #include <linux/device.h>
 #include <linux/dma-mapping.h>
+#include <linux/dmaengine.h>
 #include <linux/slab.h>
 #include <linux/module.h>
-
 #include <linux/pci-epc.h>
 #include <linux/pci-epf.h>
 #include <linux/pci-ep-cfs.h>
@@ -20,25 +20,166 @@
 static struct bus_type pci_epf_bus_type;
 static const struct device_type pci_epf_type;
 
+static void pci_epf_dma_callback(void *param)
+{
+	struct pci_epf *epf = param;
+
+	complete(&epf->transfer_complete);
+}
+
 /**
- * pci_epf_linkup() - Notify the function driver that EPC device has
- *		      established a connection with the Root Complex.
- * @epf: the EPF device bound to the EPC device which has established
- *	 the connection with the host
+ * pci_epf_data_transfer() - Helper to use dmaengine API to transfer data
+ *			     between PCIe EP and remote PCIe RC
+ * @epf: the EPF device that performs the data transfer operation
+ * @dma_dst: The destination address of the data transfer. It can be a physical
+ *	     address given by pci_epc_mem_alloc_addr or DMA mapping APIs.
+ * @dma_src: The source address of the data transfer. It can be a physical
+ *	     address given by pci_epc_mem_alloc_addr or DMA mapping APIs.
+ * @len: The size of the data transfer
  *
- * Invoke to notify the function driver that EPC device has established
- * a connection with the Root Complex.
+ * Helper to use dmaengine API to transfer data between PCIe EP and remote PCIe
+ * RC. The source and destination address can be a physical address given by
+ * pci_epc_mem_alloc_addr or the one obtained using DMA mapping APIs.
+ *
+ * The function returns '0' on success and negative value on failure.
  */
-void pci_epf_linkup(struct pci_epf *epf)
+int pci_epf_data_transfer(struct pci_epf *epf, dma_addr_t dma_dst,
+			  dma_addr_t dma_src, size_t len)
 {
-	if (!epf->driver) {
-		dev_WARN(&epf->dev, "epf device not bound to driver\n");
+	enum dma_ctrl_flags flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
+	struct dma_chan *chan = epf->dma_chan;
+	struct dma_async_tx_descriptor *tx;
+	struct device *dev = &epf->dev;
+	dma_cookie_t cookie;
+	int ret;
+
+	if (IS_ERR_OR_NULL(epf)) {
+		dev_err(dev, "Invalid EPF device\n");
+		return -EINVAL;
+	}
+
+	if (IS_ERR_OR_NULL(chan)) {
+		dev_err(dev, "Invalid DMA memcpy channel\n");
+		return -EINVAL;
+	}
+
+	tx = dmaengine_prep_dma_memcpy(chan, dma_dst, dma_src, len, flags);
+	if (!tx) {
+		dev_err(dev, "Failed to prepare DMA memcpy\n");
+		return -EIO;
+	}
+
+	tx->callback = pci_epf_dma_callback;
+	tx->callback_param = epf;
+	cookie = tx->tx_submit(tx);
+	reinit_completion(&epf->transfer_complete);
+
+	ret = dma_submit_error(cookie);
+	if (ret) {
+		dev_err(dev, "Failed to do DMA tx_submit %d\n", cookie);
+		return -EIO;
+	}
+
+	dma_async_issue_pending(chan);
+	ret = wait_for_completion_interruptible(&epf->transfer_complete);
+	if (ret < 0) {
+		dmaengine_terminate_sync(chan);
+		dev_err(dev, "DMA wait_for_completion_timeout\n");
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(pci_epf_data_transfer);
+
+/**
+ * pci_epf_init_dma_chan() - Helper to initialize EPF DMA channel
+ * @epf: the EPF device that has to perform the data transfer operation
+ *
+ * Helper to initialize EPF DMA channel.
+ */
+int pci_epf_init_dma_chan(struct pci_epf *epf)
+{
+	struct device *dev = &epf->dev;
+	struct dma_chan *dma_chan;
+	dma_cap_mask_t mask;
+	int ret;
+
+	if (IS_ERR_OR_NULL(epf)) {
+		dev_err(dev, "Invalid EPF device\n");
+		return -EINVAL;
+	}
+
+	dma_cap_zero(mask);
+	dma_cap_set(DMA_MEMCPY, mask);
+
+	dma_chan = dma_request_chan_by_mask(&mask);
+	if (IS_ERR(dma_chan)) {
+		ret = PTR_ERR(dma_chan);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "Failed to get DMA channel\n");
+		return ret;
+	}
+	init_completion(&epf->transfer_complete);
+
+	epf->dma_chan = dma_chan;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(pci_epf_init_dma_chan);
+
+/**
+ * pci_epf_clean_dma_chan() - Helper to cleanup EPF DMA channel
+ * @epf: the EPF device that performed the data transfer operation
+ *
+ * Helper to cleanup EPF DMA channel.
+ */
+void pci_epf_clean_dma_chan(struct pci_epf *epf)
+{
+	struct device *dev = &epf->dev;
+
+	if (IS_ERR_OR_NULL(epf)) {
+		dev_err(dev, "Invalid EPF device\n");
 		return;
 	}
 
-	epf->driver->ops->linkup(epf);
+	dma_release_channel(epf->dma_chan);
+	epf->dma_chan = NULL;
+}
+EXPORT_SYMBOL_GPL(pci_epf_clean_dma_chan);
+
+/**
+ * pci_epf_tx() - transfer data between EPC and remote PCIe RC
+ * @epf: the EPF device that performs the data transfer operation
+ * @dma_dst: The destination address of the data transfer. It can be a physical
+ *	     address given by pci_epc_mem_alloc_addr or DMA mapping APIs.
+ * @dma_src: The source address of the data transfer. It can be a physical
+ *	     address given by pci_epc_mem_alloc_addr or DMA mapping APIs.
+ * @len: The size of the data transfer
+ *
+ * Invoke to transfer data between EPC and remote PCIe RC. The source and
+ * destination address can be a physical address given by pci_epc_mem_alloc_addr
+ * or the one obtained using DMA mapping APIs.
+ */
+int pci_epf_tx(struct pci_epf *epf, dma_addr_t dma_dst,
+	       dma_addr_t dma_src, size_t len)
+{
+	int ret;
+	struct pci_epc *epc = epf->epc;
+
+	if (IS_ERR_OR_NULL(epc) || IS_ERR_OR_NULL(epf))
+		return -EINVAL;
+
+	if (!epc->ops->data_transfer)
+		return -EINVAL;
+
+	mutex_lock(&epf->lock);
+	ret = epc->ops->data_transfer(epc, epf, dma_dst, dma_src, len);
+	mutex_unlock(&epf->lock);
+
+	return ret;
 }
-EXPORT_SYMBOL_GPL(pci_epf_linkup);
+EXPORT_SYMBOL_GPL(pci_epf_tx);
 
 /**
  * pci_epf_unbind() - Notify the function driver that the binding between the
@@ -50,12 +191,21 @@
  */
 void pci_epf_unbind(struct pci_epf *epf)
 {
+	struct pci_epf *epf_vf;
+
 	if (!epf->driver) {
 		dev_WARN(&epf->dev, "epf device not bound to driver\n");
 		return;
 	}
 
-	epf->driver->ops->unbind(epf);
+	mutex_lock(&epf->lock);
+	list_for_each_entry(epf_vf, &epf->pci_vepf, list) {
+		if (epf_vf->is_bound)
+			epf_vf->driver->ops->unbind(epf_vf);
+	}
+	if (epf->is_bound)
+		epf->driver->ops->unbind(epf);
+	mutex_unlock(&epf->lock);
 	module_put(epf->driver->owner);
 }
 EXPORT_SYMBOL_GPL(pci_epf_unbind);
@@ -69,6 +219,9 @@
  */
 int pci_epf_bind(struct pci_epf *epf)
 {
+	struct pci_epf *epf_vf;
+	int ret;
+
 	if (!epf->driver) {
 		dev_WARN(&epf->dev, "epf device not bound to driver\n");
 		return -EINVAL;
@@ -77,31 +230,127 @@
 	if (!try_module_get(epf->driver->owner))
 		return -EAGAIN;
 
-	return epf->driver->ops->bind(epf);
+	mutex_lock(&epf->lock);
+	list_for_each_entry(epf_vf, &epf->pci_vepf, list) {
+		epf_vf->func_no = epf->func_no;
+		epf_vf->epc = epf->epc;
+		ret = epf_vf->driver->ops->bind(epf_vf);
+		if (ret)
+			goto ret;
+		epf_vf->is_bound = true;
+	}
+
+	ret = epf->driver->ops->bind(epf);
+	if (ret)
+		goto ret;
+	epf->is_bound = true;
+
+	mutex_unlock(&epf->lock);
+	return 0;
+
+ret:
+	mutex_unlock(&epf->lock);
+	pci_epf_unbind(epf);
+
+	return ret;
 }
 EXPORT_SYMBOL_GPL(pci_epf_bind);
 
 /**
+ * pci_epf_add_vepf() - associate virtual EP function to physical EP function
+ * @epf_pf: the physical EP function to which the virtual EP function should be
+ *   associated
+ * @epf_vf: the virtual EP function to be added
+ *
+ * A physical endpoint function can be associated with multiple virtual
+ * endpoint functions. Invoke pci_epf_add_epf() to add a virtual PCI endpoint
+ * function to a physical PCI endpoint function.
+ */
+int pci_epf_add_vepf(struct pci_epf *epf_pf, struct pci_epf *epf_vf)
+{
+	u32 vfunc_no;
+
+	if (IS_ERR_OR_NULL(epf_pf) || IS_ERR_OR_NULL(epf_vf))
+		return -EINVAL;
+
+	if (epf_pf->epc || epf_vf->epc || epf_vf->epf_pf)
+		return -EBUSY;
+
+	mutex_lock(&epf_pf->lock);
+	vfunc_no = find_first_zero_bit(&epf_pf->vfunction_num_map,
+				       BITS_PER_LONG);
+	if (vfunc_no >= BITS_PER_LONG)
+		return -EINVAL;
+
+	set_bit(vfunc_no, &epf_pf->vfunction_num_map);
+	epf_vf->vfunc_no = vfunc_no;
+
+	epf_vf->epf_pf = epf_pf;
+	epf_vf->is_vf = true;
+
+	list_add_tail(&epf_vf->list, &epf_pf->pci_vepf);
+	mutex_unlock(&epf_pf->lock);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(pci_epf_add_vepf);
+
+/**
+ * pci_epf_remove_vepf() - remove virtual EP function from physical EP function
+ * @epf_pf: the physical EP function from which the virtual EP function should
+ *   be removed
+ * @epf_vf: the virtual EP function to be removed
+ *
+ * Invoke to remove a virtual endpoint function from the physcial endpoint
+ * function.
+ */
+void pci_epf_remove_vepf(struct pci_epf *epf_pf, struct pci_epf *epf_vf)
+{
+	if (IS_ERR_OR_NULL(epf_pf) || IS_ERR_OR_NULL(epf_vf))
+		return;
+
+	mutex_lock(&epf_pf->lock);
+	clear_bit(epf_vf->vfunc_no, &epf_pf->vfunction_num_map);
+	list_del(&epf_vf->list);
+	mutex_unlock(&epf_pf->lock);
+}
+EXPORT_SYMBOL_GPL(pci_epf_remove_vepf);
+
+/**
  * pci_epf_free_space() - free the allocated PCI EPF register space
  * @addr: the virtual address of the PCI EPF register space
  * @bar: the BAR number corresponding to the register space
+ * @type: Identifies if the allocated space is for primary EPC or secondary EPC
  *
  * Invoke to free the allocated PCI EPF register space.
  */
-void pci_epf_free_space(struct pci_epf *epf, void *addr, enum pci_barno bar)
+void pci_epf_free_space(struct pci_epf *epf, void *addr, enum pci_barno bar,
+			enum pci_epc_interface_type type)
 {
 	struct device *dev = epf->epc->dev.parent;
+	struct pci_epf_bar *epf_bar;
+	struct pci_epc *epc;
 
 	if (!addr)
 		return;
 
-	dma_free_coherent(dev, epf->bar[bar].size, addr,
-			  epf->bar[bar].phys_addr);
+	if (type == PRIMARY_INTERFACE) {
+		epc = epf->epc;
+		epf_bar = epf->bar;
+	} else {
+		epc = epf->sec_epc;
+		epf_bar = epf->sec_epc_bar;
+	}
 
-	epf->bar[bar].phys_addr = 0;
-	epf->bar[bar].size = 0;
-	epf->bar[bar].barno = 0;
-	epf->bar[bar].flags = 0;
+	dev = epc->dev.parent;
+	dma_free_coherent(dev, epf_bar[bar].size, addr,
+			  epf_bar[bar].phys_addr);
+
+	epf_bar[bar].phys_addr = 0;
+	epf_bar[bar].addr = NULL;
+	epf_bar[bar].size = 0;
+	epf_bar[bar].barno = 0;
+	epf_bar[bar].flags = 0;
 }
 EXPORT_SYMBOL_GPL(pci_epf_free_space);
 
@@ -109,29 +358,50 @@
  * pci_epf_alloc_space() - allocate memory for the PCI EPF register space
  * @size: the size of the memory that has to be allocated
  * @bar: the BAR number corresponding to the allocated register space
+ * @align: alignment size for the allocation region
+ * @type: Identifies if the allocation is for primary EPC or secondary EPC
  *
  * Invoke to allocate memory for the PCI EPF register space.
  */
-void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar)
+void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar,
+			  size_t align, enum pci_epc_interface_type type)
 {
-	void *space;
-	struct device *dev = epf->epc->dev.parent;
+	struct pci_epf_bar *epf_bar;
 	dma_addr_t phys_addr;
+	struct pci_epc *epc;
+	struct device *dev;
+	void *space;
 
 	if (size < 128)
 		size = 128;
-	size = roundup_pow_of_two(size);
 
+	if (align)
+		size = ALIGN(size, align);
+	else
+		size = roundup_pow_of_two(size);
+
+	if (type == PRIMARY_INTERFACE) {
+		epc = epf->epc;
+		epf_bar = epf->bar;
+	} else {
+		epc = epf->sec_epc;
+		epf_bar = epf->sec_epc_bar;
+	}
+
+	dev = epc->dev.parent;
 	space = dma_alloc_coherent(dev, size, &phys_addr, GFP_KERNEL);
 	if (!space) {
 		dev_err(dev, "failed to allocate mem space\n");
 		return NULL;
 	}
 
-	epf->bar[bar].phys_addr = phys_addr;
-	epf->bar[bar].size = size;
-	epf->bar[bar].barno = bar;
-	epf->bar[bar].flags = PCI_BASE_ADDRESS_SPACE_MEMORY;
+	epf_bar[bar].phys_addr = phys_addr;
+	epf_bar[bar].addr = space;
+	epf_bar[bar].size = size;
+	epf_bar[bar].barno = bar;
+	epf_bar[bar].flags |= upper_32_bits(size) ?
+				PCI_BASE_ADDRESS_MEM_TYPE_64 :
+				PCI_BASE_ADDRESS_MEM_TYPE_32;
 
 	return space;
 }
@@ -203,11 +473,9 @@
 {
 	int ret;
 
-	if (!driver->ops)
-		return -EINVAL;
-
-	if (!driver->ops->bind || !driver->ops->unbind || !driver->ops->linkup)
-		return -EINVAL;
+	if (!driver->ops || !driver->ops->bind || !driver->ops->unbind)
+		pr_debug("%s: Supports only pci_epf device created using DT\n",
+			 driver->driver.name);
 
 	driver->driver.bus = &pci_epf_bus_type;
 	driver->driver.owner = owner;
@@ -260,10 +528,15 @@
 		return ERR_PTR(-ENOMEM);
 	}
 
+	/* VFs are numbered starting with 1. So set BIT(0) by default */
+	epf->vfunction_num_map = 1;
+	INIT_LIST_HEAD(&epf->pci_vepf);
+
 	dev = &epf->dev;
 	device_initialize(dev);
 	dev->bus = &pci_epf_bus_type;
 	dev->type = &pci_epf_type;
+	mutex_init(&epf->lock);
 
 	ret = dev_set_name(dev, "%s", name);
 	if (ret) {
@@ -281,26 +554,78 @@
 }
 EXPORT_SYMBOL_GPL(pci_epf_create);
 
-const struct pci_epf_device_id *
-pci_epf_match_device(const struct pci_epf_device_id *id, struct pci_epf *epf)
+/**
+ * pci_epf_of_create() - create a new PCI EPF device from device tree node
+ * @node: the device node of the PCI EPF device.
+ *
+ * Invoke to create a new PCI EPF device by providing a device tree node
+ * with compatible property.
+ */
+struct pci_epf *pci_epf_of_create(struct device_node *node)
 {
-	if (!id || !epf)
-		return NULL;
+	struct pci_epf *epf;
+	const char *compat;
+	int ret;
 
-	while (*id->name) {
-		if (strcmp(epf->name, id->name) == 0)
-			return id;
-		id++;
+	of_node_get(node);
+
+	ret = of_property_read_string(node, "compatible", &compat);
+	if (ret) {
+		of_node_put(node);
+		return ERR_PTR(ret);
+	}
+
+	epf = pci_epf_create(compat);
+	if (!IS_ERR(epf))
+		epf->node = node;
+
+	return epf;
+}
+EXPORT_SYMBOL_GPL(pci_epf_of_create);
+
+static void devm_epf_release(struct device *dev, void *res)
+{
+	struct pci_epf *epf = *(struct pci_epf **)res;
+
+	pci_epf_destroy(epf);
+}
+
+/**
+ * devm_pci_epf_of_create() - create a new PCI EPF device from device tree node
+ * @dev: device that is creating the new EPF
+ * @node: the device node of the PCI EPF device.
+ *
+ * Invoke to create a new PCI EPF device by providing a device tree node with
+ * compatible property. While at that, it also associates the device with the
+ * EPF using devres. On driver detach, release function is invoked on the devres
+ * data, where devres data is freed.
+ */
+struct pci_epf *devm_pci_epf_of_create(struct device *dev,
+				       struct device_node *node)
+{
+	struct pci_epf **ptr, *epf;
+
+	ptr = devres_alloc(devm_epf_release, sizeof(*ptr), GFP_KERNEL);
+	if (!ptr)
+		return ERR_PTR(-ENOMEM);
+
+	epf = pci_epf_of_create(node);
+	if (!IS_ERR(epf)) {
+		*ptr = epf;
+		devres_add(dev, ptr);
+	} else {
+		devres_free(ptr);
 	}
 
-	return NULL;
+	return epf;
 }
-EXPORT_SYMBOL_GPL(pci_epf_match_device);
+EXPORT_SYMBOL_GPL(devm_pci_epf_of_create);
 
 static void pci_epf_dev_release(struct device *dev)
 {
 	struct pci_epf *epf = to_pci_epf(dev);
 
+	of_node_put(epf->node);
 	kfree(epf->name);
 	kfree(epf);
 }
diff -Naur --no-dereference a/drivers/pci/Makefile b/drivers/pci/Makefile
--- a/drivers/pci/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/Makefile	2020-06-19 21:00:31.420227716 -0400
@@ -10,9 +10,9 @@
 ifdef CONFIG_PCI
 obj-$(CONFIG_PROC_FS)		+= proc.o
 obj-$(CONFIG_SYSFS)		+= slot.o
-obj-$(CONFIG_OF)		+= of.o
 endif
 
+obj-$(CONFIG_OF)		+= of.o
 obj-$(CONFIG_PCI_QUIRKS)	+= quirks.o
 obj-$(CONFIG_PCIEPORTBUS)	+= pcie/
 obj-$(CONFIG_HOTPLUG_PCI)	+= hotplug/
diff -Naur --no-dereference a/drivers/pci/of.c b/drivers/pci/of.c
--- a/drivers/pci/of.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pci/of.c	2020-06-19 21:00:31.420227716 -0400
@@ -15,6 +15,7 @@
 #include <linux/of_pci.h>
 #include "pci.h"
 
+#ifdef CONFIG_PCI
 void pci_set_of_node(struct pci_dev *dev)
 {
 	if (!dev->bus->dev.of_node)
@@ -197,27 +198,6 @@
 EXPORT_SYMBOL_GPL(of_get_pci_domain_nr);
 
 /**
- * This function will try to find the limitation of link speed by finding
- * a property called "max-link-speed" of the given device node.
- *
- * @node: device tree node with the max link speed information
- *
- * Returns the associated max link speed from DT, or a negative value if the
- * required property is not found or is invalid.
- */
-int of_pci_get_max_link_speed(struct device_node *node)
-{
-	u32 max_link_speed;
-
-	if (of_property_read_u32(node, "max-link-speed", &max_link_speed) ||
-	    max_link_speed > 4)
-		return -EINVAL;
-
-	return max_link_speed;
-}
-EXPORT_SYMBOL_GPL(of_pci_get_max_link_speed);
-
-/**
  * of_pci_check_probe_only - Setup probe only mode if linux,pci-probe-only
  *                           is present and valid
  */
@@ -638,3 +618,25 @@
 	return err;
 }
 
+#endif /* CONFIG_PCI */
+
+/**
+ * This function will try to find the limitation of link speed by finding
+ * a property called "max-link-speed" of the given device node.
+ *
+ * @node: device tree node with the max link speed information
+ *
+ * Returns the associated max link speed from DT, or a negative value if the
+ * required property is not found or is invalid.
+ */
+int of_pci_get_max_link_speed(struct device_node *node)
+{
+	u32 max_link_speed;
+
+	if (of_property_read_u32(node, "max-link-speed", &max_link_speed) ||
+	    max_link_speed > 4)
+		return -EINVAL;
+
+	return max_link_speed;
+}
+EXPORT_SYMBOL_GPL(of_pci_get_max_link_speed);
diff -Naur --no-dereference a/drivers/phy/allwinner/phy-sun4i-usb.c b/drivers/phy/allwinner/phy-sun4i-usb.c
--- a/drivers/phy/allwinner/phy-sun4i-usb.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/phy/allwinner/phy-sun4i-usb.c	2020-06-19 21:00:31.420227716 -0400
@@ -475,7 +475,8 @@
 	return 0;
 }
 
-static int sun4i_usb_phy_set_mode(struct phy *_phy, enum phy_mode mode)
+static int sun4i_usb_phy_set_mode(struct phy *_phy,
+				  enum phy_mode mode, int submode)
 {
 	struct sun4i_usb_phy *phy = phy_get_drvdata(_phy);
 	struct sun4i_usb_phy_data *data = to_sun4i_usb_phy_data(phy);
diff -Naur --no-dereference a/drivers/phy/amlogic/phy-meson-gxl-usb2.c b/drivers/phy/amlogic/phy-meson-gxl-usb2.c
--- a/drivers/phy/amlogic/phy-meson-gxl-usb2.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/phy/amlogic/phy-meson-gxl-usb2.c	2020-06-19 21:00:31.420227716 -0400
@@ -152,7 +152,8 @@
 	return 0;
 }
 
-static int phy_meson_gxl_usb2_set_mode(struct phy *phy, enum phy_mode mode)
+static int phy_meson_gxl_usb2_set_mode(struct phy *phy,
+				       enum phy_mode mode, int submode)
 {
 	struct phy_meson_gxl_usb2_priv *priv = phy_get_drvdata(phy);
 
@@ -209,7 +210,7 @@
 	/* power on the PHY by taking it out of reset mode */
 	regmap_update_bits(priv->regmap, U2P_R0, U2P_R0_POWER_ON_RESET, 0);
 
-	ret = phy_meson_gxl_usb2_set_mode(phy, priv->mode);
+	ret = phy_meson_gxl_usb2_set_mode(phy, priv->mode, 0);
 	if (ret) {
 		phy_meson_gxl_usb2_power_off(phy);
 
diff -Naur --no-dereference a/drivers/phy/amlogic/phy-meson-gxl-usb3.c b/drivers/phy/amlogic/phy-meson-gxl-usb3.c
--- a/drivers/phy/amlogic/phy-meson-gxl-usb3.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/phy/amlogic/phy-meson-gxl-usb3.c	2020-06-19 21:00:31.420227716 -0400
@@ -119,7 +119,8 @@
 	return 0;
 }
 
-static int phy_meson_gxl_usb3_set_mode(struct phy *phy, enum phy_mode mode)
+static int phy_meson_gxl_usb3_set_mode(struct phy *phy,
+				       enum phy_mode mode, int submode)
 {
 	struct phy_meson_gxl_usb3_priv *priv = phy_get_drvdata(phy);
 
@@ -164,7 +165,7 @@
 	if (ret)
 		goto err_disable_clk_phy;
 
-	ret = phy_meson_gxl_usb3_set_mode(phy, priv->mode);
+	ret = phy_meson_gxl_usb3_set_mode(phy, priv->mode, 0);
 	if (ret)
 		goto err_disable_clk_peripheral;
 
diff -Naur --no-dereference a/drivers/phy/cadence/Kconfig b/drivers/phy/cadence/Kconfig
--- a/drivers/phy/cadence/Kconfig	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/phy/cadence/Kconfig	2020-06-19 21:00:31.420227716 -0400
@@ -0,0 +1,17 @@
+#
+# Phy drivers for Cadence PHYs
+#
+config PHY_CADENCE_TORRENT
+	tristate "Cadence Torrent PHY driver"
+	depends on OF
+	depends on HAS_IOMEM
+	select GENERIC_PHY
+	help
+	  Support for Cadence Torrent PHY.
+
+config PHY_CADENCE_SIERRA
+	tristate "Cadence Sierra PHY Driver"
+	depends on OF && HAS_IOMEM && RESET_CONTROLLER
+	select GENERIC_PHY
+	help
+	  Enable this to support the Cadence Sierra PHY driver
diff -Naur --no-dereference a/drivers/phy/cadence/Makefile b/drivers/phy/cadence/Makefile
--- a/drivers/phy/cadence/Makefile	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/phy/cadence/Makefile	2020-06-19 21:00:31.420227716 -0400
@@ -0,0 +1,2 @@
+obj-$(CONFIG_PHY_CADENCE_TORRENT)	+= phy-cadence-torrent.o
+obj-$(CONFIG_PHY_CADENCE_SIERRA)	+= phy-cadence-sierra.o
diff -Naur --no-dereference a/drivers/phy/cadence/phy-cadence-sierra.c b/drivers/phy/cadence/phy-cadence-sierra.c
--- a/drivers/phy/cadence/phy-cadence-sierra.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/phy/cadence/phy-cadence-sierra.c	2020-06-19 21:00:31.420227716 -0400
@@ -0,0 +1,834 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Cadence Sierra PHY Driver
+ *
+ * Copyright (c) 2018 Cadence Design Systems
+ * Author: Alan Douglas <adouglas@cadence.com>
+ *
+ */
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+#include <linux/slab.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <dt-bindings/phy/phy.h>
+
+/* PHY register offsets */
+#define SIERRA_COMMON_CDB_OFFSET			0x0
+#define SIERRA_MACRO_ID_REG				0x0
+#define SIERRA_CMN_PLLLC_MODE_PREG			0x48
+#define SIERRA_CMN_PLLLC_LF_COEFF_MODE1_PREG		0x49
+#define SIERRA_CMN_PLLLC_LF_COEFF_MODE0_PREG		0x4A
+#define SIERRA_CMN_PLLLC_LOCK_CNTSTART_PREG		0x4B
+#define SIERRA_CMN_PLLLC_BWCAL_MODE1_PREG		0x4F
+#define SIERRA_CMN_PLLLC_BWCAL_MODE0_PREG		0x50
+#define SIERRA_CMN_PLLLC_SS_TIME_STEPSIZE_MODE_PREG	0x62
+
+#define SIERRA_LANE_CDB_OFFSET(ln, offset)	\
+				(0x4000 + ((ln) * (0x800 >> (2 - (offset)))))
+
+#define SIERRA_DET_STANDEC_A_PREG			0x000
+#define SIERRA_DET_STANDEC_B_PREG			0x001
+#define SIERRA_DET_STANDEC_C_PREG			0x002
+#define SIERRA_DET_STANDEC_D_PREG			0x003
+#define SIERRA_DET_STANDEC_E_PREG			0x004
+#define SIERRA_PSM_LANECAL_DLY_A1_RESETS_PREG		0x008
+#define SIERRA_PSM_A0IN_TMR_PREG			0x009
+#define SIERRA_PSM_DIAG_PREG				0x015
+#define SIERRA_PSC_TX_A0_PREG				0x028
+#define SIERRA_PSC_TX_A1_PREG				0x029
+#define SIERRA_PSC_TX_A2_PREG				0x02A
+#define SIERRA_PSC_TX_A3_PREG				0x02B
+#define SIERRA_PSC_RX_A0_PREG				0x030
+#define SIERRA_PSC_RX_A1_PREG				0x031
+#define SIERRA_PSC_RX_A2_PREG				0x032
+#define SIERRA_PSC_RX_A3_PREG				0x033
+#define SIERRA_PLLCTRL_SUBRATE_PREG			0x03A
+#define SIERRA_PLLCTRL_GEN_D_PREG			0x03E
+#define SIERRA_PLLCTRL_CPGAIN_MODE_PREG			0x03F
+#define SIERRA_PLLCTRL_STATUS_PREG			0x044
+#define SIERRA_CLKPATH_BIASTRIM_PREG			0x04B
+#define SIERRA_DFE_BIASTRIM_PREG			0x04C
+#define SIERRA_DRVCTRL_ATTEN_PREG			0x06A
+#define SIERRA_CLKPATHCTRL_TMR_PREG			0x081
+#define SIERRA_RX_CREQ_FLTR_A_MODE3_PREG		0x085
+#define SIERRA_RX_CREQ_FLTR_A_MODE2_PREG		0x086
+#define SIERRA_RX_CREQ_FLTR_A_MODE1_PREG		0x087
+#define SIERRA_RX_CREQ_FLTR_A_MODE0_PREG		0x088
+#define SIERRA_CREQ_CCLKDET_MODE01_PREG			0x08E
+#define SIERRA_RX_CTLE_MAINTENANCE_PREG			0x091
+#define SIERRA_CREQ_FSMCLK_SEL_PREG			0x092
+#define SIERRA_CREQ_EQ_CTRL_PREG			0x093
+#define SIERRA_CREQ_SPARE_PREG				0x096
+#define SIERRA_CREQ_EQ_OPEN_EYE_THRESH_PREG		0x097
+#define SIERRA_CTLELUT_CTRL_PREG			0x098
+#define SIERRA_DFE_ECMP_RATESEL_PREG			0x0C0
+#define SIERRA_DFE_SMP_RATESEL_PREG			0x0C1
+#define SIERRA_DEQ_PHALIGN_CTRL				0x0C4
+#define SIERRA_DEQ_CONCUR_CTRL1_PREG			0x0C8
+#define SIERRA_DEQ_CONCUR_CTRL2_PREG			0x0C9
+#define SIERRA_DEQ_EPIPWR_CTRL2_PREG			0x0CD
+#define SIERRA_DEQ_FAST_MAINT_CYCLES_PREG		0x0CE
+#define SIERRA_DEQ_ERRCMP_CTRL_PREG			0x0D0
+#define SIERRA_DEQ_OFFSET_CTRL_PREG			0x0D8
+#define SIERRA_DEQ_GAIN_CTRL_PREG			0x0E0
+#define SIERRA_DEQ_VGATUNE_CTRL_PREG			0x0E1
+#define SIERRA_DEQ_GLUT0				0x0E8
+#define SIERRA_DEQ_GLUT1				0x0E9
+#define SIERRA_DEQ_GLUT2				0x0EA
+#define SIERRA_DEQ_GLUT3				0x0EB
+#define SIERRA_DEQ_GLUT4				0x0EC
+#define SIERRA_DEQ_GLUT5				0x0ED
+#define SIERRA_DEQ_GLUT6				0x0EE
+#define SIERRA_DEQ_GLUT7				0x0EF
+#define SIERRA_DEQ_GLUT8				0x0F0
+#define SIERRA_DEQ_GLUT9				0x0F1
+#define SIERRA_DEQ_GLUT10				0x0F2
+#define SIERRA_DEQ_GLUT11				0x0F3
+#define SIERRA_DEQ_GLUT12				0x0F4
+#define SIERRA_DEQ_GLUT13				0x0F5
+#define SIERRA_DEQ_GLUT14				0x0F6
+#define SIERRA_DEQ_GLUT15				0x0F7
+#define SIERRA_DEQ_GLUT16				0x0F8
+#define SIERRA_DEQ_ALUT0				0x108
+#define SIERRA_DEQ_ALUT1				0x109
+#define SIERRA_DEQ_ALUT2				0x10A
+#define SIERRA_DEQ_ALUT3				0x10B
+#define SIERRA_DEQ_ALUT4				0x10C
+#define SIERRA_DEQ_ALUT5				0x10D
+#define SIERRA_DEQ_ALUT6				0x10E
+#define SIERRA_DEQ_ALUT7				0x10F
+#define SIERRA_DEQ_ALUT8				0x110
+#define SIERRA_DEQ_ALUT9				0x111
+#define SIERRA_DEQ_ALUT10				0x112
+#define SIERRA_DEQ_ALUT11				0x113
+#define SIERRA_DEQ_ALUT12				0x114
+#define SIERRA_DEQ_ALUT13				0x115
+#define SIERRA_DEQ_DFETAP_CTRL_PREG			0x128
+#define SIERRA_DFE_EN_1010_IGNORE_PREG			0x134
+#define SIERRA_DEQ_TAU_CTRL1_SLOW_MAINT_PREG		0x150
+#define SIERRA_DEQ_TAU_CTRL2_PREG			0x151
+#define SIERRA_DEQ_PICTRL_PREG				0x161
+#define SIERRA_CPICAL_TMRVAL_MODE1_PREG			0x170
+#define SIERRA_CPICAL_TMRVAL_MODE0_PREG			0x171
+#define SIERRA_CPICAL_PICNT_MODE1_PREG			0x174
+#define SIERRA_CPI_OUTBUF_RATESEL_PREG			0x17C
+#define SIERRA_CPICAL_RES_STARTCODE_MODE23_PREG		0x183
+#define SIERRA_LFPSDET_SUPPORT_PREG			0x188
+#define SIERRA_LFPSFILT_NS_PREG				0x18A
+#define SIERRA_LFPSFILT_RD_PREG				0x18B
+#define SIERRA_LFPSFILT_MP_PREG				0x18C
+#define SIERRA_SIGDET_SUPPORT_PREG			0x190
+#define SIERRA_SDFILT_H2L_A_PREG			0x191
+#define SIERRA_SDFILT_L2H_PREG				0x193
+#define SIERRA_RXBUFFER_CTLECTRL_PREG			0x19E
+#define SIERRA_RXBUFFER_RCDFECTRL_PREG			0x19F
+#define SIERRA_RXBUFFER_DFECTRL_PREG			0x1A0
+#define SIERRA_DEQ_TAU_CTRL1_FAST_MAINT_PREG		0x14F
+#define SIERRA_DEQ_TAU_CTRL1_SLOW_MAINT_PREG		0x150
+
+#define SIERRA_PHY_CONFIG_CTRL_OFFSET			0xc000
+#define SIERRA_PHY_PLL_CFG				0xe
+
+#define SIERRA_MACRO_ID					0x00007364
+#define SIERRA_MAX_LANES				16
+#define PLL_LOCK_TIME					100000
+
+static const struct reg_field macro_id_type =
+				REG_FIELD(SIERRA_MACRO_ID_REG, 0, 15);
+static const struct reg_field phy_pll_cfg_1 =
+				REG_FIELD(SIERRA_PHY_PLL_CFG, 1, 1);
+static const struct reg_field pllctrl_lock =
+				REG_FIELD(SIERRA_PLLCTRL_STATUS_PREG, 0, 0);
+
+struct cdns_sierra_inst {
+	struct phy *phy;
+	u32 phy_type;
+	u32 num_lanes;
+	u32 mlane;
+	struct reset_control *lnk_rst;
+};
+
+struct cdns_reg_pairs {
+	u16 val;
+	u32 off;
+};
+
+struct cdns_sierra_data {
+		u32 id_value;
+		u8 block_offset_shift;
+		u8 reg_offset_shift;
+		u32 pcie_cmn_regs;
+		u32 pcie_ln_regs;
+		u32 usb_cmn_regs;
+		u32 usb_ln_regs;
+		struct cdns_reg_pairs *pcie_cmn_vals;
+		struct cdns_reg_pairs *pcie_ln_vals;
+		struct cdns_reg_pairs *usb_cmn_vals;
+		struct cdns_reg_pairs *usb_ln_vals;
+};
+
+struct cdns_regmap_cdb_context {
+	struct device *dev;
+	void __iomem *base;
+	u8 reg_offset_shift;
+};
+
+struct cdns_sierra_phy {
+	struct device *dev;
+	struct regmap *regmap;
+	struct cdns_sierra_data *init_data;
+	struct cdns_sierra_inst phys[SIERRA_MAX_LANES];
+	struct reset_control *phy_rst;
+	struct reset_control *apb_rst;
+	struct regmap *regmap_lane_cdb[SIERRA_MAX_LANES];
+	struct regmap *regmap_phy_config_ctrl;
+	struct regmap *regmap_common_cdb;
+	struct regmap_field *macro_id_type;
+	struct regmap_field *phy_pll_cfg_1;
+	struct regmap_field *pllctrl_lock[SIERRA_MAX_LANES];
+	struct clk *clk;
+	struct clk *cmn_refclk;
+	struct clk *cmn_refclk1;
+	int nsubnodes;
+	u32 num_lanes;
+	bool autoconf;
+};
+
+static int cdns_regmap_write(void *context, unsigned int reg, unsigned int val)
+{
+	struct cdns_regmap_cdb_context *ctx = context;
+	u32 offset = reg << ctx->reg_offset_shift;
+
+	writew(val, ctx->base + offset);
+
+	return 0;
+}
+
+static int cdns_regmap_read(void *context, unsigned int reg, unsigned int *val)
+{
+	struct cdns_regmap_cdb_context *ctx = context;
+	u32 offset = reg << ctx->reg_offset_shift;
+
+	*val = readw(ctx->base + offset);
+	return 0;
+}
+
+#define SIERRA_LANE_CDB_REGMAP_CONF(n) \
+{ \
+	.name = "sierra_lane" n "_cdb", \
+	.reg_stride = 1, \
+	.fast_io = true, \
+	.reg_write = cdns_regmap_write, \
+	.reg_read = cdns_regmap_read, \
+}
+
+static struct regmap_config cdns_sierra_lane_cdb_config[] = {
+	SIERRA_LANE_CDB_REGMAP_CONF("0"),
+	SIERRA_LANE_CDB_REGMAP_CONF("1"),
+	SIERRA_LANE_CDB_REGMAP_CONF("2"),
+	SIERRA_LANE_CDB_REGMAP_CONF("3"),
+	SIERRA_LANE_CDB_REGMAP_CONF("4"),
+	SIERRA_LANE_CDB_REGMAP_CONF("5"),
+	SIERRA_LANE_CDB_REGMAP_CONF("6"),
+	SIERRA_LANE_CDB_REGMAP_CONF("7"),
+	SIERRA_LANE_CDB_REGMAP_CONF("8"),
+	SIERRA_LANE_CDB_REGMAP_CONF("9"),
+	SIERRA_LANE_CDB_REGMAP_CONF("10"),
+	SIERRA_LANE_CDB_REGMAP_CONF("11"),
+	SIERRA_LANE_CDB_REGMAP_CONF("12"),
+	SIERRA_LANE_CDB_REGMAP_CONF("13"),
+	SIERRA_LANE_CDB_REGMAP_CONF("14"),
+	SIERRA_LANE_CDB_REGMAP_CONF("15"),
+};
+
+static struct regmap_config cdns_sierra_common_cdb_config = {
+	.name = "sierra_common_cdb",
+	.reg_stride = 1,
+	.fast_io = true,
+	.reg_write = cdns_regmap_write,
+	.reg_read = cdns_regmap_read,
+};
+
+static struct regmap_config cdns_sierra_phy_config_ctrl_config = {
+	.name = "sierra_phy_config_ctrl",
+	.reg_stride = 1,
+	.fast_io = true,
+	.reg_write = cdns_regmap_write,
+	.reg_read = cdns_regmap_read,
+};
+
+static int cdns_sierra_phy_init(struct phy *gphy)
+{
+	struct cdns_sierra_inst *ins = phy_get_drvdata(gphy);
+	struct cdns_sierra_phy *phy = dev_get_drvdata(gphy->dev.parent);
+	struct regmap *regmap = phy->regmap;
+	int i, j;
+	struct cdns_reg_pairs *cmn_vals, *ln_vals;
+	u32 num_cmn_regs, num_ln_regs;
+
+	/* Initialise the PHY registers, unless auto configured */
+	if (phy->autoconf)
+		return 0;
+
+	clk_set_rate(phy->cmn_refclk, 25000000);
+	clk_set_rate(phy->cmn_refclk1, 25000000);
+	if (ins->phy_type == PHY_TYPE_PCIE) {
+		num_cmn_regs = phy->init_data->pcie_cmn_regs;
+		num_ln_regs = phy->init_data->pcie_ln_regs;
+		cmn_vals = phy->init_data->pcie_cmn_vals;
+		ln_vals = phy->init_data->pcie_ln_vals;
+	} else if (ins->phy_type == PHY_TYPE_USB3) {
+		num_cmn_regs = phy->init_data->usb_cmn_regs;
+		num_ln_regs = phy->init_data->usb_ln_regs;
+		cmn_vals = phy->init_data->usb_cmn_vals;
+		ln_vals = phy->init_data->usb_ln_vals;
+	} else {
+		return -EINVAL;
+	}
+
+	regmap = phy->regmap_common_cdb;
+	for (j = 0; j < num_cmn_regs ; j++)
+		regmap_write(regmap, cmn_vals[j].off, cmn_vals[j].val);
+
+	for (i = 0; i < ins->num_lanes; i++) {
+		for (j = 0; j < num_ln_regs ; j++) {
+			regmap = phy->regmap_lane_cdb[i + ins->mlane];
+			regmap_write(regmap, ln_vals[j].off, ln_vals[j].val);
+		}
+	}
+
+	return 0;
+}
+
+static int cdns_sierra_phy_on(struct phy *gphy)
+{
+	struct cdns_sierra_phy *sp = dev_get_drvdata(gphy->dev.parent);
+	struct cdns_sierra_inst *ins = phy_get_drvdata(gphy);
+	struct device *dev = sp->dev;
+	u32 val;
+	int ret;
+
+	/* Take the PHY lane group out of reset */
+	ret = reset_control_deassert(ins->lnk_rst);
+	if (ret) {
+		dev_err(dev, "Failed to take the PHY lane out of reset\n");
+		return ret;
+	}
+
+	ret = regmap_field_read_poll_timeout(sp->pllctrl_lock[ins->mlane],
+					     val, val, 1000, PLL_LOCK_TIME);
+	if (ret < 0)
+		dev_err(dev, "PLL lock of lane failed\n");
+
+	return ret;
+}
+
+static int cdns_sierra_phy_off(struct phy *gphy)
+{
+	struct cdns_sierra_inst *ins = phy_get_drvdata(gphy);
+
+	return reset_control_assert(ins->lnk_rst);
+}
+
+static int cdns_sierra_phy_reset(struct phy *gphy)
+{
+	struct cdns_sierra_phy *sp = dev_get_drvdata(gphy->dev.parent);
+
+	reset_control_assert(sp->phy_rst);
+	reset_control_deassert(sp->phy_rst);
+	return 0;
+};
+
+static const struct phy_ops ops = {
+	.init		= cdns_sierra_phy_init,
+	.power_on	= cdns_sierra_phy_on,
+	.power_off	= cdns_sierra_phy_off,
+	.reset		= cdns_sierra_phy_reset,
+	.owner		= THIS_MODULE,
+};
+
+static int cdns_sierra_get_optional(struct cdns_sierra_inst *inst,
+				    struct device_node *child)
+{
+	if (of_property_read_u32(child, "reg", &inst->mlane))
+		return -EINVAL;
+
+	if (of_property_read_u32(child, "cdns,num-lanes", &inst->num_lanes))
+		return -EINVAL;
+
+	if (of_property_read_u32(child, "cdns,phy-type", &inst->phy_type))
+		return -EINVAL;
+
+	return 0;
+}
+
+static const struct of_device_id cdns_sierra_id_table[];
+
+static struct regmap *cdns_regmap_init(struct device *dev, void __iomem *base,
+				       u32 block_offset, u8 block_offset_shift,
+				       u8 reg_offset_shift,
+				       const struct regmap_config *config)
+{
+	struct cdns_regmap_cdb_context *ctx;
+
+	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return ERR_PTR(-ENOMEM);
+
+	ctx->dev = dev;
+	ctx->base = base + (block_offset << block_offset_shift);
+	ctx->reg_offset_shift = reg_offset_shift;
+
+	return devm_regmap_init(dev, NULL, ctx, config);
+}
+
+static int cdns_regfield_init(struct cdns_sierra_phy *sp)
+{
+	struct device *dev = sp->dev;
+	struct regmap_field *field;
+	struct regmap *regmap;
+	int i;
+
+	regmap = sp->regmap_common_cdb;
+	field = devm_regmap_field_alloc(dev, regmap, macro_id_type);
+	if (IS_ERR(field)) {
+		dev_err(dev, "MACRO_ID_TYPE reg field init failed\n");
+		return PTR_ERR(field);
+	}
+	sp->macro_id_type = field;
+
+	regmap = sp->regmap_phy_config_ctrl;
+	field = devm_regmap_field_alloc(dev, regmap, phy_pll_cfg_1);
+	if (IS_ERR(field)) {
+		dev_err(dev, "PHY_PLL_CFG_1 reg field init failed\n");
+		return PTR_ERR(field);
+	}
+	sp->phy_pll_cfg_1 = field;
+
+	for (i = 0; i < SIERRA_MAX_LANES; i++) {
+		regmap = sp->regmap_lane_cdb[i];
+		field = devm_regmap_field_alloc(dev, regmap, pllctrl_lock);
+		if (IS_ERR(field)) {
+			dev_err(dev, "P%d_ENABLE reg field init failed\n", i);
+			return PTR_ERR(field);
+		}
+		sp->pllctrl_lock[i] =  field;
+	}
+
+	return 0;
+}
+
+static int cdns_regmap_init_blocks(struct cdns_sierra_phy *sp,
+				   void __iomem *base, u8 block_offset_shift,
+				   u8 reg_offset_shift)
+{
+	struct device *dev = sp->dev;
+	struct regmap *regmap;
+	u32 block_offset;
+	int i;
+
+	for (i = 0; i < SIERRA_MAX_LANES; i++) {
+		block_offset = SIERRA_LANE_CDB_OFFSET(i, reg_offset_shift);
+		regmap = cdns_regmap_init(dev, base, block_offset,
+					  block_offset_shift, reg_offset_shift,
+					  &cdns_sierra_lane_cdb_config[i]);
+		if (IS_ERR(regmap)) {
+			dev_err(dev, "Failed to init lane CDB regmap\n");
+			return PTR_ERR(regmap);
+		}
+		sp->regmap_lane_cdb[i] = regmap;
+	}
+
+	regmap = cdns_regmap_init(dev, base, SIERRA_COMMON_CDB_OFFSET,
+				  block_offset_shift, reg_offset_shift,
+				  &cdns_sierra_common_cdb_config);
+	if (IS_ERR(regmap)) {
+		dev_err(dev, "Failed to init common CDB regmap\n");
+		return PTR_ERR(regmap);
+	}
+	sp->regmap_common_cdb = regmap;
+
+	regmap = cdns_regmap_init(dev, base, SIERRA_PHY_CONFIG_CTRL_OFFSET,
+				  block_offset_shift, reg_offset_shift,
+				  &cdns_sierra_phy_config_ctrl_config);
+	if (IS_ERR(regmap)) {
+		dev_err(dev, "Failed to init PHY config and control regmap\n");
+		return PTR_ERR(regmap);
+	}
+	sp->regmap_phy_config_ctrl = regmap;
+
+	return 0;
+}
+
+static int cdns_sierra_phy_probe(struct platform_device *pdev)
+{
+	struct cdns_sierra_phy *sp;
+	struct phy_provider *phy_provider;
+	struct device *dev = &pdev->dev;
+	const struct of_device_id *match;
+	struct cdns_sierra_data *data;
+	unsigned int id_value;
+	struct resource *res;
+	int i, ret, node = 0;
+	void __iomem *base;
+	struct clk *clk;
+	struct device_node *dn = dev->of_node, *child;
+
+	if (of_get_child_count(dn) == 0)
+		return -ENODEV;
+
+	/* Get init data for this PHY */
+	match = of_match_device(cdns_sierra_id_table, dev);
+	if (!match)
+		return -EINVAL;
+
+	data = (struct cdns_sierra_data *)match->data;
+
+	sp = devm_kzalloc(dev, sizeof(*sp), GFP_KERNEL);
+	if (!sp)
+		return -ENOMEM;
+	dev_set_drvdata(dev, sp);
+	sp->dev = dev;
+	sp->init_data = data;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(base)) {
+		dev_err(dev, "missing \"reg\"\n");
+		return PTR_ERR(base);
+	}
+
+	ret = cdns_regmap_init_blocks(sp, base, data->block_offset_shift,
+				      data->reg_offset_shift);
+	if (ret)
+		return ret;
+
+	ret = cdns_regfield_init(sp);
+	if (ret)
+		return ret;
+
+	platform_set_drvdata(pdev, sp);
+
+	sp->clk = devm_clk_get_optional(dev, "phy_clk");
+	if (IS_ERR(sp->clk)) {
+		dev_err(dev, "failed to get clock phy_clk\n");
+		return PTR_ERR(sp->clk);
+	}
+
+	sp->phy_rst = devm_reset_control_get(dev, "sierra_reset");
+	if (IS_ERR(sp->phy_rst)) {
+		dev_err(dev, "failed to get reset\n");
+		return PTR_ERR(sp->phy_rst);
+	}
+
+	sp->apb_rst = devm_reset_control_get_optional(dev, "sierra_apb");
+	if (IS_ERR(sp->apb_rst)) {
+		dev_err(dev, "failed to get apb reset\n");
+		return PTR_ERR(sp->apb_rst);
+	}
+
+	clk = devm_clk_get_optional(dev, "cmn_refclk");
+	if (IS_ERR(clk)) {
+		dev_err(dev, "core_ref_clk clock not found\n");
+		ret = PTR_ERR(clk);
+		return ret;
+	}
+	sp->cmn_refclk = clk;
+
+	clk = devm_clk_get_optional(dev, "cmn_refclk1");
+	if (IS_ERR(clk)) {
+		dev_err(dev, "core_ref_clk clock not found\n");
+		ret = PTR_ERR(clk);
+		return ret;
+	}
+	sp->cmn_refclk1 = clk;
+
+	ret = clk_prepare_enable(sp->clk);
+	if (ret)
+		return ret;
+
+	/* Enable APB */
+	reset_control_deassert(sp->apb_rst);
+
+	/* Check that PHY is present */
+	regmap_field_read(sp->macro_id_type, &id_value);
+	if  (sp->init_data->id_value != id_value) {
+		ret = -EINVAL;
+		goto clk_disable;
+	}
+
+	sp->autoconf = of_property_read_bool(dn, "cdns,autoconf");
+
+	for_each_available_child_of_node(dn, child) {
+		struct phy *gphy;
+
+		sp->phys[node].lnk_rst =
+			of_reset_control_array_get_exclusive(child);
+
+		if (IS_ERR(sp->phys[node].lnk_rst)) {
+			dev_err(dev, "failed to get reset %s\n",
+				child->full_name);
+			ret = PTR_ERR(sp->phys[node].lnk_rst);
+			goto put_child2;
+		}
+
+		if (!sp->autoconf) {
+			ret = cdns_sierra_get_optional(&sp->phys[node], child);
+			if (ret) {
+				dev_err(dev, "missing property in node %s\n",
+					child->name);
+				goto put_child;
+			}
+		}
+
+		sp->num_lanes += sp->phys[node].num_lanes;
+
+		gphy = devm_phy_create(dev, child, &ops);
+
+		if (IS_ERR(gphy)) {
+			ret = PTR_ERR(gphy);
+			goto put_child;
+		}
+		sp->phys[node].phy = gphy;
+		phy_set_drvdata(gphy, &sp->phys[node]);
+
+		node++;
+	}
+	sp->nsubnodes = node;
+
+	if (sp->num_lanes > SIERRA_MAX_LANES) {
+		dev_err(dev, "Invalid lane configuration\n");
+		goto put_child2;
+	}
+
+	/* If more than one subnode, configure the PHY as multilink */
+	if (!sp->autoconf && sp->nsubnodes > 1)
+		regmap_field_write(sp->phy_pll_cfg_1, 0x1);
+
+	pm_runtime_enable(dev);
+	phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
+	reset_control_deassert(sp->phy_rst);
+	return PTR_ERR_OR_ZERO(phy_provider);
+
+put_child:
+	node++;
+put_child2:
+	for (i = 0; i < node; i++)
+		reset_control_put(sp->phys[i].lnk_rst);
+	of_node_put(child);
+clk_disable:
+	clk_disable_unprepare(sp->clk);
+	reset_control_assert(sp->apb_rst);
+	return ret;
+}
+
+static int cdns_sierra_phy_remove(struct platform_device *pdev)
+{
+	struct cdns_sierra_phy *phy = platform_get_drvdata(pdev);
+	int i;
+
+	reset_control_assert(phy->phy_rst);
+	reset_control_assert(phy->apb_rst);
+	pm_runtime_disable(&pdev->dev);
+
+	/*
+	 * The device level resets will be put automatically.
+	 * Need to put the subnode resets here though.
+	 */
+	for (i = 0; i < phy->nsubnodes; i++) {
+		reset_control_assert(phy->phys[i].lnk_rst);
+		reset_control_put(phy->phys[i].lnk_rst);
+	}
+	return 0;
+}
+
+/* refclk100MHz_32b_PCIe_cmn_pll_ext_ssc */
+static struct cdns_reg_pairs cdns_pcie_cmn_regs_ext_ssc[] = {
+	{0x2106, SIERRA_CMN_PLLLC_LF_COEFF_MODE1_PREG},
+	{0x2106, SIERRA_CMN_PLLLC_LF_COEFF_MODE0_PREG},
+	{0x8A06, SIERRA_CMN_PLLLC_BWCAL_MODE1_PREG},
+	{0x8A06, SIERRA_CMN_PLLLC_BWCAL_MODE0_PREG},
+	{0x1B1B, SIERRA_CMN_PLLLC_SS_TIME_STEPSIZE_MODE_PREG}
+};
+
+/* refclk100MHz_32b_PCIe_ln_ext_ssc */
+static struct cdns_reg_pairs cdns_pcie_ln_regs_ext_ssc[] = {
+	{0x813E, SIERRA_CLKPATHCTRL_TMR_PREG},
+	{0x8047, SIERRA_RX_CREQ_FLTR_A_MODE3_PREG},
+	{0x808F, SIERRA_RX_CREQ_FLTR_A_MODE2_PREG},
+	{0x808F, SIERRA_RX_CREQ_FLTR_A_MODE1_PREG},
+	{0x808F, SIERRA_RX_CREQ_FLTR_A_MODE0_PREG},
+	{0x033C, SIERRA_RX_CTLE_MAINTENANCE_PREG},
+	{0x44CC, SIERRA_CREQ_EQ_OPEN_EYE_THRESH_PREG}
+};
+
+/* refclk100MHz_20b_USB_cmn_pll_ext_ssc */
+static struct cdns_reg_pairs cdns_usb_cmn_regs_ext_ssc[] = {
+	{0x2085, SIERRA_CMN_PLLLC_LF_COEFF_MODE1_PREG},
+	{0x2085, SIERRA_CMN_PLLLC_LF_COEFF_MODE0_PREG},
+	{0x0000, SIERRA_CMN_PLLLC_BWCAL_MODE0_PREG},
+	{0x0000, SIERRA_CMN_PLLLC_SS_TIME_STEPSIZE_MODE_PREG}
+};
+
+/* refclk100MHz_20b_USB_ln_ext_ssc */
+static struct cdns_reg_pairs cdns_usb_ln_regs_ext_ssc[] = {
+	{0xFE0A, SIERRA_DET_STANDEC_A_PREG},
+	{0x000F, SIERRA_DET_STANDEC_B_PREG},
+	{0x00A5, SIERRA_DET_STANDEC_C_PREG},
+	{0x69ad, SIERRA_DET_STANDEC_D_PREG},
+	{0x0241, SIERRA_DET_STANDEC_E_PREG},
+	{0x0010, SIERRA_PSM_LANECAL_DLY_A1_RESETS_PREG},
+	{0x0014, SIERRA_PSM_A0IN_TMR_PREG},
+	{0xCF00, SIERRA_PSM_DIAG_PREG},
+	{0x001F, SIERRA_PSC_TX_A0_PREG},
+	{0x0007, SIERRA_PSC_TX_A1_PREG},
+	{0x0003, SIERRA_PSC_TX_A2_PREG},
+	{0x0003, SIERRA_PSC_TX_A3_PREG},
+	{0x0FFF, SIERRA_PSC_RX_A0_PREG},
+	{0x0619, SIERRA_PSC_RX_A1_PREG},
+	{0x0003, SIERRA_PSC_RX_A2_PREG},
+	{0x0001, SIERRA_PSC_RX_A3_PREG},
+	{0x0001, SIERRA_PLLCTRL_SUBRATE_PREG},
+	{0x0406, SIERRA_PLLCTRL_GEN_D_PREG},
+	{0x5233, SIERRA_PLLCTRL_CPGAIN_MODE_PREG},
+	{0x00CA, SIERRA_CLKPATH_BIASTRIM_PREG},
+	{0x2512, SIERRA_DFE_BIASTRIM_PREG},
+	{0x0000, SIERRA_DRVCTRL_ATTEN_PREG},
+	{0x873E, SIERRA_CLKPATHCTRL_TMR_PREG},
+	{0x03CF, SIERRA_RX_CREQ_FLTR_A_MODE1_PREG},
+	{0x01CE, SIERRA_RX_CREQ_FLTR_A_MODE0_PREG},
+	{0x7B3C, SIERRA_CREQ_CCLKDET_MODE01_PREG},
+	{0x033F, SIERRA_RX_CTLE_MAINTENANCE_PREG},
+	{0x3232, SIERRA_CREQ_FSMCLK_SEL_PREG},
+	{0x0000, SIERRA_CREQ_EQ_CTRL_PREG},
+	{0x8000, SIERRA_CREQ_SPARE_PREG},
+	{0xCC44, SIERRA_CREQ_EQ_OPEN_EYE_THRESH_PREG},
+	{0x8453, SIERRA_CTLELUT_CTRL_PREG},
+	{0x4110, SIERRA_DFE_ECMP_RATESEL_PREG},
+	{0x4110, SIERRA_DFE_SMP_RATESEL_PREG},
+	{0x0002, SIERRA_DEQ_PHALIGN_CTRL},
+	{0x3200, SIERRA_DEQ_CONCUR_CTRL1_PREG},
+	{0x5064, SIERRA_DEQ_CONCUR_CTRL2_PREG},
+	{0x0030, SIERRA_DEQ_EPIPWR_CTRL2_PREG},
+	{0x0048, SIERRA_DEQ_FAST_MAINT_CYCLES_PREG},
+	{0x5A5A, SIERRA_DEQ_ERRCMP_CTRL_PREG},
+	{0x02F5, SIERRA_DEQ_OFFSET_CTRL_PREG},
+	{0x02F5, SIERRA_DEQ_GAIN_CTRL_PREG},
+	{0x9A8A, SIERRA_DEQ_VGATUNE_CTRL_PREG},
+	{0x0014, SIERRA_DEQ_GLUT0},
+	{0x0014, SIERRA_DEQ_GLUT1},
+	{0x0014, SIERRA_DEQ_GLUT2},
+	{0x0014, SIERRA_DEQ_GLUT3},
+	{0x0014, SIERRA_DEQ_GLUT4},
+	{0x0014, SIERRA_DEQ_GLUT5},
+	{0x0014, SIERRA_DEQ_GLUT6},
+	{0x0014, SIERRA_DEQ_GLUT7},
+	{0x0014, SIERRA_DEQ_GLUT8},
+	{0x0014, SIERRA_DEQ_GLUT9},
+	{0x0014, SIERRA_DEQ_GLUT10},
+	{0x0014, SIERRA_DEQ_GLUT11},
+	{0x0014, SIERRA_DEQ_GLUT12},
+	{0x0014, SIERRA_DEQ_GLUT13},
+	{0x0014, SIERRA_DEQ_GLUT14},
+	{0x0014, SIERRA_DEQ_GLUT15},
+	{0x0014, SIERRA_DEQ_GLUT16},
+	{0x0BAE, SIERRA_DEQ_ALUT0},
+	{0x0AEB, SIERRA_DEQ_ALUT1},
+	{0x0A28, SIERRA_DEQ_ALUT2},
+	{0x0965, SIERRA_DEQ_ALUT3},
+	{0x08A2, SIERRA_DEQ_ALUT4},
+	{0x07DF, SIERRA_DEQ_ALUT5},
+	{0x071C, SIERRA_DEQ_ALUT6},
+	{0x0659, SIERRA_DEQ_ALUT7},
+	{0x0596, SIERRA_DEQ_ALUT8},
+	{0x0514, SIERRA_DEQ_ALUT9},
+	{0x0492, SIERRA_DEQ_ALUT10},
+	{0x0410, SIERRA_DEQ_ALUT11},
+	{0x038E, SIERRA_DEQ_ALUT12},
+	{0x030C, SIERRA_DEQ_ALUT13},
+	{0x03F4, SIERRA_DEQ_DFETAP_CTRL_PREG},
+	{0x0001, SIERRA_DFE_EN_1010_IGNORE_PREG},
+	{0x3C01, SIERRA_DEQ_TAU_CTRL1_FAST_MAINT_PREG},
+	{0x3C40, SIERRA_DEQ_TAU_CTRL1_SLOW_MAINT_PREG},
+	{0x1C08, SIERRA_DEQ_TAU_CTRL2_PREG},
+	{0x0033, SIERRA_DEQ_PICTRL_PREG},
+	{0x0400, SIERRA_CPICAL_TMRVAL_MODE1_PREG},
+	{0x0330, SIERRA_CPICAL_TMRVAL_MODE0_PREG},
+	{0x01FF, SIERRA_CPICAL_PICNT_MODE1_PREG},
+	{0x0009, SIERRA_CPI_OUTBUF_RATESEL_PREG},
+	{0x3232, SIERRA_CPICAL_RES_STARTCODE_MODE23_PREG},
+	{0x0005, SIERRA_LFPSDET_SUPPORT_PREG},
+	{0x000F, SIERRA_LFPSFILT_NS_PREG},
+	{0x0009, SIERRA_LFPSFILT_RD_PREG},
+	{0x0001, SIERRA_LFPSFILT_MP_PREG},
+	{0x8013, SIERRA_SDFILT_H2L_A_PREG},
+	{0x8009, SIERRA_SDFILT_L2H_PREG},
+	{0x0024, SIERRA_RXBUFFER_CTLECTRL_PREG},
+	{0x0020, SIERRA_RXBUFFER_RCDFECTRL_PREG},
+	{0x4243, SIERRA_RXBUFFER_DFECTRL_PREG}
+};
+
+static const struct cdns_sierra_data cdns_map_sierra = {
+	SIERRA_MACRO_ID,
+	0x2,
+	0x2,
+	ARRAY_SIZE(cdns_pcie_cmn_regs_ext_ssc),
+	ARRAY_SIZE(cdns_pcie_ln_regs_ext_ssc),
+	ARRAY_SIZE(cdns_usb_cmn_regs_ext_ssc),
+	ARRAY_SIZE(cdns_usb_ln_regs_ext_ssc),
+	cdns_pcie_cmn_regs_ext_ssc,
+	cdns_pcie_ln_regs_ext_ssc,
+	cdns_usb_cmn_regs_ext_ssc,
+	cdns_usb_ln_regs_ext_ssc,
+};
+
+static const struct cdns_sierra_data cdns_ti_map_sierra = {
+	SIERRA_MACRO_ID,
+	0x0,
+	0x1,
+	ARRAY_SIZE(cdns_pcie_cmn_regs_ext_ssc),
+	ARRAY_SIZE(cdns_pcie_ln_regs_ext_ssc),
+	ARRAY_SIZE(cdns_usb_cmn_regs_ext_ssc),
+	ARRAY_SIZE(cdns_usb_ln_regs_ext_ssc),
+	cdns_pcie_cmn_regs_ext_ssc,
+	cdns_pcie_ln_regs_ext_ssc,
+	cdns_usb_cmn_regs_ext_ssc,
+	cdns_usb_ln_regs_ext_ssc,
+};
+
+static const struct of_device_id cdns_sierra_id_table[] = {
+	{
+		.compatible = "cdns,sierra-phy-t0",
+		.data = &cdns_map_sierra,
+	},
+	{
+		.compatible = "cdns,ti,sierra-phy-t0",
+		.data = &cdns_ti_map_sierra,
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(of, cdns_sierra_id_table);
+
+static struct platform_driver cdns_sierra_driver = {
+	.probe		= cdns_sierra_phy_probe,
+	.remove		= cdns_sierra_phy_remove,
+	.driver		= {
+		.name	= "cdns-sierra-phy",
+		.of_match_table = cdns_sierra_id_table,
+	},
+};
+module_platform_driver(cdns_sierra_driver);
+
+MODULE_ALIAS("platform:cdns_sierra");
+MODULE_AUTHOR("Cadence Design Systems");
+MODULE_DESCRIPTION("CDNS sierra phy driver");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/phy/cadence/phy-cadence-torrent.c b/drivers/phy/cadence/phy-cadence-torrent.c
--- a/drivers/phy/cadence/phy-cadence-torrent.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/phy/cadence/phy-cadence-torrent.c	2020-06-19 21:00:31.420227716 -0400
@@ -0,0 +1,1392 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Cadence Torrent SD0801 PHY driver.
+ *
+ * Copyright 2018 Cadence Design Systems, Inc.
+ *
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+
+#define REF_CLK_19_2MHz		19200000
+#define REF_CLK_25MHz		25000000
+
+#define DEFAULT_NUM_LANES	4
+#define MAX_NUM_LANES		4
+#define DEFAULT_MAX_BIT_RATE	8100 /* in Mbps */
+
+#define POLL_TIMEOUT_US		5000
+#define LANE_MASK		0x7
+
+/*
+ * register offsets from DPTX PHY register block base (i.e MHDP
+ * register base + 0x30a00)
+ */
+#define PHY_AUX_CONFIG			0x00
+#define PHY_AUX_CTRL			0x04
+#define PHY_RESET			0x20
+#define PMA_TX_ELEC_IDLE_MASK		0xF0U
+#define PMA_TX_ELEC_IDLE_SHIFT		4
+#define PHY_L00_RESET_N_MASK		0x01U
+#define PHY_PMA_XCVR_PLLCLK_EN		0x24
+#define PHY_PMA_XCVR_PLLCLK_EN_ACK	0x28
+#define PHY_PMA_XCVR_POWER_STATE_REQ	0x2c
+#define PHY_POWER_STATE_LN_0	0x0000
+#define PHY_POWER_STATE_LN_1	0x0008
+#define PHY_POWER_STATE_LN_2	0x0010
+#define PHY_POWER_STATE_LN_3	0x0018
+#define PMA_XCVR_POWER_STATE_REQ_LN_MASK	0x3FU
+#define PHY_PMA_XCVR_POWER_STATE_ACK	0x30
+#define PHY_PMA_CMN_READY		0x34
+#define PHY_PMA_XCVR_TX_VMARGIN		0x38
+#define PHY_PMA_XCVR_TX_DEEMPH		0x3c
+
+/*
+ * register offsets from SD0801 PHY register block base (i.e MHDP
+ * register base + 0x500000)
+ */
+#define CMN_SSM_BANDGAP_TMR		(0x00084/2)
+#define CMN_SSM_BIAS_TMR		(0x00088/2)
+#define CMN_PLLSM0_PLLPRE_TMR		(0x000a8/2)
+#define CMN_PLLSM0_PLLLOCK_TMR		(0x000b0/2)
+#define CMN_PLLSM1_PLLPRE_TMR		(0x000c8/2)
+#define CMN_PLLSM1_PLLLOCK_TMR		(0x000d0/2)
+#define CMN_BGCAL_INIT_TMR		(0x00190/2)
+#define CMN_BGCAL_ITER_TMR		(0x00194/2)
+#define CMN_IBCAL_INIT_TMR		(0x001d0/2)
+#define CMN_PLL0_VCOCAL_TCTRL		(0x00208/2)
+#define CMN_PLL0_VCOCAL_INIT_TMR	(0x00210/2)
+#define CMN_PLL0_VCOCAL_ITER_TMR	(0x00214/2)
+#define CMN_PLL0_VCOCAL_REFTIM_START	(0x00218/2)
+#define CMN_PLL0_VCOCAL_PLLCNT_START	(0x00220/2)
+#define CMN_PLL0_INTDIV_M0		(0x00240/2)
+#define CMN_PLL0_FRACDIVL_M0		(0x00244/2)
+#define CMN_PLL0_FRACDIVH_M0		(0x00248/2)
+#define CMN_PLL0_HIGH_THR_M0		(0x0024c/2)
+#define CMN_PLL0_DSM_DIAG_M0		(0x00250/2)
+#define CMN_PLL0_SS_CTRL1_M0		(0x00260/2)
+#define CMN_PLL0_SS_CTRL2_M0            (0x00264/2)
+#define CMN_PLL0_SS_CTRL3_M0            (0x00268/2)
+#define CMN_PLL0_SS_CTRL4_M0            (0x0026C/2)
+#define CMN_PLL0_LOCK_REFCNT_START      (0x00270/2)
+#define CMN_PLL0_LOCK_PLLCNT_START	(0x00278/2)
+#define CMN_PLL0_LOCK_PLLCNT_THR        (0x0027C/2)
+#define CMN_PLL1_VCOCAL_TCTRL		(0x00308/2)
+#define CMN_PLL1_VCOCAL_INIT_TMR	(0x00310/2)
+#define CMN_PLL1_VCOCAL_ITER_TMR	(0x00314/2)
+#define CMN_PLL1_VCOCAL_REFTIM_START	(0x00318/2)
+#define CMN_PLL1_VCOCAL_PLLCNT_START	(0x00320/2)
+#define CMN_PLL1_INTDIV_M0		(0x00340/2)
+#define CMN_PLL1_FRACDIVL_M0		(0x00344/2)
+#define CMN_PLL1_FRACDIVH_M0		(0x00348/2)
+#define CMN_PLL1_HIGH_THR_M0		(0x0034c/2)
+#define CMN_PLL1_DSM_DIAG_M0		(0x00350/2)
+#define CMN_PLL1_SS_CTRL1_M0		(0x00360/2)
+#define CMN_PLL1_SS_CTRL2_M0            (0x00364/2)
+#define CMN_PLL1_SS_CTRL3_M0            (0x00368/2)
+#define CMN_PLL1_SS_CTRL4_M0            (0x0036C/2)
+#define CMN_PLL1_LOCK_REFCNT_START      (0x00370/2)
+#define CMN_PLL1_LOCK_PLLCNT_START	(0x00378/2)
+#define CMN_PLL1_LOCK_PLLCNT_THR        (0x0037C/2)
+#define CMN_TXPUCAL_INIT_TMR		(0x00410/2)
+#define CMN_TXPUCAL_ITER_TMR		(0x00414/2)
+#define CMN_TXPDCAL_INIT_TMR		(0x00430/2)
+#define CMN_TXPDCAL_ITER_TMR		(0x00434/2)
+#define CMN_RXCAL_INIT_TMR		(0x00450/2)
+#define CMN_RXCAL_ITER_TMR		(0x00454/2)
+#define CMN_SD_CAL_INIT_TMR		(0x00490/2)
+#define CMN_SD_CAL_ITER_TMR		(0x00494/2)
+#define CMN_SD_CAL_REFTIM_START		(0x00498/2)
+#define CMN_SD_CAL_PLLCNT_START		(0x004a0/2)
+#define CMN_PDIAG_PLL0_CTRL_M0		(0x00680/2)
+#define CMN_PDIAG_PLL0_CLK_SEL_M0	(0x00684/2)
+#define CMN_PDIAG_PLL0_CP_PADJ_M0	(0x00690/2)
+#define CMN_PDIAG_PLL0_CP_IADJ_M0	(0x00694/2)
+#define CMN_PDIAG_PLL0_FILT_PADJ_M0	(0x00698/2)
+#define CMN_PDIAG_PLL0_CP_PADJ_M1	(0x006d0/2)
+#define CMN_PDIAG_PLL0_CP_IADJ_M1	(0x006d4/2)
+#define CMN_PDIAG_PLL1_CTRL_M0		(0x00700/2)
+#define CMN_PDIAG_PLL1_CLK_SEL_M0	(0x00704/2)
+#define CMN_PDIAG_PLL1_CP_PADJ_M0	(0x00710/2)
+#define CMN_PDIAG_PLL1_CP_IADJ_M0	(0x00714/2)
+#define CMN_PDIAG_PLL1_FILT_PADJ_M0	(0x00718/2)
+#define CMN_PDIAG_PLL1_CP_PADJ_M1	(0x00750/2)
+#define CMN_PDIAG_PLL1_CP_IADJ_M1	(0x00754/2)
+
+#define XCVR_DIAG_PLLDRC_CTRL(j)		(0x4000 + 0x01ca + (j) * 0x400)
+#define XCVR_DIAG_HSCLK_SEL(j)			(0x4000 + 0x01cc + (j) * 0x400)
+#define XCVR_DIAG_HSCLK_DIV(j)			(0x4000 + 0x01ce + (j) * 0x400)
+#define XCVR_DIAG_BIDI_CTRL(j)			(0x4000 + 0x01d4 + (j) * 0x400)
+#define TX_PSC_A0(j)				(0x4000 + 0x0200 + (j) * 0x400)
+#define TX_PSC_A1(j)				(0x4000 + 0x0202 + (j) * 0x400)
+#define TX_PSC_A2(j)				(0x4000 + 0x0204 + (j) * 0x400)
+#define TX_PSC_A3(j)				(0x4000 + 0x0206 + (j) * 0x400)
+
+#define TX_RCVDET_ST_TMR(j)			(0x4000 + 0x0246 + (j) * 0x400)
+
+#define RX_PSC_A0(j)				(0x8000 + 0x0000 + (j) * 0x400)
+#define RX_PSC_A1(j)				(0x8000 + 0x0002 + (j) * 0x400)
+#define RX_PSC_A2(j)				(0x8000 + 0x0004 + (j) * 0x400)
+#define RX_PSC_A3(j)				(0x8000 + 0x0006 + (j) * 0x400)
+
+#define TX_TXCC_CTRL(j)				(0x4000 + 0x80 + (j) * 0x400)
+#define TX_DIAG_ACYA(j)				(0x4000 + 0x3ce + (j) * 0x400)
+#define DRV_DIAG_TX_DRV(j)			(0x4000 + 0x18c + (j) * 0x400)
+#define TX_TXCC_MGNFS_MULT_000(j)		(0x4000 + 0xa0 + (j) * 0x400)
+#define TX_TXCC_CPOST_MULT_00(j)		(0x4000 + 0x98 + (j) * 0x400)
+
+#define PHY_PLL_CFG				(0xc000 + 0x001c) // XXX is this correct?
+
+#define PHY_PMA_CMN_CTRL2			0xe002
+#define PHY_PMA_PLL_RAW_CTRL			0xe006
+
+
+#define TX_DIAG_ACYA_HBDC_MASK 0x0001U
+
+#define RX_PSC_CAL(j)				(0x8000 + 0x000c + (j) * 0x400)
+
+#define RX_REE_GCSM1_CTRL(j)			(0x8000 + 0x0210 + (j) * 0x400)
+#define RX_REE_GCSM2_CTRL(j)			(0x8000 + 0x0220 + (j) * 0x400)
+#define RX_REE_PERGCSM_CTRL(j)			(0x8000 + 0x0230 + (j) * 0x400)
+
+struct cdns_torrent_phy {
+	void __iomem *base;	/* DPTX registers base */
+	void __iomem *sd_base;	/* SD0801 registers base */
+	u32 num_lanes; /* Number of lanes to use */
+	u32 max_bit_rate; /* Maximum link bit rate to use (in Mbps) */
+	struct reset_control *phy_rst;
+	struct device *dev;
+	struct clk *clk;
+	unsigned long ref_clk_rate;
+};
+
+enum phy_powerstate {
+	POWERSTATE_A0 = 0,
+	// Powerstate A1 is unused
+	POWERSTATE_A2 = 2,
+	POWERSTATE_A3 = 3,
+};
+
+static int cdns_torrent_dp_init(struct phy *phy);
+static int cdns_torrent_dp_exit(struct phy *phy);
+static int cdns_torrent_dp_run(struct cdns_torrent_phy *cdns_phy);
+static int cdns_torrent_dp_wait_pma_cmn_ready(struct cdns_torrent_phy *cdns_phy);
+static void cdns_torrent_dp_pma_cfg(struct cdns_torrent_phy *cdns_phy);
+static void cdns_torrent_dp_pma_cmn_cfg_19_2mhz(struct cdns_torrent_phy *cdns_phy);
+static void cdns_torrent_dp_pma_cmn_vco_cfg_19_2mhz(struct cdns_torrent_phy *cdns_phy, u32 rate, bool ssc);
+static void cdns_torrent_dp_pma_cmn_cfg_25mhz(struct cdns_torrent_phy *cdns_phy);
+static void cdns_torrent_dp_pma_cmn_vco_cfg_25mhz(struct cdns_torrent_phy *cdns_phy, u32 rate, bool ssc);
+static void cdns_torrent_dp_pma_lane_cfg(struct cdns_torrent_phy *cdns_phy,
+				     unsigned int lane);
+static void cdns_torrent_dp_pma_cmn_rate(struct cdns_torrent_phy *cdns_phy,
+				     u32 rate, u32 lanes);
+static void cdns_dp_phy_write_field(struct cdns_torrent_phy *cdns_phy,
+				    unsigned int offset,
+				    unsigned char start_bit,
+				    unsigned char num_bits,
+				    unsigned int val);
+static int cdns_torrent_dp_configure(struct phy *phy, union phy_configure_opts *opts);
+static void cdns_torrent_dp_set_a0_pll(struct cdns_torrent_phy *cdns_phy, u32 num_lanes);
+static int cdns_torrent_dp_set_power_state(struct cdns_torrent_phy *cdns_phy,
+				       u32 num_lanes,
+				       enum phy_powerstate powerstate);
+
+static int cdns_torrent_phy_on(struct phy *gphy);
+static int cdns_torrent_phy_off(struct phy *gphy);
+
+static const struct phy_ops cdns_torrent_phy_ops = {
+	.init		= cdns_torrent_dp_init,
+	.exit		= cdns_torrent_dp_exit,
+	.configure	= cdns_torrent_dp_configure,
+	.power_on	= cdns_torrent_phy_on,
+	.power_off	= cdns_torrent_phy_off,
+	.owner		= THIS_MODULE,
+};
+
+//#define DEBUG_DP_WRITES
+//#define DEBUG_PHY_WRITES
+
+/* PHY mmr access functions */
+
+#ifdef DEBUG_PHY_WRITES
+#define cdns_dp_phy_write_phy(cdns_phy, offset, val)  \
+({ \
+	printk("PHY WR PHY %s (0x%08x) = 0x%08x [%s:%d]\n", \
+		#offset, offset, val, __func__, __LINE__); \
+	_cdns_dp_phy_write_phy(cdns_phy, offset, val); \
+})
+
+static void _cdns_dp_phy_write_phy(struct cdns_torrent_phy *cdns_phy, u32 offset, u16 val)
+{
+	writew(val, cdns_phy->sd_base + offset);
+}
+#else
+static void cdns_dp_phy_write_phy(struct cdns_torrent_phy *cdns_phy, u32 offset, u16 val)
+{
+	writew(val, cdns_phy->sd_base + offset);
+}
+#endif
+
+static u16 cdns_dp_phy_read_phy(struct cdns_torrent_phy *cdns_phy, u32 offset)
+{
+	return readw(cdns_phy->sd_base + offset);
+}
+
+#define cdns_phy_read_poll_timeout(offset, val, cond, delay_us, timeout_us) \
+	readw_poll_timeout(cdns_phy->sd_base + (offset), val, cond, delay_us, timeout_us)
+
+/* DPTX mmr access functions */
+
+#ifdef DEBUG_DP_WRITES
+#define cdns_dp_phy_write_dp(cdns_phy, offset, val)  \
+({ \
+	printk("PHY WR DP %s (0x%08x) = 0x%08x [%s:%d]\n", \
+		#offset, offset, val, __func__, __LINE__); \
+	_cdns_dp_phy_write_dp(cdns_phy, offset, val); \
+})
+
+static void _cdns_dp_phy_write_dp(struct cdns_torrent_phy *cdns_phy, u32 offset, u16 val)
+{
+	writel(val, cdns_phy->base + offset);
+}
+#else
+static void cdns_dp_phy_write_dp(struct cdns_torrent_phy *cdns_phy, u32 offset, u16 val)
+{
+	writel(val, cdns_phy->base + offset);
+}
+#endif
+
+static u32 cdns_dp_phy_read_dp(struct cdns_torrent_phy *cdns_phy, u32 offset)
+{
+	return readl(cdns_phy->base + offset);
+}
+
+#define cdns_phy_read_dp_poll_timeout(cdns_phy, offset, val, cond, delay_us, timeout_us) \
+	readl_poll_timeout(cdns_phy->base + (offset), val, cond, delay_us, timeout_us)
+
+
+/*
+ * Structure used to store values of PHY registers for voltage-related
+ * coefficients, for particular voltage swing and re-emphasis level. Values
+ * are shared across all physical lanes.
+ */
+struct coefficients {
+	/** Value of DRV_DIAG_TX_DRV register to use */
+	uint16_t diag_tx_drv;
+	/** Value of TX_TXCC_MGNFS_MULT_000 register to use */
+	uint16_t mgnfs_mult;
+	/** Value of TX_TXCC_CPOST_MULT_00 register to use */
+	uint16_t cpost_mult;
+};
+
+/*
+ * Array consists of values of voltage-related registers for sd0801 PHY. A value
+ * of 0xFFFF is a placeholder for invalid combination, and will never be used.
+ */
+static const struct coefficients voltage_coeffs[4][4] = {
+	// voltage swing 0, pre-emphasis 0->3
+	{	{.diag_tx_drv = 0x0003, .mgnfs_mult = 0x002A, .cpost_mult = 0x0000},
+		{.diag_tx_drv = 0x0003, .mgnfs_mult = 0x001F, .cpost_mult = 0x0014},
+		{.diag_tx_drv = 0x0003, .mgnfs_mult = 0x0012, .cpost_mult = 0x0020},
+		{.diag_tx_drv = 0x0003, .mgnfs_mult = 0x0000, .cpost_mult = 0x002A}
+	},
+
+	// voltage swing 1, pre-emphasis 0->3
+	{	{.diag_tx_drv = 0x0003, .mgnfs_mult = 0x001F, .cpost_mult = 0x0000},
+		{.diag_tx_drv = 0x0003, .mgnfs_mult = 0x0013, .cpost_mult = 0x0012},
+		{.diag_tx_drv = 0x0003, .mgnfs_mult = 0x0000, .cpost_mult = 0x001F},
+		{.diag_tx_drv = 0xFFFF, .mgnfs_mult = 0xFFFF, .cpost_mult = 0xFFFF}
+	},
+
+	// voltage swing 2, pre-emphasis 0->3
+	{	{.diag_tx_drv = 0x0003, .mgnfs_mult = 0x0013, .cpost_mult = 0x0000},
+		{.diag_tx_drv = 0x0003, .mgnfs_mult = 0x0000, .cpost_mult = 0x0013},
+		{.diag_tx_drv = 0xFFFF, .mgnfs_mult = 0xFFFF, .cpost_mult = 0xFFFF},
+		{.diag_tx_drv = 0xFFFF, .mgnfs_mult = 0xFFFF, .cpost_mult = 0xFFFF}
+	},
+
+	// voltage swing 3, pre-emphasis 0->3
+	{	{.diag_tx_drv = 0x0003, .mgnfs_mult = 0x0000, .cpost_mult = 0x0000},
+		{.diag_tx_drv = 0xFFFF, .mgnfs_mult = 0xFFFF, .cpost_mult = 0xFFFF},
+		{.diag_tx_drv = 0xFFFF, .mgnfs_mult = 0xFFFF, .cpost_mult = 0xFFFF},
+		{.diag_tx_drv = 0xFFFF, .mgnfs_mult = 0xFFFF, .cpost_mult = 0xFFFF}
+	}
+};
+
+static int cdns_torrent_dp_init(struct phy *phy)
+{
+	unsigned char lane_bits;
+	int r;
+
+	struct cdns_torrent_phy *cdns_phy = phy_get_drvdata(phy);
+
+	r = clk_prepare_enable(cdns_phy->clk);
+	if (r) {
+		dev_err(cdns_phy->dev, "Failed to prepare ref clock\n");
+		return r;
+	}
+
+	cdns_phy->ref_clk_rate = clk_get_rate(cdns_phy->clk);
+	if (!(cdns_phy->ref_clk_rate)) {
+		dev_err(cdns_phy->dev, "Failed to get ref clock rate\n");
+		clk_disable_unprepare(cdns_phy->clk);
+		return -EINVAL;
+	}
+
+	switch (cdns_phy->ref_clk_rate) {
+	case REF_CLK_19_2MHz:
+	case REF_CLK_25MHz:
+		/* Valid Ref Clock Rate */
+		break;
+	default:
+		dev_err(cdns_phy->dev, "Unsupported Ref Clock Rate\n");
+		return -EINVAL;
+	}
+
+	cdns_dp_phy_write_dp(cdns_phy, PHY_AUX_CTRL, 0x0003); /* enable AUX */
+
+	/* PHY PMA registers configuration function */
+	cdns_torrent_dp_pma_cfg(cdns_phy);
+
+	/*
+	 * Set lines power state to A0
+	 * Set lines pll clk enable to 0
+	 */
+	cdns_torrent_dp_set_a0_pll(cdns_phy, cdns_phy->num_lanes);
+
+	/*
+	 * release phy_l0*_reset_n and pma_tx_elec_idle_ln_* based on
+	 * used lanes
+	 */
+	lane_bits = (1 << cdns_phy->num_lanes) - 1;
+	cdns_dp_phy_write_dp(cdns_phy, PHY_RESET, ((0xF & ~lane_bits) << 4) | (0xF & lane_bits));
+
+	/* release pma_xcvr_pllclk_en_ln_*, only for the master lane */
+	cdns_dp_phy_write_dp(cdns_phy, PHY_PMA_XCVR_PLLCLK_EN, 0x0001);
+
+	/* PHY PMA registers configuration functions */
+	/* Initialize PHY with max supported link rate, without SSC. */
+	if (cdns_phy->ref_clk_rate ==  REF_CLK_19_2MHz)
+		cdns_torrent_dp_pma_cmn_vco_cfg_19_2mhz(cdns_phy, cdns_phy->max_bit_rate, false);
+	else if (cdns_phy->ref_clk_rate == REF_CLK_25MHz)
+		cdns_torrent_dp_pma_cmn_vco_cfg_25mhz(cdns_phy, cdns_phy->max_bit_rate, false);
+	cdns_torrent_dp_pma_cmn_rate(cdns_phy, cdns_phy->max_bit_rate, cdns_phy->num_lanes);
+
+	/* take out of reset */
+	cdns_dp_phy_write_field(cdns_phy, PHY_RESET, 8, 1, 1);
+	cdns_torrent_phy_on(phy);
+	r = cdns_torrent_dp_wait_pma_cmn_ready(cdns_phy);
+	if (r)
+		return r;
+
+	r = cdns_torrent_dp_run(cdns_phy);
+
+	return r;
+}
+
+static int cdns_torrent_dp_exit(struct phy *phy)
+{
+	struct cdns_torrent_phy *cdns_phy = phy_get_drvdata(phy);
+
+	clk_disable_unprepare(cdns_phy->clk);
+	return 0;
+}
+
+static int cdns_torrent_dp_wait_pma_cmn_ready(struct cdns_torrent_phy *cdns_phy)
+{
+	unsigned int reg;
+	int ret;
+
+	ret = cdns_phy_read_dp_poll_timeout(cdns_phy, PHY_PMA_CMN_READY, reg,
+					    reg & 1, 0, POLL_TIMEOUT_US);
+	if (ret == -ETIMEDOUT) {
+		dev_err(cdns_phy->dev,
+			"timeout waiting for PMA common ready\n");
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+static void cdns_torrent_dp_pma_cfg(struct cdns_torrent_phy *cdns_phy)
+{
+	unsigned int i;
+
+	if (cdns_phy->ref_clk_rate ==  REF_CLK_19_2MHz)
+		/* PMA common configuration 19.2MHz */
+		cdns_torrent_dp_pma_cmn_cfg_19_2mhz(cdns_phy);
+	else if (cdns_phy->ref_clk_rate == REF_CLK_25MHz)
+		/* PMA common configuration 25MHz */
+		cdns_torrent_dp_pma_cmn_cfg_25mhz(cdns_phy);
+
+	/* PMA lane configuration to deal with multi-link operation */
+	for (i = 0; i < cdns_phy->num_lanes; i++)
+		cdns_torrent_dp_pma_lane_cfg(cdns_phy, i);
+}
+
+static void cdns_torrent_dp_pma_cmn_cfg_19_2mhz(struct cdns_torrent_phy *cdns_phy)
+{
+	/* refclock registers - assumes 19.2 MHz refclock */
+	cdns_dp_phy_write_phy(cdns_phy, CMN_SSM_BIAS_TMR, 0x0014);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLLSM0_PLLPRE_TMR, 0x0027);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLLSM0_PLLLOCK_TMR, 0x00A1);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLLSM1_PLLPRE_TMR, 0x0027);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLLSM1_PLLLOCK_TMR, 0x00A1);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_BGCAL_INIT_TMR, 0x0060);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_BGCAL_ITER_TMR, 0x0060);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_IBCAL_INIT_TMR, 0x0014);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_TXPUCAL_INIT_TMR, 0x0018);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_TXPUCAL_ITER_TMR, 0x0005);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_TXPDCAL_INIT_TMR, 0x0018);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_TXPDCAL_ITER_TMR, 0x0005);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_RXCAL_INIT_TMR, 0x0240);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_RXCAL_ITER_TMR, 0x0005);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_SD_CAL_INIT_TMR, 0x0002);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_SD_CAL_ITER_TMR, 0x0002);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_SD_CAL_REFTIM_START, 0x000B);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_SD_CAL_PLLCNT_START, 0x0137);
+
+	/* PLL registers */
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PDIAG_PLL0_CP_PADJ_M0, 0x0509);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PDIAG_PLL0_CP_IADJ_M0, 0x0F00);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PDIAG_PLL0_FILT_PADJ_M0, 0x0F08);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_DSM_DIAG_M0, 0x0004);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PDIAG_PLL1_CP_PADJ_M0, 0x0509);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PDIAG_PLL1_CP_IADJ_M0, 0x0F00);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PDIAG_PLL1_FILT_PADJ_M0, 0x0F08);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_DSM_DIAG_M0, 0x0004);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_VCOCAL_INIT_TMR, 0x00C0);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_VCOCAL_ITER_TMR, 0x0004);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_VCOCAL_INIT_TMR, 0x00C0);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_VCOCAL_ITER_TMR, 0x0004);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_VCOCAL_REFTIM_START, 0x0260);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_VCOCAL_TCTRL, 0x0003);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_VCOCAL_REFTIM_START, 0x0260);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_VCOCAL_TCTRL, 0x0003);
+}
+
+/*
+ * Set registers responsible for enabling and configuring SSC, with second and
+ * third register values provided by parameters.
+ */
+static void cdns_torrent_dp_enable_ssc_19_2mhz(struct cdns_torrent_phy *cdns_phy,
+					   u32 ctrl2_val, u32 ctrl3_val)
+{
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_SS_CTRL1_M0, 0x0001);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_SS_CTRL1_M0, ctrl2_val);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_SS_CTRL1_M0, ctrl3_val);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_SS_CTRL4_M0, 0x0003);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_SS_CTRL1_M0, 0x0001);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_SS_CTRL1_M0, ctrl2_val);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_SS_CTRL1_M0, ctrl3_val);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_SS_CTRL4_M0, 0x0003);
+}
+
+static void cdns_torrent_dp_pma_cmn_vco_cfg_19_2mhz(struct cdns_torrent_phy *cdns_phy,
+						u32 rate, bool ssc)
+{
+
+	/* Assumes 19.2 MHz refclock */
+	switch (rate) {
+	/* Setting VCO for 10.8GHz */
+	case 2700:
+	case 5400:
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_INTDIV_M0, 0x0119);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_FRACDIVL_M0, 0x4000);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_FRACDIVH_M0, 0x0002);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_HIGH_THR_M0, 0x00BC);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PDIAG_PLL0_CTRL_M0, 0x0012);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_INTDIV_M0, 0x0119);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_FRACDIVL_M0, 0x4000);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_FRACDIVH_M0, 0x0002);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_HIGH_THR_M0, 0x00BC);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PDIAG_PLL1_CTRL_M0, 0x0012);
+		if (ssc)
+			cdns_torrent_dp_enable_ssc_19_2mhz(cdns_phy, 0x033A, 0x006A);
+		break;
+	/* Setting VCO for 9.72GHz */
+	case 1620:
+	case 2430:
+	case 3240:
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_INTDIV_M0, 0x01FA);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_FRACDIVL_M0, 0x4000);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_FRACDIVH_M0, 0x0002);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_HIGH_THR_M0, 0x0152);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PDIAG_PLL0_CTRL_M0, 0x0002);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_INTDIV_M0, 0x01FA);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_FRACDIVL_M0, 0x4000);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_FRACDIVH_M0, 0x0002);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_HIGH_THR_M0, 0x0152);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PDIAG_PLL1_CTRL_M0, 0x0002);
+		if (ssc)
+			cdns_torrent_dp_enable_ssc_19_2mhz(cdns_phy, 0x05DD, 0x0069);
+		break;
+	/* Setting VCO for 8.64GHz */
+	case 2160:
+	case 4320:
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_INTDIV_M0, 0x01C2);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_FRACDIVL_M0, 0x0000);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_FRACDIVH_M0, 0x0002);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_HIGH_THR_M0, 0x012C);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PDIAG_PLL0_CTRL_M0, 0x0002);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_INTDIV_M0, 0x01C2);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_FRACDIVL_M0, 0x0000);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_FRACDIVH_M0, 0x0002);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_HIGH_THR_M0, 0x012C);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PDIAG_PLL1_CTRL_M0, 0x0002);
+		if (ssc)
+			cdns_torrent_dp_enable_ssc_19_2mhz(cdns_phy, 0x0536, 0x0069);
+		break;
+	/* Setting VCO for 8.1GHz */
+	case 8100:
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_INTDIV_M0, 0x01A5);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_FRACDIVL_M0, 0xE000);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_FRACDIVH_M0, 0x0002);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_HIGH_THR_M0, 0x011A);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PDIAG_PLL0_CTRL_M0, 0x0002);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_INTDIV_M0, 0x01A5);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_FRACDIVL_M0, 0xE000);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_FRACDIVH_M0, 0x0002);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_HIGH_THR_M0, 0x011A);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PDIAG_PLL1_CTRL_M0, 0x0002);
+		if (ssc)
+			cdns_torrent_dp_enable_ssc_19_2mhz(cdns_phy, 0x04D7, 0x006A);
+		break;
+	}
+
+	if (ssc) {
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_VCOCAL_PLLCNT_START, 0x025E);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_LOCK_PLLCNT_THR, 0x0005);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_VCOCAL_PLLCNT_START, 0x025E);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_LOCK_PLLCNT_THR, 0x0005);
+	} else {
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_VCOCAL_PLLCNT_START, 0x0260);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_VCOCAL_PLLCNT_START, 0x0260);
+		/* Set reset register values to disable SSC */
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_SS_CTRL1_M0, 0x0002);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_SS_CTRL2_M0, 0x0000);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_SS_CTRL3_M0, 0x0000);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_SS_CTRL4_M0, 0x0000);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_LOCK_PLLCNT_THR, 0x0003);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_SS_CTRL1_M0, 0x0002);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_SS_CTRL2_M0, 0x0000);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_SS_CTRL3_M0, 0x0000);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_SS_CTRL4_M0, 0x0000);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_LOCK_PLLCNT_THR, 0x0003);
+	}
+
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_LOCK_REFCNT_START, 0x0099);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_LOCK_PLLCNT_START, 0x0099);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_LOCK_REFCNT_START, 0x0099);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_LOCK_PLLCNT_START, 0x0099);
+}
+
+
+static void cdns_torrent_dp_pma_cmn_cfg_25mhz(struct cdns_torrent_phy *cdns_phy)
+{
+	/* refclock registers - assumes 25 MHz refclock */
+	cdns_dp_phy_write_phy(cdns_phy, CMN_SSM_BIAS_TMR, 0x0019);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLLSM0_PLLPRE_TMR, 0x0032);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLLSM0_PLLLOCK_TMR, 0x00D1);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLLSM1_PLLPRE_TMR, 0x0032);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLLSM1_PLLLOCK_TMR, 0x00D1);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_BGCAL_INIT_TMR, 0x007D);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_BGCAL_ITER_TMR, 0x007D);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_IBCAL_INIT_TMR, 0x0019);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_TXPUCAL_INIT_TMR, 0x001E);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_TXPUCAL_ITER_TMR, 0x0006);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_TXPDCAL_INIT_TMR, 0x001E);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_TXPDCAL_ITER_TMR, 0x0006);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_RXCAL_INIT_TMR, 0x02EE);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_RXCAL_ITER_TMR, 0x0006);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_SD_CAL_INIT_TMR, 0x0002);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_SD_CAL_ITER_TMR, 0x0002);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_SD_CAL_REFTIM_START, 0x000E);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_SD_CAL_PLLCNT_START, 0x012B);
+	/* PLL registers */
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PDIAG_PLL0_CP_PADJ_M0, 0x0509);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PDIAG_PLL0_CP_IADJ_M0, 0x0F00);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PDIAG_PLL0_FILT_PADJ_M0, 0x0F08);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_DSM_DIAG_M0, 0x0004);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PDIAG_PLL1_CP_PADJ_M0, 0x0509);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PDIAG_PLL1_CP_IADJ_M0, 0x0F00);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PDIAG_PLL1_FILT_PADJ_M0, 0x0F08);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_DSM_DIAG_M0, 0x0004);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_VCOCAL_INIT_TMR, 0x00FA);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_VCOCAL_ITER_TMR, 0x0004);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_VCOCAL_INIT_TMR, 0x00FA);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_VCOCAL_ITER_TMR, 0x0004);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_VCOCAL_REFTIM_START, 0x0317);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_VCOCAL_TCTRL, 0x0003);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_VCOCAL_REFTIM_START, 0x0317);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_VCOCAL_TCTRL, 0x0003);
+}
+
+/*
+ * Set registers responsible for enabling and configuring SSC, with second
+ * register value provided by a parameter.
+ */
+static void cdns_torrent_dp_enable_ssc_25mhz(struct cdns_torrent_phy *cdns_phy, u32 ctrl2_val)
+{
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_SS_CTRL1_M0, 0x0001);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_SS_CTRL1_M0, ctrl2_val);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_SS_CTRL1_M0, 0x007F);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_SS_CTRL4_M0, 0x0003);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_SS_CTRL1_M0, 0x0001);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_SS_CTRL1_M0, ctrl2_val);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_SS_CTRL1_M0, 0x007F);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_SS_CTRL4_M0, 0x0003);
+}
+
+static void cdns_torrent_dp_pma_cmn_vco_cfg_25mhz(struct cdns_torrent_phy *cdns_phy,
+					      u32 rate, bool ssc)
+{
+	/* Assumes 25 MHz refclock */
+	switch (rate) {
+	/* Setting VCO for 10.8GHz */
+	case 2700:
+	case 5400:
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_INTDIV_M0, 0x01B0);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_FRACDIVL_M0, 0x0000);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_FRACDIVH_M0, 0x0002);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_HIGH_THR_M0, 0x0120);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_INTDIV_M0, 0x01B0);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_FRACDIVL_M0, 0x0000);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_FRACDIVH_M0, 0x0002);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_HIGH_THR_M0, 0x0120);
+		if (ssc)
+			cdns_torrent_dp_enable_ssc_25mhz(cdns_phy, 0x0423);
+		break;
+	/* Setting VCO for 9.72GHz */
+	case 1620:
+	case 2430:
+	case 3240:
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_INTDIV_M0, 0x0184);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_FRACDIVL_M0, 0xCCCD);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_FRACDIVH_M0, 0x0002);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_HIGH_THR_M0, 0x0104);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_INTDIV_M0, 0x0184);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_FRACDIVL_M0, 0xCCCD);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_FRACDIVH_M0, 0x0002);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_HIGH_THR_M0, 0x0104);
+		if (ssc)
+			cdns_torrent_dp_enable_ssc_25mhz(cdns_phy, 0x03B9);
+		break;
+	/* Setting VCO for 8.64GHz */
+	case 2160:
+	case 4320:
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_INTDIV_M0, 0x0159);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_FRACDIVL_M0, 0x999A);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_FRACDIVH_M0, 0x0002);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_HIGH_THR_M0, 0x00E7);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_INTDIV_M0, 0x0159);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_FRACDIVL_M0, 0x999A);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_FRACDIVH_M0, 0x0002);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_HIGH_THR_M0, 0x00E7);
+		if (ssc)
+			cdns_torrent_dp_enable_ssc_25mhz(cdns_phy, 0x034F);
+		break;
+	/* Setting VCO for 8.1GHz */
+	case 8100:
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_INTDIV_M0, 0x0144);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_FRACDIVL_M0, 0x0000);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_FRACDIVH_M0, 0x0002);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_HIGH_THR_M0, 0x00D8);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_INTDIV_M0, 0x0144);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_FRACDIVL_M0, 0x0000);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_FRACDIVH_M0, 0x0002);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_HIGH_THR_M0, 0x00D8);
+		if (ssc)
+			cdns_torrent_dp_enable_ssc_25mhz(cdns_phy, 0x031A);
+		break;
+	}
+
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PDIAG_PLL0_CTRL_M0, 0x0002);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PDIAG_PLL1_CTRL_M0, 0x0002);
+
+	if (ssc) {
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_VCOCAL_PLLCNT_START, 0x0315);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_LOCK_PLLCNT_THR, 0x0005);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_VCOCAL_PLLCNT_START, 0x0315);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_LOCK_PLLCNT_THR, 0x0005);
+	} else {
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_VCOCAL_PLLCNT_START, 0x0317);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_VCOCAL_PLLCNT_START, 0x0317);
+		/* Set reset register values to disable SSC */
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_SS_CTRL1_M0, 0x0002);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_SS_CTRL2_M0, 0x0000);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_SS_CTRL3_M0, 0x0000);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_SS_CTRL4_M0, 0x0000);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_LOCK_PLLCNT_THR, 0x0003);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_SS_CTRL1_M0, 0x0002);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_SS_CTRL2_M0, 0x0000);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_SS_CTRL3_M0, 0x0000);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_SS_CTRL4_M0, 0x0000);
+		cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_LOCK_PLLCNT_THR, 0x0003);
+	}
+
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_LOCK_REFCNT_START, 0x00C7);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL0_LOCK_PLLCNT_START, 0x00C7);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_LOCK_REFCNT_START, 0x00C7);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PLL1_LOCK_PLLCNT_START, 0x00C7);
+}
+
+
+static void cdns_torrent_dp_pma_cmn_rate(struct cdns_torrent_phy *cdns_phy,
+				     u32 rate, u32 lanes)
+{
+	unsigned int clk_sel_val = 0;
+	unsigned int hsclk_div_val = 0;
+	unsigned int i;
+
+	/* 16'h0000 for single DP link configuration */
+	cdns_dp_phy_write_phy(cdns_phy, PHY_PLL_CFG, 0x0000);
+
+	switch (rate) {
+	case 1620:
+		clk_sel_val = 0x0f01;
+		hsclk_div_val = 2;
+		break;
+	case 2160:
+	case 2430:
+	case 2700:
+		clk_sel_val = 0x0701;
+		hsclk_div_val = 1;
+		break;
+	case 3240:
+		clk_sel_val = 0x0b00;
+		hsclk_div_val = 2;
+		break;
+	case 4320:
+	case 5400:
+		clk_sel_val = 0x0301;
+		hsclk_div_val = 0;
+		break;
+	case 8100:
+		clk_sel_val = 0x0200;
+		hsclk_div_val = 0;
+		break;
+	}
+
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PDIAG_PLL0_CLK_SEL_M0, clk_sel_val);
+	cdns_dp_phy_write_phy(cdns_phy, CMN_PDIAG_PLL1_CLK_SEL_M0, clk_sel_val);
+
+	/* PMA lane configuration to deal with multi-link operation */
+	for (i = 0; i < cdns_phy->num_lanes; i++)
+		cdns_dp_phy_write_phy(cdns_phy, XCVR_DIAG_HSCLK_DIV(i),
+				      hsclk_div_val);
+}
+
+static void cdns_torrent_dp_pma_lane_cfg(struct cdns_torrent_phy *cdns_phy,
+				     unsigned int lane)
+{
+	/* Per lane, refclock-dependent receiver detection setting */
+	if (cdns_phy->ref_clk_rate ==  REF_CLK_19_2MHz)
+		cdns_dp_phy_write_phy(cdns_phy, TX_RCVDET_ST_TMR(lane), 0x0780);
+	else if (cdns_phy->ref_clk_rate == REF_CLK_25MHz)
+		cdns_dp_phy_write_phy(cdns_phy, TX_RCVDET_ST_TMR(lane), 0x09C4);
+
+	/* Writing Tx/Rx Power State Controllers registers */
+	cdns_dp_phy_write_phy(cdns_phy, TX_PSC_A0(lane), 0x00FB);
+	cdns_dp_phy_write_phy(cdns_phy, TX_PSC_A2(lane), 0x04AA);
+	cdns_dp_phy_write_phy(cdns_phy, TX_PSC_A3(lane), 0x04AA);
+	cdns_dp_phy_write_phy(cdns_phy, RX_PSC_A0(lane), 0x0000);
+	cdns_dp_phy_write_phy(cdns_phy, RX_PSC_A2(lane), 0x0000);
+	cdns_dp_phy_write_phy(cdns_phy, RX_PSC_A3(lane), 0x0000);
+
+	cdns_dp_phy_write_phy(cdns_phy, RX_PSC_CAL(lane), 0x0000);
+
+	cdns_dp_phy_write_phy(cdns_phy, RX_REE_GCSM1_CTRL(lane), 0x0000);
+	cdns_dp_phy_write_phy(cdns_phy, RX_REE_GCSM2_CTRL(lane), 0x0000);
+	cdns_dp_phy_write_phy(cdns_phy, RX_REE_PERGCSM_CTRL(lane), 0x0000);
+
+	cdns_dp_phy_write_phy(cdns_phy, XCVR_DIAG_BIDI_CTRL(lane), 0x000F);
+	cdns_dp_phy_write_phy(cdns_phy, XCVR_DIAG_PLLDRC_CTRL(lane), 0x0001);
+	cdns_dp_phy_write_phy(cdns_phy, XCVR_DIAG_HSCLK_SEL(lane), 0x0000);
+}
+
+static int cdns_torrent_dp_run(struct cdns_torrent_phy *cdns_phy)
+{
+	unsigned int read_val;
+	int ret;
+
+	/*
+	 * waiting for ACK of pma_xcvr_pllclk_en_ln_*, only for the
+	 * master lane
+	 */
+	ret = cdns_phy_read_dp_poll_timeout(cdns_phy, PHY_PMA_XCVR_PLLCLK_EN_ACK,
+					    read_val, read_val & 1, 0, POLL_TIMEOUT_US);
+	if (ret == -ETIMEDOUT) {
+		dev_err(cdns_phy->dev,
+			"timeout waiting for link PLL clock enable ack\n");
+		return ret;
+	}
+
+	ndelay(100);
+
+	ret = cdns_torrent_dp_set_power_state(cdns_phy, cdns_phy->num_lanes, POWERSTATE_A2);
+	if (ret)
+		return ret;
+
+	ret = cdns_torrent_dp_set_power_state(cdns_phy, cdns_phy->num_lanes, POWERSTATE_A0);
+
+	return ret;
+}
+
+static void cdns_dp_phy_write_field(struct cdns_torrent_phy *cdns_phy,
+				    unsigned int offset,
+				    unsigned char start_bit,
+				    unsigned char num_bits,
+				    unsigned int val)
+{
+	unsigned int read_val;
+
+	read_val = cdns_dp_phy_read_dp(cdns_phy, offset);
+	cdns_dp_phy_write_dp(cdns_phy, offset,
+			     ((val << start_bit) | (read_val & ~(((1 << num_bits) - 1) <<
+								 start_bit))));
+}
+
+static int cdns_torrent_phy_on(struct phy *phy)
+{
+	struct cdns_torrent_phy *cdns_phy = phy_get_drvdata(phy);
+
+	/* Take the PHY lane group out of reset */
+	return reset_control_deassert(cdns_phy->phy_rst);
+}
+
+static int cdns_torrent_phy_off(struct phy *phy)
+{
+	struct cdns_torrent_phy *cdns_phy = phy_get_drvdata(phy);
+
+	return reset_control_assert(cdns_phy->phy_rst);
+}
+
+static int cdns_torrent_phy_probe(struct platform_device *pdev)
+{
+	struct resource *regs;
+	struct cdns_torrent_phy *cdns_phy;
+	struct device *dev = &pdev->dev;
+	struct phy_provider *phy_provider;
+	struct phy *phy;
+	int err;
+
+	cdns_phy = devm_kzalloc(dev, sizeof(*cdns_phy), GFP_KERNEL);
+	if (!cdns_phy)
+		return -ENOMEM;
+
+	cdns_phy->dev = &pdev->dev;
+
+	phy = devm_phy_create(dev, NULL, &cdns_torrent_phy_ops);
+	if (IS_ERR(phy)) {
+		dev_err(dev, "failed to create Torrent PHY\n");
+		return PTR_ERR(phy);
+	}
+
+	regs = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+	cdns_phy->base = devm_ioremap_resource(&pdev->dev, regs);
+	if (IS_ERR(cdns_phy->base))
+		return PTR_ERR(cdns_phy->base);
+
+	regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	cdns_phy->sd_base = devm_ioremap_resource(&pdev->dev, regs);
+	if (IS_ERR(cdns_phy->sd_base))
+		return PTR_ERR(cdns_phy->sd_base);
+
+	cdns_phy->phy_rst = devm_reset_control_array_get_exclusive(dev);
+
+	err = device_property_read_u32(dev, "num_lanes",
+				       &cdns_phy->num_lanes);
+	if (err)
+		cdns_phy->num_lanes = DEFAULT_NUM_LANES;
+
+	switch (cdns_phy->num_lanes) {
+	case 1:
+	case 2:
+	case 4:
+		/* valid number of lanes */
+		break;
+	default:
+		dev_err(dev, "unsupported number of lanes: %d\n",
+			cdns_phy->num_lanes);
+		return -EINVAL;
+	}
+
+	err = device_property_read_u32(dev, "max_bit_rate",
+				       &cdns_phy->max_bit_rate);
+	if (err)
+		cdns_phy->max_bit_rate = DEFAULT_MAX_BIT_RATE;
+
+	switch (cdns_phy->max_bit_rate) {
+	case 1620:
+	case 2160:
+	case 2430:
+	case 2700:
+	case 3240:
+	case 4320:
+	case 5400:
+	case 8100:
+		/* valid bit rate */
+		break;
+	default:
+		dev_err(dev, "unsupported max bit rate: %dMbps\n",
+			cdns_phy->max_bit_rate);
+		return -EINVAL;
+	}
+
+	cdns_phy->clk = devm_clk_get(dev, "refclk");
+	if (IS_ERR(cdns_phy->clk)) {
+		dev_err(dev, "phy ref clock not found\n");
+		return PTR_ERR(cdns_phy->clk);
+	}
+
+	phy_set_drvdata(phy, cdns_phy);
+
+	phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
+
+	dev_info(dev, "%d lanes, max bit rate %d.%03d Gbps\n",
+		 cdns_phy->num_lanes,
+		 cdns_phy->max_bit_rate / 1000,
+		 cdns_phy->max_bit_rate % 1000);
+
+	return PTR_ERR_OR_ZERO(phy_provider);
+}
+
+static int cdns_torrent_dp_set_power_state(struct cdns_torrent_phy *cdns_phy,
+				       u32 num_lanes,
+				       enum phy_powerstate powerstate)
+{
+	/* Register value for power state for a single byte. */
+	u32 value_part;
+	u32 value;
+	u32 mask;
+	u32 read_val;
+	u32 ret;
+
+	switch (powerstate) {
+	case (POWERSTATE_A0):
+		value_part = 0x01U;
+		break;
+	case (POWERSTATE_A2):
+		value_part = 0x04U;
+		break;
+	default:
+		/* Powerstate A3 */
+		value_part = 0x08U;
+		break;
+	}
+
+	/* Select values of registers and mask, depending on enabled lane count. */
+	switch (num_lanes) {
+	// lane 0
+	case (1):
+		value = value_part;
+		mask = 0x0000003FU;
+		break;
+	// lanes 0-1
+	case (2):
+		value = (value_part
+			 | (value_part << 8));
+		mask = 0x00003F3FU;
+		break;
+	// lanes 0-3, all
+	default:
+		value = (value_part
+			 | (value_part << 8)
+			 | (value_part << 16)
+			 | (value_part << 24));
+		mask = 0x3F3F3F3FU;
+		break;
+	}
+
+	/* Set power state A<n>. */
+	cdns_dp_phy_write_dp(cdns_phy, PHY_PMA_XCVR_POWER_STATE_REQ, value);
+	/* Wait, until PHY acknowledges power state completion. */
+	ret = cdns_phy_read_dp_poll_timeout(cdns_phy, PHY_PMA_XCVR_POWER_STATE_ACK,
+					    read_val, (read_val & mask) == value, 0,
+					    POLL_TIMEOUT_US);
+	cdns_dp_phy_write_dp(cdns_phy, PHY_PMA_XCVR_POWER_STATE_REQ, 0x00000000);
+	ndelay(100);
+
+	return ret;
+}
+
+/*
+ * Enable or disable PLL for selected lanes.
+ */
+static int cdns_torrent_dp_set_pll_en(struct cdns_torrent_phy *cdns_phy,
+				  struct phy_configure_opts_dp *dp,
+				  bool enable)
+{
+	u32 read_val;
+	u32 ret;
+
+	/*
+	 * Used to determine, which bits to check for or enable in
+	 * PHY_PMA_XCVR_PLLCLK_EN register.
+	 */
+	u32 pll_reg_bits;
+	/* used to enable or disable lanes. */
+	u32 pll_reg_write_val;
+
+	/* Select values of registers and mask, depending on enabled lane count. */
+	switch (dp->lanes) {
+	// lane 0
+	case (1):
+		pll_reg_bits = 0x00000001;
+		break;
+	// lanes 0-1
+	case (2):
+		pll_reg_bits = 0x00000003;
+		break;
+	// lanes 0-3, all
+	default:
+		pll_reg_bits = 0x0000000F;
+		break;
+	}
+
+	if (enable)
+		pll_reg_write_val = pll_reg_bits;
+	else
+		pll_reg_write_val = 0x00000000;
+
+	cdns_dp_phy_write_dp(cdns_phy, PHY_PMA_XCVR_PLLCLK_EN, pll_reg_write_val);
+
+	/* Wait for acknowledgment from PHY. */
+	ret = cdns_phy_read_dp_poll_timeout(cdns_phy, PHY_PMA_XCVR_PLLCLK_EN_ACK,
+					    read_val,
+					    (read_val & pll_reg_bits) == pll_reg_write_val,
+					    0, POLL_TIMEOUT_US);
+	ndelay(100);
+	return ret;
+}
+
+
+/*
+ * Perform register operations related to setting link rate, once powerstate is
+ * set and PLL disable request was processed.
+ */
+static int cdns_torrent_dp_configure_rate(struct cdns_torrent_phy *cdns_phy,
+				      struct phy_configure_opts_dp *dp)
+{
+	u32 ret;
+	u32 read_val;
+
+	/* Disable the cmn_pll0_en before re-programming the new data rate. */
+	cdns_dp_phy_write_phy(cdns_phy, PHY_PMA_PLL_RAW_CTRL, 0);
+
+	/* Wait for PLL ready de-assertion. */
+	/* For PLL0 - PHY_PMA_CMN_CTRL2[2] == 1 */
+	ret = cdns_phy_read_poll_timeout(PHY_PMA_CMN_CTRL2,
+					 read_val,
+					 ((read_val >> 2) & 0x01) != 0,
+					 0, POLL_TIMEOUT_US);
+	if (ret)
+		return ret;
+	ndelay(200);
+
+	/* DP Rate Change - VCO Output settings. */
+	if (cdns_phy->ref_clk_rate ==  REF_CLK_19_2MHz) {
+		/* PMA common configuration 19.2MHz */
+		cdns_torrent_dp_pma_cmn_vco_cfg_19_2mhz(cdns_phy, dp->link_rate, dp->ssc);
+		cdns_torrent_dp_pma_cmn_cfg_19_2mhz(cdns_phy);
+	} else if (cdns_phy->ref_clk_rate == REF_CLK_25MHz) {
+		/* PMA common configuration 25MHz */
+		cdns_torrent_dp_pma_cmn_vco_cfg_25mhz(cdns_phy, dp->link_rate, dp->ssc);
+		cdns_torrent_dp_pma_cmn_cfg_25mhz(cdns_phy);
+	}
+	cdns_torrent_dp_pma_cmn_rate(cdns_phy, dp->link_rate, dp->lanes);
+
+	/* Enable the cmn_pll0_en. */
+	cdns_dp_phy_write_phy(cdns_phy, PHY_PMA_PLL_RAW_CTRL, 0x3);
+
+	/* Wait for PLL ready assertion. */
+	/* For PLL0 - PHY_PMA_CMN_CTRL2[0] == 1 */
+	ret = cdns_phy_read_poll_timeout(PHY_PMA_CMN_CTRL2,
+					 read_val,
+					 (read_val & 0x01) != 0,
+					 0, POLL_TIMEOUT_US);
+	return ret;
+}
+
+/*
+ * Verify, that parameters to configure PHY with are correct.
+ */
+static int cdns_torrent_dp_verify_config(struct cdns_torrent_phy *cdns_phy,
+				     struct phy_configure_opts_dp *dp)
+{
+	u8 i;
+
+	/* If changing link rate was required, verify it's supported. */
+	if (dp->set_rate) {
+		switch (dp->link_rate) {
+		case 1620:
+		case 2160:
+		case 2430:
+		case 2700:
+		case 3240:
+		case 4320:
+		case 5400:
+		case 8100:
+			/* valid bit rate */
+			break;
+		default:
+			return -EINVAL;
+		}
+	}
+
+	/* Verify lane count. */
+	switch (dp->lanes) {
+	case 1:
+	case 2:
+	case 4:
+		/* valid lane count. */
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* Check against actual number of PHY's lanes. */
+	if (dp->lanes > cdns_phy->num_lanes)
+		return -EINVAL;
+
+	/*
+	 * If changing voltages is required, check swing and pre-emphasis levels,
+	 * per-lane.
+	 */
+	if (dp->set_voltages) {
+		/* Lane count verified previously. */
+		for (i = 0; i < dp->lanes; i++) {
+			if ((dp->voltage[i] > 3) || (dp->pre[i] > 3))
+				return -EINVAL;
+
+			/* Sum of voltage swing and pre-emphasis levels cannot exceed 3. */
+			if (dp->voltage[i] + dp->pre[i] > 3)
+				return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+/* Set power state A0 and PLL clock enable to 0 on enabled lanes. */
+static void cdns_torrent_dp_set_a0_pll(struct cdns_torrent_phy *cdns_phy,
+				   u32 num_lanes)
+{
+	u32 pwr_state = cdns_dp_phy_read_dp(cdns_phy, PHY_PMA_XCVR_POWER_STATE_REQ);
+	u32 pll_clk_en = cdns_dp_phy_read_dp(cdns_phy, PHY_PMA_XCVR_PLLCLK_EN);
+
+	/* Lane 0 is always enabled. */
+	pwr_state &= ~(PMA_XCVR_POWER_STATE_REQ_LN_MASK << PHY_POWER_STATE_LN_0);
+	pll_clk_en &= ~0x01U;
+
+	if (num_lanes > 1) {
+		/* lane 1 */
+		pwr_state &= ~(PMA_XCVR_POWER_STATE_REQ_LN_MASK << PHY_POWER_STATE_LN_1);
+		pll_clk_en &= ~(0x01U << 1);
+	}
+
+	if (num_lanes > 2) {
+		/* lanes 2 and 3 */
+		pwr_state &= ~(PMA_XCVR_POWER_STATE_REQ_LN_MASK << PHY_POWER_STATE_LN_2);
+		pwr_state &= ~(PMA_XCVR_POWER_STATE_REQ_LN_MASK << PHY_POWER_STATE_LN_3);
+		pll_clk_en &= ~(0x01U << 2);
+		pll_clk_en &= ~(0x01U << 3);
+	}
+
+	cdns_dp_phy_write_dp(cdns_phy, PHY_PMA_XCVR_POWER_STATE_REQ, pwr_state);
+	cdns_dp_phy_write_dp(cdns_phy, PHY_PMA_XCVR_PLLCLK_EN, pll_clk_en);
+}
+
+/* Configure lane count as required. */
+static int cdns_torrent_dp_set_lanes(struct cdns_torrent_phy *cdns_phy,
+				 struct phy_configure_opts_dp *dp)
+{
+	u32 value;
+	u32 ret;
+	u8 lane_mask = (1 << dp->lanes) - 1;
+
+	value = cdns_dp_phy_read_dp(cdns_phy, PHY_RESET);
+	/* clear pma_tx_elec_idle_ln_* bits. */
+	value &= ~PMA_TX_ELEC_IDLE_MASK;
+	/* Assert pma_tx_elec_idle_ln_* for disabled lanes. */
+	value |= ((~lane_mask) << PMA_TX_ELEC_IDLE_SHIFT) & PMA_TX_ELEC_IDLE_MASK;
+	cdns_dp_phy_write_dp(cdns_phy, PHY_RESET, value);
+
+	/* reset the link by asserting phy_l00_reset_n low */
+	cdns_dp_phy_write_dp(cdns_phy, PHY_RESET, value & (~PHY_L00_RESET_N_MASK));
+
+	/*
+	 * Assert lane reset on unused lanes and lane 0 so they remain in reset
+	 * and powered down when re-enabling the link
+	 */
+	value = (value & 0x0000FFF0) | (0x0000000E & lane_mask);
+	cdns_dp_phy_write_dp(cdns_phy, PHY_RESET, value);
+
+	cdns_torrent_dp_set_a0_pll(cdns_phy, dp->lanes);
+
+	/* release phy_l0*_reset_n based on used laneCount */
+	value = (value & 0x0000FFF0) | (0x0000000F & lane_mask);
+	cdns_dp_phy_write_dp(cdns_phy, PHY_RESET, value);
+
+	/* Wait, until PHY gets ready after releasing PHY reset signal. */
+	ret = cdns_torrent_dp_wait_pma_cmn_ready(cdns_phy);
+	if (ret)
+		return ret;
+
+	ndelay(100);
+
+	/* release pma_xcvr_pllclk_en_ln_*, only for the master lane */
+	cdns_dp_phy_write_dp(cdns_phy, PHY_PMA_XCVR_PLLCLK_EN, 0x0001);
+
+	ret = cdns_torrent_dp_run(cdns_phy);
+
+	return ret;
+}
+
+/* Configure link rate as required. */
+static int cdns_torrent_dp_set_rate(struct cdns_torrent_phy *cdns_phy,
+				struct phy_configure_opts_dp *dp)
+{
+	u32 ret;
+
+	ret = cdns_torrent_dp_set_power_state(cdns_phy, dp->lanes, POWERSTATE_A3);
+	if (ret)
+		return ret;
+	ret = cdns_torrent_dp_set_pll_en(cdns_phy, dp, false);
+	if (ret)
+		return ret;
+	ndelay(200);
+
+	ret = cdns_torrent_dp_configure_rate(cdns_phy, dp);
+	if (ret)
+		return ret;
+	ndelay(200);
+
+	ret = cdns_torrent_dp_set_pll_en(cdns_phy, dp, true);
+	if (ret)
+		return ret;
+	ret = cdns_torrent_dp_set_power_state(cdns_phy, dp->lanes, POWERSTATE_A2);
+	if (ret)
+		return ret;
+	ret = cdns_torrent_dp_set_power_state(cdns_phy, dp->lanes, POWERSTATE_A0);
+	if (ret)
+		return ret;
+	ndelay(900);
+
+	return ret;
+}
+
+/* Configure voltage swing and pre-emphasis for all enabled lanes. */
+static void cdns_torrent_dp_set_voltages(struct cdns_torrent_phy *cdns_phy,
+				     struct phy_configure_opts_dp *dp)
+{
+	u8 lane;
+	u16 phy_reg;
+
+	for (lane = 0; lane < dp->lanes; lane++) {
+		phy_reg = cdns_dp_phy_read_phy(cdns_phy, TX_DIAG_ACYA(lane));
+		/*
+		 * Write 1 to register bit TX_DIAG_ACYA[0] to freeze the
+		 * current state of the analog TX driver.
+		 */
+		phy_reg |= TX_DIAG_ACYA_HBDC_MASK;
+		cdns_dp_phy_write_phy(cdns_phy, TX_DIAG_ACYA(lane), phy_reg);
+
+		cdns_dp_phy_write_phy(cdns_phy, TX_TXCC_CTRL(lane), 0x08A4);
+		phy_reg = voltage_coeffs[dp->voltage[lane]][dp->pre[lane]].diag_tx_drv;
+		cdns_dp_phy_write_phy(cdns_phy, DRV_DIAG_TX_DRV(lane), phy_reg);
+		phy_reg = voltage_coeffs[dp->voltage[lane]][dp->pre[lane]].mgnfs_mult;
+		cdns_dp_phy_write_phy(cdns_phy, TX_TXCC_MGNFS_MULT_000(lane), phy_reg);
+		phy_reg = voltage_coeffs[dp->voltage[lane]][dp->pre[lane]].cpost_mult;
+		cdns_dp_phy_write_phy(cdns_phy, TX_TXCC_CPOST_MULT_00(lane), phy_reg);
+
+		phy_reg = cdns_dp_phy_read_phy(cdns_phy, TX_DIAG_ACYA(lane));
+		/*
+		 * Write 0 to register bit TX_DIAG_ACYA[0] to allow the state of
+		 * analog TX driver to reflect the new programmed one.
+		 */
+		phy_reg &= ~TX_DIAG_ACYA_HBDC_MASK;
+		cdns_dp_phy_write_phy(cdns_phy, TX_DIAG_ACYA(lane), phy_reg);
+	}
+};
+
+static int cdns_torrent_dp_configure(struct phy *phy, union phy_configure_opts *opts)
+{
+	struct cdns_torrent_phy *cdns_phy = phy_get_drvdata(phy);
+	int ret;
+
+	dev_dbg(&phy->dev,
+		"cdns_dp_phy_configure rate %d, lanes %d(%d), vs %d,%d,%d,%d, pe %d,%d,%d,%d, ssc %d\n",
+		opts->dp.set_rate	? opts->dp.link_rate : -1,
+		opts->dp.lanes, opts->dp.set_lanes,
+		(opts->dp.set_voltages && opts->dp.lanes > 0)	? opts->dp.voltage[0]	: -1,
+		(opts->dp.set_voltages && opts->dp.lanes > 1)	? opts->dp.voltage[1]	: -1,
+		(opts->dp.set_voltages && opts->dp.lanes > 2)	? opts->dp.voltage[2]	: -1,
+		(opts->dp.set_voltages && opts->dp.lanes > 3)	? opts->dp.voltage[3]	: -1,
+		(opts->dp.set_voltages && opts->dp.lanes > 0)	? opts->dp.pre[0]	: -1,
+		(opts->dp.set_voltages && opts->dp.lanes > 1)	? opts->dp.pre[1]	: -1,
+		(opts->dp.set_voltages && opts->dp.lanes > 2)	? opts->dp.pre[2]	: -1,
+		(opts->dp.set_voltages && opts->dp.lanes > 3)	? opts->dp.pre[3]	: -1,
+		opts->dp.ssc);
+
+	ret = cdns_torrent_dp_verify_config(cdns_phy, &opts->dp);
+	if (ret) {
+		dev_err(&phy->dev, "invalid params for phy configure\n");
+		return ret;
+	}
+
+	if (opts->dp.set_lanes) {
+		ret = cdns_torrent_dp_set_lanes(cdns_phy, &opts->dp);
+		if (ret) {
+			dev_err(&phy->dev, "cdns_torrent_dp_set_lanes failed\n");
+			return ret;
+		}
+	}
+
+	if (opts->dp.set_rate) {
+		ret = cdns_torrent_dp_set_rate(cdns_phy, &opts->dp);
+		if (ret) {
+			dev_err(&phy->dev, "cdns_torrent_dp_set_rate failed\n");
+			return ret;
+		}
+	}
+
+	if (opts->dp.set_voltages)
+		cdns_torrent_dp_set_voltages(cdns_phy, &opts->dp);
+
+	return ret;
+}
+
+static const struct of_device_id cdns_torrent_phy_of_match[] = {
+	{
+		.compatible = "cdns,torrent-phy"
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(of, cdns_torrent_phy_of_match);
+
+static struct platform_driver cdns_torrent_phy_driver = {
+	.probe	= cdns_torrent_phy_probe,
+	.driver = {
+		.name	= "cdns-torrent-phy",
+		.of_match_table	= cdns_torrent_phy_of_match,
+	}
+};
+module_platform_driver(cdns_torrent_phy_driver);
+
+MODULE_AUTHOR("Cadence Design Systems, Inc.");
+MODULE_DESCRIPTION("Cadence Torrent PHY driver");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/phy/Kconfig b/drivers/phy/Kconfig
--- a/drivers/phy/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/phy/Kconfig	2020-06-19 21:00:31.420227716 -0400
@@ -43,6 +43,7 @@
 source "drivers/phy/allwinner/Kconfig"
 source "drivers/phy/amlogic/Kconfig"
 source "drivers/phy/broadcom/Kconfig"
+source "drivers/phy/cadence/Kconfig"
 source "drivers/phy/hisilicon/Kconfig"
 source "drivers/phy/lantiq/Kconfig"
 source "drivers/phy/marvell/Kconfig"
diff -Naur --no-dereference a/drivers/phy/Makefile b/drivers/phy/Makefile
--- a/drivers/phy/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/phy/Makefile	2020-06-19 21:00:31.420227716 -0400
@@ -15,6 +15,7 @@
 obj-$(CONFIG_ARCH_ROCKCHIP)		+= rockchip/
 obj-$(CONFIG_ARCH_TEGRA)		+= tegra/
 obj-y					+= broadcom/	\
+					   cadence/	\
 					   hisilicon/	\
 					   marvell/	\
 					   motorola/	\
diff -Naur --no-dereference a/drivers/phy/marvell/phy-mvebu-cp110-comphy.c b/drivers/phy/marvell/phy-mvebu-cp110-comphy.c
--- a/drivers/phy/marvell/phy-mvebu-cp110-comphy.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/phy/marvell/phy-mvebu-cp110-comphy.c	2020-06-19 21:00:31.420227716 -0400
@@ -9,6 +9,7 @@
 #include <linux/iopoll.h>
 #include <linux/mfd/syscon.h>
 #include <linux/module.h>
+#include <linux/phy.h>
 #include <linux/phy/phy.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
@@ -116,41 +117,43 @@
 
 struct mvebu_comhy_conf {
 	enum phy_mode mode;
+	int submode;
 	unsigned lane;
 	unsigned port;
 	u32 mux;
 };
 
-#define MVEBU_COMPHY_CONF(_lane, _port, _mode, _mux)	\
+#define MVEBU_COMPHY_CONF(_lane, _port, _submode, _mux)	\
 	{						\
 		.lane = _lane,				\
 		.port = _port,				\
-		.mode = _mode,				\
+		.mode = PHY_MODE_ETHERNET,		\
+		.submode = _submode,			\
 		.mux = _mux,				\
 	}
 
 static const struct mvebu_comhy_conf mvebu_comphy_cp110_modes[] = {
 	/* lane 0 */
-	MVEBU_COMPHY_CONF(0, 1, PHY_MODE_SGMII, 0x1),
-	MVEBU_COMPHY_CONF(0, 1, PHY_MODE_2500SGMII, 0x1),
+	MVEBU_COMPHY_CONF(0, 1, PHY_INTERFACE_MODE_SGMII, 0x1),
+	MVEBU_COMPHY_CONF(0, 1, PHY_INTERFACE_MODE_2500BASEX, 0x1),
 	/* lane 1 */
-	MVEBU_COMPHY_CONF(1, 2, PHY_MODE_SGMII, 0x1),
-	MVEBU_COMPHY_CONF(1, 2, PHY_MODE_2500SGMII, 0x1),
+	MVEBU_COMPHY_CONF(1, 2, PHY_INTERFACE_MODE_SGMII, 0x1),
+	MVEBU_COMPHY_CONF(1, 2, PHY_INTERFACE_MODE_2500BASEX, 0x1),
 	/* lane 2 */
-	MVEBU_COMPHY_CONF(2, 0, PHY_MODE_SGMII, 0x1),
-	MVEBU_COMPHY_CONF(2, 0, PHY_MODE_2500SGMII, 0x1),
-	MVEBU_COMPHY_CONF(2, 0, PHY_MODE_10GKR, 0x1),
+	MVEBU_COMPHY_CONF(2, 0, PHY_INTERFACE_MODE_SGMII, 0x1),
+	MVEBU_COMPHY_CONF(2, 0, PHY_INTERFACE_MODE_2500BASEX, 0x1),
+	MVEBU_COMPHY_CONF(2, 0, PHY_INTERFACE_MODE_10GKR, 0x1),
 	/* lane 3 */
-	MVEBU_COMPHY_CONF(3, 1, PHY_MODE_SGMII, 0x2),
-	MVEBU_COMPHY_CONF(3, 1, PHY_MODE_2500SGMII, 0x2),
+	MVEBU_COMPHY_CONF(3, 1, PHY_INTERFACE_MODE_SGMII, 0x2),
+	MVEBU_COMPHY_CONF(3, 1, PHY_INTERFACE_MODE_2500BASEX, 0x2),
 	/* lane 4 */
-	MVEBU_COMPHY_CONF(4, 0, PHY_MODE_SGMII, 0x2),
-	MVEBU_COMPHY_CONF(4, 0, PHY_MODE_2500SGMII, 0x2),
-	MVEBU_COMPHY_CONF(4, 0, PHY_MODE_10GKR, 0x2),
-	MVEBU_COMPHY_CONF(4, 1, PHY_MODE_SGMII, 0x1),
+	MVEBU_COMPHY_CONF(4, 0, PHY_INTERFACE_MODE_SGMII, 0x2),
+	MVEBU_COMPHY_CONF(4, 0, PHY_INTERFACE_MODE_2500BASEX, 0x2),
+	MVEBU_COMPHY_CONF(4, 0, PHY_INTERFACE_MODE_10GKR, 0x2),
+	MVEBU_COMPHY_CONF(4, 1, PHY_INTERFACE_MODE_SGMII, 0x1),
 	/* lane 5 */
-	MVEBU_COMPHY_CONF(5, 2, PHY_MODE_SGMII, 0x1),
-	MVEBU_COMPHY_CONF(5, 2, PHY_MODE_2500SGMII, 0x1),
+	MVEBU_COMPHY_CONF(5, 2, PHY_INTERFACE_MODE_SGMII, 0x1),
+	MVEBU_COMPHY_CONF(5, 2, PHY_INTERFACE_MODE_2500BASEX, 0x1),
 };
 
 struct mvebu_comphy_priv {
@@ -163,10 +166,12 @@
 	struct mvebu_comphy_priv *priv;
 	unsigned id;
 	enum phy_mode mode;
+	int submode;
 	int port;
 };
 
-static int mvebu_comphy_get_mux(int lane, int port, enum phy_mode mode)
+static int mvebu_comphy_get_mux(int lane, int port,
+				enum phy_mode mode, int submode)
 {
 	int i, n = ARRAY_SIZE(mvebu_comphy_cp110_modes);
 
@@ -177,7 +182,8 @@
 	for (i = 0; i < n; i++) {
 		if (mvebu_comphy_cp110_modes[i].lane == lane &&
 		    mvebu_comphy_cp110_modes[i].port == port &&
-		    mvebu_comphy_cp110_modes[i].mode == mode)
+		    mvebu_comphy_cp110_modes[i].mode == mode &&
+		    mvebu_comphy_cp110_modes[i].submode == submode)
 			break;
 	}
 
@@ -187,8 +193,7 @@
 	return mvebu_comphy_cp110_modes[i].mux;
 }
 
-static void mvebu_comphy_ethernet_init_reset(struct mvebu_comphy_lane *lane,
-					     enum phy_mode mode)
+static void mvebu_comphy_ethernet_init_reset(struct mvebu_comphy_lane *lane)
 {
 	struct mvebu_comphy_priv *priv = lane->priv;
 	u32 val;
@@ -206,14 +211,14 @@
 		 MVEBU_COMPHY_SERDES_CFG0_HALF_BUS |
 		 MVEBU_COMPHY_SERDES_CFG0_GEN_RX(0xf) |
 		 MVEBU_COMPHY_SERDES_CFG0_GEN_TX(0xf));
-	if (mode == PHY_MODE_10GKR)
+	if (lane->submode == PHY_INTERFACE_MODE_10GKR)
 		val |= MVEBU_COMPHY_SERDES_CFG0_GEN_RX(0xe) |
 		       MVEBU_COMPHY_SERDES_CFG0_GEN_TX(0xe);
-	else if (mode == PHY_MODE_2500SGMII)
+	else if (lane->submode == PHY_INTERFACE_MODE_2500BASEX)
 		val |= MVEBU_COMPHY_SERDES_CFG0_GEN_RX(0x8) |
 		       MVEBU_COMPHY_SERDES_CFG0_GEN_TX(0x8) |
 		       MVEBU_COMPHY_SERDES_CFG0_HALF_BUS;
-	else if (mode == PHY_MODE_SGMII)
+	else if (lane->submode == PHY_INTERFACE_MODE_SGMII)
 		val |= MVEBU_COMPHY_SERDES_CFG0_GEN_RX(0x6) |
 		       MVEBU_COMPHY_SERDES_CFG0_GEN_TX(0x6) |
 		       MVEBU_COMPHY_SERDES_CFG0_HALF_BUS;
@@ -243,7 +248,7 @@
 	/* refclk selection */
 	val = readl(priv->base + MVEBU_COMPHY_MISC_CTRL0(lane->id));
 	val &= ~MVEBU_COMPHY_MISC_CTRL0_REFCLK_SEL;
-	if (mode == PHY_MODE_10GKR)
+	if (lane->submode == PHY_INTERFACE_MODE_10GKR)
 		val |= MVEBU_COMPHY_MISC_CTRL0_ICP_FORCE;
 	writel(val, priv->base + MVEBU_COMPHY_MISC_CTRL0(lane->id));
 
@@ -261,8 +266,7 @@
 	writel(val, priv->base + MVEBU_COMPHY_LOOPBACK(lane->id));
 }
 
-static int mvebu_comphy_init_plls(struct mvebu_comphy_lane *lane,
-				  enum phy_mode mode)
+static int mvebu_comphy_init_plls(struct mvebu_comphy_lane *lane)
 {
 	struct mvebu_comphy_priv *priv = lane->priv;
 	u32 val;
@@ -303,13 +307,13 @@
 	return 0;
 }
 
-static int mvebu_comphy_set_mode_sgmii(struct phy *phy, enum phy_mode mode)
+static int mvebu_comphy_set_mode_sgmii(struct phy *phy)
 {
 	struct mvebu_comphy_lane *lane = phy_get_drvdata(phy);
 	struct mvebu_comphy_priv *priv = lane->priv;
 	u32 val;
 
-	mvebu_comphy_ethernet_init_reset(lane, mode);
+	mvebu_comphy_ethernet_init_reset(lane);
 
 	val = readl(priv->base + MVEBU_COMPHY_RX_CTRL1(lane->id));
 	val &= ~MVEBU_COMPHY_RX_CTRL1_CLK8T_EN;
@@ -330,7 +334,7 @@
 	val |= MVEBU_COMPHY_GEN1_S0_TX_EMPH(0x1);
 	writel(val, priv->base + MVEBU_COMPHY_GEN1_S0(lane->id));
 
-	return mvebu_comphy_init_plls(lane, PHY_MODE_SGMII);
+	return mvebu_comphy_init_plls(lane);
 }
 
 static int mvebu_comphy_set_mode_10gkr(struct phy *phy)
@@ -339,7 +343,7 @@
 	struct mvebu_comphy_priv *priv = lane->priv;
 	u32 val;
 
-	mvebu_comphy_ethernet_init_reset(lane, PHY_MODE_10GKR);
+	mvebu_comphy_ethernet_init_reset(lane);
 
 	val = readl(priv->base + MVEBU_COMPHY_RX_CTRL1(lane->id));
 	val |= MVEBU_COMPHY_RX_CTRL1_RXCLK2X_SEL |
@@ -469,7 +473,7 @@
 	val |= MVEBU_COMPHY_EXT_SELV_RX_SAMPL(0x1a);
 	writel(val, priv->base + MVEBU_COMPHY_EXT_SELV(lane->id));
 
-	return mvebu_comphy_init_plls(lane, PHY_MODE_10GKR);
+	return mvebu_comphy_init_plls(lane);
 }
 
 static int mvebu_comphy_power_on(struct phy *phy)
@@ -479,7 +483,8 @@
 	int ret, mux;
 	u32 val;
 
-	mux = mvebu_comphy_get_mux(lane->id, lane->port, lane->mode);
+	mux = mvebu_comphy_get_mux(lane->id, lane->port,
+				   lane->mode, lane->submode);
 	if (mux < 0)
 		return -ENOTSUPP;
 
@@ -492,12 +497,12 @@
 	val |= mux << MVEBU_COMPHY_SELECTOR_PHY(lane->id);
 	regmap_write(priv->regmap, MVEBU_COMPHY_SELECTOR, val);
 
-	switch (lane->mode) {
-	case PHY_MODE_SGMII:
-	case PHY_MODE_2500SGMII:
-		ret = mvebu_comphy_set_mode_sgmii(phy, lane->mode);
+	switch (lane->submode) {
+	case PHY_INTERFACE_MODE_SGMII:
+	case PHY_INTERFACE_MODE_2500BASEX:
+		ret = mvebu_comphy_set_mode_sgmii(phy);
 		break;
-	case PHY_MODE_10GKR:
+	case PHY_INTERFACE_MODE_10GKR:
 		ret = mvebu_comphy_set_mode_10gkr(phy);
 		break;
 	default:
@@ -512,14 +517,22 @@
 	return ret;
 }
 
-static int mvebu_comphy_set_mode(struct phy *phy, enum phy_mode mode)
+static int mvebu_comphy_set_mode(struct phy *phy,
+				 enum phy_mode mode, int submode)
 {
 	struct mvebu_comphy_lane *lane = phy_get_drvdata(phy);
 
-	if (mvebu_comphy_get_mux(lane->id, lane->port, mode) < 0)
+	if (mode != PHY_MODE_ETHERNET)
+		return -EINVAL;
+
+	if (submode == PHY_INTERFACE_MODE_1000BASEX)
+		submode = PHY_INTERFACE_MODE_SGMII;
+
+	if (mvebu_comphy_get_mux(lane->id, lane->port, mode, submode) < 0)
 		return -EINVAL;
 
 	lane->mode = mode;
+	lane->submode = submode;
 	return 0;
 }
 
diff -Naur --no-dereference a/drivers/phy/mediatek/phy-mtk-tphy.c b/drivers/phy/mediatek/phy-mtk-tphy.c
--- a/drivers/phy/mediatek/phy-mtk-tphy.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/phy/mediatek/phy-mtk-tphy.c	2020-06-19 21:00:31.420227716 -0400
@@ -971,7 +971,7 @@
 	return 0;
 }
 
-static int mtk_phy_set_mode(struct phy *phy, enum phy_mode mode)
+static int mtk_phy_set_mode(struct phy *phy, enum phy_mode mode, int submode)
 {
 	struct mtk_phy_instance *instance = phy_get_drvdata(phy);
 	struct mtk_tphy *tphy = dev_get_drvdata(phy->dev.parent);
diff -Naur --no-dereference a/drivers/phy/mediatek/phy-mtk-xsphy.c b/drivers/phy/mediatek/phy-mtk-xsphy.c
--- a/drivers/phy/mediatek/phy-mtk-xsphy.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/phy/mediatek/phy-mtk-xsphy.c	2020-06-19 21:00:31.424227716 -0400
@@ -426,7 +426,7 @@
 	return 0;
 }
 
-static int mtk_phy_set_mode(struct phy *phy, enum phy_mode mode)
+static int mtk_phy_set_mode(struct phy *phy, enum phy_mode mode, int submode)
 {
 	struct xsphy_instance *inst = phy_get_drvdata(phy);
 	struct mtk_xsphy *xsphy = dev_get_drvdata(phy->dev.parent);
diff -Naur --no-dereference a/drivers/phy/phy-core.c b/drivers/phy/phy-core.c
--- a/drivers/phy/phy-core.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/phy/phy-core.c	2020-06-19 21:00:31.424227716 -0400
@@ -360,7 +360,7 @@
 }
 EXPORT_SYMBOL_GPL(phy_power_off);
 
-int phy_set_mode(struct phy *phy, enum phy_mode mode)
+int phy_set_mode_ext(struct phy *phy, enum phy_mode mode, int submode)
 {
 	int ret;
 
@@ -368,14 +368,14 @@
 		return 0;
 
 	mutex_lock(&phy->mutex);
-	ret = phy->ops->set_mode(phy, mode);
+	ret = phy->ops->set_mode(phy, mode, submode);
 	if (!ret)
 		phy->attrs.mode = mode;
 	mutex_unlock(&phy->mutex);
 
 	return ret;
 }
-EXPORT_SYMBOL_GPL(phy_set_mode);
+EXPORT_SYMBOL_GPL(phy_set_mode_ext);
 
 int phy_reset(struct phy *phy)
 {
@@ -384,10 +384,16 @@
 	if (!phy || !phy->ops->reset)
 		return 0;
 
+	ret = phy_pm_runtime_get_sync(phy);
+	if (ret < 0 && ret != -ENOTSUPP)
+		return ret;
+
 	mutex_lock(&phy->mutex);
 	ret = phy->ops->reset(phy);
 	mutex_unlock(&phy->mutex);
 
+	phy_pm_runtime_put(phy);
+
 	return ret;
 }
 EXPORT_SYMBOL_GPL(phy_reset);
@@ -408,6 +414,70 @@
 EXPORT_SYMBOL_GPL(phy_calibrate);
 
 /**
+ * phy_configure() - Changes the phy parameters
+ * @phy: the phy returned by phy_get()
+ * @opts: New configuration to apply
+ *
+ * Used to change the PHY parameters. phy_init() must have been called
+ * on the phy. The configuration will be applied on the current phy
+ * mode, that can be changed using phy_set_mode().
+ *
+ * Returns: 0 if successful, an negative error code otherwise
+ */
+int phy_configure(struct phy *phy, union phy_configure_opts *opts)
+{
+	int ret;
+
+	if (!phy)
+		return -EINVAL;
+
+	if (!phy->ops->configure)
+		return -EOPNOTSUPP;
+
+	mutex_lock(&phy->mutex);
+	ret = phy->ops->configure(phy, opts);
+	mutex_unlock(&phy->mutex);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(phy_configure);
+
+/**
+ * phy_validate() - Checks the phy parameters
+ * @phy: the phy returned by phy_get()
+ * @mode: phy_mode the configuration is applicable to.
+ * @submode: PHY submode the configuration is applicable to.
+ * @opts: Configuration to check
+ *
+ * Used to check that the current set of parameters can be handled by
+ * the phy. Implementations are free to tune the parameters passed as
+ * arguments if needed by some implementation detail or
+ * constraints. It will not change any actual configuration of the
+ * PHY, so calling it as many times as deemed fit will have no side
+ * effect.
+ *
+ * Returns: 0 if successful, an negative error code otherwise
+ */
+int phy_validate(struct phy *phy, enum phy_mode mode, int submode,
+		 union phy_configure_opts *opts)
+{
+	int ret;
+
+	if (!phy)
+		return -EINVAL;
+
+	if (!phy->ops->validate)
+		return -EOPNOTSUPP;
+
+	mutex_lock(&phy->mutex);
+	ret = phy->ops->validate(phy, mode, submode, opts);
+	mutex_unlock(&phy->mutex);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(phy_validate);
+
+/**
  * _of_phy_get() - lookup and obtain a reference to a phy by phandle
  * @np: device_node for which to get the phy
  * @index: the index of the phy
@@ -500,6 +570,11 @@
 	if (!phy || IS_ERR(phy))
 		return;
 
+	mutex_lock(&phy->mutex);
+	if (phy->ops->release)
+		phy->ops->release(phy);
+	mutex_unlock(&phy->mutex);
+
 	module_put(phy->ops->owner);
 	put_device(&phy->dev);
 }
diff -Naur --no-dereference a/drivers/phy/qualcomm/phy-qcom-qmp.c b/drivers/phy/qualcomm/phy-qcom-qmp.c
--- a/drivers/phy/qualcomm/phy-qcom-qmp.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/phy/qualcomm/phy-qcom-qmp.c	2020-06-19 21:00:31.424227716 -0400
@@ -1191,7 +1191,8 @@
 	return 0;
 }
 
-static int qcom_qmp_phy_set_mode(struct phy *phy, enum phy_mode mode)
+static int qcom_qmp_phy_set_mode(struct phy *phy,
+				 enum phy_mode mode, int submode)
 {
 	struct qmp_phy *qphy = phy_get_drvdata(phy);
 	struct qcom_qmp *qmp = qphy->qmp;
diff -Naur --no-dereference a/drivers/phy/qualcomm/phy-qcom-qusb2.c b/drivers/phy/qualcomm/phy-qcom-qusb2.c
--- a/drivers/phy/qualcomm/phy-qcom-qusb2.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/phy/qualcomm/phy-qcom-qusb2.c	2020-06-19 21:00:31.424227716 -0400
@@ -425,7 +425,8 @@
 				 HSTX_TRIM_MASK);
 }
 
-static int qusb2_phy_set_mode(struct phy *phy, enum phy_mode mode)
+static int qusb2_phy_set_mode(struct phy *phy,
+			      enum phy_mode mode, int submode)
 {
 	struct qusb2_phy *qphy = phy_get_drvdata(phy);
 
diff -Naur --no-dereference a/drivers/phy/qualcomm/phy-qcom-ufs-qmp-14nm.c b/drivers/phy/qualcomm/phy-qcom-ufs-qmp-14nm.c
--- a/drivers/phy/qualcomm/phy-qcom-ufs-qmp-14nm.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/phy/qualcomm/phy-qcom-ufs-qmp-14nm.c	2020-06-19 21:00:31.424227716 -0400
@@ -65,7 +65,8 @@
 }
 
 static
-int ufs_qcom_phy_qmp_14nm_set_mode(struct phy *generic_phy, enum phy_mode mode)
+int ufs_qcom_phy_qmp_14nm_set_mode(struct phy *generic_phy,
+				   enum phy_mode mode, int submode)
 {
 	struct ufs_qcom_phy *phy_common = get_ufs_qcom_phy(generic_phy);
 
diff -Naur --no-dereference a/drivers/phy/qualcomm/phy-qcom-ufs-qmp-20nm.c b/drivers/phy/qualcomm/phy-qcom-ufs-qmp-20nm.c
--- a/drivers/phy/qualcomm/phy-qcom-ufs-qmp-20nm.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/phy/qualcomm/phy-qcom-ufs-qmp-20nm.c	2020-06-19 21:00:31.424227716 -0400
@@ -84,7 +84,8 @@
 }
 
 static
-int ufs_qcom_phy_qmp_20nm_set_mode(struct phy *generic_phy, enum phy_mode mode)
+int ufs_qcom_phy_qmp_20nm_set_mode(struct phy *generic_phy,
+				   enum phy_mode mode, int submode)
 {
 	struct ufs_qcom_phy *phy_common = get_ufs_qcom_phy(generic_phy);
 
diff -Naur --no-dereference a/drivers/phy/qualcomm/phy-qcom-usb-hs.c b/drivers/phy/qualcomm/phy-qcom-usb-hs.c
--- a/drivers/phy/qualcomm/phy-qcom-usb-hs.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/phy/qualcomm/phy-qcom-usb-hs.c	2020-06-19 21:00:31.424227716 -0400
@@ -42,7 +42,8 @@
 	struct notifier_block vbus_notify;
 };
 
-static int qcom_usb_hs_phy_set_mode(struct phy *phy, enum phy_mode mode)
+static int qcom_usb_hs_phy_set_mode(struct phy *phy,
+				    enum phy_mode mode, int submode)
 {
 	struct qcom_usb_hs_phy *uphy = phy_get_drvdata(phy);
 	u8 addr;
diff -Naur --no-dereference a/drivers/phy/ti/Kconfig b/drivers/phy/ti/Kconfig
--- a/drivers/phy/ti/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/phy/ti/Kconfig	2020-06-19 21:00:31.424227716 -0400
@@ -20,6 +20,33 @@
 	help
 	  Enable this for dm816x USB to work.
 
+config PHY_AM654_SERDES
+	tristate "TI AM654 SERDES support"
+	depends on OF && ARCH_K3 || COMPILE_TEST
+	depends on COMMON_CLK
+	select GENERIC_PHY
+	select MULTIPLEXER
+	select REGMAP_MMIO
+	select MUX_MMIO
+	help
+	  This option enables support for TI AM654 SerDes PHY used for
+	  PCIe.
+
+config PHY_J721E_WIZ
+	tristate "TI J721E WIZ (SERDES Wrapper) support"
+	depends on OF && ARCH_K3 || COMPILE_TEST
+	depends on COMMON_CLK
+	select GENERIC_PHY
+	select MULTIPLEXER
+	select REGMAP_MMIO
+	select MUX_MMIO
+	help
+	  This option enables support for WIZ module present in TI's J721E
+	  SoC. WIZ is a serdes wrapper used to configure some of the input
+	  signals to the SERDES (Sierra/Torrent). This driver configures
+	  three clock selects (pll0, pll1, dig) and resets for each of the
+	  lanes.
+
 config OMAP_CONTROL_PHY
 	tristate "OMAP CONTROL PHY Driver"
 	depends on ARCH_OMAP2PLUS || COMPILE_TEST
@@ -33,12 +60,11 @@
 
 config OMAP_USB2
 	tristate "OMAP USB2 PHY Driver"
-	depends on ARCH_OMAP2PLUS
+	depends on ARCH_OMAP2PLUS || ARCH_K3
 	depends on USB_SUPPORT
 	select GENERIC_PHY
 	select USB_PHY
-	select OMAP_CONTROL_PHY
-	depends on OMAP_OCP2SCP
+	select OMAP_CONTROL_PHY if ARCH_OMAP2PLUS
 	help
 	  Enable this to support the transceiver that is part of SOC. This
 	  driver takes care of all the PHY functionality apart from comparator.
@@ -50,7 +76,6 @@
 	depends on ARCH_OMAP2PLUS || COMPILE_TEST
 	select GENERIC_PHY
 	select OMAP_CONTROL_PHY
-	depends on OMAP_OCP2SCP
 	help
 	  Enable this to support the PIPE3 PHY that is part of TI SOCs. This
 	  driver takes care of all the PHY functionality apart from comparator.
@@ -76,3 +101,11 @@
 	  family chips (including the TWL5030 and TPS659x0 devices).
 	  This transceiver supports high and full speed devices plus,
 	  in host mode, low speed.
+
+config PHY_TI_KEYSTONE_SERDES
+	tristate "TI Keystone SerDes PHY support"
+	depends on OF && ARCH_KEYSTONE || COMPILE_TEST
+	select GENERIC_PHY
+	help
+	  This option enables support for TI Keystone SerDes PHY found
+	  in peripherals GBE, 10GBE and PCIe.
diff -Naur --no-dereference a/drivers/phy/ti/Makefile b/drivers/phy/ti/Makefile
--- a/drivers/phy/ti/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/phy/ti/Makefile	2020-06-19 21:00:31.424227716 -0400
@@ -6,3 +6,6 @@
 obj-$(CONFIG_TI_PIPE3)			+= phy-ti-pipe3.o
 obj-$(CONFIG_PHY_TUSB1210)		+= phy-tusb1210.o
 obj-$(CONFIG_TWL4030_USB)		+= phy-twl4030-usb.o
+obj-$(CONFIG_PHY_TI_KEYSTONE_SERDES)	+= phy-keystone-serdes.o
+obj-$(CONFIG_PHY_AM654_SERDES)		+= phy-am654-serdes.o
+obj-$(CONFIG_PHY_J721E_WIZ)		+= phy-j721e-wiz.o
diff -Naur --no-dereference a/drivers/phy/ti/phy-am654-serdes.c b/drivers/phy/ti/phy-am654-serdes.c
--- a/drivers/phy/ti/phy-am654-serdes.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/phy/ti/phy-am654-serdes.c	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,539 @@
+// SPDX-License-Identifier: GPL-2.0
+/**
+ * PCIe SERDES driver for AM654x SoC
+ *
+ * Copyright (C) 2018 Texas Instruments
+ * Author: Kishon Vijay Abraham I <kishon@ti.com>
+ */
+
+#include <dt-bindings/phy/phy.h>
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/mux/consumer.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+#include <linux/mfd/syscon.h>
+
+#define CMU_R07C		0x7c
+#define CMU_MASTER_CDN_O	BIT(24)
+
+#define COMLANE_R138		0xb38
+#define CONFIG_VERSION_REG_MASK	GENMASK(23, 16)
+#define CONFIG_VERSION_REG_SHIFT 16
+#define VERSION			0x70
+
+#define COMLANE_R190		0xb90
+#define L1_MASTER_CDN_O		BIT(9)
+
+#define COMLANE_R194		0xb94
+#define CMU_OK_I_0		BIT(19)
+
+#define SERDES_CTRL		0x1fd0
+#define POR_EN			BIT(29)
+
+#define WIZ_LANEXCTL_STS	0x1fe0
+#define TX0_ENABLE_OVL		BIT(31)
+#define TX0_ENABLE_MASK		GENMASK(30, 29)
+#define TX0_ENABLE_SHIFT	29
+#define TX0_DISABLE_STATE	0x0
+#define TX0_SLEEP_STATE		0x1
+#define TX0_SNOOZE_STATE	0x2
+#define TX0_ENABLE_STATE	0x3
+#define RX0_ENABLE_OVL		BIT(15)
+#define RX0_ENABLE_MASK		GENMASK(14, 13)
+#define RX0_ENABLE_SHIFT	13
+#define RX0_DISABLE_STATE	0x0
+#define RX0_SLEEP_STATE		0x1
+#define RX0_SNOOZE_STATE	0x2
+#define RX0_ENABLE_STATE	0x3
+
+#define WIZ_PLL_CTRL		0x1ff4
+#define PLL_ENABLE_OVL		BIT(31)
+#define PLL_ENABLE_MASK		GENMASK(30, 29)
+#define PLL_ENABLE_SHIFT	29
+#define PLL_DISABLE_STATE	0x0
+#define PLL_SLEEP_STATE		0x1
+#define PLL_SNOOZE_STATE	0x2
+#define PLL_ENABLE_STATE	0x3
+#define PLL_OK			BIT(28)
+
+#define PLL_LOCK_TIME		100000	/* in microseconds */
+#define SLEEP_TIME		100	/* in microseconds */
+
+#define LANE_USB3		0x0
+#define LANE_PCIE0_LANE0	0x1
+
+#define LANE_PCIE1_LANE0	0x0
+#define LANE_PCIE0_LANE1	0x1
+
+#define SERDES_NUM_CLOCKS	3
+
+struct serdes_am654_clk_mux {
+	struct clk_hw	hw;
+	struct regmap	*regmap;
+	unsigned int	reg;
+	int		*table;
+	u32		mask;
+	u8		shift;
+	struct clk_init_data clk_data;
+};
+
+#define to_serdes_am654_clk_mux(_hw)	\
+		container_of(_hw, struct serdes_am654_clk_mux, hw)
+
+static struct regmap_config serdes_am654_regmap_config = {
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = 4,
+	.fast_io = true,
+};
+
+struct serdes_am654 {
+	struct regmap		*regmap;
+	struct device		*dev;
+	struct mux_control	*control;
+	bool			busy;
+	u32			type;
+	struct device_node	*of_node;
+	struct clk_onecell_data	clk_data;
+	struct clk		*clks[SERDES_NUM_CLOCKS];
+};
+
+static int serdes_am654_enable_pll(struct serdes_am654 *phy)
+{
+	u32 mask = PLL_ENABLE_OVL | PLL_ENABLE_MASK;
+	u32 val = PLL_ENABLE_OVL | (PLL_ENABLE_STATE << PLL_ENABLE_SHIFT);
+
+	regmap_update_bits(phy->regmap, WIZ_PLL_CTRL, mask, val);
+
+	return regmap_read_poll_timeout(phy->regmap, WIZ_PLL_CTRL, val,
+					val & PLL_OK, 1000, PLL_LOCK_TIME);
+}
+
+static void serdes_am654_disable_pll(struct serdes_am654 *phy)
+{
+	u32 mask = PLL_ENABLE_OVL | PLL_ENABLE_MASK;
+
+	regmap_update_bits(phy->regmap, WIZ_PLL_CTRL, mask, PLL_ENABLE_OVL);
+}
+
+static int serdes_am654_enable_txrx(struct serdes_am654 *phy)
+{
+	u32 mask;
+	u32 val;
+
+	/* Enable TX */
+	mask = TX0_ENABLE_OVL | TX0_ENABLE_MASK;
+	val = TX0_ENABLE_OVL | (TX0_ENABLE_STATE << TX0_ENABLE_SHIFT);
+	regmap_update_bits(phy->regmap, WIZ_LANEXCTL_STS, mask, val);
+
+	/* Enable RX */
+	mask = RX0_ENABLE_OVL | RX0_ENABLE_MASK;
+	val = RX0_ENABLE_OVL | (RX0_ENABLE_STATE << RX0_ENABLE_SHIFT);
+	regmap_update_bits(phy->regmap, WIZ_LANEXCTL_STS, mask, val);
+
+	return 0;
+}
+
+static int serdes_am654_disable_txrx(struct serdes_am654 *phy)
+{
+	u32 mask;
+
+	/* Disable TX */
+	mask = TX0_ENABLE_OVL | TX0_ENABLE_MASK;
+	regmap_update_bits(phy->regmap, WIZ_LANEXCTL_STS, mask, TX0_ENABLE_OVL);
+
+	/* Disable RX */
+	mask = RX0_ENABLE_OVL | RX0_ENABLE_MASK;
+	regmap_update_bits(phy->regmap, WIZ_LANEXCTL_STS, mask, RX0_ENABLE_OVL);
+
+	return 0;
+}
+
+static int serdes_am654_power_on(struct phy *x)
+{
+	struct serdes_am654 *phy = phy_get_drvdata(x);
+	struct device *dev = phy->dev;
+	int ret;
+	u32 val;
+
+	ret = serdes_am654_enable_pll(phy);
+	if (ret) {
+		dev_err(dev, "Failed to enable PLL\n");
+		return ret;
+	}
+
+	ret = serdes_am654_enable_txrx(phy);
+	if (ret) {
+		dev_err(dev, "Failed to enable TX RX\n");
+		return ret;
+	}
+
+	return regmap_read_poll_timeout(phy->regmap, COMLANE_R194, val,
+					val & CMU_OK_I_0, SLEEP_TIME,
+					PLL_LOCK_TIME);
+}
+
+static int serdes_am654_power_off(struct phy *x)
+{
+	struct serdes_am654 *phy = phy_get_drvdata(x);
+
+	serdes_am654_disable_txrx(phy);
+	serdes_am654_disable_pll(phy);
+
+	return 0;
+}
+
+static int serdes_am654_init(struct phy *x)
+{
+	struct serdes_am654 *phy = phy_get_drvdata(x);
+	u32 mask;
+	u32 val;
+
+	mask = CONFIG_VERSION_REG_MASK;
+	val = VERSION << CONFIG_VERSION_REG_SHIFT;
+	regmap_update_bits(phy->regmap, COMLANE_R138, mask, val);
+
+	val = CMU_MASTER_CDN_O;
+	regmap_update_bits(phy->regmap, CMU_R07C, val, val);
+
+	val = L1_MASTER_CDN_O;
+	regmap_update_bits(phy->regmap, COMLANE_R190, val, val);
+
+	return 0;
+}
+
+static int serdes_am654_reset(struct phy *x)
+{
+	struct serdes_am654 *phy = phy_get_drvdata(x);
+	u32 val;
+
+	val = POR_EN;
+	regmap_update_bits(phy->regmap, SERDES_CTRL, val, val);
+	mdelay(1);
+	regmap_update_bits(phy->regmap, SERDES_CTRL, val, 0);
+
+	return 0;
+}
+
+static void serdes_am654_release(struct phy *x)
+{
+	struct serdes_am654 *phy = phy_get_drvdata(x);
+
+	phy->type = PHY_NONE;
+	phy->busy = false;
+	mux_control_deselect(phy->control);
+}
+
+struct phy *serdes_am654_xlate(struct device *dev, struct of_phandle_args
+				 *args)
+{
+	struct serdes_am654 *am654_phy;
+	struct phy *phy;
+	int ret;
+
+	phy = of_phy_simple_xlate(dev, args);
+	if (IS_ERR(phy))
+		return phy;
+
+	am654_phy = phy_get_drvdata(phy);
+	if (am654_phy->busy)
+		return ERR_PTR(-EBUSY);
+
+	ret = mux_control_select(am654_phy->control, args->args[1]);
+	if (ret) {
+		dev_err(dev, "Failed to select SERDES Lane Function\n");
+		return ERR_PTR(ret);
+	}
+
+	am654_phy->busy = true;
+	am654_phy->type = args->args[0];
+
+	return phy;
+}
+
+static const struct phy_ops ops = {
+	.reset		= serdes_am654_reset,
+	.init		= serdes_am654_init,
+	.power_on	= serdes_am654_power_on,
+	.power_off	= serdes_am654_power_off,
+	.release	= serdes_am654_release,
+	.owner		= THIS_MODULE,
+};
+
+static u8 serdes_am654_clk_mux_get_parent(struct clk_hw *hw)
+{
+	struct serdes_am654_clk_mux *mux = to_serdes_am654_clk_mux(hw);
+	unsigned int num_parents = clk_hw_get_num_parents(hw);
+	struct regmap *regmap = mux->regmap;
+	unsigned int reg = mux->reg;
+	unsigned int val;
+	int i;
+
+	regmap_read(regmap, reg, &val);
+	val >>= mux->shift;
+	val &= mux->mask;
+
+	for (i = 0; i < num_parents; i++)
+		if (mux->table[i] == val)
+			return i;
+
+	/*
+	 * No parent? This should never happen!
+	 * Verify if we set a valid parent in serdes_am654_clk_register()
+	 */
+	WARN(1, "Failed to find the parent of %s clock\n", hw->init->name);
+
+	/* Make the parent lookup to fail */
+	return num_parents;
+}
+
+static int serdes_am654_clk_mux_set_parent(struct clk_hw *hw, u8 index)
+{
+	struct serdes_am654_clk_mux *mux = to_serdes_am654_clk_mux(hw);
+	struct regmap *regmap = mux->regmap;
+	unsigned int reg = mux->reg;
+	int val;
+	int ret;
+
+	val = mux->table[index];
+
+	if (val == -1)
+		return -EINVAL;
+
+	val <<= mux->shift;
+	ret = regmap_update_bits(regmap, reg, mux->mask << mux->shift, val);
+
+	return ret;
+}
+
+static const struct clk_ops serdes_am654_clk_mux_ops = {
+	.set_parent = serdes_am654_clk_mux_set_parent,
+	.get_parent = serdes_am654_clk_mux_get_parent,
+};
+
+static int mux_table[SERDES_NUM_CLOCKS][3] = {
+	/*
+	 * The entries represent values for selecting between
+	 * {left input, external reference clock, right input}
+	 * Only one of Left Output or Right Output should be used since
+	 * both left and right output clock uses the same bits and modifying
+	 * one clock will impact the other.
+	 */
+	{ BIT(2),               0, BIT(0) }, /* Mux of CMU refclk */
+	{     -1,          BIT(3), BIT(1) }, /* Mux of Left Output */
+	{ BIT(1), BIT(3) | BIT(1),     -1 }, /* Mux of Right Output */
+};
+
+static int mux_mask[SERDES_NUM_CLOCKS] = { 0x5, 0xa, 0xa };
+
+static int serdes_am654_clk_register(struct serdes_am654 *am654_phy,
+				     const char *clock_name, int clock_num)
+{
+	struct device_node *node = am654_phy->of_node;
+	struct device *dev = am654_phy->dev;
+	struct serdes_am654_clk_mux *mux;
+	struct device_node *regmap_node;
+	const char **parent_names;
+	struct clk_init_data *init;
+	unsigned int num_parents;
+	struct regmap *regmap;
+	const __be32 *addr;
+	unsigned int reg;
+	struct clk *clk;
+
+	mux = devm_kzalloc(dev, sizeof(*mux), GFP_KERNEL);
+	if (!mux)
+		return -ENOMEM;
+
+	init = &mux->clk_data;
+
+	regmap_node = of_parse_phandle(node, "ti,serdes-clk", 0);
+	of_node_put(regmap_node);
+	if (!regmap_node) {
+		dev_err(dev, "Fail to get serdes-clk node\n");
+		return -ENODEV;
+	}
+
+	regmap = syscon_node_to_regmap(regmap_node->parent);
+	if (IS_ERR(regmap)) {
+		dev_err(dev, "Fail to get Syscon regmap\n");
+		return PTR_ERR(regmap);
+	}
+
+	num_parents = of_clk_get_parent_count(node);
+	if (num_parents < 2) {
+		dev_err(dev, "SERDES clock must have parents\n");
+		return -EINVAL;
+	}
+
+	parent_names = devm_kzalloc(dev, (sizeof(char *) * num_parents),
+				    GFP_KERNEL);
+	if (!parent_names)
+		return -ENOMEM;
+
+	of_clk_parent_fill(node, parent_names, num_parents);
+
+	addr = of_get_address(regmap_node, 0, NULL, NULL);
+	if (!addr)
+		return -EINVAL;
+
+	reg = be32_to_cpu(*addr);
+
+	init->ops = &serdes_am654_clk_mux_ops;
+	init->flags = CLK_SET_RATE_NO_REPARENT;
+	init->parent_names = parent_names;
+	init->num_parents = num_parents;
+	init->name = clock_name;
+
+	mux->table = mux_table[clock_num];
+	mux->regmap = regmap;
+	mux->reg = reg;
+	mux->shift = 4;
+	mux->mask = mux_mask[clock_num];
+	mux->hw.init = init;
+
+	/*
+	 * setup a sane default so get_parent() call evaluates
+	 * to a valid parent. Index 1 is the safest choice as
+	 * the default as it is valid value for all of serdes's
+	 * output clocks.
+	 */
+	serdes_am654_clk_mux_set_parent(&mux->hw, 1);
+	clk = devm_clk_register(dev, &mux->hw);
+	if (IS_ERR(clk))
+		return PTR_ERR(clk);
+
+	am654_phy->clks[clock_num] = clk;
+
+	return 0;
+}
+
+static const struct of_device_id serdes_am654_id_table[] = {
+	{
+		.compatible = "ti,phy-am654-serdes",
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(of, serdes_am654_id_table);
+
+static int serdes_am654_probe(struct platform_device *pdev)
+{
+	struct phy_provider *phy_provider;
+	struct device *dev = &pdev->dev;
+	struct device_node *node = dev->of_node;
+	struct clk_onecell_data *clk_data;
+	struct serdes_am654 *am654_phy;
+	struct mux_control *control;
+	const char *clock_name;
+	struct regmap *regmap;
+	struct resource *res;
+	void __iomem *base;
+	struct phy *phy;
+	int ret;
+	int i;
+
+	am654_phy = devm_kzalloc(dev, sizeof(*am654_phy), GFP_KERNEL);
+	if (!am654_phy)
+		return -ENOMEM;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "serdes");
+	base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	regmap = devm_regmap_init_mmio(dev, base, &serdes_am654_regmap_config);
+	if (IS_ERR(regmap)) {
+		dev_err(dev, "Failed to initialize regmap\n");
+		return PTR_ERR(regmap);
+	}
+
+	control = devm_mux_control_get(dev, NULL);
+	if (IS_ERR(control))
+		return PTR_ERR(control);
+
+	am654_phy->dev = dev;
+	am654_phy->of_node = node;
+	am654_phy->regmap = regmap;
+	am654_phy->control = control;
+	am654_phy->type = PHY_NONE;
+
+	platform_set_drvdata(pdev, am654_phy);
+
+	for (i = 0; i < SERDES_NUM_CLOCKS; i++) {
+		ret = of_property_read_string_index(node, "clock-output-names",
+						    i, &clock_name);
+		if (ret) {
+			dev_err(dev, "Failed to get clock name\n");
+			return ret;
+		}
+
+		ret = serdes_am654_clk_register(am654_phy, clock_name, i);
+		if (ret) {
+			dev_err(dev, "Failed to initialize clock %s\n",
+				clock_name);
+			return ret;
+		}
+	}
+
+	clk_data = &am654_phy->clk_data;
+	clk_data->clks = am654_phy->clks;
+	clk_data->clk_num = SERDES_NUM_CLOCKS;
+	ret = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+	if (ret)
+		return ret;
+
+	pm_runtime_enable(dev);
+
+	phy = devm_phy_create(dev, NULL, &ops);
+	if (IS_ERR(phy))
+		return PTR_ERR(phy);
+
+	phy_set_drvdata(phy, am654_phy);
+	phy_provider = devm_of_phy_provider_register(dev, serdes_am654_xlate);
+	if (IS_ERR(phy_provider)) {
+		ret = PTR_ERR(phy_provider);
+		goto clk_err;
+	}
+
+	return 0;
+
+clk_err:
+	of_clk_del_provider(node);
+
+	return ret;
+}
+
+static int serdes_am654_remove(struct platform_device *pdev)
+{
+	struct serdes_am654 *am654_phy = platform_get_drvdata(pdev);
+	struct device_node *node = am654_phy->of_node;
+
+	pm_runtime_disable(&pdev->dev);
+	of_clk_del_provider(node);
+
+	return 0;
+}
+
+static struct platform_driver serdes_am654_driver = {
+	.probe		= serdes_am654_probe,
+	.remove		= serdes_am654_remove,
+	.driver		= {
+		.name	= "phy-am654",
+		.of_match_table = serdes_am654_id_table,
+	},
+};
+module_platform_driver(serdes_am654_driver);
+
+MODULE_ALIAS("platform:phy-am654");
+MODULE_AUTHOR("Texas Instruments Inc.");
+MODULE_DESCRIPTION("TI AM654x SERDES driver");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/phy/ti/phy-da8xx-usb.c b/drivers/phy/ti/phy-da8xx-usb.c
--- a/drivers/phy/ti/phy-da8xx-usb.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/phy/ti/phy-da8xx-usb.c	2020-06-19 21:00:31.424227716 -0400
@@ -93,7 +93,8 @@
 	return 0;
 }
 
-static int da8xx_usb20_phy_set_mode(struct phy *phy, enum phy_mode mode)
+static int da8xx_usb20_phy_set_mode(struct phy *phy,
+				    enum phy_mode mode, int submode)
 {
 	struct da8xx_usb_phy *d_phy = phy_get_drvdata(phy);
 	u32 val;
diff -Naur --no-dereference a/drivers/phy/ti/phy-j721e-wiz.c b/drivers/phy/ti/phy-j721e-wiz.c
--- a/drivers/phy/ti/phy-j721e-wiz.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/phy/ti/phy-j721e-wiz.c	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,991 @@
+// SPDX-License-Identifier: GPL-2.0
+/**
+ * Wrapper driver for SERDES used in J721E
+ *
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Kishon Vijay Abraham I <kishon@ti.com>
+ */
+
+#include <dt-bindings/phy/phy.h>
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/mux/consumer.h>
+#include <linux/of_address.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/regmap.h>
+#include <linux/reset-controller.h>
+#include <dt-bindings/phy/phy.h>
+
+#define WIZ_SERDES_CTRL		0x404
+#define WIZ_SERDES_TOP_CTRL	0x408
+#define WIZ_SERDES_RST		0x40c
+#define WIZ_SERDES_TYPEC	0x410
+#define WIZ_LANECTL(n)		(0x480 + (0x40 * (n)))
+
+#define WIZ_MAX_LANES		4
+#define WIZ_MUX_NUM_CLOCKS	3
+#define WIZ_DIV_NUM_CLOCKS_16G	2
+#define WIZ_DIV_NUM_CLOCKS_10G	1
+
+#define WIZ_SERDES_TYPEC_LN10_SWAP	BIT(30)
+
+enum wiz_lane_standard_mode {
+	LANE_MODE_GEN1,
+	LANE_MODE_GEN2,
+	LANE_MODE_GEN3,
+	LANE_MODE_GEN4,
+};
+
+enum wiz_refclk_mux_sel {
+	PLL0_REFCLK,
+	PLL1_REFCLK,
+	REFCLK_DIG,
+};
+
+enum wiz_refclk_div_sel {
+	CMN_REFCLK,
+	CMN_REFCLK1,
+};
+
+static const struct reg_field por_en = REG_FIELD(WIZ_SERDES_CTRL, 31, 31);
+static const struct reg_field phy_reset_n = REG_FIELD(WIZ_SERDES_RST, 31, 31);
+static const struct reg_field pll1_refclk_mux_sel =
+					REG_FIELD(WIZ_SERDES_RST, 29, 29);
+static const struct reg_field pll0_refclk_mux_sel =
+					REG_FIELD(WIZ_SERDES_RST, 28, 28);
+static const struct reg_field refclk_dig_sel_16g =
+					REG_FIELD(WIZ_SERDES_RST, 24, 25);
+static const struct reg_field refclk_dig_sel_10g =
+					REG_FIELD(WIZ_SERDES_RST, 24, 24);
+static const struct reg_field pma_cmn_refclk_int_mode =
+					REG_FIELD(WIZ_SERDES_TOP_CTRL, 28, 29);
+static const struct reg_field pma_cmn_refclk_mode =
+					REG_FIELD(WIZ_SERDES_TOP_CTRL, 30, 31);
+static const struct reg_field pma_cmn_refclk_dig_div =
+					REG_FIELD(WIZ_SERDES_TOP_CTRL, 26, 27);
+static const struct reg_field pma_cmn_refclk1_dig_div =
+					REG_FIELD(WIZ_SERDES_TOP_CTRL, 24, 25);
+
+static const struct reg_field p_enable[WIZ_MAX_LANES] = {
+	REG_FIELD(WIZ_LANECTL(0), 30, 31),
+	REG_FIELD(WIZ_LANECTL(1), 30, 31),
+	REG_FIELD(WIZ_LANECTL(2), 30, 31),
+	REG_FIELD(WIZ_LANECTL(3), 30, 31),
+};
+
+enum p_enable { P_ENABLE = 2, P_ENABLE_FORCE = 1, P_ENABLE_DISABLE = 0 };
+
+static const struct reg_field p_align[WIZ_MAX_LANES] = {
+	REG_FIELD(WIZ_LANECTL(0), 29, 29),
+	REG_FIELD(WIZ_LANECTL(1), 29, 29),
+	REG_FIELD(WIZ_LANECTL(2), 29, 29),
+	REG_FIELD(WIZ_LANECTL(3), 29, 29),
+};
+
+static const struct reg_field p_raw_auto_start[WIZ_MAX_LANES] = {
+	REG_FIELD(WIZ_LANECTL(0), 28, 28),
+	REG_FIELD(WIZ_LANECTL(1), 28, 28),
+	REG_FIELD(WIZ_LANECTL(2), 28, 28),
+	REG_FIELD(WIZ_LANECTL(3), 28, 28),
+};
+
+static const struct reg_field p_standard_mode[WIZ_MAX_LANES] = {
+	REG_FIELD(WIZ_LANECTL(0), 24, 25),
+	REG_FIELD(WIZ_LANECTL(1), 24, 25),
+	REG_FIELD(WIZ_LANECTL(2), 24, 25),
+	REG_FIELD(WIZ_LANECTL(3), 24, 25),
+};
+
+struct wiz_clk_mux {
+	struct clk_hw		hw;
+	struct regmap_field	*field;
+	u32			*table;
+	struct clk_init_data	clk_data;
+};
+
+#define to_wiz_clk_mux(_hw) container_of(_hw, struct wiz_clk_mux, hw)
+
+struct wiz_clk_divider {
+	struct clk_hw		hw;
+	struct regmap_field	*field;
+	struct clk_div_table	*table;
+	struct clk_init_data	clk_data;
+};
+
+#define to_wiz_clk_div(_hw) container_of(_hw, struct wiz_clk_divider, hw)
+
+struct wiz_clk_mux_sel {
+	struct regmap_field	*field;
+	u32			table[4];
+	const char		*node_name;
+};
+
+struct wiz_clk_div_sel {
+	struct regmap_field	*field;
+	struct clk_div_table	*table;
+	const char		*node_name;
+};
+
+static struct wiz_clk_mux_sel clk_mux_sel_16g[] = {
+	{
+		/*
+		 * Mux value to be configured for each of the input clocks
+		 * in the order populated in device tree
+		 */
+		.table = { 1, 0 },
+		.node_name = "pll0_refclk",
+	},
+	{
+		.table = { 1, 0 },
+		.node_name = "pll1_refclk",
+	},
+	{
+		.table = { 1, 3, 0, 2 },
+		.node_name = "refclk_dig",
+	},
+};
+
+static struct wiz_clk_mux_sel clk_mux_sel_10g[] = {
+	{
+		/*
+		 * Mux value to be configured for each of the input clocks
+		 * in the order populated in device tree
+		 */
+		.table = { 1, 0 },
+		.node_name = "pll0_refclk",
+	},
+	{
+		.table = { 1, 0 },
+		.node_name = "pll1_refclk",
+	},
+	{
+		.table = { 1, 0 },
+		.node_name = "refclk_dig",
+	},
+};
+
+static struct clk_div_table clk_div_table[] = {
+	{ .val = 0, .div = 1, },
+	{ .val = 1, .div = 2, },
+	{ .val = 2, .div = 4, },
+	{ .val = 3, .div = 8, },
+};
+
+static struct wiz_clk_div_sel clk_div_sel[] = {
+	{
+		.table = clk_div_table,
+		.node_name = "cmn_refclk",
+	},
+	{
+		.table = clk_div_table,
+		.node_name = "cmn_refclk1",
+	},
+};
+
+enum wiz_type {
+	J721E_WIZ_16G,
+	J721E_WIZ_10G,
+};
+
+struct wiz {
+	struct regmap		*regmap;
+	struct wiz_clk_mux_sel	*clk_mux_sel;
+	struct wiz_clk_div_sel	*clk_div_sel;
+	unsigned int		clk_div_sel_num;
+	struct regmap_field	*por_en;
+	struct regmap_field	*phy_reset_n;
+	struct regmap_field	*p_enable[WIZ_MAX_LANES];
+	struct regmap_field	*p_align[WIZ_MAX_LANES];
+	struct regmap_field	*p_raw_auto_start[WIZ_MAX_LANES];
+	struct regmap_field	*p_standard_mode[WIZ_MAX_LANES];
+	struct regmap_field	*pma_cmn_refclk_int_mode;
+	struct regmap_field	*pma_cmn_refclk_mode;
+	struct regmap_field	*pma_cmn_refclk_dig_div;
+	struct regmap_field	*pma_cmn_refclk1_dig_div;
+
+	struct device		*dev;
+	u32			num_lanes;
+	struct platform_device	*serdes_pdev;
+	struct reset_controller_dev wiz_phy_reset_dev;
+	struct gpio_desc	*gpio_typec_dir;
+	int			typec_dir_delay;
+
+	enum wiz_type type;
+	u32 lane_modes[WIZ_MAX_LANES];
+};
+
+static int wiz_reset(struct wiz *wiz)
+{
+	int ret;
+
+	ret = regmap_field_write(wiz->por_en, 0x1);
+	if (ret)
+		return ret;
+
+	mdelay(1);
+
+	ret = regmap_field_write(wiz->por_en, 0x0);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int wiz_mode_select(struct wiz *wiz)
+{
+	u32 num_lanes = wiz->num_lanes;
+	enum wiz_lane_standard_mode mode;
+	int ret;
+	int i;
+
+	for (i = 0; i < num_lanes; i++) {
+		if (wiz->lane_modes[i] == PHY_TYPE_DP)
+			mode = LANE_MODE_GEN1;
+		else
+			mode = LANE_MODE_GEN4;
+
+		ret = regmap_field_write(wiz->p_standard_mode[i], mode);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int wiz_init_raw_interface(struct wiz *wiz, bool enable)
+{
+	u32 num_lanes = wiz->num_lanes;
+	int i;
+	int ret;
+
+	for (i = 0; i < num_lanes; i++) {
+		ret = regmap_field_write(wiz->p_align[i], enable);
+		if (ret)
+			return ret;
+
+		ret = regmap_field_write(wiz->p_raw_auto_start[i], enable);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int wiz_init(struct wiz *wiz)
+{
+	struct device *dev = wiz->dev;
+	int ret;
+
+	ret = wiz_reset(wiz);
+	if (ret) {
+		dev_err(dev, "WIZ reset failed\n");
+		return ret;
+	}
+
+	ret = wiz_mode_select(wiz);
+	if (ret) {
+		dev_err(dev, "WIZ mode select failed\n");
+		return ret;
+	}
+
+	ret = wiz_init_raw_interface(wiz, true);
+	if (ret) {
+		dev_err(dev, "WIZ interface initialization failed\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int wiz_regfield_init(struct wiz *wiz)
+{
+	struct wiz_clk_mux_sel *clk_mux_sel;
+	struct wiz_clk_div_sel *clk_div_sel;
+	struct regmap *regmap = wiz->regmap;
+	int num_lanes = wiz->num_lanes;
+	struct device *dev = wiz->dev;
+	int i;
+
+	wiz->por_en = devm_regmap_field_alloc(dev, regmap, por_en);
+	if (IS_ERR(wiz->por_en)) {
+		dev_err(dev, "POR_EN reg field init failed\n");
+		return PTR_ERR(wiz->por_en);
+	}
+
+	wiz->phy_reset_n = devm_regmap_field_alloc(dev, regmap,
+						   phy_reset_n);
+	if (IS_ERR(wiz->phy_reset_n)) {
+		dev_err(dev, "PHY_RESET_N reg field init failed\n");
+		return PTR_ERR(wiz->phy_reset_n);
+	}
+
+	wiz->pma_cmn_refclk_int_mode =
+		devm_regmap_field_alloc(dev, regmap, pma_cmn_refclk_int_mode);
+	if (IS_ERR(wiz->pma_cmn_refclk_int_mode)) {
+		dev_err(dev, "PMA_CMN_REFCLK_INT_MODE reg field init failed\n");
+		return PTR_ERR(wiz->pma_cmn_refclk_int_mode);
+	}
+
+	wiz->pma_cmn_refclk_mode =
+		devm_regmap_field_alloc(dev, regmap, pma_cmn_refclk_mode);
+	if (IS_ERR(wiz->pma_cmn_refclk_mode)) {
+		dev_err(dev, "PMA_CMN_REFCLK_MODE reg field init failed\n");
+		return PTR_ERR(wiz->pma_cmn_refclk_mode);
+	}
+
+	clk_div_sel = &wiz->clk_div_sel[CMN_REFCLK];
+	clk_div_sel->field = devm_regmap_field_alloc(dev, regmap,
+						     pma_cmn_refclk_dig_div);
+	if (IS_ERR(clk_div_sel->field)) {
+		dev_err(dev, "PMA_CMN_REFCLK_DIG_DIV reg field init failed\n");
+		return PTR_ERR(clk_div_sel->field);
+	}
+
+	if (wiz->type == J721E_WIZ_16G) {
+		clk_div_sel = &wiz->clk_div_sel[CMN_REFCLK1];
+		clk_div_sel->field = devm_regmap_field_alloc(dev, regmap,
+							     pma_cmn_refclk1_dig_div);
+		if (IS_ERR(clk_div_sel->field)) {
+			dev_err(dev, "PMA_CMN_REFCLK1_DIG_DIV reg field init failed\n");
+			return PTR_ERR(clk_div_sel->field);
+		}
+	}
+
+	clk_mux_sel = &wiz->clk_mux_sel[PLL0_REFCLK];
+	clk_mux_sel->field = devm_regmap_field_alloc(dev, regmap,
+						     pll0_refclk_mux_sel);
+	if (IS_ERR(clk_mux_sel->field)) {
+		dev_err(dev, "PLL0_REFCLK_SEL reg field init failed\n");
+		return PTR_ERR(clk_mux_sel->field);
+	}
+
+	clk_mux_sel = &wiz->clk_mux_sel[PLL1_REFCLK];
+	clk_mux_sel->field = devm_regmap_field_alloc(dev, regmap,
+						     pll1_refclk_mux_sel);
+	if (IS_ERR(clk_mux_sel->field)) {
+		dev_err(dev, "PLL1_REFCLK_SEL reg field init failed\n");
+		return PTR_ERR(clk_mux_sel->field);
+	}
+
+	clk_mux_sel = &wiz->clk_mux_sel[REFCLK_DIG];
+	if (wiz->type == J721E_WIZ_10G)
+		clk_mux_sel->field =
+			devm_regmap_field_alloc(dev, regmap,
+						refclk_dig_sel_10g);
+	else
+		clk_mux_sel->field =
+			devm_regmap_field_alloc(dev, regmap,
+						refclk_dig_sel_16g);
+
+	if (IS_ERR(clk_mux_sel->field)) {
+		dev_err(dev, "REFCLK_DIG_SEL reg field init failed\n");
+		return PTR_ERR(clk_mux_sel->field);
+	}
+
+	for (i = 0; i < num_lanes; i++) {
+		wiz->p_enable[i] = devm_regmap_field_alloc(dev, regmap,
+							   p_enable[i]);
+		if (IS_ERR(wiz->p_enable[i])) {
+			dev_err(dev, "P%d_ENABLE reg field init failed\n", i);
+			return PTR_ERR(wiz->p_enable[i]);
+		}
+
+		wiz->p_align[i] = devm_regmap_field_alloc(dev, regmap,
+							  p_align[i]);
+		if (IS_ERR(wiz->p_align[i])) {
+			dev_err(dev, "P%d_ALIGN reg field init failed\n", i);
+			return PTR_ERR(wiz->p_align[i]);
+		}
+
+		wiz->p_raw_auto_start[i] =
+		  devm_regmap_field_alloc(dev, regmap, p_raw_auto_start[i]);
+		if (IS_ERR(wiz->p_raw_auto_start[i])) {
+			dev_err(dev, "P%d_RAW_AUTO_START reg field init fail\n",
+				i);
+			return PTR_ERR(wiz->p_raw_auto_start[i]);
+		}
+
+		wiz->p_standard_mode[i] =
+		  devm_regmap_field_alloc(dev, regmap, p_standard_mode[i]);
+		if (IS_ERR(wiz->p_standard_mode[i])) {
+			dev_err(dev, "P%d_STANDARD_MODE reg field init fail\n",
+				i);
+			return PTR_ERR(wiz->p_standard_mode[i]);
+		}
+	}
+
+	return 0;
+}
+
+static u8 wiz_clk_mux_get_parent(struct clk_hw *hw)
+{
+	struct wiz_clk_mux *mux = to_wiz_clk_mux(hw);
+	struct regmap_field *field = mux->field;
+	unsigned int val;
+
+	regmap_field_read(field, &val);
+	return clk_mux_val_to_index(hw, mux->table, 0, val);
+}
+
+static int wiz_clk_mux_set_parent(struct clk_hw *hw, u8 index)
+{
+	struct wiz_clk_mux *mux = to_wiz_clk_mux(hw);
+	struct regmap_field *field = mux->field;
+	int val;
+
+	val = mux->table[index];
+	return regmap_field_write(field, val);
+}
+
+static const struct clk_ops wiz_clk_mux_ops = {
+	.set_parent = wiz_clk_mux_set_parent,
+	.get_parent = wiz_clk_mux_get_parent,
+};
+
+static int wiz_mux_clk_register(struct wiz *wiz, struct device_node *node,
+				struct regmap_field *field, u32 *table)
+{
+	struct device *dev = wiz->dev;
+	struct clk_init_data *init;
+	const char **parent_names;
+	unsigned int num_parents;
+	struct wiz_clk_mux *mux;
+	const char *clk_name;
+	struct clk *clk;
+	int ret;
+
+	mux = devm_kzalloc(dev, sizeof(*mux), GFP_KERNEL);
+	if (!mux)
+		return -ENOMEM;
+
+	num_parents = of_clk_get_parent_count(node);
+	if (num_parents < 2) {
+		dev_err(dev, "SERDES clock must have parents\n");
+		return -EINVAL;
+	}
+
+	parent_names = devm_kzalloc(dev, (sizeof(char *) * num_parents),
+				    GFP_KERNEL);
+	if (!parent_names)
+		return -ENOMEM;
+
+	of_clk_parent_fill(node, parent_names, num_parents);
+
+	ret = of_property_read_string(node, "clock-output-names", &clk_name);
+	if (ret) {
+		dev_err(dev, "Unable to read clock-output-names DT property\n");
+		return ret;
+	}
+
+	init = &mux->clk_data;
+
+	init->ops = &wiz_clk_mux_ops;
+	init->flags = CLK_SET_RATE_NO_REPARENT;
+	init->parent_names = parent_names;
+	init->num_parents = num_parents;
+	init->name = clk_name;
+
+	mux->field = field;
+	mux->table = table;
+	mux->hw.init = init;
+
+	clk = devm_clk_register(dev, &mux->hw);
+	if (IS_ERR(clk))
+		return PTR_ERR(clk);
+
+	ret = of_clk_add_provider(node, of_clk_src_simple_get, clk);
+	if (ret)
+		dev_err(dev, "Failed to add clock provider: %s\n", clk_name);
+
+	return ret;
+}
+
+static unsigned long wiz_clk_div_recalc_rate(struct clk_hw *hw,
+					     unsigned long parent_rate)
+{
+	struct wiz_clk_divider *div = to_wiz_clk_div(hw);
+	struct regmap_field *field = div->field;
+	int val;
+
+	regmap_field_read(field, &val);
+
+	return divider_recalc_rate(hw, parent_rate, val, div->table, 0x0, 2);
+}
+
+static long wiz_clk_div_round_rate(struct clk_hw *hw, unsigned long rate,
+				   unsigned long *prate)
+{
+	struct wiz_clk_divider *div = to_wiz_clk_div(hw);
+
+	return divider_round_rate(hw, rate, prate, div->table, 2, 0x0);
+}
+
+static int wiz_clk_div_set_rate(struct clk_hw *hw, unsigned long rate,
+				unsigned long parent_rate)
+{
+	struct wiz_clk_divider *div = to_wiz_clk_div(hw);
+	struct regmap_field *field = div->field;
+	int val;
+
+	val = divider_get_val(rate, parent_rate, div->table, 2, 0x0);
+	if (val < 0)
+		return val;
+
+	return regmap_field_write(field, val);
+}
+
+static const struct clk_ops wiz_clk_div_ops = {
+	.recalc_rate = wiz_clk_div_recalc_rate,
+	.round_rate = wiz_clk_div_round_rate,
+	.set_rate = wiz_clk_div_set_rate,
+};
+
+static int wiz_div_clk_register(struct wiz *wiz, struct device_node *node,
+				struct regmap_field *field,
+				struct clk_div_table *table)
+{
+	struct device *dev = wiz->dev;
+	struct wiz_clk_divider *div;
+	struct clk_init_data *init;
+	const char **parent_names;
+	const char *clk_name;
+	struct clk *clk;
+	int ret;
+
+	div = devm_kzalloc(dev, sizeof(*div), GFP_KERNEL);
+	if (!div)
+		return -ENOMEM;
+
+	ret = of_property_read_string(node, "clock-output-names", &clk_name);
+	if (ret) {
+		dev_err(dev, "Unable to read clock-output-names DT property\n");
+		return ret;
+	}
+
+	parent_names = devm_kzalloc(dev, sizeof(char *), GFP_KERNEL);
+	if (!parent_names)
+		return -ENOMEM;
+
+	of_clk_parent_fill(node, parent_names, 1);
+
+	init = &div->clk_data;
+
+	init->ops = &wiz_clk_div_ops;
+	init->flags = 0;
+	init->parent_names = parent_names;
+	init->num_parents = 1;
+	init->name = clk_name;
+
+	div->field = field;
+	div->table = table;
+	div->hw.init = init;
+
+	clk = devm_clk_register(dev, &div->hw);
+	if (IS_ERR(clk))
+		return PTR_ERR(clk);
+
+	ret = of_clk_add_provider(node, of_clk_src_simple_get, clk);
+	if (ret)
+		dev_err(dev, "Failed to add clock provider: %s\n", clk_name);
+
+	return ret;
+}
+
+static void wiz_clock_cleanup(struct wiz *wiz, struct device_node *node)
+{
+	struct wiz_clk_mux_sel *clk_mux_sel = wiz->clk_mux_sel;
+	struct device_node *clk_node;
+	int i;
+
+	for (i = 0; i < WIZ_MUX_NUM_CLOCKS; i++) {
+		clk_node = of_get_child_by_name(node, clk_mux_sel[i].node_name);
+		of_clk_del_provider(clk_node);
+		of_node_put(clk_node);
+	}
+}
+
+static int wiz_clock_init(struct wiz *wiz, struct device_node *node)
+{
+	struct wiz_clk_mux_sel *clk_mux_sel = wiz->clk_mux_sel;
+	struct device *dev = wiz->dev;
+	struct device_node *clk_node;
+	const char *node_name;
+	unsigned long rate;
+	struct clk *clk;
+	int ret;
+	int i;
+
+	clk = devm_clk_get(dev, "core_ref_clk");
+	if (IS_ERR(clk)) {
+		dev_err(dev, "core_ref_clk clock not found\n");
+		ret = PTR_ERR(clk);
+		return ret;
+	}
+
+	rate = clk_get_rate(clk);
+	if (rate >= 100000000)
+		regmap_field_write(wiz->pma_cmn_refclk_int_mode, 0x1);
+	else
+		regmap_field_write(wiz->pma_cmn_refclk_int_mode, 0x3);
+
+	clk = devm_clk_get(dev, "ext_ref_clk");
+	if (IS_ERR(clk)) {
+		dev_err(dev, "ext_ref_clk clock not found\n");
+		ret = PTR_ERR(clk);
+		return ret;
+	}
+
+	rate = clk_get_rate(clk);
+	if (rate >= 100000000)
+		regmap_field_write(wiz->pma_cmn_refclk_mode, 0x0);
+	else
+		regmap_field_write(wiz->pma_cmn_refclk_mode, 0x2);
+
+	for (i = 0; i < WIZ_MUX_NUM_CLOCKS; i++) {
+		node_name = clk_mux_sel[i].node_name;
+		clk_node = of_get_child_by_name(node, node_name);
+		if (!clk_node) {
+			dev_err(dev, "Unable to get %s node\n", node_name);
+			ret = -EINVAL;
+			goto err;
+		}
+
+		ret = wiz_mux_clk_register(wiz, clk_node, clk_mux_sel[i].field,
+					   clk_mux_sel[i].table);
+		if (ret) {
+			dev_err(dev, "Failed to register %s clock\n",
+				node_name);
+			of_node_put(clk_node);
+			goto err;
+		}
+
+		of_node_put(clk_node);
+	}
+
+	for (i = 0; i < wiz->clk_div_sel_num; i++) {
+		node_name = clk_div_sel[i].node_name;
+		clk_node = of_get_child_by_name(node, node_name);
+		if (!clk_node) {
+			dev_err(dev, "Unable to get %s node\n", node_name);
+			ret = -EINVAL;
+			goto err;
+		}
+
+		ret = wiz_div_clk_register(wiz, clk_node, clk_div_sel[i].field,
+					   clk_div_sel[i].table);
+		if (ret) {
+			dev_err(dev, "Failed to register %s clock\n",
+				node_name);
+			of_node_put(clk_node);
+			goto err;
+		}
+
+		of_node_put(clk_node);
+	}
+
+	return 0;
+err:
+	wiz_clock_cleanup(wiz, node);
+
+	return ret;
+}
+
+static int wiz_phy_reset_assert(struct reset_controller_dev *rcdev,
+				unsigned long id)
+{
+	struct device *dev = rcdev->dev;
+	struct wiz *wiz = dev_get_drvdata(dev);
+	int ret = 0;
+
+	if (id == 0) {
+		ret = regmap_field_write(wiz->phy_reset_n, false);
+		return ret;
+	}
+
+	ret = regmap_field_write(wiz->p_enable[id - 1], P_ENABLE_DISABLE);
+	return ret;
+}
+
+static int wiz_phy_reset_deassert(struct reset_controller_dev *rcdev,
+				  unsigned long id)
+{
+	struct device *dev = rcdev->dev;
+	struct wiz *wiz = dev_get_drvdata(dev);
+	int ret;
+
+	/* if typec-dir gpio was specified, set LN10 SWAP bit based on that */
+	if (id == 0 && wiz->gpio_typec_dir) {
+		if (wiz->typec_dir_delay)
+			msleep_interruptible(wiz->typec_dir_delay);
+
+		if (gpiod_get_value_cansleep(wiz->gpio_typec_dir)) {
+			regmap_update_bits(wiz->regmap, WIZ_SERDES_TYPEC,
+					   WIZ_SERDES_TYPEC_LN10_SWAP,
+					   WIZ_SERDES_TYPEC_LN10_SWAP);
+		} else {
+			regmap_update_bits(wiz->regmap, WIZ_SERDES_TYPEC,
+					   WIZ_SERDES_TYPEC_LN10_SWAP, 0);
+		}
+	}
+
+	if (id == 0) {
+		ret = regmap_field_write(wiz->phy_reset_n, true);
+		return ret;
+	}
+
+	if (wiz->lane_modes[id - 1] == PHY_TYPE_DP)
+		ret = regmap_field_write(wiz->p_enable[id - 1], P_ENABLE);
+	else
+		ret = regmap_field_write(wiz->p_enable[id - 1], P_ENABLE_FORCE);
+
+	return ret;
+}
+
+static const struct reset_control_ops wiz_phy_reset_ops = {
+	.assert = wiz_phy_reset_assert,
+	.deassert = wiz_phy_reset_deassert,
+};
+
+static struct regmap_config wiz_regmap_config = {
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = 4,
+	.fast_io = true,
+};
+
+static const struct of_device_id wiz_id_table[] = {
+	{
+		.compatible = "ti,j721e-wiz-16g", .data = (void *) J721E_WIZ_16G
+	},
+	{
+		.compatible = "ti,j721e-wiz-10g", .data = (void *) J721E_WIZ_10G
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(of, wiz_id_table);
+
+static int wiz_get_lane_mode(struct device *dev, int lane_number,
+			     u32 *lane_mode)
+{
+	char property_name[11]; /* 11 is length of "lane0-mode\0" */
+	int ret;
+
+	ret = snprintf(property_name, sizeof(property_name), "lane%u-mode",
+		       lane_number);
+
+	if (ret != 10) { /* 10 is length of "lane0-mode" */
+		dev_err(dev, "%s: bad lane number %d (ret = %d)\n",
+			__func__, lane_number, ret);
+		return -ENOTSUPP;
+	}
+
+	ret = of_property_read_u32(dev->of_node, property_name, lane_mode);
+	if (ret == -EINVAL) {
+		*lane_mode = PHY_NONE;
+		return 0;
+	} else if (ret) {
+		dev_err(dev, "Getting \"%s\" property failed: %d\n",
+			property_name, ret);
+	}
+
+	return ret;
+}
+
+static int wiz_probe(struct platform_device *pdev)
+{
+	struct reset_controller_dev *phy_reset_dev;
+	struct device *dev = &pdev->dev;
+	struct device_node *node = dev->of_node;
+	struct platform_device *serdes_pdev;
+	struct device_node *child_node;
+	struct regmap *regmap;
+	struct resource res;
+	void __iomem *base;
+	struct wiz *wiz;
+	u32 num_lanes;
+	int ret;
+	int i;
+
+	wiz = devm_kzalloc(dev, sizeof(*wiz), GFP_KERNEL);
+	if (!wiz)
+		return -ENOMEM;
+
+	wiz->type = (enum wiz_type) of_device_get_match_data(dev);
+
+	child_node = of_get_child_by_name(node, "serdes");
+	if (!child_node) {
+		dev_err(dev, "Failed to get SERDES child DT node\n");
+		return -ENODEV;
+	}
+
+	ret = of_address_to_resource(child_node, 0, &res);
+	if (ret) {
+		dev_err(dev, "Failed to get memory resource\n");
+		goto err_addr_to_resource;
+	}
+
+	base = devm_ioremap(dev, res.start, resource_size(&res));
+	if (IS_ERR(base))
+		goto err_addr_to_resource;
+
+	regmap = devm_regmap_init_mmio(dev, base, &wiz_regmap_config);
+	if (IS_ERR(regmap)) {
+		dev_err(dev, "Failed to initialize regmap\n");
+		ret = PTR_ERR(regmap);
+		goto err_addr_to_resource;
+	}
+
+	ret = of_property_read_u32(node, "num-lanes", &num_lanes);
+	if (ret) {
+		dev_err(dev, "Failed to read num-lanes property\n");
+		goto err_addr_to_resource;
+	}
+
+	if (num_lanes > WIZ_MAX_LANES) {
+		dev_err(dev, "Cannot support %d lanes\n", num_lanes);
+		goto err_addr_to_resource;
+	}
+
+	wiz->gpio_typec_dir = devm_gpiod_get_optional(dev, "typec-dir",
+						      GPIOD_IN);
+	if (IS_ERR(wiz->gpio_typec_dir)) {
+		ret = PTR_ERR(wiz->gpio_typec_dir);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "Failed to request typec-dir gpio: %d\n", ret);
+		goto err_addr_to_resource;
+	}
+
+	if (wiz->gpio_typec_dir) {
+		ret = of_property_read_u32(node, "typec-dir-debounce",
+					   &wiz->typec_dir_delay);
+		if (ret && ret != -EINVAL) {
+			dev_err(dev, "Invalid typec-dir-debounce property\n");
+			goto err_addr_to_resource;
+		}
+	}
+
+	for (i = 0; i < num_lanes; i++) {
+		ret = wiz_get_lane_mode(dev, i, &wiz->lane_modes[i]);
+		if (ret)
+			return ret;
+	}
+
+	wiz->dev = dev;
+	wiz->regmap = regmap;
+	wiz->num_lanes = num_lanes;
+	if (wiz->type == J721E_WIZ_10G)
+		wiz->clk_mux_sel = clk_mux_sel_10g;
+	else
+		wiz->clk_mux_sel = clk_mux_sel_16g;
+
+	wiz->clk_div_sel = clk_div_sel;
+
+	if (wiz->type == J721E_WIZ_10G)
+		wiz->clk_div_sel_num = WIZ_DIV_NUM_CLOCKS_10G;
+	else
+		wiz->clk_div_sel_num = WIZ_DIV_NUM_CLOCKS_16G;
+
+	platform_set_drvdata(pdev, wiz);
+
+	ret = wiz_regfield_init(wiz);
+	if (ret) {
+		dev_err(dev, "Failed to initialize regfields\n");
+		goto err_addr_to_resource;
+	}
+
+	phy_reset_dev = &wiz->wiz_phy_reset_dev;
+	phy_reset_dev->dev = dev;
+	phy_reset_dev->ops = &wiz_phy_reset_ops,
+	phy_reset_dev->owner = THIS_MODULE,
+	phy_reset_dev->of_node = node;
+	/* Reset for each of the lane and one for the entire SERDES */
+	phy_reset_dev->nr_resets = num_lanes + 1;
+
+	ret = devm_reset_controller_register(dev, phy_reset_dev);
+	if (ret < 0) {
+		dev_warn(dev, "Failed to register reset controller\n");
+		goto err_addr_to_resource;
+	}
+
+	pm_runtime_enable(dev);
+	ret = pm_runtime_get_sync(dev);
+	if (ret < 0) {
+		dev_err(dev, "pm_runtime_get_sync failed\n");
+		goto err_get_sync;
+	}
+
+	ret = wiz_clock_init(wiz, node);
+	if (ret < 0) {
+		dev_warn(dev, "Failed to initialize clocks\n");
+		goto err_get_sync;
+	}
+
+	serdes_pdev = of_platform_device_create(child_node, NULL, dev);
+	if (!serdes_pdev) {
+		dev_WARN(dev, "Unable to create SERDES platform device\n");
+		goto err_pdev_create;
+	}
+	wiz->serdes_pdev = serdes_pdev;
+
+	ret = wiz_init(wiz);
+	if (ret) {
+		dev_err(dev, "WIZ initialization failed\n");
+		goto err_wiz_init;
+	}
+
+	of_node_put(child_node);
+	return 0;
+
+err_wiz_init:
+	of_platform_device_destroy(&serdes_pdev->dev, NULL);
+
+err_pdev_create:
+	wiz_clock_cleanup(wiz, node);
+
+err_get_sync:
+	pm_runtime_put(dev);
+	pm_runtime_disable(dev);
+
+err_addr_to_resource:
+	of_node_put(child_node);
+
+	return ret;
+}
+
+static int wiz_remove(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *node = dev->of_node;
+	struct platform_device *serdes_pdev;
+	struct wiz *wiz;
+
+	wiz = dev_get_drvdata(dev);
+	serdes_pdev = wiz->serdes_pdev;
+
+	of_platform_device_destroy(&serdes_pdev->dev, NULL);
+	wiz_clock_cleanup(wiz, node);
+	pm_runtime_put(dev);
+	pm_runtime_disable(dev);
+
+	return 0;
+}
+
+static struct platform_driver wiz_driver = {
+	.probe		= wiz_probe,
+	.remove		= wiz_remove,
+	.driver		= {
+		.name	= "wiz",
+		.of_match_table = wiz_id_table,
+	},
+};
+module_platform_driver(wiz_driver);
+
+MODULE_AUTHOR("Texas Instruments Inc.");
+MODULE_DESCRIPTION("TI J721E WIZ driver");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/phy/ti/phy-keystone-serdes.c b/drivers/phy/ti/phy-keystone-serdes.c
--- a/drivers/phy/ti/phy-keystone-serdes.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/phy/ti/phy-keystone-serdes.c	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,2569 @@
+/*
+ * Texas Instruments Keystone SerDes driver
+ * Authors: WingMan Kwok <w-kwok2@ti.com>
+ *
+ * This is the SerDes Phy driver for Keystone devices. This is
+ * required to support PCIe RC functionality based on designware
+ * PCIe hardware, gbe and 10gbe found on these devices.
+ *
+ * Revision History:
+ *    3.3.0.2c
+ *	- Full update based on CSL version 3.3.0.2c
+ *	- This update requires the remodelling of each SerDes lane
+ *	  as a separate PHY device, as opposed to each SerDes as a
+ *	  separate PHY device prior to this patch.
+ *
+ *    1.0.0
+ *	- Initial revision.
+ *
+ * Copyright (C) 2015-18 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the
+ * distribution.
+ *
+ * Neither the name of Texas Instruments Incorporated nor the names of
+ * its contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#include <linux/module.h>
+#include <linux/io.h>
+#include <linux/mfd/syscon.h>
+#include <linux/delay.h>
+#include <linux/firmware.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_platform.h>
+#include <linux/regmap.h>
+#include <linux/sizes.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+
+#define KSERDES_SS_OFFSET	0x1fc0
+#define MOD_VER_REG		(KSERDES_SS_OFFSET + 0x00)
+#define MEM_ADR_REG		(KSERDES_SS_OFFSET + 0x04)
+#define MEM_DAT_REG		(KSERDES_SS_OFFSET + 0x08)
+#define MEM_DATINC_REG		(KSERDES_SS_OFFSET + 0x0c)
+#define CPU_CTRL_REG		(KSERDES_SS_OFFSET + 0x10)
+#define LANE_CTRL_STS_REG(x)	(KSERDES_SS_OFFSET + 0x20 + (x * 0x04))
+#define LINK_LOSS_WAIT_REG	(KSERDES_SS_OFFSET + 0x30)
+#define PLL_CTRL_REG		(KSERDES_SS_OFFSET + 0x34)
+
+#define CMU0_SS_OFFSET		0x0000
+#define CMU0_REG(x)		(CMU0_SS_OFFSET + x)
+
+#define LANE0_SS_OFFSET		0x0200
+#define LANEX_SS_OFFSET(x)	(LANE0_SS_OFFSET * (x + 1))
+#define LANEX_REG(x, y)		(LANEX_SS_OFFSET(x) + y)
+
+#define CML_SS_OFFSET		0x0a00
+#define CML_REG(x)		(CML_SS_OFFSET + x)
+
+#define CMU1_SS_OFFSET		0x0c00
+#define CMU1_REG(x)		(CMU1_SS_OFFSET + x)
+
+#define PCSR_OFFSET(x)		(x * 0x80)
+
+#define PCSR_TX_CTL(x)		(PCSR_OFFSET(x) + 0x00)
+#define PCSR_TX_STATUS(x)	(PCSR_OFFSET(x) + 0x04)
+#define PCSR_RX_CTL(x)		(PCSR_OFFSET(x) + 0x08)
+#define PCSR_RX_STATUS(x)	(PCSR_OFFSET(x) + 0x0C)
+
+#define XGE_CTRL_OFFSET		0x0c
+#define PCIE_PL_GEN2_OFFSET	0x180c
+
+#define reg_rmw(addr, value, mask) \
+	writel(((readl(addr) & (~(mask))) | (value & (mask))), (addr))
+
+#define FINSR(base, offset, msb, lsb, val) \
+	reg_rmw((base) + (offset), ((val) << (lsb)), GENMASK((msb), (lsb)))
+
+#define FEXTR(val, msb, lsb) \
+	(((val) >> (lsb)) & ((1 << ((msb) - (lsb) + 1)) - 1))
+
+#define MOD_VER(serdes) \
+	((kserdes_readl(serdes, MOD_VER_REG) >> 16) & 0xffff)
+
+#define PHY_A(serdes) (MOD_VER(serdes) != 0x4eba)
+
+#define FOUR_LANE(serdes) \
+	((MOD_VER(serdes) == 0x4eb9) || (MOD_VER(serdes) == 0x4ebd))
+
+#define KSERDES_MAX_LANES	4
+#define MAX_COEFS		5
+#define MAX_CMP			5
+#define OFFSET_SAMPLES		100
+
+#define for_each_cmp(i)	\
+	for (i = 1; i < MAX_CMP; i++)
+
+#define CPU_EN			BIT(31)
+#define CPU_GO			BIT(30)
+#define POR_EN			BIT(29)
+#define CPUREG_EN		BIT(28)
+#define AUTONEG_CTL		BIT(27)
+#define DATASPLIT		BIT(26)
+#define LNKTRN_SIG_DET		BIT(8)
+
+#define PLL_ENABLE_1P25G	0xe0000000
+#define LANE_CTRL_1P25G		0xf800f8c0
+#define XFM_FLUSH_CMD		0x00009c9c
+
+#define ANEG_LINK_CTL_10GKR_MASK	GENMASK(21, 20)
+#define ANEG_LINK_CTL_1GKX_MASK		GENMASK(17, 16)
+#define ANEG_LINK_CTL_1G10G_MASK \
+	(ANEG_LINK_CTL_10GKR_MASK | ANEG_LINK_CTL_1GKX_MASK)
+
+#define ANEG_1G_10G_OPT_MASK		GENMASK(7, 5)
+
+#define SERDES_REG_INDEX		0
+
+#define KSERDES_XFW_MEM_SIZE		SZ_64K
+#define KSERDES_XFW_CONFIG_MEM_SIZE	SZ_64
+#define KSERDES_XFW_NUM_PARAMS		5
+
+#define KSERDES_XFW_CONFIG_START_ADDR \
+	(KSERDES_XFW_MEM_SIZE - KSERDES_XFW_CONFIG_MEM_SIZE)
+
+#define KSERDES_XFW_PARAM_START_ADDR \
+	(KSERDES_XFW_MEM_SIZE - (KSERDES_XFW_NUM_PARAMS * 4))
+
+#define LANE_ENABLE(sc, n) ((sc)->lane[n].enable)
+
+enum kserdes_link_rate {
+	KSERDES_LINK_RATE_1P25G		=  1250000,
+	KSERDES_LINK_RATE_3P125G	=  3125000,
+	KSERDES_LINK_RATE_4P9152G	=  4915200,
+	KSERDES_LINK_RATE_5G		=  5000000,
+	KSERDES_LINK_RATE_6P144G	=  6144000,
+	KSERDES_LINK_RATE_6P25G		=  6250000,
+	KSERDES_LINK_RATE_7P3728G	=  7372800,
+	KSERDES_LINK_RATE_9P8304G	=  9830400,
+	KSERDES_LINK_RATE_10G		= 10000000,
+	KSERDES_LINK_RATE_10P3125G	= 10312500,
+	KSERDES_LINK_RATE_12P5G		= 12500000,
+};
+
+enum kserdes_lane_ctrl_rate {
+	KSERDES_FULL_RATE,
+	KSERDES_HALF_RATE,
+	KSERDES_QUARTER_RATE,
+};
+
+enum kserdes_phy_type {
+	KSERDES_PHY_SGMII,
+	KSERDES_PHY_XGE,
+	KSERDES_PHY_PCIE,
+	KSERDES_PHY_HYPERLINK,
+};
+
+struct kserdes_tx_coeff {
+	u32	c1;
+	u32	c2;
+	u32	cm;
+	u32	att;
+	u32	vreg;
+};
+
+struct kserdes_equalizer {
+	u32	att;
+	u32	boost;
+};
+
+struct kserdes_lane_config {
+	bool				enable;
+	u32				ctrl_rate;
+	struct kserdes_tx_coeff		tx_coeff;
+	struct kserdes_equalizer	rx_start;
+	struct kserdes_equalizer	rx_force;
+	bool				loopback;
+};
+
+struct kserdes_fw_config {
+	bool				on;
+	u32				rate;
+	u32				link_loss_wait;
+	u32				lane_seeds;
+	u32				fast_train;
+	u32				active_lane;
+	u32				c1, c2, cm, attn, boost, dlpf, rxcal;
+	u32				lane_config[KSERDES_MAX_LANES];
+};
+
+struct kserdes_lane_dlev_out {
+	u32 delay;
+	int coef_vals[MAX_COEFS];
+};
+
+struct kserdes_dlev_out {
+	struct kserdes_lane_dlev_out lane_dlev_out[KSERDES_MAX_LANES];
+};
+
+struct kserdes_cmp_coef_ofs {
+	u32 cmp;
+	u32 coef1;
+	u32 coef2;
+	u32 coef3;
+	u32 coef4;
+	u32 coef5;
+};
+
+struct kserdes_lane_ofs {
+	struct kserdes_cmp_coef_ofs ct_ofs[MAX_CMP];
+};
+
+struct kserdes_ofs {
+	struct kserdes_lane_ofs lane_ofs[KSERDES_MAX_LANES];
+};
+
+/*
+ * All firmware file names end up here. List the firmware file names below.
+ * Newest first. Search starts from the 0-th array entry until a firmware
+ * file is found.
+ */
+static const char * const ks2_gbe_serdes_firmwares[] = {"ks2_gbe_serdes.bin"};
+static const char * const ks2_xgbe_serdes_firmwares[] = {"ks2_xgbe_serdes.bin"};
+static const char * const ks2_pcie_serdes_firmwares[] = {"ks2_pcie_serdes.bin"};
+
+#define MAX_VERSION		64
+#define INIT_FW_MAGIC_1		0xfaceface
+#define INIT_FW_MAGIC_2		0xcafecafe
+
+static char *compatible_init_fw_version[] = {
+	"3.3.0.2c",
+	NULL,
+};
+
+struct serdes_cfg_header {
+	u32 magic_1;
+	char version[MAX_VERSION];
+	u32 magic_2;
+};
+
+struct serdes_cfg {
+	u32 ofs;
+	u32 msb;
+	u32 lsb;
+	u32 val;
+};
+
+struct kserdes_config {
+	struct device			*dev;
+	enum kserdes_phy_type		phy_type;
+	u32				lanes;
+	void __iomem			*regs;
+	struct regmap			*peripheral_regmap;
+	struct regmap			*pcsr_regmap;
+	const char			*init_fw;
+	struct serdes_cfg		*init_cfg;
+	int				init_cfg_len;
+	enum kserdes_link_rate		link_rate;
+	bool				rx_force_enable;
+	struct kserdes_lane_config	lane[KSERDES_MAX_LANES];
+	struct kserdes_ofs		sofs;
+	bool				firmware;
+	struct kserdes_fw_config	fw;
+};
+
+struct kserdes_phy {
+	u32 lane;
+	struct phy *phy;
+};
+
+struct kserdes_dev {
+	struct device *dev;
+	u32 nphys;
+	struct kserdes_phy *phys[KSERDES_MAX_LANES];
+	struct kserdes_config sc;
+};
+
+static struct platform_device_id kserdes_devtype[] = {
+	{
+		.name = "kserdes-gbe",
+		.driver_data = KSERDES_PHY_SGMII,
+	}, {
+		.name = "kserdes-xgbe",
+		.driver_data = KSERDES_PHY_XGE,
+	}, {
+		.name = "kserdes-pcie",
+		.driver_data = KSERDES_PHY_PCIE,
+	}
+};
+
+static inline int next_enable_lane(struct kserdes_config *sc, int i)
+{
+	int j = i;
+
+	while (++j < sc->lanes) {
+		if (sc->lane[j].enable)
+			return j;
+	}
+	return j;
+}
+
+#define for_each_lane(sc, i) \
+	for (i = 0; i < (sc)->lanes; i++)
+
+#define for_each_enable_lane(sc, i) \
+	for (i = -1; i = next_enable_lane(sc, i), i < sc->lanes; )
+
+static inline u32 kserdes_readl(void __iomem *base, u32 offset)
+{
+	return readl(base + offset);
+}
+
+static inline void kserdes_writel(void __iomem *base, u32 offset, u32 value)
+{
+	writel(value, base + offset);
+}
+
+static void kserdes_do_config(void __iomem *base,
+			      struct serdes_cfg *cfg, u32 size)
+{
+	u32 i;
+
+	for (i = 0; i < size; i++)
+		FINSR(base, cfg[i].ofs, cfg[i].msb, cfg[i].lsb, cfg[i].val);
+}
+
+static bool is_init_fw_compatible(struct kserdes_config *sc,
+				  struct serdes_cfg_header *hdr)
+{
+	int i = 0;
+
+	if ((hdr->magic_1 != INIT_FW_MAGIC_1) ||
+	    (hdr->magic_2 != INIT_FW_MAGIC_2)) {
+		dev_err(sc->dev, "incompatible fw %s\n", sc->init_fw);
+		return false;
+	}
+
+	while (compatible_init_fw_version[i]) {
+		if (!strcmp(compatible_init_fw_version[i], hdr->version)) {
+			dev_info(sc->dev, "init fw %s: version %s\n",
+				 sc->init_fw, hdr->version);
+			return true;
+		}
+
+		++i;
+	}
+
+	dev_err(sc->dev, "incompatible fw %s: version %s\n",
+		sc->init_fw, hdr->version);
+	return false;
+}
+
+static int kserdes_load_init_fw(struct kserdes_config *sc,
+				const char * const *a_firmwares,
+				int n_firmwares)
+{
+	const struct firmware *fw;
+	bool found = false;
+	int ret, i;
+	struct serdes_cfg_header hdr;
+	int hdr_sz;
+
+	for (i = 0; i < n_firmwares; i++) {
+		if (a_firmwares[i]) {
+			ret = request_firmware(&fw, a_firmwares[i], sc->dev);
+			if (!ret) {
+				found = true;
+				break;
+			}
+		}
+	}
+
+	if (!found) {
+		dev_err(sc->dev, "can't get any serdes init fw");
+		return -ENODEV;
+	}
+
+	sc->init_fw = a_firmwares[i];
+
+	memcpy((void *)&hdr, fw->data, sizeof(hdr));
+	hdr_sz = sizeof(hdr);
+	hdr.version[MAX_VERSION - 1] = 0;
+	if (!is_init_fw_compatible(sc, &hdr))
+		return -EINVAL;
+
+	sc->init_cfg = devm_kzalloc(sc->dev, fw->size - hdr_sz, GFP_KERNEL);
+	memcpy((void *)sc->init_cfg, fw->data + hdr_sz, fw->size - hdr_sz);
+	sc->init_cfg_len = fw->size - hdr_sz;
+	release_firmware(fw);
+
+	kserdes_do_config(sc->regs, sc->init_cfg,
+			  sc->init_cfg_len / sizeof(struct serdes_cfg));
+
+	return 0;
+}
+
+static inline u32 _kserdes_read_tbus_val(void __iomem *sregs)
+{
+	u32 tmp;
+
+	if (PHY_A(sregs)) {
+		tmp  = ((kserdes_readl(sregs, CMU0_REG(0xec))) >> 24) & 0x0ff;
+		tmp |= ((kserdes_readl(sregs, CMU0_REG(0xfc))) >> 16) & 0xf00;
+	} else {
+		tmp  = ((kserdes_readl(sregs, CMU0_REG(0xf8))) >> 16) & 0xfff;
+	}
+
+	return tmp;
+}
+
+static void _kserdes_write_tbus_addr(void __iomem *sregs, int select, int ofs)
+{
+	if (select && !FOUR_LANE(sregs))
+		++select;
+
+	if (PHY_A(sregs))
+		FINSR(sregs, CMU0_REG(0x8), 31, 24, ((select << 5) + ofs));
+	else
+		FINSR(sregs, CMU0_REG(0xfc), 26, 16, ((select << 8) + ofs));
+}
+
+static u32 _kserdes_read_select_tbus(void __iomem *sregs, int select, int ofs)
+{
+	_kserdes_write_tbus_addr(sregs, select, ofs);
+	return _kserdes_read_tbus_val(sregs);
+}
+
+static inline void kserdes_set_tx_idle(struct kserdes_config *sc, u32 lane)
+{
+	if (sc->phy_type != KSERDES_PHY_XGE)
+		FINSR(sc->regs, LANEX_REG(lane, 0xb8), 17, 16, 3);
+
+	FINSR(sc->regs, LANE_CTRL_STS_REG(lane), 25, 24, 3);
+	FINSR(sc->regs, LANEX_REG(lane, 0x28), 21, 20, 0);
+}
+
+static inline void kserdes_clr_tx_idle(struct kserdes_config *sc, u32 lane)
+{
+	if (sc->phy_type != KSERDES_PHY_XGE)
+		FINSR(sc->regs, LANEX_REG(lane, 0xb8), 17, 16, 0);
+
+	FINSR(sc->regs, LANE_CTRL_STS_REG(lane), 25, 24, 0);
+	FINSR(sc->regs, LANEX_REG(lane, 0x28), 21, 20, 0);
+}
+
+static void kserdes_set_lane_ov(struct kserdes_config *sc, u32 lane)
+{
+	u32 val_0, val_1, val;
+
+	val_0 = _kserdes_read_select_tbus(sc->regs, lane + 1, 0);
+	val_1 = _kserdes_read_select_tbus(sc->regs, lane + 1, 1);
+
+	val = 0;
+	val |= ((val_1 >> 9) & 0x3) << 1;
+	val |= (val_0 & 0x3) << 3;
+	val |= ((val_0 >> 2) & 0x1ff) << 5;
+	val |= (1 << 14);
+	val &= ~0x60;
+
+	FINSR(sc->regs, LANEX_REG(lane, 0x028), 29, 15, val);
+}
+
+static inline void kserdes_assert_reset(struct kserdes_config *sc)
+{
+	int lane;
+
+	for_each_enable_lane(sc, lane)
+		kserdes_set_lane_ov(sc, lane);
+}
+
+static inline void kserdes_config_c1_c2_cm(struct kserdes_config *sc, u32 lane)
+{
+	u32 c1, c2, cm;
+
+	c1 = sc->lane[lane].tx_coeff.c1;
+	c2 = sc->lane[lane].tx_coeff.c2;
+	cm = sc->lane[lane].tx_coeff.cm;
+
+	if (sc->phy_type == KSERDES_PHY_XGE) {
+		FINSR(sc->regs, LANEX_REG(lane, 0x8), 11,  8, (cm & 0xf));
+		FINSR(sc->regs, LANEX_REG(lane, 0x8),  4,  0, (c1 & 0x1f));
+		FINSR(sc->regs, LANEX_REG(lane, 0x8),  7,  5, (c2 & 0x7));
+		FINSR(sc->regs, LANEX_REG(lane, 0x4),
+		      18, 18, ((c2 >> 3) & 0x1));
+	} else {
+		FINSR(sc->regs, LANEX_REG(lane, 0x8), 15, 12, (cm & 0xf));
+		FINSR(sc->regs, LANEX_REG(lane, 0x8),  4,  0, (c1 & 0x1f));
+		FINSR(sc->regs, LANEX_REG(lane, 0x8), 11,  8, (c2 & 0xf));
+	}
+}
+
+static inline void kserdes_config_att_boost(struct kserdes_config *sc, u32 lane)
+{
+	u32 att, boost;
+
+	att = sc->lane[lane].rx_force.att;
+	boost = sc->lane[lane].rx_force.boost;
+
+	if (sc->phy_type == KSERDES_PHY_XGE) {
+		FINSR(sc->regs, LANEX_REG(lane, 0x98), 13, 13, 0);
+		FINSR(sc->regs, LANEX_REG(lane, 0x8c), 15, 12, boost);
+		FINSR(sc->regs, LANEX_REG(lane, 0x8c), 11, 8, att);
+	} else {
+		if (att != -1) {
+			FINSR(sc->regs, CML_REG(0x84), 0, 0, 0);
+			FINSR(sc->regs, CML_REG(0x8c), 24, 24, 0);
+			FINSR(sc->regs, LANEX_REG(lane, 0x8c), 11, 8, att);
+		}
+		if (boost != -1) {
+			FINSR(sc->regs, CML_REG(0x84), 1, 1, 0);
+			FINSR(sc->regs, CML_REG(0x8c), 25, 25, 0);
+			FINSR(sc->regs, LANEX_REG(lane, 0x8c), 15, 12, boost);
+		}
+	}
+}
+
+static void kserdes_set_tx_rx_fir_coeff(struct kserdes_config *sc, u32 lane)
+{
+	struct kserdes_tx_coeff *tc = &sc->lane[lane].tx_coeff;
+
+	if (sc->phy_type == KSERDES_PHY_XGE) {
+		FINSR(sc->regs, LANEX_REG(lane, 0x004), 29, 26, tc->att);
+		FINSR(sc->regs, LANEX_REG(lane, 0x0a4), 2, 0, tc->vreg);
+	} else {
+		FINSR(sc->regs, LANEX_REG(lane, 0x004), 28, 25, tc->att);
+		FINSR(sc->regs, LANEX_REG(lane, 0x084), 7, 5, tc->vreg);
+	}
+
+	kserdes_config_c1_c2_cm(sc, lane);
+
+	if (sc->rx_force_enable)
+		kserdes_config_att_boost(sc, lane);
+}
+
+static inline void
+_kserdes_force_signal_detect_low(void __iomem *sregs, u32 lane)
+{
+	FINSR(sregs, LANEX_REG(lane, 0x004), 2, 1, 0x2);
+}
+
+static inline void
+kserdes_force_signal_detect_low(struct kserdes_config *sc, u32 lane)
+{
+	_kserdes_force_signal_detect_low(sc->regs, lane);
+}
+
+static inline void
+_kserdes_force_signal_detect_high(void __iomem *sregs, u32 lane)
+{
+	FINSR(sregs, LANEX_REG(lane, 0x004), 2, 1, 0x0);
+}
+
+static inline void
+kserdes_force_signal_detect_high(struct kserdes_config *sc, u32 lane)
+{
+	_kserdes_force_signal_detect_high(sc->regs, lane);
+}
+
+static int kserdes_deassert_reset_poll_others(struct kserdes_config *sc)
+{
+	unsigned long timeout = jiffies + msecs_to_jiffies(500);
+	unsigned long time_check;
+	u32 lanes_not_ok = 0;
+	u32 ofs = 28;
+	u32 ret, i;
+
+	for_each_enable_lane(sc, i)
+		lanes_not_ok |= BIT(i);
+
+	if (!FOUR_LANE(sc->regs))
+		ofs = 29;
+
+	do {
+		time_check = jiffies;
+		for_each_enable_lane(sc, i) {
+			if (!(lanes_not_ok & (1 << i)))
+				continue;
+
+			ret = kserdes_readl(sc->regs, CML_REG(0x1f8));
+
+			if (ret & BIT(ofs + i))
+				lanes_not_ok &= ~BIT(i);
+		}
+
+		if (!lanes_not_ok)
+			return 0;
+
+		if (time_after(time_check, timeout))
+			return -ETIMEDOUT;
+
+		cpu_relax();
+	} while (true);
+}
+
+static int kserdes_deassert_reset_poll_pcie(struct kserdes_config *sc)
+{
+	unsigned long timeout = jiffies + msecs_to_jiffies(500);
+	unsigned long time_check;
+	u32 lanes_not_ok = 0;
+	u32 ret, i;
+
+	for_each_enable_lane(sc, i)
+		lanes_not_ok |= (1 << i);
+
+	do {
+		time_check = jiffies;
+		for_each_enable_lane(sc, i) {
+			if (!(lanes_not_ok & BIT(i)))
+				continue;
+
+			ret = _kserdes_read_select_tbus(sc->regs, i + 1, 0x02);
+
+			if (!(ret & BIT(4)))
+				lanes_not_ok &= ~BIT(i);
+		}
+
+		if (!lanes_not_ok)
+			return 0;
+
+		if (time_after(time_check, timeout))
+			return -ETIMEDOUT;
+
+		cpu_relax();
+	} while (true);
+}
+
+static inline void _kserdes_lane_reset(void __iomem *serdes,
+				       u32 lane, u32 reset)
+{
+	FINSR(serdes, LANEX_REG(lane, 0x28), 29, 29, !!reset);
+}
+
+static inline void kserdes_release_reset(struct kserdes_config *sc, u32 lane)
+{
+	if (sc->phy_type == KSERDES_PHY_XGE)
+		FINSR(sc->regs, LANEX_REG(lane, 0x60), 0, 0, 0x1);
+	_kserdes_lane_reset(sc->regs, lane, 0);
+}
+
+static int kserdes_deassert_reset(struct kserdes_config *sc, u32 poll)
+{
+	int ret = 0, lane;
+
+	for_each_enable_lane(sc, lane)
+		kserdes_release_reset(sc, lane);
+
+	if (!poll)
+		goto done;
+
+	if (sc->phy_type == KSERDES_PHY_PCIE)
+		ret = kserdes_deassert_reset_poll_pcie(sc);
+	else
+		ret = kserdes_deassert_reset_poll_others(sc);
+
+done:
+	return ret;
+}
+
+static inline void _kserdes_lane_enable(void __iomem *sregs, u32 lane)
+{
+	FINSR(sregs, LANE_CTRL_STS_REG(lane), 31, 29, 0x7);
+	FINSR(sregs, LANE_CTRL_STS_REG(lane), 15, 13, 0x7);
+}
+
+static inline int _kserdes_set_lane_ctrl_rate(void __iomem *sregs, u32 lane,
+					      enum kserdes_lane_ctrl_rate rate)
+{
+	u32 rate_mode;
+
+	switch (rate) {
+	case KSERDES_FULL_RATE:
+		rate_mode = 0x4;
+		break;
+	case KSERDES_QUARTER_RATE:
+		rate_mode = 0x6;
+		break;
+	case KSERDES_HALF_RATE:
+		rate_mode = 0x5;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	FINSR(sregs, LANE_CTRL_STS_REG(lane), 28, 26, rate_mode);
+	FINSR(sregs, LANE_CTRL_STS_REG(lane), 12, 10, rate_mode);
+	return 0;
+}
+
+static inline void _kserdes_set_lane_loopback(void __iomem *sregs, u32 lane,
+					      enum kserdes_phy_type phy_type)
+{
+	if (phy_type == KSERDES_PHY_XGE) {
+		FINSR(sregs, LANEX_REG(lane, 0x0), 7, 0, 0x4);
+		FINSR(sregs, LANEX_REG(lane, 0x4), 2, 1, 0x3);
+	} else {
+		FINSR(sregs, LANEX_REG(lane, 0x0), 31, 24, 0x40);
+	}
+}
+
+static inline void _kserdes_clear_wait_after(void __iomem *sregs)
+{
+	FINSR(sregs, PLL_CTRL_REG, 17, 16, 0);
+}
+
+static inline void _kserdes_clear_lane_wait_after(void __iomem *sregs, u32 lane)
+{
+	FINSR(sregs, PLL_CTRL_REG, lane + 12, lane + 12, 1);
+	FINSR(sregs, PLL_CTRL_REG, lane + 4, lane + 4, 1);
+}
+
+static inline void _kserdes_pll_enable(void __iomem *sregs)
+{
+	FINSR(sregs, PLL_CTRL_REG, 31, 29, 0x7);
+}
+
+static inline void _kserdes_pll2_enable(void __iomem *sregs)
+{
+	FINSR(sregs, PLL_CTRL_REG, 27, 25, 0x7);
+}
+
+static inline void _kserdes_pll_disable(void __iomem *sregs)
+{
+	FINSR(sregs, PLL_CTRL_REG, 31, 29, 0x4);
+}
+
+static inline void _kserdes_pll2_disable(void __iomem *sregs)
+{
+	FINSR(sregs, PLL_CTRL_REG, 27, 25, 0x4);
+}
+
+static inline u32 _kserdes_get_pll_status(void __iomem *sregs)
+{
+	return FEXTR(kserdes_readl(sregs, PLL_CTRL_REG), 28, 28);
+}
+
+static inline u32 _kserdes_get_pll2_status(void __iomem *sregs)
+{
+	return FEXTR(kserdes_readl(sregs, PLL_CTRL_REG), 24, 24);
+}
+
+static inline void kserdes_lane_enable_loopback(void __iomem *serdes, u32 lane)
+{
+	FINSR(serdes, LANEX_REG(lane, 0), 31, 24, 0x40);
+}
+
+static inline u32 _kserdes_get_lane_status(void __iomem *sregs, u32 lane,
+					   enum kserdes_phy_type phy_type)
+{
+	int d = ((phy_type == KSERDES_PHY_PCIE) ? 0 : 8);
+
+	return FEXTR(kserdes_readl(sregs, PLL_CTRL_REG), lane + d, lane + d);
+}
+
+static u32 kserdes_get_pll_lanes_status(struct kserdes_config *sc)
+{
+	u32 val, i;
+
+	val = _kserdes_get_pll_status(sc->regs);
+	if (!val)
+		goto done;
+
+	if (sc->phy_type == KSERDES_PHY_XGE) {
+		val = _kserdes_get_pll2_status(sc->regs);
+		if (!val)
+			goto done;
+	}
+
+	for_each_enable_lane(sc, i)
+		val &= _kserdes_get_lane_status(sc->regs, i, sc->phy_type);
+
+done:
+	return val;
+}
+
+static int kserdes_get_status(struct kserdes_config *sc)
+{
+	unsigned long timeout = jiffies + msecs_to_jiffies(500);
+	unsigned long time_check;
+
+	do {
+		time_check = jiffies;
+		if (kserdes_get_pll_lanes_status(sc))
+			break;
+
+		if (time_after(time_check, timeout))
+			return -ETIMEDOUT;
+
+		cpu_relax();
+	} while (true);
+
+	return 0;
+}
+
+static inline u32 _kserdes_get_tx_termination(void __iomem *sregs,
+					      enum kserdes_phy_type phy_type)
+{
+	return (_kserdes_read_select_tbus(sregs, 1,
+					  ((phy_type == KSERDES_PHY_XGE) ?
+					  0x1a : 0x1b)) & 0xff);
+}
+
+static void kserdes_set_tx_terminations(struct kserdes_config *sc, u32 term)
+{
+	int i;
+
+	for_each_lane(sc, i) {
+		FINSR(sc->regs, LANEX_REG(i, 0x7c), 31, 24, term);
+		FINSR(sc->regs, LANEX_REG(i, 0x7c), 20, 20, 0x1);
+	}
+}
+
+static void
+_kserdes_write_ofs_xge(void __iomem *sregs, u32 lane, u32 cmp,
+		       struct kserdes_cmp_coef_ofs *ofs)
+{
+	FINSR(sregs, CML_REG(0x8c), 23, 21, cmp);
+
+	FINSR(sregs, CMU0_REG(0xfc), 26, 16, ((lane + 2) << 8) + 0x11);
+	ofs->cmp = (_kserdes_read_tbus_val(sregs) & 0x0ff0) >> 4;
+
+	FINSR(sregs, CMU0_REG(0xfc), 26, 16, ((lane + 2) << 8) + 0x11);
+	ofs->coef1 = (_kserdes_read_tbus_val(sregs) & 0x000f) << 3;
+
+	FINSR(sregs, CMU0_REG(0xfc), 26, 16, ((lane + 2) << 8) + 0x12);
+	ofs->coef1 |= (_kserdes_read_tbus_val(sregs) & 0x0e00) >> 9;
+	ofs->coef2  = (_kserdes_read_tbus_val(sregs) & 0x01f8) >> 3;
+	ofs->coef3  = (_kserdes_read_tbus_val(sregs) & 0x0007) << 3;
+
+	FINSR(sregs, CMU0_REG(0xfc), 26, 16, ((lane + 2) << 8) + 0x13);
+	ofs->coef3 |= (_kserdes_read_tbus_val(sregs) & 0x0e00) >> 9;
+	ofs->coef4  = (_kserdes_read_tbus_val(sregs) & 0x01f8) >> 3;
+	ofs->coef5  = (_kserdes_read_tbus_val(sregs) & 0x0007) << 3;
+
+	FINSR(sregs, CMU0_REG(0xfc), 26, 16, ((lane + 2) << 8) + 0x14);
+	ofs->coef5 |= (_kserdes_read_tbus_val(sregs) & 0x0e00) >> 9;
+}
+
+static void kserdes_add_ofs_xge(struct kserdes_config *sc,
+				struct kserdes_ofs *sofs)
+{
+	struct kserdes_cmp_coef_ofs *ctofs;
+	struct kserdes_cmp_coef_ofs sample;
+	struct kserdes_lane_ofs *lofs;
+	u32 lane, cmp;
+
+	for_each_enable_lane(sc, lane) {
+		lofs = &sofs->lane_ofs[lane];
+		for_each_cmp(cmp) {
+			ctofs = &lofs->ct_ofs[cmp];
+
+			_kserdes_write_ofs_xge(sc->regs, lane, cmp, &sample);
+
+			ctofs->cmp  += sample.cmp;
+			ctofs->coef1 += sample.coef1;
+			ctofs->coef2 += sample.coef2;
+			ctofs->coef3 += sample.coef3;
+			ctofs->coef4 += sample.coef4;
+			ctofs->coef5 += sample.coef5;
+		}
+	}
+}
+
+static void
+kserdes_get_cmp_coef_ofs_non_xge(void __iomem *sregs, u32 lane, u32 cmp,
+				 struct kserdes_cmp_coef_ofs *ofs)
+{
+	FINSR(sregs, CML_REG(0x8c), 23, 21, cmp);
+	FINSR(sregs, CMU0_REG(0x8), 31, 24, ((lane + 1) << 5) + 0x12);
+	ofs->cmp = (_kserdes_read_tbus_val(sregs) & 0x0ff0) >> 4;
+}
+
+static void kserdes_add_ofs_non_xge(struct kserdes_config *sc,
+				    struct kserdes_ofs *sofs)
+{
+	struct kserdes_cmp_coef_ofs *ctofs;
+	struct kserdes_cmp_coef_ofs sample;
+	struct kserdes_lane_ofs *lofs;
+	u32 lane, cmp;
+
+	for_each_enable_lane(sc, lane) {
+		lofs = &sofs->lane_ofs[lane];
+		for_each_cmp(cmp) {
+			ctofs = &lofs->ct_ofs[cmp];
+
+			kserdes_get_cmp_coef_ofs_non_xge(sc->regs, lane,
+							 cmp, &sample);
+
+			ctofs->cmp  += sample.cmp;
+		}
+	}
+}
+
+static void kserdes_get_average_ofs(struct kserdes_config *sc, u32 samples,
+				    struct kserdes_ofs *sofs)
+{
+	struct kserdes_cmp_coef_ofs *ctofs;
+	struct kserdes_lane_ofs *lofs;
+	u32 i, lane, cmp;
+	int ret;
+
+	memset(sofs, 0, sizeof(*sofs));
+
+	for (i = 0; i < samples; i++) {
+		kserdes_assert_reset(sc);
+		ret = kserdes_deassert_reset(sc, 1);
+		if (ret) {
+			dev_err(sc->dev, "%s: reset failed %d\n",
+				__func__, ret);
+			return;
+		}
+
+		if (sc->phy_type == KSERDES_PHY_XGE)
+			kserdes_add_ofs_xge(sc, sofs);
+		else
+			kserdes_add_ofs_non_xge(sc, sofs);
+	}
+
+	for_each_enable_lane(sc, lane) {
+		lofs = &sofs->lane_ofs[lane];
+		for_each_cmp(cmp) {
+			ctofs = &lofs->ct_ofs[cmp];
+			if (sc->phy_type == KSERDES_PHY_XGE) {
+				ctofs->cmp  /= samples;
+				ctofs->coef1 /= samples;
+				ctofs->coef2 /= samples;
+				ctofs->coef3 /= samples;
+				ctofs->coef4 /= samples;
+				ctofs->coef5 /= samples;
+			} else {
+				ctofs->cmp  /= samples;
+			}
+		}
+	}
+}
+
+static void _kserdes_set_ofs(void __iomem *sregs, u32 lane, u32 cmp,
+			     struct kserdes_cmp_coef_ofs *ofs)
+{
+	FINSR(sregs, CML_REG(0xf0), 27, 26, (lane + 1));
+	FINSR(sregs, CML_REG(0x98), 24, 24, 0x1);
+	FINSR(sregs, LANEX_REG(lane, 0x2c), 2, 2, 0x1);
+	FINSR(sregs, LANEX_REG(lane, 0x30), 7, 5, cmp);
+	FINSR(sregs, LANEX_REG(lane, 0x5c), 31, 31, 0x1);
+	FINSR(sregs, CML_REG(0x9c), 7, 0, ofs->cmp);
+	FINSR(sregs, LANEX_REG(lane, 0x58), 30, 24, ofs->coef1);
+	FINSR(sregs, LANEX_REG(lane, 0x5c),  5,  0, ofs->coef2);
+	FINSR(sregs, LANEX_REG(lane, 0x5c), 13,  8, ofs->coef3);
+	FINSR(sregs, LANEX_REG(lane, 0x5c), 21, 16, ofs->coef4);
+	FINSR(sregs, LANEX_REG(lane, 0x5c), 29, 24, ofs->coef5);
+
+	FINSR(sregs, LANEX_REG(lane, 0x2c), 10, 10, 0x1);
+	FINSR(sregs, LANEX_REG(lane, 0x2c), 10, 10, 0x0);
+
+	FINSR(sregs, CML_REG(0x98), 24, 24, 0x0);
+	FINSR(sregs, LANEX_REG(lane, 0x2c), 2, 2, 0x0);
+	FINSR(sregs, LANEX_REG(lane, 0x5c), 31, 31, 0x0);
+}
+
+static inline void _kserdes_set_cmp_ofs_phyb(void __iomem *sregs, u32 lane,
+					     u32 cmp, u32 cmp_ofs)
+{
+	FINSR(sregs, LANEX_REG(lane, 0x58), 18, 18, 0x1);
+	FINSR(sregs, LANEX_REG(lane, 0x4c), 5, 2, (0x1 << (cmp - 1)));
+	FINSR(sregs, LANEX_REG(lane, 0x48), 24, 17, cmp_ofs);
+	FINSR(sregs, LANEX_REG(lane, 0x48), 29, 29, 0x1);
+	FINSR(sregs, LANEX_REG(lane, 0x48), 29, 29, 0x0);
+	FINSR(sregs, LANEX_REG(lane, 0x58), 18, 18, 0x0);
+}
+
+static inline void _kserdes_set_coef_ofs(void __iomem *sregs, u32 lane,
+					 u32 coef, u32 width, u32 coef_ofs)
+{
+	FINSR(sregs, LANEX_REG(lane, 0x58), 23, 19, (0x1 << (coef - 1)));
+	FINSR(sregs, LANEX_REG(lane, 0x48), 17 + (width - 1), 17, coef_ofs);
+	FINSR(sregs, LANEX_REG(lane, 0x48), 29, 29, 0x1);
+	FINSR(sregs, LANEX_REG(lane, 0x48), 29, 29, 0x0);
+}
+
+static void _kserdes_set_ofs_phyb(void __iomem *sregs, u32 lane, u32 cmp,
+				  struct kserdes_cmp_coef_ofs *ofs)
+{
+	FINSR(sregs, LANEX_REG(lane, 0x58), 16, 16, 0x1);
+	FINSR(sregs, LANEX_REG(lane, 0x48), 16, 16, 0x1);
+
+	_kserdes_set_cmp_ofs_phyb(sregs, lane, cmp, ofs->cmp);
+
+	FINSR(sregs, LANEX_REG(lane, 0x58), 17, 17, 0x1);
+
+	_kserdes_set_coef_ofs(sregs, lane, 1, 7, ofs->coef1);
+	_kserdes_set_coef_ofs(sregs, lane, 2, 6, ofs->coef2);
+	_kserdes_set_coef_ofs(sregs, lane, 3, 6, ofs->coef3);
+	_kserdes_set_coef_ofs(sregs, lane, 4, 6, ofs->coef4);
+	_kserdes_set_coef_ofs(sregs, lane, 5, 6, ofs->coef5);
+
+	FINSR(sregs, LANEX_REG(lane, 0x58), 16, 16, 0x0);
+	FINSR(sregs, LANEX_REG(lane, 0x48), 16, 16, 0x0);
+	FINSR(sregs, LANEX_REG(lane, 0x58), 18, 18, 0x0);
+	FINSR(sregs, LANEX_REG(lane, 0x58), 17, 17, 0x0);
+}
+
+static void kserdes_set_ofs_xge(struct kserdes_config *sc,
+				struct kserdes_ofs *sofs)
+{
+	struct kserdes_cmp_coef_ofs *ctofs;
+	struct kserdes_lane_ofs *lofs;
+	int lane, cmp;
+
+	for_each_enable_lane(sc, lane) {
+		lofs = &sofs->lane_ofs[lane];
+		for_each_cmp(cmp) {
+			ctofs = &lofs->ct_ofs[cmp];
+			_kserdes_set_ofs(sc->regs, lane, cmp, ctofs);
+			_kserdes_set_ofs_phyb(sc->regs, lane, cmp, ctofs);
+		}
+	}
+}
+
+static void kserdes_set_ofs_non_xge(struct kserdes_config *sc,
+				    struct kserdes_ofs *sofs)
+{
+	struct kserdes_cmp_coef_ofs *ctofs;
+	struct kserdes_lane_ofs *lofs;
+	u32 lane, cmp;
+
+	for_each_enable_lane(sc, lane) {
+		lofs = &sofs->lane_ofs[lane];
+		for_each_cmp(cmp) {
+			ctofs = &lofs->ct_ofs[cmp];
+			_kserdes_set_ofs(sc->regs, lane, cmp, ctofs);
+		}
+	}
+}
+
+static void kserdes_set_average_ofs(struct kserdes_config *sc,
+				    struct kserdes_ofs *sofs)
+{
+	if (sc->phy_type == KSERDES_PHY_XGE)
+		kserdes_set_ofs_xge(sc, sofs);
+	else
+		kserdes_set_ofs_non_xge(sc, sofs);
+}
+
+static void kserdes_phyb_init_config(struct kserdes_config *sc,
+				     struct kserdes_ofs *sofs)
+{
+	int lane;
+
+	for_each_enable_lane(sc, lane)
+		kserdes_force_signal_detect_low(sc, lane);
+
+	usleep_range(10, 20);
+
+	kserdes_get_average_ofs(sc, OFFSET_SAMPLES, sofs);
+	kserdes_set_average_ofs(sc, sofs);
+	usleep_range(10, 20);
+
+	for_each_enable_lane(sc, lane)
+		kserdes_force_signal_detect_high(sc, lane);
+
+	usleep_range(10, 20);
+}
+
+static int kserdes_wait_lane_rx_valid(struct kserdes_config *sc, u32 lane)
+{
+	unsigned long timeout = jiffies + msecs_to_jiffies(500);
+	unsigned long time_check;
+	u32 status;
+
+	do {
+		time_check = jiffies;
+		status = _kserdes_read_select_tbus(sc->regs, lane + 1, 0x02);
+
+		if (status & 0x20)
+			return 0;
+
+		if (time_after(time_check, timeout))
+			return -ETIMEDOUT;
+
+		cpu_relax();
+	} while (true);
+}
+
+static inline void _kserdes_reset(void __iomem *sregs)
+{
+	FINSR(sregs, CPU_CTRL_REG, 29, 29, 0x1);
+	usleep_range(10, 20);
+	FINSR(sregs, CPU_CTRL_REG, 29, 29, 0x0);
+	usleep_range(10, 20);
+}
+
+static inline void kserdes_xge_pll_enable(struct kserdes_config *sc)
+{
+	if (!sc->firmware)
+		FINSR(sc->regs, CML_REG(0), 7, 0, 0x1f);
+
+	if (sc->link_rate == KSERDES_LINK_RATE_10P3125G) {
+		_kserdes_pll_enable(sc->regs);
+		_kserdes_pll2_enable(sc->regs);
+	} else if (sc->link_rate == KSERDES_LINK_RATE_1P25G) {
+		kserdes_writel(sc->regs, PLL_CTRL_REG, PLL_ENABLE_1P25G);
+	}
+}
+
+static inline void _kserdes_enable_xgmii_port(struct regmap *peripheral_regmap,
+					      u32 port)
+{
+	regmap_update_bits(peripheral_regmap, XGE_CTRL_OFFSET,
+			   GENMASK(port, port), BIT(port));
+}
+
+static inline void _kserdes_reset_rx(void __iomem *sregs, int lane)
+{
+	_kserdes_force_signal_detect_low(sregs, lane);
+	usleep_range(1000, 2000);
+	_kserdes_force_signal_detect_high(sregs, lane);
+}
+
+static int kserdes_check_link_status(struct kserdes_config *sc,
+				     u32 *current_lane_state,
+				     u32 lanes_chk_mask,
+				     u32 *lanes_up_mask)
+{
+	struct device *dev = sc->dev;
+	u32 pcsr_rx_stat, blk_lock, blk_errs;
+	int loss, i, link_up = 1;
+	int ret;
+	unsigned long lmask = (unsigned long)lanes_chk_mask;
+
+	for_each_set_bit(i, &lmask, 8) {
+		loss = (kserdes_readl(sc->regs, LANE_CTRL_STS_REG(i))) & 0x01;
+
+		ret = regmap_read(sc->pcsr_regmap, PCSR_RX_STATUS(i),
+				  &pcsr_rx_stat);
+
+		if (ret)
+			return ret;
+
+		blk_lock = (pcsr_rx_stat >> 30) & 0x1;
+		blk_errs = (pcsr_rx_stat >> 16) & 0x0ff;
+
+		if (blk_errs)
+			blk_lock = 0;
+
+		switch (current_lane_state[i]) {
+		case 0:
+			if (!loss && blk_lock) {
+				dev_dbg(dev, "XGE PCSR Linked Lane: %d\n", i);
+				FINSR(sc->regs, LANEX_REG(i, 0x04), 2, 1, 0);
+				current_lane_state[i] = 1;
+			} else if (!blk_lock) {
+				dev_dbg(dev,
+					"XGE PCSR Recover Lane: %d\n", i);
+				_kserdes_reset_rx(sc->regs, i);
+			}
+			break;
+		case 1:
+			if (!blk_lock)
+				current_lane_state[i] = 2;
+
+			break;
+		case 2:
+			if (blk_lock) {
+				current_lane_state[i] = 1;
+			} else {
+				_kserdes_reset_rx(sc->regs, i);
+				current_lane_state[i] = 0;
+			}
+			break;
+		default:
+			dev_info(dev,
+				 "XGE: unknown current_lane_state[%d] %d\n",
+				 i, current_lane_state[i]);
+			break;
+		}
+
+		if (blk_errs) {
+			regmap_update_bits(sc->pcsr_regmap, PCSR_RX_CTL(i),
+					   GENMASK(7, 0), 0x19);
+			regmap_update_bits(sc->pcsr_regmap, PCSR_RX_CTL(i),
+					   GENMASK(7, 0), 0x00);
+		}
+
+		if (current_lane_state[i] == 1) {
+			*lanes_up_mask |= BIT(i);
+		} else {
+			*lanes_up_mask &= ~BIT(i);
+			link_up = 0;
+		}
+	}
+
+	return link_up;
+}
+
+static int kserdes_check_lanes_status(struct kserdes_config *sc);
+
+static int kserdes_wait_link_up(struct kserdes_config *sc,
+				u32 lanes_chk_mask,
+				u32 *lanes_up_mask)
+{
+	u32 current_state[KSERDES_MAX_LANES];
+	unsigned long time_check = 0;
+	int i, link_up, lane_status, ret = 0;
+
+	memset(current_state, 0, sizeof(current_state));
+
+	do {
+		usleep_range(10000, 20000);
+		if (sc->phy_type == KSERDES_PHY_XGE)
+			link_up = kserdes_check_link_status(sc, current_state,
+							    lanes_chk_mask,
+							    lanes_up_mask);
+		else {
+			lane_status = kserdes_check_lanes_status(sc);
+			if (lane_status)
+				link_up = 0;
+			else
+				link_up = 1;
+		}
+
+		if (link_up)
+			break;
+
+		for_each_enable_lane(sc, i) {
+			if (!(*lanes_up_mask & BIT(i))) {
+				dev_dbg(sc->dev,
+					"Lane %d down when waiting link up\n",
+					 i);
+			}
+		}
+
+		if (++time_check >= 200) {
+			ret = -ETIMEDOUT;
+			break;
+		}
+
+	} while (1);
+
+	return ret;
+}
+
+static inline void kserdes_xfw_get_lane_params(struct kserdes_config *sc,
+					       int lane)
+{
+	struct kserdes_fw_config *fw = &sc->fw;
+	u32 tx_ctrl, val_0, val_1;
+	u32 phy_a = PHY_A(sc->regs);
+
+	val_0 = kserdes_readl(sc->regs, LANEX_REG(lane, 0x04));
+	val_1 = kserdes_readl(sc->regs, LANEX_REG(lane, 0x08));
+
+	tx_ctrl = ((((val_0 >> 18) & 0x1)    << 24) |
+		   (((val_1 >> 0)  & 0xffff) <<  8) |
+		   (((val_0 >> 24) & 0xff)   <<  0));
+
+	if (phy_a) {
+		fw->cm = (val_1 >> 12) & 0xf;
+		fw->c1 = (val_1 >> 0) & 0x1f;
+		fw->c2 = (val_1 >> 8) & 0xf;
+	} else {
+		fw->cm = (tx_ctrl >> 16) & 0xf;
+		fw->c1 = (tx_ctrl >> 8) & 0x1f;
+		fw->c2 = (tx_ctrl >> 13) & 0x7;
+		fw->c2 = fw->c2 | (((tx_ctrl >> 24) & 0x1) << 3);
+	}
+
+	val_0 = _kserdes_read_select_tbus(sc->regs, lane + 1,
+					  (phy_a ? 0x11 : 0x10));
+	fw->attn = (val_0 >> 4) & 0xf;
+	fw->boost = (val_0 >> 8) & 0xf;
+
+	val_0 = _kserdes_read_select_tbus(sc->regs, lane + 1, 0x5);
+	fw->dlpf = (val_0 >> 2) & 0x3ff;
+
+	val_0 = _kserdes_read_select_tbus(sc->regs, lane + 1, 0x6);
+	fw->rxcal = (val_0 >> 3) & 0xff;
+}
+
+static inline void kserdes_xfw_mem_init(struct kserdes_config *sc)
+{
+	struct kserdes_fw_config *fw = &sc->fw;
+	u32 i, lane_config = 0;
+
+	for_each_lane(sc, i)
+		lane_config = (lane_config << 8) | (fw->lane_config[i] & 0xff);
+
+	lane_config <<= 8;
+
+	kserdes_writel(sc->regs, MEM_ADR_REG, KSERDES_XFW_CONFIG_START_ADDR);
+
+	for (i = KSERDES_XFW_CONFIG_START_ADDR;
+	     i < KSERDES_XFW_PARAM_START_ADDR; i += 4)
+		kserdes_writel(sc->regs, MEM_DATINC_REG, 0x00000000);
+
+	kserdes_writel(sc->regs, MEM_DATINC_REG, XFM_FLUSH_CMD);
+	kserdes_writel(sc->regs, MEM_DATINC_REG, fw->fast_train);
+	kserdes_writel(sc->regs, MEM_DATINC_REG, 0x00000000);
+	kserdes_writel(sc->regs, MEM_DATINC_REG, fw->lane_seeds);
+	kserdes_writel(sc->regs, MEM_DATINC_REG, lane_config);
+}
+
+static int kserdes_pcie_lanes_enable(struct kserdes_config *sc)
+{
+	int ret, i;
+	u32 lanes_enable = 0;
+
+	for_each_enable_lane(sc, i)
+		lanes_enable |= BIT(i);
+
+	for_each_lane(sc, i) {
+		kserdes_release_reset(sc, i);
+
+		if (sc->lane[i].loopback)
+			_kserdes_set_lane_loopback(sc->regs, i, sc->phy_type);
+	}
+
+	ret = kserdes_get_status(sc);
+	if (ret)
+		return ret;
+
+	return lanes_enable;
+}
+
+static void kserdes_clear_wait_after(struct kserdes_config *sc,
+				     unsigned long lanes_mask)
+{
+	u32 lane;
+
+	if (!sc->rx_force_enable) {
+		for_each_set_bit(lane, &lanes_mask, 8) {
+			if (!LANE_ENABLE(sc, lane))
+				continue;
+
+			_kserdes_clear_lane_wait_after(sc->regs, lane);
+		}
+	} else {
+		_kserdes_clear_wait_after(sc->regs);
+	}
+}
+
+static int kserdes_check_lanes_status(struct kserdes_config *sc)
+{
+	unsigned long timeout = jiffies + msecs_to_jiffies(500);
+	unsigned long time_check;
+	u32 val, i;
+
+	do {
+		time_check = jiffies;
+		val = 1;
+
+		for_each_enable_lane(sc, i)
+			val &= _kserdes_get_lane_status(sc->regs, i,
+							sc->phy_type);
+
+		if (val)
+			break;
+
+		if (time_after(time_check, timeout))
+			return -ETIMEDOUT;
+
+		cpu_relax();
+	} while (true);
+
+	return 0;
+}
+
+static void kserdes_phya_init_config(struct kserdes_config *sc, u32 lane)
+{
+	u32 coef1val, coef2val, coef3val, coef4val, coef5val;
+	void __iomem *sregs = sc->regs;
+	u32 cmp, coef1_ofs;
+
+	for_each_cmp(cmp) {
+		if (!(cmp & 0x1))
+			continue;
+
+		FINSR(sregs, CML_REG(0x8c), 23, 21, cmp);
+
+		FINSR(sregs, CMU0_REG(0x8), 31, 24, ((lane + 1) << 5) + 0x12);
+		coef1_ofs = (_kserdes_read_tbus_val(sregs) & 0x000f) << 3;
+
+		FINSR(sregs, CMU0_REG(0x8), 31, 24, ((lane + 1) << 5) + 0x13);
+		coef1_ofs |= (_kserdes_read_tbus_val(sregs) & 0x0e00) >> 9;
+
+		coef1val = coef1_ofs - 14;
+		coef2val = 31;
+		coef3val = 31;
+		coef4val = 31;
+		coef5val = 31;
+
+		FINSR(sregs, CML_REG(0xf0), 27, 26, lane + 1);
+		FINSR(sregs, LANEX_REG(lane, 0x2c), 2, 2, 0x1);
+		FINSR(sregs, LANEX_REG(lane, 0x30), 7, 5, cmp);
+		FINSR(sregs, LANEX_REG(lane, 0x5c), 31, 31, 0x1);
+
+		FINSR(sregs, LANEX_REG(lane, 0x58), 30, 24, coef1val);
+		FINSR(sregs, LANEX_REG(lane, 0x5c),  6,  0, coef2val);
+		FINSR(sregs, LANEX_REG(lane, 0x5c), 13,  8, coef3val);
+		FINSR(sregs, LANEX_REG(lane, 0x5c), 21, 16, coef4val);
+		FINSR(sregs, LANEX_REG(lane, 0x5c), 29, 24, coef5val);
+
+		FINSR(sregs, LANEX_REG(lane, 0x2c), 10, 10, 0x1);
+		FINSR(sregs, LANEX_REG(lane, 0x2c), 10, 10, 0x0);
+
+		FINSR(sregs, LANEX_REG(lane, 0x2c), 2, 2, 0x0);
+		FINSR(sregs, LANEX_REG(lane, 0x5c), 31, 31, 0x0);
+
+		FINSR(sregs, LANEX_REG(lane, 0x58), 16, 16, 0x1);
+		FINSR(sregs, LANEX_REG(lane, 0x48), 16, 16, 0x1);
+
+		FINSR(sregs, LANEX_REG(lane, 0x4c), 5, 2, (0x1 << (cmp - 1)));
+		FINSR(sregs, LANEX_REG(lane, 0x58), 17, 17, 0x1);
+
+		_kserdes_set_coef_ofs(sregs, lane, 1, 7, coef1val);
+		_kserdes_set_coef_ofs(sregs, lane, 2, 6, coef2val);
+		_kserdes_set_coef_ofs(sregs, lane, 3, 6, coef3val);
+		_kserdes_set_coef_ofs(sregs, lane, 4, 6, coef4val);
+		_kserdes_set_coef_ofs(sregs, lane, 5, 6, coef5val);
+
+		FINSR(sregs, LANEX_REG(lane, 0x58), 16, 16, 0x0);
+		FINSR(sregs, LANEX_REG(lane, 0x48), 16, 16, 0x0);
+		FINSR(sregs, LANEX_REG(lane, 0x58), 17, 17, 0x0);
+	}
+}
+
+static int kserdes_check_pll_status(struct kserdes_config *sc)
+{
+	unsigned long timeout = jiffies + msecs_to_jiffies(500);
+	unsigned long time_check;
+	u32 val;
+
+	do {
+		time_check = jiffies;
+		val = _kserdes_get_pll_status(sc->regs);
+
+		if (sc->phy_type == KSERDES_PHY_XGE)
+			val &= _kserdes_get_pll2_status(sc->regs);
+
+		if (val)
+			break;
+
+		if (time_after(time_check, timeout))
+			return -ETIMEDOUT;
+
+		cpu_relax();
+	} while (true);
+
+	return 0;
+}
+
+static void kserdes_enable_common_set_lane_rate(struct kserdes_config *sc,
+						u32 lane)
+{
+	int ret;
+
+	ret = _kserdes_set_lane_ctrl_rate(sc->regs, lane,
+					  sc->lane[lane].ctrl_rate);
+	if (ret) {
+		dev_err(sc->dev, "set_lane_rate FAILED: lane = %d err = %d\n",
+			lane, ret);
+		return;
+	}
+
+	switch (sc->phy_type) {
+	case KSERDES_PHY_SGMII:
+		FINSR(sc->regs, LANE_CTRL_STS_REG(lane),  7,  6, 0x3);
+		FINSR(sc->regs, LANE_CTRL_STS_REG(lane), 23, 21, 0x4);
+		FINSR(sc->regs, LANE_CTRL_STS_REG(lane),  5,  3, 0x4);
+		break;
+
+	case KSERDES_PHY_XGE:
+		FINSR(sc->regs, LANE_CTRL_STS_REG(lane), 23, 21, 0x7);
+		FINSR(sc->regs, LANE_CTRL_STS_REG(lane),  5,  3, 0x7);
+		if (sc->link_rate == KSERDES_LINK_RATE_10P3125G) {
+			FINSR(sc->regs, LANE_CTRL_STS_REG(lane), 16, 16, 0x1);
+			FINSR(sc->regs, LANE_CTRL_STS_REG(lane), 19, 19, 0x1);
+		}
+		break;
+
+	case KSERDES_PHY_PCIE:
+		break;
+
+	default:
+		FINSR(sc->regs, LANE_CTRL_STS_REG(lane), 23, 21, 0x6);
+		FINSR(sc->regs, LANE_CTRL_STS_REG(lane),  5,  3, 0x6);
+		break;
+	}
+
+	if (sc->lane[lane].loopback)
+		_kserdes_set_lane_loopback(sc->regs, lane, sc->phy_type);
+
+	if (sc->phy_type != KSERDES_PHY_XGE) {
+		FINSR(sc->regs, LANEX_REG(lane, 0x30), 11, 11, 0x1);
+		FINSR(sc->regs, LANEX_REG(lane, 0x30), 13, 12, 0x0);
+	}
+}
+
+static inline void kserdes_set_lane_rx_starts(struct kserdes_config *sc,
+					      u32 lane)
+{
+	FINSR(sc->regs, LANEX_REG(lane, 0x8c), 11, 8,
+	      sc->lane[lane].rx_start.att);
+	FINSR(sc->regs, LANEX_REG(lane, 0x8c), 15, 12,
+	      sc->lane[lane].rx_start.boost);
+
+	FINSR(sc->regs, LANEX_REG(lane, 0x84), 27, 24,
+	      sc->lane[lane].rx_start.att);
+	FINSR(sc->regs, LANEX_REG(lane, 0x84), 31, 28,
+	      sc->lane[lane].rx_start.boost);
+
+	FINSR(sc->regs, LANEX_REG(lane, 0x84), 19, 16,
+	      sc->lane[lane].rx_start.att);
+	FINSR(sc->regs, LANEX_REG(lane, 0x84), 23, 20,
+	      sc->lane[lane].rx_start.boost);
+}
+
+static void kserdes_hs_init_config(struct kserdes_config *sc)
+{
+	int i;
+
+	if (sc->phy_type != KSERDES_PHY_XGE) {
+		if (sc->link_rate >= KSERDES_LINK_RATE_9P8304G)
+			FINSR(sc->regs, CML_REG(0xbc), 28, 24, 0x1e);
+	}
+
+	for_each_enable_lane(sc, i)
+		kserdes_set_tx_idle(sc, i);
+
+	if (sc->link_rate >= KSERDES_LINK_RATE_9P8304G) {
+		if (sc->phy_type != KSERDES_PHY_XGE) {
+			for_each_enable_lane(sc, i)
+				kserdes_force_signal_detect_low(sc, i);
+
+			for_each_enable_lane(sc, i)
+				FINSR(sc->regs, LANEX_REG(i, 0x78),
+				      30, 24, 0x7f);
+		} else {
+			FINSR(sc->regs, CML_REG(0x10c), 7, 0, 0xff);
+		}
+	}
+}
+
+static int kserdes_lanes_enable_common(struct kserdes_config *sc,
+				       struct kserdes_ofs *sofs)
+{
+	u32 val, lane_mask = 0;
+	int i, ret;
+
+	for_each_lane(sc, i) {
+		if (sc->lane[i].enable)
+			lane_mask |= BIT(i);
+		else
+			sc->lane[i].enable = 1;
+	}
+
+	if (sc->phy_type == KSERDES_PHY_PCIE) {
+		dev_err(sc->dev, "%s: pcie TBD.\n", __func__);
+		return -EINVAL;
+	}
+
+	kserdes_hs_init_config(sc);
+
+	for_each_enable_lane(sc, i)
+		kserdes_set_lane_rx_starts(sc, i);
+
+	kserdes_assert_reset(sc);
+
+	for_each_enable_lane(sc, i)
+		kserdes_set_tx_rx_fir_coeff(sc, i);
+
+	for_each_enable_lane(sc, i)
+		kserdes_force_signal_detect_low(sc, i);
+
+	ret = kserdes_deassert_reset(sc, 0);
+	if (ret) {
+		dev_err(sc->dev, "kserdes_deassert_reset FAILED %d\n", ret);
+		return ret;
+	}
+
+	for_each_enable_lane(sc, i)
+		kserdes_enable_common_set_lane_rate(sc, i);
+
+	if (sc->phy_type == KSERDES_PHY_XGE)
+		kserdes_xge_pll_enable(sc);
+	else
+		_kserdes_pll_enable(sc->regs);
+
+	ret = kserdes_check_pll_status(sc);
+	if (ret) {
+		dev_err(sc->dev,
+			"common init: check pll status FAILED %d\n", ret);
+		return ret;
+	}
+
+	for_each_enable_lane(sc, i)
+		_kserdes_lane_enable(sc->regs, i);
+
+	ret = kserdes_check_lanes_status(sc);
+	if (ret) {
+		dev_err(sc->dev,
+			"common init: check lanes status FAILED %d\n", ret);
+		return ret;
+	}
+
+	usleep_range(5, 10);
+
+	val = _kserdes_get_tx_termination(sc->regs, sc->phy_type);
+
+	kserdes_set_tx_terminations(sc, val);
+
+	if (sc->phy_type == KSERDES_PHY_XGE)
+		kserdes_phyb_init_config(sc, sofs);
+	else if (sc->link_rate >= KSERDES_LINK_RATE_9P8304G)
+		for_each_enable_lane(sc, i)
+			kserdes_phya_init_config(sc, i);
+
+	for_each_enable_lane(sc, i)
+		kserdes_clr_tx_idle(sc, i);
+
+	for_each_lane(sc, i)
+		sc->lane[i].enable = (lane_mask & BIT(i)) >> i;
+
+	return 0;
+}
+
+static inline u32 _kserdes_get_lane_sd(void __iomem *sregs, u32 lane)
+{
+	return FEXTR(kserdes_readl(sregs, PLL_CTRL_REG), lane, lane);
+}
+
+static int _kserdes_wait_lane_sd(void __iomem *sregs, u32 lane)
+{
+	unsigned long timeout = jiffies + msecs_to_jiffies(500);
+	unsigned long time_check;
+
+	do {
+		time_check = jiffies;
+
+		if (_kserdes_get_lane_sd(sregs, lane))
+			break;
+
+		if (time_after(time_check, timeout))
+			return -ETIMEDOUT;
+
+		cpu_relax();
+	} while (true);
+
+	return 0;
+}
+
+static void kserdes_rx_att_boost_config_phyb(struct kserdes_config *sc,
+					     u32 lane)
+{
+	u32 tbus_ofs, rxeq_init_reg_ofs, rxeq_ln_reg_ofs, rxeq_ln_force_bit;
+	void __iomem *sregs = sc->regs;
+	u32 att_start, att_read, boost_read;
+	int ret;
+
+	if (sc->phy_type == KSERDES_PHY_XGE) {
+		tbus_ofs = 0x10;
+		rxeq_init_reg_ofs = 0x9c;
+		rxeq_ln_reg_ofs = 0x98;
+		rxeq_ln_force_bit = 14;
+	} else {
+		tbus_ofs = 0x11;
+		rxeq_init_reg_ofs = 0x84;
+		rxeq_ln_reg_ofs = 0xac;
+		rxeq_ln_force_bit = 11;
+	}
+
+	att_start = kserdes_readl(sregs, LANEX_REG(lane, 0x8c));
+	att_start = (att_start >> 8) & 0xf;
+
+	att_read = _kserdes_read_select_tbus(sregs, lane + 1, tbus_ofs);
+	att_read = (att_read >> 4) & 0xf;
+
+	FINSR(sregs, LANEX_REG(lane, 0x8c), 11, 8, att_read);
+	FINSR(sregs, LANEX_REG(lane, rxeq_init_reg_ofs), 0, 0, 0x0);
+	FINSR(sregs, CML_REG(0x8c), 24, 24, 0x0);
+
+	FINSR(sregs, LANEX_REG(lane, rxeq_ln_reg_ofs),
+	      rxeq_ln_force_bit, rxeq_ln_force_bit, 0x1);
+	FINSR(sregs, LANEX_REG(lane, rxeq_ln_reg_ofs),
+	      rxeq_ln_force_bit, rxeq_ln_force_bit, 0x0);
+
+	ret = kserdes_wait_lane_rx_valid(sc, lane);
+	if (ret) {
+		dev_dbg(sc->dev, "kserdes_wait_lane_rx_valid %d FAILED: %d\n",
+			lane, ret);
+	}
+	usleep_range(300, 600);
+
+	boost_read = _kserdes_read_select_tbus(sregs, lane + 1, tbus_ofs);
+	boost_read = (boost_read >> 8) & 0xf;
+
+	if (!boost_read) {
+		FINSR(sregs, LANEX_REG(lane, 0x2c),  2,  2, 0x1);
+		FINSR(sregs, LANEX_REG(lane, 0x2c), 18, 12, 0x2);
+		FINSR(sregs, LANEX_REG(lane, 0x2c),  9,  3, 0x1);
+		FINSR(sregs, LANEX_REG(lane, 0x2c), 10, 10, 0x1);
+		FINSR(sregs, LANEX_REG(lane, 0x2c), 10, 10, 0x0);
+		FINSR(sregs, LANEX_REG(lane, 0x2c),  2,  2, 0x0);
+		FINSR(sregs, LANEX_REG(lane, 0x2c), 18, 12, 0x0);
+		FINSR(sregs, LANEX_REG(lane, 0x2c),  9,  3, 0x0);
+	}
+
+	FINSR(sregs, LANEX_REG(lane, 0x8c), 11, 8, att_start);
+	FINSR(sregs, LANEX_REG(lane, rxeq_init_reg_ofs), 0, 0, 0x1);
+	FINSR(sregs, CML_REG(0x8c), 24, 24, 0x1);
+}
+
+static int kserdes_set_dlev_patt_adapt(struct kserdes_config *sc,
+				       u32 lane, u32 pattern,
+				       struct kserdes_lane_ofs *lofs)
+{
+	struct kserdes_cmp_coef_ofs *ctofs = &lofs->ct_ofs[4];
+	void __iomem *sregs = sc->regs;
+	u32 dlevp, dlevn, dlevavg;
+	int ret;
+
+	FINSR(sregs, CML_REG(0x158), 14, 8, pattern);
+	FINSR(sregs, LANEX_REG(lane, 0x98), 14, 14, 1);
+	FINSR(sregs, LANEX_REG(lane, 0x98), 14, 14, 0);
+
+	ret = kserdes_wait_lane_rx_valid(sc, lane);
+	if (ret) {
+		dev_dbg(sc->dev,
+			"set dlev patt: wait_lane_rx_valid FAILED %d\n", ret);
+		return ret;
+	}
+
+	dlevp = _kserdes_read_select_tbus(sregs, lane + 1, 0x44);
+	dlevp = (dlevp >> 4) & 0xff;
+
+	dlevn = _kserdes_read_select_tbus(sregs, lane + 1, 0x45);
+	dlevn &= 0xff;
+
+	if (ctofs->cmp <= 120)
+		dlevavg = ctofs->cmp - dlevn;
+	else if (ctofs->cmp >= 134)
+		dlevavg = dlevp - ctofs->cmp;
+	else
+		dlevavg = (dlevp - dlevn) / 2;
+
+	return dlevavg;
+}
+
+static u32 kserdes_eye_monitor_dll_ovr(struct kserdes_config *sc,
+				       u32 lane, u32 phase_num, u32 t_offset,
+				       u32 phase_shift)
+{
+	void __iomem *sregs = sc->regs;
+	u32 tbus_data, delay, partial_eye, i;
+	u32 start_bin = 0, end_bin = 0;
+	u32 eye_scan_errors_array[128];
+	u32 error_free = 0, val_0, val_1;
+	u32 max_dly = 128;
+	bool not_phy_xge = (sc->phy_type != KSERDES_PHY_XGE);
+
+	if (t_offset == 0)
+		t_offset++;
+
+	if (phase_num == 1) {
+		val_0 = 0x00400000;
+		val_1 = 0x00000011;
+	} else {
+		val_0 = 0x00800000;
+		val_1 = 0x00000009;
+	}
+	reg_rmw(sregs + LANEX_REG(lane, 0x2c), val_0, GENMASK(23, 22));
+	reg_rmw(sregs + LANEX_REG(lane, 0x30), val_1, GENMASK(4, 0));
+
+	reg_rmw(sregs + CML_REG(0xb8), 0x00004000, GENMASK(15, 8));
+
+	if (phase_num == 1)
+		val_0 = 0xffef0000;
+	else
+		val_0 = 0xfff60000;
+
+	reg_rmw(sregs + CML_REG(0xb8), val_0, GENMASK(31, 16));
+
+	reg_rmw(sregs + CML_REG(0xbc), 0x000fffff, GENMASK(19, 0));
+	tbus_data = _kserdes_read_select_tbus(sregs, lane + 1, 0x02);
+	tbus_data = tbus_data;
+	usleep_range(250, 500);
+
+	for (i = 0; i < max_dly; i = i + t_offset) {
+		reg_rmw(sregs + LANEX_REG(lane, 0x2c),
+			(i & 0xff) << 24, GENMASK(31, 24));
+
+		reg_rmw(sregs + LANEX_REG(lane, 0x30),
+			phase_shift, GENMASK(1, 0));
+		usleep_range(5, 10);
+		reg_rmw(sregs + CML_REG(0xb8), 0x0000c000, GENMASK(15, 8));
+		usleep_range(500, 1000);
+
+		val_0 = _kserdes_read_select_tbus(sregs, lane + 1,
+						  not_phy_xge ?  0x1a : 0x19);
+		val_0 <<= 4;
+
+		val_1 = _kserdes_read_select_tbus(sregs, lane + 1,
+						  not_phy_xge ?  0x1b : 0x1a);
+		val_1 = (val_1 >> 8) & 0xf;
+
+		eye_scan_errors_array[i] = (val_0 | val_1);
+
+		reg_rmw(sregs + CML_REG(0xb8), 0x00004000, GENMASK(15, 8));
+	}
+
+	partial_eye = 0;
+	error_free = 0;
+
+	for (i = 0; i < max_dly; i = i + t_offset) {
+		if (i == 0) {
+			if (eye_scan_errors_array[i] < 16384)
+				partial_eye = 1;
+		} else {
+			if (eye_scan_errors_array[i] > 16384 + 3000)
+				partial_eye = 0;
+		}
+
+		if ((eye_scan_errors_array[i] < 16384) &&
+		    (partial_eye == 0) &&
+		    (error_free  == 0)) {
+			if (!((eye_scan_errors_array[i - 1] > 16384) &&
+			      (eye_scan_errors_array[i + 1] > 16384))) {
+				error_free = 1;
+				start_bin = i;
+			}
+		} else if ((eye_scan_errors_array[i] > 16384) &&
+			   (partial_eye == 0) &&
+			   (error_free  == 1)) {
+			if (!((eye_scan_errors_array[i - 1] < 16384) &&
+			      (eye_scan_errors_array[i + 1] < 16384))) {
+				end_bin = i;
+				break;
+			}
+		}
+	}
+
+	delay = (end_bin - start_bin) / 4 + start_bin;
+	reg_rmw(sregs + LANEX_REG(lane, 0x30), 0x00000000, GENMASK(7, 0));
+	reg_rmw(sregs + LANEX_REG(lane, 0x2c), 0x00000003, GENMASK(7, 0));
+	reg_rmw(sregs + CML_REG(0x98), 0x00000000, GENMASK(31, 0));
+	reg_rmw(sregs + CML_REG(0xb8), 0x00000000, GENMASK(15, 14));
+	reg_rmw(sregs + LANEX_REG(lane, 0x2c), 0x00000000, GENMASK(23, 22));
+	return delay;
+}
+
+static void kserdes_rx_calibration_phyb(struct kserdes_config *sc, u32 lane,
+					struct kserdes_lane_ofs *lofs,
+					struct kserdes_lane_dlev_out *ldlevo)
+{
+	struct kserdes_cmp_coef_ofs *ctofs, *ctofs_temp;
+	struct kserdes_lane_ofs lofs_temp;
+	void __iomem *sregs = sc->regs;
+	u32 att, boost, comp_no, att_start, boost_start;
+	u32 delay_ovr = 0;
+	int dlevavg_temp[6];
+
+	delay_ovr = kserdes_eye_monitor_dll_ovr(sc, lane, 0, 1, 0);
+	ldlevo->delay = delay_ovr;
+
+	FINSR(sregs, CML_REG(0x164), 15, 15, 1);
+
+	FINSR(sregs, CML_REG(0x164), 16, 16, 1);
+	FINSR(sregs, CML_REG(0x164), 31, 26, (128 + delay_ovr) & 0x3f);
+	FINSR(sregs, CML_REG(0x168),  2,  0, (128 + delay_ovr) >> 6);
+	FINSR(sregs, LANEX_REG(lane, 0x9c), 1, 1, 0);
+	FINSR(sregs, LANEX_REG(lane, 0x9c), 0, 0, 0);
+
+	att_start = (kserdes_readl(sregs, LANEX_REG(lane, 0x8c)) >> 8) & 0xf;
+	boost_start = (kserdes_readl(sregs, LANEX_REG(lane, 0x8c)) >> 12) & 0xf;
+
+	att = _kserdes_read_select_tbus(sregs, lane + 1, 0x10);
+	boost = (att >> 8) & 0xf;
+	att = (att >> 4) & 0xf;
+
+	FINSR(sregs, LANEX_REG(lane, 0x8c), 11, 8, att);
+	FINSR(sregs, LANEX_REG(lane, 0x8c), 15, 12, boost);
+
+	dlevavg_temp[0] = kserdes_set_dlev_patt_adapt(sc, lane, 0x71, lofs);
+	dlevavg_temp[1] = kserdes_set_dlev_patt_adapt(sc, lane, 0x61, lofs);
+	dlevavg_temp[2] = kserdes_set_dlev_patt_adapt(sc, lane, 0x79, lofs);
+	dlevavg_temp[3] = kserdes_set_dlev_patt_adapt(sc, lane, 0x75, lofs);
+	dlevavg_temp[4] = kserdes_set_dlev_patt_adapt(sc, lane, 0x73, lofs);
+	dlevavg_temp[5] = kserdes_set_dlev_patt_adapt(sc, lane, 0x70, lofs);
+
+	ldlevo->coef_vals[0] = (dlevavg_temp[0] - dlevavg_temp[1]) /  2;
+	ldlevo->coef_vals[1] = (dlevavg_temp[0] - dlevavg_temp[2]) / -2;
+	ldlevo->coef_vals[2] = (dlevavg_temp[0] - dlevavg_temp[3]) / -2;
+	ldlevo->coef_vals[3] = (dlevavg_temp[0] - dlevavg_temp[4]) / -2;
+	ldlevo->coef_vals[4] = (dlevavg_temp[0] - dlevavg_temp[5]) /  2;
+
+	ldlevo->coef_vals[0] = ldlevo->coef_vals[0] -
+					ldlevo->coef_vals[0] / 3;
+
+	for (comp_no = 1; comp_no < 5; comp_no++) {
+		ctofs = &lofs->ct_ofs[comp_no];
+		ctofs_temp = &lofs_temp.ct_ofs[comp_no];
+
+		ctofs_temp->cmp = ctofs->cmp;
+
+		if ((comp_no == 1) || (comp_no == 3)) {
+			ctofs_temp->coef1 = ldlevo->coef_vals[0] + ctofs->coef1;
+			ctofs_temp->coef2 = ldlevo->coef_vals[1] + ctofs->coef2;
+			ctofs_temp->coef3 = ldlevo->coef_vals[2] + ctofs->coef3;
+			ctofs_temp->coef4 = ldlevo->coef_vals[3] + ctofs->coef4;
+			ctofs_temp->coef5 = ldlevo->coef_vals[4] + ctofs->coef5;
+		} else {
+			ctofs_temp->coef1 = ctofs->coef1;
+			ctofs_temp->coef2 = ctofs->coef2;
+			ctofs_temp->coef3 = ctofs->coef3;
+			ctofs_temp->coef4 = ctofs->coef4;
+			ctofs_temp->coef5 = ctofs->coef5;
+		}
+
+		_kserdes_set_ofs(sregs, lane, comp_no, ctofs_temp);
+	}
+
+	FINSR(sregs, LANEX_REG(lane, 0x8c), 11, 8, att_start);
+	FINSR(sregs, LANEX_REG(lane, 0x8c), 15, 12, boost_start);
+	FINSR(sregs, LANEX_REG(lane, 0x9c), 1, 1, 1);
+	FINSR(sregs, LANEX_REG(lane, 0x9c), 0, 0, 1);
+}
+
+static int kserdes_rx_boost_config_phya(struct kserdes_config *sc, u32 lane)
+{
+	u32 boost_read;
+	int ret;
+	bool phy_xge = (sc->phy_type == KSERDES_PHY_XGE);
+
+	ret = kserdes_wait_lane_rx_valid(sc, lane);
+	if (ret) {
+		dev_err(sc->dev,
+			"config_phya: wait_lane_rx_valid FAILED %d\n", ret);
+		return ret;
+	}
+
+	boost_read = _kserdes_read_select_tbus(sc->regs, lane + 1,
+					       phy_xge ?  0x10 : 0x11);
+
+	boost_read = (boost_read >> 8) & 0xf;
+
+	if (!boost_read) {
+		FINSR(sc->regs, LANEX_REG(lane, 0x2c),  2,  2, 0x1);
+		FINSR(sc->regs, LANEX_REG(lane, 0x2c), 18, 12, 0x2);
+		FINSR(sc->regs, LANEX_REG(lane, 0x2c),  9,  3, 0x1);
+		FINSR(sc->regs, LANEX_REG(lane, 0x2c), 10, 10, 0x1);
+		FINSR(sc->regs, LANEX_REG(lane, 0x2c), 10, 10, 0x0);
+		FINSR(sc->regs, LANEX_REG(lane, 0x2c),  2,  2, 0x0);
+		FINSR(sc->regs, LANEX_REG(lane, 0x2c), 18, 12, 0x0);
+		FINSR(sc->regs, LANEX_REG(lane, 0x2c),  9,  3, 0x0);
+	}
+	return 0;
+}
+
+static int kserdes_enable_lane_rx(struct kserdes_config *sc, u32 lane,
+				  struct kserdes_lane_ofs *lofs,
+				  struct kserdes_lane_dlev_out *ldlevo)
+{
+	int ret = 0;
+
+	_kserdes_force_signal_detect_high(sc->regs, lane);
+
+	if (!sc->rx_force_enable) {
+		ret = _kserdes_wait_lane_sd(sc->regs, lane);
+		if (ret) {
+			dev_err(sc->dev,
+				"init_lane_rx wait sd valid FAILED %d\n", ret);
+			return ret;
+		}
+
+		if ((sc->phy_type == KSERDES_PHY_XGE) ||
+		    (sc->link_rate >= KSERDES_LINK_RATE_5G)) {
+			if (sc->lane[lane].ctrl_rate == KSERDES_FULL_RATE) {
+				ret = kserdes_wait_lane_rx_valid(sc, lane);
+				if (ret) {
+					dev_err(sc->dev,
+						"init_lane_rx wait rx valid FAILED %d\n",
+					       ret);
+					return ret;
+				}
+			}
+		}
+
+		if (sc->phy_type == KSERDES_PHY_XGE) {
+			kserdes_rx_att_boost_config_phyb(sc, lane);
+		} else if ((sc->link_rate >= KSERDES_LINK_RATE_5G) &&
+			   (sc->lane[lane].ctrl_rate == KSERDES_FULL_RATE)) {
+			kserdes_rx_boost_config_phya(sc, lane);
+		}
+	}
+
+	if (sc->phy_type == KSERDES_PHY_XGE)
+		kserdes_rx_calibration_phyb(sc, lane, lofs, ldlevo);
+
+	return ret;
+}
+
+static int xge_kserdes_recover_lane_rx(struct kserdes_config *sc, u32 lane,
+				   struct kserdes_lane_ofs *lofs,
+				   struct kserdes_lane_dlev_out *ldlevo)
+{
+	int ret;
+
+	_kserdes_force_signal_detect_high(sc->regs, lane);
+
+	if (!sc->rx_force_enable) {
+		ret = _kserdes_wait_lane_sd(sc->regs, lane);
+		if (ret) {
+			dev_dbg(sc->dev,
+				"init_lane_rx wait sd valid FAILED %d\n", ret);
+			return ret;
+		}
+		dev_dbg(sc->dev, "recover_lane_rx sig detcected\n");
+
+		if ((sc->phy_type == KSERDES_PHY_XGE) ||
+		    (sc->link_rate >= KSERDES_LINK_RATE_5G)) {
+			if (sc->lane[lane].ctrl_rate == KSERDES_FULL_RATE) {
+				ret = kserdes_wait_lane_rx_valid(sc, lane);
+				if (ret) {
+					dev_err(sc->dev,
+						"init_lane_rx wait rx valid FAILED %d\n",
+					       ret);
+					return ret;
+				}
+				dev_dbg(sc->dev, "recover_lane_rx rx valid\n");
+			}
+		}
+
+		if (sc->phy_type == KSERDES_PHY_XGE) {
+			kserdes_rx_att_boost_config_phyb(sc, lane);
+		} else if ((sc->link_rate >= KSERDES_LINK_RATE_5G) &&
+			   (sc->lane[lane].ctrl_rate == KSERDES_FULL_RATE)) {
+			kserdes_rx_boost_config_phya(sc, lane);
+		}
+	}
+
+	if (sc->phy_type == KSERDES_PHY_XGE)
+		kserdes_rx_calibration_phyb(sc, lane, lofs, ldlevo);
+
+	return 0;
+}
+
+static int gbe_kserdes_recover_lane_rx(struct kserdes_config *sc, u32 lane,
+				       struct kserdes_lane_ofs *lofs,
+				       struct kserdes_lane_dlev_out *ldlevo)
+{
+	int ret;
+
+	_kserdes_force_signal_detect_high(sc->regs, lane);
+
+	if (!sc->rx_force_enable) {
+		ret = _kserdes_wait_lane_sd(sc->regs, lane);
+		if (ret) {
+			dev_dbg(sc->dev,
+				"init_lane_rx wait sd valid FAILED %d\n", ret);
+			return ret;
+		}
+		dev_dbg(sc->dev, "recover_lane_rx sig detcected\n");
+
+		if (sc->lane[lane].ctrl_rate == KSERDES_QUARTER_RATE) {
+			ret = kserdes_wait_lane_rx_valid(sc, lane);
+			if (ret) {
+				dev_err(sc->dev,
+					"init_lane_rx wait rx valid FAILED %d\n",
+					ret);
+				return ret;
+			}
+			dev_dbg(sc->dev, "recover_lane_rx rx valid\n");
+		}
+	}
+
+	return 0;
+}
+
+static int kserdes_sgmii_init(struct kserdes_config *sc)
+{
+	return kserdes_load_init_fw(sc, ks2_gbe_serdes_firmwares,
+				    ARRAY_SIZE(ks2_gbe_serdes_firmwares));
+}
+
+static int kserdes_xge_init(struct kserdes_config *sc)
+{
+	_kserdes_reset(sc->regs);
+	return kserdes_load_init_fw(sc, ks2_xgbe_serdes_firmwares,
+				    ARRAY_SIZE(ks2_xgbe_serdes_firmwares));
+}
+
+static int kserdes_pcie_init(struct kserdes_config *sc)
+{
+	return kserdes_load_init_fw(sc, ks2_pcie_serdes_firmwares,
+				    ARRAY_SIZE(ks2_pcie_serdes_firmwares));
+}
+
+static int kserdes_of_parse_lane(struct device *dev,
+				 struct device_node *np,
+				 struct kserdes_config *sc)
+{
+	struct kserdes_lane_config *lc;
+	struct kserdes_equalizer *eq;
+	struct kserdes_tx_coeff *tc;
+	int lane_num, ret;
+
+	ret = of_property_read_u32(np, "reg", &lane_num);
+	if (ret) {
+		dev_err(dev, "Failed to parse reg\n");
+		return -EINVAL;
+	}
+
+	if (lane_num >= sc->lanes) {
+		dev_err(dev, "Invalid lane number %u\n", lane_num);
+		return -EINVAL;
+	}
+
+	lc = &sc->lane[lane_num];
+
+	lc->enable = of_device_is_available(np);
+	dev_dbg(dev, "lane %d enabled\n", lane_num);
+
+	if (of_property_read_u32(np, "control-rate", &lc->ctrl_rate)) {
+		dev_info(dev, "use default lane control-rate: %u\n",
+			 lc->ctrl_rate);
+	}
+	dev_dbg(dev, "lane control-rate: %d\n", lc->ctrl_rate);
+
+	if (of_find_property(np, "loopback", NULL))
+		lc->loopback = true;
+	else
+		lc->loopback = false;
+
+	dev_dbg(dev, "lane loopback: %d\n", lc->loopback);
+
+	eq = &lc->rx_start;
+	if (of_property_read_u32_array(np, "rx-start", &eq->att, 2)) {
+		dev_info(dev, "use default lane rx-start 0 0\n");
+		eq->att = 0;
+		eq->boost = 0;
+	}
+	dev_dbg(dev, "lane rx-start: %d %d\n", eq->att, eq->boost);
+
+	eq = &lc->rx_force;
+	if (of_property_read_u32_array(np, "rx-force", &eq->att, 2)) {
+		dev_info(dev, "use default lane rx-force 0 0\n");
+		eq->att = 0;
+		eq->boost = 0;
+	}
+	dev_dbg(dev, "lane rx-force: %d %d\n", eq->att, eq->boost);
+
+	tc = &lc->tx_coeff;
+	if (of_property_read_u32_array(np, "tx-coeff", &tc->c1, 5)) {
+		dev_info(dev, "use default tx-coeff 0\n");
+		tc->c1 = 0;
+	}
+	dev_dbg(dev, "tx-coeff: %d %d %d %d %d\n",
+		tc->c1, tc->c2, tc->cm, tc->att, tc->vreg);
+
+	return lane_num;
+}
+
+static void kserdes_set_sgmii_defaults(struct kserdes_config *sc)
+{
+	int i;
+
+	sc->link_rate		= KSERDES_LINK_RATE_1P25G;
+	sc->lanes		= 4;
+	sc->rx_force_enable	= false;
+
+	for_each_lane(sc, i) {
+		memset(&sc->lane[i], 0, sizeof(sc->lane[i]));
+		sc->lane[i].ctrl_rate = KSERDES_QUARTER_RATE;
+	}
+}
+
+static void kserdes_set_xge_defaults(struct kserdes_config *sc)
+{
+	int i;
+
+	sc->link_rate		= KSERDES_LINK_RATE_10P3125G;
+	sc->lanes		= 2;
+	sc->rx_force_enable	= false;
+
+	for_each_lane(sc, i) {
+		memset(&sc->lane[i], 0, sizeof(sc->lane[i]));
+		sc->lane[i].ctrl_rate = KSERDES_FULL_RATE;
+	}
+}
+
+static void kserdes_set_pcie_defaults(struct kserdes_config *sc)
+{
+	int i;
+
+	sc->link_rate		= KSERDES_LINK_RATE_5G;
+	sc->lanes		= 2;
+	sc->rx_force_enable	= false;
+
+	for_each_lane(sc, i)
+		memset(&sc->lane[i], 0, sizeof(sc->lane[i]));
+}
+
+static void kserdes_set_defaults(struct kserdes_config *sc,
+				 enum kserdes_phy_type phy_type)
+{
+	switch (phy_type) {
+	case KSERDES_PHY_SGMII:
+		kserdes_set_sgmii_defaults(sc);
+		break;
+	case KSERDES_PHY_XGE:
+		kserdes_set_xge_defaults(sc);
+		break;
+	case KSERDES_PHY_PCIE:
+		kserdes_set_pcie_defaults(sc);
+		break;
+	default:
+		break;
+	}
+}
+
+static const struct of_device_id kserdes_of_match[] = {
+	{ .compatible	= "ti,keystone-serdes-gbe",
+	  .data		= &kserdes_devtype[KSERDES_PHY_SGMII], },
+
+	{ .compatible	= "ti,keystone-serdes-xgbe",
+	  .data		= &kserdes_devtype[KSERDES_PHY_XGE], },
+
+	{ .compatible	= "ti,keystone-serdes-pcie",
+	  .data		= &kserdes_devtype[KSERDES_PHY_PCIE], },
+
+	{ },
+};
+MODULE_DEVICE_TABLE(of, kserdes_of_match);
+
+static int kserdes_phy_enable_rx(struct phy *phy)
+{
+	struct kserdes_phy *ks_phy = phy_get_drvdata(phy);
+	struct kserdes_dev *sd = dev_get_drvdata(phy->dev.parent);
+	struct kserdes_config *sc = &sd->sc;
+	struct kserdes_ofs *sofs = &sc->sofs;
+	struct kserdes_dlev_out dlevo;
+	u32 lanes_up_map = 0;
+	u32 i = ks_phy->lane;
+	int ret;
+
+	ret = kserdes_enable_lane_rx(sc, i, &sofs->lane_ofs[i],
+				     &dlevo.lane_dlev_out[i]);
+
+	kserdes_clear_wait_after(sc, BIT(i));
+
+	if (sc->phy_type == KSERDES_PHY_XGE) {
+		_kserdes_enable_xgmii_port(sc->peripheral_regmap, i);
+		kserdes_wait_link_up(sc, BIT(i), &lanes_up_map);
+	}
+
+	return 0;
+}
+
+static int kserdes_phy_reset(struct phy *phy)
+{
+	struct kserdes_phy *ks_phy = phy_get_drvdata(phy);
+	struct kserdes_dev *sd = dev_get_drvdata(phy->dev.parent);
+	struct kserdes_config *sc = &sd->sc;
+	struct kserdes_ofs *sofs = &sc->sofs;
+	struct kserdes_dlev_out dlevo;
+	u32 i = ks_phy->lane;
+	u32 lanes_up_map = 0;
+	int ret = 0;
+
+	if (sc->phy_type == KSERDES_PHY_PCIE)
+		return ret;
+
+	if (sc->phy_type == KSERDES_PHY_XGE)
+		ret = xge_kserdes_recover_lane_rx(sc, i, &sofs->lane_ofs[i],
+			&dlevo.lane_dlev_out[i]);
+	else
+		ret = gbe_kserdes_recover_lane_rx(sc, i, &sofs->lane_ofs[i],
+			&dlevo.lane_dlev_out[i]);
+
+	kserdes_clear_wait_after(sc, BIT(i));
+
+	if (sc->phy_type == KSERDES_PHY_XGE)
+		_kserdes_enable_xgmii_port(sc->peripheral_regmap, i);
+
+	kserdes_wait_link_up(sc, BIT(i), &lanes_up_map);
+
+	dev_dbg(sd->dev, "phy reset: recover lane %u rx\n", i);
+
+	return ret;
+}
+
+static struct phy_ops kserdes_phy_ops = {
+	.init		= kserdes_phy_enable_rx,
+	.reset		= kserdes_phy_reset,
+	.owner		= THIS_MODULE,
+};
+
+static int kserdes_of_parse(struct platform_device *pdev,
+			    struct kserdes_dev *sd,
+			    struct device_node *np)
+{
+	const struct of_device_id *of_id;
+	struct kserdes_config *sc = &sd->sc;
+	struct device_node *child;
+	struct device *dev = sd->dev;
+	struct resource res;
+	void __iomem *regs;
+	int ret, lane = 0;
+
+	ret = of_address_to_resource(np, SERDES_REG_INDEX, &res);
+	if (ret) {
+		dev_err(dev, "Can't xlate serdes reg addr of node(%s)\n",
+			np->name);
+		return ret;
+	}
+
+	regs = devm_ioremap_resource(dev, &res);
+	if (IS_ERR(regs)) {
+		dev_err(dev, "Failed to map serdes register base\n");
+		return PTR_ERR(regs);
+	}
+	sc->regs = regs;
+
+	of_id = of_match_device(kserdes_of_match, dev);
+	if (!of_id) {
+		dev_err(dev, "unknown phy type\n");
+		return -EINVAL;
+	}
+	pdev->id_entry = of_id->data;
+	sc->phy_type = pdev->id_entry->driver_data;
+
+	sc->dev = dev;
+
+	kserdes_set_defaults(sc, sc->phy_type);
+
+	if (sc->phy_type == KSERDES_PHY_XGE) {
+		sc->peripheral_regmap =
+			syscon_regmap_lookup_by_phandle(np,
+							"syscon-peripheral");
+
+		if (IS_ERR(sc->peripheral_regmap)) {
+			dev_err(sc->dev,
+				"peripheral regmap lookup failed: %ld\n",
+				PTR_ERR(sc->peripheral_regmap));
+			return PTR_ERR(sc->peripheral_regmap);
+		}
+
+		sc->pcsr_regmap =
+			syscon_regmap_lookup_by_phandle(np, "syscon-link");
+
+		if (IS_ERR(sc->pcsr_regmap)) {
+			dev_err(sc->dev, "link regmap lookup failed: %ld\n",
+				PTR_ERR(sc->pcsr_regmap));
+			return PTR_ERR(sc->pcsr_regmap);
+		}
+	}
+
+	if (of_property_read_u32(np, "link-rate-kbps", &sc->link_rate)) {
+		dev_info(dev, "use default link-rate-kbps: %u\n",
+			 sc->link_rate);
+	}
+
+	if (of_property_read_u32(np, "num-lanes", &sc->lanes))
+		dev_info(dev, "use default num-lanes %d\n", sc->lanes);
+
+	if (sc->lanes > KSERDES_MAX_LANES) {
+		sc->lanes = KSERDES_MAX_LANES;
+		dev_info(dev, "use max allowed lanes %d\n", sc->lanes);
+	}
+
+	if (of_property_read_bool(np, "rx-force-enable"))
+		sc->rx_force_enable = true;
+	else
+		sc->rx_force_enable = false;
+
+	sd->nphys = sc->lanes;
+
+	for_each_child_of_node(np, child) {
+		struct kserdes_phy *ks_phy;
+		struct phy *phy;
+
+		lane = kserdes_of_parse_lane(dev, child, sc);
+		if (lane < 0) {
+			ret = lane;
+			goto err_child;
+		}
+
+		if (!sc->lane[lane].enable)
+			continue;
+
+		ks_phy = devm_kzalloc(dev, sizeof(*ks_phy), GFP_KERNEL);
+		if (!ks_phy) {
+			ret = -ENOMEM;
+			goto err_child;
+		}
+
+		sd->phys[lane] = ks_phy;
+
+		phy = devm_phy_create(dev, child, &kserdes_phy_ops);
+		if (IS_ERR(phy)) {
+			dev_err(dev, "failed to create PHY\n");
+			ret = PTR_ERR(phy);
+			goto err_child;
+		}
+
+		sd->phys[lane]->phy = phy;
+		sd->phys[lane]->lane = lane;
+		phy_set_drvdata(phy, sd->phys[lane]);
+	}
+
+	return 0;
+err_child:
+	of_node_put(child);
+	return ret;
+}
+
+static int kserdes_provider_lanes_enable_common(struct kserdes_config *sc)
+{
+	struct kserdes_ofs *sofs = &sc->sofs;
+	unsigned long lanes_needed = 0;
+	int ret, i;
+
+	if (sc->firmware)
+		return 0;
+
+	for_each_enable_lane(sc, i)
+		lanes_needed |= BIT(i);
+
+	if (sc->phy_type == KSERDES_PHY_PCIE) {
+		kserdes_pcie_lanes_enable(sc);
+		return lanes_needed;
+	}
+
+	ret = kserdes_lanes_enable_common(sc, sofs);
+	if (ret)
+		dev_err(sc->dev, "provider lanes enable: FAILED %d\n", ret);
+
+	return ret;
+}
+
+static int kserdes_provider_init(struct kserdes_dev *sd)
+{
+	struct kserdes_config *sc = &sd->sc;
+	struct device *dev = sd->dev;
+	int ret;
+
+	switch (sc->phy_type) {
+	case KSERDES_PHY_SGMII:
+		ret = kserdes_sgmii_init(sc);
+		break;
+	case KSERDES_PHY_XGE:
+		ret = kserdes_xge_init(sc);
+		break;
+	case KSERDES_PHY_PCIE:
+		ret = kserdes_pcie_init(sc);
+		break;
+	default:
+		ret = -EINVAL;
+	}
+
+	if (ret < 0) {
+		dev_err(dev, "serdes procider init failed %d\n", ret);
+		return ret;
+	}
+
+	return kserdes_provider_lanes_enable_common(sc);
+}
+
+static struct phy *kserdes_xlate(struct device *dev,
+				 struct of_phandle_args *args)
+{
+	struct kserdes_dev *sd = dev_get_drvdata(dev);
+	struct phy *phy = NULL;
+	struct device_node *phynode = args->np;
+	int i;
+
+	if (args->args_count) {
+		dev_err(dev, "invalid #cell in PHY property\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	for (i = 0; i < sd->nphys; i++) {
+		if (sd->phys[i] &&
+		    (phynode == sd->phys[i]->phy->dev.of_node)) {
+			phy = sd->phys[i]->phy;
+			break;
+		}
+	}
+
+	return phy;
+}
+
+static int kserdes_probe(struct platform_device *pdev)
+{
+	struct phy_provider *phy_provider;
+	struct kserdes_dev *sd;
+	struct device_node *np = pdev->dev.of_node;
+	struct device *dev = &pdev->dev;
+	int ret;
+
+	sd = devm_kzalloc(dev, sizeof(*sd), GFP_KERNEL);
+	if (!sd)
+		return -ENOMEM;
+
+	pm_runtime_enable(&pdev->dev);
+	ret = pm_runtime_get_sync(&pdev->dev);
+	if (ret < 0) {
+		dev_err(dev, "Failed to enable SerDes power-domain\n");
+		pm_runtime_set_suspended(dev);
+		pm_runtime_put_noidle(dev);
+		return ret;
+	}
+
+	sd->dev = dev;
+	dev_set_drvdata(dev, sd);
+
+	ret = kserdes_of_parse(pdev, sd, np);
+	if (ret)
+		goto error;
+
+	phy_provider = devm_of_phy_provider_register(sd->dev, kserdes_xlate);
+	if (IS_ERR(phy_provider)) {
+		ret = PTR_ERR_OR_ZERO(phy_provider);
+		goto error;
+	}
+
+	kserdes_provider_init(sd);
+
+	dev_vdbg(&pdev->dev, "probed");
+	return 0;
+
+error:
+	pm_runtime_put_sync(dev);
+	pm_runtime_disable(dev);
+	return ret;
+}
+
+static struct platform_driver kserdes_driver = {
+	.probe	= kserdes_probe,
+	.driver = {
+		.of_match_table	= kserdes_of_match,
+		.name  = "ti,keystone-serdes",
+	}
+};
+module_platform_driver(kserdes_driver);
+
+MODULE_AUTHOR("WingMan Kwok <w-kwok2@ti.com>");
+MODULE_DESCRIPTION("TI Keystone SerDes driver");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/phy/ti/phy-omap-usb2.c b/drivers/phy/ti/phy-omap-usb2.c
--- a/drivers/phy/ti/phy-omap-usb2.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/phy/ti/phy-omap-usb2.c	2020-06-19 21:00:31.424227716 -0400
@@ -34,8 +34,16 @@
 #include <linux/of_platform.h>
 
 #define USB2PHY_DISCON_BYP_LATCH (1 << 31)
+#define USB2PHY_CHRG_DET 0x14
 #define USB2PHY_ANA_CONFIG1 0x4c
 
+#define USB2PHY_USE_CHG_DET_REG		BIT(29)
+#define USB2PHY_DIS_CHG_DET		BIT(28)
+
+#define AM654_USB2_OTG_PD		BIT(8)
+#define AM654_USB2_VBUS_DET_EN		BIT(5)
+#define AM654_USB2_VBUSVALID_DET_EN	BIT(4)
+
 /**
  * omap_usb2_set_comparator - links the comparator present in the sytem with
  *	this phy
@@ -135,9 +143,9 @@
 
 static int omap_usb2_disable_clocks(struct omap_usb *phy)
 {
-	clk_disable(phy->wkupclk);
+	clk_disable_unprepare(phy->wkupclk);
 	if (!IS_ERR(phy->optclk))
-		clk_disable(phy->optclk);
+		clk_disable_unprepare(phy->optclk);
 
 	return 0;
 }
@@ -146,14 +154,14 @@
 {
 	int ret;
 
-	ret = clk_enable(phy->wkupclk);
+	ret = clk_prepare_enable(phy->wkupclk);
 	if (ret < 0) {
 		dev_err(phy->dev, "Failed to enable wkupclk %d\n", ret);
 		goto err0;
 	}
 
 	if (!IS_ERR(phy->optclk)) {
-		ret = clk_enable(phy->optclk);
+		ret = clk_prepare_enable(phy->optclk);
 		if (ret < 0) {
 			dev_err(phy->dev, "Failed to enable optclk %d\n", ret);
 			goto err1;
@@ -190,6 +198,12 @@
 		omap_usb_writel(phy->phy_base, USB2PHY_ANA_CONFIG1, val);
 	}
 
+	if (phy->flags & OMAP_USB2_DISABLE_CHG_DET) {
+		val = omap_usb_readl(phy->phy_base, USB2PHY_CHRG_DET);
+		val |= USB2PHY_USE_CHG_DET_REG | USB2PHY_DIS_CHG_DET;
+		omap_usb_writel(phy->phy_base, USB2PHY_CHRG_DET, val);
+	}
+
 	return 0;
 }
 
@@ -245,6 +259,15 @@
 	.power_off = AM437X_USB2_PHY_PD | AM437X_USB2_OTG_PD,
 };
 
+static const struct usb_phy_data am654_usb2_data = {
+	.label = "am654_usb2",
+	.flags = OMAP_USB2_CALIBRATE_FALSE_DISCONNECT,
+	.mask = AM654_USB2_OTG_PD | AM654_USB2_VBUS_DET_EN |
+		AM654_USB2_VBUSVALID_DET_EN,
+	.power_on = AM654_USB2_VBUS_DET_EN | AM654_USB2_VBUSVALID_DET_EN,
+	.power_off = AM654_USB2_OTG_PD,
+};
+
 static const struct of_device_id omap_usb2_id_table[] = {
 	{
 		.compatible = "ti,omap-usb2",
@@ -266,6 +289,10 @@
 		.compatible = "ti,am437x-usb2",
 		.data = &am437x_usb2_data,
 	},
+	{
+		.compatible = "ti,am654-usb2",
+		.data = &am654_usb2_data,
+	},
 	{},
 };
 MODULE_DEVICE_TABLE(of, omap_usb2_id_table);
@@ -308,13 +335,25 @@
 	phy->power_on		= phy_data->power_on;
 	phy->power_off		= phy_data->power_off;
 
-	if (phy_data->flags & OMAP_USB2_CALIBRATE_FALSE_DISCONNECT) {
-		res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-		phy->phy_base = devm_ioremap_resource(&pdev->dev, res);
-		if (IS_ERR(phy->phy_base))
-			return PTR_ERR(phy->phy_base);
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	phy->phy_base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(phy->phy_base))
+		return PTR_ERR(phy->phy_base);
+
+	if (phy_data->flags & OMAP_USB2_CALIBRATE_FALSE_DISCONNECT)
 		phy->flags |= OMAP_USB2_CALIBRATE_FALSE_DISCONNECT;
-	}
+
+	/*
+	 * AM654x PG1.0 has a silicon bug that D+ is pulled high after
+	 * POR, which could cause enumeration failure with some USB hubs.
+	 * Disabling the USB2_PHY Charger Detect function will put D+
+	 * into the normal state.
+	 *
+	 * Using property "ti,dis-chg-det-quirk" in the DT usb2-phy node
+	 * to enable this workaround for AM654x PG1.0.
+	 */
+	if (device_property_read_bool(phy->dev, "ti,dis-chg-det-quirk"))
+		phy->flags |= OMAP_USB2_DISABLE_CHG_DET;
 
 	phy->syscon_phy_power = syscon_regmap_lookup_by_phandle(node,
 							"syscon-phy-power");
@@ -346,63 +385,72 @@
 		}
 	}
 
-	otg->set_host		= omap_usb_set_host;
-	otg->set_peripheral	= omap_usb_set_peripheral;
-	if (phy_data->flags & OMAP_USB2_HAS_SET_VBUS)
-		otg->set_vbus		= omap_usb_set_vbus;
-	if (phy_data->flags & OMAP_USB2_HAS_START_SRP)
-		otg->start_srp		= omap_usb_start_srp;
-	otg->usb_phy		= &phy->phy;
-
-	platform_set_drvdata(pdev, phy);
-	pm_runtime_enable(phy->dev);
-
-	generic_phy = devm_phy_create(phy->dev, NULL, &ops);
-	if (IS_ERR(generic_phy)) {
-		pm_runtime_disable(phy->dev);
-		return PTR_ERR(generic_phy);
-	}
-
-	phy_set_drvdata(generic_phy, phy);
-	omap_usb_power_off(generic_phy);
-
-	phy_provider = devm_of_phy_provider_register(phy->dev,
-			of_phy_simple_xlate);
-	if (IS_ERR(phy_provider)) {
-		pm_runtime_disable(phy->dev);
-		return PTR_ERR(phy_provider);
-	}
 
 	phy->wkupclk = devm_clk_get(phy->dev, "wkupclk");
 	if (IS_ERR(phy->wkupclk)) {
-		dev_warn(&pdev->dev, "unable to get wkupclk, trying old name\n");
+		if (PTR_ERR(phy->wkupclk) == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
+
+		dev_warn(&pdev->dev, "unable to get wkupclk %ld, trying old name\n",
+			 PTR_ERR(phy->wkupclk));
 		phy->wkupclk = devm_clk_get(phy->dev, "usb_phy_cm_clk32k");
+
 		if (IS_ERR(phy->wkupclk)) {
-			dev_err(&pdev->dev, "unable to get usb_phy_cm_clk32k\n");
-			pm_runtime_disable(phy->dev);
+			if (PTR_ERR(phy->wkupclk) != -EPROBE_DEFER)
+				dev_err(&pdev->dev, "unable to get usb_phy_cm_clk32k\n");
 			return PTR_ERR(phy->wkupclk);
 		} else {
 			dev_warn(&pdev->dev,
 				 "found usb_phy_cm_clk32k, please fix DTS\n");
 		}
 	}
-	clk_prepare(phy->wkupclk);
 
 	phy->optclk = devm_clk_get(phy->dev, "refclk");
 	if (IS_ERR(phy->optclk)) {
+		if (PTR_ERR(phy->optclk) == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
+
 		dev_dbg(&pdev->dev, "unable to get refclk, trying old name\n");
 		phy->optclk = devm_clk_get(phy->dev, "usb_otg_ss_refclk960m");
+
 		if (IS_ERR(phy->optclk)) {
-			dev_dbg(&pdev->dev,
-				"unable to get usb_otg_ss_refclk960m\n");
+			if (PTR_ERR(phy->optclk) != -EPROBE_DEFER) {
+				dev_dbg(&pdev->dev,
+					"unable to get usb_otg_ss_refclk960m\n");
+			}
 		} else {
 			dev_warn(&pdev->dev,
 				 "found usb_otg_ss_refclk960m, please fix DTS\n");
 		}
 	}
 
-	if (!IS_ERR(phy->optclk))
-		clk_prepare(phy->optclk);
+	otg->set_host = omap_usb_set_host;
+	otg->set_peripheral = omap_usb_set_peripheral;
+	if (phy_data->flags & OMAP_USB2_HAS_SET_VBUS)
+		otg->set_vbus = omap_usb_set_vbus;
+	if (phy_data->flags & OMAP_USB2_HAS_START_SRP)
+		otg->start_srp = omap_usb_start_srp;
+	otg->usb_phy = &phy->phy;
+
+	platform_set_drvdata(pdev, phy);
+	pm_runtime_enable(phy->dev);
+
+	generic_phy = devm_phy_create(phy->dev, NULL, &ops);
+	if (IS_ERR(generic_phy)) {
+		pm_runtime_disable(phy->dev);
+		return PTR_ERR(generic_phy);
+	}
+
+	phy_set_drvdata(generic_phy, phy);
+	omap_usb_power_off(generic_phy);
+
+	phy_provider = devm_of_phy_provider_register(phy->dev,
+						     of_phy_simple_xlate);
+	if (IS_ERR(phy_provider)) {
+		pm_runtime_disable(phy->dev);
+		return PTR_ERR(phy_provider);
+	}
+
 
 	usb_add_phy_dev(&phy->phy);
 
@@ -413,9 +461,6 @@
 {
 	struct omap_usb	*phy = platform_get_drvdata(pdev);
 
-	clk_unprepare(phy->wkupclk);
-	if (!IS_ERR(phy->optclk))
-		clk_unprepare(phy->optclk);
 	usb_remove_phy(&phy->phy);
 	pm_runtime_disable(phy->dev);
 
diff -Naur --no-dereference a/drivers/phy/ti/phy-ti-pipe3.c b/drivers/phy/ti/phy-ti-pipe3.c
--- a/drivers/phy/ti/phy-ti-pipe3.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/phy/ti/phy-ti-pipe3.c	2020-06-19 21:00:31.424227716 -0400
@@ -56,51 +56,73 @@
 
 #define SATA_PLL_SOFT_RESET	BIT(18)
 
-#define PIPE3_PHY_PWRCTL_CLK_CMD_MASK	0x003FC000
+#define PIPE3_PHY_PWRCTL_CLK_CMD_MASK	GENMASK(21, 14)
 #define PIPE3_PHY_PWRCTL_CLK_CMD_SHIFT	14
 
-#define PIPE3_PHY_PWRCTL_CLK_FREQ_MASK	0xFFC00000
+#define PIPE3_PHY_PWRCTL_CLK_FREQ_MASK	GENMASK(31, 22)
 #define PIPE3_PHY_PWRCTL_CLK_FREQ_SHIFT	22
 
-#define PIPE3_PHY_TX_RX_POWERON		0x3
-#define PIPE3_PHY_TX_RX_POWEROFF	0x0
+#define PIPE3_PHY_RX_POWERON       (0x1 << PIPE3_PHY_PWRCTL_CLK_CMD_SHIFT)
+#define PIPE3_PHY_TX_POWERON       (0x2 << PIPE3_PHY_PWRCTL_CLK_CMD_SHIFT)
 
 #define PCIE_PCS_MASK			0xFF0000
 #define PCIE_PCS_DELAY_COUNT_SHIFT	0x10
 
-#define PCIEPHYRX_ANA_PROGRAMMABILITY	0x0000000C
+#define PIPE3_PHY_RX_ANA_PROGRAMMABILITY	0x0000000C
 #define INTERFACE_MASK			GENMASK(31, 27)
 #define INTERFACE_SHIFT			27
+#define INTERFACE_MODE_USBSS		BIT(4)
+#define INTERFACE_MODE_SATA_1P5		BIT(3)
+#define INTERFACE_MODE_SATA_3P0		BIT(2)
+#define INTERFACE_MODE_PCIE		BIT(0)
+
 #define LOSD_MASK			GENMASK(17, 14)
 #define LOSD_SHIFT			14
 #define MEM_PLLDIV			GENMASK(6, 5)
 
-#define PCIEPHYRX_TRIM			0x0000001C
-#define MEM_DLL_TRIM_SEL		GENMASK(31, 30)
+#define PIPE3_PHY_RX_TRIM		0x0000001C
+#define MEM_DLL_TRIM_SEL_MASK		GENMASK(31, 30)
 #define MEM_DLL_TRIM_SHIFT		30
 
-#define PCIEPHYRX_DLL			0x00000024
-#define MEM_DLL_PHINT_RATE		GENMASK(31, 30)
-
-#define PCIEPHYRX_DIGITAL_MODES		0x00000028
+#define PIPE3_PHY_RX_DLL		0x00000024
+#define MEM_DLL_PHINT_RATE_MASK		GENMASK(31, 30)
+#define MEM_DLL_PHINT_RATE_SHIFT	30
+
+#define PIPE3_PHY_RX_DIGITAL_MODES		0x00000028
+#define MEM_HS_RATE_MASK		GENMASK(28, 27)
+#define MEM_HS_RATE_SHIFT		27
+#define MEM_OVRD_HS_RATE		BIT(26)
+#define MEM_OVRD_HS_RATE_SHIFT		26
 #define MEM_CDR_FASTLOCK		BIT(23)
-#define MEM_CDR_LBW			GENMASK(22, 21)
-#define MEM_CDR_STEPCNT			GENMASK(20, 19)
+#define MEM_CDR_FASTLOCK_SHIFT		23
+#define MEM_CDR_LBW_MASK		GENMASK(22, 21)
+#define MEM_CDR_LBW_SHIFT		21
+#define MEM_CDR_STEPCNT_MASK		GENMASK(20, 19)
+#define MEM_CDR_STEPCNT_SHIFT		19
 #define MEM_CDR_STL_MASK		GENMASK(18, 16)
 #define MEM_CDR_STL_SHIFT		16
 #define MEM_CDR_THR_MASK		GENMASK(15, 13)
 #define MEM_CDR_THR_SHIFT		13
 #define MEM_CDR_THR_MODE		BIT(12)
-#define MEM_CDR_CDR_2NDO_SDM_MODE	BIT(11)
-#define MEM_OVRD_HS_RATE		BIT(26)
-
-#define PCIEPHYRX_EQUALIZER		0x00000038
-#define MEM_EQLEV			GENMASK(31, 16)
-#define MEM_EQFTC			GENMASK(15, 11)
-#define MEM_EQCTL			GENMASK(10, 7)
+#define MEM_CDR_THR_MODE_SHIFT		12
+#define MEM_CDR_2NDO_SDM_MODE		BIT(11)
+#define MEM_CDR_2NDO_SDM_MODE_SHIFT	11
+
+#define PIPE3_PHY_RX_EQUALIZER		0x00000038
+#define MEM_EQLEV_MASK			GENMASK(31, 16)
+#define MEM_EQLEV_SHIFT			16
+#define MEM_EQFTC_MASK			GENMASK(15, 11)
+#define MEM_EQFTC_SHIFT			11
+#define MEM_EQCTL_MASK			GENMASK(10, 7)
 #define MEM_EQCTL_SHIFT			7
 #define MEM_OVRD_EQLEV			BIT(2)
+#define MEM_OVRD_EQLEV_SHIFT		2
 #define MEM_OVRD_EQFTC			BIT(1)
+#define MEM_OVRD_EQFTC_SHIFT		1
+
+#define SATA_PHY_RX_IO_AND_A2D_OVERRIDES	0x44
+#define MEM_CDR_LOS_SOURCE_MASK		GENMASK(10, 9)
+#define MEM_CDR_LOS_SOURCE_SHIFT	9
 
 /*
  * This is an Empirical value that works, need to confirm the actual
@@ -110,6 +132,10 @@
 #define PLL_IDLE_TIME	100	/* in milliseconds */
 #define PLL_LOCK_TIME	100	/* in milliseconds */
 
+enum pipe3_mode { PIPE3_MODE_PCIE = 1,
+		  PIPE3_MODE_SATA,
+		  PIPE3_MODE_USBSS };
+
 struct pipe3_dpll_params {
 	u16	m;
 	u8	n;
@@ -123,6 +149,27 @@
 	struct pipe3_dpll_params params;
 };
 
+struct pipe3_settings {
+	u8 ana_interface;
+	u8 ana_losd;
+	u8 dig_fastlock;
+	u8 dig_lbw;
+	u8 dig_stepcnt;
+	u8 dig_stl;
+	u8 dig_thr;
+	u8 dig_thr_mode;
+	u8 dig_2ndo_sdm_mode;
+	u8 dig_hs_rate;
+	u8 dig_ovrd_hs_rate;
+	u8 dll_trim_sel;
+	u8 dll_phint_rate;
+	u8 eq_lev;
+	u8 eq_ftc;
+	u8 eq_ctl;
+	u8 eq_ovrd_lev;
+	u8 eq_ovrd_ftc;
+};
+
 struct ti_pipe3 {
 	void __iomem		*pll_ctrl_base;
 	void __iomem		*phy_rx;
@@ -141,6 +188,8 @@
 	unsigned int		power_reg; /* power reg. index within syscon */
 	unsigned int		pcie_pcs_reg; /* pcs reg. index in syscon */
 	bool			sata_refclk_enabled;
+	enum pipe3_mode		mode;
+	struct pipe3_settings	settings;
 };
 
 static struct pipe3_dpll_map dpll_map_usb[] = {
@@ -163,6 +212,89 @@
 	{ },					/* Terminator */
 };
 
+struct pipe3_data {
+	enum pipe3_mode mode;
+	struct pipe3_dpll_map *dpll_map;
+	struct pipe3_settings settings;
+};
+
+static struct pipe3_data data_usb = {
+	.mode = PIPE3_MODE_USBSS,
+	.dpll_map = dpll_map_usb,
+	.settings = {
+	/* DRA75x TRM Table 26-17. Preferred USB3_PHY_RX SCP Register Settings */
+		.ana_interface = INTERFACE_MODE_USBSS,
+		.ana_losd = 0xa,
+		.dig_fastlock = 1,
+		.dig_lbw = 3,
+		.dig_stepcnt = 0,
+		.dig_stl = 0x3,
+		.dig_thr = 1,
+		.dig_thr_mode = 1,
+		.dig_2ndo_sdm_mode = 0,
+		.dig_hs_rate = 0,
+		.dig_ovrd_hs_rate = 1,
+		.dll_trim_sel = 0x2,
+		.dll_phint_rate = 0x3,
+		.eq_lev = 0,
+		.eq_ftc = 0,
+		.eq_ctl = 0x9,
+		.eq_ovrd_lev = 0,
+		.eq_ovrd_ftc = 0,
+	},
+};
+
+static struct pipe3_data data_sata = {
+	.mode = PIPE3_MODE_SATA,
+	.dpll_map = dpll_map_sata,
+	.settings = {
+	/* DRA75x TRM Table 26-9. Preferred SATA_PHY_RX SCP Register Settings */
+		.ana_interface = INTERFACE_MODE_SATA_3P0,
+		.ana_losd = 0x5,
+		.dig_fastlock = 1,
+		.dig_lbw = 3,
+		.dig_stepcnt = 0,
+		.dig_stl = 0x3,
+		.dig_thr = 1,
+		.dig_thr_mode = 1,
+		.dig_2ndo_sdm_mode = 0,
+		.dig_hs_rate = 0,	/* Not in TRM preferred settings */
+		.dig_ovrd_hs_rate = 0,	/* Not in TRM preferred settings */
+		.dll_trim_sel = 0x1,
+		.dll_phint_rate = 0x2,	/* for 1.5 GHz DPLL clock */
+		.eq_lev = 0,
+		.eq_ftc = 0x1f,
+		.eq_ctl = 0,
+		.eq_ovrd_lev = 1,
+		.eq_ovrd_ftc = 1,
+	},
+};
+
+static struct pipe3_data data_pcie = {
+	.mode = PIPE3_MODE_PCIE,
+	.settings = {
+	/* DRA75x TRM Table 26-62. Preferred PCIe_PHY_RX SCP Register Settings */
+		.ana_interface = INTERFACE_MODE_PCIE,
+		.ana_losd = 0xa,
+		.dig_fastlock = 1,
+		.dig_lbw = 3,
+		.dig_stepcnt = 0,
+		.dig_stl = 0x3,
+		.dig_thr = 1,
+		.dig_thr_mode = 1,
+		.dig_2ndo_sdm_mode = 0,
+		.dig_hs_rate = 0,
+		.dig_ovrd_hs_rate = 0,
+		.dll_trim_sel = 0x2,
+		.dll_phint_rate = 0x3,
+		.eq_lev = 0,
+		.eq_ftc = 0x1f,
+		.eq_ctl = 1,
+		.eq_ovrd_lev = 0,
+		.eq_ovrd_ftc = 0,
+	},
+};
+
 static inline u32 ti_pipe3_readl(void __iomem *addr, unsigned offset)
 {
 	return __raw_readl(addr + offset);
@@ -196,7 +328,6 @@
 
 static int ti_pipe3_power_off(struct phy *x)
 {
-	u32 val;
 	int ret;
 	struct ti_pipe3 *phy = phy_get_drvdata(x);
 
@@ -205,13 +336,13 @@
 		return 0;
 	}
 
-	val = PIPE3_PHY_TX_RX_POWEROFF << PIPE3_PHY_PWRCTL_CLK_CMD_SHIFT;
-
 	ret = regmap_update_bits(phy->phy_power_syscon, phy->power_reg,
-				 PIPE3_PHY_PWRCTL_CLK_CMD_MASK, val);
+				 PIPE3_PHY_PWRCTL_CLK_CMD_MASK, 0);
 	return ret;
 }
 
+static void ti_pipe3_calibrate(struct ti_pipe3 *phy);
+
 static int ti_pipe3_power_on(struct phy *x)
 {
 	u32 val;
@@ -219,6 +350,7 @@
 	int ret;
 	unsigned long rate;
 	struct ti_pipe3 *phy = phy_get_drvdata(x);
+	bool rx_pending = false;
 
 	if (!phy->phy_power_syscon) {
 		omap_control_phy_power(phy->control_dev, 1);
@@ -231,14 +363,35 @@
 		return -EINVAL;
 	}
 	rate = rate / 1000000;
-	mask = OMAP_CTRL_PIPE3_PHY_PWRCTL_CLK_CMD_MASK |
-		  OMAP_CTRL_PIPE3_PHY_PWRCTL_CLK_FREQ_MASK;
-	val = PIPE3_PHY_TX_RX_POWERON << PIPE3_PHY_PWRCTL_CLK_CMD_SHIFT;
-	val |= rate << OMAP_CTRL_PIPE3_PHY_PWRCTL_CLK_FREQ_SHIFT;
-
+	mask = OMAP_CTRL_PIPE3_PHY_PWRCTL_CLK_FREQ_MASK;
+	val = rate << OMAP_CTRL_PIPE3_PHY_PWRCTL_CLK_FREQ_SHIFT;
 	ret = regmap_update_bits(phy->phy_power_syscon, phy->power_reg,
 				 mask, val);
-	return ret;
+	/*
+	 * For PCIe, TX and RX must be powered on simultaneously.
+	 * For USB and SATA, TX must be powered on before RX
+	 */
+	mask = OMAP_CTRL_PIPE3_PHY_PWRCTL_CLK_CMD_MASK;
+	if (phy->mode == PIPE3_MODE_SATA || phy->mode == PIPE3_MODE_USBSS) {
+		val = PIPE3_PHY_TX_POWERON;
+		rx_pending = true;
+	} else {
+		val = PIPE3_PHY_TX_POWERON | PIPE3_PHY_RX_POWERON;
+	}
+
+	regmap_update_bits(phy->phy_power_syscon, phy->power_reg,
+			   mask, val);
+
+	if (rx_pending) {
+		val = PIPE3_PHY_TX_POWERON | PIPE3_PHY_RX_POWERON;
+		regmap_update_bits(phy->phy_power_syscon, phy->power_reg,
+				   mask, val);
+	}
+
+	if (phy->mode == PIPE3_MODE_PCIE)
+		ti_pipe3_calibrate(phy);
+
+	return 0;
 }
 
 static int ti_pipe3_dpll_wait_lock(struct ti_pipe3 *phy)
@@ -300,32 +453,55 @@
 static void ti_pipe3_calibrate(struct ti_pipe3 *phy)
 {
 	u32 val;
+	struct pipe3_settings *s = &phy->settings;
 
-	val = ti_pipe3_readl(phy->phy_rx, PCIEPHYRX_ANA_PROGRAMMABILITY);
+	val = ti_pipe3_readl(phy->phy_rx, PIPE3_PHY_RX_ANA_PROGRAMMABILITY);
 	val &= ~(INTERFACE_MASK | LOSD_MASK | MEM_PLLDIV);
-	val |= (0x1 << INTERFACE_SHIFT | 0xA << LOSD_SHIFT);
-	ti_pipe3_writel(phy->phy_rx, PCIEPHYRX_ANA_PROGRAMMABILITY, val);
+	val |= (s->ana_interface << INTERFACE_SHIFT | s->ana_losd << LOSD_SHIFT);
+	ti_pipe3_writel(phy->phy_rx, PIPE3_PHY_RX_ANA_PROGRAMMABILITY, val);
 
-	val = ti_pipe3_readl(phy->phy_rx, PCIEPHYRX_DIGITAL_MODES);
-	val &= ~(MEM_CDR_STEPCNT | MEM_CDR_STL_MASK | MEM_CDR_THR_MASK |
-		 MEM_CDR_CDR_2NDO_SDM_MODE | MEM_OVRD_HS_RATE);
-	val |= (MEM_CDR_FASTLOCK | MEM_CDR_LBW | 0x3 << MEM_CDR_STL_SHIFT |
-		0x1 << MEM_CDR_THR_SHIFT | MEM_CDR_THR_MODE);
-	ti_pipe3_writel(phy->phy_rx, PCIEPHYRX_DIGITAL_MODES, val);
-
-	val = ti_pipe3_readl(phy->phy_rx, PCIEPHYRX_TRIM);
-	val &= ~MEM_DLL_TRIM_SEL;
-	val |= 0x2 << MEM_DLL_TRIM_SHIFT;
-	ti_pipe3_writel(phy->phy_rx, PCIEPHYRX_TRIM, val);
-
-	val = ti_pipe3_readl(phy->phy_rx, PCIEPHYRX_DLL);
-	val |= MEM_DLL_PHINT_RATE;
-	ti_pipe3_writel(phy->phy_rx, PCIEPHYRX_DLL, val);
-
-	val = ti_pipe3_readl(phy->phy_rx, PCIEPHYRX_EQUALIZER);
-	val &= ~(MEM_EQLEV | MEM_EQCTL | MEM_OVRD_EQLEV | MEM_OVRD_EQFTC);
-	val |= MEM_EQFTC | 0x1 << MEM_EQCTL_SHIFT;
-	ti_pipe3_writel(phy->phy_rx, PCIEPHYRX_EQUALIZER, val);
+	val = ti_pipe3_readl(phy->phy_rx, PIPE3_PHY_RX_DIGITAL_MODES);
+	val &= ~(MEM_HS_RATE_MASK | MEM_OVRD_HS_RATE | MEM_CDR_FASTLOCK |
+		 MEM_CDR_LBW_MASK | MEM_CDR_STEPCNT_MASK | MEM_CDR_STL_MASK |
+		 MEM_CDR_THR_MASK | MEM_CDR_THR_MODE | MEM_CDR_2NDO_SDM_MODE);
+	val |= s->dig_hs_rate << MEM_HS_RATE_SHIFT |
+		s->dig_ovrd_hs_rate << MEM_OVRD_HS_RATE_SHIFT |
+		s->dig_fastlock << MEM_CDR_FASTLOCK_SHIFT |
+		s->dig_lbw << MEM_CDR_LBW_SHIFT |
+		s->dig_stepcnt << MEM_CDR_STEPCNT_SHIFT |
+		s->dig_stl << MEM_CDR_STL_SHIFT |
+		s->dig_thr << MEM_CDR_THR_SHIFT |
+		s->dig_thr_mode << MEM_CDR_THR_MODE_SHIFT |
+		s->dig_2ndo_sdm_mode << MEM_CDR_2NDO_SDM_MODE_SHIFT;
+	ti_pipe3_writel(phy->phy_rx, PIPE3_PHY_RX_DIGITAL_MODES, val);
+
+	val = ti_pipe3_readl(phy->phy_rx, PIPE3_PHY_RX_TRIM);
+	val &= ~MEM_DLL_TRIM_SEL_MASK;
+	val |= s->dll_trim_sel << MEM_DLL_TRIM_SHIFT;
+	ti_pipe3_writel(phy->phy_rx, PIPE3_PHY_RX_TRIM, val);
+
+	val = ti_pipe3_readl(phy->phy_rx, PIPE3_PHY_RX_DLL);
+	val &= ~MEM_DLL_PHINT_RATE_MASK;
+	val |= s->dll_phint_rate << MEM_DLL_PHINT_RATE_SHIFT;
+	ti_pipe3_writel(phy->phy_rx, PIPE3_PHY_RX_DLL, val);
+
+	val = ti_pipe3_readl(phy->phy_rx, PIPE3_PHY_RX_EQUALIZER);
+	val &= ~(MEM_EQLEV_MASK | MEM_EQFTC_MASK | MEM_EQCTL_MASK |
+		 MEM_OVRD_EQLEV | MEM_OVRD_EQFTC);
+	val |= s->eq_lev << MEM_EQLEV_SHIFT |
+		s->eq_ftc << MEM_EQFTC_SHIFT |
+		s->eq_ctl << MEM_EQCTL_SHIFT |
+		s->eq_ovrd_lev << MEM_OVRD_EQLEV_SHIFT |
+		s->eq_ovrd_ftc << MEM_OVRD_EQFTC_SHIFT;
+	ti_pipe3_writel(phy->phy_rx, PIPE3_PHY_RX_EQUALIZER, val);
+
+	if (phy->mode == PIPE3_MODE_SATA) {
+		val = ti_pipe3_readl(phy->phy_rx,
+				     SATA_PHY_RX_IO_AND_A2D_OVERRIDES);
+		val &= ~MEM_CDR_LOS_SOURCE_MASK;
+		ti_pipe3_writel(phy->phy_rx, SATA_PHY_RX_IO_AND_A2D_OVERRIDES,
+				val);
+	}
 }
 
 static int ti_pipe3_init(struct phy *x)
@@ -340,7 +516,7 @@
 	 * as recommended in AM572x TRM SPRUHZ6, section 18.5.2.2, table
 	 * 18-1804.
 	 */
-	if (of_device_is_compatible(phy->dev->of_node, "ti,phy-pipe3-pcie")) {
+	if (phy->mode == PIPE3_MODE_PCIE) {
 		if (!phy->pcs_syscon) {
 			omap_control_pcie_pcs(phy->control_dev, 0x96);
 			return 0;
@@ -349,12 +525,7 @@
 		val = 0x96 << OMAP_CTRL_PCIE_PCS_DELAY_COUNT_SHIFT;
 		ret = regmap_update_bits(phy->pcs_syscon, phy->pcie_pcs_reg,
 					 PCIE_PCS_MASK, val);
-		if (ret)
-			return ret;
-
-		ti_pipe3_calibrate(phy);
-
-		return 0;
+		return ret;
 	}
 
 	/* Bring it out of IDLE if it is IDLE */
@@ -367,8 +538,7 @@
 
 	/* SATA has issues if re-programmed when locked */
 	val = ti_pipe3_readl(phy->pll_ctrl_base, PLL_STATUS);
-	if ((val & PLL_LOCK) && of_device_is_compatible(phy->dev->of_node,
-							"ti,phy-pipe3-sata"))
+	if ((val & PLL_LOCK) && phy->mode == PIPE3_MODE_SATA)
 		return ret;
 
 	/* Program the DPLL */
@@ -378,6 +548,8 @@
 		return -EINVAL;
 	}
 
+	ti_pipe3_calibrate(phy);
+
 	return ret;
 }
 
@@ -390,12 +562,11 @@
 	/* If dpll_reset_syscon is not present we wont power down SATA DPLL
 	 * due to Errata i783
 	 */
-	if (of_device_is_compatible(phy->dev->of_node, "ti,phy-pipe3-sata") &&
-	    !phy->dpll_reset_syscon)
+	if (phy->mode == PIPE3_MODE_SATA && !phy->dpll_reset_syscon)
 		return 0;
 
 	/* PCIe doesn't have internal DPLL */
-	if (!of_device_is_compatible(phy->dev->of_node, "ti,phy-pipe3-pcie")) {
+	if (phy->mode != PIPE3_MODE_PCIE) {
 		/* Put DPLL in IDLE mode */
 		val = ti_pipe3_readl(phy->pll_ctrl_base, PLL_CONFIGURATION2);
 		val |= PLL_IDLE;
@@ -418,7 +589,7 @@
 	}
 
 	/* i783: SATA needs control bit toggle after PLL unlock */
-	if (of_device_is_compatible(phy->dev->of_node, "ti,phy-pipe3-sata")) {
+	if (phy->mode == PIPE3_MODE_SATA) {
 		regmap_update_bits(phy->dpll_reset_syscon, phy->dpll_reset_reg,
 				   SATA_PLL_SOFT_RESET, SATA_PLL_SOFT_RESET);
 		regmap_update_bits(phy->dpll_reset_syscon, phy->dpll_reset_reg,
@@ -443,7 +614,6 @@
 {
 	struct clk *clk;
 	struct device *dev = phy->dev;
-	struct device_node *node = dev->of_node;
 
 	phy->refclk = devm_clk_get(dev, "refclk");
 	if (IS_ERR(phy->refclk)) {
@@ -451,11 +621,11 @@
 		/* older DTBs have missing refclk in SATA PHY
 		 * so don't bail out in case of SATA PHY.
 		 */
-		if (!of_device_is_compatible(node, "ti,phy-pipe3-sata"))
+		if (phy->mode != PIPE3_MODE_SATA)
 			return PTR_ERR(phy->refclk);
 	}
 
-	if (!of_device_is_compatible(node, "ti,phy-pipe3-sata")) {
+	if (phy->mode != PIPE3_MODE_SATA) {
 		phy->wkupclk = devm_clk_get(dev, "wkupclk");
 		if (IS_ERR(phy->wkupclk)) {
 			dev_err(dev, "unable to get wkupclk\n");
@@ -465,8 +635,7 @@
 		phy->wkupclk = ERR_PTR(-ENODEV);
 	}
 
-	if (!of_device_is_compatible(node, "ti,phy-pipe3-pcie") ||
-	    phy->phy_power_syscon) {
+	if (phy->mode != PIPE3_MODE_PCIE || phy->phy_power_syscon) {
 		phy->sys_clk = devm_clk_get(dev, "sysclk");
 		if (IS_ERR(phy->sys_clk)) {
 			dev_err(dev, "unable to get sysclk\n");
@@ -474,7 +643,7 @@
 		}
 	}
 
-	if (of_device_is_compatible(node, "ti,phy-pipe3-pcie")) {
+	if (phy->mode == PIPE3_MODE_PCIE) {
 		clk = devm_clk_get(dev, "dpll_ref");
 		if (IS_ERR(clk)) {
 			dev_err(dev, "unable to get dpll ref clk\n");
@@ -546,7 +715,7 @@
 		phy->control_dev = &control_pdev->dev;
 	}
 
-	if (of_device_is_compatible(node, "ti,phy-pipe3-pcie")) {
+	if (phy->mode == PIPE3_MODE_PCIE) {
 		phy->pcs_syscon = syscon_regmap_lookup_by_phandle(node,
 								  "syscon-pcs");
 		if (IS_ERR(phy->pcs_syscon)) {
@@ -564,7 +733,7 @@
 		}
 	}
 
-	if (of_device_is_compatible(node, "ti,phy-pipe3-sata")) {
+	if (phy->mode == PIPE3_MODE_SATA) {
 		phy->dpll_reset_syscon = syscon_regmap_lookup_by_phandle(node,
 							"syscon-pllreset");
 		if (IS_ERR(phy->dpll_reset_syscon)) {
@@ -589,12 +758,8 @@
 {
 	struct resource *res;
 	struct device *dev = phy->dev;
-	struct device_node *node = dev->of_node;
 	struct platform_device *pdev = to_platform_device(dev);
 
-	if (!of_device_is_compatible(node, "ti,phy-pipe3-pcie"))
-		return 0;
-
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
 					   "phy_rx");
 	phy->phy_rx = devm_ioremap_resource(dev, res);
@@ -611,24 +776,12 @@
 static int ti_pipe3_get_pll_base(struct ti_pipe3 *phy)
 {
 	struct resource *res;
-	const struct of_device_id *match;
 	struct device *dev = phy->dev;
-	struct device_node *node = dev->of_node;
 	struct platform_device *pdev = to_platform_device(dev);
 
-	if (of_device_is_compatible(node, "ti,phy-pipe3-pcie"))
+	if (phy->mode == PIPE3_MODE_PCIE)
 		return 0;
 
-	match = of_match_device(ti_pipe3_id_table, dev);
-	if (!match)
-		return -EINVAL;
-
-	phy->dpll_map = (struct pipe3_dpll_map *)match->data;
-	if (!phy->dpll_map) {
-		dev_err(dev, "no DPLL data\n");
-		return -EINVAL;
-	}
-
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
 					   "pll_ctrl");
 	phy->pll_ctrl_base = devm_ioremap_resource(dev, res);
@@ -640,15 +793,29 @@
 	struct ti_pipe3 *phy;
 	struct phy *generic_phy;
 	struct phy_provider *phy_provider;
-	struct device_node *node = pdev->dev.of_node;
 	struct device *dev = &pdev->dev;
 	int ret;
+	const struct of_device_id *match;
+	struct pipe3_data *data;
 
 	phy = devm_kzalloc(dev, sizeof(*phy), GFP_KERNEL);
 	if (!phy)
 		return -ENOMEM;
 
-	phy->dev		= dev;
+	match = of_match_device(ti_pipe3_id_table, dev);
+	if (!match)
+		return -EINVAL;
+
+	data = (struct pipe3_data *)match->data;
+	if (!data) {
+		dev_err(dev, "no driver data\n");
+		return -EINVAL;
+	}
+
+	phy->dev = dev;
+	phy->mode = data->mode;
+	phy->dpll_map = data->dpll_map;
+	phy->settings = data->settings;
 
 	ret = ti_pipe3_get_pll_base(phy);
 	if (ret)
@@ -672,7 +839,7 @@
 	/*
 	 * Prevent auto-disable of refclk for SATA PHY due to Errata i783
 	 */
-	if (of_device_is_compatible(node, "ti,phy-pipe3-sata")) {
+	if (phy->mode == PIPE3_MODE_SATA) {
 		if (!IS_ERR(phy->refclk)) {
 			clk_prepare_enable(phy->refclk);
 			phy->sata_refclk_enabled = true;
@@ -762,18 +929,19 @@
 static const struct of_device_id ti_pipe3_id_table[] = {
 	{
 		.compatible = "ti,phy-usb3",
-		.data = dpll_map_usb,
+		.data = &data_usb,
 	},
 	{
 		.compatible = "ti,omap-usb3",
-		.data = dpll_map_usb,
+		.data = &data_usb,
 	},
 	{
 		.compatible = "ti,phy-pipe3-sata",
-		.data = dpll_map_sata,
+		.data = &data_sata,
 	},
 	{
 		.compatible = "ti,phy-pipe3-pcie",
+		.data = &data_pcie,
 	},
 	{}
 };
diff -Naur --no-dereference a/drivers/phy/ti/phy-tusb1210.c b/drivers/phy/ti/phy-tusb1210.c
--- a/drivers/phy/ti/phy-tusb1210.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/phy/ti/phy-tusb1210.c	2020-06-19 21:00:31.424227716 -0400
@@ -53,7 +53,7 @@
 	return 0;
 }
 
-static int tusb1210_set_mode(struct phy *phy, enum phy_mode mode)
+static int tusb1210_set_mode(struct phy *phy, enum phy_mode mode, int submode)
 {
 	struct tusb1210 *tusb = phy_get_drvdata(phy);
 	int ret;
diff -Naur --no-dereference a/drivers/pwm/Kconfig b/drivers/pwm/Kconfig
--- a/drivers/pwm/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pwm/Kconfig	2020-06-19 21:00:31.424227716 -0400
@@ -447,20 +447,18 @@
 
 config  PWM_TIECAP
 	tristate "ECAP PWM support"
-	depends on ARCH_OMAP2PLUS || ARCH_DAVINCI_DA8XX || ARCH_KEYSTONE
+	depends on ARCH_OMAP2PLUS || ARCH_DAVINCI_DA8XX || ARCH_KEYSTONE || ARCH_K3
 	help
-	  PWM driver support for the ECAP APWM controller found on AM33XX
-	  TI SOC
+	  PWM driver support for the ECAP APWM controller found on TI SOCs
 
 	  To compile this driver as a module, choose M here: the module
 	  will be called pwm-tiecap.
 
 config  PWM_TIEHRPWM
 	tristate "EHRPWM PWM support"
-	depends on ARCH_OMAP2PLUS || ARCH_DAVINCI_DA8XX
+	depends on ARCH_OMAP2PLUS || ARCH_DAVINCI_DA8XX || ARCH_K3
 	help
-	  PWM driver support for the EHRPWM controller found on AM33XX
-	  TI SOC
+	  PWM driver support for the EHRPWM controller found on TI SOCs
 
 	  To compile this driver as a module, choose M here: the module
 	  will be called pwm-tiehrpwm.
@@ -510,4 +508,14 @@
 	  To compile this driver as a module, choose M here: the module
 	  will be called pwm-zx.
 
+config  PWM_PRU
+	tristate "PRU-ICSS PWM support"
+	depends on PRU_REMOTEPROC && ARCH_K3 || COMPILE_TEST
+	help
+	  PWM driver support for the PRU-ICSS PWM controller found on TI
+	  SOCs.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called pwm-pru.
+
 endif
diff -Naur --no-dereference a/drivers/pwm/Makefile b/drivers/pwm/Makefile
--- a/drivers/pwm/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pwm/Makefile	2020-06-19 21:00:31.424227716 -0400
@@ -51,3 +51,4 @@
 obj-$(CONFIG_PWM_TWL_LED)	+= pwm-twl-led.o
 obj-$(CONFIG_PWM_VT8500)	+= pwm-vt8500.o
 obj-$(CONFIG_PWM_ZX)		+= pwm-zx.o
+obj-$(CONFIG_PWM_PRU)		+= pwm-pru.o
diff -Naur --no-dereference a/drivers/pwm/pwm-pru.c b/drivers/pwm/pwm-pru.c
--- a/drivers/pwm/pwm-pru.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/pwm/pwm-pru.c	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,539 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * PWM driver for PRU PWM controller
+ *
+ * Copyright (C) 2019 by Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Bin Liu <b-liu@ti.com>
+ */
+
+#include <linux/module.h>
+#include <linux/of_irq.h>
+#include <linux/of_platform.h>
+#include <linux/pruss.h>
+#include <linux/pwm.h>
+#include <linux/regmap.h>
+#include <linux/remoteproc.h>
+
+#define PP_FW_MAGIC_NUMBER	0x4d575047	/* "GPWM" */
+#define PP_NUM_CHIPS		2
+#define PPC_NUM_PWMS		12
+/*
+ * PWM duty cycle and period thresholds in ns. PWM output is undefined
+ * if its duty cycle or period is out of the range. Valid settings:
+ *     period: 40us ~ 2sec
+ *     duty cycle: 400ns ~ (period - 400ns)
+ *
+ * These thresholds are determined by the timing required in PRU to clear
+ * the IEP CMP events which triggers toggling the PWM pins.
+ */
+#define PP_MIN_DUTY_NS		400
+#define PP_MIN_PERIOD_NS	40000
+#define PP_MAX_PERIOD_NS	2000000000	/* 2 sec */
+
+/* global registers */
+#define PP_FW_MAGIC		0x00
+#define PP_FW_VERSION		0x08
+#define PP_CTRL			0x14
+#define PP_STAT			0x18
+
+/* PP_CTRL bits */
+#define PP_CTRL_IEP_EN		BIT(0)
+
+#define PP_CHIP0_OFFSET		0x1c
+#define PP_CHIP1_OFFSET		0x68
+
+/* pwm chip register offsets */
+#define PPC_PWM_CFG		0x00
+#define PPC_PWM_EN		0x08
+#define PPC_PWM_PERIOD		0x0c	/* holds half of period_ns value */
+#define PPC_PWM_DC0		0x10
+#define PPC_PWM_DC(x)		(PPC_PWM_DC0 + ((x) << 2))
+
+/* PP_PWM_RECFG bits */
+#define PPC_PWM_CFG_DC0		BIT(2)
+#define PPC_PWM_CFG_DC_MASK	GENMASK(13, 2)
+
+#define PPC_PWM_CFG_COMMIT	1
+
+enum {
+	PWMEN_UPDATE,
+	PRD_UPDATE,
+
+	MAX_REGFIELDS
+};
+
+#define to_pru_pwmchip(c)	container_of((c), struct pru_pwmchip, chip)
+
+struct pru_pwmchip {
+	struct pwm_chip chip;
+	int period_owner;
+	spinlock_t period_lock;    /* lock to serialize pwm period access */
+	struct regmap *map;
+	struct regmap_field *pwmen_update;
+	struct regmap_field *period_update;
+};
+
+struct pru_pwm {
+	struct device *dev;
+	struct rproc *pru;
+	struct pruss *pruss;
+	int pru_id;
+	struct pruss_mem_region mem;
+	struct regmap *map;
+	struct regmap_field *fw_inited;
+};
+
+static const struct regmap_config ppc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = PPC_PWM_DC(PPC_NUM_PWMS - 1),
+};
+
+static const struct reg_field ppc_regfields[MAX_REGFIELDS] = {
+	[PWMEN_UPDATE] = REG_FIELD(PPC_PWM_CFG, 0, 0),
+	[PRD_UPDATE]   = REG_FIELD(PPC_PWM_CFG, 1, 1),
+};
+
+static struct regmap_config pp_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = PP_STAT,
+};
+
+static const struct reg_field pp_regfield = REG_FIELD(PP_STAT, 3, 3);
+
+static int prupwm_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
+			     int duty_ns, int period_ns)
+{
+	struct pru_pwmchip *ppc = to_pru_pwmchip(chip);
+	int duty;
+	int idx;
+	int ret = 0;
+
+	spin_lock(&ppc->period_lock);
+	if (period_ns < PP_MIN_PERIOD_NS || period_ns > PP_MAX_PERIOD_NS) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	/* check whether period has been set by another pwm */
+	if (period_ns != pwm_get_period(pwm) && ppc->period_owner != -1 &&
+	    ppc->period_owner != pwm->hwpwm) {
+		ret = -EACCES;
+		goto out;
+	}
+
+	if ((duty_ns && duty_ns < PP_MIN_DUTY_NS) ||
+	    duty_ns > period_ns - PP_MIN_DUTY_NS) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	if (period_ns != pwm_get_period(pwm)) {
+		for (idx = 0; idx < chip->npwm; idx++) {
+			/* skip current pwm device */
+			if (idx == pwm->hwpwm)
+				continue;
+			/*
+			 * the period is global to all pwms, so it cannot be
+			 * less than the duty cycle of any pwm.
+			 */
+			duty = pwm_get_duty_cycle(&chip->pwms[idx]);
+			if (period_ns >= duty)
+				continue;
+
+			dev_err(chip->dev, "Error: new period (%d) is less than pwm%d duty cycle (%d)\n",
+				period_ns, idx, duty);
+			ret = -EINVAL;
+			goto out;
+		}
+		ppc->period_owner = pwm->hwpwm;
+		kobject_uevent(&chip->dev->kobj, KOBJ_CHANGE);
+
+		/* update the new period in pwm->state for all pwms */
+		for (idx = 0; idx < chip->npwm; idx++)
+			pwm_set_period(&chip->pwms[idx], period_ns);
+
+		/* the pwm period register holds half of the period value */
+		regmap_write(ppc->map, PPC_PWM_PERIOD, period_ns >> 1);
+		regmap_field_write(ppc->period_update, PPC_PWM_CFG_COMMIT);
+	}
+
+	regmap_write(ppc->map, PPC_PWM_DC(pwm->hwpwm), duty_ns);
+	regmap_update_bits(ppc->map, PPC_PWM_CFG, PPC_PWM_CFG_DC0 << pwm->hwpwm,
+			   PPC_PWM_CFG_DC0 << pwm->hwpwm);
+
+out:
+	spin_unlock(&ppc->period_lock);
+	return ret;
+}
+
+static int prupwm_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm)
+{
+	struct pru_pwmchip *ppc = to_pru_pwmchip(chip);
+
+	regmap_update_bits(ppc->map, PPC_PWM_EN, BIT(pwm->hwpwm),
+			   BIT(pwm->hwpwm));
+	regmap_field_write(ppc->pwmen_update, PPC_PWM_CFG_COMMIT);
+
+	return 0;
+}
+
+static void prupwm_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm)
+{
+	struct pru_pwmchip *ppc = to_pru_pwmchip(chip);
+
+	regmap_update_bits(ppc->map, PPC_PWM_EN, BIT(pwm->hwpwm), 0);
+	regmap_field_write(ppc->pwmen_update, PPC_PWM_CFG_COMMIT);
+}
+
+/* default period register value might not be 0, update it in pwm->state */
+static void prupwm_pwm_get_init_state(struct pwm_chip *chip,
+				      struct pwm_device *pwm,
+				      struct pwm_state *state)
+{
+	struct pru_pwmchip *ppc = to_pru_pwmchip(chip);
+	int period;
+
+	regmap_read(ppc->map, PPC_PWM_PERIOD, &period);
+	pwm_set_period(pwm, period << 1);
+}
+
+static void prupwm_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm)
+{
+	struct pru_pwmchip *ppc = to_pru_pwmchip(chip);
+
+	/* set pwm duty cycle register to 0 */
+	regmap_write(ppc->map, PPC_PWM_DC(pwm->hwpwm), 0);
+	regmap_update_bits(ppc->map, PPC_PWM_CFG, PPC_PWM_CFG_DC0 << pwm->hwpwm,
+			   PPC_PWM_CFG_DC0 << pwm->hwpwm);
+
+	pwm_set_duty_cycle(pwm, 0);
+
+	spin_lock(&ppc->period_lock);
+	if (ppc->period_owner == pwm->hwpwm) {
+		ppc->period_owner = -1;
+		kobject_uevent(&chip->dev->kobj, KOBJ_CHANGE);
+	}
+	spin_unlock(&ppc->period_lock);
+}
+
+static const struct pwm_ops prupwm_pwm_ops = {
+	.config = prupwm_pwm_config,
+	.enable = prupwm_pwm_enable,
+	.disable = prupwm_pwm_disable,
+	.get_state = prupwm_pwm_get_init_state,
+	.free = prupwm_pwm_free,
+	.owner = THIS_MODULE,
+};
+
+static const struct of_device_id pru_pwmchip_of_match[] = {
+	{ .compatible = "ti,pru-pwmchip", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, pru_pwmchip_of_match);
+
+#ifdef CONFIG_PWM_SYSFS
+static int prupwm_uevent(struct device *dev, struct kobj_uevent_env *env)
+{
+	struct pwm_chip *chip = dev_get_drvdata(dev);
+	struct pru_pwmchip *ppc = to_pru_pwmchip(chip);
+	int ret;
+
+	ret = add_uevent_var(env, "PERIOD_OWNER=%d", ppc->period_owner);
+	return ret;
+}
+#endif
+
+static int pru_pwmchip_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct platform_device *parent = to_platform_device(dev->parent);
+	struct pru_pwm *pp = platform_get_drvdata(parent);
+	struct pru_pwmchip *ppc;
+	void __iomem *mbase;
+	int idx, ret;
+
+	ppc = devm_kzalloc(pp->dev, sizeof(*ppc), GFP_KERNEL);
+	if (!ppc)
+		return -ENOMEM;
+
+	ret = of_property_read_u32(dev->of_node, "reg", &idx);
+	if (ret || idx < 0 || idx >= PP_NUM_CHIPS)
+		return -EINVAL;
+
+	platform_set_drvdata(pdev, ppc);
+	spin_lock_init(&ppc->period_lock);
+	ppc->period_owner = -1;
+
+	mbase = pp->mem.va + (idx ? PP_CHIP1_OFFSET : PP_CHIP0_OFFSET);
+	ppc->map = devm_regmap_init_mmio(dev, mbase, &ppc_regmap_config);
+	if (IS_ERR(ppc->map)) {
+		dev_err(dev, "failed to init regmap (%d)\n", ret);
+		return PTR_ERR(ppc->map);
+	}
+
+	ppc->pwmen_update =
+		devm_regmap_field_alloc(dev, ppc->map,
+					ppc_regfields[PWMEN_UPDATE]);
+	if (IS_ERR(ppc->pwmen_update))
+		return PTR_ERR(ppc->pwmen_update);
+
+	ppc->period_update =
+		devm_regmap_field_alloc(dev, ppc->map,
+					ppc_regfields[PRD_UPDATE]);
+	if (IS_ERR(ppc->period_update))
+		return PTR_ERR(ppc->period_update);
+
+	ppc->chip.dev = dev;
+	ppc->chip.ops = &prupwm_pwm_ops;
+	ppc->chip.base = -1;
+	ppc->chip.npwm = PPC_NUM_PWMS;
+
+	/* set initial duty cycle register of all pwms to 0 */
+	for (idx = 0; idx < PPC_NUM_PWMS; idx++)
+		regmap_write(ppc->map, PPC_PWM_DC(idx), 0);
+
+	/* commit the duty cycle config changes */
+	regmap_write(ppc->map, PPC_PWM_CFG, PPC_PWM_CFG_DC_MASK);
+
+	ret = pwmchip_add(&ppc->chip);
+	if (ret)
+		dev_err(dev, "pwmchip_add() failed: %d\n", ret);
+
+#ifdef CONFIG_PWM_SYSFS
+	if (ppc->chip.sysfs_dev)
+		ppc->chip.sysfs_dev->class->dev_uevent = prupwm_uevent;
+#endif
+
+	return ret;
+}
+
+static int pru_pwmchip_remove(struct platform_device *pdev)
+{
+	struct pru_pwmchip *ppc = platform_get_drvdata(pdev);
+
+	return pwmchip_remove(&ppc->chip);
+}
+
+static struct platform_driver pru_pwmchip_driver = {
+	.driver = {
+		.name = "pru_pwmchip",
+		.of_match_table = pru_pwmchip_of_match,
+	},
+	.probe = pru_pwmchip_probe,
+	.remove = pru_pwmchip_remove,
+};
+
+static int prupwm_init_prufw(struct device_node *np, struct pru_pwm *pp)
+{
+	struct device *dev = pp->dev;
+	struct device_node *child;
+	enum pruss_mem mem_id;
+	u32 reg, id;
+	int ret = 0;
+
+	pp->pru = pru_rproc_get(np, 0);
+	if (IS_ERR(pp->pru)) {
+		ret = PTR_ERR(pp->pru);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "failed to get pru (%d)\n", ret);
+		return ret;
+	}
+
+	pp->pruss = pruss_get(pp->pru);
+	if (IS_ERR(pp->pruss)) {
+		ret = PTR_ERR(pp->pruss);
+		dev_err(dev, "failed to get pruss handle (%d)\n", ret);
+		goto put_pru;
+	}
+
+	pp->pru_id = pru_rproc_get_id(pp->pru);
+	if (pp->pru_id < 0) {
+		dev_err(dev, "failed to get pru id (%d)\n", pp->pru_id);
+		ret = -EINVAL;
+		goto put_pruss;
+	}
+
+	if (pp->pru_id > 1) {
+		dev_err(dev, "invalid pru id (%d)\n", pp->pru_id);
+		ret = -EINVAL;
+		goto put_pruss;
+	}
+
+	mem_id = pp->pru_id ? PRUSS_MEM_DRAM1 : PRUSS_MEM_DRAM0;
+	ret = pruss_request_mem_region(pp->pruss, mem_id, &pp->mem);
+	if (ret) {
+		dev_err(dev, "failed to get pruss mem region (%d)\n", ret);
+		goto put_pruss;
+	}
+
+	pp->map = devm_regmap_init_mmio(dev, pp->mem.va, &pp_regmap_config);
+	if (IS_ERR(pp->map)) {
+		ret = PTR_ERR(pp->map);
+		dev_err(dev, "failed to init register map (%d)\n", ret);
+		goto put_mem;
+	}
+
+	pp->fw_inited = devm_regmap_field_alloc(dev, pp->map, pp_regfield);
+	if (IS_ERR(pp->fw_inited)) {
+		ret = PTR_ERR(pp->fw_inited);
+		goto put_mem;
+	}
+
+	/* clear the mem region before firmware runs by rproc_boot() */
+	memset_io(pp->mem.va, 0, pp->mem.size);
+
+	ret = rproc_boot(pp->pru);
+	if (ret) {
+		dev_err(dev, "failed to boot pru (%d)\n", ret);
+		goto put_mem;
+	}
+
+	regmap_read(pp->map, PP_FW_MAGIC, &reg);
+	if (reg != PP_FW_MAGIC_NUMBER) {
+		dev_err(dev, "invalid firmware magic number\n");
+		ret = -EINVAL;
+		goto put_rproc;
+	}
+
+	regmap_read(pp->map, PP_FW_VERSION, &reg);
+	if (reg > 0x01000000) {
+		dev_err(dev, "unsupported firmware version(0x%x)\n", reg);
+		ret = -EINVAL;
+		goto put_rproc;
+	}
+
+	reg = 0;
+	for_each_available_child_of_node(np, child) {
+		ret = of_property_read_u32(child, "reg", &id);
+
+		if (ret || id >= PP_NUM_CHIPS) {
+			dev_err(dev, "invalid pwmchip id %d (%d)\n", id, ret);
+			ret = -EINVAL;
+			goto put_rproc;
+		}
+
+		reg |= 1 << (id + 1);
+	}
+
+	if (reg)
+		reg |= PP_CTRL_IEP_EN;
+	regmap_write(pp->map, PP_CTRL, reg);
+
+	/* check for firmware init completion, timeout in 100us */
+	ret = regmap_field_read_poll_timeout(pp->fw_inited, reg, reg, 0, 100);
+	if (ret == -ETIMEDOUT)
+		dev_err(dev, "failed to initialize firmware\n");
+	else if (!ret)
+		return 0;
+
+put_rproc:
+	rproc_shutdown(pp->pru);
+put_mem:
+	pruss_release_mem_region(pp->pruss, &pp->mem);
+put_pruss:
+	pruss_put(pp->pruss);
+put_pru:
+	pru_rproc_put(pp->pru);
+
+	return ret;
+}
+
+static int prupwm_exit_pruss(struct pru_pwm *pp)
+{
+	int ret;
+
+	rproc_shutdown(pp->pru);
+	ret = pruss_release_mem_region(pp->pruss, &pp->mem);
+	if (ret)
+		return ret;
+
+	pruss_put(pp->pruss);
+	pru_rproc_put(pp->pru);
+
+	return 0;
+}
+
+static const struct of_device_id prupwm_of_match[] = {
+	{ .compatible = "ti,pru-pwm", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, prupwm_of_match);
+
+static int prupwm_probe(struct platform_device *pdev)
+{
+	struct pru_pwm *pp;
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	int ret;
+
+	if (!np)
+		return -ENODEV;	/* non-DT not supported */
+
+	pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
+	if (!pp)
+		return -ENOMEM;
+
+	pp->dev = dev;
+	ret = prupwm_init_prufw(np, pp);
+	if (ret < 0)
+		return -ENODEV;
+
+	platform_set_drvdata(pdev, pp);
+
+	ret = of_platform_populate(np, NULL, NULL, dev);
+	if (ret) {
+		dev_err(dev, "failed to create pwmchip\n");
+		prupwm_exit_pruss(pp);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int prupwm_remove(struct platform_device *pdev)
+{
+	struct pru_pwm *pp = platform_get_drvdata(pdev);
+
+	of_platform_depopulate(&pdev->dev);
+	return prupwm_exit_pruss(pp);
+}
+
+static struct platform_driver prupwm_driver = {
+	.driver = {
+		.name = "prupwm",
+		.of_match_table = prupwm_of_match,
+	},
+	.probe = prupwm_probe,
+	.remove = prupwm_remove,
+};
+
+static int __init prupwm_init(void)
+{
+	int ret;
+
+	ret = platform_driver_register(&pru_pwmchip_driver);
+	if (ret)
+		return ret;
+
+	ret = platform_driver_register(&prupwm_driver);
+	if (ret)
+		platform_driver_unregister(&pru_pwmchip_driver);
+
+	return ret;
+}
+module_init(prupwm_init);
+
+static void __exit prupwm_exit(void)
+{
+	platform_driver_unregister(&prupwm_driver);
+	platform_driver_unregister(&pru_pwmchip_driver);
+}
+module_exit(prupwm_exit);
+
+MODULE_AUTHOR("Bin Liu <b-liu@ti.com>");
+MODULE_DESCRIPTION("PRU PWM Driver");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/pwm/sysfs.c b/drivers/pwm/sysfs.c
--- a/drivers/pwm/sysfs.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/pwm/sysfs.c	2020-06-19 21:00:31.424227716 -0400
@@ -23,6 +23,8 @@
 #include <linux/kdev_t.h>
 #include <linux/pwm.h>
 
+static struct class pwm_class;
+
 struct pwm_export {
 	struct device child;
 	struct pwm_device *pwm;
@@ -239,6 +241,10 @@
 };
 ATTRIBUTE_GROUPS(pwm);
 
+static const struct device_type pwm_channel_type = {
+	.name		= "pwm_channel",
+};
+
 static void pwm_export_release(struct device *child)
 {
 	struct pwm_export *export = child_to_pwm_export(child);
@@ -248,6 +254,7 @@
 
 static int pwm_export_child(struct device *parent, struct pwm_device *pwm)
 {
+	struct pwm_chip *chip = dev_get_drvdata(parent);
 	struct pwm_export *export;
 	int ret;
 
@@ -265,9 +272,11 @@
 
 	export->child.release = pwm_export_release;
 	export->child.parent = parent;
+	export->child.type = &pwm_channel_type;
 	export->child.devt = MKDEV(0, 0);
+	export->child.class = &pwm_class;
 	export->child.groups = pwm_groups;
-	dev_set_name(&export->child, "pwm%u", pwm->hwpwm);
+	dev_set_name(&export->child, "pwm-%d:%u", chip->base, pwm->hwpwm);
 
 	ret = device_register(&export->child);
 	if (ret) {
@@ -373,7 +382,6 @@
 static struct class pwm_class = {
 	.name = "pwm",
 	.owner = THIS_MODULE,
-	.dev_groups = pwm_chip_groups,
 };
 
 static int pwmchip_sysfs_match(struct device *parent, const void *data)
@@ -389,11 +397,14 @@
 	 * If device_create() fails the pwm_chip is still usable by
 	 * the kernel its just not exported.
 	 */
-	parent = device_create(&pwm_class, chip->dev, MKDEV(0, 0), chip,
+	parent = device_create_with_groups(&pwm_class, chip->dev, MKDEV(0, 0),
+			       chip, pwm_chip_groups,
 			       "pwmchip%d", chip->base);
 	if (IS_ERR(parent)) {
 		dev_warn(chip->dev,
 			 "device_create failed for pwm_chip sysfs export\n");
+	} else {
+		chip->sysfs_dev = parent;
 	}
 }
 
diff -Naur --no-dereference a/drivers/regulator/palmas-regulator.c b/drivers/regulator/palmas-regulator.c
--- a/drivers/regulator/palmas-regulator.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/regulator/palmas-regulator.c	2020-06-19 21:00:31.424227716 -0400
@@ -1024,6 +1024,7 @@
 	struct palmas_reg_init *reg_init;
 	struct palmas_regs_info *rinfo;
 	struct regulator_desc *desc;
+	unsigned int reg;
 
 	for (id = ddata->ldo_begin; id < ddata->max_reg; id++) {
 		if (pdata && pdata->reg_init[id])
@@ -1072,6 +1073,29 @@
 						TPS65917_LDO1_CTRL_BYPASS_EN;
 				desc->bypass_mask =
 						TPS65917_LDO1_CTRL_BYPASS_EN;
+
+				/*
+				 * OTP Values are set to bypass enable.
+				 * Switch to disable so that use count
+				 * does not go negative while directly
+				 * disabling bypass.
+				 */
+				ret = palmas_ldo_read(pmic->palmas,
+						      rinfo->ctrl_addr, &reg);
+				if (ret) {
+					dev_err(pmic->dev,
+						"Error reading ldo1 reg\n");
+					return ret;
+				}
+				reg &= ~TPS65917_LDO1_CTRL_BYPASS_EN;
+				ret = palmas_ldo_write(pmic->palmas,
+						       rinfo->ctrl_addr, reg);
+				if (ret) {
+					dev_err(pmic->dev,
+						"Error writing ldo1 reg\n");
+					return ret;
+				}
+
 			}
 		} else {
 			desc->n_voltages = 1;
diff -Naur --no-dereference a/drivers/remoteproc/da8xx_remoteproc.c b/drivers/remoteproc/da8xx_remoteproc.c
--- a/drivers/remoteproc/da8xx_remoteproc.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/remoteproc/da8xx_remoteproc.c	2020-06-19 21:00:31.424227716 -0400
@@ -226,7 +226,7 @@
 				res->start & DA8XX_RPROC_LOCAL_ADDRESS_MASK;
 		drproc->mem[i].size = resource_size(res);
 
-		dev_dbg(dev, "memory %8s: bus addr %pa size 0x%zx va %p da 0x%x\n",
+		dev_dbg(dev, "memory %8s: bus addr %pa size 0x%zx va %pK da 0x%x\n",
 			mem_names[i], &drproc->mem[i].bus_addr,
 			drproc->mem[i].size, drproc->mem[i].cpu_addr,
 			drproc->mem[i].dev_addr);
@@ -352,6 +352,9 @@
 		goto free_rproc;
 	}
 
+	if (rproc_get_id(rproc) < 0)
+		dev_warn(dev, "device does not have an alias id or platform device id\n");
+
 	return 0;
 
 free_rproc:
diff -Naur --no-dereference a/drivers/remoteproc/imx_rproc.c b/drivers/remoteproc/imx_rproc.c
--- a/drivers/remoteproc/imx_rproc.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/remoteproc/imx_rproc.c	2020-06-19 21:00:31.424227716 -0400
@@ -211,7 +211,7 @@
 	return -ENOENT;
 }
 
-static void *imx_rproc_da_to_va(struct rproc *rproc, u64 da, int len)
+static void *imx_rproc_da_to_va(struct rproc *rproc, u64 da, int len, u32 flags)
 {
 	struct imx_rproc *priv = rproc->priv;
 	void *va = NULL;
diff -Naur --no-dereference a/drivers/remoteproc/Kconfig b/drivers/remoteproc/Kconfig
--- a/drivers/remoteproc/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/remoteproc/Kconfig	2020-06-19 21:00:31.424227716 -0400
@@ -24,7 +24,7 @@
 
 config OMAP_REMOTEPROC
 	tristate "OMAP remoteproc support"
-	depends on ARCH_OMAP4 || SOC_OMAP5
+	depends on ARCH_OMAP4 || SOC_OMAP5 || SOC_DRA7XX
 	depends on OMAP_IOMMU
 	select MAILBOX
 	select OMAP2PLUS_MBOX
@@ -41,6 +41,17 @@
 	  It's safe to say N here if you're not interested in multimedia
 	  offloading or just want a bare minimum kernel.
 
+config OMAP_REMOTEPROC_WATCHDOG
+	bool "OMAP remoteproc watchdog timer"
+	depends on OMAP_REMOTEPROC
+	help
+	  Say Y here to enable watchdog timer for remote processors.
+
+	  This option controls the watchdog functionality for the remote
+	  processors in OMAP. Dedicated OMAP DMTimers are used by the remote
+	  processors and triggers the timer interrupt upon a watchdog
+	  detection.
+
 config WKUP_M3_RPROC
 	tristate "AMx3xx Wakeup M3 remoteproc support"
 	depends on SOC_AM33XX || SOC_AM43XX
@@ -77,6 +88,7 @@
 config KEYSTONE_REMOTEPROC
 	tristate "Keystone Remoteproc support"
 	depends on ARCH_KEYSTONE
+	depends on UIO
 	help
 	  Say Y here here to support Keystone remote processors (DSP)
 	  via the remote processor framework.
@@ -84,6 +96,20 @@
 	  It's safe to say N here if you're not interested in the Keystone
 	  DSPs or just want to use a bare minimum kernel.
 
+config PRU_REMOTEPROC
+	tristate "TI PRU remoteproc support"
+	depends on TI_PRUSS
+	default TI_PRUSS
+	select MAILBOX
+	select OMAP2PLUS_MBOX if ARCH_OMAP2PLUS || ARCH_K3
+	help
+	  Support for TI PRU remote processors present within a PRU-ICSS
+	  subsystem via the remote processor framework.
+
+	  Say Y or M here to support the Programmable Realtime Unit (PRU)
+	  processors on various TI SoCs. It's safe to say N here if you're
+	  not interested in the PRU or if you are unsure.
+
 config QCOM_ADSP_PIL
 	tristate "Qualcomm ADSP Peripheral Image Loader"
 	depends on OF && ARCH_QCOM
@@ -181,6 +207,38 @@
 config ST_SLIM_REMOTEPROC
 	tristate
 
+config TI_K3_R5_REMOTEPROC
+	tristate "TI K3 R5 remoteproc support"
+	depends on ARCH_K3
+	select MAILBOX
+	select OMAP2PLUS_MBOX
+	help
+	  Say y here to support TI's R5F remote processor subsystems
+	  on various TI K3 family of SoCs through the remote processor
+	  framework.
+
+	  You want to say y here in order to offload some processing
+	  tasks to these processors
+
+	  It's safe to say N here if you're not interested in utilizing
+	  a slave processor
+
+config TI_K3_DSP_REMOTEPROC
+	tristate "TI K3 DSP remoteproc support"
+	depends on ARCH_K3
+	select MAILBOX
+	select OMAP2PLUS_MBOX
+	help
+	  Say y here to support TI's C66x and C71x DSP remote processor
+	  subsystems on various TI K3 family of SoCs through the remote
+	  processor framework.
+
+	  You want to say m here in order to offload some processing
+	  tasks to these processors.
+
+	  It's safe to say N here if you're not interested in utilizing
+	  the DSP slave processors.
+
 endif # REMOTEPROC
 
 endmenu
diff -Naur --no-dereference a/drivers/remoteproc/keystone_remoteproc.c b/drivers/remoteproc/keystone_remoteproc.c
--- a/drivers/remoteproc/keystone_remoteproc.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/remoteproc/keystone_remoteproc.c	2020-06-19 21:00:31.424227716 -0400
@@ -1,22 +1,16 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * TI Keystone DSP remoteproc driver
  *
- * Copyright (C) 2015-2017 Texas Instruments Incorporated - http://www.ti.com/
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
+ * Copyright (C) 2015-2019 Texas Instruments Incorporated - http://www.ti.com/
  */
 
 #include <linux/module.h>
+#include <linux/uaccess.h>
 #include <linux/slab.h>
 #include <linux/io.h>
 #include <linux/interrupt.h>
+#include <linux/spinlock.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
 #include <linux/workqueue.h>
@@ -26,24 +20,40 @@
 #include <linux/regmap.h>
 #include <linux/mfd/syscon.h>
 #include <linux/remoteproc.h>
+#include <linux/miscdevice.h>
+#include <linux/uio_driver.h>
 #include <linux/reset.h>
 
+#include <uapi/linux/keystone_remoteproc.h>
+
 #include "remoteproc_internal.h"
 
+#define DRIVER_UIO_VERSION			"0.1"
+
+#define KEYSTONE_RPROC_MAX_RSC_TABLE		SZ_1K
 #define KEYSTONE_RPROC_LOCAL_ADDRESS_MASK	(SZ_16M - 1)
 
+/*
+ * XXX: evaluate if this param needs to be enhanced so that the switch between
+ * userspace and remoteproc core loaders can be controlled per device.
+ */
+static bool use_rproc_core_loader;
+module_param(use_rproc_core_loader, bool, 0444);
+
 /**
  * struct keystone_rproc_mem - internal memory structure
  * @cpu_addr: MPU virtual address of the memory region
  * @bus_addr: Bus address used to access the memory region
  * @dev_addr: Device address of the memory region from DSP view
  * @size: Size of the memory region
+ * @kobj: kobject for the sysfs directory file
  */
 struct keystone_rproc_mem {
 	void __iomem *cpu_addr;
 	phys_addr_t bus_addr;
 	u32 dev_addr;
 	size_t size;
+	struct kobject kobj;
 };
 
 /**
@@ -59,6 +69,18 @@
  * @irq_fault: irq entry for exception
  * @kick_gpio: gpio used for virtio kicks
  * @workqueue: workqueue for processing virtio interrupts
+ * @misc: misc device structure used to expose fops to user-space
+ * @uio: uio device information
+ * @mlock: lock to protect resources in fops
+ * @lock: lock to protect shared resources within UIO interrupt handlers
+ * @flags: flags to keep track of UIO interrupt occurrence
+ * @rsc_table: resource table pointer copied from userspace
+ * @rsc_table_size: size of resource table
+ * @loaded_rsc_table: kernel pointer of loaded resource table
+ * @boot_addr: remote processor boot address used with userspace loader
+ * @open_count: fops open reference counter
+ * @use_userspace_loader: flag to denote if driver is configured for userspace
+ *			  loader
  */
 struct keystone_rproc {
 	struct device *dev;
@@ -72,8 +94,283 @@
 	int irq_fault;
 	int kick_gpio;
 	struct work_struct workqueue;
+	struct miscdevice misc;
+	struct uio_info uio;
+	struct mutex mlock; /* fops lock */
+	spinlock_t lock; /* uio handler lock */
+	unsigned long flags;
+	struct resource_table *rsc_table;
+	int rsc_table_size;
+	void *loaded_rsc_table;
+	u32 boot_addr;
+	int open_count;
+	unsigned int use_userspace_loader : 1;
+};
+
+struct mem_sysfs_entry {
+	struct attribute attr;
+	ssize_t (*show)(struct keystone_rproc_mem *mem, char *buf);
+	ssize_t (*store)(struct keystone_rproc_mem *mem, const char *buf,
+			 size_t len);
 };
 
+static ssize_t mem_addr_show(struct keystone_rproc_mem *mem, char *buf)
+{
+	return sprintf(buf, "%pa\n", &mem->bus_addr);
+}
+
+static ssize_t mem_size_show(struct keystone_rproc_mem *mem, char *buf)
+{
+	return sprintf(buf, "0x%016zx\n", mem->size);
+}
+
+static struct mem_sysfs_entry addr_attribute =
+	__ATTR(addr, 0444, mem_addr_show, NULL);
+static struct mem_sysfs_entry size_attribute =
+	__ATTR(size, 0444, mem_size_show, NULL);
+
+static struct attribute *attrs[] = {
+	&addr_attribute.attr,
+	&size_attribute.attr,
+	NULL,	/* sentinel */
+};
+
+#define to_dsp_mem(m) container_of(m, struct keystone_rproc_mem, kobj)
+
+static ssize_t mem_type_show(struct kobject *kobj, struct attribute *attr,
+			     char *buf)
+{
+	struct keystone_rproc_mem *mem = to_dsp_mem(kobj);
+	struct mem_sysfs_entry *entry;
+
+	entry = container_of(attr, struct mem_sysfs_entry, attr);
+	if (!entry->show)
+		return -EIO;
+
+	return entry->show(mem, buf);
+}
+
+static const struct sysfs_ops mem_sysfs_ops = {
+	.show = mem_type_show,
+};
+
+static struct kobj_type mem_attr_type = {
+	.sysfs_ops	= &mem_sysfs_ops,
+	.default_attrs	= attrs,
+};
+
+static int keystone_rproc_mem_add_attrs(struct keystone_rproc *ksproc)
+{
+	int i, ret;
+	struct keystone_rproc_mem *mem;
+	struct kobject *kobj_parent = &ksproc->misc.this_device->kobj;
+
+	for (i = 0; i < ksproc->num_mems; i++) {
+		mem = &ksproc->mem[i];
+		kobject_init(&mem->kobj, &mem_attr_type);
+		ret = kobject_add(&mem->kobj, kobj_parent, "memory%d", i);
+		if (ret)
+			goto err_kobj;
+		ret = kobject_uevent(&mem->kobj, KOBJ_ADD);
+		if (ret)
+			goto err_kobj;
+	}
+
+	return 0;
+
+err_kobj:
+	for (; i >= 0; i--) {
+		mem = &ksproc->mem[i];
+		kobject_put(&mem->kobj);
+	}
+	return ret;
+}
+
+static void keystone_rproc_mem_del_attrs(struct keystone_rproc *ksproc)
+{
+	int i;
+	struct keystone_rproc_mem *mem;
+
+	for (i = 0; i < ksproc->num_mems; i++) {
+		mem = &ksproc->mem[i];
+		kobject_put(&mem->kobj);
+	}
+}
+
+static void *keystone_rproc_da_to_va(struct rproc *rproc, u64 da, int len,
+				     u32 flags);
+
+/* uio handler dealing with userspace controlled exception interrupt */
+static irqreturn_t keystone_rproc_uio_handler(int irq, struct uio_info *uio)
+{
+	struct keystone_rproc *ksproc = uio->priv;
+
+	spin_lock(&ksproc->lock);
+	if (!__test_and_set_bit(0, &ksproc->flags))
+		disable_irq_nosync(irq);
+	spin_unlock(&ksproc->lock);
+
+	return IRQ_HANDLED;
+}
+
+/* uio driver interrupt control dealing with exception interrupt */
+static int keystone_rproc_uio_irqcontrol(struct uio_info *uio, s32 irq_on)
+{
+	struct keystone_rproc *ksproc = uio->priv;
+	unsigned long flags;
+
+	spin_lock_irqsave(&ksproc->lock, flags);
+	if (irq_on) {
+		if (__test_and_clear_bit(0, &ksproc->flags))
+			enable_irq(uio->irq);
+	} else {
+		if (!__test_and_set_bit(0, &ksproc->flags))
+			disable_irq(uio->irq);
+	}
+	spin_unlock_irqrestore(&ksproc->lock, flags);
+
+	return 0;
+}
+
+/* Reset previously set rsc table variables */
+static void keystone_rproc_reset_rsc_table(struct keystone_rproc *ksproc)
+{
+	kfree(ksproc->rsc_table);
+	ksproc->rsc_table = NULL;
+	ksproc->loaded_rsc_table = NULL;
+	ksproc->rsc_table_size = 0;
+}
+
+/*
+ * Create/delete the virtio devices in kernel once the user-space loading is
+ * complete, configure the remoteproc states appropriately, and boot or reset
+ * the remote processor. The resource table should have been published through
+ * KEYSTONE_RPROC_IOC_SET_RSC_TABLE & KEYSTONE_RPROC_IOC_SET_LOADED_RSC_TABLE
+ * ioctls before invoking this. The boot address is passed through the
+ * KEYSTONE_RPROC_IOC_SET_STATE ioctl when setting the KEYSTONE_RPROC_RUNNING
+ * state.
+ *
+ * NOTE:
+ * The ioctls KEYSTONE_RPROC_IOC_DSP_RESET and KEYSTONE_RPROC_IOC_DSP_BOOT
+ * are restricted to support the booting or resetting the DSP devices only
+ * for firmware images without any resource table.
+ */
+static int keystone_rproc_set_state(struct keystone_rproc *ksproc,
+				    void __user *argp)
+{
+	struct rproc *rproc = ksproc->rproc;
+	struct keystone_rproc_set_state_params set_state_params;
+	int ret = 0;
+
+	if (copy_from_user(&set_state_params, argp, sizeof(set_state_params)))
+		return -EFAULT;
+
+	switch (set_state_params.state) {
+	case KEYSTONE_RPROC_RUNNING:
+		if (!ksproc->rsc_table || !ksproc->loaded_rsc_table)
+			return -EINVAL;
+
+		/*
+		 * store boot address for .get_boot_addr() rproc fw ops
+		 * XXX: validate the boot address so it is not set to a
+		 * random address
+		 */
+		ksproc->boot_addr = set_state_params.boot_addr;
+
+		/*
+		 * invoke rproc_boot to trigger the boot, the resource table
+		 * is parsed during the process and is agnostic of the presence
+		 * or absence of virtio devices
+		 */
+		ret = rproc_boot(rproc);
+		break;
+
+	case KEYSTONE_RPROC_OFFLINE:
+		if (rproc->state != RPROC_RUNNING)
+			return -EINVAL;
+
+		/* invoke rproc_shutdown to match rproc_boot */
+		rproc_shutdown(rproc);
+
+		mutex_lock(&ksproc->mlock);
+		keystone_rproc_reset_rsc_table(ksproc);
+		mutex_unlock(&ksproc->mlock);
+
+		break;
+
+	default:
+		ret = -ENOTSUPP;
+	}
+
+	return ret;
+}
+
+/* Copy the resource table from userspace into kernel */
+static int keystone_rproc_set_rsc_table(struct keystone_rproc *ksproc,
+					void __user *data)
+{
+	unsigned long len = 0;
+	void *rsc_table = NULL;
+
+	if (!data)
+		return -EFAULT;
+
+	if (copy_from_user(&len, data, sizeof(len)))
+		return -EFAULT;
+
+	if (len >= KEYSTONE_RPROC_MAX_RSC_TABLE)
+		return -EOVERFLOW;
+
+	data += sizeof(len);
+
+	rsc_table = kzalloc(len, GFP_KERNEL);
+	if (!rsc_table)
+		return -ENOMEM;
+
+	if (copy_from_user(rsc_table, data, len))
+		goto error_return;
+
+	mutex_lock(&ksproc->mlock);
+
+	kfree(ksproc->rsc_table);
+
+	ksproc->rsc_table = rsc_table;
+	ksproc->rsc_table_size = len;
+	ksproc->loaded_rsc_table = NULL;
+
+	mutex_unlock(&ksproc->mlock);
+
+	return 0;
+
+error_return:
+	kfree(rsc_table);
+	return -EFAULT;
+}
+
+/*
+ * Store the equivalent kernel virtual address of the loaded resource table in
+ * device memory. Userspace published the device address of the loaded resource
+ * table.
+ */
+static int keystone_rproc_set_loaded_rsc_table(struct keystone_rproc *ksproc,
+					       unsigned int dma_addr)
+{
+	struct rproc *rproc = ksproc->rproc;
+	void *ptr;
+
+	if (!ksproc->rsc_table_size || !ksproc->rsc_table)
+		return -EINVAL;
+
+	ptr = keystone_rproc_da_to_va(rproc, dma_addr, ksproc->rsc_table_size,
+				      RPROC_FLAGS_NONE);
+	if (!ptr)
+		return -EINVAL;
+
+	ksproc->loaded_rsc_table = ptr;
+
+	return 0;
+}
+
 /* Put the DSP processor into reset */
 static void keystone_rproc_dsp_reset(struct keystone_rproc *ksproc)
 {
@@ -81,7 +378,8 @@
 }
 
 /* Configure the boot address and boot the DSP processor */
-static int keystone_rproc_dsp_boot(struct keystone_rproc *ksproc, u32 boot_addr)
+static int keystone_rproc_dsp_boot(struct keystone_rproc *ksproc,
+				   uint32_t boot_addr)
 {
 	int ret;
 
@@ -103,6 +401,235 @@
 	return 0;
 }
 
+static long
+keystone_rproc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
+{
+	struct miscdevice *misc = filp->private_data;
+	struct keystone_rproc *ksproc =
+		container_of(misc, struct keystone_rproc, misc);
+	void __user *argp = (void __user *)arg;
+	int ret = 0;
+
+	dev_dbg(ksproc->dev, "%s: cmd 0x%.8x (%d), arg 0x%lx\n",
+		__func__, cmd, _IOC_NR(cmd), arg);
+
+	if (_IOC_TYPE(cmd) != KEYSTONE_RPROC_IOC_MAGIC)
+		return -ENOTTY;
+
+	if (_IOC_NR(cmd) >= KEYSTONE_RPROC_IOC_MAXNR)
+		return -ENOTTY;
+
+	switch (cmd) {
+	case KEYSTONE_RPROC_IOC_SET_STATE:
+		ret = keystone_rproc_set_state(ksproc, argp);
+		break;
+
+	case KEYSTONE_RPROC_IOC_SET_RSC_TABLE:
+		ret = keystone_rproc_set_rsc_table(ksproc, argp);
+		break;
+
+	case KEYSTONE_RPROC_IOC_SET_LOADED_RSC_TABLE:
+		ret = keystone_rproc_set_loaded_rsc_table(ksproc, arg);
+		break;
+
+	case KEYSTONE_RPROC_IOC_DSP_RESET:
+		if (ksproc->rsc_table) {
+			ret = -EINVAL;
+			break;
+		}
+
+		keystone_rproc_dsp_reset(ksproc);
+		break;
+
+	case KEYSTONE_RPROC_IOC_DSP_BOOT:
+		if (ksproc->rsc_table) {
+			ret = -EINVAL;
+			break;
+		}
+
+		ret = keystone_rproc_dsp_boot(ksproc, arg);
+		break;
+
+	default:
+		ret = -ENOTTY;
+		break;
+	}
+
+	if (ret) {
+		dev_err(ksproc->dev, "error in ioctl call: cmd 0x%.8x (%d), ret %d\n",
+			cmd, _IOC_NR(cmd), ret);
+	}
+
+	return ret;
+}
+
+/*
+ * Map DSP memories into userspace for supporting Userspace loading.
+ *
+ * This is a custom mmap function following semantics based on the UIO
+ * mmap implementation. The vm_pgoff passed in the vma structure is a
+ * combination of the memory region index and the actual page offset in
+ * that region. This checks if user request is in valid range before
+ * providing mmap access.
+ *
+ * XXX: Evaluate this approach, as the internal memories can be mapped in
+ * whole into userspace as they are not super-large, or switch to using
+ * direct addresses to look more like a traditional implementation.
+ */
+static int keystone_rproc_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	struct miscdevice *misc = file->private_data;
+	struct keystone_rproc *ksproc =
+		container_of(misc, struct keystone_rproc, misc);
+	size_t size = vma->vm_end - vma->vm_start;
+	size_t req_offset;
+	u32 idx;
+
+	idx = vma->vm_pgoff & KEYSTONE_RPROC_UIO_MAP_INDEX_MASK;
+
+	if (idx >= ksproc->num_mems) {
+		dev_err(ksproc->dev, "invalid mmap region index %d\n", idx);
+		return -EINVAL;
+	}
+
+	req_offset = (vma->vm_pgoff - idx) << PAGE_SHIFT;
+	if (req_offset + size < req_offset) {
+		dev_err(ksproc->dev, "invalid request - overflow, mmap offset = 0x%zx size 0x%zx region %d\n",
+			req_offset, size, idx);
+		return -EINVAL;
+	}
+
+	if ((req_offset + size) > ksproc->mem[idx].size) {
+		dev_err(ksproc->dev, "invalid request - out of range, mmap offset 0x%zx size 0x%zx region %d\n",
+			req_offset, size, idx);
+		return -EINVAL;
+	}
+
+	vma->vm_page_prot =
+		phys_mem_access_prot(file,
+				     (ksproc->mem[idx].bus_addr >> PAGE_SHIFT) +
+				     (vma->vm_pgoff - idx), size,
+				     vma->vm_page_prot);
+
+	if (remap_pfn_range(vma, vma->vm_start,
+			    (ksproc->mem[idx].bus_addr >> PAGE_SHIFT) +
+			    (vma->vm_pgoff - idx), size, vma->vm_page_prot))
+		return -EAGAIN;
+
+	return 0;
+}
+
+static int keystone_rproc_open(struct inode *inode, struct file *file)
+{
+	struct miscdevice *misc = file->private_data;
+	struct keystone_rproc *ksproc =
+		container_of(misc, struct keystone_rproc, misc);
+
+	mutex_lock(&ksproc->mlock);
+	ksproc->open_count++;
+	mutex_unlock(&ksproc->mlock);
+
+	return 0;
+}
+
+static int keystone_rproc_release(struct inode *inode, struct file *filp)
+{
+	struct miscdevice *misc = filp->private_data;
+	struct keystone_rproc *ksproc =
+		container_of(misc, struct keystone_rproc, misc);
+	struct rproc *rproc = ksproc->rproc;
+
+	mutex_lock(&ksproc->mlock);
+
+	if ((WARN_ON(ksproc->open_count == 0)))
+		goto end;
+
+	if (--ksproc->open_count > 0)
+		goto end;
+
+	if (rproc->state != RPROC_OFFLINE) {
+		rproc_shutdown(rproc);
+		WARN_ON(rproc->state != RPROC_OFFLINE);
+	}
+
+	keystone_rproc_reset_rsc_table(ksproc);
+
+end:
+	mutex_unlock(&ksproc->mlock);
+	return 0;
+}
+
+/*
+ * File operations exposed through a miscdevice for supporting
+ * the userspace loader/boot mechanism.
+ */
+static const struct file_operations keystone_rproc_fops = {
+	.owner		= THIS_MODULE,
+	.unlocked_ioctl	= keystone_rproc_ioctl,
+	.mmap		= keystone_rproc_mmap,
+	.open		= keystone_rproc_open,
+	.release	= keystone_rproc_release,
+};
+
+/*
+ * Used only with userspace loader/boot mechanism, the parsing of the firmware
+ * is done in userspace, and a copy of the resource table is added for the
+ * kernel-level access through an ioctl. Create the remoteproc cached table
+ * using this resource table and configure the table pointer and table size
+ * accordingly to allow the remoteproc core to process the resource table for
+ * creating the vrings and traces.
+ */
+static int keystone_rproc_load_rsc_table(struct rproc *rproc,
+					 const struct firmware *fw)
+{
+	struct keystone_rproc *ksproc = rproc->priv;
+
+	rproc->cached_table = kmemdup(ksproc->rsc_table, ksproc->rsc_table_size,
+				      GFP_KERNEL);
+	if (!rproc->cached_table)
+		return -ENOMEM;
+
+	rproc->table_ptr = rproc->cached_table;
+	rproc->table_sz = ksproc->rsc_table_size;
+
+	return 0;
+}
+
+/*
+ * Used only with userspace loader/boot mechanism, the device address of the
+ * loaded resource table is published to the kernel-level through an ioctl
+ * at which point the equivalent kernel virtual pointer is stored in a local
+ * variable in the keystone_rproc device structure. Return this kernel pointer
+ * to the remoteproc core for runtime publishing/modification of the resource
+ * table entries.
+ *
+ * NOTE: Only loaded resource tables in the DSP internal memories is supported
+ *       at present.
+ */
+static struct resource_table *
+keystone_rproc_find_loaded_rsc_table(struct rproc *rproc,
+				     const struct firmware *fw)
+{
+	struct keystone_rproc *ksproc = rproc->priv;
+
+	return ksproc->loaded_rsc_table;
+}
+
+/*
+ * Used only with userspace loader/boot mechanism, the boot address
+ * is published to the kernel-level through an ioctl call and is
+ * stored in a local variable in the keystone_rproc device structure.
+ * Return this address to the remoteproc core through the .get_boot_addr()
+ * remoteproc firmware ops
+ */
+static u32 keystone_rproc_get_boot_addr(struct rproc *rproc,
+					const struct firmware *fw)
+{
+	struct keystone_rproc *ksproc = rproc->priv;
+
+	return ksproc->boot_addr;
+}
+
 /*
  * Process the remoteproc exceptions
  *
@@ -112,7 +639,9 @@
  * generation register.
  *
  * This function just invokes the rproc_report_crash to report the exception
- * to the remoteproc driver core, to trigger a recovery.
+ * to the remoteproc driver core, to trigger a recovery. This is the case
+ * only when using in-kernel remoteproc core loader/boot mechanism, and is
+ * handled through an UIO interrupt otherwise.
  */
 static irqreturn_t keystone_rproc_exception_interrupt(int irq, void *dev_id)
 {
@@ -172,7 +701,11 @@
  *
  * This function will be invoked only after the firmware for this rproc
  * was loaded, parsed successfully, and all of its resource requirements
- * were met.
+ * were met. The function skips releasing the processor from reset and
+ * registering for the exception interrupt if using the userspace controlled
+ * load/boot mechanism. The processor will be started through an ioctl when
+ * controlled from userspace, but the virtio interrupt still is handled at
+ * the kernel layer.
  */
 static int keystone_rproc_start(struct rproc *rproc)
 {
@@ -189,12 +722,15 @@
 		goto out;
 	}
 
-	ret = request_irq(ksproc->irq_fault, keystone_rproc_exception_interrupt,
-			  0, dev_name(ksproc->dev), ksproc);
-	if (ret) {
-		dev_err(ksproc->dev, "failed to enable exception interrupt, ret = %d\n",
-			ret);
-		goto free_vring_irq;
+	if (!ksproc->use_userspace_loader) {
+		ret = request_irq(ksproc->irq_fault,
+				  keystone_rproc_exception_interrupt, 0,
+				  dev_name(ksproc->dev), ksproc);
+		if (ret) {
+			dev_err(ksproc->dev, "failed to enable exception interrupt, ret = %d\n",
+				ret);
+			goto free_vring_irq;
+		}
 	}
 
 	ret = keystone_rproc_dsp_boot(ksproc, rproc->bootaddr);
@@ -216,24 +752,29 @@
  * Stop the DSP remote processor.
  *
  * This function puts the DSP processor into reset, and finishes processing
- * of any pending messages.
+ * of any pending messages. The reset procedure is completed only if using
+ * kernel-mode remoteproc loading/booting mechanism, it is handled outside
+ * if using userspace load/boot mechanism either through an ioctl, or when
+ * the handle to the device is closed without triggering a reset.
  */
 static int keystone_rproc_stop(struct rproc *rproc)
 {
 	struct keystone_rproc *ksproc = rproc->priv;
 
-	keystone_rproc_dsp_reset(ksproc);
-	free_irq(ksproc->irq_fault, ksproc);
+	if (!ksproc->use_userspace_loader) {
+		keystone_rproc_dsp_reset(ksproc);
+		free_irq(ksproc->irq_fault, ksproc);
+	}
+
 	free_irq(ksproc->irq_ring, ksproc);
 	flush_work(&ksproc->workqueue);
-
 	return 0;
 }
 
 /*
  * Kick the remote processor to notify about pending unprocessed messages.
  * The vqid usage is not used and is inconsequential, as the kick is performed
- * through a simulated GPIO (a bit in an IPC interrupt-triggering register),
+ * through a simulated GPIO (an bit in an IPC interrupt-triggering register),
  * the remote processor is expected to process both its Tx and Rx virtqueues.
  */
 static void keystone_rproc_kick(struct rproc *rproc, int vqid)
@@ -254,7 +795,8 @@
  * can be used either by the remoteproc core for loading (when using kernel
  * remoteproc loader), or by any rpmsg bus drivers.
  */
-static void *keystone_rproc_da_to_va(struct rproc *rproc, u64 da, int len)
+static void *keystone_rproc_da_to_va(struct rproc *rproc, u64 da, int len,
+				     u32 flags)
 {
 	struct keystone_rproc *ksproc = rproc->priv;
 	void __iomem *va = NULL;
@@ -371,8 +913,11 @@
 	struct device *dev = &pdev->dev;
 	struct device_node *np = dev->of_node;
 	struct keystone_rproc *ksproc;
+	struct miscdevice *misc;
+	struct uio_info *uio;
 	struct rproc *rproc;
 	int dsp_id;
+	char *uio_name = NULL;
 	char *fw_name = NULL;
 	char *template = "keystone-dsp%d-fw";
 	int name_len = 0;
@@ -389,6 +934,13 @@
 		return dsp_id;
 	}
 
+	/* construct a name for uio devices - assuming a single digit alias */
+	name_len = strlen("dsp%d");
+	uio_name = devm_kzalloc(dev, name_len, GFP_KERNEL);
+	if (!uio_name)
+		return -ENOMEM;
+	snprintf(uio_name, name_len, "dsp%d", dsp_id);
+
 	/* construct a custom default fw name - subject to change in future */
 	name_len = strlen(template); /* assuming a single digit alias */
 	fw_name = devm_kzalloc(dev, name_len, GFP_KERNEL);
@@ -402,9 +954,33 @@
 		return -ENOMEM;
 
 	rproc->has_iommu = false;
+
 	ksproc = rproc->priv;
 	ksproc->rproc = rproc;
 	ksproc->dev = dev;
+	ksproc->use_userspace_loader = !use_rproc_core_loader;
+
+	/*
+	 * customize the remoteproc core config flags and ELF fw ops for
+	 * userspace loader/boot mechanism
+	 */
+	if (ksproc->use_userspace_loader) {
+		rproc->recovery_disabled = true;
+		rproc->auto_boot = false;
+		rproc->skip_firmware_request = 1;
+		rproc->skip_load = 1;
+		rproc->deny_sysfs_ops = true;
+
+		rproc->ops->parse_fw = keystone_rproc_load_rsc_table;
+		rproc->ops->find_loaded_rsc_table =
+					keystone_rproc_find_loaded_rsc_table;
+		rproc->ops->get_boot_addr = keystone_rproc_get_boot_addr;
+		rproc->ops->sanity_check = NULL;
+		rproc->ops->load = NULL;
+	}
+
+	mutex_init(&ksproc->mlock);
+	spin_lock_init(&ksproc->lock);
 
 	ret = keystone_rproc_of_get_dev_syscon(pdev, ksproc);
 	if (ret)
@@ -475,8 +1051,52 @@
 
 	platform_set_drvdata(pdev, ksproc);
 
+	if (ksproc->use_userspace_loader) {
+		uio = &ksproc->uio;
+		uio->name = uio_name;
+		uio->version = DRIVER_UIO_VERSION;
+		uio->irq = ksproc->irq_fault;
+		uio->priv = ksproc;
+		uio->handler = keystone_rproc_uio_handler;
+		uio->irqcontrol	= keystone_rproc_uio_irqcontrol;
+		ret = uio_register_device(dev, uio);
+		if (ret) {
+			dev_err(dev, "failed to register uio device, status = %d\n",
+				ret);
+			goto del_rproc;
+		}
+		dev_dbg(dev, "registered uio device %s\n", uio->name);
+
+		misc = &ksproc->misc;
+		misc->minor = MISC_DYNAMIC_MINOR;
+		misc->name = uio->name;
+		misc->fops = &keystone_rproc_fops;
+		misc->parent = dev;
+		ret = misc_register(misc);
+		if (ret) {
+			dev_err(dev, "failed to register misc device, status = %d\n",
+				ret);
+			goto unregister_uio;
+		}
+
+		ret = keystone_rproc_mem_add_attrs(ksproc);
+		if (ret) {
+			dev_err(ksproc->dev, "error creating sysfs files (%d)\n",
+				ret);
+			goto unregister_misc;
+		}
+
+		dev_dbg(dev, "registered misc device %s\n", misc->name);
+	}
+
 	return 0;
 
+unregister_misc:
+	misc_deregister(misc);
+unregister_uio:
+	uio_unregister_device(uio);
+del_rproc:
+	rproc_del(rproc);
 release_mem:
 	of_reserved_mem_device_release(dev);
 disable_clk:
@@ -492,6 +1112,11 @@
 {
 	struct keystone_rproc *ksproc = platform_get_drvdata(pdev);
 
+	if (ksproc->use_userspace_loader) {
+		keystone_rproc_mem_del_attrs(ksproc);
+		misc_deregister(&ksproc->misc);
+		uio_unregister_device(&ksproc->uio);
+	}
 	rproc_del(ksproc->rproc);
 	pm_runtime_put_sync(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
@@ -519,8 +1144,22 @@
 	},
 };
 
-module_platform_driver(keystone_rproc_driver);
+static int __init keystone_rproc_init(void)
+{
+	keystone_rproc_driver.driver.suppress_bind_attrs =
+		!use_rproc_core_loader;
+
+	return platform_driver_register(&keystone_rproc_driver);
+}
+module_init(keystone_rproc_init);
+
+static void __exit keystone_rproc_exit(void)
+{
+	platform_driver_unregister(&keystone_rproc_driver);
+}
+module_exit(keystone_rproc_exit);
 
 MODULE_AUTHOR("Suman Anna <s-anna@ti.com>");
+MODULE_AUTHOR("Sam Nelson <sam.nelson@ti.com>");
 MODULE_LICENSE("GPL v2");
 MODULE_DESCRIPTION("TI Keystone DSP Remoteproc driver");
diff -Naur --no-dereference a/drivers/remoteproc/Makefile b/drivers/remoteproc/Makefile
--- a/drivers/remoteproc/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/remoteproc/Makefile	2020-06-19 21:00:31.424227716 -0400
@@ -9,11 +9,13 @@
 remoteproc-y				+= remoteproc_sysfs.o
 remoteproc-y				+= remoteproc_virtio.o
 remoteproc-y				+= remoteproc_elf_loader.o
+remoteproc-y				+= remoteproc_elf64_loader.o
 obj-$(CONFIG_IMX_REMOTEPROC)		+= imx_rproc.o
 obj-$(CONFIG_OMAP_REMOTEPROC)		+= omap_remoteproc.o
 obj-$(CONFIG_WKUP_M3_RPROC)		+= wkup_m3_rproc.o
 obj-$(CONFIG_DA8XX_REMOTEPROC)		+= da8xx_remoteproc.o
 obj-$(CONFIG_KEYSTONE_REMOTEPROC)	+= keystone_remoteproc.o
+obj-$(CONFIG_PRU_REMOTEPROC)		+= pru_rproc.o
 obj-$(CONFIG_QCOM_ADSP_PIL)		+= qcom_adsp_pil.o
 obj-$(CONFIG_QCOM_RPROC_COMMON)		+= qcom_common.o
 obj-$(CONFIG_QCOM_Q6V5_COMMON)		+= qcom_q6v5.o
@@ -25,3 +27,5 @@
 qcom_wcnss_pil-y			+= qcom_wcnss_iris.o
 obj-$(CONFIG_ST_REMOTEPROC)		+= st_remoteproc.o
 obj-$(CONFIG_ST_SLIM_REMOTEPROC)	+= st_slim_rproc.o
+obj-$(CONFIG_TI_K3_R5_REMOTEPROC)	+= ti_k3_r5_remoteproc.o
+obj-$(CONFIG_TI_K3_DSP_REMOTEPROC)	+= ti_k3_dsp_remoteproc.o
diff -Naur --no-dereference a/drivers/remoteproc/omap_remoteproc.c b/drivers/remoteproc/omap_remoteproc.c
--- a/drivers/remoteproc/omap_remoteproc.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/remoteproc/omap_remoteproc.c	2020-06-19 21:00:31.424227716 -0400
@@ -1,7 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * OMAP Remote Processor driver
  *
- * Copyright (C) 2011 Texas Instruments, Inc.
+ * Copyright (C) 2011-2019 Texas Instruments Incorporated - http://www.ti.com/
  * Copyright (C) 2011 Google, Inc.
  *
  * Ohad Ben-Cohen <ohad@wizery.com>
@@ -10,44 +11,444 @@
  * Mark Grosen <mgrosen@ti.com>
  * Suman Anna <s-anna@ti.com>
  * Hari Kanigeri <h-kanigeri2@ti.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/err.h>
+#include <linux/io.h>
+#include <linux/of_device.h>
+#include <linux/of_address.h>
+#include <linux/of_reserved_mem.h>
 #include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
 #include <linux/dma-mapping.h>
+#include <linux/interrupt.h>
 #include <linux/remoteproc.h>
 #include <linux/mailbox_client.h>
 #include <linux/omap-mailbox.h>
+#include <linux/omap-iommu.h>
+#include <linux/regmap.h>
+#include <linux/mfd/syscon.h>
+#include <clocksource/timer-ti-dm.h>
 
 #include <linux/platform_data/remoteproc-omap.h>
+#include <linux/platform_data/dmtimer-omap.h>
 
 #include "omap_remoteproc.h"
 #include "remoteproc_internal.h"
 
+#define OMAP_RPROC_DSP_LOCAL_MEM_OFFSET		(0x00800000)
+#define OMAP_RPROC_IPU_L2RAM_DEV_ADDR		(0x20000000)
+
+/* default auto-suspend delay (ms) */
+#define DEFAULT_AUTOSUSPEND_DELAY		10000
+
+/**
+ * struct omap_rproc_boot_data - boot data structure for the DSP omap rprocs
+ * @syscon: regmap handle for the system control configuration module
+ * @boot_reg: boot register offset within the @syscon regmap
+ * @boot_reg_shift: bit-field shift required for the boot address value in
+ *		    @boot_reg
+ */
+struct omap_rproc_boot_data {
+	struct regmap *syscon;
+	unsigned int boot_reg;
+	unsigned int boot_reg_shift;
+};
+
+/**
+ * struct omap_rproc_mem - internal memory structure
+ * @cpu_addr: MPU virtual address of the memory region
+ * @bus_addr: bus address used to access the memory region
+ * @dev_addr: device address of the memory region from DSP view
+ * @size: size of the memory region
+ */
+struct omap_rproc_mem {
+	void __iomem *cpu_addr;
+	phys_addr_t bus_addr;
+	u32 dev_addr;
+	size_t size;
+};
+
+/**
+ * struct omap_rproc_timer - data structure for a timer used by a omap rproc
+ * @odt: timer pointer
+ * @timer_ops: OMAP dmtimer ops for @odt timer
+ * @irq: timer irq
+ */
+struct omap_rproc_timer {
+	struct omap_dm_timer *odt;
+	const struct omap_dm_timer_ops *timer_ops;
+	int irq;
+};
+
 /**
  * struct omap_rproc - omap remote processor state
  * @mbox: mailbox channel handle
  * @client: mailbox client to request the mailbox channel
+ * @boot_data: boot data structure for setting processor boot address
+ * @mem: internal memory regions data
+ * @num_mems: number of internal memory regions
+ * @num_timers: number of rproc timer(s)
+ * @num_wd_timers: number of rproc watchdog timers
+ * @timers: timer(s) info used by rproc
+ * @autosuspend_delay: auto-suspend delay value to be used for runtime pm
+ * @need_resume: if true a resume is needed in the system resume callback
  * @rproc: rproc handle
+ * @pm_comp: completion primitive to sync for suspend response
+ * @standby_addr: kernel address of the register having module standby status
+ * @suspend_acked: state machine flag to store the suspend request ack
  */
 struct omap_rproc {
 	struct mbox_chan *mbox;
 	struct mbox_client client;
+	struct omap_rproc_boot_data *boot_data;
+	struct omap_rproc_mem *mem;
+	int num_mems;
+	int num_timers;
+	int num_wd_timers;
+	struct omap_rproc_timer *timers;
+	int autosuspend_delay;
+	bool need_resume;
 	struct rproc *rproc;
+	struct completion pm_comp;
+	void __iomem *standby_addr;
+	bool suspend_acked;
 };
 
 /**
+ * struct omap_rproc_dev_data - device data for the omap remote processor
+ * @device_name: device name of the remote processor
+ * @fw_name: firmware name to use
+ * @autosuspend_delay: custom auto-suspend delay value in milliseconds
+ */
+struct omap_rproc_dev_data {
+	const char *device_name;
+	const char *fw_name;
+	int autosuspend_delay;
+};
+
+/**
+ * omap_rproc_request_timer - request a timer for a remoteproc
+ * @np: device node pointer to the desired timer
+ * @timer: handle to a struct omap_rproc_timer to return the timer handle
+ *
+ * This helper function is used primarily to request a timer associated with
+ * a remoteproc. The returned handle is stored in the .odt field of the
+ * @timer structure passed in, and is used to invoke other timer specific
+ * ops (like starting a timer either during device initialization or during
+ * a resume operation, or for stopping/freeing a timer).
+ *
+ * Returns 0 on success, otherwise an appropriate failure
+ */
+static int omap_rproc_request_timer(struct device_node *np,
+				    struct omap_rproc_timer *timer)
+{
+	int ret = 0;
+
+	timer->odt = timer->timer_ops->request_by_node(np);
+	if (!timer->odt) {
+		pr_err("request for timer node %p failed\n", np);
+		return -EBUSY;
+	}
+
+	ret = timer->timer_ops->set_source(timer->odt, OMAP_TIMER_SRC_SYS_CLK);
+	if (ret) {
+		pr_err("error setting OMAP_TIMER_SRC_SYS_CLK as source for timer node %p\n",
+		       np);
+		timer->timer_ops->free(timer->odt);
+		return ret;
+	}
+
+	/* clean counter, remoteproc code will set the value */
+	timer->timer_ops->set_load(timer->odt, 0, 0);
+
+	return ret;
+}
+
+/**
+ * omap_rproc_start_timer - start a timer for a remoteproc
+ * @timer: handle to a OMAP rproc timer
+ *
+ * This helper function is used to start a timer associated with a remoteproc,
+ * obtained using the request_timer ops. The helper function needs to be
+ * invoked by the driver to start the timer (during device initialization)
+ * or to just resume the timer.
+ *
+ * Returns 0 on success, otherwise a failure as returned by DMTimer ops
+ */
+static inline int omap_rproc_start_timer(struct omap_rproc_timer *timer)
+{
+	return timer->timer_ops->start(timer->odt);
+}
+
+/**
+ * omap_rproc_stop_timer - stop a timer for a remoteproc
+ * @timer: handle to a OMAP rproc timer
+ *
+ * This helper function is used to disable a timer associated with a
+ * remoteproc, and needs to be called either during a device shutdown
+ * or suspend operation. The separate helper function allows the driver
+ * to just stop a timer without having to release the timer during a
+ * suspend operation.
+ *
+ * Returns 0 on success, otherwise a failure as returned by DMTimer ops
+ */
+static inline int omap_rproc_stop_timer(struct omap_rproc_timer *timer)
+{
+	return timer->timer_ops->stop(timer->odt);
+}
+
+/**
+ * omap_rproc_release_timer - release a timer for a remoteproc
+ * @timer: handle to a OMAP rproc timer
+ *
+ * This helper function is used primarily to release a timer associated
+ * with a remoteproc. The dmtimer will be available for other clients to
+ * use once released.
+ *
+ * Returns 0 on success, otherwise a failure as returned by DMTimer ops
+ */
+static inline int omap_rproc_release_timer(struct omap_rproc_timer *timer)
+{
+	return timer->timer_ops->free(timer->odt);
+}
+
+/**
+ * omap_rproc_get_timer_irq - get the irq for a timer
+ * @timer - handle to a OMAP rproc timer
+ *
+ * This function is used to get the irq associated with a watchdog timer. The
+ * function is called by the OMAP remoteproc driver to register a interrupt
+ * handler to handle watchdog events on the remote processor.
+ *
+ * Returns the irq id on success, otherwise a failure as returned by DMTimer ops
+ */
+static inline int omap_rproc_get_timer_irq(struct omap_rproc_timer *timer)
+{
+	return timer->timer_ops->get_irq(timer->odt);
+}
+
+/**
+ * omap_rproc_ack_timer_irq - acknowledge a timer irq
+ * @timer: handle to a OMAP rproc timer
+ *
+ * This function is used to clear the irq associated with a watchdog timer. The
+ * The function is called by the OMAP remoteproc upon a watchdog event on the
+ * remote processor to clear the interrupt status of the watchdog timer.
+ *
+ * Returns the irq id on success, otherwise a failure as returned by DMTimer ops
+ */
+static inline void omap_rproc_ack_timer_irq(struct omap_rproc_timer *timer)
+{
+	timer->timer_ops->write_status(timer->odt, OMAP_TIMER_INT_OVERFLOW);
+}
+
+/**
+ * omap_rproc_watchdog_isr - Watchdog ISR handler for remoteproc device
+ * @irq: IRQ number associated with a watchdog timer
+ * @data: IRQ handler data
+ *
+ * This ISR routine executes the required necessary low-level code to
+ * acknowledge a watchdog timer interrupt. There can be multiple watchdog
+ * timers associated with a rproc (like IPUs which have 2 watchdog timers,
+ * one per Cortex M3/M4 core), so a lookup has to be performed to identify
+ * the timer to acknowledge its interrupt.
+ *
+ * The function also invokes rproc_report_crash to report the watchdog event
+ * to the remoteproc driver core, to trigger a recovery.
+ *
+ * Return: IRQ_HANDLED or IRQ_NONE
+ */
+static irqreturn_t omap_rproc_watchdog_isr(int irq, void *data)
+{
+	struct rproc *rproc = data;
+	struct omap_rproc *oproc = rproc->priv;
+	struct device *dev = rproc->dev.parent;
+	struct omap_rproc_timer *timers = oproc->timers;
+	struct omap_rproc_timer *wd_timer = NULL;
+	int num_timers = oproc->num_timers + oproc->num_wd_timers;
+	int i;
+
+	for (i = oproc->num_timers; i < num_timers; i++) {
+		if (timers[i].irq > 0 && irq == timers[i].irq) {
+			wd_timer = &timers[i];
+			break;
+		}
+	}
+
+	if (!wd_timer) {
+		dev_err(dev, "invalid timer\n");
+		return IRQ_NONE;
+	}
+
+	omap_rproc_ack_timer_irq(wd_timer);
+
+	rproc_report_crash(rproc, RPROC_WATCHDOG);
+
+	return IRQ_HANDLED;
+}
+
+/**
+ * omap_rproc_enable_timers - enable the timers for a remoteproc
+ * @rproc: handle of a remote processor
+ * @configure: boolean flag used to acquire and configure the timer handle
+ *
+ * This function is used primarily to enable the timers associated with
+ * a remoteproc. The configure flag is provided to allow the driver to
+ * to either acquire and start a timer (during device initialization) or
+ * to just start a timer (during a resume operation).
+ */
+static int omap_rproc_enable_timers(struct rproc *rproc, bool configure)
+{
+	int i;
+	int ret = 0;
+	struct platform_device *tpdev;
+	struct dmtimer_platform_data *tpdata;
+	const struct omap_dm_timer_ops *timer_ops;
+	struct omap_rproc *oproc = rproc->priv;
+	struct omap_rproc_timer *timers = oproc->timers;
+	struct device *dev = rproc->dev.parent;
+	struct device_node *np = NULL;
+	int num_timers = oproc->num_timers + oproc->num_wd_timers;
+
+	if (num_timers <= 0)
+		return 0;
+
+	if (!configure)
+		goto start_timers;
+
+	for (i = 0; i < num_timers; i++) {
+		if (i < oproc->num_timers)
+			np = of_parse_phandle(dev->of_node, "timers", i);
+		else
+			np = of_parse_phandle(dev->of_node, "watchdog-timers",
+					      (i - oproc->num_timers));
+		if (!np) {
+			ret = -ENXIO;
+			dev_err(dev, "device node lookup for timer at index %d failed: %d\n",
+				i < oproc->num_timers ? i :
+				i - oproc->num_timers, ret);
+			goto free_timers;
+		}
+
+		tpdev = of_find_device_by_node(np);
+		if (!tpdev) {
+			ret = -ENODEV;
+			dev_err(dev, "could not get timer platform device\n");
+			goto put_node;
+		}
+
+		tpdata = dev_get_platdata(&tpdev->dev);
+		put_device(&tpdev->dev);
+		if (!tpdata) {
+			ret = -EINVAL;
+			dev_err(dev, "dmtimer pdata structure NULL\n");
+			goto put_node;
+		}
+
+		timer_ops = tpdata->timer_ops;
+		if (!timer_ops || !timer_ops->request_by_node ||
+		    !timer_ops->set_source || !timer_ops->set_load ||
+		    !timer_ops->free || !timer_ops->start ||
+		    !timer_ops->stop || !timer_ops->get_irq ||
+		    !timer_ops->write_status) {
+			ret = -EINVAL;
+			dev_err(dev, "device does not have required timer ops\n");
+			goto put_node;
+		}
+
+		timers[i].irq = -1;
+		timers[i].timer_ops = timer_ops;
+		ret = omap_rproc_request_timer(np, &timers[i]);
+		if (ret) {
+			dev_err(dev, "request for timer %p failed: %d\n", np,
+				ret);
+			goto put_node;
+		}
+		of_node_put(np);
+
+		if (i >= oproc->num_timers) {
+			timers[i].irq = omap_rproc_get_timer_irq(&timers[i]);
+			if (timers[i].irq < 0) {
+				dev_err(dev, "get_irq for timer %p failed: %d\n",
+					np, timers[i].irq);
+				ret = -EBUSY;
+				goto free_timers;
+			}
+
+			ret = request_irq(timers[i].irq,
+					  omap_rproc_watchdog_isr, IRQF_SHARED,
+					  "rproc-wdt", rproc);
+			if (ret) {
+				dev_err(dev, "error requesting irq for timer %p\n",
+					np);
+				omap_rproc_release_timer(&timers[i]);
+				timers[i].odt = NULL;
+				timers[i].timer_ops = NULL;
+				timers[i].irq = -1;
+				goto free_timers;
+			}
+		}
+	}
+
+start_timers:
+	for (i = 0; i < num_timers; i++)
+		omap_rproc_start_timer(&timers[i]);
+	return 0;
+
+put_node:
+	of_node_put(np);
+free_timers:
+	while (i--) {
+		if (i >= oproc->num_timers)
+			free_irq(timers[i].irq, rproc);
+		omap_rproc_release_timer(&timers[i]);
+		timers[i].odt = NULL;
+		timers[i].timer_ops = NULL;
+		timers[i].irq = -1;
+	}
+
+	return ret;
+}
+
+/**
+ * omap_rproc_disable_timers - disable the timers for a remoteproc
+ * @rproc: handle of a remote processor
+ * @configure: boolean flag used to release the timer handle
+ *
+ * This function is used primarily to disable the timers associated with
+ * a remoteproc. The configure flag is provided to allow the driver to
+ * to either stop and release a timer (during device shutdown) or to just
+ * stop a timer (during a suspend operation).
+ */
+static int omap_rproc_disable_timers(struct rproc *rproc, bool configure)
+{
+	int i;
+	struct omap_rproc *oproc = rproc->priv;
+	struct omap_rproc_timer *timers = oproc->timers;
+	int num_timers = oproc->num_timers + oproc->num_wd_timers;
+
+	if (num_timers <= 0)
+		return 0;
+
+	for (i = 0; i < num_timers; i++) {
+		omap_rproc_stop_timer(&timers[i]);
+		if (configure) {
+			if (i >= oproc->num_timers)
+				free_irq(timers[i].irq, rproc);
+			omap_rproc_release_timer(&timers[i]);
+			timers[i].odt = NULL;
+			timers[i].timer_ops = NULL;
+			timers[i].irq = -1;
+		}
+	}
+
+	return 0;
+}
+
+/**
  * omap_rproc_mbox_callback() - inbound mailbox message handler
  * @client: mailbox client pointer used for requesting the mailbox channel
  * @data: mailbox payload
@@ -73,13 +474,28 @@
 
 	switch (msg) {
 	case RP_MBOX_CRASH:
-		/* just log this for now. later, we'll also do recovery */
+		/*
+		 * remoteproc detected an exception, notify the rproc core.
+		 * The remoteproc core will handle the recovery.
+		 */
 		dev_err(dev, "omap rproc %s crashed\n", name);
+		rproc_report_crash(oproc->rproc, RPROC_FATAL_ERROR);
 		break;
 	case RP_MBOX_ECHO_REPLY:
 		dev_info(dev, "received echo reply from %s\n", name);
 		break;
+	case RP_MBOX_SUSPEND_ACK:
+	case RP_MBOX_SUSPEND_CANCEL:
+		oproc->suspend_acked = msg == RP_MBOX_SUSPEND_ACK;
+		complete(&oproc->pm_comp);
+		break;
 	default:
+		if (msg >= RP_MBOX_READY && msg < RP_MBOX_END_MSG)
+			return;
+		if (msg > oproc->rproc->max_notifyid) {
+			dev_dbg(dev, "dropping unknown message 0x%x", msg);
+			return;
+		}
 		/* msg contains the index of the triggered vring */
 		if (rproc_vq_interrupt(oproc->rproc, msg) == IRQ_NONE)
 			dev_dbg(dev, "no message was found in vqid %d\n", msg);
@@ -93,11 +509,52 @@
 	struct device *dev = rproc->dev.parent;
 	int ret;
 
+	/* wake up the rproc before kicking it */
+	ret = pm_runtime_get_sync(dev);
+	if (WARN_ON(ret < 0)) {
+		dev_err(dev, "pm_runtime_get_sync() failed during kick, ret = %d\n",
+			ret);
+		pm_runtime_put_noidle(dev);
+		return;
+	}
+
 	/* send the index of the triggered virtqueue in the mailbox payload */
 	ret = mbox_send_message(oproc->mbox, (void *)vqid);
 	if (ret < 0)
 		dev_err(dev, "failed to send mailbox message, status = %d\n",
 			ret);
+
+	pm_runtime_mark_last_busy(dev);
+	pm_runtime_put_autosuspend(dev);
+}
+
+/**
+ * omap_rproc_write_dsp_boot_addr - set boot address for a DSP remote processor
+ * @rproc: handle of a remote processor
+ *
+ * Set boot address for a supported DSP remote processor.
+ */
+static int omap_rproc_write_dsp_boot_addr(struct rproc *rproc)
+{
+	struct device *dev = rproc->dev.parent;
+	struct omap_rproc *oproc = rproc->priv;
+	struct omap_rproc_boot_data *bdata = oproc->boot_data;
+	u32 offset = bdata->boot_reg;
+	unsigned int value = rproc->bootaddr;
+	unsigned int mask = ~(SZ_1K - 1);
+
+	if (value & (SZ_1K - 1)) {
+		dev_err(dev, "invalid boot address 0x%x, must be aligned on a 1KB boundary\n",
+			value);
+		return -EINVAL;
+	}
+
+	value >>= bdata->boot_reg_shift;
+	mask >>= bdata->boot_reg_shift;
+
+	regmap_update_bits(bdata->syscon, offset, mask, value);
+
+	return 0;
 }
 
 /*
@@ -116,8 +573,15 @@
 	int ret;
 	struct mbox_client *client = &oproc->client;
 
-	if (pdata->set_bootaddr)
-		pdata->set_bootaddr(rproc->bootaddr);
+	/*
+	 * We set boot address irrespective of the value of the late attach flag
+	 * as boot address takes effect only on a deassert of remoteproc reset.
+	 */
+	if (oproc->boot_data) {
+		ret = omap_rproc_write_dsp_boot_addr(rproc);
+		if (ret)
+			return ret;
+	}
 
 	client->dev = dev;
 	client->tx_done = NULL;
@@ -125,7 +589,7 @@
 	client->tx_block = false;
 	client->knows_txdone = false;
 
-	oproc->mbox = omap_mbox_request_channel(client, pdata->mbox_name);
+	oproc->mbox = mbox_request_channel(client, 0);
 	if (IS_ERR(oproc->mbox)) {
 		ret = -EBUSY;
 		dev_err(dev, "mbox_request_channel failed: %ld\n",
@@ -146,14 +610,37 @@
 		goto put_mbox;
 	}
 
-	ret = pdata->device_enable(pdev);
+	ret = omap_rproc_enable_timers(rproc, true);
 	if (ret) {
-		dev_err(dev, "omap_device_enable failed: %d\n", ret);
+		dev_err(dev, "omap_rproc_enable_timers failed: %d\n", ret);
 		goto put_mbox;
 	}
 
+	if (!rproc->late_attach) {
+		ret = pdata->device_enable(pdev);
+		if (ret) {
+			dev_err(dev, "omap_device_enable failed: %d\n", ret);
+			goto reset_timers;
+		}
+	}
+
+	/*
+	 * remote processor is up, so update the runtime pm status and
+	 * enable the auto-suspend. The device usage count is incremented
+	 * manually for balancing it for auto-suspend
+	 */
+	pm_runtime_set_active(dev);
+	pm_runtime_set_autosuspend_delay(dev, oproc->autosuspend_delay);
+	pm_runtime_use_autosuspend(dev);
+	pm_runtime_get_noresume(dev);
+	pm_runtime_enable(dev);
+	pm_runtime_mark_last_busy(dev);
+	pm_runtime_put_autosuspend(dev);
+
 	return 0;
 
+reset_timers:
+	omap_rproc_disable_timers(rproc, true);
 put_mbox:
 	mbox_free_channel(oproc->mbox);
 	return ret;
@@ -168,53 +655,772 @@
 	struct omap_rproc *oproc = rproc->priv;
 	int ret;
 
+	/*
+	 * cancel any possible scheduled runtime suspend by incrementing
+	 * the device usage count, and resuming the device. The remoteproc
+	 * also needs to be woken up if suspended, to avoid the remoteproc
+	 * OS to continue to remember any context that it has saved, and
+	 * avoid potential issues in misindentifying a subsequent device
+	 * reboot as a power restore boot
+	 */
+	ret = pm_runtime_get_sync(dev);
+	if (ret < 0) {
+		pm_runtime_put_noidle(dev);
+		return ret;
+	}
+
 	ret = pdata->device_shutdown(pdev);
 	if (ret)
-		return ret;
+		goto out;
+
+	ret = omap_rproc_disable_timers(rproc, true);
+	if (ret)
+		goto enable_device;
+
+	/*
+	 * During late attach, we use non-zeroing dma ops to prevent the kernel
+	 * from overwriting already loaded code and data segments. When
+	 * shutting down the processor, we restore the normal zeroing dma ops.
+	 * This allows the kernel to clear memory when loading a new remoteproc
+	 * binary or during error recovery with the current remoteproc binary.
+	 */
+	if (rproc->late_attach)
+		set_dma_ops(dev, &arm_dma_ops);
 
 	mbox_free_channel(oproc->mbox);
 
+	/*
+	 * update the runtime pm states and status now that the remoteproc
+	 * has stopped
+	 */
+	pm_runtime_disable(dev);
+	pm_runtime_dont_use_autosuspend(dev);
+	pm_runtime_put_noidle(dev);
+	pm_runtime_set_suspended(dev);
+
 	return 0;
+
+enable_device:
+	pdata->device_enable(pdev);
+out:
+	/* schedule the next auto-suspend */
+	pm_runtime_mark_last_busy(dev);
+	pm_runtime_put_autosuspend(dev);
+	return ret;
+
+}
+
+/*
+ * Internal Memory translation helper
+ *
+ * Custom function implementing the rproc .da_to_va ops to provide address
+ * translation (device address to kernel virtual address) for internal RAMs
+ * present in a DSP or IPU device). The translated addresses can be used
+ * either by the remoteproc core for loading, or by any rpmsg bus drivers.
+ */
+static void *omap_rproc_da_to_va(struct rproc *rproc, u64 da, int len,
+				 u32 flags)
+{
+	struct omap_rproc *oproc = rproc->priv;
+	void *va = NULL;
+	int i;
+	u32 offset;
+
+	if (len <= 0)
+		return NULL;
+
+	if (!oproc->num_mems)
+		return NULL;
+
+	for (i = 0; i < oproc->num_mems; i++) {
+		if (da >= oproc->mem[i].dev_addr && da + len <=
+		    oproc->mem[i].dev_addr +  oproc->mem[i].size) {
+			offset = da -  oproc->mem[i].dev_addr;
+			/* __force to make sparse happy with type conversion */
+			va = (__force void *)(oproc->mem[i].cpu_addr + offset);
+			break;
+		}
+	}
+
+	return va;
 }
 
 static const struct rproc_ops omap_rproc_ops = {
 	.start		= omap_rproc_start,
 	.stop		= omap_rproc_stop,
 	.kick		= omap_rproc_kick,
+	.da_to_va	= omap_rproc_da_to_va,
 };
 
+#ifdef CONFIG_PM
+static bool _is_rproc_in_standby(struct omap_rproc *oproc)
+{
+	static int standby_mask = (1 << 18);
+
+	return readl(oproc->standby_addr) & standby_mask;
+}
+
+/* 1 sec is long enough time to let the remoteproc side suspend the device */
+#define DEF_SUSPEND_TIMEOUT 1000
+static int _omap_rproc_suspend(struct rproc *rproc, bool auto_suspend)
+{
+	struct device *dev = rproc->dev.parent;
+	struct platform_device *pdev = to_platform_device(dev);
+	struct omap_rproc_pdata *pdata = dev_get_platdata(dev);
+	struct omap_rproc *oproc = rproc->priv;
+	unsigned long to = msecs_to_jiffies(DEF_SUSPEND_TIMEOUT);
+	unsigned long ta = jiffies + to;
+	u32 suspend_msg = auto_suspend ?
+				RP_MBOX_SUSPEND_AUTO : RP_MBOX_SUSPEND_SYSTEM;
+	int ret;
+
+	reinit_completion(&oproc->pm_comp);
+	oproc->suspend_acked = false;
+	ret = mbox_send_message(oproc->mbox, (void *)suspend_msg);
+	if (ret < 0) {
+		dev_err(dev, "PM mbox_send_message failed: %d\n", ret);
+		return ret;
+	}
+
+	ret = wait_for_completion_timeout(&oproc->pm_comp, to);
+	if (!oproc->suspend_acked)
+		return -EBUSY;
+
+	/*
+	 * The remoteproc side is returning the ACK message before saving the
+	 * context, because the context saving is performed within a SYS/BIOS
+	 * function, and it cannot have any inter-dependencies against the IPC
+	 * layer. Also, as the SYS/BIOS needs to preserve properly the processor
+	 * register set, sending this ACK or signalling the completion of the
+	 * context save through a shared memory variable can never be the
+	 * absolute last thing to be executed on the remoteproc side, and the
+	 * MPU cannot use the ACK message as a sync point to put the remoteproc
+	 * into reset. The only way to ensure that the remote processor has
+	 * completed saving the context is to check that the module has reached
+	 * STANDBY state (after saving the context, the SYS/BIOS executes the
+	 * appropriate target-specific WFI instruction causing the module to
+	 * enter STANDBY).
+	 */
+	while (!_is_rproc_in_standby(oproc)) {
+		if (time_after(jiffies, ta))
+			return -ETIME;
+		schedule();
+	}
+
+	ret = pdata->device_shutdown(pdev);
+	if (ret)
+		return ret;
+
+	ret = omap_rproc_disable_timers(rproc, false);
+	if (ret) {
+		dev_err(dev, "disabling timers during suspend failed %d\n",
+			ret);
+		goto enable_device;
+	}
+
+	/*
+	 * IOMMUs would have to be disabled specifically for runtime suspend.
+	 * They are handled automatically through System PM callbacks for
+	 * regular system suspend
+	 */
+	if (auto_suspend) {
+		ret = omap_iommu_domain_deactivate(rproc->domain);
+		if (ret) {
+			dev_err(dev, "iommu domain deactivate failed %d\n",
+				ret);
+			goto enable_timers;
+		}
+	}
+
+	return 0;
+
+enable_timers:
+	/* ignore errors on re-enabling code */
+	omap_rproc_enable_timers(rproc, false);
+enable_device:
+	pdata->device_enable(pdev);
+	return ret;
+}
+
+static int _omap_rproc_resume(struct rproc *rproc, bool auto_suspend)
+{
+	struct device *dev = rproc->dev.parent;
+	struct platform_device *pdev = to_platform_device(dev);
+	struct omap_rproc_pdata *pdata = dev_get_platdata(dev);
+	struct omap_rproc *oproc = rproc->priv;
+	int ret;
+
+	/*
+	 * IOMMUs would have to be enabled specifically for runtime resume.
+	 * They would have been already enabled automatically through System
+	 * PM callbacks for regular system resume
+	 */
+	if (auto_suspend) {
+		ret = omap_iommu_domain_activate(rproc->domain);
+		if (ret) {
+			dev_err(dev, "omap_iommu activate failed %d\n", ret);
+			goto out;
+		}
+	}
+
+	/* boot address could be lost after suspend, so restore it */
+	if (oproc->boot_data) {
+		ret = omap_rproc_write_dsp_boot_addr(rproc);
+		if (ret) {
+			dev_err(dev, "boot address restore failed %d\n", ret);
+			goto suspend_iommu;
+		}
+	}
+
+	ret = omap_rproc_enable_timers(rproc, false);
+	if (ret) {
+		dev_err(dev, "enabling timers during resume failed %d\n",
+			ret);
+		goto suspend_iommu;
+	}
+
+	ret = pdata->device_enable(pdev);
+	if (ret)
+		goto disable_timers;
+
+	return 0;
+
+disable_timers:
+	omap_rproc_disable_timers(rproc, false);
+suspend_iommu:
+	if (auto_suspend)
+		omap_iommu_domain_deactivate(rproc->domain);
+out:
+	return ret;
+}
+
+static int __maybe_unused omap_rproc_suspend(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct rproc *rproc = platform_get_drvdata(pdev);
+	struct omap_rproc *oproc = rproc->priv;
+	int ret = 0;
+
+	mutex_lock(&rproc->lock);
+	if (rproc->state == RPROC_OFFLINE)
+		goto out;
+
+	if (rproc->state == RPROC_SUSPENDED)
+		goto out;
+
+	if (rproc->state != RPROC_RUNNING) {
+		ret = -EBUSY;
+		goto out;
+	}
+
+	ret = _omap_rproc_suspend(rproc, false);
+	if (ret) {
+		dev_err(dev, "suspend failed %d\n", ret);
+		goto out;
+	}
+
+	/*
+	 * remoteproc is running at the time of system suspend, so remember
+	 * it so as to wake it up during system resume
+	 */
+	oproc->need_resume = 1;
+	rproc->state = RPROC_SUSPENDED;
+
+	/*
+	 * update the runtime pm status to be suspended, without decrementing
+	 * the device usage count
+	 */
+	pm_runtime_disable(dev);
+	pm_runtime_set_suspended(dev);
+out:
+	mutex_unlock(&rproc->lock);
+	return ret;
+}
+
+static int __maybe_unused omap_rproc_resume(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct rproc *rproc = platform_get_drvdata(pdev);
+	struct omap_rproc *oproc = rproc->priv;
+	int ret = 0;
+
+	mutex_lock(&rproc->lock);
+	if (rproc->state == RPROC_OFFLINE)
+		goto out;
+
+	if (rproc->state != RPROC_SUSPENDED) {
+		ret = -EBUSY;
+		goto out;
+	}
+
+	/*
+	 * remoteproc was auto-suspended at the time of system suspend,
+	 * so no need to wake-up the processor (leave it in suspended
+	 * state, will be woken up during a subsequent runtime_resume)
+	 */
+	if (!oproc->need_resume)
+		goto out;
+
+	ret = _omap_rproc_resume(rproc, false);
+	if (ret) {
+		dev_err(dev, "resume failed %d\n", ret);
+		goto out;
+	}
+	oproc->need_resume = false;
+
+	rproc->state = RPROC_RUNNING;
+
+	/*
+	 * update the runtime pm status to be active, without incrementing
+	 * the device usage count
+	 */
+	pm_runtime_set_active(dev);
+	pm_runtime_enable(dev);
+	pm_runtime_mark_last_busy(dev);
+out:
+	mutex_unlock(&rproc->lock);
+	return ret;
+}
+
+static int omap_rproc_runtime_suspend(struct device *dev)
+{
+	struct rproc *rproc = dev_get_drvdata(dev);
+	struct omap_rproc *oproc = rproc->priv;
+	int ret;
+
+	if (rproc->state == RPROC_CRASHED) {
+		dev_dbg(dev, "rproc cannot be runtime suspended when crashed!\n");
+		return -EBUSY;
+	}
+
+	if (WARN_ON(rproc->state != RPROC_RUNNING)) {
+		dev_err(dev, "rproc cannot be runtime suspended when not running!\n");
+		return -EBUSY;
+	}
+
+	/*
+	 * do not even attempt suspend if the remote processor is not
+	 * idled for runtime auto-suspend
+	 */
+	if (!_is_rproc_in_standby(oproc)) {
+		ret = -EBUSY;
+		goto abort;
+	}
+
+	ret = _omap_rproc_suspend(rproc, true);
+	if (ret)
+		goto abort;
+
+	rproc->state = RPROC_SUSPENDED;
+	return 0;
+
+abort:
+	pm_runtime_mark_last_busy(dev);
+	return ret;
+}
+
+static int omap_rproc_runtime_resume(struct device *dev)
+{
+	struct rproc *rproc = dev_get_drvdata(dev);
+	int ret;
+
+	if (WARN_ON(rproc->state != RPROC_SUSPENDED)) {
+		dev_err(dev, "rproc cannot be runtime resumed if not suspended!\n");
+		return -EBUSY;
+	}
+
+	ret = _omap_rproc_resume(rproc, true);
+	if (ret) {
+		dev_err(dev, "runtime resume failed %d\n", ret);
+		return ret;
+	}
+
+	rproc->state = RPROC_RUNNING;
+	return 0;
+}
+#endif /* CONFIG_PM */
+
+static const struct omap_rproc_dev_data omap4_dsp_dev_data = {
+	.device_name	= "dsp",
+	.fw_name	= "omap4-dsp-fw.xe64T",
+};
+
+static const struct omap_rproc_dev_data omap4_ipu_dev_data = {
+	.device_name	= "ipu",
+	.fw_name	= "omap4-ipu-fw.xem3",
+};
+
+static const struct omap_rproc_dev_data omap5_dsp_dev_data = {
+	.device_name	= "dsp",
+	.fw_name	= "omap5-dsp-fw.xe64T",
+};
+
+static const struct omap_rproc_dev_data omap5_ipu_dev_data = {
+	.device_name	= "ipu",
+	.fw_name	= "omap5-ipu-fw.xem4",
+};
+
+static const struct omap_rproc_dev_data dra7_rproc_dev_data[] = {
+	{
+		.device_name	= "40800000.dsp",
+		.fw_name	= "dra7-dsp1-fw.xe66",
+	},
+	{
+		.device_name	= "41000000.dsp",
+		.fw_name	= "dra7-dsp2-fw.xe66",
+	},
+	{
+		.device_name	= "55020000.ipu",
+		.fw_name	= "dra7-ipu2-fw.xem4",
+	},
+	{
+		.device_name	= "58820000.ipu",
+		.fw_name	= "dra7-ipu1-fw.xem4",
+	},
+	{
+		/* sentinel */
+	},
+};
+
+static const struct of_device_id omap_rproc_of_match[] = {
+	{
+		.compatible     = "ti,omap4-dsp",
+		.data           = &omap4_dsp_dev_data,
+	},
+	{
+		.compatible     = "ti,omap4-ipu",
+		.data           = &omap4_ipu_dev_data,
+	},
+	{
+		.compatible     = "ti,omap5-dsp",
+		.data           = &omap5_dsp_dev_data,
+	},
+	{
+		.compatible     = "ti,omap5-ipu",
+		.data           = &omap5_ipu_dev_data,
+	},
+	{
+		.compatible     = "ti,dra7-dsp",
+		.data           = dra7_rproc_dev_data,
+	},
+	{
+		.compatible     = "ti,dra7-ipu",
+		.data           = dra7_rproc_dev_data,
+	},
+	{
+		/* end */
+	},
+};
+MODULE_DEVICE_TABLE(of, omap_rproc_of_match);
+
+static int omap_rproc_get_autosuspend_delay(struct platform_device *pdev)
+{
+	struct device_node *np = pdev->dev.of_node;
+	const struct omap_rproc_dev_data *data;
+	int delay = -EINVAL;
+
+	data = of_device_get_match_data(&pdev->dev);
+	if (!data)
+		return -ENODEV;
+
+	if (!of_device_is_compatible(np, "ti,dra7-dsp") &&
+	    !of_device_is_compatible(np, "ti,dra7-ipu")) {
+		delay = data->autosuspend_delay;
+		goto out;
+	}
+
+	for (; data && data->device_name; data++) {
+		if (!strcmp(dev_name(&pdev->dev), data->device_name)) {
+			delay = data->autosuspend_delay;
+			break;
+		}
+	}
+
+out:
+	return (delay > 0) ? delay : DEFAULT_AUTOSUSPEND_DELAY;
+}
+
+static const char *omap_rproc_get_firmware(struct platform_device *pdev)
+{
+	struct device_node *np = pdev->dev.of_node;
+	const struct omap_rproc_dev_data *data;
+
+	data = of_device_get_match_data(&pdev->dev);
+	if (!data)
+		return ERR_PTR(-ENODEV);
+
+	if (!of_device_is_compatible(np, "ti,dra7-dsp") &&
+	    !of_device_is_compatible(np, "ti,dra7-ipu"))
+		return data->fw_name;
+
+	for (; data && data->device_name; data++) {
+		if (!strcmp(dev_name(&pdev->dev), data->device_name))
+			return data->fw_name;
+	}
+
+	return ERR_PTR(-ENOENT);
+}
+
+static int omap_rproc_get_boot_data(struct platform_device *pdev,
+				    struct rproc *rproc)
+{
+	struct device_node *np = pdev->dev.of_node;
+	struct omap_rproc *oproc = rproc->priv;
+	int ret;
+
+	if (!of_device_is_compatible(np, "ti,omap4-dsp") &&
+	    !of_device_is_compatible(np, "ti,omap5-dsp") &&
+	    !of_device_is_compatible(np, "ti,dra7-dsp"))
+		return 0;
+
+	oproc->boot_data = devm_kzalloc(&pdev->dev, sizeof(*oproc->boot_data),
+					GFP_KERNEL);
+	if (!oproc->boot_data)
+		return -ENOMEM;
+
+	if (!of_property_read_bool(np, "syscon-bootreg")) {
+		dev_err(&pdev->dev, "syscon-bootreg property is missing\n");
+		return -EINVAL;
+	}
+
+	oproc->boot_data->syscon =
+			syscon_regmap_lookup_by_phandle(np, "syscon-bootreg");
+	if (IS_ERR(oproc->boot_data->syscon)) {
+		ret = PTR_ERR(oproc->boot_data->syscon);
+		return ret;
+	}
+
+	if (of_property_read_u32_index(np, "syscon-bootreg", 1,
+				       &oproc->boot_data->boot_reg)) {
+		dev_err(&pdev->dev, "couldn't get the boot register\n");
+		return -EINVAL;
+	}
+
+	if (of_device_is_compatible(np, "ti,dra7-dsp"))
+		oproc->boot_data->boot_reg_shift = 10;
+
+	return 0;
+}
+
+static int omap_rproc_of_get_internal_memories(struct platform_device *pdev,
+					       struct rproc *rproc)
+{
+	static const char * const ipu_mem_names[] = {"l2ram"};
+	static const char * const dra7_dsp_mem_names[] = {"l2ram", "l1pram",
+								"l1dram"};
+	struct device_node *np = pdev->dev.of_node;
+	struct omap_rproc *oproc = rproc->priv;
+	struct device *dev = &pdev->dev;
+	const char * const *mem_names;
+	struct resource *res;
+	int num_mems;
+	const __be32 *addrp;
+	u32 l4_offset = 0;
+	u64 size;
+	int i;
+
+	/* OMAP4 and OMAP5 DSPs do not have support for flat SRAM */
+	if (of_device_is_compatible(np, "ti,omap4-dsp") ||
+	    of_device_is_compatible(np, "ti,omap5-dsp"))
+		return 0;
+
+	/* DRA7 DSPs have two additional SRAMs at L1 level */
+	if (of_device_is_compatible(np, "ti,dra7-dsp")) {
+		mem_names = dra7_dsp_mem_names;
+		num_mems = ARRAY_SIZE(dra7_dsp_mem_names);
+	} else {
+		mem_names = ipu_mem_names;
+		num_mems = ARRAY_SIZE(ipu_mem_names);
+	}
+
+	oproc->mem = devm_kcalloc(dev, num_mems, sizeof(*oproc->mem),
+				  GFP_KERNEL);
+	if (!oproc->mem)
+		return -ENOMEM;
+
+	for (i = 0; i < num_mems; i++) {
+		res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+						   mem_names[i]);
+		oproc->mem[i].cpu_addr = devm_ioremap_resource(dev, res);
+		if (IS_ERR(oproc->mem[i].cpu_addr)) {
+			dev_err(dev, "failed to parse and map %s memory\n",
+				mem_names[i]);
+			return PTR_ERR(oproc->mem[i].cpu_addr);
+		}
+		oproc->mem[i].bus_addr = res->start;
+
+		/*
+		 * The DSPs have the internal memories starting at a fixed
+		 * offset of 0x800000 from address 0, and this corresponds to
+		 * L2RAM. The L3 address view has the L2RAM bus address as the
+		 * starting address for the IP, so the L2RAM memory region needs
+		 * to be processed first, and the device addresses for each
+		 * memory region can be computed using the relative offset
+		 * from this base address.
+		 */
+		if (of_device_is_compatible(np, "ti,dra7-dsp") &&
+		    !strcmp(mem_names[i], "l2ram")) {
+			addrp = of_get_address(dev->of_node, i, &size, NULL);
+			l4_offset = be32_to_cpu(*addrp);
+		}
+		oproc->mem[i].dev_addr =
+			of_device_is_compatible(np, "ti,dra7-dsp") ?
+				res->start - l4_offset +
+				OMAP_RPROC_DSP_LOCAL_MEM_OFFSET :
+				OMAP_RPROC_IPU_L2RAM_DEV_ADDR;
+		oproc->mem[i].size = resource_size(res);
+
+		dev_dbg(dev, "memory %8s: bus addr %pa size 0x%x va %p da 0x%x\n",
+			mem_names[i], &oproc->mem[i].bus_addr,
+			oproc->mem[i].size, oproc->mem[i].cpu_addr,
+			oproc->mem[i].dev_addr);
+	}
+	oproc->num_mems = num_mems;
+
+	return 0;
+}
+
 static int omap_rproc_probe(struct platform_device *pdev)
 {
-	struct omap_rproc_pdata *pdata = pdev->dev.platform_data;
+	struct omap_rproc_pdata *pdata = dev_get_platdata(&pdev->dev);
+	struct device_node *np = pdev->dev.of_node;
 	struct omap_rproc *oproc;
 	struct rproc *rproc;
+	const char *firmware;
+	u32 standby_addr = 0;
+	int num_timers;
 	int ret;
 
+	if (!np) {
+		dev_err(&pdev->dev, "only DT-based devices are supported\n");
+		return -ENODEV;
+	}
+
+	/*
+	 * self-manage the ordering dependencies between omap_device_enable/idle
+	 * and omap_device_assert/deassert_hardreset API during runtime suspend
+	 * and resume, rather than relying on the order in omap_device layer.
+	 */
+	if (pdev->dev.pm_domain) {
+		dev_dbg(&pdev->dev, "device pm_domain is being reset for this remoteproc device\n");
+		pdev->dev.pm_domain = NULL;
+	}
+
+	if (!pdata || !pdata->device_enable || !pdata->device_shutdown) {
+		dev_err(&pdev->dev, "platform data is either missing or incomplete\n");
+		return -ENODEV;
+	}
+
+	firmware = omap_rproc_get_firmware(pdev);
+	if (IS_ERR(firmware))
+		return PTR_ERR(firmware);
+
 	ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
 	if (ret) {
 		dev_err(&pdev->dev, "dma_set_coherent_mask: %d\n", ret);
 		return ret;
 	}
 
-	rproc = rproc_alloc(&pdev->dev, pdata->name, &omap_rproc_ops,
-			    pdata->firmware, sizeof(*oproc));
+	rproc = rproc_alloc(&pdev->dev, dev_name(&pdev->dev), &omap_rproc_ops,
+			    firmware, sizeof(*oproc));
 	if (!rproc)
 		return -ENOMEM;
 
+	if (pdata->device_is_enabled && pdata->device_is_enabled(pdev)) {
+		rproc->late_attach = 1;
+		set_dma_ops(&pdev->dev, &arm_dma_m_ops);
+	}
+
 	oproc = rproc->priv;
 	oproc->rproc = rproc;
 	/* All existing OMAP IPU and DSP processors have an MMU */
 	rproc->has_iommu = true;
 
+	ret = omap_rproc_of_get_internal_memories(pdev, rproc);
+	if (ret)
+		goto free_rproc;
+
+	ret = omap_rproc_get_boot_data(pdev, rproc);
+	if (ret)
+		goto free_rproc;
+
+	/*
+	 * Timer nodes are directly used in client nodes as phandles, so
+	 * retrieve the count using appropriate size
+	 */
+	oproc->num_timers = of_property_count_elems_of_size(np, "timers",
+							    sizeof(phandle));
+	if (oproc->num_timers <= 0) {
+		dev_dbg(&pdev->dev, "device does not have timers, status = %d\n",
+			oproc->num_timers);
+		oproc->num_timers = 0;
+	}
+
+#ifdef CONFIG_OMAP_REMOTEPROC_WATCHDOG
+	oproc->num_wd_timers = of_count_phandle_with_args(np, "watchdog-timers",
+							  NULL);
+	if (oproc->num_wd_timers <= 0) {
+		dev_dbg(&pdev->dev, "device does not have watchdog timers, status = %d\n",
+			oproc->num_wd_timers);
+		oproc->num_wd_timers = 0;
+	}
+#endif
+
+	if (oproc->num_timers || oproc->num_wd_timers) {
+		num_timers = oproc->num_timers + oproc->num_wd_timers;
+		oproc->timers = devm_kzalloc(&pdev->dev, sizeof(*oproc->timers)
+					     * num_timers, GFP_KERNEL);
+		if (!oproc->timers) {
+			ret = -ENOMEM;
+			goto free_rproc;
+		}
+
+		dev_dbg(&pdev->dev, "device has %d tick timers and %d watchdog timers\n",
+			oproc->num_timers, oproc->num_wd_timers);
+	}
+
+	init_completion(&oproc->pm_comp);
+	oproc->autosuspend_delay = omap_rproc_get_autosuspend_delay(pdev);
+	if (oproc->autosuspend_delay < 0) {
+		ret = oproc->autosuspend_delay;
+		goto free_rproc;
+	}
+
+	ret = of_property_read_u32(np, "ti,rproc-standby-info", &standby_addr);
+	if (ret || !standby_addr) {
+		ret = !standby_addr ? -EINVAL : ret;
+		goto free_rproc;
+	}
+
+	oproc->standby_addr = devm_ioremap(&pdev->dev, standby_addr,
+					   sizeof(u32));
+	if (!oproc->standby_addr) {
+		ret = -ENOMEM;
+		goto free_rproc;
+	}
+
+	ret = of_reserved_mem_device_init(&pdev->dev);
+	if (ret) {
+		dev_err(&pdev->dev, "device does not have specific CMA pool\n");
+		goto free_rproc;
+	}
+
 	platform_set_drvdata(pdev, rproc);
 
 	ret = rproc_add(rproc);
 	if (ret)
-		goto free_rproc;
+		goto release_mem;
+
+	if (rproc_get_id(rproc) < 0)
+		dev_warn(&pdev->dev, "device does not have an alias id\n");
 
 	return 0;
 
+release_mem:
+	of_reserved_mem_device_release(&pdev->dev);
 free_rproc:
+	if (rproc->late_attach)
+		set_dma_ops(&pdev->dev, &arm_dma_ops);
 	rproc_free(rproc);
 	return ret;
 }
@@ -225,15 +1431,24 @@
 
 	rproc_del(rproc);
 	rproc_free(rproc);
+	of_reserved_mem_device_release(&pdev->dev);
 
 	return 0;
 }
 
+static const struct dev_pm_ops omap_rproc_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(omap_rproc_suspend, omap_rproc_resume)
+	SET_RUNTIME_PM_OPS(omap_rproc_runtime_suspend,
+			   omap_rproc_runtime_resume, NULL)
+};
+
 static struct platform_driver omap_rproc_driver = {
 	.probe = omap_rproc_probe,
 	.remove = omap_rproc_remove,
 	.driver = {
 		.name = "omap-rproc",
+		.pm = &omap_rproc_pm_ops,
+		.of_match_table = omap_rproc_of_match,
 	},
 };
 
diff -Naur --no-dereference a/drivers/remoteproc/omap_remoteproc.h b/drivers/remoteproc/omap_remoteproc.h
--- a/drivers/remoteproc/omap_remoteproc.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/remoteproc/omap_remoteproc.h	2020-06-19 21:00:31.424227716 -0400
@@ -1,35 +1,10 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */
 /*
  * Remote processor messaging
  *
- * Copyright (C) 2011 Texas Instruments, Inc.
+ * Copyright (C) 2011-2019 Texas Instruments, Inc.
  * Copyright (C) 2011 Google, Inc.
  * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- *   notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- *   notice, this list of conditions and the following disclaimer in
- *   the documentation and/or other materials provided with the
- *   distribution.
- * * Neither the name Texas Instruments nor the names of its
- *   contributors may be used to endorse or promote products derived
- *   from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
 #ifndef _OMAP_RPMSG_H
@@ -56,6 +31,22 @@
  *
  * @RP_MBOX_ABORT_REQUEST: a "please crash" request, used for testing the
  * recovery mechanism (to some extent).
+ *
+ * @RP_MBOX_SUSPEND_AUTO: auto suspend request for the remote processor
+ *
+ * @RP_MBOX_SUSPEND_SYSTEM: system suspend request for the remote processor
+ *
+ * @RP_MBOX_SUSPEND_ACK: successful response from remote processor for a
+ * suspend request
+ *
+ * @RP_MBOX_SUSPEND_CANCEL: a cancel suspend response from a remote processor
+ * on a suspend request
+ *
+ * Introduce new message definitions if any here.
+ *
+ * @RP_MBOX_END_MSG: Indicates end of known/defined messages from remote core
+ * This should be the last definition.
+ *
  */
 enum omap_rp_mbox_messages {
 	RP_MBOX_READY		= 0xFFFFFF00,
@@ -64,6 +55,11 @@
 	RP_MBOX_ECHO_REQUEST	= 0xFFFFFF03,
 	RP_MBOX_ECHO_REPLY	= 0xFFFFFF04,
 	RP_MBOX_ABORT_REQUEST	= 0xFFFFFF05,
+	RP_MBOX_SUSPEND_AUTO	= 0xFFFFFF10,
+	RP_MBOX_SUSPEND_SYSTEM	= 0xFFFFFF11,
+	RP_MBOX_SUSPEND_ACK	= 0xFFFFFF12,
+	RP_MBOX_SUSPEND_CANCEL	= 0xFFFFFF13,
+	RP_MBOX_END_MSG		= 0xFFFFFF14,
 };
 
 #endif /* _OMAP_RPMSG_H */
diff -Naur --no-dereference a/drivers/remoteproc/pru_rproc.c b/drivers/remoteproc/pru_rproc.c
--- a/drivers/remoteproc/pru_rproc.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/remoteproc/pru_rproc.c	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,1340 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * PRU-ICSS remoteproc driver for various TI SoCs
+ *
+ * Copyright (C) 2014-2019 Texas Instruments Incorporated - http://www.ti.com/
+ *	Suman Anna <s-anna@ti.com>
+ *	Andrew F. Davis <afd@ti.com>
+ */
+
+#include <linux/bitops.h>
+#include <linux/debugfs.h>
+#include <linux/interrupt.h>
+#include <linux/mailbox_client.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/omap-mailbox.h>
+#include <linux/pruss.h>
+#include <linux/pruss_driver.h>
+#include <linux/remoteproc.h>
+
+#include "remoteproc_internal.h"
+#include "pru_rproc.h"
+
+/* PRU_ICSS_PRU_CTRL registers */
+#define PRU_CTRL_CTRL		0x0000
+#define PRU_CTRL_STS		0x0004
+#define PRU_CTRL_WAKEUP_EN	0x0008
+#define PRU_CTRL_CYCLE		0x000C
+#define PRU_CTRL_STALL		0x0010
+#define PRU_CTRL_CTBIR0		0x0020
+#define PRU_CTRL_CTBIR1		0x0024
+#define PRU_CTRL_CTPPR0		0x0028
+#define PRU_CTRL_CTPPR1		0x002C
+
+/* CTRL register bit-fields */
+#define CTRL_CTRL_SOFT_RST_N	BIT(0)
+#define CTRL_CTRL_EN		BIT(1)
+#define CTRL_CTRL_SLEEPING	BIT(2)
+#define CTRL_CTRL_CTR_EN	BIT(3)
+#define CTRL_CTRL_SINGLE_STEP	BIT(8)
+#define CTRL_CTRL_RUNSTATE	BIT(15)
+
+/* PRU_ICSS_PRU_DEBUG registers */
+#define PRU_DEBUG_GPREG(x)	(0x0000 + (x) * 4)
+#define PRU_DEBUG_CT_REG(x)	(0x0080 + (x) * 4)
+
+/* PRU/RTU Core IRAM address masks */
+#define PRU0_IRAM_ADDR_MASK	0x34000
+#define PRU1_IRAM_ADDR_MASK	0x38000
+#define RTU0_IRAM_ADDR_MASK	0x4000
+#define RTU1_IRAM_ADDR_MASK	0x6000
+#define TX_PRU0_IRAM_ADDR_MASK	0xa000
+#define TX_PRU1_IRAM_ADDR_MASK	0xc000
+
+/**
+ * enum pru_iomem - PRU core memory/register range identifiers
+ */
+enum pru_iomem {
+	PRU_IOMEM_IRAM = 0,
+	PRU_IOMEM_CTRL,
+	PRU_IOMEM_DEBUG,
+	PRU_IOMEM_MAX,
+};
+
+/**
+ * enum pru_type - PRU core type identifier
+ */
+enum pru_type {
+	PRU_TYPE_PRU = 0,
+	PRU_TYPE_RTU,
+	PRU_TYPE_TX_PRU,
+	PRU_TYPE_MAX,
+};
+
+/**
+ * struct pru_rproc - PRU remoteproc structure
+ * @id: id of the PRU core within the PRUSS
+ * @pruss: back-reference to parent PRUSS structure
+ * @rproc: remoteproc pointer for this PRU core
+ * @client_np: client device node
+ * @mbox: mailbox channel handle used for vring signalling with MPU
+ * @client: mailbox client to request the mailbox channel
+ * @type: type of the PRU core (PRU, RTU, Tx_PRU)
+ * @irq_ring: IRQ number to use for processing vring buffers
+ * @irq_kick: IRQ number to use to perform virtio kick
+ * @mem_regions: data for each of the PRU memory regions
+ * @intc_config: PRU INTC configuration data
+ * @rmw_lock: lock for read, modify, write operations on registers
+ * @iram_da: device address of Instruction RAM for this PRU
+ * @pdram_da: device address of primary Data RAM for this PRU
+ * @sdram_da: device address of secondary Data RAM for this PRU
+ * @shrdram_da: device address of shared Data RAM
+ * @fw_name: name of firmware image used during loading
+ * @dt_irqs: number of irqs configured from DT
+ * @gpmux_save: saved value for gpmux config
+ * @lock: mutex to protect client usage
+ * @dbg_single_step: debug state variable to set PRU into single step mode
+ * @dbg_continuous: debug state variable to restore PRU execution mode
+ * @fw_has_intc_rsc: boolean flag to indicate INTC config through firmware
+ * @is_k3: boolean flag used to indicate the core has increased number of events
+ */
+struct pru_rproc {
+	int id;
+	struct pruss *pruss;
+	struct rproc *rproc;
+	struct device_node *client_np;
+	struct mbox_chan *mbox;
+	struct mbox_client client;
+	enum pru_type type;
+	int irq_vring;
+	int irq_kick;
+	struct pruss_mem_region mem_regions[PRU_IOMEM_MAX];
+	struct pruss_intc_config intc_config;
+	spinlock_t rmw_lock; /* register access lock */
+	u32 iram_da;
+	u32 pdram_da;
+	u32 sdram_da;
+	u32 shrdram_da;
+	const char *fw_name;
+	int dt_irqs;
+	u8 gpmux_save;
+	struct mutex lock; /* client access lock */
+	u32 dbg_single_step;
+	u32 dbg_continuous;
+	unsigned int fw_has_intc_rsc : 1;
+	unsigned int is_k3 : 1;
+};
+
+static void *pru_d_da_to_va(struct pru_rproc *pru, u32 da, int len);
+
+static inline u32 pru_control_read_reg(struct pru_rproc *pru, unsigned int reg)
+{
+	return readl_relaxed(pru->mem_regions[PRU_IOMEM_CTRL].va + reg);
+}
+
+static inline
+void pru_control_write_reg(struct pru_rproc *pru, unsigned int reg, u32 val)
+{
+	writel_relaxed(val, pru->mem_regions[PRU_IOMEM_CTRL].va + reg);
+}
+
+static inline
+void pru_control_set_reg(struct pru_rproc *pru, unsigned int reg,
+			 u32 mask, u32 set)
+{
+	u32 val;
+	unsigned long flags;
+
+	spin_lock_irqsave(&pru->rmw_lock, flags);
+
+	val = pru_control_read_reg(pru, reg);
+	val &= ~mask;
+	val |= (set & mask);
+	pru_control_write_reg(pru, reg, val);
+
+	spin_unlock_irqrestore(&pru->rmw_lock, flags);
+}
+
+/**
+ * pru_rproc_set_firmware() - set firmware for a pru core
+ * @rproc: the rproc instance of the PRU
+ * @fw_name: the new firmware name, or NULL if default is desired
+ */
+static int pru_rproc_set_firmware(struct rproc *rproc, const char *fw_name)
+{
+	struct pru_rproc *pru = rproc->priv;
+
+	if (!fw_name)
+		fw_name = pru->fw_name;
+
+	return rproc_set_firmware(rproc, fw_name);
+}
+
+static int pru_rproc_intc_dt_config(struct pru_rproc *pru, int index)
+{
+	struct device *dev = &pru->rproc->dev;
+	struct device_node *np = pru->client_np;
+	struct property *prop;
+	const char *prop_name = "ti,pru-interrupt-map";
+	u8 max_system_events, max_pru_channels, max_pru_host_ints;
+	int ret = 0, i;
+	int dt_irqs;
+	u32 *arr;
+	bool has_irqs = false;
+
+	prop = of_find_property(np, prop_name, NULL);
+	if (!prop)
+		return 0;
+
+	dt_irqs = of_property_count_u32_elems(np, prop_name);
+	if (dt_irqs <= 0 || dt_irqs % 4) {
+		dev_err(dev, "bad interrupt map data %d, expected multiple of 4\n",
+			dt_irqs);
+		return -EINVAL;
+	}
+
+	arr = kmalloc_array(dt_irqs, sizeof(u32), GFP_KERNEL);
+	if (!arr)
+		return -ENOMEM;
+
+	ret = of_property_read_u32_array(np, prop_name, arr, dt_irqs);
+	if (ret) {
+		dev_err(dev, "failed to read pru irq map: %d\n", ret);
+		goto out;
+	}
+
+	max_system_events = pru->is_k3 ?
+				MAX_PRU_SYS_EVENTS_K3 : MAX_PRU_SYS_EVENTS;
+	max_pru_channels = pru->is_k3 ? MAX_PRU_CHANNELS_K3 : MAX_PRU_CHANNELS;
+	max_pru_host_ints = pru->is_k3 ? MAX_PRU_HOST_INT_K3 : MAX_PRU_HOST_INT;
+
+	for (i = 0; i < ARRAY_SIZE(pru->intc_config.sysev_to_ch); i++)
+		pru->intc_config.sysev_to_ch[i] = -1;
+
+	for (i = 0; i < ARRAY_SIZE(pru->intc_config.ch_to_host); i++)
+		pru->intc_config.ch_to_host[i] = -1;
+
+	for (i = 0; i < dt_irqs; i += 4) {
+		if (arr[i] != index)
+			continue;
+
+		if (arr[i + 1] < 0 ||
+		    arr[i + 1] >= max_system_events) {
+			dev_err(dev, "bad sys event %d\n", arr[i + 1]);
+			ret = -EINVAL;
+			goto out;
+		}
+
+		if (arr[i + 2] < 0 ||
+		    arr[i + 2] >= max_pru_channels) {
+			dev_err(dev, "bad channel %d\n", arr[i + 2]);
+			ret = -EINVAL;
+			goto out;
+		}
+
+		if (arr[i + 3] < 0 ||
+		    arr[i + 3] >= max_pru_host_ints) {
+			dev_err(dev, "bad irq %d\n", arr[i + 3]);
+			ret = -EINVAL;
+			goto out;
+		}
+
+		pru->intc_config.sysev_to_ch[arr[i + 1]] = arr[i + 2];
+		dev_dbg(dev, "sysevt-to-ch[%d] -> %d\n", arr[i + 1],
+			arr[i + 2]);
+
+		pru->intc_config.ch_to_host[arr[i + 2]] = arr[i + 3];
+		dev_dbg(dev, "chnl-to-host[%d] -> %d\n", arr[i + 2],
+			arr[i + 3]);
+
+		has_irqs = true;
+	}
+
+	/*
+	 * The property "ti,pru-interrupt-map" is used in a consumer node, but
+	 * need not necessarily have data for all referenced PRUs. Provide a
+	 * fallback to get the interrupt data from firmware for PRUs ith no
+	 * interrupt data.
+	 */
+	if (!has_irqs) {
+		dev_dbg(dev, "no DT irqs, falling back to firmware intc rsc mode\n");
+		goto out;
+	}
+
+	pru->dt_irqs = dt_irqs;
+	ret = pruss_intc_configure(pru->pruss, &pru->intc_config);
+	if (ret) {
+		dev_err(dev, "failed to configure intc %d\n", ret);
+		pru->dt_irqs = 0;
+	}
+
+out:
+	kfree(arr);
+
+	return ret;
+}
+
+static struct rproc *__pru_rproc_get(struct device_node *np, int index)
+{
+	struct device_node *rproc_np = NULL;
+	struct platform_device *pdev;
+	struct rproc *rproc;
+
+	rproc_np = of_parse_phandle(np, "prus", index);
+	if (!rproc_np || !of_device_is_available(rproc_np))
+		return ERR_PTR(-ENODEV);
+
+	pdev = of_find_device_by_node(rproc_np);
+	of_node_put(rproc_np);
+
+	if (!pdev)
+		/* probably PRU not yet probed */
+		return ERR_PTR(-EPROBE_DEFER);
+
+	/* TODO: replace the crude string based check to make sure it is PRU */
+	if (!strstr(dev_name(&pdev->dev), "pru") &&
+	    !strstr(dev_name(&pdev->dev), "rtu")) {
+		put_device(&pdev->dev);
+		return ERR_PTR(-ENODEV);
+	}
+
+	rproc = platform_get_drvdata(pdev);
+	put_device(&pdev->dev);
+	if (!rproc)
+		return ERR_PTR(-EPROBE_DEFER);
+
+	get_device(&rproc->dev);
+
+	return rproc;
+}
+
+/**
+ * pru_rproc_get() - get the PRU rproc instance from a device node
+ * @np: the user/client device node
+ * @index: index to use for the prus property
+ *
+ * This function looks through a client device node's "prus" property at index
+ * @index and returns the rproc handle for a valid PRU remote processor if
+ * found. The function allows only one user to own the PRU rproc resource at
+ * a time. Caller must call pru_rproc_put() when done with using the rproc,
+ * not required if the function returns a failure.
+ *
+ * Returns the rproc handle on success, and an ERR_PTR on failure using one
+ * of the following error values
+ *    -ENODEV if device is not found
+ *    -EBUSY if PRU is already acquired by anyone
+ *    -EPROBE_DEFER is PRU device is not probed yet
+ */
+struct rproc *pru_rproc_get(struct device_node *np, int index)
+{
+	struct rproc *rproc;
+	struct pru_rproc *pru;
+	const char *fw_name;
+	struct device *dev;
+	int ret;
+	u32 mux;
+
+	rproc = __pru_rproc_get(np, index);
+	if (IS_ERR(rproc))
+		return rproc;
+
+	pru = rproc->priv;
+	dev = &rproc->dev;
+
+	mutex_lock(&pru->lock);
+
+	if (pru->client_np) {
+		mutex_unlock(&pru->lock);
+		put_device(&rproc->dev);
+		return ERR_PTR(-EBUSY);
+	}
+
+	pru->client_np = np;
+	rproc->deny_sysfs_ops = 1;
+
+	mutex_unlock(&pru->lock);
+
+	ret = pruss_cfg_get_gpmux(pru->pruss, pru->id, &pru->gpmux_save);
+	if (ret) {
+		dev_err(dev, "failed to get cfg gpmux: %d\n", ret);
+		goto err;
+	}
+
+	ret = of_property_read_u32_index(np, "ti,pruss-gp-mux-sel", index,
+					 &mux);
+	if (!ret) {
+		ret = pruss_cfg_set_gpmux(pru->pruss, pru->id, mux);
+		if (ret) {
+			dev_err(dev, "failed to set cfg gpmux: %d\n", ret);
+			goto err;
+		}
+	}
+
+	ret = of_property_read_string_index(np, "firmware-name", index,
+					    &fw_name);
+	if (!ret) {
+		ret = pru_rproc_set_firmware(rproc, fw_name);
+		if (ret) {
+			dev_err(dev, "failed to set firmware: %d\n", ret);
+			goto err;
+		}
+	}
+
+	ret = pru_rproc_intc_dt_config(pru, index);
+	if (ret)
+		goto err;
+
+	return rproc;
+
+err:
+	pru_rproc_put(rproc);
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(pru_rproc_get);
+
+/**
+ * pru_rproc_put() - release the PRU rproc resource
+ * @rproc: the rproc resource to release
+ *
+ * Releases the PRU rproc resource and makes it available to other
+ * users.
+ */
+void pru_rproc_put(struct rproc *rproc)
+{
+	struct pru_rproc *pru;
+
+	if (IS_ERR_OR_NULL(rproc))
+		return;
+
+	/* TODO: replace the crude string based check to make sure it is PRU */
+	if (!strstr(dev_name(rproc->dev.parent), "pru") &&
+	    !strstr(dev_name(rproc->dev.parent), "rtu"))
+		return;
+
+	pru = rproc->priv;
+	if (!pru->client_np)
+		return;
+
+	pruss_cfg_set_gpmux(pru->pruss, pru->id, pru->gpmux_save);
+
+	if (pru->dt_irqs)
+		pruss_intc_unconfigure(pru->pruss, &pru->intc_config);
+
+	pru_rproc_set_firmware(rproc, NULL);
+
+	mutex_lock(&pru->lock);
+	pru->client_np = NULL;
+	rproc->deny_sysfs_ops = 0;
+	mutex_unlock(&pru->lock);
+
+	put_device(&rproc->dev);
+}
+EXPORT_SYMBOL_GPL(pru_rproc_put);
+
+/**
+ * pru_rproc_get_id() - get PRU id from a previously acquired PRU remoteproc
+ * @rproc: the rproc instance of the PRU
+ *
+ * Returns the PRU id of the PRU remote processor that has been acquired through
+ * a pru_rproc_get(), or a negative value on error
+ */
+enum pruss_pru_id pru_rproc_get_id(struct rproc *rproc)
+{
+	struct pru_rproc *pru;
+
+	if (IS_ERR_OR_NULL(rproc) || !rproc->dev.parent)
+		return -EINVAL;
+
+	/* TODO: replace the crude string based check to make sure it is PRU */
+	if (!strstr(dev_name(rproc->dev.parent), "pru") &&
+	    !strstr(dev_name(rproc->dev.parent), "rtu"))
+		return -EINVAL;
+
+	pru = rproc->priv;
+	return pru->id;
+}
+EXPORT_SYMBOL_GPL(pru_rproc_get_id);
+
+/**
+ * pru_rproc_set_ctable() - set the constant table index for the PRU
+ * @rproc: the rproc instance of the PRU
+ * @c: constant table index to set
+ * @addr: physical address to set it to
+ */
+int pru_rproc_set_ctable(struct rproc *rproc, enum pru_ctable_idx c, u32 addr)
+{
+	struct pru_rproc *pru = rproc->priv;
+	unsigned int reg;
+	u32 mask, set;
+	u16 idx;
+	u16 idx_mask;
+
+	/* pointer is 16 bit and index is 8-bit so mask out the rest */
+	idx_mask = (c >= PRU_C28) ? 0xFFFF : 0xFF;
+
+	/* ctable uses bit 8 and upwards only */
+	idx = (addr >> 8) & idx_mask;
+
+	/* configurable ctable (i.e. C24) starts at PRU_CTRL_CTBIR0 */
+	reg = PRU_CTRL_CTBIR0 + 4 * (c >> 1);
+	mask = idx_mask << (16 * (c & 1));
+	set = idx << (16 * (c & 1));
+
+	pru_control_set_reg(pru, reg, mask, set);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(pru_rproc_set_ctable);
+
+static inline u32 pru_debug_read_reg(struct pru_rproc *pru, unsigned int reg)
+{
+	return readl_relaxed(pru->mem_regions[PRU_IOMEM_DEBUG].va + reg);
+}
+
+static inline
+void pru_debug_write_reg(struct pru_rproc *pru, unsigned int reg, u32 val)
+{
+	writel_relaxed(val, pru->mem_regions[PRU_IOMEM_DEBUG].va + reg);
+}
+
+static int regs_show(struct seq_file *s, void *data)
+{
+	struct rproc *rproc = s->private;
+	struct pru_rproc *pru = rproc->priv;
+	int i, nregs = 32;
+	u32 pru_sts;
+	int pru_is_running;
+
+	seq_puts(s, "============== Control Registers ==============\n");
+	seq_printf(s, "CTRL      := 0x%08x\n",
+		   pru_control_read_reg(pru, PRU_CTRL_CTRL));
+	pru_sts = pru_control_read_reg(pru, PRU_CTRL_STS);
+	seq_printf(s, "STS (PC)  := 0x%08x (0x%08x)\n", pru_sts, pru_sts << 2);
+	seq_printf(s, "WAKEUP_EN := 0x%08x\n",
+		   pru_control_read_reg(pru, PRU_CTRL_WAKEUP_EN));
+	seq_printf(s, "CYCLE     := 0x%08x\n",
+		   pru_control_read_reg(pru, PRU_CTRL_CYCLE));
+	seq_printf(s, "STALL     := 0x%08x\n",
+		   pru_control_read_reg(pru, PRU_CTRL_STALL));
+	seq_printf(s, "CTBIR0    := 0x%08x\n",
+		   pru_control_read_reg(pru, PRU_CTRL_CTBIR0));
+	seq_printf(s, "CTBIR1    := 0x%08x\n",
+		   pru_control_read_reg(pru, PRU_CTRL_CTBIR1));
+	seq_printf(s, "CTPPR0    := 0x%08x\n",
+		   pru_control_read_reg(pru, PRU_CTRL_CTPPR0));
+	seq_printf(s, "CTPPR1    := 0x%08x\n",
+		   pru_control_read_reg(pru, PRU_CTRL_CTPPR1));
+
+	seq_puts(s, "=============== Debug Registers ===============\n");
+	pru_is_running = pru_control_read_reg(pru, PRU_CTRL_CTRL) &
+				CTRL_CTRL_RUNSTATE;
+	if (pru_is_running) {
+		seq_puts(s, "PRU is executing, cannot print/access debug registers.\n");
+		return 0;
+	}
+
+	for (i = 0; i < nregs; i++) {
+		seq_printf(s, "GPREG%-2d := 0x%08x\tCT_REG%-2d := 0x%08x\n",
+			   i, pru_debug_read_reg(pru, PRU_DEBUG_GPREG(i)),
+			   i, pru_debug_read_reg(pru, PRU_DEBUG_CT_REG(i)));
+	}
+
+	return 0;
+}
+
+DEFINE_SHOW_ATTRIBUTE(regs);
+
+/*
+ * Control PRU single-step mode
+ *
+ * This is a debug helper function used for controlling the single-step
+ * mode of the PRU. The PRU Debug registers are not accessible when the
+ * PRU is in RUNNING state.
+ *
+ * Writing a non-zero value sets the PRU into single-step mode irrespective
+ * of its previous state. The PRU mode is saved only on the first set into
+ * a single-step mode. Writing a zero value will restore the PRU into its
+ * original mode.
+ */
+static int pru_rproc_debug_ss_set(void *data, u64 val)
+{
+	struct rproc *rproc = data;
+	struct pru_rproc *pru = rproc->priv;
+	u32 reg_val;
+
+	val = val ? 1 : 0;
+	if (!val && !pru->dbg_single_step)
+		return 0;
+
+	reg_val = pru_control_read_reg(pru, PRU_CTRL_CTRL);
+
+	if (val && !pru->dbg_single_step)
+		pru->dbg_continuous = reg_val;
+
+	if (val)
+		reg_val |= CTRL_CTRL_SINGLE_STEP | CTRL_CTRL_EN;
+	else
+		reg_val = pru->dbg_continuous;
+
+	pru->dbg_single_step = val;
+	pru_control_write_reg(pru, PRU_CTRL_CTRL, reg_val);
+
+	return 0;
+}
+
+static int pru_rproc_debug_ss_get(void *data, u64 *val)
+{
+	struct rproc *rproc = data;
+	struct pru_rproc *pru = rproc->priv;
+
+	*val = pru->dbg_single_step;
+
+	return 0;
+}
+DEFINE_SIMPLE_ATTRIBUTE(pru_rproc_debug_ss_fops, pru_rproc_debug_ss_get,
+			pru_rproc_debug_ss_set, "%llu\n");
+
+/*
+ * Create PRU-specific debugfs entries
+ *
+ * The entries are created only if the parent remoteproc debugfs directory
+ * exists, and will be cleaned up by the remoteproc core.
+ */
+static void pru_rproc_create_debug_entries(struct rproc *rproc)
+{
+	if (!rproc->dbg_dir)
+		return;
+
+	debugfs_create_file("regs", 0400, rproc->dbg_dir,
+			    rproc, &regs_fops);
+	debugfs_create_file("single_step", 0600, rproc->dbg_dir,
+			    rproc, &pru_rproc_debug_ss_fops);
+}
+
+/**
+ * pru_rproc_mbox_callback() - inbound mailbox message handler
+ * @client: mailbox client pointer used for requesting the mailbox channel
+ * @data: mailbox payload
+ *
+ * This handler is invoked by omap's mailbox driver whenever a mailbox
+ * message is received. Usually, the mailbox payload simply contains
+ * the index of the virtqueue that is kicked by the PRU remote processor,
+ * and we let remoteproc core handle it.
+ *
+ * In addition to virtqueue indices, we might also have some out-of-band
+ * values that indicates different events. Those values are deliberately
+ * very big so they don't coincide with virtqueue indices.
+ */
+static void pru_rproc_mbox_callback(struct mbox_client *client, void *data)
+{
+	struct pru_rproc *pru = container_of(client, struct pru_rproc, client);
+	struct device *dev = &pru->rproc->dev;
+	u32 msg = to_omap_mbox_msg(data);
+
+	dev_dbg(dev, "mbox msg: 0x%x\n", msg);
+
+	/* msg contains the index of the triggered vring */
+	if (rproc_vq_interrupt(pru->rproc, msg) == IRQ_NONE)
+		dev_dbg(dev, "no message was found in vqid %d\n", msg);
+}
+
+/**
+ * pru_rproc_vring_interrupt() - interrupt handler for processing vrings
+ * @irq: irq number associated with the PRU event MPU is listening on
+ * @data: interrupt handler data, will be a PRU rproc structure
+ *
+ * This handler is used by the PRU remoteproc driver when using PRU system
+ * events for processing the virtqueues. Unlike the mailbox IP, there is
+ * no payload associated with an interrupt, so either a unique event is
+ * used for each virtqueue kick, or a both virtqueues are processed on
+ * a single event. The latter is chosen to conserve the usable PRU system
+ * events.
+ */
+static irqreturn_t pru_rproc_vring_interrupt(int irq, void *data)
+{
+	struct pru_rproc *pru = data;
+
+	dev_dbg(&pru->rproc->dev, "got vring irq\n");
+
+	/* process incoming buffers on both the Rx and Tx vrings */
+	rproc_vq_interrupt(pru->rproc, 0);
+	rproc_vq_interrupt(pru->rproc, 1);
+
+	return IRQ_HANDLED;
+}
+
+/* kick a virtqueue */
+static void pru_rproc_kick(struct rproc *rproc, int vq_id)
+{
+	struct device *dev = &rproc->dev;
+	struct pru_rproc *pru = rproc->priv;
+	int ret;
+	mbox_msg_t msg = (mbox_msg_t)vq_id;
+	const char *names[PRU_TYPE_MAX] = { "PRU", "RTU", "Tx_PRU" };
+
+	dev_dbg(dev, "kicking vqid %d on %s%d\n", vq_id,
+		names[pru->type], pru->id);
+
+	if (pru->irq_kick > 0) {
+		ret = pruss_intc_trigger(pru->irq_kick);
+		if (ret < 0)
+			dev_err(dev, "pruss_intc_trigger failed: %d\n", ret);
+	} else if (pru->mbox) {
+		/*
+		 * send the index of the triggered virtqueue in the mailbox
+		 * payload
+		 */
+		ret = mbox_send_message(pru->mbox, (void *)msg);
+		if (ret < 0)
+			dev_err(dev, "mbox_send_message failed: %d\n", ret);
+	}
+}
+
+/* start a PRU core */
+static int pru_rproc_start(struct rproc *rproc)
+{
+	struct device *dev = &rproc->dev;
+	struct pru_rproc *pru = rproc->priv;
+	const char *names[PRU_TYPE_MAX] = { "PRU", "RTU", "Tx_PRU" };
+	u32 val;
+	int ret;
+
+	dev_dbg(dev, "starting %s%d: entry-point = 0x%x\n",
+		names[pru->type], pru->id, (rproc->bootaddr >> 2));
+
+	if (!list_empty(&pru->rproc->rvdevs)) {
+		if (!pru->mbox && (pru->irq_vring <= 0 || pru->irq_kick <= 0)) {
+			dev_err(dev, "virtio vring interrupt mechanisms are not provided\n");
+			ret = -EINVAL;
+			goto fail;
+		}
+
+		if (!pru->mbox && pru->irq_vring > 0) {
+			ret = request_threaded_irq(pru->irq_vring, NULL,
+						   pru_rproc_vring_interrupt,
+						   IRQF_ONESHOT, dev_name(dev),
+						   pru);
+			if (ret) {
+				dev_err(dev, "failed to enable vring interrupt, ret = %d\n",
+					ret);
+				goto fail;
+			}
+		}
+	}
+
+	val = CTRL_CTRL_EN | ((rproc->bootaddr >> 2) << 16);
+	pru_control_write_reg(pru, PRU_CTRL_CTRL, val);
+
+	return 0;
+
+fail:
+	if (!pru->dt_irqs && pru->fw_has_intc_rsc)
+		pruss_intc_unconfigure(pru->pruss, &pru->intc_config);
+	return ret;
+}
+
+/* stop/disable a PRU core */
+static int pru_rproc_stop(struct rproc *rproc)
+{
+	struct device *dev = &rproc->dev;
+	struct pru_rproc *pru = rproc->priv;
+	const char *names[PRU_TYPE_MAX] = { "PRU", "RTU", "Tx_PRU" };
+	u32 val;
+
+	dev_dbg(dev, "stopping %s%d\n", names[pru->type], pru->id);
+
+	val = pru_control_read_reg(pru, PRU_CTRL_CTRL);
+	val &= ~CTRL_CTRL_EN;
+	pru_control_write_reg(pru, PRU_CTRL_CTRL, val);
+
+	if (!list_empty(&pru->rproc->rvdevs) &&
+	    !pru->mbox && pru->irq_vring > 0)
+		free_irq(pru->irq_vring, pru);
+
+	/* undo INTC config */
+	if (!pru->dt_irqs && pru->fw_has_intc_rsc)
+		pruss_intc_unconfigure(pru->pruss, &pru->intc_config);
+
+	return 0;
+}
+
+/*
+ * parse the custom PRU interrupt map resource and configure the INTC
+ * appropriately
+ */
+static int pru_handle_vendor_intrmap(struct rproc *rproc,
+				     struct fw_rsc_vendor *rsc)
+{
+	struct device *dev = rproc->dev.parent;
+	struct pru_rproc *pru = rproc->priv;
+	struct pruss *pruss = pru->pruss;
+	struct pruss_event_chnl *event_chnl_map;
+	struct fw_rsc_pruss_intrmap *intr_rsc0;
+	struct fw_rsc_pruss_intrmap_k3 *intr_rsc1;
+	int i, ret;
+	u32 event_chnl_map_da, event_chnl_map_size;
+	s8 sys_evt, chnl, intr_no;
+	s8 *chnl_host_intr_map;
+	u8 max_system_events, max_pru_channels, max_pru_host_ints;
+
+	if (rsc->u.st.st_ver != 0 && rsc->u.st.st_ver != 1) {
+		dev_err(dev, "only PRU interrupt resource versions 0 and 1 are supported\n");
+		return -EINVAL;
+	}
+
+	if (!rsc->u.st.st_ver) {
+		intr_rsc0 = (struct fw_rsc_pruss_intrmap *)rsc->data;
+		event_chnl_map_da = intr_rsc0->event_chnl_map_addr;
+		event_chnl_map_size = intr_rsc0->event_chnl_map_size;
+		chnl_host_intr_map = intr_rsc0->chnl_host_intr_map;
+		max_system_events = MAX_PRU_SYS_EVENTS;
+		max_pru_channels = MAX_PRU_CHANNELS;
+		max_pru_host_ints = MAX_PRU_HOST_INT;
+
+		dev_dbg(dev, "version %d event_chnl_map_size %d event_chnl_map_da 0x%x\n",
+			rsc->u.st.st_ver, intr_rsc0->event_chnl_map_size,
+			event_chnl_map_da);
+	} else {
+		intr_rsc1 = (struct fw_rsc_pruss_intrmap_k3 *)rsc->data;
+		event_chnl_map_da = intr_rsc1->event_chnl_map_addr;
+		event_chnl_map_size = intr_rsc1->event_chnl_map_size;
+		chnl_host_intr_map = intr_rsc1->chnl_host_intr_map;
+		max_system_events = MAX_PRU_SYS_EVENTS_K3;
+		max_pru_channels = MAX_PRU_CHANNELS_K3;
+		max_pru_host_ints = MAX_PRU_HOST_INT_K3;
+
+		dev_dbg(dev, "version %d event_chnl_map_size %d event_chnl_map_da 0x%x\n",
+			rsc->u.st.st_ver, intr_rsc1->event_chnl_map_size,
+			event_chnl_map_da);
+	}
+
+	if (event_chnl_map_size < 0 ||
+	    event_chnl_map_size >= max_system_events) {
+		dev_err(dev, "PRU interrupt resource has more events than present on hardware\n");
+		return -EINVAL;
+	}
+
+	/*
+	 * XXX: The event_chnl_map_addr mapping is currently a pointer in device
+	 * memory, evaluate if this needs to be directly in firmware file.
+	 */
+	event_chnl_map = pru_d_da_to_va(pru, event_chnl_map_da,
+					event_chnl_map_size *
+					sizeof(*event_chnl_map));
+	if (!event_chnl_map) {
+		dev_err(dev, "PRU interrupt resource has inadequate event_chnl_map configuration\n");
+		return -EINVAL;
+	}
+
+	/* init intc_config to defaults */
+	for (i = 0; i < ARRAY_SIZE(pru->intc_config.sysev_to_ch); i++)
+		pru->intc_config.sysev_to_ch[i] = -1;
+
+	for (i = 0; i < ARRAY_SIZE(pru->intc_config.ch_to_host); i++)
+		pru->intc_config.ch_to_host[i] = -1;
+
+	/* parse and fill in system event to interrupt channel mapping */
+	for (i = 0; i < event_chnl_map_size; i++) {
+		sys_evt = event_chnl_map[i].event;
+		chnl = event_chnl_map[i].chnl;
+
+		if (sys_evt < 0 || sys_evt >= max_system_events) {
+			dev_err(dev, "[%d] bad sys event %d\n", i, sys_evt);
+			return -EINVAL;
+		}
+		if (chnl < 0 || chnl >= max_pru_channels) {
+			dev_err(dev, "[%d] bad channel value %d\n", i, chnl);
+			return -EINVAL;
+		}
+
+		pru->intc_config.sysev_to_ch[sys_evt] = chnl;
+		dev_dbg(dev, "sysevt-to-ch[%d] -> %d\n", sys_evt, chnl);
+	}
+
+	/* parse and handle interrupt channel-to-host interrupt mapping */
+	for (i = 0; i < max_pru_channels; i++) {
+		intr_no = chnl_host_intr_map[i];
+		if (intr_no < 0) {
+			dev_dbg(dev, "skip intr mapping for chnl %d\n", i);
+			continue;
+		}
+
+		if (intr_no >= max_pru_host_ints) {
+			dev_err(dev, "bad intr mapping for chnl %d, intr_no %d\n",
+				i, intr_no);
+			return -EINVAL;
+		}
+
+		pru->intc_config.ch_to_host[i] = intr_no;
+		dev_dbg(dev, "chnl-to-host[%d] -> %d\n", i, intr_no);
+	}
+
+	pru->fw_has_intc_rsc = 1;
+
+	ret = pruss_intc_configure(pruss, &pru->intc_config);
+	if (ret)
+		dev_err(dev, "failed to configure pruss intc %d\n", ret);
+
+	return ret;
+}
+
+/* PRU-specific vendor resource handler */
+static int pru_rproc_handle_vendor_rsc(struct rproc *rproc,
+				       struct fw_rsc_vendor *rsc)
+{
+	struct device *dev = rproc->dev.parent;
+	struct pru_rproc *pru = rproc->priv;
+	int ret = 0;
+
+	switch (rsc->u.st.st_type) {
+	case PRUSS_RSC_INTRS:
+		if (!pru->dt_irqs)
+			ret = pru_handle_vendor_intrmap(rproc, rsc);
+		break;
+	default:
+		dev_err(dev, "%s: cannot handle unknown type %d\n", __func__,
+			rsc->u.st.st_type);
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
+/*
+ * Convert PRU device address (data spaces only) to kernel virtual address
+ *
+ * Each PRU has access to all data memories within the PRUSS, accessible at
+ * different ranges. So, look through both its primary and secondary Data
+ * RAMs as well as any shared Data RAM to convert a PRU device address to
+ * kernel virtual address. Data RAM0 is primary Data RAM for PRU0 and Data
+ * RAM1 is primary Data RAM for PRU1.
+ */
+static void *pru_d_da_to_va(struct pru_rproc *pru, u32 da, int len)
+{
+	struct pruss_mem_region dram0, dram1, shrd_ram;
+	struct pruss *pruss = pru->pruss;
+	u32 offset;
+	void *va = NULL;
+
+	if (len <= 0)
+		return NULL;
+
+	dram0 = pruss->mem_regions[PRUSS_MEM_DRAM0];
+	dram1 = pruss->mem_regions[PRUSS_MEM_DRAM1];
+	/* PRU1 has its local RAM addresses reversed */
+	if (pru->id == 1)
+		swap(dram0, dram1);
+	shrd_ram = pruss->mem_regions[PRUSS_MEM_SHRD_RAM2];
+
+	if (da >= pru->pdram_da && da + len <= pru->pdram_da + dram0.size) {
+		offset = da - pru->pdram_da;
+		va = (__force void *)(dram0.va + offset);
+	} else if (da >= pru->sdram_da &&
+		   da + len <= pru->sdram_da + dram1.size) {
+		offset = da - pru->sdram_da;
+		va = (__force void *)(dram1.va + offset);
+	} else if (da >= pru->shrdram_da &&
+		   da + len <= pru->shrdram_da + shrd_ram.size) {
+		offset = da - pru->shrdram_da;
+		va = (__force void *)(shrd_ram.va + offset);
+	}
+
+	return va;
+}
+
+/*
+ * Convert PRU device address (instruction space) to kernel virtual address
+ *
+ * A PRU does not have an unified address space. Each PRU has its very own
+ * private Instruction RAM, and its device address is identical to that of
+ * its primary Data RAM device address.
+ */
+static void *pru_i_da_to_va(struct pru_rproc *pru, u32 da, int len)
+{
+	u32 offset;
+	void *va = NULL;
+
+	/* GNU binutils do not support multiple address spaces. The
+	 * default linker script from the official GNU pru-ld places
+	 * IRAM at an arbitrary high offset, in order to differentiate it
+	 * from DRAM. Hence we need to strip the artificial offset
+	 * from the IRAM address.
+	 */
+	da &= ~0xf0000000u;
+
+	if (len <= 0)
+		return NULL;
+
+	if (da >= pru->iram_da &&
+	    da + len <= pru->iram_da + pru->mem_regions[PRU_IOMEM_IRAM].size) {
+		offset = da - pru->iram_da;
+		va = (__force void *)(pru->mem_regions[PRU_IOMEM_IRAM].va +
+				      offset);
+	}
+
+	return va;
+}
+
+/* PRU-specific address translator */
+static void *pru_da_to_va(struct rproc *rproc, u64 da, int len, u32 flags)
+{
+	struct pru_rproc *pru = rproc->priv;
+	void *va;
+	u32 exec_flag;
+
+	exec_flag = ((flags & RPROC_FLAGS_ELF_SHDR) ? flags & SHF_EXECINSTR :
+		     ((flags & RPROC_FLAGS_ELF_PHDR) ? flags & PF_X : 0));
+
+	if (exec_flag)
+		va = pru_i_da_to_va(pru, da, len);
+	else
+		va = pru_d_da_to_va(pru, da, len);
+
+	return va;
+}
+
+static struct rproc_ops pru_rproc_ops = {
+	.start			= pru_rproc_start,
+	.stop			= pru_rproc_stop,
+	.kick			= pru_rproc_kick,
+	.handle_vendor_rsc	= pru_rproc_handle_vendor_rsc,
+	.da_to_va		= pru_da_to_va,
+};
+
+/*
+ * Custom memory copy implementation for ICSSG PRU/RTU Cores
+ *
+ * The ICSSG PRU/RTU cores have a memory copying issue with IRAM memories, that
+ * is not seen on previous generation SoCs. The data is reflected properly in
+ * the IRAM memories only for integer (4-byte) copies. Any unaligned copies
+ * result in all the other pre-existing bytes zeroed out within that 4-byte
+ * boundary, thereby resulting in wrong text/code in the IRAMs. Also, the
+ * IRAM memory port interface does not allow any 8-byte copies (as commonly
+ * used by ARM64 memcpy implementation) and throws an exception. The DRAM
+ * memory ports do not show this behavior. Use this custom copying function
+ * to properly load the PRU/RTU firmware images on all memories for simplicity.
+ *
+ * TODO: Improve the function to deal with additional corner cases like
+ * unaligned copy sizes or sub-integer trailing bytes when the need arises.
+ */
+static int pru_rproc_memcpy(void *dest, const void *src, size_t count)
+{
+	const int *s = src;
+	int *d = dest;
+	int size = count / 4;
+	int *tmp_src = NULL;
+
+	/* limited to 4-byte aligned addresses and copy sizes */
+	if ((long)dest % 4 || count % 4)
+		return -EINVAL;
+
+	/* src offsets in ELF firmware image can be non-aligned */
+	if ((long)src % 4) {
+		tmp_src = kmemdup(src, count, GFP_KERNEL);
+		if (!tmp_src)
+			return -ENOMEM;
+		s = tmp_src;
+	}
+
+	while (size--)
+		*d++ = *s++;
+
+	kfree(tmp_src);
+
+	return 0;
+}
+
+static int
+pru_rproc_load_elf_segments(struct rproc *rproc, const struct firmware *fw)
+{
+	struct device *dev = &rproc->dev;
+	struct elf32_hdr *ehdr;
+	struct elf32_phdr *phdr;
+	int i, ret = 0;
+	const u8 *elf_data = fw->data;
+
+	ehdr = (struct elf32_hdr *)elf_data;
+	phdr = (struct elf32_phdr *)(elf_data + ehdr->e_phoff);
+
+	/* go through the available ELF segments */
+	for (i = 0; i < ehdr->e_phnum; i++, phdr++) {
+		u32 da = phdr->p_paddr;
+		u32 memsz = phdr->p_memsz;
+		u32 filesz = phdr->p_filesz;
+		u32 offset = phdr->p_offset;
+		void *ptr;
+
+		if (phdr->p_type != PT_LOAD)
+			continue;
+
+		dev_dbg(dev, "phdr: type %d da 0x%x memsz 0x%x filesz 0x%x\n",
+			phdr->p_type, da, memsz, filesz);
+
+		if (filesz > memsz) {
+			dev_err(dev, "bad phdr filesz 0x%x memsz 0x%x\n",
+				filesz, memsz);
+			ret = -EINVAL;
+			break;
+		}
+
+		if (offset + filesz > fw->size) {
+			dev_err(dev, "truncated fw: need 0x%x avail 0x%zx\n",
+				offset + filesz, fw->size);
+			ret = -EINVAL;
+			break;
+		}
+
+		/* grab the kernel address for this device address */
+		ptr = rproc_da_to_va(rproc, da, memsz,
+				     RPROC_FLAGS_ELF_PHDR | phdr->p_flags);
+		if (!ptr) {
+			dev_err(dev, "bad phdr da 0x%x mem 0x%x\n", da, memsz);
+			ret = -EINVAL;
+			break;
+		}
+
+		/* skip the memzero logic performed by remoteproc ELF loader */
+		if (!phdr->p_filesz)
+			continue;
+
+		ret = pru_rproc_memcpy(ptr, elf_data + phdr->p_offset, filesz);
+		if (ret) {
+			dev_err(dev, "PRU custom memory copy failed for da 0x%x memsz 0x%x\n",
+				da, memsz);
+			break;
+		}
+	}
+
+	return ret;
+}
+
+/*
+ * compute PRU id based on the IRAM addresses. The PRU IRAMs are
+ * always at a particular offset within the PRUSS address space.
+ * The other alternative is to use static data for each core (not a
+ * hardware property to define it in DT), and the id can always be
+ * computated using this inherent address logic.
+ */
+static int pru_rproc_set_id(struct device_node *np, struct pru_rproc *pru)
+{
+	int ret = 0;
+	u32 mask1 = PRU0_IRAM_ADDR_MASK;
+	u32 mask2 = PRU1_IRAM_ADDR_MASK;
+
+	if (of_device_is_compatible(np, "ti,am654-rtu") ||
+	    of_device_is_compatible(np, "ti,j721e-rtu")) {
+		mask1 = RTU0_IRAM_ADDR_MASK;
+		mask2 = RTU1_IRAM_ADDR_MASK;
+		pru->type = PRU_TYPE_RTU;
+	}
+
+	if (of_device_is_compatible(np, "ti,j721e-tx-pru")) {
+		mask1 = TX_PRU0_IRAM_ADDR_MASK;
+		mask2 = TX_PRU1_IRAM_ADDR_MASK;
+		pru->type = PRU_TYPE_TX_PRU;
+	}
+
+	if ((pru->mem_regions[PRU_IOMEM_IRAM].pa & mask2) == mask2)
+		pru->id = PRUSS_PRU1;
+	else if ((pru->mem_regions[PRU_IOMEM_IRAM].pa & mask1) == mask1)
+		pru->id = PRUSS_PRU0;
+	else
+		ret = -EINVAL;
+
+	return ret;
+}
+
+static int pru_rproc_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct platform_device *ppdev = to_platform_device(dev->parent);
+	struct pru_rproc *pru;
+	const char *fw_name;
+	struct rproc *rproc = NULL;
+	struct mbox_client *client;
+	struct resource *res;
+	int i, ret;
+	const char *mem_names[PRU_IOMEM_MAX] = { "iram", "control", "debug" };
+
+	if (!np) {
+		dev_err(dev, "Non-DT platform device not supported\n");
+		return -ENODEV;
+	}
+
+	ret = of_property_read_string(np, "firmware-name", &fw_name);
+	if (ret) {
+		dev_err(dev, "unable to retrieve firmware-name %d\n", ret);
+		return ret;
+	}
+
+	rproc = rproc_alloc(dev, pdev->name, &pru_rproc_ops, fw_name,
+			    sizeof(*pru));
+	if (!rproc) {
+		dev_err(dev, "rproc_alloc failed\n");
+		return -ENOMEM;
+	}
+	/* error recovery is not supported for PRUs */
+	rproc->recovery_disabled = true;
+
+	/*
+	 * rproc_add will auto-boot the processor normally, but this is
+	 * not desired with PRU client driven boot-flow methodology. A PRU
+	 * application/client driver will boot the corresponding PRU
+	 * remote-processor as part of its state machine either through
+	 * the remoteproc sysfs interface or through the equivalent kernel API
+	 */
+	rproc->auto_boot = false;
+
+	pru = rproc->priv;
+	pru->pruss = platform_get_drvdata(ppdev);
+	pru->rproc = rproc;
+	pru->fw_name = fw_name;
+	spin_lock_init(&pru->rmw_lock);
+	mutex_init(&pru->lock);
+
+	if (of_device_is_compatible(np, "ti,am654-pru") ||
+	    of_device_is_compatible(np, "ti,am654-rtu") ||
+	    of_device_is_compatible(np, "ti,j721e-pru") ||
+	    of_device_is_compatible(np, "ti,j721e-rtu") ||
+	    of_device_is_compatible(np, "ti,j721e-tx-pru")) {
+		/* use generic elf ops for undefined platform driver ops */
+		rproc->ops->load = pru_rproc_load_elf_segments;
+
+		pru->is_k3 = 1;
+	}
+
+	/* XXX: get this from match data if different in the future */
+	pru->iram_da = 0;
+	pru->pdram_da = 0;
+	pru->sdram_da = 0x2000;
+	pru->shrdram_da = 0x10000;
+
+	for (i = 0; i < ARRAY_SIZE(mem_names); i++) {
+		res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+						   mem_names[i]);
+		pru->mem_regions[i].va = devm_ioremap_resource(dev, res);
+		if (IS_ERR(pru->mem_regions[i].va)) {
+			dev_err(dev, "failed to parse and map memory resource %d %s\n",
+				i, mem_names[i]);
+			ret = PTR_ERR(pru->mem_regions[i].va);
+			goto free_rproc;
+		}
+		pru->mem_regions[i].pa = res->start;
+		pru->mem_regions[i].size = resource_size(res);
+
+		dev_dbg(dev, "memory %8s: pa %pa size 0x%zx va %p\n",
+			mem_names[i], &pru->mem_regions[i].pa,
+			pru->mem_regions[i].size, pru->mem_regions[i].va);
+	}
+
+	ret = pru_rproc_set_id(np, pru);
+	if (ret < 0)
+		goto free_rproc;
+
+	platform_set_drvdata(pdev, rproc);
+
+	/* get optional vring and kick interrupts for supporting virtio rpmsg */
+	pru->irq_vring = platform_get_irq_byname(pdev, "vring");
+	if (pru->irq_vring <= 0) {
+		ret = pru->irq_vring;
+		if (ret == -EPROBE_DEFER)
+			goto free_rproc;
+		dev_dbg(dev, "unable to get vring interrupt, status = %d\n",
+			ret);
+	}
+
+	pru->irq_kick = platform_get_irq_byname(pdev, "kick");
+	if (pru->irq_kick <= 0) {
+		ret = pru->irq_kick;
+		if (ret == -EPROBE_DEFER)
+			goto free_rproc;
+		dev_dbg(dev, "unable to get kick interrupt, status = %d\n",
+			ret);
+	}
+
+	/*
+	 * get optional mailbox for virtio rpmsg signalling if vring and kick
+	 * interrupts are not specified for OMAP architecture based SoCs
+	 */
+	if (pru->irq_vring <= 0 && pru->irq_kick <= 0 &&
+	    !of_device_is_compatible(np, "ti,k2g-pru")) {
+		client = &pru->client;
+		client->dev = dev;
+		client->tx_done = NULL;
+		client->rx_callback = pru_rproc_mbox_callback;
+		client->tx_block = false;
+		client->knows_txdone = false;
+		pru->mbox = mbox_request_channel(client, 0);
+		if (IS_ERR(pru->mbox)) {
+			ret = PTR_ERR(pru->mbox);
+			pru->mbox = NULL;
+			dev_dbg(dev, "unable to get mailbox channel, status = %d\n",
+				ret);
+		}
+	}
+
+	ret = rproc_add(pru->rproc);
+	if (ret) {
+		dev_err(dev, "rproc_add failed: %d\n", ret);
+		goto put_mbox;
+	}
+
+	pru_rproc_create_debug_entries(rproc);
+
+	dev_info(dev, "PRU rproc node %s probed successfully\n", np->full_name);
+
+	return 0;
+
+put_mbox:
+	mbox_free_channel(pru->mbox);
+free_rproc:
+	rproc_free(rproc);
+	return ret;
+}
+
+static int pru_rproc_remove(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct rproc *rproc = platform_get_drvdata(pdev);
+	struct pru_rproc *pru = rproc->priv;
+
+	dev_info(dev, "%s: removing rproc %s\n", __func__, rproc->name);
+
+	mbox_free_channel(pru->mbox);
+
+	rproc_del(rproc);
+	rproc_free(rproc);
+
+	return 0;
+}
+
+static const struct of_device_id pru_rproc_match[] = {
+	{ .compatible = "ti,am3356-pru", },
+	{ .compatible = "ti,am4376-pru", },
+	{ .compatible = "ti,am5728-pru", },
+	{ .compatible = "ti,k2g-pru",    },
+	{ .compatible = "ti,am654-pru",  },
+	{ .compatible = "ti,am654-rtu",  },
+	{ .compatible = "ti,j721e-pru",  },
+	{ .compatible = "ti,j721e-rtu",  },
+	{ .compatible = "ti,j721e-tx-pru",  },
+	{},
+};
+MODULE_DEVICE_TABLE(of, pru_rproc_match);
+
+static struct platform_driver pru_rproc_driver = {
+	.driver = {
+		.name   = "pru-rproc",
+		.of_match_table = pru_rproc_match,
+		.suppress_bind_attrs = true,
+	},
+	.probe  = pru_rproc_probe,
+	.remove = pru_rproc_remove,
+};
+module_platform_driver(pru_rproc_driver);
+
+MODULE_AUTHOR("Suman Anna <s-anna@ti.com>");
+MODULE_DESCRIPTION("PRU-ICSS Remote Processor Driver");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/remoteproc/pru_rproc.h b/drivers/remoteproc/pru_rproc.h
--- a/drivers/remoteproc/pru_rproc.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/remoteproc/pru_rproc.h	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,92 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */
+/*
+ * PRUSS Remote Processor specific types
+ *
+ * Copyright (C) 2014-2019 Texas Instruments Incorporated - http://www.ti.com/
+ *	Suman Anna <s-anna@ti.com>
+ */
+
+#ifndef _PRU_RPROC_H_
+#define _PRU_RPROC_H_
+
+/**
+ * enum pruss_rsc_types - PRU specific resource types
+ *
+ * @PRUSS_RSC_INTRS: Resource holding information on PRU INTC configuration
+ * @PRUSS_RSC_MAX: Indicates end of known/defined PRU resource types.
+ *		   This should be the last definition.
+ *
+ * Introduce new vendor resource types before PRUSS_RSC_MAX.
+ */
+enum pruss_rsc_types {
+	PRUSS_RSC_INTRS	= 1,
+	PRUSS_RSC_MAX	= 2,
+};
+
+/**
+ * struct pruss_event_chnl - PRU system events _to_ channel mapping
+ * @event: number of the system event
+ * @chnl: channel number assigned to a given @event
+ *
+ * PRU system events are mapped to channels, and these channels are mapped
+ * to host interrupts. Events can be mapped to channels in a one-to-one or
+ * many-to-one ratio (multiple events per channel), and channels can be
+ * mapped to host interrupts in a one-to-one or many-to-one ratio (multiple
+ * channels per interrupt).
+ *
+ */
+struct pruss_event_chnl {
+	s8 event;
+	s8 chnl;
+};
+
+/**
+ * struct fw_rsc_pruss_intrmap - custom/vendor resource to define PRU interrupts
+ * @reserved: reserved field providing padding and alignment
+ * @chnl_host_intr_map: array of PRU channels to host interrupt mappings
+ * @event_chnl_map_size: number of event_channel mappings defined in
+ *			 @event_chnl_map_addr
+ * @event_chnl_map_addr: PRU device address of pointer to array of events to
+ *			 channel mappings (struct pruss_event_chnl elements)
+ *
+ * PRU system events are mapped to channels, and these channels are mapped
+ * to host interrupts. Events can be mapped to channels in a one-to-one or
+ * many-to-one ratio (multiple events per channel), and channels can be
+ * mapped to host interrupts in a one-to-one or many-to-one ratio (multiple
+ * channels per interrupt).
+ */
+struct fw_rsc_pruss_intrmap {
+	u16 reserved;
+	s8 chnl_host_intr_map[10];
+	u32 event_chnl_map_size;
+	u32 event_chnl_map_addr;
+};
+
+/**
+ * struct fw_rsc_pruss_intrmap_k3 - K3 custom resource to define PRU interrupts
+ * @chnl_host_intr_map: array of PRU channels to host interrupt mappings
+ * @event_chnl_map_size: number of event_channel mappings defined in
+ *			 @event_chnl_map_addr
+ * @event_chnl_map_addr: PRU device address of pointer to array of events to
+ *			 channel mappings (struct pruss_event_chnl elements)
+ *
+ * PRU system events are mapped to channels, and these channels are mapped
+ * to host interrupts. Events can be mapped to channels in a one-to-one or
+ * many-to-one ratio (multiple events per channel), and channels can be
+ * mapped to host interrupts in a one-to-one or many-to-one ratio (multiple
+ * channels per interrupt).
+ *
+ * This structure needs to be used using PRU vendor interrupt resource version
+ * number 1. This structure is to be used with firmwares dealing with the
+ * additional host interrupts on ICSSG IP instances. The firmwares for PRU
+ * cores on ICSSG can get away with the standard version (if not dealing with
+ * Task Manager), but the firmwares for RTU cores would definitely need this
+ * for mapping to the corresponding higher host interrupts.
+ */
+struct fw_rsc_pruss_intrmap_k3 {
+	s8 chnl_host_intr_map[20];
+	u32 event_chnl_map_size;
+	u32 event_chnl_map_addr;
+};
+
+#endif	/* _PRU_RPROC_H_ */
diff -Naur --no-dereference a/drivers/remoteproc/qcom_adsp_pil.c b/drivers/remoteproc/qcom_adsp_pil.c
--- a/drivers/remoteproc/qcom_adsp_pil.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/remoteproc/qcom_adsp_pil.c	2020-06-19 21:00:31.424227716 -0400
@@ -167,7 +167,7 @@
 	return ret;
 }
 
-static void *adsp_da_to_va(struct rproc *rproc, u64 da, int len)
+static void *adsp_da_to_va(struct rproc *rproc, u64 da, int len, u32 flags)
 {
 	struct qcom_adsp *adsp = (struct qcom_adsp *)rproc->priv;
 	int offset;
diff -Naur --no-dereference a/drivers/remoteproc/qcom_q6v5_pil.c b/drivers/remoteproc/qcom_q6v5_pil.c
--- a/drivers/remoteproc/qcom_q6v5_pil.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/remoteproc/qcom_q6v5_pil.c	2020-06-19 21:00:31.424227716 -0400
@@ -976,7 +976,7 @@
 	return 0;
 }
 
-static void *q6v5_da_to_va(struct rproc *rproc, u64 da, int len)
+static void *q6v5_da_to_va(struct rproc *rproc, u64 da, int len, u32 flags)
 {
 	struct q6v5 *qproc = rproc->priv;
 	int offset;
diff -Naur --no-dereference a/drivers/remoteproc/qcom_q6v5_wcss.c b/drivers/remoteproc/qcom_q6v5_wcss.c
--- a/drivers/remoteproc/qcom_q6v5_wcss.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/remoteproc/qcom_q6v5_wcss.c	2020-06-19 21:00:31.424227716 -0400
@@ -406,7 +406,7 @@
 	return 0;
 }
 
-static void *q6v5_wcss_da_to_va(struct rproc *rproc, u64 da, int len)
+static void *q6v5_wcss_da_to_va(struct rproc *rproc, u64 da, int len, u32 flags)
 {
 	struct q6v5_wcss *wcss = rproc->priv;
 	int offset;
diff -Naur --no-dereference a/drivers/remoteproc/qcom_wcnss.c b/drivers/remoteproc/qcom_wcnss.c
--- a/drivers/remoteproc/qcom_wcnss.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/remoteproc/qcom_wcnss.c	2020-06-19 21:00:31.424227716 -0400
@@ -295,7 +295,7 @@
 	return ret;
 }
 
-static void *wcnss_da_to_va(struct rproc *rproc, u64 da, int len)
+static void *wcnss_da_to_va(struct rproc *rproc, u64 da, int len, u32 flags)
 {
 	struct qcom_wcnss *wcnss = (struct qcom_wcnss *)rproc->priv;
 	int offset;
diff -Naur --no-dereference a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c
--- a/drivers/remoteproc/remoteproc_core.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/remoteproc/remoteproc_core.c	2020-06-19 21:00:31.424227716 -0400
@@ -41,6 +41,9 @@
 #include <linux/crc32.h>
 #include <linux/virtio_ids.h>
 #include <linux/virtio_ring.h>
+#include <linux/vmalloc.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
 #include <asm/byteorder.h>
 
 #include "remoteproc_internal.h"
@@ -51,7 +54,7 @@
 typedef int (*rproc_handle_resources_t)(struct rproc *rproc,
 				struct resource_table *table, int len);
 typedef int (*rproc_handle_resource_t)(struct rproc *rproc,
-				 void *, int offset, int avail);
+				 void *, int offset, int avail, u16 ver);
 
 /* Unique indices for remoteproc devices */
 static DEFINE_IDA(rproc_dev_index);
@@ -145,6 +148,7 @@
  * @rproc: handle of a remote processor
  * @da: remoteproc device address to translate
  * @len: length of the memory region @da is pointing to
+ * @flags: flags to pass onto platform implementations for aiding translations
  *
  * Some remote processors will ask us to allocate them physically contiguous
  * memory regions (which we call "carveouts"), and map them to specific
@@ -160,7 +164,10 @@
  * carveouts and translate specific device addresses to kernel virtual addresses
  * so we can access the referenced memory. This function also allows to perform
  * translations on the internal remoteproc memory regions through a platform
- * implementation specific da_to_va ops, if present.
+ * implementation specific da_to_va ops, if present. The @flags field is passed
+ * onto these ops to aid the translation within the ops implementation. The
+ * @flags field is to be passed as a combination of the RPROC_FLAGS_xxx type
+ * and the pertinent flags value for that type.
  *
  * The function returns a valid kernel address on success or NULL on failure.
  *
@@ -169,13 +176,13 @@
  * here the output of the DMA API for the carveouts, which should be more
  * correct.
  */
-void *rproc_da_to_va(struct rproc *rproc, u64 da, int len)
+void *rproc_da_to_va(struct rproc *rproc, u64 da, int len, u32 flags)
 {
 	struct rproc_mem_entry *carveout;
 	void *ptr = NULL;
 
 	if (rproc->ops->da_to_va) {
-		ptr = rproc->ops->da_to_va(rproc, da, len);
+		ptr = rproc->ops->da_to_va(rproc, da, len, flags);
 		if (ptr)
 			goto out;
 	}
@@ -201,6 +208,61 @@
 }
 EXPORT_SYMBOL(rproc_da_to_va);
 
+/**
+ * rproc_pa_to_da() - lookup the rproc device address for a physical address
+ * @rproc: handle of a remote processor
+ * @pa: physical address of the buffer to translate
+ * @da: device address to return
+ *
+ * Communication clients of remote processors usually would need a means to
+ * convert a host buffer pointer to an equivalent device virtual address pointer
+ * that the code running on the remote processor can operate on. These buffer
+ * pointers can either be from the physically contiguous memory regions (or
+ * "carveouts") or can be some memory-mapped Device IO memory. This function
+ * provides a means to translate a given physical address to its associated
+ * device address.
+ *
+ * The function looks through both the carveouts and the device memory mappings
+ * since both of them are stored in separate lists.
+ *
+ * Returns 0 on success, or an appropriate error code otherwise. The translated
+ * device address is returned through the appropriate function argument.
+ */
+int rproc_pa_to_da(struct rproc *rproc, phys_addr_t pa, u64 *da)
+{
+	int ret = -EINVAL;
+	struct rproc_mem_entry *maps = NULL;
+
+	if (!rproc || !da)
+		return -EINVAL;
+
+	if (mutex_lock_interruptible(&rproc->lock))
+		return -EINTR;
+
+	if (rproc->state == RPROC_RUNNING || rproc->state == RPROC_SUSPENDED) {
+		/* Look in the mappings first */
+		list_for_each_entry(maps, &rproc->mappings, node) {
+			if (pa >= maps->dma && pa < (maps->dma + maps->len)) {
+				*da = maps->da + (pa - maps->dma);
+				ret = 0;
+				goto exit;
+			}
+		}
+		/* If not, check in the carveouts */
+		list_for_each_entry(maps, &rproc->carveouts, node) {
+			if (pa >= maps->dma && pa < (maps->dma + maps->len)) {
+				*da = maps->da + (pa - maps->dma);
+				ret = 0;
+				break;
+			}
+		}
+	}
+exit:
+	mutex_unlock(&rproc->lock);
+	return ret;
+}
+EXPORT_SYMBOL(rproc_pa_to_da);
+
 int rproc_alloc_vring(struct rproc_vdev *rvdev, int i)
 {
 	struct rproc *rproc = rvdev->rproc;
@@ -320,6 +382,7 @@
  * @rproc: the remote processor
  * @rsc: the vring resource descriptor
  * @avail: size of available data (for sanity checking the image)
+ * @ver: version number of the resource type
  *
  * This resource entry requests the host to statically register a virtio
  * device (vdev), and setup everything needed to support it. It contains
@@ -343,7 +406,7 @@
  * Returns 0 on success, or an appropriate error code otherwise
  */
 static int rproc_handle_vdev(struct rproc *rproc, struct fw_rsc_vdev *rsc,
-			     int offset, int avail)
+			     int offset, int avail, u16 ver)
 {
 	struct device *dev = &rproc->dev;
 	struct rproc_vdev *rvdev;
@@ -435,10 +498,108 @@
 }
 
 /**
+ * rproc_handle_last_trace() - setup a buffer to capture the trace snapshot
+ *				before recovery
+ * @rproc: the remote processor
+ * @trace: the trace resource descriptor
+ * @count: the index of the trace under process
+ *
+ * The last trace is allocated and the contents of the trace buffer are
+ * copied during a recovery cleanup. Once, the contents get copied, the
+ * trace buffers are cleaned up for re-use.
+ *
+ * It might also happen that the remoteproc binary changes between the
+ * time that it was loaded and the time that it crashed. In this case,
+ * the trace descriptors might have changed too. The last traces are
+ * re-built as required in this case.
+ *
+ * Returns 0 on success, or an appropriate error code otherwise
+ */
+static int rproc_handle_last_trace(struct rproc *rproc,
+				   struct rproc_mem_entry *trace, int count)
+{
+	struct rproc_mem_entry *trace_last, *tmp_trace;
+	struct device *dev = &rproc->dev;
+	char name[15];
+	int i = 0;
+	bool new_trace = false;
+
+	if (!rproc || !trace)
+		return -EINVAL;
+
+	/* we need a new trace in this case */
+	if (count > rproc->num_last_traces) {
+		new_trace = true;
+		/*
+		 * make sure snprintf always null terminates, even if truncating
+		 */
+		snprintf(name, sizeof(name), "trace%d_last", (count - 1));
+		trace_last = kzalloc(sizeof(*trace_last), GFP_KERNEL);
+		if (!trace_last) {
+			dev_err(dev, "kzalloc failed for trace%d_last\n",
+				count);
+			return -ENOMEM;
+		}
+	} else {
+		/* try to reuse buffers here */
+		list_for_each_entry_safe(trace_last, tmp_trace,
+					 &rproc->last_traces, node) {
+			if (++i == count)
+				break;
+		}
+
+		/* if we can reuse the trace, copy buffer and exit */
+		if (trace_last->len == trace->len)
+			goto copy_and_exit;
+
+		/* can reuse the trace struct but not the buffer */
+		vfree(trace_last->va);
+		trace_last->va = NULL;
+		trace_last->len = 0;
+	}
+
+	trace_last->len = trace->len;
+	trace_last->va = vmalloc(sizeof(u32) * trace_last->len);
+	if (!trace_last->va) {
+		dev_err(dev, "vmalloc failed for trace%d_last\n", count);
+		if (!new_trace) {
+			list_del(&trace_last->node);
+			rproc->num_last_traces--;
+		}
+		kfree(trace_last);
+		return -ENOMEM;
+	}
+
+	/* create the debugfs entry */
+	if (new_trace) {
+		trace_last->priv = rproc_create_trace_file(name, rproc,
+							   trace_last);
+		if (!trace_last->priv) {
+			dev_err(dev, "trace%d_last create debugfs failed\n",
+				count);
+			vfree(trace_last->va);
+			kfree(trace_last);
+			return -EINVAL;
+		}
+
+		/* add it to the trace list */
+		list_add_tail(&trace_last->node, &rproc->last_traces);
+		rproc->num_last_traces++;
+	}
+
+copy_and_exit:
+	/* copy the trace to last trace */
+	memcpy(trace_last->va, trace->va, trace->len);
+
+	return 0;
+}
+
+/**
  * rproc_handle_trace() - handle a shared trace buffer resource
  * @rproc: the remote processor
  * @rsc: the trace resource descriptor
  * @avail: size of available data (for sanity checking the image)
+ * @ver: version number of the resource type
  *
  * In case the remote processor dumps trace logs into memory,
  * export it via debugfs.
@@ -450,27 +611,51 @@
  *
  * Returns 0 on success, or an appropriate error code otherwise
  */
-static int rproc_handle_trace(struct rproc *rproc, struct fw_rsc_trace *rsc,
-			      int offset, int avail)
+static int rproc_handle_trace(struct rproc *rproc, void *rsc,
+			      int offset, int avail, u16 ver)
 {
 	struct rproc_mem_entry *trace;
 	struct device *dev = &rproc->dev;
+	struct fw_rsc_trace *rsc1;
+	struct fw_rsc_trace2 *rsc2;
 	void *ptr;
 	char name[15];
+	size_t rsc_size;
+	u32 reserved;
+	u64 da;
+	u32 len;
+
+	if (!ver) {
+		rsc1 = (struct fw_rsc_trace *)rsc;
+		rsc_size = sizeof(*rsc1);
+		reserved = rsc1->reserved;
+		da = rsc1->da;
+		len = rsc1->len;
+	} else if (ver == 1) {
+		rsc2 = (struct fw_rsc_trace2 *)rsc;
+		rsc_size = sizeof(*rsc2);
+		reserved = rsc2->reserved;
+		da = rsc2->da;
+		len = rsc2->len;
+	} else {
+		dev_err(dev, "unsupported trace rsc version %d\n", ver);
+		return -EINVAL;
+	}
 
-	if (sizeof(*rsc) > avail) {
+	if (rsc_size > avail) {
 		dev_err(dev, "trace rsc is truncated\n");
 		return -EINVAL;
 	}
 
 	/* make sure reserved bytes are zeroes */
-	if (rsc->reserved) {
-		dev_err(dev, "trace rsc has non zero reserved bytes\n");
+	if (reserved) {
+		dev_err(dev, "trace rsc has non zero reserved bytes, value = 0x%x\n",
+			reserved);
 		return -EINVAL;
 	}
 
 	/* what's the kernel address of this resource ? */
-	ptr = rproc_da_to_va(rproc, rsc->da, rsc->len);
+	ptr = rproc_da_to_va(rproc, da, len, RPROC_FLAGS_NONE);
 	if (!ptr) {
 		dev_err(dev, "erroneous trace resource entry\n");
 		return -EINVAL;
@@ -481,7 +666,7 @@
 		return -ENOMEM;
 
 	/* set the trace buffer dma properties */
-	trace->len = rsc->len;
+	trace->len = len;
 	trace->va = ptr;
 
 	/* make sure snprintf always null terminates, even if truncating */
@@ -499,8 +684,8 @@
 
 	rproc->num_traces++;
 
-	dev_dbg(dev, "%s added: va %pK, da 0x%x, len 0x%x\n",
-		name, ptr, rsc->da, rsc->len);
+	dev_dbg(dev, "%s added: va %pK, da 0x%llx, len 0x%x\n",
+		name, ptr, da, len);
 
 	return 0;
 }
@@ -510,6 +695,7 @@
  * @rproc: remote processor handle
  * @rsc: the devmem resource entry
  * @avail: size of available data (for sanity checking the image)
+ * @ver: version number of the resource type
  *
  * Remote processors commonly need to access certain on-chip peripherals.
  *
@@ -531,7 +717,7 @@
  * are outside those ranges.
  */
 static int rproc_handle_devmem(struct rproc *rproc, struct fw_rsc_devmem *rsc,
-			       int offset, int avail)
+			       int offset, int avail, u16 ver)
 {
 	struct rproc_mem_entry *mapping;
 	struct device *dev = &rproc->dev;
@@ -539,7 +725,7 @@
 
 	/* no point in handling this resource without a valid iommu domain */
 	if (!rproc->domain)
-		return -EINVAL;
+		return 0;
 
 	if (sizeof(*rsc) > avail) {
 		dev_err(dev, "devmem rsc is truncated\n");
@@ -556,10 +742,13 @@
 	if (!mapping)
 		return -ENOMEM;
 
-	ret = iommu_map(rproc->domain, rsc->da, rsc->pa, rsc->len, rsc->flags);
-	if (ret) {
-		dev_err(dev, "failed to map devmem: %d\n", ret);
-		goto out;
+	if (!rproc->late_attach) {
+		ret = iommu_map(rproc->domain, rsc->da, rsc->pa, rsc->len,
+				rsc->flags);
+		if (ret) {
+			dev_err(dev, "failed to map devmem: %d\n", ret);
+			goto out;
+		}
 	}
 
 	/*
@@ -569,12 +758,17 @@
 	 * We can't trust the remote processor not to change the resource
 	 * table, so we must maintain this info independently.
 	 */
+	mapping->dma = rsc->pa;
 	mapping->da = rsc->da;
 	mapping->len = rsc->len;
 	list_add_tail(&mapping->node, &rproc->mappings);
 
-	dev_dbg(dev, "mapped devmem pa 0x%x, da 0x%x, len 0x%x\n",
-		rsc->pa, rsc->da, rsc->len);
+	if (!rproc->late_attach)
+		dev_dbg(dev, "mapped devmem pa 0x%x, da 0x%x, len 0x%x\n",
+			rsc->pa, rsc->da, rsc->len);
+	else
+		dev_dbg(dev, "late-attach: processed devmem pa 0x%x, da 0x%x, len 0x%x\n",
+			rsc->pa, rsc->da, rsc->len);
 
 	return 0;
 
@@ -588,6 +782,7 @@
  * @rproc: rproc handle
  * @rsc: the resource entry
  * @avail: size of available data (for image validation)
+ * @ver: version number of the resource type
  *
  * This function will handle firmware requests for allocation of physically
  * contiguous memory regions.
@@ -603,7 +798,7 @@
  */
 static int rproc_handle_carveout(struct rproc *rproc,
 				 struct fw_rsc_carveout *rsc,
-				 int offset, int avail)
+				 int offset, int avail, u16 ver)
 {
 	struct rproc_mem_entry *carveout, *mapping;
 	struct device *dev = &rproc->dev;
@@ -629,7 +824,13 @@
 	if (!carveout)
 		return -ENOMEM;
 
-	va = dma_alloc_coherent(dev->parent, rsc->len, &dma, GFP_KERNEL);
+	if (rproc->late_attach) {
+		va = dma_malloc_coherent(dev->parent, rsc->len, &dma,
+					 GFP_KERNEL);
+	} else {
+		va = dma_alloc_coherent(dev->parent, rsc->len, &dma,
+					GFP_KERNEL);
+	}
 	if (!va) {
 		dev_err(dev->parent,
 			"failed to allocate dma memory: len 0x%x\n", rsc->len);
@@ -664,11 +865,13 @@
 			goto dma_free;
 		}
 
-		ret = iommu_map(rproc->domain, rsc->da, dma, rsc->len,
-				rsc->flags);
-		if (ret) {
-			dev_err(dev, "iommu_map failed: %d\n", ret);
-			goto free_mapping;
+		if (!rproc->late_attach) {
+			ret = iommu_map(rproc->domain, rsc->da, dma, rsc->len,
+					rsc->flags);
+			if (ret) {
+				dev_err(dev, "iommu_map failed: %d\n", ret);
+				goto free_mapping;
+			}
 		}
 
 		/*
@@ -682,8 +885,13 @@
 		mapping->len = rsc->len;
 		list_add_tail(&mapping->node, &rproc->mappings);
 
-		dev_dbg(dev, "carveout mapped 0x%x to %pad\n",
-			rsc->da, &dma);
+		if (!rproc->late_attach)
+			dev_dbg(dev, "carveout mapped 0x%x to %pad\n",
+				rsc->da, &dma);
+		else
+			dev_dbg(dev, "late-attach: carveout processed 0x%x to %pad\n",
+				rsc->da, &dma);
+
 	}
 
 	/*
@@ -709,6 +917,7 @@
 	carveout->len = rsc->len;
 	carveout->dma = dma;
 	carveout->da = rsc->da;
+	strlcpy(carveout->name, rsc->name, sizeof(carveout->name));
 
 	list_add_tail(&carveout->node, &rproc->carveouts);
 
@@ -723,6 +932,42 @@
 	return ret;
 }
 
+/**
+ * rproc_handle_vendor_rsc() - provide implementation specific hook
+ *			       to handle vendor/custom resources
+ * @rproc: the remote processor
+ * @rsc: vendor resource to be handled by remoteproc drivers
+ * @offset: offset of the resource data in resource table
+ * @avail: size of available data
+ *
+ * Remoteproc implementations might want to add resource table entries
+ * that are not generic enough to be handled by the framework. This
+ * provides a hook to handle such custom resources. Note that a single
+ * hook is reused between RSC_PRELOAD_VENDOR and RSC_PRELOAD_VENDOR
+ * resources with the platform driver implementation distinguishing
+ * the two based on the sub-type resource.
+ *
+ * Returns 0 on success, or an appropriate error code otherwise
+ */
+static int rproc_handle_vendor_rsc(struct rproc *rproc,
+				   struct fw_rsc_vendor *rsc,
+				   int offset, int avail)
+{
+	struct device *dev = &rproc->dev;
+
+	if (!rproc->ops->handle_vendor_rsc) {
+		dev_err(dev, "vendor resource handler not implemented, ignoring resource\n");
+		return 0;
+	}
+
+	if (sizeof(*rsc) > avail) {
+		dev_err(dev, "vendor resource is truncated\n");
+		return -EINVAL;
+	}
+
+	return rproc->ops->handle_vendor_rsc(rproc, (void *)rsc);
+}
+
 /*
  * A lookup table for resource handlers. The indices are defined in
  * enum fw_resource_type.
@@ -731,9 +976,15 @@
 	[RSC_CARVEOUT] = (rproc_handle_resource_t)rproc_handle_carveout,
 	[RSC_DEVMEM] = (rproc_handle_resource_t)rproc_handle_devmem,
 	[RSC_TRACE] = (rproc_handle_resource_t)rproc_handle_trace,
+	[RSC_PRELOAD_VENDOR] = (rproc_handle_resource_t)rproc_handle_vendor_rsc,
 	[RSC_VDEV] = (rproc_handle_resource_t)rproc_handle_vdev,
 };
 
+static rproc_handle_resource_t rproc_post_loading_handlers[RSC_LAST] = {
+	[RSC_POSTLOAD_VENDOR] =
+			(rproc_handle_resource_t)rproc_handle_vendor_rsc,
+};
+
 /* handle firmware resource entries before booting the remote processor */
 static int rproc_handle_resources(struct rproc *rproc,
 				  rproc_handle_resource_t handlers[RSC_LAST])
@@ -757,18 +1008,19 @@
 			return -EINVAL;
 		}
 
-		dev_dbg(dev, "rsc: type %d\n", hdr->type);
+		dev_dbg(dev, "rsc: type %d vers %d\n", hdr->st.t, hdr->st.v);
 
-		if (hdr->type >= RSC_LAST) {
-			dev_warn(dev, "unsupported resource %d\n", hdr->type);
+		if (hdr->st.t >= RSC_LAST) {
+			dev_warn(dev, "unsupported resource %d\n", hdr->st.t);
 			continue;
 		}
 
-		handler = handlers[hdr->type];
+		handler = handlers[hdr->st.t];
 		if (!handler)
 			continue;
 
-		ret = handler(rproc, rsc, offset + sizeof(*hdr), avail);
+		ret = handler(rproc, rsc, offset + sizeof(*hdr), avail,
+			      hdr->st.v);
 		if (ret)
 			break;
 	}
@@ -859,6 +1111,18 @@
 }
 
 /**
+ * rproc_free_last_trace() - helper function to cleanup a last trace entry
+ * @trace: the last trace element to be cleaned up
+ */
+static void rproc_free_last_trace(struct rproc_mem_entry *trace)
+{
+	rproc_remove_trace_file(trace->priv);
+	list_del(&trace->node);
+	vfree(trace->va);
+	kfree(trace);
+}
+
+/**
  * rproc_resource_cleanup() - clean up and free all acquired resources
  * @rproc: rproc handle
  *
@@ -870,24 +1134,45 @@
 	struct rproc_mem_entry *entry, *tmp;
 	struct rproc_vdev *rvdev, *rvtmp;
 	struct device *dev = &rproc->dev;
+	int count = 0, i = rproc->num_traces;
 
 	/* clean up debugfs trace entries */
 	list_for_each_entry_safe(entry, tmp, &rproc->traces, node) {
+		/* handle last trace here */
+		if (rproc->state == RPROC_CRASHED)
+			rproc_handle_last_trace(rproc, entry, ++count);
+
 		rproc_remove_trace_file(entry->priv);
-		rproc->num_traces--;
 		list_del(&entry->node);
 		kfree(entry);
 	}
+	rproc->num_traces = 0;
+
+	/*
+	 * clean up debugfs last trace entries. This either deletes all last
+	 * trace entries during cleanup or just the remaining entries, if any,
+	 * in case of a crash.
+	 */
+	list_for_each_entry_safe(entry, tmp, &rproc->last_traces, node) {
+		/* skip the valid traces */
+		if ((i--) && rproc->state == RPROC_CRASHED)
+			continue;
+		rproc_free_last_trace(entry);
+		rproc->num_last_traces--;
+	}
 
 	/* clean up iommu mapping entries */
 	list_for_each_entry_safe(entry, tmp, &rproc->mappings, node) {
 		size_t unmapped;
 
-		unmapped = iommu_unmap(rproc->domain, entry->da, entry->len);
-		if (unmapped != entry->len) {
-			/* nothing much to do besides complaining */
-			dev_err(dev, "failed to unmap %u/%zu\n", entry->len,
-				unmapped);
+		if (!rproc->late_attach) {
+			unmapped = iommu_unmap(rproc->domain, entry->da,
+					       entry->len);
+			if (unmapped != entry->len) {
+				/* nothing much to do besides complaining */
+				dev_err(dev, "failed to unmap %u/%zu\n",
+					entry->len, unmapped);
+			}
 		}
 
 		list_del(&entry->node);
@@ -909,19 +1194,75 @@
 	rproc_coredump_cleanup(rproc);
 }
 
-static int rproc_start(struct rproc *rproc, const struct firmware *fw)
+/*
+ * take a firmware and boot a remote processor with it.
+ */
+static int rproc_fw_boot(struct rproc *rproc, const struct firmware *fw)
 {
-	struct resource_table *loaded_table;
 	struct device *dev = &rproc->dev;
+	const char *name = rproc->firmware;
+	struct resource_table *loaded_table;
 	int ret;
 
-	/* load the ELF segments to memory */
-	ret = rproc_load_segments(rproc, fw);
+	ret = rproc_fw_sanity_check(rproc, fw);
+	if (ret)
+		return ret;
+
+	if (!rproc->skip_firmware_request)
+		dev_info(dev, "Booting fw image %s, size %zd\n",
+			 name, fw->size);
+	else
+		dev_info(dev, "Booting unspecified pre-loaded fw image\n");
+
+	/*
+	 * if enabling an IOMMU isn't relevant for this rproc, this is
+	 * just a nop
+	 */
+	ret = rproc_enable_iommu(rproc);
 	if (ret) {
-		dev_err(dev, "Failed to load program segments: %d\n", ret);
+		dev_err(dev, "can't enable iommu: %d\n", ret);
 		return ret;
 	}
 
+	/* Prepare rproc for firmware loading if needed */
+	if (rproc->ops->prepare) {
+		ret = rproc->ops->prepare(rproc);
+		if (ret) {
+			dev_err(dev, "can't prepare rproc %s: %d\n",
+				rproc->name, ret);
+			goto disable_iommu;
+		}
+	}
+
+	rproc->bootaddr = rproc_get_boot_addr(rproc, fw);
+
+	/* Load resource table, core dump segment list etc from the firmware */
+	ret = rproc_parse_fw(rproc, fw);
+	if (ret)
+		goto unprepare_rproc;
+
+	/* reset max_notifyid */
+	rproc->max_notifyid = -1;
+
+	/* handle fw resources which are required to boot rproc */
+	ret = rproc_handle_resources(rproc, rproc_loading_handlers);
+	if (ret) {
+		dev_err(dev, "Failed to process resources: %d\n", ret);
+		goto clean_up_resources;
+	}
+
+	if (!rproc->skip_load && !rproc->late_attach) {
+		/* load the ELF segments to memory */
+		ret = rproc_load_segments(rproc, fw);
+		if (ret) {
+			dev_err(dev, "Failed to load program segments: %d\n",
+				ret);
+			goto clean_up_resources;
+		}
+	} else {
+		dev_dbg(dev, "Skipped program segments load for pre-booted rproc\n");
+	}
+
 	/*
 	 * The starting device has been given the rproc->cached_table as the
 	 * resource table. The address of the vring along with the other
@@ -936,6 +1277,14 @@
 		rproc->table_ptr = loaded_table;
 	}
 
+	/* handle fw resources which require fw segments to be loaded */
+	ret = rproc_handle_resources(rproc, rproc_post_loading_handlers);
+	if (ret) {
+		dev_err(dev, "Failed to process post-loading resources: %d\n",
+			ret);
+		goto reset_table_ptr;
+	}
+
 	ret = rproc_prepare_subdevices(rproc);
 	if (ret) {
 		dev_err(dev, "failed to prepare subdevices for %s: %d\n",
@@ -970,63 +1319,16 @@
 	rproc_unprepare_subdevices(rproc);
 reset_table_ptr:
 	rproc->table_ptr = rproc->cached_table;
-
-	return ret;
-}
-
-/*
- * take a firmware and boot a remote processor with it.
- */
-static int rproc_fw_boot(struct rproc *rproc, const struct firmware *fw)
-{
-	struct device *dev = &rproc->dev;
-	const char *name = rproc->firmware;
-	int ret;
-
-	ret = rproc_fw_sanity_check(rproc, fw);
-	if (ret)
-		return ret;
-
-	dev_info(dev, "Booting fw image %s, size %zd\n", name, fw->size);
-
-	/*
-	 * if enabling an IOMMU isn't relevant for this rproc, this is
-	 * just a nop
-	 */
-	ret = rproc_enable_iommu(rproc);
-	if (ret) {
-		dev_err(dev, "can't enable iommu: %d\n", ret);
-		return ret;
-	}
-
-	rproc->bootaddr = rproc_get_boot_addr(rproc, fw);
-
-	/* Load resource table, core dump segment list etc from the firmware */
-	ret = rproc_parse_fw(rproc, fw);
-	if (ret)
-		goto disable_iommu;
-
-	/* reset max_notifyid */
-	rproc->max_notifyid = -1;
-
-	/* handle fw resources which are required to boot rproc */
-	ret = rproc_handle_resources(rproc, rproc_loading_handlers);
-	if (ret) {
-		dev_err(dev, "Failed to process resources: %d\n", ret);
-		goto clean_up_resources;
-	}
-
-	ret = rproc_start(rproc, fw);
-	if (ret)
-		goto clean_up_resources;
-
-	return 0;
-
 clean_up_resources:
 	rproc_resource_cleanup(rproc);
 	kfree(rproc->cached_table);
 	rproc->cached_table = NULL;
 	rproc->table_ptr = NULL;
+	rproc->table_sz = 0;
+unprepare_rproc:
+	/* release HW resources if needed */
+	if (rproc->ops->unprepare)
+		rproc->ops->unprepare(rproc);
 disable_iommu:
 	rproc_disable_iommu(rproc);
 	return ret;
@@ -1066,33 +1368,6 @@
 	return ret;
 }
 
-static int rproc_stop(struct rproc *rproc, bool crashed)
-{
-	struct device *dev = &rproc->dev;
-	int ret;
-
-	/* Stop any subdevices for the remote processor */
-	rproc_stop_subdevices(rproc, crashed);
-
-	/* the installed resource table is no longer accessible */
-	rproc->table_ptr = rproc->cached_table;
-
-	/* power off the remote processor */
-	ret = rproc->ops->stop(rproc);
-	if (ret) {
-		dev_err(dev, "can't stop rproc: %d\n", ret);
-		return ret;
-	}
-
-	rproc_unprepare_subdevices(rproc);
-
-	rproc->state = RPROC_OFFLINE;
-
-	dev_info(dev, "stopped remote processor %s\n", rproc->name);
-
-	return 0;
-}
-
 /**
  * rproc_coredump_add_segment() - add segment of device memory to coredump
  * @rproc:	handle of a remote processor
@@ -1183,7 +1458,8 @@
 		phdr->p_flags = PF_R | PF_W | PF_X;
 		phdr->p_align = 0;
 
-		ptr = rproc_da_to_va(rproc, segment->da, segment->size);
+		ptr = rproc_da_to_va(rproc, segment->da, segment->size,
+				     RPROC_FLAGS_NONE);
 		if (!ptr) {
 			dev_err(&rproc->dev,
 				"invalid coredump segment (%pad, %zu)\n",
@@ -1212,38 +1488,23 @@
  */
 int rproc_trigger_recovery(struct rproc *rproc)
 {
-	const struct firmware *firmware_p;
-	struct device *dev = &rproc->dev;
-	int ret;
-
-	dev_err(dev, "recovering %s\n", rproc->name);
-
-	ret = mutex_lock_interruptible(&rproc->lock);
-	if (ret)
-		return ret;
-
-	ret = rproc_stop(rproc, true);
-	if (ret)
-		goto unlock_mutex;
+	dev_err(&rproc->dev, "recovering %s\n", rproc->name);
 
-	/* generate coredump */
-	rproc_coredump(rproc);
+	init_completion(&rproc->crash_comp);
 
-	/* load firmware */
-	ret = request_firmware(&firmware_p, rproc->firmware, dev);
-	if (ret < 0) {
-		dev_err(dev, "request_firmware failed: %d\n", ret);
-		goto unlock_mutex;
-	}
+	/* shut down the remote */
+	/* TODO: make sure this works with rproc->power > 1 */
+	rproc_shutdown(rproc);
 
-	/* boot the remote processor up again */
-	ret = rproc_start(rproc, firmware_p);
+	/* wait until there is no more rproc users */
+	wait_for_completion(&rproc->crash_comp);
 
-	release_firmware(firmware_p);
+	/*
+	 * boot the remote processor up again
+	 */
+	rproc_boot(rproc);
 
-unlock_mutex:
-	mutex_unlock(&rproc->lock);
-	return ret;
+	return 0;
 }
 
 /**
@@ -1278,6 +1539,36 @@
 }
 
 /**
+ * rproc_get_id() - return the id for the rproc device
+ * @rproc: handle of a remote processor
+ *
+ * Each rproc device is associated with a platform device, which is created
+ * either from device tree (majority newer platforms) or using legacy style
+ * platform device creation (fewer legacy platforms). This function retrieves
+ * an unique id for each remote processor and is useful for clients needing
+ * to distinguish each of the remoteprocs. This unique id is derived using
+ * the platform device id for non-DT devices, or an alternate alias id for
+ * DT devices (since they do not have a valid platform device id). It is
+ * assumed that the platform devices were created with known ids or were
+ * given proper alias ids using the stem "rproc".
+ *
+ * Return: alias id for DT devices or platform device id for non-DT devices
+ * associated with the rproc
+ */
+int rproc_get_id(struct rproc *rproc)
+{
+	struct device *dev = rproc->dev.parent;
+	struct device_node *np = dev->of_node;
+	struct platform_device *pdev = to_platform_device(dev);
+
+	if (np)
+		return of_alias_get_id(np, "rproc");
+	else
+		return pdev->id;
+}
+EXPORT_SYMBOL(rproc_get_id);
+
+/**
  * rproc_boot() - boot a remote processor
  * @rproc: handle of a remote processor
  *
@@ -1321,16 +1612,19 @@
 
 	dev_info(dev, "powering up %s\n", rproc->name);
 
-	/* load firmware */
-	ret = request_firmware(&firmware_p, rproc->firmware, dev);
-	if (ret < 0) {
-		dev_err(dev, "request_firmware failed: %d\n", ret);
-		goto downref_rproc;
+	if (!rproc->skip_firmware_request) {
+		/* load firmware */
+		ret = request_firmware(&firmware_p, rproc->firmware, dev);
+		if (ret < 0) {
+			dev_err(dev, "request_firmware failed: %d\n", ret);
+			goto downref_rproc;
+		}
 	}
 
 	ret = rproc_fw_boot(rproc, firmware_p);
 
-	release_firmware(firmware_p);
+	if (!rproc->skip_firmware_request)
+		release_firmware(firmware_p);
 
 downref_rproc:
 	if (ret)
@@ -1364,6 +1658,7 @@
 {
 	struct device *dev = &rproc->dev;
 	int ret;
+	bool crashed = false;
 
 	ret = mutex_lock_interruptible(&rproc->lock);
 	if (ret) {
@@ -1375,21 +1670,52 @@
 	if (!atomic_dec_and_test(&rproc->power))
 		goto out;
 
-	ret = rproc_stop(rproc, false);
+	if (rproc->state == RPROC_CRASHED)
+		crashed = true;
+
+	/* remove any subdevices for the remote processor */
+	rproc_stop_subdevices(rproc, crashed);
+
+	/* power off the remote processor */
+	ret = rproc->ops->stop(rproc);
 	if (ret) {
 		atomic_inc(&rproc->power);
+		dev_err(dev, "can't stop rproc: %d\n", ret);
 		goto out;
 	}
 
+	rproc_unprepare_subdevices(rproc);
+
+	/* generate coredump */
+	if (rproc->state == RPROC_CRASHED)
+		rproc_coredump(rproc);
+
+	/* the installed resource table may no longer be accessible */
+	rproc->table_ptr = rproc->cached_table;
+
 	/* clean up all acquired resources */
 	rproc_resource_cleanup(rproc);
 
+	/* release HW resources if needed */
+	if (rproc->ops->unprepare)
+		rproc->ops->unprepare(rproc);
+
 	rproc_disable_iommu(rproc);
 
 	/* Free the copy of the resource table */
 	kfree(rproc->cached_table);
 	rproc->cached_table = NULL;
 	rproc->table_ptr = NULL;
+
+	/* if in crash state, unlock crash handler */
+	if (rproc->state == RPROC_CRASHED)
+		complete_all(&rproc->crash_comp);
+
+	rproc->state = RPROC_OFFLINE;
+	rproc->late_attach = 0;
+
+	dev_info(dev, "stopped remote processor %s\n", rproc->name);
+
 out:
 	mutex_unlock(&rproc->lock);
 }
@@ -1517,6 +1843,7 @@
 
 	kfree(rproc->firmware);
 	kfree(rproc->ops);
+	kfree(rproc->name);
 	kfree(rproc);
 }
 
@@ -1589,7 +1916,13 @@
 	}
 
 	rproc->firmware = p;
-	rproc->name = name;
+	rproc->name = kstrdup(name, GFP_KERNEL);
+	if (!rproc->name) {
+		kfree(p);
+		kfree(rproc->ops);
+		kfree(rproc);
+		return NULL;
+	}
 	rproc->priv = &rproc[1];
 	rproc->auto_boot = true;
 
@@ -1627,11 +1960,13 @@
 	INIT_LIST_HEAD(&rproc->carveouts);
 	INIT_LIST_HEAD(&rproc->mappings);
 	INIT_LIST_HEAD(&rproc->traces);
+	INIT_LIST_HEAD(&rproc->last_traces);
 	INIT_LIST_HEAD(&rproc->rvdevs);
 	INIT_LIST_HEAD(&rproc->subdevs);
 	INIT_LIST_HEAD(&rproc->dump_segments);
 
 	INIT_WORK(&rproc->crash_handler, rproc_crash_handler_work);
+	init_completion(&rproc->crash_comp);
 
 	rproc->state = RPROC_OFFLINE;
 
@@ -1687,6 +2022,8 @@
  */
 int rproc_del(struct rproc *rproc)
 {
+	struct rproc_mem_entry *entry, *tmp;
+
 	if (!rproc)
 		return -EINVAL;
 
@@ -1699,6 +2036,12 @@
 	rproc->state = RPROC_DELETED;
 	mutex_unlock(&rproc->lock);
 
+	/* clean up debugfs last trace entries */
+	list_for_each_entry_safe(entry, tmp, &rproc->last_traces, node) {
+		rproc_free_last_trace(entry);
+		rproc->num_last_traces--;
+	}
+
 	rproc_delete_debug_dir(rproc);
 
 	/* the rproc is downref'ed as soon as it's removed from the klist */
@@ -1774,11 +2117,75 @@
 	dev_err(&rproc->dev, "crash detected in %s: type %s\n",
 		rproc->name, rproc_crash_to_string(type));
 
-	/* create a new task to handle the error */
-	schedule_work(&rproc->crash_handler);
+	/* create a new task to handle the error if not scheduled already */
+	if (!work_busy(&rproc->crash_handler))
+		schedule_work(&rproc->crash_handler);
 }
 EXPORT_SYMBOL(rproc_report_crash);
 
+/**
+ * rproc_set_firmware() - assign a new firmware
+ * @rproc: rproc handle to which the new firmware is being assigned
+ * @fw_name: new firmware name to be assigned
+ *
+ * This function allows remoteproc drivers or clients to configure a custom
+ * firmware name that is different from the default name used during remoteproc
+ * registration. The function does not trigger a remote processor boot,
+ * only sets the firmware name used for a subsequent boot. This function
+ * should also be called only when the remote processor is offline.
+ *
+ * This allows either the userspace to configure a different name through
+ * sysfs or a kernel-level remoteproc or a remoteproc client driver to set
+ * a specific firmware when it is controlling the boot and shutdown of the
+ * remote processor.
+ *
+ * Returns 0 on success or a negative value upon failure
+ */
+int rproc_set_firmware(struct rproc *rproc, const char *fw_name)
+{
+	struct device *dev;
+	int ret, len;
+	char *p;
+
+	if (!rproc || !fw_name)
+		return -EINVAL;
+
+	dev = rproc->dev.parent;
+
+	ret = mutex_lock_interruptible(&rproc->lock);
+	if (ret) {
+		dev_err(dev, "can't lock rproc %s: %d\n", rproc->name, ret);
+		return -EINVAL;
+	}
+
+	if (rproc->state != RPROC_OFFLINE) {
+		dev_err(dev, "can't change firmware while running\n");
+		ret = -EBUSY;
+		goto out;
+	}
+
+	len = strcspn(fw_name, "\n");
+	if (!len) {
+		dev_err(dev, "can't provide empty string for firmware name\n");
+		ret = -EINVAL;
+		goto out;
+	}
+
+	p = kstrndup(fw_name, len, GFP_KERNEL);
+	if (!p) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	kfree(rproc->firmware);
+	rproc->firmware = p;
+
+out:
+	mutex_unlock(&rproc->lock);
+	return ret;
+}
+EXPORT_SYMBOL(rproc_set_firmware);
+
 static int __init remoteproc_init(void)
 {
 	rproc_init_sysfs();
diff -Naur --no-dereference a/drivers/remoteproc/remoteproc_debugfs.c b/drivers/remoteproc/remoteproc_debugfs.c
--- a/drivers/remoteproc/remoteproc_debugfs.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/remoteproc/remoteproc_debugfs.c	2020-06-19 21:00:31.424227716 -0400
@@ -158,13 +158,16 @@
 /* Expose resource table content via debugfs */
 static int rproc_rsc_table_show(struct seq_file *seq, void *p)
 {
-	static const char * const types[] = {"carveout", "devmem", "trace", "vdev"};
+	static const char * const types[] = {"carveout", "devmem", "trace",
+					     "vdev", "preload", "postload"};
 	struct rproc *rproc = seq->private;
 	struct resource_table *table = rproc->table_ptr;
 	struct fw_rsc_carveout *c;
 	struct fw_rsc_devmem *d;
-	struct fw_rsc_trace *t;
+	struct fw_rsc_trace *t1;
+	struct fw_rsc_trace2 *t2;
 	struct fw_rsc_vdev *v;
+	struct fw_rsc_vendor *vr;
 	int i, j;
 
 	if (!table) {
@@ -176,11 +179,13 @@
 		int offset = table->offset[i];
 		struct fw_rsc_hdr *hdr = (void *)table + offset;
 		void *rsc = (void *)hdr + sizeof(*hdr);
+		u16 ver = hdr->st.v;
 
-		switch (hdr->type) {
+		switch (hdr->st.t) {
 		case RSC_CARVEOUT:
 			c = rsc;
-			seq_printf(seq, "Entry %d is of type %s\n", i, types[hdr->type]);
+			seq_printf(seq, "Entry %d is of type %s\n",
+				   i, types[hdr->st.t]);
 			seq_printf(seq, "  Device Address 0x%x\n", c->da);
 			seq_printf(seq, "  Physical Address 0x%x\n", c->pa);
 			seq_printf(seq, "  Length 0x%x Bytes\n", c->len);
@@ -190,7 +195,8 @@
 			break;
 		case RSC_DEVMEM:
 			d = rsc;
-			seq_printf(seq, "Entry %d is of type %s\n", i, types[hdr->type]);
+			seq_printf(seq, "Entry %d is of type %s\n",
+				   i, types[hdr->st.t]);
 			seq_printf(seq, "  Device Address 0x%x\n", d->da);
 			seq_printf(seq, "  Physical Address 0x%x\n", d->pa);
 			seq_printf(seq, "  Length 0x%x Bytes\n", d->len);
@@ -199,17 +205,37 @@
 			seq_printf(seq, "  Name %s\n\n", d->name);
 			break;
 		case RSC_TRACE:
-			t = rsc;
-			seq_printf(seq, "Entry %d is of type %s\n", i, types[hdr->type]);
-			seq_printf(seq, "  Device Address 0x%x\n", t->da);
-			seq_printf(seq, "  Length 0x%x Bytes\n", t->len);
-			seq_printf(seq, "  Reserved (should be zero) [%d]\n", t->reserved);
-			seq_printf(seq, "  Name %s\n\n", t->name);
+			if (ver == 0) {
+				t1 = rsc;
+				seq_printf(seq, "Entry %d is version %d of type %s\n",
+					   i, ver, types[hdr->st.t]);
+				seq_printf(seq, "  Device Address 0x%x\n",
+					   t1->da);
+				seq_printf(seq, "  Length 0x%x Bytes\n",
+					   t1->len);
+				seq_printf(seq, "  Reserved (should be zero) [%d]\n",
+					   t1->reserved);
+				seq_printf(seq, "  Name %s\n\n", t1->name);
+			} else if (ver == 1) {
+				t2 = rsc;
+				seq_printf(seq, "Entry %d is version %d of type %s\n",
+					   i, ver, types[hdr->st.t]);
+				seq_printf(seq, "  Device Address 0x%llx\n",
+					   t2->da);
+				seq_printf(seq, "  Length 0x%x Bytes\n",
+					   t2->len);
+				seq_printf(seq, "  Reserved (should be zero) [%d]\n",
+					   t2->reserved);
+				seq_printf(seq, "  Name %s\n\n", t2->name);
+			} else {
+				seq_printf(seq, "Entry %d is an unsupported version %d of type %s\n",
+					   i, ver, types[hdr->st.t]);
+			}
 			break;
 		case RSC_VDEV:
 			v = rsc;
-			seq_printf(seq, "Entry %d is of type %s\n", i, types[hdr->type]);
-
+			seq_printf(seq, "Entry %d is of type %s\n",
+				   i, types[hdr->st.t]);
 			seq_printf(seq, "  ID %d\n", v->id);
 			seq_printf(seq, "  Notify ID %d\n", v->notifyid);
 			seq_printf(seq, "  Device features 0x%x\n", v->dfeatures);
@@ -230,9 +256,19 @@
 					   v->vring[j].pa);
 			}
 			break;
+		case RSC_PRELOAD_VENDOR:
+		case RSC_POSTLOAD_VENDOR:
+			vr = rsc;
+			seq_printf(seq, "Entry %d is of type vendor-%s\n",
+				   i, types[hdr->type]);
+			seq_printf(seq, "  Vendor sub-type %d version %d\n",
+				   vr->u.st.st_type, vr->u.st.st_ver);
+			seq_printf(seq, "  Vendor resource size %d\n",
+				   vr->size);
+			break;
 		default:
 			seq_printf(seq, "Unknown resource type found: %d [hdr: %pK]\n",
-				   hdr->type, hdr);
+				   hdr->st.t, hdr);
 			break;
 		}
 	}
@@ -260,6 +296,7 @@
 
 	list_for_each_entry(carveout, &rproc->carveouts, node) {
 		seq_puts(seq, "Carveout memory entry:\n");
+		seq_printf(seq, "\tName: %s\n", carveout->name);
 		seq_printf(seq, "\tVirtual address: %pK\n", carveout->va);
 		seq_printf(seq, "\tDMA address: %pad\n", &carveout->dma);
 		seq_printf(seq, "\tDevice address: 0x%x\n", carveout->da);
diff -Naur --no-dereference a/drivers/remoteproc/remoteproc_elf64_loader.c b/drivers/remoteproc/remoteproc_elf64_loader.c
--- a/drivers/remoteproc/remoteproc_elf64_loader.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/remoteproc/remoteproc_elf64_loader.c	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,335 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Remote Processor Framework 64-bit Elf loader
+ * Code based on current remoteproc_elf_loader.c
+ *
+ * Copyright (C) 2018-2019 Texas Instruments, Inc.
+ *
+ * Suman Anna <s-anna@ti.com>
+ */
+
+#define pr_fmt(fmt)    "%s: " fmt, __func__
+
+#include <linux/elf.h>
+#include <linux/kernel.h>
+#include <linux/remoteproc.h>
+
+#include "remoteproc_internal.h"
+
+/**
+ * rproc_elf64_sanity_check() - Sanity Check ELF firmware image
+ * @rproc: the remote processor handle
+ * @fw: the ELF firmware image
+ *
+ * Make sure this fw image is sane.
+ */
+int rproc_elf64_sanity_check(struct rproc *rproc, const struct firmware *fw)
+{
+	const char *name = rproc->firmware;
+	struct device *dev = &rproc->dev;
+	struct elf64_hdr *ehdr;
+	char class;
+
+	if (!fw) {
+		dev_err(dev, "failed to load %s\n", name);
+		return -EINVAL;
+	}
+
+	if (fw->size < sizeof(struct elf64_hdr)) {
+		dev_err(dev, "Image is too small\n");
+		return -EINVAL;
+	}
+
+	ehdr = (struct elf64_hdr *)fw->data;
+
+	/* We only support ELF64 at this point */
+	class = ehdr->e_ident[EI_CLASS];
+	if (class != ELFCLASS64) {
+		dev_err(dev, "Unsupported class: %d\n", class);
+		return -EINVAL;
+	}
+
+	/* We assume the firmware has the same endianness as the host */
+# ifdef __LITTLE_ENDIAN
+	if (ehdr->e_ident[EI_DATA] != ELFDATA2LSB) {
+# else /* BIG ENDIAN */
+	if (ehdr->e_ident[EI_DATA] != ELFDATA2MSB) {
+# endif
+		dev_err(dev, "Unsupported firmware endianness\n");
+		return -EINVAL;
+	}
+
+	if (fw->size < ehdr->e_shoff + sizeof(struct elf64_shdr)) {
+		dev_err(dev, "Image is too small\n");
+		return -EINVAL;
+	}
+
+	if (memcmp(ehdr->e_ident, ELFMAG, SELFMAG)) {
+		dev_err(dev, "Image is corrupted (bad magic)\n");
+		return -EINVAL;
+	}
+
+	if (ehdr->e_phnum == 0) {
+		dev_err(dev, "No loadable segments\n");
+		return -EINVAL;
+	}
+
+	if (ehdr->e_phoff > fw->size) {
+		dev_err(dev, "Firmware size is too small\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(rproc_elf64_sanity_check);
+
+/**
+ * rproc_elf64_get_boot_addr() - Get rproc's boot address.
+ * @rproc: the remote processor handle
+ * @fw: the ELF firmware image
+ *
+ * This function returns the entry point address of the ELF
+ * image.
+ *
+ * Note that the boot address is not a configurable property of all remote
+ * processors. Some will always boot at a specific hard-coded address.
+ *
+ * FIXME: The entry-point with 64-bit processors will be a 64-bit address,
+ * but return a 32-bit address for now to overload and reuse the current
+ * rproc_ops. This will scale as long as the higher 32-bit addresses are
+ * zero.
+ */
+u32 rproc_elf64_get_boot_addr(struct rproc *rproc, const struct firmware *fw)
+{
+	struct elf64_hdr *ehdr = (struct elf64_hdr *)fw->data;
+
+	WARN_ON(upper_32_bits(ehdr->e_entry) != 0);
+
+	return lower_32_bits(ehdr->e_entry);
+}
+EXPORT_SYMBOL(rproc_elf64_get_boot_addr);
+
+/**
+ * rproc_elf64_load_segments() - load firmware segments to memory
+ * @rproc: remote processor which will be booted using these fw segments
+ * @fw: the ELF firmware image
+ *
+ * This function loads the firmware segments to memory, where the remote
+ * processor expects them.
+ *
+ * Some remote processors will expect their code and data to be placed
+ * in specific device addresses, and can't have them dynamically assigned.
+ *
+ * We currently support only those kind of remote processors, and expect
+ * the program header's paddr member to contain those addresses. We then
+ * request the remoteproc core to perform a lookup either in its list of
+ * allocated (and mapped) physically contiguous "carveout" memory regions
+ * and/or registered pre-fixed physically contiguous reserved memory
+ * regions, and "translate" device address to kernel addresses, so we can
+ * copy the segments where they are expected.
+ */
+int rproc_elf64_load_segments(struct rproc *rproc, const struct firmware *fw)
+{
+	struct device *dev = &rproc->dev;
+	struct elf64_hdr *ehdr;
+	struct elf64_phdr *phdr;
+	int i, ret = 0;
+	const u8 *elf_data = fw->data;
+
+	ehdr = (struct elf64_hdr *)elf_data;
+	phdr = (struct elf64_phdr *)(elf_data + ehdr->e_phoff);
+
+	/* go through the available ELF segments */
+	for (i = 0; i < ehdr->e_phnum; i++, phdr++) {
+		u32 da = phdr->p_paddr;
+		u32 memsz = phdr->p_memsz;
+		u32 filesz = phdr->p_filesz;
+		u32 offset = phdr->p_offset;
+		void *ptr;
+
+		if (phdr->p_type != PT_LOAD)
+			continue;
+
+		dev_dbg(dev, "phdr: type %d da 0x%x memsz 0x%x filesz 0x%x\n",
+			phdr->p_type, da, memsz, filesz);
+
+		if (filesz > memsz) {
+			dev_err(dev, "bad phdr filesz 0x%x memsz 0x%x\n",
+				filesz, memsz);
+			ret = -EINVAL;
+			break;
+		}
+
+		if (offset + filesz > fw->size) {
+			dev_err(dev, "truncated fw: need 0x%x avail 0x%zx\n",
+				offset + filesz, fw->size);
+			ret = -EINVAL;
+			break;
+		}
+
+		/*
+		 * grab the kernel address for this device address. Use
+		 * RPROC_FLAGS_NONE as current rproc_da_to_va() does not
+		 * scale for 64-bit flags, and there are no existing use-cases
+		 * requiring this
+		 */
+		ptr = rproc_da_to_va(rproc, da, memsz, RPROC_FLAGS_NONE);
+		if (!ptr) {
+			dev_err(dev, "bad phdr da 0x%x mem 0x%x\n", da, memsz);
+			ret = -EINVAL;
+			break;
+		}
+
+		/* put the segment where the remote processor expects it */
+		if (phdr->p_filesz)
+			memcpy(ptr, elf_data + phdr->p_offset, filesz);
+
+		/*
+		 * Zero out remaining memory for this segment.
+		 *
+		 * This isn't strictly required since dma_alloc_coherent already
+		 * did this for us. albeit harmless, we may consider removing
+		 * this.
+		 */
+		if (memsz > filesz)
+			memset(ptr + filesz, 0, memsz - filesz);
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL(rproc_elf64_load_segments);
+
+static struct elf64_shdr *
+find_table(struct device *dev, struct elf64_hdr *ehdr, size_t fw_size)
+{
+	struct elf64_shdr *shdr;
+	int i;
+	const char *name_table;
+	struct resource_table *table = NULL;
+	const u8 *elf_data = (void *)ehdr;
+
+	/* look for the resource table and handle it */
+	shdr = (struct elf64_shdr *)(elf_data + ehdr->e_shoff);
+	name_table = elf_data + shdr[ehdr->e_shstrndx].sh_offset;
+
+	for (i = 0; i < ehdr->e_shnum; i++, shdr++) {
+		u32 size = shdr->sh_size;
+		u32 offset = shdr->sh_offset;
+
+		if (strcmp(name_table + shdr->sh_name, ".resource_table"))
+			continue;
+
+		table = (struct resource_table *)(elf_data + offset);
+
+		/* make sure we have the entire table */
+		if (offset + size > fw_size || offset + size < size) {
+			dev_err(dev, "resource table truncated\n");
+			return NULL;
+		}
+
+		/* make sure table has at least the header */
+		if (sizeof(struct resource_table) > size) {
+			dev_err(dev, "header-less resource table\n");
+			return NULL;
+		}
+
+		/* we don't support any version beyond the first */
+		if (table->ver != 1) {
+			dev_err(dev, "unsupported fw ver: %d\n", table->ver);
+			return NULL;
+		}
+
+		/* make sure reserved bytes are zeroes */
+		if (table->reserved[0] || table->reserved[1]) {
+			dev_err(dev, "non zero reserved bytes\n");
+			return NULL;
+		}
+
+		/* make sure the offsets array isn't truncated */
+		if (table->num * sizeof(table->offset[0]) +
+				sizeof(struct resource_table) > size) {
+			dev_err(dev, "resource table incomplete\n");
+			return NULL;
+		}
+
+		return shdr;
+	}
+
+	return NULL;
+}
+
+/**
+ * rproc_elf64_load_rsc_table() - load the resource table
+ * @rproc: the rproc handle
+ * @fw: the ELF firmware image
+ *
+ * This function finds the resource table inside the remote processor's
+ * firmware, load it into the @cached_table and update @table_ptr.
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int rproc_elf64_load_rsc_table(struct rproc *rproc, const struct firmware *fw)
+{
+	struct elf64_hdr *ehdr;
+	struct elf64_shdr *shdr;
+	struct device *dev = &rproc->dev;
+	struct resource_table *table = NULL;
+	const u8 *elf_data = fw->data;
+	size_t tablesz;
+
+	ehdr = (struct elf64_hdr *)elf_data;
+
+	shdr = find_table(dev, ehdr, fw->size);
+	if (!shdr)
+		return -EINVAL;
+
+	table = (struct resource_table *)(elf_data + shdr->sh_offset);
+	tablesz = shdr->sh_size;
+
+	/*
+	 * Create a copy of the resource table. When a virtio device starts
+	 * and calls vring_new_virtqueue() the address of the allocated vring
+	 * will be stored in the cached_table. Before the device is started,
+	 * cached_table will be copied into device memory.
+	 */
+	rproc->cached_table = kmemdup(table, tablesz, GFP_KERNEL);
+	if (!rproc->cached_table)
+		return -ENOMEM;
+
+	rproc->table_ptr = rproc->cached_table;
+	rproc->table_sz = tablesz;
+
+	return 0;
+}
+EXPORT_SYMBOL(rproc_elf64_load_rsc_table);
+
+/**
+ * rproc_elf64_find_loaded_rsc_table() - find the loaded resource table
+ * @rproc: the rproc handle
+ * @fw: the ELF firmware image
+ *
+ * This function finds the location of the loaded resource table. Don't
+ * call this function if the table wasn't loaded yet - it's a bug if you do.
+ *
+ * Returns the pointer to the resource table if it is found or NULL otherwise.
+ * If the table wasn't loaded yet the result is unspecified.
+ */
+struct resource_table *
+rproc_elf64_find_loaded_rsc_table(struct rproc *rproc,
+				  const struct firmware *fw)
+{
+	struct elf64_hdr *ehdr = (struct elf64_hdr *)fw->data;
+	struct elf64_shdr *shdr;
+
+	shdr = find_table(&rproc->dev, ehdr, fw->size);
+	if (!shdr)
+		return NULL;
+
+	/*
+	 * Use RPROC_FLAGS_NONE as current rproc_da_to_va() does not
+	 * scale for 64-bit flags, and no current use-cases require this
+	 */
+	return rproc_da_to_va(rproc, shdr->sh_addr, shdr->sh_size,
+			      RPROC_FLAGS_NONE);
+}
+EXPORT_SYMBOL(rproc_elf64_find_loaded_rsc_table);
diff -Naur --no-dereference a/drivers/remoteproc/remoteproc_elf_loader.c b/drivers/remoteproc/remoteproc_elf_loader.c
--- a/drivers/remoteproc/remoteproc_elf_loader.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/remoteproc/remoteproc_elf_loader.c	2020-06-19 21:00:31.424227716 -0400
@@ -182,7 +182,8 @@
 		}
 
 		/* grab the kernel address for this device address */
-		ptr = rproc_da_to_va(rproc, da, memsz);
+		ptr = rproc_da_to_va(rproc, da, memsz,
+				     RPROC_FLAGS_ELF_PHDR | phdr->p_flags);
 		if (!ptr) {
 			dev_err(dev, "bad phdr da 0x%x mem 0x%x\n", da, memsz);
 			ret = -EINVAL;
@@ -333,6 +334,7 @@
 	if (!shdr)
 		return NULL;
 
-	return rproc_da_to_va(rproc, shdr->sh_addr, shdr->sh_size);
+	return rproc_da_to_va(rproc, shdr->sh_addr, shdr->sh_size,
+			      RPROC_FLAGS_ELF_SHDR | shdr->sh_flags);
 }
 EXPORT_SYMBOL(rproc_elf_find_loaded_rsc_table);
diff -Naur --no-dereference a/drivers/remoteproc/remoteproc_internal.h b/drivers/remoteproc/remoteproc_internal.h
--- a/drivers/remoteproc/remoteproc_internal.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/remoteproc/remoteproc_internal.h	2020-06-19 21:00:31.424227716 -0400
@@ -51,7 +51,6 @@
 void rproc_free_vring(struct rproc_vring *rvring);
 int rproc_alloc_vring(struct rproc_vdev *rvdev, int i);
 
-void *rproc_da_to_va(struct rproc *rproc, u64 da, int len);
 int rproc_trigger_recovery(struct rproc *rproc);
 
 int rproc_elf_sanity_check(struct rproc *rproc, const struct firmware *fw);
@@ -61,6 +60,14 @@
 struct resource_table *rproc_elf_find_loaded_rsc_table(struct rproc *rproc,
 						       const struct firmware *fw);
 
+int rproc_elf64_sanity_check(struct rproc *rproc, const struct firmware *fw);
+u32 rproc_elf64_get_boot_addr(struct rproc *rproc, const struct firmware *fw);
+int rproc_elf64_load_segments(struct rproc *rproc, const struct firmware *fw);
+int rproc_elf64_load_rsc_table(struct rproc *rproc, const struct firmware *fw);
+struct resource_table *
+rproc_elf64_find_loaded_rsc_table(struct rproc *rproc,
+				  const struct firmware *fw);
+
 static inline
 int rproc_fw_sanity_check(struct rproc *rproc, const struct firmware *fw)
 {
diff -Naur --no-dereference a/drivers/remoteproc/remoteproc_sysfs.c b/drivers/remoteproc/remoteproc_sysfs.c
--- a/drivers/remoteproc/remoteproc_sysfs.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/remoteproc/remoteproc_sysfs.c	2020-06-19 21:00:31.424227716 -0400
@@ -11,6 +11,7 @@
  * GNU General Public License for more details.
  */
 
+#include <linux/module.h>
 #include <linux/remoteproc.h>
 
 #include "remoteproc_internal.h"
@@ -32,38 +33,13 @@
 			      const char *buf, size_t count)
 {
 	struct rproc *rproc = to_rproc(dev);
-	char *p;
-	int err, len = count;
+	int err;
 
-	err = mutex_lock_interruptible(&rproc->lock);
-	if (err) {
-		dev_err(dev, "can't lock rproc %s: %d\n", rproc->name, err);
-		return -EINVAL;
-	}
-
-	if (rproc->state != RPROC_OFFLINE) {
-		dev_err(dev, "can't change firmware while running\n");
-		err = -EBUSY;
-		goto out;
-	}
-
-	len = strcspn(buf, "\n");
-	if (!len) {
-		dev_err(dev, "can't provide a NULL firmware\n");
-		err = -EINVAL;
-		goto out;
-	}
-
-	p = kstrndup(buf, len, GFP_KERNEL);
-	if (!p) {
-		err = -ENOMEM;
-		goto out;
-	}
+	/* restrict sysfs operations if not allowed by remoteproc drivers */
+	if (rproc->deny_sysfs_ops)
+		return -EPERM;
 
-	kfree(rproc->firmware);
-	rproc->firmware = p;
-out:
-	mutex_unlock(&rproc->lock);
+	err = rproc_set_firmware(rproc, buf);
 
 	return err ? err : count;
 }
@@ -101,18 +77,36 @@
 	struct rproc *rproc = to_rproc(dev);
 	int ret = 0;
 
+	/* restrict sysfs operations if not allowed by remoteproc drivers */
+	if (rproc->deny_sysfs_ops)
+		return -EPERM;
+
 	if (sysfs_streq(buf, "start")) {
 		if (rproc->state == RPROC_RUNNING)
 			return -EBUSY;
 
+		/*
+		 * prevent underlying implementation from being removed
+		 * when remoteproc does not support auto-boot
+		 */
+		if (!rproc->auto_boot &&
+		    !try_module_get(dev->parent->driver->owner))
+			return -EINVAL;
+
 		ret = rproc_boot(rproc);
-		if (ret)
+		if (ret) {
 			dev_err(&rproc->dev, "Boot failed: %d\n", ret);
+			if (!rproc->auto_boot)
+				module_put(dev->parent->driver->owner);
+		}
 	} else if (sysfs_streq(buf, "stop")) {
-		if (rproc->state != RPROC_RUNNING)
+		if (rproc->state != RPROC_RUNNING &&
+		    rproc->state != RPROC_SUSPENDED)
 			return -EINVAL;
 
 		rproc_shutdown(rproc);
+		if (!rproc->auto_boot)
+			module_put(dev->parent->driver->owner);
 	} else {
 		dev_err(&rproc->dev, "Unrecognised option: %s\n", buf);
 		ret = -EINVAL;
@@ -121,9 +115,20 @@
 }
 static DEVICE_ATTR_RW(state);
 
+/* Expose the name of the remote processor via sysfs */
+static ssize_t name_show(struct device *dev, struct device_attribute *attr,
+			 char *buf)
+{
+	struct rproc *rproc = to_rproc(dev);
+
+	return sprintf(buf, "%s\n", rproc->name);
+}
+static DEVICE_ATTR_RO(name);
+
 static struct attribute *rproc_attrs[] = {
 	&dev_attr_firmware.attr,
 	&dev_attr_state.attr,
+	&dev_attr_name.attr,
 	NULL
 };
 
diff -Naur --no-dereference a/drivers/remoteproc/st_slim_rproc.c b/drivers/remoteproc/st_slim_rproc.c
--- a/drivers/remoteproc/st_slim_rproc.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/remoteproc/st_slim_rproc.c	2020-06-19 21:00:31.424227716 -0400
@@ -178,7 +178,8 @@
 	return 0;
 }
 
-static void *slim_rproc_da_to_va(struct rproc *rproc, u64 da, int len)
+static void *slim_rproc_da_to_va(struct rproc *rproc, u64 da, int len,
+				 u32 flags)
 {
 	struct st_slim_rproc *slim_rproc = rproc->priv;
 	void *va = NULL;
diff -Naur --no-dereference a/drivers/remoteproc/ti_k3_dsp_remoteproc.c b/drivers/remoteproc/ti_k3_dsp_remoteproc.c
--- a/drivers/remoteproc/ti_k3_dsp_remoteproc.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/remoteproc/ti_k3_dsp_remoteproc.c	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,888 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * TI K3 DSP Remote Processor(s) driver
+ *
+ * Copyright (C) 2018-2019 Texas Instruments Incorporated - http://www.ti.com/
+ *	Suman Anna <s-anna@ti.com>
+ */
+
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/of_reserved_mem.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/remoteproc.h>
+#include <linux/mailbox_client.h>
+#include <linux/omap-mailbox.h>
+#include <linux/reset.h>
+#include <linux/soc/ti/ti_sci_protocol.h>
+
+#include "omap_remoteproc.h"
+#include "remoteproc_internal.h"
+#include "ti_sci_proc.h"
+
+#define KEYSTONE_RPROC_LOCAL_ADDRESS_MASK	(SZ_16M - 1)
+
+/**
+ * struct k3_dsp_rproc_mem - internal memory structure
+ * @cpu_addr: MPU virtual address of the memory region
+ * @bus_addr: Bus address used to access the memory region
+ * @dev_addr: Device address of the memory region from DSP view
+ * @size: Size of the memory region
+ */
+struct k3_dsp_rproc_mem {
+	void __iomem *cpu_addr;
+	phys_addr_t bus_addr;
+	u32 dev_addr;
+	size_t size;
+};
+
+/**
+ * struct k3_dsp_rproc - k3 DSP remote processor driver structure
+ * @dev: cached device pointer
+ * @rproc: remoteproc device handle
+ * @mem: internal memory regions data
+ * @num_mems: number of internal memory regions
+ * @rmem: reserved memory regions data
+ * @num_rmems: number of reserved memory regions
+ * @reset: reset control handle
+ * @tsp: TI-SCI processor control handle
+ * @ti_sci: TI-SCI handle
+ * @ti_sci_id: TI-SCI device identifier
+ * @mbox: mailbox channel handle
+ * @client: mailbox client to request the mailbox channel
+ * @uses_lreset: flag to denote the need for local reset management
+ * @ipc_only: flag to indicate IPC-only mode
+ */
+struct k3_dsp_rproc {
+	struct device *dev;
+	struct rproc *rproc;
+	struct k3_dsp_rproc_mem *mem;
+	int num_mems;
+	struct k3_dsp_rproc_mem *rmem;
+	int num_rmems;
+	struct reset_control *reset;
+	struct ti_sci_proc *tsp;
+	const struct ti_sci_handle *ti_sci;
+	u32 ti_sci_id;
+	struct mbox_chan *mbox;
+	struct mbox_client client;
+	unsigned int uses_lreset : 1;
+	unsigned int ipc_only : 1;
+};
+
+/**
+ * struct k3_dsp_rproc_dev_data - device data for the remote processor
+ * @device_name: device name of the remote processor
+ * @fw_name: firmware name to use
+ */
+struct k3_dsp_rproc_dev_data {
+	const char *device_name;
+	const char *fw_name;
+};
+
+/**
+ * k3_dsp_rproc_mbox_callback() - inbound mailbox message handler
+ * @client: mailbox client pointer used for requesting the mailbox channel
+ * @data: mailbox payload
+ *
+ * This handler is invoked by the OMAP mailbox driver whenever a mailbox
+ * message is received. Usually, the mailbox payload simply contains
+ * the index of the virtqueue that is kicked by the remote processor,
+ * and we let remoteproc core handle it.
+ *
+ * In addition to virtqueue indices, we also have some out-of-band values
+ * that indicate different events. Those values are deliberately very
+ * large so they don't coincide with virtqueue indices.
+ */
+static void k3_dsp_rproc_mbox_callback(struct mbox_client *client, void *data)
+{
+	struct k3_dsp_rproc *kproc = container_of(client, struct k3_dsp_rproc,
+						client);
+	struct device *dev = kproc->rproc->dev.parent;
+	const char *name = kproc->rproc->name;
+	u32 msg = to_omap_mbox_msg(data);
+
+	dev_dbg(dev, "mbox msg: 0x%x\n", msg);
+
+	switch (msg) {
+	case RP_MBOX_CRASH:
+		/*
+		 * remoteproc detected an exception, but error recovery is not
+		 * supported. So, just log this for now
+		 */
+		dev_err(dev, "K3 DSP rproc %s crashed\n", name);
+		break;
+	case RP_MBOX_ECHO_REPLY:
+		dev_info(dev, "received echo reply from %s\n", name);
+		break;
+	default:
+		/* silently handle all other valid messages */
+		if (msg >= RP_MBOX_READY && msg < RP_MBOX_END_MSG)
+			return;
+		if (msg > kproc->rproc->max_notifyid) {
+			dev_dbg(dev, "dropping unknown message 0x%x", msg);
+			return;
+		}
+		/* msg contains the index of the triggered vring */
+		if (rproc_vq_interrupt(kproc->rproc, msg) == IRQ_NONE)
+			dev_dbg(dev, "no message was found in vqid %d\n", msg);
+	}
+}
+
+/*
+ * Kick the remote processor to notify about pending unprocessed messages.
+ * The vqid usage is not used and is inconsequential, as the kick is performed
+ * through a simulated GPIO (a bit in an IPC interrupt-triggering register),
+ * the remote processor is expected to process both its Tx and Rx virtqueues.
+ */
+static void k3_dsp_rproc_kick(struct rproc *rproc, int vqid)
+{
+	struct k3_dsp_rproc *kproc = rproc->priv;
+	struct device *dev = rproc->dev.parent;
+	mbox_msg_t msg = (mbox_msg_t)vqid;
+	int ret;
+
+	/* send the index of the triggered virtqueue in the mailbox payload */
+	ret = mbox_send_message(kproc->mbox, (void *)msg);
+	if (ret < 0)
+		dev_err(dev, "failed to send mailbox message, status = %d\n",
+			ret);
+}
+
+/* Put the DSP processor into reset */
+static int k3_dsp_rproc_reset(struct k3_dsp_rproc *kproc)
+{
+	struct device *dev = kproc->dev;
+	int ret;
+
+	ret = reset_control_assert(kproc->reset);
+	if (ret) {
+		dev_err(dev, "local-reset assert failed, ret = %d\n", ret);
+		return ret;
+	}
+
+	if (kproc->uses_lreset)
+		return ret;
+
+	ret = kproc->ti_sci->ops.dev_ops.put_device(kproc->ti_sci,
+						    kproc->ti_sci_id);
+	if (ret) {
+		dev_err(dev, "module-reset assert failed, ret = %d\n", ret);
+		if (reset_control_deassert(kproc->reset))
+			dev_warn(dev, "local-reset deassert back failed\n");
+	}
+
+	return ret;
+}
+
+/* Release the DSP processor from reset */
+static int k3_dsp_rproc_release(struct k3_dsp_rproc *kproc)
+{
+	struct device *dev = kproc->dev;
+	int ret;
+
+	if (kproc->uses_lreset)
+		goto lreset;
+
+	ret = kproc->ti_sci->ops.dev_ops.get_device(kproc->ti_sci,
+						   kproc->ti_sci_id);
+	if (ret) {
+		dev_err(dev, "module-reset deassert failed, ret = %d\n", ret);
+		return ret;
+	}
+
+lreset:
+	ret = reset_control_deassert(kproc->reset);
+	if (ret) {
+		dev_err(dev, "local-reset deassert failed, ret = %d\n", ret);
+		if (kproc->ti_sci->ops.dev_ops.put_device(kproc->ti_sci,
+							  kproc->ti_sci_id))
+			dev_warn(dev, "module-reset assert back failed\n");
+	}
+
+	return ret;
+}
+
+/*
+ * The C66x DSP cores have a local reset that affects only the CPU, and a
+ * generic module reset that powers on the device and allows the DSP internal
+ * memories to be accessed while the local reset is asserted. This function is
+ * used to release the global reset on C66x DSPs to allow loading into the DSP
+ * internal RAMs. The .prepare() ops is invoked by remoteproc core before any
+ * firmware loading, and is followed by the .start() ops after loading to
+ * actually let the C66x DSP cores run. The local reset on C71x cores is a
+ * no-op and the global reset cannot be released on C71x cores until after
+ * the firmware images are loaded, so this function does nothing for C71x cores.
+ */
+static int k3_dsp_rproc_prepare(struct rproc *rproc)
+{
+	struct k3_dsp_rproc *kproc = rproc->priv;
+	struct device *dev = kproc->dev;
+	int ret;
+
+	/* IPC-only mode does not require the core to be released from reset */
+	if (kproc->ipc_only)
+		return 0;
+
+	/* local reset is no-op on C71x processors */
+	if (!kproc->uses_lreset)
+		return 0;
+
+	ret = kproc->ti_sci->ops.dev_ops.get_device(kproc->ti_sci,
+						    kproc->ti_sci_id);
+	if (ret)
+		dev_err(dev, "module-reset deassert failed, cannot enable internal RAM loading, ret = %d\n",
+			ret);
+
+	return ret;
+}
+
+/*
+ * This function implements the .unprepare() ops and performs the complimentary
+ * operations to that of the .prepare() ops. The function is used to assert the
+ * global reset on applicable C66x cores. This completes the second portion of
+ * powering down the C66x DSP cores. The cores themselves are only halted in the
+ * .stop() callback through the local reset, and the .unprepare() ops is invoked
+ * by the remoteproc core after the remoteproc is stopped to balance the global
+ * reset.
+ */
+static int k3_dsp_rproc_unprepare(struct rproc *rproc)
+{
+	struct k3_dsp_rproc *kproc = rproc->priv;
+	struct device *dev = kproc->dev;
+	int ret;
+
+	/* do not put back the cores into reset in IPC-only mode */
+	if (kproc->ipc_only)
+		return 0;
+
+	/* local reset is no-op on C71x processors */
+	if (!kproc->uses_lreset)
+		return 0;
+
+	ret = kproc->ti_sci->ops.dev_ops.put_device(kproc->ti_sci,
+						    kproc->ti_sci_id);
+	if (ret)
+		dev_err(dev, "module-reset assert failed, ret = %d\n", ret);
+
+	return ret;
+}
+
+/*
+ * Power up the DSP remote processor.
+ *
+ * This function will be invoked only after the firmware for this rproc
+ * was loaded, parsed successfully, and all of its resource requirements
+ * were met.
+ */
+static int k3_dsp_rproc_start(struct rproc *rproc)
+{
+	struct k3_dsp_rproc *kproc = rproc->priv;
+	struct mbox_client *client = &kproc->client;
+	struct device *dev = kproc->dev;
+	u32 boot_addr;
+	int ret;
+
+	client->dev = dev;
+	client->tx_done = NULL;
+	client->rx_callback = k3_dsp_rproc_mbox_callback;
+	client->tx_block = false;
+	client->knows_txdone = false;
+
+	kproc->mbox = mbox_request_channel(client, 0);
+	if (IS_ERR(kproc->mbox)) {
+		ret = -EBUSY;
+		dev_err(dev, "mbox_request_channel failed: %ld\n",
+			PTR_ERR(kproc->mbox));
+		return ret;
+	}
+
+	/*
+	 * Ping the remote processor, this is only for sanity-sake for now;
+	 * there is no functional effect whatsoever.
+	 *
+	 * Note that the reply will _not_ arrive immediately: this message
+	 * will wait in the mailbox fifo until the remote processor is booted.
+	 */
+	ret = mbox_send_message(kproc->mbox, (void *)RP_MBOX_ECHO_REQUEST);
+	if (ret < 0) {
+		dev_err(dev, "mbox_send_message failed: %d\n", ret);
+		goto put_mbox;
+	}
+
+	/*
+	 * no need to issue TI-SCI commands to configure and boot the DSP cores
+	 * in IPC-only mode.
+	 */
+	if (kproc->ipc_only) {
+		dev_err(dev, "DSP initialized in IPC-only mode\n");
+		return 0;
+	}
+
+	boot_addr = rproc->bootaddr;
+	if (boot_addr & (SZ_1K - 1)) {
+		dev_err(dev, "invalid boot address 0x%x, must be aligned on a 1KB boundary\n",
+			boot_addr);
+		ret = -EINVAL;
+		goto put_mbox;
+	}
+
+	dev_err(dev, "booting DSP core using boot addr = 0x%x\n", boot_addr);
+	ret = ti_sci_proc_set_config(kproc->tsp, boot_addr, 0, 0);
+	if (ret)
+		goto put_mbox;
+
+	ret = k3_dsp_rproc_release(kproc);
+	if (ret)
+		goto put_mbox;
+
+	return 0;
+
+put_mbox:
+	mbox_free_channel(kproc->mbox);
+	return ret;
+}
+
+/*
+ * Stop the DSP remote processor.
+ *
+ * This function puts the DSP processor into reset, and finishes processing
+ * of any pending messages.
+ */
+static int k3_dsp_rproc_stop(struct rproc *rproc)
+{
+	struct k3_dsp_rproc *kproc = rproc->priv;
+
+	mbox_free_channel(kproc->mbox);
+
+	/*
+	 * no need to issue TI-SCI commands to stop the DSP core
+	 * in IPC-only mode.
+	 */
+	if (kproc->ipc_only) {
+		dev_err(kproc->dev, "DSP deinitialized in IPC-only mode\n");
+		return 0;
+	}
+
+	k3_dsp_rproc_reset(kproc);
+
+	return 0;
+}
+
+/*
+ * Custom function to translate a DSP device address (internal RAMs only) to a
+ * kernel virtual address.  The DSPs can access their RAMs at either an internal
+ * address visible only from a DSP, or at the SoC-level bus address. Both these
+ * addresses need to be looked through for translation. The translated addresses
+ * can be used either by the remoteproc core for loading (when using kernel
+ * remoteproc loader), or by any rpmsg bus drivers.
+ */
+static void *k3_dsp_rproc_da_to_va(struct rproc *rproc, u64 da, int len,
+				   u32 flags)
+{
+	struct k3_dsp_rproc *kproc = rproc->priv;
+	void __iomem *va = NULL;
+	phys_addr_t bus_addr;
+	u32 dev_addr, offset;
+	size_t size;
+	int i;
+
+	if (len <= 0)
+		return NULL;
+
+	for (i = 0; i < kproc->num_mems; i++) {
+		bus_addr = kproc->mem[i].bus_addr;
+		dev_addr = kproc->mem[i].dev_addr;
+		size = kproc->mem[i].size;
+
+		if (da < KEYSTONE_RPROC_LOCAL_ADDRESS_MASK) {
+			/* handle DSP-view addresses */
+			if (da >= dev_addr &&
+			    ((da + len) <= (dev_addr + size))) {
+				offset = da - dev_addr;
+				va = kproc->mem[i].cpu_addr + offset;
+				return (__force void *)va;
+			}
+		} else {
+			/* handle SoC-view addresses */
+			if (da >= bus_addr &&
+			    (da + len) <= (bus_addr + size)) {
+				offset = da - bus_addr;
+				va = kproc->mem[i].cpu_addr + offset;
+				return (__force void *)va;
+			}
+		}
+	}
+
+	/* handle static DDR reserved memory regions */
+	for (i = 0; i < kproc->num_rmems; i++) {
+		dev_addr = kproc->rmem[i].dev_addr;
+		size = kproc->rmem[i].size;
+
+		if (da >= dev_addr && ((da + len) <= (dev_addr + size))) {
+			offset = da - dev_addr;
+			va = kproc->rmem[i].cpu_addr + offset;
+			return (__force void *)va;
+		}
+	}
+
+	return NULL;
+}
+
+static const struct rproc_ops k3_dsp_rproc_ops = {
+	.prepare	= k3_dsp_rproc_prepare,
+	.unprepare	= k3_dsp_rproc_unprepare,
+	.start		= k3_dsp_rproc_start,
+	.stop		= k3_dsp_rproc_stop,
+	.kick		= k3_dsp_rproc_kick,
+	.da_to_va	= k3_dsp_rproc_da_to_va,
+};
+
+static const char *k3_dsp_rproc_get_firmware(struct device *dev)
+{
+	const struct k3_dsp_rproc_dev_data *data =
+						of_device_get_match_data(dev);
+
+	if (!data) {
+		dev_err(dev, "data is NULL, %s\n", dev_name(dev));
+		return ERR_PTR(-ENODEV);
+	}
+
+	for (; data && data->device_name; data++) {
+		if (!strcmp(dev_name(dev), data->device_name))
+			return data->fw_name;
+	}
+
+	dev_err(dev, "No matching DSP device found, %s\n", dev_name(dev));
+	return ERR_PTR(-ENODEV);
+}
+
+static int k3_dsp_rproc_of_get_memories(struct platform_device *pdev,
+					struct k3_dsp_rproc *kproc)
+{
+	static const char * const mem_names[] = {"l2sram", "l1pram", "l1dram"};
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct resource *res;
+	int num_mems = 0;
+	int i;
+
+	num_mems = ARRAY_SIZE(mem_names);
+	kproc->mem = devm_kcalloc(kproc->dev, num_mems,
+				  sizeof(*kproc->mem), GFP_KERNEL);
+	if (!kproc->mem)
+		return -ENOMEM;
+
+	for (i = 0; i < num_mems; i++) {
+		/* C71x cores only have a L1P Cache, there are no L1P SRAMs */
+		if (of_device_is_compatible(np, "ti,j721e-c71-dsp") &&
+		    !strcmp(mem_names[i], "l1pram"))
+			continue;
+
+		res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+						   mem_names[i]);
+		if (!res) {
+			dev_err(dev, "found no memory resource for %s\n",
+				mem_names[i]);
+			return -EINVAL;
+		}
+		if (!devm_request_mem_region(dev, res->start,
+					     resource_size(res),
+					     dev_name(dev))) {
+			dev_err(dev, "could not request %s region for resource\n",
+				mem_names[i]);
+			return -EBUSY;
+		}
+
+		kproc->mem[i].cpu_addr = devm_ioremap_wc(dev, res->start,
+							 resource_size(res));
+		if (IS_ERR(kproc->mem[i].cpu_addr)) {
+			dev_err(dev, "failed to map %s memory\n", mem_names[i]);
+			return PTR_ERR(kproc->mem[i].cpu_addr);
+		}
+		kproc->mem[i].bus_addr = res->start;
+		kproc->mem[i].dev_addr =
+				res->start & KEYSTONE_RPROC_LOCAL_ADDRESS_MASK;
+		kproc->mem[i].size = resource_size(res);
+
+		dev_dbg(dev, "memory %8s: bus addr %pa size 0x%zx va %pK da 0x%x\n",
+			mem_names[i], &kproc->mem[i].bus_addr,
+			kproc->mem[i].size, kproc->mem[i].cpu_addr,
+			kproc->mem[i].dev_addr);
+
+		/* zero out memories to start in a pristine state */
+		/*
+		 * FIXME: comment out until kernel crash is fixed, possible
+		 * issue with local resets.
+		 * memset((__force void *)kproc->mem[i].cpu_addr, 0,
+		 *      kproc->mem[i].size);
+		 */
+	}
+	kproc->num_mems = num_mems;
+
+	return 0;
+}
+
+static int k3_dsp_reserved_mem_init(struct k3_dsp_rproc *kproc)
+{
+	struct device *dev = kproc->dev;
+	struct device_node *np = dev->of_node;
+	struct device_node *rmem_np;
+	struct reserved_mem *rmem;
+	int num_rmems;
+	int ret, i;
+
+	num_rmems = of_property_count_elems_of_size(np, "memory-region",
+						    sizeof(phandle));
+	if (num_rmems <= 0) {
+		dev_err(dev, "device does not reserved memory regions, ret = %d\n",
+			num_rmems);
+		return -EINVAL;
+	}
+	if (num_rmems < 2) {
+		dev_err(dev, "device needs atleast two memory regions to be defined, num = %d\n",
+			num_rmems);
+		return -EINVAL;
+	}
+
+	/* use reserved memory region 0 for vring DMA allocations */
+	ret = of_reserved_mem_device_init_by_idx(dev, np, 0);
+	if (ret) {
+		dev_err(dev, "device cannot initialize DMA pool, ret = %d\n",
+			ret);
+		return ret;
+	}
+
+	num_rmems--;
+	kproc->rmem = kcalloc(num_rmems, sizeof(*kproc->rmem), GFP_KERNEL);
+	if (!kproc->rmem) {
+		ret = -ENOMEM;
+		goto release_rmem;
+	}
+
+	/* use remaining reserved memory regions for static carveouts */
+	for (i = 0; i < num_rmems; i++) {
+		rmem_np = of_parse_phandle(np, "memory-region", i + 1);
+		if (!rmem_np) {
+			ret = -EINVAL;
+			goto unmap_rmem;
+		}
+
+		rmem = of_reserved_mem_lookup(rmem_np);
+		if (!rmem) {
+			of_node_put(rmem_np);
+			ret = -EINVAL;
+			goto unmap_rmem;
+		}
+		of_node_put(rmem_np);
+
+		kproc->rmem[i].bus_addr = rmem->base;
+		/* 64-bit address regions currently not supported */
+		kproc->rmem[i].dev_addr = (u32)rmem->base;
+		kproc->rmem[i].size = rmem->size;
+		kproc->rmem[i].cpu_addr = ioremap_wc(rmem->base, rmem->size);
+		if (!kproc->rmem[i].cpu_addr) {
+			dev_err(dev, "failed to map reserved memory#%d at %pa of size %pa\n",
+				i + 1, &rmem->base, &rmem->size);
+			ret = -ENOMEM;
+			goto unmap_rmem;
+		}
+
+		dev_dbg(dev, "reserved memory%d: bus addr %pa size 0x%zx va %pK da 0x%x\n",
+			i + 1, &kproc->rmem[i].bus_addr,
+			kproc->rmem[i].size, kproc->rmem[i].cpu_addr,
+			kproc->rmem[i].dev_addr);
+	}
+	kproc->num_rmems = num_rmems;
+
+	return 0;
+
+unmap_rmem:
+	for (i--; i >= 0; i--) {
+		if (kproc->rmem[i].cpu_addr)
+			iounmap(kproc->rmem[i].cpu_addr);
+	}
+	kfree(kproc->rmem);
+release_rmem:
+	of_reserved_mem_device_release(kproc->dev);
+	return ret;
+}
+
+static void k3_dsp_reserved_mem_exit(struct k3_dsp_rproc *kproc)
+{
+	int i;
+
+	for (i = 0; i < kproc->num_rmems; i++)
+		iounmap(kproc->rmem[i].cpu_addr);
+	kfree(kproc->rmem);
+
+	of_reserved_mem_device_release(kproc->dev);
+}
+
+static
+struct ti_sci_proc *k3_dsp_rproc_of_get_tsp(struct device *dev,
+					    const struct ti_sci_handle *sci)
+{
+	struct ti_sci_proc *tsp;
+	u32 temp[2];
+	int ret;
+
+	ret = of_property_read_u32_array(dev->of_node, "ti,sci-proc-ids",
+					 temp, 2);
+	if (ret < 0)
+		return ERR_PTR(ret);
+
+	tsp = kzalloc(sizeof(*tsp), GFP_KERNEL);
+	if (!tsp)
+		return ERR_PTR(-ENOMEM);
+
+	tsp->dev = dev;
+	tsp->sci = sci;
+	tsp->ops = &sci->ops.proc_ops;
+	tsp->proc_id = temp[0];
+	tsp->host_id = temp[1];
+
+	return tsp;
+}
+
+static int k3_dsp_rproc_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct k3_dsp_rproc *kproc;
+	struct rproc *rproc;
+	const char *fw_name;
+	bool r_state = false;
+	bool p_state = false;
+	int ret = 0;
+	int ret1;
+
+	fw_name = k3_dsp_rproc_get_firmware(dev);
+	if (IS_ERR(fw_name))
+		return PTR_ERR(fw_name);
+
+	rproc = rproc_alloc(dev, dev_name(dev), &k3_dsp_rproc_ops, fw_name,
+			    sizeof(*kproc));
+	if (!rproc)
+		return -ENOMEM;
+
+	rproc->has_iommu = false;
+	rproc->recovery_disabled = true;
+	kproc = rproc->priv;
+	kproc->rproc = rproc;
+	kproc->dev = dev;
+	kproc->uses_lreset = 1;
+
+	/* C71x is a 64-bit processor, so customize rproc elf loader ops */
+	if (of_device_is_compatible(np, "ti,j721e-c71-dsp")) {
+		rproc->ops->load = rproc_elf64_load_segments;
+		rproc->ops->sanity_check = rproc_elf64_sanity_check;
+		rproc->ops->parse_fw = rproc_elf64_load_rsc_table;
+		rproc->ops->find_loaded_rsc_table =
+				rproc_elf64_find_loaded_rsc_table;
+		rproc->ops->get_boot_addr = rproc_elf64_get_boot_addr;
+		rproc->ops->load = rproc_elf64_load_segments;
+
+		kproc->uses_lreset = 0;
+	}
+
+	kproc->ti_sci = ti_sci_get_by_phandle(np, "ti,sci");
+	if (IS_ERR(kproc->ti_sci)) {
+		ret = PTR_ERR(kproc->ti_sci);
+		if (ret != -EPROBE_DEFER) {
+			dev_err(dev, "failed to get ti-sci handle, ret = %d\n",
+				ret);
+		}
+		kproc->ti_sci = NULL;
+		goto free_rproc;
+	}
+
+	ret = of_property_read_u32(np, "ti,sci-dev-id", &kproc->ti_sci_id);
+	if (ret) {
+		dev_err(dev, "missing 'ti,sci-dev-id' property\n");
+		goto put_sci;
+	}
+
+	kproc->reset = devm_reset_control_get_exclusive(dev, NULL);
+	if (IS_ERR(kproc->reset)) {
+		ret = PTR_ERR(kproc->reset);
+		dev_err(dev, "failed to get reset, status = %d\n", ret);
+		goto put_sci;
+	}
+
+	kproc->tsp = k3_dsp_rproc_of_get_tsp(dev, kproc->ti_sci);
+	if (IS_ERR(kproc->tsp)) {
+		dev_err(dev, "failed to construct ti-sci proc control, ret = %d\n",
+			ret);
+		ret = PTR_ERR(kproc->tsp);
+		goto put_sci;
+	}
+
+	ret = ti_sci_proc_request(kproc->tsp);
+	if (ret < 0) {
+		dev_err(dev, "ti_sci_proc_request failed, ret = %d\n", ret);
+		goto free_tsp;
+	}
+
+	/* enable clock for accessing DSP internal memories */
+	pm_runtime_enable(dev);
+	ret = pm_runtime_get_sync(dev);
+	if (ret < 0) {
+		dev_err(dev, "failed to enable clock, status = %d\n", ret);
+		pm_runtime_put_noidle(dev);
+		goto disable_rpm;
+	}
+
+	ret = k3_dsp_rproc_of_get_memories(pdev, kproc);
+	if (ret)
+		goto disable_clk;
+
+	ret = k3_dsp_reserved_mem_init(kproc);
+	if (ret) {
+		dev_err(dev, "reserved memory init failed, ret = %d\n", ret);
+		goto disable_clk;
+	}
+
+	ret = kproc->ti_sci->ops.dev_ops.is_on(kproc->ti_sci, kproc->ti_sci_id,
+					       &r_state, &p_state);
+	if (ret) {
+		dev_err(dev, "failed to get initial state, mode cannot be determined, ret = %d\n",
+			ret);
+		goto release_mem;
+	}
+
+	/* configure J721E devices for either remoteproc or IPC-only mode */
+	if (p_state) {
+		dev_err(dev, "configured DSP for IPC-only mode\n");
+		rproc->skip_load = 1;
+		kproc->ipc_only = 1;
+	} else {
+		dev_err(dev, "configured DSP for remoteproc mode\n");
+		/*
+		 * ensure the DSP local reset is asserted to ensure the DSP
+		 * doesn't execute bogus code in .prepare() when the module
+		 * reset is released.
+		 */
+		if (kproc->uses_lreset) {
+			ret = reset_control_status(kproc->reset);
+			if (ret < 0) {
+				dev_err(dev, "failed to get reset status, status = %d\n",
+					ret);
+				goto release_mem;
+			} else if (ret == 0) {
+				dev_warn(dev, "local reset is deasserted for device\n");
+				k3_dsp_rproc_reset(kproc);
+			}
+		}
+	}
+
+	ret = rproc_add(rproc);
+	if (ret) {
+		dev_err(dev, "failed to add register device with remoteproc core, status = %d\n",
+			ret);
+		goto release_mem;
+	}
+
+	platform_set_drvdata(pdev, kproc);
+
+	return 0;
+
+release_mem:
+	k3_dsp_reserved_mem_exit(kproc);
+disable_clk:
+	pm_runtime_put_sync(dev);
+disable_rpm:
+	pm_runtime_disable(dev);
+	ret1 = ti_sci_proc_release(kproc->tsp);
+	if (ret1)
+		dev_err(dev, "failed to release proc, ret = %d\n", ret1);
+free_tsp:
+	kfree(kproc->tsp);
+put_sci:
+	ret1 = ti_sci_put_handle(kproc->ti_sci);
+	if (ret1)
+		dev_err(dev, "failed to put ti_sci handle, ret = %d\n", ret1);
+free_rproc:
+	rproc_free(rproc);
+	return ret;
+}
+
+static int k3_dsp_rproc_remove(struct platform_device *pdev)
+{
+	struct k3_dsp_rproc *kproc = platform_get_drvdata(pdev);
+	struct device *dev = &pdev->dev;
+	int ret;
+
+	rproc_del(kproc->rproc);
+	pm_runtime_put_sync(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+
+	ret = ti_sci_proc_release(kproc->tsp);
+	if (ret)
+		dev_err(dev, "failed to release proc, ret = %d\n", ret);
+
+	kfree(kproc->tsp);
+
+	ret = ti_sci_put_handle(kproc->ti_sci);
+	if (ret)
+		dev_err(dev, "failed to put ti_sci handle, ret = %d\n", ret);
+
+	k3_dsp_reserved_mem_exit(kproc);
+	rproc_free(kproc->rproc);
+
+	return 0;
+}
+
+static const struct k3_dsp_rproc_dev_data j721e_c66_dsp_dev_data[] = {
+	{
+		.device_name	= "4d80800000.dsp",
+		.fw_name	= "j7-c66_0-fw",
+	},
+	{
+		.device_name	= "4d81800000.dsp",
+		.fw_name	= "j7-c66_1-fw",
+	},
+	{
+		/* sentinel */
+	},
+};
+
+static const struct k3_dsp_rproc_dev_data j721e_c71_dsp_dev_data[] = {
+	{
+		.device_name	= "64800000.dsp",
+		.fw_name	= "j7-c71_0-fw",
+	},
+	{
+		/* sentinel */
+	},
+};
+
+static const struct of_device_id k3_dsp_of_match[] = {
+	{
+		.compatible = "ti,j721e-c66-dsp",
+		.data = j721e_c66_dsp_dev_data,
+	},
+	{
+		.compatible = "ti,j721e-c71-dsp",
+		.data = j721e_c71_dsp_dev_data,
+	},
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, k3_dsp_of_match);
+
+static struct platform_driver k3_dsp_rproc_driver = {
+	.probe	= k3_dsp_rproc_probe,
+	.remove	= k3_dsp_rproc_remove,
+	.driver	= {
+		.name = "k3-dsp-rproc",
+		.of_match_table = k3_dsp_of_match,
+	},
+};
+
+module_platform_driver(k3_dsp_rproc_driver);
+
+MODULE_AUTHOR("Suman Anna <s-anna@ti.com>");
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("TI K3 DSP Remoteproc driver");
diff -Naur --no-dereference a/drivers/remoteproc/ti_k3_r5_remoteproc.c b/drivers/remoteproc/ti_k3_r5_remoteproc.c
--- a/drivers/remoteproc/ti_k3_r5_remoteproc.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/remoteproc/ti_k3_r5_remoteproc.c	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,1643 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * TI K3 R5F (MCU) Remote Processor driver
+ *
+ * Copyright (C) 2017-2019 Texas Instruments Incorporated - http://www.ti.com/
+ *	Suman Anna <s-anna@ti.com>
+ */
+
+#include <linux/dma-mapping.h>
+#include <linux/err.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/mailbox_client.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/of_address.h>
+#include <linux/of_reserved_mem.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/remoteproc.h>
+#include <linux/omap-mailbox.h>
+#include <linux/reset.h>
+#include <linux/soc/ti/ti_sci_protocol.h>
+
+#include "omap_remoteproc.h"
+#include "remoteproc_internal.h"
+#include "ti_sci_proc.h"
+
+/* This address can either be for ATCM or BTCM with the other at address 0x0 */
+#define K3_R5_TCM_DEV_ADDR	0x41010000
+
+/* R5 TI-SCI Processor Configuration Flags */
+#define PROC_BOOT_CFG_FLAG_R5_DBG_EN			0x00000001
+#define PROC_BOOT_CFG_FLAG_R5_DBG_NIDEN			0x00000002
+#define PROC_BOOT_CFG_FLAG_R5_LOCKSTEP			0x00000100
+#define PROC_BOOT_CFG_FLAG_R5_TEINIT			0x00000200
+#define PROC_BOOT_CFG_FLAG_R5_NMFI_EN			0x00000400
+#define PROC_BOOT_CFG_FLAG_R5_TCM_RSTBASE		0x00000800
+#define PROC_BOOT_CFG_FLAG_R5_BTCM_EN			0x00001000
+#define PROC_BOOT_CFG_FLAG_R5_ATCM_EN			0x00002000
+
+/* R5 TI-SCI Processor Control Flags */
+#define PROC_BOOT_CTRL_FLAG_R5_CORE_HALT		0x00000001
+
+/* R5 TI-SCI Processor Status Flags */
+#define PROC_BOOT_STATUS_FLAG_R5_WFE			0x00000001
+#define PROC_BOOT_STATUS_FLAG_R5_WFI			0x00000002
+#define PROC_BOOT_STATUS_FLAG_R5_CLK_GATED		0x00000004
+#define PROC_BOOT_STATUS_FLAG_R5_LOCKSTEP_PERMITTED	0x00000100
+
+/**
+ * struct k3_r5_mem - internal memory structure
+ * @cpu_addr: MPU virtual address of the memory region
+ * @bus_addr: Bus address used to access the memory region
+ * @dev_addr: Device address from remoteproc view
+ * @size: Size of the memory region
+ */
+struct k3_r5_mem {
+	void __iomem *cpu_addr;
+	phys_addr_t bus_addr;
+	u32 dev_addr;
+	size_t size;
+};
+
+enum cluster_mode {
+	CLUSTER_MODE_SPLIT = 0,
+	CLUSTER_MODE_LOCKSTEP,
+};
+
+/**
+ * struct k3_r5_cluster - K3 R5F Cluster structure
+ * @dev: cached device pointer
+ * @mode: Mode to configure the Cluster - Split or LockStep
+ * @cores: list of R5 cores within the cluster
+ */
+struct k3_r5_cluster {
+	struct device *dev;
+	enum cluster_mode mode;
+	struct list_head cores;
+};
+
+/**
+ * struct k3_r5_core - K3 R5 core structure
+ * @dev: cached device pointer
+ * @rproc: rproc handle representing this core
+ * @mem: internal memory regions data
+ * @sram: on-chip SRAM memory regions data
+ * @num_mems: number of internal memory regions
+ * @num_sram: number of on-chip SRAM memory regions
+ * @reset: reset control handle
+ * @tsp: TI-SCI processor control handle
+ * @ti_sci: TI-SCI handle
+ * @ti_sci_id: TI-SCI device identifier
+ * @atcm_enable: flag to control ATCM enablement
+ * @btcm_enable: flag to control BTCM enablement
+ * @loczrama: flag to dictate which TCM is at device address 0x0
+ */
+struct k3_r5_core {
+	struct list_head elem;
+	struct device *dev;
+	struct rproc *rproc;
+	struct k3_r5_mem *mem;
+	struct k3_r5_mem *sram;
+	int num_mems;
+	int num_sram;
+	struct reset_control *reset;
+	struct ti_sci_proc *tsp;
+	const struct ti_sci_handle *ti_sci;
+	u32 ti_sci_id;
+	u32 atcm_enable;
+	u32 btcm_enable;
+	u32 loczrama;
+};
+
+/**
+ * struct k3_r5_rproc - K3 remote processor state
+ * @dev: cached device pointer
+ * @cluster: cached pointer to parent cluster structure
+ * @mbox: mailbox channel handle
+ * @client: mailbox client to request the mailbox channel
+ * @rproc: rproc handle
+ * @core: cached pointer to r5 core structure being used
+ * @rmem: reserved memory regions data
+ * @num_rmems: number of reserved memory regions
+ * @ipc_only: flag to indicate IPC-only mode
+ */
+struct k3_r5_rproc {
+	struct device *dev;
+	struct k3_r5_cluster *cluster;
+	struct mbox_chan *mbox;
+	struct mbox_client client;
+	struct rproc *rproc;
+	struct k3_r5_core *core;
+	struct k3_r5_mem *rmem;
+	int num_rmems;
+	unsigned int ipc_only : 1;
+};
+
+/**
+ * struct k3_r5_rproc_dev_data - device data for the remote processor
+ * @device_name: device name of the remote processor
+ * @fw_name: firmware name to use
+ */
+struct k3_r5_rproc_dev_data {
+	const char *device_name;
+	const char *fw_name;
+};
+
+/**
+ * k3_r5_rproc_mbox_callback() - inbound mailbox message handler
+ * @client: mailbox client pointer used for requesting the mailbox channel
+ * @data: mailbox payload
+ *
+ * This handler is invoked by the OMAP mailbox driver whenever a mailbox
+ * message is received. Usually, the mailbox payload simply contains
+ * the index of the virtqueue that is kicked by the remote processor,
+ * and we let remoteproc core handle it.
+ *
+ * In addition to virtqueue indices, we also have some out-of-band values
+ * that indicate different events. Those values are deliberately very
+ * large so they don't coincide with virtqueue indices.
+ */
+static void k3_r5_rproc_mbox_callback(struct mbox_client *client, void *data)
+{
+	struct k3_r5_rproc *kproc = container_of(client, struct k3_r5_rproc,
+						client);
+	struct device *dev = kproc->rproc->dev.parent;
+	const char *name = kproc->rproc->name;
+	u32 msg = to_omap_mbox_msg(data);
+
+	dev_dbg(dev, "mbox msg: 0x%x\n", msg);
+
+	switch (msg) {
+	case RP_MBOX_CRASH:
+		/*
+		 * remoteproc detected an exception, but error recovery is not
+		 * supported. So, just log this for now
+		 */
+		dev_err(dev, "K3 R5F rproc %s crashed\n", name);
+		break;
+	case RP_MBOX_ECHO_REPLY:
+		dev_info(dev, "received echo reply from %s\n", name);
+		break;
+	default:
+		/* silently handle all other valid messages */
+		if (msg >= RP_MBOX_READY && msg < RP_MBOX_END_MSG)
+			return;
+		if (msg > kproc->rproc->max_notifyid) {
+			dev_dbg(dev, "dropping unknown message 0x%x", msg);
+			return;
+		}
+		/* msg contains the index of the triggered vring */
+		if (rproc_vq_interrupt(kproc->rproc, msg) == IRQ_NONE)
+			dev_dbg(dev, "no message was found in vqid %d\n", msg);
+	}
+}
+
+/* kick a virtqueue */
+static void k3_r5_rproc_kick(struct rproc *rproc, int vqid)
+{
+	struct k3_r5_rproc *kproc = rproc->priv;
+	struct device *dev = rproc->dev.parent;
+	mbox_msg_t msg = (mbox_msg_t)vqid;
+	int ret;
+
+	/* send the index of the triggered virtqueue in the mailbox payload */
+	ret = mbox_send_message(kproc->mbox, (void *)msg);
+	if (ret < 0)
+		dev_err(dev, "failed to send mailbox message, status = %d\n",
+			ret);
+}
+
+static int k3_r5_split_reset(struct k3_r5_core *core)
+{
+	int ret;
+
+	ret = reset_control_assert(core->reset);
+	if (ret) {
+		dev_err(core->dev, "local-reset assert failed, ret = %d\n",
+			ret);
+		return ret;
+	}
+
+	ret = core->ti_sci->ops.dev_ops.put_device(core->ti_sci,
+						   core->ti_sci_id);
+	if (ret) {
+		dev_err(core->dev, "module-reset assert failed, ret = %d\n",
+			ret);
+		if (reset_control_deassert(core->reset))
+			dev_warn(core->dev, "local-reset deassert back failed\n");
+	}
+
+	return ret;
+}
+
+static int k3_r5_split_release(struct k3_r5_core *core)
+{
+	int ret;
+
+	ret = core->ti_sci->ops.dev_ops.get_device(core->ti_sci,
+						   core->ti_sci_id);
+	if (ret) {
+		dev_err(core->dev, "module-reset deassert failed, ret = %d\n",
+			ret);
+		return ret;
+	}
+
+	ret = reset_control_deassert(core->reset);
+	if (ret) {
+		dev_err(core->dev, "local-reset deassert failed, ret = %d\n",
+			ret);
+		if (core->ti_sci->ops.dev_ops.put_device(core->ti_sci,
+							 core->ti_sci_id))
+			dev_warn(core->dev, "module-reset assert back failed\n");
+	}
+
+	return ret;
+}
+
+static int k3_r5_lockstep_reset(struct k3_r5_cluster *cluster)
+{
+	struct k3_r5_core *core;
+	int ret;
+
+	/* assert local reset on all applicable cores */
+	list_for_each_entry(core, &cluster->cores, elem) {
+		ret = reset_control_assert(core->reset);
+		if (ret) {
+			dev_err(core->dev, "local-reset assert failed, ret = %d\n",
+				ret);
+			core = list_prev_entry(core, elem);
+			goto unroll_local_reset;
+		}
+	}
+
+	/* disable PSC modules on all applicable cores */
+	list_for_each_entry(core, &cluster->cores, elem) {
+		ret = core->ti_sci->ops.dev_ops.put_device(core->ti_sci,
+							   core->ti_sci_id);
+		if (ret) {
+			dev_err(core->dev, "module-reset assert failed, ret = %d\n",
+				ret);
+			goto unroll_module_reset;
+		}
+	}
+
+	return 0;
+
+unroll_module_reset:
+	list_for_each_entry_continue_reverse(core, &cluster->cores, elem) {
+		if (core->ti_sci->ops.dev_ops.put_device(core->ti_sci,
+							 core->ti_sci_id))
+			dev_warn(core->dev, "module-reset assert back failed\n");
+	}
+	core = list_last_entry(&cluster->cores, struct k3_r5_core, elem);
+unroll_local_reset:
+	list_for_each_entry_from_reverse(core, &cluster->cores, elem) {
+		if (reset_control_deassert(core->reset))
+			dev_warn(core->dev, "local-reset deassert back failed\n");
+	}
+
+	return ret;
+}
+
+static int k3_r5_lockstep_release(struct k3_r5_cluster *cluster)
+{
+	struct k3_r5_core *core;
+	int ret;
+
+	/* enable PSC modules on all applicable cores */
+	list_for_each_entry_reverse(core, &cluster->cores, elem) {
+		ret = core->ti_sci->ops.dev_ops.get_device(core->ti_sci,
+							   core->ti_sci_id);
+		if (ret) {
+			dev_err(core->dev, "module-reset deassert failed, ret = %d\n",
+				ret);
+			core = list_next_entry(core, elem);
+			goto unroll_module_reset;
+		}
+	}
+
+	/* deassert local reset on all applicable cores */
+	list_for_each_entry_reverse(core, &cluster->cores, elem) {
+		ret = reset_control_deassert(core->reset);
+		if (ret) {
+			dev_err(core->dev, "module-reset deassert failed, ret = %d\n",
+				ret);
+			goto unroll_local_reset;
+		}
+	}
+
+	return 0;
+
+unroll_local_reset:
+	list_for_each_entry_continue(core, &cluster->cores, elem) {
+		if (reset_control_assert(core->reset))
+			dev_warn(core->dev, "local-reset assert back failed\n");
+	}
+	core = list_first_entry(&cluster->cores, struct k3_r5_core, elem);
+unroll_module_reset:
+	list_for_each_entry_from(core, &cluster->cores, elem) {
+		if (core->ti_sci->ops.dev_ops.put_device(core->ti_sci,
+							 core->ti_sci_id))
+			dev_warn(core->dev, "module-reset assert back failed\n");
+	}
+
+	return ret;
+}
+
+static inline int k3_r5_core_halt(struct k3_r5_core *core)
+{
+	return ti_sci_proc_set_control(core->tsp,
+				       PROC_BOOT_CTRL_FLAG_R5_CORE_HALT, 0);
+}
+
+static inline int k3_r5_core_run(struct k3_r5_core *core)
+{
+	return ti_sci_proc_set_control(core->tsp,
+				       0, PROC_BOOT_CTRL_FLAG_R5_CORE_HALT);
+}
+
+/*
+ * The R5F cores have controls for both a reset and a halt/run. The code
+ * execution from DDR requires the initial boot-strapping code to be run
+ * from the internal TCMs. This function is used to release the resets on
+ * applicable cores to allow loading into the TCMs. The .prepare() ops is
+ * invoked by remoteproc core before any firmware loading, and is followed
+ * by the .start() ops after loading to actually let the R5 cores run.
+ */
+static int k3_r5_rproc_prepare(struct rproc *rproc)
+{
+	struct k3_r5_rproc *kproc = rproc->priv;
+	struct k3_r5_cluster *cluster = kproc->cluster;
+	struct k3_r5_core *core = kproc->core;
+	struct device *dev = kproc->dev;
+	int ret;
+
+	/* IPC-only mode does not require the cores to be released from reset */
+	if (kproc->ipc_only)
+		return 0;
+
+	ret = cluster->mode ? k3_r5_lockstep_release(cluster) :
+			      k3_r5_split_release(core);
+	if (ret)
+		dev_err(dev, "unable to enable cores for TCM loading, ret = %d\n",
+			ret);
+
+	/*
+	 * Zero out both TCMs unconditionally (access from v8 Arm core is not
+	 * affected by ATCM & BTCM enable configuration values) so that ECC
+	 * can be effective on all TCM addresses.
+	 */
+	dev_dbg(dev, "zeroing out ATCM memory\n");
+	memset(core->mem[0].cpu_addr, 0x00, core->mem[0].size);
+
+	dev_dbg(dev, "zeroing out BTCM memory\n");
+	memset(core->mem[1].cpu_addr, 0x00, core->mem[1].size);
+
+	return ret;
+}
+
+/*
+ * This function implements the .unprepare() ops and performs the complimentary
+ * operations to that of the .prepare() ops. The function is used to assert the
+ * resets on all applicable cores for the rproc device (depending on LockStep
+ * or Split mode). This completes the second portion of powering down the R5F
+ * cores. The cores themselves are only halted in the .stop() ops, and the
+ * .unprepare() ops is invoked by the remoteproc core after the remoteproc is
+ * stopped.
+ */
+static int k3_r5_rproc_unprepare(struct rproc *rproc)
+{
+	struct k3_r5_rproc *kproc = rproc->priv;
+	struct k3_r5_cluster *cluster = kproc->cluster;
+	struct k3_r5_core *core = kproc->core;
+	struct device *dev = kproc->dev;
+	int ret;
+
+	/* do not put back the cores into reset in IPC-only mode */
+	if (kproc->ipc_only)
+		return 0;
+
+	ret = cluster->mode ? k3_r5_lockstep_reset(cluster) :
+			      k3_r5_split_reset(core);
+	if (ret)
+		dev_err(dev, "unable to disable cores, ret = %d\n", ret);
+
+	return ret;
+}
+
+/*
+ * The R5F start sequence includes two different operations
+ * 1. Configure the boot vector for R5F core(s)
+ * 2. Unhalt/Run the R5F core(s)
+ *
+ * The sequence is different between LockStep and Split modes. The LockStep
+ * mode requires the boot vector to be configured only for Core0, and then
+ * unhalt both the cores to start the execution - Core1 needs to be unhalted
+ * first followed by Core0. The Split-mode requires that Core0 to be maintained
+ * always in a higher power state that Core1 (implying Core1 needs to be started
+ * always only after Core0 is started).
+ */
+static int k3_r5_rproc_start(struct rproc *rproc)
+{
+	struct k3_r5_rproc *kproc = rproc->priv;
+	struct k3_r5_cluster *cluster = kproc->cluster;
+	struct mbox_client *client = &kproc->client;
+	struct device *dev = kproc->dev;
+	struct k3_r5_core *core;
+	u32 boot_addr;
+	int ret;
+
+	client->dev = dev;
+	client->tx_done = NULL;
+	client->rx_callback = k3_r5_rproc_mbox_callback;
+	client->tx_block = false;
+	client->knows_txdone = false;
+
+	kproc->mbox = mbox_request_channel(client, 0);
+	if (IS_ERR(kproc->mbox)) {
+		ret = -EBUSY;
+		dev_err(dev, "mbox_request_channel failed: %ld\n",
+			PTR_ERR(kproc->mbox));
+		return ret;
+	}
+
+	/*
+	 * Ping the remote processor, this is only for sanity-sake for now;
+	 * there is no functional effect whatsoever.
+	 *
+	 * Note that the reply will _not_ arrive immediately: this message
+	 * will wait in the mailbox fifo until the remote processor is booted.
+	 */
+	ret = mbox_send_message(kproc->mbox, (void *)RP_MBOX_ECHO_REQUEST);
+	if (ret < 0) {
+		dev_err(dev, "mbox_send_message failed: %d\n", ret);
+		goto put_mbox;
+	}
+
+	/*
+	 * no need to issue TI-SCI commands to configure and boot the R5F cores
+	 * in IPC-only mode.
+	 */
+	if (kproc->ipc_only) {
+		dev_err(dev, "R5F core initialized in IPC-only mode\n");
+		return 0;
+	}
+
+	boot_addr = rproc->bootaddr;
+	/* TODO: add boot_addr sanity checking */
+	dev_err(dev, "booting R5F core using boot addr = 0x%x\n", boot_addr);
+
+	/* boot vector need not be programmed for Core1 in LockStep mode */
+	core = kproc->core;
+	ret = ti_sci_proc_set_config(core->tsp, boot_addr, 0, 0);
+	if (ret)
+		goto put_mbox;
+
+	/* unhalt/run all applicable cores */
+	if (cluster->mode) {
+		list_for_each_entry_reverse(core, &cluster->cores, elem) {
+			ret = k3_r5_core_run(core);
+			if (ret)
+				goto unroll_core_run;
+		}
+	} else {
+		ret = k3_r5_core_run(core);
+		if (ret)
+			goto put_mbox;
+	}
+
+	return 0;
+
+unroll_core_run:
+	list_for_each_entry_continue(core, &cluster->cores, elem) {
+		if (k3_r5_core_halt(core))
+			dev_warn(core->dev, "core halt back failed\n");
+	}
+put_mbox:
+	mbox_free_channel(kproc->mbox);
+	return ret;
+}
+
+/*
+ * The R5F stop function includes the following operations
+ * 1. Halt R5F core(s)
+ *
+ * The sequence is different between LockStep and Split modes, and the order
+ * of cores the operations are performed are also in general reverse to that
+ * of the start function. The LockStep mode requires each operation to be
+ * performed first on Core0 followed by Core1. The Split-mode requires that
+ * Core0 to be maintained always in a higher power state that Core1 (implying
+ * Core1 needs to be stopped first before Core0).
+ *
+ * Note that the R5F halt operation in general is not effective when the R5F
+ * core is running, but is needed to make sure the core won't run after
+ * deasserting the reset the subsequent time. The asserting of reset can
+ * be done here, but is preferred to be done in the .unprepare() ops - this
+ * maintains the symmetric behavior between the .start(), .stop(), .prepare()
+ * and .unprepare() ops, and also balances them well between sysfs 'state'
+ * flow and device bind/unbind or module removal.
+ */
+static int k3_r5_rproc_stop(struct rproc *rproc)
+{
+	struct k3_r5_rproc *kproc = rproc->priv;
+	struct k3_r5_cluster *cluster = kproc->cluster;
+	struct k3_r5_core *core = kproc->core;
+	int ret;
+
+	/*
+	 * no need to issue TI-SCI commands to stop the R5F cores
+	 * in IPC-only mode.
+	 */
+	if (kproc->ipc_only) {
+		mbox_free_channel(kproc->mbox);
+		dev_err(kproc->dev, "R5F core deinitialized in IPC-only mode\n");
+		return 0;
+	}
+
+	/* halt all applicable cores */
+	if (cluster->mode) {
+		list_for_each_entry(core, &cluster->cores, elem) {
+			ret = k3_r5_core_halt(core);
+			if (ret) {
+				core = list_prev_entry(core, elem);
+				goto unroll_core_halt;
+			}
+		}
+	} else {
+		ret = k3_r5_core_halt(core);
+		if (ret)
+			goto out;
+	}
+
+	mbox_free_channel(kproc->mbox);
+
+	return 0;
+
+unroll_core_halt:
+	list_for_each_entry_from_reverse(core, &cluster->cores, elem) {
+		if (k3_r5_core_run(core))
+			dev_warn(core->dev, "core run back failed\n");
+	}
+out:
+	return ret;
+}
+
+/*
+ * Internal Memory translation helper
+ *
+ * Custom function implementing the rproc .da_to_va ops to provide address
+ * translation (device address to kernel virtual address) for internal RAMs
+ * present in a DSP or IPU device). The translated addresses can be used
+ * either by the remoteproc core for loading, or by any rpmsg bus drivers.
+ */
+static void *k3_r5_rproc_da_to_va(struct rproc *rproc, u64 da, int len,
+				  u32 flags)
+{
+	struct k3_r5_rproc *kproc = rproc->priv;
+	struct k3_r5_core *core = kproc->core;
+	void __iomem *va = NULL;
+	phys_addr_t bus_addr;
+	u32 dev_addr, offset;
+	size_t size;
+	int i;
+
+	if (len <= 0)
+		return NULL;
+
+	/* handle both R5 and SoC views of ATCM and BTCM */
+	for (i = 0; i < core->num_mems; i++) {
+		bus_addr = core->mem[i].bus_addr;
+		dev_addr = core->mem[i].dev_addr;
+		size = core->mem[i].size;
+
+		/* handle R5-view addresses of TCMs */
+		if (da >= dev_addr && ((da + len) <= (dev_addr + size))) {
+			offset = da - dev_addr;
+			va = core->mem[i].cpu_addr + offset;
+			return (__force void *)va;
+		}
+
+		/* handle SoC-view addresses of TCMs */
+		if (da >= bus_addr && ((da + len) <= (bus_addr + size))) {
+			offset = da - bus_addr;
+			va = core->mem[i].cpu_addr + offset;
+			return (__force void *)va;
+		}
+	}
+
+	/* handle any SRAM regions using SoC-view addresses */
+	for (i = 0; i < core->num_sram; i++) {
+		dev_addr = core->sram[i].dev_addr;
+		size = core->sram[i].size;
+
+		if (da >= dev_addr && ((da + len) <= (dev_addr + size))) {
+			offset = da - dev_addr;
+			va = core->sram[i].cpu_addr + offset;
+			return (__force void *)va;
+		}
+	}
+
+	/* handle static DDR reserved memory regions */
+	for (i = 0; i < kproc->num_rmems; i++) {
+		dev_addr = kproc->rmem[i].dev_addr;
+		size = kproc->rmem[i].size;
+
+		if (da >= dev_addr && ((da + len) <= (dev_addr + size))) {
+			offset = da - dev_addr;
+			va = kproc->rmem[i].cpu_addr + offset;
+			return (__force void *)va;
+		}
+	}
+
+	return NULL;
+}
+
+static const struct rproc_ops k3_r5_rproc_ops = {
+	.prepare	= k3_r5_rproc_prepare,
+	.unprepare	= k3_r5_rproc_unprepare,
+	.start		= k3_r5_rproc_start,
+	.stop		= k3_r5_rproc_stop,
+	.kick		= k3_r5_rproc_kick,
+	.da_to_va	= k3_r5_rproc_da_to_va,
+};
+
+static const char *k3_r5_rproc_get_firmware(struct device *dev)
+{
+	const struct k3_r5_rproc_dev_data *data =
+				of_device_get_match_data(dev->parent);
+
+	if (!data) {
+		dev_err(dev, "data is NULL, %s\n", dev_name(dev));
+		return ERR_PTR(-ENODEV);
+	}
+
+	for (; data && data->device_name; data++) {
+		if (!strcmp(dev_name(dev), data->device_name))
+			return data->fw_name;
+	}
+
+	return ERR_PTR(-ENODEV);
+}
+
+static int k3_r5_rproc_configure(struct k3_r5_rproc *kproc)
+{
+	struct k3_r5_cluster *cluster = kproc->cluster;
+	struct device *dev = kproc->dev;
+	struct k3_r5_core *core0, *core, *temp;
+	u32 ctrl = 0, cfg = 0, stat = 0;
+	u32 set_cfg = 0, clr_cfg = 0;
+	u64 boot_vec = 0;
+	bool lockstep_en;
+	int ret;
+
+	core0 = list_first_entry(&cluster->cores, struct k3_r5_core, elem);
+	core = cluster->mode ? core0 : kproc->core;
+
+	ret = ti_sci_proc_get_status(core->tsp, &boot_vec, &cfg, &ctrl,
+				     &stat);
+	if (ret < 0)
+		return ret;
+
+	dev_dbg(dev, "boot_vector = 0x%llx, cfg = 0x%x ctrl = 0x%x stat = 0x%x\n",
+		boot_vec, cfg, ctrl, stat);
+
+	lockstep_en = !!(stat & PROC_BOOT_STATUS_FLAG_R5_LOCKSTEP_PERMITTED);
+	if (!lockstep_en && cluster->mode) {
+		dev_err(cluster->dev, "lockstep mode not permitted, force configuring for split-mode\n");
+		cluster->mode = 0;
+	}
+
+	/* always enable ARM mode and set boot vector to 0 */
+	boot_vec = 0x0;
+	if (core == core0) {
+		clr_cfg = PROC_BOOT_CFG_FLAG_R5_TEINIT;
+		if (lockstep_en)
+			clr_cfg |= PROC_BOOT_CFG_FLAG_R5_LOCKSTEP;
+	}
+
+	if (core->atcm_enable)
+		set_cfg |= PROC_BOOT_CFG_FLAG_R5_ATCM_EN;
+	else
+		clr_cfg |= PROC_BOOT_CFG_FLAG_R5_ATCM_EN;
+
+	if (core->btcm_enable)
+		set_cfg |= PROC_BOOT_CFG_FLAG_R5_BTCM_EN;
+	else
+		clr_cfg |= PROC_BOOT_CFG_FLAG_R5_BTCM_EN;
+
+	if (core->loczrama)
+		set_cfg |= PROC_BOOT_CFG_FLAG_R5_TCM_RSTBASE;
+	else
+		clr_cfg |= PROC_BOOT_CFG_FLAG_R5_TCM_RSTBASE;
+
+	if (cluster->mode) {
+		/*
+		 * work around system firmware limitations to make sure both
+		 * cores are programmed symmetrically in LockStep. LockStep
+		 * and TEINIT config is only allowed with Core0.
+		 */
+		list_for_each_entry(temp, &cluster->cores, elem) {
+			ret = k3_r5_core_halt(core);
+			if (ret)
+				goto out;
+
+			if (temp != core) {
+				clr_cfg &= ~PROC_BOOT_CFG_FLAG_R5_LOCKSTEP;
+				clr_cfg &= ~PROC_BOOT_CFG_FLAG_R5_TEINIT;
+			}
+			ret = ti_sci_proc_set_config(temp->tsp, boot_vec,
+						     set_cfg, clr_cfg);
+			if (ret)
+				goto out;
+		}
+
+		set_cfg = PROC_BOOT_CFG_FLAG_R5_LOCKSTEP;
+		clr_cfg = 0;
+		ret = ti_sci_proc_set_config(core->tsp, boot_vec,
+					     set_cfg, clr_cfg);
+	} else {
+		ret = k3_r5_core_halt(core);
+		if (ret)
+			goto out;
+
+		ret = ti_sci_proc_set_config(core->tsp, boot_vec,
+					     set_cfg, clr_cfg);
+	}
+
+out:
+	return ret;
+}
+
+static int k3_r5_reserved_mem_init(struct k3_r5_rproc *kproc)
+{
+	struct device *dev = kproc->dev;
+	struct device_node *np = dev->of_node;
+	struct device_node *rmem_np;
+	struct reserved_mem *rmem;
+	int num_rmems;
+	int ret, i;
+
+	num_rmems = of_property_count_elems_of_size(np, "memory-region",
+						    sizeof(phandle));
+	if (num_rmems <= 0) {
+		dev_err(dev, "device does not have reserved memory regions, ret = %d\n",
+			num_rmems);
+		return -EINVAL;
+	}
+	if (num_rmems < 2) {
+		dev_err(dev, "device needs atleast two memory regions to be defined, num = %d\n",
+			num_rmems);
+		return -EINVAL;
+	}
+
+	/* use reserved memory region 0 for vring DMA allocations */
+	ret = of_reserved_mem_device_init_by_idx(dev, np, 0);
+	if (ret) {
+		dev_err(dev, "device cannot initialize DMA pool, ret = %d\n",
+			ret);
+		return ret;
+	}
+
+	num_rmems--;
+	kproc->rmem = kcalloc(num_rmems, sizeof(*kproc->rmem), GFP_KERNEL);
+	if (!kproc->rmem) {
+		ret = -ENOMEM;
+		goto release_rmem;
+	}
+
+	/* use remaining reserved memory regions for static carveouts */
+	for (i = 0; i < num_rmems; i++) {
+		rmem_np = of_parse_phandle(np, "memory-region", i + 1);
+		if (!rmem_np) {
+			ret = -EINVAL;
+			goto unmap_rmem;
+		}
+
+		rmem = of_reserved_mem_lookup(rmem_np);
+		if (!rmem) {
+			of_node_put(rmem_np);
+			ret = -EINVAL;
+			goto unmap_rmem;
+		}
+		of_node_put(rmem_np);
+
+		kproc->rmem[i].bus_addr = rmem->base;
+		/* 64-bit address regions currently not supported */
+		kproc->rmem[i].dev_addr = (u32)rmem->base;
+		kproc->rmem[i].size = rmem->size;
+		kproc->rmem[i].cpu_addr = ioremap_wc(rmem->base, rmem->size);
+		if (!kproc->rmem[i].cpu_addr) {
+			dev_err(dev, "failed to map reserved memory#%d at %pa of size %pa\n",
+				i + 1, &rmem->base, &rmem->size);
+			ret = -ENOMEM;
+			goto unmap_rmem;
+		}
+
+		dev_dbg(dev, "reserved memory%d: bus addr %pa size 0x%zx va %pK da 0x%x\n",
+			i + 1, &kproc->rmem[i].bus_addr,
+			kproc->rmem[i].size, kproc->rmem[i].cpu_addr,
+			kproc->rmem[i].dev_addr);
+	}
+	kproc->num_rmems = num_rmems;
+
+	return 0;
+
+unmap_rmem:
+	for (i--; i >= 0; i--) {
+		if (kproc->rmem[i].cpu_addr)
+			iounmap(kproc->rmem[i].cpu_addr);
+	}
+	kfree(kproc->rmem);
+release_rmem:
+	of_reserved_mem_device_release(dev);
+	return ret;
+}
+
+static void k3_r5_reserved_mem_exit(struct k3_r5_rproc *kproc)
+{
+	int i;
+
+	for (i = 0; i < kproc->num_rmems; i++)
+		iounmap(kproc->rmem[i].cpu_addr);
+	kfree(kproc->rmem);
+
+	of_reserved_mem_device_release(kproc->dev);
+}
+
+/*
+ * This function checks and configures a R5F core for IPC-only or remoteproc
+ * mode. The driver is configured to be in IPC-only mode for a R5F core when
+ * the core has been loaded and started by a bootloader. The IPC-only mode is
+ * detected by querying the System Firmware for reset, power on and halt status
+ * and ensuring that the core is running. Any incomplete steps at bootloader
+ * are validated and errored out.
+ *
+ * In IPC-only mode, the driver state flags for ATCM, BTCM and LOCZRAMA settings
+ * and cluster mode parsed originally from kernel DT are updated to reflect the
+ * actual values configured by bootloader. The driver internal device memory
+ * addresses for TCMs are also updated.
+ */
+static int k3_r5_rproc_configure_mode(struct k3_r5_rproc *kproc)
+{
+	struct k3_r5_cluster *cluster = kproc->cluster;
+	struct k3_r5_core *core = kproc->core;
+	struct device *cdev = core->dev;
+	bool r_state = false, c_state = false;
+	u32 ctrl = 0, cfg = 0, stat = 0, halted = 0;
+	u64 boot_vec = 0;
+	u32 atcm_enable, btcm_enable, loczrama;
+	struct k3_r5_core *core0;
+	enum cluster_mode mode;
+	int ret;
+
+	core0 = list_first_entry(&cluster->cores, struct k3_r5_core, elem);
+
+	ret = core->ti_sci->ops.dev_ops.is_on(core->ti_sci, core->ti_sci_id,
+					      &r_state, &c_state);
+	if (ret) {
+		dev_err(cdev, "failed to get initial state, mode cannot be determined, ret = %d\n",
+			ret);
+		return ret;
+	}
+	if (r_state != c_state) {
+		dev_warn(cdev, "R5F core may have been powered on by a different host, programmed state (%d) != actual state (%d)\n",
+			 r_state, c_state);
+	}
+
+	ret = reset_control_status(core->reset);
+	if (ret < 0) {
+		dev_err(cdev, "failed to get initial local reset status, ret = %d\n",
+			ret);
+		return ret;
+	}
+
+	ret = ti_sci_proc_get_status(core->tsp, &boot_vec, &cfg, &ctrl,
+				     &stat);
+	if (ret < 0) {
+		dev_err(cdev, "failed to get initial processor status, ret = %d\n",
+			ret);
+		return ret;
+	}
+	atcm_enable = cfg & PROC_BOOT_CFG_FLAG_R5_ATCM_EN ?  1 : 0;
+	btcm_enable = cfg & PROC_BOOT_CFG_FLAG_R5_BTCM_EN ?  1 : 0;
+	loczrama = cfg & PROC_BOOT_CFG_FLAG_R5_TCM_RSTBASE ?  1 : 0;
+	mode = cfg & PROC_BOOT_CFG_FLAG_R5_LOCKSTEP ?  1 : 0;
+	halted = ctrl & PROC_BOOT_CTRL_FLAG_R5_CORE_HALT;
+
+	/*
+	 * IPC-only mode detection requires both local and module resets to
+	 * be deasserted and R5F core to be unhalted. Local reset status is
+	 * irrelevant if module reset is asserted (POR value has local reset
+	 * deasserted), and is deemed as remoteproc mode
+	 */
+	if (c_state && !ret && !halted) {
+		dev_err(cdev, "configured R5F for IPC-only mode\n");
+		kproc->rproc->skip_load = 1;
+		kproc->ipc_only = 1;
+		ret = 1;
+	} else if (!c_state) {
+		dev_err(cdev, "configured R5F for remoteproc mode\n");
+		ret = 0;
+	} else {
+		dev_err(cdev, "mismatched mode: local_reset = %s, module_reset = %s, core_state = %s\n",
+			!ret ? "deasserted" : "asserted",
+			c_state ? "deasserted" : "asserted",
+			halted ? "halted" : "unhalted");
+		ret = -EINVAL;
+	}
+
+	/* fixup TCMs, cluster & core flags to actual values in IPC-only mode */
+	if (ret > 0) {
+		if (core == core0)
+			cluster->mode = mode;
+		core->atcm_enable = atcm_enable;
+		core->btcm_enable = btcm_enable;
+		core->loczrama = loczrama;
+		core->mem[0].dev_addr = loczrama ? 0 : K3_R5_TCM_DEV_ADDR;
+		core->mem[1].dev_addr = loczrama ? K3_R5_TCM_DEV_ADDR : 0;
+	}
+
+	return ret;
+}
+
+static int k3_r5_cluster_rproc_init(struct platform_device *pdev)
+{
+	struct k3_r5_cluster *cluster = platform_get_drvdata(pdev);
+	struct device *dev = &pdev->dev;
+	struct k3_r5_rproc *kproc;
+	struct k3_r5_core *core, *core1;
+	struct device *cdev;
+	const char *fw_name;
+	struct rproc *rproc;
+	int ret;
+
+	core1 = list_last_entry(&cluster->cores, struct k3_r5_core, elem);
+	list_for_each_entry(core, &cluster->cores, elem) {
+		cdev = core->dev;
+		fw_name = k3_r5_rproc_get_firmware(cdev);
+		if (IS_ERR(fw_name)) {
+			ret = PTR_ERR(fw_name);
+			goto out;
+		}
+
+		rproc = rproc_alloc(cdev, dev_name(cdev), &k3_r5_rproc_ops,
+				    fw_name, sizeof(*kproc));
+		if (!rproc) {
+			ret = -ENOMEM;
+			goto out;
+		}
+
+		/* K3 R5s have a Region Address Translator (RAT) but no MMU */
+		rproc->has_iommu = false;
+		/* error recovery is not supported at present */
+		rproc->recovery_disabled = true;
+
+		kproc = rproc->priv;
+		kproc->cluster = cluster;
+		kproc->core = core;
+		kproc->dev = cdev;
+		kproc->rproc = rproc;
+		core->rproc = rproc;
+
+		ret = k3_r5_rproc_configure_mode(kproc);
+		if (ret < 0)
+			goto err_config;
+		if (ret)
+			goto init_rmem;
+
+		ret = k3_r5_rproc_configure(kproc);
+		if (ret) {
+			dev_err(dev, "initial configure failed, ret = %d\n",
+				ret);
+			goto err_config;
+		}
+
+init_rmem:
+		ret = k3_r5_reserved_mem_init(kproc);
+		if (ret) {
+			dev_err(dev, "reserved memory init failed, ret = %d\n",
+				ret);
+			goto err_config;
+		}
+
+		ret = rproc_add(rproc);
+		if (ret) {
+			dev_err(dev, "rproc_add failed, ret = %d\n", ret);
+			goto err_add;
+		}
+
+		/* create only one rproc in lockstep mode */
+		if (cluster->mode)
+			break;
+	}
+
+	return 0;
+
+err_split:
+	rproc_del(rproc);
+err_add:
+	k3_r5_reserved_mem_exit(kproc);
+err_config:
+	rproc_free(rproc);
+	core->rproc = NULL;
+out:
+	/* undo core0 upon any failures on core1 in split-mode */
+	if (!cluster->mode && core == core1) {
+		core = list_prev_entry(core, elem);
+		rproc = core->rproc;
+		kproc = rproc->priv;
+		goto err_split;
+	}
+	return ret;
+}
+
+static int k3_r5_cluster_rproc_exit(struct platform_device *pdev)
+{
+	struct k3_r5_cluster *cluster = platform_get_drvdata(pdev);
+	struct k3_r5_rproc *kproc;
+	struct k3_r5_core *core;
+	struct rproc *rproc;
+
+	/*
+	 * lockstep mode has only one rproc associated with first core, whereas
+	 * split-mode has two rprocs associated with each core, and requires
+	 * that core1 be powered down first
+	 */
+	core = cluster->mode ?
+		list_first_entry(&cluster->cores, struct k3_r5_core, elem) :
+		list_last_entry(&cluster->cores, struct k3_r5_core, elem);
+
+	list_for_each_entry_from_reverse(core, &cluster->cores, elem) {
+		rproc = core->rproc;
+		kproc = rproc->priv;
+
+		rproc_del(rproc);
+
+		k3_r5_reserved_mem_exit(kproc);
+
+		rproc_free(rproc);
+		core->rproc = NULL;
+	}
+
+	return 0;
+}
+
+static int k3_r5_core_of_get_internal_memories(struct platform_device *pdev,
+					       struct k3_r5_core *core)
+{
+	static const char * const mem_names[] = {"atcm", "btcm"};
+	struct device *dev = &pdev->dev;
+	struct resource *res;
+	int num_mems;
+	int i, ret;
+
+	num_mems = ARRAY_SIZE(mem_names);
+	core->mem = devm_kcalloc(dev, num_mems, sizeof(*core->mem), GFP_KERNEL);
+	if (!core->mem)
+		return -ENOMEM;
+
+	for (i = 0; i < num_mems; i++) {
+		res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+						   mem_names[i]);
+		if (!res) {
+			dev_err(dev, "found no memory resource for %s\n",
+				mem_names[i]);
+			ret = -EINVAL;
+			goto fail;
+		}
+		if (!devm_request_mem_region(dev, res->start,
+					     resource_size(res),
+					     dev_name(dev))) {
+			dev_err(dev, "could not request %s region for resource\n",
+				mem_names[i]);
+			ret = -EBUSY;
+			goto fail;
+		}
+
+		core->mem[i].cpu_addr = devm_ioremap_wc(dev, res->start,
+							resource_size(res));
+		if (IS_ERR(core->mem[i].cpu_addr)) {
+			dev_err(dev, "failed to map %s memory\n", mem_names[i]);
+			ret = PTR_ERR(core->mem[i].cpu_addr);
+			devm_release_mem_region(dev, res->start,
+						resource_size(res));
+			goto fail;
+		}
+		core->mem[i].bus_addr = res->start;
+
+		/*
+		 * TODO:
+		 * The R5F cores can place ATCM & BTCM anywhere in its address
+		 * based on the corresponding Region Registers in the System
+		 * Control coprocessor. For now, place ATCM and BTCM at
+		 * addresses 0 and 0x41010000 (same as the bus address on AM65x
+		 * SoCs) based on loczrama setting
+		 */
+		if (!strcmp(mem_names[i], "atcm")) {
+			core->mem[i].dev_addr = core->loczrama ?
+							0 : K3_R5_TCM_DEV_ADDR;
+		} else {
+			core->mem[i].dev_addr = core->loczrama ?
+							K3_R5_TCM_DEV_ADDR : 0;
+		}
+		core->mem[i].size = resource_size(res);
+
+		dev_dbg(dev, "memory %8s: bus addr %pa size 0x%zx va %p da 0x%x\n",
+			mem_names[i], &core->mem[i].bus_addr,
+			core->mem[i].size, core->mem[i].cpu_addr,
+			core->mem[i].dev_addr);
+	}
+	core->num_mems = num_mems;
+
+	return 0;
+
+fail:
+	for (i--; i >= 0; i--) {
+		devm_release_mem_region(dev, core->mem[i].bus_addr,
+					core->mem[i].size);
+		devm_iounmap(dev, core->mem[i].cpu_addr);
+	}
+	if (core->mem)
+		devm_kfree(dev, core->mem);
+	return ret;
+}
+
+static int k3_r5_core_of_get_sram_memories(struct platform_device *pdev,
+					   struct k3_r5_core *core)
+{
+	struct device_node *np = pdev->dev.of_node;
+	struct device *dev = &pdev->dev;
+	struct device_node *sram_np;
+	struct resource res;
+	int num_sram;
+	int i, ret;
+
+	num_sram = of_property_count_elems_of_size(np, "sram", sizeof(phandle));
+	if (num_sram <= 0) {
+		dev_dbg(dev, "device does not use reserved on-chip memories, num_sram = %d\n",
+			num_sram);
+		return 0;
+	}
+
+	core->sram = kcalloc(num_sram, sizeof(*core->sram), GFP_KERNEL);
+	if (!core->sram)
+		return -ENOMEM;
+
+	for (i = 0; i < num_sram; i++) {
+		sram_np = of_parse_phandle(np, "sram", i);
+		if (!sram_np) {
+			ret = -EINVAL;
+			goto fail;
+		}
+
+		if (!of_device_is_available(sram_np)) {
+			of_node_put(sram_np);
+			ret = -EINVAL;
+			goto fail;
+		}
+
+		ret = of_address_to_resource(sram_np, 0, &res);
+		of_node_put(sram_np);
+		if (ret) {
+			ret = -EINVAL;
+			goto fail;
+		}
+		core->sram[i].bus_addr = res.start;
+		core->sram[i].dev_addr = res.start;
+		core->sram[i].size = resource_size(&res);
+		core->sram[i].cpu_addr = ioremap_wc(res.start,
+						    resource_size(&res));
+		if (!core->sram[i].cpu_addr) {
+			dev_err(dev, "failed to parse and map sram%d memory at %pad\n",
+				i, &res.start);
+			ret = -ENOMEM;
+			goto fail;
+		}
+
+		dev_dbg(dev, "memory    sram%d: bus addr %pa size 0x%zx va %pK da 0x%x\n",
+			i, &core->sram[i].bus_addr,
+			core->sram[i].size, core->sram[i].cpu_addr,
+			core->sram[i].dev_addr);
+	}
+	core->num_sram = num_sram;
+
+	return 0;
+
+fail:
+	for (i--; i >= 0; i--) {
+		if (core->sram[i].cpu_addr)
+			iounmap(core->sram[i].cpu_addr);
+	}
+	kfree(core->sram);
+
+	return ret;
+}
+
+static
+struct ti_sci_proc *k3_r5_core_of_get_tsp(struct device *dev,
+					  const struct ti_sci_handle *sci)
+{
+	struct ti_sci_proc *tsp;
+	u32 temp[2];
+	int ret;
+
+	ret = of_property_read_u32_array(dev->of_node, "ti,sci-proc-ids",
+					 temp, 2);
+	if (ret < 0)
+		return ERR_PTR(ret);
+
+	tsp = kzalloc(sizeof(*tsp), GFP_KERNEL);
+	if (!tsp)
+		return ERR_PTR(-ENOMEM);
+
+	tsp->dev = dev;
+	tsp->sci = sci;
+	tsp->ops = &sci->ops.proc_ops;
+	tsp->proc_id = temp[0];
+	tsp->host_id = temp[1];
+
+	return tsp;
+}
+
+static int k3_r5_core_of_init(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct k3_r5_core *core;
+	int ret, ret1, i;
+
+	core = devm_kzalloc(dev, sizeof(*core), GFP_KERNEL);
+	if (!core)
+		return -ENOMEM;
+
+	core->dev = dev;
+	core->atcm_enable = 0;
+	core->btcm_enable = 1;
+	core->loczrama = 1;
+
+	ret = of_property_read_u32(np, "atcm-enable", &core->atcm_enable);
+	if (ret < 0 && ret != -EINVAL) {
+		dev_err(dev, "invalid format for atcm-enable, ret = %d\n", ret);
+		goto err_of;
+	}
+
+	ret = of_property_read_u32(np, "btcm-enable", &core->btcm_enable);
+	if (ret < 0 && ret != -EINVAL) {
+		dev_err(dev, "invalid format for btcm-enable, ret = %d\n", ret);
+		goto err_of;
+	}
+
+	ret = of_property_read_u32(np, "loczrama", &core->loczrama);
+	if (ret < 0 && ret != -EINVAL) {
+		dev_err(dev, "invalid format for loczrama, ret = %d\n", ret);
+		goto err_of;
+	}
+
+	core->ti_sci = ti_sci_get_by_phandle(np, "ti,sci");
+	if (IS_ERR(core->ti_sci)) {
+		ret = PTR_ERR(core->ti_sci);
+		if (ret != -EPROBE_DEFER) {
+			dev_err(dev, "failed to get ti-sci handle, ret = %d\n",
+				ret);
+		}
+		core->ti_sci = NULL;
+		goto err_of;
+	}
+
+	ret = of_property_read_u32(np, "ti,sci-dev-id", &core->ti_sci_id);
+	if (ret) {
+		dev_err(dev, "missing 'ti,sci-dev-id' property\n");
+		goto err_sci_id;
+	}
+
+	core->reset = reset_control_get_exclusive(dev, NULL);
+	if (IS_ERR(core->reset)) {
+		ret = PTR_ERR(core->reset);
+		if (ret != -EPROBE_DEFER) {
+			dev_err(dev, "failed to get reset handle, ret = %d\n",
+				ret);
+		}
+		goto err_sci_id;
+	}
+
+	core->tsp = k3_r5_core_of_get_tsp(dev, core->ti_sci);
+	if (IS_ERR(core->tsp)) {
+		dev_err(dev, "failed to construct ti-sci proc control, ret = %d\n",
+			ret);
+		ret = PTR_ERR(core->tsp);
+		goto err_sci_proc;
+	}
+
+	ret = ti_sci_proc_request(core->tsp);
+	if (ret < 0) {
+		dev_err(dev, "ti_sci_proc_request failed, ret = %d\n", ret);
+		goto err_proc;
+	}
+
+	ret = k3_r5_core_of_get_internal_memories(pdev, core);
+	if (ret) {
+		dev_err(dev, "failed to get internal memories, ret = %d\n",
+			ret);
+		goto err_intmem;
+	}
+
+	ret = k3_r5_core_of_get_sram_memories(pdev, core);
+	if (ret) {
+		dev_err(dev, "failed to get sram memories, ret = %d\n", ret);
+		goto err_sram;
+	}
+
+	platform_set_drvdata(pdev, core);
+
+	return 0;
+
+err_sram:
+	for (i = 0; i < core->num_mems; i++) {
+		devm_release_mem_region(dev, core->mem[i].bus_addr,
+					core->mem[i].size);
+		devm_iounmap(dev, core->mem[i].cpu_addr);
+	}
+	devm_kfree(dev, core->mem);
+err_intmem:
+	ret1 = ti_sci_proc_release(core->tsp);
+	if (ret1)
+		dev_err(dev, "failed to release proc, ret1 = %d\n", ret1);
+err_proc:
+	kfree(core->tsp);
+err_sci_proc:
+	reset_control_put(core->reset);
+err_sci_id:
+	ret1 = ti_sci_put_handle(core->ti_sci);
+	if (ret1)
+		dev_err(dev, "failed to put ti_sci handle, ret = %d\n", ret1);
+err_of:
+	devm_kfree(dev, core);
+	return ret;
+}
+
+/*
+ * free the resources explicitly since driver model is not being used
+ * for the child R5F devices
+ */
+static int k3_r5_core_of_exit(struct platform_device *pdev)
+{
+	struct k3_r5_core *core = platform_get_drvdata(pdev);
+	struct device *dev = &pdev->dev;
+	int i, ret;
+
+	for (i = 0; i < core->num_sram; i++)
+		iounmap(core->sram[i].cpu_addr);
+	kfree(core->sram);
+
+	for (i = 0; i < core->num_mems; i++) {
+		devm_release_mem_region(dev, core->mem[i].bus_addr,
+					core->mem[i].size);
+		devm_iounmap(dev, core->mem[i].cpu_addr);
+	}
+	if (core->mem)
+		devm_kfree(dev, core->mem);
+
+	ret = ti_sci_proc_release(core->tsp);
+	if (ret)
+		dev_err(dev, "failed to release proc, ret = %d\n", ret);
+
+	kfree(core->tsp);
+	reset_control_put(core->reset);
+
+	ret = ti_sci_put_handle(core->ti_sci);
+	if (ret)
+		dev_err(dev, "failed to put ti_sci handle, ret = %d\n", ret);
+
+	platform_set_drvdata(pdev, NULL);
+	devm_kfree(dev, core);
+
+	return ret;
+}
+
+static int k3_r5_cluster_of_init(struct platform_device *pdev)
+{
+	struct k3_r5_cluster *cluster = platform_get_drvdata(pdev);
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct platform_device *cpdev;
+	struct device_node *child;
+	struct k3_r5_core *core, *temp;
+	int ret;
+
+	for_each_available_child_of_node(np, child) {
+		cpdev = of_find_device_by_node(child);
+		if (!cpdev) {
+			ret = -ENODEV;
+			dev_err(dev, "could not get R5 core platform device\n");
+			goto fail;
+		}
+
+		ret = k3_r5_core_of_init(cpdev);
+		if (ret) {
+			dev_err(dev, "k3_r5_core_of_init failed, ret = %d\n",
+				ret);
+			put_device(&cpdev->dev);
+			goto fail;
+		}
+
+		core = platform_get_drvdata(cpdev);
+		put_device(&cpdev->dev);
+		list_add_tail(&core->elem, &cluster->cores);
+	}
+
+	return 0;
+
+fail:
+	list_for_each_entry_safe_reverse(core, temp, &cluster->cores, elem) {
+		list_del(&core->elem);
+		cpdev = to_platform_device(core->dev);
+		if (k3_r5_core_of_exit(cpdev))
+			dev_err(dev, "k3_r5_core_of_exit cleanup failed\n");
+	}
+	return ret;
+}
+
+static int k3_r5_cluster_of_exit(struct platform_device *pdev)
+{
+	struct k3_r5_cluster *cluster = platform_get_drvdata(pdev);
+	struct device *dev = &pdev->dev;
+	struct platform_device *cpdev;
+	struct k3_r5_core *core, *temp;
+	int ret;
+
+	list_for_each_entry_safe_reverse(core, temp, &cluster->cores, elem) {
+		list_del(&core->elem);
+		cpdev = to_platform_device(core->dev);
+		ret = k3_r5_core_of_exit(cpdev);
+		if (ret) {
+			dev_err(dev, "k3_r5_core_of_exit failed, ret = %d\n",
+				ret);
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static int k3_r5_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct k3_r5_cluster *cluster;
+	int ret, ret1;
+	int num_cores;
+
+	cluster = devm_kzalloc(dev, sizeof(*cluster), GFP_KERNEL);
+	if (!cluster)
+		return -ENOMEM;
+
+	cluster->dev = dev;
+	cluster->mode = CLUSTER_MODE_LOCKSTEP;
+	INIT_LIST_HEAD(&cluster->cores);
+
+	ret = of_property_read_u32(np, "lockstep-mode", &cluster->mode);
+	if (ret < 0 && ret != -EINVAL) {
+		dev_err(dev, "invalid format for lockstep-mode, ret = %d\n",
+			ret);
+		return ret;
+	}
+
+	num_cores = of_get_available_child_count(np);
+	if (num_cores != 2) {
+		dev_err(dev, "MCU cluster requires both R5F cores to be enabled, num_cores = %d\n",
+			num_cores);
+		return -ENODEV;
+	}
+
+	platform_set_drvdata(pdev, cluster);
+
+	dev_info(dev, "creating child devices for R5F cores\n");
+	ret = of_platform_populate(np, NULL, NULL, dev);
+	if (ret) {
+		dev_err(dev, "of_platform_populate failed, ret = %d\n", ret);
+		return ret;
+	}
+
+	ret = k3_r5_cluster_of_init(pdev);
+	if (ret) {
+		dev_err(dev, "k3_r5_cluster_of_init failed, ret = %d\n", ret);
+		goto fail_of;
+	}
+
+	ret = k3_r5_cluster_rproc_init(pdev);
+	if (ret) {
+		dev_err(dev, "k3_r5_cluster_rproc_init failed, ret = %d\n",
+			ret);
+		goto fail_rproc;
+	}
+
+	return 0;
+
+fail_rproc:
+	ret1 = k3_r5_cluster_of_exit(pdev);
+	if (ret1)
+		dev_err(dev, "k3_r5_cluster_of_exit failed, ret = %d\n", ret1);
+fail_of:
+	of_platform_depopulate(dev);
+	return ret;
+}
+
+static int k3_r5_remove(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	int ret;
+
+	ret = k3_r5_cluster_rproc_exit(pdev);
+	if (ret) {
+		dev_err(dev, "k3_r5_cluster_rproc_exit failed, ret = %d\n",
+			ret);
+		goto fail;
+	}
+
+	ret = k3_r5_cluster_of_exit(pdev);
+	if (ret) {
+		dev_err(dev, "k3_r5_cluster_of_exit failed, ret = %d\n", ret);
+		goto fail;
+	}
+
+	dev_info(dev, "removing child devices for R5F cores\n");
+	of_platform_depopulate(dev);
+
+fail:
+	return ret;
+}
+
+static const struct k3_r5_rproc_dev_data am65x_r5f_dev_data[] = {
+	{
+		.device_name	= "41000000.r5f",
+		.fw_name	= "am65x-mcu-r5f0_0-fw",
+	},
+	{
+		.device_name	= "41400000.r5f",
+		.fw_name	= "am65x-mcu-r5f0_1-fw",
+	},
+	{
+		/* sentinel */
+	},
+};
+
+static const struct k3_r5_rproc_dev_data j721e_r5f_dev_data[] = {
+	{
+		.device_name	= "41000000.r5f",
+		.fw_name	= "j7-mcu-r5f0_0-fw",
+	},
+	{
+		.device_name	= "41400000.r5f",
+		.fw_name	= "j7-mcu-r5f0_1-fw",
+	},
+	{
+		.device_name	= "5c00000.r5f",
+		.fw_name	= "j7-main-r5f0_0-fw",
+	},
+	{
+		.device_name	= "5d00000.r5f",
+		.fw_name	= "j7-main-r5f0_1-fw",
+	},
+	{
+		.device_name	= "5e00000.r5f",
+		.fw_name	= "j7-main-r5f1_0-fw",
+	},
+	{
+		.device_name	= "5f00000.r5f",
+		.fw_name	= "j7-main-r5f1_1-fw",
+	},
+	{
+		/* sentinel */
+	},
+};
+
+static const struct of_device_id k3_r5_of_match[] = {
+	{
+		.compatible     = "ti,am654-r5fss",
+		.data           = am65x_r5f_dev_data,
+	},
+	{
+		.compatible     = "ti,j721e-r5fss",
+		.data           = j721e_r5f_dev_data,
+	},
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, k3_r5_of_match);
+
+static struct platform_driver k3_r5_rproc_driver = {
+	.probe = k3_r5_probe,
+	.remove = k3_r5_remove,
+	.driver = {
+		.name = "k3_r5_rproc",
+		.of_match_table = k3_r5_of_match,
+	},
+};
+
+module_platform_driver(k3_r5_rproc_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("TI K3 R5F remote processor driver");
+MODULE_AUTHOR("Suman Anna <s-anna@ti.com>");
diff -Naur --no-dereference a/drivers/remoteproc/ti_sci_proc.h b/drivers/remoteproc/ti_sci_proc.h
--- a/drivers/remoteproc/ti_sci_proc.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/remoteproc/ti_sci_proc.h	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,102 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Texas Instruments TI-SCI Processor Controller Helper Functions
+ *
+ * Copyright (C) 2018-2019 Texas Instruments Incorporated - http://www.ti.com/
+ *	Suman Anna
+ */
+
+#ifndef REMOTEPROC_TI_SCI_PROC_H
+#define REMOTEPROC_TI_SCI_PROC_H
+
+/**
+ * struct ti_sci_proc - structure representing a processor control client
+ * @sci: cached TI-SCI protocol handle
+ * @ops: cached TI-SCI proc ops
+ * @dev: cached client device pointer
+ * @proc_id: processor id for the consumer remoteproc device
+ * @host_id: host id to pass the control over for this consumer remoteproc
+ *	     device
+ */
+struct ti_sci_proc {
+	const struct ti_sci_handle *sci;
+	const struct ti_sci_proc_ops *ops;
+	struct device *dev;
+	u8 proc_id;
+	u8 host_id;
+};
+
+static inline int ti_sci_proc_request(struct ti_sci_proc *tsp)
+{
+	int ret;
+
+	ret = tsp->ops->request(tsp->sci, tsp->proc_id);
+	if (ret)
+		dev_err(tsp->dev, "ti-sci processor request failed: %d\n",
+			ret);
+	return ret;
+}
+
+static inline int ti_sci_proc_release(struct ti_sci_proc *tsp)
+{
+	int ret;
+
+	ret = tsp->ops->release(tsp->sci, tsp->proc_id);
+	if (ret)
+		dev_err(tsp->dev, "ti-sci processor release failed: %d\n",
+			ret);
+	return ret;
+}
+
+static inline int ti_sci_proc_handover(struct ti_sci_proc *tsp)
+{
+	int ret;
+
+	ret = tsp->ops->handover(tsp->sci, tsp->proc_id, tsp->host_id);
+	if (ret)
+		dev_err(tsp->dev, "ti-sci processor handover of %d to %d failed: %d\n",
+			tsp->proc_id, tsp->host_id, ret);
+	return ret;
+}
+
+static inline int ti_sci_proc_set_config(struct ti_sci_proc *tsp,
+					 u64 boot_vector,
+					 u32 cfg_set, u32 cfg_clr)
+{
+	int ret;
+
+	ret = tsp->ops->set_config(tsp->sci, tsp->proc_id, boot_vector,
+				   cfg_set, cfg_clr);
+	if (ret)
+		dev_err(tsp->dev, "ti-sci processor set_config failed: %d\n",
+			ret);
+	return ret;
+}
+
+static inline int ti_sci_proc_set_control(struct ti_sci_proc *tsp,
+					  u32 ctrl_set, u32 ctrl_clr)
+{
+	int ret;
+
+	ret = tsp->ops->set_control(tsp->sci, tsp->proc_id, ctrl_set, ctrl_clr);
+	if (ret)
+		dev_err(tsp->dev, "ti-sci processor set_control failed: %d\n",
+			ret);
+	return ret;
+}
+
+static inline int ti_sci_proc_get_status(struct ti_sci_proc *tsp,
+					 u64 *boot_vector, u32 *cfg_flags,
+					 u32 *ctrl_flags, u32 *status_flags)
+{
+	int ret;
+
+	ret = tsp->ops->get_status(tsp->sci, tsp->proc_id, boot_vector,
+				   cfg_flags, ctrl_flags, status_flags);
+	if (ret)
+		dev_err(tsp->dev, "ti-sci processor get_status failed: %d\n",
+			ret);
+	return ret;
+}
+
+#endif /* REMOTEPROC_TI_SCI_PROC_H */
diff -Naur --no-dereference a/drivers/remoteproc/wkup_m3_rproc.c b/drivers/remoteproc/wkup_m3_rproc.c
--- a/drivers/remoteproc/wkup_m3_rproc.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/remoteproc/wkup_m3_rproc.c	2020-06-19 21:00:31.424227716 -0400
@@ -88,7 +88,8 @@
 	return 0;
 }
 
-static void *wkup_m3_rproc_da_to_va(struct rproc *rproc, u64 da, int len)
+static void *wkup_m3_rproc_da_to_va(struct rproc *rproc, u64 da, int len,
+				    u32 flags)
 {
 	struct wkup_m3_rproc *wkupm3 = rproc->priv;
 	void *va = NULL;
@@ -168,6 +169,7 @@
 	}
 
 	rproc->auto_boot = false;
+	rproc->deny_sysfs_ops = 1;
 
 	wkupm3 = rproc->priv;
 	wkupm3->rproc = rproc;
diff -Naur --no-dereference a/drivers/rpmsg/Kconfig b/drivers/rpmsg/Kconfig
--- a/drivers/rpmsg/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/rpmsg/Kconfig	2020-06-19 21:00:31.424227716 -0400
@@ -55,4 +55,30 @@
 	select RPMSG
 	select VIRTIO
 
+config RPMSG_RPC
+	tristate "rpmsg Remote Procedure Call driver"
+	default n
+	depends on RPMSG_VIRTIO
+	depends on REMOTEPROC
+	depends on OMAP_REMOTEPROC
+	select DMA_SHARED_BUFFER
+	help
+	  An rpmsg driver that exposes the Remote Procedure Call API to
+	  user space, in order to allow applications to distribute
+	  remote calls to more power-efficient remote processors. This is
+	  currently available only on OMAP4+ systems.
+
+config RPMSG_PRU
+	tristate "PRU RPMsg Communication driver"
+	depends on RPMSG_VIRTIO
+	depends on REMOTEPROC
+	depends on PRU_REMOTEPROC
+	help
+	  An rpmsg driver that exposes interfaces to user space, to allow
+	  applications to communicate with the PRU processors on available
+	  TI SoCs. This is restricted to SoCs that have the PRUSS remoteproc
+	  support.
+
+	  If unsure, say N.
+
 endmenu
diff -Naur --no-dereference a/drivers/rpmsg/Makefile b/drivers/rpmsg/Makefile
--- a/drivers/rpmsg/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/rpmsg/Makefile	2020-06-19 21:00:31.424227716 -0400
@@ -1,8 +1,12 @@
 # SPDX-License-Identifier: GPL-2.0
 obj-$(CONFIG_RPMSG)		+= rpmsg_core.o
 obj-$(CONFIG_RPMSG_CHAR)	+= rpmsg_char.o
+obj-$(CONFIG_RPMSG_PRU)		+= rpmsg_pru.o
 obj-$(CONFIG_RPMSG_QCOM_GLINK_RPM) += qcom_glink_rpm.o
 obj-$(CONFIG_RPMSG_QCOM_GLINK_NATIVE) += qcom_glink_native.o
 obj-$(CONFIG_RPMSG_QCOM_GLINK_SMEM) += qcom_glink_smem.o
 obj-$(CONFIG_RPMSG_QCOM_SMD)	+= qcom_smd.o
 obj-$(CONFIG_RPMSG_VIRTIO)	+= virtio_rpmsg_bus.o
+
+obj-$(CONFIG_RPMSG_RPC)		+= rpmsg-rpc.o
+rpmsg-rpc-y			:= rpmsg_rpc.o rpmsg_rpc_sysfs.o rpmsg_rpc_dmabuf.o
diff -Naur --no-dereference a/drivers/rpmsg/qcom_glink_native.c b/drivers/rpmsg/qcom_glink_native.c
--- a/drivers/rpmsg/qcom_glink_native.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/rpmsg/qcom_glink_native.c	2020-06-19 21:00:31.424227716 -0400
@@ -1477,6 +1477,7 @@
 		strncpy(chinfo.name, channel->name, sizeof(chinfo.name));
 		chinfo.src = RPMSG_ADDR_ANY;
 		chinfo.dst = RPMSG_ADDR_ANY;
+		chinfo.desc[0] = '\0';
 
 		rpmsg_unregister_device(glink->dev, &chinfo);
 	}
diff -Naur --no-dereference a/drivers/rpmsg/qcom_smd.c b/drivers/rpmsg/qcom_smd.c
--- a/drivers/rpmsg/qcom_smd.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/rpmsg/qcom_smd.c	2020-06-19 21:00:31.424227716 -0400
@@ -1307,6 +1307,7 @@
 		strncpy(chinfo.name, channel->name, sizeof(chinfo.name));
 		chinfo.src = RPMSG_ADDR_ANY;
 		chinfo.dst = RPMSG_ADDR_ANY;
+		chinfo.desc[0] = '\0';
 		rpmsg_unregister_device(&edge->dev, &chinfo);
 		channel->registered = false;
 		spin_lock_irqsave(&edge->channels_lock, flags);
diff -Naur --no-dereference a/drivers/rpmsg/rpmsg_char.c b/drivers/rpmsg/rpmsg_char.c
--- a/drivers/rpmsg/rpmsg_char.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/rpmsg/rpmsg_char.c	2020-06-19 21:00:31.424227716 -0400
@@ -167,9 +167,9 @@
 	return 0;
 }
 
-static ssize_t rpmsg_eptdev_read(struct file *filp, char __user *buf,
-				 size_t len, loff_t *f_pos)
+static ssize_t rpmsg_eptdev_read_iter(struct kiocb *iocb, struct iov_iter *to)
 {
+	struct file *filp = iocb->ki_filp;
 	struct rpmsg_eptdev *eptdev = filp->private_data;
 	unsigned long flags;
 	struct sk_buff *skb;
@@ -205,8 +205,8 @@
 	if (!skb)
 		return -EFAULT;
 
-	use = min_t(size_t, len, skb->len);
-	if (copy_to_user(buf, skb->data, use))
+	use = min_t(size_t, iov_iter_count(to), skb->len);
+	if (copy_to_iter(skb->data, use, to) != use)
 		use = -EFAULT;
 
 	kfree_skb(skb);
@@ -214,16 +214,21 @@
 	return use;
 }
 
-static ssize_t rpmsg_eptdev_write(struct file *filp, const char __user *buf,
-				  size_t len, loff_t *f_pos)
+static ssize_t rpmsg_eptdev_write_iter(struct kiocb *iocb,
+				       struct iov_iter *from)
 {
+	struct file *filp = iocb->ki_filp;
 	struct rpmsg_eptdev *eptdev = filp->private_data;
+	size_t len = iov_iter_count(from);
 	void *kbuf;
 	int ret;
 
-	kbuf = memdup_user(buf, len);
-	if (IS_ERR(kbuf))
-		return PTR_ERR(kbuf);
+	kbuf = kzalloc(len, GFP_KERNEL);
+	if (!kbuf)
+		return -ENOMEM;
+
+	if (!copy_from_iter_full(kbuf, len, from))
+		return -EFAULT;
 
 	if (mutex_lock_interruptible(&eptdev->ept_lock)) {
 		ret = -ERESTARTSYS;
@@ -281,8 +286,8 @@
 	.owner = THIS_MODULE,
 	.open = rpmsg_eptdev_open,
 	.release = rpmsg_eptdev_release,
-	.read = rpmsg_eptdev_read,
-	.write = rpmsg_eptdev_write,
+	.read_iter = rpmsg_eptdev_read_iter,
+	.write_iter = rpmsg_eptdev_write_iter,
 	.poll = rpmsg_eptdev_poll,
 	.unlocked_ioctl = rpmsg_eptdev_ioctl,
 	.compat_ioctl = rpmsg_eptdev_ioctl,
@@ -437,6 +442,7 @@
 	chinfo.name[RPMSG_NAME_SIZE-1] = '\0';
 	chinfo.src = eptinfo.src;
 	chinfo.dst = eptinfo.dst;
+	chinfo.desc[0] = '\0';
 
 	return rpmsg_eptdev_create(ctrldev, chinfo);
 };
diff -Naur --no-dereference a/drivers/rpmsg/rpmsg_core.c b/drivers/rpmsg/rpmsg_core.c
--- a/drivers/rpmsg/rpmsg_core.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/rpmsg/rpmsg_core.c	2020-06-19 21:00:31.424227716 -0400
@@ -46,7 +46,7 @@
  * equals to the src address of their rpmsg channel), the driver's handler
  * is invoked to process it.
  *
- * That said, more complicated drivers might do need to allocate
+ * That said, more complicated drivers might need to allocate
  * additional rpmsg addresses, and bind them to different rx callbacks.
  * To accomplish that, those drivers need to call this function.
  *
@@ -177,7 +177,7 @@
 EXPORT_SYMBOL(rpmsg_send_offchannel);
 
 /**
- * rpmsg_send() - send a message across to the remote processor
+ * rpmsg_trysend() - send a message across to the remote processor
  * @ept: the rpmsg endpoint
  * @data: payload of message
  * @len: length of payload
@@ -205,7 +205,7 @@
 EXPORT_SYMBOL(rpmsg_trysend);
 
 /**
- * rpmsg_sendto() - send a message across to the remote processor, specify dst
+ * rpmsg_trysendto() - send a message across to the remote processor, specify dst
  * @ept: the rpmsg endpoint
  * @data: payload of message
  * @len: length of payload
@@ -253,7 +253,7 @@
 EXPORT_SYMBOL(rpmsg_poll);
 
 /**
- * rpmsg_send_offchannel() - send a message using explicit src/dst addresses
+ * rpmsg_trysend_offchannel() - send a message using explicit src/dst addresses
  * @ept: the rpmsg endpoint
  * @src: source address
  * @dst: destination address
@@ -302,6 +302,10 @@
 	if (strncmp(chinfo->name, rpdev->id.name, RPMSG_NAME_SIZE))
 		return 0;
 
+	if (chinfo->desc && chinfo->desc != rpdev->desc &&
+	    strncmp(chinfo->desc, rpdev->desc, RPMSG_NAME_SIZE))
+		return 0;
+
 	/* found a match ! */
 	return 1;
 }
@@ -365,6 +369,7 @@
 
 /* for more info, see Documentation/ABI/testing/sysfs-bus-rpmsg */
 rpmsg_show_attr(name, id.name, "%s\n");
+rpmsg_show_attr(desc, desc, "%s\n");
 rpmsg_show_attr(src, src, "0x%x\n");
 rpmsg_show_attr(dst, dst, "0x%x\n");
 rpmsg_show_attr(announce, announce ? "true" : "false", "%s\n");
@@ -386,6 +391,7 @@
 
 static struct attribute *rpmsg_dev_attrs[] = {
 	&dev_attr_name.attr,
+	&dev_attr_desc.attr,
 	&dev_attr_modalias.attr,
 	&dev_attr_dst.attr,
 	&dev_attr_src.attr,
diff -Naur --no-dereference a/drivers/rpmsg/rpmsg_internal.h b/drivers/rpmsg/rpmsg_internal.h
--- a/drivers/rpmsg/rpmsg_internal.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/rpmsg/rpmsg_internal.h	2020-06-19 21:00:31.424227716 -0400
@@ -20,7 +20,7 @@
 
 /**
  * struct rpmsg_device_ops - indirection table for the rpmsg_device operations
- * @create_ept:		create backend-specific endpoint, requried
+ * @create_ept:		create backend-specific endpoint, required
  * @announce_create:	announce presence of new channel, optional
  * @announce_destroy:	announce destruction of channel, optional
  *
@@ -39,13 +39,14 @@
 
 /**
  * struct rpmsg_endpoint_ops - indirection table for rpmsg_endpoint operations
- * @destroy_ept:	destroy the given endpoint, required
+ * @destroy_ept:	see @rpmsg_destroy_ept(), required
  * @send:		see @rpmsg_send(), required
  * @sendto:		see @rpmsg_sendto(), optional
  * @send_offchannel:	see @rpmsg_send_offchannel(), optional
  * @trysend:		see @rpmsg_trysend(), required
  * @trysendto:		see @rpmsg_trysendto(), optional
  * @trysend_offchannel:	see @rpmsg_trysend_offchannel(), optional
+ * @poll:		see @rpmsg_poll(), optional
  *
  * Indirection table for the operations that a rpmsg backend should implement.
  * In addition to @destroy_ept, the backend must at least implement @send and
diff -Naur --no-dereference a/drivers/rpmsg/rpmsg_pru.c b/drivers/rpmsg/rpmsg_pru.c
--- a/drivers/rpmsg/rpmsg_pru.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/rpmsg/rpmsg_pru.c	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,350 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * PRU Remote Processor Messaging Driver
+ *
+ * Copyright (C) 2015-2019 Texas Instruments Incorporated - http://www.ti.com/
+ *	Jason Reeder <jreeder@ti.com>
+ *	Suman Anna <s-anna@ti.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/rpmsg.h>
+#include <linux/slab.h>
+#include <linux/fs.h>
+#include <linux/init.h>
+#include <linux/cdev.h>
+#include <linux/module.h>
+#include <linux/kfifo.h>
+#include <linux/uaccess.h>
+#include <linux/mutex.h>
+#include <linux/poll.h>
+#include <linux/rpmsg/virtio_rpmsg.h>
+
+#define PRU_MAX_DEVICES				(16)
+/* Matches the definition in virtio_rpmsg_bus.c */
+#define RPMSG_BUF_SIZE				(512)
+#define MAX_FIFO_MSG				(32)
+#define FIFO_MSG_SIZE				RPMSG_BUF_SIZE
+
+/**
+ * struct rpmsg_pru_dev - Structure that contains the per-device data
+ * @rpdev: rpmsg channel device that is associated with this rpmsg_pru device
+ * @dev: device
+ * @cdev: character device
+ * @locked: boolean used to determine whether or not the device file is in use
+ * @devt: dev_t structure for the rpmsg_pru device
+ * @msg_fifo: kernel fifo used to buffer the messages between userspace and PRU
+ * @msg_len: array storing the lengths of each message in the kernel fifo
+ * @msg_idx_rd: kernel fifo read index
+ * @msg_idx_wr: kernel fifo write index
+ * @wait_list: wait queue used to implement the poll operation of the character
+ *             device
+ *
+ * Each rpmsg_pru device provides an interface, using an rpmsg channel (rpdev),
+ * between a user space character device (cdev) and a PRU core. A kernel fifo
+ * (msg_fifo) is used to buffer the messages in the kernel that are
+ * being passed between the character device and the PRU.
+ */
+struct rpmsg_pru_dev {
+	struct rpmsg_device *rpdev;
+	struct device *dev;
+	struct cdev cdev;
+	bool locked;
+	dev_t devt;
+	struct kfifo msg_fifo;
+	u32 msg_len[MAX_FIFO_MSG];
+	int msg_idx_rd;
+	int msg_idx_wr;
+	wait_queue_head_t wait_list;
+};
+
+static struct class *rpmsg_pru_class;
+static dev_t rpmsg_pru_devt;
+static DEFINE_MUTEX(rpmsg_pru_lock);
+static DEFINE_IDR(rpmsg_pru_minors);
+
+static int rpmsg_pru_open(struct inode *inode, struct file *filp)
+{
+	struct rpmsg_pru_dev *prudev;
+	int ret = -EACCES;
+
+	prudev = container_of(inode->i_cdev, struct rpmsg_pru_dev, cdev);
+
+	mutex_lock(&rpmsg_pru_lock);
+	if (!prudev->locked) {
+		prudev->locked = true;
+		filp->private_data = prudev;
+		ret = 0;
+	}
+	mutex_unlock(&rpmsg_pru_lock);
+
+	if (ret)
+		dev_err(prudev->dev, "Device already open\n");
+
+	return ret;
+}
+
+static int rpmsg_pru_release(struct inode *inode, struct file *filp)
+{
+	struct rpmsg_pru_dev *prudev;
+
+	prudev = container_of(inode->i_cdev, struct rpmsg_pru_dev, cdev);
+	mutex_lock(&rpmsg_pru_lock);
+	prudev->locked = false;
+	mutex_unlock(&rpmsg_pru_lock);
+	return 0;
+}
+
+static ssize_t rpmsg_pru_read(struct file *filp, char __user *buf,
+			      size_t count, loff_t *f_pos)
+{
+	int ret;
+	u32 length;
+	struct rpmsg_pru_dev *prudev;
+
+	prudev = filp->private_data;
+
+	if (kfifo_is_empty(&prudev->msg_fifo) &&
+	    (filp->f_flags & O_NONBLOCK))
+		return -EAGAIN;
+
+	ret = wait_event_interruptible(prudev->wait_list,
+				       !kfifo_is_empty(&prudev->msg_fifo));
+	if (ret)
+		return -EINTR;
+
+	ret = kfifo_to_user(&prudev->msg_fifo, buf,
+			    prudev->msg_len[prudev->msg_idx_rd], &length);
+	prudev->msg_idx_rd = (prudev->msg_idx_rd + 1) % MAX_FIFO_MSG;
+
+	return ret ? ret : length;
+}
+
+static ssize_t rpmsg_pru_write(struct file *filp, const char __user *buf,
+			       size_t count, loff_t *f_pos)
+{
+	int ret;
+	struct rpmsg_pru_dev *prudev;
+	static char rpmsg_pru_buf[RPMSG_BUF_SIZE];
+
+	prudev = filp->private_data;
+
+	if (count > RPMSG_BUF_SIZE - sizeof(struct rpmsg_hdr)) {
+		dev_err(prudev->dev, "Data too large for RPMsg Buffer\n");
+		return -EINVAL;
+	}
+
+	if (copy_from_user(rpmsg_pru_buf, buf, count)) {
+		dev_err(prudev->dev, "Error copying buffer from user space");
+		return -EFAULT;
+	}
+
+	ret = rpmsg_send(prudev->rpdev->ept, (void *)rpmsg_pru_buf, count);
+	if (ret)
+		dev_err(prudev->dev, "rpmsg_send failed: %d\n", ret);
+
+	return ret ? ret : count;
+}
+
+static unsigned int rpmsg_pru_poll(struct file *filp,
+				   struct poll_table_struct *wait)
+{
+	int mask;
+	struct rpmsg_pru_dev *prudev;
+
+	prudev = filp->private_data;
+
+	poll_wait(filp, &prudev->wait_list, wait);
+
+	mask = POLLOUT | POLLWRNORM;
+
+	if (!kfifo_is_empty(&prudev->msg_fifo))
+		mask |= POLLIN | POLLRDNORM;
+
+	return mask;
+}
+
+static const struct file_operations rpmsg_pru_fops = {
+	.owner = THIS_MODULE,
+	.open = rpmsg_pru_open,
+	.release = rpmsg_pru_release,
+	.read = rpmsg_pru_read,
+	.write = rpmsg_pru_write,
+	.poll = rpmsg_pru_poll,
+};
+
+static int rpmsg_pru_cb(struct rpmsg_device *rpdev, void *data, int len,
+			void *priv, u32 src)
+{
+	u32 length;
+	struct rpmsg_pru_dev *prudev;
+
+	prudev = dev_get_drvdata(&rpdev->dev);
+
+	if (kfifo_avail(&prudev->msg_fifo) < len) {
+		dev_err(&rpdev->dev, "Not enough space on the FIFO\n");
+		return -ENOSPC;
+	}
+
+	if ((prudev->msg_idx_wr + 1) % MAX_FIFO_MSG ==
+		prudev->msg_idx_rd) {
+		dev_err(&rpdev->dev, "Message length table is full\n");
+		return -ENOSPC;
+	}
+
+	length = kfifo_in(&prudev->msg_fifo, data, len);
+	prudev->msg_len[prudev->msg_idx_wr] = length;
+	prudev->msg_idx_wr = (prudev->msg_idx_wr + 1) % MAX_FIFO_MSG;
+
+	wake_up_interruptible(&prudev->wait_list);
+
+	return 0;
+}
+
+static int rpmsg_pru_probe(struct rpmsg_device *rpdev)
+{
+	int ret;
+	struct rpmsg_pru_dev *prudev;
+	int minor_got;
+
+	prudev = devm_kzalloc(&rpdev->dev, sizeof(*prudev), GFP_KERNEL);
+	if (!prudev)
+		return -ENOMEM;
+
+	mutex_lock(&rpmsg_pru_lock);
+	minor_got = idr_alloc(&rpmsg_pru_minors, prudev, 0, PRU_MAX_DEVICES,
+			      GFP_KERNEL);
+	mutex_unlock(&rpmsg_pru_lock);
+	if (minor_got < 0) {
+		ret = minor_got;
+		dev_err(&rpdev->dev, "Failed to get a minor number for the rpmsg_pru device: %d\n",
+			ret);
+		goto fail_alloc_minor;
+	}
+
+	prudev->devt = MKDEV(MAJOR(rpmsg_pru_devt), minor_got);
+
+	cdev_init(&prudev->cdev, &rpmsg_pru_fops);
+	prudev->cdev.owner = THIS_MODULE;
+	ret = cdev_add(&prudev->cdev, prudev->devt, 1);
+	if (ret) {
+		dev_err(&rpdev->dev, "Unable to add cdev for the rpmsg_pru device\n");
+		goto fail_add_cdev;
+	}
+
+	prudev->dev = device_create(rpmsg_pru_class, &rpdev->dev, prudev->devt,
+				    NULL, "rpmsg_pru%d", rpdev->dst);
+	if (IS_ERR(prudev->dev)) {
+		dev_err(&rpdev->dev, "Unable to create the rpmsg_pru device\n");
+		ret = PTR_ERR(prudev->dev);
+		goto fail_create_device;
+	}
+
+	prudev->rpdev = rpdev;
+
+	ret = kfifo_alloc(&prudev->msg_fifo, MAX_FIFO_MSG * FIFO_MSG_SIZE,
+			  GFP_KERNEL);
+	if (ret) {
+		dev_err(&rpdev->dev, "Unable to allocate fifo for the rpmsg_pru device\n");
+		goto fail_alloc_fifo;
+	}
+
+	init_waitqueue_head(&prudev->wait_list);
+
+	dev_set_drvdata(&rpdev->dev, prudev);
+
+	dev_info(&rpdev->dev, "new rpmsg_pru device: /dev/rpmsg_pru%d",
+		 rpdev->dst);
+
+	return 0;
+
+fail_alloc_fifo:
+	device_destroy(rpmsg_pru_class, prudev->devt);
+fail_create_device:
+	cdev_del(&prudev->cdev);
+fail_add_cdev:
+	mutex_lock(&rpmsg_pru_lock);
+	idr_remove(&rpmsg_pru_minors, minor_got);
+	mutex_unlock(&rpmsg_pru_lock);
+fail_alloc_minor:
+	return ret;
+}
+
+static void rpmsg_pru_remove(struct rpmsg_device *rpdev)
+{
+	struct rpmsg_pru_dev *prudev;
+
+	prudev = dev_get_drvdata(&rpdev->dev);
+
+	kfifo_free(&prudev->msg_fifo);
+	device_destroy(rpmsg_pru_class, prudev->devt);
+	cdev_del(&prudev->cdev);
+	mutex_lock(&rpmsg_pru_lock);
+	idr_remove(&rpmsg_pru_minors, MINOR(prudev->devt));
+	mutex_unlock(&rpmsg_pru_lock);
+}
+
+/* .name matches on RPMsg Channels and causes a probe */
+static const struct rpmsg_device_id rpmsg_driver_pru_id_table[] = {
+	{ .name	= "rpmsg-pru" },
+	{ },
+};
+MODULE_DEVICE_TABLE(rpmsg, rpmsg_driver_pru_id_table);
+
+static struct rpmsg_driver rpmsg_pru_driver = {
+	.drv.name	= KBUILD_MODNAME,
+	.id_table	= rpmsg_driver_pru_id_table,
+	.probe		= rpmsg_pru_probe,
+	.callback	= rpmsg_pru_cb,
+	.remove		= rpmsg_pru_remove,
+};
+
+static int __init rpmsg_pru_init(void)
+{
+	int ret;
+
+	rpmsg_pru_class = class_create(THIS_MODULE, "rpmsg_pru");
+	if (IS_ERR(rpmsg_pru_class)) {
+		pr_err("Unable to create class\n");
+		ret = PTR_ERR(rpmsg_pru_class);
+		goto fail_create_class;
+	}
+
+	ret = alloc_chrdev_region(&rpmsg_pru_devt, 0, PRU_MAX_DEVICES,
+				  "rpmsg_pru");
+	if (ret) {
+		pr_err("Unable to allocate chrdev region\n");
+		goto fail_alloc_region;
+	}
+
+	ret = register_rpmsg_driver(&rpmsg_pru_driver);
+	if (ret) {
+		pr_err("Unable to register rpmsg driver");
+		goto fail_register_rpmsg_driver;
+	}
+
+	return 0;
+
+fail_register_rpmsg_driver:
+	unregister_chrdev_region(rpmsg_pru_devt, PRU_MAX_DEVICES);
+fail_alloc_region:
+	class_destroy(rpmsg_pru_class);
+fail_create_class:
+	return ret;
+}
+
+static void __exit rpmsg_pru_exit(void)
+{
+	unregister_rpmsg_driver(&rpmsg_pru_driver);
+	idr_destroy(&rpmsg_pru_minors);
+	mutex_destroy(&rpmsg_pru_lock);
+	class_destroy(rpmsg_pru_class);
+	unregister_chrdev_region(rpmsg_pru_devt, PRU_MAX_DEVICES);
+}
+
+module_init(rpmsg_pru_init);
+module_exit(rpmsg_pru_exit);
+
+MODULE_AUTHOR("Jason Reeder <jreeder@ti.com>");
+MODULE_DESCRIPTION("PRU Remote Processor Messaging Driver");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/rpmsg/rpmsg_rpc.c b/drivers/rpmsg/rpmsg_rpc.c
--- a/drivers/rpmsg/rpmsg_rpc.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/rpmsg/rpmsg_rpc.c	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,1409 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Remote Processor Procedure Call Driver
+ *
+ * Copyright (C) 2012-2019 Texas Instruments Incorporated - http://www.ti.com/
+ *	Erik Rainey <erik.rainey@ti.com>
+ *	Suman Anna <s-anna@ti.com>
+ */
+
+#define pr_fmt(fmt) "%s: " fmt, __func__
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/idr.h>
+#include <linux/poll.h>
+#include <linux/mutex.h>
+#include <linux/sched.h>
+#include <linux/fdtable.h>
+#include <linux/remoteproc.h>
+#include <linux/rpmsg.h>
+#include <linux/rpmsg_rpc.h>
+#include <linux/rpmsg/virtio_rpmsg.h>
+#include <linux/sched/signal.h>
+
+#include "rpmsg_rpc_internal.h"
+
+#define RPPC_MAX_DEVICES	(8)
+#define RPPC_MAX_REG_FDS	(10)
+
+#define RPPC_SIG_NUM_PARAM(sig) ((sig).num_param - 1)
+
+/* TODO: remove these fields */
+#define RPPC_JOBID_DISCRETE	(0)
+#define RPPC_POOLID_DEFAULT	(0x8000)
+
+static struct class *rppc_class;
+static dev_t rppc_dev;
+
+/* store all remote rpc connection services (usually one per remoteproc) */
+static DEFINE_IDR(rppc_devices);
+static DEFINE_MUTEX(rppc_devices_lock);
+
+/*
+ * Retrieve the rproc instance so that it can be used for performing
+ * address translations
+ */
+static inline struct rproc *rpdev_to_rproc(struct rpmsg_device *rpdev)
+{
+	return rproc_get_by_child(&rpdev->dev);
+}
+
+/*
+ * A wrapper function to translate local physical addresses to the remote core
+ * device addresses (virtual addresses that a code on remote processor can use
+ * directly.
+ *
+ * XXX: Fix this to return negative values on errors to follow normal kernel
+ *      conventions, and since 0 can also be a valid remote processor address
+ *
+ * Returns a remote processor device address on success, 0 otherwise
+ */
+dev_addr_t rppc_local_to_remote_da(struct rppc_instance *rpc, phys_addr_t pa)
+{
+	int ret;
+	struct rproc *rproc;
+	u64 da = 0;
+	dev_addr_t rda;
+	struct device *dev = rpc->dev;
+
+	if (mutex_lock_interruptible(&rpc->rppcdev->lock))
+		return 0;
+
+	rproc = rpdev_to_rproc(rpc->rppcdev->rpdev);
+	if (!rproc) {
+		dev_err(dev, "error getting rproc for rpdev 0x%x\n",
+			(u32)rpc->rppcdev->rpdev);
+	} else {
+		ret = rproc_pa_to_da(rproc, pa, &da);
+		if (ret) {
+			dev_err(dev, "error from rproc_pa_to_da, rproc = %p, pa = %pa ret = %d\n",
+				rproc, &pa, ret);
+		}
+	}
+	rda = (dev_addr_t)da;
+
+	mutex_unlock(&rpc->rppcdev->lock);
+
+	return rda;
+}
+
+static void rppc_print_msg(struct rppc_instance *rpc, char *prefix,
+			   char buffer[512])
+{
+	struct rppc_msg_header *hdr = (struct rppc_msg_header *)buffer;
+	struct rppc_instance_handle *hdl = NULL;
+	struct rppc_query_function *info = NULL;
+	struct rppc_packet *packet = NULL;
+	struct rppc_param_data *param = NULL;
+	struct device *dev = rpc->dev;
+	u32 i = 0, paramsz = sizeof(*param);
+
+	dev_dbg(dev, "%s HDR: msg_type = %d msg_len = %d\n",
+		prefix, hdr->msg_type, hdr->msg_len);
+
+	switch (hdr->msg_type) {
+	case RPPC_MSGTYPE_CREATE_RESP:
+	case RPPC_MSGTYPE_DELETE_RESP:
+		hdl = RPPC_PAYLOAD(buffer, rppc_instance_handle);
+		dev_dbg(dev, "%s endpoint = %d status = %d\n",
+			prefix, hdl->endpoint_address, hdl->status);
+		break;
+	case RPPC_MSGTYPE_FUNCTION_INFO:
+		info = RPPC_PAYLOAD(buffer, rppc_query_function);
+		dev_dbg(dev, "%s (info not yet implemented)\n", prefix);
+		break;
+	case RPPC_MSGTYPE_FUNCTION_CALL:
+		packet = RPPC_PAYLOAD(buffer, rppc_packet);
+		dev_dbg(dev, "%s PACKET: desc = %04x msg_id = %04x flags = %08x func = 0x%08x result = %d size = %u\n",
+			prefix, packet->desc, packet->msg_id,
+			packet->flags, packet->fxn_id,
+			packet->result, packet->data_size);
+		param = (struct rppc_param_data *)packet->data;
+		for (i = 0; i < (packet->data_size / paramsz); i++) {
+			dev_dbg(dev, "%s param[%u] size = %zu data = %zu (0x%08x)",
+				prefix, i, param[i].size, param[i].data,
+				param[i].data);
+		}
+		break;
+	default:
+		break;
+	}
+}
+
+/* free any outstanding function calls */
+static void rppc_delete_fxns(struct rppc_instance *rpc)
+{
+	struct rppc_function_list *pos, *n;
+
+	if (!list_empty(&rpc->fxn_list)) {
+		mutex_lock(&rpc->lock);
+		list_for_each_entry_safe(pos, n, &rpc->fxn_list, list) {
+			list_del(&pos->list);
+			kfree(pos->function);
+			kfree(pos);
+		}
+		mutex_unlock(&rpc->lock);
+	}
+}
+
+static
+struct rppc_function *rppc_find_fxn(struct rppc_instance *rpc, u16 msg_id)
+{
+	struct rppc_function *function = NULL;
+	struct rppc_function_list *pos, *n;
+	struct device *dev = rpc->dev;
+
+	mutex_lock(&rpc->lock);
+	list_for_each_entry_safe(pos, n, &rpc->fxn_list, list) {
+		dev_dbg(dev, "looking for msg %u, found msg %u\n",
+			msg_id, pos->msg_id);
+		if (pos->msg_id == msg_id) {
+			function = pos->function;
+			list_del(&pos->list);
+			kfree(pos);
+			break;
+		}
+	}
+	mutex_unlock(&rpc->lock);
+
+	return function;
+}
+
+static int rppc_add_fxn(struct rppc_instance *rpc,
+			struct rppc_function *function, u16 msg_id)
+{
+	struct rppc_function_list *fxn = NULL;
+	struct device *dev = rpc->dev;
+
+	fxn = kzalloc(sizeof(*fxn), GFP_KERNEL);
+	if (!fxn)
+		return -ENOMEM;
+
+	fxn->function = function;
+	fxn->msg_id = msg_id;
+	mutex_lock(&rpc->lock);
+	list_add(&fxn->list, &rpc->fxn_list);
+	mutex_unlock(&rpc->lock);
+	dev_dbg(dev, "added msg id %u to list", msg_id);
+
+	return 0;
+}
+
+static
+void rppc_handle_create_resp(struct rppc_instance *rpc, char *data, int len)
+{
+	struct device *dev = rpc->dev;
+	struct rppc_msg_header *hdr = (struct rppc_msg_header *)data;
+	struct rppc_instance_handle *hdl;
+	u32 exp_len = sizeof(*hdl) + sizeof(*hdr);
+
+	if (len != exp_len) {
+		dev_err(dev, "invalid response message length %d (expected %d bytes)",
+			len, exp_len);
+		rpc->state = RPPC_STATE_STALE;
+		return;
+	}
+
+	hdl = RPPC_PAYLOAD(data, rppc_instance_handle);
+
+	mutex_lock(&rpc->lock);
+	if (rpc->state != RPPC_STATE_STALE && hdl->status == 0) {
+		rpc->dst = hdl->endpoint_address;
+		rpc->state = RPPC_STATE_CONNECTED;
+	} else {
+		rpc->state = RPPC_STATE_STALE;
+	}
+	rpc->in_transition = 0;
+	dev_dbg(dev, "creation response: status %d addr 0x%x\n",
+		hdl->status, hdl->endpoint_address);
+
+	complete(&rpc->reply_arrived);
+	mutex_unlock(&rpc->lock);
+}
+
+static
+void rppc_handle_delete_resp(struct rppc_instance *rpc, char *data, int len)
+{
+	struct device *dev = rpc->dev;
+	struct rppc_msg_header *hdr = (struct rppc_msg_header *)data;
+	struct rppc_instance_handle *hdl;
+	u32 exp_len = sizeof(*hdl) + sizeof(*hdr);
+
+	if (len != exp_len) {
+		dev_err(dev, "invalid response message length %d (expected %d bytes)",
+			len, exp_len);
+		rpc->state = RPPC_STATE_STALE;
+		return;
+	}
+	if (hdr->msg_len != sizeof(*hdl)) {
+		dev_err(dev, "disconnect message was incorrect size!\n");
+		rpc->state = RPPC_STATE_STALE;
+		return;
+	}
+
+	hdl = RPPC_PAYLOAD(data, rppc_instance_handle);
+	dev_dbg(dev, "deletion response: status %d addr 0x%x\n",
+		hdl->status, hdl->endpoint_address);
+	mutex_lock(&rpc->lock);
+	rpc->dst = 0;
+	rpc->state = RPPC_STATE_DISCONNECTED;
+	rpc->in_transition = 0;
+	complete(&rpc->reply_arrived);
+	mutex_unlock(&rpc->lock);
+}
+
+/*
+ * store the received message and wake up any blocking processes,
+ * waiting for new data. The allocated buffer would be freed after
+ * the user-space reads the packet.
+ */
+static void rppc_handle_fxn_resp(struct rppc_instance *rpc, char *data, int len)
+{
+	struct rppc_msg_header *hdr = (struct rppc_msg_header *)data;
+	struct sk_buff *skb;
+	char *skbdata;
+
+	/* TODO: need to check the response length? */
+	skb = alloc_skb(hdr->msg_len, GFP_KERNEL);
+	if (!skb)
+		return;
+	skbdata = skb_put(skb, hdr->msg_len);
+	memcpy(skbdata, hdr->msg_data, hdr->msg_len);
+
+	mutex_lock(&rpc->lock);
+	skb_queue_tail(&rpc->queue, skb);
+	mutex_unlock(&rpc->lock);
+
+	wake_up_interruptible(&rpc->readq);
+}
+
+/*
+ * callback function for processing the different responses
+ * from the remote processor on a particular rpmsg channel
+ * instance.
+ */
+static int rppc_cb(struct rpmsg_device *rpdev,
+		   void *data, int len, void *priv, u32 src)
+{
+	struct rppc_msg_header *hdr = data;
+	struct rppc_instance *rpc = priv;
+	struct device *dev = rpc->dev;
+	char *buf = (char *)data;
+
+	dev_dbg(dev, "<== incoming msg src %d len %d msg_type %d msg_len %d\n",
+		src, len, hdr->msg_type, hdr->msg_len);
+	rppc_print_msg(rpc, "RX:", buf);
+
+	if (len <= sizeof(*hdr)) {
+		dev_err(dev, "message truncated\n");
+		rpc->state = RPPC_STATE_STALE;
+		return -EINVAL;
+	}
+
+	switch (hdr->msg_type) {
+	case RPPC_MSGTYPE_CREATE_RESP:
+		rppc_handle_create_resp(rpc, data, len);
+		break;
+	case RPPC_MSGTYPE_DELETE_RESP:
+		rppc_handle_delete_resp(rpc, data, len);
+		break;
+	case RPPC_MSGTYPE_FUNCTION_CALL:
+	case RPPC_MSGTYPE_FUNCTION_RET:
+		rppc_handle_fxn_resp(rpc, data, len);
+		break;
+	default:
+		dev_warn(dev, "unexpected msg type: %d\n", hdr->msg_type);
+		break;
+	}
+
+	return 0;
+}
+
+/*
+ * send a connection request to the remote rpc connection service. Use
+ * the new local address created during .open for this instance as the
+ * source address to complete the connection.
+ */
+static int rppc_connect(struct rppc_instance *rpc,
+			struct rppc_create_instance *connect)
+{
+	int ret = 0;
+	u32 len = 0;
+	char kbuf[512];
+	struct rppc_device *rppcdev = rpc->rppcdev;
+	struct rppc_msg_header *hdr = (struct rppc_msg_header *)&kbuf[0];
+
+	if (rpc->state == RPPC_STATE_CONNECTED) {
+		dev_dbg(rpc->dev, "endpoint already connected\n");
+		return -EISCONN;
+	}
+
+	hdr->msg_type = RPPC_MSGTYPE_CREATE_REQ;
+	hdr->msg_len = sizeof(*connect);
+	memcpy(hdr->msg_data, connect, hdr->msg_len);
+	len = sizeof(struct rppc_msg_header) + hdr->msg_len;
+
+	init_completion(&rpc->reply_arrived);
+	rpc->in_transition = 1;
+	ret = rpmsg_send_offchannel(rppcdev->rpdev->ept, rpc->ept->addr,
+				    rppcdev->rpdev->dst, (char *)kbuf, len);
+	if (ret > 0) {
+		dev_err(rpc->dev, "rpmsg_send failed: %d\n", ret);
+		return ret;
+	}
+
+	ret = wait_for_completion_interruptible_timeout(&rpc->reply_arrived,
+							msecs_to_jiffies(5000));
+	if (rpc->state == RPPC_STATE_CONNECTED)
+		return 0;
+
+	if (rpc->state == RPPC_STATE_STALE)
+		return -ENXIO;
+
+	if (ret > 0) {
+		dev_err(rpc->dev, "premature wakeup: %d\n", ret);
+		return -EIO;
+	}
+
+	return -ETIMEDOUT;
+}
+
+static void rppc_disconnect(struct rppc_instance *rpc)
+{
+	int ret;
+	size_t len;
+	char kbuf[512];
+	struct rppc_device *rppcdev = rpc->rppcdev;
+	struct rppc_msg_header *hdr = (struct rppc_msg_header *)&kbuf[0];
+	struct rppc_instance_handle *handle =
+				RPPC_PAYLOAD(kbuf, rppc_instance_handle);
+
+	if (rpc->state != RPPC_STATE_CONNECTED)
+		return;
+
+	hdr->msg_type = RPPC_MSGTYPE_DELETE_REQ;
+	hdr->msg_len = sizeof(u32);
+	handle->endpoint_address = rpc->dst;
+	handle->status = 0;
+	len = sizeof(struct rppc_msg_header) + hdr->msg_len;
+
+	dev_dbg(rpc->dev, "disconnecting from RPC service at %d\n",
+		rpc->dst);
+	ret = rpmsg_send_offchannel(rppcdev->rpdev->ept, rpc->ept->addr,
+				    rppcdev->rpdev->dst, kbuf, len);
+	if (ret)
+		dev_err(rpc->dev, "rpmsg_send failed: %d\n", ret);
+
+	/*
+	 * TODO: should we wait for a message to come back?
+	 * For now, no.
+	 */
+	wait_for_completion_interruptible(&rpc->reply_arrived);
+}
+
+static int rppc_register_buffers(struct rppc_instance *rpc,
+				 unsigned long arg)
+{
+	struct rppc_buf_fds data;
+	int *fds = NULL;
+	struct rppc_dma_buf **bufs = NULL;
+	struct rppc_dma_buf *tmp;
+	int i = 0, ret = 0;
+
+	if (copy_from_user(&data, (char __user *)arg, sizeof(data)))
+		return -EFAULT;
+
+	/* impose a maximum number of buffers for now */
+	if (data.num > RPPC_MAX_REG_FDS)
+		return -EINVAL;
+
+	fds = kcalloc(data.num, sizeof(*fds), GFP_KERNEL);
+	if (!fds)
+		return -ENOMEM;
+
+	if (copy_from_user(fds, (char __user *)data.fds,
+			   sizeof(*fds) * data.num)) {
+		ret = -EFAULT;
+		goto free_fds;
+	}
+
+	for (i = 0; i < data.num; i++) {
+		rcu_read_lock();
+		if (!fcheck(fds[i])) {
+			rcu_read_unlock();
+			ret = -EBADF;
+			goto free_fds;
+		}
+		rcu_read_unlock();
+
+		tmp = rppc_find_dmabuf(rpc, fds[i]);
+		if (!IS_ERR_OR_NULL(tmp)) {
+			ret = -EEXIST;
+			goto free_fds;
+		}
+	}
+
+	bufs = kcalloc(data.num, sizeof(*bufs), GFP_KERNEL);
+	if (!bufs) {
+		ret = -ENOMEM;
+		goto free_fds;
+	}
+
+	for (i = 0; i < data.num; i++) {
+		bufs[i] = rppc_alloc_dmabuf(rpc, fds[i], false);
+		if (IS_ERR(bufs[i])) {
+			ret = PTR_ERR(bufs[i]);
+			break;
+		}
+	}
+	if (i == data.num)
+		goto free_bufs;
+
+	for (i -= 1; i >= 0; i--)
+		rppc_free_dmabuf(bufs[i]->id, bufs[i], rpc);
+
+free_bufs:
+	kfree(bufs);
+free_fds:
+	kfree(fds);
+	return ret;
+}
+
+static int rppc_unregister_buffers(struct rppc_instance *rpc,
+				   unsigned long arg)
+{
+	struct rppc_buf_fds data;
+	int *fds = NULL;
+	struct rppc_dma_buf **bufs = NULL;
+	int i = 0, ret = 0;
+
+	if (copy_from_user(&data, (char __user *)arg, sizeof(data)))
+		return -EFAULT;
+
+	/* impose a maximum number of buffers for now */
+	if (data.num > RPPC_MAX_REG_FDS)
+		return -EINVAL;
+
+	fds = kcalloc(data.num, sizeof(*fds), GFP_KERNEL);
+	if (!fds)
+		return -ENOMEM;
+
+	if (copy_from_user(fds, (char __user *)data.fds,
+			   sizeof(*fds) * data.num)) {
+		ret = -EFAULT;
+		goto free_fds;
+	}
+
+	bufs = kcalloc(data.num, sizeof(*bufs), GFP_KERNEL);
+	if (!bufs) {
+		ret = -ENOMEM;
+		goto free_fds;
+	}
+
+	for (i = 0; i < data.num; i++) {
+		rcu_read_lock();
+		if (!fcheck(fds[i])) {
+			rcu_read_unlock();
+			ret = -EBADF;
+			goto free_bufs;
+		}
+		rcu_read_unlock();
+
+		bufs[i] = rppc_find_dmabuf(rpc, fds[i]);
+		if (IS_ERR_OR_NULL(bufs[i])) {
+			ret = -EEXIST;
+			goto free_bufs;
+		}
+	}
+
+	for (i = 0; i < data.num; i++)
+		rppc_free_dmabuf(bufs[i]->id, bufs[i], rpc);
+
+free_bufs:
+	kfree(bufs);
+free_fds:
+	kfree(fds);
+	return ret;
+}
+
+/*
+ * create a new rpc instance that a user-space client can use to invoke
+ * remote functions. A new local address would be created and tied with
+ * this instance for uniquely identifying the messages communicated by
+ * this instance with the remote side.
+ *
+ * The function is blocking if there is no underlying connection manager
+ * channel, unless the device is opened with non-blocking flags specifically.
+ */
+static int rppc_open(struct inode *inode, struct file *filp)
+{
+	struct rppc_device *rppcdev;
+	struct rppc_instance *rpc;
+	struct rpmsg_channel_info chinfo = {};
+
+	rppcdev = container_of(inode->i_cdev, struct rppc_device, cdev);
+
+	if (!rppcdev->rpdev)
+		if ((filp->f_flags & O_NONBLOCK) ||
+		    wait_for_completion_interruptible(&rppcdev->comp))
+			return -EBUSY;
+
+	rpc = kzalloc(sizeof(*rpc), GFP_KERNEL);
+	if (!rpc)
+		return -ENOMEM;
+
+	mutex_init(&rpc->lock);
+	skb_queue_head_init(&rpc->queue);
+	init_waitqueue_head(&rpc->readq);
+	INIT_LIST_HEAD(&rpc->fxn_list);
+	idr_init(&rpc->dma_idr);
+	rpc->in_transition = 0;
+	rpc->msg_id = 0;
+	rpc->state = RPPC_STATE_DISCONNECTED;
+	rpc->rppcdev = rppcdev;
+
+	rpc->dev = get_device(rppcdev->dev);
+	chinfo.src = RPMSG_ADDR_ANY;
+	chinfo.dst = RPMSG_ADDR_ANY;
+	rpc->ept = rpmsg_create_ept(rppcdev->rpdev, rppc_cb, rpc, chinfo);
+	if (!rpc->ept) {
+		dev_err(rpc->dev, "create ept failed\n");
+		put_device(rpc->dev);
+		kfree(rpc);
+		return -ENOMEM;
+	}
+	filp->private_data = rpc;
+
+	mutex_lock(&rppcdev->lock);
+	list_add(&rpc->list, &rppcdev->instances);
+	mutex_unlock(&rppcdev->lock);
+
+	dev_dbg(rpc->dev, "local addr assigned: 0x%x\n", rpc->ept->addr);
+
+	return 0;
+}
+
+/*
+ * release and free all the resources associated with a particular rpc
+ * instance. This includes the data structures maintaining the current
+ * outstanding function invocations, and all the buffers registered for
+ * use with this instance. Send a disconnect message and cleanup the
+ * local end-point only if the instance is in a normal state, with the
+ * remote connection manager functional.
+ */
+static int rppc_release(struct inode *inode, struct file *filp)
+{
+	struct rppc_instance *rpc = filp->private_data;
+	struct rppc_device *rppcdev = rpc->rppcdev;
+	struct sk_buff *skb = NULL;
+
+	dev_dbg(rpc->dev, "releasing Instance %p, in state %d\n", rpc,
+		rpc->state);
+
+	if (rpc->state != RPPC_STATE_STALE) {
+		if (rpc->ept) {
+			rppc_disconnect(rpc);
+			rpmsg_destroy_ept(rpc->ept);
+			rpc->ept = NULL;
+		}
+	}
+
+	rppc_delete_fxns(rpc);
+
+	while (!skb_queue_empty(&rpc->queue)) {
+		skb = skb_dequeue(&rpc->queue);
+		kfree_skb(skb);
+	}
+
+	mutex_lock(&rpc->lock);
+	idr_for_each(&rpc->dma_idr, rppc_free_dmabuf, rpc);
+	idr_destroy(&rpc->dma_idr);
+	mutex_unlock(&rpc->lock);
+
+	mutex_lock(&rppcdev->lock);
+	list_del(&rpc->list);
+	mutex_unlock(&rppcdev->lock);
+
+	dev_dbg(rpc->dev, "instance %p has been deleted!\n", rpc);
+	if (list_empty(&rppcdev->instances))
+		dev_dbg(rpc->dev, "all instances have been removed!\n");
+
+	put_device(rpc->dev);
+	kfree(rpc);
+	return 0;
+}
+
+static long rppc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
+{
+	struct rppc_instance *rpc = filp->private_data;
+	struct rppc_create_instance connect;
+	int ret = 0;
+
+	dev_dbg(rpc->dev, "%s: cmd %d, arg 0x%lx\n", __func__, cmd, arg);
+
+	if (_IOC_TYPE(cmd) != RPPC_IOC_MAGIC)
+		return -ENOTTY;
+
+	if (_IOC_NR(cmd) > RPPC_IOC_MAXNR)
+		return -ENOTTY;
+
+	switch (cmd) {
+	case RPPC_IOC_CREATE:
+		ret = copy_from_user(&connect, (char __user *)arg,
+				     sizeof(connect));
+		if (ret) {
+			dev_err(rpc->dev, "%s: %d: copy_from_user fail: %d\n",
+				__func__, _IOC_NR(cmd), ret);
+			ret = -EFAULT;
+		} else {
+			connect.name[sizeof(connect.name) - 1] = '\0';
+			ret = rppc_connect(rpc, &connect);
+		}
+		break;
+	case RPPC_IOC_BUFREGISTER:
+		ret = rppc_register_buffers(rpc, arg);
+		break;
+	case RPPC_IOC_BUFUNREGISTER:
+		ret = rppc_unregister_buffers(rpc, arg);
+		break;
+	default:
+		dev_err(rpc->dev, "unhandled ioctl cmd: %d\n", cmd);
+		break;
+	}
+
+	return ret;
+}
+
+static ssize_t rppc_read(struct file *filp, char __user *buf, size_t len,
+			 loff_t *offp)
+{
+	struct rppc_instance *rpc = filp->private_data;
+	struct rppc_packet *packet = NULL;
+	struct rppc_param_data *parameters = NULL;
+	struct rppc_function *function = NULL;
+	struct rppc_function_return returned;
+	struct sk_buff *skb = NULL;
+	int ret = 0;
+	int use = sizeof(returned);
+	DEFINE_WAIT(wait);
+
+	if (mutex_lock_interruptible(&rpc->lock))
+		return -ERESTARTSYS;
+
+	/* instance is invalid */
+	if (rpc->state == RPPC_STATE_STALE) {
+		mutex_unlock(&rpc->lock);
+		return -ENXIO;
+	}
+
+	/* not yet connected to the remote side */
+	if (rpc->state == RPPC_STATE_DISCONNECTED) {
+		mutex_unlock(&rpc->lock);
+		return -ENOTCONN;
+	}
+
+	if (len > use) {
+		mutex_unlock(&rpc->lock);
+		return -EOVERFLOW;
+	}
+	if (len < use) {
+		mutex_unlock(&rpc->lock);
+		return -EINVAL;
+	}
+
+	/* TODO: Use the much simpler wait_event_interruptible API */
+	while (skb_queue_empty(&rpc->queue)) {
+		mutex_unlock(&rpc->lock);
+		/* non-blocking requested ? return now */
+		if (filp->f_flags & O_NONBLOCK)
+			return -EAGAIN;
+
+		prepare_to_wait_exclusive(&rpc->readq, &wait,
+					  TASK_INTERRUPTIBLE);
+		if (skb_queue_empty(&rpc->queue) &&
+		    rpc->state != RPPC_STATE_STALE)
+			schedule();
+		finish_wait(&rpc->readq, &wait);
+		if (signal_pending(current))
+			return -ERESTARTSYS;
+
+		ret = mutex_lock_interruptible(&rpc->lock);
+		if (ret < 0)
+			return -ERESTARTSYS;
+
+		if (rpc->state == RPPC_STATE_STALE) {
+			mutex_unlock(&rpc->lock);
+			return -ENXIO;
+		}
+
+		/* make sure state is sane while we waited */
+		if (rpc->state != RPPC_STATE_CONNECTED) {
+			mutex_unlock(&rpc->lock);
+			ret = -EIO;
+			goto out;
+		}
+	}
+
+	skb = skb_dequeue(&rpc->queue);
+	if (WARN_ON(!skb)) {
+		mutex_unlock(&rpc->lock);
+		ret = -EIO;
+		goto out;
+	}
+
+	mutex_unlock(&rpc->lock);
+
+	packet = (struct rppc_packet *)skb->data;
+	parameters = (struct rppc_param_data *)packet->data;
+
+	/*
+	 * pull the function memory from the list and untranslate
+	 * the remote device address pointers in the packet back
+	 * to MPU pointers.
+	 */
+	function = rppc_find_fxn(rpc, packet->msg_id);
+	if (function && function->num_translations > 0) {
+		ret = rppc_xlate_buffers(rpc, function, RPPC_RPA_TO_UVA);
+		if (ret < 0) {
+			dev_err(rpc->dev, "failed to translate back pointers from remote core!\n");
+			goto failure;
+		}
+	}
+	returned.fxn_id = RPPC_FXN_MASK(packet->fxn_id);
+	returned.status = packet->result;
+
+	if (copy_to_user(buf, &returned, use)) {
+		dev_err(rpc->dev, "%s: copy_to_user fail\n", __func__);
+		ret = -EFAULT;
+	} else {
+		ret = use;
+	}
+
+failure:
+	kfree(function);
+	kfree_skb(skb);
+out:
+	return ret;
+}
+
+static ssize_t rppc_write(struct file *filp, const char __user *ubuf,
+			  size_t len, loff_t *offp)
+{
+	struct rppc_instance *rpc = filp->private_data;
+	struct rppc_device *rppcdev = rpc->rppcdev;
+	struct device *dev = rpc->dev;
+	struct rppc_msg_header *hdr = NULL;
+	struct rppc_function *function = NULL;
+	struct rppc_packet *packet = NULL;
+	struct rppc_param_data *parameters = NULL;
+	char kbuf[512];
+	int use = 0, ret = 0, param = 0;
+	u32 sig_idx = 0;
+	u32 sig_prm = 0;
+	static u32 rppc_atomic_size[RPPC_PARAM_ATOMIC_MAX] = {
+		0, /* RPPC_PARAM_VOID */
+		1, /* RPPC_PARAM_S08 */
+		1, /* RPPC_PARAM_U08 */
+		2, /* RPPC_PARAM_S16 */
+		2, /* RPPC_PARAM_U16 */
+		4, /* RPPC_PARAM_S32 */
+		4, /* RPPC_PARAM_U32 */
+		8, /* RPPC_PARAM_S64 */
+		8  /* RPPC_PARAM_U64 */
+	};
+
+	if (len < sizeof(*function)) {
+		ret = -ENOTSUPP;
+		goto failure;
+	}
+
+	if (len > (sizeof(*function) + RPPC_MAX_TRANSLATIONS *
+				sizeof(struct rppc_param_translation))) {
+		ret = -ENOTSUPP;
+		goto failure;
+	}
+
+	if (rpc->state != RPPC_STATE_CONNECTED) {
+		ret = -ENOTCONN;
+		goto failure;
+	}
+
+	function = kzalloc(len, GFP_KERNEL);
+	if (!function) {
+		ret = -ENOMEM;
+		goto failure;
+	}
+
+	if (copy_from_user(function, ubuf, len)) {
+		ret = -EMSGSIZE;
+		goto failure;
+	}
+
+	if (function->fxn_id >= rppcdev->num_funcs - 1) {
+		ret = -EINVAL;
+		goto failure;
+	}
+
+	/* increment the message id and wrap if needed */
+	rpc->msg_id = (rpc->msg_id + 1) & 0xFFFF;
+
+	memset(kbuf, 0, sizeof(kbuf));
+	sig_idx = function->fxn_id + 1;
+	hdr = (struct rppc_msg_header *)kbuf;
+	hdr->msg_type = RPPC_MSGTYPE_FUNCTION_CALL;
+	hdr->msg_len = sizeof(*packet);
+	packet = RPPC_PAYLOAD(kbuf, rppc_packet);
+	packet->desc = RPPC_DESC_EXEC_SYNC;
+	packet->msg_id = rpc->msg_id;
+	packet->flags = (RPPC_JOBID_DISCRETE << 16) | RPPC_POOLID_DEFAULT;
+	packet->fxn_id = RPPC_SET_FXN_IDX(function->fxn_id);
+	packet->result = 0;
+	packet->data_size = sizeof(*parameters) * function->num_params;
+
+	/* check the signatures against what were published */
+	if (RPPC_SIG_NUM_PARAM(rppcdev->signatures[sig_idx]) !=
+		function->num_params) {
+		dev_err(dev, "number of parameters mismatch! params = %u expected = %u\n",
+			function->num_params,
+			RPPC_SIG_NUM_PARAM(rppcdev->signatures[sig_idx]));
+		ret = -EINVAL;
+		goto failure;
+	}
+
+	/*
+	 * compute the parameter pointer changes last since this will cause the
+	 * cache operations
+	 */
+	parameters = (struct rppc_param_data *)packet->data;
+	for (param = 0; param < function->num_params; param++) {
+		u32 param_type;
+
+		sig_prm = param + 1;
+		param_type = rppcdev->signatures[sig_idx].params[sig_prm].type;
+		/*
+		 * check to make sure the parameter description matches the
+		 * signature published from the other side.
+		 */
+		if (function->params[param].type == RPPC_PARAM_TYPE_PTR &&
+		    !RPPC_IS_PTR(param_type)) {
+			dev_err(dev, "parameter %u Pointer Type Mismatch sig type:%x func %u\n",
+				param, param_type, sig_idx);
+			ret = -EINVAL;
+			goto failure;
+		} else if (param > 0 && function->params[param].type ==
+			RPPC_PARAM_TYPE_ATOMIC) {
+			if (!RPPC_IS_ATOMIC(param_type)) {
+				dev_err(dev, "parameter Atomic Type Mismatch\n");
+				ret = -EINVAL;
+				goto failure;
+			} else {
+				if (rppc_atomic_size[param_type] !=
+					function->params[param].size) {
+					dev_err(dev, "size mismatch! u:%u sig:%u\n",
+						function->params[param].size,
+						rppc_atomic_size[param_type]);
+					ret = -EINVAL;
+					goto failure;
+				}
+			}
+		}
+
+		parameters[param].size = function->params[param].size;
+
+		/* check the type and lookup if it's a pointer */
+		if (function->params[param].type == RPPC_PARAM_TYPE_PTR) {
+			/*
+			 * internally the buffer translations takes care of the
+			 * offsets.
+			 */
+			int fd = function->params[param].fd;
+
+			parameters[param].data = (size_t)rppc_buffer_lookup(rpc,
+				(virt_addr_t)function->params[param].data,
+				(virt_addr_t)function->params[param].base, fd);
+		} else if (function->params[param].type ==
+			   RPPC_PARAM_TYPE_ATOMIC) {
+			parameters[param].data = function->params[param].data;
+		} else {
+			ret = -ENOTSUPP;
+			goto failure;
+		}
+	}
+
+	/* compute the size of the rpmsg packet */
+	use = sizeof(*hdr) + hdr->msg_len + packet->data_size;
+
+	/* failed to provide the translation data */
+	if (function->num_translations > 0 &&
+	    len < (sizeof(*function) + (function->num_translations *
+				sizeof(struct rppc_param_translation)))) {
+		ret = -EINVAL;
+		goto failure;
+	}
+
+	/*
+	 * if there are pointers to translate for the user, do so now.
+	 * alter our copy of function and the user's parameters so that
+	 * the proper pointers can be sent to remote cores
+	 */
+	if (function->num_translations > 0) {
+		ret = rppc_xlate_buffers(rpc, function, RPPC_UVA_TO_RPA);
+		if (ret < 0) {
+			dev_err(dev, "failed to translate all pointers for remote core!\n");
+			goto failure;
+		}
+	}
+
+	ret = rppc_add_fxn(rpc, function, rpc->msg_id);
+	if (ret < 0) {
+		rppc_xlate_buffers(rpc, function, RPPC_RPA_TO_UVA);
+		goto failure;
+	}
+
+	rppc_print_msg(rpc, "TX:", kbuf);
+
+	ret = rpmsg_send_offchannel(rppcdev->rpdev->ept, rpc->ept->addr,
+				    rpc->dst, kbuf, use);
+	if (ret) {
+		dev_err(dev, "rpmsg_send failed: %d\n", ret);
+		rppc_find_fxn(rpc, rpc->msg_id);
+		rppc_xlate_buffers(rpc, function, RPPC_RPA_TO_UVA);
+		goto failure;
+	}
+	dev_dbg(dev, "==> sent msg to remote endpoint %u\n", rpc->dst);
+
+failure:
+	if (ret >= 0)
+		ret = len;
+	else
+		kfree(function);
+
+	return ret;
+}
+
+static __poll_t rppc_poll(struct file *filp, struct poll_table_struct *wait)
+{
+	struct rppc_instance *rpc = filp->private_data;
+	__poll_t mask = 0;
+
+	poll_wait(filp, &rpc->readq, wait);
+	if (rpc->state == RPPC_STATE_STALE) {
+		mask = EPOLLERR;
+		goto out;
+	}
+
+	/* if the queue is not empty set the poll bit correctly */
+	if (!skb_queue_empty(&rpc->queue))
+		mask |= (EPOLLIN | EPOLLRDNORM);
+
+	/* TODO: writes are deemed to be successful always, fix this later */
+	if (true)
+		mask |= EPOLLOUT | EPOLLWRNORM;
+
+out:
+	return mask;
+}
+
+static const struct file_operations rppc_fops = {
+	.owner = THIS_MODULE,
+	.open = rppc_open,
+	.release = rppc_release,
+	.unlocked_ioctl = rppc_ioctl,
+	.read = rppc_read,
+	.write = rppc_write,
+	.poll = rppc_poll,
+};
+
+/*
+ * send a function query message, the sysfs entry will be created
+ * during the processing of the response message
+ */
+static int rppc_query_function(struct rpmsg_device *rpdev)
+{
+	int ret = 0;
+	u32 len = 0;
+	char kbuf[512];
+	struct rppc_device *rppcdev = dev_get_drvdata(&rpdev->dev);
+	struct rppc_msg_header *hdr = (struct rppc_msg_header *)&kbuf[0];
+	struct rppc_query_function *fxn_info =
+				(struct rppc_query_function *)hdr->msg_data;
+
+	if (rppcdev->cur_func >= rppcdev->num_funcs)
+		return -EINVAL;
+
+	hdr->msg_type = RPPC_MSGTYPE_FUNCTION_QUERY;
+	hdr->msg_len = sizeof(*fxn_info);
+	len = sizeof(*hdr) + hdr->msg_len;
+	fxn_info->info_type = RPPC_INFOTYPE_FUNC_SIGNATURE;
+	fxn_info->fxn_id = rppcdev->cur_func++;
+
+	dev_dbg(&rpdev->dev, "sending function query type %u for function %u\n",
+		fxn_info->info_type, fxn_info->fxn_id);
+	ret = rpmsg_send(rpdev->ept, (char *)kbuf, len);
+	if (ret) {
+		dev_err(&rpdev->dev, "rpmsg_send failed: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void
+rppc_handle_devinfo_resp(struct rpmsg_device *rpdev, char *data, int len)
+{
+	struct rppc_device *rppcdev = dev_get_drvdata(&rpdev->dev);
+	struct rppc_device_info *info;
+	u32 exp_len = sizeof(*info) + sizeof(struct rppc_msg_header);
+
+	if (len != exp_len) {
+		dev_err(&rpdev->dev, "invalid message length %d (expected %d bytes)",
+			len, exp_len);
+		return;
+	}
+
+	info = RPPC_PAYLOAD(data, rppc_device_info);
+	if (info->num_funcs > RPPC_MAX_NUM_FUNCS) {
+		rppcdev->num_funcs = 0;
+		dev_err(&rpdev->dev, "number of functions (%d) exceeds the limit supported(%d)\n",
+			info->num_funcs, RPPC_MAX_NUM_FUNCS);
+		return;
+	}
+
+	rppcdev->num_funcs = info->num_funcs;
+	rppcdev->signatures = kcalloc(rppcdev->num_funcs,
+				      sizeof(struct rppc_func_signature),
+				      GFP_KERNEL);
+	if (!rppcdev->signatures)
+		return;
+
+	dev_info(&rpdev->dev, "published functions = %u\n", info->num_funcs);
+
+	/* send the function query for first function */
+	if (rppc_query_function(rpdev) == -EINVAL)
+		dev_err(&rpdev->dev, "failed to get a reasonable number of functions!\n");
+}
+
+static void
+rppc_handle_fxninfo_resp(struct rpmsg_device *rpdev, char *data, int len)
+{
+	struct rppc_device *rppcdev = dev_get_drvdata(&rpdev->dev);
+	struct rppc_query_function *fxn_info;
+	struct rppc_func_signature *signature;
+	u32 exp_len = sizeof(*fxn_info) + sizeof(struct rppc_msg_header);
+	int i;
+
+	if (len != exp_len) {
+		dev_err(&rpdev->dev, "invalid message length %d (expected %d bytes)",
+			len, exp_len);
+		return;
+	}
+
+	fxn_info = RPPC_PAYLOAD(data, rppc_query_function);
+	dev_dbg(&rpdev->dev, "response for function query of type %u\n",
+		fxn_info->info_type);
+
+	switch (fxn_info->info_type) {
+	case RPPC_INFOTYPE_FUNC_SIGNATURE:
+		if (fxn_info->fxn_id >= rppcdev->num_funcs) {
+			dev_err(&rpdev->dev, "function(%d) is out of range!\n",
+				fxn_info->fxn_id);
+			break;
+		}
+
+		memcpy(&rppcdev->signatures[fxn_info->fxn_id],
+		       &fxn_info->info.signature, sizeof(*signature));
+
+		/* TODO: delete these debug prints later */
+		dev_dbg(&rpdev->dev, "received info for func(%d); name = %s #params = %u\n",
+			fxn_info->fxn_id, fxn_info->info.signature.name,
+			fxn_info->info.signature.num_param);
+		signature = &rppcdev->signatures[fxn_info->fxn_id];
+		for (i = 0; i < signature->num_param; i++) {
+			dev_dbg(&rpdev->dev, "param[%u] type = %x dir = %u\n",
+				i, signature->params[i].type,
+				signature->params[i].direction);
+		}
+
+		/* query again until we've hit our limit */
+		if (rppc_query_function(rpdev) == -EINVAL) {
+			dev_dbg(&rpdev->dev, "reached end of function list!\n");
+			rppc_create_sysfs(rppcdev);
+		}
+		break;
+	default:
+		dev_err(&rpdev->dev, "unrecognized fxn query response %u\n",
+			fxn_info->info_type);
+		break;
+	}
+}
+
+static int rppc_driver_cb(struct rpmsg_device *rpdev, void *data, int len,
+			  void *priv, u32 src)
+{
+	struct rppc_msg_header *hdr = data;
+	char *buf = (char *)data;
+
+	dev_dbg(&rpdev->dev, "<== incoming drv msg src %d len %d msg_type %d msg_len %d\n",
+		src, len, hdr->msg_type, hdr->msg_len);
+
+	if (len <= sizeof(*hdr)) {
+		dev_err(&rpdev->dev, "message truncated\n");
+		return -EINVAL;
+	}
+
+	switch (hdr->msg_type) {
+	case RPPC_MSGTYPE_DEVINFO_RESP:
+		rppc_handle_devinfo_resp(rpdev, buf, len);
+		break;
+	case RPPC_MSGTYPE_FUNCTION_INFO:
+		rppc_handle_fxninfo_resp(rpdev, buf, len);
+		break;
+	default:
+		dev_err(&rpdev->dev, "unrecognized message type %u\n",
+			hdr->msg_type);
+		break;
+	}
+
+	return 0;
+}
+
+static int find_rpccdev_by_name(int id, void *p, void *data)
+{
+	struct rppc_device *rppcdev = p;
+
+	return strcmp(rppcdev->desc, data) ? 0 : (int)p;
+}
+
+/*
+ * send a device info query message, the device will be created
+ * during the processing of the response message
+ */
+static int rppc_device_create(struct rpmsg_device *rpdev)
+{
+	int ret;
+	u32 len;
+	char kbuf[512];
+	struct rppc_msg_header *hdr = (struct rppc_msg_header *)&kbuf[0];
+
+	hdr->msg_type = RPPC_MSGTYPE_DEVINFO_REQ;
+	hdr->msg_len = 0;
+	len = sizeof(*hdr);
+	ret = rpmsg_send(rpdev->ept, (char *)kbuf, len);
+	if (ret) {
+		dev_err(&rpdev->dev, "rpmsg_send failed: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int rppc_probe(struct rpmsg_device *rpdev)
+{
+	int ret, minor;
+	int major = MAJOR(rppc_dev);
+	struct rppc_device *rppcdev = NULL;
+	dev_t dev;
+	char namedesc[RPMSG_NAME_SIZE];
+
+	dev_info(&rpdev->dev, "probing service %s with src %u dst %u\n",
+		 rpdev->desc, rpdev->src, rpdev->dst);
+
+	mutex_lock(&rppc_devices_lock);
+	snprintf(namedesc, sizeof(namedesc), "%s", rpdev->desc);
+	rppcdev = (struct rppc_device *)idr_for_each(&rppc_devices,
+						find_rpccdev_by_name, namedesc);
+	if (rppcdev) {
+		rppcdev->rpdev = rpdev;
+		dev_set_drvdata(&rpdev->dev, rppcdev);
+		goto serv_up;
+	}
+
+	rppcdev = kzalloc(sizeof(*rppcdev), GFP_KERNEL);
+	if (!rppcdev) {
+		ret = -ENOMEM;
+		goto exit;
+	}
+
+	minor = idr_alloc(&rppc_devices, rppcdev, 0, 0, GFP_KERNEL);
+	if (minor < 0) {
+		ret = minor;
+		dev_err(&rpdev->dev, "failed to get a minor number: %d\n", ret);
+		goto free_rppcdev;
+	}
+
+	INIT_LIST_HEAD(&rppcdev->instances);
+	mutex_init(&rppcdev->lock);
+	init_completion(&rppcdev->comp);
+
+	rppcdev->minor = minor;
+	rppcdev->rpdev = rpdev;
+	strncpy(rppcdev->desc, namedesc, RPMSG_NAME_SIZE);
+	dev_set_drvdata(&rpdev->dev, rppcdev);
+
+	cdev_init(&rppcdev->cdev, &rppc_fops);
+	rppcdev->cdev.owner = THIS_MODULE;
+	dev = MKDEV(major, minor);
+	ret = cdev_add(&rppcdev->cdev, dev, 1);
+	if (ret) {
+		dev_err(&rpdev->dev, "cdev_add failed: %d\n", ret);
+		goto free_id;
+	}
+
+serv_up:
+	rppcdev->dev = device_create(rppc_class, &rpdev->dev,
+				     MKDEV(major, rppcdev->minor), NULL,
+				     namedesc);
+	if (IS_ERR(rppcdev->dev)) {
+		ret = PTR_ERR(rppcdev->dev);
+
+		dev_err(&rpdev->dev, "device_create failed: %d\n", ret);
+		goto free_cdev;
+	}
+	dev_set_drvdata(rppcdev->dev, rppcdev);
+
+	ret = rppc_device_create(rpdev);
+	if (ret) {
+		dev_err(&rpdev->dev, "failed to query channel info: %d\n", ret);
+		dev = MKDEV(MAJOR(rppc_dev), rppcdev->minor);
+		goto free_dev;
+	}
+
+	complete_all(&rppcdev->comp);
+
+	dev_dbg(&rpdev->dev, "new RPPC connection srv channel: %u -> %u!\n",
+		rpdev->src, rpdev->dst);
+
+	mutex_unlock(&rppc_devices_lock);
+	return 0;
+
+free_dev:
+	device_destroy(rppc_class, dev);
+free_cdev:
+	cdev_del(&rppcdev->cdev);
+free_id:
+	idr_remove(&rppc_devices, rppcdev->minor);
+free_rppcdev:
+	kfree(rppcdev);
+exit:
+	mutex_unlock(&rppc_devices_lock);
+	return ret;
+}
+
+static void rppc_remove(struct rpmsg_device *rpdev)
+{
+	struct rppc_device *rppcdev = dev_get_drvdata(&rpdev->dev);
+	struct rppc_instance *rpc = NULL;
+	int major = MAJOR(rppc_dev);
+
+	dev_dbg(&rpdev->dev, "removing rpmsg-rpc device %u.%u\n",
+		major, rppcdev->minor);
+
+	mutex_lock(&rppc_devices_lock);
+
+	rppc_remove_sysfs(rppcdev);
+	rppcdev->cur_func = 0;
+	kfree(rppcdev->signatures);
+
+	/* if there are no instances in the list, just teardown */
+	if (list_empty(&rppcdev->instances)) {
+		dev_dbg(&rpdev->dev, "no instances, removing device!\n");
+		device_destroy(rppc_class, MKDEV(major, rppcdev->minor));
+		cdev_del(&rppcdev->cdev);
+		idr_remove(&rppc_devices, rppcdev->minor);
+		kfree(rppcdev);
+		mutex_unlock(&rppc_devices_lock);
+		return;
+	}
+
+	/*
+	 * if there are rpc instances that means that this is a recovery
+	 * operation. Don't clean the rppcdev, and retain it for reuse.
+	 * mark each instance as invalid, and complete any on-going transactions
+	 */
+	init_completion(&rppcdev->comp);
+	mutex_lock(&rppcdev->lock);
+	list_for_each_entry(rpc, &rppcdev->instances, list) {
+		dev_dbg(&rpdev->dev, "instance %p in state %d\n",
+			rpc, rpc->state);
+		if (rpc->state == RPPC_STATE_CONNECTED && rpc->in_transition)
+			complete_all(&rpc->reply_arrived);
+		rpc->state = RPPC_STATE_STALE;
+		if (rpc->ept) {
+			rpmsg_destroy_ept(rpc->ept);
+			rpc->ept = NULL;
+		}
+		wake_up_interruptible(&rpc->readq);
+	}
+	device_destroy(rppc_class, MKDEV(major, rppcdev->minor));
+	rppcdev->dev = NULL;
+	rppcdev->rpdev = NULL;
+	mutex_unlock(&rppcdev->lock);
+	mutex_unlock(&rppc_devices_lock);
+	dev_dbg(&rpdev->dev, "removed rpmsg rpmsg-rpc service %s\n",
+		rpdev->desc);
+}
+
+static struct rpmsg_device_id rppc_id_table[] = {
+	{.name = "rpmsg-rpc"},
+	{},
+};
+
+static struct rpmsg_driver rppc_driver = {
+	.drv.name = KBUILD_MODNAME,
+	.id_table = rppc_id_table,
+	.probe = rppc_probe,
+	.remove = rppc_remove,
+	.callback = rppc_driver_cb,
+};
+
+static int __init rppc_init(void)
+{
+	int ret;
+
+	ret = alloc_chrdev_region(&rppc_dev, 0, RPPC_MAX_DEVICES,
+				  KBUILD_MODNAME);
+	if (ret) {
+		pr_err("alloc_chrdev_region failed: %d\n", ret);
+		goto out;
+	}
+
+	rppc_class = class_create(THIS_MODULE, KBUILD_MODNAME);
+	if (IS_ERR(rppc_class)) {
+		ret = PTR_ERR(rppc_class);
+		pr_err("class_create failed: %d\n", ret);
+		goto unreg_region;
+	}
+
+	ret = register_rpmsg_driver(&rppc_driver);
+	if (ret) {
+		pr_err("register_rpmsg_driver failed: %d\n", ret);
+		goto destroy_class;
+	}
+	return 0;
+
+destroy_class:
+	class_destroy(rppc_class);
+unreg_region:
+	unregister_chrdev_region(rppc_dev, RPPC_MAX_DEVICES);
+out:
+	return ret;
+}
+
+static void __exit rppc_exit(void)
+{
+	unregister_rpmsg_driver(&rppc_driver);
+	class_destroy(rppc_class);
+	unregister_chrdev_region(rppc_dev, RPPC_MAX_DEVICES);
+}
+
+module_init(rppc_init);
+module_exit(rppc_exit);
+MODULE_DEVICE_TABLE(rpmsg, rppc_id_table);
+
+MODULE_AUTHOR("Suman Anna <s-anna@ti.com>");
+MODULE_AUTHOR("Erik Rainey <erik.rainey@ti.com>");
+MODULE_DESCRIPTION("Remote Processor Procedure Call Driver");
+MODULE_ALIAS("rpmsg:rpmsg-rpc");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/rpmsg/rpmsg_rpc_dmabuf.c b/drivers/rpmsg/rpmsg_rpc_dmabuf.c
--- a/drivers/rpmsg/rpmsg_rpc_dmabuf.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/rpmsg/rpmsg_rpc_dmabuf.c	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,654 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Remote Processor Procedure Call Driver
+ *
+ * Copyright (C) 2012-2019 Texas Instruments Incorporated - http://www.ti.com/
+ *	Erik Rainey <erik.rainey@ti.com>
+ *	Suman Anna <s-anna@ti.com>
+ */
+
+#include <linux/dma-buf.h>
+#include <linux/rpmsg_rpc.h>
+
+#include "rpmsg_rpc_internal.h"
+
+#if defined(CONFIG_ARCH_OMAP4) || defined(CONFIG_SOC_OMAP5) || \
+	defined(CONFIG_SOC_DRA7XX)
+/*
+ * TODO: Remove tiler_stride_from_region & rppc_recalc_off from here, and
+ *	 rely on OMAPDRM/TILER code for OMAP dependencies
+ */
+
+/**
+ * tiler_stride_from_region() - calculate stride value for OMAP TILER
+ * @localphys:	The local physical address.
+ *
+ * Returns the stride value as seen by remote processors based on the local
+ * address given to the function. This stride value is calculated based on the
+ * actual bus address, and is assumed that the TILER regions are mapped in a
+ * in a linear fashion.
+ *
+ * The physical address range decoding of local addresses is as follows:
+ *
+ * 0x60000000 - 0x67FFFFFF : 8-bit region (Stride is 16K bytes)
+ * 0x68000000 - 0x6FFFFFFF : 16-bit region (Stride is 32K bytes)
+ * 0x70000000 - 0x77FFFFFF : 32-bit region (Stride is 32K bytes)
+ * 0x78000000 - 0x7FFFFFFF : Page mode region (Stride is 0 bytes)
+ *
+ * Return: stride value
+ */
+static long tiler_stride_from_region(phys_addr_t localphys)
+{
+	switch (localphys & 0xf8000000) {
+	case 0x60000000:
+		return 0x4000;
+	case 0x68000000:
+	case 0x70000000:
+		return 0x8000;
+	default:
+		return 0;
+	}
+}
+
+/**
+ * rppc_recalc_off() - Recalculate the unsigned offset in a buffer due to
+ *		       it's location in the TILER.
+ * @lpa:	local physical address
+ * @uoff:	unsigned offset
+ *
+ * Return: adjusted offset accounting for TILER region
+ */
+static long rppc_recalc_off(phys_addr_t lpa, long uoff)
+{
+	long stride = tiler_stride_from_region(lpa);
+
+	return (stride != 0) ? (stride * (uoff / PAGE_SIZE)) +
+				(uoff & (PAGE_SIZE - 1)) : uoff;
+}
+#else
+static inline long rppc_recalc_off(phys_addr_t lpa, long uoff)
+{
+	return uoff;
+}
+#endif
+
+/**
+ * rppc_alloc_dmabuf - import a buffer and store in a rppc buffer descriptor
+ * @rpc - rppc instance handle
+ * @fd - dma_buf file descriptor
+ * @autoreg: flag indicating the mode of creation
+ *
+ * This function primarily imports a buffer into the driver and holds
+ * a reference to the buffer on behalf of the remote processor. The
+ * buffer to be imported is represented by a dma-buf file descriptor,
+ * and as such is agnostic of the buffer allocator and/or exporter.
+ * The buffer is imported using the dma-buf api, and a driver specific
+ * buffer descriptor is used to store the imported buffer properties.
+ * The imported buffers are all stored in a rppc instance specific
+ * idr, to be used for looking up and cleaning up the driver buffer
+ * descriptors.
+ *
+ * The @autoreg field is used to dictate the manner in which the buffer
+ * is imported. The user-side can pre-register the buffers with the driver
+ * (which will import the buffers) if the application is going to use
+ * these repeatedly in consecutive function invocations. The buffers
+ * are auto-imported if the user-side has not registered them previously
+ * and are un-imported once the remote function call returns.
+ *
+ * This function is to be called only after checking that buffer has
+ * not been imported already (see rppc_find_dmabuf).
+ *
+ * Return: allocated rppc_dma_buf or error
+ */
+struct rppc_dma_buf *rppc_alloc_dmabuf(struct rppc_instance *rpc, int fd,
+				       bool autoreg)
+{
+	struct rppc_dma_buf *dma;
+	void *ret;
+	int id;
+
+	dma = kzalloc(sizeof(*dma), GFP_KERNEL);
+	if (!dma)
+		return ERR_PTR(-ENOMEM);
+
+	dma->fd = fd;
+	dma->autoreg = !!autoreg;
+	dma->buf = dma_buf_get(dma->fd);
+	if (IS_ERR(dma->buf)) {
+		ret = dma->buf;
+		goto free_dma;
+	}
+
+	dma->attach = dma_buf_attach(dma->buf, rpc->dev);
+	if (IS_ERR(dma->attach)) {
+		ret = dma->attach;
+		goto put_buf;
+	}
+
+	dma->sgt = dma_buf_map_attachment(dma->attach, DMA_BIDIRECTIONAL);
+	if (IS_ERR(dma->sgt)) {
+		ret = dma->sgt;
+		goto detach_buf;
+	}
+
+	dma->pa = sg_dma_address(dma->sgt->sgl);
+	mutex_lock(&rpc->lock);
+	id = idr_alloc(&rpc->dma_idr, dma, 0, 0, GFP_KERNEL);
+	dma->id = id;
+	mutex_unlock(&rpc->lock);
+	if (id < 0) {
+		ret = ERR_PTR(id);
+		goto unmap_buf;
+	}
+
+	return dma;
+
+unmap_buf:
+	dma_buf_unmap_attachment(dma->attach, dma->sgt, DMA_BIDIRECTIONAL);
+detach_buf:
+	dma_buf_detach(dma->buf, dma->attach);
+put_buf:
+	dma_buf_put(dma->buf);
+free_dma:
+	kfree(dma);
+
+	return ret;
+}
+
+/**
+ * rppc_free_dmabuf - release the imported buffer
+ * @id: idr index of the imported buffer descriptor
+ * @p: imported buffer descriptor allocated during rppc_alloc_dmabuf
+ * @data: rpc instance handle
+ *
+ * This function is used to release a buffer that has been previously
+ * imported through a rppc_alloc_dmabuf call. The function can be used
+ * either individually for releasing a specific buffer or in a loop iterator
+ * for releasing all the buffers associated with a remote function call, or
+ * during cleanup of the rpc instance.
+ *
+ * Return: 0 on success, and -ENOENT if invalid pointers passed in
+ */
+int rppc_free_dmabuf(int id, void *p, void *data)
+{
+	struct rppc_dma_buf *dma = p;
+	struct rppc_instance *rpc = data;
+
+	if (!dma || !rpc)
+		return -ENOENT;
+
+	dma_buf_unmap_attachment(dma->attach, dma->sgt, DMA_BIDIRECTIONAL);
+	dma_buf_detach(dma->buf, dma->attach);
+	dma_buf_put(dma->buf);
+	WARN_ON(id != dma->id);
+	idr_remove(&rpc->dma_idr, id);
+	kfree(dma);
+
+	return 0;
+}
+
+/**
+ * rppc_free_auto_dmabuf - release an auto-registered imported buffer
+ * @id: idr index of the imported buffer descriptor
+ * @p: imported buffer descriptor allocated during the rppc_alloc_dmabuf
+ * @data: rpc instance handle
+ *
+ * This function is used to release a buffer that has been previously
+ * imported automatically in the remote function invocation path (for
+ * rppc_alloc_dmabuf invocations with autoreg set as true). The function
+ * is used as a loop iterator for releasing all such buffers associated
+ * with a remote function call, and is called after processing the
+ * translations while handling the return message of an executed function
+ * call.
+ *
+ * Return: 0 on success or if the buffer is not auto-imported, and -ENOENT
+ *	   if invalid pointers passed in
+ */
+static int rppc_free_auto_dmabuf(int id, void *p, void *data)
+{
+	struct rppc_dma_buf *dma = p;
+	struct rppc_instance *rpc = data;
+
+	if (WARN_ON(!dma || !rpc))
+		return -ENOENT;
+
+	if (!dma->autoreg)
+		return 0;
+
+	rppc_free_dmabuf(id, p, data);
+	return 0;
+}
+
+/**
+ * find_dma_by_fd - find the allocated buffer descriptor
+ * @id: idr loop index
+ * @p: imported buffer descriptor associated with each idr index @id
+ * @data: dma-buf file descriptor of the buffer
+ *
+ * This is a idr iterator helper function, used for checking if a buffer
+ * has been imported before and present within the rpc instance's idr.
+ *
+ * Return: rpc buffer descriptor if file descriptor matches, and 0 otherwise
+ */
+static int find_dma_by_fd(int id, void *p, void *data)
+{
+	struct rppc_dma_buf *dma = p;
+	int fd = (int)data;
+
+	if (dma->fd == fd)
+		return (int)p;
+
+	return 0;
+}
+
+/**
+ * rppc_find_dmabuf - find and return the rppc buffer descriptor of an imported
+ *		      buffer
+ * @rpc: rpc instance
+ * @fd: dma-buf file descriptor of the buffer
+ *
+ * This function is used to find and return the rppc buffer descriptor of an
+ * imported buffer. The function is used to check if ia buffer has already
+ * been imported (during manual registration to return an error), and to return
+ * the rppc buffer descriptor to be used for freeing (during manual
+ * deregistration). It is also used during auto-registration to see if the
+ * buffer needs to be imported through a rppc_alloc_dmabuf if not found.
+ *
+ * Return: rppc buffer descriptor of the buffer if it has already been imported,
+ *	   or NULL otherwise.
+ */
+struct rppc_dma_buf *rppc_find_dmabuf(struct rppc_instance *rpc, int fd)
+{
+	struct rppc_dma_buf *node = NULL;
+	void *data = (void *)fd;
+
+	dev_dbg(rpc->dev, "looking for fd %u\n", fd);
+
+	mutex_lock(&rpc->lock);
+	node = (struct rppc_dma_buf *)
+			idr_for_each(&rpc->dma_idr, find_dma_by_fd, data);
+	mutex_unlock(&rpc->lock);
+
+	dev_dbg(rpc->dev, "returning node %p for fd %u\n",
+		node, fd);
+
+	return node;
+}
+
+/**
+ * rppc_map_page - import and map a kernel page in a dma_buf
+ * @rpc - rppc instance handle
+ * @fd: file descriptor of the dma_buf to import
+ * @offset: offset of the translate location within the buffer
+ * @base_ptr: pointer for returning mapped kernel address
+ * @dmabuf: pointer for returning the imported dma_buf
+ *
+ * A helper function to import the dma_buf buffer and map into kernel
+ * the page containing the offset within the buffer. The function is
+ * called by rppc_xlate_buffers and returns the pointers to the kernel
+ * mapped address and the imported dma_buf handle in arguments. The
+ * mapping is used for performing in-place translation of the user
+ * provided pointer at location @offset within the buffer.
+ *
+ * The mapping is achieved through the appropriate dma_buf ops, and
+ * the page will be unmapped after performing the translation. See
+ * also rppc_unmap_page.
+ *
+ * Return: 0 on success, or an appropriate failure code otherwise
+ */
+static int rppc_map_page(struct rppc_instance *rpc, int fd, u32 offset,
+			 u8 **base_ptr, struct dma_buf **dmabuf)
+{
+	int ret = 0;
+	u8 *ptr = NULL;
+	struct dma_buf *dbuf = NULL;
+	u32 pg_offset;
+	unsigned long pg_num;
+	size_t begin, end = PAGE_SIZE;
+	struct device *dev = rpc->dev;
+
+	if (!base_ptr || !dmabuf)
+		return -EINVAL;
+
+	pg_offset = (offset & (PAGE_SIZE - 1));
+	begin = offset & PAGE_MASK;
+	pg_num = offset >> PAGE_SHIFT;
+
+	dbuf = dma_buf_get(fd);
+	if (IS_ERR(dbuf)) {
+		ret = PTR_ERR(dbuf);
+		dev_err(dev, "invalid dma_buf file descriptor passed! fd = %d ret = %d\n",
+			fd, ret);
+		goto out;
+	}
+
+	ret = dma_buf_begin_cpu_access(dbuf, DMA_BIDIRECTIONAL);
+	if (ret < 0) {
+		dev_err(dev, "failed to acquire cpu access to the dma buf fd = %d offset = 0x%x, ret = %d\n",
+			fd, offset, ret);
+		goto put_dmabuf;
+	}
+
+	ptr = dma_buf_kmap(dbuf, pg_num);
+	if (!ptr) {
+		ret = -ENOBUFS;
+		dev_err(dev, "failed to map the page containing the translation into kernel fd = %d offset = 0x%x\n",
+			fd, offset);
+		goto end_cpuaccess;
+	}
+
+	*base_ptr = ptr;
+	*dmabuf = dbuf;
+	dev_dbg(dev, "kmap'd base_ptr = %p buf = %p into kernel from %zu for %zu bytes, pg_offset = 0x%x\n",
+		ptr, dbuf, begin, end, pg_offset);
+	return 0;
+
+end_cpuaccess:
+	dma_buf_end_cpu_access(dbuf, DMA_BIDIRECTIONAL);
+put_dmabuf:
+	dma_buf_put(dbuf);
+out:
+	return ret;
+}
+
+/**
+ * rppc_unmap_page - unmap and release a previously mapped page
+ * @rpc - rppc instance handle
+ * @offset: offset of the translate location within the buffer
+ * @base_ptr: kernel mapped address for the page to be unmapped
+ * @dmabuf: imported dma_buf to be released
+ *
+ * This function is called by rppc_xlate_buffers to unmap the
+ * page and release the imported buffer. It essentially undoes
+ * the functionality of rppc_map_page.
+ */
+static void rppc_unmap_page(struct rppc_instance *rpc, u32 offset,
+			    u8 *base_ptr, struct dma_buf *dmabuf)
+{
+	u32 pg_offset;
+	unsigned long pg_num;
+	size_t begin, end = PAGE_SIZE;
+	struct device *dev = rpc->dev;
+
+	if (!base_ptr || !dmabuf)
+		return;
+
+	pg_offset = (offset & (PAGE_SIZE - 1));
+	begin = offset & PAGE_MASK;
+	pg_num = offset >> PAGE_SHIFT;
+
+	dev_dbg(dev, "Unkmaping base_ptr = %p of buf = %p from %zu to %zu bytes\n",
+		base_ptr, dmabuf, begin, end);
+	dma_buf_kunmap(dmabuf, pg_num, base_ptr);
+	dma_buf_end_cpu_access(dmabuf, DMA_BIDIRECTIONAL);
+	dma_buf_put(dmabuf);
+}
+
+/**
+ * rppc_buffer_lookup - convert a buffer pointer to a remote processor pointer
+ * @rpc: rpc instance
+ * @uva: buffer pointer that needs to be translated
+ * @buva: base pointer of the allocated buffer
+ * @fd: dma-buf file descriptor of the allocated buffer
+ *
+ * This function is used for converting a pointer value in the function
+ * arguments to its appropriate remote processor device address value.
+ * The @uva and @buva are used for identifying the offset of the function
+ * argument pointer in an original allocation. This supports the cases where
+ * an offset pointer (eg: alignment, packed buffers etc) needs to be passed
+ * as the argument rather than the actual allocated pointer.
+ *
+ * The remote processor device address is done by retrieving the base physical
+ * address of the buffer by importing the buffer and converting it to the
+ * remote processor device address using a remoteproc api, with adjustments
+ * to the offset.
+ *
+ * The offset is specifically adjusted for OMAP TILER to account for the stride
+ * and mapping onto the remote processor.
+ *
+ * Return: remote processor device address, 0 on failure (implies invalid
+ *	   arguments)
+ */
+dev_addr_t rppc_buffer_lookup(struct rppc_instance *rpc, virt_addr_t uva,
+			      virt_addr_t buva, int fd)
+{
+	phys_addr_t lpa = 0;
+	dev_addr_t rda = 0;
+	long uoff = uva - buva;
+	struct device *dev = rpc->dev;
+	struct rppc_dma_buf *buf;
+
+	dev_dbg(dev, "buva = %p uva = %p offset = %ld [0x%016lx] fd = %d\n",
+		(void *)buva, (void *)uva, uoff, (ulong)uoff, fd);
+
+	if (uoff < 0) {
+		dev_err(dev, "invalid pointer values for uva = %p from buva = %p\n",
+			(void *)uva, (void *)buva);
+		return rda;
+	}
+
+	buf = rppc_find_dmabuf(rpc, fd);
+	if (IS_ERR_OR_NULL(buf)) {
+		buf = rppc_alloc_dmabuf(rpc, fd, true);
+		if (IS_ERR(buf))
+			goto out;
+	}
+
+	lpa = buf->pa;
+	WARN_ON(lpa != sg_dma_address(buf->sgt->sgl));
+	uoff = rppc_recalc_off(lpa, uoff);
+	lpa += uoff;
+	rda = rppc_local_to_remote_da(rpc, lpa);
+
+out:
+	dev_dbg(dev, "host uva %p == host pa %pa => remote da %p (fd %d)\n",
+		(void *)uva, &lpa, (void *)rda, fd);
+	return rda;
+}
+
+/**
+ * rppc_xlate_buffers - translate argument pointers in the marshalled packet
+ * @rpc: rppc instance
+ * @func: rppc function packet being acted upon
+ * @direction: direction of translation
+ *
+ * This function translates all the pointers within the function call packet
+ * structure, based on the translation descriptor structures. The translation
+ * replaces the pointers to the appropriate pointers based on the direction.
+ * The function is invoked in preparing the packet to be sent to the remote
+ * processor-side and replaces the pointers to the remote processor device
+ * address pointers; and in processing the packet back after executing the
+ * function and replacing back the remote processor device addresses with
+ * the original pointers.
+ *
+ * Return: 0 on success, or an appropriate failure code otherwise
+ */
+int rppc_xlate_buffers(struct rppc_instance *rpc, struct rppc_function *func,
+		       int direction)
+{
+	u8 *base_ptr = NULL;
+	struct dma_buf *dbuf = NULL;
+	struct device *dev = rpc->dev;
+	u32 ptr_idx, pri_offset, sec_offset, offset, pg_offset, size;
+	int i, limit, inc = 1;
+	virt_addr_t kva, uva, buva;
+	dev_addr_t rda;
+	int ret = 0, final_ret = 0;
+	int xlate_fd;
+
+	limit = func->num_translations;
+	if (WARN_ON(!limit))
+		return 0;
+
+	dev_dbg(dev, "operating on %d pointers\n", func->num_translations);
+
+	/* sanity check the translation elements */
+	for (i = 0; i < limit; i++) {
+		ptr_idx = func->translations[i].index;
+
+		if (ptr_idx >= RPPC_MAX_PARAMETERS) {
+			dev_err(dev, "xlate[%d] - invalid parameter pointer index %u\n",
+				i, ptr_idx);
+			return -EINVAL;
+		}
+		if (func->params[ptr_idx].type != RPPC_PARAM_TYPE_PTR) {
+			dev_err(dev, "xlate[%d] - parameter index %u is not a pointer (type %u)\n",
+				i, ptr_idx, func->params[ptr_idx].type);
+			return -EINVAL;
+		}
+		if (func->params[ptr_idx].data == 0) {
+			dev_err(dev, "xlate[%d] - supplied user pointer is NULL!\n",
+				i);
+			return -EINVAL;
+		}
+
+		pri_offset = func->params[ptr_idx].data -
+					func->params[ptr_idx].base;
+		sec_offset = func->translations[i].offset;
+		size = func->params[ptr_idx].size;
+
+		if (sec_offset > (size - sizeof(virt_addr_t))) {
+			dev_err(dev, "xlate[%d] offset is larger than data area! (sec_offset = %u size = %u)\n",
+				i, sec_offset, size);
+			return -ENOSPC;
+		}
+	}
+
+	/*
+	 * we may have a failure during translation, in which case use the same
+	 * loop to unwind the whole operation
+	 */
+	for (i = 0; i != limit; i += inc) {
+		dev_dbg(dev, "starting translation %d of %d by %d\n",
+			i, limit, inc);
+
+		ptr_idx = func->translations[i].index;
+		pri_offset = func->params[ptr_idx].data -
+						func->params[ptr_idx].base;
+		sec_offset = func->translations[i].offset;
+		offset = pri_offset + sec_offset;
+		pg_offset = (offset & (PAGE_SIZE - 1));
+
+		/*
+		 * map into kernel the page containing the offset, where the
+		 * pointer needs to be translated.
+		 */
+		ret = rppc_map_page(rpc, func->params[ptr_idx].fd, offset,
+				    &base_ptr, &dbuf);
+		if (ret) {
+			dev_err(dev, "rppc_map_page failed, translation = %d param_index = %d fd = %d ret = %d\n",
+				i, ptr_idx, func->params[ptr_idx].fd, ret);
+			goto unwind;
+		}
+
+		/*
+		 * perform the actual translation as per the direction.
+		 */
+		if (direction == RPPC_UVA_TO_RPA) {
+			kva = (virt_addr_t)&base_ptr[pg_offset];
+			if (kva & 0x3) {
+				dev_err(dev, "kernel virtual address %p is not aligned for translation = %d\n",
+					(void *)kva, i);
+				ret = -EADDRNOTAVAIL;
+				goto unmap;
+			}
+
+			uva = *(virt_addr_t *)kva;
+			if (!uva) {
+				dev_err(dev, "user pointer in the translated offset location is NULL for translation = %d\n",
+					i);
+				print_hex_dump(KERN_DEBUG, "KMAP: ",
+					       DUMP_PREFIX_NONE, 16, 1,
+					       base_ptr, PAGE_SIZE, true);
+				ret = -EADDRNOTAVAIL;
+				goto unmap;
+			}
+
+			buva = (virt_addr_t)func->translations[i].base;
+			xlate_fd = func->translations[i].fd;
+
+			dev_dbg(dev, "replacing UVA %p at KVA %p prt_idx = %u pg_offset = 0x%x fd = %d\n",
+				(void *)uva, (void *)kva, ptr_idx,
+				pg_offset, xlate_fd);
+
+			/* compute the corresponding remote device address */
+			rda = rppc_buffer_lookup(rpc, uva, buva, xlate_fd);
+			if (!rda) {
+				ret = -ENODATA;
+				goto unmap;
+			}
+
+			/*
+			 * replace the pointer, save the old value for replacing
+			 * it back on the function return path
+			 */
+			func->translations[i].fd = (int32_t)uva;
+			*(virt_addr_t *)kva = rda;
+			dev_dbg(dev, "replaced UVA %p with RDA %p at KVA %p\n",
+				(void *)uva, (void *)rda, (void *)kva);
+		} else if (direction == RPPC_RPA_TO_UVA) {
+			kva = (virt_addr_t)&base_ptr[pg_offset];
+			if (kva & 0x3) {
+				ret = -EADDRNOTAVAIL;
+				goto unmap;
+			}
+
+			rda = *(virt_addr_t *)kva;
+			uva = (virt_addr_t)func->translations[i].fd;
+			WARN_ON(!uva);
+			*(virt_addr_t *)kva = uva;
+
+			dev_dbg(dev, "replaced RDA %p with UVA %p at KVA %p\n",
+				(void *)rda, (void *)uva, (void *)kva);
+		}
+
+unmap:
+		/*
+		 * unmap the page containing the translation from kernel, the
+		 * next translation acting on the same fd might be in a
+		 * different page altogether from the current one
+		 */
+		rppc_unmap_page(rpc, offset, base_ptr, dbuf);
+		dbuf = NULL;
+		base_ptr = NULL;
+
+		if (!ret)
+			continue;
+
+unwind:
+		/*
+		 * unwind all the previous translations if the failure occurs
+		 * while sending a message to the remote-side. There's nothing
+		 * to do but to continue if the failure occurs during the
+		 * processing of a function response.
+		 */
+		if (direction == RPPC_UVA_TO_RPA) {
+			dev_err(dev, "unwinding UVA to RDA translations! translation = %d\n",
+				i);
+			direction = RPPC_RPA_TO_UVA;
+			inc = -1;
+			limit = -1;
+		} else if (direction == RPPC_RPA_TO_UVA) {
+			dev_err(dev, "error during UVA to RDA translations!! current translation = %d\n",
+				i);
+		}
+		/*
+		 * store away the return value to return back to caller
+		 * in case of an error, record only the first error
+		 */
+		if (!final_ret)
+			final_ret = ret;
+	}
+
+	/*
+	 * all the in-place pointer replacements are done, release all the
+	 * imported buffers during the remote function return path
+	 */
+	if (direction == RPPC_RPA_TO_UVA) {
+		mutex_lock(&rpc->lock);
+		idr_for_each(&rpc->dma_idr, rppc_free_auto_dmabuf, rpc);
+		mutex_unlock(&rpc->lock);
+	}
+
+	return final_ret;
+}
diff -Naur --no-dereference a/drivers/rpmsg/rpmsg_rpc_internal.h b/drivers/rpmsg/rpmsg_rpc_internal.h
--- a/drivers/rpmsg/rpmsg_rpc_internal.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/rpmsg/rpmsg_rpc_internal.h	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,392 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */
+/*
+ * Remote Processor Procedure Call Driver
+ *
+ * Copyright (C) 2012-2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+#ifndef _RPMSG_RPC_INTERNAL_H_
+#define _RPMSG_RPC_INTERNAL_H_
+
+#include <linux/cdev.h>
+#include <linux/idr.h>
+#include <linux/wait.h>
+#include <linux/fs.h>
+#include <linux/skbuff.h>
+#include <linux/rpmsg.h>
+
+typedef u32 virt_addr_t;
+typedef u32 dev_addr_t;
+
+/**
+ * struct rppc_device - The per-device (server) data
+ * @cdev: character device
+ * @dev: device
+ * @rpdev: rpmsg channel device associated with the remote server
+ * @instances: list of currently opened/connected instances
+ * @lock: mutex for protection of device variables
+ * @comp: completion signal used for unblocking users during a
+ *	  remote processor recovery
+ * @sig_attr: array of device attributes to use with the publishing of
+ *	      function information in sysfs for all the functions
+ *	      associated with this remote server device.
+ * @signatures: function signatures for the functions published by this
+ *		remote server device
+ * @minor: minor number for the character device
+ * @num_funcs: number of functions published by this remote server device
+ * @cur_func: counter used while querying information for each function
+ *	      associated with this remote server device
+ * @desc: description of the exposed service
+ *
+ * A rppc_device indicates the base remote server device that supports the
+ * execution of a bunch of remote functions. Each such remote server device
+ * has an associated character device that is used by the userland apps to
+ * connect to it, and request the execution of any of these remote functions.
+ */
+struct rppc_device {
+	struct cdev cdev;
+	struct device *dev;
+	struct rpmsg_device *rpdev;
+	struct list_head instances;
+	struct mutex lock; /* device state variables lock */
+	struct completion comp;
+	struct device_attribute *sig_attr;
+	struct rppc_func_signature *signatures;
+	unsigned int minor;
+	u32 num_funcs;
+	u32 cur_func;
+	char desc[RPMSG_NAME_SIZE];
+};
+
+/**
+ * struct rppc_instance - The per-instance data structure (per user)
+ * @list: list node
+ * @rppcdev: the rppc device (remote server instance) handle
+ * @dev: local device reference pointer of the rppc device
+ * @queue: queue of buffers waiting to be read by the user
+ * @lock: mutex for protecting instance variables
+ * @readq: wait queue of blocked user threads waiting to read data
+ * @reply_arrived: signal for unblocking the user thread
+ * @ept: rpmsg endpoint associated with the rppc device
+ * @in_transition: flag for storing a pending connection request
+ * @dst: destination end-point of the remote server instance
+ * @state: state of the opened instance, see enum rppc_state
+ * @dma_idr: idr structure storing the imported buffers
+ * @msg_id: last/current active message id tagged on a message sent
+ *	    to the remote processor
+ * @fxn_list: list of functions published by the remote server instance
+ *
+ * This structure is created whenever the user opens the driver. The
+ * various elements of the structure are used to store its state and
+ * information about the remote server it is connected to.
+ */
+struct rppc_instance {
+	struct list_head list;
+	struct rppc_device *rppcdev;
+	struct device *dev;
+	struct sk_buff_head queue;
+	struct mutex lock; /* instance state variables lock */
+	wait_queue_head_t readq;
+	struct completion reply_arrived;
+	struct rpmsg_endpoint *ept;
+	int in_transition;
+	u32 dst;
+	int state;
+	struct idr dma_idr;
+	u16 msg_id;
+	struct list_head fxn_list;
+};
+
+/**
+ * struct rppc_function_list - outstanding function descriptor
+ * @list: list node
+ * @function: current remote function descriptor
+ * @msg_id: message id for the function invocation
+ *
+ * This structure is used for storing the information about outstanding
+ * functions that the remote side is executing. This provides the host
+ * side a means to track every outstanding function, and a means to process
+ * the responses received from the remote processor.
+ */
+struct rppc_function_list {
+	struct list_head list;
+	struct rppc_function *function;
+	u16 msg_id;
+};
+
+/**
+ * struct rppc_dma_buf - a rppc dma_buf descriptor for buffers imported by rppc
+ * @fd: file descriptor of a buffer used to import the dma_buf
+ * @id: idr index value for this descriptor
+ * @buf: imported dma_buf handle for the buffer
+ * @attach: attachment structure returned by exporter upon attaching to
+ *	    the buffer by the rppc driver
+ * @sgt: the scatter-gather structure associated with @buf
+ * @pa: the physical address associated with the imported buffer
+ * @autoreg: mode of how the descriptor is created
+ *
+ * This structure is used for storing the information relevant to the imported
+ * buffer. The rpmsg rpc driver acts as a proxy on behalf of the remote core
+ * and attaches itself to the driver while the remote processor/accelerators are
+ * operating on the buffer.
+ */
+struct rppc_dma_buf {
+	int fd;
+	int id;
+	struct dma_buf *buf;
+	struct dma_buf_attachment *attach;
+	struct sg_table *sgt;
+	phys_addr_t pa;
+	int autoreg;
+};
+
+/**
+ * enum rppc_msg_type - message types exchanged between host and remote server
+ * @RPPC_MSGTYPE_DEVINFO_REQ: request remote server for channel information
+ * @RPPC_MSGTYPE_DEVINFO_RESP: response message from remote server for a
+ *			       request of type RPPC_MSGTYPE_DEVINFO_REQ
+ * @RPPC_MSGTYPE_FUNCTION_QUERY: request remote server for information about a
+ *				 specific function
+ * @RPPC_MSGTYPE_FUNCTION_INFO: response message from remote server for a prior
+ *				request of type RPPC_MSGTYPE_FUNCTION_QUERY
+ * @RPPC_MSGTYPE_CREATE_REQ: request the remote server manager to create a new
+ *			     remote server instance. No secondary data is
+ *			     needed
+ * @RPPC_MSGTYPE_CREATE_RESP: response message from remote server manager for a
+ *			      request of type RPPC_MSGTYPE_CREATE_REQ. The
+ *			      message contains the new endpoint address in the
+ *			      rppc_instance_handle
+ * @RPPC_MSGTYPE_DELETE_REQ: request the remote server manager to delete a
+ *			     remote server instance
+ * @RPPC_MSGTYPE_DELETE_RESP: response message from remote server manager to a
+ *			      request of type RPPC_MSGTYPE_DELETE_REQ. The
+ *			      message contains the old endpoint address in the
+ *			      rppc_instance_handle
+ * @RPPC_MSGTYPE_FUNCTION_CALL: request remote server to execute a specific
+ *				function
+ * @RPPC_MSGTYPE_FUNCTION_RET: response message carrying the return status of a
+ *			       specific function execution
+ * @RPPC_MSGTYPE_ERROR: an error response message sent by either the remote
+ *			server manager or remote server instance while
+ *			processing any request messages
+ * @RPPC_MSGTYPE_MAX: limit value to define the maximum message type value
+ *
+ * Every message exchanged between the host-side and the remote-side is
+ * identified through a message type defined in this enum. The message type
+ * is specified through the msg_type field of the struct rppc_msg_header,
+ * which is the common header for rppc messages.
+ */
+enum rppc_msg_type {
+	RPPC_MSGTYPE_DEVINFO_REQ	= 0,
+	RPPC_MSGTYPE_DEVINFO_RESP	= 1,
+	RPPC_MSGTYPE_FUNCTION_QUERY	= 2,
+	RPPC_MSGTYPE_FUNCTION_INFO	= 3,
+	RPPC_MSGTYPE_CREATE_REQ		= 6,
+	RPPC_MSGTYPE_CREATE_RESP	= 8,
+	RPPC_MSGTYPE_DELETE_REQ		= 4,
+	RPPC_MSGTYPE_DELETE_RESP	= 7,
+	RPPC_MSGTYPE_FUNCTION_CALL	= 5,
+	RPPC_MSGTYPE_FUNCTION_RET	= 9,
+	RPPC_MSGTYPE_ERROR = 10,
+	RPPC_MSGTYPE_MAX
+};
+
+/**
+ * enum rppc_infotype - function information query type
+ * @RPPC_INFOTYPE_FUNC_SIGNATURE: function signature
+ * @RPPC_INFOTYPE_NUM_CALLS: the number of times a function has been invoked
+ * @RPPC_INFOTYPE_MAX: limit value to define the maximum info type
+ *
+ * This enum is used for identifying the type of information queried
+ * from the remote processor. Only RPPC_INFOTYPE_FUNC_SIGNATURE is
+ * currently used.
+ */
+enum rppc_infotype {
+	RPPC_INFOTYPE_FUNC_SIGNATURE = 1,
+	RPPC_INFOTYPE_NUM_CALLS,
+	RPPC_INFOTYPE_MAX
+};
+
+/**
+ * struct rppc_instance_handle - rppc instance information
+ * @endpoint_address: end-point address of the remote server instance
+ * @status: status of the request
+ *
+ * This structure indicates the format of the message payload exchanged
+ * between the host and the remote sides for messages pertaining to
+ * creation and deletion of the remote server instances. This payload
+ * is associated with messages of type RPPC_MSGTYPE_CREATE_RESP and
+ * RPPC_MSGTYPE_DELETE_RESP.
+ */
+struct rppc_instance_handle {
+	u32 endpoint_address;
+	u32 status;
+} __packed;
+
+/**
+ * struct rppc_param_signature - parameter descriptor
+ * @direction: input or output classifier, see enum rppc_param_direction
+ * @type: parameter data type, see enum rppc_param_type
+ * @count: used to do some basic sanity checking on array bounds
+ */
+struct rppc_param_signature {
+	u32 direction;
+	u32 type;
+	u32 count;
+};
+
+/**
+ * struct rppc_func_signature - remote function signature descriptor
+ * @name: name of the function
+ * @num_param: number of parameters to the function
+ * @params: parameter descriptors for each of the parameters
+ *
+ * This structure contains the indicates the format of the message payload
+ * exchanged between the host and the remote sides for messages pertaining
+ * to creation and deletion of the remote server instances. This payload
+ * is associated with messages of type RPPC_MSGTYPE_CREATE_RESP and
+ * RPPC_MSGTYPE_FUNCTION_INFO.
+ */
+struct rppc_func_signature {
+	char name[RPPC_MAX_CHANNEL_NAMELEN];
+	u32 num_param;
+	struct rppc_param_signature params[RPPC_MAX_NUM_PARAMS + 1];
+};
+
+/**
+ * struct rppc_query_function - function info packet structure
+ * @info_type: type of the function information requested, see
+ *	       enum rppc_infotype
+ * @fxn_id: function identifier on this specific remote server instance
+ * @num_calls: number of types function is invoked, filled in during a response
+ *	       (only valid for rppc_infotype RPPC_INFOTYPE_NUM_CALLS)
+ * @signature: the signature of the function including its return type,
+ *	       parameters and their description
+ *	       (only valid for rppc_infotype RPPC_INFOTYPE_FUNC_SIGNATURE)
+ *
+ * This structure indicates the format of the message payload exchanged
+ * between the host and the remote sides for messages pertaining to
+ * information about each function supported by the remote server instance.
+ * This payload is associated with messages of type RPPC_MSGTYPE_FUNCTION_QUERY
+ * and RPPC_MSGTYPE_FUNCTION_INFO.
+ */
+struct rppc_query_function {
+	u32 info_type;
+	u32 fxn_id;
+	union {
+		u32 num_calls;
+		struct rppc_func_signature signature;
+	} info;
+};
+
+/**
+ * enum rppc_translate_direction - pointer translation direction
+ * @RPPC_UVA_TO_RPA: user virtual address to remote device address translation
+ * @RPPC_RPA_TO_UVA: remote device address to user virtual address translation
+ *
+ * An enum used for identifying the rppc function message direction, whether
+ * it is going to the remote side, or is a response from the remote side. This
+ * is used in translating the pointers from the host-side to the remote-side
+ * and vice versa depending on the packet direction.
+ */
+enum rppc_translate_direction {
+	RPPC_UVA_TO_RPA,
+	RPPC_RPA_TO_UVA,
+};
+
+/**
+ * enum rppc_state - rppc instance state
+ * @RPPC_STATE_DISCONNECTED: uninitialized state
+ * @RPPC_STATE_CONNECTED: initialized state
+ * @RPPC_STATE_STALE: invalid or stale state
+ * @RPPC_STATE_MAX: limit value for the different state values
+ *
+ * This enum value is used to define the status values of a
+ * rppc_instance object.
+ */
+enum rppc_state {
+	RPPC_STATE_DISCONNECTED,
+	RPPC_STATE_CONNECTED,
+	RPPC_STATE_STALE,
+	RPPC_STATE_MAX
+};
+
+/**
+ * struct rppc_device_info - rppc remote server device info
+ * @num_funcs: number of functions supported by a remote server instance
+ *
+ * This structure indicates the format of the message payload responded by
+ * the remote side upon a request for message type RPPC_MSGTYPE_DEVINFO_REQ.
+ * This payload is associated with messages of type RPPC_MSGTYPE_DEVINFO_RESP.
+ */
+struct rppc_device_info {
+	u32 num_funcs;
+};
+
+/**
+ * struct rppc_error - rppc error information
+ * @endpoint_address: end-point address of the remote server instance
+ * @status: status of the request
+ *
+ * This structure indicates the format of the message payload exchanged
+ * between the host and the remote sides for error messages. This payload
+ * is associated with messages of type RPPC_MSGTYPE_ERROR
+ * XXX: check if this is needed still, not used anywhere at present
+ */
+struct rppc_error {
+	u32 endpoint_address;
+	u32 status;
+} __packed;
+
+/**
+ * struct rppc_param_data - marshalled parameter data structure
+ * @size: size of the parameter data type
+ * @data: actual parameter data
+ *
+ * Each function parameter is marshalled in this form between the host
+ * and remote sides. The @data field would contain the actual value of
+ * of the parameter if it is a scalar argument type, or the remote-side
+ * device address (virtual address) of the pointer if the argument is
+ * of pointer type.
+ */
+struct rppc_param_data {
+	size_t size;
+	size_t data;
+} __packed;
+
+/**
+ * struct rppc_msg_header - generic rpmsg rpc message header
+ * @msg_type: message type, see enum rppc_msg_type
+ * @msg_len: length of the message payload in bytes
+ * @msg_data: the actual message payload (depends on message type)
+ *
+ * All RPPC messages will start with this common header (which will begin
+ * right after the standard rpmsg header ends).
+ */
+struct rppc_msg_header {
+	u32 msg_type;
+	u32 msg_len;
+	u8  msg_data[0];
+} __packed;
+
+#define RPPC_PAYLOAD(ptr, type)	\
+		((struct type *)&(ptr)[sizeof(struct rppc_msg_header)])
+
+/* from rpmsg_rpc.c */
+dev_addr_t rppc_local_to_remote_da(struct rppc_instance *rpc, phys_addr_t pa);
+
+/* from rpmsg_rpc_dmabuf.c */
+struct rppc_dma_buf *rppc_alloc_dmabuf(struct rppc_instance *rpc,
+				       int fd, bool autoreg);
+struct rppc_dma_buf *rppc_find_dmabuf(struct rppc_instance *rpc, int fd);
+int rppc_free_dmabuf(int id, void *p, void *data);
+dev_addr_t rppc_buffer_lookup(struct rppc_instance *rpc, virt_addr_t uva,
+			      virt_addr_t buva, int fd);
+int rppc_xlate_buffers(struct rppc_instance *rpc, struct rppc_function *func,
+		       int direction);
+
+/* from rpmsg_rpc_sysfs.c */
+int rppc_create_sysfs(struct rppc_device *rppcdev);
+int rppc_remove_sysfs(struct rppc_device *rppcdev);
+
+#endif
diff -Naur --no-dereference a/drivers/rpmsg/rpmsg_rpc_sysfs.c b/drivers/rpmsg/rpmsg_rpc_sysfs.c
--- a/drivers/rpmsg/rpmsg_rpc_sysfs.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/rpmsg/rpmsg_rpc_sysfs.c	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,247 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Remote Processor Procedure Call Driver
+ *
+ * Copyright (C) 2012-2019 Texas Instruments Incorporated - http://www.ti.com/
+ *	Erik Rainey <erik.rainey@ti.com>
+ *	Suman Anna <s-anna@ti.com>
+ */
+
+#include <linux/device.h>
+#include <linux/slab.h>
+#include <linux/rpmsg_rpc.h>
+
+#include "rpmsg_rpc_internal.h"
+
+static ssize_t show_numfuncs(struct device *dev, struct device_attribute *attr,
+			     char *buf)
+{
+	struct rppc_device *rppcdev = dev_get_drvdata(dev);
+
+	return snprintf(buf, PAGE_SIZE, "%u\n", rppcdev->num_funcs - 1);
+}
+
+static ssize_t set_type_c(char *buf, uint32_t len,
+			  struct rppc_param_signature *psig)
+{
+	char *isptr = (psig->type & RPPC_PARAM_PTR ? " *" : "");
+
+	switch (psig->type & RPPC_PARAM_MASK) {
+	case RPPC_PARAM_S08:
+		return snprintf(buf, len, "int8_t%s", isptr);
+	case RPPC_PARAM_U08:
+		return snprintf(buf, len, "uint8_t%s", isptr);
+	case RPPC_PARAM_S16:
+		return snprintf(buf, len, "int16_t%s", isptr);
+	case RPPC_PARAM_U16:
+		return snprintf(buf, len, "uint16_t%s", isptr);
+	case RPPC_PARAM_S32:
+		return snprintf(buf, len, "int32_t%s", isptr);
+	case RPPC_PARAM_U32:
+		return snprintf(buf, len, "uint32_t%s", isptr);
+	case RPPC_PARAM_S64:
+		return snprintf(buf, len, "int64_t%s", isptr);
+	case RPPC_PARAM_U64:
+		return snprintf(buf, len, "uint64_t%s", isptr);
+	default:
+		return snprintf(buf, len, "<unknown>%s", isptr);
+	}
+}
+
+static ssize_t set_type_doxy(char *buf, uint32_t len,
+			     struct rppc_param_signature *psig)
+{
+	char *isptr = (psig->type & RPPC_PARAM_PTR ? " *" : "");
+	char dir[10];
+
+	switch (psig->direction) {
+	case RPPC_PARAMDIR_IN:
+		snprintf(dir, sizeof(dir), "[in]");
+		break;
+	case RPPC_PARAMDIR_OUT:
+		snprintf(dir, sizeof(dir), "[out]");
+		break;
+	case RPPC_PARAMDIR_BI:
+		snprintf(dir, sizeof(dir), "[in,out]");
+		break;
+	default:
+		snprintf(dir, sizeof(dir), "[unknown]");
+		break;
+	}
+
+	switch (psig->type & RPPC_PARAM_MASK) {
+	case RPPC_PARAM_S08:
+		return snprintf(buf, len, "%s int8_t%s", dir, isptr);
+	case RPPC_PARAM_U08:
+		return snprintf(buf, len, "%s uint8_t%s", dir, isptr);
+	case RPPC_PARAM_S16:
+		return snprintf(buf, len, "%s int16_t%s", dir, isptr);
+	case RPPC_PARAM_U16:
+		return snprintf(buf, len, "%s uint16_t%s", dir, isptr);
+	case RPPC_PARAM_S32:
+		return snprintf(buf, len, "%s int32_t%s", dir, isptr);
+	case RPPC_PARAM_U32:
+		return snprintf(buf, len, "%s uint32_t%s", dir, isptr);
+	case RPPC_PARAM_S64:
+		return snprintf(buf, len, "%s int64_t%s", dir, isptr);
+	case RPPC_PARAM_U64:
+		return snprintf(buf, len, "%s uint64_t%s", dir, isptr);
+	default:
+		return snprintf(buf, len, "%s <unknown>%s", dir, isptr);
+	}
+}
+
+static ssize_t show_c_function(struct device *dev,
+			       struct device_attribute *attr, char *buf)
+{
+	struct rppc_device *rppcdev = dev_get_drvdata(dev);
+	char return_value[11]; /* longest string is strlen("uintXX_t *") = 10 */
+	char parameters[110]; /* longest string * 10 + 9(,) */
+	char comment[300];
+	int p;
+	ssize_t pidx = 0;
+	ssize_t cidx = 0;
+	__u32 index = 0;
+
+	if (sscanf(attr->attr.name, "c_function%u\n", &index) != 1)
+		return -EIO;
+
+	memset(return_value, 0, sizeof(return_value));
+	memset(parameters, 0, sizeof(parameters));
+
+	strcpy(return_value, "void");
+	strcpy(parameters, "void");
+	cidx += snprintf(&comment[cidx], sizeof(comment) - cidx, "/**\n");
+	cidx += snprintf(&comment[cidx], sizeof(comment) - cidx,
+		" * \\fn %s\n", rppcdev->signatures[index].name);
+	for (p = 0; p < rppcdev->signatures[index].num_param; p++) {
+		if (p == 0) {
+			set_type_c(return_value, sizeof(return_value),
+				   &rppcdev->signatures[index].params[0]);
+			cidx += snprintf(&comment[cidx], sizeof(comment) - cidx,
+					" * \\return %s\n", return_value);
+		} else {
+			pidx += set_type_c(&parameters[pidx],
+					sizeof(parameters) - pidx,
+					&rppcdev->signatures[index].params[p]);
+			if (p != rppcdev->signatures[index].num_param - 1)
+				parameters[pidx++] = ',';
+			cidx += snprintf(&comment[cidx], sizeof(comment) - cidx,
+						" * \\param ");
+			cidx += set_type_doxy(&comment[cidx],
+					sizeof(comment) - cidx,
+					&rppcdev->signatures[index].params[p]);
+			cidx += snprintf(&comment[cidx], sizeof(comment) - cidx,
+						"\n");
+		}
+	}
+	if (p <= 1)
+		pidx += strlen("void");
+	if (pidx < sizeof(parameters))
+		parameters[pidx] = '\0';
+	cidx += snprintf(&comment[cidx], sizeof(comment) - cidx, " */");
+	return snprintf(buf, PAGE_SIZE, "%s\nextern \"C\" %s %s(%s);\n",
+			comment, return_value, rppcdev->signatures[index].name,
+			parameters);
+}
+
+static struct device_attribute rppc_attrs[] = {
+	__ATTR(num_funcs, 0444, show_numfuncs, NULL),
+};
+
+/**
+ * rppc_create_sysfs - Creates the sysfs entry structures for the instance
+ * @rppcdev: the rppc device (remote server instance) handle
+ *
+ * Helper function to create all the sysfs entries associated with a rppc
+ * device. Each device is associated with a number of remote procedure
+ * functions. The number of such functions and the signatures of those
+ * functions are created in sysfs. Function is invoked after querying
+ * the remote side about the supported functions on this device.
+ *
+ * The entries are split into a set of static entries, which are common
+ * between all rppc devices, and a set of dynamic entries specific to
+ * each rppc device.
+ *
+ * Return: 0 on success, or an appropriate error code otherwise
+ */
+int rppc_create_sysfs(struct rppc_device *rppcdev)
+{
+	int i;
+	int ret;
+
+	rppcdev->sig_attr = kcalloc(rppcdev->num_funcs,
+				    sizeof(*rppcdev->sig_attr), GFP_KERNEL);
+	if (!rppcdev->sig_attr)
+		return -ENOMEM;
+
+	for (i = 0; i < ARRAY_SIZE(rppc_attrs); i++) {
+		ret = device_create_file(rppcdev->dev, &rppc_attrs[i]);
+		if (ret) {
+			dev_err(rppcdev->dev, "failed to create sysfs entry\n");
+			goto clean_static_entries;
+		}
+	}
+
+	for (i = 1; i < rppcdev->num_funcs; i++) {
+		sysfs_attr_init(&rppcdev->sig_attr[i].attr);
+		rppcdev->sig_attr[i].attr.name =
+				kzalloc(RPPC_MAX_FUNC_NAMELEN, GFP_KERNEL);
+		if (!rppcdev->sig_attr[i].attr.name) {
+			ret = -ENOMEM;
+			goto clean_dynamic_entries;
+		}
+		snprintf((char *)rppcdev->sig_attr[i].attr.name,
+			 RPPC_MAX_FUNC_NAMELEN, "c_function%u", i);
+		rppcdev->sig_attr[i].attr.mode = 0444;
+		rppcdev->sig_attr[i].show = show_c_function;
+		rppcdev->sig_attr[i].store = NULL;
+
+		ret = device_create_file(rppcdev->dev, &rppcdev->sig_attr[i]);
+		if (ret) {
+			dev_err(rppcdev->dev, "failed to create sysfs function entry (%d)\n",
+				ret);
+			goto clean_dynamic_entries;
+		}
+	}
+	return 0;
+
+clean_dynamic_entries:
+	while (i-- > 1) {
+		device_remove_file(rppcdev->dev, &rppcdev->sig_attr[i]);
+		kfree(rppcdev->sig_attr[i].attr.name);
+	}
+	i = ARRAY_SIZE(rppc_attrs);
+clean_static_entries:
+	while (i-- > 0)
+		device_remove_file(rppcdev->dev, &rppc_attrs[i]);
+	kfree(rppcdev->sig_attr);
+	return ret;
+}
+
+/**
+ * rppc_remove_sysfs: Removes the sysfs entry structures for the instance
+ * @rppcdev: the rppc device (remote server instance) handle
+ *
+ * Helper function to remove all the sysfs entries associated with the
+ * rppc device.
+ *
+ * Return: 0 on success, or an appropriate error code otherwise
+ */
+int rppc_remove_sysfs(struct rppc_device *rppcdev)
+{
+	int i;
+
+	if (rppcdev->sig_attr) {
+		for (i = 1; i < rppcdev->num_funcs; i++) {
+			device_remove_file(rppcdev->dev, &rppcdev->sig_attr[i]);
+			kfree(rppcdev->sig_attr[i].attr.name);
+		}
+	}
+	kfree(rppcdev->sig_attr);
+
+	for (i = 0; i < ARRAY_SIZE(rppc_attrs); i++)
+		device_remove_file(rppcdev->dev, &rppc_attrs[i]);
+
+	return 0;
+}
diff -Naur --no-dereference a/drivers/rpmsg/virtio_rpmsg_bus.c b/drivers/rpmsg/virtio_rpmsg_bus.c
--- a/drivers/rpmsg/virtio_rpmsg_bus.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/rpmsg/virtio_rpmsg_bus.c	2020-06-19 21:00:31.424227716 -0400
@@ -26,6 +26,7 @@
 #include <linux/rpmsg.h>
 #include <linux/mutex.h>
 #include <linux/of_device.h>
+#include <linux/rpmsg/virtio_rpmsg.h>
 
 #include "rpmsg_internal.h"
 
@@ -73,26 +74,6 @@
 #define VIRTIO_RPMSG_F_NS	0 /* RP supports name service notifications */
 
 /**
- * struct rpmsg_hdr - common header for all rpmsg messages
- * @src: source address
- * @dst: destination address
- * @reserved: reserved for future use
- * @len: length of payload (in bytes)
- * @flags: message flags
- * @data: @len bytes of message payload data
- *
- * Every message sent(/received) on the rpmsg bus begins with this header.
- */
-struct rpmsg_hdr {
-	u32 src;
-	u32 dst;
-	u32 reserved;
-	u16 len;
-	u16 flags;
-	u8 data[0];
-} __packed;
-
-/**
  * struct rpmsg_ns_msg - dynamic name service announcement message
  * @name: name of remote service that is published
  * @addr: address of remote service that is published
@@ -111,6 +92,23 @@
 } __packed;
 
 /**
+ * struct rpmsg_ns_msg_ext - dynamic name service announcement message v2
+ * @name: name of remote service that is published
+ * @desc: description of remote service
+ * @addr: address of remote service that is published
+ * @flags: indicates whether service is created or destroyed
+ *
+ * Interchangeable nameservice message with rpmsg_ns_msg. This one has
+ * the addition of the desc field for extra flexibility.
+ */
+struct rpmsg_ns_msg_ext {
+	char name[RPMSG_NAME_SIZE];
+	char desc[RPMSG_NAME_SIZE];
+	u32 addr;
+	u32 flags;
+} __packed;
+
+/**
  * enum rpmsg_ns_flags - dynamic name service announcement flags
  *
  * @RPMSG_NS_CREATE: a new remote service was just created
@@ -188,6 +186,7 @@
 
 /**
  * rpmsg_sg_init - initialize scatterlist according to cpu address location
+ * @vrp: virtual remoteproc structure used with this buffer
  * @sg: scatterlist to fill
  * @cpu_addr: virtual address of the buffer
  * @len: buffer length
@@ -196,9 +195,17 @@
  * location (in vmalloc or in kernel).
  */
 static void
-rpmsg_sg_init(struct scatterlist *sg, void *cpu_addr, unsigned int len)
+rpmsg_sg_init(struct virtproc_info *vrp, struct scatterlist *sg,
+	      void *cpu_addr, unsigned int len)
 {
-	if (is_vmalloc_addr(cpu_addr)) {
+	if (of_machine_is_compatible("ti,keystone")) {
+		unsigned long offset = cpu_addr - vrp->rbufs;
+		dma_addr_t dma_addr = vrp->bufs_dma + offset;
+
+		sg_init_table(sg, 1);
+		sg_set_page(sg, pfn_to_page(PHYS_PFN(dma_addr)), len,
+			    offset_in_page(dma_addr));
+	} else if (is_vmalloc_addr(cpu_addr)) {
 		sg_init_table(sg, 1);
 		sg_set_page(sg, vmalloc_to_page(cpu_addr), len,
 			    offset_in_page(cpu_addr));
@@ -268,6 +275,9 @@
 		goto free_ept;
 	}
 	ept->addr = id;
+	ept->cb_lockdep_class = ((ept->addr == RPMSG_NS_ADDR) ?
+				 RPMSG_LOCKDEP_SUBCLASS_NS :
+				 RPMSG_LOCKDEP_SUBCLASS_NORMAL);
 
 	mutex_unlock(&vrp->endpoints_lock);
 
@@ -308,7 +318,7 @@
 	mutex_unlock(&vrp->endpoints_lock);
 
 	/* make sure in-flight inbound messages won't invoke cb anymore */
-	mutex_lock(&ept->cb_lock);
+	mutex_lock_nested(&ept->cb_lock, ept->cb_lockdep_class);
 	ept->cb = NULL;
 	mutex_unlock(&ept->cb_lock);
 
@@ -402,8 +412,9 @@
 	if (tmp) {
 		/* decrement the matched device's refcount back */
 		put_device(tmp);
-		dev_err(dev, "channel %s:%x:%x already exist\n",
-				chinfo->name, chinfo->src, chinfo->dst);
+		dev_err(dev, "channel %s:%s:%x:%x already exist\n",
+			chinfo->name, chinfo->desc,
+			chinfo->src, chinfo->dst);
 		return NULL;
 	}
 
@@ -419,6 +430,7 @@
 	rpdev->src = chinfo->src;
 	rpdev->dst = chinfo->dst;
 	rpdev->ops = &virtio_rpmsg_ops;
+	strncpy(rpdev->desc, chinfo->desc, RPMSG_NAME_SIZE);
 
 	/*
 	 * rpmsg server channels has predefined local address (for now),
@@ -626,7 +638,7 @@
 			 msg, sizeof(*msg) + msg->len, true);
 #endif
 
-	rpmsg_sg_init(&sg, msg, sizeof(*msg) + len);
+	rpmsg_sg_init(vrp, &sg, msg, sizeof(*msg) + len);
 
 	mutex_lock(&vrp->tx_lock);
 
@@ -736,7 +748,7 @@
 
 	if (ept) {
 		/* make sure ept->cb doesn't go away while we use it */
-		mutex_lock(&ept->cb_lock);
+		mutex_lock_nested(&ept->cb_lock, ept->cb_lockdep_class);
 
 		if (ept->cb)
 			ept->cb(ept->rpdev, msg->data, msg->len, ept->priv,
@@ -750,7 +762,7 @@
 		dev_warn(dev, "msg received with no recipient\n");
 
 	/* publish the real size of the buffer */
-	rpmsg_sg_init(&sg, msg, vrp->buf_size);
+	rpmsg_sg_init(vrp, &sg, msg, vrp->buf_size);
 
 	/* add the buffer back to the remote processor's virtqueue */
 	err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, msg, GFP_KERNEL);
@@ -816,18 +828,29 @@
 		       void *priv, u32 src)
 {
 	struct rpmsg_ns_msg *msg = data;
+	struct rpmsg_ns_msg_ext *msg_ext = data;
 	struct rpmsg_device *newch;
 	struct rpmsg_channel_info chinfo;
 	struct virtproc_info *vrp = priv;
 	struct device *dev = &vrp->vdev->dev;
 	int ret;
+	u32 addr;
+	u32 flags;
 
 #if defined(CONFIG_DYNAMIC_DEBUG)
 	dynamic_hex_dump("NS announcement: ", DUMP_PREFIX_NONE, 16, 1,
 			 data, len, true);
 #endif
 
-	if (len != sizeof(*msg)) {
+	if (len == sizeof(*msg)) {
+		addr = msg->addr;
+		flags = msg->flags;
+		chinfo.desc[0] = '\0';
+	} else if (len == sizeof(*msg_ext)) {
+		addr = msg_ext->addr;
+		flags = msg_ext->flags;
+		strncpy(chinfo.desc, msg_ext->desc, sizeof(chinfo.desc));
+	} else if (len != sizeof(*msg)) {
 		dev_err(dev, "malformed ns msg (%d)\n", len);
 		return -EINVAL;
 	}
@@ -847,14 +870,14 @@
 	msg->name[RPMSG_NAME_SIZE - 1] = '\0';
 
 	dev_info(dev, "%sing channel %s addr 0x%x\n",
-		 msg->flags & RPMSG_NS_DESTROY ? "destroy" : "creat",
-		 msg->name, msg->addr);
+		 flags & RPMSG_NS_DESTROY ? "destroy" : "creat",
+		 msg->name, addr);
 
 	strncpy(chinfo.name, msg->name, sizeof(chinfo.name));
 	chinfo.src = RPMSG_ADDR_ANY;
-	chinfo.dst = msg->addr;
+	chinfo.dst = addr;
 
-	if (msg->flags & RPMSG_NS_DESTROY) {
+	if (flags & RPMSG_NS_DESTROY) {
 		ret = rpmsg_unregister_device(&vrp->vdev->dev, &chinfo);
 		if (ret)
 			dev_err(dev, "rpmsg_destroy_channel failed: %d\n", ret);
@@ -920,7 +943,7 @@
 		goto vqs_del;
 	}
 
-	dev_dbg(&vdev->dev, "buffers: va %p, dma %pad\n",
+	dev_dbg(&vdev->dev, "buffers: va %pK, dma %pad\n",
 		bufs_va, &vrp->bufs_dma);
 
 	/* half of the buffers is dedicated for RX */
@@ -934,7 +957,7 @@
 		struct scatterlist sg;
 		void *cpu_addr = vrp->rbufs + i * vrp->buf_size;
 
-		rpmsg_sg_init(&sg, cpu_addr, vrp->buf_size);
+		rpmsg_sg_init(vrp, &sg, cpu_addr, vrp->buf_size);
 
 		err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, cpu_addr,
 					  GFP_KERNEL);
diff -Naur --no-dereference a/drivers/rpmsg-kdrv/Kconfig b/drivers/rpmsg-kdrv/Kconfig
--- a/drivers/rpmsg-kdrv/Kconfig	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/rpmsg-kdrv/Kconfig	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,35 @@
+# SPDX-License-Identifier: GPL-2.0
+
+menu "Rpmsg virtual device drivers"
+
+# RPMSG always gets selected by whoever wants it
+config RPMSG_KDRV
+	tristate "RPMSG virtual device interface"
+	select RPMSG
+	help
+	  Say Y here enable support for RPMSG based remote devices, usually
+	  exported by a firmware running rpmsg stack and remote_device stack.
+	  This feature enables the framework for para-virtualizing entire H/W
+	  or specific resources of a hardware
+
+
+config RPMSG_KDRV_DISPLAY
+	tristate "RPMSG virtual display device support"
+	select RPMSG_KDRV
+	help
+	  Say Y here to enable support for remote device based display
+	  virtualization. This setup expects that the display will be driven
+	  by a remoteproc and DRM driver will be able to use display features
+	  using remote_device framework
+
+config RPMSG_KDRV_ETH_SWITCH
+	tristate "RPMSG virtual eth switch device support"
+	select RPMSG_KDRV
+	default m
+	help
+	  Say Y here to enable support for remote device based Eth switch
+	  virtualization. This setup expects that the Eth switch will be driven
+	  by a remoteproc and virtual Network device will be able to use
+	  Eth switch features using remote_device framework
+
+endmenu
diff -Naur --no-dereference a/drivers/rpmsg-kdrv/Makefile b/drivers/rpmsg-kdrv/Makefile
--- a/drivers/rpmsg-kdrv/Makefile	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/rpmsg-kdrv/Makefile	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_RPMSG_KDRV)		+= rpmsg_kdrv.o
+obj-$(CONFIG_RPMSG_KDRV_DISPLAY)	+= rpmsg_kdrv_display.o
+obj-$(CONFIG_RPMSG_KDRV_ETH_SWITCH)	+= rpmsg_kdrv_switch.o
diff -Naur --no-dereference a/drivers/rpmsg-kdrv/rpmsg_kdrv.c b/drivers/rpmsg-kdrv/rpmsg_kdrv.c
--- a/drivers/rpmsg-kdrv/rpmsg_kdrv.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/rpmsg-kdrv/rpmsg_kdrv.c	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,742 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Subhajit Paul <subahjit_paul@ti.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/list.h>
+#include <linux/mutex.h>
+
+#include <linux/rpmsg.h>
+#include <linux/rpmsg-remotedev/rpmsg-remotedev.h>
+#include "shared/rpmsg-kdrv-transport.h"
+#include "rpmsg_kdrv_internal.h"
+
+struct rpmsg_kdrv_priv {
+	struct rpmsg_device *rpdev;
+
+	struct idr message_idr;
+	struct mutex message_lock;
+
+	int num_raw_devices;
+	struct rpmsg_kdrv_init_device_info raw_devices[RPMSG_KDRV_TP_MAX_DEVICES];
+	void *raw_device_data[RPMSG_KDRV_TP_MAX_DEVICES];
+	int raw_device_data_size[RPMSG_KDRV_TP_MAX_DEVICES];
+};
+
+struct rpmsg_kdrv_ctx {
+	struct rpmsg_device *rpdev;
+	bool wait_for_response;
+	request_cb_t callback;
+	void *cb_data;
+	bool response_recv;
+	struct wait_queue_head response_wq;
+
+	struct rpmsg_kdrv_device_header *dev_hdr;
+	void *req;
+	void *resp;
+	int req_size;
+	int resp_size;
+};
+
+static struct bus_type rpmsg_kdrv_bus;
+
+#define to_rpmsg_kdrv_device(d) container_of(d, struct rpmsg_kdrv_device, dev)
+#define to_rpmsg_kdrv_driver(d) container_of(d, struct rpmsg_kdrv_driver, drv)
+
+static int rpmsg_kdrv_match_id(struct device *dev, void *data)
+{
+	uint32_t *idptr = data;
+	struct rpmsg_kdrv_device *kddev = container_of(dev, struct rpmsg_kdrv_device, dev);
+
+	if (kddev->device_id == *idptr)
+		return 1;
+	return 0;
+}
+
+static int rpmsg_kdrv_match_remotedev(struct device *dev, void *data)
+{
+	struct rpmsg_remotedev *rdev = data;
+	struct rpmsg_kdrv_device *kddev = container_of(dev, struct rpmsg_kdrv_device, dev);
+
+	if (kddev->remotedev == rdev)
+		return 1;
+	return 0;
+}
+
+static int rpmsg_kdrv_match_name(struct device *dev, void *data)
+{
+	const char *name = data;
+	struct rpmsg_kdrv_device *kddev = container_of(dev, struct rpmsg_kdrv_device, dev);
+
+	if (strcmp(kddev->device_name, name) == 0)
+		return 1;
+	return 0;
+}
+
+int rpmsg_kdrv_register_driver(struct rpmsg_kdrv_driver *drv)
+{
+	int ret;
+
+	drv->drv.bus = &rpmsg_kdrv_bus;
+	drv->drv.owner = THIS_MODULE;
+
+	ret = driver_register(&drv->drv);
+	if (ret)
+		pr_err("%s: driver_register failed\n", __func__);
+
+	return ret;
+}
+EXPORT_SYMBOL(rpmsg_kdrv_register_driver);
+
+static void rpmsg_kdrv_driver_handle_data(struct rpmsg_device *rpdev, void *data, int len, void *private, u32 src)
+{
+	struct device *dev;
+	struct rpmsg_kdrv_device_header *hdr = data;
+	struct rpmsg_kdrv_device *kddev = NULL;
+	struct rpmsg_kdrv_driver *kddrv = NULL;
+	void *message;
+	int message_size;
+	uint32_t msg_device_id;
+	int ret;
+
+	msg_device_id = hdr->device_id;
+	dev = bus_find_device(&rpmsg_kdrv_bus, NULL, &(msg_device_id), rpmsg_kdrv_match_id);
+	if (!dev) {
+		dev_err(&rpdev->dev, "%s: message received for unknown device\n", __func__);
+		return;
+	}
+	kddev = container_of(dev, struct rpmsg_kdrv_device, dev);
+	kddrv = to_rpmsg_kdrv_driver(kddev->dev.driver);
+	if (!kddrv) {
+		dev_err(&rpdev->dev, "%s: message received for device with no driver\n", __func__);
+		return;
+	}
+
+	message = (void *)(&hdr[1]);
+	message_size = len - sizeof(*hdr);
+	ret = kddrv->callback(kddev, message, message_size);
+	if (ret)
+		dev_err(&rpdev->dev, "%s: message callback returns %d\n", __func__, ret);
+
+}
+
+static int rpmsg_kdrv_connect(struct rpmsg_device *rpdev, struct rpmsg_kdrv_device *kddev)
+{
+	int ret;
+	struct rpmsg_kdrv_init_connect_message *connect_req;
+
+	connect_req = devm_kzalloc(&rpdev->dev, sizeof(*connect_req), GFP_KERNEL);
+	if (!connect_req)
+		return -ENOMEM;
+
+	connect_req->header.message_type = RPMSG_KDRV_TP_INIT_CONNECT_MESSAGE;
+	connect_req->device_id = kddev->device_id;
+
+	ret = rpmsg_kdrv_send_message(rpdev, RPMSG_KDRV_TP_DEVICE_ID_INIT,
+			connect_req, sizeof(*connect_req));
+
+	devm_kfree(&rpdev->dev, connect_req);
+	return ret;
+}
+
+static int rpmsg_kdrv_disconnect(struct rpmsg_device *rpdev, struct rpmsg_kdrv_device *kddev)
+{
+	int ret;
+	struct rpmsg_kdrv_init_disconnect_message *disconnect_req;
+
+	disconnect_req = devm_kzalloc(&rpdev->dev, sizeof(*disconnect_req), GFP_KERNEL);
+	if (!disconnect_req)
+		return -ENOMEM;
+
+	disconnect_req->header.message_type = RPMSG_KDRV_TP_INIT_DISCONNECT_MESSAGE;
+	disconnect_req->device_id = kddev->device_id;
+
+	ret = rpmsg_kdrv_send_message(rpdev, RPMSG_KDRV_TP_DEVICE_ID_INIT,
+			disconnect_req, sizeof(*disconnect_req));
+
+	devm_kfree(&rpdev->dev, disconnect_req);
+	return ret;
+}
+
+struct rpmsg_remotedev *rpmsg_remotedev_get_named_device(const char *device_name)
+{
+	struct device *dev;
+	struct rpmsg_kdrv_device *kddev = NULL;
+
+	dev = bus_find_device(&rpmsg_kdrv_bus, NULL, (void *)device_name, rpmsg_kdrv_match_name);
+	if (!dev)
+		return ERR_PTR(-EPROBE_DEFER);
+
+	kddev = container_of(dev, struct rpmsg_kdrv_device, dev);
+	if (!kddev->remotedev)
+		return ERR_PTR(-EPROBE_DEFER);
+
+	rpmsg_kdrv_connect(kddev->rpdev, kddev);
+
+	return kddev->remotedev;
+}
+EXPORT_SYMBOL(rpmsg_remotedev_get_named_device);
+
+void rpmsg_remotedev_put_device(struct rpmsg_remotedev *rdev)
+{
+	struct device *dev;
+	struct rpmsg_kdrv_device *kddev = NULL;
+
+	dev = bus_find_device(&rpmsg_kdrv_bus, NULL, (void *)rdev, rpmsg_kdrv_match_remotedev);
+	if (!dev) {
+		pr_err("%s: could not find device for remotedev\n", __func__);
+		return;
+	}
+
+	kddev = container_of(dev, struct rpmsg_kdrv_device, dev);
+
+	rpmsg_kdrv_disconnect(kddev->rpdev, kddev);
+}
+EXPORT_SYMBOL(rpmsg_remotedev_put_device);
+
+static void rpmsg_kdrv_release_device(struct device *dev)
+{
+	struct rpmsg_kdrv_device *kddev = to_rpmsg_kdrv_device(dev);
+
+	dev_dbg(dev, "%s\n", __func__);
+
+	devm_kfree(&kddev->rpdev->dev, kddev);
+}
+
+static struct rpmsg_kdrv_device *rpmsg_kdrv_device_create(struct rpmsg_device *rpdev, int index)
+{
+	struct rpmsg_kdrv_device *kddev = devm_kzalloc(&rpdev->dev, sizeof(*kddev), GFP_KERNEL);
+	struct rpmsg_kdrv_priv *priv = dev_get_drvdata(&rpdev->dev);
+	struct rpmsg_kdrv_init_device_info *dev = &priv->raw_devices[index];
+	int ret;
+
+	if (!kddev) {
+		dev_err(&rpdev->dev, "%s: could not allocate kddev\n", __func__);
+		return NULL;
+	}
+
+	kddev->rpdev = rpdev;
+	kddev->device_id = dev->device_id;
+	kddev->device_type = dev->device_type;
+	kddev->device_data_len = priv->raw_device_data_size[index];
+	kddev->device_data = priv->raw_device_data[index];
+	kddev->device_name = devm_kstrdup(&rpdev->dev, dev->device_name, GFP_KERNEL);
+	if (!kddev->device_name) {
+		dev_err(&rpdev->dev, "%s: could not allocate device name\n", __func__);
+		devm_kfree(&rpdev->dev, kddev);
+		return NULL;
+	}
+
+	kddev->dev.parent = &rpdev->dev;
+	kddev->dev.release = rpmsg_kdrv_release_device;
+	kddev->dev.bus = &rpmsg_kdrv_bus;
+
+	dev_set_name(&kddev->dev, "rpmsg-kdrv-%u-%s", dev->device_id, dev->device_name);
+
+	ret = device_register(&kddev->dev);
+	if (ret) {
+		dev_err(&rpdev->dev, "%s: device_register failed: %d\n", __func__, ret);
+		put_device(&kddev->dev);
+		return NULL;
+	}
+	dev_dbg(&rpdev->dev, "%s: registered new device : %s\n", __func__, dev_name(&kddev->dev));
+
+	return kddev;
+}
+
+static int rpmsg_kdrv_get_devices_cb(void *cb_data, void *req, int req_sz, void *resp, int resp_sz)
+{
+	int i, cnt;
+	struct rpmsg_device *rpdev = cb_data;
+	struct rpmsg_kdrv_priv *priv = dev_get_drvdata(&rpdev->dev);
+	struct rpmsg_kdrv_init_dev_info_response *info_resp = resp;
+	struct rpmsg_kdrv_init_device_info *dev;
+	int ret;
+
+	if (info_resp->header.message_type != RPMSG_KDRV_TP_INIT_DEV_INFO_RESPONSE) {
+		dev_err(&rpdev->dev, "%s: wrong response type\n", __func__);
+		ret = -EINVAL;
+		goto out;
+	}
+
+	for (i = 0; i < info_resp->num_devices; i++) {
+		dev = &info_resp->devices[i];
+		cnt = priv->num_raw_devices;
+
+		priv->raw_device_data_size[cnt] = dev->device_data_len;
+		priv->raw_device_data[cnt] = devm_kzalloc(&rpdev->dev, dev->device_data_len, GFP_KERNEL);
+		if (!priv->raw_device_data[cnt])
+			goto out;
+		memcpy(priv->raw_device_data[cnt],
+				&info_resp->device_data[dev->device_data_offset],
+				dev->device_data_len);
+		memcpy(&priv->raw_devices[cnt], dev, sizeof(*dev));
+		priv->num_raw_devices++;
+
+		dev_dbg(&rpdev->dev, "new device: %s\n", dev->device_name);
+	}
+
+	for (i = 0; i < priv->num_raw_devices; i++)
+		rpmsg_kdrv_device_create(rpdev, i);
+
+out:
+	devm_kfree(&rpdev->dev, req);
+	return ret;
+}
+
+static int rpmsg_kdrv_get_devices(struct rpmsg_device *rpdev)
+{
+	int ret;
+	struct rpmsg_kdrv_init_dev_info_request *info_req;
+
+	info_req = devm_kzalloc(&rpdev->dev, sizeof(*info_req), GFP_KERNEL);
+	if (!info_req)
+		return -ENOMEM;
+
+	info_req->header.message_type = RPMSG_KDRV_TP_INIT_DEV_INFO_REQUEST;
+
+	ret = rpmsg_kdrv_send_request_with_callback(rpdev, RPMSG_KDRV_TP_DEVICE_ID_INIT,
+			info_req, sizeof(*info_req), rpdev, rpmsg_kdrv_get_devices_cb);
+	if (ret)
+		goto nosend;
+
+	return 0;
+
+nosend:
+	devm_kfree(&rpdev->dev, info_req);
+	return ret;
+}
+
+static void rpmsg_kdrv_del_packet_id(struct rpmsg_device *rpdev, int id)
+{
+	struct rpmsg_kdrv_priv *priv = dev_get_drvdata(&rpdev->dev);
+
+	mutex_lock(&priv->message_lock);
+	idr_remove(&priv->message_idr, id);
+	mutex_unlock(&priv->message_lock);
+}
+
+static uint32_t rpmsg_kdrv_new_packet_id(struct rpmsg_device *rpdev, void *data)
+{
+	struct rpmsg_kdrv_priv *priv = dev_get_drvdata(&rpdev->dev);
+	int id;
+
+	mutex_lock(&priv->message_lock);
+	id = idr_alloc(&priv->message_idr, data, RPMSG_KDRV_TP_PACKET_ID_FIRST, 0, GFP_KERNEL);
+	mutex_unlock(&priv->message_lock);
+
+	if (id < 0)
+		return 0;
+
+	return id;
+}
+
+static void rpmsg_kdrv_dev_hdr_delete(struct rpmsg_device *rpdev, struct rpmsg_kdrv_device_header *hdr)
+{
+	rpmsg_kdrv_del_packet_id(rpdev, hdr->packet_id);
+	devm_kfree(&rpdev->dev, hdr);
+}
+
+static struct rpmsg_kdrv_device_header *rpmsg_kdrv_dev_hdr_alloc(struct rpmsg_device *rpdev,
+		int device_id, int size, int pkt_type, int pkt_src, void *msg, int len, struct rpmsg_kdrv_ctx *ctx)
+{
+	struct rpmsg_kdrv_device_header *dev_hdr;
+	void *dst;
+
+	dev_hdr = devm_kzalloc(&rpdev->dev, size, GFP_KERNEL);
+	if (!dev_hdr)
+		return NULL;
+
+	dev_hdr->device_id = device_id;
+	dev_hdr->packet_type = pkt_type;
+	dev_hdr->packet_source = pkt_src;
+	dev_hdr->packet_size = size;
+	dev_hdr->packet_id = RPMSG_KDRV_TP_PACKET_ID_NONE;
+
+
+	dst = (void *)(&dev_hdr[1]);
+	memcpy(dst, msg, len);
+
+	if (pkt_type == RPMSG_KDRV_TP_PACKET_TYPE_MESSAGE)
+		return dev_hdr;
+
+	dev_hdr->packet_id = rpmsg_kdrv_new_packet_id(rpdev, ctx);
+	if (!dev_hdr->packet_id) {
+		devm_kfree(&rpdev->dev, dev_hdr);
+		return NULL;
+	}
+
+	ctx->dev_hdr = dev_hdr;
+
+	return dev_hdr;
+}
+
+static struct rpmsg_kdrv_ctx *rpmsg_kdrv_ctx_alloc(struct rpmsg_device *rpdev, bool blocking,
+		request_cb_t callback, void *cb_data, void *req, int req_size, void *resp, int resp_size)
+{
+	struct rpmsg_kdrv_ctx *ctx;
+
+	ctx = devm_kzalloc(&rpdev->dev, sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return NULL;
+
+	ctx->rpdev = rpdev;
+	if (blocking) {
+		ctx->wait_for_response = true;
+		ctx->response_recv = false;
+		init_waitqueue_head(&ctx->response_wq);
+	} else {
+		ctx->wait_for_response = false;
+		ctx->callback = callback;
+	}
+
+	ctx->cb_data = cb_data;
+	ctx->req = req;
+	ctx->req_size = req_size;
+	ctx->resp = resp;
+	ctx->resp_size = resp_size;
+
+	return ctx;
+}
+
+static int rpmsg_kdrv_send_packet(struct rpmsg_device *rpdev, void *data, int len)
+{
+	return rpmsg_send(rpdev->ept, data, len);
+}
+
+/*
+ * rpmsg_kdrv_send_request_with_callback
+ *
+ * Send a message where
+ * a) the caller does not block
+ * b) the caller expects multile responses
+ *
+ * The callback function must return
+ * a) RRMSG_KDRV_CALLBACK_DONE when no more responses are expected
+ * b) RPMSG_KDRV_CALLBACK_MORE when more responses are awaited
+ *
+ * The caller is expected to destroy message when it does not
+ * expect any more responses
+ */
+int rpmsg_kdrv_send_request_with_callback(struct rpmsg_device *rpdev, uint32_t device_id,
+		void *message, uint32_t message_size,
+		void *cb_data, request_cb_t callback)
+{
+	struct rpmsg_kdrv_device_header *dev_hdr;
+	int total_size = message_size + sizeof(*dev_hdr);
+	struct rpmsg_kdrv_ctx *ctx = NULL;
+	int ret;
+
+	ctx = rpmsg_kdrv_ctx_alloc(rpdev, false, callback, cb_data, message, message_size, NULL, 0);
+	if (!ctx) {
+		dev_err(&rpdev->dev, "%s: ctx allocation failed\n", __func__);
+		return -ENOMEM;
+	}
+
+	dev_hdr = rpmsg_kdrv_dev_hdr_alloc(rpdev, device_id, total_size,
+			RPMSG_KDRV_TP_PACKET_TYPE_REQUEST,
+			RPMSG_KDRV_TP_PACKET_SOURCE_CLIENT,
+			message, message_size,
+			ctx);
+	if (!dev_hdr) {
+		dev_err(&rpdev->dev, "%s: device header allocation failed\n", __func__);
+		ret = -ENOMEM;
+		goto dev_hdr_fail;
+	}
+
+	ret = rpmsg_kdrv_send_packet(rpdev, dev_hdr, total_size);
+	if (ret) {
+		dev_err(&rpdev->dev, "rpmsg_send failed: %d\n", ret);
+		goto nosend;
+	}
+
+	return 0;
+
+nosend:
+	rpmsg_kdrv_dev_hdr_delete(rpdev, dev_hdr);
+dev_hdr_fail:
+	devm_kfree(&rpdev->dev, ctx);
+	return ret;
+}
+EXPORT_SYMBOL(rpmsg_kdrv_send_request_with_callback);
+
+/*
+ * rpmsg_kdrv_send_request_with_response
+ *
+ * Send a message where the caller will block for a response
+ *
+ * The caller is expected to destroy message and response
+ * when this function returns
+ */
+int rpmsg_kdrv_send_request_with_response(struct rpmsg_device *rpdev, uint32_t device_id,
+		void *message, uint32_t message_size,
+		void *response, uint32_t response_size)
+{
+	struct rpmsg_kdrv_device_header *dev_hdr;
+	int total_size = message_size + sizeof(*dev_hdr);
+	struct rpmsg_kdrv_ctx *ctx = NULL;
+	int ret;
+
+	ctx = rpmsg_kdrv_ctx_alloc(rpdev, true, NULL, NULL, message, message_size, response, response_size);
+	if (!ctx) {
+		dev_err(&rpdev->dev, "%s: ctx allocation failed\n", __func__);
+		return -ENOMEM;
+	}
+
+	dev_hdr = rpmsg_kdrv_dev_hdr_alloc(rpdev, device_id, total_size,
+			RPMSG_KDRV_TP_PACKET_TYPE_REQUEST,
+			RPMSG_KDRV_TP_PACKET_SOURCE_CLIENT,
+			message, message_size,
+			ctx);
+	if (!dev_hdr) {
+		dev_err(&rpdev->dev, "%s: device header allocation failed\n", __func__);
+		ret = -ENOMEM;
+		goto dev_hdr_fail;
+	}
+
+	ret = rpmsg_kdrv_send_packet(rpdev, dev_hdr, total_size);
+	if (ret) {
+		dev_err(&rpdev->dev, "rpmsg_send failed: %d\n", ret);
+		goto nosend;
+	}
+
+	wait_event(ctx->response_wq, ctx->response_recv == true);
+
+nosend:
+	rpmsg_kdrv_dev_hdr_delete(rpdev, dev_hdr);
+dev_hdr_fail:
+	devm_kfree(&rpdev->dev, ctx);
+	return ret;
+}
+EXPORT_SYMBOL(rpmsg_kdrv_send_request_with_response);
+
+/*
+ * rpmsg_kdrv_send_message
+ *
+ * Send a message and dont expect a response
+ *
+ * The caller is expected to destroy message when
+ * this function returns
+ */
+int rpmsg_kdrv_send_message(struct rpmsg_device *rpdev, uint32_t device_id,
+		void *message, uint32_t message_size)
+{
+	struct rpmsg_kdrv_device_header *dev_hdr;
+	int total_size = message_size + sizeof(*dev_hdr);
+	int ret;
+
+	/* We dont need a ctx for direct messages */
+
+	dev_hdr = rpmsg_kdrv_dev_hdr_alloc(rpdev, device_id, total_size,
+			RPMSG_KDRV_TP_PACKET_TYPE_MESSAGE,
+			RPMSG_KDRV_TP_PACKET_SOURCE_CLIENT,
+			message, message_size,
+			NULL);
+	if (!dev_hdr) {
+		dev_err(&rpdev->dev, "%s: device header allocation failed\n", __func__);
+		return -ENOMEM;
+	}
+
+	ret = rpmsg_kdrv_send_packet(rpdev, dev_hdr, total_size);
+	if (ret) {
+		dev_err(&rpdev->dev, "%s: rpmsg_send failed: %d\n", __func__, ret);
+		goto out;
+	}
+
+out:
+	rpmsg_kdrv_dev_hdr_delete(rpdev, dev_hdr);
+	return ret;
+}
+EXPORT_SYMBOL(rpmsg_kdrv_send_message);
+
+static int rpmsg_kdrv_cb(struct rpmsg_device *rpdev, void *data, int len,
+						void *private, u32 src)
+{
+	struct rpmsg_kdrv_priv *priv = dev_get_drvdata(&rpdev->dev);
+	struct rpmsg_kdrv_device_header *hdr = data;
+	struct rpmsg_kdrv_message_header *msg;
+	int msg_len;
+	struct rpmsg_kdrv_ctx *ctx;
+	int ret;
+
+	if (hdr->packet_type != RPMSG_KDRV_TP_PACKET_TYPE_RESPONSE) {
+		rpmsg_kdrv_driver_handle_data(rpdev, data, len, private, src);
+		return 0;
+	}
+
+	mutex_lock(&priv->message_lock);
+	ctx = idr_find(&priv->message_idr, hdr->packet_id);
+	mutex_unlock(&priv->message_lock);
+
+	if (!ctx) {
+		dev_err(&rpdev->dev, "%s: response received with no pending request\n", __func__);
+		return 0;
+	}
+
+	msg = (struct rpmsg_kdrv_message_header *)((void *)(&hdr[1]));
+	msg_len = len - sizeof(*hdr);
+
+	/* process callback if expected */
+	if (ctx->callback) {
+		ret = ctx->callback(ctx->cb_data, ctx->req, ctx->req_size, msg, msg_len);
+		if (ret == RRMSG_KDRV_CALLBACK_DONE) {
+			/* No need to keep the ctx alive */
+			rpmsg_kdrv_dev_hdr_delete(rpdev, ctx->dev_hdr);
+			devm_kfree(&rpdev->dev, ctx);
+		}
+		return 0;
+	}
+
+	/* copy the response and wake up caller, caller will destroy ctx & dev_hdr */
+	memcpy(ctx->resp, msg, min(msg_len, ctx->resp_size));
+
+	ctx->response_recv = true;
+	wake_up(&ctx->response_wq);
+
+	return 0;
+}
+
+static int rpmsg_kdrv_dev_match(struct device *dev, struct device_driver *drv)
+{
+	struct rpmsg_kdrv_device *kddev = to_rpmsg_kdrv_device(dev);
+	struct rpmsg_kdrv_driver *kddrv = to_rpmsg_kdrv_driver(drv);
+
+	if (kddrv->device_type == kddev->device_type) {
+		dev_dbg(dev, "%s: matching with driver %s\n", __func__, drv->name);
+		return 1;
+	}
+
+	dev_dbg(dev, "%s: does not match driver %s\n", __func__, drv->name);
+	return 0;
+}
+
+static int rpmsg_kdrv_dev_probe(struct device *dev)
+{
+	struct rpmsg_kdrv_device *kddev = to_rpmsg_kdrv_device(dev);
+	struct rpmsg_kdrv_driver *kddrv = to_rpmsg_kdrv_driver(kddev->dev.driver);
+	int ret;
+
+	dev_dbg(dev, "%s: probe\n", __func__);
+
+	ret = kddrv->probe(kddev);
+	if (ret) {
+		dev_err(dev, "%s: child probe failed\n", __func__);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int rpmsg_kdrv_dev_remove(struct device *dev)
+{
+	struct rpmsg_kdrv_device *kddev = to_rpmsg_kdrv_device(dev);
+	struct rpmsg_kdrv_driver *kddrv = to_rpmsg_kdrv_driver(kddev->dev.driver);
+
+	dev_dbg(dev, "%s: remove\n", __func__);
+
+	kddrv->remove(kddev);
+	return 0;
+}
+
+static int rpmsg_kdrv_probe(struct rpmsg_device *rpdev)
+{
+	int ret;
+	struct rpmsg_kdrv_priv *priv;
+
+	dev_dbg(&rpdev->dev, "%s: probing rpmsg kdrv driver\n", __func__);
+
+	priv = devm_kzalloc(&rpdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	dev_set_drvdata(&rpdev->dev, priv);
+	priv->rpdev = rpdev;
+
+	idr_init(&priv->message_idr);
+	mutex_init(&priv->message_lock);
+
+	dev_dbg(&rpdev->dev, "%s: sending device info request\n", __func__);
+	ret = rpmsg_kdrv_get_devices(rpdev);
+	if (ret) {
+		dev_err(&rpdev->dev, "%s: error collecting device info\n", __func__);
+		goto out;
+	}
+
+	return 0;
+
+out:
+	dev_set_drvdata(&rpdev->dev, NULL);
+	devm_kfree(&rpdev->dev, priv);
+	return ret;
+}
+
+static void rpmsg_kdrv_remove(struct rpmsg_device *rpdev)
+{
+	dev_dbg(&rpdev->dev, "removing rpmsg kdrv driver\n");
+
+	/* TODO check for pending responses for any of the child devices */
+	/* TODO disconnect them all */
+}
+
+static struct bus_type rpmsg_kdrv_bus = {
+	.name		= "rpmsg_kdrv",
+	.match		= rpmsg_kdrv_dev_match,
+	.probe		= rpmsg_kdrv_dev_probe,
+	.remove		= rpmsg_kdrv_dev_remove,
+};
+
+static struct rpmsg_device_id rpmsg_kdrv_id_table[] = {
+	{ .name	= "rpmsg-kdrv" },
+	{ },
+};
+
+static struct rpmsg_driver rpmsg_kdrv = {
+	.drv.name	= "rpmsg-kdrv",
+	.id_table	= rpmsg_kdrv_id_table,
+	.probe		= rpmsg_kdrv_probe,
+	.callback	= rpmsg_kdrv_cb,
+	.remove		= rpmsg_kdrv_remove,
+};
+
+static int __init rpmsg_kdrv_init(void)
+{
+	int ret;
+
+	ret = bus_register(&rpmsg_kdrv_bus);
+	if (ret) {
+		pr_err("failed to register rpmsg kdrv bus: %d\n", ret);
+		goto out;
+	}
+
+	ret = register_rpmsg_driver(&rpmsg_kdrv);
+	if (ret) {
+		pr_err("failed to register rpmsg kdrv driver: %d\n", ret);
+		goto rpdrv_fail;
+	}
+
+	pr_debug("registered rpmsg kdrv driver\n");
+
+	return 0;
+
+rpdrv_fail:
+	bus_unregister(&rpmsg_kdrv_bus);
+out:
+	return ret;
+}
+module_init(rpmsg_kdrv_init);
+
+static void __exit rpmsg_kdrv_fini(void)
+{
+	pr_debug("unregistering rpmsg kdrv driver\n");
+
+	unregister_rpmsg_driver(&rpmsg_kdrv);
+	bus_unregister(&rpmsg_kdrv_bus);
+}
+module_exit(rpmsg_kdrv_fini);
+
+MODULE_AUTHOR("Subhajit Paul <subhajit_paul@ti.com>");
+MODULE_DESCRIPTION("TI Remote-device framework Driver");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/rpmsg-kdrv/rpmsg_kdrv_display.c b/drivers/rpmsg-kdrv/rpmsg_kdrv_display.c
--- a/drivers/rpmsg-kdrv/rpmsg_kdrv_display.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/rpmsg-kdrv/rpmsg_kdrv_display.c	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,473 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Subhajit Paul <subahjit_paul@ti.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+
+#include <drm/drm_fourcc.h>
+#include <linux/rpmsg.h>
+#include <linux/rpmsg-remotedev/rpmsg-remotedev.h>
+
+#include "shared/rpmsg-kdrv-transport-display.h"
+#include "rpmsg_kdrv_internal.h"
+
+#define RPMSG_KDRV_DISPLAY_RES_ID_FIRST		(0x10)
+
+struct rpmsg_kdrv_display_private {
+	struct rpmsg_kdrv_device *kddev;
+
+	struct rpmsg_remotedev rdev;
+
+	struct idr res_idr;
+	struct mutex res_lock;
+
+};
+
+static uint32_t check_min(uint32_t a, uint32_t b, int line)
+{
+	uint32_t res = min(a, b);
+
+	if (res != b) {
+		pr_err("Copy mismatch at Line %d\n", line);
+		WARN_ON(1);
+	}
+
+	return res;
+}
+
+static inline enum rpmsg_kdrv_display_format rpmsg_kdrv_display_fmt_to_rpmsg_fmt(uint32_t in_fmt)
+{
+	switch (in_fmt) {
+	case DRM_FORMAT_ARGB8888:
+		return RPMSG_KDRV_TP_DISPLAY_FORMAT_ARGB8888;
+	case DRM_FORMAT_XRGB8888:
+		return RPMSG_KDRV_TP_DISPLAY_FORMAT_XRGB8888;
+	default:
+		return RPMSG_KDRV_TP_DISPLAY_FORMAT_MAX;
+	}
+}
+
+static inline uint32_t rpmsg_kdrv_display_fmt_to_drm_fmt(uint32_t in_fmt)
+{
+	switch (in_fmt) {
+	case RPMSG_KDRV_TP_DISPLAY_FORMAT_ARGB8888:
+		return DRM_FORMAT_ARGB8888;
+	case RPMSG_KDRV_TP_DISPLAY_FORMAT_XRGB8888:
+		return DRM_FORMAT_XRGB8888;
+	default:
+		return 0;
+	}
+}
+
+static bool rpmsg_kdrv_display_ready(struct rpmsg_remotedev *rdev)
+{
+	struct rpmsg_kdrv_display_private *priv = container_of(rdev, struct rpmsg_kdrv_display_private, rdev);
+	struct rpmsg_kdrv_device *kddev = priv->kddev;
+	struct rpmsg_device *rpdev = kddev->rpdev;
+	struct rpmsg_kdrv_display_ready_query_request *req;
+	struct rpmsg_kdrv_display_ready_query_response *resp;
+	int ret;
+	bool retval;
+
+	req = devm_kzalloc(&kddev->dev, sizeof(*req), GFP_KERNEL);
+	if (!req)
+		return false;
+
+	resp = devm_kzalloc(&kddev->dev, sizeof(*resp), GFP_KERNEL);
+	if (!resp) {
+		devm_kfree(&kddev->dev, req);
+		return false;
+	}
+
+	req->header.message_type = RPMSG_KDRV_TP_DISPLAY_READY_QUERY_REQUEST;
+
+	ret = rpmsg_kdrv_send_request_with_response(rpdev, kddev->device_id, req, sizeof(*req), resp, sizeof(*resp));
+	if (ret) {
+		dev_err(&kddev->dev, "%s: rpmsg_kdrv_send_request_with_response\n", __func__);
+		retval = false;
+		goto out;
+	}
+
+	if (resp->header.message_type != RPMSG_KDRV_TP_DISPLAY_READY_QUERY_RESPONSE) {
+		dev_err(&kddev->dev, "%s: wrong response type\n", __func__);
+		retval = false;
+		goto out;
+	}
+
+	retval = resp->ready ? true : false;
+
+out:
+	devm_kfree(&kddev->dev, resp);
+	devm_kfree(&kddev->dev, req);
+	return retval;
+
+}
+
+static void rpmsg_kdrv_display_copy_vid_info(struct rpmsg_remotedev_display_pipe *dst, struct rpmsg_kdrv_display_vid_info *src)
+{
+	int cnt;
+	uint32_t out_fmt;
+
+	dst->pipe_id = src->id;
+	dst->can_scale = src->can_scale ? true : false;
+	dst->can_mod_win = src->mutable_window ? true : false;
+	if (dst->can_mod_win)
+		dst->fixed_win_x = dst->fixed_win_y = dst->fixed_win_w = dst->fixed_win_h = 0;
+	else {
+		dst->fixed_win_x = src->fixed_window_x;
+		dst->fixed_win_y = src->fixed_window_y;
+		dst->fixed_win_w = src->fixed_window_w;
+		dst->fixed_win_h = src->fixed_window_h;
+	}
+	dst->initial_zorder = src->init_zorder;
+	dst->num_formats = check_min(RPMSG_REMOTEDEV_DISPLAY_MAX_FORMATS, src->num_formats, __LINE__);
+
+	dst->num_allowed_zorders = check_min(RPMSG_REMOTEDEV_DISPLAY_MAX_ZORDERS, src->num_zorders, __LINE__);
+
+	for (cnt = 0; cnt < dst->num_formats; cnt++) {
+		out_fmt = rpmsg_kdrv_display_fmt_to_drm_fmt(src->format[cnt]);
+		WARN_ON(out_fmt == 0);
+		dst->formats[cnt] = out_fmt;
+	}
+
+	for (cnt = 0; cnt < dst->num_allowed_zorders; cnt++)
+		dst->allowed_zorders[cnt] = src->zorder[cnt];
+}
+
+static void rpmsg_kdrv_display_copy_vp_info(struct rpmsg_remotedev_display_disp *dst, struct rpmsg_kdrv_display_vp_info *src)
+{
+	int vidcnt;
+
+	dst->disp_id = src->id;
+	dst->width = src->width;
+	dst->height = src->height;
+	dst->refresh = src->refresh;
+	dst->num_pipes = check_min(RPMSG_REMOTEDEV_DISPLAY_MAX_PIPES, src->num_vids, __LINE__);
+
+	for (vidcnt = 0; vidcnt < dst->num_pipes; vidcnt++)
+		rpmsg_kdrv_display_copy_vid_info(&dst->pipes[vidcnt], &src->vid[vidcnt]);
+}
+
+static int rpmsg_kdrv_display_get_res(struct rpmsg_remotedev *rdev, struct rpmsg_remotedev_display_resinfo *res)
+{
+	struct rpmsg_kdrv_display_private *priv = container_of(rdev, struct rpmsg_kdrv_display_private, rdev);
+	struct rpmsg_kdrv_device *kddev = priv->kddev;
+	struct rpmsg_device *rpdev = kddev->rpdev;
+	struct rpmsg_kdrv_display_res_info_request *req;
+	struct rpmsg_kdrv_display_res_info_response *resp;
+	int ret, vpcnt;
+
+	req = devm_kzalloc(&kddev->dev, sizeof(*req), GFP_KERNEL);
+	if (!req)
+		return -ENOMEM;
+
+	resp = devm_kzalloc(&kddev->dev, sizeof(*resp), GFP_KERNEL);
+	if (!resp) {
+		devm_kfree(&kddev->dev, req);
+		return -ENOMEM;
+	}
+
+	req->header.message_type = RPMSG_KDRV_TP_DISPLAY_RES_INFO_REQUEST;
+
+	ret = rpmsg_kdrv_send_request_with_response(rpdev, kddev->device_id, req, sizeof(*req), resp, sizeof(*resp));
+	if (ret) {
+		dev_err(&kddev->dev, "%s: rpmsg_kdrv_send_request_with_response\n", __func__);
+		goto out;
+	}
+
+	if (resp->header.message_type != RPMSG_KDRV_TP_DISPLAY_RES_INFO_RESPONSE) {
+		dev_err(&kddev->dev, "%s: wrong response type\n", __func__);
+		ret = -EINVAL;
+		goto out;
+	}
+
+	res->num_disps = check_min(RPMSG_REMOTEDEV_DISPLAY_MAX_DISPS, resp->num_vps, __LINE__);
+
+	for (vpcnt = 0; vpcnt < res->num_disps; vpcnt++)
+		rpmsg_kdrv_display_copy_vp_info(&res->disps[vpcnt], &resp->vp[vpcnt]);
+
+out:
+	devm_kfree(&kddev->dev, resp);
+	devm_kfree(&kddev->dev, req);
+	return ret;
+}
+
+static uint32_t rpmsg_kdrv_display_res_id_new(struct rpmsg_kdrv_device *kddev, void *data)
+{
+	struct rpmsg_kdrv_display_private *priv = kddev->driver_private;
+	int id;
+
+	mutex_lock(&priv->res_lock);
+	id = idr_alloc(&priv->res_idr, data, RPMSG_KDRV_DISPLAY_RES_ID_FIRST, 0, GFP_KERNEL);
+	mutex_unlock(&priv->res_lock);
+
+	if (id < 0)
+		return 0;
+
+	return id;
+}
+
+static void rpmsg_kdrv_display_free_res_id(struct rpmsg_kdrv_device *kddev, uint32_t id)
+{
+	struct rpmsg_kdrv_display_private *priv = kddev->driver_private;
+
+	mutex_lock(&priv->res_lock);
+	idr_remove(&priv->res_idr, id);
+	mutex_unlock(&priv->res_lock);
+}
+
+static void rpmsg_kdrv_free_request_res(struct rpmsg_kdrv_device *kddev, struct rpmsg_kdrv_display_commit_request *req)
+{
+	int i;
+
+	rpmsg_kdrv_display_free_res_id(kddev, req->commit_id);
+
+	for (i = 0; i < req->num_vid_updates; i++)
+		if (req->vid[i].enabled)
+			rpmsg_kdrv_display_free_res_id(kddev, req->vid[i].buffer.buffer_id);
+
+}
+
+static bool rpmsg_kdrv_display_copy_buffer(struct rpmsg_kdrv_device *kddev, struct rpmsg_kdrv_display_buffer_info *dst,
+		struct rpmsg_remotedev_display_buffer *src)
+{
+	int i;
+
+	dst->width = src->width;
+	dst->height = src->height;
+
+	dst->format = rpmsg_kdrv_display_fmt_to_rpmsg_fmt(src->format);
+	if (WARN_ON(dst->format == RPMSG_KDRV_TP_DISPLAY_FORMAT_MAX))
+		return false;
+
+	dst->num_planes = check_min(RPMSG_KDRV_TP_DISPLAY_MAX_PLANES, src->num_planes, __LINE__);
+	if (dst->num_planes != src->num_planes)
+		return false;
+
+	for (i = 0; i < dst->num_planes; i++) {
+		dst->plane[i] = (uint64_t)src->planes[i];
+		dst->pitch[i] = src->pitches[i];
+	}
+
+	dst->buffer_id = rpmsg_kdrv_display_res_id_new(kddev, src);
+	if (!dst->buffer_id)
+		return false;
+
+	return true;
+}
+
+static bool rpmsg_kdrv_display_copy_vid_commit(struct rpmsg_kdrv_device *kddev, struct rpmsg_kdrv_display_vid_update_info *dst,
+		struct rpmsg_remotedev_display_pipe_update *src)
+{
+	dst->id = src->pipe_id;
+	dst->enabled = src->enabled ? 1 : 0;
+	if (dst->enabled) {
+		dst->dst_w = src->dst_w;
+		dst->dst_h = src->dst_h;
+		dst->dst_x = src->dst_x;
+		dst->dst_y = src->dst_y;
+
+		if (!rpmsg_kdrv_display_copy_buffer(kddev, &dst->buffer, src->buffer))
+			return false;
+	}
+
+	return true;
+}
+
+static bool rpmsg_kdrv_display_copy_commit(struct rpmsg_kdrv_device *kddev, struct rpmsg_kdrv_display_commit_request *dst,
+		struct rpmsg_remotedev_display_commit *src)
+{
+	int i, copied_vids;
+
+	dst->id = src->disp_id;
+	dst->num_vid_updates = check_min(RPMSG_KDRV_TP_DISPLAY_MAX_VIDS, src->num_pipe_updates, __LINE__);
+
+	for (i = 0, copied_vids = 0; i < dst->num_vid_updates; i++, copied_vids++)
+		if (!rpmsg_kdrv_display_copy_vid_commit(kddev, &dst->vid[i], &src->pipes[i]))
+			goto free_vid_res;
+
+	dst->commit_id = rpmsg_kdrv_display_res_id_new(kddev, src);
+	if (!dst->commit_id)
+		goto free_vid_res;
+
+	return true;
+
+free_vid_res:
+	for (i = 0; i < copied_vids; i++)
+		if (dst->vid[i].enabled)
+			rpmsg_kdrv_display_free_res_id(kddev, dst->vid[i].buffer.buffer_id);
+	return false;
+
+}
+
+static int rpmsg_kdrv_display_commit(struct rpmsg_remotedev *rdev, struct rpmsg_remotedev_display_commit *commit)
+{
+	struct rpmsg_kdrv_display_private *priv = container_of(rdev, struct rpmsg_kdrv_display_private, rdev);
+	struct rpmsg_kdrv_device *kddev = priv->kddev;
+	struct rpmsg_device *rpdev = kddev->rpdev;
+	struct rpmsg_kdrv_display_commit_request *req;
+	struct rpmsg_kdrv_display_commit_response *resp;
+	int ret;
+
+	req = devm_kzalloc(&kddev->dev, sizeof(*req), GFP_KERNEL);
+	if (!req)
+		return -ENOMEM;
+
+	resp = devm_kzalloc(&kddev->dev, sizeof(*resp), GFP_KERNEL);
+	if (!resp) {
+		devm_kfree(&kddev->dev, req);
+		return -ENOMEM;
+	}
+
+	req->header.message_type = RPMSG_KDRV_TP_DISPLAY_COMMIT_REQUEST;
+
+	if (!rpmsg_kdrv_display_copy_commit(kddev, req, commit)) {
+		dev_err(&kddev->dev, "%s: failed to copy commit request\n", __func__);
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	ret = rpmsg_kdrv_send_request_with_response(rpdev, kddev->device_id, req, sizeof(*req),
+			resp, sizeof(*resp));
+	if (ret) {
+		dev_err(&kddev->dev, "%s: rpmsg_kdrv_send_request_with_response\n", __func__);
+		goto nosend;
+	}
+
+
+	if (resp->header.message_type != RPMSG_KDRV_TP_DISPLAY_COMMIT_RESPONSE) {
+		dev_err(&kddev->dev, "%s: wrong response type\n", __func__);
+		goto out;
+	}
+
+	ret = ((resp->status == 0) ? 0 : -EINVAL);
+	goto out;
+
+nosend:
+	rpmsg_kdrv_free_request_res(kddev, req);
+out:
+	devm_kfree(&kddev->dev, req);
+	devm_kfree(&kddev->dev, resp);
+	return ret;
+}
+
+
+struct rpmsg_remotedev_display_ops disp_ops = {
+	.ready = rpmsg_kdrv_display_ready,
+	.get_res_info = rpmsg_kdrv_display_get_res,
+	.commit = rpmsg_kdrv_display_commit,
+};
+
+static void rpmsg_kdrv_display_device_init(struct rpmsg_kdrv_device *kddev, void *data, int len)
+{
+}
+
+static int rpmsg_kdrv_display_probe(struct rpmsg_kdrv_device *dev)
+{
+	struct rpmsg_kdrv_display_private *priv;
+
+	dev_dbg(&dev->dev, "%s\n", __func__);
+
+	priv = devm_kzalloc(&dev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->rdev.type = RPMSG_REMOTEDEV_DISPLAY_DEVICE;
+	priv->rdev.device.display.ops = &disp_ops;
+
+	mutex_init(&priv->res_lock);
+	idr_init(&priv->res_idr);
+
+	priv->kddev = dev;
+	dev->driver_private = priv;
+	dev->remotedev = &priv->rdev;
+
+	rpmsg_kdrv_display_device_init(dev, dev->device_data, dev->device_data_len);
+
+	return 0;
+}
+
+static void rpmsg_kdrv_display_remove(struct rpmsg_kdrv_device *dev)
+{
+	dev_dbg(&dev->dev, "%s\n", __func__);
+}
+
+static void rpmsg_kdrv_display_handle_commit(struct rpmsg_kdrv_device *dev, struct rpmsg_kdrv_display_commit_done_message *msg)
+{
+	struct rpmsg_kdrv_display_private *priv = dev->driver_private;
+	struct rpmsg_remotedev *rdev = &priv->rdev;
+	struct rpmsg_remotedev_display_commit *commit;
+
+	mutex_lock(&priv->res_lock);
+	commit = idr_find(&priv->res_idr, msg->commit_id);
+	idr_remove(&priv->res_idr, msg->commit_id);
+	mutex_unlock(&priv->res_lock);
+
+	if (!commit) {
+		dev_err(&dev->dev, "%s: no pending commit found\n", __func__);
+		return;
+	}
+
+	if (rdev->device.display.cb_ops && rdev->device.display.cb_ops->commit_done)
+		rdev->device.display.cb_ops->commit_done(commit, rdev->cb_data);
+}
+
+static void rpmsg_kdrv_display_handle_buffer(struct rpmsg_kdrv_device *dev, struct rpmsg_kdrv_display_buffer_done_message *msg)
+{
+	struct rpmsg_kdrv_display_private *priv = dev->driver_private;
+	struct rpmsg_remotedev *rdev = &priv->rdev;
+	struct rpmsg_remotedev_display_buffer *buffer;
+
+	mutex_lock(&priv->res_lock);
+	buffer = idr_find(&priv->res_idr, msg->buffer_id);
+	idr_remove(&priv->res_idr, msg->buffer_id);
+	mutex_unlock(&priv->res_lock);
+
+	if (!buffer) {
+		dev_err(&dev->dev, "%s: no pending buffer found\n", __func__);
+		return;
+	}
+
+	if (rdev->device.display.cb_ops && rdev->device.display.cb_ops->buffer_done)
+		rdev->device.display.cb_ops->buffer_done(buffer, rdev->cb_data);
+}
+
+static int rpmsg_kdrv_display_callback(struct rpmsg_kdrv_device *dev, void *msg, int len)
+{
+	struct rpmsg_kdrv_display_message_header *hdr = msg;
+
+	if (hdr->message_type == RPMSG_KDRV_TP_DISPLAY_COMMIT_DONE_MESSAGE)
+		rpmsg_kdrv_display_handle_commit(dev, msg);
+	else if (hdr->message_type == RPMSG_KDRV_TP_DISPLAY_BUFFER_DONE_MESSAGE)
+		rpmsg_kdrv_display_handle_buffer(dev, msg);
+
+	return 0;
+}
+
+
+struct rpmsg_kdrv_driver rpmsg_kdrv_display = {
+	.drv.name = "rpmsg-kdrv-display",
+	.device_type = RPMSG_KDRV_TP_DEVICE_TYPE_DISPLAY,
+	.probe = rpmsg_kdrv_display_probe,
+	.remove = rpmsg_kdrv_display_remove,
+	.callback = rpmsg_kdrv_display_callback,
+};
+
+static int __init rpmsg_kdrv_display_driver_init(void)
+{
+	return rpmsg_kdrv_register_driver(&rpmsg_kdrv_display);
+}
+module_init(rpmsg_kdrv_display_driver_init);
+
+static void rpmsg_kdrv_display_driver_fini(void)
+{
+}
+module_exit(rpmsg_kdrv_display_driver_fini);
+
+MODULE_AUTHOR("Subhajit Paul <subhajit_paul@ti.com>");
+MODULE_DESCRIPTION("TI Remote-device Virtual Display Driver");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/rpmsg-kdrv/rpmsg_kdrv_internal.h b/drivers/rpmsg-kdrv/rpmsg_kdrv_internal.h
--- a/drivers/rpmsg-kdrv/rpmsg_kdrv_internal.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/rpmsg-kdrv/rpmsg_kdrv_internal.h	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Subhajit Paul <subhajit_paul@ti.com>
+ */
+
+#ifndef __RPMSG_KDRV_INTERNAL_H__
+#define __RPMSG_KDRV_INTERNAL_H__
+
+#define RRMSG_KDRV_CALLBACK_DONE		(0)
+#define RRMSG_KDRV_CALLBACK_MORE		(1)
+
+struct rpmsg_kdrv_device {
+	struct device dev;
+	struct rpmsg_device *rpdev;
+	int device_type;
+	int device_id;
+	void *device_data;
+	int device_data_len;
+	char *device_name;
+	void *device_private;
+	void *driver_private;
+	struct rpmsg_remotedev *remotedev;
+};
+
+struct rpmsg_kdrv_driver {
+	struct device_driver drv;
+	int device_type;
+	int (*probe)(struct rpmsg_kdrv_device *dev);
+	void (*remove)(struct rpmsg_kdrv_device *dev);
+	int (*callback)(struct rpmsg_kdrv_device *dev, void *msg, int len);
+};
+
+typedef int (*request_cb_t)(void *data, void *req, int req_sz, void *resp, int resp_sz);
+
+extern int rpmsg_kdrv_register_driver(struct rpmsg_kdrv_driver *drv);
+
+extern int rpmsg_kdrv_send_request_with_callback(struct rpmsg_device *rpdev,
+		uint32_t device_id, void *message, uint32_t message_size, void *cb_data,
+		request_cb_t callback);
+extern int rpmsg_kdrv_send_request_with_response(struct rpmsg_device *rpdev,
+		uint32_t device_id, void *message, uint32_t message_size,
+		void *response, uint32_t response_size);
+extern int rpmsg_kdrv_send_message(struct rpmsg_device *rpdev,
+		uint32_t device_id, void *message, uint32_t message_size);
+
+
+#endif
diff -Naur --no-dereference a/drivers/rpmsg-kdrv/rpmsg_kdrv_switch.c b/drivers/rpmsg-kdrv/rpmsg_kdrv_switch.c
--- a/drivers/rpmsg-kdrv/rpmsg_kdrv_switch.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/rpmsg-kdrv/rpmsg_kdrv_switch.c	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,828 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Grygorii Strashko <grygorii.strashko@ti.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/rpmsg.h>
+#include <linux/rpmsg-remotedev/rpmsg-remotedev.h>
+
+#include "rpmsg_kdrv_internal.h"
+#include "shared/rpmsg-kdrv-transport-switch.h"
+
+struct rpmsg_kdrv_switch_private {
+	struct rpmsg_kdrv_device *kddev;
+	struct rpmsg_remotedev rdev;
+	u64	session_id;
+	u32	core_key;
+	u32	permissions;
+	u32	uart_id;
+	u32	attached:1;
+	u32	uart_connected:1;
+};
+
+static bool
+rpmsg_kdrv_switch_check_perm(struct rpmsg_kdrv_switch_private *priv,
+			     enum rpmsg_kdrv_ethswitch_message_type msg_type)
+{
+	struct rpmsg_kdrv_device *kddev = priv->kddev;
+
+	if (priv->permissions & BIT(msg_type))
+		return true;
+
+	dev_err_ratelimited(&kddev->dev, "permission denied msg: 0x%02X\n",
+			    msg_type);
+	return false;
+}
+
+static int
+rpmsg_kdrv_switch_check_resp_status(struct rpmsg_kdrv_ethswitch_common_resp_info *info)
+{
+	if (info->status == RPMSG_KDRV_TP_ETHSWITCH_CMDSTATUS_EAGAIN)
+		return -EAGAIN;
+	if (info->status == RPMSG_KDRV_TP_ETHSWITCH_CMDSTATUS_EFAIL)
+		return -EIO;
+	return 0;
+}
+
+static int rpmsg_kdrv_switch_ping(struct rpmsg_remotedev *rdev,
+				  const u8 *data, int size)
+{
+	struct rpmsg_kdrv_switch_private *priv =
+		container_of(rdev, struct rpmsg_kdrv_switch_private, rdev);
+	struct rpmsg_kdrv_device *kddev = priv->kddev;
+	struct rpmsg_device *rpdev = kddev->rpdev;
+	struct rpmsg_kdrv_ethswitch_ping_req *req;
+	struct rpmsg_kdrv_ethswitch_ping_resp *resp;
+	int ret;
+
+	if (!rpmsg_kdrv_switch_check_perm(priv,
+					  RPMSG_KDRV_TP_ETHSWITCH_PING_REQUEST))
+		return -EPERM;
+
+	req = devm_kzalloc(&kddev->dev, sizeof(*req), GFP_KERNEL);
+	if (!req)
+		return -ENOMEM;
+
+	resp = devm_kzalloc(&kddev->dev, sizeof(*resp), GFP_KERNEL);
+	if (!resp) {
+		devm_kfree(&kddev->dev, req);
+		return -ENOMEM;
+	}
+
+	if (size > RPMSG_KDRV_TP_ETHSWITCH_MESSAGE_DATA_LEN)
+		size = RPMSG_KDRV_TP_ETHSWITCH_MESSAGE_DATA_LEN;
+
+	req->header.message_type = RPMSG_KDRV_TP_ETHSWITCH_PING_REQUEST;
+	memcpy(req->data, data, size);
+
+	ret = rpmsg_kdrv_send_request_with_response(rpdev, kddev->device_id,
+						    req, sizeof(*req),
+						    resp, sizeof(*resp));
+	if (ret) {
+		dev_dbg(&kddev->dev, "%s: send: %d\n", __func__, ret);
+		goto out;
+	}
+
+	if (memcmp(req->data, resp->data, size)) {
+		dev_dbg(&kddev->dev, "%s: ping fail - data\n", __func__);
+		ret = -EINVAL;
+	}
+
+out:
+	devm_kfree(&kddev->dev, resp);
+	devm_kfree(&kddev->dev, req);
+	return ret;
+}
+
+static int
+rpmsg_kdrv_switch_attach(struct rpmsg_remotedev *rdev,
+			 struct rpmsg_rdev_eth_switch_attach_info *attach_info)
+{
+	struct rpmsg_kdrv_switch_private *priv =
+		container_of(rdev, struct rpmsg_kdrv_switch_private, rdev);
+	struct rpmsg_kdrv_device *kddev = priv->kddev;
+	struct rpmsg_device *rpdev = kddev->rpdev;
+	struct rpmsg_kdrv_ethswitch_attach_req *req;
+	struct rpmsg_kdrv_ethswitch_attach_resp *resp;
+	int ret;
+
+	if (priv->attached)
+		return -EBUSY;
+
+	req = devm_kzalloc(&kddev->dev, sizeof(*req), GFP_KERNEL);
+	if (!req)
+		return -ENOMEM;
+
+	resp = devm_kzalloc(&kddev->dev, sizeof(*resp), GFP_KERNEL);
+	if (!resp) {
+		devm_kfree(&kddev->dev, req);
+		return -ENOMEM;
+	}
+
+	req->header.message_type = RPMSG_KDRV_TP_ETHSWITCH_ATTACH;
+	req->cpsw_type = RPMSG_KDRV_TP_ETHSWITCH_CPSWTYPE_9G;
+
+	ret = rpmsg_kdrv_send_request_with_response(rpdev, kddev->device_id,
+						    req, sizeof(*req),
+						    resp, sizeof(*resp));
+	if (ret) {
+		dev_dbg(&kddev->dev, "%s: send: %d\n", __func__, ret);
+		goto out;
+	}
+
+	ret = rpmsg_kdrv_switch_check_resp_status(&resp->info);
+	if (ret)
+		goto out;
+
+	priv->session_id = resp->id;
+	priv->core_key = resp->core_key;
+	priv->attached = true;
+
+	dev_dbg(&kddev->dev, "%s: done id:%llX core_key:%08X\n",
+		__func__, priv->session_id, priv->core_key);
+
+	attach_info->rx_mtu = resp->rx_mtu;
+	memcpy(attach_info->tx_mtu, resp->tx_mtu, sizeof(attach_info->tx_mtu));
+	attach_info->features = resp->features;
+	if (priv->uart_connected)
+		attach_info->features |=
+				RPMSG_KDRV_ETHSWITCH_FEATURE_DUMP_STATS;
+
+out:
+	devm_kfree(&kddev->dev, resp);
+	devm_kfree(&kddev->dev, req);
+	return ret;
+}
+
+static int
+rpmsg_kdrv_switch_attach_ext(struct rpmsg_remotedev *rdev,
+			     struct rpmsg_rdev_eth_switch_attach_ext_info *attach_ext_info)
+{
+	struct rpmsg_kdrv_switch_private *priv =
+		container_of(rdev, struct rpmsg_kdrv_switch_private, rdev);
+	struct rpmsg_kdrv_device *kddev = priv->kddev;
+	struct rpmsg_device *rpdev = kddev->rpdev;
+	struct rpmsg_kdrv_ethswitch_attach_extended_req *req;
+	struct rpmsg_kdrv_ethswitch_attach_extended_resp *resp;
+	int ret;
+
+	if (priv->attached)
+		return -EBUSY;
+
+	req = devm_kzalloc(&kddev->dev, sizeof(*req), GFP_KERNEL);
+	if (!req)
+		return -ENOMEM;
+
+	resp = devm_kzalloc(&kddev->dev, sizeof(*resp), GFP_KERNEL);
+	if (!resp) {
+		devm_kfree(&kddev->dev, req);
+		return -ENOMEM;
+	}
+
+	req->header.message_type = RPMSG_KDRV_TP_ETHSWITCH_ATTACH_EXT;
+	req->cpsw_type = RPMSG_KDRV_TP_ETHSWITCH_CPSWTYPE_9G;
+
+	ret = rpmsg_kdrv_send_request_with_response(rpdev, kddev->device_id,
+						    req, sizeof(*req),
+						    resp, sizeof(*resp));
+	if (ret) {
+		dev_dbg(&kddev->dev, "%s: send: %d\n", __func__, ret);
+		goto out;
+	}
+
+	ret = rpmsg_kdrv_switch_check_resp_status(&resp->info);
+	if (ret)
+		goto out;
+
+	priv->session_id = resp->id;
+	priv->core_key = resp->core_key;
+	priv->attached = true;
+
+	dev_dbg(&kddev->dev, "%s: done id:%llX core_key:%08X\n",
+		__func__, priv->session_id, priv->core_key);
+
+	attach_ext_info->rx_mtu = resp->rx_mtu;
+	memcpy(attach_ext_info->tx_mtu, resp->tx_mtu,
+	       sizeof(attach_ext_info->tx_mtu));
+	attach_ext_info->features = resp->features;
+	if (priv->uart_connected)
+		attach_ext_info->features |=
+				RPMSG_KDRV_ETHSWITCH_FEATURE_DUMP_STATS;
+	attach_ext_info->flow_idx = resp->alloc_flow_idx;
+	attach_ext_info->tx_cpsw_psil_dst_id = resp->tx_cpsw_psil_dst_id;
+	ether_addr_copy(attach_ext_info->mac_addr, resp->mac_address);
+
+out:
+	devm_kfree(&kddev->dev, resp);
+	devm_kfree(&kddev->dev, req);
+	return ret;
+}
+
+static int rpmsg_kdrv_switch_detach(struct rpmsg_remotedev *rdev)
+{
+	struct rpmsg_kdrv_switch_private *priv =
+		container_of(rdev, struct rpmsg_kdrv_switch_private, rdev);
+	struct rpmsg_kdrv_device *kddev = priv->kddev;
+	struct rpmsg_device *rpdev = kddev->rpdev;
+	struct rpmsg_kdrv_ethswitch_detach_req *req;
+	struct rpmsg_kdrv_ethswitch_detach_resp *resp;
+	int ret;
+
+	if (!priv->attached)
+		return -EINVAL;
+
+	req = devm_kzalloc(&kddev->dev, sizeof(*req), GFP_KERNEL);
+	if (!req)
+		return -ENOMEM;
+
+	resp = devm_kzalloc(&kddev->dev, sizeof(*resp), GFP_KERNEL);
+	if (!resp) {
+		devm_kfree(&kddev->dev, req);
+		return -ENOMEM;
+	}
+
+	req->header.message_type = RPMSG_KDRV_TP_ETHSWITCH_DETACH;
+	req->info.id = priv->session_id;
+	req->info.core_key = priv->core_key;
+
+	ret = rpmsg_kdrv_send_request_with_response(rpdev, kddev->device_id,
+						    req, sizeof(*req),
+						    resp, sizeof(*resp));
+	if (ret) {
+		dev_dbg(&kddev->dev, "%s: send: %d\n", __func__, ret);
+		goto out;
+	}
+
+	ret = rpmsg_kdrv_switch_check_resp_status(&resp->info);
+
+	priv->attached = false;
+
+	dev_dbg(&kddev->dev, "%s: done ret:%d\n", __func__, ret);
+
+out:
+	devm_kfree(&kddev->dev, resp);
+	devm_kfree(&kddev->dev, req);
+	return ret;
+}
+
+static int
+rpmsg_kdrv_switch_get_tx_info(struct rpmsg_remotedev *rdev,
+			      struct rpmsg_rdev_eth_switch_tx_info *info)
+{
+	struct rpmsg_kdrv_switch_private *priv =
+		container_of(rdev, struct rpmsg_kdrv_switch_private, rdev);
+	struct rpmsg_kdrv_device *kddev = priv->kddev;
+	struct rpmsg_device *rpdev = kddev->rpdev;
+	struct rpmsg_kdrv_ethswitch_alloc_req *req;
+	struct rpmsg_kdrv_ethswitch_alloc_tx_resp *resp;
+	int ret;
+
+	if (!priv->attached)
+		return -EINVAL;
+
+	req = devm_kzalloc(&kddev->dev, sizeof(*req), GFP_KERNEL);
+	if (!req)
+		return -ENOMEM;
+
+	resp = devm_kzalloc(&kddev->dev, sizeof(*resp), GFP_KERNEL);
+	if (!resp) {
+		devm_kfree(&kddev->dev, req);
+		return -ENOMEM;
+	}
+
+	req->header.message_type = RPMSG_KDRV_TP_ETHSWITCH_ALLOC_TX;
+	req->info.id = priv->session_id;
+	req->info.core_key = priv->core_key;
+
+	ret = rpmsg_kdrv_send_request_with_response(rpdev, kddev->device_id,
+						    req, sizeof(*req),
+						    resp, sizeof(*resp));
+	if (ret) {
+		dev_dbg(&kddev->dev, "%s: send: %d\n", __func__, ret);
+		goto out;
+	}
+
+	ret = rpmsg_kdrv_switch_check_resp_status(&resp->info);
+	if (ret)
+		goto out;
+
+	info->tx_cpsw_psil_dst_id = resp->tx_cpsw_psil_dst_id;
+
+	dev_dbg(&kddev->dev, "%s: done\n", __func__);
+
+out:
+	devm_kfree(&kddev->dev, resp);
+	devm_kfree(&kddev->dev, req);
+	return ret;
+}
+
+static int
+rpmsg_kdrv_switch_get_rx_info(struct rpmsg_remotedev *rdev,
+			      struct rpmsg_rdev_eth_switch_rx_info *info)
+{
+	struct rpmsg_kdrv_switch_private *priv =
+		container_of(rdev, struct rpmsg_kdrv_switch_private, rdev);
+	struct rpmsg_kdrv_device *kddev = priv->kddev;
+	struct rpmsg_device *rpdev = kddev->rpdev;
+	struct rpmsg_kdrv_ethswitch_alloc_req *req;
+	struct rpmsg_kdrv_ethswitch_alloc_rx_resp *resp;
+	int ret;
+
+	if (!priv->attached)
+		return -EINVAL;
+
+	req = devm_kzalloc(&kddev->dev, sizeof(*req), GFP_KERNEL);
+	if (!req)
+		return -ENOMEM;
+
+	resp = devm_kzalloc(&kddev->dev, sizeof(*resp), GFP_KERNEL);
+	if (!resp) {
+		devm_kfree(&kddev->dev, req);
+		return -ENOMEM;
+	}
+
+	req->header.message_type = RPMSG_KDRV_TP_ETHSWITCH_ALLOC_RX;
+	req->info.id = priv->session_id;
+	req->info.core_key = priv->core_key;
+
+	ret = rpmsg_kdrv_send_request_with_response(rpdev, kddev->device_id,
+						    req, sizeof(*req),
+						    resp, sizeof(*resp));
+	if (ret) {
+		dev_dbg(&kddev->dev, "%s: send: %d\n", __func__, ret);
+		goto out;
+	}
+
+	ret = rpmsg_kdrv_switch_check_resp_status(&resp->info);
+	if (ret)
+		goto out;
+
+	info->flow_idx = resp->alloc_flow_idx;
+
+	dev_dbg(&kddev->dev, "%s: done\n", __func__);
+
+out:
+	devm_kfree(&kddev->dev, resp);
+	devm_kfree(&kddev->dev, req);
+	return ret;
+}
+
+static int rpmsg_kdrv_switch_get_mac(struct rpmsg_remotedev *rdev,
+				     void *mac_addr)
+{
+	struct rpmsg_kdrv_switch_private *priv =
+		container_of(rdev, struct rpmsg_kdrv_switch_private, rdev);
+	struct rpmsg_kdrv_device *kddev = priv->kddev;
+	struct rpmsg_device *rpdev = kddev->rpdev;
+	struct rpmsg_kdrv_ethswitch_alloc_req *req;
+	struct rpmsg_kdrv_ethswitch_alloc_mac_resp *resp;
+	int ret;
+
+	if (!priv->attached)
+		return -EINVAL;
+
+	req = devm_kzalloc(&kddev->dev, sizeof(*req), GFP_KERNEL);
+	if (!req)
+		return -ENOMEM;
+
+	resp = devm_kzalloc(&kddev->dev, sizeof(*resp), GFP_KERNEL);
+	if (!resp) {
+		devm_kfree(&kddev->dev, req);
+		return -ENOMEM;
+	}
+
+	req->header.message_type = RPMSG_KDRV_TP_ETHSWITCH_ALLOC_MAC;
+	req->info.id = priv->session_id;
+	req->info.core_key = priv->core_key;
+
+	ret = rpmsg_kdrv_send_request_with_response(rpdev, kddev->device_id,
+						    req, sizeof(*req),
+						    resp, sizeof(*resp));
+	if (ret) {
+		dev_dbg(&kddev->dev, "%s: send: %d\n", __func__, ret);
+		goto out;
+	}
+
+	ret = rpmsg_kdrv_switch_check_resp_status(&resp->info);
+	if (ret)
+		goto out;
+
+	ether_addr_copy(mac_addr, resp->mac_address);
+
+	dev_dbg(&kddev->dev, "%s: done\n", __func__);
+
+out:
+	devm_kfree(&kddev->dev, resp);
+	devm_kfree(&kddev->dev, req);
+	return ret;
+}
+
+static int rpmsg_kdrv_switch_register_mac(struct rpmsg_remotedev *rdev,
+					  void *mac_addr, u32 flow_idx_offset)
+{
+	struct rpmsg_kdrv_switch_private *priv =
+		container_of(rdev, struct rpmsg_kdrv_switch_private, rdev);
+	struct rpmsg_kdrv_device *kddev = priv->kddev;
+	struct rpmsg_device *rpdev = kddev->rpdev;
+	struct rpmsg_kdrv_ethswitch_register_mac_req *req;
+	struct rpmsg_kdrv_ethswitch_register_mac_resp *resp;
+	int ret;
+
+	if (!priv->attached)
+		return -EINVAL;
+
+	req = devm_kzalloc(&kddev->dev, sizeof(*req), GFP_KERNEL);
+	if (!req)
+		return -ENOMEM;
+
+	resp = devm_kzalloc(&kddev->dev, sizeof(*resp), GFP_KERNEL);
+	if (!resp) {
+		devm_kfree(&kddev->dev, req);
+		return -ENOMEM;
+	}
+
+	req->header.message_type = RPMSG_KDRV_TP_ETHSWITCH_REGISTER_MAC;
+	req->info.id = priv->session_id;
+	req->info.core_key = priv->core_key;
+	ether_addr_copy(req->mac_address, mac_addr);
+	req->flow_idx = flow_idx_offset;
+
+	ret = rpmsg_kdrv_send_request_with_response(rpdev, kddev->device_id,
+						    req, sizeof(*req),
+						    resp, sizeof(*resp));
+	if (ret) {
+		dev_dbg(&kddev->dev, "%s: send: %d\n", __func__, ret);
+		goto out;
+	}
+
+	ret = rpmsg_kdrv_switch_check_resp_status(&resp->info);
+
+	dev_dbg(&kddev->dev, "%s: done\n", __func__);
+
+out:
+	devm_kfree(&kddev->dev, resp);
+	devm_kfree(&kddev->dev, req);
+	return ret;
+}
+
+static int
+rpmsg_kdrv_switch_unregister_mac(struct rpmsg_remotedev *rdev,
+				 void *mac_addr, u32 flow_idx_offset)
+{
+	struct rpmsg_kdrv_switch_private *priv =
+		container_of(rdev, struct rpmsg_kdrv_switch_private, rdev);
+	struct rpmsg_kdrv_device *kddev = priv->kddev;
+	struct rpmsg_device *rpdev = kddev->rpdev;
+	struct rpmsg_kdrv_ethswitch_unregister_mac_req *req;
+	struct rpmsg_kdrv_ethswitch_unregister_mac_resp *resp;
+	int ret;
+
+	if (!priv->attached)
+		return -EINVAL;
+
+	req = devm_kzalloc(&kddev->dev, sizeof(*req), GFP_KERNEL);
+	if (!req)
+		return -ENOMEM;
+
+	resp = devm_kzalloc(&kddev->dev, sizeof(*resp), GFP_KERNEL);
+	if (!resp) {
+		devm_kfree(&kddev->dev, req);
+		return -ENOMEM;
+	}
+
+	req->header.message_type = RPMSG_KDRV_TP_ETHSWITCH_UNREGISTER_MAC;
+	req->info.id = priv->session_id;
+	req->info.core_key = priv->core_key;
+	ether_addr_copy(req->mac_address, mac_addr);
+	req->flow_idx = flow_idx_offset;
+
+	ret = rpmsg_kdrv_send_request_with_response(rpdev, kddev->device_id,
+						    req, sizeof(*req),
+						    resp, sizeof(*resp));
+	if (ret) {
+		dev_dbg(&kddev->dev, "%s: send: %d\n", __func__, ret);
+		goto out;
+	}
+
+	ret = rpmsg_kdrv_switch_check_resp_status(&resp->info);
+
+	dev_dbg(&kddev->dev, "%s: done\n", __func__);
+
+out:
+	devm_kfree(&kddev->dev, resp);
+	devm_kfree(&kddev->dev, req);
+	return ret;
+}
+
+static int rpmsg_kdrv_switch_reg_ipv4(struct rpmsg_remotedev *rdev,
+				      void *mac_addr, __be32 ipv4)
+{
+	struct rpmsg_kdrv_switch_private *priv =
+		container_of(rdev, struct rpmsg_kdrv_switch_private, rdev);
+	struct rpmsg_kdrv_device *kddev = priv->kddev;
+	struct rpmsg_device *rpdev = kddev->rpdev;
+	struct rpmsg_kdrv_ethswitch_ipv4_register_mac_req *req;
+	struct rpmsg_kdrv_ethswitch_ipv4_register_mac_resp *resp;
+	int ret;
+
+	if (!priv->attached)
+		return -EINVAL;
+
+	req = devm_kzalloc(&kddev->dev, sizeof(*req), GFP_KERNEL);
+	if (!req)
+		return -ENOMEM;
+
+	resp = devm_kzalloc(&kddev->dev, sizeof(*resp), GFP_KERNEL);
+	if (!resp) {
+		devm_kfree(&kddev->dev, req);
+		return -ENOMEM;
+	}
+
+	req->header.message_type = RPMSG_KDRV_TP_ETHSWITCH_IPV4_MAC_REGISTER;
+	req->info.id = priv->session_id;
+	req->info.core_key = priv->core_key;
+	ether_addr_copy(req->mac_address, mac_addr);
+	memcpy(req->ipv4_addr, &ipv4, RPMSG_KDRV_TP_ETHSWITCH_IPV4ADDRLEN);
+
+	ret = rpmsg_kdrv_send_request_with_response(rpdev, kddev->device_id,
+						    req, sizeof(*req),
+						    resp, sizeof(*resp));
+	if (ret) {
+		dev_dbg(&kddev->dev, "%s: send: %d\n", __func__, ret);
+		goto out;
+	}
+
+	ret = rpmsg_kdrv_switch_check_resp_status(&resp->info);
+
+	dev_dbg(&kddev->dev, "%s: done\n", __func__);
+
+out:
+	devm_kfree(&kddev->dev, resp);
+	devm_kfree(&kddev->dev, req);
+	return ret;
+}
+
+static int rpmsg_kdrv_switch_unreg_ipv4(struct rpmsg_remotedev *rdev,
+					__be32 ipv4)
+{
+	struct rpmsg_kdrv_switch_private *priv =
+		container_of(rdev, struct rpmsg_kdrv_switch_private, rdev);
+	struct rpmsg_kdrv_device *kddev = priv->kddev;
+	struct rpmsg_device *rpdev = kddev->rpdev;
+	struct rpmsg_kdrv_ethswitch_ipv4_unregister_mac_req *req;
+	struct rpmsg_kdrv_ethswitch_ipv4_unregister_mac_resp *resp;
+	int ret;
+
+	if (!priv->attached)
+		return -EINVAL;
+
+	req = devm_kzalloc(&kddev->dev, sizeof(*req), GFP_KERNEL);
+	if (!req)
+		return -ENOMEM;
+
+	resp = devm_kzalloc(&kddev->dev, sizeof(*resp), GFP_KERNEL);
+	if (!resp) {
+		devm_kfree(&kddev->dev, req);
+		return -ENOMEM;
+	}
+
+	req->header.message_type = RPMSG_KDRV_TP_ETHSWITCH_IPV4_MAC_UNREGISTER;
+	req->info.id = priv->session_id;
+	req->info.core_key = priv->core_key;
+	memcpy(req->ipv4_addr, &ipv4, RPMSG_KDRV_TP_ETHSWITCH_IPV4ADDRLEN);
+
+	ret = rpmsg_kdrv_send_request_with_response(rpdev, kddev->device_id,
+						    req, sizeof(*req),
+						    resp, sizeof(*resp));
+	if (ret) {
+		dev_dbg(&kddev->dev, "%s: send: %d\n", __func__, ret);
+		goto out;
+	}
+
+	ret = rpmsg_kdrv_switch_check_resp_status(&resp->info);
+
+	dev_dbg(&kddev->dev, "%s: done\n", __func__);
+
+out:
+	devm_kfree(&kddev->dev, resp);
+	devm_kfree(&kddev->dev, req);
+	return ret;
+}
+
+static int rpmsg_kdrv_switch_reg_read(struct rpmsg_remotedev *rdev,
+				      u32 reg_addr, u32 *val)
+{
+	struct rpmsg_kdrv_switch_private *priv =
+		container_of(rdev, struct rpmsg_kdrv_switch_private, rdev);
+	struct rpmsg_kdrv_device *kddev = priv->kddev;
+	struct rpmsg_device *rpdev = kddev->rpdev;
+	struct rpmsg_kdrv_ethswitch_regrd_req *req;
+	struct rpmsg_kdrv_ethswitch_regrd_resp *resp;
+	int ret;
+
+	if (!rpmsg_kdrv_switch_check_perm(priv, RPMSG_KDRV_TP_ETHSWITCH_REGRD))
+		return -EPERM;
+
+	req = devm_kzalloc(&kddev->dev, sizeof(*req), GFP_KERNEL);
+	if (!req)
+		return -ENOMEM;
+
+	resp = devm_kzalloc(&kddev->dev, sizeof(*resp), GFP_KERNEL);
+	if (!resp) {
+		devm_kfree(&kddev->dev, req);
+		return -ENOMEM;
+	}
+
+	req->header.message_type = RPMSG_KDRV_TP_ETHSWITCH_REGRD;
+	req->regaddr = reg_addr;
+
+	ret = rpmsg_kdrv_send_request_with_response(rpdev, kddev->device_id,
+						    req, sizeof(*req),
+						    resp, sizeof(*resp));
+	if (ret) {
+		dev_dbg(&kddev->dev, "%s: send: %d\n", __func__, ret);
+		goto out;
+	}
+
+	ret = rpmsg_kdrv_switch_check_resp_status(&resp->info);
+	if (ret)
+		goto out;
+
+	*val = resp->regval;
+	dev_dbg(&kddev->dev, "%s: done\n", __func__);
+
+out:
+	devm_kfree(&kddev->dev, resp);
+	devm_kfree(&kddev->dev, req);
+	return ret;
+}
+
+static int rpmsg_kdrv_switch_c2s_dbg_dump_stats(struct rpmsg_remotedev *rdev)
+{
+	struct rpmsg_kdrv_switch_private *priv =
+		container_of(rdev, struct rpmsg_kdrv_switch_private, rdev);
+	struct rpmsg_kdrv_device *kddev = priv->kddev;
+	struct rpmsg_device *rpdev = kddev->rpdev;
+	struct rpmsg_kdrv_ethswitch_c2s_notify *req;
+	int ret;
+
+	if (!priv->attached)
+		return -EINVAL;
+
+	req = devm_kzalloc(&kddev->dev, sizeof(*req), GFP_KERNEL);
+	if (!req)
+		return -ENOMEM;
+
+	req->header.message_type = RPMSG_KDRV_TP_ETHSWITCH_C2S_NOTIFY;
+	req->info.id = priv->session_id;
+	req->info.core_key = priv->core_key;
+	req->notifyid = RPMSG_KDRV_TP_ETHSWITCH_CLIENTNOTIFY_DUMPSTATS;
+
+	ret = rpmsg_kdrv_send_message(rpdev, kddev->device_id,
+				      req, sizeof(*req));
+
+	dev_dbg(&kddev->dev, "%s: done ret:%d\n", __func__, ret);
+	devm_kfree(&kddev->dev, req);
+	return ret;
+}
+
+static void rpmsg_kdrv_switch_get_fw_ver(struct rpmsg_remotedev *rdev,
+					 char *buf, size_t size)
+{
+	struct rpmsg_kdrv_switch_private *priv =
+		container_of(rdev, struct rpmsg_kdrv_switch_private, rdev);
+	struct rpmsg_kdrv_ethswitch_fw_version_info *fw_info;
+	struct rpmsg_kdrv_ethswitch_device_data *kddev_data;
+
+	kddev_data = priv->kddev->device_data;
+	fw_info = &kddev_data->fw_ver;
+
+	snprintf(buf, size, "%u.%u.%u %.*s/%.*s/%.*s SHA:%.*s",
+		 fw_info->major, fw_info->minor, fw_info->rev,
+		 RPMSG_KDRV_TP_ETHSWITCH_DATELEN, fw_info->date,
+		 RPMSG_KDRV_TP_ETHSWITCH_MONTHLEN, fw_info->month,
+		 RPMSG_KDRV_TP_ETHSWITCH_YEARLEN, fw_info->year,
+		 RPMSG_KDRV_TP_ETHSWITCH_COMMITSHALEN, fw_info->commit_hash);
+}
+
+struct rpmsg_remotedev_eth_switch_ops switch_ops = {
+	.get_fw_ver = rpmsg_kdrv_switch_get_fw_ver,
+	.attach = rpmsg_kdrv_switch_attach,
+	.attach_ext = rpmsg_kdrv_switch_attach_ext,
+	.detach = rpmsg_kdrv_switch_detach,
+	.get_tx_info = rpmsg_kdrv_switch_get_tx_info,
+	.get_rx_info = rpmsg_kdrv_switch_get_rx_info,
+	.get_mac = rpmsg_kdrv_switch_get_mac,
+	.register_mac = rpmsg_kdrv_switch_register_mac,
+	.unregister_mac = rpmsg_kdrv_switch_unregister_mac,
+	.register_ipv4 = rpmsg_kdrv_switch_reg_ipv4,
+	.unregister_ipv4 = rpmsg_kdrv_switch_unreg_ipv4,
+	.ping = rpmsg_kdrv_switch_ping,
+	.read_reg = rpmsg_kdrv_switch_reg_read,
+	.dbg_dump_stats = rpmsg_kdrv_switch_c2s_dbg_dump_stats,
+};
+
+static int rpmsg_kdrv_switch_callback(struct rpmsg_kdrv_device *dev,
+				      void *msg, int len)
+{
+	return 0;
+}
+
+static int
+rpmsg_kdrv_switch_dev_data_parse(struct rpmsg_kdrv_device *kddev,
+				 void *data, int len,
+				 struct rpmsg_kdrv_switch_private *priv)
+{
+	struct rpmsg_kdrv_ethswitch_device_data *kddev_data = data;
+	struct rpmsg_kdrv_ethswitch_fw_version_info *fw_info;
+
+	if (sizeof(*kddev_data) != len)
+		return -EINVAL;
+
+	dev_info(&kddev->dev, "Device info: permissions: %08X uart_id: %d\n",
+		 kddev_data->permission_flags,
+		 kddev_data->uart_connected ? kddev_data->uart_id : -1);
+
+	fw_info = &kddev_data->fw_ver;
+
+	dev_info(&kddev->dev, "FW ver %u.%u (rev %u) %.*s/%.*s/%.*s SHA:%.*s\n",
+		 fw_info->major, fw_info->minor, fw_info->rev,
+		 RPMSG_KDRV_TP_ETHSWITCH_DATELEN, fw_info->date,
+		 RPMSG_KDRV_TP_ETHSWITCH_MONTHLEN, fw_info->month,
+		 RPMSG_KDRV_TP_ETHSWITCH_YEARLEN, fw_info->year,
+		 RPMSG_KDRV_TP_ETHSWITCH_COMMITSHALEN, fw_info->commit_hash);
+
+	if (fw_info->major != RPMSG_KDRV_TP_ETHSWITCH_VERSION_MAJOR &&
+	    fw_info->minor != RPMSG_KDRV_TP_ETHSWITCH_VERSION_MINOR) {
+		dev_err(&kddev->dev, "Unsupported EthSwitch FW version\n");
+		return -EOPNOTSUPP;
+	}
+
+	priv->uart_connected = kddev_data->uart_connected;
+	priv->uart_id = kddev_data->uart_connected ? kddev_data->uart_id : -1;
+	priv->permissions = kddev_data->permission_flags;
+
+	return 0;
+}
+
+static int rpmsg_kdrv_switch_probe(struct rpmsg_kdrv_device *dev)
+{
+	struct rpmsg_kdrv_switch_private *priv;
+	int ret;
+
+	dev_dbg(&dev->dev, "%s\n", __func__);
+
+	priv = devm_kzalloc(&dev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->rdev.type = RPMSG_REMOTEDEV_ETH_SWITCH_DEVICE;
+	priv->rdev.device.eth_switch.ops = &switch_ops;
+
+	priv->kddev = dev;
+
+	ret = rpmsg_kdrv_switch_dev_data_parse(dev, dev->device_data,
+					       dev->device_data_len, priv);
+	if (ret)
+		return ret;
+
+	dev->driver_private = priv;
+	dev->remotedev = &priv->rdev;
+
+	return 0;
+}
+
+static void rpmsg_kdrv_switch_remove(struct rpmsg_kdrv_device *dev)
+{
+	dev->driver_private = NULL;
+	dev->remotedev = NULL;
+
+	dev_dbg(&dev->dev, "%s\n", __func__);
+}
+
+struct rpmsg_kdrv_driver rpmsg_kdrv_switch = {
+	.drv = {
+		.name = "rpmsg-kdrv-eth-switch",
+	},
+	.device_type = RPMSG_KDRV_TP_DEVICE_TYPE_ETHSWITCH,
+	.probe = rpmsg_kdrv_switch_probe,
+	.remove = rpmsg_kdrv_switch_remove,
+	.callback = rpmsg_kdrv_switch_callback,
+};
+
+static int __init rpmsg_kdrv_display_driver_init(void)
+{
+	return rpmsg_kdrv_register_driver(&rpmsg_kdrv_switch);
+}
+module_init(rpmsg_kdrv_display_driver_init);
+
+static void rpmsg_kdrv_display_driver_fini(void)
+{
+}
+module_exit(rpmsg_kdrv_display_driver_fini);
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Grygorii Strashko <grygorii.strashko@ti.com>");
+MODULE_DESCRIPTION("TI J721E RPMSG KDRV Ethernet switch driver");
diff -Naur --no-dereference a/drivers/rpmsg-kdrv/shared/rpmsg-kdrv-transport-common.h b/drivers/rpmsg-kdrv/shared/rpmsg-kdrv-transport-common.h
--- a/drivers/rpmsg-kdrv/shared/rpmsg-kdrv-transport-common.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/rpmsg-kdrv/shared/rpmsg-kdrv-transport-common.h	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,80 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Subhajit Paul <subhajit_paul@ti.com>
+ */
+
+#ifndef __RPMSG_KDRV_TRANSPORT_COMMON_H__
+#define __RPMSG_KDRV_TRANSPORT_COMMON_H__
+
+/*
+ * Device types supported by RPMSG-KDRV framework
+ * Currently supported device types: display
+ * Planned future support for capture and i2c devices
+ */
+#define RPMSG_KDRV_TP_DEVICE_TYPE_INIT		(0x0)
+#define RPMSG_KDRV_TP_DEVICE_TYPE_DISPLAY	(0x1)
+#define RPMSG_KDRV_TP_DEVICE_TYPE_DEMO		(0x2)
+#define RPMSG_KDRV_TP_DEVICE_TYPE_ETHSWITCH	(0x3)
+
+/* More device types here*/
+#define RPMSG_KDRV_TP_DEVICE_TYPE_MAX		(0x4)
+
+/*
+ * Maximum number of proxy devices per remotecore
+ */
+#define RPMSG_KDRV_TP_MAX_DEVICES		(2)
+
+/*
+ * Maximum length of proxy device name
+ */
+#define RPMSG_KDRV_TP_DEVICE_NAME_LEN		(32)
+
+/*
+ * Statically assigned device ID for init device
+ * Remote device framework dynamically assigns device
+ * IDs for other devices. All dynamically assigned IDs
+ * are greater than RPMSG_KDRV_TP_DEVICE_ID_INIT
+ */
+#define RPMSG_KDRV_TP_DEVICE_ID_INIT		(0)
+
+/*
+ * Packet IDs are assigned dynamically (for REQUEST packets)
+ * starting from RPMSG_KDRV_TP_PACKET_ID_FIRST
+ * For MESSAGE packets, framework can use RPMSG_KDRV_TP_PACKET_ID_NONE
+ */
+#define RPMSG_KDRV_TP_PACKET_ID_NONE		(0x10)
+#define RPMSG_KDRV_TP_PACKET_ID_FIRST		(RPMSG_KDRV_TP_PACKET_ID_NONE + 1)
+
+enum rpmsg_kdrv_packet_source {
+	RPMSG_KDRV_TP_PACKET_SOURCE_SERVER,
+	RPMSG_KDRV_TP_PACKET_SOURCE_CLIENT,
+	RPMSG_KDRV_TP_PACKET_SOURCE_MAX,
+};
+
+enum rpmsg_kdrv_packet_type {
+	RPMSG_KDRV_TP_PACKET_TYPE_REQUEST,
+	RPMSG_KDRV_TP_PACKET_TYPE_RESPONSE,
+	RPMSG_KDRV_TP_PACKET_TYPE_MESSAGE,
+	RPMSG_KDRV_TP_PACKET_TYPE_MAX,
+};
+
+/*RPMSG_KDRV message :
+ * => device_header
+ * => message_header : defined by each device type
+ * => request / response / message payload
+ */
+struct rpmsg_kdrv_device_header {
+	/* ID of device sending the packet */
+	u8 device_id;
+	/* enum: rpmsg_kdrv_packet_type */
+	u8 packet_type;
+	/* enum: rpmsg_kdrv_packet_source */
+	u8 packet_source;
+	/* dynamically assigned packet ID for response matching */
+	u32 packet_id;
+	/* size of packet */
+	u32 packet_size;
+} __packed;
+
+#endif
diff -Naur --no-dereference a/drivers/rpmsg-kdrv/shared/rpmsg-kdrv-transport-demo.h b/drivers/rpmsg-kdrv/shared/rpmsg-kdrv-transport-demo.h
--- a/drivers/rpmsg-kdrv/shared/rpmsg-kdrv-transport-demo.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/rpmsg-kdrv/shared/rpmsg-kdrv-transport-demo.h	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,78 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Subhajit Paul <subhajit_paul@ti.com>
+ */
+
+#ifndef __RPMSG_KDRV_TRANSPORT_DEMODEV_H__
+#define __RPMSG_KDRV_TRANSPORT_DEMODEV_H__
+
+#include "rpmsg-kdrv-transport-common.h"
+
+enum rpmsg_kdrv_display_message_type {
+	RPMSG_KDRV_TP_DEMODEV_PING_REQUEST,
+	RPMSG_KDRV_TP_DEMODEV_PING_RESPONSE,
+	RPMSG_KDRV_TP_DEMODEV_S2C_MESSAGE,
+	RPMSG_KDRV_TP_DEMODEV_C2S_MESSAGE,
+	RPMSG_KDRV_TP_DEMODEV_MAX,
+};
+
+/*
+ * Maximum length of demo device data
+ */
+#define RPMSG_KDRV_TP_DEMODEV_DEVICE_DATA_LEN	(32)
+
+/*
+ * Maximum length of demo device message data
+ */
+#define RPMSG_KDRV_TP_DEMODEV_MESSAGE_DATA_LEN	(128)
+
+/*
+ * per-device data for demo device
+ */
+struct rpmsg_kdrv_demodev_device_data {
+	/* Does the device send all vsyncs? */
+	u8 charString[RPMSG_KDRV_TP_DEMODEV_DEVICE_DATA_LEN];
+} __packed;
+
+/*
+ * message header for demo device
+ */
+struct rpmsg_kdrv_demodev_message_header {
+	/* enum: rpmsg_kdrv_demodev_message_type */
+	u8 message_type;
+} __packed;
+
+/* demo device ping request - always client to server */
+struct rpmsg_kdrv_demodev_ping_request {
+	/* message header */
+	struct rpmsg_kdrv_demodev_message_header header;
+	/* ping data */
+	u8 data[RPMSG_KDRV_TP_DEMODEV_MESSAGE_DATA_LEN];
+} __packed;
+
+/* demo device ping response - always server to client */
+struct rpmsg_kdrv_demodev_ping_response {
+	/* message header */
+	struct rpmsg_kdrv_demodev_message_header header;
+	/* ping data */
+	u8 data[RPMSG_KDRV_TP_DEMODEV_MESSAGE_DATA_LEN];
+} __packed;
+
+/* demo device server to client one-way message */
+struct rpmsg_kdrv_demodev_s2c_message {
+	/* message header */
+	struct rpmsg_kdrv_demodev_message_header header;
+	/* message data */
+	u8 data[RPMSG_KDRV_TP_DEMODEV_MESSAGE_DATA_LEN];
+} __packed;
+
+/* demo device client to server one-way message */
+struct rpmsg_kdrv_demodev_c2s_message {
+	/* message header */
+	struct rpmsg_kdrv_demodev_message_header header;
+	/* message data */
+	u8 data[RPMSG_KDRV_TP_DEMODEV_MESSAGE_DATA_LEN];
+} __packed;
+
+#endif
diff -Naur --no-dereference a/drivers/rpmsg-kdrv/shared/rpmsg-kdrv-transport-display.h b/drivers/rpmsg-kdrv/shared/rpmsg-kdrv-transport-display.h
--- a/drivers/rpmsg-kdrv/shared/rpmsg-kdrv-transport-display.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/rpmsg-kdrv/shared/rpmsg-kdrv-transport-display.h	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,223 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Subhajit Paul <subhajit_paul@ti.com>
+ */
+
+#ifndef __RPMSG_KDRV_TRANSPORT_DISPLAY_H__
+#define __RPMSG_KDRV_TRANSPORT_DISPLAY_H__
+
+#include "rpmsg-kdrv-transport-common.h"
+
+/*
+ * Maximum number of planes per buffer
+ */
+#define RPMSG_KDRV_TP_DISPLAY_MAX_PLANES	(2)
+
+/*
+ * Maximum number of shared displays
+ */
+#define RPMSG_KDRV_TP_DISPLAY_MAX_VPS		(2)
+
+/*
+ * Maximum number of pipes per shared display
+ */
+#define RPMSG_KDRV_TP_DISPLAY_MAX_VIDS		(4)
+
+/*
+ * Maximum number of formats supported per pipe
+ */
+#define RPMSG_KDRV_TP_DISPLAY_MAX_FORMATS	(2)
+
+/*
+ * Maximum number of zorders supported per pipe
+ */
+#define RPMSG_KDRV_TP_DISPLAY_MAX_ZORDERS	(4)
+
+enum rpmsg_kdrv_display_format {
+	RPMSG_KDRV_TP_DISPLAY_FORMAT_ARGB8888,
+	RPMSG_KDRV_TP_DISPLAY_FORMAT_XRGB8888,
+	RPMSG_KDRV_TP_DISPLAY_FORMAT_MAX,
+};
+
+enum rpmsg_kdrv_display_message_type {
+	RPMSG_KDRV_TP_DISPLAY_READY_QUERY_REQUEST,
+	RPMSG_KDRV_TP_DISPLAY_READY_QUERY_RESPONSE,
+	RPMSG_KDRV_TP_DISPLAY_RES_INFO_REQUEST,
+	RPMSG_KDRV_TP_DISPLAY_RES_INFO_RESPONSE,
+	RPMSG_KDRV_TP_DISPLAY_COMMIT_REQUEST,
+	RPMSG_KDRV_TP_DISPLAY_COMMIT_RESPONSE,
+	RPMSG_KDRV_TP_DISPLAY_COMMIT_DONE_MESSAGE,
+	RPMSG_KDRV_TP_DISPLAY_BUFFER_DONE_MESSAGE,
+	RPMSG_KDRV_TP_DISPLAY_MAX,
+};
+
+/*
+ * per-device data for display device
+ */
+struct rpmsg_kdrv_display_device_data {
+	/* Does the device send all vsyncs? */
+	u8 periodic_vsync;
+	/*Does the device defer the use of buffers? */
+	u8 deferred_buffer_usage;
+} __packed;
+
+/*
+ * message header for display device
+ */
+struct rpmsg_kdrv_display_message_header {
+	/* enum: rpmsg_kdrv_display_message_type */
+	u8 message_type;
+} __packed;
+
+/* display device request to provide ready / not-ready info */
+struct rpmsg_kdrv_display_ready_query_request {
+	/* message header */
+	struct rpmsg_kdrv_display_message_header header;
+} __packed;
+
+/* display device response indicating ready / not-ready status */
+struct rpmsg_kdrv_display_ready_query_response {
+	/* message header */
+	struct rpmsg_kdrv_display_message_header header;
+	/* can be 0 : if not ready 1: if ready */
+	u8 ready;
+} __packed;
+
+/* display device buffer update info */
+struct rpmsg_kdrv_display_buffer_info {
+	/* buffer width */
+	u16 width;
+	/* buffer height */
+	u16 height;
+	/* enum: rpmsg_kdrv_display_format */
+	u8 format;
+	/* number of planes */
+	u8 num_planes;
+	/* per plane start addresses */
+	u64 plane[RPMSG_KDRV_TP_DISPLAY_MAX_PLANES];
+	/* per plane pitch */
+	u16 pitch[RPMSG_KDRV_TP_DISPLAY_MAX_PLANES];
+	/* buffer id : to be used in buffer-done message */
+	u32 buffer_id;
+} __packed;
+
+/* display device pipe update info */
+struct rpmsg_kdrv_display_vid_update_info {
+	/* pipe ID */
+	u8 id;
+	/*enable / disable request */
+	u8 enabled;
+	/* window width */
+	u16 dst_w;
+	/* window height */
+	u16 dst_h;
+	/* window position X */
+	u16 dst_x;
+	/* window position Y */
+	u16 dst_y;
+	/* buffer */
+	struct rpmsg_kdrv_display_buffer_info buffer;
+} __packed;
+
+/* display device commit request */
+struct rpmsg_kdrv_display_commit_request {
+	/* message header */
+	struct rpmsg_kdrv_display_message_header header;
+	/*ID of shared display */
+	u8 id;
+	/* number of pipe updates in the commit */
+	u8 num_vid_updates;
+	/* list of pipe updates */
+	struct rpmsg_kdrv_display_vid_update_info vid[RPMSG_KDRV_TP_DISPLAY_MAX_VIDS];
+	/*commit id : to be used in commit-done message */
+	u32 commit_id;
+} __packed;
+
+/* display device commit response */
+struct rpmsg_kdrv_display_commit_response {
+	/* message header */
+	struct rpmsg_kdrv_display_message_header header;
+	/*commit id : from commit request */
+	u32 commit_id;
+	/*status : 0 = accepted, 1 = rejected */
+	u8 status;
+} __packed;
+
+/* display device commit done message */
+struct rpmsg_kdrv_display_commit_done_message {
+	/* message header */
+	struct rpmsg_kdrv_display_message_header header;
+	/* commit id : from commit request */
+	u32 commit_id;
+} __packed;
+
+/*display device buffer deferred release message */
+struct rpmsg_kdrv_display_buffer_done_message {
+	/* message header */
+	struct rpmsg_kdrv_display_message_header header;
+	/* buffer id: from bufer_info */
+	u32 buffer_id;
+} __packed;
+
+/* display device request to provide list of shared resources */
+struct rpmsg_kdrv_display_res_info_request {
+	/* message header */
+	struct rpmsg_kdrv_display_message_header header;
+} __packed;
+
+/* display device shared pipe */
+struct rpmsg_kdrv_display_vid_info {
+	/* pipe ID */
+	u8 id;
+	/* is pipe window fixed on display? */
+	u8 mutable_window;
+	/* fixed window position X, if applicable */
+	u16 fixed_window_x;
+	/* fixed window position Y, if applicable */
+	u16 fixed_window_y;
+	/* fixed window width, if applicable */
+	u16 fixed_window_w;
+	/* fixed window height, if applicable */
+	u16 fixed_window_h;
+	/* can pipe scale buffers? */
+	u8 can_scale;
+	/* number of formats supported */
+	u8 num_formats;
+	/*enum: rpmsg_kdrv_display_format */
+	u8 format[RPMSG_KDRV_TP_DISPLAY_MAX_FORMATS];
+	/* initial zorder of pipe */
+	u8 init_zorder;
+	/* number of allowed zorders */
+	u8 num_zorders;
+	/* list of allowed zorders */
+	u8 zorder[RPMSG_KDRV_TP_DISPLAY_MAX_ZORDERS];
+} __packed;
+
+/* display device shared display */
+struct rpmsg_kdrv_display_vp_info {
+	/* ID of shared display */
+	u8 id;
+	/* raster width */
+	u16 width;
+	/* raster height */
+	u16 height;
+	/* refresh rate */
+	u8 refresh;
+	/* number of pipes for this display */
+	u8 num_vids;
+	/* list of pipes */
+	struct rpmsg_kdrv_display_vid_info vid[RPMSG_KDRV_TP_DISPLAY_MAX_VIDS];
+} __packed;
+
+/* display device response providing list of shared resources */
+struct rpmsg_kdrv_display_res_info_response {
+	/* message header */
+	struct rpmsg_kdrv_display_message_header header;
+	/* number of shared displays */
+	u8 num_vps;
+	/* list of shared displays */
+	struct rpmsg_kdrv_display_vp_info vp[RPMSG_KDRV_TP_DISPLAY_MAX_VPS];
+} __packed;
+
+#endif
diff -Naur --no-dereference a/drivers/rpmsg-kdrv/shared/rpmsg-kdrv-transport.h b/drivers/rpmsg-kdrv/shared/rpmsg-kdrv-transport.h
--- a/drivers/rpmsg-kdrv/shared/rpmsg-kdrv-transport.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/rpmsg-kdrv/shared/rpmsg-kdrv-transport.h	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,83 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Subhajit Paul <subhajit_paul@ti.com>
+ */
+
+#ifndef __RPMSG_KDRV_TRANSPORT_H__
+#define __RPMSG_KDRV_TRANSPORT_H__
+
+#include "rpmsg-kdrv-transport-common.h"
+
+enum rpmsg_kdrv_init_message_type {
+	RPMSG_KDRV_TP_INIT_DEV_INFO_REQUEST,
+	RPMSG_KDRV_TP_INIT_DEV_INFO_RESPONSE,
+	RPMSG_KDRV_TP_INIT_CONNECT_MESSAGE,
+	RPMSG_KDRV_TP_INIT_DISCONNECT_MESSAGE,
+	RPMSG_KDRV_TP_INIT_MAX,
+};
+
+/*
+ * message header for init device
+ */
+struct rpmsg_kdrv_init_message_header {
+	/* enum: rpmsg_kdrv_init_message_type */
+	u8 message_type;
+} __packed;
+
+/*
+ * init device request to provide list of devices
+ */
+struct rpmsg_kdrv_init_dev_info_request {
+	/* message header */
+	struct rpmsg_kdrv_init_message_header header;
+} __packed;
+
+struct rpmsg_kdrv_init_device_info {
+	/* device id */
+	u8 device_id;
+	/* device type (display, capture etc) */
+	u8 device_type;
+	/* name of device */
+	u8 device_name[RPMSG_KDRV_TP_DEVICE_NAME_LEN];
+	/* device specific info length */
+	u16 device_data_len;
+	/* per device-type info offset */
+	u16 device_data_offset;
+} __packed;
+
+/*
+ * init device response with list of devices
+ */
+struct rpmsg_kdrv_init_dev_info_response {
+	/* message header */
+	struct rpmsg_kdrv_init_message_header header;
+	/*number of exported devices */
+	u8 num_devices;
+	/* list of exported devices */
+	struct rpmsg_kdrv_init_device_info devices[RPMSG_KDRV_TP_MAX_DEVICES];
+	/* device specific data */
+	u8 device_data[0];
+} __packed;
+
+/*
+ * init device per-device connect message
+ */
+struct rpmsg_kdrv_init_connect_message {
+	/* message header */
+	struct rpmsg_kdrv_init_message_header header;
+	/* device ID to connect */
+	u8 device_id;
+} __packed;
+
+/*
+ * init device per-device disconnect message
+ */
+struct rpmsg_kdrv_init_disconnect_message {
+	/* message header */
+	struct rpmsg_kdrv_init_message_header header;
+	/* device ID to disconnect */
+	u8 device_id;
+} __packed;
+
+#endif
diff -Naur --no-dereference a/drivers/rpmsg-kdrv/shared/rpmsg-kdrv-transport-switch.h b/drivers/rpmsg-kdrv/shared/rpmsg-kdrv-transport-switch.h
--- a/drivers/rpmsg-kdrv/shared/rpmsg-kdrv-transport-switch.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/rpmsg-kdrv/shared/rpmsg-kdrv-transport-switch.h	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,563 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Grygorii Strashko <grygorii.strashko@ti.com>
+ */
+
+#ifndef DRIVERS_RPMSG_KDRV_SHARED_RPMSG_KDRV_TRANSPORT_SWITCH_H_
+#define DRIVERS_RPMSG_KDRV_SHARED_RPMSG_KDRV_TRANSPORT_SWITCH_H_
+
+#include <linux/etherdevice.h>
+#include "rpmsg-kdrv-transport-common.h"
+
+#define RPMSG_KDRV_TP_ETHSWITCH_VERSION_MAJOR             (0)
+#define RPMSG_KDRV_TP_ETHSWITCH_VERSION_MINOR             (1)
+#define RPMSG_KDRV_TP_ETHSWITCH_VERSION_REVISION          (1)
+
+/**
+ * enum rpmsg_kdrv_ethswitch_message_type - Eth switch rpmsg protocol messages
+ */
+enum rpmsg_kdrv_ethswitch_message_type {
+	RPMSG_KDRV_TP_ETHSWITCH_ATTACH = 0x00,
+	RPMSG_KDRV_TP_ETHSWITCH_ATTACH_EXT = 0x01,
+	RPMSG_KDRV_TP_ETHSWITCH_ALLOC_TX = 0x02,
+	RPMSG_KDRV_TP_ETHSWITCH_ALLOC_RX = 0x03,
+	RPMSG_KDRV_TP_ETHSWITCH_REGISTER_DEFAULTFLOW = 0x04,
+	RPMSG_KDRV_TP_ETHSWITCH_ALLOC_MAC = 0x05,
+	RPMSG_KDRV_TP_ETHSWITCH_REGISTER_MAC = 0x06,
+	RPMSG_KDRV_TP_ETHSWITCH_UNREGISTER_MAC = 0x07,
+	RPMSG_KDRV_TP_ETHSWITCH_UNREGISTER_DEFAULTFLOW = 0x08,
+	RPMSG_KDRV_TP_ETHSWITCH_FREE_MAC = 0x09,
+	RPMSG_KDRV_TP_ETHSWITCH_FREE_TX = 0x0A,
+	RPMSG_KDRV_TP_ETHSWITCH_FREE_RX = 0x0B,
+	RPMSG_KDRV_TP_ETHSWITCH_DETACH = 0x0C,
+	RPMSG_KDRV_TP_ETHSWITCH_IOCTL = 0x0D,
+	RPMSG_KDRV_TP_ETHSWITCH_REGWR = 0x0E,
+	RPMSG_KDRV_TP_ETHSWITCH_REGRD = 0x0F,
+	RPMSG_KDRV_TP_ETHSWITCH_IPV4_MAC_REGISTER = 0x10,
+	RPMSG_KDRV_TP_ETHSWITCH_IPV6_MAC_REGISTER = 0x11,
+	RPMSG_KDRV_TP_ETHSWITCH_IPV4_MAC_UNREGISTER = 0x12,
+	RPMSG_KDRV_TP_ETHSWITCH_IPV6_MAC_UNREGISTER = 0x13,
+	RPMSG_KDRV_TP_ETHSWITCH_PING_REQUEST = 0x14,
+	RPMSG_KDRV_TP_ETHSWITCH_S2C_NOTIFY = 0x15,
+	RPMSG_KDRV_TP_ETHSWITCH_C2S_NOTIFY = 0x16,
+	RPMSG_KDRV_TP_ETHSWITCH_MAX = 0x17,
+};
+
+/**
+ * Client to Eth switch notification events @RPMSG_KDRV_TP_ETHSWITCH_C2S_NOTIFY
+ */
+enum rpmsg_kdrv_ethswitch_c2s_notify_type {
+	RPMSG_KDRV_TP_ETHSWITCH_CLIENTNOTIFY_DUMPSTATS = 0x00,
+	RPMSG_KDRV_TP_ETHSWITCH_CLIENTNOTIFY_MAX,
+};
+
+/**
+ * Eth switch HW ID
+ */
+enum rpmsg_kdrv_ethswitch_cpsw_type {
+	RPMSG_KDRV_TP_ETHSWITCH_CPSWTYPE_2G,
+	RPMSG_KDRV_TP_ETHSWITCH_CPSWTYPE_9G,
+	RPMSG_KDRV_TP_ETHSWITCH_CPSWTYPE_MAX,
+};
+
+/**
+ * Response status codes returned by Eth switch FW in
+ * struct @rpmsg_kdrv_ethswitch_common_resp_info
+ */
+#define RPMSG_KDRV_TP_ETHSWITCH_CMDSTATUS_OK		(0)
+#define RPMSG_KDRV_TP_ETHSWITCH_CMDSTATUS_EAGAIN	(-1)
+#define RPMSG_KDRV_TP_ETHSWITCH_CMDSTATUS_EFAIL		(-2)
+#define RPMSG_KDRV_TP_ETHSWITCH_CMDSTATUS_EACCESS	(-3)
+
+/* Maximum length of message data */
+#define RPMSG_KDRV_TP_ETHSWITCH_MESSAGE_DATA_LEN	(128)
+
+/* Number of priorities supported by CPSW */
+#define RPMSG_KDRV_TP_ETHSWITCH_PRIORITY_NUM   (8)
+
+/* IPv4 Address length in octets */
+#define RPMSG_KDRV_TP_ETHSWITCH_IPV4ADDRLEN         (4)
+
+/**
+ * struct rpmsg_kdrv_ethswitch_msg_header - Message Header for outgoing messages
+ *
+ * @message_type: Type of messages: One of
+ *	enum @rpmsg_kdrv_ethswitch_message_type values
+ */
+struct rpmsg_kdrv_ethswitch_msg_header {
+	u8 message_type;
+} __packed;
+
+/**
+ * struct rpmsg_kdrv_ethswitch_common_req_info - common request msgs data
+ *
+ * @id: unique handle
+ * @core_key: core specific key to indicate attached core
+ *
+ * Common structure used for all Eth switch FW request msgs except
+ * @RPMSG_KDRV_TP_ETHSWITCH_ATTACH. It has to be filled with values returned
+ * by @RPMSG_KDRV_TP_ETHSWITCH_ATTACH.
+ */
+struct rpmsg_kdrv_ethswitch_common_req_info {
+	u64 id;
+	u32 core_key;
+} __packed;
+
+/**
+ * struct rpmsg_kdrv_ethswitch_common_resp_info - common response data
+ *
+ * @status: status of request
+ *
+ * Common data returned by Eth switch FW in all response messages to identify
+ * status of request message processing.
+ */
+struct rpmsg_kdrv_ethswitch_common_resp_info {
+	s32 status;
+} __packed;
+
+/**
+ * struct rpmsg_kdrv_ethswitch_attach_req - attach cmd client request msg
+ *
+ * @header: msg header
+ * @cpsw_type: CPSW HW type enum @rpmsg_kdrv_ethswitch_cpsw_type
+ *
+ * Client attach message @RPMSG_KDRV_TP_ETHSWITCH_ATTACH. it should be always
+ * sent first before other requests to Eth switch FW.
+ */
+struct rpmsg_kdrv_ethswitch_attach_req {
+	struct rpmsg_kdrv_ethswitch_msg_header header;
+	u8 cpsw_type;
+} __packed;
+
+/**
+ * struct rpmsg_kdrv_ethswitch_attach_resp - attach client response msg
+ *
+ * @info: common response data. Status of the request processing
+ * @id: unique handle used by all further CMDs
+ * @core_key: core specific key to indicate attached core
+ * @rx_mtu: MTU of rx packets
+ * @tx_mtu: MTU of tx packet per priority
+ * @features: supported features mask
+ *
+ * Attach client response msg received as response to client attach request
+ * @RPMSG_KDRV_TP_ETHSWITCH_ATTACH. The @id and @core_key should be used to
+ * fill struct @rpmsg_kdrv_ethswitch_common_req_info in all further request
+ * messages.
+ * The @info.status field is @RPMSG_KDRV_TP_ETHSWITCH_CMDSTATUS_OK on success.
+ */
+struct rpmsg_kdrv_ethswitch_attach_resp {
+	struct rpmsg_kdrv_ethswitch_common_resp_info info;
+	u64 id;
+	u32 core_key;
+	u32 rx_mtu;
+	u32 tx_mtu[RPMSG_KDRV_TP_ETHSWITCH_PRIORITY_NUM];
+	u32 features;
+#define RPMSG_KDRV_TP_ETHSWITCH_FEATURE_TXCSUM BIT(0)
+} __packed;
+
+/**
+ * struct rpmsg_kdrv_ethswitch_attach_extended_req - extended attach request msg
+ *
+ * @header: msg header
+ * @cpsw_type: CPSW HW type enum @rpmsg_kdrv_ethswitch_cpsw_type
+ *
+ * Client extended attach request @RPMSG_KDRV_TP_ETHSWITCH_ATTACH_EXT. It can
+ * be used instead of @RPMSG_KDRV_TP_ETHSWITCH_ATTACH and has to sent first
+ * before other requests to Eth switch FW.
+ */
+struct rpmsg_kdrv_ethswitch_attach_extended_req {
+	struct rpmsg_kdrv_ethswitch_msg_header header;
+	u8 cpsw_type;
+} __packed;
+
+/**
+ * struct rpmsg_kdrv_ethswitch_common_response_info - extended attach resp msg
+ *
+ * @info: common response data. Status of the request processing
+ * @id: unique handle used by all further CMDs
+ * @core_key: core specific key to indicate attached core
+ * @rx_mtu: MTU of rx packets
+ * @tx_mtu: MTU of tx packet per priority
+ * @features: supported features mask
+ * @alloc_flow_idx: RX UDMA flow ID
+ * @tx_cpsw_psil_dst_id: PSI-L dest thread id
+ * @mac_address: default eth MAC address assigned to this client
+ *
+ * Extended attach response msg received as response to client extended attach
+ * request @RPMSG_KDRV_TP_ETHSWITCH_ATTACH_EXT. The @id and @core_key should be
+ * used to fill struct @rpmsg_kdrv_ethswitch_common_req_info in all further
+ * request messages. In addition, it provides allocated DMA resources and
+ * MAC address.
+ *
+ * The @info.status field is @RPMSG_KDRV_TP_ETHSWITCH_CMDSTATUS_OK on success.
+ */
+struct rpmsg_kdrv_ethswitch_attach_extended_resp {
+	struct rpmsg_kdrv_ethswitch_common_resp_info info;
+	u64 id;
+	u32 core_key;
+	u32 rx_mtu;
+	u32 tx_mtu[RPMSG_KDRV_TP_ETHSWITCH_PRIORITY_NUM];
+	u32 features;
+	u32 alloc_flow_idx;
+	u32 tx_cpsw_psil_dst_id;
+	u8 mac_address[ETH_ALEN];
+} __packed;
+
+/**
+ * struct rpmsg_kdrv_ethswitch_detach_req  - detach client request msg
+ *
+ * @header: msg header
+ * @info: common request msgs data
+ *
+ * Client detach request message @RPMSG_KDRV_TP_ETHSWITCH_DETACH.
+ * it should be always sent as the last message to Eth switch FW.
+ */
+struct rpmsg_kdrv_ethswitch_detach_req {
+	struct rpmsg_kdrv_ethswitch_msg_header header;
+	struct rpmsg_kdrv_ethswitch_common_req_info info;
+} __packed;
+
+/**
+ * struct rpmsg_kdrv_ethswitch_detach_resp - detach client response msg
+ *
+ * @info: common response data. Status of the request processing
+ *
+ * Client detach response msg received as response to client detach request
+ * @RPMSG_KDRV_TP_ETHSWITCH_DETACH.
+ * The @info.status field is @RPMSG_KDRV_TP_ETHSWITCH_CMDSTATUS_OK on success.
+ */
+struct rpmsg_kdrv_ethswitch_detach_resp {
+	struct rpmsg_kdrv_ethswitch_common_resp_info info;
+} __packed;
+
+/**
+ * struct rpmsg_kdrv_ethswitch_alloc_req  - alloc resources request msg
+ *
+ * @header: msg header
+ * @info: common request msgs data
+ *
+ * Client resources allocation request messages
+ * @RPMSG_KDRV_TP_ETHSWITCH_ALLOC_RX: get RX DMA resources
+ * @RPMSG_KDRV_TP_ETHSWITCH_ALLOC_TX: get TX DMA resources
+ * @RPMSG_KDRV_TP_ETHSWITCH_ALLOC_MAC: get MAC address
+ */
+struct rpmsg_kdrv_ethswitch_alloc_req {
+	struct rpmsg_kdrv_ethswitch_msg_header header;
+	struct rpmsg_kdrv_ethswitch_common_req_info info;
+} __packed;
+
+/**
+ * struct rpmsg_kdrv_ethswitch_alloc_rx_resp - alloc rx resources response msg
+ *
+ * @info: common response data. Status of the request processing
+ * @alloc_flow_idx: RX UDMA flow ID
+ *
+ * Client alloc rx resources response msg received as response to request
+ * @RPMSG_KDRV_TP_ETHSWITCH_ALLOC_RX. The @alloc_flow_idx is RX UDMA flow ID
+ * to be used for ingress packets reception.
+ * The @info.status field is @RPMSG_KDRV_TP_ETHSWITCH_CMDSTATUS_OK on success.
+ */
+struct rpmsg_kdrv_ethswitch_alloc_rx_resp {
+	struct rpmsg_kdrv_ethswitch_common_resp_info info;
+	u32 alloc_flow_idx;
+} __packed;
+
+/**
+ * struct rpmsg_kdrv_ethswitch_alloc_tx_resp - alloc tx resources response msg
+ *
+ * @info: common response data. Status of the request processing
+ * @tx_cpsw_psil_dst_id: PSI-L dest thread id
+ *
+ * Client alloc tx resources response msg received as response to request
+ * @RPMSG_KDRV_TP_ETHSWITCH_ALLOC_TX. The @tx_cpsw_psil_dst_id is TX PSI-L dest
+ * thread ID to be used for TX UDMA channel setup.
+ * The @info.status field is @RPMSG_KDRV_TP_ETHSWITCH_CMDSTATUS_OK on success.
+ */
+struct rpmsg_kdrv_ethswitch_alloc_tx_resp {
+	struct rpmsg_kdrv_ethswitch_common_resp_info info;
+	u32 tx_cpsw_psil_dst_id;
+} __packed;
+
+/**
+ * struct rpmsg_kdrv_ethswitch_alloc_mac_resp - alloc MAC resources response msg
+ *
+ * @info: common response data. Status of the request processing
+ * @mac_address: default eth MAC address assigned to this client
+ *
+ * Client alloc MAC resources response msg received as response to request
+ * @RPMSG_KDRV_TP_ETHSWITCH_ALLOC_MAC.
+ * The @info.status field is @RPMSG_KDRV_TP_ETHSWITCH_CMDSTATUS_OK on success.
+ */
+struct rpmsg_kdrv_ethswitch_alloc_mac_resp {
+	struct rpmsg_kdrv_ethswitch_common_resp_info info;
+	u8 mac_address[ETH_ALEN];
+} __packed;
+
+/**
+ * struct rpmsg_kdrv_ethswitch_register_mac_req - register MAC addr
+ *
+ * @header: msg header
+ * @info: common request msgs data
+ * @mac_address: eth MAC address used by client
+ * @flow_idx: RX UDMA flow ID
+ *
+ * Client register MAC addr message @RPMSG_KDRV_TP_ETHSWITCH_REGISTER_MAC.
+ * it should be sent to Eth switch FW to configure HW network traffic
+ * classifiers so all network traffic directed to @mac_address will be
+ * redirected to this client and can be received through allocated RX UDMA flow
+ * @flow_idx.
+ *
+ * This message has to be sent by client when it's ready to receive network
+ * traffic.
+ */
+struct rpmsg_kdrv_ethswitch_register_mac_req {
+	struct rpmsg_kdrv_ethswitch_msg_header header;
+	struct rpmsg_kdrv_ethswitch_common_req_info info;
+	u8 mac_address[ETH_ALEN];
+	u32 flow_idx;
+} __packed;
+
+/**
+ * struct rpmsg_kdrv_ethswitch_register_mac_resp - register MAC addr response
+ *
+ * @info: common response data. Status of the request processing
+ *
+ * Client register MAC addr response response msg received as response to
+ * request @RPMSG_KDRV_TP_ETHSWITCH_REGISTER_MAC.
+ * The @info.status field is @RPMSG_KDRV_TP_ETHSWITCH_CMDSTATUS_OK on success.
+ */
+struct rpmsg_kdrv_ethswitch_register_mac_resp {
+	struct rpmsg_kdrv_ethswitch_common_resp_info info;
+} __packed;
+
+/**
+ * struct rpmsg_kdrv_ethswitch_unregister_mac_req - unregister MAC addr
+ *
+ * @header: msg header
+ * @info: common request msgs data
+ * @mac_address: eth MAC address used by client
+ * @flow_idx: RX UDMA flow ID
+ *
+ * Client unregister MAC addr message @RPMSG_KDRV_TP_ETHSWITCH_UNREGISTER_MAC.
+ * it should be sent to Eth switch FW to disable HW network traffic
+ * classifiers so all network traffic directed to @mac_address will be dropped.
+ *
+ * This message has to be sent by client when it does not want to receive any
+ * more network traffic.
+ */
+struct rpmsg_kdrv_ethswitch_unregister_mac_req {
+	struct rpmsg_kdrv_ethswitch_msg_header header;
+	struct rpmsg_kdrv_ethswitch_common_req_info info;
+	u8 mac_address[ETH_ALEN];
+	u32 flow_idx;
+} __packed;
+
+/**
+ * struct rpmsg_kdrv_ethswitch_unregister_mac_resp - unregister MAC addr resp
+ *
+ * @info: common response data. Status of the request processing
+ *
+ * Client unregister MAC addr response msg received as response to
+ * request @RPMSG_KDRV_TP_ETHSWITCH_UNREGISTER_MAC.
+ * The @info.status field is @RPMSG_KDRV_TP_ETHSWITCH_CMDSTATUS_OK on success.
+ */
+struct rpmsg_kdrv_ethswitch_unregister_mac_resp {
+	struct rpmsg_kdrv_ethswitch_common_resp_info info;
+} __packed;
+
+/**
+ * struct rpmsg_kdrv_ethswitch_ipv4_register_mac_req - register IPv4:MAC pair
+ *
+ * @header: msg header
+ * @info: common request msgs data
+ * @mac_address: eth MAC address used by client
+ * @ipv4_addr: IPv4 addr
+ *
+ * Client register IPv4:MAC addr pair message
+ * @RPMSG_KDRV_TP_ETHSWITCH_IPV4_MAC_REGISTER registers pair of IPv4 @ipv4_addr
+ * and Eth MAC @mac_address addresses in Eth switch FW ARP database.
+ *
+ * This message has to be sent by client when there is new IPv4 addr assigned.
+ */
+struct rpmsg_kdrv_ethswitch_ipv4_register_mac_req {
+	struct rpmsg_kdrv_ethswitch_msg_header header;
+	struct rpmsg_kdrv_ethswitch_common_req_info info;
+	u8 mac_address[ETH_ALEN];
+	u8 ipv4_addr[RPMSG_KDRV_TP_ETHSWITCH_IPV4ADDRLEN];
+} __packed;
+
+/**
+ * struct rpmsg_kdrv_ethswitch_ipv4_register_mac_resp - register IPv4:MAC pair
+ *	response
+ *
+ * @info: common response data. Status of the request processing
+ *
+ * Client register IPv4:MAC addr pair response msg received as response to
+ * request @RPMSG_KDRV_TP_ETHSWITCH_IPV4_MAC_REGISTER.
+ * The @info.status field is @RPMSG_KDRV_TP_ETHSWITCH_CMDSTATUS_OK on success.
+ */
+struct rpmsg_kdrv_ethswitch_ipv4_register_mac_resp {
+	struct rpmsg_kdrv_ethswitch_common_resp_info info;
+} __packed;
+
+/**
+ * struct rpmsg_kdrv_ethswitch_ipv4_unregister_mac_req - unregister IPv4 addr
+ *
+ * @header: msg header
+ * @info: common request msgs data
+ * @ipv4_addr: IPv4 addr
+ *
+ * Client unregister IPv4 addr message
+ * @RPMSG_KDRV_TP_ETHSWITCH_IPV4_MAC_UNREGISTER. It removes IPv4 @ipv4_addr
+ * address from Eth switch FW ARP database.
+ *
+ * This message has to be sent by client when there is IPv4 addr unassigned.
+ */
+struct rpmsg_kdrv_ethswitch_ipv4_unregister_mac_req {
+	struct rpmsg_kdrv_ethswitch_msg_header header;
+	struct rpmsg_kdrv_ethswitch_common_req_info info;
+	u8 ipv4_addr[RPMSG_KDRV_TP_ETHSWITCH_IPV4ADDRLEN];
+} __packed;
+
+/**
+ * struct rpmsg_kdrv_ethswitch_ipv4_unregister_mac_resp - unregister IPv4 addr
+ *
+ * @info: common response data. Status of the request processing
+ *
+ * Client unregister IPv4 addr response msg received as response to
+ * request @RPMSG_KDRV_TP_ETHSWITCH_IPV4_MAC_UNREGISTER.
+ * The @info.status field is @RPMSG_KDRV_TP_ETHSWITCH_CMDSTATUS_OK on success.
+ */
+struct rpmsg_kdrv_ethswitch_ipv4_unregister_mac_resp {
+	struct rpmsg_kdrv_ethswitch_common_resp_info info;
+} __packed;
+
+/**
+ * struct rpmsg_kdrv_ethswitch_ping_req - ping request
+ *
+ * @header: msg header
+ * @data: custom data
+ *
+ * Client ping request @RPMSG_KDRV_TP_ETHSWITCH_PING_REQUEST. The Eth switch FW
+ * should return the same @data in struct @rpmsg_kdrv_ethswitch_ping_resp.
+ * Can be used any time - no attach required.
+ */
+struct rpmsg_kdrv_ethswitch_ping_req {
+	struct rpmsg_kdrv_ethswitch_msg_header header;
+	u8 data[RPMSG_KDRV_TP_ETHSWITCH_MESSAGE_DATA_LEN];
+} __packed;
+
+/**
+ * struct rpmsg_kdrv_ethswitch_ping_resp - ping response
+ *
+ * @data: custom data
+ *
+ * The ping response msg received as response to request
+ * @RPMSG_KDRV_TP_ETHSWITCH_PING_REQUEST. The Eth switch FW should return
+ * the same @data as was provided in struct @rpmsg_kdrv_ethswitch_ping_req.
+ */
+struct rpmsg_kdrv_ethswitch_ping_resp {
+	u8 data[RPMSG_KDRV_TP_ETHSWITCH_MESSAGE_DATA_LEN];
+} __packed;
+
+/**
+ * struct rpmsg_kdrv_ethswitch_regrd_req - read hw register request
+ *
+ * @header: msg header
+ * @regaddr: phys register address
+ *
+ * The read hw register request @RPMSG_KDRV_TP_ETHSWITCH_REGRD.
+ * The Eth switch FW should return the @regaddr register value.
+ * Can be used any time - no attach required.
+ */
+struct rpmsg_kdrv_ethswitch_regrd_req {
+	struct rpmsg_kdrv_ethswitch_msg_header header;
+	u32 regaddr;
+} __packed;
+
+/**
+ * struct rpmsg_kdrv_ethswitch_regrd_resp - read hw register response
+ *
+ * @info: common response data. Status of the request processing
+ * @regval: register value
+ *
+ * The read hw register response received as response to request
+ * @RPMSG_KDRV_TP_ETHSWITCH_REGRD. The @regval is hw register value from
+ * @regaddr phys register address provided in
+ * struct @rpmsg_kdrv_ethswitch_regrd_req
+ *
+ * The @info.status field is @RPMSG_KDRV_TP_ETHSWITCH_CMDSTATUS_OK on success.
+ */
+struct rpmsg_kdrv_ethswitch_regrd_resp {
+	struct rpmsg_kdrv_ethswitch_common_resp_info info;
+	u32 regval;
+} __packed;
+
+/**
+ * struct rpmsg_kdrv_ethswitch_c2s_notify - notification request
+ *
+ * @header: msg header
+ * @info: common request msg data
+ * @notifyid: enum @rpmsg_kdrv_ethswitch_c2s_notify_type
+ * @notify_info_len: length of @notify_info
+ * @notify_info: notification message data
+ *
+ * The notification request message @RPMSG_KDRV_TP_ETHSWITCH_C2S_NOTIFY is one
+ * way message to Eth switch FW without response.
+ */
+struct rpmsg_kdrv_ethswitch_c2s_notify {
+	struct rpmsg_kdrv_ethswitch_msg_header header;
+	struct rpmsg_kdrv_ethswitch_common_req_info info;
+	u8 notifyid;
+	u32 notify_info_len;
+	u8 notify_info[RPMSG_KDRV_TP_ETHSWITCH_MESSAGE_DATA_LEN];
+} __packed;
+
+/**
+ * struct rpmsg_kdrv_ethswitch_fw_version_info - fw version info
+ *
+ * @major: major
+ * @minor: minor
+ * @rev: revision
+ * @year:
+ * @month:
+ * @date: build date
+ * @commit_hash: commit hash
+ */
+struct rpmsg_kdrv_ethswitch_fw_version_info {
+#define RPMSG_KDRV_TP_ETHSWITCH_YEARLEN		(4)
+#define RPMSG_KDRV_TP_ETHSWITCH_MONTHLEN	(3)
+#define RPMSG_KDRV_TP_ETHSWITCH_DATELEN		(2)
+#define RPMSG_KDRV_TP_ETHSWITCH_COMMITSHALEN	(8)
+	u32 major;
+	u32 minor;
+	u32 rev;
+	char year[RPMSG_KDRV_TP_ETHSWITCH_YEARLEN];
+	char month[RPMSG_KDRV_TP_ETHSWITCH_MONTHLEN];
+	char date[RPMSG_KDRV_TP_ETHSWITCH_DATELEN];
+	char commit_hash[RPMSG_KDRV_TP_ETHSWITCH_COMMITSHALEN];
+} __packed;
+
+/*
+ * per-device data for ethswitch device
+ */
+/**
+ * struct rpmsg_kdrv_ethswitch_device_data - rpmsg_kdrv_device data
+ *
+ * @fw_ver: fw version info
+ * @permission_flags: permission enabled for each
+ *	enum @rpmsg_kdrv_ethswitch_message_type command
+ * @uart_connected: flag indicating if UART is connected
+ * @uart_id: UART ID used by firmware for log prints
+ *
+ * Provided as part of RPMSG KDRV device discovery protocol
+ */
+struct rpmsg_kdrv_ethswitch_device_data {
+	struct rpmsg_kdrv_ethswitch_fw_version_info fw_ver;
+	u32 permission_flags;
+	u32 uart_connected;
+	u32 uart_id;
+} __packed;
+
+#endif /* DRIVERS_RPMSG_KDRV_SHARED_RPMSG_KDRV_TRANSPORT_SWITCH_H_ */
diff -Naur --no-dereference a/drivers/rtc/interface.c b/drivers/rtc/interface.c
--- a/drivers/rtc/interface.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/rtc/interface.c	2020-06-19 21:00:31.424227716 -0400
@@ -1093,3 +1093,15 @@
 	trace_rtc_set_offset(offset, ret);
 	return ret;
 }
+
+/**
+ * rtc_power_off_program - Some of the rtc are hooked on to PMIC_EN
+ * line and can be used to power off the SoC.
+ *
+ * Kernel interface to program rtc to power off
+ */
+int rtc_power_off_program(struct rtc_device *rtc)
+{
+	return rtc->ops->power_off_program(rtc->dev.parent);
+}
+EXPORT_SYMBOL_GPL(rtc_power_off_program);
diff -Naur --no-dereference a/drivers/rtc/rtc-omap.c b/drivers/rtc/rtc-omap.c
--- a/drivers/rtc/rtc-omap.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/rtc/rtc-omap.c	2020-06-19 21:00:31.424227716 -0400
@@ -415,25 +415,17 @@
 
 static struct omap_rtc *omap_rtc_power_off_rtc;
 
-/*
- * omap_rtc_poweroff: RTC-controlled power off
- *
- * The RTC can be used to control an external PMIC via the pmic_power_en pin,
- * which can be configured to transition to OFF on ALARM2 events.
- *
- * Notes:
- * The two-second alarm offset is the shortest offset possible as the alarm
- * registers must be set before the next timer update and the offset
- * calculation is too heavy for everything to be done within a single access
- * period (~15 us).
- *
- * Called with local interrupts disabled.
+/**
+ * omap_rtc_power_off_program: Set the pmic power off sequence. The RTC
+ * generates pmic_pwr_enable control, which can be used to control an external
+ * PMIC.
  */
-static void omap_rtc_power_off(void)
+static int omap_rtc_power_off_program(struct device *dev)
 {
 	struct omap_rtc *rtc = omap_rtc_power_off_rtc;
 	struct rtc_time tm;
 	unsigned long now;
+	int seconds;
 	u32 val;
 
 	rtc->type->unlock(rtc);
@@ -441,16 +433,21 @@
 	val = rtc_readl(rtc, OMAP_RTC_PMIC_REG);
 	rtc_writel(rtc, OMAP_RTC_PMIC_REG, val | OMAP_RTC_PMIC_POWER_EN_EN);
 
-	/* set alarm two seconds from now */
+again:
+	/* Clear any existing ALARM2 event */
+	rtc_writel(rtc, OMAP_RTC_STATUS_REG, OMAP_RTC_STATUS_ALARM2);
+
+	/* set alarm one second from now */
 	omap_rtc_read_time_raw(rtc, &tm);
+	seconds = tm.tm_sec;
 	bcd2tm(&tm);
 	rtc_tm_to_time(&tm, &now);
-	rtc_time_to_tm(now + 2, &tm);
+	rtc_time_to_tm(now + 1, &tm);
 
 	if (tm2bcd(&tm) < 0) {
 		dev_err(&rtc->rtc->dev, "power off failed\n");
 		rtc->type->lock(rtc);
-		return;
+		return -EINVAL;
 	}
 
 	rtc_wait_not_busy(rtc);
@@ -470,14 +467,54 @@
 	val = rtc_read(rtc, OMAP_RTC_INTERRUPTS_REG);
 	rtc_writel(rtc, OMAP_RTC_INTERRUPTS_REG,
 			val | OMAP_RTC_INTERRUPTS_IT_ALARM2);
+
+	/* Retry in case roll over happened before alarm was armed. */
+	if (rtc_read(rtc, OMAP_RTC_SECONDS_REG) != seconds) {
+		val = rtc_read(rtc, OMAP_RTC_STATUS_REG);
+		if (!(val & OMAP_RTC_STATUS_ALARM2))
+			goto again;
+	}
+
 	rtc->type->lock(rtc);
 
+	return 0;
+}
+
+/*
+ * omap_rtc_poweroff: RTC-controlled power off
+ *
+ * The RTC can be used to control an external PMIC via the pmic_power_en pin,
+ * which can be configured to transition to OFF on ALARM2 events.
+ *
+ * Notes:
+ * The one-second alarm offset is the shortest offset possible as the alarm
+ * registers must be set before the next timer update and the offset
+ * calculation is too heavy for everything to be done within a single access
+ * period (~15 us).
+ *
+ * Called with local interrupts disabled.
+ */
+static void omap_rtc_power_off(void)
+{
+	struct rtc_device *rtc = omap_rtc_power_off_rtc->rtc;
+	u32 val;
+
+	omap_rtc_power_off_program(rtc->dev.parent);
+
+	/* Set PMIC power enable and EXT_WAKEUP in case PB power on is used */
+	omap_rtc_power_off_rtc->type->unlock(omap_rtc_power_off_rtc);
+	val = rtc_readl(omap_rtc_power_off_rtc, OMAP_RTC_PMIC_REG);
+	val |= OMAP_RTC_PMIC_POWER_EN_EN | OMAP_RTC_PMIC_EXT_WKUP_POL(0) |
+			OMAP_RTC_PMIC_EXT_WKUP_EN(0);
+	rtc_writel(omap_rtc_power_off_rtc, OMAP_RTC_PMIC_REG, val);
+	omap_rtc_power_off_rtc->type->lock(omap_rtc_power_off_rtc);
+
 	/*
-	 * Wait for alarm to trigger (within two seconds) and external PMIC to
+	 * Wait for alarm to trigger (within one second) and external PMIC to
 	 * power off the system. Add a 500 ms margin for external latencies
 	 * (e.g. debounce circuits).
 	 */
-	mdelay(2500);
+	mdelay(1500);
 }
 
 static const struct rtc_class_ops omap_rtc_ops = {
@@ -486,6 +523,7 @@
 	.read_alarm	= omap_rtc_read_alarm,
 	.set_alarm	= omap_rtc_set_alarm,
 	.alarm_irq_enable = omap_rtc_alarm_irq_enable,
+	.power_off_program = omap_rtc_power_off_program,
 };
 
 static const struct omap_rtc_device_type omap_rtc_default_type = {
@@ -721,8 +759,7 @@
 	if (of_id) {
 		rtc->type = of_id->data;
 		rtc->is_pmic_controller = rtc->type->has_pmic_mode &&
-				of_property_read_bool(pdev->dev.of_node,
-						"system-power-controller");
+			of_device_is_system_power_controller(pdev->dev.of_node);
 	} else {
 		id_entry = platform_get_device_id(pdev);
 		rtc->type = (void *)id_entry->driver_data;
diff -Naur --no-dereference a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
--- a/drivers/scsi/scsi_lib.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/scsi/scsi_lib.c	2020-06-19 21:00:31.424227716 -0400
@@ -2228,6 +2228,21 @@
 	return blk_mq_map_queues(set);
 }
 
+static u64 scsi_calculate_bounce_limit(struct Scsi_Host *shost)
+{
+	struct device *dma_dev = shost->dma_dev;
+	u64 bounce_limit = 0xffffffff;
+
+	if (shost->unchecked_isa_dma)
+		return BLK_BOUNCE_ISA;
+
+	if (dma_dev && dma_dev->dma_mask)
+		bounce_limit = (u64)dma_max_pfn(dma_dev) << PAGE_SHIFT;
+
+
+	return bounce_limit;
+}
+
 void __scsi_init_queue(struct Scsi_Host *shost, struct request_queue *q)
 {
 	struct device *dev = shost->dma_dev;
@@ -2247,8 +2262,7 @@
 	}
 
 	blk_queue_max_hw_sectors(q, shost->max_sectors);
-	if (shost->unchecked_isa_dma)
-		blk_queue_bounce_limit(q, BLK_BOUNCE_ISA);
+	blk_queue_bounce_limit(q, scsi_calculate_bounce_limit(shost));
 	blk_queue_segment_boundary(q, shost->dma_boundary);
 	dma_set_seg_boundary(dev, shost->dma_boundary);
 
diff -Naur --no-dereference a/drivers/scsi/scsi_transport_fc.c b/drivers/scsi/scsi_transport_fc.c
--- a/drivers/scsi/scsi_transport_fc.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/scsi/scsi_transport_fc.c	2020-06-19 21:00:31.424227716 -0400
@@ -3780,7 +3780,8 @@
 	snprintf(bsg_name, sizeof(bsg_name),
 		 "fc_host%d", shost->host_no);
 
-	q = bsg_setup_queue(dev, bsg_name, fc_bsg_dispatch, i->f->dd_bsg_size);
+	q = bsg_setup_queue(dev, bsg_name, fc_bsg_dispatch, fc_bsg_job_timeout,
+				i->f->dd_bsg_size);
 	if (IS_ERR(q)) {
 		dev_err(dev,
 			"fc_host%d: bsg interface failed to initialize - setup queue\n",
@@ -3788,7 +3789,6 @@
 		return PTR_ERR(q);
 	}
 	__scsi_init_queue(shost, q);
-	blk_queue_rq_timed_out(q, fc_bsg_job_timeout);
 	blk_queue_rq_timeout(q, FC_DEFAULT_BSG_TIMEOUT);
 	fc_host->rqst_q = q;
 	return 0;
@@ -3826,14 +3826,13 @@
 		return -ENOTSUPP;
 
 	q = bsg_setup_queue(dev, dev_name(dev), fc_bsg_dispatch,
-			i->f->dd_bsg_size);
+				fc_bsg_job_timeout, i->f->dd_bsg_size);
 	if (IS_ERR(q)) {
 		dev_err(dev, "failed to setup bsg queue\n");
 		return PTR_ERR(q);
 	}
 	__scsi_init_queue(shost, q);
 	blk_queue_prep_rq(q, fc_bsg_rport_prep);
-	blk_queue_rq_timed_out(q, fc_bsg_job_timeout);
 	blk_queue_rq_timeout(q, BLK_DEFAULT_SG_TIMEOUT);
 	rport->rqst_q = q;
 	return 0;
@@ -3852,10 +3851,7 @@
 static void
 fc_bsg_remove(struct request_queue *q)
 {
-	if (q) {
-		bsg_unregister_queue(q);
-		blk_cleanup_queue(q);
-	}
+	bsg_remove_queue(q);
 }
 
 
diff -Naur --no-dereference a/drivers/scsi/scsi_transport_iscsi.c b/drivers/scsi/scsi_transport_iscsi.c
--- a/drivers/scsi/scsi_transport_iscsi.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/scsi/scsi_transport_iscsi.c	2020-06-19 21:00:31.424227716 -0400
@@ -1542,7 +1542,7 @@
 		return -ENOTSUPP;
 
 	snprintf(bsg_name, sizeof(bsg_name), "iscsi_host%d", shost->host_no);
-	q = bsg_setup_queue(dev, bsg_name, iscsi_bsg_host_dispatch, 0);
+	q = bsg_setup_queue(dev, bsg_name, iscsi_bsg_host_dispatch, NULL, 0);
 	if (IS_ERR(q)) {
 		shost_printk(KERN_ERR, shost, "bsg interface failed to "
 			     "initialize - no request queue\n");
@@ -1576,10 +1576,7 @@
 	struct Scsi_Host *shost = dev_to_shost(dev);
 	struct iscsi_cls_host *ihost = shost->shost_data;
 
-	if (ihost->bsg_q) {
-		bsg_unregister_queue(ihost->bsg_q);
-		blk_cleanup_queue(ihost->bsg_q);
-	}
+	bsg_remove_queue(ihost->bsg_q);
 	return 0;
 }
 
diff -Naur --no-dereference a/drivers/scsi/scsi_transport_sas.c b/drivers/scsi/scsi_transport_sas.c
--- a/drivers/scsi/scsi_transport_sas.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/scsi/scsi_transport_sas.c	2020-06-19 21:00:31.424227716 -0400
@@ -198,7 +198,7 @@
 
 	if (rphy) {
 		q = bsg_setup_queue(&rphy->dev, dev_name(&rphy->dev),
-				sas_smp_dispatch, 0);
+				sas_smp_dispatch, NULL, 0);
 		if (IS_ERR(q))
 			return PTR_ERR(q);
 		rphy->q = q;
@@ -207,7 +207,7 @@
 
 		snprintf(name, sizeof(name), "sas_host%d", shost->host_no);
 		q = bsg_setup_queue(&shost->shost_gendev, name,
-				sas_smp_dispatch, 0);
+				sas_smp_dispatch, NULL, 0);
 		if (IS_ERR(q))
 			return PTR_ERR(q);
 		to_sas_host_attrs(shost)->q = q;
@@ -246,11 +246,7 @@
 	struct Scsi_Host *shost = dev_to_shost(dev);
 	struct request_queue *q = to_sas_host_attrs(shost)->q;
 
-	if (q) {
-		bsg_unregister_queue(q);
-		blk_cleanup_queue(q);
-	}
-
+	bsg_remove_queue(q);
 	return 0;
 }
 
diff -Naur --no-dereference a/drivers/scsi/ufs/cdns-pltfrm.c b/drivers/scsi/ufs/cdns-pltfrm.c
--- a/drivers/scsi/ufs/cdns-pltfrm.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/scsi/ufs/cdns-pltfrm.c	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,229 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Platform UFS Host driver for Cadence controller
+ *
+ * Copyright (C) 2018 Cadence Design Systems, Inc.
+ *
+ * Authors:
+ *	Jan Kotas <jank@cadence.com>
+ *
+ */
+
+#include <linux/bitfield.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/of.h>
+#include <linux/time.h>
+
+#include "ufshcd-pltfrm.h"
+
+#define CDNS_UFS_REG_HCLKDIV	0xFC
+#define CDNS_UFS_REG_PHY_XCFGD1	0x113C
+
+/**
+ * Sets HCLKDIV register value based on the core_clk
+ * @hba: host controller instance
+ *
+ * Return zero for success and non-zero for failure
+ */
+static int cdns_ufs_set_hclkdiv(struct ufs_hba *hba)
+{
+	struct ufs_clk_info *clki;
+	struct list_head *head = &hba->clk_list_head;
+	unsigned long core_clk_rate = 0;
+	u32 core_clk_div = 0;
+
+	if (list_empty(head))
+		return 0;
+
+	list_for_each_entry(clki, head, list) {
+		if (IS_ERR_OR_NULL(clki->clk))
+			continue;
+		if (!strcmp(clki->name, "core_clk"))
+			core_clk_rate = clk_get_rate(clki->clk);
+	}
+
+	if (!core_clk_rate) {
+		dev_err(hba->dev, "%s: unable to find core_clk rate\n",
+			__func__);
+		return -EINVAL;
+	}
+
+	core_clk_div = core_clk_rate / USEC_PER_SEC;
+
+	ufshcd_writel(hba, core_clk_div, CDNS_UFS_REG_HCLKDIV);
+	/**
+	 * Make sure the register was updated,
+	 * UniPro layer will not work with an incorrect value.
+	 */
+	mb();
+
+	return 0;
+}
+
+/**
+ * Sets vendor specific clock divider register for Controller
+ * @hba: host controller instance
+ * @status: notify stage (pre, post change)
+ *
+ * Return zero for success and non-zero for failure
+ */
+static int cdns_ufs_hce_enable_notify(struct ufs_hba *hba,
+				      enum ufs_notify_change_status status)
+{
+	switch (status) {
+	case PRE_CHANGE:
+		return cdns_ufs_set_hclkdiv(hba);
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+/**
+ * cdns_ufs_init - performs additional ufs initialization
+ * @hba: host controller instance
+ *
+ * Returns status of initialization
+ */
+static int cdns_ufs_init(struct ufs_hba *hba)
+{
+	int status = 0;
+
+	/* Disable Auto-Hibernate Idle Timer by setting it to 0ms */
+	hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 0) |
+		    FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3);
+
+	if (hba->vops && hba->vops->phy_initialization)
+		status = hba->vops->phy_initialization(hba);
+
+	return status;
+}
+
+/**
+ * cdns_ufs_m31_16nm_phy_initialization - performs m31 phy initialization
+ * @hba: host controller instance
+ *
+ * Always returns 0
+ */
+static int cdns_ufs_m31_16nm_phy_initialization(struct ufs_hba *hba)
+{
+	u32 data;
+
+	/* Increase RX_Advanced_Min_ActivateTime_Capability */
+	data = ufshcd_readl(hba, CDNS_UFS_REG_PHY_XCFGD1);
+	data |= BIT(24);
+	ufshcd_writel(hba, data, CDNS_UFS_REG_PHY_XCFGD1);
+
+	return 0;
+}
+
+static int cdns_ufs_link_startup_notify(struct ufs_hba *hba,
+					enum ufs_notify_change_status status)
+{
+	hba->quirks |= UFSHCD_QUIRK_BROKEN_LCC;
+
+	switch (status) {
+	case PRE_CHANGE:
+		return ufshcd_dme_set(hba,
+				      UIC_ARG_MIB(PA_LOCAL_TX_LCC_ENABLE),
+				      0);
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static const struct ufs_hba_variant_ops cdns_ufs_pltfm_hba_vops = {
+	.name = "cdns-ufs-pltfm",
+	.hce_enable_notify = cdns_ufs_hce_enable_notify,
+};
+
+static const struct ufs_hba_variant_ops cdns_ufs_m31_16nm_pltfm_hba_vops = {
+	.name = "cdns-ufs-pltfm",
+	.init = cdns_ufs_init,
+	.hce_enable_notify = cdns_ufs_hce_enable_notify,
+	.phy_initialization = cdns_ufs_m31_16nm_phy_initialization,
+	.link_startup_notify = cdns_ufs_link_startup_notify,
+};
+
+static const struct of_device_id cdns_ufs_of_match[] = {
+	{
+		.compatible = "cdns,ufshc",
+		.data =  &cdns_ufs_pltfm_hba_vops,
+	},
+	{
+		.compatible = "cdns,ufshc-m31-16nm",
+		.data =  &cdns_ufs_m31_16nm_pltfm_hba_vops,
+	},
+	{ },
+};
+
+MODULE_DEVICE_TABLE(of, cdns_ufs_of_match);
+
+/**
+ * cdns_ufs_pltfrm_probe - probe routine of the driver
+ * @pdev: pointer to platform device handle
+ *
+ * Return zero for success and non-zero for failure
+ */
+static int cdns_ufs_pltfrm_probe(struct platform_device *pdev)
+{
+	int err;
+	const struct of_device_id *of_id;
+	struct ufs_hba_variant_ops *vops;
+	struct device *dev = &pdev->dev;
+
+	of_id = of_match_node(cdns_ufs_of_match, dev->of_node);
+	vops = (struct ufs_hba_variant_ops *)of_id->data;
+
+	/* Perform generic probe */
+	err = ufshcd_pltfrm_init(pdev, vops);
+	if (err)
+		dev_err(dev, "ufshcd_pltfrm_init() failed %d\n", err);
+
+	return err;
+}
+
+/**
+ * cdns_ufs_pltfrm_remove - removes the ufs driver
+ * @pdev: pointer to platform device handle
+ *
+ * Always returns 0
+ */
+static int cdns_ufs_pltfrm_remove(struct platform_device *pdev)
+{
+	struct ufs_hba *hba =  platform_get_drvdata(pdev);
+
+	ufshcd_remove(hba);
+	return 0;
+}
+
+static const struct dev_pm_ops cdns_ufs_dev_pm_ops = {
+	.suspend         = ufshcd_pltfrm_suspend,
+	.resume          = ufshcd_pltfrm_resume,
+	.runtime_suspend = ufshcd_pltfrm_runtime_suspend,
+	.runtime_resume  = ufshcd_pltfrm_runtime_resume,
+	.runtime_idle    = ufshcd_pltfrm_runtime_idle,
+};
+
+static struct platform_driver cdns_ufs_pltfrm_driver = {
+	.probe	= cdns_ufs_pltfrm_probe,
+	.remove	= cdns_ufs_pltfrm_remove,
+	.driver	= {
+		.name   = "cdns-ufshcd",
+		.owner  = THIS_MODULE,
+		.pm     = &cdns_ufs_dev_pm_ops,
+		.of_match_table = cdns_ufs_of_match,
+	},
+};
+
+module_platform_driver(cdns_ufs_pltfrm_driver);
+
+MODULE_AUTHOR("Jan Kotas <jank@cadence.com>");
+MODULE_DESCRIPTION("Cadence UFS host controller platform driver");
+MODULE_LICENSE("GPL v2");
+MODULE_VERSION(UFSHCD_DRIVER_VERSION);
diff -Naur --no-dereference a/drivers/scsi/ufs/Kconfig b/drivers/scsi/ufs/Kconfig
--- a/drivers/scsi/ufs/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/scsi/ufs/Kconfig	2020-06-19 21:00:31.424227716 -0400
@@ -80,6 +80,14 @@
 
 	  If unsure, say N.
 
+config SCSI_UFS_CDNS_PLATFORM
+	tristate "Cadence UFS Controller platform driver"
+	depends on SCSI_UFSHCD_PLATFORM
+	help
+	This selects the Cadence-specific additions to UFSHCD platform driver.
+
+	  If unsure, say N.
+
 config SCSI_UFS_DWC_TC_PLATFORM
 	tristate "DesignWare platform support using a G210 Test Chip"
 	depends on SCSI_UFSHCD_PLATFORM
@@ -109,3 +117,26 @@
 
 	  Select this if you have UFS controller on Hisilicon chipset.
 	  If unsure, say N.
+
+config SCSI_UFS_BSG
+	bool "Universal Flash Storage BSG device node"
+	depends on SCSI_UFSHCD
+	select BLK_DEV_BSGLIB
+	help
+	  Universal Flash Storage (UFS) is SCSI transport specification for
+	  accessing flash storage on digital cameras, mobile phones and
+	  consumer electronic devices.
+	  A UFS controller communicates with a UFS device by exchanging
+	  UFS Protocol Information Units (UPIUs).
+	  UPIUs can not only be used as a transport layer for the SCSI protocol
+	  but are also used by the UFS native command set.
+	  This transport driver supports exchanging UFS protocol information units
+	  with a UFS device. See also the ufshcd driver, which is a SCSI driver
+	  that supports UFS devices.
+
+	  Select this if you need a bsg device node for your UFS controller.
+	  If unsure, say N.
+
+config SCSI_UFS_TI_J721E
+	tristate "TI glue layer for Cadence UFS Controller"
+	depends on OF && (ARCH_K3 || COMPILE_TEST)
diff -Naur --no-dereference a/drivers/scsi/ufs/Makefile b/drivers/scsi/ufs/Makefile
--- a/drivers/scsi/ufs/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/scsi/ufs/Makefile	2020-06-19 21:00:31.424227716 -0400
@@ -2,9 +2,12 @@
 # UFSHCD makefile
 obj-$(CONFIG_SCSI_UFS_DWC_TC_PCI) += tc-dwc-g210-pci.o ufshcd-dwc.o tc-dwc-g210.o
 obj-$(CONFIG_SCSI_UFS_DWC_TC_PLATFORM) += tc-dwc-g210-pltfrm.o ufshcd-dwc.o tc-dwc-g210.o
+obj-$(CONFIG_SCSI_UFS_CDNS_PLATFORM) += cdns-pltfrm.o
 obj-$(CONFIG_SCSI_UFS_QCOM) += ufs-qcom.o
 obj-$(CONFIG_SCSI_UFSHCD) += ufshcd-core.o
-ufshcd-core-objs := ufshcd.o ufs-sysfs.o
+ufshcd-core-y				+= ufshcd.o ufs-sysfs.o
+ufshcd-core-$(CONFIG_SCSI_UFS_BSG)	+= ufs_bsg.o
 obj-$(CONFIG_SCSI_UFSHCD_PCI) += ufshcd-pci.o
 obj-$(CONFIG_SCSI_UFSHCD_PLATFORM) += ufshcd-pltfrm.o
 obj-$(CONFIG_SCSI_UFS_HISI) += ufs-hisi.o
+obj-$(CONFIG_SCSI_UFS_TI_J721E) += ti-j721e-ufs.o
diff -Naur --no-dereference a/drivers/scsi/ufs/ti-j721e-ufs.c b/drivers/scsi/ufs/ti-j721e-ufs.c
--- a/drivers/scsi/ufs/ti-j721e-ufs.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/scsi/ufs/ti-j721e-ufs.c	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+//
+
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+
+#define UFS_SS_CTRL		0x4
+#define UFS_SS_RST_N_PCS	BIT(0)
+#define UFS_SS_CLK_26MHZ	BIT(4)
+
+static int ti_j721e_ufs_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	unsigned long clk_rate;
+	void __iomem *regbase;
+	struct resource *res;
+	struct clk *clk;
+	u32 reg = 0;
+	int ret;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	regbase = devm_ioremap_resource(dev, res);
+	if (IS_ERR(regbase))
+		return PTR_ERR(regbase);
+
+	/* Select M-PHY refclk frequency */
+	clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(clk)) {
+		dev_err(dev, "Cannot claim M-PHY clock.\n");
+		return PTR_ERR(clk);
+	}
+	clk_rate = clk_get_rate(clk);
+	if (clk_rate == 26000000)
+		reg |= UFS_SS_CLK_26MHZ;
+	devm_clk_put(&pdev->dev, clk);
+
+	/*  Take UFS slave device out of reset */
+	reg |= UFS_SS_RST_N_PCS;
+
+	writel(reg, regbase + UFS_SS_CTRL);
+
+	ret = of_platform_populate(pdev->dev.of_node, NULL, NULL,
+				   &pdev->dev);
+	if (ret)
+		dev_err(dev, "failed to populate child nodes %d\n", ret);
+
+	return ret;
+}
+
+static int ti_j721e_ufs_remove(struct platform_device *pdev)
+{
+	of_platform_depopulate(&pdev->dev);
+
+	return 0;
+}
+
+static const struct of_device_id ti_j721e_ufs_of_match[] = {
+	{
+		.compatible = "ti,j721e-ufs",
+	},
+	{ },
+};
+
+static struct platform_driver ti_j721e_ufs_driver = {
+	.probe	= ti_j721e_ufs_probe,
+	.remove	= ti_j721e_ufs_remove,
+	.driver	= {
+		.name   = "ti-j721e-ufs",
+		.of_match_table = ti_j721e_ufs_of_match,
+	},
+};
+module_platform_driver(ti_j721e_ufs_driver);
+
+MODULE_AUTHOR("Vignesh Raghavendra <vigneshr@ti.com>");
+MODULE_DESCRIPTION("TI UFS host controller glue driver");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/scsi/ufs/ufs_bsg.c b/drivers/scsi/ufs/ufs_bsg.c
--- a/drivers/scsi/ufs/ufs_bsg.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/scsi/ufs/ufs_bsg.c	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,221 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * bsg endpoint that supports UPIUs
+ *
+ * Copyright (C) 2018 Western Digital Corporation
+ */
+#include "ufs_bsg.h"
+
+static int ufs_bsg_get_query_desc_size(struct ufs_hba *hba, int *desc_len,
+				       struct utp_upiu_query *qr)
+{
+	int desc_size = be16_to_cpu(qr->length);
+	int desc_id = qr->idn;
+	int ret;
+
+	if (desc_size <= 0)
+		return -EINVAL;
+
+	ret = ufshcd_map_desc_id_to_length(hba, desc_id, desc_len);
+	if (ret || !*desc_len)
+		return -EINVAL;
+
+	*desc_len = min_t(int, *desc_len, desc_size);
+
+	return 0;
+}
+
+static int ufs_bsg_verify_query_size(struct ufs_hba *hba,
+				     unsigned int request_len,
+				     unsigned int reply_len)
+{
+	int min_req_len = sizeof(struct ufs_bsg_request);
+	int min_rsp_len = sizeof(struct ufs_bsg_reply);
+
+	if (min_req_len > request_len || min_rsp_len > reply_len) {
+		dev_err(hba->dev, "not enough space assigned\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int ufs_bsg_alloc_desc_buffer(struct ufs_hba *hba, struct bsg_job *job,
+				     uint8_t **desc_buff, int *desc_len,
+				     enum query_opcode desc_op)
+{
+	struct ufs_bsg_request *bsg_request = job->request;
+	struct utp_upiu_query *qr;
+	u8 *descp;
+
+	if (desc_op != UPIU_QUERY_OPCODE_WRITE_DESC &&
+	    desc_op != UPIU_QUERY_OPCODE_READ_DESC)
+		goto out;
+
+	qr = &bsg_request->upiu_req.qr;
+	if (ufs_bsg_get_query_desc_size(hba, desc_len, qr)) {
+		dev_err(hba->dev, "Illegal desc size\n");
+		return -EINVAL;
+	}
+
+	if (*desc_len > job->request_payload.payload_len) {
+		dev_err(hba->dev, "Illegal desc size\n");
+		return -EINVAL;
+	}
+
+	descp = kzalloc(*desc_len, GFP_KERNEL);
+	if (!descp)
+		return -ENOMEM;
+
+	if (desc_op == UPIU_QUERY_OPCODE_WRITE_DESC)
+		sg_copy_to_buffer(job->request_payload.sg_list,
+				  job->request_payload.sg_cnt, descp,
+				  *desc_len);
+
+	*desc_buff = descp;
+
+out:
+	return 0;
+}
+
+static int ufs_bsg_request(struct bsg_job *job)
+{
+	struct ufs_bsg_request *bsg_request = job->request;
+	struct ufs_bsg_reply *bsg_reply = job->reply;
+	struct ufs_hba *hba = shost_priv(dev_to_shost(job->dev->parent));
+	unsigned int req_len = job->request_len;
+	unsigned int reply_len = job->reply_len;
+	struct uic_command uc = {};
+	int msgcode;
+	uint8_t *desc_buff = NULL;
+	int desc_len = 0;
+	enum query_opcode desc_op = UPIU_QUERY_OPCODE_NOP;
+	int ret;
+
+	ret = ufs_bsg_verify_query_size(hba, req_len, reply_len);
+	if (ret)
+		goto out;
+
+	bsg_reply->reply_payload_rcv_len = 0;
+
+	msgcode = bsg_request->msgcode;
+	switch (msgcode) {
+	case UPIU_TRANSACTION_QUERY_REQ:
+		desc_op = bsg_request->upiu_req.qr.opcode;
+		ret = ufs_bsg_alloc_desc_buffer(hba, job, &desc_buff,
+						&desc_len, desc_op);
+		if (ret)
+			goto out;
+
+		/* fall through */
+	case UPIU_TRANSACTION_NOP_OUT:
+	case UPIU_TRANSACTION_TASK_REQ:
+		ret = ufshcd_exec_raw_upiu_cmd(hba, &bsg_request->upiu_req,
+					       &bsg_reply->upiu_rsp, msgcode,
+					       desc_buff, &desc_len, desc_op);
+		if (ret)
+			dev_err(hba->dev,
+				"exe raw upiu: error code %d\n", ret);
+
+		break;
+	case UPIU_TRANSACTION_UIC_CMD:
+		memcpy(&uc, &bsg_request->upiu_req.uc, UIC_CMD_SIZE);
+		ret = ufshcd_send_uic_cmd(hba, &uc);
+		if (ret)
+			dev_err(hba->dev,
+				"send uic cmd: error code %d\n", ret);
+
+		memcpy(&bsg_reply->upiu_rsp.uc, &uc, UIC_CMD_SIZE);
+
+		break;
+	default:
+		ret = -ENOTSUPP;
+		dev_err(hba->dev, "unsupported msgcode 0x%x\n", msgcode);
+
+		break;
+	}
+
+	if (!desc_buff)
+		goto out;
+
+	if (desc_op == UPIU_QUERY_OPCODE_READ_DESC && desc_len)
+		bsg_reply->reply_payload_rcv_len =
+			sg_copy_from_buffer(job->request_payload.sg_list,
+					    job->request_payload.sg_cnt,
+					    desc_buff, desc_len);
+
+	kfree(desc_buff);
+
+out:
+	bsg_reply->result = ret;
+	job->reply_len = sizeof(struct ufs_bsg_reply);
+	/* complete the job here only if no error */
+	if (ret == 0)
+		bsg_job_done(job, ret, bsg_reply->reply_payload_rcv_len);
+
+	return ret;
+}
+
+/**
+ * ufs_bsg_remove - detach and remove the added ufs-bsg node
+ *
+ * Should be called when unloading the driver.
+ */
+void ufs_bsg_remove(struct ufs_hba *hba)
+{
+	struct device *bsg_dev = &hba->bsg_dev;
+
+	if (!hba->bsg_queue)
+		return;
+
+	bsg_remove_queue(hba->bsg_queue);
+
+	device_del(bsg_dev);
+	put_device(bsg_dev);
+}
+
+static inline void ufs_bsg_node_release(struct device *dev)
+{
+	put_device(dev->parent);
+}
+
+/**
+ * ufs_bsg_probe - Add ufs bsg device node
+ * @hba: per adapter object
+ *
+ * Called during initial loading of the driver, and before scsi_scan_host.
+ */
+int ufs_bsg_probe(struct ufs_hba *hba)
+{
+	struct device *bsg_dev = &hba->bsg_dev;
+	struct Scsi_Host *shost = hba->host;
+	struct device *parent = &shost->shost_gendev;
+	struct request_queue *q;
+	int ret;
+
+	device_initialize(bsg_dev);
+
+	bsg_dev->parent = get_device(parent);
+	bsg_dev->release = ufs_bsg_node_release;
+
+	dev_set_name(bsg_dev, "ufs-bsg");
+
+	ret = device_add(bsg_dev);
+	if (ret)
+		goto out;
+
+	q = bsg_setup_queue(bsg_dev, dev_name(bsg_dev), ufs_bsg_request, NULL, 0);
+	if (IS_ERR(q)) {
+		ret = PTR_ERR(q);
+		goto out;
+	}
+
+	hba->bsg_queue = q;
+
+	return 0;
+
+out:
+	dev_err(bsg_dev, "fail to initialize a bsg dev %d\n", shost->host_no);
+	put_device(bsg_dev);
+	return ret;
+}
diff -Naur --no-dereference a/drivers/scsi/ufs/ufs_bsg.h b/drivers/scsi/ufs/ufs_bsg.h
--- a/drivers/scsi/ufs/ufs_bsg.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/scsi/ufs/ufs_bsg.h	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2018 Western Digital Corporation
+ */
+#ifndef UFS_BSG_H
+#define UFS_BSG_H
+
+#include <linux/bsg-lib.h>
+#include <scsi/scsi.h>
+#include <scsi/scsi_host.h>
+
+#include "ufshcd.h"
+#include "ufs.h"
+
+#ifdef CONFIG_SCSI_UFS_BSG
+void ufs_bsg_remove(struct ufs_hba *hba);
+int ufs_bsg_probe(struct ufs_hba *hba);
+#else
+static inline void ufs_bsg_remove(struct ufs_hba *hba) {}
+static inline int ufs_bsg_probe(struct ufs_hba *hba) {return 0; }
+#endif
+
+#endif /* UFS_BSG_H */
diff -Naur --no-dereference a/drivers/scsi/ufs/ufs.h b/drivers/scsi/ufs/ufs.h
--- a/drivers/scsi/ufs/ufs.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/scsi/ufs/ufs.h	2020-06-19 21:00:31.424227716 -0400
@@ -38,9 +38,9 @@
 
 #include <linux/mutex.h>
 #include <linux/types.h>
+#include <uapi/scsi/scsi_bsg_ufs.h>
 
-#define MAX_CDB_SIZE	16
-#define GENERAL_UPIU_REQUEST_SIZE 32
+#define GENERAL_UPIU_REQUEST_SIZE (sizeof(struct utp_upiu_req))
 #define QUERY_DESC_MAX_SIZE       255
 #define QUERY_DESC_MIN_SIZE       2
 #define QUERY_DESC_HDR_SIZE       2
@@ -378,6 +378,20 @@
 	UPIU_QUERY_OPCODE_TOGGLE_FLAG	= 0x8,
 };
 
+/* bRefClkFreq attribute values */
+enum ufs_ref_clk_freq {
+	REF_CLK_FREQ_19_2_MHZ	= 0,
+	REF_CLK_FREQ_26_MHZ	= 1,
+	REF_CLK_FREQ_38_4_MHZ	= 2,
+	REF_CLK_FREQ_52_MHZ	= 3,
+	REF_CLK_FREQ_INVAL	= -1,
+};
+
+struct ufs_ref_clk {
+	unsigned long freq_hz;
+	enum ufs_ref_clk_freq val;
+};
+
 /* Query response result code */
 enum {
 	QUERY_RESULT_SUCCESS                    = 0x00,
@@ -414,6 +428,7 @@
 	MASK_RSP_UPIU_DATA_SEG_LEN	= 0xFFFF,
 	MASK_RSP_EXCEPTION_EVENT        = 0x10000,
 	MASK_TM_SERVICE_RESP		= 0xFF,
+	MASK_TM_FUNC			= 0xFF,
 };
 
 /* Task management service response */
@@ -433,65 +448,6 @@
 };
 
 /**
- * struct utp_upiu_header - UPIU header structure
- * @dword_0: UPIU header DW-0
- * @dword_1: UPIU header DW-1
- * @dword_2: UPIU header DW-2
- */
-struct utp_upiu_header {
-	__be32 dword_0;
-	__be32 dword_1;
-	__be32 dword_2;
-};
-
-/**
- * struct utp_upiu_cmd - Command UPIU structure
- * @data_transfer_len: Data Transfer Length DW-3
- * @cdb: Command Descriptor Block CDB DW-4 to DW-7
- */
-struct utp_upiu_cmd {
-	__be32 exp_data_transfer_len;
-	u8 cdb[MAX_CDB_SIZE];
-};
-
-/**
- * struct utp_upiu_query - upiu request buffer structure for
- * query request.
- * @opcode: command to perform B-0
- * @idn: a value that indicates the particular type of data B-1
- * @index: Index to further identify data B-2
- * @selector: Index to further identify data B-3
- * @reserved_osf: spec reserved field B-4,5
- * @length: number of descriptor bytes to read/write B-6,7
- * @value: Attribute value to be written DW-5
- * @reserved: spec reserved DW-6,7
- */
-struct utp_upiu_query {
-	u8 opcode;
-	u8 idn;
-	u8 index;
-	u8 selector;
-	__be16 reserved_osf;
-	__be16 length;
-	__be32 value;
-	__be32 reserved[2];
-};
-
-/**
- * struct utp_upiu_req - general upiu request structure
- * @header:UPIU header structure DW-0 to DW-2
- * @sc: fields structure for scsi command DW-3 to DW-7
- * @qr: fields structure for query request DW-3 to DW-7
- */
-struct utp_upiu_req {
-	struct utp_upiu_header header;
-	union {
-		struct utp_upiu_cmd sc;
-		struct utp_upiu_query qr;
-	};
-};
-
-/**
  * struct utp_cmd_rsp - Response UPIU structure
  * @residual_transfer_count: Residual transfer count DW-3
  * @reserved: Reserved double words DW-4 to DW-7
@@ -520,36 +476,6 @@
 };
 
 /**
- * struct utp_upiu_task_req - Task request UPIU structure
- * @header - UPIU header structure DW0 to DW-2
- * @input_param1: Input parameter 1 DW-3
- * @input_param2: Input parameter 2 DW-4
- * @input_param3: Input parameter 3 DW-5
- * @reserved: Reserved double words DW-6 to DW-7
- */
-struct utp_upiu_task_req {
-	struct utp_upiu_header header;
-	__be32 input_param1;
-	__be32 input_param2;
-	__be32 input_param3;
-	__be32 reserved[2];
-};
-
-/**
- * struct utp_upiu_task_rsp - Task Management Response UPIU structure
- * @header: UPIU header structure DW0-DW-2
- * @output_param1: Ouput parameter 1 DW3
- * @output_param2: Output parameter 2 DW4
- * @reserved: Reserved double words DW-5 to DW-7
- */
-struct utp_upiu_task_rsp {
-	struct utp_upiu_header header;
-	__be32 output_param1;
-	__be32 output_param2;
-	__be32 reserved[3];
-};
-
-/**
  * struct ufs_query_req - parameters for building a query request
  * @query_func: UPIU header query function
  * @upiu_req: the query request data
diff -Naur --no-dereference a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c
--- a/drivers/scsi/ufs/ufshcd.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/scsi/ufs/ufshcd.c	2020-06-19 21:00:31.424227716 -0400
@@ -46,6 +46,7 @@
 #include "ufs_quirks.h"
 #include "unipro.h"
 #include "ufs-sysfs.h"
+#include "ufs_bsg.h"
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/ufs.h>
@@ -334,14 +335,11 @@
 static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag,
 		const char *str)
 {
-	struct utp_task_req_desc *descp;
-	struct utp_upiu_task_req *task_req;
 	int off = (int)tag - hba->nutrs;
+	struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[off];
 
-	descp = &hba->utmrdl_base_addr[off];
-	task_req = (struct utp_upiu_task_req *)descp->task_req_upiu;
-	trace_ufshcd_upiu(dev_name(hba->dev), str, &task_req->header,
-			&task_req->input_param1);
+	trace_ufshcd_upiu(dev_name(hba->dev), str, &descp->req_header,
+			&descp->input_param1);
 }
 
 static void ufshcd_add_command_trace(struct ufs_hba *hba,
@@ -483,22 +481,13 @@
 
 static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap)
 {
-	struct utp_task_req_desc *tmrdp;
 	int tag;
 
 	for_each_set_bit(tag, &bitmap, hba->nutmrs) {
-		tmrdp = &hba->utmrdl_base_addr[tag];
+		struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag];
+
 		dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag);
-		ufshcd_hex_dump("TM TRD: ", &tmrdp->header,
-				sizeof(struct request_desc_header));
-		dev_err(hba->dev, "TM[%d] - Task Management Request UPIU\n",
-				tag);
-		ufshcd_hex_dump("TM REQ: ", tmrdp->task_req_upiu,
-				sizeof(struct utp_upiu_req));
-		dev_err(hba->dev, "TM[%d] - Task Management Response UPIU\n",
-				tag);
-		ufshcd_hex_dump("TM RSP: ", tmrdp->task_rsp_upiu,
-				sizeof(struct utp_task_req_desc));
+		ufshcd_hex_dump("", tmrdp, sizeof(*tmrdp));
 	}
 }
 
@@ -654,19 +643,6 @@
 }
 
 /**
- * ufshcd_get_tmr_ocs - Get the UTMRD Overall Command Status
- * @task_req_descp: pointer to utp_task_req_desc structure
- *
- * This function is used to get the OCS field from UTMRD
- * Returns the OCS field in the UTMRD
- */
-static inline int
-ufshcd_get_tmr_ocs(struct utp_task_req_desc *task_req_descp)
-{
-	return le32_to_cpu(task_req_descp->header.dword_2) & MASK_OCS;
-}
-
-/**
  * ufshcd_get_tm_free_slot - get a free slot for task management request
  * @hba: per adapter instance
  * @free_slot: pointer to variable with available slot value
@@ -2093,8 +2069,7 @@
  *
  * Returns 0 only if success.
  */
-static int
-ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
+int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
 {
 	int ret;
 	unsigned long flags;
@@ -2276,8 +2251,8 @@
 	ucd_req_ptr->sc.exp_data_transfer_len =
 		cpu_to_be32(lrbp->cmd->sdb.length);
 
-	cdb_len = min_t(unsigned short, lrbp->cmd->cmd_len, MAX_CDB_SIZE);
-	memset(ucd_req_ptr->sc.cdb, 0, MAX_CDB_SIZE);
+	cdb_len = min_t(unsigned short, lrbp->cmd->cmd_len, UFS_CDB_SIZE);
+	memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE);
 	memcpy(ucd_req_ptr->sc.cdb, lrbp->cmd->cmnd, cdb_len);
 
 	memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
@@ -2296,7 +2271,6 @@
 	struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
 	struct ufs_query *query = &hba->dev_cmd.query;
 	u16 len = be16_to_cpu(query->request.upiu_req.length);
-	u8 *descp = (u8 *)lrbp->ucd_req_ptr + GENERAL_UPIU_REQUEST_SIZE;
 
 	/* Query request header */
 	ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
@@ -2318,7 +2292,7 @@
 
 	/* Copy the Descriptor */
 	if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
-		memcpy(descp, query->descriptor, len);
+		memcpy(ucd_req_ptr + 1, query->descriptor, len);
 
 	memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
 }
@@ -3944,7 +3918,7 @@
 {
 	unsigned long flags;
 
-	if (!(hba->capabilities & MASK_AUTO_HIBERN8_SUPPORT) || !hba->ahit)
+	if (!ufshcd_is_auto_hibern8_supported(hba) || !hba->ahit)
 		return;
 
 	spin_lock_irqsave(hba->host->host_lock, flags);
@@ -4648,46 +4622,6 @@
 }
 
 /**
- * ufshcd_task_req_compl - handle task management request completion
- * @hba: per adapter instance
- * @index: index of the completed request
- * @resp: task management service response
- *
- * Returns non-zero value on error, zero on success
- */
-static int ufshcd_task_req_compl(struct ufs_hba *hba, u32 index, u8 *resp)
-{
-	struct utp_task_req_desc *task_req_descp;
-	struct utp_upiu_task_rsp *task_rsp_upiup;
-	unsigned long flags;
-	int ocs_value;
-	int task_result;
-
-	spin_lock_irqsave(hba->host->host_lock, flags);
-
-	/* Clear completed tasks from outstanding_tasks */
-	__clear_bit(index, &hba->outstanding_tasks);
-
-	task_req_descp = hba->utmrdl_base_addr;
-	ocs_value = ufshcd_get_tmr_ocs(&task_req_descp[index]);
-
-	if (ocs_value == OCS_SUCCESS) {
-		task_rsp_upiup = (struct utp_upiu_task_rsp *)
-				task_req_descp[index].task_rsp_upiu;
-		task_result = be32_to_cpu(task_rsp_upiup->output_param1);
-		task_result = task_result & MASK_TM_SERVICE_RESP;
-		if (resp)
-			*resp = (u8)task_result;
-	} else {
-		dev_err(hba->dev, "%s: failed, ocs = 0x%x\n",
-				__func__, ocs_value);
-	}
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
-
-	return ocs_value;
-}
-
-/**
  * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
  * @lrbp: pointer to local reference block of completed command
  * @scsi_status: SCSI command status
@@ -5330,6 +5264,7 @@
 			goto skip_err_handling;
 	}
 	if ((hba->saved_err & INT_FATAL_ERRORS) ||
+	    (hba->saved_err & UFSHCD_UIC_HIBERN8_MASK) ||
 	    ((hba->saved_err & UIC_ERROR) &&
 	    (hba->saved_uic_err & (UFSHCD_UIC_DL_PA_INIT_ERROR |
 				   UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
@@ -5489,6 +5424,23 @@
 			__func__, hba->uic_error);
 }
 
+static bool ufshcd_is_auto_hibern8_error(struct ufs_hba *hba,
+					 u32 intr_mask)
+{
+	if (!ufshcd_is_auto_hibern8_supported(hba))
+		return false;
+
+	if (!(intr_mask & UFSHCD_UIC_HIBERN8_MASK))
+		return false;
+
+	if (hba->active_uic_cmd &&
+	    (hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_ENTER ||
+	    hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_EXIT))
+		return false;
+
+	return true;
+}
+
 /**
  * ufshcd_check_errors - Check for errors that need s/w attention
  * @hba: per-adapter instance
@@ -5507,6 +5459,15 @@
 			queue_eh_work = true;
 	}
 
+	if (hba->errors & UFSHCD_UIC_HIBERN8_MASK) {
+		dev_err(hba->dev,
+			"%s: Auto Hibern8 %s failed - status: 0x%08x, upmcrs: 0x%08x\n",
+			__func__, (hba->errors & UIC_HIBERNATE_ENTER) ?
+			"Enter" : "Exit",
+			hba->errors, ufshcd_get_upmcrs(hba));
+		queue_eh_work = true;
+	}
+
 	if (queue_eh_work) {
 		/*
 		 * update the transfer error masks to sticky bits, let's do this
@@ -5569,6 +5530,10 @@
 static void ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
 {
 	hba->errors = UFSHCD_ERROR_MASK & intr_status;
+
+	if (ufshcd_is_auto_hibern8_error(hba, intr_status))
+		hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status);
+
 	if (hba->errors)
 		ufshcd_check_errors(hba);
 
@@ -5644,28 +5609,12 @@
 	return err;
 }
 
-/**
- * ufshcd_issue_tm_cmd - issues task management commands to controller
- * @hba: per adapter instance
- * @lun_id: LUN ID to which TM command is sent
- * @task_id: task ID to which the TM command is applicable
- * @tm_function: task management function opcode
- * @tm_response: task management service response return value
- *
- * Returns non-zero value on error, zero on success.
- */
-static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id,
-		u8 tm_function, u8 *tm_response)
+static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba,
+		struct utp_task_req_desc *treq, u8 tm_function)
 {
-	struct utp_task_req_desc *task_req_descp;
-	struct utp_upiu_task_req *task_req_upiup;
-	struct Scsi_Host *host;
+	struct Scsi_Host *host = hba->host;
 	unsigned long flags;
-	int free_slot;
-	int err;
-	int task_tag;
-
-	host = hba->host;
+	int free_slot, task_tag, err;
 
 	/*
 	 * Get free slot, sleep if slots are unavailable.
@@ -5676,30 +5625,11 @@
 	ufshcd_hold(hba, false);
 
 	spin_lock_irqsave(host->host_lock, flags);
-	task_req_descp = hba->utmrdl_base_addr;
-	task_req_descp += free_slot;
-
-	/* Configure task request descriptor */
-	task_req_descp->header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
-	task_req_descp->header.dword_2 =
-			cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
-
-	/* Configure task request UPIU */
-	task_req_upiup =
-		(struct utp_upiu_task_req *) task_req_descp->task_req_upiu;
 	task_tag = hba->nutrs + free_slot;
-	task_req_upiup->header.dword_0 =
-		UPIU_HEADER_DWORD(UPIU_TRANSACTION_TASK_REQ, 0,
-					      lun_id, task_tag);
-	task_req_upiup->header.dword_1 =
-		UPIU_HEADER_DWORD(0, tm_function, 0, 0);
-	/*
-	 * The host shall provide the same value for LUN field in the basic
-	 * header and for Input Parameter.
-	 */
-	task_req_upiup->input_param1 = cpu_to_be32(lun_id);
-	task_req_upiup->input_param2 = cpu_to_be32(task_id);
 
+	treq->req_header.dword_0 |= cpu_to_be32(task_tag);
+
+	memcpy(hba->utmrdl_base_addr + free_slot, treq, sizeof(*treq));
 	ufshcd_vops_setup_task_mgmt(hba, free_slot, tm_function);
 
 	/* send command to the controller */
@@ -5729,8 +5659,15 @@
 					__func__, free_slot);
 		err = -ETIMEDOUT;
 	} else {
-		err = ufshcd_task_req_compl(hba, free_slot, tm_response);
+		err = 0;
+		memcpy(treq, hba->utmrdl_base_addr + free_slot, sizeof(*treq));
+
 		ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_complete");
+
+		spin_lock_irqsave(hba->host->host_lock, flags);
+		__clear_bit(free_slot, &hba->outstanding_tasks);
+		spin_unlock_irqrestore(hba->host->host_lock, flags);
+
 	}
 
 	clear_bit(free_slot, &hba->tm_condition);
@@ -5742,6 +5679,236 @@
 }
 
 /**
+ * ufshcd_issue_tm_cmd - issues task management commands to controller
+ * @hba: per adapter instance
+ * @lun_id: LUN ID to which TM command is sent
+ * @task_id: task ID to which the TM command is applicable
+ * @tm_function: task management function opcode
+ * @tm_response: task management service response return value
+ *
+ * Returns non-zero value on error, zero on success.
+ */
+static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id,
+		u8 tm_function, u8 *tm_response)
+{
+	struct utp_task_req_desc treq = { { 0 }, };
+	int ocs_value, err;
+
+	/* Configure task request descriptor */
+	treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
+	treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
+
+	/* Configure task request UPIU */
+	treq.req_header.dword_0 = cpu_to_be32(lun_id << 8) |
+				  cpu_to_be32(UPIU_TRANSACTION_TASK_REQ << 24);
+	treq.req_header.dword_1 = cpu_to_be32(tm_function << 16);
+
+	/*
+	 * The host shall provide the same value for LUN field in the basic
+	 * header and for Input Parameter.
+	 */
+	treq.input_param1 = cpu_to_be32(lun_id);
+	treq.input_param2 = cpu_to_be32(task_id);
+
+	err = __ufshcd_issue_tm_cmd(hba, &treq, tm_function);
+	if (err == -ETIMEDOUT)
+		return err;
+
+	ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
+	if (ocs_value != OCS_SUCCESS)
+		dev_err(hba->dev, "%s: failed, ocs = 0x%x\n",
+				__func__, ocs_value);
+	else if (tm_response)
+		*tm_response = be32_to_cpu(treq.output_param1) &
+				MASK_TM_SERVICE_RESP;
+	return err;
+}
+
+/**
+ * ufshcd_issue_devman_upiu_cmd - API for sending "utrd" type requests
+ * @hba:	per-adapter instance
+ * @req_upiu:	upiu request
+ * @rsp_upiu:	upiu reply
+ * @msgcode:	message code, one of UPIU Transaction Codes Initiator to Target
+ * @desc_buff:	pointer to descriptor buffer, NULL if NA
+ * @buff_len:	descriptor size, 0 if NA
+ * @desc_op:	descriptor operation
+ *
+ * Those type of requests uses UTP Transfer Request Descriptor - utrd.
+ * Therefore, it "rides" the device management infrastructure: uses its tag and
+ * tasks work queues.
+ *
+ * Since there is only one available tag for device management commands,
+ * the caller is expected to hold the hba->dev_cmd.lock mutex.
+ */
+static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
+					struct utp_upiu_req *req_upiu,
+					struct utp_upiu_req *rsp_upiu,
+					u8 *desc_buff, int *buff_len,
+					int cmd_type,
+					enum query_opcode desc_op)
+{
+	struct ufshcd_lrb *lrbp;
+	int err = 0;
+	int tag;
+	struct completion wait;
+	unsigned long flags;
+	u32 upiu_flags;
+
+	down_read(&hba->clk_scaling_lock);
+
+	wait_event(hba->dev_cmd.tag_wq, ufshcd_get_dev_cmd_tag(hba, &tag));
+
+	init_completion(&wait);
+	lrbp = &hba->lrb[tag];
+	WARN_ON(lrbp->cmd);
+
+	lrbp->cmd = NULL;
+	lrbp->sense_bufflen = 0;
+	lrbp->sense_buffer = NULL;
+	lrbp->task_tag = tag;
+	lrbp->lun = 0;
+	lrbp->intr_cmd = true;
+	hba->dev_cmd.type = cmd_type;
+
+	switch (hba->ufs_version) {
+	case UFSHCI_VERSION_10:
+	case UFSHCI_VERSION_11:
+		lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
+		break;
+	default:
+		lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
+		break;
+	}
+
+	/* update the task tag in the request upiu */
+	req_upiu->header.dword_0 |= cpu_to_be32(tag);
+
+	ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
+
+	/* just copy the upiu request as it is */
+	memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr));
+	if (desc_buff && desc_op == UPIU_QUERY_OPCODE_WRITE_DESC) {
+		/* The Data Segment Area is optional depending upon the query
+		 * function value. for WRITE DESCRIPTOR, the data segment
+		 * follows right after the tsf.
+		 */
+		memcpy(lrbp->ucd_req_ptr + 1, desc_buff, *buff_len);
+		*buff_len = 0;
+	}
+
+	memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
+
+	hba->dev_cmd.complete = &wait;
+
+	/* Make sure descriptors are ready before ringing the doorbell */
+	wmb();
+	spin_lock_irqsave(hba->host->host_lock, flags);
+	ufshcd_send_command(hba, tag);
+	spin_unlock_irqrestore(hba->host->host_lock, flags);
+
+	/*
+	 * ignore the returning value here - ufshcd_check_query_response is
+	 * bound to fail since dev_cmd.query and dev_cmd.type were left empty.
+	 * read the response directly ignoring all errors.
+	 */
+	ufshcd_wait_for_dev_cmd(hba, lrbp, QUERY_REQ_TIMEOUT);
+
+	/* just copy the upiu response as it is */
+	memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
+	if (desc_buff && desc_op == UPIU_QUERY_OPCODE_READ_DESC) {
+		u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu);
+		u16 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
+			       MASK_QUERY_DATA_SEG_LEN;
+
+		if (*buff_len >= resp_len) {
+			memcpy(desc_buff, descp, resp_len);
+			*buff_len = resp_len;
+		} else {
+			dev_warn(hba->dev, "rsp size is bigger than buffer");
+			*buff_len = 0;
+			err = -EINVAL;
+		}
+	}
+
+	ufshcd_put_dev_cmd_tag(hba, tag);
+	wake_up(&hba->dev_cmd.tag_wq);
+	up_read(&hba->clk_scaling_lock);
+	return err;
+}
+
+/**
+ * ufshcd_exec_raw_upiu_cmd - API function for sending raw upiu commands
+ * @hba:	per-adapter instance
+ * @req_upiu:	upiu request
+ * @rsp_upiu:	upiu reply - only 8 DW as we do not support scsi commands
+ * @msgcode:	message code, one of UPIU Transaction Codes Initiator to Target
+ * @desc_buff:	pointer to descriptor buffer, NULL if NA
+ * @buff_len:	descriptor size, 0 if NA
+ * @desc_op:	descriptor operation
+ *
+ * Supports UTP Transfer requests (nop and query), and UTP Task
+ * Management requests.
+ * It is up to the caller to fill the upiu conent properly, as it will
+ * be copied without any further input validations.
+ */
+int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba,
+			     struct utp_upiu_req *req_upiu,
+			     struct utp_upiu_req *rsp_upiu,
+			     int msgcode,
+			     u8 *desc_buff, int *buff_len,
+			     enum query_opcode desc_op)
+{
+	int err;
+	int cmd_type = DEV_CMD_TYPE_QUERY;
+	struct utp_task_req_desc treq = { { 0 }, };
+	int ocs_value;
+	u8 tm_f = be32_to_cpu(req_upiu->header.dword_1) >> 16 & MASK_TM_FUNC;
+
+	switch (msgcode) {
+	case UPIU_TRANSACTION_NOP_OUT:
+		cmd_type = DEV_CMD_TYPE_NOP;
+		/* fall through */
+	case UPIU_TRANSACTION_QUERY_REQ:
+		ufshcd_hold(hba, false);
+		mutex_lock(&hba->dev_cmd.lock);
+		err = ufshcd_issue_devman_upiu_cmd(hba, req_upiu, rsp_upiu,
+						   desc_buff, buff_len,
+						   cmd_type, desc_op);
+		mutex_unlock(&hba->dev_cmd.lock);
+		ufshcd_release(hba);
+
+		break;
+	case UPIU_TRANSACTION_TASK_REQ:
+		treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
+		treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
+
+		memcpy(&treq.req_header, req_upiu, sizeof(*req_upiu));
+
+		err = __ufshcd_issue_tm_cmd(hba, &treq, tm_f);
+		if (err == -ETIMEDOUT)
+			break;
+
+		ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
+		if (ocs_value != OCS_SUCCESS) {
+			dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__,
+				ocs_value);
+			break;
+		}
+
+		memcpy(rsp_upiu, &treq.rsp_header, sizeof(*rsp_upiu));
+
+		break;
+	default:
+		err = -EINVAL;
+
+		break;
+	}
+
+	return err;
+}
+
+/**
  * ufshcd_eh_device_reset_handler - device reset handler registered to
  *                                    scsi layer.
  * @cmd: SCSI command pointer
@@ -6589,6 +6756,74 @@
 	hba->desc_size.hlth_desc = QUERY_DESC_HEALTH_DEF_SIZE;
 }
 
+static struct ufs_ref_clk ufs_ref_clk_freqs[] = {
+	{19200000, REF_CLK_FREQ_19_2_MHZ},
+	{26000000, REF_CLK_FREQ_26_MHZ},
+	{38400000, REF_CLK_FREQ_38_4_MHZ},
+	{52000000, REF_CLK_FREQ_52_MHZ},
+	{0, REF_CLK_FREQ_INVAL},
+};
+
+static enum ufs_ref_clk_freq
+ufs_get_bref_clk_from_hz(unsigned long freq)
+{
+	int i;
+
+	for (i = 0; ufs_ref_clk_freqs[i].freq_hz; i++)
+		if (ufs_ref_clk_freqs[i].freq_hz == freq)
+			return ufs_ref_clk_freqs[i].val;
+
+	return REF_CLK_FREQ_INVAL;
+}
+
+void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk)
+{
+	unsigned long freq;
+
+	freq = clk_get_rate(refclk);
+
+	hba->dev_ref_clk_freq =
+		ufs_get_bref_clk_from_hz(freq);
+
+	if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL)
+		dev_err(hba->dev,
+		"invalid ref_clk setting = %ld\n", freq);
+}
+
+static int ufshcd_set_dev_ref_clk(struct ufs_hba *hba)
+{
+	int err;
+	u32 ref_clk;
+	u32 freq = hba->dev_ref_clk_freq;
+
+	err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
+			QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &ref_clk);
+
+	if (err) {
+		dev_err(hba->dev, "failed reading bRefClkFreq. err = %d\n",
+			err);
+		goto out;
+	}
+
+	if (ref_clk == freq)
+		goto out; /* nothing to update */
+
+	err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
+			QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &freq);
+
+	if (err) {
+		dev_err(hba->dev, "bRefClkFreq setting to %lu Hz failed\n",
+			ufs_ref_clk_freqs[freq].freq_hz);
+		goto out;
+	}
+
+	dev_dbg(hba->dev, "bRefClkFreq setting to %lu Hz succeeded\n",
+			ufs_ref_clk_freqs[freq].freq_hz);
+
+out:
+	return err;
+}
+
 /**
  * ufshcd_probe_hba - probe hba to detect device and initialize
  * @hba: per-adapter instance
@@ -6654,6 +6889,12 @@
 			"%s: Failed getting max supported power mode\n",
 			__func__);
 	} else {
+		/*
+		 * Set the right value to bRefClkFreq before attempting to
+		 * switch to HS gears.
+		 */
+		if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL)
+			ufshcd_set_dev_ref_clk(hba);
 		ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info);
 		if (ret) {
 			dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n",
@@ -6699,6 +6940,8 @@
 			hba->clk_scaling.is_allowed = true;
 		}
 
+		ufs_bsg_probe(hba);
+
 		scsi_scan_host(hba->host);
 		pm_runtime_put_sync(hba->dev);
 	}
@@ -7150,6 +7393,14 @@
 			goto out;
 		}
 
+		/*
+		 * Parse device ref clk freq as per device tree "ref_clk".
+		 * Default dev_ref_clk_freq is set to REF_CLK_FREQ_INVAL
+		 * in ufshcd_alloc_host().
+		 */
+		if (!strcmp(clki->name, "ref_clk"))
+			ufshcd_parse_dev_ref_clk_freq(hba, clki->clk);
+
 		if (clki->max_freq) {
 			ret = clk_set_rate(clki->clk, clki->max_freq);
 			if (ret) {
@@ -7937,6 +8188,7 @@
  */
 void ufshcd_remove(struct ufs_hba *hba)
 {
+	ufs_bsg_remove(hba);
 	ufs_sysfs_remove_nodes(hba->dev);
 	scsi_remove_host(hba->host);
 	/* disable interrupts */
@@ -8014,6 +8266,7 @@
 	hba->host = host;
 	hba->dev = dev;
 	*hba_handle = hba;
+	hba->dev_ref_clk_freq = REF_CLK_FREQ_INVAL;
 
 	INIT_LIST_HEAD(&hba->clk_list_head);
 
@@ -8090,7 +8343,7 @@
 	host->max_lun = UFS_MAX_LUNS;
 	host->max_channel = UFSHCD_MAX_CHANNEL;
 	host->unique_id = host->host_no;
-	host->max_cmd_len = MAX_CDB_SIZE;
+	host->max_cmd_len = UFS_CDB_SIZE;
 
 	hba->max_pwr_info.is_valid = false;
 
@@ -8168,7 +8421,7 @@
 						UIC_LINK_HIBERN8_STATE);
 
 	/* Set the default auto-hiberate idle timer value to 150 ms */
-	if (hba->capabilities & MASK_AUTO_HIBERN8_SUPPORT) {
+	if (ufshcd_is_auto_hibern8_supported(hba) && !hba->ahit) {
 		hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 150) |
 			    FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3);
 	}
diff -Naur --no-dereference a/drivers/scsi/ufs/ufshcd.h b/drivers/scsi/ufs/ufshcd.h
--- a/drivers/scsi/ufs/ufshcd.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/scsi/ufs/ufshcd.h	2020-06-19 21:00:31.424227716 -0400
@@ -550,6 +550,7 @@
 	void *priv;
 	unsigned int irq;
 	bool is_irq_enabled;
+	enum ufs_ref_clk_freq dev_ref_clk_freq;
 
 	/* Interrupt aggregation support is broken */
 	#define UFSHCD_QUIRK_BROKEN_INTR_AGGR			0x1
@@ -702,6 +703,9 @@
 	struct rw_semaphore clk_scaling_lock;
 	struct ufs_desc_size desc_size;
 	atomic_t scsi_block_reqs_cnt;
+
+	struct device		bsg_dev;
+	struct request_queue	*bsg_queue;
 };
 
 /* Returns true if clocks can be gated. Otherwise false */
@@ -736,6 +740,11 @@
 #endif
 }
 
+static inline bool ufshcd_is_auto_hibern8_supported(struct ufs_hba *hba)
+{
+	return (hba->capabilities & MASK_AUTO_HIBERN8_SUPPORT);
+}
+
 #define ufshcd_writel(hba, val, reg)	\
 	writel((val), (hba)->mmio_base + (reg))
 #define ufshcd_readl(hba, reg)	\
@@ -765,6 +774,7 @@
 int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
 				u32 val, unsigned long interval_us,
 				unsigned long timeout_ms, bool can_sleep);
+void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk);
 
 static inline void check_upiu_size(void)
 {
@@ -892,6 +902,15 @@
 
 u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba);
 
+int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd);
+
+int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba,
+			     struct utp_upiu_req *req_upiu,
+			     struct utp_upiu_req *rsp_upiu,
+			     int msgcode,
+			     u8 *desc_buff, int *buff_len,
+			     enum query_opcode desc_op);
+
 /* Wrapper functions for safely calling variant operations */
 static inline const char *ufshcd_get_var_name(struct ufs_hba *hba)
 {
diff -Naur --no-dereference a/drivers/scsi/ufs/ufshci.h b/drivers/scsi/ufs/ufshci.h
--- a/drivers/scsi/ufs/ufshci.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/scsi/ufs/ufshci.h	2020-06-19 21:00:31.424227716 -0400
@@ -144,8 +144,10 @@
 #define CONTROLLER_FATAL_ERROR			0x10000
 #define SYSTEM_BUS_FATAL_ERROR			0x20000
 
-#define UFSHCD_UIC_PWR_MASK	(UIC_HIBERNATE_ENTER |\
-				UIC_HIBERNATE_EXIT |\
+#define UFSHCD_UIC_HIBERN8_MASK	(UIC_HIBERNATE_ENTER |\
+				UIC_HIBERNATE_EXIT)
+
+#define UFSHCD_UIC_PWR_MASK	(UFSHCD_UIC_HIBERN8_MASK |\
 				UIC_POWER_MODE)
 
 #define UFSHCD_UIC_MASK		(UIC_COMMAND_COMPL | UFSHCD_UIC_PWR_MASK)
@@ -433,22 +435,25 @@
 	__le16  prd_table_offset;
 };
 
-/**
- * struct utp_task_req_desc - UTMRD structure
- * @header: UTMRD header DW-0 to DW-3
- * @task_req_upiu: Pointer to task request UPIU DW-4 to DW-11
- * @task_rsp_upiu: Pointer to task response UPIU DW12 to DW-19
+/*
+ * UTMRD structure.
  */
 struct utp_task_req_desc {
-
 	/* DW 0-3 */
 	struct request_desc_header header;
 
-	/* DW 4-11 */
-	__le32 task_req_upiu[TASK_REQ_UPIU_SIZE_DWORDS];
-
-	/* DW 12-19 */
-	__le32 task_rsp_upiu[TASK_RSP_UPIU_SIZE_DWORDS];
+	/* DW 4-11 - Task request UPIU structure */
+	struct utp_upiu_header	req_header;
+	__be32			input_param1;
+	__be32			input_param2;
+	__be32			input_param3;
+	__be32			__reserved1[2];
+
+	/* DW 12-19 - Task Management Response UPIU structure */
+	struct utp_upiu_header	rsp_header;
+	__be32			output_param1;
+	__be32			output_param2;
+	__be32			__reserved2[3];
 };
 
 #endif /* End of Header */
diff -Naur --no-dereference a/drivers/scsi/ufs/ufs-sysfs.c b/drivers/scsi/ufs/ufs-sysfs.c
--- a/drivers/scsi/ufs/ufs-sysfs.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/scsi/ufs/ufs-sysfs.c	2020-06-19 21:00:31.424227716 -0400
@@ -122,7 +122,7 @@
 {
 	unsigned long flags;
 
-	if (!(hba->capabilities & MASK_AUTO_HIBERN8_SUPPORT))
+	if (!ufshcd_is_auto_hibern8_supported(hba))
 		return;
 
 	spin_lock_irqsave(hba->host->host_lock, flags);
@@ -164,7 +164,7 @@
 {
 	struct ufs_hba *hba = dev_get_drvdata(dev);
 
-	if (!(hba->capabilities & MASK_AUTO_HIBERN8_SUPPORT))
+	if (!ufshcd_is_auto_hibern8_supported(hba))
 		return -EOPNOTSUPP;
 
 	return snprintf(buf, PAGE_SIZE, "%d\n", ufshcd_ahit_to_us(hba->ahit));
@@ -177,7 +177,7 @@
 	struct ufs_hba *hba = dev_get_drvdata(dev);
 	unsigned int timer;
 
-	if (!(hba->capabilities & MASK_AUTO_HIBERN8_SUPPORT))
+	if (!ufshcd_is_auto_hibern8_supported(hba))
 		return -EOPNOTSUPP;
 
 	if (kstrtouint(buf, 0, &timer))
diff -Naur --no-dereference a/drivers/soc/qcom/wcnss_ctrl.c b/drivers/soc/qcom/wcnss_ctrl.c
--- a/drivers/soc/qcom/wcnss_ctrl.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/soc/qcom/wcnss_ctrl.c	2020-06-19 21:00:31.424227716 -0400
@@ -284,6 +284,7 @@
 	strscpy(chinfo.name, name, sizeof(chinfo.name));
 	chinfo.src = RPMSG_ADDR_ANY;
 	chinfo.dst = RPMSG_ADDR_ANY;
+	chinfo.desc[0] = '\0';
 
 	return rpmsg_create_ept(_wcnss->channel->rpdev, cb, priv, chinfo);
 }
diff -Naur --no-dereference a/drivers/soc/ti/k3-navss-desc-pool.c b/drivers/soc/ti/k3-navss-desc-pool.c
--- a/drivers/soc/ti/k3-navss-desc-pool.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/soc/ti/k3-navss-desc-pool.c	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,129 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * TI K3 CPPI5 descriptors pool API
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/err.h>
+#include <linux/genalloc.h>
+#include <linux/kernel.h>
+
+struct k3_knav_desc_pool {
+	struct device		*dev;
+	dma_addr_t		dma_addr;
+	void			*cpumem;	/* dma_alloc map */
+	size_t			desc_size, mem_size;
+	size_t			num_desc;
+	struct gen_pool		*gen_pool;
+};
+
+void k3_knav_pool_destroy(struct k3_knav_desc_pool *pool)
+{
+	if (!pool)
+		return;
+
+	WARN(gen_pool_size(pool->gen_pool) != gen_pool_avail(pool->gen_pool),
+	     "k3_knav_desc_pool size %zu != avail %zu",
+	     gen_pool_size(pool->gen_pool),
+	     gen_pool_avail(pool->gen_pool));
+	if (pool->cpumem)
+		dma_free_coherent(pool->dev, pool->mem_size, pool->cpumem,
+				  pool->dma_addr);
+
+	gen_pool_destroy(pool->gen_pool);	/* frees pool->name */
+}
+EXPORT_SYMBOL_GPL(k3_knav_pool_destroy);
+
+struct k3_knav_desc_pool *k3_knav_pool_create_name(struct device *dev,
+						   size_t size,
+						   size_t desc_size,
+						   const char *name)
+{
+	struct k3_knav_desc_pool *pool;
+	int ret = -ENOMEM;
+	const char *pool_name = NULL;
+
+	pool = devm_kzalloc(dev, sizeof(*pool), GFP_KERNEL);
+	if (!pool)
+		return ERR_PTR(ret);
+
+	pool->dev = dev;
+	pool->desc_size	= roundup_pow_of_two(desc_size);
+	pool->num_desc	= size;
+	pool->mem_size	= pool->num_desc * pool->desc_size;
+
+	pool_name = kstrdup_const(name ? name : dev_name(pool->dev), GFP_KERNEL);
+	if (!pool_name)
+		return ERR_PTR(-ENOMEM);
+
+	pool->gen_pool = gen_pool_create(ilog2(pool->desc_size), -1);
+	if (IS_ERR(pool->gen_pool)) {
+		ret = PTR_ERR(pool->gen_pool);
+		dev_err(pool->dev, "pool create failed %d\n", ret);
+		kfree_const(pool_name);
+		goto gen_pool_create_fail;
+	}
+
+	pool->gen_pool->name = pool_name;
+
+	pool->cpumem = dma_alloc_coherent(pool->dev, pool->mem_size,
+					  &pool->dma_addr, GFP_KERNEL);
+
+	if (!pool->cpumem)
+		goto dma_alloc_fail;
+
+	ret = gen_pool_add_virt(pool->gen_pool, (unsigned long)pool->cpumem,
+				(phys_addr_t)pool->dma_addr, pool->mem_size,
+				-1);
+	if (ret < 0) {
+		dev_err(pool->dev, "pool add failed %d\n", ret);
+		goto gen_pool_add_virt_fail;
+	}
+
+	return pool;
+
+gen_pool_add_virt_fail:
+	dma_free_coherent(pool->dev, pool->mem_size, pool->cpumem,
+			  pool->dma_addr);
+dma_alloc_fail:
+	gen_pool_destroy(pool->gen_pool);	/* frees pool->name */
+gen_pool_create_fail:
+	devm_kfree(pool->dev, pool);
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(k3_knav_pool_create_name);
+
+dma_addr_t k3_knav_pool_virt2dma(struct k3_knav_desc_pool *pool, void *addr)
+{
+	return addr ? pool->dma_addr + (addr - pool->cpumem) : 0;
+}
+EXPORT_SYMBOL_GPL(k3_knav_pool_virt2dma);
+
+void *k3_knav_pool_dma2virt(struct k3_knav_desc_pool *pool, dma_addr_t dma)
+{
+	return dma ? pool->cpumem + (dma - pool->dma_addr) : NULL;
+}
+EXPORT_SYMBOL_GPL(k3_knav_pool_dma2virt);
+
+void *k3_knav_pool_alloc(struct k3_knav_desc_pool *pool)
+{
+	return (void *)gen_pool_alloc(pool->gen_pool, pool->desc_size);
+}
+EXPORT_SYMBOL_GPL(k3_knav_pool_alloc);
+
+void k3_knav_pool_free(struct k3_knav_desc_pool *pool, void *addr)
+{
+	gen_pool_free(pool->gen_pool, (unsigned long)addr, pool->desc_size);
+}
+EXPORT_SYMBOL_GPL(k3_knav_pool_free);
+
+size_t k3_knav_pool_avail(struct k3_knav_desc_pool *pool)
+{
+	return gen_pool_avail(pool->gen_pool) / pool->desc_size;
+}
+EXPORT_SYMBOL_GPL(k3_knav_pool_avail);
diff -Naur --no-dereference a/drivers/soc/ti/k3-ringacc.c b/drivers/soc/ti/k3-ringacc.c
--- a/drivers/soc/ti/k3-ringacc.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/soc/ti/k3-ringacc.c	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,1162 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * TI K3 NAVSS Ring Accelerator subsystem driver
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
+ */
+
+#include <linux/dma-mapping.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/soc/ti/k3-ringacc.h>
+#include <linux/soc/ti/ti_sci_protocol.h>
+
+static LIST_HEAD(k3_ringacc_list);
+static DEFINE_MUTEX(k3_ringacc_list_lock);
+
+#ifdef CONFIG_TI_K3_RINGACC_DEBUG
+#define	k3_nav_dbg(dev, arg...) dev_err(dev, arg)
+static	void dbg_writel(u32 v, void __iomem *reg)
+{
+	pr_err("WRITEL(32): v(%08X)-->reg(%p)\n", v, reg);
+	writel(v, reg);
+}
+
+static	u32 dbg_readl(void __iomem *reg)
+{
+	u32 v;
+
+	v = readl(reg);
+	pr_err("READL(32): v(%08X)<--reg(%p)\n", v, reg);
+	return v;
+}
+#else
+#define	k3_nav_dbg(dev, arg...) dev_dbg(dev, arg)
+#define dbg_writel(v, reg) writel(v, reg)
+
+#define dbg_readl(reg) readl(reg)
+#endif
+
+#define K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK		GENMASK(19, 0)
+
+/**
+ * struct k3_ring_rt_regs -  The RA Control/Status Registers region
+ */
+struct k3_ring_rt_regs {
+	u32	resv_16[4];
+	u32	db;		/* RT Ring N Doorbell Register */
+	u32	resv_4[1];
+	u32	occ;		/* RT Ring N Occupancy Register */
+	u32	indx;		/* RT Ring N Current Index Register */
+	u32	hwocc;		/* RT Ring N Hardware Occupancy Register */
+	u32	hwindx;		/* RT Ring N Current Index Register */
+};
+
+#define K3_RINGACC_RT_REGS_STEP	0x1000
+
+/**
+ * struct k3_ring_fifo_regs -  The Ring Accelerator Queues Registers region
+ */
+struct k3_ring_fifo_regs {
+	u32	head_data[128];		/* Ring Head Entry Data Registers */
+	u32	tail_data[128];		/* Ring Tail Entry Data Registers */
+	u32	peek_head_data[128];	/* Ring Peek Head Entry Data Regs */
+	u32	peek_tail_data[128];	/* Ring Peek Tail Entry Data Regs */
+};
+
+/**
+ * struct k3_ringacc_proxy_gcfg_regs - RA Proxy Global Config MMIO Region
+ */
+struct k3_ringacc_proxy_gcfg_regs {
+	u32	revision;	/* Revision Register */
+	u32	config;		/* Config Register */
+};
+
+#define K3_RINGACC_PROXY_CFG_THREADS_MASK		GENMASK(15, 0)
+
+/**
+ * struct k3_ringacc_proxy_target_regs -  Proxy Datapath MMIO Region
+ */
+struct k3_ringacc_proxy_target_regs {
+	u32	control;	/* Proxy Control Register */
+	u32	status;		/* Proxy Status Register */
+	u8	resv_512[504];
+	u32	data[128];	/* Proxy Data Register */
+};
+
+#define K3_RINGACC_PROXY_TARGET_STEP	0x1000
+#define K3_RINGACC_PROXY_NOT_USED	(-1)
+
+enum k3_ringacc_proxy_access_mode {
+	PROXY_ACCESS_MODE_HEAD = 0,
+	PROXY_ACCESS_MODE_TAIL = 1,
+	PROXY_ACCESS_MODE_PEEK_HEAD = 2,
+	PROXY_ACCESS_MODE_PEEK_TAIL = 3,
+};
+
+#define K3_RINGACC_FIFO_WINDOW_SIZE_BYTES  (512U)
+#define K3_RINGACC_FIFO_REGS_STEP	0x1000
+#define K3_RINGACC_MAX_DB_RING_CNT    (127U)
+
+/**
+ * struct k3_ring_ops -  Ring operations
+ */
+struct k3_ring_ops {
+	int (*push_tail)(struct k3_ring *ring, void *elm);
+	int (*push_head)(struct k3_ring *ring, void *elm);
+	int (*pop_tail)(struct k3_ring *ring, void *elm);
+	int (*pop_head)(struct k3_ring *ring, void *elm);
+};
+
+/**
+ * struct k3_ring - RA Ring descriptor
+ *
+ * @rt - Ring control/status registers
+ * @fifos - Ring queues registers
+ * @proxy - Ring Proxy Datapath registers
+ * @ring_mem_dma - Ring buffer dma address
+ * @ring_mem_virt - Ring buffer virt address
+ * @ops - Ring operations
+ * @size - Ring size in elements
+ * @elm_size - Size of the ring element
+ * @mode - Ring mode
+ * @flags - flags
+ * @free - Number of free elements
+ * @occ - Ring occupancy
+ * @windex - Write index (only for @K3_RINGACC_RING_MODE_RING)
+ * @rindex - Read index (only for @K3_RINGACC_RING_MODE_RING)
+ * @ring_id - Ring Id
+ * @parent - Pointer on struct @k3_ringacc
+ * @use_count - Use count for shared rings
+ * @proxy_id - RA Ring Proxy Id (only if @K3_RINGACC_RING_USE_PROXY)
+ */
+struct k3_ring {
+	struct k3_ring_rt_regs __iomem *rt;
+	struct k3_ring_fifo_regs __iomem *fifos;
+	struct k3_ringacc_proxy_target_regs  __iomem *proxy;
+	dma_addr_t	ring_mem_dma;
+	void		*ring_mem_virt;
+	struct k3_ring_ops *ops;
+	u32		size;
+	enum k3_ring_size elm_size;
+	enum k3_ring_mode mode;
+	u32		flags;
+#define K3_RING_FLAG_BUSY	BIT(1)
+#define K3_RING_FLAG_SHARED	BIT(2)
+	u32		free;
+	u32		occ;
+	u32		windex;
+	u32		rindex;
+	u32		ring_id;
+	struct k3_ringacc	*parent;
+	u32		use_count;
+	int		proxy_id;
+};
+
+/**
+ * struct k3_ringacc - Rings accelerator descriptor
+ *
+ * @dev - pointer on RA device
+ * @proxy_gcfg - RA proxy global config registers
+ * @proxy_target_base - RA proxy datapath region
+ * @num_rings - number of ring in RA
+ * @rm_gp_range - general purpose rings range from tisci
+ * @dma_ring_reset_quirk - DMA reset w/a enable
+ * @num_proxies - number of RA proxies
+ * @rings - array of rings descriptors (struct @k3_ring)
+ * @list - list of RAs in the system
+ * @tisci - pointer ti-sci handle
+ * @tisci_ring_ops - ti-sci rings ops
+ * @tisci_dev_id - ti-sci device id
+ */
+struct k3_ringacc {
+	struct device *dev;
+	struct k3_ringacc_proxy_gcfg_regs __iomem *proxy_gcfg;
+	void __iomem *proxy_target_base;
+	u32 num_rings; /* number of rings in Ringacc module */
+	unsigned long *rings_inuse;
+	struct ti_sci_resource *rm_gp_range;
+
+	bool dma_ring_reset_quirk;
+	u32 num_proxies;
+	unsigned long *proxy_inuse;
+
+	struct k3_ring *rings;
+	struct list_head list;
+	struct mutex req_lock; /* protect rings allocation */
+
+	const struct ti_sci_handle *tisci;
+	const struct ti_sci_rm_ringacc_ops *tisci_ring_ops;
+	u32  tisci_dev_id;
+};
+
+static long k3_ringacc_ring_get_fifo_pos(struct k3_ring *ring)
+{
+	return K3_RINGACC_FIFO_WINDOW_SIZE_BYTES -
+	       (4 << ring->elm_size);
+}
+
+static void *k3_ringacc_get_elm_addr(struct k3_ring *ring, u32 idx)
+{
+	return (idx * (4 << ring->elm_size) + ring->ring_mem_virt);
+}
+
+static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem);
+static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem);
+
+static struct k3_ring_ops k3_ring_mode_ring_ops = {
+		.push_tail = k3_ringacc_ring_push_mem,
+		.pop_head = k3_ringacc_ring_pop_mem,
+};
+
+static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem);
+static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem);
+static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem);
+static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem);
+
+static struct k3_ring_ops k3_ring_mode_msg_ops = {
+		.push_tail = k3_ringacc_ring_push_io,
+		.push_head = k3_ringacc_ring_push_head_io,
+		.pop_tail = k3_ringacc_ring_pop_tail_io,
+		.pop_head = k3_ringacc_ring_pop_io,
+};
+
+static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem);
+static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem);
+static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem);
+static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem);
+
+static struct k3_ring_ops k3_ring_mode_proxy_ops = {
+		.push_tail = k3_ringacc_ring_push_tail_proxy,
+		.push_head = k3_ringacc_ring_push_head_proxy,
+		.pop_tail = k3_ringacc_ring_pop_tail_proxy,
+		.pop_head = k3_ringacc_ring_pop_head_proxy,
+};
+
+#ifdef CONFIG_TI_K3_RINGACC_DEBUG
+void k3_ringacc_ring_dump(struct k3_ring *ring)
+{
+	struct device *dev = ring->parent->dev;
+
+	k3_nav_dbg(dev, "dump ring: %d\n", ring->ring_id);
+	k3_nav_dbg(dev, "dump mem virt %p, dma %pad\n",
+		   ring->ring_mem_virt, &ring->ring_mem_dma);
+	k3_nav_dbg(dev, "dump elmsize %d, size %d, mode %d, proxy_id %d\n",
+		   ring->elm_size, ring->size, ring->mode, ring->proxy_id);
+
+	k3_nav_dbg(dev, "dump ring_rt_regs: db%08x\n",
+		   readl(&ring->rt->db));
+	k3_nav_dbg(dev, "dump occ%08x\n",
+		   readl(&ring->rt->occ));
+	k3_nav_dbg(dev, "dump indx%08x\n",
+		   readl(&ring->rt->indx));
+	k3_nav_dbg(dev, "dump hwocc%08x\n",
+		   readl(&ring->rt->hwocc));
+	k3_nav_dbg(dev, "dump hwindx%08x\n",
+		   readl(&ring->rt->hwindx));
+
+	if (ring->ring_mem_virt)
+		print_hex_dump(KERN_ERR, "dump ring_mem_virt ",
+			       DUMP_PREFIX_NONE, 16, 1,
+			       ring->ring_mem_virt, 16 * 8, false);
+}
+EXPORT_SYMBOL_GPL(k3_ringacc_ring_dump);
+#endif
+
+struct device *k3_ringacc_get_dev(struct k3_ringacc *ringacc)
+{
+	return ringacc->dev;
+}
+EXPORT_SYMBOL_GPL(k3_ringacc_get_dev);
+
+struct k3_ring *k3_ringacc_request_ring(struct k3_ringacc *ringacc,
+					int id, u32 flags)
+{
+	int proxy_id = K3_RINGACC_PROXY_NOT_USED;
+
+	mutex_lock(&ringacc->req_lock);
+
+	if (id == K3_RINGACC_RING_ID_ANY) {
+		/* Request for any general purpose ring */
+		struct ti_sci_resource_desc *gp_rings =
+						&ringacc->rm_gp_range->desc[0];
+		unsigned long size;
+
+		size = gp_rings->start + gp_rings->num;
+		id = find_next_zero_bit(ringacc->rings_inuse, size,
+					gp_rings->start);
+		if (id == size)
+			goto error;
+	} else if (id < 0) {
+		goto error;
+	}
+
+	if (test_bit(id, ringacc->rings_inuse) &&
+	    !(ringacc->rings[id].flags & K3_RING_FLAG_SHARED))
+		goto error;
+	else if (ringacc->rings[id].flags & K3_RING_FLAG_SHARED)
+		goto out;
+
+	if (flags & K3_RINGACC_RING_USE_PROXY) {
+		proxy_id = find_next_zero_bit(ringacc->proxy_inuse,
+					      ringacc->num_proxies, 0);
+		if (proxy_id == ringacc->num_proxies)
+			goto error;
+	}
+
+	if (!try_module_get(ringacc->dev->driver->owner))
+		goto error;
+
+	if (proxy_id != K3_RINGACC_PROXY_NOT_USED) {
+		set_bit(proxy_id, ringacc->proxy_inuse);
+		ringacc->rings[id].proxy_id = proxy_id;
+		k3_nav_dbg(ringacc->dev, "Giving ring#%d proxy#%d\n",
+			   id, proxy_id);
+	} else {
+		k3_nav_dbg(ringacc->dev, "Giving ring#%d\n", id);
+	}
+
+	set_bit(id, ringacc->rings_inuse);
+out:
+	ringacc->rings[id].use_count++;
+	mutex_unlock(&ringacc->req_lock);
+	return &ringacc->rings[id];
+
+error:
+	mutex_unlock(&ringacc->req_lock);
+	return NULL;
+}
+EXPORT_SYMBOL_GPL(k3_ringacc_request_ring);
+
+static void k3_ringacc_ring_reset_sci(struct k3_ring *ring)
+{
+	struct k3_ringacc *ringacc = ring->parent;
+	int ret;
+
+	ret = ringacc->tisci_ring_ops->config(ringacc->tisci,
+					TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID,
+					ringacc->tisci_dev_id,
+					ring->ring_id,
+					0,
+					0,
+					ring->size,
+					0,
+					0,
+					0);
+	if (ret)
+		dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n",
+			ret, ring->ring_id);
+}
+
+void k3_ringacc_ring_reset(struct k3_ring *ring)
+{
+	if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
+		return;
+
+	ring->occ = 0;
+	ring->free = 0;
+	ring->rindex = 0;
+	ring->windex = 0;
+
+	k3_ringacc_ring_reset_sci(ring);
+}
+EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset);
+
+static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_ring *ring,
+					       enum k3_ring_mode mode)
+{
+	struct k3_ringacc *ringacc = ring->parent;
+	int ret;
+
+	ret = ringacc->tisci_ring_ops->config(ringacc->tisci,
+					TI_SCI_MSG_VALUE_RM_RING_MODE_VALID,
+					ringacc->tisci_dev_id,
+					ring->ring_id,
+					0,
+					0,
+					0,
+					mode,
+					0,
+					0);
+	if (ret)
+		dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n",
+			ret, ring->ring_id);
+}
+
+void k3_ringacc_ring_reset_dma(struct k3_ring *ring, u32 occ)
+{
+	if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
+		return;
+
+	if (!ring->parent->dma_ring_reset_quirk) {
+		k3_ringacc_ring_reset(ring);
+		return;
+	}
+
+	if (!occ)
+		occ = dbg_readl(&ring->rt->occ);
+
+	if (occ) {
+		u32 db_ring_cnt, db_ring_cnt_cur;
+
+		k3_nav_dbg(ring->parent->dev, "%s %u occ: %u\n", __func__,
+			   ring->ring_id, occ);
+		/* 2. Reset the ring */
+		k3_ringacc_ring_reset_sci(ring);
+
+		/*
+		 * 3. Setup the ring in ring/doorbell mode
+		 * (if not already in this mode)
+		 */
+		if (ring->mode != K3_RINGACC_RING_MODE_RING)
+			k3_ringacc_ring_reconfig_qmode_sci(ring,
+						K3_RINGACC_RING_MODE_RING);
+		/*
+		 * 4. Ring the doorbell 2**22 – ringOcc times.
+		 * This will wrap the internal UDMAP ring state occupancy
+		 * counter (which is 21-bits wide) to 0.
+		 */
+		db_ring_cnt = (1U << 22) - occ;
+
+		while (db_ring_cnt != 0) {
+			/*
+			 * Ring the doorbell with the maximum count each
+			 * iteration if possible to minimize the total
+			 * of writes
+			 */
+			if (db_ring_cnt > K3_RINGACC_MAX_DB_RING_CNT)
+				db_ring_cnt_cur = K3_RINGACC_MAX_DB_RING_CNT;
+			else
+				db_ring_cnt_cur = db_ring_cnt;
+
+			writel(db_ring_cnt_cur, &ring->rt->db);
+			db_ring_cnt -= db_ring_cnt_cur;
+		}
+
+		/* 5. Restore the original ring mode (if not ring mode) */
+		if (ring->mode != K3_RINGACC_RING_MODE_RING)
+			k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode);
+	}
+
+	/* 2. Reset the ring */
+	k3_ringacc_ring_reset(ring);
+}
+EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset_dma);
+
+static void k3_ringacc_ring_free_sci(struct k3_ring *ring)
+{
+	struct k3_ringacc *ringacc = ring->parent;
+	int ret;
+
+	ret = ringacc->tisci_ring_ops->config(ringacc->tisci,
+					      TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER,
+					      ringacc->tisci_dev_id,
+					      ring->ring_id,
+					      0,
+					      0,
+					      0,
+					      0,
+					      0,
+					      0);
+	if (ret)
+		dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n",
+			ret, ring->ring_id);
+}
+
+int k3_ringacc_ring_free(struct k3_ring *ring)
+{
+	struct k3_ringacc *ringacc;
+
+	if (!ring)
+		return -EINVAL;
+
+	ringacc = ring->parent;
+
+	k3_nav_dbg(ring->parent->dev, "flags: 0x%08x\n", ring->flags);
+
+	if (!test_bit(ring->ring_id, ringacc->rings_inuse))
+		return -EINVAL;
+
+	mutex_lock(&ringacc->req_lock);
+
+	if (--ring->use_count)
+		goto out;
+
+	if (!(ring->flags & K3_RING_FLAG_BUSY))
+		goto no_init;
+
+	k3_ringacc_ring_free_sci(ring);
+
+	dma_free_coherent(ringacc->dev,
+			  ring->size * (4 << ring->elm_size),
+			  ring->ring_mem_virt, ring->ring_mem_dma);
+	ring->flags = 0;
+	ring->ops = NULL;
+	if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) {
+		clear_bit(ring->proxy_id, ringacc->proxy_inuse);
+		ring->proxy = NULL;
+		ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
+	}
+
+no_init:
+	clear_bit(ring->ring_id, ringacc->rings_inuse);
+
+	module_put(ringacc->dev->driver->owner);
+
+out:
+	mutex_unlock(&ringacc->req_lock);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(k3_ringacc_ring_free);
+
+u32 k3_ringacc_get_ring_id(struct k3_ring *ring)
+{
+	if (!ring)
+		return -EINVAL;
+
+	return ring->ring_id;
+}
+EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_id);
+
+u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring)
+{
+	if (!ring)
+		return -EINVAL;
+
+	return ring->parent->tisci_dev_id;
+}
+EXPORT_SYMBOL_GPL(k3_ringacc_get_tisci_dev_id);
+
+static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring)
+{
+	struct k3_ringacc *ringacc = ring->parent;
+	u32 ring_idx;
+	int ret;
+
+	if (!ringacc->tisci)
+		return -EINVAL;
+
+	ring_idx = ring->ring_id;
+	ret = ringacc->tisci_ring_ops->config(ringacc->tisci,
+					      TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER,
+					      ringacc->tisci_dev_id,
+					      ring_idx,
+					      lower_32_bits(ring->ring_mem_dma),
+					      upper_32_bits(ring->ring_mem_dma),
+					      ring->size,
+					      ring->mode,
+					      ring->elm_size,
+					      0);
+	if (ret)
+		dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n",
+			ret, ring_idx);
+
+	return ret;
+}
+
+int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg)
+{
+	struct k3_ringacc *ringacc = ring->parent;
+	int ret = 0;
+
+	if (!ring || !cfg)
+		return -EINVAL;
+	if (cfg->elm_size > K3_RINGACC_RING_ELSIZE_256 ||
+	    cfg->mode > K3_RINGACC_RING_MODE_QM ||
+	    cfg->size & ~K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK ||
+	    !test_bit(ring->ring_id, ringacc->rings_inuse))
+		return -EINVAL;
+
+	if (ring->use_count != 1)
+		return 0;
+
+	ring->size = cfg->size;
+	ring->elm_size = cfg->elm_size;
+	ring->mode = cfg->mode;
+	ring->occ = 0;
+	ring->free = 0;
+	ring->rindex = 0;
+	ring->windex = 0;
+
+	if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED)
+		ring->proxy = ringacc->proxy_target_base +
+			      ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP;
+
+	switch (ring->mode) {
+	case K3_RINGACC_RING_MODE_RING:
+		ring->ops = &k3_ring_mode_ring_ops;
+		break;
+	case K3_RINGACC_RING_MODE_QM:
+		/*
+		 * In Queue mode elm_size can be 8 only and each operation
+		 * uses 2 element slots
+		 */
+		if (cfg->elm_size != K3_RINGACC_RING_ELSIZE_8 ||
+		    cfg->size % 2)
+			goto err_free_proxy;
+	case K3_RINGACC_RING_MODE_MESSAGE:
+		if (ring->proxy)
+			ring->ops = &k3_ring_mode_proxy_ops;
+		else
+			ring->ops = &k3_ring_mode_msg_ops;
+		break;
+	default:
+		ring->ops = NULL;
+		ret = -EINVAL;
+		goto err_free_proxy;
+	};
+
+	ring->ring_mem_virt =
+			dma_zalloc_coherent(ringacc->dev,
+					    ring->size * (4 << ring->elm_size),
+					    &ring->ring_mem_dma, GFP_KERNEL);
+	if (!ring->ring_mem_virt) {
+		dev_err(ringacc->dev, "Failed to alloc ring mem\n");
+		ret = -ENOMEM;
+		goto err_free_ops;
+	}
+
+	ret = k3_ringacc_ring_cfg_sci(ring);
+
+	if (ret)
+		goto err_free_mem;
+
+	ring->flags |= K3_RING_FLAG_BUSY;
+	ring->flags |= (cfg->flags & K3_RINGACC_RING_SHARED) ?
+			K3_RING_FLAG_SHARED : 0;
+
+	k3_ringacc_ring_dump(ring);
+
+	return 0;
+
+err_free_mem:
+	dma_free_coherent(ringacc->dev,
+			  ring->size * (4 << ring->elm_size),
+			  ring->ring_mem_virt,
+			  ring->ring_mem_dma);
+err_free_ops:
+	ring->ops = NULL;
+err_free_proxy:
+	ring->proxy = NULL;
+	return ret;
+}
+EXPORT_SYMBOL_GPL(k3_ringacc_ring_cfg);
+
+u32 k3_ringacc_ring_get_size(struct k3_ring *ring)
+{
+	if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
+		return -EINVAL;
+
+	return ring->size;
+}
+EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_size);
+
+u32 k3_ringacc_ring_get_free(struct k3_ring *ring)
+{
+	if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
+		return -EINVAL;
+
+	if (!ring->free)
+		ring->free = ring->size - dbg_readl(&ring->rt->occ);
+
+	return ring->free;
+}
+EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_free);
+
+u32 k3_ringacc_ring_get_occ(struct k3_ring *ring)
+{
+	if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
+		return -EINVAL;
+
+	return dbg_readl(&ring->rt->occ);
+}
+EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_occ);
+
+u32 k3_ringacc_ring_is_full(struct k3_ring *ring)
+{
+	return !k3_ringacc_ring_get_free(ring);
+}
+EXPORT_SYMBOL_GPL(k3_ringacc_ring_is_full);
+
+enum k3_ringacc_access_mode {
+	K3_RINGACC_ACCESS_MODE_PUSH_HEAD,
+	K3_RINGACC_ACCESS_MODE_POP_HEAD,
+	K3_RINGACC_ACCESS_MODE_PUSH_TAIL,
+	K3_RINGACC_ACCESS_MODE_POP_TAIL,
+	K3_RINGACC_ACCESS_MODE_PEEK_HEAD,
+	K3_RINGACC_ACCESS_MODE_PEEK_TAIL,
+};
+
+static int k3_ringacc_ring_cfg_proxy(struct k3_ring *ring,
+				     enum k3_ringacc_proxy_access_mode mode)
+{
+	u32 val;
+
+	val = ring->ring_id;
+	val |= mode << 16;
+	val |= ring->elm_size << 24;
+	dbg_writel(val, &ring->proxy->control);
+	return 0;
+}
+
+static int k3_ringacc_ring_access_proxy(struct k3_ring *ring, void *elem,
+					enum k3_ringacc_access_mode access_mode)
+{
+	void __iomem *ptr;
+
+	ptr = (void __iomem *)&ring->proxy->data;
+
+	switch (access_mode) {
+	case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
+	case K3_RINGACC_ACCESS_MODE_POP_HEAD:
+		k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD);
+		break;
+	case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
+	case K3_RINGACC_ACCESS_MODE_POP_TAIL:
+		k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	ptr += k3_ringacc_ring_get_fifo_pos(ring);
+
+	switch (access_mode) {
+	case K3_RINGACC_ACCESS_MODE_POP_HEAD:
+	case K3_RINGACC_ACCESS_MODE_POP_TAIL:
+		k3_nav_dbg(ring->parent->dev, "proxy:memcpy_fromio(x): --> ptr(%p), mode:%d\n",
+			   ptr, access_mode);
+		memcpy_fromio(elem, ptr, (4 << ring->elm_size));
+		ring->occ--;
+		break;
+	case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
+	case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
+		k3_nav_dbg(ring->parent->dev, "proxy:memcpy_toio(x): --> ptr(%p), mode:%d\n",
+			   ptr, access_mode);
+		memcpy_toio(ptr, elem, (4 << ring->elm_size));
+		ring->free--;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	k3_nav_dbg(ring->parent->dev, "proxy: free%d occ%d\n",
+		   ring->free, ring->occ);
+	return 0;
+}
+
+static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem)
+{
+	return k3_ringacc_ring_access_proxy(ring, elem,
+					    K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
+}
+
+static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem)
+{
+	return k3_ringacc_ring_access_proxy(ring, elem,
+					    K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
+}
+
+static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem)
+{
+	return k3_ringacc_ring_access_proxy(ring, elem,
+					    K3_RINGACC_ACCESS_MODE_POP_HEAD);
+}
+
+static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem)
+{
+	return k3_ringacc_ring_access_proxy(ring, elem,
+					    K3_RINGACC_ACCESS_MODE_POP_HEAD);
+}
+
+static int k3_ringacc_ring_access_io(struct k3_ring *ring, void *elem,
+				     enum k3_ringacc_access_mode access_mode)
+{
+	void __iomem *ptr;
+
+	switch (access_mode) {
+	case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
+	case K3_RINGACC_ACCESS_MODE_POP_HEAD:
+		ptr = (void __iomem *)&ring->fifos->head_data;
+		break;
+	case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
+	case K3_RINGACC_ACCESS_MODE_POP_TAIL:
+		ptr = (void __iomem *)&ring->fifos->tail_data;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	ptr += k3_ringacc_ring_get_fifo_pos(ring);
+
+	switch (access_mode) {
+	case K3_RINGACC_ACCESS_MODE_POP_HEAD:
+	case K3_RINGACC_ACCESS_MODE_POP_TAIL:
+		k3_nav_dbg(ring->parent->dev, "memcpy_fromio(x): --> ptr(%p), mode:%d\n",
+			   ptr, access_mode);
+		memcpy_fromio(elem, ptr, (4 << ring->elm_size));
+		ring->occ--;
+		break;
+	case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
+	case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
+		k3_nav_dbg(ring->parent->dev, "memcpy_toio(x): --> ptr(%p), mode:%d\n",
+			   ptr, access_mode);
+		memcpy_toio(ptr, elem, (4 << ring->elm_size));
+		ring->free--;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	k3_nav_dbg(ring->parent->dev, "free%d index%d occ%d index%d\n",
+		   ring->free, ring->windex, ring->occ, ring->rindex);
+	return 0;
+}
+
+static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem)
+{
+	return k3_ringacc_ring_access_io(ring, elem,
+					 K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
+}
+
+static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem)
+{
+	return k3_ringacc_ring_access_io(ring, elem,
+					 K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
+}
+
+static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem)
+{
+	return k3_ringacc_ring_access_io(ring, elem,
+					 K3_RINGACC_ACCESS_MODE_POP_HEAD);
+}
+
+static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem)
+{
+	return k3_ringacc_ring_access_io(ring, elem,
+					 K3_RINGACC_ACCESS_MODE_POP_HEAD);
+}
+
+static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem)
+{
+	void *elem_ptr;
+
+	elem_ptr = k3_ringacc_get_elm_addr(ring, ring->windex);
+
+	memcpy(elem_ptr, elem, (4 << ring->elm_size));
+
+	ring->windex = (ring->windex + 1) % ring->size;
+	ring->free--;
+	dbg_writel(1, &ring->rt->db);
+
+	k3_nav_dbg(ring->parent->dev, "ring_push_mem: free%d index%d\n",
+		   ring->free, ring->windex);
+
+	return 0;
+}
+
+static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem)
+{
+	void *elem_ptr;
+
+	elem_ptr = k3_ringacc_get_elm_addr(ring, ring->rindex);
+
+	memcpy(elem, elem_ptr, (4 << ring->elm_size));
+
+	ring->rindex = (ring->rindex + 1) % ring->size;
+	ring->occ--;
+	dbg_writel(-1, &ring->rt->db);
+
+	k3_nav_dbg(ring->parent->dev, "ring_pop_mem: occ%d index%d pos_ptr%p\n",
+		   ring->occ, ring->rindex, elem_ptr);
+	return 0;
+}
+
+int k3_ringacc_ring_push(struct k3_ring *ring, void *elem)
+{
+	int ret = -EOPNOTSUPP;
+
+	if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
+		return -EINVAL;
+
+	k3_nav_dbg(ring->parent->dev, "ring_push: free%d index%d\n",
+		   ring->free, ring->windex);
+
+	if (k3_ringacc_ring_is_full(ring))
+		return -ENOMEM;
+
+	if (ring->ops && ring->ops->push_tail)
+		ret = ring->ops->push_tail(ring, elem);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(k3_ringacc_ring_push);
+
+int k3_ringacc_ring_push_head(struct k3_ring *ring, void *elem)
+{
+	int ret = -EOPNOTSUPP;
+
+	if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
+		return -EINVAL;
+
+	k3_nav_dbg(ring->parent->dev, "ring_push_head: free%d index%d\n",
+		   ring->free, ring->windex);
+
+	if (k3_ringacc_ring_is_full(ring))
+		return -ENOMEM;
+
+	if (ring->ops && ring->ops->push_head)
+		ret = ring->ops->push_head(ring, elem);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(k3_ringacc_ring_push_head);
+
+int k3_ringacc_ring_pop(struct k3_ring *ring, void *elem)
+{
+	int ret = -EOPNOTSUPP;
+
+	if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
+		return -EINVAL;
+
+	if (!ring->occ)
+		ring->occ = k3_ringacc_ring_get_occ(ring);
+
+	k3_nav_dbg(ring->parent->dev, "ring_pop: occ%d index%d\n",
+		   ring->occ, ring->rindex);
+
+	if (!ring->occ)
+		return -ENODATA;
+
+	if (ring->ops && ring->ops->pop_head)
+		ret = ring->ops->pop_head(ring, elem);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop);
+
+int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem)
+{
+	int ret = -EOPNOTSUPP;
+
+	if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
+		return -EINVAL;
+
+	if (!ring->occ)
+		ring->occ = k3_ringacc_ring_get_occ(ring);
+
+	k3_nav_dbg(ring->parent->dev, "ring_pop_tail: occ%d index%d\n",
+		   ring->occ, ring->rindex);
+
+	if (!ring->occ)
+		return -ENODATA;
+
+	if (ring->ops && ring->ops->pop_tail)
+		ret = ring->ops->pop_tail(ring, elem);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop_tail);
+
+struct k3_ringacc *of_k3_ringacc_get_by_phandle(struct device_node *np,
+						const char *property)
+{
+	struct device_node *ringacc_np;
+	struct k3_ringacc *ringacc = ERR_PTR(-EPROBE_DEFER);
+	struct k3_ringacc *entry;
+
+	ringacc_np = of_parse_phandle(np, property, 0);
+	if (!ringacc_np)
+		return ERR_PTR(-ENODEV);
+
+	mutex_lock(&k3_ringacc_list_lock);
+	list_for_each_entry(entry, &k3_ringacc_list, list)
+		if (entry->dev->of_node == ringacc_np) {
+			ringacc = entry;
+			break;
+		}
+	mutex_unlock(&k3_ringacc_list_lock);
+	of_node_put(ringacc_np);
+
+	return ringacc;
+}
+EXPORT_SYMBOL_GPL(of_k3_ringacc_get_by_phandle);
+
+static int k3_ringacc_probe_dt(struct k3_ringacc *ringacc)
+{
+	struct device_node *node = ringacc->dev->of_node;
+	struct device *dev = ringacc->dev;
+	int ret;
+
+	if (!node) {
+		dev_err(dev, "device tree info unavailable\n");
+		return -ENODEV;
+	}
+
+	ret = of_property_read_u32(node, "ti,num-rings", &ringacc->num_rings);
+	if (ret) {
+		dev_err(dev, "ti,num-rings read failure %d\n", ret);
+		return ret;
+	}
+
+	ringacc->dma_ring_reset_quirk =
+			of_property_read_bool(node, "ti,dma-ring-reset-quirk");
+
+	ringacc->tisci = ti_sci_get_by_phandle(node, "ti,sci");
+	if (IS_ERR(ringacc->tisci)) {
+		ret = PTR_ERR(ringacc->tisci);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "ti,sci read fail %d\n", ret);
+		ringacc->tisci = NULL;
+		return ret;
+	}
+
+	ret = of_property_read_u32(node, "ti,sci-dev-id",
+				   &ringacc->tisci_dev_id);
+	if (ret)
+		dev_err(dev, "ti,sci-dev-id read fail %d\n", ret);
+
+	ringacc->rm_gp_range = devm_ti_sci_get_of_resource(ringacc->tisci, dev,
+						ringacc->tisci_dev_id,
+						"ti,sci-rm-range-gp-rings");
+	if (IS_ERR(ringacc->rm_gp_range))
+		ret = PTR_ERR(ringacc->rm_gp_range);
+
+	return ret;
+}
+
+static int k3_ringacc_probe(struct platform_device *pdev)
+{
+	struct k3_ringacc *ringacc;
+	void __iomem *base_fifo, *base_rt;
+	struct device *dev = &pdev->dev;
+	struct resource *res;
+	int ret, i;
+
+	ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL);
+	if (!ringacc)
+		return -ENOMEM;
+
+	ringacc->dev = dev;
+	mutex_init(&ringacc->req_lock);
+
+	ret = k3_ringacc_probe_dt(ringacc);
+	if (ret)
+		return ret;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "rt");
+	base_rt = devm_ioremap_resource(dev, res);
+	if (IS_ERR(base_rt))
+		return PTR_ERR(base_rt);
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "fifos");
+	base_fifo = devm_ioremap_resource(dev, res);
+	if (IS_ERR(base_fifo))
+		return PTR_ERR(base_fifo);
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "proxy_gcfg");
+	ringacc->proxy_gcfg = devm_ioremap_resource(dev, res);
+	if (IS_ERR(ringacc->proxy_gcfg))
+		return PTR_ERR(ringacc->proxy_gcfg);
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+					   "proxy_target");
+	ringacc->proxy_target_base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(ringacc->proxy_target_base))
+		return PTR_ERR(ringacc->proxy_target_base);
+
+	ringacc->num_proxies = dbg_readl(&ringacc->proxy_gcfg->config) &
+					 K3_RINGACC_PROXY_CFG_THREADS_MASK;
+
+	ringacc->rings = devm_kzalloc(dev,
+				      sizeof(*ringacc->rings) *
+				      ringacc->num_rings,
+				      GFP_KERNEL);
+	ringacc->rings_inuse = devm_kcalloc(dev,
+					    BITS_TO_LONGS(ringacc->num_rings),
+					    sizeof(unsigned long), GFP_KERNEL);
+	ringacc->proxy_inuse = devm_kcalloc(dev,
+					    BITS_TO_LONGS(ringacc->num_proxies),
+					    sizeof(unsigned long), GFP_KERNEL);
+
+	if (!ringacc->rings || !ringacc->rings_inuse || !ringacc->proxy_inuse)
+		return -ENOMEM;
+
+	for (i = 0; i < ringacc->num_rings; i++) {
+		ringacc->rings[i].rt = base_rt +
+				       K3_RINGACC_RT_REGS_STEP * i;
+		ringacc->rings[i].fifos = base_fifo +
+					  K3_RINGACC_FIFO_REGS_STEP * i;
+		ringacc->rings[i].parent = ringacc;
+		ringacc->rings[i].ring_id = i;
+		ringacc->rings[i].proxy_id = K3_RINGACC_PROXY_NOT_USED;
+	}
+	dev_set_drvdata(dev, ringacc);
+
+	ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;
+
+	pm_runtime_enable(dev);
+	ret = pm_runtime_get_sync(dev);
+	if (ret < 0) {
+		pm_runtime_put_noidle(dev);
+		dev_err(dev, "Failed to enable pm %d\n", ret);
+		goto err;
+	}
+
+	mutex_lock(&k3_ringacc_list_lock);
+	list_add_tail(&ringacc->list, &k3_ringacc_list);
+	mutex_unlock(&k3_ringacc_list_lock);
+
+	dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n",
+		 ringacc->num_rings,
+		 ringacc->rm_gp_range->desc[0].start,
+		 ringacc->rm_gp_range->desc[0].num,
+		 ringacc->tisci_dev_id);
+	dev_info(dev, "dma-ring-reset-quirk: %s\n",
+		 ringacc->dma_ring_reset_quirk ? "enabled" : "disabled");
+	dev_info(dev, "RA Proxy rev. %08x, num_proxies:%u\n",
+		 dbg_readl(&ringacc->proxy_gcfg->revision),
+		 ringacc->num_proxies);
+	return 0;
+
+err:
+	pm_runtime_disable(dev);
+	return ret;
+}
+
+static int k3_ringacc_remove(struct platform_device *pdev)
+{
+	struct k3_ringacc *ringacc = dev_get_drvdata(&pdev->dev);
+
+	pm_runtime_put_sync(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+
+	mutex_lock(&k3_ringacc_list_lock);
+	list_del(&ringacc->list);
+	mutex_unlock(&k3_ringacc_list_lock);
+	return 0;
+}
+
+/* Match table for of_platform binding */
+static const struct of_device_id k3_ringacc_of_match[] = {
+	{ .compatible = "ti,am654-navss-ringacc", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, k3_ringacc_of_match);
+
+static struct platform_driver k3_ringacc_driver = {
+	.probe		= k3_ringacc_probe,
+	.remove		= k3_ringacc_remove,
+	.driver		= {
+		.name	= "k3-ringacc",
+		.of_match_table = k3_ringacc_of_match,
+	},
+};
+module_platform_driver(k3_ringacc_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("TI Ringacc driver for K3 SOCs");
+MODULE_AUTHOR("Grygorii Strashko <grygorii.strashko@ti.com>");
diff -Naur --no-dereference a/drivers/soc/ti/Kconfig b/drivers/soc/ti/Kconfig
--- a/drivers/soc/ti/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/soc/ti/Kconfig	2020-06-19 21:00:31.424227716 -0400
@@ -8,6 +8,11 @@
 	help
 	  Enable support for TI's AM6 SoC Family support
 
+config ARCH_K3_J721E_SOC
+	bool "K3 J721E SoC"
+	help
+	  Enable support for TI's J721E SoC Family support
+
 endif
 
 endif
@@ -20,6 +25,17 @@
 
 if SOC_TI
 
+config KEYSTONE_DSP_MEM
+	tristate "TI Keystone DSP Memory Mapping Driver"
+	depends on ARCH_KEYSTONE
+	help
+	  Userspace memory mapping interface driver for TI Keystone SoCs.
+	  Provides access to MSM SRAM memory regions and dedicated DDR
+	  carveout memory regions to user space to aid userspace loading
+	  of the DSPs within the SoC.
+
+	  If unsure, say N.
+
 config KEYSTONE_NAVIGATOR_QMSS
 	tristate "Keystone Queue Manager Sub System"
 	depends on ARCH_KEYSTONE
@@ -45,11 +61,12 @@
 config AMX3_PM
 	tristate "AMx3 Power Management"
 	depends on SOC_AM33XX || SOC_AM43XX
-	depends on WKUP_M3_IPC && TI_EMIF_SRAM && SRAM
+	depends on WKUP_M3_IPC && TI_EMIF_SRAM && SRAM && RTC_DRV_OMAP
 	help
 	  Enable power management on AM335x and AM437x. Required for suspend to mem
 	  and standby states on both AM335x and AM437x platforms and for deeper cpuidle
-	  c-states on AM335x.
+	  c-states on AM335x. Also required for rtc and ddr in self-refresh low
+	  power mode on AM437x platforms.
 
 config WKUP_M3_IPC
 	tristate "TI AMx3 Wkup-M3 IPC Driver"
@@ -73,4 +90,49 @@
 	  called ti_sci_pm_domains. Note this is needed early in boot before
 	  rootfs may be available.
 
+config TI_PRUSS
+	tristate "TI PRU-ICSS Subsystem Platform drivers"
+	depends on SOC_AM33XX || SOC_AM43XX || SOC_DRA7XX || ARCH_KEYSTONE || ARCH_K3
+	select MFD_SYSCON
+	help
+	  TI PRU-ICSS Subsystem platform specific support.
+
+	  Say Y or M here to support the Programmable Realtime Unit (PRU)
+	  processors on various TI SoCs. It's safe to say N here if you're
+	  not interested in the PRU or if you are unsure.
+
+config TI_K3_RINGACC
+	tristate "K3 Ring accelerator Sub System"
+	depends on ARCH_K3 || COMPILE_TEST
+	default y
+	help
+	  Say y here to support the K3 Ring accelerator module.
+	  The Ring Accelerator (RINGACC or RA)  provides hardware acceleration
+	  to enable straightforward passing of work between a producer
+	  and a consumer. There is one RINGACC module per NAVSS on TI AM65x SoCs
+	  If unsure, say N.
+
+config TI_K3_RINGACC_DEBUG
+	tristate "K3 Ring accelerator Sub System tests and debug"
+	depends on TI_K3_RINGACC
+	default n
+
+config TI_K3_UDMA_DESC_POOL
+	tristate "K3 CPPI5 DMA descriptors pool"
+	depends on ARCH_K3
+	help
+	  Say y here to enable support for creating and maintaining a
+	  pool of descriptors usable with CPPI5 DMA. Such pools are
+	  required for networking usecases.
+
+config TI_PAT
+	tristate "TI PAT DMA-BUF exporter"
+	depends on ARCH_K3
+	select REGMAP
+	help
+	  Driver for TI Page-based Address Translator (PAT). This driver
+	  provides the an API allowing the remapping of a non-contiguous
+	  DMA-BUF into a contiguous one that is sutable for devices needing
+	  contiguous memory.
+
 endif # SOC_TI
diff -Naur --no-dereference a/drivers/soc/ti/keystone_dsp_mem.c b/drivers/soc/ti/keystone_dsp_mem.c
--- a/drivers/soc/ti/keystone_dsp_mem.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/soc/ti/keystone_dsp_mem.c	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,402 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * TI Keystone DSP Memory Mapping Driver
+ *
+ * Copyright (C) 2015-2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+#include <linux/module.h>
+#include <linux/uaccess.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+#include <linux/miscdevice.h>
+#include <linux/fs.h>
+#include <linux/mm.h>
+#include <linux/io.h>
+#include <linux/slab.h>
+#include <uapi/linux/keystone_dsp_mem.h>
+
+#define KEYSTONE_ALIAS_PHYS_START	0x80000000ULL
+#define KEYSTONE_ALIAS_PHYS_SIZE	0x80000000ULL /* 2G */
+
+#define KEYSTONE_HIGH_PHYS_START	0x800000000ULL
+#define KEYSTONE_HIGH_PHYS_LIMIT	(KEYSTONE_HIGH_PHYS_START + \
+					 KEYSTONE_ALIAS_PHYS_SIZE)
+
+#define to_alias_addr(addr)	(((addr) - KEYSTONE_HIGH_PHYS_START) + \
+				 KEYSTONE_ALIAS_PHYS_START)
+
+/**
+ * struct keystone_dsp_mem - internal memory structure
+ * @addr: physical address on the bus to access the memory region
+ * @size: size of the memory region
+ * @kobj: kobject for the sysfs directory file
+ */
+struct keystone_dsp_mem {
+	phys_addr_t addr;
+	resource_size_t size;
+	struct kobject kobj;
+};
+
+#define to_dsp_mem(obj) container_of(obj, struct keystone_dsp_mem, kobj)
+
+/**
+ * struct keystone_dsp_mem_info - Keystone DSP Memory device structure
+ * @misc: child miscdevice structure
+ * @mem: memory region array pointer
+ * @num_maps: number of memory regions
+ */
+struct keystone_dsp_mem_info {
+	struct miscdevice misc;
+	struct keystone_dsp_mem *mem;
+	int num_maps;
+};
+
+static struct keystone_dsp_mem_info *dsp_mem;
+
+#define to_dsp_mem_info(m) container_of(m, struct keystone_dsp_mem_info, misc)
+
+static ssize_t mem_addr_show(struct keystone_dsp_mem *mem, char *buf)
+{
+	return sprintf(buf, "%pa\n", &mem->addr);
+}
+
+static ssize_t mem_size_show(struct keystone_dsp_mem *mem, char *buf)
+{
+	return sprintf(buf, "%pa\n", &mem->size);
+}
+
+struct mem_sysfs_entry {
+	struct attribute attr;
+	ssize_t (*show)(struct keystone_dsp_mem *mem, char *buf);
+	ssize_t (*store)(struct keystone_dsp_mem *mem, const char *buf,
+			 size_t len);
+};
+
+static struct mem_sysfs_entry addr_attribute =
+	__ATTR(addr, 0444, mem_addr_show, NULL);
+static struct mem_sysfs_entry size_attribute =
+	__ATTR(size, 0444, mem_size_show, NULL);
+
+static struct attribute *attrs[] = {
+	&addr_attribute.attr,
+	&size_attribute.attr,
+	NULL,	/* sentinel */
+};
+
+static ssize_t mem_type_show(struct kobject *kobj, struct attribute *attr,
+			     char *buf)
+{
+	struct keystone_dsp_mem *mem = to_dsp_mem(kobj);
+	struct mem_sysfs_entry *entry;
+
+	entry = container_of(attr, struct mem_sysfs_entry, attr);
+	if (!entry->show)
+		return -EIO;
+
+	return entry->show(mem, buf);
+}
+
+static const struct sysfs_ops mem_sysfs_ops = {
+	.show = mem_type_show,
+};
+
+static struct kobj_type mem_attr_type = {
+	.sysfs_ops	= &mem_sysfs_ops,
+	.default_attrs	= attrs,
+};
+
+static int keystone_dsp_mem_add_attrs(struct keystone_dsp_mem_info *dsp_mem)
+{
+	int i, ret;
+	struct keystone_dsp_mem *mem;
+	struct kobject *kobj_parent = &dsp_mem->misc.this_device->kobj;
+
+	for (i = 0; i < dsp_mem->num_maps; i++) {
+		mem = &dsp_mem->mem[i];
+		kobject_init(&mem->kobj, &mem_attr_type);
+		ret = kobject_add(&mem->kobj, kobj_parent, "memory%d", i);
+		if (ret)
+			goto err_kobj;
+		ret = kobject_uevent(&mem->kobj, KOBJ_ADD);
+		if (ret)
+			goto err_kobj;
+	}
+
+	return 0;
+
+err_kobj:
+	for (; i >= 0; i--) {
+		mem = &dsp_mem->mem[i];
+		kobject_put(&mem->kobj);
+	}
+	return ret;
+}
+
+static void keystone_dsp_mem_del_attrs(struct keystone_dsp_mem_info *dsp_mem)
+{
+	int i;
+	struct keystone_dsp_mem *mem;
+
+	for (i = 0; i < dsp_mem->num_maps; i++) {
+		mem = &dsp_mem->mem[i];
+		kobject_put(&mem->kobj);
+	}
+}
+
+static int keystone_dsp_mem_check_addr(struct keystone_dsp_mem_info *dsp_mem,
+				       int mask, size_t size)
+{
+	size_t req_offset;
+	u32 index;
+
+	index = mask & KEYSTONE_DSP_MEM_MAP_INDEX_MASK;
+	if (index >= dsp_mem->num_maps) {
+		pr_err("%s: invalid mmap region index %d\n", __func__, index);
+		return -EINVAL;
+	}
+
+	req_offset = (mask - index) << PAGE_SHIFT;
+	if (req_offset + size < req_offset) {
+		pr_err("%s: invalid request - overflow, mmap offset = 0x%zx size 0x%zx region %d\n",
+		       __func__, req_offset, size, index);
+		return -EINVAL;
+	}
+
+	if ((req_offset + size) > dsp_mem->mem[index].size) {
+		pr_err("%s: invalid request - out of range, mmap offset 0x%zx size 0x%zx region %d\n",
+		       __func__, req_offset, size, index);
+		return -EINVAL;
+	}
+
+	return index;
+}
+
+/*
+ * This is a custom mmap function following semantics based on the UIO
+ * mmap implementation. The vm_pgoff passed in the vma structure is a
+ * combination of the memory region index and the actual page offset in
+ * that region. This checks if user request is in valid range before
+ * providing mmap access.
+ *
+ * XXX: Evaluate this approach, as the MSMC memory can be mapped in whole
+ * into userspace as it is not super-large, and the allowable kernel
+ * unmapped DDR memory can be mmaped using traditional mmap semantics.
+ */
+static int keystone_dsp_mem_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	size_t size = vma->vm_end - vma->vm_start;
+	struct miscdevice *misc = file->private_data;
+	struct keystone_dsp_mem_info *dsp_mem = to_dsp_mem_info(misc);
+	int index;
+
+	index = keystone_dsp_mem_check_addr(dsp_mem, vma->vm_pgoff, size);
+	if (index < 0)
+		return index;
+
+	vma->vm_page_prot =
+		phys_mem_access_prot(file,
+				     (dsp_mem->mem[index].addr >> PAGE_SHIFT) +
+				     (vma->vm_pgoff - index), size,
+				     vma->vm_page_prot);
+
+	if (remap_pfn_range(vma, vma->vm_start,
+			    (dsp_mem->mem[index].addr >> PAGE_SHIFT) +
+			    (vma->vm_pgoff - index), size, vma->vm_page_prot))
+		return -EAGAIN;
+
+	return 0;
+}
+
+static const struct file_operations keystone_dsp_mem_fops = {
+	.owner	= THIS_MODULE,
+	.mmap	= keystone_dsp_mem_mmap,
+};
+
+static int keystone_dsp_mem_parse(struct device_node *np, int index)
+{
+	phys_addr_t start, end, addr, size;
+	struct resource res;
+	resource_size_t rsize;
+	int ret, j;
+
+	if (!of_find_property(np, "no-map", NULL)) {
+		pr_err("dsp reserved memory regions without no-map are not supported\n");
+		return -EINVAL;
+	}
+
+	ret = of_address_to_resource(np, 0, &res);
+	if (ret)
+		return ret;
+
+	/* make sure only aliased addresses are covered */
+	rsize = resource_size(&res);
+	start = res.start;
+	end = res.start + rsize;
+	if (start < KEYSTONE_HIGH_PHYS_START ||
+	    start >= KEYSTONE_HIGH_PHYS_LIMIT ||
+	    end > KEYSTONE_HIGH_PHYS_LIMIT) {
+		pr_err("invalid address/size for keystone dsp memory carveout: %pa of size %pa\n",
+		       &start, &rsize);
+		return -EINVAL;
+	}
+
+	/* check for overlaps */
+	start = to_alias_addr(start);
+	end = to_alias_addr(end);
+	for (j = 0; j < index; j++) {
+		addr = dsp_mem->mem[j].addr;
+		size = dsp_mem->mem[j].size;
+		if ((end > addr && end <= addr + size) ||
+		    (start >= addr && start < addr + size) ||
+		    (start < addr && end > addr + size)) {
+			pr_err("dsp memory carveout (%pa of size %pa) overlaps with (%pa of size %pa)\n",
+			       &start, &rsize, &addr, &size);
+			return -EINVAL;
+		}
+	}
+
+	dsp_mem->mem[index].addr = to_alias_addr(res.start);
+	dsp_mem->mem[index].size = resource_size(&res);
+
+	return 0;
+}
+
+static int keystone_dsp_mem_init(void)
+{
+	struct miscdevice *misc;
+	struct resource res;
+	struct device_node *rmem_np, *sram_np, *np;
+	int ret, i = 0;
+	int num_maps = 0, num_sram = 0;
+
+	if (!of_have_populated_dt())
+		return -ENOTSUPP;
+
+	/* module is supported only on TI Keystone SoCs */
+	if (!of_machine_is_compatible("ti,keystone"))
+		return -ENOTSUPP;
+
+	/* count the number of DDR regions */
+	rmem_np = of_find_node_by_path("/reserved-memory");
+	if (rmem_np) {
+		for_each_available_child_of_node(rmem_np, np) {
+			if (of_device_is_compatible(np,
+						    "ti,keystone-dsp-mem-pool"))
+				num_maps++;
+		}
+	}
+
+	for_each_compatible_node(sram_np, NULL, "ti,keystone-dsp-msm-ram") {
+		if (!of_device_is_available(sram_np))
+			continue;
+		num_sram++;
+	}
+
+	if ((!num_maps && !num_sram) ||
+	    (num_maps + num_sram > KEYSTONE_DSP_MEM_MAP_INDEX_MASK)) {
+		ret = -EINVAL;
+		goto put_rmem;
+	}
+
+	dsp_mem = kzalloc(sizeof(*dsp_mem), GFP_KERNEL);
+	if (!dsp_mem) {
+		ret = -ENOMEM;
+		goto put_rmem;
+	}
+
+	dsp_mem->mem = kcalloc(num_maps + num_sram, sizeof(*dsp_mem->mem),
+			       GFP_KERNEL);
+	if (!dsp_mem->mem) {
+		ret = -ENOMEM;
+		goto free_dsp;
+	}
+
+	/* handle reserved-memory carveouts */
+	if (num_maps) {
+		for_each_available_child_of_node(rmem_np, np) {
+			if (!of_device_is_compatible(np,
+						"ti,keystone-dsp-mem-pool"))
+				continue;
+
+			ret = keystone_dsp_mem_parse(np, i);
+			if (ret) {
+				of_node_put(np);
+				goto free_mem;
+			}
+			i++;
+			dsp_mem->num_maps++;
+		}
+	}
+
+	/* handle on-chip SRAM reserved regions */
+	if (num_sram) {
+		for_each_compatible_node(sram_np, NULL,
+					 "ti,keystone-dsp-msm-ram") {
+			if (!of_device_is_available(sram_np))
+				continue;
+
+			ret = of_address_to_resource(sram_np, 0, &res);
+			if (ret) {
+				ret = -EINVAL;
+				of_node_put(sram_np);
+				goto free_mem;
+			}
+			dsp_mem->mem[i].addr = res.start;
+			dsp_mem->mem[i].size = resource_size(&res);
+			i++;
+			dsp_mem->num_maps++;
+		}
+	}
+
+	misc = &dsp_mem->misc;
+	misc->minor = MISC_DYNAMIC_MINOR;
+	misc->name = "dspmem";
+	misc->fops = &keystone_dsp_mem_fops;
+	misc->parent = NULL;
+	ret = misc_register(misc);
+	if (ret) {
+		pr_err("%s: could not register dspmem misc device\n", __func__);
+		goto free_mem;
+	}
+
+	ret = keystone_dsp_mem_add_attrs(dsp_mem);
+	if (ret) {
+		pr_err("%s: error creating sysfs files (%d)\n", __func__, ret);
+		goto unregister_misc;
+	}
+	of_node_put(rmem_np);
+
+	pr_info("registered dspmem misc device\n");
+
+	return 0;
+
+unregister_misc:
+	misc_deregister(&dsp_mem->misc);
+free_mem:
+	kfree(dsp_mem->mem);
+free_dsp:
+	kfree(dsp_mem);
+	dsp_mem = NULL;
+put_rmem:
+	of_node_put(rmem_np);
+	return ret;
+}
+
+static void keystone_dsp_mem_exit(void)
+{
+	keystone_dsp_mem_del_attrs(dsp_mem);
+
+	misc_deregister(&dsp_mem->misc);
+
+	kfree(dsp_mem->mem);
+	kfree(dsp_mem);
+	dsp_mem = NULL;
+}
+
+module_init(keystone_dsp_mem_init);
+module_exit(keystone_dsp_mem_exit);
+
+MODULE_AUTHOR("Suman Anna <s-anna@ti.com>");
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("TI Keystone DSP Memory Mapping Driver");
diff -Naur --no-dereference a/drivers/soc/ti/Makefile b/drivers/soc/ti/Makefile
--- a/drivers/soc/ti/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/soc/ti/Makefile	2020-06-19 21:00:31.424227716 -0400
@@ -2,9 +2,14 @@
 #
 # TI Keystone SOC drivers
 #
+obj-$(CONFIG_KEYSTONE_DSP_MEM)		+= keystone_dsp_mem.o
 obj-$(CONFIG_KEYSTONE_NAVIGATOR_QMSS)	+= knav_qmss.o
 knav_qmss-y := knav_qmss_queue.o knav_qmss_acc.o
 obj-$(CONFIG_KEYSTONE_NAVIGATOR_DMA)	+= knav_dma.o
 obj-$(CONFIG_AMX3_PM)			+= pm33xx.o
 obj-$(CONFIG_WKUP_M3_IPC)		+= wkup_m3_ipc.o
 obj-$(CONFIG_TI_SCI_PM_DOMAINS)		+= ti_sci_pm_domains.o
+obj-$(CONFIG_TI_PRUSS)			+= pruss_soc_bus.o pruss.o
+obj-$(CONFIG_TI_K3_RINGACC)		+= k3-ringacc.o
+obj-$(CONFIG_TI_K3_UDMA_DESC_POOL)	+= k3-navss-desc-pool.o
+obj-$(CONFIG_TI_PAT)			+= ti-pat.o
diff -Naur --no-dereference a/drivers/soc/ti/pm33xx.c b/drivers/soc/ti/pm33xx.c
--- a/drivers/soc/ti/pm33xx.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/soc/ti/pm33xx.c	2020-06-19 21:00:31.424227716 -0400
@@ -6,6 +6,7 @@
  *	Vaibhav Bedia, Dave Gerlach
  */
 
+#include <linux/clk.h>
 #include <linux/cpu.h>
 #include <linux/err.h>
 #include <linux/genalloc.h>
@@ -13,9 +14,11 @@
 #include <linux/init.h>
 #include <linux/io.h>
 #include <linux/module.h>
+#include <linux/nvmem-consumer.h>
 #include <linux/of.h>
 #include <linux/platform_data/pm33xx.h>
 #include <linux/platform_device.h>
+#include <linux/rtc.h>
 #include <linux/sizes.h>
 #include <linux/sram.h>
 #include <linux/suspend.h>
@@ -29,33 +32,182 @@
 #define AMX3_PM_SRAM_SYMBOL_OFFSET(sym) ((unsigned long)(sym) - \
 					 (unsigned long)pm_sram->do_wfi)
 
+#define RTC_SCRATCH_RESUME_REG	0
+#define RTC_SCRATCH_MAGIC_REG	1
+#define RTC_REG_BOOT_MAGIC	0x8cd0 /* RTC */
+#define GIC_INT_SET_PENDING_BASE 0x200
+#define AM43XX_GIC_DIST_BASE	0x48241000
+
+static u32 rtc_magic_val;
+
 static int (*am33xx_do_wfi_sram)(unsigned long unused);
 static phys_addr_t am33xx_do_wfi_sram_phys;
 
 static struct gen_pool *sram_pool, *sram_pool_data;
 static unsigned long ocmcram_location, ocmcram_location_data;
 
+static struct rtc_device *omap_rtc;
+static void __iomem *gic_dist_base;
+
 static struct am33xx_pm_platform_data *pm_ops;
 static struct am33xx_pm_sram_addr *pm_sram;
 
 static struct device *pm33xx_dev;
 static struct wkup_m3_ipc *m3_ipc;
 
+#ifdef CONFIG_SUSPEND
+
+static int rtc_only_idle;
+static int retrigger_irq;
+
 static unsigned long suspend_wfi_flags;
 
+static struct wkup_m3_wakeup_src wakeup_src = {.irq_nr = 0,
+	.src = "Unknown",
+};
+
+static struct wkup_m3_wakeup_src rtc_alarm_wakeup = {
+	.irq_nr = 108, .src = "RTC Alarm",
+};
+
+static struct wkup_m3_wakeup_src rtc_ext_wakeup = {
+	.irq_nr = 0, .src = "Ext wakeup",
+};
+
+#endif
+
 static u32 sram_suspend_address(unsigned long addr)
 {
 	return ((unsigned long)am33xx_do_wfi_sram +
 		AMX3_PM_SRAM_SYMBOL_OFFSET(addr));
 }
 
+static int am33xx_push_sram_idle(void)
+{
+	struct am33xx_pm_ro_sram_data ro_sram_data;
+	int ret;
+	u32 table_addr, ro_data_addr;
+	void *copy_addr;
+
+	ro_sram_data.amx3_pm_sram_data_virt = ocmcram_location_data;
+	ro_sram_data.amx3_pm_sram_data_phys =
+		gen_pool_virt_to_phys(sram_pool_data, ocmcram_location_data);
+	ro_sram_data.rtc_base_virt = pm_ops->get_rtc_base_addr();
+
+	/* Save physical address to calculate resume offset during pm init */
+	am33xx_do_wfi_sram_phys = gen_pool_virt_to_phys(sram_pool,
+							ocmcram_location);
+
+	am33xx_do_wfi_sram = sram_exec_copy(sram_pool, (void *)ocmcram_location,
+					    pm_sram->do_wfi,
+					    *pm_sram->do_wfi_sz);
+	if (!am33xx_do_wfi_sram) {
+		dev_err(pm33xx_dev,
+			"PM: %s: am33xx_do_wfi copy to sram failed\n",
+			__func__);
+		return -ENODEV;
+	}
+
+	table_addr =
+		sram_suspend_address((unsigned long)pm_sram->emif_sram_table);
+	ret = ti_emif_copy_pm_function_table(sram_pool, (void *)table_addr);
+	if (ret) {
+		dev_dbg(pm33xx_dev,
+			"PM: %s: EMIF function copy failed\n", __func__);
+		return -EPROBE_DEFER;
+	}
+
+	ro_data_addr =
+		sram_suspend_address((unsigned long)pm_sram->ro_sram_data);
+	copy_addr = sram_exec_copy(sram_pool, (void *)ro_data_addr,
+				   &ro_sram_data,
+				   sizeof(ro_sram_data));
+	if (!copy_addr) {
+		dev_err(pm33xx_dev,
+			"PM: %s: ro_sram_data copy to sram failed\n",
+			__func__);
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+static int am33xx_do_sram_idle(u32 wfi_flags)
+{
+	int ret = 0;
+
+	if (!m3_ipc || !pm_ops)
+		return 0;
+
+	if (wfi_flags & WFI_FLAG_WAKE_M3)
+		ret = m3_ipc->ops->prepare_low_power(m3_ipc, WKUP_M3_IDLE);
+
+	return pm_ops->cpu_suspend(am33xx_do_wfi_sram, wfi_flags);
+}
+
+static int __init am43xx_map_gic(void)
+{
+	gic_dist_base = ioremap(AM43XX_GIC_DIST_BASE, SZ_4K);
+
+	if (!gic_dist_base)
+		return -ENOMEM;
+
+	return 0;
+}
+
 #ifdef CONFIG_SUSPEND
+
+static int rtc_only_idle;
+static int retrigger_irq;
+
+struct wkup_m3_wakeup_src rtc_wake_src(void)
+{
+	u32 i;
+
+	i = __raw_readl(pm_ops->get_rtc_base_addr() + 0x44) & 0x40;
+
+	if (i) {
+		retrigger_irq = rtc_alarm_wakeup.irq_nr;
+		return rtc_alarm_wakeup;
+	}
+
+	retrigger_irq = rtc_ext_wakeup.irq_nr;
+
+	return rtc_ext_wakeup;
+}
+
+int am33xx_rtc_only_idle(unsigned long wfi_flags)
+{
+	rtc_power_off_program(omap_rtc);
+	am33xx_do_wfi_sram(wfi_flags);
+	return 0;
+}
+
 static int am33xx_pm_suspend(suspend_state_t suspend_state)
 {
 	int i, ret = 0;
 
-	ret = pm_ops->soc_suspend((unsigned long)suspend_state,
-				  am33xx_do_wfi_sram, suspend_wfi_flags);
+	if (suspend_state == PM_SUSPEND_MEM &&
+	    pm_ops->check_off_mode_enable()) {
+		pm_ops->prepare_rtc_suspend();
+		pm_ops->save_context();
+		suspend_wfi_flags |= WFI_FLAG_RTC_ONLY;
+		clk_save_context();
+		ret = pm_ops->soc_suspend(suspend_state, am33xx_rtc_only_idle,
+					  suspend_wfi_flags);
+
+		suspend_wfi_flags &= ~WFI_FLAG_RTC_ONLY;
+
+		if (!ret) {
+			clk_restore_context();
+			pm_ops->restore_context();
+			m3_ipc->ops->set_rtc_only(m3_ipc);
+			am33xx_push_sram_idle();
+		}
+	} else {
+		ret = pm_ops->soc_suspend(suspend_state, am33xx_do_wfi_sram,
+					  suspend_wfi_flags);
+	}
 
 	if (ret) {
 		dev_err(pm33xx_dev, "PM: Kernel suspend failure\n");
@@ -77,8 +229,20 @@
 				"PM: CM3 returned unknown result = %d\n", i);
 			ret = -1;
 		}
+
+		/* print the wakeup reason */
+		if (rtc_only_idle) {
+			wakeup_src = rtc_wake_src();
+			pr_info("PM: Wakeup source %s\n", wakeup_src.src);
+		} else {
+			pr_info("PM: Wakeup source %s\n",
+				m3_ipc->ops->request_wake_src(m3_ipc));
+		}
 	}
 
+	if (suspend_state == PM_SUSPEND_MEM && pm_ops->check_off_mode_enable())
+		pm_ops->prepare_rtc_resume();
+
 	return ret;
 }
 
@@ -102,6 +266,16 @@
 {
 	int ret = -EINVAL;
 
+	if (state == PM_SUSPEND_MEM && pm_ops->check_off_mode_enable()) {
+		nvmem_device_write(omap_rtc->nvmem, RTC_SCRATCH_MAGIC_REG * 4,
+				   4, (void *)&rtc_magic_val);
+		rtc_only_idle = 1;
+	} else {
+		rtc_only_idle = 0;
+	}
+
+	cpu_idle_poll_ctrl(true);
+
 	switch (state) {
 	case PM_SUSPEND_MEM:
 		ret = m3_ipc->ops->prepare_low_power(m3_ipc, WKUP_M3_DEEPSLEEP);
@@ -116,7 +290,29 @@
 
 static void am33xx_pm_end(void)
 {
+	u32 val = 0;
+
 	m3_ipc->ops->finish_low_power(m3_ipc);
+	if (rtc_only_idle) {
+		if (retrigger_irq)
+			/*
+			 * 32 bits of Interrupt Set-Pending correspond to 32
+			 * 32 interupts. Compute the bit offset of the
+			 * Interrupt and set that particular bit
+			 * Compute the register offset by dividing interrupt
+			 * number by 32 and mutiplying by 4
+			 */
+			writel_relaxed(1 << (retrigger_irq & 31),
+				       gic_dist_base + GIC_INT_SET_PENDING_BASE
+				       + retrigger_irq / 32 * 4);
+			nvmem_device_write(omap_rtc->nvmem,
+					   RTC_SCRATCH_MAGIC_REG * 4,
+					   4, (void *)&val);
+	}
+
+	rtc_only_idle = 0;
+
+	cpu_idle_poll_ctrl(false);
 }
 
 static int am33xx_pm_valid(suspend_state_t state)
@@ -219,51 +415,32 @@
 	return ret;
 }
 
-static int am33xx_push_sram_idle(void)
+static int am33xx_pm_rtc_setup(void)
 {
-	struct am33xx_pm_ro_sram_data ro_sram_data;
-	int ret;
-	u32 table_addr, ro_data_addr;
-	void *copy_addr;
-
-	ro_sram_data.amx3_pm_sram_data_virt = ocmcram_location_data;
-	ro_sram_data.amx3_pm_sram_data_phys =
-		gen_pool_virt_to_phys(sram_pool_data, ocmcram_location_data);
-	ro_sram_data.rtc_base_virt = pm_ops->get_rtc_base_addr();
+	struct device_node *np;
+	unsigned long val = 0;
 
-	/* Save physical address to calculate resume offset during pm init */
-	am33xx_do_wfi_sram_phys = gen_pool_virt_to_phys(sram_pool,
-							ocmcram_location);
+	np = of_find_node_by_name(NULL, "rtc");
 
-	am33xx_do_wfi_sram = sram_exec_copy(sram_pool, (void *)ocmcram_location,
-					    pm_sram->do_wfi,
-					    *pm_sram->do_wfi_sz);
-	if (!am33xx_do_wfi_sram) {
-		dev_err(pm33xx_dev,
-			"PM: %s: am33xx_do_wfi copy to sram failed\n",
-			__func__);
-		return -ENODEV;
-	}
-
-	table_addr =
-		sram_suspend_address((unsigned long)pm_sram->emif_sram_table);
-	ret = ti_emif_copy_pm_function_table(sram_pool, (void *)table_addr);
-	if (ret) {
-		dev_dbg(pm33xx_dev,
-			"PM: %s: EMIF function copy failed\n", __func__);
-		return -EPROBE_DEFER;
-	}
+	if (of_device_is_available(np)) {
+		omap_rtc = rtc_class_open("rtc0");
+		if (!omap_rtc) {
+			pr_warn("PM: rtc0 not available");
+			return -EPROBE_DEFER;
+		}
 
-	ro_data_addr =
-		sram_suspend_address((unsigned long)pm_sram->ro_sram_data);
-	copy_addr = sram_exec_copy(sram_pool, (void *)ro_data_addr,
-				   &ro_sram_data,
-				   sizeof(ro_sram_data));
-	if (!copy_addr) {
-		dev_err(pm33xx_dev,
-			"PM: %s: ro_sram_data copy to sram failed\n",
-			__func__);
-		return -ENODEV;
+		nvmem_device_read(omap_rtc->nvmem, RTC_SCRATCH_MAGIC_REG * 4,
+				  4, (void *)&rtc_magic_val);
+		if ((rtc_magic_val & 0xffff) != RTC_REG_BOOT_MAGIC)
+			pr_warn("PM: bootloader does not support rtc-only!\n");
+
+		nvmem_device_write(omap_rtc->nvmem, RTC_SCRATCH_MAGIC_REG * 4,
+				   4, (void *)&val);
+		val = pm_sram->resume_address;
+		nvmem_device_write(omap_rtc->nvmem, RTC_SCRATCH_RESUME_REG * 4,
+				   4, (void *)&val);
+	} else {
+		pr_warn("PM: no-rtc available, rtc-only mode disabled.\n");
 	}
 
 	return 0;
@@ -284,34 +461,42 @@
 		return -ENODEV;
 	}
 
+	ret = am43xx_map_gic();
+	if (ret) {
+		pr_err("PM: Could not ioremap GIC base\n");
+		return ret;
+	}
+
 	pm_sram = pm_ops->get_sram_addrs();
 	if (!pm_sram) {
 		dev_err(dev, "PM: Cannot get PM asm function addresses!!\n");
 		return -ENODEV;
 	}
 
+	m3_ipc = wkup_m3_ipc_get();
+	if (!m3_ipc) {
+		pr_err("PM: Cannot get wkup_m3_ipc handle\n");
+		return -EPROBE_DEFER;
+	}
+
 	pm33xx_dev = dev;
 
 	ret = am33xx_pm_alloc_sram();
 	if (ret)
 		return ret;
 
-	ret = am33xx_push_sram_idle();
+	ret = am33xx_pm_rtc_setup();
 	if (ret)
 		goto err_free_sram;
 
-	m3_ipc = wkup_m3_ipc_get();
-	if (!m3_ipc) {
-		dev_dbg(dev, "PM: Cannot get wkup_m3_ipc handle\n");
-		ret = -EPROBE_DEFER;
+	ret = am33xx_push_sram_idle();
+	if (ret)
 		goto err_free_sram;
-	}
 
 	am33xx_pm_set_ipc_ops();
 
 #ifdef CONFIG_SUSPEND
 	suspend_set_ops(&am33xx_pm_ops);
-#endif /* CONFIG_SUSPEND */
 
 	/*
 	 * For a system suspend we must flush the caches, we want
@@ -323,8 +508,9 @@
 	suspend_wfi_flags |= WFI_FLAG_SELF_REFRESH;
 	suspend_wfi_flags |= WFI_FLAG_SAVE_EMIF;
 	suspend_wfi_flags |= WFI_FLAG_WAKE_M3;
+#endif /* CONFIG_SUSPEND */
 
-	ret = pm_ops->init();
+	ret = pm_ops->init(am33xx_do_sram_idle);
 	if (ret) {
 		dev_err(dev, "Unable to call core pm init!\n");
 		ret = -ENODEV;
@@ -343,6 +529,8 @@
 
 static int am33xx_pm_remove(struct platform_device *pdev)
 {
+	if (pm_ops->deinit)
+		pm_ops->deinit();
 	suspend_set_ops(NULL);
 	wkup_m3_ipc_put(m3_ipc);
 	am33xx_pm_free_sram();
diff -Naur --no-dereference a/drivers/soc/ti/pruss.c b/drivers/soc/ti/pruss.c
--- a/drivers/soc/ti/pruss.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/soc/ti/pruss.c	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,499 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * PRU-ICSS platform driver for various TI SoCs
+ *
+ * Copyright (C) 2014-2019 Texas Instruments Incorporated - http://www.ti.com/
+ *	Suman Anna <s-anna@ti.com>
+ *	Andrew F. Davis <afd@ti.com>
+ *	Tero Kristo <t-kristo@ti.com>
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/dma-mapping.h>
+#include <linux/io.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/pruss_driver.h>
+#include <linux/regmap.h>
+#include <linux/remoteproc.h>
+
+/**
+ * struct pruss_private_data - PRUSS driver private data
+ * @has_no_sharedram: flag to indicate the absence of PRUSS Shared Data RAM
+ */
+struct pruss_private_data {
+	bool has_no_sharedram;
+};
+
+/**
+ * struct pruss_match_private_data - private data to handle multiple instances
+ * @device_name: device name of the PRUSS instance
+ * @priv_data: PRUSS driver private data for this PRUSS instance
+ */
+struct pruss_match_private_data {
+	const char *device_name;
+	const struct pruss_private_data *priv_data;
+};
+
+/**
+ * pruss_get() - get the pruss for a given PRU remoteproc
+ * @rproc: remoteproc handle of a PRU instance
+ *
+ * Finds the parent pruss device for a PRU given the @rproc handle of the
+ * PRU remote processor. This function increments the pruss device's refcount,
+ * so always use pruss_put() to decrement it back once pruss isn't needed
+ * anymore.
+ *
+ * Returns the pruss handle on success, and an ERR_PTR on failure using one
+ * of the following error values
+ *    -EINVAL if invalid parameter
+ *    -ENODEV if PRU device or PRUSS device is not found
+ */
+struct pruss *pruss_get(struct rproc *rproc)
+{
+	struct pruss *pruss;
+	struct device *dev;
+	struct platform_device *ppdev;
+
+	if (IS_ERR_OR_NULL(rproc))
+		return ERR_PTR(-EINVAL);
+
+	dev = &rproc->dev;
+	if (!dev->parent)
+		return ERR_PTR(-ENODEV);
+
+	/* rudimentary check to make sure rproc handle is for a PRU or RTU */
+	if (!strstr(dev_name(dev->parent), "pru") &&
+	    !strstr(dev_name(dev->parent), "rtu"))
+		return ERR_PTR(-ENODEV);
+
+	ppdev = to_platform_device(dev->parent->parent);
+	pruss = platform_get_drvdata(ppdev);
+	if (!pruss)
+		return ERR_PTR(-ENODEV);
+
+	get_device(pruss->dev);
+
+	return pruss;
+}
+EXPORT_SYMBOL_GPL(pruss_get);
+
+/**
+ * pruss_put() - decrement pruss device's usecount
+ * @pruss: pruss handle
+ *
+ * Complimentary function for pruss_get(). Needs to be called
+ * after the PRUSS is used, and only if the pruss_get() succeeds.
+ */
+void pruss_put(struct pruss *pruss)
+{
+	if (IS_ERR_OR_NULL(pruss))
+		return;
+
+	put_device(pruss->dev);
+}
+EXPORT_SYMBOL_GPL(pruss_put);
+
+/**
+ * pruss_request_mem_region() - request a memory resource
+ * @pruss: the pruss instance
+ * @mem_id: the memory resource id
+ * @region: pointer to memory region structure to be filled in
+ *
+ * This function allows a client driver to request a memory resource,
+ * and if successful, will let the client driver own the particular
+ * memory region until released using the pruss_release_mem_region()
+ * API.
+ *
+ * Returns the memory region if requested resource is available, an
+ * error otherwise
+ */
+int pruss_request_mem_region(struct pruss *pruss, enum pruss_mem mem_id,
+			     struct pruss_mem_region *region)
+{
+	if (!pruss || !region)
+		return -EINVAL;
+
+	if (mem_id >= PRUSS_MEM_MAX)
+		return -EINVAL;
+
+	mutex_lock(&pruss->lock);
+
+	if (pruss->mem_in_use[mem_id]) {
+		mutex_unlock(&pruss->lock);
+		return -EBUSY;
+	}
+
+	*region = pruss->mem_regions[mem_id];
+	pruss->mem_in_use[mem_id] = region;
+
+	mutex_unlock(&pruss->lock);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(pruss_request_mem_region);
+
+/**
+ * pruss_release_mem_region() - release a memory resource
+ * @pruss: the pruss instance
+ * @region: the memory region to release
+ *
+ * This function is the complimentary function to
+ * pruss_request_mem_region(), and allows the client drivers to
+ * release back a memory resource.
+ *
+ * Returns 0 on success, an error code otherwise
+ */
+int pruss_release_mem_region(struct pruss *pruss,
+			     struct pruss_mem_region *region)
+{
+	int id;
+
+	if (!pruss || !region)
+		return -EINVAL;
+
+	mutex_lock(&pruss->lock);
+
+	/* find out the memory region being released */
+	for (id = 0; id < PRUSS_MEM_MAX; id++) {
+		if (pruss->mem_in_use[id] == region)
+			break;
+	}
+
+	if (id == PRUSS_MEM_MAX) {
+		mutex_unlock(&pruss->lock);
+		return -EINVAL;
+	}
+
+	pruss->mem_in_use[id] = NULL;
+
+	mutex_unlock(&pruss->lock);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(pruss_release_mem_region);
+
+/**
+ * pruss_cfg_read() - read a PRUSS CFG sub-module register
+ * @pruss: the pruss instance handle
+ * @reg: register offset within the CFG sub-module
+ * @val: pointer to return the value in
+ *
+ * Reads a given register within the PRUSS CFG sub-module and
+ * returns it through the passed-in @val pointer
+ *
+ * Returns 0 on success, or an error code otherwise
+ */
+int pruss_cfg_read(struct pruss *pruss, unsigned int reg, unsigned int *val)
+{
+	if (IS_ERR_OR_NULL(pruss))
+		return -EINVAL;
+
+	return regmap_read(pruss->cfg, reg, val);
+}
+EXPORT_SYMBOL_GPL(pruss_cfg_read);
+
+/**
+ * pruss_cfg_update() - configure a PRUSS CFG sub-module register
+ * @pruss: the pruss instance handle
+ * @reg: register offset within the CFG sub-module
+ * @mask: bit mask to use for programming the @val
+ * @val: value to write
+ *
+ * Programs a given register within the PRUSS CFG sub-module
+ *
+ * Returns 0 on success, or an error code otherwise
+ */
+int pruss_cfg_update(struct pruss *pruss, unsigned int reg,
+		     unsigned int mask, unsigned int val)
+{
+	if (IS_ERR_OR_NULL(pruss))
+		return -EINVAL;
+
+	return regmap_update_bits(pruss->cfg, reg, mask, val);
+}
+EXPORT_SYMBOL_GPL(pruss_cfg_update);
+
+static const
+struct pruss_private_data *pruss_get_private_data(struct platform_device *pdev)
+{
+	const struct pruss_match_private_data *data;
+
+	if (!of_device_is_compatible(pdev->dev.of_node, "ti,am4376-pruss"))
+		return NULL;
+
+	data = of_device_get_match_data(&pdev->dev);
+	for (; data && data->device_name; data++) {
+		if (!strcmp(dev_name(&pdev->dev), data->device_name))
+			return data->priv_data;
+	}
+
+	return ERR_PTR(-ENODEV);
+}
+
+static const struct regmap_config syscon_regmap_config = {
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = 4,
+};
+
+static void pruss_of_free_clk_provider(void *data)
+{
+	struct device_node *clk_mux_np = data;
+
+	of_clk_del_provider(clk_mux_np);
+	of_node_put(clk_mux_np);
+}
+
+static int pruss_clk_mux_setup(struct pruss *pruss, struct clk *clk_mux,
+			       char *mux_name, unsigned int reg_offset)
+{
+	unsigned int num_parents;
+	const char **parent_names;
+	void __iomem *reg;
+	int ret;
+	char *clk_mux_name = NULL;
+	struct device_node *clk_mux_np;
+
+	clk_mux_np = of_get_child_by_name(pruss->dev->of_node, mux_name);
+	if (!clk_mux_np)
+		return -EINVAL;
+
+	num_parents = of_clk_get_parent_count(clk_mux_np);
+	if (num_parents < 1) {
+		dev_err(pruss->dev, "mux-clock %pOF must have parents\n",
+			clk_mux_np);
+		return -EINVAL;
+	}
+
+	parent_names = devm_kcalloc(pruss->dev, sizeof(char *), num_parents,
+				    GFP_KERNEL);
+	if (!parent_names)
+		return -ENOMEM;
+
+	of_clk_parent_fill(clk_mux_np, parent_names, num_parents);
+
+	clk_mux_name = devm_kasprintf(pruss->dev, GFP_KERNEL, "%s.%pOFn",
+				      dev_name(pruss->dev), clk_mux_np);
+	if (!clk_mux_name)
+		return -ENOMEM;
+
+	reg = pruss->cfg_base + reg_offset;
+	/* WARN: dev must be NULL to avoid recursive incrementing
+	 * of module refcnt
+	 */
+	clk_mux = clk_register_mux(NULL, clk_mux_name,
+				   parent_names, num_parents,
+				   0, reg, 0, 1, 0, NULL);
+	if (IS_ERR(clk_mux))
+		return PTR_ERR(clk_mux);
+
+	ret = devm_add_action_or_reset(pruss->dev,
+				       (void(*)(void *))clk_unregister_mux,
+				       clk_mux);
+	if (ret) {
+		dev_err(pruss->dev, "failed to add clkmux reset action %d",
+			ret);
+		return ret;
+	}
+
+	ret = of_clk_add_provider(clk_mux_np, of_clk_src_simple_get,
+				  clk_mux);
+	if (ret)
+		return ret;
+
+	ret = devm_add_action_or_reset(pruss->dev,
+				       pruss_of_free_clk_provider,
+				       clk_mux_np);
+	if (ret)
+		dev_err(pruss->dev, "failed to add clkmux reset action %d",
+			ret);
+
+	return ret;
+}
+
+static int pruss_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *node = dev->of_node;
+	struct device_node *np;
+	struct pruss *pruss;
+	struct resource res;
+	int ret, i, index;
+	const struct pruss_private_data *data;
+	const char *mem_names[PRUSS_MEM_MAX] = { "dram0", "dram1", "shrdram2" };
+	struct regmap_config syscon_config = syscon_regmap_config;
+
+	if (!node) {
+		dev_err(dev, "Non-DT platform device not supported\n");
+		return -ENODEV;
+	}
+
+	data = pruss_get_private_data(pdev);
+	if (IS_ERR(data)) {
+		dev_err(dev, "missing private data\n");
+		return -ENODEV;
+	}
+
+	ret = dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
+	if (ret) {
+		dev_err(dev, "dma_set_coherent_mask: %d\n", ret);
+		return ret;
+	}
+
+	pruss = devm_kzalloc(dev, sizeof(*pruss), GFP_KERNEL);
+	if (!pruss)
+		return -ENOMEM;
+
+	pruss->dev = dev;
+	mutex_init(&pruss->lock);
+
+	np = of_get_child_by_name(node, "cfg");
+	if (!np) {
+		dev_err(dev, "%pOF is missing cfg node\n", np);
+		return -ENODEV;
+	}
+
+	if (of_address_to_resource(np, 0, &res))
+		return -ENOMEM;
+
+	pruss->cfg_base = devm_ioremap(dev, res.start, resource_size(&res));
+	if (!pruss->cfg_base)
+		return -ENOMEM;
+
+	if (!of_device_is_compatible(pdev->dev.of_node, "ti,am654-icssg") &&
+	    !of_device_is_compatible(pdev->dev.of_node, "ti,j721e-icssg"))
+		goto skip_mux;
+
+	ret = pruss_clk_mux_setup(pruss, pruss->core_clk_mux, "coreclk_mux",
+				  ICSSG_CFG_CORE_SYNC);
+	if (ret) {
+		dev_err(dev, "failed to setup coreclk_mux\n");
+		return ret;
+	}
+
+	ret = pruss_clk_mux_setup(pruss, pruss->core_clk_mux, "iepclk_mux",
+				  PRUSS_CFG_IEPCLK);
+	if (ret) {
+		dev_err(dev, "failed to setup iepclk_mux\n");
+		return ret;
+	}
+
+skip_mux:
+	syscon_config.name = of_node_full_name(np);
+	syscon_config.max_register = resource_size(&res) - 4;
+
+	pruss->cfg = regmap_init_mmio(NULL, pruss->cfg_base, &syscon_config);
+	if (IS_ERR(pruss->cfg)) {
+		dev_err(dev, "cfg regmap init failed\n");
+		return PTR_ERR(pruss->cfg);
+	}
+
+	np = of_get_child_by_name(node, "memories");
+	if (!np) {
+		dev_err(dev, "%pOF is missing memories node\n", np);
+		return -ENODEV;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(mem_names); i++) {
+		if (data && data->has_no_sharedram &&
+		    !strcmp(mem_names[i], "shrdram2"))
+			continue;
+
+		index = of_property_match_string(np, "reg-names", mem_names[i]);
+		if (index < 0) {
+			of_node_put(np);
+			return index;
+		}
+
+		if (of_address_to_resource(np, index, &res)) {
+			of_node_put(np);
+			return -EINVAL;
+		}
+
+		pruss->mem_regions[i].va = devm_ioremap(dev, res.start,
+							resource_size(&res));
+		if (!pruss->mem_regions[i].va) {
+			dev_err(dev, "failed to parse and map memory resource %d %s\n",
+				i, mem_names[i]);
+			of_node_put(np);
+			return -ENOMEM;
+		}
+		pruss->mem_regions[i].pa = res.start;
+		pruss->mem_regions[i].size = resource_size(&res);
+
+		dev_dbg(dev, "memory %8s: pa %pa size 0x%zx va %pK\n",
+			mem_names[i], &pruss->mem_regions[i].pa,
+			pruss->mem_regions[i].size, pruss->mem_regions[i].va);
+	}
+	of_node_put(np);
+
+	platform_set_drvdata(pdev, pruss);
+
+	dev_dbg(&pdev->dev, "creating PRU cores and other child platform devices\n");
+	ret = of_platform_populate(node, NULL, NULL, &pdev->dev);
+	if (ret)
+		dev_err(dev, "of_platform_populate failed\n");
+
+	return ret;
+}
+
+static int pruss_remove(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+
+	dev_dbg(dev, "remove PRU cores and other child platform devices\n");
+	of_platform_depopulate(dev);
+
+	return 0;
+}
+
+/* instance-specific driver private data */
+static const struct pruss_private_data am437x_pruss1_priv_data = {
+	.has_no_sharedram = false,
+};
+
+static const struct pruss_private_data am437x_pruss0_priv_data = {
+	.has_no_sharedram = true,
+};
+
+static const struct pruss_match_private_data am437x_match_data[] = {
+	{
+		.device_name	= "54400000.pruss",
+		.priv_data	= &am437x_pruss1_priv_data,
+	},
+	{
+		.device_name	= "54440000.pruss",
+		.priv_data	= &am437x_pruss0_priv_data,
+	},
+	{
+		/* sentinel */
+	},
+};
+
+static const struct of_device_id pruss_of_match[] = {
+	{ .compatible = "ti,am3356-pruss", .data = NULL, },
+	{ .compatible = "ti,am4376-pruss", .data = &am437x_match_data, },
+	{ .compatible = "ti,am5728-pruss", .data = NULL, },
+	{ .compatible = "ti,k2g-pruss", .data = NULL, },
+	{ .compatible = "ti,am654-icssg", .data = NULL, },
+	{ .compatible = "ti,j721e-icssg", .data = NULL, },
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, pruss_of_match);
+
+static struct platform_driver pruss_driver = {
+	.driver = {
+		.name = "pruss",
+		.of_match_table = pruss_of_match,
+	},
+	.probe  = pruss_probe,
+	.remove = pruss_remove,
+};
+module_platform_driver(pruss_driver);
+
+MODULE_AUTHOR("Suman Anna <s-anna@ti.com>");
+MODULE_DESCRIPTION("PRU-ICSS Subsystem Driver");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/soc/ti/pruss_soc_bus.c b/drivers/soc/ti/pruss_soc_bus.c
--- a/drivers/soc/ti/pruss_soc_bus.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/soc/ti/pruss_soc_bus.c	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,316 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * PRU-ICSS SoC bus driver for various TI SoCs
+ *
+ * Copyright (C) 2016-2019 Texas Instruments Incorporated - http://www.ti.com/
+ *	Suman Anna <s-anna@ti.com>
+ *	Keerthy <j-keerthy@ti.com>
+ */
+
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+
+#include <linux/platform_data/ti-pruss.h>
+
+#define SYSCFG_STANDBY_INIT	BIT(4)
+#define SYSCFG_SUB_MWAIT_READY	BIT(5)
+
+#define SYSCFG_STANDBY_MODE_FORCE	(0 << 2)
+#define SYSCFG_STANDBY_MODE_NO		(1 << 2)
+#define SYSCFG_STANDBY_MODE_SMART	(2 << 2)
+#define SYSCFG_STANDBY_MODE_MASK	(3 << 2)
+
+#define SYSCFG_IDLE_MODE_FORCE		0
+#define SYSCFG_IDLE_MODE_NO		1
+#define SYSCFG_IDLE_MODE_SMART		2
+#define SYSCFG_IDLE_MODE_MASK		3
+
+/**
+ * struct pruss_soc_bus - PRUSS SoC bus structure
+ * @syscfg: kernel mapped address for SYSCFG register
+ * @in_standby: flag for storing standby status
+ * @has_reset: cached variable for storing global module reset flag
+ * @skip_syscfg: flag to indicate if PRCM master standby/slave idle is needed
+ */
+struct pruss_soc_bus {
+	void __iomem *syscfg;
+	bool in_standby;
+	bool has_reset;
+	bool skip_syscfg;
+};
+
+/**
+ * struct pruss_soc_bus_match_data - PRUSS SoC bus driver match data
+ * @has_reset: flag to indicate the presence of global module reset
+ * @uses_prcm: flag to indicate the usage of PRCM master standby/slave idle
+ *	       protocol
+ */
+struct pruss_soc_bus_match_data {
+	bool has_reset;
+	bool uses_prcm;
+};
+
+static inline void pruss_soc_bus_rmw(void __iomem *reg, u32 mask, u32 set)
+{
+	u32 val;
+
+	val = readl_relaxed(reg);
+	val &= ~mask;
+	val |= (set & mask);
+	writel_relaxed(val, reg);
+}
+
+/*
+ * This function programs the PRUSS_SYSCFG.STANDBY_INIT bit to achieve dual
+ * functionalities - one is to deassert the MStandby signal to the device
+ * PRCM, and the other is to enable OCP master ports to allow accesses
+ * outside of the PRU-ICSS. The function has to wait for the PRCM to
+ * acknowledge through the monitoring of the PRUSS_SYSCFG.SUB_MWAIT bit.
+ */
+static int pruss_soc_bus_enable_ocp_master_ports(struct device *dev)
+{
+	struct pruss_soc_bus *psoc_bus = dev_get_drvdata(dev);
+	u32 syscfg_val, i;
+	bool ready = false;
+
+	pruss_soc_bus_rmw(psoc_bus->syscfg, SYSCFG_STANDBY_INIT, 0);
+
+	/* wait till we are ready for transactions - delay is arbitrary */
+	for (i = 0; i < 10; i++) {
+		syscfg_val = readl_relaxed(psoc_bus->syscfg);
+		ready = !(syscfg_val & SYSCFG_SUB_MWAIT_READY);
+		if (ready)
+			break;
+		udelay(5);
+	}
+
+	if (!ready) {
+		dev_err(dev, "timeout waiting for SUB_MWAIT_READY\n");
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+static int __maybe_unused pruss_soc_bus_suspend(struct device *dev)
+{
+	struct pruss_soc_bus *psoc_bus = dev_get_drvdata(dev);
+	u32 syscfg_val;
+
+	if (psoc_bus->skip_syscfg)
+		return 0;
+
+	syscfg_val = readl_relaxed(psoc_bus->syscfg);
+	psoc_bus->in_standby = syscfg_val & SYSCFG_STANDBY_INIT;
+
+	/* initiate MStandby, undo the MStandby config in probe */
+	if (!psoc_bus->in_standby) {
+		pruss_soc_bus_rmw(psoc_bus->syscfg, SYSCFG_STANDBY_INIT,
+				  SYSCFG_STANDBY_INIT);
+	}
+
+	return 0;
+}
+
+static int __maybe_unused pruss_soc_bus_resume(struct device *dev)
+{
+	struct pruss_soc_bus *psoc_bus = dev_get_drvdata(dev);
+	int ret = 0;
+
+	/* re-enable OCP master ports/disable MStandby */
+	if (!psoc_bus->skip_syscfg && !psoc_bus->in_standby) {
+		ret = pruss_soc_bus_enable_ocp_master_ports(dev);
+		if (ret)
+			dev_err(dev, "%s failed\n", __func__);
+	}
+
+	return ret;
+}
+
+/* firmware must be idle when calling this function */
+static void pruss_disable_module(struct device *dev)
+{
+	struct pruss_soc_bus *psoc_bus = dev_get_drvdata(dev);
+
+	if (psoc_bus->skip_syscfg)
+		goto put_sync;
+
+	/* configure Smart Standby */
+	pruss_soc_bus_rmw(psoc_bus->syscfg, SYSCFG_STANDBY_MODE_MASK,
+			  SYSCFG_STANDBY_MODE_SMART);
+
+	/* initiate MStandby */
+	pruss_soc_bus_rmw(psoc_bus->syscfg, SYSCFG_STANDBY_INIT,
+			  SYSCFG_STANDBY_INIT);
+
+put_sync:
+	/* initiate IDLE request, disable clocks */
+	pm_runtime_put_sync(dev);
+}
+
+static int pruss_enable_module(struct device *dev)
+{
+	struct pruss_soc_bus *psoc_bus = dev_get_drvdata(dev);
+	int ret;
+
+	/* enable clocks, de-assert IDLE request */
+	ret = pm_runtime_get_sync(dev);
+	if (ret < 0) {
+		pm_runtime_put_noidle(dev);
+		return ret;
+	}
+
+	if (psoc_bus->skip_syscfg)
+		return ret;
+
+	/* configure for Smart Idle & Smart Standby */
+	pruss_soc_bus_rmw(psoc_bus->syscfg, SYSCFG_IDLE_MODE_MASK,
+			  SYSCFG_IDLE_MODE_SMART);
+	pruss_soc_bus_rmw(psoc_bus->syscfg, SYSCFG_STANDBY_MODE_MASK,
+			  SYSCFG_STANDBY_MODE_SMART);
+
+	/* enable OCP master ports/disable MStandby */
+	ret = pruss_soc_bus_enable_ocp_master_ports(dev);
+	if (ret)
+		pruss_disable_module(dev);
+
+	return ret;
+}
+
+static int pruss_soc_bus_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *node = dev->of_node;
+	struct pruss_platform_data *pdata = dev_get_platdata(dev);
+	struct pruss_soc_bus *psoc_bus;
+	const struct pruss_soc_bus_match_data *data;
+	int ret;
+
+	psoc_bus = devm_kzalloc(dev, sizeof(*psoc_bus), GFP_KERNEL);
+	if (!psoc_bus)
+		return -ENOMEM;
+
+	psoc_bus->syscfg = of_iomap(node, 0);
+	if (!psoc_bus->syscfg)
+		return -ENOMEM;
+
+	data = of_device_get_match_data(dev);
+	if (!data) {
+		dev_err(dev, "missing match data\n");
+		return -ENODEV;
+	}
+
+	if (data->has_reset && (!pdata || !pdata->deassert_reset ||
+				!pdata->assert_reset || !pdata->reset_name)) {
+		dev_err(dev, "platform data (reset configuration information) missing\n");
+		return -ENODEV;
+	}
+	psoc_bus->has_reset = data->has_reset;
+	psoc_bus->skip_syscfg = !data->uses_prcm;
+	platform_set_drvdata(pdev, psoc_bus);
+
+	if (psoc_bus->has_reset) {
+		ret = pdata->deassert_reset(pdev, pdata->reset_name);
+		if (ret) {
+			dev_err(dev, "deassert_reset failed: %d\n", ret);
+			goto fail_reset;
+		}
+	}
+
+	pm_runtime_enable(dev);
+	ret = pruss_enable_module(dev);
+	if (ret < 0) {
+		dev_err(dev, "couldn't enable module\n");
+		goto fail_module;
+	}
+
+	ret = of_platform_populate(node, NULL, NULL, dev);
+	if (ret)
+		goto fail_of;
+
+	return 0;
+
+fail_of:
+	pruss_disable_module(dev);
+fail_module:
+	pm_runtime_disable(dev);
+	if (psoc_bus->has_reset)
+		pdata->assert_reset(pdev, pdata->reset_name);
+fail_reset:
+	iounmap(psoc_bus->syscfg);
+	return ret;
+}
+
+static int pruss_soc_bus_remove(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct pruss_platform_data *pdata = dev_get_platdata(dev);
+	struct pruss_soc_bus *psoc_bus = platform_get_drvdata(pdev);
+
+	of_platform_depopulate(dev);
+
+	pruss_disable_module(dev);
+	pm_runtime_disable(dev);
+
+	if (psoc_bus->has_reset)
+		pdata->assert_reset(pdev, pdata->reset_name);
+	iounmap(psoc_bus->syscfg);
+
+	return 0;
+}
+
+/* instance-specific driver private data */
+static const struct pruss_soc_bus_match_data am335x_data = {
+	.has_reset = true,
+	.uses_prcm = true,
+};
+
+static const struct pruss_soc_bus_match_data am437x_data = {
+	.has_reset = true,
+	.uses_prcm = true,
+};
+
+static const struct pruss_soc_bus_match_data am57xx_data = {
+	.has_reset = false,
+	.uses_prcm = true,
+};
+
+static const struct pruss_soc_bus_match_data k2g_data = {
+	.has_reset = false,
+	.uses_prcm = false,
+};
+
+static const struct of_device_id pruss_soc_bus_of_match[] = {
+	{ .compatible = "ti,am3356-pruss-soc-bus", .data = &am335x_data, },
+	{ .compatible = "ti,am4376-pruss-soc-bus", .data = &am437x_data, },
+	{ .compatible = "ti,am5728-pruss-soc-bus", .data = &am57xx_data, },
+	{ .compatible = "ti,k2g-pruss-soc-bus", .data = &k2g_data, },
+	{ .compatible = "ti,am654-icssg-soc-bus", .data = &k2g_data, },
+	{ .compatible = "ti,j721e-icssg-soc-bus", .data = &k2g_data, },
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, pruss_soc_bus_of_match);
+
+static SIMPLE_DEV_PM_OPS(pruss_soc_bus_pm_ops,
+			 pruss_soc_bus_suspend, pruss_soc_bus_resume);
+
+static struct platform_driver pruss_soc_bus_driver = {
+	.driver	= {
+		.name = "pruss-soc-bus",
+		.pm = &pruss_soc_bus_pm_ops,
+		.of_match_table = pruss_soc_bus_of_match,
+	},
+	.probe	= pruss_soc_bus_probe,
+	.remove	= pruss_soc_bus_remove,
+};
+module_platform_driver(pruss_soc_bus_driver);
+
+MODULE_AUTHOR("Suman Anna <s-anna@ti.com>");
+MODULE_AUTHOR("Keerthy <j-keerthy@ti.com>");
+MODULE_DESCRIPTION("PRU-ICSS SoC Bus Driver for TI SoCs");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/soc/ti/ti-pat.c b/drivers/soc/ti/ti-pat.c
--- a/drivers/soc/ti/ti-pat.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/soc/ti/ti-pat.c	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,629 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * TI PAT mapped DMA-BUF memory re-exporter
+ *
+ * Copyright (C) 2018-2019 Texas Instruments Incorporated - http://www.ti.com/
+ *	Andrew F. Davis <afd@ti.com>
+ */
+
+#include <linux/dma-buf.h>
+#include <linux/fs.h>
+#include <linux/genalloc.h>
+#include <linux/miscdevice.h>
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/platform_device.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include <linux/vmalloc.h>
+
+#include <linux/ti-pat.h>
+
+/* TI PAT MMRS registers */
+#define TI_PAT_MMRS_PID		0x0 /* Revision Register */
+#define TI_PAT_MMRS_CONFIG	0x4 /* Config Register */
+#define TI_PAT_MMRS_CONTROL	0x10 /* Control Register */
+
+/* TI PAT CONTROL register field values */
+#define TI_PAT_CONTROL_ARB_MODE_UF	0x0 /* Updates first */
+#define TI_PAT_CONTROL_ARB_MODE_RR	0x2 /* Round-robin */
+
+#define TI_PAT_CONTROL_PAGE_SIZE_4KB	0x0
+#define TI_PAT_CONTROL_PAGE_SIZE_16KB	0x1
+#define TI_PAT_CONTROL_PAGE_SIZE_64KB	0x2
+#define TI_PAT_CONTROL_PAGE_SIZE_1MB	0x3
+
+/* TI PAT TABLE registers */
+#define TI_PAT_TABLE_ADDRL	0x0 /* Low address Register */
+#define TI_PAT_TABLE_ADDRH	0x4 /* High address and enable Register */
+
+static unsigned int ti_pat_page_sizes[] = {
+	[TI_PAT_CONTROL_PAGE_SIZE_4KB]  = 4 * 1024,
+	[TI_PAT_CONTROL_PAGE_SIZE_16KB] = 16 * 1024,
+	[TI_PAT_CONTROL_PAGE_SIZE_64KB] = 64 * 1024,
+	[TI_PAT_CONTROL_PAGE_SIZE_1MB]  = 1024 * 1024,
+};
+
+enum ti_pat_fields {
+	/* Revision */
+	F_PID_MAJOR,
+	F_PID_MINOR,
+
+	/* Controls */
+	F_CONTROL_ARB_MODE,
+	F_CONTROL_PAGE_SIZE,
+	F_CONTROL_REPLACE_OID_EN,
+	F_CONTROL_EN,
+
+	/* sentinel */
+	F_MMRS_FIELDS,
+
+	/* Table */
+	F_TABLE_ADDRL,
+	F_TABLE_ADDRH,
+	F_TABLE_ENABLE,
+
+	/* sentinel */
+	F_MAX_FIELDS
+};
+
+static struct reg_field ti_pat_reg_fields[] = {
+	/* Revision */
+	[F_PID_MAJOR]			= REG_FIELD(TI_PAT_MMRS_PID, 8, 10),
+	[F_PID_MINOR]			= REG_FIELD(TI_PAT_MMRS_PID, 0, 5),
+	/* Controls */
+	[F_CONTROL_ARB_MODE]		= REG_FIELD(TI_PAT_MMRS_CONTROL, 6, 7),
+	[F_CONTROL_PAGE_SIZE]		= REG_FIELD(TI_PAT_MMRS_CONTROL, 4, 5),
+	[F_CONTROL_REPLACE_OID_EN]	= REG_FIELD(TI_PAT_MMRS_CONTROL, 1, 1),
+	[F_CONTROL_EN]			= REG_FIELD(TI_PAT_MMRS_CONTROL, 0, 0),
+	/* Table */
+	[F_TABLE_ADDRL]			= REG_FIELD(TI_PAT_TABLE_ADDRL, 0, 31),
+	[F_TABLE_ADDRH]			= REG_FIELD(TI_PAT_TABLE_ADDRH, 0, 3),
+	[F_TABLE_ENABLE]		= REG_FIELD(TI_PAT_TABLE_ADDRH, 31, 31),
+};
+
+/**
+ * struct ti_pat_data - PAT device instance data
+ * @dev: PAT device structure
+ * @mdev: misc device
+ * @mmrs_fields: Register fields for both MMRS and TABLE
+ * @page_count: Total number of pages in this PAT
+ * @page_size: Size of region mapped by each page in bytes
+ * @window_base: Base address of WINDOW region
+ * @pool: Pool for managing translation space
+ */
+struct ti_pat_data {
+	struct device *dev;
+	struct miscdevice mdev;
+	struct regmap_field *fields[F_MAX_FIELDS];
+	unsigned int page_count;
+	unsigned int page_size;
+	phys_addr_t window_base;
+	struct gen_pool *pool;
+};
+
+struct ti_pat_dma_buf_attachment {
+	struct device *dev;
+	struct sg_table *table;
+	struct ti_pat_buffer *buffer;
+	struct list_head list;
+};
+
+/**
+ * struct ti_pat_buffer - Single buffer instance data
+ * @pat: PAT instance for whom this buffer belongs
+ * @i_dma_buf: Imported DMA-BUF buffer
+ * @size: Total allocated size of this buffer
+ * @offset: Allocated offset into the PAT window
+ * @e_dma_buf: Exported DMA-BUF buffer
+ * @attachment: Our attachment to the imported buffer
+ * @sgt: DMA map of our imported buffer
+ * @attachments: Attachments to this buffer
+ * @map_count: Reference count of mappings to this buffer
+ * @lock: Protect the attach list and map count
+ */
+struct ti_pat_buffer {
+	struct ti_pat_data *pat;
+	struct dma_buf *i_dma_buf;
+	size_t size;
+	unsigned long offset;
+	struct dma_buf *e_dma_buf;
+
+	struct dma_buf_attachment *attachment;
+	struct sg_table *sgt;
+
+	struct list_head attachments;
+	int map_count;
+
+	struct mutex lock;
+};
+
+static const struct regmap_config ti_pat_regmap_config = {
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = 4,
+};
+
+static int ti_pat_dma_buf_attach(struct dma_buf *dmabuf,
+				 struct dma_buf_attachment *attachment)
+{
+	struct ti_pat_dma_buf_attachment *a;
+	struct ti_pat_buffer *buffer = dmabuf->priv;
+
+	a = kzalloc(sizeof(*a), GFP_KERNEL);
+	if (!a)
+		return -ENOMEM;
+
+	a->dev = attachment->dev;
+	a->buffer = buffer;
+	INIT_LIST_HEAD(&a->list);
+
+	a->table = kzalloc(sizeof(*a->table), GFP_KERNEL);
+	if (!a->table) {
+		kfree(a);
+		return -ENOMEM;
+	}
+
+	if (sg_alloc_table(a->table, 1, GFP_KERNEL)) {
+		kfree(a->table);
+		kfree(a);
+		return -ENOMEM;
+	}
+
+	sg_set_page(a->table->sgl, pfn_to_page(PFN_DOWN(buffer->offset)), buffer->size, 0);
+
+	attachment->priv = a;
+
+	mutex_lock(&buffer->lock);
+	/* First time attachment we attach to parent */
+	if (list_empty(&buffer->attachments)) {
+		buffer->attachment = dma_buf_attach(buffer->i_dma_buf, buffer->pat->dev);
+		if (IS_ERR(buffer->attachment)) {
+			dev_err(buffer->pat->dev, "Unable to attach to parent DMA-BUF\n");
+			mutex_unlock(&buffer->lock);
+			sg_free_table(a->table);
+			kfree(a->table);
+			kfree(a);
+			return PTR_ERR(buffer->attachment);
+		}
+	}
+	list_add(&a->list, &buffer->attachments);
+	mutex_unlock(&buffer->lock);
+
+	return 0;
+}
+
+static void ti_pat_dma_buf_detach(struct dma_buf *dmabuf,
+				  struct dma_buf_attachment *attachment)
+{
+	struct ti_pat_dma_buf_attachment *a = attachment->priv;
+	struct ti_pat_buffer *buffer = dmabuf->priv;
+
+	mutex_lock(&buffer->lock);
+	list_del(&a->list);
+	/* Last attachment we detach from parent */
+	if (list_empty(&buffer->attachments)) {
+		dma_buf_detach(buffer->i_dma_buf, buffer->attachment);
+		buffer->attachment = NULL;
+	}
+	mutex_unlock(&buffer->lock);
+
+	sg_free_table(a->table);
+	kfree(a->table);
+	kfree(a);
+}
+
+static unsigned int ti_pat_table_index_from_page(size_t page)
+{
+	/* Every 256 pages start on a 4k boundary */
+	unsigned int boundery = page / 256;
+	unsigned int offset = page % 256;
+	/* Each page occupies 8 bytes in the table */
+	return (boundery * (4096 / 8)) + offset;
+}
+
+static void ti_pat_set_page(struct ti_pat_data *pat, size_t page_id, dma_addr_t dma_address)
+{
+	unsigned int index = ti_pat_table_index_from_page(page_id);
+
+	/*
+	 * Addresses will always be at least 4K aligned, so both high and low
+	 * addresses are shifted by an additional 12 bits before being written
+	 * to the PAT.
+	 */
+	u32 base_l = dma_address >> 12;
+	u32 base_h = dma_address >> 44;
+
+	dev_dbg(pat->dev, "Enabling PAT index: %zu pointing to %pad\n", page_id, &dma_address);
+
+	regmap_fields_write(pat->fields[F_TABLE_ADDRL], index, base_l);
+	regmap_fields_write(pat->fields[F_TABLE_ADDRH], index, base_h);
+	regmap_fields_write(pat->fields[F_TABLE_ENABLE], index, 1);
+}
+
+static void ti_pat_unset_page(struct ti_pat_data *pat, size_t page_id)
+{
+	unsigned int index = ti_pat_table_index_from_page(page_id);
+
+	dev_dbg(pat->dev, "Disabling PAT index: %zu\n", page_id);
+
+	regmap_fields_write(pat->fields[F_TABLE_ENABLE], index, 0);
+}
+
+static struct sg_table *ti_pat_map_dma_buf(struct dma_buf_attachment *attachment,
+					   enum dma_data_direction direction)
+{
+	struct ti_pat_dma_buf_attachment *a = attachment->priv;
+	struct ti_pat_buffer *buffer = a->buffer;
+	struct ti_pat_data *pat = buffer->pat;
+	struct sg_table *table = a->table;
+	struct scatterlist *s;
+	unsigned int i, s_len;
+	size_t page_id;
+	int ret;
+
+	mutex_lock(&buffer->lock);
+	/* First time mapping we map to parent */
+	if (!buffer->map_count) {
+		buffer->sgt = dma_buf_map_attachment(buffer->attachment, DMA_BIDIRECTIONAL);
+		if (IS_ERR(buffer->sgt)) {
+			dev_err(pat->dev, "Unable to map parent DMA-BUF\n");
+			return buffer->sgt;
+		}
+
+		/* And program PAT area for this set of pages */
+		page_id = (buffer->offset - pat->window_base) / pat->page_size;
+		for_each_sg(buffer->sgt->sgl, s, buffer->sgt->nents, i) {
+			if (s->offset) {
+				dev_err(pat->dev, "Cannot use offset buffers\n");
+				ret = -EINVAL;
+				goto unmap;
+			}
+
+			if (s->length % pat->page_size) {
+				dev_err(pat->dev, "Cannot use buffers not a multiple of page size\n");
+				ret = -EINVAL;
+				goto unmap;
+			}
+
+			for (s_len = 0; s_len < s->length; s_len += pat->page_size)
+				ti_pat_set_page(pat, page_id++, s->dma_address + s_len);
+		}
+	}
+	buffer->map_count++;
+	mutex_unlock(&buffer->lock);
+
+	/* Map the attached device's table to get DMA addresses */
+	if (!dma_map_sg_attrs(attachment->dev, table->sgl, table->nents, direction, DMA_ATTR_SKIP_CPU_SYNC))
+		return ERR_PTR(-ENOMEM);
+
+	return table;
+
+unmap:
+	dma_buf_unmap_attachment(buffer->attachment, buffer->sgt, DMA_BIDIRECTIONAL);
+	mutex_unlock(&buffer->lock);
+	return ERR_PTR(ret);
+}
+
+static void ti_pat_unmap_dma_buf(struct dma_buf_attachment *attachment,
+				 struct sg_table *table,
+				 enum dma_data_direction direction)
+{
+	struct ti_pat_dma_buf_attachment *a = attachment->priv;
+	struct ti_pat_buffer *buffer = a->buffer;
+	struct ti_pat_data *pat = buffer->pat;
+
+	/* Unmap the attached device's table */
+	dma_unmap_sg_attrs(attachment->dev, table->sgl, table->nents, direction, DMA_ATTR_SKIP_CPU_SYNC);
+
+	mutex_lock(&buffer->lock);
+	buffer->map_count--;
+	/* Last mapping we unmap from parent */
+	if (!buffer->map_count) {
+		/* Disable PAT pages for this area */
+		size_t page_start = (buffer->offset - pat->window_base) / pat->page_size;
+		size_t page_end = page_start + (buffer->size / pat->page_size);
+
+		for (; page_start < page_end; page_start++)
+			ti_pat_unset_page(pat, page_start);
+
+		dma_buf_unmap_attachment(buffer->attachment, buffer->sgt, DMA_BIDIRECTIONAL);
+		buffer->sgt = NULL;
+	}
+	mutex_unlock(&buffer->lock);
+}
+
+static void ti_pat_dma_buf_release(struct dma_buf *dmabuf)
+{
+	struct ti_pat_buffer *buffer = dmabuf->priv;
+
+	if (buffer->attachment && buffer->sgt)
+		dma_buf_unmap_attachment(buffer->attachment, buffer->sgt, DMA_BIDIRECTIONAL);
+	if (buffer->i_dma_buf && !IS_ERR_OR_NULL(buffer->attachment))
+		dma_buf_detach(buffer->i_dma_buf, buffer->attachment);
+	if (buffer->i_dma_buf)
+		dma_buf_put(buffer->i_dma_buf);
+
+	if (buffer->offset)
+		gen_pool_free(buffer->pat->pool, buffer->offset, buffer->size);
+
+	kfree(buffer);
+}
+
+static const struct dma_buf_ops dma_buf_ops = {
+	.attach = ti_pat_dma_buf_attach,
+	.detach = ti_pat_dma_buf_detach,
+
+	.map_dma_buf = ti_pat_map_dma_buf,
+	.unmap_dma_buf = ti_pat_unmap_dma_buf,
+
+	.release = ti_pat_dma_buf_release,
+};
+
+int ti_pat_export(struct ti_pat_data *pat,
+		  struct dma_buf *i_dma_buf,
+		  struct dma_buf **e_dma_buf)
+{
+	struct ti_pat_buffer *buffer;
+	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
+	int ret;
+
+	buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
+	if (!buffer)
+		return -ENOMEM;
+
+	buffer->pat = pat;
+	buffer->i_dma_buf = i_dma_buf;
+	buffer->size = buffer->i_dma_buf->size;
+	mutex_init(&buffer->lock);
+	INIT_LIST_HEAD(&buffer->attachments);
+	buffer->map_count = 0;
+
+	/* Reserve PAT space */
+	buffer->offset = gen_pool_alloc(buffer->pat->pool, buffer->size);
+	if (!buffer->offset) {
+		ret = -ENOMEM;
+		goto free_buffer;
+	}
+
+	exp_info.ops = &dma_buf_ops;
+	exp_info.size = buffer->size;
+	exp_info.flags = O_RDWR;
+	exp_info.priv = buffer;
+
+	*e_dma_buf = dma_buf_export(&exp_info);
+	if (IS_ERR(*e_dma_buf)) {
+		ret = PTR_ERR(*e_dma_buf);
+		goto free_pool;
+	}
+
+	return 0;
+
+free_pool:
+	gen_pool_free(buffer->pat->pool, buffer->offset, buffer->size);
+free_buffer:
+	kfree(buffer);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(ti_pat_export);
+
+static long ti_pat_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	struct ti_pat_data *pat = container_of(file->private_data, struct ti_pat_data, mdev);
+
+	switch (cmd) {
+	case TI_PAT_IOC_EXPORT:
+	{
+		struct ti_pat_export_data export;
+		struct dma_buf *i_dma_buf;
+		struct dma_buf *e_dma_buf;
+		int ret;
+
+		if (_IOC_SIZE(cmd) > sizeof(export))
+			return -EINVAL;
+
+		if (copy_from_user(&export, (void __user *)arg, _IOC_SIZE(cmd)))
+			return -EFAULT;
+
+		i_dma_buf = dma_buf_get(export.fd);
+		if (IS_ERR(i_dma_buf))
+			return PTR_ERR(i_dma_buf);
+
+		ret = ti_pat_export(pat, i_dma_buf, &e_dma_buf);
+		if (ret) {
+			dma_buf_put(i_dma_buf);
+			return ret;
+		}
+
+		export.fd = dma_buf_fd(e_dma_buf, O_CLOEXEC);
+		if (export.fd < 0) {
+			dma_buf_put(e_dma_buf);
+			dma_buf_put(i_dma_buf);
+			return export.fd;
+		}
+
+		if (copy_to_user((void __user *)arg, &export, _IOC_SIZE(cmd)))
+			return -EFAULT;
+
+		break;
+	}
+	default:
+		return -ENOTTY;
+	}
+
+	return 0;
+}
+
+static const struct file_operations ti_pat_fops = {
+	.owner          = THIS_MODULE,
+	.unlocked_ioctl = ti_pat_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl	= ti_pat_ioctl,
+#endif
+};
+
+static const struct of_device_id ti_pat_of_match[] = {
+	{ .compatible = "ti,j721e-pat", },
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, ti_pat_of_match);
+
+static int ti_pat_probe(struct platform_device *pdev)
+{
+	struct ti_pat_data *pat;
+	struct resource *res;
+	void __iomem *base;
+	struct regmap *mmrs_map;
+	struct regmap *table_map;
+	unsigned int revision_major;
+	unsigned int revision_minor;
+	resource_size_t size;
+	size_t page_size;
+	int i, ret;
+
+	pat = devm_kzalloc(&pdev->dev, sizeof(*pat), GFP_KERNEL);
+	if (!pat)
+		return -ENOMEM;
+	platform_set_drvdata(pdev, pat);
+	pat->dev = &pdev->dev;
+
+	/* Set DMA mask to 64 bits */
+	ret = dma_set_mask_and_coherent(pat->dev, DMA_BIT_MASK(64));
+	if (ret) {
+		dev_err(pat->dev, "Unable to set coherent mask to 64");
+		return ret;
+	}
+
+	/* MMRS */
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mmrs");
+	if (!res) {
+		dev_err(pat->dev, "Unable to find MMRS IO resource\n");
+		return -ENOENT;
+	}
+	base = devm_ioremap_resource(pat->dev, res);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	mmrs_map = devm_regmap_init_mmio(pat->dev, base, &ti_pat_regmap_config);
+	if (IS_ERR(mmrs_map)) {
+		dev_err(pat->dev, "Unable to allocate MMRS register map\n");
+		return PTR_ERR(mmrs_map);
+	}
+
+	for (i = 0; i < F_MMRS_FIELDS; i++) {
+		pat->fields[i] = devm_regmap_field_alloc(pat->dev, mmrs_map, ti_pat_reg_fields[i]);
+		if (IS_ERR(pat->fields[i])) {
+			dev_err(pat->dev, "Unable to allocate Regmap fields\n");
+			return PTR_ERR(pat->fields[i]);
+		}
+	}
+
+	ret = regmap_read(mmrs_map, TI_PAT_MMRS_CONFIG, &pat->page_count);
+	if (ret) {
+		dev_err(pat->dev, "Unable to read device page count\n");
+		return ret;
+	}
+
+	ret = regmap_field_read(pat->fields[F_PID_MAJOR], &revision_major);
+	if (ret) {
+		dev_err(pat->dev, "Unable to read device major revision\n");
+		return ret;
+	}
+
+	ret = regmap_field_read(pat->fields[F_PID_MINOR], &revision_minor);
+	if (ret) {
+		dev_err(pat->dev, "Unable to read device minor revision\n");
+		return ret;
+	}
+
+	dev_info(pat->dev, "Found PAT Rev %d.%d with %d pages\n", revision_major, revision_minor, pat->page_count);
+
+	/* TABLE */
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "table");
+	if (!res) {
+		dev_err(pat->dev, "Unable to find TABLE IO resource\n");
+		return -ENOENT;
+	}
+	base = devm_ioremap_resource(pat->dev, res);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	/* 256 pages per 4KB of table space */
+	size = resource_size(res);
+	if (size != (pat->page_count << 4))
+		dev_warn(pat->dev, "TABLE region size (%llu) does not match reported page count\n", size);
+
+	table_map = devm_regmap_init_mmio(pat->dev, base, &ti_pat_regmap_config);
+	if (IS_ERR(table_map)) {
+		dev_err(pat->dev, "Unable to allocate TABLE register map\n");
+		return PTR_ERR(table_map);
+	}
+
+	for (i = F_MMRS_FIELDS + 1; i < F_MAX_FIELDS; i++) {
+		ti_pat_reg_fields[i].id_size = ti_pat_table_index_from_page(pat->page_count);
+		ti_pat_reg_fields[i].id_offset = 8; /* 8 bytes per entry */
+		pat->fields[i] = devm_regmap_field_alloc(pat->dev, table_map, ti_pat_reg_fields[i]);
+		if (IS_ERR(pat->fields[i])) {
+			dev_err(pat->dev, "Unable to allocate Regmap fields\n");
+			return PTR_ERR(pat->fields[i]);
+		}
+	}
+
+	/* WINDOW */
+	ret = device_property_read_u64(pat->dev, "ti,pat-window-base", &pat->window_base);
+	if (ret) {
+		dev_err(pat->dev, "Unable to find ti,pat-window-base\n");
+		return -ENOENT;
+	}
+
+	ret = device_property_read_u64(pat->dev, "ti,pat-window-size", &size);
+	if (ret) {
+		dev_err(pat->dev, "Unable to find ti,pat-window-size\n");
+		return -ENOENT;
+	}
+
+	pat->page_size = PAGE_SIZE;
+	for (page_size = 0; page_size < ARRAY_SIZE(ti_pat_page_sizes); page_size++)
+		if (ti_pat_page_sizes[page_size] == pat->page_size)
+			break;
+	if (page_size == ARRAY_SIZE(ti_pat_page_sizes)) {
+		dev_err(pat->dev, "Unsupported PAGE_SIZE (%d)\n", pat->page_size);
+		return -EINVAL;
+	}
+	regmap_field_write(pat->fields[F_CONTROL_PAGE_SIZE], page_size);
+
+	/* Enable this PAT module */
+	regmap_field_write(pat->fields[F_CONTROL_EN], 1);
+
+	pat->pool = gen_pool_create(PAGE_SHIFT, -1);
+	if (!pat->pool)
+		return -ENOMEM;
+	gen_pool_add(pat->pool, pat->window_base, size, -1);
+
+	pat->mdev.minor = MISC_DYNAMIC_MINOR;
+	pat->mdev.name = pdev->name;
+	pat->mdev.fops = &ti_pat_fops;
+	pat->mdev.parent = NULL;
+	ret = misc_register(&pat->mdev);
+	if (ret) {
+		dev_err(pat->dev, "Unable to register misc device\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static struct platform_driver ti_pat_driver = {
+	.probe = ti_pat_probe,
+	.driver = {
+		.name = "ti-pat",
+		.of_match_table = ti_pat_of_match,
+	},
+};
+module_platform_driver(ti_pat_driver);
+
+MODULE_AUTHOR("Andrew F. Davis <afd@ti.com>");
+MODULE_DESCRIPTION("TI PAT mapped DMA-BUF memory exporter");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/soc/ti/ti_sci_pm_domains.c b/drivers/soc/ti/ti_sci_pm_domains.c
--- a/drivers/soc/ti/ti_sci_pm_domains.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/soc/ti/ti_sci_pm_domains.c	2020-06-19 21:00:31.424227716 -0400
@@ -1,156 +1,108 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * TI SCI Generic Power Domain Driver
  *
  * Copyright (C) 2015-2017 Texas Instruments Incorporated - http://www.ti.com/
  *	J Keerthy <j-keerthy@ti.com>
  *	Dave Gerlach <d-gerlach@ti.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <linux/err.h>
 #include <linux/module.h>
-#include <linux/mutex.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/pm_domain.h>
 #include <linux/slab.h>
 #include <linux/soc/ti/ti_sci_protocol.h>
+#include <dt-bindings/soc/ti,sci_pm_domain.h>
 
 /**
- * struct ti_sci_genpd_dev_data: holds data needed for every device attached
- *				 to this genpd
- * @idx: index of the device that identifies it with the system
- *	 control processor.
+ * struct ti_sci_genpd_provider: holds common TI SCI genpd provider data
+ * @ti_sci: handle to TI SCI protocol driver that provides ops to
+ *	    communicate with system control processor.
+ * @dev: pointer to dev for the driver for devm allocs
+ * @pd_list: list of all the power domains on the device
+ * @data: onecell data for genpd core
  */
-struct ti_sci_genpd_dev_data {
-	int idx;
+struct ti_sci_genpd_provider {
+	const struct ti_sci_handle *ti_sci;
+	struct device *dev;
+	struct list_head pd_list;
+	struct genpd_onecell_data data;
 };
 
 /**
  * struct ti_sci_pm_domain: TI specific data needed for power domain
- * @ti_sci: handle to TI SCI protocol driver that provides ops to
- *	    communicate with system control processor.
- * @dev: pointer to dev for the driver for devm allocs
+ * @idx: index of the device that identifies it with the system
+ *	 control processor.
+ * @exclusive: Permissions for exclusive request or shared request of the
+ *	       device.
  * @pd: generic_pm_domain for use with the genpd framework
+ * @node: link for the genpd list
+ * @parent: link to the parent TI SCI genpd provider
  */
 struct ti_sci_pm_domain {
-	const struct ti_sci_handle *ti_sci;
-	struct device *dev;
+	int idx;
+	u8 exclusive;
 	struct generic_pm_domain pd;
+	struct list_head node;
+	struct ti_sci_genpd_provider *parent;
 };
 
 #define genpd_to_ti_sci_pd(gpd) container_of(gpd, struct ti_sci_pm_domain, pd)
 
-/**
- * ti_sci_dev_id(): get prepopulated ti_sci id from struct dev
- * @dev: pointer to device associated with this genpd
- *
- * Returns device_id stored from ti,sci_id property
- */
-static int ti_sci_dev_id(struct device *dev)
-{
-	struct generic_pm_domain_data *genpd_data = dev_gpd_data(dev);
-	struct ti_sci_genpd_dev_data *sci_dev_data = genpd_data->data;
-
-	return sci_dev_data->idx;
-}
-
-/**
- * ti_sci_dev_to_sci_handle(): get pointer to ti_sci_handle
- * @dev: pointer to device associated with this genpd
- *
- * Returns ti_sci_handle to be used to communicate with system
- *	   control processor.
+/*
+ * ti_sci_pd_power_off(): genpd power down hook
+ * @domain: pointer to the powerdomain to power off
  */
-static const struct ti_sci_handle *ti_sci_dev_to_sci_handle(struct device *dev)
+static int ti_sci_pd_power_off(struct generic_pm_domain *domain)
 {
-	struct generic_pm_domain *pd = pd_to_genpd(dev->pm_domain);
-	struct ti_sci_pm_domain *ti_sci_genpd = genpd_to_ti_sci_pd(pd);
+	struct ti_sci_pm_domain *pd = genpd_to_ti_sci_pd(domain);
+	const struct ti_sci_handle *ti_sci = pd->parent->ti_sci;
 
-	return ti_sci_genpd->ti_sci;
+	return ti_sci->ops.dev_ops.put_device(ti_sci, pd->idx);
 }
 
-/**
- * ti_sci_dev_start(): genpd device start hook called to turn device on
- * @dev: pointer to device associated with this genpd to be powered on
+/*
+ * ti_sci_pd_power_on(): genpd power up hook
+ * @domain: pointer to the powerdomain to power on
  */
-static int ti_sci_dev_start(struct device *dev)
+static int ti_sci_pd_power_on(struct generic_pm_domain *domain)
 {
-	const struct ti_sci_handle *ti_sci = ti_sci_dev_to_sci_handle(dev);
-	int idx = ti_sci_dev_id(dev);
+	struct ti_sci_pm_domain *pd = genpd_to_ti_sci_pd(domain);
+	const struct ti_sci_handle *ti_sci = pd->parent->ti_sci;
 
-	return ti_sci->ops.dev_ops.get_device(ti_sci, idx);
+	if (pd->exclusive)
+		return ti_sci->ops.dev_ops.get_device_exclusive(ti_sci,
+								pd->idx);
+	else
+		return ti_sci->ops.dev_ops.get_device(ti_sci, pd->idx);
 }
 
-/**
- * ti_sci_dev_stop(): genpd device stop hook called to turn device off
- * @dev: pointer to device associated with this genpd to be powered off
+/*
+ * ti_sci_pd_xlate(): translation service for TI SCI genpds
+ * @genpdspec: DT identification data for the genpd
+ * @data: genpd core data for all the powerdomains on the device
  */
-static int ti_sci_dev_stop(struct device *dev)
-{
-	const struct ti_sci_handle *ti_sci = ti_sci_dev_to_sci_handle(dev);
-	int idx = ti_sci_dev_id(dev);
-
-	return ti_sci->ops.dev_ops.put_device(ti_sci, idx);
-}
-
-static int ti_sci_pd_attach_dev(struct generic_pm_domain *domain,
-				struct device *dev)
+static struct generic_pm_domain *ti_sci_pd_xlate(
+					struct of_phandle_args *genpdspec,
+					void *data)
 {
-	struct device_node *np = dev->of_node;
-	struct of_phandle_args pd_args;
-	struct ti_sci_pm_domain *ti_sci_genpd = genpd_to_ti_sci_pd(domain);
-	const struct ti_sci_handle *ti_sci = ti_sci_genpd->ti_sci;
-	struct ti_sci_genpd_dev_data *sci_dev_data;
-	struct generic_pm_domain_data *genpd_data;
-	int idx, ret = 0;
-
-	ret = of_parse_phandle_with_args(np, "power-domains",
-					 "#power-domain-cells", 0, &pd_args);
-	if (ret < 0)
-		return ret;
-
-	if (pd_args.args_count != 1)
-		return -EINVAL;
-
-	idx = pd_args.args[0];
-
-	/*
-	 * Check the validity of the requested idx, if the index is not valid
-	 * the PMMC will return a NAK here and we will not allocate it.
-	 */
-	ret = ti_sci->ops.dev_ops.is_valid(ti_sci, idx);
-	if (ret)
-		return -EINVAL;
+	struct genpd_onecell_data *genpd_data = data;
+	unsigned int idx = genpdspec->args[0];
 
-	sci_dev_data = kzalloc(sizeof(*sci_dev_data), GFP_KERNEL);
-	if (!sci_dev_data)
-		return -ENOMEM;
+	if (genpdspec->args_count < 1)
+		return ERR_PTR(-EINVAL);
 
-	sci_dev_data->idx = idx;
+	if (idx >= genpd_data->num_domains) {
+		pr_err("%s: invalid domain index %u\n", __func__, idx);
+		return ERR_PTR(-EINVAL);
+	}
 
-	genpd_data = dev_gpd_data(dev);
-	genpd_data->data = sci_dev_data;
+	if (!genpd_data->domains[idx])
+		return ERR_PTR(-ENOENT);
 
-	return 0;
-}
-
-static void ti_sci_pd_detach_dev(struct generic_pm_domain *domain,
-				 struct device *dev)
-{
-	struct generic_pm_domain_data *genpd_data = dev_gpd_data(dev);
-	struct ti_sci_genpd_dev_data *sci_dev_data = genpd_data->data;
-
-	kfree(sci_dev_data);
-	genpd_data->data = NULL;
+	return genpd_data->domains[idx];
 }
 
 static const struct of_device_id ti_sci_pm_domain_matches[] = {
@@ -162,33 +114,80 @@
 static int ti_sci_pm_domain_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	struct device_node *np = dev->of_node;
-	struct ti_sci_pm_domain *ti_sci_pd;
+	struct ti_sci_genpd_provider *pd_provider;
+	struct ti_sci_pm_domain *pd;
+	struct device_node *np = NULL;
+	struct of_phandle_args args;
 	int ret;
+	u32 max_id = 0;
+	int index;
 
-	ti_sci_pd = devm_kzalloc(dev, sizeof(*ti_sci_pd), GFP_KERNEL);
-	if (!ti_sci_pd)
+	pd_provider = devm_kzalloc(dev, sizeof(*pd_provider), GFP_KERNEL);
+	if (!pd_provider)
 		return -ENOMEM;
 
-	ti_sci_pd->ti_sci = devm_ti_sci_get_handle(dev);
-	if (IS_ERR(ti_sci_pd->ti_sci))
-		return PTR_ERR(ti_sci_pd->ti_sci);
-
-	ti_sci_pd->dev = dev;
-
-	ti_sci_pd->pd.name = "ti_sci_pd";
-
-	ti_sci_pd->pd.attach_dev = ti_sci_pd_attach_dev;
-	ti_sci_pd->pd.detach_dev = ti_sci_pd_detach_dev;
-
-	ti_sci_pd->pd.dev_ops.start = ti_sci_dev_start;
-	ti_sci_pd->pd.dev_ops.stop = ti_sci_dev_stop;
+	pd_provider->ti_sci = devm_ti_sci_get_handle(dev);
+	if (IS_ERR(pd_provider->ti_sci))
+		return PTR_ERR(pd_provider->ti_sci);
+
+	pd_provider->dev = dev;
+
+	INIT_LIST_HEAD(&pd_provider->pd_list);
+
+	/* Find highest device ID used for power domains */
+	while (1) {
+		np = of_find_node_with_property(np, "power-domains");
+		if (!np)
+			break;
+
+		index = 0;
+
+		while (1) {
+			ret = of_parse_phandle_with_args(np, "power-domains",
+							 "#power-domain-cells",
+							 index, &args);
+			if (ret)
+				break;
+
+			if (args.args_count >= 1 && args.np == dev->of_node) {
+				if (args.args[0] > max_id)
+					max_id = args.args[0];
+
+				pd = devm_kzalloc(dev, sizeof(*pd), GFP_KERNEL);
+				if (!pd)
+					return -ENOMEM;
+
+				pd->pd.name = devm_kasprintf(dev, GFP_KERNEL,
+							     "pd:%d",
+							     args.args[0]);
+				if (!pd->pd.name)
+					return -ENOMEM;
+
+				pd->pd.power_off = ti_sci_pd_power_off;
+				pd->pd.power_on = ti_sci_pd_power_on;
+				pd->idx = args.args[0];
+				pd->parent = pd_provider;
+
+				pm_genpd_init(&pd->pd, NULL, true);
+
+				list_add(&pd->node, &pd_provider->pd_list);
+			}
+			index++;
+		}
+	}
+
+	pd_provider->data.domains =
+		devm_kcalloc(dev, max_id + 1,
+			     sizeof(*pd_provider->data.domains),
+			     GFP_KERNEL);
 
-	pm_genpd_init(&ti_sci_pd->pd, NULL, true);
+	pd_provider->data.num_domains = max_id + 1;
+	pd_provider->data.xlate = ti_sci_pd_xlate;
 
-	ret = of_genpd_add_provider_simple(np, &ti_sci_pd->pd);
+	list_for_each_entry(pd, &pd_provider->pd_list, node)
+		pd_provider->data.domains[pd->idx] = &pd->pd;
 
-	return ret;
+	return of_genpd_add_provider_onecell(dev->of_node, &pd_provider->data);
 }
 
 static struct platform_driver ti_sci_pm_domains_driver = {
diff -Naur --no-dereference a/drivers/soc/ti/wkup_m3_ipc.c b/drivers/soc/ti/wkup_m3_ipc.c
--- a/drivers/soc/ti/wkup_m3_ipc.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/soc/ti/wkup_m3_ipc.c	2020-06-19 21:00:31.424227716 -0400
@@ -15,7 +15,9 @@
  * GNU General Public License for more details.
  */
 
+#include <linux/debugfs.h>
 #include <linux/err.h>
+#include <linux/firmware.h>
 #include <linux/kernel.h>
 #include <linux/kthread.h>
 #include <linux/interrupt.h>
@@ -48,15 +50,34 @@
 #define M3_FW_VERSION_MASK		0xffff
 #define M3_WAKE_SRC_MASK		0xff
 
+#define IPC_MEM_TYPE_SHIFT		(0x0)
+#define IPC_MEM_TYPE_MASK		(0x7 << 0)
+#define IPC_VTT_STAT_SHIFT		(0x3)
+#define IPC_VTT_STAT_MASK		(0x1 << 3)
+#define IPC_VTT_GPIO_PIN_SHIFT		(0x4)
+#define IPC_VTT_GPIO_PIN_MASK		(0x3f << 4)
+#define IPC_IO_ISOLATION_STAT_SHIFT	(10)
+#define IPC_IO_ISOLATION_STAT_MASK	(0x1 << 10)
+
+#define IPC_DBG_HALT_SHIFT		(11)
+#define IPC_DBG_HALT_MASK		(0x1 << 11)
+
 #define M3_STATE_UNKNOWN		0
 #define M3_STATE_RESET			1
 #define M3_STATE_INITED			2
 #define M3_STATE_MSG_FOR_LP		3
 #define M3_STATE_MSG_FOR_RESET		4
 
+#define WKUP_M3_SD_FW_MAGIC		0x570C
+
+#define WKUP_M3_DMEM_START		0x80000
+#define WKUP_M3_AUXDATA_OFFSET		0x1000
+#define WKUP_M3_AUXDATA_SIZE		0xFF
+
 static struct wkup_m3_ipc *m3_ipc_state;
 
 static const struct wkup_m3_wakeup_src wakeups[] = {
+	{.irq_nr = 16,	.src = "PRCM"},
 	{.irq_nr = 35,	.src = "USB0_PHY"},
 	{.irq_nr = 36,	.src = "USB1_PHY"},
 	{.irq_nr = 40,	.src = "I2C0"},
@@ -73,6 +94,146 @@
 	{.irq_nr = 0,	.src = "Unknown"},
 };
 
+/**
+ * wkup_m3_copy_aux_data - Copy auxiliary data to special region of m3 dmem
+ * @data - pointer to data
+ * @sz - size of data to copy (limit 256 bytes)
+ *
+ * Copies any additional blob of data to the wkup_m3 dmem to be used by the
+ * firmware
+ */
+static unsigned long wkup_m3_copy_aux_data(struct wkup_m3_ipc *m3_ipc,
+					   const void *data, int sz)
+{
+	unsigned long aux_data_dev_addr;
+	void *aux_data_addr;
+
+	aux_data_dev_addr = WKUP_M3_DMEM_START + WKUP_M3_AUXDATA_OFFSET;
+	aux_data_addr = rproc_da_to_va(m3_ipc->rproc, aux_data_dev_addr,
+				       WKUP_M3_AUXDATA_SIZE, RPROC_FLAGS_NONE);
+	memcpy(aux_data_addr, data, sz);
+
+	return WKUP_M3_AUXDATA_OFFSET;
+}
+
+static void wkup_m3_scale_data_fw_cb(const struct firmware *fw, void *context)
+{
+	unsigned long val, aux_base;
+	struct wkup_m3_scale_data_header hdr;
+	struct wkup_m3_ipc *m3_ipc = context;
+	struct device *dev = m3_ipc->dev;
+
+	if (!fw) {
+		dev_err(dev, "Voltage scale fw name given but file missing.\n");
+		return;
+	}
+
+	memcpy(&hdr, fw->data, sizeof(hdr));
+
+	if (hdr.magic != WKUP_M3_SD_FW_MAGIC) {
+		dev_err(dev, "PM: Voltage Scale Data binary does not appear valid.\n");
+		goto release_sd_fw;
+	}
+
+	aux_base = wkup_m3_copy_aux_data(m3_ipc, fw->data + sizeof(hdr),
+					 fw->size - sizeof(hdr));
+
+	val = (aux_base + hdr.sleep_offset);
+	val |= ((aux_base + hdr.wake_offset) << 16);
+
+	m3_ipc->volt_scale_offsets = val;
+
+release_sd_fw:
+	release_firmware(fw);
+};
+
+static int wkup_m3_init_scale_data(struct wkup_m3_ipc *m3_ipc,
+				   struct device *dev)
+{
+	int ret = 0;
+
+	/*
+	 * If no name is provided, user has already been warned, pm will
+	 * still work so return 0
+	 */
+
+	if (!m3_ipc->sd_fw_name)
+		return ret;
+
+	ret = request_firmware_nowait(THIS_MODULE, FW_ACTION_HOTPLUG,
+				      m3_ipc->sd_fw_name, dev, GFP_ATOMIC,
+				      m3_ipc, wkup_m3_scale_data_fw_cb);
+
+	return ret;
+}
+
+#ifdef CONFIG_DEBUG_FS
+static void wkup_m3_set_halt_late(bool enabled)
+{
+	if (enabled)
+		m3_ipc_state->halt = (1 << IPC_DBG_HALT_SHIFT);
+	else
+		m3_ipc_state->halt = 0;
+}
+
+static int option_get(void *data, u64 *val)
+{
+	u32 *option = data;
+
+	*val = *option;
+
+	return 0;
+}
+
+static int option_set(void *data, u64 val)
+{
+	u32 *option = data;
+
+	*option = val;
+
+	if (option == &m3_ipc_state->halt) {
+		if (val)
+			wkup_m3_set_halt_late(true);
+		else
+			wkup_m3_set_halt_late(false);
+	}
+
+	return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(wkup_m3_ipc_option_fops, option_get, option_set,
+			"%llu\n");
+
+static int wkup_m3_ipc_dbg_init(struct wkup_m3_ipc *m3_ipc)
+{
+	m3_ipc->dbg_path = debugfs_create_dir("wkup_m3_ipc", NULL);
+
+	if (!m3_ipc->dbg_path)
+		return -EINVAL;
+
+	(void)debugfs_create_file("enable_late_halt", 0644,
+				  m3_ipc->dbg_path,
+				  &m3_ipc->halt,
+				  &wkup_m3_ipc_option_fops);
+
+	return 0;
+}
+
+static inline void wkup_m3_ipc_dbg_destroy(struct wkup_m3_ipc *m3_ipc)
+{
+	debugfs_remove_recursive(m3_ipc->dbg_path);
+}
+#else
+static inline int wkup_m3_ipc_dbg_init(struct wkup_m3_ipc *m3_ipc)
+{
+	return 0;
+}
+
+static inline void wkup_m3_ipc_dbg_destroy(struct wkup_m3_ipc *m3_ipc)
+{
+}
+#endif /* CONFIG_DEBUG_FS */
+
 static void am33xx_txev_eoi(struct wkup_m3_ipc *m3_ipc)
 {
 	writel(AM33XX_M3_TXEV_ACK,
@@ -137,6 +298,7 @@
 		}
 
 		m3_ipc->state = M3_STATE_INITED;
+		wkup_m3_init_scale_data(m3_ipc, dev);
 		complete(&m3_ipc->sync_complete);
 		break;
 	case M3_STATE_MSG_FOR_RESET:
@@ -222,6 +384,17 @@
 		(m3_ipc->state != M3_STATE_UNKNOWN));
 }
 
+static void wkup_m3_set_vtt_gpio(struct wkup_m3_ipc *m3_ipc, int gpio)
+{
+	m3_ipc->vtt_conf = (1 << IPC_VTT_STAT_SHIFT) |
+			    (gpio << IPC_VTT_GPIO_PIN_SHIFT);
+}
+
+static void wkup_m3_set_io_isolation(struct wkup_m3_ipc *m3_ipc)
+{
+	m3_ipc->isolation_conf = (1 << IPC_IO_ISOLATION_STAT_SHIFT);
+}
+
 /* Public functions */
 /**
  * wkup_m3_set_mem_type - Pass wkup_m3 which type of memory is in use
@@ -283,12 +456,15 @@
 	switch (state) {
 	case WKUP_M3_DEEPSLEEP:
 		m3_power_state = IPC_CMD_DS0;
+		wkup_m3_ctrl_ipc_write(m3_ipc, m3_ipc->volt_scale_offsets, 5);
 		break;
 	case WKUP_M3_STANDBY:
 		m3_power_state = IPC_CMD_STANDBY;
+		wkup_m3_ctrl_ipc_write(m3_ipc, DS_IPC_DEFAULT, 5);
 		break;
 	case WKUP_M3_IDLE:
 		m3_power_state = IPC_CMD_IDLE;
+		wkup_m3_ctrl_ipc_write(m3_ipc, DS_IPC_DEFAULT, 5);
 		break;
 	default:
 		return 1;
@@ -297,11 +473,13 @@
 	/* Program each required IPC register then write defaults to others */
 	wkup_m3_ctrl_ipc_write(m3_ipc, m3_ipc->resume_addr, 0);
 	wkup_m3_ctrl_ipc_write(m3_ipc, m3_power_state, 1);
-	wkup_m3_ctrl_ipc_write(m3_ipc, m3_ipc->mem_type, 4);
+	wkup_m3_ctrl_ipc_write(m3_ipc, m3_ipc->mem_type |
+			       m3_ipc->vtt_conf |
+			       m3_ipc->isolation_conf |
+			       m3_ipc->halt, 4);
 
 	wkup_m3_ctrl_ipc_write(m3_ipc, DS_IPC_DEFAULT, 2);
 	wkup_m3_ctrl_ipc_write(m3_ipc, DS_IPC_DEFAULT, 3);
-	wkup_m3_ctrl_ipc_write(m3_ipc, DS_IPC_DEFAULT, 5);
 	wkup_m3_ctrl_ipc_write(m3_ipc, DS_IPC_DEFAULT, 6);
 	wkup_m3_ctrl_ipc_write(m3_ipc, DS_IPC_DEFAULT, 7);
 
@@ -426,6 +604,8 @@
 	ret = rproc_boot(m3_ipc->rproc);
 	if (ret)
 		dev_err(dev, "rproc_boot failed\n");
+	else
+		m3_ipc_state = m3_ipc;
 
 	do_exit(0);
 }
@@ -433,12 +613,13 @@
 static int wkup_m3_ipc_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	int irq, ret;
+	int irq, ret, temp;
 	phandle rproc_phandle;
 	struct rproc *m3_rproc;
 	struct resource *res;
 	struct task_struct *task;
 	struct wkup_m3_ipc *m3_ipc;
+	struct device_node *np = dev->of_node;
 
 	m3_ipc = devm_kzalloc(dev, sizeof(*m3_ipc), GFP_KERNEL);
 	if (!m3_ipc)
@@ -498,6 +679,23 @@
 
 	m3_ipc->ops = &ipc_ops;
 
+	if (of_find_property(np, "ti,needs-vtt-toggle", NULL) &&
+	    !(of_property_read_u32(np, "ti,vtt-gpio-pin", &temp))) {
+		if (temp >= 0 && temp <= 31)
+			wkup_m3_set_vtt_gpio(m3_ipc, temp);
+		else
+			dev_warn(dev, "Invalid VTT GPIO(%d) pin\n", temp);
+	}
+
+	if (of_find_property(np, "ti,set-io-isolation", NULL))
+		wkup_m3_set_io_isolation(m3_ipc);
+
+	ret = of_property_read_string(np, "ti,scale-data-fw",
+				      &m3_ipc->sd_fw_name);
+	if (ret) {
+		dev_dbg(dev, "Voltage scaling data blob not provided from DT.\n");
+	};
+
 	/*
 	 * Wait for firmware loading completion in a thread so we
 	 * can boot the wkup_m3 as soon as it's ready without holding
@@ -512,7 +710,7 @@
 		goto err_put_rproc;
 	}
 
-	m3_ipc_state = m3_ipc;
+	wkup_m3_ipc_dbg_init(m3_ipc);
 
 	return 0;
 
@@ -525,6 +723,8 @@
 
 static int wkup_m3_ipc_remove(struct platform_device *pdev)
 {
+	wkup_m3_ipc_dbg_destroy(m3_ipc_state);
+
 	mbox_free_channel(m3_ipc_state->mbox);
 
 	rproc_shutdown(m3_ipc_state->rproc);
diff -Naur --no-dereference a/drivers/spi/Kconfig b/drivers/spi/Kconfig
--- a/drivers/spi/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/spi/Kconfig	2020-06-19 21:00:31.424227716 -0400
@@ -427,7 +427,7 @@
 
 config SPI_OMAP24XX
 	tristate "McSPI driver for OMAP"
-	depends on ARCH_OMAP2PLUS || COMPILE_TEST
+	depends on ARCH_OMAP2PLUS || ARCH_K3 || COMPILE_TEST
 	select SG_SPLIT
 	help
 	  SPI master controller for OMAP24XX and later Multichannel SPI
diff -Naur --no-dereference a/drivers/spi/spi.c b/drivers/spi/spi.c
--- a/drivers/spi/spi.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/spi/spi.c	2020-06-19 21:00:31.424227716 -0400
@@ -1575,6 +1575,9 @@
 		case 4:
 			spi->mode |= SPI_TX_QUAD;
 			break;
+		case 8:
+			spi->mode |= SPI_TX_OCTAL;
+			break;
 		default:
 			dev_warn(&ctlr->dev,
 				"spi-tx-bus-width %d not supported\n",
@@ -1593,6 +1596,9 @@
 		case 4:
 			spi->mode |= SPI_RX_QUAD;
 			break;
+		case 8:
+			spi->mode |= SPI_RX_OCTAL;
+			break;
 		default:
 			dev_warn(&ctlr->dev,
 				"spi-rx-bus-width %d not supported\n",
@@ -2781,14 +2787,16 @@
 	/* if it is SPI_3WIRE mode, DUAL and QUAD should be forbidden
 	 */
 	if ((spi->mode & SPI_3WIRE) && (spi->mode &
-		(SPI_TX_DUAL | SPI_TX_QUAD | SPI_RX_DUAL | SPI_RX_QUAD)))
+		(SPI_TX_DUAL | SPI_TX_QUAD | SPI_TX_OCTAL |
+		 SPI_RX_DUAL | SPI_RX_QUAD | SPI_RX_OCTAL)))
 		return -EINVAL;
 	/* help drivers fail *cleanly* when they need options
 	 * that aren't supported with their current controller
 	 */
 	bad_bits = spi->mode & ~spi->controller->mode_bits;
 	ugly_bits = bad_bits &
-		    (SPI_TX_DUAL | SPI_TX_QUAD | SPI_RX_DUAL | SPI_RX_QUAD);
+		    (SPI_TX_DUAL | SPI_TX_QUAD | SPI_TX_OCTAL |
+		     SPI_RX_DUAL | SPI_RX_QUAD | SPI_RX_OCTAL);
 	if (ugly_bits) {
 		dev_warn(&spi->dev,
 			 "setup: ignoring unsupported mode bits %x\n",
diff -Naur --no-dereference a/drivers/spi/spidev.c b/drivers/spi/spidev.c
--- a/drivers/spi/spidev.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/spi/spidev.c	2020-06-19 21:00:31.424227716 -0400
@@ -727,9 +727,9 @@
 	 * compatible string, it is a Linux implementation thing
 	 * rather than a description of the hardware.
 	 */
-	WARN(spi->dev.of_node &&
-	     of_device_is_compatible(spi->dev.of_node, "spidev"),
-	     "%pOF: buggy DT: spidev listed directly in DT\n", spi->dev.of_node);
+//	WARN(spi->dev.of_node &&
+//	     of_device_is_compatible(spi->dev.of_node, "spidev"),
+//	     "%pOF: buggy DT: spidev listed directly in DT\n", spi->dev.of_node);
 
 	spidev_probe_acpi(spi);
 
diff -Naur --no-dereference a/drivers/spi/spi-mem.c b/drivers/spi/spi-mem.c
--- a/drivers/spi/spi-mem.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/spi/spi-mem.c	2020-06-19 21:00:31.424227716 -0400
@@ -12,6 +12,8 @@
 
 #include "internals.h"
 
+#define SPI_MEM_MAX_BUSWIDTH		8
+
 /**
  * spi_controller_dma_map_mem_op_data() - DMA-map the buffer attached to a
  *					  memory operation
@@ -118,6 +120,13 @@
 			return 0;
 
 		break;
+
+	case 8:
+		if ((tx && (mode & SPI_TX_OCTAL)) ||
+		    (!tx && (mode & SPI_RX_OCTAL)))
+			return 0;
+
+		break;
 
 	default:
 		break;
diff -Naur --no-dereference a/drivers/spi/spi-omap2-mcspi.c b/drivers/spi/spi-omap2-mcspi.c
--- a/drivers/spi/spi-omap2-mcspi.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/spi/spi-omap2-mcspi.c	2020-06-19 21:00:31.424227716 -0400
@@ -33,6 +33,7 @@
 #include <linux/of.h>
 #include <linux/of_device.h>
 #include <linux/gcd.h>
+#include <linux/iopoll.h>
 
 #include <linux/spi/spi.h>
 #include <linux/gpio.h>
@@ -126,6 +127,7 @@
 };
 
 struct omap2_mcspi {
+	struct completion	txdone;
 	struct spi_master	*master;
 	/* Virtual base address of the controller */
 	void __iomem		*base;
@@ -135,7 +137,9 @@
 	struct device		*dev;
 	struct omap2_mcspi_regs ctx;
 	int			fifo_depth;
+	bool			slave_aborted;
 	unsigned int		pin_dir:1;
+	size_t			max_xfer_len;
 };
 
 struct omap2_mcspi_cs {
@@ -274,19 +278,23 @@
 	}
 }
 
-static void omap2_mcspi_set_master_mode(struct spi_master *master)
+static void omap2_mcspi_set_mode(struct spi_master *master)
 {
 	struct omap2_mcspi	*mcspi = spi_master_get_devdata(master);
 	struct omap2_mcspi_regs	*ctx = &mcspi->ctx;
 	u32 l;
 
 	/*
-	 * Setup when switching from (reset default) slave mode
-	 * to single-channel master mode
+	 * Choose master or slave mode
 	 */
 	l = mcspi_read_reg(master, OMAP2_MCSPI_MODULCTRL);
-	l &= ~(OMAP2_MCSPI_MODULCTRL_STEST | OMAP2_MCSPI_MODULCTRL_MS);
-	l |= OMAP2_MCSPI_MODULCTRL_SINGLE;
+	l &= ~(OMAP2_MCSPI_MODULCTRL_STEST);
+	if (spi_controller_is_slave(master)) {
+		l |= (OMAP2_MCSPI_MODULCTRL_MS);
+	} else {
+		l &= ~(OMAP2_MCSPI_MODULCTRL_MS);
+		l |= OMAP2_MCSPI_MODULCTRL_SINGLE;
+	}
 	mcspi_write_reg(master, OMAP2_MCSPI_MODULCTRL, l);
 
 	ctx->modulctrl = l;
@@ -350,18 +358,22 @@
 
 static int mcspi_wait_for_reg_bit(void __iomem *reg, unsigned long bit)
 {
-	unsigned long timeout;
+	u32 val;
 
-	timeout = jiffies + msecs_to_jiffies(1000);
-	while (!(readl_relaxed(reg) & bit)) {
-		if (time_after(jiffies, timeout)) {
-			if (!(readl_relaxed(reg) & bit))
-				return -ETIMEDOUT;
-			else
-				return 0;
-		}
-		cpu_relax();
+	return readl_poll_timeout(reg, val, val & bit, 1, MSEC_PER_SEC);
+}
+
+static int mcspi_wait_for_completion(struct  omap2_mcspi *mcspi,
+				     struct completion *x)
+{
+	if (spi_controller_is_slave(mcspi->master)) {
+		if (wait_for_completion_interruptible(x) ||
+		    mcspi->slave_aborted)
+			return -EINTR;
+	} else {
+		wait_for_completion(x);
 	}
+
 	return 0;
 }
 
@@ -514,7 +526,12 @@
 	dma_async_issue_pending(mcspi_dma->dma_rx);
 	omap2_mcspi_set_dma_req(spi, 1, 1);
 
-	wait_for_completion(&mcspi_dma->dma_rx_completion);
+	ret = mcspi_wait_for_completion(mcspi, &mcspi_dma->dma_rx_completion);
+	if (ret || mcspi->slave_aborted) {
+		dmaengine_terminate_sync(mcspi_dma->dma_rx);
+		omap2_mcspi_set_dma_req(spi, 1, 0);
+		return 0;
+	}
 
 	for (x = 0; x < nb_sizes; x++)
 		kfree(sg_out[x]);
@@ -613,14 +630,37 @@
 	rx = xfer->rx_buf;
 	tx = xfer->tx_buf;
 
-	if (tx != NULL)
+	mcspi->slave_aborted = false;
+	reinit_completion(&mcspi_dma->dma_tx_completion);
+	reinit_completion(&mcspi_dma->dma_rx_completion);
+	reinit_completion(&mcspi->txdone);
+	if (tx) {
+		/* Enable EOW IRQ to know end of tx in slave mode */
+		if (spi_controller_is_slave(spi->master))
+			mcspi_write_reg(spi->master,
+					OMAP2_MCSPI_IRQENABLE,
+					OMAP2_MCSPI_IRQSTATUS_EOW);
 		omap2_mcspi_tx_dma(spi, xfer, cfg);
+	}
 
 	if (rx != NULL)
 		count = omap2_mcspi_rx_dma(spi, xfer, cfg, es);
 
 	if (tx != NULL) {
-		wait_for_completion(&mcspi_dma->dma_tx_completion);
+		int ret;
+
+		ret = mcspi_wait_for_completion(mcspi, &mcspi_dma->dma_tx_completion);
+		if (ret || mcspi->slave_aborted) {
+			dmaengine_terminate_sync(mcspi_dma->dma_tx);
+			omap2_mcspi_set_dma_req(spi, 0, 0);
+			return 0;
+		}
+
+		if (spi_controller_is_slave(mcspi->master)) {
+			ret = mcspi_wait_for_completion(mcspi, &mcspi->txdone);
+			if (ret || mcspi->slave_aborted)
+				return 0;
+		}
 
 		if (mcspi->fifo_depth > 0) {
 			irqstat_reg = mcspi->base + OMAP2_MCSPI_IRQSTATUS;
@@ -955,19 +995,11 @@
  * Note that we currently allow DMA only if we get a channel
  * for both rx and tx. Otherwise we'll do PIO for both rx and tx.
  */
-static int omap2_mcspi_request_dma(struct spi_device *spi)
+static int omap2_mcspi_request_dma(struct spi_master *master,
+				   struct omap2_mcspi_dma *mcspi_dma)
 {
-	struct spi_master	*master = spi->master;
-	struct omap2_mcspi	*mcspi;
-	struct omap2_mcspi_dma	*mcspi_dma;
 	int ret = 0;
 
-	mcspi = spi_master_get_devdata(master);
-	mcspi_dma = mcspi->dma_channels + spi->chip_select;
-
-	init_completion(&mcspi_dma->dma_rx_completion);
-	init_completion(&mcspi_dma->dma_tx_completion);
-
 	mcspi_dma->dma_rx = dma_request_chan(&master->dev,
 					     mcspi_dma->dma_rx_ch_name);
 	if (IS_ERR(mcspi_dma->dma_rx)) {
@@ -985,20 +1017,40 @@
 		mcspi_dma->dma_rx = NULL;
 	}
 
+	init_completion(&mcspi_dma->dma_rx_completion);
+	init_completion(&mcspi_dma->dma_tx_completion);
+
 no_dma:
 	return ret;
 }
 
+static void omap2_mcspi_release_dma(struct spi_master *master)
+{
+	struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
+	struct omap2_mcspi_dma	*mcspi_dma;
+	int i;
+
+	for (i = 0; i < master->num_chipselect; i++) {
+		mcspi_dma = &mcspi->dma_channels[i];
+
+		if (mcspi_dma->dma_rx) {
+			dma_release_channel(mcspi_dma->dma_rx);
+			mcspi_dma->dma_rx = NULL;
+		}
+		if (mcspi_dma->dma_tx) {
+			dma_release_channel(mcspi_dma->dma_tx);
+			mcspi_dma->dma_tx = NULL;
+		}
+	}
+}
+
 static int omap2_mcspi_setup(struct spi_device *spi)
 {
 	int			ret;
 	struct omap2_mcspi	*mcspi = spi_master_get_devdata(spi->master);
 	struct omap2_mcspi_regs	*ctx = &mcspi->ctx;
-	struct omap2_mcspi_dma	*mcspi_dma;
 	struct omap2_mcspi_cs	*cs = spi->controller_state;
 
-	mcspi_dma = &mcspi->dma_channels[spi->chip_select];
-
 	if (!cs) {
 		cs = kzalloc(sizeof *cs, GFP_KERNEL);
 		if (!cs)
@@ -1023,13 +1075,6 @@
 		}
 	}
 
-	if (!mcspi_dma->dma_rx || !mcspi_dma->dma_tx) {
-		ret = omap2_mcspi_request_dma(spi);
-		if (ret)
-			dev_warn(&spi->dev, "not using DMA for McSPI (%d)\n",
-				 ret);
-	}
-
 	ret = pm_runtime_get_sync(mcspi->dev);
 	if (ret < 0) {
 		pm_runtime_put_noidle(mcspi->dev);
@@ -1046,12 +1091,8 @@
 
 static void omap2_mcspi_cleanup(struct spi_device *spi)
 {
-	struct omap2_mcspi	*mcspi;
-	struct omap2_mcspi_dma	*mcspi_dma;
 	struct omap2_mcspi_cs	*cs;
 
-	mcspi = spi_master_get_devdata(spi->master);
-
 	if (spi->controller_state) {
 		/* Unlink controller state from context save list */
 		cs = spi->controller_state;
@@ -1060,23 +1101,40 @@
 		kfree(cs);
 	}
 
-	if (spi->chip_select < spi->master->num_chipselect) {
-		mcspi_dma = &mcspi->dma_channels[spi->chip_select];
-
-		if (mcspi_dma->dma_rx) {
-			dma_release_channel(mcspi_dma->dma_rx);
-			mcspi_dma->dma_rx = NULL;
-		}
-		if (mcspi_dma->dma_tx) {
-			dma_release_channel(mcspi_dma->dma_tx);
-			mcspi_dma->dma_tx = NULL;
-		}
-	}
-
 	if (gpio_is_valid(spi->cs_gpio))
 		gpio_free(spi->cs_gpio);
 }
 
+static irqreturn_t omap2_mcspi_irq_handler(int irq, void *data)
+{
+	struct omap2_mcspi *mcspi = data;
+	u32 irqstat;
+
+	irqstat	= mcspi_read_reg(mcspi->master, OMAP2_MCSPI_IRQSTATUS);
+	if (!irqstat)
+		return IRQ_NONE;
+
+	/* Disable IRQ and wakeup slave xfer task */
+	mcspi_write_reg(mcspi->master, OMAP2_MCSPI_IRQENABLE, 0);
+	if (irqstat & OMAP2_MCSPI_IRQSTATUS_EOW)
+		complete(&mcspi->txdone);
+
+	return IRQ_HANDLED;
+}
+
+static int omap2_mcspi_slave_abort(struct spi_master *master)
+{
+	struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
+	struct omap2_mcspi_dma *mcspi_dma = mcspi->dma_channels;
+
+	mcspi->slave_aborted = true;
+	complete(&mcspi_dma->dma_rx_completion);
+	complete(&mcspi_dma->dma_tx_completion);
+	complete(&mcspi->txdone);
+
+	return 0;
+}
+
 static int omap2_mcspi_transfer_one(struct spi_master *master,
 				    struct spi_device *spi,
 				    struct spi_transfer *t)
@@ -1243,10 +1301,35 @@
 				struct spi_device *spi,
 				struct spi_transfer *xfer)
 {
+	struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
+	struct omap2_mcspi_dma *mcspi_dma =
+		&mcspi->dma_channels[spi->chip_select];
+
+	if (!mcspi_dma->dma_rx || !mcspi_dma->dma_tx)
+		return false;
+
+	if (spi_controller_is_slave(master))
+		return true;
+
+	master->dma_rx = mcspi_dma->dma_rx;
+	master->dma_tx = mcspi_dma->dma_tx;
+
 	return (xfer->len >= DMA_MIN_BYTES);
 }
 
-static int omap2_mcspi_master_setup(struct omap2_mcspi *mcspi)
+static size_t omap2_mcspi_max_xfer_size(struct spi_device *spi)
+{
+	struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
+	struct omap2_mcspi_dma *mcspi_dma =
+		&mcspi->dma_channels[spi->chip_select];
+
+	if (mcspi->max_xfer_len && mcspi_dma->dma_rx)
+		return mcspi->max_xfer_len;
+
+	return SIZE_MAX;
+}
+
+static int omap2_mcspi_controller_setup(struct omap2_mcspi *mcspi)
 {
 	struct spi_master	*master = mcspi->master;
 	struct omap2_mcspi_regs	*ctx = &mcspi->ctx;
@@ -1263,7 +1346,7 @@
 			OMAP2_MCSPI_WAKEUPENABLE_WKEN);
 	ctx->wakeupenable = OMAP2_MCSPI_WAKEUPENABLE_WKEN;
 
-	omap2_mcspi_set_master_mode(master);
+	omap2_mcspi_set_mode(master);
 	pm_runtime_mark_last_busy(mcspi->dev);
 	pm_runtime_put_autosuspend(mcspi->dev);
 	return 0;
@@ -1314,6 +1397,11 @@
 	.regs_offset = OMAP4_MCSPI_REG_OFFSET,
 };
 
+static struct omap2_mcspi_platform_config am654_pdata = {
+	.regs_offset = OMAP4_MCSPI_REG_OFFSET,
+	.max_xfer_len = SZ_4K - 1,
+};
+
 static const struct of_device_id omap_mcspi_of_match[] = {
 	{
 		.compatible = "ti,omap2-mcspi",
@@ -1323,6 +1411,10 @@
 		.compatible = "ti,omap4-mcspi",
 		.data = &omap4_pdata,
 	},
+	{
+		.compatible = "ti,am654-mcspi",
+		.data = &am654_pdata,
+	},
 	{ },
 };
 MODULE_DEVICE_TABLE(of, omap_mcspi_of_match);
@@ -1338,11 +1430,12 @@
 	struct device_node	*node = pdev->dev.of_node;
 	const struct of_device_id *match;
 
-	master = spi_alloc_master(&pdev->dev, sizeof *mcspi);
-	if (master == NULL) {
-		dev_dbg(&pdev->dev, "master allocation failed\n");
+	if (of_property_read_bool(node, "spi-slave"))
+		master = spi_alloc_slave(&pdev->dev, sizeof(*mcspi));
+	else
+		master = spi_alloc_master(&pdev->dev, sizeof(*mcspi));
+	if (!master)
 		return -ENOMEM;
-	}
 
 	/* the spi->mode bits understood by this driver: */
 	master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH;
@@ -1354,6 +1447,7 @@
 	master->transfer_one = omap2_mcspi_transfer_one;
 	master->set_cs = omap2_mcspi_set_cs;
 	master->cleanup = omap2_mcspi_cleanup;
+	master->slave_abort = omap2_mcspi_slave_abort;
 	master->dev.of_node = node;
 	master->max_speed_hz = OMAP2_MCSPI_MAX_FREQ;
 	master->min_speed_hz = OMAP2_MCSPI_MAX_FREQ >> 15;
@@ -1378,6 +1472,10 @@
 		mcspi->pin_dir = pdata->pin_dir;
 	}
 	regs_offset = pdata->regs_offset;
+	if (pdata->max_xfer_len) {
+		mcspi->max_xfer_len = pdata->max_xfer_len;
+		master->max_transfer_size = omap2_mcspi_max_xfer_size;
+	}
 
 	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	mcspi->base = devm_ioremap_resource(&pdev->dev, r);
@@ -1403,17 +1501,38 @@
 	for (i = 0; i < master->num_chipselect; i++) {
 		sprintf(mcspi->dma_channels[i].dma_rx_ch_name, "rx%d", i);
 		sprintf(mcspi->dma_channels[i].dma_tx_ch_name, "tx%d", i);
+
+		status = omap2_mcspi_request_dma(master,
+						 &mcspi->dma_channels[i]);
+		if (status == -EPROBE_DEFER)
+			goto free_master;
+	}
+
+	status = platform_get_irq(pdev, 0);
+	if (status == -EPROBE_DEFER)
+		goto free_master;
+	if (status < 0) {
+		dev_err(&pdev->dev, "no irq resource found\n");
+		goto free_master;
+	}
+	init_completion(&mcspi->txdone);
+	status = devm_request_irq(&pdev->dev, status,
+				  omap2_mcspi_irq_handler, 0, pdev->name,
+				  mcspi);
+	if (status) {
+		dev_err(&pdev->dev, "Cannot request IRQ");
+		goto free_master;
 	}
 
 	pm_runtime_use_autosuspend(&pdev->dev);
 	pm_runtime_set_autosuspend_delay(&pdev->dev, SPI_AUTOSUSPEND_TIMEOUT);
 	pm_runtime_enable(&pdev->dev);
 
-	status = omap2_mcspi_master_setup(mcspi);
+	status = omap2_mcspi_controller_setup(mcspi);
 	if (status < 0)
 		goto disable_pm;
 
-	status = devm_spi_register_master(&pdev->dev, master);
+	status = devm_spi_register_controller(&pdev->dev, master);
 	if (status < 0)
 		goto disable_pm;
 
@@ -1424,6 +1543,7 @@
 	pm_runtime_put_sync(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
 free_master:
+	omap2_mcspi_release_dma(master);
 	spi_master_put(master);
 	return status;
 }
@@ -1433,6 +1553,8 @@
 	struct spi_master *master = platform_get_drvdata(pdev);
 	struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
 
+	omap2_mcspi_release_dma(master);
+
 	pm_runtime_dont_use_autosuspend(mcspi->dev);
 	pm_runtime_put_sync(mcspi->dev);
 	pm_runtime_disable(&pdev->dev);
diff -Naur --no-dereference a/drivers/spi/spi-ti-qspi.c b/drivers/spi/spi-ti-qspi.c
--- a/drivers/spi/spi-ti-qspi.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/spi/spi-ti-qspi.c	2020-06-19 21:00:31.424227716 -0400
@@ -69,6 +69,7 @@
 	u32 dc;
 
 	bool mmap_enabled;
+	int current_cs;
 };
 
 #define QSPI_PID			(0x0)
@@ -494,6 +495,7 @@
 				   MEM_CS_EN(spi->chip_select));
 	}
 	qspi->mmap_enabled = true;
+	qspi->current_cs = spi->chip_select;
 }
 
 static void ti_qspi_disable_memory_map(struct spi_device *spi)
@@ -505,6 +507,7 @@
 		regmap_update_bits(qspi->ctrl_base, qspi->ctrl_reg,
 				   MEM_CS_MASK, 0);
 	qspi->mmap_enabled = false;
+	qspi->current_cs = -1;
 }
 
 static void ti_qspi_setup_mmap_read(struct spi_device *spi, u8 opcode,
@@ -550,7 +553,7 @@
 
 	mutex_lock(&qspi->list_lock);
 
-	if (!qspi->mmap_enabled)
+	if (!qspi->mmap_enabled || qspi->current_cs != mem->spi->chip_select)
 		ti_qspi_enable_memory_map(mem->spi);
 	ti_qspi_setup_mmap_read(mem->spi, op->cmd.opcode, op->data.buswidth,
 				op->addr.nbytes, op->dummy.nbytes);
@@ -807,6 +810,7 @@
 		}
 	}
 	qspi->mmap_enabled = false;
+	qspi->current_cs = -1;
 
 	ret = devm_spi_register_master(&pdev->dev, master);
 	if (!ret)
diff -Naur --no-dereference a/drivers/staging/android/ion/ion.c b/drivers/staging/android/ion/ion.c
--- a/drivers/staging/android/ion/ion.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/staging/android/ion/ion.c	2020-06-19 21:00:31.424227716 -0400
@@ -1,11 +1,10 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * drivers/staging/android/ion/ion.c
+ * ION Memory Allocator
  *
  * Copyright (C) 2011 Google, Inc.
  */
 
-#include <linux/anon_inodes.h>
 #include <linux/debugfs.h>
 #include <linux/device.h>
 #include <linux/dma-buf.h>
@@ -14,16 +13,13 @@
 #include <linux/file.h>
 #include <linux/freezer.h>
 #include <linux/fs.h>
-#include <linux/idr.h>
 #include <linux/kthread.h>
 #include <linux/list.h>
-#include <linux/memblock.h>
 #include <linux/miscdevice.h>
 #include <linux/mm.h>
 #include <linux/mm_types.h>
 #include <linux/rbtree.h>
 #include <linux/sched/task.h>
-#include <linux/seq_file.h>
 #include <linux/slab.h>
 #include <linux/uaccess.h>
 #include <linux/vmalloc.h>
@@ -95,6 +91,13 @@
 		goto err1;
 	}
 
+	spin_lock(&heap->stat_lock);
+	heap->num_of_buffers++;
+	heap->num_of_alloc_bytes += len;
+	if (heap->num_of_alloc_bytes > heap->alloc_bytes_wm)
+		heap->alloc_bytes_wm = heap->num_of_alloc_bytes;
+	spin_unlock(&heap->stat_lock);
+
 	INIT_LIST_HEAD(&buffer->attachments);
 	mutex_init(&buffer->lock);
 	mutex_lock(&dev->buffer_lock);
@@ -117,6 +120,11 @@
 		buffer->heap->ops->unmap_kernel(buffer->heap, buffer);
 	}
 	buffer->heap->ops->free(buffer);
+	spin_lock(&buffer->heap->stat_lock);
+	buffer->heap->num_of_buffers--;
+	buffer->heap->num_of_alloc_bytes -= buffer->size;
+	spin_unlock(&buffer->heap->stat_lock);
+
 	kfree(buffer);
 }
 
@@ -253,8 +261,8 @@
 
 	table = a->table;
 
-	if (!dma_map_sg(attachment->dev, table->sgl, table->nents,
-			direction))
+	if (!dma_map_sg_attrs(attachment->dev, table->sgl, table->nents,
+			      direction, DMA_ATTR_SKIP_CPU_SYNC))
 		return ERR_PTR(-ENOMEM);
 
 	return table;
@@ -264,7 +272,8 @@
 			      struct sg_table *table,
 			      enum dma_data_direction direction)
 {
-	dma_unmap_sg(attachment->dev, table->sgl, table->nents, direction);
+	dma_unmap_sg_attrs(attachment->dev, table->sgl, table->nents,
+			   direction, DMA_ATTR_SKIP_CPU_SYNC);
 }
 
 static int ion_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
@@ -379,7 +388,7 @@
 	.unmap = ion_dma_buf_kunmap,
 };
 
-int ion_alloc(size_t len, unsigned int heap_id_mask, unsigned int flags)
+static int ion_alloc(size_t len, unsigned int heap_id_mask, unsigned int flags)
 {
 	struct ion_device *dev = internal_dev;
 	struct ion_buffer *buffer = NULL;
@@ -436,7 +445,7 @@
 	return fd;
 }
 
-int ion_query_heaps(struct ion_heap_query *query)
+static int ion_query_heaps(struct ion_heap_query *query)
 {
 	struct ion_device *dev = internal_dev;
 	struct ion_heap_data __user *buffer = u64_to_user_ptr(query->heaps);
@@ -459,7 +468,7 @@
 	max_cnt = query->cnt;
 
 	plist_for_each_entry(heap, &dev->heaps, node) {
-		strncpy(hdata.name, heap->name, MAX_HEAP_NAME);
+		strncpy(hdata.name, heap->name ?: "(null)", MAX_HEAP_NAME);
 		hdata.name[sizeof(hdata.name) - 1] = '\0';
 		hdata.type = heap->type;
 		hdata.heap_id = heap->id;
@@ -481,6 +490,81 @@
 	return ret;
 }
 
+union ion_ioctl_arg {
+	struct ion_allocation_data allocation;
+	struct ion_heap_query query;
+};
+
+static int validate_ioctl_arg(unsigned int cmd, union ion_ioctl_arg *arg)
+{
+	switch (cmd) {
+	case ION_IOC_HEAP_QUERY:
+		if (arg->query.reserved0 ||
+		    arg->query.reserved1 ||
+		    arg->query.reserved2)
+			return -EINVAL;
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static long ion_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
+{
+	int ret = 0;
+	union ion_ioctl_arg data;
+
+	if (_IOC_SIZE(cmd) > sizeof(data))
+		return -EINVAL;
+
+	/*
+	 * The copy_from_user is unconditional here for both read and write
+	 * to do the validate. If there is no write for the ioctl, the
+	 * buffer is cleared
+	 */
+	if (copy_from_user(&data, (void __user *)arg, _IOC_SIZE(cmd)))
+		return -EFAULT;
+
+	ret = validate_ioctl_arg(cmd, &data);
+	if (ret) {
+		pr_warn_once("%s: ioctl validate failed\n", __func__);
+		return ret;
+	}
+
+	if (!(_IOC_DIR(cmd) & _IOC_WRITE))
+		memset(&data, 0, sizeof(data));
+
+	switch (cmd) {
+	case ION_IOC_ALLOC:
+	{
+		int fd;
+
+		fd = ion_alloc(data.allocation.len,
+			       data.allocation.heap_id_mask,
+			       data.allocation.flags);
+		if (fd < 0)
+			return fd;
+
+		data.allocation.fd = fd;
+
+		break;
+	}
+	case ION_IOC_HEAP_QUERY:
+		ret = ion_query_heaps(&data.query);
+		break;
+	default:
+		return -ENOTTY;
+	}
+
+	if (_IOC_DIR(cmd) & _IOC_READ) {
+		if (copy_to_user((void __user *)arg, &data, _IOC_SIZE(cmd)))
+			return -EFAULT;
+	}
+	return ret;
+}
+
 static const struct file_operations ion_fops = {
 	.owner          = THIS_MODULE,
 	.unlocked_ioctl = ion_ioctl,
@@ -528,12 +612,15 @@
 {
 	struct ion_device *dev = internal_dev;
 	int ret;
+	struct dentry *heap_root;
+	char debug_name[64];
 
 	if (!heap->ops->allocate || !heap->ops->free)
 		pr_err("%s: can not add heap with invalid ops struct.\n",
 		       __func__);
 
 	spin_lock_init(&heap->free_lock);
+	spin_lock_init(&heap->stat_lock);
 	heap->free_list_size = 0;
 
 	if (heap->flags & ION_HEAP_FLAG_DEFER_FREE)
@@ -546,6 +633,33 @@
 	}
 
 	heap->dev = dev;
+	heap->num_of_buffers = 0;
+	heap->num_of_alloc_bytes = 0;
+	heap->alloc_bytes_wm = 0;
+
+	heap_root = debugfs_create_dir(heap->name, dev->debug_root);
+	debugfs_create_u64("num_of_buffers",
+			   0444, heap_root,
+			   &heap->num_of_buffers);
+	debugfs_create_u64("num_of_alloc_bytes",
+			   0444,
+			   heap_root,
+			   &heap->num_of_alloc_bytes);
+	debugfs_create_u64("alloc_bytes_wm",
+			   0444,
+			   heap_root,
+			   &heap->alloc_bytes_wm);
+
+	if (heap->shrinker.count_objects &&
+	    heap->shrinker.scan_objects) {
+		snprintf(debug_name, 64, "%s_shrink", heap->name);
+		debugfs_create_file(debug_name,
+				    0644,
+				    heap_root,
+				    heap,
+				    &debug_shrink_fops);
+	}
+
 	down_write(&dev->lock);
 	heap->id = heap_id++;
 	/*
@@ -555,14 +669,6 @@
 	plist_node_init(&heap->node, -heap->id);
 	plist_add(&heap->node, &dev->heaps);
 
-	if (heap->shrinker.count_objects && heap->shrinker.scan_objects) {
-		char debug_name[64];
-
-		snprintf(debug_name, 64, "%s_shrink", heap->name);
-		debugfs_create_file(debug_name, 0644, dev->debug_root,
-				    heap, &debug_shrink_fops);
-	}
-
 	dev->heap_cnt++;
 	up_write(&dev->lock);
 }
diff -Naur --no-dereference a/drivers/staging/android/ion/ion_carveout_heap.c b/drivers/staging/android/ion/ion_carveout_heap.c
--- a/drivers/staging/android/ion/ion_carveout_heap.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/staging/android/ion/ion_carveout_heap.c	2020-06-19 21:00:31.424227716 -0400
@@ -1,10 +1,10 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * drivers/staging/android/ion/ion_carveout_heap.c
+ * ION Memory Allocator carveout heap helper
  *
  * Copyright (C) 2011 Google, Inc.
  */
-#include <linux/spinlock.h>
+
 #include <linux/dma-mapping.h>
 #include <linux/err.h>
 #include <linux/genalloc.h>
@@ -12,7 +12,7 @@
 #include <linux/mm.h>
 #include <linux/scatterlist.h>
 #include <linux/slab.h>
-#include <linux/vmalloc.h>
+
 #include "ion.h"
 
 #define ION_CARVEOUT_ALLOCATE_FAIL	-1
@@ -20,7 +20,6 @@
 struct ion_carveout_heap {
 	struct ion_heap heap;
 	struct gen_pool *pool;
-	phys_addr_t base;
 };
 
 static phys_addr_t ion_carveout_allocate(struct ion_heap *heap,
@@ -44,6 +43,7 @@
 
 	if (addr == ION_CARVEOUT_ALLOCATE_FAIL)
 		return;
+
 	gen_pool_free(carveout_heap->pool, addr, size);
 }
 
@@ -103,17 +103,14 @@
 	.unmap_kernel = ion_heap_unmap_kernel,
 };
 
-struct ion_heap *ion_carveout_heap_create(struct ion_platform_heap *heap_data)
+struct ion_heap *ion_carveout_heap_create(phys_addr_t base, size_t size)
 {
 	struct ion_carveout_heap *carveout_heap;
 	int ret;
 
 	struct page *page;
-	size_t size;
-
-	page = pfn_to_page(PFN_DOWN(heap_data->base));
-	size = heap_data->size;
 
+	page = pfn_to_page(PFN_DOWN(base));
 	ret = ion_heap_pages_zero(page, size, pgprot_writecombine(PAGE_KERNEL));
 	if (ret)
 		return ERR_PTR(ret);
@@ -127,9 +124,7 @@
 		kfree(carveout_heap);
 		return ERR_PTR(-ENOMEM);
 	}
-	carveout_heap->base = heap_data->base;
-	gen_pool_add(carveout_heap->pool, carveout_heap->base, heap_data->size,
-		     -1);
+	gen_pool_add(carveout_heap->pool, base, size, -1);
 	carveout_heap->heap.ops = &carveout_heap_ops;
 	carveout_heap->heap.type = ION_HEAP_TYPE_CARVEOUT;
 	carveout_heap->heap.flags = ION_HEAP_FLAG_DEFER_FREE;
diff -Naur --no-dereference a/drivers/staging/android/ion/ion_chunk_heap.c b/drivers/staging/android/ion/ion_chunk_heap.c
--- a/drivers/staging/android/ion/ion_chunk_heap.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/staging/android/ion/ion_chunk_heap.c	2020-06-19 21:00:31.424227716 -0400
@@ -1,23 +1,22 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * drivers/staging/android/ion/ion_chunk_heap.c
+ * ION memory allocator chunk heap helper
  *
  * Copyright (C) 2012 Google, Inc.
  */
+
 #include <linux/dma-mapping.h>
 #include <linux/err.h>
 #include <linux/genalloc.h>
-#include <linux/io.h>
 #include <linux/mm.h>
 #include <linux/scatterlist.h>
 #include <linux/slab.h>
-#include <linux/vmalloc.h>
+
 #include "ion.h"
 
 struct ion_chunk_heap {
 	struct ion_heap heap;
 	struct gen_pool *pool;
-	phys_addr_t base;
 	unsigned long chunk_size;
 	unsigned long size;
 	unsigned long allocated;
@@ -108,16 +107,13 @@
 	.unmap_kernel = ion_heap_unmap_kernel,
 };
 
-struct ion_heap *ion_chunk_heap_create(struct ion_platform_heap *heap_data)
+struct ion_heap *ion_chunk_heap_create(phys_addr_t base, size_t size, size_t chunk_size)
 {
 	struct ion_chunk_heap *chunk_heap;
 	int ret;
 	struct page *page;
-	size_t size;
-
-	page = pfn_to_page(PFN_DOWN(heap_data->base));
-	size = heap_data->size;
 
+	page = pfn_to_page(PFN_DOWN(base));
 	ret = ion_heap_pages_zero(page, size, pgprot_writecombine(PAGE_KERNEL));
 	if (ret)
 		return ERR_PTR(ret);
@@ -126,23 +122,21 @@
 	if (!chunk_heap)
 		return ERR_PTR(-ENOMEM);
 
-	chunk_heap->chunk_size = (unsigned long)heap_data->priv;
+	chunk_heap->chunk_size = chunk_size;
 	chunk_heap->pool = gen_pool_create(get_order(chunk_heap->chunk_size) +
 					   PAGE_SHIFT, -1);
 	if (!chunk_heap->pool) {
 		ret = -ENOMEM;
 		goto error_gen_pool_create;
 	}
-	chunk_heap->base = heap_data->base;
-	chunk_heap->size = heap_data->size;
+	chunk_heap->size = size;
 	chunk_heap->allocated = 0;
 
-	gen_pool_add(chunk_heap->pool, chunk_heap->base, heap_data->size, -1);
+	gen_pool_add(chunk_heap->pool, base, size, -1);
 	chunk_heap->heap.ops = &chunk_heap_ops;
 	chunk_heap->heap.type = ION_HEAP_TYPE_CHUNK;
 	chunk_heap->heap.flags = ION_HEAP_FLAG_DEFER_FREE;
-	pr_debug("%s: base %pa size %zu\n", __func__,
-		 &chunk_heap->base, heap_data->size);
+	pr_debug("%s: base %pa size %zu\n", __func__, &base, size);
 
 	return &chunk_heap->heap;
 
@@ -150,4 +144,3 @@
 	kfree(chunk_heap);
 	return ERR_PTR(ret);
 }
-
diff -Naur --no-dereference a/drivers/staging/android/ion/ion_cma_heap.c b/drivers/staging/android/ion/ion_cma_heap.c
--- a/drivers/staging/android/ion/ion_cma_heap.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/staging/android/ion/ion_cma_heap.c	2020-06-19 21:00:31.424227716 -0400
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * drivers/staging/android/ion/ion_cma_heap.c
+ * ION Memory Allocator CMA heap exporter
  *
  * Copyright (C) Linaro 2012
  * Author: <benjamin.gaignard@linaro.org> for ST-Ericsson.
@@ -111,10 +111,6 @@
 		return ERR_PTR(-ENOMEM);
 
 	cma_heap->heap.ops = &ion_cma_ops;
-	/*
-	 * get device from private heaps data, later it will be
-	 * used to make the link with reserved CMA memory
-	 */
 	cma_heap->cma = cma;
 	cma_heap->heap.type = ION_HEAP_TYPE_DMA;
 	return &cma_heap->heap;
diff -Naur --no-dereference a/drivers/staging/android/ion/ion.h b/drivers/staging/android/ion/ion.h
--- a/drivers/staging/android/ion/ion.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/staging/android/ion/ion.h	2020-06-19 21:00:31.424227716 -0400
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /*
- * drivers/staging/android/ion/ion.h
+ * ION Memory Allocator kernel interface header
  *
  * Copyright (C) 2011 Google, Inc.
  */
@@ -22,32 +22,9 @@
 #include "../uapi/ion.h"
 
 /**
- * struct ion_platform_heap - defines a heap in the given platform
- * @type:	type of the heap from ion_heap_type enum
- * @id:		unique identifier for heap.  When allocating higher numb ers
- *		will be allocated from first.  At allocation these are passed
- *		as a bit mask and therefore can not exceed ION_NUM_HEAP_IDS.
- * @name:	used for debug purposes
- * @base:	base address of heap in physical memory if applicable
- * @size:	size of the heap in bytes if applicable
- * @priv:	private info passed from the board file
- *
- * Provided by the board file.
- */
-struct ion_platform_heap {
-	enum ion_heap_type type;
-	unsigned int id;
-	const char *name;
-	phys_addr_t base;
-	size_t size;
-	phys_addr_t align;
-	void *priv;
-};
-
-/**
  * struct ion_buffer - metadata for a particular buffer
- * @ref:		reference count
  * @node:		node in the ion_device buffers tree
+ * @list:		element in list of deferred freeable buffers
  * @dev:		back pointer to the ion_device
  * @heap:		back pointer to the heap the buffer came from
  * @flags:		buffer specific flags
@@ -58,7 +35,8 @@
  * @lock:		protects the buffers cnt fields
  * @kmap_cnt:		number of times the buffer is mapped to the kernel
  * @vaddr:		the kernel mapping if kmap_cnt is not zero
- * @sg_table:		the sg table for the buffer if dmap_cnt is not zero
+ * @sg_table:		the sg table for the buffer
+ * @attachments:	list of devices attached to this buffer
  */
 struct ion_buffer {
 	union {
@@ -157,8 +135,9 @@
  * @lock:		protects the free list
  * @waitqueue:		queue to wait on from deferred free thread
  * @task:		task struct of deferred free thread
- * @debug_show:		called when heap debug file is read to add any
- *			heap specific debug info to output
+ * @num_of_buffers	the number of currently allocated buffers
+ * @num_of_alloc_bytes	the number of allocated bytes
+ * @alloc_bytes_wm	the number of allocated bytes watermark
  *
  * Represents a pool of memory from which buffers can be made.  In some
  * systems the only heap is regular system memory allocated via vmalloc.
@@ -173,6 +152,8 @@
 	unsigned long flags;
 	unsigned int id;
 	const char *name;
+
+	/* deferred free support */
 	struct shrinker shrinker;
 	struct list_head free_list;
 	size_t free_list_size;
@@ -180,8 +161,13 @@
 	wait_queue_head_t waitqueue;
 	struct task_struct *task;
 
-	int (*debug_show)(struct ion_heap *heap, struct seq_file *s,
-			  void *unused);
+	/* heap statistics */
+	u64 num_of_buffers;
+	u64 num_of_alloc_bytes;
+	u64 alloc_bytes_wm;
+
+	/* protect heap statistics */
+	spinlock_t stat_lock;
 };
 
 /**
@@ -201,10 +187,6 @@
 int ion_heap_buffer_zero(struct ion_buffer *buffer);
 int ion_heap_pages_zero(struct page *page, size_t size, pgprot_t pgprot);
 
-int ion_alloc(size_t len,
-	      unsigned int heap_id_mask,
-	      unsigned int flags);
-
 /**
  * ion_heap_init_shrinker
  * @heap:		the heap
@@ -326,8 +308,37 @@
 int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask,
 			 int nr_to_scan);
 
-long ion_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
-
-int ion_query_heaps(struct ion_heap_query *query);
+#ifdef CONFIG_ION_CARVEOUT_HEAP
+/**
+ * ion_carveout_heap_create
+ * @base:		base address of carveout memory
+ * @size:		size of carveout memory region
+ *
+ * Creates a carveout ion_heap using the passed in data
+ */
+struct ion_heap *ion_carveout_heap_create(phys_addr_t base, size_t size);
+#else
+static inline struct ion_heap *ion_carveout_heap_create(phys_addr_t base, size_t size)
+{
+	return ERR_PTR(-ENODEV);
+}
+#endif
+
+#ifdef CONFIG_ION_CHUNK_HEAP
+/**
+ * ion_chunk_heap_create
+ * @base:		base address of carveout memory
+ * @size:		size of carveout memory region
+ * @chunk_size:		minimum allocation granularity
+ *
+ * Creates a chunk ion_heap using the passed in data
+ */
+struct ion_heap *ion_chunk_heap_create(phys_addr_t base, size_t size, size_t chunk_size);
+#else
+static inline struct ion_heap *ion_chunk_heap_create(phys_addr_t base, size_t size, size_t chunk_size)
+{
+	return ERR_PTR(-ENODEV);
+}
+#endif
 
 #endif /* _ION_H */
diff -Naur --no-dereference a/drivers/staging/android/ion/ion_heap.c b/drivers/staging/android/ion/ion_heap.c
--- a/drivers/staging/android/ion/ion_heap.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/staging/android/ion/ion_heap.c	2020-06-19 21:00:31.424227716 -0400
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * drivers/staging/android/ion/ion_heap.c
+ * ION Memory Allocator generic heap helpers
  *
  * Copyright (C) 2011 Google, Inc.
  */
@@ -14,6 +14,7 @@
 #include <uapi/linux/sched/types.h>
 #include <linux/scatterlist.h>
 #include <linux/vmalloc.h>
+
 #include "ion.h"
 
 void *ion_heap_map_kernel(struct ion_heap *heap,
@@ -92,6 +93,7 @@
 		if (addr >= vma->vm_end)
 			return 0;
 	}
+
 	return 0;
 }
 
@@ -254,6 +256,7 @@
 		return PTR_ERR_OR_ZERO(heap->task);
 	}
 	sched_setscheduler(heap->task, SCHED_IDLE, &param);
+
 	return 0;
 }
 
@@ -265,8 +268,10 @@
 	int total = 0;
 
 	total = ion_heap_freelist_size(heap) / PAGE_SIZE;
+
 	if (heap->ops->shrink)
 		total += heap->ops->shrink(heap, sc->gfp_mask, 0);
+
 	return total;
 }
 
@@ -295,6 +300,7 @@
 
 	if (heap->ops->shrink)
 		freed += heap->ops->shrink(heap, sc->gfp_mask, to_scan);
+
 	return freed;
 }
 
diff -Naur --no-dereference a/drivers/staging/android/ion/ion-ioctl.c b/drivers/staging/android/ion/ion-ioctl.c
--- a/drivers/staging/android/ion/ion-ioctl.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/staging/android/ion/ion-ioctl.c	1969-12-31 19:00:00.000000000 -0500
@@ -1,98 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (C) 2011 Google, Inc.
- */
-
-#include <linux/kernel.h>
-#include <linux/file.h>
-#include <linux/fs.h>
-#include <linux/uaccess.h>
-
-#include "ion.h"
-
-union ion_ioctl_arg {
-	struct ion_allocation_data allocation;
-	struct ion_heap_query query;
-};
-
-static int validate_ioctl_arg(unsigned int cmd, union ion_ioctl_arg *arg)
-{
-	switch (cmd) {
-	case ION_IOC_HEAP_QUERY:
-		if (arg->query.reserved0 ||
-		    arg->query.reserved1 ||
-		    arg->query.reserved2)
-			return -EINVAL;
-		break;
-	default:
-		break;
-	}
-
-	return 0;
-}
-
-/* fix up the cases where the ioctl direction bits are incorrect */
-static unsigned int ion_ioctl_dir(unsigned int cmd)
-{
-	switch (cmd) {
-	default:
-		return _IOC_DIR(cmd);
-	}
-}
-
-long ion_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
-{
-	int ret = 0;
-	unsigned int dir;
-	union ion_ioctl_arg data;
-
-	dir = ion_ioctl_dir(cmd);
-
-	if (_IOC_SIZE(cmd) > sizeof(data))
-		return -EINVAL;
-
-	/*
-	 * The copy_from_user is unconditional here for both read and write
-	 * to do the validate. If there is no write for the ioctl, the
-	 * buffer is cleared
-	 */
-	if (copy_from_user(&data, (void __user *)arg, _IOC_SIZE(cmd)))
-		return -EFAULT;
-
-	ret = validate_ioctl_arg(cmd, &data);
-	if (ret) {
-		pr_warn_once("%s: ioctl validate failed\n", __func__);
-		return ret;
-	}
-
-	if (!(dir & _IOC_WRITE))
-		memset(&data, 0, sizeof(data));
-
-	switch (cmd) {
-	case ION_IOC_ALLOC:
-	{
-		int fd;
-
-		fd = ion_alloc(data.allocation.len,
-			       data.allocation.heap_id_mask,
-			       data.allocation.flags);
-		if (fd < 0)
-			return fd;
-
-		data.allocation.fd = fd;
-
-		break;
-	}
-	case ION_IOC_HEAP_QUERY:
-		ret = ion_query_heaps(&data.query);
-		break;
-	default:
-		return -ENOTTY;
-	}
-
-	if (dir & _IOC_READ) {
-		if (copy_to_user((void __user *)arg, &data, _IOC_SIZE(cmd)))
-			return -EFAULT;
-	}
-	return ret;
-}
diff -Naur --no-dereference a/drivers/staging/android/ion/ion_page_pool.c b/drivers/staging/android/ion/ion_page_pool.c
--- a/drivers/staging/android/ion/ion_page_pool.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/staging/android/ion/ion_page_pool.c	2020-06-19 21:00:31.424227716 -0400
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * drivers/staging/android/ion/ion_mem_pool.c
+ * ION Memory Allocator page pool helpers
  *
  * Copyright (C) 2011 Google, Inc.
  */
diff -Naur --no-dereference a/drivers/staging/android/ion/ion_system_heap.c b/drivers/staging/android/ion/ion_system_heap.c
--- a/drivers/staging/android/ion/ion_system_heap.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/staging/android/ion/ion_system_heap.c	2020-06-19 21:00:31.424227716 -0400
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * drivers/staging/android/ion/ion_system_heap.c
+ * ION Memory Allocator system heap exporter
  *
  * Copyright (C) 2011 Google, Inc.
  */
@@ -11,9 +11,9 @@
 #include <linux/highmem.h>
 #include <linux/mm.h>
 #include <linux/scatterlist.h>
-#include <linux/seq_file.h>
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
+
 #include "ion.h"
 
 #define NUM_ORDERS ARRAY_SIZE(orders)
@@ -212,29 +212,6 @@
 	.shrink = ion_system_heap_shrink,
 };
 
-static int ion_system_heap_debug_show(struct ion_heap *heap, struct seq_file *s,
-				      void *unused)
-{
-	struct ion_system_heap *sys_heap = container_of(heap,
-							struct ion_system_heap,
-							heap);
-	int i;
-	struct ion_page_pool *pool;
-
-	for (i = 0; i < NUM_ORDERS; i++) {
-		pool = sys_heap->pools[i];
-
-		seq_printf(s, "%d order %u highmem pages %lu total\n",
-			   pool->high_count, pool->order,
-			   (PAGE_SIZE << pool->order) * pool->high_count);
-		seq_printf(s, "%d order %u lowmem pages %lu total\n",
-			   pool->low_count, pool->order,
-			   (PAGE_SIZE << pool->order) * pool->low_count);
-	}
-
-	return 0;
-}
-
 static void ion_system_heap_destroy_pools(struct ion_page_pool **pools)
 {
 	int i;
@@ -260,6 +237,7 @@
 			goto err_create_pool;
 		pools[i] = pool;
 	}
+
 	return 0;
 
 err_create_pool:
@@ -281,7 +259,6 @@
 	if (ion_system_heap_create_pools(heap->pools))
 		goto free_heap;
 
-	heap->heap.debug_show = ion_system_heap_debug_show;
 	return &heap->heap;
 
 free_heap:
@@ -299,6 +276,7 @@
 	heap->name = "ion_system_heap";
 
 	ion_device_add_heap(heap);
+
 	return 0;
 }
 device_initcall(ion_system_heap_create);
@@ -380,6 +358,7 @@
 	heap->ops = &kmalloc_ops;
 	heap->type = ION_HEAP_TYPE_SYSTEM_CONTIG;
 	heap->name = "ion_system_contig_heap";
+
 	return heap;
 }
 
@@ -392,7 +371,7 @@
 		return PTR_ERR(heap);
 
 	ion_device_add_heap(heap);
+
 	return 0;
 }
 device_initcall(ion_system_contig_heap_create);
-
diff -Naur --no-dereference a/drivers/staging/android/ion/ion-ti.c b/drivers/staging/android/ion/ion-ti.c
--- a/drivers/staging/android/ion/ion-ti.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/staging/android/ion/ion-ti.c	2020-06-19 21:00:31.424227716 -0400
@@ -0,0 +1,92 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Texas Instruments ION Driver
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ *	Andrew F. Davis <afd@ti.com>
+ */
+
+#include <linux/of.h>
+#include <linux/of_address.h>
+
+#include "ion.h"
+
+static int __init ti_ion_init(void)
+{
+	struct device_node *node, *mem, *child = NULL;
+	struct resource res;
+	phys_addr_t base;
+	size_t size;
+	struct ion_heap *heap;
+	u32 ion_heap_type, ion_chunk_size;
+	int ret;
+
+	if (!of_have_populated_dt())
+		return -ENODEV;
+
+	node = of_find_compatible_node(NULL, NULL, "ti,ion");
+	if (!node)
+		return -ENODEV;
+
+	if (of_get_available_child_count(node) == 0) {
+		pr_err("No child block node(s) found under ti,ion node\n");
+		return -EINVAL;
+	}
+
+	while ((child = of_get_next_available_child(node, child))) {
+		mem = of_parse_phandle(child, "memory-region", 0);
+		if (!mem) {
+			pr_err("No memory-region phandle for node %pOF\n", child);
+			return -EINVAL;
+		}
+
+		ret = of_address_to_resource(mem, 0, &res);
+		if (ret) {
+			pr_err("Could not get resource address for node %pOF\n", child);
+			return -EINVAL;
+		}
+
+		of_node_put(mem);
+
+		base = res.start;
+		size = resource_size(&res);
+
+		ret = of_property_read_u32(child, "ion-heap-type", &ion_heap_type);
+		if (ret) {
+			pr_err("No ion-heap-type property for node %pOF\n", child);
+			return ret;
+		}
+
+		switch (ion_heap_type) {
+		case ION_HEAP_TYPE_CHUNK:
+			ret = of_property_read_u32(child, "ion-chunk-size", &ion_chunk_size);
+			if (ret) {
+				pr_err("No ion-chunk-size property for node %pOF\n", child);
+				return ret;
+			}
+			heap = ion_chunk_heap_create(base, size, ion_chunk_size);
+			if (IS_ERR(heap)) {
+				pr_err("Could not chunk heap for node %pOF\n", child);
+				return PTR_ERR(heap);
+			}
+			break;
+		case ION_HEAP_TYPE_CARVEOUT:
+			heap = ion_carveout_heap_create(base, size);
+			if (IS_ERR(heap)) {
+				pr_err("Could not carveout heap for node %pOF\n", child);
+				return PTR_ERR(heap);
+			}
+			break;
+		default:
+			pr_err("Unknown heap type for node %pOF\n", child);
+			continue;
+		}
+
+		heap->name = child->name;
+
+		ion_device_add_heap(heap);
+	}
+
+	return 0;
+}
+device_initcall(ti_ion_init);
diff -Naur --no-dereference a/drivers/staging/android/ion/Kconfig b/drivers/staging/android/ion/Kconfig
--- a/drivers/staging/android/ion/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/staging/android/ion/Kconfig	2020-06-19 21:00:31.424227716 -0400
@@ -42,3 +42,9 @@
 	  Choose this option to enable CMA heaps with Ion. This heap is backed
 	  by the Contiguous Memory Allocator (CMA). If your system has these
 	  regions, you should say Y here.
+
+config ION_TI
+	bool "Ion for Texas Instruments"
+	depends on ION
+	help
+	  Choose this option if you wish to use ION on Texas Instruments platforms.
diff -Naur --no-dereference a/drivers/staging/android/ion/Makefile b/drivers/staging/android/ion/Makefile
--- a/drivers/staging/android/ion/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/staging/android/ion/Makefile	2020-06-19 21:00:31.424227716 -0400
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0
-obj-$(CONFIG_ION) +=	ion.o ion-ioctl.o ion_heap.o
+obj-$(CONFIG_ION) += ion.o ion_heap.o
 obj-$(CONFIG_ION_SYSTEM_HEAP) += ion_system_heap.o ion_page_pool.o
 obj-$(CONFIG_ION_CARVEOUT_HEAP) += ion_carveout_heap.o
 obj-$(CONFIG_ION_CHUNK_HEAP) += ion_chunk_heap.o
 obj-$(CONFIG_ION_CMA_HEAP) += ion_cma_heap.o
+obj-$(CONFIG_ION_TI) += ion-ti.o
diff -Naur --no-dereference a/drivers/staging/fbtft/fb_ssd1306.c b/drivers/staging/fbtft/fb_ssd1306.c
--- a/drivers/staging/fbtft/fb_ssd1306.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/staging/fbtft/fb_ssd1306.c	2020-06-19 21:00:31.424227716 -0400
@@ -55,6 +55,8 @@
 		write_reg(par, 0x3F);
 	else if (par->info->var.yres == 48)
 		write_reg(par, 0x2F);
+	else if (par->info->var.yres == 39)
+		write_reg(par, 0x27);	/* https://libstock.mikroe.com/projects/download/1111/2577/1411057038_oled_b_click___e_mikroc_arm.zip */
 	else
 		write_reg(par, 0x1F);
 
@@ -72,16 +74,24 @@
 
 	/* Set Memory Addressing Mode */
 	write_reg(par, 0x20);
-	/* Vertical addressing mode  */
+	/* Vertical Addressing mode  */
 	write_reg(par, 0x01);
 
 	/* Set Segment Re-map */
-	/* column address 127 is mapped to SEG0 */
-	write_reg(par, 0xA0 | 0x1);
+	if (par->info->var.yres == 39)
+		/* no segment re-map */
+		write_reg(par, 0xA0 | 0x0);
+	else
+		/* column address 127 is mapped to SEG0 */
+		write_reg(par, 0xA0 | 0x1);
 
 	/* Set COM Output Scan Direction */
-	/* remapped mode. Scan from COM[N-1] to COM0 */
-	write_reg(par, 0xC8);
+	if (par->info->var.yres == 39)
+		/* no columnt re-map mode. Scan from COM0 to COM[N-1] */
+		write_reg(par, 0xC0 | 0x0);
+	else
+		/* remapped mode. Scan from COM[N-1] to COM0 */
+		write_reg(par, 0xC0 | 0x8);
 
 	/* Set COM Pins Hardware Configuration */
 	write_reg(par, 0xDA);
@@ -91,18 +101,27 @@
 	else if (par->info->var.yres == 48)
 		/* A[4]=1b, Alternative COM pin configuration */
 		write_reg(par, 0x12);
+	else if (par->info->var.yres == 39)
+		/* A[4]=1b, Alternative COM pin configuration */
+		write_reg(par, 0x12);
 	else
 		/* A[4]=0b, Sequential COM pin configuration */
 		write_reg(par, 0x02);
 
 	/* Set Pre-charge Period */
 	write_reg(par, 0xD9);
-	write_reg(par, 0xF1);
+	if (par->info->var.yres == 39)
+		write_reg(par, 0x25);
+	else
+		write_reg(par, 0xF1);
 
 	/* Set VCOMH Deselect Level */
 	write_reg(par, 0xDB);
-	/* according to the datasheet, this value is out of bounds */
-	write_reg(par, 0x40);
+	if (par->info->var.yres == 39)
+		write_reg(par, 0x20);
+	else
+		/* according to the datasheet, this value is out of bounds */
+		write_reg(par, 0x40);
 
 	/* Entire Display ON */
 	/* Resume to RAM content display. Output follows RAM content */
@@ -133,17 +152,35 @@
 	write_reg(par, 0x5);
 }
 
+static void set_addr_win_96x39(struct fbtft_par *par)
+{
+	/* Set Page Address */
+	write_reg(par, 0xB0);
+
+	/* Set Column Address */
+	write_reg(par, 0x21);
+	write_reg(par, 0x00);
+	write_reg(par, 0x5F);
+
+	/* Set Page Address Range */
+	write_reg(par, 0x22);
+	write_reg(par, 0x0);
+	write_reg(par, 0x4);
+}
+
 static void set_addr_win(struct fbtft_par *par, int xs, int ys, int xe, int ye)
 {
-	/* Set Lower Column Start Address for Page Addressing Mode */
+	/* Set Lower Column Start Address for Vertical Addressing Mode */
 	write_reg(par, 0x00 | 0x0);
-	/* Set Higher Column Start Address for Page Addressing Mode */
+	/* Set Higher Column Start Address for Vertical Addressing Mode */
 	write_reg(par, 0x10 | 0x0);
 	/* Set Display Start Line */
 	write_reg(par, 0x40 | 0x0);
 
 	if (par->info->var.xres == 64 && par->info->var.yres == 48)
 		set_addr_win_64x48(par);
+	else if (par->info->var.xres == 96 && par->info->var.yres == 39)
+		set_addr_win_96x39(par);
 }
 
 static int blank(struct fbtft_par *par, bool on)
@@ -187,11 +224,19 @@
 				*buf |= (vmem16[(y * 8 + i) * xres + x] ? 1 : 0) << i;
 			buf++;
 		}
+		if (yres % 8) {
+			*buf = 0x00;
+			for (i = 0; i < (yres - (y * 8)); i++)
+				*buf |= (vmem16[(y * 8 + i) * xres + x] ? 1 : 0) << i;
+			buf++;
+			y++;
+		}
 	}
 
 	/* Write data */
 	gpio_set_value(par->gpio.dc, 1);
-	ret = par->fbtftops.write(par, par->txbuf.buf, xres * yres / 8);
+	ret = par->fbtftops.write(par, par->txbuf.buf, xres * (yres / 8 + (yres % 8 != 0)));
+	//ret = par->fbtftops.write(par, par->txbuf.buf, xres * 128 / 8);
 	if (ret < 0)
 		dev_err(par->info->device, "write failed and returned: %d\n",
 			ret);
diff -Naur --no-dereference a/drivers/staging/fbtft/fbtft_device.c b/drivers/staging/fbtft/fbtft_device.c
--- a/drivers/staging/fbtft/fbtft_device.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/staging/fbtft/fbtft_device.c	2020-06-19 21:00:31.424227716 -0400
@@ -388,6 +388,25 @@
 			}
 		}
 	}, {
+		.name = "mikroe_oledb",
+		.spi = &(struct spi_board_info) {
+			.modalias = "fb_ssd1306",
+			.max_speed_hz = 16000000,
+			.mode = SPI_MODE_0,
+			.platform_data = &(struct fbtft_platform_data) {
+				.display = {
+					.width = 96,
+					.height = 39,
+					.buswidth = 8,
+				},
+				.gpios = (const struct fbtft_gpio []) {
+					{ "reset", 45 },
+					{ "dc", 110 },
+					{},
+				},
+			}
+		}
+	}, {
 		.name = "admatec_c-berry28",
 		.spi = &(struct spi_board_info) {
 			.modalias = "fb_st7789v",
diff -Naur --no-dereference a/drivers/staging/media/zoran/zoran.h b/drivers/staging/media/zoran/zoran.h
--- a/drivers/staging/media/zoran/zoran.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/staging/media/zoran/zoran.h	2020-06-19 21:00:31.424227716 -0400
@@ -60,6 +60,8 @@
 #   define   V4L_MAX_FRAME   32
 #elif VIDEO_MAX_FRAME <= 64
 #   define   V4L_MAX_FRAME   64
+#elif VIDEO_MAX_FRAME <= 128
+#   define   V4L_MAX_FRAME   128
 #else
 #   error   "Too many video frame buffers to handle"
 #endif
diff -Naur --no-dereference a/drivers/thermal/k3_bandgap.c b/drivers/thermal/k3_bandgap.c
--- a/drivers/thermal/k3_bandgap.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/thermal/k3_bandgap.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,289 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * TI Bandgap temperature sensor driver for K3 SoC Family
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/pm_runtime.h>
+#include <linux/err.h>
+#include <linux/types.h>
+#include <linux/spinlock.h>
+#include <linux/of_platform.h>
+#include <linux/io.h>
+#include <linux/workqueue.h>
+#include <linux/thermal.h>
+#include <linux/of.h>
+
+#define K3_VTM_DEVINFO_PWR0_OFFSET		0x4
+#define K3_VTM_DEVINFO_PWR0_CVD_CT_MASK	0xf
+#define K3_VTM_DEVINFO_PWR0_TEMPSENS_CT_MASK	0xf0
+#define K3_VTM_TMPSENS0_CTRL_OFFSET	0x80
+#define K3_VTM_REGS_PER_TS			0x10
+#define K3_VTM_TS_STAT_DTEMP_MASK	0x3ff
+#define K3_VTM_MAX_NUM_TS		8
+#define K3_VTM_TMPSENS_CTRL_CBIASSEL	BIT(0)
+#define K3_VTM_TMPSENS_CTRL_SOC		BIT(5)
+#define K3_VTM_TMPSENS_CTRL_CLRZ		BIT(6)
+#define K3_VTM_TMPSENS_CTRL_CLKON_REQ	BIT(7)
+
+#define K3_VTM_ADC_BEGIN_VAL		540
+#define K3_VTM_ADC_END_VAL		944
+
+static const int k3_adc_to_temp[K3_VTM_ADC_END_VAL - K3_VTM_ADC_BEGIN_VAL
+				+ 1] = {
+	-40000, -40000, -40000, -40000, -39800, -39400, -39000, -38600, -38200,
+	-37800, -37400, -37000, -36600, -36200, -35800, -35300, -34700, -34200,
+	-33800, -33400, -33000, -32600, -32200, -31800, -31400, -31000, -30600,
+	-30200, -29800, -29400, -29000, -28600, -28200, -27700, -27100, -26600,
+	-26200, -25800, -25400, -25000, -24600, -24200, -23800, -23400, -23000,
+	-22600, -22200, -21800, -21400, -21000, -20500, -19900, -19400, -19000,
+	-18600, -18200, -17800, -17400, -17000, -16600, -16200, -15800, -15400,
+	-15000, -14600, -14200, -13800, -13400, -13000, -12500, -11900, -11400,
+	-11000, -10600, -10200, -9800, -9400, -9000, -8600, -8200, -7800, -7400,
+	-7000, -6600, -6200, -5800, -5400, -5000, -4500, -3900, -3400, -3000,
+	-2600, -2200, -1800, -1400, -1000, -600, -200, 200, 600, 1000, 1400,
+	1800, 2200, 2600, 3000, 3400, 3900, 4500, 5000, 5400, 5800, 6200, 6600,
+	7000, 7400, 7800, 8200, 8600, 9000, 9400, 9800, 10200, 10600, 11000,
+	11400, 11800, 12200, 12700, 13300, 13800, 14200, 14600, 15000, 15400,
+	15800, 16200, 16600, 17000, 17400, 17800, 18200, 18600, 19000, 19400,
+	19800, 20200, 20600, 21000, 21400, 21900, 22500, 23000, 23400, 23800,
+	24200, 24600, 25000, 25400, 25800, 26200, 26600, 27000, 27400, 27800,
+	28200, 28600, 29000, 29400, 29800, 30200, 30600, 31000, 31400, 31900,
+	32500, 33000, 33400, 33800, 34200, 34600, 35000, 35400, 35800, 36200,
+	36600, 37000, 37400, 37800, 38200, 38600, 39000, 39400, 39800, 40200,
+	40600, 41000, 41400, 41800, 42200, 42600, 43100, 43700, 44200, 44600,
+	45000, 45400, 45800, 46200, 46600, 47000, 47400, 47800, 48200, 48600,
+	49000, 49400, 49800, 50200, 50600, 51000, 51400, 51800, 52200, 52600,
+	53000, 53400, 53800, 54200, 54600, 55000, 55400, 55900, 56500, 57000,
+	57400, 57800, 58200, 58600, 59000, 59400, 59800, 60200, 60600, 61000,
+	61400, 61800, 62200, 62600, 63000, 63400, 63800, 64200, 64600, 65000,
+	65400, 65800, 66200, 66600, 67000, 67400, 67800, 68200, 68600, 69000,
+	69400, 69800, 70200, 70600, 71000, 71500, 72100, 72600, 73000, 73400,
+	73800, 74200, 74600, 75000, 75400, 75800, 76200, 76600, 77000, 77400,
+	77800, 78200, 78600, 79000, 79400, 79800, 80200, 80600, 81000, 81400,
+	81800, 82200, 82600, 83000, 83400, 83800, 84200, 84600, 85000, 85400,
+	85800, 86200, 86600, 87000, 87400, 87800, 88200, 88600, 89000, 89400,
+	89800, 90200, 90600, 91000, 91400, 91800, 92200, 92600, 93000, 93400,
+	93800, 94200, 94600, 95000, 95400, 95800, 96200, 96600, 97000, 97500,
+	98100, 98600, 99000, 99400, 99800, 100200, 100600, 101000, 101400,
+	101800, 102200, 102600, 103000, 103400, 103800, 104200, 104600, 105000,
+	105400, 105800, 106200, 106600, 107000, 107400, 107800, 108200, 108600,
+	109000, 109400, 109800, 110200, 110600, 111000, 111400, 111800, 112200,
+	112600, 113000, 113400, 113800, 114200, 114600, 115000, 115400, 115800,
+	116200, 116600, 117000, 117400, 117800, 118200, 118600, 119000, 119400,
+	119800, 120200, 120600, 121000, 121400, 121800, 122200, 122600, 123000,
+	123400, 123800, 124200, 124600, 124900, 125000,
+};
+
+struct k3_thermal_data;
+
+struct k3_bandgap {
+	struct device *dev;
+	void __iomem *base;
+	const struct k3_bandgap_data	*conf;
+	spinlock_t lock; /* shields this struct */
+	int ts_cnt;
+	struct k3_thermal_data *ts_data[K3_VTM_MAX_NUM_TS];
+};
+
+/* common data structures */
+struct k3_thermal_data {
+	struct thermal_zone_device *ti_thermal;
+	struct thermal_cooling_device *cool_dev;
+	struct k3_bandgap *bgp;
+	enum thermal_device_mode mode;
+	struct work_struct thermal_wq;
+	int sensor_id;
+	u32 ctrl_offset;
+	u32 stat_offset;
+	int prev_temp;
+};
+
+static inline int k3_bgp_read_temp(struct k3_thermal_data *devdata,
+				   int *temp)
+{
+	struct k3_bandgap *bgp;
+	int dtemp;
+
+	bgp = devdata->bgp;
+	dtemp = readl(bgp->base + devdata->stat_offset);
+	dtemp = dtemp & K3_VTM_TS_STAT_DTEMP_MASK;
+
+	if (dtemp < K3_VTM_ADC_BEGIN_VAL || dtemp > K3_VTM_ADC_END_VAL)
+		return -EINVAL;
+
+	*temp = k3_adc_to_temp[dtemp - K3_VTM_ADC_BEGIN_VAL];
+
+	return 0;
+}
+
+/* thermal zone ops */
+/* Get temperature callback function for thermal zone */
+static int k3_thermal_get_temp(void *devdata, int *temp)
+{
+	struct k3_thermal_data *data = devdata;
+	int ret = 0;
+
+	ret = k3_bgp_read_temp(data, temp);
+	if (ret)
+		return ret;
+
+	data->prev_temp = *temp;
+
+	return ret;
+}
+
+static int k3_thermal_get_trend(void *p, int trip, enum thermal_trend *trend)
+{
+	struct k3_thermal_data *data = p;
+	struct k3_bandgap *bgp;
+	int ret = 0, temp = 0;
+
+	bgp = data->bgp;
+
+	ret = k3_bgp_read_temp(data, &temp);
+	if (ret)
+		return ret;
+
+	if (temp > data->prev_temp)
+		*trend = THERMAL_TREND_RAISING;
+	else if (temp < data->prev_temp)
+		*trend = THERMAL_TREND_DROPPING;
+	else
+		*trend = THERMAL_TREND_STABLE;
+
+	return 0;
+}
+
+static const struct thermal_zone_of_device_ops k3_of_thermal_ops = {
+	.get_temp = k3_thermal_get_temp,
+	.get_trend = k3_thermal_get_trend,
+};
+
+static void k3_thermal_work(struct work_struct *work)
+{
+	struct k3_thermal_data *data = container_of(work,
+					struct k3_thermal_data, thermal_wq);
+
+	thermal_zone_device_update(data->ti_thermal, THERMAL_EVENT_UNSPECIFIED);
+
+	dev_dbg(&data->ti_thermal->device, "updated thermal zone %s\n",
+		data->ti_thermal->type);
+}
+
+static int k3_bandgap_probe(struct platform_device *pdev)
+{
+	int ret = 0, cnt, val, id, reg_cnt = 0;
+	struct resource *res;
+	struct device *dev = &pdev->dev;
+	struct k3_bandgap *bgp;
+	struct k3_thermal_data *data;
+
+	bgp = devm_kzalloc(&pdev->dev, sizeof(*bgp), GFP_KERNEL);
+	if (!bgp)
+		return -ENOMEM;
+
+	bgp->dev = dev;
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	bgp->base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(bgp->base))
+		return PTR_ERR(bgp->base);
+
+	pm_runtime_enable(dev);
+	ret = pm_runtime_get_sync(dev);
+	if (ret < 0) {
+		pm_runtime_put_noidle(dev);
+		pm_runtime_disable(dev);
+		return ret;
+	}
+
+	/* Get the sensor count in the VTM */
+	val = readl(bgp->base + K3_VTM_DEVINFO_PWR0_OFFSET);
+	cnt = val & K3_VTM_DEVINFO_PWR0_TEMPSENS_CT_MASK;
+	cnt >>= __ffs(K3_VTM_DEVINFO_PWR0_TEMPSENS_CT_MASK);
+	bgp->ts_cnt = cnt;
+
+	data = devm_kcalloc(bgp->dev, cnt, sizeof(*data), GFP_KERNEL);
+	if (!data) {
+		ret = -ENOMEM;
+		goto err_alloc;
+	}
+
+	/* Register the thermal sensors */
+	for (id = 0; id < cnt; id++) {
+		data[id].sensor_id = id;
+		data[id].bgp = bgp;
+		data[id].ctrl_offset = K3_VTM_TMPSENS0_CTRL_OFFSET +
+					id * K3_VTM_REGS_PER_TS;
+		data[id].stat_offset = data[id].ctrl_offset + 0x8;
+		INIT_WORK(&data[id].thermal_wq, k3_thermal_work);
+
+		val = readl(data[id].bgp->base + data[id].ctrl_offset);
+		val |= (K3_VTM_TMPSENS_CTRL_SOC |
+			K3_VTM_TMPSENS_CTRL_CLRZ |
+			K3_VTM_TMPSENS_CTRL_CLKON_REQ);
+		val &= ~K3_VTM_TMPSENS_CTRL_CBIASSEL;
+		writel(val, data[id].bgp->base + data[id].ctrl_offset);
+
+		bgp->ts_data[id] = &data[id];
+		data[id].ti_thermal =
+		devm_thermal_zone_of_sensor_register(bgp->dev, id,
+						     &data[id],
+						     &k3_of_thermal_ops);
+		if (IS_ERR(data[id].ti_thermal)) {
+			dev_err(bgp->dev, "thermal zone device is NULL\n");
+			ret = PTR_ERR(data[id].ti_thermal);
+			goto err_alloc;
+		}
+
+		reg_cnt++;
+
+		/* Initialize Previous temp */
+		k3_thermal_get_temp(&data[id], &data[id].prev_temp);
+	}
+
+	platform_set_drvdata(pdev, bgp);
+
+	return 0;
+
+err_alloc:
+	pm_runtime_put_sync(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+
+	return ret;
+}
+
+static int k3_bandgap_remove(struct platform_device *pdev)
+{
+	pm_runtime_put_sync(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+
+	return 0;
+}
+
+static const struct of_device_id of_k3_bandgap_match[] = {
+	{
+		.compatible = "ti,am654-vtm",
+	},
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, of_k3_bandgap_match);
+
+static struct platform_driver k3_bandgap_sensor_driver = {
+	.probe = k3_bandgap_probe,
+	.remove = k3_bandgap_remove,
+	.driver = {
+		.name = "k3-soc-thermal",
+		.of_match_table	= of_k3_bandgap_match,
+	},
+};
+
+module_platform_driver(k3_bandgap_sensor_driver);
+
+MODULE_DESCRIPTION("K3 bandgap temperature sensor driver");
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("J Keerthy <j-keerthy@ti.com>");
diff -Naur --no-dereference a/drivers/thermal/Kconfig b/drivers/thermal/Kconfig
--- a/drivers/thermal/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/thermal/Kconfig	2020-06-19 21:00:31.424227716 -0400
@@ -222,6 +222,18 @@
 	  cpufreq is used as the cooling device to throttle CPUs when the
 	  passive trip is crossed.
 
+config K3_THERMAL
+	bool "Texas Instruments K3 thermal support"
+	depends on THERMAL
+	depends on ARCH_K3 || COMPILE_TEST
+	help
+	  If you say yes here you get thermal support for the Texas Instruments
+	  K3 SoC family. The current chip supported is:
+	   - AM654
+
+	  This includes temperature reading functionality and also trend
+	  computation.
+
 config MAX77620_THERMAL
 	tristate "Temperature sensor driver for Maxim MAX77620 PMIC"
 	depends on MFD_MAX77620
diff -Naur --no-dereference a/drivers/thermal/Makefile b/drivers/thermal/Makefile
--- a/drivers/thermal/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/thermal/Makefile	2020-06-19 21:00:31.428227716 -0400
@@ -27,6 +27,7 @@
 # devfreq cooling
 thermal_sys-$(CONFIG_DEVFREQ_THERMAL) += devfreq_cooling.o
 
+obj-$(CONFIG_K3_THERMAL)	+= k3_bandgap.o
 # platform thermal drivers
 obj-y				+= broadcom/
 obj-$(CONFIG_QCOM_SPMI_TEMP_ALARM)	+= qcom-spmi-temp-alarm.o
diff -Naur --no-dereference a/drivers/tty/serial/8250/8250_omap.c b/drivers/tty/serial/8250/8250_omap.c
--- a/drivers/tty/serial/8250/8250_omap.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/tty/serial/8250/8250_omap.c	2020-06-19 21:00:31.428227716 -0400
@@ -8,6 +8,7 @@
  *
  */
 
+#include <linux/clk.h>
 #include <linux/device.h>
 #include <linux/io.h>
 #include <linux/module.h>
@@ -39,6 +40,7 @@
  * The same errata is applicable to AM335x and DRA7x processors too.
  */
 #define UART_ERRATA_CLOCK_DISABLE	(1 << 3)
+#define	UART_HAS_EFR2			BIT(4)
 
 #define OMAP_UART_FCR_RX_TRIG		6
 #define OMAP_UART_FCR_TX_TRIG		4
@@ -92,6 +94,10 @@
 #define OMAP_UART_REV_52 0x0502
 #define OMAP_UART_REV_63 0x0603
 
+/* Enhanced features register 2 */
+#define UART_OMAP_EFR2			0x23
+#define UART_OMAP_EFR2_TIMEOUT_BEHAVE	BIT(6)
+
 struct omap8250_priv {
 	int line;
 	u8 habit;
@@ -104,6 +110,8 @@
 	u8 delayed_restore;
 	u16 quot;
 
+	u8 tx_trigger;
+	u8 rx_trigger;
 	bool is_suspending;
 	int wakeirq;
 	int wakeups_enabled;
@@ -117,6 +125,17 @@
 	bool throttled;
 };
 
+struct omap8250_dma_params {
+	u32 rx_size;
+	u8 rx_trigger;
+	u8 tx_trigger;
+};
+
+struct omap8250_platdata {
+	struct omap8250_dma_params *dma_params;
+	u8 habit;
+};
+
 #ifdef CONFIG_SERIAL_8250_DMA
 static void omap_8250_rx_dma_flush(struct uart_8250_port *p);
 #else
@@ -292,8 +311,8 @@
 	serial_out(up, UART_TI752_TCR, OMAP_UART_TCR_RESTORE(16) |
 			OMAP_UART_TCR_HALT(52));
 	serial_out(up, UART_TI752_TLR,
-		   TRIGGER_TLR_MASK(TX_TRIGGER) << UART_TI752_TLR_TX |
-		   TRIGGER_TLR_MASK(RX_TRIGGER) << UART_TI752_TLR_RX);
+		   TRIGGER_TLR_MASK(priv->tx_trigger) << UART_TI752_TLR_TX |
+		   TRIGGER_TLR_MASK(priv->rx_trigger) << UART_TI752_TLR_RX);
 
 	serial_out(up, UART_LCR, 0);
 
@@ -432,8 +451,8 @@
 	 * This is because threshold and trigger values are the same.
 	 */
 	up->fcr = UART_FCR_ENABLE_FIFO;
-	up->fcr |= TRIGGER_FCR_MASK(TX_TRIGGER) << OMAP_UART_FCR_TX_TRIG;
-	up->fcr |= TRIGGER_FCR_MASK(RX_TRIGGER) << OMAP_UART_FCR_RX_TRIG;
+	up->fcr |= TRIGGER_FCR_MASK(priv->tx_trigger) << OMAP_UART_FCR_TX_TRIG;
+	up->fcr |= TRIGGER_FCR_MASK(priv->rx_trigger) << OMAP_UART_FCR_RX_TRIG;
 
 	priv->scr = OMAP_UART_SCR_RX_TRIG_GRANU1_MASK | OMAP_UART_SCR_TX_EMPTY |
 		OMAP_UART_SCR_TX_TRIG_GRANU1_MASK;
@@ -646,7 +665,7 @@
 		priv->wer |= OMAP_UART_TX_WAKEUP_EN;
 	serial_out(up, UART_OMAP_WER, priv->wer);
 
-	if (up->dma)
+	if (up->dma && !(priv->habit & UART_HAS_EFR2))
 		up->dma->rx_dma(up);
 
 	pm_runtime_mark_last_busy(port->dev);
@@ -671,6 +690,8 @@
 	pm_runtime_get_sync(port->dev);
 
 	serial_out(up, UART_OMAP_WER, 0);
+	if (priv->habit & UART_HAS_EFR2)
+		serial_out(up, UART_OMAP_EFR2, 0x0);
 
 	up->ier = 0;
 	serial_out(up, UART_IER, 0);
@@ -767,6 +788,8 @@
 	struct omap8250_priv	*priv = p->port.private_data;
 	struct uart_8250_dma    *dma = p->dma;
 	struct tty_port         *tty_port = &p->port.state->port;
+	struct dma_chan		*rxchan = dma->rxchan;
+	dma_cookie_t		cookie;
 	struct dma_tx_state     state;
 	int                     count;
 	unsigned long		flags;
@@ -777,11 +800,31 @@
 	if (!dma->rx_running)
 		goto unlock;
 
+	cookie = dma->rx_cookie;
 	dma->rx_running = 0;
-	dmaengine_tx_status(dma->rxchan, dma->rx_cookie, &state);
+	dmaengine_tx_status(rxchan, cookie, &state);
 
-	count = dma->rx_size - state.residue;
+	count = dma->rx_size - state.residue + state.in_flight_bytes;
+	if (count < dma->rx_size) {
+		dmaengine_terminate_async(rxchan);
 
+		/*
+		 * Poll for teardown to complete which guarantees in
+		 * flight data is drained.
+		 */
+		if (state.in_flight_bytes) {
+			int poll_count = 25;
+
+			while (dmaengine_tx_status(rxchan, cookie, NULL) &&
+			       poll_count--)
+				cpu_relax();
+
+			if (!poll_count)
+				dev_err(p->port.dev, "teardown incomplete\n");
+		}
+	}
+	if (!count)
+		goto unlock;
 	ret = tty_insert_flip_string(tty_port, dma->rx_buf, count);
 
 	p->port.icount.rx += ret;
@@ -813,7 +856,7 @@
 		return;
 	}
 	__dma_rx_do_complete(p);
-	if (!priv->throttled)
+	if (!priv->throttled && !(priv->habit & UART_HAS_EFR2))
 		omap_8250_rx_dma(p);
 
 	spin_unlock_irqrestore(&p->port.lock, flags);
@@ -843,7 +886,6 @@
 	spin_unlock_irqrestore(&priv->rx_dma_lock, flags);
 
 	__dma_rx_do_complete(p);
-	dmaengine_terminate_all(dma->rxchan);
 }
 
 static int omap_8250_rx_dma(struct uart_8250_port *p)
@@ -1039,6 +1081,44 @@
 	return omap_8250_rx_dma(up);
 }
 
+static unsigned char omap_8250_handle_rx_dma(struct uart_8250_port *up,
+					     u8 iir, unsigned char status)
+{
+	if ((status & (UART_LSR_DR | UART_LSR_BI)) &&
+	    (iir & UART_IIR_RDI)) {
+		if (handle_rx_dma(up, iir)) {
+			status = serial8250_rx_chars(up, status);
+			omap_8250_rx_dma(up);
+		}
+	}
+
+	return status;
+}
+
+static unsigned char am654_8250_handle_rx_dma(struct uart_8250_port *up,
+					      u8 iir, unsigned char status)
+{
+	if ((status & (UART_LSR_DR | UART_LSR_BI)) && (iir & UART_IIR_RDI)) {
+		omap_8250_rx_dma(up);
+		serial_out(up, UART_OMAP_EFR2, UART_OMAP_EFR2_TIMEOUT_BEHAVE);
+	} else if ((iir & 0x3f) == UART_IIR_RX_TIMEOUT) {
+		/*
+		 * Disable RX timeout, read IIR to clear
+		 * current timeout condition, clear EFR2 to
+		 * periodic timeouts, re-enable interrupts.
+		 */
+		up->ier &= ~(UART_IER_RLSI | UART_IER_RDI);
+		serial_out(up, UART_IER, up->ier);
+		omap_8250_rx_dma_flush(up);
+		serial_in(up, UART_IIR);
+		serial_out(up, UART_OMAP_EFR2, 0x0);
+		up->ier |= UART_IER_RLSI | UART_IER_RDI;
+		serial_out(up, UART_IER, up->ier);
+	}
+
+	return status;
+}
+
 /*
  * This is mostly serial8250_handle_irq(). We have a slightly different DMA
  * hoook for RX/TX and need different logic for them in the ISR. Therefore we
@@ -1047,6 +1127,7 @@
 static int omap_8250_dma_handle_irq(struct uart_port *port)
 {
 	struct uart_8250_port *up = up_to_u8250p(port);
+	struct omap8250_priv *priv = up->port.private_data;
 	unsigned char status;
 	unsigned long flags;
 	u8 iir;
@@ -1056,19 +1137,18 @@
 	iir = serial_port_in(port, UART_IIR);
 	if (iir & UART_IIR_NO_INT) {
 		serial8250_rpm_put(up);
-		return 0;
+		return IRQ_HANDLED;
 	}
 
 	spin_lock_irqsave(&port->lock, flags);
 
 	status = serial_port_in(port, UART_LSR);
 
-	if (status & (UART_LSR_DR | UART_LSR_BI)) {
-		if (handle_rx_dma(up, iir)) {
-			status = serial8250_rx_chars(up, status);
-			omap_8250_rx_dma(up);
-		}
-	}
+	if (priv->habit & UART_HAS_EFR2)
+		status = am654_8250_handle_rx_dma(up, iir, status);
+	else
+		status = omap_8250_handle_rx_dma(up, iir, status);
+
 	serial8250_modem_status(up);
 	if (status & UART_LSR_THRE && up->dma->tx_err) {
 		if (uart_tx_stopped(&up->port) ||
@@ -1110,18 +1190,41 @@
 	return 0;
 }
 
-static const u8 omap4_habit = UART_ERRATA_CLOCK_DISABLE;
-static const u8 am3352_habit = OMAP_DMA_TX_KICK | UART_ERRATA_CLOCK_DISABLE;
-static const u8 dra742_habit = UART_ERRATA_CLOCK_DISABLE;
+static struct omap8250_dma_params am654_dma = {
+	.rx_size = SZ_2K,
+	.rx_trigger = 1,
+	.tx_trigger = TX_TRIGGER,
+};
+
+static struct omap8250_dma_params am33xx_dma = {
+	.rx_size = RX_TRIGGER,
+	.rx_trigger = RX_TRIGGER,
+	.tx_trigger = TX_TRIGGER,
+};
+
+static struct omap8250_platdata am654_platdata = {
+	.dma_params	= &am654_dma,
+	.habit		= UART_HAS_EFR2,
+};
+
+static struct omap8250_platdata am33xx_platdata = {
+	.dma_params	= &am33xx_dma,
+	.habit		= OMAP_DMA_TX_KICK | UART_ERRATA_CLOCK_DISABLE,
+};
+
+static struct omap8250_platdata omap4_platdata = {
+	.dma_params	= &am33xx_dma,
+	.habit		= UART_ERRATA_CLOCK_DISABLE,
+};
 
 static const struct of_device_id omap8250_dt_ids[] = {
-	{ .compatible = "ti,am654-uart" },
+	{ .compatible = "ti,am654-uart", .data = &am654_platdata, },
 	{ .compatible = "ti,omap2-uart" },
 	{ .compatible = "ti,omap3-uart" },
-	{ .compatible = "ti,omap4-uart", .data = &omap4_habit, },
-	{ .compatible = "ti,am3352-uart", .data = &am3352_habit, },
-	{ .compatible = "ti,am4372-uart", .data = &am3352_habit, },
-	{ .compatible = "ti,dra742-uart", .data = &dra742_habit, },
+	{ .compatible = "ti,omap4-uart", .data = &omap4_platdata, },
+	{ .compatible = "ti,am3352-uart", .data = &am33xx_platdata, },
+	{ .compatible = "ti,am4372-uart", .data = &am33xx_platdata, },
+	{ .compatible = "ti,dra742-uart", .data = &omap4_platdata, },
 	{},
 };
 MODULE_DEVICE_TABLE(of, omap8250_dt_ids);
@@ -1130,7 +1233,9 @@
 {
 	struct resource *regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	struct resource *irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
+	struct device_node *np = pdev->dev.of_node;
 	struct omap8250_priv *priv;
+	const struct omap8250_platdata *pdata;
 	struct uart_8250_port up;
 	int ret;
 	void __iomem *membase;
@@ -1190,27 +1295,31 @@
 	up.port.unthrottle = omap_8250_unthrottle;
 	up.port.rs485_config = omap_8250_rs485_config;
 
-	if (pdev->dev.of_node) {
-		const struct of_device_id *id;
-
-		ret = of_alias_get_id(pdev->dev.of_node, "serial");
-
-		of_property_read_u32(pdev->dev.of_node, "clock-frequency",
-				     &up.port.uartclk);
-		priv->wakeirq = irq_of_parse_and_map(pdev->dev.of_node, 1);
-
-		id = of_match_device(of_match_ptr(omap8250_dt_ids), &pdev->dev);
-		if (id && id->data)
-			priv->habit |= *(u8 *)id->data;
-	} else {
-		ret = pdev->id;
-	}
+	ret = of_alias_get_id(np, "serial");
 	if (ret < 0) {
-		dev_err(&pdev->dev, "failed to get alias/pdev id\n");
+		dev_err(&pdev->dev, "failed to get alias\n");
 		return ret;
 	}
 	up.port.line = ret;
 
+	if (of_property_read_u32(np, "clock-frequency", &up.port.uartclk)) {
+		struct clk *clk;
+
+		clk = devm_clk_get(&pdev->dev, NULL);
+		if (IS_ERR(clk)) {
+			if (PTR_ERR(clk) == -EPROBE_DEFER)
+				return -EPROBE_DEFER;
+		} else {
+			up.port.uartclk = clk_get_rate(clk);
+		}
+	}
+
+	priv->wakeirq = irq_of_parse_and_map(np, 1);
+
+	pdata = of_device_get_match_data(&pdev->dev);
+	if (pdata)
+		priv->habit |= pdata->habit;
+
 	if (!up.port.uartclk) {
 		up.port.uartclk = DEFAULT_CLK_SPEED;
 		dev_warn(&pdev->dev,
@@ -1237,25 +1346,38 @@
 
 	omap_serial_fill_features_erratas(&up, priv);
 	up.port.handle_irq = omap8250_no_handle_irq;
+	priv->rx_trigger = RX_TRIGGER;
+	priv->tx_trigger = TX_TRIGGER;
 #ifdef CONFIG_SERIAL_8250_DMA
-	if (pdev->dev.of_node) {
-		/*
-		 * Oh DMA support. If there are no DMA properties in the DT then
-		 * we will fall back to a generic DMA channel which does not
-		 * really work here. To ensure that we do not get a generic DMA
-		 * channel assigned, we have the the_no_dma_filter_fn() here.
-		 * To avoid "failed to request DMA" messages we check for DMA
-		 * properties in DT.
-		 */
-		ret = of_property_count_strings(pdev->dev.of_node, "dma-names");
-		if (ret == 2) {
-			up.dma = &priv->omap8250_dma;
-			priv->omap8250_dma.fn = the_no_dma_filter_fn;
-			priv->omap8250_dma.tx_dma = omap_8250_tx_dma;
-			priv->omap8250_dma.rx_dma = omap_8250_rx_dma;
-			priv->omap8250_dma.rx_size = RX_TRIGGER;
-			priv->omap8250_dma.rxconf.src_maxburst = RX_TRIGGER;
-			priv->omap8250_dma.txconf.dst_maxburst = TX_TRIGGER;
+	/*
+	 * Oh DMA support. If there are no DMA properties in the DT then
+	 * we will fall back to a generic DMA channel which does not
+	 * really work here. To ensure that we do not get a generic DMA
+	 * channel assigned, we have the the_no_dma_filter_fn() here.
+	 * To avoid "failed to request DMA" messages we check for DMA
+	 * properties in DT.
+	 */
+	ret = of_property_count_strings(np, "dma-names");
+	if (ret == 2) {
+		struct omap8250_dma_params *dma_params = NULL;
+
+		up.dma = &priv->omap8250_dma;
+		up.dma->fn = the_no_dma_filter_fn;
+		up.dma->tx_dma = omap_8250_tx_dma;
+		up.dma->rx_dma = omap_8250_rx_dma;
+		if (pdata)
+			dma_params = pdata->dma_params;
+
+		if (dma_params) {
+			up.dma->rx_size = dma_params->rx_size;
+			up.dma->rxconf.src_maxburst = dma_params->rx_trigger;
+			up.dma->txconf.dst_maxburst = dma_params->tx_trigger;
+			priv->rx_trigger = dma_params->rx_trigger;
+			priv->tx_trigger = dma_params->tx_trigger;
+		} else {
+			up.dma->rx_size = RX_TRIGGER;
+			up.dma->rxconf.src_maxburst = RX_TRIGGER;
+			up.dma->txconf.dst_maxburst = TX_TRIGGER;
 		}
 	}
 #endif
@@ -1454,7 +1576,7 @@
 	if (omap8250_lost_context(up))
 		omap8250_restore_regs(up);
 
-	if (up->dma && up->dma->rxchan)
+	if (up->dma && up->dma->rxchan && !(priv->habit & UART_HAS_EFR2))
 		omap_8250_rx_dma(up);
 
 	priv->latency = priv->calc_latency;
@@ -1494,10 +1616,10 @@
 	}
 
 	add_preferred_console("ttyS", idx, options);
-	pr_err("WARNING: Your 'console=ttyO%d' has been replaced by 'ttyS%d'\n",
+	pr_info("WARNING: Your 'console=ttyO%d' has been replaced by 'ttyS%d'\n",
 	       idx, idx);
-	pr_err("This ensures that you still see kernel messages. Please\n");
-	pr_err("update your kernel commandline.\n");
+	pr_info("This ensures that you still see kernel messages. Please\n");
+	pr_info("update your kernel commandline.\n");
 	return 0;
 }
 console_initcall(omap8250_console_fixup);
diff -Naur --no-dereference a/drivers/tty/serial/8250/Kconfig b/drivers/tty/serial/8250/Kconfig
--- a/drivers/tty/serial/8250/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/tty/serial/8250/Kconfig	2020-06-19 21:00:31.428227716 -0400
@@ -375,7 +375,7 @@
 
 config SERIAL_8250_OMAP
 	tristate "Support for OMAP internal UART (8250 based driver)"
-	depends on SERIAL_8250 && ARCH_OMAP2PLUS
+	depends on SERIAL_8250 && (ARCH_OMAP2PLUS || ARCH_K3)
 	help
 	  If you have a machine based on an Texas Instruments OMAP CPU you
 	  can enable its onboard serial ports by enabling this option.
diff -Naur --no-dereference a/drivers/tty/serial/Kconfig b/drivers/tty/serial/Kconfig
--- a/drivers/tty/serial/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/tty/serial/Kconfig	2020-06-19 21:00:31.428227716 -0400
@@ -1528,6 +1528,16 @@
 	  Say 'Y' here if you wish to use Actions Semiconductor S500/S900 UART
 	  as the system console.
 
+config SERIAL_PRU_SUART
+	tristate "TI PRU Software UART support"
+	depends on PRU_REMOTEPROC && !ARCH_K3
+	select SERIAL_CORE
+	help
+	  This driver is to support software UART over PRUSS, provided through
+	  PRU firmware.
+	  Say 'Y' here if you wish to use PRU software based UART.
+	  Otherwise, say 'N'.
+
 endmenu
 
 config SERIAL_MCTRL_GPIO
diff -Naur --no-dereference a/drivers/tty/serial/Makefile b/drivers/tty/serial/Makefile
--- a/drivers/tty/serial/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/tty/serial/Makefile	2020-06-19 21:00:31.428227716 -0400
@@ -89,6 +89,7 @@
 obj-$(CONFIG_SERIAL_PIC32)	+= pic32_uart.o
 obj-$(CONFIG_SERIAL_MPS2_UART)	+= mps2-uart.o
 obj-$(CONFIG_SERIAL_OWL)	+= owl-uart.o
+obj-$(CONFIG_SERIAL_PRU_SUART)	+= pru_suart.o
 
 # GPIOLIB helpers for modem control lines
 obj-$(CONFIG_SERIAL_MCTRL_GPIO)	+= serial_mctrl_gpio.o
diff -Naur --no-dereference a/drivers/tty/serial/pru_suart.c b/drivers/tty/serial/pru_suart.c
--- a/drivers/tty/serial/pru_suart.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/tty/serial/pru_suart.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,753 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * UART driver for PRU software based UARTs
+ *
+ * Copyright (C) 2019 by Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Bin Liu <b-liu@ti.com>
+ */
+
+#include <linux/module.h>
+#include <linux/of_irq.h>
+#include <linux/of_platform.h>
+#include <linux/pruss.h>
+#include <linux/remoteproc.h>
+#include <linux/serial_core.h>
+#include <linux/tty_flip.h>
+
+#define PSUART_NAME		"ttySPRU"
+
+#define PSUART_FW_MAGIC_NUMBER	0x54524155	/* "UART" */
+#define MAX_UART_PORTS		3
+#define DRV_TOTAL_PORTS		(MAX_UART_PORTS * 2)	/* 2 PRUs */
+#define PORT_MMR_BASE		0x14
+#define PORT_MMR_LEN		0x14
+#define FIFO_BASE		0x100
+#define FIFO_SIZE		256
+#define BPC			2	/* bytes per char */
+
+/* hw flow control threshold */
+#define RX_FIFO_THRES_SHIFT	16
+#define RX_FIFO_THRES		((FIFO_SIZE - 16) << RX_FIFO_THRES_SHIFT)
+#define RX_FIFO_THRES_MASK	(0xff << RX_FIFO_THRES_SHIFT)
+
+/* global registers */
+#define PSUART_FW_MAGIC		0x00
+#define PSUART_FW_VERSION		0x08
+#define PSUART_FW_GCFG			0x50
+#define PSUART_FW_INITED		BIT(1)
+
+/* uart port registers */
+#define PPORT_ENABLE		0x00
+#define PPORT_STATUS		0x01
+#define PPORT_ENABLED		((BIT(1)) | (BIT(0)))
+
+#define PPORT_CFG		0x04
+#define PPORT_FIFO_POS		0x08
+#define PPORT_TXFIFO_POS	0x08	/* 16-bit register */
+#define PPORT_TXFIFO_WRITE	0x09	/* 8-bit register */
+#define PPORT_RXFIFO_POS	0x0a	/* 16-bit register */
+#define PPORT_RXFIFO_READ	0x0a	/* 8-bit register */
+#define PPORT_TX_CFG		0x0c
+#define PPORT_TX_INTR_CTRL	0x0d	/* 8-bit register */
+#define PPORT_RX_CFG		0x10
+#define PPORT_RX_INTR_CTRL	0x11	/* 8-bit register */
+
+/* PPORT_CFG register bits */
+#define PPORT_CFG_HWFLOW_EN	BIT(12)
+#define PPORT_CFG_PARADD	BIT(10)
+#define PPORT_CFG_PAR_EN	BIT(9)
+#define PPORT_CFG_CSTOPB	BIT(8)
+
+#define PPORT_CFG_CS6		0x1
+#define PPORT_CFG_CS7		0x2
+#define PPORT_CFG_CS8		0x3
+#define PPORT_CFG_CSSHIFT	4
+
+#define PPORT_CFG_B600		0x1
+#define PPORT_CFG_B1200		0x2
+#define PPORT_CFG_B2400		0x3
+#define PPORT_CFG_B4800		0x4
+#define PPORT_CFG_B9600		0x5
+#define PPORT_CFG_B19200	0x7
+#define PPORT_CFG_B38400	0x9
+#define PPORT_CFG_B57600	0xa
+#define PPORT_CFG_B115200	0xb
+#define PPORT_CFG_BSHIFT	0
+#define PPORT_MAX_BAUD		115200
+
+/* rx character info bits */
+#define PPORT_RX_CHAR_PE	BIT(15)
+#define PPORT_RX_CHAR_FE	BIT(14)
+
+struct psuart_port {
+	struct uart_port port;
+	void __iomem *mbase;
+	void __iomem *tx_fifo;
+	void __iomem *rx_fifo;
+};
+
+struct pru_suart {
+	struct device *dev;
+	struct rproc *pru;
+	struct pruss *pruss;
+	int pru_id;
+	struct pruss_mem_region mem;
+};
+
+struct pport_pins {
+	u8 tx;
+	u8 rx;
+	u8 cts;
+	u8 rts;
+};
+
+union fifo_pos {
+	u16 pos;
+	struct {
+		u8 tail;	/* read pointer */
+		u8 head;	/* write pointer */
+	} s;
+};
+
+static struct psuart_port pports[DRV_TOTAL_PORTS];
+
+static inline struct psuart_port *up_to_pport(struct uart_port *up)
+{
+	return  container_of(up, struct psuart_port, port);
+}
+
+static inline u32 psuart_readl(struct pru_suart *pu, u32 reg)
+{
+	return readl(pu->mem.va + reg);
+}
+
+static inline void psuart_writel(struct pru_suart *pu, u32 reg, u32 val)
+{
+	writel(val, pu->mem.va + reg);
+}
+
+static inline u8 pport_readb(struct psuart_port *pp, u32 reg)
+{
+	return readb(pp->mbase + reg);
+}
+
+static inline void pport_writeb(struct psuart_port *pp, u32 reg, u8 val)
+{
+	writeb(val, pp->mbase + reg);
+}
+
+static inline u16 pport_readw(struct psuart_port *pp, u32 reg)
+{
+	return readw(pp->mbase + reg);
+}
+
+static inline u32 pport_readl(struct psuart_port *pp, u32 reg)
+{
+	return readl(pp->mbase + reg);
+}
+
+static inline void pport_writel(struct psuart_port *pp, u32 reg, u32 val)
+{
+	writel(val, pp->mbase + reg);
+}
+
+static inline int pport_is_fifo_empty(union fifo_pos *pos)
+{
+	return pos->s.head == pos->s.tail;
+}
+
+static void pport_rx_chars(struct psuart_port *pp)
+{
+	struct uart_port *up = &pp->port;
+	union fifo_pos fifo;
+	u16 ch;
+	int i, total;
+
+	fifo.pos = pport_readw(pp, PPORT_RXFIFO_POS);
+	total = CIRC_CNT(fifo.s.head, fifo.s.tail, FIFO_SIZE) / BPC;
+	if (!total)
+		return;
+
+	for (i = 0; i < total; i++) {
+		ch = readw(pp->rx_fifo + fifo.s.tail);
+		fifo.s.tail += BPC;
+
+		if (ch & PPORT_RX_CHAR_PE)
+			up->icount.parity++;
+		if (ch & PPORT_RX_CHAR_FE)
+			up->icount.frame++;
+
+		uart_insert_char(up, 0, 0, ch, TTY_NORMAL);
+	}
+
+	up->icount.rx += total;
+	pport_writeb(pp, PPORT_RXFIFO_READ, fifo.s.tail);
+	tty_flip_buffer_push(&up->state->port);
+}
+
+static void pport_tx_chars(struct psuart_port *pp)
+{
+	struct uart_port *up = &pp->port;
+	union fifo_pos fifo;
+	struct circ_buf *xmit = &up->state->xmit;
+	int count;
+
+	fifo.pos = pport_readw(pp, PPORT_TXFIFO_POS);
+	count = CIRC_SPACE(fifo.s.head, fifo.s.tail, FIFO_SIZE) / BPC;
+	if (!count)
+		return;
+
+	if (up->x_char) {
+		writew(up->x_char, pp->tx_fifo + fifo.s.head);
+		fifo.s.head += BPC;
+		pport_writeb(pp, PPORT_TXFIFO_WRITE, fifo.s.head);
+		up->icount.tx++;
+		up->x_char = 0;
+		return;
+	}
+
+	if (uart_circ_empty(xmit) || uart_tx_stopped(up))
+		return;
+
+	do {
+		writew(xmit->buf[xmit->tail], pp->tx_fifo + fifo.s.head);
+		fifo.s.head += BPC;
+		xmit->tail = (xmit->tail + 1) & (UART_XMIT_SIZE - 1);
+		up->icount.tx++;
+		if (uart_circ_empty(xmit))
+			break;
+	} while (--count > 0);
+
+	pport_writeb(pp, PPORT_TXFIFO_WRITE, fifo.s.head);
+
+	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
+		uart_write_wakeup(up);
+}
+
+static irqreturn_t pport_handle_irq(int irq, void *pru_port)
+{
+	struct psuart_port *pp = (struct psuart_port *)pru_port;
+	int rx, tx;
+
+	rx = pport_readb(pp, PPORT_RX_INTR_CTRL);
+	if (rx) {
+		pport_writeb(pp, PPORT_RX_INTR_CTRL, 0);
+		pport_rx_chars(pp);
+	}
+
+	tx = pport_readb(pp, PPORT_TX_INTR_CTRL);
+	if (tx) {
+		pport_tx_chars(pp);
+		pport_writeb(pp, PPORT_TX_INTR_CTRL, 1);
+	}
+
+	if (rx)
+		pport_writeb(pp, PPORT_RX_INTR_CTRL, 1);
+
+	return IRQ_HANDLED;
+}
+
+static unsigned int pport_tx_empty(struct uart_port *up)
+{
+	struct psuart_port *pp = up_to_pport(up);
+	union fifo_pos tx;
+
+	tx.pos = pport_readw(pp, PPORT_TXFIFO_POS);
+
+	return pport_is_fifo_empty(&tx) ? TIOCSER_TEMT : 0;
+}
+
+static unsigned int pport_get_mctrl(struct uart_port *up)
+{
+	return up->mctrl;
+}
+
+/* the hardware flow control doesn't require any software assistance */
+static void pport_set_mctrl(struct uart_port *up, unsigned int mctrl)
+{
+};
+
+static void pport_stop_tx(struct uart_port *up)
+{
+	struct psuart_port *pp = up_to_pport(up);
+
+	pport_writeb(pp, PPORT_TX_INTR_CTRL, 0);
+}
+
+static void pport_start_tx(struct uart_port *up)
+{
+	struct psuart_port *pp = up_to_pport(up);
+
+	pport_writeb(pp, PPORT_TX_INTR_CTRL, 0);
+	pport_tx_chars(pp);
+	pport_writeb(pp, PPORT_TX_INTR_CTRL, 1);
+}
+
+static void pport_stop_rx(struct uart_port *up)
+{
+	struct psuart_port *pp = up_to_pport(up);
+
+	pport_writeb(pp, PPORT_RX_INTR_CTRL, 0);
+}
+
+static void pport_start_rx(struct uart_port *up)
+{
+	struct psuart_port *pp = up_to_pport(up);
+
+	pport_writeb(pp, PPORT_RX_INTR_CTRL, 0);
+	pport_rx_chars(pp);
+	pport_writeb(pp, PPORT_RX_INTR_CTRL, 1);
+}
+
+static void pport_throttle(struct uart_port *up)
+{
+	pport_stop_rx(up);
+}
+
+static void pport_unthrottle(struct uart_port *up)
+{
+	pport_start_rx(up);
+}
+
+/* line break is not supported */
+static void pport_break_ctl(struct uart_port *up, int break_state)
+{
+}
+
+/* software flow control currently not supported */
+static void pport_set_termios(struct uart_port *up, struct ktermios *termios,
+			struct ktermios *old)
+{
+	struct psuart_port *pp = up_to_pport(up);
+	tcflag_t cflag;
+	unsigned int baud;
+	u32 cfg;
+
+	/* reset all fields except hw flow control settings */
+	cfg = pport_readl(pp, PPORT_CFG);
+	cfg &= RX_FIFO_THRES_MASK;
+
+	cflag = termios->c_cflag;
+	switch (cflag & CSIZE) {
+	case CS5:
+		break;
+	case CS6:
+		cfg |= (PPORT_CFG_CS6 << PPORT_CFG_CSSHIFT);
+		break;
+	case CS7:
+		cfg |= (PPORT_CFG_CS7 << PPORT_CFG_CSSHIFT);
+		break;
+	case CS8:
+	default:
+		cfg |= (PPORT_CFG_CS8 << PPORT_CFG_CSSHIFT);
+		break;
+	}
+
+	if (cflag & PARENB) {
+		cfg |= PPORT_CFG_PAR_EN;
+		cfg |= (cflag & PARODD) ? PPORT_CFG_PARADD : 0;
+	}
+
+	cfg |= (cflag & CSTOPB) ? PPORT_CFG_CSTOPB : 0;
+
+	if (cflag & CRTSCTS) {
+		cfg |= PPORT_CFG_HWFLOW_EN;
+		/*
+		 * Setting TIOCM_CTS here to prevent core uart_change_speed()
+		 * calls ops->stop_tx() when hw flow control is enabled.
+		 */
+		up->mctrl |= TIOCM_CTS;
+	} else {
+		up->mctrl &= ~TIOCM_CTS;
+	}
+
+	switch (cflag & CBAUD) {
+	case B300:
+		break;
+	case B600:
+		cfg |= (PPORT_CFG_B600 << PPORT_CFG_BSHIFT);
+		break;
+	case B1200:
+		cfg |= (PPORT_CFG_B1200 << PPORT_CFG_BSHIFT);
+		break;
+	case B2400:
+		cfg |= (PPORT_CFG_B2400 << PPORT_CFG_BSHIFT);
+		break;
+	case B4800:
+		cfg |= (PPORT_CFG_B4800 << PPORT_CFG_BSHIFT);
+		break;
+	case B9600:
+	default:
+		cfg |= (PPORT_CFG_B9600 << PPORT_CFG_BSHIFT);
+		break;
+	case B19200:
+		cfg |= (PPORT_CFG_B19200 << PPORT_CFG_BSHIFT);
+		break;
+	case B38400:
+		cfg |= (PPORT_CFG_B38400 << PPORT_CFG_BSHIFT);
+		break;
+	case B57600:
+		cfg |= (PPORT_CFG_B57600 << PPORT_CFG_BSHIFT);
+		break;
+	case B115200:
+		cfg |= (PPORT_CFG_B115200 << PPORT_CFG_BSHIFT);
+		break;
+	}
+
+	baud = uart_get_baud_rate(up, termios, old, 0, PPORT_MAX_BAUD);
+	uart_update_timeout(up, cflag, baud);
+
+	pport_writeb(pp, PPORT_ENABLE, 0);
+	pport_writel(pp, PPORT_CFG, cfg);
+	pport_writeb(pp, PPORT_ENABLE, 1);
+}
+
+static int pport_startup(struct uart_port *up)
+{
+	struct psuart_port *pp = up_to_pport(up);
+	int timeout = 100;
+
+	if (up->flags & UPF_HARD_FLOW) {
+		/*
+		 * CTS is a input-only pin in the firmware, so AUTOCTS is
+		 * not supported.
+		 */
+		up->status |= UPSTAT_AUTORTS;
+		pport_writel(pp, PPORT_CFG, RX_FIFO_THRES);
+	}
+
+	pport_writeb(pp, PPORT_ENABLE, 1);
+	while (!(pport_readb(pp, PPORT_STATUS) & PPORT_ENABLED)) {
+		if (--timeout < 0) {
+			dev_err(up->dev, "failed to enable port\n");
+			return 1;
+		}
+	}
+
+	pport_start_rx(up);
+	return 0;
+};
+
+static void pport_shutdown(struct uart_port *up)
+{
+	struct psuart_port *pp = up_to_pport(up);
+
+	pport_writeb(pp, PPORT_ENABLE, 0);
+};
+
+static void pport_config_port(struct uart_port *up, int flags)
+{
+	up->type = PORT_PSUART;
+	up->flags |= UPF_HARD_FLOW;
+}
+
+/* rs485 is unsupported */
+static int pport_rs485_config(struct uart_port *up, struct serial_rs485 *rs485)
+{
+	return rs485->flags & SER_RS485_ENABLED ? -EOPNOTSUPP : 0;
+}
+
+static const struct uart_ops psuart_port_ops = {
+	.tx_empty	= pport_tx_empty,
+	.get_mctrl	= pport_get_mctrl,
+	.set_mctrl	= pport_set_mctrl,
+	.stop_tx	= pport_stop_tx,
+	.start_tx	= pport_start_tx,
+	.throttle	= pport_throttle,
+	.unthrottle	= pport_unthrottle,
+	.stop_rx	= pport_stop_rx,
+	.break_ctl	= pport_break_ctl,
+	.startup	= pport_startup,
+	.shutdown	= pport_shutdown,
+	.set_termios	= pport_set_termios,
+	.config_port	= pport_config_port,
+};
+
+static struct uart_driver psuart_port_drv = {
+	.owner		= THIS_MODULE,
+	.driver_name	= "PRU-SUART",
+	.dev_name	= PSUART_NAME,
+	.nr		= DRV_TOTAL_PORTS,
+};
+
+static int pport_config_port_pins(struct psuart_port *pp,
+			struct device_node *np)
+{
+	struct device *dev = pp->port.dev;
+	struct pport_pins *pins;
+	int nr_pins;
+	int ret;
+	u32 val;
+
+	nr_pins = of_property_count_u8_elems(np, "ti,pru-suart-pins");
+
+	/* CTS/RTS pins are optional */
+	if (nr_pins != 2 && nr_pins != 4) {
+		dev_err(dev, "unexpected number of pins\n");
+		return -EINVAL;
+	}
+	pp->port.flags = (nr_pins == 4) ?  UPF_HARD_FLOW : 0;
+
+	pins = devm_kmalloc(dev, sizeof(*pins), GFP_KERNEL);
+	if (!pins)
+		return -ENOMEM;
+
+	/* set non-configured pin value to 0xff */
+	memset(pins, 0xff, sizeof(*pins));
+	ret = of_property_read_u8_array(np, "ti,pru-suart-pins",
+			(u8 *)pins, nr_pins);
+	if (ret)
+		return ret;
+
+	ret = of_property_read_u32(np, "interrupts", &val);
+	if (ret)
+		return ret;
+
+	val = pins->cts << 24 | pins->tx << 16 | (val & 0xff);
+	pport_writel(pp, PPORT_TX_CFG, val);
+
+	val = pins->rts << 24 | pins->rx << 16 | (val & 0xff);
+	pport_writel(pp, PPORT_RX_CFG, val);
+
+	return 0;
+}
+
+static void psuart_init_port(struct pru_suart *pu, struct device_node *np,
+			int index)
+{
+	struct psuart_port *pp;
+	int port_id;
+	int ret;
+
+	port_id = pu->pru_id * MAX_UART_PORTS + index;
+	pp = &pports[port_id];
+	if (pp->mbase) {
+		dev_err(pu->dev, "Error: port[%d] is already initialized\n",
+				index);
+		return;
+	}
+
+	pp->mbase = pu->mem.va + PORT_MMR_BASE + PORT_MMR_LEN * index;
+	pp->tx_fifo = pu->mem.va + FIFO_BASE + FIFO_SIZE * 2 * index;
+	pp->rx_fifo = pp->tx_fifo + FIFO_SIZE;
+
+	pp->port.dev = pu->dev;
+	pp->port.type = PORT_PSUART;
+	pp->port.iotype = UPIO_MEM;
+	pp->port.fifosize = FIFO_SIZE / BPC;
+	pp->port.ops = &psuart_port_ops;
+	pp->port.line = port_id;
+	pp->port.rs485_config = pport_rs485_config;
+
+	ret = pport_config_port_pins(pp, np);
+	if (ret)
+		return;
+
+	ret = of_irq_get(np, 0);
+	if (ret < 0) {
+		dev_err(pu->dev, "port[%d]: failed to get irq (%d)\n",
+				index, ret);
+		return;
+	}
+	pp->port.irq = ret;
+
+	ret = request_irq(pp->port.irq, pport_handle_irq,
+			  IRQF_TRIGGER_HIGH | IRQF_ONESHOT,
+			  dev_name(pp->port.dev), pp);
+	if (ret) {
+		dev_err(pu->dev, "port[%d]: failed to to request irq (%d)\n",
+				index, ret);
+		return;
+	}
+
+	ret = uart_add_one_port(&psuart_port_drv, &pp->port);
+	if (ret) {
+		dev_err(pu->dev, "adding port[%d] failed (%d)\n", index, ret);
+		free_irq(pp->port.irq, pp);
+	}
+}
+
+static int psuart_init_pruss(struct device_node *np, struct pru_suart *pu)
+{
+	u32 reg;
+	int ret = 0;
+
+	pu->pru = pru_rproc_get(np, 0);
+	if (IS_ERR(pu->pru)) {
+		ret = PTR_ERR(pu->pru);
+		if (ret != -EPROBE_DEFER)
+			dev_err(pu->dev, "failed to get pru (%d)\n", ret);
+		return ret;
+	}
+
+	pu->pruss = pruss_get(pu->pru);
+	if (IS_ERR(pu->pruss)) {
+		ret = PTR_ERR(pu->pruss);
+		dev_err(pu->dev, "failed to get pruss handle (%d)\n", ret);
+		goto put_pru;
+	}
+
+	pu->pru_id = pru_rproc_get_id(pu->pru);
+	if (pu->pru_id < 0) {
+		dev_err(pu->dev, "failed to get pru id (%d)\n", pu->pru_id);
+		ret = -EINVAL;
+		goto put_pruss;
+	}
+
+	if (pu->pru_id > 1) {
+		dev_err(pu->dev, "invalid pru id (%d)\n", pu->pru_id);
+		ret = -EINVAL;
+		goto put_pruss;
+	}
+
+	ret = pruss_request_mem_region(pu->pruss,
+			pu->pru_id ? PRUSS_MEM_DRAM1 : PRUSS_MEM_DRAM0,
+			&pu->mem);
+	if (ret) {
+		dev_err(pu->dev, "failed to get pruss mem region (%d)\n", ret);
+		goto put_pruss;
+	}
+
+	/* clear the mem region before firmware runs by rproc_boot() */
+	memset_io(pu->mem.va, 0, pu->mem.size);
+
+	ret = rproc_boot(pu->pru);
+	if (ret) {
+		dev_err(pu->dev, "failed to boot pru (%d)\n", ret);
+		goto put_mem;
+	}
+
+	reg = psuart_readl(pu, PSUART_FW_MAGIC);
+	if (reg != PSUART_FW_MAGIC_NUMBER) {
+		dev_err(pu->dev, "invalid firmware magic number\n");
+		ret = -EINVAL;
+		goto put_rproc;
+	}
+
+	reg = psuart_readl(pu, PSUART_FW_VERSION);
+	if (reg > 0x01000000) {
+		dev_err(pu->dev, "unsupported firmware version(0x%x)\n",
+				reg);
+		ret = -EINVAL;
+		goto put_rproc;
+	}
+
+	reg = psuart_readl(pu, PSUART_FW_GCFG);
+	if (!(reg & PSUART_FW_INITED)) {
+		dev_err(pu->dev, "failed to initialize firmware\n");
+		ret = -EINVAL;
+		goto put_rproc;
+	}
+
+	return ret;
+
+put_rproc:
+	rproc_shutdown(pu->pru);
+put_mem:
+	pruss_release_mem_region(pu->pruss, &pu->mem);
+put_pruss:
+	pruss_put(pu->pruss);
+put_pru:
+	pru_rproc_put(pu->pru);
+
+	return ret;
+}
+
+static const struct of_device_id psuart_dt_ids[] = {
+	{ .compatible = "ti,pru-soft-uart", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, psuart_dt_ids);
+
+static int psuart_probe(struct platform_device *pdev)
+{
+	struct pru_suart *pu;
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct device_node *child;
+	int id, ret;
+
+	if (!np)
+		return -ENODEV;	/* we don't support non DT */
+
+	pu = devm_kzalloc(dev, sizeof(*pu), GFP_KERNEL);
+	if (!pu)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, pu);
+	pu->dev = dev;
+
+	ret = psuart_init_pruss(np, pu);
+	if (ret < 0)
+		return -ENODEV;
+
+	for_each_available_child_of_node(np, child) {
+		ret = of_property_read_u32(child, "reg", &id);
+
+		if (ret || id < 0 || id >= MAX_UART_PORTS)
+			continue;
+
+		psuart_init_port(pu, child, id);
+	}
+	return 0;
+}
+
+static int psuart_remove(struct platform_device *pdev)
+{
+	struct pru_suart *pu = platform_get_drvdata(pdev);
+	struct psuart_port *pp;
+	int i;
+
+	for (i = 0; i < MAX_UART_PORTS; i++) {
+		pp = &pports[pu->pru_id * MAX_UART_PORTS + i];
+		if (!pp->mbase)
+			continue;
+
+		uart_remove_one_port(&psuart_port_drv, &pp->port);
+		pp->mbase = NULL;
+		free_irq(pp->port.irq, pp);
+	}
+
+	rproc_shutdown(pu->pru);
+	pruss_release_mem_region(pu->pruss, &pu->mem);
+	pruss_put(pu->pruss);
+	pru_rproc_put(pu->pru);
+
+	return 0;
+}
+
+static struct platform_driver psuart_driver = {
+	.probe = psuart_probe,
+	.remove = psuart_remove,
+	.driver = {
+		.name = "pru_suart",
+		.of_match_table = psuart_dt_ids,
+	},
+};
+
+static int __init psuart_init(void)
+{
+	int ret;
+
+	ret = uart_register_driver(&psuart_port_drv);
+	if (ret)
+		return ret;
+
+	ret = platform_driver_register(&psuart_driver);
+	if (ret)
+		uart_unregister_driver(&psuart_port_drv);
+
+	return ret;
+}
+module_init(psuart_init);
+
+static void __exit psuart_exit(void)
+{
+	platform_driver_unregister(&psuart_driver);
+	uart_unregister_driver(&psuart_port_drv);
+}
+module_exit(psuart_exit);
+
+MODULE_AUTHOR("Bin Liu <b-liu@ti.com>");
+MODULE_DESCRIPTION("PRU SW UART Driver");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/drivers/uio/Kconfig b/drivers/uio/Kconfig
--- a/drivers/uio/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/uio/Kconfig	2020-06-19 21:00:31.428227716 -0400
@@ -142,6 +142,14 @@
 	  To compile this driver as a module, choose M here: the module
 	  will be called uio_pruss.
 
+config UIO_PRUSS_SHMEM
+	tristate "Texas Instruments PRUSS_SHMEM driver"
+	help
+	  PRUSS SHMEM driver for AM3X and AM5X devices
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called uio_pruss_shmem.
+
 config UIO_MF624
 	tristate "Humusoft MF624 DAQ PCI card driver"
 	depends on PCI
diff -Naur --no-dereference a/drivers/uio/Makefile b/drivers/uio/Makefile
--- a/drivers/uio/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/uio/Makefile	2020-06-19 21:00:31.428227716 -0400
@@ -8,6 +8,7 @@
 obj-$(CONFIG_UIO_PCI_GENERIC)	+= uio_pci_generic.o
 obj-$(CONFIG_UIO_NETX)	+= uio_netx.o
 obj-$(CONFIG_UIO_PRUSS)         += uio_pruss.o
+obj-$(CONFIG_UIO_PRUSS_SHMEM)	+= uio_pruss_shmem.o
 obj-$(CONFIG_UIO_MF624)         += uio_mf624.o
 obj-$(CONFIG_UIO_FSL_ELBC_GPCM)	+= uio_fsl_elbc_gpcm.o
 obj-$(CONFIG_UIO_HV_GENERIC)	+= uio_hv_generic.o
diff -Naur --no-dereference a/drivers/uio/uio_pruss.c b/drivers/uio/uio_pruss.c
--- a/drivers/uio/uio_pruss.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/uio/uio_pruss.c	2020-06-19 21:00:31.428227716 -0400
@@ -26,14 +26,22 @@
 #include <linux/dma-mapping.h>
 #include <linux/sizes.h>
 #include <linux/slab.h>
+#include <linux/of.h>
+#ifdef CONFIG_ARCH_DAVINCI_DA850
 #include <linux/genalloc.h>
+#else
+#include <linux/pm_runtime.h>
+#endif
 
 #define DRV_NAME "pruss_uio"
 #define DRV_VERSION "1.0"
 
+#ifdef CONFIG_ARCH_DAVINCI_DA850
 static int sram_pool_sz = SZ_16K;
 module_param(sram_pool_sz, int, 0);
-MODULE_PARM_DESC(sram_pool_sz, "sram pool size to allocate ");
+MODULE_PARM_DESC(sram_pool_sz, "sram pool size to allocate");
+#endif
+
 
 static int extram_pool_sz = SZ_256K;
 module_param(extram_pool_sz, int, 0);
@@ -61,22 +69,23 @@
 #define PINTC_HIER	0x1500
 
 struct uio_pruss_dev {
-	struct uio_info *info;
-	struct clk *pruss_clk;
-	dma_addr_t sram_paddr;
+	struct uio_info info[MAX_PRUSS_EVT];
 	dma_addr_t ddr_paddr;
 	void __iomem *prussio_vaddr;
-	unsigned long sram_vaddr;
 	void *ddr_vaddr;
-	unsigned int hostirq_start;
 	unsigned int pintc_base;
+#ifdef CONFIG_ARCH_DAVINCI_DA850
+	struct clk *pruss_clk;
 	struct gen_pool *sram_pool;
+	dma_addr_t sram_paddr;
+	unsigned long sram_vaddr;
+#endif
 };
 
 static irqreturn_t pruss_handler(int irq, struct uio_info *info)
 {
 	struct uio_pruss_dev *gdev = info->priv;
-	int intr_bit = (irq - gdev->hostirq_start + 2);
+	int intr_bit = 2 + (info - gdev->info);
 	int val, intr_mask = (1 << intr_bit);
 	void __iomem *base = gdev->prussio_vaddr + gdev->pintc_base;
 	void __iomem *intren_reg = base + PINTC_HIER;
@@ -94,53 +103,50 @@
 
 static void pruss_cleanup(struct device *dev, struct uio_pruss_dev *gdev)
 {
-	int cnt;
-	struct uio_info *p = gdev->info;
+	int i;
 
-	for (cnt = 0; cnt < MAX_PRUSS_EVT; cnt++, p++) {
-		uio_unregister_device(p);
-		kfree(p->name);
+	for (i = 0; i < MAX_PRUSS_EVT; i++) {
+		uio_unregister_device(&gdev->info[i]);
+		kfree(gdev->info[i].name);
 	}
 	iounmap(gdev->prussio_vaddr);
 	if (gdev->ddr_vaddr) {
 		dma_free_coherent(dev, extram_pool_sz, gdev->ddr_vaddr,
 			gdev->ddr_paddr);
 	}
+#ifdef CONFIG_ARCH_DAVINCI_DA850
 	if (gdev->sram_vaddr)
 		gen_pool_free(gdev->sram_pool,
 			      gdev->sram_vaddr,
 			      sram_pool_sz);
-	kfree(gdev->info);
 	clk_disable(gdev->pruss_clk);
 	clk_put(gdev->pruss_clk);
+#else
+	pm_runtime_put(dev);
+	pm_runtime_disable(dev);
+#endif
 	kfree(gdev);
 }
 
 static int pruss_probe(struct platform_device *pdev)
 {
-	struct uio_info *p;
 	struct uio_pruss_dev *gdev;
 	struct resource *regs_prussio;
 	struct device *dev = &pdev->dev;
-	int ret, cnt, i, len;
+	int ret, i, len;
 	struct uio_pruss_pdata *pdata = dev_get_platdata(dev);
 
 	gdev = kzalloc(sizeof(struct uio_pruss_dev), GFP_KERNEL);
 	if (!gdev)
 		return -ENOMEM;
 
-	gdev->info = kcalloc(MAX_PRUSS_EVT, sizeof(*p), GFP_KERNEL);
-	if (!gdev->info) {
-		ret = -ENOMEM;
-		goto err_free_gdev;
-	}
-
+#ifdef CONFIG_ARCH_DAVINCI_DA850
 	/* Power on PRU in case its not done as part of boot-loader */
 	gdev->pruss_clk = clk_get(dev, "pruss");
 	if (IS_ERR(gdev->pruss_clk)) {
 		dev_err(dev, "Failed to get clock\n");
 		ret = PTR_ERR(gdev->pruss_clk);
-		goto err_free_info;
+		goto err_free_gdev;
 	}
 
 	ret = clk_enable(gdev->pruss_clk);
@@ -148,6 +154,15 @@
 		dev_err(dev, "Failed to enable clock\n");
 		goto err_clk_put;
 	}
+#else
+	pm_runtime_enable(dev);
+	ret = pm_runtime_get_sync(dev);
+	if (ret < 0) {
+		dev_err(dev, "pm_runtime_get_sync() failed\n");
+		pm_runtime_disable(dev);
+		goto err_free_gdev;
+	}
+#endif
 
 	regs_prussio = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	if (!regs_prussio) {
@@ -162,6 +177,18 @@
 		goto err_clk_disable;
 	}
 
+	if (dev->of_node) {
+		ret = of_property_read_u32(dev->of_node,
+					   "ti,pintc-offset",
+					   &gdev->pintc_base);
+		if (ret < 0) {
+			dev_err(dev, "Can't parse ti,pintc-offset property\n");
+			goto err_clk_disable;
+		}
+	} else
+		gdev->pintc_base = pdata->pintc_base;
+
+#ifdef CONFIG_ARCH_DAVINCI_DA850
 	if (pdata->sram_pool) {
 		gdev->sram_pool = pdata->sram_pool;
 		gdev->sram_vaddr =
@@ -173,9 +200,10 @@
 			goto err_clk_disable;
 		}
 	}
+#endif
 
 	gdev->ddr_vaddr = dma_alloc_coherent(dev, extram_pool_sz,
-				&(gdev->ddr_paddr), GFP_KERNEL | GFP_DMA);
+				&gdev->ddr_paddr, GFP_KERNEL | GFP_DMA);
 	if (!gdev->ddr_vaddr) {
 		dev_err(dev, "Could not allocate external memory\n");
 		ret = -ENOMEM;
@@ -185,32 +213,48 @@
 	len = resource_size(regs_prussio);
 	gdev->prussio_vaddr = ioremap(regs_prussio->start, len);
 	if (!gdev->prussio_vaddr) {
-		dev_err(dev, "Can't remap PRUSS I/O  address range\n");
+		dev_err(dev, "Can't remap PRUSS I/O address range\n");
 		ret = -ENOMEM;
 		goto err_free_ddr_vaddr;
 	}
 
-	gdev->pintc_base = pdata->pintc_base;
-	gdev->hostirq_start = platform_get_irq(pdev, 0);
+	for (i = 0; i < MAX_PRUSS_EVT; i++) {
+		struct uio_info *p = &gdev->info[i];
 
-	for (cnt = 0, p = gdev->info; cnt < MAX_PRUSS_EVT; cnt++, p++) {
+		p->mem[0].name = "pruss";
 		p->mem[0].addr = regs_prussio->start;
 		p->mem[0].size = resource_size(regs_prussio);
 		p->mem[0].memtype = UIO_MEM_PHYS;
 
+		/* note: some userspace code uses hardcoded mem indices... */
+#ifdef CONFIG_ARCH_DAVINCI_DA850
+		p->mem[1].name = "sram";
 		p->mem[1].addr = gdev->sram_paddr;
 		p->mem[1].size = sram_pool_sz;
 		p->mem[1].memtype = UIO_MEM_PHYS;
 
+		p->mem[2].name = "ddr";
 		p->mem[2].addr = gdev->ddr_paddr;
 		p->mem[2].size = extram_pool_sz;
 		p->mem[2].memtype = UIO_MEM_PHYS;
+#else
+		p->mem[1].name = "ddr";
+		p->mem[1].addr = gdev->ddr_paddr;
+		p->mem[1].size = extram_pool_sz;
+		p->mem[1].memtype = UIO_MEM_PHYS;
+#endif
 
-		p->name = kasprintf(GFP_KERNEL, "pruss_evt%d", cnt);
+		ret = platform_get_irq(pdev, i);
+		if (ret < 0) {
+			dev_err(dev, "Failed to obtain irq %d (%d)\n", i, ret);
+			goto err_unloop;
+		}
+
+		p->name = kasprintf(GFP_KERNEL, "pruss_evt%d", i);
 		p->version = DRV_VERSION;
 
 		/* Register PRUSS IRQ lines */
-		p->irq = gdev->hostirq_start + cnt;
+		p->irq = ret;
 		p->handler = pruss_handler;
 		p->priv = gdev;
 
@@ -225,23 +269,27 @@
 	return 0;
 
 err_unloop:
-	for (i = 0, p = gdev->info; i < cnt; i++, p++) {
-		uio_unregister_device(p);
-		kfree(p->name);
+	while( --i >= 0 ) {
+		uio_unregister_device(&gdev->info[i]);
+		kfree(gdev->info[i].name);
 	}
 	iounmap(gdev->prussio_vaddr);
 err_free_ddr_vaddr:
 	dma_free_coherent(dev, extram_pool_sz, gdev->ddr_vaddr,
 			  gdev->ddr_paddr);
 err_free_sram:
+#ifdef CONFIG_ARCH_DAVINCI_DA850
 	if (pdata->sram_pool)
 		gen_pool_free(gdev->sram_pool, gdev->sram_vaddr, sram_pool_sz);
 err_clk_disable:
 	clk_disable(gdev->pruss_clk);
 err_clk_put:
 	clk_put(gdev->pruss_clk);
-err_free_info:
-	kfree(gdev->info);
+#else
+err_clk_disable:
+	pm_runtime_put(dev);
+	pm_runtime_disable(dev);
+#endif
 err_free_gdev:
 	kfree(gdev);
 
@@ -256,11 +304,19 @@
 	return 0;
 }
 
+static const struct of_device_id pruss_dt_ids[] = {
+	{ .compatible = "ti,pruss-v1" },
+	{ .compatible = "ti,pruss-v2" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, pruss_dt_ids);
+
 static struct platform_driver pruss_driver = {
 	.probe = pruss_probe,
 	.remove = pruss_remove,
 	.driver = {
 		   .name = DRV_NAME,
+		   .of_match_table = pruss_dt_ids,
 		   },
 };
 
diff -Naur --no-dereference a/drivers/uio/uio_pruss_shmem.c b/drivers/uio/uio_pruss_shmem.c
--- a/drivers/uio/uio_pruss_shmem.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/uio/uio_pruss_shmem.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,163 @@
+/*
+ * Programmable Real-Time Unit Sub System (PRUSS) UIO driver (uio_pruss)
+ *
+ * This driver exports PRUSS host event out interrupts and PRUSS, L3 RAM,
+ * and DDR RAM to user space for applications interacting with PRUSS firmware
+ *
+ * Copyright (C) 2010-11 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation version 2.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#include <linux/device.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/platform_device.h>
+#include <linux/uio_driver.h>
+#include <linux/io.h>
+#include <linux/clk.h>
+#include <linux/dma-mapping.h>
+#include <linux/sizes.h>
+#include <linux/slab.h>
+#include <linux/genalloc.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+
+#define DRV_NAME "pruss_uio_shmem"
+#define DRV_VERSION "1.0"
+
+struct uio_pruss_shmem_dev {
+	struct uio_info *info;
+	void __iomem *prussio_vaddr;
+};
+
+static void pruss_shmem_cleanup(struct device *dev, struct uio_pruss_shmem_dev *gdev)
+{
+	struct uio_info *p = gdev->info;
+
+	uio_unregister_device(p);
+	iounmap(gdev->prussio_vaddr);
+	kfree(gdev->info);
+	kfree(gdev);
+}
+
+static int pruss_shmem_probe(struct platform_device *pdev)
+{
+	struct uio_pruss_shmem_dev *gdev;
+	struct resource *regs_prussio;
+	struct resource res;
+	struct device *dev = &pdev->dev;
+	int ret, len;
+	struct uio_info *p;
+
+	dev_info(dev, "Allocating gdev\n");
+	gdev = kzalloc(sizeof(struct uio_pruss_shmem_dev), GFP_KERNEL);
+	if (!gdev)
+		return -ENOMEM;
+
+	dev_info(dev, "Allocating info\n");
+	gdev->info = kzalloc(sizeof(struct uio_info), GFP_KERNEL);
+	if (!gdev->info) {
+		ret = -ENOMEM;
+		goto err_free_gdev;
+	}
+
+	dev_info(dev, "Requesting resource\n");
+	if (dev->of_node) {
+		ret = of_address_to_resource(dev->of_node, 0, &res);
+		if (IS_ERR_VALUE(ret)) {
+			dev_err(dev, "Failed to parse DT reg\n");
+			goto err_free_info;
+		}
+		regs_prussio = &res;
+	} else {
+		regs_prussio = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+		if (!regs_prussio) {
+			dev_err(dev, "No PRUSS I/O resource specified\n");
+			goto err_free_info;
+		}
+	}
+	if (!regs_prussio) {
+		dev_err(dev, "No PRUSS I/O resource specified\n");
+		ret = -EIO;
+		goto err_free_info;
+	}
+	if (!regs_prussio->start) {
+		dev_err(dev, "Invalid memory resource\n");
+		ret = -EIO;
+		goto err_free_info;
+	}
+	dev_info(dev, "Mapping resource\n");
+	len = resource_size(regs_prussio);
+	gdev->prussio_vaddr = ioremap(regs_prussio->start, len);
+	if (!gdev->prussio_vaddr) {
+		dev_err(dev, "Can't remap PRUSS I/O  address range\n");
+		ret = -ENOMEM;
+		goto err_free_info;
+	}
+
+	p = gdev->info;
+	p->mem[0].addr = regs_prussio->start;
+	p->mem[0].size = resource_size(regs_prussio);
+	p->mem[0].memtype = UIO_MEM_PHYS;
+
+	p->name = "pruss_shmem";
+	p->version = DRV_VERSION;
+
+	dev_info(dev, "Registering with uio driver\n");
+	ret = uio_register_device(dev, p);
+	if (ret < 0) {
+		goto err_free_register;
+	}
+
+	dev_info(dev, "Saving platform data\n");
+	platform_set_drvdata(pdev, gdev);
+	return 0;
+
+err_free_register:
+	uio_unregister_device(p);
+	iounmap(gdev->prussio_vaddr);
+err_free_info:
+	kfree(gdev->info);
+err_free_gdev:
+	kfree(gdev);
+
+	return ret;
+}
+
+static int pruss_shmem_remove(struct platform_device *dev)
+{
+	struct uio_pruss_shmem_dev *gdev = platform_get_drvdata(dev);
+
+	pruss_shmem_cleanup(&dev->dev, gdev);
+	return 0;
+}
+
+static const struct of_device_id pruss_shmem_dt_ids[] = {
+	{ .compatible = "ti,pruss-shmem", .data = NULL, },
+	{},
+};
+MODULE_DEVICE_TABLE(of, pruss_shmem_dt_ids);
+
+static struct platform_driver pruss_shmem_driver = {
+	.probe = pruss_shmem_probe,
+	.remove = pruss_shmem_remove,
+	.driver = {
+		   .name = DRV_NAME,
+		   .of_match_table = pruss_shmem_dt_ids,
+		   },
+};
+
+module_platform_driver(pruss_shmem_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_VERSION(DRV_VERSION);
+MODULE_AUTHOR("Amit Chatterjee <amit.chatterjee@ti.com>");
+MODULE_AUTHOR("Pratheesh Gangadhar <pratheesh@ti.com>");
+MODULE_AUTHOR("Jason Kridner <jdk@ti.com>");
diff -Naur --no-dereference a/drivers/usb/cdns3/cdns3-pci-wrap.c b/drivers/usb/cdns3/cdns3-pci-wrap.c
--- a/drivers/usb/cdns3/cdns3-pci-wrap.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/usb/cdns3/cdns3-pci-wrap.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,204 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Cadence USBSS PCI Glue driver
+ *
+ * Copyright (C) 2018-2019 Cadence.
+ *
+ * Author: Pawel Laszczak <pawell@cadence.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+#include <linux/dma-mapping.h>
+#include <linux/slab.h>
+
+struct cdns3_wrap {
+	struct platform_device *plat_dev;
+	struct resource dev_res[6];
+	int devfn;
+};
+
+#define RES_IRQ_HOST_ID		0
+#define RES_IRQ_PERIPHERAL_ID	1
+#define RES_IRQ_OTG_ID		2
+#define RES_HOST_ID		3
+#define RES_DEV_ID		4
+#define RES_DRD_ID		5
+
+#define PCI_BAR_HOST		0
+#define PCI_BAR_DEV		2
+#define PCI_BAR_OTG		0
+
+#define PCI_DEV_FN_HOST_DEVICE	0
+#define PCI_DEV_FN_OTG		1
+
+#define PCI_DRIVER_NAME		"cdns3-pci-usbss"
+#define PLAT_DRIVER_NAME	"cdns-usb3"
+
+#define CDNS_VENDOR_ID		0x17cd
+#define CDNS_DEVICE_ID		0x0100
+
+static struct pci_dev *cdns3_get_second_fun(struct pci_dev *pdev)
+{
+	struct pci_dev *func;
+
+	/*
+	 * Gets the second function.
+	 * It's little tricky, but this platform has two function.
+	 * The fist keeps resources for Host/Device while the second
+	 * keeps resources for DRD/OTG.
+	 */
+	func = pci_get_device(pdev->vendor, pdev->device, NULL);
+	if (unlikely(!func))
+		return NULL;
+
+	if (func->devfn == pdev->devfn) {
+		func = pci_get_device(pdev->vendor, pdev->device, func);
+		if (unlikely(!func))
+			return NULL;
+	}
+
+	return func;
+}
+
+static int cdns3_pci_probe(struct pci_dev *pdev,
+			   const struct pci_device_id *id)
+{
+	struct platform_device_info plat_info;
+	struct cdns3_wrap *wrap;
+	struct resource *res;
+	struct pci_dev *func;
+	int err;
+
+	/*
+	 * for GADGET/HOST PCI (devfn) function number is 0,
+	 * for OTG PCI (devfn) function number is 1
+	 */
+	if (!id || (pdev->devfn != PCI_DEV_FN_HOST_DEVICE &&
+		    pdev->devfn != PCI_DEV_FN_OTG))
+		return -EINVAL;
+
+	func = cdns3_get_second_fun(pdev);
+	if (unlikely(!func))
+		return -EINVAL;
+
+	err = pcim_enable_device(pdev);
+	if (err) {
+		dev_err(&pdev->dev, "Enabling PCI device has failed %d\n", err);
+		return err;
+	}
+
+	pci_set_master(pdev);
+
+	if (pci_is_enabled(func)) {
+		wrap = pci_get_drvdata(func);
+	} else {
+		wrap = kzalloc(sizeof(*wrap), GFP_KERNEL);
+		if (!wrap) {
+			pci_disable_device(pdev);
+			return -ENOMEM;
+		}
+	}
+
+	res = wrap->dev_res;
+
+	if (pdev->devfn == PCI_DEV_FN_HOST_DEVICE) {
+		/* function 0: host(BAR_0) + device(BAR_1).*/
+		dev_dbg(&pdev->dev, "Initialize Device resources\n");
+		res[RES_DEV_ID].start = pci_resource_start(pdev, PCI_BAR_DEV);
+		res[RES_DEV_ID].end =   pci_resource_end(pdev, PCI_BAR_DEV);
+		res[RES_DEV_ID].name = "dev";
+		res[RES_DEV_ID].flags = IORESOURCE_MEM;
+		dev_dbg(&pdev->dev, "USBSS-DEV physical base addr: %pa\n",
+			&res[RES_DEV_ID].start);
+
+		res[RES_HOST_ID].start = pci_resource_start(pdev, PCI_BAR_HOST);
+		res[RES_HOST_ID].end = pci_resource_end(pdev, PCI_BAR_HOST);
+		res[RES_HOST_ID].name = "xhci";
+		res[RES_HOST_ID].flags = IORESOURCE_MEM;
+		dev_dbg(&pdev->dev, "USBSS-XHCI physical base addr: %pa\n",
+			&res[RES_HOST_ID].start);
+
+		/* Interrupt for XHCI */
+		wrap->dev_res[RES_IRQ_HOST_ID].start = pdev->irq;
+		wrap->dev_res[RES_IRQ_HOST_ID].name = "host";
+		wrap->dev_res[RES_IRQ_HOST_ID].flags = IORESOURCE_IRQ;
+
+		/* Interrupt device. It's the same as for HOST. */
+		wrap->dev_res[RES_IRQ_PERIPHERAL_ID].start = pdev->irq;
+		wrap->dev_res[RES_IRQ_PERIPHERAL_ID].name = "peripheral";
+		wrap->dev_res[RES_IRQ_PERIPHERAL_ID].flags = IORESOURCE_IRQ;
+	} else {
+		res[RES_DRD_ID].start = pci_resource_start(pdev, PCI_BAR_OTG);
+		res[RES_DRD_ID].end =   pci_resource_end(pdev, PCI_BAR_OTG);
+		res[RES_DRD_ID].name = "otg";
+		res[RES_DRD_ID].flags = IORESOURCE_MEM;
+		dev_dbg(&pdev->dev, "USBSS-DRD physical base addr: %pa\n",
+			&res[RES_DRD_ID].start);
+
+		/* Interrupt for OTG/DRD. */
+		wrap->dev_res[RES_IRQ_OTG_ID].start = pdev->irq;
+		wrap->dev_res[RES_IRQ_OTG_ID].name = "otg";
+		wrap->dev_res[RES_IRQ_OTG_ID].flags = IORESOURCE_IRQ;
+	}
+
+	if (pci_is_enabled(func)) {
+		/* set up platform device info */
+		memset(&plat_info, 0, sizeof(plat_info));
+		plat_info.parent = &pdev->dev;
+		plat_info.fwnode = pdev->dev.fwnode;
+		plat_info.name = PLAT_DRIVER_NAME;
+		plat_info.id = pdev->devfn;
+		wrap->devfn  = pdev->devfn;
+		plat_info.res = wrap->dev_res;
+		plat_info.num_res = ARRAY_SIZE(wrap->dev_res);
+		plat_info.dma_mask = pdev->dma_mask;
+		/* register platform device */
+		wrap->plat_dev = platform_device_register_full(&plat_info);
+		if (IS_ERR(wrap->plat_dev)) {
+			pci_disable_device(pdev);
+			err = PTR_ERR(wrap->plat_dev);
+			kfree(wrap);
+			return err;
+		}
+	}
+
+	pci_set_drvdata(pdev, wrap);
+	return err;
+}
+
+static void cdns3_pci_remove(struct pci_dev *pdev)
+{
+	struct cdns3_wrap *wrap;
+	struct pci_dev *func;
+
+	func = cdns3_get_second_fun(pdev);
+
+	wrap = (struct cdns3_wrap *)pci_get_drvdata(pdev);
+	if (wrap->devfn == pdev->devfn)
+		platform_device_unregister(wrap->plat_dev);
+
+	if (!pci_is_enabled(func))
+		kfree(wrap);
+}
+
+static const struct pci_device_id cdns3_pci_ids[] = {
+	{ PCI_DEVICE(CDNS_VENDOR_ID, CDNS_DEVICE_ID), },
+	{ 0, }
+};
+
+static struct pci_driver cdns3_pci_driver = {
+	.name = PCI_DRIVER_NAME,
+	.id_table = cdns3_pci_ids,
+	.probe = cdns3_pci_probe,
+	.remove = cdns3_pci_remove,
+};
+
+module_pci_driver(cdns3_pci_driver);
+MODULE_DEVICE_TABLE(pci, cdns3_pci_ids);
+
+MODULE_AUTHOR("Pawel Laszczak <pawell@cadence.com>");
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("Cadence USBSS PCI wrapperr");
diff -Naur --no-dereference a/drivers/usb/cdns3/cdns3-ti.c b/drivers/usb/cdns3/cdns3-ti.c
--- a/drivers/usb/cdns3/cdns3-ti.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/usb/cdns3/cdns3-ti.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,244 @@
+// SPDX-License-Identifier: GPL-2.0
+/**
+ * cdns_ti-ti.c - TI specific Glue layer for Cadence USB Controller
+ *
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
+ */
+
+#include <linux/bits.h>
+#include <linux/clk.h>
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/interrupt.h>
+#include <linux/platform_device.h>
+#include <linux/dma-mapping.h>
+#include <linux/io.h>
+#include <linux/of_platform.h>
+#include <linux/pm_runtime.h>
+
+/* USB Wrapper register offsets */
+#define USBSS_PID		0x0
+#define	USBSS_W1		0x4
+#define USBSS_STATIC_CONFIG	0x8
+#define USBSS_PHY_TEST		0xc
+#define	USBSS_DEBUG_CTRL	0x10
+#define	USBSS_DEBUG_INFO	0x14
+#define	USBSS_DEBUG_LINK_STATE	0x18
+#define	USBSS_DEVICE_CTRL	0x1c
+
+/* Wrapper 1 register bits */
+#define USBSS_W1_PWRUP_RST		BIT(0)
+#define USBSS_W1_OVERCURRENT_SEL	BIT(8)
+#define USBSS_W1_MODESTRAP_SEL		BIT(9)
+#define USBSS_W1_OVERCURRENT		BIT(16)
+#define USBSS_W1_MODESTRAP_MASK		GENMASK(18, 17)
+#define USBSS_W1_MODESTRAP_SHIFT	17
+#define USBSS_W1_USB2_ONLY		BIT(19)
+
+/* Static config register bits */
+#define USBSS1_STATIC_PLL_REF_SEL_MASK	GENMASK(8, 5)
+#define USBSS1_STATIC_PLL_REF_SEL_SHIFT	5
+#define USBSS1_STATIC_LOOPBACK_MODE_MASK	GENMASK(4, 3)
+#define USBSS1_STATIC_LOOPBACK_MODE_SHIFT	3
+#define USBSS1_STATIC_VBUS_SEL_MASK	GENMASK(2, 1)
+#define USBSS1_STATIC_VBUS_SEL_SHIFT	1
+#define USBSS1_STATIC_LANE_REVERSE	BIT(0)
+
+/* Modestrap modes */
+enum modestrap_mode { USBSS_MODESTRAP_MODE_NONE,
+		      USBSS_MODESTRAP_MODE_HOST,
+		      USBSS_MODESTRAP_MODE_PERIPHERAL};
+
+struct cdns_ti {
+	struct device *dev;
+	void __iomem *usbss;
+	int usb2_only:1;
+	int vbus_divider:1;
+	struct clk *usb2_refclk;
+	struct clk *lpm_clk;
+};
+
+static const int cdns_ti_rate_table[] = {	/* in KHZ */
+	9600,
+	10000,
+	12000,
+	19200,
+	20000,
+	24000,
+	25000,
+	26000,
+	38400,
+	40000,
+	58000,
+	50000,
+	52000,
+};
+
+static inline u32 cdns_ti_readl(struct cdns_ti *data, u32 offset)
+{
+	return readl(data->usbss + offset);
+}
+
+static inline void cdns_ti_writel(struct cdns_ti *data, u32 offset, u32 value)
+{
+	writel(value, data->usbss + offset);
+}
+
+static int cdns_ti_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *node = pdev->dev.of_node;
+	struct cdns_ti *data;
+	struct resource	*res;
+	int error;
+	u32 reg;
+	int modestrap_mode;
+	int rate_code, i;
+	unsigned long rate;
+
+	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, data);
+
+	data->dev = dev;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	data->usbss = devm_ioremap_resource(dev, res);
+	if (IS_ERR(data->usbss))
+		return PTR_ERR(data->usbss);
+
+	data->usb2_refclk = devm_clk_get(dev, "usb2_refclk");
+	if (IS_ERR(data->usb2_refclk)) {
+		dev_err(dev, "can't get usb2_refclk\n");
+		return PTR_ERR(data->usb2_refclk);
+	}
+
+	data->lpm_clk = devm_clk_get(dev, "lpm_clk");
+	if (IS_ERR(data->lpm_clk)) {
+		dev_err(dev, "can't get lpm_clk\n");
+		return PTR_ERR(data->lpm_clk);
+	}
+
+	rate = clk_get_rate(data->usb2_refclk);
+	rate /= 1000;	/* To KHz */
+	for (i = 0; i < ARRAY_SIZE(cdns_ti_rate_table); i++) {
+		if (cdns_ti_rate_table[i] == rate)
+			break;
+	}
+
+	if (i == ARRAY_SIZE(cdns_ti_rate_table)) {
+		dev_err(dev, "unsupported usb2_refclk rate: %lu KHz\n", rate);
+		return -EINVAL;
+	}
+
+	rate_code = i;
+
+	pm_runtime_enable(dev);
+	error = pm_runtime_get_sync(dev);
+	if (error < 0) {
+		dev_err(dev, "pm_runtime_get_sync failed: %d\n", error);
+		goto err_get;
+	}
+
+	/* assert RESET */
+	reg = cdns_ti_readl(data, USBSS_W1);
+	reg &= ~USBSS_W1_PWRUP_RST;
+	cdns_ti_writel(data, USBSS_W1, reg);
+
+	/* set static config */
+	reg = cdns_ti_readl(data, USBSS_STATIC_CONFIG);
+	reg &= ~USBSS1_STATIC_PLL_REF_SEL_MASK;
+	reg |= rate_code << USBSS1_STATIC_PLL_REF_SEL_SHIFT;
+
+	reg &= ~USBSS1_STATIC_VBUS_SEL_MASK;
+	data->vbus_divider = device_property_read_bool(dev, "ti,vbus-divider");
+	if (data->vbus_divider)
+		reg |= 1 << USBSS1_STATIC_VBUS_SEL_SHIFT;
+
+	cdns_ti_writel(data, USBSS_STATIC_CONFIG, reg);
+	reg = cdns_ti_readl(data, USBSS_STATIC_CONFIG);
+
+	/* set USB2_ONLY mode if requested */
+	reg = cdns_ti_readl(data, USBSS_W1);
+	data->usb2_only = device_property_read_bool(dev, "ti,usb2-only");
+	if (data->usb2_only)
+		reg |= USBSS_W1_USB2_ONLY;
+
+	/* set modestrap  */
+	if (device_property_read_bool(dev, "ti,modestrap-host"))
+		modestrap_mode = USBSS_MODESTRAP_MODE_HOST;
+	else if (device_property_read_bool(dev, "ti,modestrap-peripheral"))
+		modestrap_mode = USBSS_MODESTRAP_MODE_PERIPHERAL;
+	else
+		modestrap_mode = USBSS_MODESTRAP_MODE_NONE;
+
+	reg |= USBSS_W1_MODESTRAP_SEL;
+	reg &= ~USBSS_W1_MODESTRAP_MASK;
+	reg |= modestrap_mode << USBSS_W1_MODESTRAP_SHIFT;
+	cdns_ti_writel(data, USBSS_W1, reg);
+
+	/* de-assert RESET */
+	reg |= USBSS_W1_PWRUP_RST;
+	cdns_ti_writel(data, USBSS_W1, reg);
+
+	error = of_platform_populate(node, NULL, NULL, dev);
+	if (error) {
+		dev_err(dev, "failed to create children: %d\n", error);
+		goto err;
+	}
+
+	return 0;
+
+err:
+	pm_runtime_put_sync(data->dev);
+err_get:
+	pm_runtime_disable(data->dev);
+
+	return error;
+}
+
+static int cdns_ti_remove_core(struct device *dev, void *c)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+
+	platform_device_unregister(pdev);
+
+	return 0;
+}
+
+static int cdns_ti_remove(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+
+	device_for_each_child(dev, NULL, cdns_ti_remove_core);
+	pm_runtime_put_sync(dev);
+	pm_runtime_disable(dev);
+
+	platform_set_drvdata(pdev, NULL);
+
+	return 0;
+}
+
+static const struct of_device_id cdns_ti_of_match[] = {
+	{ .compatible = "ti,j721e-usb", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, cdns_ti_of_match);
+
+static struct platform_driver cdns_ti_driver = {
+	.probe		= cdns_ti_probe,
+	.remove		= cdns_ti_remove,
+	.driver		= {
+		.name	= "cdns3-ti",
+		.of_match_table	= cdns_ti_of_match,
+	},
+};
+
+module_platform_driver(cdns_ti_driver);
+
+MODULE_ALIAS("platform:cdns3-ti");
+MODULE_AUTHOR("Roger Quadros <rogerq@ti.com>");
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("Cadence USB3 TI Glue Layer");
diff -Naur --no-dereference a/drivers/usb/cdns3/core.c b/drivers/usb/cdns3/core.c
--- a/drivers/usb/cdns3/core.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/usb/cdns3/core.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,671 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Cadence USBSS DRD Driver.
+ *
+ * Copyright (C) 2018-2019 Cadence.
+ * Copyright (C) 2017-2018 NXP
+ * Copyright (C) 2019 Texas Instruments
+ *
+ * Author: Peter Chen <peter.chen@nxp.com>
+ *         Pawel Laszczak <pawell@cadence.com>
+ *         Roger Quadros <rogerq@ti.com>
+ */
+
+#include <linux/dma-mapping.h>
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/platform_device.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/pm_runtime.h>
+
+#include "gadget.h"
+#include "core.h"
+#include "host-export.h"
+#include "gadget-export.h"
+#include "drd.h"
+
+static int cdns3_idle_init(struct cdns3 *cdns);
+
+static inline
+struct cdns3_role_driver *cdns3_get_current_role_driver(struct cdns3 *cdns)
+{
+	WARN_ON(!cdns->roles[cdns->role]);
+	return cdns->roles[cdns->role];
+}
+
+static int cdns3_role_start(struct cdns3 *cdns, enum usb_role role)
+{
+	int ret;
+
+	if (WARN_ON(role > USB_ROLE_DEVICE))
+		return 0;
+
+	mutex_lock(&cdns->mutex);
+	cdns->role = role;
+	mutex_unlock(&cdns->mutex);
+
+	if (!cdns->roles[role])
+		return -ENXIO;
+
+	if (cdns->roles[role]->state == CDNS3_ROLE_STATE_ACTIVE)
+		return 0;
+
+	mutex_lock(&cdns->mutex);
+	ret = cdns->roles[role]->start(cdns);
+	if (!ret)
+		cdns->roles[role]->state = CDNS3_ROLE_STATE_ACTIVE;
+	mutex_unlock(&cdns->mutex);
+
+	return ret;
+}
+
+static void cdns3_role_stop(struct cdns3 *cdns)
+{
+	enum usb_role role = cdns->role;
+
+	if (WARN_ON(role > USB_ROLE_DEVICE))
+		return;
+
+	if (cdns->roles[role]->state == CDNS3_ROLE_STATE_INACTIVE)
+		return;
+
+	mutex_lock(&cdns->mutex);
+	cdns->roles[role]->stop(cdns);
+	cdns->roles[role]->state = CDNS3_ROLE_STATE_INACTIVE;
+	mutex_unlock(&cdns->mutex);
+}
+
+static void cdns3_exit_roles(struct cdns3 *cdns)
+{
+	cdns3_role_stop(cdns);
+	cdns3_drd_exit(cdns);
+}
+
+static enum usb_role cdsn3_hw_role_state_machine(struct cdns3 *cdns);
+
+/**
+ * cdns3_core_init_role - initialize role of operation
+ * @cdns: Pointer to cdns3 structure
+ *
+ * Returns 0 on success otherwise negative errno
+ */
+static int cdns3_core_init_role(struct cdns3 *cdns)
+{
+	struct device *dev = cdns->dev;
+	enum usb_dr_mode best_dr_mode;
+	enum usb_dr_mode dr_mode;
+	int ret = 0;
+
+	dr_mode = usb_get_dr_mode(dev);
+	cdns->role = USB_ROLE_NONE;
+
+	/*
+	 * If driver can't read mode by means of usb_get_dr_mode function then
+	 * chooses mode according with Kernel configuration. This setting
+	 * can be restricted later depending on strap pin configuration.
+	 */
+	if (dr_mode == USB_DR_MODE_UNKNOWN) {
+		if (IS_ENABLED(CONFIG_USB_CDNS3_HOST) &&
+		    IS_ENABLED(CONFIG_USB_CDNS3_GADGET))
+			dr_mode = USB_DR_MODE_OTG;
+		else if (IS_ENABLED(CONFIG_USB_CDNS3_HOST))
+			dr_mode = USB_DR_MODE_HOST;
+		else if (IS_ENABLED(CONFIG_USB_CDNS3_GADGET))
+			dr_mode = USB_DR_MODE_PERIPHERAL;
+	}
+
+	/*
+	 * At this point cdns->dr_mode contains strap configuration.
+	 * Driver try update this setting considering kernel configuration
+	 */
+	best_dr_mode = cdns->dr_mode;
+
+	ret = cdns3_idle_init(cdns);
+	if (ret)
+		return ret;
+
+	if (dr_mode == USB_DR_MODE_OTG) {
+		best_dr_mode = cdns->dr_mode;
+	} else if (cdns->dr_mode == USB_DR_MODE_OTG) {
+		best_dr_mode = dr_mode;
+	} else if (cdns->dr_mode != dr_mode) {
+		dev_err(dev, "Incorrect DRD configuration\n");
+		return -EINVAL;
+	}
+
+	dr_mode = best_dr_mode;
+
+	if (dr_mode == USB_DR_MODE_OTG || dr_mode == USB_DR_MODE_HOST) {
+		ret = cdns3_host_init(cdns);
+		if (ret) {
+			dev_err(dev, "Host initialization failed with %d\n",
+				ret);
+			goto err;
+		}
+	}
+
+	if (dr_mode == USB_DR_MODE_OTG || dr_mode == USB_DR_MODE_PERIPHERAL) {
+		ret = cdns3_gadget_init(cdns);
+		if (ret) {
+			dev_err(dev, "Device initialization failed with %d\n",
+				ret);
+			goto err;
+		}
+	}
+
+	cdns->dr_mode = dr_mode;
+
+	ret = cdns3_drd_update_mode(cdns);
+	if (ret)
+		goto err;
+
+	/* Initialize idle role to start with */
+	ret = cdns3_role_start(cdns, USB_ROLE_NONE);
+	if (ret)
+		goto err;
+
+	switch (cdns->dr_mode) {
+	case USB_DR_MODE_OTG:
+		ret = cdns3_hw_role_switch(cdns);
+		if (ret)
+			goto err;
+		break;
+	case USB_DR_MODE_PERIPHERAL:
+		ret = cdns3_role_start(cdns, USB_ROLE_DEVICE);
+		if (ret)
+			goto err;
+		break;
+	case USB_DR_MODE_HOST:
+		ret = cdns3_role_start(cdns, USB_ROLE_HOST);
+		if (ret)
+			goto err;
+		break;
+	default:
+		ret = -EINVAL;
+		goto err;
+	}
+
+	return ret;
+err:
+	cdns3_exit_roles(cdns);
+	return ret;
+}
+
+/**
+ * cdsn3_hw_role_state_machine  - role switch state machine based on hw events.
+ * @cdns: Pointer to controller structure.
+ *
+ * Returns next role to be entered based on hw events.
+ */
+static enum usb_role cdsn3_hw_role_state_machine(struct cdns3 *cdns)
+{
+	enum usb_role role;
+	int id, vbus;
+
+	if (cdns->dr_mode != USB_DR_MODE_OTG)
+		goto not_otg;
+
+	id = cdns3_get_id(cdns);
+	vbus = cdns3_get_vbus(cdns);
+
+	/*
+	 * Role change state machine
+	 * Inputs: ID, VBUS
+	 * Previous state: cdns->role
+	 * Next state: role
+	 */
+	role = cdns->role;
+
+	switch (role) {
+	case USB_ROLE_NONE:
+		/*
+		 * Driver treats USB_ROLE_NONE synonymous to IDLE state from
+		 * controller specification.
+		 */
+		if (!id)
+			role = USB_ROLE_HOST;
+		else if (vbus)
+			role = USB_ROLE_DEVICE;
+		break;
+	case USB_ROLE_HOST: /* from HOST, we can only change to NONE */
+		if (id)
+			role = USB_ROLE_NONE;
+		break;
+	case USB_ROLE_DEVICE: /* from GADGET, we can only change to NONE*/
+		if (!vbus)
+			role = USB_ROLE_NONE;
+		break;
+	}
+
+	dev_dbg(cdns->dev, "role %d -> %d\n", cdns->role, role);
+
+	return role;
+
+not_otg:
+	if (cdns3_is_host(cdns))
+		role = USB_ROLE_HOST;
+	if (cdns3_is_device(cdns))
+		role = USB_ROLE_DEVICE;
+
+	return role;
+}
+
+static int cdns3_idle_role_start(struct cdns3 *cdns)
+{
+	return 0;
+}
+
+static void cdns3_idle_role_stop(struct cdns3 *cdns)
+{
+	/* Program Lane swap and bring PHY out of RESET */
+	phy_reset(cdns->usb3_phy);
+}
+
+static int cdns3_idle_init(struct cdns3 *cdns)
+{
+	struct cdns3_role_driver *rdrv;
+
+	rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
+	if (!rdrv)
+		return -ENOMEM;
+
+	rdrv->start = cdns3_idle_role_start;
+	rdrv->stop = cdns3_idle_role_stop;
+	rdrv->state = CDNS3_ROLE_STATE_INACTIVE;
+	rdrv->suspend = NULL;
+	rdrv->resume = NULL;
+	rdrv->name = "idle";
+
+	cdns->roles[USB_ROLE_NONE] = rdrv;
+
+	return 0;
+}
+
+/**
+ * cdns3_hw_role_switch - switch roles based on HW state
+ * @cdns3: controller
+ */
+int cdns3_hw_role_switch(struct cdns3 *cdns)
+{
+	enum usb_role real_role, current_role;
+	int ret = 0;
+
+	/* Do nothing if role based on syfs. */
+	if (cdns->role_override)
+		return 0;
+
+	pm_runtime_get_sync(cdns->dev);
+
+	current_role = cdns->role;
+	real_role = cdsn3_hw_role_state_machine(cdns);
+
+	/* Do nothing if nothing changed */
+	if (current_role == real_role)
+		goto exit;
+
+	cdns3_role_stop(cdns);
+
+	dev_dbg(cdns->dev, "Switching role %d -> %d", current_role, real_role);
+
+	ret = cdns3_role_start(cdns, real_role);
+	if (ret) {
+		/* Back to current role */
+		dev_err(cdns->dev, "set %d has failed, back to %d\n",
+			real_role, current_role);
+		ret = cdns3_role_start(cdns, current_role);
+		if (ret)
+			dev_err(cdns->dev, "back to %d failed too\n",
+				current_role);
+	}
+exit:
+	pm_runtime_put_sync(cdns->dev);
+	return ret;
+}
+
+/**
+ * cdsn3_role_get - get current role of controller.
+ *
+ * @dev: Pointer to device structure
+ *
+ * Returns role
+ */
+static enum usb_role cdns3_role_get(struct device *dev)
+{
+	struct cdns3 *cdns = dev_get_drvdata(dev);
+
+	return cdns->role;
+}
+
+/**
+ * cdns3_role_set - set current role of controller.
+ *
+ * @dev: pointer to device object
+ * @role - the previous role
+ * Handles below events:
+ * - Role switch for dual-role devices
+ * - USB_ROLE_GADGET <--> USB_ROLE_NONE for peripheral-only devices
+ */
+static int cdns3_role_set(struct device *dev, enum usb_role role)
+{
+	struct cdns3 *cdns = dev_get_drvdata(dev);
+	int ret = 0;
+
+	pm_runtime_get_sync(cdns->dev);
+
+	/*
+	 * FIXME: switch role framework should be extended to meet
+	 * requirements. Driver assumes that role can be controlled
+	 * by SW or HW. Temporary workaround is to use USB_ROLE_NONE to
+	 * switch from SW to HW control.
+	 *
+	 * For dr_mode == USB_DR_MODE_OTG:
+	 *	if user sets USB_ROLE_HOST or USB_ROLE_DEVICE then driver
+	 *	sets role_override flag and forces that role.
+	 *	if user sets USB_ROLE_NONE, driver clears role_override and lets
+	 *	HW state machine take over.
+	 *
+	 * For dr_mode != USB_DR_MODE_OTG:
+	 *	Assumptions:
+	 *	1. Restricted user control between NONE and dr_mode.
+	 *	2. Driver doesn't need to rely on role_override flag.
+	 *	3. Driver needs to ensure that HW state machine is never called
+	 *	   if dr_mode != USB_DR_MODE_OTG.
+	 */
+	if (role == USB_ROLE_NONE)
+		cdns->role_override = 0;
+	else
+		cdns->role_override = 1;
+
+	/*
+	 * HW state might have changed so driver need to trigger
+	 * HW state machine if dr_mode == USB_DR_MODE_OTG.
+	 */
+	if (!cdns->role_override && cdns->dr_mode == USB_DR_MODE_OTG) {
+		cdns3_hw_role_switch(cdns);
+		goto pm_put;
+	}
+
+	if (cdns->role == role)
+		goto pm_put;
+
+	if (cdns->dr_mode == USB_DR_MODE_HOST) {
+		switch (role) {
+		case USB_ROLE_NONE:
+		case USB_ROLE_HOST:
+			break;
+		default:
+			ret = -EPERM;
+			goto pm_put;
+		}
+	}
+
+	if (cdns->dr_mode == USB_DR_MODE_PERIPHERAL) {
+		switch (role) {
+		case USB_ROLE_NONE:
+		case USB_ROLE_DEVICE:
+			break;
+		default:
+			ret = -EPERM;
+			goto pm_put;
+		}
+	}
+
+	cdns3_role_stop(cdns);
+	ret = cdns3_role_start(cdns, role);
+	if (ret) {
+		dev_err(cdns->dev, "set role %d has failed\n", role);
+		ret = -EPERM;
+	}
+
+pm_put:
+	pm_runtime_put_sync(cdns->dev);
+	return ret;
+}
+
+static const struct usb_role_switch_desc cdns3_switch_desc = {
+	.set = cdns3_role_set,
+	.get = cdns3_role_get,
+	.allow_userspace_control = true,
+};
+
+/**
+ * cdns3_probe - probe for cdns3 core device
+ * @pdev: Pointer to cdns3 core platform device
+ *
+ * Returns 0 on success otherwise negative errno
+ */
+static int cdns3_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct resource	*res;
+	struct cdns3 *cdns;
+	void __iomem *regs;
+	int ret;
+
+	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
+	if (ret) {
+		dev_err(dev, "error setting dma mask: %d\n", ret);
+		return -ENODEV;
+	}
+
+	cdns = devm_kzalloc(dev, sizeof(*cdns), GFP_KERNEL);
+	if (!cdns)
+		return -ENOMEM;
+
+	cdns->dev = dev;
+
+	platform_set_drvdata(pdev, cdns);
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_IRQ, "host");
+	if (!res) {
+		dev_err(dev, "missing host IRQ\n");
+		return -ENODEV;
+	}
+
+	cdns->xhci_res[0] = *res;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "xhci");
+	if (!res) {
+		dev_err(dev, "couldn't get xhci resource\n");
+		return -ENXIO;
+	}
+
+	cdns->xhci_res[1] = *res;
+
+	cdns->dev_irq = platform_get_irq_byname(pdev, "peripheral");
+	if (cdns->dev_irq == -EPROBE_DEFER)
+		return cdns->dev_irq;
+
+	if (cdns->dev_irq < 0)
+		dev_err(dev, "couldn't get peripheral irq\n");
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dev");
+	regs = devm_ioremap_resource(dev, res);
+	if (IS_ERR(regs))
+		return PTR_ERR(regs);
+	cdns->dev_regs	= regs;
+
+	cdns->otg_irq = platform_get_irq_byname(pdev, "otg");
+	if (cdns->otg_irq == -EPROBE_DEFER)
+		return cdns->otg_irq;
+
+	if (cdns->otg_irq < 0) {
+		dev_err(dev, "couldn't get otg irq\n");
+		return cdns->otg_irq;
+	}
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "otg");
+	if (!res) {
+		dev_err(dev, "couldn't get otg resource\n");
+		return -ENXIO;
+	}
+
+	cdns->otg_res = *res;
+
+	mutex_init(&cdns->mutex);
+
+	cdns->usb2_phy = devm_phy_optional_get(dev, "cdns3,usb2-phy");
+	if (IS_ERR(cdns->usb2_phy))
+		return PTR_ERR(cdns->usb2_phy);
+
+	ret = phy_init(cdns->usb2_phy);
+	if (ret)
+		return ret;
+
+	cdns->usb3_phy = devm_phy_optional_get(dev, "cdns3,usb3-phy");
+	if (IS_ERR(cdns->usb3_phy))
+		return PTR_ERR(cdns->usb3_phy);
+
+	ret = phy_init(cdns->usb3_phy);
+	if (ret)
+		goto err1;
+
+	ret = phy_power_on(cdns->usb2_phy);
+	if (ret)
+		goto err2;
+
+	ret = phy_power_on(cdns->usb3_phy);
+	if (ret)
+		goto err3;
+
+	cdns->role_sw = usb_role_switch_register(dev, &cdns3_switch_desc);
+	if (IS_ERR(cdns->role_sw)) {
+		ret = PTR_ERR(cdns->role_sw);
+		dev_warn(dev, "Unable to register Role Switch\n");
+		goto err4;
+	}
+
+	ret = cdns3_drd_init(cdns);
+	if (ret)
+		goto err5;
+
+	ret = cdns3_core_init_role(cdns);
+	if (ret)
+		goto err5;
+
+	device_set_wakeup_capable(dev, true);
+	pm_runtime_set_active(dev);
+	pm_runtime_enable(dev);
+
+	/*
+	 * The controller needs less time between bus and controller suspend,
+	 * and we also needs a small delay to avoid frequently entering low
+	 * power mode.
+	 */
+	pm_runtime_set_autosuspend_delay(dev, 20);
+	pm_runtime_mark_last_busy(dev);
+	pm_runtime_use_autosuspend(dev);
+	dev_dbg(dev, "Cadence USB3 core: probe succeed\n");
+
+	return 0;
+err5:
+	cdns3_drd_exit(cdns);
+	usb_role_switch_unregister(cdns->role_sw);
+err4:
+	phy_power_off(cdns->usb3_phy);
+
+err3:
+	phy_power_off(cdns->usb2_phy);
+err2:
+	phy_exit(cdns->usb3_phy);
+err1:
+	phy_exit(cdns->usb2_phy);
+
+	return ret;
+}
+
+/**
+ * cdns3_remove - unbind drd driver and clean up
+ * @pdev: Pointer to Linux platform device
+ *
+ * Returns 0 on success otherwise negative errno
+ */
+static int cdns3_remove(struct platform_device *pdev)
+{
+	struct cdns3 *cdns = platform_get_drvdata(pdev);
+
+	pm_runtime_get_sync(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+	pm_runtime_put_noidle(&pdev->dev);
+	cdns3_exit_roles(cdns);
+	usb_role_switch_unregister(cdns->role_sw);
+	phy_power_off(cdns->usb2_phy);
+	phy_power_off(cdns->usb3_phy);
+	phy_exit(cdns->usb2_phy);
+	phy_exit(cdns->usb3_phy);
+	return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+
+static int cdns3_suspend(struct device *dev)
+{
+	struct cdns3 *cdns = dev_get_drvdata(dev);
+	unsigned long flags;
+
+	if (cdns->role == USB_ROLE_HOST)
+		return 0;
+
+	if (pm_runtime_status_suspended(dev))
+		pm_runtime_resume(dev);
+
+	if (cdns->roles[cdns->role]->suspend) {
+		spin_lock_irqsave(&cdns->gadget_dev->lock, flags);
+		cdns->roles[cdns->role]->suspend(cdns, false);
+		spin_unlock_irqrestore(&cdns->gadget_dev->lock, flags);
+	}
+
+	return 0;
+}
+
+static int cdns3_resume(struct device *dev)
+{
+	struct cdns3 *cdns = dev_get_drvdata(dev);
+	unsigned long flags;
+
+	if (cdns->role == USB_ROLE_HOST)
+		return 0;
+
+	if (cdns->roles[cdns->role]->resume) {
+		spin_lock_irqsave(&cdns->gadget_dev->lock, flags);
+		cdns->roles[cdns->role]->resume(cdns, false);
+		spin_unlock_irqrestore(&cdns->gadget_dev->lock, flags);
+	}
+
+	pm_runtime_disable(dev);
+	pm_runtime_set_active(dev);
+	pm_runtime_enable(dev);
+
+	return 0;
+}
+#endif
+
+static const struct dev_pm_ops cdns3_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(cdns3_suspend, cdns3_resume)
+};
+
+#ifdef CONFIG_OF
+static const struct of_device_id of_cdns3_match[] = {
+	{ .compatible = "cdns,usb3" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, of_cdns3_match);
+#endif
+
+static struct platform_driver cdns3_driver = {
+	.probe		= cdns3_probe,
+	.remove		= cdns3_remove,
+	.driver		= {
+		.name	= "cdns-usb3",
+		.of_match_table	= of_match_ptr(of_cdns3_match),
+		.pm	= &cdns3_pm_ops,
+	},
+};
+
+module_platform_driver(cdns3_driver);
+
+MODULE_ALIAS("platform:cdns3");
+MODULE_AUTHOR("Pawel Laszczak <pawell@cadence.com>");
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("Cadence USB3 DRD Controller Driver");
diff -Naur --no-dereference a/drivers/usb/cdns3/core.h b/drivers/usb/cdns3/core.h
--- a/drivers/usb/cdns3/core.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/usb/cdns3/core.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,98 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Cadence USBSS DRD Header File.
+ *
+ * Copyright (C) 2017-2018 NXP
+ * Copyright (C) 2018-2019 Cadence.
+ *
+ * Authors: Peter Chen <peter.chen@nxp.com>
+ *          Pawel Laszczak <pawell@cadence.com>
+ */
+#include <linux/usb/otg.h>
+#include <linux/usb/role.h>
+
+#ifndef __LINUX_CDNS3_CORE_H
+#define __LINUX_CDNS3_CORE_H
+
+struct cdns3;
+
+/**
+ * struct cdns3_role_driver - host/gadget role driver
+ * @start: start this role
+ * @stop: stop this role
+ * @suspend: suspend callback for this role
+ * @resume: resume callback for this role
+ * @irq: irq handler for this role
+ * @name: role name string (host/gadget)
+ * @state: current state
+ */
+struct cdns3_role_driver {
+	int (*start)(struct cdns3 *cdns);
+	void (*stop)(struct cdns3 *cdns);
+	int (*suspend)(struct cdns3 *cdns, bool do_wakeup);
+	int (*resume)(struct cdns3 *cdns, bool hibernated);
+	const char *name;
+#define CDNS3_ROLE_STATE_INACTIVE	0
+#define CDNS3_ROLE_STATE_ACTIVE		1
+	int state;
+};
+
+#define CDNS3_XHCI_RESOURCES_NUM	2
+/**
+ * struct cdns3 - Representation of Cadence USB3 DRD controller.
+ * @dev: pointer to Cadence device struct
+ * @xhci_regs: pointer to base of xhci registers
+ * @xhci_res: the resource for xhci
+ * @dev_regs: pointer to base of dev registers
+ * @otg_res: the resource for otg
+ * @otg_v0_regs: pointer to base of v0 otg registers
+ * @otg_v1_regs: pointer to base of v1 otg registers
+ * @otg_regs: pointer to base of otg registers
+ * @otg_irq: irq number for otg controller
+ * @dev_irq: irq number for device controller
+ * @roles: array of supported roles for this controller
+ * @role: current role
+ * @host_dev: the child host device pointer for cdns3 core
+ * @gadget_dev: the child gadget device pointer for cdns3 core
+ * @usb2_phy: pointer to USB2 PHY
+ * @usb3_phy: pointer to USB3 PHY
+ * @mutex: the mutex for concurrent code at driver
+ * @dr_mode: supported mode of operation it can be only Host, only Device
+ *           or OTG mode that allow to switch between Device and Host mode.
+ *           This field based on firmware setting, kernel configuration
+ *           and hardware configuration.
+ * @role_sw: pointer to role switch object.
+ * @role_override: set 1 if role rely on SW.
+ */
+struct cdns3 {
+	struct device			*dev;
+	void __iomem			*xhci_regs;
+	struct resource			xhci_res[CDNS3_XHCI_RESOURCES_NUM];
+	struct cdns3_usb_regs __iomem	*dev_regs;
+
+	struct resource			otg_res;
+	struct cdns3_otg_legacy_regs	*otg_v0_regs;
+	struct cdns3_otg_regs		*otg_v1_regs;
+	struct cdns3_otg_common_regs	*otg_regs;
+#define CDNS3_CONTROLLER_V0	0
+#define CDNS3_CONTROLLER_V1	1
+	u32				version;
+
+	int				otg_irq;
+	int				dev_irq;
+	struct cdns3_role_driver	*roles[USB_ROLE_DEVICE + 1];
+	enum usb_role			role;
+	struct platform_device		*host_dev;
+	struct cdns3_device		*gadget_dev;
+	struct phy			*usb2_phy;
+	struct phy			*usb3_phy;
+	/* mutext used in workqueue*/
+	struct mutex			mutex;
+	enum usb_dr_mode		dr_mode;
+	struct usb_role_switch		*role_sw;
+	int				role_override;
+};
+
+int cdns3_hw_role_switch(struct cdns3 *cdns);
+
+#endif /* __LINUX_CDNS3_CORE_H */
diff -Naur --no-dereference a/drivers/usb/cdns3/debug.h b/drivers/usb/cdns3/debug.h
--- a/drivers/usb/cdns3/debug.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/usb/cdns3/debug.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,161 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Cadence USBSS DRD Driver.
+ * Debug header file.
+ *
+ * Copyright (C) 2018-2019 Cadence.
+ *
+ * Author: Pawel Laszczak <pawell@cadence.com>
+ */
+#ifndef __LINUX_CDNS3_DEBUG
+#define __LINUX_CDNS3_DEBUG
+
+#include "core.h"
+
+static inline char *cdns3_decode_usb_irq(char *str,
+					 enum usb_device_speed speed,
+					 u32 usb_ists)
+{
+	int ret;
+
+	ret = sprintf(str, "IRQ %08x = ", usb_ists);
+
+	if (usb_ists & (USB_ISTS_CON2I | USB_ISTS_CONI)) {
+		ret += sprintf(str + ret, "Connection %s\n",
+			       usb_speed_string(speed));
+	}
+	if (usb_ists & USB_ISTS_DIS2I || usb_ists & USB_ISTS_DISI)
+		ret += sprintf(str + ret, "Disconnection ");
+	if (usb_ists & USB_ISTS_L2ENTI)
+		ret += sprintf(str + ret, "suspended ");
+	if (usb_ists & USB_ISTS_L1ENTI)
+		ret += sprintf(str + ret, "L1 enter ");
+	if (usb_ists & USB_ISTS_L1EXTI)
+		ret += sprintf(str + ret, "L1 exit ");
+	if (usb_ists & USB_ISTS_L2ENTI)
+		ret += sprintf(str + ret, "L2 enter ");
+	if (usb_ists & USB_ISTS_L2EXTI)
+		ret += sprintf(str + ret, "L2 exit ");
+	if (usb_ists & USB_ISTS_U3EXTI)
+		ret += sprintf(str + ret, "U3 exit ");
+	if (usb_ists & USB_ISTS_UWRESI)
+		ret += sprintf(str + ret, "Warm Reset ");
+	if (usb_ists & USB_ISTS_UHRESI)
+		ret += sprintf(str + ret, "Hot Reset ");
+	if (usb_ists & USB_ISTS_U2RESI)
+		ret += sprintf(str + ret, "Reset");
+
+	return str;
+}
+
+static inline  char *cdns3_decode_ep_irq(char *str,
+					 u32 ep_sts,
+					 const char *ep_name)
+{
+	int ret;
+
+	ret = sprintf(str, "IRQ for %s: %08x ", ep_name, ep_sts);
+
+	if (ep_sts & EP_STS_SETUP)
+		ret += sprintf(str + ret, "SETUP ");
+	if (ep_sts & EP_STS_IOC)
+		ret += sprintf(str + ret, "IOC ");
+	if (ep_sts & EP_STS_ISP)
+		ret += sprintf(str + ret, "ISP ");
+	if (ep_sts & EP_STS_DESCMIS)
+		ret += sprintf(str + ret, "DESCMIS ");
+	if (ep_sts & EP_STS_STREAMR)
+		ret += sprintf(str + ret, "STREAMR ");
+	if (ep_sts & EP_STS_MD_EXIT)
+		ret += sprintf(str + ret, "MD_EXIT ");
+	if (ep_sts & EP_STS_TRBERR)
+		ret += sprintf(str + ret, "TRBERR ");
+	if (ep_sts & EP_STS_NRDY)
+		ret += sprintf(str + ret, "NRDY ");
+	if (ep_sts & EP_STS_PRIME)
+		ret += sprintf(str + ret, "PRIME ");
+	if (ep_sts & EP_STS_SIDERR)
+		ret += sprintf(str + ret, "SIDERRT ");
+	if (ep_sts & EP_STS_OUTSMM)
+		ret += sprintf(str + ret, "OUTSMM ");
+	if (ep_sts & EP_STS_ISOERR)
+		ret += sprintf(str + ret, "ISOERR ");
+	if (ep_sts & EP_STS_IOT)
+		ret += sprintf(str + ret, "IOT ");
+
+	return str;
+}
+
+static inline char *cdns3_decode_epx_irq(char *str,
+					 char *ep_name,
+					 u32 ep_sts)
+{
+	return cdns3_decode_ep_irq(str, ep_sts, ep_name);
+}
+
+static inline char *cdns3_decode_ep0_irq(char *str,
+					 int dir,
+					 u32 ep_sts)
+{
+	return cdns3_decode_ep_irq(str, ep_sts,
+				   dir ? "ep0IN" : "ep0OUT");
+}
+
+/**
+ * Debug a transfer ring.
+ *
+ * Prints out all TRBs in the endpoint ring, even those after the Link TRB.
+ *.
+ */
+static inline char *cdns3_dbg_ring(struct cdns3_endpoint *priv_ep,
+				   struct cdns3_trb *ring, char *str)
+{
+	dma_addr_t addr = priv_ep->trb_pool_dma;
+	struct cdns3_trb *trb;
+	int trb_per_sector;
+	int ret = 0;
+	int i;
+
+	trb_per_sector = GET_TRBS_PER_SEGMENT(priv_ep->type);
+
+	trb = &priv_ep->trb_pool[priv_ep->dequeue];
+	ret += sprintf(str + ret, "\n\t\tRing contents for %s:", priv_ep->name);
+
+	ret += sprintf(str + ret,
+		       "\n\t\tRing deq index: %d, trb: %p (virt), 0x%llx (dma)\n",
+		       priv_ep->dequeue, trb,
+		       (unsigned long long)cdns3_trb_virt_to_dma(priv_ep, trb));
+
+	trb = &priv_ep->trb_pool[priv_ep->enqueue];
+	ret += sprintf(str + ret,
+		       "\t\tRing enq index: %d, trb: %p (virt), 0x%llx (dma)\n",
+		       priv_ep->enqueue, trb,
+		       (unsigned long long)cdns3_trb_virt_to_dma(priv_ep, trb));
+
+	ret += sprintf(str + ret,
+		       "\t\tfree trbs: %d, CCS=%d, PCS=%d\n",
+		       priv_ep->free_trbs, priv_ep->ccs, priv_ep->pcs);
+
+	if (trb_per_sector > TRBS_PER_SEGMENT)
+		trb_per_sector = TRBS_PER_SEGMENT;
+
+	if (trb_per_sector > TRBS_PER_SEGMENT) {
+		sprintf(str + ret, "\t\tTo big transfer ring %d\n",
+			trb_per_sector);
+		return str;
+	}
+
+	for (i = 0; i < trb_per_sector; ++i) {
+		trb = &ring[i];
+		ret += sprintf(str + ret,
+			"\t\t@%pad %08x %08x %08x\n", &addr,
+			le32_to_cpu(trb->buffer),
+			le32_to_cpu(trb->length),
+			le32_to_cpu(trb->control));
+		addr += sizeof(*trb);
+	}
+
+	return str;
+}
+
+#endif /*__LINUX_CDNS3_DEBUG*/
diff -Naur --no-dereference a/drivers/usb/cdns3/drd.c b/drivers/usb/cdns3/drd.c
--- a/drivers/usb/cdns3/drd.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/usb/cdns3/drd.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,381 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Cadence USBSS DRD Driver.
+ *
+ * Copyright (C) 2018-2019 Cadence.
+ * Copyright (C) 2019 Texas Instruments
+ *
+ * Author: Pawel Laszczak <pawell@cadence.com>
+ *         Roger Quadros <rogerq@ti.com>
+ *
+ *
+ */
+#include <linux/kernel.h>
+#include <linux/interrupt.h>
+#include <linux/delay.h>
+#include <linux/iopoll.h>
+#include <linux/usb/otg.h>
+
+#include "gadget.h"
+#include "drd.h"
+#include "core.h"
+
+/**
+ * cdns3_set_mode - change mode of OTG Core
+ * @cdns: pointer to context structure
+ * @mode: selected mode from cdns_role
+ *
+ * Returns 0 on success otherwise negative errno
+ */
+int cdns3_set_mode(struct cdns3 *cdns, enum usb_dr_mode mode)
+{
+	int ret = 0;
+	u32 reg;
+
+	switch (mode) {
+	case USB_DR_MODE_PERIPHERAL:
+		break;
+	case USB_DR_MODE_HOST:
+		break;
+	case USB_DR_MODE_OTG:
+		dev_dbg(cdns->dev, "Set controller to OTG mode\n");
+		if (cdns->version == CDNS3_CONTROLLER_V1) {
+			reg = readl(&cdns->otg_v1_regs->override);
+			reg |= OVERRIDE_IDPULLUP;
+			writel(reg, &cdns->otg_v1_regs->override);
+		} else {
+			reg = readl(&cdns->otg_v0_regs->ctrl1);
+			reg |= OVERRIDE_IDPULLUP_V0;
+			writel(reg, &cdns->otg_v0_regs->ctrl1);
+		}
+
+		/*
+		 * Hardware specification says: "ID_VALUE must be valid within
+		 * 50ms after idpullup is set to '1" so driver must wait
+		 * 50ms before reading this pin.
+		 */
+		usleep_range(50000, 60000);
+		break;
+	default:
+		dev_err(cdns->dev, "Unsupported mode of operation %d\n", mode);
+		return -EINVAL;
+	}
+
+	return ret;
+}
+
+int cdns3_get_id(struct cdns3 *cdns)
+{
+	int id;
+
+	id = readl(&cdns->otg_regs->sts) & OTGSTS_ID_VALUE;
+	dev_dbg(cdns->dev, "OTG ID: %d", id);
+
+	return id;
+}
+
+int cdns3_get_vbus(struct cdns3 *cdns)
+{
+	int vbus;
+
+	vbus = !!(readl(&cdns->otg_regs->sts) & OTGSTS_VBUS_VALID);
+	dev_dbg(cdns->dev, "OTG VBUS: %d", vbus);
+
+	return vbus;
+}
+
+int cdns3_is_host(struct cdns3 *cdns)
+{
+	if (cdns->dr_mode == USB_DR_MODE_HOST)
+		return 1;
+	else if (!cdns3_get_id(cdns))
+		return 1;
+
+	return 0;
+}
+
+int cdns3_is_device(struct cdns3 *cdns)
+{
+	if (cdns->dr_mode == USB_DR_MODE_PERIPHERAL)
+		return 1;
+	else if (cdns->dr_mode == USB_DR_MODE_OTG)
+		if (cdns3_get_id(cdns))
+			return 1;
+
+	return 0;
+}
+
+/**
+ * cdns3_otg_disable_irq - Disable all OTG interrupts
+ * @cdns: Pointer to controller context structure
+ */
+static void cdns3_otg_disable_irq(struct cdns3 *cdns)
+{
+	writel(0, &cdns->otg_regs->ien);
+}
+
+/**
+ * cdns3_otg_enable_irq - enable id and sess_valid interrupts
+ * @cdns: Pointer to controller context structure
+ */
+static void cdns3_otg_enable_irq(struct cdns3 *cdns)
+{
+	writel(OTGIEN_ID_CHANGE_INT | OTGIEN_VBUSVALID_RISE_INT |
+	       OTGIEN_VBUSVALID_FALL_INT, &cdns->otg_regs->ien);
+}
+
+/**
+ * cdns3_drd_switch_host - start/stop host
+ * @cdns: Pointer to controller context structure
+ * @on: 1 for start, 0 for stop
+ *
+ * Returns 0 on success otherwise negative errno
+ */
+int cdns3_drd_switch_host(struct cdns3 *cdns, int on)
+{
+	int ret, val;
+	u32 reg = OTGCMD_OTG_DIS;
+
+	/* switch OTG core */
+	if (on) {
+		writel(OTGCMD_HOST_BUS_REQ | reg, &cdns->otg_regs->cmd);
+
+		dev_dbg(cdns->dev, "Waiting till Host mode is turned on\n");
+		ret = readl_poll_timeout_atomic(&cdns->otg_regs->sts, val,
+						val & OTGSTS_XHCI_READY,
+						1, 100000);
+		if (ret) {
+			dev_err(cdns->dev, "timeout waiting for xhci_ready\n");
+			return ret;
+		}
+	} else {
+		writel(OTGCMD_HOST_BUS_DROP | OTGCMD_DEV_BUS_DROP |
+		       OTGCMD_DEV_POWER_OFF | OTGCMD_HOST_POWER_OFF,
+		       &cdns->otg_regs->cmd);
+		/* Waiting till H_IDLE state.*/
+		readl_poll_timeout_atomic(&cdns->otg_regs->state, val,
+					  !(val & OTGSTATE_HOST_STATE_MASK),
+					  1, 2000000);
+	}
+
+	return 0;
+}
+
+/**
+ * cdns3_drd_switch_gadget - start/stop gadget
+ * @cdns: Pointer to controller context structure
+ * @on: 1 for start, 0 for stop
+ *
+ * Returns 0 on success otherwise negative errno
+ */
+int cdns3_drd_switch_gadget(struct cdns3 *cdns, int on)
+{
+	int ret, val;
+	u32 reg = OTGCMD_OTG_DIS;
+
+	/* switch OTG core */
+	if (on) {
+		writel(OTGCMD_DEV_BUS_REQ | reg, &cdns->otg_regs->cmd);
+
+		dev_dbg(cdns->dev, "Waiting till Device mode is turned on\n");
+
+		ret = readl_poll_timeout_atomic(&cdns->otg_regs->sts, val,
+						val & OTGSTS_DEV_READY,
+						1, 100000);
+		if (ret) {
+			dev_err(cdns->dev, "timeout waiting for dev_ready\n");
+			return ret;
+		}
+	} else {
+		/*
+		 * driver should wait at least 10us after disabling Device
+		 * before turning-off Device (DEV_BUS_DROP)
+		 */
+		usleep_range(20, 30);
+		writel(OTGCMD_HOST_BUS_DROP | OTGCMD_DEV_BUS_DROP |
+		       OTGCMD_DEV_POWER_OFF | OTGCMD_HOST_POWER_OFF,
+		       &cdns->otg_regs->cmd);
+		/* Waiting till DEV_IDLE state.*/
+		readl_poll_timeout_atomic(&cdns->otg_regs->state, val,
+					  !(val & OTGSTATE_DEV_STATE_MASK),
+					  1, 2000000);
+	}
+
+	return 0;
+}
+
+/**
+ * cdns3_init_otg_mode - initialize drd controller
+ * @cdns: Pointer to controller context structure
+ *
+ * Returns 0 on success otherwise negative errno
+ */
+static int cdns3_init_otg_mode(struct cdns3 *cdns)
+{
+	int ret = 0;
+
+	cdns3_otg_disable_irq(cdns);
+	/* clear all interrupts */
+	writel(~0, &cdns->otg_regs->ivect);
+
+	ret = cdns3_set_mode(cdns, USB_DR_MODE_OTG);
+	if (ret)
+		return ret;
+
+	cdns3_otg_enable_irq(cdns);
+	return ret;
+}
+
+/**
+ * cdns3_drd_update_mode - initialize mode of operation
+ * @cdns: Pointer to controller context structure
+ *
+ * Returns 0 on success otherwise negative errno
+ */
+int cdns3_drd_update_mode(struct cdns3 *cdns)
+{
+	int ret = 0;
+
+	switch (cdns->dr_mode) {
+	case USB_DR_MODE_PERIPHERAL:
+		ret = cdns3_set_mode(cdns, USB_DR_MODE_PERIPHERAL);
+		break;
+	case USB_DR_MODE_HOST:
+		ret = cdns3_set_mode(cdns, USB_DR_MODE_HOST);
+		break;
+	case USB_DR_MODE_OTG:
+		ret = cdns3_init_otg_mode(cdns);
+		break;
+	default:
+		dev_err(cdns->dev, "Unsupported mode of operation %d\n",
+			cdns->dr_mode);
+		return -EINVAL;
+	}
+
+	return ret;
+}
+
+static irqreturn_t cdns3_drd_thread_irq(int irq, void *data)
+{
+	struct cdns3 *cdns = data;
+
+	cdns3_hw_role_switch(cdns);
+
+	return IRQ_HANDLED;
+}
+
+/**
+ * cdns3_drd_irq - interrupt handler for OTG events
+ *
+ * @irq: irq number for cdns3 core device
+ * @data: structure of cdns3
+ *
+ * Returns IRQ_HANDLED or IRQ_NONE
+ */
+static irqreturn_t cdns3_drd_irq(int irq, void *data)
+{
+	irqreturn_t ret = IRQ_NONE;
+	struct cdns3 *cdns = data;
+	u32 reg;
+
+	if (cdns->dr_mode != USB_DR_MODE_OTG)
+		return ret;
+
+	reg = readl(&cdns->otg_regs->ivect);
+
+	if (!reg)
+		return ret;
+
+	if (reg & OTGIEN_ID_CHANGE_INT) {
+		dev_dbg(cdns->dev, "OTG IRQ: new ID: %d\n",
+			cdns3_get_id(cdns));
+
+		ret = IRQ_WAKE_THREAD;
+	}
+
+	if (reg & (OTGIEN_VBUSVALID_RISE_INT | OTGIEN_VBUSVALID_FALL_INT)) {
+		dev_dbg(cdns->dev, "OTG IRQ: new VBUS: %d\n",
+			cdns3_get_vbus(cdns));
+
+		ret = IRQ_WAKE_THREAD;
+	}
+
+	writel(~0, &cdns->otg_regs->ivect);
+	return ret;
+}
+
+int cdns3_drd_init(struct cdns3 *cdns)
+{
+	void __iomem *regs;
+	int ret = 0;
+	u32 state;
+
+	regs = devm_ioremap_resource(cdns->dev, &cdns->otg_res);
+	if (IS_ERR(regs))
+		return PTR_ERR(regs);
+
+	/* Detection of DRD version. Controller has been released
+	 * in two versions. Both are similar, but they have same changes
+	 * in register maps.
+	 * The first register in old version is command register and it's read
+	 * only, so driver should read 0 from it. On the other hand, in v1
+	 * the first register contains device ID number which is not set to 0.
+	 * Driver uses this fact to detect the proper version of
+	 * controller.
+	 */
+	cdns->otg_v0_regs = regs;
+	if (!readl(&cdns->otg_v0_regs->cmd)) {
+		cdns->version  = CDNS3_CONTROLLER_V0;
+		cdns->otg_v1_regs = NULL;
+		cdns->otg_regs = regs;
+		writel(1, &cdns->otg_v0_regs->simulate);
+		dev_info(cdns->dev, "DRD version v0 (%08x)\n",
+			 readl(&cdns->otg_v0_regs->version));
+	} else {
+		cdns->otg_v0_regs = NULL;
+		cdns->otg_v1_regs = regs;
+		cdns->otg_regs = (void *)&cdns->otg_v1_regs->cmd;
+		cdns->version  = CDNS3_CONTROLLER_V1;
+		writel(1, &cdns->otg_v1_regs->simulate);
+		dev_info(cdns->dev, "DRD version v1 (ID: %08x, rev: %08x)\n",
+			 readl(&cdns->otg_v1_regs->did),
+			 readl(&cdns->otg_v1_regs->rid));
+	}
+
+	state = OTGSTS_STRAP(readl(&cdns->otg_regs->sts));
+
+	/* Update dr_mode according to STRAP configuration. */
+	cdns->dr_mode = USB_DR_MODE_OTG;
+	if (state == OTGSTS_STRAP_HOST) {
+		dev_dbg(cdns->dev, "Controller strapped to HOST\n");
+		cdns->dr_mode = USB_DR_MODE_HOST;
+	} else if (state == OTGSTS_STRAP_GADGET) {
+		dev_dbg(cdns->dev, "Controller strapped to PERIPHERAL\n");
+		cdns->dr_mode = USB_DR_MODE_PERIPHERAL;
+	}
+
+	ret = devm_request_threaded_irq(cdns->dev, cdns->otg_irq,
+					cdns3_drd_irq,
+					cdns3_drd_thread_irq,
+					IRQF_SHARED,
+					dev_name(cdns->dev), cdns);
+
+	if (ret) {
+		dev_err(cdns->dev, "couldn't get otg_irq\n");
+		return ret;
+	}
+
+	state = readl(&cdns->otg_regs->sts);
+	if (OTGSTS_OTG_NRDY(state) != 0) {
+		dev_err(cdns->dev, "Cadence USB3 OTG device not ready\n");
+		return -ENODEV;
+	}
+
+	return ret;
+}
+
+int cdns3_drd_exit(struct cdns3 *cdns)
+{
+	cdns3_otg_disable_irq(cdns);
+	return 0;
+}
diff -Naur --no-dereference a/drivers/usb/cdns3/drd.h b/drivers/usb/cdns3/drd.h
--- a/drivers/usb/cdns3/drd.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/usb/cdns3/drd.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,167 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Cadence USB3 DRD header file.
+ *
+ * Copyright (C) 2018-2019 Cadence.
+ *
+ * Author: Pawel Laszczak <pawell@cadence.com>
+ */
+#ifndef __LINUX_CDNS3_DRD
+#define __LINUX_CDNS3_DRD
+
+#include <linux/usb/otg.h>
+#include <linux/phy/phy.h>
+#include "core.h"
+
+/*  DRD register interface for version v1. */
+struct cdns3_otg_regs {
+	__le32 did;
+	__le32 rid;
+	__le32 capabilities;
+	__le32 reserved1;
+	__le32 cmd;
+	__le32 sts;
+	__le32 state;
+	__le32 reserved2;
+	__le32 ien;
+	__le32 ivect;
+	__le32 refclk;
+	__le32 tmr;
+	__le32 reserved3[4];
+	__le32 simulate;
+	__le32 override;
+	__le32 susp_ctrl;
+	__le32 reserved4;
+	__le32 anasts;
+	__le32 adp_ramp_time;
+	__le32 ctrl1;
+	__le32 ctrl2;
+};
+
+/*  DRD register interface for version v0. */
+struct cdns3_otg_legacy_regs {
+	__le32 cmd;
+	__le32 sts;
+	__le32 state;
+	__le32 refclk;
+	__le32 ien;
+	__le32 ivect;
+	__le32 reserved1[3];
+	__le32 tmr;
+	__le32 reserved2[2];
+	__le32 version;
+	__le32 capabilities;
+	__le32 reserved3[2];
+	__le32 simulate;
+	__le32 reserved4[5];
+	__le32 ctrl1;
+};
+
+/*
+ * Common registers interface for both version of DRD.
+ */
+struct cdns3_otg_common_regs {
+	__le32 cmd;
+	__le32 sts;
+	__le32 state;
+	__le32 different1;
+	__le32 ien;
+	__le32 ivect;
+};
+
+/* CDNS_RID - bitmasks */
+#define CDNS_RID(p)			((p) & GENMASK(15, 0))
+
+/* CDNS_VID - bitmasks */
+#define CDNS_DID(p)			((p) & GENMASK(31, 0))
+
+/* OTGCMD - bitmasks */
+/* "Request the bus for Device mode. */
+#define OTGCMD_DEV_BUS_REQ		BIT(0)
+/* Request the bus for Host mode */
+#define OTGCMD_HOST_BUS_REQ		BIT(1)
+/* Enable OTG mode. */
+#define OTGCMD_OTG_EN			BIT(2)
+/* Disable OTG mode */
+#define OTGCMD_OTG_DIS			BIT(3)
+/*"Configure OTG as A-Device. */
+#define OTGCMD_A_DEV_EN			BIT(4)
+/*"Configure OTG as A-Device. */
+#define OTGCMD_A_DEV_DIS		BIT(5)
+/* Drop the bus for Device mod	e. */
+#define OTGCMD_DEV_BUS_DROP		BIT(8)
+/* Drop the bus for Host mode*/
+#define OTGCMD_HOST_BUS_DROP		BIT(9)
+/* Power Down USBSS-DEV. */
+#define OTGCMD_DEV_POWER_OFF		BIT(11)
+/* Power Down CDNSXHCI. */
+#define OTGCMD_HOST_POWER_OFF		BIT(12)
+
+/* OTGIEN - bitmasks */
+/* ID change interrupt enable */
+#define OTGIEN_ID_CHANGE_INT		BIT(0)
+/* Vbusvalid fall detected interrupt enable.*/
+#define OTGIEN_VBUSVALID_RISE_INT	BIT(4)
+/* Vbusvalid fall detected interrupt enable */
+#define OTGIEN_VBUSVALID_FALL_INT	BIT(5)
+
+/* OTGSTS - bitmasks */
+/*
+ * Current value of the ID pin. It is only valid when idpullup in
+ *  OTGCTRL1_TYPE register is set to '1'.
+ */
+#define OTGSTS_ID_VALUE			BIT(0)
+/* Current value of the vbus_valid */
+#define OTGSTS_VBUS_VALID		BIT(1)
+/* Current value of the b_sess_vld */
+#define OTGSTS_SESSION_VALID		BIT(2)
+/*Device mode is active*/
+#define OTGSTS_DEV_ACTIVE		BIT(3)
+/* Host mode is active. */
+#define OTGSTS_HOST_ACTIVE		BIT(4)
+/* OTG Controller not ready. */
+#define OTGSTS_OTG_NRDY_MASK		BIT(11)
+#define OTGSTS_OTG_NRDY(p)		((p) & OTGSTS_OTG_NRDY_MASK)
+/*
+ * Value of the strap pins.
+ * 000 - no default configuration
+ * 010 - Controller initiall configured as Host
+ * 100 - Controller initially configured as Device
+ */
+#define OTGSTS_STRAP(p)			(((p) & GENMASK(14, 12)) >> 12)
+#define OTGSTS_STRAP_NO_DEFAULT_CFG	0x00
+#define OTGSTS_STRAP_HOST_OTG		0x01
+#define OTGSTS_STRAP_HOST		0x02
+#define OTGSTS_STRAP_GADGET		0x04
+/* Host mode is turned on. */
+#define OTGSTS_XHCI_READY		BIT(26)
+/* "Device mode is turned on .*/
+#define OTGSTS_DEV_READY		BIT(27)
+
+/* OTGSTATE- bitmasks */
+#define OTGSTATE_DEV_STATE_MASK		GENMASK(2, 0)
+#define OTGSTATE_HOST_STATE_MASK	GENMASK(5, 3)
+#define OTGSTATE_HOST_STATE_IDLE	0x0
+#define OTGSTATE_HOST_STATE_VBUS_FALL	0x7
+#define OTGSTATE_HOST_STATE(p)		(((p) & OTGSTATE_HOST_STATE_MASK) >> 3)
+
+/* OTGREFCLK - bitmasks */
+#define OTGREFCLK_STB_CLK_SWITCH_EN	BIT(31)
+
+/* OVERRIDE - bitmasks */
+#define OVERRIDE_IDPULLUP		BIT(0)
+/* Only for CDNS3_CONTROLLER_V0 version */
+#define OVERRIDE_IDPULLUP_V0		BIT(24)
+
+int cdns3_is_host(struct cdns3 *cdns);
+int cdns3_is_device(struct cdns3 *cdns);
+int cdns3_get_id(struct cdns3 *cdns);
+int cdns3_get_vbus(struct cdns3 *cdns);
+int cdns3_drd_init(struct cdns3 *cdns);
+int cdns3_drd_exit(struct cdns3 *cdns);
+int cdns3_drd_update_mode(struct cdns3 *cdns);
+int cdns3_drd_switch_gadget(struct cdns3 *cdns, int on);
+int cdns3_drd_switch_host(struct cdns3 *cdns, int on);
+int cdns3_set_mode(struct cdns3 *cdns, enum usb_dr_mode mode);
+
+#endif /* __LINUX_CDNS3_DRD */
diff -Naur --no-dereference a/drivers/usb/cdns3/ep0.c b/drivers/usb/cdns3/ep0.c
--- a/drivers/usb/cdns3/ep0.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/usb/cdns3/ep0.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,892 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Cadence USBSS DRD Driver - gadget side.
+ *
+ * Copyright (C) 2018 Cadence Design Systems.
+ * Copyright (C) 2017-2018 NXP
+ *
+ * Authors: Pawel Jez <pjez@cadence.com>,
+ *          Pawel Laszczak <pawell@cadence.com>
+ *          Peter Chen <peter.chen@nxp.com>
+ */
+
+#include <linux/usb/composite.h>
+#include <linux/iopoll.h>
+
+#include "gadget.h"
+#include "trace.h"
+
+static struct usb_endpoint_descriptor cdns3_gadget_ep0_desc = {
+	.bLength = USB_DT_ENDPOINT_SIZE,
+	.bDescriptorType = USB_DT_ENDPOINT,
+	.bmAttributes =	USB_ENDPOINT_XFER_CONTROL,
+};
+
+/**
+ * cdns3_ep0_run_transfer - Do transfer on default endpoint hardware
+ * @priv_dev: extended gadget object
+ * @dma_addr: physical address where data is/will be stored
+ * @length: data length
+ * @erdy: set it to 1 when ERDY packet should be sent -
+ *        exit from flow control state
+ */
+static void cdns3_ep0_run_transfer(struct cdns3_device *priv_dev,
+				   dma_addr_t dma_addr,
+				   unsigned int length, int erdy, int zlp)
+{
+	struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
+	struct cdns3_endpoint *priv_ep = priv_dev->eps[0];
+
+	priv_ep->trb_pool[0].buffer = TRB_BUFFER(dma_addr);
+	priv_ep->trb_pool[0].length = TRB_LEN(length);
+
+	if (zlp) {
+		priv_ep->trb_pool[0].control = TRB_CYCLE | TRB_TYPE(TRB_NORMAL);
+		priv_ep->trb_pool[1].buffer = TRB_BUFFER(dma_addr);
+		priv_ep->trb_pool[1].length = TRB_LEN(0);
+		priv_ep->trb_pool[1].control = TRB_CYCLE | TRB_IOC |
+		    TRB_TYPE(TRB_NORMAL);
+	} else {
+		priv_ep->trb_pool[0].control = TRB_CYCLE | TRB_IOC |
+		    TRB_TYPE(TRB_NORMAL);
+		priv_ep->trb_pool[1].control = 0;
+	}
+
+	trace_cdns3_prepare_trb(priv_ep, priv_ep->trb_pool);
+
+	cdns3_select_ep(priv_dev, priv_dev->ep0_data_dir);
+
+	writel(EP_STS_TRBERR, &regs->ep_sts);
+	writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma), &regs->ep_traddr);
+	trace_cdns3_doorbell_ep0(priv_dev->ep0_data_dir ? "ep0in" : "ep0out",
+				 readl(&regs->ep_traddr));
+
+	/* TRB should be prepared before starting transfer. */
+	writel(EP_CMD_DRDY, &regs->ep_cmd);
+
+	/* Resume controller before arming transfer. */
+	__cdns3_gadget_wakeup(priv_dev);
+
+	if (erdy)
+		writel(EP_CMD_ERDY, &priv_dev->regs->ep_cmd);
+}
+
+/**
+ * cdns3_ep0_delegate_req - Returns status of handling setup packet
+ * Setup is handled by gadget driver
+ * @priv_dev: extended gadget object
+ * @ctrl_req: pointer to received setup packet
+ *
+ * Returns zero on success or negative value on failure
+ */
+static int cdns3_ep0_delegate_req(struct cdns3_device *priv_dev,
+				  struct usb_ctrlrequest *ctrl_req)
+{
+	int ret;
+
+	spin_unlock(&priv_dev->lock);
+	priv_dev->setup_pending = 1;
+	ret = priv_dev->gadget_driver->setup(&priv_dev->gadget, ctrl_req);
+	priv_dev->setup_pending = 0;
+	spin_lock(&priv_dev->lock);
+	return ret;
+}
+
+static void cdns3_prepare_setup_packet(struct cdns3_device *priv_dev)
+{
+	priv_dev->ep0_data_dir = 0;
+	priv_dev->ep0_stage = CDNS3_SETUP_STAGE;
+	cdns3_ep0_run_transfer(priv_dev, priv_dev->setup_dma,
+			       sizeof(struct usb_ctrlrequest), 0, 0);
+}
+
+static void cdns3_ep0_complete_setup(struct cdns3_device *priv_dev,
+				     u8 send_stall, u8 send_erdy)
+{
+	struct cdns3_endpoint *priv_ep = priv_dev->eps[0];
+	struct usb_request *request;
+
+	request = cdns3_next_request(&priv_ep->pending_req_list);
+	if (request)
+		list_del_init(&request->list);
+
+	if (send_stall) {
+		trace_cdns3_halt(priv_ep, send_stall, 0);
+		/* set_stall on ep0 */
+		cdns3_select_ep(priv_dev, 0x00);
+		writel(EP_CMD_SSTALL, &priv_dev->regs->ep_cmd);
+	} else {
+		cdns3_prepare_setup_packet(priv_dev);
+	}
+
+	priv_dev->ep0_stage = CDNS3_SETUP_STAGE;
+	writel((send_erdy ? EP_CMD_ERDY : 0) | EP_CMD_REQ_CMPL,
+	       &priv_dev->regs->ep_cmd);
+
+	cdns3_allow_enable_l1(priv_dev, 1);
+}
+
+/**
+ * cdns3_req_ep0_set_configuration - Handling of SET_CONFIG standard USB request
+ * @priv_dev: extended gadget object
+ * @ctrl_req: pointer to received setup packet
+ *
+ * Returns 0 if success, USB_GADGET_DELAYED_STATUS on deferred status stage,
+ * error code on error
+ */
+static int cdns3_req_ep0_set_configuration(struct cdns3_device *priv_dev,
+					   struct usb_ctrlrequest *ctrl_req)
+{
+	enum usb_device_state device_state = priv_dev->gadget.state;
+	struct cdns3_endpoint *priv_ep;
+	u32 config = le16_to_cpu(ctrl_req->wValue);
+	int result = 0;
+	int i;
+
+	switch (device_state) {
+	case USB_STATE_ADDRESS:
+		/* Configure non-control EPs */
+		for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) {
+			priv_ep = priv_dev->eps[i];
+			if (!priv_ep)
+				continue;
+
+			if (priv_ep->flags & EP_CLAIMED)
+				cdns3_ep_config(priv_ep);
+		}
+
+		result = cdns3_ep0_delegate_req(priv_dev, ctrl_req);
+
+		if (result)
+			return result;
+
+		if (config) {
+			cdns3_set_hw_configuration(priv_dev);
+		} else {
+			cdns3_hw_reset_eps_config(priv_dev);
+			usb_gadget_set_state(&priv_dev->gadget,
+					     USB_STATE_ADDRESS);
+		}
+		break;
+	case USB_STATE_CONFIGURED:
+		result = cdns3_ep0_delegate_req(priv_dev, ctrl_req);
+
+		if (!config && !result) {
+			cdns3_hw_reset_eps_config(priv_dev);
+			usb_gadget_set_state(&priv_dev->gadget,
+					     USB_STATE_ADDRESS);
+		}
+		break;
+	default:
+		result = -EINVAL;
+	}
+
+	return result;
+}
+
+/**
+ * cdns3_req_ep0_set_address - Handling of SET_ADDRESS standard USB request
+ * @priv_dev: extended gadget object
+ * @ctrl_req: pointer to received setup packet
+ *
+ * Returns 0 if success, error code on error
+ */
+static int cdns3_req_ep0_set_address(struct cdns3_device *priv_dev,
+				     struct usb_ctrlrequest *ctrl_req)
+{
+	enum usb_device_state device_state = priv_dev->gadget.state;
+	u32 reg;
+	u32 addr;
+
+	addr = le16_to_cpu(ctrl_req->wValue);
+
+	if (addr > USB_DEVICE_MAX_ADDRESS) {
+		dev_err(priv_dev->dev,
+			"Device address (%d) cannot be greater than %d\n",
+			addr, USB_DEVICE_MAX_ADDRESS);
+		return -EINVAL;
+	}
+
+	if (device_state == USB_STATE_CONFIGURED) {
+		dev_err(priv_dev->dev,
+			"can't set_address from configured state\n");
+		return -EINVAL;
+	}
+
+	reg = readl(&priv_dev->regs->usb_cmd);
+
+	writel(reg | USB_CMD_FADDR(addr) | USB_CMD_SET_ADDR,
+	       &priv_dev->regs->usb_cmd);
+
+	usb_gadget_set_state(&priv_dev->gadget,
+			     (addr ? USB_STATE_ADDRESS : USB_STATE_DEFAULT));
+
+	return 0;
+}
+
+/**
+ * cdns3_req_ep0_get_status - Handling of GET_STATUS standard USB request
+ * @priv_dev: extended gadget object
+ * @ctrl_req: pointer to received setup packet
+ *
+ * Returns 0 if success, error code on error
+ */
+static int cdns3_req_ep0_get_status(struct cdns3_device *priv_dev,
+				    struct usb_ctrlrequest *ctrl)
+{
+	struct cdns3_endpoint *priv_ep;
+	__le16 *response_pkt;
+	u16 usb_status = 0;
+	u32 recip;
+	u8 index;
+
+	recip = ctrl->bRequestType & USB_RECIP_MASK;
+
+	switch (recip) {
+	case USB_RECIP_DEVICE:
+		/* self powered */
+		if (priv_dev->is_selfpowered)
+			usb_status = BIT(USB_DEVICE_SELF_POWERED);
+
+		if (priv_dev->wake_up_flag)
+			usb_status |= BIT(USB_DEVICE_REMOTE_WAKEUP);
+
+		if (priv_dev->gadget.speed != USB_SPEED_SUPER)
+			break;
+
+		if (priv_dev->u1_allowed)
+			usb_status |= BIT(USB_DEV_STAT_U1_ENABLED);
+
+		if (priv_dev->u2_allowed)
+			usb_status |= BIT(USB_DEV_STAT_U2_ENABLED);
+
+		break;
+	case USB_RECIP_INTERFACE:
+		return cdns3_ep0_delegate_req(priv_dev, ctrl);
+	case USB_RECIP_ENDPOINT:
+		index = cdns3_ep_addr_to_index(ctrl->wIndex);
+		priv_ep = priv_dev->eps[index];
+
+		/* check if endpoint is stalled or stall is pending */
+		cdns3_select_ep(priv_dev, ctrl->wIndex);
+		if (EP_STS_STALL(readl(&priv_dev->regs->ep_sts)) ||
+		    (priv_ep->flags & EP_STALL_PENDING))
+			usb_status =  BIT(USB_ENDPOINT_HALT);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	response_pkt = (__le16 *)priv_dev->setup_buf;
+	*response_pkt = cpu_to_le16(usb_status);
+
+	cdns3_ep0_run_transfer(priv_dev, priv_dev->setup_dma,
+			       sizeof(*response_pkt), 1, 0);
+	return 0;
+}
+
+static int cdns3_ep0_feature_handle_device(struct cdns3_device *priv_dev,
+					   struct usb_ctrlrequest *ctrl,
+					   int set)
+{
+	enum usb_device_state state;
+	enum usb_device_speed speed;
+	int ret = 0;
+	u32 wValue;
+	u16 tmode;
+
+	wValue = le16_to_cpu(ctrl->wValue);
+	state = priv_dev->gadget.state;
+	speed = priv_dev->gadget.speed;
+
+	switch (wValue) {
+	case USB_DEVICE_REMOTE_WAKEUP:
+		priv_dev->wake_up_flag = !!set;
+		break;
+	case USB_DEVICE_U1_ENABLE:
+		if (state != USB_STATE_CONFIGURED || speed != USB_SPEED_SUPER)
+			return -EINVAL;
+
+		priv_dev->u1_allowed = !!set;
+		break;
+	case USB_DEVICE_U2_ENABLE:
+		if (state != USB_STATE_CONFIGURED || speed != USB_SPEED_SUPER)
+			return -EINVAL;
+
+		priv_dev->u2_allowed = !!set;
+		break;
+	case USB_DEVICE_LTM_ENABLE:
+		ret = -EINVAL;
+		break;
+	case USB_DEVICE_TEST_MODE:
+		if (state != USB_STATE_CONFIGURED || speed > USB_SPEED_HIGH)
+			return -EINVAL;
+
+		tmode = le16_to_cpu(ctrl->wIndex);
+
+		if (!set || (tmode & 0xff) != 0)
+			return -EINVAL;
+
+		switch (tmode >> 8) {
+		case TEST_J:
+		case TEST_K:
+		case TEST_SE0_NAK:
+		case TEST_PACKET:
+			cdns3_ep0_complete_setup(priv_dev, 0, 1);
+			/**
+			 *  Little delay to give the controller some time
+			 * for sending status stage.
+			 * This time should be less then 3ms.
+			 */
+			mdelay(1);
+			cdns3_set_register_bit(&priv_dev->regs->usb_cmd,
+					       USB_CMD_STMODE |
+					       USB_STS_TMODE_SEL(tmode - 1));
+			break;
+		default:
+			ret = -EINVAL;
+		}
+		break;
+	default:
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
+static int cdns3_ep0_feature_handle_intf(struct cdns3_device *priv_dev,
+					 struct usb_ctrlrequest *ctrl,
+					 int set)
+{
+	u32 wValue;
+	int ret = 0;
+
+	wValue = le16_to_cpu(ctrl->wValue);
+
+	switch (wValue) {
+	case USB_INTRF_FUNC_SUSPEND:
+		break;
+	default:
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
+static int cdns3_ep0_feature_handle_endpoint(struct cdns3_device *priv_dev,
+					     struct usb_ctrlrequest *ctrl,
+					     int set)
+{
+	struct cdns3_endpoint *priv_ep;
+	int ret = 0;
+	u8 index;
+
+	if (le16_to_cpu(ctrl->wValue) != USB_ENDPOINT_HALT)
+		return -EINVAL;
+
+	if (!(ctrl->wIndex & ~USB_DIR_IN))
+		return 0;
+
+	index = cdns3_ep_addr_to_index(ctrl->wIndex);
+	priv_ep = priv_dev->eps[index];
+
+	cdns3_select_ep(priv_dev, ctrl->wIndex);
+
+	if (set)
+		__cdns3_gadget_ep_set_halt(priv_ep);
+	else if (!(priv_ep->flags & EP_WEDGE))
+		ret = __cdns3_gadget_ep_clear_halt(priv_ep);
+
+	cdns3_select_ep(priv_dev, 0x00);
+
+	return ret;
+}
+
+/**
+ * cdns3_req_ep0_handle_feature -
+ * Handling of GET/SET_FEATURE standard USB request
+ *
+ * @priv_dev: extended gadget object
+ * @ctrl_req: pointer to received setup packet
+ * @set: must be set to 1 for SET_FEATURE request
+ *
+ * Returns 0 if success, error code on error
+ */
+static int cdns3_req_ep0_handle_feature(struct cdns3_device *priv_dev,
+					struct usb_ctrlrequest *ctrl,
+					int set)
+{
+	int ret = 0;
+	u32 recip;
+
+	recip = ctrl->bRequestType & USB_RECIP_MASK;
+
+	switch (recip) {
+	case USB_RECIP_DEVICE:
+		ret = cdns3_ep0_feature_handle_device(priv_dev, ctrl, set);
+		break;
+	case USB_RECIP_INTERFACE:
+		ret = cdns3_ep0_feature_handle_intf(priv_dev, ctrl, set);
+		break;
+	case USB_RECIP_ENDPOINT:
+		ret = cdns3_ep0_feature_handle_endpoint(priv_dev, ctrl, set);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return ret;
+}
+
+/**
+ * cdns3_req_ep0_set_sel - Handling of SET_SEL standard USB request
+ * @priv_dev: extended gadget object
+ * @ctrl_req: pointer to received setup packet
+ *
+ * Returns 0 if success, error code on error
+ */
+static int cdns3_req_ep0_set_sel(struct cdns3_device *priv_dev,
+				 struct usb_ctrlrequest *ctrl_req)
+{
+	if (priv_dev->gadget.state < USB_STATE_ADDRESS)
+		return -EINVAL;
+
+	if (ctrl_req->wLength != 6) {
+		dev_err(priv_dev->dev, "Set SEL should be 6 bytes, got %d\n",
+			ctrl_req->wLength);
+		return -EINVAL;
+	}
+
+	cdns3_ep0_run_transfer(priv_dev, priv_dev->setup_dma, 6, 1, 0);
+	return 0;
+}
+
+/**
+ * cdns3_req_ep0_set_isoch_delay -
+ * Handling of GET_ISOCH_DELAY standard USB request
+ * @priv_dev: extended gadget object
+ * @ctrl_req: pointer to received setup packet
+ *
+ * Returns 0 if success, error code on error
+ */
+static int cdns3_req_ep0_set_isoch_delay(struct cdns3_device *priv_dev,
+					 struct usb_ctrlrequest *ctrl_req)
+{
+	if (ctrl_req->wIndex || ctrl_req->wLength)
+		return -EINVAL;
+
+	priv_dev->isoch_delay = ctrl_req->wValue;
+
+	return 0;
+}
+
+/**
+ * cdns3_ep0_standard_request - Handling standard USB requests
+ * @priv_dev: extended gadget object
+ * @ctrl_req: pointer to received setup packet
+ *
+ * Returns 0 if success, error code on error
+ */
+static int cdns3_ep0_standard_request(struct cdns3_device *priv_dev,
+				      struct usb_ctrlrequest *ctrl_req)
+{
+	int ret;
+
+	switch (ctrl_req->bRequest) {
+	case USB_REQ_SET_ADDRESS:
+		ret = cdns3_req_ep0_set_address(priv_dev, ctrl_req);
+		break;
+	case USB_REQ_SET_CONFIGURATION:
+		ret = cdns3_req_ep0_set_configuration(priv_dev, ctrl_req);
+		break;
+	case USB_REQ_GET_STATUS:
+		ret = cdns3_req_ep0_get_status(priv_dev, ctrl_req);
+		break;
+	case USB_REQ_CLEAR_FEATURE:
+		ret = cdns3_req_ep0_handle_feature(priv_dev, ctrl_req, 0);
+		break;
+	case USB_REQ_SET_FEATURE:
+		ret = cdns3_req_ep0_handle_feature(priv_dev, ctrl_req, 1);
+		break;
+	case USB_REQ_SET_SEL:
+		ret = cdns3_req_ep0_set_sel(priv_dev, ctrl_req);
+		break;
+	case USB_REQ_SET_ISOCH_DELAY:
+		ret = cdns3_req_ep0_set_isoch_delay(priv_dev, ctrl_req);
+		break;
+	default:
+		ret = cdns3_ep0_delegate_req(priv_dev, ctrl_req);
+		break;
+	}
+
+	return ret;
+}
+
+static void __pending_setup_status_handler(struct cdns3_device *priv_dev)
+{
+	struct usb_request *request = priv_dev->pending_status_request;
+
+	if (priv_dev->status_completion_no_call && request &&
+	    request->complete) {
+		request->complete(&priv_dev->eps[0]->endpoint, request);
+		priv_dev->status_completion_no_call = 0;
+	}
+}
+
+void cdns3_pending_setup_status_handler(struct work_struct *work)
+{
+	struct cdns3_device *priv_dev = container_of(work, struct cdns3_device,
+			pending_status_wq);
+	unsigned long flags;
+
+	spin_lock_irqsave(&priv_dev->lock, flags);
+	__pending_setup_status_handler(priv_dev);
+	spin_unlock_irqrestore(&priv_dev->lock, flags);
+}
+
+/**
+ * cdns3_ep0_setup_phase - Handling setup USB requests
+ * @priv_dev: extended gadget object
+ */
+static void cdns3_ep0_setup_phase(struct cdns3_device *priv_dev)
+{
+	struct usb_ctrlrequest *ctrl = priv_dev->setup_buf;
+	struct cdns3_endpoint *priv_ep = priv_dev->eps[0];
+	int result;
+
+	priv_dev->ep0_data_dir = ctrl->bRequestType & USB_DIR_IN;
+
+	trace_cdns3_ctrl_req(ctrl);
+
+	if (!list_empty(&priv_ep->pending_req_list)) {
+		struct usb_request *request;
+
+		request = cdns3_next_request(&priv_ep->pending_req_list);
+		priv_ep->dir = priv_dev->ep0_data_dir;
+		cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
+				      -ECONNRESET);
+	}
+
+	if (le16_to_cpu(ctrl->wLength))
+		priv_dev->ep0_stage = CDNS3_DATA_STAGE;
+	else
+		priv_dev->ep0_stage = CDNS3_STATUS_STAGE;
+
+	if ((ctrl->bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD)
+		result = cdns3_ep0_standard_request(priv_dev, ctrl);
+	else
+		result = cdns3_ep0_delegate_req(priv_dev, ctrl);
+
+	if (result == USB_GADGET_DELAYED_STATUS)
+		return;
+
+	if (result < 0)
+		cdns3_ep0_complete_setup(priv_dev, 1, 1);
+	else if (priv_dev->ep0_stage == CDNS3_STATUS_STAGE)
+		cdns3_ep0_complete_setup(priv_dev, 0, 1);
+}
+
+static void cdns3_transfer_completed(struct cdns3_device *priv_dev)
+{
+	struct cdns3_endpoint *priv_ep = priv_dev->eps[0];
+
+	if (!list_empty(&priv_ep->pending_req_list)) {
+		struct usb_request *request;
+
+		trace_cdns3_complete_trb(priv_ep, priv_ep->trb_pool);
+		request = cdns3_next_request(&priv_ep->pending_req_list);
+
+		request->actual =
+			TRB_LEN(le32_to_cpu(priv_ep->trb_pool->length));
+
+		priv_ep->dir = priv_dev->ep0_data_dir;
+		cdns3_gadget_giveback(priv_ep, to_cdns3_request(request), 0);
+	}
+
+	cdns3_ep0_complete_setup(priv_dev, 0, 0);
+}
+
+/**
+ * cdns3_check_new_setup - Check if controller receive new SETUP packet.
+ * @priv_dev: extended gadget object
+ *
+ * The SETUP packet can be kept in on-chip memory or in system memory.
+ */
+static bool cdns3_check_new_setup(struct cdns3_device *priv_dev)
+{
+	u32 ep_sts_reg;
+
+	cdns3_select_ep(priv_dev, 0 | USB_DIR_OUT);
+	ep_sts_reg = readl(&priv_dev->regs->ep_sts);
+
+	return !!(ep_sts_reg & (EP_STS_SETUP | EP_STS_STPWAIT));
+}
+
+/**
+ * cdns3_check_ep0_interrupt_proceed - Processes interrupt related to endpoint 0
+ * @priv_dev: extended gadget object
+ * @dir: USB_DIR_IN for IN direction, USB_DIR_OUT for OUT direction
+ */
+void cdns3_check_ep0_interrupt_proceed(struct cdns3_device *priv_dev, int dir)
+{
+	u32 ep_sts_reg;
+
+	cdns3_select_ep(priv_dev, dir);
+
+	ep_sts_reg = readl(&priv_dev->regs->ep_sts);
+	writel(ep_sts_reg, &priv_dev->regs->ep_sts);
+
+	trace_cdns3_ep0_irq(priv_dev, ep_sts_reg);
+
+	__pending_setup_status_handler(priv_dev);
+
+	if (ep_sts_reg & EP_STS_SETUP)
+		priv_dev->wait_for_setup = 1;
+
+	if (priv_dev->wait_for_setup && ep_sts_reg & EP_STS_IOC) {
+		priv_dev->wait_for_setup = 0;
+		cdns3_allow_enable_l1(priv_dev, 0);
+		cdns3_ep0_setup_phase(priv_dev);
+	} else if ((ep_sts_reg & EP_STS_IOC) || (ep_sts_reg & EP_STS_ISP)) {
+		priv_dev->ep0_data_dir = dir;
+		cdns3_transfer_completed(priv_dev);
+	}
+
+	if (ep_sts_reg & EP_STS_DESCMIS) {
+		if (dir == 0 && !priv_dev->setup_pending)
+			cdns3_prepare_setup_packet(priv_dev);
+	}
+}
+
+/**
+ * cdns3_gadget_ep0_enable
+ * Function shouldn't be called by gadget driver,
+ * endpoint 0 is allways active
+ */
+static int cdns3_gadget_ep0_enable(struct usb_ep *ep,
+				   const struct usb_endpoint_descriptor *desc)
+{
+	return -EINVAL;
+}
+
+/**
+ * cdns3_gadget_ep0_disable
+ * Function shouldn't be called by gadget driver,
+ * endpoint 0 is allways active
+ */
+static int cdns3_gadget_ep0_disable(struct usb_ep *ep)
+{
+	return -EINVAL;
+}
+
+/**
+ * cdns3_gadget_ep0_set_halt
+ * @ep: pointer to endpoint zero object
+ * @value: 1 for set stall, 0 for clear stall
+ *
+ * Returns 0
+ */
+static int cdns3_gadget_ep0_set_halt(struct usb_ep *ep, int value)
+{
+	/* TODO */
+	return 0;
+}
+
+/**
+ * cdns3_gadget_ep0_queue Transfer data on endpoint zero
+ * @ep: pointer to endpoint zero object
+ * @request: pointer to request object
+ * @gfp_flags: gfp flags
+ *
+ * Returns 0 on success, error code elsewhere
+ */
+static int cdns3_gadget_ep0_queue(struct usb_ep *ep,
+				  struct usb_request *request,
+				  gfp_t gfp_flags)
+{
+	struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+	unsigned long flags;
+	int erdy_sent = 0;
+	int ret = 0;
+	u8 zlp = 0;
+
+	trace_cdns3_ep0_queue(priv_dev, request);
+
+	/* cancel the request if controller receive new SETUP packet. */
+	if (cdns3_check_new_setup(priv_dev))
+		return -ECONNRESET;
+
+	/* send STATUS stage. Should be called only for SET_CONFIGURATION */
+	if (priv_dev->ep0_stage == CDNS3_STATUS_STAGE) {
+		spin_lock_irqsave(&priv_dev->lock, flags);
+		cdns3_select_ep(priv_dev, 0x00);
+
+		erdy_sent = !priv_dev->hw_configured_flag;
+		cdns3_set_hw_configuration(priv_dev);
+
+		if (!erdy_sent)
+			cdns3_ep0_complete_setup(priv_dev, 0, 1);
+
+		cdns3_allow_enable_l1(priv_dev, 1);
+
+		request->actual = 0;
+		priv_dev->status_completion_no_call = true;
+		priv_dev->pending_status_request = request;
+		spin_unlock_irqrestore(&priv_dev->lock, flags);
+
+		/*
+		 * Since there is no completion interrupt for status stage,
+		 * it needs to call ->completion in software after
+		 * ep0_queue is back.
+		 */
+		queue_work(system_freezable_wq, &priv_dev->pending_status_wq);
+		return 0;
+	}
+
+	spin_lock_irqsave(&priv_dev->lock, flags);
+	if (!list_empty(&priv_ep->pending_req_list)) {
+		dev_err(priv_dev->dev,
+			"can't handle multiple requests for ep0\n");
+		spin_unlock_irqrestore(&priv_dev->lock, flags);
+		return -EBUSY;
+	}
+
+	ret = usb_gadget_map_request_by_dev(priv_dev->sysdev, request,
+					    priv_dev->ep0_data_dir);
+	if (ret) {
+		spin_unlock_irqrestore(&priv_dev->lock, flags);
+		dev_err(priv_dev->dev, "failed to map request\n");
+		return -EINVAL;
+	}
+
+	request->status = -EINPROGRESS;
+	list_add_tail(&request->list, &priv_ep->pending_req_list);
+
+	if (request->zero && request->length &&
+	    (request->length % ep->maxpacket == 0))
+		zlp = 1;
+
+	cdns3_ep0_run_transfer(priv_dev, request->dma, request->length, 1, zlp);
+
+	spin_unlock_irqrestore(&priv_dev->lock, flags);
+
+	return ret;
+}
+
+/**
+ * cdns3_gadget_ep_set_wedge Set wedge on selected endpoint
+ * @ep: endpoint object
+ *
+ * Returns 0
+ */
+int cdns3_gadget_ep_set_wedge(struct usb_ep *ep)
+{
+	struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+
+	dev_dbg(priv_dev->dev, "Wedge for %s\n", ep->name);
+	cdns3_gadget_ep_set_halt(ep, 1);
+	priv_ep->flags |= EP_WEDGE;
+
+	return 0;
+}
+
+const struct usb_ep_ops cdns3_gadget_ep0_ops = {
+	.enable = cdns3_gadget_ep0_enable,
+	.disable = cdns3_gadget_ep0_disable,
+	.alloc_request = cdns3_gadget_ep_alloc_request,
+	.free_request = cdns3_gadget_ep_free_request,
+	.queue = cdns3_gadget_ep0_queue,
+	.dequeue = cdns3_gadget_ep_dequeue,
+	.set_halt = cdns3_gadget_ep0_set_halt,
+	.set_wedge = cdns3_gadget_ep_set_wedge,
+};
+
+/**
+ * cdns3_ep0_config - Configures default endpoint
+ * @priv_dev: extended gadget object
+ *
+ * Functions sets parameters: maximal packet size and enables interrupts
+ */
+void cdns3_ep0_config(struct cdns3_device *priv_dev)
+{
+	struct cdns3_usb_regs __iomem *regs;
+	struct cdns3_endpoint *priv_ep;
+	u32 max_packet_size = 64;
+
+	regs = priv_dev->regs;
+
+	if (priv_dev->gadget.speed == USB_SPEED_SUPER)
+		max_packet_size = 512;
+
+	priv_ep = priv_dev->eps[0];
+
+	if (!list_empty(&priv_ep->pending_req_list)) {
+		struct usb_request *request;
+
+		request = cdns3_next_request(&priv_ep->pending_req_list);
+		list_del_init(&request->list);
+	}
+
+	priv_dev->u1_allowed = 0;
+	priv_dev->u2_allowed = 0;
+
+	priv_dev->gadget.ep0->maxpacket = max_packet_size;
+	cdns3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(max_packet_size);
+
+	/* init ep out */
+	cdns3_select_ep(priv_dev, USB_DIR_OUT);
+
+	if (priv_dev->dev_ver >= DEV_VER_V3) {
+		cdns3_set_register_bit(&priv_dev->regs->dtrans,
+				       BIT(0) | BIT(16));
+		cdns3_set_register_bit(&priv_dev->regs->tdl_from_trb,
+				       BIT(0) | BIT(16));
+	}
+
+	writel(EP_CFG_ENABLE | EP_CFG_MAXPKTSIZE(max_packet_size),
+	       &regs->ep_cfg);
+
+	writel(EP_STS_EN_SETUPEN | EP_STS_EN_DESCMISEN | EP_STS_EN_TRBERREN,
+	       &regs->ep_sts_en);
+
+	/* init ep in */
+	cdns3_select_ep(priv_dev, USB_DIR_IN);
+
+	writel(EP_CFG_ENABLE | EP_CFG_MAXPKTSIZE(max_packet_size),
+	       &regs->ep_cfg);
+
+	writel(EP_STS_EN_SETUPEN | EP_STS_EN_TRBERREN, &regs->ep_sts_en);
+
+	cdns3_set_register_bit(&regs->usb_conf, USB_CONF_U1DS | USB_CONF_U2DS);
+}
+
+/**
+ * cdns3_init_ep0 Initializes software endpoint 0 of gadget
+ * @priv_dev: extended gadget object
+ * @ep_priv: extended endpoint object
+ *
+ * Returns 0 on success else error code.
+ */
+int cdns3_init_ep0(struct cdns3_device *priv_dev,
+		   struct cdns3_endpoint *priv_ep)
+{
+	sprintf(priv_ep->name, "ep0");
+
+	/* fill linux fields */
+	priv_ep->endpoint.ops = &cdns3_gadget_ep0_ops;
+	priv_ep->endpoint.maxburst = 1;
+	usb_ep_set_maxpacket_limit(&priv_ep->endpoint,
+				   CDNS3_EP0_MAX_PACKET_LIMIT);
+	priv_ep->endpoint.address = 0;
+	priv_ep->endpoint.caps.type_control = 1;
+	priv_ep->endpoint.caps.dir_in = 1;
+	priv_ep->endpoint.caps.dir_out = 1;
+	priv_ep->endpoint.name = priv_ep->name;
+	priv_ep->endpoint.desc = &cdns3_gadget_ep0_desc;
+	priv_dev->gadget.ep0 = &priv_ep->endpoint;
+	priv_ep->type = USB_ENDPOINT_XFER_CONTROL;
+
+	return cdns3_allocate_trb_pool(priv_ep);
+}
diff -Naur --no-dereference a/drivers/usb/cdns3/gadget.c b/drivers/usb/cdns3/gadget.c
--- a/drivers/usb/cdns3/gadget.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/usb/cdns3/gadget.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,2776 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Cadence USBSS DRD Driver - gadget side.
+ *
+ * Copyright (C) 2018-2019 Cadence Design Systems.
+ * Copyright (C) 2017-2018 NXP
+ *
+ * Authors: Pawel Jez <pjez@cadence.com>,
+ *          Pawel Laszczak <pawell@cadence.com>
+ *          Peter Chen <peter.chen@nxp.com>
+ */
+
+/*
+ * Work around 1:
+ * At some situations, the controller may get stale data address in TRB
+ * at below sequences:
+ * 1. Controller read TRB includes data address
+ * 2. Software updates TRBs includes data address and Cycle bit
+ * 3. Controller read TRB which includes Cycle bit
+ * 4. DMA run with stale data address
+ *
+ * To fix this problem, driver needs to make the first TRB in TD as invalid.
+ * After preparing all TRBs driver needs to check the position of DMA and
+ * if the DMA point to the first just added TRB and doorbell is 1,
+ * then driver must defer making this TRB as valid. This TRB will be make
+ * as valid during adding next TRB only if DMA is stopped or at TRBERR
+ * interrupt.
+ *
+ * Issue has been fixed in DEV_VER_V3 version of controller.
+ *
+ * Work around 2:
+ * Controller for OUT endpoints has shared on-chip buffers for all incoming
+ * packets, including ep0out. It's FIFO buffer, so packets must be handle by DMA
+ * in correct order. If the first packet in the buffer will not be handled,
+ * then the following packets directed for other endpoints and  functions
+ * will be blocked.
+ * Additionally the packets directed to one endpoint can block entire on-chip
+ * buffers. In this case transfer to other endpoints also will blocked.
+ *
+ * To resolve this issue after raising the descriptor missing interrupt
+ * driver prepares internal usb_request object and use it to arm DMA transfer.
+ *
+ * The problematic situation was observed in case when endpoint has been enabled
+ * but no usb_request were queued. Driver try detects such endpoints and will
+ * use this workaround only for these endpoint.
+ *
+ * Driver use limited number of buffer. This number can be set by macro
+ * CDNS3_WA2_NUM_BUFFERS.
+ *
+ * Such blocking situation was observed on ACM gadget. For this function
+ * host send OUT data packet but ACM function is not prepared for this packet.
+ * It's cause that buffer placed in on chip memory block transfer to other
+ * endpoints.
+ *
+ * Issue has been fixed in DEV_VER_V2 version of controller.
+ *
+ */
+
+#include <linux/dma-mapping.h>
+#include <linux/usb/gadget.h>
+#include <linux/module.h>
+#include <linux/iopoll.h>
+
+#include "core.h"
+#include "gadget-export.h"
+#include "gadget.h"
+#include "trace.h"
+#include "drd.h"
+
+static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
+				   struct usb_request *request,
+				   gfp_t gfp_flags);
+
+/**
+ * cdns3_set_register_bit - set bit in given register.
+ * @ptr: address of device controller register to be read and changed
+ * @mask: bits requested to set
+ */
+void cdns3_set_register_bit(void __iomem *ptr, u32 mask)
+{
+	mask = readl(ptr) | mask;
+	writel(mask, ptr);
+}
+
+/**
+ * cdns3_ep_addr_to_index - Macro converts endpoint address to
+ * index of endpoint object in cdns3_device.eps[] container
+ * @ep_addr: endpoint address for which endpoint object is required
+ *
+ */
+u8 cdns3_ep_addr_to_index(u8 ep_addr)
+{
+	return (((ep_addr & 0x7F)) + ((ep_addr & USB_DIR_IN) ? 16 : 0));
+}
+
+static int cdns3_get_dma_pos(struct cdns3_device *priv_dev,
+			     struct cdns3_endpoint *priv_ep)
+{
+	int dma_index;
+
+	dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma;
+
+	return dma_index / TRB_SIZE;
+}
+
+/**
+ * cdns3_next_request - returns next request from list
+ * @list: list containing requests
+ *
+ * Returns request or NULL if no requests in list
+ */
+struct usb_request *cdns3_next_request(struct list_head *list)
+{
+	return list_first_entry_or_null(list, struct usb_request, list);
+}
+
+/**
+ * cdns3_next_align_buf - returns next buffer from list
+ * @list: list containing buffers
+ *
+ * Returns buffer or NULL if no buffers in list
+ */
+struct cdns3_aligned_buf *cdns3_next_align_buf(struct list_head *list)
+{
+	return list_first_entry_or_null(list, struct cdns3_aligned_buf, list);
+}
+
+/**
+ * cdns3_next_priv_request - returns next request from list
+ * @list: list containing requests
+ *
+ * Returns request or NULL if no requests in list
+ */
+struct cdns3_request *cdns3_next_priv_request(struct list_head *list)
+{
+	return list_first_entry_or_null(list, struct cdns3_request, list);
+}
+
+/**
+ * select_ep - selects endpoint
+ * @priv_dev:  extended gadget object
+ * @ep: endpoint address
+ */
+void cdns3_select_ep(struct cdns3_device *priv_dev, u32 ep)
+{
+	if (priv_dev->selected_ep == ep)
+		return;
+
+	priv_dev->selected_ep = ep;
+	writel(ep, &priv_dev->regs->ep_sel);
+}
+
+dma_addr_t cdns3_trb_virt_to_dma(struct cdns3_endpoint *priv_ep,
+				 struct cdns3_trb *trb)
+{
+	u32 offset = (char *)trb - (char *)priv_ep->trb_pool;
+
+	return priv_ep->trb_pool_dma + offset;
+}
+
+int cdns3_ring_size(struct cdns3_endpoint *priv_ep)
+{
+	switch (priv_ep->type) {
+	case USB_ENDPOINT_XFER_ISOC:
+		return TRB_ISO_RING_SIZE;
+	case USB_ENDPOINT_XFER_CONTROL:
+		return TRB_CTRL_RING_SIZE;
+	default:
+		return TRB_RING_SIZE;
+	}
+}
+
+/**
+ * cdns3_allocate_trb_pool - Allocates TRB's pool for selected endpoint
+ * @priv_ep:  endpoint object
+ *
+ * Function will return 0 on success or -ENOMEM on allocation error
+ */
+int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep)
+{
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+	int ring_size = cdns3_ring_size(priv_ep);
+	struct cdns3_trb *link_trb;
+
+	if (!priv_ep->trb_pool) {
+		priv_ep->trb_pool = dma_alloc_coherent(priv_dev->sysdev,
+						       ring_size,
+						       &priv_ep->trb_pool_dma,
+						       GFP_DMA32 | GFP_ATOMIC);
+		if (!priv_ep->trb_pool)
+			return -ENOMEM;
+	} else {
+		memset(priv_ep->trb_pool, 0, ring_size);
+	}
+
+	if (!priv_ep->num)
+		return 0;
+
+	priv_ep->num_trbs = ring_size / TRB_SIZE;
+	/* Initialize the last TRB as Link TRB. */
+	link_trb = (priv_ep->trb_pool + (priv_ep->num_trbs - 1));
+	link_trb->buffer = TRB_BUFFER(priv_ep->trb_pool_dma);
+	link_trb->control = TRB_CYCLE | TRB_TYPE(TRB_LINK) | TRB_TOGGLE;
+
+	return 0;
+}
+
+static void cdns3_free_trb_pool(struct cdns3_endpoint *priv_ep)
+{
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+
+	if (priv_ep->trb_pool) {
+		dma_free_coherent(priv_dev->sysdev,
+				  cdns3_ring_size(priv_ep),
+				  priv_ep->trb_pool, priv_ep->trb_pool_dma);
+		priv_ep->trb_pool = NULL;
+	}
+}
+
+/**
+ * cdns3_ep_stall_flush - Stalls and flushes selected endpoint
+ * @priv_ep: endpoint object
+ *
+ * Endpoint must be selected before call to this function
+ */
+static void cdns3_ep_stall_flush(struct cdns3_endpoint *priv_ep)
+{
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+	int val;
+
+	trace_cdns3_halt(priv_ep, 1, 1);
+
+	writel(EP_CMD_DFLUSH | EP_CMD_ERDY | EP_CMD_SSTALL,
+	       &priv_dev->regs->ep_cmd);
+
+	/* wait for DFLUSH cleared */
+	readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
+				  !(val & EP_CMD_DFLUSH), 1, 1000);
+	priv_ep->flags |= EP_STALLED;
+	priv_ep->flags &= ~EP_STALL_PENDING;
+}
+
+/**
+ * cdns3_hw_reset_eps_config - reset endpoints configuration kept by controller.
+ * @priv_dev: extended gadget object
+ */
+void cdns3_hw_reset_eps_config(struct cdns3_device *priv_dev)
+{
+	writel(USB_CONF_CFGRST, &priv_dev->regs->usb_conf);
+
+	cdns3_allow_enable_l1(priv_dev, 0);
+	priv_dev->hw_configured_flag = 0;
+	priv_dev->onchip_used_size = 0;
+	priv_dev->out_mem_is_allocated = 0;
+	priv_dev->wait_for_setup = 0;
+}
+
+/**
+ * cdns3_ep_inc_trb - increment a trb index.
+ * @index: Pointer to the TRB index to increment.
+ * @cs: Cycle state
+ * @trb_in_seg: number of TRBs in segment
+ *
+ * The index should never point to the link TRB. After incrementing,
+ * if it is point to the link TRB, wrap around to the beginning and revert
+ * cycle state bit The
+ * link TRB is always at the last TRB entry.
+ */
+static void cdns3_ep_inc_trb(int *index, u8 *cs, int trb_in_seg)
+{
+	(*index)++;
+	if (*index == (trb_in_seg - 1)) {
+		*index = 0;
+		*cs ^=  1;
+	}
+}
+
+/**
+ * cdns3_ep_inc_enq - increment endpoint's enqueue pointer
+ * @priv_ep: The endpoint whose enqueue pointer we're incrementing
+ */
+static void cdns3_ep_inc_enq(struct cdns3_endpoint *priv_ep)
+{
+	priv_ep->free_trbs--;
+	cdns3_ep_inc_trb(&priv_ep->enqueue, &priv_ep->pcs, priv_ep->num_trbs);
+}
+
+/**
+ * cdns3_ep_inc_deq - increment endpoint's dequeue pointer
+ * @priv_ep: The endpoint whose dequeue pointer we're incrementing
+ */
+static void cdns3_ep_inc_deq(struct cdns3_endpoint *priv_ep)
+{
+	priv_ep->free_trbs++;
+	cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs);
+}
+
+void cdns3_move_deq_to_next_trb(struct cdns3_request *priv_req)
+{
+	struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
+	int current_trb = priv_req->start_trb;
+
+	while (current_trb != priv_req->end_trb) {
+		cdns3_ep_inc_deq(priv_ep);
+		current_trb = priv_ep->dequeue;
+	}
+
+	cdns3_ep_inc_deq(priv_ep);
+}
+
+/**
+ * cdns3_allow_enable_l1 - enable/disable permits to transition to L1.
+ * @priv_dev: Extended gadget object
+ * @enable: Enable/disable permit to transition to L1.
+ *
+ * If bit USB_CONF_L1EN is set and device receive Extended Token packet,
+ * then controller answer with ACK handshake.
+ * If bit USB_CONF_L1DS is set and device receive Extended Token packet,
+ * then controller answer with NYET handshake.
+ */
+void cdns3_allow_enable_l1(struct cdns3_device *priv_dev, int enable)
+{
+	if (enable)
+		writel(USB_CONF_L1EN, &priv_dev->regs->usb_conf);
+	else
+		writel(USB_CONF_L1DS, &priv_dev->regs->usb_conf);
+}
+
+enum usb_device_speed cdns3_get_speed(struct cdns3_device *priv_dev)
+{
+	u32 reg;
+
+	reg = readl(&priv_dev->regs->usb_sts);
+
+	if (DEV_SUPERSPEED(reg))
+		return USB_SPEED_SUPER;
+	else if (DEV_HIGHSPEED(reg))
+		return USB_SPEED_HIGH;
+	else if (DEV_FULLSPEED(reg))
+		return USB_SPEED_FULL;
+	else if (DEV_LOWSPEED(reg))
+		return USB_SPEED_LOW;
+	return USB_SPEED_UNKNOWN;
+}
+
+/**
+ * cdns3_start_all_request - add to ring all request not started
+ * @priv_dev: Extended gadget object
+ * @priv_ep: The endpoint for whom request will be started.
+ *
+ * Returns return ENOMEM if transfer ring i not enough TRBs to start
+ *         all requests.
+ */
+static int cdns3_start_all_request(struct cdns3_device *priv_dev,
+				   struct cdns3_endpoint *priv_ep)
+{
+	struct usb_request *request;
+	int ret = 0;
+
+	while (!list_empty(&priv_ep->deferred_req_list)) {
+		request = cdns3_next_request(&priv_ep->deferred_req_list);
+
+		ret = cdns3_ep_run_transfer(priv_ep, request);
+		if (ret)
+			return ret;
+
+		list_del(&request->list);
+		list_add_tail(&request->list,
+			      &priv_ep->pending_req_list);
+	}
+
+	priv_ep->flags &= ~EP_RING_FULL;
+	return ret;
+}
+
+/*
+ * WA2: Set flag for all not ISOC OUT endpoints. If this flag is set
+ * driver try to detect whether endpoint need additional internal
+ * buffer for unblocking on-chip FIFO buffer. This flag will be cleared
+ * if before first DESCMISS interrupt the DMA will be armed.
+ */
+#define cdns3_wa2_enable_detection(priv_dev, ep_priv, reg) do { \
+	if (!priv_ep->dir && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { \
+		priv_ep->flags |= EP_QUIRK_EXTRA_BUF_DET; \
+		(reg) |= EP_STS_EN_DESCMISEN; \
+	} } while (0)
+
+/**
+ * cdns3_wa2_descmiss_copy_data copy data from internal requests to
+ * request queued by class driver.
+ * @priv_ep: extended endpoint object
+ * @request: request object
+ */
+static void cdns3_wa2_descmiss_copy_data(struct cdns3_endpoint *priv_ep,
+					 struct usb_request *request)
+{
+	struct usb_request *descmiss_req;
+	struct cdns3_request *descmiss_priv_req;
+
+	while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
+		int chunk_end;
+		int length;
+
+		descmiss_priv_req =
+			cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
+		descmiss_req = &descmiss_priv_req->request;
+
+		/* driver can't touch pending request */
+		if (descmiss_priv_req->flags & REQUEST_PENDING)
+			break;
+
+		chunk_end = descmiss_priv_req->flags & REQUEST_INTERNAL_CH;
+		length = request->actual + descmiss_req->actual;
+
+		request->status = descmiss_req->status;
+
+		if (length <= request->length) {
+			memcpy(&((u8 *)request->buf)[request->actual],
+			       descmiss_req->buf,
+			       descmiss_req->actual);
+			request->actual = length;
+		} else {
+			/* It should never occures */
+			request->status = -ENOMEM;
+		}
+
+		list_del_init(&descmiss_priv_req->list);
+
+		kfree(descmiss_req->buf);
+		cdns3_gadget_ep_free_request(&priv_ep->endpoint, descmiss_req);
+		--priv_ep->wa2_counter;
+
+		if (!chunk_end)
+			break;
+	}
+}
+
+struct usb_request *cdns3_wa2_gadget_giveback(struct cdns3_device *priv_dev,
+					      struct cdns3_endpoint *priv_ep,
+					      struct cdns3_request *priv_req)
+{
+	if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN &&
+	    priv_req->flags & REQUEST_INTERNAL) {
+		struct usb_request *req;
+
+		req = cdns3_next_request(&priv_ep->deferred_req_list);
+
+		priv_ep->descmis_req = NULL;
+
+		if (!req)
+			return NULL;
+
+		cdns3_wa2_descmiss_copy_data(priv_ep, req);
+		if (!(priv_ep->flags & EP_QUIRK_END_TRANSFER) &&
+		    req->length != req->actual) {
+			/* wait for next part of transfer */
+			return NULL;
+		}
+
+		if (req->status == -EINPROGRESS)
+			req->status = 0;
+
+		list_del_init(&req->list);
+		cdns3_start_all_request(priv_dev, priv_ep);
+		return req;
+	}
+
+	return &priv_req->request;
+}
+
+int cdns3_wa2_gadget_ep_queue(struct cdns3_device *priv_dev,
+			      struct cdns3_endpoint *priv_ep,
+			      struct cdns3_request *priv_req)
+{
+	int deferred = 0;
+
+	/*
+	 * If transfer was queued before DESCMISS appear than we
+	 * can disable handling of DESCMISS interrupt. Driver assumes that it
+	 * can disable special treatment for this endpoint.
+	 */
+	if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
+		u32 reg;
+
+		cdns3_select_ep(priv_dev, priv_ep->num | priv_ep->dir);
+		priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
+		reg = readl(&priv_dev->regs->ep_sts_en);
+		reg &= ~EP_STS_EN_DESCMISEN;
+		trace_cdns3_wa2(priv_ep, "workaround disabled\n");
+		writel(reg, &priv_dev->regs->ep_sts_en);
+	}
+
+	if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
+		u8 pending_empty = list_empty(&priv_ep->pending_req_list);
+		u8 descmiss_empty = list_empty(&priv_ep->wa2_descmiss_req_list);
+
+		/*
+		 *  DESCMISS transfer has been finished, so data will be
+		 *  directly copied from internal allocated usb_request
+		 *  objects.
+		 */
+		if (pending_empty && !descmiss_empty &&
+		    !(priv_req->flags & REQUEST_INTERNAL)) {
+			cdns3_wa2_descmiss_copy_data(priv_ep,
+						     &priv_req->request);
+
+			trace_cdns3_wa2(priv_ep, "get internal stored data");
+
+			list_add_tail(&priv_req->request.list,
+				      &priv_ep->pending_req_list);
+			cdns3_gadget_giveback(priv_ep, priv_req,
+					      priv_req->request.status);
+
+			/*
+			 * Intentionally driver returns positive value as
+			 * correct value. It informs that transfer has
+			 * been finished.
+			 */
+			return EINPROGRESS;
+		}
+
+		/*
+		 * Driver will wait for completion DESCMISS transfer,
+		 * before starts new, not DESCMISS transfer.
+		 */
+		if (!pending_empty && !descmiss_empty) {
+			trace_cdns3_wa2(priv_ep, "wait for pending transfer\n");
+			deferred = 1;
+		}
+
+		if (priv_req->flags & REQUEST_INTERNAL)
+			list_add_tail(&priv_req->list,
+				      &priv_ep->wa2_descmiss_req_list);
+	}
+
+	return deferred;
+}
+
+static void cdns3_wa2_remove_old_request(struct cdns3_endpoint *priv_ep)
+{
+	struct cdns3_request *priv_req;
+
+	while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
+		u8 chain;
+
+		priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
+		chain = !!(priv_req->flags & REQUEST_INTERNAL_CH);
+
+		trace_cdns3_wa2(priv_ep, "removes eldest request");
+
+		kfree(priv_req->request.buf);
+		cdns3_gadget_ep_free_request(&priv_ep->endpoint,
+					     &priv_req->request);
+		list_del_init(&priv_req->list);
+		--priv_ep->wa2_counter;
+
+		if (!chain)
+			break;
+	}
+}
+
+/**
+ * cdns3_wa2_descmissing_packet - handles descriptor missing event.
+ * @priv_dev: extended gadget object
+ *
+ * This function is used only for WA2. For more information see Work around 2
+ * description.
+ */
+static void cdns3_wa2_descmissing_packet(struct cdns3_endpoint *priv_ep)
+{
+	struct cdns3_request *priv_req;
+	struct usb_request *request;
+
+	if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
+		priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
+		priv_ep->flags |= EP_QUIRK_EXTRA_BUF_EN;
+	}
+
+	trace_cdns3_wa2(priv_ep, "Description Missing detected\n");
+
+	if (priv_ep->wa2_counter >= CDNS3_WA2_NUM_BUFFERS)
+		cdns3_wa2_remove_old_request(priv_ep);
+
+	request = cdns3_gadget_ep_alloc_request(&priv_ep->endpoint,
+						GFP_ATOMIC);
+	if (!request)
+		goto err;
+
+	priv_req = to_cdns3_request(request);
+	priv_req->flags |= REQUEST_INTERNAL;
+
+	/* if this field is still assigned it indicate that transfer related
+	 * with this request has not been finished yet. Driver in this
+	 * case simply allocate next request and assign flag REQUEST_INTERNAL_CH
+	 * flag to previous one. It will indicate that current request is
+	 * part of the previous one.
+	 */
+	if (priv_ep->descmis_req)
+		priv_ep->descmis_req->flags |= REQUEST_INTERNAL_CH;
+
+	priv_req->request.buf = kzalloc(CDNS3_DESCMIS_BUF_SIZE,
+					GFP_ATOMIC);
+	priv_ep->wa2_counter++;
+
+	if (!priv_req->request.buf) {
+		cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
+		goto err;
+	}
+
+	priv_req->request.length = CDNS3_DESCMIS_BUF_SIZE;
+	priv_ep->descmis_req = priv_req;
+
+	__cdns3_gadget_ep_queue(&priv_ep->endpoint,
+				&priv_ep->descmis_req->request,
+				GFP_ATOMIC);
+
+	return;
+
+err:
+	dev_err(priv_ep->cdns3_dev->dev,
+		"Failed: No sufficient memory for DESCMIS\n");
+}
+
+/**
+ * cdns3_gadget_giveback - call struct usb_request's ->complete callback
+ * @priv_ep: The endpoint to whom the request belongs to
+ * @priv_req: The request we're giving back
+ * @status: completion code for the request
+ *
+ * Must be called with controller's lock held and interrupts disabled. This
+ * function will unmap @req and call its ->complete() callback to notify upper
+ * layers that it has completed.
+ */
+void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep,
+			   struct cdns3_request *priv_req,
+			   int status)
+{
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+	struct usb_request *request = &priv_req->request;
+
+	list_del_init(&request->list);
+
+	if (request->status == -EINPROGRESS)
+		request->status = status;
+
+	usb_gadget_unmap_request_by_dev(priv_dev->sysdev, request,
+					priv_ep->dir);
+
+	if ((priv_req->flags & REQUEST_UNALIGNED) &&
+	    priv_ep->dir == USB_DIR_OUT && !request->status)
+		memcpy(request->buf, priv_req->aligned_buf->buf,
+		       request->length);
+
+	priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED);
+	trace_cdns3_gadget_giveback(priv_req);
+
+	if (priv_dev->dev_ver < DEV_VER_V2) {
+		request = cdns3_wa2_gadget_giveback(priv_dev, priv_ep,
+						    priv_req);
+		if (!request)
+			return;
+	}
+
+	if (request->complete) {
+		spin_unlock(&priv_dev->lock);
+		usb_gadget_giveback_request(&priv_ep->endpoint,
+					    request);
+		spin_lock(&priv_dev->lock);
+	}
+
+	if (request->buf == priv_dev->zlp_buf)
+		cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
+}
+
+void cdns3_wa1_restore_cycle_bit(struct cdns3_endpoint *priv_ep)
+{
+	/* Work around for stale data address in TRB*/
+	if (priv_ep->wa1_set) {
+		trace_cdns3_wa1(priv_ep, "restore cycle bit");
+
+		priv_ep->wa1_set = 0;
+		priv_ep->wa1_trb_index = 0xFFFF;
+		if (priv_ep->wa1_cycle_bit) {
+			priv_ep->wa1_trb->control =
+				priv_ep->wa1_trb->control | 0x1;
+		} else {
+			priv_ep->wa1_trb->control =
+				priv_ep->wa1_trb->control & ~0x1;
+		}
+	}
+}
+
+static void cdns3_free_aligned_request_buf(struct work_struct *work)
+{
+	struct cdns3_device *priv_dev = container_of(work, struct cdns3_device,
+					aligned_buf_wq);
+	struct cdns3_aligned_buf *buf, *tmp;
+	unsigned long flags;
+
+	spin_lock_irqsave(&priv_dev->lock, flags);
+
+	list_for_each_entry_safe(buf, tmp, &priv_dev->aligned_buf_list, list) {
+		if (!buf->in_use) {
+			list_del(&buf->list);
+
+			/*
+			 * Re-enable interrupts to free DMA capable memory.
+			 * Driver can't free this memory with disabled
+			 * interrupts.
+			 */
+			spin_unlock_irqrestore(&priv_dev->lock, flags);
+			dma_free_coherent(priv_dev->sysdev, buf->size,
+					  buf->buf, buf->dma);
+			kfree(buf);
+			spin_lock_irqsave(&priv_dev->lock, flags);
+		}
+	}
+
+	spin_unlock_irqrestore(&priv_dev->lock, flags);
+}
+
+static int cdns3_prepare_aligned_request_buf(struct cdns3_request *priv_req)
+{
+	struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+	struct cdns3_aligned_buf *buf;
+
+	/* check if buffer is aligned to 8. */
+	if (!((uintptr_t)priv_req->request.buf & 0x7))
+		return 0;
+
+	buf = priv_req->aligned_buf;
+
+	if (!buf || priv_req->request.length > buf->size) {
+		buf = kzalloc(sizeof(*buf), GFP_ATOMIC);
+		if (!buf)
+			return -ENOMEM;
+
+		buf->size = priv_req->request.length;
+
+		buf->buf = dma_alloc_coherent(priv_dev->sysdev,
+					      buf->size,
+					      &buf->dma,
+					      GFP_ATOMIC);
+		if (!buf->buf) {
+			kfree(buf);
+			return -ENOMEM;
+		}
+
+		if (priv_req->aligned_buf) {
+			trace_cdns3_free_aligned_request(priv_req);
+			priv_req->aligned_buf->in_use = 0;
+			queue_work(system_freezable_wq,
+				   &priv_dev->aligned_buf_wq);
+		}
+
+		buf->in_use = 1;
+		priv_req->aligned_buf = buf;
+
+		list_add_tail(&buf->list,
+			      &priv_dev->aligned_buf_list);
+	}
+
+	if (priv_ep->dir == USB_DIR_IN) {
+		memcpy(buf->buf, priv_req->request.buf,
+		       priv_req->request.length);
+	}
+
+	priv_req->flags |= REQUEST_UNALIGNED;
+	trace_cdns3_prepare_aligned_request(priv_req);
+
+	return 0;
+}
+
+static int cdns3_wa1_update_guard(struct cdns3_endpoint *priv_ep,
+				  struct cdns3_trb *trb)
+{
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+
+	if (!priv_ep->wa1_set) {
+		u32 doorbell;
+
+		doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
+
+		if (doorbell) {
+			priv_ep->wa1_cycle_bit = priv_ep->pcs ? TRB_CYCLE : 0;
+			priv_ep->wa1_set = 1;
+			priv_ep->wa1_trb = trb;
+			priv_ep->wa1_trb_index = priv_ep->enqueue;
+			trace_cdns3_wa1(priv_ep, "set guard");
+			return 0;
+		}
+	}
+	return 1;
+}
+
+static void cdns3_wa1_tray_restore_cycle_bit(struct cdns3_device *priv_dev,
+					     struct cdns3_endpoint *priv_ep)
+{
+	int dma_index;
+	u32 doorbell;
+
+	doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
+	dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
+
+	if (!doorbell || dma_index != priv_ep->wa1_trb_index)
+		cdns3_wa1_restore_cycle_bit(priv_ep);
+}
+
+/**
+ * cdns3_ep_run_transfer - start transfer on no-default endpoint hardware
+ * @priv_ep: endpoint object
+ *
+ * Returns zero on success or negative value on failure
+ */
+int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
+			  struct usb_request *request)
+{
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+	struct cdns3_request *priv_req;
+	struct cdns3_trb *trb;
+	dma_addr_t trb_dma;
+	u32 togle_pcs = 1;
+	int sg_iter = 0;
+	int num_trb;
+	int address;
+	u32 control;
+	int pcs;
+
+	if (priv_ep->type == USB_ENDPOINT_XFER_ISOC)
+		num_trb = priv_ep->interval;
+	else
+		num_trb = request->num_sgs ? request->num_sgs : 1;
+
+	if (num_trb > priv_ep->free_trbs) {
+		priv_ep->flags |= EP_RING_FULL;
+		return -ENOBUFS;
+	}
+
+	priv_req = to_cdns3_request(request);
+	address = priv_ep->endpoint.desc->bEndpointAddress;
+
+	priv_ep->flags |= EP_PENDING_REQUEST;
+
+	/* must allocate buffer aligned to 8 */
+	if (priv_req->flags & REQUEST_UNALIGNED)
+		trb_dma = priv_req->aligned_buf->dma;
+	else
+		trb_dma = request->dma;
+
+	trb = priv_ep->trb_pool + priv_ep->enqueue;
+	priv_req->start_trb = priv_ep->enqueue;
+	priv_req->trb = trb;
+
+	cdns3_select_ep(priv_ep->cdns3_dev, address);
+
+	/* prepare ring */
+	if ((priv_ep->enqueue + num_trb)  >= (priv_ep->num_trbs - 1)) {
+		struct cdns3_trb *link_trb;
+		int doorbell, dma_index;
+		u32 ch_bit = 0;
+
+		doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
+		dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
+
+		/* Driver can't update LINK TRB if it is current processed. */
+		if (doorbell && dma_index == priv_ep->num_trbs - 1) {
+			priv_ep->flags |= EP_DEFERRED_DRDY;
+			return -ENOBUFS;
+		}
+
+		/*updating C bt in  Link TRB before starting DMA*/
+		link_trb = priv_ep->trb_pool + (priv_ep->num_trbs - 1);
+		/*
+		 * For TRs size equal 2 enabling TRB_CHAIN for epXin causes
+		 * that DMA stuck at the LINK TRB.
+		 * On the other hand, removing TRB_CHAIN for longer TRs for
+		 * epXout cause that DMA stuck after handling LINK TRB.
+		 * To eliminate this strange behavioral driver set TRB_CHAIN
+		 * bit only for TR size > 2.
+		 */
+		if (priv_ep->type == USB_ENDPOINT_XFER_ISOC ||
+		    TRBS_PER_SEGMENT > 2)
+			ch_bit = TRB_CHAIN;
+
+		link_trb->control = ((priv_ep->pcs) ? TRB_CYCLE : 0) |
+				    TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit;
+	}
+
+	if (priv_dev->dev_ver <= DEV_VER_V2)
+		togle_pcs = cdns3_wa1_update_guard(priv_ep, trb);
+
+	/* set incorrect Cycle Bit for first trb*/
+	control = priv_ep->pcs ? 0 : TRB_CYCLE;
+
+	do {
+		u32 length;
+		u16 td_size = 0;
+
+		/* fill TRB */
+		control |= TRB_TYPE(TRB_NORMAL);
+		trb->buffer = TRB_BUFFER(request->num_sgs == 0
+				? trb_dma : request->sg[sg_iter].dma_address);
+
+		if (likely(!request->num_sgs))
+			length = request->length;
+		else
+			length = request->sg[sg_iter].length;
+
+		if (likely(priv_dev->dev_ver >= DEV_VER_V2))
+			td_size = DIV_ROUND_UP(length,
+					       priv_ep->endpoint.maxpacket);
+
+		trb->length = TRB_BURST_LEN(priv_ep->trb_burst_size) |
+					TRB_LEN(length);
+		if (priv_dev->gadget.speed == USB_SPEED_SUPER)
+			trb->length |= TRB_TDL_SS_SIZE(td_size);
+		else
+			control |= TRB_TDL_HS_SIZE(td_size);
+
+		pcs = priv_ep->pcs ? TRB_CYCLE : 0;
+
+		/*
+		 * first trb should be prepared as last to avoid processing
+		 *  transfer to early
+		 */
+		if (sg_iter != 0)
+			control |= pcs;
+
+		if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir) {
+			control |= TRB_IOC | TRB_ISP;
+		} else {
+			/* for last element in TD or in SG list */
+			if (sg_iter == (num_trb - 1) && sg_iter != 0)
+				control |= pcs | TRB_IOC | TRB_ISP;
+		}
+
+		if (sg_iter)
+			trb->control = control;
+		else
+			priv_req->trb->control = control;
+
+		control = 0;
+		++sg_iter;
+		priv_req->end_trb = priv_ep->enqueue;
+		cdns3_ep_inc_enq(priv_ep);
+		trb = priv_ep->trb_pool + priv_ep->enqueue;
+	} while (sg_iter < num_trb);
+
+	trb = priv_req->trb;
+
+	priv_req->flags |= REQUEST_PENDING;
+
+	if (sg_iter == 1)
+		trb->control |= TRB_IOC | TRB_ISP;
+
+	/*
+	 * Memory barrier - cycle bit must be set before other filds in trb.
+	 */
+	wmb();
+
+	/* give the TD to the consumer*/
+	if (togle_pcs)
+		trb->control =  trb->control ^ 1;
+
+	if (priv_dev->dev_ver <= DEV_VER_V2)
+		cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep);
+
+	trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
+
+	/*
+	 * Memory barrier - Cycle Bit must be set before trb->length  and
+	 * trb->buffer fields.
+	 */
+	wmb();
+
+	/*
+	 * For DMULT mode we can set address to transfer ring only once after
+	 * enabling endpoint.
+	 */
+	if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) {
+		/*
+		 * Until SW is not ready to handle the OUT transfer the ISO OUT
+		 * Endpoint should be disabled (EP_CFG.ENABLE = 0).
+		 * EP_CFG_ENABLE must be set before updating ep_traddr.
+		 */
+		if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir &&
+		    !(priv_ep->flags & EP_QUIRK_ISO_OUT_EN)) {
+			priv_ep->flags |= EP_QUIRK_ISO_OUT_EN;
+			cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
+					       EP_CFG_ENABLE);
+		}
+
+		writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma +
+					priv_req->start_trb * TRB_SIZE),
+					&priv_dev->regs->ep_traddr);
+
+		priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR;
+	}
+
+	if (!priv_ep->wa1_set && !(priv_ep->flags & EP_STALLED)) {
+		trace_cdns3_ring(priv_ep);
+		/*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
+		writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
+		writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
+		trace_cdns3_doorbell_epx(priv_ep->name,
+					 readl(&priv_dev->regs->ep_traddr));
+	}
+
+	/* WORKAROUND for transition to L0 */
+	__cdns3_gadget_wakeup(priv_dev);
+
+	return 0;
+}
+
+void cdns3_set_hw_configuration(struct cdns3_device *priv_dev)
+{
+	struct cdns3_endpoint *priv_ep;
+	struct usb_ep *ep;
+	int val;
+
+	if (priv_dev->hw_configured_flag)
+		return;
+
+	writel(USB_CONF_CFGSET, &priv_dev->regs->usb_conf);
+	writel(EP_CMD_ERDY | EP_CMD_REQ_CMPL, &priv_dev->regs->ep_cmd);
+
+	cdns3_set_register_bit(&priv_dev->regs->usb_conf,
+			       USB_CONF_U1EN | USB_CONF_U2EN);
+
+	/* wait until configuration set */
+	readl_poll_timeout_atomic(&priv_dev->regs->usb_sts, val,
+				  val & USB_STS_CFGSTS_MASK, 1, 100);
+
+	priv_dev->hw_configured_flag = 1;
+
+	list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
+		if (ep->enabled) {
+			priv_ep = ep_to_cdns3_ep(ep);
+			cdns3_start_all_request(priv_dev, priv_ep);
+		}
+	}
+}
+
+/**
+ * cdns3_request_handled - check whether request has been handled by DMA
+ *
+ * @priv_ep: extended endpoint object.
+ * @priv_req: request object for checking
+ *
+ * Endpoint must be selected before invoking this function.
+ *
+ * Returns false if request has not been handled by DMA, else returns true.
+ *
+ * SR - start ring
+ * ER -  end ring
+ * DQ = priv_ep->dequeue - dequeue position
+ * EQ = priv_ep->enqueue -  enqueue position
+ * ST = priv_req->start_trb - index of first TRB in transfer ring
+ * ET = priv_req->end_trb - index of last TRB in transfer ring
+ * CI = current_index - index of processed TRB by DMA.
+ *
+ * As first step, function checks if cycle bit for priv_req->start_trb is
+ * correct.
+ *
+ * some rules:
+ * 1. priv_ep->dequeue never exceed current_index.
+ * 2  priv_ep->enqueue never exceed priv_ep->dequeue
+ * 3. exception: priv_ep->enqueue == priv_ep->dequeue
+ *    and priv_ep->free_trbs is zero.
+ *    This case indicate that TR is full.
+ *
+ * Then We can split recognition into two parts:
+ * Case 1 - priv_ep->dequeue < current_index
+ *      SR ... EQ ... DQ ... CI ... ER
+ *      SR ... DQ ... CI ... EQ ... ER
+ *
+ *      Request has been handled by DMA if ST and ET is between DQ and CI.
+ *
+ * Case 2 - priv_ep->dequeue > current_index
+ * This situation take place when CI go through the LINK TRB at the end of
+ * transfer ring.
+ *      SR ... CI ... EQ ... DQ ... ER
+ *
+ *      Request has been handled by DMA if ET is less then CI or
+ *      ET is greater or equal DQ.
+ */
+static bool cdns3_request_handled(struct cdns3_endpoint *priv_ep,
+				  struct cdns3_request *priv_req)
+{
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+	struct cdns3_trb *trb = priv_req->trb;
+	int current_index = 0;
+	int handled = 0;
+	int doorbell;
+
+	current_index = cdns3_get_dma_pos(priv_dev, priv_ep);
+	doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
+
+	trb = &priv_ep->trb_pool[priv_req->start_trb];
+
+	if ((trb->control  & TRB_CYCLE) != priv_ep->ccs)
+		goto finish;
+
+	if (doorbell == 1 && current_index == priv_ep->dequeue)
+		goto finish;
+
+	/* The corner case for TRBS_PER_SEGMENT equal 2). */
+	if (TRBS_PER_SEGMENT == 2 && priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
+		handled = 1;
+		goto finish;
+	}
+
+	if (priv_ep->enqueue == priv_ep->dequeue &&
+	    priv_ep->free_trbs == 0) {
+		handled = 1;
+	} else if (priv_ep->dequeue < current_index) {
+		if ((current_index == (priv_ep->num_trbs - 1)) &&
+		    !priv_ep->dequeue)
+			goto finish;
+
+		if (priv_req->end_trb >= priv_ep->dequeue &&
+		    priv_req->end_trb < current_index)
+			handled = 1;
+	} else if (priv_ep->dequeue  > current_index) {
+		if (priv_req->end_trb  < current_index ||
+		    priv_req->end_trb >= priv_ep->dequeue)
+			handled = 1;
+	}
+
+finish:
+	trace_cdns3_request_handled(priv_req, current_index, handled);
+
+	return handled;
+}
+
+static void cdns3_transfer_completed(struct cdns3_device *priv_dev,
+				     struct cdns3_endpoint *priv_ep)
+{
+	struct cdns3_request *priv_req;
+	struct usb_request *request;
+	struct cdns3_trb *trb;
+
+	while (!list_empty(&priv_ep->pending_req_list)) {
+		request = cdns3_next_request(&priv_ep->pending_req_list);
+		priv_req = to_cdns3_request(request);
+
+		trb = priv_ep->trb_pool + priv_ep->dequeue;
+
+		/* Request was dequeued and TRB was changed to TRB_LINK. */
+		if (TRB_FIELD_TO_TYPE(trb->control) == TRB_LINK) {
+			trace_cdns3_complete_trb(priv_ep, trb);
+			cdns3_move_deq_to_next_trb(priv_req);
+		}
+
+		/* Re-select endpoint. It could be changed by other CPU during
+		 * handling usb_gadget_giveback_request.
+		 */
+		cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
+
+		if (!cdns3_request_handled(priv_ep, priv_req))
+			goto prepare_next_td;
+
+		trb = priv_ep->trb_pool + priv_ep->dequeue;
+		trace_cdns3_complete_trb(priv_ep, trb);
+
+		if (trb != priv_req->trb)
+			dev_warn(priv_dev->dev,
+				 "request_trb=0x%p, queue_trb=0x%p\n",
+				 priv_req->trb, trb);
+
+		request->actual = TRB_LEN(le32_to_cpu(trb->length));
+		cdns3_move_deq_to_next_trb(priv_req);
+		cdns3_gadget_giveback(priv_ep, priv_req, 0);
+
+		if (priv_ep->type != USB_ENDPOINT_XFER_ISOC &&
+		    TRBS_PER_SEGMENT == 2)
+			break;
+	}
+	priv_ep->flags &= ~EP_PENDING_REQUEST;
+
+prepare_next_td:
+	if (!(priv_ep->flags & EP_STALLED) &&
+	    !(priv_ep->flags & EP_STALL_PENDING))
+		cdns3_start_all_request(priv_dev, priv_ep);
+}
+
+void cdns3_rearm_transfer(struct cdns3_endpoint *priv_ep, u8 rearm)
+{
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+
+	cdns3_wa1_restore_cycle_bit(priv_ep);
+
+	if (rearm) {
+		trace_cdns3_ring(priv_ep);
+
+		/* Cycle Bit must be updated before arming DMA. */
+		wmb();
+		writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
+
+		__cdns3_gadget_wakeup(priv_dev);
+
+		trace_cdns3_doorbell_epx(priv_ep->name,
+					 readl(&priv_dev->regs->ep_traddr));
+	}
+}
+
+/**
+ * cdns3_check_ep_interrupt_proceed - Processes interrupt related to endpoint
+ * @priv_ep: endpoint object
+ *
+ * Returns 0
+ */
+static int cdns3_check_ep_interrupt_proceed(struct cdns3_endpoint *priv_ep)
+{
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+	u32 ep_sts_reg;
+
+	cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
+
+	trace_cdns3_epx_irq(priv_dev, priv_ep);
+
+	ep_sts_reg = readl(&priv_dev->regs->ep_sts);
+	writel(ep_sts_reg, &priv_dev->regs->ep_sts);
+
+	if (ep_sts_reg & EP_STS_TRBERR) {
+		if (priv_ep->flags & EP_STALL_PENDING &&
+		    !(ep_sts_reg & EP_STS_DESCMIS &&
+		    priv_dev->dev_ver < DEV_VER_V2)) {
+			cdns3_ep_stall_flush(priv_ep);
+		}
+
+		/*
+		 * For isochronous transfer driver completes request on
+		 * IOC or on TRBERR. IOC appears only when device receive
+		 * OUT data packet. If host disable stream or lost some packet
+		 * then the only way to finish all queued transfer is to do it
+		 * on TRBERR event.
+		 */
+		if (priv_ep->type == USB_ENDPOINT_XFER_ISOC &&
+		    !priv_ep->wa1_set) {
+			if (!priv_ep->dir) {
+				u32 ep_cfg = readl(&priv_dev->regs->ep_cfg);
+
+				ep_cfg &= ~EP_CFG_ENABLE;
+				writel(ep_cfg, &priv_dev->regs->ep_cfg);
+				priv_ep->flags &= ~EP_QUIRK_ISO_OUT_EN;
+			}
+			cdns3_transfer_completed(priv_dev, priv_ep);
+		} else if (!(priv_ep->flags & EP_STALLED) &&
+			  !(priv_ep->flags & EP_STALL_PENDING)) {
+			if (priv_ep->flags & EP_DEFERRED_DRDY) {
+				priv_ep->flags &= ~EP_DEFERRED_DRDY;
+				cdns3_start_all_request(priv_dev, priv_ep);
+			} else {
+				cdns3_rearm_transfer(priv_ep,
+						     priv_ep->wa1_set);
+			}
+		}
+	}
+
+	if ((ep_sts_reg & EP_STS_IOC) || (ep_sts_reg & EP_STS_ISP)) {
+		if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
+			if (ep_sts_reg & EP_STS_ISP)
+				priv_ep->flags |= EP_QUIRK_END_TRANSFER;
+			else
+				priv_ep->flags &= ~EP_QUIRK_END_TRANSFER;
+		}
+
+		cdns3_transfer_completed(priv_dev, priv_ep);
+	}
+
+	/*
+	 * WA2: this condition should only be meet when
+	 * priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET or
+	 * priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN.
+	 * In other cases this interrupt will be disabled/
+	 */
+	if (ep_sts_reg & EP_STS_DESCMIS && priv_dev->dev_ver < DEV_VER_V2 &&
+	    !(priv_ep->flags & EP_STALLED))
+		cdns3_wa2_descmissing_packet(priv_ep);
+
+	return 0;
+}
+
+static void cdns3_disconnect_gadget(struct cdns3_device *priv_dev)
+{
+	if (priv_dev->gadget_driver && priv_dev->gadget_driver->disconnect) {
+		spin_unlock(&priv_dev->lock);
+		priv_dev->gadget_driver->disconnect(&priv_dev->gadget);
+		spin_lock(&priv_dev->lock);
+	}
+}
+
+/**
+ * cdns3_check_usb_interrupt_proceed - Processes interrupt related to device
+ * @priv_dev: extended gadget object
+ * @usb_ists: bitmap representation of device's reported interrupts
+ * (usb_ists register value)
+ */
+static void cdns3_check_usb_interrupt_proceed(struct cdns3_device *priv_dev,
+					      u32 usb_ists)
+{
+	int speed = 0;
+
+	trace_cdns3_usb_irq(priv_dev, usb_ists);
+	if (usb_ists & USB_ISTS_L1ENTI) {
+		/*
+		 * WORKAROUND: CDNS3 controller has issue with hardware resuming
+		 * from L1. To fix it, if any DMA transfer is pending driver
+		 * must starts driving resume signal immediately.
+		 */
+		if (readl(&priv_dev->regs->drbl))
+			__cdns3_gadget_wakeup(priv_dev);
+	}
+
+	/* Connection detected */
+	if (usb_ists & (USB_ISTS_CON2I | USB_ISTS_CONI)) {
+		speed = cdns3_get_speed(priv_dev);
+		priv_dev->gadget.speed = speed;
+		usb_gadget_set_state(&priv_dev->gadget, USB_STATE_POWERED);
+		cdns3_ep0_config(priv_dev);
+	}
+
+	/* Disconnection detected */
+	if (usb_ists & (USB_ISTS_DIS2I | USB_ISTS_DISI)) {
+		cdns3_disconnect_gadget(priv_dev);
+		priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
+		usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
+		cdns3_hw_reset_eps_config(priv_dev);
+	}
+
+	if (usb_ists & (USB_ISTS_L2ENTI | USB_ISTS_U3ENTI)) {
+		if (priv_dev->gadget_driver &&
+		    priv_dev->gadget_driver->suspend) {
+			spin_unlock(&priv_dev->lock);
+			priv_dev->gadget_driver->suspend(&priv_dev->gadget);
+			spin_lock(&priv_dev->lock);
+		}
+	}
+
+	if (usb_ists & (USB_ISTS_L2EXTI | USB_ISTS_U3EXTI)) {
+		if (priv_dev->gadget_driver &&
+		    priv_dev->gadget_driver->resume) {
+			spin_unlock(&priv_dev->lock);
+			priv_dev->gadget_driver->resume(&priv_dev->gadget);
+			spin_lock(&priv_dev->lock);
+		}
+	}
+
+	/* reset*/
+	if (usb_ists & (USB_ISTS_UWRESI | USB_ISTS_UHRESI | USB_ISTS_U2RESI)) {
+		if (priv_dev->gadget_driver) {
+			spin_unlock(&priv_dev->lock);
+			usb_gadget_udc_reset(&priv_dev->gadget,
+					     priv_dev->gadget_driver);
+			spin_lock(&priv_dev->lock);
+
+			/*read again to check the actual speed*/
+			speed = cdns3_get_speed(priv_dev);
+			priv_dev->gadget.speed = speed;
+			cdns3_hw_reset_eps_config(priv_dev);
+			cdns3_ep0_config(priv_dev);
+		}
+	}
+}
+
+/**
+ * cdns3_device_irq_handler- interrupt handler for device part of controller
+ *
+ * @irq: irq number for cdns3 core device
+ * @data: structure of cdns3
+ *
+ * Returns IRQ_HANDLED or IRQ_NONE
+ */
+static irqreturn_t cdns3_device_irq_handler(int irq, void *data)
+{
+	struct cdns3_device *priv_dev;
+	struct cdns3 *cdns = data;
+	irqreturn_t ret = IRQ_NONE;
+	u32 reg;
+
+	priv_dev = cdns->gadget_dev;
+
+	/* check USB device interrupt */
+	reg = readl(&priv_dev->regs->usb_ists);
+	if (reg) {
+		/* After masking interrupts the new interrupts won't be
+		 * reported in usb_ists/ep_ists. In order to not lose some
+		 * of them driver disables only detected interrupts.
+		 * They will be enabled ASAP after clearing source of
+		 * interrupt. This an unusual behavior only applies to
+		 * usb_ists register.
+		 */
+		reg = ~reg & readl(&priv_dev->regs->usb_ien);
+		/* mask deferred interrupt. */
+		writel(reg, &priv_dev->regs->usb_ien);
+		ret = IRQ_WAKE_THREAD;
+	}
+
+	/* check endpoint interrupt */
+	reg = readl(&priv_dev->regs->ep_ists);
+	if (reg) {
+		writel(0, &priv_dev->regs->ep_ien);
+		ret = IRQ_WAKE_THREAD;
+	}
+
+	return ret;
+}
+
+/**
+ * cdns3_device_thread_irq_handler- interrupt handler for device part
+ * of controller
+ *
+ * @irq: irq number for cdns3 core device
+ * @data: structure of cdns3
+ *
+ * Returns IRQ_HANDLED or IRQ_NONE
+ */
+static irqreturn_t cdns3_device_thread_irq_handler(int irq, void *data)
+{
+	struct cdns3_device *priv_dev;
+	struct cdns3 *cdns = data;
+	irqreturn_t ret = IRQ_NONE;
+	unsigned long flags;
+	int bit;
+	u32 reg;
+
+	priv_dev = cdns->gadget_dev;
+	spin_lock_irqsave(&priv_dev->lock, flags);
+
+	reg = readl(&priv_dev->regs->usb_ists);
+	if (reg) {
+		writel(reg, &priv_dev->regs->usb_ists);
+		writel(USB_IEN_INIT, &priv_dev->regs->usb_ien);
+		cdns3_check_usb_interrupt_proceed(priv_dev, reg);
+		ret = IRQ_HANDLED;
+	}
+
+	reg = readl(&priv_dev->regs->ep_ists);
+
+	/* handle default endpoint OUT */
+	if (reg & EP_ISTS_EP_OUT0) {
+		cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_OUT);
+		ret = IRQ_HANDLED;
+	}
+
+	/* handle default endpoint IN */
+	if (reg & EP_ISTS_EP_IN0) {
+		cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_IN);
+		ret = IRQ_HANDLED;
+	}
+
+	/* check if interrupt from non default endpoint, if no exit */
+	reg &= ~(EP_ISTS_EP_OUT0 | EP_ISTS_EP_IN0);
+	if (!reg)
+		goto irqend;
+
+	for_each_set_bit(bit, (unsigned long *)&reg,
+			 sizeof(u32) * BITS_PER_BYTE) {
+		cdns3_check_ep_interrupt_proceed(priv_dev->eps[bit]);
+		ret = IRQ_HANDLED;
+	}
+
+irqend:
+	writel(~0, &priv_dev->regs->ep_ien);
+	spin_unlock_irqrestore(&priv_dev->lock, flags);
+
+	return ret;
+}
+
+/**
+ * cdns3_ep_onchip_buffer_reserve - Try to reserve onchip buf for EP
+ *
+ * The real reservation will occur during write to EP_CFG register,
+ * this function is used to check if the 'size' reservation is allowed.
+ *
+ * @priv_dev: extended gadget object
+ * @size: the size (KB) for EP would like to allocate
+ * @is_in: endpoint direction
+ *
+ * Return 0 if the required size can met or negative value on failure
+ */
+static int cdns3_ep_onchip_buffer_reserve(struct cdns3_device *priv_dev,
+					  int size, int is_in)
+{
+	int remained;
+
+	/* 2KB are reserved for EP0*/
+	remained = priv_dev->onchip_buffers - priv_dev->onchip_used_size - 2;
+
+	if (is_in) {
+		if (remained < size)
+			return -EPERM;
+
+		priv_dev->onchip_used_size += size;
+	} else {
+		int required;
+
+		/**
+		 *  ALL OUT EPs are shared the same chunk onchip memory, so
+		 * driver checks if it already has assigned enough buffers
+		 */
+		if (priv_dev->out_mem_is_allocated >= size)
+			return 0;
+
+		required = size - priv_dev->out_mem_is_allocated;
+
+		if (required > remained)
+			return -EPERM;
+
+		priv_dev->out_mem_is_allocated += required;
+		priv_dev->onchip_used_size += required;
+	}
+
+	return 0;
+}
+
+void cdns3_configure_dmult(struct cdns3_device *priv_dev,
+			   struct cdns3_endpoint *priv_ep)
+{
+	struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
+
+	/* For dev_ver > DEV_VER_V2 DMULT is configured per endpoint */
+	if (priv_dev->dev_ver <= DEV_VER_V2)
+		writel(USB_CONF_DMULT, &regs->usb_conf);
+
+	if (priv_dev->dev_ver == DEV_VER_V2)
+		writel(USB_CONF2_EN_TDL_TRB, &regs->usb_conf2);
+
+	if (priv_dev->dev_ver >= DEV_VER_V3 && priv_ep) {
+		u32 mask;
+
+		if (priv_ep->dir)
+			mask = BIT(priv_ep->num + 16);
+		else
+			mask = BIT(priv_ep->num);
+
+		if (priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
+			cdns3_set_register_bit(&regs->tdl_from_trb, mask);
+			cdns3_set_register_bit(&regs->tdl_beh, mask);
+			cdns3_set_register_bit(&regs->tdl_beh2, mask);
+			cdns3_set_register_bit(&regs->dma_adv_td, mask);
+		}
+
+		if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
+			cdns3_set_register_bit(&regs->tdl_from_trb, mask);
+
+		cdns3_set_register_bit(&regs->dtrans, mask);
+	}
+}
+
+/**
+ * cdns3_ep_config Configure hardware endpoint
+ * @priv_ep: extended endpoint object
+ */
+void cdns3_ep_config(struct cdns3_endpoint *priv_ep)
+{
+	bool is_iso_ep = (priv_ep->type == USB_ENDPOINT_XFER_ISOC);
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+	u32 bEndpointAddress = priv_ep->num | priv_ep->dir;
+	u32 max_packet_size = 0;
+	u8 maxburst = 0;
+	u32 ep_cfg = 0;
+	u8 buffering;
+	u8 mult = 0;
+	int ret;
+
+	buffering = CDNS3_EP_BUF_SIZE - 1;
+
+	cdns3_configure_dmult(priv_dev, priv_ep);
+
+	switch (priv_ep->type) {
+	case USB_ENDPOINT_XFER_INT:
+		ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_INT);
+
+		if ((priv_dev->dev_ver == DEV_VER_V2 && !priv_ep->dir) ||
+		    priv_dev->dev_ver > DEV_VER_V2)
+			ep_cfg |= EP_CFG_TDL_CHK;
+		break;
+	case USB_ENDPOINT_XFER_BULK:
+		ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_BULK);
+
+		if ((priv_dev->dev_ver == DEV_VER_V2  && !priv_ep->dir) ||
+		    priv_dev->dev_ver > DEV_VER_V2)
+			ep_cfg |= EP_CFG_TDL_CHK;
+		break;
+	default:
+		ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_ISOC);
+		mult = CDNS3_EP_ISO_HS_MULT - 1;
+		buffering = mult + 1;
+	}
+
+	switch (priv_dev->gadget.speed) {
+	case USB_SPEED_FULL:
+		max_packet_size = is_iso_ep ? 1023 : 64;
+		break;
+	case USB_SPEED_HIGH:
+		max_packet_size = is_iso_ep ? 1024 : 512;
+		break;
+	case USB_SPEED_SUPER:
+		/* It's limitation that driver assumes in driver. */
+		mult = 0;
+		max_packet_size = 1024;
+		if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
+			maxburst = CDNS3_EP_ISO_SS_BURST - 1;
+			buffering = (mult + 1) *
+				    (maxburst + 1);
+
+			if (priv_ep->interval > 1)
+				buffering++;
+		} else {
+			maxburst = CDNS3_EP_BUF_SIZE - 1;
+		}
+		break;
+	default:
+		/* all other speed are not supported */
+		return;
+	}
+
+	if (max_packet_size == 1024)
+		priv_ep->trb_burst_size = 128;
+	else if (max_packet_size >= 512)
+		priv_ep->trb_burst_size = 64;
+	else
+		priv_ep->trb_burst_size = 16;
+
+	ret = cdns3_ep_onchip_buffer_reserve(priv_dev, buffering + 1,
+					     !!priv_ep->dir);
+	if (ret) {
+		dev_err(priv_dev->dev, "onchip mem is full, ep is invalid\n");
+		return;
+	}
+
+	ep_cfg |= EP_CFG_MAXPKTSIZE(max_packet_size) |
+		  EP_CFG_MULT(mult) |
+		  EP_CFG_BUFFERING(buffering) |
+		  EP_CFG_MAXBURST(maxburst);
+
+	cdns3_select_ep(priv_dev, bEndpointAddress);
+	writel(ep_cfg, &priv_dev->regs->ep_cfg);
+
+	dev_dbg(priv_dev->dev, "Configure %s: with val %08x\n",
+		priv_ep->name, ep_cfg);
+}
+
+/* Find correct direction for HW endpoint according to description */
+static int cdns3_ep_dir_is_correct(struct usb_endpoint_descriptor *desc,
+				   struct cdns3_endpoint *priv_ep)
+{
+	return (priv_ep->endpoint.caps.dir_in && usb_endpoint_dir_in(desc)) ||
+	       (priv_ep->endpoint.caps.dir_out && usb_endpoint_dir_out(desc));
+}
+
+static struct
+cdns3_endpoint *cdns3_find_available_ep(struct cdns3_device *priv_dev,
+					struct usb_endpoint_descriptor *desc)
+{
+	struct usb_ep *ep;
+	struct cdns3_endpoint *priv_ep;
+
+	list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
+		unsigned long num;
+		int ret;
+		/* ep name pattern likes epXin or epXout */
+		char c[2] = {ep->name[2], '\0'};
+
+		ret = kstrtoul(c, 10, &num);
+		if (ret)
+			return ERR_PTR(ret);
+
+		priv_ep = ep_to_cdns3_ep(ep);
+		if (cdns3_ep_dir_is_correct(desc, priv_ep)) {
+			if (!(priv_ep->flags & EP_CLAIMED)) {
+				priv_ep->num  = num;
+				return priv_ep;
+			}
+		}
+	}
+
+	return ERR_PTR(-ENOENT);
+}
+
+/*
+ *  Cadence IP has one limitation that all endpoints must be configured
+ * (Type & MaxPacketSize) before setting configuration through hardware
+ * register, it means we can't change endpoints configuration after
+ * set_configuration.
+ *
+ * This function set EP_CLAIMED flag which is added when the gadget driver
+ * uses usb_ep_autoconfig to configure specific endpoint;
+ * When the udc driver receives set_configurion request,
+ * it goes through all claimed endpoints, and configure all endpoints
+ * accordingly.
+ *
+ * At usb_ep_ops.enable/disable, we only enable and disable endpoint through
+ * ep_cfg register which can be changed after set_configuration, and do
+ * some software operation accordingly.
+ */
+static struct
+usb_ep *cdns3_gadget_match_ep(struct usb_gadget *gadget,
+			      struct usb_endpoint_descriptor *desc,
+			      struct usb_ss_ep_comp_descriptor *comp_desc)
+{
+	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
+	struct cdns3_endpoint *priv_ep;
+	unsigned long flags;
+
+	priv_ep = cdns3_find_available_ep(priv_dev, desc);
+	if (IS_ERR(priv_ep)) {
+		dev_err(priv_dev->dev, "no available ep\n");
+		return NULL;
+	}
+
+	dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name);
+
+	spin_lock_irqsave(&priv_dev->lock, flags);
+	priv_ep->endpoint.desc = desc;
+	priv_ep->dir  = usb_endpoint_dir_in(desc) ? USB_DIR_IN : USB_DIR_OUT;
+	priv_ep->type = usb_endpoint_type(desc);
+	priv_ep->flags |= EP_CLAIMED;
+	priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
+
+	spin_unlock_irqrestore(&priv_dev->lock, flags);
+	return &priv_ep->endpoint;
+}
+
+/**
+ * cdns3_gadget_ep_alloc_request Allocates request
+ * @ep: endpoint object associated with request
+ * @gfp_flags: gfp flags
+ *
+ * Returns allocated request address, NULL on allocation error
+ */
+struct usb_request *cdns3_gadget_ep_alloc_request(struct usb_ep *ep,
+						  gfp_t gfp_flags)
+{
+	struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
+	struct cdns3_request *priv_req;
+
+	priv_req = kzalloc(sizeof(*priv_req), gfp_flags);
+	if (!priv_req)
+		return NULL;
+
+	priv_req->priv_ep = priv_ep;
+
+	trace_cdns3_alloc_request(priv_req);
+	return &priv_req->request;
+}
+
+/**
+ * cdns3_gadget_ep_free_request Free memory occupied by request
+ * @ep: endpoint object associated with request
+ * @request: request to free memory
+ */
+void cdns3_gadget_ep_free_request(struct usb_ep *ep,
+				  struct usb_request *request)
+{
+	struct cdns3_request *priv_req = to_cdns3_request(request);
+
+	if (priv_req->aligned_buf)
+		priv_req->aligned_buf->in_use = 0;
+
+	trace_cdns3_free_request(priv_req);
+	kfree(priv_req);
+}
+
+/**
+ * cdns3_gadget_ep_enable Enable endpoint
+ * @ep: endpoint object
+ * @desc: endpoint descriptor
+ *
+ * Returns 0 on success, error code elsewhere
+ */
+static int cdns3_gadget_ep_enable(struct usb_ep *ep,
+				  const struct usb_endpoint_descriptor *desc)
+{
+	struct cdns3_endpoint *priv_ep;
+	struct cdns3_device *priv_dev;
+	u32 reg = EP_STS_EN_TRBERREN;
+	u32 bEndpointAddress;
+	unsigned long flags;
+	int enable = 1;
+	int ret;
+	int val;
+
+	priv_ep = ep_to_cdns3_ep(ep);
+	priv_dev = priv_ep->cdns3_dev;
+
+	if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
+		dev_dbg(priv_dev->dev, "usbss: invalid parameters\n");
+		return -EINVAL;
+	}
+
+	if (!desc->wMaxPacketSize) {
+		dev_err(priv_dev->dev, "usbss: missing wMaxPacketSize\n");
+		return -EINVAL;
+	}
+
+	if (dev_WARN_ONCE(priv_dev->dev, priv_ep->flags & EP_ENABLED,
+			  "%s is already enabled\n", priv_ep->name))
+		return 0;
+
+	spin_lock_irqsave(&priv_dev->lock, flags);
+
+	priv_ep->endpoint.desc = desc;
+	priv_ep->type = usb_endpoint_type(desc);
+	priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
+
+	if (priv_ep->interval > ISO_MAX_INTERVAL &&
+	    priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
+		dev_err(priv_dev->dev, "Driver is limited to %d period\n",
+			ISO_MAX_INTERVAL);
+
+		ret =  -EINVAL;
+		goto exit;
+	}
+
+	ret = cdns3_allocate_trb_pool(priv_ep);
+
+	if (ret)
+		goto exit;
+
+	bEndpointAddress = priv_ep->num | priv_ep->dir;
+	cdns3_select_ep(priv_dev, bEndpointAddress);
+
+	trace_cdns3_gadget_ep_enable(priv_ep);
+
+	writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
+
+	ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
+					!(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
+					1, 1000);
+
+	if (unlikely(ret)) {
+		cdns3_free_trb_pool(priv_ep);
+		ret =  -EINVAL;
+		goto exit;
+	}
+
+	/* enable interrupt for selected endpoint */
+	cdns3_set_register_bit(&priv_dev->regs->ep_ien,
+			       BIT(cdns3_ep_addr_to_index(bEndpointAddress)));
+
+	if (priv_dev->dev_ver < DEV_VER_V2)
+		cdns3_wa2_enable_detection(priv_dev, priv_ep, reg);
+
+	writel(reg, &priv_dev->regs->ep_sts_en);
+
+	/*
+	 * For some versions of controller at some point during ISO OUT traffic
+	 * DMA reads Transfer Ring for the EP which has never got doorbell.
+	 * This issue was detected only on simulation, but to avoid this issue
+	 * driver add protection against it. To fix it driver enable ISO OUT
+	 * endpoint before setting DRBL. This special treatment of ISO OUT
+	 * endpoints are recommended by controller specification.
+	 */
+	if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir)
+		enable = 0;
+
+	if (enable)
+		cdns3_set_register_bit(&priv_dev->regs->ep_cfg, EP_CFG_ENABLE);
+
+	ep->desc = desc;
+	priv_ep->flags &= ~(EP_PENDING_REQUEST | EP_STALLED | EP_STALL_PENDING |
+			    EP_QUIRK_ISO_OUT_EN | EP_QUIRK_EXTRA_BUF_EN);
+	priv_ep->flags |= EP_ENABLED | EP_UPDATE_EP_TRBADDR;
+	priv_ep->wa1_set = 0;
+	priv_ep->enqueue = 0;
+	priv_ep->dequeue = 0;
+	reg = readl(&priv_dev->regs->ep_sts);
+	priv_ep->pcs = !!EP_STS_CCS(reg);
+	priv_ep->ccs = !!EP_STS_CCS(reg);
+	/* one TRB is reserved for link TRB used in DMULT mode*/
+	priv_ep->free_trbs = priv_ep->num_trbs - 1;
+exit:
+	spin_unlock_irqrestore(&priv_dev->lock, flags);
+
+	return ret;
+}
+
+/**
+ * cdns3_gadget_ep_disable Disable endpoint
+ * @ep: endpoint object
+ *
+ * Returns 0 on success, error code elsewhere
+ */
+static int cdns3_gadget_ep_disable(struct usb_ep *ep)
+{
+	struct cdns3_endpoint *priv_ep;
+	struct cdns3_request *priv_req;
+	struct cdns3_device *priv_dev;
+	struct usb_request *request;
+	unsigned long flags;
+	int ret = 0;
+	u32 ep_cfg;
+	int val;
+
+	if (!ep) {
+		pr_err("usbss: invalid parameters\n");
+		return -EINVAL;
+	}
+
+	priv_ep = ep_to_cdns3_ep(ep);
+	priv_dev = priv_ep->cdns3_dev;
+
+	if (dev_WARN_ONCE(priv_dev->dev, !(priv_ep->flags & EP_ENABLED),
+			  "%s is already disabled\n", priv_ep->name))
+		return 0;
+
+	spin_lock_irqsave(&priv_dev->lock, flags);
+
+	trace_cdns3_gadget_ep_disable(priv_ep);
+
+	cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
+
+	ep_cfg = readl(&priv_dev->regs->ep_cfg);
+	ep_cfg &= ~EP_CFG_ENABLE;
+	writel(ep_cfg, &priv_dev->regs->ep_cfg);
+
+	/**
+	 * Driver needs some time before resetting endpoint.
+	 * It need waits for clearing DBUSY bit or for timeout expired.
+	 * 10us is enough time for controller to stop transfer.
+	 */
+	readl_poll_timeout_atomic(&priv_dev->regs->ep_sts, val,
+				  !(val & EP_STS_DBUSY), 1, 10);
+	writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
+
+	readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
+				  !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
+				  1, 1000);
+	if (unlikely(ret))
+		dev_err(priv_dev->dev, "Timeout: %s resetting failed.\n",
+			priv_ep->name);
+
+	while (!list_empty(&priv_ep->pending_req_list)) {
+		request = cdns3_next_request(&priv_ep->pending_req_list);
+
+		cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
+				      -ESHUTDOWN);
+	}
+
+	while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
+		priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
+
+		kfree(priv_req->request.buf);
+		cdns3_gadget_ep_free_request(&priv_ep->endpoint,
+					     &priv_req->request);
+		list_del_init(&priv_req->list);
+		--priv_ep->wa2_counter;
+	}
+
+	while (!list_empty(&priv_ep->deferred_req_list)) {
+		request = cdns3_next_request(&priv_ep->deferred_req_list);
+
+		cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
+				      -ESHUTDOWN);
+	}
+
+	priv_ep->descmis_req = NULL;
+
+	ep->desc = NULL;
+	priv_ep->flags &= ~EP_ENABLED;
+
+	spin_unlock_irqrestore(&priv_dev->lock, flags);
+
+	return ret;
+}
+
+/**
+ * cdns3_gadget_ep_queue Transfer data on endpoint
+ * @ep: endpoint object
+ * @request: request object
+ * @gfp_flags: gfp flags
+ *
+ * Returns 0 on success, error code elsewhere
+ */
+static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
+				   struct usb_request *request,
+				   gfp_t gfp_flags)
+{
+	struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+	struct cdns3_request *priv_req;
+	int ret = 0;
+
+	request->actual = 0;
+	request->status = -EINPROGRESS;
+	priv_req = to_cdns3_request(request);
+	trace_cdns3_ep_queue(priv_req);
+
+	if (priv_dev->dev_ver < DEV_VER_V2) {
+		ret = cdns3_wa2_gadget_ep_queue(priv_dev, priv_ep,
+						priv_req);
+
+		if (ret == EINPROGRESS)
+			return 0;
+	}
+
+	ret = cdns3_prepare_aligned_request_buf(priv_req);
+	if (ret < 0)
+		return ret;
+
+	ret = usb_gadget_map_request_by_dev(priv_dev->sysdev, request,
+					    usb_endpoint_dir_in(ep->desc));
+	if (ret)
+		return ret;
+
+	list_add_tail(&request->list, &priv_ep->deferred_req_list);
+
+	/*
+	 * If hardware endpoint configuration has not been set yet then
+	 * just queue request in deferred list. Transfer will be started in
+	 * cdns3_set_hw_configuration.
+	 */
+	if (priv_dev->hw_configured_flag && !(priv_ep->flags & EP_STALLED) &&
+	    !(priv_ep->flags & EP_STALL_PENDING))
+		cdns3_start_all_request(priv_dev, priv_ep);
+
+	return 0;
+}
+
+static int cdns3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
+				 gfp_t gfp_flags)
+{
+	struct usb_request *zlp_request;
+	struct cdns3_endpoint *priv_ep;
+	struct cdns3_device *priv_dev;
+	unsigned long flags;
+	int ret;
+
+	if (!request || !ep)
+		return -EINVAL;
+
+	priv_ep = ep_to_cdns3_ep(ep);
+	priv_dev = priv_ep->cdns3_dev;
+
+	spin_lock_irqsave(&priv_dev->lock, flags);
+
+	ret = __cdns3_gadget_ep_queue(ep, request, gfp_flags);
+
+	if (ret == 0 && request->zero && request->length &&
+	    (request->length % ep->maxpacket == 0)) {
+		struct cdns3_request *priv_req;
+
+		zlp_request = cdns3_gadget_ep_alloc_request(ep, GFP_ATOMIC);
+		zlp_request->buf = priv_dev->zlp_buf;
+		zlp_request->length = 0;
+
+		priv_req = to_cdns3_request(zlp_request);
+		priv_req->flags |= REQUEST_ZLP;
+
+		dev_dbg(priv_dev->dev, "Queuing ZLP for endpoint: %s\n",
+			priv_ep->name);
+		ret = __cdns3_gadget_ep_queue(ep, zlp_request, gfp_flags);
+	}
+
+	spin_unlock_irqrestore(&priv_dev->lock, flags);
+	return ret;
+}
+
+/**
+ * cdns3_gadget_ep_dequeue Remove request from transfer queue
+ * @ep: endpoint object associated with request
+ * @request: request object
+ *
+ * Returns 0 on success, error code elsewhere
+ */
+int cdns3_gadget_ep_dequeue(struct usb_ep *ep,
+			    struct usb_request *request)
+{
+	struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+	struct usb_request *req, *req_temp;
+	struct cdns3_request *priv_req;
+	struct cdns3_trb *link_trb;
+	u8 req_on_hw_ring = 0;
+	unsigned long flags;
+	int ret = 0;
+
+	if (!ep || !request || !ep->desc)
+		return -EINVAL;
+
+	spin_lock_irqsave(&priv_dev->lock, flags);
+
+	priv_req = to_cdns3_request(request);
+
+	trace_cdns3_ep_dequeue(priv_req);
+
+	cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
+
+	list_for_each_entry_safe(req, req_temp, &priv_ep->pending_req_list,
+				 list) {
+		if (request == req) {
+			req_on_hw_ring = 1;
+			goto found;
+		}
+	}
+
+	list_for_each_entry_safe(req, req_temp, &priv_ep->deferred_req_list,
+				 list) {
+		if (request == req)
+			goto found;
+	}
+
+	goto not_found;
+
+found:
+	link_trb = priv_req->trb;
+
+	/* Update ring only if removed request is on pending_req_list list */
+	if (req_on_hw_ring) {
+		link_trb->buffer = TRB_BUFFER(priv_ep->trb_pool_dma +
+					      (priv_req->start_trb * TRB_SIZE));
+		link_trb->control = (link_trb->control & TRB_CYCLE) |
+				    TRB_TYPE(TRB_LINK) | TRB_CHAIN;
+
+		if (priv_ep->wa1_trb == priv_req->trb)
+			cdns3_wa1_restore_cycle_bit(priv_ep);
+	}
+
+	cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET);
+
+not_found:
+	spin_unlock_irqrestore(&priv_dev->lock, flags);
+	return ret;
+}
+
+/**
+ * __cdns3_gadget_ep_set_halt Sets stall on selected endpoint
+ * Should be called after acquiring spin_lock and selecting ep
+ * @ep: endpoint object to set stall on.
+ */
+void __cdns3_gadget_ep_set_halt(struct cdns3_endpoint *priv_ep)
+{
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+
+	trace_cdns3_halt(priv_ep, 1, 0);
+
+	if (!(priv_ep->flags & EP_STALLED)) {
+		u32 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
+
+		if (!(ep_sts_reg & EP_STS_DBUSY))
+			cdns3_ep_stall_flush(priv_ep);
+		else
+			priv_ep->flags |= EP_STALL_PENDING;
+	}
+}
+
+/**
+ * __cdns3_gadget_ep_clear_halt Clears stall on selected endpoint
+ * Should be called after acquiring spin_lock and selecting ep
+ * @ep: endpoint object to clear stall on
+ */
+int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep)
+{
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+	struct usb_request *request;
+	int ret;
+	int val;
+
+	trace_cdns3_halt(priv_ep, 0, 0);
+
+	writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
+
+	/* wait for EPRST cleared */
+	ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
+					!(val & EP_CMD_EPRST), 1, 100);
+	if (ret)
+		return -EINVAL;
+
+	priv_ep->flags &= ~(EP_STALLED | EP_STALL_PENDING);
+
+	request = cdns3_next_request(&priv_ep->pending_req_list);
+
+	if (request)
+		cdns3_rearm_transfer(priv_ep, 1);
+
+	cdns3_start_all_request(priv_dev, priv_ep);
+	return ret;
+}
+
+/**
+ * cdns3_gadget_ep_set_halt Sets/clears stall on selected endpoint
+ * @ep: endpoint object to set/clear stall on
+ * @value: 1 for set stall, 0 for clear stall
+ *
+ * Returns 0 on success, error code elsewhere
+ */
+int cdns3_gadget_ep_set_halt(struct usb_ep *ep, int value)
+{
+	struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+	unsigned long flags;
+	int ret = 0;
+
+	if (!(priv_ep->flags & EP_ENABLED))
+		return -EPERM;
+
+	spin_lock_irqsave(&priv_dev->lock, flags);
+
+	cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
+
+	if (!value) {
+		priv_ep->flags &= ~EP_WEDGE;
+		ret = __cdns3_gadget_ep_clear_halt(priv_ep);
+	} else {
+		__cdns3_gadget_ep_set_halt(priv_ep);
+	}
+
+	spin_unlock_irqrestore(&priv_dev->lock, flags);
+
+	return ret;
+}
+
+extern const struct usb_ep_ops cdns3_gadget_ep0_ops;
+
+static const struct usb_ep_ops cdns3_gadget_ep_ops = {
+	.enable = cdns3_gadget_ep_enable,
+	.disable = cdns3_gadget_ep_disable,
+	.alloc_request = cdns3_gadget_ep_alloc_request,
+	.free_request = cdns3_gadget_ep_free_request,
+	.queue = cdns3_gadget_ep_queue,
+	.dequeue = cdns3_gadget_ep_dequeue,
+	.set_halt = cdns3_gadget_ep_set_halt,
+	.set_wedge = cdns3_gadget_ep_set_wedge,
+};
+
+/**
+ * cdns3_gadget_get_frame Returns number of actual ITP frame
+ * @gadget: gadget object
+ *
+ * Returns number of actual ITP frame
+ */
+static int cdns3_gadget_get_frame(struct usb_gadget *gadget)
+{
+	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
+
+	return readl(&priv_dev->regs->usb_itpn);
+}
+
+int __cdns3_gadget_wakeup(struct cdns3_device *priv_dev)
+{
+	enum usb_device_speed speed;
+
+	speed = cdns3_get_speed(priv_dev);
+
+	if (speed >= USB_SPEED_SUPER)
+		return 0;
+
+	/* Start driving resume signaling to indicate remote wakeup. */
+	writel(USB_CONF_LGO_L0, &priv_dev->regs->usb_conf);
+
+	return 0;
+}
+
+static int cdns3_gadget_wakeup(struct usb_gadget *gadget)
+{
+	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
+	unsigned long flags;
+	int ret = 0;
+
+	spin_lock_irqsave(&priv_dev->lock, flags);
+	ret = __cdns3_gadget_wakeup(priv_dev);
+	spin_unlock_irqrestore(&priv_dev->lock, flags);
+	return ret;
+}
+
+static int cdns3_gadget_set_selfpowered(struct usb_gadget *gadget,
+					int is_selfpowered)
+{
+	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
+	unsigned long flags;
+
+	spin_lock_irqsave(&priv_dev->lock, flags);
+	priv_dev->is_selfpowered = !!is_selfpowered;
+	spin_unlock_irqrestore(&priv_dev->lock, flags);
+	return 0;
+}
+
+static int cdns3_gadget_pullup(struct usb_gadget *gadget, int is_on)
+{
+	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
+
+	if (is_on)
+		writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);
+	else
+		writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
+
+	return 0;
+}
+
+static void cdns3_gadget_config(struct cdns3_device *priv_dev)
+{
+	struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
+	u32 reg;
+
+	cdns3_ep0_config(priv_dev);
+
+	/* enable interrupts for endpoint 0 (in and out) */
+	writel(EP_IEN_EP_OUT0 | EP_IEN_EP_IN0, &regs->ep_ien);
+
+	/*
+	 * Driver needs to modify LFPS minimal U1 Exit time for DEV_VER_TI_V1
+	 * revision of controller.
+	 */
+	if (priv_dev->dev_ver == DEV_VER_TI_V1) {
+		reg = readl(&regs->dbg_link1);
+
+		reg &= ~DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_MASK;
+		reg |= DBG_LINK1_LFPS_MIN_GEN_U1_EXIT(0x55) |
+		       DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_SET;
+		writel(reg, &regs->dbg_link1);
+	}
+
+	/*
+	 * By default some platforms has set protected access to memory.
+	 * This cause problem with cache, so driver restore non-secure
+	 * access to memory.
+	 */
+	reg = readl(&regs->dma_axi_ctrl);
+	reg |= DMA_AXI_CTRL_MARPROT(DMA_AXI_CTRL_NON_SECURE) |
+	       DMA_AXI_CTRL_MAWPROT(DMA_AXI_CTRL_NON_SECURE);
+	writel(reg, &regs->dma_axi_ctrl);
+
+	/* enable generic interrupt*/
+	writel(USB_IEN_INIT, &regs->usb_ien);
+	writel(USB_CONF_CLK2OFFDS | USB_CONF_L1DS, &regs->usb_conf);
+
+	cdns3_configure_dmult(priv_dev, NULL);
+}
+
+/**
+ * cdns3_gadget_udc_start Gadget start
+ * @gadget: gadget object
+ * @driver: driver which operates on this gadget
+ *
+ * Returns 0 on success, error code elsewhere
+ */
+static int cdns3_gadget_udc_start(struct usb_gadget *gadget,
+				  struct usb_gadget_driver *driver)
+{
+	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
+	unsigned long flags;
+	enum usb_device_speed max_speed = driver->max_speed;
+
+	spin_lock_irqsave(&priv_dev->lock, flags);
+	priv_dev->gadget_driver = driver;
+
+	/* limit speed if necessary */
+	max_speed = min(driver->max_speed, gadget->max_speed);
+
+	switch (max_speed) {
+	case USB_SPEED_FULL:
+		writel(USB_CONF_SFORCE_FS, &priv_dev->regs->usb_conf);
+		writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
+		break;
+	case USB_SPEED_HIGH:
+		writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
+		break;
+	case USB_SPEED_SUPER:
+		break;
+	default:
+		dev_err(priv_dev->dev,
+			"invalid maximum_speed parameter %d\n",
+			max_speed);
+		/* fall through */
+	case USB_SPEED_UNKNOWN:
+		/* default to superspeed */
+		max_speed = USB_SPEED_SUPER;
+		break;
+	}
+
+	cdns3_gadget_config(priv_dev);
+	spin_unlock_irqrestore(&priv_dev->lock, flags);
+	return 0;
+}
+
+/**
+ * cdns3_gadget_udc_stop Stops gadget
+ * @gadget: gadget object
+ *
+ * Returns 0
+ */
+static int cdns3_gadget_udc_stop(struct usb_gadget *gadget)
+{
+	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
+	struct cdns3_endpoint *priv_ep;
+	u32 bEndpointAddress;
+	struct usb_ep *ep;
+	int ret = 0;
+	int val;
+
+	priv_dev->gadget_driver = NULL;
+
+	priv_dev->onchip_used_size = 0;
+	priv_dev->out_mem_is_allocated = 0;
+	priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
+
+	list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
+		priv_ep = ep_to_cdns3_ep(ep);
+		bEndpointAddress = priv_ep->num | priv_ep->dir;
+		cdns3_select_ep(priv_dev, bEndpointAddress);
+		writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
+		readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
+					  !(val & EP_CMD_EPRST), 1, 100);
+
+		priv_ep->flags &= ~EP_CLAIMED;
+	}
+
+	/* disable interrupt for device */
+	writel(0, &priv_dev->regs->usb_ien);
+	writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
+
+	return ret;
+}
+
+static const struct usb_gadget_ops cdns3_gadget_ops = {
+	.get_frame = cdns3_gadget_get_frame,
+	.wakeup = cdns3_gadget_wakeup,
+	.set_selfpowered = cdns3_gadget_set_selfpowered,
+	.pullup = cdns3_gadget_pullup,
+	.udc_start = cdns3_gadget_udc_start,
+	.udc_stop = cdns3_gadget_udc_stop,
+	.match_ep = cdns3_gadget_match_ep,
+};
+
+static void cdns3_free_all_eps(struct cdns3_device *priv_dev)
+{
+	int i;
+
+	/* ep0 OUT point to ep0 IN. */
+	priv_dev->eps[16] = NULL;
+
+	for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++)
+		if (priv_dev->eps[i]) {
+			cdns3_free_trb_pool(priv_dev->eps[i]);
+			devm_kfree(priv_dev->dev, priv_dev->eps[i]);
+		}
+}
+
+/**
+ * cdns3_init_eps Initializes software endpoints of gadget
+ * @cdns3: extended gadget object
+ *
+ * Returns 0 on success, error code elsewhere
+ */
+static int cdns3_init_eps(struct cdns3_device *priv_dev)
+{
+	u32 ep_enabled_reg, iso_ep_reg;
+	struct cdns3_endpoint *priv_ep;
+	int ep_dir, ep_number;
+	u32 ep_mask;
+	int ret = 0;
+	int i;
+
+	/* Read it from USB_CAP3 to USB_CAP5 */
+	ep_enabled_reg = readl(&priv_dev->regs->usb_cap3);
+	iso_ep_reg = readl(&priv_dev->regs->usb_cap4);
+
+	dev_dbg(priv_dev->dev, "Initializing non-zero endpoints\n");
+
+	for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) {
+		ep_dir = i >> 4;	/* i div 16 */
+		ep_number = i & 0xF;	/* i % 16 */
+		ep_mask = BIT(i);
+
+		if (!(ep_enabled_reg & ep_mask))
+			continue;
+
+		if (ep_dir && !ep_number) {
+			priv_dev->eps[i] = priv_dev->eps[0];
+			continue;
+		}
+
+		priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep),
+				       GFP_KERNEL);
+		if (!priv_ep)
+			goto err;
+
+		/* set parent of endpoint object */
+		priv_ep->cdns3_dev = priv_dev;
+		priv_dev->eps[i] = priv_ep;
+		priv_ep->num = ep_number;
+		priv_ep->dir = ep_dir ? USB_DIR_IN : USB_DIR_OUT;
+
+		if (!ep_number) {
+			ret = cdns3_init_ep0(priv_dev, priv_ep);
+			if (ret) {
+				dev_err(priv_dev->dev, "Failed to init ep0\n");
+				goto err;
+			}
+		} else {
+			snprintf(priv_ep->name, sizeof(priv_ep->name), "ep%d%s",
+				 ep_number, !!ep_dir ? "in" : "out");
+			priv_ep->endpoint.name = priv_ep->name;
+
+			usb_ep_set_maxpacket_limit(&priv_ep->endpoint,
+						   CDNS3_EP_MAX_PACKET_LIMIT);
+			priv_ep->endpoint.max_streams = CDNS3_EP_MAX_STREAMS;
+			priv_ep->endpoint.ops = &cdns3_gadget_ep_ops;
+			if (ep_dir)
+				priv_ep->endpoint.caps.dir_in = 1;
+			else
+				priv_ep->endpoint.caps.dir_out = 1;
+
+			if (iso_ep_reg & ep_mask)
+				priv_ep->endpoint.caps.type_iso = 1;
+
+			priv_ep->endpoint.caps.type_bulk = 1;
+			priv_ep->endpoint.caps.type_int = 1;
+
+			list_add_tail(&priv_ep->endpoint.ep_list,
+				      &priv_dev->gadget.ep_list);
+		}
+
+		priv_ep->flags = 0;
+
+		dev_info(priv_dev->dev, "Initialized  %s support: %s %s\n",
+			 priv_ep->name,
+			 priv_ep->endpoint.caps.type_bulk ? "BULK, INT" : "",
+			 priv_ep->endpoint.caps.type_iso ? "ISO" : "");
+
+		INIT_LIST_HEAD(&priv_ep->pending_req_list);
+		INIT_LIST_HEAD(&priv_ep->deferred_req_list);
+		INIT_LIST_HEAD(&priv_ep->wa2_descmiss_req_list);
+	}
+
+	return 0;
+err:
+	cdns3_free_all_eps(priv_dev);
+	return -ENOMEM;
+}
+
+void cdns3_gadget_exit(struct cdns3 *cdns)
+{
+	struct cdns3_device *priv_dev;
+
+	priv_dev = cdns->gadget_dev;
+
+	devm_free_irq(cdns->dev, cdns->dev_irq, cdns);
+
+	pm_runtime_mark_last_busy(cdns->dev);
+	pm_runtime_put_autosuspend(cdns->dev);
+
+	usb_del_gadget_udc(&priv_dev->gadget);
+
+	cdns3_free_all_eps(priv_dev);
+
+	while (!list_empty(&priv_dev->aligned_buf_list)) {
+		struct cdns3_aligned_buf *buf;
+
+		buf = cdns3_next_align_buf(&priv_dev->aligned_buf_list);
+		dma_free_coherent(priv_dev->sysdev, buf->size,
+				  buf->buf,
+				  buf->dma);
+
+		list_del(&buf->list);
+		kfree(buf);
+	}
+
+	dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
+			  priv_dev->setup_dma);
+
+	kfree(priv_dev->zlp_buf);
+	kfree(priv_dev);
+	cdns->gadget_dev = NULL;
+	cdns3_drd_switch_gadget(cdns, 0);
+}
+
+static int cdns3_gadget_start(struct cdns3 *cdns)
+{
+	struct cdns3_device *priv_dev;
+	u32 max_speed;
+	int ret;
+
+	priv_dev = kzalloc(sizeof(*priv_dev), GFP_KERNEL);
+	if (!priv_dev)
+		return -ENOMEM;
+
+	cdns->gadget_dev = priv_dev;
+	priv_dev->sysdev = cdns->dev;
+	priv_dev->dev = cdns->dev;
+	priv_dev->regs = cdns->dev_regs;
+
+	device_property_read_u16(priv_dev->dev, "cdns,on-chip-buff-size",
+				 &priv_dev->onchip_buffers);
+
+	if (priv_dev->onchip_buffers <=  0) {
+		u32 reg = readl(&priv_dev->regs->usb_cap2);
+
+		priv_dev->onchip_buffers = USB_CAP2_ACTUAL_MEM_SIZE(reg);
+	}
+
+	if (!priv_dev->onchip_buffers)
+		priv_dev->onchip_buffers = 256;
+
+	max_speed = usb_get_maximum_speed(cdns->dev);
+
+	/* Check the maximum_speed parameter */
+	switch (max_speed) {
+	case USB_SPEED_FULL:
+	case USB_SPEED_HIGH:
+	case USB_SPEED_SUPER:
+		break;
+	default:
+		dev_err(cdns->dev, "invalid maximum_speed parameter %d\n",
+			max_speed);
+		/* fall through */
+	case USB_SPEED_UNKNOWN:
+		/* default to superspeed */
+		max_speed = USB_SPEED_SUPER;
+		break;
+	}
+
+	/* fill gadget fields */
+	priv_dev->gadget.max_speed = max_speed;
+	priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
+	priv_dev->gadget.ops = &cdns3_gadget_ops;
+	priv_dev->gadget.name = "usb-ss-gadget";
+	priv_dev->gadget.sg_supported = 1;
+	priv_dev->gadget.quirk_avoids_skb_reserve = 1;
+
+	spin_lock_init(&priv_dev->lock);
+	INIT_WORK(&priv_dev->pending_status_wq,
+		  cdns3_pending_setup_status_handler);
+
+	INIT_WORK(&priv_dev->aligned_buf_wq,
+		  cdns3_free_aligned_request_buf);
+
+	/* initialize endpoint container */
+	INIT_LIST_HEAD(&priv_dev->gadget.ep_list);
+	INIT_LIST_HEAD(&priv_dev->aligned_buf_list);
+
+	ret = cdns3_init_eps(priv_dev);
+	if (ret) {
+		dev_err(priv_dev->dev, "Failed to create endpoints\n");
+		goto err1;
+	}
+
+	/* allocate memory for setup packet buffer */
+	priv_dev->setup_buf = dma_alloc_coherent(priv_dev->sysdev, 8,
+						 &priv_dev->setup_dma, GFP_DMA);
+	if (!priv_dev->setup_buf) {
+		ret = -ENOMEM;
+		goto err2;
+	}
+
+	priv_dev->dev_ver = readl(&priv_dev->regs->usb_cap6);
+
+	dev_dbg(priv_dev->dev, "Device Controller version: %08x\n",
+		readl(&priv_dev->regs->usb_cap6));
+	dev_dbg(priv_dev->dev, "USB Capabilities:: %08x\n",
+		readl(&priv_dev->regs->usb_cap1));
+	dev_dbg(priv_dev->dev, "On-Chip memory configuration: %08x\n",
+		readl(&priv_dev->regs->usb_cap2));
+
+	priv_dev->dev_ver = GET_DEV_BASE_VERSION(priv_dev->dev_ver);
+
+	priv_dev->zlp_buf = kzalloc(CDNS3_EP_ZLP_BUF_SIZE, GFP_KERNEL);
+	if (!priv_dev->zlp_buf) {
+		ret = -ENOMEM;
+		goto err3;
+	}
+
+	/* add USB gadget device */
+	ret = usb_add_gadget_udc(priv_dev->dev, &priv_dev->gadget);
+	if (ret < 0) {
+		dev_err(priv_dev->dev,
+			"Failed to register USB device controller\n");
+		goto err4;
+	}
+
+	return 0;
+err4:
+	kfree(priv_dev->zlp_buf);
+err3:
+	dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
+			  priv_dev->setup_dma);
+err2:
+	cdns3_free_all_eps(priv_dev);
+err1:
+	cdns->gadget_dev = NULL;
+	return ret;
+}
+
+static int __cdns3_gadget_init(struct cdns3 *cdns)
+{
+	int ret = 0;
+
+	/* Restore 32-bit DMA Mask in case we switched from Host mode */
+	ret = dma_set_mask_and_coherent(cdns->dev, DMA_BIT_MASK(32));
+	if (ret) {
+		dev_err(cdns->dev, "Failed to set dma mask: %d\n", ret);
+		return ret;
+	}
+
+	cdns3_drd_switch_gadget(cdns, 1);
+	pm_runtime_get_sync(cdns->dev);
+
+	ret = cdns3_gadget_start(cdns);
+	if (ret)
+		return ret;
+
+	/*
+	 * Because interrupt line can be shared with other components in
+	 * driver it can't use IRQF_ONESHOT flag here.
+	 */
+	ret = devm_request_threaded_irq(cdns->dev, cdns->dev_irq,
+					cdns3_device_irq_handler,
+					cdns3_device_thread_irq_handler,
+					IRQF_SHARED, dev_name(cdns->dev), cdns);
+
+	if (ret)
+		goto err0;
+
+	return 0;
+err0:
+	cdns3_gadget_exit(cdns);
+	return ret;
+}
+
+static int cdns3_gadget_suspend(struct cdns3 *cdns, bool do_wakeup)
+{
+	struct cdns3_device *priv_dev = cdns->gadget_dev;
+
+	cdns3_disconnect_gadget(priv_dev);
+
+	priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
+	usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
+	cdns3_hw_reset_eps_config(priv_dev);
+
+	/* disable interrupt for device */
+	writel(0, &priv_dev->regs->usb_ien);
+
+	return 0;
+}
+
+static int cdns3_gadget_resume(struct cdns3 *cdns, bool hibernated)
+{
+	struct cdns3_device *priv_dev = cdns->gadget_dev;
+
+	if (!priv_dev->gadget_driver)
+		return 0;
+
+	cdns3_gadget_config(priv_dev);
+
+	return 0;
+}
+
+/**
+ * cdns3_gadget_init - initialize device structure
+ *
+ * cdns: cdns3 instance
+ *
+ * This function initializes the gadget.
+ */
+int cdns3_gadget_init(struct cdns3 *cdns)
+{
+	struct cdns3_role_driver *rdrv;
+
+	rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
+	if (!rdrv)
+		return -ENOMEM;
+
+	rdrv->start	= __cdns3_gadget_init;
+	rdrv->stop	= cdns3_gadget_exit;
+	rdrv->suspend	= cdns3_gadget_suspend;
+	rdrv->resume	= cdns3_gadget_resume;
+	rdrv->state	= CDNS3_ROLE_STATE_INACTIVE;
+	rdrv->name	= "gadget";
+	cdns->roles[USB_ROLE_DEVICE] = rdrv;
+
+	return 0;
+}
diff -Naur --no-dereference a/drivers/usb/cdns3/gadget-export.h b/drivers/usb/cdns3/gadget-export.h
--- a/drivers/usb/cdns3/gadget-export.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/usb/cdns3/gadget-export.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Cadence USBSS DRD Driver - Gadget Export APIs.
+ *
+ * Copyright (C) 2017 NXP
+ * Copyright (C) 2017-2018 NXP
+ *
+ * Authors: Peter Chen <peter.chen@nxp.com>
+ */
+#ifndef __LINUX_CDNS3_GADGET_EXPORT
+#define __LINUX_CDNS3_GADGET_EXPORT
+
+#ifdef CONFIG_USB_CDNS3_GADGET
+
+int cdns3_gadget_init(struct cdns3 *cdns);
+void cdns3_gadget_exit(struct cdns3 *cdns);
+#else
+
+static inline int cdns3_gadget_init(struct cdns3 *cdns)
+{
+	return -ENXIO;
+}
+
+static inline void cdns3_gadget_exit(struct cdns3 *cdns) { }
+
+#endif
+
+#endif /* __LINUX_CDNS3_GADGET_EXPORT */
diff -Naur --no-dereference a/drivers/usb/cdns3/gadget.h b/drivers/usb/cdns3/gadget.h
--- a/drivers/usb/cdns3/gadget.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/usb/cdns3/gadget.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,1338 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * USBSS device controller driver header file
+ *
+ * Copyright (C) 2018-2019 Cadence.
+ * Copyright (C) 2017-2018 NXP
+ *
+ * Author: Pawel Laszczak <pawell@cadence.com>
+ *         Pawel Jez <pjez@cadence.com>
+ *         Peter Chen <peter.chen@nxp.com>
+ */
+#ifndef __LINUX_CDNS3_GADGET
+#define __LINUX_CDNS3_GADGET
+#include <linux/usb/gadget.h>
+
+/*
+ * USBSS-DEV register interface.
+ * This corresponds to the USBSS Device Controller Interface
+ */
+
+/**
+ * struct cdns3_usb_regs - device controller registers.
+ * @usb_conf:      Global Configuration.
+ * @usb_sts:       Global Status.
+ * @usb_cmd:       Global Command.
+ * @usb_itpn:      ITP/SOF number.
+ * @usb_lpm:       Global Command.
+ * @usb_ien:       USB Interrupt Enable.
+ * @usb_ists:      USB Interrupt Status.
+ * @ep_sel:        Endpoint Select.
+ * @ep_traddr:     Endpoint Transfer Ring Address.
+ * @ep_cfg:        Endpoint Configuration.
+ * @ep_cmd:        Endpoint Command.
+ * @ep_sts:        Endpoint Status.
+ * @ep_sts_sid:    Endpoint Status.
+ * @ep_sts_en:     Endpoint Status Enable.
+ * @drbl:          Doorbell.
+ * @ep_ien:        EP Interrupt Enable.
+ * @ep_ists:       EP Interrupt Status.
+ * @usb_pwr:       Global Power Configuration.
+ * @usb_conf2:     Global Configuration 2.
+ * @usb_cap1:      Capability 1.
+ * @usb_cap2:      Capability 2.
+ * @usb_cap3:      Capability 3.
+ * @usb_cap4:      Capability 4.
+ * @usb_cap5:      Capability 5.
+ * @usb_cap6:      Capability 6.
+ * @usb_cpkt1:     Custom Packet 1.
+ * @usb_cpkt2:     Custom Packet 2.
+ * @usb_cpkt3:     Custom Packet 3.
+ * @ep_dma_ext_addr: Upper address for DMA operations.
+ * @buf_addr:      Address for On-chip Buffer operations.
+ * @buf_data:      Data for On-chip Buffer operations.
+ * @buf_ctrl:      On-chip Buffer Access Control.
+ * @dtrans:        DMA Transfer Mode.
+ * @tdl_from_trb:  Source of TD Configuration.
+ * @tdl_beh:       TDL Behavior Configuration.
+ * @ep_tdl:        Endpoint TDL.
+ * @tdl_beh2:      TDL Behavior 2 Configuration.
+ * @dma_adv_td:    DMA Advance TD Configuration.
+ * @reserved1:     Reserved.
+ * @cfg_regs:      Configuration.
+ * @reserved2:     Reserved.
+ * @dma_axi_ctrl:  AXI Control.
+ * @dma_axi_id:    AXI ID register.
+ * @dma_axi_cap:   AXI Capability.
+ * @dma_axi_ctrl0: AXI Control 0.
+ * @dma_axi_ctrl1: AXI Control 1.
+ */
+struct cdns3_usb_regs {
+	__le32 usb_conf;
+	__le32 usb_sts;
+	__le32 usb_cmd;
+	__le32 usb_itpn;
+	__le32 usb_lpm;
+	__le32 usb_ien;
+	__le32 usb_ists;
+	__le32 ep_sel;
+	__le32 ep_traddr;
+	__le32 ep_cfg;
+	__le32 ep_cmd;
+	__le32 ep_sts;
+	__le32 ep_sts_sid;
+	__le32 ep_sts_en;
+	__le32 drbl;
+	__le32 ep_ien;
+	__le32 ep_ists;
+	__le32 usb_pwr;
+	__le32 usb_conf2;
+	__le32 usb_cap1;
+	__le32 usb_cap2;
+	__le32 usb_cap3;
+	__le32 usb_cap4;
+	__le32 usb_cap5;
+	__le32 usb_cap6;
+	__le32 usb_cpkt1;
+	__le32 usb_cpkt2;
+	__le32 usb_cpkt3;
+	__le32 ep_dma_ext_addr;
+	__le32 buf_addr;
+	__le32 buf_data;
+	__le32 buf_ctrl;
+	__le32 dtrans;
+	__le32 tdl_from_trb;
+	__le32 tdl_beh;
+	__le32 ep_tdl;
+	__le32 tdl_beh2;
+	__le32 dma_adv_td;
+	__le32 reserved1[26];
+	__le32 cfg_reg1;
+	__le32 dbg_link1;
+	__le32 dbg_link2;
+	__le32 cfg_regs[74];
+	__le32 reserved2[51];
+	__le32 dma_axi_ctrl;
+	__le32 dma_axi_id;
+	__le32 dma_axi_cap;
+	__le32 dma_axi_ctrl0;
+	__le32 dma_axi_ctrl1;
+};
+
+/* USB_CONF - bitmasks */
+/* Reset USB device configuration. */
+#define USB_CONF_CFGRST		BIT(0)
+/* Set Configuration. */
+#define USB_CONF_CFGSET		BIT(1)
+/* Disconnect USB device in SuperSpeed. */
+#define USB_CONF_USB3DIS	BIT(3)
+/* Disconnect USB device in HS/FS */
+#define USB_CONF_USB2DIS	BIT(4)
+/* Little Endian access - default */
+#define USB_CONF_LENDIAN	BIT(5)
+/*
+ * Big Endian access. Driver assume that byte order for
+ * SFRs access always is as Little Endian so this bit
+ * is not used.
+ */
+#define USB_CONF_BENDIAN	BIT(6)
+/* Device software reset. */
+#define USB_CONF_SWRST		BIT(7)
+/* Singular DMA transfer mode. Only for VER < DEV_VER_V3*/
+#define USB_CONF_DSING		BIT(8)
+/* Multiple DMA transfers mode. Only for VER < DEV_VER_V3 */
+#define USB_CONF_DMULT		BIT(9)
+/* DMA clock turn-off enable. */
+#define USB_CONF_DMAOFFEN	BIT(10)
+/* DMA clock turn-off disable. */
+#define USB_CONF_DMAOFFDS	BIT(11)
+/* Clear Force Full Speed. */
+#define USB_CONF_CFORCE_FS	BIT(12)
+/* Set Force Full Speed. */
+#define USB_CONF_SFORCE_FS	BIT(13)
+/* Device enable. */
+#define USB_CONF_DEVEN		BIT(14)
+/* Device disable. */
+#define USB_CONF_DEVDS		BIT(15)
+/* L1 LPM state entry enable (used in HS/FS mode). */
+#define USB_CONF_L1EN		BIT(16)
+/* L1 LPM state entry disable (used in HS/FS mode). */
+#define USB_CONF_L1DS		BIT(17)
+/* USB 2.0 clock gate disable. */
+#define USB_CONF_CLK2OFFEN	BIT(18)
+/* USB 2.0 clock gate enable. */
+#define USB_CONF_CLK2OFFDS	BIT(19)
+/* L0 LPM state entry request (used in HS/FS mode). */
+#define USB_CONF_LGO_L0		BIT(20)
+/* USB 3.0 clock gate disable. */
+#define USB_CONF_CLK3OFFEN	BIT(21)
+/* USB 3.0 clock gate enable. */
+#define USB_CONF_CLK3OFFDS	BIT(22)
+/* Bit 23 is reserved*/
+/* U1 state entry enable (used in SS mode). */
+#define USB_CONF_U1EN		BIT(24)
+/* U1 state entry disable (used in SS mode). */
+#define USB_CONF_U1DS		BIT(25)
+/* U2 state entry enable (used in SS mode). */
+#define USB_CONF_U2EN		BIT(26)
+/* U2 state entry disable (used in SS mode). */
+#define USB_CONF_U2DS		BIT(27)
+/* U0 state entry request (used in SS mode). */
+#define USB_CONF_LGO_U0		BIT(28)
+/* U1 state entry request (used in SS mode). */
+#define USB_CONF_LGO_U1		BIT(29)
+/* U2 state entry request (used in SS mode). */
+#define USB_CONF_LGO_U2		BIT(30)
+/* SS.Inactive state entry request (used in SS mode) */
+#define USB_CONF_LGO_SSINACT	BIT(31)
+
+/* USB_STS - bitmasks */
+/*
+ * Configuration status.
+ * 1 - device is in the configured state.
+ * 0 - device is not configured.
+ */
+#define USB_STS_CFGSTS_MASK	BIT(0)
+#define USB_STS_CFGSTS(p)	((p) & USB_STS_CFGSTS_MASK)
+/*
+ * On-chip memory overflow.
+ * 0 - On-chip memory status OK.
+ * 1 - On-chip memory overflow.
+ */
+#define USB_STS_OV_MASK		BIT(1)
+#define USB_STS_OV(p)		((p) & USB_STS_OV_MASK)
+/*
+ * SuperSpeed connection status.
+ * 0 - USB in SuperSpeed mode disconnected.
+ * 1 - USB in SuperSpeed mode connected.
+ */
+#define USB_STS_USB3CONS_MASK	BIT(2)
+#define USB_STS_USB3CONS(p)	((p) & USB_STS_USB3CONS_MASK)
+/*
+ * DMA transfer configuration status.
+ * 0 - single request.
+ * 1 - multiple TRB chain
+ * Supported only for controller version <  DEV_VER_V3
+ */
+#define USB_STS_DTRANS_MASK	BIT(3)
+#define USB_STS_DTRANS(p)	((p) & USB_STS_DTRANS_MASK)
+/*
+ * Device speed.
+ * 0 - Undefined (value after reset).
+ * 1 - Low speed
+ * 2 - Full speed
+ * 3 - High speed
+ * 4 - Super speed
+ */
+#define USB_STS_USBSPEED_MASK	GENMASK(6, 4)
+#define USB_STS_USBSPEED(p)	(((p) & USB_STS_USBSPEED_MASK) >> 4)
+#define USB_STS_LS		(0x1 << 4)
+#define USB_STS_FS		(0x2 << 4)
+#define USB_STS_HS		(0x3 << 4)
+#define USB_STS_SS		(0x4 << 4)
+#define DEV_UNDEFSPEED(p)	(((p) & USB_STS_USBSPEED_MASK) == (0x0 << 4))
+#define DEV_LOWSPEED(p)		(((p) & USB_STS_USBSPEED_MASK) == USB_STS_LS)
+#define DEV_FULLSPEED(p)	(((p) & USB_STS_USBSPEED_MASK) == USB_STS_FS)
+#define DEV_HIGHSPEED(p)	(((p) & USB_STS_USBSPEED_MASK) == USB_STS_HS)
+#define DEV_SUPERSPEED(p)	(((p) & USB_STS_USBSPEED_MASK) == USB_STS_SS)
+/*
+ * Endianness for SFR access.
+ * 0 - Little Endian order (default after hardware reset).
+ * 1 - Big Endian order
+ */
+#define USB_STS_ENDIAN_MASK	BIT(7)
+#define USB_STS_ENDIAN(p)	((p) & USB_STS_ENDIAN_MASK)
+/*
+ * HS/FS clock turn-off status.
+ * 0 - hsfs clock is always on.
+ * 1 - hsfs clock turn-off in L2 (HS/FS mode) is enabled
+ *          (default after hardware reset).
+ */
+#define USB_STS_CLK2OFF_MASK	BIT(8)
+#define USB_STS_CLK2OFF(p)	((p) & USB_STS_CLK2OFF_MASK)
+/*
+ * PCLK clock turn-off status.
+ * 0 - pclk clock is always on.
+ * 1 - pclk clock turn-off in U3 (SS mode) is enabled
+ *          (default after hardware reset).
+ */
+#define USB_STS_CLK3OFF_MASK	BIT(9)
+#define USB_STS_CLK3OFF(p)	((p) & USB_STS_CLK3OFF_MASK)
+/*
+ * Controller in reset state.
+ * 0 - Internal reset is active.
+ * 1 - Internal reset is not active and controller is fully operational.
+ */
+#define USB_STS_IN_RST_MASK	BIT(10)
+#define USB_STS_IN_RST(p)	((p) & USB_STS_IN_RST_MASK)
+/*
+ * Status of the "TDL calculation basing on TRB" feature.
+ * 0 - disabled
+ * 1 - enabled
+ * Supported only for DEV_VER_V2 controller version.
+ */
+#define USB_STS_TDL_TRB_ENABLED	BIT(11)
+/*
+ * Device enable Status.
+ * 0 - USB device is disabled (VBUS input is disconnected from internal logic).
+ * 1 - USB device is enabled (VBUS input is connected to the internal logic).
+ */
+#define USB_STS_DEVS_MASK	BIT(14)
+#define USB_STS_DEVS(p)		((p) & USB_STS_DEVS_MASK)
+/*
+ * Address status.
+ * 0 - USB device is default state.
+ * 1 - USB device is at least in address state.
+ */
+#define USB_STS_ADDRESSED_MASK	BIT(15)
+#define USB_STS_ADDRESSED(p)	((p) & USB_STS_ADDRESSED_MASK)
+/*
+ * L1 LPM state enable status (used in HS/FS mode).
+ * 0 - Entering to L1 LPM state disabled.
+ * 1 - Entering to L1 LPM state enabled.
+ */
+#define USB_STS_L1ENS_MASK	BIT(16)
+#define USB_STS_L1ENS(p)	((p) & USB_STS_L1ENS_MASK)
+/*
+ * Internal VBUS connection status (used both in HS/FS  and SS mode).
+ * 0 - internal VBUS is not detected.
+ * 1 - internal VBUS is detected.
+ */
+#define USB_STS_VBUSS_MASK	BIT(17)
+#define USB_STS_VBUSS(p)	((p) & USB_STS_VBUSS_MASK)
+/*
+ * HS/FS LPM  state (used in FS/HS mode).
+ * 0 - L0 State
+ * 1 - L1 State
+ * 2 - L2 State
+ * 3 - L3 State
+ */
+#define USB_STS_LPMST_MASK	GENMASK(19, 18)
+#define DEV_L0_STATE(p)		(((p) & USB_STS_LPMST_MASK) == (0x0 << 18))
+#define DEV_L1_STATE(p)		(((p) & USB_STS_LPMST_MASK) == (0x1 << 18))
+#define DEV_L2_STATE(p)		(((p) & USB_STS_LPMST_MASK) == (0x2 << 18))
+#define DEV_L3_STATE(p)		(((p) & USB_STS_LPMST_MASK) == (0x3 << 18))
+/*
+ * Disable HS status (used in FS/HS mode).
+ * 0 - the disconnect bit for HS/FS mode is set .
+ * 1 - the disconnect bit for HS/FS mode is not set.
+ */
+#define USB_STS_USB2CONS_MASK	BIT(20)
+#define USB_STS_USB2CONS(p)	((p) & USB_STS_USB2CONS_MASK)
+/*
+ * HS/FS mode connection status (used in FS/HS mode).
+ * 0 - High Speed operations in USB2.0 (FS/HS) mode not disabled.
+ * 1 - High Speed operations in USB2.0 (FS/HS).
+ */
+#define USB_STS_DISABLE_HS_MASK	BIT(21)
+#define USB_STS_DISABLE_HS(p)	((p) & USB_STS_DISABLE_HS_MASK)
+/*
+ * U1 state enable status (used in SS mode).
+ * 0 - Entering to  U1 state disabled.
+ * 1 - Entering to  U1 state enabled.
+ */
+#define USB_STS_U1ENS_MASK	BIT(24)
+#define USB_STS_U1ENS(p)	((p) & USB_STS_U1ENS_MASK)
+/*
+ * U2 state enable status (used in SS mode).
+ * 0 - Entering to  U2 state disabled.
+ * 1 - Entering to  U2 state enabled.
+ */
+#define USB_STS_U2ENS_MASK	BIT(25)
+#define USB_STS_U2ENS(p)	((p) & USB_STS_U2ENS_MASK)
+/*
+ * SuperSpeed Link LTSSM state. This field reflects USBSS-DEV current
+ * SuperSpeed link state
+ */
+#define USB_STS_LST_MASK	GENMASK(29, 26)
+#define DEV_LST_U0		(((p) & USB_STS_LST_MASK) == (0x0 << 26))
+#define DEV_LST_U1		(((p) & USB_STS_LST_MASK) == (0x1 << 26))
+#define DEV_LST_U2		(((p) & USB_STS_LST_MASK) == (0x2 << 26))
+#define DEV_LST_U3		(((p) & USB_STS_LST_MASK) == (0x3 << 26))
+#define DEV_LST_DISABLED	(((p) & USB_STS_LST_MASK) == (0x4 << 26))
+#define DEV_LST_RXDETECT	(((p) & USB_STS_LST_MASK) == (0x5 << 26))
+#define DEV_LST_INACTIVE	(((p) & USB_STS_LST_MASK) == (0x6 << 26))
+#define DEV_LST_POLLING		(((p) & USB_STS_LST_MASK) == (0x7 << 26))
+#define DEV_LST_RECOVERY	(((p) & USB_STS_LST_MASK) == (0x8 << 26))
+#define DEV_LST_HOT_RESET	(((p) & USB_STS_LST_MASK) == (0x9 << 26))
+#define DEV_LST_COMP_MODE	(((p) & USB_STS_LST_MASK) == (0xa << 26))
+#define DEV_LST_LB_STATE	(((p) & USB_STS_LST_MASK) == (0xb << 26))
+/*
+ * DMA clock turn-off status.
+ * 0 - DMA clock is always on (default after hardware reset).
+ * 1 - DMA clock turn-off in U1, U2 and U3 (SS mode) is enabled.
+ */
+#define USB_STS_DMAOFF_MASK	BIT(30)
+#define USB_STS_DMAOFF(p)	((p) & USB_STS_DMAOFF_MASK)
+/*
+ * SFR Endian status.
+ * 0 - Little Endian order (default after hardware reset).
+ * 1 - Big Endian order.
+ */
+#define USB_STS_ENDIAN2_MASK	BIT(31)
+#define USB_STS_ENDIAN2(p)	((p) & USB_STS_ENDIAN2_MASK)
+
+/* USB_CMD -  bitmasks */
+/* Set Function Address */
+#define USB_CMD_SET_ADDR	BIT(0)
+/*
+ * Function Address This field is saved to the device only when the field
+ * SET_ADDR is set '1 ' during write to USB_CMD register.
+ * Software is responsible for entering the address of the device during
+ * SET_ADDRESS request service. This field should be set immediately after
+ * the SETUP packet is decoded, and prior to confirmation of the status phase
+ */
+#define USB_CMD_FADDR_MASK	GENMASK(7, 1)
+#define USB_CMD_FADDR(p)	(((p) << 1) & USB_CMD_FADDR_MASK)
+/* Send Function Wake Device Notification TP (used only in SS mode). */
+#define USB_CMD_SDNFW		BIT(8)
+/* Set Test Mode (used only in HS/FS mode). */
+#define USB_CMD_STMODE		BIT(9)
+/* Test mode selector (used only in HS/FS mode) */
+#define USB_STS_TMODE_SEL_MASK	GENMASK(11, 10)
+#define USB_STS_TMODE_SEL(p)	(((p) << 10) & USB_STS_TMODE_SEL_MASK)
+/*
+ *  Send Latency Tolerance Message Device Notification TP (used only
+ *  in SS mode).
+ */
+#define USB_CMD_SDNLTM		BIT(12)
+/* Send Custom Transaction Packet (used only in SS mode) */
+#define USB_CMD_SPKT		BIT(13)
+/*Device Notification 'Function Wake' - Interface value (only in SS mode. */
+#define USB_CMD_DNFW_INT_MASK	GENMASK(23, 16)
+#define USB_STS_DNFW_INT(p)	(((p) << 16) & USB_CMD_DNFW_INT_MASK)
+/*
+ * Device Notification 'Latency Tolerance Message' -373 BELT value [7:0]
+ * (used only in SS mode).
+ */
+#define USB_CMD_DNLTM_BELT_MASK	GENMASK(27, 16)
+#define USB_STS_DNLTM_BELT(p)	(((p) << 16) & USB_CMD_DNLTM_BELT_MASK)
+
+/* USB_ITPN - bitmasks */
+/*
+ * ITP(SS) / SOF (HS/FS) number
+ * In SS mode this field represent number of last ITP received from host.
+ * In HS/FS mode this field represent number of last SOF received from host.
+ */
+#define USB_ITPN_MASK		GENMASK(13, 0)
+#define USB_ITPN(p)		((p) & USB_ITPN_MASK)
+
+/* USB_LPM - bitmasks */
+/* Host Initiated Resume Duration. */
+#define USB_LPM_HIRD_MASK	GENMASK(3, 0)
+#define USB_LPM_HIRD(p)		((p) & USB_LPM_HIRD_MASK)
+/* Remote Wakeup Enable (bRemoteWake). */
+#define USB_LPM_BRW		BIT(4)
+
+/* USB_IEN - bitmasks */
+/* SS connection interrupt enable */
+#define USB_IEN_CONIEN		BIT(0)
+/* SS disconnection interrupt enable. */
+#define USB_IEN_DISIEN		BIT(1)
+/* USB SS warm reset interrupt enable. */
+#define USB_IEN_UWRESIEN	BIT(2)
+/* USB SS hot reset interrupt enable */
+#define USB_IEN_UHRESIEN	BIT(3)
+/* SS link U3 state enter interrupt enable (suspend).*/
+#define USB_IEN_U3ENTIEN	BIT(4)
+/* SS link U3 state exit interrupt enable (wakeup). */
+#define USB_IEN_U3EXTIEN	BIT(5)
+/* SS link U2 state enter interrupt enable.*/
+#define USB_IEN_U2ENTIEN	BIT(6)
+/* SS link U2 state exit interrupt enable.*/
+#define USB_IEN_U2EXTIEN	BIT(7)
+/* SS link U1 state enter interrupt enable.*/
+#define USB_IEN_U1ENTIEN	BIT(8)
+/* SS link U1 state exit interrupt enable.*/
+#define USB_IEN_U1EXTIEN	BIT(9)
+/* ITP/SOF packet detected interrupt enable.*/
+#define USB_IEN_ITPIEN		BIT(10)
+/* Wakeup interrupt enable.*/
+#define USB_IEN_WAKEIEN		BIT(11)
+/* Send Custom Packet interrupt enable.*/
+#define USB_IEN_SPKTIEN		BIT(12)
+/* HS/FS mode connection interrupt enable.*/
+#define USB_IEN_CON2IEN		BIT(16)
+/* HS/FS mode disconnection interrupt enable.*/
+#define USB_IEN_DIS2IEN		BIT(17)
+/* USB reset (HS/FS mode) interrupt enable.*/
+#define USB_IEN_U2RESIEN	BIT(18)
+/* LPM L2 state enter interrupt enable.*/
+#define USB_IEN_L2ENTIEN	BIT(20)
+/* LPM  L2 state exit interrupt enable.*/
+#define USB_IEN_L2EXTIEN	BIT(21)
+/* LPM L1 state enter interrupt enable.*/
+#define USB_IEN_L1ENTIEN	BIT(24)
+/* LPM  L1 state exit interrupt enable.*/
+#define USB_IEN_L1EXTIEN	BIT(25)
+/* Configuration reset interrupt enable.*/
+#define USB_IEN_CFGRESIEN	BIT(26)
+/* Start of the USB SS warm reset interrupt enable.*/
+#define USB_IEN_UWRESSIEN	BIT(28)
+/* End of the USB SS warm reset interrupt enable.*/
+#define USB_IEN_UWRESEIEN	BIT(29)
+
+#define USB_IEN_INIT  (USB_IEN_U2RESIEN | USB_ISTS_DIS2I | USB_IEN_CON2IEN \
+		       | USB_IEN_UHRESIEN | USB_IEN_UWRESIEN | USB_IEN_DISIEN \
+		       | USB_IEN_CONIEN | USB_IEN_U3EXTIEN | USB_IEN_L2ENTIEN \
+		       | USB_IEN_L2EXTIEN | USB_IEN_L1ENTIEN | USB_IEN_U3ENTIEN)
+
+/* USB_ISTS - bitmasks */
+/* SS Connection detected. */
+#define USB_ISTS_CONI		BIT(0)
+/* SS Disconnection detected. */
+#define USB_ISTS_DISI		BIT(1)
+/* UUSB warm reset detectede. */
+#define USB_ISTS_UWRESI		BIT(2)
+/* USB hot reset detected. */
+#define USB_ISTS_UHRESI		BIT(3)
+/* U3 link state enter detected (suspend).*/
+#define USB_ISTS_U3ENTI		BIT(4)
+/* U3 link state exit detected (wakeup). */
+#define USB_ISTS_U3EXTI		BIT(5)
+/* U2 link state enter detected.*/
+#define USB_ISTS_U2ENTI		BIT(6)
+/* U2 link state exit detected.*/
+#define USB_ISTS_U2EXTI		BIT(7)
+/* U1 link state enter detected.*/
+#define USB_ISTS_U1ENTI		BIT(8)
+/* U1 link state exit detected.*/
+#define USB_ISTS_U1EXTI		BIT(9)
+/* ITP/SOF packet detected.*/
+#define USB_ISTS_ITPI		BIT(10)
+/* Wakeup detected.*/
+#define USB_ISTS_WAKEI		BIT(11)
+/* Send Custom Packet detected.*/
+#define USB_ISTS_SPKTI		BIT(12)
+/* HS/FS mode connection detected.*/
+#define USB_ISTS_CON2I		BIT(16)
+/* HS/FS mode disconnection detected.*/
+#define USB_ISTS_DIS2I		BIT(17)
+/* USB reset (HS/FS mode) detected.*/
+#define USB_ISTS_U2RESI		BIT(18)
+/* LPM L2 state enter detected.*/
+#define USB_ISTS_L2ENTI		BIT(20)
+/* LPM  L2 state exit detected.*/
+#define USB_ISTS_L2EXTI		BIT(21)
+/* LPM L1 state enter detected.*/
+#define USB_ISTS_L1ENTI		BIT(24)
+/* LPM L1 state exit detected.*/
+#define USB_ISTS_L1EXTI		BIT(25)
+/* USB configuration reset detected.*/
+#define USB_ISTS_CFGRESI	BIT(26)
+/* Start of the USB warm reset detected.*/
+#define USB_ISTS_UWRESSI	BIT(28)
+/* End of the USB warm reset detected.*/
+#define USB_ISTS_UWRESEI	BIT(29)
+
+/* USB_SEL - bitmasks */
+#define EP_SEL_EPNO_MASK	GENMASK(3, 0)
+/* Endpoint number. */
+#define EP_SEL_EPNO(p)		((p) & EP_SEL_EPNO_MASK)
+/* Endpoint direction bit - 0 - OUT, 1 - IN. */
+#define EP_SEL_DIR		BIT(7)
+
+#define select_ep_in(nr)	(EP_SEL_EPNO(p) | EP_SEL_DIR)
+#define select_ep_out		(EP_SEL_EPNO(p))
+
+/* EP_TRADDR - bitmasks */
+/* Transfer Ring address. */
+#define EP_TRADDR_TRADDR(p)	((p))
+
+/* EP_CFG - bitmasks */
+/* Endpoint enable */
+#define EP_CFG_ENABLE		BIT(0)
+/*
+ *  Endpoint type.
+ * 1 - isochronous
+ * 2 - bulk
+ * 3 - interrupt
+ */
+#define EP_CFG_EPTYPE_MASK	GENMASK(2, 1)
+#define EP_CFG_EPTYPE(p)	(((p) << 1)  & EP_CFG_EPTYPE_MASK)
+/* Stream support enable (only in SS mode). */
+#define EP_CFG_STREAM_EN	BIT(3)
+/* TDL check (only in SS mode for BULK EP). */
+#define EP_CFG_TDL_CHK		BIT(4)
+/* SID check (only in SS mode for BULK OUT EP). */
+#define EP_CFG_SID_CHK		BIT(5)
+/* DMA transfer endianness. */
+#define EP_CFG_EPENDIAN		BIT(7)
+/* Max burst size (used only in SS mode). */
+#define EP_CFG_MAXBURST_MASK	GENMASK(11, 8)
+#define EP_CFG_MAXBURST(p)	(((p) << 8) & EP_CFG_MAXBURST_MASK)
+/* ISO max burst. */
+#define EP_CFG_MULT_MASK	GENMASK(15, 14)
+#define EP_CFG_MULT(p)		(((p) << 14) & EP_CFG_MULT_MASK)
+/* ISO max burst. */
+#define EP_CFG_MAXPKTSIZE_MASK	GENMASK(26, 16)
+#define EP_CFG_MAXPKTSIZE(p)	(((p) << 16) & EP_CFG_MAXPKTSIZE_MASK)
+/* Max number of buffered packets. */
+#define EP_CFG_BUFFERING_MASK	GENMASK(31, 27)
+#define EP_CFG_BUFFERING(p)	(((p) << 27) & EP_CFG_BUFFERING_MASK)
+
+/* EP_CMD - bitmasks */
+/* Endpoint reset. */
+#define EP_CMD_EPRST		BIT(0)
+/* Endpoint STALL set. */
+#define EP_CMD_SSTALL		BIT(1)
+/* Endpoint STALL clear. */
+#define EP_CMD_CSTALL		BIT(2)
+/* Send ERDY TP. */
+#define EP_CMD_ERDY		BIT(3)
+/* Request complete. */
+#define EP_CMD_REQ_CMPL		BIT(5)
+/* Transfer descriptor ready. */
+#define EP_CMD_DRDY		BIT(6)
+/* Data flush. */
+#define EP_CMD_DFLUSH		BIT(7)
+/*
+ * Transfer Descriptor Length write  (used only for Bulk Stream capable
+ * endpoints in SS mode).
+ * Bit Removed from DEV_VER_V3 controller version.
+ */
+#define EP_CMD_STDL		BIT(8)
+/*
+ * Transfer Descriptor Length (used only in SS mode for bulk endpoints).
+ * Bits Removed from DEV_VER_V3 controller version.
+ */
+#define EP_CMD_TDL_MASK		GENMASK(15, 9)
+#define EP_CMD_TDL_SET(p)	(((p) << 9) & EP_CMD_TDL_MASK)
+#define EP_CMD_TDL_GET(p)	(((p) & EP_CMD_TDL_MASK) >> 9)
+
+/* ERDY Stream ID value (used in SS mode). */
+#define EP_CMD_ERDY_SID_MASK	GENMASK(31, 16)
+#define EP_CMD_ERDY_SID(p)	(((p) << 16) & EP_CMD_ERDY_SID_MASK)
+
+/* EP_STS - bitmasks */
+/* Setup transfer complete. */
+#define EP_STS_SETUP		BIT(0)
+/* Endpoint STALL status. */
+#define EP_STS_STALL(p)		((p) & BIT(1))
+/* Interrupt On Complete. */
+#define EP_STS_IOC		BIT(2)
+/* Interrupt on Short Packet. */
+#define EP_STS_ISP		BIT(3)
+/* Transfer descriptor missing. */
+#define EP_STS_DESCMIS		BIT(4)
+/* Stream Rejected (used only in SS mode) */
+#define EP_STS_STREAMR		BIT(5)
+/* EXIT from MOVE DATA State (used only for stream transfers in SS mode). */
+#define EP_STS_MD_EXIT		BIT(6)
+/* TRB error. */
+#define EP_STS_TRBERR		BIT(7)
+/* Not ready (used only in SS mode). */
+#define EP_STS_NRDY		BIT(8)
+/* DMA busy bit. */
+#define EP_STS_DBUSY		BIT(9)
+/* Endpoint Buffer Empty */
+#define EP_STS_BUFFEMPTY(p)	((p) & BIT(10))
+/* Current Cycle Status */
+#define EP_STS_CCS(p)		((p) & BIT(11))
+/* Prime (used only in SS mode. */
+#define EP_STS_PRIME		BIT(12)
+/* Stream error (used only in SS mode). */
+#define EP_STS_SIDERR		BIT(13)
+/* OUT size mismatch. */
+#define EP_STS_OUTSMM		BIT(14)
+/* ISO transmission error. */
+#define EP_STS_ISOERR		BIT(15)
+/* Host Packet Pending (only for SS mode). */
+#define EP_STS_HOSTPP(p)	((p) & BIT(16))
+/* Stream Protocol State Machine State (only for Bulk stream endpoints). */
+#define EP_STS_SPSMST_MASK		GENMASK(18, 17)
+#define EP_STS_SPSMST_DISABLED(p)	(((p) & EP_STS_SPSMST_MASK) >> 17)
+#define EP_STS_SPSMST_IDLE(p)		(((p) & EP_STS_SPSMST_MASK) >> 17)
+#define EP_STS_SPSMST_START_STREAM(p)	(((p) & EP_STS_SPSMST_MASK) >> 17)
+#define EP_STS_SPSMST_MOVE_DATA(p)	(((p) & EP_STS_SPSMST_MASK) >> 17)
+/* Interrupt On Transfer complete. */
+#define EP_STS_IOT		BIT(19)
+/* OUT queue endpoint number. */
+#define EP_STS_OUTQ_NO_MASK	GENMASK(27, 24)
+#define EP_STS_OUTQ_NO(p)	(((p) & EP_STS_OUTQ_NO_MASK) >> 24)
+/* OUT queue valid flag. */
+#define EP_STS_OUTQ_VAL_MASK	BIT(28)
+#define EP_STS_OUTQ_VAL(p)	((p) & EP_STS_OUTQ_VAL_MASK)
+/* SETUP WAIT. */
+#define EP_STS_STPWAIT		BIT(31)
+
+/* EP_STS_SID - bitmasks */
+/* Stream ID (used only in SS mode). */
+#define EP_STS_SID_MASK		GENMASK(15, 0)
+#define EP_STS_SID(p)		((p) & EP_STS_SID_MASK)
+
+/* EP_STS_EN - bitmasks */
+/* SETUP interrupt enable. */
+#define EP_STS_EN_SETUPEN	BIT(0)
+/* OUT transfer missing descriptor enable. */
+#define EP_STS_EN_DESCMISEN	BIT(4)
+/* Stream Rejected enable. */
+#define EP_STS_EN_STREAMREN	BIT(5)
+/* Move Data Exit enable.*/
+#define EP_STS_EN_MD_EXITEN	BIT(6)
+/* TRB enable. */
+#define EP_STS_EN_TRBERREN	BIT(7)
+/* NRDY enable. */
+#define EP_STS_EN_NRDYEN	BIT(8)
+/* Prime enable. */
+#define EP_STS_EN_PRIMEEEN	BIT(12)
+/* Stream error enable. */
+#define EP_STS_EN_SIDERREN	BIT(13)
+/* OUT size mismatch enable. */
+#define EP_STS_EN_OUTSMMEN	BIT(14)
+/* ISO transmission error enable. */
+#define EP_STS_EN_ISOERREN	BIT(15)
+/* Interrupt on Transmission complete enable. */
+#define EP_STS_EN_IOTEN		BIT(19)
+/* Setup Wait interrupt enable. */
+#define EP_STS_EN_STPWAITEN	BIT(31)
+
+/* DRBL- bitmasks */
+#define DB_VALUE_BY_INDEX(index) (1 << (index))
+#define DB_VALUE_EP0_OUT	BIT(0)
+#define DB_VALUE_EP0_IN		BIT(16)
+
+/* EP_IEN - bitmasks */
+#define EP_IEN(index)		(1 << (index))
+#define EP_IEN_EP_OUT0		BIT(0)
+#define EP_IEN_EP_IN0		BIT(16)
+
+/* EP_ISTS - bitmasks */
+#define EP_ISTS(index)		(1 << (index))
+#define EP_ISTS_EP_OUT0		BIT(0)
+#define EP_ISTS_EP_IN0		BIT(16)
+
+/* USB_PWR- bitmasks */
+/*Power Shut Off capability enable*/
+#define PUSB_PWR_PSO_EN		BIT(0)
+/*Power Shut Off capability disable*/
+#define PUSB_PWR_PSO_DS		BIT(1)
+/*
+ * Enables turning-off Reference Clock.
+ * This bit is optional and implemented only when support for OTG is
+ * implemented (indicated by OTG_READY bit set to '1').
+ */
+#define PUSB_PWR_STB_CLK_SWITCH_EN	BIT(8)
+/*
+ * Status bit indicating that operation required by STB_CLK_SWITCH_EN write
+ * is completed
+ */
+#define PUSB_PWR_STB_CLK_SWITCH_DONE	BIT(9)
+/* This bit informs if Fast Registers Access is enabled. */
+#define PUSB_PWR_FST_REG_ACCESS_STAT	BIT(30)
+/* Fast Registers Access Enable. */
+#define PUSB_PWR_FST_REG_ACCESS		BIT(31)
+
+/* USB_CONF2- bitmasks */
+/*
+ * Writing 1 disables TDL calculation basing on TRB feature in controller
+ * for DMULT mode.
+ * Bit supported only for DEV_VER_V2 version.
+ */
+#define USB_CONF2_DIS_TDL_TRB		BIT(1)
+/*
+ * Writing 1 enables TDL calculation basing on TRB feature in controller
+ * for DMULT mode.
+ * Bit supported only for DEV_VER_V2 version.
+ */
+#define USB_CONF2_EN_TDL_TRB		BIT(2)
+
+/* USB_CAP1- bitmasks */
+/*
+ * SFR Interface type
+ * These field reflects type of SFR interface implemented:
+ * 0x0 - OCP
+ * 0x1 - AHB,
+ * 0x2 - PLB
+ * 0x3 - AXI
+ * 0x4-0xF - reserved
+ */
+#define USB_CAP1_SFR_TYPE_MASK	GENMASK(3, 0)
+#define DEV_SFR_TYPE_OCP(p)	(((p) & USB_CAP1_SFR_TYPE_MASK) == 0x0)
+#define DEV_SFR_TYPE_AHB(p)	(((p) & USB_CAP1_SFR_TYPE_MASK) == 0x1)
+#define DEV_SFR_TYPE_PLB(p)	(((p) & USB_CAP1_SFR_TYPE_MASK) == 0x2)
+#define DEV_SFR_TYPE_AXI(p)	(((p) & USB_CAP1_SFR_TYPE_MASK) == 0x3)
+/*
+ * SFR Interface width
+ * These field reflects width of SFR interface implemented:
+ * 0x0 - 8 bit interface,
+ * 0x1 - 16 bit interface,
+ * 0x2 - 32 bit interface
+ * 0x3 - 64 bit interface
+ * 0x4-0xF - reserved
+ */
+#define USB_CAP1_SFR_WIDTH_MASK	GENMASK(7, 4)
+#define DEV_SFR_WIDTH_8(p)	(((p) & USB_CAP1_SFR_WIDTH_MASK) == (0x0 << 4))
+#define DEV_SFR_WIDTH_16(p)	(((p) & USB_CAP1_SFR_WIDTH_MASK) == (0x1 << 4))
+#define DEV_SFR_WIDTH_32(p)	(((p) & USB_CAP1_SFR_WIDTH_MASK) == (0x2 << 4))
+#define DEV_SFR_WIDTH_64(p)	(((p) & USB_CAP1_SFR_WIDTH_MASK) == (0x3 << 4))
+/*
+ * DMA Interface type
+ * These field reflects type of DMA interface implemented:
+ * 0x0 - OCP
+ * 0x1 - AHB,
+ * 0x2 - PLB
+ * 0x3 - AXI
+ * 0x4-0xF - reserved
+ */
+#define USB_CAP1_DMA_TYPE_MASK	GENMASK(11, 8)
+#define DEV_DMA_TYPE_OCP(p)	(((p) & USB_CAP1_DMA_TYPE_MASK) == (0x0 << 8))
+#define DEV_DMA_TYPE_AHB(p)	(((p) & USB_CAP1_DMA_TYPE_MASK) == (0x1 << 8))
+#define DEV_DMA_TYPE_PLB(p)	(((p) & USB_CAP1_DMA_TYPE_MASK) == (0x2 << 8))
+#define DEV_DMA_TYPE_AXI(p)	(((p) & USB_CAP1_DMA_TYPE_MASK) == (0x3 << 8))
+/*
+ * DMA Interface width
+ * These field reflects width of DMA interface implemented:
+ * 0x0 - reserved,
+ * 0x1 - reserved,
+ * 0x2 - 32 bit interface
+ * 0x3 - 64 bit interface
+ * 0x4-0xF - reserved
+ */
+#define USB_CAP1_DMA_WIDTH_MASK	GENMASK(15, 12)
+#define DEV_DMA_WIDTH_32(p)	(((p) & USB_CAP1_DMA_WIDTH_MASK) == (0x2 << 12))
+#define DEV_DMA_WIDTH_64(p)	(((p) & USB_CAP1_DMA_WIDTH_MASK) == (0x3 << 12))
+/*
+ * USB3 PHY Interface type
+ * These field reflects type of USB3 PHY interface implemented:
+ * 0x0 - USB PIPE,
+ * 0x1 - RMMI,
+ * 0x2-0xF - reserved
+ */
+#define USB_CAP1_U3PHY_TYPE_MASK GENMASK(19, 16)
+#define DEV_U3PHY_PIPE(p) (((p) & USB_CAP1_U3PHY_TYPE_MASK) == (0x0 << 16))
+#define DEV_U3PHY_RMMI(p) (((p) & USB_CAP1_U3PHY_TYPE_MASK) == (0x1 << 16))
+/*
+ * USB3 PHY Interface width
+ * These field reflects width of USB3 PHY interface implemented:
+ * 0x0 - 8 bit PIPE interface,
+ * 0x1 - 16 bit PIPE interface,
+ * 0x2 - 32 bit PIPE interface,
+ * 0x3 - 64 bit PIPE interface
+ * 0x4-0xF - reserved
+ * Note: When SSIC interface is implemented this field shows the width of
+ * internal PIPE interface. The RMMI interface is always 20bit wide.
+ */
+#define USB_CAP1_U3PHY_WIDTH_MASK GENMASK(23, 20)
+#define DEV_U3PHY_WIDTH_8(p) \
+	(((p) & USB_CAP1_U3PHY_WIDTH_MASK) == (0x0 << 20))
+#define DEV_U3PHY_WIDTH_16(p) \
+	(((p) & USB_CAP1_U3PHY_WIDTH_MASK) == (0x1 << 16))
+#define DEV_U3PHY_WIDTH_32(p) \
+	(((p) & USB_CAP1_U3PHY_WIDTH_MASK) == (0x2 << 20))
+#define DEV_U3PHY_WIDTH_64(p) \
+	(((p) & USB_CAP1_U3PHY_WIDTH_MASK) == (0x3 << 16))
+
+/*
+ * USB2 PHY Interface enable
+ * These field informs if USB2 PHY interface is implemented:
+ * 0x0 - interface NOT implemented,
+ * 0x1 - interface implemented
+ */
+#define USB_CAP1_U2PHY_EN(p)	((p) & BIT(24))
+/*
+ * USB2 PHY Interface type
+ * These field reflects type of USB2 PHY interface implemented:
+ * 0x0 - UTMI,
+ * 0x1 - ULPI
+ */
+#define DEV_U2PHY_ULPI(p)	((p) & BIT(25))
+/*
+ * USB2 PHY Interface width
+ * These field reflects width of USB2 PHY interface implemented:
+ * 0x0 - 8 bit interface,
+ * 0x1 - 16 bit interface,
+ * Note: The ULPI interface is always 8bit wide.
+ */
+#define DEV_U2PHY_WIDTH_16(p)	((p) & BIT(26))
+/*
+ * OTG Ready
+ * 0x0 - pure device mode
+ * 0x1 - some features and ports for CDNS USB OTG controller are implemented.
+ */
+#define USB_CAP1_OTG_READY(p)	((p) & BIT(27))
+
+/*
+ * When set, indicates that controller supports automatic internal TDL
+ * calculation basing on the size provided in TRB (TRB[22:17]) for DMULT mode
+ * Supported only for DEV_VER_V2 controller version.
+ */
+#define USB_CAP1_TDL_FROM_TRB(p)	((p) & BIT(28))
+
+/* USB_CAP2- bitmasks */
+/*
+ * The actual size of the connected On-chip RAM memory in kB:
+ * - 0 means 256 kB (max supported mem size)
+ * - value other than 0 reflects the mem size in kB
+ */
+#define USB_CAP2_ACTUAL_MEM_SIZE(p) ((p) & GENMASK(7, 0))
+/*
+ * Max supported mem size
+ * These field reflects width of on-chip RAM address bus width,
+ * which determines max supported mem size:
+ * 0x0-0x7 - reserved,
+ * 0x8 - support for 4kB mem,
+ * 0x9 - support for 8kB mem,
+ * 0xA - support for 16kB mem,
+ * 0xB - support for 32kB mem,
+ * 0xC - support for 64kB mem,
+ * 0xD - support for 128kB mem,
+ * 0xE - support for 256kB mem,
+ * 0xF - reserved
+ */
+#define USB_CAP2_MAX_MEM_SIZE(p) ((p) & GENMASK(11, 8))
+
+/* USB_CAP3- bitmasks */
+#define EP_IS_IMPLEMENTED(reg, index) ((reg) & (1 << (index)))
+
+/* USB_CAP4- bitmasks */
+#define EP_SUPPORT_ISO(reg, index) ((reg) & (1 << (index)))
+
+/* USB_CAP5- bitmasks */
+#define EP_SUPPORT_STREAM(reg, index) ((reg) & (1 << (index)))
+
+/* USB_CAP6- bitmasks */
+/* The USBSS-DEV Controller  Internal build number. */
+#define GET_DEV_BASE_VERSION(p) ((p) & GENMASK(23, 0))
+/* The USBSS-DEV Controller version number. */
+#define GET_DEV_CUSTOM_VERSION(p) ((p) & GENMASK(31, 24))
+
+#define DEV_VER_NXP_V1		0x00024502
+#define DEV_VER_TI_V1		0x00024509
+#define DEV_VER_V2		0x0002450C
+#define DEV_VER_V3		0x0002450d
+
+/* DBG_LINK1- bitmasks */
+/*
+ * LFPS_MIN_DET_U1_EXIT value This parameter configures the minimum
+ * time required for decoding the received LFPS as an LFPS.U1_Exit.
+ */
+#define DBG_LINK1_LFPS_MIN_DET_U1_EXIT(p)	((p) & GENMASK(7, 0))
+/*
+ * LFPS_MIN_GEN_U1_EXIT value This parameter configures the minimum time for
+ * phytxelecidle deassertion when LFPS.U1_Exit
+ */
+#define DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_MASK	GENMASK(15, 8)
+#define DBG_LINK1_LFPS_MIN_GEN_U1_EXIT(p)	(((p) << 8) & GENMASK(15, 8))
+/*
+ * RXDET_BREAK_DIS value This parameter configures terminating the Far-end
+ * Receiver termination detection sequence:
+ * 0: it is possible that USBSS_DEV will terminate Farend receiver
+ *    termination detection sequence
+ * 1: USBSS_DEV will not terminate Far-end receiver termination
+ *    detection sequence
+ */
+#define DBG_LINK1_RXDET_BREAK_DIS		BIT(16)
+/* LFPS_GEN_PING value This parameter configures the LFPS.Ping generation */
+#define DBG_LINK1_LFPS_GEN_PING(p)		(((p) << 17) & GENMASK(21, 17))
+/*
+ * Set the LFPS_MIN_DET_U1_EXIT value Writing '1' to this bit writes the
+ * LFPS_MIN_DET_U1_EXIT field value to the device. This bit is automatically
+ * cleared. Writing '0' has no effect
+ */
+#define DBG_LINK1_LFPS_MIN_DET_U1_EXIT_SET	BIT(24)
+/*
+ * Set the LFPS_MIN_GEN_U1_EXIT value. Writing '1' to this bit writes the
+ * LFPS_MIN_GEN_U1_EXIT field value to the device. This bit is automatically
+ * cleared. Writing '0' has no effect
+ */
+#define DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_SET	BIT(25)
+/*
+ * Set the RXDET_BREAK_DIS value Writing '1' to this bit writes
+ * the RXDET_BREAK_DIS field value to the device. This bit is automatically
+ * cleared. Writing '0' has no effect
+ */
+#define DBG_LINK1_RXDET_BREAK_DIS_SET		BIT(26)
+/*
+ * Set the LFPS_GEN_PING_SET value Writing '1' to this bit writes
+ * the LFPS_GEN_PING field value to the device. This bit is automatically
+ * cleared. Writing '0' has no effect."
+ */
+#define DBG_LINK1_LFPS_GEN_PING_SET		BIT(27)
+
+/* DMA_AXI_CTRL- bitmasks */
+/* The mawprot pin configuration. */
+#define DMA_AXI_CTRL_MARPROT(p) ((p) & GENMASK(2, 0))
+/* The marprot pin configuration. */
+#define DMA_AXI_CTRL_MAWPROT(p) (((p) & GENMASK(2, 0)) << 16)
+#define DMA_AXI_CTRL_NON_SECURE 0x02
+
+#define gadget_to_cdns3_device(g) (container_of(g, struct cdns3_device, gadget))
+
+#define ep_to_cdns3_ep(ep) (container_of(ep, struct cdns3_endpoint, endpoint))
+
+/*-------------------------------------------------------------------------*/
+/*
+ * USBSS-DEV DMA interface.
+ */
+#define TRBS_PER_SEGMENT	40
+
+#define ISO_MAX_INTERVAL	10
+
+#if TRBS_PER_SEGMENT < 2
+#error "Incorrect TRBS_PER_SEGMENT. Minimal Transfer Ring size is 2."
+#endif
+
+/*
+ *Only for ISOC endpoints - maximum number of TRBs is calculated as
+ * pow(2, bInterval-1) * number of usb requests. It is limitation made by
+ * driver to save memory. Controller must prepare TRB for each ITP even
+ * if bInterval > 1. It's the reason why driver needs so many TRBs for
+ * isochronous endpoints.
+ */
+#define TRBS_PER_ISOC_SEGMENT	(ISO_MAX_INTERVAL * 8)
+
+#define GET_TRBS_PER_SEGMENT(ep_type) ((ep_type) == USB_ENDPOINT_XFER_ISOC ? \
+				      TRBS_PER_ISOC_SEGMENT : TRBS_PER_SEGMENT)
+/**
+ * struct cdns3_trb - represent Transfer Descriptor block.
+ * @buffer:	pointer to buffer data
+ * @length:	length of data
+ * @control:	control flags.
+ *
+ * This structure describes transfer block serviced by DMA module.
+ */
+struct cdns3_trb {
+	__le32 buffer;
+	__le32 length;
+	__le32 control;
+};
+
+#define TRB_SIZE		(sizeof(struct cdns3_trb))
+#define TRB_RING_SIZE		(TRB_SIZE * TRBS_PER_SEGMENT)
+#define TRB_ISO_RING_SIZE	(TRB_SIZE * TRBS_PER_ISOC_SEGMENT)
+#define TRB_CTRL_RING_SIZE	(TRB_SIZE * 2)
+
+/* TRB bit mask */
+#define TRB_TYPE_BITMASK	GENMASK(15, 10)
+#define TRB_TYPE(p)		((p) << 10)
+#define TRB_FIELD_TO_TYPE(p)	(((p) & TRB_TYPE_BITMASK) >> 10)
+
+/* TRB type IDs */
+/* bulk, interrupt, isoc , and control data stage */
+#define TRB_NORMAL		1
+/* TRB for linking ring segments */
+#define TRB_LINK		6
+
+/* Cycle bit - indicates TRB ownership by driver or hw*/
+#define TRB_CYCLE		BIT(0)
+/*
+ * When set to '1', the device will toggle its interpretation of the Cycle bit
+ */
+#define TRB_TOGGLE		BIT(1)
+
+/*
+ * Short Packet (SP). OUT EPs at DMULT=1 only. Indicates if the TRB was
+ * processed while USB short packet was received. No more buffers defined by
+ * the TD will be used. DMA will automatically advance to next TD.
+ * - Shall be set to 0 by Software when putting TRB on the Transfer Ring
+ * - Shall be set to 1 by Controller when Short Packet condition for this TRB
+ *   is detected independent if ISP is set or not.
+ */
+#define TRB_SP			BIT(1)
+
+/* Interrupt on short packet*/
+#define TRB_ISP			BIT(2)
+/*Setting this bit enables FIFO DMA operation mode*/
+#define TRB_FIFO_MODE		BIT(3)
+/* Set PCIe no snoop attribute */
+#define TRB_CHAIN		BIT(4)
+/* Interrupt on completion */
+#define TRB_IOC			BIT(5)
+
+/* stream ID bitmasks. */
+#define TRB_STREAM_ID_BITMASK		GENMASK(31, 16)
+#define TRB_STREAM_ID(p)		((p) << 16)
+#define TRB_FIELD_TO_STREAMID(p)	(((p) & TRB_STREAM_ID_BITMASK) >> 16)
+
+/* Size of TD expressed in USB packets for HS/FS mode. */
+#define TRB_TDL_HS_SIZE(p)	(((p) << 16) & GENMASK(31, 16))
+#define TRB_TDL_HS_SIZE_GET(p)	(((p) & GENMASK(31, 16)) >> 16)
+
+/* transfer_len bitmasks. */
+#define TRB_LEN(p)		((p) & GENMASK(16, 0))
+
+/* Size of TD expressed in USB packets for SS mode. */
+#define TRB_TDL_SS_SIZE(p)	(((p) << 17) & GENMASK(23, 17))
+#define TRB_TDL_SS_SIZE_GET(p)	(((p) & GENMASK(23, 17)) >> 17)
+
+/* transfer_len bitmasks - bits 31:24 */
+#define TRB_BURST_LEN(p)	(((p) << 24) & GENMASK(31, 24))
+#define TRB_BURST_LEN_GET(p)	(((p) & GENMASK(31, 24)) >> 24)
+
+/* Data buffer pointer bitmasks*/
+#define TRB_BUFFER(p)		((p) & GENMASK(31, 0))
+
+/*-------------------------------------------------------------------------*/
+/* Driver numeric constants */
+
+/* Such declaration should be added to ch9.h */
+#define USB_DEVICE_MAX_ADDRESS		127
+
+/* Endpoint init values */
+#define CDNS3_EP_MAX_PACKET_LIMIT	1024
+#define CDNS3_EP_MAX_STREAMS		15
+#define CDNS3_EP0_MAX_PACKET_LIMIT	512
+
+/* All endpoints including EP0 */
+#define CDNS3_ENDPOINTS_MAX_COUNT	32
+#define CDNS3_EP_ZLP_BUF_SIZE		1024
+
+#define CDNS3_EP_BUF_SIZE		2	/* KB */
+#define CDNS3_EP_ISO_HS_MULT		3
+#define CDNS3_EP_ISO_SS_BURST		3
+#define CDNS3_MAX_NUM_DESCMISS_BUF	32
+#define CDNS3_DESCMIS_BUF_SIZE		2048	/* Bytes */
+#define CDNS3_WA2_NUM_BUFFERS		128
+/*-------------------------------------------------------------------------*/
+/* Used structs */
+
+struct cdns3_device;
+
+/**
+ * struct cdns3_endpoint - extended device side representation of USB endpoint.
+ * @endpoint: usb endpoint
+ * @pending_req_list: list of requests queuing on transfer ring.
+ * @deferred_req_list: list of requests waiting for queuing on transfer ring.
+ * @wa2_descmiss_req_list: list of requests internally allocated by driver.
+ * @trb_pool: transfer ring - array of transaction buffers
+ * @trb_pool_dma: dma address of transfer ring
+ * @cdns3_dev: device associated with this endpoint
+ * @name: a human readable name e.g. ep1out
+ * @flags: specify the current state of endpoint
+ * @descmis_req: internal transfer object used for getting data from on-chip
+ *     buffer. It can happen only if function driver doesn't send usb_request
+ *     object on time.
+ * @dir: endpoint direction
+ * @num: endpoint number (1 - 15)
+ * @type: set to bmAttributes & USB_ENDPOINT_XFERTYPE_MASK
+ * @interval: interval between packets used for ISOC endpoint.
+ * @free_trbs: number of free TRBs in transfer ring
+ * @num_trbs: number of all TRBs in transfer ring
+ * @pcs: producer cycle state
+ * @ccs: consumer cycle state
+ * @enqueue: enqueue index in transfer ring
+ * @dequeue: dequeue index in transfer ring
+ * @trb_burst_size: number of burst used in trb.
+ */
+struct cdns3_endpoint {
+	struct usb_ep		endpoint;
+	struct list_head	pending_req_list;
+	struct list_head	deferred_req_list;
+	struct list_head	wa2_descmiss_req_list;
+	int			wa2_counter;
+
+	struct cdns3_trb	*trb_pool;
+	dma_addr_t		trb_pool_dma;
+
+	struct cdns3_device	*cdns3_dev;
+	char			name[20];
+
+#define EP_ENABLED		BIT(0)
+#define EP_STALLED		BIT(1)
+#define EP_STALL_PENDING	BIT(2)
+#define EP_WEDGE		BIT(3)
+#define EP_TRANSFER_STARTED	BIT(4)
+#define EP_UPDATE_EP_TRBADDR	BIT(5)
+#define EP_PENDING_REQUEST	BIT(6)
+#define EP_RING_FULL		BIT(7)
+#define EP_CLAIMED		BIT(8)
+#define EP_DEFERRED_DRDY	BIT(9)
+#define EP_QUIRK_ISO_OUT_EN	BIT(10)
+#define EP_QUIRK_END_TRANSFER	BIT(11)
+#define EP_QUIRK_EXTRA_BUF_DET	BIT(12)
+#define EP_QUIRK_EXTRA_BUF_EN	BIT(13)
+	u32			flags;
+
+	struct cdns3_request	*descmis_req;
+
+	u8			dir;
+	u8			num;
+	u8			type;
+	int			interval;
+
+	int			free_trbs;
+	int			num_trbs;
+	u8			pcs;
+	u8			ccs;
+	int			enqueue;
+	int			dequeue;
+	u8			trb_burst_size;
+
+	unsigned int		wa1_set:1;
+	struct cdns3_trb	*wa1_trb;
+	unsigned int		wa1_trb_index;
+	unsigned int		wa1_cycle_bit:1;
+};
+
+/**
+ * struct cdns3_aligned_buf - represent aligned buffer used for DMA transfer
+ * @buf: aligned to 8 bytes data buffer. Buffer address used in
+ *       TRB shall be aligned to 8.
+ * @dma: dma address
+ * @size: size of buffer
+ * @in_use: inform if this buffer is associated with usb_request
+ * @list: used to adding instance of this object to list
+ */
+struct cdns3_aligned_buf {
+	void			*buf;
+	dma_addr_t		dma;
+	u32			size;
+	int			in_use:1;
+	struct list_head	list;
+};
+
+/**
+ * struct cdns3_request - extended device side representation of usb_request
+ *                        object .
+ * @request: generic usb_request object describing single I/O request.
+ * @priv_ep: extended representation of usb_ep object
+ * @trb: the first TRB association with this request
+ * @start_trb: number of the first TRB in transfer ring
+ * @end_trb: number of the last TRB in transfer ring
+ * @aligned_buf: object holds information about aligned buffer associated whit
+ *               this endpoint
+ * @flags: flag specifying special usage of request
+ * @list: used by internally allocated request to add to wa2_descmiss_req_list.
+ */
+struct cdns3_request {
+	struct usb_request		request;
+	struct cdns3_endpoint		*priv_ep;
+	struct cdns3_trb		*trb;
+	int				start_trb;
+	int				end_trb;
+	struct cdns3_aligned_buf	*aligned_buf;
+#define REQUEST_PENDING			BIT(0)
+#define REQUEST_INTERNAL		BIT(1)
+#define REQUEST_INTERNAL_CH		BIT(2)
+#define REQUEST_ZLP			BIT(3)
+#define REQUEST_UNALIGNED		BIT(4)
+	u32				flags;
+	struct list_head		list;
+};
+
+#define to_cdns3_request(r) (container_of(r, struct cdns3_request, request))
+
+/*Stages used during enumeration process.*/
+#define CDNS3_SETUP_STAGE		0x0
+#define CDNS3_DATA_STAGE		0x1
+#define CDNS3_STATUS_STAGE		0x2
+
+/**
+ * struct cdns3_device - represent USB device.
+ * @dev: pointer to device structure associated whit this controller
+ * @sysdev: pointer to the DMA capable device
+ * @gadget: device side representation of the peripheral controller
+ * @gadget_driver: pointer to the gadget driver
+ * @dev_ver: device controller version.
+ * @lock: for synchronizing
+ * @regs: base address for device side registers
+ * @setup_buf: used while processing usb control requests
+ * @setup_dma: dma address for setup_buf
+ * @zlp_buf - zlp buffer
+ * @ep0_stage: ep0 stage during enumeration process.
+ * @ep0_data_dir: direction for control transfer
+ * @eps: array of pointers to all endpoints with exclusion ep0
+ * @aligned_buf_list: list of aligned buffers internally allocated by driver
+ * @aligned_buf_wq: workqueue freeing  no longer used aligned buf.
+ * @selected_ep: actually selected endpoint. It's used only to improve
+ *               performance.
+ * @isoch_delay: value from Set Isoch Delay request. Only valid on SS/SSP.
+ * @u1_allowed: allow device transition to u1 state
+ * @u2_allowed: allow device transition to u2 state
+ * @is_selfpowered: device is self powered
+ * @setup_pending: setup packet is processing by gadget driver
+ * @hw_configured_flag: hardware endpoint configuration was set.
+ * @wake_up_flag: allow device to remote up the host
+ * @status_completion_no_call: indicate that driver is waiting for status s
+ *     stage completion. It's used in deferred SET_CONFIGURATION request.
+ * @onchip_buffers: number of available on-chip buffers.
+ * @onchip_used_size: actual size of on-chip memory assigned to endpoints.
+ * @pending_status_wq: workqueue handling status stage for deferred requests.
+ * @pending_status_request: request for which status stage was deferred
+ */
+struct cdns3_device {
+	struct device			*dev;
+	struct device			*sysdev;
+
+	struct usb_gadget		gadget;
+	struct usb_gadget_driver	*gadget_driver;
+
+#define CDNS_REVISION_V0		0x00024501
+#define CDNS_REVISION_V1		0x00024509
+	u32				dev_ver;
+
+	/* generic spin-lock for drivers */
+	spinlock_t			lock;
+
+	struct cdns3_usb_regs		__iomem *regs;
+
+	struct usb_ctrlrequest		*setup_buf;
+	dma_addr_t			setup_dma;
+	void				*zlp_buf;
+
+	u8				ep0_stage;
+	int				ep0_data_dir;
+
+	struct cdns3_endpoint		*eps[CDNS3_ENDPOINTS_MAX_COUNT];
+
+	struct list_head		aligned_buf_list;
+	struct work_struct		aligned_buf_wq;
+
+	u32				selected_ep;
+	u16				isoch_delay;
+
+	unsigned			wait_for_setup:1;
+	unsigned			u1_allowed:1;
+	unsigned			u2_allowed:1;
+	unsigned			is_selfpowered:1;
+	unsigned			setup_pending:1;
+	int				hw_configured_flag:1;
+	int				wake_up_flag:1;
+	unsigned			status_completion_no_call:1;
+	int				out_mem_is_allocated;
+
+	struct work_struct		pending_status_wq;
+	struct usb_request		*pending_status_request;
+
+	/*in KB */
+	u16				onchip_buffers;
+	u16				onchip_used_size;
+};
+
+void cdns3_set_register_bit(void __iomem *ptr, u32 mask);
+dma_addr_t cdns3_trb_virt_to_dma(struct cdns3_endpoint *priv_ep,
+				 struct cdns3_trb *trb);
+enum usb_device_speed cdns3_get_speed(struct cdns3_device *priv_dev);
+void cdns3_pending_setup_status_handler(struct work_struct *work);
+void cdns3_hw_reset_eps_config(struct cdns3_device *priv_dev);
+void cdns3_set_hw_configuration(struct cdns3_device *priv_dev);
+void cdns3_select_ep(struct cdns3_device *priv_dev, u32 ep);
+void cdns3_allow_enable_l1(struct cdns3_device *priv_dev, int enable);
+struct usb_request *cdns3_next_request(struct list_head *list);
+int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
+			  struct usb_request *request);
+void cdns3_rearm_transfer(struct cdns3_endpoint *priv_ep, u8 rearm);
+int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep);
+u8 cdns3_ep_addr_to_index(u8 ep_addr);
+int cdns3_gadget_ep_set_wedge(struct usb_ep *ep);
+int cdns3_gadget_ep_set_halt(struct usb_ep *ep, int value);
+void __cdns3_gadget_ep_set_halt(struct cdns3_endpoint *priv_ep);
+int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep);
+struct usb_request *cdns3_gadget_ep_alloc_request(struct usb_ep *ep,
+						  gfp_t gfp_flags);
+void cdns3_gadget_ep_free_request(struct usb_ep *ep,
+				  struct usb_request *request);
+int cdns3_gadget_ep_dequeue(struct usb_ep *ep, struct usb_request *request);
+void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep,
+			   struct cdns3_request *priv_req,
+			   int status);
+
+int cdns3_init_ep0(struct cdns3_device *priv_dev,
+		   struct cdns3_endpoint *priv_ep);
+void cdns3_ep0_config(struct cdns3_device *priv_dev);
+void cdns3_ep_config(struct cdns3_endpoint *priv_ep);
+void cdns3_check_ep0_interrupt_proceed(struct cdns3_device *priv_dev, int dir);
+int __cdns3_gadget_wakeup(struct cdns3_device *priv_dev);
+
+#endif /* __LINUX_CDNS3_GADGET */
diff -Naur --no-dereference a/drivers/usb/cdns3/host.c b/drivers/usb/cdns3/host.c
--- a/drivers/usb/cdns3/host.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/usb/cdns3/host.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Cadence USBSS DRD Driver - host side
+ *
+ * Copyright (C) 2018-2019 Cadence Design Systems.
+ * Copyright (C) 2017-2018 NXP
+ *
+ * Authors: Peter Chen <peter.chen@nxp.com>
+ *          Pawel Laszczak <pawell@cadence.com>
+ */
+
+#include <linux/platform_device.h>
+#include "core.h"
+#include "drd.h"
+#include "host-export.h"
+
+static int __cdns3_host_init(struct cdns3 *cdns)
+{
+	struct platform_device *xhci;
+	int ret;
+
+	cdns3_drd_switch_host(cdns, 1);
+
+	xhci = platform_device_alloc("xhci-hcd", PLATFORM_DEVID_AUTO);
+	if (!xhci) {
+		dev_err(cdns->dev, "couldn't allocate xHCI device\n");
+		return -ENOMEM;
+	}
+
+	xhci->dev.parent = cdns->dev;
+	cdns->host_dev = xhci;
+
+	ret = platform_device_add_resources(xhci, cdns->xhci_res,
+					    CDNS3_XHCI_RESOURCES_NUM);
+	if (ret) {
+		dev_err(cdns->dev, "couldn't add resources to xHCI device\n");
+		goto err1;
+	}
+
+	ret = platform_device_add(xhci);
+	if (ret) {
+		dev_err(cdns->dev, "failed to register xHCI device\n");
+		goto err1;
+	}
+
+	return 0;
+err1:
+	platform_device_put(xhci);
+	return ret;
+}
+
+static void cdns3_host_exit(struct cdns3 *cdns)
+{
+	platform_device_unregister(cdns->host_dev);
+	cdns->host_dev = NULL;
+	cdns3_drd_switch_host(cdns, 0);
+}
+
+int cdns3_host_init(struct cdns3 *cdns)
+{
+	struct cdns3_role_driver *rdrv;
+
+	rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
+	if (!rdrv)
+		return -ENOMEM;
+
+	rdrv->start	= __cdns3_host_init;
+	rdrv->stop	= cdns3_host_exit;
+	rdrv->state	= CDNS3_ROLE_STATE_INACTIVE;
+	rdrv->name	= "host";
+
+	cdns->roles[USB_ROLE_HOST] = rdrv;
+
+	return 0;
+}
diff -Naur --no-dereference a/drivers/usb/cdns3/host-export.h b/drivers/usb/cdns3/host-export.h
--- a/drivers/usb/cdns3/host-export.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/usb/cdns3/host-export.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Cadence USBSS DRD Driver - Host Export APIs
+ *
+ * Copyright (C) 2017-2018 NXP
+ *
+ * Authors: Peter Chen <peter.chen@nxp.com>
+ */
+#ifndef __LINUX_CDNS3_HOST_EXPORT
+#define __LINUX_CDNS3_HOST_EXPORT
+
+#ifdef CONFIG_USB_CDNS3_HOST
+
+int cdns3_host_init(struct cdns3 *cdns);
+
+#else
+
+static inline int cdns3_host_init(struct cdns3 *cdns)
+{
+	return -ENXIO;
+}
+
+static inline void cdns3_host_exit(struct cdns3 *cdns) { }
+
+#endif /* CONFIG_USB_CDNS3_HOST */
+
+#endif /* __LINUX_CDNS3_HOST_EXPORT */
diff -Naur --no-dereference a/drivers/usb/cdns3/Kconfig b/drivers/usb/cdns3/Kconfig
--- a/drivers/usb/cdns3/Kconfig	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/usb/cdns3/Kconfig	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,56 @@
+config USB_CDNS3
+	tristate "Cadence USB3 Dual-Role Controller"
+	depends on USB_SUPPORT && (USB || USB_GADGET) && HAS_DMA
+	select USB_XHCI_PLATFORM if USB_XHCI_HCD
+	select USB_ROLE_SWITCH
+	help
+	  Say Y here if your system has a Cadence USB3 dual-role controller.
+	  It supports: dual-role switch, Host-only, and Peripheral-only.
+
+	  If you choose to build this driver is a dynamically linked
+	  as module, the module will be called cdns3.ko.
+
+if USB_CDNS3
+
+config USB_CDNS3_GADGET
+	bool "Cadence USB3 device controller"
+	depends on USB_GADGET=y || USB_GADGET=USB_CDNS3
+	help
+	  Say Y here to enable device controller functionality of the
+	  Cadence USBSS-DEV driver.
+
+	  This controller supports FF, HS and SS mode. It doesn't support
+	  LS and SSP mode.
+
+config USB_CDNS3_HOST
+	bool "Cadence USB3 host controller"
+	depends on USB=y || USB=USB_CDNS3
+	help
+	  Say Y here to enable host controller functionality of the
+	  Cadence driver.
+
+	  Host controller is compliant with XHCI so it will use
+	  standard XHCI driver.
+
+config USB_CDNS3_PCI_WRAP
+	tristate "Cadence USB3 support on PCIe-based platforms"
+	depends on USB_PCI && ACPI
+	default USB_CDNS3
+	help
+	  If you're using the USBSS Core IP with a PCIe, please say
+	  'Y' or 'M' here.
+
+	  If you choose to build this driver as module it will
+	  be dynamically linked and module will be called cdns3-pci.ko
+
+config USB_CDNS3_TI
+	tristate "Cadence USB3 support on TI platforms"
+	depends on ARCH_K3 || COMPILE_TEST
+	default USB_CDNS3
+	help
+	  Say 'Y' or 'M' here if you are building for Texas Instruments
+	  platforms that contain Cadence USB3 controller core.
+
+	  e.g. J721e.
+
+endif
diff -Naur --no-dereference a/drivers/usb/cdns3/Makefile b/drivers/usb/cdns3/Makefile
--- a/drivers/usb/cdns3/Makefile	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/usb/cdns3/Makefile	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: GPL-2.0
+# define_trace.h needs to know how to find our header
+CFLAGS_trace.o				:= -I$(src)
+
+cdns3-y					:= core.o drd.o
+
+obj-$(CONFIG_USB_CDNS3)			+= cdns3.o
+cdns3-$(CONFIG_USB_CDNS3_GADGET)	+= gadget.o ep0.o
+
+ifneq ($(CONFIG_USB_CDNS3_GADGET),)
+cdns3-$(CONFIG_TRACING)			+= trace.o
+endif
+
+cdns3-$(CONFIG_USB_CDNS3_HOST)		+= host.o
+
+obj-$(CONFIG_USB_CDNS3_PCI_WRAP)	+= cdns3-pci-wrap.o
+obj-$(CONFIG_USB_CDNS3_TI)		+= cdns3-ti.o
diff -Naur --no-dereference a/drivers/usb/cdns3/trace.c b/drivers/usb/cdns3/trace.c
--- a/drivers/usb/cdns3/trace.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/usb/cdns3/trace.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,11 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * USBSS device controller driver Trace Support
+ *
+ * Copyright (C) 2018-2019 Cadence.
+ *
+ * Author: Pawel Laszczak <pawell@cadence.com>
+ */
+
+#define CREATE_TRACE_POINTS
+#include "trace.h"
diff -Naur --no-dereference a/drivers/usb/cdns3/trace.h b/drivers/usb/cdns3/trace.h
--- a/drivers/usb/cdns3/trace.h	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/usb/cdns3/trace.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,493 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * USBSS device controller driver.
+ * Trace support header file.
+ *
+ * Copyright (C) 2018-2019 Cadence.
+ *
+ * Author: Pawel Laszczak <pawell@cadence.com>
+ */
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM cdns3
+
+#if !defined(__LINUX_CDNS3_TRACE) || defined(TRACE_HEADER_MULTI_READ)
+#define __LINUX_CDNS3_TRACE
+
+#include <linux/types.h>
+#include <linux/tracepoint.h>
+#include <asm/byteorder.h>
+#include <linux/usb/ch9.h>
+#include "core.h"
+#include "gadget.h"
+#include "debug.h"
+
+#define CDNS3_MSG_MAX	500
+
+TRACE_EVENT(cdns3_halt,
+	TP_PROTO(struct cdns3_endpoint *ep_priv, u8 halt, u8 flush),
+	TP_ARGS(ep_priv, halt, flush),
+	TP_STRUCT__entry(
+		__string(name, ep_priv->name)
+		__field(u8, halt)
+		__field(u8, flush)
+	),
+	TP_fast_assign(
+		__assign_str(name, ep_priv->name);
+		__entry->halt = halt;
+		__entry->flush = flush;
+	),
+	TP_printk("Halt %s for %s: %s", __entry->flush ? " and flush" : "",
+		  __get_str(name), __entry->halt ? "set" : "cleared")
+);
+
+TRACE_EVENT(cdns3_wa1,
+	TP_PROTO(struct cdns3_endpoint *ep_priv, char *msg),
+	TP_ARGS(ep_priv, msg),
+	TP_STRUCT__entry(
+		__string(ep_name, ep_priv->name)
+		__string(msg, msg)
+	),
+	TP_fast_assign(
+		__assign_str(ep_name, ep_priv->name);
+		__assign_str(msg, msg);
+	),
+	TP_printk("WA1: %s %s", __get_str(ep_name), __get_str(msg))
+);
+
+TRACE_EVENT(cdns3_wa2,
+	TP_PROTO(struct cdns3_endpoint *ep_priv, char *msg),
+	TP_ARGS(ep_priv, msg),
+	TP_STRUCT__entry(
+		__string(ep_name, ep_priv->name)
+		__string(msg, msg)
+	),
+	TP_fast_assign(
+		__assign_str(ep_name, ep_priv->name);
+		__assign_str(msg, msg);
+	),
+	TP_printk("WA2: %s %s", __get_str(ep_name), __get_str(msg))
+);
+
+DECLARE_EVENT_CLASS(cdns3_log_doorbell,
+	TP_PROTO(const char *ep_name, u32 ep_trbaddr),
+	TP_ARGS(ep_name, ep_trbaddr),
+	TP_STRUCT__entry(
+		__string(name, ep_name)
+		__field(u32, ep_trbaddr)
+	),
+	TP_fast_assign(
+		__assign_str(name, ep_name);
+		__entry->ep_trbaddr = ep_trbaddr;
+	),
+	TP_printk("%s, ep_trbaddr %08x", __get_str(name),
+		  __entry->ep_trbaddr)
+);
+
+DEFINE_EVENT(cdns3_log_doorbell, cdns3_doorbell_ep0,
+	TP_PROTO(const char *ep_name, u32 ep_trbaddr),
+	TP_ARGS(ep_name, ep_trbaddr)
+);
+
+DEFINE_EVENT(cdns3_log_doorbell, cdns3_doorbell_epx,
+	TP_PROTO(const char *ep_name, u32 ep_trbaddr),
+	TP_ARGS(ep_name, ep_trbaddr)
+);
+
+DECLARE_EVENT_CLASS(cdns3_log_usb_irq,
+	TP_PROTO(struct cdns3_device *priv_dev, u32 usb_ists),
+	TP_ARGS(priv_dev, usb_ists),
+	TP_STRUCT__entry(
+		__field(enum usb_device_speed, speed)
+		__field(u32, usb_ists)
+		__dynamic_array(char, str, CDNS3_MSG_MAX)
+	),
+	TP_fast_assign(
+		__entry->speed = cdns3_get_speed(priv_dev);
+		__entry->usb_ists = usb_ists;
+	),
+	TP_printk("%s", cdns3_decode_usb_irq(__get_str(str), __entry->speed,
+					     __entry->usb_ists))
+);
+
+DEFINE_EVENT(cdns3_log_usb_irq, cdns3_usb_irq,
+	TP_PROTO(struct cdns3_device *priv_dev, u32 usb_ists),
+	TP_ARGS(priv_dev, usb_ists)
+);
+
+DECLARE_EVENT_CLASS(cdns3_log_epx_irq,
+	TP_PROTO(struct cdns3_device *priv_dev, struct cdns3_endpoint *priv_ep),
+	TP_ARGS(priv_dev, priv_ep),
+	TP_STRUCT__entry(
+		__string(ep_name, priv_ep->name)
+		__field(u32, ep_sts)
+		__field(u32, ep_traddr)
+		__dynamic_array(char, str, CDNS3_MSG_MAX)
+	),
+	TP_fast_assign(
+		__assign_str(ep_name, priv_ep->name);
+		__entry->ep_sts = readl(&priv_dev->regs->ep_sts);
+		__entry->ep_traddr = readl(&priv_dev->regs->ep_traddr);
+	),
+	TP_printk("%s, ep_traddr: %08x",
+		  cdns3_decode_epx_irq(__get_str(str),
+				       __get_str(ep_name),
+				       __entry->ep_sts),
+		  __entry->ep_traddr)
+);
+
+DEFINE_EVENT(cdns3_log_epx_irq, cdns3_epx_irq,
+	TP_PROTO(struct cdns3_device *priv_dev, struct cdns3_endpoint *priv_ep),
+	TP_ARGS(priv_dev, priv_ep)
+);
+
+DECLARE_EVENT_CLASS(cdns3_log_ep0_irq,
+	TP_PROTO(struct cdns3_device *priv_dev,  u32 ep_sts),
+	TP_ARGS(priv_dev, ep_sts),
+	TP_STRUCT__entry(
+		__field(int, ep_dir)
+		__field(u32, ep_sts)
+		__dynamic_array(char, str, CDNS3_MSG_MAX)
+	),
+	TP_fast_assign(
+		__entry->ep_dir = priv_dev->ep0_data_dir;
+		__entry->ep_sts = ep_sts;
+	),
+	TP_printk("%s", cdns3_decode_ep0_irq(__get_str(str),
+					     __entry->ep_dir,
+					     __entry->ep_sts))
+);
+
+DEFINE_EVENT(cdns3_log_ep0_irq, cdns3_ep0_irq,
+	TP_PROTO(struct cdns3_device *priv_dev, u32 ep_sts),
+	TP_ARGS(priv_dev, ep_sts)
+);
+
+DECLARE_EVENT_CLASS(cdns3_log_ctrl,
+	TP_PROTO(struct usb_ctrlrequest *ctrl),
+	TP_ARGS(ctrl),
+	TP_STRUCT__entry(
+		__field(u8, bRequestType)
+		__field(u8, bRequest)
+		__field(u16, wValue)
+		__field(u16, wIndex)
+		__field(u16, wLength)
+		__dynamic_array(char, str, CDNS3_MSG_MAX)
+	),
+	TP_fast_assign(
+		__entry->bRequestType = ctrl->bRequestType;
+		__entry->bRequest = ctrl->bRequest;
+		__entry->wValue = le16_to_cpu(ctrl->wValue);
+		__entry->wIndex = le16_to_cpu(ctrl->wIndex);
+		__entry->wLength = le16_to_cpu(ctrl->wLength);
+	),
+	TP_printk("%s", usb_decode_ctrl(__get_str(str), CDNS3_MSG_MAX,
+					__entry->bRequestType,
+					__entry->bRequest, __entry->wValue,
+					__entry->wIndex, __entry->wLength)
+	)
+);
+
+DEFINE_EVENT(cdns3_log_ctrl, cdns3_ctrl_req,
+	TP_PROTO(struct usb_ctrlrequest *ctrl),
+	TP_ARGS(ctrl)
+);
+
+DECLARE_EVENT_CLASS(cdns3_log_request,
+	TP_PROTO(struct cdns3_request *req),
+	TP_ARGS(req),
+	TP_STRUCT__entry(
+		__string(name, req->priv_ep->name)
+		__field(struct cdns3_request *, req)
+		__field(void *, buf)
+		__field(unsigned int, actual)
+		__field(unsigned int, length)
+		__field(int, status)
+		__field(int, zero)
+		__field(int, short_not_ok)
+		__field(int, no_interrupt)
+		__field(int, start_trb)
+		__field(int, end_trb)
+		__field(struct cdns3_trb *, start_trb_addr)
+		__field(int, flags)
+	),
+	TP_fast_assign(
+		__assign_str(name, req->priv_ep->name);
+		__entry->req = req;
+		__entry->buf = req->request.buf;
+		__entry->actual = req->request.actual;
+		__entry->length = req->request.length;
+		__entry->status = req->request.status;
+		__entry->zero = req->request.zero;
+		__entry->short_not_ok = req->request.short_not_ok;
+		__entry->no_interrupt = req->request.no_interrupt;
+		__entry->start_trb = req->start_trb;
+		__entry->end_trb = req->end_trb;
+		__entry->start_trb_addr = req->trb;
+		__entry->flags = req->flags;
+	),
+	TP_printk("%s: req: %p, req buff %p, length: %u/%u %s%s%s, status: %d,"
+		  " trb: [start:%d, end:%d: virt addr %pa], flags:%x ",
+		__get_str(name), __entry->req, __entry->buf, __entry->actual,
+		__entry->length,
+		__entry->zero ? "Z" : "z",
+		__entry->short_not_ok ? "S" : "s",
+		__entry->no_interrupt ? "I" : "i",
+		__entry->status,
+		__entry->start_trb,
+		__entry->end_trb,
+		__entry->start_trb_addr,
+		__entry->flags
+	)
+);
+
+DEFINE_EVENT(cdns3_log_request, cdns3_alloc_request,
+	TP_PROTO(struct cdns3_request *req),
+	TP_ARGS(req)
+);
+
+DEFINE_EVENT(cdns3_log_request, cdns3_free_request,
+	TP_PROTO(struct cdns3_request *req),
+	TP_ARGS(req)
+);
+
+DEFINE_EVENT(cdns3_log_request, cdns3_ep_queue,
+	TP_PROTO(struct cdns3_request *req),
+	TP_ARGS(req)
+);
+
+DEFINE_EVENT(cdns3_log_request, cdns3_ep_dequeue,
+	TP_PROTO(struct cdns3_request *req),
+	TP_ARGS(req)
+);
+
+DEFINE_EVENT(cdns3_log_request, cdns3_gadget_giveback,
+	TP_PROTO(struct cdns3_request *req),
+	TP_ARGS(req)
+);
+
+TRACE_EVENT(cdns3_ep0_queue,
+	TP_PROTO(struct cdns3_device *dev_priv, struct usb_request *request),
+	TP_ARGS(dev_priv, request),
+	TP_STRUCT__entry(
+		__field(int, dir)
+		__field(int, length)
+	),
+	TP_fast_assign(
+		__entry->dir = dev_priv->ep0_data_dir;
+		__entry->length = request->length;
+	),
+	TP_printk("Queue to ep0%s length: %u", __entry->dir ? "in" : "out",
+		  __entry->length)
+);
+
+DECLARE_EVENT_CLASS(cdns3_log_aligned_request,
+	TP_PROTO(struct cdns3_request *priv_req),
+	TP_ARGS(priv_req),
+	TP_STRUCT__entry(
+		__string(name, priv_req->priv_ep->name)
+		__field(struct usb_request *, req)
+		__field(void *, buf)
+		__field(dma_addr_t, dma)
+		__field(void *, aligned_buf)
+		__field(dma_addr_t, aligned_dma)
+		__field(u32, aligned_buf_size)
+	),
+	TP_fast_assign(
+		__assign_str(name, priv_req->priv_ep->name);
+		__entry->req = &priv_req->request;
+		__entry->buf = priv_req->request.buf;
+		__entry->dma = priv_req->request.dma;
+		__entry->aligned_buf = priv_req->aligned_buf->buf;
+		__entry->aligned_dma = priv_req->aligned_buf->dma;
+		__entry->aligned_buf_size = priv_req->aligned_buf->size;
+	),
+	TP_printk("%s: req: %p, req buf %p, dma %pad a_buf %p a_dma %pad, size %d",
+		__get_str(name), __entry->req, __entry->buf, &__entry->dma,
+		__entry->aligned_buf, &__entry->aligned_dma,
+		__entry->aligned_buf_size
+	)
+);
+
+DEFINE_EVENT(cdns3_log_aligned_request, cdns3_free_aligned_request,
+	TP_PROTO(struct cdns3_request *req),
+	TP_ARGS(req)
+);
+
+DEFINE_EVENT(cdns3_log_aligned_request, cdns3_prepare_aligned_request,
+	TP_PROTO(struct cdns3_request *req),
+	TP_ARGS(req)
+);
+
+DECLARE_EVENT_CLASS(cdns3_log_trb,
+	TP_PROTO(struct cdns3_endpoint *priv_ep, struct cdns3_trb *trb),
+	TP_ARGS(priv_ep, trb),
+	TP_STRUCT__entry(
+		__string(name, priv_ep->name)
+		__field(struct cdns3_trb *, trb)
+		__field(u32, buffer)
+		__field(u32, length)
+		__field(u32, control)
+		__field(u32, type)
+	),
+	TP_fast_assign(
+		__assign_str(name, priv_ep->name);
+		__entry->trb = trb;
+		__entry->buffer = trb->buffer;
+		__entry->length = trb->length;
+		__entry->control = trb->control;
+		__entry->type = usb_endpoint_type(priv_ep->endpoint.desc);
+	),
+	TP_printk("%s: trb %p, dma buf: 0x%08x, size: %ld, burst: %d ctrl: 0x%08x (%s%s%s%s%s%s%s)",
+		__get_str(name), __entry->trb, __entry->buffer,
+		TRB_LEN(__entry->length),
+		(u8)TRB_BURST_LEN_GET(__entry->length),
+		__entry->control,
+		__entry->control & TRB_CYCLE ? "C=1, " : "C=0, ",
+		__entry->control & TRB_TOGGLE ? "T=1, " : "T=0, ",
+		__entry->control & TRB_ISP ? "ISP, " : "",
+		__entry->control & TRB_FIFO_MODE ? "FIFO, " : "",
+		__entry->control & TRB_CHAIN ? "CHAIN, " : "",
+		__entry->control & TRB_IOC ? "IOC, " : "",
+		TRB_FIELD_TO_TYPE(__entry->control) == TRB_NORMAL ? "Normal" : "LINK"
+	)
+);
+
+DEFINE_EVENT(cdns3_log_trb, cdns3_prepare_trb,
+	TP_PROTO(struct cdns3_endpoint *priv_ep, struct cdns3_trb *trb),
+	TP_ARGS(priv_ep, trb)
+);
+
+DEFINE_EVENT(cdns3_log_trb, cdns3_complete_trb,
+	TP_PROTO(struct cdns3_endpoint *priv_ep, struct cdns3_trb *trb),
+	TP_ARGS(priv_ep, trb)
+);
+
+DECLARE_EVENT_CLASS(cdns3_log_ring,
+	TP_PROTO(struct cdns3_endpoint *priv_ep),
+	TP_ARGS(priv_ep),
+	TP_STRUCT__entry(
+		__dynamic_array(u8, ring, TRB_RING_SIZE)
+		__dynamic_array(u8, priv_ep, sizeof(struct cdns3_endpoint))
+		__dynamic_array(char, buffer,
+				(TRBS_PER_SEGMENT * 65) + CDNS3_MSG_MAX)
+	),
+	TP_fast_assign(
+		memcpy(__get_dynamic_array(priv_ep), priv_ep,
+		       sizeof(struct cdns3_endpoint));
+		memcpy(__get_dynamic_array(ring), priv_ep->trb_pool,
+		       TRB_RING_SIZE);
+	),
+
+	TP_printk("%s",
+		  cdns3_dbg_ring((struct cdns3_endpoint *)__get_str(priv_ep),
+				 (struct cdns3_trb *)__get_str(ring),
+				 __get_str(buffer)))
+);
+
+DEFINE_EVENT(cdns3_log_ring, cdns3_ring,
+	TP_PROTO(struct cdns3_endpoint *priv_ep),
+	TP_ARGS(priv_ep)
+);
+
+DECLARE_EVENT_CLASS(cdns3_log_ep,
+	TP_PROTO(struct cdns3_endpoint *priv_ep),
+	TP_ARGS(priv_ep),
+	TP_STRUCT__entry(
+		__string(name, priv_ep->name)
+		__field(unsigned int, maxpacket)
+		__field(unsigned int, maxpacket_limit)
+		__field(unsigned int, max_streams)
+		__field(unsigned int, maxburst)
+		__field(unsigned int, flags)
+		__field(unsigned int, dir)
+		__field(u8, enqueue)
+		__field(u8, dequeue)
+	),
+	TP_fast_assign(
+		__assign_str(name, priv_ep->name);
+		__entry->maxpacket = priv_ep->endpoint.maxpacket;
+		__entry->maxpacket_limit = priv_ep->endpoint.maxpacket_limit;
+		__entry->max_streams = priv_ep->endpoint.max_streams;
+		__entry->maxburst = priv_ep->endpoint.maxburst;
+		__entry->flags = priv_ep->flags;
+		__entry->dir = priv_ep->dir;
+		__entry->enqueue = priv_ep->enqueue;
+		__entry->dequeue = priv_ep->dequeue;
+	),
+	TP_printk("%s: mps: %d/%d. streams: %d, burst: %d, enq idx: %d, "
+		  "deq idx: %d, flags %s%s%s%s%s%s%s%s, dir: %s",
+		__get_str(name), __entry->maxpacket,
+		__entry->maxpacket_limit, __entry->max_streams,
+		__entry->maxburst, __entry->enqueue,
+		__entry->dequeue,
+		__entry->flags & EP_ENABLED ? "EN | " : "",
+		__entry->flags & EP_STALLED ? "STALLED | " : "",
+		__entry->flags & EP_WEDGE ? "WEDGE | " : "",
+		__entry->flags & EP_TRANSFER_STARTED ? "STARTED | " : "",
+		__entry->flags & EP_UPDATE_EP_TRBADDR ? "UPD TRB | " : "",
+		__entry->flags & EP_PENDING_REQUEST ? "REQ PEN | " : "",
+		__entry->flags & EP_RING_FULL ? "RING FULL |" : "",
+		__entry->flags & EP_CLAIMED ?  "CLAIMED " : "",
+		__entry->dir ? "IN" : "OUT"
+	)
+);
+
+DEFINE_EVENT(cdns3_log_ep, cdns3_gadget_ep_enable,
+	TP_PROTO(struct cdns3_endpoint *priv_ep),
+	TP_ARGS(priv_ep)
+);
+
+DEFINE_EVENT(cdns3_log_ep, cdns3_gadget_ep_disable,
+	TP_PROTO(struct cdns3_endpoint *priv_ep),
+	TP_ARGS(priv_ep)
+);
+
+DECLARE_EVENT_CLASS(cdns3_log_request_handled,
+	TP_PROTO(struct cdns3_request *priv_req, int current_index,
+		 int handled),
+	TP_ARGS(priv_req, current_index, handled),
+	TP_STRUCT__entry(
+		__field(struct cdns3_request *, priv_req)
+		__field(unsigned int, dma_position)
+		__field(unsigned int, handled)
+		__field(unsigned int, dequeue_idx)
+		__field(unsigned int, enqueue_idx)
+		__field(unsigned int, start_trb)
+		__field(unsigned int, end_trb)
+	),
+	TP_fast_assign(
+		__entry->priv_req = priv_req;
+		__entry->dma_position = current_index;
+		__entry->handled = handled;
+		__entry->dequeue_idx = priv_req->priv_ep->dequeue;
+		__entry->enqueue_idx = priv_req->priv_ep->enqueue;
+		__entry->start_trb = priv_req->start_trb;
+		__entry->end_trb = priv_req->end_trb;
+	),
+	TP_printk("Req: %p %s, DMA pos: %d, ep deq: %d, ep enq: %d,"
+		  " start trb: %d, end trb: %d",
+		__entry->priv_req,
+		__entry->handled ? "handled" : "not handled",
+		__entry->dma_position, __entry->dequeue_idx,
+		__entry->enqueue_idx, __entry->start_trb,
+		__entry->end_trb
+	)
+);
+
+DEFINE_EVENT(cdns3_log_request_handled, cdns3_request_handled,
+	TP_PROTO(struct cdns3_request *priv_req, int current_index,
+		 int handled),
+	TP_ARGS(priv_req, current_index, handled)
+);
+#endif /* __LINUX_CDNS3_TRACE */
+
+/* this part must be outside header guard */
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE trace
+
+#include <trace/define_trace.h>
diff -Naur --no-dereference a/drivers/usb/common/debug.c b/drivers/usb/common/debug.c
--- a/drivers/usb/common/debug.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/usb/common/debug.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,268 @@
+// SPDX-License-Identifier: GPL-2.0
+/**
+ * Common USB debugging functions
+ *
+ * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com
+ *
+ * Authors: Felipe Balbi <balbi@ti.com>,
+ *	    Sebastian Andrzej Siewior <bigeasy@linutronix.de>
+ */
+
+#include <linux/usb/ch9.h>
+
+static void usb_decode_get_status(__u8 bRequestType, __u16 wIndex,
+				  __u16 wLength, char *str, size_t size)
+{
+	switch (bRequestType & USB_RECIP_MASK) {
+	case USB_RECIP_DEVICE:
+		snprintf(str, size, "Get Device Status(Length = %d)", wLength);
+		break;
+	case USB_RECIP_INTERFACE:
+		snprintf(str, size,
+			 "Get Interface Status(Intf = %d, Length = %d)",
+			 wIndex, wLength);
+		break;
+	case USB_RECIP_ENDPOINT:
+		snprintf(str, size, "Get Endpoint Status(ep%d%s)",
+			 wIndex & ~USB_DIR_IN,
+			 wIndex & USB_DIR_IN ? "in" : "out");
+		break;
+	}
+}
+
+static const char *usb_decode_device_feature(u16 wValue)
+{
+	switch (wValue) {
+	case USB_DEVICE_SELF_POWERED:
+		return "Self Powered";
+	case USB_DEVICE_REMOTE_WAKEUP:
+		return "Remote Wakeup";
+	case USB_DEVICE_TEST_MODE:
+		return "Test Mode";
+	case USB_DEVICE_U1_ENABLE:
+		return "U1 Enable";
+	case USB_DEVICE_U2_ENABLE:
+		return "U2 Enable";
+	case USB_DEVICE_LTM_ENABLE:
+		return "LTM Enable";
+	default:
+		return "UNKNOWN";
+	}
+}
+
+static const char *usb_decode_test_mode(u16 wIndex)
+{
+	switch (wIndex) {
+	case TEST_J:
+		return ": TEST_J";
+	case TEST_K:
+		return ": TEST_K";
+	case TEST_SE0_NAK:
+		return ": TEST_SE0_NAK";
+	case TEST_PACKET:
+		return ": TEST_PACKET";
+	case TEST_FORCE_EN:
+		return ": TEST_FORCE_EN";
+	default:
+		return ": UNKNOWN";
+	}
+}
+
+static void usb_decode_set_clear_feature(__u8 bRequestType,
+					 __u8 bRequest, __u16 wValue,
+					 __u16 wIndex, char *str, size_t size)
+{
+	switch (bRequestType & USB_RECIP_MASK) {
+	case USB_RECIP_DEVICE:
+		snprintf(str, size, "%s Device Feature(%s%s)",
+			 bRequest == USB_REQ_CLEAR_FEATURE ? "Clear" : "Set",
+			 usb_decode_device_feature(wValue),
+			 wValue == USB_DEVICE_TEST_MODE ?
+			 usb_decode_test_mode(wIndex) : "");
+		break;
+	case USB_RECIP_INTERFACE:
+		snprintf(str, size, "%s Interface Feature(%s)",
+			 bRequest == USB_REQ_CLEAR_FEATURE ? "Clear" : "Set",
+			 wValue == USB_INTRF_FUNC_SUSPEND ?
+			 "Function Suspend" : "UNKNOWN");
+		break;
+	case USB_RECIP_ENDPOINT:
+		snprintf(str, size, "%s Endpoint Feature(%s ep%d%s)",
+			 bRequest == USB_REQ_CLEAR_FEATURE ? "Clear" : "Set",
+			 wValue == USB_ENDPOINT_HALT ? "Halt" : "UNKNOWN",
+			 wIndex & ~USB_DIR_IN,
+			 wIndex & USB_DIR_IN ? "in" : "out");
+		break;
+	}
+}
+
+static void usb_decode_set_address(__u16 wValue, char *str, size_t size)
+{
+	snprintf(str, size, "Set Address(Addr = %02x)", wValue);
+}
+
+static void usb_decode_get_set_descriptor(__u8 bRequestType, __u8 bRequest,
+					  __u16 wValue, __u16 wIndex,
+					  __u16 wLength, char *str, size_t size)
+{
+	char *s;
+
+	switch (wValue >> 8) {
+	case USB_DT_DEVICE:
+		s = "Device";
+		break;
+	case USB_DT_CONFIG:
+		s = "Configuration";
+		break;
+	case USB_DT_STRING:
+		s = "String";
+		break;
+	case USB_DT_INTERFACE:
+		s = "Interface";
+		break;
+	case USB_DT_ENDPOINT:
+		s = "Endpoint";
+		break;
+	case USB_DT_DEVICE_QUALIFIER:
+		s = "Device Qualifier";
+		break;
+	case USB_DT_OTHER_SPEED_CONFIG:
+		s = "Other Speed Config";
+		break;
+	case USB_DT_INTERFACE_POWER:
+		s = "Interface Power";
+		break;
+	case USB_DT_OTG:
+		s = "OTG";
+		break;
+	case USB_DT_DEBUG:
+		s = "Debug";
+		break;
+	case USB_DT_INTERFACE_ASSOCIATION:
+		s = "Interface Association";
+		break;
+	case USB_DT_BOS:
+		s = "BOS";
+		break;
+	case USB_DT_DEVICE_CAPABILITY:
+		s = "Device Capability";
+		break;
+	case USB_DT_PIPE_USAGE:
+		s = "Pipe Usage";
+		break;
+	case USB_DT_SS_ENDPOINT_COMP:
+		s = "SS Endpoint Companion";
+		break;
+	case USB_DT_SSP_ISOC_ENDPOINT_COMP:
+		s = "SSP Isochronous Endpoint Companion";
+		break;
+	default:
+		s = "UNKNOWN";
+		break;
+	}
+
+	snprintf(str, size, "%s %s Descriptor(Index = %d, Length = %d)",
+		bRequest == USB_REQ_GET_DESCRIPTOR ? "Get" : "Set",
+		s, wValue & 0xff, wLength);
+}
+
+static void usb_decode_get_configuration(__u16 wLength, char *str, size_t size)
+{
+	snprintf(str, size, "Get Configuration(Length = %d)", wLength);
+}
+
+static void usb_decode_set_configuration(__u8 wValue, char *str, size_t size)
+{
+	snprintf(str, size, "Set Configuration(Config = %d)", wValue);
+}
+
+static void usb_decode_get_intf(__u16 wIndex, __u16 wLength, char *str,
+				size_t size)
+{
+	snprintf(str, size, "Get Interface(Intf = %d, Length = %d)",
+		 wIndex, wLength);
+}
+
+static void usb_decode_set_intf(__u8 wValue, __u16 wIndex, char *str,
+				size_t size)
+{
+	snprintf(str, size, "Set Interface(Intf = %d, Alt.Setting = %d)",
+		 wIndex, wValue);
+}
+
+static void usb_decode_synch_frame(__u16 wIndex, __u16 wLength,
+				   char *str, size_t size)
+{
+	snprintf(str, size, "Synch Frame(Endpoint = %d, Length = %d)",
+		 wIndex, wLength);
+}
+
+static void usb_decode_set_sel(__u16 wLength, char *str, size_t size)
+{
+	snprintf(str, size, "Set SEL(Length = %d)", wLength);
+}
+
+static void usb_decode_set_isoch_delay(__u8 wValue, char *str, size_t size)
+{
+	snprintf(str, size, "Set Isochronous Delay(Delay = %d ns)", wValue);
+}
+
+/**
+ * usb_decode_ctrl - returns a string representation of ctrl request
+ */
+const char *usb_decode_ctrl(char *str, size_t size, __u8 bRequestType,
+			    __u8 bRequest, __u16 wValue, __u16 wIndex,
+			    __u16 wLength)
+{
+	switch (bRequest) {
+	case USB_REQ_GET_STATUS:
+		usb_decode_get_status(bRequestType, wIndex, wLength, str, size);
+		break;
+	case USB_REQ_CLEAR_FEATURE:
+	case USB_REQ_SET_FEATURE:
+		usb_decode_set_clear_feature(bRequestType, bRequest, wValue,
+					     wIndex, str, size);
+		break;
+	case USB_REQ_SET_ADDRESS:
+		usb_decode_set_address(wValue, str, size);
+		break;
+	case USB_REQ_GET_DESCRIPTOR:
+	case USB_REQ_SET_DESCRIPTOR:
+		usb_decode_get_set_descriptor(bRequestType, bRequest, wValue,
+					      wIndex, wLength, str, size);
+		break;
+	case USB_REQ_GET_CONFIGURATION:
+		usb_decode_get_configuration(wLength, str, size);
+		break;
+	case USB_REQ_SET_CONFIGURATION:
+		usb_decode_set_configuration(wValue, str, size);
+		break;
+	case USB_REQ_GET_INTERFACE:
+		usb_decode_get_intf(wIndex, wLength, str, size);
+		break;
+	case USB_REQ_SET_INTERFACE:
+		usb_decode_set_intf(wValue, wIndex, str, size);
+		break;
+	case USB_REQ_SYNCH_FRAME:
+		usb_decode_synch_frame(wIndex, wLength, str, size);
+		break;
+	case USB_REQ_SET_SEL:
+		usb_decode_set_sel(wLength, str, size);
+		break;
+	case USB_REQ_SET_ISOCH_DELAY:
+		usb_decode_set_isoch_delay(wValue, str, size);
+		break;
+	default:
+		snprintf(str, size, "%02x %02x %02x %02x %02x %02x %02x %02x",
+			 bRequestType, bRequest,
+			 (u8)(cpu_to_le16(wValue) & 0xff),
+			 (u8)(cpu_to_le16(wValue) >> 8),
+			 (u8)(cpu_to_le16(wIndex) & 0xff),
+			 (u8)(cpu_to_le16(wIndex) >> 8),
+			 (u8)(cpu_to_le16(wLength) & 0xff),
+			 (u8)(cpu_to_le16(wLength) >> 8));
+	}
+
+	return str;
+}
+EXPORT_SYMBOL_GPL(usb_decode_ctrl);
diff -Naur --no-dereference a/drivers/usb/common/Makefile b/drivers/usb/common/Makefile
--- a/drivers/usb/common/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/usb/common/Makefile	2020-06-19 21:00:31.428227716 -0400
@@ -5,6 +5,7 @@
 
 obj-$(CONFIG_USB_COMMON)	  += usb-common.o
 usb-common-y			  += common.o
+usb-common-$(CONFIG_TRACING)	  += debug.o
 usb-common-$(CONFIG_USB_LED_TRIG) += led.o
 
 obj-$(CONFIG_USB_OTG_FSM) += usb-otg-fsm.o
diff -Naur --no-dereference a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h
--- a/drivers/usb/dwc3/core.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/usb/dwc3/core.h	2020-06-19 21:00:31.428227716 -0400
@@ -677,7 +677,6 @@
 #define DWC3_EP_WEDGE		BIT(2)
 #define DWC3_EP_TRANSFER_STARTED BIT(3)
 #define DWC3_EP_PENDING_REQUEST	BIT(5)
-#define DWC3_EP_END_TRANSFER_PENDING	BIT(7)
 
 	/* This last one is specific to EP0 */
 #define DWC3_EP0_DIR_IN		BIT(31)
diff -Naur --no-dereference a/drivers/usb/dwc3/debug.h b/drivers/usb/dwc3/debug.h
--- a/drivers/usb/dwc3/debug.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/usb/dwc3/debug.h	2020-06-19 21:00:31.428227716 -0400
@@ -193,294 +193,54 @@
  * dwc3_gadget_event_string - returns event name
  * @event: the event code
  */
-static inline const char *
-dwc3_gadget_event_string(char *str, const struct dwc3_event_devt *event)
+static inline const char *dwc3_gadget_event_string(char *str, size_t size,
+		const struct dwc3_event_devt *event)
 {
 	enum dwc3_link_state state = event->event_info & DWC3_LINK_STATE_MASK;
 
 	switch (event->type) {
 	case DWC3_DEVICE_EVENT_DISCONNECT:
-		sprintf(str, "Disconnect: [%s]",
+		snprintf(str, size, "Disconnect: [%s]",
 				dwc3_gadget_link_string(state));
 		break;
 	case DWC3_DEVICE_EVENT_RESET:
-		sprintf(str, "Reset [%s]", dwc3_gadget_link_string(state));
+		snprintf(str, size, "Reset [%s]",
+				dwc3_gadget_link_string(state));
 		break;
 	case DWC3_DEVICE_EVENT_CONNECT_DONE:
-		sprintf(str, "Connection Done [%s]",
+		snprintf(str, size, "Connection Done [%s]",
 				dwc3_gadget_link_string(state));
 		break;
 	case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
-		sprintf(str, "Link Change [%s]",
+		snprintf(str, size, "Link Change [%s]",
 				dwc3_gadget_link_string(state));
 		break;
 	case DWC3_DEVICE_EVENT_WAKEUP:
-		sprintf(str, "WakeUp [%s]", dwc3_gadget_link_string(state));
+		snprintf(str, size, "WakeUp [%s]",
+				dwc3_gadget_link_string(state));
 		break;
 	case DWC3_DEVICE_EVENT_EOPF:
-		sprintf(str, "End-Of-Frame [%s]",
+		snprintf(str, size, "End-Of-Frame [%s]",
 				dwc3_gadget_link_string(state));
 		break;
 	case DWC3_DEVICE_EVENT_SOF:
-		sprintf(str, "Start-Of-Frame [%s]",
+		snprintf(str, size, "Start-Of-Frame [%s]",
 				dwc3_gadget_link_string(state));
 		break;
 	case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
-		sprintf(str, "Erratic Error [%s]",
+		snprintf(str, size, "Erratic Error [%s]",
 				dwc3_gadget_link_string(state));
 		break;
 	case DWC3_DEVICE_EVENT_CMD_CMPL:
-		sprintf(str, "Command Complete [%s]",
+		snprintf(str, size, "Command Complete [%s]",
 				dwc3_gadget_link_string(state));
 		break;
 	case DWC3_DEVICE_EVENT_OVERFLOW:
-		sprintf(str, "Overflow [%s]", dwc3_gadget_link_string(state));
-		break;
-	default:
-		sprintf(str, "UNKNOWN");
-	}
-
-	return str;
-}
-
-static inline void dwc3_decode_get_status(__u8 t, __u16 i, __u16 l, char *str)
-{
-	switch (t & USB_RECIP_MASK) {
-	case USB_RECIP_INTERFACE:
-		sprintf(str, "Get Interface Status(Intf = %d, Length = %d)",
-			i, l);
-		break;
-	case USB_RECIP_ENDPOINT:
-		sprintf(str, "Get Endpoint Status(ep%d%s)",
-			i & ~USB_DIR_IN,
-			i & USB_DIR_IN ? "in" : "out");
-		break;
-	}
-}
-
-static inline void dwc3_decode_set_clear_feature(__u8 t, __u8 b, __u16 v,
-						 __u16 i, char *str)
-{
-	switch (t & USB_RECIP_MASK) {
-	case USB_RECIP_DEVICE:
-		sprintf(str, "%s Device Feature(%s%s)",
-			b == USB_REQ_CLEAR_FEATURE ? "Clear" : "Set",
-			({char *s;
-				switch (v) {
-				case USB_DEVICE_SELF_POWERED:
-					s = "Self Powered";
-					break;
-				case USB_DEVICE_REMOTE_WAKEUP:
-					s = "Remote Wakeup";
-					break;
-				case USB_DEVICE_TEST_MODE:
-					s = "Test Mode";
-					break;
-				case USB_DEVICE_U1_ENABLE:
-					s = "U1 Enable";
-					break;
-				case USB_DEVICE_U2_ENABLE:
-					s = "U2 Enable";
-					break;
-				case USB_DEVICE_LTM_ENABLE:
-					s = "LTM Enable";
-					break;
-				default:
-					s = "UNKNOWN";
-				} s; }),
-			v == USB_DEVICE_TEST_MODE ?
-			({ char *s;
-				switch (i) {
-				case TEST_J:
-					s = ": TEST_J";
-					break;
-				case TEST_K:
-					s = ": TEST_K";
-					break;
-				case TEST_SE0_NAK:
-					s = ": TEST_SE0_NAK";
-					break;
-				case TEST_PACKET:
-					s = ": TEST_PACKET";
-					break;
-				case TEST_FORCE_EN:
-					s = ": TEST_FORCE_EN";
-					break;
-				default:
-					s = ": UNKNOWN";
-				} s; }) : "");
-		break;
-	case USB_RECIP_INTERFACE:
-		sprintf(str, "%s Interface Feature(%s)",
-			b == USB_REQ_CLEAR_FEATURE ? "Clear" : "Set",
-			v == USB_INTRF_FUNC_SUSPEND ?
-			"Function Suspend" : "UNKNOWN");
-		break;
-	case USB_RECIP_ENDPOINT:
-		sprintf(str, "%s Endpoint Feature(%s ep%d%s)",
-			b == USB_REQ_CLEAR_FEATURE ? "Clear" : "Set",
-			v == USB_ENDPOINT_HALT ? "Halt" : "UNKNOWN",
-			i & ~USB_DIR_IN,
-			i & USB_DIR_IN ? "in" : "out");
-		break;
-	}
-}
-
-static inline void dwc3_decode_set_address(__u16 v, char *str)
-{
-	sprintf(str, "Set Address(Addr = %02x)", v);
-}
-
-static inline void dwc3_decode_get_set_descriptor(__u8 t, __u8 b, __u16 v,
-						  __u16 i, __u16 l, char *str)
-{
-	sprintf(str, "%s %s Descriptor(Index = %d, Length = %d)",
-		b == USB_REQ_GET_DESCRIPTOR ? "Get" : "Set",
-		({ char *s;
-			switch (v >> 8) {
-			case USB_DT_DEVICE:
-				s = "Device";
-				break;
-			case USB_DT_CONFIG:
-				s = "Configuration";
-				break;
-			case USB_DT_STRING:
-				s = "String";
-				break;
-			case USB_DT_INTERFACE:
-				s = "Interface";
-				break;
-			case USB_DT_ENDPOINT:
-				s = "Endpoint";
-				break;
-			case USB_DT_DEVICE_QUALIFIER:
-				s = "Device Qualifier";
-				break;
-			case USB_DT_OTHER_SPEED_CONFIG:
-				s = "Other Speed Config";
-				break;
-			case USB_DT_INTERFACE_POWER:
-				s = "Interface Power";
-				break;
-			case USB_DT_OTG:
-				s = "OTG";
-				break;
-			case USB_DT_DEBUG:
-				s = "Debug";
-				break;
-			case USB_DT_INTERFACE_ASSOCIATION:
-				s = "Interface Association";
-				break;
-			case USB_DT_BOS:
-				s = "BOS";
-				break;
-			case USB_DT_DEVICE_CAPABILITY:
-				s = "Device Capability";
-				break;
-			case USB_DT_PIPE_USAGE:
-				s = "Pipe Usage";
-				break;
-			case USB_DT_SS_ENDPOINT_COMP:
-				s = "SS Endpoint Companion";
-				break;
-			case USB_DT_SSP_ISOC_ENDPOINT_COMP:
-				s = "SSP Isochronous Endpoint Companion";
-				break;
-			default:
-				s = "UNKNOWN";
-				break;
-			} s; }), v & 0xff, l);
-}
-
-
-static inline void dwc3_decode_get_configuration(__u16 l, char *str)
-{
-	sprintf(str, "Get Configuration(Length = %d)", l);
-}
-
-static inline void dwc3_decode_set_configuration(__u8 v, char *str)
-{
-	sprintf(str, "Set Configuration(Config = %d)", v);
-}
-
-static inline void dwc3_decode_get_intf(__u16 i, __u16 l, char *str)
-{
-	sprintf(str, "Get Interface(Intf = %d, Length = %d)", i, l);
-}
-
-static inline void dwc3_decode_set_intf(__u8 v, __u16 i, char *str)
-{
-	sprintf(str, "Set Interface(Intf = %d, Alt.Setting = %d)", i, v);
-}
-
-static inline void dwc3_decode_synch_frame(__u16 i, __u16 l, char *str)
-{
-	sprintf(str, "Synch Frame(Endpoint = %d, Length = %d)", i, l);
-}
-
-static inline void dwc3_decode_set_sel(__u16 l, char *str)
-{
-	sprintf(str, "Set SEL(Length = %d)", l);
-}
-
-static inline void dwc3_decode_set_isoch_delay(__u8 v, char *str)
-{
-	sprintf(str, "Set Isochronous Delay(Delay = %d ns)", v);
-}
-
-/**
- * dwc3_decode_ctrl - returns a string represetion of ctrl request
- */
-static inline const char *dwc3_decode_ctrl(char *str, __u8 bRequestType,
-		__u8 bRequest, __u16 wValue, __u16 wIndex, __u16 wLength)
-{
-	switch (bRequest) {
-	case USB_REQ_GET_STATUS:
-		dwc3_decode_get_status(bRequestType, wIndex, wLength, str);
-		break;
-	case USB_REQ_CLEAR_FEATURE:
-	case USB_REQ_SET_FEATURE:
-		dwc3_decode_set_clear_feature(bRequestType, bRequest, wValue,
-					      wIndex, str);
-		break;
-	case USB_REQ_SET_ADDRESS:
-		dwc3_decode_set_address(wValue, str);
-		break;
-	case USB_REQ_GET_DESCRIPTOR:
-	case USB_REQ_SET_DESCRIPTOR:
-		dwc3_decode_get_set_descriptor(bRequestType, bRequest, wValue,
-					       wIndex, wLength, str);
-		break;
-	case USB_REQ_GET_CONFIGURATION:
-		dwc3_decode_get_configuration(wLength, str);
-		break;
-	case USB_REQ_SET_CONFIGURATION:
-		dwc3_decode_set_configuration(wValue, str);
-		break;
-	case USB_REQ_GET_INTERFACE:
-		dwc3_decode_get_intf(wIndex, wLength, str);
-		break;
-	case USB_REQ_SET_INTERFACE:
-		dwc3_decode_set_intf(wValue, wIndex, str);
-		break;
-	case USB_REQ_SYNCH_FRAME:
-		dwc3_decode_synch_frame(wIndex, wLength, str);
-		break;
-	case USB_REQ_SET_SEL:
-		dwc3_decode_set_sel(wLength, str);
-		break;
-	case USB_REQ_SET_ISOCH_DELAY:
-		dwc3_decode_set_isoch_delay(wValue, str);
+		snprintf(str, size, "Overflow [%s]",
+				dwc3_gadget_link_string(state));
 		break;
 	default:
-		sprintf(str, "%02x %02x %02x %02x %02x %02x %02x %02x",
-			bRequestType, bRequest,
-			cpu_to_le16(wValue) & 0xff,
-			cpu_to_le16(wValue) >> 8,
-			cpu_to_le16(wIndex) & 0xff,
-			cpu_to_le16(wIndex) >> 8,
-			cpu_to_le16(wLength) & 0xff,
-			cpu_to_le16(wLength) >> 8);
+		snprintf(str, size, "UNKNOWN");
 	}
 
 	return str;
@@ -490,16 +250,15 @@
  * dwc3_ep_event_string - returns event name
  * @event: then event code
  */
-static inline const char *
-dwc3_ep_event_string(char *str, const struct dwc3_event_depevt *event,
-		     u32 ep0state)
+static inline const char *dwc3_ep_event_string(char *str, size_t size,
+		const struct dwc3_event_depevt *event, u32 ep0state)
 {
 	u8 epnum = event->endpoint_number;
 	size_t len;
 	int status;
 	int ret;
 
-	ret = sprintf(str, "ep%d%s: ", epnum >> 1,
+	ret = snprintf(str, size, "ep%d%s: ", epnum >> 1,
 			(epnum & 1) ? "in" : "out");
 	if (ret < 0)
 		return "UNKNOWN";
@@ -509,7 +268,7 @@
 	switch (event->endpoint_event) {
 	case DWC3_DEPEVT_XFERCOMPLETE:
 		len = strlen(str);
-		sprintf(str + len, "Transfer Complete (%c%c%c)",
+		snprintf(str + len, size - len, "Transfer Complete (%c%c%c)",
 				status & DEPEVT_STATUS_SHORT ? 'S' : 's',
 				status & DEPEVT_STATUS_IOC ? 'I' : 'i',
 				status & DEPEVT_STATUS_LST ? 'L' : 'l');
@@ -517,12 +276,13 @@
 		len = strlen(str);
 
 		if (epnum <= 1)
-			sprintf(str + len, " [%s]", dwc3_ep0_state_string(ep0state));
+			snprintf(str + len, size - len, " [%s]",
+					dwc3_ep0_state_string(ep0state));
 		break;
 	case DWC3_DEPEVT_XFERINPROGRESS:
 		len = strlen(str);
 
-		sprintf(str + len, "Transfer In Progress [%d] (%c%c%c)",
+		snprintf(str + len, size - len, "Transfer In Progress [%d] (%c%c%c)",
 				event->parameters,
 				status & DEPEVT_STATUS_SHORT ? 'S' : 's',
 				status & DEPEVT_STATUS_IOC ? 'I' : 'i',
@@ -531,47 +291,51 @@
 	case DWC3_DEPEVT_XFERNOTREADY:
 		len = strlen(str);
 
-		sprintf(str + len, "Transfer Not Ready [%d]%s",
+		snprintf(str + len, size - len, "Transfer Not Ready [%d]%s",
 				event->parameters,
 				status & DEPEVT_STATUS_TRANSFER_ACTIVE ?
 				" (Active)" : " (Not Active)");
 
+		len = strlen(str);
+
 		/* Control Endpoints */
 		if (epnum <= 1) {
 			int phase = DEPEVT_STATUS_CONTROL_PHASE(event->status);
 
 			switch (phase) {
 			case DEPEVT_STATUS_CONTROL_DATA:
-				strcat(str, " [Data Phase]");
+				snprintf(str + ret, size - ret,
+						" [Data Phase]");
 				break;
 			case DEPEVT_STATUS_CONTROL_STATUS:
-				strcat(str, " [Status Phase]");
+				snprintf(str + ret, size - ret,
+						" [Status Phase]");
 			}
 		}
 		break;
 	case DWC3_DEPEVT_RXTXFIFOEVT:
-		strcat(str, "FIFO");
+		snprintf(str + ret, size - ret, "FIFO");
 		break;
 	case DWC3_DEPEVT_STREAMEVT:
 		status = event->status;
 
 		switch (status) {
 		case DEPEVT_STREAMEVT_FOUND:
-			sprintf(str + ret, " Stream %d Found",
+			snprintf(str + ret, size - ret, " Stream %d Found",
 					event->parameters);
 			break;
 		case DEPEVT_STREAMEVT_NOTFOUND:
 		default:
-			strcat(str, " Stream Not Found");
+			snprintf(str + ret, size - ret, " Stream Not Found");
 			break;
 		}
 
 		break;
 	case DWC3_DEPEVT_EPCMDCMPLT:
-		strcat(str, "Endpoint Command Complete");
+		snprintf(str + ret, size - ret, "Endpoint Command Complete");
 		break;
 	default:
-		sprintf(str, "UNKNOWN");
+		snprintf(str, size, "UNKNOWN");
 	}
 
 	return str;
@@ -611,14 +375,15 @@
 	}
 }
 
-static inline const char *dwc3_decode_event(char *str, u32 event, u32 ep0state)
+static inline const char *dwc3_decode_event(char *str, size_t size, u32 event,
+		u32 ep0state)
 {
 	const union dwc3_event evt = (union dwc3_event) event;
 
 	if (evt.type.is_devspec)
-		return dwc3_gadget_event_string(str, &evt.devt);
+		return dwc3_gadget_event_string(str, size, &evt.devt);
 	else
-		return dwc3_ep_event_string(str, &evt.depevt, ep0state);
+		return dwc3_ep_event_string(str, size, &evt.depevt, ep0state);
 }
 
 static inline const char *dwc3_ep_cmd_status_string(int status)
diff -Naur --no-dereference a/drivers/usb/dwc3/dwc3-keystone.c b/drivers/usb/dwc3/dwc3-keystone.c
--- a/drivers/usb/dwc3/dwc3-keystone.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/usb/dwc3/dwc3-keystone.c	2020-06-19 21:00:31.428227716 -0400
@@ -106,6 +106,10 @@
 		goto err_irq;
 	}
 
+	/* IRQ processing not required currently for AM65 */
+	if (of_device_is_compatible(node, "ti,am654-dwc3"))
+		goto skip_irq;
+
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0) {
 		dev_err(&pdev->dev, "missing irq\n");
@@ -123,6 +127,7 @@
 
 	kdwc3_enable_irqs(kdwc);
 
+skip_irq:
 	error = of_platform_populate(node, NULL, NULL, dev);
 	if (error) {
 		dev_err(&pdev->dev, "failed to create dwc3 core\n");
@@ -152,8 +157,11 @@
 static int kdwc3_remove(struct platform_device *pdev)
 {
 	struct dwc3_keystone *kdwc = platform_get_drvdata(pdev);
+	struct device_node *node = pdev->dev.of_node;
+
+	if (!of_device_is_compatible(node, "ti,am654-dwc3"))
+		kdwc3_disable_irqs(kdwc);
 
-	kdwc3_disable_irqs(kdwc);
 	device_for_each_child(&pdev->dev, NULL, kdwc3_remove_core);
 	pm_runtime_put_sync(kdwc->dev);
 	pm_runtime_disable(kdwc->dev);
@@ -165,6 +173,7 @@
 
 static const struct of_device_id kdwc3_of_match[] = {
 	{ .compatible = "ti,keystone-dwc3", },
+	{ .compatible = "ti,am654-dwc3" },
 	{},
 };
 MODULE_DEVICE_TABLE(of, kdwc3_of_match);
diff -Naur --no-dereference a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c
--- a/drivers/usb/dwc3/gadget.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/usb/dwc3/gadget.c	2020-06-19 21:00:31.428227716 -0400
@@ -632,7 +632,6 @@
 
 		dep->type = usb_endpoint_type(desc);
 		dep->flags |= DWC3_EP_ENABLED;
-		dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
 
 		reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 		reg |= DWC3_DALEPENA_EP(dep->number);
@@ -744,7 +743,7 @@
 
 	dep->stream_capable = false;
 	dep->type = 0;
-	dep->flags &= DWC3_EP_END_TRANSFER_PENDING;
+	dep->flags = 0;
 
 	/* Clear out the ep descriptors for non-ep0 */
 	if (dep->number > 1) {
@@ -2271,6 +2270,13 @@
 
 static bool dwc3_gadget_ep_request_completed(struct dwc3_request *req)
 {
+	/*
+	 * For OUT direction, host may send less than the setup
+	 * length. Return true for all OUT requests.
+	 */
+	if (!req->direction)
+		return true;
+
 	return req->request.actual == req->request.length;
 }
 
@@ -2398,7 +2404,7 @@
 	dep = dwc->eps[epnum];
 
 	if (!(dep->flags & DWC3_EP_ENABLED)) {
-		if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
+		if (!(dep->flags & DWC3_EP_TRANSFER_STARTED))
 			return;
 
 		/* Handle only EPCMDCMPLT when EP disabled */
@@ -2422,8 +2428,7 @@
 		cmd = DEPEVT_PARAMETER_CMD(event->parameters);
 
 		if (cmd == DWC3_DEPCMD_ENDTRANSFER) {
-			dep->flags &= ~(DWC3_EP_END_TRANSFER_PENDING |
-					DWC3_EP_TRANSFER_STARTED);
+			dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
 			dwc3_gadget_ep_cleanup_cancelled_requests(dep);
 		}
 		break;
@@ -2480,8 +2485,7 @@
 	u32 cmd;
 	int ret;
 
-	if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) ||
-	    !dep->resource_index)
+	if (!(dep->flags & DWC3_EP_TRANSFER_STARTED))
 		return;
 
 	/*
@@ -2524,10 +2528,8 @@
 	WARN_ON_ONCE(ret);
 	dep->resource_index = 0;
 
-	if (dwc3_is_usb31(dwc) || dwc->revision < DWC3_REVISION_310A) {
-		dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
+	if (dwc3_is_usb31(dwc) || dwc->revision < DWC3_REVISION_310A)
 		udelay(100);
-	}
 }
 
 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
@@ -3151,7 +3153,6 @@
 	dwc->gadget.speed		= USB_SPEED_UNKNOWN;
 	dwc->gadget.sg_supported	= true;
 	dwc->gadget.name		= "dwc3-gadget";
-	dwc->gadget.is_otg		= dwc->dr_mode == USB_DR_MODE_OTG;
 
 	/*
 	 * FIXME We might be setting max_speed to <SUPER, however versions
diff -Naur --no-dereference a/drivers/usb/dwc3/Kconfig b/drivers/usb/dwc3/Kconfig
--- a/drivers/usb/dwc3/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/usb/dwc3/Kconfig	2020-06-19 21:00:31.428227716 -0400
@@ -86,11 +86,11 @@
 	  platform, please say 'Y' or 'M' here.
 
 config USB_DWC3_KEYSTONE
-	tristate "Texas Instruments Keystone2 Platforms"
-	depends on ARCH_KEYSTONE || COMPILE_TEST
+	tristate "Texas Instruments Keystone2/AM654 Platforms"
+	depends on ARCH_KEYSTONE || ARCH_K3 || COMPILE_TEST
 	default USB_DWC3
 	help
-	  Support of USB2/3 functionality in TI Keystone2 platforms.
+	  Support of USB2/3 functionality in TI Keystone2 and AM654 platforms.
 	  Say 'Y' or 'M' here if you have one such device
 
 config USB_DWC3_OF_SIMPLE
diff -Naur --no-dereference a/drivers/usb/dwc3/trace.h b/drivers/usb/dwc3/trace.h
--- a/drivers/usb/dwc3/trace.h	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/usb/dwc3/trace.h	2020-06-19 21:00:31.428227716 -0400
@@ -59,8 +59,8 @@
 		__entry->ep0state = dwc->ep0state;
 	),
 	TP_printk("event (%08x): %s", __entry->event,
-			dwc3_decode_event(__get_str(str), __entry->event,
-					  __entry->ep0state))
+			dwc3_decode_event(__get_str(str), DWC3_MSG_MAX,
+					__entry->event, __entry->ep0state))
 );
 
 DEFINE_EVENT(dwc3_log_event, dwc3_event,
@@ -86,7 +86,8 @@
 		__entry->wIndex = le16_to_cpu(ctrl->wIndex);
 		__entry->wLength = le16_to_cpu(ctrl->wLength);
 	),
-	TP_printk("%s", dwc3_decode_ctrl(__get_str(str), __entry->bRequestType,
+	TP_printk("%s", usb_decode_ctrl(__get_str(str), DWC3_MSG_MAX,
+					__entry->bRequestType,
 					__entry->bRequest, __entry->wValue,
 					__entry->wIndex, __entry->wLength)
 	)
@@ -199,7 +200,7 @@
 		__entry->param2 = params->param2;
 		__entry->cmd_status = cmd_status;
 	),
-	TP_printk("%s: cmd '%s' [%d] params %08x %08x %08x --> status: %s",
+	TP_printk("%s: cmd '%s' [%x] params %08x %08x %08x --> status: %s",
 		__get_str(name), dwc3_gadget_ep_cmd_string(__entry->cmd),
 		__entry->cmd, __entry->param0,
 		__entry->param1, __entry->param2,
@@ -305,7 +306,7 @@
 		__entry->trb_enqueue = dep->trb_enqueue;
 		__entry->trb_dequeue = dep->trb_dequeue;
 	),
-	TP_printk("%s: mps %d/%d streams %d burst %d ring %d/%d flags %c:%c%c%c%c:%c:%c",
+	TP_printk("%s: mps %d/%d streams %d burst %d ring %d/%d flags %c:%c%c%c%c:%c",
 		__get_str(name), __entry->maxpacket,
 		__entry->maxpacket_limit, __entry->max_streams,
 		__entry->maxburst, __entry->trb_enqueue,
@@ -315,7 +316,6 @@
 		__entry->flags & DWC3_EP_WEDGE ? 'W' : 'w',
 		__entry->flags & DWC3_EP_TRANSFER_STARTED ? 'B' : 'b',
 		__entry->flags & DWC3_EP_PENDING_REQUEST ? 'P' : 'p',
-		__entry->flags & DWC3_EP_END_TRANSFER_PENDING ? 'E' : 'e',
 		__entry->direction ? '<' : '>'
 	)
 );
diff -Naur --no-dereference a/drivers/usb/gadget/legacy/cdc2.c b/drivers/usb/gadget/legacy/cdc2.c
--- a/drivers/usb/gadget/legacy/cdc2.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/usb/gadget/legacy/cdc2.c	2020-06-19 21:00:31.428227716 -0400
@@ -225,7 +225,7 @@
 	.name		= "g_cdc",
 	.dev		= &device_desc,
 	.strings	= dev_strings,
-	.max_speed	= USB_SPEED_HIGH,
+	.max_speed	= USB_SPEED_SUPER,
 	.bind		= cdc_bind,
 	.unbind		= cdc_unbind,
 };
diff -Naur --no-dereference a/drivers/usb/gadget/legacy/g_ffs.c b/drivers/usb/gadget/legacy/g_ffs.c
--- a/drivers/usb/gadget/legacy/g_ffs.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/usb/gadget/legacy/g_ffs.c	2020-06-19 21:00:31.428227716 -0400
@@ -149,7 +149,7 @@
 	.name		= DRIVER_NAME,
 	.dev		= &gfs_dev_desc,
 	.strings	= gfs_dev_strings,
-	.max_speed	= USB_SPEED_HIGH,
+	.max_speed	= USB_SPEED_SUPER,
 	.bind		= gfs_bind,
 	.unbind		= gfs_unbind,
 };
diff -Naur --no-dereference a/drivers/usb/gadget/legacy/multi.c b/drivers/usb/gadget/legacy/multi.c
--- a/drivers/usb/gadget/legacy/multi.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/usb/gadget/legacy/multi.c	2020-06-19 21:00:31.428227716 -0400
@@ -482,7 +482,7 @@
 	.name		= "g_multi",
 	.dev		= &device_desc,
 	.strings	= dev_strings,
-	.max_speed	= USB_SPEED_HIGH,
+	.max_speed	= USB_SPEED_SUPER,
 	.bind		= multi_bind,
 	.unbind		= multi_unbind,
 	.needs_serial	= 1,
diff -Naur --no-dereference a/drivers/usb/gadget/legacy/ncm.c b/drivers/usb/gadget/legacy/ncm.c
--- a/drivers/usb/gadget/legacy/ncm.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/usb/gadget/legacy/ncm.c	2020-06-19 21:00:31.428227716 -0400
@@ -197,7 +197,7 @@
 	.name		= "g_ncm",
 	.dev		= &device_desc,
 	.strings	= dev_strings,
-	.max_speed	= USB_SPEED_HIGH,
+	.max_speed	= USB_SPEED_SUPER,
 	.bind		= gncm_bind,
 	.unbind		= gncm_unbind,
 };
diff -Naur --no-dereference a/drivers/usb/Kconfig b/drivers/usb/Kconfig
--- a/drivers/usb/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/usb/Kconfig	2020-06-19 21:00:31.428227716 -0400
@@ -112,6 +112,8 @@
 
 endif
 
+source "drivers/usb/cdns3/Kconfig"
+
 source "drivers/usb/mtu3/Kconfig"
 
 source "drivers/usb/musb/Kconfig"
diff -Naur --no-dereference a/drivers/usb/Makefile b/drivers/usb/Makefile
--- a/drivers/usb/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/usb/Makefile	2020-06-19 21:00:31.428227716 -0400
@@ -12,6 +12,8 @@
 obj-$(CONFIG_USB_DWC2)		+= dwc2/
 obj-$(CONFIG_USB_ISP1760)	+= isp1760/
 
+obj-$(CONFIG_USB_CDNS3)		+= cdns3/
+
 obj-$(CONFIG_USB_MON)		+= mon/
 obj-$(CONFIG_USB_MTU3)		+= mtu3/
 
diff -Naur --no-dereference a/drivers/video/backlight/Kconfig b/drivers/video/backlight/Kconfig
--- a/drivers/video/backlight/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/video/backlight/Kconfig	2020-06-19 21:00:31.428227716 -0400
@@ -480,6 +480,13 @@
 	help
 	  Support for backlight control on RAVE SP device.
 
+config BACKLIGHT_LED
+	tristate "Generic LED based Backlight Driver"
+	depends on LEDS_CLASS && OF
+	help
+	  If you have a LCD backlight adjustable by LED class driver, say Y
+	  to enable this driver.
+
 endif # BACKLIGHT_CLASS_DEVICE
 
 endif # BACKLIGHT_LCD_SUPPORT
diff -Naur --no-dereference a/drivers/video/backlight/led_bl.c b/drivers/video/backlight/led_bl.c
--- a/drivers/video/backlight/led_bl.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/video/backlight/led_bl.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,224 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2015-2018 Texas Instruments Incorporated -  http://www.ti.com/
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ *
+ * Based on pwm_bl.c
+ */
+
+#include <linux/backlight.h>
+#include <linux/gpio/consumer.h>
+#include <linux/leds.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regulator/consumer.h>
+#include <linux/slab.h>
+
+struct led_bl_data {
+	struct device		*dev;
+	struct backlight_device	*bl_dev;
+
+	unsigned int		*levels;
+	bool			enabled;
+	struct regulator	*power_supply;
+	struct gpio_desc	*enable_gpio;
+
+	struct led_classdev *led_cdev;
+
+	unsigned int max_brightness;
+	unsigned int default_brightness;
+};
+
+static void led_bl_set_brightness(struct led_bl_data *priv, int brightness)
+{
+	int err;
+
+	if (!priv->enabled) {
+		err = regulator_enable(priv->power_supply);
+		if (err < 0)
+			dev_err(priv->dev, "failed to enable power supply\n");
+
+		if (priv->enable_gpio)
+			gpiod_set_value_cansleep(priv->enable_gpio, 1);
+	}
+
+	led_set_brightness(priv->led_cdev, priv->levels[brightness]);
+
+	priv->enabled = true;
+}
+
+static void led_bl_power_off(struct led_bl_data *priv)
+{
+	if (!priv->enabled)
+		return;
+
+	led_set_brightness(priv->led_cdev, LED_OFF);
+
+	if (priv->enable_gpio)
+		gpiod_set_value_cansleep(priv->enable_gpio, 0);
+
+	regulator_disable(priv->power_supply);
+
+	priv->enabled = false;
+}
+
+static int led_bl_update_status(struct backlight_device *bl)
+{
+	struct led_bl_data *priv = bl_get_data(bl);
+	int brightness = bl->props.brightness;
+
+	if (bl->props.power != FB_BLANK_UNBLANK ||
+	    bl->props.fb_blank != FB_BLANK_UNBLANK ||
+	    bl->props.state & BL_CORE_FBBLANK)
+		brightness = 0;
+
+	if (brightness > 0)
+		led_bl_set_brightness(priv, brightness);
+	else
+		led_bl_power_off(priv);
+
+	return 0;
+}
+
+static const struct backlight_ops led_bl_ops = {
+	.update_status	= led_bl_update_status,
+};
+
+static int led_bl_parse_dt(struct device *dev,
+			   struct led_bl_data *priv)
+{
+	struct device_node *node = dev->of_node;
+	int num_levels;
+	u32 *levels;
+	u32 value;
+	int ret;
+
+	if (!node)
+		return -ENODEV;
+
+	num_levels = of_property_count_u32_elems(node, "brightness-levels");
+	if (num_levels < 0)
+		return num_levels;
+
+	levels = devm_kzalloc(dev, sizeof(u32) * num_levels, GFP_KERNEL);
+	if (!levels)
+		return -ENOMEM;
+
+	ret = of_property_read_u32_array(node, "brightness-levels",
+					 levels,
+					 num_levels);
+	if (ret < 0)
+		return ret;
+
+	ret = of_property_read_u32(node, "default-brightness-level", &value);
+	if (ret < 0)
+		return ret;
+
+	if (value >= num_levels) {
+		dev_err(dev, "invalid default-brightness-level\n");
+		return -EINVAL;
+	}
+
+	priv->levels = levels;
+	priv->max_brightness = num_levels - 1;
+	priv->default_brightness = value;
+
+	priv->led_cdev = of_led_get(node);
+	if (IS_ERR(priv->led_cdev))
+		return PTR_ERR(priv->led_cdev);
+
+	return 0;
+}
+
+static int led_bl_probe(struct platform_device *pdev)
+{
+	struct backlight_properties props;
+	struct led_bl_data *priv;
+	int ret;
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, priv);
+
+	priv->dev = &pdev->dev;
+
+	ret = led_bl_parse_dt(&pdev->dev, priv);
+	if (ret < 0) {
+		if (ret != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "failed to parse DT data\n");
+		return ret;
+	}
+
+	priv->enable_gpio = devm_gpiod_get_optional(&pdev->dev, "enable",
+			    GPIOD_OUT_LOW);
+	if (IS_ERR(priv->enable_gpio)) {
+		ret = PTR_ERR(priv->enable_gpio);
+		goto err;
+	}
+
+	priv->power_supply = devm_regulator_get(&pdev->dev, "power");
+	if (IS_ERR(priv->power_supply)) {
+		ret = PTR_ERR(priv->power_supply);
+		goto err;
+	}
+
+	memset(&props, 0, sizeof(struct backlight_properties));
+	props.type = BACKLIGHT_RAW;
+	props.max_brightness = priv->max_brightness;
+	priv->bl_dev = backlight_device_register(dev_name(&pdev->dev),
+			&pdev->dev, priv, &led_bl_ops, &props);
+	if (IS_ERR(priv->bl_dev)) {
+		dev_err(&pdev->dev, "failed to register backlight\n");
+		ret = PTR_ERR(priv->bl_dev);
+		goto err;
+	}
+
+	priv->bl_dev->props.brightness = priv->default_brightness;
+	backlight_update_status(priv->bl_dev);
+
+	return 0;
+
+err:
+	if (priv->led_cdev)
+		led_put(priv->led_cdev);
+
+	return ret;
+}
+
+static int led_bl_remove(struct platform_device *pdev)
+{
+	struct led_bl_data *priv = platform_get_drvdata(pdev);
+	struct backlight_device *bl = priv->bl_dev;
+
+	backlight_device_unregister(bl);
+
+	led_bl_power_off(priv);
+
+	led_put(priv->led_cdev);
+
+	return 0;
+}
+
+static const struct of_device_id led_bl_of_match[] = {
+	{ .compatible = "led-backlight" },
+	{ }
+};
+
+MODULE_DEVICE_TABLE(of, led_bl_of_match);
+
+static struct platform_driver led_bl_driver = {
+	.driver		= {
+		.name		= "led-backlight",
+		.of_match_table	= of_match_ptr(led_bl_of_match),
+	},
+	.probe		= led_bl_probe,
+	.remove		= led_bl_remove,
+};
+
+module_platform_driver(led_bl_driver);
+
+MODULE_DESCRIPTION("LED based Backlight Driver");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:led-backlight");
diff -Naur --no-dereference a/drivers/video/backlight/Makefile b/drivers/video/backlight/Makefile
--- a/drivers/video/backlight/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/video/backlight/Makefile	2020-06-19 21:00:31.428227716 -0400
@@ -59,3 +59,4 @@
 obj-$(CONFIG_BACKLIGHT_WM831X)		+= wm831x_bl.o
 obj-$(CONFIG_BACKLIGHT_ARCXCNN) 	+= arcxcnn_bl.o
 obj-$(CONFIG_BACKLIGHT_RAVE_SP)		+= rave-sp-backlight.o
+obj-$(CONFIG_BACKLIGHT_LED)		+= led_bl.o
diff -Naur --no-dereference a/drivers/watchdog/Kconfig b/drivers/watchdog/Kconfig
--- a/drivers/watchdog/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/watchdog/Kconfig	2020-06-19 21:00:31.428227716 -0400
@@ -481,6 +481,14 @@
 	  NOTE: once enabled, this timer cannot be disabled.
 	  Say N if you are unsure.
 
+config K3_RTI_WATCHDOG
+	tristate "Texas Instruments K3 RTI watchdog"
+	depends on ARCH_K3 || COMPILE_TEST
+	select WATCHDOG_CORE
+	help
+	  Say Y here if you want to include support for the K3 watchdog
+	  timer (RTI module) available in the K3 generation of processors.
+
 config ORION_WATCHDOG
 	tristate "Orion watchdog"
 	depends on ARCH_ORION5X || ARCH_DOVE || MACH_DOVE || ARCH_MVEBU || (COMPILE_TEST && !ARCH_EBSA110)
diff -Naur --no-dereference a/drivers/watchdog/Makefile b/drivers/watchdog/Makefile
--- a/drivers/watchdog/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/watchdog/Makefile	2020-06-19 21:00:31.428227716 -0400
@@ -57,6 +57,7 @@
 obj-$(CONFIG_PNX4008_WATCHDOG) += pnx4008_wdt.o
 obj-$(CONFIG_IOP_WATCHDOG) += iop_wdt.o
 obj-$(CONFIG_DAVINCI_WATCHDOG) += davinci_wdt.o
+obj-$(CONFIG_K3_RTI_WATCHDOG) += rti_wdt.o
 obj-$(CONFIG_ORION_WATCHDOG) += orion_wdt.o
 obj-$(CONFIG_SUNXI_WATCHDOG) += sunxi_wdt.o
 obj-$(CONFIG_RN5T618_WATCHDOG) += rn5t618_wdt.o
diff -Naur --no-dereference a/drivers/watchdog/rti_wdt.c b/drivers/watchdog/rti_wdt.c
--- a/drivers/watchdog/rti_wdt.c	1969-12-31 19:00:00.000000000 -0500
+++ b/drivers/watchdog/rti_wdt.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,252 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Watchdog driver for the K3 RTI module
+ *
+ * (c) Copyright 2019 Texas Instruments Inc.
+ * All rights reserved.
+ */
+
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/mod_devicetable.h>
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/watchdog.h>
+#include <linux/platform_device.h>
+#include <linux/io.h>
+#include <linux/device.h>
+#include <linux/clk.h>
+#include <linux/err.h>
+#include <linux/pm_runtime.h>
+
+#define MODULE_NAME "rti-wdt"
+#define DEFAULT_HEARTBEAT 60
+#define MAX_HEARTBEAT     1000
+
+/* Timer register set definition */
+#define RTIDWDCTRL	0x90
+#define RTIDWDPRLD	0x94
+#define RTIWDSTATUS	0x98
+#define RTIWDKEY	0x9c
+#define RTIDWDCNTR	0xa0
+#define RTIWWDRXCTRL	0xa4
+#define RTIWWDSIZECTRL	0xa8
+
+#define RTIWWDRX_NMI	0xa
+
+#define RTIWWDSIZE_50P	0x50
+
+#define WDENABLE_KEY	0xa98559da
+
+#define WDKEY_SEQ0		0xe51a
+#define WDKEY_SEQ1		0xa35c
+
+#define WDT_PRELOAD_SHIFT	13
+
+#define WDT_PRELOAD_MAX		0xfff
+
+#define DWDST			BIT(1)
+
+static int heartbeat;
+
+/*
+ * struct to hold data for each WDT device
+ * @base - base io address of WD device
+ * @clk - source clock of WDT
+ * @wdd - hold watchdog device as is in WDT core
+ */
+struct rti_wdt_device {
+	void __iomem		*base;
+	struct clk		*clk;
+	struct watchdog_device	wdd;
+};
+
+static int rti_wdt_start(struct watchdog_device *wdd)
+{
+	u32 timer_margin;
+	unsigned long freq;
+	struct rti_wdt_device *wdt = watchdog_get_drvdata(wdd);
+
+	freq = clk_get_rate(wdt->clk);
+
+	/* set timeout period */
+	timer_margin = (u64)wdd->timeout * freq;
+	timer_margin >>= WDT_PRELOAD_SHIFT;
+	if (timer_margin > WDT_PRELOAD_MAX)
+		timer_margin = WDT_PRELOAD_MAX;
+	writel_relaxed(timer_margin, wdt->base + RTIDWDPRLD);
+
+	/* Set min heartbeat to 1.1x window size */
+	wdd->min_hw_heartbeat_ms = 11 * wdd->timeout * 1000 / 20;
+
+	/* Generate NMI when wdt expires */
+	writel_relaxed(RTIWWDRX_NMI, wdt->base + RTIWWDRXCTRL);
+
+	/* Window size 50% */
+	writel_relaxed(RTIWWDSIZE_50P, wdt->base + RTIWWDSIZECTRL);
+
+	readl_relaxed(wdt->base + RTIWWDSIZECTRL);
+
+	/* enable watchdog */
+	writel_relaxed(WDENABLE_KEY, wdt->base + RTIDWDCTRL);
+	return 0;
+}
+
+static int rti_wdt_ping(struct watchdog_device *wdd)
+{
+	struct rti_wdt_device *wdt = watchdog_get_drvdata(wdd);
+
+	/* put watchdog in service state */
+	writel_relaxed(WDKEY_SEQ0, wdt->base + RTIWDKEY);
+	/* put watchdog in active state */
+	writel_relaxed(WDKEY_SEQ1, wdt->base + RTIWDKEY);
+
+	if (readl_relaxed(wdt->base + RTIWDSTATUS))
+		WARN_ON_ONCE(1);
+
+	return 0;
+}
+
+static unsigned int rti_wdt_get_timeleft(struct watchdog_device *wdd)
+{
+	u64 timer_counter;
+	unsigned long freq;
+	u32 val;
+	struct rti_wdt_device *wdt = watchdog_get_drvdata(wdd);
+
+	/* if timeout has occurred then return 0 */
+	val = readl_relaxed(wdt->base + RTIWDSTATUS);
+	if (val & DWDST)
+		return 0;
+
+	freq = clk_get_rate(wdt->clk);
+	if (!freq)
+		return 0;
+
+	timer_counter = readl_relaxed(wdt->base + RTIDWDCNTR);
+
+	do_div(timer_counter, freq);
+
+	return timer_counter;
+}
+
+static const struct watchdog_info rti_wdt_info = {
+	.options = WDIOF_KEEPALIVEPING,
+	.identity = "K3 RTI Watchdog",
+};
+
+static const struct watchdog_ops rti_wdt_ops = {
+	.owner		= THIS_MODULE,
+	.start		= rti_wdt_start,
+	.ping		= rti_wdt_ping,
+	.get_timeleft	= rti_wdt_get_timeleft,
+};
+
+static int rti_wdt_probe(struct platform_device *pdev)
+{
+	int ret = 0;
+	struct device *dev = &pdev->dev;
+	struct resource *wdt_mem;
+	struct watchdog_device *wdd;
+	struct rti_wdt_device *wdt;
+
+	wdt = devm_kzalloc(dev, sizeof(*wdt), GFP_KERNEL);
+	if (!wdt)
+		return -ENOMEM;
+
+	wdt->clk = devm_clk_get(dev, NULL);
+	if (IS_ERR(wdt->clk)) {
+		if (PTR_ERR(wdt->clk) != -EPROBE_DEFER)
+			dev_err(dev, "failed to get clock\n");
+		return PTR_ERR(wdt->clk);
+	}
+
+	pm_runtime_enable(dev);
+	ret = pm_runtime_get_sync(dev);
+	if (ret) {
+		if (ret != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "runtime pm failed\n");
+		return ret;
+	}
+
+	platform_set_drvdata(pdev, wdt);
+
+	wdd = &wdt->wdd;
+	wdd->info = &rti_wdt_info;
+	wdd->ops = &rti_wdt_ops;
+	wdd->min_timeout = 1;
+	/* Set min heartbeat to 1.1x window size */
+	wdd->min_hw_heartbeat_ms = 11 * DEFAULT_HEARTBEAT * 1000 / 20;
+	wdd->max_hw_heartbeat_ms = MAX_HEARTBEAT * 1000;
+	wdd->timeout = DEFAULT_HEARTBEAT;
+	wdd->parent = dev;
+
+	set_bit(WDOG_RESET_KEEPALIVE, &wdd->status);
+
+	watchdog_init_timeout(wdd, heartbeat, dev);
+
+	dev_info(dev, "heartbeat %d sec\n", wdd->timeout);
+
+	watchdog_set_drvdata(wdd, wdt);
+	watchdog_set_nowayout(wdd, 1);
+	watchdog_set_restart_priority(wdd, 128);
+
+	wdt_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	wdt->base = devm_ioremap_resource(dev, wdt_mem);
+	if (IS_ERR(wdt->base)) {
+		ret = PTR_ERR(wdt->base);
+		goto err_iomap;
+	}
+
+	ret = watchdog_register_device(wdd);
+	if (ret) {
+		dev_err(dev, "cannot register watchdog device\n");
+		goto err_iomap;
+	}
+
+	return 0;
+
+err_iomap:
+	pm_runtime_put_sync(&pdev->dev);
+
+	return ret;
+}
+
+static int rti_wdt_remove(struct platform_device *pdev)
+{
+	struct rti_wdt_device *wdt = platform_get_drvdata(pdev);
+
+	watchdog_unregister_device(&wdt->wdd);
+	pm_runtime_put(&pdev->dev);
+
+	return 0;
+}
+
+static const struct of_device_id rti_wdt_of_match[] = {
+	{ .compatible = "ti,rti-wdt", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, rti_wdt_of_match);
+
+static struct platform_driver rti_wdt_driver = {
+	.driver = {
+		.name = "rti-wdt",
+		.of_match_table = rti_wdt_of_match,
+	},
+	.probe = rti_wdt_probe,
+	.remove = rti_wdt_remove,
+};
+
+module_platform_driver(rti_wdt_driver);
+
+MODULE_AUTHOR("Tero Kristo <t-kristo@ti.com>");
+MODULE_DESCRIPTION("K3 RTI Watchdog Driver");
+
+module_param(heartbeat, int, 0);
+MODULE_PARM_DESC(heartbeat,
+		 "Watchdog heartbeat period in seconds from 1 to "
+		 __MODULE_STRING(MAX_HEARTBEAT) ", default "
+		 __MODULE_STRING(DEFAULT_HEARTBEAT));
+
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:rti-wdt");
diff -Naur --no-dereference a/drivers/watchdog/watchdog_dev.c b/drivers/watchdog/watchdog_dev.c
--- a/drivers/watchdog/watchdog_dev.c	2020-01-09 04:19:10.000000000 -0500
+++ b/drivers/watchdog/watchdog_dev.c	2020-06-19 21:00:31.428227716 -0400
@@ -266,6 +266,8 @@
 		set_bit(WDOG_ACTIVE, &wdd->status);
 		wd_data->last_keepalive = started_at;
 		watchdog_update_worker(wdd);
+		if (test_bit(WDOG_RESET_KEEPALIVE, &wdd->status))
+			wd_data->last_hw_keepalive = started_at;
 	}
 
 	return err;
diff -Naur --no-dereference a/firmware/am335x-bone-scale-data.bin b/firmware/am335x-bone-scale-data.bin
--- a/firmware/am335x-bone-scale-data.bin	1969-12-31 19:00:00.000000000 -0500
+++ b/firmware/am335x-bone-scale-data.bin	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1 @@
+W#d$m$$m$$l$�$l$�d$m$$m$$l$�$l$�
\ No newline at end of file
diff -Naur --no-dereference a/firmware/am335x-evm-scale-data.bin b/firmware/am335x-evm-scale-data.bin
--- a/firmware/am335x-evm-scale-data.bin	1969-12-31 19:00:00.000000000 -0500
+++ b/firmware/am335x-evm-scale-data.bin	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1 @@
+Wd-%d-%+
\ No newline at end of file
diff -Naur --no-dereference a/firmware/am335x-pm-firmware.bin b/firmware/am335x-pm-firmware.bin
--- a/firmware/am335x-pm-firmware.bin	1969-12-31 19:00:00.000000000 -0500
+++ b/firmware/am335x-pm-firmware.bin	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,34 @@
+�����	
����������������������������������M
a
�
�
����
�
�
	1EYm�����JhC��sC�`pG0�D�>�KD�hpG��D8KD�`pG��D8KJh��`pG,�D�KhZ*	�J���h0� pG pG��#J$Khh�#M�"J/`"N���p!L"H��!J��%7`���#p%pHhJh`S��h�HI��0P`�D��hH�Gi��I	h)ИI��dh�T`��I��Q�$@,����<�D8�D�����0�D4�D��O��3�HLIJ#``]�K``pG4�D0�D8�D<�DJhC��sC�`pG0�DJh��C�@`pG�,�DJh#��C`pG�@�DKJh��|pG��KJh��X|pG������$��� F4��4,��K��Jh�B������r������� ���5 �~����� ��@�I �� � F�������T$�����0���������@��������@�" �}���@" �P�����" �r���@" �E���$ F�S� F4�c�4,��K��Jh�B������ �,�5 �)� ��� ��U�@�� F�2� F�C���pG�pG�pG�pG�pG�2�FɲD��B��pG�xC!��+*��FpG��LF!h
+MI�K�B#`�Kp�#` ��O�8F����9F��� 8�Fx8�%F�����(��`8����Fxx�i�F<F}�B	������(�����F��0F����K0�h���B� ���0@�pG���r�KJhR�  LH`�>��FD����D
+ ������� F���0@�pG�T|$�$K`pG�T-��O$F�F'F���Fe�(t���r��FvF��%(!F��/��;FO� :F�����#u��W������������ �%.........������������������������������P����������U������^�c�m|����\����� F�����
+ ��#�i�ځ�@���#������oк���I�y��
+	�^#rF��0�F
�SF�F�fF�����3�L�\Y���a����O�O� ��۲�F�;F*�������sqF�
+к���+���+���s��E����	�����- L�����S�����+����+�����M�0xP���~��
4(�ѹ�?����+?������	����	��6>@F��d�s��LD�� �c�pxC�@.F��pxC�s.F����Q�4���c����(���px.F��pxC�c.F��pxC��.F��pxC�c.F���j@�c�pxO�0.F������0��0	(���j�.F�	�L����0��0	(��HF����eFbF��� C�S�����4�����F�pxC�.F��@�3�#�pxC�.F������@�C�"F�����#�a� ����*@��C�s ����#�i���pxC�.Fc���������&�HF��������ՐB�F�F�F���"��v�À��;>���;@F�������?��QFHF����4DD����pxC� .F+��C�i��@�Ӏ
+ �#��������@�#�C�,��pF����'D��H����D��HF.F�jO���$�$��	AѸ�0������+����9	��9@F��F����?���D�+@�Ӏ�+����O�	�������+����9	��9@F��'����?���D0x(?���O�	�������+��+?􏮺�?�}��0��p��Ԛ��^��{ջ������0�`{�
+ ������pxC�.F��px�� .F���HFQF��
+���9D>�	��9@F�������?��4D���T�#�s��vx��0��0	(�F���0FbF.F]�pxC�.FX��3���3���Ԙ��Z��O�
+*ջ��������Z��'���0��� ��X���0 4�����px����vB�.F#�^�g�������Z�^��������X�u���0�p��!FO�	���$�����S�F������]���pG����JK�B
��CD
H!�1FDS��BB���H	IO��B��@�+����ǻ��0*�������������������������� �0���� ��A���`pG�@���hpG#AB��X�JB(��0@��`@�@�a@`pG#AB��X�JB(��0J@B� 0pG���#AB��X�JB(��0J@B� 0pG���JhC�`pG��JhC�`pG��� ���� ���Kh`���D� ������.���6�������FH���������B���@ ���� ��0���@ ���� ������@ ��� F��,���@ ��w���! ������@! ���$ F��{� F4����4,��K��	Jh�Bн�@��� F��h� F��y���@�������# ��Z���@# �����$ ��P���@$ �����( ��F���@( �y���) ��<���@) �o���* ��2���@* �e���+ ��(���@+ �[���, �����@, �Q���- �����@- �G���. ��
+���@. �=���/ �����@/ �3���0 ������@0 �)���1 ������@1 ����2 ������@2 ����3 ������@3 ����4 ������@4 ����5 ������<���D�������0���T���@5 ���� ��B���@5 ���� ������@5 �����Khhh���pG�Khhh���pG�KhS�  h#�`pG�KhS�  hC�`pG�K�DKh�B�K���B�pGKJ`pGKJ`pG���)�&"K`Z`�`�`aZapG� K�DKh�B�K���B�pG	K	H
+I
+J�a�abpGKH	I	J�a�abpG�� \)$))�)�)�)
+Jp�j�A��mhT�1�h-h��Pf`�``p� pG� Kp��i��EyYk�i�i����@��@$��@hXh!C��!��@�h�h
+C���i"��@C1�[i���@ �(Cp�pG Jy�ip��j
hHh��@��$�iMi�@"C����h"��@�hC�� ��@Cp�pG K�@j�R�0 [h`pG� Kj��Chh�pG� 
+Kj��	hhJ@�� pG�h�hhhX@�� pG� �Dh�����Y� x�!��O�0�$��U�� ��X��! D��N�Kh,���M��� F�~���@ �y���8�7K7JhF���|Dh+Y�(��b�������� h.K@���O��� F��.�F F��X�!���!(F����������� �B��\�#x��Kx+���F�u ������K�DKh�B�K���B�� ��]�(��J hC�`��{� ��b�Kx+�ѽ�8@ �������Kh+���������� �D�8�"K"JhF���|Dh��(����#x�(������y� hK@ӱ��O�Q� F����F F����!����!(F�����K� ���Kx[���F�u �6��������:��� ��8@𲾜���8�!K!JhF���|Dhs�(���#x�%������+� hK@����O�� F��`�F F����!��A�!(F��=����
Kx[���F�u ��������������� ��8@�g������8�KJhF���|DhS�(��K�������� hK@˱��O�� F���F F��A�!����!(F���� �1�	Kx���8�F�u ������������Ch؈������@ ��8�F lh��(��������� F����F F���!����(F!����#y�+� �~���8@ ���pG�p�F��H�(D�"M#L+h���h+:ИG0F����khx���8�h���+h���|��d�$ F���� F4����4,��K��Jh�B� F���� F������`���� ��q�5 ��n���p@ �����pG�K�DK�h�B!�K���B!�����F ���� ���� ������;��G��1� F��p�����@ �u�Kh+�� ���� ������������F ��{� ��x� �Y����� F��F��v���@ �K�����}�F ��_� ��\� �=���� F��,�@���@�Y�����e�F ��G� ��D�����>� F����E���@ ����=���@ �����E�F ��'� ��$����� F�����%� �����@ ���KFhA��q`h���pG$@�DKFh!��q`h���pG$@�D"�K`Z`��Q���@����
KHhI�J��4``hJ��#`�N�����@�����@����D��@�D�J hC�3`pG�p�DJ h#�3`pG�p�DJhC`pG�p�DO�qKJ`hC`pG��D�DJh#�s`pG�D8�IK�t`d#h+,�ch+/�#j+2�#i+5�ci+8ѣi+;ѣh+>��h+A��i+D�K��9M*h�BGУj+d�#k+]��j+M�K��*h�BP�8�! ����ch+��( ����#j+��3 ����#i+��- ����ci+��. �����i+��/ �����h+��* �����h+��, �����i+��2 ����K��M*h�B��ck+�� ����O�qKJ`hC`��0 ����K��*h�B�ѽ�8@4 ����$ ������# �������D��D�D"8�"K! `����( ����) ����* ����+ ����, ����- ����. ����/ ����0 ����1 ����2 ����3 ����# ����$ ���K��M,h�B�8�	J h#�s`��p�+h�B�ѽ�8@4 ��h�D��D�����+����
Kx������� ���� �z� �w� �t� �q���@ �l��� ��� �}� �z� �w� �t� �����O�Kh+��q�KxC����K�D	Jh�B����U����@� ������8�O��s�B�F ��F ��� �?�F
� F8� �� F8�$ F8�����Kx��pG��Kx��pG��K��I�hINMLH	h6hK-h$hh�B`^`�`�`a�IJ	hhYa�a�pG��D��D�D@�DD�DHH�DL�D������Kh#�
Kh;+
�JK
O
NMLIH:`2`+`#``��
JhC��S`������?���@�DD�D�D�D�D�	���D8�����MKLHIJ+`#```h+�+�8�O�`o�bIK``8�%O��0
JhC�s`D�<\A�@8��@�D�?D�DH�DL�D��D�D`�DK�DKh�B�K���B�pG���������K�DJ�h�B�K��KHh]hL�h`�h%`iM�`.`dh�BњiXiIK``�pGKh+��
Kh;+��Ih#��S`����H�D�D@�DH�DL�D���D !	KF``hA�@`h!�A�``h���pG��DKh+�"KJK`pG��D�	Kx�� �F�#IJ	h �@`��@�M�����q�D�	Kx�� �,�#IJ	h �@`��@�3�����q�DKh+�JhC�C`pG��DKh+�Jh#�C`pG��D J�h,:�#hY7�Ph���P�3P��.hJh2C*`P�3PJi.h2C*`P�3`�h�h2h=C*C2`P�3P
+i.h2C*`P�3P�i*h"�*`P�3 �ih#�`Kj�j	jChB��T�?Z���pGd#J�h.@�3hY=жFUh���U�4 ��h�iC`j�jhB��U�4 �ihC`jBjhB��U�4i
+h^�?"�
+`U�4p�h�h:h1C"�:`U�4Gi
+h"�
+`U�4@h
+h"�
+`Z����dK���Zh����j� h �A�!`k�`h+��]�KpGd
+K���Zh��h���j�)`�pGkh+��pG�dK���[h��Ckh��pGd"K�`�`aZa�apGdK�DKh�B
+�K���B�pG	K	I
+JY``pG	JK	IhY`*��J`pG��d''�l%\%�K��
���L�#h+����As��S�S�����&@�DK�DKh�B�K���B�pGKH	I	J��pGKHI	J��pG���`(8(0(�(�(�(K(hO�S�! h#��C�C�`pG��8�F	hJ�F�8�(F����T�K��8�8� ���KhS� 8�h�������@	pG�KhS�  h#�C�`pG��KhS�  h#�C�`pG��K Yh��K Yh��K �h��-��OO�	FFhJiK���������fMgHgI���(�-����	�`������������sx�{������G�'���%�FHF����-��5/�;F#�������rRQ��)@��!O�r�;��+@��#%O�BO��wBH�BM�8+����>KAI����x*DМFH�k���������Fl09O�zs�;P�
���Ԯ�pfxU��`����� 2F������#D��p�K�B���5O�zrt�[�:�������O��rO��0 K��������pr]*�� ��PKI�J
���PL���P%���@���0!��<B��������O��rO��0K������#��!"u牲s盲y����D���D���D���D���D(��D,��D���D���DhZ5��F%Jh`�W�#@O��r `�:�#h��C+��V�?j\�F���
�<�B�(��"hO��sW�"`�;��
+h�@?��<�B�� �pG pG��K�DKh�B�K���B�pGKH	I	J��pGKHI	J��pG����&�&�& %%�$"IO��s	hQ� 
+`�;�
+h�@?��pG�#IO��r	hQ� `�:�h��C+��pG��KhS� 0h�������@	pG���K Yh��\� ���K Yh��R� ���K �h��H� ���K �h��>� ��Booted version 0x%x of CM3 PM Firmware.�$�$�$�$�$[%08x]%s:   ERR WARN INFO DEBUG0123456789abcdefghijklmnopqrstuvwxyz(null)0x
+	��������L�D(�D��D�D��D��D��D��Dd�D �D`�D�D,�D�����D� �D��Dp�D��D�D� �D��DH�DT�D�D� �D��D4�D@�D��D �D,�D��D\�Dh�D�D�D
+	����
����8*�D ��D0��Dh+�D@+�D(��D ��D@��D���D���D ��D ��DP��D8��D ��DK�]�a�e�����D� �D�-�D�-�D�-�D�D� �D .�D$.�D,.�D�D� �D�-�D�-�D�-�D`-�Dd-�Dl-�D -�D$-�D,-�D����
����)�D��D��D��D��D��D��D��D��D��D��D+�D����

+	�����DP�DD�D@�D�D�D�D�D�DH�D,�D�D�D�D�D�D�D	
+
+	@�D@�D�D�D�D�D ��@GNUs	��i���dW��7#=ub�Qu�Qu���y���y��������|E!|E!�p�Qp�QXy�1da�� � ������������������������������log_buf
\ No newline at end of file
diff -Naur --no-dereference a/firmware/am335x-pm-firmware.elf b/firmware/am335x-pm-firmware.elf
--- a/firmware/am335x-pm-firmware.elf	1969-12-31 19:00:00.000000000 -0500
+++ b/firmware/am335x-pm-firmware.elf	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,396 @@
+ELF(40M4 (0*0*0*�Q�td�����	
����������������������������������M
a
�
�
����
�
�
	1EYm�����JhC��sC�`pG0�D�>�KD�hpG��D8KD�`pG��D8KJh��`pG,�D�KhZ*	�J���h0� pG pG��#J$Khh�#M�"J/`"N���p!L"H��!J��%7`���#p%pHhJh`S��h�HI��0P`�D��hH�Gi��I	h)ИI��dh�T`��I��Q�$@,����<�D8�D�����0�D4�D��O��3�HLIJ#``]�K``pG4�D0�D8�D<�DJhC��sC�`pG0�DJh��C�@`pG�,�DJh#��C`pG�@�DKJh��|pG��KJh��X|pG������$��� F4��4,��K��Jh�B������r������� ���5 �~����� ��@�I �� � F�������T$�����0���������@��������@�" �}���@" �P�����" �r���@" �E���$ F�S� F4�c�4,��K��Jh�B������ �,�5 �)� ��� ��U�@�� F�2� F�C���pG�pG�pG�pG�pG�2�FɲD��B��pG�xC!��+*��FpG��LF!h
+MI�K�B#`�Kp�#` ��O�8F����9F��� 8�Fx8�%F�����(��`8����Fxx�i�F<F}�B	������(�����F��0F����K0�h���B� ���0@�pG���r�KJhR�  LH`�>��FD����D
+ ������� F���0@�pG�T|$�$K`pG�T-��O$F�F'F���Fe�(t���r��FvF��%(!F��/��;FO� :F�����#u��W������������ �%.........������������������������������P����������U������^�c�m|����\����� F�����
+ ��#�i�ځ�@���#������oк���I�y��
+	�^#rF��0�F
�SF�F�fF�����3�L�\Y���a����O�O� ��۲�F�;F*�������sqF�
+к���+���+���s��E����	�����- L�����S�����+����+�����M�0xP���~��
4(�ѹ�?����+?������	����	��6>@F��d�s��LD�� �c�pxC�@.F��pxC�s.F����Q�4���c����(���px.F��pxC�c.F��pxC��.F��pxC�c.F���j@�c�pxO�0.F������0��0	(���j�.F�	�L����0��0	(��HF����eFbF��� C�S�����4�����F�pxC�.F��@�3�#�pxC�.F������@�C�"F�����#�a� ����*@��C�s ����#�i���pxC�.Fc���������&�HF��������ՐB�F�F�F���"��v�À��;>���;@F�������?��QFHF����4DD����pxC� .F+��C�i��@�Ӏ
+ �#��������@�#�C�,��pF����'D��H����D��HF.F�jO���$�$��	AѸ�0������+����9	��9@F��F����?���D�+@�Ӏ�+����O�	�������+����9	��9@F��'����?���D0x(?���O�	�������+��+?􏮺�?�}��0��p��Ԛ��^��{ջ������0�`{�
+ ������pxC�.F��px�� .F���HFQF��
+���9D>�	��9@F�������?��4D���T�#�s��vx��0��0	(�F���0FbF.F]�pxC�.FX��3���3���Ԙ��Z��O�
+*ջ��������Z��'���0��� ��X���0 4�����px����vB�.F#�^�g�������Z�^��������X�u���0�p��!FO�	���$�����S�F������]���pG����JK�B
��CD
H!�1FDS��BB���H	IO��B��@�+����ǻ��0*�������������������������� �0���� ��A���`pG�@���hpG#AB��X�JB(��0@��`@�@�a@`pG#AB��X�JB(��0J@B� 0pG���#AB��X�JB(��0J@B� 0pG���JhC�`pG��JhC�`pG��� ���� ���Kh`���D� ������.���6�������FH���������B���@ ���� ��0���@ ���� ������@ ��� F��,���@ ��w���! ������@! ���$ F��{� F4����4,��K��	Jh�Bн�@��� F��h� F��y���@�������# ��Z���@# �����$ ��P���@$ �����( ��F���@( �y���) ��<���@) �o���* ��2���@* �e���+ ��(���@+ �[���, �����@, �Q���- �����@- �G���. ��
+���@. �=���/ �����@/ �3���0 ������@0 �)���1 ������@1 ����2 ������@2 ����3 ������@3 ����4 ������@4 ����5 ������<���D�������0���T���@5 ���� ��B���@5 ���� ������@5 �����Khhh���pG�Khhh���pG�KhS�  h#�`pG�KhS�  hC�`pG�K�DKh�B�K���B�pGKJ`pGKJ`pG���)�&"K`Z`�`�`aZapG� K�DKh�B�K���B�pG	K	H
+I
+J�a�abpGKH	I	J�a�abpG�� \)$))�)�)�)
+Jp�j�A��mhT�1�h-h��Pf`�``p� pG� Kp��i��EyYk�i�i����@��@$��@hXh!C��!��@�h�h
+C���i"��@C1�[i���@ �(Cp�pG Jy�ip��j
hHh��@��$�iMi�@"C����h"��@�hC�� ��@Cp�pG K�@j�R�0 [h`pG� Kj��Chh�pG� 
+Kj��	hhJ@�� pG�h�hhhX@�� pG� �Dh�����Y� x�!��O�0�$��U�� ��X��! D��N�Kh,���M��� F�~���@ �y���8�7K7JhF���|Dh+Y�(��b�������� h.K@���O��� F��.�F F��X�!���!(F����������� �B��\�#x��Kx+���F�u ������K�DKh�B�K���B�� ��]�(��J hC�`��{� ��b�Kx+�ѽ�8@ �������Kh+���������� �D�8�"K"JhF���|Dh��(����#x�(������y� hK@ӱ��O�Q� F����F F����!����!(F�����K� ���Kx[���F�u �6��������:��� ��8@𲾜���8�!K!JhF���|Dhs�(���#x�%������+� hK@����O�� F��`�F F����!��A�!(F��=����
Kx[���F�u ��������������� ��8@�g������8�KJhF���|DhS�(��K�������� hK@˱��O�� F���F F��A�!����!(F���� �1�	Kx���8�F�u ������������Ch؈������@ ��8�F lh��(��������� F����F F���!����(F!����#y�+� �~���8@ ���pG�p�F��H�(D�"M#L+h���h+:ИG0F����khx���8�h���+h���|��d�$ F���� F4����4,��K��Jh�B� F���� F������`���� ��q�5 ��n���p@ �����pG�K�DK�h�B!�K���B!�����F ���� ���� ������;��G��1� F��p�����@ �u�Kh+�� ���� ������������F ��{� ��x� �Y����� F��F��v���@ �K�����}�F ��_� ��\� �=���� F��,�@���@�Y�����e�F ��G� ��D�����>� F����E���@ ����=���@ �����E�F ��'� ��$����� F�����%� �����@ ���KFhA��q`h���pG$@�DKFh!��q`h���pG$@�D"�K`Z`��Q���@����
KHhI�J��4``hJ��#`�N�����@�����@����D��@�D�J hC�3`pG�p�DJ h#�3`pG�p�DJhC`pG�p�DO�qKJ`hC`pG��D�DJh#�s`pG�D8�IK�t`d#h+,�ch+/�#j+2�#i+5�ci+8ѣi+;ѣh+>��h+A��i+D�K��9M*h�BGУj+d�#k+]��j+M�K��*h�BP�8�! ����ch+��( ����#j+��3 ����#i+��- ����ci+��. �����i+��/ �����h+��* �����h+��, �����i+��2 ����K��M*h�B��ck+�� ����O�qKJ`hC`��0 ����K��*h�B�ѽ�8@4 ����$ ������# �������D��D�D"8�"K! `����( ����) ����* ����+ ����, ����- ����. ����/ ����0 ����1 ����2 ����3 ����# ����$ ���K��M,h�B�8�	J h#�s`��p�+h�B�ѽ�8@4 ��h�D��D�����+����
Kx������� ���� �z� �w� �t� �q���@ �l��� ��� �}� �z� �w� �t� �����O�Kh+��q�KxC����K�D	Jh�B����U����@� ������8�O��s�B�F ��F ��� �?�F
� F8� �� F8�$ F8�����Kx��pG��Kx��pG��K��I�hINMLH	h6hK-h$hh�B`^`�`�`a�IJ	hhYa�a�pG��D��D�D@�DD�DHH�DL�D������Kh#�
Kh;+
�JK
O
NMLIH:`2`+`#``��
JhC��S`������?���@�DD�D�D�D�D�	���D8�����MKLHIJ+`#```h+�+�8�O�`o�bIK``8�%O��0
JhC�s`D�<\A�@8��@�D�?D�DH�DL�D��D�D`�DK�DKh�B�K���B�pG���������K�DJ�h�B�K��KHh]hL�h`�h%`iM�`.`dh�BњiXiIK``�pGKh+��
Kh;+��Ih#��S`����H�D�D@�DH�DL�D���D !	KF``hA�@`h!�A�``h���pG��DKh+�"KJK`pG��D�	Kx�� �F�#IJ	h �@`��@�M�����q�D�	Kx�� �,�#IJ	h �@`��@�3�����q�DKh+�JhC�C`pG��DKh+�Jh#�C`pG��D J�h,:�#hY7�Ph���P�3P��.hJh2C*`P�3PJi.h2C*`P�3`�h�h2h=C*C2`P�3P
+i.h2C*`P�3P�i*h"�*`P�3 �ih#�`Kj�j	jChB��T�?Z���pGd#J�h.@�3hY=жFUh���U�4 ��h�iC`j�jhB��U�4 �ihC`jBjhB��U�4i
+h^�?"�
+`U�4p�h�h:h1C"�:`U�4Gi
+h"�
+`U�4@h
+h"�
+`Z����dK���Zh����j� h �A�!`k�`h+��]�KpGd
+K���Zh��h���j�)`�pGkh+��pG�dK���[h��Ckh��pGd"K�`�`aZa�apGdK�DKh�B
+�K���B�pG	K	I
+JY``pG	JK	IhY`*��J`pG��d''�l%\%�K��
���L�#h+����As��S�S�����&@�DK�DKh�B�K���B�pGKH	I	J��pGKHI	J��pG���`(8(0(�(�(�(K(hO�S�! h#��C�C�`pG��8�F	hJ�F�8�(F����T�K��8�8� ���KhS� 8�h�������@	pG�KhS�  h#�C�`pG��KhS�  h#�C�`pG��K Yh��K Yh��K �h��-��OO�	FFhJiK���������fMgHgI���(�-����	�`������������sx�{������G�'���%�FHF����-��5/�;F#�������rRQ��)@��!O�r�;��+@��#%O�BO��wBH�BM�8+����>KAI����x*DМFH�k���������Fl09O�zs�;P�
���Ԯ�pfxU��`����� 2F������#D��p�K�B���5O�zrt�[�:�������O��rO��0 K��������pr]*�� ��PKI�J
���PL���P%���@���0!��<B��������O��rO��0K������#��!"u牲s盲y����D���D���D���D���D(��D,��D���D���DhZ5��F%Jh`�W�#@O��r `�:�#h��C+��V�?j\�F���
�<�B�(��"hO��sW�"`�;��
+h�@?��<�B�� �pG pG��K�DKh�B�K���B�pGKH	I	J��pGKHI	J��pG����&�&�& %%�$"IO��s	hQ� 
+`�;�
+h�@?��pG�#IO��r	hQ� `�:�h��C+��pG��KhS� 0h�������@	pG���K Yh��\� ���K Yh��R� ���K �h��H� ���K �h��>� ��Booted version 0x%x of CM3 PM Firmware.�$�$�$�$�$[%08x]%s:   ERR WARN INFO DEBUG0123456789abcdefghijklmnopqrstuvwxyz(null)0x
+	��������L�D(�D��D�D��D��D��D��Dd�D �D`�D�D,�D�����D� �D��Dp�D��D�D� �D��DH�DT�D�D� �D��D4�D@�D��D �D,�D��D\�Dh�D�D�D
+	����
����8*�D ��D0��Dh+�D@+�D(��D ��D@��D���D���D ��D ��DP��D8��D ��DK�]�a�e�����D� �D�-�D�-�D�-�D�D� �D .�D$.�D,.�D�D� �D�-�D�-�D�-�D`-�Dd-�Dl-�D -�D$-�D,-�D����
����)�D��D��D��D��D��D��D��D��D��D��D+�D����

+	�����DP�DD�D@�D�D�D�D�D�DH�D,�D�D�D�D�D�D�D	
+
+	@�D@�D�D�D�D�D ��@GNUs	��i���dW��7#=ub�Qu�Qu���y���y��������|E!|E!�p�Qp�QXy�1da�� � ������������������������������log_buf������0*PP�%`GP9�!	�olz��@5	]N
[xS�M��NPQ��RPgSPCVP�XPYP�ZPO[P`\P�^P)_P0`PzaP�bPecP�ePTjP	p
+�qP
+crPHm6n�
rtco�
raws�vs
+EwW
+4xs
+Cyy
+�zy
+�}�
+�~s
+�s
+���
+M��
+��%
+��%
+h�%��6����Preg�Q��E��Preg���]��P���4�PE!Q�۽"(�92.F`$0�4$�Y4� �4�-z "D�O!.P!9Q"0�T�~#�T��$��%h,�"����Z%id�Z���Pc�Pj�Pq�P&�����&��0�'&��H�1'&��`�J'(�)P�*int"��X,�&"Xxќ9�.&"^���9.-&"b���9B.V#"d��9k. E��gQ����D�"��9�.�"������Z�+��P���,"�
+�9@.T"��Mj�Zg+��P����*�"�
+�9�.�$��%��$��%��,A-�7P.X6�/��0�~.q)�.�Z�.�%�.Z�.�%�.�%�-�-�-0-O-x-L-�-9-41jjF����U*`J�int��0aHm3,V��verhnum hT!��"�h�	�~h�
+��%�&h4'V
+��,*Mda+hlen,hT-h
.MV]	��D1��2z�4�*5��6h�	��H�l��iH�
�
���Pt
$�
(�
,
06-PO<AP5
@(F3]P0R>�P3QT$R
+�\�Pt�GHh
+��
n�v5,�9]���+��,��)���}}��9��:88(�������	
+ ����t��*`���UJ�e��;0�	�
+��
$�int����	�t
+��
+�z�P�P
���u��8��P
+�Y��P�P��dP"��P"
�&����&�z���P�P���P"��P"
j/�X��i1����
+N!
+����P0��5Pt��IPt��
��nPO���P5��P;���Pt�Pt�R�v5,���qq*88(��)}}��9��(9(��z2!%��]�O���H0[g���s�Pc��n%cs�2	int
+�Q`%0��s�Pret
%{
+��b����D�H0[n	~D<�~c	~�n\PwRuwint�~���str��	��
+
+�>�Q��~�.�(str�A
�%�	���
+�>�9?H�?v?gg������x�[���l��lvl��fmt��t�%!ret�4	ap�d
+��P�$QtR�P2$|$"
+��P�\Q�`�P:
intQ��		)8�Blvl)�P��T���tt~��|$d����nno�Q	�DV[�>*^^�%`1	R��padR�qR�	retT�
+intQ	B��L	B�numB^�B�n	B	C3	pE�	cE����Z�D8��
fmtZ�R
apZ��L	`���pa���	a�chb�)nb�O
+numc^�
+�d�Ttmpd�h	d��g	e��
b	f��9	g3�reth�-�i%��	y%	
+2	>	�R�	r��R ��(�]�u�����R���sepA���up�|FPxe����u��
+	FPxe�	,W��u�	��	FPxe�	9��(u�	��	FPxe�
+���^u��
+FPx `F!F�P- HF �F NF!�Q"Py!	\<PyQz l	\!x	g\P�$!�
+\vPyQz*FP0�"�#��$	�%&��'� ~(�Q�|� )fmtQ��p*+apS��d+retT�P�P�PQ�T&�0'�$s	;A�$" ,nn,``,��,���F
+��	��L�*`�	Ȝ�c��H�}�	�}��	�}��,	�	,	�	,	�	,	�	,
+���E~
�	\�����
+��N�
+��^
+��L
+��;
+ ���
+&��
+,�p
+2�,
+8�CH�p�(�*`����
+Q�
+9�
+[��
+�
+	
+��
f.�
+$int����;	P>
+C �val,Preg���
+ ,,�;reg ��
��.5���8�	88(�f�f,�fM�mSint`�`(���fv�f��mSq&�(���&f��(f��)mS�1���3mS�;���=mS���	
��Fj	*`���=
R�:oP@�EP@int('�^�^�:�PO	�P	�[	
f	
+
q	
|
+
��PO 
�P2
+*
�PO0
�'P1
+:
�:PO@
�NPtJ
�POJCL
��T
:�P!^
EP![J`
D�iL
+j
:�Ptr
��Pt
�
|�
:�Pt�
�Pt
�
|l]�
�W�
:FP#�
EP#}d�
���
:�P$�
EP$g
k�
���
:�P(�
EP(x
r�
��
:�P)�
EP)�
y�
�?�
:.P*EP*��y:hP+EP+���$:�P,.EP,9�0��8:�P-BEP-V
�D�'L:P.VEP.E
�X�a`:PP/jEP/,
�l��t:�P0~EP0������:�P1�EP1�����:�P2�EP2����I�:8P3�EP3������:rP4�EP4���F�/�:�P5	�P	�[	�f	�|
+���P5���P2
+�P5�P1�P5v5,��)��00�����������88(�������qq*Qy
��
���
+ HH�
�
`�
�qid%P'!,��id!%P�
'@��id'%Pval)HS�
1X��id1%Pval3HS�
;p8�	
+Hv5H
4?)�
O))@�y�9*`���UJ�e��;0�	�
+��
$�int���~	$
+P� %	����	+
+8!%	���
+^)'�$	��������*`Q0�R,gS,CV,�X,Y,�Z,O[,`\,�^,)_,0`,za,�b,ec,�e,Tj,C,T���0"��#,�$,�%,�&,�',�(,�),�*,�+, #,,$m-,(�.,,�81��2,3,<4,45,L6,`7,,8,�9,�:, �;,$�<,(#=,,m>,0�?,4�B��C,qD,o��,�,�,,.	var,
+�,
+),Em,S	valm,	vm,Rg,x	valg,	vg,�a,�	vala,	va,�[,�	val[,	v[,�t,�	valt,	vt,�U,	valU,	vU,�O,1	valO,	vO,TI,V	valI,	vI,\z,{	valz,	vz,.���	pd�7��,t�,int
1(��
C2�X��?,0�val?,Wpd?7v�,H\��4���v�,��H �c���SR8��nc��RPi"�5.�h�I>I��p"k3��,�H�L4��~v�,�V���q�f�
���npd�7P�,��pd�7�l�� 0�$���{ ���� �
� {6��6�S�3�58; T�Q<W �m@s �!��"�~�!x #v5,#;#�W!s�$#� �#�;#W#x� �|��=��*`�ff�����int�8"�$�$Y $�!$�"$�#$1$$'%$�'$ �($$�)$(+$,y,$0�-$4	92
+2~m�Ol%"	dd
+2+eOz\%Tv:���
 @@�
�
`^WW~@t�&G�����Pvj
�0*`G79�!	�olz��@5	]N
[xS�M��N7Q��R7gS7CV7�X7Y7�Z7O[7`\7�^7)_70`7za7�b7ec7�e7Tj7p�	�q7	cr7
+Hmn�rtco�raws�vZ	Ew>	4xZ	Cy`	�z`
+
��}�	�~Z	�Z	���	M��	��%	��%	h�%��

�
��cc�
+Q�
+9�
+[��
+�
+	
+��
f.�
+$int�c�UJ�e��;0�	�
+��
$�C7����c9����� 7\�
�
�!P\�)4!�F�#)g$7zZ^v�P@<$Q�|!�P���Q��,�7�BPt�BP?
��P����4P���R���S7DT7P�Ucc��M�X�c�Pt�n�Pt�y�Q0y�PuQ1�
+,�B,P17.XJP
+uj4�N�fP1b�yP0h��P0zB�P?~�
������4���������7�D�7��M�X�c/Pt�nCPt�yVQ0yoPuQ1�B�P1X�P
+uj"�(�4BP?K�D���4��/�ͩ���7�D�7�brM�X�cBPt�nVPt�yiQ0�y�PuQ1���X�P
+uj�����BP?k�����	4������TD�7&��79�MXc@	Pt"nT	Pt(yg	Q00y�	PuQ16B�	P1HX�	P
+ujN��`�
+4�W��xnMxBP1�xR��
+4���!���"7�D#7���j
+P3��M�c�
+Pt�n�
+Pt�y�
+Q0�y�
+PuQ1���
+P5�BP1�;��4;�P �A���hAc!iCcB����iPv�
+$�Pt,)�PtB�PtH)�PtL4P?VJ�PO\J
P5fUP;"$ut���xp�
��c��`�kqP1�k�P0�v�P?�����������Pt���v�P1���P0#��P0���6��
��c��`�kP
P1�kc
P0vv
P?�
+���
Pt�vP1�� .�D��c�&`.k�
P14k
+P0:vP?>�D�:Pt$N��P0����c�V`^k�P1dk�P0h�l�r��Ptv��vP1,������vP1���6���c��`�kIP1�k\P0�������Pt���v�P1��P3%��&%'v57'�67'X�(g�)'q��'��%*wwF*��	G*CC	I*�
+�
+	H*��,*��+*WW)*//@*]]?*N*��O*��J*88-*��0*QQ<*L*�
�
*''*''C*~~3*��)*���*�*��=*FFG*��D*��
+)*qq
+**}}*��9*88
+(*��(*llQ*

K***��,*ZZ1*		+*��.*���*vvA*�
�
*�
�
*��-*��4��������0*PP�%`���UJ�e��;0�	�
+��
$�int������� P3�
�
GP9�!	�olz��@5	]N
[xS	�M�
+�NP	Q�
+�RP
+gSP
+CVP
+�XP
+YP
+�ZP
+O[P
+`\P
+�^P
+)_P
+0`P
+zaP
+�bP
+ecP
+�eP
+TjPp��qPcrP
Hmn�rtco�raws�	vOEw34xOCyU�zU
+�P�varP�P)P~u�=
w�&��=
��AB��D�WP��R��������,�>�;��T���DL��varFPd8�<@DL%QV�`��vXP�\fZ �� w� l��a�t�JvcP�\ze �� w� l�]l���!�l��vnP	\�p � w/ l	�����E�����X/��8�@!���|"���,#���$:0@%P!$F0T%P($R0h%P3$^0|%P-$j0�%P.$v0�%P/$�0�%P*$�0�%P,$�0�%P2$�0�%P@$�0%P0&�0%P4$�0/%P$'�0%P#v����(�p��x)���$;�%P!$;�%P($;�%P)$ ;�%P*$&;�%P+$,;�%P,$2;%P-$8;%P.$>;,%P/$D;@%P0$J;T%P1$P;h%P2$V;|%P3$\;�%P#$b;�%P$$�;�%P@*�;%P4'�H��	"��
+�		#�
++����F�Q�\�g$�r@	%P1$�}S	%P4$�}f	%P2$�}y	%P1$�}�	%P0*�}%P3,��p��
+-���	)�+�6$���	%P3$��
+%P0$��
+%P1$��'
+%P2$�:
+%P4$
+�M
+%P1��$�@�'F�%P1.F2�T:�-/�2U�0n4-1ret5�6)�2�8%w$f��
+%P4$n�%P4$v�%Pt�� '��%P43H���\+������~+��3�4PD4v6P�4�7P�4�8P�5X�5���5��%611G6ss86��(6��'6CCH6��96�
�
688	(6��	)6((
+6��
+6
+6��
+6�
�
6~~36��46�
�
666
+6��
+6
+
+
+6
+6��(6ll*6��6��)�L��*`���UJ�e��;0�	�
+��
$�int���~	$
+i��&	����	+
+w#�&	�
+1,�&	�n��������0*PP�%`���UJ�e��;0�	�
+��
$�int	"=
+:#=
+�$=
+�%=
+�'=
+(=
+F)=
+�*=[�/�Zd
var\PD�d�$>l��var?PSDL�x��X�@�X��D�z�$�Ls
+� ��#var�PS6��0�Hvar�P���� ����4��[�P3"fP3�04��@[�P3VfP3(�d��var�P���var�P��-�Hv5P�6P�����%����(��)�;����86*`�nn�����int�8*�,�,Y ,�!,�",�#,1$,'%,�', �(,$�),(+,,y,,0�-,4	a!M
+�!3var#,�v����
ixn�*�f{6��fA 	SJ�
+�J3varL,Z$��
+
i�n� �8���� ��� ~��4�/��3� ���0�T��3!��,����3)!
val�,J!s�4��
i�nb!��HT���,�8�����h��n�!~n��)h/upEdK3,`�Plv5,�6,/����K�e�����q��|����P*`�ff�����int�8"�$�$Y $�!$�"$�#$1$$'%$�'$ �($$�)$(+$,y,$0�-$4	92
+2~m�O'"	dd
+2+qOz'T0d���`*�MG�NQD�RgSCV�XY�ZO[`\�^)_0`za�bec�eTjpe�qcrHm�nG	rtco+	rawsD
+�e�
+�nep
+0e�
+O-e�
+x@e�
+LQe�
+�ae|
+9~ed
+4�eX����� ��0*PP�%`����
+Q�
+9�
+[��
+�
+	
+��
f.�
+$int	P

+varP�P)PV,0
+reg,�
var.P�5S
+reg5�
var7P�>�y�>�
+id>W��T�S( (�Uc�!nQ�cn
2 A2 $"�8 
+1��"0B C<B GS�B :��qH�P &���H�"idsH�k"retJ��"iK��"f SPu �!W"�Q%x  ��idQW#varSP0##W^� �t$id^WP
� `_#� $|#�� 1�#��#�|##c� ��$idcWP0� e<�#� G�#�� :�#��#��##�h� �%� UP1#�m� �7%� UP2#�r� �\%� UP2&�m� s!P&���&3��'v5P(s�)'��!�(��)'��!�',�!�'c�!�'B!�'�!�/0���*`����
+Q�
+9�
+[��
+�
+	
+��
f.�
+$int���~	$
+c�`(	����		+
+B!8(	���
+�/-0(	/���u*`����
+Q�
+9�
+[��
+�
+	
+��
f.�
+$int���~	$
+���(	����
+	+
+�!�(	���
+,0-�(	�B��C::��`�0"��#:�$:�%:�&:�':�(:�):�*:�+: #,:$m-:(�.:,�81��2:3:<4:45:L6:`7:,8:�9:�:: �;:$�<:(#=:,m>:0�?:4�B��C:qD:��\)A#�$)�	��
+�~�x4)������� �5*`�nn�����into,�val,�reg,n		1��reg1n
+D6	}=n�ret?ni@n	�NnretPniQn
�T�3b�bn
�dn
�en
�fn
�gnsclhn
%in
jn��n� ������$len���$����$e��X
*��
���
���
 ��
��u� �'���!0�D��$�!X�a��$�!x�~��$�!����%�!����)%N!���&>%�1h%<G�%R�%]�%h�%s&u�!�;�h&�}&u�!0�]��&�u�!���&�R!�Z!�Pyu�!H����&��&u�!h����&��&��!��u�!�9�'�%'u�!��B�:'�O'u�!��`��u�!��~����!�����e'��'��!I�u�!9��'��'��!@C�u�!X��(�+(�"�Qu"9�e(�z(�"p��p��(�( ��!
�""U��(�,"�^u,"�9�)�3)uH"����U)�j)uJ"��#�~)��)uT"�I��)��)u\"(�o��)��)uf"ԕ��)�*ul"��"*�7*"�H#�[$��;$��6v`!�P�k @@�
�
`^WW~@�
t�)G�!�}��"���0*`���UJ�e��;0�	�
+��
$�int�9�	reg9�
+i;��/�	reg/�
+i1�kC%	regC��H��"|��
�H�K*idsH��*iJ��*��"N���*�"�
+�#R�`+#�z+�>�D#T��W�#$�!idW>P��#@Y��+@��+�\�#(�kid\>P��#`^�=,`�Z,la%�# ��ida>�,��#c�,8k�$��$P0�s�$��$$P2�{�,$�'8$P0	��@$�PL$P2Ia�g7?��e�� v57!g�" P��!��" 8�� ^�� i�� w�� 1
+��#����C::��`�0"��#:�$:�%:�&:�':�(:�):�*:�+: #,:$m-:(�.:,�81��2:3:<4:45:L6:`7:,8:�9:�:: �;:$�<:(#=:,m>:0�?:4�B��C:qD:��)A�#��)�	��
+�~�2�)�%:;I$>I:;(I:;:;
:;I
8	:;
+
:;I8:;
:;I

:;II'I'.?:;'I :;I.?:;' :;I4:;I.:;'@�B:;I1XY411XY1:;I���B1 .1@�B!1".?:;'@�B#1RUXY$.?:;'I@�B%4:;I&1RUXY'1(���B)���B*$>+4:;I,4:;I-4:;I?<.4:;I?/I0!I/1.?<n:;%U$>$>:;I:;
:;I8
:;I8I	!I/
+!I.?:;'I@�B4:;I
��1��1���B��1.?:;'I�@�B4:;I?<4:;I?.?<n:;%$>I:;((
$>.?:;' :;I	.1@�B
+1��1���B
.?:;'@�B:;I1XY��1���B14:;I��1.?:;'@�B4:;I?<.?<n:;%.?:;' .1@�B%U:;I$>.?:;'I@�B:;I:;I4:;I	$>
+I&I%:;I$>.?:;'I@�B:;I��1���B$>	4:;I
+��1I&I
:;I4:;I4:;I?<.?<n%$>.?:;'I@�B:;I:;I4:;I4:;I	4:;I
+��1���B��1
$>I&I:;
I84.?:;'@�B4:;II!I/.?<n:;%$>:;II:;(.:;'I :;I:;I	4:;I
+$>I.?:;'I@�B
:;I4:;I4:;I4:;I4:;I
+:;
+:;
+:;1XY141141��1���B1XY1RUXYU ��1!��1"&I#:;$
I84%&I'!I/(.?:;'I@�B):;I*+4:;I,.?<n:;%$>.?:;'�@�B.?:;'@�B4:;I4:;I���B1I	4:;I?<
+I!I/
4:;I?.?<n:;%.?:;'�@�B%$>I:;((
$>.?:;'I@�B��1	���B
+.?:;'@�B:;I:;I
.?<n:;%.?:;'@�B:;I4:;I4:;I$>$>.?:;'@�B%$>.?:;'@�B4:;I��1���B��1$>	��1
+���B1���B14:;I
���B14:;I?<.?<n:;%I:;($>.?:;'@�B:;I4:;I.?:;'@�B	4:;I
+I&I4:;I?<
I!%$>I:;((
$>I!I/	&I
+4:;I?%$>:;
:;I
8I:;(
:;I8.:;'I 	:;I
+:;I4:;I$>
.?:;'@�B.?:;'I@�B:;I:;I4:;I1RUXY111RUXY1XYI.?:;'@�B:;I1XY411414:;I &I!I"!I/#4:;I?<$!%$>I:;((
$>:;
:;I8	I
+!I/&I4:;I?%I:;($>I!I/&I4:;I?%:;I$>I:;(:;
:;I
8:;	
:;I8
+:;
:;I
:;I
I'I'(
$>.?:;'@�B:;I4:;I��1��1���B���B1���B14:;I.?:;'@�B:;I4:;I:;I .?:;'@�B!4:;I".?:;'@�B#��1$���B1%4:;I&&I'4:;I?<(I)!*.?<n:;%:;I$>I:;(I:;(
$>	:;
+
:;I
8:;
:;I8
:;
:;I
:;II.:;'I :;I:;I.?:;' 4:;I.?:;' 4:;I.1@�B41.?:;'@�B��1���B14:;I.?:;'I@�B1XY 1!:;I"1XY#$��1%���B&���B1'��1(1RUXY)U*���B1+41,.?:;'@�B-1RUXY..?:;'I@�B/:;I04:;I14:;I24:;I34:;I44:;I?54:;I?<6.?<n:;%$>I:;((
$>I!I/	&I
+4:;I?%:;I$>I:;(I:;(
$>	:;
+
:;I8.?:;' .?:;' 
4:;I.1@�B.?:;'@�B4:;I��1.1@�B4141���B14:;I.?:;'@�B��1���B���B14:;I4:;I?<.?<n:;%$>I:;((
$>:;
:;I8	.:;' 
+:;I4:;I.?:;'@�B
4:;I1XY1411RUXYU:;I.?:;'I@�B.?:;'@�B4:;I4:;II!I/&II4:;I?<!%$>I:;((
$>:;
:;I8	I
+!I/&I4:;I?%$>:;
:;I
8:;
:;I8:;
:;I	
:;I
+4:;I?%:;I$>I:;(I:;(
$>	.:;'I 
+:;I:;I.:;' 
4:;I.?:;'I .?:;'@�B.1@�B11U111XY4141.:;'I@�B:;I:;I4:;I��1���B I!&I".?:;'I@�B#.?:;'@�B$:;I%���B1&4:;I'4:;I?<(I)!%$>I:;((
$>I!I/	&I
+4:;I?%$>I:;((
$>I!I/	&I
+4:;I?%I:;($>:;
:;I84:;I?&I	I
+!I/%$>I:;((
$>.:;' :;I	.:;'I 
+.:;' 4:;I
4:;I:;I.?:;'I@�B:;I4:;I4:;I4:;I1RUXY11U41411XY��1��1���B1RUXY1XY U!41"I#&I$.?<n:;%I:;($>I!I/&I4:;I?%:;I$>I:;((
$>.:;' 	:;I
+4:;I.:;'I .:;'I@�B
:;I:;I4:;I1XY1411XYI&I.?:;'@�B.?:;'@�B:;I1RUXYU.?:;'I@�B��1���B4:;I 4:;I?<!I"!%I:;($>:;
:;I84:;I?&I	I
+!I/SS2�2�SP�P�$P$,�P�<B1���S����@S|�5�@P2�@P	��PR3�PR	��RX4�RX	��XZ5�XZ	��txS|�Slp2�|~2�|~S��P���P���S��P��1���1���P��P���P���S��P��6���6���PTPt�hTP
+�P�P$�P�P�P�$)P):�P�<@0�@JTJMPMNt�N�Tx�0�"Q".�Q�R$r�$P$,S04P4<S<@s�060�68p s"�8@QP V &P&<V<BPBNTNTt�@BPBFTFTUTV�P�X^P^jWjnt�nvTv~t�~�W��t�X`Q`jq�~�q�\^P^�WP�P�.P.l�P� VU:VTVlPPU^@FVFU
+�U��V��v��8UQ,Q�Vv�V��Y��V<HYP,>P>@~���FP
+~���
+P\|#�@BPBF|#�FHPHP|#�PRPRZ|#�Z]PbdPd~|#�~�P��|#���P��|#���P��|#���P��|#���P��Y��Y��P��|#��P	|#�P|#�P,|#�,4P4J|#�JLPLP|#�P^P^n|#�npPpx|#�x�P��|#���P��|#���P�|#�P|#�$P$'|#�8LY��|#��<|#�dtPtx|0�x�V��P��|#���P��|#���P��|#��,|#���0���\��\��R8L\jp0�p�\
+0�.@ZhnZ
+Z.8ZL�Z��Z
+Z$Z��Z��Z��ZZZ,8Z@��Pn@��:�.8��L�����:��
+��$:�dj@�������:��������:�,8����S������S�@��.8��L���������S������S������S����,8��@0�@��
+0�
+������$@Y@p��pxPx���������V�.��8d��djSj�����S���<��<dVd�����P������V�8��@R
+R@]Rb�R�	R:RJRRn�R������R��R'R8LR$<Rj�R��R��R��R��R@ �
+ �
+0�*W,.1�.8W
+0�QT,ZQZ\T\�^��Q��t���T
+t�
+T<>ty"�>@T@]Q]bTb	Q	
+t�
+T>Q>JTJ�Q��T�"Q".T68T8jQ��t��"���Q��T�
+Q
+T<Q<dTd�Q��T�Q,T@S
+S�Y@]Sb	S@SJhShnYn�S������S��S�YS'S8LSLPY��Y�
+Y
+S$Y$<Sd�S��Y��S��S��S�Y$SFP~0.�P\r0.�\�q0.�F���F���F\Z\^R^fS��S��RF������S�����F\���\^V^jv�jV.8VL�V��V��V��V,8Vj�S,0Y02V28v�8<V��V��[��{���[fjSjnYnty�tzY��S��Y��y���YJPYPVy�V\Y S &s|�&(SQq����{�q����{� P $p����{�$&p����{�&(�P�P(�P�p �(�P �(BPBP�P�(Bp �BP�P �PjPjx�P�Pjp �jx�P ���0���T��P��t���T��Tp�P��qp�Q���Q���P���P���0���
+p�3&����u3&����p�3&�s$����P#�3&�s$���s���P#�	�$3&����Q�
+P
+D�P��0��p���S`hPhx�P�x�0���Q��R��PP\�P�T,0	t�p"�\vPv�U���P���U�*�P�*-P-0U0X�P�r$T*XT��P��U�$U0XU��Ps2!�SXpPp�U���P���U��P��U���P�n�T��P��U��U��P�P:U:Z�P�ZjUjmPmpUp��P�
+vT:<P<ZUpvU@EP��P��U���P���U��P�U��P��P��UP(�P�p�P#(.P.LULz�P�0tTLNPNtURWP��P�V�P�$V��0���T��P��t��TDFPFtT��P��T��P��T
+P
+*THJPJpT
+q@D$!�
+Q"	q���w�"0QR^SdjS��S��@=$���S��0���@=$���P���P���S��P��
+�����S��s����SPl�P�lnPnqDq8�P���S��s�����S��P���P���P���P���0���P��T��0���T��U"S
+QQ��S��S0�S $V$,R.0V0HRHJvJLVL`R`budlSlnr��0���S��Q��S��Q��S�R2P2P�P�PdPd��P���P���P���p����0���1���2���3���4���5�sHJ$�"sHC$�"&ST^P^|�P�dhS|�P��U���P���U��P���P�|�Q��T���Q���T|�0���P��0���P��P��0�|�0���0���P���P���p3���P��p���p2$�"��S��1���3���p2$�"��S��2���3� P LTLZVZ�T�t�Rr�"RNTt���t���T��RPTtZ�W��W>��>D,�DN��NR��RZ��Z�w
+�����w
+���`bPblUflUv�U��Uz�S��S��sy���S��S��S��r{���Q��r{���Q��r{���Q��r{�������0�������������������0���	����(���	��������0���	����
+�s���
+�s���
+�s���
+�s���(���(���	����	��T��$T$@p@D���DJNp(�	��8D	��JN0�&,
+�r�,.
+�r�.8
+�r�JN
+�r�.2$�8D(�JN(�8D	��JN	��T`��T`0�`d��`d�Ldl��dl�Hlr��lr�Drx��rx�Tx�,�x��PnPnr�P�rtPt|�P�BQBpu�pr�Q�r|Q0�2U2BRr|0�s2$w"0�1� @<$r� "
+��r�",@<$r�LXt2$w"LX0�XZ1�Z\@<$s�\^
+��s�^l@<$s���p2$�"��0���1���@<$s���
+��s���@<$s��p2$�"�0�1�@<$r�
+
+��r�
+@<$r�&P&<�P�&p2$�"�$��lh
+�t��;(D���x7DV��L��=(�8���F|#��$�%��.�/�/Pv�A���N�O��qS�8OW�X[� `Ra�b�d� ��k�k�"�p<>@B|~��~�����������@DHJNPFHPRJLVXLNXZ�hhrDDHJ\����������������������RXjp�������TVXZ^n8<48<>"&>D*,DFLN .0NR "02RTXZZ������������������������������������������������������������������&<@DJN&*.88<@DJNTVZ\^`VZ\^bd`bdhjlhjlr�����������Qv�
src/sys_execsrc/includemsg.cmsg.hprcm_core.hpm_state_data.hstddef.hsync.h��./�J�J�.KQ�s q  |\�#j_!J
 =">!P.1 -";O ).W *  r 
 "&//1Yu /JwI�B>.B �f�.�.���./�J�J��.!�.�X��./� �Xuwu^�
src/sys_exec/home/a0274052local/toolchains/gcc-linaro-5.3-2016.02-x86_64_arm-linux-gnueabihf/arm-linux-gnueabihf/libc/usr/includesrc/includesys_init.cstdint.hrsc_table.hprcm_core.hcm3.hmsg.htrace.hsync.hdebug.h�
+ 22!-2j0001=>3>ke.	h�'2�m�
src/sys_execsrc/includesync.cmsg.hcm3.htrace.hprcm_core.hhwmod.ht!/Az /
+ >-@!1>-N$"=#+5k01=CJ<=$.=f@+�
src/sys_exectrace.c!X>�
src/libsrc/includestring.ctypes.hK-11�u<�
src/libsrc/includeputs.ctypes.hD	!!"$!","0z.	� "=-[1 "t/-w�V�
src/libsrc/includedebug.cprintf.hputs.h<built-in>�%%%/ sX?�!"!>?0!��c�
src/libsrc/includeprintf.cputs.hstring.hstddef.h<built-in>D�
.s .t  =0+�.	t�a�~J�  /0= �</=0�~.�g�>�JX=LMY",>2!1.=MMM/-03JJ�~t=;�.�~.� ?�.� �.)&(/!"=K!l<Dx.	 0u. v.,# Yc.#.-0\�" K-0I$��>�.K>.C.> "==�.�.>=��� 2K3y &-//0�.� "�.</"L�~.Y;�<I!z J�.�   � �.0K��2=/�<_y (j �~X\ ��<</�~XK;�</N�.�~XK;�<0�~t� Xy�SJ000/IX�t�.� �JF.<g. �<�~J�.�~ =;�<x x<E<0;!:^ �<�.�.LjJ000K�.� �<00/t�<!�~J/j.�.=X=�.� �<=��<A!2S/�
src/foundationstartup.c��_.�,",	 
�
V:�
src/foundationexception_handlers.c�4444444dH�
src/foundationsrc/includertc.cclockdomain.h!]1jYV+�
src/foundationcm3.c8�r"O�0��0�/0N/0,��
src/foundationsrc/includeext_intr_handlers.ccm3.hpm_handlers.hmsg.hsync.hprcm_core.h�"=A/]!>0CMA2-v<<-s<<-y<D-O!>-O$"=#+3l-*=@-]!>-O!>-O!>-O!>-O!>-O!>-O!>-O!>-O!>-O!>-O!>-O!>-O!>-O!>-O!>-O!>4?B2-y<D-u<<-�l�
src/pm_servicessrc/includeldo.cprcm_core.hldo_335x.hldo_43xx.hldo.h>=M>=MY/\Y/\hYH
PJ�
src/pm_servicessrc/includehwmod_335x.chwmod.h7��
src/pm_servicessrc/includepowerdomain.cio.hmsg.hpowerdomain.hprcm_core.hpowerdomain_335x.hpowerdomain_43xx.h�(=!!!!\jY/!!!z./!!!	."//-!!/!!#:fu u  !� �t� �J� �tMn k  (�[�gfv/-!"V/"XNH�
src/includesrc/pm_servicesdpll.hdpll_335x.cLF�
src/includesrc/pm_servicesldo.hldo_335x.c4��
src/pm_servicessrc/includepm_handlers.cmsg.hclockdomain.hhwmod.hpowerdomain.hdpll.hldo.hprcm_core.hrtc.hcm3.htrace.hstddef.hP!!#?0&z !Ah>LZ11	<y.f&z<&S%1>[KNKA?M010>1<UJX0fLZ$">/AN-H<'<<%7%T$#??[KNK??M0>pJxXz< <#+�%7%T$#??[KNK??M0rJxXz<<#+�%7%T$#>[KNK??L>sXvX�#i-@ %$0>ZK??LY?-O4 	 K�2">K0>ui. =",4g=?01=@-�.9#<P>=>>0000>1-g<L= �#>=>>00>1-
+J#>=>>0?#-	<#>=>/0>1-C!1-	<#>=>00>0?-���
src/pm_servicessrc/includeprcm_core.cio.hhwmod.hdpll.hldo.hmsg.hpowerdomain.hclockdomain.hcm3.hddr.hi2c.hstddef.h��$">=>$">=�}<!/"0-\"#&/!"!L///0-�'y @ �.#M'y � �.#M�.�<\?#!�1	f!#+#M????????f???Mf>=>=>=>=>=>=>=>=>f=H<?#!�<>Z-z<G
�z &##/=============>�t� � #�<1@+�!00�.�X0>===>-\!====>>1M<.H<0�-w.�$YK=Y%*@u. [�PJ�
src/pm_servicessrc/includehwmod_43xx.chwmod.hSg�
src/pm_servicessrc/includeddr.cprcm_core.hmsg.hhwmod.hstddef.h�/y.!'z !"!"z ""!"y !!"!"!!!!"L#1<`$!#!&u.!#!&o.//�#1/!!"!!!"D>4z.&z &v.!)//"/XhY,^8$<y.=/$$"!">/!"m.KL/0f# p ##!?"/-!#=N?,0�!M>L1-�!M>L1-�L//kL//p�
src/pm_servicessrc/includedpll.cdpll.hprcm_core.hdpll_335x.hdpll_43xx.h��#�tu/#/!!#/-!/#/!!#/!/#/!/[6<LN#L�u/?@/!!?@/" _./#/-!=#/!/#/!/ kI!ge%=[�:">x�M�kY/!)/!/	 !![(x !"?0/	NH�
src/includesrc/pm_servicesdpll.hdpll_43xx.cQK�
src/includesrc/pm_servicesmsg.hpm_state_data.c��
src/pm_servicessrc/includeclockdomain.cio.hprcm_core.hclockdomain_335x.hclockdomain_43xx.hstddef.hclockdomain.h�jY/!zJ/! J!%j.h<#t M #X!;M)D[#u?P.h<J2XT._<#J.Xjj\V�
src/pm_servicessrc/includeclockdomain_43xx.cclockdomain.h\V�
src/pm_servicessrc/includeclockdomain_335x.cclockdomain.h\V�
src/includesrc/pm_servicespowerdomain.hpowerdomain_43xx.c2B�
src/pm_servicessrc/includei2c.cdpll.h� ��.�. s � � �.���.� � �.� )3�X�<�J�.: ><>H"/=>h!>�J��~<�� �<q. k.�.�~.�.�~ �.�~<�X-�~ � �f)`.#.X<�.�~.�.�~<�.�~<�X1�~<�.�~.�.�<)LF�
src/includesrc/pm_servicesldo.hldo_43xx.c���
src/pm_servicessrc/includehwmod.cprcm_core.hhwmod_335x.hhwmod_43xx.hstddef.hhwmod.h�"�93;o h/J�//` h/J3G�jY/!zJ/!8JZ' Y '.Y L/)f_" ^ ".^ L/$�b.</�"[M"[M![M![\V�
src/includesrc/pm_servicespowerdomain.hpowerdomain_335x.cCMD_ID_CPUIDLE_V2cmd_datapd_mpu_ram_ret_statemsg_cmd_read_idCMD_ID_DS2_V2CMD_ID_STANDBY_V2a8_reset_handlercmd_handlerssizetypeGNU C11 5.3.1 20160113 -mcpu=cortex-m3 -mthumb -mfloat-abi=hard -mfpu=vfpv3-d16 -mtls-dialect=gnu -g -O2 -fno-delete-null-pointer-checks -fno-strict-overflow -fno-exceptionsCMD_ID_STANDALONEcmd_idspd_mpu_l2_ret_stateparam2param3param4CMD_ID_CPUIDLE/home/a0274052local/git-repos/amx3-cm3m3_firmware_versiondebug_haltpd_mpu_l1_ret_stateboolcmd_stat_valuemsg_cmd_is_validmsg_cmd_dispatcherdeep_sleep_datapd_per_icss_mem_ret_statevdd_mpu_valds0_data_hsCMD_ID_VERSIONinit_m3_state_machinemsg_cmd_stat_updatea8_version_handlersoc_typemsg_cmd_needs_triggerCMD_ID_RTC_FASTds2_datastate_handlerrtc_datasrc/sys_exec/msg.cCMD_ID_RTCCMD_ID_RESETCMD_ID_INVALIDds0_dataidle_datai2c_sleep_offsetreservedCMD_ID_STANDBYCMD_ID_DS0_V2pd_per_ocmc_ret_statefalsei2c_wake_offsetpd_per_stateparam1io_isolationrtc_mode_datartc_timeout_valmsg_cmd_fast_triggermosc_statevtt_gpio_pinshort unsigned inttrue_Boolpd_per_mem_ret_statecmd_idds1_data_hscmd_global_datado_ddrCMD_ID_DS0CMD_ID_DS1CMD_ID_DS2msg_writemsg_readcmd_handlerCMD_ID_DS1_V2m3_param_reseths_datavtt_togglepd_per_ocmc2_ret_statemem_typepd_mpu_statedeep_sleepmsg_cmd_wakeup_reason_updateidle_v2_datacustom_state_dataCMD_ID_COUNTres1res2wakeup_sourceds1_datagp_datastandby_datadebug_printfshort intmainnvic_disable_irqa8_notifyam335_inituint8_tscr_enable_sleeponexitrsc_tablong long intnamesrc/sys_exec/sys_init.ccm3_trace_hdrnvic_enable_irqunsigned charlong long unsigned intuint32_tsoc_idtrace_initscr_enable_sleepdeepfw_rsc_hdrsetup_socpm_resetbasefw_rsc_tracecm3_resource_tablecm3_tracehwmod_ida8_m3_low_power_syncHWMOD_COUNTHWMOD_OTFA_EMIFHWMOD_OCPWPHWMOD_L4FWHWMOD_L3_INSTRHWMOD_EMIFHWMOD_CLKDIV32KHWMOD_GPIO0nvic_clear_irqHWMOD_L3HWMOD_MPUHWMOD_I2C0HWMOD_OCMCRAMHWMOD_L4HSHWMOD_L4LSa8_m3_low_power_fastHWMOD_ENDHWMOD_EMIF_FWtimer_syncsrc/sys_exec/sync.cHWMOD_IEEE5000src/sys_exec/trace.ctrace_set_current_possize_tsrc/lib/string.cstrlenmemsetputchar_logbuf_endsrc/lib/puts.c_logbuf_startmaxlenputsnputslogbuf_posflags__va_listdebug_levelsrc/lib/debug.clevelsvprintfdebug_idxdebug_set_level__apksprintnupperhandle_signput_padstophandle_nosignnbufsrc/lib/printf.cpadcdwidthlenphex2ascii_datapercentreswitchnumberdummy_handlersrc/foundation/startup.cpulsrc_end_textpuldestvector_table_end_stack_start_data_end_datasrc/foundation/exception_handlers.cpendsv_handlerbusfault_handlermemmanage_handlerhardfault_handlerdebugmon_handlersvc_handlerusagefault_handlernmi_handlerCLKDM_L3S_TSCrtc_reg_readCLKDM_L3SCLKDM_COUNTCLKDM_CLK_24MHZCLKDM_EMIFCLKDM_L4FWCLKDM_CPSWCLKDM_LCDCCLKDM_L4HSCLKDM_ENDCLKDM_WKUPCLKDM_DSSrtc_reg_writeCLKDM_MPUCLKDM_ICSSCLKDM_RTCsrc/foundation/rtc.cclkdm_idCLKDM_L4LSclkdm_activeCLKDM_L3rtc_enable_checkCLKDM_OCPWP_L3irq_noscr_regsrc/foundation/cm3.creg_offsetirq_regextint43_handlerextint44_handlerextint31_handlerextint45_handlerextint33_handlerextint34_handlerextint35_handlerextint36_handlergeneric_wake_handlerextint50_handlerextint51_handlerextint52_handlerextint49_handlerextint53_handlerextint16_handlersrc/foundation/ext_intr_handlers.cextint48_handlerreg_valextint47_handlerextint46_handlerextint40_handlerextint41_handlerextint42_handlerldo_power_upldo_wait_for_onam43xx_ldo_regssrc/pm_services/ldo.cLDO_COUNTldo_power_downldo_initLDO_CORELDO_MPUam335x_ldo_regsldo_idldo_wait_for_retam335x_essential_hwmodsam335x_hwmodsam335x_interconnect_hwmodssrc/pm_services/hwmod_335x.cpowerdomain_regsPD_MPUicss_retst_shiftl1_retst_shiftper_powerst_changestctrl_next_valsrc/pm_services/powerdomain.cam335x_pd_regsper_retst_shiftverify_pd_transitionpowerdomain_initmpu_ram_ret_state_changepwrst_maskam43xx_pd_regspwrstst_prev_valstctrlpd_mpu_bitsl2_retst_shiftstctrl_prev_valget_pd_mpu_stctrl_valresultram_retst_shiftpd_per_bitsget_pd_per_stctrl_valam335x_mpu_bitsbit_valpowerdomain_resetpowerdomain_idocmc_mem_ret_state_changeverify_pd_transitionspd_state_changelogicretst_maskam335x_per_bitsam43xx_mpu_bitspd_statesper_mem_ret_state_changeicss_mem_ret_state_changelpstchg_shiftpd_state_restorepd_read_stateicss_retst_maskram1_retst_maskram2_retst_maskmpu_powerst_changepowerdomain_statelpstchg_maskPD_PERram_retst_maskmpu_l2_ret_state_changeper_retst_maskl1_retst_maskmpu_l1_ret_state_changepwrst_shiftl2_retst_maskam43xx_per_bitsvar_modlogicretst_shiftram1_retst_shiftocmc2_mem_ret_state_changeram2_retst_shiftpwrststidlest_regpgoodout_status_bitret_bitponout_status_bitDPLL_COREDPLL_COUNTiso_bitdpll_pwr_sw_ctrl_regDPLL_DDRreset_bitsw_ctrl_dpll_bitclkmode_regDPLL_DISPDPLL_PERponin_bitpgoodin_bitsrc/pm_services/dpll_335x.cisoscan_bitam335x_pg2_power_down_pllsDPLL_ENDclksel_regdpll_idam335x_dpll_regsdpll_pwr_sw_status_regDPLL_MPUsrc/pm_services/ldo_335x.ca8_wake_ds2_handlera8_cpuidle_handlera8_wake_standby_handlerds_saveconfigure_wake_sourcesa8_i2c_wake_handlerplls_power_upwakeup_reasonclear_wake_sourcesper_stclkdms_standby_wakea8_standalone_handlera8_lp_ds0_handlerclkdms_waketempa8_lp_rtc_handleressential_hwmods_enableclkdm_waketimeouta8_lp_ds1_handlera8_wake_cpuidle_handlermpu_sta8_lp_ds2_handlerconfigure_deepsleep_counta8_i2c_sleep_handlera8_cpuidle_v2_handlerclkdms_sleepds_restoreplls_power_downhalt_on_resumea8_wake_ds0_handlerinterconnect_hwmods_disableinterconnect_hwmods_enablea8_wake_rtc_handleressential_hwmods_disabledisable_master_oscillatora8_standby_handlerpll_bypassa8_wake_ds1_handlera8_wake_cpuidle_v2_handlersoc_revenable_master_oscillatorpll_locklocal_cmdsrc/pm_services/pm_handlers.cddr_io_resumeclear_ddr_resetset_ddr_resetvtp_enableprcm_enable_isolationprcm_disable_isolationdmemdpll_resetenable_iodpll_initddr_io_suspendi2c_writewas_enabledvtp_disablehwmod_initvtt_highclockdomain_initcmd_wake_sourcesds_countsrc/pm_services/prcm_core.cvtt_lowdisable_ioam43xx_interconnect_hwmodssrc/pm_services/hwmod_43xx.cam43xx_hwmodsam43xx_essential_hwmodsddr_am437x_io_suspendcmd1_ioctrlio_ctxdata0_ioctrldata3_ioctrlsrc/pm_services/ddr.ccmd2_ioctrlddr_io_save_contextdata1_ioctrlddr_io_contextddr_am335x_io_suspendcmd0_ioctrldata2_ioctrldpll_power_updpll_power_downam43xx_power_down_pllsdpll_get_divpll_modeam43xx_dpll_regsxtal_freqssrc/pm_services/dpll.cdpllindexget_master_xtal_khzsrc/pm_services/dpll_43xx.csrc/pm_services/pm_state_data.cam335x_standby_clkdmsam43xx_sleep_clkdms_clkdm_sleepam43xx_clkdmsclkdms_state_changeclkdm_state_changeam335x_clkdmsam335x_sleep_clkdmssrc/pm_services/clockdomain.cam43xx_standby_clkdms_clkdm_wakeupsrc/pm_services/clockdomain_43xx.csrc/pm_services/clockdomain_335x.csrc/pm_services/powerdomain_43xx.corig_sysci2c_reg_writei2c_wait_for_bborig_irq_enn2_divorig_pscper_clkoutm2sequencei2c_wait_for_ardyspeed_khzstati2c_fclksrc/pm_services/i2c.cxtal_freqi2c_reg_readorig_sclhorig_scllorig_coni2c_program_freqi2c_ack_allsrc/pm_services/ldo_43xx.c_hwmod_is_enabledsrc/pm_services/hwmod.c_hwmod_disable_hwmod_enablehwmods_state_changesrc/pm_services/powerdomain_335x.cGCC: (Linaro GCC 5.3-2016.02) 5.3.1 20160113A6aeabi,Cortex-M3
+M	
+"����|
04DTh,$��E����k
+����AX,C�H����������|
�lA��h
+A������|
Dt
+A��D�A��H��D�A��G��D�XA��D����|
��������|
0����|
LD<A������L�A����L�.A����������|
D��lA���B���B D
+B���A���A\B���A����8����|
, D8B$�	��������Eh|
+$B0 |A����A�A HB�A��������|
����H����|
����������������|
TA��T T,����|
�8(�`(��(��������|
��A��0��^A��S
+��CE
+��CE
+��CE���L
A��E�� �`
DA��R
+��BH����
A��E����
A��E����
A��E����
A��E����
A��E���A��E���A��E���0A��E���DA��E���XA��E���lA��E����A��E����A��E����A��E����A��E��(��FA��O
+��CE
+��CE������|
hh,h@hXhp8����|
����X�0B���Q����H\B���i�����HD���]������� 0����|
�P\A��g��$���A����c
+����C ���A����C���� �D�A����@�������A�����`A��H�� �xRA����e������$���A����G
+����C�t�xpD��b
+��C��6A��W��� .A��T���P0A��T����A��D����6A��W������|
t
+�t
+�t
+�B��G��t
+LE��U��t
+`t
+tt
+�t
+�t
+�$t
+�8A�����
+����C t
+�B����C����t
+�HA��^��t
+�pA��k
+��Bt
+T:A����t
+�t
+�t
+�����|
 �dD����Z����lA�������xA�����$$ �D����]
+����A�0� �4A��P��04A��P��d�����|
  
��B�������� 
$�B����� 
�4E�R� 
�0 
 
4 
HT 
�8A��BS����|
�
�T�
( (�
P &A�����
x  �
� �
� �
� �
� �
� ����|
8�� �B$�	��������U@�
+$B^
+$BG
+$B����|
 ��"|D����t�����D#T��#$��#(��# �$A���$A���,$A���@$A��.symtab.strtab.shstrtab.text.data.resource_table.bss.aux_data.logbuf.debug_info.debug_abbrev.debug_loc.debug_aranges.debug_ranges.debug_line.debug_str.comment.ARM.attributes.debug_frame�	
+
���������!,041@DPT`h��$Xt����������EW���`h�T$b��tTi���q��,z��Dt��\����(8@|$�|$T�T�$�����D�F�	�	x|�$��$%�$0��������d���8���������������
�
���(,<@TXlp�x
���$������DH���� L ( 28;<D@�L��\%X���&c��P����4D��P`ht���q���}�������L`pt�������������HT���D�D�
+����&�����X��� ������$0Xdx��H�H���� $����04DH����&d�d�hl0

��'��X)��� ( L P 7Q &� � � � � � � � � � � �K�R�_�n��0(����(���)���� �"�����)����"��"|@#D#x#�#�#�#�#�#$$$$($,$<$@$P$�����������)E<�&6�A�W� c!0y�"�	��)8�����H�)����p��H�}'�)%�.>TK�)Z\%u$������"���� ��
�lX
�"0.�07a
DHmY� �c	q%���
�����x��"����"%�(@3�"C�TanE#Ty��9�1�������"����"��4�'��
.�"?5H�"Y�nY,}
�p����"����yp��"�$�.�;�^La(]�8qi,��4��"��T�����U:��F�u
+	�"	�+	� 7	�"H	eV	0*`	�p	Y�	�8�	-$�	�"�	��	�X�	��	�
+l%
+�
+(
+E80
+!�>
+�"N
+$)8^
+�l
+�"}
+�
+�"�
+��
+Y�
+ %<�
+�(�
+� �
+�"�
+�"�($a7�L�"]5h�
y���"������.�14��"��$A�)0/�"@	Q��\�dp!~�6����&�Q0�E�����#$�i
+
L


	.
?
UO
�Xe
do
�"�
�# �
��
A$�
��
) (�
�
�"�
�0y   !�*�;� G�"X�"i�Hq�&�9(����$������l�0(�`(@��
+u2HyR^�r-�����q8���
���%��"E�Q\)� 6��FM
W�"h�~E��|����(,����D��6�\)0�l1#!.7I\M-Z�"k�"|�����u��"�8((��&<��#(��IT#�,�"startup.c$tzero_loop$dmsg.ca8_version_handlera8_reset_handlercustom_state_datasys_init.csync.ctrace.cstring.cputs.clogbuf_posdebug.clevelsdebug_leveldebug_idxprintf.chex2ascii_dataexception_handlers.crtc.ccm3.cext_intr_handlers.cldo.cldo_regshwmod_335x.cpowerdomain.cpd_statesmpu_bitsper_bitspd_regsdpll_335x.cldo_335x.cpm_handlers.cprcm_core.cprcm_enable_isolation.part.0prcm_disable_isolation.part.1cmd_wake_sourceshwmod_43xx.cddr.cio_ctxdpll.cpower_down_pllsdpll_regspll_modedpll_43xx.cpm_state_data.cclockdomain.cclkdms_state_changeclkdmssleep_clkdmsstandby_clkdmsclockdomain_43xx.cclockdomain_335x.cpowerdomain_43xx.ci2c.cldo_43xx.chwmod.chwmods_state_changehwmodsessential_hwmodsinterconnect_hwmodspowerdomain_335x.cputcharam43xx_interconnect_hwmodsdisable_ioa8_standalone_handlervtp_disableverify_pd_transitionsextint8_handlertimer_syncam335x_per_bitsa8_lp_ds0_handlerreset_handleram43xx_pd_regspm_resetds_restoreget_pd_mpu_stctrl_valprintfam43xx_dpll_regsam43xx_ldo_regsds1_data_aux_data_startcmd_handlersam335x_pd_regsam335x_pg2_power_down_pllsessential_hwmods_enablea8_lp_ds1_handlerdpll_get_divextint27_handlersoc_revclkdms_standby_wakeextint36_handlerddr_am335x_io_suspendidle_v2_dataextint26_handlerpd_state_changepll_lockextint34_handlerextint48_handleri2c_writepd_read_stateplls_power_up_start_dataextint41_handlerprcm_disable_isolationtrace_set_current_posddr_am437x_io_suspendextint4_handlera8_m3_low_power_fastextint50_handlerextint0_handleram335x_clkdmsextint7_handlerbusfault_handlerdisable_master_oscillatorhwmod_init_end_datadebug_set_levelextint45_handlerclear_wake_sourcesa8_standby_handlerextint9_handlerputssystick_handlerpll_bypassam43xx_power_down_pllsds1_data_hsextint40_handlerextint37_handlermsg_readextint12_handlera8_m3_low_power_syncm3_param_resettrace_initstandby_datamemmanage_handlerextint30_handlerm3_firmware_versiontrace_updatea8_wake_ds0_handlerextint5_handleressential_hwmods_disablea8_wake_cpuidle_handlerio_isolationextint31_handlernvic_disable_irqget_master_xtal_khzmsg_cmd_is_validvtt_highextint19_handlerclockdomain_initextint16_handler_start_textsvc_handlera8_i2c_wake_handlerextint53_handlera8_notifyextint1_handlermsg_cmd_needs_triggerclkdms_wakeextint15_handlerset_ddr_reset_end_textclear_ddr_resetextint47_handlerconfigure_wake_sourcesinterconnect_hwmods_disableextint18_handlerdummy_handlerpowerdomain_initds0_data_hsextint52_handleram335x_dpll_regssoc_typevector_tablevprintfddr_io_resumeextint6_handleram43xx_per_bitsrtc_mode_dataextint23_handlerldo_wait_for_onextint20_handlerscr_enable_sleeponexitldo_power_downam335x_hwmodsnvic_clear_irqclkdm_wakeextint38_handlerextint22_handleram335x_standby_clkdmsa8_cpuidle_handlerscr_enable_sleepdeepextint39_handlerdpll_resetextint42_handlermsg_cmd_wakeup_reason_updateextint21_handler_logbuf_startgeneric_wake_handlerputsnvtt_lowextint32_handleram335x_interconnect_hwmods_aux_data_endldo_power_upam335x_mpu_bitsextint28_handlerdebugmon_handler_end_stackddr_io_save_contextrtc_reg_writea8_wake_ds1_handlerextint49_handleram43xx_essential_hwmodsa8_wake_standby_handlerextint46_handlermemsetsoc_idhwmod_enablemainsetup_socpendsv_handlerextint43_handlerrtc_enable_checkmsg_cmd_read_idinit_m3_state_machineidle_dataextint25_handlerhwmod_is_enabledvtt_toggleinterconnect_hwmods_enablecmd_global_dataclkdm_state_changeusagefault_handlerextint3_handlervtp_enableclkdm_active_logbuf_endds0_dataextint51_handlerclkdm_sleepextint24_handlerextint10_handlerds_saveam335x_ldo_regsnvic_enable_irqpowerdomain_resetddr_io_suspendpd_state_restoredebug_haltam335_initam43xx_standby_clkdmsam43xx_clkdmsvtt_gpio_pin_end_resource_tablea8_wake_rtc_handlertrace_get_current_posa8_cpuidle_v2_handlermsg_cmd_stat_updateldo_wait_for_ret_end_bsshardfault_handlerldo_init_start_bssextint35_handlermsg_cmd_fast_triggeram335x_essential_hwmodsextint14_handlermsg_writenmi_handlera8_lp_rtc_handlerclkdms_sleepplls_power_downextint33_handlerextint13_handlerprcm_enable_isolationa8_lp_ds2_handlerds2_dataa8_i2c_sleep_handleram335x_sleep_clkdmsconfigure_deepsleep_countrsc_taba8_wake_cpuidle_v2_handleram43xx_mpu_bitsdebug_printfstrlena8_wake_ds2_handlerget_pd_per_stctrl_valrtc_reg_readextint29_handlerextint11_handler_start_resource_tableextint44_handlerenable_ioenable_master_oscillatorextint2_handleram43xx_sleep_clkdmsam43xx_hwmodshwmod_disablemsg_cmd_dispatcherdpll_initmem_typeextint17_handler0*!�'��D7�<FN%rZ-t$hK��,s6���������0����0�-�p�7������p$U	�:=
\ No newline at end of file
diff -Naur --no-dereference a/firmware/am43x-evm-scale-data.bin b/firmware/am43x-evm-scale-data.bin
--- a/firmware/am43x-evm-scale-data.bin	1969-12-31 19:00:00.000000000 -0500
+++ b/firmware/am43x-evm-scale-data.bin	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1 @@
+Wd$k$�$g$�d$k$�$g$�
\ No newline at end of file
diff -Naur --no-dereference a/fs/aufs/aufs.h b/fs/aufs/aufs.h
--- a/fs/aufs/aufs.h	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/aufs.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,62 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * all header files
+ */
+
+#ifndef __AUFS_H__
+#define __AUFS_H__
+
+#ifdef __KERNEL__
+
+#define AuStub(type, name, body, ...) \
+	static inline type name(__VA_ARGS__) { body; }
+
+#define AuStubVoid(name, ...) \
+	AuStub(void, name, , __VA_ARGS__)
+#define AuStubInt0(name, ...) \
+	AuStub(int, name, return 0, __VA_ARGS__)
+
+#include "debug.h"
+
+#include "branch.h"
+#include "cpup.h"
+#include "dcsub.h"
+#include "dbgaufs.h"
+#include "dentry.h"
+#include "dir.h"
+#include "dirren.h"
+#include "dynop.h"
+#include "file.h"
+#include "fstype.h"
+#include "hbl.h"
+#include "inode.h"
+#include "lcnt.h"
+#include "loop.h"
+#include "module.h"
+#include "opts.h"
+#include "rwsem.h"
+#include "super.h"
+#include "sysaufs.h"
+#include "vfsub.h"
+#include "whout.h"
+#include "wkq.h"
+
+#endif /* __KERNEL__ */
+#endif /* __AUFS_H__ */
diff -Naur --no-dereference a/fs/aufs/branch.c b/fs/aufs/branch.c
--- a/fs/aufs/branch.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/branch.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,1422 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * branch management
+ */
+
+#include <linux/compat.h>
+#include <linux/statfs.h>
+#include "aufs.h"
+
+/*
+ * free a single branch
+ */
+static void au_br_do_free(struct au_branch *br)
+{
+	int i;
+	struct au_wbr *wbr;
+	struct au_dykey **key;
+
+	au_hnotify_fin_br(br);
+	/* always, regardless the mount option */
+	au_dr_hino_free(&br->br_dirren);
+	au_xino_put(br);
+
+	AuLCntZero(au_lcnt_read(&br->br_nfiles, /*do_rev*/0));
+	au_lcnt_fin(&br->br_nfiles, /*do_sync*/0);
+	AuLCntZero(au_lcnt_read(&br->br_count, /*do_rev*/0));
+	au_lcnt_fin(&br->br_count, /*do_sync*/0);
+
+	wbr = br->br_wbr;
+	if (wbr) {
+		for (i = 0; i < AuBrWh_Last; i++)
+			dput(wbr->wbr_wh[i]);
+		AuDebugOn(atomic_read(&wbr->wbr_wh_running));
+		AuRwDestroy(&wbr->wbr_wh_rwsem);
+	}
+
+	if (br->br_fhsm) {
+		au_br_fhsm_fin(br->br_fhsm);
+		au_kfree_try_rcu(br->br_fhsm);
+	}
+
+	key = br->br_dykey;
+	for (i = 0; i < AuBrDynOp; i++, key++)
+		if (*key)
+			au_dy_put(*key);
+		else
+			break;
+
+	/* recursive lock, s_umount of branch's */
+	/* synchronize_rcu(); */ /* why? */
+	lockdep_off();
+	path_put(&br->br_path);
+	lockdep_on();
+	au_kfree_rcu(wbr);
+	au_lcnt_wait_for_fin(&br->br_nfiles);
+	au_lcnt_wait_for_fin(&br->br_count);
+	/* I don't know why, but percpu_refcount requires this */
+	/* synchronize_rcu(); */
+	au_kfree_rcu(br);
+}
+
+/*
+ * frees all branches
+ */
+void au_br_free(struct au_sbinfo *sbinfo)
+{
+	aufs_bindex_t bmax;
+	struct au_branch **br;
+
+	AuRwMustWriteLock(&sbinfo->si_rwsem);
+
+	bmax = sbinfo->si_bbot + 1;
+	br = sbinfo->si_branch;
+	while (bmax--)
+		au_br_do_free(*br++);
+}
+
+/*
+ * find the index of a branch which is specified by @br_id.
+ */
+int au_br_index(struct super_block *sb, aufs_bindex_t br_id)
+{
+	aufs_bindex_t bindex, bbot;
+
+	bbot = au_sbbot(sb);
+	for (bindex = 0; bindex <= bbot; bindex++)
+		if (au_sbr_id(sb, bindex) == br_id)
+			return bindex;
+	return -1;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * add a branch
+ */
+
+static int test_overlap(struct super_block *sb, struct dentry *h_adding,
+			struct dentry *h_root)
+{
+	if (unlikely(h_adding == h_root
+		     || au_test_loopback_overlap(sb, h_adding)))
+		return 1;
+	if (h_adding->d_sb != h_root->d_sb)
+		return 0;
+	return au_test_subdir(h_adding, h_root)
+		|| au_test_subdir(h_root, h_adding);
+}
+
+/*
+ * returns a newly allocated branch. @new_nbranch is a number of branches
+ * after adding a branch.
+ */
+static struct au_branch *au_br_alloc(struct super_block *sb, int new_nbranch,
+				     int perm)
+{
+	struct au_branch *add_branch;
+	struct dentry *root;
+	struct inode *inode;
+	int err;
+
+	err = -ENOMEM;
+	add_branch = kzalloc(sizeof(*add_branch), GFP_NOFS);
+	if (unlikely(!add_branch))
+		goto out;
+	add_branch->br_xino = au_xino_alloc(/*nfile*/1);
+	if (unlikely(!add_branch->br_xino))
+		goto out_br;
+	err = au_hnotify_init_br(add_branch, perm);
+	if (unlikely(err))
+		goto out_xino;
+
+	if (au_br_writable(perm)) {
+		/* may be freed separately at changing the branch permission */
+		add_branch->br_wbr = kzalloc(sizeof(*add_branch->br_wbr),
+					     GFP_NOFS);
+		if (unlikely(!add_branch->br_wbr))
+			goto out_hnotify;
+	}
+
+	if (au_br_fhsm(perm)) {
+		err = au_fhsm_br_alloc(add_branch);
+		if (unlikely(err))
+			goto out_wbr;
+	}
+
+	root = sb->s_root;
+	err = au_sbr_realloc(au_sbi(sb), new_nbranch, /*may_shrink*/0);
+	if (!err)
+		err = au_di_realloc(au_di(root), new_nbranch, /*may_shrink*/0);
+	if (!err) {
+		inode = d_inode(root);
+		err = au_hinode_realloc(au_ii(inode), new_nbranch,
+					/*may_shrink*/0);
+	}
+	if (!err)
+		return add_branch; /* success */
+
+out_wbr:
+	au_kfree_rcu(add_branch->br_wbr);
+out_hnotify:
+	au_hnotify_fin_br(add_branch);
+out_xino:
+	au_xino_put(add_branch);
+out_br:
+	au_kfree_rcu(add_branch);
+out:
+	return ERR_PTR(err);
+}
+
+/*
+ * test if the branch permission is legal or not.
+ */
+static int test_br(struct inode *inode, int brperm, char *path)
+{
+	int err;
+
+	err = (au_br_writable(brperm) && IS_RDONLY(inode));
+	if (!err)
+		goto out;
+
+	err = -EINVAL;
+	pr_err("write permission for readonly mount or inode, %s\n", path);
+
+out:
+	return err;
+}
+
+/*
+ * returns:
+ * 0: success, the caller will add it
+ * plus: success, it is already unified, the caller should ignore it
+ * minus: error
+ */
+static int test_add(struct super_block *sb, struct au_opt_add *add, int remount)
+{
+	int err;
+	aufs_bindex_t bbot, bindex;
+	struct dentry *root, *h_dentry;
+	struct inode *inode, *h_inode;
+
+	root = sb->s_root;
+	bbot = au_sbbot(sb);
+	if (unlikely(bbot >= 0
+		     && au_find_dbindex(root, add->path.dentry) >= 0)) {
+		err = 1;
+		if (!remount) {
+			err = -EINVAL;
+			pr_err("%s duplicated\n", add->pathname);
+		}
+		goto out;
+	}
+
+	err = -ENOSPC; /* -E2BIG; */
+	if (unlikely(AUFS_BRANCH_MAX <= add->bindex
+		     || AUFS_BRANCH_MAX - 1 <= bbot)) {
+		pr_err("number of branches exceeded %s\n", add->pathname);
+		goto out;
+	}
+
+	err = -EDOM;
+	if (unlikely(add->bindex < 0 || bbot + 1 < add->bindex)) {
+		pr_err("bad index %d\n", add->bindex);
+		goto out;
+	}
+
+	inode = d_inode(add->path.dentry);
+	err = -ENOENT;
+	if (unlikely(!inode->i_nlink)) {
+		pr_err("no existence %s\n", add->pathname);
+		goto out;
+	}
+
+	err = -EINVAL;
+	if (unlikely(inode->i_sb == sb)) {
+		pr_err("%s must be outside\n", add->pathname);
+		goto out;
+	}
+
+	if (unlikely(au_test_fs_unsuppoted(inode->i_sb))) {
+		pr_err("unsupported filesystem, %s (%s)\n",
+		       add->pathname, au_sbtype(inode->i_sb));
+		goto out;
+	}
+
+	if (unlikely(inode->i_sb->s_stack_depth)) {
+		pr_err("already stacked, %s (%s)\n",
+		       add->pathname, au_sbtype(inode->i_sb));
+		goto out;
+	}
+
+	err = test_br(d_inode(add->path.dentry), add->perm, add->pathname);
+	if (unlikely(err))
+		goto out;
+
+	if (bbot < 0)
+		return 0; /* success */
+
+	err = -EINVAL;
+	for (bindex = 0; bindex <= bbot; bindex++)
+		if (unlikely(test_overlap(sb, add->path.dentry,
+					  au_h_dptr(root, bindex)))) {
+			pr_err("%s is overlapped\n", add->pathname);
+			goto out;
+		}
+
+	err = 0;
+	if (au_opt_test(au_mntflags(sb), WARN_PERM)) {
+		h_dentry = au_h_dptr(root, 0);
+		h_inode = d_inode(h_dentry);
+		if ((h_inode->i_mode & S_IALLUGO) != (inode->i_mode & S_IALLUGO)
+		    || !uid_eq(h_inode->i_uid, inode->i_uid)
+		    || !gid_eq(h_inode->i_gid, inode->i_gid))
+			pr_warn("uid/gid/perm %s %u/%u/0%o, %u/%u/0%o\n",
+				add->pathname,
+				i_uid_read(inode), i_gid_read(inode),
+				(inode->i_mode & S_IALLUGO),
+				i_uid_read(h_inode), i_gid_read(h_inode),
+				(h_inode->i_mode & S_IALLUGO));
+	}
+
+out:
+	return err;
+}
+
+/*
+ * initialize or clean the whiteouts for an adding branch
+ */
+static int au_br_init_wh(struct super_block *sb, struct au_branch *br,
+			 int new_perm)
+{
+	int err, old_perm;
+	aufs_bindex_t bindex;
+	struct inode *h_inode;
+	struct au_wbr *wbr;
+	struct au_hinode *hdir;
+	struct dentry *h_dentry;
+
+	err = vfsub_mnt_want_write(au_br_mnt(br));
+	if (unlikely(err))
+		goto out;
+
+	wbr = br->br_wbr;
+	old_perm = br->br_perm;
+	br->br_perm = new_perm;
+	hdir = NULL;
+	h_inode = NULL;
+	bindex = au_br_index(sb, br->br_id);
+	if (0 <= bindex) {
+		hdir = au_hi(d_inode(sb->s_root), bindex);
+		au_hn_inode_lock_nested(hdir, AuLsc_I_PARENT);
+	} else {
+		h_dentry = au_br_dentry(br);
+		h_inode = d_inode(h_dentry);
+		inode_lock_nested(h_inode, AuLsc_I_PARENT);
+	}
+	if (!wbr)
+		err = au_wh_init(br, sb);
+	else {
+		wbr_wh_write_lock(wbr);
+		err = au_wh_init(br, sb);
+		wbr_wh_write_unlock(wbr);
+	}
+	if (hdir)
+		au_hn_inode_unlock(hdir);
+	else
+		inode_unlock(h_inode);
+	vfsub_mnt_drop_write(au_br_mnt(br));
+	br->br_perm = old_perm;
+
+	if (!err && wbr && !au_br_writable(new_perm)) {
+		au_kfree_rcu(wbr);
+		br->br_wbr = NULL;
+	}
+
+out:
+	return err;
+}
+
+static int au_wbr_init(struct au_branch *br, struct super_block *sb,
+		       int perm)
+{
+	int err;
+	struct kstatfs kst;
+	struct au_wbr *wbr;
+
+	wbr = br->br_wbr;
+	au_rw_init(&wbr->wbr_wh_rwsem);
+	atomic_set(&wbr->wbr_wh_running, 0);
+
+	/*
+	 * a limit for rmdir/rename a dir
+	 * cf. AUFS_MAX_NAMELEN in include/uapi/linux/aufs_type.h
+	 */
+	err = vfs_statfs(&br->br_path, &kst);
+	if (unlikely(err))
+		goto out;
+	err = -EINVAL;
+	if (kst.f_namelen >= NAME_MAX)
+		err = au_br_init_wh(sb, br, perm);
+	else
+		pr_err("%pd(%s), unsupported namelen %ld\n",
+		       au_br_dentry(br),
+		       au_sbtype(au_br_dentry(br)->d_sb), kst.f_namelen);
+
+out:
+	return err;
+}
+
+/* initialize a new branch */
+static int au_br_init(struct au_branch *br, struct super_block *sb,
+		      struct au_opt_add *add)
+{
+	int err;
+	struct au_branch *brbase;
+	struct file *xf;
+	struct inode *h_inode;
+
+	err = 0;
+	br->br_perm = add->perm;
+	br->br_path = add->path; /* set first, path_get() later */
+	spin_lock_init(&br->br_dykey_lock);
+	au_lcnt_init(&br->br_nfiles, /*release*/NULL);
+	au_lcnt_init(&br->br_count, /*release*/NULL);
+	br->br_id = au_new_br_id(sb);
+	AuDebugOn(br->br_id < 0);
+
+	/* always, regardless the given option */
+	err = au_dr_br_init(sb, br, &add->path);
+	if (unlikely(err))
+		goto out_err;
+
+	if (au_br_writable(add->perm)) {
+		err = au_wbr_init(br, sb, add->perm);
+		if (unlikely(err))
+			goto out_err;
+	}
+
+	if (au_opt_test(au_mntflags(sb), XINO)) {
+		brbase = au_sbr(sb, 0);
+		xf = au_xino_file(brbase->br_xino, /*idx*/-1);
+		AuDebugOn(!xf);
+		h_inode = d_inode(add->path.dentry);
+		err = au_xino_init_br(sb, br, h_inode->i_ino, &xf->f_path);
+		if (unlikely(err)) {
+			AuDebugOn(au_xino_file(br->br_xino, /*idx*/-1));
+			goto out_err;
+		}
+	}
+
+	sysaufs_br_init(br);
+	path_get(&br->br_path);
+	goto out; /* success */
+
+out_err:
+	memset(&br->br_path, 0, sizeof(br->br_path));
+out:
+	return err;
+}
+
+static void au_br_do_add_brp(struct au_sbinfo *sbinfo, aufs_bindex_t bindex,
+			     struct au_branch *br, aufs_bindex_t bbot,
+			     aufs_bindex_t amount)
+{
+	struct au_branch **brp;
+
+	AuRwMustWriteLock(&sbinfo->si_rwsem);
+
+	brp = sbinfo->si_branch + bindex;
+	memmove(brp + 1, brp, sizeof(*brp) * amount);
+	*brp = br;
+	sbinfo->si_bbot++;
+	if (unlikely(bbot < 0))
+		sbinfo->si_bbot = 0;
+}
+
+static void au_br_do_add_hdp(struct au_dinfo *dinfo, aufs_bindex_t bindex,
+			     aufs_bindex_t bbot, aufs_bindex_t amount)
+{
+	struct au_hdentry *hdp;
+
+	AuRwMustWriteLock(&dinfo->di_rwsem);
+
+	hdp = au_hdentry(dinfo, bindex);
+	memmove(hdp + 1, hdp, sizeof(*hdp) * amount);
+	au_h_dentry_init(hdp);
+	dinfo->di_bbot++;
+	if (unlikely(bbot < 0))
+		dinfo->di_btop = 0;
+}
+
+static void au_br_do_add_hip(struct au_iinfo *iinfo, aufs_bindex_t bindex,
+			     aufs_bindex_t bbot, aufs_bindex_t amount)
+{
+	struct au_hinode *hip;
+
+	AuRwMustWriteLock(&iinfo->ii_rwsem);
+
+	hip = au_hinode(iinfo, bindex);
+	memmove(hip + 1, hip, sizeof(*hip) * amount);
+	au_hinode_init(hip);
+	iinfo->ii_bbot++;
+	if (unlikely(bbot < 0))
+		iinfo->ii_btop = 0;
+}
+
+static void au_br_do_add(struct super_block *sb, struct au_branch *br,
+			 aufs_bindex_t bindex)
+{
+	struct dentry *root, *h_dentry;
+	struct inode *root_inode, *h_inode;
+	aufs_bindex_t bbot, amount;
+
+	root = sb->s_root;
+	root_inode = d_inode(root);
+	bbot = au_sbbot(sb);
+	amount = bbot + 1 - bindex;
+	h_dentry = au_br_dentry(br);
+	au_sbilist_lock();
+	au_br_do_add_brp(au_sbi(sb), bindex, br, bbot, amount);
+	au_br_do_add_hdp(au_di(root), bindex, bbot, amount);
+	au_br_do_add_hip(au_ii(root_inode), bindex, bbot, amount);
+	au_set_h_dptr(root, bindex, dget(h_dentry));
+	h_inode = d_inode(h_dentry);
+	au_set_h_iptr(root_inode, bindex, au_igrab(h_inode), /*flags*/0);
+	au_sbilist_unlock();
+}
+
+int au_br_add(struct super_block *sb, struct au_opt_add *add, int remount)
+{
+	int err;
+	aufs_bindex_t bbot, add_bindex;
+	struct dentry *root, *h_dentry;
+	struct inode *root_inode;
+	struct au_branch *add_branch;
+
+	root = sb->s_root;
+	root_inode = d_inode(root);
+	IMustLock(root_inode);
+	IiMustWriteLock(root_inode);
+	err = test_add(sb, add, remount);
+	if (unlikely(err < 0))
+		goto out;
+	if (err) {
+		err = 0;
+		goto out; /* success */
+	}
+
+	bbot = au_sbbot(sb);
+	add_branch = au_br_alloc(sb, bbot + 2, add->perm);
+	err = PTR_ERR(add_branch);
+	if (IS_ERR(add_branch))
+		goto out;
+
+	err = au_br_init(add_branch, sb, add);
+	if (unlikely(err)) {
+		au_br_do_free(add_branch);
+		goto out;
+	}
+
+	add_bindex = add->bindex;
+	sysaufs_brs_del(sb, add_bindex);	/* remove successors */
+	au_br_do_add(sb, add_branch, add_bindex);
+	sysaufs_brs_add(sb, add_bindex);	/* append successors */
+	dbgaufs_brs_add(sb, add_bindex, /*topdown*/0);	/* rename successors */
+
+	h_dentry = add->path.dentry;
+	if (!add_bindex) {
+		au_cpup_attr_all(root_inode, /*force*/1);
+		sb->s_maxbytes = h_dentry->d_sb->s_maxbytes;
+	} else
+		au_add_nlink(root_inode, d_inode(h_dentry));
+
+out:
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static unsigned long long au_farray_cb(struct super_block *sb, void *a,
+				       unsigned long long max __maybe_unused,
+				       void *arg)
+{
+	unsigned long long n;
+	struct file **p, *f;
+	struct hlist_bl_head *files;
+	struct hlist_bl_node *pos;
+	struct au_finfo *finfo;
+
+	n = 0;
+	p = a;
+	files = &au_sbi(sb)->si_files;
+	hlist_bl_lock(files);
+	hlist_bl_for_each_entry(finfo, pos, files, fi_hlist) {
+		f = finfo->fi_file;
+		if (file_count(f)
+		    && !special_file(file_inode(f)->i_mode)) {
+			get_file(f);
+			*p++ = f;
+			n++;
+			AuDebugOn(n > max);
+		}
+	}
+	hlist_bl_unlock(files);
+
+	return n;
+}
+
+static struct file **au_farray_alloc(struct super_block *sb,
+				     unsigned long long *max)
+{
+	struct au_sbinfo *sbi;
+
+	sbi = au_sbi(sb);
+	*max = au_lcnt_read(&sbi->si_nfiles, /*do_rev*/1);
+	return au_array_alloc(max, au_farray_cb, sb, /*arg*/NULL);
+}
+
+static void au_farray_free(struct file **a, unsigned long long max)
+{
+	unsigned long long ull;
+
+	for (ull = 0; ull < max; ull++)
+		if (a[ull])
+			fput(a[ull]);
+	kvfree(a);
+}
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * delete a branch
+ */
+
+/* to show the line number, do not make it inlined function */
+#define AuVerbose(do_info, fmt, ...) do { \
+	if (do_info) \
+		pr_info(fmt, ##__VA_ARGS__); \
+} while (0)
+
+static int au_test_ibusy(struct inode *inode, aufs_bindex_t btop,
+			 aufs_bindex_t bbot)
+{
+	return (inode && !S_ISDIR(inode->i_mode)) || btop == bbot;
+}
+
+static int au_test_dbusy(struct dentry *dentry, aufs_bindex_t btop,
+			 aufs_bindex_t bbot)
+{
+	return au_test_ibusy(d_inode(dentry), btop, bbot);
+}
+
+/*
+ * test if the branch is deletable or not.
+ */
+static int test_dentry_busy(struct dentry *root, aufs_bindex_t bindex,
+			    unsigned int sigen, const unsigned int verbose)
+{
+	int err, i, j, ndentry;
+	aufs_bindex_t btop, bbot;
+	struct au_dcsub_pages dpages;
+	struct au_dpage *dpage;
+	struct dentry *d;
+
+	err = au_dpages_init(&dpages, GFP_NOFS);
+	if (unlikely(err))
+		goto out;
+	err = au_dcsub_pages(&dpages, root, NULL, NULL);
+	if (unlikely(err))
+		goto out_dpages;
+
+	for (i = 0; !err && i < dpages.ndpage; i++) {
+		dpage = dpages.dpages + i;
+		ndentry = dpage->ndentry;
+		for (j = 0; !err && j < ndentry; j++) {
+			d = dpage->dentries[j];
+			AuDebugOn(au_dcount(d) <= 0);
+			if (!au_digen_test(d, sigen)) {
+				di_read_lock_child(d, AuLock_IR);
+				if (unlikely(au_dbrange_test(d))) {
+					di_read_unlock(d, AuLock_IR);
+					continue;
+				}
+			} else {
+				di_write_lock_child(d);
+				if (unlikely(au_dbrange_test(d))) {
+					di_write_unlock(d);
+					continue;
+				}
+				err = au_reval_dpath(d, sigen);
+				if (!err)
+					di_downgrade_lock(d, AuLock_IR);
+				else {
+					di_write_unlock(d);
+					break;
+				}
+			}
+
+			/* AuDbgDentry(d); */
+			btop = au_dbtop(d);
+			bbot = au_dbbot(d);
+			if (btop <= bindex
+			    && bindex <= bbot
+			    && au_h_dptr(d, bindex)
+			    && au_test_dbusy(d, btop, bbot)) {
+				err = -EBUSY;
+				AuVerbose(verbose, "busy %pd\n", d);
+				AuDbgDentry(d);
+			}
+			di_read_unlock(d, AuLock_IR);
+		}
+	}
+
+out_dpages:
+	au_dpages_free(&dpages);
+out:
+	return err;
+}
+
+static int test_inode_busy(struct super_block *sb, aufs_bindex_t bindex,
+			   unsigned int sigen, const unsigned int verbose)
+{
+	int err;
+	unsigned long long max, ull;
+	struct inode *i, **array;
+	aufs_bindex_t btop, bbot;
+
+	array = au_iarray_alloc(sb, &max);
+	err = PTR_ERR(array);
+	if (IS_ERR(array))
+		goto out;
+
+	err = 0;
+	AuDbg("b%d\n", bindex);
+	for (ull = 0; !err && ull < max; ull++) {
+		i = array[ull];
+		if (unlikely(!i))
+			break;
+		if (i->i_ino == AUFS_ROOT_INO)
+			continue;
+
+		/* AuDbgInode(i); */
+		if (au_iigen(i, NULL) == sigen)
+			ii_read_lock_child(i);
+		else {
+			ii_write_lock_child(i);
+			err = au_refresh_hinode_self(i);
+			au_iigen_dec(i);
+			if (!err)
+				ii_downgrade_lock(i);
+			else {
+				ii_write_unlock(i);
+				break;
+			}
+		}
+
+		btop = au_ibtop(i);
+		bbot = au_ibbot(i);
+		if (btop <= bindex
+		    && bindex <= bbot
+		    && au_h_iptr(i, bindex)
+		    && au_test_ibusy(i, btop, bbot)) {
+			err = -EBUSY;
+			AuVerbose(verbose, "busy i%lu\n", i->i_ino);
+			AuDbgInode(i);
+		}
+		ii_read_unlock(i);
+	}
+	au_iarray_free(array, max);
+
+out:
+	return err;
+}
+
+static int test_children_busy(struct dentry *root, aufs_bindex_t bindex,
+			      const unsigned int verbose)
+{
+	int err;
+	unsigned int sigen;
+
+	sigen = au_sigen(root->d_sb);
+	DiMustNoWaiters(root);
+	IiMustNoWaiters(d_inode(root));
+	di_write_unlock(root);
+	err = test_dentry_busy(root, bindex, sigen, verbose);
+	if (!err)
+		err = test_inode_busy(root->d_sb, bindex, sigen, verbose);
+	di_write_lock_child(root); /* aufs_write_lock() calls ..._child() */
+
+	return err;
+}
+
+static int test_dir_busy(struct file *file, aufs_bindex_t br_id,
+			 struct file **to_free, int *idx)
+{
+	int err;
+	unsigned char matched, root;
+	aufs_bindex_t bindex, bbot;
+	struct au_fidir *fidir;
+	struct au_hfile *hfile;
+
+	err = 0;
+	root = IS_ROOT(file->f_path.dentry);
+	if (root) {
+		get_file(file);
+		to_free[*idx] = file;
+		(*idx)++;
+		goto out;
+	}
+
+	matched = 0;
+	fidir = au_fi(file)->fi_hdir;
+	AuDebugOn(!fidir);
+	bbot = au_fbbot_dir(file);
+	for (bindex = au_fbtop(file); bindex <= bbot; bindex++) {
+		hfile = fidir->fd_hfile + bindex;
+		if (!hfile->hf_file)
+			continue;
+
+		if (hfile->hf_br->br_id == br_id) {
+			matched = 1;
+			break;
+		}
+	}
+	if (matched)
+		err = -EBUSY;
+
+out:
+	return err;
+}
+
+static int test_file_busy(struct super_block *sb, aufs_bindex_t br_id,
+			  struct file **to_free, int opened)
+{
+	int err, idx;
+	unsigned long long ull, max;
+	aufs_bindex_t btop;
+	struct file *file, **array;
+	struct dentry *root;
+	struct au_hfile *hfile;
+
+	array = au_farray_alloc(sb, &max);
+	err = PTR_ERR(array);
+	if (IS_ERR(array))
+		goto out;
+
+	err = 0;
+	idx = 0;
+	root = sb->s_root;
+	di_write_unlock(root);
+	for (ull = 0; ull < max; ull++) {
+		file = array[ull];
+		if (unlikely(!file))
+			break;
+
+		/* AuDbg("%pD\n", file); */
+		fi_read_lock(file);
+		btop = au_fbtop(file);
+		if (!d_is_dir(file->f_path.dentry)) {
+			hfile = &au_fi(file)->fi_htop;
+			if (hfile->hf_br->br_id == br_id)
+				err = -EBUSY;
+		} else
+			err = test_dir_busy(file, br_id, to_free, &idx);
+		fi_read_unlock(file);
+		if (unlikely(err))
+			break;
+	}
+	di_write_lock_child(root);
+	au_farray_free(array, max);
+	AuDebugOn(idx > opened);
+
+out:
+	return err;
+}
+
+static void br_del_file(struct file **to_free, unsigned long long opened,
+			aufs_bindex_t br_id)
+{
+	unsigned long long ull;
+	aufs_bindex_t bindex, btop, bbot, bfound;
+	struct file *file;
+	struct au_fidir *fidir;
+	struct au_hfile *hfile;
+
+	for (ull = 0; ull < opened; ull++) {
+		file = to_free[ull];
+		if (unlikely(!file))
+			break;
+
+		/* AuDbg("%pD\n", file); */
+		AuDebugOn(!d_is_dir(file->f_path.dentry));
+		bfound = -1;
+		fidir = au_fi(file)->fi_hdir;
+		AuDebugOn(!fidir);
+		fi_write_lock(file);
+		btop = au_fbtop(file);
+		bbot = au_fbbot_dir(file);
+		for (bindex = btop; bindex <= bbot; bindex++) {
+			hfile = fidir->fd_hfile + bindex;
+			if (!hfile->hf_file)
+				continue;
+
+			if (hfile->hf_br->br_id == br_id) {
+				bfound = bindex;
+				break;
+			}
+		}
+		AuDebugOn(bfound < 0);
+		au_set_h_fptr(file, bfound, NULL);
+		if (bfound == btop) {
+			for (btop++; btop <= bbot; btop++)
+				if (au_hf_dir(file, btop)) {
+					au_set_fbtop(file, btop);
+					break;
+				}
+		}
+		fi_write_unlock(file);
+	}
+}
+
+static void au_br_do_del_brp(struct au_sbinfo *sbinfo,
+			     const aufs_bindex_t bindex,
+			     const aufs_bindex_t bbot)
+{
+	struct au_branch **brp, **p;
+
+	AuRwMustWriteLock(&sbinfo->si_rwsem);
+
+	brp = sbinfo->si_branch + bindex;
+	if (bindex < bbot)
+		memmove(brp, brp + 1, sizeof(*brp) * (bbot - bindex));
+	sbinfo->si_branch[0 + bbot] = NULL;
+	sbinfo->si_bbot--;
+
+	p = au_krealloc(sbinfo->si_branch, sizeof(*p) * bbot, AuGFP_SBILIST,
+			/*may_shrink*/1);
+	if (p)
+		sbinfo->si_branch = p;
+	/* harmless error */
+}
+
+static void au_br_do_del_hdp(struct au_dinfo *dinfo, const aufs_bindex_t bindex,
+			     const aufs_bindex_t bbot)
+{
+	struct au_hdentry *hdp, *p;
+
+	AuRwMustWriteLock(&dinfo->di_rwsem);
+
+	hdp = au_hdentry(dinfo, bindex);
+	if (bindex < bbot)
+		memmove(hdp, hdp + 1, sizeof(*hdp) * (bbot - bindex));
+	/* au_h_dentry_init(au_hdentry(dinfo, bbot); */
+	dinfo->di_bbot--;
+
+	p = au_krealloc(dinfo->di_hdentry, sizeof(*p) * bbot, AuGFP_SBILIST,
+			/*may_shrink*/1);
+	if (p)
+		dinfo->di_hdentry = p;
+	/* harmless error */
+}
+
+static void au_br_do_del_hip(struct au_iinfo *iinfo, const aufs_bindex_t bindex,
+			     const aufs_bindex_t bbot)
+{
+	struct au_hinode *hip, *p;
+
+	AuRwMustWriteLock(&iinfo->ii_rwsem);
+
+	hip = au_hinode(iinfo, bindex);
+	if (bindex < bbot)
+		memmove(hip, hip + 1, sizeof(*hip) * (bbot - bindex));
+	/* au_hinode_init(au_hinode(iinfo, bbot)); */
+	iinfo->ii_bbot--;
+
+	p = au_krealloc(iinfo->ii_hinode, sizeof(*p) * bbot, AuGFP_SBILIST,
+			/*may_shrink*/1);
+	if (p)
+		iinfo->ii_hinode = p;
+	/* harmless error */
+}
+
+static void au_br_do_del(struct super_block *sb, aufs_bindex_t bindex,
+			 struct au_branch *br)
+{
+	aufs_bindex_t bbot;
+	struct au_sbinfo *sbinfo;
+	struct dentry *root, *h_root;
+	struct inode *inode, *h_inode;
+	struct au_hinode *hinode;
+
+	SiMustWriteLock(sb);
+
+	root = sb->s_root;
+	inode = d_inode(root);
+	sbinfo = au_sbi(sb);
+	bbot = sbinfo->si_bbot;
+
+	h_root = au_h_dptr(root, bindex);
+	hinode = au_hi(inode, bindex);
+	h_inode = au_igrab(hinode->hi_inode);
+	au_hiput(hinode);
+
+	au_sbilist_lock();
+	au_br_do_del_brp(sbinfo, bindex, bbot);
+	au_br_do_del_hdp(au_di(root), bindex, bbot);
+	au_br_do_del_hip(au_ii(inode), bindex, bbot);
+	au_sbilist_unlock();
+
+	/* ignore an error */
+	au_dr_br_fin(sb, br); /* always, regardless the mount option */
+
+	dput(h_root);
+	iput(h_inode);
+	au_br_do_free(br);
+}
+
+static unsigned long long empty_cb(struct super_block *sb, void *array,
+				   unsigned long long max, void *arg)
+{
+	return max;
+}
+
+int au_br_del(struct super_block *sb, struct au_opt_del *del, int remount)
+{
+	int err, rerr, i;
+	unsigned long long opened;
+	unsigned int mnt_flags;
+	aufs_bindex_t bindex, bbot, br_id;
+	unsigned char do_wh, verbose;
+	struct au_branch *br;
+	struct au_wbr *wbr;
+	struct dentry *root;
+	struct file **to_free;
+
+	err = 0;
+	opened = 0;
+	to_free = NULL;
+	root = sb->s_root;
+	bindex = au_find_dbindex(root, del->h_path.dentry);
+	if (bindex < 0) {
+		if (remount)
+			goto out; /* success */
+		err = -ENOENT;
+		pr_err("%s no such branch\n", del->pathname);
+		goto out;
+	}
+	AuDbg("bindex b%d\n", bindex);
+
+	err = -EBUSY;
+	mnt_flags = au_mntflags(sb);
+	verbose = !!au_opt_test(mnt_flags, VERBOSE);
+	bbot = au_sbbot(sb);
+	if (unlikely(!bbot)) {
+		AuVerbose(verbose, "no more branches left\n");
+		goto out;
+	}
+
+	br = au_sbr(sb, bindex);
+	AuDebugOn(!path_equal(&br->br_path, &del->h_path));
+	if (unlikely(au_lcnt_read(&br->br_count, /*do_rev*/1))) {
+		AuVerbose(verbose, "br %pd2 is busy now\n", del->h_path.dentry);
+		goto out;
+	}
+
+	br_id = br->br_id;
+	opened = au_lcnt_read(&br->br_nfiles, /*do_rev*/1);
+	if (unlikely(opened)) {
+		to_free = au_array_alloc(&opened, empty_cb, sb, NULL);
+		err = PTR_ERR(to_free);
+		if (IS_ERR(to_free))
+			goto out;
+
+		err = test_file_busy(sb, br_id, to_free, opened);
+		if (unlikely(err)) {
+			AuVerbose(verbose, "%llu file(s) opened\n", opened);
+			goto out;
+		}
+	}
+
+	wbr = br->br_wbr;
+	do_wh = wbr && (wbr->wbr_whbase || wbr->wbr_plink || wbr->wbr_orph);
+	if (do_wh) {
+		/* instead of WbrWhMustWriteLock(wbr) */
+		SiMustWriteLock(sb);
+		for (i = 0; i < AuBrWh_Last; i++) {
+			dput(wbr->wbr_wh[i]);
+			wbr->wbr_wh[i] = NULL;
+		}
+	}
+
+	err = test_children_busy(root, bindex, verbose);
+	if (unlikely(err)) {
+		if (do_wh)
+			goto out_wh;
+		goto out;
+	}
+
+	err = 0;
+	if (to_free) {
+		/*
+		 * now we confirmed the branch is deletable.
+		 * let's free the remaining opened dirs on the branch.
+		 */
+		di_write_unlock(root);
+		br_del_file(to_free, opened, br_id);
+		di_write_lock_child(root);
+	}
+
+	sysaufs_brs_del(sb, bindex);	/* remove successors */
+	dbgaufs_xino_del(br);		/* remove one */
+	au_br_do_del(sb, bindex, br);
+	sysaufs_brs_add(sb, bindex);	/* append successors */
+	dbgaufs_brs_add(sb, bindex, /*topdown*/1);	/* rename successors */
+
+	if (!bindex) {
+		au_cpup_attr_all(d_inode(root), /*force*/1);
+		sb->s_maxbytes = au_sbr_sb(sb, 0)->s_maxbytes;
+	} else
+		au_sub_nlink(d_inode(root), d_inode(del->h_path.dentry));
+	if (au_opt_test(mnt_flags, PLINK))
+		au_plink_half_refresh(sb, br_id);
+
+	goto out; /* success */
+
+out_wh:
+	/* revert */
+	rerr = au_br_init_wh(sb, br, br->br_perm);
+	if (rerr)
+		pr_warn("failed re-creating base whiteout, %s. (%d)\n",
+			del->pathname, rerr);
+out:
+	if (to_free)
+		au_farray_free(to_free, opened);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static int au_ibusy(struct super_block *sb, struct aufs_ibusy __user *arg)
+{
+	int err;
+	aufs_bindex_t btop, bbot;
+	struct aufs_ibusy ibusy;
+	struct inode *inode, *h_inode;
+
+	err = -EPERM;
+	if (unlikely(!capable(CAP_SYS_ADMIN)))
+		goto out;
+
+	err = copy_from_user(&ibusy, arg, sizeof(ibusy));
+	if (!err)
+		err = !access_ok(VERIFY_WRITE, &arg->h_ino, sizeof(arg->h_ino));
+	if (unlikely(err)) {
+		err = -EFAULT;
+		AuTraceErr(err);
+		goto out;
+	}
+
+	err = -EINVAL;
+	si_read_lock(sb, AuLock_FLUSH);
+	if (unlikely(ibusy.bindex < 0 || ibusy.bindex > au_sbbot(sb)))
+		goto out_unlock;
+
+	err = 0;
+	ibusy.h_ino = 0; /* invalid */
+	inode = ilookup(sb, ibusy.ino);
+	if (!inode
+	    || inode->i_ino == AUFS_ROOT_INO
+	    || au_is_bad_inode(inode))
+		goto out_unlock;
+
+	ii_read_lock_child(inode);
+	btop = au_ibtop(inode);
+	bbot = au_ibbot(inode);
+	if (btop <= ibusy.bindex && ibusy.bindex <= bbot) {
+		h_inode = au_h_iptr(inode, ibusy.bindex);
+		if (h_inode && au_test_ibusy(inode, btop, bbot))
+			ibusy.h_ino = h_inode->i_ino;
+	}
+	ii_read_unlock(inode);
+	iput(inode);
+
+out_unlock:
+	si_read_unlock(sb);
+	if (!err) {
+		err = __put_user(ibusy.h_ino, &arg->h_ino);
+		if (unlikely(err)) {
+			err = -EFAULT;
+			AuTraceErr(err);
+		}
+	}
+out:
+	return err;
+}
+
+long au_ibusy_ioctl(struct file *file, unsigned long arg)
+{
+	return au_ibusy(file->f_path.dentry->d_sb, (void __user *)arg);
+}
+
+#ifdef CONFIG_COMPAT
+long au_ibusy_compat_ioctl(struct file *file, unsigned long arg)
+{
+	return au_ibusy(file->f_path.dentry->d_sb, compat_ptr(arg));
+}
+#endif
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * change a branch permission
+ */
+
+static void au_warn_ima(void)
+{
+#ifdef CONFIG_IMA
+	/* since it doesn't support mark_files_ro() */
+	AuWarn1("RW -> RO makes IMA to produce wrong message\n");
+#endif
+}
+
+static int do_need_sigen_inc(int a, int b)
+{
+	return au_br_whable(a) && !au_br_whable(b);
+}
+
+static int need_sigen_inc(int old, int new)
+{
+	return do_need_sigen_inc(old, new)
+		|| do_need_sigen_inc(new, old);
+}
+
+static int au_br_mod_files_ro(struct super_block *sb, aufs_bindex_t bindex)
+{
+	int err, do_warn;
+	unsigned int mnt_flags;
+	unsigned long long ull, max;
+	aufs_bindex_t br_id;
+	unsigned char verbose, writer;
+	struct file *file, *hf, **array;
+	struct au_hfile *hfile;
+
+	mnt_flags = au_mntflags(sb);
+	verbose = !!au_opt_test(mnt_flags, VERBOSE);
+
+	array = au_farray_alloc(sb, &max);
+	err = PTR_ERR(array);
+	if (IS_ERR(array))
+		goto out;
+
+	do_warn = 0;
+	br_id = au_sbr_id(sb, bindex);
+	for (ull = 0; ull < max; ull++) {
+		file = array[ull];
+		if (unlikely(!file))
+			break;
+
+		/* AuDbg("%pD\n", file); */
+		fi_read_lock(file);
+		if (unlikely(au_test_mmapped(file))) {
+			err = -EBUSY;
+			AuVerbose(verbose, "mmapped %pD\n", file);
+			AuDbgFile(file);
+			FiMustNoWaiters(file);
+			fi_read_unlock(file);
+			goto out_array;
+		}
+
+		hfile = &au_fi(file)->fi_htop;
+		hf = hfile->hf_file;
+		if (!d_is_reg(file->f_path.dentry)
+		    || !(file->f_mode & FMODE_WRITE)
+		    || hfile->hf_br->br_id != br_id
+		    || !(hf->f_mode & FMODE_WRITE))
+			array[ull] = NULL;
+		else {
+			do_warn = 1;
+			get_file(file);
+		}
+
+		FiMustNoWaiters(file);
+		fi_read_unlock(file);
+		fput(file);
+	}
+
+	err = 0;
+	if (do_warn)
+		au_warn_ima();
+
+	for (ull = 0; ull < max; ull++) {
+		file = array[ull];
+		if (!file)
+			continue;
+
+		/* todo: already flushed? */
+		/*
+		 * fs/super.c:mark_files_ro() is gone, but aufs keeps its
+		 * approach which resets f_mode and calls mnt_drop_write() and
+		 * file_release_write() for each file, because the branch
+		 * attribute in aufs world is totally different from the native
+		 * fs rw/ro mode.
+		*/
+		/* fi_read_lock(file); */
+		hfile = &au_fi(file)->fi_htop;
+		hf = hfile->hf_file;
+		/* fi_read_unlock(file); */
+		spin_lock(&hf->f_lock);
+		writer = !!(hf->f_mode & FMODE_WRITER);
+		hf->f_mode &= ~(FMODE_WRITE | FMODE_WRITER);
+		spin_unlock(&hf->f_lock);
+		if (writer) {
+			put_write_access(file_inode(hf));
+			__mnt_drop_write(hf->f_path.mnt);
+		}
+	}
+
+out_array:
+	au_farray_free(array, max);
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+int au_br_mod(struct super_block *sb, struct au_opt_mod *mod, int remount,
+	      int *do_refresh)
+{
+	int err, rerr;
+	aufs_bindex_t bindex;
+	struct dentry *root;
+	struct au_branch *br;
+	struct au_br_fhsm *bf;
+
+	root = sb->s_root;
+	bindex = au_find_dbindex(root, mod->h_root);
+	if (bindex < 0) {
+		if (remount)
+			return 0; /* success */
+		err = -ENOENT;
+		pr_err("%s no such branch\n", mod->path);
+		goto out;
+	}
+	AuDbg("bindex b%d\n", bindex);
+
+	err = test_br(d_inode(mod->h_root), mod->perm, mod->path);
+	if (unlikely(err))
+		goto out;
+
+	br = au_sbr(sb, bindex);
+	AuDebugOn(mod->h_root != au_br_dentry(br));
+	if (br->br_perm == mod->perm)
+		return 0; /* success */
+
+	/* pre-allocate for non-fhsm --> fhsm */
+	bf = NULL;
+	if (!au_br_fhsm(br->br_perm) && au_br_fhsm(mod->perm)) {
+		err = au_fhsm_br_alloc(br);
+		if (unlikely(err))
+			goto out;
+		bf = br->br_fhsm;
+		br->br_fhsm = NULL;
+	}
+
+	if (au_br_writable(br->br_perm)) {
+		/* remove whiteout base */
+		err = au_br_init_wh(sb, br, mod->perm);
+		if (unlikely(err))
+			goto out_bf;
+
+		if (!au_br_writable(mod->perm)) {
+			/* rw --> ro, file might be mmapped */
+			DiMustNoWaiters(root);
+			IiMustNoWaiters(d_inode(root));
+			di_write_unlock(root);
+			err = au_br_mod_files_ro(sb, bindex);
+			/* aufs_write_lock() calls ..._child() */
+			di_write_lock_child(root);
+
+			if (unlikely(err)) {
+				rerr = -ENOMEM;
+				br->br_wbr = kzalloc(sizeof(*br->br_wbr),
+						     GFP_NOFS);
+				if (br->br_wbr)
+					rerr = au_wbr_init(br, sb, br->br_perm);
+				if (unlikely(rerr)) {
+					AuIOErr("nested error %d (%d)\n",
+						rerr, err);
+					br->br_perm = mod->perm;
+				}
+			}
+		}
+	} else if (au_br_writable(mod->perm)) {
+		/* ro --> rw */
+		err = -ENOMEM;
+		br->br_wbr = kzalloc(sizeof(*br->br_wbr), GFP_NOFS);
+		if (br->br_wbr) {
+			err = au_wbr_init(br, sb, mod->perm);
+			if (unlikely(err)) {
+				au_kfree_rcu(br->br_wbr);
+				br->br_wbr = NULL;
+			}
+		}
+	}
+	if (unlikely(err))
+		goto out_bf;
+
+	if (au_br_fhsm(br->br_perm)) {
+		if (!au_br_fhsm(mod->perm)) {
+			/* fhsm --> non-fhsm */
+			au_br_fhsm_fin(br->br_fhsm);
+			au_kfree_rcu(br->br_fhsm);
+			br->br_fhsm = NULL;
+		}
+	} else if (au_br_fhsm(mod->perm))
+		/* non-fhsm --> fhsm */
+		br->br_fhsm = bf;
+
+	*do_refresh |= need_sigen_inc(br->br_perm, mod->perm);
+	br->br_perm = mod->perm;
+	goto out; /* success */
+
+out_bf:
+	au_kfree_try_rcu(bf);
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+int au_br_stfs(struct au_branch *br, struct aufs_stfs *stfs)
+{
+	int err;
+	struct kstatfs kstfs;
+
+	err = vfs_statfs(&br->br_path, &kstfs);
+	if (!err) {
+		stfs->f_blocks = kstfs.f_blocks;
+		stfs->f_bavail = kstfs.f_bavail;
+		stfs->f_files = kstfs.f_files;
+		stfs->f_ffree = kstfs.f_ffree;
+	}
+
+	return err;
+}
diff -Naur --no-dereference a/fs/aufs/branch.h b/fs/aufs/branch.h
--- a/fs/aufs/branch.h	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/branch.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,366 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * branch filesystems and xino for them
+ */
+
+#ifndef __AUFS_BRANCH_H__
+#define __AUFS_BRANCH_H__
+
+#ifdef __KERNEL__
+
+#include <linux/mount.h>
+#include "dirren.h"
+#include "dynop.h"
+#include "lcnt.h"
+#include "rwsem.h"
+#include "super.h"
+
+/* ---------------------------------------------------------------------- */
+
+/* a xino file */
+struct au_xino {
+	struct file		**xi_file;
+	unsigned int		xi_nfile;
+
+	struct {
+		spinlock_t		spin;
+		ino_t			*array;
+		int			total;
+		/* reserved for future use */
+		/* unsigned long	*bitmap; */
+		wait_queue_head_t	wqh;
+	} xi_nondir;
+
+	struct mutex		xi_mtx;	/* protects xi_file array */
+	struct hlist_bl_head	xi_writing;
+
+	atomic_t		xi_truncating;
+
+	struct kref		xi_kref;
+};
+
+/* File-based Hierarchical Storage Management */
+struct au_br_fhsm {
+#ifdef CONFIG_AUFS_FHSM
+	struct mutex		bf_lock;
+	unsigned long		bf_jiffy;
+	struct aufs_stfs	bf_stfs;
+	int			bf_readable;
+#endif
+};
+
+/* members for writable branch only */
+enum {AuBrWh_BASE, AuBrWh_PLINK, AuBrWh_ORPH, AuBrWh_Last};
+struct au_wbr {
+	struct au_rwsem		wbr_wh_rwsem;
+	struct dentry		*wbr_wh[AuBrWh_Last];
+	atomic_t		wbr_wh_running;
+#define wbr_whbase		wbr_wh[AuBrWh_BASE]	/* whiteout base */
+#define wbr_plink		wbr_wh[AuBrWh_PLINK]	/* pseudo-link dir */
+#define wbr_orph		wbr_wh[AuBrWh_ORPH]	/* dir for orphans */
+
+	/* mfs mode */
+	unsigned long long	wbr_bytes;
+};
+
+/* ext2 has 3 types of operations at least, ext3 has 4 */
+#define AuBrDynOp (AuDyLast * 4)
+
+#ifdef CONFIG_AUFS_HFSNOTIFY
+/* support for asynchronous destruction */
+struct au_br_hfsnotify {
+	struct fsnotify_group	*hfsn_group;
+};
+#endif
+
+/* sysfs entries */
+struct au_brsysfs {
+	char			name[16];
+	struct attribute	attr;
+};
+
+enum {
+	AuBrSysfs_BR,
+	AuBrSysfs_BRID,
+	AuBrSysfs_Last
+};
+
+/* protected by superblock rwsem */
+struct au_branch {
+	struct au_xino		*br_xino;
+
+	aufs_bindex_t		br_id;
+
+	int			br_perm;
+	struct path		br_path;
+	spinlock_t		br_dykey_lock;
+	struct au_dykey		*br_dykey[AuBrDynOp];
+	au_lcnt_t		br_nfiles;	/* opened files */
+	au_lcnt_t		br_count;	/* in-use for other */
+
+	struct au_wbr		*br_wbr;
+	struct au_br_fhsm	*br_fhsm;
+
+#ifdef CONFIG_AUFS_HFSNOTIFY
+	struct au_br_hfsnotify	*br_hfsn;
+#endif
+
+#ifdef CONFIG_SYSFS
+	/* entries under sysfs per mount-point */
+	struct au_brsysfs	br_sysfs[AuBrSysfs_Last];
+#endif
+
+#ifdef CONFIG_DEBUG_FS
+	struct dentry		 *br_dbgaufs; /* xino */
+#endif
+
+	struct au_dr_br		br_dirren;
+};
+
+/* ---------------------------------------------------------------------- */
+
+static inline struct vfsmount *au_br_mnt(struct au_branch *br)
+{
+	return br->br_path.mnt;
+}
+
+static inline struct dentry *au_br_dentry(struct au_branch *br)
+{
+	return br->br_path.dentry;
+}
+
+static inline struct super_block *au_br_sb(struct au_branch *br)
+{
+	return au_br_mnt(br)->mnt_sb;
+}
+
+static inline int au_br_rdonly(struct au_branch *br)
+{
+	return (sb_rdonly(au_br_sb(br))
+		|| !au_br_writable(br->br_perm))
+		? -EROFS : 0;
+}
+
+static inline int au_br_hnotifyable(int brperm __maybe_unused)
+{
+#ifdef CONFIG_AUFS_HNOTIFY
+	return !(brperm & AuBrPerm_RR);
+#else
+	return 0;
+#endif
+}
+
+static inline int au_br_test_oflag(int oflag, struct au_branch *br)
+{
+	int err, exec_flag;
+
+	err = 0;
+	exec_flag = oflag & __FMODE_EXEC;
+	if (unlikely(exec_flag && path_noexec(&br->br_path)))
+		err = -EACCES;
+
+	return err;
+}
+
+static inline void au_xino_get(struct au_branch *br)
+{
+	struct au_xino *xi;
+
+	xi = br->br_xino;
+	if (xi)
+		kref_get(&xi->xi_kref);
+}
+
+static inline int au_xino_count(struct au_branch *br)
+{
+	int v;
+	struct au_xino *xi;
+
+	v = 0;
+	xi = br->br_xino;
+	if (xi)
+		v = kref_read(&xi->xi_kref);
+
+	return v;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* branch.c */
+struct au_sbinfo;
+void au_br_free(struct au_sbinfo *sinfo);
+int au_br_index(struct super_block *sb, aufs_bindex_t br_id);
+struct au_opt_add;
+int au_br_add(struct super_block *sb, struct au_opt_add *add, int remount);
+struct au_opt_del;
+int au_br_del(struct super_block *sb, struct au_opt_del *del, int remount);
+long au_ibusy_ioctl(struct file *file, unsigned long arg);
+#ifdef CONFIG_COMPAT
+long au_ibusy_compat_ioctl(struct file *file, unsigned long arg);
+#endif
+struct au_opt_mod;
+int au_br_mod(struct super_block *sb, struct au_opt_mod *mod, int remount,
+	      int *do_refresh);
+struct aufs_stfs;
+int au_br_stfs(struct au_branch *br, struct aufs_stfs *stfs);
+
+/* xino.c */
+static const loff_t au_loff_max = LLONG_MAX;
+
+aufs_bindex_t au_xi_root(struct super_block *sb, struct dentry *dentry);
+struct file *au_xino_create(struct super_block *sb, char *fpath, int silent,
+			    int wbrtop);
+struct file *au_xino_create2(struct super_block *sb, struct path *base,
+			     struct file *copy_src);
+struct au_xi_new {
+	struct au_xino *xi;	/* switch between xino and xigen */
+	int idx;
+	struct path *base;
+	struct file *copy_src;
+};
+struct file *au_xi_new(struct super_block *sb, struct au_xi_new *xinew);
+
+int au_xino_read(struct super_block *sb, aufs_bindex_t bindex, ino_t h_ino,
+		 ino_t *ino);
+int au_xino_write(struct super_block *sb, aufs_bindex_t bindex, ino_t h_ino,
+		  ino_t ino);
+ssize_t xino_fread(vfs_readf_t func, struct file *file, void *buf, size_t size,
+		   loff_t *pos);
+ssize_t xino_fwrite(vfs_writef_t func, struct file *file, void *buf,
+		    size_t size, loff_t *pos);
+
+int au_xib_trunc(struct super_block *sb);
+int au_xino_trunc(struct super_block *sb, aufs_bindex_t bindex, int idx_begin);
+
+struct au_xino *au_xino_alloc(unsigned int nfile);
+int au_xino_put(struct au_branch *br);
+struct file *au_xino_file1(struct au_xino *xi);
+
+struct au_opt_xino;
+void au_xino_clr(struct super_block *sb);
+int au_xino_set(struct super_block *sb, struct au_opt_xino *xiopt, int remount);
+struct file *au_xino_def(struct super_block *sb);
+int au_xino_init_br(struct super_block *sb, struct au_branch *br, ino_t hino,
+		    struct path *base);
+
+ino_t au_xino_new_ino(struct super_block *sb);
+void au_xino_delete_inode(struct inode *inode, const int unlinked);
+
+void au_xinondir_leave(struct super_block *sb, aufs_bindex_t bindex,
+		       ino_t h_ino, int idx);
+int au_xinondir_enter(struct super_block *sb, aufs_bindex_t bindex, ino_t h_ino,
+		      int *idx);
+
+int au_xino_path(struct seq_file *seq, struct file *file);
+
+/* ---------------------------------------------------------------------- */
+
+/* @idx is signed to accept -1 meaning the first file */
+static inline struct file *au_xino_file(struct au_xino *xi, int idx)
+{
+	struct file *file;
+
+	file = NULL;
+	if (!xi)
+		goto out;
+
+	if (idx >= 0) {
+		if (idx < xi->xi_nfile)
+			file = xi->xi_file[idx];
+	} else
+		file = au_xino_file1(xi);
+
+out:
+	return file;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* Superblock to branch */
+static inline
+aufs_bindex_t au_sbr_id(struct super_block *sb, aufs_bindex_t bindex)
+{
+	return au_sbr(sb, bindex)->br_id;
+}
+
+static inline
+struct vfsmount *au_sbr_mnt(struct super_block *sb, aufs_bindex_t bindex)
+{
+	return au_br_mnt(au_sbr(sb, bindex));
+}
+
+static inline
+struct super_block *au_sbr_sb(struct super_block *sb, aufs_bindex_t bindex)
+{
+	return au_br_sb(au_sbr(sb, bindex));
+}
+
+static inline int au_sbr_perm(struct super_block *sb, aufs_bindex_t bindex)
+{
+	return au_sbr(sb, bindex)->br_perm;
+}
+
+static inline int au_sbr_whable(struct super_block *sb, aufs_bindex_t bindex)
+{
+	return au_br_whable(au_sbr_perm(sb, bindex));
+}
+
+/* ---------------------------------------------------------------------- */
+
+#define wbr_wh_read_lock(wbr)	au_rw_read_lock(&(wbr)->wbr_wh_rwsem)
+#define wbr_wh_write_lock(wbr)	au_rw_write_lock(&(wbr)->wbr_wh_rwsem)
+#define wbr_wh_read_trylock(wbr)	au_rw_read_trylock(&(wbr)->wbr_wh_rwsem)
+#define wbr_wh_write_trylock(wbr) au_rw_write_trylock(&(wbr)->wbr_wh_rwsem)
+/*
+#define wbr_wh_read_trylock_nested(wbr) \
+	au_rw_read_trylock_nested(&(wbr)->wbr_wh_rwsem)
+#define wbr_wh_write_trylock_nested(wbr) \
+	au_rw_write_trylock_nested(&(wbr)->wbr_wh_rwsem)
+*/
+
+#define wbr_wh_read_unlock(wbr)	au_rw_read_unlock(&(wbr)->wbr_wh_rwsem)
+#define wbr_wh_write_unlock(wbr)	au_rw_write_unlock(&(wbr)->wbr_wh_rwsem)
+#define wbr_wh_downgrade_lock(wbr)	au_rw_dgrade_lock(&(wbr)->wbr_wh_rwsem)
+
+#define WbrWhMustNoWaiters(wbr)	AuRwMustNoWaiters(&(wbr)->wbr_wh_rwsem)
+#define WbrWhMustAnyLock(wbr)	AuRwMustAnyLock(&(wbr)->wbr_wh_rwsem)
+#define WbrWhMustWriteLock(wbr)	AuRwMustWriteLock(&(wbr)->wbr_wh_rwsem)
+
+/* ---------------------------------------------------------------------- */
+
+#ifdef CONFIG_AUFS_FHSM
+static inline void au_br_fhsm_init(struct au_br_fhsm *brfhsm)
+{
+	mutex_init(&brfhsm->bf_lock);
+	brfhsm->bf_jiffy = 0;
+	brfhsm->bf_readable = 0;
+}
+
+static inline void au_br_fhsm_fin(struct au_br_fhsm *brfhsm)
+{
+	mutex_destroy(&brfhsm->bf_lock);
+}
+#else
+AuStubVoid(au_br_fhsm_init, struct au_br_fhsm *brfhsm)
+AuStubVoid(au_br_fhsm_fin, struct au_br_fhsm *brfhsm)
+#endif
+
+#endif /* __KERNEL__ */
+#endif /* __AUFS_BRANCH_H__ */
diff -Naur --no-dereference a/fs/aufs/conf.mk b/fs/aufs/conf.mk
--- a/fs/aufs/conf.mk	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/conf.mk	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,40 @@
+# SPDX-License-Identifier: GPL-2.0
+
+AuConfStr = CONFIG_AUFS_FS=${CONFIG_AUFS_FS}
+
+define AuConf
+ifdef ${1}
+AuConfStr += ${1}=${${1}}
+endif
+endef
+
+AuConfAll = BRANCH_MAX_127 BRANCH_MAX_511 BRANCH_MAX_1023 BRANCH_MAX_32767 \
+	SBILIST \
+	HNOTIFY HFSNOTIFY \
+	EXPORT INO_T_64 \
+	XATTR \
+	FHSM \
+	RDU \
+	DIRREN \
+	SHWH \
+	BR_RAMFS \
+	BR_FUSE POLL \
+	BR_HFSPLUS \
+	BDEV_LOOP \
+	DEBUG MAGIC_SYSRQ
+$(foreach i, ${AuConfAll}, \
+	$(eval $(call AuConf,CONFIG_AUFS_${i})))
+
+AuConfName = ${obj}/conf.str
+${AuConfName}.tmp: FORCE
+	@echo ${AuConfStr} | tr ' ' '\n' | sed -e 's/^/"/' -e 's/$$/\\n"/' > $@
+${AuConfName}: ${AuConfName}.tmp
+	@diff -q $< $@ > /dev/null 2>&1 || { \
+	echo '  GEN    ' $@; \
+	cp -p $< $@; \
+	}
+FORCE:
+clean-files += ${AuConfName} ${AuConfName}.tmp
+${obj}/sysfs.o: ${AuConfName}
+
+-include ${srctree}/${src}/conf_priv.mk
diff -Naur --no-dereference a/fs/aufs/cpup.c b/fs/aufs/cpup.c
--- a/fs/aufs/cpup.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/cpup.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,1448 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * copy-up functions, see wbr_policy.c for copy-down
+ */
+
+#include <linux/fs_stack.h>
+#include <linux/mm.h>
+#include <linux/task_work.h>
+#include "aufs.h"
+
+void au_cpup_attr_flags(struct inode *dst, unsigned int iflags)
+{
+	const unsigned int mask = S_DEAD | S_SWAPFILE | S_PRIVATE
+		| S_NOATIME | S_NOCMTIME | S_AUTOMOUNT;
+
+	BUILD_BUG_ON(sizeof(iflags) != sizeof(dst->i_flags));
+
+	dst->i_flags |= iflags & ~mask;
+	if (au_test_fs_notime(dst->i_sb))
+		dst->i_flags |= S_NOATIME | S_NOCMTIME;
+}
+
+void au_cpup_attr_timesizes(struct inode *inode)
+{
+	struct inode *h_inode;
+
+	h_inode = au_h_iptr(inode, au_ibtop(inode));
+	fsstack_copy_attr_times(inode, h_inode);
+	fsstack_copy_inode_size(inode, h_inode);
+}
+
+void au_cpup_attr_nlink(struct inode *inode, int force)
+{
+	struct inode *h_inode;
+	struct super_block *sb;
+	aufs_bindex_t bindex, bbot;
+
+	sb = inode->i_sb;
+	bindex = au_ibtop(inode);
+	h_inode = au_h_iptr(inode, bindex);
+	if (!force
+	    && !S_ISDIR(h_inode->i_mode)
+	    && au_opt_test(au_mntflags(sb), PLINK)
+	    && au_plink_test(inode))
+		return;
+
+	/*
+	 * 0 can happen in revalidating.
+	 * h_inode->i_mutex may not be held here, but it is harmless since once
+	 * i_nlink reaches 0, it will never become positive except O_TMPFILE
+	 * case.
+	 * todo: O_TMPFILE+linkat(AT_SYMLINK_FOLLOW) bypassing aufs may cause
+	 *	 the incorrect link count.
+	 */
+	set_nlink(inode, h_inode->i_nlink);
+
+	/*
+	 * fewer nlink makes find(1) noisy, but larger nlink doesn't.
+	 * it may includes whplink directory.
+	 */
+	if (S_ISDIR(h_inode->i_mode)) {
+		bbot = au_ibbot(inode);
+		for (bindex++; bindex <= bbot; bindex++) {
+			h_inode = au_h_iptr(inode, bindex);
+			if (h_inode)
+				au_add_nlink(inode, h_inode);
+		}
+	}
+}
+
+void au_cpup_attr_changeable(struct inode *inode)
+{
+	struct inode *h_inode;
+
+	h_inode = au_h_iptr(inode, au_ibtop(inode));
+	inode->i_mode = h_inode->i_mode;
+	inode->i_uid = h_inode->i_uid;
+	inode->i_gid = h_inode->i_gid;
+	au_cpup_attr_timesizes(inode);
+	au_cpup_attr_flags(inode, h_inode->i_flags);
+}
+
+void au_cpup_igen(struct inode *inode, struct inode *h_inode)
+{
+	struct au_iinfo *iinfo = au_ii(inode);
+
+	IiMustWriteLock(inode);
+
+	iinfo->ii_higen = h_inode->i_generation;
+	iinfo->ii_hsb1 = h_inode->i_sb;
+}
+
+void au_cpup_attr_all(struct inode *inode, int force)
+{
+	struct inode *h_inode;
+
+	h_inode = au_h_iptr(inode, au_ibtop(inode));
+	au_cpup_attr_changeable(inode);
+	if (inode->i_nlink > 0)
+		au_cpup_attr_nlink(inode, force);
+	inode->i_rdev = h_inode->i_rdev;
+	inode->i_blkbits = h_inode->i_blkbits;
+	au_cpup_igen(inode, h_inode);
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* Note: dt_dentry and dt_h_dentry are not dget/dput-ed */
+
+/* keep the timestamps of the parent dir when cpup */
+void au_dtime_store(struct au_dtime *dt, struct dentry *dentry,
+		    struct path *h_path)
+{
+	struct inode *h_inode;
+
+	dt->dt_dentry = dentry;
+	dt->dt_h_path = *h_path;
+	h_inode = d_inode(h_path->dentry);
+	dt->dt_atime = h_inode->i_atime;
+	dt->dt_mtime = h_inode->i_mtime;
+	/* smp_mb(); */
+}
+
+void au_dtime_revert(struct au_dtime *dt)
+{
+	struct iattr attr;
+	int err;
+
+	attr.ia_atime = dt->dt_atime;
+	attr.ia_mtime = dt->dt_mtime;
+	attr.ia_valid = ATTR_FORCE | ATTR_MTIME | ATTR_MTIME_SET
+		| ATTR_ATIME | ATTR_ATIME_SET;
+
+	/* no delegation since this is a directory */
+	err = vfsub_notify_change(&dt->dt_h_path, &attr, /*delegated*/NULL);
+	if (unlikely(err))
+		pr_warn("restoring timestamps failed(%d). ignored\n", err);
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* internal use only */
+struct au_cpup_reg_attr {
+	int		valid;
+	struct kstat	st;
+	unsigned int	iflags; /* inode->i_flags */
+};
+
+static noinline_for_stack
+int cpup_iattr(struct dentry *dst, aufs_bindex_t bindex, struct dentry *h_src,
+	       struct au_cpup_reg_attr *h_src_attr)
+{
+	int err, sbits, icex;
+	unsigned int mnt_flags;
+	unsigned char verbose;
+	struct iattr ia;
+	struct path h_path;
+	struct inode *h_isrc, *h_idst;
+	struct kstat *h_st;
+	struct au_branch *br;
+
+	h_path.dentry = au_h_dptr(dst, bindex);
+	h_idst = d_inode(h_path.dentry);
+	br = au_sbr(dst->d_sb, bindex);
+	h_path.mnt = au_br_mnt(br);
+	h_isrc = d_inode(h_src);
+	ia.ia_valid = ATTR_FORCE | ATTR_UID | ATTR_GID
+		| ATTR_ATIME | ATTR_MTIME
+		| ATTR_ATIME_SET | ATTR_MTIME_SET;
+	if (h_src_attr && h_src_attr->valid) {
+		h_st = &h_src_attr->st;
+		ia.ia_uid = h_st->uid;
+		ia.ia_gid = h_st->gid;
+		ia.ia_atime = h_st->atime;
+		ia.ia_mtime = h_st->mtime;
+		if (h_idst->i_mode != h_st->mode
+		    && !S_ISLNK(h_idst->i_mode)) {
+			ia.ia_valid |= ATTR_MODE;
+			ia.ia_mode = h_st->mode;
+		}
+		sbits = !!(h_st->mode & (S_ISUID | S_ISGID));
+		au_cpup_attr_flags(h_idst, h_src_attr->iflags);
+	} else {
+		ia.ia_uid = h_isrc->i_uid;
+		ia.ia_gid = h_isrc->i_gid;
+		ia.ia_atime = h_isrc->i_atime;
+		ia.ia_mtime = h_isrc->i_mtime;
+		if (h_idst->i_mode != h_isrc->i_mode
+		    && !S_ISLNK(h_idst->i_mode)) {
+			ia.ia_valid |= ATTR_MODE;
+			ia.ia_mode = h_isrc->i_mode;
+		}
+		sbits = !!(h_isrc->i_mode & (S_ISUID | S_ISGID));
+		au_cpup_attr_flags(h_idst, h_isrc->i_flags);
+	}
+	/* no delegation since it is just created */
+	err = vfsub_notify_change(&h_path, &ia, /*delegated*/NULL);
+
+	/* is this nfs only? */
+	if (!err && sbits && au_test_nfs(h_path.dentry->d_sb)) {
+		ia.ia_valid = ATTR_FORCE | ATTR_MODE;
+		ia.ia_mode = h_isrc->i_mode;
+		err = vfsub_notify_change(&h_path, &ia, /*delegated*/NULL);
+	}
+
+	icex = br->br_perm & AuBrAttr_ICEX;
+	if (!err) {
+		mnt_flags = au_mntflags(dst->d_sb);
+		verbose = !!au_opt_test(mnt_flags, VERBOSE);
+		err = au_cpup_xattr(h_path.dentry, h_src, icex, verbose);
+	}
+
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static int au_do_copy_file(struct file *dst, struct file *src, loff_t len,
+			   char *buf, unsigned long blksize)
+{
+	int err;
+	size_t sz, rbytes, wbytes;
+	unsigned char all_zero;
+	char *p, *zp;
+	struct inode *h_inode;
+	/* reduce stack usage */
+	struct iattr *ia;
+
+	zp = page_address(ZERO_PAGE(0));
+	if (unlikely(!zp))
+		return -ENOMEM; /* possible? */
+
+	err = 0;
+	all_zero = 0;
+	while (len) {
+		AuDbg("len %lld\n", len);
+		sz = blksize;
+		if (len < blksize)
+			sz = len;
+
+		rbytes = 0;
+		/* todo: signal_pending? */
+		while (!rbytes || err == -EAGAIN || err == -EINTR) {
+			rbytes = vfsub_read_k(src, buf, sz, &src->f_pos);
+			err = rbytes;
+		}
+		if (unlikely(err < 0))
+			break;
+
+		all_zero = 0;
+		if (len >= rbytes && rbytes == blksize)
+			all_zero = !memcmp(buf, zp, rbytes);
+		if (!all_zero) {
+			wbytes = rbytes;
+			p = buf;
+			while (wbytes) {
+				size_t b;
+
+				b = vfsub_write_k(dst, p, wbytes, &dst->f_pos);
+				err = b;
+				/* todo: signal_pending? */
+				if (unlikely(err == -EAGAIN || err == -EINTR))
+					continue;
+				if (unlikely(err < 0))
+					break;
+				wbytes -= b;
+				p += b;
+			}
+			if (unlikely(err < 0))
+				break;
+		} else {
+			loff_t res;
+
+			AuLabel(hole);
+			res = vfsub_llseek(dst, rbytes, SEEK_CUR);
+			err = res;
+			if (unlikely(res < 0))
+				break;
+		}
+		len -= rbytes;
+		err = 0;
+	}
+
+	/* the last block may be a hole */
+	if (!err && all_zero) {
+		AuLabel(last hole);
+
+		err = 1;
+		if (au_test_nfs(dst->f_path.dentry->d_sb)) {
+			/* nfs requires this step to make last hole */
+			/* is this only nfs? */
+			do {
+				/* todo: signal_pending? */
+				err = vfsub_write_k(dst, "\0", 1, &dst->f_pos);
+			} while (err == -EAGAIN || err == -EINTR);
+			if (err == 1)
+				dst->f_pos--;
+		}
+
+		if (err == 1) {
+			ia = (void *)buf;
+			ia->ia_size = dst->f_pos;
+			ia->ia_valid = ATTR_SIZE | ATTR_FILE;
+			ia->ia_file = dst;
+			h_inode = file_inode(dst);
+			inode_lock_nested(h_inode, AuLsc_I_CHILD2);
+			/* no delegation since it is just created */
+			err = vfsub_notify_change(&dst->f_path, ia,
+						  /*delegated*/NULL);
+			inode_unlock(h_inode);
+		}
+	}
+
+	return err;
+}
+
+int au_copy_file(struct file *dst, struct file *src, loff_t len)
+{
+	int err;
+	unsigned long blksize;
+	unsigned char do_kfree;
+	char *buf;
+	struct super_block *h_sb;
+
+	err = -ENOMEM;
+	h_sb = file_inode(dst)->i_sb;
+	blksize = h_sb->s_blocksize;
+	if (!blksize || PAGE_SIZE < blksize)
+		blksize = PAGE_SIZE;
+	AuDbg("blksize %lu\n", blksize);
+	do_kfree = (blksize != PAGE_SIZE && blksize >= sizeof(struct iattr *));
+	if (do_kfree)
+		buf = kmalloc(blksize, GFP_NOFS);
+	else
+		buf = (void *)__get_free_page(GFP_NOFS);
+	if (unlikely(!buf))
+		goto out;
+
+	if (len > (1 << 22))
+		AuDbg("copying a large file %lld\n", (long long)len);
+
+	src->f_pos = 0;
+	dst->f_pos = 0;
+	err = au_do_copy_file(dst, src, len, buf, blksize);
+	if (do_kfree) {
+		AuDebugOn(!au_kfree_do_sz_test(blksize));
+		au_kfree_do_rcu(buf);
+	} else
+		free_page((unsigned long)buf);
+
+out:
+	return err;
+}
+
+static int au_do_copy(struct file *dst, struct file *src, loff_t len)
+{
+	int err;
+	struct super_block *h_src_sb;
+	struct inode *h_src_inode;
+
+	h_src_inode = file_inode(src);
+	h_src_sb = h_src_inode->i_sb;
+
+	/* XFS acquires inode_lock */
+	if (!au_test_xfs(h_src_sb))
+		err = au_copy_file(dst, src, len);
+	else {
+		inode_unlock_shared(h_src_inode);
+		err = au_copy_file(dst, src, len);
+		inode_lock_shared_nested(h_src_inode, AuLsc_I_CHILD);
+	}
+
+	return err;
+}
+
+static int au_clone_or_copy(struct file *dst, struct file *src, loff_t len)
+{
+	int err;
+	struct super_block *h_src_sb;
+	struct inode *h_src_inode;
+
+	h_src_inode = file_inode(src);
+	h_src_sb = h_src_inode->i_sb;
+	if (h_src_sb != file_inode(dst)->i_sb
+	    || !dst->f_op->clone_file_range) {
+		err = au_do_copy(dst, src, len);
+		goto out;
+	}
+
+	if (!au_test_nfs(h_src_sb)) {
+		inode_unlock_shared(h_src_inode);
+		err = vfsub_clone_file_range(src, dst, len);
+		inode_lock_shared_nested(h_src_inode, AuLsc_I_CHILD);
+	} else
+		err = vfsub_clone_file_range(src, dst, len);
+	/* older XFS has a condition in cloning */
+	if (unlikely(err != -EOPNOTSUPP))
+		goto out;
+
+	/* the backend fs on NFS may not support cloning */
+	err = au_do_copy(dst, src, len);
+
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+/*
+ * to support a sparse file which is opened with O_APPEND,
+ * we need to close the file.
+ */
+static int au_cp_regular(struct au_cp_generic *cpg)
+{
+	int err, i;
+	enum { SRC, DST };
+	struct {
+		aufs_bindex_t bindex;
+		unsigned int flags;
+		struct dentry *dentry;
+		int force_wr;
+		struct file *file;
+	} *f, file[] = {
+		{
+			.bindex = cpg->bsrc,
+			.flags = O_RDONLY | O_NOATIME | O_LARGEFILE,
+		},
+		{
+			.bindex = cpg->bdst,
+			.flags = O_WRONLY | O_NOATIME | O_LARGEFILE,
+			.force_wr = !!au_ftest_cpup(cpg->flags, RWDST),
+		}
+	};
+	struct au_branch *br;
+	struct super_block *sb, *h_src_sb;
+	struct inode *h_src_inode;
+	struct task_struct *tsk = current;
+
+	/* bsrc branch can be ro/rw. */
+	sb = cpg->dentry->d_sb;
+	f = file;
+	for (i = 0; i < 2; i++, f++) {
+		f->dentry = au_h_dptr(cpg->dentry, f->bindex);
+		f->file = au_h_open(cpg->dentry, f->bindex, f->flags,
+				    /*file*/NULL, f->force_wr);
+		if (IS_ERR(f->file)) {
+			err = PTR_ERR(f->file);
+			if (i == SRC)
+				goto out;
+			else
+				goto out_src;
+		}
+	}
+
+	/* try stopping to update while we copyup */
+	h_src_inode = d_inode(file[SRC].dentry);
+	h_src_sb = h_src_inode->i_sb;
+	if (!au_test_nfs(h_src_sb))
+		IMustLock(h_src_inode);
+	err = au_clone_or_copy(file[DST].file, file[SRC].file, cpg->len);
+
+	/* i wonder if we had O_NO_DELAY_FPUT flag */
+	if (tsk->flags & PF_KTHREAD)
+		__fput_sync(file[DST].file);
+	else {
+		/* it happened actually */
+		fput(file[DST].file);
+		/*
+		 * too bad.
+		 * we have to call both since we don't know which place the file
+		 * was added to.
+		 */
+		task_work_run();
+		flush_delayed_fput();
+	}
+	br = au_sbr(sb, file[DST].bindex);
+	au_lcnt_dec(&br->br_nfiles);
+
+out_src:
+	fput(file[SRC].file);
+	br = au_sbr(sb, file[SRC].bindex);
+	au_lcnt_dec(&br->br_nfiles);
+out:
+	return err;
+}
+
+static int au_do_cpup_regular(struct au_cp_generic *cpg,
+			      struct au_cpup_reg_attr *h_src_attr)
+{
+	int err, rerr;
+	loff_t l;
+	struct path h_path;
+	struct inode *h_src_inode, *h_dst_inode;
+
+	err = 0;
+	h_src_inode = au_h_iptr(d_inode(cpg->dentry), cpg->bsrc);
+	l = i_size_read(h_src_inode);
+	if (cpg->len == -1 || l < cpg->len)
+		cpg->len = l;
+	if (cpg->len) {
+		/* try stopping to update while we are referencing */
+		inode_lock_shared_nested(h_src_inode, AuLsc_I_CHILD);
+		au_pin_hdir_unlock(cpg->pin);
+
+		h_path.dentry = au_h_dptr(cpg->dentry, cpg->bsrc);
+		h_path.mnt = au_sbr_mnt(cpg->dentry->d_sb, cpg->bsrc);
+		h_src_attr->iflags = h_src_inode->i_flags;
+		if (!au_test_nfs(h_src_inode->i_sb))
+			err = vfsub_getattr(&h_path, &h_src_attr->st);
+		else {
+			inode_unlock_shared(h_src_inode);
+			err = vfsub_getattr(&h_path, &h_src_attr->st);
+			inode_lock_shared_nested(h_src_inode, AuLsc_I_CHILD);
+		}
+		if (unlikely(err)) {
+			inode_unlock_shared(h_src_inode);
+			goto out;
+		}
+		h_src_attr->valid = 1;
+		if (!au_test_nfs(h_src_inode->i_sb)) {
+			err = au_cp_regular(cpg);
+			inode_unlock_shared(h_src_inode);
+		} else {
+			inode_unlock_shared(h_src_inode);
+			err = au_cp_regular(cpg);
+		}
+		rerr = au_pin_hdir_relock(cpg->pin);
+		if (!err && rerr)
+			err = rerr;
+	}
+	if (!err && (h_src_inode->i_state & I_LINKABLE)) {
+		h_path.dentry = au_h_dptr(cpg->dentry, cpg->bdst);
+		h_dst_inode = d_inode(h_path.dentry);
+		spin_lock(&h_dst_inode->i_lock);
+		h_dst_inode->i_state |= I_LINKABLE;
+		spin_unlock(&h_dst_inode->i_lock);
+	}
+
+out:
+	return err;
+}
+
+static int au_do_cpup_symlink(struct path *h_path, struct dentry *h_src,
+			      struct inode *h_dir)
+{
+	int err, symlen;
+	mm_segment_t old_fs;
+	union {
+		char *k;
+		char __user *u;
+	} sym;
+
+	err = -ENOMEM;
+	sym.k = (void *)__get_free_page(GFP_NOFS);
+	if (unlikely(!sym.k))
+		goto out;
+
+	/* unnecessary to support mmap_sem since symlink is not mmap-able */
+	old_fs = get_fs();
+	set_fs(KERNEL_DS);
+	symlen = vfs_readlink(h_src, sym.u, PATH_MAX);
+	err = symlen;
+	set_fs(old_fs);
+
+	if (symlen > 0) {
+		sym.k[symlen] = 0;
+		err = vfsub_symlink(h_dir, h_path, sym.k);
+	}
+	free_page((unsigned long)sym.k);
+
+out:
+	return err;
+}
+
+/*
+ * regardless 'acl' option, reset all ACL.
+ * All ACL will be copied up later from the original entry on the lower branch.
+ */
+static int au_reset_acl(struct inode *h_dir, struct path *h_path, umode_t mode)
+{
+	int err;
+	struct dentry *h_dentry;
+	struct inode *h_inode;
+
+	h_dentry = h_path->dentry;
+	h_inode = d_inode(h_dentry);
+	/* forget_all_cached_acls(h_inode)); */
+	err = vfsub_removexattr(h_dentry, XATTR_NAME_POSIX_ACL_ACCESS);
+	AuTraceErr(err);
+	if (err == -EOPNOTSUPP)
+		err = 0;
+	if (!err)
+		err = vfsub_acl_chmod(h_inode, mode);
+
+	AuTraceErr(err);
+	return err;
+}
+
+static int au_do_cpup_dir(struct au_cp_generic *cpg, struct dentry *dst_parent,
+			  struct inode *h_dir, struct path *h_path)
+{
+	int err;
+	struct inode *dir, *inode;
+
+	err = vfsub_removexattr(h_path->dentry, XATTR_NAME_POSIX_ACL_DEFAULT);
+	AuTraceErr(err);
+	if (err == -EOPNOTSUPP)
+		err = 0;
+	if (unlikely(err))
+		goto out;
+
+	/*
+	 * strange behaviour from the users view,
+	 * particularly setattr case
+	 */
+	dir = d_inode(dst_parent);
+	if (au_ibtop(dir) == cpg->bdst)
+		au_cpup_attr_nlink(dir, /*force*/1);
+	inode = d_inode(cpg->dentry);
+	au_cpup_attr_nlink(inode, /*force*/1);
+
+out:
+	return err;
+}
+
+static noinline_for_stack
+int cpup_entry(struct au_cp_generic *cpg, struct dentry *dst_parent,
+	       struct au_cpup_reg_attr *h_src_attr)
+{
+	int err;
+	umode_t mode;
+	unsigned int mnt_flags;
+	unsigned char isdir, isreg, force;
+	const unsigned char do_dt = !!au_ftest_cpup(cpg->flags, DTIME);
+	struct au_dtime dt;
+	struct path h_path;
+	struct dentry *h_src, *h_dst, *h_parent;
+	struct inode *h_inode, *h_dir;
+	struct super_block *sb;
+
+	/* bsrc branch can be ro/rw. */
+	h_src = au_h_dptr(cpg->dentry, cpg->bsrc);
+	h_inode = d_inode(h_src);
+	AuDebugOn(h_inode != au_h_iptr(d_inode(cpg->dentry), cpg->bsrc));
+
+	/* try stopping to be referenced while we are creating */
+	h_dst = au_h_dptr(cpg->dentry, cpg->bdst);
+	if (au_ftest_cpup(cpg->flags, RENAME))
+		AuDebugOn(strncmp(h_dst->d_name.name, AUFS_WH_PFX,
+				  AUFS_WH_PFX_LEN));
+	h_parent = h_dst->d_parent; /* dir inode is locked */
+	h_dir = d_inode(h_parent);
+	IMustLock(h_dir);
+	AuDebugOn(h_parent != h_dst->d_parent);
+
+	sb = cpg->dentry->d_sb;
+	h_path.mnt = au_sbr_mnt(sb, cpg->bdst);
+	if (do_dt) {
+		h_path.dentry = h_parent;
+		au_dtime_store(&dt, dst_parent, &h_path);
+	}
+	h_path.dentry = h_dst;
+
+	isreg = 0;
+	isdir = 0;
+	mode = h_inode->i_mode;
+	switch (mode & S_IFMT) {
+	case S_IFREG:
+		isreg = 1;
+		err = vfsub_create(h_dir, &h_path, 0600, /*want_excl*/true);
+		if (!err)
+			err = au_do_cpup_regular(cpg, h_src_attr);
+		break;
+	case S_IFDIR:
+		isdir = 1;
+		err = vfsub_mkdir(h_dir, &h_path, mode);
+		if (!err)
+			err = au_do_cpup_dir(cpg, dst_parent, h_dir, &h_path);
+		break;
+	case S_IFLNK:
+		err = au_do_cpup_symlink(&h_path, h_src, h_dir);
+		break;
+	case S_IFCHR:
+	case S_IFBLK:
+		AuDebugOn(!capable(CAP_MKNOD));
+		/*FALLTHROUGH*/
+	case S_IFIFO:
+	case S_IFSOCK:
+		err = vfsub_mknod(h_dir, &h_path, mode, h_inode->i_rdev);
+		break;
+	default:
+		AuIOErr("Unknown inode type 0%o\n", mode);
+		err = -EIO;
+	}
+	if (!err)
+		err = au_reset_acl(h_dir, &h_path, mode);
+
+	mnt_flags = au_mntflags(sb);
+	if (!au_opt_test(mnt_flags, UDBA_NONE)
+	    && !isdir
+	    && au_opt_test(mnt_flags, XINO)
+	    && (h_inode->i_nlink == 1
+		|| (h_inode->i_state & I_LINKABLE))
+	    /* todo: unnecessary? */
+	    /* && d_inode(cpg->dentry)->i_nlink == 1 */
+	    && cpg->bdst < cpg->bsrc
+	    && !au_ftest_cpup(cpg->flags, KEEPLINO))
+		au_xino_write(sb, cpg->bsrc, h_inode->i_ino, /*ino*/0);
+		/* ignore this error */
+
+	if (!err) {
+		force = 0;
+		if (isreg) {
+			force = !!cpg->len;
+			if (cpg->len == -1)
+				force = !!i_size_read(h_inode);
+		}
+		au_fhsm_wrote(sb, cpg->bdst, force);
+	}
+
+	if (do_dt)
+		au_dtime_revert(&dt);
+	return err;
+}
+
+static int au_do_ren_after_cpup(struct au_cp_generic *cpg, struct path *h_path)
+{
+	int err;
+	struct dentry *dentry, *h_dentry, *h_parent, *parent;
+	struct inode *h_dir;
+	aufs_bindex_t bdst;
+
+	dentry = cpg->dentry;
+	bdst = cpg->bdst;
+	h_dentry = au_h_dptr(dentry, bdst);
+	if (!au_ftest_cpup(cpg->flags, OVERWRITE)) {
+		dget(h_dentry);
+		au_set_h_dptr(dentry, bdst, NULL);
+		err = au_lkup_neg(dentry, bdst, /*wh*/0);
+		if (!err)
+			h_path->dentry = dget(au_h_dptr(dentry, bdst));
+		au_set_h_dptr(dentry, bdst, h_dentry);
+	} else {
+		err = 0;
+		parent = dget_parent(dentry);
+		h_parent = au_h_dptr(parent, bdst);
+		dput(parent);
+		h_path->dentry = vfsub_lkup_one(&dentry->d_name, h_parent);
+		if (IS_ERR(h_path->dentry))
+			err = PTR_ERR(h_path->dentry);
+	}
+	if (unlikely(err))
+		goto out;
+
+	h_parent = h_dentry->d_parent; /* dir inode is locked */
+	h_dir = d_inode(h_parent);
+	IMustLock(h_dir);
+	AuDbg("%pd %pd\n", h_dentry, h_path->dentry);
+	/* no delegation since it is just created */
+	err = vfsub_rename(h_dir, h_dentry, h_dir, h_path, /*delegated*/NULL,
+			   /*flags*/0);
+	dput(h_path->dentry);
+
+out:
+	return err;
+}
+
+/*
+ * copyup the @dentry from @bsrc to @bdst.
+ * the caller must set the both of lower dentries.
+ * @len is for truncating when it is -1 copyup the entire file.
+ * in link/rename cases, @dst_parent may be different from the real one.
+ * basic->bsrc can be larger than basic->bdst.
+ * aufs doesn't touch the credential so
+ * security_inode_copy_up{,_xattr}() are unnecessary.
+ */
+static int au_cpup_single(struct au_cp_generic *cpg, struct dentry *dst_parent)
+{
+	int err, rerr;
+	aufs_bindex_t old_ibtop;
+	unsigned char isdir, plink;
+	struct dentry *h_src, *h_dst, *h_parent;
+	struct inode *dst_inode, *h_dir, *inode, *delegated, *src_inode;
+	struct super_block *sb;
+	struct au_branch *br;
+	/* to reduce stack size */
+	struct {
+		struct au_dtime dt;
+		struct path h_path;
+		struct au_cpup_reg_attr h_src_attr;
+	} *a;
+
+	err = -ENOMEM;
+	a = kmalloc(sizeof(*a), GFP_NOFS);
+	if (unlikely(!a))
+		goto out;
+	a->h_src_attr.valid = 0;
+
+	sb = cpg->dentry->d_sb;
+	br = au_sbr(sb, cpg->bdst);
+	a->h_path.mnt = au_br_mnt(br);
+	h_dst = au_h_dptr(cpg->dentry, cpg->bdst);
+	h_parent = h_dst->d_parent; /* dir inode is locked */
+	h_dir = d_inode(h_parent);
+	IMustLock(h_dir);
+
+	h_src = au_h_dptr(cpg->dentry, cpg->bsrc);
+	inode = d_inode(cpg->dentry);
+
+	if (!dst_parent)
+		dst_parent = dget_parent(cpg->dentry);
+	else
+		dget(dst_parent);
+
+	plink = !!au_opt_test(au_mntflags(sb), PLINK);
+	dst_inode = au_h_iptr(inode, cpg->bdst);
+	if (dst_inode) {
+		if (unlikely(!plink)) {
+			err = -EIO;
+			AuIOErr("hi%lu(i%lu) exists on b%d "
+				"but plink is disabled\n",
+				dst_inode->i_ino, inode->i_ino, cpg->bdst);
+			goto out_parent;
+		}
+
+		if (dst_inode->i_nlink) {
+			const int do_dt = au_ftest_cpup(cpg->flags, DTIME);
+
+			h_src = au_plink_lkup(inode, cpg->bdst);
+			err = PTR_ERR(h_src);
+			if (IS_ERR(h_src))
+				goto out_parent;
+			if (unlikely(d_is_negative(h_src))) {
+				err = -EIO;
+				AuIOErr("i%lu exists on b%d "
+					"but not pseudo-linked\n",
+					inode->i_ino, cpg->bdst);
+				dput(h_src);
+				goto out_parent;
+			}
+
+			if (do_dt) {
+				a->h_path.dentry = h_parent;
+				au_dtime_store(&a->dt, dst_parent, &a->h_path);
+			}
+
+			a->h_path.dentry = h_dst;
+			delegated = NULL;
+			err = vfsub_link(h_src, h_dir, &a->h_path, &delegated);
+			if (!err && au_ftest_cpup(cpg->flags, RENAME))
+				err = au_do_ren_after_cpup(cpg, &a->h_path);
+			if (do_dt)
+				au_dtime_revert(&a->dt);
+			if (unlikely(err == -EWOULDBLOCK)) {
+				pr_warn("cannot retry for NFSv4 delegation"
+					" for an internal link\n");
+				iput(delegated);
+			}
+			dput(h_src);
+			goto out_parent;
+		} else
+			/* todo: cpup_wh_file? */
+			/* udba work */
+			au_update_ibrange(inode, /*do_put_zero*/1);
+	}
+
+	isdir = S_ISDIR(inode->i_mode);
+	old_ibtop = au_ibtop(inode);
+	err = cpup_entry(cpg, dst_parent, &a->h_src_attr);
+	if (unlikely(err))
+		goto out_rev;
+	dst_inode = d_inode(h_dst);
+	inode_lock_nested(dst_inode, AuLsc_I_CHILD2);
+	/* todo: necessary? */
+	/* au_pin_hdir_unlock(cpg->pin); */
+
+	err = cpup_iattr(cpg->dentry, cpg->bdst, h_src, &a->h_src_attr);
+	if (unlikely(err)) {
+		/* todo: necessary? */
+		/* au_pin_hdir_relock(cpg->pin); */ /* ignore an error */
+		inode_unlock(dst_inode);
+		goto out_rev;
+	}
+
+	if (cpg->bdst < old_ibtop) {
+		if (S_ISREG(inode->i_mode)) {
+			err = au_dy_iaop(inode, cpg->bdst, dst_inode);
+			if (unlikely(err)) {
+				/* ignore an error */
+				/* au_pin_hdir_relock(cpg->pin); */
+				inode_unlock(dst_inode);
+				goto out_rev;
+			}
+		}
+		au_set_ibtop(inode, cpg->bdst);
+	} else
+		au_set_ibbot(inode, cpg->bdst);
+	au_set_h_iptr(inode, cpg->bdst, au_igrab(dst_inode),
+		      au_hi_flags(inode, isdir));
+
+	/* todo: necessary? */
+	/* err = au_pin_hdir_relock(cpg->pin); */
+	inode_unlock(dst_inode);
+	if (unlikely(err))
+		goto out_rev;
+
+	src_inode = d_inode(h_src);
+	if (!isdir
+	    && (src_inode->i_nlink > 1
+		|| src_inode->i_state & I_LINKABLE)
+	    && plink)
+		au_plink_append(inode, cpg->bdst, h_dst);
+
+	if (au_ftest_cpup(cpg->flags, RENAME)) {
+		a->h_path.dentry = h_dst;
+		err = au_do_ren_after_cpup(cpg, &a->h_path);
+	}
+	if (!err)
+		goto out_parent; /* success */
+
+	/* revert */
+out_rev:
+	a->h_path.dentry = h_parent;
+	au_dtime_store(&a->dt, dst_parent, &a->h_path);
+	a->h_path.dentry = h_dst;
+	rerr = 0;
+	if (d_is_positive(h_dst)) {
+		if (!isdir) {
+			/* no delegation since it is just created */
+			rerr = vfsub_unlink(h_dir, &a->h_path,
+					    /*delegated*/NULL, /*force*/0);
+		} else
+			rerr = vfsub_rmdir(h_dir, &a->h_path);
+	}
+	au_dtime_revert(&a->dt);
+	if (rerr) {
+		AuIOErr("failed removing broken entry(%d, %d)\n", err, rerr);
+		err = -EIO;
+	}
+out_parent:
+	dput(dst_parent);
+	au_kfree_rcu(a);
+out:
+	return err;
+}
+
+#if 0 /* reserved */
+struct au_cpup_single_args {
+	int *errp;
+	struct au_cp_generic *cpg;
+	struct dentry *dst_parent;
+};
+
+static void au_call_cpup_single(void *args)
+{
+	struct au_cpup_single_args *a = args;
+
+	au_pin_hdir_acquire_nest(a->cpg->pin);
+	*a->errp = au_cpup_single(a->cpg, a->dst_parent);
+	au_pin_hdir_release(a->cpg->pin);
+}
+#endif
+
+/*
+ * prevent SIGXFSZ in copy-up.
+ * testing CAP_MKNOD is for generic fs,
+ * but CAP_FSETID is for xfs only, currently.
+ */
+static int au_cpup_sio_test(struct au_pin *pin, umode_t mode)
+{
+	int do_sio;
+	struct super_block *sb;
+	struct inode *h_dir;
+
+	do_sio = 0;
+	sb = au_pinned_parent(pin)->d_sb;
+	if (!au_wkq_test()
+	    && (!au_sbi(sb)->si_plink_maint_pid
+		|| au_plink_maint(sb, AuLock_NOPLM))) {
+		switch (mode & S_IFMT) {
+		case S_IFREG:
+			/* no condition about RLIMIT_FSIZE and the file size */
+			do_sio = 1;
+			break;
+		case S_IFCHR:
+		case S_IFBLK:
+			do_sio = !capable(CAP_MKNOD);
+			break;
+		}
+		if (!do_sio)
+			do_sio = ((mode & (S_ISUID | S_ISGID))
+				  && !capable(CAP_FSETID));
+		/* this workaround may be removed in the future */
+		if (!do_sio) {
+			h_dir = au_pinned_h_dir(pin);
+			do_sio = h_dir->i_mode & S_ISVTX;
+		}
+	}
+
+	return do_sio;
+}
+
+#if 0 /* reserved */
+int au_sio_cpup_single(struct au_cp_generic *cpg, struct dentry *dst_parent)
+{
+	int err, wkq_err;
+	struct dentry *h_dentry;
+
+	h_dentry = au_h_dptr(cpg->dentry, cpg->bsrc);
+	if (!au_cpup_sio_test(pin, d_inode(h_dentry)->i_mode))
+		err = au_cpup_single(cpg, dst_parent);
+	else {
+		struct au_cpup_single_args args = {
+			.errp		= &err,
+			.cpg		= cpg,
+			.dst_parent	= dst_parent
+		};
+		wkq_err = au_wkq_wait(au_call_cpup_single, &args);
+		if (unlikely(wkq_err))
+			err = wkq_err;
+	}
+
+	return err;
+}
+#endif
+
+/*
+ * copyup the @dentry from the first active lower branch to @bdst,
+ * using au_cpup_single().
+ */
+static int au_cpup_simple(struct au_cp_generic *cpg)
+{
+	int err;
+	unsigned int flags_orig;
+	struct dentry *dentry;
+
+	AuDebugOn(cpg->bsrc < 0);
+
+	dentry = cpg->dentry;
+	DiMustWriteLock(dentry);
+
+	err = au_lkup_neg(dentry, cpg->bdst, /*wh*/1);
+	if (!err) {
+		flags_orig = cpg->flags;
+		au_fset_cpup(cpg->flags, RENAME);
+		err = au_cpup_single(cpg, NULL);
+		cpg->flags = flags_orig;
+		if (!err)
+			return 0; /* success */
+
+		/* revert */
+		au_set_h_dptr(dentry, cpg->bdst, NULL);
+		au_set_dbtop(dentry, cpg->bsrc);
+	}
+
+	return err;
+}
+
+struct au_cpup_simple_args {
+	int *errp;
+	struct au_cp_generic *cpg;
+};
+
+static void au_call_cpup_simple(void *args)
+{
+	struct au_cpup_simple_args *a = args;
+
+	au_pin_hdir_acquire_nest(a->cpg->pin);
+	*a->errp = au_cpup_simple(a->cpg);
+	au_pin_hdir_release(a->cpg->pin);
+}
+
+static int au_do_sio_cpup_simple(struct au_cp_generic *cpg)
+{
+	int err, wkq_err;
+	struct dentry *dentry, *parent;
+	struct file *h_file;
+	struct inode *h_dir;
+
+	dentry = cpg->dentry;
+	h_file = NULL;
+	if (au_ftest_cpup(cpg->flags, HOPEN)) {
+		AuDebugOn(cpg->bsrc < 0);
+		h_file = au_h_open_pre(dentry, cpg->bsrc, /*force_wr*/0);
+		err = PTR_ERR(h_file);
+		if (IS_ERR(h_file))
+			goto out;
+	}
+
+	parent = dget_parent(dentry);
+	h_dir = au_h_iptr(d_inode(parent), cpg->bdst);
+	if (!au_test_h_perm_sio(h_dir, MAY_EXEC | MAY_WRITE)
+	    && !au_cpup_sio_test(cpg->pin, d_inode(dentry)->i_mode))
+		err = au_cpup_simple(cpg);
+	else {
+		struct au_cpup_simple_args args = {
+			.errp		= &err,
+			.cpg		= cpg
+		};
+		wkq_err = au_wkq_wait(au_call_cpup_simple, &args);
+		if (unlikely(wkq_err))
+			err = wkq_err;
+	}
+
+	dput(parent);
+	if (h_file)
+		au_h_open_post(dentry, cpg->bsrc, h_file);
+
+out:
+	return err;
+}
+
+int au_sio_cpup_simple(struct au_cp_generic *cpg)
+{
+	aufs_bindex_t bsrc, bbot;
+	struct dentry *dentry, *h_dentry;
+
+	if (cpg->bsrc < 0) {
+		dentry = cpg->dentry;
+		bbot = au_dbbot(dentry);
+		for (bsrc = cpg->bdst + 1; bsrc <= bbot; bsrc++) {
+			h_dentry = au_h_dptr(dentry, bsrc);
+			if (h_dentry) {
+				AuDebugOn(d_is_negative(h_dentry));
+				break;
+			}
+		}
+		AuDebugOn(bsrc > bbot);
+		cpg->bsrc = bsrc;
+	}
+	AuDebugOn(cpg->bsrc <= cpg->bdst);
+	return au_do_sio_cpup_simple(cpg);
+}
+
+int au_sio_cpdown_simple(struct au_cp_generic *cpg)
+{
+	AuDebugOn(cpg->bdst <= cpg->bsrc);
+	return au_do_sio_cpup_simple(cpg);
+}
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * copyup the deleted file for writing.
+ */
+static int au_do_cpup_wh(struct au_cp_generic *cpg, struct dentry *wh_dentry,
+			 struct file *file)
+{
+	int err;
+	unsigned int flags_orig;
+	aufs_bindex_t bsrc_orig;
+	struct au_dinfo *dinfo;
+	struct {
+		struct au_hdentry *hd;
+		struct dentry *h_dentry;
+	} hdst, hsrc;
+
+	dinfo = au_di(cpg->dentry);
+	AuRwMustWriteLock(&dinfo->di_rwsem);
+
+	bsrc_orig = cpg->bsrc;
+	cpg->bsrc = dinfo->di_btop;
+	hdst.hd = au_hdentry(dinfo, cpg->bdst);
+	hdst.h_dentry = hdst.hd->hd_dentry;
+	hdst.hd->hd_dentry = wh_dentry;
+	dinfo->di_btop = cpg->bdst;
+
+	hsrc.h_dentry = NULL;
+	if (file) {
+		hsrc.hd = au_hdentry(dinfo, cpg->bsrc);
+		hsrc.h_dentry = hsrc.hd->hd_dentry;
+		hsrc.hd->hd_dentry = au_hf_top(file)->f_path.dentry;
+	}
+	flags_orig = cpg->flags;
+	cpg->flags = !AuCpup_DTIME;
+	err = au_cpup_single(cpg, /*h_parent*/NULL);
+	cpg->flags = flags_orig;
+	if (file) {
+		if (!err)
+			err = au_reopen_nondir(file);
+		hsrc.hd->hd_dentry = hsrc.h_dentry;
+	}
+	hdst.hd->hd_dentry = hdst.h_dentry;
+	dinfo->di_btop = cpg->bsrc;
+	cpg->bsrc = bsrc_orig;
+
+	return err;
+}
+
+static int au_cpup_wh(struct au_cp_generic *cpg, struct file *file)
+{
+	int err;
+	aufs_bindex_t bdst;
+	struct au_dtime dt;
+	struct dentry *dentry, *parent, *h_parent, *wh_dentry;
+	struct au_branch *br;
+	struct path h_path;
+
+	dentry = cpg->dentry;
+	bdst = cpg->bdst;
+	br = au_sbr(dentry->d_sb, bdst);
+	parent = dget_parent(dentry);
+	h_parent = au_h_dptr(parent, bdst);
+	wh_dentry = au_whtmp_lkup(h_parent, br, &dentry->d_name);
+	err = PTR_ERR(wh_dentry);
+	if (IS_ERR(wh_dentry))
+		goto out;
+
+	h_path.dentry = h_parent;
+	h_path.mnt = au_br_mnt(br);
+	au_dtime_store(&dt, parent, &h_path);
+	err = au_do_cpup_wh(cpg, wh_dentry, file);
+	if (unlikely(err))
+		goto out_wh;
+
+	dget(wh_dentry);
+	h_path.dentry = wh_dentry;
+	if (!d_is_dir(wh_dentry)) {
+		/* no delegation since it is just created */
+		err = vfsub_unlink(d_inode(h_parent), &h_path,
+				   /*delegated*/NULL, /*force*/0);
+	} else
+		err = vfsub_rmdir(d_inode(h_parent), &h_path);
+	if (unlikely(err)) {
+		AuIOErr("failed remove copied-up tmp file %pd(%d)\n",
+			wh_dentry, err);
+		err = -EIO;
+	}
+	au_dtime_revert(&dt);
+	au_set_hi_wh(d_inode(dentry), bdst, wh_dentry);
+
+out_wh:
+	dput(wh_dentry);
+out:
+	dput(parent);
+	return err;
+}
+
+struct au_cpup_wh_args {
+	int *errp;
+	struct au_cp_generic *cpg;
+	struct file *file;
+};
+
+static void au_call_cpup_wh(void *args)
+{
+	struct au_cpup_wh_args *a = args;
+
+	au_pin_hdir_acquire_nest(a->cpg->pin);
+	*a->errp = au_cpup_wh(a->cpg, a->file);
+	au_pin_hdir_release(a->cpg->pin);
+}
+
+int au_sio_cpup_wh(struct au_cp_generic *cpg, struct file *file)
+{
+	int err, wkq_err;
+	aufs_bindex_t bdst;
+	struct dentry *dentry, *parent, *h_orph, *h_parent;
+	struct inode *dir, *h_dir, *h_tmpdir;
+	struct au_wbr *wbr;
+	struct au_pin wh_pin, *pin_orig;
+
+	dentry = cpg->dentry;
+	bdst = cpg->bdst;
+	parent = dget_parent(dentry);
+	dir = d_inode(parent);
+	h_orph = NULL;
+	h_parent = NULL;
+	h_dir = au_igrab(au_h_iptr(dir, bdst));
+	h_tmpdir = h_dir;
+	pin_orig = NULL;
+	if (!h_dir->i_nlink) {
+		wbr = au_sbr(dentry->d_sb, bdst)->br_wbr;
+		h_orph = wbr->wbr_orph;
+
+		h_parent = dget(au_h_dptr(parent, bdst));
+		au_set_h_dptr(parent, bdst, dget(h_orph));
+		h_tmpdir = d_inode(h_orph);
+		au_set_h_iptr(dir, bdst, au_igrab(h_tmpdir), /*flags*/0);
+
+		inode_lock_nested(h_tmpdir, AuLsc_I_PARENT3);
+		/* todo: au_h_open_pre()? */
+
+		pin_orig = cpg->pin;
+		au_pin_init(&wh_pin, dentry, bdst, AuLsc_DI_PARENT,
+			    AuLsc_I_PARENT3, cpg->pin->udba, AuPin_DI_LOCKED);
+		cpg->pin = &wh_pin;
+	}
+
+	if (!au_test_h_perm_sio(h_tmpdir, MAY_EXEC | MAY_WRITE)
+	    && !au_cpup_sio_test(cpg->pin, d_inode(dentry)->i_mode))
+		err = au_cpup_wh(cpg, file);
+	else {
+		struct au_cpup_wh_args args = {
+			.errp	= &err,
+			.cpg	= cpg,
+			.file	= file
+		};
+		wkq_err = au_wkq_wait(au_call_cpup_wh, &args);
+		if (unlikely(wkq_err))
+			err = wkq_err;
+	}
+
+	if (h_orph) {
+		inode_unlock(h_tmpdir);
+		/* todo: au_h_open_post()? */
+		au_set_h_iptr(dir, bdst, au_igrab(h_dir), /*flags*/0);
+		au_set_h_dptr(parent, bdst, h_parent);
+		AuDebugOn(!pin_orig);
+		cpg->pin = pin_orig;
+	}
+	iput(h_dir);
+	dput(parent);
+
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * generic routine for both of copy-up and copy-down.
+ */
+/* cf. revalidate function in file.c */
+int au_cp_dirs(struct dentry *dentry, aufs_bindex_t bdst,
+	       int (*cp)(struct dentry *dentry, aufs_bindex_t bdst,
+			 struct au_pin *pin,
+			 struct dentry *h_parent, void *arg),
+	       void *arg)
+{
+	int err;
+	struct au_pin pin;
+	struct dentry *d, *parent, *h_parent, *real_parent, *h_dentry;
+
+	err = 0;
+	parent = dget_parent(dentry);
+	if (IS_ROOT(parent))
+		goto out;
+
+	au_pin_init(&pin, dentry, bdst, AuLsc_DI_PARENT2, AuLsc_I_PARENT2,
+		    au_opt_udba(dentry->d_sb), AuPin_MNT_WRITE);
+
+	/* do not use au_dpage */
+	real_parent = parent;
+	while (1) {
+		dput(parent);
+		parent = dget_parent(dentry);
+		h_parent = au_h_dptr(parent, bdst);
+		if (h_parent)
+			goto out; /* success */
+
+		/* find top dir which is necessary to cpup */
+		do {
+			d = parent;
+			dput(parent);
+			parent = dget_parent(d);
+			di_read_lock_parent3(parent, !AuLock_IR);
+			h_parent = au_h_dptr(parent, bdst);
+			di_read_unlock(parent, !AuLock_IR);
+		} while (!h_parent);
+
+		if (d != real_parent)
+			di_write_lock_child3(d);
+
+		/* somebody else might create while we were sleeping */
+		h_dentry = au_h_dptr(d, bdst);
+		if (!h_dentry || d_is_negative(h_dentry)) {
+			if (h_dentry)
+				au_update_dbtop(d);
+
+			au_pin_set_dentry(&pin, d);
+			err = au_do_pin(&pin);
+			if (!err) {
+				err = cp(d, bdst, &pin, h_parent, arg);
+				au_unpin(&pin);
+			}
+		}
+
+		if (d != real_parent)
+			di_write_unlock(d);
+		if (unlikely(err))
+			break;
+	}
+
+out:
+	dput(parent);
+	return err;
+}
+
+static int au_cpup_dir(struct dentry *dentry, aufs_bindex_t bdst,
+		       struct au_pin *pin,
+		       struct dentry *h_parent __maybe_unused,
+		       void *arg __maybe_unused)
+{
+	struct au_cp_generic cpg = {
+		.dentry	= dentry,
+		.bdst	= bdst,
+		.bsrc	= -1,
+		.len	= 0,
+		.pin	= pin,
+		.flags	= AuCpup_DTIME
+	};
+	return au_sio_cpup_simple(&cpg);
+}
+
+int au_cpup_dirs(struct dentry *dentry, aufs_bindex_t bdst)
+{
+	return au_cp_dirs(dentry, bdst, au_cpup_dir, NULL);
+}
+
+int au_test_and_cpup_dirs(struct dentry *dentry, aufs_bindex_t bdst)
+{
+	int err;
+	struct dentry *parent;
+	struct inode *dir;
+
+	parent = dget_parent(dentry);
+	dir = d_inode(parent);
+	err = 0;
+	if (au_h_iptr(dir, bdst))
+		goto out;
+
+	di_read_unlock(parent, AuLock_IR);
+	di_write_lock_parent(parent);
+	/* someone else might change our inode while we were sleeping */
+	if (!au_h_iptr(dir, bdst))
+		err = au_cpup_dirs(dentry, bdst);
+	di_downgrade_lock(parent, AuLock_IR);
+
+out:
+	dput(parent);
+	return err;
+}
diff -Naur --no-dereference a/fs/aufs/cpup.h b/fs/aufs/cpup.h
--- a/fs/aufs/cpup.h	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/cpup.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,100 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * copy-up/down functions
+ */
+
+#ifndef __AUFS_CPUP_H__
+#define __AUFS_CPUP_H__
+
+#ifdef __KERNEL__
+
+#include <linux/path.h>
+
+struct inode;
+struct file;
+struct au_pin;
+
+void au_cpup_attr_flags(struct inode *dst, unsigned int iflags);
+void au_cpup_attr_timesizes(struct inode *inode);
+void au_cpup_attr_nlink(struct inode *inode, int force);
+void au_cpup_attr_changeable(struct inode *inode);
+void au_cpup_igen(struct inode *inode, struct inode *h_inode);
+void au_cpup_attr_all(struct inode *inode, int force);
+
+/* ---------------------------------------------------------------------- */
+
+struct au_cp_generic {
+	struct dentry	*dentry;
+	aufs_bindex_t	bdst, bsrc;
+	loff_t		len;
+	struct au_pin	*pin;
+	unsigned int	flags;
+};
+
+/* cpup flags */
+#define AuCpup_DTIME		1		/* do dtime_store/revert */
+#define AuCpup_KEEPLINO		(1 << 1)	/* do not clear the lower xino,
+						   for link(2) */
+#define AuCpup_RENAME		(1 << 2)	/* rename after cpup */
+#define AuCpup_HOPEN		(1 << 3)	/* call h_open_pre/post() in
+						   cpup */
+#define AuCpup_OVERWRITE	(1 << 4)	/* allow overwriting the
+						   existing entry */
+#define AuCpup_RWDST		(1 << 5)	/* force write target even if
+						   the branch is marked as RO */
+
+#ifndef CONFIG_AUFS_BR_HFSPLUS
+#undef AuCpup_HOPEN
+#define AuCpup_HOPEN		0
+#endif
+
+#define au_ftest_cpup(flags, name)	((flags) & AuCpup_##name)
+#define au_fset_cpup(flags, name) \
+	do { (flags) |= AuCpup_##name; } while (0)
+#define au_fclr_cpup(flags, name) \
+	do { (flags) &= ~AuCpup_##name; } while (0)
+
+int au_copy_file(struct file *dst, struct file *src, loff_t len);
+int au_sio_cpup_simple(struct au_cp_generic *cpg);
+int au_sio_cpdown_simple(struct au_cp_generic *cpg);
+int au_sio_cpup_wh(struct au_cp_generic *cpg, struct file *file);
+
+int au_cp_dirs(struct dentry *dentry, aufs_bindex_t bdst,
+	       int (*cp)(struct dentry *dentry, aufs_bindex_t bdst,
+			 struct au_pin *pin,
+			 struct dentry *h_parent, void *arg),
+	       void *arg);
+int au_cpup_dirs(struct dentry *dentry, aufs_bindex_t bdst);
+int au_test_and_cpup_dirs(struct dentry *dentry, aufs_bindex_t bdst);
+
+/* ---------------------------------------------------------------------- */
+
+/* keep timestamps when copyup */
+struct au_dtime {
+	struct dentry *dt_dentry;
+	struct path dt_h_path;
+	struct timespec64 dt_atime, dt_mtime;
+};
+void au_dtime_store(struct au_dtime *dt, struct dentry *dentry,
+		    struct path *h_path);
+void au_dtime_revert(struct au_dtime *dt);
+
+#endif /* __KERNEL__ */
+#endif /* __AUFS_CPUP_H__ */
diff -Naur --no-dereference a/fs/aufs/dbgaufs.c b/fs/aufs/dbgaufs.c
--- a/fs/aufs/dbgaufs.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/dbgaufs.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,526 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * debugfs interface
+ */
+
+#include <linux/debugfs.h>
+#include "aufs.h"
+
+#ifndef CONFIG_SYSFS
+#error DEBUG_FS depends upon SYSFS
+#endif
+
+static struct dentry *dbgaufs;
+static const mode_t dbgaufs_mode = 0444;
+
+/* 20 is max digits length of ulong 64 */
+struct dbgaufs_arg {
+	int n;
+	char a[20 * 4];
+};
+
+/*
+ * common function for all XINO files
+ */
+static int dbgaufs_xi_release(struct inode *inode __maybe_unused,
+			      struct file *file)
+{
+	void *p;
+
+	p = file->private_data;
+	if (p) {
+		/* this is struct dbgaufs_arg */
+		AuDebugOn(!au_kfree_sz_test(p));
+		au_kfree_do_rcu(p);
+	}
+	return 0;
+}
+
+static int dbgaufs_xi_open(struct file *xf, struct file *file, int do_fcnt,
+			   int cnt)
+{
+	int err;
+	struct kstat st;
+	struct dbgaufs_arg *p;
+
+	err = -ENOMEM;
+	p = kmalloc(sizeof(*p), GFP_NOFS);
+	if (unlikely(!p))
+		goto out;
+
+	err = 0;
+	p->n = 0;
+	file->private_data = p;
+	if (!xf)
+		goto out;
+
+	err = vfsub_getattr(&xf->f_path, &st);
+	if (!err) {
+		if (do_fcnt)
+			p->n = snprintf
+				(p->a, sizeof(p->a), "%d, %llux%u %lld\n",
+				 cnt, st.blocks, st.blksize,
+				 (long long)st.size);
+		else
+			p->n = snprintf(p->a, sizeof(p->a), "%llux%u %lld\n",
+					st.blocks, st.blksize,
+					(long long)st.size);
+		AuDebugOn(p->n >= sizeof(p->a));
+	} else {
+		p->n = snprintf(p->a, sizeof(p->a), "err %d\n", err);
+		err = 0;
+	}
+
+out:
+	return err;
+}
+
+static ssize_t dbgaufs_xi_read(struct file *file, char __user *buf,
+			       size_t count, loff_t *ppos)
+{
+	struct dbgaufs_arg *p;
+
+	p = file->private_data;
+	return simple_read_from_buffer(buf, count, ppos, p->a, p->n);
+}
+
+/* ---------------------------------------------------------------------- */
+
+struct dbgaufs_plink_arg {
+	int n;
+	char a[];
+};
+
+static int dbgaufs_plink_release(struct inode *inode __maybe_unused,
+				 struct file *file)
+{
+	free_page((unsigned long)file->private_data);
+	return 0;
+}
+
+static int dbgaufs_plink_open(struct inode *inode, struct file *file)
+{
+	int err, i, limit;
+	unsigned long n, sum;
+	struct dbgaufs_plink_arg *p;
+	struct au_sbinfo *sbinfo;
+	struct super_block *sb;
+	struct hlist_bl_head *hbl;
+
+	err = -ENOMEM;
+	p = (void *)get_zeroed_page(GFP_NOFS);
+	if (unlikely(!p))
+		goto out;
+
+	err = -EFBIG;
+	sbinfo = inode->i_private;
+	sb = sbinfo->si_sb;
+	si_noflush_read_lock(sb);
+	if (au_opt_test(au_mntflags(sb), PLINK)) {
+		limit = PAGE_SIZE - sizeof(p->n);
+
+		/* the number of buckets */
+		n = snprintf(p->a + p->n, limit, "%d\n", AuPlink_NHASH);
+		p->n += n;
+		limit -= n;
+
+		sum = 0;
+		for (i = 0, hbl = sbinfo->si_plink; i < AuPlink_NHASH;
+		     i++, hbl++) {
+			n = au_hbl_count(hbl);
+			sum += n;
+
+			n = snprintf(p->a + p->n, limit, "%lu ", n);
+			p->n += n;
+			limit -= n;
+			if (unlikely(limit <= 0))
+				goto out_free;
+		}
+		p->a[p->n - 1] = '\n';
+
+		/* the sum of plinks */
+		n = snprintf(p->a + p->n, limit, "%lu\n", sum);
+		p->n += n;
+		limit -= n;
+		if (unlikely(limit <= 0))
+			goto out_free;
+	} else {
+#define str "1\n0\n0\n"
+		p->n = sizeof(str) - 1;
+		strcpy(p->a, str);
+#undef str
+	}
+	si_read_unlock(sb);
+
+	err = 0;
+	file->private_data = p;
+	goto out; /* success */
+
+out_free:
+	free_page((unsigned long)p);
+out:
+	return err;
+}
+
+static ssize_t dbgaufs_plink_read(struct file *file, char __user *buf,
+				  size_t count, loff_t *ppos)
+{
+	struct dbgaufs_plink_arg *p;
+
+	p = file->private_data;
+	return simple_read_from_buffer(buf, count, ppos, p->a, p->n);
+}
+
+static const struct file_operations dbgaufs_plink_fop = {
+	.owner		= THIS_MODULE,
+	.open		= dbgaufs_plink_open,
+	.release	= dbgaufs_plink_release,
+	.read		= dbgaufs_plink_read
+};
+
+/* ---------------------------------------------------------------------- */
+
+static int dbgaufs_xib_open(struct inode *inode, struct file *file)
+{
+	int err;
+	struct au_sbinfo *sbinfo;
+	struct super_block *sb;
+
+	sbinfo = inode->i_private;
+	sb = sbinfo->si_sb;
+	si_noflush_read_lock(sb);
+	err = dbgaufs_xi_open(sbinfo->si_xib, file, /*do_fcnt*/0, /*cnt*/0);
+	si_read_unlock(sb);
+	return err;
+}
+
+static const struct file_operations dbgaufs_xib_fop = {
+	.owner		= THIS_MODULE,
+	.open		= dbgaufs_xib_open,
+	.release	= dbgaufs_xi_release,
+	.read		= dbgaufs_xi_read
+};
+
+/* ---------------------------------------------------------------------- */
+
+#define DbgaufsXi_PREFIX "xi"
+
+static int dbgaufs_xino_open(struct inode *inode, struct file *file)
+{
+	int err, idx;
+	long l;
+	aufs_bindex_t bindex;
+	char *p, a[sizeof(DbgaufsXi_PREFIX) + 8];
+	struct au_sbinfo *sbinfo;
+	struct super_block *sb;
+	struct au_xino *xi;
+	struct file *xf;
+	struct qstr *name;
+	struct au_branch *br;
+
+	err = -ENOENT;
+	name = &file->f_path.dentry->d_name;
+	if (unlikely(name->len < sizeof(DbgaufsXi_PREFIX)
+		     || memcmp(name->name, DbgaufsXi_PREFIX,
+			       sizeof(DbgaufsXi_PREFIX) - 1)))
+		goto out;
+
+	AuDebugOn(name->len >= sizeof(a));
+	memcpy(a, name->name, name->len);
+	a[name->len] = '\0';
+	p = strchr(a, '-');
+	if (p)
+		*p = '\0';
+	err = kstrtol(a + sizeof(DbgaufsXi_PREFIX) - 1, 10, &l);
+	if (unlikely(err))
+		goto out;
+	bindex = l;
+	idx = 0;
+	if (p) {
+		err = kstrtol(p + 1, 10, &l);
+		if (unlikely(err))
+			goto out;
+		idx = l;
+	}
+
+	err = -ENOENT;
+	sbinfo = inode->i_private;
+	sb = sbinfo->si_sb;
+	si_noflush_read_lock(sb);
+	if (unlikely(bindex < 0 || bindex > au_sbbot(sb)))
+		goto out_si;
+	br = au_sbr(sb, bindex);
+	xi = br->br_xino;
+	if (unlikely(idx >= xi->xi_nfile))
+		goto out_si;
+	xf = au_xino_file(xi, idx);
+	if (xf)
+		err = dbgaufs_xi_open(xf, file, /*do_fcnt*/1,
+				      au_xino_count(br));
+
+out_si:
+	si_read_unlock(sb);
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+static const struct file_operations dbgaufs_xino_fop = {
+	.owner		= THIS_MODULE,
+	.open		= dbgaufs_xino_open,
+	.release	= dbgaufs_xi_release,
+	.read		= dbgaufs_xi_read
+};
+
+void dbgaufs_xino_del(struct au_branch *br)
+{
+	struct dentry *dbgaufs;
+
+	dbgaufs = br->br_dbgaufs;
+	if (!dbgaufs)
+		return;
+
+	br->br_dbgaufs = NULL;
+	/* debugfs acquires the parent i_mutex */
+	lockdep_off();
+	debugfs_remove(dbgaufs);
+	lockdep_on();
+}
+
+void dbgaufs_brs_del(struct super_block *sb, aufs_bindex_t bindex)
+{
+	aufs_bindex_t bbot;
+	struct au_branch *br;
+
+	if (!au_sbi(sb)->si_dbgaufs)
+		return;
+
+	bbot = au_sbbot(sb);
+	for (; bindex <= bbot; bindex++) {
+		br = au_sbr(sb, bindex);
+		dbgaufs_xino_del(br);
+	}
+}
+
+static void dbgaufs_br_do_add(struct super_block *sb, aufs_bindex_t bindex,
+			      unsigned int idx, struct dentry *parent,
+			      struct au_sbinfo *sbinfo)
+{
+	struct au_branch *br;
+	struct dentry *d;
+	/* "xi" bindex(5) "-" idx(2) NULL */
+	char name[sizeof(DbgaufsXi_PREFIX) + 8];
+
+	if (!idx)
+		snprintf(name, sizeof(name), DbgaufsXi_PREFIX "%d", bindex);
+	else
+		snprintf(name, sizeof(name), DbgaufsXi_PREFIX "%d-%u",
+			 bindex, idx);
+	br = au_sbr(sb, bindex);
+	if (br->br_dbgaufs) {
+		struct qstr qstr = QSTR_INIT(name, strlen(name));
+
+		if (!au_qstreq(&br->br_dbgaufs->d_name, &qstr)) {
+			/* debugfs acquires the parent i_mutex */
+			lockdep_off();
+			d = debugfs_rename(parent, br->br_dbgaufs, parent,
+					   name);
+			lockdep_on();
+			if (unlikely(!d))
+				pr_warn("failed renaming %pd/%s, ignored.\n",
+					parent, name);
+		}
+	} else {
+		lockdep_off();
+		br->br_dbgaufs = debugfs_create_file(name, dbgaufs_mode, parent,
+						     sbinfo, &dbgaufs_xino_fop);
+		lockdep_on();
+		if (unlikely(!br->br_dbgaufs))
+			pr_warn("failed creating %pd/%s, ignored.\n",
+				parent, name);
+	}
+}
+
+static void dbgaufs_br_add(struct super_block *sb, aufs_bindex_t bindex,
+			   struct dentry *parent, struct au_sbinfo *sbinfo)
+{
+	struct au_branch *br;
+	struct au_xino *xi;
+	unsigned int u;
+
+	br = au_sbr(sb, bindex);
+	xi = br->br_xino;
+	for (u = 0; u < xi->xi_nfile; u++)
+		dbgaufs_br_do_add(sb, bindex, u, parent, sbinfo);
+}
+
+void dbgaufs_brs_add(struct super_block *sb, aufs_bindex_t bindex, int topdown)
+{
+	struct au_sbinfo *sbinfo;
+	struct dentry *parent;
+	aufs_bindex_t bbot;
+
+	if (!au_opt_test(au_mntflags(sb), XINO))
+		return;
+
+	sbinfo = au_sbi(sb);
+	parent = sbinfo->si_dbgaufs;
+	if (!parent)
+		return;
+
+	bbot = au_sbbot(sb);
+	if (topdown)
+		for (; bindex <= bbot; bindex++)
+			dbgaufs_br_add(sb, bindex, parent, sbinfo);
+	else
+		for (; bbot >= bindex; bbot--)
+			dbgaufs_br_add(sb, bbot, parent, sbinfo);
+}
+
+/* ---------------------------------------------------------------------- */
+
+#ifdef CONFIG_AUFS_EXPORT
+static int dbgaufs_xigen_open(struct inode *inode, struct file *file)
+{
+	int err;
+	struct au_sbinfo *sbinfo;
+	struct super_block *sb;
+
+	sbinfo = inode->i_private;
+	sb = sbinfo->si_sb;
+	si_noflush_read_lock(sb);
+	err = dbgaufs_xi_open(sbinfo->si_xigen, file, /*do_fcnt*/0, /*cnt*/0);
+	si_read_unlock(sb);
+	return err;
+}
+
+static const struct file_operations dbgaufs_xigen_fop = {
+	.owner		= THIS_MODULE,
+	.open		= dbgaufs_xigen_open,
+	.release	= dbgaufs_xi_release,
+	.read		= dbgaufs_xi_read
+};
+
+static int dbgaufs_xigen_init(struct au_sbinfo *sbinfo)
+{
+	int err;
+
+	/*
+	 * This function is a dynamic '__init' function actually,
+	 * so the tiny check for si_rwsem is unnecessary.
+	 */
+	/* AuRwMustWriteLock(&sbinfo->si_rwsem); */
+
+	err = -EIO;
+	sbinfo->si_dbgaufs_xigen = debugfs_create_file
+		("xigen", dbgaufs_mode, sbinfo->si_dbgaufs, sbinfo,
+		 &dbgaufs_xigen_fop);
+	if (sbinfo->si_dbgaufs_xigen)
+		err = 0;
+
+	return err;
+}
+#else
+static int dbgaufs_xigen_init(struct au_sbinfo *sbinfo)
+{
+	return 0;
+}
+#endif /* CONFIG_AUFS_EXPORT */
+
+/* ---------------------------------------------------------------------- */
+
+void dbgaufs_si_fin(struct au_sbinfo *sbinfo)
+{
+	/*
+	 * This function is a dynamic '__fin' function actually,
+	 * so the tiny check for si_rwsem is unnecessary.
+	 */
+	/* AuRwMustWriteLock(&sbinfo->si_rwsem); */
+
+	debugfs_remove_recursive(sbinfo->si_dbgaufs);
+	sbinfo->si_dbgaufs = NULL;
+}
+
+int dbgaufs_si_init(struct au_sbinfo *sbinfo)
+{
+	int err;
+	char name[SysaufsSiNameLen];
+
+	/*
+	 * This function is a dynamic '__init' function actually,
+	 * so the tiny check for si_rwsem is unnecessary.
+	 */
+	/* AuRwMustWriteLock(&sbinfo->si_rwsem); */
+
+	err = -ENOENT;
+	if (!dbgaufs) {
+		AuErr1("/debug/aufs is uninitialized\n");
+		goto out;
+	}
+
+	err = -EIO;
+	sysaufs_name(sbinfo, name);
+	sbinfo->si_dbgaufs = debugfs_create_dir(name, dbgaufs);
+	if (unlikely(!sbinfo->si_dbgaufs))
+		goto out;
+
+	/* regardless plink/noplink option */
+	sbinfo->si_dbgaufs_plink = debugfs_create_file
+		("plink", dbgaufs_mode, sbinfo->si_dbgaufs, sbinfo,
+		 &dbgaufs_plink_fop);
+	if (unlikely(!sbinfo->si_dbgaufs_plink))
+		goto out_dir;
+
+	/* regardless xino/noxino option */
+	sbinfo->si_dbgaufs_xib = debugfs_create_file
+		("xib", dbgaufs_mode, sbinfo->si_dbgaufs, sbinfo,
+		 &dbgaufs_xib_fop);
+	if (unlikely(!sbinfo->si_dbgaufs_xib))
+		goto out_dir;
+
+	err = dbgaufs_xigen_init(sbinfo);
+	if (!err)
+		goto out; /* success */
+
+out_dir:
+	dbgaufs_si_fin(sbinfo);
+out:
+	if (unlikely(err))
+		pr_err("debugfs/aufs failed\n");
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+void dbgaufs_fin(void)
+{
+	debugfs_remove(dbgaufs);
+}
+
+int __init dbgaufs_init(void)
+{
+	int err;
+
+	err = -EIO;
+	dbgaufs = debugfs_create_dir(AUFS_NAME, NULL);
+	if (dbgaufs)
+		err = 0;
+	return err;
+}
diff -Naur --no-dereference a/fs/aufs/dbgaufs.h b/fs/aufs/dbgaufs.h
--- a/fs/aufs/dbgaufs.h	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/dbgaufs.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,53 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * debugfs interface
+ */
+
+#ifndef __DBGAUFS_H__
+#define __DBGAUFS_H__
+
+#ifdef __KERNEL__
+
+struct super_block;
+struct au_sbinfo;
+struct au_branch;
+
+#ifdef CONFIG_DEBUG_FS
+/* dbgaufs.c */
+void dbgaufs_xino_del(struct au_branch *br);
+void dbgaufs_brs_del(struct super_block *sb, aufs_bindex_t bindex);
+void dbgaufs_brs_add(struct super_block *sb, aufs_bindex_t bindex, int topdown);
+void dbgaufs_si_fin(struct au_sbinfo *sbinfo);
+int dbgaufs_si_init(struct au_sbinfo *sbinfo);
+void dbgaufs_fin(void);
+int __init dbgaufs_init(void);
+#else
+AuStubVoid(dbgaufs_xino_del, struct au_branch *br)
+AuStubVoid(dbgaufs_brs_del, struct super_block *sb, aufs_bindex_t bindex)
+AuStubVoid(dbgaufs_brs_add, struct super_block *sb, aufs_bindex_t bindex,
+	   int topdown)
+AuStubVoid(dbgaufs_si_fin, struct au_sbinfo *sbinfo)
+AuStubInt0(dbgaufs_si_init, struct au_sbinfo *sbinfo)
+AuStubVoid(dbgaufs_fin, void)
+AuStubInt0(__init dbgaufs_init, void)
+#endif /* CONFIG_DEBUG_FS */
+
+#endif /* __KERNEL__ */
+#endif /* __DBGAUFS_H__ */
diff -Naur --no-dereference a/fs/aufs/dcsub.c b/fs/aufs/dcsub.c
--- a/fs/aufs/dcsub.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/dcsub.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,225 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * sub-routines for dentry cache
+ */
+
+#include "aufs.h"
+
+static void au_dpage_free(struct au_dpage *dpage)
+{
+	int i;
+	struct dentry **p;
+
+	p = dpage->dentries;
+	for (i = 0; i < dpage->ndentry; i++)
+		dput(*p++);
+	free_page((unsigned long)dpage->dentries);
+}
+
+int au_dpages_init(struct au_dcsub_pages *dpages, gfp_t gfp)
+{
+	int err;
+	void *p;
+
+	err = -ENOMEM;
+	dpages->dpages = kmalloc(sizeof(*dpages->dpages), gfp);
+	if (unlikely(!dpages->dpages))
+		goto out;
+
+	p = (void *)__get_free_page(gfp);
+	if (unlikely(!p))
+		goto out_dpages;
+
+	dpages->dpages[0].ndentry = 0;
+	dpages->dpages[0].dentries = p;
+	dpages->ndpage = 1;
+	return 0; /* success */
+
+out_dpages:
+	au_kfree_try_rcu(dpages->dpages);
+out:
+	return err;
+}
+
+void au_dpages_free(struct au_dcsub_pages *dpages)
+{
+	int i;
+	struct au_dpage *p;
+
+	p = dpages->dpages;
+	for (i = 0; i < dpages->ndpage; i++)
+		au_dpage_free(p++);
+	au_kfree_try_rcu(dpages->dpages);
+}
+
+static int au_dpages_append(struct au_dcsub_pages *dpages,
+			    struct dentry *dentry, gfp_t gfp)
+{
+	int err, sz;
+	struct au_dpage *dpage;
+	void *p;
+
+	dpage = dpages->dpages + dpages->ndpage - 1;
+	sz = PAGE_SIZE / sizeof(dentry);
+	if (unlikely(dpage->ndentry >= sz)) {
+		AuLabel(new dpage);
+		err = -ENOMEM;
+		sz = dpages->ndpage * sizeof(*dpages->dpages);
+		p = au_kzrealloc(dpages->dpages, sz,
+				 sz + sizeof(*dpages->dpages), gfp,
+				 /*may_shrink*/0);
+		if (unlikely(!p))
+			goto out;
+
+		dpages->dpages = p;
+		dpage = dpages->dpages + dpages->ndpage;
+		p = (void *)__get_free_page(gfp);
+		if (unlikely(!p))
+			goto out;
+
+		dpage->ndentry = 0;
+		dpage->dentries = p;
+		dpages->ndpage++;
+	}
+
+	AuDebugOn(au_dcount(dentry) <= 0);
+	dpage->dentries[dpage->ndentry++] = dget_dlock(dentry);
+	return 0; /* success */
+
+out:
+	return err;
+}
+
+/* todo: BAD approach */
+/* copied from linux/fs/dcache.c */
+enum d_walk_ret {
+	D_WALK_CONTINUE,
+	D_WALK_QUIT,
+	D_WALK_NORETRY,
+	D_WALK_SKIP,
+};
+
+extern void d_walk(struct dentry *parent, void *data,
+		   enum d_walk_ret (*enter)(void *, struct dentry *));
+
+struct ac_dpages_arg {
+	int err;
+	struct au_dcsub_pages *dpages;
+	struct super_block *sb;
+	au_dpages_test test;
+	void *arg;
+};
+
+static enum d_walk_ret au_call_dpages_append(void *_arg, struct dentry *dentry)
+{
+	enum d_walk_ret ret;
+	struct ac_dpages_arg *arg = _arg;
+
+	ret = D_WALK_CONTINUE;
+	if (dentry->d_sb == arg->sb
+	    && !IS_ROOT(dentry)
+	    && au_dcount(dentry) > 0
+	    && au_di(dentry)
+	    && (!arg->test || arg->test(dentry, arg->arg))) {
+		arg->err = au_dpages_append(arg->dpages, dentry, GFP_ATOMIC);
+		if (unlikely(arg->err))
+			ret = D_WALK_QUIT;
+	}
+
+	return ret;
+}
+
+int au_dcsub_pages(struct au_dcsub_pages *dpages, struct dentry *root,
+		   au_dpages_test test, void *arg)
+{
+	struct ac_dpages_arg args = {
+		.err	= 0,
+		.dpages	= dpages,
+		.sb	= root->d_sb,
+		.test	= test,
+		.arg	= arg
+	};
+
+	d_walk(root, &args, au_call_dpages_append);
+
+	return args.err;
+}
+
+int au_dcsub_pages_rev(struct au_dcsub_pages *dpages, struct dentry *dentry,
+		       int do_include, au_dpages_test test, void *arg)
+{
+	int err;
+
+	err = 0;
+	write_seqlock(&rename_lock);
+	spin_lock(&dentry->d_lock);
+	if (do_include
+	    && au_dcount(dentry) > 0
+	    && (!test || test(dentry, arg)))
+		err = au_dpages_append(dpages, dentry, GFP_ATOMIC);
+	spin_unlock(&dentry->d_lock);
+	if (unlikely(err))
+		goto out;
+
+	/*
+	 * RCU for vfsmount is unnecessary since this is a traverse in a single
+	 * mount
+	 */
+	while (!IS_ROOT(dentry)) {
+		dentry = dentry->d_parent; /* rename_lock is locked */
+		spin_lock(&dentry->d_lock);
+		if (au_dcount(dentry) > 0
+		    && (!test || test(dentry, arg)))
+			err = au_dpages_append(dpages, dentry, GFP_ATOMIC);
+		spin_unlock(&dentry->d_lock);
+		if (unlikely(err))
+			break;
+	}
+
+out:
+	write_sequnlock(&rename_lock);
+	return err;
+}
+
+static inline int au_dcsub_dpages_aufs(struct dentry *dentry, void *arg)
+{
+	return au_di(dentry) && dentry->d_sb == arg;
+}
+
+int au_dcsub_pages_rev_aufs(struct au_dcsub_pages *dpages,
+			    struct dentry *dentry, int do_include)
+{
+	return au_dcsub_pages_rev(dpages, dentry, do_include,
+				  au_dcsub_dpages_aufs, dentry->d_sb);
+}
+
+int au_test_subdir(struct dentry *d1, struct dentry *d2)
+{
+	struct path path[2] = {
+		{
+			.dentry = d1
+		},
+		{
+			.dentry = d2
+		}
+	};
+
+	return path_is_under(path + 0, path + 1);
+}
diff -Naur --no-dereference a/fs/aufs/dcsub.h b/fs/aufs/dcsub.h
--- a/fs/aufs/dcsub.h	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/dcsub.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,137 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * sub-routines for dentry cache
+ */
+
+#ifndef __AUFS_DCSUB_H__
+#define __AUFS_DCSUB_H__
+
+#ifdef __KERNEL__
+
+#include <linux/dcache.h>
+#include <linux/fs.h>
+
+struct au_dpage {
+	int ndentry;
+	struct dentry **dentries;
+};
+
+struct au_dcsub_pages {
+	int ndpage;
+	struct au_dpage *dpages;
+};
+
+/* ---------------------------------------------------------------------- */
+
+/* dcsub.c */
+int au_dpages_init(struct au_dcsub_pages *dpages, gfp_t gfp);
+void au_dpages_free(struct au_dcsub_pages *dpages);
+typedef int (*au_dpages_test)(struct dentry *dentry, void *arg);
+int au_dcsub_pages(struct au_dcsub_pages *dpages, struct dentry *root,
+		   au_dpages_test test, void *arg);
+int au_dcsub_pages_rev(struct au_dcsub_pages *dpages, struct dentry *dentry,
+		       int do_include, au_dpages_test test, void *arg);
+int au_dcsub_pages_rev_aufs(struct au_dcsub_pages *dpages,
+			    struct dentry *dentry, int do_include);
+int au_test_subdir(struct dentry *d1, struct dentry *d2);
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * todo: in linux-3.13, several similar (but faster) helpers are added to
+ * include/linux/dcache.h. Try them (in the future).
+ */
+
+static inline int au_d_hashed_positive(struct dentry *d)
+{
+	int err;
+	struct inode *inode = d_inode(d);
+
+	err = 0;
+	if (unlikely(d_unhashed(d)
+		     || d_is_negative(d)
+		     || !inode->i_nlink))
+		err = -ENOENT;
+	return err;
+}
+
+static inline int au_d_linkable(struct dentry *d)
+{
+	int err;
+	struct inode *inode = d_inode(d);
+
+	err = au_d_hashed_positive(d);
+	if (err
+	    && d_is_positive(d)
+	    && (inode->i_state & I_LINKABLE))
+		err = 0;
+	return err;
+}
+
+static inline int au_d_alive(struct dentry *d)
+{
+	int err;
+	struct inode *inode;
+
+	err = 0;
+	if (!IS_ROOT(d))
+		err = au_d_hashed_positive(d);
+	else {
+		inode = d_inode(d);
+		if (unlikely(d_unlinked(d)
+			     || d_is_negative(d)
+			     || !inode->i_nlink))
+			err = -ENOENT;
+	}
+	return err;
+}
+
+static inline int au_alive_dir(struct dentry *d)
+{
+	int err;
+
+	err = au_d_alive(d);
+	if (unlikely(err || IS_DEADDIR(d_inode(d))))
+		err = -ENOENT;
+	return err;
+}
+
+static inline int au_qstreq(struct qstr *a, struct qstr *b)
+{
+	return a->len == b->len
+		&& !memcmp(a->name, b->name, a->len);
+}
+
+/*
+ * by the commit
+ * 360f547 2015-01-25 dcache: let the dentry count go down to zero without
+ *			taking d_lock
+ * the type of d_lockref.count became int, but the inlined function d_count()
+ * still returns unsigned int.
+ * I don't know why. Maybe it is for every d_count() users?
+ * Anyway au_dcount() lives on.
+ */
+static inline int au_dcount(struct dentry *d)
+{
+	return (int)d_count(d);
+}
+
+#endif /* __KERNEL__ */
+#endif /* __AUFS_DCSUB_H__ */
diff -Naur --no-dereference a/fs/aufs/debug.c b/fs/aufs/debug.c
--- a/fs/aufs/debug.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/debug.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,441 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * debug print functions
+ */
+
+#include <linux/iversion.h>
+#include "aufs.h"
+
+/* Returns 0, or -errno.  arg is in kp->arg. */
+static int param_atomic_t_set(const char *val, const struct kernel_param *kp)
+{
+	int err, n;
+
+	err = kstrtoint(val, 0, &n);
+	if (!err) {
+		if (n > 0)
+			au_debug_on();
+		else
+			au_debug_off();
+	}
+	return err;
+}
+
+/* Returns length written or -errno.  Buffer is 4k (ie. be short!) */
+static int param_atomic_t_get(char *buffer, const struct kernel_param *kp)
+{
+	atomic_t *a;
+
+	a = kp->arg;
+	return sprintf(buffer, "%d", atomic_read(a));
+}
+
+static struct kernel_param_ops param_ops_atomic_t = {
+	.set = param_atomic_t_set,
+	.get = param_atomic_t_get
+	/* void (*free)(void *arg) */
+};
+
+atomic_t aufs_debug = ATOMIC_INIT(0);
+MODULE_PARM_DESC(debug, "debug print");
+module_param_named(debug, aufs_debug, atomic_t, 0664);
+
+DEFINE_MUTEX(au_dbg_mtx);	/* just to serialize the dbg msgs */
+char *au_plevel = KERN_DEBUG;
+#define dpri(fmt, ...) do {					\
+	if ((au_plevel						\
+	     && strcmp(au_plevel, KERN_DEBUG))			\
+	    || au_debug_test())					\
+		printk("%s" fmt, au_plevel, ##__VA_ARGS__);	\
+} while (0)
+
+/* ---------------------------------------------------------------------- */
+
+void au_dpri_whlist(struct au_nhash *whlist)
+{
+	unsigned long ul, n;
+	struct hlist_head *head;
+	struct au_vdir_wh *pos;
+
+	n = whlist->nh_num;
+	head = whlist->nh_head;
+	for (ul = 0; ul < n; ul++) {
+		hlist_for_each_entry(pos, head, wh_hash)
+			dpri("b%d, %.*s, %d\n",
+			     pos->wh_bindex,
+			     pos->wh_str.len, pos->wh_str.name,
+			     pos->wh_str.len);
+		head++;
+	}
+}
+
+void au_dpri_vdir(struct au_vdir *vdir)
+{
+	unsigned long ul;
+	union au_vdir_deblk_p p;
+	unsigned char *o;
+
+	if (!vdir || IS_ERR(vdir)) {
+		dpri("err %ld\n", PTR_ERR(vdir));
+		return;
+	}
+
+	dpri("deblk %u, nblk %lu, deblk %p, last{%lu, %p}, ver %llu\n",
+	     vdir->vd_deblk_sz, vdir->vd_nblk, vdir->vd_deblk,
+	     vdir->vd_last.ul, vdir->vd_last.p.deblk, vdir->vd_version);
+	for (ul = 0; ul < vdir->vd_nblk; ul++) {
+		p.deblk = vdir->vd_deblk[ul];
+		o = p.deblk;
+		dpri("[%lu]: %p\n", ul, o);
+	}
+}
+
+static int do_pri_inode(aufs_bindex_t bindex, struct inode *inode, int hn,
+			struct dentry *wh)
+{
+	char *n = NULL;
+	int l = 0;
+
+	if (!inode || IS_ERR(inode)) {
+		dpri("i%d: err %ld\n", bindex, PTR_ERR(inode));
+		return -1;
+	}
+
+	/* the type of i_blocks depends upon CONFIG_LBDAF */
+	BUILD_BUG_ON(sizeof(inode->i_blocks) != sizeof(unsigned long)
+		     && sizeof(inode->i_blocks) != sizeof(u64));
+	if (wh) {
+		n = (void *)wh->d_name.name;
+		l = wh->d_name.len;
+	}
+
+	dpri("i%d: %p, i%lu, %s, cnt %d, nl %u, 0%o, sz %llu, blk %llu,"
+	     " hn %d, ct %lld, np %lu, st 0x%lx, f 0x%x, v %llu, g %x%s%.*s\n",
+	     bindex, inode,
+	     inode->i_ino, inode->i_sb ? au_sbtype(inode->i_sb) : "??",
+	     atomic_read(&inode->i_count), inode->i_nlink, inode->i_mode,
+	     i_size_read(inode), (unsigned long long)inode->i_blocks,
+	     hn, (long long)timespec64_to_ns(&inode->i_ctime) & 0x0ffff,
+	     inode->i_mapping ? inode->i_mapping->nrpages : 0,
+	     inode->i_state, inode->i_flags, inode_peek_iversion(inode),
+	     inode->i_generation,
+	     l ? ", wh " : "", l, n);
+	return 0;
+}
+
+void au_dpri_inode(struct inode *inode)
+{
+	struct au_iinfo *iinfo;
+	struct au_hinode *hi;
+	aufs_bindex_t bindex;
+	int err, hn;
+
+	err = do_pri_inode(-1, inode, -1, NULL);
+	if (err || !au_test_aufs(inode->i_sb) || au_is_bad_inode(inode))
+		return;
+
+	iinfo = au_ii(inode);
+	dpri("i-1: btop %d, bbot %d, gen %d\n",
+	     iinfo->ii_btop, iinfo->ii_bbot, au_iigen(inode, NULL));
+	if (iinfo->ii_btop < 0)
+		return;
+	hn = 0;
+	for (bindex = iinfo->ii_btop; bindex <= iinfo->ii_bbot; bindex++) {
+		hi = au_hinode(iinfo, bindex);
+		hn = !!au_hn(hi);
+		do_pri_inode(bindex, hi->hi_inode, hn, hi->hi_whdentry);
+	}
+}
+
+void au_dpri_dalias(struct inode *inode)
+{
+	struct dentry *d;
+
+	spin_lock(&inode->i_lock);
+	hlist_for_each_entry(d, &inode->i_dentry, d_u.d_alias)
+		au_dpri_dentry(d);
+	spin_unlock(&inode->i_lock);
+}
+
+static int do_pri_dentry(aufs_bindex_t bindex, struct dentry *dentry)
+{
+	struct dentry *wh = NULL;
+	int hn;
+	struct inode *inode;
+	struct au_iinfo *iinfo;
+	struct au_hinode *hi;
+
+	if (!dentry || IS_ERR(dentry)) {
+		dpri("d%d: err %ld\n", bindex, PTR_ERR(dentry));
+		return -1;
+	}
+	/* do not call dget_parent() here */
+	/* note: access d_xxx without d_lock */
+	dpri("d%d: %p, %pd2?, %s, cnt %d, flags 0x%x, %shashed\n",
+	     bindex, dentry, dentry,
+	     dentry->d_sb ? au_sbtype(dentry->d_sb) : "??",
+	     au_dcount(dentry), dentry->d_flags,
+	     d_unhashed(dentry) ? "un" : "");
+	hn = -1;
+	inode = NULL;
+	if (d_is_positive(dentry))
+		inode = d_inode(dentry);
+	if (inode
+	    && au_test_aufs(dentry->d_sb)
+	    && bindex >= 0
+	    && !au_is_bad_inode(inode)) {
+		iinfo = au_ii(inode);
+		hi = au_hinode(iinfo, bindex);
+		hn = !!au_hn(hi);
+		wh = hi->hi_whdentry;
+	}
+	do_pri_inode(bindex, inode, hn, wh);
+	return 0;
+}
+
+void au_dpri_dentry(struct dentry *dentry)
+{
+	struct au_dinfo *dinfo;
+	aufs_bindex_t bindex;
+	int err;
+
+	err = do_pri_dentry(-1, dentry);
+	if (err || !au_test_aufs(dentry->d_sb))
+		return;
+
+	dinfo = au_di(dentry);
+	if (!dinfo)
+		return;
+	dpri("d-1: btop %d, bbot %d, bwh %d, bdiropq %d, gen %d, tmp %d\n",
+	     dinfo->di_btop, dinfo->di_bbot,
+	     dinfo->di_bwh, dinfo->di_bdiropq, au_digen(dentry),
+	     dinfo->di_tmpfile);
+	if (dinfo->di_btop < 0)
+		return;
+	for (bindex = dinfo->di_btop; bindex <= dinfo->di_bbot; bindex++)
+		do_pri_dentry(bindex, au_hdentry(dinfo, bindex)->hd_dentry);
+}
+
+static int do_pri_file(aufs_bindex_t bindex, struct file *file)
+{
+	char a[32];
+
+	if (!file || IS_ERR(file)) {
+		dpri("f%d: err %ld\n", bindex, PTR_ERR(file));
+		return -1;
+	}
+	a[0] = 0;
+	if (bindex < 0
+	    && !IS_ERR_OR_NULL(file->f_path.dentry)
+	    && au_test_aufs(file->f_path.dentry->d_sb)
+	    && au_fi(file))
+		snprintf(a, sizeof(a), ", gen %d, mmapped %d",
+			 au_figen(file), atomic_read(&au_fi(file)->fi_mmapped));
+	dpri("f%d: mode 0x%x, flags 0%o, cnt %ld, v %llu, pos %llu%s\n",
+	     bindex, file->f_mode, file->f_flags, (long)file_count(file),
+	     file->f_version, file->f_pos, a);
+	if (!IS_ERR_OR_NULL(file->f_path.dentry))
+		do_pri_dentry(bindex, file->f_path.dentry);
+	return 0;
+}
+
+void au_dpri_file(struct file *file)
+{
+	struct au_finfo *finfo;
+	struct au_fidir *fidir;
+	struct au_hfile *hfile;
+	aufs_bindex_t bindex;
+	int err;
+
+	err = do_pri_file(-1, file);
+	if (err
+	    || IS_ERR_OR_NULL(file->f_path.dentry)
+	    || !au_test_aufs(file->f_path.dentry->d_sb))
+		return;
+
+	finfo = au_fi(file);
+	if (!finfo)
+		return;
+	if (finfo->fi_btop < 0)
+		return;
+	fidir = finfo->fi_hdir;
+	if (!fidir)
+		do_pri_file(finfo->fi_btop, finfo->fi_htop.hf_file);
+	else
+		for (bindex = finfo->fi_btop;
+		     bindex >= 0 && bindex <= fidir->fd_bbot;
+		     bindex++) {
+			hfile = fidir->fd_hfile + bindex;
+			do_pri_file(bindex, hfile ? hfile->hf_file : NULL);
+		}
+}
+
+static int do_pri_br(aufs_bindex_t bindex, struct au_branch *br)
+{
+	struct vfsmount *mnt;
+	struct super_block *sb;
+
+	if (!br || IS_ERR(br))
+		goto out;
+	mnt = au_br_mnt(br);
+	if (!mnt || IS_ERR(mnt))
+		goto out;
+	sb = mnt->mnt_sb;
+	if (!sb || IS_ERR(sb))
+		goto out;
+
+	dpri("s%d: {perm 0x%x, id %d, wbr %p}, "
+	     "%s, dev 0x%02x%02x, flags 0x%lx, cnt %d, active %d, "
+	     "xino %d\n",
+	     bindex, br->br_perm, br->br_id, br->br_wbr,
+	     au_sbtype(sb), MAJOR(sb->s_dev), MINOR(sb->s_dev),
+	     sb->s_flags, sb->s_count,
+	     atomic_read(&sb->s_active),
+	     !!au_xino_file(br->br_xino, /*idx*/-1));
+	return 0;
+
+out:
+	dpri("s%d: err %ld\n", bindex, PTR_ERR(br));
+	return -1;
+}
+
+void au_dpri_sb(struct super_block *sb)
+{
+	struct au_sbinfo *sbinfo;
+	aufs_bindex_t bindex;
+	int err;
+	/* to reduce stack size */
+	struct {
+		struct vfsmount mnt;
+		struct au_branch fake;
+	} *a;
+
+	/* this function can be called from magic sysrq */
+	a = kzalloc(sizeof(*a), GFP_ATOMIC);
+	if (unlikely(!a)) {
+		dpri("no memory\n");
+		return;
+	}
+
+	a->mnt.mnt_sb = sb;
+	a->fake.br_path.mnt = &a->mnt;
+	err = do_pri_br(-1, &a->fake);
+	au_kfree_rcu(a);
+	dpri("dev 0x%x\n", sb->s_dev);
+	if (err || !au_test_aufs(sb))
+		return;
+
+	sbinfo = au_sbi(sb);
+	if (!sbinfo)
+		return;
+	dpri("nw %d, gen %u, kobj %d\n",
+	     atomic_read(&sbinfo->si_nowait.nw_len), sbinfo->si_generation,
+	     kref_read(&sbinfo->si_kobj.kref));
+	for (bindex = 0; bindex <= sbinfo->si_bbot; bindex++)
+		do_pri_br(bindex, sbinfo->si_branch[0 + bindex]);
+}
+
+/* ---------------------------------------------------------------------- */
+
+void __au_dbg_verify_dinode(struct dentry *dentry, const char *func, int line)
+{
+	struct inode *h_inode, *inode = d_inode(dentry);
+	struct dentry *h_dentry;
+	aufs_bindex_t bindex, bbot, bi;
+
+	if (!inode /* || au_di(dentry)->di_lsc == AuLsc_DI_TMP */)
+		return;
+
+	bbot = au_dbbot(dentry);
+	bi = au_ibbot(inode);
+	if (bi < bbot)
+		bbot = bi;
+	bindex = au_dbtop(dentry);
+	bi = au_ibtop(inode);
+	if (bi > bindex)
+		bindex = bi;
+
+	for (; bindex <= bbot; bindex++) {
+		h_dentry = au_h_dptr(dentry, bindex);
+		if (!h_dentry)
+			continue;
+		h_inode = au_h_iptr(inode, bindex);
+		if (unlikely(h_inode != d_inode(h_dentry))) {
+			au_debug_on();
+			AuDbg("b%d, %s:%d\n", bindex, func, line);
+			AuDbgDentry(dentry);
+			AuDbgInode(inode);
+			au_debug_off();
+			BUG();
+		}
+	}
+}
+
+void au_dbg_verify_gen(struct dentry *parent, unsigned int sigen)
+{
+	int err, i, j;
+	struct au_dcsub_pages dpages;
+	struct au_dpage *dpage;
+	struct dentry **dentries;
+
+	err = au_dpages_init(&dpages, GFP_NOFS);
+	AuDebugOn(err);
+	err = au_dcsub_pages_rev_aufs(&dpages, parent, /*do_include*/1);
+	AuDebugOn(err);
+	for (i = dpages.ndpage - 1; !err && i >= 0; i--) {
+		dpage = dpages.dpages + i;
+		dentries = dpage->dentries;
+		for (j = dpage->ndentry - 1; !err && j >= 0; j--)
+			AuDebugOn(au_digen_test(dentries[j], sigen));
+	}
+	au_dpages_free(&dpages);
+}
+
+void au_dbg_verify_kthread(void)
+{
+	if (au_wkq_test()) {
+		au_dbg_blocked();
+		/*
+		 * It may be recursive, but udba=notify between two aufs mounts,
+		 * where a single ro branch is shared, is not a problem.
+		 */
+		/* WARN_ON(1); */
+	}
+}
+
+/* ---------------------------------------------------------------------- */
+
+int __init au_debug_init(void)
+{
+	aufs_bindex_t bindex;
+	struct au_vdir_destr destr;
+
+	bindex = -1;
+	AuDebugOn(bindex >= 0);
+
+	destr.len = -1;
+	AuDebugOn(destr.len < NAME_MAX);
+
+#ifdef CONFIG_4KSTACKS
+	pr_warn("CONFIG_4KSTACKS is defined.\n");
+#endif
+
+	return 0;
+}
diff -Naur --no-dereference a/fs/aufs/debug.h b/fs/aufs/debug.h
--- a/fs/aufs/debug.h	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/debug.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,226 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * debug print functions
+ */
+
+#ifndef __AUFS_DEBUG_H__
+#define __AUFS_DEBUG_H__
+
+#ifdef __KERNEL__
+
+#include <linux/atomic.h>
+#include <linux/module.h>
+#include <linux/kallsyms.h>
+#include <linux/sysrq.h>
+
+#ifdef CONFIG_AUFS_DEBUG
+#define AuDebugOn(a)		BUG_ON(a)
+
+/* module parameter */
+extern atomic_t aufs_debug;
+static inline void au_debug_on(void)
+{
+	atomic_inc(&aufs_debug);
+}
+static inline void au_debug_off(void)
+{
+	atomic_dec_if_positive(&aufs_debug);
+}
+
+static inline int au_debug_test(void)
+{
+	return atomic_read(&aufs_debug) > 0;
+}
+#else
+#define AuDebugOn(a)		do {} while (0)
+AuStubVoid(au_debug_on, void)
+AuStubVoid(au_debug_off, void)
+AuStubInt0(au_debug_test, void)
+#endif /* CONFIG_AUFS_DEBUG */
+
+#define param_check_atomic_t(name, p) __param_check(name, p, atomic_t)
+
+/* ---------------------------------------------------------------------- */
+
+/* debug print */
+
+#define AuDbg(fmt, ...) do { \
+	if (au_debug_test()) \
+		pr_debug("DEBUG: " fmt, ##__VA_ARGS__); \
+} while (0)
+#define AuLabel(l)		AuDbg(#l "\n")
+#define AuIOErr(fmt, ...)	pr_err("I/O Error, " fmt, ##__VA_ARGS__)
+#define AuWarn1(fmt, ...) do { \
+	static unsigned char _c; \
+	if (!_c++) \
+		pr_warn(fmt, ##__VA_ARGS__); \
+} while (0)
+
+#define AuErr1(fmt, ...) do { \
+	static unsigned char _c; \
+	if (!_c++) \
+		pr_err(fmt, ##__VA_ARGS__); \
+} while (0)
+
+#define AuIOErr1(fmt, ...) do { \
+	static unsigned char _c; \
+	if (!_c++) \
+		AuIOErr(fmt, ##__VA_ARGS__); \
+} while (0)
+
+#define AuUnsupportMsg	"This operation is not supported." \
+			" Please report this application to aufs-users ML."
+#define AuUnsupport(fmt, ...) do { \
+	pr_err(AuUnsupportMsg "\n" fmt, ##__VA_ARGS__); \
+	dump_stack(); \
+} while (0)
+
+#define AuTraceErr(e) do { \
+	if (unlikely((e) < 0)) \
+		AuDbg("err %d\n", (int)(e)); \
+} while (0)
+
+#define AuTraceErrPtr(p) do { \
+	if (IS_ERR(p)) \
+		AuDbg("err %ld\n", PTR_ERR(p)); \
+} while (0)
+
+/* dirty macros for debug print, use with "%.*s" and caution */
+#define AuLNPair(qstr)		(qstr)->len, (qstr)->name
+
+/* ---------------------------------------------------------------------- */
+
+struct dentry;
+#ifdef CONFIG_AUFS_DEBUG
+extern struct mutex au_dbg_mtx;
+extern char *au_plevel;
+struct au_nhash;
+void au_dpri_whlist(struct au_nhash *whlist);
+struct au_vdir;
+void au_dpri_vdir(struct au_vdir *vdir);
+struct inode;
+void au_dpri_inode(struct inode *inode);
+void au_dpri_dalias(struct inode *inode);
+void au_dpri_dentry(struct dentry *dentry);
+struct file;
+void au_dpri_file(struct file *filp);
+struct super_block;
+void au_dpri_sb(struct super_block *sb);
+
+#define au_dbg_verify_dinode(d) __au_dbg_verify_dinode(d, __func__, __LINE__)
+void __au_dbg_verify_dinode(struct dentry *dentry, const char *func, int line);
+void au_dbg_verify_gen(struct dentry *parent, unsigned int sigen);
+void au_dbg_verify_kthread(void);
+
+int __init au_debug_init(void);
+
+#define AuDbgWhlist(w) do { \
+	mutex_lock(&au_dbg_mtx); \
+	AuDbg(#w "\n"); \
+	au_dpri_whlist(w); \
+	mutex_unlock(&au_dbg_mtx); \
+} while (0)
+
+#define AuDbgVdir(v) do { \
+	mutex_lock(&au_dbg_mtx); \
+	AuDbg(#v "\n"); \
+	au_dpri_vdir(v); \
+	mutex_unlock(&au_dbg_mtx); \
+} while (0)
+
+#define AuDbgInode(i) do { \
+	mutex_lock(&au_dbg_mtx); \
+	AuDbg(#i "\n"); \
+	au_dpri_inode(i); \
+	mutex_unlock(&au_dbg_mtx); \
+} while (0)
+
+#define AuDbgDAlias(i) do { \
+	mutex_lock(&au_dbg_mtx); \
+	AuDbg(#i "\n"); \
+	au_dpri_dalias(i); \
+	mutex_unlock(&au_dbg_mtx); \
+} while (0)
+
+#define AuDbgDentry(d) do { \
+	mutex_lock(&au_dbg_mtx); \
+	AuDbg(#d "\n"); \
+	au_dpri_dentry(d); \
+	mutex_unlock(&au_dbg_mtx); \
+} while (0)
+
+#define AuDbgFile(f) do { \
+	mutex_lock(&au_dbg_mtx); \
+	AuDbg(#f "\n"); \
+	au_dpri_file(f); \
+	mutex_unlock(&au_dbg_mtx); \
+} while (0)
+
+#define AuDbgSb(sb) do { \
+	mutex_lock(&au_dbg_mtx); \
+	AuDbg(#sb "\n"); \
+	au_dpri_sb(sb); \
+	mutex_unlock(&au_dbg_mtx); \
+} while (0)
+
+#define AuDbgSym(addr) do {				\
+	char sym[KSYM_SYMBOL_LEN];			\
+	sprint_symbol(sym, (unsigned long)addr);	\
+	AuDbg("%s\n", sym);				\
+} while (0)
+#else
+AuStubVoid(au_dbg_verify_dinode, struct dentry *dentry)
+AuStubVoid(au_dbg_verify_gen, struct dentry *parent, unsigned int sigen)
+AuStubVoid(au_dbg_verify_kthread, void)
+AuStubInt0(__init au_debug_init, void)
+
+#define AuDbgWhlist(w)		do {} while (0)
+#define AuDbgVdir(v)		do {} while (0)
+#define AuDbgInode(i)		do {} while (0)
+#define AuDbgDAlias(i)		do {} while (0)
+#define AuDbgDentry(d)		do {} while (0)
+#define AuDbgFile(f)		do {} while (0)
+#define AuDbgSb(sb)		do {} while (0)
+#define AuDbgSym(addr)		do {} while (0)
+#endif /* CONFIG_AUFS_DEBUG */
+
+/* ---------------------------------------------------------------------- */
+
+#ifdef CONFIG_AUFS_MAGIC_SYSRQ
+int __init au_sysrq_init(void);
+void au_sysrq_fin(void);
+
+#ifdef CONFIG_HW_CONSOLE
+#define au_dbg_blocked() do { \
+	WARN_ON(1); \
+	handle_sysrq('w'); \
+} while (0)
+#else
+AuStubVoid(au_dbg_blocked, void)
+#endif
+
+#else
+AuStubInt0(__init au_sysrq_init, void)
+AuStubVoid(au_sysrq_fin, void)
+AuStubVoid(au_dbg_blocked, void)
+#endif /* CONFIG_AUFS_MAGIC_SYSRQ */
+
+#endif /* __KERNEL__ */
+#endif /* __AUFS_DEBUG_H__ */
diff -Naur --no-dereference a/fs/aufs/dentry.c b/fs/aufs/dentry.c
--- a/fs/aufs/dentry.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/dentry.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,1154 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * lookup and dentry operations
+ */
+
+#include <linux/iversion.h>
+#include <linux/namei.h>
+#include "aufs.h"
+
+/*
+ * returns positive/negative dentry, NULL or an error.
+ * NULL means whiteout-ed or not-found.
+ */
+static struct dentry*
+au_do_lookup(struct dentry *h_parent, struct dentry *dentry,
+	     aufs_bindex_t bindex, struct au_do_lookup_args *args)
+{
+	struct dentry *h_dentry;
+	struct inode *h_inode;
+	struct au_branch *br;
+	int wh_found, opq;
+	unsigned char wh_able;
+	const unsigned char allow_neg = !!au_ftest_lkup(args->flags, ALLOW_NEG);
+	const unsigned char ignore_perm = !!au_ftest_lkup(args->flags,
+							  IGNORE_PERM);
+
+	wh_found = 0;
+	br = au_sbr(dentry->d_sb, bindex);
+	wh_able = !!au_br_whable(br->br_perm);
+	if (wh_able)
+		wh_found = au_wh_test(h_parent, &args->whname, ignore_perm);
+	h_dentry = ERR_PTR(wh_found);
+	if (!wh_found)
+		goto real_lookup;
+	if (unlikely(wh_found < 0))
+		goto out;
+
+	/* We found a whiteout */
+	/* au_set_dbbot(dentry, bindex); */
+	au_set_dbwh(dentry, bindex);
+	if (!allow_neg)
+		return NULL; /* success */
+
+real_lookup:
+	if (!ignore_perm)
+		h_dentry = vfsub_lkup_one(args->name, h_parent);
+	else
+		h_dentry = au_sio_lkup_one(args->name, h_parent);
+	if (IS_ERR(h_dentry)) {
+		if (PTR_ERR(h_dentry) == -ENAMETOOLONG
+		    && !allow_neg)
+			h_dentry = NULL;
+		goto out;
+	}
+
+	h_inode = d_inode(h_dentry);
+	if (d_is_negative(h_dentry)) {
+		if (!allow_neg)
+			goto out_neg;
+	} else if (wh_found
+		   || (args->type && args->type != (h_inode->i_mode & S_IFMT)))
+		goto out_neg;
+	else if (au_ftest_lkup(args->flags, DIRREN)
+		 /* && h_inode */
+		 && !au_dr_lkup_h_ino(args, bindex, h_inode->i_ino)) {
+		AuDbg("b%d %pd ignored hi%llu\n", bindex, h_dentry,
+		      (unsigned long long)h_inode->i_ino);
+		goto out_neg;
+	}
+
+	if (au_dbbot(dentry) <= bindex)
+		au_set_dbbot(dentry, bindex);
+	if (au_dbtop(dentry) < 0 || bindex < au_dbtop(dentry))
+		au_set_dbtop(dentry, bindex);
+	au_set_h_dptr(dentry, bindex, h_dentry);
+
+	if (!d_is_dir(h_dentry)
+	    || !wh_able
+	    || (d_really_is_positive(dentry) && !d_is_dir(dentry)))
+		goto out; /* success */
+
+	inode_lock_shared_nested(h_inode, AuLsc_I_CHILD);
+	opq = au_diropq_test(h_dentry);
+	inode_unlock_shared(h_inode);
+	if (opq > 0)
+		au_set_dbdiropq(dentry, bindex);
+	else if (unlikely(opq < 0)) {
+		au_set_h_dptr(dentry, bindex, NULL);
+		h_dentry = ERR_PTR(opq);
+	}
+	goto out;
+
+out_neg:
+	dput(h_dentry);
+	h_dentry = NULL;
+out:
+	return h_dentry;
+}
+
+static int au_test_shwh(struct super_block *sb, const struct qstr *name)
+{
+	if (unlikely(!au_opt_test(au_mntflags(sb), SHWH)
+		     && !strncmp(name->name, AUFS_WH_PFX, AUFS_WH_PFX_LEN)))
+		return -EPERM;
+	return 0;
+}
+
+/*
+ * returns the number of lower positive dentries,
+ * otherwise an error.
+ * can be called at unlinking with @type is zero.
+ */
+int au_lkup_dentry(struct dentry *dentry, aufs_bindex_t btop,
+		   unsigned int flags)
+{
+	int npositive, err;
+	aufs_bindex_t bindex, btail, bdiropq;
+	unsigned char isdir, dirperm1, dirren;
+	struct au_do_lookup_args args = {
+		.flags		= flags,
+		.name		= &dentry->d_name
+	};
+	struct dentry *parent;
+	struct super_block *sb;
+
+	sb = dentry->d_sb;
+	err = au_test_shwh(sb, args.name);
+	if (unlikely(err))
+		goto out;
+
+	err = au_wh_name_alloc(&args.whname, args.name);
+	if (unlikely(err))
+		goto out;
+
+	isdir = !!d_is_dir(dentry);
+	dirperm1 = !!au_opt_test(au_mntflags(sb), DIRPERM1);
+	dirren = !!au_opt_test(au_mntflags(sb), DIRREN);
+	if (dirren)
+		au_fset_lkup(args.flags, DIRREN);
+
+	npositive = 0;
+	parent = dget_parent(dentry);
+	btail = au_dbtaildir(parent);
+	for (bindex = btop; bindex <= btail; bindex++) {
+		struct dentry *h_parent, *h_dentry;
+		struct inode *h_inode, *h_dir;
+		struct au_branch *br;
+
+		h_dentry = au_h_dptr(dentry, bindex);
+		if (h_dentry) {
+			if (d_is_positive(h_dentry))
+				npositive++;
+			break;
+		}
+		h_parent = au_h_dptr(parent, bindex);
+		if (!h_parent || !d_is_dir(h_parent))
+			continue;
+
+		if (dirren) {
+			/* if the inum matches, then use the prepared name */
+			err = au_dr_lkup_name(&args, bindex);
+			if (unlikely(err))
+				goto out_parent;
+		}
+
+		h_dir = d_inode(h_parent);
+		inode_lock_shared_nested(h_dir, AuLsc_I_PARENT);
+		h_dentry = au_do_lookup(h_parent, dentry, bindex, &args);
+		inode_unlock_shared(h_dir);
+		err = PTR_ERR(h_dentry);
+		if (IS_ERR(h_dentry))
+			goto out_parent;
+		if (h_dentry)
+			au_fclr_lkup(args.flags, ALLOW_NEG);
+		if (dirperm1)
+			au_fset_lkup(args.flags, IGNORE_PERM);
+
+		if (au_dbwh(dentry) == bindex)
+			break;
+		if (!h_dentry)
+			continue;
+		if (d_is_negative(h_dentry))
+			continue;
+		h_inode = d_inode(h_dentry);
+		npositive++;
+		if (!args.type)
+			args.type = h_inode->i_mode & S_IFMT;
+		if (args.type != S_IFDIR)
+			break;
+		else if (isdir) {
+			/* the type of lower may be different */
+			bdiropq = au_dbdiropq(dentry);
+			if (bdiropq >= 0 && bdiropq <= bindex)
+				break;
+		}
+		br = au_sbr(sb, bindex);
+		if (dirren
+		    && au_dr_hino_test_add(&br->br_dirren, h_inode->i_ino,
+					   /*add_ent*/NULL)) {
+			/* prepare next name to lookup */
+			err = au_dr_lkup(&args, dentry, bindex);
+			if (unlikely(err))
+				goto out_parent;
+		}
+	}
+
+	if (npositive) {
+		AuLabel(positive);
+		au_update_dbtop(dentry);
+	}
+	err = npositive;
+	if (unlikely(!au_opt_test(au_mntflags(sb), UDBA_NONE)
+		     && au_dbtop(dentry) < 0)) {
+		err = -EIO;
+		AuIOErr("both of real entry and whiteout found, %pd, err %d\n",
+			dentry, err);
+	}
+
+out_parent:
+	dput(parent);
+	au_kfree_try_rcu(args.whname.name);
+	if (dirren)
+		au_dr_lkup_fin(&args);
+out:
+	return err;
+}
+
+struct dentry *au_sio_lkup_one(struct qstr *name, struct dentry *parent)
+{
+	struct dentry *dentry;
+	int wkq_err;
+
+	if (!au_test_h_perm_sio(d_inode(parent), MAY_EXEC))
+		dentry = vfsub_lkup_one(name, parent);
+	else {
+		struct vfsub_lkup_one_args args = {
+			.errp	= &dentry,
+			.name	= name,
+			.parent	= parent
+		};
+
+		wkq_err = au_wkq_wait(vfsub_call_lkup_one, &args);
+		if (unlikely(wkq_err))
+			dentry = ERR_PTR(wkq_err);
+	}
+
+	return dentry;
+}
+
+/*
+ * lookup @dentry on @bindex which should be negative.
+ */
+int au_lkup_neg(struct dentry *dentry, aufs_bindex_t bindex, int wh)
+{
+	int err;
+	struct dentry *parent, *h_parent, *h_dentry;
+	struct au_branch *br;
+
+	parent = dget_parent(dentry);
+	h_parent = au_h_dptr(parent, bindex);
+	br = au_sbr(dentry->d_sb, bindex);
+	if (wh)
+		h_dentry = au_whtmp_lkup(h_parent, br, &dentry->d_name);
+	else
+		h_dentry = au_sio_lkup_one(&dentry->d_name, h_parent);
+	err = PTR_ERR(h_dentry);
+	if (IS_ERR(h_dentry))
+		goto out;
+	if (unlikely(d_is_positive(h_dentry))) {
+		err = -EIO;
+		AuIOErr("%pd should be negative on b%d.\n", h_dentry, bindex);
+		dput(h_dentry);
+		goto out;
+	}
+
+	err = 0;
+	if (bindex < au_dbtop(dentry))
+		au_set_dbtop(dentry, bindex);
+	if (au_dbbot(dentry) < bindex)
+		au_set_dbbot(dentry, bindex);
+	au_set_h_dptr(dentry, bindex, h_dentry);
+
+out:
+	dput(parent);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* subset of struct inode */
+struct au_iattr {
+	unsigned long		i_ino;
+	/* unsigned int		i_nlink; */
+	kuid_t			i_uid;
+	kgid_t			i_gid;
+	u64			i_version;
+/*
+	loff_t			i_size;
+	blkcnt_t		i_blocks;
+*/
+	umode_t			i_mode;
+};
+
+static void au_iattr_save(struct au_iattr *ia, struct inode *h_inode)
+{
+	ia->i_ino = h_inode->i_ino;
+	/* ia->i_nlink = h_inode->i_nlink; */
+	ia->i_uid = h_inode->i_uid;
+	ia->i_gid = h_inode->i_gid;
+	ia->i_version = inode_query_iversion(h_inode);
+/*
+	ia->i_size = h_inode->i_size;
+	ia->i_blocks = h_inode->i_blocks;
+*/
+	ia->i_mode = (h_inode->i_mode & S_IFMT);
+}
+
+static int au_iattr_test(struct au_iattr *ia, struct inode *h_inode)
+{
+	return ia->i_ino != h_inode->i_ino
+		/* || ia->i_nlink != h_inode->i_nlink */
+		|| !uid_eq(ia->i_uid, h_inode->i_uid)
+		|| !gid_eq(ia->i_gid, h_inode->i_gid)
+		|| !inode_eq_iversion(h_inode, ia->i_version)
+/*
+		|| ia->i_size != h_inode->i_size
+		|| ia->i_blocks != h_inode->i_blocks
+*/
+		|| ia->i_mode != (h_inode->i_mode & S_IFMT);
+}
+
+static int au_h_verify_dentry(struct dentry *h_dentry, struct dentry *h_parent,
+			      struct au_branch *br)
+{
+	int err;
+	struct au_iattr ia;
+	struct inode *h_inode;
+	struct dentry *h_d;
+	struct super_block *h_sb;
+
+	err = 0;
+	memset(&ia, -1, sizeof(ia));
+	h_sb = h_dentry->d_sb;
+	h_inode = NULL;
+	if (d_is_positive(h_dentry)) {
+		h_inode = d_inode(h_dentry);
+		au_iattr_save(&ia, h_inode);
+	} else if (au_test_nfs(h_sb) || au_test_fuse(h_sb))
+		/* nfs d_revalidate may return 0 for negative dentry */
+		/* fuse d_revalidate always return 0 for negative dentry */
+		goto out;
+
+	/* main purpose is namei.c:cached_lookup() and d_revalidate */
+	h_d = vfsub_lkup_one(&h_dentry->d_name, h_parent);
+	err = PTR_ERR(h_d);
+	if (IS_ERR(h_d))
+		goto out;
+
+	err = 0;
+	if (unlikely(h_d != h_dentry
+		     || d_inode(h_d) != h_inode
+		     || (h_inode && au_iattr_test(&ia, h_inode))))
+		err = au_busy_or_stale();
+	dput(h_d);
+
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+int au_h_verify(struct dentry *h_dentry, unsigned int udba, struct inode *h_dir,
+		struct dentry *h_parent, struct au_branch *br)
+{
+	int err;
+
+	err = 0;
+	if (udba == AuOpt_UDBA_REVAL
+	    && !au_test_fs_remote(h_dentry->d_sb)) {
+		IMustLock(h_dir);
+		err = (d_inode(h_dentry->d_parent) != h_dir);
+	} else if (udba != AuOpt_UDBA_NONE)
+		err = au_h_verify_dentry(h_dentry, h_parent, br);
+
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static int au_do_refresh_hdentry(struct dentry *dentry, struct dentry *parent)
+{
+	int err;
+	aufs_bindex_t new_bindex, bindex, bbot, bwh, bdiropq;
+	struct au_hdentry tmp, *p, *q;
+	struct au_dinfo *dinfo;
+	struct super_block *sb;
+
+	DiMustWriteLock(dentry);
+
+	sb = dentry->d_sb;
+	dinfo = au_di(dentry);
+	bbot = dinfo->di_bbot;
+	bwh = dinfo->di_bwh;
+	bdiropq = dinfo->di_bdiropq;
+	bindex = dinfo->di_btop;
+	p = au_hdentry(dinfo, bindex);
+	for (; bindex <= bbot; bindex++, p++) {
+		if (!p->hd_dentry)
+			continue;
+
+		new_bindex = au_br_index(sb, p->hd_id);
+		if (new_bindex == bindex)
+			continue;
+
+		if (dinfo->di_bwh == bindex)
+			bwh = new_bindex;
+		if (dinfo->di_bdiropq == bindex)
+			bdiropq = new_bindex;
+		if (new_bindex < 0) {
+			au_hdput(p);
+			p->hd_dentry = NULL;
+			continue;
+		}
+
+		/* swap two lower dentries, and loop again */
+		q = au_hdentry(dinfo, new_bindex);
+		tmp = *q;
+		*q = *p;
+		*p = tmp;
+		if (tmp.hd_dentry) {
+			bindex--;
+			p--;
+		}
+	}
+
+	dinfo->di_bwh = -1;
+	if (bwh >= 0 && bwh <= au_sbbot(sb) && au_sbr_whable(sb, bwh))
+		dinfo->di_bwh = bwh;
+
+	dinfo->di_bdiropq = -1;
+	if (bdiropq >= 0
+	    && bdiropq <= au_sbbot(sb)
+	    && au_sbr_whable(sb, bdiropq))
+		dinfo->di_bdiropq = bdiropq;
+
+	err = -EIO;
+	dinfo->di_btop = -1;
+	dinfo->di_bbot = -1;
+	bbot = au_dbbot(parent);
+	bindex = 0;
+	p = au_hdentry(dinfo, bindex);
+	for (; bindex <= bbot; bindex++, p++)
+		if (p->hd_dentry) {
+			dinfo->di_btop = bindex;
+			break;
+		}
+
+	if (dinfo->di_btop >= 0) {
+		bindex = bbot;
+		p = au_hdentry(dinfo, bindex);
+		for (; bindex >= 0; bindex--, p--)
+			if (p->hd_dentry) {
+				dinfo->di_bbot = bindex;
+				err = 0;
+				break;
+			}
+	}
+
+	return err;
+}
+
+static void au_do_hide(struct dentry *dentry)
+{
+	struct inode *inode;
+
+	if (d_really_is_positive(dentry)) {
+		inode = d_inode(dentry);
+		if (!d_is_dir(dentry)) {
+			if (inode->i_nlink && !d_unhashed(dentry))
+				drop_nlink(inode);
+		} else {
+			clear_nlink(inode);
+			/* stop next lookup */
+			inode->i_flags |= S_DEAD;
+		}
+		smp_mb(); /* necessary? */
+	}
+	d_drop(dentry);
+}
+
+static int au_hide_children(struct dentry *parent)
+{
+	int err, i, j, ndentry;
+	struct au_dcsub_pages dpages;
+	struct au_dpage *dpage;
+	struct dentry *dentry;
+
+	err = au_dpages_init(&dpages, GFP_NOFS);
+	if (unlikely(err))
+		goto out;
+	err = au_dcsub_pages(&dpages, parent, NULL, NULL);
+	if (unlikely(err))
+		goto out_dpages;
+
+	/* in reverse order */
+	for (i = dpages.ndpage - 1; i >= 0; i--) {
+		dpage = dpages.dpages + i;
+		ndentry = dpage->ndentry;
+		for (j = ndentry - 1; j >= 0; j--) {
+			dentry = dpage->dentries[j];
+			if (dentry != parent)
+				au_do_hide(dentry);
+		}
+	}
+
+out_dpages:
+	au_dpages_free(&dpages);
+out:
+	return err;
+}
+
+static void au_hide(struct dentry *dentry)
+{
+	int err;
+
+	AuDbgDentry(dentry);
+	if (d_is_dir(dentry)) {
+		/* shrink_dcache_parent(dentry); */
+		err = au_hide_children(dentry);
+		if (unlikely(err))
+			AuIOErr("%pd, failed hiding children, ignored %d\n",
+				dentry, err);
+	}
+	au_do_hide(dentry);
+}
+
+/*
+ * By adding a dirty branch, a cached dentry may be affected in various ways.
+ *
+ * a dirty branch is added
+ * - on the top of layers
+ * - in the middle of layers
+ * - to the bottom of layers
+ *
+ * on the added branch there exists
+ * - a whiteout
+ * - a diropq
+ * - a same named entry
+ *   + exist
+ *     * negative --> positive
+ *     * positive --> positive
+ *	 - type is unchanged
+ *	 - type is changed
+ *   + doesn't exist
+ *     * negative --> negative
+ *     * positive --> negative (rejected by au_br_del() for non-dir case)
+ * - none
+ */
+static int au_refresh_by_dinfo(struct dentry *dentry, struct au_dinfo *dinfo,
+			       struct au_dinfo *tmp)
+{
+	int err;
+	aufs_bindex_t bindex, bbot;
+	struct {
+		struct dentry *dentry;
+		struct inode *inode;
+		mode_t mode;
+	} orig_h, tmp_h = {
+		.dentry = NULL
+	};
+	struct au_hdentry *hd;
+	struct inode *inode, *h_inode;
+	struct dentry *h_dentry;
+
+	err = 0;
+	AuDebugOn(dinfo->di_btop < 0);
+	orig_h.mode = 0;
+	orig_h.dentry = au_hdentry(dinfo, dinfo->di_btop)->hd_dentry;
+	orig_h.inode = NULL;
+	if (d_is_positive(orig_h.dentry)) {
+		orig_h.inode = d_inode(orig_h.dentry);
+		orig_h.mode = orig_h.inode->i_mode & S_IFMT;
+	}
+	if (tmp->di_btop >= 0) {
+		tmp_h.dentry = au_hdentry(tmp, tmp->di_btop)->hd_dentry;
+		if (d_is_positive(tmp_h.dentry)) {
+			tmp_h.inode = d_inode(tmp_h.dentry);
+			tmp_h.mode = tmp_h.inode->i_mode & S_IFMT;
+		}
+	}
+
+	inode = NULL;
+	if (d_really_is_positive(dentry))
+		inode = d_inode(dentry);
+	if (!orig_h.inode) {
+		AuDbg("negative originally\n");
+		if (inode) {
+			au_hide(dentry);
+			goto out;
+		}
+		AuDebugOn(inode);
+		AuDebugOn(dinfo->di_btop != dinfo->di_bbot);
+		AuDebugOn(dinfo->di_bdiropq != -1);
+
+		if (!tmp_h.inode) {
+			AuDbg("negative --> negative\n");
+			/* should have only one negative lower */
+			if (tmp->di_btop >= 0
+			    && tmp->di_btop < dinfo->di_btop) {
+				AuDebugOn(tmp->di_btop != tmp->di_bbot);
+				AuDebugOn(dinfo->di_btop != dinfo->di_bbot);
+				au_set_h_dptr(dentry, dinfo->di_btop, NULL);
+				au_di_cp(dinfo, tmp);
+				hd = au_hdentry(tmp, tmp->di_btop);
+				au_set_h_dptr(dentry, tmp->di_btop,
+					      dget(hd->hd_dentry));
+			}
+			au_dbg_verify_dinode(dentry);
+		} else {
+			AuDbg("negative --> positive\n");
+			/*
+			 * similar to the behaviour of creating with bypassing
+			 * aufs.
+			 * unhash it in order to force an error in the
+			 * succeeding create operation.
+			 * we should not set S_DEAD here.
+			 */
+			d_drop(dentry);
+			/* au_di_swap(tmp, dinfo); */
+			au_dbg_verify_dinode(dentry);
+		}
+	} else {
+		AuDbg("positive originally\n");
+		/* inode may be NULL */
+		AuDebugOn(inode && (inode->i_mode & S_IFMT) != orig_h.mode);
+		if (!tmp_h.inode) {
+			AuDbg("positive --> negative\n");
+			/* or bypassing aufs */
+			au_hide(dentry);
+			if (tmp->di_bwh >= 0 && tmp->di_bwh <= dinfo->di_btop)
+				dinfo->di_bwh = tmp->di_bwh;
+			if (inode)
+				err = au_refresh_hinode_self(inode);
+			au_dbg_verify_dinode(dentry);
+		} else if (orig_h.mode == tmp_h.mode) {
+			AuDbg("positive --> positive, same type\n");
+			if (!S_ISDIR(orig_h.mode)
+			    && dinfo->di_btop > tmp->di_btop) {
+				/*
+				 * similar to the behaviour of removing and
+				 * creating.
+				 */
+				au_hide(dentry);
+				if (inode)
+					err = au_refresh_hinode_self(inode);
+				au_dbg_verify_dinode(dentry);
+			} else {
+				/* fill empty slots */
+				if (dinfo->di_btop > tmp->di_btop)
+					dinfo->di_btop = tmp->di_btop;
+				if (dinfo->di_bbot < tmp->di_bbot)
+					dinfo->di_bbot = tmp->di_bbot;
+				dinfo->di_bwh = tmp->di_bwh;
+				dinfo->di_bdiropq = tmp->di_bdiropq;
+				bbot = dinfo->di_bbot;
+				bindex = tmp->di_btop;
+				hd = au_hdentry(tmp, bindex);
+				for (; bindex <= bbot; bindex++, hd++) {
+					if (au_h_dptr(dentry, bindex))
+						continue;
+					h_dentry = hd->hd_dentry;
+					if (!h_dentry)
+						continue;
+					AuDebugOn(d_is_negative(h_dentry));
+					h_inode = d_inode(h_dentry);
+					AuDebugOn(orig_h.mode
+						  != (h_inode->i_mode
+						      & S_IFMT));
+					au_set_h_dptr(dentry, bindex,
+						      dget(h_dentry));
+				}
+				if (inode)
+					err = au_refresh_hinode(inode, dentry);
+				au_dbg_verify_dinode(dentry);
+			}
+		} else {
+			AuDbg("positive --> positive, different type\n");
+			/* similar to the behaviour of removing and creating */
+			au_hide(dentry);
+			if (inode)
+				err = au_refresh_hinode_self(inode);
+			au_dbg_verify_dinode(dentry);
+		}
+	}
+
+out:
+	return err;
+}
+
+void au_refresh_dop(struct dentry *dentry, int force_reval)
+{
+	const struct dentry_operations *dop
+		= force_reval ? &aufs_dop : dentry->d_sb->s_d_op;
+	static const unsigned int mask
+		= DCACHE_OP_REVALIDATE | DCACHE_OP_WEAK_REVALIDATE;
+
+	BUILD_BUG_ON(sizeof(mask) != sizeof(dentry->d_flags));
+
+	if (dentry->d_op == dop)
+		return;
+
+	AuDbg("%pd\n", dentry);
+	spin_lock(&dentry->d_lock);
+	if (dop == &aufs_dop)
+		dentry->d_flags |= mask;
+	else
+		dentry->d_flags &= ~mask;
+	dentry->d_op = dop;
+	spin_unlock(&dentry->d_lock);
+}
+
+int au_refresh_dentry(struct dentry *dentry, struct dentry *parent)
+{
+	int err, ebrange, nbr;
+	unsigned int sigen;
+	struct au_dinfo *dinfo, *tmp;
+	struct super_block *sb;
+	struct inode *inode;
+
+	DiMustWriteLock(dentry);
+	AuDebugOn(IS_ROOT(dentry));
+	AuDebugOn(d_really_is_negative(parent));
+
+	sb = dentry->d_sb;
+	sigen = au_sigen(sb);
+	err = au_digen_test(parent, sigen);
+	if (unlikely(err))
+		goto out;
+
+	nbr = au_sbbot(sb) + 1;
+	dinfo = au_di(dentry);
+	err = au_di_realloc(dinfo, nbr, /*may_shrink*/0);
+	if (unlikely(err))
+		goto out;
+	ebrange = au_dbrange_test(dentry);
+	if (!ebrange)
+		ebrange = au_do_refresh_hdentry(dentry, parent);
+
+	if (d_unhashed(dentry) || ebrange /* || dinfo->di_tmpfile */) {
+		AuDebugOn(au_dbtop(dentry) < 0 && au_dbbot(dentry) >= 0);
+		if (d_really_is_positive(dentry)) {
+			inode = d_inode(dentry);
+			err = au_refresh_hinode_self(inode);
+		}
+		au_dbg_verify_dinode(dentry);
+		if (!err)
+			goto out_dgen; /* success */
+		goto out;
+	}
+
+	/* temporary dinfo */
+	AuDbgDentry(dentry);
+	err = -ENOMEM;
+	tmp = au_di_alloc(sb, AuLsc_DI_TMP);
+	if (unlikely(!tmp))
+		goto out;
+	au_di_swap(tmp, dinfo);
+	/* returns the number of positive dentries */
+	/*
+	 * if current working dir is removed, it returns an error.
+	 * but the dentry is legal.
+	 */
+	err = au_lkup_dentry(dentry, /*btop*/0, AuLkup_ALLOW_NEG);
+	AuDbgDentry(dentry);
+	au_di_swap(tmp, dinfo);
+	if (err == -ENOENT)
+		err = 0;
+	if (err >= 0) {
+		/* compare/refresh by dinfo */
+		AuDbgDentry(dentry);
+		err = au_refresh_by_dinfo(dentry, dinfo, tmp);
+		au_dbg_verify_dinode(dentry);
+		AuTraceErr(err);
+	}
+	au_di_realloc(dinfo, nbr, /*may_shrink*/1); /* harmless if err */
+	au_rw_write_unlock(&tmp->di_rwsem);
+	au_di_free(tmp);
+	if (unlikely(err))
+		goto out;
+
+out_dgen:
+	au_update_digen(dentry);
+out:
+	if (unlikely(err && !(dentry->d_flags & DCACHE_NFSFS_RENAMED))) {
+		AuIOErr("failed refreshing %pd, %d\n", dentry, err);
+		AuDbgDentry(dentry);
+	}
+	AuTraceErr(err);
+	return err;
+}
+
+static int au_do_h_d_reval(struct dentry *h_dentry, unsigned int flags,
+			   struct dentry *dentry, aufs_bindex_t bindex)
+{
+	int err, valid;
+
+	err = 0;
+	if (!(h_dentry->d_flags & DCACHE_OP_REVALIDATE))
+		goto out;
+
+	AuDbg("b%d\n", bindex);
+	/*
+	 * gave up supporting LOOKUP_CREATE/OPEN for lower fs,
+	 * due to whiteout and branch permission.
+	 */
+	flags &= ~(/*LOOKUP_PARENT |*/ LOOKUP_OPEN | LOOKUP_CREATE
+		   | LOOKUP_FOLLOW | LOOKUP_EXCL);
+	/* it may return tri-state */
+	valid = h_dentry->d_op->d_revalidate(h_dentry, flags);
+
+	if (unlikely(valid < 0))
+		err = valid;
+	else if (!valid)
+		err = -EINVAL;
+
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+/* todo: remove this */
+static int h_d_revalidate(struct dentry *dentry, struct inode *inode,
+			  unsigned int flags, int do_udba, int dirren)
+{
+	int err;
+	umode_t mode, h_mode;
+	aufs_bindex_t bindex, btail, btop, ibs, ibe;
+	unsigned char plus, unhashed, is_root, h_plus, h_nfs, tmpfile;
+	struct inode *h_inode, *h_cached_inode;
+	struct dentry *h_dentry;
+	struct qstr *name, *h_name;
+
+	err = 0;
+	plus = 0;
+	mode = 0;
+	ibs = -1;
+	ibe = -1;
+	unhashed = !!d_unhashed(dentry);
+	is_root = !!IS_ROOT(dentry);
+	name = &dentry->d_name;
+	tmpfile = au_di(dentry)->di_tmpfile;
+
+	/*
+	 * Theoretically, REVAL test should be unnecessary in case of
+	 * {FS,I}NOTIFY.
+	 * But {fs,i}notify doesn't fire some necessary events,
+	 *	IN_ATTRIB for atime/nlink/pageio
+	 * Let's do REVAL test too.
+	 */
+	if (do_udba && inode) {
+		mode = (inode->i_mode & S_IFMT);
+		plus = (inode->i_nlink > 0);
+		ibs = au_ibtop(inode);
+		ibe = au_ibbot(inode);
+	}
+
+	btop = au_dbtop(dentry);
+	btail = btop;
+	if (inode && S_ISDIR(inode->i_mode))
+		btail = au_dbtaildir(dentry);
+	for (bindex = btop; bindex <= btail; bindex++) {
+		h_dentry = au_h_dptr(dentry, bindex);
+		if (!h_dentry)
+			continue;
+
+		AuDbg("b%d, %pd\n", bindex, h_dentry);
+		h_nfs = !!au_test_nfs(h_dentry->d_sb);
+		spin_lock(&h_dentry->d_lock);
+		h_name = &h_dentry->d_name;
+		if (unlikely(do_udba
+			     && !is_root
+			     && ((!h_nfs
+				  && (unhashed != !!d_unhashed(h_dentry)
+				      || (!tmpfile && !dirren
+					  && !au_qstreq(name, h_name))
+					  ))
+				 || (h_nfs
+				     && !(flags & LOOKUP_OPEN)
+				     && (h_dentry->d_flags
+					 & DCACHE_NFSFS_RENAMED)))
+			    )) {
+			int h_unhashed;
+
+			h_unhashed = d_unhashed(h_dentry);
+			spin_unlock(&h_dentry->d_lock);
+			AuDbg("unhash 0x%x 0x%x, %pd %pd\n",
+			      unhashed, h_unhashed, dentry, h_dentry);
+			goto err;
+		}
+		spin_unlock(&h_dentry->d_lock);
+
+		err = au_do_h_d_reval(h_dentry, flags, dentry, bindex);
+		if (unlikely(err))
+			/* do not goto err, to keep the errno */
+			break;
+
+		/* todo: plink too? */
+		if (!do_udba)
+			continue;
+
+		/* UDBA tests */
+		if (unlikely(!!inode != d_is_positive(h_dentry)))
+			goto err;
+
+		h_inode = NULL;
+		if (d_is_positive(h_dentry))
+			h_inode = d_inode(h_dentry);
+		h_plus = plus;
+		h_mode = mode;
+		h_cached_inode = h_inode;
+		if (h_inode) {
+			h_mode = (h_inode->i_mode & S_IFMT);
+			h_plus = (h_inode->i_nlink > 0);
+		}
+		if (inode && ibs <= bindex && bindex <= ibe)
+			h_cached_inode = au_h_iptr(inode, bindex);
+
+		if (!h_nfs) {
+			if (unlikely(plus != h_plus && !tmpfile))
+				goto err;
+		} else {
+			if (unlikely(!(h_dentry->d_flags & DCACHE_NFSFS_RENAMED)
+				     && !is_root
+				     && !IS_ROOT(h_dentry)
+				     && unhashed != d_unhashed(h_dentry)))
+				goto err;
+		}
+		if (unlikely(mode != h_mode
+			     || h_cached_inode != h_inode))
+			goto err;
+		continue;
+
+err:
+		err = -EINVAL;
+		break;
+	}
+
+	AuTraceErr(err);
+	return err;
+}
+
+/* todo: consolidate with do_refresh() and au_reval_for_attr() */
+static int simple_reval_dpath(struct dentry *dentry, unsigned int sigen)
+{
+	int err;
+	struct dentry *parent;
+
+	if (!au_digen_test(dentry, sigen))
+		return 0;
+
+	parent = dget_parent(dentry);
+	di_read_lock_parent(parent, AuLock_IR);
+	AuDebugOn(au_digen_test(parent, sigen));
+	au_dbg_verify_gen(parent, sigen);
+	err = au_refresh_dentry(dentry, parent);
+	di_read_unlock(parent, AuLock_IR);
+	dput(parent);
+	AuTraceErr(err);
+	return err;
+}
+
+int au_reval_dpath(struct dentry *dentry, unsigned int sigen)
+{
+	int err;
+	struct dentry *d, *parent;
+
+	if (!au_ftest_si(au_sbi(dentry->d_sb), FAILED_REFRESH_DIR))
+		return simple_reval_dpath(dentry, sigen);
+
+	/* slow loop, keep it simple and stupid */
+	/* cf: au_cpup_dirs() */
+	err = 0;
+	parent = NULL;
+	while (au_digen_test(dentry, sigen)) {
+		d = dentry;
+		while (1) {
+			dput(parent);
+			parent = dget_parent(d);
+			if (!au_digen_test(parent, sigen))
+				break;
+			d = parent;
+		}
+
+		if (d != dentry)
+			di_write_lock_child2(d);
+
+		/* someone might update our dentry while we were sleeping */
+		if (au_digen_test(d, sigen)) {
+			/*
+			 * todo: consolidate with simple_reval_dpath(),
+			 * do_refresh() and au_reval_for_attr().
+			 */
+			di_read_lock_parent(parent, AuLock_IR);
+			err = au_refresh_dentry(d, parent);
+			di_read_unlock(parent, AuLock_IR);
+		}
+
+		if (d != dentry)
+			di_write_unlock(d);
+		dput(parent);
+		if (unlikely(err))
+			break;
+	}
+
+	return err;
+}
+
+/*
+ * if valid returns 1, otherwise 0.
+ */
+static int aufs_d_revalidate(struct dentry *dentry, unsigned int flags)
+{
+	int valid, err;
+	unsigned int sigen;
+	unsigned char do_udba, dirren;
+	struct super_block *sb;
+	struct inode *inode;
+
+	/* todo: support rcu-walk? */
+	if (flags & LOOKUP_RCU)
+		return -ECHILD;
+
+	valid = 0;
+	if (unlikely(!au_di(dentry)))
+		goto out;
+
+	valid = 1;
+	sb = dentry->d_sb;
+	/*
+	 * todo: very ugly
+	 * i_mutex of parent dir may be held,
+	 * but we should not return 'invalid' due to busy.
+	 */
+	err = aufs_read_lock(dentry, AuLock_FLUSH | AuLock_DW | AuLock_NOPLM);
+	if (unlikely(err)) {
+		valid = err;
+		AuTraceErr(err);
+		goto out;
+	}
+	inode = NULL;
+	if (d_really_is_positive(dentry))
+		inode = d_inode(dentry);
+	if (unlikely(inode && au_is_bad_inode(inode))) {
+		err = -EINVAL;
+		AuTraceErr(err);
+		goto out_dgrade;
+	}
+	if (unlikely(au_dbrange_test(dentry))) {
+		err = -EINVAL;
+		AuTraceErr(err);
+		goto out_dgrade;
+	}
+
+	sigen = au_sigen(sb);
+	if (au_digen_test(dentry, sigen)) {
+		AuDebugOn(IS_ROOT(dentry));
+		err = au_reval_dpath(dentry, sigen);
+		if (unlikely(err)) {
+			AuTraceErr(err);
+			goto out_dgrade;
+		}
+	}
+	di_downgrade_lock(dentry, AuLock_IR);
+
+	err = -EINVAL;
+	if (!(flags & (LOOKUP_OPEN | LOOKUP_EMPTY))
+	    && inode
+	    && !(inode->i_state && I_LINKABLE)
+	    && (IS_DEADDIR(inode) || !inode->i_nlink)) {
+		AuTraceErr(err);
+		goto out_inval;
+	}
+
+	do_udba = !au_opt_test(au_mntflags(sb), UDBA_NONE);
+	if (do_udba && inode) {
+		aufs_bindex_t btop = au_ibtop(inode);
+		struct inode *h_inode;
+
+		if (btop >= 0) {
+			h_inode = au_h_iptr(inode, btop);
+			if (h_inode && au_test_higen(inode, h_inode)) {
+				AuTraceErr(err);
+				goto out_inval;
+			}
+		}
+	}
+
+	dirren = !!au_opt_test(au_mntflags(sb), DIRREN);
+	err = h_d_revalidate(dentry, inode, flags, do_udba, dirren);
+	if (unlikely(!err && do_udba && au_dbtop(dentry) < 0)) {
+		err = -EIO;
+		AuDbg("both of real entry and whiteout found, %p, err %d\n",
+		      dentry, err);
+	}
+	goto out_inval;
+
+out_dgrade:
+	di_downgrade_lock(dentry, AuLock_IR);
+out_inval:
+	aufs_read_unlock(dentry, AuLock_IR);
+	AuTraceErr(err);
+	valid = !err;
+out:
+	if (!valid) {
+		AuDbg("%pd invalid, %d\n", dentry, valid);
+		d_drop(dentry);
+	}
+	return valid;
+}
+
+static void aufs_d_release(struct dentry *dentry)
+{
+	if (au_di(dentry)) {
+		au_di_fin(dentry);
+		au_hn_di_reinit(dentry);
+	}
+}
+
+const struct dentry_operations aufs_dop = {
+	.d_revalidate		= aufs_d_revalidate,
+	.d_weak_revalidate	= aufs_d_revalidate,
+	.d_release		= aufs_d_release
+};
+
+/* aufs_dop without d_revalidate */
+const struct dentry_operations aufs_dop_noreval = {
+	.d_release		= aufs_d_release
+};
diff -Naur --no-dereference a/fs/aufs/dentry.h b/fs/aufs/dentry.h
--- a/fs/aufs/dentry.h	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/dentry.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,268 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * lookup and dentry operations
+ */
+
+#ifndef __AUFS_DENTRY_H__
+#define __AUFS_DENTRY_H__
+
+#ifdef __KERNEL__
+
+#include <linux/dcache.h>
+#include "dirren.h"
+#include "rwsem.h"
+
+struct au_hdentry {
+	struct dentry		*hd_dentry;
+	aufs_bindex_t		hd_id;
+};
+
+struct au_dinfo {
+	atomic_t		di_generation;
+
+	struct au_rwsem		di_rwsem;
+	aufs_bindex_t		di_btop, di_bbot, di_bwh, di_bdiropq;
+	unsigned char		di_tmpfile; /* to allow the different name */
+	struct au_hdentry	*di_hdentry;
+	struct rcu_head		rcu;
+} ____cacheline_aligned_in_smp;
+
+/* ---------------------------------------------------------------------- */
+
+/* flags for au_lkup_dentry() */
+#define AuLkup_ALLOW_NEG	1
+#define AuLkup_IGNORE_PERM	(1 << 1)
+#define AuLkup_DIRREN		(1 << 2)
+#define au_ftest_lkup(flags, name)	((flags) & AuLkup_##name)
+#define au_fset_lkup(flags, name) \
+	do { (flags) |= AuLkup_##name; } while (0)
+#define au_fclr_lkup(flags, name) \
+	do { (flags) &= ~AuLkup_##name; } while (0)
+
+#ifndef CONFIG_AUFS_DIRREN
+#undef AuLkup_DIRREN
+#define AuLkup_DIRREN 0
+#endif
+
+struct au_do_lookup_args {
+	unsigned int		flags;
+	mode_t			type;
+	struct qstr		whname, *name;
+	struct au_dr_lookup	dirren;
+};
+
+/* ---------------------------------------------------------------------- */
+
+/* dentry.c */
+extern const struct dentry_operations aufs_dop, aufs_dop_noreval;
+struct au_branch;
+struct dentry *au_sio_lkup_one(struct qstr *name, struct dentry *parent);
+int au_h_verify(struct dentry *h_dentry, unsigned int udba, struct inode *h_dir,
+		struct dentry *h_parent, struct au_branch *br);
+
+int au_lkup_dentry(struct dentry *dentry, aufs_bindex_t btop,
+		   unsigned int flags);
+int au_lkup_neg(struct dentry *dentry, aufs_bindex_t bindex, int wh);
+int au_refresh_dentry(struct dentry *dentry, struct dentry *parent);
+int au_reval_dpath(struct dentry *dentry, unsigned int sigen);
+void au_refresh_dop(struct dentry *dentry, int force_reval);
+
+/* dinfo.c */
+void au_di_init_once(void *_di);
+struct au_dinfo *au_di_alloc(struct super_block *sb, unsigned int lsc);
+void au_di_free(struct au_dinfo *dinfo);
+void au_di_swap(struct au_dinfo *a, struct au_dinfo *b);
+void au_di_cp(struct au_dinfo *dst, struct au_dinfo *src);
+int au_di_init(struct dentry *dentry);
+void au_di_fin(struct dentry *dentry);
+int au_di_realloc(struct au_dinfo *dinfo, int nbr, int may_shrink);
+
+void di_read_lock(struct dentry *d, int flags, unsigned int lsc);
+void di_read_unlock(struct dentry *d, int flags);
+void di_downgrade_lock(struct dentry *d, int flags);
+void di_write_lock(struct dentry *d, unsigned int lsc);
+void di_write_unlock(struct dentry *d);
+void di_write_lock2_child(struct dentry *d1, struct dentry *d2, int isdir);
+void di_write_lock2_parent(struct dentry *d1, struct dentry *d2, int isdir);
+void di_write_unlock2(struct dentry *d1, struct dentry *d2);
+
+struct dentry *au_h_dptr(struct dentry *dentry, aufs_bindex_t bindex);
+struct dentry *au_h_d_alias(struct dentry *dentry, aufs_bindex_t bindex);
+aufs_bindex_t au_dbtail(struct dentry *dentry);
+aufs_bindex_t au_dbtaildir(struct dentry *dentry);
+
+void au_set_h_dptr(struct dentry *dentry, aufs_bindex_t bindex,
+		   struct dentry *h_dentry);
+int au_digen_test(struct dentry *dentry, unsigned int sigen);
+int au_dbrange_test(struct dentry *dentry);
+void au_update_digen(struct dentry *dentry);
+void au_update_dbrange(struct dentry *dentry, int do_put_zero);
+void au_update_dbtop(struct dentry *dentry);
+void au_update_dbbot(struct dentry *dentry);
+int au_find_dbindex(struct dentry *dentry, struct dentry *h_dentry);
+
+/* ---------------------------------------------------------------------- */
+
+static inline struct au_dinfo *au_di(struct dentry *dentry)
+{
+	return dentry->d_fsdata;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* lock subclass for dinfo */
+enum {
+	AuLsc_DI_CHILD,		/* child first */
+	AuLsc_DI_CHILD2,	/* rename(2), link(2), and cpup at hnotify */
+	AuLsc_DI_CHILD3,	/* copyup dirs */
+	AuLsc_DI_PARENT,
+	AuLsc_DI_PARENT2,
+	AuLsc_DI_PARENT3,
+	AuLsc_DI_TMP		/* temp for replacing dinfo */
+};
+
+/*
+ * di_read_lock_child, di_write_lock_child,
+ * di_read_lock_child2, di_write_lock_child2,
+ * di_read_lock_child3, di_write_lock_child3,
+ * di_read_lock_parent, di_write_lock_parent,
+ * di_read_lock_parent2, di_write_lock_parent2,
+ * di_read_lock_parent3, di_write_lock_parent3,
+ */
+#define AuReadLockFunc(name, lsc) \
+static inline void di_read_lock_##name(struct dentry *d, int flags) \
+{ di_read_lock(d, flags, AuLsc_DI_##lsc); }
+
+#define AuWriteLockFunc(name, lsc) \
+static inline void di_write_lock_##name(struct dentry *d) \
+{ di_write_lock(d, AuLsc_DI_##lsc); }
+
+#define AuRWLockFuncs(name, lsc) \
+	AuReadLockFunc(name, lsc) \
+	AuWriteLockFunc(name, lsc)
+
+AuRWLockFuncs(child, CHILD);
+AuRWLockFuncs(child2, CHILD2);
+AuRWLockFuncs(child3, CHILD3);
+AuRWLockFuncs(parent, PARENT);
+AuRWLockFuncs(parent2, PARENT2);
+AuRWLockFuncs(parent3, PARENT3);
+
+#undef AuReadLockFunc
+#undef AuWriteLockFunc
+#undef AuRWLockFuncs
+
+#define DiMustNoWaiters(d)	AuRwMustNoWaiters(&au_di(d)->di_rwsem)
+#define DiMustAnyLock(d)	AuRwMustAnyLock(&au_di(d)->di_rwsem)
+#define DiMustWriteLock(d)	AuRwMustWriteLock(&au_di(d)->di_rwsem)
+
+/* ---------------------------------------------------------------------- */
+
+/* todo: memory barrier? */
+static inline unsigned int au_digen(struct dentry *d)
+{
+	return atomic_read(&au_di(d)->di_generation);
+}
+
+static inline void au_h_dentry_init(struct au_hdentry *hdentry)
+{
+	hdentry->hd_dentry = NULL;
+}
+
+static inline struct au_hdentry *au_hdentry(struct au_dinfo *di,
+					    aufs_bindex_t bindex)
+{
+	return di->di_hdentry + bindex;
+}
+
+static inline void au_hdput(struct au_hdentry *hd)
+{
+	if (hd)
+		dput(hd->hd_dentry);
+}
+
+static inline aufs_bindex_t au_dbtop(struct dentry *dentry)
+{
+	DiMustAnyLock(dentry);
+	return au_di(dentry)->di_btop;
+}
+
+static inline aufs_bindex_t au_dbbot(struct dentry *dentry)
+{
+	DiMustAnyLock(dentry);
+	return au_di(dentry)->di_bbot;
+}
+
+static inline aufs_bindex_t au_dbwh(struct dentry *dentry)
+{
+	DiMustAnyLock(dentry);
+	return au_di(dentry)->di_bwh;
+}
+
+static inline aufs_bindex_t au_dbdiropq(struct dentry *dentry)
+{
+	DiMustAnyLock(dentry);
+	return au_di(dentry)->di_bdiropq;
+}
+
+/* todo: hard/soft set? */
+static inline void au_set_dbtop(struct dentry *dentry, aufs_bindex_t bindex)
+{
+	DiMustWriteLock(dentry);
+	au_di(dentry)->di_btop = bindex;
+}
+
+static inline void au_set_dbbot(struct dentry *dentry, aufs_bindex_t bindex)
+{
+	DiMustWriteLock(dentry);
+	au_di(dentry)->di_bbot = bindex;
+}
+
+static inline void au_set_dbwh(struct dentry *dentry, aufs_bindex_t bindex)
+{
+	DiMustWriteLock(dentry);
+	/* dbwh can be outside of btop - bbot range */
+	au_di(dentry)->di_bwh = bindex;
+}
+
+static inline void au_set_dbdiropq(struct dentry *dentry, aufs_bindex_t bindex)
+{
+	DiMustWriteLock(dentry);
+	au_di(dentry)->di_bdiropq = bindex;
+}
+
+/* ---------------------------------------------------------------------- */
+
+#ifdef CONFIG_AUFS_HNOTIFY
+static inline void au_digen_dec(struct dentry *d)
+{
+	atomic_dec(&au_di(d)->di_generation);
+}
+
+static inline void au_hn_di_reinit(struct dentry *dentry)
+{
+	dentry->d_fsdata = NULL;
+}
+#else
+AuStubVoid(au_hn_di_reinit, struct dentry *dentry __maybe_unused)
+#endif /* CONFIG_AUFS_HNOTIFY */
+
+#endif /* __KERNEL__ */
+#endif /* __AUFS_DENTRY_H__ */
diff -Naur --no-dereference a/fs/aufs/dinfo.c b/fs/aufs/dinfo.c
--- a/fs/aufs/dinfo.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/dinfo.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,554 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * dentry private data
+ */
+
+#include "aufs.h"
+
+void au_di_init_once(void *_dinfo)
+{
+	struct au_dinfo *dinfo = _dinfo;
+
+	au_rw_init(&dinfo->di_rwsem);
+}
+
+struct au_dinfo *au_di_alloc(struct super_block *sb, unsigned int lsc)
+{
+	struct au_dinfo *dinfo;
+	int nbr, i;
+
+	dinfo = au_cache_alloc_dinfo();
+	if (unlikely(!dinfo))
+		goto out;
+
+	nbr = au_sbbot(sb) + 1;
+	if (nbr <= 0)
+		nbr = 1;
+	dinfo->di_hdentry = kcalloc(nbr, sizeof(*dinfo->di_hdentry), GFP_NOFS);
+	if (dinfo->di_hdentry) {
+		au_rw_write_lock_nested(&dinfo->di_rwsem, lsc);
+		dinfo->di_btop = -1;
+		dinfo->di_bbot = -1;
+		dinfo->di_bwh = -1;
+		dinfo->di_bdiropq = -1;
+		dinfo->di_tmpfile = 0;
+		for (i = 0; i < nbr; i++)
+			dinfo->di_hdentry[i].hd_id = -1;
+		goto out;
+	}
+
+	au_cache_free_dinfo(dinfo);
+	dinfo = NULL;
+
+out:
+	return dinfo;
+}
+
+void au_di_free(struct au_dinfo *dinfo)
+{
+	struct au_hdentry *p;
+	aufs_bindex_t bbot, bindex;
+
+	/* dentry may not be revalidated */
+	bindex = dinfo->di_btop;
+	if (bindex >= 0) {
+		bbot = dinfo->di_bbot;
+		p = au_hdentry(dinfo, bindex);
+		while (bindex++ <= bbot)
+			au_hdput(p++);
+	}
+	au_kfree_try_rcu(dinfo->di_hdentry);
+	au_cache_free_dinfo(dinfo);
+}
+
+void au_di_swap(struct au_dinfo *a, struct au_dinfo *b)
+{
+	struct au_hdentry *p;
+	aufs_bindex_t bi;
+
+	AuRwMustWriteLock(&a->di_rwsem);
+	AuRwMustWriteLock(&b->di_rwsem);
+
+#define DiSwap(v, name)				\
+	do {					\
+		v = a->di_##name;		\
+		a->di_##name = b->di_##name;	\
+		b->di_##name = v;		\
+	} while (0)
+
+	DiSwap(p, hdentry);
+	DiSwap(bi, btop);
+	DiSwap(bi, bbot);
+	DiSwap(bi, bwh);
+	DiSwap(bi, bdiropq);
+	/* smp_mb(); */
+
+#undef DiSwap
+}
+
+void au_di_cp(struct au_dinfo *dst, struct au_dinfo *src)
+{
+	AuRwMustWriteLock(&dst->di_rwsem);
+	AuRwMustWriteLock(&src->di_rwsem);
+
+	dst->di_btop = src->di_btop;
+	dst->di_bbot = src->di_bbot;
+	dst->di_bwh = src->di_bwh;
+	dst->di_bdiropq = src->di_bdiropq;
+	/* smp_mb(); */
+}
+
+int au_di_init(struct dentry *dentry)
+{
+	int err;
+	struct super_block *sb;
+	struct au_dinfo *dinfo;
+
+	err = 0;
+	sb = dentry->d_sb;
+	dinfo = au_di_alloc(sb, AuLsc_DI_CHILD);
+	if (dinfo) {
+		atomic_set(&dinfo->di_generation, au_sigen(sb));
+		/* smp_mb(); */ /* atomic_set */
+		dentry->d_fsdata = dinfo;
+	} else
+		err = -ENOMEM;
+
+	return err;
+}
+
+void au_di_fin(struct dentry *dentry)
+{
+	struct au_dinfo *dinfo;
+
+	dinfo = au_di(dentry);
+	AuRwDestroy(&dinfo->di_rwsem);
+	au_di_free(dinfo);
+}
+
+int au_di_realloc(struct au_dinfo *dinfo, int nbr, int may_shrink)
+{
+	int err, sz;
+	struct au_hdentry *hdp;
+
+	AuRwMustWriteLock(&dinfo->di_rwsem);
+
+	err = -ENOMEM;
+	sz = sizeof(*hdp) * (dinfo->di_bbot + 1);
+	if (!sz)
+		sz = sizeof(*hdp);
+	hdp = au_kzrealloc(dinfo->di_hdentry, sz, sizeof(*hdp) * nbr, GFP_NOFS,
+			   may_shrink);
+	if (hdp) {
+		dinfo->di_hdentry = hdp;
+		err = 0;
+	}
+
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static void do_ii_write_lock(struct inode *inode, unsigned int lsc)
+{
+	switch (lsc) {
+	case AuLsc_DI_CHILD:
+		ii_write_lock_child(inode);
+		break;
+	case AuLsc_DI_CHILD2:
+		ii_write_lock_child2(inode);
+		break;
+	case AuLsc_DI_CHILD3:
+		ii_write_lock_child3(inode);
+		break;
+	case AuLsc_DI_PARENT:
+		ii_write_lock_parent(inode);
+		break;
+	case AuLsc_DI_PARENT2:
+		ii_write_lock_parent2(inode);
+		break;
+	case AuLsc_DI_PARENT3:
+		ii_write_lock_parent3(inode);
+		break;
+	default:
+		BUG();
+	}
+}
+
+static void do_ii_read_lock(struct inode *inode, unsigned int lsc)
+{
+	switch (lsc) {
+	case AuLsc_DI_CHILD:
+		ii_read_lock_child(inode);
+		break;
+	case AuLsc_DI_CHILD2:
+		ii_read_lock_child2(inode);
+		break;
+	case AuLsc_DI_CHILD3:
+		ii_read_lock_child3(inode);
+		break;
+	case AuLsc_DI_PARENT:
+		ii_read_lock_parent(inode);
+		break;
+	case AuLsc_DI_PARENT2:
+		ii_read_lock_parent2(inode);
+		break;
+	case AuLsc_DI_PARENT3:
+		ii_read_lock_parent3(inode);
+		break;
+	default:
+		BUG();
+	}
+}
+
+void di_read_lock(struct dentry *d, int flags, unsigned int lsc)
+{
+	struct inode *inode;
+
+	au_rw_read_lock_nested(&au_di(d)->di_rwsem, lsc);
+	if (d_really_is_positive(d)) {
+		inode = d_inode(d);
+		if (au_ftest_lock(flags, IW))
+			do_ii_write_lock(inode, lsc);
+		else if (au_ftest_lock(flags, IR))
+			do_ii_read_lock(inode, lsc);
+	}
+}
+
+void di_read_unlock(struct dentry *d, int flags)
+{
+	struct inode *inode;
+
+	if (d_really_is_positive(d)) {
+		inode = d_inode(d);
+		if (au_ftest_lock(flags, IW)) {
+			au_dbg_verify_dinode(d);
+			ii_write_unlock(inode);
+		} else if (au_ftest_lock(flags, IR)) {
+			au_dbg_verify_dinode(d);
+			ii_read_unlock(inode);
+		}
+	}
+	au_rw_read_unlock(&au_di(d)->di_rwsem);
+}
+
+void di_downgrade_lock(struct dentry *d, int flags)
+{
+	if (d_really_is_positive(d) && au_ftest_lock(flags, IR))
+		ii_downgrade_lock(d_inode(d));
+	au_rw_dgrade_lock(&au_di(d)->di_rwsem);
+}
+
+void di_write_lock(struct dentry *d, unsigned int lsc)
+{
+	au_rw_write_lock_nested(&au_di(d)->di_rwsem, lsc);
+	if (d_really_is_positive(d))
+		do_ii_write_lock(d_inode(d), lsc);
+}
+
+void di_write_unlock(struct dentry *d)
+{
+	au_dbg_verify_dinode(d);
+	if (d_really_is_positive(d))
+		ii_write_unlock(d_inode(d));
+	au_rw_write_unlock(&au_di(d)->di_rwsem);
+}
+
+void di_write_lock2_child(struct dentry *d1, struct dentry *d2, int isdir)
+{
+	AuDebugOn(d1 == d2
+		  || d_inode(d1) == d_inode(d2)
+		  || d1->d_sb != d2->d_sb);
+
+	if ((isdir && au_test_subdir(d1, d2))
+	    || d1 < d2) {
+		di_write_lock_child(d1);
+		di_write_lock_child2(d2);
+	} else {
+		di_write_lock_child(d2);
+		di_write_lock_child2(d1);
+	}
+}
+
+void di_write_lock2_parent(struct dentry *d1, struct dentry *d2, int isdir)
+{
+	AuDebugOn(d1 == d2
+		  || d_inode(d1) == d_inode(d2)
+		  || d1->d_sb != d2->d_sb);
+
+	if ((isdir && au_test_subdir(d1, d2))
+	    || d1 < d2) {
+		di_write_lock_parent(d1);
+		di_write_lock_parent2(d2);
+	} else {
+		di_write_lock_parent(d2);
+		di_write_lock_parent2(d1);
+	}
+}
+
+void di_write_unlock2(struct dentry *d1, struct dentry *d2)
+{
+	di_write_unlock(d1);
+	if (d_inode(d1) == d_inode(d2))
+		au_rw_write_unlock(&au_di(d2)->di_rwsem);
+	else
+		di_write_unlock(d2);
+}
+
+/* ---------------------------------------------------------------------- */
+
+struct dentry *au_h_dptr(struct dentry *dentry, aufs_bindex_t bindex)
+{
+	struct dentry *d;
+
+	DiMustAnyLock(dentry);
+
+	if (au_dbtop(dentry) < 0 || bindex < au_dbtop(dentry))
+		return NULL;
+	AuDebugOn(bindex < 0);
+	d = au_hdentry(au_di(dentry), bindex)->hd_dentry;
+	AuDebugOn(d && au_dcount(d) <= 0);
+	return d;
+}
+
+/*
+ * extended version of au_h_dptr().
+ * returns a hashed and positive (or linkable) h_dentry in bindex, NULL, or
+ * error.
+ */
+struct dentry *au_h_d_alias(struct dentry *dentry, aufs_bindex_t bindex)
+{
+	struct dentry *h_dentry;
+	struct inode *inode, *h_inode;
+
+	AuDebugOn(d_really_is_negative(dentry));
+
+	h_dentry = NULL;
+	if (au_dbtop(dentry) <= bindex
+	    && bindex <= au_dbbot(dentry))
+		h_dentry = au_h_dptr(dentry, bindex);
+	if (h_dentry && !au_d_linkable(h_dentry)) {
+		dget(h_dentry);
+		goto out; /* success */
+	}
+
+	inode = d_inode(dentry);
+	AuDebugOn(bindex < au_ibtop(inode));
+	AuDebugOn(au_ibbot(inode) < bindex);
+	h_inode = au_h_iptr(inode, bindex);
+	h_dentry = d_find_alias(h_inode);
+	if (h_dentry) {
+		if (!IS_ERR(h_dentry)) {
+			if (!au_d_linkable(h_dentry))
+				goto out; /* success */
+			dput(h_dentry);
+		} else
+			goto out;
+	}
+
+	if (au_opt_test(au_mntflags(dentry->d_sb), PLINK)) {
+		h_dentry = au_plink_lkup(inode, bindex);
+		AuDebugOn(!h_dentry);
+		if (!IS_ERR(h_dentry)) {
+			if (!au_d_hashed_positive(h_dentry))
+				goto out; /* success */
+			dput(h_dentry);
+			h_dentry = NULL;
+		}
+	}
+
+out:
+	AuDbgDentry(h_dentry);
+	return h_dentry;
+}
+
+aufs_bindex_t au_dbtail(struct dentry *dentry)
+{
+	aufs_bindex_t bbot, bwh;
+
+	bbot = au_dbbot(dentry);
+	if (0 <= bbot) {
+		bwh = au_dbwh(dentry);
+		if (!bwh)
+			return bwh;
+		if (0 < bwh && bwh < bbot)
+			return bwh - 1;
+	}
+	return bbot;
+}
+
+aufs_bindex_t au_dbtaildir(struct dentry *dentry)
+{
+	aufs_bindex_t bbot, bopq;
+
+	bbot = au_dbtail(dentry);
+	if (0 <= bbot) {
+		bopq = au_dbdiropq(dentry);
+		if (0 <= bopq && bopq < bbot)
+			bbot = bopq;
+	}
+	return bbot;
+}
+
+/* ---------------------------------------------------------------------- */
+
+void au_set_h_dptr(struct dentry *dentry, aufs_bindex_t bindex,
+		   struct dentry *h_dentry)
+{
+	struct au_dinfo *dinfo;
+	struct au_hdentry *hd;
+	struct au_branch *br;
+
+	DiMustWriteLock(dentry);
+
+	dinfo = au_di(dentry);
+	hd = au_hdentry(dinfo, bindex);
+	au_hdput(hd);
+	hd->hd_dentry = h_dentry;
+	if (h_dentry) {
+		br = au_sbr(dentry->d_sb, bindex);
+		hd->hd_id = br->br_id;
+	}
+}
+
+int au_dbrange_test(struct dentry *dentry)
+{
+	int err;
+	aufs_bindex_t btop, bbot;
+
+	err = 0;
+	btop = au_dbtop(dentry);
+	bbot = au_dbbot(dentry);
+	if (btop >= 0)
+		AuDebugOn(bbot < 0 && btop > bbot);
+	else {
+		err = -EIO;
+		AuDebugOn(bbot >= 0);
+	}
+
+	return err;
+}
+
+int au_digen_test(struct dentry *dentry, unsigned int sigen)
+{
+	int err;
+
+	err = 0;
+	if (unlikely(au_digen(dentry) != sigen
+		     || au_iigen_test(d_inode(dentry), sigen)))
+		err = -EIO;
+
+	return err;
+}
+
+void au_update_digen(struct dentry *dentry)
+{
+	atomic_set(&au_di(dentry)->di_generation, au_sigen(dentry->d_sb));
+	/* smp_mb(); */ /* atomic_set */
+}
+
+void au_update_dbrange(struct dentry *dentry, int do_put_zero)
+{
+	struct au_dinfo *dinfo;
+	struct dentry *h_d;
+	struct au_hdentry *hdp;
+	aufs_bindex_t bindex, bbot;
+
+	DiMustWriteLock(dentry);
+
+	dinfo = au_di(dentry);
+	if (!dinfo || dinfo->di_btop < 0)
+		return;
+
+	if (do_put_zero) {
+		bbot = dinfo->di_bbot;
+		bindex = dinfo->di_btop;
+		hdp = au_hdentry(dinfo, bindex);
+		for (; bindex <= bbot; bindex++, hdp++) {
+			h_d = hdp->hd_dentry;
+			if (h_d && d_is_negative(h_d))
+				au_set_h_dptr(dentry, bindex, NULL);
+		}
+	}
+
+	dinfo->di_btop = 0;
+	hdp = au_hdentry(dinfo, dinfo->di_btop);
+	for (; dinfo->di_btop <= dinfo->di_bbot; dinfo->di_btop++, hdp++)
+		if (hdp->hd_dentry)
+			break;
+	if (dinfo->di_btop > dinfo->di_bbot) {
+		dinfo->di_btop = -1;
+		dinfo->di_bbot = -1;
+		return;
+	}
+
+	hdp = au_hdentry(dinfo, dinfo->di_bbot);
+	for (; dinfo->di_bbot >= 0; dinfo->di_bbot--, hdp--)
+		if (hdp->hd_dentry)
+			break;
+	AuDebugOn(dinfo->di_btop > dinfo->di_bbot || dinfo->di_bbot < 0);
+}
+
+void au_update_dbtop(struct dentry *dentry)
+{
+	aufs_bindex_t bindex, bbot;
+	struct dentry *h_dentry;
+
+	bbot = au_dbbot(dentry);
+	for (bindex = au_dbtop(dentry); bindex <= bbot; bindex++) {
+		h_dentry = au_h_dptr(dentry, bindex);
+		if (!h_dentry)
+			continue;
+		if (d_is_positive(h_dentry)) {
+			au_set_dbtop(dentry, bindex);
+			return;
+		}
+		au_set_h_dptr(dentry, bindex, NULL);
+	}
+}
+
+void au_update_dbbot(struct dentry *dentry)
+{
+	aufs_bindex_t bindex, btop;
+	struct dentry *h_dentry;
+
+	btop = au_dbtop(dentry);
+	for (bindex = au_dbbot(dentry); bindex >= btop; bindex--) {
+		h_dentry = au_h_dptr(dentry, bindex);
+		if (!h_dentry)
+			continue;
+		if (d_is_positive(h_dentry)) {
+			au_set_dbbot(dentry, bindex);
+			return;
+		}
+		au_set_h_dptr(dentry, bindex, NULL);
+	}
+}
+
+int au_find_dbindex(struct dentry *dentry, struct dentry *h_dentry)
+{
+	aufs_bindex_t bindex, bbot;
+
+	bbot = au_dbbot(dentry);
+	for (bindex = au_dbtop(dentry); bindex <= bbot; bindex++)
+		if (au_h_dptr(dentry, bindex) == h_dentry)
+			return bindex;
+	return -1;
+}
diff -Naur --no-dereference a/fs/aufs/dir.c b/fs/aufs/dir.c
--- a/fs/aufs/dir.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/dir.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,763 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * directory operations
+ */
+
+#include <linux/fs_stack.h>
+#include <linux/iversion.h>
+#include "aufs.h"
+
+void au_add_nlink(struct inode *dir, struct inode *h_dir)
+{
+	unsigned int nlink;
+
+	AuDebugOn(!S_ISDIR(dir->i_mode) || !S_ISDIR(h_dir->i_mode));
+
+	nlink = dir->i_nlink;
+	nlink += h_dir->i_nlink - 2;
+	if (h_dir->i_nlink < 2)
+		nlink += 2;
+	smp_mb(); /* for i_nlink */
+	/* 0 can happen in revaliding */
+	set_nlink(dir, nlink);
+}
+
+void au_sub_nlink(struct inode *dir, struct inode *h_dir)
+{
+	unsigned int nlink;
+
+	AuDebugOn(!S_ISDIR(dir->i_mode) || !S_ISDIR(h_dir->i_mode));
+
+	nlink = dir->i_nlink;
+	nlink -= h_dir->i_nlink - 2;
+	if (h_dir->i_nlink < 2)
+		nlink -= 2;
+	smp_mb(); /* for i_nlink */
+	/* nlink == 0 means the branch-fs is broken */
+	set_nlink(dir, nlink);
+}
+
+loff_t au_dir_size(struct file *file, struct dentry *dentry)
+{
+	loff_t sz;
+	aufs_bindex_t bindex, bbot;
+	struct file *h_file;
+	struct dentry *h_dentry;
+
+	sz = 0;
+	if (file) {
+		AuDebugOn(!d_is_dir(file->f_path.dentry));
+
+		bbot = au_fbbot_dir(file);
+		for (bindex = au_fbtop(file);
+		     bindex <= bbot && sz < KMALLOC_MAX_SIZE;
+		     bindex++) {
+			h_file = au_hf_dir(file, bindex);
+			if (h_file && file_inode(h_file))
+				sz += vfsub_f_size_read(h_file);
+		}
+	} else {
+		AuDebugOn(!dentry);
+		AuDebugOn(!d_is_dir(dentry));
+
+		bbot = au_dbtaildir(dentry);
+		for (bindex = au_dbtop(dentry);
+		     bindex <= bbot && sz < KMALLOC_MAX_SIZE;
+		     bindex++) {
+			h_dentry = au_h_dptr(dentry, bindex);
+			if (h_dentry && d_is_positive(h_dentry))
+				sz += i_size_read(d_inode(h_dentry));
+		}
+	}
+	if (sz < KMALLOC_MAX_SIZE)
+		sz = roundup_pow_of_two(sz);
+	if (sz > KMALLOC_MAX_SIZE)
+		sz = KMALLOC_MAX_SIZE;
+	else if (sz < NAME_MAX) {
+		BUILD_BUG_ON(AUFS_RDBLK_DEF < NAME_MAX);
+		sz = AUFS_RDBLK_DEF;
+	}
+	return sz;
+}
+
+struct au_dir_ts_arg {
+	struct dentry *dentry;
+	aufs_bindex_t brid;
+};
+
+static void au_do_dir_ts(void *arg)
+{
+	struct au_dir_ts_arg *a = arg;
+	struct au_dtime dt;
+	struct path h_path;
+	struct inode *dir, *h_dir;
+	struct super_block *sb;
+	struct au_branch *br;
+	struct au_hinode *hdir;
+	int err;
+	aufs_bindex_t btop, bindex;
+
+	sb = a->dentry->d_sb;
+	if (d_really_is_negative(a->dentry))
+		goto out;
+	/* no dir->i_mutex lock */
+	aufs_read_lock(a->dentry, AuLock_DW); /* noflush */
+
+	dir = d_inode(a->dentry);
+	btop = au_ibtop(dir);
+	bindex = au_br_index(sb, a->brid);
+	if (bindex < btop)
+		goto out_unlock;
+
+	br = au_sbr(sb, bindex);
+	h_path.dentry = au_h_dptr(a->dentry, bindex);
+	if (!h_path.dentry)
+		goto out_unlock;
+	h_path.mnt = au_br_mnt(br);
+	au_dtime_store(&dt, a->dentry, &h_path);
+
+	br = au_sbr(sb, btop);
+	if (!au_br_writable(br->br_perm))
+		goto out_unlock;
+	h_path.dentry = au_h_dptr(a->dentry, btop);
+	h_path.mnt = au_br_mnt(br);
+	err = vfsub_mnt_want_write(h_path.mnt);
+	if (err)
+		goto out_unlock;
+	hdir = au_hi(dir, btop);
+	au_hn_inode_lock_nested(hdir, AuLsc_I_PARENT);
+	h_dir = au_h_iptr(dir, btop);
+	if (h_dir->i_nlink
+	    && timespec64_compare(&h_dir->i_mtime, &dt.dt_mtime) < 0) {
+		dt.dt_h_path = h_path;
+		au_dtime_revert(&dt);
+	}
+	au_hn_inode_unlock(hdir);
+	vfsub_mnt_drop_write(h_path.mnt);
+	au_cpup_attr_timesizes(dir);
+
+out_unlock:
+	aufs_read_unlock(a->dentry, AuLock_DW);
+out:
+	dput(a->dentry);
+	au_nwt_done(&au_sbi(sb)->si_nowait);
+	au_kfree_try_rcu(arg);
+}
+
+void au_dir_ts(struct inode *dir, aufs_bindex_t bindex)
+{
+	int perm, wkq_err;
+	aufs_bindex_t btop;
+	struct au_dir_ts_arg *arg;
+	struct dentry *dentry;
+	struct super_block *sb;
+
+	IMustLock(dir);
+
+	dentry = d_find_any_alias(dir);
+	AuDebugOn(!dentry);
+	sb = dentry->d_sb;
+	btop = au_ibtop(dir);
+	if (btop == bindex) {
+		au_cpup_attr_timesizes(dir);
+		goto out;
+	}
+
+	perm = au_sbr_perm(sb, btop);
+	if (!au_br_writable(perm))
+		goto out;
+
+	arg = kmalloc(sizeof(*arg), GFP_NOFS);
+	if (!arg)
+		goto out;
+
+	arg->dentry = dget(dentry); /* will be dput-ted by au_do_dir_ts() */
+	arg->brid = au_sbr_id(sb, bindex);
+	wkq_err = au_wkq_nowait(au_do_dir_ts, arg, sb, /*flags*/0);
+	if (unlikely(wkq_err)) {
+		pr_err("wkq %d\n", wkq_err);
+		dput(dentry);
+		au_kfree_try_rcu(arg);
+	}
+
+out:
+	dput(dentry);
+}
+
+/* ---------------------------------------------------------------------- */
+
+static int reopen_dir(struct file *file)
+{
+	int err;
+	unsigned int flags;
+	aufs_bindex_t bindex, btail, btop;
+	struct dentry *dentry, *h_dentry;
+	struct file *h_file;
+
+	/* open all lower dirs */
+	dentry = file->f_path.dentry;
+	btop = au_dbtop(dentry);
+	for (bindex = au_fbtop(file); bindex < btop; bindex++)
+		au_set_h_fptr(file, bindex, NULL);
+	au_set_fbtop(file, btop);
+
+	btail = au_dbtaildir(dentry);
+	for (bindex = au_fbbot_dir(file); btail < bindex; bindex--)
+		au_set_h_fptr(file, bindex, NULL);
+	au_set_fbbot_dir(file, btail);
+
+	flags = vfsub_file_flags(file);
+	for (bindex = btop; bindex <= btail; bindex++) {
+		h_dentry = au_h_dptr(dentry, bindex);
+		if (!h_dentry)
+			continue;
+		h_file = au_hf_dir(file, bindex);
+		if (h_file)
+			continue;
+
+		h_file = au_h_open(dentry, bindex, flags, file, /*force_wr*/0);
+		err = PTR_ERR(h_file);
+		if (IS_ERR(h_file))
+			goto out; /* close all? */
+		au_set_h_fptr(file, bindex, h_file);
+	}
+	au_update_figen(file);
+	/* todo: necessary? */
+	/* file->f_ra = h_file->f_ra; */
+	err = 0;
+
+out:
+	return err;
+}
+
+static int do_open_dir(struct file *file, int flags, struct file *h_file)
+{
+	int err;
+	aufs_bindex_t bindex, btail;
+	struct dentry *dentry, *h_dentry;
+	struct vfsmount *mnt;
+
+	FiMustWriteLock(file);
+	AuDebugOn(h_file);
+
+	err = 0;
+	mnt = file->f_path.mnt;
+	dentry = file->f_path.dentry;
+	file->f_version = inode_query_iversion(d_inode(dentry));
+	bindex = au_dbtop(dentry);
+	au_set_fbtop(file, bindex);
+	btail = au_dbtaildir(dentry);
+	au_set_fbbot_dir(file, btail);
+	for (; !err && bindex <= btail; bindex++) {
+		h_dentry = au_h_dptr(dentry, bindex);
+		if (!h_dentry)
+			continue;
+
+		err = vfsub_test_mntns(mnt, h_dentry->d_sb);
+		if (unlikely(err))
+			break;
+		h_file = au_h_open(dentry, bindex, flags, file, /*force_wr*/0);
+		if (IS_ERR(h_file)) {
+			err = PTR_ERR(h_file);
+			break;
+		}
+		au_set_h_fptr(file, bindex, h_file);
+	}
+	au_update_figen(file);
+	/* todo: necessary? */
+	/* file->f_ra = h_file->f_ra; */
+	if (!err)
+		return 0; /* success */
+
+	/* close all */
+	for (bindex = au_fbtop(file); bindex <= btail; bindex++)
+		au_set_h_fptr(file, bindex, NULL);
+	au_set_fbtop(file, -1);
+	au_set_fbbot_dir(file, -1);
+
+	return err;
+}
+
+static int aufs_open_dir(struct inode *inode __maybe_unused,
+			 struct file *file)
+{
+	int err;
+	struct super_block *sb;
+	struct au_fidir *fidir;
+
+	err = -ENOMEM;
+	sb = file->f_path.dentry->d_sb;
+	si_read_lock(sb, AuLock_FLUSH);
+	fidir = au_fidir_alloc(sb);
+	if (fidir) {
+		struct au_do_open_args args = {
+			.open	= do_open_dir,
+			.fidir	= fidir
+		};
+		err = au_do_open(file, &args);
+		if (unlikely(err))
+			au_kfree_rcu(fidir);
+	}
+	si_read_unlock(sb);
+	return err;
+}
+
+static int aufs_release_dir(struct inode *inode __maybe_unused,
+			    struct file *file)
+{
+	struct au_vdir *vdir_cache;
+	struct au_finfo *finfo;
+	struct au_fidir *fidir;
+	struct au_hfile *hf;
+	aufs_bindex_t bindex, bbot;
+
+	finfo = au_fi(file);
+	fidir = finfo->fi_hdir;
+	if (fidir) {
+		au_hbl_del(&finfo->fi_hlist,
+			   &au_sbi(file->f_path.dentry->d_sb)->si_files);
+		vdir_cache = fidir->fd_vdir_cache; /* lock-free */
+		if (vdir_cache)
+			au_vdir_free(vdir_cache);
+
+		bindex = finfo->fi_btop;
+		if (bindex >= 0) {
+			hf = fidir->fd_hfile + bindex;
+			/*
+			 * calls fput() instead of filp_close(),
+			 * since no dnotify or lock for the lower file.
+			 */
+			bbot = fidir->fd_bbot;
+			for (; bindex <= bbot; bindex++, hf++)
+				if (hf->hf_file)
+					au_hfput(hf, /*execed*/0);
+		}
+		au_kfree_rcu(fidir);
+		finfo->fi_hdir = NULL;
+	}
+	au_finfo_fin(file);
+	return 0;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static int au_do_flush_dir(struct file *file, fl_owner_t id)
+{
+	int err;
+	aufs_bindex_t bindex, bbot;
+	struct file *h_file;
+
+	err = 0;
+	bbot = au_fbbot_dir(file);
+	for (bindex = au_fbtop(file); !err && bindex <= bbot; bindex++) {
+		h_file = au_hf_dir(file, bindex);
+		if (h_file)
+			err = vfsub_flush(h_file, id);
+	}
+	return err;
+}
+
+static int aufs_flush_dir(struct file *file, fl_owner_t id)
+{
+	return au_do_flush(file, id, au_do_flush_dir);
+}
+
+/* ---------------------------------------------------------------------- */
+
+static int au_do_fsync_dir_no_file(struct dentry *dentry, int datasync)
+{
+	int err;
+	aufs_bindex_t bbot, bindex;
+	struct inode *inode;
+	struct super_block *sb;
+
+	err = 0;
+	sb = dentry->d_sb;
+	inode = d_inode(dentry);
+	IMustLock(inode);
+	bbot = au_dbbot(dentry);
+	for (bindex = au_dbtop(dentry); !err && bindex <= bbot; bindex++) {
+		struct path h_path;
+
+		if (au_test_ro(sb, bindex, inode))
+			continue;
+		h_path.dentry = au_h_dptr(dentry, bindex);
+		if (!h_path.dentry)
+			continue;
+
+		h_path.mnt = au_sbr_mnt(sb, bindex);
+		err = vfsub_fsync(NULL, &h_path, datasync);
+	}
+
+	return err;
+}
+
+static int au_do_fsync_dir(struct file *file, int datasync)
+{
+	int err;
+	aufs_bindex_t bbot, bindex;
+	struct file *h_file;
+	struct super_block *sb;
+	struct inode *inode;
+
+	err = au_reval_and_lock_fdi(file, reopen_dir, /*wlock*/1, /*fi_lsc*/0);
+	if (unlikely(err))
+		goto out;
+
+	inode = file_inode(file);
+	sb = inode->i_sb;
+	bbot = au_fbbot_dir(file);
+	for (bindex = au_fbtop(file); !err && bindex <= bbot; bindex++) {
+		h_file = au_hf_dir(file, bindex);
+		if (!h_file || au_test_ro(sb, bindex, inode))
+			continue;
+
+		err = vfsub_fsync(h_file, &h_file->f_path, datasync);
+	}
+
+out:
+	return err;
+}
+
+/*
+ * @file may be NULL
+ */
+static int aufs_fsync_dir(struct file *file, loff_t start, loff_t end,
+			  int datasync)
+{
+	int err;
+	struct dentry *dentry;
+	struct inode *inode;
+	struct super_block *sb;
+
+	err = 0;
+	dentry = file->f_path.dentry;
+	inode = d_inode(dentry);
+	inode_lock(inode);
+	sb = dentry->d_sb;
+	si_noflush_read_lock(sb);
+	if (file)
+		err = au_do_fsync_dir(file, datasync);
+	else {
+		di_write_lock_child(dentry);
+		err = au_do_fsync_dir_no_file(dentry, datasync);
+	}
+	au_cpup_attr_timesizes(inode);
+	di_write_unlock(dentry);
+	if (file)
+		fi_write_unlock(file);
+
+	si_read_unlock(sb);
+	inode_unlock(inode);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static int aufs_iterate_shared(struct file *file, struct dir_context *ctx)
+{
+	int err;
+	struct dentry *dentry;
+	struct inode *inode, *h_inode;
+	struct super_block *sb;
+
+	AuDbg("%pD, ctx{%ps, %llu}\n", file, ctx->actor, ctx->pos);
+
+	dentry = file->f_path.dentry;
+	inode = d_inode(dentry);
+	IMustLock(inode);
+
+	sb = dentry->d_sb;
+	si_read_lock(sb, AuLock_FLUSH);
+	err = au_reval_and_lock_fdi(file, reopen_dir, /*wlock*/1, /*fi_lsc*/0);
+	if (unlikely(err))
+		goto out;
+	err = au_alive_dir(dentry);
+	if (!err)
+		err = au_vdir_init(file);
+	di_downgrade_lock(dentry, AuLock_IR);
+	if (unlikely(err))
+		goto out_unlock;
+
+	h_inode = au_h_iptr(inode, au_ibtop(inode));
+	if (!au_test_nfsd()) {
+		err = au_vdir_fill_de(file, ctx);
+		fsstack_copy_attr_atime(inode, h_inode);
+	} else {
+		/*
+		 * nfsd filldir may call lookup_one_len(), vfs_getattr(),
+		 * encode_fh() and others.
+		 */
+		atomic_inc(&h_inode->i_count);
+		di_read_unlock(dentry, AuLock_IR);
+		si_read_unlock(sb);
+		err = au_vdir_fill_de(file, ctx);
+		fsstack_copy_attr_atime(inode, h_inode);
+		fi_write_unlock(file);
+		iput(h_inode);
+
+		AuTraceErr(err);
+		return err;
+	}
+
+out_unlock:
+	di_read_unlock(dentry, AuLock_IR);
+	fi_write_unlock(file);
+out:
+	si_read_unlock(sb);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+#define AuTestEmpty_WHONLY	1
+#define AuTestEmpty_CALLED	(1 << 1)
+#define AuTestEmpty_SHWH	(1 << 2)
+#define au_ftest_testempty(flags, name)	((flags) & AuTestEmpty_##name)
+#define au_fset_testempty(flags, name) \
+	do { (flags) |= AuTestEmpty_##name; } while (0)
+#define au_fclr_testempty(flags, name) \
+	do { (flags) &= ~AuTestEmpty_##name; } while (0)
+
+#ifndef CONFIG_AUFS_SHWH
+#undef AuTestEmpty_SHWH
+#define AuTestEmpty_SHWH	0
+#endif
+
+struct test_empty_arg {
+	struct dir_context ctx;
+	struct au_nhash *whlist;
+	unsigned int flags;
+	int err;
+	aufs_bindex_t bindex;
+};
+
+static int test_empty_cb(struct dir_context *ctx, const char *__name,
+			 int namelen, loff_t offset __maybe_unused, u64 ino,
+			 unsigned int d_type)
+{
+	struct test_empty_arg *arg = container_of(ctx, struct test_empty_arg,
+						  ctx);
+	char *name = (void *)__name;
+
+	arg->err = 0;
+	au_fset_testempty(arg->flags, CALLED);
+	/* smp_mb(); */
+	if (name[0] == '.'
+	    && (namelen == 1 || (name[1] == '.' && namelen == 2)))
+		goto out; /* success */
+
+	if (namelen <= AUFS_WH_PFX_LEN
+	    || memcmp(name, AUFS_WH_PFX, AUFS_WH_PFX_LEN)) {
+		if (au_ftest_testempty(arg->flags, WHONLY)
+		    && !au_nhash_test_known_wh(arg->whlist, name, namelen))
+			arg->err = -ENOTEMPTY;
+		goto out;
+	}
+
+	name += AUFS_WH_PFX_LEN;
+	namelen -= AUFS_WH_PFX_LEN;
+	if (!au_nhash_test_known_wh(arg->whlist, name, namelen))
+		arg->err = au_nhash_append_wh
+			(arg->whlist, name, namelen, ino, d_type, arg->bindex,
+			 au_ftest_testempty(arg->flags, SHWH));
+
+out:
+	/* smp_mb(); */
+	AuTraceErr(arg->err);
+	return arg->err;
+}
+
+static int do_test_empty(struct dentry *dentry, struct test_empty_arg *arg)
+{
+	int err;
+	struct file *h_file;
+	struct au_branch *br;
+
+	h_file = au_h_open(dentry, arg->bindex,
+			   O_RDONLY | O_NONBLOCK | O_DIRECTORY | O_LARGEFILE,
+			   /*file*/NULL, /*force_wr*/0);
+	err = PTR_ERR(h_file);
+	if (IS_ERR(h_file))
+		goto out;
+
+	err = 0;
+	if (!au_opt_test(au_mntflags(dentry->d_sb), UDBA_NONE)
+	    && !file_inode(h_file)->i_nlink)
+		goto out_put;
+
+	do {
+		arg->err = 0;
+		au_fclr_testempty(arg->flags, CALLED);
+		/* smp_mb(); */
+		err = vfsub_iterate_dir(h_file, &arg->ctx);
+		if (err >= 0)
+			err = arg->err;
+	} while (!err && au_ftest_testempty(arg->flags, CALLED));
+
+out_put:
+	fput(h_file);
+	br = au_sbr(dentry->d_sb, arg->bindex);
+	au_lcnt_dec(&br->br_nfiles);
+out:
+	return err;
+}
+
+struct do_test_empty_args {
+	int *errp;
+	struct dentry *dentry;
+	struct test_empty_arg *arg;
+};
+
+static void call_do_test_empty(void *args)
+{
+	struct do_test_empty_args *a = args;
+	*a->errp = do_test_empty(a->dentry, a->arg);
+}
+
+static int sio_test_empty(struct dentry *dentry, struct test_empty_arg *arg)
+{
+	int err, wkq_err;
+	struct dentry *h_dentry;
+	struct inode *h_inode;
+
+	h_dentry = au_h_dptr(dentry, arg->bindex);
+	h_inode = d_inode(h_dentry);
+	/* todo: i_mode changes anytime? */
+	inode_lock_shared_nested(h_inode, AuLsc_I_CHILD);
+	err = au_test_h_perm_sio(h_inode, MAY_EXEC | MAY_READ);
+	inode_unlock_shared(h_inode);
+	if (!err)
+		err = do_test_empty(dentry, arg);
+	else {
+		struct do_test_empty_args args = {
+			.errp	= &err,
+			.dentry	= dentry,
+			.arg	= arg
+		};
+		unsigned int flags = arg->flags;
+
+		wkq_err = au_wkq_wait(call_do_test_empty, &args);
+		if (unlikely(wkq_err))
+			err = wkq_err;
+		arg->flags = flags;
+	}
+
+	return err;
+}
+
+int au_test_empty_lower(struct dentry *dentry)
+{
+	int err;
+	unsigned int rdhash;
+	aufs_bindex_t bindex, btop, btail;
+	struct au_nhash whlist;
+	struct test_empty_arg arg = {
+		.ctx = {
+			.actor = test_empty_cb
+		}
+	};
+	int (*test_empty)(struct dentry *dentry, struct test_empty_arg *arg);
+
+	SiMustAnyLock(dentry->d_sb);
+
+	rdhash = au_sbi(dentry->d_sb)->si_rdhash;
+	if (!rdhash)
+		rdhash = au_rdhash_est(au_dir_size(/*file*/NULL, dentry));
+	err = au_nhash_alloc(&whlist, rdhash, GFP_NOFS);
+	if (unlikely(err))
+		goto out;
+
+	arg.flags = 0;
+	arg.whlist = &whlist;
+	btop = au_dbtop(dentry);
+	if (au_opt_test(au_mntflags(dentry->d_sb), SHWH))
+		au_fset_testempty(arg.flags, SHWH);
+	test_empty = do_test_empty;
+	if (au_opt_test(au_mntflags(dentry->d_sb), DIRPERM1))
+		test_empty = sio_test_empty;
+	arg.bindex = btop;
+	err = test_empty(dentry, &arg);
+	if (unlikely(err))
+		goto out_whlist;
+
+	au_fset_testempty(arg.flags, WHONLY);
+	btail = au_dbtaildir(dentry);
+	for (bindex = btop + 1; !err && bindex <= btail; bindex++) {
+		struct dentry *h_dentry;
+
+		h_dentry = au_h_dptr(dentry, bindex);
+		if (h_dentry && d_is_positive(h_dentry)) {
+			arg.bindex = bindex;
+			err = test_empty(dentry, &arg);
+		}
+	}
+
+out_whlist:
+	au_nhash_wh_free(&whlist);
+out:
+	return err;
+}
+
+int au_test_empty(struct dentry *dentry, struct au_nhash *whlist)
+{
+	int err;
+	struct test_empty_arg arg = {
+		.ctx = {
+			.actor = test_empty_cb
+		}
+	};
+	aufs_bindex_t bindex, btail;
+
+	err = 0;
+	arg.whlist = whlist;
+	arg.flags = AuTestEmpty_WHONLY;
+	if (au_opt_test(au_mntflags(dentry->d_sb), SHWH))
+		au_fset_testempty(arg.flags, SHWH);
+	btail = au_dbtaildir(dentry);
+	for (bindex = au_dbtop(dentry); !err && bindex <= btail; bindex++) {
+		struct dentry *h_dentry;
+
+		h_dentry = au_h_dptr(dentry, bindex);
+		if (h_dentry && d_is_positive(h_dentry)) {
+			arg.bindex = bindex;
+			err = sio_test_empty(dentry, &arg);
+		}
+	}
+
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+const struct file_operations aufs_dir_fop = {
+	.owner		= THIS_MODULE,
+	.llseek		= default_llseek,
+	.read		= generic_read_dir,
+	.iterate_shared	= aufs_iterate_shared,
+	.unlocked_ioctl	= aufs_ioctl_dir,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl	= aufs_compat_ioctl_dir,
+#endif
+	.open		= aufs_open_dir,
+	.release	= aufs_release_dir,
+	.flush		= aufs_flush_dir,
+	.fsync		= aufs_fsync_dir
+};
diff -Naur --no-dereference a/fs/aufs/dir.h b/fs/aufs/dir.h
--- a/fs/aufs/dir.h	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/dir.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,134 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * directory operations
+ */
+
+#ifndef __AUFS_DIR_H__
+#define __AUFS_DIR_H__
+
+#ifdef __KERNEL__
+
+#include <linux/fs.h>
+
+/* ---------------------------------------------------------------------- */
+
+/* need to be faster and smaller */
+
+struct au_nhash {
+	unsigned int		nh_num;
+	struct hlist_head	*nh_head;
+};
+
+struct au_vdir_destr {
+	unsigned char	len;
+	unsigned char	name[];
+} __packed;
+
+struct au_vdir_dehstr {
+	struct hlist_node	hash;
+	struct au_vdir_destr	*str;
+	struct rcu_head		rcu;
+} ____cacheline_aligned_in_smp;
+
+struct au_vdir_de {
+	ino_t			de_ino;
+	unsigned char		de_type;
+	/* caution: packed */
+	struct au_vdir_destr	de_str;
+} __packed;
+
+struct au_vdir_wh {
+	struct hlist_node	wh_hash;
+#ifdef CONFIG_AUFS_SHWH
+	ino_t			wh_ino;
+	aufs_bindex_t		wh_bindex;
+	unsigned char		wh_type;
+#else
+	aufs_bindex_t		wh_bindex;
+#endif
+	/* caution: packed */
+	struct au_vdir_destr	wh_str;
+} __packed;
+
+union au_vdir_deblk_p {
+	unsigned char		*deblk;
+	struct au_vdir_de	*de;
+};
+
+struct au_vdir {
+	unsigned char	**vd_deblk;
+	unsigned long	vd_nblk;
+	struct {
+		unsigned long		ul;
+		union au_vdir_deblk_p	p;
+	} vd_last;
+
+	u64		vd_version;
+	unsigned int	vd_deblk_sz;
+	unsigned long	vd_jiffy;
+	struct rcu_head	rcu;
+} ____cacheline_aligned_in_smp;
+
+/* ---------------------------------------------------------------------- */
+
+/* dir.c */
+extern const struct file_operations aufs_dir_fop;
+void au_add_nlink(struct inode *dir, struct inode *h_dir);
+void au_sub_nlink(struct inode *dir, struct inode *h_dir);
+loff_t au_dir_size(struct file *file, struct dentry *dentry);
+void au_dir_ts(struct inode *dir, aufs_bindex_t bsrc);
+int au_test_empty_lower(struct dentry *dentry);
+int au_test_empty(struct dentry *dentry, struct au_nhash *whlist);
+
+/* vdir.c */
+unsigned int au_rdhash_est(loff_t sz);
+int au_nhash_alloc(struct au_nhash *nhash, unsigned int num_hash, gfp_t gfp);
+void au_nhash_wh_free(struct au_nhash *whlist);
+int au_nhash_test_longer_wh(struct au_nhash *whlist, aufs_bindex_t btgt,
+			    int limit);
+int au_nhash_test_known_wh(struct au_nhash *whlist, char *name, int nlen);
+int au_nhash_append_wh(struct au_nhash *whlist, char *name, int nlen, ino_t ino,
+		       unsigned int d_type, aufs_bindex_t bindex,
+		       unsigned char shwh);
+void au_vdir_free(struct au_vdir *vdir);
+int au_vdir_init(struct file *file);
+int au_vdir_fill_de(struct file *file, struct dir_context *ctx);
+
+/* ioctl.c */
+long aufs_ioctl_dir(struct file *file, unsigned int cmd, unsigned long arg);
+
+#ifdef CONFIG_AUFS_RDU
+/* rdu.c */
+long au_rdu_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
+#ifdef CONFIG_COMPAT
+long au_rdu_compat_ioctl(struct file *file, unsigned int cmd,
+			 unsigned long arg);
+#endif
+#else
+AuStub(long, au_rdu_ioctl, return -EINVAL, struct file *file,
+       unsigned int cmd, unsigned long arg)
+#ifdef CONFIG_COMPAT
+AuStub(long, au_rdu_compat_ioctl, return -EINVAL, struct file *file,
+       unsigned int cmd, unsigned long arg)
+#endif
+#endif
+
+#endif /* __KERNEL__ */
+#endif /* __AUFS_DIR_H__ */
diff -Naur --no-dereference a/fs/aufs/dirren.c b/fs/aufs/dirren.c
--- a/fs/aufs/dirren.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/dirren.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,1316 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2017-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * special handling in renaming a directory
+ * in order to support looking-up the before-renamed name on the lower readonly
+ * branches
+ */
+
+#include <linux/byteorder/generic.h>
+#include "aufs.h"
+
+static void au_dr_hino_del(struct au_dr_br *dr, struct au_dr_hino *ent)
+{
+	int idx;
+
+	idx = au_dr_ihash(ent->dr_h_ino);
+	au_hbl_del(&ent->dr_hnode, dr->dr_h_ino + idx);
+}
+
+static int au_dr_hino_test_empty(struct au_dr_br *dr)
+{
+	int ret, i;
+	struct hlist_bl_head *hbl;
+
+	ret = 1;
+	for (i = 0; ret && i < AuDirren_NHASH; i++) {
+		hbl = dr->dr_h_ino + i;
+		hlist_bl_lock(hbl);
+		ret &= hlist_bl_empty(hbl);
+		hlist_bl_unlock(hbl);
+	}
+
+	return ret;
+}
+
+static struct au_dr_hino *au_dr_hino_find(struct au_dr_br *dr, ino_t ino)
+{
+	struct au_dr_hino *found, *ent;
+	struct hlist_bl_head *hbl;
+	struct hlist_bl_node *pos;
+	int idx;
+
+	found = NULL;
+	idx = au_dr_ihash(ino);
+	hbl = dr->dr_h_ino + idx;
+	hlist_bl_lock(hbl);
+	hlist_bl_for_each_entry(ent, pos, hbl, dr_hnode)
+		if (ent->dr_h_ino == ino) {
+			found = ent;
+			break;
+		}
+	hlist_bl_unlock(hbl);
+
+	return found;
+}
+
+int au_dr_hino_test_add(struct au_dr_br *dr, ino_t ino,
+			struct au_dr_hino *add_ent)
+{
+	int found, idx;
+	struct hlist_bl_head *hbl;
+	struct hlist_bl_node *pos;
+	struct au_dr_hino *ent;
+
+	found = 0;
+	idx = au_dr_ihash(ino);
+	hbl = dr->dr_h_ino + idx;
+#if 0 /* debug print */
+	{
+		struct hlist_bl_node *tmp;
+
+		hlist_bl_for_each_entry_safe(ent, pos, tmp, hbl, dr_hnode)
+			AuDbg("hi%llu\n", (unsigned long long)ent->dr_h_ino);
+	}
+#endif
+	hlist_bl_lock(hbl);
+	hlist_bl_for_each_entry(ent, pos, hbl, dr_hnode)
+		if (ent->dr_h_ino == ino) {
+			found = 1;
+			break;
+		}
+	if (!found && add_ent)
+		hlist_bl_add_head(&add_ent->dr_hnode, hbl);
+	hlist_bl_unlock(hbl);
+
+	if (!found && add_ent)
+		AuDbg("i%llu added\n", (unsigned long long)add_ent->dr_h_ino);
+
+	return found;
+}
+
+void au_dr_hino_free(struct au_dr_br *dr)
+{
+	int i;
+	struct hlist_bl_head *hbl;
+	struct hlist_bl_node *pos, *tmp;
+	struct au_dr_hino *ent;
+
+	/* SiMustWriteLock(sb); */
+
+	for (i = 0; i < AuDirren_NHASH; i++) {
+		hbl = dr->dr_h_ino + i;
+		/* no spinlock since sbinfo must be write-locked */
+		hlist_bl_for_each_entry_safe(ent, pos, tmp, hbl, dr_hnode)
+			au_kfree_rcu(ent);
+		INIT_HLIST_BL_HEAD(hbl);
+	}
+}
+
+/* returns the number of inodes or an error */
+static int au_dr_hino_store(struct super_block *sb, struct au_branch *br,
+			    struct file *hinofile)
+{
+	int err, i;
+	ssize_t ssz;
+	loff_t pos, oldsize;
+	__be64 u64;
+	struct inode *hinoinode;
+	struct hlist_bl_head *hbl;
+	struct hlist_bl_node *n1, *n2;
+	struct au_dr_hino *ent;
+
+	SiMustWriteLock(sb);
+	AuDebugOn(!au_br_writable(br->br_perm));
+
+	hinoinode = file_inode(hinofile);
+	oldsize = i_size_read(hinoinode);
+
+	err = 0;
+	pos = 0;
+	hbl = br->br_dirren.dr_h_ino;
+	for (i = 0; !err && i < AuDirren_NHASH; i++, hbl++) {
+		/* no bit-lock since sbinfo must be write-locked */
+		hlist_bl_for_each_entry_safe(ent, n1, n2, hbl, dr_hnode) {
+			AuDbg("hi%llu, %pD2\n",
+			      (unsigned long long)ent->dr_h_ino, hinofile);
+			u64 = cpu_to_be64(ent->dr_h_ino);
+			ssz = vfsub_write_k(hinofile, &u64, sizeof(u64), &pos);
+			if (ssz == sizeof(u64))
+				continue;
+
+			/* write error */
+			pr_err("ssz %zd, %pD2\n", ssz, hinofile);
+			err = -ENOSPC;
+			if (ssz < 0)
+				err = ssz;
+			break;
+		}
+	}
+	/* regardless the error */
+	if (pos < oldsize) {
+		err = vfsub_trunc(&hinofile->f_path, pos, /*attr*/0, hinofile);
+		AuTraceErr(err);
+	}
+
+	AuTraceErr(err);
+	return err;
+}
+
+static int au_dr_hino_load(struct au_dr_br *dr, struct file *hinofile)
+{
+	int err, hidx;
+	ssize_t ssz;
+	size_t sz, n;
+	loff_t pos;
+	uint64_t u64;
+	struct au_dr_hino *ent;
+	struct inode *hinoinode;
+	struct hlist_bl_head *hbl;
+
+	err = 0;
+	pos = 0;
+	hbl = dr->dr_h_ino;
+	hinoinode = file_inode(hinofile);
+	sz = i_size_read(hinoinode);
+	AuDebugOn(sz % sizeof(u64));
+	n = sz / sizeof(u64);
+	while (n--) {
+		ssz = vfsub_read_k(hinofile, &u64, sizeof(u64), &pos);
+		if (unlikely(ssz != sizeof(u64))) {
+			pr_err("ssz %zd, %pD2\n", ssz, hinofile);
+			err = -EINVAL;
+			if (ssz < 0)
+				err = ssz;
+			goto out_free;
+		}
+
+		ent = kmalloc(sizeof(*ent), GFP_NOFS);
+		if (!ent) {
+			err = -ENOMEM;
+			AuTraceErr(err);
+			goto out_free;
+		}
+		ent->dr_h_ino = be64_to_cpu((__force __be64)u64);
+		AuDbg("hi%llu, %pD2\n",
+		      (unsigned long long)ent->dr_h_ino, hinofile);
+		hidx = au_dr_ihash(ent->dr_h_ino);
+		au_hbl_add(&ent->dr_hnode, hbl + hidx);
+	}
+	goto out; /* success */
+
+out_free:
+	au_dr_hino_free(dr);
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+/*
+ * @bindex/@br is a switch to distinguish whether suspending hnotify or not.
+ * @path is a switch to distinguish load and store.
+ */
+static int au_dr_hino(struct super_block *sb, aufs_bindex_t bindex,
+		      struct au_branch *br, const struct path *path)
+{
+	int err, flags;
+	unsigned char load, suspend;
+	struct file *hinofile;
+	struct au_hinode *hdir;
+	struct inode *dir, *delegated;
+	struct path hinopath;
+	struct qstr hinoname = QSTR_INIT(AUFS_WH_DR_BRHINO,
+					 sizeof(AUFS_WH_DR_BRHINO) - 1);
+
+	AuDebugOn(bindex < 0 && !br);
+	AuDebugOn(bindex >= 0 && br);
+
+	err = -EINVAL;
+	suspend = !br;
+	if (suspend)
+		br = au_sbr(sb, bindex);
+	load = !!path;
+	if (!load) {
+		path = &br->br_path;
+		AuDebugOn(!au_br_writable(br->br_perm));
+		if (unlikely(!au_br_writable(br->br_perm)))
+			goto out;
+	}
+
+	hdir = NULL;
+	if (suspend) {
+		dir = d_inode(sb->s_root);
+		hdir = au_hinode(au_ii(dir), bindex);
+		dir = hdir->hi_inode;
+		au_hn_inode_lock_nested(hdir, AuLsc_I_CHILD);
+	} else {
+		dir = d_inode(path->dentry);
+		inode_lock_nested(dir, AuLsc_I_CHILD);
+	}
+	hinopath.dentry = vfsub_lkup_one(&hinoname, path->dentry);
+	err = PTR_ERR(hinopath.dentry);
+	if (IS_ERR(hinopath.dentry))
+		goto out_unlock;
+
+	err = 0;
+	flags = O_RDONLY;
+	if (load) {
+		if (d_is_negative(hinopath.dentry))
+			goto out_dput; /* success */
+	} else {
+		if (au_dr_hino_test_empty(&br->br_dirren)) {
+			if (d_is_positive(hinopath.dentry)) {
+				delegated = NULL;
+				err = vfsub_unlink(dir, &hinopath, &delegated,
+						   /*force*/0);
+				AuTraceErr(err);
+				if (unlikely(err))
+					pr_err("ignored err %d, %pd2\n",
+					       err, hinopath.dentry);
+				if (unlikely(err == -EWOULDBLOCK))
+					iput(delegated);
+				err = 0;
+			}
+			goto out_dput;
+		} else if (!d_is_positive(hinopath.dentry)) {
+			err = vfsub_create(dir, &hinopath, 0600,
+					   /*want_excl*/false);
+			AuTraceErr(err);
+			if (unlikely(err))
+				goto out_dput;
+		}
+		flags = O_WRONLY;
+	}
+	hinopath.mnt = path->mnt;
+	hinofile = vfsub_dentry_open(&hinopath, flags);
+	if (suspend)
+		au_hn_inode_unlock(hdir);
+	else
+		inode_unlock(dir);
+	dput(hinopath.dentry);
+	AuTraceErrPtr(hinofile);
+	if (IS_ERR(hinofile)) {
+		err = PTR_ERR(hinofile);
+		goto out;
+	}
+
+	if (load)
+		err = au_dr_hino_load(&br->br_dirren, hinofile);
+	else
+		err = au_dr_hino_store(sb, br, hinofile);
+	fput(hinofile);
+	goto out;
+
+out_dput:
+	dput(hinopath.dentry);
+out_unlock:
+	if (suspend)
+		au_hn_inode_unlock(hdir);
+	else
+		inode_unlock(dir);
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static int au_dr_brid_init(struct au_dr_brid *brid, const struct path *path)
+{
+	int err;
+	struct kstatfs kstfs;
+	dev_t dev;
+	struct dentry *dentry;
+	struct super_block *sb;
+
+	err = vfs_statfs((void *)path, &kstfs);
+	AuTraceErr(err);
+	if (unlikely(err))
+		goto out;
+
+	/* todo: support for UUID */
+
+	if (kstfs.f_fsid.val[0] || kstfs.f_fsid.val[1]) {
+		brid->type = AuBrid_FSID;
+		brid->fsid = kstfs.f_fsid;
+	} else {
+		dentry = path->dentry;
+		sb = dentry->d_sb;
+		dev = sb->s_dev;
+		if (dev) {
+			brid->type = AuBrid_DEV;
+			brid->dev = dev;
+		}
+	}
+
+out:
+	return err;
+}
+
+int au_dr_br_init(struct super_block *sb, struct au_branch *br,
+		  const struct path *path)
+{
+	int err, i;
+	struct au_dr_br *dr;
+	struct hlist_bl_head *hbl;
+
+	dr = &br->br_dirren;
+	hbl = dr->dr_h_ino;
+	for (i = 0; i < AuDirren_NHASH; i++, hbl++)
+		INIT_HLIST_BL_HEAD(hbl);
+
+	err = au_dr_brid_init(&dr->dr_brid, path);
+	if (unlikely(err))
+		goto out;
+
+	if (au_opt_test(au_mntflags(sb), DIRREN))
+		err = au_dr_hino(sb, /*bindex*/-1, br, path);
+
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+int au_dr_br_fin(struct super_block *sb, struct au_branch *br)
+{
+	int err;
+
+	err = 0;
+	if (au_br_writable(br->br_perm))
+		err = au_dr_hino(sb, /*bindex*/-1, br, /*path*/NULL);
+	if (!err)
+		au_dr_hino_free(&br->br_dirren);
+
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static int au_brid_str(struct au_dr_brid *brid, struct inode *h_inode,
+		       char *buf, size_t sz)
+{
+	int err;
+	unsigned int major, minor;
+	char *p;
+
+	p = buf;
+	err = snprintf(p, sz, "%d_", brid->type);
+	AuDebugOn(err > sz);
+	p += err;
+	sz -= err;
+	switch (brid->type) {
+	case AuBrid_Unset:
+		return -EINVAL;
+	case AuBrid_UUID:
+		err = snprintf(p, sz, "%pU", brid->uuid.b);
+		break;
+	case AuBrid_FSID:
+		err = snprintf(p, sz, "%08x-%08x",
+			       brid->fsid.val[0], brid->fsid.val[1]);
+		break;
+	case AuBrid_DEV:
+		major = MAJOR(brid->dev);
+		minor = MINOR(brid->dev);
+		if (major <= 0xff && minor <= 0xff)
+			err = snprintf(p, sz, "%02x%02x", major, minor);
+		else
+			err = snprintf(p, sz, "%03x:%05x", major, minor);
+		break;
+	}
+	AuDebugOn(err > sz);
+	p += err;
+	sz -= err;
+	err = snprintf(p, sz, "_%llu", (unsigned long long)h_inode->i_ino);
+	AuDebugOn(err > sz);
+	p += err;
+	sz -= err;
+
+	return p - buf;
+}
+
+static int au_drinfo_name(struct au_branch *br, char *name, int len)
+{
+	int rlen;
+	struct dentry *br_dentry;
+	struct inode *br_inode;
+
+	br_dentry = au_br_dentry(br);
+	br_inode = d_inode(br_dentry);
+	rlen = au_brid_str(&br->br_dirren.dr_brid, br_inode, name, len);
+	AuDebugOn(rlen >= AUFS_DIRREN_ENV_VAL_SZ);
+	AuDebugOn(rlen > len);
+
+	return rlen;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * from the given @h_dentry, construct drinfo at @*fdata.
+ * when the size of @*fdata is not enough, reallocate and return new @fdata and
+ * @allocated.
+ */
+static int au_drinfo_construct(struct au_drinfo_fdata **fdata,
+			       struct dentry *h_dentry,
+			       unsigned char *allocated)
+{
+	int err, v;
+	struct au_drinfo_fdata *f, *p;
+	struct au_drinfo *drinfo;
+	struct inode *h_inode;
+	struct qstr *qname;
+
+	err = 0;
+	f = *fdata;
+	h_inode = d_inode(h_dentry);
+	qname = &h_dentry->d_name;
+	drinfo = &f->drinfo;
+	drinfo->ino = (__force uint64_t)cpu_to_be64(h_inode->i_ino);
+	drinfo->oldnamelen = qname->len;
+	if (*allocated < sizeof(*f) + qname->len) {
+		v = roundup_pow_of_two(*allocated + qname->len);
+		p = au_krealloc(f, v, GFP_NOFS, /*may_shrink*/0);
+		if (unlikely(!p)) {
+			err = -ENOMEM;
+			AuTraceErr(err);
+			goto out;
+		}
+		f = p;
+		*fdata = f;
+		*allocated = v;
+		drinfo = &f->drinfo;
+	}
+	memcpy(drinfo->oldname, qname->name, qname->len);
+	AuDbg("i%llu, %.*s\n",
+	      be64_to_cpu((__force __be64)drinfo->ino), drinfo->oldnamelen,
+	      drinfo->oldname);
+
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+/* callers have to free the return value */
+static struct au_drinfo *au_drinfo_read_k(struct file *file, ino_t h_ino)
+{
+	struct au_drinfo *ret, *drinfo;
+	struct au_drinfo_fdata fdata;
+	int len;
+	loff_t pos;
+	ssize_t ssz;
+
+	ret = ERR_PTR(-EIO);
+	pos = 0;
+	ssz = vfsub_read_k(file, &fdata, sizeof(fdata), &pos);
+	if (unlikely(ssz != sizeof(fdata))) {
+		AuIOErr("ssz %zd, %u, %pD2\n",
+			ssz, (unsigned int)sizeof(fdata), file);
+		goto out;
+	}
+
+	fdata.magic = ntohl((__force __be32)fdata.magic);
+	switch (fdata.magic) {
+	case AUFS_DRINFO_MAGIC_V1:
+		break;
+	default:
+		AuIOErr("magic-num 0x%x, 0x%x, %pD2\n",
+			fdata.magic, AUFS_DRINFO_MAGIC_V1, file);
+		goto out;
+	}
+
+	drinfo = &fdata.drinfo;
+	len = drinfo->oldnamelen;
+	if (!len) {
+		AuIOErr("broken drinfo %pD2\n", file);
+		goto out;
+	}
+
+	ret = NULL;
+	drinfo->ino = be64_to_cpu((__force __be64)drinfo->ino);
+	if (unlikely(h_ino && drinfo->ino != h_ino)) {
+		AuDbg("ignored i%llu, i%llu, %pD2\n",
+		      (unsigned long long)drinfo->ino,
+		      (unsigned long long)h_ino, file);
+		goto out; /* success */
+	}
+
+	ret = kmalloc(sizeof(*ret) + len, GFP_NOFS);
+	if (unlikely(!ret)) {
+		ret = ERR_PTR(-ENOMEM);
+		AuTraceErrPtr(ret);
+		goto out;
+	}
+
+	*ret = *drinfo;
+	ssz = vfsub_read_k(file, (void *)ret->oldname, len, &pos);
+	if (unlikely(ssz != len)) {
+		au_kfree_rcu(ret);
+		ret = ERR_PTR(-EIO);
+		AuIOErr("ssz %zd, %u, %pD2\n", ssz, len, file);
+		goto out;
+	}
+
+	AuDbg("oldname %.*s\n", ret->oldnamelen, ret->oldname);
+
+out:
+	return ret;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* in order to be revertible */
+struct au_drinfo_rev_elm {
+	int			created;
+	struct dentry		*info_dentry;
+	struct au_drinfo	*info_last;
+};
+
+struct au_drinfo_rev {
+	unsigned char			already;
+	aufs_bindex_t			nelm;
+	struct au_drinfo_rev_elm	elm[];
+};
+
+/* todo: isn't it too large? */
+struct au_drinfo_store {
+	struct path h_ppath;
+	struct dentry *h_dentry;
+	struct au_drinfo_fdata *fdata;
+	char *infoname;			/* inside of whname, just after PFX */
+	char whname[sizeof(AUFS_WH_DR_INFO_PFX) + AUFS_DIRREN_ENV_VAL_SZ];
+	aufs_bindex_t btgt, btail;
+	unsigned char no_sio,
+		allocated,		/* current size of *fdata */
+		infonamelen,		/* room size for p */
+		whnamelen,		/* length of the generated name */
+		renameback;		/* renamed back */
+};
+
+/* on rename(2) error, the caller should revert it using @elm */
+static int au_drinfo_do_store(struct au_drinfo_store *w,
+			      struct au_drinfo_rev_elm *elm)
+{
+	int err, len;
+	ssize_t ssz;
+	loff_t pos;
+	struct path infopath = {
+		.mnt = w->h_ppath.mnt
+	};
+	struct inode *h_dir, *h_inode, *delegated;
+	struct file *infofile;
+	struct qstr *qname;
+
+	AuDebugOn(elm
+		  && memcmp(elm, page_address(ZERO_PAGE(0)), sizeof(*elm)));
+
+	infopath.dentry = vfsub_lookup_one_len(w->whname, w->h_ppath.dentry,
+					       w->whnamelen);
+	AuTraceErrPtr(infopath.dentry);
+	if (IS_ERR(infopath.dentry)) {
+		err = PTR_ERR(infopath.dentry);
+		goto out;
+	}
+
+	err = 0;
+	h_dir = d_inode(w->h_ppath.dentry);
+	if (elm && d_is_negative(infopath.dentry)) {
+		err = vfsub_create(h_dir, &infopath, 0600, /*want_excl*/true);
+		AuTraceErr(err);
+		if (unlikely(err))
+			goto out_dput;
+		elm->created = 1;
+		elm->info_dentry = dget(infopath.dentry);
+	}
+
+	infofile = vfsub_dentry_open(&infopath, O_RDWR);
+	AuTraceErrPtr(infofile);
+	if (IS_ERR(infofile)) {
+		err = PTR_ERR(infofile);
+		goto out_dput;
+	}
+
+	h_inode = d_inode(infopath.dentry);
+	if (elm && i_size_read(h_inode)) {
+		h_inode = d_inode(w->h_dentry);
+		elm->info_last = au_drinfo_read_k(infofile, h_inode->i_ino);
+		AuTraceErrPtr(elm->info_last);
+		if (IS_ERR(elm->info_last)) {
+			err = PTR_ERR(elm->info_last);
+			elm->info_last = NULL;
+			AuDebugOn(elm->info_dentry);
+			goto out_fput;
+		}
+	}
+
+	if (elm && w->renameback) {
+		delegated = NULL;
+		err = vfsub_unlink(h_dir, &infopath, &delegated, /*force*/0);
+		AuTraceErr(err);
+		if (unlikely(err == -EWOULDBLOCK))
+			iput(delegated);
+		goto out_fput;
+	}
+
+	pos = 0;
+	qname = &w->h_dentry->d_name;
+	len = sizeof(*w->fdata) + qname->len;
+	if (!elm)
+		len = sizeof(*w->fdata) + w->fdata->drinfo.oldnamelen;
+	ssz = vfsub_write_k(infofile, w->fdata, len, &pos);
+	if (ssz == len) {
+		AuDbg("hi%llu, %.*s\n", w->fdata->drinfo.ino,
+		      w->fdata->drinfo.oldnamelen, w->fdata->drinfo.oldname);
+		goto out_fput; /* success */
+	} else {
+		err = -EIO;
+		if (ssz < 0)
+			err = ssz;
+		/* the caller should revert it using @elm */
+	}
+
+out_fput:
+	fput(infofile);
+out_dput:
+	dput(infopath.dentry);
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+struct au_call_drinfo_do_store_args {
+	int *errp;
+	struct au_drinfo_store *w;
+	struct au_drinfo_rev_elm *elm;
+};
+
+static void au_call_drinfo_do_store(void *args)
+{
+	struct au_call_drinfo_do_store_args *a = args;
+
+	*a->errp = au_drinfo_do_store(a->w, a->elm);
+}
+
+static int au_drinfo_store_sio(struct au_drinfo_store *w,
+			       struct au_drinfo_rev_elm *elm)
+{
+	int err, wkq_err;
+
+	if (w->no_sio)
+		err = au_drinfo_do_store(w, elm);
+	else {
+		struct au_call_drinfo_do_store_args a = {
+			.errp	= &err,
+			.w	= w,
+			.elm	= elm
+		};
+		wkq_err = au_wkq_wait(au_call_drinfo_do_store, &a);
+		if (unlikely(wkq_err))
+			err = wkq_err;
+	}
+	AuTraceErr(err);
+
+	return err;
+}
+
+static int au_drinfo_store_work_init(struct au_drinfo_store *w,
+				     aufs_bindex_t btgt)
+{
+	int err;
+
+	memset(w, 0, sizeof(*w));
+	w->allocated = roundup_pow_of_two(sizeof(*w->fdata) + 40);
+	strcpy(w->whname, AUFS_WH_DR_INFO_PFX);
+	w->infoname = w->whname + sizeof(AUFS_WH_DR_INFO_PFX) - 1;
+	w->infonamelen = sizeof(w->whname) - sizeof(AUFS_WH_DR_INFO_PFX);
+	w->btgt = btgt;
+	w->no_sio = !!uid_eq(current_fsuid(), GLOBAL_ROOT_UID);
+
+	err = -ENOMEM;
+	w->fdata = kcalloc(1, w->allocated, GFP_NOFS);
+	if (unlikely(!w->fdata)) {
+		AuTraceErr(err);
+		goto out;
+	}
+	w->fdata->magic = (__force uint32_t)htonl(AUFS_DRINFO_MAGIC_V1);
+	err = 0;
+
+out:
+	return err;
+}
+
+static void au_drinfo_store_work_fin(struct au_drinfo_store *w)
+{
+	au_kfree_rcu(w->fdata);
+}
+
+static void au_drinfo_store_rev(struct au_drinfo_rev *rev,
+				struct au_drinfo_store *w)
+{
+	struct au_drinfo_rev_elm *elm;
+	struct inode *h_dir, *delegated;
+	int err, nelm;
+	struct path infopath = {
+		.mnt = w->h_ppath.mnt
+	};
+
+	h_dir = d_inode(w->h_ppath.dentry);
+	IMustLock(h_dir);
+
+	err = 0;
+	elm = rev->elm;
+	for (nelm = rev->nelm; nelm > 0; nelm--, elm++) {
+		AuDebugOn(elm->created && elm->info_last);
+		if (elm->created) {
+			AuDbg("here\n");
+			delegated = NULL;
+			infopath.dentry = elm->info_dentry;
+			err = vfsub_unlink(h_dir, &infopath, &delegated,
+					   !w->no_sio);
+			AuTraceErr(err);
+			if (unlikely(err == -EWOULDBLOCK))
+				iput(delegated);
+			dput(elm->info_dentry);
+		} else if (elm->info_last) {
+			AuDbg("here\n");
+			w->fdata->drinfo = *elm->info_last;
+			memcpy(w->fdata->drinfo.oldname,
+			       elm->info_last->oldname,
+			       elm->info_last->oldnamelen);
+			err = au_drinfo_store_sio(w, /*elm*/NULL);
+			au_kfree_rcu(elm->info_last);
+		}
+		if (unlikely(err))
+			AuIOErr("%d, %s\n", err, w->whname);
+		/* go on even if err */
+	}
+}
+
+/* caller has to call au_dr_rename_fin() later */
+static int au_drinfo_store(struct dentry *dentry, aufs_bindex_t btgt,
+			   struct qstr *dst_name, void *_rev)
+{
+	int err, sz, nelm;
+	aufs_bindex_t bindex, btail;
+	struct au_drinfo_store work;
+	struct au_drinfo_rev *rev, **p;
+	struct au_drinfo_rev_elm *elm;
+	struct super_block *sb;
+	struct au_branch *br;
+	struct au_hinode *hdir;
+
+	err = au_drinfo_store_work_init(&work, btgt);
+	AuTraceErr(err);
+	if (unlikely(err))
+		goto out;
+
+	err = -ENOMEM;
+	btail = au_dbtaildir(dentry);
+	nelm = btail - btgt;
+	sz = sizeof(*rev) + sizeof(*elm) * nelm;
+	rev = kcalloc(1, sz, GFP_NOFS);
+	if (unlikely(!rev)) {
+		AuTraceErr(err);
+		goto out_args;
+	}
+	rev->nelm = nelm;
+	elm = rev->elm;
+	p = _rev;
+	*p = rev;
+
+	err = 0;
+	sb = dentry->d_sb;
+	work.h_ppath.dentry = au_h_dptr(dentry, btgt);
+	work.h_ppath.mnt = au_sbr_mnt(sb, btgt);
+	hdir = au_hi(d_inode(dentry), btgt);
+	au_hn_inode_lock_nested(hdir, AuLsc_I_CHILD);
+	for (bindex = btgt + 1; bindex <= btail; bindex++, elm++) {
+		work.h_dentry = au_h_dptr(dentry, bindex);
+		if (!work.h_dentry)
+			continue;
+
+		err = au_drinfo_construct(&work.fdata, work.h_dentry,
+					  &work.allocated);
+		AuTraceErr(err);
+		if (unlikely(err))
+			break;
+
+		work.renameback = au_qstreq(&work.h_dentry->d_name, dst_name);
+		br = au_sbr(sb, bindex);
+		work.whnamelen = sizeof(AUFS_WH_DR_INFO_PFX) - 1;
+		work.whnamelen += au_drinfo_name(br, work.infoname,
+						 work.infonamelen);
+		AuDbg("whname %.*s, i%llu, %.*s\n",
+		      work.whnamelen, work.whname,
+		      be64_to_cpu((__force __be64)work.fdata->drinfo.ino),
+		      work.fdata->drinfo.oldnamelen,
+		      work.fdata->drinfo.oldname);
+
+		err = au_drinfo_store_sio(&work, elm);
+		AuTraceErr(err);
+		if (unlikely(err))
+			break;
+	}
+	if (unlikely(err)) {
+		/* revert all drinfo */
+		au_drinfo_store_rev(rev, &work);
+		au_kfree_try_rcu(rev);
+		*p = NULL;
+	}
+	au_hn_inode_unlock(hdir);
+
+out_args:
+	au_drinfo_store_work_fin(&work);
+out:
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+int au_dr_rename(struct dentry *src, aufs_bindex_t bindex,
+		 struct qstr *dst_name, void *_rev)
+{
+	int err, already;
+	ino_t ino;
+	struct super_block *sb;
+	struct au_branch *br;
+	struct au_dr_br *dr;
+	struct dentry *h_dentry;
+	struct inode *h_inode;
+	struct au_dr_hino *ent;
+	struct au_drinfo_rev *rev, **p;
+
+	AuDbg("bindex %d\n", bindex);
+
+	err = -ENOMEM;
+	ent = kmalloc(sizeof(*ent), GFP_NOFS);
+	if (unlikely(!ent))
+		goto out;
+
+	sb = src->d_sb;
+	br = au_sbr(sb, bindex);
+	dr = &br->br_dirren;
+	h_dentry = au_h_dptr(src, bindex);
+	h_inode = d_inode(h_dentry);
+	ino = h_inode->i_ino;
+	ent->dr_h_ino = ino;
+	already = au_dr_hino_test_add(dr, ino, ent);
+	AuDbg("b%d, hi%llu, already %d\n",
+	      bindex, (unsigned long long)ino, already);
+
+	err = au_drinfo_store(src, bindex, dst_name, _rev);
+	AuTraceErr(err);
+	if (!err) {
+		p = _rev;
+		rev = *p;
+		rev->already = already;
+		goto out; /* success */
+	}
+
+	/* revert */
+	if (!already)
+		au_dr_hino_del(dr, ent);
+	au_kfree_rcu(ent);
+
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+void au_dr_rename_fin(struct dentry *src, aufs_bindex_t btgt, void *_rev)
+{
+	struct au_drinfo_rev *rev;
+	struct au_drinfo_rev_elm *elm;
+	int nelm;
+
+	rev = _rev;
+	elm = rev->elm;
+	for (nelm = rev->nelm; nelm > 0; nelm--, elm++) {
+		dput(elm->info_dentry);
+		au_kfree_rcu(elm->info_last);
+	}
+	au_kfree_try_rcu(rev);
+}
+
+void au_dr_rename_rev(struct dentry *src, aufs_bindex_t btgt, void *_rev)
+{
+	int err;
+	struct au_drinfo_store work;
+	struct au_drinfo_rev *rev = _rev;
+	struct super_block *sb;
+	struct au_branch *br;
+	struct inode *h_inode;
+	struct au_dr_br *dr;
+	struct au_dr_hino *ent;
+
+	err = au_drinfo_store_work_init(&work, btgt);
+	if (unlikely(err))
+		goto out;
+
+	sb = src->d_sb;
+	br = au_sbr(sb, btgt);
+	work.h_ppath.dentry = au_h_dptr(src, btgt);
+	work.h_ppath.mnt = au_br_mnt(br);
+	au_drinfo_store_rev(rev, &work);
+	au_drinfo_store_work_fin(&work);
+	if (rev->already)
+		goto out;
+
+	dr = &br->br_dirren;
+	h_inode = d_inode(work.h_ppath.dentry);
+	ent = au_dr_hino_find(dr, h_inode->i_ino);
+	BUG_ON(!ent);
+	au_dr_hino_del(dr, ent);
+	au_kfree_rcu(ent);
+
+out:
+	au_kfree_try_rcu(rev);
+	if (unlikely(err))
+		pr_err("failed to remove dirren info\n");
+}
+
+/* ---------------------------------------------------------------------- */
+
+static struct au_drinfo *au_drinfo_do_load(struct path *h_ppath,
+					   char *whname, int whnamelen,
+					   struct dentry **info_dentry)
+{
+	struct au_drinfo *drinfo;
+	struct file *f;
+	struct inode *h_dir;
+	struct path infopath;
+	int unlocked;
+
+	AuDbg("%pd/%.*s\n", h_ppath->dentry, whnamelen, whname);
+
+	*info_dentry = NULL;
+	drinfo = NULL;
+	unlocked = 0;
+	h_dir = d_inode(h_ppath->dentry);
+	inode_lock_shared_nested(h_dir, AuLsc_I_PARENT);
+	infopath.dentry = vfsub_lookup_one_len(whname, h_ppath->dentry,
+					       whnamelen);
+	if (IS_ERR(infopath.dentry)) {
+		drinfo = (void *)infopath.dentry;
+		goto out;
+	}
+
+	if (d_is_negative(infopath.dentry))
+		goto out_dput; /* success */
+
+	infopath.mnt = h_ppath->mnt;
+	f = vfsub_dentry_open(&infopath, O_RDONLY);
+	inode_unlock_shared(h_dir);
+	unlocked = 1;
+	if (IS_ERR(f)) {
+		drinfo = (void *)f;
+		goto out_dput;
+	}
+
+	drinfo = au_drinfo_read_k(f, /*h_ino*/0);
+	if (IS_ERR_OR_NULL(drinfo))
+		goto out_fput;
+
+	AuDbg("oldname %.*s\n", drinfo->oldnamelen, drinfo->oldname);
+	*info_dentry = dget(infopath.dentry); /* keep it alive */
+
+out_fput:
+	fput(f);
+out_dput:
+	dput(infopath.dentry);
+out:
+	if (!unlocked)
+		inode_unlock_shared(h_dir);
+	AuTraceErrPtr(drinfo);
+	return drinfo;
+}
+
+struct au_drinfo_do_load_args {
+	struct au_drinfo **drinfop;
+	struct path *h_ppath;
+	char *whname;
+	int whnamelen;
+	struct dentry **info_dentry;
+};
+
+static void au_call_drinfo_do_load(void *args)
+{
+	struct au_drinfo_do_load_args *a = args;
+
+	*a->drinfop = au_drinfo_do_load(a->h_ppath, a->whname, a->whnamelen,
+					a->info_dentry);
+}
+
+struct au_drinfo_load {
+	struct path h_ppath;
+	struct qstr *qname;
+	unsigned char no_sio;
+
+	aufs_bindex_t ninfo;
+	struct au_drinfo **drinfo;
+};
+
+static int au_drinfo_load(struct au_drinfo_load *w, aufs_bindex_t bindex,
+			  struct au_branch *br)
+{
+	int err, wkq_err, whnamelen, e;
+	char whname[sizeof(AUFS_WH_DR_INFO_PFX) + AUFS_DIRREN_ENV_VAL_SZ]
+		= AUFS_WH_DR_INFO_PFX;
+	struct au_drinfo *drinfo;
+	struct qstr oldname;
+	struct inode *h_dir, *delegated;
+	struct dentry *info_dentry;
+	struct path infopath;
+
+	whnamelen = sizeof(AUFS_WH_DR_INFO_PFX) - 1;
+	whnamelen += au_drinfo_name(br, whname + whnamelen,
+				    sizeof(whname) - whnamelen);
+	if (w->no_sio)
+		drinfo = au_drinfo_do_load(&w->h_ppath, whname, whnamelen,
+					   &info_dentry);
+	else {
+		struct au_drinfo_do_load_args args = {
+			.drinfop	= &drinfo,
+			.h_ppath	= &w->h_ppath,
+			.whname		= whname,
+			.whnamelen	= whnamelen,
+			.info_dentry	= &info_dentry
+		};
+		wkq_err = au_wkq_wait(au_call_drinfo_do_load, &args);
+		if (unlikely(wkq_err))
+			drinfo = ERR_PTR(wkq_err);
+	}
+	err = PTR_ERR(drinfo);
+	if (IS_ERR_OR_NULL(drinfo))
+		goto out;
+
+	err = 0;
+	oldname.len = drinfo->oldnamelen;
+	oldname.name = drinfo->oldname;
+	if (au_qstreq(w->qname, &oldname)) {
+		/* the name is renamed back */
+		au_kfree_rcu(drinfo);
+		drinfo = NULL;
+
+		infopath.dentry = info_dentry;
+		infopath.mnt = w->h_ppath.mnt;
+		h_dir = d_inode(w->h_ppath.dentry);
+		delegated = NULL;
+		inode_lock_nested(h_dir, AuLsc_I_PARENT);
+		e = vfsub_unlink(h_dir, &infopath, &delegated, !w->no_sio);
+		inode_unlock(h_dir);
+		if (unlikely(e))
+			AuIOErr("ignored %d, %pd2\n", e, &infopath.dentry);
+		if (unlikely(e == -EWOULDBLOCK))
+			iput(delegated);
+	}
+	au_kfree_rcu(w->drinfo[bindex]);
+	w->drinfo[bindex] = drinfo;
+	dput(info_dentry);
+
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static void au_dr_lkup_free(struct au_drinfo **drinfo, int n)
+{
+	struct au_drinfo **p = drinfo;
+
+	while (n-- > 0)
+		au_kfree_rcu(*drinfo++);
+	au_kfree_try_rcu(p);
+}
+
+int au_dr_lkup(struct au_do_lookup_args *lkup, struct dentry *dentry,
+	       aufs_bindex_t btgt)
+{
+	int err, ninfo;
+	struct au_drinfo_load w;
+	aufs_bindex_t bindex, bbot;
+	struct au_branch *br;
+	struct inode *h_dir;
+	struct au_dr_hino *ent;
+	struct super_block *sb;
+
+	AuDbg("%.*s, name %.*s, whname %.*s, b%d\n",
+	      AuLNPair(&dentry->d_name), AuLNPair(&lkup->dirren.dr_name),
+	      AuLNPair(&lkup->whname), btgt);
+
+	sb = dentry->d_sb;
+	bbot = au_sbbot(sb);
+	w.ninfo = bbot + 1;
+	if (!lkup->dirren.drinfo) {
+		lkup->dirren.drinfo = kcalloc(w.ninfo,
+					      sizeof(*lkup->dirren.drinfo),
+					      GFP_NOFS);
+		if (unlikely(!lkup->dirren.drinfo)) {
+			err = -ENOMEM;
+			goto out;
+		}
+		lkup->dirren.ninfo = w.ninfo;
+	}
+	w.drinfo = lkup->dirren.drinfo;
+	w.no_sio = !!uid_eq(current_fsuid(), GLOBAL_ROOT_UID);
+	w.h_ppath.dentry = au_h_dptr(dentry, btgt);
+	AuDebugOn(!w.h_ppath.dentry);
+	w.h_ppath.mnt = au_sbr_mnt(sb, btgt);
+	w.qname = &dentry->d_name;
+
+	ninfo = 0;
+	for (bindex = btgt + 1; bindex <= bbot; bindex++) {
+		br = au_sbr(sb, bindex);
+		err = au_drinfo_load(&w, bindex, br);
+		if (unlikely(err))
+			goto out_free;
+		if (w.drinfo[bindex])
+			ninfo++;
+	}
+	if (!ninfo) {
+		br = au_sbr(sb, btgt);
+		h_dir = d_inode(w.h_ppath.dentry);
+		ent = au_dr_hino_find(&br->br_dirren, h_dir->i_ino);
+		AuDebugOn(!ent);
+		au_dr_hino_del(&br->br_dirren, ent);
+		au_kfree_rcu(ent);
+	}
+	goto out; /* success */
+
+out_free:
+	au_dr_lkup_free(lkup->dirren.drinfo, lkup->dirren.ninfo);
+	lkup->dirren.ninfo = 0;
+	lkup->dirren.drinfo = NULL;
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+void au_dr_lkup_fin(struct au_do_lookup_args *lkup)
+{
+	au_dr_lkup_free(lkup->dirren.drinfo, lkup->dirren.ninfo);
+}
+
+int au_dr_lkup_name(struct au_do_lookup_args *lkup, aufs_bindex_t btgt)
+{
+	int err;
+	struct au_drinfo *drinfo;
+
+	err = 0;
+	if (!lkup->dirren.drinfo)
+		goto out;
+	AuDebugOn(lkup->dirren.ninfo <= btgt);
+	drinfo = lkup->dirren.drinfo[btgt];
+	if (!drinfo)
+		goto out;
+
+	au_kfree_try_rcu(lkup->whname.name);
+	lkup->whname.name = NULL;
+	lkup->dirren.dr_name.len = drinfo->oldnamelen;
+	lkup->dirren.dr_name.name = drinfo->oldname;
+	lkup->name = &lkup->dirren.dr_name;
+	err = au_wh_name_alloc(&lkup->whname, lkup->name);
+	if (!err)
+		AuDbg("name %.*s, whname %.*s, b%d\n",
+		      AuLNPair(lkup->name), AuLNPair(&lkup->whname),
+		      btgt);
+
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+int au_dr_lkup_h_ino(struct au_do_lookup_args *lkup, aufs_bindex_t bindex,
+		     ino_t h_ino)
+{
+	int match;
+	struct au_drinfo *drinfo;
+
+	match = 1;
+	if (!lkup->dirren.drinfo)
+		goto out;
+	AuDebugOn(lkup->dirren.ninfo <= bindex);
+	drinfo = lkup->dirren.drinfo[bindex];
+	if (!drinfo)
+		goto out;
+
+	match = (drinfo->ino == h_ino);
+	AuDbg("match %d\n", match);
+
+out:
+	return match;
+}
+
+/* ---------------------------------------------------------------------- */
+
+int au_dr_opt_set(struct super_block *sb)
+{
+	int err;
+	aufs_bindex_t bindex, bbot;
+	struct au_branch *br;
+
+	err = 0;
+	bbot = au_sbbot(sb);
+	for (bindex = 0; !err && bindex <= bbot; bindex++) {
+		br = au_sbr(sb, bindex);
+		err = au_dr_hino(sb, bindex, /*br*/NULL, &br->br_path);
+	}
+
+	return err;
+}
+
+int au_dr_opt_flush(struct super_block *sb)
+{
+	int err;
+	aufs_bindex_t bindex, bbot;
+	struct au_branch *br;
+
+	err = 0;
+	bbot = au_sbbot(sb);
+	for (bindex = 0; !err && bindex <= bbot; bindex++) {
+		br = au_sbr(sb, bindex);
+		if (au_br_writable(br->br_perm))
+			err = au_dr_hino(sb, bindex, /*br*/NULL, /*path*/NULL);
+	}
+
+	return err;
+}
+
+int au_dr_opt_clr(struct super_block *sb, int no_flush)
+{
+	int err;
+	aufs_bindex_t bindex, bbot;
+	struct au_branch *br;
+
+	err = 0;
+	if (!no_flush) {
+		err = au_dr_opt_flush(sb);
+		if (unlikely(err))
+			goto out;
+	}
+
+	bbot = au_sbbot(sb);
+	for (bindex = 0; bindex <= bbot; bindex++) {
+		br = au_sbr(sb, bindex);
+		au_dr_hino_free(&br->br_dirren);
+	}
+
+out:
+	return err;
+}
diff -Naur --no-dereference a/fs/aufs/dirren.h b/fs/aufs/dirren.h
--- a/fs/aufs/dirren.h	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/dirren.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,140 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2017-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * renamed dir info
+ */
+
+#ifndef __AUFS_DIRREN_H__
+#define __AUFS_DIRREN_H__
+
+#ifdef __KERNEL__
+
+#include <linux/dcache.h>
+#include <linux/statfs.h>
+#include <linux/uuid.h>
+#include "hbl.h"
+
+#define AuDirren_NHASH 100
+
+#ifdef CONFIG_AUFS_DIRREN
+enum au_brid_type {
+	AuBrid_Unset,
+	AuBrid_UUID,
+	AuBrid_FSID,
+	AuBrid_DEV
+};
+
+struct au_dr_brid {
+	enum au_brid_type	type;
+	union {
+		uuid_t	uuid;	/* unimplemented yet */
+		fsid_t	fsid;
+		dev_t	dev;
+	};
+};
+
+/* 20 is the max digits length of ulong 64 */
+/* brid-type "_" uuid "_" inum */
+#define AUFS_DIRREN_FNAME_SZ	(1 + 1 + UUID_STRING_LEN + 20)
+#define AUFS_DIRREN_ENV_VAL_SZ	(AUFS_DIRREN_FNAME_SZ + 1 + 20)
+
+struct au_dr_hino {
+	struct hlist_bl_node	dr_hnode;
+	ino_t			dr_h_ino;
+};
+
+struct au_dr_br {
+	struct hlist_bl_head	dr_h_ino[AuDirren_NHASH];
+	struct au_dr_brid	dr_brid;
+};
+
+struct au_dr_lookup {
+	/* dr_name is pointed by struct au_do_lookup_args.name */
+	struct qstr		dr_name; /* subset of dr_info */
+	aufs_bindex_t		ninfo;
+	struct au_drinfo	**drinfo;
+};
+#else
+struct au_dr_hino;
+/* empty */
+struct au_dr_br { };
+struct au_dr_lookup { };
+#endif
+
+/* ---------------------------------------------------------------------- */
+
+struct au_branch;
+struct au_do_lookup_args;
+struct au_hinode;
+#ifdef CONFIG_AUFS_DIRREN
+int au_dr_hino_test_add(struct au_dr_br *dr, ino_t h_ino,
+			struct au_dr_hino *add_ent);
+void au_dr_hino_free(struct au_dr_br *dr);
+int au_dr_br_init(struct super_block *sb, struct au_branch *br,
+		  const struct path *path);
+int au_dr_br_fin(struct super_block *sb, struct au_branch *br);
+int au_dr_rename(struct dentry *src, aufs_bindex_t bindex,
+		 struct qstr *dst_name, void *_rev);
+void au_dr_rename_fin(struct dentry *src, aufs_bindex_t btgt, void *rev);
+void au_dr_rename_rev(struct dentry *src, aufs_bindex_t bindex, void *rev);
+int au_dr_lkup(struct au_do_lookup_args *lkup, struct dentry *dentry,
+	       aufs_bindex_t bindex);
+int au_dr_lkup_name(struct au_do_lookup_args *lkup, aufs_bindex_t btgt);
+int au_dr_lkup_h_ino(struct au_do_lookup_args *lkup, aufs_bindex_t bindex,
+		     ino_t h_ino);
+void au_dr_lkup_fin(struct au_do_lookup_args *lkup);
+int au_dr_opt_set(struct super_block *sb);
+int au_dr_opt_flush(struct super_block *sb);
+int au_dr_opt_clr(struct super_block *sb, int no_flush);
+#else
+AuStubInt0(au_dr_hino_test_add, struct au_dr_br *dr, ino_t h_ino,
+	   struct au_dr_hino *add_ent);
+AuStubVoid(au_dr_hino_free, struct au_dr_br *dr);
+AuStubInt0(au_dr_br_init, struct super_block *sb, struct au_branch *br,
+	   const struct path *path);
+AuStubInt0(au_dr_br_fin, struct super_block *sb, struct au_branch *br);
+AuStubInt0(au_dr_rename, struct dentry *src, aufs_bindex_t bindex,
+	   struct qstr *dst_name, void *_rev);
+AuStubVoid(au_dr_rename_fin, struct dentry *src, aufs_bindex_t btgt, void *rev);
+AuStubVoid(au_dr_rename_rev, struct dentry *src, aufs_bindex_t bindex,
+	   void *rev);
+AuStubInt0(au_dr_lkup, struct au_do_lookup_args *lkup, struct dentry *dentry,
+	   aufs_bindex_t bindex);
+AuStubInt0(au_dr_lkup_name, struct au_do_lookup_args *lkup, aufs_bindex_t btgt);
+AuStubInt0(au_dr_lkup_h_ino, struct au_do_lookup_args *lkup,
+	   aufs_bindex_t bindex, ino_t h_ino);
+AuStubVoid(au_dr_lkup_fin, struct au_do_lookup_args *lkup);
+AuStubInt0(au_dr_opt_set, struct super_block *sb);
+AuStubInt0(au_dr_opt_flush, struct super_block *sb);
+AuStubInt0(au_dr_opt_clr, struct super_block *sb, int no_flush);
+#endif
+
+/* ---------------------------------------------------------------------- */
+
+#ifdef CONFIG_AUFS_DIRREN
+static inline int au_dr_ihash(ino_t h_ino)
+{
+	return h_ino % AuDirren_NHASH;
+}
+#else
+AuStubInt0(au_dr_ihash, ino_t h_ino);
+#endif
+
+#endif /* __KERNEL__ */
+#endif /* __AUFS_DIRREN_H__ */
diff -Naur --no-dereference a/fs/aufs/dynop.c b/fs/aufs/dynop.c
--- a/fs/aufs/dynop.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/dynop.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,370 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2010-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * dynamically customizable operations for regular files
+ */
+
+#include "aufs.h"
+
+#define DyPrSym(key)	AuDbgSym(key->dk_op.dy_hop)
+
+/*
+ * How large will these lists be?
+ * Usually just a few elements, 20-30 at most for each, I guess.
+ */
+static struct hlist_bl_head dynop[AuDyLast];
+
+static struct au_dykey *dy_gfind_get(struct hlist_bl_head *hbl,
+				     const void *h_op)
+{
+	struct au_dykey *key, *tmp;
+	struct hlist_bl_node *pos;
+
+	key = NULL;
+	hlist_bl_lock(hbl);
+	hlist_bl_for_each_entry(tmp, pos, hbl, dk_hnode)
+		if (tmp->dk_op.dy_hop == h_op) {
+			if (kref_get_unless_zero(&tmp->dk_kref))
+				key = tmp;
+			break;
+		}
+	hlist_bl_unlock(hbl);
+
+	return key;
+}
+
+static struct au_dykey *dy_bradd(struct au_branch *br, struct au_dykey *key)
+{
+	struct au_dykey **k, *found;
+	const void *h_op = key->dk_op.dy_hop;
+	int i;
+
+	found = NULL;
+	k = br->br_dykey;
+	for (i = 0; i < AuBrDynOp; i++)
+		if (k[i]) {
+			if (k[i]->dk_op.dy_hop == h_op) {
+				found = k[i];
+				break;
+			}
+		} else
+			break;
+	if (!found) {
+		spin_lock(&br->br_dykey_lock);
+		for (; i < AuBrDynOp; i++)
+			if (k[i]) {
+				if (k[i]->dk_op.dy_hop == h_op) {
+					found = k[i];
+					break;
+				}
+			} else {
+				k[i] = key;
+				break;
+			}
+		spin_unlock(&br->br_dykey_lock);
+		BUG_ON(i == AuBrDynOp); /* expand the array */
+	}
+
+	return found;
+}
+
+/* kref_get() if @key is already added */
+static struct au_dykey *dy_gadd(struct hlist_bl_head *hbl, struct au_dykey *key)
+{
+	struct au_dykey *tmp, *found;
+	struct hlist_bl_node *pos;
+	const void *h_op = key->dk_op.dy_hop;
+
+	found = NULL;
+	hlist_bl_lock(hbl);
+	hlist_bl_for_each_entry(tmp, pos, hbl, dk_hnode)
+		if (tmp->dk_op.dy_hop == h_op) {
+			if (kref_get_unless_zero(&tmp->dk_kref))
+				found = tmp;
+			break;
+		}
+	if (!found)
+		hlist_bl_add_head(&key->dk_hnode, hbl);
+	hlist_bl_unlock(hbl);
+
+	if (!found)
+		DyPrSym(key);
+	return found;
+}
+
+static void dy_free_rcu(struct rcu_head *rcu)
+{
+	struct au_dykey *key;
+
+	key = container_of(rcu, struct au_dykey, dk_rcu);
+	DyPrSym(key);
+	kfree(key);
+}
+
+static void dy_free(struct kref *kref)
+{
+	struct au_dykey *key;
+	struct hlist_bl_head *hbl;
+
+	key = container_of(kref, struct au_dykey, dk_kref);
+	hbl = dynop + key->dk_op.dy_type;
+	au_hbl_del(&key->dk_hnode, hbl);
+	call_rcu(&key->dk_rcu, dy_free_rcu);
+}
+
+void au_dy_put(struct au_dykey *key)
+{
+	kref_put(&key->dk_kref, dy_free);
+}
+
+/* ---------------------------------------------------------------------- */
+
+#define DyDbgSize(cnt, op)	AuDebugOn(cnt != sizeof(op)/sizeof(void *))
+
+#ifdef CONFIG_AUFS_DEBUG
+#define DyDbgDeclare(cnt)	unsigned int cnt = 0
+#define DyDbgInc(cnt)		do { cnt++; } while (0)
+#else
+#define DyDbgDeclare(cnt)	do {} while (0)
+#define DyDbgInc(cnt)		do {} while (0)
+#endif
+
+#define DySet(func, dst, src, h_op, h_sb) do {				\
+	DyDbgInc(cnt);							\
+	if (h_op->func) {						\
+		if (src.func)						\
+			dst.func = src.func;				\
+		else							\
+			AuDbg("%s %s\n", au_sbtype(h_sb), #func);	\
+	}								\
+} while (0)
+
+#define DySetForce(func, dst, src) do {		\
+	AuDebugOn(!src.func);			\
+	DyDbgInc(cnt);				\
+	dst.func = src.func;			\
+} while (0)
+
+#define DySetAop(func) \
+	DySet(func, dyaop->da_op, aufs_aop, h_aop, h_sb)
+#define DySetAopForce(func) \
+	DySetForce(func, dyaop->da_op, aufs_aop)
+
+static void dy_aop(struct au_dykey *key, const void *h_op,
+		   struct super_block *h_sb __maybe_unused)
+{
+	struct au_dyaop *dyaop = (void *)key;
+	const struct address_space_operations *h_aop = h_op;
+	DyDbgDeclare(cnt);
+
+	AuDbg("%s\n", au_sbtype(h_sb));
+
+	DySetAop(writepage);
+	DySetAopForce(readpage);	/* force */
+	DySetAop(writepages);
+	DySetAop(set_page_dirty);
+	DySetAop(readpages);
+	DySetAop(write_begin);
+	DySetAop(write_end);
+	DySetAop(bmap);
+	DySetAop(invalidatepage);
+	DySetAop(releasepage);
+	DySetAop(freepage);
+	/* this one will be changed according to an aufs mount option */
+	DySetAop(direct_IO);
+	DySetAop(migratepage);
+	DySetAop(isolate_page);
+	DySetAop(putback_page);
+	DySetAop(launder_page);
+	DySetAop(is_partially_uptodate);
+	DySetAop(is_dirty_writeback);
+	DySetAop(error_remove_page);
+	DySetAop(swap_activate);
+	DySetAop(swap_deactivate);
+
+	DyDbgSize(cnt, *h_aop);
+}
+
+/* ---------------------------------------------------------------------- */
+
+static void dy_bug(struct kref *kref)
+{
+	BUG();
+}
+
+static struct au_dykey *dy_get(struct au_dynop *op, struct au_branch *br)
+{
+	struct au_dykey *key, *old;
+	struct hlist_bl_head *hbl;
+	struct op {
+		unsigned int sz;
+		void (*set)(struct au_dykey *key, const void *h_op,
+			    struct super_block *h_sb __maybe_unused);
+	};
+	static const struct op a[] = {
+		[AuDy_AOP] = {
+			.sz	= sizeof(struct au_dyaop),
+			.set	= dy_aop
+		}
+	};
+	const struct op *p;
+
+	hbl = dynop + op->dy_type;
+	key = dy_gfind_get(hbl, op->dy_hop);
+	if (key)
+		goto out_add; /* success */
+
+	p = a + op->dy_type;
+	key = kzalloc(p->sz, GFP_NOFS);
+	if (unlikely(!key)) {
+		key = ERR_PTR(-ENOMEM);
+		goto out;
+	}
+
+	key->dk_op.dy_hop = op->dy_hop;
+	kref_init(&key->dk_kref);
+	p->set(key, op->dy_hop, au_br_sb(br));
+	old = dy_gadd(hbl, key);
+	if (old) {
+		au_kfree_rcu(key);
+		key = old;
+	}
+
+out_add:
+	old = dy_bradd(br, key);
+	if (old)
+		/* its ref-count should never be zero here */
+		kref_put(&key->dk_kref, dy_bug);
+out:
+	return key;
+}
+
+/* ---------------------------------------------------------------------- */
+/*
+ * Aufs prohibits O_DIRECT by default even if the branch supports it.
+ * This behaviour is necessary to return an error from open(O_DIRECT) instead
+ * of the succeeding I/O. The dio mount option enables O_DIRECT and makes
+ * open(O_DIRECT) always succeed, but the succeeding I/O may return an error.
+ * See the aufs manual in detail.
+ */
+static void dy_adx(struct au_dyaop *dyaop, int do_dx)
+{
+	if (!do_dx)
+		dyaop->da_op.direct_IO = NULL;
+	else
+		dyaop->da_op.direct_IO = aufs_aop.direct_IO;
+}
+
+static struct au_dyaop *dy_aget(struct au_branch *br,
+				const struct address_space_operations *h_aop,
+				int do_dx)
+{
+	struct au_dyaop *dyaop;
+	struct au_dynop op;
+
+	op.dy_type = AuDy_AOP;
+	op.dy_haop = h_aop;
+	dyaop = (void *)dy_get(&op, br);
+	if (IS_ERR(dyaop))
+		goto out;
+	dy_adx(dyaop, do_dx);
+
+out:
+	return dyaop;
+}
+
+int au_dy_iaop(struct inode *inode, aufs_bindex_t bindex,
+		struct inode *h_inode)
+{
+	int err, do_dx;
+	struct super_block *sb;
+	struct au_branch *br;
+	struct au_dyaop *dyaop;
+
+	AuDebugOn(!S_ISREG(h_inode->i_mode));
+	IiMustWriteLock(inode);
+
+	sb = inode->i_sb;
+	br = au_sbr(sb, bindex);
+	do_dx = !!au_opt_test(au_mntflags(sb), DIO);
+	dyaop = dy_aget(br, h_inode->i_mapping->a_ops, do_dx);
+	err = PTR_ERR(dyaop);
+	if (IS_ERR(dyaop))
+		/* unnecessary to call dy_fput() */
+		goto out;
+
+	err = 0;
+	inode->i_mapping->a_ops = &dyaop->da_op;
+
+out:
+	return err;
+}
+
+/*
+ * Is it safe to replace a_ops during the inode/file is in operation?
+ * Yes, I hope so.
+ */
+int au_dy_irefresh(struct inode *inode)
+{
+	int err;
+	aufs_bindex_t btop;
+	struct inode *h_inode;
+
+	err = 0;
+	if (S_ISREG(inode->i_mode)) {
+		btop = au_ibtop(inode);
+		h_inode = au_h_iptr(inode, btop);
+		err = au_dy_iaop(inode, btop, h_inode);
+	}
+	return err;
+}
+
+void au_dy_arefresh(int do_dx)
+{
+	struct hlist_bl_head *hbl;
+	struct hlist_bl_node *pos;
+	struct au_dykey *key;
+
+	hbl = dynop + AuDy_AOP;
+	hlist_bl_lock(hbl);
+	hlist_bl_for_each_entry(key, pos, hbl, dk_hnode)
+		dy_adx((void *)key, do_dx);
+	hlist_bl_unlock(hbl);
+}
+
+/* ---------------------------------------------------------------------- */
+
+void __init au_dy_init(void)
+{
+	int i;
+
+	/* make sure that 'struct au_dykey *' can be any type */
+	BUILD_BUG_ON(offsetof(struct au_dyaop, da_key));
+
+	for (i = 0; i < AuDyLast; i++)
+		INIT_HLIST_BL_HEAD(dynop + i);
+}
+
+void au_dy_fin(void)
+{
+	int i;
+
+	for (i = 0; i < AuDyLast; i++)
+		WARN_ON(!hlist_bl_empty(dynop + i));
+}
diff -Naur --no-dereference a/fs/aufs/dynop.h b/fs/aufs/dynop.h
--- a/fs/aufs/dynop.h	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/dynop.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,75 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2010-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * dynamically customizable operations (for regular files only)
+ */
+
+#ifndef __AUFS_DYNOP_H__
+#define __AUFS_DYNOP_H__
+
+#ifdef __KERNEL__
+
+#include <linux/fs.h>
+#include <linux/kref.h>
+
+enum {AuDy_AOP, AuDyLast};
+
+struct au_dynop {
+	int						dy_type;
+	union {
+		const void				*dy_hop;
+		const struct address_space_operations	*dy_haop;
+	};
+};
+
+struct au_dykey {
+	union {
+		struct hlist_bl_node	dk_hnode;
+		struct rcu_head		dk_rcu;
+	};
+	struct au_dynop		dk_op;
+
+	/*
+	 * during I am in the branch local array, kref is gotten. when the
+	 * branch is removed, kref is put.
+	 */
+	struct kref		dk_kref;
+};
+
+/* stop unioning since their sizes are very different from each other */
+struct au_dyaop {
+	struct au_dykey			da_key;
+	struct address_space_operations	da_op; /* not const */
+};
+
+/* ---------------------------------------------------------------------- */
+
+/* dynop.c */
+struct au_branch;
+void au_dy_put(struct au_dykey *key);
+int au_dy_iaop(struct inode *inode, aufs_bindex_t bindex,
+		struct inode *h_inode);
+int au_dy_irefresh(struct inode *inode);
+void au_dy_arefresh(int do_dio);
+
+void __init au_dy_init(void);
+void au_dy_fin(void);
+
+#endif /* __KERNEL__ */
+#endif /* __AUFS_DYNOP_H__ */
diff -Naur --no-dereference a/fs/aufs/export.c b/fs/aufs/export.c
--- a/fs/aufs/export.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/export.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,838 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * export via nfs
+ */
+
+#include <linux/exportfs.h>
+#include <linux/fs_struct.h>
+#include <linux/namei.h>
+#include <linux/nsproxy.h>
+#include <linux/random.h>
+#include <linux/writeback.h>
+#include "aufs.h"
+
+union conv {
+#ifdef CONFIG_AUFS_INO_T_64
+	__u32 a[2];
+#else
+	__u32 a[1];
+#endif
+	ino_t ino;
+};
+
+static ino_t decode_ino(__u32 *a)
+{
+	union conv u;
+
+	BUILD_BUG_ON(sizeof(u.ino) != sizeof(u.a));
+	u.a[0] = a[0];
+#ifdef CONFIG_AUFS_INO_T_64
+	u.a[1] = a[1];
+#endif
+	return u.ino;
+}
+
+static void encode_ino(__u32 *a, ino_t ino)
+{
+	union conv u;
+
+	u.ino = ino;
+	a[0] = u.a[0];
+#ifdef CONFIG_AUFS_INO_T_64
+	a[1] = u.a[1];
+#endif
+}
+
+/* NFS file handle */
+enum {
+	Fh_br_id,
+	Fh_sigen,
+#ifdef CONFIG_AUFS_INO_T_64
+	/* support 64bit inode number */
+	Fh_ino1,
+	Fh_ino2,
+	Fh_dir_ino1,
+	Fh_dir_ino2,
+#else
+	Fh_ino1,
+	Fh_dir_ino1,
+#endif
+	Fh_igen,
+	Fh_h_type,
+	Fh_tail,
+
+	Fh_ino = Fh_ino1,
+	Fh_dir_ino = Fh_dir_ino1
+};
+
+static int au_test_anon(struct dentry *dentry)
+{
+	/* note: read d_flags without d_lock */
+	return !!(dentry->d_flags & DCACHE_DISCONNECTED);
+}
+
+int au_test_nfsd(void)
+{
+	int ret;
+	struct task_struct *tsk = current;
+	char comm[sizeof(tsk->comm)];
+
+	ret = 0;
+	if (tsk->flags & PF_KTHREAD) {
+		get_task_comm(comm, tsk);
+		ret = !strcmp(comm, "nfsd");
+	}
+
+	return ret;
+}
+
+/* ---------------------------------------------------------------------- */
+/* inode generation external table */
+
+void au_xigen_inc(struct inode *inode)
+{
+	loff_t pos;
+	ssize_t sz;
+	__u32 igen;
+	struct super_block *sb;
+	struct au_sbinfo *sbinfo;
+
+	sb = inode->i_sb;
+	AuDebugOn(!au_opt_test(au_mntflags(sb), XINO));
+
+	sbinfo = au_sbi(sb);
+	pos = inode->i_ino;
+	pos *= sizeof(igen);
+	igen = inode->i_generation + 1;
+	sz = xino_fwrite(sbinfo->si_xwrite, sbinfo->si_xigen, &igen,
+			 sizeof(igen), &pos);
+	if (sz == sizeof(igen))
+		return; /* success */
+
+	if (unlikely(sz >= 0))
+		AuIOErr("xigen error (%zd)\n", sz);
+}
+
+int au_xigen_new(struct inode *inode)
+{
+	int err;
+	loff_t pos;
+	ssize_t sz;
+	struct super_block *sb;
+	struct au_sbinfo *sbinfo;
+	struct file *file;
+
+	err = 0;
+	/* todo: dirty, at mount time */
+	if (inode->i_ino == AUFS_ROOT_INO)
+		goto out;
+	sb = inode->i_sb;
+	SiMustAnyLock(sb);
+	if (unlikely(!au_opt_test(au_mntflags(sb), XINO)))
+		goto out;
+
+	err = -EFBIG;
+	pos = inode->i_ino;
+	if (unlikely(au_loff_max / sizeof(inode->i_generation) - 1 < pos)) {
+		AuIOErr1("too large i%lld\n", pos);
+		goto out;
+	}
+	pos *= sizeof(inode->i_generation);
+
+	err = 0;
+	sbinfo = au_sbi(sb);
+	file = sbinfo->si_xigen;
+	BUG_ON(!file);
+
+	if (vfsub_f_size_read(file)
+	    < pos + sizeof(inode->i_generation)) {
+		inode->i_generation = atomic_inc_return(&sbinfo->si_xigen_next);
+		sz = xino_fwrite(sbinfo->si_xwrite, file, &inode->i_generation,
+				 sizeof(inode->i_generation), &pos);
+	} else
+		sz = xino_fread(sbinfo->si_xread, file, &inode->i_generation,
+				sizeof(inode->i_generation), &pos);
+	if (sz == sizeof(inode->i_generation))
+		goto out; /* success */
+
+	err = sz;
+	if (unlikely(sz >= 0)) {
+		err = -EIO;
+		AuIOErr("xigen error (%zd)\n", sz);
+	}
+
+out:
+	return err;
+}
+
+int au_xigen_set(struct super_block *sb, struct path *path)
+{
+	int err;
+	struct au_sbinfo *sbinfo;
+	struct file *file;
+
+	SiMustWriteLock(sb);
+
+	sbinfo = au_sbi(sb);
+	file = au_xino_create2(sb, path, sbinfo->si_xigen);
+	err = PTR_ERR(file);
+	if (IS_ERR(file))
+		goto out;
+	err = 0;
+	if (sbinfo->si_xigen)
+		fput(sbinfo->si_xigen);
+	sbinfo->si_xigen = file;
+
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+void au_xigen_clr(struct super_block *sb)
+{
+	struct au_sbinfo *sbinfo;
+
+	SiMustWriteLock(sb);
+
+	sbinfo = au_sbi(sb);
+	if (sbinfo->si_xigen) {
+		fput(sbinfo->si_xigen);
+		sbinfo->si_xigen = NULL;
+	}
+}
+
+/* ---------------------------------------------------------------------- */
+
+static struct dentry *decode_by_ino(struct super_block *sb, ino_t ino,
+				    ino_t dir_ino)
+{
+	struct dentry *dentry, *d;
+	struct inode *inode;
+	unsigned int sigen;
+
+	dentry = NULL;
+	inode = ilookup(sb, ino);
+	if (!inode)
+		goto out;
+
+	dentry = ERR_PTR(-ESTALE);
+	sigen = au_sigen(sb);
+	if (unlikely(au_is_bad_inode(inode)
+		     || IS_DEADDIR(inode)
+		     || sigen != au_iigen(inode, NULL)))
+		goto out_iput;
+
+	dentry = NULL;
+	if (!dir_ino || S_ISDIR(inode->i_mode))
+		dentry = d_find_alias(inode);
+	else {
+		spin_lock(&inode->i_lock);
+		hlist_for_each_entry(d, &inode->i_dentry, d_u.d_alias) {
+			spin_lock(&d->d_lock);
+			if (!au_test_anon(d)
+			    && d_inode(d->d_parent)->i_ino == dir_ino) {
+				dentry = dget_dlock(d);
+				spin_unlock(&d->d_lock);
+				break;
+			}
+			spin_unlock(&d->d_lock);
+		}
+		spin_unlock(&inode->i_lock);
+	}
+	if (unlikely(dentry && au_digen_test(dentry, sigen))) {
+		/* need to refresh */
+		dput(dentry);
+		dentry = NULL;
+	}
+
+out_iput:
+	iput(inode);
+out:
+	AuTraceErrPtr(dentry);
+	return dentry;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* todo: dirty? */
+/* if exportfs_decode_fh() passed vfsmount*, we could be happy */
+
+struct au_compare_mnt_args {
+	/* input */
+	struct super_block *sb;
+
+	/* output */
+	struct vfsmount *mnt;
+};
+
+static int au_compare_mnt(struct vfsmount *mnt, void *arg)
+{
+	struct au_compare_mnt_args *a = arg;
+
+	if (mnt->mnt_sb != a->sb)
+		return 0;
+	a->mnt = mntget(mnt);
+	return 1;
+}
+
+static struct vfsmount *au_mnt_get(struct super_block *sb)
+{
+	int err;
+	struct path root;
+	struct au_compare_mnt_args args = {
+		.sb = sb
+	};
+
+	get_fs_root(current->fs, &root);
+	rcu_read_lock();
+	err = iterate_mounts(au_compare_mnt, &args, root.mnt);
+	rcu_read_unlock();
+	path_put(&root);
+	AuDebugOn(!err);
+	AuDebugOn(!args.mnt);
+	return args.mnt;
+}
+
+struct au_nfsd_si_lock {
+	unsigned int sigen;
+	aufs_bindex_t bindex, br_id;
+	unsigned char force_lock;
+};
+
+static int si_nfsd_read_lock(struct super_block *sb,
+			     struct au_nfsd_si_lock *nsi_lock)
+{
+	int err;
+	aufs_bindex_t bindex;
+
+	si_read_lock(sb, AuLock_FLUSH);
+
+	/* branch id may be wrapped around */
+	err = 0;
+	bindex = au_br_index(sb, nsi_lock->br_id);
+	if (bindex >= 0 && nsi_lock->sigen + AUFS_BRANCH_MAX > au_sigen(sb))
+		goto out; /* success */
+
+	err = -ESTALE;
+	bindex = -1;
+	if (!nsi_lock->force_lock)
+		si_read_unlock(sb);
+
+out:
+	nsi_lock->bindex = bindex;
+	return err;
+}
+
+struct find_name_by_ino {
+	struct dir_context ctx;
+	int called, found;
+	ino_t ino;
+	char *name;
+	int namelen;
+};
+
+static int
+find_name_by_ino(struct dir_context *ctx, const char *name, int namelen,
+		 loff_t offset, u64 ino, unsigned int d_type)
+{
+	struct find_name_by_ino *a = container_of(ctx, struct find_name_by_ino,
+						  ctx);
+
+	a->called++;
+	if (a->ino != ino)
+		return 0;
+
+	memcpy(a->name, name, namelen);
+	a->namelen = namelen;
+	a->found = 1;
+	return 1;
+}
+
+static struct dentry *au_lkup_by_ino(struct path *path, ino_t ino,
+				     struct au_nfsd_si_lock *nsi_lock)
+{
+	struct dentry *dentry, *parent;
+	struct file *file;
+	struct inode *dir;
+	struct find_name_by_ino arg = {
+		.ctx = {
+			.actor = find_name_by_ino
+		}
+	};
+	int err;
+
+	parent = path->dentry;
+	if (nsi_lock)
+		si_read_unlock(parent->d_sb);
+	file = vfsub_dentry_open(path, au_dir_roflags);
+	dentry = (void *)file;
+	if (IS_ERR(file))
+		goto out;
+
+	dentry = ERR_PTR(-ENOMEM);
+	arg.name = (void *)__get_free_page(GFP_NOFS);
+	if (unlikely(!arg.name))
+		goto out_file;
+	arg.ino = ino;
+	arg.found = 0;
+	do {
+		arg.called = 0;
+		/* smp_mb(); */
+		err = vfsub_iterate_dir(file, &arg.ctx);
+	} while (!err && !arg.found && arg.called);
+	dentry = ERR_PTR(err);
+	if (unlikely(err))
+		goto out_name;
+	/* instead of ENOENT */
+	dentry = ERR_PTR(-ESTALE);
+	if (!arg.found)
+		goto out_name;
+
+	/* do not call vfsub_lkup_one() */
+	dir = d_inode(parent);
+	dentry = vfsub_lookup_one_len_unlocked(arg.name, parent, arg.namelen);
+	AuTraceErrPtr(dentry);
+	if (IS_ERR(dentry))
+		goto out_name;
+	AuDebugOn(au_test_anon(dentry));
+	if (unlikely(d_really_is_negative(dentry))) {
+		dput(dentry);
+		dentry = ERR_PTR(-ENOENT);
+	}
+
+out_name:
+	free_page((unsigned long)arg.name);
+out_file:
+	fput(file);
+out:
+	if (unlikely(nsi_lock
+		     && si_nfsd_read_lock(parent->d_sb, nsi_lock) < 0))
+		if (!IS_ERR(dentry)) {
+			dput(dentry);
+			dentry = ERR_PTR(-ESTALE);
+		}
+	AuTraceErrPtr(dentry);
+	return dentry;
+}
+
+static struct dentry *decode_by_dir_ino(struct super_block *sb, ino_t ino,
+					ino_t dir_ino,
+					struct au_nfsd_si_lock *nsi_lock)
+{
+	struct dentry *dentry;
+	struct path path;
+
+	if (dir_ino != AUFS_ROOT_INO) {
+		path.dentry = decode_by_ino(sb, dir_ino, 0);
+		dentry = path.dentry;
+		if (!path.dentry || IS_ERR(path.dentry))
+			goto out;
+		AuDebugOn(au_test_anon(path.dentry));
+	} else
+		path.dentry = dget(sb->s_root);
+
+	path.mnt = au_mnt_get(sb);
+	dentry = au_lkup_by_ino(&path, ino, nsi_lock);
+	path_put(&path);
+
+out:
+	AuTraceErrPtr(dentry);
+	return dentry;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static int h_acceptable(void *expv, struct dentry *dentry)
+{
+	return 1;
+}
+
+static char *au_build_path(struct dentry *h_parent, struct path *h_rootpath,
+			   char *buf, int len, struct super_block *sb)
+{
+	char *p;
+	int n;
+	struct path path;
+
+	p = d_path(h_rootpath, buf, len);
+	if (IS_ERR(p))
+		goto out;
+	n = strlen(p);
+
+	path.mnt = h_rootpath->mnt;
+	path.dentry = h_parent;
+	p = d_path(&path, buf, len);
+	if (IS_ERR(p))
+		goto out;
+	if (n != 1)
+		p += n;
+
+	path.mnt = au_mnt_get(sb);
+	path.dentry = sb->s_root;
+	p = d_path(&path, buf, len - strlen(p));
+	mntput(path.mnt);
+	if (IS_ERR(p))
+		goto out;
+	if (n != 1)
+		p[strlen(p)] = '/';
+
+out:
+	AuTraceErrPtr(p);
+	return p;
+}
+
+static
+struct dentry *decode_by_path(struct super_block *sb, ino_t ino, __u32 *fh,
+			      int fh_len, struct au_nfsd_si_lock *nsi_lock)
+{
+	struct dentry *dentry, *h_parent, *root;
+	struct super_block *h_sb;
+	char *pathname, *p;
+	struct vfsmount *h_mnt;
+	struct au_branch *br;
+	int err;
+	struct path path;
+
+	br = au_sbr(sb, nsi_lock->bindex);
+	h_mnt = au_br_mnt(br);
+	h_sb = h_mnt->mnt_sb;
+	/* todo: call lower fh_to_dentry()? fh_to_parent()? */
+	lockdep_off();
+	h_parent = exportfs_decode_fh(h_mnt, (void *)(fh + Fh_tail),
+				      fh_len - Fh_tail, fh[Fh_h_type],
+				      h_acceptable, /*context*/NULL);
+	lockdep_on();
+	dentry = h_parent;
+	if (unlikely(!h_parent || IS_ERR(h_parent))) {
+		AuWarn1("%s decode_fh failed, %ld\n",
+			au_sbtype(h_sb), PTR_ERR(h_parent));
+		goto out;
+	}
+	dentry = NULL;
+	if (unlikely(au_test_anon(h_parent))) {
+		AuWarn1("%s decode_fh returned a disconnected dentry\n",
+			au_sbtype(h_sb));
+		goto out_h_parent;
+	}
+
+	dentry = ERR_PTR(-ENOMEM);
+	pathname = (void *)__get_free_page(GFP_NOFS);
+	if (unlikely(!pathname))
+		goto out_h_parent;
+
+	root = sb->s_root;
+	path.mnt = h_mnt;
+	di_read_lock_parent(root, !AuLock_IR);
+	path.dentry = au_h_dptr(root, nsi_lock->bindex);
+	di_read_unlock(root, !AuLock_IR);
+	p = au_build_path(h_parent, &path, pathname, PAGE_SIZE, sb);
+	dentry = (void *)p;
+	if (IS_ERR(p))
+		goto out_pathname;
+
+	si_read_unlock(sb);
+	err = vfsub_kern_path(p, LOOKUP_FOLLOW | LOOKUP_DIRECTORY, &path);
+	dentry = ERR_PTR(err);
+	if (unlikely(err))
+		goto out_relock;
+
+	dentry = ERR_PTR(-ENOENT);
+	AuDebugOn(au_test_anon(path.dentry));
+	if (unlikely(d_really_is_negative(path.dentry)))
+		goto out_path;
+
+	if (ino != d_inode(path.dentry)->i_ino)
+		dentry = au_lkup_by_ino(&path, ino, /*nsi_lock*/NULL);
+	else
+		dentry = dget(path.dentry);
+
+out_path:
+	path_put(&path);
+out_relock:
+	if (unlikely(si_nfsd_read_lock(sb, nsi_lock) < 0))
+		if (!IS_ERR(dentry)) {
+			dput(dentry);
+			dentry = ERR_PTR(-ESTALE);
+		}
+out_pathname:
+	free_page((unsigned long)pathname);
+out_h_parent:
+	dput(h_parent);
+out:
+	AuTraceErrPtr(dentry);
+	return dentry;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static struct dentry *
+aufs_fh_to_dentry(struct super_block *sb, struct fid *fid, int fh_len,
+		  int fh_type)
+{
+	struct dentry *dentry;
+	__u32 *fh = fid->raw;
+	struct au_branch *br;
+	ino_t ino, dir_ino;
+	struct au_nfsd_si_lock nsi_lock = {
+		.force_lock	= 0
+	};
+
+	dentry = ERR_PTR(-ESTALE);
+	/* it should never happen, but the file handle is unreliable */
+	if (unlikely(fh_len < Fh_tail))
+		goto out;
+	nsi_lock.sigen = fh[Fh_sigen];
+	nsi_lock.br_id = fh[Fh_br_id];
+
+	/* branch id may be wrapped around */
+	br = NULL;
+	if (unlikely(si_nfsd_read_lock(sb, &nsi_lock)))
+		goto out;
+	nsi_lock.force_lock = 1;
+
+	/* is this inode still cached? */
+	ino = decode_ino(fh + Fh_ino);
+	/* it should never happen */
+	if (unlikely(ino == AUFS_ROOT_INO))
+		goto out_unlock;
+
+	dir_ino = decode_ino(fh + Fh_dir_ino);
+	dentry = decode_by_ino(sb, ino, dir_ino);
+	if (IS_ERR(dentry))
+		goto out_unlock;
+	if (dentry)
+		goto accept;
+
+	/* is the parent dir cached? */
+	br = au_sbr(sb, nsi_lock.bindex);
+	au_lcnt_inc(&br->br_nfiles);
+	dentry = decode_by_dir_ino(sb, ino, dir_ino, &nsi_lock);
+	if (IS_ERR(dentry))
+		goto out_unlock;
+	if (dentry)
+		goto accept;
+
+	/* lookup path */
+	dentry = decode_by_path(sb, ino, fh, fh_len, &nsi_lock);
+	if (IS_ERR(dentry))
+		goto out_unlock;
+	if (unlikely(!dentry))
+		/* todo?: make it ESTALE */
+		goto out_unlock;
+
+accept:
+	if (!au_digen_test(dentry, au_sigen(sb))
+	    && d_inode(dentry)->i_generation == fh[Fh_igen])
+		goto out_unlock; /* success */
+
+	dput(dentry);
+	dentry = ERR_PTR(-ESTALE);
+out_unlock:
+	if (br)
+		au_lcnt_dec(&br->br_nfiles);
+	si_read_unlock(sb);
+out:
+	AuTraceErrPtr(dentry);
+	return dentry;
+}
+
+#if 0 /* reserved for future use */
+/* support subtreecheck option */
+static struct dentry *aufs_fh_to_parent(struct super_block *sb, struct fid *fid,
+					int fh_len, int fh_type)
+{
+	struct dentry *parent;
+	__u32 *fh = fid->raw;
+	ino_t dir_ino;
+
+	dir_ino = decode_ino(fh + Fh_dir_ino);
+	parent = decode_by_ino(sb, dir_ino, 0);
+	if (IS_ERR(parent))
+		goto out;
+	if (!parent)
+		parent = decode_by_path(sb, au_br_index(sb, fh[Fh_br_id]),
+					dir_ino, fh, fh_len);
+
+out:
+	AuTraceErrPtr(parent);
+	return parent;
+}
+#endif
+
+/* ---------------------------------------------------------------------- */
+
+static int aufs_encode_fh(struct inode *inode, __u32 *fh, int *max_len,
+			  struct inode *dir)
+{
+	int err;
+	aufs_bindex_t bindex;
+	struct super_block *sb, *h_sb;
+	struct dentry *dentry, *parent, *h_parent;
+	struct inode *h_dir;
+	struct au_branch *br;
+
+	err = -ENOSPC;
+	if (unlikely(*max_len <= Fh_tail)) {
+		AuWarn1("NFSv2 client (max_len %d)?\n", *max_len);
+		goto out;
+	}
+
+	err = FILEID_ROOT;
+	if (inode->i_ino == AUFS_ROOT_INO) {
+		AuDebugOn(inode->i_ino != AUFS_ROOT_INO);
+		goto out;
+	}
+
+	h_parent = NULL;
+	sb = inode->i_sb;
+	err = si_read_lock(sb, AuLock_FLUSH);
+	if (unlikely(err))
+		goto out;
+
+#ifdef CONFIG_AUFS_DEBUG
+	if (unlikely(!au_opt_test(au_mntflags(sb), XINO)))
+		AuWarn1("NFS-exporting requires xino\n");
+#endif
+	err = -EIO;
+	parent = NULL;
+	ii_read_lock_child(inode);
+	bindex = au_ibtop(inode);
+	if (!dir) {
+		dentry = d_find_any_alias(inode);
+		if (unlikely(!dentry))
+			goto out_unlock;
+		AuDebugOn(au_test_anon(dentry));
+		parent = dget_parent(dentry);
+		dput(dentry);
+		if (unlikely(!parent))
+			goto out_unlock;
+		if (d_really_is_positive(parent))
+			dir = d_inode(parent);
+	}
+
+	ii_read_lock_parent(dir);
+	h_dir = au_h_iptr(dir, bindex);
+	ii_read_unlock(dir);
+	if (unlikely(!h_dir))
+		goto out_parent;
+	h_parent = d_find_any_alias(h_dir);
+	if (unlikely(!h_parent))
+		goto out_hparent;
+
+	err = -EPERM;
+	br = au_sbr(sb, bindex);
+	h_sb = au_br_sb(br);
+	if (unlikely(!h_sb->s_export_op)) {
+		AuErr1("%s branch is not exportable\n", au_sbtype(h_sb));
+		goto out_hparent;
+	}
+
+	fh[Fh_br_id] = br->br_id;
+	fh[Fh_sigen] = au_sigen(sb);
+	encode_ino(fh + Fh_ino, inode->i_ino);
+	encode_ino(fh + Fh_dir_ino, dir->i_ino);
+	fh[Fh_igen] = inode->i_generation;
+
+	*max_len -= Fh_tail;
+	fh[Fh_h_type] = exportfs_encode_fh(h_parent, (void *)(fh + Fh_tail),
+					   max_len,
+					   /*connectable or subtreecheck*/0);
+	err = fh[Fh_h_type];
+	*max_len += Fh_tail;
+	/* todo: macros? */
+	if (err != FILEID_INVALID)
+		err = 99;
+	else
+		AuWarn1("%s encode_fh failed\n", au_sbtype(h_sb));
+
+out_hparent:
+	dput(h_parent);
+out_parent:
+	dput(parent);
+out_unlock:
+	ii_read_unlock(inode);
+	si_read_unlock(sb);
+out:
+	if (unlikely(err < 0))
+		err = FILEID_INVALID;
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static int aufs_commit_metadata(struct inode *inode)
+{
+	int err;
+	aufs_bindex_t bindex;
+	struct super_block *sb;
+	struct inode *h_inode;
+	int (*f)(struct inode *inode);
+
+	sb = inode->i_sb;
+	si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLMW);
+	ii_write_lock_child(inode);
+	bindex = au_ibtop(inode);
+	AuDebugOn(bindex < 0);
+	h_inode = au_h_iptr(inode, bindex);
+
+	f = h_inode->i_sb->s_export_op->commit_metadata;
+	if (f)
+		err = f(h_inode);
+	else {
+		struct writeback_control wbc = {
+			.sync_mode	= WB_SYNC_ALL,
+			.nr_to_write	= 0 /* metadata only */
+		};
+
+		err = sync_inode(h_inode, &wbc);
+	}
+
+	au_cpup_attr_timesizes(inode);
+	ii_write_unlock(inode);
+	si_read_unlock(sb);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static struct export_operations aufs_export_op = {
+	.fh_to_dentry		= aufs_fh_to_dentry,
+	/* .fh_to_parent	= aufs_fh_to_parent, */
+	.encode_fh		= aufs_encode_fh,
+	.commit_metadata	= aufs_commit_metadata
+};
+
+void au_export_init(struct super_block *sb)
+{
+	struct au_sbinfo *sbinfo;
+	__u32 u;
+
+	BUILD_BUG_ON_MSG(IS_BUILTIN(CONFIG_AUFS_FS)
+			 && IS_MODULE(CONFIG_EXPORTFS),
+			 AUFS_NAME ": unsupported configuration "
+			 "CONFIG_EXPORTFS=m and CONFIG_AUFS_FS=y");
+
+	sb->s_export_op = &aufs_export_op;
+	sbinfo = au_sbi(sb);
+	sbinfo->si_xigen = NULL;
+	get_random_bytes(&u, sizeof(u));
+	BUILD_BUG_ON(sizeof(u) != sizeof(int));
+	atomic_set(&sbinfo->si_xigen_next, u);
+}
diff -Naur --no-dereference a/fs/aufs/fhsm.c b/fs/aufs/fhsm.c
--- a/fs/aufs/fhsm.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/fhsm.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,427 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2011-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+/*
+ * File-based Hierarchy Storage Management
+ */
+
+#include <linux/anon_inodes.h>
+#include <linux/poll.h>
+#include <linux/seq_file.h>
+#include <linux/statfs.h>
+#include "aufs.h"
+
+static aufs_bindex_t au_fhsm_bottom(struct super_block *sb)
+{
+	struct au_sbinfo *sbinfo;
+	struct au_fhsm *fhsm;
+
+	SiMustAnyLock(sb);
+
+	sbinfo = au_sbi(sb);
+	fhsm = &sbinfo->si_fhsm;
+	AuDebugOn(!fhsm);
+	return fhsm->fhsm_bottom;
+}
+
+void au_fhsm_set_bottom(struct super_block *sb, aufs_bindex_t bindex)
+{
+	struct au_sbinfo *sbinfo;
+	struct au_fhsm *fhsm;
+
+	SiMustWriteLock(sb);
+
+	sbinfo = au_sbi(sb);
+	fhsm = &sbinfo->si_fhsm;
+	AuDebugOn(!fhsm);
+	fhsm->fhsm_bottom = bindex;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static int au_fhsm_test_jiffy(struct au_sbinfo *sbinfo, struct au_branch *br)
+{
+	struct au_br_fhsm *bf;
+
+	bf = br->br_fhsm;
+	MtxMustLock(&bf->bf_lock);
+
+	return !bf->bf_readable
+		|| time_after(jiffies,
+			      bf->bf_jiffy + sbinfo->si_fhsm.fhsm_expire);
+}
+
+/* ---------------------------------------------------------------------- */
+
+static void au_fhsm_notify(struct super_block *sb, int val)
+{
+	struct au_sbinfo *sbinfo;
+	struct au_fhsm *fhsm;
+
+	SiMustAnyLock(sb);
+
+	sbinfo = au_sbi(sb);
+	fhsm = &sbinfo->si_fhsm;
+	if (au_fhsm_pid(fhsm)
+	    && atomic_read(&fhsm->fhsm_readable) != -1) {
+		atomic_set(&fhsm->fhsm_readable, val);
+		if (val)
+			wake_up(&fhsm->fhsm_wqh);
+	}
+}
+
+static int au_fhsm_stfs(struct super_block *sb, aufs_bindex_t bindex,
+			struct aufs_stfs *rstfs, int do_lock, int do_notify)
+{
+	int err;
+	struct au_branch *br;
+	struct au_br_fhsm *bf;
+
+	br = au_sbr(sb, bindex);
+	AuDebugOn(au_br_rdonly(br));
+	bf = br->br_fhsm;
+	AuDebugOn(!bf);
+
+	if (do_lock)
+		mutex_lock(&bf->bf_lock);
+	else
+		MtxMustLock(&bf->bf_lock);
+
+	/* sb->s_root for NFS is unreliable */
+	err = au_br_stfs(br, &bf->bf_stfs);
+	if (unlikely(err)) {
+		AuErr1("FHSM failed (%d), b%d, ignored.\n", bindex, err);
+		goto out;
+	}
+
+	bf->bf_jiffy = jiffies;
+	bf->bf_readable = 1;
+	if (do_notify)
+		au_fhsm_notify(sb, /*val*/1);
+	if (rstfs)
+		*rstfs = bf->bf_stfs;
+
+out:
+	if (do_lock)
+		mutex_unlock(&bf->bf_lock);
+	au_fhsm_notify(sb, /*val*/1);
+
+	return err;
+}
+
+void au_fhsm_wrote(struct super_block *sb, aufs_bindex_t bindex, int force)
+{
+	int err;
+	struct au_sbinfo *sbinfo;
+	struct au_fhsm *fhsm;
+	struct au_branch *br;
+	struct au_br_fhsm *bf;
+
+	AuDbg("b%d, force %d\n", bindex, force);
+	SiMustAnyLock(sb);
+
+	sbinfo = au_sbi(sb);
+	fhsm = &sbinfo->si_fhsm;
+	if (!au_ftest_si(sbinfo, FHSM)
+	    || fhsm->fhsm_bottom == bindex)
+		return;
+
+	br = au_sbr(sb, bindex);
+	bf = br->br_fhsm;
+	AuDebugOn(!bf);
+	mutex_lock(&bf->bf_lock);
+	if (force
+	    || au_fhsm_pid(fhsm)
+	    || au_fhsm_test_jiffy(sbinfo, br))
+		err = au_fhsm_stfs(sb, bindex, /*rstfs*/NULL, /*do_lock*/0,
+				  /*do_notify*/1);
+	mutex_unlock(&bf->bf_lock);
+}
+
+void au_fhsm_wrote_all(struct super_block *sb, int force)
+{
+	aufs_bindex_t bindex, bbot;
+	struct au_branch *br;
+
+	/* exclude the bottom */
+	bbot = au_fhsm_bottom(sb);
+	for (bindex = 0; bindex < bbot; bindex++) {
+		br = au_sbr(sb, bindex);
+		if (au_br_fhsm(br->br_perm))
+			au_fhsm_wrote(sb, bindex, force);
+	}
+}
+
+/* ---------------------------------------------------------------------- */
+
+static __poll_t au_fhsm_poll(struct file *file, struct poll_table_struct *wait)
+{
+	__poll_t mask;
+	struct au_sbinfo *sbinfo;
+	struct au_fhsm *fhsm;
+
+	mask = 0;
+	sbinfo = file->private_data;
+	fhsm = &sbinfo->si_fhsm;
+	poll_wait(file, &fhsm->fhsm_wqh, wait);
+	if (atomic_read(&fhsm->fhsm_readable))
+		mask = EPOLLIN /* | EPOLLRDNORM */;
+
+	if (!mask)
+		AuDbg("mask 0x%x\n", mask);
+	return mask;
+}
+
+static int au_fhsm_do_read_one(struct aufs_stbr __user *stbr,
+			      struct aufs_stfs *stfs, __s16 brid)
+{
+	int err;
+
+	err = copy_to_user(&stbr->stfs, stfs, sizeof(*stfs));
+	if (!err)
+		err = __put_user(brid, &stbr->brid);
+	if (unlikely(err))
+		err = -EFAULT;
+
+	return err;
+}
+
+static ssize_t au_fhsm_do_read(struct super_block *sb,
+			       struct aufs_stbr __user *stbr, size_t count)
+{
+	ssize_t err;
+	int nstbr;
+	aufs_bindex_t bindex, bbot;
+	struct au_branch *br;
+	struct au_br_fhsm *bf;
+
+	/* except the bottom branch */
+	err = 0;
+	nstbr = 0;
+	bbot = au_fhsm_bottom(sb);
+	for (bindex = 0; !err && bindex < bbot; bindex++) {
+		br = au_sbr(sb, bindex);
+		if (!au_br_fhsm(br->br_perm))
+			continue;
+
+		bf = br->br_fhsm;
+		mutex_lock(&bf->bf_lock);
+		if (bf->bf_readable) {
+			err = -EFAULT;
+			if (count >= sizeof(*stbr))
+				err = au_fhsm_do_read_one(stbr++, &bf->bf_stfs,
+							  br->br_id);
+			if (!err) {
+				bf->bf_readable = 0;
+				count -= sizeof(*stbr);
+				nstbr++;
+			}
+		}
+		mutex_unlock(&bf->bf_lock);
+	}
+	if (!err)
+		err = sizeof(*stbr) * nstbr;
+
+	return err;
+}
+
+static ssize_t au_fhsm_read(struct file *file, char __user *buf, size_t count,
+			   loff_t *pos)
+{
+	ssize_t err;
+	int readable;
+	aufs_bindex_t nfhsm, bindex, bbot;
+	struct au_sbinfo *sbinfo;
+	struct au_fhsm *fhsm;
+	struct au_branch *br;
+	struct super_block *sb;
+
+	err = 0;
+	sbinfo = file->private_data;
+	fhsm = &sbinfo->si_fhsm;
+need_data:
+	spin_lock_irq(&fhsm->fhsm_wqh.lock);
+	if (!atomic_read(&fhsm->fhsm_readable)) {
+		if (vfsub_file_flags(file) & O_NONBLOCK)
+			err = -EAGAIN;
+		else
+			err = wait_event_interruptible_locked_irq
+				(fhsm->fhsm_wqh,
+				 atomic_read(&fhsm->fhsm_readable));
+	}
+	spin_unlock_irq(&fhsm->fhsm_wqh.lock);
+	if (unlikely(err))
+		goto out;
+
+	/* sb may already be dead */
+	au_rw_read_lock(&sbinfo->si_rwsem);
+	readable = atomic_read(&fhsm->fhsm_readable);
+	if (readable > 0) {
+		sb = sbinfo->si_sb;
+		AuDebugOn(!sb);
+		/* exclude the bottom branch */
+		nfhsm = 0;
+		bbot = au_fhsm_bottom(sb);
+		for (bindex = 0; bindex < bbot; bindex++) {
+			br = au_sbr(sb, bindex);
+			if (au_br_fhsm(br->br_perm))
+				nfhsm++;
+		}
+		err = -EMSGSIZE;
+		if (nfhsm * sizeof(struct aufs_stbr) <= count) {
+			atomic_set(&fhsm->fhsm_readable, 0);
+			err = au_fhsm_do_read(sbinfo->si_sb, (void __user *)buf,
+					     count);
+		}
+	}
+	au_rw_read_unlock(&sbinfo->si_rwsem);
+	if (!readable)
+		goto need_data;
+
+out:
+	return err;
+}
+
+static int au_fhsm_release(struct inode *inode, struct file *file)
+{
+	struct au_sbinfo *sbinfo;
+	struct au_fhsm *fhsm;
+
+	/* sb may already be dead */
+	sbinfo = file->private_data;
+	fhsm = &sbinfo->si_fhsm;
+	spin_lock(&fhsm->fhsm_spin);
+	fhsm->fhsm_pid = 0;
+	spin_unlock(&fhsm->fhsm_spin);
+	kobject_put(&sbinfo->si_kobj);
+
+	return 0;
+}
+
+static const struct file_operations au_fhsm_fops = {
+	.owner		= THIS_MODULE,
+	.llseek		= noop_llseek,
+	.read		= au_fhsm_read,
+	.poll		= au_fhsm_poll,
+	.release	= au_fhsm_release
+};
+
+int au_fhsm_fd(struct super_block *sb, int oflags)
+{
+	int err, fd;
+	struct au_sbinfo *sbinfo;
+	struct au_fhsm *fhsm;
+
+	err = -EPERM;
+	if (unlikely(!capable(CAP_SYS_ADMIN)))
+		goto out;
+
+	err = -EINVAL;
+	if (unlikely(oflags & ~(O_CLOEXEC | O_NONBLOCK)))
+		goto out;
+
+	err = 0;
+	sbinfo = au_sbi(sb);
+	fhsm = &sbinfo->si_fhsm;
+	spin_lock(&fhsm->fhsm_spin);
+	if (!fhsm->fhsm_pid)
+		fhsm->fhsm_pid = current->pid;
+	else
+		err = -EBUSY;
+	spin_unlock(&fhsm->fhsm_spin);
+	if (unlikely(err))
+		goto out;
+
+	oflags |= O_RDONLY;
+	/* oflags |= FMODE_NONOTIFY; */
+	fd = anon_inode_getfd("[aufs_fhsm]", &au_fhsm_fops, sbinfo, oflags);
+	err = fd;
+	if (unlikely(fd < 0))
+		goto out_pid;
+
+	/* succeed regardless 'fhsm' status */
+	kobject_get(&sbinfo->si_kobj);
+	si_noflush_read_lock(sb);
+	if (au_ftest_si(sbinfo, FHSM))
+		au_fhsm_wrote_all(sb, /*force*/0);
+	si_read_unlock(sb);
+	goto out; /* success */
+
+out_pid:
+	spin_lock(&fhsm->fhsm_spin);
+	fhsm->fhsm_pid = 0;
+	spin_unlock(&fhsm->fhsm_spin);
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+int au_fhsm_br_alloc(struct au_branch *br)
+{
+	int err;
+
+	err = 0;
+	br->br_fhsm = kmalloc(sizeof(*br->br_fhsm), GFP_NOFS);
+	if (br->br_fhsm)
+		au_br_fhsm_init(br->br_fhsm);
+	else
+		err = -ENOMEM;
+
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+void au_fhsm_fin(struct super_block *sb)
+{
+	au_fhsm_notify(sb, /*val*/-1);
+}
+
+void au_fhsm_init(struct au_sbinfo *sbinfo)
+{
+	struct au_fhsm *fhsm;
+
+	fhsm = &sbinfo->si_fhsm;
+	spin_lock_init(&fhsm->fhsm_spin);
+	init_waitqueue_head(&fhsm->fhsm_wqh);
+	atomic_set(&fhsm->fhsm_readable, 0);
+	fhsm->fhsm_expire
+		= msecs_to_jiffies(AUFS_FHSM_CACHE_DEF_SEC * MSEC_PER_SEC);
+	fhsm->fhsm_bottom = -1;
+}
+
+void au_fhsm_set(struct au_sbinfo *sbinfo, unsigned int sec)
+{
+	sbinfo->si_fhsm.fhsm_expire
+		= msecs_to_jiffies(sec * MSEC_PER_SEC);
+}
+
+void au_fhsm_show(struct seq_file *seq, struct au_sbinfo *sbinfo)
+{
+	unsigned int u;
+
+	if (!au_ftest_si(sbinfo, FHSM))
+		return;
+
+	u = jiffies_to_msecs(sbinfo->si_fhsm.fhsm_expire) / MSEC_PER_SEC;
+	if (u != AUFS_FHSM_CACHE_DEF_SEC)
+		seq_printf(seq, ",fhsm_sec=%u", u);
+}
diff -Naur --no-dereference a/fs/aufs/file.c b/fs/aufs/file.c
--- a/fs/aufs/file.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/file.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,863 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * handling file/dir, and address_space operation
+ */
+
+#ifdef CONFIG_AUFS_DEBUG
+#include <linux/migrate.h>
+#endif
+#include <linux/pagemap.h>
+#include "aufs.h"
+
+/* drop flags for writing */
+unsigned int au_file_roflags(unsigned int flags)
+{
+	flags &= ~(O_WRONLY | O_RDWR | O_APPEND | O_CREAT | O_TRUNC);
+	flags |= O_RDONLY | O_NOATIME;
+	return flags;
+}
+
+/* common functions to regular file and dir */
+struct file *au_h_open(struct dentry *dentry, aufs_bindex_t bindex, int flags,
+		       struct file *file, int force_wr)
+{
+	struct file *h_file;
+	struct dentry *h_dentry;
+	struct inode *h_inode;
+	struct super_block *sb;
+	struct au_branch *br;
+	struct path h_path;
+	int err;
+
+	/* a race condition can happen between open and unlink/rmdir */
+	h_file = ERR_PTR(-ENOENT);
+	h_dentry = au_h_dptr(dentry, bindex);
+	if (au_test_nfsd() && (!h_dentry || d_is_negative(h_dentry)))
+		goto out;
+	h_inode = d_inode(h_dentry);
+	spin_lock(&h_dentry->d_lock);
+	err = (!d_unhashed(dentry) && d_unlinked(h_dentry))
+		/* || !d_inode(dentry)->i_nlink */
+		;
+	spin_unlock(&h_dentry->d_lock);
+	if (unlikely(err))
+		goto out;
+
+	sb = dentry->d_sb;
+	br = au_sbr(sb, bindex);
+	err = au_br_test_oflag(flags, br);
+	h_file = ERR_PTR(err);
+	if (unlikely(err))
+		goto out;
+
+	/* drop flags for writing */
+	if (au_test_ro(sb, bindex, d_inode(dentry))) {
+		if (force_wr && !(flags & O_WRONLY))
+			force_wr = 0;
+		flags = au_file_roflags(flags);
+		if (force_wr) {
+			h_file = ERR_PTR(-EROFS);
+			flags = au_file_roflags(flags);
+			if (unlikely(vfsub_native_ro(h_inode)
+				     || IS_APPEND(h_inode)))
+				goto out;
+			flags &= ~O_ACCMODE;
+			flags |= O_WRONLY;
+		}
+	}
+	flags &= ~O_CREAT;
+	au_lcnt_inc(&br->br_nfiles);
+	h_path.dentry = h_dentry;
+	h_path.mnt = au_br_mnt(br);
+	h_file = vfsub_dentry_open(&h_path, flags);
+	if (IS_ERR(h_file))
+		goto out_br;
+
+	if (flags & __FMODE_EXEC) {
+		err = deny_write_access(h_file);
+		if (unlikely(err)) {
+			fput(h_file);
+			h_file = ERR_PTR(err);
+			goto out_br;
+		}
+	}
+	fsnotify_open(h_file);
+	goto out; /* success */
+
+out_br:
+	au_lcnt_dec(&br->br_nfiles);
+out:
+	return h_file;
+}
+
+static int au_cmoo(struct dentry *dentry)
+{
+	int err, cmoo, matched;
+	unsigned int udba;
+	struct path h_path;
+	struct au_pin pin;
+	struct au_cp_generic cpg = {
+		.dentry	= dentry,
+		.bdst	= -1,
+		.bsrc	= -1,
+		.len	= -1,
+		.pin	= &pin,
+		.flags	= AuCpup_DTIME | AuCpup_HOPEN
+	};
+	struct inode *delegated;
+	struct super_block *sb;
+	struct au_sbinfo *sbinfo;
+	struct au_fhsm *fhsm;
+	pid_t pid;
+	struct au_branch *br;
+	struct dentry *parent;
+	struct au_hinode *hdir;
+
+	DiMustWriteLock(dentry);
+	IiMustWriteLock(d_inode(dentry));
+
+	err = 0;
+	if (IS_ROOT(dentry))
+		goto out;
+	cpg.bsrc = au_dbtop(dentry);
+	if (!cpg.bsrc)
+		goto out;
+
+	sb = dentry->d_sb;
+	sbinfo = au_sbi(sb);
+	fhsm = &sbinfo->si_fhsm;
+	pid = au_fhsm_pid(fhsm);
+	rcu_read_lock();
+	matched = (pid
+		   && (current->pid == pid
+		       || rcu_dereference(current->real_parent)->pid == pid));
+	rcu_read_unlock();
+	if (matched)
+		goto out;
+
+	br = au_sbr(sb, cpg.bsrc);
+	cmoo = au_br_cmoo(br->br_perm);
+	if (!cmoo)
+		goto out;
+	if (!d_is_reg(dentry))
+		cmoo &= AuBrAttr_COO_ALL;
+	if (!cmoo)
+		goto out;
+
+	parent = dget_parent(dentry);
+	di_write_lock_parent(parent);
+	err = au_wbr_do_copyup_bu(dentry, cpg.bsrc - 1);
+	cpg.bdst = err;
+	if (unlikely(err < 0)) {
+		err = 0;	/* there is no upper writable branch */
+		goto out_dgrade;
+	}
+	AuDbg("bsrc %d, bdst %d\n", cpg.bsrc, cpg.bdst);
+
+	/* do not respect the coo attrib for the target branch */
+	err = au_cpup_dirs(dentry, cpg.bdst);
+	if (unlikely(err))
+		goto out_dgrade;
+
+	di_downgrade_lock(parent, AuLock_IR);
+	udba = au_opt_udba(sb);
+	err = au_pin(&pin, dentry, cpg.bdst, udba,
+		     AuPin_DI_LOCKED | AuPin_MNT_WRITE);
+	if (unlikely(err))
+		goto out_parent;
+
+	err = au_sio_cpup_simple(&cpg);
+	au_unpin(&pin);
+	if (unlikely(err))
+		goto out_parent;
+	if (!(cmoo & AuBrWAttr_MOO))
+		goto out_parent; /* success */
+
+	err = au_pin(&pin, dentry, cpg.bsrc, udba,
+		     AuPin_DI_LOCKED | AuPin_MNT_WRITE);
+	if (unlikely(err))
+		goto out_parent;
+
+	h_path.mnt = au_br_mnt(br);
+	h_path.dentry = au_h_dptr(dentry, cpg.bsrc);
+	hdir = au_hi(d_inode(parent), cpg.bsrc);
+	delegated = NULL;
+	err = vfsub_unlink(hdir->hi_inode, &h_path, &delegated, /*force*/1);
+	au_unpin(&pin);
+	/* todo: keep h_dentry or not? */
+	if (unlikely(err == -EWOULDBLOCK)) {
+		pr_warn("cannot retry for NFSv4 delegation"
+			" for an internal unlink\n");
+		iput(delegated);
+	}
+	if (unlikely(err)) {
+		pr_err("unlink %pd after coo failed (%d), ignored\n",
+		       dentry, err);
+		err = 0;
+	}
+	goto out_parent; /* success */
+
+out_dgrade:
+	di_downgrade_lock(parent, AuLock_IR);
+out_parent:
+	di_read_unlock(parent, AuLock_IR);
+	dput(parent);
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+int au_do_open(struct file *file, struct au_do_open_args *args)
+{
+	int err, aopen = args->aopen;
+	struct dentry *dentry;
+	struct au_finfo *finfo;
+
+	if (!aopen)
+		err = au_finfo_init(file, args->fidir);
+	else {
+		lockdep_off();
+		err = au_finfo_init(file, args->fidir);
+		lockdep_on();
+	}
+	if (unlikely(err))
+		goto out;
+
+	dentry = file->f_path.dentry;
+	AuDebugOn(IS_ERR_OR_NULL(dentry));
+	di_write_lock_child(dentry);
+	err = au_cmoo(dentry);
+	di_downgrade_lock(dentry, AuLock_IR);
+	if (!err) {
+		if (!aopen)
+			err = args->open(file, vfsub_file_flags(file), NULL);
+		else {
+			lockdep_off();
+			err = args->open(file, vfsub_file_flags(file),
+					 args->h_file);
+			lockdep_on();
+		}
+	}
+	di_read_unlock(dentry, AuLock_IR);
+
+	finfo = au_fi(file);
+	if (!err) {
+		finfo->fi_file = file;
+		au_hbl_add(&finfo->fi_hlist,
+			   &au_sbi(file->f_path.dentry->d_sb)->si_files);
+	}
+	if (!aopen)
+		fi_write_unlock(file);
+	else {
+		lockdep_off();
+		fi_write_unlock(file);
+		lockdep_on();
+	}
+	if (unlikely(err)) {
+		finfo->fi_hdir = NULL;
+		au_finfo_fin(file);
+	}
+
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+int au_reopen_nondir(struct file *file)
+{
+	int err;
+	aufs_bindex_t btop;
+	struct dentry *dentry;
+	struct au_branch *br;
+	struct file *h_file, *h_file_tmp;
+
+	dentry = file->f_path.dentry;
+	btop = au_dbtop(dentry);
+	br = au_sbr(dentry->d_sb, btop);
+	h_file_tmp = NULL;
+	if (au_fbtop(file) == btop) {
+		h_file = au_hf_top(file);
+		if (file->f_mode == h_file->f_mode)
+			return 0; /* success */
+		h_file_tmp = h_file;
+		get_file(h_file_tmp);
+		au_lcnt_inc(&br->br_nfiles);
+		au_set_h_fptr(file, btop, NULL);
+	}
+	AuDebugOn(au_fi(file)->fi_hdir);
+	/*
+	 * it can happen
+	 * file exists on both of rw and ro
+	 * open --> dbtop and fbtop are both 0
+	 * prepend a branch as rw, "rw" become ro
+	 * remove rw/file
+	 * delete the top branch, "rw" becomes rw again
+	 *	--> dbtop is 1, fbtop is still 0
+	 * write --> fbtop is 0 but dbtop is 1
+	 */
+	/* AuDebugOn(au_fbtop(file) < btop); */
+
+	h_file = au_h_open(dentry, btop, vfsub_file_flags(file) & ~O_TRUNC,
+			   file, /*force_wr*/0);
+	err = PTR_ERR(h_file);
+	if (IS_ERR(h_file)) {
+		if (h_file_tmp) {
+			/* revert */
+			au_set_h_fptr(file, btop, h_file_tmp);
+			h_file_tmp = NULL;
+		}
+		goto out; /* todo: close all? */
+	}
+
+	err = 0;
+	au_set_fbtop(file, btop);
+	au_set_h_fptr(file, btop, h_file);
+	au_update_figen(file);
+	/* todo: necessary? */
+	/* file->f_ra = h_file->f_ra; */
+
+out:
+	if (h_file_tmp) {
+		fput(h_file_tmp);
+		au_lcnt_dec(&br->br_nfiles);
+	}
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static int au_reopen_wh(struct file *file, aufs_bindex_t btgt,
+			struct dentry *hi_wh)
+{
+	int err;
+	aufs_bindex_t btop;
+	struct au_dinfo *dinfo;
+	struct dentry *h_dentry;
+	struct au_hdentry *hdp;
+
+	dinfo = au_di(file->f_path.dentry);
+	AuRwMustWriteLock(&dinfo->di_rwsem);
+
+	btop = dinfo->di_btop;
+	dinfo->di_btop = btgt;
+	hdp = au_hdentry(dinfo, btgt);
+	h_dentry = hdp->hd_dentry;
+	hdp->hd_dentry = hi_wh;
+	err = au_reopen_nondir(file);
+	hdp->hd_dentry = h_dentry;
+	dinfo->di_btop = btop;
+
+	return err;
+}
+
+static int au_ready_to_write_wh(struct file *file, loff_t len,
+				aufs_bindex_t bcpup, struct au_pin *pin)
+{
+	int err;
+	struct inode *inode, *h_inode;
+	struct dentry *h_dentry, *hi_wh;
+	struct au_cp_generic cpg = {
+		.dentry	= file->f_path.dentry,
+		.bdst	= bcpup,
+		.bsrc	= -1,
+		.len	= len,
+		.pin	= pin
+	};
+
+	au_update_dbtop(cpg.dentry);
+	inode = d_inode(cpg.dentry);
+	h_inode = NULL;
+	if (au_dbtop(cpg.dentry) <= bcpup
+	    && au_dbbot(cpg.dentry) >= bcpup) {
+		h_dentry = au_h_dptr(cpg.dentry, bcpup);
+		if (h_dentry && d_is_positive(h_dentry))
+			h_inode = d_inode(h_dentry);
+	}
+	hi_wh = au_hi_wh(inode, bcpup);
+	if (!hi_wh && !h_inode)
+		err = au_sio_cpup_wh(&cpg, file);
+	else
+		/* already copied-up after unlink */
+		err = au_reopen_wh(file, bcpup, hi_wh);
+
+	if (!err
+	    && (inode->i_nlink > 1
+		|| (inode->i_state & I_LINKABLE))
+	    && au_opt_test(au_mntflags(cpg.dentry->d_sb), PLINK))
+		au_plink_append(inode, bcpup, au_h_dptr(cpg.dentry, bcpup));
+
+	return err;
+}
+
+/*
+ * prepare the @file for writing.
+ */
+int au_ready_to_write(struct file *file, loff_t len, struct au_pin *pin)
+{
+	int err;
+	aufs_bindex_t dbtop;
+	struct dentry *parent;
+	struct inode *inode;
+	struct super_block *sb;
+	struct file *h_file;
+	struct au_cp_generic cpg = {
+		.dentry	= file->f_path.dentry,
+		.bdst	= -1,
+		.bsrc	= -1,
+		.len	= len,
+		.pin	= pin,
+		.flags	= AuCpup_DTIME
+	};
+
+	sb = cpg.dentry->d_sb;
+	inode = d_inode(cpg.dentry);
+	cpg.bsrc = au_fbtop(file);
+	err = au_test_ro(sb, cpg.bsrc, inode);
+	if (!err && (au_hf_top(file)->f_mode & FMODE_WRITE)) {
+		err = au_pin(pin, cpg.dentry, cpg.bsrc, AuOpt_UDBA_NONE,
+			     /*flags*/0);
+		goto out;
+	}
+
+	/* need to cpup or reopen */
+	parent = dget_parent(cpg.dentry);
+	di_write_lock_parent(parent);
+	err = AuWbrCopyup(au_sbi(sb), cpg.dentry);
+	cpg.bdst = err;
+	if (unlikely(err < 0))
+		goto out_dgrade;
+	err = 0;
+
+	if (!d_unhashed(cpg.dentry) && !au_h_dptr(parent, cpg.bdst)) {
+		err = au_cpup_dirs(cpg.dentry, cpg.bdst);
+		if (unlikely(err))
+			goto out_dgrade;
+	}
+
+	err = au_pin(pin, cpg.dentry, cpg.bdst, AuOpt_UDBA_NONE,
+		     AuPin_DI_LOCKED | AuPin_MNT_WRITE);
+	if (unlikely(err))
+		goto out_dgrade;
+
+	dbtop = au_dbtop(cpg.dentry);
+	if (dbtop <= cpg.bdst)
+		cpg.bsrc = cpg.bdst;
+
+	if (dbtop <= cpg.bdst		/* just reopen */
+	    || !d_unhashed(cpg.dentry)	/* copyup and reopen */
+		) {
+		h_file = au_h_open_pre(cpg.dentry, cpg.bsrc, /*force_wr*/0);
+		if (IS_ERR(h_file))
+			err = PTR_ERR(h_file);
+		else {
+			di_downgrade_lock(parent, AuLock_IR);
+			if (dbtop > cpg.bdst)
+				err = au_sio_cpup_simple(&cpg);
+			if (!err)
+				err = au_reopen_nondir(file);
+			au_h_open_post(cpg.dentry, cpg.bsrc, h_file);
+		}
+	} else {			/* copyup as wh and reopen */
+		/*
+		 * since writable hfsplus branch is not supported,
+		 * h_open_pre/post() are unnecessary.
+		 */
+		err = au_ready_to_write_wh(file, len, cpg.bdst, pin);
+		di_downgrade_lock(parent, AuLock_IR);
+	}
+
+	if (!err) {
+		au_pin_set_parent_lflag(pin, /*lflag*/0);
+		goto out_dput; /* success */
+	}
+	au_unpin(pin);
+	goto out_unlock;
+
+out_dgrade:
+	di_downgrade_lock(parent, AuLock_IR);
+out_unlock:
+	di_read_unlock(parent, AuLock_IR);
+out_dput:
+	dput(parent);
+out:
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+int au_do_flush(struct file *file, fl_owner_t id,
+		int (*flush)(struct file *file, fl_owner_t id))
+{
+	int err;
+	struct super_block *sb;
+	struct inode *inode;
+
+	inode = file_inode(file);
+	sb = inode->i_sb;
+	si_noflush_read_lock(sb);
+	fi_read_lock(file);
+	ii_read_lock_child(inode);
+
+	err = flush(file, id);
+	au_cpup_attr_timesizes(inode);
+
+	ii_read_unlock(inode);
+	fi_read_unlock(file);
+	si_read_unlock(sb);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static int au_file_refresh_by_inode(struct file *file, int *need_reopen)
+{
+	int err;
+	struct au_pin pin;
+	struct au_finfo *finfo;
+	struct dentry *parent, *hi_wh;
+	struct inode *inode;
+	struct super_block *sb;
+	struct au_cp_generic cpg = {
+		.dentry	= file->f_path.dentry,
+		.bdst	= -1,
+		.bsrc	= -1,
+		.len	= -1,
+		.pin	= &pin,
+		.flags	= AuCpup_DTIME
+	};
+
+	FiMustWriteLock(file);
+
+	err = 0;
+	finfo = au_fi(file);
+	sb = cpg.dentry->d_sb;
+	inode = d_inode(cpg.dentry);
+	cpg.bdst = au_ibtop(inode);
+	if (cpg.bdst == finfo->fi_btop || IS_ROOT(cpg.dentry))
+		goto out;
+
+	parent = dget_parent(cpg.dentry);
+	if (au_test_ro(sb, cpg.bdst, inode)) {
+		di_read_lock_parent(parent, !AuLock_IR);
+		err = AuWbrCopyup(au_sbi(sb), cpg.dentry);
+		cpg.bdst = err;
+		di_read_unlock(parent, !AuLock_IR);
+		if (unlikely(err < 0))
+			goto out_parent;
+		err = 0;
+	}
+
+	di_read_lock_parent(parent, AuLock_IR);
+	hi_wh = au_hi_wh(inode, cpg.bdst);
+	if (!S_ISDIR(inode->i_mode)
+	    && au_opt_test(au_mntflags(sb), PLINK)
+	    && au_plink_test(inode)
+	    && !d_unhashed(cpg.dentry)
+	    && cpg.bdst < au_dbtop(cpg.dentry)) {
+		err = au_test_and_cpup_dirs(cpg.dentry, cpg.bdst);
+		if (unlikely(err))
+			goto out_unlock;
+
+		/* always superio. */
+		err = au_pin(&pin, cpg.dentry, cpg.bdst, AuOpt_UDBA_NONE,
+			     AuPin_DI_LOCKED | AuPin_MNT_WRITE);
+		if (!err) {
+			err = au_sio_cpup_simple(&cpg);
+			au_unpin(&pin);
+		}
+	} else if (hi_wh) {
+		/* already copied-up after unlink */
+		err = au_reopen_wh(file, cpg.bdst, hi_wh);
+		*need_reopen = 0;
+	}
+
+out_unlock:
+	di_read_unlock(parent, AuLock_IR);
+out_parent:
+	dput(parent);
+out:
+	return err;
+}
+
+static void au_do_refresh_dir(struct file *file)
+{
+	aufs_bindex_t bindex, bbot, new_bindex, brid;
+	struct au_hfile *p, tmp, *q;
+	struct au_finfo *finfo;
+	struct super_block *sb;
+	struct au_fidir *fidir;
+
+	FiMustWriteLock(file);
+
+	sb = file->f_path.dentry->d_sb;
+	finfo = au_fi(file);
+	fidir = finfo->fi_hdir;
+	AuDebugOn(!fidir);
+	p = fidir->fd_hfile + finfo->fi_btop;
+	brid = p->hf_br->br_id;
+	bbot = fidir->fd_bbot;
+	for (bindex = finfo->fi_btop; bindex <= bbot; bindex++, p++) {
+		if (!p->hf_file)
+			continue;
+
+		new_bindex = au_br_index(sb, p->hf_br->br_id);
+		if (new_bindex == bindex)
+			continue;
+		if (new_bindex < 0) {
+			au_set_h_fptr(file, bindex, NULL);
+			continue;
+		}
+
+		/* swap two lower inode, and loop again */
+		q = fidir->fd_hfile + new_bindex;
+		tmp = *q;
+		*q = *p;
+		*p = tmp;
+		if (tmp.hf_file) {
+			bindex--;
+			p--;
+		}
+	}
+
+	p = fidir->fd_hfile;
+	if (!au_test_mmapped(file) && !d_unlinked(file->f_path.dentry)) {
+		bbot = au_sbbot(sb);
+		for (finfo->fi_btop = 0; finfo->fi_btop <= bbot;
+		     finfo->fi_btop++, p++)
+			if (p->hf_file) {
+				if (file_inode(p->hf_file))
+					break;
+				au_hfput(p, /*execed*/0);
+			}
+	} else {
+		bbot = au_br_index(sb, brid);
+		for (finfo->fi_btop = 0; finfo->fi_btop < bbot;
+		     finfo->fi_btop++, p++)
+			if (p->hf_file)
+				au_hfput(p, /*execed*/0);
+		bbot = au_sbbot(sb);
+	}
+
+	p = fidir->fd_hfile + bbot;
+	for (fidir->fd_bbot = bbot; fidir->fd_bbot >= finfo->fi_btop;
+	     fidir->fd_bbot--, p--)
+		if (p->hf_file) {
+			if (file_inode(p->hf_file))
+				break;
+			au_hfput(p, /*execed*/0);
+		}
+	AuDebugOn(fidir->fd_bbot < finfo->fi_btop);
+}
+
+/*
+ * after branch manipulating, refresh the file.
+ */
+static int refresh_file(struct file *file, int (*reopen)(struct file *file))
+{
+	int err, need_reopen, nbr;
+	aufs_bindex_t bbot, bindex;
+	struct dentry *dentry;
+	struct super_block *sb;
+	struct au_finfo *finfo;
+	struct au_hfile *hfile;
+
+	dentry = file->f_path.dentry;
+	sb = dentry->d_sb;
+	nbr = au_sbbot(sb) + 1;
+	finfo = au_fi(file);
+	if (!finfo->fi_hdir) {
+		hfile = &finfo->fi_htop;
+		AuDebugOn(!hfile->hf_file);
+		bindex = au_br_index(sb, hfile->hf_br->br_id);
+		AuDebugOn(bindex < 0);
+		if (bindex != finfo->fi_btop)
+			au_set_fbtop(file, bindex);
+	} else {
+		err = au_fidir_realloc(finfo, nbr, /*may_shrink*/0);
+		if (unlikely(err))
+			goto out;
+		au_do_refresh_dir(file);
+	}
+
+	err = 0;
+	need_reopen = 1;
+	if (!au_test_mmapped(file))
+		err = au_file_refresh_by_inode(file, &need_reopen);
+	if (finfo->fi_hdir)
+		/* harmless if err */
+		au_fidir_realloc(finfo, nbr, /*may_shrink*/1);
+	if (!err && need_reopen && !d_unlinked(dentry))
+		err = reopen(file);
+	if (!err) {
+		au_update_figen(file);
+		goto out; /* success */
+	}
+
+	/* error, close all lower files */
+	if (finfo->fi_hdir) {
+		bbot = au_fbbot_dir(file);
+		for (bindex = au_fbtop(file); bindex <= bbot; bindex++)
+			au_set_h_fptr(file, bindex, NULL);
+	}
+
+out:
+	return err;
+}
+
+/* common function to regular file and dir */
+int au_reval_and_lock_fdi(struct file *file, int (*reopen)(struct file *file),
+			  int wlock, unsigned int fi_lsc)
+{
+	int err;
+	unsigned int sigen, figen;
+	aufs_bindex_t btop;
+	unsigned char pseudo_link;
+	struct dentry *dentry;
+	struct inode *inode;
+
+	err = 0;
+	dentry = file->f_path.dentry;
+	inode = d_inode(dentry);
+	sigen = au_sigen(dentry->d_sb);
+	fi_write_lock_nested(file, fi_lsc);
+	figen = au_figen(file);
+	if (!fi_lsc)
+		di_write_lock_child(dentry);
+	else
+		di_write_lock_child2(dentry);
+	btop = au_dbtop(dentry);
+	pseudo_link = (btop != au_ibtop(inode));
+	if (sigen == figen && !pseudo_link && au_fbtop(file) == btop) {
+		if (!wlock) {
+			di_downgrade_lock(dentry, AuLock_IR);
+			fi_downgrade_lock(file);
+		}
+		goto out; /* success */
+	}
+
+	AuDbg("sigen %d, figen %d\n", sigen, figen);
+	if (au_digen_test(dentry, sigen)) {
+		err = au_reval_dpath(dentry, sigen);
+		AuDebugOn(!err && au_digen_test(dentry, sigen));
+	}
+
+	if (!err)
+		err = refresh_file(file, reopen);
+	if (!err) {
+		if (!wlock) {
+			di_downgrade_lock(dentry, AuLock_IR);
+			fi_downgrade_lock(file);
+		}
+	} else {
+		di_write_unlock(dentry);
+		fi_write_unlock(file);
+	}
+
+out:
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* cf. aufs_nopage() */
+/* for madvise(2) */
+static int aufs_readpage(struct file *file __maybe_unused, struct page *page)
+{
+	unlock_page(page);
+	return 0;
+}
+
+/* it will never be called, but necessary to support O_DIRECT */
+static ssize_t aufs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
+{ BUG(); return 0; }
+
+/* they will never be called. */
+#ifdef CONFIG_AUFS_DEBUG
+static int aufs_write_begin(struct file *file, struct address_space *mapping,
+			    loff_t pos, unsigned len, unsigned flags,
+			    struct page **pagep, void **fsdata)
+{ AuUnsupport(); return 0; }
+static int aufs_write_end(struct file *file, struct address_space *mapping,
+			  loff_t pos, unsigned len, unsigned copied,
+			  struct page *page, void *fsdata)
+{ AuUnsupport(); return 0; }
+static int aufs_writepage(struct page *page, struct writeback_control *wbc)
+{ AuUnsupport(); return 0; }
+
+static int aufs_set_page_dirty(struct page *page)
+{ AuUnsupport(); return 0; }
+static void aufs_invalidatepage(struct page *page, unsigned int offset,
+				unsigned int length)
+{ AuUnsupport(); }
+static int aufs_releasepage(struct page *page, gfp_t gfp)
+{ AuUnsupport(); return 0; }
+#if 0 /* called by memory compaction regardless file */
+static int aufs_migratepage(struct address_space *mapping, struct page *newpage,
+			    struct page *page, enum migrate_mode mode)
+{ AuUnsupport(); return 0; }
+#endif
+static bool aufs_isolate_page(struct page *page, isolate_mode_t mode)
+{ AuUnsupport(); return true; }
+static void aufs_putback_page(struct page *page)
+{ AuUnsupport(); }
+static int aufs_launder_page(struct page *page)
+{ AuUnsupport(); return 0; }
+static int aufs_is_partially_uptodate(struct page *page,
+				      unsigned long from,
+				      unsigned long count)
+{ AuUnsupport(); return 0; }
+static void aufs_is_dirty_writeback(struct page *page, bool *dirty,
+				    bool *writeback)
+{ AuUnsupport(); }
+static int aufs_error_remove_page(struct address_space *mapping,
+				  struct page *page)
+{ AuUnsupport(); return 0; }
+static int aufs_swap_activate(struct swap_info_struct *sis, struct file *file,
+			      sector_t *span)
+{ AuUnsupport(); return 0; }
+static void aufs_swap_deactivate(struct file *file)
+{ AuUnsupport(); }
+#endif /* CONFIG_AUFS_DEBUG */
+
+const struct address_space_operations aufs_aop = {
+	.readpage		= aufs_readpage,
+	.direct_IO		= aufs_direct_IO,
+#ifdef CONFIG_AUFS_DEBUG
+	.writepage		= aufs_writepage,
+	/* no writepages, because of writepage */
+	.set_page_dirty		= aufs_set_page_dirty,
+	/* no readpages, because of readpage */
+	.write_begin		= aufs_write_begin,
+	.write_end		= aufs_write_end,
+	/* no bmap, no block device */
+	.invalidatepage		= aufs_invalidatepage,
+	.releasepage		= aufs_releasepage,
+	/* is fallback_migrate_page ok? */
+	/* .migratepage		= aufs_migratepage, */
+	.isolate_page		= aufs_isolate_page,
+	.putback_page		= aufs_putback_page,
+	.launder_page		= aufs_launder_page,
+	.is_partially_uptodate	= aufs_is_partially_uptodate,
+	.is_dirty_writeback	= aufs_is_dirty_writeback,
+	.error_remove_page	= aufs_error_remove_page,
+	.swap_activate		= aufs_swap_activate,
+	.swap_deactivate	= aufs_swap_deactivate
+#endif /* CONFIG_AUFS_DEBUG */
+};
diff -Naur --no-dereference a/fs/aufs/file.h b/fs/aufs/file.h
--- a/fs/aufs/file.h	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/file.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,342 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * file operations
+ */
+
+#ifndef __AUFS_FILE_H__
+#define __AUFS_FILE_H__
+
+#ifdef __KERNEL__
+
+#include <linux/file.h>
+#include <linux/fs.h>
+#include <linux/mm_types.h>
+#include <linux/poll.h>
+#include "rwsem.h"
+
+struct au_branch;
+struct au_hfile {
+	struct file		*hf_file;
+	struct au_branch	*hf_br;
+};
+
+struct au_vdir;
+struct au_fidir {
+	aufs_bindex_t		fd_bbot;
+	aufs_bindex_t		fd_nent;
+	struct au_vdir		*fd_vdir_cache;
+	struct au_hfile		fd_hfile[];
+};
+
+static inline int au_fidir_sz(int nent)
+{
+	AuDebugOn(nent < 0);
+	return sizeof(struct au_fidir) + sizeof(struct au_hfile) * nent;
+}
+
+struct au_finfo {
+	atomic_t		fi_generation;
+
+	struct au_rwsem		fi_rwsem;
+	aufs_bindex_t		fi_btop;
+
+	/* do not union them */
+	struct {				/* for non-dir */
+		struct au_hfile			fi_htop;
+		atomic_t			fi_mmapped;
+	};
+	struct au_fidir		*fi_hdir;	/* for dir only */
+
+	struct hlist_bl_node	fi_hlist;
+	struct file		*fi_file;	/* very ugly */
+	struct rcu_head		rcu;
+} ____cacheline_aligned_in_smp;
+
+/* ---------------------------------------------------------------------- */
+
+/* file.c */
+extern const struct address_space_operations aufs_aop;
+unsigned int au_file_roflags(unsigned int flags);
+struct file *au_h_open(struct dentry *dentry, aufs_bindex_t bindex, int flags,
+		       struct file *file, int force_wr);
+struct au_do_open_args {
+	int		aopen;
+	int		(*open)(struct file *file, int flags,
+				struct file *h_file);
+	struct au_fidir	*fidir;
+	struct file	*h_file;
+};
+int au_do_open(struct file *file, struct au_do_open_args *args);
+int au_reopen_nondir(struct file *file);
+struct au_pin;
+int au_ready_to_write(struct file *file, loff_t len, struct au_pin *pin);
+int au_reval_and_lock_fdi(struct file *file, int (*reopen)(struct file *file),
+			  int wlock, unsigned int fi_lsc);
+int au_do_flush(struct file *file, fl_owner_t id,
+		int (*flush)(struct file *file, fl_owner_t id));
+
+/* poll.c */
+#ifdef CONFIG_AUFS_POLL
+__poll_t aufs_poll(struct file *file, struct poll_table_struct *pt);
+#endif
+
+#ifdef CONFIG_AUFS_BR_HFSPLUS
+/* hfsplus.c */
+struct file *au_h_open_pre(struct dentry *dentry, aufs_bindex_t bindex,
+			   int force_wr);
+void au_h_open_post(struct dentry *dentry, aufs_bindex_t bindex,
+		    struct file *h_file);
+#else
+AuStub(struct file *, au_h_open_pre, return NULL, struct dentry *dentry,
+       aufs_bindex_t bindex, int force_wr)
+AuStubVoid(au_h_open_post, struct dentry *dentry, aufs_bindex_t bindex,
+	   struct file *h_file);
+#endif
+
+/* f_op.c */
+extern const struct file_operations aufs_file_fop;
+int au_do_open_nondir(struct file *file, int flags, struct file *h_file);
+int aufs_release_nondir(struct inode *inode __maybe_unused, struct file *file);
+struct file *au_read_pre(struct file *file, int keep_fi, unsigned int lsc);
+
+/* finfo.c */
+void au_hfput(struct au_hfile *hf, int execed);
+void au_set_h_fptr(struct file *file, aufs_bindex_t bindex,
+		   struct file *h_file);
+
+void au_update_figen(struct file *file);
+struct au_fidir *au_fidir_alloc(struct super_block *sb);
+int au_fidir_realloc(struct au_finfo *finfo, int nbr, int may_shrink);
+
+void au_fi_init_once(void *_fi);
+void au_finfo_fin(struct file *file);
+int au_finfo_init(struct file *file, struct au_fidir *fidir);
+
+/* ioctl.c */
+long aufs_ioctl_nondir(struct file *file, unsigned int cmd, unsigned long arg);
+#ifdef CONFIG_COMPAT
+long aufs_compat_ioctl_dir(struct file *file, unsigned int cmd,
+			   unsigned long arg);
+long aufs_compat_ioctl_nondir(struct file *file, unsigned int cmd,
+			      unsigned long arg);
+#endif
+
+/* ---------------------------------------------------------------------- */
+
+static inline struct au_finfo *au_fi(struct file *file)
+{
+	return file->private_data;
+}
+
+/* ---------------------------------------------------------------------- */
+
+#define fi_read_lock(f)	au_rw_read_lock(&au_fi(f)->fi_rwsem)
+#define fi_write_lock(f)	au_rw_write_lock(&au_fi(f)->fi_rwsem)
+#define fi_read_trylock(f)	au_rw_read_trylock(&au_fi(f)->fi_rwsem)
+#define fi_write_trylock(f)	au_rw_write_trylock(&au_fi(f)->fi_rwsem)
+/*
+#define fi_read_trylock_nested(f) \
+	au_rw_read_trylock_nested(&au_fi(f)->fi_rwsem)
+#define fi_write_trylock_nested(f) \
+	au_rw_write_trylock_nested(&au_fi(f)->fi_rwsem)
+*/
+
+#define fi_read_unlock(f)	au_rw_read_unlock(&au_fi(f)->fi_rwsem)
+#define fi_write_unlock(f)	au_rw_write_unlock(&au_fi(f)->fi_rwsem)
+#define fi_downgrade_lock(f)	au_rw_dgrade_lock(&au_fi(f)->fi_rwsem)
+
+/* lock subclass for finfo */
+enum {
+	AuLsc_FI_1,
+	AuLsc_FI_2
+};
+
+static inline void fi_read_lock_nested(struct file *f, unsigned int lsc)
+{
+	au_rw_read_lock_nested(&au_fi(f)->fi_rwsem, lsc);
+}
+
+static inline void fi_write_lock_nested(struct file *f, unsigned int lsc)
+{
+	au_rw_write_lock_nested(&au_fi(f)->fi_rwsem, lsc);
+}
+
+/*
+ * fi_read_lock_1, fi_write_lock_1,
+ * fi_read_lock_2, fi_write_lock_2
+ */
+#define AuReadLockFunc(name) \
+static inline void fi_read_lock_##name(struct file *f) \
+{ fi_read_lock_nested(f, AuLsc_FI_##name); }
+
+#define AuWriteLockFunc(name) \
+static inline void fi_write_lock_##name(struct file *f) \
+{ fi_write_lock_nested(f, AuLsc_FI_##name); }
+
+#define AuRWLockFuncs(name) \
+	AuReadLockFunc(name) \
+	AuWriteLockFunc(name)
+
+AuRWLockFuncs(1);
+AuRWLockFuncs(2);
+
+#undef AuReadLockFunc
+#undef AuWriteLockFunc
+#undef AuRWLockFuncs
+
+#define FiMustNoWaiters(f)	AuRwMustNoWaiters(&au_fi(f)->fi_rwsem)
+#define FiMustAnyLock(f)	AuRwMustAnyLock(&au_fi(f)->fi_rwsem)
+#define FiMustWriteLock(f)	AuRwMustWriteLock(&au_fi(f)->fi_rwsem)
+
+/* ---------------------------------------------------------------------- */
+
+/* todo: hard/soft set? */
+static inline aufs_bindex_t au_fbtop(struct file *file)
+{
+	FiMustAnyLock(file);
+	return au_fi(file)->fi_btop;
+}
+
+static inline aufs_bindex_t au_fbbot_dir(struct file *file)
+{
+	FiMustAnyLock(file);
+	AuDebugOn(!au_fi(file)->fi_hdir);
+	return au_fi(file)->fi_hdir->fd_bbot;
+}
+
+static inline struct au_vdir *au_fvdir_cache(struct file *file)
+{
+	FiMustAnyLock(file);
+	AuDebugOn(!au_fi(file)->fi_hdir);
+	return au_fi(file)->fi_hdir->fd_vdir_cache;
+}
+
+static inline void au_set_fbtop(struct file *file, aufs_bindex_t bindex)
+{
+	FiMustWriteLock(file);
+	au_fi(file)->fi_btop = bindex;
+}
+
+static inline void au_set_fbbot_dir(struct file *file, aufs_bindex_t bindex)
+{
+	FiMustWriteLock(file);
+	AuDebugOn(!au_fi(file)->fi_hdir);
+	au_fi(file)->fi_hdir->fd_bbot = bindex;
+}
+
+static inline void au_set_fvdir_cache(struct file *file,
+				      struct au_vdir *vdir_cache)
+{
+	FiMustWriteLock(file);
+	AuDebugOn(!au_fi(file)->fi_hdir);
+	au_fi(file)->fi_hdir->fd_vdir_cache = vdir_cache;
+}
+
+static inline struct file *au_hf_top(struct file *file)
+{
+	FiMustAnyLock(file);
+	AuDebugOn(au_fi(file)->fi_hdir);
+	return au_fi(file)->fi_htop.hf_file;
+}
+
+static inline struct file *au_hf_dir(struct file *file, aufs_bindex_t bindex)
+{
+	FiMustAnyLock(file);
+	AuDebugOn(!au_fi(file)->fi_hdir);
+	return au_fi(file)->fi_hdir->fd_hfile[0 + bindex].hf_file;
+}
+
+/* todo: memory barrier? */
+static inline unsigned int au_figen(struct file *f)
+{
+	return atomic_read(&au_fi(f)->fi_generation);
+}
+
+static inline void au_set_mmapped(struct file *f)
+{
+	if (atomic_inc_return(&au_fi(f)->fi_mmapped))
+		return;
+	pr_warn("fi_mmapped wrapped around\n");
+	while (!atomic_inc_return(&au_fi(f)->fi_mmapped))
+		;
+}
+
+static inline void au_unset_mmapped(struct file *f)
+{
+	atomic_dec(&au_fi(f)->fi_mmapped);
+}
+
+static inline int au_test_mmapped(struct file *f)
+{
+	return atomic_read(&au_fi(f)->fi_mmapped);
+}
+
+/* customize vma->vm_file */
+
+static inline void au_do_vm_file_reset(struct vm_area_struct *vma,
+				       struct file *file)
+{
+	struct file *f;
+
+	f = vma->vm_file;
+	get_file(file);
+	vma->vm_file = file;
+	fput(f);
+}
+
+#ifdef CONFIG_MMU
+#define AuDbgVmRegion(file, vma) do {} while (0)
+
+static inline void au_vm_file_reset(struct vm_area_struct *vma,
+				    struct file *file)
+{
+	au_do_vm_file_reset(vma, file);
+}
+#else
+#define AuDbgVmRegion(file, vma) \
+	AuDebugOn((vma)->vm_region && (vma)->vm_region->vm_file != (file))
+
+static inline void au_vm_file_reset(struct vm_area_struct *vma,
+				    struct file *file)
+{
+	struct file *f;
+
+	au_do_vm_file_reset(vma, file);
+	f = vma->vm_region->vm_file;
+	get_file(file);
+	vma->vm_region->vm_file = file;
+	fput(f);
+}
+#endif /* CONFIG_MMU */
+
+/* handle vma->vm_prfile */
+static inline void au_vm_prfile_set(struct vm_area_struct *vma,
+				    struct file *file)
+{
+	get_file(file);
+	vma->vm_prfile = file;
+#ifndef CONFIG_MMU
+	get_file(file);
+	vma->vm_region->vm_prfile = file;
+#endif
+}
+
+#endif /* __KERNEL__ */
+#endif /* __AUFS_FILE_H__ */
diff -Naur --no-dereference a/fs/aufs/finfo.c b/fs/aufs/finfo.c
--- a/fs/aufs/finfo.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/finfo.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,149 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * file private data
+ */
+
+#include "aufs.h"
+
+void au_hfput(struct au_hfile *hf, int execed)
+{
+	if (execed)
+		allow_write_access(hf->hf_file);
+	fput(hf->hf_file);
+	hf->hf_file = NULL;
+	au_lcnt_dec(&hf->hf_br->br_nfiles);
+	hf->hf_br = NULL;
+}
+
+void au_set_h_fptr(struct file *file, aufs_bindex_t bindex, struct file *val)
+{
+	struct au_finfo *finfo = au_fi(file);
+	struct au_hfile *hf;
+	struct au_fidir *fidir;
+
+	fidir = finfo->fi_hdir;
+	if (!fidir) {
+		AuDebugOn(finfo->fi_btop != bindex);
+		hf = &finfo->fi_htop;
+	} else
+		hf = fidir->fd_hfile + bindex;
+
+	if (hf && hf->hf_file)
+		au_hfput(hf, vfsub_file_execed(file));
+	if (val) {
+		FiMustWriteLock(file);
+		AuDebugOn(IS_ERR_OR_NULL(file->f_path.dentry));
+		hf->hf_file = val;
+		hf->hf_br = au_sbr(file->f_path.dentry->d_sb, bindex);
+	}
+}
+
+void au_update_figen(struct file *file)
+{
+	atomic_set(&au_fi(file)->fi_generation, au_digen(file->f_path.dentry));
+	/* smp_mb(); */ /* atomic_set */
+}
+
+/* ---------------------------------------------------------------------- */
+
+struct au_fidir *au_fidir_alloc(struct super_block *sb)
+{
+	struct au_fidir *fidir;
+	int nbr;
+
+	nbr = au_sbbot(sb) + 1;
+	if (nbr < 2)
+		nbr = 2; /* initial allocate for 2 branches */
+	fidir = kzalloc(au_fidir_sz(nbr), GFP_NOFS);
+	if (fidir) {
+		fidir->fd_bbot = -1;
+		fidir->fd_nent = nbr;
+	}
+
+	return fidir;
+}
+
+int au_fidir_realloc(struct au_finfo *finfo, int nbr, int may_shrink)
+{
+	int err;
+	struct au_fidir *fidir, *p;
+
+	AuRwMustWriteLock(&finfo->fi_rwsem);
+	fidir = finfo->fi_hdir;
+	AuDebugOn(!fidir);
+
+	err = -ENOMEM;
+	p = au_kzrealloc(fidir, au_fidir_sz(fidir->fd_nent), au_fidir_sz(nbr),
+			 GFP_NOFS, may_shrink);
+	if (p) {
+		p->fd_nent = nbr;
+		finfo->fi_hdir = p;
+		err = 0;
+	}
+
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+void au_finfo_fin(struct file *file)
+{
+	struct au_finfo *finfo;
+
+	au_lcnt_dec(&au_sbi(file->f_path.dentry->d_sb)->si_nfiles);
+
+	finfo = au_fi(file);
+	AuDebugOn(finfo->fi_hdir);
+	AuRwDestroy(&finfo->fi_rwsem);
+	au_cache_free_finfo(finfo);
+}
+
+void au_fi_init_once(void *_finfo)
+{
+	struct au_finfo *finfo = _finfo;
+
+	au_rw_init(&finfo->fi_rwsem);
+}
+
+int au_finfo_init(struct file *file, struct au_fidir *fidir)
+{
+	int err;
+	struct au_finfo *finfo;
+	struct dentry *dentry;
+
+	err = -ENOMEM;
+	dentry = file->f_path.dentry;
+	finfo = au_cache_alloc_finfo();
+	if (unlikely(!finfo))
+		goto out;
+
+	err = 0;
+	au_lcnt_inc(&au_sbi(dentry->d_sb)->si_nfiles);
+	au_rw_write_lock(&finfo->fi_rwsem);
+	finfo->fi_btop = -1;
+	finfo->fi_hdir = fidir;
+	atomic_set(&finfo->fi_generation, au_digen(dentry));
+	/* smp_mb(); */ /* atomic_set */
+
+	file->private_data = finfo;
+
+out:
+	return err;
+}
diff -Naur --no-dereference a/fs/aufs/f_op.c b/fs/aufs/f_op.c
--- a/fs/aufs/f_op.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/f_op.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,819 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * file and vm operations
+ */
+
+#include <linux/aio.h>
+#include <linux/fs_stack.h>
+#include <linux/mman.h>
+#include <linux/security.h>
+#include "aufs.h"
+
+int au_do_open_nondir(struct file *file, int flags, struct file *h_file)
+{
+	int err;
+	aufs_bindex_t bindex;
+	struct dentry *dentry, *h_dentry;
+	struct au_finfo *finfo;
+	struct inode *h_inode;
+
+	FiMustWriteLock(file);
+
+	err = 0;
+	dentry = file->f_path.dentry;
+	AuDebugOn(IS_ERR_OR_NULL(dentry));
+	finfo = au_fi(file);
+	memset(&finfo->fi_htop, 0, sizeof(finfo->fi_htop));
+	atomic_set(&finfo->fi_mmapped, 0);
+	bindex = au_dbtop(dentry);
+	if (!h_file) {
+		h_dentry = au_h_dptr(dentry, bindex);
+		err = vfsub_test_mntns(file->f_path.mnt, h_dentry->d_sb);
+		if (unlikely(err))
+			goto out;
+		h_file = au_h_open(dentry, bindex, flags, file, /*force_wr*/0);
+		if (IS_ERR(h_file)) {
+			err = PTR_ERR(h_file);
+			goto out;
+		}
+	} else {
+		h_dentry = h_file->f_path.dentry;
+		err = vfsub_test_mntns(file->f_path.mnt, h_dentry->d_sb);
+		if (unlikely(err))
+			goto out;
+		/* br ref is already inc-ed */
+	}
+
+	if ((flags & __O_TMPFILE)
+	    && !(flags & O_EXCL)) {
+		h_inode = file_inode(h_file);
+		spin_lock(&h_inode->i_lock);
+		h_inode->i_state |= I_LINKABLE;
+		spin_unlock(&h_inode->i_lock);
+	}
+	au_set_fbtop(file, bindex);
+	au_set_h_fptr(file, bindex, h_file);
+	au_update_figen(file);
+	/* todo: necessary? */
+	/* file->f_ra = h_file->f_ra; */
+
+out:
+	return err;
+}
+
+static int aufs_open_nondir(struct inode *inode __maybe_unused,
+			    struct file *file)
+{
+	int err;
+	struct super_block *sb;
+	struct au_do_open_args args = {
+		.open	= au_do_open_nondir
+	};
+
+	AuDbg("%pD, f_flags 0x%x, f_mode 0x%x\n",
+	      file, vfsub_file_flags(file), file->f_mode);
+
+	sb = file->f_path.dentry->d_sb;
+	si_read_lock(sb, AuLock_FLUSH);
+	err = au_do_open(file, &args);
+	si_read_unlock(sb);
+	return err;
+}
+
+int aufs_release_nondir(struct inode *inode __maybe_unused, struct file *file)
+{
+	struct au_finfo *finfo;
+	aufs_bindex_t bindex;
+
+	finfo = au_fi(file);
+	au_hbl_del(&finfo->fi_hlist,
+		   &au_sbi(file->f_path.dentry->d_sb)->si_files);
+	bindex = finfo->fi_btop;
+	if (bindex >= 0)
+		au_set_h_fptr(file, bindex, NULL);
+
+	au_finfo_fin(file);
+	return 0;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static int au_do_flush_nondir(struct file *file, fl_owner_t id)
+{
+	int err;
+	struct file *h_file;
+
+	err = 0;
+	h_file = au_hf_top(file);
+	if (h_file)
+		err = vfsub_flush(h_file, id);
+	return err;
+}
+
+static int aufs_flush_nondir(struct file *file, fl_owner_t id)
+{
+	return au_do_flush(file, id, au_do_flush_nondir);
+}
+
+/* ---------------------------------------------------------------------- */
+/*
+ * read and write functions acquire [fdi]_rwsem once, but release before
+ * mmap_sem. This is because to stop a race condition between mmap(2).
+ * Releasing these aufs-rwsem should be safe, no branch-management (by keeping
+ * si_rwsem), no harmful copy-up should happen. Actually copy-up may happen in
+ * read functions after [fdi]_rwsem are released, but it should be harmless.
+ */
+
+/* Callers should call au_read_post() or fput() in the end */
+struct file *au_read_pre(struct file *file, int keep_fi, unsigned int lsc)
+{
+	struct file *h_file;
+	int err;
+
+	err = au_reval_and_lock_fdi(file, au_reopen_nondir, /*wlock*/0, lsc);
+	if (!err) {
+		di_read_unlock(file->f_path.dentry, AuLock_IR);
+		h_file = au_hf_top(file);
+		get_file(h_file);
+		if (!keep_fi)
+			fi_read_unlock(file);
+	} else
+		h_file = ERR_PTR(err);
+
+	return h_file;
+}
+
+static void au_read_post(struct inode *inode, struct file *h_file)
+{
+	/* update without lock, I don't think it a problem */
+	fsstack_copy_attr_atime(inode, file_inode(h_file));
+	fput(h_file);
+}
+
+struct au_write_pre {
+	/* input */
+	unsigned int lsc;
+
+	/* output */
+	blkcnt_t blks;
+	aufs_bindex_t btop;
+};
+
+/*
+ * return with iinfo is write-locked
+ * callers should call au_write_post() or iinfo_write_unlock() + fput() in the
+ * end
+ */
+static struct file *au_write_pre(struct file *file, int do_ready,
+				 struct au_write_pre *wpre)
+{
+	struct file *h_file;
+	struct dentry *dentry;
+	int err;
+	unsigned int lsc;
+	struct au_pin pin;
+
+	lsc = 0;
+	if (wpre)
+		lsc = wpre->lsc;
+	err = au_reval_and_lock_fdi(file, au_reopen_nondir, /*wlock*/1, lsc);
+	h_file = ERR_PTR(err);
+	if (unlikely(err))
+		goto out;
+
+	dentry = file->f_path.dentry;
+	if (do_ready) {
+		err = au_ready_to_write(file, -1, &pin);
+		if (unlikely(err)) {
+			h_file = ERR_PTR(err);
+			di_write_unlock(dentry);
+			goto out_fi;
+		}
+	}
+
+	di_downgrade_lock(dentry, /*flags*/0);
+	if (wpre)
+		wpre->btop = au_fbtop(file);
+	h_file = au_hf_top(file);
+	get_file(h_file);
+	if (wpre)
+		wpre->blks = file_inode(h_file)->i_blocks;
+	if (do_ready)
+		au_unpin(&pin);
+	di_read_unlock(dentry, /*flags*/0);
+
+out_fi:
+	fi_write_unlock(file);
+out:
+	return h_file;
+}
+
+static void au_write_post(struct inode *inode, struct file *h_file,
+			  struct au_write_pre *wpre, ssize_t written)
+{
+	struct inode *h_inode;
+
+	au_cpup_attr_timesizes(inode);
+	AuDebugOn(au_ibtop(inode) != wpre->btop);
+	h_inode = file_inode(h_file);
+	inode->i_mode = h_inode->i_mode;
+	ii_write_unlock(inode);
+	/* AuDbg("blks %llu, %llu\n", (u64)blks, (u64)h_inode->i_blocks); */
+	if (written > 0)
+		au_fhsm_wrote(inode->i_sb, wpre->btop,
+			      /*force*/h_inode->i_blocks > wpre->blks);
+	fput(h_file);
+}
+
+static ssize_t aufs_read(struct file *file, char __user *buf, size_t count,
+			 loff_t *ppos)
+{
+	ssize_t err;
+	struct inode *inode;
+	struct file *h_file;
+	struct super_block *sb;
+
+	inode = file_inode(file);
+	sb = inode->i_sb;
+	si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLMW);
+
+	h_file = au_read_pre(file, /*keep_fi*/0, /*lsc*/0);
+	err = PTR_ERR(h_file);
+	if (IS_ERR(h_file))
+		goto out;
+
+	/* filedata may be obsoleted by concurrent copyup, but no problem */
+	err = vfsub_read_u(h_file, buf, count, ppos);
+	/* todo: necessary? */
+	/* file->f_ra = h_file->f_ra; */
+	au_read_post(inode, h_file);
+
+out:
+	si_read_unlock(sb);
+	return err;
+}
+
+/*
+ * todo: very ugly
+ * it locks both of i_mutex and si_rwsem for read in safe.
+ * if the plink maintenance mode continues forever (that is the problem),
+ * may loop forever.
+ */
+static void au_mtx_and_read_lock(struct inode *inode)
+{
+	int err;
+	struct super_block *sb = inode->i_sb;
+
+	while (1) {
+		inode_lock(inode);
+		err = si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLM);
+		if (!err)
+			break;
+		inode_unlock(inode);
+		si_read_lock(sb, AuLock_NOPLMW);
+		si_read_unlock(sb);
+	}
+}
+
+static ssize_t aufs_write(struct file *file, const char __user *ubuf,
+			  size_t count, loff_t *ppos)
+{
+	ssize_t err;
+	struct au_write_pre wpre;
+	struct inode *inode;
+	struct file *h_file;
+	char __user *buf = (char __user *)ubuf;
+
+	inode = file_inode(file);
+	au_mtx_and_read_lock(inode);
+
+	wpre.lsc = 0;
+	h_file = au_write_pre(file, /*do_ready*/1, &wpre);
+	err = PTR_ERR(h_file);
+	if (IS_ERR(h_file))
+		goto out;
+
+	err = vfsub_write_u(h_file, buf, count, ppos);
+	au_write_post(inode, h_file, &wpre, err);
+
+out:
+	si_read_unlock(inode->i_sb);
+	inode_unlock(inode);
+	return err;
+}
+
+static ssize_t au_do_iter(struct file *h_file, int rw, struct kiocb *kio,
+			  struct iov_iter *iov_iter)
+{
+	ssize_t err;
+	struct file *file;
+	ssize_t (*iter)(struct kiocb *, struct iov_iter *);
+
+	err = security_file_permission(h_file, rw);
+	if (unlikely(err))
+		goto out;
+
+	err = -ENOSYS;	/* the branch doesn't have its ->(read|write)_iter() */
+	iter = NULL;
+	if (rw == MAY_READ)
+		iter = h_file->f_op->read_iter;
+	else if (rw == MAY_WRITE)
+		iter = h_file->f_op->write_iter;
+
+	file = kio->ki_filp;
+	kio->ki_filp = h_file;
+	if (iter) {
+		lockdep_off();
+		err = iter(kio, iov_iter);
+		lockdep_on();
+	} else
+		/* currently there is no such fs */
+		WARN_ON_ONCE(1);
+	kio->ki_filp = file;
+
+out:
+	return err;
+}
+
+static ssize_t aufs_read_iter(struct kiocb *kio, struct iov_iter *iov_iter)
+{
+	ssize_t err;
+	struct file *file, *h_file;
+	struct inode *inode;
+	struct super_block *sb;
+
+	file = kio->ki_filp;
+	inode = file_inode(file);
+	sb = inode->i_sb;
+	si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLMW);
+
+	h_file = au_read_pre(file, /*keep_fi*/1, /*lsc*/0);
+	err = PTR_ERR(h_file);
+	if (IS_ERR(h_file))
+		goto out;
+
+	if (au_test_loopback_kthread()) {
+		au_warn_loopback(h_file->f_path.dentry->d_sb);
+		if (file->f_mapping != h_file->f_mapping) {
+			file->f_mapping = h_file->f_mapping;
+			smp_mb(); /* unnecessary? */
+		}
+	}
+	fi_read_unlock(file);
+
+	err = au_do_iter(h_file, MAY_READ, kio, iov_iter);
+	/* todo: necessary? */
+	/* file->f_ra = h_file->f_ra; */
+	au_read_post(inode, h_file);
+
+out:
+	si_read_unlock(sb);
+	return err;
+}
+
+static ssize_t aufs_write_iter(struct kiocb *kio, struct iov_iter *iov_iter)
+{
+	ssize_t err;
+	struct au_write_pre wpre;
+	struct inode *inode;
+	struct file *file, *h_file;
+
+	file = kio->ki_filp;
+	inode = file_inode(file);
+	au_mtx_and_read_lock(inode);
+
+	wpre.lsc = 0;
+	h_file = au_write_pre(file, /*do_ready*/1, &wpre);
+	err = PTR_ERR(h_file);
+	if (IS_ERR(h_file))
+		goto out;
+
+	err = au_do_iter(h_file, MAY_WRITE, kio, iov_iter);
+	au_write_post(inode, h_file, &wpre, err);
+
+out:
+	si_read_unlock(inode->i_sb);
+	inode_unlock(inode);
+	return err;
+}
+
+static ssize_t aufs_splice_read(struct file *file, loff_t *ppos,
+				struct pipe_inode_info *pipe, size_t len,
+				unsigned int flags)
+{
+	ssize_t err;
+	struct file *h_file;
+	struct inode *inode;
+	struct super_block *sb;
+
+	inode = file_inode(file);
+	sb = inode->i_sb;
+	si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLMW);
+
+	h_file = au_read_pre(file, /*keep_fi*/0, /*lsc*/0);
+	err = PTR_ERR(h_file);
+	if (IS_ERR(h_file))
+		goto out;
+
+	err = vfsub_splice_to(h_file, ppos, pipe, len, flags);
+	/* todo: necessary? */
+	/* file->f_ra = h_file->f_ra; */
+	au_read_post(inode, h_file);
+
+out:
+	si_read_unlock(sb);
+	return err;
+}
+
+static ssize_t
+aufs_splice_write(struct pipe_inode_info *pipe, struct file *file, loff_t *ppos,
+		  size_t len, unsigned int flags)
+{
+	ssize_t err;
+	struct au_write_pre wpre;
+	struct inode *inode;
+	struct file *h_file;
+
+	inode = file_inode(file);
+	au_mtx_and_read_lock(inode);
+
+	wpre.lsc = 0;
+	h_file = au_write_pre(file, /*do_ready*/1, &wpre);
+	err = PTR_ERR(h_file);
+	if (IS_ERR(h_file))
+		goto out;
+
+	err = vfsub_splice_from(pipe, h_file, ppos, len, flags);
+	au_write_post(inode, h_file, &wpre, err);
+
+out:
+	si_read_unlock(inode->i_sb);
+	inode_unlock(inode);
+	return err;
+}
+
+static long aufs_fallocate(struct file *file, int mode, loff_t offset,
+			   loff_t len)
+{
+	long err;
+	struct au_write_pre wpre;
+	struct inode *inode;
+	struct file *h_file;
+
+	inode = file_inode(file);
+	au_mtx_and_read_lock(inode);
+
+	wpre.lsc = 0;
+	h_file = au_write_pre(file, /*do_ready*/1, &wpre);
+	err = PTR_ERR(h_file);
+	if (IS_ERR(h_file))
+		goto out;
+
+	lockdep_off();
+	err = vfs_fallocate(h_file, mode, offset, len);
+	lockdep_on();
+	au_write_post(inode, h_file, &wpre, /*written*/1);
+
+out:
+	si_read_unlock(inode->i_sb);
+	inode_unlock(inode);
+	return err;
+}
+
+static ssize_t aufs_copy_file_range(struct file *src, loff_t src_pos,
+				    struct file *dst, loff_t dst_pos,
+				    size_t len, unsigned int flags)
+{
+	ssize_t err;
+	struct au_write_pre wpre;
+	enum { SRC, DST };
+	struct {
+		struct inode *inode;
+		struct file *h_file;
+		struct super_block *h_sb;
+	} a[2];
+#define a_src	a[SRC]
+#define a_dst	a[DST]
+
+	err = -EINVAL;
+	a_src.inode = file_inode(src);
+	if (unlikely(!S_ISREG(a_src.inode->i_mode)))
+		goto out;
+	a_dst.inode = file_inode(dst);
+	if (unlikely(!S_ISREG(a_dst.inode->i_mode)))
+		goto out;
+
+	au_mtx_and_read_lock(a_dst.inode);
+	/*
+	 * in order to match the order in di_write_lock2_{child,parent}(),
+	 * use f_path.dentry for this comparison.
+	 */
+	if (src->f_path.dentry < dst->f_path.dentry) {
+		a_src.h_file = au_read_pre(src, /*keep_fi*/1, AuLsc_FI_1);
+		err = PTR_ERR(a_src.h_file);
+		if (IS_ERR(a_src.h_file))
+			goto out_si;
+
+		wpre.lsc = AuLsc_FI_2;
+		a_dst.h_file = au_write_pre(dst, /*do_ready*/1, &wpre);
+		err = PTR_ERR(a_dst.h_file);
+		if (IS_ERR(a_dst.h_file)) {
+			au_read_post(a_src.inode, a_src.h_file);
+			goto out_si;
+		}
+	} else {
+		wpre.lsc = AuLsc_FI_1;
+		a_dst.h_file = au_write_pre(dst, /*do_ready*/1, &wpre);
+		err = PTR_ERR(a_dst.h_file);
+		if (IS_ERR(a_dst.h_file))
+			goto out_si;
+
+		a_src.h_file = au_read_pre(src, /*keep_fi*/1, AuLsc_FI_2);
+		err = PTR_ERR(a_src.h_file);
+		if (IS_ERR(a_src.h_file)) {
+			au_write_post(a_dst.inode, a_dst.h_file, &wpre,
+				      /*written*/0);
+			goto out_si;
+		}
+	}
+
+	err = -EXDEV;
+	a_src.h_sb = file_inode(a_src.h_file)->i_sb;
+	a_dst.h_sb = file_inode(a_dst.h_file)->i_sb;
+	if (unlikely(a_src.h_sb != a_dst.h_sb)) {
+		AuDbgFile(src);
+		AuDbgFile(dst);
+		goto out_file;
+	}
+
+	err = vfsub_copy_file_range(a_src.h_file, src_pos, a_dst.h_file,
+				    dst_pos, len, flags);
+
+out_file:
+	au_write_post(a_dst.inode, a_dst.h_file, &wpre, err);
+	fi_read_unlock(src);
+	au_read_post(a_src.inode, a_src.h_file);
+out_si:
+	si_read_unlock(a_dst.inode->i_sb);
+	inode_unlock(a_dst.inode);
+out:
+	return err;
+#undef a_src
+#undef a_dst
+}
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * The locking order around current->mmap_sem.
+ * - in most and regular cases
+ *   file I/O syscall -- aufs_read() or something
+ *	-- si_rwsem for read -- mmap_sem
+ *	(Note that [fdi]i_rwsem are released before mmap_sem).
+ * - in mmap case
+ *   mmap(2) -- mmap_sem -- aufs_mmap() -- si_rwsem for read -- [fdi]i_rwsem
+ * This AB-BA order is definitely bad, but is not a problem since "si_rwsem for
+ * read" allows multiple processes to acquire it and [fdi]i_rwsem are not held
+ * in file I/O. Aufs needs to stop lockdep in aufs_mmap() though.
+ * It means that when aufs acquires si_rwsem for write, the process should never
+ * acquire mmap_sem.
+ *
+ * Actually aufs_iterate() holds [fdi]i_rwsem before mmap_sem, but this is not a
+ * problem either since any directory is not able to be mmap-ed.
+ * The similar scenario is applied to aufs_readlink() too.
+ */
+
+#if 0 /* stop calling security_file_mmap() */
+/* cf. linux/include/linux/mman.h: calc_vm_prot_bits() */
+#define AuConv_VM_PROT(f, b)	_calc_vm_trans(f, VM_##b, PROT_##b)
+
+static unsigned long au_arch_prot_conv(unsigned long flags)
+{
+	/* currently ppc64 only */
+#ifdef CONFIG_PPC64
+	/* cf. linux/arch/powerpc/include/asm/mman.h */
+	AuDebugOn(arch_calc_vm_prot_bits(-1) != VM_SAO);
+	return AuConv_VM_PROT(flags, SAO);
+#else
+	AuDebugOn(arch_calc_vm_prot_bits(-1));
+	return 0;
+#endif
+}
+
+static unsigned long au_prot_conv(unsigned long flags)
+{
+	return AuConv_VM_PROT(flags, READ)
+		| AuConv_VM_PROT(flags, WRITE)
+		| AuConv_VM_PROT(flags, EXEC)
+		| au_arch_prot_conv(flags);
+}
+
+/* cf. linux/include/linux/mman.h: calc_vm_flag_bits() */
+#define AuConv_VM_MAP(f, b)	_calc_vm_trans(f, VM_##b, MAP_##b)
+
+static unsigned long au_flag_conv(unsigned long flags)
+{
+	return AuConv_VM_MAP(flags, GROWSDOWN)
+		| AuConv_VM_MAP(flags, DENYWRITE)
+		| AuConv_VM_MAP(flags, LOCKED);
+}
+#endif
+
+static int aufs_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	int err;
+	const unsigned char wlock
+		= (file->f_mode & FMODE_WRITE) && (vma->vm_flags & VM_SHARED);
+	struct super_block *sb;
+	struct file *h_file;
+	struct inode *inode;
+
+	AuDbgVmRegion(file, vma);
+
+	inode = file_inode(file);
+	sb = inode->i_sb;
+	lockdep_off();
+	si_read_lock(sb, AuLock_NOPLMW);
+
+	h_file = au_write_pre(file, wlock, /*wpre*/NULL);
+	lockdep_on();
+	err = PTR_ERR(h_file);
+	if (IS_ERR(h_file))
+		goto out;
+
+	err = 0;
+	au_set_mmapped(file);
+	au_vm_file_reset(vma, h_file);
+	/*
+	 * we cannot call security_mmap_file() here since it may acquire
+	 * mmap_sem or i_mutex.
+	 *
+	 * err = security_mmap_file(h_file, au_prot_conv(vma->vm_flags),
+	 *			 au_flag_conv(vma->vm_flags));
+	 */
+	if (!err)
+		err = call_mmap(h_file, vma);
+	if (!err) {
+		au_vm_prfile_set(vma, file);
+		fsstack_copy_attr_atime(inode, file_inode(h_file));
+		goto out_fput; /* success */
+	}
+	au_unset_mmapped(file);
+	au_vm_file_reset(vma, file);
+
+out_fput:
+	lockdep_off();
+	ii_write_unlock(inode);
+	lockdep_on();
+	fput(h_file);
+out:
+	lockdep_off();
+	si_read_unlock(sb);
+	lockdep_on();
+	AuTraceErr(err);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static int aufs_fsync_nondir(struct file *file, loff_t start, loff_t end,
+			     int datasync)
+{
+	int err;
+	struct au_write_pre wpre;
+	struct inode *inode;
+	struct file *h_file;
+
+	err = 0; /* -EBADF; */ /* posix? */
+	if (unlikely(!(file->f_mode & FMODE_WRITE)))
+		goto out;
+
+	inode = file_inode(file);
+	au_mtx_and_read_lock(inode);
+
+	wpre.lsc = 0;
+	h_file = au_write_pre(file, /*do_ready*/1, &wpre);
+	err = PTR_ERR(h_file);
+	if (IS_ERR(h_file))
+		goto out_unlock;
+
+	err = vfsub_fsync(h_file, &h_file->f_path, datasync);
+	au_write_post(inode, h_file, &wpre, /*written*/0);
+
+out_unlock:
+	si_read_unlock(inode->i_sb);
+	inode_unlock(inode);
+out:
+	return err;
+}
+
+static int aufs_fasync(int fd, struct file *file, int flag)
+{
+	int err;
+	struct file *h_file;
+	struct super_block *sb;
+
+	sb = file->f_path.dentry->d_sb;
+	si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLMW);
+
+	h_file = au_read_pre(file, /*keep_fi*/0, /*lsc*/0);
+	err = PTR_ERR(h_file);
+	if (IS_ERR(h_file))
+		goto out;
+
+	if (h_file->f_op->fasync)
+		err = h_file->f_op->fasync(fd, h_file, flag);
+	fput(h_file); /* instead of au_read_post() */
+
+out:
+	si_read_unlock(sb);
+	return err;
+}
+
+static int aufs_setfl(struct file *file, unsigned long arg)
+{
+	int err;
+	struct file *h_file;
+	struct super_block *sb;
+
+	sb = file->f_path.dentry->d_sb;
+	si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLMW);
+
+	h_file = au_read_pre(file, /*keep_fi*/0, /*lsc*/0);
+	err = PTR_ERR(h_file);
+	if (IS_ERR(h_file))
+		goto out;
+
+	/* stop calling h_file->fasync */
+	arg |= vfsub_file_flags(file) & FASYNC;
+	err = setfl(/*unused fd*/-1, h_file, arg);
+	fput(h_file); /* instead of au_read_post() */
+
+out:
+	si_read_unlock(sb);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* no one supports this operation, currently */
+#if 0 /* reserved for future use */
+static ssize_t aufs_sendpage(struct file *file, struct page *page, int offset,
+			     size_t len, loff_t *pos, int more)
+{
+}
+#endif
+
+/* ---------------------------------------------------------------------- */
+
+const struct file_operations aufs_file_fop = {
+	.owner		= THIS_MODULE,
+
+	.llseek		= default_llseek,
+
+	.read		= aufs_read,
+	.write		= aufs_write,
+	.read_iter	= aufs_read_iter,
+	.write_iter	= aufs_write_iter,
+
+#ifdef CONFIG_AUFS_POLL
+	.poll		= aufs_poll,
+#endif
+	.unlocked_ioctl	= aufs_ioctl_nondir,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl	= aufs_compat_ioctl_nondir,
+#endif
+	.mmap		= aufs_mmap,
+	.open		= aufs_open_nondir,
+	.flush		= aufs_flush_nondir,
+	.release	= aufs_release_nondir,
+	.fsync		= aufs_fsync_nondir,
+	.fasync		= aufs_fasync,
+	/* .sendpage	= aufs_sendpage, */
+	.setfl		= aufs_setfl,
+	.splice_write	= aufs_splice_write,
+	.splice_read	= aufs_splice_read,
+#if 0 /* reserved for future use */
+	.aio_splice_write = aufs_aio_splice_write,
+	.aio_splice_read  = aufs_aio_splice_read,
+#endif
+	.fallocate	= aufs_fallocate,
+	.copy_file_range = aufs_copy_file_range
+};
diff -Naur --no-dereference a/fs/aufs/fstype.h b/fs/aufs/fstype.h
--- a/fs/aufs/fstype.h	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/fstype.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,401 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * judging filesystem type
+ */
+
+#ifndef __AUFS_FSTYPE_H__
+#define __AUFS_FSTYPE_H__
+
+#ifdef __KERNEL__
+
+#include <linux/fs.h>
+#include <linux/magic.h>
+#include <linux/nfs_fs.h>
+#include <linux/romfs_fs.h>
+
+static inline int au_test_aufs(struct super_block *sb)
+{
+	return sb->s_magic == AUFS_SUPER_MAGIC;
+}
+
+static inline const char *au_sbtype(struct super_block *sb)
+{
+	return sb->s_type->name;
+}
+
+static inline int au_test_iso9660(struct super_block *sb __maybe_unused)
+{
+#if IS_ENABLED(CONFIG_ISO9660_FS)
+	return sb->s_magic == ISOFS_SUPER_MAGIC;
+#else
+	return 0;
+#endif
+}
+
+static inline int au_test_romfs(struct super_block *sb __maybe_unused)
+{
+#if IS_ENABLED(CONFIG_ROMFS_FS)
+	return sb->s_magic == ROMFS_MAGIC;
+#else
+	return 0;
+#endif
+}
+
+static inline int au_test_cramfs(struct super_block *sb __maybe_unused)
+{
+#if IS_ENABLED(CONFIG_CRAMFS)
+	return sb->s_magic == CRAMFS_MAGIC;
+#endif
+	return 0;
+}
+
+static inline int au_test_nfs(struct super_block *sb __maybe_unused)
+{
+#if IS_ENABLED(CONFIG_NFS_FS)
+	return sb->s_magic == NFS_SUPER_MAGIC;
+#else
+	return 0;
+#endif
+}
+
+static inline int au_test_fuse(struct super_block *sb __maybe_unused)
+{
+#if IS_ENABLED(CONFIG_FUSE_FS)
+	return sb->s_magic == FUSE_SUPER_MAGIC;
+#else
+	return 0;
+#endif
+}
+
+static inline int au_test_xfs(struct super_block *sb __maybe_unused)
+{
+#if IS_ENABLED(CONFIG_XFS_FS)
+	return sb->s_magic == XFS_SB_MAGIC;
+#else
+	return 0;
+#endif
+}
+
+static inline int au_test_tmpfs(struct super_block *sb __maybe_unused)
+{
+#ifdef CONFIG_TMPFS
+	return sb->s_magic == TMPFS_MAGIC;
+#else
+	return 0;
+#endif
+}
+
+static inline int au_test_ecryptfs(struct super_block *sb __maybe_unused)
+{
+#if IS_ENABLED(CONFIG_ECRYPT_FS)
+	return !strcmp(au_sbtype(sb), "ecryptfs");
+#else
+	return 0;
+#endif
+}
+
+static inline int au_test_ramfs(struct super_block *sb)
+{
+	return sb->s_magic == RAMFS_MAGIC;
+}
+
+static inline int au_test_ubifs(struct super_block *sb __maybe_unused)
+{
+#if IS_ENABLED(CONFIG_UBIFS_FS)
+	return sb->s_magic == UBIFS_SUPER_MAGIC;
+#else
+	return 0;
+#endif
+}
+
+static inline int au_test_procfs(struct super_block *sb __maybe_unused)
+{
+#ifdef CONFIG_PROC_FS
+	return sb->s_magic == PROC_SUPER_MAGIC;
+#else
+	return 0;
+#endif
+}
+
+static inline int au_test_sysfs(struct super_block *sb __maybe_unused)
+{
+#ifdef CONFIG_SYSFS
+	return sb->s_magic == SYSFS_MAGIC;
+#else
+	return 0;
+#endif
+}
+
+static inline int au_test_configfs(struct super_block *sb __maybe_unused)
+{
+#if IS_ENABLED(CONFIG_CONFIGFS_FS)
+	return sb->s_magic == CONFIGFS_MAGIC;
+#else
+	return 0;
+#endif
+}
+
+static inline int au_test_minix(struct super_block *sb __maybe_unused)
+{
+#if IS_ENABLED(CONFIG_MINIX_FS)
+	return sb->s_magic == MINIX3_SUPER_MAGIC
+		|| sb->s_magic == MINIX2_SUPER_MAGIC
+		|| sb->s_magic == MINIX2_SUPER_MAGIC2
+		|| sb->s_magic == MINIX_SUPER_MAGIC
+		|| sb->s_magic == MINIX_SUPER_MAGIC2;
+#else
+	return 0;
+#endif
+}
+
+static inline int au_test_fat(struct super_block *sb __maybe_unused)
+{
+#if IS_ENABLED(CONFIG_FAT_FS)
+	return sb->s_magic == MSDOS_SUPER_MAGIC;
+#else
+	return 0;
+#endif
+}
+
+static inline int au_test_msdos(struct super_block *sb)
+{
+	return au_test_fat(sb);
+}
+
+static inline int au_test_vfat(struct super_block *sb)
+{
+	return au_test_fat(sb);
+}
+
+static inline int au_test_securityfs(struct super_block *sb __maybe_unused)
+{
+#ifdef CONFIG_SECURITYFS
+	return sb->s_magic == SECURITYFS_MAGIC;
+#else
+	return 0;
+#endif
+}
+
+static inline int au_test_squashfs(struct super_block *sb __maybe_unused)
+{
+#if IS_ENABLED(CONFIG_SQUASHFS)
+	return sb->s_magic == SQUASHFS_MAGIC;
+#else
+	return 0;
+#endif
+}
+
+static inline int au_test_btrfs(struct super_block *sb __maybe_unused)
+{
+#if IS_ENABLED(CONFIG_BTRFS_FS)
+	return sb->s_magic == BTRFS_SUPER_MAGIC;
+#else
+	return 0;
+#endif
+}
+
+static inline int au_test_xenfs(struct super_block *sb __maybe_unused)
+{
+#if IS_ENABLED(CONFIG_XENFS)
+	return sb->s_magic == XENFS_SUPER_MAGIC;
+#else
+	return 0;
+#endif
+}
+
+static inline int au_test_debugfs(struct super_block *sb __maybe_unused)
+{
+#ifdef CONFIG_DEBUG_FS
+	return sb->s_magic == DEBUGFS_MAGIC;
+#else
+	return 0;
+#endif
+}
+
+static inline int au_test_nilfs(struct super_block *sb __maybe_unused)
+{
+#if IS_ENABLED(CONFIG_NILFS)
+	return sb->s_magic == NILFS_SUPER_MAGIC;
+#else
+	return 0;
+#endif
+}
+
+static inline int au_test_hfsplus(struct super_block *sb __maybe_unused)
+{
+#if IS_ENABLED(CONFIG_HFSPLUS_FS)
+	return sb->s_magic == HFSPLUS_SUPER_MAGIC;
+#else
+	return 0;
+#endif
+}
+
+/* ---------------------------------------------------------------------- */
+/*
+ * they can't be an aufs branch.
+ */
+static inline int au_test_fs_unsuppoted(struct super_block *sb)
+{
+	return
+#ifndef CONFIG_AUFS_BR_RAMFS
+		au_test_ramfs(sb) ||
+#endif
+		au_test_procfs(sb)
+		|| au_test_sysfs(sb)
+		|| au_test_configfs(sb)
+		|| au_test_debugfs(sb)
+		|| au_test_securityfs(sb)
+		|| au_test_xenfs(sb)
+		|| au_test_ecryptfs(sb)
+		/* || !strcmp(au_sbtype(sb), "unionfs") */
+		|| au_test_aufs(sb); /* will be supported in next version */
+}
+
+static inline int au_test_fs_remote(struct super_block *sb)
+{
+	return !au_test_tmpfs(sb)
+#ifdef CONFIG_AUFS_BR_RAMFS
+		&& !au_test_ramfs(sb)
+#endif
+		&& !(sb->s_type->fs_flags & FS_REQUIRES_DEV);
+}
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * Note: these functions (below) are created after reading ->getattr() in all
+ * filesystems under linux/fs. it means we have to do so in every update...
+ */
+
+/*
+ * some filesystems require getattr to refresh the inode attributes before
+ * referencing.
+ * in most cases, we can rely on the inode attribute in NFS (or every remote fs)
+ * and leave the work for d_revalidate()
+ */
+static inline int au_test_fs_refresh_iattr(struct super_block *sb)
+{
+	return au_test_nfs(sb)
+		|| au_test_fuse(sb)
+		/* || au_test_btrfs(sb) */	/* untested */
+		;
+}
+
+/*
+ * filesystems which don't maintain i_size or i_blocks.
+ */
+static inline int au_test_fs_bad_iattr_size(struct super_block *sb)
+{
+	return au_test_xfs(sb)
+		|| au_test_btrfs(sb)
+		|| au_test_ubifs(sb)
+		|| au_test_hfsplus(sb)	/* maintained, but incorrect */
+		/* || au_test_minix(sb) */	/* untested */
+		;
+}
+
+/*
+ * filesystems which don't store the correct value in some of their inode
+ * attributes.
+ */
+static inline int au_test_fs_bad_iattr(struct super_block *sb)
+{
+	return au_test_fs_bad_iattr_size(sb)
+		|| au_test_fat(sb)
+		|| au_test_msdos(sb)
+		|| au_test_vfat(sb);
+}
+
+/* they don't check i_nlink in link(2) */
+static inline int au_test_fs_no_limit_nlink(struct super_block *sb)
+{
+	return au_test_tmpfs(sb)
+#ifdef CONFIG_AUFS_BR_RAMFS
+		|| au_test_ramfs(sb)
+#endif
+		|| au_test_ubifs(sb)
+		|| au_test_hfsplus(sb);
+}
+
+/*
+ * filesystems which sets S_NOATIME and S_NOCMTIME.
+ */
+static inline int au_test_fs_notime(struct super_block *sb)
+{
+	return au_test_nfs(sb)
+		|| au_test_fuse(sb)
+		|| au_test_ubifs(sb)
+		;
+}
+
+/* temporary support for i#1 in cramfs */
+static inline int au_test_fs_unique_ino(struct inode *inode)
+{
+	if (au_test_cramfs(inode->i_sb))
+		return inode->i_ino != 1;
+	return 1;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * the filesystem where the xino files placed must support i/o after unlink and
+ * maintain i_size and i_blocks.
+ */
+static inline int au_test_fs_bad_xino(struct super_block *sb)
+{
+	return au_test_fs_remote(sb)
+		|| au_test_fs_bad_iattr_size(sb)
+		/* don't want unnecessary work for xino */
+		|| au_test_aufs(sb)
+		|| au_test_ecryptfs(sb)
+		|| au_test_nilfs(sb);
+}
+
+static inline int au_test_fs_trunc_xino(struct super_block *sb)
+{
+	return au_test_tmpfs(sb)
+		|| au_test_ramfs(sb);
+}
+
+/*
+ * test if the @sb is real-readonly.
+ */
+static inline int au_test_fs_rr(struct super_block *sb)
+{
+	return au_test_squashfs(sb)
+		|| au_test_iso9660(sb)
+		|| au_test_cramfs(sb)
+		|| au_test_romfs(sb);
+}
+
+/*
+ * test if the @inode is nfs with 'noacl' option
+ * NFS always sets SB_POSIXACL regardless its mount option 'noacl.'
+ */
+static inline int au_test_nfs_noacl(struct inode *inode)
+{
+	return au_test_nfs(inode->i_sb)
+		/* && IS_POSIXACL(inode) */
+		&& !nfs_server_capable(inode, NFS_CAP_ACLS);
+}
+
+#endif /* __KERNEL__ */
+#endif /* __AUFS_FSTYPE_H__ */
diff -Naur --no-dereference a/fs/aufs/hbl.h b/fs/aufs/hbl.h
--- a/fs/aufs/hbl.h	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/hbl.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,65 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2017-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * helpers for hlist_bl.h
+ */
+
+#ifndef __AUFS_HBL_H__
+#define __AUFS_HBL_H__
+
+#ifdef __KERNEL__
+
+#include <linux/list_bl.h>
+
+static inline void au_hbl_add(struct hlist_bl_node *node,
+			      struct hlist_bl_head *hbl)
+{
+	hlist_bl_lock(hbl);
+	hlist_bl_add_head(node, hbl);
+	hlist_bl_unlock(hbl);
+}
+
+static inline void au_hbl_del(struct hlist_bl_node *node,
+			      struct hlist_bl_head *hbl)
+{
+	hlist_bl_lock(hbl);
+	hlist_bl_del(node);
+	hlist_bl_unlock(hbl);
+}
+
+#define au_hbl_for_each(pos, head)					\
+	for (pos = hlist_bl_first(head);				\
+	     pos;							\
+	     pos = pos->next)
+
+static inline unsigned long au_hbl_count(struct hlist_bl_head *hbl)
+{
+	unsigned long cnt;
+	struct hlist_bl_node *pos;
+
+	cnt = 0;
+	hlist_bl_lock(hbl);
+	au_hbl_for_each(pos, hbl)
+		cnt++;
+	hlist_bl_unlock(hbl);
+	return cnt;
+}
+
+#endif /* __KERNEL__ */
+#endif /* __AUFS_HBL_H__ */
diff -Naur --no-dereference a/fs/aufs/hfsnotify.c b/fs/aufs/hfsnotify.c
--- a/fs/aufs/hfsnotify.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/hfsnotify.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,289 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * fsnotify for the lower directories
+ */
+
+#include "aufs.h"
+
+/* FS_IN_IGNORED is unnecessary */
+static const __u32 AuHfsnMask = (FS_MOVED_TO | FS_MOVED_FROM | FS_DELETE
+				 | FS_CREATE | FS_EVENT_ON_CHILD);
+static DECLARE_WAIT_QUEUE_HEAD(au_hfsn_wq);
+static __cacheline_aligned_in_smp atomic64_t au_hfsn_ifree = ATOMIC64_INIT(0);
+
+static void au_hfsn_free_mark(struct fsnotify_mark *mark)
+{
+	struct au_hnotify *hn = container_of(mark, struct au_hnotify,
+					     hn_mark);
+	/* AuDbg("here\n"); */
+	au_cache_free_hnotify(hn);
+	smp_mb__before_atomic(); /* for atomic64_dec */
+	if (atomic64_dec_and_test(&au_hfsn_ifree))
+		wake_up(&au_hfsn_wq);
+}
+
+static int au_hfsn_alloc(struct au_hinode *hinode)
+{
+	int err;
+	struct au_hnotify *hn;
+	struct super_block *sb;
+	struct au_branch *br;
+	struct fsnotify_mark *mark;
+	aufs_bindex_t bindex;
+
+	hn = hinode->hi_notify;
+	sb = hn->hn_aufs_inode->i_sb;
+	bindex = au_br_index(sb, hinode->hi_id);
+	br = au_sbr(sb, bindex);
+	AuDebugOn(!br->br_hfsn);
+
+	mark = &hn->hn_mark;
+	fsnotify_init_mark(mark, br->br_hfsn->hfsn_group);
+	mark->mask = AuHfsnMask;
+	/*
+	 * by udba rename or rmdir, aufs assign a new inode to the known
+	 * h_inode, so specify 1 to allow dups.
+	 */
+	lockdep_off();
+	err = fsnotify_add_inode_mark(mark, hinode->hi_inode, /*allow_dups*/1);
+	lockdep_on();
+
+	return err;
+}
+
+static int au_hfsn_free(struct au_hinode *hinode, struct au_hnotify *hn)
+{
+	struct fsnotify_mark *mark;
+	unsigned long long ull;
+	struct fsnotify_group *group;
+
+	ull = atomic64_inc_return(&au_hfsn_ifree);
+	BUG_ON(!ull);
+
+	mark = &hn->hn_mark;
+	spin_lock(&mark->lock);
+	group = mark->group;
+	fsnotify_get_group(group);
+	spin_unlock(&mark->lock);
+	lockdep_off();
+	fsnotify_destroy_mark(mark, group);
+	fsnotify_put_mark(mark);
+	fsnotify_put_group(group);
+	lockdep_on();
+
+	/* free hn by myself */
+	return 0;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static void au_hfsn_ctl(struct au_hinode *hinode, int do_set)
+{
+	struct fsnotify_mark *mark;
+
+	mark = &hinode->hi_notify->hn_mark;
+	spin_lock(&mark->lock);
+	if (do_set) {
+		AuDebugOn(mark->mask & AuHfsnMask);
+		mark->mask |= AuHfsnMask;
+	} else {
+		AuDebugOn(!(mark->mask & AuHfsnMask));
+		mark->mask &= ~AuHfsnMask;
+	}
+	spin_unlock(&mark->lock);
+	/* fsnotify_recalc_inode_mask(hinode->hi_inode); */
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* #define AuDbgHnotify */
+#ifdef AuDbgHnotify
+static char *au_hfsn_name(u32 mask)
+{
+#ifdef CONFIG_AUFS_DEBUG
+#define test_ret(flag)				\
+	do {					\
+		if (mask & flag)		\
+			return #flag;		\
+	} while (0)
+	test_ret(FS_ACCESS);
+	test_ret(FS_MODIFY);
+	test_ret(FS_ATTRIB);
+	test_ret(FS_CLOSE_WRITE);
+	test_ret(FS_CLOSE_NOWRITE);
+	test_ret(FS_OPEN);
+	test_ret(FS_MOVED_FROM);
+	test_ret(FS_MOVED_TO);
+	test_ret(FS_CREATE);
+	test_ret(FS_DELETE);
+	test_ret(FS_DELETE_SELF);
+	test_ret(FS_MOVE_SELF);
+	test_ret(FS_UNMOUNT);
+	test_ret(FS_Q_OVERFLOW);
+	test_ret(FS_IN_IGNORED);
+	test_ret(FS_ISDIR);
+	test_ret(FS_IN_ONESHOT);
+	test_ret(FS_EVENT_ON_CHILD);
+	return "";
+#undef test_ret
+#else
+	return "??";
+#endif
+}
+#endif
+
+/* ---------------------------------------------------------------------- */
+
+static void au_hfsn_free_group(struct fsnotify_group *group)
+{
+	struct au_br_hfsnotify *hfsn = group->private;
+
+	/* AuDbg("here\n"); */
+	au_kfree_try_rcu(hfsn);
+}
+
+static int au_hfsn_handle_event(struct fsnotify_group *group,
+				struct inode *inode,
+				u32 mask, const void *data, int data_type,
+				const unsigned char *file_name, u32 cookie,
+				struct fsnotify_iter_info *iter_info)
+{
+	int err;
+	struct au_hnotify *hnotify;
+	struct inode *h_dir, *h_inode;
+	struct qstr h_child_qstr = QSTR_INIT(file_name, strlen(file_name));
+	struct fsnotify_mark *inode_mark;
+
+	AuDebugOn(data_type != FSNOTIFY_EVENT_INODE);
+
+	err = 0;
+	/* if FS_UNMOUNT happens, there must be another bug */
+	AuDebugOn(mask & FS_UNMOUNT);
+	if (mask & (FS_IN_IGNORED | FS_UNMOUNT))
+		goto out;
+
+	h_dir = inode;
+	h_inode = NULL;
+#ifdef AuDbgHnotify
+	au_debug_on();
+	if (1 || h_child_qstr.len != sizeof(AUFS_XINO_FNAME) - 1
+	    || strncmp(h_child_qstr.name, AUFS_XINO_FNAME, h_child_qstr.len)) {
+		AuDbg("i%lu, mask 0x%x %s, hcname %.*s, hi%lu\n",
+		      h_dir->i_ino, mask, au_hfsn_name(mask),
+		      AuLNPair(&h_child_qstr), h_inode ? h_inode->i_ino : 0);
+		/* WARN_ON(1); */
+	}
+	au_debug_off();
+#endif
+
+	inode_mark = fsnotify_iter_inode_mark(iter_info);
+	AuDebugOn(!inode_mark);
+	hnotify = container_of(inode_mark, struct au_hnotify, hn_mark);
+	err = au_hnotify(h_dir, hnotify, mask, &h_child_qstr, h_inode);
+
+out:
+	return err;
+}
+
+static struct fsnotify_ops au_hfsn_ops = {
+	.handle_event		= au_hfsn_handle_event,
+	.free_group_priv	= au_hfsn_free_group,
+	.free_mark		= au_hfsn_free_mark
+};
+
+/* ---------------------------------------------------------------------- */
+
+static void au_hfsn_fin_br(struct au_branch *br)
+{
+	struct au_br_hfsnotify *hfsn;
+
+	hfsn = br->br_hfsn;
+	if (hfsn) {
+		lockdep_off();
+		fsnotify_put_group(hfsn->hfsn_group);
+		lockdep_on();
+	}
+}
+
+static int au_hfsn_init_br(struct au_branch *br, int perm)
+{
+	int err;
+	struct fsnotify_group *group;
+	struct au_br_hfsnotify *hfsn;
+
+	err = 0;
+	br->br_hfsn = NULL;
+	if (!au_br_hnotifyable(perm))
+		goto out;
+
+	err = -ENOMEM;
+	hfsn = kmalloc(sizeof(*hfsn), GFP_NOFS);
+	if (unlikely(!hfsn))
+		goto out;
+
+	err = 0;
+	group = fsnotify_alloc_group(&au_hfsn_ops);
+	if (IS_ERR(group)) {
+		err = PTR_ERR(group);
+		pr_err("fsnotify_alloc_group() failed, %d\n", err);
+		goto out_hfsn;
+	}
+
+	group->private = hfsn;
+	hfsn->hfsn_group = group;
+	br->br_hfsn = hfsn;
+	goto out; /* success */
+
+out_hfsn:
+	au_kfree_try_rcu(hfsn);
+out:
+	return err;
+}
+
+static int au_hfsn_reset_br(unsigned int udba, struct au_branch *br, int perm)
+{
+	int err;
+
+	err = 0;
+	if (!br->br_hfsn)
+		err = au_hfsn_init_br(br, perm);
+
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static void au_hfsn_fin(void)
+{
+	AuDbg("au_hfsn_ifree %lld\n", (long long)atomic64_read(&au_hfsn_ifree));
+	wait_event(au_hfsn_wq, !atomic64_read(&au_hfsn_ifree));
+}
+
+const struct au_hnotify_op au_hnotify_op = {
+	.ctl		= au_hfsn_ctl,
+	.alloc		= au_hfsn_alloc,
+	.free		= au_hfsn_free,
+
+	.fin		= au_hfsn_fin,
+
+	.reset_br	= au_hfsn_reset_br,
+	.fin_br		= au_hfsn_fin_br,
+	.init_br	= au_hfsn_init_br
+};
diff -Naur --no-dereference a/fs/aufs/hfsplus.c b/fs/aufs/hfsplus.c
--- a/fs/aufs/hfsplus.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/hfsplus.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2010-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * special support for filesystems which acquires an inode mutex
+ * at final closing a file, eg, hfsplus.
+ *
+ * This trick is very simple and stupid, just to open the file before really
+ * necessary open to tell hfsplus that this is not the final closing.
+ * The caller should call au_h_open_pre() after acquiring the inode mutex,
+ * and au_h_open_post() after releasing it.
+ */
+
+#include "aufs.h"
+
+struct file *au_h_open_pre(struct dentry *dentry, aufs_bindex_t bindex,
+			   int force_wr)
+{
+	struct file *h_file;
+	struct dentry *h_dentry;
+
+	h_dentry = au_h_dptr(dentry, bindex);
+	AuDebugOn(!h_dentry);
+	AuDebugOn(d_is_negative(h_dentry));
+
+	h_file = NULL;
+	if (au_test_hfsplus(h_dentry->d_sb)
+	    && d_is_reg(h_dentry))
+		h_file = au_h_open(dentry, bindex,
+				   O_RDONLY | O_NOATIME | O_LARGEFILE,
+				   /*file*/NULL, force_wr);
+	return h_file;
+}
+
+void au_h_open_post(struct dentry *dentry, aufs_bindex_t bindex,
+		    struct file *h_file)
+{
+	struct au_branch *br;
+
+	if (h_file) {
+		fput(h_file);
+		br = au_sbr(dentry->d_sb, bindex);
+		au_lcnt_dec(&br->br_nfiles);
+	}
+}
diff -Naur --no-dereference a/fs/aufs/hnotify.c b/fs/aufs/hnotify.c
--- a/fs/aufs/hnotify.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/hnotify.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,715 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * abstraction to notify the direct changes on lower directories
+ */
+
+/* #include <linux/iversion.h> */
+#include "aufs.h"
+
+int au_hn_alloc(struct au_hinode *hinode, struct inode *inode)
+{
+	int err;
+	struct au_hnotify *hn;
+
+	err = -ENOMEM;
+	hn = au_cache_alloc_hnotify();
+	if (hn) {
+		hn->hn_aufs_inode = inode;
+		hinode->hi_notify = hn;
+		err = au_hnotify_op.alloc(hinode);
+		AuTraceErr(err);
+		if (unlikely(err)) {
+			hinode->hi_notify = NULL;
+			au_cache_free_hnotify(hn);
+			/*
+			 * The upper dir was removed by udba, but the same named
+			 * dir left. In this case, aufs assigns a new inode
+			 * number and set the monitor again.
+			 * For the lower dir, the old monitor is still left.
+			 */
+			if (err == -EEXIST)
+				err = 0;
+		}
+	}
+
+	AuTraceErr(err);
+	return err;
+}
+
+void au_hn_free(struct au_hinode *hinode)
+{
+	struct au_hnotify *hn;
+
+	hn = hinode->hi_notify;
+	if (hn) {
+		hinode->hi_notify = NULL;
+		if (au_hnotify_op.free(hinode, hn))
+			au_cache_free_hnotify(hn);
+	}
+}
+
+/* ---------------------------------------------------------------------- */
+
+void au_hn_ctl(struct au_hinode *hinode, int do_set)
+{
+	if (hinode->hi_notify)
+		au_hnotify_op.ctl(hinode, do_set);
+}
+
+void au_hn_reset(struct inode *inode, unsigned int flags)
+{
+	aufs_bindex_t bindex, bbot;
+	struct inode *hi;
+	struct dentry *iwhdentry;
+
+	bbot = au_ibbot(inode);
+	for (bindex = au_ibtop(inode); bindex <= bbot; bindex++) {
+		hi = au_h_iptr(inode, bindex);
+		if (!hi)
+			continue;
+
+		/* inode_lock_nested(hi, AuLsc_I_CHILD); */
+		iwhdentry = au_hi_wh(inode, bindex);
+		if (iwhdentry)
+			dget(iwhdentry);
+		au_igrab(hi);
+		au_set_h_iptr(inode, bindex, NULL, 0);
+		au_set_h_iptr(inode, bindex, au_igrab(hi),
+			      flags & ~AuHi_XINO);
+		iput(hi);
+		dput(iwhdentry);
+		/* inode_unlock(hi); */
+	}
+}
+
+/* ---------------------------------------------------------------------- */
+
+static int hn_xino(struct inode *inode, struct inode *h_inode)
+{
+	int err;
+	aufs_bindex_t bindex, bbot, bfound, btop;
+	struct inode *h_i;
+
+	err = 0;
+	if (unlikely(inode->i_ino == AUFS_ROOT_INO)) {
+		pr_warn("branch root dir was changed\n");
+		goto out;
+	}
+
+	bfound = -1;
+	bbot = au_ibbot(inode);
+	btop = au_ibtop(inode);
+#if 0 /* reserved for future use */
+	if (bindex == bbot) {
+		/* keep this ino in rename case */
+		goto out;
+	}
+#endif
+	for (bindex = btop; bindex <= bbot; bindex++)
+		if (au_h_iptr(inode, bindex) == h_inode) {
+			bfound = bindex;
+			break;
+		}
+	if (bfound < 0)
+		goto out;
+
+	for (bindex = btop; bindex <= bbot; bindex++) {
+		h_i = au_h_iptr(inode, bindex);
+		if (!h_i)
+			continue;
+
+		err = au_xino_write(inode->i_sb, bindex, h_i->i_ino, /*ino*/0);
+		/* ignore this error */
+		/* bad action? */
+	}
+
+	/* children inode number will be broken */
+
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+static int hn_gen_tree(struct dentry *dentry)
+{
+	int err, i, j, ndentry;
+	struct au_dcsub_pages dpages;
+	struct au_dpage *dpage;
+	struct dentry **dentries;
+
+	err = au_dpages_init(&dpages, GFP_NOFS);
+	if (unlikely(err))
+		goto out;
+	err = au_dcsub_pages(&dpages, dentry, NULL, NULL);
+	if (unlikely(err))
+		goto out_dpages;
+
+	for (i = 0; i < dpages.ndpage; i++) {
+		dpage = dpages.dpages + i;
+		dentries = dpage->dentries;
+		ndentry = dpage->ndentry;
+		for (j = 0; j < ndentry; j++) {
+			struct dentry *d;
+
+			d = dentries[j];
+			if (IS_ROOT(d))
+				continue;
+
+			au_digen_dec(d);
+			if (d_really_is_positive(d))
+				/* todo: reset children xino?
+				   cached children only? */
+				au_iigen_dec(d_inode(d));
+		}
+	}
+
+out_dpages:
+	au_dpages_free(&dpages);
+out:
+	return err;
+}
+
+/*
+ * return 0 if processed.
+ */
+static int hn_gen_by_inode(char *name, unsigned int nlen, struct inode *inode,
+			   const unsigned int isdir)
+{
+	int err;
+	struct dentry *d;
+	struct qstr *dname;
+
+	err = 1;
+	if (unlikely(inode->i_ino == AUFS_ROOT_INO)) {
+		pr_warn("branch root dir was changed\n");
+		err = 0;
+		goto out;
+	}
+
+	if (!isdir) {
+		AuDebugOn(!name);
+		au_iigen_dec(inode);
+		spin_lock(&inode->i_lock);
+		hlist_for_each_entry(d, &inode->i_dentry, d_u.d_alias) {
+			spin_lock(&d->d_lock);
+			dname = &d->d_name;
+			if (dname->len != nlen
+			    && memcmp(dname->name, name, nlen)) {
+				spin_unlock(&d->d_lock);
+				continue;
+			}
+			err = 0;
+			au_digen_dec(d);
+			spin_unlock(&d->d_lock);
+			break;
+		}
+		spin_unlock(&inode->i_lock);
+	} else {
+		au_fset_si(au_sbi(inode->i_sb), FAILED_REFRESH_DIR);
+		d = d_find_any_alias(inode);
+		if (!d) {
+			au_iigen_dec(inode);
+			goto out;
+		}
+
+		spin_lock(&d->d_lock);
+		dname = &d->d_name;
+		if (dname->len == nlen && !memcmp(dname->name, name, nlen)) {
+			spin_unlock(&d->d_lock);
+			err = hn_gen_tree(d);
+			spin_lock(&d->d_lock);
+		}
+		spin_unlock(&d->d_lock);
+		dput(d);
+	}
+
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+static int hn_gen_by_name(struct dentry *dentry, const unsigned int isdir)
+{
+	int err;
+
+	if (IS_ROOT(dentry)) {
+		pr_warn("branch root dir was changed\n");
+		return 0;
+	}
+
+	err = 0;
+	if (!isdir) {
+		au_digen_dec(dentry);
+		if (d_really_is_positive(dentry))
+			au_iigen_dec(d_inode(dentry));
+	} else {
+		au_fset_si(au_sbi(dentry->d_sb), FAILED_REFRESH_DIR);
+		if (d_really_is_positive(dentry))
+			err = hn_gen_tree(dentry);
+	}
+
+	AuTraceErr(err);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* hnotify job flags */
+#define AuHnJob_XINO0		1
+#define AuHnJob_GEN		(1 << 1)
+#define AuHnJob_DIRENT		(1 << 2)
+#define AuHnJob_ISDIR		(1 << 3)
+#define AuHnJob_TRYXINO0	(1 << 4)
+#define AuHnJob_MNTPNT		(1 << 5)
+#define au_ftest_hnjob(flags, name)	((flags) & AuHnJob_##name)
+#define au_fset_hnjob(flags, name) \
+	do { (flags) |= AuHnJob_##name; } while (0)
+#define au_fclr_hnjob(flags, name) \
+	do { (flags) &= ~AuHnJob_##name; } while (0)
+
+enum {
+	AuHn_CHILD,
+	AuHn_PARENT,
+	AuHnLast
+};
+
+struct au_hnotify_args {
+	struct inode *h_dir, *dir, *h_child_inode;
+	u32 mask;
+	unsigned int flags[AuHnLast];
+	unsigned int h_child_nlen;
+	char h_child_name[];
+};
+
+struct hn_job_args {
+	unsigned int flags;
+	struct inode *inode, *h_inode, *dir, *h_dir;
+	struct dentry *dentry;
+	char *h_name;
+	int h_nlen;
+};
+
+static int hn_job(struct hn_job_args *a)
+{
+	const unsigned int isdir = au_ftest_hnjob(a->flags, ISDIR);
+	int e;
+
+	/* reset xino */
+	if (au_ftest_hnjob(a->flags, XINO0) && a->inode)
+		hn_xino(a->inode, a->h_inode); /* ignore this error */
+
+	if (au_ftest_hnjob(a->flags, TRYXINO0)
+	    && a->inode
+	    && a->h_inode) {
+		inode_lock_shared_nested(a->h_inode, AuLsc_I_CHILD);
+		if (!a->h_inode->i_nlink
+		    && !(a->h_inode->i_state & I_LINKABLE))
+			hn_xino(a->inode, a->h_inode); /* ignore this error */
+		inode_unlock_shared(a->h_inode);
+	}
+
+	/* make the generation obsolete */
+	if (au_ftest_hnjob(a->flags, GEN)) {
+		e = -1;
+		if (a->inode)
+			e = hn_gen_by_inode(a->h_name, a->h_nlen, a->inode,
+					      isdir);
+		if (e && a->dentry)
+			hn_gen_by_name(a->dentry, isdir);
+		/* ignore this error */
+	}
+
+	/* make dir entries obsolete */
+	if (au_ftest_hnjob(a->flags, DIRENT) && a->inode) {
+		struct au_vdir *vdir;
+
+		vdir = au_ivdir(a->inode);
+		if (vdir)
+			vdir->vd_jiffy = 0;
+		/* IMustLock(a->inode); */
+		/* inode_inc_iversion(a->inode); */
+	}
+
+	/* can do nothing but warn */
+	if (au_ftest_hnjob(a->flags, MNTPNT)
+	    && a->dentry
+	    && d_mountpoint(a->dentry))
+		pr_warn("mount-point %pd is removed or renamed\n", a->dentry);
+
+	return 0;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static struct dentry *lookup_wlock_by_name(char *name, unsigned int nlen,
+					   struct inode *dir)
+{
+	struct dentry *dentry, *d, *parent;
+	struct qstr *dname;
+
+	parent = d_find_any_alias(dir);
+	if (!parent)
+		return NULL;
+
+	dentry = NULL;
+	spin_lock(&parent->d_lock);
+	list_for_each_entry(d, &parent->d_subdirs, d_child) {
+		/* AuDbg("%pd\n", d); */
+		spin_lock_nested(&d->d_lock, DENTRY_D_LOCK_NESTED);
+		dname = &d->d_name;
+		if (dname->len != nlen || memcmp(dname->name, name, nlen))
+			goto cont_unlock;
+		if (au_di(d))
+			au_digen_dec(d);
+		else
+			goto cont_unlock;
+		if (au_dcount(d) > 0) {
+			dentry = dget_dlock(d);
+			spin_unlock(&d->d_lock);
+			break;
+		}
+
+cont_unlock:
+		spin_unlock(&d->d_lock);
+	}
+	spin_unlock(&parent->d_lock);
+	dput(parent);
+
+	if (dentry)
+		di_write_lock_child(dentry);
+
+	return dentry;
+}
+
+static struct inode *lookup_wlock_by_ino(struct super_block *sb,
+					 aufs_bindex_t bindex, ino_t h_ino)
+{
+	struct inode *inode;
+	ino_t ino;
+	int err;
+
+	inode = NULL;
+	err = au_xino_read(sb, bindex, h_ino, &ino);
+	if (!err && ino)
+		inode = ilookup(sb, ino);
+	if (!inode)
+		goto out;
+
+	if (unlikely(inode->i_ino == AUFS_ROOT_INO)) {
+		pr_warn("wrong root branch\n");
+		iput(inode);
+		inode = NULL;
+		goto out;
+	}
+
+	ii_write_lock_child(inode);
+
+out:
+	return inode;
+}
+
+static void au_hn_bh(void *_args)
+{
+	struct au_hnotify_args *a = _args;
+	struct super_block *sb;
+	aufs_bindex_t bindex, bbot, bfound;
+	unsigned char xino, try_iput;
+	int err;
+	struct inode *inode;
+	ino_t h_ino;
+	struct hn_job_args args;
+	struct dentry *dentry;
+	struct au_sbinfo *sbinfo;
+
+	AuDebugOn(!_args);
+	AuDebugOn(!a->h_dir);
+	AuDebugOn(!a->dir);
+	AuDebugOn(!a->mask);
+	AuDbg("mask 0x%x, i%lu, hi%lu, hci%lu\n",
+	      a->mask, a->dir->i_ino, a->h_dir->i_ino,
+	      a->h_child_inode ? a->h_child_inode->i_ino : 0);
+
+	inode = NULL;
+	dentry = NULL;
+	/*
+	 * do not lock a->dir->i_mutex here
+	 * because of d_revalidate() may cause a deadlock.
+	 */
+	sb = a->dir->i_sb;
+	AuDebugOn(!sb);
+	sbinfo = au_sbi(sb);
+	AuDebugOn(!sbinfo);
+	si_write_lock(sb, AuLock_NOPLMW);
+
+	if (au_opt_test(sbinfo->si_mntflags, DIRREN))
+		switch (a->mask & FS_EVENTS_POSS_ON_CHILD) {
+		case FS_MOVED_FROM:
+		case FS_MOVED_TO:
+			AuWarn1("DIRREN with UDBA may not work correctly "
+				"for the direct rename(2)\n");
+		}
+
+	ii_read_lock_parent(a->dir);
+	bfound = -1;
+	bbot = au_ibbot(a->dir);
+	for (bindex = au_ibtop(a->dir); bindex <= bbot; bindex++)
+		if (au_h_iptr(a->dir, bindex) == a->h_dir) {
+			bfound = bindex;
+			break;
+		}
+	ii_read_unlock(a->dir);
+	if (unlikely(bfound < 0))
+		goto out;
+
+	xino = !!au_opt_test(au_mntflags(sb), XINO);
+	h_ino = 0;
+	if (a->h_child_inode)
+		h_ino = a->h_child_inode->i_ino;
+
+	if (a->h_child_nlen
+	    && (au_ftest_hnjob(a->flags[AuHn_CHILD], GEN)
+		|| au_ftest_hnjob(a->flags[AuHn_CHILD], MNTPNT)))
+		dentry = lookup_wlock_by_name(a->h_child_name, a->h_child_nlen,
+					      a->dir);
+	try_iput = 0;
+	if (dentry && d_really_is_positive(dentry))
+		inode = d_inode(dentry);
+	if (xino && !inode && h_ino
+	    && (au_ftest_hnjob(a->flags[AuHn_CHILD], XINO0)
+		|| au_ftest_hnjob(a->flags[AuHn_CHILD], TRYXINO0)
+		|| au_ftest_hnjob(a->flags[AuHn_CHILD], GEN))) {
+		inode = lookup_wlock_by_ino(sb, bfound, h_ino);
+		try_iput = 1;
+	}
+
+	args.flags = a->flags[AuHn_CHILD];
+	args.dentry = dentry;
+	args.inode = inode;
+	args.h_inode = a->h_child_inode;
+	args.dir = a->dir;
+	args.h_dir = a->h_dir;
+	args.h_name = a->h_child_name;
+	args.h_nlen = a->h_child_nlen;
+	err = hn_job(&args);
+	if (dentry) {
+		if (au_di(dentry))
+			di_write_unlock(dentry);
+		dput(dentry);
+	}
+	if (inode && try_iput) {
+		ii_write_unlock(inode);
+		iput(inode);
+	}
+
+	ii_write_lock_parent(a->dir);
+	args.flags = a->flags[AuHn_PARENT];
+	args.dentry = NULL;
+	args.inode = a->dir;
+	args.h_inode = a->h_dir;
+	args.dir = NULL;
+	args.h_dir = NULL;
+	args.h_name = NULL;
+	args.h_nlen = 0;
+	err = hn_job(&args);
+	ii_write_unlock(a->dir);
+
+out:
+	iput(a->h_child_inode);
+	iput(a->h_dir);
+	iput(a->dir);
+	si_write_unlock(sb);
+	au_nwt_done(&sbinfo->si_nowait);
+	au_kfree_rcu(a);
+}
+
+/* ---------------------------------------------------------------------- */
+
+int au_hnotify(struct inode *h_dir, struct au_hnotify *hnotify, u32 mask,
+	       struct qstr *h_child_qstr, struct inode *h_child_inode)
+{
+	int err, len;
+	unsigned int flags[AuHnLast], f;
+	unsigned char isdir, isroot, wh;
+	struct inode *dir;
+	struct au_hnotify_args *args;
+	char *p, *h_child_name;
+
+	err = 0;
+	AuDebugOn(!hnotify || !hnotify->hn_aufs_inode);
+	dir = igrab(hnotify->hn_aufs_inode);
+	if (!dir)
+		goto out;
+
+	isroot = (dir->i_ino == AUFS_ROOT_INO);
+	wh = 0;
+	h_child_name = (void *)h_child_qstr->name;
+	len = h_child_qstr->len;
+	if (h_child_name) {
+		if (len > AUFS_WH_PFX_LEN
+		    && !memcmp(h_child_name, AUFS_WH_PFX, AUFS_WH_PFX_LEN)) {
+			h_child_name += AUFS_WH_PFX_LEN;
+			len -= AUFS_WH_PFX_LEN;
+			wh = 1;
+		}
+	}
+
+	isdir = 0;
+	if (h_child_inode)
+		isdir = !!S_ISDIR(h_child_inode->i_mode);
+	flags[AuHn_PARENT] = AuHnJob_ISDIR;
+	flags[AuHn_CHILD] = 0;
+	if (isdir)
+		flags[AuHn_CHILD] = AuHnJob_ISDIR;
+	au_fset_hnjob(flags[AuHn_PARENT], DIRENT);
+	au_fset_hnjob(flags[AuHn_CHILD], GEN);
+	switch (mask & FS_EVENTS_POSS_ON_CHILD) {
+	case FS_MOVED_FROM:
+	case FS_MOVED_TO:
+		au_fset_hnjob(flags[AuHn_CHILD], XINO0);
+		au_fset_hnjob(flags[AuHn_CHILD], MNTPNT);
+		/*FALLTHROUGH*/
+	case FS_CREATE:
+		AuDebugOn(!h_child_name);
+		break;
+
+	case FS_DELETE:
+		/*
+		 * aufs never be able to get this child inode.
+		 * revalidation should be in d_revalidate()
+		 * by checking i_nlink, i_generation or d_unhashed().
+		 */
+		AuDebugOn(!h_child_name);
+		au_fset_hnjob(flags[AuHn_CHILD], TRYXINO0);
+		au_fset_hnjob(flags[AuHn_CHILD], MNTPNT);
+		break;
+
+	default:
+		AuDebugOn(1);
+	}
+
+	if (wh)
+		h_child_inode = NULL;
+
+	err = -ENOMEM;
+	/* iput() and kfree() will be called in au_hnotify() */
+	args = kmalloc(sizeof(*args) + len + 1, GFP_NOFS);
+	if (unlikely(!args)) {
+		AuErr1("no memory\n");
+		iput(dir);
+		goto out;
+	}
+	args->flags[AuHn_PARENT] = flags[AuHn_PARENT];
+	args->flags[AuHn_CHILD] = flags[AuHn_CHILD];
+	args->mask = mask;
+	args->dir = dir;
+	args->h_dir = igrab(h_dir);
+	if (h_child_inode)
+		h_child_inode = igrab(h_child_inode); /* can be NULL */
+	args->h_child_inode = h_child_inode;
+	args->h_child_nlen = len;
+	if (len) {
+		p = (void *)args;
+		p += sizeof(*args);
+		memcpy(p, h_child_name, len);
+		p[len] = 0;
+	}
+
+	/* NFS fires the event for silly-renamed one from kworker */
+	f = 0;
+	if (!dir->i_nlink
+	    || (au_test_nfs(h_dir->i_sb) && (mask & FS_DELETE)))
+		f = AuWkq_NEST;
+	err = au_wkq_nowait(au_hn_bh, args, dir->i_sb, f);
+	if (unlikely(err)) {
+		pr_err("wkq %d\n", err);
+		iput(args->h_child_inode);
+		iput(args->h_dir);
+		iput(args->dir);
+		au_kfree_rcu(args);
+	}
+
+out:
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+int au_hnotify_reset_br(unsigned int udba, struct au_branch *br, int perm)
+{
+	int err;
+
+	AuDebugOn(!(udba & AuOptMask_UDBA));
+
+	err = 0;
+	if (au_hnotify_op.reset_br)
+		err = au_hnotify_op.reset_br(udba, br, perm);
+
+	return err;
+}
+
+int au_hnotify_init_br(struct au_branch *br, int perm)
+{
+	int err;
+
+	err = 0;
+	if (au_hnotify_op.init_br)
+		err = au_hnotify_op.init_br(br, perm);
+
+	return err;
+}
+
+void au_hnotify_fin_br(struct au_branch *br)
+{
+	if (au_hnotify_op.fin_br)
+		au_hnotify_op.fin_br(br);
+}
+
+static void au_hn_destroy_cache(void)
+{
+	kmem_cache_destroy(au_cache[AuCache_HNOTIFY]);
+	au_cache[AuCache_HNOTIFY] = NULL;
+}
+
+int __init au_hnotify_init(void)
+{
+	int err;
+
+	err = -ENOMEM;
+	au_cache[AuCache_HNOTIFY] = AuCache(au_hnotify);
+	if (au_cache[AuCache_HNOTIFY]) {
+		err = 0;
+		if (au_hnotify_op.init)
+			err = au_hnotify_op.init();
+		if (unlikely(err))
+			au_hn_destroy_cache();
+	}
+	AuTraceErr(err);
+	return err;
+}
+
+void au_hnotify_fin(void)
+{
+	if (au_hnotify_op.fin)
+		au_hnotify_op.fin();
+
+	/* cf. au_cache_fin() */
+	if (au_cache[AuCache_HNOTIFY])
+		au_hn_destroy_cache();
+}
diff -Naur --no-dereference a/fs/aufs/iinfo.c b/fs/aufs/iinfo.c
--- a/fs/aufs/iinfo.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/iinfo.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,286 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * inode private data
+ */
+
+#include "aufs.h"
+
+struct inode *au_h_iptr(struct inode *inode, aufs_bindex_t bindex)
+{
+	struct inode *h_inode;
+	struct au_hinode *hinode;
+
+	IiMustAnyLock(inode);
+
+	hinode = au_hinode(au_ii(inode), bindex);
+	h_inode = hinode->hi_inode;
+	AuDebugOn(h_inode && atomic_read(&h_inode->i_count) <= 0);
+	return h_inode;
+}
+
+/* todo: hard/soft set? */
+void au_hiput(struct au_hinode *hinode)
+{
+	au_hn_free(hinode);
+	dput(hinode->hi_whdentry);
+	iput(hinode->hi_inode);
+}
+
+unsigned int au_hi_flags(struct inode *inode, int isdir)
+{
+	unsigned int flags;
+	const unsigned int mnt_flags = au_mntflags(inode->i_sb);
+
+	flags = 0;
+	if (au_opt_test(mnt_flags, XINO))
+		au_fset_hi(flags, XINO);
+	if (isdir && au_opt_test(mnt_flags, UDBA_HNOTIFY))
+		au_fset_hi(flags, HNOTIFY);
+	return flags;
+}
+
+void au_set_h_iptr(struct inode *inode, aufs_bindex_t bindex,
+		   struct inode *h_inode, unsigned int flags)
+{
+	struct au_hinode *hinode;
+	struct inode *hi;
+	struct au_iinfo *iinfo = au_ii(inode);
+
+	IiMustWriteLock(inode);
+
+	hinode = au_hinode(iinfo, bindex);
+	hi = hinode->hi_inode;
+	AuDebugOn(h_inode && atomic_read(&h_inode->i_count) <= 0);
+
+	if (hi)
+		au_hiput(hinode);
+	hinode->hi_inode = h_inode;
+	if (h_inode) {
+		int err;
+		struct super_block *sb = inode->i_sb;
+		struct au_branch *br;
+
+		AuDebugOn(inode->i_mode
+			  && (h_inode->i_mode & S_IFMT)
+			  != (inode->i_mode & S_IFMT));
+		if (bindex == iinfo->ii_btop)
+			au_cpup_igen(inode, h_inode);
+		br = au_sbr(sb, bindex);
+		hinode->hi_id = br->br_id;
+		if (au_ftest_hi(flags, XINO)) {
+			err = au_xino_write(sb, bindex, h_inode->i_ino,
+					    inode->i_ino);
+			if (unlikely(err))
+				AuIOErr1("failed au_xino_write() %d\n", err);
+		}
+
+		if (au_ftest_hi(flags, HNOTIFY)
+		    && au_br_hnotifyable(br->br_perm)) {
+			err = au_hn_alloc(hinode, inode);
+			if (unlikely(err))
+				AuIOErr1("au_hn_alloc() %d\n", err);
+		}
+	}
+}
+
+void au_set_hi_wh(struct inode *inode, aufs_bindex_t bindex,
+		  struct dentry *h_wh)
+{
+	struct au_hinode *hinode;
+
+	IiMustWriteLock(inode);
+
+	hinode = au_hinode(au_ii(inode), bindex);
+	AuDebugOn(hinode->hi_whdentry);
+	hinode->hi_whdentry = h_wh;
+}
+
+void au_update_iigen(struct inode *inode, int half)
+{
+	struct au_iinfo *iinfo;
+	struct au_iigen *iigen;
+	unsigned int sigen;
+
+	sigen = au_sigen(inode->i_sb);
+	iinfo = au_ii(inode);
+	iigen = &iinfo->ii_generation;
+	spin_lock(&iigen->ig_spin);
+	iigen->ig_generation = sigen;
+	if (half)
+		au_ig_fset(iigen->ig_flags, HALF_REFRESHED);
+	else
+		au_ig_fclr(iigen->ig_flags, HALF_REFRESHED);
+	spin_unlock(&iigen->ig_spin);
+}
+
+/* it may be called at remount time, too */
+void au_update_ibrange(struct inode *inode, int do_put_zero)
+{
+	struct au_iinfo *iinfo;
+	aufs_bindex_t bindex, bbot;
+
+	AuDebugOn(au_is_bad_inode(inode));
+	IiMustWriteLock(inode);
+
+	iinfo = au_ii(inode);
+	if (do_put_zero && iinfo->ii_btop >= 0) {
+		for (bindex = iinfo->ii_btop; bindex <= iinfo->ii_bbot;
+		     bindex++) {
+			struct inode *h_i;
+
+			h_i = au_hinode(iinfo, bindex)->hi_inode;
+			if (h_i
+			    && !h_i->i_nlink
+			    && !(h_i->i_state & I_LINKABLE))
+				au_set_h_iptr(inode, bindex, NULL, 0);
+		}
+	}
+
+	iinfo->ii_btop = -1;
+	iinfo->ii_bbot = -1;
+	bbot = au_sbbot(inode->i_sb);
+	for (bindex = 0; bindex <= bbot; bindex++)
+		if (au_hinode(iinfo, bindex)->hi_inode) {
+			iinfo->ii_btop = bindex;
+			break;
+		}
+	if (iinfo->ii_btop >= 0)
+		for (bindex = bbot; bindex >= iinfo->ii_btop; bindex--)
+			if (au_hinode(iinfo, bindex)->hi_inode) {
+				iinfo->ii_bbot = bindex;
+				break;
+			}
+	AuDebugOn(iinfo->ii_btop > iinfo->ii_bbot);
+}
+
+/* ---------------------------------------------------------------------- */
+
+void au_icntnr_init_once(void *_c)
+{
+	struct au_icntnr *c = _c;
+	struct au_iinfo *iinfo = &c->iinfo;
+
+	spin_lock_init(&iinfo->ii_generation.ig_spin);
+	au_rw_init(&iinfo->ii_rwsem);
+	inode_init_once(&c->vfs_inode);
+}
+
+void au_hinode_init(struct au_hinode *hinode)
+{
+	hinode->hi_inode = NULL;
+	hinode->hi_id = -1;
+	au_hn_init(hinode);
+	hinode->hi_whdentry = NULL;
+}
+
+int au_iinfo_init(struct inode *inode)
+{
+	struct au_iinfo *iinfo;
+	struct super_block *sb;
+	struct au_hinode *hi;
+	int nbr, i;
+
+	sb = inode->i_sb;
+	iinfo = &(container_of(inode, struct au_icntnr, vfs_inode)->iinfo);
+	nbr = au_sbbot(sb) + 1;
+	if (unlikely(nbr <= 0))
+		nbr = 1;
+	hi = kmalloc_array(nbr, sizeof(*iinfo->ii_hinode), GFP_NOFS);
+	if (hi) {
+		au_lcnt_inc(&au_sbi(sb)->si_ninodes);
+
+		iinfo->ii_hinode = hi;
+		for (i = 0; i < nbr; i++, hi++)
+			au_hinode_init(hi);
+
+		iinfo->ii_generation.ig_generation = au_sigen(sb);
+		iinfo->ii_btop = -1;
+		iinfo->ii_bbot = -1;
+		iinfo->ii_vdir = NULL;
+		return 0;
+	}
+	return -ENOMEM;
+}
+
+int au_hinode_realloc(struct au_iinfo *iinfo, int nbr, int may_shrink)
+{
+	int err, i;
+	struct au_hinode *hip;
+
+	AuRwMustWriteLock(&iinfo->ii_rwsem);
+
+	err = -ENOMEM;
+	hip = au_krealloc(iinfo->ii_hinode, sizeof(*hip) * nbr, GFP_NOFS,
+			  may_shrink);
+	if (hip) {
+		iinfo->ii_hinode = hip;
+		i = iinfo->ii_bbot + 1;
+		hip += i;
+		for (; i < nbr; i++, hip++)
+			au_hinode_init(hip);
+		err = 0;
+	}
+
+	return err;
+}
+
+void au_iinfo_fin(struct inode *inode)
+{
+	struct au_iinfo *iinfo;
+	struct au_hinode *hi;
+	struct super_block *sb;
+	aufs_bindex_t bindex, bbot;
+	const unsigned char unlinked = !inode->i_nlink;
+
+	AuDebugOn(au_is_bad_inode(inode));
+
+	sb = inode->i_sb;
+	au_lcnt_dec(&au_sbi(sb)->si_ninodes);
+	if (si_pid_test(sb))
+		au_xino_delete_inode(inode, unlinked);
+	else {
+		/*
+		 * it is safe to hide the dependency between sbinfo and
+		 * sb->s_umount.
+		 */
+		lockdep_off();
+		si_noflush_read_lock(sb);
+		au_xino_delete_inode(inode, unlinked);
+		si_read_unlock(sb);
+		lockdep_on();
+	}
+
+	iinfo = au_ii(inode);
+	if (iinfo->ii_vdir)
+		au_vdir_free(iinfo->ii_vdir);
+
+	bindex = iinfo->ii_btop;
+	if (bindex >= 0) {
+		hi = au_hinode(iinfo, bindex);
+		bbot = iinfo->ii_bbot;
+		while (bindex++ <= bbot) {
+			if (hi->hi_inode)
+				au_hiput(hi);
+			hi++;
+		}
+	}
+	au_kfree_rcu(iinfo->ii_hinode);
+	AuRwDestroy(&iinfo->ii_rwsem);
+}
diff -Naur --no-dereference a/fs/aufs/inode.c b/fs/aufs/inode.c
--- a/fs/aufs/inode.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/inode.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,529 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * inode functions
+ */
+
+#include <linux/iversion.h>
+#include "aufs.h"
+
+struct inode *au_igrab(struct inode *inode)
+{
+	if (inode) {
+		AuDebugOn(!atomic_read(&inode->i_count));
+		ihold(inode);
+	}
+	return inode;
+}
+
+static void au_refresh_hinode_attr(struct inode *inode, int do_version)
+{
+	au_cpup_attr_all(inode, /*force*/0);
+	au_update_iigen(inode, /*half*/1);
+	if (do_version)
+		inode_inc_iversion(inode);
+}
+
+static int au_ii_refresh(struct inode *inode, int *update)
+{
+	int err, e, nbr;
+	umode_t type;
+	aufs_bindex_t bindex, new_bindex;
+	struct super_block *sb;
+	struct au_iinfo *iinfo;
+	struct au_hinode *p, *q, tmp;
+
+	AuDebugOn(au_is_bad_inode(inode));
+	IiMustWriteLock(inode);
+
+	*update = 0;
+	sb = inode->i_sb;
+	nbr = au_sbbot(sb) + 1;
+	type = inode->i_mode & S_IFMT;
+	iinfo = au_ii(inode);
+	err = au_hinode_realloc(iinfo, nbr, /*may_shrink*/0);
+	if (unlikely(err))
+		goto out;
+
+	AuDebugOn(iinfo->ii_btop < 0);
+	p = au_hinode(iinfo, iinfo->ii_btop);
+	for (bindex = iinfo->ii_btop; bindex <= iinfo->ii_bbot;
+	     bindex++, p++) {
+		if (!p->hi_inode)
+			continue;
+
+		AuDebugOn(type != (p->hi_inode->i_mode & S_IFMT));
+		new_bindex = au_br_index(sb, p->hi_id);
+		if (new_bindex == bindex)
+			continue;
+
+		if (new_bindex < 0) {
+			*update = 1;
+			au_hiput(p);
+			p->hi_inode = NULL;
+			continue;
+		}
+
+		if (new_bindex < iinfo->ii_btop)
+			iinfo->ii_btop = new_bindex;
+		if (iinfo->ii_bbot < new_bindex)
+			iinfo->ii_bbot = new_bindex;
+		/* swap two lower inode, and loop again */
+		q = au_hinode(iinfo, new_bindex);
+		tmp = *q;
+		*q = *p;
+		*p = tmp;
+		if (tmp.hi_inode) {
+			bindex--;
+			p--;
+		}
+	}
+	au_update_ibrange(inode, /*do_put_zero*/0);
+	au_hinode_realloc(iinfo, nbr, /*may_shrink*/1); /* harmless if err */
+	e = au_dy_irefresh(inode);
+	if (unlikely(e && !err))
+		err = e;
+
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+void au_refresh_iop(struct inode *inode, int force_getattr)
+{
+	int type;
+	struct au_sbinfo *sbi = au_sbi(inode->i_sb);
+	const struct inode_operations *iop
+		= force_getattr ? aufs_iop : sbi->si_iop_array;
+
+	if (inode->i_op == iop)
+		return;
+
+	switch (inode->i_mode & S_IFMT) {
+	case S_IFDIR:
+		type = AuIop_DIR;
+		break;
+	case S_IFLNK:
+		type = AuIop_SYMLINK;
+		break;
+	default:
+		type = AuIop_OTHER;
+		break;
+	}
+
+	inode->i_op = iop + type;
+	/* unnecessary smp_wmb() */
+}
+
+int au_refresh_hinode_self(struct inode *inode)
+{
+	int err, update;
+
+	err = au_ii_refresh(inode, &update);
+	if (!err)
+		au_refresh_hinode_attr(inode, update && S_ISDIR(inode->i_mode));
+
+	AuTraceErr(err);
+	return err;
+}
+
+int au_refresh_hinode(struct inode *inode, struct dentry *dentry)
+{
+	int err, e, update;
+	unsigned int flags;
+	umode_t mode;
+	aufs_bindex_t bindex, bbot;
+	unsigned char isdir;
+	struct au_hinode *p;
+	struct au_iinfo *iinfo;
+
+	err = au_ii_refresh(inode, &update);
+	if (unlikely(err))
+		goto out;
+
+	update = 0;
+	iinfo = au_ii(inode);
+	p = au_hinode(iinfo, iinfo->ii_btop);
+	mode = (inode->i_mode & S_IFMT);
+	isdir = S_ISDIR(mode);
+	flags = au_hi_flags(inode, isdir);
+	bbot = au_dbbot(dentry);
+	for (bindex = au_dbtop(dentry); bindex <= bbot; bindex++) {
+		struct inode *h_i, *h_inode;
+		struct dentry *h_d;
+
+		h_d = au_h_dptr(dentry, bindex);
+		if (!h_d || d_is_negative(h_d))
+			continue;
+
+		h_inode = d_inode(h_d);
+		AuDebugOn(mode != (h_inode->i_mode & S_IFMT));
+		if (iinfo->ii_btop <= bindex && bindex <= iinfo->ii_bbot) {
+			h_i = au_h_iptr(inode, bindex);
+			if (h_i) {
+				if (h_i == h_inode)
+					continue;
+				err = -EIO;
+				break;
+			}
+		}
+		if (bindex < iinfo->ii_btop)
+			iinfo->ii_btop = bindex;
+		if (iinfo->ii_bbot < bindex)
+			iinfo->ii_bbot = bindex;
+		au_set_h_iptr(inode, bindex, au_igrab(h_inode), flags);
+		update = 1;
+	}
+	au_update_ibrange(inode, /*do_put_zero*/0);
+	e = au_dy_irefresh(inode);
+	if (unlikely(e && !err))
+		err = e;
+	if (!err)
+		au_refresh_hinode_attr(inode, update && isdir);
+
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+static int set_inode(struct inode *inode, struct dentry *dentry)
+{
+	int err;
+	unsigned int flags;
+	umode_t mode;
+	aufs_bindex_t bindex, btop, btail;
+	unsigned char isdir;
+	struct dentry *h_dentry;
+	struct inode *h_inode;
+	struct au_iinfo *iinfo;
+	const struct inode_operations *iop;
+
+	IiMustWriteLock(inode);
+
+	err = 0;
+	isdir = 0;
+	iop = au_sbi(inode->i_sb)->si_iop_array;
+	btop = au_dbtop(dentry);
+	h_dentry = au_h_dptr(dentry, btop);
+	h_inode = d_inode(h_dentry);
+	mode = h_inode->i_mode;
+	switch (mode & S_IFMT) {
+	case S_IFREG:
+		btail = au_dbtail(dentry);
+		inode->i_op = iop + AuIop_OTHER;
+		inode->i_fop = &aufs_file_fop;
+		err = au_dy_iaop(inode, btop, h_inode);
+		if (unlikely(err))
+			goto out;
+		break;
+	case S_IFDIR:
+		isdir = 1;
+		btail = au_dbtaildir(dentry);
+		inode->i_op = iop + AuIop_DIR;
+		inode->i_fop = &aufs_dir_fop;
+		break;
+	case S_IFLNK:
+		btail = au_dbtail(dentry);
+		inode->i_op = iop + AuIop_SYMLINK;
+		break;
+	case S_IFBLK:
+	case S_IFCHR:
+	case S_IFIFO:
+	case S_IFSOCK:
+		btail = au_dbtail(dentry);
+		inode->i_op = iop + AuIop_OTHER;
+		init_special_inode(inode, mode, h_inode->i_rdev);
+		break;
+	default:
+		AuIOErr("Unknown file type 0%o\n", mode);
+		err = -EIO;
+		goto out;
+	}
+
+	/* do not set hnotify for whiteouted dirs (SHWH mode) */
+	flags = au_hi_flags(inode, isdir);
+	if (au_opt_test(au_mntflags(dentry->d_sb), SHWH)
+	    && au_ftest_hi(flags, HNOTIFY)
+	    && dentry->d_name.len > AUFS_WH_PFX_LEN
+	    && !memcmp(dentry->d_name.name, AUFS_WH_PFX, AUFS_WH_PFX_LEN))
+		au_fclr_hi(flags, HNOTIFY);
+	iinfo = au_ii(inode);
+	iinfo->ii_btop = btop;
+	iinfo->ii_bbot = btail;
+	for (bindex = btop; bindex <= btail; bindex++) {
+		h_dentry = au_h_dptr(dentry, bindex);
+		if (h_dentry)
+			au_set_h_iptr(inode, bindex,
+				      au_igrab(d_inode(h_dentry)), flags);
+	}
+	au_cpup_attr_all(inode, /*force*/1);
+	/*
+	 * to force calling aufs_get_acl() every time,
+	 * do not call cache_no_acl() for aufs inode.
+	 */
+
+out:
+	return err;
+}
+
+/*
+ * successful returns with iinfo write_locked
+ * minus: errno
+ * zero: success, matched
+ * plus: no error, but unmatched
+ */
+static int reval_inode(struct inode *inode, struct dentry *dentry)
+{
+	int err;
+	unsigned int gen, igflags;
+	aufs_bindex_t bindex, bbot;
+	struct inode *h_inode, *h_dinode;
+	struct dentry *h_dentry;
+
+	/*
+	 * before this function, if aufs got any iinfo lock, it must be only
+	 * one, the parent dir.
+	 * it can happen by UDBA and the obsoleted inode number.
+	 */
+	err = -EIO;
+	if (unlikely(inode->i_ino == parent_ino(dentry)))
+		goto out;
+
+	err = 1;
+	ii_write_lock_new_child(inode);
+	h_dentry = au_h_dptr(dentry, au_dbtop(dentry));
+	h_dinode = d_inode(h_dentry);
+	bbot = au_ibbot(inode);
+	for (bindex = au_ibtop(inode); bindex <= bbot; bindex++) {
+		h_inode = au_h_iptr(inode, bindex);
+		if (!h_inode || h_inode != h_dinode)
+			continue;
+
+		err = 0;
+		gen = au_iigen(inode, &igflags);
+		if (gen == au_digen(dentry)
+		    && !au_ig_ftest(igflags, HALF_REFRESHED))
+			break;
+
+		/* fully refresh inode using dentry */
+		err = au_refresh_hinode(inode, dentry);
+		if (!err)
+			au_update_iigen(inode, /*half*/0);
+		break;
+	}
+
+	if (unlikely(err))
+		ii_write_unlock(inode);
+out:
+	return err;
+}
+
+int au_ino(struct super_block *sb, aufs_bindex_t bindex, ino_t h_ino,
+	   unsigned int d_type, ino_t *ino)
+{
+	int err, idx;
+	const int isnondir = d_type != DT_DIR;
+
+	/* prevent hardlinked inode number from race condition */
+	if (isnondir) {
+		err = au_xinondir_enter(sb, bindex, h_ino, &idx);
+		if (unlikely(err))
+			goto out;
+	}
+
+	err = au_xino_read(sb, bindex, h_ino, ino);
+	if (unlikely(err))
+		goto out_xinondir;
+
+	if (!*ino) {
+		err = -EIO;
+		*ino = au_xino_new_ino(sb);
+		if (unlikely(!*ino))
+			goto out_xinondir;
+		err = au_xino_write(sb, bindex, h_ino, *ino);
+		if (unlikely(err))
+			goto out_xinondir;
+	}
+
+out_xinondir:
+	if (isnondir && idx >= 0)
+		au_xinondir_leave(sb, bindex, h_ino, idx);
+out:
+	return err;
+}
+
+/* successful returns with iinfo write_locked */
+/* todo: return with unlocked? */
+struct inode *au_new_inode(struct dentry *dentry, int must_new)
+{
+	struct inode *inode, *h_inode;
+	struct dentry *h_dentry;
+	struct super_block *sb;
+	ino_t h_ino, ino;
+	int err, idx, hlinked;
+	aufs_bindex_t btop;
+
+	sb = dentry->d_sb;
+	btop = au_dbtop(dentry);
+	h_dentry = au_h_dptr(dentry, btop);
+	h_inode = d_inode(h_dentry);
+	h_ino = h_inode->i_ino;
+	hlinked = !d_is_dir(h_dentry) && h_inode->i_nlink > 1;
+
+new_ino:
+	/*
+	 * stop 'race'-ing between hardlinks under different
+	 * parents.
+	 */
+	if (hlinked) {
+		err = au_xinondir_enter(sb, btop, h_ino, &idx);
+		inode = ERR_PTR(err);
+		if (unlikely(err))
+			goto out;
+	}
+
+	err = au_xino_read(sb, btop, h_ino, &ino);
+	inode = ERR_PTR(err);
+	if (unlikely(err))
+		goto out_xinondir;
+
+	if (!ino) {
+		ino = au_xino_new_ino(sb);
+		if (unlikely(!ino)) {
+			inode = ERR_PTR(-EIO);
+			goto out_xinondir;
+		}
+	}
+
+	AuDbg("i%lu\n", (unsigned long)ino);
+	inode = au_iget_locked(sb, ino);
+	err = PTR_ERR(inode);
+	if (IS_ERR(inode))
+		goto out_xinondir;
+
+	AuDbg("%lx, new %d\n", inode->i_state, !!(inode->i_state & I_NEW));
+	if (inode->i_state & I_NEW) {
+		ii_write_lock_new_child(inode);
+		err = set_inode(inode, dentry);
+		if (!err) {
+			unlock_new_inode(inode);
+			goto out_xinondir; /* success */
+		}
+
+		/*
+		 * iget_failed() calls iput(), but we need to call
+		 * ii_write_unlock() after iget_failed(). so dirty hack for
+		 * i_count.
+		 */
+		atomic_inc(&inode->i_count);
+		iget_failed(inode);
+		ii_write_unlock(inode);
+		au_xino_write(sb, btop, h_ino, /*ino*/0);
+		/* ignore this error */
+		goto out_iput;
+	} else if (!must_new && !IS_DEADDIR(inode) && inode->i_nlink) {
+		/*
+		 * horrible race condition between lookup, readdir and copyup
+		 * (or something).
+		 */
+		if (hlinked && idx >= 0)
+			au_xinondir_leave(sb, btop, h_ino, idx);
+		err = reval_inode(inode, dentry);
+		if (unlikely(err < 0)) {
+			hlinked = 0;
+			goto out_iput;
+		}
+		if (!err)
+			goto out; /* success */
+		else if (hlinked && idx >= 0) {
+			err = au_xinondir_enter(sb, btop, h_ino, &idx);
+			if (unlikely(err)) {
+				iput(inode);
+				inode = ERR_PTR(err);
+				goto out;
+			}
+		}
+	}
+
+	if (unlikely(au_test_fs_unique_ino(h_inode)))
+		AuWarn1("Warning: Un-notified UDBA or repeatedly renamed dir,"
+			" b%d, %s, %pd, hi%lu, i%lu.\n",
+			btop, au_sbtype(h_dentry->d_sb), dentry,
+			(unsigned long)h_ino, (unsigned long)ino);
+	ino = 0;
+	err = au_xino_write(sb, btop, h_ino, /*ino*/0);
+	if (!err) {
+		iput(inode);
+		if (hlinked && idx >= 0)
+			au_xinondir_leave(sb, btop, h_ino, idx);
+		goto new_ino;
+	}
+
+out_iput:
+	iput(inode);
+	inode = ERR_PTR(err);
+out_xinondir:
+	if (hlinked && idx >= 0)
+		au_xinondir_leave(sb, btop, h_ino, idx);
+out:
+	return inode;
+}
+
+/* ---------------------------------------------------------------------- */
+
+int au_test_ro(struct super_block *sb, aufs_bindex_t bindex,
+	       struct inode *inode)
+{
+	int err;
+	struct inode *hi;
+
+	err = au_br_rdonly(au_sbr(sb, bindex));
+
+	/* pseudo-link after flushed may happen out of bounds */
+	if (!err
+	    && inode
+	    && au_ibtop(inode) <= bindex
+	    && bindex <= au_ibbot(inode)) {
+		/*
+		 * permission check is unnecessary since vfsub routine
+		 * will be called later
+		 */
+		hi = au_h_iptr(inode, bindex);
+		if (hi)
+			err = IS_IMMUTABLE(hi) ? -EROFS : 0;
+	}
+
+	return err;
+}
+
+int au_test_h_perm(struct inode *h_inode, int mask)
+{
+	if (uid_eq(current_fsuid(), GLOBAL_ROOT_UID))
+		return 0;
+	return inode_permission(h_inode, mask);
+}
+
+int au_test_h_perm_sio(struct inode *h_inode, int mask)
+{
+	if (au_test_nfs(h_inode->i_sb)
+	    && (mask & MAY_WRITE)
+	    && S_ISDIR(h_inode->i_mode))
+		mask |= MAY_READ; /* force permission check */
+	return au_test_h_perm(h_inode, mask);
+}
diff -Naur --no-dereference a/fs/aufs/inode.h b/fs/aufs/inode.h
--- a/fs/aufs/inode.h	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/inode.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,698 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * inode operations
+ */
+
+#ifndef __AUFS_INODE_H__
+#define __AUFS_INODE_H__
+
+#ifdef __KERNEL__
+
+#include <linux/fsnotify.h>
+#include "rwsem.h"
+
+struct vfsmount;
+
+struct au_hnotify {
+#ifdef CONFIG_AUFS_HNOTIFY
+#ifdef CONFIG_AUFS_HFSNOTIFY
+	/* never use fsnotify_add_vfsmount_mark() */
+	struct fsnotify_mark		hn_mark;
+#endif
+	struct inode		*hn_aufs_inode;	/* no get/put */
+	struct rcu_head		rcu;
+#endif
+} ____cacheline_aligned_in_smp;
+
+struct au_hinode {
+	struct inode		*hi_inode;
+	aufs_bindex_t		hi_id;
+#ifdef CONFIG_AUFS_HNOTIFY
+	struct au_hnotify	*hi_notify;
+#endif
+
+	/* reference to the copied-up whiteout with get/put */
+	struct dentry		*hi_whdentry;
+};
+
+/* ig_flags */
+#define AuIG_HALF_REFRESHED		1
+#define au_ig_ftest(flags, name)	((flags) & AuIG_##name)
+#define au_ig_fset(flags, name) \
+	do { (flags) |= AuIG_##name; } while (0)
+#define au_ig_fclr(flags, name) \
+	do { (flags) &= ~AuIG_##name; } while (0)
+
+struct au_iigen {
+	spinlock_t	ig_spin;
+	__u32		ig_generation, ig_flags;
+};
+
+struct au_vdir;
+struct au_iinfo {
+	struct au_iigen		ii_generation;
+	struct super_block	*ii_hsb1;	/* no get/put */
+
+	struct au_rwsem		ii_rwsem;
+	aufs_bindex_t		ii_btop, ii_bbot;
+	__u32			ii_higen;
+	struct au_hinode	*ii_hinode;
+	struct au_vdir		*ii_vdir;
+};
+
+struct au_icntnr {
+	struct au_iinfo		iinfo;
+	struct inode		vfs_inode;
+	struct hlist_bl_node	plink;
+	struct rcu_head		rcu;
+} ____cacheline_aligned_in_smp;
+
+/* au_pin flags */
+#define AuPin_DI_LOCKED		1
+#define AuPin_MNT_WRITE		(1 << 1)
+#define au_ftest_pin(flags, name)	((flags) & AuPin_##name)
+#define au_fset_pin(flags, name) \
+	do { (flags) |= AuPin_##name; } while (0)
+#define au_fclr_pin(flags, name) \
+	do { (flags) &= ~AuPin_##name; } while (0)
+
+struct au_pin {
+	/* input */
+	struct dentry *dentry;
+	unsigned int udba;
+	unsigned char lsc_di, lsc_hi, flags;
+	aufs_bindex_t bindex;
+
+	/* output */
+	struct dentry *parent;
+	struct au_hinode *hdir;
+	struct vfsmount *h_mnt;
+
+	/* temporary unlock/relock for copyup */
+	struct dentry *h_dentry, *h_parent;
+	struct au_branch *br;
+	struct task_struct *task;
+};
+
+void au_pin_hdir_unlock(struct au_pin *p);
+int au_pin_hdir_lock(struct au_pin *p);
+int au_pin_hdir_relock(struct au_pin *p);
+void au_pin_hdir_acquire_nest(struct au_pin *p);
+void au_pin_hdir_release(struct au_pin *p);
+
+/* ---------------------------------------------------------------------- */
+
+static inline struct au_iinfo *au_ii(struct inode *inode)
+{
+	BUG_ON(is_bad_inode(inode));
+	return &(container_of(inode, struct au_icntnr, vfs_inode)->iinfo);
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* inode.c */
+struct inode *au_igrab(struct inode *inode);
+void au_refresh_iop(struct inode *inode, int force_getattr);
+int au_refresh_hinode_self(struct inode *inode);
+int au_refresh_hinode(struct inode *inode, struct dentry *dentry);
+int au_ino(struct super_block *sb, aufs_bindex_t bindex, ino_t h_ino,
+	   unsigned int d_type, ino_t *ino);
+struct inode *au_new_inode(struct dentry *dentry, int must_new);
+int au_test_ro(struct super_block *sb, aufs_bindex_t bindex,
+	       struct inode *inode);
+int au_test_h_perm(struct inode *h_inode, int mask);
+int au_test_h_perm_sio(struct inode *h_inode, int mask);
+
+static inline int au_wh_ino(struct super_block *sb, aufs_bindex_t bindex,
+			    ino_t h_ino, unsigned int d_type, ino_t *ino)
+{
+#ifdef CONFIG_AUFS_SHWH
+	return au_ino(sb, bindex, h_ino, d_type, ino);
+#else
+	return 0;
+#endif
+}
+
+/* i_op.c */
+enum {
+	AuIop_SYMLINK,
+	AuIop_DIR,
+	AuIop_OTHER,
+	AuIop_Last
+};
+extern struct inode_operations aufs_iop[AuIop_Last], /* not const */
+	aufs_iop_nogetattr[AuIop_Last];
+
+/* au_wr_dir flags */
+#define AuWrDir_ADD_ENTRY	1
+#define AuWrDir_ISDIR		(1 << 1)
+#define AuWrDir_TMPFILE		(1 << 2)
+#define au_ftest_wrdir(flags, name)	((flags) & AuWrDir_##name)
+#define au_fset_wrdir(flags, name) \
+	do { (flags) |= AuWrDir_##name; } while (0)
+#define au_fclr_wrdir(flags, name) \
+	do { (flags) &= ~AuWrDir_##name; } while (0)
+
+struct au_wr_dir_args {
+	aufs_bindex_t force_btgt;
+	unsigned char flags;
+};
+int au_wr_dir(struct dentry *dentry, struct dentry *src_dentry,
+	      struct au_wr_dir_args *args);
+
+struct dentry *au_pinned_h_parent(struct au_pin *pin);
+void au_pin_init(struct au_pin *pin, struct dentry *dentry,
+		 aufs_bindex_t bindex, int lsc_di, int lsc_hi,
+		 unsigned int udba, unsigned char flags);
+int au_pin(struct au_pin *pin, struct dentry *dentry, aufs_bindex_t bindex,
+	   unsigned int udba, unsigned char flags) __must_check;
+int au_do_pin(struct au_pin *pin) __must_check;
+void au_unpin(struct au_pin *pin);
+int au_reval_for_attr(struct dentry *dentry, unsigned int sigen);
+
+#define AuIcpup_DID_CPUP	1
+#define au_ftest_icpup(flags, name)	((flags) & AuIcpup_##name)
+#define au_fset_icpup(flags, name) \
+	do { (flags) |= AuIcpup_##name; } while (0)
+#define au_fclr_icpup(flags, name) \
+	do { (flags) &= ~AuIcpup_##name; } while (0)
+
+struct au_icpup_args {
+	unsigned char flags;
+	unsigned char pin_flags;
+	aufs_bindex_t btgt;
+	unsigned int udba;
+	struct au_pin pin;
+	struct path h_path;
+	struct inode *h_inode;
+};
+
+int au_pin_and_icpup(struct dentry *dentry, struct iattr *ia,
+		     struct au_icpup_args *a);
+
+int au_h_path_getattr(struct dentry *dentry, struct inode *inode, int force,
+		      struct path *h_path, int locked);
+
+/* i_op_add.c */
+int au_may_add(struct dentry *dentry, aufs_bindex_t bindex,
+	       struct dentry *h_parent, int isdir);
+int aufs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode,
+	       dev_t dev);
+int aufs_symlink(struct inode *dir, struct dentry *dentry, const char *symname);
+int aufs_create(struct inode *dir, struct dentry *dentry, umode_t mode,
+		bool want_excl);
+struct vfsub_aopen_args;
+int au_aopen_or_create(struct inode *dir, struct dentry *dentry,
+		       struct vfsub_aopen_args *args);
+int aufs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode);
+int aufs_link(struct dentry *src_dentry, struct inode *dir,
+	      struct dentry *dentry);
+int aufs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode);
+
+/* i_op_del.c */
+int au_wr_dir_need_wh(struct dentry *dentry, int isdir, aufs_bindex_t *bcpup);
+int au_may_del(struct dentry *dentry, aufs_bindex_t bindex,
+	       struct dentry *h_parent, int isdir);
+int aufs_unlink(struct inode *dir, struct dentry *dentry);
+int aufs_rmdir(struct inode *dir, struct dentry *dentry);
+
+/* i_op_ren.c */
+int au_wbr(struct dentry *dentry, aufs_bindex_t btgt);
+int aufs_rename(struct inode *src_dir, struct dentry *src_dentry,
+		struct inode *dir, struct dentry *dentry,
+		unsigned int flags);
+
+/* iinfo.c */
+struct inode *au_h_iptr(struct inode *inode, aufs_bindex_t bindex);
+void au_hiput(struct au_hinode *hinode);
+void au_set_hi_wh(struct inode *inode, aufs_bindex_t bindex,
+		  struct dentry *h_wh);
+unsigned int au_hi_flags(struct inode *inode, int isdir);
+
+/* hinode flags */
+#define AuHi_XINO	1
+#define AuHi_HNOTIFY	(1 << 1)
+#define au_ftest_hi(flags, name)	((flags) & AuHi_##name)
+#define au_fset_hi(flags, name) \
+	do { (flags) |= AuHi_##name; } while (0)
+#define au_fclr_hi(flags, name) \
+	do { (flags) &= ~AuHi_##name; } while (0)
+
+#ifndef CONFIG_AUFS_HNOTIFY
+#undef AuHi_HNOTIFY
+#define AuHi_HNOTIFY	0
+#endif
+
+void au_set_h_iptr(struct inode *inode, aufs_bindex_t bindex,
+		   struct inode *h_inode, unsigned int flags);
+
+void au_update_iigen(struct inode *inode, int half);
+void au_update_ibrange(struct inode *inode, int do_put_zero);
+
+void au_icntnr_init_once(void *_c);
+void au_hinode_init(struct au_hinode *hinode);
+int au_iinfo_init(struct inode *inode);
+void au_iinfo_fin(struct inode *inode);
+int au_hinode_realloc(struct au_iinfo *iinfo, int nbr, int may_shrink);
+
+#ifdef CONFIG_PROC_FS
+/* plink.c */
+int au_plink_maint(struct super_block *sb, int flags);
+struct au_sbinfo;
+void au_plink_maint_leave(struct au_sbinfo *sbinfo);
+int au_plink_maint_enter(struct super_block *sb);
+#ifdef CONFIG_AUFS_DEBUG
+void au_plink_list(struct super_block *sb);
+#else
+AuStubVoid(au_plink_list, struct super_block *sb)
+#endif
+int au_plink_test(struct inode *inode);
+struct dentry *au_plink_lkup(struct inode *inode, aufs_bindex_t bindex);
+void au_plink_append(struct inode *inode, aufs_bindex_t bindex,
+		     struct dentry *h_dentry);
+void au_plink_put(struct super_block *sb, int verbose);
+void au_plink_clean(struct super_block *sb, int verbose);
+void au_plink_half_refresh(struct super_block *sb, aufs_bindex_t br_id);
+#else
+AuStubInt0(au_plink_maint, struct super_block *sb, int flags);
+AuStubVoid(au_plink_maint_leave, struct au_sbinfo *sbinfo);
+AuStubInt0(au_plink_maint_enter, struct super_block *sb);
+AuStubVoid(au_plink_list, struct super_block *sb);
+AuStubInt0(au_plink_test, struct inode *inode);
+AuStub(struct dentry *, au_plink_lkup, return NULL,
+       struct inode *inode, aufs_bindex_t bindex);
+AuStubVoid(au_plink_append, struct inode *inode, aufs_bindex_t bindex,
+	   struct dentry *h_dentry);
+AuStubVoid(au_plink_put, struct super_block *sb, int verbose);
+AuStubVoid(au_plink_clean, struct super_block *sb, int verbose);
+AuStubVoid(au_plink_half_refresh, struct super_block *sb, aufs_bindex_t br_id);
+#endif /* CONFIG_PROC_FS */
+
+#ifdef CONFIG_AUFS_XATTR
+/* xattr.c */
+int au_cpup_xattr(struct dentry *h_dst, struct dentry *h_src, int ignore_flags,
+		  unsigned int verbose);
+ssize_t aufs_listxattr(struct dentry *dentry, char *list, size_t size);
+void au_xattr_init(struct super_block *sb);
+#else
+AuStubInt0(au_cpup_xattr, struct dentry *h_dst, struct dentry *h_src,
+	   int ignore_flags, unsigned int verbose);
+AuStubVoid(au_xattr_init, struct super_block *sb);
+#endif
+
+#ifdef CONFIG_FS_POSIX_ACL
+struct posix_acl *aufs_get_acl(struct inode *inode, int type);
+int aufs_set_acl(struct inode *inode, struct posix_acl *acl, int type);
+#endif
+
+#if IS_ENABLED(CONFIG_AUFS_XATTR) || IS_ENABLED(CONFIG_FS_POSIX_ACL)
+enum {
+	AU_XATTR_SET,
+	AU_ACL_SET
+};
+
+struct au_sxattr {
+	int type;
+	union {
+		struct {
+			const char	*name;
+			const void	*value;
+			size_t		size;
+			int		flags;
+		} set;
+		struct {
+			struct posix_acl *acl;
+			int		type;
+		} acl_set;
+	} u;
+};
+ssize_t au_sxattr(struct dentry *dentry, struct inode *inode,
+		  struct au_sxattr *arg);
+#endif
+
+/* ---------------------------------------------------------------------- */
+
+/* lock subclass for iinfo */
+enum {
+	AuLsc_II_CHILD,		/* child first */
+	AuLsc_II_CHILD2,	/* rename(2), link(2), and cpup at hnotify */
+	AuLsc_II_CHILD3,	/* copyup dirs */
+	AuLsc_II_PARENT,	/* see AuLsc_I_PARENT in vfsub.h */
+	AuLsc_II_PARENT2,
+	AuLsc_II_PARENT3,	/* copyup dirs */
+	AuLsc_II_NEW_CHILD
+};
+
+/*
+ * ii_read_lock_child, ii_write_lock_child,
+ * ii_read_lock_child2, ii_write_lock_child2,
+ * ii_read_lock_child3, ii_write_lock_child3,
+ * ii_read_lock_parent, ii_write_lock_parent,
+ * ii_read_lock_parent2, ii_write_lock_parent2,
+ * ii_read_lock_parent3, ii_write_lock_parent3,
+ * ii_read_lock_new_child, ii_write_lock_new_child,
+ */
+#define AuReadLockFunc(name, lsc) \
+static inline void ii_read_lock_##name(struct inode *i) \
+{ \
+	au_rw_read_lock_nested(&au_ii(i)->ii_rwsem, AuLsc_II_##lsc); \
+}
+
+#define AuWriteLockFunc(name, lsc) \
+static inline void ii_write_lock_##name(struct inode *i) \
+{ \
+	au_rw_write_lock_nested(&au_ii(i)->ii_rwsem, AuLsc_II_##lsc); \
+}
+
+#define AuRWLockFuncs(name, lsc) \
+	AuReadLockFunc(name, lsc) \
+	AuWriteLockFunc(name, lsc)
+
+AuRWLockFuncs(child, CHILD);
+AuRWLockFuncs(child2, CHILD2);
+AuRWLockFuncs(child3, CHILD3);
+AuRWLockFuncs(parent, PARENT);
+AuRWLockFuncs(parent2, PARENT2);
+AuRWLockFuncs(parent3, PARENT3);
+AuRWLockFuncs(new_child, NEW_CHILD);
+
+#undef AuReadLockFunc
+#undef AuWriteLockFunc
+#undef AuRWLockFuncs
+
+#define ii_read_unlock(i)	au_rw_read_unlock(&au_ii(i)->ii_rwsem)
+#define ii_write_unlock(i)	au_rw_write_unlock(&au_ii(i)->ii_rwsem)
+#define ii_downgrade_lock(i)	au_rw_dgrade_lock(&au_ii(i)->ii_rwsem)
+
+#define IiMustNoWaiters(i)	AuRwMustNoWaiters(&au_ii(i)->ii_rwsem)
+#define IiMustAnyLock(i)	AuRwMustAnyLock(&au_ii(i)->ii_rwsem)
+#define IiMustWriteLock(i)	AuRwMustWriteLock(&au_ii(i)->ii_rwsem)
+
+/* ---------------------------------------------------------------------- */
+
+static inline void au_icntnr_init(struct au_icntnr *c)
+{
+#ifdef CONFIG_AUFS_DEBUG
+	c->vfs_inode.i_mode = 0;
+#endif
+}
+
+static inline unsigned int au_iigen(struct inode *inode, unsigned int *igflags)
+{
+	unsigned int gen;
+	struct au_iinfo *iinfo;
+	struct au_iigen *iigen;
+
+	iinfo = au_ii(inode);
+	iigen = &iinfo->ii_generation;
+	spin_lock(&iigen->ig_spin);
+	if (igflags)
+		*igflags = iigen->ig_flags;
+	gen = iigen->ig_generation;
+	spin_unlock(&iigen->ig_spin);
+
+	return gen;
+}
+
+/* tiny test for inode number */
+/* tmpfs generation is too rough */
+static inline int au_test_higen(struct inode *inode, struct inode *h_inode)
+{
+	struct au_iinfo *iinfo;
+
+	iinfo = au_ii(inode);
+	AuRwMustAnyLock(&iinfo->ii_rwsem);
+	return !(iinfo->ii_hsb1 == h_inode->i_sb
+		 && iinfo->ii_higen == h_inode->i_generation);
+}
+
+static inline void au_iigen_dec(struct inode *inode)
+{
+	struct au_iinfo *iinfo;
+	struct au_iigen *iigen;
+
+	iinfo = au_ii(inode);
+	iigen = &iinfo->ii_generation;
+	spin_lock(&iigen->ig_spin);
+	iigen->ig_generation--;
+	spin_unlock(&iigen->ig_spin);
+}
+
+static inline int au_iigen_test(struct inode *inode, unsigned int sigen)
+{
+	int err;
+
+	err = 0;
+	if (unlikely(inode && au_iigen(inode, NULL) != sigen))
+		err = -EIO;
+
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static inline struct au_hinode *au_hinode(struct au_iinfo *iinfo,
+					  aufs_bindex_t bindex)
+{
+	return iinfo->ii_hinode + bindex;
+}
+
+static inline int au_is_bad_inode(struct inode *inode)
+{
+	return !!(is_bad_inode(inode) || !au_hinode(au_ii(inode), 0));
+}
+
+static inline aufs_bindex_t au_ii_br_id(struct inode *inode,
+					aufs_bindex_t bindex)
+{
+	IiMustAnyLock(inode);
+	return au_hinode(au_ii(inode), bindex)->hi_id;
+}
+
+static inline aufs_bindex_t au_ibtop(struct inode *inode)
+{
+	IiMustAnyLock(inode);
+	return au_ii(inode)->ii_btop;
+}
+
+static inline aufs_bindex_t au_ibbot(struct inode *inode)
+{
+	IiMustAnyLock(inode);
+	return au_ii(inode)->ii_bbot;
+}
+
+static inline struct au_vdir *au_ivdir(struct inode *inode)
+{
+	IiMustAnyLock(inode);
+	return au_ii(inode)->ii_vdir;
+}
+
+static inline struct dentry *au_hi_wh(struct inode *inode, aufs_bindex_t bindex)
+{
+	IiMustAnyLock(inode);
+	return au_hinode(au_ii(inode), bindex)->hi_whdentry;
+}
+
+static inline void au_set_ibtop(struct inode *inode, aufs_bindex_t bindex)
+{
+	IiMustWriteLock(inode);
+	au_ii(inode)->ii_btop = bindex;
+}
+
+static inline void au_set_ibbot(struct inode *inode, aufs_bindex_t bindex)
+{
+	IiMustWriteLock(inode);
+	au_ii(inode)->ii_bbot = bindex;
+}
+
+static inline void au_set_ivdir(struct inode *inode, struct au_vdir *vdir)
+{
+	IiMustWriteLock(inode);
+	au_ii(inode)->ii_vdir = vdir;
+}
+
+static inline struct au_hinode *au_hi(struct inode *inode, aufs_bindex_t bindex)
+{
+	IiMustAnyLock(inode);
+	return au_hinode(au_ii(inode), bindex);
+}
+
+/* ---------------------------------------------------------------------- */
+
+static inline struct dentry *au_pinned_parent(struct au_pin *pin)
+{
+	if (pin)
+		return pin->parent;
+	return NULL;
+}
+
+static inline struct inode *au_pinned_h_dir(struct au_pin *pin)
+{
+	if (pin && pin->hdir)
+		return pin->hdir->hi_inode;
+	return NULL;
+}
+
+static inline struct au_hinode *au_pinned_hdir(struct au_pin *pin)
+{
+	if (pin)
+		return pin->hdir;
+	return NULL;
+}
+
+static inline void au_pin_set_dentry(struct au_pin *pin, struct dentry *dentry)
+{
+	if (pin)
+		pin->dentry = dentry;
+}
+
+static inline void au_pin_set_parent_lflag(struct au_pin *pin,
+					   unsigned char lflag)
+{
+	if (pin) {
+		if (lflag)
+			au_fset_pin(pin->flags, DI_LOCKED);
+		else
+			au_fclr_pin(pin->flags, DI_LOCKED);
+	}
+}
+
+#if 0 /* reserved */
+static inline void au_pin_set_parent(struct au_pin *pin, struct dentry *parent)
+{
+	if (pin) {
+		dput(pin->parent);
+		pin->parent = dget(parent);
+	}
+}
+#endif
+
+/* ---------------------------------------------------------------------- */
+
+struct au_branch;
+#ifdef CONFIG_AUFS_HNOTIFY
+struct au_hnotify_op {
+	void (*ctl)(struct au_hinode *hinode, int do_set);
+	int (*alloc)(struct au_hinode *hinode);
+
+	/*
+	 * if it returns true, the the caller should free hinode->hi_notify,
+	 * otherwise ->free() frees it.
+	 */
+	int (*free)(struct au_hinode *hinode,
+		    struct au_hnotify *hn) __must_check;
+
+	void (*fin)(void);
+	int (*init)(void);
+
+	int (*reset_br)(unsigned int udba, struct au_branch *br, int perm);
+	void (*fin_br)(struct au_branch *br);
+	int (*init_br)(struct au_branch *br, int perm);
+};
+
+/* hnotify.c */
+int au_hn_alloc(struct au_hinode *hinode, struct inode *inode);
+void au_hn_free(struct au_hinode *hinode);
+void au_hn_ctl(struct au_hinode *hinode, int do_set);
+void au_hn_reset(struct inode *inode, unsigned int flags);
+int au_hnotify(struct inode *h_dir, struct au_hnotify *hnotify, u32 mask,
+	       struct qstr *h_child_qstr, struct inode *h_child_inode);
+int au_hnotify_reset_br(unsigned int udba, struct au_branch *br, int perm);
+int au_hnotify_init_br(struct au_branch *br, int perm);
+void au_hnotify_fin_br(struct au_branch *br);
+int __init au_hnotify_init(void);
+void au_hnotify_fin(void);
+
+/* hfsnotify.c */
+extern const struct au_hnotify_op au_hnotify_op;
+
+static inline
+void au_hn_init(struct au_hinode *hinode)
+{
+	hinode->hi_notify = NULL;
+}
+
+static inline struct au_hnotify *au_hn(struct au_hinode *hinode)
+{
+	return hinode->hi_notify;
+}
+
+#else
+AuStub(int, au_hn_alloc, return -EOPNOTSUPP,
+       struct au_hinode *hinode __maybe_unused,
+       struct inode *inode __maybe_unused)
+AuStub(struct au_hnotify *, au_hn, return NULL, struct au_hinode *hinode)
+AuStubVoid(au_hn_free, struct au_hinode *hinode __maybe_unused)
+AuStubVoid(au_hn_ctl, struct au_hinode *hinode __maybe_unused,
+	   int do_set __maybe_unused)
+AuStubVoid(au_hn_reset, struct inode *inode __maybe_unused,
+	   unsigned int flags __maybe_unused)
+AuStubInt0(au_hnotify_reset_br, unsigned int udba __maybe_unused,
+	   struct au_branch *br __maybe_unused,
+	   int perm __maybe_unused)
+AuStubInt0(au_hnotify_init_br, struct au_branch *br __maybe_unused,
+	   int perm __maybe_unused)
+AuStubVoid(au_hnotify_fin_br, struct au_branch *br __maybe_unused)
+AuStubInt0(__init au_hnotify_init, void)
+AuStubVoid(au_hnotify_fin, void)
+AuStubVoid(au_hn_init, struct au_hinode *hinode __maybe_unused)
+#endif /* CONFIG_AUFS_HNOTIFY */
+
+static inline void au_hn_suspend(struct au_hinode *hdir)
+{
+	au_hn_ctl(hdir, /*do_set*/0);
+}
+
+static inline void au_hn_resume(struct au_hinode *hdir)
+{
+	au_hn_ctl(hdir, /*do_set*/1);
+}
+
+static inline void au_hn_inode_lock(struct au_hinode *hdir)
+{
+	inode_lock(hdir->hi_inode);
+	au_hn_suspend(hdir);
+}
+
+static inline void au_hn_inode_lock_nested(struct au_hinode *hdir,
+					  unsigned int sc __maybe_unused)
+{
+	inode_lock_nested(hdir->hi_inode, sc);
+	au_hn_suspend(hdir);
+}
+
+#if 0 /* unused */
+#include "vfsub.h"
+static inline void au_hn_inode_lock_shared_nested(struct au_hinode *hdir,
+						  unsigned int sc)
+{
+	inode_lock_shared_nested(hdir->hi_inode, sc);
+	au_hn_suspend(hdir);
+}
+#endif
+
+static inline void au_hn_inode_unlock(struct au_hinode *hdir)
+{
+	au_hn_resume(hdir);
+	inode_unlock(hdir->hi_inode);
+}
+
+#endif /* __KERNEL__ */
+#endif /* __AUFS_INODE_H__ */
diff -Naur --no-dereference a/fs/aufs/ioctl.c b/fs/aufs/ioctl.c
--- a/fs/aufs/ioctl.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/ioctl.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,220 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * ioctl
+ * plink-management and readdir in userspace.
+ * assist the pathconf(3) wrapper library.
+ * move-down
+ * File-based Hierarchical Storage Management.
+ */
+
+#include <linux/compat.h>
+#include <linux/file.h>
+#include "aufs.h"
+
+static int au_wbr_fd(struct path *path, struct aufs_wbr_fd __user *arg)
+{
+	int err, fd;
+	aufs_bindex_t wbi, bindex, bbot;
+	struct file *h_file;
+	struct super_block *sb;
+	struct dentry *root;
+	struct au_branch *br;
+	struct aufs_wbr_fd wbrfd = {
+		.oflags	= au_dir_roflags,
+		.brid	= -1
+	};
+	const int valid = O_RDONLY | O_NONBLOCK | O_LARGEFILE | O_DIRECTORY
+		| O_NOATIME | O_CLOEXEC;
+
+	AuDebugOn(wbrfd.oflags & ~valid);
+
+	if (arg) {
+		err = copy_from_user(&wbrfd, arg, sizeof(wbrfd));
+		if (unlikely(err)) {
+			err = -EFAULT;
+			goto out;
+		}
+
+		err = -EINVAL;
+		AuDbg("wbrfd{0%o, %d}\n", wbrfd.oflags, wbrfd.brid);
+		wbrfd.oflags |= au_dir_roflags;
+		AuDbg("0%o\n", wbrfd.oflags);
+		if (unlikely(wbrfd.oflags & ~valid))
+			goto out;
+	}
+
+	fd = get_unused_fd_flags(0);
+	err = fd;
+	if (unlikely(fd < 0))
+		goto out;
+
+	h_file = ERR_PTR(-EINVAL);
+	wbi = 0;
+	br = NULL;
+	sb = path->dentry->d_sb;
+	root = sb->s_root;
+	aufs_read_lock(root, AuLock_IR);
+	bbot = au_sbbot(sb);
+	if (wbrfd.brid >= 0) {
+		wbi = au_br_index(sb, wbrfd.brid);
+		if (unlikely(wbi < 0 || wbi > bbot))
+			goto out_unlock;
+	}
+
+	h_file = ERR_PTR(-ENOENT);
+	br = au_sbr(sb, wbi);
+	if (!au_br_writable(br->br_perm)) {
+		if (arg)
+			goto out_unlock;
+
+		bindex = wbi + 1;
+		wbi = -1;
+		for (; bindex <= bbot; bindex++) {
+			br = au_sbr(sb, bindex);
+			if (au_br_writable(br->br_perm)) {
+				wbi = bindex;
+				br = au_sbr(sb, wbi);
+				break;
+			}
+		}
+	}
+	AuDbg("wbi %d\n", wbi);
+	if (wbi >= 0)
+		h_file = au_h_open(root, wbi, wbrfd.oflags, NULL,
+				   /*force_wr*/0);
+
+out_unlock:
+	aufs_read_unlock(root, AuLock_IR);
+	err = PTR_ERR(h_file);
+	if (IS_ERR(h_file))
+		goto out_fd;
+
+	au_lcnt_dec(&br->br_nfiles); /* cf. au_h_open() */
+	fd_install(fd, h_file);
+	err = fd;
+	goto out; /* success */
+
+out_fd:
+	put_unused_fd(fd);
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+long aufs_ioctl_dir(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	long err;
+	struct dentry *dentry;
+
+	switch (cmd) {
+	case AUFS_CTL_RDU:
+	case AUFS_CTL_RDU_INO:
+		err = au_rdu_ioctl(file, cmd, arg);
+		break;
+
+	case AUFS_CTL_WBR_FD:
+		err = au_wbr_fd(&file->f_path, (void __user *)arg);
+		break;
+
+	case AUFS_CTL_IBUSY:
+		err = au_ibusy_ioctl(file, arg);
+		break;
+
+	case AUFS_CTL_BRINFO:
+		err = au_brinfo_ioctl(file, arg);
+		break;
+
+	case AUFS_CTL_FHSM_FD:
+		dentry = file->f_path.dentry;
+		if (IS_ROOT(dentry))
+			err = au_fhsm_fd(dentry->d_sb, arg);
+		else
+			err = -ENOTTY;
+		break;
+
+	default:
+		/* do not call the lower */
+		AuDbg("0x%x\n", cmd);
+		err = -ENOTTY;
+	}
+
+	AuTraceErr(err);
+	return err;
+}
+
+long aufs_ioctl_nondir(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	long err;
+
+	switch (cmd) {
+	case AUFS_CTL_MVDOWN:
+		err = au_mvdown(file->f_path.dentry, (void __user *)arg);
+		break;
+
+	case AUFS_CTL_WBR_FD:
+		err = au_wbr_fd(&file->f_path, (void __user *)arg);
+		break;
+
+	default:
+		/* do not call the lower */
+		AuDbg("0x%x\n", cmd);
+		err = -ENOTTY;
+	}
+
+	AuTraceErr(err);
+	return err;
+}
+
+#ifdef CONFIG_COMPAT
+long aufs_compat_ioctl_dir(struct file *file, unsigned int cmd,
+			   unsigned long arg)
+{
+	long err;
+
+	switch (cmd) {
+	case AUFS_CTL_RDU:
+	case AUFS_CTL_RDU_INO:
+		err = au_rdu_compat_ioctl(file, cmd, arg);
+		break;
+
+	case AUFS_CTL_IBUSY:
+		err = au_ibusy_compat_ioctl(file, arg);
+		break;
+
+	case AUFS_CTL_BRINFO:
+		err = au_brinfo_compat_ioctl(file, arg);
+		break;
+
+	default:
+		err = aufs_ioctl_dir(file, cmd, arg);
+	}
+
+	AuTraceErr(err);
+	return err;
+}
+
+long aufs_compat_ioctl_nondir(struct file *file, unsigned int cmd,
+			      unsigned long arg)
+{
+	return aufs_ioctl_nondir(file, cmd, (unsigned long)compat_ptr(arg));
+}
+#endif
diff -Naur --no-dereference a/fs/aufs/i_op_add.c b/fs/aufs/i_op_add.c
--- a/fs/aufs/i_op_add.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/i_op_add.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,936 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * inode operations (add entry)
+ */
+
+#include <linux/iversion.h>
+#include "aufs.h"
+
+/*
+ * final procedure of adding a new entry, except link(2).
+ * remove whiteout, instantiate, copyup the parent dir's times and size
+ * and update version.
+ * if it failed, re-create the removed whiteout.
+ */
+static int epilog(struct inode *dir, aufs_bindex_t bindex,
+		  struct dentry *wh_dentry, struct dentry *dentry)
+{
+	int err, rerr;
+	aufs_bindex_t bwh;
+	struct path h_path;
+	struct super_block *sb;
+	struct inode *inode, *h_dir;
+	struct dentry *wh;
+
+	bwh = -1;
+	sb = dir->i_sb;
+	if (wh_dentry) {
+		h_dir = d_inode(wh_dentry->d_parent); /* dir inode is locked */
+		IMustLock(h_dir);
+		AuDebugOn(au_h_iptr(dir, bindex) != h_dir);
+		bwh = au_dbwh(dentry);
+		h_path.dentry = wh_dentry;
+		h_path.mnt = au_sbr_mnt(sb, bindex);
+		err = au_wh_unlink_dentry(au_h_iptr(dir, bindex), &h_path,
+					  dentry);
+		if (unlikely(err))
+			goto out;
+	}
+
+	inode = au_new_inode(dentry, /*must_new*/1);
+	if (!IS_ERR(inode)) {
+		d_instantiate(dentry, inode);
+		dir = d_inode(dentry->d_parent); /* dir inode is locked */
+		IMustLock(dir);
+		au_dir_ts(dir, bindex);
+		inode_inc_iversion(dir);
+		au_fhsm_wrote(sb, bindex, /*force*/0);
+		return 0; /* success */
+	}
+
+	err = PTR_ERR(inode);
+	if (!wh_dentry)
+		goto out;
+
+	/* revert */
+	/* dir inode is locked */
+	wh = au_wh_create(dentry, bwh, wh_dentry->d_parent);
+	rerr = PTR_ERR(wh);
+	if (IS_ERR(wh)) {
+		AuIOErr("%pd reverting whiteout failed(%d, %d)\n",
+			dentry, err, rerr);
+		err = -EIO;
+	} else
+		dput(wh);
+
+out:
+	return err;
+}
+
+static int au_d_may_add(struct dentry *dentry)
+{
+	int err;
+
+	err = 0;
+	if (unlikely(d_unhashed(dentry)))
+		err = -ENOENT;
+	if (unlikely(d_really_is_positive(dentry)))
+		err = -EEXIST;
+	return err;
+}
+
+/*
+ * simple tests for the adding inode operations.
+ * following the checks in vfs, plus the parent-child relationship.
+ */
+int au_may_add(struct dentry *dentry, aufs_bindex_t bindex,
+	       struct dentry *h_parent, int isdir)
+{
+	int err;
+	umode_t h_mode;
+	struct dentry *h_dentry;
+	struct inode *h_inode;
+
+	err = -ENAMETOOLONG;
+	if (unlikely(dentry->d_name.len > AUFS_MAX_NAMELEN))
+		goto out;
+
+	h_dentry = au_h_dptr(dentry, bindex);
+	if (d_really_is_negative(dentry)) {
+		err = -EEXIST;
+		if (unlikely(d_is_positive(h_dentry)))
+			goto out;
+	} else {
+		/* rename(2) case */
+		err = -EIO;
+		if (unlikely(d_is_negative(h_dentry)))
+			goto out;
+		h_inode = d_inode(h_dentry);
+		if (unlikely(!h_inode->i_nlink))
+			goto out;
+
+		h_mode = h_inode->i_mode;
+		if (!isdir) {
+			err = -EISDIR;
+			if (unlikely(S_ISDIR(h_mode)))
+				goto out;
+		} else if (unlikely(!S_ISDIR(h_mode))) {
+			err = -ENOTDIR;
+			goto out;
+		}
+	}
+
+	err = 0;
+	/* expected parent dir is locked */
+	if (unlikely(h_parent != h_dentry->d_parent))
+		err = -EIO;
+
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+/*
+ * initial procedure of adding a new entry.
+ * prepare writable branch and the parent dir, lock it,
+ * and lookup whiteout for the new entry.
+ */
+static struct dentry*
+lock_hdir_lkup_wh(struct dentry *dentry, struct au_dtime *dt,
+		  struct dentry *src_dentry, struct au_pin *pin,
+		  struct au_wr_dir_args *wr_dir_args)
+{
+	struct dentry *wh_dentry, *h_parent;
+	struct super_block *sb;
+	struct au_branch *br;
+	int err;
+	unsigned int udba;
+	aufs_bindex_t bcpup;
+
+	AuDbg("%pd\n", dentry);
+
+	err = au_wr_dir(dentry, src_dentry, wr_dir_args);
+	bcpup = err;
+	wh_dentry = ERR_PTR(err);
+	if (unlikely(err < 0))
+		goto out;
+
+	sb = dentry->d_sb;
+	udba = au_opt_udba(sb);
+	err = au_pin(pin, dentry, bcpup, udba,
+		     AuPin_DI_LOCKED | AuPin_MNT_WRITE);
+	wh_dentry = ERR_PTR(err);
+	if (unlikely(err))
+		goto out;
+
+	h_parent = au_pinned_h_parent(pin);
+	if (udba != AuOpt_UDBA_NONE
+	    && au_dbtop(dentry) == bcpup)
+		err = au_may_add(dentry, bcpup, h_parent,
+				 au_ftest_wrdir(wr_dir_args->flags, ISDIR));
+	else if (unlikely(dentry->d_name.len > AUFS_MAX_NAMELEN))
+		err = -ENAMETOOLONG;
+	wh_dentry = ERR_PTR(err);
+	if (unlikely(err))
+		goto out_unpin;
+
+	br = au_sbr(sb, bcpup);
+	if (dt) {
+		struct path tmp = {
+			.dentry	= h_parent,
+			.mnt	= au_br_mnt(br)
+		};
+		au_dtime_store(dt, au_pinned_parent(pin), &tmp);
+	}
+
+	wh_dentry = NULL;
+	if (bcpup != au_dbwh(dentry))
+		goto out; /* success */
+
+	/*
+	 * ENAMETOOLONG here means that if we allowed create such name, then it
+	 * would not be able to removed in the future. So we don't allow such
+	 * name here and we don't handle ENAMETOOLONG differently here.
+	 */
+	wh_dentry = au_wh_lkup(h_parent, &dentry->d_name, br);
+
+out_unpin:
+	if (IS_ERR(wh_dentry))
+		au_unpin(pin);
+out:
+	return wh_dentry;
+}
+
+/* ---------------------------------------------------------------------- */
+
+enum { Mknod, Symlink, Creat };
+struct simple_arg {
+	int type;
+	union {
+		struct {
+			umode_t			mode;
+			bool			want_excl;
+			bool			try_aopen;
+			struct vfsub_aopen_args	*aopen;
+		} c;
+		struct {
+			const char *symname;
+		} s;
+		struct {
+			umode_t mode;
+			dev_t dev;
+		} m;
+	} u;
+};
+
+static int add_simple(struct inode *dir, struct dentry *dentry,
+		      struct simple_arg *arg)
+{
+	int err, rerr;
+	aufs_bindex_t btop;
+	unsigned char created;
+	const unsigned char try_aopen
+		= (arg->type == Creat && arg->u.c.try_aopen);
+	struct vfsub_aopen_args *aopen = arg->u.c.aopen;
+	struct dentry *wh_dentry, *parent;
+	struct inode *h_dir;
+	struct super_block *sb;
+	struct au_branch *br;
+	/* to reduce stack size */
+	struct {
+		struct au_dtime dt;
+		struct au_pin pin;
+		struct path h_path;
+		struct au_wr_dir_args wr_dir_args;
+	} *a;
+
+	AuDbg("%pd\n", dentry);
+	IMustLock(dir);
+
+	err = -ENOMEM;
+	a = kmalloc(sizeof(*a), GFP_NOFS);
+	if (unlikely(!a))
+		goto out;
+	a->wr_dir_args.force_btgt = -1;
+	a->wr_dir_args.flags = AuWrDir_ADD_ENTRY;
+
+	parent = dentry->d_parent; /* dir inode is locked */
+	if (!try_aopen) {
+		err = aufs_read_lock(dentry, AuLock_DW | AuLock_GEN);
+		if (unlikely(err))
+			goto out_free;
+	}
+	err = au_d_may_add(dentry);
+	if (unlikely(err))
+		goto out_unlock;
+	if (!try_aopen)
+		di_write_lock_parent(parent);
+	wh_dentry = lock_hdir_lkup_wh(dentry, &a->dt, /*src_dentry*/NULL,
+				      &a->pin, &a->wr_dir_args);
+	err = PTR_ERR(wh_dentry);
+	if (IS_ERR(wh_dentry))
+		goto out_parent;
+
+	btop = au_dbtop(dentry);
+	sb = dentry->d_sb;
+	br = au_sbr(sb, btop);
+	a->h_path.dentry = au_h_dptr(dentry, btop);
+	a->h_path.mnt = au_br_mnt(br);
+	h_dir = au_pinned_h_dir(&a->pin);
+	switch (arg->type) {
+	case Creat:
+		if (!try_aopen || !h_dir->i_op->atomic_open) {
+			err = vfsub_create(h_dir, &a->h_path, arg->u.c.mode,
+					   arg->u.c.want_excl);
+			created = !err;
+			if (!err && try_aopen)
+				aopen->file->f_mode |= FMODE_CREATED;
+		} else {
+			aopen->br = br;
+			err = vfsub_atomic_open(h_dir, a->h_path.dentry, aopen);
+			AuDbg("err %d\n", err);
+			AuDbgFile(aopen->file);
+			created = err >= 0
+				&& !!(aopen->file->f_mode & FMODE_CREATED);
+		}
+		break;
+	case Symlink:
+		err = vfsub_symlink(h_dir, &a->h_path, arg->u.s.symname);
+		created = !err;
+		break;
+	case Mknod:
+		err = vfsub_mknod(h_dir, &a->h_path, arg->u.m.mode,
+				  arg->u.m.dev);
+		created = !err;
+		break;
+	default:
+		BUG();
+	}
+	if (unlikely(err < 0))
+		goto out_unpin;
+
+	err = epilog(dir, btop, wh_dentry, dentry);
+	if (!err)
+		goto out_unpin; /* success */
+
+	/* revert */
+	if (created /* && d_is_positive(a->h_path.dentry) */) {
+		/* no delegation since it is just created */
+		rerr = vfsub_unlink(h_dir, &a->h_path, /*delegated*/NULL,
+				    /*force*/0);
+		if (rerr) {
+			AuIOErr("%pd revert failure(%d, %d)\n",
+				dentry, err, rerr);
+			err = -EIO;
+		}
+		au_dtime_revert(&a->dt);
+	}
+	if (try_aopen && h_dir->i_op->atomic_open
+	    && (aopen->file->f_mode & FMODE_OPENED))
+		/* aopen->file is still opened */
+		au_lcnt_dec(&aopen->br->br_nfiles);
+
+out_unpin:
+	au_unpin(&a->pin);
+	dput(wh_dentry);
+out_parent:
+	if (!try_aopen)
+		di_write_unlock(parent);
+out_unlock:
+	if (unlikely(err)) {
+		au_update_dbtop(dentry);
+		d_drop(dentry);
+	}
+	if (!try_aopen)
+		aufs_read_unlock(dentry, AuLock_DW);
+out_free:
+	au_kfree_rcu(a);
+out:
+	return err;
+}
+
+int aufs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode,
+	       dev_t dev)
+{
+	struct simple_arg arg = {
+		.type = Mknod,
+		.u.m = {
+			.mode	= mode,
+			.dev	= dev
+		}
+	};
+	return add_simple(dir, dentry, &arg);
+}
+
+int aufs_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
+{
+	struct simple_arg arg = {
+		.type = Symlink,
+		.u.s.symname = symname
+	};
+	return add_simple(dir, dentry, &arg);
+}
+
+int aufs_create(struct inode *dir, struct dentry *dentry, umode_t mode,
+		bool want_excl)
+{
+	struct simple_arg arg = {
+		.type = Creat,
+		.u.c = {
+			.mode		= mode,
+			.want_excl	= want_excl
+		}
+	};
+	return add_simple(dir, dentry, &arg);
+}
+
+int au_aopen_or_create(struct inode *dir, struct dentry *dentry,
+		       struct vfsub_aopen_args *aopen_args)
+{
+	struct simple_arg arg = {
+		.type = Creat,
+		.u.c = {
+			.mode		= aopen_args->create_mode,
+			.want_excl	= aopen_args->open_flag & O_EXCL,
+			.try_aopen	= true,
+			.aopen		= aopen_args
+		}
+	};
+	return add_simple(dir, dentry, &arg);
+}
+
+int aufs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode)
+{
+	int err;
+	aufs_bindex_t bindex;
+	struct super_block *sb;
+	struct dentry *parent, *h_parent, *h_dentry;
+	struct inode *h_dir, *inode;
+	struct vfsmount *h_mnt;
+	struct au_wr_dir_args wr_dir_args = {
+		.force_btgt	= -1,
+		.flags		= AuWrDir_TMPFILE
+	};
+
+	/* copy-up may happen */
+	inode_lock(dir);
+
+	sb = dir->i_sb;
+	err = si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLM);
+	if (unlikely(err))
+		goto out;
+
+	err = au_di_init(dentry);
+	if (unlikely(err))
+		goto out_si;
+
+	err = -EBUSY;
+	parent = d_find_any_alias(dir);
+	AuDebugOn(!parent);
+	di_write_lock_parent(parent);
+	if (unlikely(d_inode(parent) != dir))
+		goto out_parent;
+
+	err = au_digen_test(parent, au_sigen(sb));
+	if (unlikely(err))
+		goto out_parent;
+
+	bindex = au_dbtop(parent);
+	au_set_dbtop(dentry, bindex);
+	au_set_dbbot(dentry, bindex);
+	err = au_wr_dir(dentry, /*src_dentry*/NULL, &wr_dir_args);
+	bindex = err;
+	if (unlikely(err < 0))
+		goto out_parent;
+
+	err = -EOPNOTSUPP;
+	h_dir = au_h_iptr(dir, bindex);
+	if (unlikely(!h_dir->i_op->tmpfile))
+		goto out_parent;
+
+	h_mnt = au_sbr_mnt(sb, bindex);
+	err = vfsub_mnt_want_write(h_mnt);
+	if (unlikely(err))
+		goto out_parent;
+
+	h_parent = au_h_dptr(parent, bindex);
+	h_dentry = vfs_tmpfile(h_parent, mode, /*open_flag*/0);
+	if (IS_ERR(h_dentry)) {
+		err = PTR_ERR(h_dentry);
+		goto out_mnt;
+	}
+
+	au_set_dbtop(dentry, bindex);
+	au_set_dbbot(dentry, bindex);
+	au_set_h_dptr(dentry, bindex, dget(h_dentry));
+	inode = au_new_inode(dentry, /*must_new*/1);
+	if (IS_ERR(inode)) {
+		err = PTR_ERR(inode);
+		au_set_h_dptr(dentry, bindex, NULL);
+		au_set_dbtop(dentry, -1);
+		au_set_dbbot(dentry, -1);
+	} else {
+		if (!inode->i_nlink)
+			set_nlink(inode, 1);
+		d_tmpfile(dentry, inode);
+		au_di(dentry)->di_tmpfile = 1;
+
+		/* update without i_mutex */
+		if (au_ibtop(dir) == au_dbtop(dentry))
+			au_cpup_attr_timesizes(dir);
+	}
+	dput(h_dentry);
+
+out_mnt:
+	vfsub_mnt_drop_write(h_mnt);
+out_parent:
+	di_write_unlock(parent);
+	dput(parent);
+	di_write_unlock(dentry);
+	if (unlikely(err)) {
+		au_di_fin(dentry);
+		dentry->d_fsdata = NULL;
+	}
+out_si:
+	si_read_unlock(sb);
+out:
+	inode_unlock(dir);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+struct au_link_args {
+	aufs_bindex_t bdst, bsrc;
+	struct au_pin pin;
+	struct path h_path;
+	struct dentry *src_parent, *parent;
+};
+
+static int au_cpup_before_link(struct dentry *src_dentry,
+			       struct au_link_args *a)
+{
+	int err;
+	struct dentry *h_src_dentry;
+	struct au_cp_generic cpg = {
+		.dentry	= src_dentry,
+		.bdst	= a->bdst,
+		.bsrc	= a->bsrc,
+		.len	= -1,
+		.pin	= &a->pin,
+		.flags	= AuCpup_DTIME | AuCpup_HOPEN /* | AuCpup_KEEPLINO */
+	};
+
+	di_read_lock_parent(a->src_parent, AuLock_IR);
+	err = au_test_and_cpup_dirs(src_dentry, a->bdst);
+	if (unlikely(err))
+		goto out;
+
+	h_src_dentry = au_h_dptr(src_dentry, a->bsrc);
+	err = au_pin(&a->pin, src_dentry, a->bdst,
+		     au_opt_udba(src_dentry->d_sb),
+		     AuPin_DI_LOCKED | AuPin_MNT_WRITE);
+	if (unlikely(err))
+		goto out;
+
+	err = au_sio_cpup_simple(&cpg);
+	au_unpin(&a->pin);
+
+out:
+	di_read_unlock(a->src_parent, AuLock_IR);
+	return err;
+}
+
+static int au_cpup_or_link(struct dentry *src_dentry, struct dentry *dentry,
+			   struct au_link_args *a)
+{
+	int err;
+	unsigned char plink;
+	aufs_bindex_t bbot;
+	struct dentry *h_src_dentry;
+	struct inode *h_inode, *inode, *delegated;
+	struct super_block *sb;
+	struct file *h_file;
+
+	plink = 0;
+	h_inode = NULL;
+	sb = src_dentry->d_sb;
+	inode = d_inode(src_dentry);
+	if (au_ibtop(inode) <= a->bdst)
+		h_inode = au_h_iptr(inode, a->bdst);
+	if (!h_inode || !h_inode->i_nlink) {
+		/* copyup src_dentry as the name of dentry. */
+		bbot = au_dbbot(dentry);
+		if (bbot < a->bsrc)
+			au_set_dbbot(dentry, a->bsrc);
+		au_set_h_dptr(dentry, a->bsrc,
+			      dget(au_h_dptr(src_dentry, a->bsrc)));
+		dget(a->h_path.dentry);
+		au_set_h_dptr(dentry, a->bdst, NULL);
+		AuDbg("temporary d_inode...\n");
+		spin_lock(&dentry->d_lock);
+		dentry->d_inode = d_inode(src_dentry); /* tmp */
+		spin_unlock(&dentry->d_lock);
+		h_file = au_h_open_pre(dentry, a->bsrc, /*force_wr*/0);
+		if (IS_ERR(h_file))
+			err = PTR_ERR(h_file);
+		else {
+			struct au_cp_generic cpg = {
+				.dentry	= dentry,
+				.bdst	= a->bdst,
+				.bsrc	= -1,
+				.len	= -1,
+				.pin	= &a->pin,
+				.flags	= AuCpup_KEEPLINO
+			};
+			err = au_sio_cpup_simple(&cpg);
+			au_h_open_post(dentry, a->bsrc, h_file);
+			if (!err) {
+				dput(a->h_path.dentry);
+				a->h_path.dentry = au_h_dptr(dentry, a->bdst);
+			} else
+				au_set_h_dptr(dentry, a->bdst,
+					      a->h_path.dentry);
+		}
+		spin_lock(&dentry->d_lock);
+		dentry->d_inode = NULL; /* restore */
+		spin_unlock(&dentry->d_lock);
+		AuDbg("temporary d_inode...done\n");
+		au_set_h_dptr(dentry, a->bsrc, NULL);
+		au_set_dbbot(dentry, bbot);
+	} else {
+		/* the inode of src_dentry already exists on a.bdst branch */
+		h_src_dentry = d_find_alias(h_inode);
+		if (!h_src_dentry && au_plink_test(inode)) {
+			plink = 1;
+			h_src_dentry = au_plink_lkup(inode, a->bdst);
+			err = PTR_ERR(h_src_dentry);
+			if (IS_ERR(h_src_dentry))
+				goto out;
+
+			if (unlikely(d_is_negative(h_src_dentry))) {
+				dput(h_src_dentry);
+				h_src_dentry = NULL;
+			}
+
+		}
+		if (h_src_dentry) {
+			delegated = NULL;
+			err = vfsub_link(h_src_dentry, au_pinned_h_dir(&a->pin),
+					 &a->h_path, &delegated);
+			if (unlikely(err == -EWOULDBLOCK)) {
+				pr_warn("cannot retry for NFSv4 delegation"
+					" for an internal link\n");
+				iput(delegated);
+			}
+			dput(h_src_dentry);
+		} else {
+			AuIOErr("no dentry found for hi%lu on b%d\n",
+				h_inode->i_ino, a->bdst);
+			err = -EIO;
+		}
+	}
+
+	if (!err && !plink)
+		au_plink_append(inode, a->bdst, a->h_path.dentry);
+
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+int aufs_link(struct dentry *src_dentry, struct inode *dir,
+	      struct dentry *dentry)
+{
+	int err, rerr;
+	struct au_dtime dt;
+	struct au_link_args *a;
+	struct dentry *wh_dentry, *h_src_dentry;
+	struct inode *inode, *delegated;
+	struct super_block *sb;
+	struct au_wr_dir_args wr_dir_args = {
+		/* .force_btgt	= -1, */
+		.flags		= AuWrDir_ADD_ENTRY
+	};
+
+	IMustLock(dir);
+	inode = d_inode(src_dentry);
+	IMustLock(inode);
+
+	err = -ENOMEM;
+	a = kzalloc(sizeof(*a), GFP_NOFS);
+	if (unlikely(!a))
+		goto out;
+
+	a->parent = dentry->d_parent; /* dir inode is locked */
+	err = aufs_read_and_write_lock2(dentry, src_dentry,
+					AuLock_NOPLM | AuLock_GEN);
+	if (unlikely(err))
+		goto out_kfree;
+	err = au_d_linkable(src_dentry);
+	if (unlikely(err))
+		goto out_unlock;
+	err = au_d_may_add(dentry);
+	if (unlikely(err))
+		goto out_unlock;
+
+	a->src_parent = dget_parent(src_dentry);
+	wr_dir_args.force_btgt = au_ibtop(inode);
+
+	di_write_lock_parent(a->parent);
+	wr_dir_args.force_btgt = au_wbr(dentry, wr_dir_args.force_btgt);
+	wh_dentry = lock_hdir_lkup_wh(dentry, &dt, src_dentry, &a->pin,
+				      &wr_dir_args);
+	err = PTR_ERR(wh_dentry);
+	if (IS_ERR(wh_dentry))
+		goto out_parent;
+
+	err = 0;
+	sb = dentry->d_sb;
+	a->bdst = au_dbtop(dentry);
+	a->h_path.dentry = au_h_dptr(dentry, a->bdst);
+	a->h_path.mnt = au_sbr_mnt(sb, a->bdst);
+	a->bsrc = au_ibtop(inode);
+	h_src_dentry = au_h_d_alias(src_dentry, a->bsrc);
+	if (!h_src_dentry && au_di(src_dentry)->di_tmpfile)
+		h_src_dentry = dget(au_hi_wh(inode, a->bsrc));
+	if (!h_src_dentry) {
+		a->bsrc = au_dbtop(src_dentry);
+		h_src_dentry = au_h_d_alias(src_dentry, a->bsrc);
+		AuDebugOn(!h_src_dentry);
+	} else if (IS_ERR(h_src_dentry)) {
+		err = PTR_ERR(h_src_dentry);
+		goto out_parent;
+	}
+
+	/*
+	 * aufs doesn't touch the credential so
+	 * security_dentry_create_files_as() is unnecessary.
+	 */
+	if (au_opt_test(au_mntflags(sb), PLINK)) {
+		if (a->bdst < a->bsrc
+		    /* && h_src_dentry->d_sb != a->h_path.dentry->d_sb */)
+			err = au_cpup_or_link(src_dentry, dentry, a);
+		else {
+			delegated = NULL;
+			err = vfsub_link(h_src_dentry, au_pinned_h_dir(&a->pin),
+					 &a->h_path, &delegated);
+			if (unlikely(err == -EWOULDBLOCK)) {
+				pr_warn("cannot retry for NFSv4 delegation"
+					" for an internal link\n");
+				iput(delegated);
+			}
+		}
+		dput(h_src_dentry);
+	} else {
+		/*
+		 * copyup src_dentry to the branch we process,
+		 * and then link(2) to it.
+		 */
+		dput(h_src_dentry);
+		if (a->bdst < a->bsrc
+		    /* && h_src_dentry->d_sb != a->h_path.dentry->d_sb */) {
+			au_unpin(&a->pin);
+			di_write_unlock(a->parent);
+			err = au_cpup_before_link(src_dentry, a);
+			di_write_lock_parent(a->parent);
+			if (!err)
+				err = au_pin(&a->pin, dentry, a->bdst,
+					     au_opt_udba(sb),
+					     AuPin_DI_LOCKED | AuPin_MNT_WRITE);
+			if (unlikely(err))
+				goto out_wh;
+		}
+		if (!err) {
+			h_src_dentry = au_h_dptr(src_dentry, a->bdst);
+			err = -ENOENT;
+			if (h_src_dentry && d_is_positive(h_src_dentry)) {
+				delegated = NULL;
+				err = vfsub_link(h_src_dentry,
+						 au_pinned_h_dir(&a->pin),
+						 &a->h_path, &delegated);
+				if (unlikely(err == -EWOULDBLOCK)) {
+					pr_warn("cannot retry"
+						" for NFSv4 delegation"
+						" for an internal link\n");
+					iput(delegated);
+				}
+			}
+		}
+	}
+	if (unlikely(err))
+		goto out_unpin;
+
+	if (wh_dentry) {
+		a->h_path.dentry = wh_dentry;
+		err = au_wh_unlink_dentry(au_pinned_h_dir(&a->pin), &a->h_path,
+					  dentry);
+		if (unlikely(err))
+			goto out_revert;
+	}
+
+	au_dir_ts(dir, a->bdst);
+	inode_inc_iversion(dir);
+	inc_nlink(inode);
+	inode->i_ctime = dir->i_ctime;
+	d_instantiate(dentry, au_igrab(inode));
+	if (d_unhashed(a->h_path.dentry))
+		/* some filesystem calls d_drop() */
+		d_drop(dentry);
+	/* some filesystems consume an inode even hardlink */
+	au_fhsm_wrote(sb, a->bdst, /*force*/0);
+	goto out_unpin; /* success */
+
+out_revert:
+	/* no delegation since it is just created */
+	rerr = vfsub_unlink(au_pinned_h_dir(&a->pin), &a->h_path,
+			    /*delegated*/NULL, /*force*/0);
+	if (unlikely(rerr)) {
+		AuIOErr("%pd reverting failed(%d, %d)\n", dentry, err, rerr);
+		err = -EIO;
+	}
+	au_dtime_revert(&dt);
+out_unpin:
+	au_unpin(&a->pin);
+out_wh:
+	dput(wh_dentry);
+out_parent:
+	di_write_unlock(a->parent);
+	dput(a->src_parent);
+out_unlock:
+	if (unlikely(err)) {
+		au_update_dbtop(dentry);
+		d_drop(dentry);
+	}
+	aufs_read_and_write_unlock2(dentry, src_dentry);
+out_kfree:
+	au_kfree_rcu(a);
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+int aufs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
+{
+	int err, rerr;
+	aufs_bindex_t bindex;
+	unsigned char diropq;
+	struct path h_path;
+	struct dentry *wh_dentry, *parent, *opq_dentry;
+	struct inode *h_inode;
+	struct super_block *sb;
+	struct {
+		struct au_pin pin;
+		struct au_dtime dt;
+	} *a; /* reduce the stack usage */
+	struct au_wr_dir_args wr_dir_args = {
+		.force_btgt	= -1,
+		.flags		= AuWrDir_ADD_ENTRY | AuWrDir_ISDIR
+	};
+
+	IMustLock(dir);
+
+	err = -ENOMEM;
+	a = kmalloc(sizeof(*a), GFP_NOFS);
+	if (unlikely(!a))
+		goto out;
+
+	err = aufs_read_lock(dentry, AuLock_DW | AuLock_GEN);
+	if (unlikely(err))
+		goto out_free;
+	err = au_d_may_add(dentry);
+	if (unlikely(err))
+		goto out_unlock;
+
+	parent = dentry->d_parent; /* dir inode is locked */
+	di_write_lock_parent(parent);
+	wh_dentry = lock_hdir_lkup_wh(dentry, &a->dt, /*src_dentry*/NULL,
+				      &a->pin, &wr_dir_args);
+	err = PTR_ERR(wh_dentry);
+	if (IS_ERR(wh_dentry))
+		goto out_parent;
+
+	sb = dentry->d_sb;
+	bindex = au_dbtop(dentry);
+	h_path.dentry = au_h_dptr(dentry, bindex);
+	h_path.mnt = au_sbr_mnt(sb, bindex);
+	err = vfsub_mkdir(au_pinned_h_dir(&a->pin), &h_path, mode);
+	if (unlikely(err))
+		goto out_unpin;
+
+	/* make the dir opaque */
+	diropq = 0;
+	h_inode = d_inode(h_path.dentry);
+	if (wh_dentry
+	    || au_opt_test(au_mntflags(sb), ALWAYS_DIROPQ)) {
+		inode_lock_nested(h_inode, AuLsc_I_CHILD);
+		opq_dentry = au_diropq_create(dentry, bindex);
+		inode_unlock(h_inode);
+		err = PTR_ERR(opq_dentry);
+		if (IS_ERR(opq_dentry))
+			goto out_dir;
+		dput(opq_dentry);
+		diropq = 1;
+	}
+
+	err = epilog(dir, bindex, wh_dentry, dentry);
+	if (!err) {
+		inc_nlink(dir);
+		goto out_unpin; /* success */
+	}
+
+	/* revert */
+	if (diropq) {
+		AuLabel(revert opq);
+		inode_lock_nested(h_inode, AuLsc_I_CHILD);
+		rerr = au_diropq_remove(dentry, bindex);
+		inode_unlock(h_inode);
+		if (rerr) {
+			AuIOErr("%pd reverting diropq failed(%d, %d)\n",
+				dentry, err, rerr);
+			err = -EIO;
+		}
+	}
+
+out_dir:
+	AuLabel(revert dir);
+	rerr = vfsub_rmdir(au_pinned_h_dir(&a->pin), &h_path);
+	if (rerr) {
+		AuIOErr("%pd reverting dir failed(%d, %d)\n",
+			dentry, err, rerr);
+		err = -EIO;
+	}
+	au_dtime_revert(&a->dt);
+out_unpin:
+	au_unpin(&a->pin);
+	dput(wh_dentry);
+out_parent:
+	di_write_unlock(parent);
+out_unlock:
+	if (unlikely(err)) {
+		au_update_dbtop(dentry);
+		d_drop(dentry);
+	}
+	aufs_read_unlock(dentry, AuLock_DW);
+out_free:
+	au_kfree_rcu(a);
+out:
+	return err;
+}
diff -Naur --no-dereference a/fs/aufs/i_op.c b/fs/aufs/i_op.c
--- a/fs/aufs/i_op.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/i_op.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,1505 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * inode operations (except add/del/rename)
+ */
+
+#include <linux/device_cgroup.h>
+#include <linux/fs_stack.h>
+#include <linux/iversion.h>
+#include <linux/namei.h>
+#include <linux/security.h>
+#include "aufs.h"
+
+static int h_permission(struct inode *h_inode, int mask,
+			struct path *h_path, int brperm)
+{
+	int err;
+	const unsigned char write_mask = !!(mask & (MAY_WRITE | MAY_APPEND));
+
+	err = -EPERM;
+	if (write_mask && IS_IMMUTABLE(h_inode))
+		goto out;
+
+	err = -EACCES;
+	if (((mask & MAY_EXEC)
+	     && S_ISREG(h_inode->i_mode)
+	     && (path_noexec(h_path)
+		 || !(h_inode->i_mode & 0111))))
+		goto out;
+
+	/*
+	 * - skip the lower fs test in the case of write to ro branch.
+	 * - nfs dir permission write check is optimized, but a policy for
+	 *   link/rename requires a real check.
+	 * - nfs always sets SB_POSIXACL regardless its mount option 'noacl.'
+	 *   in this case, generic_permission() returns -EOPNOTSUPP.
+	 */
+	if ((write_mask && !au_br_writable(brperm))
+	    || (au_test_nfs(h_inode->i_sb) && S_ISDIR(h_inode->i_mode)
+		&& write_mask && !(mask & MAY_READ))
+	    || !h_inode->i_op->permission) {
+		/* AuLabel(generic_permission); */
+		/* AuDbg("get_acl %ps\n", h_inode->i_op->get_acl); */
+		err = generic_permission(h_inode, mask);
+		if (err == -EOPNOTSUPP && au_test_nfs_noacl(h_inode))
+			err = h_inode->i_op->permission(h_inode, mask);
+		AuTraceErr(err);
+	} else {
+		/* AuLabel(h_inode->permission); */
+		err = h_inode->i_op->permission(h_inode, mask);
+		AuTraceErr(err);
+	}
+
+	if (!err)
+		err = devcgroup_inode_permission(h_inode, mask);
+	if (!err)
+		err = security_inode_permission(h_inode, mask);
+
+out:
+	return err;
+}
+
+static int aufs_permission(struct inode *inode, int mask)
+{
+	int err;
+	aufs_bindex_t bindex, bbot;
+	const unsigned char isdir = !!S_ISDIR(inode->i_mode),
+		write_mask = !!(mask & (MAY_WRITE | MAY_APPEND));
+	struct inode *h_inode;
+	struct super_block *sb;
+	struct au_branch *br;
+
+	/* todo: support rcu-walk? */
+	if (mask & MAY_NOT_BLOCK)
+		return -ECHILD;
+
+	sb = inode->i_sb;
+	si_read_lock(sb, AuLock_FLUSH);
+	ii_read_lock_child(inode);
+#if 0 /* reserved for future use */
+	/*
+	 * This test may be rather 'too much' since the test is essentially done
+	 * in the aufs_lookup().  Theoretically it is possible that the inode
+	 * generation doesn't match to the superblock's here.  But it isn't a
+	 * big deal I suppose.
+	 */
+	err = au_iigen_test(inode, au_sigen(sb));
+	if (unlikely(err))
+		goto out;
+#endif
+
+	if (!isdir
+	    || write_mask
+	    || au_opt_test(au_mntflags(sb), DIRPERM1)) {
+		err = au_busy_or_stale();
+		h_inode = au_h_iptr(inode, au_ibtop(inode));
+		if (unlikely(!h_inode
+			     || (h_inode->i_mode & S_IFMT)
+			     != (inode->i_mode & S_IFMT)))
+			goto out;
+
+		err = 0;
+		bindex = au_ibtop(inode);
+		br = au_sbr(sb, bindex);
+		err = h_permission(h_inode, mask, &br->br_path, br->br_perm);
+		if (write_mask
+		    && !err
+		    && !special_file(h_inode->i_mode)) {
+			/* test whether the upper writable branch exists */
+			err = -EROFS;
+			for (; bindex >= 0; bindex--)
+				if (!au_br_rdonly(au_sbr(sb, bindex))) {
+					err = 0;
+					break;
+				}
+		}
+		goto out;
+	}
+
+	/* non-write to dir */
+	err = 0;
+	bbot = au_ibbot(inode);
+	for (bindex = au_ibtop(inode); !err && bindex <= bbot; bindex++) {
+		h_inode = au_h_iptr(inode, bindex);
+		if (h_inode) {
+			err = au_busy_or_stale();
+			if (unlikely(!S_ISDIR(h_inode->i_mode)))
+				break;
+
+			br = au_sbr(sb, bindex);
+			err = h_permission(h_inode, mask, &br->br_path,
+					   br->br_perm);
+		}
+	}
+
+out:
+	ii_read_unlock(inode);
+	si_read_unlock(sb);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static struct dentry *aufs_lookup(struct inode *dir, struct dentry *dentry,
+				  unsigned int flags)
+{
+	struct dentry *ret, *parent;
+	struct inode *inode;
+	struct super_block *sb;
+	int err, npositive;
+
+	IMustLock(dir);
+
+	/* todo: support rcu-walk? */
+	ret = ERR_PTR(-ECHILD);
+	if (flags & LOOKUP_RCU)
+		goto out;
+
+	ret = ERR_PTR(-ENAMETOOLONG);
+	if (unlikely(dentry->d_name.len > AUFS_MAX_NAMELEN))
+		goto out;
+
+	sb = dir->i_sb;
+	err = si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLM);
+	ret = ERR_PTR(err);
+	if (unlikely(err))
+		goto out;
+
+	err = au_di_init(dentry);
+	ret = ERR_PTR(err);
+	if (unlikely(err))
+		goto out_si;
+
+	inode = NULL;
+	npositive = 0; /* suppress a warning */
+	parent = dentry->d_parent; /* dir inode is locked */
+	di_read_lock_parent(parent, AuLock_IR);
+	err = au_alive_dir(parent);
+	if (!err)
+		err = au_digen_test(parent, au_sigen(sb));
+	if (!err) {
+		/* regardless LOOKUP_CREATE, always ALLOW_NEG */
+		npositive = au_lkup_dentry(dentry, au_dbtop(parent),
+					   AuLkup_ALLOW_NEG);
+		err = npositive;
+	}
+	di_read_unlock(parent, AuLock_IR);
+	ret = ERR_PTR(err);
+	if (unlikely(err < 0))
+		goto out_unlock;
+
+	if (npositive) {
+		inode = au_new_inode(dentry, /*must_new*/0);
+		if (IS_ERR(inode)) {
+			ret = (void *)inode;
+			inode = NULL;
+			goto out_unlock;
+		}
+	}
+
+	if (inode)
+		atomic_inc(&inode->i_count);
+	ret = d_splice_alias(inode, dentry);
+#if 0 /* reserved for future use */
+	if (unlikely(d_need_lookup(dentry))) {
+		spin_lock(&dentry->d_lock);
+		dentry->d_flags &= ~DCACHE_NEED_LOOKUP;
+		spin_unlock(&dentry->d_lock);
+	} else
+#endif
+	if (inode) {
+		if (!IS_ERR(ret)) {
+			iput(inode);
+			if (ret && ret != dentry)
+				ii_write_unlock(inode);
+		} else {
+			ii_write_unlock(inode);
+			iput(inode);
+			inode = NULL;
+		}
+	}
+
+out_unlock:
+	di_write_unlock(dentry);
+out_si:
+	si_read_unlock(sb);
+out:
+	return ret;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * very dirty and complicated aufs ->atomic_open().
+ * aufs_atomic_open()
+ * + au_aopen_or_create()
+ *   + add_simple()
+ *     + vfsub_atomic_open()
+ *       + branch fs ->atomic_open()
+ *	   may call the actual 'open' for h_file
+ *       + inc br_nfiles only if opened
+ * + au_aopen_no_open() or au_aopen_do_open()
+ *
+ * au_aopen_do_open()
+ * + finish_open()
+ *   + au_do_aopen()
+ *     + au_do_open() the body of all 'open'
+ *       + au_do_open_nondir()
+ *	   set the passed h_file
+ *
+ * au_aopen_no_open()
+ * + finish_no_open()
+ */
+
+struct aopen_node {
+	struct hlist_bl_node hblist;
+	struct file *file, *h_file;
+};
+
+static int au_do_aopen(struct inode *inode, struct file *file)
+{
+	struct hlist_bl_head *aopen;
+	struct hlist_bl_node *pos;
+	struct aopen_node *node;
+	struct au_do_open_args args = {
+		.aopen	= 1,
+		.open	= au_do_open_nondir
+	};
+
+	aopen = &au_sbi(inode->i_sb)->si_aopen;
+	hlist_bl_lock(aopen);
+	hlist_bl_for_each_entry(node, pos, aopen, hblist)
+		if (node->file == file) {
+			args.h_file = node->h_file;
+			break;
+		}
+	hlist_bl_unlock(aopen);
+	/* AuDebugOn(!args.h_file); */
+
+	return au_do_open(file, &args);
+}
+
+static int au_aopen_do_open(struct file *file, struct dentry *dentry,
+			    struct aopen_node *aopen_node)
+{
+	int err;
+	struct hlist_bl_head *aopen;
+
+	AuLabel(here);
+	aopen = &au_sbi(dentry->d_sb)->si_aopen;
+	au_hbl_add(&aopen_node->hblist, aopen);
+	err = finish_open(file, dentry, au_do_aopen);
+	au_hbl_del(&aopen_node->hblist, aopen);
+	/* AuDbgFile(file); */
+	AuDbg("%pd%s%s\n", dentry,
+	      (file->f_mode & FMODE_CREATED) ? " created" : "",
+	      (file->f_mode & FMODE_OPENED) ? " opened" : "");
+
+	AuTraceErr(err);
+	return err;
+}
+
+static int au_aopen_no_open(struct file *file, struct dentry *dentry)
+{
+	int err;
+
+	AuLabel(here);
+	dget(dentry);
+	err = finish_no_open(file, dentry);
+
+	AuTraceErr(err);
+	return err;
+}
+
+static int aufs_atomic_open(struct inode *dir, struct dentry *dentry,
+			    struct file *file, unsigned int open_flag,
+			    umode_t create_mode)
+{
+	int err, did_open;
+	unsigned int lkup_flags;
+	aufs_bindex_t bindex;
+	struct super_block *sb;
+	struct dentry *parent, *d;
+	struct vfsub_aopen_args args = {
+		.open_flag	= open_flag,
+		.create_mode	= create_mode
+	};
+	struct aopen_node aopen_node = {
+		.file	= file
+	};
+
+	IMustLock(dir);
+	AuDbg("open_flag 0%o\n", open_flag);
+	AuDbgDentry(dentry);
+
+	err = 0;
+	if (!au_di(dentry)) {
+		lkup_flags = LOOKUP_OPEN;
+		if (open_flag & O_CREAT)
+			lkup_flags |= LOOKUP_CREATE;
+		d = aufs_lookup(dir, dentry, lkup_flags);
+		if (IS_ERR(d)) {
+			err = PTR_ERR(d);
+			AuTraceErr(err);
+			goto out;
+		} else if (d) {
+			/*
+			 * obsoleted dentry found.
+			 * another error will be returned later.
+			 */
+			d_drop(d);
+			AuDbgDentry(d);
+			dput(d);
+		}
+		AuDbgDentry(dentry);
+	}
+
+	if (d_is_positive(dentry)
+	    || d_unhashed(dentry)
+	    || d_unlinked(dentry)
+	    || !(open_flag & O_CREAT)) {
+		err = au_aopen_no_open(file, dentry);
+		goto out; /* success */
+	}
+
+	err = aufs_read_lock(dentry, AuLock_DW | AuLock_FLUSH | AuLock_GEN);
+	if (unlikely(err))
+		goto out;
+
+	sb = dentry->d_sb;
+	parent = dentry->d_parent;	/* dir is locked */
+	di_write_lock_parent(parent);
+	err = au_lkup_dentry(dentry, /*btop*/0, AuLkup_ALLOW_NEG);
+	if (unlikely(err < 0))
+		goto out_parent;
+
+	AuDbgDentry(dentry);
+	if (d_is_positive(dentry)) {
+		err = au_aopen_no_open(file, dentry);
+		goto out_parent; /* success */
+	}
+
+	args.file = alloc_empty_file(file->f_flags, current_cred());
+	err = PTR_ERR(args.file);
+	if (IS_ERR(args.file))
+		goto out_parent;
+
+	bindex = au_dbtop(dentry);
+	err = au_aopen_or_create(dir, dentry, &args);
+	AuTraceErr(err);
+	AuDbgFile(args.file);
+	file->f_mode = args.file->f_mode & ~FMODE_OPENED;
+	did_open = !!(args.file->f_mode & FMODE_OPENED);
+	if (!did_open) {
+		fput(args.file);
+		args.file = NULL;
+	}
+	di_write_unlock(parent);
+	di_write_unlock(dentry);
+	if (unlikely(err < 0)) {
+		if (args.file)
+			fput(args.file);
+		goto out_sb;
+	}
+
+	if (!did_open)
+		err = au_aopen_no_open(file, dentry);
+	else {
+		aopen_node.h_file = args.file;
+		err = au_aopen_do_open(file, dentry, &aopen_node);
+	}
+	if (unlikely(err < 0)) {
+		if (args.file)
+			fput(args.file);
+		if (did_open)
+			au_lcnt_dec(&args.br->br_nfiles);
+	}
+	goto out_sb; /* success */
+
+out_parent:
+	di_write_unlock(parent);
+	di_write_unlock(dentry);
+out_sb:
+	si_read_unlock(sb);
+out:
+	AuTraceErr(err);
+	AuDbgFile(file);
+	return err;
+}
+
+
+/* ---------------------------------------------------------------------- */
+
+static int au_wr_dir_cpup(struct dentry *dentry, struct dentry *parent,
+			  const unsigned char add_entry, aufs_bindex_t bcpup,
+			  aufs_bindex_t btop)
+{
+	int err;
+	struct dentry *h_parent;
+	struct inode *h_dir;
+
+	if (add_entry)
+		IMustLock(d_inode(parent));
+	else
+		di_write_lock_parent(parent);
+
+	err = 0;
+	if (!au_h_dptr(parent, bcpup)) {
+		if (btop > bcpup)
+			err = au_cpup_dirs(dentry, bcpup);
+		else if (btop < bcpup)
+			err = au_cpdown_dirs(dentry, bcpup);
+		else
+			BUG();
+	}
+	if (!err && add_entry && !au_ftest_wrdir(add_entry, TMPFILE)) {
+		h_parent = au_h_dptr(parent, bcpup);
+		h_dir = d_inode(h_parent);
+		inode_lock_shared_nested(h_dir, AuLsc_I_PARENT);
+		err = au_lkup_neg(dentry, bcpup, /*wh*/0);
+		/* todo: no unlock here */
+		inode_unlock_shared(h_dir);
+
+		AuDbg("bcpup %d\n", bcpup);
+		if (!err) {
+			if (d_really_is_negative(dentry))
+				au_set_h_dptr(dentry, btop, NULL);
+			au_update_dbrange(dentry, /*do_put_zero*/0);
+		}
+	}
+
+	if (!add_entry)
+		di_write_unlock(parent);
+	if (!err)
+		err = bcpup; /* success */
+
+	AuTraceErr(err);
+	return err;
+}
+
+/*
+ * decide the branch and the parent dir where we will create a new entry.
+ * returns new bindex or an error.
+ * copyup the parent dir if needed.
+ */
+int au_wr_dir(struct dentry *dentry, struct dentry *src_dentry,
+	      struct au_wr_dir_args *args)
+{
+	int err;
+	unsigned int flags;
+	aufs_bindex_t bcpup, btop, src_btop;
+	const unsigned char add_entry
+		= au_ftest_wrdir(args->flags, ADD_ENTRY)
+		| au_ftest_wrdir(args->flags, TMPFILE);
+	struct super_block *sb;
+	struct dentry *parent;
+	struct au_sbinfo *sbinfo;
+
+	sb = dentry->d_sb;
+	sbinfo = au_sbi(sb);
+	parent = dget_parent(dentry);
+	btop = au_dbtop(dentry);
+	bcpup = btop;
+	if (args->force_btgt < 0) {
+		if (src_dentry) {
+			src_btop = au_dbtop(src_dentry);
+			if (src_btop < btop)
+				bcpup = src_btop;
+		} else if (add_entry) {
+			flags = 0;
+			if (au_ftest_wrdir(args->flags, ISDIR))
+				au_fset_wbr(flags, DIR);
+			err = AuWbrCreate(sbinfo, dentry, flags);
+			bcpup = err;
+		}
+
+		if (bcpup < 0 || au_test_ro(sb, bcpup, d_inode(dentry))) {
+			if (add_entry)
+				err = AuWbrCopyup(sbinfo, dentry);
+			else {
+				if (!IS_ROOT(dentry)) {
+					di_read_lock_parent(parent, !AuLock_IR);
+					err = AuWbrCopyup(sbinfo, dentry);
+					di_read_unlock(parent, !AuLock_IR);
+				} else
+					err = AuWbrCopyup(sbinfo, dentry);
+			}
+			bcpup = err;
+			if (unlikely(err < 0))
+				goto out;
+		}
+	} else {
+		bcpup = args->force_btgt;
+		AuDebugOn(au_test_ro(sb, bcpup, d_inode(dentry)));
+	}
+
+	AuDbg("btop %d, bcpup %d\n", btop, bcpup);
+	err = bcpup;
+	if (bcpup == btop)
+		goto out; /* success */
+
+	/* copyup the new parent into the branch we process */
+	err = au_wr_dir_cpup(dentry, parent, add_entry, bcpup, btop);
+	if (err >= 0) {
+		if (d_really_is_negative(dentry)) {
+			au_set_h_dptr(dentry, btop, NULL);
+			au_set_dbtop(dentry, bcpup);
+			au_set_dbbot(dentry, bcpup);
+		}
+		AuDebugOn(add_entry
+			  && !au_ftest_wrdir(args->flags, TMPFILE)
+			  && !au_h_dptr(dentry, bcpup));
+	}
+
+out:
+	dput(parent);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+void au_pin_hdir_unlock(struct au_pin *p)
+{
+	if (p->hdir)
+		au_hn_inode_unlock(p->hdir);
+}
+
+int au_pin_hdir_lock(struct au_pin *p)
+{
+	int err;
+
+	err = 0;
+	if (!p->hdir)
+		goto out;
+
+	/* even if an error happens later, keep this lock */
+	au_hn_inode_lock_nested(p->hdir, p->lsc_hi);
+
+	err = -EBUSY;
+	if (unlikely(p->hdir->hi_inode != d_inode(p->h_parent)))
+		goto out;
+
+	err = 0;
+	if (p->h_dentry)
+		err = au_h_verify(p->h_dentry, p->udba, p->hdir->hi_inode,
+				  p->h_parent, p->br);
+
+out:
+	return err;
+}
+
+int au_pin_hdir_relock(struct au_pin *p)
+{
+	int err, i;
+	struct inode *h_i;
+	struct dentry *h_d[] = {
+		p->h_dentry,
+		p->h_parent
+	};
+
+	err = au_pin_hdir_lock(p);
+	if (unlikely(err))
+		goto out;
+
+	for (i = 0; !err && i < sizeof(h_d)/sizeof(*h_d); i++) {
+		if (!h_d[i])
+			continue;
+		if (d_is_positive(h_d[i])) {
+			h_i = d_inode(h_d[i]);
+			err = !h_i->i_nlink;
+		}
+	}
+
+out:
+	return err;
+}
+
+static void au_pin_hdir_set_owner(struct au_pin *p, struct task_struct *task)
+{
+#if !defined(CONFIG_RWSEM_GENERIC_SPINLOCK) \
+	&& defined(CONFIG_RWSEM_SPIN_ON_OWNER)
+	p->hdir->hi_inode->i_rwsem.owner = task;
+#endif
+}
+
+void au_pin_hdir_acquire_nest(struct au_pin *p)
+{
+	if (p->hdir) {
+		rwsem_acquire_nest(&p->hdir->hi_inode->i_rwsem.dep_map,
+				   p->lsc_hi, 0, NULL, _RET_IP_);
+		au_pin_hdir_set_owner(p, current);
+	}
+}
+
+void au_pin_hdir_release(struct au_pin *p)
+{
+	if (p->hdir) {
+		au_pin_hdir_set_owner(p, p->task);
+		rwsem_release(&p->hdir->hi_inode->i_rwsem.dep_map, 1, _RET_IP_);
+	}
+}
+
+struct dentry *au_pinned_h_parent(struct au_pin *pin)
+{
+	if (pin && pin->parent)
+		return au_h_dptr(pin->parent, pin->bindex);
+	return NULL;
+}
+
+void au_unpin(struct au_pin *p)
+{
+	if (p->hdir)
+		au_pin_hdir_unlock(p);
+	if (p->h_mnt && au_ftest_pin(p->flags, MNT_WRITE))
+		vfsub_mnt_drop_write(p->h_mnt);
+	if (!p->hdir)
+		return;
+
+	if (!au_ftest_pin(p->flags, DI_LOCKED))
+		di_read_unlock(p->parent, AuLock_IR);
+	iput(p->hdir->hi_inode);
+	dput(p->parent);
+	p->parent = NULL;
+	p->hdir = NULL;
+	p->h_mnt = NULL;
+	/* do not clear p->task */
+}
+
+int au_do_pin(struct au_pin *p)
+{
+	int err;
+	struct super_block *sb;
+	struct inode *h_dir;
+
+	err = 0;
+	sb = p->dentry->d_sb;
+	p->br = au_sbr(sb, p->bindex);
+	if (IS_ROOT(p->dentry)) {
+		if (au_ftest_pin(p->flags, MNT_WRITE)) {
+			p->h_mnt = au_br_mnt(p->br);
+			err = vfsub_mnt_want_write(p->h_mnt);
+			if (unlikely(err)) {
+				au_fclr_pin(p->flags, MNT_WRITE);
+				goto out_err;
+			}
+		}
+		goto out;
+	}
+
+	p->h_dentry = NULL;
+	if (p->bindex <= au_dbbot(p->dentry))
+		p->h_dentry = au_h_dptr(p->dentry, p->bindex);
+
+	p->parent = dget_parent(p->dentry);
+	if (!au_ftest_pin(p->flags, DI_LOCKED))
+		di_read_lock(p->parent, AuLock_IR, p->lsc_di);
+
+	h_dir = NULL;
+	p->h_parent = au_h_dptr(p->parent, p->bindex);
+	p->hdir = au_hi(d_inode(p->parent), p->bindex);
+	if (p->hdir)
+		h_dir = p->hdir->hi_inode;
+
+	/*
+	 * udba case, or
+	 * if DI_LOCKED is not set, then p->parent may be different
+	 * and h_parent can be NULL.
+	 */
+	if (unlikely(!p->hdir || !h_dir || !p->h_parent)) {
+		err = -EBUSY;
+		if (!au_ftest_pin(p->flags, DI_LOCKED))
+			di_read_unlock(p->parent, AuLock_IR);
+		dput(p->parent);
+		p->parent = NULL;
+		goto out_err;
+	}
+
+	if (au_ftest_pin(p->flags, MNT_WRITE)) {
+		p->h_mnt = au_br_mnt(p->br);
+		err = vfsub_mnt_want_write(p->h_mnt);
+		if (unlikely(err)) {
+			au_fclr_pin(p->flags, MNT_WRITE);
+			if (!au_ftest_pin(p->flags, DI_LOCKED))
+				di_read_unlock(p->parent, AuLock_IR);
+			dput(p->parent);
+			p->parent = NULL;
+			goto out_err;
+		}
+	}
+
+	au_igrab(h_dir);
+	err = au_pin_hdir_lock(p);
+	if (!err)
+		goto out; /* success */
+
+	au_unpin(p);
+
+out_err:
+	pr_err("err %d\n", err);
+	err = au_busy_or_stale();
+out:
+	return err;
+}
+
+void au_pin_init(struct au_pin *p, struct dentry *dentry,
+		 aufs_bindex_t bindex, int lsc_di, int lsc_hi,
+		 unsigned int udba, unsigned char flags)
+{
+	p->dentry = dentry;
+	p->udba = udba;
+	p->lsc_di = lsc_di;
+	p->lsc_hi = lsc_hi;
+	p->flags = flags;
+	p->bindex = bindex;
+
+	p->parent = NULL;
+	p->hdir = NULL;
+	p->h_mnt = NULL;
+
+	p->h_dentry = NULL;
+	p->h_parent = NULL;
+	p->br = NULL;
+	p->task = current;
+}
+
+int au_pin(struct au_pin *pin, struct dentry *dentry, aufs_bindex_t bindex,
+	   unsigned int udba, unsigned char flags)
+{
+	au_pin_init(pin, dentry, bindex, AuLsc_DI_PARENT, AuLsc_I_PARENT2,
+		    udba, flags);
+	return au_do_pin(pin);
+}
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * ->setattr() and ->getattr() are called in various cases.
+ * chmod, stat: dentry is revalidated.
+ * fchmod, fstat: file and dentry are not revalidated, additionally they may be
+ *		  unhashed.
+ * for ->setattr(), ia->ia_file is passed from ftruncate only.
+ */
+/* todo: consolidate with do_refresh() and simple_reval_dpath() */
+int au_reval_for_attr(struct dentry *dentry, unsigned int sigen)
+{
+	int err;
+	struct dentry *parent;
+
+	err = 0;
+	if (au_digen_test(dentry, sigen)) {
+		parent = dget_parent(dentry);
+		di_read_lock_parent(parent, AuLock_IR);
+		err = au_refresh_dentry(dentry, parent);
+		di_read_unlock(parent, AuLock_IR);
+		dput(parent);
+	}
+
+	AuTraceErr(err);
+	return err;
+}
+
+int au_pin_and_icpup(struct dentry *dentry, struct iattr *ia,
+		     struct au_icpup_args *a)
+{
+	int err;
+	loff_t sz;
+	aufs_bindex_t btop, ibtop;
+	struct dentry *hi_wh, *parent;
+	struct inode *inode;
+	struct au_wr_dir_args wr_dir_args = {
+		.force_btgt	= -1,
+		.flags		= 0
+	};
+
+	if (d_is_dir(dentry))
+		au_fset_wrdir(wr_dir_args.flags, ISDIR);
+	/* plink or hi_wh() case */
+	btop = au_dbtop(dentry);
+	inode = d_inode(dentry);
+	ibtop = au_ibtop(inode);
+	if (btop != ibtop && !au_test_ro(inode->i_sb, ibtop, inode))
+		wr_dir_args.force_btgt = ibtop;
+	err = au_wr_dir(dentry, /*src_dentry*/NULL, &wr_dir_args);
+	if (unlikely(err < 0))
+		goto out;
+	a->btgt = err;
+	if (err != btop)
+		au_fset_icpup(a->flags, DID_CPUP);
+
+	err = 0;
+	a->pin_flags = AuPin_MNT_WRITE;
+	parent = NULL;
+	if (!IS_ROOT(dentry)) {
+		au_fset_pin(a->pin_flags, DI_LOCKED);
+		parent = dget_parent(dentry);
+		di_write_lock_parent(parent);
+	}
+
+	err = au_pin(&a->pin, dentry, a->btgt, a->udba, a->pin_flags);
+	if (unlikely(err))
+		goto out_parent;
+
+	sz = -1;
+	a->h_path.dentry = au_h_dptr(dentry, btop);
+	a->h_inode = d_inode(a->h_path.dentry);
+	if (ia && (ia->ia_valid & ATTR_SIZE)) {
+		inode_lock_shared_nested(a->h_inode, AuLsc_I_CHILD);
+		if (ia->ia_size < i_size_read(a->h_inode))
+			sz = ia->ia_size;
+		inode_unlock_shared(a->h_inode);
+	}
+
+	hi_wh = NULL;
+	if (au_ftest_icpup(a->flags, DID_CPUP) && d_unlinked(dentry)) {
+		hi_wh = au_hi_wh(inode, a->btgt);
+		if (!hi_wh) {
+			struct au_cp_generic cpg = {
+				.dentry	= dentry,
+				.bdst	= a->btgt,
+				.bsrc	= -1,
+				.len	= sz,
+				.pin	= &a->pin
+			};
+			err = au_sio_cpup_wh(&cpg, /*file*/NULL);
+			if (unlikely(err))
+				goto out_unlock;
+			hi_wh = au_hi_wh(inode, a->btgt);
+			/* todo: revalidate hi_wh? */
+		}
+	}
+
+	if (parent) {
+		au_pin_set_parent_lflag(&a->pin, /*lflag*/0);
+		di_downgrade_lock(parent, AuLock_IR);
+		dput(parent);
+		parent = NULL;
+	}
+	if (!au_ftest_icpup(a->flags, DID_CPUP))
+		goto out; /* success */
+
+	if (!d_unhashed(dentry)) {
+		struct au_cp_generic cpg = {
+			.dentry	= dentry,
+			.bdst	= a->btgt,
+			.bsrc	= btop,
+			.len	= sz,
+			.pin	= &a->pin,
+			.flags	= AuCpup_DTIME | AuCpup_HOPEN
+		};
+		err = au_sio_cpup_simple(&cpg);
+		if (!err)
+			a->h_path.dentry = au_h_dptr(dentry, a->btgt);
+	} else if (!hi_wh)
+		a->h_path.dentry = au_h_dptr(dentry, a->btgt);
+	else
+		a->h_path.dentry = hi_wh; /* do not dget here */
+
+out_unlock:
+	a->h_inode = d_inode(a->h_path.dentry);
+	if (!err)
+		goto out; /* success */
+	au_unpin(&a->pin);
+out_parent:
+	if (parent) {
+		di_write_unlock(parent);
+		dput(parent);
+	}
+out:
+	if (!err)
+		inode_lock_nested(a->h_inode, AuLsc_I_CHILD);
+	return err;
+}
+
+static int aufs_setattr(struct dentry *dentry, struct iattr *ia)
+{
+	int err;
+	struct inode *inode, *delegated;
+	struct super_block *sb;
+	struct file *file;
+	struct au_icpup_args *a;
+
+	inode = d_inode(dentry);
+	IMustLock(inode);
+
+	err = setattr_prepare(dentry, ia);
+	if (unlikely(err))
+		goto out;
+
+	err = -ENOMEM;
+	a = kzalloc(sizeof(*a), GFP_NOFS);
+	if (unlikely(!a))
+		goto out;
+
+	if (ia->ia_valid & (ATTR_KILL_SUID | ATTR_KILL_SGID))
+		ia->ia_valid &= ~ATTR_MODE;
+
+	file = NULL;
+	sb = dentry->d_sb;
+	err = si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLM);
+	if (unlikely(err))
+		goto out_kfree;
+
+	if (ia->ia_valid & ATTR_FILE) {
+		/* currently ftruncate(2) only */
+		AuDebugOn(!d_is_reg(dentry));
+		file = ia->ia_file;
+		err = au_reval_and_lock_fdi(file, au_reopen_nondir, /*wlock*/1,
+					    /*fi_lsc*/0);
+		if (unlikely(err))
+			goto out_si;
+		ia->ia_file = au_hf_top(file);
+		a->udba = AuOpt_UDBA_NONE;
+	} else {
+		/* fchmod() doesn't pass ia_file */
+		a->udba = au_opt_udba(sb);
+		di_write_lock_child(dentry);
+		/* no d_unlinked(), to set UDBA_NONE for root */
+		if (d_unhashed(dentry))
+			a->udba = AuOpt_UDBA_NONE;
+		if (a->udba != AuOpt_UDBA_NONE) {
+			AuDebugOn(IS_ROOT(dentry));
+			err = au_reval_for_attr(dentry, au_sigen(sb));
+			if (unlikely(err))
+				goto out_dentry;
+		}
+	}
+
+	err = au_pin_and_icpup(dentry, ia, a);
+	if (unlikely(err < 0))
+		goto out_dentry;
+	if (au_ftest_icpup(a->flags, DID_CPUP)) {
+		ia->ia_file = NULL;
+		ia->ia_valid &= ~ATTR_FILE;
+	}
+
+	a->h_path.mnt = au_sbr_mnt(sb, a->btgt);
+	if ((ia->ia_valid & (ATTR_MODE | ATTR_CTIME))
+	    == (ATTR_MODE | ATTR_CTIME)) {
+		err = security_path_chmod(&a->h_path, ia->ia_mode);
+		if (unlikely(err))
+			goto out_unlock;
+	} else if ((ia->ia_valid & (ATTR_UID | ATTR_GID))
+		   && (ia->ia_valid & ATTR_CTIME)) {
+		err = security_path_chown(&a->h_path, ia->ia_uid, ia->ia_gid);
+		if (unlikely(err))
+			goto out_unlock;
+	}
+
+	if (ia->ia_valid & ATTR_SIZE) {
+		struct file *f;
+
+		if (ia->ia_size < i_size_read(inode))
+			/* unmap only */
+			truncate_setsize(inode, ia->ia_size);
+
+		f = NULL;
+		if (ia->ia_valid & ATTR_FILE)
+			f = ia->ia_file;
+		inode_unlock(a->h_inode);
+		err = vfsub_trunc(&a->h_path, ia->ia_size, ia->ia_valid, f);
+		inode_lock_nested(a->h_inode, AuLsc_I_CHILD);
+	} else {
+		delegated = NULL;
+		while (1) {
+			err = vfsub_notify_change(&a->h_path, ia, &delegated);
+			if (delegated) {
+				err = break_deleg_wait(&delegated);
+				if (!err)
+					continue;
+			}
+			break;
+		}
+	}
+	/*
+	 * regardless aufs 'acl' option setting.
+	 * why don't all acl-aware fs call this func from their ->setattr()?
+	 */
+	if (!err && (ia->ia_valid & ATTR_MODE))
+		err = vfsub_acl_chmod(a->h_inode, ia->ia_mode);
+	if (!err)
+		au_cpup_attr_changeable(inode);
+
+out_unlock:
+	inode_unlock(a->h_inode);
+	au_unpin(&a->pin);
+	if (unlikely(err))
+		au_update_dbtop(dentry);
+out_dentry:
+	di_write_unlock(dentry);
+	if (file) {
+		fi_write_unlock(file);
+		ia->ia_file = file;
+		ia->ia_valid |= ATTR_FILE;
+	}
+out_si:
+	si_read_unlock(sb);
+out_kfree:
+	au_kfree_rcu(a);
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+#if IS_ENABLED(CONFIG_AUFS_XATTR) || IS_ENABLED(CONFIG_FS_POSIX_ACL)
+static int au_h_path_to_set_attr(struct dentry *dentry,
+				 struct au_icpup_args *a, struct path *h_path)
+{
+	int err;
+	struct super_block *sb;
+
+	sb = dentry->d_sb;
+	a->udba = au_opt_udba(sb);
+	/* no d_unlinked(), to set UDBA_NONE for root */
+	if (d_unhashed(dentry))
+		a->udba = AuOpt_UDBA_NONE;
+	if (a->udba != AuOpt_UDBA_NONE) {
+		AuDebugOn(IS_ROOT(dentry));
+		err = au_reval_for_attr(dentry, au_sigen(sb));
+		if (unlikely(err))
+			goto out;
+	}
+	err = au_pin_and_icpup(dentry, /*ia*/NULL, a);
+	if (unlikely(err < 0))
+		goto out;
+
+	h_path->dentry = a->h_path.dentry;
+	h_path->mnt = au_sbr_mnt(sb, a->btgt);
+
+out:
+	return err;
+}
+
+ssize_t au_sxattr(struct dentry *dentry, struct inode *inode,
+		  struct au_sxattr *arg)
+{
+	int err;
+	struct path h_path;
+	struct super_block *sb;
+	struct au_icpup_args *a;
+	struct inode *h_inode;
+
+	IMustLock(inode);
+
+	err = -ENOMEM;
+	a = kzalloc(sizeof(*a), GFP_NOFS);
+	if (unlikely(!a))
+		goto out;
+
+	sb = dentry->d_sb;
+	err = si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLM);
+	if (unlikely(err))
+		goto out_kfree;
+
+	h_path.dentry = NULL;	/* silence gcc */
+	di_write_lock_child(dentry);
+	err = au_h_path_to_set_attr(dentry, a, &h_path);
+	if (unlikely(err))
+		goto out_di;
+
+	inode_unlock(a->h_inode);
+	switch (arg->type) {
+	case AU_XATTR_SET:
+		AuDebugOn(d_is_negative(h_path.dentry));
+		err = vfsub_setxattr(h_path.dentry,
+				     arg->u.set.name, arg->u.set.value,
+				     arg->u.set.size, arg->u.set.flags);
+		break;
+	case AU_ACL_SET:
+		err = -EOPNOTSUPP;
+		h_inode = d_inode(h_path.dentry);
+		if (h_inode->i_op->set_acl)
+			/* this will call posix_acl_update_mode */
+			err = h_inode->i_op->set_acl(h_inode,
+						     arg->u.acl_set.acl,
+						     arg->u.acl_set.type);
+		break;
+	}
+	if (!err)
+		au_cpup_attr_timesizes(inode);
+
+	au_unpin(&a->pin);
+	if (unlikely(err))
+		au_update_dbtop(dentry);
+
+out_di:
+	di_write_unlock(dentry);
+	si_read_unlock(sb);
+out_kfree:
+	au_kfree_rcu(a);
+out:
+	AuTraceErr(err);
+	return err;
+}
+#endif
+
+static void au_refresh_iattr(struct inode *inode, struct kstat *st,
+			     unsigned int nlink)
+{
+	unsigned int n;
+
+	inode->i_mode = st->mode;
+	/* don't i_[ug]id_write() here */
+	inode->i_uid = st->uid;
+	inode->i_gid = st->gid;
+	inode->i_atime = st->atime;
+	inode->i_mtime = st->mtime;
+	inode->i_ctime = st->ctime;
+
+	au_cpup_attr_nlink(inode, /*force*/0);
+	if (S_ISDIR(inode->i_mode)) {
+		n = inode->i_nlink;
+		n -= nlink;
+		n += st->nlink;
+		smp_mb(); /* for i_nlink */
+		/* 0 can happen */
+		set_nlink(inode, n);
+	}
+
+	spin_lock(&inode->i_lock);
+	inode->i_blocks = st->blocks;
+	i_size_write(inode, st->size);
+	spin_unlock(&inode->i_lock);
+}
+
+/*
+ * common routine for aufs_getattr() and au_getxattr().
+ * returns zero or negative (an error).
+ * @dentry will be read-locked in success.
+ */
+int au_h_path_getattr(struct dentry *dentry, struct inode *inode, int force,
+		      struct path *h_path, int locked)
+{
+	int err;
+	unsigned int mnt_flags, sigen;
+	unsigned char udba_none;
+	aufs_bindex_t bindex;
+	struct super_block *sb, *h_sb;
+
+	h_path->mnt = NULL;
+	h_path->dentry = NULL;
+
+	err = 0;
+	sb = dentry->d_sb;
+	mnt_flags = au_mntflags(sb);
+	udba_none = !!au_opt_test(mnt_flags, UDBA_NONE);
+
+	if (unlikely(locked))
+		goto body; /* skip locking dinfo */
+
+	/* support fstat(2) */
+	if (!d_unlinked(dentry) && !udba_none) {
+		sigen = au_sigen(sb);
+		err = au_digen_test(dentry, sigen);
+		if (!err) {
+			di_read_lock_child(dentry, AuLock_IR);
+			err = au_dbrange_test(dentry);
+			if (unlikely(err)) {
+				di_read_unlock(dentry, AuLock_IR);
+				goto out;
+			}
+		} else {
+			AuDebugOn(IS_ROOT(dentry));
+			di_write_lock_child(dentry);
+			err = au_dbrange_test(dentry);
+			if (!err)
+				err = au_reval_for_attr(dentry, sigen);
+			if (!err)
+				di_downgrade_lock(dentry, AuLock_IR);
+			else {
+				di_write_unlock(dentry);
+				goto out;
+			}
+		}
+	} else
+		di_read_lock_child(dentry, AuLock_IR);
+
+body:
+	if (!inode) {
+		inode = d_inode(dentry);
+		if (unlikely(!inode))
+			goto out;
+	}
+	bindex = au_ibtop(inode);
+	h_path->mnt = au_sbr_mnt(sb, bindex);
+	h_sb = h_path->mnt->mnt_sb;
+	if (!force
+	    && !au_test_fs_bad_iattr(h_sb)
+	    && udba_none)
+		goto out; /* success */
+
+	if (au_dbtop(dentry) == bindex)
+		h_path->dentry = au_h_dptr(dentry, bindex);
+	else if (au_opt_test(mnt_flags, PLINK) && au_plink_test(inode)) {
+		h_path->dentry = au_plink_lkup(inode, bindex);
+		if (IS_ERR(h_path->dentry))
+			/* pretending success */
+			h_path->dentry = NULL;
+		else
+			dput(h_path->dentry);
+	}
+
+out:
+	return err;
+}
+
+static int aufs_getattr(const struct path *path, struct kstat *st,
+			u32 request, unsigned int query)
+{
+	int err;
+	unsigned char positive;
+	struct path h_path;
+	struct dentry *dentry;
+	struct inode *inode;
+	struct super_block *sb;
+
+	dentry = path->dentry;
+	inode = d_inode(dentry);
+	sb = dentry->d_sb;
+	err = si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLM);
+	if (unlikely(err))
+		goto out;
+	err = au_h_path_getattr(dentry, /*inode*/NULL, /*force*/0, &h_path,
+				/*locked*/0);
+	if (unlikely(err))
+		goto out_si;
+	if (unlikely(!h_path.dentry))
+		/* illegally overlapped or something */
+		goto out_fill; /* pretending success */
+
+	positive = d_is_positive(h_path.dentry);
+	if (positive)
+		/* no vfsub version */
+		err = vfs_getattr(&h_path, st, request, query);
+	if (!err) {
+		if (positive)
+			au_refresh_iattr(inode, st,
+					 d_inode(h_path.dentry)->i_nlink);
+		goto out_fill; /* success */
+	}
+	AuTraceErr(err);
+	goto out_di;
+
+out_fill:
+	generic_fillattr(inode, st);
+out_di:
+	di_read_unlock(dentry, AuLock_IR);
+out_si:
+	si_read_unlock(sb);
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static const char *aufs_get_link(struct dentry *dentry, struct inode *inode,
+				 struct delayed_call *done)
+{
+	const char *ret;
+	struct dentry *h_dentry;
+	struct inode *h_inode;
+	int err;
+	aufs_bindex_t bindex;
+
+	ret = NULL; /* suppress a warning */
+	err = -ECHILD;
+	if (!dentry)
+		goto out;
+
+	err = aufs_read_lock(dentry, AuLock_IR | AuLock_GEN);
+	if (unlikely(err))
+		goto out;
+
+	err = au_d_hashed_positive(dentry);
+	if (unlikely(err))
+		goto out_unlock;
+
+	err = -EINVAL;
+	inode = d_inode(dentry);
+	bindex = au_ibtop(inode);
+	h_inode = au_h_iptr(inode, bindex);
+	if (unlikely(!h_inode->i_op->get_link))
+		goto out_unlock;
+
+	err = -EBUSY;
+	h_dentry = NULL;
+	if (au_dbtop(dentry) <= bindex) {
+		h_dentry = au_h_dptr(dentry, bindex);
+		if (h_dentry)
+			dget(h_dentry);
+	}
+	if (!h_dentry) {
+		h_dentry = d_find_any_alias(h_inode);
+		if (IS_ERR(h_dentry)) {
+			err = PTR_ERR(h_dentry);
+			goto out_unlock;
+		}
+	}
+	if (unlikely(!h_dentry))
+		goto out_unlock;
+
+	err = 0;
+	AuDbg("%ps\n", h_inode->i_op->get_link);
+	AuDbgDentry(h_dentry);
+	ret = vfs_get_link(h_dentry, done);
+	dput(h_dentry);
+	if (IS_ERR(ret))
+		err = PTR_ERR(ret);
+
+out_unlock:
+	aufs_read_unlock(dentry, AuLock_IR);
+out:
+	if (unlikely(err))
+		ret = ERR_PTR(err);
+	AuTraceErrPtr(ret);
+	return ret;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static int au_is_special(struct inode *inode)
+{
+	return (inode->i_mode & (S_IFBLK | S_IFCHR | S_IFIFO | S_IFSOCK));
+}
+
+static int aufs_update_time(struct inode *inode, struct timespec64 *ts,
+			    int flags)
+{
+	int err;
+	aufs_bindex_t bindex;
+	struct super_block *sb;
+	struct inode *h_inode;
+	struct vfsmount *h_mnt;
+
+	sb = inode->i_sb;
+	WARN_ONCE((flags & S_ATIME) && !IS_NOATIME(inode),
+		  "unexpected s_flags 0x%lx", sb->s_flags);
+
+	/* mmap_sem might be acquired already, cf. aufs_mmap() */
+	lockdep_off();
+	si_read_lock(sb, AuLock_FLUSH);
+	ii_write_lock_child(inode);
+
+	err = 0;
+	bindex = au_ibtop(inode);
+	h_inode = au_h_iptr(inode, bindex);
+	if (!au_test_ro(sb, bindex, inode)) {
+		h_mnt = au_sbr_mnt(sb, bindex);
+		err = vfsub_mnt_want_write(h_mnt);
+		if (!err) {
+			err = vfsub_update_time(h_inode, ts, flags);
+			vfsub_mnt_drop_write(h_mnt);
+		}
+	} else if (au_is_special(h_inode)) {
+		/*
+		 * Never copy-up here.
+		 * These special files may already be opened and used for
+		 * communicating. If we copied it up, then the communication
+		 * would be corrupted.
+		 */
+		AuWarn1("timestamps for i%lu are ignored "
+			"since it is on readonly branch (hi%lu).\n",
+			inode->i_ino, h_inode->i_ino);
+	} else if (flags & ~S_ATIME) {
+		err = -EIO;
+		AuIOErr1("unexpected flags 0x%x\n", flags);
+		AuDebugOn(1);
+	}
+
+	if (!err)
+		au_cpup_attr_timesizes(inode);
+	ii_write_unlock(inode);
+	si_read_unlock(sb);
+	lockdep_on();
+
+	if (!err && (flags & S_VERSION))
+		inode_inc_iversion(inode);
+
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* no getattr version will be set by module.c:aufs_init() */
+struct inode_operations aufs_iop_nogetattr[AuIop_Last],
+	aufs_iop[] = {
+	[AuIop_SYMLINK] = {
+		.permission	= aufs_permission,
+#ifdef CONFIG_FS_POSIX_ACL
+		.get_acl	= aufs_get_acl,
+		.set_acl	= aufs_set_acl, /* unsupport for symlink? */
+#endif
+
+		.setattr	= aufs_setattr,
+		.getattr	= aufs_getattr,
+
+#ifdef CONFIG_AUFS_XATTR
+		.listxattr	= aufs_listxattr,
+#endif
+
+		.get_link	= aufs_get_link,
+
+		/* .update_time	= aufs_update_time */
+	},
+	[AuIop_DIR] = {
+		.create		= aufs_create,
+		.lookup		= aufs_lookup,
+		.link		= aufs_link,
+		.unlink		= aufs_unlink,
+		.symlink	= aufs_symlink,
+		.mkdir		= aufs_mkdir,
+		.rmdir		= aufs_rmdir,
+		.mknod		= aufs_mknod,
+		.rename		= aufs_rename,
+
+		.permission	= aufs_permission,
+#ifdef CONFIG_FS_POSIX_ACL
+		.get_acl	= aufs_get_acl,
+		.set_acl	= aufs_set_acl,
+#endif
+
+		.setattr	= aufs_setattr,
+		.getattr	= aufs_getattr,
+
+#ifdef CONFIG_AUFS_XATTR
+		.listxattr	= aufs_listxattr,
+#endif
+
+		.update_time	= aufs_update_time,
+		.atomic_open	= aufs_atomic_open,
+		.tmpfile	= aufs_tmpfile
+	},
+	[AuIop_OTHER] = {
+		.permission	= aufs_permission,
+#ifdef CONFIG_FS_POSIX_ACL
+		.get_acl	= aufs_get_acl,
+		.set_acl	= aufs_set_acl,
+#endif
+
+		.setattr	= aufs_setattr,
+		.getattr	= aufs_getattr,
+
+#ifdef CONFIG_AUFS_XATTR
+		.listxattr	= aufs_listxattr,
+#endif
+
+		.update_time	= aufs_update_time
+	}
+};
diff -Naur --no-dereference a/fs/aufs/i_op_del.c b/fs/aufs/i_op_del.c
--- a/fs/aufs/i_op_del.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/i_op_del.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,513 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * inode operations (del entry)
+ */
+
+#include <linux/iversion.h>
+#include "aufs.h"
+
+/*
+ * decide if a new whiteout for @dentry is necessary or not.
+ * when it is necessary, prepare the parent dir for the upper branch whose
+ * branch index is @bcpup for creation. the actual creation of the whiteout will
+ * be done by caller.
+ * return value:
+ * 0: wh is unnecessary
+ * plus: wh is necessary
+ * minus: error
+ */
+int au_wr_dir_need_wh(struct dentry *dentry, int isdir, aufs_bindex_t *bcpup)
+{
+	int need_wh, err;
+	aufs_bindex_t btop;
+	struct super_block *sb;
+
+	sb = dentry->d_sb;
+	btop = au_dbtop(dentry);
+	if (*bcpup < 0) {
+		*bcpup = btop;
+		if (au_test_ro(sb, btop, d_inode(dentry))) {
+			err = AuWbrCopyup(au_sbi(sb), dentry);
+			*bcpup = err;
+			if (unlikely(err < 0))
+				goto out;
+		}
+	} else
+		AuDebugOn(btop < *bcpup
+			  || au_test_ro(sb, *bcpup, d_inode(dentry)));
+	AuDbg("bcpup %d, btop %d\n", *bcpup, btop);
+
+	if (*bcpup != btop) {
+		err = au_cpup_dirs(dentry, *bcpup);
+		if (unlikely(err))
+			goto out;
+		need_wh = 1;
+	} else {
+		struct au_dinfo *dinfo, *tmp;
+
+		need_wh = -ENOMEM;
+		dinfo = au_di(dentry);
+		tmp = au_di_alloc(sb, AuLsc_DI_TMP);
+		if (tmp) {
+			au_di_cp(tmp, dinfo);
+			au_di_swap(tmp, dinfo);
+			/* returns the number of positive dentries */
+			need_wh = au_lkup_dentry(dentry, btop + 1,
+						 /* AuLkup_IGNORE_PERM */ 0);
+			au_di_swap(tmp, dinfo);
+			au_rw_write_unlock(&tmp->di_rwsem);
+			au_di_free(tmp);
+		}
+	}
+	AuDbg("need_wh %d\n", need_wh);
+	err = need_wh;
+
+out:
+	return err;
+}
+
+/*
+ * simple tests for the del-entry operations.
+ * following the checks in vfs, plus the parent-child relationship.
+ */
+int au_may_del(struct dentry *dentry, aufs_bindex_t bindex,
+	       struct dentry *h_parent, int isdir)
+{
+	int err;
+	umode_t h_mode;
+	struct dentry *h_dentry, *h_latest;
+	struct inode *h_inode;
+
+	h_dentry = au_h_dptr(dentry, bindex);
+	if (d_really_is_positive(dentry)) {
+		err = -ENOENT;
+		if (unlikely(d_is_negative(h_dentry)))
+			goto out;
+		h_inode = d_inode(h_dentry);
+		if (unlikely(!h_inode->i_nlink))
+			goto out;
+
+		h_mode = h_inode->i_mode;
+		if (!isdir) {
+			err = -EISDIR;
+			if (unlikely(S_ISDIR(h_mode)))
+				goto out;
+		} else if (unlikely(!S_ISDIR(h_mode))) {
+			err = -ENOTDIR;
+			goto out;
+		}
+	} else {
+		/* rename(2) case */
+		err = -EIO;
+		if (unlikely(d_is_positive(h_dentry)))
+			goto out;
+	}
+
+	err = -ENOENT;
+	/* expected parent dir is locked */
+	if (unlikely(h_parent != h_dentry->d_parent))
+		goto out;
+	err = 0;
+
+	/*
+	 * rmdir a dir may break the consistency on some filesystem.
+	 * let's try heavy test.
+	 */
+	err = -EACCES;
+	if (unlikely(!au_opt_test(au_mntflags(dentry->d_sb), DIRPERM1)
+		     && au_test_h_perm(d_inode(h_parent),
+				       MAY_EXEC | MAY_WRITE)))
+		goto out;
+
+	h_latest = au_sio_lkup_one(&dentry->d_name, h_parent);
+	err = -EIO;
+	if (IS_ERR(h_latest))
+		goto out;
+	if (h_latest == h_dentry)
+		err = 0;
+	dput(h_latest);
+
+out:
+	return err;
+}
+
+/*
+ * decide the branch where we operate for @dentry. the branch index will be set
+ * @rbcpup. after deciding it, 'pin' it and store the timestamps of the parent
+ * dir for reverting.
+ * when a new whiteout is necessary, create it.
+ */
+static struct dentry*
+lock_hdir_create_wh(struct dentry *dentry, int isdir, aufs_bindex_t *rbcpup,
+		    struct au_dtime *dt, struct au_pin *pin)
+{
+	struct dentry *wh_dentry;
+	struct super_block *sb;
+	struct path h_path;
+	int err, need_wh;
+	unsigned int udba;
+	aufs_bindex_t bcpup;
+
+	need_wh = au_wr_dir_need_wh(dentry, isdir, rbcpup);
+	wh_dentry = ERR_PTR(need_wh);
+	if (unlikely(need_wh < 0))
+		goto out;
+
+	sb = dentry->d_sb;
+	udba = au_opt_udba(sb);
+	bcpup = *rbcpup;
+	err = au_pin(pin, dentry, bcpup, udba,
+		     AuPin_DI_LOCKED | AuPin_MNT_WRITE);
+	wh_dentry = ERR_PTR(err);
+	if (unlikely(err))
+		goto out;
+
+	h_path.dentry = au_pinned_h_parent(pin);
+	if (udba != AuOpt_UDBA_NONE
+	    && au_dbtop(dentry) == bcpup) {
+		err = au_may_del(dentry, bcpup, h_path.dentry, isdir);
+		wh_dentry = ERR_PTR(err);
+		if (unlikely(err))
+			goto out_unpin;
+	}
+
+	h_path.mnt = au_sbr_mnt(sb, bcpup);
+	au_dtime_store(dt, au_pinned_parent(pin), &h_path);
+	wh_dentry = NULL;
+	if (!need_wh)
+		goto out; /* success, no need to create whiteout */
+
+	wh_dentry = au_wh_create(dentry, bcpup, h_path.dentry);
+	if (IS_ERR(wh_dentry))
+		goto out_unpin;
+
+	/* returns with the parent is locked and wh_dentry is dget-ed */
+	goto out; /* success */
+
+out_unpin:
+	au_unpin(pin);
+out:
+	return wh_dentry;
+}
+
+/*
+ * when removing a dir, rename it to a unique temporary whiteout-ed name first
+ * in order to be revertible and save time for removing many child whiteouts
+ * under the dir.
+ * returns 1 when there are too many child whiteout and caller should remove
+ * them asynchronously. returns 0 when the number of children is enough small to
+ * remove now or the branch fs is a remote fs.
+ * otherwise return an error.
+ */
+static int renwh_and_rmdir(struct dentry *dentry, aufs_bindex_t bindex,
+			   struct au_nhash *whlist, struct inode *dir)
+{
+	int rmdir_later, err, dirwh;
+	struct dentry *h_dentry;
+	struct super_block *sb;
+	struct inode *inode;
+
+	sb = dentry->d_sb;
+	SiMustAnyLock(sb);
+	h_dentry = au_h_dptr(dentry, bindex);
+	err = au_whtmp_ren(h_dentry, au_sbr(sb, bindex));
+	if (unlikely(err))
+		goto out;
+
+	/* stop monitoring */
+	inode = d_inode(dentry);
+	au_hn_free(au_hi(inode, bindex));
+
+	if (!au_test_fs_remote(h_dentry->d_sb)) {
+		dirwh = au_sbi(sb)->si_dirwh;
+		rmdir_later = (dirwh <= 1);
+		if (!rmdir_later)
+			rmdir_later = au_nhash_test_longer_wh(whlist, bindex,
+							      dirwh);
+		if (rmdir_later)
+			return rmdir_later;
+	}
+
+	err = au_whtmp_rmdir(dir, bindex, h_dentry, whlist);
+	if (unlikely(err)) {
+		AuIOErr("rmdir %pd, b%d failed, %d. ignored\n",
+			h_dentry, bindex, err);
+		err = 0;
+	}
+
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+/*
+ * final procedure for deleting a entry.
+ * maintain dentry and iattr.
+ */
+static void epilog(struct inode *dir, struct dentry *dentry,
+		   aufs_bindex_t bindex)
+{
+	struct inode *inode;
+
+	inode = d_inode(dentry);
+	d_drop(dentry);
+	inode->i_ctime = dir->i_ctime;
+
+	au_dir_ts(dir, bindex);
+	inode_inc_iversion(dir);
+}
+
+/*
+ * when an error happened, remove the created whiteout and revert everything.
+ */
+static int do_revert(int err, struct inode *dir, aufs_bindex_t bindex,
+		     aufs_bindex_t bwh, struct dentry *wh_dentry,
+		     struct dentry *dentry, struct au_dtime *dt)
+{
+	int rerr;
+	struct path h_path = {
+		.dentry	= wh_dentry,
+		.mnt	= au_sbr_mnt(dir->i_sb, bindex)
+	};
+
+	rerr = au_wh_unlink_dentry(au_h_iptr(dir, bindex), &h_path, dentry);
+	if (!rerr) {
+		au_set_dbwh(dentry, bwh);
+		au_dtime_revert(dt);
+		return 0;
+	}
+
+	AuIOErr("%pd reverting whiteout failed(%d, %d)\n", dentry, err, rerr);
+	return -EIO;
+}
+
+/* ---------------------------------------------------------------------- */
+
+int aufs_unlink(struct inode *dir, struct dentry *dentry)
+{
+	int err;
+	aufs_bindex_t bwh, bindex, btop;
+	struct inode *inode, *h_dir, *delegated;
+	struct dentry *parent, *wh_dentry;
+	/* to reduce stack size */
+	struct {
+		struct au_dtime dt;
+		struct au_pin pin;
+		struct path h_path;
+	} *a;
+
+	IMustLock(dir);
+
+	err = -ENOMEM;
+	a = kmalloc(sizeof(*a), GFP_NOFS);
+	if (unlikely(!a))
+		goto out;
+
+	err = aufs_read_lock(dentry, AuLock_DW | AuLock_GEN);
+	if (unlikely(err))
+		goto out_free;
+	err = au_d_hashed_positive(dentry);
+	if (unlikely(err))
+		goto out_unlock;
+	inode = d_inode(dentry);
+	IMustLock(inode);
+	err = -EISDIR;
+	if (unlikely(d_is_dir(dentry)))
+		goto out_unlock; /* possible? */
+
+	btop = au_dbtop(dentry);
+	bwh = au_dbwh(dentry);
+	bindex = -1;
+	parent = dentry->d_parent; /* dir inode is locked */
+	di_write_lock_parent(parent);
+	wh_dentry = lock_hdir_create_wh(dentry, /*isdir*/0, &bindex, &a->dt,
+					&a->pin);
+	err = PTR_ERR(wh_dentry);
+	if (IS_ERR(wh_dentry))
+		goto out_parent;
+
+	a->h_path.mnt = au_sbr_mnt(dentry->d_sb, btop);
+	a->h_path.dentry = au_h_dptr(dentry, btop);
+	dget(a->h_path.dentry);
+	if (bindex == btop) {
+		h_dir = au_pinned_h_dir(&a->pin);
+		delegated = NULL;
+		err = vfsub_unlink(h_dir, &a->h_path, &delegated, /*force*/0);
+		if (unlikely(err == -EWOULDBLOCK)) {
+			pr_warn("cannot retry for NFSv4 delegation"
+				" for an internal unlink\n");
+			iput(delegated);
+		}
+	} else {
+		/* dir inode is locked */
+		h_dir = d_inode(wh_dentry->d_parent);
+		IMustLock(h_dir);
+		err = 0;
+	}
+
+	if (!err) {
+		vfsub_drop_nlink(inode);
+		epilog(dir, dentry, bindex);
+
+		/* update target timestamps */
+		if (bindex == btop) {
+			vfsub_update_h_iattr(&a->h_path, /*did*/NULL);
+			/*ignore*/
+			inode->i_ctime = d_inode(a->h_path.dentry)->i_ctime;
+		} else
+			/* todo: this timestamp may be reverted later */
+			inode->i_ctime = h_dir->i_ctime;
+		goto out_unpin; /* success */
+	}
+
+	/* revert */
+	if (wh_dentry) {
+		int rerr;
+
+		rerr = do_revert(err, dir, bindex, bwh, wh_dentry, dentry,
+				 &a->dt);
+		if (rerr)
+			err = rerr;
+	}
+
+out_unpin:
+	au_unpin(&a->pin);
+	dput(wh_dentry);
+	dput(a->h_path.dentry);
+out_parent:
+	di_write_unlock(parent);
+out_unlock:
+	aufs_read_unlock(dentry, AuLock_DW);
+out_free:
+	au_kfree_rcu(a);
+out:
+	return err;
+}
+
+int aufs_rmdir(struct inode *dir, struct dentry *dentry)
+{
+	int err, rmdir_later;
+	aufs_bindex_t bwh, bindex, btop;
+	struct inode *inode;
+	struct dentry *parent, *wh_dentry, *h_dentry;
+	struct au_whtmp_rmdir *args;
+	/* to reduce stack size */
+	struct {
+		struct au_dtime dt;
+		struct au_pin pin;
+	} *a;
+
+	IMustLock(dir);
+
+	err = -ENOMEM;
+	a = kmalloc(sizeof(*a), GFP_NOFS);
+	if (unlikely(!a))
+		goto out;
+
+	err = aufs_read_lock(dentry, AuLock_DW | AuLock_FLUSH | AuLock_GEN);
+	if (unlikely(err))
+		goto out_free;
+	err = au_alive_dir(dentry);
+	if (unlikely(err))
+		goto out_unlock;
+	inode = d_inode(dentry);
+	IMustLock(inode);
+	err = -ENOTDIR;
+	if (unlikely(!d_is_dir(dentry)))
+		goto out_unlock; /* possible? */
+
+	err = -ENOMEM;
+	args = au_whtmp_rmdir_alloc(dir->i_sb, GFP_NOFS);
+	if (unlikely(!args))
+		goto out_unlock;
+
+	parent = dentry->d_parent; /* dir inode is locked */
+	di_write_lock_parent(parent);
+	err = au_test_empty(dentry, &args->whlist);
+	if (unlikely(err))
+		goto out_parent;
+
+	btop = au_dbtop(dentry);
+	bwh = au_dbwh(dentry);
+	bindex = -1;
+	wh_dentry = lock_hdir_create_wh(dentry, /*isdir*/1, &bindex, &a->dt,
+					&a->pin);
+	err = PTR_ERR(wh_dentry);
+	if (IS_ERR(wh_dentry))
+		goto out_parent;
+
+	h_dentry = au_h_dptr(dentry, btop);
+	dget(h_dentry);
+	rmdir_later = 0;
+	if (bindex == btop) {
+		err = renwh_and_rmdir(dentry, btop, &args->whlist, dir);
+		if (err > 0) {
+			rmdir_later = err;
+			err = 0;
+		}
+	} else {
+		/* stop monitoring */
+		au_hn_free(au_hi(inode, btop));
+
+		/* dir inode is locked */
+		IMustLock(d_inode(wh_dentry->d_parent));
+		err = 0;
+	}
+
+	if (!err) {
+		vfsub_dead_dir(inode);
+		au_set_dbdiropq(dentry, -1);
+		epilog(dir, dentry, bindex);
+
+		if (rmdir_later) {
+			au_whtmp_kick_rmdir(dir, btop, h_dentry, args);
+			args = NULL;
+		}
+
+		goto out_unpin; /* success */
+	}
+
+	/* revert */
+	AuLabel(revert);
+	if (wh_dentry) {
+		int rerr;
+
+		rerr = do_revert(err, dir, bindex, bwh, wh_dentry, dentry,
+				 &a->dt);
+		if (rerr)
+			err = rerr;
+	}
+
+out_unpin:
+	au_unpin(&a->pin);
+	dput(wh_dentry);
+	dput(h_dentry);
+out_parent:
+	di_write_unlock(parent);
+	if (args)
+		au_whtmp_rmdir_free(args);
+out_unlock:
+	aufs_read_unlock(dentry, AuLock_DW);
+out_free:
+	au_kfree_rcu(a);
+out:
+	AuTraceErr(err);
+	return err;
+}
diff -Naur --no-dereference a/fs/aufs/i_op_ren.c b/fs/aufs/i_op_ren.c
--- a/fs/aufs/i_op_ren.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/i_op_ren.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,1250 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * inode operation (rename entry)
+ * todo: this is crazy monster
+ */
+
+#include <linux/iversion.h>
+#include "aufs.h"
+
+enum { AuSRC, AuDST, AuSrcDst };
+enum { AuPARENT, AuCHILD, AuParentChild };
+
+#define AuRen_ISDIR_SRC		1
+#define AuRen_ISDIR_DST		(1 << 1)
+#define AuRen_ISSAMEDIR		(1 << 2)
+#define AuRen_WHSRC		(1 << 3)
+#define AuRen_WHDST		(1 << 4)
+#define AuRen_MNT_WRITE		(1 << 5)
+#define AuRen_DT_DSTDIR		(1 << 6)
+#define AuRen_DIROPQ_SRC	(1 << 7)
+#define AuRen_DIROPQ_DST	(1 << 8)
+#define AuRen_DIRREN		(1 << 9)
+#define AuRen_DROPPED_SRC	(1 << 10)
+#define AuRen_DROPPED_DST	(1 << 11)
+#define au_ftest_ren(flags, name)	((flags) & AuRen_##name)
+#define au_fset_ren(flags, name) \
+	do { (flags) |= AuRen_##name; } while (0)
+#define au_fclr_ren(flags, name) \
+	do { (flags) &= ~AuRen_##name; } while (0)
+
+#ifndef CONFIG_AUFS_DIRREN
+#undef AuRen_DIRREN
+#define AuRen_DIRREN		0
+#endif
+
+struct au_ren_args {
+	struct {
+		struct dentry *dentry, *h_dentry, *parent, *h_parent,
+			*wh_dentry;
+		struct inode *dir, *inode;
+		struct au_hinode *hdir, *hinode;
+		struct au_dtime dt[AuParentChild];
+		aufs_bindex_t btop, bdiropq;
+	} sd[AuSrcDst];
+
+#define src_dentry	sd[AuSRC].dentry
+#define src_dir		sd[AuSRC].dir
+#define src_inode	sd[AuSRC].inode
+#define src_h_dentry	sd[AuSRC].h_dentry
+#define src_parent	sd[AuSRC].parent
+#define src_h_parent	sd[AuSRC].h_parent
+#define src_wh_dentry	sd[AuSRC].wh_dentry
+#define src_hdir	sd[AuSRC].hdir
+#define src_hinode	sd[AuSRC].hinode
+#define src_h_dir	sd[AuSRC].hdir->hi_inode
+#define src_dt		sd[AuSRC].dt
+#define src_btop	sd[AuSRC].btop
+#define src_bdiropq	sd[AuSRC].bdiropq
+
+#define dst_dentry	sd[AuDST].dentry
+#define dst_dir		sd[AuDST].dir
+#define dst_inode	sd[AuDST].inode
+#define dst_h_dentry	sd[AuDST].h_dentry
+#define dst_parent	sd[AuDST].parent
+#define dst_h_parent	sd[AuDST].h_parent
+#define dst_wh_dentry	sd[AuDST].wh_dentry
+#define dst_hdir	sd[AuDST].hdir
+#define dst_hinode	sd[AuDST].hinode
+#define dst_h_dir	sd[AuDST].hdir->hi_inode
+#define dst_dt		sd[AuDST].dt
+#define dst_btop	sd[AuDST].btop
+#define dst_bdiropq	sd[AuDST].bdiropq
+
+	struct dentry *h_trap;
+	struct au_branch *br;
+	struct path h_path;
+	struct au_nhash whlist;
+	aufs_bindex_t btgt, src_bwh;
+
+	struct {
+		unsigned short auren_flags;
+		unsigned char flags;	/* syscall parameter */
+		unsigned char exchange;
+	} __packed;
+
+	struct au_whtmp_rmdir *thargs;
+	struct dentry *h_dst;
+	struct au_hinode *h_root;
+};
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * functions for reverting.
+ * when an error happened in a single rename systemcall, we should revert
+ * everything as if nothing happened.
+ * we don't need to revert the copied-up/down the parent dir since they are
+ * harmless.
+ */
+
+#define RevertFailure(fmt, ...) do { \
+	AuIOErr("revert failure: " fmt " (%d, %d)\n", \
+		##__VA_ARGS__, err, rerr); \
+	err = -EIO; \
+} while (0)
+
+static void au_ren_do_rev_diropq(int err, struct au_ren_args *a, int idx)
+{
+	int rerr;
+	struct dentry *d;
+#define src_or_dst(member) a->sd[idx].member
+
+	d = src_or_dst(dentry); /* {src,dst}_dentry */
+	au_hn_inode_lock_nested(src_or_dst(hinode), AuLsc_I_CHILD);
+	rerr = au_diropq_remove(d, a->btgt);
+	au_hn_inode_unlock(src_or_dst(hinode));
+	au_set_dbdiropq(d, src_or_dst(bdiropq));
+	if (rerr)
+		RevertFailure("remove diropq %pd", d);
+
+#undef src_or_dst_
+}
+
+static void au_ren_rev_diropq(int err, struct au_ren_args *a)
+{
+	if (au_ftest_ren(a->auren_flags, DIROPQ_SRC))
+		au_ren_do_rev_diropq(err, a, AuSRC);
+	if (au_ftest_ren(a->auren_flags, DIROPQ_DST))
+		au_ren_do_rev_diropq(err, a, AuDST);
+}
+
+static void au_ren_rev_rename(int err, struct au_ren_args *a)
+{
+	int rerr;
+	struct inode *delegated;
+
+	a->h_path.dentry = vfsub_lkup_one(&a->src_dentry->d_name,
+					  a->src_h_parent);
+	rerr = PTR_ERR(a->h_path.dentry);
+	if (IS_ERR(a->h_path.dentry)) {
+		RevertFailure("lkup one %pd", a->src_dentry);
+		return;
+	}
+
+	delegated = NULL;
+	rerr = vfsub_rename(a->dst_h_dir,
+			    au_h_dptr(a->src_dentry, a->btgt),
+			    a->src_h_dir, &a->h_path, &delegated, a->flags);
+	if (unlikely(rerr == -EWOULDBLOCK)) {
+		pr_warn("cannot retry for NFSv4 delegation"
+			" for an internal rename\n");
+		iput(delegated);
+	}
+	d_drop(a->h_path.dentry);
+	dput(a->h_path.dentry);
+	/* au_set_h_dptr(a->src_dentry, a->btgt, NULL); */
+	if (rerr)
+		RevertFailure("rename %pd", a->src_dentry);
+}
+
+static void au_ren_rev_whtmp(int err, struct au_ren_args *a)
+{
+	int rerr;
+	struct inode *delegated;
+
+	a->h_path.dentry = vfsub_lkup_one(&a->dst_dentry->d_name,
+					  a->dst_h_parent);
+	rerr = PTR_ERR(a->h_path.dentry);
+	if (IS_ERR(a->h_path.dentry)) {
+		RevertFailure("lkup one %pd", a->dst_dentry);
+		return;
+	}
+	if (d_is_positive(a->h_path.dentry)) {
+		d_drop(a->h_path.dentry);
+		dput(a->h_path.dentry);
+		return;
+	}
+
+	delegated = NULL;
+	rerr = vfsub_rename(a->dst_h_dir, a->h_dst, a->dst_h_dir, &a->h_path,
+			    &delegated, a->flags);
+	if (unlikely(rerr == -EWOULDBLOCK)) {
+		pr_warn("cannot retry for NFSv4 delegation"
+			" for an internal rename\n");
+		iput(delegated);
+	}
+	d_drop(a->h_path.dentry);
+	dput(a->h_path.dentry);
+	if (!rerr)
+		au_set_h_dptr(a->dst_dentry, a->btgt, dget(a->h_dst));
+	else
+		RevertFailure("rename %pd", a->h_dst);
+}
+
+static void au_ren_rev_whsrc(int err, struct au_ren_args *a)
+{
+	int rerr;
+
+	a->h_path.dentry = a->src_wh_dentry;
+	rerr = au_wh_unlink_dentry(a->src_h_dir, &a->h_path, a->src_dentry);
+	au_set_dbwh(a->src_dentry, a->src_bwh);
+	if (rerr)
+		RevertFailure("unlink %pd", a->src_wh_dentry);
+}
+#undef RevertFailure
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * when we have to copyup the renaming entry, do it with the rename-target name
+ * in order to minimize the cost (the later actual rename is unnecessary).
+ * otherwise rename it on the target branch.
+ */
+static int au_ren_or_cpup(struct au_ren_args *a)
+{
+	int err;
+	struct dentry *d;
+	struct inode *delegated;
+
+	d = a->src_dentry;
+	if (au_dbtop(d) == a->btgt) {
+		a->h_path.dentry = a->dst_h_dentry;
+		AuDebugOn(au_dbtop(d) != a->btgt);
+		delegated = NULL;
+		err = vfsub_rename(a->src_h_dir, au_h_dptr(d, a->btgt),
+				   a->dst_h_dir, &a->h_path, &delegated,
+				   a->flags);
+		if (unlikely(err == -EWOULDBLOCK)) {
+			pr_warn("cannot retry for NFSv4 delegation"
+				" for an internal rename\n");
+			iput(delegated);
+		}
+	} else
+		BUG();
+
+	if (!err && a->h_dst)
+		/* it will be set to dinfo later */
+		dget(a->h_dst);
+
+	return err;
+}
+
+/* cf. aufs_rmdir() */
+static int au_ren_del_whtmp(struct au_ren_args *a)
+{
+	int err;
+	struct inode *dir;
+
+	dir = a->dst_dir;
+	SiMustAnyLock(dir->i_sb);
+	if (!au_nhash_test_longer_wh(&a->whlist, a->btgt,
+				     au_sbi(dir->i_sb)->si_dirwh)
+	    || au_test_fs_remote(a->h_dst->d_sb)) {
+		err = au_whtmp_rmdir(dir, a->btgt, a->h_dst, &a->whlist);
+		if (unlikely(err))
+			pr_warn("failed removing whtmp dir %pd (%d), "
+				"ignored.\n", a->h_dst, err);
+	} else {
+		au_nhash_wh_free(&a->thargs->whlist);
+		a->thargs->whlist = a->whlist;
+		a->whlist.nh_num = 0;
+		au_whtmp_kick_rmdir(dir, a->btgt, a->h_dst, a->thargs);
+		dput(a->h_dst);
+		a->thargs = NULL;
+	}
+
+	return 0;
+}
+
+/* make it 'opaque' dir. */
+static int au_ren_do_diropq(struct au_ren_args *a, int idx)
+{
+	int err;
+	struct dentry *d, *diropq;
+#define src_or_dst(member) a->sd[idx].member
+
+	err = 0;
+	d = src_or_dst(dentry); /* {src,dst}_dentry */
+	src_or_dst(bdiropq) = au_dbdiropq(d);
+	src_or_dst(hinode) = au_hi(src_or_dst(inode), a->btgt);
+	au_hn_inode_lock_nested(src_or_dst(hinode), AuLsc_I_CHILD);
+	diropq = au_diropq_create(d, a->btgt);
+	au_hn_inode_unlock(src_or_dst(hinode));
+	if (IS_ERR(diropq))
+		err = PTR_ERR(diropq);
+	else
+		dput(diropq);
+
+#undef src_or_dst_
+	return err;
+}
+
+static int au_ren_diropq(struct au_ren_args *a)
+{
+	int err;
+	unsigned char always;
+	struct dentry *d;
+
+	err = 0;
+	d = a->dst_dentry; /* already renamed on the branch */
+	always = !!au_opt_test(au_mntflags(d->d_sb), ALWAYS_DIROPQ);
+	if (au_ftest_ren(a->auren_flags, ISDIR_SRC)
+	    && !au_ftest_ren(a->auren_flags, DIRREN)
+	    && a->btgt != au_dbdiropq(a->src_dentry)
+	    && (a->dst_wh_dentry
+		|| a->btgt <= au_dbdiropq(d)
+		/* hide the lower to keep xino */
+		/* the lowers may not be a dir, but we hide them anyway */
+		|| a->btgt < au_dbbot(d)
+		|| always)) {
+		AuDbg("here\n");
+		err = au_ren_do_diropq(a, AuSRC);
+		if (unlikely(err))
+			goto out;
+		au_fset_ren(a->auren_flags, DIROPQ_SRC);
+	}
+	if (!a->exchange)
+		goto out; /* success */
+
+	d = a->src_dentry; /* already renamed on the branch */
+	if (au_ftest_ren(a->auren_flags, ISDIR_DST)
+	    && a->btgt != au_dbdiropq(a->dst_dentry)
+	    && (a->btgt < au_dbdiropq(d)
+		|| a->btgt < au_dbbot(d)
+		|| always)) {
+		AuDbgDentry(a->src_dentry);
+		AuDbgDentry(a->dst_dentry);
+		err = au_ren_do_diropq(a, AuDST);
+		if (unlikely(err))
+			goto out_rev_src;
+		au_fset_ren(a->auren_flags, DIROPQ_DST);
+	}
+	goto out; /* success */
+
+out_rev_src:
+	AuDbg("err %d, reverting src\n", err);
+	au_ren_rev_diropq(err, a);
+out:
+	return err;
+}
+
+static int do_rename(struct au_ren_args *a)
+{
+	int err;
+	struct dentry *d, *h_d;
+
+	if (!a->exchange) {
+		/* prepare workqueue args for asynchronous rmdir */
+		h_d = a->dst_h_dentry;
+		if (au_ftest_ren(a->auren_flags, ISDIR_DST)
+		    /* && !au_ftest_ren(a->auren_flags, DIRREN) */
+		    && d_is_positive(h_d)) {
+			err = -ENOMEM;
+			a->thargs = au_whtmp_rmdir_alloc(a->src_dentry->d_sb,
+							 GFP_NOFS);
+			if (unlikely(!a->thargs))
+				goto out;
+			a->h_dst = dget(h_d);
+		}
+
+		/* create whiteout for src_dentry */
+		if (au_ftest_ren(a->auren_flags, WHSRC)) {
+			a->src_bwh = au_dbwh(a->src_dentry);
+			AuDebugOn(a->src_bwh >= 0);
+			a->src_wh_dentry = au_wh_create(a->src_dentry, a->btgt,
+							a->src_h_parent);
+			err = PTR_ERR(a->src_wh_dentry);
+			if (IS_ERR(a->src_wh_dentry))
+				goto out_thargs;
+		}
+
+		/* lookup whiteout for dentry */
+		if (au_ftest_ren(a->auren_flags, WHDST)) {
+			h_d = au_wh_lkup(a->dst_h_parent,
+					 &a->dst_dentry->d_name, a->br);
+			err = PTR_ERR(h_d);
+			if (IS_ERR(h_d))
+				goto out_whsrc;
+			if (d_is_negative(h_d))
+				dput(h_d);
+			else
+				a->dst_wh_dentry = h_d;
+		}
+
+		/* rename dentry to tmpwh */
+		if (a->thargs) {
+			err = au_whtmp_ren(a->dst_h_dentry, a->br);
+			if (unlikely(err))
+				goto out_whdst;
+
+			d = a->dst_dentry;
+			au_set_h_dptr(d, a->btgt, NULL);
+			err = au_lkup_neg(d, a->btgt, /*wh*/0);
+			if (unlikely(err))
+				goto out_whtmp;
+			a->dst_h_dentry = au_h_dptr(d, a->btgt);
+		}
+	}
+
+	BUG_ON(d_is_positive(a->dst_h_dentry) && a->src_btop != a->btgt);
+#if 0 /* debugging */
+	BUG_ON(!au_ftest_ren(a->auren_flags, DIRREN)
+	       && d_is_positive(a->dst_h_dentry)
+	       && a->src_btop != a->btgt);
+#endif
+
+	/* rename by vfs_rename or cpup */
+	err = au_ren_or_cpup(a);
+	if (unlikely(err))
+		/* leave the copied-up one */
+		goto out_whtmp;
+
+	/* make dir opaque */
+	err = au_ren_diropq(a);
+	if (unlikely(err))
+		goto out_rename;
+
+	/* update target timestamps */
+	if (a->exchange) {
+		AuDebugOn(au_dbtop(a->dst_dentry) != a->btgt);
+		a->h_path.dentry = au_h_dptr(a->dst_dentry, a->btgt);
+		vfsub_update_h_iattr(&a->h_path, /*did*/NULL); /*ignore*/
+		a->dst_inode->i_ctime = d_inode(a->h_path.dentry)->i_ctime;
+	}
+	AuDebugOn(au_dbtop(a->src_dentry) != a->btgt);
+	a->h_path.dentry = au_h_dptr(a->src_dentry, a->btgt);
+	vfsub_update_h_iattr(&a->h_path, /*did*/NULL); /*ignore*/
+	a->src_inode->i_ctime = d_inode(a->h_path.dentry)->i_ctime;
+
+	if (!a->exchange) {
+		/* remove whiteout for dentry */
+		if (a->dst_wh_dentry) {
+			a->h_path.dentry = a->dst_wh_dentry;
+			err = au_wh_unlink_dentry(a->dst_h_dir, &a->h_path,
+						  a->dst_dentry);
+			if (unlikely(err))
+				goto out_diropq;
+		}
+
+		/* remove whtmp */
+		if (a->thargs)
+			au_ren_del_whtmp(a); /* ignore this error */
+
+		au_fhsm_wrote(a->src_dentry->d_sb, a->btgt, /*force*/0);
+	}
+	err = 0;
+	goto out_success;
+
+out_diropq:
+	au_ren_rev_diropq(err, a);
+out_rename:
+	au_ren_rev_rename(err, a);
+	dput(a->h_dst);
+out_whtmp:
+	if (a->thargs)
+		au_ren_rev_whtmp(err, a);
+out_whdst:
+	dput(a->dst_wh_dentry);
+	a->dst_wh_dentry = NULL;
+out_whsrc:
+	if (a->src_wh_dentry)
+		au_ren_rev_whsrc(err, a);
+out_success:
+	dput(a->src_wh_dentry);
+	dput(a->dst_wh_dentry);
+out_thargs:
+	if (a->thargs) {
+		dput(a->h_dst);
+		au_whtmp_rmdir_free(a->thargs);
+		a->thargs = NULL;
+	}
+out:
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * test if @dentry dir can be rename destination or not.
+ * success means, it is a logically empty dir.
+ */
+static int may_rename_dstdir(struct dentry *dentry, struct au_nhash *whlist)
+{
+	return au_test_empty(dentry, whlist);
+}
+
+/*
+ * test if @a->src_dentry dir can be rename source or not.
+ * if it can, return 0.
+ * success means,
+ * - it is a logically empty dir.
+ * - or, it exists on writable branch and has no children including whiteouts
+ *   on the lower branch unless DIRREN is on.
+ */
+static int may_rename_srcdir(struct au_ren_args *a)
+{
+	int err;
+	unsigned int rdhash;
+	aufs_bindex_t btop, btgt;
+	struct dentry *dentry;
+	struct super_block *sb;
+	struct au_sbinfo *sbinfo;
+
+	dentry = a->src_dentry;
+	sb = dentry->d_sb;
+	sbinfo = au_sbi(sb);
+	if (au_opt_test(sbinfo->si_mntflags, DIRREN))
+		au_fset_ren(a->auren_flags, DIRREN);
+
+	btgt = a->btgt;
+	btop = au_dbtop(dentry);
+	if (btop != btgt) {
+		struct au_nhash whlist;
+
+		SiMustAnyLock(sb);
+		rdhash = sbinfo->si_rdhash;
+		if (!rdhash)
+			rdhash = au_rdhash_est(au_dir_size(/*file*/NULL,
+							   dentry));
+		err = au_nhash_alloc(&whlist, rdhash, GFP_NOFS);
+		if (unlikely(err))
+			goto out;
+		err = au_test_empty(dentry, &whlist);
+		au_nhash_wh_free(&whlist);
+		goto out;
+	}
+
+	if (btop == au_dbtaildir(dentry))
+		return 0; /* success */
+
+	err = au_test_empty_lower(dentry);
+
+out:
+	if (err == -ENOTEMPTY) {
+		if (au_ftest_ren(a->auren_flags, DIRREN)) {
+			err = 0;
+		} else {
+			AuWarn1("renaming dir who has child(ren) on multiple "
+				"branches, is not supported\n");
+			err = -EXDEV;
+		}
+	}
+	return err;
+}
+
+/* side effect: sets whlist and h_dentry */
+static int au_ren_may_dir(struct au_ren_args *a)
+{
+	int err;
+	unsigned int rdhash;
+	struct dentry *d;
+
+	d = a->dst_dentry;
+	SiMustAnyLock(d->d_sb);
+
+	err = 0;
+	if (au_ftest_ren(a->auren_flags, ISDIR_DST) && a->dst_inode) {
+		rdhash = au_sbi(d->d_sb)->si_rdhash;
+		if (!rdhash)
+			rdhash = au_rdhash_est(au_dir_size(/*file*/NULL, d));
+		err = au_nhash_alloc(&a->whlist, rdhash, GFP_NOFS);
+		if (unlikely(err))
+			goto out;
+
+		if (!a->exchange) {
+			au_set_dbtop(d, a->dst_btop);
+			err = may_rename_dstdir(d, &a->whlist);
+			au_set_dbtop(d, a->btgt);
+		} else
+			err = may_rename_srcdir(a);
+	}
+	a->dst_h_dentry = au_h_dptr(d, au_dbtop(d));
+	if (unlikely(err))
+		goto out;
+
+	d = a->src_dentry;
+	a->src_h_dentry = au_h_dptr(d, au_dbtop(d));
+	if (au_ftest_ren(a->auren_flags, ISDIR_SRC)) {
+		err = may_rename_srcdir(a);
+		if (unlikely(err)) {
+			au_nhash_wh_free(&a->whlist);
+			a->whlist.nh_num = 0;
+		}
+	}
+out:
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * simple tests for rename.
+ * following the checks in vfs, plus the parent-child relationship.
+ */
+static int au_may_ren(struct au_ren_args *a)
+{
+	int err, isdir;
+	struct inode *h_inode;
+
+	if (a->src_btop == a->btgt) {
+		err = au_may_del(a->src_dentry, a->btgt, a->src_h_parent,
+				 au_ftest_ren(a->auren_flags, ISDIR_SRC));
+		if (unlikely(err))
+			goto out;
+		err = -EINVAL;
+		if (unlikely(a->src_h_dentry == a->h_trap))
+			goto out;
+	}
+
+	err = 0;
+	if (a->dst_btop != a->btgt)
+		goto out;
+
+	err = -ENOTEMPTY;
+	if (unlikely(a->dst_h_dentry == a->h_trap))
+		goto out;
+
+	err = -EIO;
+	isdir = !!au_ftest_ren(a->auren_flags, ISDIR_DST);
+	if (d_really_is_negative(a->dst_dentry)) {
+		if (d_is_negative(a->dst_h_dentry))
+			err = au_may_add(a->dst_dentry, a->btgt,
+					 a->dst_h_parent, isdir);
+	} else {
+		if (unlikely(d_is_negative(a->dst_h_dentry)))
+			goto out;
+		h_inode = d_inode(a->dst_h_dentry);
+		if (h_inode->i_nlink)
+			err = au_may_del(a->dst_dentry, a->btgt,
+					 a->dst_h_parent, isdir);
+	}
+
+out:
+	if (unlikely(err == -ENOENT || err == -EEXIST))
+		err = -EIO;
+	AuTraceErr(err);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * locking order
+ * (VFS)
+ * - src_dir and dir by lock_rename()
+ * - inode if exists
+ * (aufs)
+ * - lock all
+ *   + src_dentry and dentry by aufs_read_and_write_lock2() which calls,
+ *     + si_read_lock
+ *     + di_write_lock2_child()
+ *       + di_write_lock_child()
+ *	   + ii_write_lock_child()
+ *       + di_write_lock_child2()
+ *	   + ii_write_lock_child2()
+ *     + src_parent and parent
+ *       + di_write_lock_parent()
+ *	   + ii_write_lock_parent()
+ *       + di_write_lock_parent2()
+ *	   + ii_write_lock_parent2()
+ *   + lower src_dir and dir by vfsub_lock_rename()
+ *   + verify the every relationships between child and parent. if any
+ *     of them failed, unlock all and return -EBUSY.
+ */
+static void au_ren_unlock(struct au_ren_args *a)
+{
+	vfsub_unlock_rename(a->src_h_parent, a->src_hdir,
+			    a->dst_h_parent, a->dst_hdir);
+	if (au_ftest_ren(a->auren_flags, DIRREN)
+	    && a->h_root)
+		au_hn_inode_unlock(a->h_root);
+	if (au_ftest_ren(a->auren_flags, MNT_WRITE))
+		vfsub_mnt_drop_write(au_br_mnt(a->br));
+}
+
+static int au_ren_lock(struct au_ren_args *a)
+{
+	int err;
+	unsigned int udba;
+
+	err = 0;
+	a->src_h_parent = au_h_dptr(a->src_parent, a->btgt);
+	a->src_hdir = au_hi(a->src_dir, a->btgt);
+	a->dst_h_parent = au_h_dptr(a->dst_parent, a->btgt);
+	a->dst_hdir = au_hi(a->dst_dir, a->btgt);
+
+	err = vfsub_mnt_want_write(au_br_mnt(a->br));
+	if (unlikely(err))
+		goto out;
+	au_fset_ren(a->auren_flags, MNT_WRITE);
+	if (au_ftest_ren(a->auren_flags, DIRREN)) {
+		struct dentry *root;
+		struct inode *dir;
+
+		/*
+		 * sbinfo is already locked, so this ii_read_lock is
+		 * unnecessary. but our debugging feature checks it.
+		 */
+		root = a->src_inode->i_sb->s_root;
+		if (root != a->src_parent && root != a->dst_parent) {
+			dir = d_inode(root);
+			ii_read_lock_parent3(dir);
+			a->h_root = au_hi(dir, a->btgt);
+			ii_read_unlock(dir);
+			au_hn_inode_lock_nested(a->h_root, AuLsc_I_PARENT3);
+		}
+	}
+	a->h_trap = vfsub_lock_rename(a->src_h_parent, a->src_hdir,
+				      a->dst_h_parent, a->dst_hdir);
+	udba = au_opt_udba(a->src_dentry->d_sb);
+	if (unlikely(a->src_hdir->hi_inode != d_inode(a->src_h_parent)
+		     || a->dst_hdir->hi_inode != d_inode(a->dst_h_parent)))
+		err = au_busy_or_stale();
+	if (!err && au_dbtop(a->src_dentry) == a->btgt)
+		err = au_h_verify(a->src_h_dentry, udba,
+				  d_inode(a->src_h_parent), a->src_h_parent,
+				  a->br);
+	if (!err && au_dbtop(a->dst_dentry) == a->btgt)
+		err = au_h_verify(a->dst_h_dentry, udba,
+				  d_inode(a->dst_h_parent), a->dst_h_parent,
+				  a->br);
+	if (!err)
+		goto out; /* success */
+
+	err = au_busy_or_stale();
+	au_ren_unlock(a);
+
+out:
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static void au_ren_refresh_dir(struct au_ren_args *a)
+{
+	struct inode *dir;
+
+	dir = a->dst_dir;
+	inode_inc_iversion(dir);
+	if (au_ftest_ren(a->auren_flags, ISDIR_SRC)) {
+		/* is this updating defined in POSIX? */
+		au_cpup_attr_timesizes(a->src_inode);
+		au_cpup_attr_nlink(dir, /*force*/1);
+	}
+	au_dir_ts(dir, a->btgt);
+
+	if (a->exchange) {
+		dir = a->src_dir;
+		inode_inc_iversion(dir);
+		if (au_ftest_ren(a->auren_flags, ISDIR_DST)) {
+			/* is this updating defined in POSIX? */
+			au_cpup_attr_timesizes(a->dst_inode);
+			au_cpup_attr_nlink(dir, /*force*/1);
+		}
+		au_dir_ts(dir, a->btgt);
+	}
+
+	if (au_ftest_ren(a->auren_flags, ISSAMEDIR))
+		return;
+
+	dir = a->src_dir;
+	inode_inc_iversion(dir);
+	if (au_ftest_ren(a->auren_flags, ISDIR_SRC))
+		au_cpup_attr_nlink(dir, /*force*/1);
+	au_dir_ts(dir, a->btgt);
+}
+
+static void au_ren_refresh(struct au_ren_args *a)
+{
+	aufs_bindex_t bbot, bindex;
+	struct dentry *d, *h_d;
+	struct inode *i, *h_i;
+	struct super_block *sb;
+
+	d = a->dst_dentry;
+	d_drop(d);
+	if (a->h_dst)
+		/* already dget-ed by au_ren_or_cpup() */
+		au_set_h_dptr(d, a->btgt, a->h_dst);
+
+	i = a->dst_inode;
+	if (i) {
+		if (!a->exchange) {
+			if (!au_ftest_ren(a->auren_flags, ISDIR_DST))
+				vfsub_drop_nlink(i);
+			else {
+				vfsub_dead_dir(i);
+				au_cpup_attr_timesizes(i);
+			}
+			au_update_dbrange(d, /*do_put_zero*/1);
+		} else
+			au_cpup_attr_nlink(i, /*force*/1);
+	} else {
+		bbot = a->btgt;
+		for (bindex = au_dbtop(d); bindex < bbot; bindex++)
+			au_set_h_dptr(d, bindex, NULL);
+		bbot = au_dbbot(d);
+		for (bindex = a->btgt + 1; bindex <= bbot; bindex++)
+			au_set_h_dptr(d, bindex, NULL);
+		au_update_dbrange(d, /*do_put_zero*/0);
+	}
+
+	if (a->exchange
+	    || au_ftest_ren(a->auren_flags, DIRREN)) {
+		d_drop(a->src_dentry);
+		if (au_ftest_ren(a->auren_flags, DIRREN))
+			au_set_dbwh(a->src_dentry, -1);
+		return;
+	}
+
+	d = a->src_dentry;
+	au_set_dbwh(d, -1);
+	bbot = au_dbbot(d);
+	for (bindex = a->btgt + 1; bindex <= bbot; bindex++) {
+		h_d = au_h_dptr(d, bindex);
+		if (h_d)
+			au_set_h_dptr(d, bindex, NULL);
+	}
+	au_set_dbbot(d, a->btgt);
+
+	sb = d->d_sb;
+	i = a->src_inode;
+	if (au_opt_test(au_mntflags(sb), PLINK) && au_plink_test(i))
+		return; /* success */
+
+	bbot = au_ibbot(i);
+	for (bindex = a->btgt + 1; bindex <= bbot; bindex++) {
+		h_i = au_h_iptr(i, bindex);
+		if (h_i) {
+			au_xino_write(sb, bindex, h_i->i_ino, /*ino*/0);
+			/* ignore this error */
+			au_set_h_iptr(i, bindex, NULL, 0);
+		}
+	}
+	au_set_ibbot(i, a->btgt);
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* mainly for link(2) and rename(2) */
+int au_wbr(struct dentry *dentry, aufs_bindex_t btgt)
+{
+	aufs_bindex_t bdiropq, bwh;
+	struct dentry *parent;
+	struct au_branch *br;
+
+	parent = dentry->d_parent;
+	IMustLock(d_inode(parent)); /* dir is locked */
+
+	bdiropq = au_dbdiropq(parent);
+	bwh = au_dbwh(dentry);
+	br = au_sbr(dentry->d_sb, btgt);
+	if (au_br_rdonly(br)
+	    || (0 <= bdiropq && bdiropq < btgt)
+	    || (0 <= bwh && bwh < btgt))
+		btgt = -1;
+
+	AuDbg("btgt %d\n", btgt);
+	return btgt;
+}
+
+/* sets src_btop, dst_btop and btgt */
+static int au_ren_wbr(struct au_ren_args *a)
+{
+	int err;
+	struct au_wr_dir_args wr_dir_args = {
+		/* .force_btgt	= -1, */
+		.flags		= AuWrDir_ADD_ENTRY
+	};
+
+	a->src_btop = au_dbtop(a->src_dentry);
+	a->dst_btop = au_dbtop(a->dst_dentry);
+	if (au_ftest_ren(a->auren_flags, ISDIR_SRC)
+	    || au_ftest_ren(a->auren_flags, ISDIR_DST))
+		au_fset_wrdir(wr_dir_args.flags, ISDIR);
+	wr_dir_args.force_btgt = a->src_btop;
+	if (a->dst_inode && a->dst_btop < a->src_btop)
+		wr_dir_args.force_btgt = a->dst_btop;
+	wr_dir_args.force_btgt = au_wbr(a->dst_dentry, wr_dir_args.force_btgt);
+	err = au_wr_dir(a->dst_dentry, a->src_dentry, &wr_dir_args);
+	a->btgt = err;
+	if (a->exchange)
+		au_update_dbtop(a->dst_dentry);
+
+	return err;
+}
+
+static void au_ren_dt(struct au_ren_args *a)
+{
+	a->h_path.dentry = a->src_h_parent;
+	au_dtime_store(a->src_dt + AuPARENT, a->src_parent, &a->h_path);
+	if (!au_ftest_ren(a->auren_flags, ISSAMEDIR)) {
+		a->h_path.dentry = a->dst_h_parent;
+		au_dtime_store(a->dst_dt + AuPARENT, a->dst_parent, &a->h_path);
+	}
+
+	au_fclr_ren(a->auren_flags, DT_DSTDIR);
+	if (!au_ftest_ren(a->auren_flags, ISDIR_SRC)
+	    && !a->exchange)
+		return;
+
+	a->h_path.dentry = a->src_h_dentry;
+	au_dtime_store(a->src_dt + AuCHILD, a->src_dentry, &a->h_path);
+	if (d_is_positive(a->dst_h_dentry)) {
+		au_fset_ren(a->auren_flags, DT_DSTDIR);
+		a->h_path.dentry = a->dst_h_dentry;
+		au_dtime_store(a->dst_dt + AuCHILD, a->dst_dentry, &a->h_path);
+	}
+}
+
+static void au_ren_rev_dt(int err, struct au_ren_args *a)
+{
+	struct dentry *h_d;
+	struct inode *h_inode;
+
+	au_dtime_revert(a->src_dt + AuPARENT);
+	if (!au_ftest_ren(a->auren_flags, ISSAMEDIR))
+		au_dtime_revert(a->dst_dt + AuPARENT);
+
+	if (au_ftest_ren(a->auren_flags, ISDIR_SRC) && err != -EIO) {
+		h_d = a->src_dt[AuCHILD].dt_h_path.dentry;
+		h_inode = d_inode(h_d);
+		inode_lock_nested(h_inode, AuLsc_I_CHILD);
+		au_dtime_revert(a->src_dt + AuCHILD);
+		inode_unlock(h_inode);
+
+		if (au_ftest_ren(a->auren_flags, DT_DSTDIR)) {
+			h_d = a->dst_dt[AuCHILD].dt_h_path.dentry;
+			h_inode = d_inode(h_d);
+			inode_lock_nested(h_inode, AuLsc_I_CHILD);
+			au_dtime_revert(a->dst_dt + AuCHILD);
+			inode_unlock(h_inode);
+		}
+	}
+}
+
+/* ---------------------------------------------------------------------- */
+
+int aufs_rename(struct inode *_src_dir, struct dentry *_src_dentry,
+		struct inode *_dst_dir, struct dentry *_dst_dentry,
+		unsigned int _flags)
+{
+	int err, lock_flags;
+	void *rev;
+	/* reduce stack space */
+	struct au_ren_args *a;
+	struct au_pin pin;
+
+	AuDbg("%pd, %pd, 0x%x\n", _src_dentry, _dst_dentry, _flags);
+	IMustLock(_src_dir);
+	IMustLock(_dst_dir);
+
+	err = -EINVAL;
+	if (unlikely(_flags & RENAME_WHITEOUT))
+		goto out;
+
+	err = -ENOMEM;
+	BUILD_BUG_ON(sizeof(*a) > PAGE_SIZE);
+	a = kzalloc(sizeof(*a), GFP_NOFS);
+	if (unlikely(!a))
+		goto out;
+
+	a->flags = _flags;
+	BUILD_BUG_ON(sizeof(a->exchange) == sizeof(u8)
+		     && RENAME_EXCHANGE > U8_MAX);
+	a->exchange = _flags & RENAME_EXCHANGE;
+	a->src_dir = _src_dir;
+	a->src_dentry = _src_dentry;
+	a->src_inode = NULL;
+	if (d_really_is_positive(a->src_dentry))
+		a->src_inode = d_inode(a->src_dentry);
+	a->src_parent = a->src_dentry->d_parent; /* dir inode is locked */
+	a->dst_dir = _dst_dir;
+	a->dst_dentry = _dst_dentry;
+	a->dst_inode = NULL;
+	if (d_really_is_positive(a->dst_dentry))
+		a->dst_inode = d_inode(a->dst_dentry);
+	a->dst_parent = a->dst_dentry->d_parent; /* dir inode is locked */
+	if (a->dst_inode) {
+		/*
+		 * if EXCHANGE && src is non-dir && dst is dir,
+		 * dst is not locked.
+		 */
+		/* IMustLock(a->dst_inode); */
+		au_igrab(a->dst_inode);
+	}
+
+	err = -ENOTDIR;
+	lock_flags = AuLock_FLUSH | AuLock_NOPLM | AuLock_GEN;
+	if (d_is_dir(a->src_dentry)) {
+		au_fset_ren(a->auren_flags, ISDIR_SRC);
+		if (unlikely(!a->exchange
+			     && d_really_is_positive(a->dst_dentry)
+			     && !d_is_dir(a->dst_dentry)))
+			goto out_free;
+		lock_flags |= AuLock_DIRS;
+	}
+	if (a->dst_inode && d_is_dir(a->dst_dentry)) {
+		au_fset_ren(a->auren_flags, ISDIR_DST);
+		if (unlikely(!a->exchange
+			     && d_really_is_positive(a->src_dentry)
+			     && !d_is_dir(a->src_dentry)))
+			goto out_free;
+		lock_flags |= AuLock_DIRS;
+	}
+	err = aufs_read_and_write_lock2(a->dst_dentry, a->src_dentry,
+					lock_flags);
+	if (unlikely(err))
+		goto out_free;
+
+	err = au_d_hashed_positive(a->src_dentry);
+	if (unlikely(err))
+		goto out_unlock;
+	err = -ENOENT;
+	if (a->dst_inode) {
+		/*
+		 * If it is a dir, VFS unhash it before this
+		 * function. It means we cannot rely upon d_unhashed().
+		 */
+		if (unlikely(!a->dst_inode->i_nlink))
+			goto out_unlock;
+		if (!au_ftest_ren(a->auren_flags, ISDIR_DST)) {
+			err = au_d_hashed_positive(a->dst_dentry);
+			if (unlikely(err && !a->exchange))
+				goto out_unlock;
+		} else if (unlikely(IS_DEADDIR(a->dst_inode)))
+			goto out_unlock;
+	} else if (unlikely(d_unhashed(a->dst_dentry)))
+		goto out_unlock;
+
+	/*
+	 * is it possible?
+	 * yes, it happened (in linux-3.3-rcN) but I don't know why.
+	 * there may exist a problem somewhere else.
+	 */
+	err = -EINVAL;
+	if (unlikely(d_inode(a->dst_parent) == d_inode(a->src_dentry)))
+		goto out_unlock;
+
+	au_fset_ren(a->auren_flags, ISSAMEDIR); /* temporary */
+	di_write_lock_parent(a->dst_parent);
+
+	/* which branch we process */
+	err = au_ren_wbr(a);
+	if (unlikely(err < 0))
+		goto out_parent;
+	a->br = au_sbr(a->dst_dentry->d_sb, a->btgt);
+	a->h_path.mnt = au_br_mnt(a->br);
+
+	/* are they available to be renamed */
+	err = au_ren_may_dir(a);
+	if (unlikely(err))
+		goto out_children;
+
+	/* prepare the writable parent dir on the same branch */
+	if (a->dst_btop == a->btgt) {
+		au_fset_ren(a->auren_flags, WHDST);
+	} else {
+		err = au_cpup_dirs(a->dst_dentry, a->btgt);
+		if (unlikely(err))
+			goto out_children;
+	}
+
+	err = 0;
+	if (!a->exchange) {
+		if (a->src_dir != a->dst_dir) {
+			/*
+			 * this temporary unlock is safe,
+			 * because both dir->i_mutex are locked.
+			 */
+			di_write_unlock(a->dst_parent);
+			di_write_lock_parent(a->src_parent);
+			err = au_wr_dir_need_wh(a->src_dentry,
+						au_ftest_ren(a->auren_flags,
+							     ISDIR_SRC),
+						&a->btgt);
+			di_write_unlock(a->src_parent);
+			di_write_lock2_parent(a->src_parent, a->dst_parent,
+					      /*isdir*/1);
+			au_fclr_ren(a->auren_flags, ISSAMEDIR);
+		} else
+			err = au_wr_dir_need_wh(a->src_dentry,
+						au_ftest_ren(a->auren_flags,
+							     ISDIR_SRC),
+						&a->btgt);
+	}
+	if (unlikely(err < 0))
+		goto out_children;
+	if (err)
+		au_fset_ren(a->auren_flags, WHSRC);
+
+	/* cpup src */
+	if (a->src_btop != a->btgt) {
+		err = au_pin(&pin, a->src_dentry, a->btgt,
+			     au_opt_udba(a->src_dentry->d_sb),
+			     AuPin_DI_LOCKED | AuPin_MNT_WRITE);
+		if (!err) {
+			struct au_cp_generic cpg = {
+				.dentry	= a->src_dentry,
+				.bdst	= a->btgt,
+				.bsrc	= a->src_btop,
+				.len	= -1,
+				.pin	= &pin,
+				.flags	= AuCpup_DTIME | AuCpup_HOPEN
+			};
+			AuDebugOn(au_dbtop(a->src_dentry) != a->src_btop);
+			err = au_sio_cpup_simple(&cpg);
+			au_unpin(&pin);
+		}
+		if (unlikely(err))
+			goto out_children;
+		a->src_btop = a->btgt;
+		a->src_h_dentry = au_h_dptr(a->src_dentry, a->btgt);
+		if (!a->exchange)
+			au_fset_ren(a->auren_flags, WHSRC);
+	}
+
+	/* cpup dst */
+	if (a->exchange && a->dst_inode
+	    && a->dst_btop != a->btgt) {
+		err = au_pin(&pin, a->dst_dentry, a->btgt,
+			     au_opt_udba(a->dst_dentry->d_sb),
+			     AuPin_DI_LOCKED | AuPin_MNT_WRITE);
+		if (!err) {
+			struct au_cp_generic cpg = {
+				.dentry	= a->dst_dentry,
+				.bdst	= a->btgt,
+				.bsrc	= a->dst_btop,
+				.len	= -1,
+				.pin	= &pin,
+				.flags	= AuCpup_DTIME | AuCpup_HOPEN
+			};
+			err = au_sio_cpup_simple(&cpg);
+			au_unpin(&pin);
+		}
+		if (unlikely(err))
+			goto out_children;
+		a->dst_btop = a->btgt;
+		a->dst_h_dentry = au_h_dptr(a->dst_dentry, a->btgt);
+	}
+
+	/* lock them all */
+	err = au_ren_lock(a);
+	if (unlikely(err))
+		/* leave the copied-up one */
+		goto out_children;
+
+	if (!a->exchange) {
+		if (!au_opt_test(au_mntflags(a->dst_dir->i_sb), UDBA_NONE))
+			err = au_may_ren(a);
+		else if (unlikely(a->dst_dentry->d_name.len > AUFS_MAX_NAMELEN))
+			err = -ENAMETOOLONG;
+		if (unlikely(err))
+			goto out_hdir;
+	}
+
+	/* store timestamps to be revertible */
+	au_ren_dt(a);
+
+	/* store dirren info */
+	if (au_ftest_ren(a->auren_flags, DIRREN)) {
+		err = au_dr_rename(a->src_dentry, a->btgt,
+				   &a->dst_dentry->d_name, &rev);
+		AuTraceErr(err);
+		if (unlikely(err))
+			goto out_dt;
+	}
+
+	/* here we go */
+	err = do_rename(a);
+	if (unlikely(err))
+		goto out_dirren;
+
+	if (au_ftest_ren(a->auren_flags, DIRREN))
+		au_dr_rename_fin(a->src_dentry, a->btgt, rev);
+
+	/* update dir attributes */
+	au_ren_refresh_dir(a);
+
+	/* dput/iput all lower dentries */
+	au_ren_refresh(a);
+
+	goto out_hdir; /* success */
+
+out_dirren:
+	if (au_ftest_ren(a->auren_flags, DIRREN))
+		au_dr_rename_rev(a->src_dentry, a->btgt, rev);
+out_dt:
+	au_ren_rev_dt(err, a);
+out_hdir:
+	au_ren_unlock(a);
+out_children:
+	au_nhash_wh_free(&a->whlist);
+	if (err && a->dst_inode && a->dst_btop != a->btgt) {
+		AuDbg("btop %d, btgt %d\n", a->dst_btop, a->btgt);
+		au_set_h_dptr(a->dst_dentry, a->btgt, NULL);
+		au_set_dbtop(a->dst_dentry, a->dst_btop);
+	}
+out_parent:
+	if (!err) {
+		if (d_unhashed(a->src_dentry))
+			au_fset_ren(a->auren_flags, DROPPED_SRC);
+		if (d_unhashed(a->dst_dentry))
+			au_fset_ren(a->auren_flags, DROPPED_DST);
+		if (!a->exchange)
+			d_move(a->src_dentry, a->dst_dentry);
+		else {
+			d_exchange(a->src_dentry, a->dst_dentry);
+			if (au_ftest_ren(a->auren_flags, DROPPED_DST))
+				d_drop(a->dst_dentry);
+		}
+		if (au_ftest_ren(a->auren_flags, DROPPED_SRC))
+			d_drop(a->src_dentry);
+	} else {
+		au_update_dbtop(a->dst_dentry);
+		if (!a->dst_inode)
+			d_drop(a->dst_dentry);
+	}
+	if (au_ftest_ren(a->auren_flags, ISSAMEDIR))
+		di_write_unlock(a->dst_parent);
+	else
+		di_write_unlock2(a->src_parent, a->dst_parent);
+out_unlock:
+	aufs_read_and_write_unlock2(a->dst_dentry, a->src_dentry);
+out_free:
+	iput(a->dst_inode);
+	if (a->thargs)
+		au_whtmp_rmdir_free(a->thargs);
+	au_kfree_rcu(a);
+out:
+	AuTraceErr(err);
+	return err;
+}
diff -Naur --no-dereference a/fs/aufs/Kconfig b/fs/aufs/Kconfig
--- a/fs/aufs/Kconfig	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/Kconfig	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,199 @@
+# SPDX-License-Identifier: GPL-2.0
+config AUFS_FS
+	tristate "Aufs (Advanced multi layered unification filesystem) support"
+	help
+	Aufs is a stackable unification filesystem such as Unionfs,
+	which unifies several directories and provides a merged single
+	directory.
+	In the early days, aufs was entirely re-designed and
+	re-implemented Unionfs Version 1.x series. Introducing many
+	original ideas, approaches and improvements, it becomes totally
+	different from Unionfs while keeping the basic features.
+
+if AUFS_FS
+choice
+	prompt "Maximum number of branches"
+	default AUFS_BRANCH_MAX_127
+	help
+	Specifies the maximum number of branches (or member directories)
+	in a single aufs. The larger value consumes more system
+	resources and has a minor impact to performance.
+config AUFS_BRANCH_MAX_127
+	bool "127"
+	help
+	Specifies the maximum number of branches (or member directories)
+	in a single aufs. The larger value consumes more system
+	resources and has a minor impact to performance.
+config AUFS_BRANCH_MAX_511
+	bool "511"
+	help
+	Specifies the maximum number of branches (or member directories)
+	in a single aufs. The larger value consumes more system
+	resources and has a minor impact to performance.
+config AUFS_BRANCH_MAX_1023
+	bool "1023"
+	help
+	Specifies the maximum number of branches (or member directories)
+	in a single aufs. The larger value consumes more system
+	resources and has a minor impact to performance.
+config AUFS_BRANCH_MAX_32767
+	bool "32767"
+	help
+	Specifies the maximum number of branches (or member directories)
+	in a single aufs. The larger value consumes more system
+	resources and has a minor impact to performance.
+endchoice
+
+config AUFS_SBILIST
+	bool
+	depends on AUFS_MAGIC_SYSRQ || PROC_FS
+	default y
+	help
+	Automatic configuration for internal use.
+	When aufs supports Magic SysRq or /proc, enabled automatically.
+
+config AUFS_HNOTIFY
+	bool "Detect direct branch access (bypassing aufs)"
+	help
+	If you want to modify files on branches directly, eg. bypassing aufs,
+	and want aufs to detect the changes of them fully, then enable this
+	option and use 'udba=notify' mount option.
+	Currently there is only one available configuration, "fsnotify".
+	It will have a negative impact to the performance.
+	See detail in aufs.5.
+
+choice
+	prompt "method" if AUFS_HNOTIFY
+	default AUFS_HFSNOTIFY
+config AUFS_HFSNOTIFY
+	bool "fsnotify"
+	select FSNOTIFY
+endchoice
+
+config AUFS_EXPORT
+	bool "NFS-exportable aufs"
+	depends on EXPORTFS
+	help
+	If you want to export your mounted aufs via NFS, then enable this
+	option. There are several requirements for this configuration.
+	See detail in aufs.5.
+
+config AUFS_INO_T_64
+	bool
+	depends on AUFS_EXPORT
+	depends on 64BIT && !(ALPHA || S390)
+	default y
+	help
+	Automatic configuration for internal use.
+	/* typedef unsigned long/int __kernel_ino_t */
+	/* alpha and s390x are int */
+
+config AUFS_XATTR
+	bool "support for XATTR/EA (including Security Labels)"
+	help
+	If your branch fs supports XATTR/EA and you want to make them
+	available in aufs too, then enable this opsion and specify the
+	branch attributes for EA.
+	See detail in aufs.5.
+
+config AUFS_FHSM
+	bool "File-based Hierarchical Storage Management"
+	help
+	Hierarchical Storage Management (or HSM) is a well-known feature
+	in the storage world. Aufs provides this feature as file-based.
+	with multiple branches.
+	These multiple branches are prioritized, ie. the topmost one
+	should be the fastest drive and be used heavily.
+
+config AUFS_RDU
+	bool "Readdir in userspace"
+	help
+	Aufs has two methods to provide a merged view for a directory,
+	by a user-space library and by kernel-space natively. The latter
+	is always enabled but sometimes large and slow.
+	If you enable this option, install the library in aufs2-util
+	package, and set some environment variables for your readdir(3),
+	then the work will be handled in user-space which generally
+	shows better performance in most cases.
+	See detail in aufs.5.
+
+config AUFS_DIRREN
+	bool "Workaround for rename(2)-ing a directory"
+	help
+	By default, aufs returns EXDEV error in renameing a dir who has
+	his child on the lower branch, since it is a bad idea to issue
+	rename(2) internally for every lower branch. But user may not
+	accept this behaviour. So here is a workaround to allow such
+	rename(2) and store some extra infromation on the writable
+	branch. Obviously this costs high (and I don't like it).
+	To use this feature, you need to enable this configuration AND
+	to specify the mount option `dirren.'
+	See details in aufs.5 and the design documents.
+
+config AUFS_SHWH
+	bool "Show whiteouts"
+	help
+	If you want to make the whiteouts in aufs visible, then enable
+	this option and specify 'shwh' mount option. Although it may
+	sounds like philosophy or something, but in technically it
+	simply shows the name of whiteout with keeping its behaviour.
+
+config AUFS_BR_RAMFS
+	bool "Ramfs (initramfs/rootfs) as an aufs branch"
+	help
+	If you want to use ramfs as an aufs branch fs, then enable this
+	option. Generally tmpfs is recommended.
+	Aufs prohibited them to be a branch fs by default, because
+	initramfs becomes unusable after switch_root or something
+	generally. If you sets initramfs as an aufs branch and boot your
+	system by switch_root, you will meet a problem easily since the
+	files in initramfs may be inaccessible.
+	Unless you are going to use ramfs as an aufs branch fs without
+	switch_root or something, leave it N.
+
+config AUFS_BR_FUSE
+	bool "Fuse fs as an aufs branch"
+	depends on FUSE_FS
+	select AUFS_POLL
+	help
+	If you want to use fuse-based userspace filesystem as an aufs
+	branch fs, then enable this option.
+	It implements the internal poll(2) operation which is
+	implemented by fuse only (curretnly).
+
+config AUFS_POLL
+	bool
+	help
+	Automatic configuration for internal use.
+
+config AUFS_BR_HFSPLUS
+	bool "Hfsplus as an aufs branch"
+	depends on HFSPLUS_FS
+	default y
+	help
+	If you want to use hfsplus fs as an aufs branch fs, then enable
+	this option. This option introduces a small overhead at
+	copying-up a file on hfsplus.
+
+config AUFS_BDEV_LOOP
+	bool
+	depends on BLK_DEV_LOOP
+	default y
+	help
+	Automatic configuration for internal use.
+	Convert =[ym] into =y.
+
+config AUFS_DEBUG
+	bool "Debug aufs"
+	help
+	Enable this to compile aufs internal debug code.
+	It will have a negative impact to the performance.
+
+config AUFS_MAGIC_SYSRQ
+	bool
+	depends on AUFS_DEBUG && MAGIC_SYSRQ
+	default y
+	help
+	Automatic configuration for internal use.
+	When aufs supports Magic SysRq, enabled automatically.
+endif
diff -Naur --no-dereference a/fs/aufs/lcnt.h b/fs/aufs/lcnt.h
--- a/fs/aufs/lcnt.h	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/lcnt.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,186 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2018-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * simple long counter wrapper
+ */
+
+#ifndef __AUFS_LCNT_H__
+#define __AUFS_LCNT_H__
+
+#ifdef __KERNEL__
+
+#include "debug.h"
+
+#define AuLCntATOMIC	1
+#define AuLCntPCPUCNT	2
+/*
+ * why does percpu_refcount require extra synchronize_rcu()s in
+ * au_br_do_free()
+ */
+#define AuLCntPCPUREF	3
+
+/* #define AuLCntChosen	AuLCntATOMIC */
+#define AuLCntChosen	AuLCntPCPUCNT
+/* #define AuLCntChosen	AuLCntPCPUREF */
+
+#if AuLCntChosen == AuLCntATOMIC
+#include <linux/atomic.h>
+
+typedef atomic_long_t au_lcnt_t;
+
+static inline int au_lcnt_init(au_lcnt_t *cnt, void *release __maybe_unused)
+{
+	atomic_long_set(cnt, 0);
+	return 0;
+}
+
+static inline void au_lcnt_wait_for_fin(au_lcnt_t *cnt __maybe_unused)
+{
+	/* empty */
+}
+
+static inline void au_lcnt_fin(au_lcnt_t *cnt __maybe_unused,
+			       int do_sync __maybe_unused)
+{
+	/* empty */
+}
+
+static inline void au_lcnt_inc(au_lcnt_t *cnt)
+{
+	atomic_long_inc(cnt);
+}
+
+static inline void au_lcnt_dec(au_lcnt_t *cnt)
+{
+	atomic_long_dec(cnt);
+}
+
+static inline long au_lcnt_read(au_lcnt_t *cnt, int do_rev __maybe_unused)
+{
+	return atomic_long_read(cnt);
+}
+#endif
+
+#if AuLCntChosen == AuLCntPCPUCNT
+#include <linux/percpu_counter.h>
+
+typedef struct percpu_counter au_lcnt_t;
+
+static inline int au_lcnt_init(au_lcnt_t *cnt, void *release __maybe_unused)
+{
+	return percpu_counter_init(cnt, 0, GFP_NOFS);
+}
+
+static inline void au_lcnt_wait_for_fin(au_lcnt_t *cnt __maybe_unused)
+{
+	/* empty */
+}
+
+static inline void au_lcnt_fin(au_lcnt_t *cnt, int do_sync __maybe_unused)
+{
+	percpu_counter_destroy(cnt);
+}
+
+static inline void au_lcnt_inc(au_lcnt_t *cnt)
+{
+	percpu_counter_inc(cnt);
+}
+
+static inline void au_lcnt_dec(au_lcnt_t *cnt)
+{
+	percpu_counter_dec(cnt);
+}
+
+static inline long au_lcnt_read(au_lcnt_t *cnt, int do_rev __maybe_unused)
+{
+	s64 n;
+
+	n = percpu_counter_sum(cnt);
+	BUG_ON(n < 0);
+	if (LONG_MAX != LLONG_MAX
+	    && n > LONG_MAX)
+		AuWarn1("%s\n", "wrap-around");
+
+	return n;
+}
+#endif
+
+#if AuLCntChosen == AuLCntPCPUREF
+#include <linux/percpu-refcount.h>
+
+typedef struct percpu_ref au_lcnt_t;
+
+static inline int au_lcnt_init(au_lcnt_t *cnt, percpu_ref_func_t *release)
+{
+	if (!release)
+		release = percpu_ref_exit;
+	return percpu_ref_init(cnt, release, /*percpu mode*/0, GFP_NOFS);
+}
+
+static inline void au_lcnt_wait_for_fin(au_lcnt_t *cnt __maybe_unused)
+{
+	synchronize_rcu();
+}
+
+static inline void au_lcnt_fin(au_lcnt_t *cnt, int do_sync)
+{
+	percpu_ref_kill(cnt);
+	if (do_sync)
+		au_lcnt_wait_for_fin(cnt);
+}
+
+static inline void au_lcnt_inc(au_lcnt_t *cnt)
+{
+	percpu_ref_get(cnt);
+}
+
+static inline void au_lcnt_dec(au_lcnt_t *cnt)
+{
+	percpu_ref_put(cnt);
+}
+
+/*
+ * avoid calling this func as possible.
+ */
+static inline long au_lcnt_read(au_lcnt_t *cnt, int do_rev)
+{
+	long l;
+
+	percpu_ref_switch_to_atomic_sync(cnt);
+	l = atomic_long_read(&cnt->count);
+	if (do_rev)
+		percpu_ref_switch_to_percpu(cnt);
+
+	/* percpu_ref is initialized by 1 instead of 0 */
+	return l - 1;
+}
+#endif
+
+#ifdef CONFIG_AUFS_DEBUG
+#define AuLCntZero(val) do {			\
+	long l = val;				\
+	if (l)					\
+		AuDbg("%s = %ld\n", #val, l);	\
+} while (0)
+#else
+#define AuLCntZero(val)		do {} while (0)
+#endif
+
+#endif /* __KERNEL__ */
+#endif /* __AUFS_LCNT_H__ */
diff -Naur --no-dereference a/fs/aufs/loop.c b/fs/aufs/loop.c
--- a/fs/aufs/loop.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/loop.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,148 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * support for loopback block device as a branch
+ */
+
+#include "aufs.h"
+
+/* added into drivers/block/loop.c */
+static struct file *(*backing_file_func)(struct super_block *sb);
+
+/*
+ * test if two lower dentries have overlapping branches.
+ */
+int au_test_loopback_overlap(struct super_block *sb, struct dentry *h_adding)
+{
+	struct super_block *h_sb;
+	struct file *backing_file;
+
+	if (unlikely(!backing_file_func)) {
+		/* don't load "loop" module here */
+		backing_file_func = symbol_get(loop_backing_file);
+		if (unlikely(!backing_file_func))
+			/* "loop" module is not loaded */
+			return 0;
+	}
+
+	h_sb = h_adding->d_sb;
+	backing_file = backing_file_func(h_sb);
+	if (!backing_file)
+		return 0;
+
+	h_adding = backing_file->f_path.dentry;
+	/*
+	 * h_adding can be local NFS.
+	 * in this case aufs cannot detect the loop.
+	 */
+	if (unlikely(h_adding->d_sb == sb))
+		return 1;
+	return !!au_test_subdir(h_adding, sb->s_root);
+}
+
+/* true if a kernel thread named 'loop[0-9].*' accesses a file */
+int au_test_loopback_kthread(void)
+{
+	int ret;
+	struct task_struct *tsk = current;
+	char c, comm[sizeof(tsk->comm)];
+
+	ret = 0;
+	if (tsk->flags & PF_KTHREAD) {
+		get_task_comm(comm, tsk);
+		c = comm[4];
+		ret = ('0' <= c && c <= '9'
+		       && !strncmp(comm, "loop", 4));
+	}
+
+	return ret;
+}
+
+/* ---------------------------------------------------------------------- */
+
+#define au_warn_loopback_step	16
+static int au_warn_loopback_nelem = au_warn_loopback_step;
+static unsigned long *au_warn_loopback_array;
+
+void au_warn_loopback(struct super_block *h_sb)
+{
+	int i, new_nelem;
+	unsigned long *a, magic;
+	static DEFINE_SPINLOCK(spin);
+
+	magic = h_sb->s_magic;
+	spin_lock(&spin);
+	a = au_warn_loopback_array;
+	for (i = 0; i < au_warn_loopback_nelem && *a; i++)
+		if (a[i] == magic) {
+			spin_unlock(&spin);
+			return;
+		}
+
+	/* h_sb is new to us, print it */
+	if (i < au_warn_loopback_nelem) {
+		a[i] = magic;
+		goto pr;
+	}
+
+	/* expand the array */
+	new_nelem = au_warn_loopback_nelem + au_warn_loopback_step;
+	a = au_kzrealloc(au_warn_loopback_array,
+			 au_warn_loopback_nelem * sizeof(unsigned long),
+			 new_nelem * sizeof(unsigned long), GFP_ATOMIC,
+			 /*may_shrink*/0);
+	if (a) {
+		au_warn_loopback_nelem = new_nelem;
+		au_warn_loopback_array = a;
+		a[i] = magic;
+		goto pr;
+	}
+
+	spin_unlock(&spin);
+	AuWarn1("realloc failed, ignored\n");
+	return;
+
+pr:
+	spin_unlock(&spin);
+	pr_warn("you may want to try another patch for loopback file "
+		"on %s(0x%lx) branch\n", au_sbtype(h_sb), magic);
+}
+
+int au_loopback_init(void)
+{
+	int err;
+	struct super_block *sb __maybe_unused;
+
+	BUILD_BUG_ON(sizeof(sb->s_magic) != sizeof(unsigned long));
+
+	err = 0;
+	au_warn_loopback_array = kcalloc(au_warn_loopback_step,
+					 sizeof(unsigned long), GFP_NOFS);
+	if (unlikely(!au_warn_loopback_array))
+		err = -ENOMEM;
+
+	return err;
+}
+
+void au_loopback_fin(void)
+{
+	if (backing_file_func)
+		symbol_put(loop_backing_file);
+	au_kfree_try_rcu(au_warn_loopback_array);
+}
diff -Naur --no-dereference a/fs/aufs/loop.h b/fs/aufs/loop.h
--- a/fs/aufs/loop.h	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/loop.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,55 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * support for loopback mount as a branch
+ */
+
+#ifndef __AUFS_LOOP_H__
+#define __AUFS_LOOP_H__
+
+#ifdef __KERNEL__
+
+struct dentry;
+struct super_block;
+
+#ifdef CONFIG_AUFS_BDEV_LOOP
+/* drivers/block/loop.c */
+struct file *loop_backing_file(struct super_block *sb);
+
+/* loop.c */
+int au_test_loopback_overlap(struct super_block *sb, struct dentry *h_adding);
+int au_test_loopback_kthread(void);
+void au_warn_loopback(struct super_block *h_sb);
+
+int au_loopback_init(void);
+void au_loopback_fin(void);
+#else
+AuStub(struct file *, loop_backing_file, return NULL, struct super_block *sb)
+
+AuStubInt0(au_test_loopback_overlap, struct super_block *sb,
+	   struct dentry *h_adding)
+AuStubInt0(au_test_loopback_kthread, void)
+AuStubVoid(au_warn_loopback, struct super_block *h_sb)
+
+AuStubInt0(au_loopback_init, void)
+AuStubVoid(au_loopback_fin, void)
+#endif /* BLK_DEV_LOOP */
+
+#endif /* __KERNEL__ */
+#endif /* __AUFS_LOOP_H__ */
diff -Naur --no-dereference a/fs/aufs/magic.mk b/fs/aufs/magic.mk
--- a/fs/aufs/magic.mk	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/magic.mk	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,31 @@
+# SPDX-License-Identifier: GPL-2.0
+
+# defined in ${srctree}/fs/fuse/inode.c
+# tristate
+ifdef CONFIG_FUSE_FS
+ccflags-y += -DFUSE_SUPER_MAGIC=0x65735546
+endif
+
+# defined in ${srctree}/fs/xfs/xfs_sb.h
+# tristate
+ifdef CONFIG_XFS_FS
+ccflags-y += -DXFS_SB_MAGIC=0x58465342
+endif
+
+# defined in ${srctree}/fs/configfs/mount.c
+# tristate
+ifdef CONFIG_CONFIGFS_FS
+ccflags-y += -DCONFIGFS_MAGIC=0x62656570
+endif
+
+# defined in ${srctree}/fs/ubifs/ubifs.h
+# tristate
+ifdef CONFIG_UBIFS_FS
+ccflags-y += -DUBIFS_SUPER_MAGIC=0x24051905
+endif
+
+# defined in ${srctree}/fs/hfsplus/hfsplus_raw.h
+# tristate
+ifdef CONFIG_HFSPLUS_FS
+ccflags-y += -DHFSPLUS_SUPER_MAGIC=0x482b
+endif
diff -Naur --no-dereference a/fs/aufs/Makefile b/fs/aufs/Makefile
--- a/fs/aufs/Makefile	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/Makefile	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,46 @@
+# SPDX-License-Identifier: GPL-2.0
+
+include ${src}/magic.mk
+ifeq (${CONFIG_AUFS_FS},m)
+include ${src}/conf.mk
+endif
+-include ${src}/priv_def.mk
+
+# cf. include/linux/kernel.h
+# enable pr_debug
+ccflags-y += -DDEBUG
+# sparse requires the full pathname
+ifdef M
+ccflags-y += -include ${M}/../../include/uapi/linux/aufs_type.h
+else
+ccflags-y += -include ${srctree}/include/uapi/linux/aufs_type.h
+endif
+
+obj-$(CONFIG_AUFS_FS) += aufs.o
+aufs-y := module.o sbinfo.o super.o branch.o xino.o sysaufs.o opts.o \
+	wkq.o vfsub.o dcsub.o \
+	cpup.o whout.o wbr_policy.o \
+	dinfo.o dentry.o \
+	dynop.o \
+	finfo.o file.o f_op.o \
+	dir.o vdir.o \
+	iinfo.o inode.o i_op.o i_op_add.o i_op_del.o i_op_ren.o \
+	mvdown.o ioctl.o
+
+# all are boolean
+aufs-$(CONFIG_PROC_FS) += procfs.o plink.o
+aufs-$(CONFIG_SYSFS) += sysfs.o
+aufs-$(CONFIG_DEBUG_FS) += dbgaufs.o
+aufs-$(CONFIG_AUFS_BDEV_LOOP) += loop.o
+aufs-$(CONFIG_AUFS_HNOTIFY) += hnotify.o
+aufs-$(CONFIG_AUFS_HFSNOTIFY) += hfsnotify.o
+aufs-$(CONFIG_AUFS_EXPORT) += export.o
+aufs-$(CONFIG_AUFS_XATTR) += xattr.o
+aufs-$(CONFIG_FS_POSIX_ACL) += posix_acl.o
+aufs-$(CONFIG_AUFS_DIRREN) += dirren.o
+aufs-$(CONFIG_AUFS_FHSM) += fhsm.o
+aufs-$(CONFIG_AUFS_POLL) += poll.o
+aufs-$(CONFIG_AUFS_RDU) += rdu.o
+aufs-$(CONFIG_AUFS_BR_HFSPLUS) += hfsplus.o
+aufs-$(CONFIG_AUFS_DEBUG) += debug.o
+aufs-$(CONFIG_AUFS_MAGIC_SYSRQ) += sysrq.o
diff -Naur --no-dereference a/fs/aufs/module.c b/fs/aufs/module.c
--- a/fs/aufs/module.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/module.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,273 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * module global variables and operations
+ */
+
+#include <linux/module.h>
+#include <linux/seq_file.h>
+#include "aufs.h"
+
+/* shrinkable realloc */
+void *au_krealloc(void *p, unsigned int new_sz, gfp_t gfp, int may_shrink)
+{
+	size_t sz;
+	int diff;
+
+	sz = 0;
+	diff = -1;
+	if (p) {
+#if 0 /* unused */
+		if (!new_sz) {
+			au_kfree_rcu(p);
+			p = NULL;
+			goto out;
+		}
+#else
+		AuDebugOn(!new_sz);
+#endif
+		sz = ksize(p);
+		diff = au_kmidx_sub(sz, new_sz);
+	}
+	if (sz && !diff)
+		goto out;
+
+	if (sz < new_sz)
+		/* expand or SLOB */
+		p = krealloc(p, new_sz, gfp);
+	else if (new_sz < sz && may_shrink) {
+		/* shrink */
+		void *q;
+
+		q = kmalloc(new_sz, gfp);
+		if (q) {
+			if (p) {
+				memcpy(q, p, new_sz);
+				au_kfree_try_rcu(p);
+			}
+			p = q;
+		} else
+			p = NULL;
+	}
+
+out:
+	return p;
+}
+
+void *au_kzrealloc(void *p, unsigned int nused, unsigned int new_sz, gfp_t gfp,
+		   int may_shrink)
+{
+	p = au_krealloc(p, new_sz, gfp, may_shrink);
+	if (p && new_sz > nused)
+		memset(p + nused, 0, new_sz - nused);
+	return p;
+}
+
+/* ---------------------------------------------------------------------- */
+/*
+ * aufs caches
+ */
+struct kmem_cache *au_cache[AuCache_Last];
+
+static void au_cache_fin(void)
+{
+	int i;
+
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
+
+	/* excluding AuCache_HNOTIFY */
+	BUILD_BUG_ON(AuCache_HNOTIFY + 1 != AuCache_Last);
+	for (i = 0; i < AuCache_HNOTIFY; i++) {
+		kmem_cache_destroy(au_cache[i]);
+		au_cache[i] = NULL;
+	}
+}
+
+static int __init au_cache_init(void)
+{
+	au_cache[AuCache_DINFO] = AuCacheCtor(au_dinfo, au_di_init_once);
+	if (au_cache[AuCache_DINFO])
+		/* SLAB_DESTROY_BY_RCU */
+		au_cache[AuCache_ICNTNR] = AuCacheCtor(au_icntnr,
+						       au_icntnr_init_once);
+	if (au_cache[AuCache_ICNTNR])
+		au_cache[AuCache_FINFO] = AuCacheCtor(au_finfo,
+						      au_fi_init_once);
+	if (au_cache[AuCache_FINFO])
+		au_cache[AuCache_VDIR] = AuCache(au_vdir);
+	if (au_cache[AuCache_VDIR])
+		au_cache[AuCache_DEHSTR] = AuCache(au_vdir_dehstr);
+	if (au_cache[AuCache_DEHSTR])
+		return 0;
+
+	au_cache_fin();
+	return -ENOMEM;
+}
+
+/* ---------------------------------------------------------------------- */
+
+int au_dir_roflags;
+
+#ifdef CONFIG_AUFS_SBILIST
+/*
+ * iterate_supers_type() doesn't protect us from
+ * remounting (branch management)
+ */
+struct hlist_bl_head au_sbilist;
+#endif
+
+/*
+ * functions for module interface.
+ */
+MODULE_LICENSE("GPL");
+/* MODULE_LICENSE("GPL v2"); */
+MODULE_AUTHOR("Junjiro R. Okajima <aufs-users@lists.sourceforge.net>");
+MODULE_DESCRIPTION(AUFS_NAME
+	" -- Advanced multi layered unification filesystem");
+MODULE_VERSION(AUFS_VERSION);
+MODULE_ALIAS_FS(AUFS_NAME);
+
+/* this module parameter has no meaning when SYSFS is disabled */
+int sysaufs_brs = 1;
+MODULE_PARM_DESC(brs, "use <sysfs>/fs/aufs/si_*/brN");
+module_param_named(brs, sysaufs_brs, int, 0444);
+
+/* this module parameter has no meaning when USER_NS is disabled */
+bool au_userns;
+MODULE_PARM_DESC(allow_userns, "allow unprivileged to mount under userns");
+module_param_named(allow_userns, au_userns, bool, 0444);
+
+/* ---------------------------------------------------------------------- */
+
+static char au_esc_chars[0x20 + 3]; /* 0x01-0x20, backslash, del, and NULL */
+
+int au_seq_path(struct seq_file *seq, struct path *path)
+{
+	int err;
+
+	err = seq_path(seq, path, au_esc_chars);
+	if (err >= 0)
+		err = 0;
+	else
+		err = -ENOMEM;
+
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static int __init aufs_init(void)
+{
+	int err, i;
+	char *p;
+
+	p = au_esc_chars;
+	for (i = 1; i <= ' '; i++)
+		*p++ = i;
+	*p++ = '\\';
+	*p++ = '\x7f';
+	*p = 0;
+
+	au_dir_roflags = au_file_roflags(O_DIRECTORY | O_LARGEFILE);
+
+	memcpy(aufs_iop_nogetattr, aufs_iop, sizeof(aufs_iop));
+	for (i = 0; i < AuIop_Last; i++)
+		aufs_iop_nogetattr[i].getattr = NULL;
+
+	memset(au_cache, 0, sizeof(au_cache));	/* including hnotify */
+
+	au_sbilist_init();
+	sysaufs_brs_init();
+	au_debug_init();
+	au_dy_init();
+	err = sysaufs_init();
+	if (unlikely(err))
+		goto out;
+	err = dbgaufs_init();
+	if (unlikely(err))
+		goto out_sysaufs;
+	err = au_procfs_init();
+	if (unlikely(err))
+		goto out_dbgaufs;
+	err = au_wkq_init();
+	if (unlikely(err))
+		goto out_procfs;
+	err = au_loopback_init();
+	if (unlikely(err))
+		goto out_wkq;
+	err = au_hnotify_init();
+	if (unlikely(err))
+		goto out_loopback;
+	err = au_sysrq_init();
+	if (unlikely(err))
+		goto out_hin;
+	err = au_cache_init();
+	if (unlikely(err))
+		goto out_sysrq;
+
+	aufs_fs_type.fs_flags |= au_userns ? FS_USERNS_MOUNT : 0;
+	err = register_filesystem(&aufs_fs_type);
+	if (unlikely(err))
+		goto out_cache;
+
+	/* since we define pr_fmt, call printk directly */
+	printk(KERN_INFO AUFS_NAME " " AUFS_VERSION "\n");
+	goto out; /* success */
+
+out_cache:
+	au_cache_fin();
+out_sysrq:
+	au_sysrq_fin();
+out_hin:
+	au_hnotify_fin();
+out_loopback:
+	au_loopback_fin();
+out_wkq:
+	au_wkq_fin();
+out_procfs:
+	au_procfs_fin();
+out_dbgaufs:
+	dbgaufs_fin();
+out_sysaufs:
+	sysaufs_fin();
+	au_dy_fin();
+out:
+	return err;
+}
+
+static void __exit aufs_exit(void)
+{
+	unregister_filesystem(&aufs_fs_type);
+	au_cache_fin();
+	au_sysrq_fin();
+	au_hnotify_fin();
+	au_loopback_fin();
+	au_wkq_fin();
+	au_procfs_fin();
+	dbgaufs_fin();
+	sysaufs_fin();
+	au_dy_fin();
+}
+
+module_init(aufs_init);
+module_exit(aufs_exit);
diff -Naur --no-dereference a/fs/aufs/module.h b/fs/aufs/module.h
--- a/fs/aufs/module.h	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/module.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,166 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * module initialization and module-global
+ */
+
+#ifndef __AUFS_MODULE_H__
+#define __AUFS_MODULE_H__
+
+#ifdef __KERNEL__
+
+#include <linux/slab.h>
+#include "debug.h"
+#include "dentry.h"
+#include "dir.h"
+#include "file.h"
+#include "inode.h"
+
+struct path;
+struct seq_file;
+
+/* module parameters */
+extern int sysaufs_brs;
+extern bool au_userns;
+
+/* ---------------------------------------------------------------------- */
+
+extern int au_dir_roflags;
+
+void *au_krealloc(void *p, unsigned int new_sz, gfp_t gfp, int may_shrink);
+void *au_kzrealloc(void *p, unsigned int nused, unsigned int new_sz, gfp_t gfp,
+		   int may_shrink);
+
+/*
+ * Comparing the size of the object with sizeof(struct rcu_head)
+ * case 1: object is always larger
+ *	--> au_kfree_rcu() or au_kfree_do_rcu()
+ * case 2: object is always smaller
+ *	--> au_kfree_small()
+ * case 3: object can be any size
+ *	--> au_kfree_try_rcu()
+ */
+
+static inline void au_kfree_do_rcu(const void *p)
+{
+	struct {
+		struct rcu_head rcu;
+	} *a = (void *)p;
+
+	kfree_rcu(a, rcu);
+}
+
+#define au_kfree_rcu(_p) do {						\
+		typeof(_p) p = (_p);					\
+		BUILD_BUG_ON(sizeof(*p) < sizeof(struct rcu_head));	\
+		if (p)							\
+			au_kfree_do_rcu(p);				\
+	} while (0)
+
+#define au_kfree_do_sz_test(sz)	(sz >= sizeof(struct rcu_head))
+#define au_kfree_sz_test(p)	(p && au_kfree_do_sz_test(ksize(p)))
+
+static inline void au_kfree_try_rcu(const void *p)
+{
+	if (!p)
+		return;
+	if (au_kfree_sz_test(p))
+		au_kfree_do_rcu(p);
+	else
+		kfree(p);
+}
+
+static inline void au_kfree_small(const void *p)
+{
+	if (!p)
+		return;
+	AuDebugOn(au_kfree_sz_test(p));
+	kfree(p);
+}
+
+static inline int au_kmidx_sub(size_t sz, size_t new_sz)
+{
+#ifndef CONFIG_SLOB
+	return kmalloc_index(sz) - kmalloc_index(new_sz);
+#else
+	return -1; /* SLOB is untested */
+#endif
+}
+
+int au_seq_path(struct seq_file *seq, struct path *path);
+
+#ifdef CONFIG_PROC_FS
+/* procfs.c */
+int __init au_procfs_init(void);
+void au_procfs_fin(void);
+#else
+AuStubInt0(au_procfs_init, void);
+AuStubVoid(au_procfs_fin, void);
+#endif
+
+/* ---------------------------------------------------------------------- */
+
+/* kmem cache */
+enum {
+	AuCache_DINFO,
+	AuCache_ICNTNR,
+	AuCache_FINFO,
+	AuCache_VDIR,
+	AuCache_DEHSTR,
+	AuCache_HNOTIFY, /* must be last */
+	AuCache_Last
+};
+
+extern struct kmem_cache *au_cache[AuCache_Last];
+
+#define AuCacheFlags		(SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD)
+#define AuCache(type)		KMEM_CACHE(type, AuCacheFlags)
+#define AuCacheCtor(type, ctor)	\
+	kmem_cache_create(#type, sizeof(struct type), \
+			  __alignof__(struct type), AuCacheFlags, ctor)
+
+#define AuCacheFuncs(name, index)					\
+	static inline struct au_##name *au_cache_alloc_##name(void)	\
+	{ return kmem_cache_alloc(au_cache[AuCache_##index], GFP_NOFS); } \
+	static inline void au_cache_free_##name##_norcu(struct au_##name *p) \
+	{ kmem_cache_free(au_cache[AuCache_##index], p); }		\
+									\
+	static inline void au_cache_free_##name##_rcu_cb(struct rcu_head *rcu) \
+	{ void *p = rcu;						\
+		p -= offsetof(struct au_##name, rcu);			\
+		kmem_cache_free(au_cache[AuCache_##index], p); }	\
+	static inline void au_cache_free_##name##_rcu(struct au_##name *p) \
+	{ BUILD_BUG_ON(sizeof(struct au_##name) < sizeof(struct rcu_head)); \
+		call_rcu(&p->rcu, au_cache_free_##name##_rcu_cb); }	\
+									\
+	static inline void au_cache_free_##name(struct au_##name *p)	\
+	{ /* au_cache_free_##name##_norcu(p); */			\
+		au_cache_free_##name##_rcu(p); }
+
+AuCacheFuncs(dinfo, DINFO);
+AuCacheFuncs(icntnr, ICNTNR);
+AuCacheFuncs(finfo, FINFO);
+AuCacheFuncs(vdir, VDIR);
+AuCacheFuncs(vdir_dehstr, DEHSTR);
+#ifdef CONFIG_AUFS_HNOTIFY
+AuCacheFuncs(hnotify, HNOTIFY);
+#endif
+
+#endif /* __KERNEL__ */
+#endif /* __AUFS_MODULE_H__ */
diff -Naur --no-dereference a/fs/aufs/mvdown.c b/fs/aufs/mvdown.c
--- a/fs/aufs/mvdown.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/mvdown.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,705 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2011-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * move-down, opposite of copy-up
+ */
+
+#include "aufs.h"
+
+struct au_mvd_args {
+	struct {
+		struct super_block *h_sb;
+		struct dentry *h_parent;
+		struct au_hinode *hdir;
+		struct inode *h_dir, *h_inode;
+		struct au_pin pin;
+	} info[AUFS_MVDOWN_NARRAY];
+
+	struct aufs_mvdown mvdown;
+	struct dentry *dentry, *parent;
+	struct inode *inode, *dir;
+	struct super_block *sb;
+	aufs_bindex_t bopq, bwh, bfound;
+	unsigned char rename_lock;
+};
+
+#define mvd_errno		mvdown.au_errno
+#define mvd_bsrc		mvdown.stbr[AUFS_MVDOWN_UPPER].bindex
+#define mvd_src_brid		mvdown.stbr[AUFS_MVDOWN_UPPER].brid
+#define mvd_bdst		mvdown.stbr[AUFS_MVDOWN_LOWER].bindex
+#define mvd_dst_brid		mvdown.stbr[AUFS_MVDOWN_LOWER].brid
+
+#define mvd_h_src_sb		info[AUFS_MVDOWN_UPPER].h_sb
+#define mvd_h_src_parent	info[AUFS_MVDOWN_UPPER].h_parent
+#define mvd_hdir_src		info[AUFS_MVDOWN_UPPER].hdir
+#define mvd_h_src_dir		info[AUFS_MVDOWN_UPPER].h_dir
+#define mvd_h_src_inode		info[AUFS_MVDOWN_UPPER].h_inode
+#define mvd_pin_src		info[AUFS_MVDOWN_UPPER].pin
+
+#define mvd_h_dst_sb		info[AUFS_MVDOWN_LOWER].h_sb
+#define mvd_h_dst_parent	info[AUFS_MVDOWN_LOWER].h_parent
+#define mvd_hdir_dst		info[AUFS_MVDOWN_LOWER].hdir
+#define mvd_h_dst_dir		info[AUFS_MVDOWN_LOWER].h_dir
+#define mvd_h_dst_inode		info[AUFS_MVDOWN_LOWER].h_inode
+#define mvd_pin_dst		info[AUFS_MVDOWN_LOWER].pin
+
+#define AU_MVD_PR(flag, ...) do {			\
+		if (flag)				\
+			pr_err(__VA_ARGS__);		\
+	} while (0)
+
+static int find_lower_writable(struct au_mvd_args *a)
+{
+	struct super_block *sb;
+	aufs_bindex_t bindex, bbot;
+	struct au_branch *br;
+
+	sb = a->sb;
+	bindex = a->mvd_bsrc;
+	bbot = au_sbbot(sb);
+	if (a->mvdown.flags & AUFS_MVDOWN_FHSM_LOWER)
+		for (bindex++; bindex <= bbot; bindex++) {
+			br = au_sbr(sb, bindex);
+			if (au_br_fhsm(br->br_perm)
+			    && !sb_rdonly(au_br_sb(br)))
+				return bindex;
+		}
+	else if (!(a->mvdown.flags & AUFS_MVDOWN_ROLOWER))
+		for (bindex++; bindex <= bbot; bindex++) {
+			br = au_sbr(sb, bindex);
+			if (!au_br_rdonly(br))
+				return bindex;
+		}
+	else
+		for (bindex++; bindex <= bbot; bindex++) {
+			br = au_sbr(sb, bindex);
+			if (!sb_rdonly(au_br_sb(br))) {
+				if (au_br_rdonly(br))
+					a->mvdown.flags
+						|= AUFS_MVDOWN_ROLOWER_R;
+				return bindex;
+			}
+		}
+
+	return -1;
+}
+
+/* make the parent dir on bdst */
+static int au_do_mkdir(const unsigned char dmsg, struct au_mvd_args *a)
+{
+	int err;
+
+	err = 0;
+	a->mvd_hdir_src = au_hi(a->dir, a->mvd_bsrc);
+	a->mvd_hdir_dst = au_hi(a->dir, a->mvd_bdst);
+	a->mvd_h_src_parent = au_h_dptr(a->parent, a->mvd_bsrc);
+	a->mvd_h_dst_parent = NULL;
+	if (au_dbbot(a->parent) >= a->mvd_bdst)
+		a->mvd_h_dst_parent = au_h_dptr(a->parent, a->mvd_bdst);
+	if (!a->mvd_h_dst_parent) {
+		err = au_cpdown_dirs(a->dentry, a->mvd_bdst);
+		if (unlikely(err)) {
+			AU_MVD_PR(dmsg, "cpdown_dirs failed\n");
+			goto out;
+		}
+		a->mvd_h_dst_parent = au_h_dptr(a->parent, a->mvd_bdst);
+	}
+
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+/* lock them all */
+static int au_do_lock(const unsigned char dmsg, struct au_mvd_args *a)
+{
+	int err;
+	struct dentry *h_trap;
+
+	a->mvd_h_src_sb = au_sbr_sb(a->sb, a->mvd_bsrc);
+	a->mvd_h_dst_sb = au_sbr_sb(a->sb, a->mvd_bdst);
+	err = au_pin(&a->mvd_pin_dst, a->dentry, a->mvd_bdst,
+		     au_opt_udba(a->sb),
+		     AuPin_MNT_WRITE | AuPin_DI_LOCKED);
+	AuTraceErr(err);
+	if (unlikely(err)) {
+		AU_MVD_PR(dmsg, "pin_dst failed\n");
+		goto out;
+	}
+
+	if (a->mvd_h_src_sb != a->mvd_h_dst_sb) {
+		a->rename_lock = 0;
+		au_pin_init(&a->mvd_pin_src, a->dentry, a->mvd_bsrc,
+			    AuLsc_DI_PARENT, AuLsc_I_PARENT3,
+			    au_opt_udba(a->sb),
+			    AuPin_MNT_WRITE | AuPin_DI_LOCKED);
+		err = au_do_pin(&a->mvd_pin_src);
+		AuTraceErr(err);
+		a->mvd_h_src_dir = d_inode(a->mvd_h_src_parent);
+		if (unlikely(err)) {
+			AU_MVD_PR(dmsg, "pin_src failed\n");
+			goto out_dst;
+		}
+		goto out; /* success */
+	}
+
+	a->rename_lock = 1;
+	au_pin_hdir_unlock(&a->mvd_pin_dst);
+	err = au_pin(&a->mvd_pin_src, a->dentry, a->mvd_bsrc,
+		     au_opt_udba(a->sb),
+		     AuPin_MNT_WRITE | AuPin_DI_LOCKED);
+	AuTraceErr(err);
+	a->mvd_h_src_dir = d_inode(a->mvd_h_src_parent);
+	if (unlikely(err)) {
+		AU_MVD_PR(dmsg, "pin_src failed\n");
+		au_pin_hdir_lock(&a->mvd_pin_dst);
+		goto out_dst;
+	}
+	au_pin_hdir_unlock(&a->mvd_pin_src);
+	h_trap = vfsub_lock_rename(a->mvd_h_src_parent, a->mvd_hdir_src,
+				   a->mvd_h_dst_parent, a->mvd_hdir_dst);
+	if (h_trap) {
+		err = (h_trap != a->mvd_h_src_parent);
+		if (err)
+			err = (h_trap != a->mvd_h_dst_parent);
+	}
+	BUG_ON(err); /* it should never happen */
+	if (unlikely(a->mvd_h_src_dir != au_pinned_h_dir(&a->mvd_pin_src))) {
+		err = -EBUSY;
+		AuTraceErr(err);
+		vfsub_unlock_rename(a->mvd_h_src_parent, a->mvd_hdir_src,
+				    a->mvd_h_dst_parent, a->mvd_hdir_dst);
+		au_pin_hdir_lock(&a->mvd_pin_src);
+		au_unpin(&a->mvd_pin_src);
+		au_pin_hdir_lock(&a->mvd_pin_dst);
+		goto out_dst;
+	}
+	goto out; /* success */
+
+out_dst:
+	au_unpin(&a->mvd_pin_dst);
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+static void au_do_unlock(const unsigned char dmsg, struct au_mvd_args *a)
+{
+	if (!a->rename_lock)
+		au_unpin(&a->mvd_pin_src);
+	else {
+		vfsub_unlock_rename(a->mvd_h_src_parent, a->mvd_hdir_src,
+				    a->mvd_h_dst_parent, a->mvd_hdir_dst);
+		au_pin_hdir_lock(&a->mvd_pin_src);
+		au_unpin(&a->mvd_pin_src);
+		au_pin_hdir_lock(&a->mvd_pin_dst);
+	}
+	au_unpin(&a->mvd_pin_dst);
+}
+
+/* copy-down the file */
+static int au_do_cpdown(const unsigned char dmsg, struct au_mvd_args *a)
+{
+	int err;
+	struct au_cp_generic cpg = {
+		.dentry	= a->dentry,
+		.bdst	= a->mvd_bdst,
+		.bsrc	= a->mvd_bsrc,
+		.len	= -1,
+		.pin	= &a->mvd_pin_dst,
+		.flags	= AuCpup_DTIME | AuCpup_HOPEN
+	};
+
+	AuDbg("b%d, b%d\n", cpg.bsrc, cpg.bdst);
+	if (a->mvdown.flags & AUFS_MVDOWN_OWLOWER)
+		au_fset_cpup(cpg.flags, OVERWRITE);
+	if (a->mvdown.flags & AUFS_MVDOWN_ROLOWER)
+		au_fset_cpup(cpg.flags, RWDST);
+	err = au_sio_cpdown_simple(&cpg);
+	if (unlikely(err))
+		AU_MVD_PR(dmsg, "cpdown failed\n");
+
+	AuTraceErr(err);
+	return err;
+}
+
+/*
+ * unlink the whiteout on bdst if exist which may be created by UDBA while we
+ * were sleeping
+ */
+static int au_do_unlink_wh(const unsigned char dmsg, struct au_mvd_args *a)
+{
+	int err;
+	struct path h_path;
+	struct au_branch *br;
+	struct inode *delegated;
+
+	br = au_sbr(a->sb, a->mvd_bdst);
+	h_path.dentry = au_wh_lkup(a->mvd_h_dst_parent, &a->dentry->d_name, br);
+	err = PTR_ERR(h_path.dentry);
+	if (IS_ERR(h_path.dentry)) {
+		AU_MVD_PR(dmsg, "wh_lkup failed\n");
+		goto out;
+	}
+
+	err = 0;
+	if (d_is_positive(h_path.dentry)) {
+		h_path.mnt = au_br_mnt(br);
+		delegated = NULL;
+		err = vfsub_unlink(d_inode(a->mvd_h_dst_parent), &h_path,
+				   &delegated, /*force*/0);
+		if (unlikely(err == -EWOULDBLOCK)) {
+			pr_warn("cannot retry for NFSv4 delegation"
+				" for an internal unlink\n");
+			iput(delegated);
+		}
+		if (unlikely(err))
+			AU_MVD_PR(dmsg, "wh_unlink failed\n");
+	}
+	dput(h_path.dentry);
+
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+/*
+ * unlink the topmost h_dentry
+ */
+static int au_do_unlink(const unsigned char dmsg, struct au_mvd_args *a)
+{
+	int err;
+	struct path h_path;
+	struct inode *delegated;
+
+	h_path.mnt = au_sbr_mnt(a->sb, a->mvd_bsrc);
+	h_path.dentry = au_h_dptr(a->dentry, a->mvd_bsrc);
+	delegated = NULL;
+	err = vfsub_unlink(a->mvd_h_src_dir, &h_path, &delegated, /*force*/0);
+	if (unlikely(err == -EWOULDBLOCK)) {
+		pr_warn("cannot retry for NFSv4 delegation"
+			" for an internal unlink\n");
+		iput(delegated);
+	}
+	if (unlikely(err))
+		AU_MVD_PR(dmsg, "unlink failed\n");
+
+	AuTraceErr(err);
+	return err;
+}
+
+/* Since mvdown succeeded, we ignore an error of this function */
+static void au_do_stfs(const unsigned char dmsg, struct au_mvd_args *a)
+{
+	int err;
+	struct au_branch *br;
+
+	a->mvdown.flags |= AUFS_MVDOWN_STFS_FAILED;
+	br = au_sbr(a->sb, a->mvd_bsrc);
+	err = au_br_stfs(br, &a->mvdown.stbr[AUFS_MVDOWN_UPPER].stfs);
+	if (!err) {
+		br = au_sbr(a->sb, a->mvd_bdst);
+		a->mvdown.stbr[AUFS_MVDOWN_LOWER].brid = br->br_id;
+		err = au_br_stfs(br, &a->mvdown.stbr[AUFS_MVDOWN_LOWER].stfs);
+	}
+	if (!err)
+		a->mvdown.flags &= ~AUFS_MVDOWN_STFS_FAILED;
+	else
+		AU_MVD_PR(dmsg, "statfs failed (%d), ignored\n", err);
+}
+
+/*
+ * copy-down the file and unlink the bsrc file.
+ * - unlink the bdst whout if exist
+ * - copy-down the file (with whtmp name and rename)
+ * - unlink the bsrc file
+ */
+static int au_do_mvdown(const unsigned char dmsg, struct au_mvd_args *a)
+{
+	int err;
+
+	err = au_do_mkdir(dmsg, a);
+	if (!err)
+		err = au_do_lock(dmsg, a);
+	if (unlikely(err))
+		goto out;
+
+	/*
+	 * do not revert the activities we made on bdst since they should be
+	 * harmless in aufs.
+	 */
+
+	err = au_do_cpdown(dmsg, a);
+	if (!err)
+		err = au_do_unlink_wh(dmsg, a);
+	if (!err && !(a->mvdown.flags & AUFS_MVDOWN_KUPPER))
+		err = au_do_unlink(dmsg, a);
+	if (unlikely(err))
+		goto out_unlock;
+
+	AuDbg("%pd2, 0x%x, %d --> %d\n",
+	      a->dentry, a->mvdown.flags, a->mvd_bsrc, a->mvd_bdst);
+	if (find_lower_writable(a) < 0)
+		a->mvdown.flags |= AUFS_MVDOWN_BOTTOM;
+
+	if (a->mvdown.flags & AUFS_MVDOWN_STFS)
+		au_do_stfs(dmsg, a);
+
+	/* maintain internal array */
+	if (!(a->mvdown.flags & AUFS_MVDOWN_KUPPER)) {
+		au_set_h_dptr(a->dentry, a->mvd_bsrc, NULL);
+		au_set_dbtop(a->dentry, a->mvd_bdst);
+		au_set_h_iptr(a->inode, a->mvd_bsrc, NULL, /*flags*/0);
+		au_set_ibtop(a->inode, a->mvd_bdst);
+	} else {
+		/* hide the lower */
+		au_set_h_dptr(a->dentry, a->mvd_bdst, NULL);
+		au_set_dbbot(a->dentry, a->mvd_bsrc);
+		au_set_h_iptr(a->inode, a->mvd_bdst, NULL, /*flags*/0);
+		au_set_ibbot(a->inode, a->mvd_bsrc);
+	}
+	if (au_dbbot(a->dentry) < a->mvd_bdst)
+		au_set_dbbot(a->dentry, a->mvd_bdst);
+	if (au_ibbot(a->inode) < a->mvd_bdst)
+		au_set_ibbot(a->inode, a->mvd_bdst);
+
+out_unlock:
+	au_do_unlock(dmsg, a);
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* make sure the file is idle */
+static int au_mvd_args_busy(const unsigned char dmsg, struct au_mvd_args *a)
+{
+	int err, plinked;
+
+	err = 0;
+	plinked = !!au_opt_test(au_mntflags(a->sb), PLINK);
+	if (au_dbtop(a->dentry) == a->mvd_bsrc
+	    && au_dcount(a->dentry) == 1
+	    && atomic_read(&a->inode->i_count) == 1
+	    /* && a->mvd_h_src_inode->i_nlink == 1 */
+	    && (!plinked || !au_plink_test(a->inode))
+	    && a->inode->i_nlink == 1)
+		goto out;
+
+	err = -EBUSY;
+	AU_MVD_PR(dmsg,
+		  "b%d, d{b%d, c%d?}, i{c%d?, l%u}, hi{l%u}, p{%d, %d}\n",
+		  a->mvd_bsrc, au_dbtop(a->dentry), au_dcount(a->dentry),
+		  atomic_read(&a->inode->i_count), a->inode->i_nlink,
+		  a->mvd_h_src_inode->i_nlink,
+		  plinked, plinked ? au_plink_test(a->inode) : 0);
+
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+/* make sure the parent dir is fine */
+static int au_mvd_args_parent(const unsigned char dmsg,
+			      struct au_mvd_args *a)
+{
+	int err;
+	aufs_bindex_t bindex;
+
+	err = 0;
+	if (unlikely(au_alive_dir(a->parent))) {
+		err = -ENOENT;
+		AU_MVD_PR(dmsg, "parent dir is dead\n");
+		goto out;
+	}
+
+	a->bopq = au_dbdiropq(a->parent);
+	bindex = au_wbr_nonopq(a->dentry, a->mvd_bdst);
+	AuDbg("b%d\n", bindex);
+	if (unlikely((bindex >= 0 && bindex < a->mvd_bdst)
+		     || (a->bopq != -1 && a->bopq < a->mvd_bdst))) {
+		err = -EINVAL;
+		a->mvd_errno = EAU_MVDOWN_OPAQUE;
+		AU_MVD_PR(dmsg, "ancestor is opaque b%d, b%d\n",
+			  a->bopq, a->mvd_bdst);
+	}
+
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+static int au_mvd_args_intermediate(const unsigned char dmsg,
+				    struct au_mvd_args *a)
+{
+	int err;
+	struct au_dinfo *dinfo, *tmp;
+
+	/* lookup the next lower positive entry */
+	err = -ENOMEM;
+	tmp = au_di_alloc(a->sb, AuLsc_DI_TMP);
+	if (unlikely(!tmp))
+		goto out;
+
+	a->bfound = -1;
+	a->bwh = -1;
+	dinfo = au_di(a->dentry);
+	au_di_cp(tmp, dinfo);
+	au_di_swap(tmp, dinfo);
+
+	/* returns the number of positive dentries */
+	err = au_lkup_dentry(a->dentry, a->mvd_bsrc + 1,
+			     /* AuLkup_IGNORE_PERM */ 0);
+	if (!err)
+		a->bwh = au_dbwh(a->dentry);
+	else if (err > 0)
+		a->bfound = au_dbtop(a->dentry);
+
+	au_di_swap(tmp, dinfo);
+	au_rw_write_unlock(&tmp->di_rwsem);
+	au_di_free(tmp);
+	if (unlikely(err < 0))
+		AU_MVD_PR(dmsg, "failed look-up lower\n");
+
+	/*
+	 * here, we have these cases.
+	 * bfound == -1
+	 *	no positive dentry under bsrc. there are more sub-cases.
+	 *	bwh < 0
+	 *		there no whiteout, we can safely move-down.
+	 *	bwh <= bsrc
+	 *		impossible
+	 *	bsrc < bwh && bwh < bdst
+	 *		there is a whiteout on RO branch. cannot proceed.
+	 *	bwh == bdst
+	 *		there is a whiteout on the RW target branch. it should
+	 *		be removed.
+	 *	bdst < bwh
+	 *		there is a whiteout somewhere unrelated branch.
+	 * -1 < bfound && bfound <= bsrc
+	 *	impossible.
+	 * bfound < bdst
+	 *	found, but it is on RO branch between bsrc and bdst. cannot
+	 *	proceed.
+	 * bfound == bdst
+	 *	found, replace it if AUFS_MVDOWN_FORCE is set. otherwise return
+	 *	error.
+	 * bdst < bfound
+	 *	found, after we create the file on bdst, it will be hidden.
+	 */
+
+	AuDebugOn(a->bfound == -1
+		  && a->bwh != -1
+		  && a->bwh <= a->mvd_bsrc);
+	AuDebugOn(-1 < a->bfound
+		  && a->bfound <= a->mvd_bsrc);
+
+	err = -EINVAL;
+	if (a->bfound == -1
+	    && a->mvd_bsrc < a->bwh
+	    && a->bwh != -1
+	    && a->bwh < a->mvd_bdst) {
+		a->mvd_errno = EAU_MVDOWN_WHITEOUT;
+		AU_MVD_PR(dmsg, "bsrc %d, bdst %d, bfound %d, bwh %d\n",
+			  a->mvd_bsrc, a->mvd_bdst, a->bfound, a->bwh);
+		goto out;
+	} else if (a->bfound != -1 && a->bfound < a->mvd_bdst) {
+		a->mvd_errno = EAU_MVDOWN_UPPER;
+		AU_MVD_PR(dmsg, "bdst %d, bfound %d\n",
+			  a->mvd_bdst, a->bfound);
+		goto out;
+	}
+
+	err = 0; /* success */
+
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+static int au_mvd_args_exist(const unsigned char dmsg, struct au_mvd_args *a)
+{
+	int err;
+
+	err = 0;
+	if (!(a->mvdown.flags & AUFS_MVDOWN_OWLOWER)
+	    && a->bfound == a->mvd_bdst)
+		err = -EEXIST;
+	AuTraceErr(err);
+	return err;
+}
+
+static int au_mvd_args(const unsigned char dmsg, struct au_mvd_args *a)
+{
+	int err;
+	struct au_branch *br;
+
+	err = -EISDIR;
+	if (unlikely(S_ISDIR(a->inode->i_mode)))
+		goto out;
+
+	err = -EINVAL;
+	if (!(a->mvdown.flags & AUFS_MVDOWN_BRID_UPPER))
+		a->mvd_bsrc = au_ibtop(a->inode);
+	else {
+		a->mvd_bsrc = au_br_index(a->sb, a->mvd_src_brid);
+		if (unlikely(a->mvd_bsrc < 0
+			     || (a->mvd_bsrc < au_dbtop(a->dentry)
+				 || au_dbbot(a->dentry) < a->mvd_bsrc
+				 || !au_h_dptr(a->dentry, a->mvd_bsrc))
+			     || (a->mvd_bsrc < au_ibtop(a->inode)
+				 || au_ibbot(a->inode) < a->mvd_bsrc
+				 || !au_h_iptr(a->inode, a->mvd_bsrc)))) {
+			a->mvd_errno = EAU_MVDOWN_NOUPPER;
+			AU_MVD_PR(dmsg, "no upper\n");
+			goto out;
+		}
+	}
+	if (unlikely(a->mvd_bsrc == au_sbbot(a->sb))) {
+		a->mvd_errno = EAU_MVDOWN_BOTTOM;
+		AU_MVD_PR(dmsg, "on the bottom\n");
+		goto out;
+	}
+	a->mvd_h_src_inode = au_h_iptr(a->inode, a->mvd_bsrc);
+	br = au_sbr(a->sb, a->mvd_bsrc);
+	err = au_br_rdonly(br);
+	if (!(a->mvdown.flags & AUFS_MVDOWN_ROUPPER)) {
+		if (unlikely(err))
+			goto out;
+	} else if (!(vfsub_native_ro(a->mvd_h_src_inode)
+		     || IS_APPEND(a->mvd_h_src_inode))) {
+		if (err)
+			a->mvdown.flags |= AUFS_MVDOWN_ROUPPER_R;
+		/* go on */
+	} else
+		goto out;
+
+	err = -EINVAL;
+	if (!(a->mvdown.flags & AUFS_MVDOWN_BRID_LOWER)) {
+		a->mvd_bdst = find_lower_writable(a);
+		if (unlikely(a->mvd_bdst < 0)) {
+			a->mvd_errno = EAU_MVDOWN_BOTTOM;
+			AU_MVD_PR(dmsg, "no writable lower branch\n");
+			goto out;
+		}
+	} else {
+		a->mvd_bdst = au_br_index(a->sb, a->mvd_dst_brid);
+		if (unlikely(a->mvd_bdst < 0
+			     || au_sbbot(a->sb) < a->mvd_bdst)) {
+			a->mvd_errno = EAU_MVDOWN_NOLOWERBR;
+			AU_MVD_PR(dmsg, "no lower brid\n");
+			goto out;
+		}
+	}
+
+	err = au_mvd_args_busy(dmsg, a);
+	if (!err)
+		err = au_mvd_args_parent(dmsg, a);
+	if (!err)
+		err = au_mvd_args_intermediate(dmsg, a);
+	if (!err)
+		err = au_mvd_args_exist(dmsg, a);
+	if (!err)
+		AuDbg("b%d, b%d\n", a->mvd_bsrc, a->mvd_bdst);
+
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+int au_mvdown(struct dentry *dentry, struct aufs_mvdown __user *uarg)
+{
+	int err, e;
+	unsigned char dmsg;
+	struct au_mvd_args *args;
+	struct inode *inode;
+
+	inode = d_inode(dentry);
+	err = -EPERM;
+	if (unlikely(!capable(CAP_SYS_ADMIN)))
+		goto out;
+
+	err = -ENOMEM;
+	args = kmalloc(sizeof(*args), GFP_NOFS);
+	if (unlikely(!args))
+		goto out;
+
+	err = copy_from_user(&args->mvdown, uarg, sizeof(args->mvdown));
+	if (!err)
+		err = !access_ok(VERIFY_WRITE, uarg, sizeof(*uarg));
+	if (unlikely(err)) {
+		err = -EFAULT;
+		AuTraceErr(err);
+		goto out_free;
+	}
+	AuDbg("flags 0x%x\n", args->mvdown.flags);
+	args->mvdown.flags &= ~(AUFS_MVDOWN_ROLOWER_R | AUFS_MVDOWN_ROUPPER_R);
+	args->mvdown.au_errno = 0;
+	args->dentry = dentry;
+	args->inode = inode;
+	args->sb = dentry->d_sb;
+
+	err = -ENOENT;
+	dmsg = !!(args->mvdown.flags & AUFS_MVDOWN_DMSG);
+	args->parent = dget_parent(dentry);
+	args->dir = d_inode(args->parent);
+	inode_lock_nested(args->dir, I_MUTEX_PARENT);
+	dput(args->parent);
+	if (unlikely(args->parent != dentry->d_parent)) {
+		AU_MVD_PR(dmsg, "parent dir is moved\n");
+		goto out_dir;
+	}
+
+	inode_lock_nested(inode, I_MUTEX_CHILD);
+	err = aufs_read_lock(dentry, AuLock_DW | AuLock_FLUSH | AuLock_NOPLMW);
+	if (unlikely(err))
+		goto out_inode;
+
+	di_write_lock_parent(args->parent);
+	err = au_mvd_args(dmsg, args);
+	if (unlikely(err))
+		goto out_parent;
+
+	err = au_do_mvdown(dmsg, args);
+	if (unlikely(err))
+		goto out_parent;
+
+	au_cpup_attr_timesizes(args->dir);
+	au_cpup_attr_timesizes(inode);
+	if (!(args->mvdown.flags & AUFS_MVDOWN_KUPPER))
+		au_cpup_igen(inode, au_h_iptr(inode, args->mvd_bdst));
+	/* au_digen_dec(dentry); */
+
+out_parent:
+	di_write_unlock(args->parent);
+	aufs_read_unlock(dentry, AuLock_DW);
+out_inode:
+	inode_unlock(inode);
+out_dir:
+	inode_unlock(args->dir);
+out_free:
+	e = copy_to_user(uarg, &args->mvdown, sizeof(args->mvdown));
+	if (unlikely(e))
+		err = -EFAULT;
+	au_kfree_rcu(args);
+out:
+	AuTraceErr(err);
+	return err;
+}
diff -Naur --no-dereference a/fs/aufs/opts.c b/fs/aufs/opts.c
--- a/fs/aufs/opts.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/opts.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,1880 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * mount options/flags
+ */
+
+#include <linux/namei.h>
+#include <linux/types.h> /* a distribution requires */
+#include <linux/parser.h>
+#include "aufs.h"
+
+/* ---------------------------------------------------------------------- */
+
+enum {
+	Opt_br,
+	Opt_add, Opt_del, Opt_mod, Opt_append, Opt_prepend,
+	Opt_idel, Opt_imod,
+	Opt_dirwh, Opt_rdcache, Opt_rdblk, Opt_rdhash,
+	Opt_rdblk_def, Opt_rdhash_def,
+	Opt_xino, Opt_noxino,
+	Opt_trunc_xino, Opt_trunc_xino_v, Opt_notrunc_xino,
+	Opt_trunc_xino_path, Opt_itrunc_xino,
+	Opt_trunc_xib, Opt_notrunc_xib,
+	Opt_shwh, Opt_noshwh,
+	Opt_plink, Opt_noplink, Opt_list_plink,
+	Opt_udba,
+	Opt_dio, Opt_nodio,
+	Opt_diropq_a, Opt_diropq_w,
+	Opt_warn_perm, Opt_nowarn_perm,
+	Opt_wbr_copyup, Opt_wbr_create,
+	Opt_fhsm_sec,
+	Opt_verbose, Opt_noverbose,
+	Opt_sum, Opt_nosum, Opt_wsum,
+	Opt_dirperm1, Opt_nodirperm1,
+	Opt_dirren, Opt_nodirren,
+	Opt_acl, Opt_noacl,
+	Opt_tail, Opt_ignore, Opt_ignore_silent, Opt_err
+};
+
+static match_table_t options = {
+	{Opt_br, "br=%s"},
+	{Opt_br, "br:%s"},
+
+	{Opt_add, "add=%d:%s"},
+	{Opt_add, "add:%d:%s"},
+	{Opt_add, "ins=%d:%s"},
+	{Opt_add, "ins:%d:%s"},
+	{Opt_append, "append=%s"},
+	{Opt_append, "append:%s"},
+	{Opt_prepend, "prepend=%s"},
+	{Opt_prepend, "prepend:%s"},
+
+	{Opt_del, "del=%s"},
+	{Opt_del, "del:%s"},
+	/* {Opt_idel, "idel:%d"}, */
+	{Opt_mod, "mod=%s"},
+	{Opt_mod, "mod:%s"},
+	/* {Opt_imod, "imod:%d:%s"}, */
+
+	{Opt_dirwh, "dirwh=%d"},
+
+	{Opt_xino, "xino=%s"},
+	{Opt_noxino, "noxino"},
+	{Opt_trunc_xino, "trunc_xino"},
+	{Opt_trunc_xino_v, "trunc_xino_v=%d:%d"},
+	{Opt_notrunc_xino, "notrunc_xino"},
+	{Opt_trunc_xino_path, "trunc_xino=%s"},
+	{Opt_itrunc_xino, "itrunc_xino=%d"},
+	/* {Opt_zxino, "zxino=%s"}, */
+	{Opt_trunc_xib, "trunc_xib"},
+	{Opt_notrunc_xib, "notrunc_xib"},
+
+#ifdef CONFIG_PROC_FS
+	{Opt_plink, "plink"},
+#else
+	{Opt_ignore_silent, "plink"},
+#endif
+
+	{Opt_noplink, "noplink"},
+
+#ifdef CONFIG_AUFS_DEBUG
+	{Opt_list_plink, "list_plink"},
+#endif
+
+	{Opt_udba, "udba=%s"},
+
+	{Opt_dio, "dio"},
+	{Opt_nodio, "nodio"},
+
+#ifdef CONFIG_AUFS_DIRREN
+	{Opt_dirren, "dirren"},
+	{Opt_nodirren, "nodirren"},
+#else
+	{Opt_ignore, "dirren"},
+	{Opt_ignore_silent, "nodirren"},
+#endif
+
+#ifdef CONFIG_AUFS_FHSM
+	{Opt_fhsm_sec, "fhsm_sec=%d"},
+#else
+	{Opt_ignore, "fhsm_sec=%d"},
+#endif
+
+	{Opt_diropq_a, "diropq=always"},
+	{Opt_diropq_a, "diropq=a"},
+	{Opt_diropq_w, "diropq=whiteouted"},
+	{Opt_diropq_w, "diropq=w"},
+
+	{Opt_warn_perm, "warn_perm"},
+	{Opt_nowarn_perm, "nowarn_perm"},
+
+	/* keep them temporary */
+	{Opt_ignore_silent, "nodlgt"},
+	{Opt_ignore, "clean_plink"},
+
+#ifdef CONFIG_AUFS_SHWH
+	{Opt_shwh, "shwh"},
+#endif
+	{Opt_noshwh, "noshwh"},
+
+	{Opt_dirperm1, "dirperm1"},
+	{Opt_nodirperm1, "nodirperm1"},
+
+	{Opt_verbose, "verbose"},
+	{Opt_verbose, "v"},
+	{Opt_noverbose, "noverbose"},
+	{Opt_noverbose, "quiet"},
+	{Opt_noverbose, "q"},
+	{Opt_noverbose, "silent"},
+
+	{Opt_sum, "sum"},
+	{Opt_nosum, "nosum"},
+	{Opt_wsum, "wsum"},
+
+	{Opt_rdcache, "rdcache=%d"},
+	{Opt_rdblk, "rdblk=%d"},
+	{Opt_rdblk_def, "rdblk=def"},
+	{Opt_rdhash, "rdhash=%d"},
+	{Opt_rdhash_def, "rdhash=def"},
+
+	{Opt_wbr_create, "create=%s"},
+	{Opt_wbr_create, "create_policy=%s"},
+	{Opt_wbr_copyup, "cpup=%s"},
+	{Opt_wbr_copyup, "copyup=%s"},
+	{Opt_wbr_copyup, "copyup_policy=%s"},
+
+	/* generic VFS flag */
+#ifdef CONFIG_FS_POSIX_ACL
+	{Opt_acl, "acl"},
+	{Opt_noacl, "noacl"},
+#else
+	{Opt_ignore, "acl"},
+	{Opt_ignore_silent, "noacl"},
+#endif
+
+	/* internal use for the scripts */
+	{Opt_ignore_silent, "si=%s"},
+
+	{Opt_br, "dirs=%s"},
+	{Opt_ignore, "debug=%d"},
+	{Opt_ignore, "delete=whiteout"},
+	{Opt_ignore, "delete=all"},
+	{Opt_ignore, "imap=%s"},
+
+	/* temporary workaround, due to old mount(8)? */
+	{Opt_ignore_silent, "relatime"},
+
+	{Opt_err, NULL}
+};
+
+/* ---------------------------------------------------------------------- */
+
+static const char *au_parser_pattern(int val, match_table_t tbl)
+{
+	struct match_token *p;
+
+	p = tbl;
+	while (p->pattern) {
+		if (p->token == val)
+			return p->pattern;
+		p++;
+	}
+	BUG();
+	return "??";
+}
+
+static const char *au_optstr(int *val, match_table_t tbl)
+{
+	struct match_token *p;
+	int v;
+
+	v = *val;
+	if (!v)
+		goto out;
+	p = tbl;
+	while (p->pattern) {
+		if (p->token
+		    && (v & p->token) == p->token) {
+			*val &= ~p->token;
+			return p->pattern;
+		}
+		p++;
+	}
+
+out:
+	return NULL;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static match_table_t brperm = {
+	{AuBrPerm_RO, AUFS_BRPERM_RO},
+	{AuBrPerm_RR, AUFS_BRPERM_RR},
+	{AuBrPerm_RW, AUFS_BRPERM_RW},
+	{0, NULL}
+};
+
+static match_table_t brattr = {
+	/* general */
+	{AuBrAttr_COO_REG, AUFS_BRATTR_COO_REG},
+	{AuBrAttr_COO_ALL, AUFS_BRATTR_COO_ALL},
+	/* 'unpin' attrib is meaningless since linux-3.18-rc1 */
+	{AuBrAttr_UNPIN, AUFS_BRATTR_UNPIN},
+#ifdef CONFIG_AUFS_FHSM
+	{AuBrAttr_FHSM, AUFS_BRATTR_FHSM},
+#endif
+#ifdef CONFIG_AUFS_XATTR
+	{AuBrAttr_ICEX, AUFS_BRATTR_ICEX},
+	{AuBrAttr_ICEX_SEC, AUFS_BRATTR_ICEX_SEC},
+	{AuBrAttr_ICEX_SYS, AUFS_BRATTR_ICEX_SYS},
+	{AuBrAttr_ICEX_TR, AUFS_BRATTR_ICEX_TR},
+	{AuBrAttr_ICEX_USR, AUFS_BRATTR_ICEX_USR},
+	{AuBrAttr_ICEX_OTH, AUFS_BRATTR_ICEX_OTH},
+#endif
+
+	/* ro/rr branch */
+	{AuBrRAttr_WH, AUFS_BRRATTR_WH},
+
+	/* rw branch */
+	{AuBrWAttr_MOO, AUFS_BRWATTR_MOO},
+	{AuBrWAttr_NoLinkWH, AUFS_BRWATTR_NLWH},
+
+	{0, NULL}
+};
+
+static int br_attr_val(char *str, match_table_t table, substring_t args[])
+{
+	int attr, v;
+	char *p;
+
+	attr = 0;
+	do {
+		p = strchr(str, '+');
+		if (p)
+			*p = 0;
+		v = match_token(str, table, args);
+		if (v) {
+			if (v & AuBrAttr_CMOO_Mask)
+				attr &= ~AuBrAttr_CMOO_Mask;
+			attr |= v;
+		} else {
+			if (p)
+				*p = '+';
+			pr_warn("ignored branch attribute %s\n", str);
+			break;
+		}
+		if (p)
+			str = p + 1;
+	} while (p);
+
+	return attr;
+}
+
+static int au_do_optstr_br_attr(au_br_perm_str_t *str, int perm)
+{
+	int sz;
+	const char *p;
+	char *q;
+
+	q = str->a;
+	*q = 0;
+	p = au_optstr(&perm, brattr);
+	if (p) {
+		sz = strlen(p);
+		memcpy(q, p, sz + 1);
+		q += sz;
+	} else
+		goto out;
+
+	do {
+		p = au_optstr(&perm, brattr);
+		if (p) {
+			*q++ = '+';
+			sz = strlen(p);
+			memcpy(q, p, sz + 1);
+			q += sz;
+		}
+	} while (p);
+
+out:
+	return q - str->a;
+}
+
+static int noinline_for_stack br_perm_val(char *perm)
+{
+	int val, bad, sz;
+	char *p;
+	substring_t args[MAX_OPT_ARGS];
+	au_br_perm_str_t attr;
+
+	p = strchr(perm, '+');
+	if (p)
+		*p = 0;
+	val = match_token(perm, brperm, args);
+	if (!val) {
+		if (p)
+			*p = '+';
+		pr_warn("ignored branch permission %s\n", perm);
+		val = AuBrPerm_RO;
+		goto out;
+	}
+	if (!p)
+		goto out;
+
+	val |= br_attr_val(p + 1, brattr, args);
+
+	bad = 0;
+	switch (val & AuBrPerm_Mask) {
+	case AuBrPerm_RO:
+	case AuBrPerm_RR:
+		bad = val & AuBrWAttr_Mask;
+		val &= ~AuBrWAttr_Mask;
+		break;
+	case AuBrPerm_RW:
+		bad = val & AuBrRAttr_Mask;
+		val &= ~AuBrRAttr_Mask;
+		break;
+	}
+
+	/*
+	 * 'unpin' attrib becomes meaningless since linux-3.18-rc1, but aufs
+	 * does not treat it as an error, just warning.
+	 * this is a tiny guard for the user operation.
+	 */
+	if (val & AuBrAttr_UNPIN) {
+		bad |= AuBrAttr_UNPIN;
+		val &= ~AuBrAttr_UNPIN;
+	}
+
+	if (unlikely(bad)) {
+		sz = au_do_optstr_br_attr(&attr, bad);
+		AuDebugOn(!sz);
+		pr_warn("ignored branch attribute %s\n", attr.a);
+	}
+
+out:
+	return val;
+}
+
+void au_optstr_br_perm(au_br_perm_str_t *str, int perm)
+{
+	au_br_perm_str_t attr;
+	const char *p;
+	char *q;
+	int sz;
+
+	q = str->a;
+	p = au_optstr(&perm, brperm);
+	AuDebugOn(!p || !*p);
+	sz = strlen(p);
+	memcpy(q, p, sz + 1);
+	q += sz;
+
+	sz = au_do_optstr_br_attr(&attr, perm);
+	if (sz) {
+		*q++ = '+';
+		memcpy(q, attr.a, sz + 1);
+	}
+
+	AuDebugOn(strlen(str->a) >= sizeof(str->a));
+}
+
+/* ---------------------------------------------------------------------- */
+
+static match_table_t udbalevel = {
+	{AuOpt_UDBA_REVAL, "reval"},
+	{AuOpt_UDBA_NONE, "none"},
+#ifdef CONFIG_AUFS_HNOTIFY
+	{AuOpt_UDBA_HNOTIFY, "notify"}, /* abstraction */
+#ifdef CONFIG_AUFS_HFSNOTIFY
+	{AuOpt_UDBA_HNOTIFY, "fsnotify"},
+#endif
+#endif
+	{-1, NULL}
+};
+
+static int noinline_for_stack udba_val(char *str)
+{
+	substring_t args[MAX_OPT_ARGS];
+
+	return match_token(str, udbalevel, args);
+}
+
+const char *au_optstr_udba(int udba)
+{
+	return au_parser_pattern(udba, udbalevel);
+}
+
+/* ---------------------------------------------------------------------- */
+
+static match_table_t au_wbr_create_policy = {
+	{AuWbrCreate_TDP, "tdp"},
+	{AuWbrCreate_TDP, "top-down-parent"},
+	{AuWbrCreate_RR, "rr"},
+	{AuWbrCreate_RR, "round-robin"},
+	{AuWbrCreate_MFS, "mfs"},
+	{AuWbrCreate_MFS, "most-free-space"},
+	{AuWbrCreate_MFSV, "mfs:%d"},
+	{AuWbrCreate_MFSV, "most-free-space:%d"},
+
+	/* top-down regardless the parent, and then mfs */
+	{AuWbrCreate_TDMFS, "tdmfs:%d"},
+	{AuWbrCreate_TDMFSV, "tdmfs:%d:%d"},
+
+	{AuWbrCreate_MFSRR, "mfsrr:%d"},
+	{AuWbrCreate_MFSRRV, "mfsrr:%d:%d"},
+	{AuWbrCreate_PMFS, "pmfs"},
+	{AuWbrCreate_PMFSV, "pmfs:%d"},
+	{AuWbrCreate_PMFSRR, "pmfsrr:%d"},
+	{AuWbrCreate_PMFSRRV, "pmfsrr:%d:%d"},
+
+	{-1, NULL}
+};
+
+static int au_wbr_mfs_wmark(substring_t *arg, char *str,
+			    struct au_opt_wbr_create *create)
+{
+	int err;
+	unsigned long long ull;
+
+	err = 0;
+	if (!match_u64(arg, &ull))
+		create->mfsrr_watermark = ull;
+	else {
+		pr_err("bad integer in %s\n", str);
+		err = -EINVAL;
+	}
+
+	return err;
+}
+
+static int au_wbr_mfs_sec(substring_t *arg, char *str,
+			  struct au_opt_wbr_create *create)
+{
+	int n, err;
+
+	err = 0;
+	if (!match_int(arg, &n) && 0 <= n && n <= AUFS_MFS_MAX_SEC)
+		create->mfs_second = n;
+	else {
+		pr_err("bad integer in %s\n", str);
+		err = -EINVAL;
+	}
+
+	return err;
+}
+
+static int noinline_for_stack
+au_wbr_create_val(char *str, struct au_opt_wbr_create *create)
+{
+	int err, e;
+	substring_t args[MAX_OPT_ARGS];
+
+	err = match_token(str, au_wbr_create_policy, args);
+	create->wbr_create = err;
+	switch (err) {
+	case AuWbrCreate_MFSRRV:
+	case AuWbrCreate_TDMFSV:
+	case AuWbrCreate_PMFSRRV:
+		e = au_wbr_mfs_wmark(&args[0], str, create);
+		if (!e)
+			e = au_wbr_mfs_sec(&args[1], str, create);
+		if (unlikely(e))
+			err = e;
+		break;
+	case AuWbrCreate_MFSRR:
+	case AuWbrCreate_TDMFS:
+	case AuWbrCreate_PMFSRR:
+		e = au_wbr_mfs_wmark(&args[0], str, create);
+		if (unlikely(e)) {
+			err = e;
+			break;
+		}
+		/*FALLTHROUGH*/
+	case AuWbrCreate_MFS:
+	case AuWbrCreate_PMFS:
+		create->mfs_second = AUFS_MFS_DEF_SEC;
+		break;
+	case AuWbrCreate_MFSV:
+	case AuWbrCreate_PMFSV:
+		e = au_wbr_mfs_sec(&args[0], str, create);
+		if (unlikely(e))
+			err = e;
+		break;
+	}
+
+	return err;
+}
+
+const char *au_optstr_wbr_create(int wbr_create)
+{
+	return au_parser_pattern(wbr_create, au_wbr_create_policy);
+}
+
+static match_table_t au_wbr_copyup_policy = {
+	{AuWbrCopyup_TDP, "tdp"},
+	{AuWbrCopyup_TDP, "top-down-parent"},
+	{AuWbrCopyup_BUP, "bup"},
+	{AuWbrCopyup_BUP, "bottom-up-parent"},
+	{AuWbrCopyup_BU, "bu"},
+	{AuWbrCopyup_BU, "bottom-up"},
+	{-1, NULL}
+};
+
+static int noinline_for_stack au_wbr_copyup_val(char *str)
+{
+	substring_t args[MAX_OPT_ARGS];
+
+	return match_token(str, au_wbr_copyup_policy, args);
+}
+
+const char *au_optstr_wbr_copyup(int wbr_copyup)
+{
+	return au_parser_pattern(wbr_copyup, au_wbr_copyup_policy);
+}
+
+/* ---------------------------------------------------------------------- */
+
+static const int lkup_dirflags = LOOKUP_FOLLOW | LOOKUP_DIRECTORY;
+
+static void dump_opts(struct au_opts *opts)
+{
+#ifdef CONFIG_AUFS_DEBUG
+	/* reduce stack space */
+	union {
+		struct au_opt_add *add;
+		struct au_opt_del *del;
+		struct au_opt_mod *mod;
+		struct au_opt_xino *xino;
+		struct au_opt_xino_itrunc *xino_itrunc;
+		struct au_opt_wbr_create *create;
+	} u;
+	struct au_opt *opt;
+
+	opt = opts->opt;
+	while (opt->type != Opt_tail) {
+		switch (opt->type) {
+		case Opt_add:
+			u.add = &opt->add;
+			AuDbg("add {b%d, %s, 0x%x, %p}\n",
+				  u.add->bindex, u.add->pathname, u.add->perm,
+				  u.add->path.dentry);
+			break;
+		case Opt_del:
+		case Opt_idel:
+			u.del = &opt->del;
+			AuDbg("del {%s, %p}\n",
+			      u.del->pathname, u.del->h_path.dentry);
+			break;
+		case Opt_mod:
+		case Opt_imod:
+			u.mod = &opt->mod;
+			AuDbg("mod {%s, 0x%x, %p}\n",
+				  u.mod->path, u.mod->perm, u.mod->h_root);
+			break;
+		case Opt_append:
+			u.add = &opt->add;
+			AuDbg("append {b%d, %s, 0x%x, %p}\n",
+				  u.add->bindex, u.add->pathname, u.add->perm,
+				  u.add->path.dentry);
+			break;
+		case Opt_prepend:
+			u.add = &opt->add;
+			AuDbg("prepend {b%d, %s, 0x%x, %p}\n",
+				  u.add->bindex, u.add->pathname, u.add->perm,
+				  u.add->path.dentry);
+			break;
+		case Opt_dirwh:
+			AuDbg("dirwh %d\n", opt->dirwh);
+			break;
+		case Opt_rdcache:
+			AuDbg("rdcache %d\n", opt->rdcache);
+			break;
+		case Opt_rdblk:
+			AuDbg("rdblk %u\n", opt->rdblk);
+			break;
+		case Opt_rdblk_def:
+			AuDbg("rdblk_def\n");
+			break;
+		case Opt_rdhash:
+			AuDbg("rdhash %u\n", opt->rdhash);
+			break;
+		case Opt_rdhash_def:
+			AuDbg("rdhash_def\n");
+			break;
+		case Opt_xino:
+			u.xino = &opt->xino;
+			AuDbg("xino {%s %pD}\n", u.xino->path, u.xino->file);
+			break;
+		case Opt_trunc_xino:
+			AuLabel(trunc_xino);
+			break;
+		case Opt_notrunc_xino:
+			AuLabel(notrunc_xino);
+			break;
+		case Opt_trunc_xino_path:
+		case Opt_itrunc_xino:
+			u.xino_itrunc = &opt->xino_itrunc;
+			AuDbg("trunc_xino %d\n", u.xino_itrunc->bindex);
+			break;
+		case Opt_noxino:
+			AuLabel(noxino);
+			break;
+		case Opt_trunc_xib:
+			AuLabel(trunc_xib);
+			break;
+		case Opt_notrunc_xib:
+			AuLabel(notrunc_xib);
+			break;
+		case Opt_shwh:
+			AuLabel(shwh);
+			break;
+		case Opt_noshwh:
+			AuLabel(noshwh);
+			break;
+		case Opt_dirperm1:
+			AuLabel(dirperm1);
+			break;
+		case Opt_nodirperm1:
+			AuLabel(nodirperm1);
+			break;
+		case Opt_plink:
+			AuLabel(plink);
+			break;
+		case Opt_noplink:
+			AuLabel(noplink);
+			break;
+		case Opt_list_plink:
+			AuLabel(list_plink);
+			break;
+		case Opt_udba:
+			AuDbg("udba %d, %s\n",
+				  opt->udba, au_optstr_udba(opt->udba));
+			break;
+		case Opt_dio:
+			AuLabel(dio);
+			break;
+		case Opt_nodio:
+			AuLabel(nodio);
+			break;
+		case Opt_diropq_a:
+			AuLabel(diropq_a);
+			break;
+		case Opt_diropq_w:
+			AuLabel(diropq_w);
+			break;
+		case Opt_warn_perm:
+			AuLabel(warn_perm);
+			break;
+		case Opt_nowarn_perm:
+			AuLabel(nowarn_perm);
+			break;
+		case Opt_verbose:
+			AuLabel(verbose);
+			break;
+		case Opt_noverbose:
+			AuLabel(noverbose);
+			break;
+		case Opt_sum:
+			AuLabel(sum);
+			break;
+		case Opt_nosum:
+			AuLabel(nosum);
+			break;
+		case Opt_wsum:
+			AuLabel(wsum);
+			break;
+		case Opt_wbr_create:
+			u.create = &opt->wbr_create;
+			AuDbg("create %d, %s\n", u.create->wbr_create,
+				  au_optstr_wbr_create(u.create->wbr_create));
+			switch (u.create->wbr_create) {
+			case AuWbrCreate_MFSV:
+			case AuWbrCreate_PMFSV:
+				AuDbg("%d sec\n", u.create->mfs_second);
+				break;
+			case AuWbrCreate_MFSRR:
+			case AuWbrCreate_TDMFS:
+				AuDbg("%llu watermark\n",
+					  u.create->mfsrr_watermark);
+				break;
+			case AuWbrCreate_MFSRRV:
+			case AuWbrCreate_TDMFSV:
+			case AuWbrCreate_PMFSRRV:
+				AuDbg("%llu watermark, %d sec\n",
+					  u.create->mfsrr_watermark,
+					  u.create->mfs_second);
+				break;
+			}
+			break;
+		case Opt_wbr_copyup:
+			AuDbg("copyup %d, %s\n", opt->wbr_copyup,
+				  au_optstr_wbr_copyup(opt->wbr_copyup));
+			break;
+		case Opt_fhsm_sec:
+			AuDbg("fhsm_sec %u\n", opt->fhsm_second);
+			break;
+		case Opt_dirren:
+			AuLabel(dirren);
+			break;
+		case Opt_nodirren:
+			AuLabel(nodirren);
+			break;
+		case Opt_acl:
+			AuLabel(acl);
+			break;
+		case Opt_noacl:
+			AuLabel(noacl);
+			break;
+		default:
+			BUG();
+		}
+		opt++;
+	}
+#endif
+}
+
+void au_opts_free(struct au_opts *opts)
+{
+	struct au_opt *opt;
+
+	opt = opts->opt;
+	while (opt->type != Opt_tail) {
+		switch (opt->type) {
+		case Opt_add:
+		case Opt_append:
+		case Opt_prepend:
+			path_put(&opt->add.path);
+			break;
+		case Opt_del:
+		case Opt_idel:
+			path_put(&opt->del.h_path);
+			break;
+		case Opt_mod:
+		case Opt_imod:
+			dput(opt->mod.h_root);
+			break;
+		case Opt_xino:
+			fput(opt->xino.file);
+			break;
+		}
+		opt++;
+	}
+}
+
+static int opt_add(struct au_opt *opt, char *opt_str, unsigned long sb_flags,
+		   aufs_bindex_t bindex)
+{
+	int err;
+	struct au_opt_add *add = &opt->add;
+	char *p;
+
+	add->bindex = bindex;
+	add->perm = AuBrPerm_RO;
+	add->pathname = opt_str;
+	p = strchr(opt_str, '=');
+	if (p) {
+		*p++ = 0;
+		if (*p)
+			add->perm = br_perm_val(p);
+	}
+
+	err = vfsub_kern_path(add->pathname, lkup_dirflags, &add->path);
+	if (!err) {
+		if (!p) {
+			add->perm = AuBrPerm_RO;
+			if (au_test_fs_rr(add->path.dentry->d_sb))
+				add->perm = AuBrPerm_RR;
+			else if (!bindex && !(sb_flags & SB_RDONLY))
+				add->perm = AuBrPerm_RW;
+		}
+		opt->type = Opt_add;
+		goto out;
+	}
+	pr_err("lookup failed %s (%d)\n", add->pathname, err);
+	err = -EINVAL;
+
+out:
+	return err;
+}
+
+static int au_opts_parse_del(struct au_opt_del *del, substring_t args[])
+{
+	int err;
+
+	del->pathname = args[0].from;
+	AuDbg("del path %s\n", del->pathname);
+
+	err = vfsub_kern_path(del->pathname, lkup_dirflags, &del->h_path);
+	if (unlikely(err))
+		pr_err("lookup failed %s (%d)\n", del->pathname, err);
+
+	return err;
+}
+
+#if 0 /* reserved for future use */
+static int au_opts_parse_idel(struct super_block *sb, aufs_bindex_t bindex,
+			      struct au_opt_del *del, substring_t args[])
+{
+	int err;
+	struct dentry *root;
+
+	err = -EINVAL;
+	root = sb->s_root;
+	aufs_read_lock(root, AuLock_FLUSH);
+	if (bindex < 0 || au_sbbot(sb) < bindex) {
+		pr_err("out of bounds, %d\n", bindex);
+		goto out;
+	}
+
+	err = 0;
+	del->h_path.dentry = dget(au_h_dptr(root, bindex));
+	del->h_path.mnt = mntget(au_sbr_mnt(sb, bindex));
+
+out:
+	aufs_read_unlock(root, !AuLock_IR);
+	return err;
+}
+#endif
+
+static int noinline_for_stack
+au_opts_parse_mod(struct au_opt_mod *mod, substring_t args[])
+{
+	int err;
+	struct path path;
+	char *p;
+
+	err = -EINVAL;
+	mod->path = args[0].from;
+	p = strchr(mod->path, '=');
+	if (unlikely(!p)) {
+		pr_err("no permission %s\n", args[0].from);
+		goto out;
+	}
+
+	*p++ = 0;
+	err = vfsub_kern_path(mod->path, lkup_dirflags, &path);
+	if (unlikely(err)) {
+		pr_err("lookup failed %s (%d)\n", mod->path, err);
+		goto out;
+	}
+
+	mod->perm = br_perm_val(p);
+	AuDbg("mod path %s, perm 0x%x, %s\n", mod->path, mod->perm, p);
+	mod->h_root = dget(path.dentry);
+	path_put(&path);
+
+out:
+	return err;
+}
+
+#if 0 /* reserved for future use */
+static int au_opts_parse_imod(struct super_block *sb, aufs_bindex_t bindex,
+			      struct au_opt_mod *mod, substring_t args[])
+{
+	int err;
+	struct dentry *root;
+
+	err = -EINVAL;
+	root = sb->s_root;
+	aufs_read_lock(root, AuLock_FLUSH);
+	if (bindex < 0 || au_sbbot(sb) < bindex) {
+		pr_err("out of bounds, %d\n", bindex);
+		goto out;
+	}
+
+	err = 0;
+	mod->perm = br_perm_val(args[1].from);
+	AuDbg("mod path %s, perm 0x%x, %s\n",
+	      mod->path, mod->perm, args[1].from);
+	mod->h_root = dget(au_h_dptr(root, bindex));
+
+out:
+	aufs_read_unlock(root, !AuLock_IR);
+	return err;
+}
+#endif
+
+static int au_opts_parse_xino(struct super_block *sb, struct au_opt_xino *xino,
+			      substring_t args[])
+{
+	int err;
+	struct file *file;
+
+	file = au_xino_create(sb, args[0].from, /*silent*/0, /*wbrtop*/0);
+	err = PTR_ERR(file);
+	if (IS_ERR(file))
+		goto out;
+
+	err = -EINVAL;
+	if (unlikely(file->f_path.dentry->d_sb == sb)) {
+		fput(file);
+		pr_err("%s must be outside\n", args[0].from);
+		goto out;
+	}
+
+	err = 0;
+	xino->file = file;
+	xino->path = args[0].from;
+
+out:
+	return err;
+}
+
+static int noinline_for_stack
+au_opts_parse_xino_itrunc_path(struct super_block *sb,
+			       struct au_opt_xino_itrunc *xino_itrunc,
+			       substring_t args[])
+{
+	int err;
+	aufs_bindex_t bbot, bindex;
+	struct path path;
+	struct dentry *root;
+
+	err = vfsub_kern_path(args[0].from, lkup_dirflags, &path);
+	if (unlikely(err)) {
+		pr_err("lookup failed %s (%d)\n", args[0].from, err);
+		goto out;
+	}
+
+	xino_itrunc->bindex = -1;
+	root = sb->s_root;
+	aufs_read_lock(root, AuLock_FLUSH);
+	bbot = au_sbbot(sb);
+	for (bindex = 0; bindex <= bbot; bindex++) {
+		if (au_h_dptr(root, bindex) == path.dentry) {
+			xino_itrunc->bindex = bindex;
+			break;
+		}
+	}
+	aufs_read_unlock(root, !AuLock_IR);
+	path_put(&path);
+
+	if (unlikely(xino_itrunc->bindex < 0)) {
+		pr_err("no such branch %s\n", args[0].from);
+		err = -EINVAL;
+	}
+
+out:
+	return err;
+}
+
+/* called without aufs lock */
+int au_opts_parse(struct super_block *sb, char *str, struct au_opts *opts)
+{
+	int err, n, token;
+	aufs_bindex_t bindex;
+	unsigned char skipped;
+	struct dentry *root;
+	struct au_opt *opt, *opt_tail;
+	char *opt_str;
+	/* reduce the stack space */
+	union {
+		struct au_opt_xino_itrunc *xino_itrunc;
+		struct au_opt_wbr_create *create;
+	} u;
+	struct {
+		substring_t args[MAX_OPT_ARGS];
+	} *a;
+
+	err = -ENOMEM;
+	a = kmalloc(sizeof(*a), GFP_NOFS);
+	if (unlikely(!a))
+		goto out;
+
+	root = sb->s_root;
+	err = 0;
+	bindex = 0;
+	opt = opts->opt;
+	opt_tail = opt + opts->max_opt - 1;
+	opt->type = Opt_tail;
+	while (!err && (opt_str = strsep(&str, ",")) && *opt_str) {
+		err = -EINVAL;
+		skipped = 0;
+		token = match_token(opt_str, options, a->args);
+		switch (token) {
+		case Opt_br:
+			err = 0;
+			while (!err && (opt_str = strsep(&a->args[0].from, ":"))
+			       && *opt_str) {
+				err = opt_add(opt, opt_str, opts->sb_flags,
+					      bindex++);
+				if (unlikely(!err && ++opt > opt_tail)) {
+					err = -E2BIG;
+					break;
+				}
+				opt->type = Opt_tail;
+				skipped = 1;
+			}
+			break;
+		case Opt_add:
+			if (unlikely(match_int(&a->args[0], &n))) {
+				pr_err("bad integer in %s\n", opt_str);
+				break;
+			}
+			bindex = n;
+			err = opt_add(opt, a->args[1].from, opts->sb_flags,
+				      bindex);
+			if (!err)
+				opt->type = token;
+			break;
+		case Opt_append:
+			err = opt_add(opt, a->args[0].from, opts->sb_flags,
+				      /*dummy bindex*/1);
+			if (!err)
+				opt->type = token;
+			break;
+		case Opt_prepend:
+			err = opt_add(opt, a->args[0].from, opts->sb_flags,
+				      /*bindex*/0);
+			if (!err)
+				opt->type = token;
+			break;
+		case Opt_del:
+			err = au_opts_parse_del(&opt->del, a->args);
+			if (!err)
+				opt->type = token;
+			break;
+#if 0 /* reserved for future use */
+		case Opt_idel:
+			del->pathname = "(indexed)";
+			if (unlikely(match_int(&args[0], &n))) {
+				pr_err("bad integer in %s\n", opt_str);
+				break;
+			}
+			err = au_opts_parse_idel(sb, n, &opt->del, a->args);
+			if (!err)
+				opt->type = token;
+			break;
+#endif
+		case Opt_mod:
+			err = au_opts_parse_mod(&opt->mod, a->args);
+			if (!err)
+				opt->type = token;
+			break;
+#ifdef IMOD /* reserved for future use */
+		case Opt_imod:
+			u.mod->path = "(indexed)";
+			if (unlikely(match_int(&a->args[0], &n))) {
+				pr_err("bad integer in %s\n", opt_str);
+				break;
+			}
+			err = au_opts_parse_imod(sb, n, &opt->mod, a->args);
+			if (!err)
+				opt->type = token;
+			break;
+#endif
+		case Opt_xino:
+			err = au_opts_parse_xino(sb, &opt->xino, a->args);
+			if (!err)
+				opt->type = token;
+			break;
+
+		case Opt_trunc_xino_path:
+			err = au_opts_parse_xino_itrunc_path
+				(sb, &opt->xino_itrunc, a->args);
+			if (!err)
+				opt->type = token;
+			break;
+
+		case Opt_itrunc_xino:
+			u.xino_itrunc = &opt->xino_itrunc;
+			if (unlikely(match_int(&a->args[0], &n))) {
+				pr_err("bad integer in %s\n", opt_str);
+				break;
+			}
+			u.xino_itrunc->bindex = n;
+			aufs_read_lock(root, AuLock_FLUSH);
+			if (n < 0 || au_sbbot(sb) < n) {
+				pr_err("out of bounds, %d\n", n);
+				aufs_read_unlock(root, !AuLock_IR);
+				break;
+			}
+			aufs_read_unlock(root, !AuLock_IR);
+			err = 0;
+			opt->type = token;
+			break;
+
+		case Opt_dirwh:
+			if (unlikely(match_int(&a->args[0], &opt->dirwh)))
+				break;
+			err = 0;
+			opt->type = token;
+			break;
+
+		case Opt_rdcache:
+			if (unlikely(match_int(&a->args[0], &n))) {
+				pr_err("bad integer in %s\n", opt_str);
+				break;
+			}
+			if (unlikely(n > AUFS_RDCACHE_MAX)) {
+				pr_err("rdcache must be smaller than %d\n",
+				       AUFS_RDCACHE_MAX);
+				break;
+			}
+			opt->rdcache = n;
+			err = 0;
+			opt->type = token;
+			break;
+		case Opt_rdblk:
+			if (unlikely(match_int(&a->args[0], &n)
+				     || n < 0
+				     || n > KMALLOC_MAX_SIZE)) {
+				pr_err("bad integer in %s\n", opt_str);
+				break;
+			}
+			if (unlikely(n && n < NAME_MAX)) {
+				pr_err("rdblk must be larger than %d\n",
+				       NAME_MAX);
+				break;
+			}
+			opt->rdblk = n;
+			err = 0;
+			opt->type = token;
+			break;
+		case Opt_rdhash:
+			if (unlikely(match_int(&a->args[0], &n)
+				     || n < 0
+				     || n * sizeof(struct hlist_head)
+				     > KMALLOC_MAX_SIZE)) {
+				pr_err("bad integer in %s\n", opt_str);
+				break;
+			}
+			opt->rdhash = n;
+			err = 0;
+			opt->type = token;
+			break;
+
+		case Opt_trunc_xino:
+		case Opt_notrunc_xino:
+		case Opt_noxino:
+		case Opt_trunc_xib:
+		case Opt_notrunc_xib:
+		case Opt_shwh:
+		case Opt_noshwh:
+		case Opt_dirperm1:
+		case Opt_nodirperm1:
+		case Opt_plink:
+		case Opt_noplink:
+		case Opt_list_plink:
+		case Opt_dio:
+		case Opt_nodio:
+		case Opt_diropq_a:
+		case Opt_diropq_w:
+		case Opt_warn_perm:
+		case Opt_nowarn_perm:
+		case Opt_verbose:
+		case Opt_noverbose:
+		case Opt_sum:
+		case Opt_nosum:
+		case Opt_wsum:
+		case Opt_rdblk_def:
+		case Opt_rdhash_def:
+		case Opt_dirren:
+		case Opt_nodirren:
+		case Opt_acl:
+		case Opt_noacl:
+			err = 0;
+			opt->type = token;
+			break;
+
+		case Opt_udba:
+			opt->udba = udba_val(a->args[0].from);
+			if (opt->udba >= 0) {
+				err = 0;
+				opt->type = token;
+			} else
+				pr_err("wrong value, %s\n", opt_str);
+			break;
+
+		case Opt_wbr_create:
+			u.create = &opt->wbr_create;
+			u.create->wbr_create
+				= au_wbr_create_val(a->args[0].from, u.create);
+			if (u.create->wbr_create >= 0) {
+				err = 0;
+				opt->type = token;
+			} else
+				pr_err("wrong value, %s\n", opt_str);
+			break;
+		case Opt_wbr_copyup:
+			opt->wbr_copyup = au_wbr_copyup_val(a->args[0].from);
+			if (opt->wbr_copyup >= 0) {
+				err = 0;
+				opt->type = token;
+			} else
+				pr_err("wrong value, %s\n", opt_str);
+			break;
+
+		case Opt_fhsm_sec:
+			if (unlikely(match_int(&a->args[0], &n)
+				     || n < 0)) {
+				pr_err("bad integer in %s\n", opt_str);
+				break;
+			}
+			if (sysaufs_brs) {
+				opt->fhsm_second = n;
+				opt->type = token;
+			} else
+				pr_warn("ignored %s\n", opt_str);
+			err = 0;
+			break;
+
+		case Opt_ignore:
+			pr_warn("ignored %s\n", opt_str);
+			/*FALLTHROUGH*/
+		case Opt_ignore_silent:
+			skipped = 1;
+			err = 0;
+			break;
+		case Opt_err:
+			pr_err("unknown option %s\n", opt_str);
+			break;
+		}
+
+		if (!err && !skipped) {
+			if (unlikely(++opt > opt_tail)) {
+				err = -E2BIG;
+				opt--;
+				opt->type = Opt_tail;
+				break;
+			}
+			opt->type = Opt_tail;
+		}
+	}
+
+	au_kfree_rcu(a);
+	dump_opts(opts);
+	if (unlikely(err))
+		au_opts_free(opts);
+
+out:
+	return err;
+}
+
+static int au_opt_wbr_create(struct super_block *sb,
+			     struct au_opt_wbr_create *create)
+{
+	int err;
+	struct au_sbinfo *sbinfo;
+
+	SiMustWriteLock(sb);
+
+	err = 1; /* handled */
+	sbinfo = au_sbi(sb);
+	if (sbinfo->si_wbr_create_ops->fin) {
+		err = sbinfo->si_wbr_create_ops->fin(sb);
+		if (!err)
+			err = 1;
+	}
+
+	sbinfo->si_wbr_create = create->wbr_create;
+	sbinfo->si_wbr_create_ops = au_wbr_create_ops + create->wbr_create;
+	switch (create->wbr_create) {
+	case AuWbrCreate_MFSRRV:
+	case AuWbrCreate_MFSRR:
+	case AuWbrCreate_TDMFS:
+	case AuWbrCreate_TDMFSV:
+	case AuWbrCreate_PMFSRR:
+	case AuWbrCreate_PMFSRRV:
+		sbinfo->si_wbr_mfs.mfsrr_watermark = create->mfsrr_watermark;
+		/*FALLTHROUGH*/
+	case AuWbrCreate_MFS:
+	case AuWbrCreate_MFSV:
+	case AuWbrCreate_PMFS:
+	case AuWbrCreate_PMFSV:
+		sbinfo->si_wbr_mfs.mfs_expire
+			= msecs_to_jiffies(create->mfs_second * MSEC_PER_SEC);
+		break;
+	}
+
+	if (sbinfo->si_wbr_create_ops->init)
+		sbinfo->si_wbr_create_ops->init(sb); /* ignore */
+
+	return err;
+}
+
+/*
+ * returns,
+ * plus: processed without an error
+ * zero: unprocessed
+ */
+static int au_opt_simple(struct super_block *sb, struct au_opt *opt,
+			 struct au_opts *opts)
+{
+	int err;
+	struct au_sbinfo *sbinfo;
+
+	SiMustWriteLock(sb);
+
+	err = 1; /* handled */
+	sbinfo = au_sbi(sb);
+	switch (opt->type) {
+	case Opt_udba:
+		sbinfo->si_mntflags &= ~AuOptMask_UDBA;
+		sbinfo->si_mntflags |= opt->udba;
+		opts->given_udba |= opt->udba;
+		break;
+
+	case Opt_plink:
+		au_opt_set(sbinfo->si_mntflags, PLINK);
+		break;
+	case Opt_noplink:
+		if (au_opt_test(sbinfo->si_mntflags, PLINK))
+			au_plink_put(sb, /*verbose*/1);
+		au_opt_clr(sbinfo->si_mntflags, PLINK);
+		break;
+	case Opt_list_plink:
+		if (au_opt_test(sbinfo->si_mntflags, PLINK))
+			au_plink_list(sb);
+		break;
+
+	case Opt_dio:
+		au_opt_set(sbinfo->si_mntflags, DIO);
+		au_fset_opts(opts->flags, REFRESH_DYAOP);
+		break;
+	case Opt_nodio:
+		au_opt_clr(sbinfo->si_mntflags, DIO);
+		au_fset_opts(opts->flags, REFRESH_DYAOP);
+		break;
+
+	case Opt_fhsm_sec:
+		au_fhsm_set(sbinfo, opt->fhsm_second);
+		break;
+
+	case Opt_diropq_a:
+		au_opt_set(sbinfo->si_mntflags, ALWAYS_DIROPQ);
+		break;
+	case Opt_diropq_w:
+		au_opt_clr(sbinfo->si_mntflags, ALWAYS_DIROPQ);
+		break;
+
+	case Opt_warn_perm:
+		au_opt_set(sbinfo->si_mntflags, WARN_PERM);
+		break;
+	case Opt_nowarn_perm:
+		au_opt_clr(sbinfo->si_mntflags, WARN_PERM);
+		break;
+
+	case Opt_verbose:
+		au_opt_set(sbinfo->si_mntflags, VERBOSE);
+		break;
+	case Opt_noverbose:
+		au_opt_clr(sbinfo->si_mntflags, VERBOSE);
+		break;
+
+	case Opt_sum:
+		au_opt_set(sbinfo->si_mntflags, SUM);
+		break;
+	case Opt_wsum:
+		au_opt_clr(sbinfo->si_mntflags, SUM);
+		au_opt_set(sbinfo->si_mntflags, SUM_W);
+		break;
+	case Opt_nosum:
+		au_opt_clr(sbinfo->si_mntflags, SUM);
+		au_opt_clr(sbinfo->si_mntflags, SUM_W);
+		break;
+
+	case Opt_wbr_create:
+		err = au_opt_wbr_create(sb, &opt->wbr_create);
+		break;
+	case Opt_wbr_copyup:
+		sbinfo->si_wbr_copyup = opt->wbr_copyup;
+		sbinfo->si_wbr_copyup_ops = au_wbr_copyup_ops + opt->wbr_copyup;
+		break;
+
+	case Opt_dirwh:
+		sbinfo->si_dirwh = opt->dirwh;
+		break;
+
+	case Opt_rdcache:
+		sbinfo->si_rdcache
+			= msecs_to_jiffies(opt->rdcache * MSEC_PER_SEC);
+		break;
+	case Opt_rdblk:
+		sbinfo->si_rdblk = opt->rdblk;
+		break;
+	case Opt_rdblk_def:
+		sbinfo->si_rdblk = AUFS_RDBLK_DEF;
+		break;
+	case Opt_rdhash:
+		sbinfo->si_rdhash = opt->rdhash;
+		break;
+	case Opt_rdhash_def:
+		sbinfo->si_rdhash = AUFS_RDHASH_DEF;
+		break;
+
+	case Opt_shwh:
+		au_opt_set(sbinfo->si_mntflags, SHWH);
+		break;
+	case Opt_noshwh:
+		au_opt_clr(sbinfo->si_mntflags, SHWH);
+		break;
+
+	case Opt_dirperm1:
+		au_opt_set(sbinfo->si_mntflags, DIRPERM1);
+		break;
+	case Opt_nodirperm1:
+		au_opt_clr(sbinfo->si_mntflags, DIRPERM1);
+		break;
+
+	case Opt_trunc_xino:
+		au_opt_set(sbinfo->si_mntflags, TRUNC_XINO);
+		break;
+	case Opt_notrunc_xino:
+		au_opt_clr(sbinfo->si_mntflags, TRUNC_XINO);
+		break;
+
+	case Opt_trunc_xino_path:
+	case Opt_itrunc_xino:
+		err = au_xino_trunc(sb, opt->xino_itrunc.bindex,
+				    /*idx_begin*/0);
+		if (!err)
+			err = 1;
+		break;
+
+	case Opt_trunc_xib:
+		au_fset_opts(opts->flags, TRUNC_XIB);
+		break;
+	case Opt_notrunc_xib:
+		au_fclr_opts(opts->flags, TRUNC_XIB);
+		break;
+
+	case Opt_dirren:
+		err = 1;
+		if (!au_opt_test(sbinfo->si_mntflags, DIRREN)) {
+			err = au_dr_opt_set(sb);
+			if (!err)
+				err = 1;
+		}
+		if (err == 1)
+			au_opt_set(sbinfo->si_mntflags, DIRREN);
+		break;
+	case Opt_nodirren:
+		err = 1;
+		if (au_opt_test(sbinfo->si_mntflags, DIRREN)) {
+			err = au_dr_opt_clr(sb, au_ftest_opts(opts->flags,
+							      DR_FLUSHED));
+			if (!err)
+				err = 1;
+		}
+		if (err == 1)
+			au_opt_clr(sbinfo->si_mntflags, DIRREN);
+		break;
+
+	case Opt_acl:
+		sb->s_flags |= SB_POSIXACL;
+		break;
+	case Opt_noacl:
+		sb->s_flags &= ~SB_POSIXACL;
+		break;
+
+	default:
+		err = 0;
+		break;
+	}
+
+	return err;
+}
+
+/*
+ * returns tri-state.
+ * plus: processed without an error
+ * zero: unprocessed
+ * minus: error
+ */
+static int au_opt_br(struct super_block *sb, struct au_opt *opt,
+		     struct au_opts *opts)
+{
+	int err, do_refresh;
+
+	err = 0;
+	switch (opt->type) {
+	case Opt_append:
+		opt->add.bindex = au_sbbot(sb) + 1;
+		if (opt->add.bindex < 0)
+			opt->add.bindex = 0;
+		goto add;
+		/* Always goto add, not fallthrough */
+	case Opt_prepend:
+		opt->add.bindex = 0;
+		/* fallthrough */
+	add: /* indented label */
+	case Opt_add:
+		err = au_br_add(sb, &opt->add,
+				au_ftest_opts(opts->flags, REMOUNT));
+		if (!err) {
+			err = 1;
+			au_fset_opts(opts->flags, REFRESH);
+		}
+		break;
+
+	case Opt_del:
+	case Opt_idel:
+		err = au_br_del(sb, &opt->del,
+				au_ftest_opts(opts->flags, REMOUNT));
+		if (!err) {
+			err = 1;
+			au_fset_opts(opts->flags, TRUNC_XIB);
+			au_fset_opts(opts->flags, REFRESH);
+		}
+		break;
+
+	case Opt_mod:
+	case Opt_imod:
+		err = au_br_mod(sb, &opt->mod,
+				au_ftest_opts(opts->flags, REMOUNT),
+				&do_refresh);
+		if (!err) {
+			err = 1;
+			if (do_refresh)
+				au_fset_opts(opts->flags, REFRESH);
+		}
+		break;
+	}
+	return err;
+}
+
+static int au_opt_xino(struct super_block *sb, struct au_opt *opt,
+		       struct au_opt_xino **opt_xino,
+		       struct au_opts *opts)
+{
+	int err;
+
+	err = 0;
+	switch (opt->type) {
+	case Opt_xino:
+		err = au_xino_set(sb, &opt->xino,
+				  !!au_ftest_opts(opts->flags, REMOUNT));
+		if (unlikely(err))
+			break;
+
+		*opt_xino = &opt->xino;
+		break;
+
+	case Opt_noxino:
+		au_xino_clr(sb);
+		*opt_xino = (void *)-1;
+		break;
+	}
+
+	return err;
+}
+
+int au_opts_verify(struct super_block *sb, unsigned long sb_flags,
+		   unsigned int pending)
+{
+	int err, fhsm;
+	aufs_bindex_t bindex, bbot;
+	unsigned char do_plink, skip, do_free, can_no_dreval;
+	struct au_branch *br;
+	struct au_wbr *wbr;
+	struct dentry *root, *dentry;
+	struct inode *dir, *h_dir;
+	struct au_sbinfo *sbinfo;
+	struct au_hinode *hdir;
+
+	SiMustAnyLock(sb);
+
+	sbinfo = au_sbi(sb);
+	AuDebugOn(!(sbinfo->si_mntflags & AuOptMask_UDBA));
+
+	if (!(sb_flags & SB_RDONLY)) {
+		if (unlikely(!au_br_writable(au_sbr_perm(sb, 0))))
+			pr_warn("first branch should be rw\n");
+		if (unlikely(au_opt_test(sbinfo->si_mntflags, SHWH)))
+			pr_warn_once("shwh should be used with ro\n");
+	}
+
+	if (au_opt_test((sbinfo->si_mntflags | pending), UDBA_HNOTIFY)
+	    && !au_opt_test(sbinfo->si_mntflags, XINO))
+		pr_warn_once("udba=*notify requires xino\n");
+
+	if (au_opt_test(sbinfo->si_mntflags, DIRPERM1))
+		pr_warn_once("dirperm1 breaks the protection"
+			     " by the permission bits on the lower branch\n");
+
+	err = 0;
+	fhsm = 0;
+	root = sb->s_root;
+	dir = d_inode(root);
+	do_plink = !!au_opt_test(sbinfo->si_mntflags, PLINK);
+	can_no_dreval = !!au_opt_test((sbinfo->si_mntflags | pending),
+				      UDBA_NONE);
+	bbot = au_sbbot(sb);
+	for (bindex = 0; !err && bindex <= bbot; bindex++) {
+		skip = 0;
+		h_dir = au_h_iptr(dir, bindex);
+		br = au_sbr(sb, bindex);
+
+		if ((br->br_perm & AuBrAttr_ICEX)
+		    && !h_dir->i_op->listxattr)
+			br->br_perm &= ~AuBrAttr_ICEX;
+#if 0 /* untested */
+		if ((br->br_perm & AuBrAttr_ICEX_SEC)
+		    && (au_br_sb(br)->s_flags & SB_NOSEC))
+			br->br_perm &= ~AuBrAttr_ICEX_SEC;
+#endif
+
+		do_free = 0;
+		wbr = br->br_wbr;
+		if (wbr)
+			wbr_wh_read_lock(wbr);
+
+		if (!au_br_writable(br->br_perm)) {
+			do_free = !!wbr;
+			skip = (!wbr
+				|| (!wbr->wbr_whbase
+				    && !wbr->wbr_plink
+				    && !wbr->wbr_orph));
+		} else if (!au_br_wh_linkable(br->br_perm)) {
+			/* skip = (!br->br_whbase && !br->br_orph); */
+			skip = (!wbr || !wbr->wbr_whbase);
+			if (skip && wbr) {
+				if (do_plink)
+					skip = !!wbr->wbr_plink;
+				else
+					skip = !wbr->wbr_plink;
+			}
+		} else {
+			/* skip = (br->br_whbase && br->br_ohph); */
+			skip = (wbr && wbr->wbr_whbase);
+			if (skip) {
+				if (do_plink)
+					skip = !!wbr->wbr_plink;
+				else
+					skip = !wbr->wbr_plink;
+			}
+		}
+		if (wbr)
+			wbr_wh_read_unlock(wbr);
+
+		if (can_no_dreval) {
+			dentry = br->br_path.dentry;
+			spin_lock(&dentry->d_lock);
+			if (dentry->d_flags &
+			    (DCACHE_OP_REVALIDATE | DCACHE_OP_WEAK_REVALIDATE))
+				can_no_dreval = 0;
+			spin_unlock(&dentry->d_lock);
+		}
+
+		if (au_br_fhsm(br->br_perm)) {
+			fhsm++;
+			AuDebugOn(!br->br_fhsm);
+		}
+
+		if (skip)
+			continue;
+
+		hdir = au_hi(dir, bindex);
+		au_hn_inode_lock_nested(hdir, AuLsc_I_PARENT);
+		if (wbr)
+			wbr_wh_write_lock(wbr);
+		err = au_wh_init(br, sb);
+		if (wbr)
+			wbr_wh_write_unlock(wbr);
+		au_hn_inode_unlock(hdir);
+
+		if (!err && do_free) {
+			au_kfree_rcu(wbr);
+			br->br_wbr = NULL;
+		}
+	}
+
+	if (can_no_dreval)
+		au_fset_si(sbinfo, NO_DREVAL);
+	else
+		au_fclr_si(sbinfo, NO_DREVAL);
+
+	if (fhsm >= 2) {
+		au_fset_si(sbinfo, FHSM);
+		for (bindex = bbot; bindex >= 0; bindex--) {
+			br = au_sbr(sb, bindex);
+			if (au_br_fhsm(br->br_perm)) {
+				au_fhsm_set_bottom(sb, bindex);
+				break;
+			}
+		}
+	} else {
+		au_fclr_si(sbinfo, FHSM);
+		au_fhsm_set_bottom(sb, -1);
+	}
+
+	return err;
+}
+
+int au_opts_mount(struct super_block *sb, struct au_opts *opts)
+{
+	int err;
+	unsigned int tmp;
+	aufs_bindex_t bindex, bbot;
+	struct au_opt *opt;
+	struct au_opt_xino *opt_xino, xino;
+	struct au_sbinfo *sbinfo;
+	struct au_branch *br;
+	struct inode *dir;
+
+	SiMustWriteLock(sb);
+
+	err = 0;
+	opt_xino = NULL;
+	opt = opts->opt;
+	while (err >= 0 && opt->type != Opt_tail)
+		err = au_opt_simple(sb, opt++, opts);
+	if (err > 0)
+		err = 0;
+	else if (unlikely(err < 0))
+		goto out;
+
+	/* disable xino and udba temporary */
+	sbinfo = au_sbi(sb);
+	tmp = sbinfo->si_mntflags;
+	au_opt_clr(sbinfo->si_mntflags, XINO);
+	au_opt_set_udba(sbinfo->si_mntflags, UDBA_REVAL);
+
+	opt = opts->opt;
+	while (err >= 0 && opt->type != Opt_tail)
+		err = au_opt_br(sb, opt++, opts);
+	if (err > 0)
+		err = 0;
+	else if (unlikely(err < 0))
+		goto out;
+
+	bbot = au_sbbot(sb);
+	if (unlikely(bbot < 0)) {
+		err = -EINVAL;
+		pr_err("no branches\n");
+		goto out;
+	}
+
+	if (au_opt_test(tmp, XINO))
+		au_opt_set(sbinfo->si_mntflags, XINO);
+	opt = opts->opt;
+	while (!err && opt->type != Opt_tail)
+		err = au_opt_xino(sb, opt++, &opt_xino, opts);
+	if (unlikely(err))
+		goto out;
+
+	err = au_opts_verify(sb, sb->s_flags, tmp);
+	if (unlikely(err))
+		goto out;
+
+	/* restore xino */
+	if (au_opt_test(tmp, XINO) && !opt_xino) {
+		xino.file = au_xino_def(sb);
+		err = PTR_ERR(xino.file);
+		if (IS_ERR(xino.file))
+			goto out;
+
+		err = au_xino_set(sb, &xino, /*remount*/0);
+		fput(xino.file);
+		if (unlikely(err))
+			goto out;
+	}
+
+	/* restore udba */
+	tmp &= AuOptMask_UDBA;
+	sbinfo->si_mntflags &= ~AuOptMask_UDBA;
+	sbinfo->si_mntflags |= tmp;
+	bbot = au_sbbot(sb);
+	for (bindex = 0; bindex <= bbot; bindex++) {
+		br = au_sbr(sb, bindex);
+		err = au_hnotify_reset_br(tmp, br, br->br_perm);
+		if (unlikely(err))
+			AuIOErr("hnotify failed on br %d, %d, ignored\n",
+				bindex, err);
+		/* go on even if err */
+	}
+	if (au_opt_test(tmp, UDBA_HNOTIFY)) {
+		dir = d_inode(sb->s_root);
+		au_hn_reset(dir, au_hi_flags(dir, /*isdir*/1) & ~AuHi_XINO);
+	}
+
+out:
+	return err;
+}
+
+int au_opts_remount(struct super_block *sb, struct au_opts *opts)
+{
+	int err, rerr;
+	unsigned char no_dreval;
+	struct inode *dir;
+	struct au_opt_xino *opt_xino;
+	struct au_opt *opt;
+	struct au_sbinfo *sbinfo;
+
+	SiMustWriteLock(sb);
+
+	err = au_dr_opt_flush(sb);
+	if (unlikely(err))
+		goto out;
+	au_fset_opts(opts->flags, DR_FLUSHED);
+
+	dir = d_inode(sb->s_root);
+	sbinfo = au_sbi(sb);
+	opt_xino = NULL;
+	opt = opts->opt;
+	while (err >= 0 && opt->type != Opt_tail) {
+		err = au_opt_simple(sb, opt, opts);
+		if (!err)
+			err = au_opt_br(sb, opt, opts);
+		if (!err)
+			err = au_opt_xino(sb, opt, &opt_xino, opts);
+		opt++;
+	}
+	if (err > 0)
+		err = 0;
+	AuTraceErr(err);
+	/* go on even err */
+
+	no_dreval = !!au_ftest_si(sbinfo, NO_DREVAL);
+	rerr = au_opts_verify(sb, opts->sb_flags, /*pending*/0);
+	if (unlikely(rerr && !err))
+		err = rerr;
+
+	if (no_dreval != !!au_ftest_si(sbinfo, NO_DREVAL))
+		au_fset_opts(opts->flags, REFRESH_IDOP);
+
+	if (au_ftest_opts(opts->flags, TRUNC_XIB)) {
+		rerr = au_xib_trunc(sb);
+		if (unlikely(rerr && !err))
+			err = rerr;
+	}
+
+	/* will be handled by the caller */
+	if (!au_ftest_opts(opts->flags, REFRESH)
+	    && (opts->given_udba
+		|| au_opt_test(sbinfo->si_mntflags, XINO)
+		|| au_ftest_opts(opts->flags, REFRESH_IDOP)
+		    ))
+		au_fset_opts(opts->flags, REFRESH);
+
+	AuDbg("status 0x%x\n", opts->flags);
+
+out:
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+unsigned int au_opt_udba(struct super_block *sb)
+{
+	return au_mntflags(sb) & AuOptMask_UDBA;
+}
diff -Naur --no-dereference a/fs/aufs/opts.h b/fs/aufs/opts.h
--- a/fs/aufs/opts.h	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/opts.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,225 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * mount options/flags
+ */
+
+#ifndef __AUFS_OPTS_H__
+#define __AUFS_OPTS_H__
+
+#ifdef __KERNEL__
+
+#include <linux/path.h>
+
+struct file;
+
+/* ---------------------------------------------------------------------- */
+
+/* mount flags */
+#define AuOpt_XINO		1		/* external inode number bitmap
+						   and translation table */
+#define AuOpt_TRUNC_XINO	(1 << 1)	/* truncate xino files */
+#define AuOpt_UDBA_NONE		(1 << 2)	/* users direct branch access */
+#define AuOpt_UDBA_REVAL	(1 << 3)
+#define AuOpt_UDBA_HNOTIFY	(1 << 4)
+#define AuOpt_SHWH		(1 << 5)	/* show whiteout */
+#define AuOpt_PLINK		(1 << 6)	/* pseudo-link */
+#define AuOpt_DIRPERM1		(1 << 7)	/* ignore the lower dir's perm
+						   bits */
+#define AuOpt_ALWAYS_DIROPQ	(1 << 9)	/* policy to creating diropq */
+#define AuOpt_SUM		(1 << 10)	/* summation for statfs(2) */
+#define AuOpt_SUM_W		(1 << 11)	/* unimplemented */
+#define AuOpt_WARN_PERM		(1 << 12)	/* warn when add-branch */
+#define AuOpt_VERBOSE		(1 << 13)	/* print the cause of error */
+#define AuOpt_DIO		(1 << 14)	/* direct io */
+#define AuOpt_DIRREN		(1 << 15)	/* directory rename */
+
+#ifndef CONFIG_AUFS_HNOTIFY
+#undef AuOpt_UDBA_HNOTIFY
+#define AuOpt_UDBA_HNOTIFY	0
+#endif
+#ifndef CONFIG_AUFS_DIRREN
+#undef AuOpt_DIRREN
+#define AuOpt_DIRREN		0
+#endif
+#ifndef CONFIG_AUFS_SHWH
+#undef AuOpt_SHWH
+#define AuOpt_SHWH		0
+#endif
+
+#define AuOpt_Def	(AuOpt_XINO \
+			 | AuOpt_UDBA_REVAL \
+			 | AuOpt_PLINK \
+			 /* | AuOpt_DIRPERM1 */ \
+			 | AuOpt_WARN_PERM)
+#define AuOptMask_UDBA	(AuOpt_UDBA_NONE \
+			 | AuOpt_UDBA_REVAL \
+			 | AuOpt_UDBA_HNOTIFY)
+
+#define au_opt_test(flags, name)	(flags & AuOpt_##name)
+#define au_opt_set(flags, name) do { \
+	BUILD_BUG_ON(AuOpt_##name & AuOptMask_UDBA); \
+	((flags) |= AuOpt_##name); \
+} while (0)
+#define au_opt_set_udba(flags, name) do { \
+	(flags) &= ~AuOptMask_UDBA; \
+	((flags) |= AuOpt_##name); \
+} while (0)
+#define au_opt_clr(flags, name) do { \
+	((flags) &= ~AuOpt_##name); \
+} while (0)
+
+static inline unsigned int au_opts_plink(unsigned int mntflags)
+{
+#ifdef CONFIG_PROC_FS
+	return mntflags;
+#else
+	return mntflags & ~AuOpt_PLINK;
+#endif
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* policies to select one among multiple writable branches */
+enum {
+	AuWbrCreate_TDP,	/* top down parent */
+	AuWbrCreate_RR,		/* round robin */
+	AuWbrCreate_MFS,	/* most free space */
+	AuWbrCreate_MFSV,	/* mfs with seconds */
+	AuWbrCreate_MFSRR,	/* mfs then rr */
+	AuWbrCreate_MFSRRV,	/* mfs then rr with seconds */
+	AuWbrCreate_TDMFS,	/* top down regardless parent and mfs */
+	AuWbrCreate_TDMFSV,	/* top down regardless parent and mfs */
+	AuWbrCreate_PMFS,	/* parent and mfs */
+	AuWbrCreate_PMFSV,	/* parent and mfs with seconds */
+	AuWbrCreate_PMFSRR,	/* parent, mfs and round-robin */
+	AuWbrCreate_PMFSRRV,	/* plus seconds */
+
+	AuWbrCreate_Def = AuWbrCreate_TDP
+};
+
+enum {
+	AuWbrCopyup_TDP,	/* top down parent */
+	AuWbrCopyup_BUP,	/* bottom up parent */
+	AuWbrCopyup_BU,		/* bottom up */
+
+	AuWbrCopyup_Def = AuWbrCopyup_TDP
+};
+
+/* ---------------------------------------------------------------------- */
+
+struct au_opt_add {
+	aufs_bindex_t	bindex;
+	char		*pathname;
+	int		perm;
+	struct path	path;
+};
+
+struct au_opt_del {
+	char		*pathname;
+	struct path	h_path;
+};
+
+struct au_opt_mod {
+	char		*path;
+	int		perm;
+	struct dentry	*h_root;
+};
+
+struct au_opt_xino {
+	char		*path;
+	struct file	*file;
+};
+
+struct au_opt_xino_itrunc {
+	aufs_bindex_t	bindex;
+};
+
+struct au_opt_wbr_create {
+	int			wbr_create;
+	int			mfs_second;
+	unsigned long long	mfsrr_watermark;
+};
+
+struct au_opt {
+	int type;
+	union {
+		struct au_opt_xino	xino;
+		struct au_opt_xino_itrunc xino_itrunc;
+		struct au_opt_add	add;
+		struct au_opt_del	del;
+		struct au_opt_mod	mod;
+		int			dirwh;
+		int			rdcache;
+		unsigned int		rdblk;
+		unsigned int		rdhash;
+		int			udba;
+		struct au_opt_wbr_create wbr_create;
+		int			wbr_copyup;
+		unsigned int		fhsm_second;
+	};
+};
+
+/* opts flags */
+#define AuOpts_REMOUNT		1
+#define AuOpts_REFRESH		(1 << 1)
+#define AuOpts_TRUNC_XIB	(1 << 2)
+#define AuOpts_REFRESH_DYAOP	(1 << 3)
+#define AuOpts_REFRESH_IDOP	(1 << 4)
+#define AuOpts_DR_FLUSHED	(1 << 5)
+#define au_ftest_opts(flags, name)	((flags) & AuOpts_##name)
+#define au_fset_opts(flags, name) \
+	do { (flags) |= AuOpts_##name; } while (0)
+#define au_fclr_opts(flags, name) \
+	do { (flags) &= ~AuOpts_##name; } while (0)
+
+#ifndef CONFIG_AUFS_DIRREN
+#undef AuOpts_DR_FLUSHED
+#define AuOpts_DR_FLUSHED	0
+#endif
+
+struct au_opts {
+	struct au_opt	*opt;
+	int		max_opt;
+
+	unsigned int	given_udba;
+	unsigned int	flags;
+	unsigned long	sb_flags;
+};
+
+/* ---------------------------------------------------------------------- */
+
+/* opts.c */
+void au_optstr_br_perm(au_br_perm_str_t *str, int perm);
+const char *au_optstr_udba(int udba);
+const char *au_optstr_wbr_copyup(int wbr_copyup);
+const char *au_optstr_wbr_create(int wbr_create);
+
+void au_opts_free(struct au_opts *opts);
+struct super_block;
+int au_opts_parse(struct super_block *sb, char *str, struct au_opts *opts);
+int au_opts_verify(struct super_block *sb, unsigned long sb_flags,
+		   unsigned int pending);
+int au_opts_mount(struct super_block *sb, struct au_opts *opts);
+int au_opts_remount(struct super_block *sb, struct au_opts *opts);
+
+unsigned int au_opt_udba(struct super_block *sb);
+
+#endif /* __KERNEL__ */
+#endif /* __AUFS_OPTS_H__ */
diff -Naur --no-dereference a/fs/aufs/plink.c b/fs/aufs/plink.c
--- a/fs/aufs/plink.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/plink.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,516 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * pseudo-link
+ */
+
+#include "aufs.h"
+
+/*
+ * the pseudo-link maintenance mode.
+ * during a user process maintains the pseudo-links,
+ * prohibit adding a new plink and branch manipulation.
+ *
+ * Flags
+ * NOPLM:
+ *	For entry functions which will handle plink, and i_mutex is already held
+ *	in VFS.
+ *	They cannot wait and should return an error at once.
+ *	Callers has to check the error.
+ * NOPLMW:
+ *	For entry functions which will handle plink, but i_mutex is not held
+ *	in VFS.
+ *	They can wait the plink maintenance mode to finish.
+ *
+ * They behave like F_SETLK and F_SETLKW.
+ * If the caller never handle plink, then both flags are unnecessary.
+ */
+
+int au_plink_maint(struct super_block *sb, int flags)
+{
+	int err;
+	pid_t pid, ppid;
+	struct task_struct *parent, *prev;
+	struct au_sbinfo *sbi;
+
+	SiMustAnyLock(sb);
+
+	err = 0;
+	if (!au_opt_test(au_mntflags(sb), PLINK))
+		goto out;
+
+	sbi = au_sbi(sb);
+	pid = sbi->si_plink_maint_pid;
+	if (!pid || pid == current->pid)
+		goto out;
+
+	/* todo: it highly depends upon /sbin/mount.aufs */
+	prev = NULL;
+	parent = current;
+	ppid = 0;
+	rcu_read_lock();
+	while (1) {
+		parent = rcu_dereference(parent->real_parent);
+		if (parent == prev)
+			break;
+		ppid = task_pid_vnr(parent);
+		if (pid == ppid) {
+			rcu_read_unlock();
+			goto out;
+		}
+		prev = parent;
+	}
+	rcu_read_unlock();
+
+	if (au_ftest_lock(flags, NOPLMW)) {
+		/* if there is no i_mutex lock in VFS, we don't need to wait */
+		/* AuDebugOn(!lockdep_depth(current)); */
+		while (sbi->si_plink_maint_pid) {
+			si_read_unlock(sb);
+			/* gave up wake_up_bit() */
+			wait_event(sbi->si_plink_wq, !sbi->si_plink_maint_pid);
+
+			if (au_ftest_lock(flags, FLUSH))
+				au_nwt_flush(&sbi->si_nowait);
+			si_noflush_read_lock(sb);
+		}
+	} else if (au_ftest_lock(flags, NOPLM)) {
+		AuDbg("ppid %d, pid %d\n", ppid, pid);
+		err = -EAGAIN;
+	}
+
+out:
+	return err;
+}
+
+void au_plink_maint_leave(struct au_sbinfo *sbinfo)
+{
+	spin_lock(&sbinfo->si_plink_maint_lock);
+	sbinfo->si_plink_maint_pid = 0;
+	spin_unlock(&sbinfo->si_plink_maint_lock);
+	wake_up_all(&sbinfo->si_plink_wq);
+}
+
+int au_plink_maint_enter(struct super_block *sb)
+{
+	int err;
+	struct au_sbinfo *sbinfo;
+
+	err = 0;
+	sbinfo = au_sbi(sb);
+	/* make sure i am the only one in this fs */
+	si_write_lock(sb, AuLock_FLUSH);
+	if (au_opt_test(au_mntflags(sb), PLINK)) {
+		spin_lock(&sbinfo->si_plink_maint_lock);
+		if (!sbinfo->si_plink_maint_pid)
+			sbinfo->si_plink_maint_pid = current->pid;
+		else
+			err = -EBUSY;
+		spin_unlock(&sbinfo->si_plink_maint_lock);
+	}
+	si_write_unlock(sb);
+
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+#ifdef CONFIG_AUFS_DEBUG
+void au_plink_list(struct super_block *sb)
+{
+	int i;
+	struct au_sbinfo *sbinfo;
+	struct hlist_bl_head *hbl;
+	struct hlist_bl_node *pos;
+	struct au_icntnr *icntnr;
+
+	SiMustAnyLock(sb);
+
+	sbinfo = au_sbi(sb);
+	AuDebugOn(!au_opt_test(au_mntflags(sb), PLINK));
+	AuDebugOn(au_plink_maint(sb, AuLock_NOPLM));
+
+	for (i = 0; i < AuPlink_NHASH; i++) {
+		hbl = sbinfo->si_plink + i;
+		hlist_bl_lock(hbl);
+		hlist_bl_for_each_entry(icntnr, pos, hbl, plink)
+			AuDbg("%lu\n", icntnr->vfs_inode.i_ino);
+		hlist_bl_unlock(hbl);
+	}
+}
+#endif
+
+/* is the inode pseudo-linked? */
+int au_plink_test(struct inode *inode)
+{
+	int found, i;
+	struct au_sbinfo *sbinfo;
+	struct hlist_bl_head *hbl;
+	struct hlist_bl_node *pos;
+	struct au_icntnr *icntnr;
+
+	sbinfo = au_sbi(inode->i_sb);
+	AuRwMustAnyLock(&sbinfo->si_rwsem);
+	AuDebugOn(!au_opt_test(au_mntflags(inode->i_sb), PLINK));
+	AuDebugOn(au_plink_maint(inode->i_sb, AuLock_NOPLM));
+
+	found = 0;
+	i = au_plink_hash(inode->i_ino);
+	hbl =  sbinfo->si_plink + i;
+	hlist_bl_lock(hbl);
+	hlist_bl_for_each_entry(icntnr, pos, hbl, plink)
+		if (&icntnr->vfs_inode == inode) {
+			found = 1;
+			break;
+		}
+	hlist_bl_unlock(hbl);
+	return found;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * generate a name for plink.
+ * the file will be stored under AUFS_WH_PLINKDIR.
+ */
+/* 20 is max digits length of ulong 64 */
+#define PLINK_NAME_LEN	((20 + 1) * 2)
+
+static int plink_name(char *name, int len, struct inode *inode,
+		      aufs_bindex_t bindex)
+{
+	int rlen;
+	struct inode *h_inode;
+
+	h_inode = au_h_iptr(inode, bindex);
+	rlen = snprintf(name, len, "%lu.%lu", inode->i_ino, h_inode->i_ino);
+	return rlen;
+}
+
+struct au_do_plink_lkup_args {
+	struct dentry **errp;
+	struct qstr *tgtname;
+	struct dentry *h_parent;
+	struct au_branch *br;
+};
+
+static struct dentry *au_do_plink_lkup(struct qstr *tgtname,
+				       struct dentry *h_parent,
+				       struct au_branch *br)
+{
+	struct dentry *h_dentry;
+	struct inode *h_inode;
+
+	h_inode = d_inode(h_parent);
+	inode_lock_shared_nested(h_inode, AuLsc_I_CHILD2);
+	h_dentry = vfsub_lkup_one(tgtname, h_parent);
+	inode_unlock_shared(h_inode);
+	return h_dentry;
+}
+
+static void au_call_do_plink_lkup(void *args)
+{
+	struct au_do_plink_lkup_args *a = args;
+	*a->errp = au_do_plink_lkup(a->tgtname, a->h_parent, a->br);
+}
+
+/* lookup the plink-ed @inode under the branch at @bindex */
+struct dentry *au_plink_lkup(struct inode *inode, aufs_bindex_t bindex)
+{
+	struct dentry *h_dentry, *h_parent;
+	struct au_branch *br;
+	int wkq_err;
+	char a[PLINK_NAME_LEN];
+	struct qstr tgtname = QSTR_INIT(a, 0);
+
+	AuDebugOn(au_plink_maint(inode->i_sb, AuLock_NOPLM));
+
+	br = au_sbr(inode->i_sb, bindex);
+	h_parent = br->br_wbr->wbr_plink;
+	tgtname.len = plink_name(a, sizeof(a), inode, bindex);
+
+	if (!uid_eq(current_fsuid(), GLOBAL_ROOT_UID)) {
+		struct au_do_plink_lkup_args args = {
+			.errp		= &h_dentry,
+			.tgtname	= &tgtname,
+			.h_parent	= h_parent,
+			.br		= br
+		};
+
+		wkq_err = au_wkq_wait(au_call_do_plink_lkup, &args);
+		if (unlikely(wkq_err))
+			h_dentry = ERR_PTR(wkq_err);
+	} else
+		h_dentry = au_do_plink_lkup(&tgtname, h_parent, br);
+
+	return h_dentry;
+}
+
+/* create a pseudo-link */
+static int do_whplink(struct qstr *tgt, struct dentry *h_parent,
+		      struct dentry *h_dentry, struct au_branch *br)
+{
+	int err;
+	struct path h_path = {
+		.mnt = au_br_mnt(br)
+	};
+	struct inode *h_dir, *delegated;
+
+	h_dir = d_inode(h_parent);
+	inode_lock_nested(h_dir, AuLsc_I_CHILD2);
+again:
+	h_path.dentry = vfsub_lkup_one(tgt, h_parent);
+	err = PTR_ERR(h_path.dentry);
+	if (IS_ERR(h_path.dentry))
+		goto out;
+
+	err = 0;
+	/* wh.plink dir is not monitored */
+	/* todo: is it really safe? */
+	if (d_is_positive(h_path.dentry)
+	    && d_inode(h_path.dentry) != d_inode(h_dentry)) {
+		delegated = NULL;
+		err = vfsub_unlink(h_dir, &h_path, &delegated, /*force*/0);
+		if (unlikely(err == -EWOULDBLOCK)) {
+			pr_warn("cannot retry for NFSv4 delegation"
+				" for an internal unlink\n");
+			iput(delegated);
+		}
+		dput(h_path.dentry);
+		h_path.dentry = NULL;
+		if (!err)
+			goto again;
+	}
+	if (!err && d_is_negative(h_path.dentry)) {
+		delegated = NULL;
+		err = vfsub_link(h_dentry, h_dir, &h_path, &delegated);
+		if (unlikely(err == -EWOULDBLOCK)) {
+			pr_warn("cannot retry for NFSv4 delegation"
+				" for an internal link\n");
+			iput(delegated);
+		}
+	}
+	dput(h_path.dentry);
+
+out:
+	inode_unlock(h_dir);
+	return err;
+}
+
+struct do_whplink_args {
+	int *errp;
+	struct qstr *tgt;
+	struct dentry *h_parent;
+	struct dentry *h_dentry;
+	struct au_branch *br;
+};
+
+static void call_do_whplink(void *args)
+{
+	struct do_whplink_args *a = args;
+	*a->errp = do_whplink(a->tgt, a->h_parent, a->h_dentry, a->br);
+}
+
+static int whplink(struct dentry *h_dentry, struct inode *inode,
+		   aufs_bindex_t bindex, struct au_branch *br)
+{
+	int err, wkq_err;
+	struct au_wbr *wbr;
+	struct dentry *h_parent;
+	char a[PLINK_NAME_LEN];
+	struct qstr tgtname = QSTR_INIT(a, 0);
+
+	wbr = au_sbr(inode->i_sb, bindex)->br_wbr;
+	h_parent = wbr->wbr_plink;
+	tgtname.len = plink_name(a, sizeof(a), inode, bindex);
+
+	/* always superio. */
+	if (!uid_eq(current_fsuid(), GLOBAL_ROOT_UID)) {
+		struct do_whplink_args args = {
+			.errp		= &err,
+			.tgt		= &tgtname,
+			.h_parent	= h_parent,
+			.h_dentry	= h_dentry,
+			.br		= br
+		};
+		wkq_err = au_wkq_wait(call_do_whplink, &args);
+		if (unlikely(wkq_err))
+			err = wkq_err;
+	} else
+		err = do_whplink(&tgtname, h_parent, h_dentry, br);
+
+	return err;
+}
+
+/*
+ * create a new pseudo-link for @h_dentry on @bindex.
+ * the linked inode is held in aufs @inode.
+ */
+void au_plink_append(struct inode *inode, aufs_bindex_t bindex,
+		     struct dentry *h_dentry)
+{
+	struct super_block *sb;
+	struct au_sbinfo *sbinfo;
+	struct hlist_bl_head *hbl;
+	struct hlist_bl_node *pos;
+	struct au_icntnr *icntnr;
+	int found, err, cnt, i;
+
+	sb = inode->i_sb;
+	sbinfo = au_sbi(sb);
+	AuDebugOn(!au_opt_test(au_mntflags(sb), PLINK));
+	AuDebugOn(au_plink_maint(sb, AuLock_NOPLM));
+
+	found = au_plink_test(inode);
+	if (found)
+		return;
+
+	i = au_plink_hash(inode->i_ino);
+	hbl = sbinfo->si_plink + i;
+	au_igrab(inode);
+
+	hlist_bl_lock(hbl);
+	hlist_bl_for_each_entry(icntnr, pos, hbl, plink) {
+		if (&icntnr->vfs_inode == inode) {
+			found = 1;
+			break;
+		}
+	}
+	if (!found) {
+		icntnr = container_of(inode, struct au_icntnr, vfs_inode);
+		hlist_bl_add_head(&icntnr->plink, hbl);
+	}
+	hlist_bl_unlock(hbl);
+	if (!found) {
+		cnt = au_hbl_count(hbl);
+#define msg "unexpectedly unbalanced or too many pseudo-links"
+		if (cnt > AUFS_PLINK_WARN)
+			AuWarn1(msg ", %d\n", cnt);
+#undef msg
+		err = whplink(h_dentry, inode, bindex, au_sbr(sb, bindex));
+		if (unlikely(err)) {
+			pr_warn("err %d, damaged pseudo link.\n", err);
+			au_hbl_del(&icntnr->plink, hbl);
+			iput(&icntnr->vfs_inode);
+		}
+	} else
+		iput(&icntnr->vfs_inode);
+}
+
+/* free all plinks */
+void au_plink_put(struct super_block *sb, int verbose)
+{
+	int i, warned;
+	struct au_sbinfo *sbinfo;
+	struct hlist_bl_head *hbl;
+	struct hlist_bl_node *pos, *tmp;
+	struct au_icntnr *icntnr;
+
+	SiMustWriteLock(sb);
+
+	sbinfo = au_sbi(sb);
+	AuDebugOn(!au_opt_test(au_mntflags(sb), PLINK));
+	AuDebugOn(au_plink_maint(sb, AuLock_NOPLM));
+
+	/* no spin_lock since sbinfo is write-locked */
+	warned = 0;
+	for (i = 0; i < AuPlink_NHASH; i++) {
+		hbl = sbinfo->si_plink + i;
+		if (!warned && verbose && !hlist_bl_empty(hbl)) {
+			pr_warn("pseudo-link is not flushed");
+			warned = 1;
+		}
+		hlist_bl_for_each_entry_safe(icntnr, pos, tmp, hbl, plink)
+			iput(&icntnr->vfs_inode);
+		INIT_HLIST_BL_HEAD(hbl);
+	}
+}
+
+void au_plink_clean(struct super_block *sb, int verbose)
+{
+	struct dentry *root;
+
+	root = sb->s_root;
+	aufs_write_lock(root);
+	if (au_opt_test(au_mntflags(sb), PLINK))
+		au_plink_put(sb, verbose);
+	aufs_write_unlock(root);
+}
+
+static int au_plink_do_half_refresh(struct inode *inode, aufs_bindex_t br_id)
+{
+	int do_put;
+	aufs_bindex_t btop, bbot, bindex;
+
+	do_put = 0;
+	btop = au_ibtop(inode);
+	bbot = au_ibbot(inode);
+	if (btop >= 0) {
+		for (bindex = btop; bindex <= bbot; bindex++) {
+			if (!au_h_iptr(inode, bindex)
+			    || au_ii_br_id(inode, bindex) != br_id)
+				continue;
+			au_set_h_iptr(inode, bindex, NULL, 0);
+			do_put = 1;
+			break;
+		}
+		if (do_put)
+			for (bindex = btop; bindex <= bbot; bindex++)
+				if (au_h_iptr(inode, bindex)) {
+					do_put = 0;
+					break;
+				}
+	} else
+		do_put = 1;
+
+	return do_put;
+}
+
+/* free the plinks on a branch specified by @br_id */
+void au_plink_half_refresh(struct super_block *sb, aufs_bindex_t br_id)
+{
+	struct au_sbinfo *sbinfo;
+	struct hlist_bl_head *hbl;
+	struct hlist_bl_node *pos, *tmp;
+	struct au_icntnr *icntnr;
+	struct inode *inode;
+	int i, do_put;
+
+	SiMustWriteLock(sb);
+
+	sbinfo = au_sbi(sb);
+	AuDebugOn(!au_opt_test(au_mntflags(sb), PLINK));
+	AuDebugOn(au_plink_maint(sb, AuLock_NOPLM));
+
+	/* no bit_lock since sbinfo is write-locked */
+	for (i = 0; i < AuPlink_NHASH; i++) {
+		hbl = sbinfo->si_plink + i;
+		hlist_bl_for_each_entry_safe(icntnr, pos, tmp, hbl, plink) {
+			inode = au_igrab(&icntnr->vfs_inode);
+			ii_write_lock_child(inode);
+			do_put = au_plink_do_half_refresh(inode, br_id);
+			if (do_put) {
+				hlist_bl_del(&icntnr->plink);
+				iput(inode);
+			}
+			ii_write_unlock(inode);
+			iput(inode);
+		}
+	}
+}
diff -Naur --no-dereference a/fs/aufs/poll.c b/fs/aufs/poll.c
--- a/fs/aufs/poll.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/poll.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,51 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * poll operation
+ * There is only one filesystem which implements ->poll operation, currently.
+ */
+
+#include "aufs.h"
+
+__poll_t aufs_poll(struct file *file, struct poll_table_struct *pt)
+{
+	__poll_t mask;
+	struct file *h_file;
+	struct super_block *sb;
+
+	/* We should pretend an error happened. */
+	mask = EPOLLERR /* | EPOLLIN | EPOLLOUT */;
+	sb = file->f_path.dentry->d_sb;
+	si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLMW);
+
+	h_file = au_read_pre(file, /*keep_fi*/0, /*lsc*/0);
+	if (IS_ERR(h_file)) {
+		AuDbg("h_file %ld\n", PTR_ERR(h_file));
+		goto out;
+	}
+
+	mask = vfs_poll(h_file, pt);
+	fput(h_file); /* instead of au_read_post() */
+
+out:
+	si_read_unlock(sb);
+	if (mask & EPOLLERR)
+		AuDbg("mask 0x%x\n", mask);
+	return mask;
+}
diff -Naur --no-dereference a/fs/aufs/posix_acl.c b/fs/aufs/posix_acl.c
--- a/fs/aufs/posix_acl.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/posix_acl.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,105 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2014-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * posix acl operations
+ */
+
+#include <linux/fs.h>
+#include "aufs.h"
+
+struct posix_acl *aufs_get_acl(struct inode *inode, int type)
+{
+	struct posix_acl *acl;
+	int err;
+	aufs_bindex_t bindex;
+	struct inode *h_inode;
+	struct super_block *sb;
+
+	acl = NULL;
+	sb = inode->i_sb;
+	si_read_lock(sb, AuLock_FLUSH);
+	ii_read_lock_child(inode);
+	if (!(sb->s_flags & SB_POSIXACL))
+		goto out;
+
+	bindex = au_ibtop(inode);
+	h_inode = au_h_iptr(inode, bindex);
+	if (unlikely(!h_inode
+		     || ((h_inode->i_mode & S_IFMT)
+			 != (inode->i_mode & S_IFMT)))) {
+		err = au_busy_or_stale();
+		acl = ERR_PTR(err);
+		goto out;
+	}
+
+	/* always topmost only */
+	acl = get_acl(h_inode, type);
+	if (IS_ERR(acl))
+		forget_cached_acl(inode, type);
+	else
+		set_cached_acl(inode, type, acl);
+
+out:
+	ii_read_unlock(inode);
+	si_read_unlock(sb);
+
+	AuTraceErrPtr(acl);
+	return acl;
+}
+
+int aufs_set_acl(struct inode *inode, struct posix_acl *acl, int type)
+{
+	int err;
+	ssize_t ssz;
+	struct dentry *dentry;
+	struct au_sxattr arg = {
+		.type = AU_ACL_SET,
+		.u.acl_set = {
+			.acl	= acl,
+			.type	= type
+		},
+	};
+
+	IMustLock(inode);
+
+	if (inode->i_ino == AUFS_ROOT_INO)
+		dentry = dget(inode->i_sb->s_root);
+	else {
+		dentry = d_find_alias(inode);
+		if (!dentry)
+			dentry = d_find_any_alias(inode);
+		if (!dentry) {
+			pr_warn("cannot handle this inode, "
+				"please report to aufs-users ML\n");
+			err = -ENOENT;
+			goto out;
+		}
+	}
+
+	ssz = au_sxattr(dentry, inode, &arg);
+	/* forget even it if succeeds since the branch might set differently */
+	forget_cached_acl(inode, type);
+	dput(dentry);
+	err = ssz;
+	if (ssz >= 0)
+		err = 0;
+
+out:
+	return err;
+}
diff -Naur --no-dereference a/fs/aufs/procfs.c b/fs/aufs/procfs.c
--- a/fs/aufs/procfs.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/procfs.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,171 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2010-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * procfs interfaces
+ */
+
+#include <linux/proc_fs.h>
+#include "aufs.h"
+
+static int au_procfs_plm_release(struct inode *inode, struct file *file)
+{
+	struct au_sbinfo *sbinfo;
+
+	sbinfo = file->private_data;
+	if (sbinfo) {
+		au_plink_maint_leave(sbinfo);
+		kobject_put(&sbinfo->si_kobj);
+	}
+
+	return 0;
+}
+
+static void au_procfs_plm_write_clean(struct file *file)
+{
+	struct au_sbinfo *sbinfo;
+
+	sbinfo = file->private_data;
+	if (sbinfo)
+		au_plink_clean(sbinfo->si_sb, /*verbose*/0);
+}
+
+static int au_procfs_plm_write_si(struct file *file, unsigned long id)
+{
+	int err;
+	struct super_block *sb;
+	struct au_sbinfo *sbinfo;
+	struct hlist_bl_node *pos;
+
+	err = -EBUSY;
+	if (unlikely(file->private_data))
+		goto out;
+
+	sb = NULL;
+	/* don't use au_sbilist_lock() here */
+	hlist_bl_lock(&au_sbilist);
+	hlist_bl_for_each_entry(sbinfo, pos, &au_sbilist, si_list)
+		if (id == sysaufs_si_id(sbinfo)) {
+			if (kobject_get_unless_zero(&sbinfo->si_kobj))
+				sb = sbinfo->si_sb;
+			break;
+		}
+	hlist_bl_unlock(&au_sbilist);
+
+	err = -EINVAL;
+	if (unlikely(!sb))
+		goto out;
+
+	err = au_plink_maint_enter(sb);
+	if (!err)
+		/* keep kobject_get() */
+		file->private_data = sbinfo;
+	else
+		kobject_put(&sbinfo->si_kobj);
+out:
+	return err;
+}
+
+/*
+ * Accept a valid "si=xxxx" only.
+ * Once it is accepted successfully, accept "clean" too.
+ */
+static ssize_t au_procfs_plm_write(struct file *file, const char __user *ubuf,
+				   size_t count, loff_t *ppos)
+{
+	ssize_t err;
+	unsigned long id;
+	/* last newline is allowed */
+	char buf[3 + sizeof(unsigned long) * 2 + 1];
+
+	err = -EACCES;
+	if (unlikely(!capable(CAP_SYS_ADMIN)))
+		goto out;
+
+	err = -EINVAL;
+	if (unlikely(count > sizeof(buf)))
+		goto out;
+
+	err = copy_from_user(buf, ubuf, count);
+	if (unlikely(err)) {
+		err = -EFAULT;
+		goto out;
+	}
+	buf[count] = 0;
+
+	err = -EINVAL;
+	if (!strcmp("clean", buf)) {
+		au_procfs_plm_write_clean(file);
+		goto out_success;
+	} else if (unlikely(strncmp("si=", buf, 3)))
+		goto out;
+
+	err = kstrtoul(buf + 3, 16, &id);
+	if (unlikely(err))
+		goto out;
+
+	err = au_procfs_plm_write_si(file, id);
+	if (unlikely(err))
+		goto out;
+
+out_success:
+	err = count; /* success */
+out:
+	return err;
+}
+
+static const struct file_operations au_procfs_plm_fop = {
+	.write		= au_procfs_plm_write,
+	.release	= au_procfs_plm_release,
+	.owner		= THIS_MODULE
+};
+
+/* ---------------------------------------------------------------------- */
+
+static struct proc_dir_entry *au_procfs_dir;
+
+void au_procfs_fin(void)
+{
+	remove_proc_entry(AUFS_PLINK_MAINT_NAME, au_procfs_dir);
+	remove_proc_entry(AUFS_PLINK_MAINT_DIR, NULL);
+}
+
+int __init au_procfs_init(void)
+{
+	int err;
+	struct proc_dir_entry *entry;
+
+	err = -ENOMEM;
+	au_procfs_dir = proc_mkdir(AUFS_PLINK_MAINT_DIR, NULL);
+	if (unlikely(!au_procfs_dir))
+		goto out;
+
+	entry = proc_create(AUFS_PLINK_MAINT_NAME, S_IFREG | 0200,
+			    au_procfs_dir, &au_procfs_plm_fop);
+	if (unlikely(!entry))
+		goto out_dir;
+
+	err = 0;
+	goto out; /* success */
+
+
+out_dir:
+	remove_proc_entry(AUFS_PLINK_MAINT_DIR, NULL);
+out:
+	return err;
+}
diff -Naur --no-dereference a/fs/aufs/rdu.c b/fs/aufs/rdu.c
--- a/fs/aufs/rdu.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/rdu.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,382 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * readdir in userspace.
+ */
+
+#include <linux/compat.h>
+#include <linux/fs_stack.h>
+#include <linux/security.h>
+#include "aufs.h"
+
+/* bits for struct aufs_rdu.flags */
+#define	AuRdu_CALLED	1
+#define	AuRdu_CONT	(1 << 1)
+#define	AuRdu_FULL	(1 << 2)
+#define au_ftest_rdu(flags, name)	((flags) & AuRdu_##name)
+#define au_fset_rdu(flags, name) \
+	do { (flags) |= AuRdu_##name; } while (0)
+#define au_fclr_rdu(flags, name) \
+	do { (flags) &= ~AuRdu_##name; } while (0)
+
+struct au_rdu_arg {
+	struct dir_context		ctx;
+	struct aufs_rdu			*rdu;
+	union au_rdu_ent_ul		ent;
+	unsigned long			end;
+
+	struct super_block		*sb;
+	int				err;
+};
+
+static int au_rdu_fill(struct dir_context *ctx, const char *name, int nlen,
+		       loff_t offset, u64 h_ino, unsigned int d_type)
+{
+	int err, len;
+	struct au_rdu_arg *arg = container_of(ctx, struct au_rdu_arg, ctx);
+	struct aufs_rdu *rdu = arg->rdu;
+	struct au_rdu_ent ent;
+
+	err = 0;
+	arg->err = 0;
+	au_fset_rdu(rdu->cookie.flags, CALLED);
+	len = au_rdu_len(nlen);
+	if (arg->ent.ul + len  < arg->end) {
+		ent.ino = h_ino;
+		ent.bindex = rdu->cookie.bindex;
+		ent.type = d_type;
+		ent.nlen = nlen;
+		if (unlikely(nlen > AUFS_MAX_NAMELEN))
+			ent.type = DT_UNKNOWN;
+
+		/* unnecessary to support mmap_sem since this is a dir */
+		err = -EFAULT;
+		if (copy_to_user(arg->ent.e, &ent, sizeof(ent)))
+			goto out;
+		if (copy_to_user(arg->ent.e->name, name, nlen))
+			goto out;
+		/* the terminating NULL */
+		if (__put_user(0, arg->ent.e->name + nlen))
+			goto out;
+		err = 0;
+		/* AuDbg("%p, %.*s\n", arg->ent.p, nlen, name); */
+		arg->ent.ul += len;
+		rdu->rent++;
+	} else {
+		err = -EFAULT;
+		au_fset_rdu(rdu->cookie.flags, FULL);
+		rdu->full = 1;
+		rdu->tail = arg->ent;
+	}
+
+out:
+	/* AuTraceErr(err); */
+	return err;
+}
+
+static int au_rdu_do(struct file *h_file, struct au_rdu_arg *arg)
+{
+	int err;
+	loff_t offset;
+	struct au_rdu_cookie *cookie = &arg->rdu->cookie;
+
+	/* we don't have to care (FMODE_32BITHASH | FMODE_64BITHASH) for ext4 */
+	offset = vfsub_llseek(h_file, cookie->h_pos, SEEK_SET);
+	err = offset;
+	if (unlikely(offset != cookie->h_pos))
+		goto out;
+
+	err = 0;
+	do {
+		arg->err = 0;
+		au_fclr_rdu(cookie->flags, CALLED);
+		/* smp_mb(); */
+		err = vfsub_iterate_dir(h_file, &arg->ctx);
+		if (err >= 0)
+			err = arg->err;
+	} while (!err
+		 && au_ftest_rdu(cookie->flags, CALLED)
+		 && !au_ftest_rdu(cookie->flags, FULL));
+	cookie->h_pos = h_file->f_pos;
+
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+static int au_rdu(struct file *file, struct aufs_rdu *rdu)
+{
+	int err;
+	aufs_bindex_t bbot;
+	struct au_rdu_arg arg = {
+		.ctx = {
+			.actor = au_rdu_fill
+		}
+	};
+	struct dentry *dentry;
+	struct inode *inode;
+	struct file *h_file;
+	struct au_rdu_cookie *cookie = &rdu->cookie;
+
+	err = !access_ok(VERIFY_WRITE, rdu->ent.e, rdu->sz);
+	if (unlikely(err)) {
+		err = -EFAULT;
+		AuTraceErr(err);
+		goto out;
+	}
+	rdu->rent = 0;
+	rdu->tail = rdu->ent;
+	rdu->full = 0;
+	arg.rdu = rdu;
+	arg.ent = rdu->ent;
+	arg.end = arg.ent.ul;
+	arg.end += rdu->sz;
+
+	err = -ENOTDIR;
+	if (unlikely(!file->f_op->iterate && !file->f_op->iterate_shared))
+		goto out;
+
+	err = security_file_permission(file, MAY_READ);
+	AuTraceErr(err);
+	if (unlikely(err))
+		goto out;
+
+	dentry = file->f_path.dentry;
+	inode = d_inode(dentry);
+	inode_lock_shared(inode);
+
+	arg.sb = inode->i_sb;
+	err = si_read_lock(arg.sb, AuLock_FLUSH | AuLock_NOPLM);
+	if (unlikely(err))
+		goto out_mtx;
+	err = au_alive_dir(dentry);
+	if (unlikely(err))
+		goto out_si;
+	/* todo: reval? */
+	fi_read_lock(file);
+
+	err = -EAGAIN;
+	if (unlikely(au_ftest_rdu(cookie->flags, CONT)
+		     && cookie->generation != au_figen(file)))
+		goto out_unlock;
+
+	err = 0;
+	if (!rdu->blk) {
+		rdu->blk = au_sbi(arg.sb)->si_rdblk;
+		if (!rdu->blk)
+			rdu->blk = au_dir_size(file, /*dentry*/NULL);
+	}
+	bbot = au_fbtop(file);
+	if (cookie->bindex < bbot)
+		cookie->bindex = bbot;
+	bbot = au_fbbot_dir(file);
+	/* AuDbg("b%d, b%d\n", cookie->bindex, bbot); */
+	for (; !err && cookie->bindex <= bbot;
+	     cookie->bindex++, cookie->h_pos = 0) {
+		h_file = au_hf_dir(file, cookie->bindex);
+		if (!h_file)
+			continue;
+
+		au_fclr_rdu(cookie->flags, FULL);
+		err = au_rdu_do(h_file, &arg);
+		AuTraceErr(err);
+		if (unlikely(au_ftest_rdu(cookie->flags, FULL) || err))
+			break;
+	}
+	AuDbg("rent %llu\n", rdu->rent);
+
+	if (!err && !au_ftest_rdu(cookie->flags, CONT)) {
+		rdu->shwh = !!au_opt_test(au_sbi(arg.sb)->si_mntflags, SHWH);
+		au_fset_rdu(cookie->flags, CONT);
+		cookie->generation = au_figen(file);
+	}
+
+	ii_read_lock_child(inode);
+	fsstack_copy_attr_atime(inode, au_h_iptr(inode, au_ibtop(inode)));
+	ii_read_unlock(inode);
+
+out_unlock:
+	fi_read_unlock(file);
+out_si:
+	si_read_unlock(arg.sb);
+out_mtx:
+	inode_unlock_shared(inode);
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+static int au_rdu_ino(struct file *file, struct aufs_rdu *rdu)
+{
+	int err;
+	ino_t ino;
+	unsigned long long nent;
+	union au_rdu_ent_ul *u;
+	struct au_rdu_ent ent;
+	struct super_block *sb;
+
+	err = 0;
+	nent = rdu->nent;
+	u = &rdu->ent;
+	sb = file->f_path.dentry->d_sb;
+	si_read_lock(sb, AuLock_FLUSH);
+	while (nent-- > 0) {
+		/* unnecessary to support mmap_sem since this is a dir */
+		err = copy_from_user(&ent, u->e, sizeof(ent));
+		if (!err)
+			err = !access_ok(VERIFY_WRITE, &u->e->ino, sizeof(ino));
+		if (unlikely(err)) {
+			err = -EFAULT;
+			AuTraceErr(err);
+			break;
+		}
+
+		/* AuDbg("b%d, i%llu\n", ent.bindex, ent.ino); */
+		if (!ent.wh)
+			err = au_ino(sb, ent.bindex, ent.ino, ent.type, &ino);
+		else
+			err = au_wh_ino(sb, ent.bindex, ent.ino, ent.type,
+					&ino);
+		if (unlikely(err)) {
+			AuTraceErr(err);
+			break;
+		}
+
+		err = __put_user(ino, &u->e->ino);
+		if (unlikely(err)) {
+			err = -EFAULT;
+			AuTraceErr(err);
+			break;
+		}
+		u->ul += au_rdu_len(ent.nlen);
+	}
+	si_read_unlock(sb);
+
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static int au_rdu_verify(struct aufs_rdu *rdu)
+{
+	AuDbg("rdu{%llu, %p, %u | %u | %llu, %u, %u | "
+	      "%llu, b%d, 0x%x, g%u}\n",
+	      rdu->sz, rdu->ent.e, rdu->verify[AufsCtlRduV_SZ],
+	      rdu->blk,
+	      rdu->rent, rdu->shwh, rdu->full,
+	      rdu->cookie.h_pos, rdu->cookie.bindex, rdu->cookie.flags,
+	      rdu->cookie.generation);
+
+	if (rdu->verify[AufsCtlRduV_SZ] == sizeof(*rdu))
+		return 0;
+
+	AuDbg("%u:%u\n",
+	      rdu->verify[AufsCtlRduV_SZ], (unsigned int)sizeof(*rdu));
+	return -EINVAL;
+}
+
+long au_rdu_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	long err, e;
+	struct aufs_rdu rdu;
+	void __user *p = (void __user *)arg;
+
+	err = copy_from_user(&rdu, p, sizeof(rdu));
+	if (unlikely(err)) {
+		err = -EFAULT;
+		AuTraceErr(err);
+		goto out;
+	}
+	err = au_rdu_verify(&rdu);
+	if (unlikely(err))
+		goto out;
+
+	switch (cmd) {
+	case AUFS_CTL_RDU:
+		err = au_rdu(file, &rdu);
+		if (unlikely(err))
+			break;
+
+		e = copy_to_user(p, &rdu, sizeof(rdu));
+		if (unlikely(e)) {
+			err = -EFAULT;
+			AuTraceErr(err);
+		}
+		break;
+	case AUFS_CTL_RDU_INO:
+		err = au_rdu_ino(file, &rdu);
+		break;
+
+	default:
+		/* err = -ENOTTY; */
+		err = -EINVAL;
+	}
+
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+#ifdef CONFIG_COMPAT
+long au_rdu_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	long err, e;
+	struct aufs_rdu rdu;
+	void __user *p = compat_ptr(arg);
+
+	/* todo: get_user()? */
+	err = copy_from_user(&rdu, p, sizeof(rdu));
+	if (unlikely(err)) {
+		err = -EFAULT;
+		AuTraceErr(err);
+		goto out;
+	}
+	rdu.ent.e = compat_ptr(rdu.ent.ul);
+	err = au_rdu_verify(&rdu);
+	if (unlikely(err))
+		goto out;
+
+	switch (cmd) {
+	case AUFS_CTL_RDU:
+		err = au_rdu(file, &rdu);
+		if (unlikely(err))
+			break;
+
+		rdu.ent.ul = ptr_to_compat(rdu.ent.e);
+		rdu.tail.ul = ptr_to_compat(rdu.tail.e);
+		e = copy_to_user(p, &rdu, sizeof(rdu));
+		if (unlikely(e)) {
+			err = -EFAULT;
+			AuTraceErr(err);
+		}
+		break;
+	case AUFS_CTL_RDU_INO:
+		err = au_rdu_ino(file, &rdu);
+		break;
+
+	default:
+		/* err = -ENOTTY; */
+		err = -EINVAL;
+	}
+
+out:
+	AuTraceErr(err);
+	return err;
+}
+#endif
diff -Naur --no-dereference a/fs/aufs/rwsem.h b/fs/aufs/rwsem.h
--- a/fs/aufs/rwsem.h	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/rwsem.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,73 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * simple read-write semaphore wrappers
+ */
+
+#ifndef __AUFS_RWSEM_H__
+#define __AUFS_RWSEM_H__
+
+#ifdef __KERNEL__
+
+#include "debug.h"
+
+/* in the future, the name 'au_rwsem' will be totally gone */
+#define au_rwsem	rw_semaphore
+
+/* to debug easier, do not make them inlined functions */
+#define AuRwMustNoWaiters(rw)	AuDebugOn(rwsem_is_contended(rw))
+/* rwsem_is_locked() is unusable */
+#define AuRwMustReadLock(rw)	AuDebugOn(!lockdep_recursing(current) \
+					  && debug_locks \
+					  && !lockdep_is_held_type(rw, 1))
+#define AuRwMustWriteLock(rw)	AuDebugOn(!lockdep_recursing(current) \
+					  && debug_locks \
+					  && !lockdep_is_held_type(rw, 0))
+#define AuRwMustAnyLock(rw)	AuDebugOn(!lockdep_recursing(current) \
+					  && debug_locks \
+					  && !lockdep_is_held(rw))
+#define AuRwDestroy(rw)		AuDebugOn(!lockdep_recursing(current) \
+					  && debug_locks \
+					  && lockdep_is_held(rw))
+
+#define au_rw_init(rw)	init_rwsem(rw)
+
+#define au_rw_init_wlock(rw) do {		\
+		au_rw_init(rw);			\
+		down_write(rw);			\
+	} while (0)
+
+#define au_rw_init_wlock_nested(rw, lsc) do {	\
+		au_rw_init(rw);			\
+		down_write_nested(rw, lsc);	\
+	} while (0)
+
+#define au_rw_read_lock(rw)		down_read(rw)
+#define au_rw_read_lock_nested(rw, lsc)	down_read_nested(rw, lsc)
+#define au_rw_read_unlock(rw)		up_read(rw)
+#define au_rw_dgrade_lock(rw)		downgrade_write(rw)
+#define au_rw_write_lock(rw)		down_write(rw)
+#define au_rw_write_lock_nested(rw, lsc) down_write_nested(rw, lsc)
+#define au_rw_write_unlock(rw)		up_write(rw)
+/* why is not _nested version defined? */
+#define au_rw_read_trylock(rw)		down_read_trylock(rw)
+#define au_rw_write_trylock(rw)		down_write_trylock(rw)
+
+#endif /* __KERNEL__ */
+#endif /* __AUFS_RWSEM_H__ */
diff -Naur --no-dereference a/fs/aufs/sbinfo.c b/fs/aufs/sbinfo.c
--- a/fs/aufs/sbinfo.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/sbinfo.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,314 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * superblock private data
+ */
+
+#include <linux/iversion.h>
+#include "aufs.h"
+
+/*
+ * they are necessary regardless sysfs is disabled.
+ */
+void au_si_free(struct kobject *kobj)
+{
+	int i;
+	struct au_sbinfo *sbinfo;
+	char *locked __maybe_unused; /* debug only */
+
+	sbinfo = container_of(kobj, struct au_sbinfo, si_kobj);
+	for (i = 0; i < AuPlink_NHASH; i++)
+		AuDebugOn(!hlist_bl_empty(sbinfo->si_plink + i));
+	AuDebugOn(atomic_read(&sbinfo->si_nowait.nw_len));
+
+	AuLCntZero(au_lcnt_read(&sbinfo->si_ninodes, /*do_rev*/0));
+	au_lcnt_fin(&sbinfo->si_ninodes, /*do_sync*/0);
+	AuLCntZero(au_lcnt_read(&sbinfo->si_nfiles, /*do_rev*/0));
+	au_lcnt_fin(&sbinfo->si_nfiles, /*do_sync*/0);
+
+	dbgaufs_si_fin(sbinfo);
+	au_rw_write_lock(&sbinfo->si_rwsem);
+	au_br_free(sbinfo);
+	au_rw_write_unlock(&sbinfo->si_rwsem);
+
+	au_kfree_try_rcu(sbinfo->si_branch);
+	mutex_destroy(&sbinfo->si_xib_mtx);
+	AuRwDestroy(&sbinfo->si_rwsem);
+
+	au_lcnt_wait_for_fin(&sbinfo->si_ninodes);
+	/* si_nfiles is waited too */
+	au_kfree_rcu(sbinfo);
+}
+
+int au_si_alloc(struct super_block *sb)
+{
+	int err, i;
+	struct au_sbinfo *sbinfo;
+
+	err = -ENOMEM;
+	sbinfo = kzalloc(sizeof(*sbinfo), GFP_NOFS);
+	if (unlikely(!sbinfo))
+		goto out;
+
+	/* will be reallocated separately */
+	sbinfo->si_branch = kzalloc(sizeof(*sbinfo->si_branch), GFP_NOFS);
+	if (unlikely(!sbinfo->si_branch))
+		goto out_sbinfo;
+
+	err = sysaufs_si_init(sbinfo);
+	if (!err) {
+		dbgaufs_si_null(sbinfo);
+		err = dbgaufs_si_init(sbinfo);
+		if (unlikely(err))
+			kobject_put(&sbinfo->si_kobj);
+	}
+	if (unlikely(err))
+		goto out_br;
+
+	au_nwt_init(&sbinfo->si_nowait);
+	au_rw_init_wlock(&sbinfo->si_rwsem);
+
+	au_lcnt_init(&sbinfo->si_ninodes, /*release*/NULL);
+	au_lcnt_init(&sbinfo->si_nfiles, /*release*/NULL);
+
+	sbinfo->si_bbot = -1;
+	sbinfo->si_last_br_id = AUFS_BRANCH_MAX / 2;
+
+	sbinfo->si_wbr_copyup = AuWbrCopyup_Def;
+	sbinfo->si_wbr_create = AuWbrCreate_Def;
+	sbinfo->si_wbr_copyup_ops = au_wbr_copyup_ops + sbinfo->si_wbr_copyup;
+	sbinfo->si_wbr_create_ops = au_wbr_create_ops + sbinfo->si_wbr_create;
+
+	au_fhsm_init(sbinfo);
+
+	sbinfo->si_mntflags = au_opts_plink(AuOpt_Def);
+
+	sbinfo->si_xino_jiffy = jiffies;
+	sbinfo->si_xino_expire
+		= msecs_to_jiffies(AUFS_XINO_DEF_SEC * MSEC_PER_SEC);
+	mutex_init(&sbinfo->si_xib_mtx);
+	/* leave si_xib_last_pindex and si_xib_next_bit */
+
+	INIT_HLIST_BL_HEAD(&sbinfo->si_aopen);
+
+	sbinfo->si_rdcache = msecs_to_jiffies(AUFS_RDCACHE_DEF * MSEC_PER_SEC);
+	sbinfo->si_rdblk = AUFS_RDBLK_DEF;
+	sbinfo->si_rdhash = AUFS_RDHASH_DEF;
+	sbinfo->si_dirwh = AUFS_DIRWH_DEF;
+
+	for (i = 0; i < AuPlink_NHASH; i++)
+		INIT_HLIST_BL_HEAD(sbinfo->si_plink + i);
+	init_waitqueue_head(&sbinfo->si_plink_wq);
+	spin_lock_init(&sbinfo->si_plink_maint_lock);
+
+	INIT_HLIST_BL_HEAD(&sbinfo->si_files);
+
+	/* with getattr by default */
+	sbinfo->si_iop_array = aufs_iop;
+
+	/* leave other members for sysaufs and si_mnt. */
+	sbinfo->si_sb = sb;
+	sb->s_fs_info = sbinfo;
+	si_pid_set(sb);
+	return 0; /* success */
+
+out_br:
+	au_kfree_try_rcu(sbinfo->si_branch);
+out_sbinfo:
+	au_kfree_rcu(sbinfo);
+out:
+	return err;
+}
+
+int au_sbr_realloc(struct au_sbinfo *sbinfo, int nbr, int may_shrink)
+{
+	int err, sz;
+	struct au_branch **brp;
+
+	AuRwMustWriteLock(&sbinfo->si_rwsem);
+
+	err = -ENOMEM;
+	sz = sizeof(*brp) * (sbinfo->si_bbot + 1);
+	if (unlikely(!sz))
+		sz = sizeof(*brp);
+	brp = au_kzrealloc(sbinfo->si_branch, sz, sizeof(*brp) * nbr, GFP_NOFS,
+			   may_shrink);
+	if (brp) {
+		sbinfo->si_branch = brp;
+		err = 0;
+	}
+
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+unsigned int au_sigen_inc(struct super_block *sb)
+{
+	unsigned int gen;
+	struct inode *inode;
+
+	SiMustWriteLock(sb);
+
+	gen = ++au_sbi(sb)->si_generation;
+	au_update_digen(sb->s_root);
+	inode = d_inode(sb->s_root);
+	au_update_iigen(inode, /*half*/0);
+	inode_inc_iversion(inode);
+	return gen;
+}
+
+aufs_bindex_t au_new_br_id(struct super_block *sb)
+{
+	aufs_bindex_t br_id;
+	int i;
+	struct au_sbinfo *sbinfo;
+
+	SiMustWriteLock(sb);
+
+	sbinfo = au_sbi(sb);
+	for (i = 0; i <= AUFS_BRANCH_MAX; i++) {
+		br_id = ++sbinfo->si_last_br_id;
+		AuDebugOn(br_id < 0);
+		if (br_id && au_br_index(sb, br_id) < 0)
+			return br_id;
+	}
+
+	return -1;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* it is ok that new 'nwt' tasks are appended while we are sleeping */
+int si_read_lock(struct super_block *sb, int flags)
+{
+	int err;
+
+	err = 0;
+	if (au_ftest_lock(flags, FLUSH))
+		au_nwt_flush(&au_sbi(sb)->si_nowait);
+
+	si_noflush_read_lock(sb);
+	err = au_plink_maint(sb, flags);
+	if (unlikely(err))
+		si_read_unlock(sb);
+
+	return err;
+}
+
+int si_write_lock(struct super_block *sb, int flags)
+{
+	int err;
+
+	if (au_ftest_lock(flags, FLUSH))
+		au_nwt_flush(&au_sbi(sb)->si_nowait);
+
+	si_noflush_write_lock(sb);
+	err = au_plink_maint(sb, flags);
+	if (unlikely(err))
+		si_write_unlock(sb);
+
+	return err;
+}
+
+/* dentry and super_block lock. call at entry point */
+int aufs_read_lock(struct dentry *dentry, int flags)
+{
+	int err;
+	struct super_block *sb;
+
+	sb = dentry->d_sb;
+	err = si_read_lock(sb, flags);
+	if (unlikely(err))
+		goto out;
+
+	if (au_ftest_lock(flags, DW))
+		di_write_lock_child(dentry);
+	else
+		di_read_lock_child(dentry, flags);
+
+	if (au_ftest_lock(flags, GEN)) {
+		err = au_digen_test(dentry, au_sigen(sb));
+		if (!au_opt_test(au_mntflags(sb), UDBA_NONE))
+			AuDebugOn(!err && au_dbrange_test(dentry));
+		else if (!err)
+			err = au_dbrange_test(dentry);
+		if (unlikely(err))
+			aufs_read_unlock(dentry, flags);
+	}
+
+out:
+	return err;
+}
+
+void aufs_read_unlock(struct dentry *dentry, int flags)
+{
+	if (au_ftest_lock(flags, DW))
+		di_write_unlock(dentry);
+	else
+		di_read_unlock(dentry, flags);
+	si_read_unlock(dentry->d_sb);
+}
+
+void aufs_write_lock(struct dentry *dentry)
+{
+	si_write_lock(dentry->d_sb, AuLock_FLUSH | AuLock_NOPLMW);
+	di_write_lock_child(dentry);
+}
+
+void aufs_write_unlock(struct dentry *dentry)
+{
+	di_write_unlock(dentry);
+	si_write_unlock(dentry->d_sb);
+}
+
+int aufs_read_and_write_lock2(struct dentry *d1, struct dentry *d2, int flags)
+{
+	int err;
+	unsigned int sigen;
+	struct super_block *sb;
+
+	sb = d1->d_sb;
+	err = si_read_lock(sb, flags);
+	if (unlikely(err))
+		goto out;
+
+	di_write_lock2_child(d1, d2, au_ftest_lock(flags, DIRS));
+
+	if (au_ftest_lock(flags, GEN)) {
+		sigen = au_sigen(sb);
+		err = au_digen_test(d1, sigen);
+		AuDebugOn(!err && au_dbrange_test(d1));
+		if (!err) {
+			err = au_digen_test(d2, sigen);
+			AuDebugOn(!err && au_dbrange_test(d2));
+		}
+		if (unlikely(err))
+			aufs_read_and_write_unlock2(d1, d2);
+	}
+
+out:
+	return err;
+}
+
+void aufs_read_and_write_unlock2(struct dentry *d1, struct dentry *d2)
+{
+	di_write_unlock2(d1, d2);
+	si_read_unlock(d1->d_sb);
+}
diff -Naur --no-dereference a/fs/aufs/super.c b/fs/aufs/super.c
--- a/fs/aufs/super.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/super.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,1049 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * mount and super_block operations
+ */
+
+#include <linux/iversion.h>
+#include <linux/mm.h>
+#include <linux/seq_file.h>
+#include <linux/statfs.h>
+#include <linux/vmalloc.h>
+#include "aufs.h"
+
+/*
+ * super_operations
+ */
+static struct inode *aufs_alloc_inode(struct super_block *sb __maybe_unused)
+{
+	struct au_icntnr *c;
+
+	c = au_cache_alloc_icntnr();
+	if (c) {
+		au_icntnr_init(c);
+		inode_set_iversion(&c->vfs_inode, 1); /* sigen(sb); */
+		c->iinfo.ii_hinode = NULL;
+		return &c->vfs_inode;
+	}
+	return NULL;
+}
+
+static void aufs_destroy_inode_cb(struct rcu_head *head)
+{
+	struct inode *inode = container_of(head, struct inode, i_rcu);
+
+	au_cache_free_icntnr(container_of(inode, struct au_icntnr, vfs_inode));
+}
+
+static void aufs_destroy_inode(struct inode *inode)
+{
+	if (!au_is_bad_inode(inode))
+		au_iinfo_fin(inode);
+	call_rcu(&inode->i_rcu, aufs_destroy_inode_cb);
+}
+
+struct inode *au_iget_locked(struct super_block *sb, ino_t ino)
+{
+	struct inode *inode;
+	int err;
+
+	inode = iget_locked(sb, ino);
+	if (unlikely(!inode)) {
+		inode = ERR_PTR(-ENOMEM);
+		goto out;
+	}
+	if (!(inode->i_state & I_NEW))
+		goto out;
+
+	err = au_xigen_new(inode);
+	if (!err)
+		err = au_iinfo_init(inode);
+	if (!err)
+		inode_inc_iversion(inode);
+	else {
+		iget_failed(inode);
+		inode = ERR_PTR(err);
+	}
+
+out:
+	/* never return NULL */
+	AuDebugOn(!inode);
+	AuTraceErrPtr(inode);
+	return inode;
+}
+
+/* lock free root dinfo */
+static int au_show_brs(struct seq_file *seq, struct super_block *sb)
+{
+	int err;
+	aufs_bindex_t bindex, bbot;
+	struct path path;
+	struct au_hdentry *hdp;
+	struct au_branch *br;
+	au_br_perm_str_t perm;
+
+	err = 0;
+	bbot = au_sbbot(sb);
+	bindex = 0;
+	hdp = au_hdentry(au_di(sb->s_root), bindex);
+	for (; !err && bindex <= bbot; bindex++, hdp++) {
+		br = au_sbr(sb, bindex);
+		path.mnt = au_br_mnt(br);
+		path.dentry = hdp->hd_dentry;
+		err = au_seq_path(seq, &path);
+		if (!err) {
+			au_optstr_br_perm(&perm, br->br_perm);
+			seq_printf(seq, "=%s", perm.a);
+			if (bindex != bbot)
+				seq_putc(seq, ':');
+		}
+	}
+	if (unlikely(err || seq_has_overflowed(seq)))
+		err = -E2BIG;
+
+	return err;
+}
+
+static void au_gen_fmt(char *fmt, int len __maybe_unused, const char *pat,
+		       const char *append)
+{
+	char *p;
+
+	p = fmt;
+	while (*pat != ':')
+		*p++ = *pat++;
+	*p++ = *pat++;
+	strcpy(p, append);
+	AuDebugOn(strlen(fmt) >= len);
+}
+
+static void au_show_wbr_create(struct seq_file *m, int v,
+			       struct au_sbinfo *sbinfo)
+{
+	const char *pat;
+	char fmt[32];
+	struct au_wbr_mfs *mfs;
+
+	AuRwMustAnyLock(&sbinfo->si_rwsem);
+
+	seq_puts(m, ",create=");
+	pat = au_optstr_wbr_create(v);
+	mfs = &sbinfo->si_wbr_mfs;
+	switch (v) {
+	case AuWbrCreate_TDP:
+	case AuWbrCreate_RR:
+	case AuWbrCreate_MFS:
+	case AuWbrCreate_PMFS:
+		seq_puts(m, pat);
+		break;
+	case AuWbrCreate_MFSRR:
+	case AuWbrCreate_TDMFS:
+	case AuWbrCreate_PMFSRR:
+		au_gen_fmt(fmt, sizeof(fmt), pat, "%llu");
+		seq_printf(m, fmt, mfs->mfsrr_watermark);
+		break;
+	case AuWbrCreate_MFSV:
+	case AuWbrCreate_PMFSV:
+		au_gen_fmt(fmt, sizeof(fmt), pat, "%lu");
+		seq_printf(m, fmt,
+			   jiffies_to_msecs(mfs->mfs_expire)
+			   / MSEC_PER_SEC);
+		break;
+	case AuWbrCreate_MFSRRV:
+	case AuWbrCreate_TDMFSV:
+	case AuWbrCreate_PMFSRRV:
+		au_gen_fmt(fmt, sizeof(fmt), pat, "%llu:%lu");
+		seq_printf(m, fmt, mfs->mfsrr_watermark,
+			   jiffies_to_msecs(mfs->mfs_expire) / MSEC_PER_SEC);
+		break;
+	default:
+		BUG();
+	}
+}
+
+static int au_show_xino(struct seq_file *seq, struct super_block *sb)
+{
+#ifdef CONFIG_SYSFS
+	return 0;
+#else
+	int err;
+	const int len = sizeof(AUFS_XINO_FNAME) - 1;
+	aufs_bindex_t bindex, brid;
+	struct qstr *name;
+	struct file *f;
+	struct dentry *d, *h_root;
+	struct au_branch *br;
+
+	AuRwMustAnyLock(&sbinfo->si_rwsem);
+
+	err = 0;
+	f = au_sbi(sb)->si_xib;
+	if (!f)
+		goto out;
+
+	/* stop printing the default xino path on the first writable branch */
+	h_root = NULL;
+	bindex = au_xi_root(sb, f->f_path.dentry);
+	if (bindex >= 0) {
+		br = au_sbr_sb(sb, bindex);
+		h_root = au_br_dentry(br);
+	}
+
+	d = f->f_path.dentry;
+	name = &d->d_name;
+	/* safe ->d_parent because the file is unlinked */
+	if (d->d_parent == h_root
+	    && name->len == len
+	    && !memcmp(name->name, AUFS_XINO_FNAME, len))
+		goto out;
+
+	seq_puts(seq, ",xino=");
+	err = au_xino_path(seq, f);
+
+out:
+	return err;
+#endif
+}
+
+/* seq_file will re-call me in case of too long string */
+static int aufs_show_options(struct seq_file *m, struct dentry *dentry)
+{
+	int err;
+	unsigned int mnt_flags, v;
+	struct super_block *sb;
+	struct au_sbinfo *sbinfo;
+
+#define AuBool(name, str) do { \
+	v = au_opt_test(mnt_flags, name); \
+	if (v != au_opt_test(AuOpt_Def, name)) \
+		seq_printf(m, ",%s" #str, v ? "" : "no"); \
+} while (0)
+
+#define AuStr(name, str) do { \
+	v = mnt_flags & AuOptMask_##name; \
+	if (v != (AuOpt_Def & AuOptMask_##name)) \
+		seq_printf(m, "," #str "=%s", au_optstr_##str(v)); \
+} while (0)
+
+#define AuUInt(name, str, val) do { \
+	if (val != AUFS_##name##_DEF) \
+		seq_printf(m, "," #str "=%u", val); \
+} while (0)
+
+	sb = dentry->d_sb;
+	if (sb->s_flags & SB_POSIXACL)
+		seq_puts(m, ",acl");
+#if 0 /* reserved for future use */
+	if (sb->s_flags & SB_I_VERSION)
+		seq_puts(m, ",i_version");
+#endif
+
+	/* lock free root dinfo */
+	si_noflush_read_lock(sb);
+	sbinfo = au_sbi(sb);
+	seq_printf(m, ",si=%lx", sysaufs_si_id(sbinfo));
+
+	mnt_flags = au_mntflags(sb);
+	if (au_opt_test(mnt_flags, XINO)) {
+		err = au_show_xino(m, sb);
+		if (unlikely(err))
+			goto out;
+	} else
+		seq_puts(m, ",noxino");
+
+	AuBool(TRUNC_XINO, trunc_xino);
+	AuStr(UDBA, udba);
+	AuBool(SHWH, shwh);
+	AuBool(PLINK, plink);
+	AuBool(DIO, dio);
+	AuBool(DIRPERM1, dirperm1);
+
+	v = sbinfo->si_wbr_create;
+	if (v != AuWbrCreate_Def)
+		au_show_wbr_create(m, v, sbinfo);
+
+	v = sbinfo->si_wbr_copyup;
+	if (v != AuWbrCopyup_Def)
+		seq_printf(m, ",cpup=%s", au_optstr_wbr_copyup(v));
+
+	v = au_opt_test(mnt_flags, ALWAYS_DIROPQ);
+	if (v != au_opt_test(AuOpt_Def, ALWAYS_DIROPQ))
+		seq_printf(m, ",diropq=%c", v ? 'a' : 'w');
+
+	AuUInt(DIRWH, dirwh, sbinfo->si_dirwh);
+
+	v = jiffies_to_msecs(sbinfo->si_rdcache) / MSEC_PER_SEC;
+	AuUInt(RDCACHE, rdcache, v);
+
+	AuUInt(RDBLK, rdblk, sbinfo->si_rdblk);
+	AuUInt(RDHASH, rdhash, sbinfo->si_rdhash);
+
+	au_fhsm_show(m, sbinfo);
+
+	AuBool(DIRREN, dirren);
+	AuBool(SUM, sum);
+	/* AuBool(SUM_W, wsum); */
+	AuBool(WARN_PERM, warn_perm);
+	AuBool(VERBOSE, verbose);
+
+out:
+	/* be sure to print "br:" last */
+	if (!sysaufs_brs) {
+		seq_puts(m, ",br:");
+		au_show_brs(m, sb);
+	}
+	si_read_unlock(sb);
+	return 0;
+
+#undef AuBool
+#undef AuStr
+#undef AuUInt
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* sum mode which returns the summation for statfs(2) */
+
+static u64 au_add_till_max(u64 a, u64 b)
+{
+	u64 old;
+
+	old = a;
+	a += b;
+	if (old <= a)
+		return a;
+	return ULLONG_MAX;
+}
+
+static u64 au_mul_till_max(u64 a, long mul)
+{
+	u64 old;
+
+	old = a;
+	a *= mul;
+	if (old <= a)
+		return a;
+	return ULLONG_MAX;
+}
+
+static int au_statfs_sum(struct super_block *sb, struct kstatfs *buf)
+{
+	int err;
+	long bsize, factor;
+	u64 blocks, bfree, bavail, files, ffree;
+	aufs_bindex_t bbot, bindex, i;
+	unsigned char shared;
+	struct path h_path;
+	struct super_block *h_sb;
+
+	err = 0;
+	bsize = LONG_MAX;
+	files = 0;
+	ffree = 0;
+	blocks = 0;
+	bfree = 0;
+	bavail = 0;
+	bbot = au_sbbot(sb);
+	for (bindex = 0; bindex <= bbot; bindex++) {
+		h_path.mnt = au_sbr_mnt(sb, bindex);
+		h_sb = h_path.mnt->mnt_sb;
+		shared = 0;
+		for (i = 0; !shared && i < bindex; i++)
+			shared = (au_sbr_sb(sb, i) == h_sb);
+		if (shared)
+			continue;
+
+		/* sb->s_root for NFS is unreliable */
+		h_path.dentry = h_path.mnt->mnt_root;
+		err = vfs_statfs(&h_path, buf);
+		if (unlikely(err))
+			goto out;
+
+		if (bsize > buf->f_bsize) {
+			/*
+			 * we will reduce bsize, so we have to expand blocks
+			 * etc. to match them again
+			 */
+			factor = (bsize / buf->f_bsize);
+			blocks = au_mul_till_max(blocks, factor);
+			bfree = au_mul_till_max(bfree, factor);
+			bavail = au_mul_till_max(bavail, factor);
+			bsize = buf->f_bsize;
+		}
+
+		factor = (buf->f_bsize / bsize);
+		blocks = au_add_till_max(blocks,
+				au_mul_till_max(buf->f_blocks, factor));
+		bfree = au_add_till_max(bfree,
+				au_mul_till_max(buf->f_bfree, factor));
+		bavail = au_add_till_max(bavail,
+				au_mul_till_max(buf->f_bavail, factor));
+		files = au_add_till_max(files, buf->f_files);
+		ffree = au_add_till_max(ffree, buf->f_ffree);
+	}
+
+	buf->f_bsize = bsize;
+	buf->f_blocks = blocks;
+	buf->f_bfree = bfree;
+	buf->f_bavail = bavail;
+	buf->f_files = files;
+	buf->f_ffree = ffree;
+	buf->f_frsize = 0;
+
+out:
+	return err;
+}
+
+static int aufs_statfs(struct dentry *dentry, struct kstatfs *buf)
+{
+	int err;
+	struct path h_path;
+	struct super_block *sb;
+
+	/* lock free root dinfo */
+	sb = dentry->d_sb;
+	si_noflush_read_lock(sb);
+	if (!au_opt_test(au_mntflags(sb), SUM)) {
+		/* sb->s_root for NFS is unreliable */
+		h_path.mnt = au_sbr_mnt(sb, 0);
+		h_path.dentry = h_path.mnt->mnt_root;
+		err = vfs_statfs(&h_path, buf);
+	} else
+		err = au_statfs_sum(sb, buf);
+	si_read_unlock(sb);
+
+	if (!err) {
+		buf->f_type = AUFS_SUPER_MAGIC;
+		buf->f_namelen = AUFS_MAX_NAMELEN;
+		memset(&buf->f_fsid, 0, sizeof(buf->f_fsid));
+	}
+	/* buf->f_bsize = buf->f_blocks = buf->f_bfree = buf->f_bavail = -1; */
+
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static int aufs_sync_fs(struct super_block *sb, int wait)
+{
+	int err, e;
+	aufs_bindex_t bbot, bindex;
+	struct au_branch *br;
+	struct super_block *h_sb;
+
+	err = 0;
+	si_noflush_read_lock(sb);
+	bbot = au_sbbot(sb);
+	for (bindex = 0; bindex <= bbot; bindex++) {
+		br = au_sbr(sb, bindex);
+		if (!au_br_writable(br->br_perm))
+			continue;
+
+		h_sb = au_sbr_sb(sb, bindex);
+		e = vfsub_sync_filesystem(h_sb, wait);
+		if (unlikely(e && !err))
+			err = e;
+		/* go on even if an error happens */
+	}
+	si_read_unlock(sb);
+
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* final actions when unmounting a file system */
+static void aufs_put_super(struct super_block *sb)
+{
+	struct au_sbinfo *sbinfo;
+
+	sbinfo = au_sbi(sb);
+	if (sbinfo)
+		kobject_put(&sbinfo->si_kobj);
+}
+
+/* ---------------------------------------------------------------------- */
+
+void *au_array_alloc(unsigned long long *hint, au_arraycb_t cb,
+		     struct super_block *sb, void *arg)
+{
+	void *array;
+	unsigned long long n, sz;
+
+	array = NULL;
+	n = 0;
+	if (!*hint)
+		goto out;
+
+	if (*hint > ULLONG_MAX / sizeof(array)) {
+		array = ERR_PTR(-EMFILE);
+		pr_err("hint %llu\n", *hint);
+		goto out;
+	}
+
+	sz = sizeof(array) * *hint;
+	array = kzalloc(sz, GFP_NOFS);
+	if (unlikely(!array))
+		array = vzalloc(sz);
+	if (unlikely(!array)) {
+		array = ERR_PTR(-ENOMEM);
+		goto out;
+	}
+
+	n = cb(sb, array, *hint, arg);
+	AuDebugOn(n > *hint);
+
+out:
+	*hint = n;
+	return array;
+}
+
+static unsigned long long au_iarray_cb(struct super_block *sb, void *a,
+				       unsigned long long max __maybe_unused,
+				       void *arg)
+{
+	unsigned long long n;
+	struct inode **p, *inode;
+	struct list_head *head;
+
+	n = 0;
+	p = a;
+	head = arg;
+	spin_lock(&sb->s_inode_list_lock);
+	list_for_each_entry(inode, head, i_sb_list) {
+		if (!au_is_bad_inode(inode)
+		    && au_ii(inode)->ii_btop >= 0) {
+			spin_lock(&inode->i_lock);
+			if (atomic_read(&inode->i_count)) {
+				au_igrab(inode);
+				*p++ = inode;
+				n++;
+				AuDebugOn(n > max);
+			}
+			spin_unlock(&inode->i_lock);
+		}
+	}
+	spin_unlock(&sb->s_inode_list_lock);
+
+	return n;
+}
+
+struct inode **au_iarray_alloc(struct super_block *sb, unsigned long long *max)
+{
+	struct au_sbinfo *sbi;
+
+	sbi = au_sbi(sb);
+	*max = au_lcnt_read(&sbi->si_ninodes, /*do_rev*/1);
+	return au_array_alloc(max, au_iarray_cb, sb, &sb->s_inodes);
+}
+
+void au_iarray_free(struct inode **a, unsigned long long max)
+{
+	unsigned long long ull;
+
+	for (ull = 0; ull < max; ull++)
+		iput(a[ull]);
+	kvfree(a);
+}
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * refresh dentry and inode at remount time.
+ */
+/* todo: consolidate with simple_reval_dpath() and au_reval_for_attr() */
+static int au_do_refresh(struct dentry *dentry, unsigned int dir_flags,
+		      struct dentry *parent)
+{
+	int err;
+
+	di_write_lock_child(dentry);
+	di_read_lock_parent(parent, AuLock_IR);
+	err = au_refresh_dentry(dentry, parent);
+	if (!err && dir_flags)
+		au_hn_reset(d_inode(dentry), dir_flags);
+	di_read_unlock(parent, AuLock_IR);
+	di_write_unlock(dentry);
+
+	return err;
+}
+
+static int au_do_refresh_d(struct dentry *dentry, unsigned int sigen,
+			   struct au_sbinfo *sbinfo,
+			   const unsigned int dir_flags, unsigned int do_idop)
+{
+	int err;
+	struct dentry *parent;
+
+	err = 0;
+	parent = dget_parent(dentry);
+	if (!au_digen_test(parent, sigen) && au_digen_test(dentry, sigen)) {
+		if (d_really_is_positive(dentry)) {
+			if (!d_is_dir(dentry))
+				err = au_do_refresh(dentry, /*dir_flags*/0,
+						 parent);
+			else {
+				err = au_do_refresh(dentry, dir_flags, parent);
+				if (unlikely(err))
+					au_fset_si(sbinfo, FAILED_REFRESH_DIR);
+			}
+		} else
+			err = au_do_refresh(dentry, /*dir_flags*/0, parent);
+		AuDbgDentry(dentry);
+	}
+	dput(parent);
+
+	if (!err) {
+		if (do_idop)
+			au_refresh_dop(dentry, /*force_reval*/0);
+	} else
+		au_refresh_dop(dentry, /*force_reval*/1);
+
+	AuTraceErr(err);
+	return err;
+}
+
+static int au_refresh_d(struct super_block *sb, unsigned int do_idop)
+{
+	int err, i, j, ndentry, e;
+	unsigned int sigen;
+	struct au_dcsub_pages dpages;
+	struct au_dpage *dpage;
+	struct dentry **dentries, *d;
+	struct au_sbinfo *sbinfo;
+	struct dentry *root = sb->s_root;
+	const unsigned int dir_flags = au_hi_flags(d_inode(root), /*isdir*/1);
+
+	if (do_idop)
+		au_refresh_dop(root, /*force_reval*/0);
+
+	err = au_dpages_init(&dpages, GFP_NOFS);
+	if (unlikely(err))
+		goto out;
+	err = au_dcsub_pages(&dpages, root, NULL, NULL);
+	if (unlikely(err))
+		goto out_dpages;
+
+	sigen = au_sigen(sb);
+	sbinfo = au_sbi(sb);
+	for (i = 0; i < dpages.ndpage; i++) {
+		dpage = dpages.dpages + i;
+		dentries = dpage->dentries;
+		ndentry = dpage->ndentry;
+		for (j = 0; j < ndentry; j++) {
+			d = dentries[j];
+			e = au_do_refresh_d(d, sigen, sbinfo, dir_flags,
+					    do_idop);
+			if (unlikely(e && !err))
+				err = e;
+			/* go on even err */
+		}
+	}
+
+out_dpages:
+	au_dpages_free(&dpages);
+out:
+	return err;
+}
+
+static int au_refresh_i(struct super_block *sb, unsigned int do_idop)
+{
+	int err, e;
+	unsigned int sigen;
+	unsigned long long max, ull;
+	struct inode *inode, **array;
+
+	array = au_iarray_alloc(sb, &max);
+	err = PTR_ERR(array);
+	if (IS_ERR(array))
+		goto out;
+
+	err = 0;
+	sigen = au_sigen(sb);
+	for (ull = 0; ull < max; ull++) {
+		inode = array[ull];
+		if (unlikely(!inode))
+			break;
+
+		e = 0;
+		ii_write_lock_child(inode);
+		if (au_iigen(inode, NULL) != sigen) {
+			e = au_refresh_hinode_self(inode);
+			if (unlikely(e)) {
+				au_refresh_iop(inode, /*force_getattr*/1);
+				pr_err("error %d, i%lu\n", e, inode->i_ino);
+				if (!err)
+					err = e;
+				/* go on even if err */
+			}
+		}
+		if (!e && do_idop)
+			au_refresh_iop(inode, /*force_getattr*/0);
+		ii_write_unlock(inode);
+	}
+
+	au_iarray_free(array, max);
+
+out:
+	return err;
+}
+
+static void au_remount_refresh(struct super_block *sb, unsigned int do_idop)
+{
+	int err, e;
+	unsigned int udba;
+	aufs_bindex_t bindex, bbot;
+	struct dentry *root;
+	struct inode *inode;
+	struct au_branch *br;
+	struct au_sbinfo *sbi;
+
+	au_sigen_inc(sb);
+	sbi = au_sbi(sb);
+	au_fclr_si(sbi, FAILED_REFRESH_DIR);
+
+	root = sb->s_root;
+	DiMustNoWaiters(root);
+	inode = d_inode(root);
+	IiMustNoWaiters(inode);
+
+	udba = au_opt_udba(sb);
+	bbot = au_sbbot(sb);
+	for (bindex = 0; bindex <= bbot; bindex++) {
+		br = au_sbr(sb, bindex);
+		err = au_hnotify_reset_br(udba, br, br->br_perm);
+		if (unlikely(err))
+			AuIOErr("hnotify failed on br %d, %d, ignored\n",
+				bindex, err);
+		/* go on even if err */
+	}
+	au_hn_reset(inode, au_hi_flags(inode, /*isdir*/1));
+
+	if (do_idop) {
+		if (au_ftest_si(sbi, NO_DREVAL)) {
+			AuDebugOn(sb->s_d_op == &aufs_dop_noreval);
+			sb->s_d_op = &aufs_dop_noreval;
+			AuDebugOn(sbi->si_iop_array == aufs_iop_nogetattr);
+			sbi->si_iop_array = aufs_iop_nogetattr;
+		} else {
+			AuDebugOn(sb->s_d_op == &aufs_dop);
+			sb->s_d_op = &aufs_dop;
+			AuDebugOn(sbi->si_iop_array == aufs_iop);
+			sbi->si_iop_array = aufs_iop;
+		}
+		pr_info("reset to %ps and %ps\n",
+			sb->s_d_op, sbi->si_iop_array);
+	}
+
+	di_write_unlock(root);
+	err = au_refresh_d(sb, do_idop);
+	e = au_refresh_i(sb, do_idop);
+	if (unlikely(e && !err))
+		err = e;
+	/* aufs_write_lock() calls ..._child() */
+	di_write_lock_child(root);
+
+	au_cpup_attr_all(inode, /*force*/1);
+
+	if (unlikely(err))
+		AuIOErr("refresh failed, ignored, %d\n", err);
+}
+
+/* stop extra interpretation of errno in mount(8), and strange error messages */
+static int cvt_err(int err)
+{
+	AuTraceErr(err);
+
+	switch (err) {
+	case -ENOENT:
+	case -ENOTDIR:
+	case -EEXIST:
+	case -EIO:
+		err = -EINVAL;
+	}
+	return err;
+}
+
+static int aufs_remount_fs(struct super_block *sb, int *flags, char *data)
+{
+	int err, do_dx;
+	unsigned int mntflags;
+	struct au_opts opts = {
+		.opt = NULL
+	};
+	struct dentry *root;
+	struct inode *inode;
+	struct au_sbinfo *sbinfo;
+
+	err = 0;
+	root = sb->s_root;
+	if (!data || !*data) {
+		err = si_write_lock(sb, AuLock_FLUSH | AuLock_NOPLM);
+		if (!err) {
+			di_write_lock_child(root);
+			err = au_opts_verify(sb, *flags, /*pending*/0);
+			aufs_write_unlock(root);
+		}
+		goto out;
+	}
+
+	err = -ENOMEM;
+	opts.opt = (void *)__get_free_page(GFP_NOFS);
+	if (unlikely(!opts.opt))
+		goto out;
+	opts.max_opt = PAGE_SIZE / sizeof(*opts.opt);
+	opts.flags = AuOpts_REMOUNT;
+	opts.sb_flags = *flags;
+
+	/* parse it before aufs lock */
+	err = au_opts_parse(sb, data, &opts);
+	if (unlikely(err))
+		goto out_opts;
+
+	sbinfo = au_sbi(sb);
+	inode = d_inode(root);
+	inode_lock(inode);
+	err = si_write_lock(sb, AuLock_FLUSH | AuLock_NOPLM);
+	if (unlikely(err))
+		goto out_mtx;
+	di_write_lock_child(root);
+
+	/* au_opts_remount() may return an error */
+	err = au_opts_remount(sb, &opts);
+	au_opts_free(&opts);
+
+	if (au_ftest_opts(opts.flags, REFRESH))
+		au_remount_refresh(sb, au_ftest_opts(opts.flags, REFRESH_IDOP));
+
+	if (au_ftest_opts(opts.flags, REFRESH_DYAOP)) {
+		mntflags = au_mntflags(sb);
+		do_dx = !!au_opt_test(mntflags, DIO);
+		au_dy_arefresh(do_dx);
+	}
+
+	au_fhsm_wrote_all(sb, /*force*/1); /* ?? */
+	aufs_write_unlock(root);
+
+out_mtx:
+	inode_unlock(inode);
+out_opts:
+	free_page((unsigned long)opts.opt);
+out:
+	err = cvt_err(err);
+	AuTraceErr(err);
+	return err;
+}
+
+static const struct super_operations aufs_sop = {
+	.alloc_inode	= aufs_alloc_inode,
+	.destroy_inode	= aufs_destroy_inode,
+	/* always deleting, no clearing */
+	.drop_inode	= generic_delete_inode,
+	.show_options	= aufs_show_options,
+	.statfs		= aufs_statfs,
+	.put_super	= aufs_put_super,
+	.sync_fs	= aufs_sync_fs,
+	.remount_fs	= aufs_remount_fs
+};
+
+/* ---------------------------------------------------------------------- */
+
+static int alloc_root(struct super_block *sb)
+{
+	int err;
+	struct inode *inode;
+	struct dentry *root;
+
+	err = -ENOMEM;
+	inode = au_iget_locked(sb, AUFS_ROOT_INO);
+	err = PTR_ERR(inode);
+	if (IS_ERR(inode))
+		goto out;
+
+	inode->i_op = aufs_iop + AuIop_DIR; /* with getattr by default */
+	inode->i_fop = &aufs_dir_fop;
+	inode->i_mode = S_IFDIR;
+	set_nlink(inode, 2);
+	unlock_new_inode(inode);
+
+	root = d_make_root(inode);
+	if (unlikely(!root))
+		goto out;
+	err = PTR_ERR(root);
+	if (IS_ERR(root))
+		goto out;
+
+	err = au_di_init(root);
+	if (!err) {
+		sb->s_root = root;
+		return 0; /* success */
+	}
+	dput(root);
+
+out:
+	return err;
+}
+
+static int aufs_fill_super(struct super_block *sb, void *raw_data,
+			   int silent __maybe_unused)
+{
+	int err;
+	struct au_opts opts = {
+		.opt = NULL
+	};
+	struct au_sbinfo *sbinfo;
+	struct dentry *root;
+	struct inode *inode;
+	char *arg = raw_data;
+
+	if (unlikely(!arg || !*arg)) {
+		err = -EINVAL;
+		pr_err("no arg\n");
+		goto out;
+	}
+
+	err = -ENOMEM;
+	opts.opt = (void *)__get_free_page(GFP_NOFS);
+	if (unlikely(!opts.opt))
+		goto out;
+	opts.max_opt = PAGE_SIZE / sizeof(*opts.opt);
+	opts.sb_flags = sb->s_flags;
+
+	err = au_si_alloc(sb);
+	if (unlikely(err))
+		goto out_opts;
+	sbinfo = au_sbi(sb);
+
+	/* all timestamps always follow the ones on the branch */
+	sb->s_flags |= SB_NOATIME | SB_NODIRATIME;
+	sb->s_flags |= SB_I_VERSION; /* do we really need this? */
+	sb->s_op = &aufs_sop;
+	sb->s_d_op = &aufs_dop;
+	sb->s_magic = AUFS_SUPER_MAGIC;
+	sb->s_maxbytes = 0;
+	sb->s_stack_depth = 1;
+	au_export_init(sb);
+	au_xattr_init(sb);
+
+	err = alloc_root(sb);
+	if (unlikely(err)) {
+		si_write_unlock(sb);
+		goto out_info;
+	}
+	root = sb->s_root;
+	inode = d_inode(root);
+
+	/*
+	 * actually we can parse options regardless aufs lock here.
+	 * but at remount time, parsing must be done before aufs lock.
+	 * so we follow the same rule.
+	 */
+	ii_write_lock_parent(inode);
+	aufs_write_unlock(root);
+	err = au_opts_parse(sb, arg, &opts);
+	if (unlikely(err))
+		goto out_root;
+
+	/* lock vfs_inode first, then aufs. */
+	inode_lock(inode);
+	aufs_write_lock(root);
+	err = au_opts_mount(sb, &opts);
+	au_opts_free(&opts);
+	if (!err && au_ftest_si(sbinfo, NO_DREVAL)) {
+		sb->s_d_op = &aufs_dop_noreval;
+		pr_info("%ps\n", sb->s_d_op);
+		au_refresh_dop(root, /*force_reval*/0);
+		sbinfo->si_iop_array = aufs_iop_nogetattr;
+		au_refresh_iop(inode, /*force_getattr*/0);
+	}
+	aufs_write_unlock(root);
+	inode_unlock(inode);
+	if (!err)
+		goto out_opts; /* success */
+
+out_root:
+	dput(root);
+	sb->s_root = NULL;
+out_info:
+	kobject_put(&sbinfo->si_kobj);
+	sb->s_fs_info = NULL;
+out_opts:
+	free_page((unsigned long)opts.opt);
+out:
+	AuTraceErr(err);
+	err = cvt_err(err);
+	AuTraceErr(err);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static struct dentry *aufs_mount(struct file_system_type *fs_type, int flags,
+				 const char *dev_name __maybe_unused,
+				 void *raw_data)
+{
+	struct dentry *root;
+
+	/* all timestamps always follow the ones on the branch */
+	/* mnt->mnt_flags |= MNT_NOATIME | MNT_NODIRATIME; */
+	root = mount_nodev(fs_type, flags, raw_data, aufs_fill_super);
+	if (IS_ERR(root))
+		goto out;
+
+	au_sbilist_add(root->d_sb);
+
+out:
+	return root;
+}
+
+static void aufs_kill_sb(struct super_block *sb)
+{
+	struct au_sbinfo *sbinfo;
+
+	sbinfo = au_sbi(sb);
+	if (sbinfo) {
+		au_sbilist_del(sb);
+		aufs_write_lock(sb->s_root);
+		au_fhsm_fin(sb);
+		if (sbinfo->si_wbr_create_ops->fin)
+			sbinfo->si_wbr_create_ops->fin(sb);
+		if (au_opt_test(sbinfo->si_mntflags, UDBA_HNOTIFY)) {
+			au_opt_set_udba(sbinfo->si_mntflags, UDBA_NONE);
+			au_remount_refresh(sb, /*do_idop*/0);
+		}
+		if (au_opt_test(sbinfo->si_mntflags, PLINK))
+			au_plink_put(sb, /*verbose*/1);
+		au_xino_clr(sb);
+		au_dr_opt_flush(sb);
+		sbinfo->si_sb = NULL;
+		aufs_write_unlock(sb->s_root);
+		au_nwt_flush(&sbinfo->si_nowait);
+	}
+	kill_anon_super(sb);
+}
+
+struct file_system_type aufs_fs_type = {
+	.name		= AUFS_FSTYPE,
+	/* a race between rename and others */
+	.fs_flags	= FS_RENAME_DOES_D_MOVE,
+	.mount		= aufs_mount,
+	.kill_sb	= aufs_kill_sb,
+	/* no need to __module_get() and module_put(). */
+	.owner		= THIS_MODULE,
+};
diff -Naur --no-dereference a/fs/aufs/super.h b/fs/aufs/super.h
--- a/fs/aufs/super.h	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/super.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,589 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * super_block operations
+ */
+
+#ifndef __AUFS_SUPER_H__
+#define __AUFS_SUPER_H__
+
+#ifdef __KERNEL__
+
+#include <linux/fs.h>
+#include <linux/kobject.h>
+#include "hbl.h"
+#include "lcnt.h"
+#include "rwsem.h"
+#include "wkq.h"
+
+/* policies to select one among multiple writable branches */
+struct au_wbr_copyup_operations {
+	int (*copyup)(struct dentry *dentry);
+};
+
+#define AuWbr_DIR	1		/* target is a dir */
+#define AuWbr_PARENT	(1 << 1)	/* always require a parent */
+
+#define au_ftest_wbr(flags, name)	((flags) & AuWbr_##name)
+#define au_fset_wbr(flags, name)	{ (flags) |= AuWbr_##name; }
+#define au_fclr_wbr(flags, name)	{ (flags) &= ~AuWbr_##name; }
+
+struct au_wbr_create_operations {
+	int (*create)(struct dentry *dentry, unsigned int flags);
+	int (*init)(struct super_block *sb);
+	int (*fin)(struct super_block *sb);
+};
+
+struct au_wbr_mfs {
+	struct mutex	mfs_lock; /* protect this structure */
+	unsigned long	mfs_jiffy;
+	unsigned long	mfs_expire;
+	aufs_bindex_t	mfs_bindex;
+
+	unsigned long long	mfsrr_bytes;
+	unsigned long long	mfsrr_watermark;
+};
+
+#define AuPlink_NHASH 100
+static inline int au_plink_hash(ino_t ino)
+{
+	return ino % AuPlink_NHASH;
+}
+
+/* File-based Hierarchical Storage Management */
+struct au_fhsm {
+#ifdef CONFIG_AUFS_FHSM
+	/* allow only one process who can receive the notification */
+	spinlock_t		fhsm_spin;
+	pid_t			fhsm_pid;
+	wait_queue_head_t	fhsm_wqh;
+	atomic_t		fhsm_readable;
+
+	/* these are protected by si_rwsem */
+	unsigned long		fhsm_expire;
+	aufs_bindex_t		fhsm_bottom;
+#endif
+};
+
+struct au_branch;
+struct au_sbinfo {
+	/* nowait tasks in the system-wide workqueue */
+	struct au_nowait_tasks	si_nowait;
+
+	/*
+	 * tried sb->s_umount, but failed due to the dependency between i_mutex.
+	 * rwsem for au_sbinfo is necessary.
+	 */
+	struct au_rwsem		si_rwsem;
+
+	/*
+	 * dirty approach to protect sb->sb_inodes and ->s_files (gone) from
+	 * remount.
+	 */
+	au_lcnt_t		si_ninodes, si_nfiles;
+
+	/* branch management */
+	unsigned int		si_generation;
+
+	/* see AuSi_ flags */
+	unsigned char		au_si_status;
+
+	aufs_bindex_t		si_bbot;
+
+	/* dirty trick to keep br_id plus */
+	unsigned int		si_last_br_id :
+				sizeof(aufs_bindex_t) * BITS_PER_BYTE - 1;
+	struct au_branch	**si_branch;
+
+	/* policy to select a writable branch */
+	unsigned char		si_wbr_copyup;
+	unsigned char		si_wbr_create;
+	struct au_wbr_copyup_operations *si_wbr_copyup_ops;
+	struct au_wbr_create_operations *si_wbr_create_ops;
+
+	/* round robin */
+	atomic_t		si_wbr_rr_next;
+
+	/* most free space */
+	struct au_wbr_mfs	si_wbr_mfs;
+
+	/* File-based Hierarchical Storage Management */
+	struct au_fhsm		si_fhsm;
+
+	/* mount flags */
+	/* include/asm-ia64/siginfo.h defines a macro named si_flags */
+	unsigned int		si_mntflags;
+
+	/* external inode number (bitmap and translation table) */
+	vfs_readf_t		si_xread;
+	vfs_writef_t		si_xwrite;
+	loff_t			si_ximaxent;	/* max entries in a xino */
+
+	struct file		*si_xib;
+	struct mutex		si_xib_mtx; /* protect xib members */
+	unsigned long		*si_xib_buf;
+	unsigned long		si_xib_last_pindex;
+	int			si_xib_next_bit;
+
+	unsigned long		si_xino_jiffy;
+	unsigned long		si_xino_expire;
+	/* reserved for future use */
+	/* unsigned long long	si_xib_limit; */	/* Max xib file size */
+
+#ifdef CONFIG_AUFS_EXPORT
+	/* i_generation */
+	/* todo: make xigen file an array to support many inode numbers */
+	struct file		*si_xigen;
+	atomic_t		si_xigen_next;
+#endif
+
+	/* dirty trick to support atomic_open */
+	struct hlist_bl_head	si_aopen;
+
+	/* vdir parameters */
+	unsigned long		si_rdcache;	/* max cache time in jiffies */
+	unsigned int		si_rdblk;	/* deblk size */
+	unsigned int		si_rdhash;	/* hash size */
+
+	/*
+	 * If the number of whiteouts are larger than si_dirwh, leave all of
+	 * them after au_whtmp_ren to reduce the cost of rmdir(2).
+	 * future fsck.aufs or kernel thread will remove them later.
+	 * Otherwise, remove all whiteouts and the dir in rmdir(2).
+	 */
+	unsigned int		si_dirwh;
+
+	/* pseudo_link list */
+	struct hlist_bl_head	si_plink[AuPlink_NHASH];
+	wait_queue_head_t	si_plink_wq;
+	spinlock_t		si_plink_maint_lock;
+	pid_t			si_plink_maint_pid;
+
+	/* file list */
+	struct hlist_bl_head	si_files;
+
+	/* with/without getattr, brother of sb->s_d_op */
+	const struct inode_operations *si_iop_array;
+
+	/*
+	 * sysfs and lifetime management.
+	 * this is not a small structure and it may be a waste of memory in case
+	 * of sysfs is disabled, particularly when many aufs-es are mounted.
+	 * but using sysfs is majority.
+	 */
+	struct kobject		si_kobj;
+#ifdef CONFIG_DEBUG_FS
+	struct dentry		 *si_dbgaufs;
+	struct dentry		 *si_dbgaufs_plink;
+	struct dentry		 *si_dbgaufs_xib;
+#ifdef CONFIG_AUFS_EXPORT
+	struct dentry		 *si_dbgaufs_xigen;
+#endif
+#endif
+
+#ifdef CONFIG_AUFS_SBILIST
+	struct hlist_bl_node	si_list;
+#endif
+
+	/* dirty, necessary for unmounting, sysfs and sysrq */
+	struct super_block	*si_sb;
+};
+
+/* sbinfo status flags */
+/*
+ * set true when refresh_dirs() failed at remount time.
+ * then try refreshing dirs at access time again.
+ * if it is false, refreshing dirs at access time is unnecessary
+ */
+#define AuSi_FAILED_REFRESH_DIR	1
+#define AuSi_FHSM		(1 << 1)	/* fhsm is active now */
+#define AuSi_NO_DREVAL		(1 << 2)	/* disable all d_revalidate */
+
+#ifndef CONFIG_AUFS_FHSM
+#undef AuSi_FHSM
+#define AuSi_FHSM		0
+#endif
+
+static inline unsigned char au_do_ftest_si(struct au_sbinfo *sbi,
+					   unsigned int flag)
+{
+	AuRwMustAnyLock(&sbi->si_rwsem);
+	return sbi->au_si_status & flag;
+}
+#define au_ftest_si(sbinfo, name)	au_do_ftest_si(sbinfo, AuSi_##name)
+#define au_fset_si(sbinfo, name) do { \
+	AuRwMustWriteLock(&(sbinfo)->si_rwsem); \
+	(sbinfo)->au_si_status |= AuSi_##name; \
+} while (0)
+#define au_fclr_si(sbinfo, name) do { \
+	AuRwMustWriteLock(&(sbinfo)->si_rwsem); \
+	(sbinfo)->au_si_status &= ~AuSi_##name; \
+} while (0)
+
+/* ---------------------------------------------------------------------- */
+
+/* policy to select one among writable branches */
+#define AuWbrCopyup(sbinfo, ...) \
+	((sbinfo)->si_wbr_copyup_ops->copyup(__VA_ARGS__))
+#define AuWbrCreate(sbinfo, ...) \
+	((sbinfo)->si_wbr_create_ops->create(__VA_ARGS__))
+
+/* flags for si_read_lock()/aufs_read_lock()/di_read_lock() */
+#define AuLock_DW		1		/* write-lock dentry */
+#define AuLock_IR		(1 << 1)	/* read-lock inode */
+#define AuLock_IW		(1 << 2)	/* write-lock inode */
+#define AuLock_FLUSH		(1 << 3)	/* wait for 'nowait' tasks */
+#define AuLock_DIRS		(1 << 4)	/* target is a pair of dirs */
+						/* except RENAME_EXCHANGE */
+#define AuLock_NOPLM		(1 << 5)	/* return err in plm mode */
+#define AuLock_NOPLMW		(1 << 6)	/* wait for plm mode ends */
+#define AuLock_GEN		(1 << 7)	/* test digen/iigen */
+#define au_ftest_lock(flags, name)	((flags) & AuLock_##name)
+#define au_fset_lock(flags, name) \
+	do { (flags) |= AuLock_##name; } while (0)
+#define au_fclr_lock(flags, name) \
+	do { (flags) &= ~AuLock_##name; } while (0)
+
+/* ---------------------------------------------------------------------- */
+
+/* super.c */
+extern struct file_system_type aufs_fs_type;
+struct inode *au_iget_locked(struct super_block *sb, ino_t ino);
+typedef unsigned long long (*au_arraycb_t)(struct super_block *sb, void *array,
+					   unsigned long long max, void *arg);
+void *au_array_alloc(unsigned long long *hint, au_arraycb_t cb,
+		     struct super_block *sb, void *arg);
+struct inode **au_iarray_alloc(struct super_block *sb, unsigned long long *max);
+void au_iarray_free(struct inode **a, unsigned long long max);
+
+/* sbinfo.c */
+void au_si_free(struct kobject *kobj);
+int au_si_alloc(struct super_block *sb);
+int au_sbr_realloc(struct au_sbinfo *sbinfo, int nbr, int may_shrink);
+
+unsigned int au_sigen_inc(struct super_block *sb);
+aufs_bindex_t au_new_br_id(struct super_block *sb);
+
+int si_read_lock(struct super_block *sb, int flags);
+int si_write_lock(struct super_block *sb, int flags);
+int aufs_read_lock(struct dentry *dentry, int flags);
+void aufs_read_unlock(struct dentry *dentry, int flags);
+void aufs_write_lock(struct dentry *dentry);
+void aufs_write_unlock(struct dentry *dentry);
+int aufs_read_and_write_lock2(struct dentry *d1, struct dentry *d2, int flags);
+void aufs_read_and_write_unlock2(struct dentry *d1, struct dentry *d2);
+
+/* wbr_policy.c */
+extern struct au_wbr_copyup_operations au_wbr_copyup_ops[];
+extern struct au_wbr_create_operations au_wbr_create_ops[];
+int au_cpdown_dirs(struct dentry *dentry, aufs_bindex_t bdst);
+int au_wbr_nonopq(struct dentry *dentry, aufs_bindex_t bindex);
+int au_wbr_do_copyup_bu(struct dentry *dentry, aufs_bindex_t btop);
+
+/* mvdown.c */
+int au_mvdown(struct dentry *dentry, struct aufs_mvdown __user *arg);
+
+#ifdef CONFIG_AUFS_FHSM
+/* fhsm.c */
+
+static inline pid_t au_fhsm_pid(struct au_fhsm *fhsm)
+{
+	pid_t pid;
+
+	spin_lock(&fhsm->fhsm_spin);
+	pid = fhsm->fhsm_pid;
+	spin_unlock(&fhsm->fhsm_spin);
+
+	return pid;
+}
+
+void au_fhsm_wrote(struct super_block *sb, aufs_bindex_t bindex, int force);
+void au_fhsm_wrote_all(struct super_block *sb, int force);
+int au_fhsm_fd(struct super_block *sb, int oflags);
+int au_fhsm_br_alloc(struct au_branch *br);
+void au_fhsm_set_bottom(struct super_block *sb, aufs_bindex_t bindex);
+void au_fhsm_fin(struct super_block *sb);
+void au_fhsm_init(struct au_sbinfo *sbinfo);
+void au_fhsm_set(struct au_sbinfo *sbinfo, unsigned int sec);
+void au_fhsm_show(struct seq_file *seq, struct au_sbinfo *sbinfo);
+#else
+AuStubVoid(au_fhsm_wrote, struct super_block *sb, aufs_bindex_t bindex,
+	   int force)
+AuStubVoid(au_fhsm_wrote_all, struct super_block *sb, int force)
+AuStub(int, au_fhsm_fd, return -EOPNOTSUPP, struct super_block *sb, int oflags)
+AuStub(pid_t, au_fhsm_pid, return 0, struct au_fhsm *fhsm)
+AuStubInt0(au_fhsm_br_alloc, struct au_branch *br)
+AuStubVoid(au_fhsm_set_bottom, struct super_block *sb, aufs_bindex_t bindex)
+AuStubVoid(au_fhsm_fin, struct super_block *sb)
+AuStubVoid(au_fhsm_init, struct au_sbinfo *sbinfo)
+AuStubVoid(au_fhsm_set, struct au_sbinfo *sbinfo, unsigned int sec)
+AuStubVoid(au_fhsm_show, struct seq_file *seq, struct au_sbinfo *sbinfo)
+#endif
+
+/* ---------------------------------------------------------------------- */
+
+static inline struct au_sbinfo *au_sbi(struct super_block *sb)
+{
+	return sb->s_fs_info;
+}
+
+/* ---------------------------------------------------------------------- */
+
+#ifdef CONFIG_AUFS_EXPORT
+int au_test_nfsd(void);
+void au_export_init(struct super_block *sb);
+void au_xigen_inc(struct inode *inode);
+int au_xigen_new(struct inode *inode);
+int au_xigen_set(struct super_block *sb, struct path *path);
+void au_xigen_clr(struct super_block *sb);
+
+static inline int au_busy_or_stale(void)
+{
+	if (!au_test_nfsd())
+		return -EBUSY;
+	return -ESTALE;
+}
+#else
+AuStubInt0(au_test_nfsd, void)
+AuStubVoid(au_export_init, struct super_block *sb)
+AuStubVoid(au_xigen_inc, struct inode *inode)
+AuStubInt0(au_xigen_new, struct inode *inode)
+AuStubInt0(au_xigen_set, struct super_block *sb, struct path *path)
+AuStubVoid(au_xigen_clr, struct super_block *sb)
+AuStub(int, au_busy_or_stale, return -EBUSY, void)
+#endif /* CONFIG_AUFS_EXPORT */
+
+/* ---------------------------------------------------------------------- */
+
+#ifdef CONFIG_AUFS_SBILIST
+/* module.c */
+extern struct hlist_bl_head au_sbilist;
+
+static inline void au_sbilist_init(void)
+{
+	INIT_HLIST_BL_HEAD(&au_sbilist);
+}
+
+static inline void au_sbilist_add(struct super_block *sb)
+{
+	au_hbl_add(&au_sbi(sb)->si_list, &au_sbilist);
+}
+
+static inline void au_sbilist_del(struct super_block *sb)
+{
+	au_hbl_del(&au_sbi(sb)->si_list, &au_sbilist);
+}
+
+#ifdef CONFIG_AUFS_MAGIC_SYSRQ
+static inline void au_sbilist_lock(void)
+{
+	hlist_bl_lock(&au_sbilist);
+}
+
+static inline void au_sbilist_unlock(void)
+{
+	hlist_bl_unlock(&au_sbilist);
+}
+#define AuGFP_SBILIST	GFP_ATOMIC
+#else
+AuStubVoid(au_sbilist_lock, void)
+AuStubVoid(au_sbilist_unlock, void)
+#define AuGFP_SBILIST	GFP_NOFS
+#endif /* CONFIG_AUFS_MAGIC_SYSRQ */
+#else
+AuStubVoid(au_sbilist_init, void)
+AuStubVoid(au_sbilist_add, struct super_block *sb)
+AuStubVoid(au_sbilist_del, struct super_block *sb)
+AuStubVoid(au_sbilist_lock, void)
+AuStubVoid(au_sbilist_unlock, void)
+#define AuGFP_SBILIST	GFP_NOFS
+#endif
+
+/* ---------------------------------------------------------------------- */
+
+static inline void dbgaufs_si_null(struct au_sbinfo *sbinfo)
+{
+	/*
+	 * This function is a dynamic '__init' function actually,
+	 * so the tiny check for si_rwsem is unnecessary.
+	 */
+	/* AuRwMustWriteLock(&sbinfo->si_rwsem); */
+#ifdef CONFIG_DEBUG_FS
+	sbinfo->si_dbgaufs = NULL;
+	sbinfo->si_dbgaufs_plink = NULL;
+	sbinfo->si_dbgaufs_xib = NULL;
+#ifdef CONFIG_AUFS_EXPORT
+	sbinfo->si_dbgaufs_xigen = NULL;
+#endif
+#endif
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* current->atomic_flags */
+/* this value should never corrupt the ones defined in linux/sched.h */
+#define PFA_AUFS	7
+
+TASK_PFA_TEST(AUFS, test_aufs)	/* task_test_aufs */
+TASK_PFA_SET(AUFS, aufs)	/* task_set_aufs */
+TASK_PFA_CLEAR(AUFS, aufs)	/* task_clear_aufs */
+
+static inline int si_pid_test(struct super_block *sb)
+{
+	return !!task_test_aufs(current);
+}
+
+static inline void si_pid_clr(struct super_block *sb)
+{
+	AuDebugOn(!task_test_aufs(current));
+	task_clear_aufs(current);
+}
+
+static inline void si_pid_set(struct super_block *sb)
+{
+	AuDebugOn(task_test_aufs(current));
+	task_set_aufs(current);
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* lock superblock. mainly for entry point functions */
+#define __si_read_lock(sb)	au_rw_read_lock(&au_sbi(sb)->si_rwsem)
+#define __si_write_lock(sb)	au_rw_write_lock(&au_sbi(sb)->si_rwsem)
+#define __si_read_trylock(sb)	au_rw_read_trylock(&au_sbi(sb)->si_rwsem)
+#define __si_write_trylock(sb)	au_rw_write_trylock(&au_sbi(sb)->si_rwsem)
+/*
+#define __si_read_trylock_nested(sb) \
+	au_rw_read_trylock_nested(&au_sbi(sb)->si_rwsem)
+#define __si_write_trylock_nested(sb) \
+	au_rw_write_trylock_nested(&au_sbi(sb)->si_rwsem)
+*/
+
+#define __si_read_unlock(sb)	au_rw_read_unlock(&au_sbi(sb)->si_rwsem)
+#define __si_write_unlock(sb)	au_rw_write_unlock(&au_sbi(sb)->si_rwsem)
+#define __si_downgrade_lock(sb)	au_rw_dgrade_lock(&au_sbi(sb)->si_rwsem)
+
+#define SiMustNoWaiters(sb)	AuRwMustNoWaiters(&au_sbi(sb)->si_rwsem)
+#define SiMustAnyLock(sb)	AuRwMustAnyLock(&au_sbi(sb)->si_rwsem)
+#define SiMustWriteLock(sb)	AuRwMustWriteLock(&au_sbi(sb)->si_rwsem)
+
+static inline void si_noflush_read_lock(struct super_block *sb)
+{
+	__si_read_lock(sb);
+	si_pid_set(sb);
+}
+
+static inline int si_noflush_read_trylock(struct super_block *sb)
+{
+	int locked;
+
+	locked = __si_read_trylock(sb);
+	if (locked)
+		si_pid_set(sb);
+	return locked;
+}
+
+static inline void si_noflush_write_lock(struct super_block *sb)
+{
+	__si_write_lock(sb);
+	si_pid_set(sb);
+}
+
+static inline int si_noflush_write_trylock(struct super_block *sb)
+{
+	int locked;
+
+	locked = __si_write_trylock(sb);
+	if (locked)
+		si_pid_set(sb);
+	return locked;
+}
+
+#if 0 /* reserved */
+static inline int si_read_trylock(struct super_block *sb, int flags)
+{
+	if (au_ftest_lock(flags, FLUSH))
+		au_nwt_flush(&au_sbi(sb)->si_nowait);
+	return si_noflush_read_trylock(sb);
+}
+#endif
+
+static inline void si_read_unlock(struct super_block *sb)
+{
+	si_pid_clr(sb);
+	__si_read_unlock(sb);
+}
+
+#if 0 /* reserved */
+static inline int si_write_trylock(struct super_block *sb, int flags)
+{
+	if (au_ftest_lock(flags, FLUSH))
+		au_nwt_flush(&au_sbi(sb)->si_nowait);
+	return si_noflush_write_trylock(sb);
+}
+#endif
+
+static inline void si_write_unlock(struct super_block *sb)
+{
+	si_pid_clr(sb);
+	__si_write_unlock(sb);
+}
+
+#if 0 /* reserved */
+static inline void si_downgrade_lock(struct super_block *sb)
+{
+	__si_downgrade_lock(sb);
+}
+#endif
+
+/* ---------------------------------------------------------------------- */
+
+static inline aufs_bindex_t au_sbbot(struct super_block *sb)
+{
+	SiMustAnyLock(sb);
+	return au_sbi(sb)->si_bbot;
+}
+
+static inline unsigned int au_mntflags(struct super_block *sb)
+{
+	SiMustAnyLock(sb);
+	return au_sbi(sb)->si_mntflags;
+}
+
+static inline unsigned int au_sigen(struct super_block *sb)
+{
+	SiMustAnyLock(sb);
+	return au_sbi(sb)->si_generation;
+}
+
+static inline struct au_branch *au_sbr(struct super_block *sb,
+				       aufs_bindex_t bindex)
+{
+	SiMustAnyLock(sb);
+	return au_sbi(sb)->si_branch[0 + bindex];
+}
+
+static inline loff_t au_xi_maxent(struct super_block *sb)
+{
+	SiMustAnyLock(sb);
+	return au_sbi(sb)->si_ximaxent;
+}
+
+#endif /* __KERNEL__ */
+#endif /* __AUFS_SUPER_H__ */
diff -Naur --no-dereference a/fs/aufs/sysaufs.c b/fs/aufs/sysaufs.c
--- a/fs/aufs/sysaufs.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/sysaufs.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,93 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * sysfs interface and lifetime management
+ * they are necessary regardless sysfs is disabled.
+ */
+
+#include <linux/random.h>
+#include "aufs.h"
+
+unsigned long sysaufs_si_mask;
+struct kset *sysaufs_kset;
+
+#define AuSiAttr(_name) { \
+	.attr   = { .name = __stringify(_name), .mode = 0444 },	\
+	.show   = sysaufs_si_##_name,				\
+}
+
+static struct sysaufs_si_attr sysaufs_si_attr_xi_path = AuSiAttr(xi_path);
+struct attribute *sysaufs_si_attrs[] = {
+	&sysaufs_si_attr_xi_path.attr,
+	NULL,
+};
+
+static const struct sysfs_ops au_sbi_ops = {
+	.show   = sysaufs_si_show
+};
+
+static struct kobj_type au_sbi_ktype = {
+	.release	= au_si_free,
+	.sysfs_ops	= &au_sbi_ops,
+	.default_attrs	= sysaufs_si_attrs
+};
+
+/* ---------------------------------------------------------------------- */
+
+int sysaufs_si_init(struct au_sbinfo *sbinfo)
+{
+	int err;
+
+	sbinfo->si_kobj.kset = sysaufs_kset;
+	/* cf. sysaufs_name() */
+	err = kobject_init_and_add
+		(&sbinfo->si_kobj, &au_sbi_ktype, /*&sysaufs_kset->kobj*/NULL,
+		 SysaufsSiNamePrefix "%lx", sysaufs_si_id(sbinfo));
+
+	return err;
+}
+
+void sysaufs_fin(void)
+{
+	sysfs_remove_group(&sysaufs_kset->kobj, sysaufs_attr_group);
+	kset_unregister(sysaufs_kset);
+}
+
+int __init sysaufs_init(void)
+{
+	int err;
+
+	do {
+		get_random_bytes(&sysaufs_si_mask, sizeof(sysaufs_si_mask));
+	} while (!sysaufs_si_mask);
+
+	err = -EINVAL;
+	sysaufs_kset = kset_create_and_add(AUFS_NAME, NULL, fs_kobj);
+	if (unlikely(!sysaufs_kset))
+		goto out;
+	err = PTR_ERR(sysaufs_kset);
+	if (IS_ERR(sysaufs_kset))
+		goto out;
+	err = sysfs_create_group(&sysaufs_kset->kobj, sysaufs_attr_group);
+	if (unlikely(err))
+		kset_unregister(sysaufs_kset);
+
+out:
+	return err;
+}
diff -Naur --no-dereference a/fs/aufs/sysaufs.h b/fs/aufs/sysaufs.h
--- a/fs/aufs/sysaufs.h	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/sysaufs.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,102 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * sysfs interface and mount lifetime management
+ */
+
+#ifndef __SYSAUFS_H__
+#define __SYSAUFS_H__
+
+#ifdef __KERNEL__
+
+#include <linux/sysfs.h>
+#include "module.h"
+
+struct super_block;
+struct au_sbinfo;
+
+struct sysaufs_si_attr {
+	struct attribute attr;
+	int (*show)(struct seq_file *seq, struct super_block *sb);
+};
+
+/* ---------------------------------------------------------------------- */
+
+/* sysaufs.c */
+extern unsigned long sysaufs_si_mask;
+extern struct kset *sysaufs_kset;
+extern struct attribute *sysaufs_si_attrs[];
+int sysaufs_si_init(struct au_sbinfo *sbinfo);
+int __init sysaufs_init(void);
+void sysaufs_fin(void);
+
+/* ---------------------------------------------------------------------- */
+
+/* some people doesn't like to show a pointer in kernel */
+static inline unsigned long sysaufs_si_id(struct au_sbinfo *sbinfo)
+{
+	return sysaufs_si_mask ^ (unsigned long)sbinfo;
+}
+
+#define SysaufsSiNamePrefix	"si_"
+#define SysaufsSiNameLen	(sizeof(SysaufsSiNamePrefix) + 16)
+static inline void sysaufs_name(struct au_sbinfo *sbinfo, char *name)
+{
+	snprintf(name, SysaufsSiNameLen, SysaufsSiNamePrefix "%lx",
+		 sysaufs_si_id(sbinfo));
+}
+
+struct au_branch;
+#ifdef CONFIG_SYSFS
+/* sysfs.c */
+extern struct attribute_group *sysaufs_attr_group;
+
+int sysaufs_si_xi_path(struct seq_file *seq, struct super_block *sb);
+ssize_t sysaufs_si_show(struct kobject *kobj, struct attribute *attr,
+			 char *buf);
+long au_brinfo_ioctl(struct file *file, unsigned long arg);
+#ifdef CONFIG_COMPAT
+long au_brinfo_compat_ioctl(struct file *file, unsigned long arg);
+#endif
+
+void sysaufs_br_init(struct au_branch *br);
+void sysaufs_brs_add(struct super_block *sb, aufs_bindex_t bindex);
+void sysaufs_brs_del(struct super_block *sb, aufs_bindex_t bindex);
+
+#define sysaufs_brs_init()	do {} while (0)
+
+#else
+#define sysaufs_attr_group	NULL
+
+AuStubInt0(sysaufs_si_xi_path, struct seq_file *seq, struct super_block *sb)
+AuStub(ssize_t, sysaufs_si_show, return 0, struct kobject *kobj,
+       struct attribute *attr, char *buf)
+AuStubVoid(sysaufs_br_init, struct au_branch *br)
+AuStubVoid(sysaufs_brs_add, struct super_block *sb, aufs_bindex_t bindex)
+AuStubVoid(sysaufs_brs_del, struct super_block *sb, aufs_bindex_t bindex)
+
+static inline void sysaufs_brs_init(void)
+{
+	sysaufs_brs = 0;
+}
+
+#endif /* CONFIG_SYSFS */
+
+#endif /* __KERNEL__ */
+#endif /* __SYSAUFS_H__ */
diff -Naur --no-dereference a/fs/aufs/sysfs.c b/fs/aufs/sysfs.c
--- a/fs/aufs/sysfs.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/sysfs.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,373 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * sysfs interface
+ */
+
+#include <linux/compat.h>
+#include <linux/seq_file.h>
+#include "aufs.h"
+
+#ifdef CONFIG_AUFS_FS_MODULE
+/* this entry violates the "one line per file" policy of sysfs */
+static ssize_t config_show(struct kobject *kobj, struct kobj_attribute *attr,
+			   char *buf)
+{
+	ssize_t err;
+	static char *conf =
+/* this file is generated at compiling */
+#include "conf.str"
+		;
+
+	err = snprintf(buf, PAGE_SIZE, conf);
+	if (unlikely(err >= PAGE_SIZE))
+		err = -EFBIG;
+	return err;
+}
+
+static struct kobj_attribute au_config_attr = __ATTR_RO(config);
+#endif
+
+static struct attribute *au_attr[] = {
+#ifdef CONFIG_AUFS_FS_MODULE
+	&au_config_attr.attr,
+#endif
+	NULL,	/* need to NULL terminate the list of attributes */
+};
+
+static struct attribute_group sysaufs_attr_group_body = {
+	.attrs = au_attr
+};
+
+struct attribute_group *sysaufs_attr_group = &sysaufs_attr_group_body;
+
+/* ---------------------------------------------------------------------- */
+
+int sysaufs_si_xi_path(struct seq_file *seq, struct super_block *sb)
+{
+	int err;
+
+	SiMustAnyLock(sb);
+
+	err = 0;
+	if (au_opt_test(au_mntflags(sb), XINO)) {
+		err = au_xino_path(seq, au_sbi(sb)->si_xib);
+		seq_putc(seq, '\n');
+	}
+	return err;
+}
+
+/*
+ * the lifetime of branch is independent from the entry under sysfs.
+ * sysfs handles the lifetime of the entry, and never call ->show() after it is
+ * unlinked.
+ */
+static int sysaufs_si_br(struct seq_file *seq, struct super_block *sb,
+			 aufs_bindex_t bindex, int idx)
+{
+	int err;
+	struct path path;
+	struct dentry *root;
+	struct au_branch *br;
+	au_br_perm_str_t perm;
+
+	AuDbg("b%d\n", bindex);
+
+	err = 0;
+	root = sb->s_root;
+	di_read_lock_parent(root, !AuLock_IR);
+	br = au_sbr(sb, bindex);
+
+	switch (idx) {
+	case AuBrSysfs_BR:
+		path.mnt = au_br_mnt(br);
+		path.dentry = au_h_dptr(root, bindex);
+		err = au_seq_path(seq, &path);
+		if (!err) {
+			au_optstr_br_perm(&perm, br->br_perm);
+			seq_printf(seq, "=%s\n", perm.a);
+		}
+		break;
+	case AuBrSysfs_BRID:
+		seq_printf(seq, "%d\n", br->br_id);
+		break;
+	}
+	di_read_unlock(root, !AuLock_IR);
+	if (unlikely(err || seq_has_overflowed(seq)))
+		err = -E2BIG;
+
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static struct seq_file *au_seq(char *p, ssize_t len)
+{
+	struct seq_file *seq;
+
+	seq = kzalloc(sizeof(*seq), GFP_NOFS);
+	if (seq) {
+		/* mutex_init(&seq.lock); */
+		seq->buf = p;
+		seq->size = len;
+		return seq; /* success */
+	}
+
+	seq = ERR_PTR(-ENOMEM);
+	return seq;
+}
+
+#define SysaufsBr_PREFIX	"br"
+#define SysaufsBrid_PREFIX	"brid"
+
+/* todo: file size may exceed PAGE_SIZE */
+ssize_t sysaufs_si_show(struct kobject *kobj, struct attribute *attr,
+			char *buf)
+{
+	ssize_t err;
+	int idx;
+	long l;
+	aufs_bindex_t bbot;
+	struct au_sbinfo *sbinfo;
+	struct super_block *sb;
+	struct seq_file *seq;
+	char *name;
+	struct attribute **cattr;
+
+	sbinfo = container_of(kobj, struct au_sbinfo, si_kobj);
+	sb = sbinfo->si_sb;
+
+	/*
+	 * prevent a race condition between sysfs and aufs.
+	 * for instance, sysfs_file_read() calls sysfs_get_active_two() which
+	 * prohibits maintaining the sysfs entries.
+	 * hew we acquire read lock after sysfs_get_active_two().
+	 * on the other hand, the remount process may maintain the sysfs/aufs
+	 * entries after acquiring write lock.
+	 * it can cause a deadlock.
+	 * simply we gave up processing read here.
+	 */
+	err = -EBUSY;
+	if (unlikely(!si_noflush_read_trylock(sb)))
+		goto out;
+
+	seq = au_seq(buf, PAGE_SIZE);
+	err = PTR_ERR(seq);
+	if (IS_ERR(seq))
+		goto out_unlock;
+
+	name = (void *)attr->name;
+	cattr = sysaufs_si_attrs;
+	while (*cattr) {
+		if (!strcmp(name, (*cattr)->name)) {
+			err = container_of(*cattr, struct sysaufs_si_attr, attr)
+				->show(seq, sb);
+			goto out_seq;
+		}
+		cattr++;
+	}
+
+	if (!strncmp(name, SysaufsBrid_PREFIX,
+		     sizeof(SysaufsBrid_PREFIX) - 1)) {
+		idx = AuBrSysfs_BRID;
+		name += sizeof(SysaufsBrid_PREFIX) - 1;
+	} else if (!strncmp(name, SysaufsBr_PREFIX,
+			    sizeof(SysaufsBr_PREFIX) - 1)) {
+		idx = AuBrSysfs_BR;
+		name += sizeof(SysaufsBr_PREFIX) - 1;
+	} else
+		  BUG();
+
+	err = kstrtol(name, 10, &l);
+	if (!err) {
+		bbot = au_sbbot(sb);
+		if (l <= bbot)
+			err = sysaufs_si_br(seq, sb, (aufs_bindex_t)l, idx);
+		else
+			err = -ENOENT;
+	}
+
+out_seq:
+	if (!err) {
+		err = seq->count;
+		/* sysfs limit */
+		if (unlikely(err == PAGE_SIZE))
+			err = -EFBIG;
+	}
+	au_kfree_rcu(seq);
+out_unlock:
+	si_read_unlock(sb);
+out:
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static int au_brinfo(struct super_block *sb, union aufs_brinfo __user *arg)
+{
+	int err;
+	int16_t brid;
+	aufs_bindex_t bindex, bbot;
+	size_t sz;
+	char *buf;
+	struct seq_file *seq;
+	struct au_branch *br;
+
+	si_read_lock(sb, AuLock_FLUSH);
+	bbot = au_sbbot(sb);
+	err = bbot + 1;
+	if (!arg)
+		goto out;
+
+	err = -ENOMEM;
+	buf = (void *)__get_free_page(GFP_NOFS);
+	if (unlikely(!buf))
+		goto out;
+
+	seq = au_seq(buf, PAGE_SIZE);
+	err = PTR_ERR(seq);
+	if (IS_ERR(seq))
+		goto out_buf;
+
+	sz = sizeof(*arg) - offsetof(union aufs_brinfo, path);
+	for (bindex = 0; bindex <= bbot; bindex++, arg++) {
+		err = !access_ok(VERIFY_WRITE, arg, sizeof(*arg));
+		if (unlikely(err))
+			break;
+
+		br = au_sbr(sb, bindex);
+		brid = br->br_id;
+		BUILD_BUG_ON(sizeof(brid) != sizeof(arg->id));
+		err = __put_user(brid, &arg->id);
+		if (unlikely(err))
+			break;
+
+		BUILD_BUG_ON(sizeof(br->br_perm) != sizeof(arg->perm));
+		err = __put_user(br->br_perm, &arg->perm);
+		if (unlikely(err))
+			break;
+
+		err = au_seq_path(seq, &br->br_path);
+		if (unlikely(err))
+			break;
+		seq_putc(seq, '\0');
+		if (!seq_has_overflowed(seq)) {
+			err = copy_to_user(arg->path, seq->buf, seq->count);
+			seq->count = 0;
+			if (unlikely(err))
+				break;
+		} else {
+			err = -E2BIG;
+			goto out_seq;
+		}
+	}
+	if (unlikely(err))
+		err = -EFAULT;
+
+out_seq:
+	au_kfree_rcu(seq);
+out_buf:
+	free_page((unsigned long)buf);
+out:
+	si_read_unlock(sb);
+	return err;
+}
+
+long au_brinfo_ioctl(struct file *file, unsigned long arg)
+{
+	return au_brinfo(file->f_path.dentry->d_sb, (void __user *)arg);
+}
+
+#ifdef CONFIG_COMPAT
+long au_brinfo_compat_ioctl(struct file *file, unsigned long arg)
+{
+	return au_brinfo(file->f_path.dentry->d_sb, compat_ptr(arg));
+}
+#endif
+
+/* ---------------------------------------------------------------------- */
+
+void sysaufs_br_init(struct au_branch *br)
+{
+	int i;
+	struct au_brsysfs *br_sysfs;
+	struct attribute *attr;
+
+	br_sysfs = br->br_sysfs;
+	for (i = 0; i < ARRAY_SIZE(br->br_sysfs); i++) {
+		attr = &br_sysfs->attr;
+		sysfs_attr_init(attr);
+		attr->name = br_sysfs->name;
+		attr->mode = 0444;
+		br_sysfs++;
+	}
+}
+
+void sysaufs_brs_del(struct super_block *sb, aufs_bindex_t bindex)
+{
+	struct au_branch *br;
+	struct kobject *kobj;
+	struct au_brsysfs *br_sysfs;
+	int i;
+	aufs_bindex_t bbot;
+
+	if (!sysaufs_brs)
+		return;
+
+	kobj = &au_sbi(sb)->si_kobj;
+	bbot = au_sbbot(sb);
+	for (; bindex <= bbot; bindex++) {
+		br = au_sbr(sb, bindex);
+		br_sysfs = br->br_sysfs;
+		for (i = 0; i < ARRAY_SIZE(br->br_sysfs); i++) {
+			sysfs_remove_file(kobj, &br_sysfs->attr);
+			br_sysfs++;
+		}
+	}
+}
+
+void sysaufs_brs_add(struct super_block *sb, aufs_bindex_t bindex)
+{
+	int err, i;
+	aufs_bindex_t bbot;
+	struct kobject *kobj;
+	struct au_branch *br;
+	struct au_brsysfs *br_sysfs;
+
+	if (!sysaufs_brs)
+		return;
+
+	kobj = &au_sbi(sb)->si_kobj;
+	bbot = au_sbbot(sb);
+	for (; bindex <= bbot; bindex++) {
+		br = au_sbr(sb, bindex);
+		br_sysfs = br->br_sysfs;
+		snprintf(br_sysfs[AuBrSysfs_BR].name, sizeof(br_sysfs->name),
+			 SysaufsBr_PREFIX "%d", bindex);
+		snprintf(br_sysfs[AuBrSysfs_BRID].name, sizeof(br_sysfs->name),
+			 SysaufsBrid_PREFIX "%d", bindex);
+		for (i = 0; i < ARRAY_SIZE(br->br_sysfs); i++) {
+			err = sysfs_create_file(kobj, &br_sysfs->attr);
+			if (unlikely(err))
+				pr_warn("failed %s under sysfs(%d)\n",
+					br_sysfs->name, err);
+			br_sysfs++;
+		}
+	}
+}
diff -Naur --no-dereference a/fs/aufs/sysrq.c b/fs/aufs/sysrq.c
--- a/fs/aufs/sysrq.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/sysrq.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,149 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * magic sysrq handler
+ */
+
+/* #include <linux/sysrq.h> */
+#include <linux/writeback.h>
+#include "aufs.h"
+
+/* ---------------------------------------------------------------------- */
+
+static void sysrq_sb(struct super_block *sb)
+{
+	char *plevel;
+	struct au_sbinfo *sbinfo;
+	struct file *file;
+	struct hlist_bl_head *files;
+	struct hlist_bl_node *pos;
+	struct au_finfo *finfo;
+	struct inode *i;
+
+	plevel = au_plevel;
+	au_plevel = KERN_WARNING;
+
+	/* since we define pr_fmt, call printk directly */
+#define pr(str) printk(KERN_WARNING AUFS_NAME ": " str)
+
+	sbinfo = au_sbi(sb);
+	printk(KERN_WARNING "si=%lx\n", sysaufs_si_id(sbinfo));
+	pr("superblock\n");
+	au_dpri_sb(sb);
+
+#if 0 /* reserved */
+	do {
+		int err, i, j, ndentry;
+		struct au_dcsub_pages dpages;
+		struct au_dpage *dpage;
+
+		err = au_dpages_init(&dpages, GFP_ATOMIC);
+		if (unlikely(err))
+			break;
+		err = au_dcsub_pages(&dpages, sb->s_root, NULL, NULL);
+		if (!err)
+			for (i = 0; i < dpages.ndpage; i++) {
+				dpage = dpages.dpages + i;
+				ndentry = dpage->ndentry;
+				for (j = 0; j < ndentry; j++)
+					au_dpri_dentry(dpage->dentries[j]);
+			}
+		au_dpages_free(&dpages);
+	} while (0);
+#endif
+
+	pr("isolated inode\n");
+	spin_lock(&sb->s_inode_list_lock);
+	list_for_each_entry(i, &sb->s_inodes, i_sb_list) {
+		spin_lock(&i->i_lock);
+		if (hlist_empty(&i->i_dentry))
+			au_dpri_inode(i);
+		spin_unlock(&i->i_lock);
+	}
+	spin_unlock(&sb->s_inode_list_lock);
+
+	pr("files\n");
+	files = &au_sbi(sb)->si_files;
+	hlist_bl_lock(files);
+	hlist_bl_for_each_entry(finfo, pos, files, fi_hlist) {
+		umode_t mode;
+
+		file = finfo->fi_file;
+		mode = file_inode(file)->i_mode;
+		if (!special_file(mode))
+			au_dpri_file(file);
+	}
+	hlist_bl_unlock(files);
+	pr("done\n");
+
+#undef pr
+	au_plevel = plevel;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* module parameter */
+static char *aufs_sysrq_key = "a";
+module_param_named(sysrq, aufs_sysrq_key, charp, 0444);
+MODULE_PARM_DESC(sysrq, "MagicSysRq key for " AUFS_NAME);
+
+static void au_sysrq(int key __maybe_unused)
+{
+	struct au_sbinfo *sbinfo;
+	struct hlist_bl_node *pos;
+
+	lockdep_off();
+	au_sbilist_lock();
+	hlist_bl_for_each_entry(sbinfo, pos, &au_sbilist, si_list)
+		sysrq_sb(sbinfo->si_sb);
+	au_sbilist_unlock();
+	lockdep_on();
+}
+
+static struct sysrq_key_op au_sysrq_op = {
+	.handler	= au_sysrq,
+	.help_msg	= "Aufs",
+	.action_msg	= "Aufs",
+	.enable_mask	= SYSRQ_ENABLE_DUMP
+};
+
+/* ---------------------------------------------------------------------- */
+
+int __init au_sysrq_init(void)
+{
+	int err;
+	char key;
+
+	err = -1;
+	key = *aufs_sysrq_key;
+	if ('a' <= key && key <= 'z')
+		err = register_sysrq_key(key, &au_sysrq_op);
+	if (unlikely(err))
+		pr_err("err %d, sysrq=%c\n", err, key);
+	return err;
+}
+
+void au_sysrq_fin(void)
+{
+	int err;
+
+	err = unregister_sysrq_key(*aufs_sysrq_key, &au_sysrq_op);
+	if (unlikely(err))
+		pr_err("err %d (ignored)\n", err);
+}
diff -Naur --no-dereference a/fs/aufs/vdir.c b/fs/aufs/vdir.c
--- a/fs/aufs/vdir.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/vdir.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,896 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * virtual or vertical directory
+ */
+
+#include <linux/iversion.h>
+#include "aufs.h"
+
+static unsigned int calc_size(int nlen)
+{
+	return ALIGN(sizeof(struct au_vdir_de) + nlen, sizeof(ino_t));
+}
+
+static int set_deblk_end(union au_vdir_deblk_p *p,
+			 union au_vdir_deblk_p *deblk_end)
+{
+	if (calc_size(0) <= deblk_end->deblk - p->deblk) {
+		p->de->de_str.len = 0;
+		/* smp_mb(); */
+		return 0;
+	}
+	return -1; /* error */
+}
+
+/* returns true or false */
+static int is_deblk_end(union au_vdir_deblk_p *p,
+			union au_vdir_deblk_p *deblk_end)
+{
+	if (calc_size(0) <= deblk_end->deblk - p->deblk)
+		return !p->de->de_str.len;
+	return 1;
+}
+
+static unsigned char *last_deblk(struct au_vdir *vdir)
+{
+	return vdir->vd_deblk[vdir->vd_nblk - 1];
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* estimate the appropriate size for name hash table */
+unsigned int au_rdhash_est(loff_t sz)
+{
+	unsigned int n;
+
+	n = UINT_MAX;
+	sz >>= 10;
+	if (sz < n)
+		n = sz;
+	if (sz < AUFS_RDHASH_DEF)
+		n = AUFS_RDHASH_DEF;
+	/* pr_info("n %u\n", n); */
+	return n;
+}
+
+/*
+ * the allocated memory has to be freed by
+ * au_nhash_wh_free() or au_nhash_de_free().
+ */
+int au_nhash_alloc(struct au_nhash *nhash, unsigned int num_hash, gfp_t gfp)
+{
+	struct hlist_head *head;
+	unsigned int u;
+	size_t sz;
+
+	sz = sizeof(*nhash->nh_head) * num_hash;
+	head = kmalloc(sz, gfp);
+	if (head) {
+		nhash->nh_num = num_hash;
+		nhash->nh_head = head;
+		for (u = 0; u < num_hash; u++)
+			INIT_HLIST_HEAD(head++);
+		return 0; /* success */
+	}
+
+	return -ENOMEM;
+}
+
+static void nhash_count(struct hlist_head *head)
+{
+#if 0 /* debugging */
+	unsigned long n;
+	struct hlist_node *pos;
+
+	n = 0;
+	hlist_for_each(pos, head)
+		n++;
+	pr_info("%lu\n", n);
+#endif
+}
+
+static void au_nhash_wh_do_free(struct hlist_head *head)
+{
+	struct au_vdir_wh *pos;
+	struct hlist_node *node;
+
+	hlist_for_each_entry_safe(pos, node, head, wh_hash)
+		au_kfree_rcu(pos);
+}
+
+static void au_nhash_de_do_free(struct hlist_head *head)
+{
+	struct au_vdir_dehstr *pos;
+	struct hlist_node *node;
+
+	hlist_for_each_entry_safe(pos, node, head, hash)
+		au_cache_free_vdir_dehstr(pos);
+}
+
+static void au_nhash_do_free(struct au_nhash *nhash,
+			     void (*free)(struct hlist_head *head))
+{
+	unsigned int n;
+	struct hlist_head *head;
+
+	n = nhash->nh_num;
+	if (!n)
+		return;
+
+	head = nhash->nh_head;
+	while (n-- > 0) {
+		nhash_count(head);
+		free(head++);
+	}
+	au_kfree_try_rcu(nhash->nh_head);
+}
+
+void au_nhash_wh_free(struct au_nhash *whlist)
+{
+	au_nhash_do_free(whlist, au_nhash_wh_do_free);
+}
+
+static void au_nhash_de_free(struct au_nhash *delist)
+{
+	au_nhash_do_free(delist, au_nhash_de_do_free);
+}
+
+/* ---------------------------------------------------------------------- */
+
+int au_nhash_test_longer_wh(struct au_nhash *whlist, aufs_bindex_t btgt,
+			    int limit)
+{
+	int num;
+	unsigned int u, n;
+	struct hlist_head *head;
+	struct au_vdir_wh *pos;
+
+	num = 0;
+	n = whlist->nh_num;
+	head = whlist->nh_head;
+	for (u = 0; u < n; u++, head++)
+		hlist_for_each_entry(pos, head, wh_hash)
+			if (pos->wh_bindex == btgt && ++num > limit)
+				return 1;
+	return 0;
+}
+
+static struct hlist_head *au_name_hash(struct au_nhash *nhash,
+				       unsigned char *name,
+				       unsigned int len)
+{
+	unsigned int v;
+	/* const unsigned int magic_bit = 12; */
+
+	AuDebugOn(!nhash->nh_num || !nhash->nh_head);
+
+	v = 0;
+	if (len > 8)
+		len = 8;
+	while (len--)
+		v += *name++;
+	/* v = hash_long(v, magic_bit); */
+	v %= nhash->nh_num;
+	return nhash->nh_head + v;
+}
+
+static int au_nhash_test_name(struct au_vdir_destr *str, const char *name,
+			      int nlen)
+{
+	return str->len == nlen && !memcmp(str->name, name, nlen);
+}
+
+/* returns found or not */
+int au_nhash_test_known_wh(struct au_nhash *whlist, char *name, int nlen)
+{
+	struct hlist_head *head;
+	struct au_vdir_wh *pos;
+	struct au_vdir_destr *str;
+
+	head = au_name_hash(whlist, name, nlen);
+	hlist_for_each_entry(pos, head, wh_hash) {
+		str = &pos->wh_str;
+		AuDbg("%.*s\n", str->len, str->name);
+		if (au_nhash_test_name(str, name, nlen))
+			return 1;
+	}
+	return 0;
+}
+
+/* returns found(true) or not */
+static int test_known(struct au_nhash *delist, char *name, int nlen)
+{
+	struct hlist_head *head;
+	struct au_vdir_dehstr *pos;
+	struct au_vdir_destr *str;
+
+	head = au_name_hash(delist, name, nlen);
+	hlist_for_each_entry(pos, head, hash) {
+		str = pos->str;
+		AuDbg("%.*s\n", str->len, str->name);
+		if (au_nhash_test_name(str, name, nlen))
+			return 1;
+	}
+	return 0;
+}
+
+static void au_shwh_init_wh(struct au_vdir_wh *wh, ino_t ino,
+			    unsigned char d_type)
+{
+#ifdef CONFIG_AUFS_SHWH
+	wh->wh_ino = ino;
+	wh->wh_type = d_type;
+#endif
+}
+
+/* ---------------------------------------------------------------------- */
+
+int au_nhash_append_wh(struct au_nhash *whlist, char *name, int nlen, ino_t ino,
+		       unsigned int d_type, aufs_bindex_t bindex,
+		       unsigned char shwh)
+{
+	int err;
+	struct au_vdir_destr *str;
+	struct au_vdir_wh *wh;
+
+	AuDbg("%.*s\n", nlen, name);
+	AuDebugOn(!whlist->nh_num || !whlist->nh_head);
+
+	err = -ENOMEM;
+	wh = kmalloc(sizeof(*wh) + nlen, GFP_NOFS);
+	if (unlikely(!wh))
+		goto out;
+
+	err = 0;
+	wh->wh_bindex = bindex;
+	if (shwh)
+		au_shwh_init_wh(wh, ino, d_type);
+	str = &wh->wh_str;
+	str->len = nlen;
+	memcpy(str->name, name, nlen);
+	hlist_add_head(&wh->wh_hash, au_name_hash(whlist, name, nlen));
+	/* smp_mb(); */
+
+out:
+	return err;
+}
+
+static int append_deblk(struct au_vdir *vdir)
+{
+	int err;
+	unsigned long ul;
+	const unsigned int deblk_sz = vdir->vd_deblk_sz;
+	union au_vdir_deblk_p p, deblk_end;
+	unsigned char **o;
+
+	err = -ENOMEM;
+	o = au_krealloc(vdir->vd_deblk, sizeof(*o) * (vdir->vd_nblk + 1),
+			GFP_NOFS, /*may_shrink*/0);
+	if (unlikely(!o))
+		goto out;
+
+	vdir->vd_deblk = o;
+	p.deblk = kmalloc(deblk_sz, GFP_NOFS);
+	if (p.deblk) {
+		ul = vdir->vd_nblk++;
+		vdir->vd_deblk[ul] = p.deblk;
+		vdir->vd_last.ul = ul;
+		vdir->vd_last.p.deblk = p.deblk;
+		deblk_end.deblk = p.deblk + deblk_sz;
+		err = set_deblk_end(&p, &deblk_end);
+	}
+
+out:
+	return err;
+}
+
+static int append_de(struct au_vdir *vdir, char *name, int nlen, ino_t ino,
+		     unsigned int d_type, struct au_nhash *delist)
+{
+	int err;
+	unsigned int sz;
+	const unsigned int deblk_sz = vdir->vd_deblk_sz;
+	union au_vdir_deblk_p p, *room, deblk_end;
+	struct au_vdir_dehstr *dehstr;
+
+	p.deblk = last_deblk(vdir);
+	deblk_end.deblk = p.deblk + deblk_sz;
+	room = &vdir->vd_last.p;
+	AuDebugOn(room->deblk < p.deblk || deblk_end.deblk <= room->deblk
+		  || !is_deblk_end(room, &deblk_end));
+
+	sz = calc_size(nlen);
+	if (unlikely(sz > deblk_end.deblk - room->deblk)) {
+		err = append_deblk(vdir);
+		if (unlikely(err))
+			goto out;
+
+		p.deblk = last_deblk(vdir);
+		deblk_end.deblk = p.deblk + deblk_sz;
+		/* smp_mb(); */
+		AuDebugOn(room->deblk != p.deblk);
+	}
+
+	err = -ENOMEM;
+	dehstr = au_cache_alloc_vdir_dehstr();
+	if (unlikely(!dehstr))
+		goto out;
+
+	dehstr->str = &room->de->de_str;
+	hlist_add_head(&dehstr->hash, au_name_hash(delist, name, nlen));
+	room->de->de_ino = ino;
+	room->de->de_type = d_type;
+	room->de->de_str.len = nlen;
+	memcpy(room->de->de_str.name, name, nlen);
+
+	err = 0;
+	room->deblk += sz;
+	if (unlikely(set_deblk_end(room, &deblk_end)))
+		err = append_deblk(vdir);
+	/* smp_mb(); */
+
+out:
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+void au_vdir_free(struct au_vdir *vdir)
+{
+	unsigned char **deblk;
+
+	deblk = vdir->vd_deblk;
+	while (vdir->vd_nblk--)
+		au_kfree_try_rcu(*deblk++);
+	au_kfree_try_rcu(vdir->vd_deblk);
+	au_cache_free_vdir(vdir);
+}
+
+static struct au_vdir *alloc_vdir(struct file *file)
+{
+	struct au_vdir *vdir;
+	struct super_block *sb;
+	int err;
+
+	sb = file->f_path.dentry->d_sb;
+	SiMustAnyLock(sb);
+
+	err = -ENOMEM;
+	vdir = au_cache_alloc_vdir();
+	if (unlikely(!vdir))
+		goto out;
+
+	vdir->vd_deblk = kzalloc(sizeof(*vdir->vd_deblk), GFP_NOFS);
+	if (unlikely(!vdir->vd_deblk))
+		goto out_free;
+
+	vdir->vd_deblk_sz = au_sbi(sb)->si_rdblk;
+	if (!vdir->vd_deblk_sz) {
+		/* estimate the appropriate size for deblk */
+		vdir->vd_deblk_sz = au_dir_size(file, /*dentry*/NULL);
+		/* pr_info("vd_deblk_sz %u\n", vdir->vd_deblk_sz); */
+	}
+	vdir->vd_nblk = 0;
+	vdir->vd_version = 0;
+	vdir->vd_jiffy = 0;
+	err = append_deblk(vdir);
+	if (!err)
+		return vdir; /* success */
+
+	au_kfree_try_rcu(vdir->vd_deblk);
+
+out_free:
+	au_cache_free_vdir(vdir);
+out:
+	vdir = ERR_PTR(err);
+	return vdir;
+}
+
+static int reinit_vdir(struct au_vdir *vdir)
+{
+	int err;
+	union au_vdir_deblk_p p, deblk_end;
+
+	while (vdir->vd_nblk > 1) {
+		au_kfree_try_rcu(vdir->vd_deblk[vdir->vd_nblk - 1]);
+		/* vdir->vd_deblk[vdir->vd_nblk - 1] = NULL; */
+		vdir->vd_nblk--;
+	}
+	p.deblk = vdir->vd_deblk[0];
+	deblk_end.deblk = p.deblk + vdir->vd_deblk_sz;
+	err = set_deblk_end(&p, &deblk_end);
+	/* keep vd_dblk_sz */
+	vdir->vd_last.ul = 0;
+	vdir->vd_last.p.deblk = vdir->vd_deblk[0];
+	vdir->vd_version = 0;
+	vdir->vd_jiffy = 0;
+	/* smp_mb(); */
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+#define AuFillVdir_CALLED	1
+#define AuFillVdir_WHABLE	(1 << 1)
+#define AuFillVdir_SHWH		(1 << 2)
+#define au_ftest_fillvdir(flags, name)	((flags) & AuFillVdir_##name)
+#define au_fset_fillvdir(flags, name) \
+	do { (flags) |= AuFillVdir_##name; } while (0)
+#define au_fclr_fillvdir(flags, name) \
+	do { (flags) &= ~AuFillVdir_##name; } while (0)
+
+#ifndef CONFIG_AUFS_SHWH
+#undef AuFillVdir_SHWH
+#define AuFillVdir_SHWH		0
+#endif
+
+struct fillvdir_arg {
+	struct dir_context	ctx;
+	struct file		*file;
+	struct au_vdir		*vdir;
+	struct au_nhash		delist;
+	struct au_nhash		whlist;
+	aufs_bindex_t		bindex;
+	unsigned int		flags;
+	int			err;
+};
+
+static int fillvdir(struct dir_context *ctx, const char *__name, int nlen,
+		    loff_t offset __maybe_unused, u64 h_ino,
+		    unsigned int d_type)
+{
+	struct fillvdir_arg *arg = container_of(ctx, struct fillvdir_arg, ctx);
+	char *name = (void *)__name;
+	struct super_block *sb;
+	ino_t ino;
+	const unsigned char shwh = !!au_ftest_fillvdir(arg->flags, SHWH);
+
+	arg->err = 0;
+	sb = arg->file->f_path.dentry->d_sb;
+	au_fset_fillvdir(arg->flags, CALLED);
+	/* smp_mb(); */
+	if (nlen <= AUFS_WH_PFX_LEN
+	    || memcmp(name, AUFS_WH_PFX, AUFS_WH_PFX_LEN)) {
+		if (test_known(&arg->delist, name, nlen)
+		    || au_nhash_test_known_wh(&arg->whlist, name, nlen))
+			goto out; /* already exists or whiteouted */
+
+		arg->err = au_ino(sb, arg->bindex, h_ino, d_type, &ino);
+		if (!arg->err) {
+			if (unlikely(nlen > AUFS_MAX_NAMELEN))
+				d_type = DT_UNKNOWN;
+			arg->err = append_de(arg->vdir, name, nlen, ino,
+					     d_type, &arg->delist);
+		}
+	} else if (au_ftest_fillvdir(arg->flags, WHABLE)) {
+		name += AUFS_WH_PFX_LEN;
+		nlen -= AUFS_WH_PFX_LEN;
+		if (au_nhash_test_known_wh(&arg->whlist, name, nlen))
+			goto out; /* already whiteouted */
+
+		ino = 0; /* just to suppress a warning */
+		if (shwh)
+			arg->err = au_wh_ino(sb, arg->bindex, h_ino, d_type,
+					     &ino);
+		if (!arg->err) {
+			if (nlen <= AUFS_MAX_NAMELEN + AUFS_WH_PFX_LEN)
+				d_type = DT_UNKNOWN;
+			arg->err = au_nhash_append_wh
+				(&arg->whlist, name, nlen, ino, d_type,
+				 arg->bindex, shwh);
+		}
+	}
+
+out:
+	if (!arg->err)
+		arg->vdir->vd_jiffy = jiffies;
+	/* smp_mb(); */
+	AuTraceErr(arg->err);
+	return arg->err;
+}
+
+static int au_handle_shwh(struct super_block *sb, struct au_vdir *vdir,
+			  struct au_nhash *whlist, struct au_nhash *delist)
+{
+#ifdef CONFIG_AUFS_SHWH
+	int err;
+	unsigned int nh, u;
+	struct hlist_head *head;
+	struct au_vdir_wh *pos;
+	struct hlist_node *n;
+	char *p, *o;
+	struct au_vdir_destr *destr;
+
+	AuDebugOn(!au_opt_test(au_mntflags(sb), SHWH));
+
+	err = -ENOMEM;
+	o = p = (void *)__get_free_page(GFP_NOFS);
+	if (unlikely(!p))
+		goto out;
+
+	err = 0;
+	nh = whlist->nh_num;
+	memcpy(p, AUFS_WH_PFX, AUFS_WH_PFX_LEN);
+	p += AUFS_WH_PFX_LEN;
+	for (u = 0; u < nh; u++) {
+		head = whlist->nh_head + u;
+		hlist_for_each_entry_safe(pos, n, head, wh_hash) {
+			destr = &pos->wh_str;
+			memcpy(p, destr->name, destr->len);
+			err = append_de(vdir, o, destr->len + AUFS_WH_PFX_LEN,
+					pos->wh_ino, pos->wh_type, delist);
+			if (unlikely(err))
+				break;
+		}
+	}
+
+	free_page((unsigned long)o);
+
+out:
+	AuTraceErr(err);
+	return err;
+#else
+	return 0;
+#endif
+}
+
+static int au_do_read_vdir(struct fillvdir_arg *arg)
+{
+	int err;
+	unsigned int rdhash;
+	loff_t offset;
+	aufs_bindex_t bbot, bindex, btop;
+	unsigned char shwh;
+	struct file *hf, *file;
+	struct super_block *sb;
+
+	file = arg->file;
+	sb = file->f_path.dentry->d_sb;
+	SiMustAnyLock(sb);
+
+	rdhash = au_sbi(sb)->si_rdhash;
+	if (!rdhash)
+		rdhash = au_rdhash_est(au_dir_size(file, /*dentry*/NULL));
+	err = au_nhash_alloc(&arg->delist, rdhash, GFP_NOFS);
+	if (unlikely(err))
+		goto out;
+	err = au_nhash_alloc(&arg->whlist, rdhash, GFP_NOFS);
+	if (unlikely(err))
+		goto out_delist;
+
+	err = 0;
+	arg->flags = 0;
+	shwh = 0;
+	if (au_opt_test(au_mntflags(sb), SHWH)) {
+		shwh = 1;
+		au_fset_fillvdir(arg->flags, SHWH);
+	}
+	btop = au_fbtop(file);
+	bbot = au_fbbot_dir(file);
+	for (bindex = btop; !err && bindex <= bbot; bindex++) {
+		hf = au_hf_dir(file, bindex);
+		if (!hf)
+			continue;
+
+		offset = vfsub_llseek(hf, 0, SEEK_SET);
+		err = offset;
+		if (unlikely(offset))
+			break;
+
+		arg->bindex = bindex;
+		au_fclr_fillvdir(arg->flags, WHABLE);
+		if (shwh
+		    || (bindex != bbot
+			&& au_br_whable(au_sbr_perm(sb, bindex))))
+			au_fset_fillvdir(arg->flags, WHABLE);
+		do {
+			arg->err = 0;
+			au_fclr_fillvdir(arg->flags, CALLED);
+			/* smp_mb(); */
+			err = vfsub_iterate_dir(hf, &arg->ctx);
+			if (err >= 0)
+				err = arg->err;
+		} while (!err && au_ftest_fillvdir(arg->flags, CALLED));
+
+		/*
+		 * dir_relax() may be good for concurrency, but aufs should not
+		 * use it since it will cause a lockdep problem.
+		 */
+	}
+
+	if (!err && shwh)
+		err = au_handle_shwh(sb, arg->vdir, &arg->whlist, &arg->delist);
+
+	au_nhash_wh_free(&arg->whlist);
+
+out_delist:
+	au_nhash_de_free(&arg->delist);
+out:
+	return err;
+}
+
+static int read_vdir(struct file *file, int may_read)
+{
+	int err;
+	unsigned long expire;
+	unsigned char do_read;
+	struct fillvdir_arg arg = {
+		.ctx = {
+			.actor = fillvdir
+		}
+	};
+	struct inode *inode;
+	struct au_vdir *vdir, *allocated;
+
+	err = 0;
+	inode = file_inode(file);
+	IMustLock(inode);
+	IiMustWriteLock(inode);
+	SiMustAnyLock(inode->i_sb);
+
+	allocated = NULL;
+	do_read = 0;
+	expire = au_sbi(inode->i_sb)->si_rdcache;
+	vdir = au_ivdir(inode);
+	if (!vdir) {
+		do_read = 1;
+		vdir = alloc_vdir(file);
+		err = PTR_ERR(vdir);
+		if (IS_ERR(vdir))
+			goto out;
+		err = 0;
+		allocated = vdir;
+	} else if (may_read
+		   && (!inode_eq_iversion(inode, vdir->vd_version)
+		       || time_after(jiffies, vdir->vd_jiffy + expire))) {
+		do_read = 1;
+		err = reinit_vdir(vdir);
+		if (unlikely(err))
+			goto out;
+	}
+
+	if (!do_read)
+		return 0; /* success */
+
+	arg.file = file;
+	arg.vdir = vdir;
+	err = au_do_read_vdir(&arg);
+	if (!err) {
+		/* file->f_pos = 0; */ /* todo: ctx->pos? */
+		vdir->vd_version = inode_query_iversion(inode);
+		vdir->vd_last.ul = 0;
+		vdir->vd_last.p.deblk = vdir->vd_deblk[0];
+		if (allocated)
+			au_set_ivdir(inode, allocated);
+	} else if (allocated)
+		au_vdir_free(allocated);
+
+out:
+	return err;
+}
+
+static int copy_vdir(struct au_vdir *tgt, struct au_vdir *src)
+{
+	int err, rerr;
+	unsigned long ul, n;
+	const unsigned int deblk_sz = src->vd_deblk_sz;
+
+	AuDebugOn(tgt->vd_nblk != 1);
+
+	err = -ENOMEM;
+	if (tgt->vd_nblk < src->vd_nblk) {
+		unsigned char **p;
+
+		p = au_krealloc(tgt->vd_deblk, sizeof(*p) * src->vd_nblk,
+				GFP_NOFS, /*may_shrink*/0);
+		if (unlikely(!p))
+			goto out;
+		tgt->vd_deblk = p;
+	}
+
+	if (tgt->vd_deblk_sz != deblk_sz) {
+		unsigned char *p;
+
+		tgt->vd_deblk_sz = deblk_sz;
+		p = au_krealloc(tgt->vd_deblk[0], deblk_sz, GFP_NOFS,
+				/*may_shrink*/1);
+		if (unlikely(!p))
+			goto out;
+		tgt->vd_deblk[0] = p;
+	}
+	memcpy(tgt->vd_deblk[0], src->vd_deblk[0], deblk_sz);
+	tgt->vd_version = src->vd_version;
+	tgt->vd_jiffy = src->vd_jiffy;
+
+	n = src->vd_nblk;
+	for (ul = 1; ul < n; ul++) {
+		tgt->vd_deblk[ul] = kmemdup(src->vd_deblk[ul], deblk_sz,
+					    GFP_NOFS);
+		if (unlikely(!tgt->vd_deblk[ul]))
+			goto out;
+		tgt->vd_nblk++;
+	}
+	tgt->vd_nblk = n;
+	tgt->vd_last.ul = tgt->vd_last.ul;
+	tgt->vd_last.p.deblk = tgt->vd_deblk[tgt->vd_last.ul];
+	tgt->vd_last.p.deblk += src->vd_last.p.deblk
+		- src->vd_deblk[src->vd_last.ul];
+	/* smp_mb(); */
+	return 0; /* success */
+
+out:
+	rerr = reinit_vdir(tgt);
+	BUG_ON(rerr);
+	return err;
+}
+
+int au_vdir_init(struct file *file)
+{
+	int err;
+	struct inode *inode;
+	struct au_vdir *vdir_cache, *allocated;
+
+	/* test file->f_pos here instead of ctx->pos */
+	err = read_vdir(file, !file->f_pos);
+	if (unlikely(err))
+		goto out;
+
+	allocated = NULL;
+	vdir_cache = au_fvdir_cache(file);
+	if (!vdir_cache) {
+		vdir_cache = alloc_vdir(file);
+		err = PTR_ERR(vdir_cache);
+		if (IS_ERR(vdir_cache))
+			goto out;
+		allocated = vdir_cache;
+	} else if (!file->f_pos && vdir_cache->vd_version != file->f_version) {
+		/* test file->f_pos here instead of ctx->pos */
+		err = reinit_vdir(vdir_cache);
+		if (unlikely(err))
+			goto out;
+	} else
+		return 0; /* success */
+
+	inode = file_inode(file);
+	err = copy_vdir(vdir_cache, au_ivdir(inode));
+	if (!err) {
+		file->f_version = inode_query_iversion(inode);
+		if (allocated)
+			au_set_fvdir_cache(file, allocated);
+	} else if (allocated)
+		au_vdir_free(allocated);
+
+out:
+	return err;
+}
+
+static loff_t calc_offset(struct au_vdir *vdir)
+{
+	loff_t offset;
+	union au_vdir_deblk_p p;
+
+	p.deblk = vdir->vd_deblk[vdir->vd_last.ul];
+	offset = vdir->vd_last.p.deblk - p.deblk;
+	offset += vdir->vd_deblk_sz * vdir->vd_last.ul;
+	return offset;
+}
+
+/* returns true or false */
+static int seek_vdir(struct file *file, struct dir_context *ctx)
+{
+	int valid;
+	unsigned int deblk_sz;
+	unsigned long ul, n;
+	loff_t offset;
+	union au_vdir_deblk_p p, deblk_end;
+	struct au_vdir *vdir_cache;
+
+	valid = 1;
+	vdir_cache = au_fvdir_cache(file);
+	offset = calc_offset(vdir_cache);
+	AuDbg("offset %lld\n", offset);
+	if (ctx->pos == offset)
+		goto out;
+
+	vdir_cache->vd_last.ul = 0;
+	vdir_cache->vd_last.p.deblk = vdir_cache->vd_deblk[0];
+	if (!ctx->pos)
+		goto out;
+
+	valid = 0;
+	deblk_sz = vdir_cache->vd_deblk_sz;
+	ul = div64_u64(ctx->pos, deblk_sz);
+	AuDbg("ul %lu\n", ul);
+	if (ul >= vdir_cache->vd_nblk)
+		goto out;
+
+	n = vdir_cache->vd_nblk;
+	for (; ul < n; ul++) {
+		p.deblk = vdir_cache->vd_deblk[ul];
+		deblk_end.deblk = p.deblk + deblk_sz;
+		offset = ul;
+		offset *= deblk_sz;
+		while (!is_deblk_end(&p, &deblk_end) && offset < ctx->pos) {
+			unsigned int l;
+
+			l = calc_size(p.de->de_str.len);
+			offset += l;
+			p.deblk += l;
+		}
+		if (!is_deblk_end(&p, &deblk_end)) {
+			valid = 1;
+			vdir_cache->vd_last.ul = ul;
+			vdir_cache->vd_last.p = p;
+			break;
+		}
+	}
+
+out:
+	/* smp_mb(); */
+	if (!valid)
+		AuDbg("valid %d\n", !valid);
+	return valid;
+}
+
+int au_vdir_fill_de(struct file *file, struct dir_context *ctx)
+{
+	unsigned int l, deblk_sz;
+	union au_vdir_deblk_p deblk_end;
+	struct au_vdir *vdir_cache;
+	struct au_vdir_de *de;
+
+	if (!seek_vdir(file, ctx))
+		return 0;
+
+	vdir_cache = au_fvdir_cache(file);
+	deblk_sz = vdir_cache->vd_deblk_sz;
+	while (1) {
+		deblk_end.deblk = vdir_cache->vd_deblk[vdir_cache->vd_last.ul];
+		deblk_end.deblk += deblk_sz;
+		while (!is_deblk_end(&vdir_cache->vd_last.p, &deblk_end)) {
+			de = vdir_cache->vd_last.p.de;
+			AuDbg("%.*s, off%lld, i%lu, dt%d\n",
+			      de->de_str.len, de->de_str.name, ctx->pos,
+			      (unsigned long)de->de_ino, de->de_type);
+			if (unlikely(!dir_emit(ctx, de->de_str.name,
+					       de->de_str.len, de->de_ino,
+					       de->de_type))) {
+				/* todo: ignore the error caused by udba? */
+				/* return err; */
+				return 0;
+			}
+
+			l = calc_size(de->de_str.len);
+			vdir_cache->vd_last.p.deblk += l;
+			ctx->pos += l;
+		}
+		if (vdir_cache->vd_last.ul < vdir_cache->vd_nblk - 1) {
+			vdir_cache->vd_last.ul++;
+			vdir_cache->vd_last.p.deblk
+				= vdir_cache->vd_deblk[vdir_cache->vd_last.ul];
+			ctx->pos = deblk_sz * vdir_cache->vd_last.ul;
+			continue;
+		}
+		break;
+	}
+
+	/* smp_mb(); */
+	return 0;
+}
diff -Naur --no-dereference a/fs/aufs/vfsub.c b/fs/aufs/vfsub.c
--- a/fs/aufs/vfsub.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/vfsub.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,902 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * sub-routines for VFS
+ */
+
+#include <linux/mnt_namespace.h>
+#include <linux/namei.h>
+#include <linux/nsproxy.h>
+#include <linux/security.h>
+#include <linux/splice.h>
+#include "aufs.h"
+
+#ifdef CONFIG_AUFS_BR_FUSE
+int vfsub_test_mntns(struct vfsmount *mnt, struct super_block *h_sb)
+{
+	if (!au_test_fuse(h_sb) || !au_userns)
+		return 0;
+
+	return is_current_mnt_ns(mnt) ? 0 : -EACCES;
+}
+#endif
+
+int vfsub_sync_filesystem(struct super_block *h_sb, int wait)
+{
+	int err;
+
+	lockdep_off();
+	down_read(&h_sb->s_umount);
+	err = __sync_filesystem(h_sb, wait);
+	up_read(&h_sb->s_umount);
+	lockdep_on();
+
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+int vfsub_update_h_iattr(struct path *h_path, int *did)
+{
+	int err;
+	struct kstat st;
+	struct super_block *h_sb;
+
+	/* for remote fs, leave work for its getattr or d_revalidate */
+	/* for bad i_attr fs, handle them in aufs_getattr() */
+	/* still some fs may acquire i_mutex. we need to skip them */
+	err = 0;
+	if (!did)
+		did = &err;
+	h_sb = h_path->dentry->d_sb;
+	*did = (!au_test_fs_remote(h_sb) && au_test_fs_refresh_iattr(h_sb));
+	if (*did)
+		err = vfsub_getattr(h_path, &st);
+
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+struct file *vfsub_dentry_open(struct path *path, int flags)
+{
+	struct file *file;
+
+	file = dentry_open(path, flags /* | __FMODE_NONOTIFY */,
+			   current_cred());
+	if (!IS_ERR_OR_NULL(file)
+	    && (file->f_mode & (FMODE_READ | FMODE_WRITE)) == FMODE_READ)
+		i_readcount_inc(d_inode(path->dentry));
+
+	return file;
+}
+
+struct file *vfsub_filp_open(const char *path, int oflags, int mode)
+{
+	struct file *file;
+
+	lockdep_off();
+	file = filp_open(path,
+			 oflags /* | __FMODE_NONOTIFY */,
+			 mode);
+	lockdep_on();
+	if (IS_ERR(file))
+		goto out;
+	vfsub_update_h_iattr(&file->f_path, /*did*/NULL); /*ignore*/
+
+out:
+	return file;
+}
+
+/*
+ * Ideally this function should call VFS:do_last() in order to keep all its
+ * checkings. But it is very hard for aufs to regenerate several VFS internal
+ * structure such as nameidata. This is a second (or third) best approach.
+ * cf. linux/fs/namei.c:do_last(), lookup_open() and atomic_open().
+ */
+int vfsub_atomic_open(struct inode *dir, struct dentry *dentry,
+		      struct vfsub_aopen_args *args)
+{
+	int err;
+	struct au_branch *br = args->br;
+	struct file *file = args->file;
+	/* copied from linux/fs/namei.c:atomic_open() */
+	struct dentry *const DENTRY_NOT_SET = (void *)-1UL;
+
+	IMustLock(dir);
+	AuDebugOn(!dir->i_op->atomic_open);
+
+	err = au_br_test_oflag(args->open_flag, br);
+	if (unlikely(err))
+		goto out;
+
+	au_lcnt_inc(&br->br_nfiles);
+	file->f_path.dentry = DENTRY_NOT_SET;
+	file->f_path.mnt = au_br_mnt(br);
+	AuDbg("%ps\n", dir->i_op->atomic_open);
+	err = dir->i_op->atomic_open(dir, dentry, file, args->open_flag,
+				     args->create_mode);
+	if (unlikely(err < 0)) {
+		au_lcnt_dec(&br->br_nfiles);
+		goto out;
+	}
+
+	/* temporary workaround for nfsv4 branch */
+	if (au_test_nfs(dir->i_sb))
+		nfs_mark_for_revalidate(dir);
+
+	if (file->f_mode & FMODE_CREATED)
+		fsnotify_create(dir, dentry);
+	if (!(file->f_mode & FMODE_OPENED)) {
+		au_lcnt_dec(&br->br_nfiles);
+		goto out;
+	}
+
+	/* todo: call VFS:may_open() here */
+	/* todo: ima_file_check() too? */
+	if (!err && (args->open_flag & __FMODE_EXEC))
+		err = deny_write_access(file);
+	if (!err)
+		fsnotify_open(file);
+	else
+		au_lcnt_dec(&br->br_nfiles);
+	/* note that the file is created and still opened */
+
+out:
+	return err;
+}
+
+int vfsub_kern_path(const char *name, unsigned int flags, struct path *path)
+{
+	int err;
+
+	err = kern_path(name, flags, path);
+	if (!err && d_is_positive(path->dentry))
+		vfsub_update_h_iattr(path, /*did*/NULL); /*ignore*/
+	return err;
+}
+
+struct dentry *vfsub_lookup_one_len_unlocked(const char *name,
+					     struct dentry *parent, int len)
+{
+	struct path path = {
+		.mnt = NULL
+	};
+
+	path.dentry = lookup_one_len_unlocked(name, parent, len);
+	if (IS_ERR(path.dentry))
+		goto out;
+	if (d_is_positive(path.dentry))
+		vfsub_update_h_iattr(&path, /*did*/NULL); /*ignore*/
+
+out:
+	AuTraceErrPtr(path.dentry);
+	return path.dentry;
+}
+
+struct dentry *vfsub_lookup_one_len(const char *name, struct dentry *parent,
+				    int len)
+{
+	struct path path = {
+		.mnt = NULL
+	};
+
+	/* VFS checks it too, but by WARN_ON_ONCE() */
+	IMustLock(d_inode(parent));
+
+	path.dentry = lookup_one_len(name, parent, len);
+	if (IS_ERR(path.dentry))
+		goto out;
+	if (d_is_positive(path.dentry))
+		vfsub_update_h_iattr(&path, /*did*/NULL); /*ignore*/
+
+out:
+	AuTraceErrPtr(path.dentry);
+	return path.dentry;
+}
+
+void vfsub_call_lkup_one(void *args)
+{
+	struct vfsub_lkup_one_args *a = args;
+	*a->errp = vfsub_lkup_one(a->name, a->parent);
+}
+
+/* ---------------------------------------------------------------------- */
+
+struct dentry *vfsub_lock_rename(struct dentry *d1, struct au_hinode *hdir1,
+				 struct dentry *d2, struct au_hinode *hdir2)
+{
+	struct dentry *d;
+
+	lockdep_off();
+	d = lock_rename(d1, d2);
+	lockdep_on();
+	au_hn_suspend(hdir1);
+	if (hdir1 != hdir2)
+		au_hn_suspend(hdir2);
+
+	return d;
+}
+
+void vfsub_unlock_rename(struct dentry *d1, struct au_hinode *hdir1,
+			 struct dentry *d2, struct au_hinode *hdir2)
+{
+	au_hn_resume(hdir1);
+	if (hdir1 != hdir2)
+		au_hn_resume(hdir2);
+	lockdep_off();
+	unlock_rename(d1, d2);
+	lockdep_on();
+}
+
+/* ---------------------------------------------------------------------- */
+
+int vfsub_create(struct inode *dir, struct path *path, int mode, bool want_excl)
+{
+	int err;
+	struct dentry *d;
+
+	IMustLock(dir);
+
+	d = path->dentry;
+	path->dentry = d->d_parent;
+	err = security_path_mknod(path, d, mode, 0);
+	path->dentry = d;
+	if (unlikely(err))
+		goto out;
+
+	lockdep_off();
+	err = vfs_create(dir, path->dentry, mode, want_excl);
+	lockdep_on();
+	if (!err) {
+		struct path tmp = *path;
+		int did;
+
+		vfsub_update_h_iattr(&tmp, &did);
+		if (did) {
+			tmp.dentry = path->dentry->d_parent;
+			vfsub_update_h_iattr(&tmp, /*did*/NULL);
+		}
+		/*ignore*/
+	}
+
+out:
+	return err;
+}
+
+int vfsub_symlink(struct inode *dir, struct path *path, const char *symname)
+{
+	int err;
+	struct dentry *d;
+
+	IMustLock(dir);
+
+	d = path->dentry;
+	path->dentry = d->d_parent;
+	err = security_path_symlink(path, d, symname);
+	path->dentry = d;
+	if (unlikely(err))
+		goto out;
+
+	lockdep_off();
+	err = vfs_symlink(dir, path->dentry, symname);
+	lockdep_on();
+	if (!err) {
+		struct path tmp = *path;
+		int did;
+
+		vfsub_update_h_iattr(&tmp, &did);
+		if (did) {
+			tmp.dentry = path->dentry->d_parent;
+			vfsub_update_h_iattr(&tmp, /*did*/NULL);
+		}
+		/*ignore*/
+	}
+
+out:
+	return err;
+}
+
+int vfsub_mknod(struct inode *dir, struct path *path, int mode, dev_t dev)
+{
+	int err;
+	struct dentry *d;
+
+	IMustLock(dir);
+
+	d = path->dentry;
+	path->dentry = d->d_parent;
+	err = security_path_mknod(path, d, mode, new_encode_dev(dev));
+	path->dentry = d;
+	if (unlikely(err))
+		goto out;
+
+	lockdep_off();
+	err = vfs_mknod(dir, path->dentry, mode, dev);
+	lockdep_on();
+	if (!err) {
+		struct path tmp = *path;
+		int did;
+
+		vfsub_update_h_iattr(&tmp, &did);
+		if (did) {
+			tmp.dentry = path->dentry->d_parent;
+			vfsub_update_h_iattr(&tmp, /*did*/NULL);
+		}
+		/*ignore*/
+	}
+
+out:
+	return err;
+}
+
+static int au_test_nlink(struct inode *inode)
+{
+	const unsigned int link_max = UINT_MAX >> 1; /* rough margin */
+
+	if (!au_test_fs_no_limit_nlink(inode->i_sb)
+	    || inode->i_nlink < link_max)
+		return 0;
+	return -EMLINK;
+}
+
+int vfsub_link(struct dentry *src_dentry, struct inode *dir, struct path *path,
+	       struct inode **delegated_inode)
+{
+	int err;
+	struct dentry *d;
+
+	IMustLock(dir);
+
+	err = au_test_nlink(d_inode(src_dentry));
+	if (unlikely(err))
+		return err;
+
+	/* we don't call may_linkat() */
+	d = path->dentry;
+	path->dentry = d->d_parent;
+	err = security_path_link(src_dentry, path, d);
+	path->dentry = d;
+	if (unlikely(err))
+		goto out;
+
+	lockdep_off();
+	err = vfs_link(src_dentry, dir, path->dentry, delegated_inode);
+	lockdep_on();
+	if (!err) {
+		struct path tmp = *path;
+		int did;
+
+		/* fuse has different memory inode for the same inumber */
+		vfsub_update_h_iattr(&tmp, &did);
+		if (did) {
+			tmp.dentry = path->dentry->d_parent;
+			vfsub_update_h_iattr(&tmp, /*did*/NULL);
+			tmp.dentry = src_dentry;
+			vfsub_update_h_iattr(&tmp, /*did*/NULL);
+		}
+		/*ignore*/
+	}
+
+out:
+	return err;
+}
+
+int vfsub_rename(struct inode *src_dir, struct dentry *src_dentry,
+		 struct inode *dir, struct path *path,
+		 struct inode **delegated_inode, unsigned int flags)
+{
+	int err;
+	struct path tmp = {
+		.mnt	= path->mnt
+	};
+	struct dentry *d;
+
+	IMustLock(dir);
+	IMustLock(src_dir);
+
+	d = path->dentry;
+	path->dentry = d->d_parent;
+	tmp.dentry = src_dentry->d_parent;
+	err = security_path_rename(&tmp, src_dentry, path, d, /*flags*/0);
+	path->dentry = d;
+	if (unlikely(err))
+		goto out;
+
+	lockdep_off();
+	err = vfs_rename(src_dir, src_dentry, dir, path->dentry,
+			 delegated_inode, flags);
+	lockdep_on();
+	if (!err) {
+		int did;
+
+		tmp.dentry = d->d_parent;
+		vfsub_update_h_iattr(&tmp, &did);
+		if (did) {
+			tmp.dentry = src_dentry;
+			vfsub_update_h_iattr(&tmp, /*did*/NULL);
+			tmp.dentry = src_dentry->d_parent;
+			vfsub_update_h_iattr(&tmp, /*did*/NULL);
+		}
+		/*ignore*/
+	}
+
+out:
+	return err;
+}
+
+int vfsub_mkdir(struct inode *dir, struct path *path, int mode)
+{
+	int err;
+	struct dentry *d;
+
+	IMustLock(dir);
+
+	d = path->dentry;
+	path->dentry = d->d_parent;
+	err = security_path_mkdir(path, d, mode);
+	path->dentry = d;
+	if (unlikely(err))
+		goto out;
+
+	lockdep_off();
+	err = vfs_mkdir(dir, path->dentry, mode);
+	lockdep_on();
+	if (!err) {
+		struct path tmp = *path;
+		int did;
+
+		vfsub_update_h_iattr(&tmp, &did);
+		if (did) {
+			tmp.dentry = path->dentry->d_parent;
+			vfsub_update_h_iattr(&tmp, /*did*/NULL);
+		}
+		/*ignore*/
+	}
+
+out:
+	return err;
+}
+
+int vfsub_rmdir(struct inode *dir, struct path *path)
+{
+	int err;
+	struct dentry *d;
+
+	IMustLock(dir);
+
+	d = path->dentry;
+	path->dentry = d->d_parent;
+	err = security_path_rmdir(path, d);
+	path->dentry = d;
+	if (unlikely(err))
+		goto out;
+
+	lockdep_off();
+	err = vfs_rmdir(dir, path->dentry);
+	lockdep_on();
+	if (!err) {
+		struct path tmp = {
+			.dentry	= path->dentry->d_parent,
+			.mnt	= path->mnt
+		};
+
+		vfsub_update_h_iattr(&tmp, /*did*/NULL); /*ignore*/
+	}
+
+out:
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* todo: support mmap_sem? */
+ssize_t vfsub_read_u(struct file *file, char __user *ubuf, size_t count,
+		     loff_t *ppos)
+{
+	ssize_t err;
+
+	lockdep_off();
+	err = vfs_read(file, ubuf, count, ppos);
+	lockdep_on();
+	if (err >= 0)
+		vfsub_update_h_iattr(&file->f_path, /*did*/NULL); /*ignore*/
+	return err;
+}
+
+/* todo: kernel_read()? */
+ssize_t vfsub_read_k(struct file *file, void *kbuf, size_t count,
+		     loff_t *ppos)
+{
+	ssize_t err;
+	mm_segment_t oldfs;
+	union {
+		void *k;
+		char __user *u;
+	} buf;
+
+	buf.k = kbuf;
+	oldfs = get_fs();
+	set_fs(KERNEL_DS);
+	err = vfsub_read_u(file, buf.u, count, ppos);
+	set_fs(oldfs);
+	return err;
+}
+
+ssize_t vfsub_write_u(struct file *file, const char __user *ubuf, size_t count,
+		      loff_t *ppos)
+{
+	ssize_t err;
+
+	lockdep_off();
+	err = vfs_write(file, ubuf, count, ppos);
+	lockdep_on();
+	if (err >= 0)
+		vfsub_update_h_iattr(&file->f_path, /*did*/NULL); /*ignore*/
+	return err;
+}
+
+ssize_t vfsub_write_k(struct file *file, void *kbuf, size_t count, loff_t *ppos)
+{
+	ssize_t err;
+	mm_segment_t oldfs;
+	union {
+		void *k;
+		const char __user *u;
+	} buf;
+
+	buf.k = kbuf;
+	oldfs = get_fs();
+	set_fs(KERNEL_DS);
+	err = vfsub_write_u(file, buf.u, count, ppos);
+	set_fs(oldfs);
+	return err;
+}
+
+int vfsub_flush(struct file *file, fl_owner_t id)
+{
+	int err;
+
+	err = 0;
+	if (file->f_op->flush) {
+		if (!au_test_nfs(file->f_path.dentry->d_sb))
+			err = file->f_op->flush(file, id);
+		else {
+			lockdep_off();
+			err = file->f_op->flush(file, id);
+			lockdep_on();
+		}
+		if (!err)
+			vfsub_update_h_iattr(&file->f_path, /*did*/NULL);
+		/*ignore*/
+	}
+	return err;
+}
+
+int vfsub_iterate_dir(struct file *file, struct dir_context *ctx)
+{
+	int err;
+
+	AuDbg("%pD, ctx{%ps, %llu}\n", file, ctx->actor, ctx->pos);
+
+	lockdep_off();
+	err = iterate_dir(file, ctx);
+	lockdep_on();
+	if (err >= 0)
+		vfsub_update_h_iattr(&file->f_path, /*did*/NULL); /*ignore*/
+
+	return err;
+}
+
+long vfsub_splice_to(struct file *in, loff_t *ppos,
+		     struct pipe_inode_info *pipe, size_t len,
+		     unsigned int flags)
+{
+	long err;
+
+	lockdep_off();
+	err = do_splice_to(in, ppos, pipe, len, flags);
+	lockdep_on();
+	file_accessed(in);
+	if (err >= 0)
+		vfsub_update_h_iattr(&in->f_path, /*did*/NULL); /*ignore*/
+	return err;
+}
+
+long vfsub_splice_from(struct pipe_inode_info *pipe, struct file *out,
+		       loff_t *ppos, size_t len, unsigned int flags)
+{
+	long err;
+
+	lockdep_off();
+	err = do_splice_from(pipe, out, ppos, len, flags);
+	lockdep_on();
+	if (err >= 0)
+		vfsub_update_h_iattr(&out->f_path, /*did*/NULL); /*ignore*/
+	return err;
+}
+
+int vfsub_fsync(struct file *file, struct path *path, int datasync)
+{
+	int err;
+
+	/* file can be NULL */
+	lockdep_off();
+	err = vfs_fsync(file, datasync);
+	lockdep_on();
+	if (!err) {
+		if (!path) {
+			AuDebugOn(!file);
+			path = &file->f_path;
+		}
+		vfsub_update_h_iattr(path, /*did*/NULL); /*ignore*/
+	}
+	return err;
+}
+
+/* cf. open.c:do_sys_truncate() and do_sys_ftruncate() */
+int vfsub_trunc(struct path *h_path, loff_t length, unsigned int attr,
+		struct file *h_file)
+{
+	int err;
+	struct inode *h_inode;
+	struct super_block *h_sb;
+
+	if (!h_file) {
+		err = vfsub_truncate(h_path, length);
+		goto out;
+	}
+
+	h_inode = d_inode(h_path->dentry);
+	h_sb = h_inode->i_sb;
+	lockdep_off();
+	sb_start_write(h_sb);
+	lockdep_on();
+	err = locks_verify_truncate(h_inode, h_file, length);
+	if (!err)
+		err = security_path_truncate(h_path);
+	if (!err) {
+		lockdep_off();
+		err = do_truncate(h_path->dentry, length, attr, h_file);
+		lockdep_on();
+	}
+	lockdep_off();
+	sb_end_write(h_sb);
+	lockdep_on();
+
+out:
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+struct au_vfsub_mkdir_args {
+	int *errp;
+	struct inode *dir;
+	struct path *path;
+	int mode;
+};
+
+static void au_call_vfsub_mkdir(void *args)
+{
+	struct au_vfsub_mkdir_args *a = args;
+	*a->errp = vfsub_mkdir(a->dir, a->path, a->mode);
+}
+
+int vfsub_sio_mkdir(struct inode *dir, struct path *path, int mode)
+{
+	int err, do_sio, wkq_err;
+
+	do_sio = au_test_h_perm_sio(dir, MAY_EXEC | MAY_WRITE);
+	if (!do_sio) {
+		lockdep_off();
+		err = vfsub_mkdir(dir, path, mode);
+		lockdep_on();
+	} else {
+		struct au_vfsub_mkdir_args args = {
+			.errp	= &err,
+			.dir	= dir,
+			.path	= path,
+			.mode	= mode
+		};
+		wkq_err = au_wkq_wait(au_call_vfsub_mkdir, &args);
+		if (unlikely(wkq_err))
+			err = wkq_err;
+	}
+
+	return err;
+}
+
+struct au_vfsub_rmdir_args {
+	int *errp;
+	struct inode *dir;
+	struct path *path;
+};
+
+static void au_call_vfsub_rmdir(void *args)
+{
+	struct au_vfsub_rmdir_args *a = args;
+	*a->errp = vfsub_rmdir(a->dir, a->path);
+}
+
+int vfsub_sio_rmdir(struct inode *dir, struct path *path)
+{
+	int err, do_sio, wkq_err;
+
+	do_sio = au_test_h_perm_sio(dir, MAY_EXEC | MAY_WRITE);
+	if (!do_sio) {
+		lockdep_off();
+		err = vfsub_rmdir(dir, path);
+		lockdep_on();
+	} else {
+		struct au_vfsub_rmdir_args args = {
+			.errp	= &err,
+			.dir	= dir,
+			.path	= path
+		};
+		wkq_err = au_wkq_wait(au_call_vfsub_rmdir, &args);
+		if (unlikely(wkq_err))
+			err = wkq_err;
+	}
+
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+struct notify_change_args {
+	int *errp;
+	struct path *path;
+	struct iattr *ia;
+	struct inode **delegated_inode;
+};
+
+static void call_notify_change(void *args)
+{
+	struct notify_change_args *a = args;
+	struct inode *h_inode;
+
+	h_inode = d_inode(a->path->dentry);
+	IMustLock(h_inode);
+
+	*a->errp = -EPERM;
+	if (!IS_IMMUTABLE(h_inode) && !IS_APPEND(h_inode)) {
+		lockdep_off();
+		*a->errp = notify_change(a->path->dentry, a->ia,
+					 a->delegated_inode);
+		lockdep_on();
+		if (!*a->errp)
+			vfsub_update_h_iattr(a->path, /*did*/NULL); /*ignore*/
+	}
+	AuTraceErr(*a->errp);
+}
+
+int vfsub_notify_change(struct path *path, struct iattr *ia,
+			struct inode **delegated_inode)
+{
+	int err;
+	struct notify_change_args args = {
+		.errp			= &err,
+		.path			= path,
+		.ia			= ia,
+		.delegated_inode	= delegated_inode
+	};
+
+	call_notify_change(&args);
+
+	return err;
+}
+
+int vfsub_sio_notify_change(struct path *path, struct iattr *ia,
+			    struct inode **delegated_inode)
+{
+	int err, wkq_err;
+	struct notify_change_args args = {
+		.errp			= &err,
+		.path			= path,
+		.ia			= ia,
+		.delegated_inode	= delegated_inode
+	};
+
+	wkq_err = au_wkq_wait(call_notify_change, &args);
+	if (unlikely(wkq_err))
+		err = wkq_err;
+
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+struct unlink_args {
+	int *errp;
+	struct inode *dir;
+	struct path *path;
+	struct inode **delegated_inode;
+};
+
+static void call_unlink(void *args)
+{
+	struct unlink_args *a = args;
+	struct dentry *d = a->path->dentry;
+	struct inode *h_inode;
+	const int stop_sillyrename = (au_test_nfs(d->d_sb)
+				      && au_dcount(d) == 1);
+
+	IMustLock(a->dir);
+
+	a->path->dentry = d->d_parent;
+	*a->errp = security_path_unlink(a->path, d);
+	a->path->dentry = d;
+	if (unlikely(*a->errp))
+		return;
+
+	if (!stop_sillyrename)
+		dget(d);
+	h_inode = NULL;
+	if (d_is_positive(d)) {
+		h_inode = d_inode(d);
+		ihold(h_inode);
+	}
+
+	lockdep_off();
+	*a->errp = vfs_unlink(a->dir, d, a->delegated_inode);
+	lockdep_on();
+	if (!*a->errp) {
+		struct path tmp = {
+			.dentry = d->d_parent,
+			.mnt	= a->path->mnt
+		};
+		vfsub_update_h_iattr(&tmp, /*did*/NULL); /*ignore*/
+	}
+
+	if (!stop_sillyrename)
+		dput(d);
+	if (h_inode)
+		iput(h_inode);
+
+	AuTraceErr(*a->errp);
+}
+
+/*
+ * @dir: must be locked.
+ * @dentry: target dentry.
+ */
+int vfsub_unlink(struct inode *dir, struct path *path,
+		 struct inode **delegated_inode, int force)
+{
+	int err;
+	struct unlink_args args = {
+		.errp			= &err,
+		.dir			= dir,
+		.path			= path,
+		.delegated_inode	= delegated_inode
+	};
+
+	if (!force)
+		call_unlink(&args);
+	else {
+		int wkq_err;
+
+		wkq_err = au_wkq_wait(call_unlink, &args);
+		if (unlikely(wkq_err))
+			err = wkq_err;
+	}
+
+	return err;
+}
diff -Naur --no-dereference a/fs/aufs/vfsub.h b/fs/aufs/vfsub.h
--- a/fs/aufs/vfsub.h	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/vfsub.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,354 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * sub-routines for VFS
+ */
+
+#ifndef __AUFS_VFSUB_H__
+#define __AUFS_VFSUB_H__
+
+#ifdef __KERNEL__
+
+#include <linux/fs.h>
+#include <linux/mount.h>
+#include <linux/posix_acl.h>
+#include <linux/xattr.h>
+#include "debug.h"
+
+/* copied from linux/fs/internal.h */
+/* todo: BAD approach!! */
+extern void __mnt_drop_write(struct vfsmount *);
+extern struct file *alloc_empty_file(int, const struct cred *);
+
+/* ---------------------------------------------------------------------- */
+
+/* lock subclass for lower inode */
+/* default MAX_LOCKDEP_SUBCLASSES(8) is not enough */
+/* reduce? gave up. */
+enum {
+	AuLsc_I_Begin = I_MUTEX_PARENT2, /* 5 */
+	AuLsc_I_PARENT,		/* lower inode, parent first */
+	AuLsc_I_PARENT2,	/* copyup dirs */
+	AuLsc_I_PARENT3,	/* copyup wh */
+	AuLsc_I_CHILD,
+	AuLsc_I_CHILD2,
+	AuLsc_I_End
+};
+
+/* to debug easier, do not make them inlined functions */
+#define MtxMustLock(mtx)	AuDebugOn(!mutex_is_locked(mtx))
+#define IMustLock(i)		AuDebugOn(!inode_is_locked(i))
+
+/* ---------------------------------------------------------------------- */
+
+static inline void vfsub_drop_nlink(struct inode *inode)
+{
+	AuDebugOn(!inode->i_nlink);
+	drop_nlink(inode);
+}
+
+static inline void vfsub_dead_dir(struct inode *inode)
+{
+	AuDebugOn(!S_ISDIR(inode->i_mode));
+	inode->i_flags |= S_DEAD;
+	clear_nlink(inode);
+}
+
+static inline int vfsub_native_ro(struct inode *inode)
+{
+	return sb_rdonly(inode->i_sb)
+		|| IS_RDONLY(inode)
+		/* || IS_APPEND(inode) */
+		|| IS_IMMUTABLE(inode);
+}
+
+#ifdef CONFIG_AUFS_BR_FUSE
+int vfsub_test_mntns(struct vfsmount *mnt, struct super_block *h_sb);
+#else
+AuStubInt0(vfsub_test_mntns, struct vfsmount *mnt, struct super_block *h_sb);
+#endif
+
+int vfsub_sync_filesystem(struct super_block *h_sb, int wait);
+
+/* ---------------------------------------------------------------------- */
+
+int vfsub_update_h_iattr(struct path *h_path, int *did);
+struct file *vfsub_dentry_open(struct path *path, int flags);
+struct file *vfsub_filp_open(const char *path, int oflags, int mode);
+struct au_branch;
+struct vfsub_aopen_args {
+	struct file		*file;
+	unsigned int		open_flag;
+	umode_t			create_mode;
+	struct au_branch	*br;
+};
+int vfsub_atomic_open(struct inode *dir, struct dentry *dentry,
+		      struct vfsub_aopen_args *args);
+int vfsub_kern_path(const char *name, unsigned int flags, struct path *path);
+
+struct dentry *vfsub_lookup_one_len_unlocked(const char *name,
+					     struct dentry *parent, int len);
+struct dentry *vfsub_lookup_one_len(const char *name, struct dentry *parent,
+				    int len);
+
+struct vfsub_lkup_one_args {
+	struct dentry **errp;
+	struct qstr *name;
+	struct dentry *parent;
+};
+
+static inline struct dentry *vfsub_lkup_one(struct qstr *name,
+					    struct dentry *parent)
+{
+	return vfsub_lookup_one_len(name->name, parent, name->len);
+}
+
+void vfsub_call_lkup_one(void *args);
+
+/* ---------------------------------------------------------------------- */
+
+static inline int vfsub_mnt_want_write(struct vfsmount *mnt)
+{
+	int err;
+
+	lockdep_off();
+	err = mnt_want_write(mnt);
+	lockdep_on();
+	return err;
+}
+
+static inline void vfsub_mnt_drop_write(struct vfsmount *mnt)
+{
+	lockdep_off();
+	mnt_drop_write(mnt);
+	lockdep_on();
+}
+
+#if 0 /* reserved */
+static inline void vfsub_mnt_drop_write_file(struct file *file)
+{
+	lockdep_off();
+	mnt_drop_write_file(file);
+	lockdep_on();
+}
+#endif
+
+/* ---------------------------------------------------------------------- */
+
+struct au_hinode;
+struct dentry *vfsub_lock_rename(struct dentry *d1, struct au_hinode *hdir1,
+				 struct dentry *d2, struct au_hinode *hdir2);
+void vfsub_unlock_rename(struct dentry *d1, struct au_hinode *hdir1,
+			 struct dentry *d2, struct au_hinode *hdir2);
+
+int vfsub_create(struct inode *dir, struct path *path, int mode,
+		 bool want_excl);
+int vfsub_symlink(struct inode *dir, struct path *path,
+		  const char *symname);
+int vfsub_mknod(struct inode *dir, struct path *path, int mode, dev_t dev);
+int vfsub_link(struct dentry *src_dentry, struct inode *dir,
+	       struct path *path, struct inode **delegated_inode);
+int vfsub_rename(struct inode *src_hdir, struct dentry *src_dentry,
+		 struct inode *hdir, struct path *path,
+		 struct inode **delegated_inode, unsigned int flags);
+int vfsub_mkdir(struct inode *dir, struct path *path, int mode);
+int vfsub_rmdir(struct inode *dir, struct path *path);
+
+/* ---------------------------------------------------------------------- */
+
+ssize_t vfsub_read_u(struct file *file, char __user *ubuf, size_t count,
+		     loff_t *ppos);
+ssize_t vfsub_read_k(struct file *file, void *kbuf, size_t count,
+			loff_t *ppos);
+ssize_t vfsub_write_u(struct file *file, const char __user *ubuf, size_t count,
+		      loff_t *ppos);
+ssize_t vfsub_write_k(struct file *file, void *kbuf, size_t count,
+		      loff_t *ppos);
+int vfsub_flush(struct file *file, fl_owner_t id);
+int vfsub_iterate_dir(struct file *file, struct dir_context *ctx);
+
+static inline loff_t vfsub_f_size_read(struct file *file)
+{
+	return i_size_read(file_inode(file));
+}
+
+static inline unsigned int vfsub_file_flags(struct file *file)
+{
+	unsigned int flags;
+
+	spin_lock(&file->f_lock);
+	flags = file->f_flags;
+	spin_unlock(&file->f_lock);
+
+	return flags;
+}
+
+static inline int vfsub_file_execed(struct file *file)
+{
+	/* todo: direct access f_flags */
+	return !!(vfsub_file_flags(file) & __FMODE_EXEC);
+}
+
+#if 0 /* reserved */
+static inline void vfsub_file_accessed(struct file *h_file)
+{
+	file_accessed(h_file);
+	vfsub_update_h_iattr(&h_file->f_path, /*did*/NULL); /*ignore*/
+}
+#endif
+
+#if 0 /* reserved */
+static inline void vfsub_touch_atime(struct vfsmount *h_mnt,
+				     struct dentry *h_dentry)
+{
+	struct path h_path = {
+		.dentry	= h_dentry,
+		.mnt	= h_mnt
+	};
+	touch_atime(&h_path);
+	vfsub_update_h_iattr(&h_path, /*did*/NULL); /*ignore*/
+}
+#endif
+
+static inline int vfsub_update_time(struct inode *h_inode,
+				    struct timespec64 *ts, int flags)
+{
+	return update_time(h_inode, ts, flags);
+	/* no vfsub_update_h_iattr() since we don't have struct path */
+}
+
+#ifdef CONFIG_FS_POSIX_ACL
+static inline int vfsub_acl_chmod(struct inode *h_inode, umode_t h_mode)
+{
+	int err;
+
+	err = posix_acl_chmod(h_inode, h_mode);
+	if (err == -EOPNOTSUPP)
+		err = 0;
+	return err;
+}
+#else
+AuStubInt0(vfsub_acl_chmod, struct inode *h_inode, umode_t h_mode);
+#endif
+
+long vfsub_splice_to(struct file *in, loff_t *ppos,
+		     struct pipe_inode_info *pipe, size_t len,
+		     unsigned int flags);
+long vfsub_splice_from(struct pipe_inode_info *pipe, struct file *out,
+		       loff_t *ppos, size_t len, unsigned int flags);
+
+static inline long vfsub_truncate(struct path *path, loff_t length)
+{
+	long err;
+
+	lockdep_off();
+	err = vfs_truncate(path, length);
+	lockdep_on();
+	return err;
+}
+
+int vfsub_trunc(struct path *h_path, loff_t length, unsigned int attr,
+		struct file *h_file);
+int vfsub_fsync(struct file *file, struct path *path, int datasync);
+
+/*
+ * re-use branch fs's ioctl(FICLONE) while aufs itself doesn't support such
+ * ioctl.
+ */
+static inline int vfsub_clone_file_range(struct file *src, struct file *dst,
+					 u64 len)
+{
+	int err;
+
+	lockdep_off();
+	err = vfs_clone_file_range(src, 0, dst, 0, len);
+	lockdep_on();
+
+	return err;
+}
+
+/* copy_file_range(2) is a systemcall */
+static inline ssize_t vfsub_copy_file_range(struct file *src, loff_t src_pos,
+					    struct file *dst, loff_t dst_pos,
+					    size_t len, unsigned int flags)
+{
+	ssize_t ssz;
+
+	lockdep_off();
+	ssz = vfs_copy_file_range(src, src_pos, dst, dst_pos, len, flags);
+	lockdep_on();
+
+	return ssz;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static inline loff_t vfsub_llseek(struct file *file, loff_t offset, int origin)
+{
+	loff_t err;
+
+	lockdep_off();
+	err = vfs_llseek(file, offset, origin);
+	lockdep_on();
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+int vfsub_sio_mkdir(struct inode *dir, struct path *path, int mode);
+int vfsub_sio_rmdir(struct inode *dir, struct path *path);
+int vfsub_sio_notify_change(struct path *path, struct iattr *ia,
+			    struct inode **delegated_inode);
+int vfsub_notify_change(struct path *path, struct iattr *ia,
+			struct inode **delegated_inode);
+int vfsub_unlink(struct inode *dir, struct path *path,
+		 struct inode **delegated_inode, int force);
+
+static inline int vfsub_getattr(const struct path *path, struct kstat *st)
+{
+	return vfs_getattr(path, st, STATX_BASIC_STATS, AT_STATX_SYNC_AS_STAT);
+}
+
+/* ---------------------------------------------------------------------- */
+
+static inline int vfsub_setxattr(struct dentry *dentry, const char *name,
+				 const void *value, size_t size, int flags)
+{
+	int err;
+
+	lockdep_off();
+	err = vfs_setxattr(dentry, name, value, size, flags);
+	lockdep_on();
+
+	return err;
+}
+
+static inline int vfsub_removexattr(struct dentry *dentry, const char *name)
+{
+	int err;
+
+	lockdep_off();
+	err = vfs_removexattr(dentry, name);
+	lockdep_on();
+
+	return err;
+}
+
+#endif /* __KERNEL__ */
+#endif /* __AUFS_VFSUB_H__ */
diff -Naur --no-dereference a/fs/aufs/wbr_policy.c b/fs/aufs/wbr_policy.c
--- a/fs/aufs/wbr_policy.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/wbr_policy.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,830 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * policies for selecting one among multiple writable branches
+ */
+
+#include <linux/statfs.h>
+#include "aufs.h"
+
+/* subset of cpup_attr() */
+static noinline_for_stack
+int au_cpdown_attr(struct path *h_path, struct dentry *h_src)
+{
+	int err, sbits;
+	struct iattr ia;
+	struct inode *h_isrc;
+
+	h_isrc = d_inode(h_src);
+	ia.ia_valid = ATTR_FORCE | ATTR_MODE | ATTR_UID | ATTR_GID;
+	ia.ia_mode = h_isrc->i_mode;
+	ia.ia_uid = h_isrc->i_uid;
+	ia.ia_gid = h_isrc->i_gid;
+	sbits = !!(ia.ia_mode & (S_ISUID | S_ISGID));
+	au_cpup_attr_flags(d_inode(h_path->dentry), h_isrc->i_flags);
+	/* no delegation since it is just created */
+	err = vfsub_sio_notify_change(h_path, &ia, /*delegated*/NULL);
+
+	/* is this nfs only? */
+	if (!err && sbits && au_test_nfs(h_path->dentry->d_sb)) {
+		ia.ia_valid = ATTR_FORCE | ATTR_MODE;
+		ia.ia_mode = h_isrc->i_mode;
+		err = vfsub_sio_notify_change(h_path, &ia, /*delegated*/NULL);
+	}
+
+	return err;
+}
+
+#define AuCpdown_PARENT_OPQ	1
+#define AuCpdown_WHED		(1 << 1)
+#define AuCpdown_MADE_DIR	(1 << 2)
+#define AuCpdown_DIROPQ		(1 << 3)
+#define au_ftest_cpdown(flags, name)	((flags) & AuCpdown_##name)
+#define au_fset_cpdown(flags, name) \
+	do { (flags) |= AuCpdown_##name; } while (0)
+#define au_fclr_cpdown(flags, name) \
+	do { (flags) &= ~AuCpdown_##name; } while (0)
+
+static int au_cpdown_dir_opq(struct dentry *dentry, aufs_bindex_t bdst,
+			     unsigned int *flags)
+{
+	int err;
+	struct dentry *opq_dentry;
+
+	opq_dentry = au_diropq_create(dentry, bdst);
+	err = PTR_ERR(opq_dentry);
+	if (IS_ERR(opq_dentry))
+		goto out;
+	dput(opq_dentry);
+	au_fset_cpdown(*flags, DIROPQ);
+
+out:
+	return err;
+}
+
+static int au_cpdown_dir_wh(struct dentry *dentry, struct dentry *h_parent,
+			    struct inode *dir, aufs_bindex_t bdst)
+{
+	int err;
+	struct path h_path;
+	struct au_branch *br;
+
+	br = au_sbr(dentry->d_sb, bdst);
+	h_path.dentry = au_wh_lkup(h_parent, &dentry->d_name, br);
+	err = PTR_ERR(h_path.dentry);
+	if (IS_ERR(h_path.dentry))
+		goto out;
+
+	err = 0;
+	if (d_is_positive(h_path.dentry)) {
+		h_path.mnt = au_br_mnt(br);
+		err = au_wh_unlink_dentry(au_h_iptr(dir, bdst), &h_path,
+					  dentry);
+	}
+	dput(h_path.dentry);
+
+out:
+	return err;
+}
+
+static int au_cpdown_dir(struct dentry *dentry, aufs_bindex_t bdst,
+			 struct au_pin *pin,
+			 struct dentry *h_parent, void *arg)
+{
+	int err, rerr;
+	aufs_bindex_t bopq, btop;
+	struct path h_path;
+	struct dentry *parent;
+	struct inode *h_dir, *h_inode, *inode, *dir;
+	unsigned int *flags = arg;
+
+	btop = au_dbtop(dentry);
+	/* dentry is di-locked */
+	parent = dget_parent(dentry);
+	dir = d_inode(parent);
+	h_dir = d_inode(h_parent);
+	AuDebugOn(h_dir != au_h_iptr(dir, bdst));
+	IMustLock(h_dir);
+
+	err = au_lkup_neg(dentry, bdst, /*wh*/0);
+	if (unlikely(err < 0))
+		goto out;
+	h_path.dentry = au_h_dptr(dentry, bdst);
+	h_path.mnt = au_sbr_mnt(dentry->d_sb, bdst);
+	err = vfsub_sio_mkdir(au_h_iptr(dir, bdst), &h_path, 0755);
+	if (unlikely(err))
+		goto out_put;
+	au_fset_cpdown(*flags, MADE_DIR);
+
+	bopq = au_dbdiropq(dentry);
+	au_fclr_cpdown(*flags, WHED);
+	au_fclr_cpdown(*flags, DIROPQ);
+	if (au_dbwh(dentry) == bdst)
+		au_fset_cpdown(*flags, WHED);
+	if (!au_ftest_cpdown(*flags, PARENT_OPQ) && bopq <= bdst)
+		au_fset_cpdown(*flags, PARENT_OPQ);
+	h_inode = d_inode(h_path.dentry);
+	inode_lock_nested(h_inode, AuLsc_I_CHILD);
+	if (au_ftest_cpdown(*flags, WHED)) {
+		err = au_cpdown_dir_opq(dentry, bdst, flags);
+		if (unlikely(err)) {
+			inode_unlock(h_inode);
+			goto out_dir;
+		}
+	}
+
+	err = au_cpdown_attr(&h_path, au_h_dptr(dentry, btop));
+	inode_unlock(h_inode);
+	if (unlikely(err))
+		goto out_opq;
+
+	if (au_ftest_cpdown(*flags, WHED)) {
+		err = au_cpdown_dir_wh(dentry, h_parent, dir, bdst);
+		if (unlikely(err))
+			goto out_opq;
+	}
+
+	inode = d_inode(dentry);
+	if (au_ibbot(inode) < bdst)
+		au_set_ibbot(inode, bdst);
+	au_set_h_iptr(inode, bdst, au_igrab(h_inode),
+		      au_hi_flags(inode, /*isdir*/1));
+	au_fhsm_wrote(dentry->d_sb, bdst, /*force*/0);
+	goto out; /* success */
+
+	/* revert */
+out_opq:
+	if (au_ftest_cpdown(*flags, DIROPQ)) {
+		inode_lock_nested(h_inode, AuLsc_I_CHILD);
+		rerr = au_diropq_remove(dentry, bdst);
+		inode_unlock(h_inode);
+		if (unlikely(rerr)) {
+			AuIOErr("failed removing diropq for %pd b%d (%d)\n",
+				dentry, bdst, rerr);
+			err = -EIO;
+			goto out;
+		}
+	}
+out_dir:
+	if (au_ftest_cpdown(*flags, MADE_DIR)) {
+		rerr = vfsub_sio_rmdir(au_h_iptr(dir, bdst), &h_path);
+		if (unlikely(rerr)) {
+			AuIOErr("failed removing %pd b%d (%d)\n",
+				dentry, bdst, rerr);
+			err = -EIO;
+		}
+	}
+out_put:
+	au_set_h_dptr(dentry, bdst, NULL);
+	if (au_dbbot(dentry) == bdst)
+		au_update_dbbot(dentry);
+out:
+	dput(parent);
+	return err;
+}
+
+int au_cpdown_dirs(struct dentry *dentry, aufs_bindex_t bdst)
+{
+	int err;
+	unsigned int flags;
+
+	flags = 0;
+	err = au_cp_dirs(dentry, bdst, au_cpdown_dir, &flags);
+
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* policies for create */
+
+int au_wbr_nonopq(struct dentry *dentry, aufs_bindex_t bindex)
+{
+	int err, i, j, ndentry;
+	aufs_bindex_t bopq;
+	struct au_dcsub_pages dpages;
+	struct au_dpage *dpage;
+	struct dentry **dentries, *parent, *d;
+
+	err = au_dpages_init(&dpages, GFP_NOFS);
+	if (unlikely(err))
+		goto out;
+	parent = dget_parent(dentry);
+	err = au_dcsub_pages_rev_aufs(&dpages, parent, /*do_include*/0);
+	if (unlikely(err))
+		goto out_free;
+
+	err = bindex;
+	for (i = 0; i < dpages.ndpage; i++) {
+		dpage = dpages.dpages + i;
+		dentries = dpage->dentries;
+		ndentry = dpage->ndentry;
+		for (j = 0; j < ndentry; j++) {
+			d = dentries[j];
+			di_read_lock_parent2(d, !AuLock_IR);
+			bopq = au_dbdiropq(d);
+			di_read_unlock(d, !AuLock_IR);
+			if (bopq >= 0 && bopq < err)
+				err = bopq;
+		}
+	}
+
+out_free:
+	dput(parent);
+	au_dpages_free(&dpages);
+out:
+	return err;
+}
+
+static int au_wbr_bu(struct super_block *sb, aufs_bindex_t bindex)
+{
+	for (; bindex >= 0; bindex--)
+		if (!au_br_rdonly(au_sbr(sb, bindex)))
+			return bindex;
+	return -EROFS;
+}
+
+/* top down parent */
+static int au_wbr_create_tdp(struct dentry *dentry,
+			     unsigned int flags __maybe_unused)
+{
+	int err;
+	aufs_bindex_t btop, bindex;
+	struct super_block *sb;
+	struct dentry *parent, *h_parent;
+
+	sb = dentry->d_sb;
+	btop = au_dbtop(dentry);
+	err = btop;
+	if (!au_br_rdonly(au_sbr(sb, btop)))
+		goto out;
+
+	err = -EROFS;
+	parent = dget_parent(dentry);
+	for (bindex = au_dbtop(parent); bindex < btop; bindex++) {
+		h_parent = au_h_dptr(parent, bindex);
+		if (!h_parent || d_is_negative(h_parent))
+			continue;
+
+		if (!au_br_rdonly(au_sbr(sb, bindex))) {
+			err = bindex;
+			break;
+		}
+	}
+	dput(parent);
+
+	/* bottom up here */
+	if (unlikely(err < 0)) {
+		err = au_wbr_bu(sb, btop - 1);
+		if (err >= 0)
+			err = au_wbr_nonopq(dentry, err);
+	}
+
+out:
+	AuDbg("b%d\n", err);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* an exception for the policy other than tdp */
+static int au_wbr_create_exp(struct dentry *dentry)
+{
+	int err;
+	aufs_bindex_t bwh, bdiropq;
+	struct dentry *parent;
+
+	err = -1;
+	bwh = au_dbwh(dentry);
+	parent = dget_parent(dentry);
+	bdiropq = au_dbdiropq(parent);
+	if (bwh >= 0) {
+		if (bdiropq >= 0)
+			err = min(bdiropq, bwh);
+		else
+			err = bwh;
+		AuDbg("%d\n", err);
+	} else if (bdiropq >= 0) {
+		err = bdiropq;
+		AuDbg("%d\n", err);
+	}
+	dput(parent);
+
+	if (err >= 0)
+		err = au_wbr_nonopq(dentry, err);
+
+	if (err >= 0 && au_br_rdonly(au_sbr(dentry->d_sb, err)))
+		err = -1;
+
+	AuDbg("%d\n", err);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* round robin */
+static int au_wbr_create_init_rr(struct super_block *sb)
+{
+	int err;
+
+	err = au_wbr_bu(sb, au_sbbot(sb));
+	atomic_set(&au_sbi(sb)->si_wbr_rr_next, -err); /* less important */
+	/* smp_mb(); */
+
+	AuDbg("b%d\n", err);
+	return err;
+}
+
+static int au_wbr_create_rr(struct dentry *dentry, unsigned int flags)
+{
+	int err, nbr;
+	unsigned int u;
+	aufs_bindex_t bindex, bbot;
+	struct super_block *sb;
+	atomic_t *next;
+
+	err = au_wbr_create_exp(dentry);
+	if (err >= 0)
+		goto out;
+
+	sb = dentry->d_sb;
+	next = &au_sbi(sb)->si_wbr_rr_next;
+	bbot = au_sbbot(sb);
+	nbr = bbot + 1;
+	for (bindex = 0; bindex <= bbot; bindex++) {
+		if (!au_ftest_wbr(flags, DIR)) {
+			err = atomic_dec_return(next) + 1;
+			/* modulo for 0 is meaningless */
+			if (unlikely(!err))
+				err = atomic_dec_return(next) + 1;
+		} else
+			err = atomic_read(next);
+		AuDbg("%d\n", err);
+		u = err;
+		err = u % nbr;
+		AuDbg("%d\n", err);
+		if (!au_br_rdonly(au_sbr(sb, err)))
+			break;
+		err = -EROFS;
+	}
+
+	if (err >= 0)
+		err = au_wbr_nonopq(dentry, err);
+
+out:
+	AuDbg("%d\n", err);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* most free space */
+static void au_mfs(struct dentry *dentry, struct dentry *parent)
+{
+	struct super_block *sb;
+	struct au_branch *br;
+	struct au_wbr_mfs *mfs;
+	struct dentry *h_parent;
+	aufs_bindex_t bindex, bbot;
+	int err;
+	unsigned long long b, bavail;
+	struct path h_path;
+	/* reduce the stack usage */
+	struct kstatfs *st;
+
+	st = kmalloc(sizeof(*st), GFP_NOFS);
+	if (unlikely(!st)) {
+		AuWarn1("failed updating mfs(%d), ignored\n", -ENOMEM);
+		return;
+	}
+
+	bavail = 0;
+	sb = dentry->d_sb;
+	mfs = &au_sbi(sb)->si_wbr_mfs;
+	MtxMustLock(&mfs->mfs_lock);
+	mfs->mfs_bindex = -EROFS;
+	mfs->mfsrr_bytes = 0;
+	if (!parent) {
+		bindex = 0;
+		bbot = au_sbbot(sb);
+	} else {
+		bindex = au_dbtop(parent);
+		bbot = au_dbtaildir(parent);
+	}
+
+	for (; bindex <= bbot; bindex++) {
+		if (parent) {
+			h_parent = au_h_dptr(parent, bindex);
+			if (!h_parent || d_is_negative(h_parent))
+				continue;
+		}
+		br = au_sbr(sb, bindex);
+		if (au_br_rdonly(br))
+			continue;
+
+		/* sb->s_root for NFS is unreliable */
+		h_path.mnt = au_br_mnt(br);
+		h_path.dentry = h_path.mnt->mnt_root;
+		err = vfs_statfs(&h_path, st);
+		if (unlikely(err)) {
+			AuWarn1("failed statfs, b%d, %d\n", bindex, err);
+			continue;
+		}
+
+		/* when the available size is equal, select the lower one */
+		BUILD_BUG_ON(sizeof(b) < sizeof(st->f_bavail)
+			     || sizeof(b) < sizeof(st->f_bsize));
+		b = st->f_bavail * st->f_bsize;
+		br->br_wbr->wbr_bytes = b;
+		if (b >= bavail) {
+			bavail = b;
+			mfs->mfs_bindex = bindex;
+			mfs->mfs_jiffy = jiffies;
+		}
+	}
+
+	mfs->mfsrr_bytes = bavail;
+	AuDbg("b%d\n", mfs->mfs_bindex);
+	au_kfree_rcu(st);
+}
+
+static int au_wbr_create_mfs(struct dentry *dentry, unsigned int flags)
+{
+	int err;
+	struct dentry *parent;
+	struct super_block *sb;
+	struct au_wbr_mfs *mfs;
+
+	err = au_wbr_create_exp(dentry);
+	if (err >= 0)
+		goto out;
+
+	sb = dentry->d_sb;
+	parent = NULL;
+	if (au_ftest_wbr(flags, PARENT))
+		parent = dget_parent(dentry);
+	mfs = &au_sbi(sb)->si_wbr_mfs;
+	mutex_lock(&mfs->mfs_lock);
+	if (time_after(jiffies, mfs->mfs_jiffy + mfs->mfs_expire)
+	    || mfs->mfs_bindex < 0
+	    || au_br_rdonly(au_sbr(sb, mfs->mfs_bindex)))
+		au_mfs(dentry, parent);
+	mutex_unlock(&mfs->mfs_lock);
+	err = mfs->mfs_bindex;
+	dput(parent);
+
+	if (err >= 0)
+		err = au_wbr_nonopq(dentry, err);
+
+out:
+	AuDbg("b%d\n", err);
+	return err;
+}
+
+static int au_wbr_create_init_mfs(struct super_block *sb)
+{
+	struct au_wbr_mfs *mfs;
+
+	mfs = &au_sbi(sb)->si_wbr_mfs;
+	mutex_init(&mfs->mfs_lock);
+	mfs->mfs_jiffy = 0;
+	mfs->mfs_bindex = -EROFS;
+
+	return 0;
+}
+
+static int au_wbr_create_fin_mfs(struct super_block *sb __maybe_unused)
+{
+	mutex_destroy(&au_sbi(sb)->si_wbr_mfs.mfs_lock);
+	return 0;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* top down regardless parent, and then mfs */
+static int au_wbr_create_tdmfs(struct dentry *dentry,
+			       unsigned int flags __maybe_unused)
+{
+	int err;
+	aufs_bindex_t bwh, btail, bindex, bfound, bmfs;
+	unsigned long long watermark;
+	struct super_block *sb;
+	struct au_wbr_mfs *mfs;
+	struct au_branch *br;
+	struct dentry *parent;
+
+	sb = dentry->d_sb;
+	mfs = &au_sbi(sb)->si_wbr_mfs;
+	mutex_lock(&mfs->mfs_lock);
+	if (time_after(jiffies, mfs->mfs_jiffy + mfs->mfs_expire)
+	    || mfs->mfs_bindex < 0)
+		au_mfs(dentry, /*parent*/NULL);
+	watermark = mfs->mfsrr_watermark;
+	bmfs = mfs->mfs_bindex;
+	mutex_unlock(&mfs->mfs_lock);
+
+	/* another style of au_wbr_create_exp() */
+	bwh = au_dbwh(dentry);
+	parent = dget_parent(dentry);
+	btail = au_dbtaildir(parent);
+	if (bwh >= 0 && bwh < btail)
+		btail = bwh;
+
+	err = au_wbr_nonopq(dentry, btail);
+	if (unlikely(err < 0))
+		goto out;
+	btail = err;
+	bfound = -1;
+	for (bindex = 0; bindex <= btail; bindex++) {
+		br = au_sbr(sb, bindex);
+		if (au_br_rdonly(br))
+			continue;
+		if (br->br_wbr->wbr_bytes > watermark) {
+			bfound = bindex;
+			break;
+		}
+	}
+	err = bfound;
+	if (err < 0)
+		err = bmfs;
+
+out:
+	dput(parent);
+	AuDbg("b%d\n", err);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* most free space and then round robin */
+static int au_wbr_create_mfsrr(struct dentry *dentry, unsigned int flags)
+{
+	int err;
+	struct au_wbr_mfs *mfs;
+
+	err = au_wbr_create_mfs(dentry, flags);
+	if (err >= 0) {
+		mfs = &au_sbi(dentry->d_sb)->si_wbr_mfs;
+		mutex_lock(&mfs->mfs_lock);
+		if (mfs->mfsrr_bytes < mfs->mfsrr_watermark)
+			err = au_wbr_create_rr(dentry, flags);
+		mutex_unlock(&mfs->mfs_lock);
+	}
+
+	AuDbg("b%d\n", err);
+	return err;
+}
+
+static int au_wbr_create_init_mfsrr(struct super_block *sb)
+{
+	int err;
+
+	au_wbr_create_init_mfs(sb); /* ignore */
+	err = au_wbr_create_init_rr(sb);
+
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* top down parent and most free space */
+static int au_wbr_create_pmfs(struct dentry *dentry, unsigned int flags)
+{
+	int err, e2;
+	unsigned long long b;
+	aufs_bindex_t bindex, btop, bbot;
+	struct super_block *sb;
+	struct dentry *parent, *h_parent;
+	struct au_branch *br;
+
+	err = au_wbr_create_tdp(dentry, flags);
+	if (unlikely(err < 0))
+		goto out;
+	parent = dget_parent(dentry);
+	btop = au_dbtop(parent);
+	bbot = au_dbtaildir(parent);
+	if (btop == bbot)
+		goto out_parent; /* success */
+
+	e2 = au_wbr_create_mfs(dentry, flags);
+	if (e2 < 0)
+		goto out_parent; /* success */
+
+	/* when the available size is equal, select upper one */
+	sb = dentry->d_sb;
+	br = au_sbr(sb, err);
+	b = br->br_wbr->wbr_bytes;
+	AuDbg("b%d, %llu\n", err, b);
+
+	for (bindex = btop; bindex <= bbot; bindex++) {
+		h_parent = au_h_dptr(parent, bindex);
+		if (!h_parent || d_is_negative(h_parent))
+			continue;
+
+		br = au_sbr(sb, bindex);
+		if (!au_br_rdonly(br) && br->br_wbr->wbr_bytes > b) {
+			b = br->br_wbr->wbr_bytes;
+			err = bindex;
+			AuDbg("b%d, %llu\n", err, b);
+		}
+	}
+
+	if (err >= 0)
+		err = au_wbr_nonopq(dentry, err);
+
+out_parent:
+	dput(parent);
+out:
+	AuDbg("b%d\n", err);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * - top down parent
+ * - most free space with parent
+ * - most free space round-robin regardless parent
+ */
+static int au_wbr_create_pmfsrr(struct dentry *dentry, unsigned int flags)
+{
+	int err;
+	unsigned long long watermark;
+	struct super_block *sb;
+	struct au_branch *br;
+	struct au_wbr_mfs *mfs;
+
+	err = au_wbr_create_pmfs(dentry, flags | AuWbr_PARENT);
+	if (unlikely(err < 0))
+		goto out;
+
+	sb = dentry->d_sb;
+	br = au_sbr(sb, err);
+	mfs = &au_sbi(sb)->si_wbr_mfs;
+	mutex_lock(&mfs->mfs_lock);
+	watermark = mfs->mfsrr_watermark;
+	mutex_unlock(&mfs->mfs_lock);
+	if (br->br_wbr->wbr_bytes < watermark)
+		/* regardless the parent dir */
+		err = au_wbr_create_mfsrr(dentry, flags);
+
+out:
+	AuDbg("b%d\n", err);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* policies for copyup */
+
+/* top down parent */
+static int au_wbr_copyup_tdp(struct dentry *dentry)
+{
+	return au_wbr_create_tdp(dentry, /*flags, anything is ok*/0);
+}
+
+/* bottom up parent */
+static int au_wbr_copyup_bup(struct dentry *dentry)
+{
+	int err;
+	aufs_bindex_t bindex, btop;
+	struct dentry *parent, *h_parent;
+	struct super_block *sb;
+
+	err = -EROFS;
+	sb = dentry->d_sb;
+	parent = dget_parent(dentry);
+	btop = au_dbtop(parent);
+	for (bindex = au_dbtop(dentry); bindex >= btop; bindex--) {
+		h_parent = au_h_dptr(parent, bindex);
+		if (!h_parent || d_is_negative(h_parent))
+			continue;
+
+		if (!au_br_rdonly(au_sbr(sb, bindex))) {
+			err = bindex;
+			break;
+		}
+	}
+	dput(parent);
+
+	/* bottom up here */
+	if (unlikely(err < 0))
+		err = au_wbr_bu(sb, btop - 1);
+
+	AuDbg("b%d\n", err);
+	return err;
+}
+
+/* bottom up */
+int au_wbr_do_copyup_bu(struct dentry *dentry, aufs_bindex_t btop)
+{
+	int err;
+
+	err = au_wbr_bu(dentry->d_sb, btop);
+	AuDbg("b%d\n", err);
+	if (err > btop)
+		err = au_wbr_nonopq(dentry, err);
+
+	AuDbg("b%d\n", err);
+	return err;
+}
+
+static int au_wbr_copyup_bu(struct dentry *dentry)
+{
+	int err;
+	aufs_bindex_t btop;
+
+	btop = au_dbtop(dentry);
+	err = au_wbr_do_copyup_bu(dentry, btop);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+struct au_wbr_copyup_operations au_wbr_copyup_ops[] = {
+	[AuWbrCopyup_TDP] = {
+		.copyup	= au_wbr_copyup_tdp
+	},
+	[AuWbrCopyup_BUP] = {
+		.copyup	= au_wbr_copyup_bup
+	},
+	[AuWbrCopyup_BU] = {
+		.copyup	= au_wbr_copyup_bu
+	}
+};
+
+struct au_wbr_create_operations au_wbr_create_ops[] = {
+	[AuWbrCreate_TDP] = {
+		.create	= au_wbr_create_tdp
+	},
+	[AuWbrCreate_RR] = {
+		.create	= au_wbr_create_rr,
+		.init	= au_wbr_create_init_rr
+	},
+	[AuWbrCreate_MFS] = {
+		.create	= au_wbr_create_mfs,
+		.init	= au_wbr_create_init_mfs,
+		.fin	= au_wbr_create_fin_mfs
+	},
+	[AuWbrCreate_MFSV] = {
+		.create	= au_wbr_create_mfs,
+		.init	= au_wbr_create_init_mfs,
+		.fin	= au_wbr_create_fin_mfs
+	},
+	[AuWbrCreate_MFSRR] = {
+		.create	= au_wbr_create_mfsrr,
+		.init	= au_wbr_create_init_mfsrr,
+		.fin	= au_wbr_create_fin_mfs
+	},
+	[AuWbrCreate_MFSRRV] = {
+		.create	= au_wbr_create_mfsrr,
+		.init	= au_wbr_create_init_mfsrr,
+		.fin	= au_wbr_create_fin_mfs
+	},
+	[AuWbrCreate_TDMFS] = {
+		.create	= au_wbr_create_tdmfs,
+		.init	= au_wbr_create_init_mfs,
+		.fin	= au_wbr_create_fin_mfs
+	},
+	[AuWbrCreate_TDMFSV] = {
+		.create	= au_wbr_create_tdmfs,
+		.init	= au_wbr_create_init_mfs,
+		.fin	= au_wbr_create_fin_mfs
+	},
+	[AuWbrCreate_PMFS] = {
+		.create	= au_wbr_create_pmfs,
+		.init	= au_wbr_create_init_mfs,
+		.fin	= au_wbr_create_fin_mfs
+	},
+	[AuWbrCreate_PMFSV] = {
+		.create	= au_wbr_create_pmfs,
+		.init	= au_wbr_create_init_mfs,
+		.fin	= au_wbr_create_fin_mfs
+	},
+	[AuWbrCreate_PMFSRR] = {
+		.create	= au_wbr_create_pmfsrr,
+		.init	= au_wbr_create_init_mfsrr,
+		.fin	= au_wbr_create_fin_mfs
+	},
+	[AuWbrCreate_PMFSRRV] = {
+		.create	= au_wbr_create_pmfsrr,
+		.init	= au_wbr_create_init_mfsrr,
+		.fin	= au_wbr_create_fin_mfs
+	}
+};
diff -Naur --no-dereference a/fs/aufs/whout.c b/fs/aufs/whout.c
--- a/fs/aufs/whout.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/whout.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,1062 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * whiteout for logical deletion and opaque directory
+ */
+
+#include "aufs.h"
+
+#define WH_MASK			0444
+
+/*
+ * If a directory contains this file, then it is opaque.  We start with the
+ * .wh. flag so that it is blocked by lookup.
+ */
+static struct qstr diropq_name = QSTR_INIT(AUFS_WH_DIROPQ,
+					   sizeof(AUFS_WH_DIROPQ) - 1);
+
+/*
+ * generate whiteout name, which is NOT terminated by NULL.
+ * @name: original d_name.name
+ * @len: original d_name.len
+ * @wh: whiteout qstr
+ * returns zero when succeeds, otherwise error.
+ * succeeded value as wh->name should be freed by kfree().
+ */
+int au_wh_name_alloc(struct qstr *wh, const struct qstr *name)
+{
+	char *p;
+
+	if (unlikely(name->len > PATH_MAX - AUFS_WH_PFX_LEN))
+		return -ENAMETOOLONG;
+
+	wh->len = name->len + AUFS_WH_PFX_LEN;
+	p = kmalloc(wh->len, GFP_NOFS);
+	wh->name = p;
+	if (p) {
+		memcpy(p, AUFS_WH_PFX, AUFS_WH_PFX_LEN);
+		memcpy(p + AUFS_WH_PFX_LEN, name->name, name->len);
+		/* smp_mb(); */
+		return 0;
+	}
+	return -ENOMEM;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * test if the @wh_name exists under @h_parent.
+ * @try_sio specifies the necessary of super-io.
+ */
+int au_wh_test(struct dentry *h_parent, struct qstr *wh_name, int try_sio)
+{
+	int err;
+	struct dentry *wh_dentry;
+
+	if (!try_sio)
+		wh_dentry = vfsub_lkup_one(wh_name, h_parent);
+	else
+		wh_dentry = au_sio_lkup_one(wh_name, h_parent);
+	err = PTR_ERR(wh_dentry);
+	if (IS_ERR(wh_dentry)) {
+		if (err == -ENAMETOOLONG)
+			err = 0;
+		goto out;
+	}
+
+	err = 0;
+	if (d_is_negative(wh_dentry))
+		goto out_wh; /* success */
+
+	err = 1;
+	if (d_is_reg(wh_dentry))
+		goto out_wh; /* success */
+
+	err = -EIO;
+	AuIOErr("%pd Invalid whiteout entry type 0%o.\n",
+		wh_dentry, d_inode(wh_dentry)->i_mode);
+
+out_wh:
+	dput(wh_dentry);
+out:
+	return err;
+}
+
+/*
+ * test if the @h_dentry sets opaque or not.
+ */
+int au_diropq_test(struct dentry *h_dentry)
+{
+	int err;
+	struct inode *h_dir;
+
+	h_dir = d_inode(h_dentry);
+	err = au_wh_test(h_dentry, &diropq_name,
+			 au_test_h_perm_sio(h_dir, MAY_EXEC));
+	return err;
+}
+
+/*
+ * returns a negative dentry whose name is unique and temporary.
+ */
+struct dentry *au_whtmp_lkup(struct dentry *h_parent, struct au_branch *br,
+			     struct qstr *prefix)
+{
+	struct dentry *dentry;
+	int i;
+	char defname[NAME_MAX - AUFS_MAX_NAMELEN + DNAME_INLINE_LEN + 1],
+		*name, *p;
+	/* strict atomic_t is unnecessary here */
+	static unsigned short cnt;
+	struct qstr qs;
+
+	BUILD_BUG_ON(sizeof(cnt) * 2 > AUFS_WH_TMP_LEN);
+
+	name = defname;
+	qs.len = sizeof(defname) - DNAME_INLINE_LEN + prefix->len - 1;
+	if (unlikely(prefix->len > DNAME_INLINE_LEN)) {
+		dentry = ERR_PTR(-ENAMETOOLONG);
+		if (unlikely(qs.len > NAME_MAX))
+			goto out;
+		dentry = ERR_PTR(-ENOMEM);
+		name = kmalloc(qs.len + 1, GFP_NOFS);
+		if (unlikely(!name))
+			goto out;
+	}
+
+	/* doubly whiteout-ed */
+	memcpy(name, AUFS_WH_PFX AUFS_WH_PFX, AUFS_WH_PFX_LEN * 2);
+	p = name + AUFS_WH_PFX_LEN * 2;
+	memcpy(p, prefix->name, prefix->len);
+	p += prefix->len;
+	*p++ = '.';
+	AuDebugOn(name + qs.len + 1 - p <= AUFS_WH_TMP_LEN);
+
+	qs.name = name;
+	for (i = 0; i < 3; i++) {
+		sprintf(p, "%.*x", AUFS_WH_TMP_LEN, cnt++);
+		dentry = au_sio_lkup_one(&qs, h_parent);
+		if (IS_ERR(dentry) || d_is_negative(dentry))
+			goto out_name;
+		dput(dentry);
+	}
+	/* pr_warn("could not get random name\n"); */
+	dentry = ERR_PTR(-EEXIST);
+	AuDbg("%.*s\n", AuLNPair(&qs));
+	BUG();
+
+out_name:
+	if (name != defname)
+		au_kfree_try_rcu(name);
+out:
+	AuTraceErrPtr(dentry);
+	return dentry;
+}
+
+/*
+ * rename the @h_dentry on @br to the whiteouted temporary name.
+ */
+int au_whtmp_ren(struct dentry *h_dentry, struct au_branch *br)
+{
+	int err;
+	struct path h_path = {
+		.mnt = au_br_mnt(br)
+	};
+	struct inode *h_dir, *delegated;
+	struct dentry *h_parent;
+
+	h_parent = h_dentry->d_parent; /* dir inode is locked */
+	h_dir = d_inode(h_parent);
+	IMustLock(h_dir);
+
+	h_path.dentry = au_whtmp_lkup(h_parent, br, &h_dentry->d_name);
+	err = PTR_ERR(h_path.dentry);
+	if (IS_ERR(h_path.dentry))
+		goto out;
+
+	/* under the same dir, no need to lock_rename() */
+	delegated = NULL;
+	err = vfsub_rename(h_dir, h_dentry, h_dir, &h_path, &delegated,
+			   /*flags*/0);
+	AuTraceErr(err);
+	if (unlikely(err == -EWOULDBLOCK)) {
+		pr_warn("cannot retry for NFSv4 delegation"
+			" for an internal rename\n");
+		iput(delegated);
+	}
+	dput(h_path.dentry);
+
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+/*
+ * functions for removing a whiteout
+ */
+
+static int do_unlink_wh(struct inode *h_dir, struct path *h_path)
+{
+	int err, force;
+	struct inode *delegated;
+
+	/*
+	 * forces superio when the dir has a sticky bit.
+	 * this may be a violation of unix fs semantics.
+	 */
+	force = (h_dir->i_mode & S_ISVTX)
+		&& !uid_eq(current_fsuid(), d_inode(h_path->dentry)->i_uid);
+	delegated = NULL;
+	err = vfsub_unlink(h_dir, h_path, &delegated, force);
+	if (unlikely(err == -EWOULDBLOCK)) {
+		pr_warn("cannot retry for NFSv4 delegation"
+			" for an internal unlink\n");
+		iput(delegated);
+	}
+	return err;
+}
+
+int au_wh_unlink_dentry(struct inode *h_dir, struct path *h_path,
+			struct dentry *dentry)
+{
+	int err;
+
+	err = do_unlink_wh(h_dir, h_path);
+	if (!err && dentry)
+		au_set_dbwh(dentry, -1);
+
+	return err;
+}
+
+static int unlink_wh_name(struct dentry *h_parent, struct qstr *wh,
+			  struct au_branch *br)
+{
+	int err;
+	struct path h_path = {
+		.mnt = au_br_mnt(br)
+	};
+
+	err = 0;
+	h_path.dentry = vfsub_lkup_one(wh, h_parent);
+	if (IS_ERR(h_path.dentry))
+		err = PTR_ERR(h_path.dentry);
+	else {
+		if (d_is_reg(h_path.dentry))
+			err = do_unlink_wh(d_inode(h_parent), &h_path);
+		dput(h_path.dentry);
+	}
+
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+/*
+ * initialize/clean whiteout for a branch
+ */
+
+static void au_wh_clean(struct inode *h_dir, struct path *whpath,
+			const int isdir)
+{
+	int err;
+	struct inode *delegated;
+
+	if (d_is_negative(whpath->dentry))
+		return;
+
+	if (isdir)
+		err = vfsub_rmdir(h_dir, whpath);
+	else {
+		delegated = NULL;
+		err = vfsub_unlink(h_dir, whpath, &delegated, /*force*/0);
+		if (unlikely(err == -EWOULDBLOCK)) {
+			pr_warn("cannot retry for NFSv4 delegation"
+				" for an internal unlink\n");
+			iput(delegated);
+		}
+	}
+	if (unlikely(err))
+		pr_warn("failed removing %pd (%d), ignored.\n",
+			whpath->dentry, err);
+}
+
+static int test_linkable(struct dentry *h_root)
+{
+	struct inode *h_dir = d_inode(h_root);
+
+	if (h_dir->i_op->link)
+		return 0;
+
+	pr_err("%pd (%s) doesn't support link(2), use noplink and rw+nolwh\n",
+	       h_root, au_sbtype(h_root->d_sb));
+	return -ENOSYS;	/* the branch doesn't have its ->link() */
+}
+
+/* todo: should this mkdir be done in /sbin/mount.aufs helper? */
+static int au_whdir(struct inode *h_dir, struct path *path)
+{
+	int err;
+
+	err = -EEXIST;
+	if (d_is_negative(path->dentry)) {
+		int mode = 0700;
+
+		if (au_test_nfs(path->dentry->d_sb))
+			mode |= 0111;
+		err = vfsub_mkdir(h_dir, path, mode);
+	} else if (d_is_dir(path->dentry))
+		err = 0;
+	else
+		pr_err("unknown %pd exists\n", path->dentry);
+
+	return err;
+}
+
+struct au_wh_base {
+	const struct qstr *name;
+	struct dentry *dentry;
+};
+
+static void au_wh_init_ro(struct inode *h_dir, struct au_wh_base base[],
+			  struct path *h_path)
+{
+	h_path->dentry = base[AuBrWh_BASE].dentry;
+	au_wh_clean(h_dir, h_path, /*isdir*/0);
+	h_path->dentry = base[AuBrWh_PLINK].dentry;
+	au_wh_clean(h_dir, h_path, /*isdir*/1);
+	h_path->dentry = base[AuBrWh_ORPH].dentry;
+	au_wh_clean(h_dir, h_path, /*isdir*/1);
+}
+
+/*
+ * returns tri-state,
+ * minus: error, caller should print the message
+ * zero: success
+ * plus: error, caller should NOT print the message
+ */
+static int au_wh_init_rw_nolink(struct dentry *h_root, struct au_wbr *wbr,
+				int do_plink, struct au_wh_base base[],
+				struct path *h_path)
+{
+	int err;
+	struct inode *h_dir;
+
+	h_dir = d_inode(h_root);
+	h_path->dentry = base[AuBrWh_BASE].dentry;
+	au_wh_clean(h_dir, h_path, /*isdir*/0);
+	h_path->dentry = base[AuBrWh_PLINK].dentry;
+	if (do_plink) {
+		err = test_linkable(h_root);
+		if (unlikely(err)) {
+			err = 1;
+			goto out;
+		}
+
+		err = au_whdir(h_dir, h_path);
+		if (unlikely(err))
+			goto out;
+		wbr->wbr_plink = dget(base[AuBrWh_PLINK].dentry);
+	} else
+		au_wh_clean(h_dir, h_path, /*isdir*/1);
+	h_path->dentry = base[AuBrWh_ORPH].dentry;
+	err = au_whdir(h_dir, h_path);
+	if (unlikely(err))
+		goto out;
+	wbr->wbr_orph = dget(base[AuBrWh_ORPH].dentry);
+
+out:
+	return err;
+}
+
+/*
+ * for the moment, aufs supports the branch filesystem which does not support
+ * link(2). testing on FAT which does not support i_op->setattr() fully either,
+ * copyup failed. finally, such filesystem will not be used as the writable
+ * branch.
+ *
+ * returns tri-state, see above.
+ */
+static int au_wh_init_rw(struct dentry *h_root, struct au_wbr *wbr,
+			 int do_plink, struct au_wh_base base[],
+			 struct path *h_path)
+{
+	int err;
+	struct inode *h_dir;
+
+	WbrWhMustWriteLock(wbr);
+
+	err = test_linkable(h_root);
+	if (unlikely(err)) {
+		err = 1;
+		goto out;
+	}
+
+	/*
+	 * todo: should this create be done in /sbin/mount.aufs helper?
+	 */
+	err = -EEXIST;
+	h_dir = d_inode(h_root);
+	if (d_is_negative(base[AuBrWh_BASE].dentry)) {
+		h_path->dentry = base[AuBrWh_BASE].dentry;
+		err = vfsub_create(h_dir, h_path, WH_MASK, /*want_excl*/true);
+	} else if (d_is_reg(base[AuBrWh_BASE].dentry))
+		err = 0;
+	else
+		pr_err("unknown %pd2 exists\n", base[AuBrWh_BASE].dentry);
+	if (unlikely(err))
+		goto out;
+
+	h_path->dentry = base[AuBrWh_PLINK].dentry;
+	if (do_plink) {
+		err = au_whdir(h_dir, h_path);
+		if (unlikely(err))
+			goto out;
+		wbr->wbr_plink = dget(base[AuBrWh_PLINK].dentry);
+	} else
+		au_wh_clean(h_dir, h_path, /*isdir*/1);
+	wbr->wbr_whbase = dget(base[AuBrWh_BASE].dentry);
+
+	h_path->dentry = base[AuBrWh_ORPH].dentry;
+	err = au_whdir(h_dir, h_path);
+	if (unlikely(err))
+		goto out;
+	wbr->wbr_orph = dget(base[AuBrWh_ORPH].dentry);
+
+out:
+	return err;
+}
+
+/*
+ * initialize the whiteout base file/dir for @br.
+ */
+int au_wh_init(struct au_branch *br, struct super_block *sb)
+{
+	int err, i;
+	const unsigned char do_plink
+		= !!au_opt_test(au_mntflags(sb), PLINK);
+	struct inode *h_dir;
+	struct path path = br->br_path;
+	struct dentry *h_root = path.dentry;
+	struct au_wbr *wbr = br->br_wbr;
+	static const struct qstr base_name[] = {
+		[AuBrWh_BASE] = QSTR_INIT(AUFS_BASE_NAME,
+					  sizeof(AUFS_BASE_NAME) - 1),
+		[AuBrWh_PLINK] = QSTR_INIT(AUFS_PLINKDIR_NAME,
+					   sizeof(AUFS_PLINKDIR_NAME) - 1),
+		[AuBrWh_ORPH] = QSTR_INIT(AUFS_ORPHDIR_NAME,
+					  sizeof(AUFS_ORPHDIR_NAME) - 1)
+	};
+	struct au_wh_base base[] = {
+		[AuBrWh_BASE] = {
+			.name	= base_name + AuBrWh_BASE,
+			.dentry	= NULL
+		},
+		[AuBrWh_PLINK] = {
+			.name	= base_name + AuBrWh_PLINK,
+			.dentry	= NULL
+		},
+		[AuBrWh_ORPH] = {
+			.name	= base_name + AuBrWh_ORPH,
+			.dentry	= NULL
+		}
+	};
+
+	if (wbr)
+		WbrWhMustWriteLock(wbr);
+
+	for (i = 0; i < AuBrWh_Last; i++) {
+		/* doubly whiteouted */
+		struct dentry *d;
+
+		d = au_wh_lkup(h_root, (void *)base[i].name, br);
+		err = PTR_ERR(d);
+		if (IS_ERR(d))
+			goto out;
+
+		base[i].dentry = d;
+		AuDebugOn(wbr
+			  && wbr->wbr_wh[i]
+			  && wbr->wbr_wh[i] != base[i].dentry);
+	}
+
+	if (wbr)
+		for (i = 0; i < AuBrWh_Last; i++) {
+			dput(wbr->wbr_wh[i]);
+			wbr->wbr_wh[i] = NULL;
+		}
+
+	err = 0;
+	if (!au_br_writable(br->br_perm)) {
+		h_dir = d_inode(h_root);
+		au_wh_init_ro(h_dir, base, &path);
+	} else if (!au_br_wh_linkable(br->br_perm)) {
+		err = au_wh_init_rw_nolink(h_root, wbr, do_plink, base, &path);
+		if (err > 0)
+			goto out;
+		else if (err)
+			goto out_err;
+	} else {
+		err = au_wh_init_rw(h_root, wbr, do_plink, base, &path);
+		if (err > 0)
+			goto out;
+		else if (err)
+			goto out_err;
+	}
+	goto out; /* success */
+
+out_err:
+	pr_err("an error(%d) on the writable branch %pd(%s)\n",
+	       err, h_root, au_sbtype(h_root->d_sb));
+out:
+	for (i = 0; i < AuBrWh_Last; i++)
+		dput(base[i].dentry);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+/*
+ * whiteouts are all hard-linked usually.
+ * when its link count reaches a ceiling, we create a new whiteout base
+ * asynchronously.
+ */
+
+struct reinit_br_wh {
+	struct super_block *sb;
+	struct au_branch *br;
+};
+
+static void reinit_br_wh(void *arg)
+{
+	int err;
+	aufs_bindex_t bindex;
+	struct path h_path;
+	struct reinit_br_wh *a = arg;
+	struct au_wbr *wbr;
+	struct inode *dir, *delegated;
+	struct dentry *h_root;
+	struct au_hinode *hdir;
+
+	err = 0;
+	wbr = a->br->br_wbr;
+	/* big aufs lock */
+	si_noflush_write_lock(a->sb);
+	if (!au_br_writable(a->br->br_perm))
+		goto out;
+	bindex = au_br_index(a->sb, a->br->br_id);
+	if (unlikely(bindex < 0))
+		goto out;
+
+	di_read_lock_parent(a->sb->s_root, AuLock_IR);
+	dir = d_inode(a->sb->s_root);
+	hdir = au_hi(dir, bindex);
+	h_root = au_h_dptr(a->sb->s_root, bindex);
+	AuDebugOn(h_root != au_br_dentry(a->br));
+
+	au_hn_inode_lock_nested(hdir, AuLsc_I_PARENT);
+	wbr_wh_write_lock(wbr);
+	err = au_h_verify(wbr->wbr_whbase, au_opt_udba(a->sb), hdir->hi_inode,
+			  h_root, a->br);
+	if (!err) {
+		h_path.dentry = wbr->wbr_whbase;
+		h_path.mnt = au_br_mnt(a->br);
+		delegated = NULL;
+		err = vfsub_unlink(hdir->hi_inode, &h_path, &delegated,
+				   /*force*/0);
+		if (unlikely(err == -EWOULDBLOCK)) {
+			pr_warn("cannot retry for NFSv4 delegation"
+				" for an internal unlink\n");
+			iput(delegated);
+		}
+	} else {
+		pr_warn("%pd is moved, ignored\n", wbr->wbr_whbase);
+		err = 0;
+	}
+	dput(wbr->wbr_whbase);
+	wbr->wbr_whbase = NULL;
+	if (!err)
+		err = au_wh_init(a->br, a->sb);
+	wbr_wh_write_unlock(wbr);
+	au_hn_inode_unlock(hdir);
+	di_read_unlock(a->sb->s_root, AuLock_IR);
+	if (!err)
+		au_fhsm_wrote(a->sb, bindex, /*force*/0);
+
+out:
+	if (wbr)
+		atomic_dec(&wbr->wbr_wh_running);
+	au_lcnt_dec(&a->br->br_count);
+	si_write_unlock(a->sb);
+	au_nwt_done(&au_sbi(a->sb)->si_nowait);
+	au_kfree_rcu(a);
+	if (unlikely(err))
+		AuIOErr("err %d\n", err);
+}
+
+static void kick_reinit_br_wh(struct super_block *sb, struct au_branch *br)
+{
+	int do_dec, wkq_err;
+	struct reinit_br_wh *arg;
+
+	do_dec = 1;
+	if (atomic_inc_return(&br->br_wbr->wbr_wh_running) != 1)
+		goto out;
+
+	/* ignore ENOMEM */
+	arg = kmalloc(sizeof(*arg), GFP_NOFS);
+	if (arg) {
+		/*
+		 * dec(wh_running), kfree(arg) and dec(br_count)
+		 * in reinit function
+		 */
+		arg->sb = sb;
+		arg->br = br;
+		au_lcnt_inc(&br->br_count);
+		wkq_err = au_wkq_nowait(reinit_br_wh, arg, sb, /*flags*/0);
+		if (unlikely(wkq_err)) {
+			atomic_dec(&br->br_wbr->wbr_wh_running);
+			au_lcnt_dec(&br->br_count);
+			au_kfree_rcu(arg);
+		}
+		do_dec = 0;
+	}
+
+out:
+	if (do_dec)
+		atomic_dec(&br->br_wbr->wbr_wh_running);
+}
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * create the whiteout @wh.
+ */
+static int link_or_create_wh(struct super_block *sb, aufs_bindex_t bindex,
+			     struct dentry *wh)
+{
+	int err;
+	struct path h_path = {
+		.dentry = wh
+	};
+	struct au_branch *br;
+	struct au_wbr *wbr;
+	struct dentry *h_parent;
+	struct inode *h_dir, *delegated;
+
+	h_parent = wh->d_parent; /* dir inode is locked */
+	h_dir = d_inode(h_parent);
+	IMustLock(h_dir);
+
+	br = au_sbr(sb, bindex);
+	h_path.mnt = au_br_mnt(br);
+	wbr = br->br_wbr;
+	wbr_wh_read_lock(wbr);
+	if (wbr->wbr_whbase) {
+		delegated = NULL;
+		err = vfsub_link(wbr->wbr_whbase, h_dir, &h_path, &delegated);
+		if (unlikely(err == -EWOULDBLOCK)) {
+			pr_warn("cannot retry for NFSv4 delegation"
+				" for an internal link\n");
+			iput(delegated);
+		}
+		if (!err || err != -EMLINK)
+			goto out;
+
+		/* link count full. re-initialize br_whbase. */
+		kick_reinit_br_wh(sb, br);
+	}
+
+	/* return this error in this context */
+	err = vfsub_create(h_dir, &h_path, WH_MASK, /*want_excl*/true);
+	if (!err)
+		au_fhsm_wrote(sb, bindex, /*force*/0);
+
+out:
+	wbr_wh_read_unlock(wbr);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * create or remove the diropq.
+ */
+static struct dentry *do_diropq(struct dentry *dentry, aufs_bindex_t bindex,
+				unsigned int flags)
+{
+	struct dentry *opq_dentry, *h_dentry;
+	struct super_block *sb;
+	struct au_branch *br;
+	int err;
+
+	sb = dentry->d_sb;
+	br = au_sbr(sb, bindex);
+	h_dentry = au_h_dptr(dentry, bindex);
+	opq_dentry = vfsub_lkup_one(&diropq_name, h_dentry);
+	if (IS_ERR(opq_dentry))
+		goto out;
+
+	if (au_ftest_diropq(flags, CREATE)) {
+		err = link_or_create_wh(sb, bindex, opq_dentry);
+		if (!err) {
+			au_set_dbdiropq(dentry, bindex);
+			goto out; /* success */
+		}
+	} else {
+		struct path tmp = {
+			.dentry = opq_dentry,
+			.mnt	= au_br_mnt(br)
+		};
+		err = do_unlink_wh(au_h_iptr(d_inode(dentry), bindex), &tmp);
+		if (!err)
+			au_set_dbdiropq(dentry, -1);
+	}
+	dput(opq_dentry);
+	opq_dentry = ERR_PTR(err);
+
+out:
+	return opq_dentry;
+}
+
+struct do_diropq_args {
+	struct dentry **errp;
+	struct dentry *dentry;
+	aufs_bindex_t bindex;
+	unsigned int flags;
+};
+
+static void call_do_diropq(void *args)
+{
+	struct do_diropq_args *a = args;
+	*a->errp = do_diropq(a->dentry, a->bindex, a->flags);
+}
+
+struct dentry *au_diropq_sio(struct dentry *dentry, aufs_bindex_t bindex,
+			     unsigned int flags)
+{
+	struct dentry *diropq, *h_dentry;
+
+	h_dentry = au_h_dptr(dentry, bindex);
+	if (!au_test_h_perm_sio(d_inode(h_dentry), MAY_EXEC | MAY_WRITE))
+		diropq = do_diropq(dentry, bindex, flags);
+	else {
+		int wkq_err;
+		struct do_diropq_args args = {
+			.errp		= &diropq,
+			.dentry		= dentry,
+			.bindex		= bindex,
+			.flags		= flags
+		};
+
+		wkq_err = au_wkq_wait(call_do_diropq, &args);
+		if (unlikely(wkq_err))
+			diropq = ERR_PTR(wkq_err);
+	}
+
+	return diropq;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * lookup whiteout dentry.
+ * @h_parent: lower parent dentry which must exist and be locked
+ * @base_name: name of dentry which will be whiteouted
+ * returns dentry for whiteout.
+ */
+struct dentry *au_wh_lkup(struct dentry *h_parent, struct qstr *base_name,
+			  struct au_branch *br)
+{
+	int err;
+	struct qstr wh_name;
+	struct dentry *wh_dentry;
+
+	err = au_wh_name_alloc(&wh_name, base_name);
+	wh_dentry = ERR_PTR(err);
+	if (!err) {
+		wh_dentry = vfsub_lkup_one(&wh_name, h_parent);
+		au_kfree_try_rcu(wh_name.name);
+	}
+	return wh_dentry;
+}
+
+/*
+ * link/create a whiteout for @dentry on @bindex.
+ */
+struct dentry *au_wh_create(struct dentry *dentry, aufs_bindex_t bindex,
+			    struct dentry *h_parent)
+{
+	struct dentry *wh_dentry;
+	struct super_block *sb;
+	int err;
+
+	sb = dentry->d_sb;
+	wh_dentry = au_wh_lkup(h_parent, &dentry->d_name, au_sbr(sb, bindex));
+	if (!IS_ERR(wh_dentry) && d_is_negative(wh_dentry)) {
+		err = link_or_create_wh(sb, bindex, wh_dentry);
+		if (!err) {
+			au_set_dbwh(dentry, bindex);
+			au_fhsm_wrote(sb, bindex, /*force*/0);
+		} else {
+			dput(wh_dentry);
+			wh_dentry = ERR_PTR(err);
+		}
+	}
+
+	return wh_dentry;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* Delete all whiteouts in this directory on branch bindex. */
+static int del_wh_children(struct dentry *h_dentry, struct au_nhash *whlist,
+			   aufs_bindex_t bindex, struct au_branch *br)
+{
+	int err;
+	unsigned long ul, n;
+	struct qstr wh_name;
+	char *p;
+	struct hlist_head *head;
+	struct au_vdir_wh *pos;
+	struct au_vdir_destr *str;
+
+	err = -ENOMEM;
+	p = (void *)__get_free_page(GFP_NOFS);
+	wh_name.name = p;
+	if (unlikely(!wh_name.name))
+		goto out;
+
+	err = 0;
+	memcpy(p, AUFS_WH_PFX, AUFS_WH_PFX_LEN);
+	p += AUFS_WH_PFX_LEN;
+	n = whlist->nh_num;
+	head = whlist->nh_head;
+	for (ul = 0; !err && ul < n; ul++, head++) {
+		hlist_for_each_entry(pos, head, wh_hash) {
+			if (pos->wh_bindex != bindex)
+				continue;
+
+			str = &pos->wh_str;
+			if (str->len + AUFS_WH_PFX_LEN <= PATH_MAX) {
+				memcpy(p, str->name, str->len);
+				wh_name.len = AUFS_WH_PFX_LEN + str->len;
+				err = unlink_wh_name(h_dentry, &wh_name, br);
+				if (!err)
+					continue;
+				break;
+			}
+			AuIOErr("whiteout name too long %.*s\n",
+				str->len, str->name);
+			err = -EIO;
+			break;
+		}
+	}
+	free_page((unsigned long)wh_name.name);
+
+out:
+	return err;
+}
+
+struct del_wh_children_args {
+	int *errp;
+	struct dentry *h_dentry;
+	struct au_nhash *whlist;
+	aufs_bindex_t bindex;
+	struct au_branch *br;
+};
+
+static void call_del_wh_children(void *args)
+{
+	struct del_wh_children_args *a = args;
+	*a->errp = del_wh_children(a->h_dentry, a->whlist, a->bindex, a->br);
+}
+
+/* ---------------------------------------------------------------------- */
+
+struct au_whtmp_rmdir *au_whtmp_rmdir_alloc(struct super_block *sb, gfp_t gfp)
+{
+	struct au_whtmp_rmdir *whtmp;
+	int err;
+	unsigned int rdhash;
+
+	SiMustAnyLock(sb);
+
+	whtmp = kzalloc(sizeof(*whtmp), gfp);
+	if (unlikely(!whtmp)) {
+		whtmp = ERR_PTR(-ENOMEM);
+		goto out;
+	}
+
+	/* no estimation for dir size */
+	rdhash = au_sbi(sb)->si_rdhash;
+	if (!rdhash)
+		rdhash = AUFS_RDHASH_DEF;
+	err = au_nhash_alloc(&whtmp->whlist, rdhash, gfp);
+	if (unlikely(err)) {
+		au_kfree_rcu(whtmp);
+		whtmp = ERR_PTR(err);
+	}
+
+out:
+	return whtmp;
+}
+
+void au_whtmp_rmdir_free(struct au_whtmp_rmdir *whtmp)
+{
+	if (whtmp->br)
+		au_lcnt_dec(&whtmp->br->br_count);
+	dput(whtmp->wh_dentry);
+	iput(whtmp->dir);
+	au_nhash_wh_free(&whtmp->whlist);
+	au_kfree_rcu(whtmp);
+}
+
+/*
+ * rmdir the whiteouted temporary named dir @h_dentry.
+ * @whlist: whiteouted children.
+ */
+int au_whtmp_rmdir(struct inode *dir, aufs_bindex_t bindex,
+		   struct dentry *wh_dentry, struct au_nhash *whlist)
+{
+	int err;
+	unsigned int h_nlink;
+	struct path h_tmp;
+	struct inode *wh_inode, *h_dir;
+	struct au_branch *br;
+
+	h_dir = d_inode(wh_dentry->d_parent); /* dir inode is locked */
+	IMustLock(h_dir);
+
+	br = au_sbr(dir->i_sb, bindex);
+	wh_inode = d_inode(wh_dentry);
+	inode_lock_nested(wh_inode, AuLsc_I_CHILD);
+
+	/*
+	 * someone else might change some whiteouts while we were sleeping.
+	 * it means this whlist may have an obsoleted entry.
+	 */
+	if (!au_test_h_perm_sio(wh_inode, MAY_EXEC | MAY_WRITE))
+		err = del_wh_children(wh_dentry, whlist, bindex, br);
+	else {
+		int wkq_err;
+		struct del_wh_children_args args = {
+			.errp		= &err,
+			.h_dentry	= wh_dentry,
+			.whlist		= whlist,
+			.bindex		= bindex,
+			.br		= br
+		};
+
+		wkq_err = au_wkq_wait(call_del_wh_children, &args);
+		if (unlikely(wkq_err))
+			err = wkq_err;
+	}
+	inode_unlock(wh_inode);
+
+	if (!err) {
+		h_tmp.dentry = wh_dentry;
+		h_tmp.mnt = au_br_mnt(br);
+		h_nlink = h_dir->i_nlink;
+		err = vfsub_rmdir(h_dir, &h_tmp);
+		/* some fs doesn't change the parent nlink in some cases */
+		h_nlink -= h_dir->i_nlink;
+	}
+
+	if (!err) {
+		if (au_ibtop(dir) == bindex) {
+			/* todo: dir->i_mutex is necessary */
+			au_cpup_attr_timesizes(dir);
+			if (h_nlink)
+				vfsub_drop_nlink(dir);
+		}
+		return 0; /* success */
+	}
+
+	pr_warn("failed removing %pd(%d), ignored\n", wh_dentry, err);
+	return err;
+}
+
+static void call_rmdir_whtmp(void *args)
+{
+	int err;
+	aufs_bindex_t bindex;
+	struct au_whtmp_rmdir *a = args;
+	struct super_block *sb;
+	struct dentry *h_parent;
+	struct inode *h_dir;
+	struct au_hinode *hdir;
+
+	/* rmdir by nfsd may cause deadlock with this i_mutex */
+	/* inode_lock(a->dir); */
+	err = -EROFS;
+	sb = a->dir->i_sb;
+	si_read_lock(sb, !AuLock_FLUSH);
+	if (!au_br_writable(a->br->br_perm))
+		goto out;
+	bindex = au_br_index(sb, a->br->br_id);
+	if (unlikely(bindex < 0))
+		goto out;
+
+	err = -EIO;
+	ii_write_lock_parent(a->dir);
+	h_parent = dget_parent(a->wh_dentry);
+	h_dir = d_inode(h_parent);
+	hdir = au_hi(a->dir, bindex);
+	err = vfsub_mnt_want_write(au_br_mnt(a->br));
+	if (unlikely(err))
+		goto out_mnt;
+	au_hn_inode_lock_nested(hdir, AuLsc_I_PARENT);
+	err = au_h_verify(a->wh_dentry, au_opt_udba(sb), h_dir, h_parent,
+			  a->br);
+	if (!err)
+		err = au_whtmp_rmdir(a->dir, bindex, a->wh_dentry, &a->whlist);
+	au_hn_inode_unlock(hdir);
+	vfsub_mnt_drop_write(au_br_mnt(a->br));
+
+out_mnt:
+	dput(h_parent);
+	ii_write_unlock(a->dir);
+out:
+	/* inode_unlock(a->dir); */
+	au_whtmp_rmdir_free(a);
+	si_read_unlock(sb);
+	au_nwt_done(&au_sbi(sb)->si_nowait);
+	if (unlikely(err))
+		AuIOErr("err %d\n", err);
+}
+
+void au_whtmp_kick_rmdir(struct inode *dir, aufs_bindex_t bindex,
+			 struct dentry *wh_dentry, struct au_whtmp_rmdir *args)
+{
+	int wkq_err;
+	struct super_block *sb;
+
+	IMustLock(dir);
+
+	/* all post-process will be done in do_rmdir_whtmp(). */
+	sb = dir->i_sb;
+	args->dir = au_igrab(dir);
+	args->br = au_sbr(sb, bindex);
+	au_lcnt_inc(&args->br->br_count);
+	args->wh_dentry = dget(wh_dentry);
+	wkq_err = au_wkq_nowait(call_rmdir_whtmp, args, sb, /*flags*/0);
+	if (unlikely(wkq_err)) {
+		pr_warn("rmdir error %pd (%d), ignored\n", wh_dentry, wkq_err);
+		au_whtmp_rmdir_free(args);
+	}
+}
diff -Naur --no-dereference a/fs/aufs/whout.h b/fs/aufs/whout.h
--- a/fs/aufs/whout.h	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/whout.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,86 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * whiteout for logical deletion and opaque directory
+ */
+
+#ifndef __AUFS_WHOUT_H__
+#define __AUFS_WHOUT_H__
+
+#ifdef __KERNEL__
+
+#include "dir.h"
+
+/* whout.c */
+int au_wh_name_alloc(struct qstr *wh, const struct qstr *name);
+int au_wh_test(struct dentry *h_parent, struct qstr *wh_name, int try_sio);
+int au_diropq_test(struct dentry *h_dentry);
+struct au_branch;
+struct dentry *au_whtmp_lkup(struct dentry *h_parent, struct au_branch *br,
+			     struct qstr *prefix);
+int au_whtmp_ren(struct dentry *h_dentry, struct au_branch *br);
+int au_wh_unlink_dentry(struct inode *h_dir, struct path *h_path,
+			struct dentry *dentry);
+int au_wh_init(struct au_branch *br, struct super_block *sb);
+
+/* diropq flags */
+#define AuDiropq_CREATE	1
+#define au_ftest_diropq(flags, name)	((flags) & AuDiropq_##name)
+#define au_fset_diropq(flags, name) \
+	do { (flags) |= AuDiropq_##name; } while (0)
+#define au_fclr_diropq(flags, name) \
+	do { (flags) &= ~AuDiropq_##name; } while (0)
+
+struct dentry *au_diropq_sio(struct dentry *dentry, aufs_bindex_t bindex,
+			     unsigned int flags);
+struct dentry *au_wh_lkup(struct dentry *h_parent, struct qstr *base_name,
+			  struct au_branch *br);
+struct dentry *au_wh_create(struct dentry *dentry, aufs_bindex_t bindex,
+			    struct dentry *h_parent);
+
+/* real rmdir for the whiteout-ed dir */
+struct au_whtmp_rmdir {
+	struct inode *dir;
+	struct au_branch *br;
+	struct dentry *wh_dentry;
+	struct au_nhash whlist;
+};
+
+struct au_whtmp_rmdir *au_whtmp_rmdir_alloc(struct super_block *sb, gfp_t gfp);
+void au_whtmp_rmdir_free(struct au_whtmp_rmdir *whtmp);
+int au_whtmp_rmdir(struct inode *dir, aufs_bindex_t bindex,
+		   struct dentry *wh_dentry, struct au_nhash *whlist);
+void au_whtmp_kick_rmdir(struct inode *dir, aufs_bindex_t bindex,
+			 struct dentry *wh_dentry, struct au_whtmp_rmdir *args);
+
+/* ---------------------------------------------------------------------- */
+
+static inline struct dentry *au_diropq_create(struct dentry *dentry,
+					      aufs_bindex_t bindex)
+{
+	return au_diropq_sio(dentry, bindex, AuDiropq_CREATE);
+}
+
+static inline int au_diropq_remove(struct dentry *dentry, aufs_bindex_t bindex)
+{
+	return PTR_ERR(au_diropq_sio(dentry, bindex, !AuDiropq_CREATE));
+}
+
+#endif /* __KERNEL__ */
+#endif /* __AUFS_WHOUT_H__ */
diff -Naur --no-dereference a/fs/aufs/wkq.c b/fs/aufs/wkq.c
--- a/fs/aufs/wkq.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/wkq.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,372 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * workqueue for asynchronous/super-io operations
+ * todo: try new credential scheme
+ */
+
+#include <linux/module.h>
+#include "aufs.h"
+
+/* internal workqueue named AUFS_WKQ_NAME */
+
+static struct workqueue_struct *au_wkq;
+
+struct au_wkinfo {
+	struct work_struct wk;
+	struct kobject *kobj;
+
+	unsigned int flags; /* see wkq.h */
+
+	au_wkq_func_t func;
+	void *args;
+
+#ifdef CONFIG_LOCKDEP
+	int dont_check;
+	struct held_lock **hlock;
+#endif
+
+	struct completion *comp;
+};
+
+/* ---------------------------------------------------------------------- */
+/*
+ * Aufs passes some operations to the workqueue such as the internal copyup.
+ * This scheme looks rather unnatural for LOCKDEP debugging feature, since the
+ * job run by workqueue depends upon the locks acquired in the other task.
+ * Delegating a small operation to the workqueue, aufs passes its lockdep
+ * information too. And the job in the workqueue restores the info in order to
+ * pretend as if it acquired those locks. This is just to make LOCKDEP work
+ * correctly and expectedly.
+ */
+
+#ifndef CONFIG_LOCKDEP
+AuStubInt0(au_wkq_lockdep_alloc, struct au_wkinfo *wkinfo);
+AuStubVoid(au_wkq_lockdep_free, struct au_wkinfo *wkinfo);
+AuStubVoid(au_wkq_lockdep_pre, struct au_wkinfo *wkinfo);
+AuStubVoid(au_wkq_lockdep_post, struct au_wkinfo *wkinfo);
+AuStubVoid(au_wkq_lockdep_init, struct au_wkinfo *wkinfo);
+#else
+static void au_wkq_lockdep_init(struct au_wkinfo *wkinfo)
+{
+	wkinfo->hlock = NULL;
+	wkinfo->dont_check = 0;
+}
+
+/*
+ * 1: matched
+ * 0: unmatched
+ */
+static int au_wkq_lockdep_test(struct lock_class_key *key, const char *name)
+{
+	static DEFINE_SPINLOCK(spin);
+	static struct {
+		char *name;
+		struct lock_class_key *key;
+	} a[] = {
+		{ .name = "&sbinfo->si_rwsem" },
+		{ .name = "&finfo->fi_rwsem" },
+		{ .name = "&dinfo->di_rwsem" },
+		{ .name = "&iinfo->ii_rwsem" }
+	};
+	static int set;
+	int i;
+
+	/* lockless read from 'set.' see below */
+	if (set == ARRAY_SIZE(a)) {
+		for (i = 0; i < ARRAY_SIZE(a); i++)
+			if (a[i].key == key)
+				goto match;
+		goto unmatch;
+	}
+
+	spin_lock(&spin);
+	if (set)
+		for (i = 0; i < ARRAY_SIZE(a); i++)
+			if (a[i].key == key) {
+				spin_unlock(&spin);
+				goto match;
+			}
+	for (i = 0; i < ARRAY_SIZE(a); i++) {
+		if (a[i].key) {
+			if (unlikely(a[i].key == key)) { /* rare but possible */
+				spin_unlock(&spin);
+				goto match;
+			} else
+				continue;
+		}
+		if (strstr(a[i].name, name)) {
+			/*
+			 * the order of these three lines is important for the
+			 * lockless read above.
+			 */
+			a[i].key = key;
+			spin_unlock(&spin);
+			set++;
+			/* AuDbg("%d, %s\n", set, name); */
+			goto match;
+		}
+	}
+	spin_unlock(&spin);
+	goto unmatch;
+
+match:
+	return 1;
+unmatch:
+	return 0;
+}
+
+static int au_wkq_lockdep_alloc(struct au_wkinfo *wkinfo)
+{
+	int err, n;
+	struct task_struct *curr;
+	struct held_lock **hl, *held_locks, *p;
+
+	err = 0;
+	curr = current;
+	wkinfo->dont_check = lockdep_recursing(curr);
+	if (wkinfo->dont_check)
+		goto out;
+	n = curr->lockdep_depth;
+	if (!n)
+		goto out;
+
+	err = -ENOMEM;
+	wkinfo->hlock = kmalloc_array(n + 1, sizeof(*wkinfo->hlock), GFP_NOFS);
+	if (unlikely(!wkinfo->hlock))
+		goto out;
+
+	err = 0;
+#if 0 /* left for debugging */
+	if (0 && au_debug_test())
+		lockdep_print_held_locks(curr);
+#endif
+	held_locks = curr->held_locks;
+	hl = wkinfo->hlock;
+	while (n--) {
+		p = held_locks++;
+		if (au_wkq_lockdep_test(p->instance->key, p->instance->name))
+			*hl++ = p;
+	}
+	*hl = NULL;
+
+out:
+	return err;
+}
+
+static void au_wkq_lockdep_free(struct au_wkinfo *wkinfo)
+{
+	au_kfree_try_rcu(wkinfo->hlock);
+}
+
+static void au_wkq_lockdep_pre(struct au_wkinfo *wkinfo)
+{
+	struct held_lock *p, **hl = wkinfo->hlock;
+	int subclass;
+
+	if (wkinfo->dont_check)
+		lockdep_off();
+	if (!hl)
+		return;
+	while ((p = *hl++)) { /* assignment */
+		subclass = lockdep_hlock_class(p)->subclass;
+		/* AuDbg("%s, %d\n", p->instance->name, subclass); */
+		if (p->read)
+			rwsem_acquire_read(p->instance, subclass, 0,
+					   /*p->acquire_ip*/_RET_IP_);
+		else
+			rwsem_acquire(p->instance, subclass, 0,
+				      /*p->acquire_ip*/_RET_IP_);
+	}
+}
+
+static void au_wkq_lockdep_post(struct au_wkinfo *wkinfo)
+{
+	struct held_lock *p, **hl = wkinfo->hlock;
+
+	if (wkinfo->dont_check)
+		lockdep_on();
+	if (!hl)
+		return;
+	while ((p = *hl++)) /* assignment */
+		rwsem_release(p->instance, 0, /*p->acquire_ip*/_RET_IP_);
+}
+#endif
+
+static void wkq_func(struct work_struct *wk)
+{
+	struct au_wkinfo *wkinfo = container_of(wk, struct au_wkinfo, wk);
+
+	AuDebugOn(!uid_eq(current_fsuid(), GLOBAL_ROOT_UID));
+	AuDebugOn(rlimit(RLIMIT_FSIZE) != RLIM_INFINITY);
+
+	au_wkq_lockdep_pre(wkinfo);
+	wkinfo->func(wkinfo->args);
+	au_wkq_lockdep_post(wkinfo);
+	if (au_ftest_wkq(wkinfo->flags, WAIT))
+		complete(wkinfo->comp);
+	else {
+		kobject_put(wkinfo->kobj);
+		module_put(THIS_MODULE); /* todo: ?? */
+		au_kfree_rcu(wkinfo);
+	}
+}
+
+/*
+ * Since struct completion is large, try allocating it dynamically.
+ */
+#define AuWkqCompDeclare(name)	struct completion *comp = NULL
+
+static int au_wkq_comp_alloc(struct au_wkinfo *wkinfo, struct completion **comp)
+{
+	*comp = kmalloc(sizeof(**comp), GFP_NOFS);
+	if (*comp) {
+		init_completion(*comp);
+		wkinfo->comp = *comp;
+		return 0;
+	}
+	return -ENOMEM;
+}
+
+static void au_wkq_comp_free(struct completion *comp)
+{
+	au_kfree_rcu(comp);
+}
+
+static void au_wkq_run(struct au_wkinfo *wkinfo)
+{
+	if (au_ftest_wkq(wkinfo->flags, NEST)) {
+		if (au_wkq_test()) {
+			AuWarn1("wkq from wkq, unless silly-rename on NFS,"
+				" due to a dead dir by UDBA,"
+				" or async xino write?\n");
+			AuDebugOn(au_ftest_wkq(wkinfo->flags, WAIT));
+		}
+	} else
+		au_dbg_verify_kthread();
+
+	if (au_ftest_wkq(wkinfo->flags, WAIT)) {
+		INIT_WORK_ONSTACK(&wkinfo->wk, wkq_func);
+		queue_work(au_wkq, &wkinfo->wk);
+	} else {
+		INIT_WORK(&wkinfo->wk, wkq_func);
+		schedule_work(&wkinfo->wk);
+	}
+}
+
+/*
+ * Be careful. It is easy to make deadlock happen.
+ * processA: lock, wkq and wait
+ * processB: wkq and wait, lock in wkq
+ * --> deadlock
+ */
+int au_wkq_do_wait(unsigned int flags, au_wkq_func_t func, void *args)
+{
+	int err;
+	AuWkqCompDeclare(comp);
+	struct au_wkinfo wkinfo = {
+		.flags	= flags,
+		.func	= func,
+		.args	= args
+	};
+
+	err = au_wkq_comp_alloc(&wkinfo, &comp);
+	if (unlikely(err))
+		goto out;
+	err = au_wkq_lockdep_alloc(&wkinfo);
+	if (unlikely(err))
+		goto out_comp;
+	if (!err) {
+		au_wkq_run(&wkinfo);
+		/* no timeout, no interrupt */
+		wait_for_completion(wkinfo.comp);
+	}
+	au_wkq_lockdep_free(&wkinfo);
+
+out_comp:
+	au_wkq_comp_free(comp);
+out:
+	destroy_work_on_stack(&wkinfo.wk);
+	return err;
+}
+
+/*
+ * Note: dget/dput() in func for aufs dentries are not supported. It will be a
+ * problem in a concurrent umounting.
+ */
+int au_wkq_nowait(au_wkq_func_t func, void *args, struct super_block *sb,
+		  unsigned int flags)
+{
+	int err;
+	struct au_wkinfo *wkinfo;
+
+	atomic_inc(&au_sbi(sb)->si_nowait.nw_len);
+
+	/*
+	 * wkq_func() must free this wkinfo.
+	 * it highly depends upon the implementation of workqueue.
+	 */
+	err = 0;
+	wkinfo = kmalloc(sizeof(*wkinfo), GFP_NOFS);
+	if (wkinfo) {
+		wkinfo->kobj = &au_sbi(sb)->si_kobj;
+		wkinfo->flags = flags & ~AuWkq_WAIT;
+		wkinfo->func = func;
+		wkinfo->args = args;
+		wkinfo->comp = NULL;
+		au_wkq_lockdep_init(wkinfo);
+		kobject_get(wkinfo->kobj);
+		__module_get(THIS_MODULE); /* todo: ?? */
+
+		au_wkq_run(wkinfo);
+	} else {
+		err = -ENOMEM;
+		au_nwt_done(&au_sbi(sb)->si_nowait);
+	}
+
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+void au_nwt_init(struct au_nowait_tasks *nwt)
+{
+	atomic_set(&nwt->nw_len, 0);
+	/* smp_mb(); */ /* atomic_set */
+	init_waitqueue_head(&nwt->nw_wq);
+}
+
+void au_wkq_fin(void)
+{
+	destroy_workqueue(au_wkq);
+}
+
+int __init au_wkq_init(void)
+{
+	int err;
+
+	err = 0;
+	au_wkq = alloc_workqueue(AUFS_WKQ_NAME, 0, WQ_DFL_ACTIVE);
+	if (IS_ERR(au_wkq))
+		err = PTR_ERR(au_wkq);
+	else if (!au_wkq)
+		err = -ENOMEM;
+
+	return err;
+}
diff -Naur --no-dereference a/fs/aufs/wkq.h b/fs/aufs/wkq.h
--- a/fs/aufs/wkq.h	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/wkq.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,89 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * workqueue for asynchronous/super-io operations
+ * todo: try new credentials management scheme
+ */
+
+#ifndef __AUFS_WKQ_H__
+#define __AUFS_WKQ_H__
+
+#ifdef __KERNEL__
+
+#include <linux/wait.h>
+
+struct super_block;
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * in the next operation, wait for the 'nowait' tasks in system-wide workqueue
+ */
+struct au_nowait_tasks {
+	atomic_t		nw_len;
+	wait_queue_head_t	nw_wq;
+};
+
+/* ---------------------------------------------------------------------- */
+
+typedef void (*au_wkq_func_t)(void *args);
+
+/* wkq flags */
+#define AuWkq_WAIT	1
+#define AuWkq_NEST	(1 << 1)
+#define au_ftest_wkq(flags, name)	((flags) & AuWkq_##name)
+#define au_fset_wkq(flags, name) \
+	do { (flags) |= AuWkq_##name; } while (0)
+#define au_fclr_wkq(flags, name) \
+	do { (flags) &= ~AuWkq_##name; } while (0)
+
+/* wkq.c */
+int au_wkq_do_wait(unsigned int flags, au_wkq_func_t func, void *args);
+int au_wkq_nowait(au_wkq_func_t func, void *args, struct super_block *sb,
+		  unsigned int flags);
+void au_nwt_init(struct au_nowait_tasks *nwt);
+int __init au_wkq_init(void);
+void au_wkq_fin(void);
+
+/* ---------------------------------------------------------------------- */
+
+static inline int au_wkq_test(void)
+{
+	return current->flags & PF_WQ_WORKER;
+}
+
+static inline int au_wkq_wait(au_wkq_func_t func, void *args)
+{
+	return au_wkq_do_wait(AuWkq_WAIT, func, args);
+}
+
+static inline void au_nwt_done(struct au_nowait_tasks *nwt)
+{
+	if (atomic_dec_and_test(&nwt->nw_len))
+		wake_up_all(&nwt->nw_wq);
+}
+
+static inline int au_nwt_flush(struct au_nowait_tasks *nwt)
+{
+	wait_event(nwt->nw_wq, !atomic_read(&nwt->nw_len));
+	return 0;
+}
+
+#endif /* __KERNEL__ */
+#endif /* __AUFS_WKQ_H__ */
diff -Naur --no-dereference a/fs/aufs/xattr.c b/fs/aufs/xattr.c
--- a/fs/aufs/xattr.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/xattr.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,356 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2014-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * handling xattr functions
+ */
+
+#include <linux/fs.h>
+#include <linux/posix_acl_xattr.h>
+#include <linux/xattr.h>
+#include "aufs.h"
+
+static int au_xattr_ignore(int err, char *name, unsigned int ignore_flags)
+{
+	if (!ignore_flags)
+		goto out;
+	switch (err) {
+	case -ENOMEM:
+	case -EDQUOT:
+		goto out;
+	}
+
+	if ((ignore_flags & AuBrAttr_ICEX) == AuBrAttr_ICEX) {
+		err = 0;
+		goto out;
+	}
+
+#define cmp(brattr, prefix) do {					\
+		if (!strncmp(name, XATTR_##prefix##_PREFIX,		\
+			     XATTR_##prefix##_PREFIX_LEN)) {		\
+			if (ignore_flags & AuBrAttr_ICEX_##brattr)	\
+				err = 0;				\
+			goto out;					\
+		}							\
+	} while (0)
+
+	cmp(SEC, SECURITY);
+	cmp(SYS, SYSTEM);
+	cmp(TR, TRUSTED);
+	cmp(USR, USER);
+#undef cmp
+
+	if (ignore_flags & AuBrAttr_ICEX_OTH)
+		err = 0;
+
+out:
+	return err;
+}
+
+static const int au_xattr_out_of_list = AuBrAttr_ICEX_OTH << 1;
+
+static int au_do_cpup_xattr(struct dentry *h_dst, struct dentry *h_src,
+			    char *name, char **buf, unsigned int ignore_flags,
+			    unsigned int verbose)
+{
+	int err;
+	ssize_t ssz;
+	struct inode *h_idst;
+
+	ssz = vfs_getxattr_alloc(h_src, name, buf, 0, GFP_NOFS);
+	err = ssz;
+	if (unlikely(err <= 0)) {
+		if (err == -ENODATA
+		    || (err == -EOPNOTSUPP
+			&& ((ignore_flags & au_xattr_out_of_list)
+			    || (au_test_nfs_noacl(d_inode(h_src))
+				&& (!strcmp(name, XATTR_NAME_POSIX_ACL_ACCESS)
+				    || !strcmp(name,
+					       XATTR_NAME_POSIX_ACL_DEFAULT))))
+			    ))
+			err = 0;
+		if (err && (verbose || au_debug_test()))
+			pr_err("%s, err %d\n", name, err);
+		goto out;
+	}
+
+	/* unlock it temporary */
+	h_idst = d_inode(h_dst);
+	inode_unlock(h_idst);
+	err = vfsub_setxattr(h_dst, name, *buf, ssz, /*flags*/0);
+	inode_lock_nested(h_idst, AuLsc_I_CHILD2);
+	if (unlikely(err)) {
+		if (verbose || au_debug_test())
+			pr_err("%s, err %d\n", name, err);
+		err = au_xattr_ignore(err, name, ignore_flags);
+	}
+
+out:
+	return err;
+}
+
+int au_cpup_xattr(struct dentry *h_dst, struct dentry *h_src, int ignore_flags,
+		  unsigned int verbose)
+{
+	int err, unlocked, acl_access, acl_default;
+	ssize_t ssz;
+	struct inode *h_isrc, *h_idst;
+	char *value, *p, *o, *e;
+
+	/* try stopping to update the source inode while we are referencing */
+	/* there should not be the parent-child relationship between them */
+	h_isrc = d_inode(h_src);
+	h_idst = d_inode(h_dst);
+	inode_unlock(h_idst);
+	inode_lock_shared_nested(h_isrc, AuLsc_I_CHILD);
+	inode_lock_nested(h_idst, AuLsc_I_CHILD2);
+	unlocked = 0;
+
+	/* some filesystems don't list POSIX ACL, for example tmpfs */
+	ssz = vfs_listxattr(h_src, NULL, 0);
+	err = ssz;
+	if (unlikely(err < 0)) {
+		AuTraceErr(err);
+		if (err == -ENODATA
+		    || err == -EOPNOTSUPP)
+			err = 0;	/* ignore */
+		goto out;
+	}
+
+	err = 0;
+	p = NULL;
+	o = NULL;
+	if (ssz) {
+		err = -ENOMEM;
+		p = kmalloc(ssz, GFP_NOFS);
+		o = p;
+		if (unlikely(!p))
+			goto out;
+		err = vfs_listxattr(h_src, p, ssz);
+	}
+	inode_unlock_shared(h_isrc);
+	unlocked = 1;
+	AuDbg("err %d, ssz %zd\n", err, ssz);
+	if (unlikely(err < 0))
+		goto out_free;
+
+	err = 0;
+	e = p + ssz;
+	value = NULL;
+	acl_access = 0;
+	acl_default = 0;
+	while (!err && p < e) {
+		acl_access |= !strncmp(p, XATTR_NAME_POSIX_ACL_ACCESS,
+				       sizeof(XATTR_NAME_POSIX_ACL_ACCESS) - 1);
+		acl_default |= !strncmp(p, XATTR_NAME_POSIX_ACL_DEFAULT,
+					sizeof(XATTR_NAME_POSIX_ACL_DEFAULT)
+					- 1);
+		err = au_do_cpup_xattr(h_dst, h_src, p, &value, ignore_flags,
+				       verbose);
+		p += strlen(p) + 1;
+	}
+	AuTraceErr(err);
+	ignore_flags |= au_xattr_out_of_list;
+	if (!err && !acl_access) {
+		err = au_do_cpup_xattr(h_dst, h_src,
+				       XATTR_NAME_POSIX_ACL_ACCESS, &value,
+				       ignore_flags, verbose);
+		AuTraceErr(err);
+	}
+	if (!err && !acl_default) {
+		err = au_do_cpup_xattr(h_dst, h_src,
+				       XATTR_NAME_POSIX_ACL_DEFAULT, &value,
+				       ignore_flags, verbose);
+		AuTraceErr(err);
+	}
+
+	au_kfree_try_rcu(value);
+
+out_free:
+	au_kfree_try_rcu(o);
+out:
+	if (!unlocked)
+		inode_unlock_shared(h_isrc);
+	AuTraceErr(err);
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+static int au_smack_reentering(struct super_block *sb)
+{
+#if IS_ENABLED(CONFIG_SECURITY_SMACK) || IS_ENABLED(CONFIG_SECURITY_SELINUX)
+	/*
+	 * as a part of lookup, smack_d_instantiate() is called, and it calls
+	 * i_op->getxattr(). ouch.
+	 */
+	return si_pid_test(sb);
+#else
+	return 0;
+#endif
+}
+
+enum {
+	AU_XATTR_LIST,
+	AU_XATTR_GET
+};
+
+struct au_lgxattr {
+	int type;
+	union {
+		struct {
+			char	*list;
+			size_t	size;
+		} list;
+		struct {
+			const char	*name;
+			void		*value;
+			size_t		size;
+		} get;
+	} u;
+};
+
+static ssize_t au_lgxattr(struct dentry *dentry, struct inode *inode,
+			  struct au_lgxattr *arg)
+{
+	ssize_t err;
+	int reenter;
+	struct path h_path;
+	struct super_block *sb;
+
+	sb = dentry->d_sb;
+	reenter = au_smack_reentering(sb);
+	if (!reenter) {
+		err = si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLM);
+		if (unlikely(err))
+			goto out;
+	}
+	err = au_h_path_getattr(dentry, inode, /*force*/1, &h_path, reenter);
+	if (unlikely(err))
+		goto out_si;
+	if (unlikely(!h_path.dentry))
+		/* illegally overlapped or something */
+		goto out_di; /* pretending success */
+
+	/* always topmost entry only */
+	switch (arg->type) {
+	case AU_XATTR_LIST:
+		err = vfs_listxattr(h_path.dentry,
+				    arg->u.list.list, arg->u.list.size);
+		break;
+	case AU_XATTR_GET:
+		AuDebugOn(d_is_negative(h_path.dentry));
+		err = vfs_getxattr(h_path.dentry,
+				   arg->u.get.name, arg->u.get.value,
+				   arg->u.get.size);
+		break;
+	}
+
+out_di:
+	if (!reenter)
+		di_read_unlock(dentry, AuLock_IR);
+out_si:
+	if (!reenter)
+		si_read_unlock(sb);
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+ssize_t aufs_listxattr(struct dentry *dentry, char *list, size_t size)
+{
+	struct au_lgxattr arg = {
+		.type = AU_XATTR_LIST,
+		.u.list = {
+			.list	= list,
+			.size	= size
+		},
+	};
+
+	return au_lgxattr(dentry, /*inode*/NULL, &arg);
+}
+
+static ssize_t au_getxattr(struct dentry *dentry, struct inode *inode,
+			   const char *name, void *value, size_t size)
+{
+	struct au_lgxattr arg = {
+		.type = AU_XATTR_GET,
+		.u.get = {
+			.name	= name,
+			.value	= value,
+			.size	= size
+		},
+	};
+
+	return au_lgxattr(dentry, inode, &arg);
+}
+
+static int au_setxattr(struct dentry *dentry, struct inode *inode,
+		       const char *name, const void *value, size_t size,
+		       int flags)
+{
+	struct au_sxattr arg = {
+		.type = AU_XATTR_SET,
+		.u.set = {
+			.name	= name,
+			.value	= value,
+			.size	= size,
+			.flags	= flags
+		},
+	};
+
+	return au_sxattr(dentry, inode, &arg);
+}
+
+/* ---------------------------------------------------------------------- */
+
+static int au_xattr_get(const struct xattr_handler *handler,
+			struct dentry *dentry, struct inode *inode,
+			const char *name, void *buffer, size_t size)
+{
+	return au_getxattr(dentry, inode, name, buffer, size);
+}
+
+static int au_xattr_set(const struct xattr_handler *handler,
+			struct dentry *dentry, struct inode *inode,
+			const char *name, const void *value, size_t size,
+			int flags)
+{
+	return au_setxattr(dentry, inode, name, value, size, flags);
+}
+
+static const struct xattr_handler au_xattr_handler = {
+	.name	= "",
+	.prefix	= "",
+	.get	= au_xattr_get,
+	.set	= au_xattr_set
+};
+
+static const struct xattr_handler *au_xattr_handlers[] = {
+#ifdef CONFIG_FS_POSIX_ACL
+	&posix_acl_access_xattr_handler,
+	&posix_acl_default_xattr_handler,
+#endif
+	&au_xattr_handler, /* must be last */
+	NULL
+};
+
+void au_xattr_init(struct super_block *sb)
+{
+	sb->s_xattr = au_xattr_handlers;
+}
diff -Naur --no-dereference a/fs/aufs/xino.c b/fs/aufs/xino.c
--- a/fs/aufs/xino.c	1969-12-31 19:00:00.000000000 -0500
+++ b/fs/aufs/xino.c	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,1966 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * external inode number translation table and bitmap
+ *
+ * things to consider
+ * - the lifetime
+ *   + au_xino object
+ *   + XINO files (xino, xib, xigen)
+ *   + dynamic debugfs entries (xiN)
+ *   + static debugfs entries (xib, xigen)
+ *   + static sysfs entry (xi_path)
+ * - several entry points to handle them.
+ *   + mount(2) without xino option (default)
+ *   + mount(2) with xino option
+ *   + mount(2) with noxino option
+ *   + umount(2)
+ *   + remount with add/del branches
+ *   + remount with xino/noxino options
+ */
+
+#include <linux/seq_file.h>
+#include <linux/statfs.h>
+#include "aufs.h"
+
+static aufs_bindex_t sbr_find_shared(struct super_block *sb, aufs_bindex_t btop,
+				     aufs_bindex_t bbot,
+				     struct super_block *h_sb)
+{
+	/* todo: try binary-search if the branches are many */
+	for (; btop <= bbot; btop++)
+		if (h_sb == au_sbr_sb(sb, btop))
+			return btop;
+	return -1;
+}
+
+/*
+ * find another branch who is on the same filesystem of the specified
+ * branch{@btgt}. search until @bbot.
+ */
+static aufs_bindex_t is_sb_shared(struct super_block *sb, aufs_bindex_t btgt,
+				  aufs_bindex_t bbot)
+{
+	aufs_bindex_t bindex;
+	struct super_block *tgt_sb;
+
+	tgt_sb = au_sbr_sb(sb, btgt);
+	bindex = sbr_find_shared(sb, /*btop*/0, btgt - 1, tgt_sb);
+	if (bindex < 0)
+		bindex = sbr_find_shared(sb, btgt + 1, bbot, tgt_sb);
+
+	return bindex;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * stop unnecessary notify events at creating xino files
+ */
+
+aufs_bindex_t au_xi_root(struct super_block *sb, struct dentry *dentry)
+{
+	aufs_bindex_t bfound, bindex, bbot;
+	struct dentry *parent;
+	struct au_branch *br;
+
+	bfound = -1;
+	parent = dentry->d_parent; /* safe d_parent access */
+	bbot = au_sbbot(sb);
+	for (bindex = 0; bindex <= bbot; bindex++) {
+		br = au_sbr(sb, bindex);
+		if (au_br_dentry(br) == parent) {
+			bfound = bindex;
+			break;
+		}
+	}
+
+	AuDbg("bfound b%d\n", bfound);
+	return bfound;
+}
+
+struct au_xino_lock_dir {
+	struct au_hinode *hdir;
+	struct dentry *parent;
+	struct inode *dir;
+};
+
+static struct dentry *au_dget_parent_lock(struct dentry *dentry,
+					  unsigned int lsc)
+{
+	struct dentry *parent;
+	struct inode *dir;
+
+	parent = dget_parent(dentry);
+	dir = d_inode(parent);
+	inode_lock_nested(dir, lsc);
+#if 0 /* it should not happen */
+	spin_lock(&dentry->d_lock);
+	if (unlikely(dentry->d_parent != parent)) {
+		spin_unlock(&dentry->d_lock);
+		inode_unlock(dir);
+		dput(parent);
+		parent = NULL;
+		goto out;
+	}
+	spin_unlock(&dentry->d_lock);
+
+out:
+#endif
+	return parent;
+}
+
+static void au_xino_lock_dir(struct super_block *sb, struct path *xipath,
+			     struct au_xino_lock_dir *ldir)
+{
+	aufs_bindex_t bindex;
+
+	ldir->hdir = NULL;
+	bindex = au_xi_root(sb, xipath->dentry);
+	if (bindex >= 0) {
+		/* rw branch root */
+		ldir->hdir = au_hi(d_inode(sb->s_root), bindex);
+		au_hn_inode_lock_nested(ldir->hdir, AuLsc_I_PARENT);
+	} else {
+		/* other */
+		ldir->parent = au_dget_parent_lock(xipath->dentry,
+						   AuLsc_I_PARENT);
+		ldir->dir = d_inode(ldir->parent);
+	}
+}
+
+static void au_xino_unlock_dir(struct au_xino_lock_dir *ldir)
+{
+	if (ldir->hdir)
+		au_hn_inode_unlock(ldir->hdir);
+	else {
+		inode_unlock(ldir->dir);
+		dput(ldir->parent);
+	}
+}
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * create and set a new xino file
+ */
+struct file *au_xino_create(struct super_block *sb, char *fpath, int silent,
+			    int wbrtop)
+{
+	struct file *file;
+	struct dentry *h_parent, *d;
+	struct inode *h_dir, *inode;
+	int err;
+	static DEFINE_MUTEX(mtx);
+
+	/*
+	 * at mount-time, and the xino file is the default path,
+	 * hnotify is disabled so we have no notify events to ignore.
+	 * when a user specified the xino, we cannot get au_hdir to be ignored.
+	 */
+	if (!wbrtop)
+		mutex_lock(&mtx);
+	file = vfsub_filp_open(fpath, O_RDWR | O_CREAT | O_EXCL | O_LARGEFILE
+			       /* | __FMODE_NONOTIFY */,
+			       0666);
+	if (IS_ERR(file)) {
+		if (!wbrtop)
+			mutex_unlock(&mtx);
+		if (!silent)
+			pr_err("open %s(%ld)\n", fpath, PTR_ERR(file));
+		return file;
+	}
+
+	/* keep file count */
+	err = 0;
+	d = file->f_path.dentry;
+	h_parent = au_dget_parent_lock(d, AuLsc_I_PARENT);
+	if (!wbrtop)
+		mutex_unlock(&mtx);
+	/* mnt_want_write() is unnecessary here */
+	h_dir = d_inode(h_parent);
+	inode = file_inode(file);
+	/* no delegation since it is just created */
+	if (inode->i_nlink)
+		err = vfsub_unlink(h_dir, &file->f_path, /*delegated*/NULL,
+				   /*force*/0);
+	inode_unlock(h_dir);
+	dput(h_parent);
+	if (unlikely(err)) {
+		if (!silent)
+			pr_err("unlink %s(%d)\n", fpath, err);
+		goto out;
+	}
+
+	err = -EINVAL;
+	if (unlikely(sb == d->d_sb)) {
+		if (!silent)
+			pr_err("%s must be outside\n", fpath);
+		goto out;
+	}
+	if (unlikely(au_test_fs_bad_xino(d->d_sb))) {
+		if (!silent)
+			pr_err("xino doesn't support %s(%s)\n",
+			       fpath, au_sbtype(d->d_sb));
+		goto out;
+	}
+	return file; /* success */
+
+out:
+	fput(file);
+	file = ERR_PTR(err);
+	return file;
+}
+
+/*
+ * create a new xinofile at the same place/path as @base.
+ */
+struct file *au_xino_create2(struct super_block *sb, struct path *base,
+			     struct file *copy_src)
+{
+	struct file *file;
+	struct dentry *dentry, *parent;
+	struct inode *dir, *delegated;
+	struct qstr *name;
+	struct path path;
+	int err, do_unlock;
+	struct au_xino_lock_dir ldir;
+
+	do_unlock = 1;
+	au_xino_lock_dir(sb, base, &ldir);
+	dentry = base->dentry;
+	parent = dentry->d_parent; /* dir inode is locked */
+	dir = d_inode(parent);
+	IMustLock(dir);
+
+	name = &dentry->d_name;
+	path.dentry = vfsub_lookup_one_len(name->name, parent, name->len);
+	if (IS_ERR(path.dentry)) {
+		file = (void *)path.dentry;
+		pr_err("%pd lookup err %ld\n", dentry, PTR_ERR(path.dentry));
+		goto out;
+	}
+
+	/* no need to mnt_want_write() since we call dentry_open() later */
+	err = vfs_create(dir, path.dentry, 0666, NULL);
+	if (unlikely(err)) {
+		file = ERR_PTR(err);
+		pr_err("%pd create err %d\n", dentry, err);
+		goto out_dput;
+	}
+
+	path.mnt = base->mnt;
+	file = vfsub_dentry_open(&path,
+				 O_RDWR | O_CREAT | O_EXCL | O_LARGEFILE
+				 /* | __FMODE_NONOTIFY */);
+	if (IS_ERR(file)) {
+		pr_err("%pd open err %ld\n", dentry, PTR_ERR(file));
+		goto out_dput;
+	}
+
+	delegated = NULL;
+	err = vfsub_unlink(dir, &file->f_path, &delegated, /*force*/0);
+	au_xino_unlock_dir(&ldir);
+	do_unlock = 0;
+	if (unlikely(err == -EWOULDBLOCK)) {
+		pr_warn("cannot retry for NFSv4 delegation"
+			" for an internal unlink\n");
+		iput(delegated);
+	}
+	if (unlikely(err)) {
+		pr_err("%pd unlink err %d\n", dentry, err);
+		goto out_fput;
+	}
+
+	if (copy_src) {
+		/* no one can touch copy_src xino */
+		err = au_copy_file(file, copy_src, vfsub_f_size_read(copy_src));
+		if (unlikely(err)) {
+			pr_err("%pd copy err %d\n", dentry, err);
+			goto out_fput;
+		}
+	}
+	goto out_dput; /* success */
+
+out_fput:
+	fput(file);
+	file = ERR_PTR(err);
+out_dput:
+	dput(path.dentry);
+out:
+	if (do_unlock)
+		au_xino_unlock_dir(&ldir);
+	return file;
+}
+
+struct file *au_xino_file1(struct au_xino *xi)
+{
+	struct file *file;
+	unsigned int u, nfile;
+
+	file = NULL;
+	nfile = xi->xi_nfile;
+	for (u = 0; u < nfile; u++) {
+		file = xi->xi_file[u];
+		if (file)
+			break;
+	}
+
+	return file;
+}
+
+static int au_xino_file_set(struct au_xino *xi, int idx, struct file *file)
+{
+	int err;
+	struct file *f;
+	void *p;
+
+	if (file)
+		get_file(file);
+
+	err = 0;
+	f = NULL;
+	if (idx < xi->xi_nfile) {
+		f = xi->xi_file[idx];
+		if (f)
+			fput(f);
+	} else {
+		p = au_kzrealloc(xi->xi_file,
+				 sizeof(*xi->xi_file) * xi->xi_nfile,
+				 sizeof(*xi->xi_file) * (idx + 1),
+				 GFP_NOFS, /*may_shrink*/0);
+		if (p) {
+			MtxMustLock(&xi->xi_mtx);
+			xi->xi_file = p;
+			xi->xi_nfile = idx + 1;
+		} else {
+			err = -ENOMEM;
+			if (file)
+				fput(file);
+			goto out;
+		}
+	}
+	xi->xi_file[idx] = file;
+
+out:
+	return err;
+}
+
+/*
+ * if @xinew->xi is not set, then create new xigen file.
+ */
+struct file *au_xi_new(struct super_block *sb, struct au_xi_new *xinew)
+{
+	struct file *file;
+	int err;
+
+	SiMustAnyLock(sb);
+
+	file = au_xino_create2(sb, xinew->base, xinew->copy_src);
+	if (IS_ERR(file)) {
+		err = PTR_ERR(file);
+		pr_err("%s[%d], err %d\n",
+		       xinew->xi ? "xino" : "xigen",
+		       xinew->idx, err);
+		goto out;
+	}
+
+	if (xinew->xi)
+		err = au_xino_file_set(xinew->xi, xinew->idx, file);
+	else {
+		BUG();
+		/* todo: make xigen file an array */
+		/* err = au_xigen_file_set(sb, xinew->idx, file); */
+	}
+	fput(file);
+	if (unlikely(err))
+		file = ERR_PTR(err);
+
+out:
+	return file;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * truncate xino files
+ */
+static int au_xino_do_trunc(struct super_block *sb, aufs_bindex_t bindex,
+			    int idx, struct kstatfs *st)
+{
+	int err;
+	blkcnt_t blocks;
+	struct file *file, *new_xino;
+	struct au_xi_new xinew = {
+		.idx = idx
+	};
+
+	err = 0;
+	xinew.xi = au_sbr(sb, bindex)->br_xino;
+	file = au_xino_file(xinew.xi, idx);
+	if (!file)
+		goto out;
+
+	xinew.base = &file->f_path;
+	err = vfs_statfs(xinew.base, st);
+	if (unlikely(err)) {
+		AuErr1("statfs err %d, ignored\n", err);
+		err = 0;
+		goto out;
+	}
+
+	blocks = file_inode(file)->i_blocks;
+	pr_info("begin truncating xino(b%d-%d), ib%llu, %llu/%llu free blks\n",
+		bindex, idx, (u64)blocks, st->f_bfree, st->f_blocks);
+
+	xinew.copy_src = file;
+	new_xino = au_xi_new(sb, &xinew);
+	if (IS_ERR(new_xino)) {
+		err = PTR_ERR(new_xino);
+		pr_err("xino(b%d-%d), err %d, ignored\n", bindex, idx, err);
+		goto out;
+	}
+
+	err = vfs_statfs(&new_xino->f_path, st);
+	if (!err)
+		pr_info("end truncating xino(b%d-%d), ib%llu, %llu/%llu free blks\n",
+			bindex, idx, (u64)file_inode(new_xino)->i_blocks,
+			st->f_bfree, st->f_blocks);
+	else {
+		AuErr1("statfs err %d, ignored\n", err);
+		err = 0;
+	}
+
+out:
+	return err;
+}
+
+int au_xino_trunc(struct super_block *sb, aufs_bindex_t bindex, int idx_begin)
+{
+	int err, i;
+	unsigned long jiffy;
+	aufs_bindex_t bbot;
+	struct kstatfs *st;
+	struct au_branch *br;
+	struct au_xino *xi;
+
+	err = -ENOMEM;
+	st = kmalloc(sizeof(*st), GFP_NOFS);
+	if (unlikely(!st))
+		goto out;
+
+	err = -EINVAL;
+	bbot = au_sbbot(sb);
+	if (unlikely(bindex < 0 || bbot < bindex))
+		goto out_st;
+
+	err = 0;
+	jiffy = jiffies;
+	br = au_sbr(sb, bindex);
+	xi = br->br_xino;
+	for (i = idx_begin; !err && i < xi->xi_nfile; i++)
+		err = au_xino_do_trunc(sb, bindex, i, st);
+	if (!err)
+		au_sbi(sb)->si_xino_jiffy = jiffy;
+
+out_st:
+	au_kfree_rcu(st);
+out:
+	return err;
+}
+
+struct xino_do_trunc_args {
+	struct super_block *sb;
+	struct au_branch *br;
+	int idx;
+};
+
+static void xino_do_trunc(void *_args)
+{
+	struct xino_do_trunc_args *args = _args;
+	struct super_block *sb;
+	struct au_branch *br;
+	struct inode *dir;
+	int err, idx;
+	aufs_bindex_t bindex;
+
+	err = 0;
+	sb = args->sb;
+	dir = d_inode(sb->s_root);
+	br = args->br;
+	idx = args->idx;
+
+	si_noflush_write_lock(sb);
+	ii_read_lock_parent(dir);
+	bindex = au_br_index(sb, br->br_id);
+	err = au_xino_trunc(sb, bindex, idx);
+	ii_read_unlock(dir);
+	if (unlikely(err))
+		pr_warn("err b%d, (%d)\n", bindex, err);
+	atomic_dec(&br->br_xino->xi_truncating);
+	au_lcnt_dec(&br->br_count);
+	si_write_unlock(sb);
+	au_nwt_done(&au_sbi(sb)->si_nowait);
+	au_kfree_rcu(args);
+}
+
+/*
+ * returns the index in the xi_file array whose corresponding file is necessary
+ * to truncate, or -1 which means no need to truncate.
+ */
+static int xino_trunc_test(struct super_block *sb, struct au_branch *br)
+{
+	int err;
+	unsigned int u;
+	struct kstatfs st;
+	struct au_sbinfo *sbinfo;
+	struct au_xino *xi;
+	struct file *file;
+
+	/* todo: si_xino_expire and the ratio should be customizable */
+	sbinfo = au_sbi(sb);
+	if (time_before(jiffies,
+			sbinfo->si_xino_jiffy + sbinfo->si_xino_expire))
+		return -1;
+
+	/* truncation border */
+	xi = br->br_xino;
+	for (u = 0; u < xi->xi_nfile; u++) {
+		file = au_xino_file(xi, u);
+		if (!file)
+			continue;
+
+		err = vfs_statfs(&file->f_path, &st);
+		if (unlikely(err)) {
+			AuErr1("statfs err %d, ignored\n", err);
+			return -1;
+		}
+		if (div64_u64(st.f_bfree * 100, st.f_blocks)
+		    >= AUFS_XINO_DEF_TRUNC)
+			return u;
+	}
+
+	return -1;
+}
+
+static void xino_try_trunc(struct super_block *sb, struct au_branch *br)
+{
+	int idx;
+	struct xino_do_trunc_args *args;
+	int wkq_err;
+
+	idx = xino_trunc_test(sb, br);
+	if (idx < 0)
+		return;
+
+	if (atomic_inc_return(&br->br_xino->xi_truncating) > 1)
+		goto out;
+
+	/* lock and kfree() will be called in trunc_xino() */
+	args = kmalloc(sizeof(*args), GFP_NOFS);
+	if (unlikely(!args)) {
+		AuErr1("no memory\n");
+		goto out;
+	}
+
+	au_lcnt_inc(&br->br_count);
+	args->sb = sb;
+	args->br = br;
+	args->idx = idx;
+	wkq_err = au_wkq_nowait(xino_do_trunc, args, sb, /*flags*/0);
+	if (!wkq_err)
+		return; /* success */
+
+	pr_err("wkq %d\n", wkq_err);
+	au_lcnt_dec(&br->br_count);
+	au_kfree_rcu(args);
+
+out:
+	atomic_dec(&br->br_xino->xi_truncating);
+}
+
+/* ---------------------------------------------------------------------- */
+
+struct au_xi_calc {
+	int idx;
+	loff_t pos;
+};
+
+static void au_xi_calc(struct super_block *sb, ino_t h_ino,
+		       struct au_xi_calc *calc)
+{
+	loff_t maxent;
+
+	maxent = au_xi_maxent(sb);
+	calc->idx = div64_u64_rem(h_ino, maxent, &calc->pos);
+	calc->pos *= sizeof(ino_t);
+}
+
+static int au_xino_do_new_async(struct super_block *sb, struct au_branch *br,
+				struct au_xi_calc *calc)
+{
+	int err;
+	struct file *file;
+	struct au_xino *xi = br->br_xino;
+	struct au_xi_new xinew = {
+		.xi = xi
+	};
+
+	SiMustAnyLock(sb);
+
+	err = 0;
+	if (!xi)
+		goto out;
+
+	mutex_lock(&xi->xi_mtx);
+	file = au_xino_file(xi, calc->idx);
+	if (file)
+		goto out_mtx;
+
+	file = au_xino_file(xi, /*idx*/-1);
+	AuDebugOn(!file);
+	xinew.idx = calc->idx;
+	xinew.base = &file->f_path;
+	/* xinew.copy_src = NULL; */
+	file = au_xi_new(sb, &xinew);
+	if (IS_ERR(file))
+		err = PTR_ERR(file);
+
+out_mtx:
+	mutex_unlock(&xi->xi_mtx);
+out:
+	return err;
+}
+
+struct au_xino_do_new_async_args {
+	struct super_block *sb;
+	struct au_branch *br;
+	struct au_xi_calc calc;
+	ino_t ino;
+};
+
+struct au_xi_writing {
+	struct hlist_bl_node node;
+	ino_t h_ino, ino;
+};
+
+static int au_xino_do_write(vfs_writef_t write, struct file *file,
+			    struct au_xi_calc *calc, ino_t ino);
+
+static void au_xino_call_do_new_async(void *args)
+{
+	struct au_xino_do_new_async_args *a = args;
+	struct au_branch *br;
+	struct super_block *sb;
+	struct au_sbinfo *sbi;
+	struct inode *root;
+	struct file *file;
+	struct au_xi_writing *del, *p;
+	struct hlist_bl_head *hbl;
+	struct hlist_bl_node *pos;
+	int err;
+
+	br = a->br;
+	sb = a->sb;
+	sbi = au_sbi(sb);
+	si_noflush_read_lock(sb);
+	root = d_inode(sb->s_root);
+	ii_read_lock_child(root);
+	err = au_xino_do_new_async(sb, br, &a->calc);
+	if (unlikely(err)) {
+		AuIOErr("err %d\n", err);
+		goto out;
+	}
+
+	file = au_xino_file(br->br_xino, a->calc.idx);
+	AuDebugOn(!file);
+	err = au_xino_do_write(sbi->si_xwrite, file, &a->calc, a->ino);
+	if (unlikely(err)) {
+		AuIOErr("err %d\n", err);
+		goto out;
+	}
+
+	del = NULL;
+	hbl = &br->br_xino->xi_writing;
+	hlist_bl_lock(hbl);
+	au_hbl_for_each(pos, hbl) {
+		p = container_of(pos, struct au_xi_writing, node);
+		if (p->ino == a->ino) {
+			del = p;
+			hlist_bl_del(&p->node);
+			break;
+		}
+	}
+	hlist_bl_unlock(hbl);
+	au_kfree_rcu(del);
+
+out:
+	au_lcnt_dec(&br->br_count);
+	ii_read_unlock(root);
+	si_read_unlock(sb);
+	au_nwt_done(&sbi->si_nowait);
+	au_kfree_rcu(a);
+}
+
+/*
+ * create a new xino file asynchronously
+ */
+static int au_xino_new_async(struct super_block *sb, struct au_branch *br,
+			     struct au_xi_calc *calc, ino_t ino)
+{
+	int err;
+	struct au_xino_do_new_async_args *arg;
+
+	err = -ENOMEM;
+	arg = kmalloc(sizeof(*arg), GFP_NOFS);
+	if (unlikely(!arg))
+		goto out;
+
+	arg->sb = sb;
+	arg->br = br;
+	arg->calc = *calc;
+	arg->ino = ino;
+	au_lcnt_inc(&br->br_count);
+	err = au_wkq_nowait(au_xino_call_do_new_async, arg, sb, AuWkq_NEST);
+	if (unlikely(err)) {
+		pr_err("wkq %d\n", err);
+		au_lcnt_dec(&br->br_count);
+		au_kfree_rcu(arg);
+	}
+
+out:
+	return err;
+}
+
+/*
+ * read @ino from xinofile for the specified branch{@sb, @bindex}
+ * at the position of @h_ino.
+ */
+int au_xino_read(struct super_block *sb, aufs_bindex_t bindex, ino_t h_ino,
+		 ino_t *ino)
+{
+	int err;
+	ssize_t sz;
+	struct au_xi_calc calc;
+	struct au_sbinfo *sbinfo;
+	struct file *file;
+	struct au_xino *xi;
+	struct hlist_bl_head *hbl;
+	struct hlist_bl_node *pos;
+	struct au_xi_writing *p;
+
+	*ino = 0;
+	if (!au_opt_test(au_mntflags(sb), XINO))
+		return 0; /* no xino */
+
+	err = 0;
+	au_xi_calc(sb, h_ino, &calc);
+	xi = au_sbr(sb, bindex)->br_xino;
+	file = au_xino_file(xi, calc.idx);
+	if (!file) {
+		hbl = &xi->xi_writing;
+		hlist_bl_lock(hbl);
+		au_hbl_for_each(pos, hbl) {
+			p = container_of(pos, struct au_xi_writing, node);
+			if (p->h_ino == h_ino) {
+				AuDbg("hi%llu, i%llu, found\n",
+				      (u64)p->h_ino, (u64)p->ino);
+				*ino = p->ino;
+				break;
+			}
+		}
+		hlist_bl_unlock(hbl);
+		return 0;
+	} else if (vfsub_f_size_read(file) < calc.pos + sizeof(*ino))
+		return 0; /* no xino */
+
+	sbinfo = au_sbi(sb);
+	sz = xino_fread(sbinfo->si_xread, file, ino, sizeof(*ino), &calc.pos);
+	if (sz == sizeof(*ino))
+		return 0; /* success */
+
+	err = sz;
+	if (unlikely(sz >= 0)) {
+		err = -EIO;
+		AuIOErr("xino read error (%zd)\n", sz);
+	}
+	return err;
+}
+
+static int au_xino_do_write(vfs_writef_t write, struct file *file,
+			    struct au_xi_calc *calc, ino_t ino)
+{
+	ssize_t sz;
+
+	sz = xino_fwrite(write, file, &ino, sizeof(ino), &calc->pos);
+	if (sz == sizeof(ino))
+		return 0; /* success */
+
+	AuIOErr("write failed (%zd)\n", sz);
+	return -EIO;
+}
+
+/*
+ * write @ino to the xinofile for the specified branch{@sb, @bindex}
+ * at the position of @h_ino.
+ * even if @ino is zero, it is written to the xinofile and means no entry.
+ * if the size of the xino file on a specific filesystem exceeds the watermark,
+ * try truncating it.
+ */
+int au_xino_write(struct super_block *sb, aufs_bindex_t bindex, ino_t h_ino,
+		  ino_t ino)
+{
+	int err;
+	unsigned int mnt_flags;
+	struct au_xi_calc calc;
+	struct file *file;
+	struct au_branch *br;
+	struct au_xino *xi;
+	struct au_xi_writing *p;
+
+	SiMustAnyLock(sb);
+
+	mnt_flags = au_mntflags(sb);
+	if (!au_opt_test(mnt_flags, XINO))
+		return 0;
+
+	au_xi_calc(sb, h_ino, &calc);
+	br = au_sbr(sb, bindex);
+	xi = br->br_xino;
+	file = au_xino_file(xi, calc.idx);
+	if (!file) {
+		/* store the inum pair into the list */
+		p = kmalloc(sizeof(*p), GFP_NOFS | __GFP_NOFAIL);
+		p->h_ino = h_ino;
+		p->ino = ino;
+		au_hbl_add(&p->node, &xi->xi_writing);
+
+		/* create and write a new xino file asynchronously */
+		err = au_xino_new_async(sb, br, &calc, ino);
+		if (!err)
+			return 0; /* success */
+		goto out;
+	}
+
+	err = au_xino_do_write(au_sbi(sb)->si_xwrite, file, &calc, ino);
+	if (!err) {
+		br = au_sbr(sb, bindex);
+		if (au_opt_test(mnt_flags, TRUNC_XINO)
+		    && au_test_fs_trunc_xino(au_br_sb(br)))
+			xino_try_trunc(sb, br);
+		return 0; /* success */
+	}
+
+out:
+	AuIOErr("write failed (%d)\n", err);
+	return -EIO;
+}
+
+static ssize_t xino_fread_wkq(vfs_readf_t func, struct file *file, void *buf,
+			      size_t size, loff_t *pos);
+
+/* todo: unnecessary to support mmap_sem since kernel-space? */
+ssize_t xino_fread(vfs_readf_t func, struct file *file, void *kbuf, size_t size,
+		   loff_t *pos)
+{
+	ssize_t err;
+	mm_segment_t oldfs;
+	union {
+		void *k;
+		char __user *u;
+	} buf;
+	int i;
+	const int prevent_endless = 10;
+
+	i = 0;
+	buf.k = kbuf;
+	oldfs = get_fs();
+	set_fs(KERNEL_DS);
+	do {
+		err = func(file, buf.u, size, pos);
+		if (err == -EINTR
+		    && !au_wkq_test()
+		    && fatal_signal_pending(current)) {
+			set_fs(oldfs);
+			err = xino_fread_wkq(func, file, kbuf, size, pos);
+			BUG_ON(err == -EINTR);
+			oldfs = get_fs();
+			set_fs(KERNEL_DS);
+		}
+	} while (i++ < prevent_endless
+		 && (err == -EAGAIN || err == -EINTR));
+	set_fs(oldfs);
+
+#if 0 /* reserved for future use */
+	if (err > 0)
+		fsnotify_access(file->f_path.dentry);
+#endif
+
+	return err;
+}
+
+struct xino_fread_args {
+	ssize_t *errp;
+	vfs_readf_t func;
+	struct file *file;
+	void *buf;
+	size_t size;
+	loff_t *pos;
+};
+
+static void call_xino_fread(void *args)
+{
+	struct xino_fread_args *a = args;
+	*a->errp = xino_fread(a->func, a->file, a->buf, a->size, a->pos);
+}
+
+static ssize_t xino_fread_wkq(vfs_readf_t func, struct file *file, void *buf,
+			      size_t size, loff_t *pos)
+{
+	ssize_t err;
+	int wkq_err;
+	struct xino_fread_args args = {
+		.errp	= &err,
+		.func	= func,
+		.file	= file,
+		.buf	= buf,
+		.size	= size,
+		.pos	= pos
+	};
+
+	wkq_err = au_wkq_wait(call_xino_fread, &args);
+	if (unlikely(wkq_err))
+		err = wkq_err;
+
+	return err;
+}
+
+static ssize_t xino_fwrite_wkq(vfs_writef_t func, struct file *file, void *buf,
+			       size_t size, loff_t *pos);
+
+static ssize_t do_xino_fwrite(vfs_writef_t func, struct file *file, void *kbuf,
+			      size_t size, loff_t *pos)
+{
+	ssize_t err;
+	mm_segment_t oldfs;
+	union {
+		void *k;
+		const char __user *u;
+	} buf;
+	int i;
+	const int prevent_endless = 10;
+
+	i = 0;
+	buf.k = kbuf;
+	oldfs = get_fs();
+	set_fs(KERNEL_DS);
+	do {
+		err = func(file, buf.u, size, pos);
+		if (err == -EINTR
+		    && !au_wkq_test()
+		    && fatal_signal_pending(current)) {
+			set_fs(oldfs);
+			err = xino_fwrite_wkq(func, file, kbuf, size, pos);
+			BUG_ON(err == -EINTR);
+			oldfs = get_fs();
+			set_fs(KERNEL_DS);
+		}
+	} while (i++ < prevent_endless
+		 && (err == -EAGAIN || err == -EINTR));
+	set_fs(oldfs);
+
+#if 0 /* reserved for future use */
+	if (err > 0)
+		fsnotify_modify(file->f_path.dentry);
+#endif
+
+	return err;
+}
+
+struct do_xino_fwrite_args {
+	ssize_t *errp;
+	vfs_writef_t func;
+	struct file *file;
+	void *buf;
+	size_t size;
+	loff_t *pos;
+};
+
+static void call_do_xino_fwrite(void *args)
+{
+	struct do_xino_fwrite_args *a = args;
+	*a->errp = do_xino_fwrite(a->func, a->file, a->buf, a->size, a->pos);
+}
+
+static ssize_t xino_fwrite_wkq(vfs_writef_t func, struct file *file, void *buf,
+			       size_t size, loff_t *pos)
+{
+	ssize_t err;
+	int wkq_err;
+	struct do_xino_fwrite_args args = {
+		.errp	= &err,
+		.func	= func,
+		.file	= file,
+		.buf	= buf,
+		.size	= size,
+		.pos	= pos
+	};
+
+	/*
+	 * it breaks RLIMIT_FSIZE and normal user's limit,
+	 * users should care about quota and real 'filesystem full.'
+	 */
+	wkq_err = au_wkq_wait(call_do_xino_fwrite, &args);
+	if (unlikely(wkq_err))
+		err = wkq_err;
+
+	return err;
+}
+
+ssize_t xino_fwrite(vfs_writef_t func, struct file *file, void *buf,
+		    size_t size, loff_t *pos)
+{
+	ssize_t err;
+
+	if (rlimit(RLIMIT_FSIZE) == RLIM_INFINITY) {
+		lockdep_off();
+		err = do_xino_fwrite(func, file, buf, size, pos);
+		lockdep_on();
+	} else {
+		lockdep_off();
+		err = xino_fwrite_wkq(func, file, buf, size, pos);
+		lockdep_on();
+	}
+
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * inode number bitmap
+ */
+static const int page_bits = (int)PAGE_SIZE * BITS_PER_BYTE;
+static ino_t xib_calc_ino(unsigned long pindex, int bit)
+{
+	ino_t ino;
+
+	AuDebugOn(bit < 0 || page_bits <= bit);
+	ino = AUFS_FIRST_INO + pindex * page_bits + bit;
+	return ino;
+}
+
+static void xib_calc_bit(ino_t ino, unsigned long *pindex, int *bit)
+{
+	AuDebugOn(ino < AUFS_FIRST_INO);
+	ino -= AUFS_FIRST_INO;
+	*pindex = ino / page_bits;
+	*bit = ino % page_bits;
+}
+
+static int xib_pindex(struct super_block *sb, unsigned long pindex)
+{
+	int err;
+	loff_t pos;
+	ssize_t sz;
+	struct au_sbinfo *sbinfo;
+	struct file *xib;
+	unsigned long *p;
+
+	sbinfo = au_sbi(sb);
+	MtxMustLock(&sbinfo->si_xib_mtx);
+	AuDebugOn(pindex > ULONG_MAX / PAGE_SIZE
+		  || !au_opt_test(sbinfo->si_mntflags, XINO));
+
+	if (pindex == sbinfo->si_xib_last_pindex)
+		return 0;
+
+	xib = sbinfo->si_xib;
+	p = sbinfo->si_xib_buf;
+	pos = sbinfo->si_xib_last_pindex;
+	pos *= PAGE_SIZE;
+	sz = xino_fwrite(sbinfo->si_xwrite, xib, p, PAGE_SIZE, &pos);
+	if (unlikely(sz != PAGE_SIZE))
+		goto out;
+
+	pos = pindex;
+	pos *= PAGE_SIZE;
+	if (vfsub_f_size_read(xib) >= pos + PAGE_SIZE)
+		sz = xino_fread(sbinfo->si_xread, xib, p, PAGE_SIZE, &pos);
+	else {
+		memset(p, 0, PAGE_SIZE);
+		sz = xino_fwrite(sbinfo->si_xwrite, xib, p, PAGE_SIZE, &pos);
+	}
+	if (sz == PAGE_SIZE) {
+		sbinfo->si_xib_last_pindex = pindex;
+		return 0; /* success */
+	}
+
+out:
+	AuIOErr1("write failed (%zd)\n", sz);
+	err = sz;
+	if (sz >= 0)
+		err = -EIO;
+	return err;
+}
+
+static void au_xib_clear_bit(struct inode *inode)
+{
+	int err, bit;
+	unsigned long pindex;
+	struct super_block *sb;
+	struct au_sbinfo *sbinfo;
+
+	AuDebugOn(inode->i_nlink);
+
+	sb = inode->i_sb;
+	xib_calc_bit(inode->i_ino, &pindex, &bit);
+	AuDebugOn(page_bits <= bit);
+	sbinfo = au_sbi(sb);
+	mutex_lock(&sbinfo->si_xib_mtx);
+	err = xib_pindex(sb, pindex);
+	if (!err) {
+		clear_bit(bit, sbinfo->si_xib_buf);
+		sbinfo->si_xib_next_bit = bit;
+	}
+	mutex_unlock(&sbinfo->si_xib_mtx);
+}
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * truncate a xino bitmap file
+ */
+
+/* todo: slow */
+static int do_xib_restore(struct super_block *sb, struct file *file, void *page)
+{
+	int err, bit;
+	ssize_t sz;
+	unsigned long pindex;
+	loff_t pos, pend;
+	struct au_sbinfo *sbinfo;
+	vfs_readf_t func;
+	ino_t *ino;
+	unsigned long *p;
+
+	err = 0;
+	sbinfo = au_sbi(sb);
+	MtxMustLock(&sbinfo->si_xib_mtx);
+	p = sbinfo->si_xib_buf;
+	func = sbinfo->si_xread;
+	pend = vfsub_f_size_read(file);
+	pos = 0;
+	while (pos < pend) {
+		sz = xino_fread(func, file, page, PAGE_SIZE, &pos);
+		err = sz;
+		if (unlikely(sz <= 0))
+			goto out;
+
+		err = 0;
+		for (ino = page; sz > 0; ino++, sz -= sizeof(ino)) {
+			if (unlikely(*ino < AUFS_FIRST_INO))
+				continue;
+
+			xib_calc_bit(*ino, &pindex, &bit);
+			AuDebugOn(page_bits <= bit);
+			err = xib_pindex(sb, pindex);
+			if (!err)
+				set_bit(bit, p);
+			else
+				goto out;
+		}
+	}
+
+out:
+	return err;
+}
+
+static int xib_restore(struct super_block *sb)
+{
+	int err, i;
+	unsigned int nfile;
+	aufs_bindex_t bindex, bbot;
+	void *page;
+	struct au_branch *br;
+	struct au_xino *xi;
+	struct file *file;
+
+	err = -ENOMEM;
+	page = (void *)__get_free_page(GFP_NOFS);
+	if (unlikely(!page))
+		goto out;
+
+	err = 0;
+	bbot = au_sbbot(sb);
+	for (bindex = 0; !err && bindex <= bbot; bindex++)
+		if (!bindex || is_sb_shared(sb, bindex, bindex - 1) < 0) {
+			br = au_sbr(sb, bindex);
+			xi = br->br_xino;
+			nfile = xi->xi_nfile;
+			for (i = 0; i < nfile; i++) {
+				file = au_xino_file(xi, i);
+				if (file)
+					err = do_xib_restore(sb, file, page);
+			}
+		} else
+			AuDbg("skip shared b%d\n", bindex);
+	free_page((unsigned long)page);
+
+out:
+	return err;
+}
+
+int au_xib_trunc(struct super_block *sb)
+{
+	int err;
+	ssize_t sz;
+	loff_t pos;
+	struct au_sbinfo *sbinfo;
+	unsigned long *p;
+	struct file *file;
+
+	SiMustWriteLock(sb);
+
+	err = 0;
+	sbinfo = au_sbi(sb);
+	if (!au_opt_test(sbinfo->si_mntflags, XINO))
+		goto out;
+
+	file = sbinfo->si_xib;
+	if (vfsub_f_size_read(file) <= PAGE_SIZE)
+		goto out;
+
+	file = au_xino_create2(sb, &sbinfo->si_xib->f_path, NULL);
+	err = PTR_ERR(file);
+	if (IS_ERR(file))
+		goto out;
+	fput(sbinfo->si_xib);
+	sbinfo->si_xib = file;
+
+	p = sbinfo->si_xib_buf;
+	memset(p, 0, PAGE_SIZE);
+	pos = 0;
+	sz = xino_fwrite(sbinfo->si_xwrite, sbinfo->si_xib, p, PAGE_SIZE, &pos);
+	if (unlikely(sz != PAGE_SIZE)) {
+		err = sz;
+		AuIOErr("err %d\n", err);
+		if (sz >= 0)
+			err = -EIO;
+		goto out;
+	}
+
+	mutex_lock(&sbinfo->si_xib_mtx);
+	/* mnt_want_write() is unnecessary here */
+	err = xib_restore(sb);
+	mutex_unlock(&sbinfo->si_xib_mtx);
+
+out:
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+struct au_xino *au_xino_alloc(unsigned int nfile)
+{
+	struct au_xino *xi;
+
+	xi = kzalloc(sizeof(*xi), GFP_NOFS);
+	if (unlikely(!xi))
+		goto out;
+	xi->xi_nfile = nfile;
+	xi->xi_file = kcalloc(nfile, sizeof(*xi->xi_file), GFP_NOFS);
+	if (unlikely(!xi->xi_file))
+		goto out_free;
+
+	xi->xi_nondir.total = 8; /* initial size */
+	xi->xi_nondir.array = kcalloc(xi->xi_nondir.total, sizeof(ino_t),
+				      GFP_NOFS);
+	if (unlikely(!xi->xi_nondir.array))
+		goto out_file;
+
+	spin_lock_init(&xi->xi_nondir.spin);
+	init_waitqueue_head(&xi->xi_nondir.wqh);
+	mutex_init(&xi->xi_mtx);
+	INIT_HLIST_BL_HEAD(&xi->xi_writing);
+	atomic_set(&xi->xi_truncating, 0);
+	kref_init(&xi->xi_kref);
+	goto out; /* success */
+
+out_file:
+	au_kfree_try_rcu(xi->xi_file);
+out_free:
+	au_kfree_rcu(xi);
+	xi = NULL;
+out:
+	return xi;
+}
+
+static int au_xino_init(struct au_branch *br, int idx, struct file *file)
+{
+	int err;
+	struct au_xino *xi;
+
+	err = 0;
+	xi = au_xino_alloc(idx + 1);
+	if (unlikely(!xi)) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	if (file)
+		get_file(file);
+	xi->xi_file[idx] = file;
+	AuDebugOn(br->br_xino);
+	br->br_xino = xi;
+
+out:
+	return err;
+}
+
+static void au_xino_release(struct kref *kref)
+{
+	struct au_xino *xi;
+	int i;
+	unsigned long ul;
+	struct hlist_bl_head *hbl;
+	struct hlist_bl_node *pos, *n;
+	struct au_xi_writing *p;
+
+	xi = container_of(kref, struct au_xino, xi_kref);
+	for (i = 0; i < xi->xi_nfile; i++)
+		if (xi->xi_file[i])
+			fput(xi->xi_file[i]);
+	for (i = xi->xi_nondir.total - 1; i >= 0; i--)
+		AuDebugOn(xi->xi_nondir.array[i]);
+	mutex_destroy(&xi->xi_mtx);
+	hbl = &xi->xi_writing;
+	ul = au_hbl_count(hbl);
+	if (unlikely(ul)) {
+		pr_warn("xi_writing %lu\n", ul);
+		hlist_bl_lock(hbl);
+		hlist_bl_for_each_entry_safe(p, pos, n, hbl, node) {
+			hlist_bl_del(&p->node);
+			/* kmemleak reported au_kfree_rcu() doesn't free it */
+			kfree(p);
+		}
+		hlist_bl_unlock(hbl);
+	}
+	au_kfree_try_rcu(xi->xi_file);
+	au_kfree_try_rcu(xi->xi_nondir.array);
+	au_kfree_rcu(xi);
+}
+
+int au_xino_put(struct au_branch *br)
+{
+	int ret;
+	struct au_xino *xi;
+
+	ret = 0;
+	xi = br->br_xino;
+	if (xi) {
+		br->br_xino = NULL;
+		ret = kref_put(&xi->xi_kref, au_xino_release);
+	}
+
+	return ret;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * xino mount option handlers
+ */
+
+/* xino bitmap */
+static void xino_clear_xib(struct super_block *sb)
+{
+	struct au_sbinfo *sbinfo;
+
+	SiMustWriteLock(sb);
+
+	sbinfo = au_sbi(sb);
+	/* unnecessary to clear sbinfo->si_xread and ->si_xwrite */
+	if (sbinfo->si_xib)
+		fput(sbinfo->si_xib);
+	sbinfo->si_xib = NULL;
+	if (sbinfo->si_xib_buf)
+		free_page((unsigned long)sbinfo->si_xib_buf);
+	sbinfo->si_xib_buf = NULL;
+}
+
+static int au_xino_set_xib(struct super_block *sb, struct path *path)
+{
+	int err;
+	loff_t pos;
+	struct au_sbinfo *sbinfo;
+	struct file *file;
+	struct super_block *xi_sb;
+
+	SiMustWriteLock(sb);
+
+	sbinfo = au_sbi(sb);
+	file = au_xino_create2(sb, path, sbinfo->si_xib);
+	err = PTR_ERR(file);
+	if (IS_ERR(file))
+		goto out;
+	if (sbinfo->si_xib)
+		fput(sbinfo->si_xib);
+	sbinfo->si_xib = file;
+	sbinfo->si_xread = vfs_readf(file);
+	sbinfo->si_xwrite = vfs_writef(file);
+	xi_sb = file_inode(file)->i_sb;
+	sbinfo->si_ximaxent = xi_sb->s_maxbytes;
+	if (unlikely(sbinfo->si_ximaxent < PAGE_SIZE)) {
+		err = -EIO;
+		pr_err("s_maxbytes(%llu) on %s is too small\n",
+		       (u64)sbinfo->si_ximaxent, au_sbtype(xi_sb));
+		goto out_unset;
+	}
+	sbinfo->si_ximaxent /= sizeof(ino_t);
+
+	err = -ENOMEM;
+	if (!sbinfo->si_xib_buf)
+		sbinfo->si_xib_buf = (void *)get_zeroed_page(GFP_NOFS);
+	if (unlikely(!sbinfo->si_xib_buf))
+		goto out_unset;
+
+	sbinfo->si_xib_last_pindex = 0;
+	sbinfo->si_xib_next_bit = 0;
+	if (vfsub_f_size_read(file) < PAGE_SIZE) {
+		pos = 0;
+		err = xino_fwrite(sbinfo->si_xwrite, file, sbinfo->si_xib_buf,
+				  PAGE_SIZE, &pos);
+		if (unlikely(err != PAGE_SIZE))
+			goto out_free;
+	}
+	err = 0;
+	goto out; /* success */
+
+out_free:
+	if (sbinfo->si_xib_buf)
+		free_page((unsigned long)sbinfo->si_xib_buf);
+	sbinfo->si_xib_buf = NULL;
+	if (err >= 0)
+		err = -EIO;
+out_unset:
+	fput(sbinfo->si_xib);
+	sbinfo->si_xib = NULL;
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+/* xino for each branch */
+static void xino_clear_br(struct super_block *sb)
+{
+	aufs_bindex_t bindex, bbot;
+	struct au_branch *br;
+
+	bbot = au_sbbot(sb);
+	for (bindex = 0; bindex <= bbot; bindex++) {
+		br = au_sbr(sb, bindex);
+		AuDebugOn(!br);
+		au_xino_put(br);
+	}
+}
+
+static void au_xino_set_br_shared(struct super_block *sb, struct au_branch *br,
+				  aufs_bindex_t bshared)
+{
+	struct au_branch *brshared;
+
+	brshared = au_sbr(sb, bshared);
+	AuDebugOn(!brshared->br_xino);
+	AuDebugOn(!brshared->br_xino->xi_file);
+	if (br->br_xino != brshared->br_xino) {
+		au_xino_get(brshared);
+		au_xino_put(br);
+		br->br_xino = brshared->br_xino;
+	}
+}
+
+struct au_xino_do_set_br {
+	vfs_writef_t writef;
+	struct au_branch *br;
+	ino_t h_ino;
+	aufs_bindex_t bshared;
+};
+
+static int au_xino_do_set_br(struct super_block *sb, struct path *path,
+			     struct au_xino_do_set_br *args)
+{
+	int err;
+	struct au_xi_calc calc;
+	struct file *file;
+	struct au_branch *br;
+	struct au_xi_new xinew = {
+		.base = path
+	};
+
+	br = args->br;
+	xinew.xi = br->br_xino;
+	au_xi_calc(sb, args->h_ino, &calc);
+	xinew.copy_src = au_xino_file(xinew.xi, calc.idx);
+	if (args->bshared >= 0)
+		/* shared xino */
+		au_xino_set_br_shared(sb, br, args->bshared);
+	else if (!xinew.xi) {
+		/* new xino */
+		err = au_xino_init(br, calc.idx, xinew.copy_src);
+		if (unlikely(err))
+			goto out;
+	}
+
+	/* force re-creating */
+	xinew.xi = br->br_xino;
+	xinew.idx = calc.idx;
+	mutex_lock(&xinew.xi->xi_mtx);
+	file = au_xi_new(sb, &xinew);
+	mutex_unlock(&xinew.xi->xi_mtx);
+	err = PTR_ERR(file);
+	if (IS_ERR(file))
+		goto out;
+	AuDebugOn(!file);
+
+	err = au_xino_do_write(args->writef, file, &calc, AUFS_ROOT_INO);
+	if (unlikely(err))
+		au_xino_put(br);
+
+out:
+	AuTraceErr(err);
+	return err;
+}
+
+static int au_xino_set_br(struct super_block *sb, struct path *path)
+{
+	int err;
+	aufs_bindex_t bindex, bbot;
+	struct au_xino_do_set_br args;
+	struct inode *inode;
+
+	SiMustWriteLock(sb);
+
+	bbot = au_sbbot(sb);
+	inode = d_inode(sb->s_root);
+	args.writef = au_sbi(sb)->si_xwrite;
+	for (bindex = 0; bindex <= bbot; bindex++) {
+		args.h_ino = au_h_iptr(inode, bindex)->i_ino;
+		args.br = au_sbr(sb, bindex);
+		args.bshared = is_sb_shared(sb, bindex, bindex - 1);
+		err = au_xino_do_set_br(sb, path, &args);
+		if (unlikely(err))
+			break;
+	}
+
+	AuTraceErr(err);
+	return err;
+}
+
+void au_xino_clr(struct super_block *sb)
+{
+	struct au_sbinfo *sbinfo;
+
+	au_xigen_clr(sb);
+	xino_clear_xib(sb);
+	xino_clear_br(sb);
+	dbgaufs_brs_del(sb, 0);
+	sbinfo = au_sbi(sb);
+	/* lvalue, do not call au_mntflags() */
+	au_opt_clr(sbinfo->si_mntflags, XINO);
+}
+
+int au_xino_set(struct super_block *sb, struct au_opt_xino *xiopt, int remount)
+{
+	int err, skip;
+	struct dentry *dentry, *parent, *cur_dentry, *cur_parent;
+	struct qstr *dname, *cur_name;
+	struct file *cur_xino;
+	struct au_sbinfo *sbinfo;
+	struct path *path, *cur_path;
+
+	SiMustWriteLock(sb);
+
+	err = 0;
+	sbinfo = au_sbi(sb);
+	path = &xiopt->file->f_path;
+	dentry = path->dentry;
+	parent = dget_parent(dentry);
+	if (remount) {
+		skip = 0;
+		cur_xino = sbinfo->si_xib;
+		if (cur_xino) {
+			cur_path = &cur_xino->f_path;
+			cur_dentry = cur_path->dentry;
+			cur_parent = dget_parent(cur_dentry);
+			cur_name = &cur_dentry->d_name;
+			dname = &dentry->d_name;
+			skip = (cur_parent == parent
+				&& au_qstreq(dname, cur_name));
+			dput(cur_parent);
+		}
+		if (skip)
+			goto out;
+	}
+
+	au_opt_set(sbinfo->si_mntflags, XINO);
+	err = au_xino_set_xib(sb, path);
+	/* si_x{read,write} are set */
+	if (!err)
+		err = au_xigen_set(sb, path);
+	if (!err)
+		err = au_xino_set_br(sb, path);
+	if (!err) {
+		dbgaufs_brs_add(sb, 0, /*topdown*/1);
+		goto out; /* success */
+	}
+
+	/* reset all */
+	AuIOErr("failed setting xino(%d).\n", err);
+	au_xino_clr(sb);
+
+out:
+	dput(parent);
+	return err;
+}
+
+/*
+ * create a xinofile at the default place/path.
+ */
+struct file *au_xino_def(struct super_block *sb)
+{
+	struct file *file;
+	char *page, *p;
+	struct au_branch *br;
+	struct super_block *h_sb;
+	struct path path;
+	aufs_bindex_t bbot, bindex, bwr;
+
+	br = NULL;
+	bbot = au_sbbot(sb);
+	bwr = -1;
+	for (bindex = 0; bindex <= bbot; bindex++) {
+		br = au_sbr(sb, bindex);
+		if (au_br_writable(br->br_perm)
+		    && !au_test_fs_bad_xino(au_br_sb(br))) {
+			bwr = bindex;
+			break;
+		}
+	}
+
+	if (bwr >= 0) {
+		file = ERR_PTR(-ENOMEM);
+		page = (void *)__get_free_page(GFP_NOFS);
+		if (unlikely(!page))
+			goto out;
+		path.mnt = au_br_mnt(br);
+		path.dentry = au_h_dptr(sb->s_root, bwr);
+		p = d_path(&path, page, PATH_MAX - sizeof(AUFS_XINO_FNAME));
+		file = (void *)p;
+		if (!IS_ERR(p)) {
+			strcat(p, "/" AUFS_XINO_FNAME);
+			AuDbg("%s\n", p);
+			file = au_xino_create(sb, p, /*silent*/0, /*wbrtop*/1);
+		}
+		free_page((unsigned long)page);
+	} else {
+		file = au_xino_create(sb, AUFS_XINO_DEFPATH, /*silent*/0,
+				      /*wbrtop*/0);
+		if (IS_ERR(file))
+			goto out;
+		h_sb = file->f_path.dentry->d_sb;
+		if (unlikely(au_test_fs_bad_xino(h_sb))) {
+			pr_err("xino doesn't support %s(%s)\n",
+			       AUFS_XINO_DEFPATH, au_sbtype(h_sb));
+			fput(file);
+			file = ERR_PTR(-EINVAL);
+		}
+	}
+
+out:
+	return file;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * initialize the xinofile for the specified branch @br
+ * at the place/path where @base_file indicates.
+ * test whether another branch is on the same filesystem or not,
+ * if found then share the xinofile with another branch.
+ */
+int au_xino_init_br(struct super_block *sb, struct au_branch *br, ino_t h_ino,
+		    struct path *base)
+{
+	int err;
+	struct au_xino_do_set_br args = {
+		.h_ino	= h_ino,
+		.br	= br
+	};
+
+	args.writef = au_sbi(sb)->si_xwrite;
+	args.bshared = sbr_find_shared(sb, /*btop*/0, au_sbbot(sb),
+				       au_br_sb(br));
+	err = au_xino_do_set_br(sb, base, &args);
+	if (unlikely(err))
+		au_xino_put(br);
+
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/*
+ * get an unused inode number from bitmap
+ */
+ino_t au_xino_new_ino(struct super_block *sb)
+{
+	ino_t ino;
+	unsigned long *p, pindex, ul, pend;
+	struct au_sbinfo *sbinfo;
+	struct file *file;
+	int free_bit, err;
+
+	if (!au_opt_test(au_mntflags(sb), XINO))
+		return iunique(sb, AUFS_FIRST_INO);
+
+	sbinfo = au_sbi(sb);
+	mutex_lock(&sbinfo->si_xib_mtx);
+	p = sbinfo->si_xib_buf;
+	free_bit = sbinfo->si_xib_next_bit;
+	if (free_bit < page_bits && !test_bit(free_bit, p))
+		goto out; /* success */
+	free_bit = find_first_zero_bit(p, page_bits);
+	if (free_bit < page_bits)
+		goto out; /* success */
+
+	pindex = sbinfo->si_xib_last_pindex;
+	for (ul = pindex - 1; ul < ULONG_MAX; ul--) {
+		err = xib_pindex(sb, ul);
+		if (unlikely(err))
+			goto out_err;
+		free_bit = find_first_zero_bit(p, page_bits);
+		if (free_bit < page_bits)
+			goto out; /* success */
+	}
+
+	file = sbinfo->si_xib;
+	pend = vfsub_f_size_read(file) / PAGE_SIZE;
+	for (ul = pindex + 1; ul <= pend; ul++) {
+		err = xib_pindex(sb, ul);
+		if (unlikely(err))
+			goto out_err;
+		free_bit = find_first_zero_bit(p, page_bits);
+		if (free_bit < page_bits)
+			goto out; /* success */
+	}
+	BUG();
+
+out:
+	set_bit(free_bit, p);
+	sbinfo->si_xib_next_bit = free_bit + 1;
+	pindex = sbinfo->si_xib_last_pindex;
+	mutex_unlock(&sbinfo->si_xib_mtx);
+	ino = xib_calc_ino(pindex, free_bit);
+	AuDbg("i%lu\n", (unsigned long)ino);
+	return ino;
+out_err:
+	mutex_unlock(&sbinfo->si_xib_mtx);
+	AuDbg("i0\n");
+	return 0;
+}
+
+/* for s_op->delete_inode() */
+void au_xino_delete_inode(struct inode *inode, const int unlinked)
+{
+	int err;
+	unsigned int mnt_flags;
+	aufs_bindex_t bindex, bbot, bi;
+	unsigned char try_trunc;
+	struct au_iinfo *iinfo;
+	struct super_block *sb;
+	struct au_hinode *hi;
+	struct inode *h_inode;
+	struct au_branch *br;
+	vfs_writef_t xwrite;
+	struct au_xi_calc calc;
+	struct file *file;
+
+	AuDebugOn(au_is_bad_inode(inode));
+
+	sb = inode->i_sb;
+	mnt_flags = au_mntflags(sb);
+	if (!au_opt_test(mnt_flags, XINO)
+	    || inode->i_ino == AUFS_ROOT_INO)
+		return;
+
+	if (unlinked) {
+		au_xigen_inc(inode);
+		au_xib_clear_bit(inode);
+	}
+
+	iinfo = au_ii(inode);
+	bindex = iinfo->ii_btop;
+	if (bindex < 0)
+		return;
+
+	xwrite = au_sbi(sb)->si_xwrite;
+	try_trunc = !!au_opt_test(mnt_flags, TRUNC_XINO);
+	hi = au_hinode(iinfo, bindex);
+	bbot = iinfo->ii_bbot;
+	for (; bindex <= bbot; bindex++, hi++) {
+		h_inode = hi->hi_inode;
+		if (!h_inode
+		    || (!unlinked && h_inode->i_nlink))
+			continue;
+
+		/* inode may not be revalidated */
+		bi = au_br_index(sb, hi->hi_id);
+		if (bi < 0)
+			continue;
+
+		br = au_sbr(sb, bi);
+		au_xi_calc(sb, h_inode->i_ino, &calc);
+		file = au_xino_file(br->br_xino, calc.idx);
+		if (IS_ERR_OR_NULL(file))
+			continue;
+
+		err = au_xino_do_write(xwrite, file, &calc, /*ino*/0);
+		if (!err && try_trunc
+		    && au_test_fs_trunc_xino(au_br_sb(br)))
+			xino_try_trunc(sb, br);
+	}
+}
+
+/* ---------------------------------------------------------------------- */
+
+static int au_xinondir_find(struct au_xino *xi, ino_t h_ino)
+{
+	int found, total, i;
+
+	found = -1;
+	total = xi->xi_nondir.total;
+	for (i = 0; i < total; i++) {
+		if (xi->xi_nondir.array[i] != h_ino)
+			continue;
+		found = i;
+		break;
+	}
+
+	return found;
+}
+
+static int au_xinondir_expand(struct au_xino *xi)
+{
+	int err, sz;
+	ino_t *p;
+
+	BUILD_BUG_ON(KMALLOC_MAX_SIZE > INT_MAX);
+
+	err = -ENOMEM;
+	sz = xi->xi_nondir.total * sizeof(ino_t);
+	if (unlikely(sz > KMALLOC_MAX_SIZE / 2))
+		goto out;
+	p = au_kzrealloc(xi->xi_nondir.array, sz, sz << 1, GFP_ATOMIC,
+			 /*may_shrink*/0);
+	if (p) {
+		xi->xi_nondir.array = p;
+		xi->xi_nondir.total <<= 1;
+		AuDbg("xi_nondir.total %d\n", xi->xi_nondir.total);
+		err = 0;
+	}
+
+out:
+	return err;
+}
+
+void au_xinondir_leave(struct super_block *sb, aufs_bindex_t bindex,
+		       ino_t h_ino, int idx)
+{
+	struct au_xino *xi;
+
+	AuDebugOn(!au_opt_test(au_mntflags(sb), XINO));
+	xi = au_sbr(sb, bindex)->br_xino;
+	AuDebugOn(idx < 0 || xi->xi_nondir.total <= idx);
+
+	spin_lock(&xi->xi_nondir.spin);
+	AuDebugOn(xi->xi_nondir.array[idx] != h_ino);
+	xi->xi_nondir.array[idx] = 0;
+	spin_unlock(&xi->xi_nondir.spin);
+	wake_up_all(&xi->xi_nondir.wqh);
+}
+
+int au_xinondir_enter(struct super_block *sb, aufs_bindex_t bindex, ino_t h_ino,
+		      int *idx)
+{
+	int err, found, empty;
+	struct au_xino *xi;
+
+	err = 0;
+	*idx = -1;
+	if (!au_opt_test(au_mntflags(sb), XINO))
+		goto out; /* no xino */
+
+	xi = au_sbr(sb, bindex)->br_xino;
+
+again:
+	spin_lock(&xi->xi_nondir.spin);
+	found = au_xinondir_find(xi, h_ino);
+	if (found == -1) {
+		empty = au_xinondir_find(xi, /*h_ino*/0);
+		if (empty == -1) {
+			empty = xi->xi_nondir.total;
+			err = au_xinondir_expand(xi);
+			if (unlikely(err))
+				goto out_unlock;
+		}
+		xi->xi_nondir.array[empty] = h_ino;
+		*idx = empty;
+	} else {
+		spin_unlock(&xi->xi_nondir.spin);
+		wait_event(xi->xi_nondir.wqh,
+			   xi->xi_nondir.array[found] != h_ino);
+		goto again;
+	}
+
+out_unlock:
+	spin_unlock(&xi->xi_nondir.spin);
+out:
+	return err;
+}
+
+/* ---------------------------------------------------------------------- */
+
+int au_xino_path(struct seq_file *seq, struct file *file)
+{
+	int err;
+
+	err = au_seq_path(seq, &file->f_path);
+	if (unlikely(err))
+		goto out;
+
+#define Deleted "\\040(deleted)"
+	seq->count -= sizeof(Deleted) - 1;
+	AuDebugOn(memcmp(seq->buf + seq->count, Deleted,
+			 sizeof(Deleted) - 1));
+#undef Deleted
+
+out:
+	return err;
+}
diff -Naur --no-dereference a/fs/dcache.c b/fs/dcache.c
--- a/fs/dcache.c	2020-01-09 04:19:10.000000000 -0500
+++ b/fs/dcache.c	2020-06-19 21:00:31.428227716 -0400
@@ -1234,7 +1234,7 @@
  *
  * The @enter() callbacks are called with d_lock held.
  */
-static void d_walk(struct dentry *parent, void *data,
+void d_walk(struct dentry *parent, void *data,
 		   enum d_walk_ret (*enter)(void *, struct dentry *))
 {
 	struct dentry *this_parent;
@@ -1339,6 +1339,7 @@
 	seq = 1;
 	goto again;
 }
+EXPORT_SYMBOL_GPL(d_walk);
 
 struct check_mount {
 	struct vfsmount *mnt;
@@ -2835,6 +2836,7 @@
 
 	write_sequnlock(&rename_lock);
 }
+EXPORT_SYMBOL_GPL(d_exchange);
 
 /**
  * d_ancestor - search for an ancestor
diff -Naur --no-dereference a/fs/exec.c b/fs/exec.c
--- a/fs/exec.c	2020-01-09 04:19:10.000000000 -0500
+++ b/fs/exec.c	2020-06-19 21:00:31.428227716 -0400
@@ -109,6 +109,7 @@
 	return (path->mnt->mnt_flags & MNT_NOEXEC) ||
 	       (path->mnt->mnt_sb->s_iflags & SB_I_NOEXEC);
 }
+EXPORT_SYMBOL_GPL(path_noexec);
 
 #ifdef CONFIG_USELIB
 /*
diff -Naur --no-dereference a/fs/fcntl.c b/fs/fcntl.c
--- a/fs/fcntl.c	2020-01-09 04:19:10.000000000 -0500
+++ b/fs/fcntl.c	2020-06-19 21:00:31.428227716 -0400
@@ -32,7 +32,7 @@
 
 #define SETFL_MASK (O_APPEND | O_NONBLOCK | O_NDELAY | O_DIRECT | O_NOATIME)
 
-static int setfl(int fd, struct file * filp, unsigned long arg)
+int setfl(int fd, struct file *filp, unsigned long arg)
 {
 	struct inode * inode = file_inode(filp);
 	int error = 0;
@@ -63,6 +63,8 @@
 
 	if (filp->f_op->check_flags)
 		error = filp->f_op->check_flags(arg);
+	if (!error && filp->f_op->setfl)
+		error = filp->f_op->setfl(filp, arg);
 	if (error)
 		return error;
 
@@ -83,6 +85,7 @@
  out:
 	return error;
 }
+EXPORT_SYMBOL_GPL(setfl);
 
 static void f_modown(struct file *filp, struct pid *pid, enum pid_type type,
                      int force)
diff -Naur --no-dereference a/fs/file_table.c b/fs/file_table.c
--- a/fs/file_table.c	2020-01-09 04:19:10.000000000 -0500
+++ b/fs/file_table.c	2020-06-19 21:00:31.428227716 -0400
@@ -161,6 +161,7 @@
 	}
 	return ERR_PTR(-ENFILE);
 }
+EXPORT_SYMBOL_GPL(alloc_empty_file);
 
 /*
  * Variant of alloc_empty_file() that doesn't check and modify nr_files.
@@ -323,6 +324,7 @@
 {
 	delayed_fput(NULL);
 }
+EXPORT_SYMBOL_GPL(flush_delayed_fput);
 
 static DECLARE_DELAYED_WORK(delayed_fput_work, delayed_fput);
 
@@ -365,6 +367,7 @@
 }
 
 EXPORT_SYMBOL(fput);
+EXPORT_SYMBOL_GPL(__fput_sync);
 
 void __init files_init(void)
 {
diff -Naur --no-dereference a/fs/inode.c b/fs/inode.c
--- a/fs/inode.c	2020-01-09 04:19:10.000000000 -0500
+++ b/fs/inode.c	2020-06-19 21:00:31.428227716 -0400
@@ -1657,7 +1657,7 @@
  * This does the actual work of updating an inodes time or version.  Must have
  * had called mnt_want_write() before calling this.
  */
-static int update_time(struct inode *inode, struct timespec64 *time, int flags)
+int update_time(struct inode *inode, struct timespec64 *time, int flags)
 {
 	int (*update_time)(struct inode *, struct timespec64 *, int);
 
@@ -1666,6 +1666,7 @@
 
 	return update_time(inode, time, flags);
 }
+EXPORT_SYMBOL_GPL(update_time);
 
 /**
  *	touch_atime	-	update the access time
diff -Naur --no-dereference a/fs/Kconfig b/fs/Kconfig
--- a/fs/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/fs/Kconfig	2020-06-19 21:00:31.428227716 -0400
@@ -255,6 +255,7 @@
 source "fs/sysv/Kconfig"
 source "fs/ufs/Kconfig"
 source "fs/exofs/Kconfig"
+source "fs/aufs/Kconfig"
 
 endif # MISC_FILESYSTEMS
 
diff -Naur --no-dereference a/fs/Makefile b/fs/Makefile
--- a/fs/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/fs/Makefile	2020-06-19 21:00:31.428227716 -0400
@@ -128,3 +128,4 @@
 obj-$(CONFIG_CEPH_FS)		+= ceph/
 obj-$(CONFIG_PSTORE)		+= pstore/
 obj-$(CONFIG_EFIVAR_FS)		+= efivarfs/
+obj-$(CONFIG_AUFS_FS)           += aufs/
diff -Naur --no-dereference a/fs/namespace.c b/fs/namespace.c
--- a/fs/namespace.c	2020-01-09 04:19:10.000000000 -0500
+++ b/fs/namespace.c	2020-06-19 21:00:31.428227716 -0400
@@ -437,6 +437,7 @@
 	mnt_dec_writers(real_mount(mnt));
 	preempt_enable();
 }
+EXPORT_SYMBOL_GPL(__mnt_drop_write);
 
 /**
  * mnt_drop_write - give up write access to a mount
@@ -770,6 +771,13 @@
 	return mnt->mnt_ns == current->nsproxy->mnt_ns;
 }
 
+/* for aufs, CONFIG_AUFS_BR_FUSE */
+int is_current_mnt_ns(struct vfsmount *mnt)
+{
+	return check_mnt(real_mount(mnt));
+}
+EXPORT_SYMBOL_GPL(is_current_mnt_ns);
+
 /*
  * vfsmount lock must be held for write
  */
@@ -1838,6 +1846,7 @@
 	}
 	return 0;
 }
+EXPORT_SYMBOL_GPL(iterate_mounts);
 
 static void cleanup_group_ids(struct mount *mnt, struct mount *end)
 {
diff -Naur --no-dereference a/fs/notify/group.c b/fs/notify/group.c
--- a/fs/notify/group.c	2020-01-09 04:19:10.000000000 -0500
+++ b/fs/notify/group.c	2020-06-19 21:00:31.428227716 -0400
@@ -112,6 +112,7 @@
 {
 	refcount_inc(&group->refcnt);
 }
+EXPORT_SYMBOL_GPL(fsnotify_get_group);
 
 /*
  * Drop a reference to a group.  Free it if it's through.
@@ -121,6 +122,7 @@
 	if (refcount_dec_and_test(&group->refcnt))
 		fsnotify_final_destroy_group(group);
 }
+EXPORT_SYMBOL_GPL(fsnotify_put_group);
 
 /*
  * Create a new fsnotify_group and hold a reference for the group returned.
@@ -150,6 +152,7 @@
 
 	return group;
 }
+EXPORT_SYMBOL_GPL(fsnotify_alloc_group);
 
 int fsnotify_fasync(int fd, struct file *file, int on)
 {
diff -Naur --no-dereference a/fs/notify/mark.c b/fs/notify/mark.c
--- a/fs/notify/mark.c	2020-01-09 04:19:10.000000000 -0500
+++ b/fs/notify/mark.c	2020-06-19 21:00:31.428227716 -0400
@@ -285,6 +285,7 @@
 	queue_delayed_work(system_unbound_wq, &reaper_work,
 			   FSNOTIFY_REAPER_DELAY);
 }
+EXPORT_SYMBOL_GPL(fsnotify_put_mark);
 
 /*
  * Get mark reference when we found the mark via lockless traversal of object
@@ -439,6 +440,7 @@
 	mutex_unlock(&group->mark_mutex);
 	fsnotify_free_mark(mark);
 }
+EXPORT_SYMBOL_GPL(fsnotify_destroy_mark);
 
 /*
  * Sorting function for lists of fsnotify marks.
@@ -654,6 +656,7 @@
 	mutex_unlock(&group->mark_mutex);
 	return ret;
 }
+EXPORT_SYMBOL_GPL(fsnotify_add_mark);
 
 /*
  * Given a list of marks, find the mark associated with given group. If found
@@ -777,6 +780,7 @@
 	fsnotify_get_group(group);
 	mark->group = group;
 }
+EXPORT_SYMBOL_GPL(fsnotify_init_mark);
 
 /*
  * Destroy all marks in destroy_list, waits for SRCU period to finish before
diff -Naur --no-dereference a/fs/open.c b/fs/open.c
--- a/fs/open.c	2020-01-09 04:19:10.000000000 -0500
+++ b/fs/open.c	2020-06-19 21:00:31.428227716 -0400
@@ -64,6 +64,7 @@
 	inode_unlock(dentry->d_inode);
 	return ret;
 }
+EXPORT_SYMBOL_GPL(do_truncate);
 
 long vfs_truncate(const struct path *path, loff_t length)
 {
diff -Naur --no-dereference a/fs/proc/base.c b/fs/proc/base.c
--- a/fs/proc/base.c	2020-01-09 04:19:10.000000000 -0500
+++ b/fs/proc/base.c	2020-06-19 21:00:31.428227716 -0400
@@ -2036,7 +2036,7 @@
 	rc = -ENOENT;
 	vma = find_exact_vma(mm, vm_start, vm_end);
 	if (vma && vma->vm_file) {
-		*path = vma->vm_file->f_path;
+		*path = vma_pr_or_file(vma)->f_path;
 		path_get(path);
 		rc = 0;
 	}
diff -Naur --no-dereference a/fs/proc/nommu.c b/fs/proc/nommu.c
--- a/fs/proc/nommu.c	2020-01-09 04:19:10.000000000 -0500
+++ b/fs/proc/nommu.c	2020-06-19 21:00:31.428227716 -0400
@@ -45,7 +45,10 @@
 	file = region->vm_file;
 
 	if (file) {
-		struct inode *inode = file_inode(region->vm_file);
+		struct inode *inode;
+
+		file = vmr_pr_or_file(region);
+		inode = file_inode(file);
 		dev = inode->i_sb->s_dev;
 		ino = inode->i_ino;
 	}
diff -Naur --no-dereference a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
--- a/fs/proc/task_mmu.c	2020-01-09 04:19:10.000000000 -0500
+++ b/fs/proc/task_mmu.c	2020-06-19 21:00:31.428227716 -0400
@@ -309,7 +309,10 @@
 	const char *name = NULL;
 
 	if (file) {
-		struct inode *inode = file_inode(vma->vm_file);
+		struct inode *inode;
+
+		file = vma_pr_or_file(vma);
+		inode = file_inode(file);
 		dev = inode->i_sb->s_dev;
 		ino = inode->i_ino;
 		pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT;
@@ -1768,7 +1771,7 @@
 	struct proc_maps_private *proc_priv = &numa_priv->proc_maps;
 	struct vm_area_struct *vma = v;
 	struct numa_maps *md = &numa_priv->md;
-	struct file *file = vma->vm_file;
+	struct file *file = vma_pr_or_file(vma);
 	struct mm_struct *mm = vma->vm_mm;
 	struct mm_walk walk = {
 		.hugetlb_entry = gather_hugetlb_stats,
diff -Naur --no-dereference a/fs/proc/task_nommu.c b/fs/proc/task_nommu.c
--- a/fs/proc/task_nommu.c	2020-01-09 04:19:10.000000000 -0500
+++ b/fs/proc/task_nommu.c	2020-06-19 21:00:31.428227716 -0400
@@ -155,7 +155,10 @@
 	file = vma->vm_file;
 
 	if (file) {
-		struct inode *inode = file_inode(vma->vm_file);
+		struct inode *inode;
+
+		file = vma_pr_or_file(vma);
+		inode = file_inode(file);
 		dev = inode->i_sb->s_dev;
 		ino = inode->i_ino;
 		pgoff = (loff_t)vma->vm_pgoff << PAGE_SHIFT;
diff -Naur --no-dereference a/fs/read_write.c b/fs/read_write.c
--- a/fs/read_write.c	2020-01-09 04:19:10.000000000 -0500
+++ b/fs/read_write.c	2020-06-19 21:00:31.428227716 -0400
@@ -459,6 +459,7 @@
 
 	return ret;
 }
+EXPORT_SYMBOL_GPL(vfs_read);
 
 static ssize_t new_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
 {
@@ -489,6 +490,30 @@
 		return -EINVAL;
 }
 
+vfs_readf_t vfs_readf(struct file *file)
+{
+	const struct file_operations *fop = file->f_op;
+
+	if (fop->read)
+		return fop->read;
+	if (fop->read_iter)
+		return new_sync_read;
+	return ERR_PTR(-ENOSYS); /* doesn't have ->read(|_iter)() op */
+}
+EXPORT_SYMBOL_GPL(vfs_readf);
+
+vfs_writef_t vfs_writef(struct file *file)
+{
+	const struct file_operations *fop = file->f_op;
+
+	if (fop->write)
+		return fop->write;
+	if (fop->write_iter)
+		return new_sync_write;
+	return ERR_PTR(-ENOSYS); /* doesn't have ->write(|_iter)() op */
+}
+EXPORT_SYMBOL_GPL(vfs_writef);
+
 ssize_t __kernel_write(struct file *file, const void *buf, size_t count, loff_t *pos)
 {
 	mm_segment_t old_fs;
@@ -557,6 +582,7 @@
 
 	return ret;
 }
+EXPORT_SYMBOL_GPL(vfs_write);
 
 static inline loff_t file_pos_read(struct file *file)
 {
diff -Naur --no-dereference a/fs/splice.c b/fs/splice.c
--- a/fs/splice.c	2020-01-09 04:19:10.000000000 -0500
+++ b/fs/splice.c	2020-06-19 21:00:31.428227716 -0400
@@ -838,8 +838,8 @@
 /*
  * Attempt to initiate a splice from pipe to file.
  */
-static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
-			   loff_t *ppos, size_t len, unsigned int flags)
+long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
+		    loff_t *ppos, size_t len, unsigned int flags)
 {
 	ssize_t (*splice_write)(struct pipe_inode_info *, struct file *,
 				loff_t *, size_t, unsigned int);
@@ -851,13 +851,14 @@
 
 	return splice_write(pipe, out, ppos, len, flags);
 }
+EXPORT_SYMBOL_GPL(do_splice_from);
 
 /*
  * Attempt to initiate a splice from a file to a pipe.
  */
-static long do_splice_to(struct file *in, loff_t *ppos,
-			 struct pipe_inode_info *pipe, size_t len,
-			 unsigned int flags)
+long do_splice_to(struct file *in, loff_t *ppos,
+		  struct pipe_inode_info *pipe, size_t len,
+		  unsigned int flags)
 {
 	ssize_t (*splice_read)(struct file *, loff_t *,
 			       struct pipe_inode_info *, size_t, unsigned int);
@@ -880,6 +881,7 @@
 
 	return splice_read(in, ppos, pipe, len, flags);
 }
+EXPORT_SYMBOL_GPL(do_splice_to);
 
 /**
  * splice_direct_to_actor - splices data directly between two non-pipes
diff -Naur --no-dereference a/fs/sync.c b/fs/sync.c
--- a/fs/sync.c	2020-01-09 04:19:10.000000000 -0500
+++ b/fs/sync.c	2020-06-19 21:00:31.428227716 -0400
@@ -28,7 +28,7 @@
  * wait == 1 case since in that case write_inode() functions do
  * sync_dirty_buffer() and thus effectively write one block at a time.
  */
-static int __sync_filesystem(struct super_block *sb, int wait)
+int __sync_filesystem(struct super_block *sb, int wait)
 {
 	if (wait)
 		sync_inodes_sb(sb);
@@ -39,6 +39,7 @@
 		sb->s_op->sync_fs(sb, wait);
 	return __sync_blockdev(sb->s_bdev, wait);
 }
+EXPORT_SYMBOL_GPL(__sync_filesystem);
 
 /*
  * Write out and wait upon all dirty data associated with this
diff -Naur --no-dereference a/fs/xattr.c b/fs/xattr.c
--- a/fs/xattr.c	2020-01-09 04:19:10.000000000 -0500
+++ b/fs/xattr.c	2020-06-19 21:00:31.428227716 -0400
@@ -295,6 +295,7 @@
 	*xattr_value = value;
 	return error;
 }
+EXPORT_SYMBOL_GPL(vfs_getxattr_alloc);
 
 ssize_t
 __vfs_getxattr(struct dentry *dentry, struct inode *inode, const char *name,
diff -Naur --no-dereference a/.gitignore b/.gitignore
--- a/.gitignore	2020-01-09 04:19:10.000000000 -0500
+++ b/.gitignore	2020-06-19 21:00:31.400227715 -0400
@@ -17,11 +17,13 @@
 *.c.[012]*.*
 *.dtb
 *.dtb.S
+*.dtbo
 *.dwo
 *.elf
 *.gcno
 *.gz
 *.i
+*.itb
 *.ko
 *.lex.c
 *.ll
diff -Naur --no-dereference a/include/clocksource/timer-ti-dm.h b/include/clocksource/timer-ti-dm.h
--- a/include/clocksource/timer-ti-dm.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/clocksource/timer-ti-dm.h	2020-06-19 21:00:31.428227716 -0400
@@ -116,6 +116,7 @@
 	u32 errata;
 	struct platform_device *pdev;
 	struct list_head node;
+	u32 late_attach;
 };
 
 int omap_dm_timer_reserve_systimer(int id);
diff -Naur --no-dereference a/include/crypto/sha.h b/include/crypto/sha.h
--- a/include/crypto/sha.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/crypto/sha.h	2020-06-19 21:00:31.428227716 -0400
@@ -95,6 +95,7 @@
 
 struct shash_desc;
 
+extern void sha256_transform(u32 *state, const u8 *input);
 extern int crypto_sha1_update(struct shash_desc *desc, const u8 *data,
 			      unsigned int len);
 
diff -Naur --no-dereference a/include/drm/drm_atomic.h b/include/drm/drm_atomic.h
--- a/include/drm/drm_atomic.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/drm/drm_atomic.h	2020-06-19 21:00:31.428227716 -0400
@@ -227,9 +227,31 @@
  * Currently only tracks the state update functions and the opaque driver
  * private state itself, but in the future might also track which
  * &drm_modeset_lock is required to duplicate and update this object's state.
+ *
+ * All private objects must be initialized before the DRM device they are
+ * attached to is registered to the DRM subsystem (call to drm_dev_register())
+ * and should stay around until this DRM device is unregistered (call to
+ * drm_dev_unregister()). In other words, private objects lifetime is tied
+ * to the DRM device lifetime. This implies that:
+ *
+ * 1/ all calls to drm_atomic_private_obj_init() must be done before calling
+ *    drm_dev_register()
+ * 2/ all calls to drm_atomic_private_obj_fini() must be done after calling
+ *    drm_dev_unregister()
  */
 struct drm_private_obj {
 	/**
+	 * @head: List entry used to attach a private object to a &drm_device
+	 * (queued to &drm_mode_config.privobj_list).
+	 */
+	struct list_head head;
+
+	/**
+	 * @lock: Modeset lock to protect the state object.
+	 */
+	struct drm_modeset_lock lock;
+
+	/**
 	 * @state: Current atomic state for this driver private object.
 	 */
 	struct drm_private_state *state;
@@ -244,6 +266,18 @@
 };
 
 /**
+ * drm_for_each_privobj() - private object iterator
+ *
+ * @privobj: pointer to the current private object. Updated after each
+ *	     iteration
+ * @dev: the DRM device we want get private objects from
+ *
+ * Allows one to iterate over all private objects attached to @dev
+ */
+#define drm_for_each_privobj(privobj, dev) \
+	list_for_each_entry(privobj, &(dev)->mode_config.privobj_list, head)
+
+/**
  * struct drm_private_state - base struct for driver private object state
  * @state: backpointer to global drm_atomic_state
  *
@@ -394,7 +428,8 @@
 drm_atomic_get_connector_state(struct drm_atomic_state *state,
 			       struct drm_connector *connector);
 
-void drm_atomic_private_obj_init(struct drm_private_obj *obj,
+void drm_atomic_private_obj_init(struct drm_device *dev,
+				 struct drm_private_obj *obj,
 				 struct drm_private_state *state,
 				 const struct drm_private_state_funcs *funcs);
 void drm_atomic_private_obj_fini(struct drm_private_obj *obj);
diff -Naur --no-dereference a/include/drm/drm_bridge.h b/include/drm/drm_bridge.h
--- a/include/drm/drm_bridge.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/drm/drm_bridge.h	2020-06-19 21:00:31.428227716 -0400
@@ -244,14 +244,13 @@
  */
 struct drm_bridge_timings {
 	/**
-	 * @sampling_edge:
+	 * @input_bus_flags:
 	 *
-	 * Tells whether the bridge samples the digital input signal
-	 * from the display engine on the positive or negative edge of the
-	 * clock, this should reuse the DRM_BUS_FLAG_PIXDATA_[POS|NEG]EDGE
-	 * bitwise flags from the DRM connector (bit 2 and 3 valid).
+	 * Tells what additional settings for the pixel data on the bus
+	 * this bridge requires (like pixel signal polarity). See also
+	 * &drm_display_info->bus_flags.
 	 */
-	u32 sampling_edge;
+	u32 input_bus_flags;
 	/**
 	 * @setup_time_ps:
 	 *
diff -Naur --no-dereference a/include/drm/drm_connector.h b/include/drm/drm_connector.h
--- a/include/drm/drm_connector.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/drm/drm_connector.h	2020-06-19 21:00:31.428227716 -0400
@@ -206,6 +206,68 @@
 };
 
 /**
+ * enum drm_bus_flags - bus_flags info for &drm_display_info
+ *
+ * This enum defines signal polarities and clock edge information for signals on
+ * a bus as bitmask flags.
+ *
+ * The clock edge information is conveyed by two sets of symbols,
+ * DRM_BUS_FLAGS_*_DRIVE_\* and DRM_BUS_FLAGS_*_SAMPLE_\*. When this enum is
+ * used to describe a bus from the point of view of the transmitter, the
+ * \*_DRIVE_\* flags should be used. When used from the point of view of the
+ * receiver, the \*_SAMPLE_\* flags should be used. The \*_DRIVE_\* and
+ * \*_SAMPLE_\* flags alias each other, with the \*_SAMPLE_POSEDGE and
+ * \*_SAMPLE_NEGEDGE flags being equal to \*_DRIVE_NEGEDGE and \*_DRIVE_POSEDGE
+ * respectively. This simplifies code as signals are usually sampled on the
+ * opposite edge of the driving edge. Transmitters and receivers may however
+ * need to take other signal timings into account to convert between driving
+ * and sample edges.
+ *
+ * @DRM_BUS_FLAG_DE_LOW:		The Data Enable signal is active low
+ * @DRM_BUS_FLAG_DE_HIGH:		The Data Enable signal is active high
+ * @DRM_BUS_FLAG_PIXDATA_POSEDGE:	Legacy value, do not use
+ * @DRM_BUS_FLAG_PIXDATA_NEGEDGE:	Legacy value, do not use
+ * @DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE:	Data is driven on the rising edge of
+ *					the pixel clock
+ * @DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE:	Data is driven on the falling edge of
+ *					the pixel clock
+ * @DRM_BUS_FLAG_PIXDATA_SAMPLE_POSEDGE: Data is sampled on the rising edge of
+ *					the pixel clock
+ * @DRM_BUS_FLAG_PIXDATA_SAMPLE_NEGEDGE: Data is sampled on the falling edge of
+ *					the pixel clock
+ * @DRM_BUS_FLAG_DATA_MSB_TO_LSB:	Data is transmitted MSB to LSB on the bus
+ * @DRM_BUS_FLAG_DATA_LSB_TO_MSB:	Data is transmitted LSB to MSB on the bus
+ * @DRM_BUS_FLAG_SYNC_POSEDGE:		Legacy value, do not use
+ * @DRM_BUS_FLAG_SYNC_NEGEDGE:		Legacy value, do not use
+ * @DRM_BUS_FLAG_SYNC_DRIVE_POSEDGE:	Sync signals are driven on the rising
+ *					edge of the pixel clock
+ * @DRM_BUS_FLAG_SYNC_DRIVE_NEGEDGE:	Sync signals are driven on the falling
+ *					edge of the pixel clock
+ * @DRM_BUS_FLAG_SYNC_SAMPLE_POSEDGE:	Sync signals are sampled on the rising
+ *					edge of the pixel clock
+ * @DRM_BUS_FLAG_SYNC_SAMPLE_NEGEDGE:	Sync signals are sampled on the falling
+ *					edge of the pixel clock
+ */
+enum drm_bus_flags {
+	DRM_BUS_FLAG_DE_LOW = BIT(0),
+	DRM_BUS_FLAG_DE_HIGH = BIT(1),
+	DRM_BUS_FLAG_PIXDATA_POSEDGE = BIT(2),
+	DRM_BUS_FLAG_PIXDATA_NEGEDGE = BIT(3),
+	DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE = DRM_BUS_FLAG_PIXDATA_POSEDGE,
+	DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE = DRM_BUS_FLAG_PIXDATA_NEGEDGE,
+	DRM_BUS_FLAG_PIXDATA_SAMPLE_POSEDGE = DRM_BUS_FLAG_PIXDATA_NEGEDGE,
+	DRM_BUS_FLAG_PIXDATA_SAMPLE_NEGEDGE = DRM_BUS_FLAG_PIXDATA_POSEDGE,
+	DRM_BUS_FLAG_DATA_MSB_TO_LSB = BIT(4),
+	DRM_BUS_FLAG_DATA_LSB_TO_MSB = BIT(5),
+	DRM_BUS_FLAG_SYNC_POSEDGE = BIT(6),
+	DRM_BUS_FLAG_SYNC_NEGEDGE = BIT(7),
+	DRM_BUS_FLAG_SYNC_DRIVE_POSEDGE = DRM_BUS_FLAG_SYNC_POSEDGE,
+	DRM_BUS_FLAG_SYNC_DRIVE_NEGEDGE = DRM_BUS_FLAG_SYNC_NEGEDGE,
+	DRM_BUS_FLAG_SYNC_SAMPLE_POSEDGE = DRM_BUS_FLAG_SYNC_NEGEDGE,
+	DRM_BUS_FLAG_SYNC_SAMPLE_NEGEDGE = DRM_BUS_FLAG_SYNC_POSEDGE,
+};
+
+/**
  * struct drm_display_info - runtime data about the connected sink
  *
  * Describes a given display (e.g. CRT or flat panel) and its limitations. For
@@ -280,24 +342,10 @@
 	 */
 	unsigned int num_bus_formats;
 
-#define DRM_BUS_FLAG_DE_LOW		(1<<0)
-#define DRM_BUS_FLAG_DE_HIGH		(1<<1)
-/* drive data on pos. edge */
-#define DRM_BUS_FLAG_PIXDATA_POSEDGE	(1<<2)
-/* drive data on neg. edge */
-#define DRM_BUS_FLAG_PIXDATA_NEGEDGE	(1<<3)
-/* data is transmitted MSB to LSB on the bus */
-#define DRM_BUS_FLAG_DATA_MSB_TO_LSB	(1<<4)
-/* data is transmitted LSB to MSB on the bus */
-#define DRM_BUS_FLAG_DATA_LSB_TO_MSB	(1<<5)
-/* drive sync on pos. edge */
-#define DRM_BUS_FLAG_SYNC_POSEDGE	(1<<6)
-/* drive sync on neg. edge */
-#define DRM_BUS_FLAG_SYNC_NEGEDGE	(1<<7)
-
 	/**
 	 * @bus_flags: Additional information (like pixel signal polarity) for
-	 * the pixel data on the bus, using DRM_BUS_FLAGS\_ defines.
+	 * the pixel data on the bus, using &enum drm_bus_flags values
+	 * DRM_BUS_FLAGS\_.
 	 */
 	u32 bus_flags;
 
diff -Naur --no-dereference a/include/drm/drm_dp_helper.h b/include/drm/drm_dp_helper.h
--- a/include/drm/drm_dp_helper.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/drm/drm_dp_helper.h	2020-06-19 21:00:31.428227716 -0400
@@ -123,8 +123,9 @@
 # define DP_FRAMING_CHANGE_CAP		    (1 << 1)
 # define DP_DPCD_DISPLAY_CONTROL_CAPABLE     (1 << 3) /* edp v1.2 or higher */
 
-#define DP_TRAINING_AUX_RD_INTERVAL         0x00e   /* XXX 1.2? */
-# define DP_TRAINING_AUX_RD_MASK            0x7F    /* XXX 1.2? */
+#define DP_TRAINING_AUX_RD_INTERVAL             0x00e   /* XXX 1.2? */
+# define DP_TRAINING_AUX_RD_MASK                0x7F    /* DP 1.3 */
+# define DP_EXTENDED_RECEIVER_CAP_FIELD_PRESENT	(1 << 7) /* DP 1.3 */
 
 #define DP_ADAPTER_CAP			    0x00f   /* 1.2 */
 # define DP_FORCE_LOAD_SENSE_CAP	    (1 << 0)
diff -Naur --no-dereference a/include/drm/drm_mode_config.h b/include/drm/drm_mode_config.h
--- a/include/drm/drm_mode_config.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/drm/drm_mode_config.h	2020-06-19 21:00:31.428227716 -0400
@@ -506,6 +506,15 @@
 	 */
 	struct list_head property_list;
 
+	/**
+	 * @privobj_list:
+	 *
+	 * List of private objects linked with &drm_private_obj.head. This is
+	 * invariant over the lifetime of a device and hence doesn't need any
+	 * locks.
+	 */
+	struct list_head privobj_list;
+
 	int min_width, min_height;
 	int max_width, max_height;
 	const struct drm_mode_config_funcs *funcs;
diff -Naur --no-dereference a/include/dt-bindings/board/am335x-bbw-bbb-base.h b/include/dt-bindings/board/am335x-bbw-bbb-base.h
--- a/include/dt-bindings/board/am335x-bbw-bbb-base.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/dt-bindings/board/am335x-bbw-bbb-base.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,103 @@
+/*
+ * This header provides constants for bbw/bbb pinctrl bindings.
+ *
+ * Copyright (C) 2014 Robert Nelson <robertcnelson@gmail.com>
+ *
+ * Numbers Based on: https://github.com/derekmolloy/boneDeviceTree/tree/master/docs
+ */
+
+#ifndef _DT_BINDINGS_BOARD_AM335X_BBW_BBB_BASE_H
+#define _DT_BINDINGS_BOARD_AM335X_BBW_BBB_BASE_H
+
+#define BONE_P8_03 0x018
+#define BONE_P8_04 0x01C
+
+#define BONE_P8_05 0x008
+#define BONE_P8_06 0x00C
+#define BONE_P8_07 0x090
+#define BONE_P8_08 0x094
+
+#define BONE_P8_09 0x09C
+#define BONE_P8_10 0x098
+#define BONE_P8_11 0x034
+#define BONE_P8_12 0x030
+
+#define BONE_P8_13 0x024
+#define BONE_P8_14 0x028
+#define BONE_P8_15 0x03C
+#define BONE_P8_16 0x038
+
+#define BONE_P8_17 0x02C
+#define BONE_P8_18 0x08C
+#define BONE_P8_19 0x020
+#define BONE_P8_20 0x084
+
+#define BONE_P8_21 0x080
+#define BONE_P8_22 0x014
+#define BONE_P8_23 0x010
+#define BONE_P8_24 0x004
+
+#define BONE_P8_25 0x000
+#define BONE_P8_26 0x07C
+#define BONE_P8_27 0x0E0
+#define BONE_P8_28 0x0E8
+
+#define BONE_P8_29 0x0E4
+#define BONE_P8_30 0x0EC
+#define BONE_P8_31 0x0D8
+#define BONE_P8_32 0x0DC
+
+#define BONE_P8_33 0x0D4
+#define BONE_P8_34 0x0CC
+#define BONE_P8_35 0x0D0
+#define BONE_P8_36 0x0C8
+
+#define BONE_P8_37 0x0C0
+#define BONE_P8_38 0x0C4
+#define BONE_P8_39 0x0B8
+#define BONE_P8_40 0x0BC
+
+#define BONE_P8_41 0x0B0
+#define BONE_P8_42 0x0B4
+#define BONE_P8_43 0x0A8
+#define BONE_P8_44 0x0AC
+
+#define BONE_P8_45 0x0A0
+#define BONE_P8_46 0x0A4
+
+#define BONE_P9_11 0x070
+#define BONE_P9_12 0x078
+
+#define BONE_P9_13 0x074
+#define BONE_P9_14 0x048
+#define BONE_P9_15 0x040
+#define BONE_P9_16 0x04C
+
+#define BONE_P9_17 0x15C
+#define BONE_P9_18 0x158
+#define BONE_P9_19 0x17C
+#define BONE_P9_20 0x178
+
+#define BONE_P9_21 0x154
+#define BONE_P9_22 0x150
+#define BONE_P9_23 0x044
+#define BONE_P9_24 0x184
+
+#define BONE_P9_25 0x1AC
+#define BONE_P9_26 0x180
+#define BONE_P9_27 0x1A4
+#define BONE_P9_28 0x19C
+
+#define BONE_P9_29 0x194
+#define BONE_P9_30 0x198
+#define BONE_P9_31 0x190
+
+/* Shared P21 of P11 */
+#define BONE_P9_41A 0x1B4
+#define BONE_P9_41B 0x1A8
+
+/* Shared P22 of P11 */
+#define BONE_P9_42A 0x164
+#define BONE_P9_42B 0x1A0
+
+#endif
diff -Naur --no-dereference a/include/dt-bindings/bus/ti-sysc.h b/include/dt-bindings/bus/ti-sysc.h
--- a/include/dt-bindings/bus/ti-sysc.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/dt-bindings/bus/ti-sysc.h	2020-06-19 21:00:31.428227716 -0400
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /* TI sysc interconnect target module defines */
 
 /* Generic sysc found on omap2 and later, also known as type1 */
diff -Naur --no-dereference a/include/dt-bindings/clock/am3.h b/include/dt-bindings/clock/am3.h
--- a/include/dt-bindings/clock/am3.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/dt-bindings/clock/am3.h	2020-06-19 21:00:31.428227716 -0400
@@ -1,14 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
 /*
  * Copyright 2017 Texas Instruments, Inc.
- *
- * This software is licensed under the terms of the GNU General Public
- * License version 2, as published by the Free Software Foundation, and
- * may be copied, distributed, and modified under those terms.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 #ifndef __DT_BINDINGS_CLK_AM3_H
 #define __DT_BINDINGS_CLK_AM3_H
@@ -16,6 +8,8 @@
 #define AM3_CLKCTRL_OFFSET	0x0
 #define AM3_CLKCTRL_INDEX(offset)	((offset) - AM3_CLKCTRL_OFFSET)
 
+/* XXX: Compatibility part begin, remove this once compatibility support is no longer needed */
+
 /* l4_per clocks */
 #define AM3_L4_PER_CLKCTRL_OFFSET	0x14
 #define AM3_L4_PER_CLKCTRL_INDEX(offset)	((offset) - AM3_L4_PER_CLKCTRL_OFFSET)
diff -Naur --no-dereference a/include/dt-bindings/clock/dra7.h b/include/dt-bindings/clock/dra7.h
--- a/include/dt-bindings/clock/dra7.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/dt-bindings/clock/dra7.h	2020-06-19 21:00:31.428227716 -0400
@@ -1,14 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
 /*
  * Copyright 2017 Texas Instruments, Inc.
- *
- * This software is licensed under the terms of the GNU General Public
- * License version 2, as published by the Free Software Foundation, and
- * may be copied, distributed, and modified under those terms.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 #ifndef __DT_BINDINGS_CLK_DRA7_H
 #define __DT_BINDINGS_CLK_DRA7_H
@@ -16,9 +8,18 @@
 #define DRA7_CLKCTRL_OFFSET	0x20
 #define DRA7_CLKCTRL_INDEX(offset)	((offset) - DRA7_CLKCTRL_OFFSET)
 
+/* XXX: Compatibility part begin, remove this once compatibility support is no longer needed */
+
 /* mpu clocks */
 #define DRA7_MPU_CLKCTRL	DRA7_CLKCTRL_INDEX(0x20)
 
+/* dsp clocks */
+#define DRA7_DSP1_CLKCTRL	DRA7_CLKCTRL_INDEX(0x20)
+#define DRA7_DSP2_CLKCTRL	DRA7_CLKCTRL_INDEX(0x20)
+
+/* ipu1 clocks */
+#define DRA7_IPU1_CLKCTRL	DRA7_CLKCTRL_INDEX(0x20)
+
 /* ipu clocks */
 #define DRA7_IPU_CLKCTRL_OFFSET	0x40
 #define DRA7_IPU_CLKCTRL_INDEX(offset)	((offset) - DRA7_IPU_CLKCTRL_OFFSET)
@@ -48,6 +49,9 @@
 #define DRA7_VCP1_CLKCTRL	DRA7_CLKCTRL_INDEX(0x88)
 #define DRA7_VCP2_CLKCTRL	DRA7_CLKCTRL_INDEX(0x90)
 
+/* ipu2 clocks */
+#define DRA7_IPU2_CLKCTRL	DRA7_CLKCTRL_INDEX(0x20)
+
 /* dma clocks */
 #define DRA7_DMA_SYSTEM_CLKCTRL	DRA7_CLKCTRL_INDEX(0x20)
 
@@ -103,6 +107,8 @@
 #define DRA7_L4PER_CLKCTRL_INDEX(offset)	((offset) - DRA7_L4PER_CLKCTRL_OFFSET)
 #define DRA7_L4_PER2_CLKCTRL	DRA7_L4PER_CLKCTRL_INDEX(0xc)
 #define DRA7_L4_PER3_CLKCTRL	DRA7_L4PER_CLKCTRL_INDEX(0x14)
+#define DRA7_PRUSS1_CLKCTRL	DRA7_L4PER_CLKCTRL_INDEX(0x18)
+#define DRA7_PRUSS2_CLKCTRL	DRA7_L4PER_CLKCTRL_INDEX(0x20)
 #define DRA7_TIMER10_CLKCTRL	DRA7_L4PER_CLKCTRL_INDEX(0x28)
 #define DRA7_TIMER11_CLKCTRL	DRA7_L4PER_CLKCTRL_INDEX(0x30)
 #define DRA7_TIMER2_CLKCTRL	DRA7_L4PER_CLKCTRL_INDEX(0x38)
diff -Naur --no-dereference a/include/dt-bindings/clock/omap5.h b/include/dt-bindings/clock/omap5.h
--- a/include/dt-bindings/clock/omap5.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/dt-bindings/clock/omap5.h	2020-06-19 21:00:31.428227716 -0400
@@ -1,14 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
 /*
  * Copyright 2017 Texas Instruments, Inc.
- *
- * This software is licensed under the terms of the GNU General Public
- * License version 2, as published by the Free Software Foundation, and
- * may be copied, distributed, and modified under those terms.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 #ifndef __DT_BINDINGS_CLK_OMAP5_H
 #define __DT_BINDINGS_CLK_OMAP5_H
diff -Naur --no-dereference a/include/dt-bindings/dma/k3-udma.h b/include/dt-bindings/dma/k3-udma.h
--- a/include/dt-bindings/dma/k3-udma.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/dt-bindings/dma/k3-udma.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,35 @@
+#ifndef __DT_TI_UDMA_H
+#define __DT_TI_UDMA_H
+
+#define UDMA_TR_MODE		0
+#define UDMA_PKT_MODE		1
+
+#define UDMA_DIR_TX		0
+#define UDMA_DIR_RX		1
+
+#define PSIL_STATIC_TR_NONE	0
+#define PSIL_STATIC_TR_XY	1
+#define PSIL_STATIC_TR_MCAN	2
+
+#define UDMA_PDMA_TR_XY(id)				\
+	ti,psil-config##id {				\
+		linux,udma-mode = <UDMA_TR_MODE>;	\
+		statictr-type = <PSIL_STATIC_TR_XY>;	\
+	}
+
+/* j721e have support for ACC32 and burst flags for McASP */
+#define UDMA_PDMA_J721E_MCASP(id)			\
+	ti,psil-config##id {				\
+		linux,udma-mode = <UDMA_TR_MODE>;	\
+		statictr-type = <PSIL_STATIC_TR_XY>;	\
+		enable-acc32;				\
+		enable-burst;				\
+	}
+
+#define UDMA_PDMA_PKT_XY(id)				\
+	ti,psil-config##id {				\
+		linux,udma-mode = <UDMA_PKT_MODE>;	\
+		statictr-type = <PSIL_STATIC_TR_XY>;	\
+	}
+
+#endif /* __DT_TI_UDMA_H */
diff -Naur --no-dereference a/include/dt-bindings/gpio/gpio.h b/include/dt-bindings/gpio/gpio.h
--- a/include/dt-bindings/gpio/gpio.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/dt-bindings/gpio/gpio.h	2020-06-19 21:00:31.428227716 -0400
@@ -33,4 +33,10 @@
 #define GPIO_PERSISTENT 0
 #define GPIO_TRANSITORY 8
 
+/* Bit 4 express pull up */
+#define GPIO_PULL_UP 16
+
+/* Bit 5 express pull down */
+#define GPIO_PULL_DOWN 32
+
 #endif
diff -Naur --no-dereference a/include/dt-bindings/interrupt-controller/arm-gic.h b/include/dt-bindings/interrupt-controller/arm-gic.h
--- a/include/dt-bindings/interrupt-controller/arm-gic.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/dt-bindings/interrupt-controller/arm-gic.h	2020-06-19 21:00:31.428227716 -0400
@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
 /*
  * This header provides constants for the ARM GIC.
  */
diff -Naur --no-dereference a/include/dt-bindings/interrupt-controller/irq.h b/include/dt-bindings/interrupt-controller/irq.h
--- a/include/dt-bindings/interrupt-controller/irq.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/dt-bindings/interrupt-controller/irq.h	2020-06-19 21:00:31.428227716 -0400
@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
 /*
  * This header provides constants for most IRQ bindings.
  *
diff -Naur --no-dereference a/include/dt-bindings/media/omap3-isp.h b/include/dt-bindings/media/omap3-isp.h
--- a/include/dt-bindings/media/omap3-isp.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/dt-bindings/media/omap3-isp.h	2020-06-19 21:00:31.428227716 -0400
@@ -1,16 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
 /*
  * include/dt-bindings/media/omap3-isp.h
  *
  * Copyright (C) 2015 Sakari Ailus
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
  */
 
 #ifndef __DT_BINDINGS_OMAP3_ISP_H__
diff -Naur --no-dereference a/include/dt-bindings/mux/mux-j721e-wiz.h b/include/dt-bindings/mux/mux-j721e-wiz.h
--- a/include/dt-bindings/mux/mux-j721e-wiz.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/dt-bindings/mux/mux-j721e-wiz.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,53 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * This header provides constants for J721E WIZ.
+ */
+
+#ifndef _DT_BINDINGS_J721E_WIZ
+#define _DT_BINDINGS_J721E_WIZ
+
+#define SERDES0_LANE0_QSGMII_LANE1	0x0
+#define SERDES0_LANE0_PCIE0_LANE0	0x1
+#define SERDES0_LANE0_USB3_0_SWAP	0x2
+
+#define SERDES0_LANE1_QSGMII_LANE2	0x0
+#define SERDES0_LANE1_PCIE0_LANE1	0x1
+#define SERDES0_LANE1_USB3_0		0x2
+
+#define SERDES1_LANE0_QSGMII_LANE3	0x0
+#define SERDES1_LANE0_PCIE1_LANE0	0x1
+#define SERDES1_LANE0_USB3_1_SWAP	0x2
+#define SERDES1_LANE0_SGMII_LANE0	0x3
+
+#define SERDES1_LANE1_QSGMII_LANE4	0x0
+#define SERDES1_LANE1_PCIE1_LANE1	0x1
+#define SERDES1_LANE1_USB3_1		0x2
+#define SERDES1_LANE1_SGMII_LANE1	0x3
+
+#define SERDES2_LANE0_PCIE2_LANE0	0x1
+#define SERDES2_LANE0_SGMII_LANE0	0x3
+#define SERDES2_LANE0_USB3_1_SWAP	0x2
+
+#define SERDES2_LANE1_PCIE2_LANE1	0x1
+#define SERDES2_LANE1_USB3_1		0x2
+#define SERDES2_LANE1_SGMII_LANE1	0x3
+
+#define SERDES3_LANE0_PCIE3_LANE0	0x1
+#define SERDES3_LANE0_USB3_0_SWAP	0x2
+
+#define SERDES3_LANE1_PCIE3_LANE1	0x1
+#define SERDES3_LANE1_USB3_0		0x2
+
+#define SERDES4_LANE0_EDP_LANE0		0x0
+#define SERDES4_LANE0_QSGMII_LANE5	0x2
+
+#define SERDES4_LANE1_EDP_LANE1		0x0
+#define SERDES4_LANE1_QSGMII_LANE6	0x2
+
+#define SERDES4_LANE2_EDP_LANE2		0x0
+#define SERDES4_LANE2_QSGMII_LANE7	0x2
+
+#define SERDES4_LANE3_EDP_LANE3		0x0
+#define SERDES4_LANE3_QSGMII_LANE8	0x2
+
+#endif /* _DT_BINDINGS_J721E_WIZ */
diff -Naur --no-dereference a/include/dt-bindings/pci/pci.h b/include/dt-bindings/pci/pci.h
--- a/include/dt-bindings/pci/pci.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/dt-bindings/pci/pci.h	2020-06-19 21:00:31.428227716 -0400
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * This header provides constants for PCI bindings.
+ */
+
+#ifndef _DT_BINDINGS_PCI_H
+#define _DT_BINDINGS_PCI_H
+
+#define PCI_MODE_RC		1
+#define PCI_MODE_EP		2
+
+#endif
diff -Naur --no-dereference a/include/dt-bindings/phy/phy-am654-serdes.h b/include/dt-bindings/phy/phy-am654-serdes.h
--- a/include/dt-bindings/phy/phy-am654-serdes.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/dt-bindings/phy/phy-am654-serdes.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * This header provides constants for AM654 SERDES.
+ */
+
+#ifndef _DT_BINDINGS_AM654_SERDES
+#define _DT_BINDINGS_AM654_SERDES
+
+#define AM654_SERDES_CMU_REFCLK	0
+#define AM654_SERDES_LO_REFCLK	1
+#define AM654_SERDES_RO_REFCLK	2
+
+#endif /* _DT_BINDINGS_AM654_SERDES */
diff -Naur --no-dereference a/include/dt-bindings/phy/phy.h b/include/dt-bindings/phy/phy.h
--- a/include/dt-bindings/phy/phy.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/dt-bindings/phy/phy.h	2020-06-19 21:00:31.432227716 -0400
@@ -16,5 +16,6 @@
 #define PHY_TYPE_USB2		3
 #define PHY_TYPE_USB3		4
 #define PHY_TYPE_UFS		5
+#define PHY_TYPE_DP		6
 
 #endif /* _DT_BINDINGS_PHY */
diff -Naur --no-dereference a/include/dt-bindings/pinctrl/am33xx.h b/include/dt-bindings/pinctrl/am33xx.h
--- a/include/dt-bindings/pinctrl/am33xx.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/dt-bindings/pinctrl/am33xx.h	2020-06-19 21:00:31.432227716 -0400
@@ -40,5 +40,133 @@
 #undef PIN_OFF_INPUT_PULLDOWN
 #undef PIN_OFF_WAKEUPENABLE
 
-#endif
+#define AM335X_PIN_OFFSET_MIN			0x0800U
+
+#define AM335X_PIN_GPMC_AD0			0x800
+#define AM335X_PIN_GPMC_AD1			0x804
+#define AM335X_PIN_GPMC_AD2			0x808
+#define AM335X_PIN_GPMC_AD3			0x80c
+#define AM335X_PIN_GPMC_AD4			0x810
+#define AM335X_PIN_GPMC_AD5			0x814
+#define AM335X_PIN_GPMC_AD6			0x818
+#define AM335X_PIN_GPMC_AD7			0x81c
+#define AM335X_PIN_GPMC_AD8			0x820
+#define AM335X_PIN_GPMC_AD9			0x824
+#define AM335X_PIN_GPMC_AD10			0x828
+#define AM335X_PIN_GPMC_AD11			0x82c
+#define AM335X_PIN_GPMC_AD12			0x830
+#define AM335X_PIN_GPMC_AD13			0x834
+#define AM335X_PIN_GPMC_AD14			0x838
+#define AM335X_PIN_GPMC_AD15			0x83c
+#define AM335X_PIN_GPMC_A0			0x840
+#define AM335X_PIN_GPMC_A1			0x844
+#define AM335X_PIN_GPMC_A2			0x848
+#define AM335X_PIN_GPMC_A3			0x84c
+#define AM335X_PIN_GPMC_A4			0x850
+#define AM335X_PIN_GPMC_A5			0x854
+#define AM335X_PIN_GPMC_A6			0x858
+#define AM335X_PIN_GPMC_A7			0x85c
+#define AM335X_PIN_GPMC_A8			0x860
+#define AM335X_PIN_GPMC_A9			0x864
+#define AM335X_PIN_GPMC_A10			0x868
+#define AM335X_PIN_GPMC_A11			0x86c
+#define AM335X_PIN_GPMC_WAIT0			0x870
+#define AM335X_PIN_GPMC_WPN			0x874
+#define AM335X_PIN_GPMC_BEN1			0x878
+#define AM335X_PIN_GPMC_CSN0			0x87c
+#define AM335X_PIN_GPMC_CSN1			0x880
+#define AM335X_PIN_GPMC_CSN2			0x884
+#define AM335X_PIN_GPMC_CSN3			0x888
+#define AM335X_PIN_GPMC_CLK			0x88c
+#define AM335X_PIN_GPMC_ADVN_ALE		0x890
+#define AM335X_PIN_GPMC_OEN_REN			0x894
+#define AM335X_PIN_GPMC_WEN			0x898
+#define AM335X_PIN_GPMC_BEN0_CLE		0x89c
+#define AM335X_PIN_LCD_DATA0			0x8a0
+#define AM335X_PIN_LCD_DATA1			0x8a4
+#define AM335X_PIN_LCD_DATA2			0x8a8
+#define AM335X_PIN_LCD_DATA3			0x8ac
+#define AM335X_PIN_LCD_DATA4			0x8b0
+#define AM335X_PIN_LCD_DATA5			0x8b4
+#define AM335X_PIN_LCD_DATA6			0x8b8
+#define AM335X_PIN_LCD_DATA7			0x8bc
+#define AM335X_PIN_LCD_DATA8			0x8c0
+#define AM335X_PIN_LCD_DATA9			0x8c4
+#define AM335X_PIN_LCD_DATA10			0x8c8
+#define AM335X_PIN_LCD_DATA11			0x8cc
+#define AM335X_PIN_LCD_DATA12			0x8d0
+#define AM335X_PIN_LCD_DATA13			0x8d4
+#define AM335X_PIN_LCD_DATA14			0x8d8
+#define AM335X_PIN_LCD_DATA15			0x8dc
+#define AM335X_PIN_LCD_VSYNC			0x8e0
+#define AM335X_PIN_LCD_HSYNC			0x8e4
+#define AM335X_PIN_LCD_PCLK			0x8e8
+#define AM335X_PIN_LCD_AC_BIAS_EN		0x8ec
+#define AM335X_PIN_MMC0_DAT3			0x8f0
+#define AM335X_PIN_MMC0_DAT2			0x8f4
+#define AM335X_PIN_MMC0_DAT1			0x8f8
+#define AM335X_PIN_MMC0_DAT0			0x8fc
+#define AM335X_PIN_MMC0_CLK			0x900
+#define AM335X_PIN_MMC0_CMD			0x904
+#define AM335X_PIN_MII1_COL			0x908
+#define AM335X_PIN_MII1_CRS			0x90c
+#define AM335X_PIN_MII1_RX_ER			0x910
+#define AM335X_PIN_MII1_TX_EN			0x914
+#define AM335X_PIN_MII1_RX_DV			0x918
+#define AM335X_PIN_MII1_TXD3			0x91c
+#define AM335X_PIN_MII1_TXD2			0x920
+#define AM335X_PIN_MII1_TXD1			0x924
+#define AM335X_PIN_MII1_TXD0			0x928
+#define AM335X_PIN_MII1_TX_CLK			0x92c
+#define AM335X_PIN_MII1_RX_CLK			0x930
+#define AM335X_PIN_MII1_RXD3			0x934
+#define AM335X_PIN_MII1_RXD2			0x938
+#define AM335X_PIN_MII1_RXD1			0x93c
+#define AM335X_PIN_MII1_RXD0			0x940
+#define AM335X_PIN_RMII1_REF_CLK		0x944
+#define AM335X_PIN_MDIO				0x948
+#define AM335X_PIN_MDC				0x94c
+#define AM335X_PIN_SPI0_SCLK			0x950
+#define AM335X_PIN_SPI0_D0			0x954
+#define AM335X_PIN_SPI0_D1			0x958
+#define AM335X_PIN_SPI0_CS0			0x95c
+#define AM335X_PIN_SPI0_CS1			0x960
+#define AM335X_PIN_ECAP0_IN_PWM0_OUT		0x964
+#define AM335X_PIN_UART0_CTSN			0x968
+#define AM335X_PIN_UART0_RTSN			0x96c
+#define AM335X_PIN_UART0_RXD			0x970
+#define AM335X_PIN_UART0_TXD			0x974
+#define AM335X_PIN_UART1_CTSN			0x978
+#define AM335X_PIN_UART1_RTSN			0x97c
+#define AM335X_PIN_UART1_RXD			0x980
+#define AM335X_PIN_UART1_TXD			0x984
+#define AM335X_PIN_I2C0_SDA			0x988
+#define AM335X_PIN_I2C0_SCL			0x98c
+#define AM335X_PIN_MCASP0_ACLKX			0x990
+#define AM335X_PIN_MCASP0_FSX			0x994
+#define AM335X_PIN_MCASP0_AXR0			0x998
+#define AM335X_PIN_MCASP0_AHCLKR		0x99c
+#define AM335X_PIN_MCASP0_ACLKR			0x9a0
+#define AM335X_PIN_MCASP0_FSR			0x9a4
+#define AM335X_PIN_MCASP0_AXR1			0x9a8
+#define AM335X_PIN_MCASP0_AHCLKX		0x9ac
+#define AM335X_PIN_XDMA_EVENT_INTR0		0x9b0
+#define AM335X_PIN_XDMA_EVENT_INTR1		0x9b4
+#define AM335X_PIN_WARMRSTN			0x9b8
+#define AM335X_PIN_NNMI				0x9c0
+#define AM335X_PIN_TMS				0x9d0
+#define AM335X_PIN_TDI				0x9d4
+#define AM335X_PIN_TDO				0x9d8
+#define AM335X_PIN_TCK				0x9dc
+#define AM335X_PIN_TRSTN			0x9e0
+#define AM335X_PIN_EMU0				0x9e4
+#define AM335X_PIN_EMU1				0x9e8
+#define AM335X_PIN_RTC_PWRONRSTN		0x9f8
+#define AM335X_PIN_PMIC_POWER_EN		0x9fc
+#define AM335X_PIN_EXT_WAKEUP			0xa00
+#define AM335X_PIN_USB0_DRVVBUS			0xa1c
+#define AM335X_PIN_USB1_DRVVBUS			0xa34
 
+#define AM335X_PIN_OFFSET_MAX			0x0a34U
+
+#endif
diff -Naur --no-dereference a/include/dt-bindings/pinctrl/dra.h b/include/dt-bindings/pinctrl/dra.h
--- a/include/dt-bindings/pinctrl/dra.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/dt-bindings/pinctrl/dra.h	2020-06-19 21:00:31.432227716 -0400
@@ -1,12 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
 /*
  * This header provides constants for DRA pinctrl bindings.
  *
  * Copyright (C) 2013 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Rajendra Nayak <rnayak@ti.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #ifndef _DT_BINDINGS_PINCTRL_DRA_H
@@ -76,5 +73,19 @@
 /* DRA7 IODELAY configuration parameters */
 #define A_DELAY_PS(val)			((val) & 0xffff)
 #define G_DELAY_PS(val)			((val) & 0xffff)
+
+/* DRA72 VIP MUX selection parameters */
+#define VIP_VIN3A		(0x0 << 4)
+#define VIP_VIN5A		(0x1 << 4)
+#define VIP_VIN6A		(0x2 << 4)
+
+#define VIP_VIN4B		(0x0 << 3)
+
+#define VIP_VIN2A		(0x0 << 1)
+#define VIP_VIN4A		(0x1 << 1)
+
+#define VIP_VIN3B		(0x0 << 0)
+#define VIP_VIN2B		(0x1 << 0)
+
 #endif
 
diff -Naur --no-dereference a/include/dt-bindings/pinctrl/k3.h b/include/dt-bindings/pinctrl/k3.h
--- a/include/dt-bindings/pinctrl/k3.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/dt-bindings/pinctrl/k3.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * This header provides constants for pinctrl bindings for TI's K3 SoC
+ * family.
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ */
+#ifndef _DT_BINDINGS_PINCTRL_TI_K3_H
+#define _DT_BINDINGS_PINCTRL_TI_K3_H
+
+#define PULLUDEN_SHIFT		(16)
+#define PULLTYPESEL_SHIFT	(17)
+#define RXACTIVE_SHIFT		(18)
+
+#define PULL_DISABLE		(1 << PULLUDEN_SHIFT)
+#define PULL_ENABLE		(0 << PULLUDEN_SHIFT)
+
+#define PULL_UP			(1 << PULLTYPESEL_SHIFT | PULL_ENABLE)
+#define PULL_DOWN		(0 << PULLTYPESEL_SHIFT | PULL_ENABLE)
+
+#define INPUT_EN		(1 << RXACTIVE_SHIFT)
+#define INPUT_DISABLE		(0 << RXACTIVE_SHIFT)
+
+/* Only these macros are expected be used directly in device tree files */
+#define PIN_OUTPUT		(INPUT_DISABLE | PULL_DISABLE)
+#define PIN_OUTPUT_PULLUP	(INPUT_DISABLE | PULL_UP)
+#define PIN_OUTPUT_PULLDOWN	(INPUT_DISABLE | PULL_DOWN)
+#define PIN_INPUT		(INPUT_EN | PULL_DISABLE)
+#define PIN_INPUT_PULLUP	(INPUT_EN | PULL_UP)
+#define PIN_INPUT_PULLDOWN	(INPUT_EN | PULL_DOWN)
+
+#define AM65X_IOPAD(pa, val, muxmode)		(((pa) & 0x1fff)) ((val) | (muxmode))
+#define AM65X_WKUP_IOPAD(pa, val, muxmode)	(((pa) & 0x1fff)) ((val) | (muxmode))
+
+#define J721E_IOPAD(pa, val, muxmode)		(((pa) & 0x1fff)) ((val) | (muxmode))
+#define J721E_WKUP_IOPAD(pa, val, muxmode)	(((pa) & 0x1fff)) ((val) | (muxmode))
+
+#endif
diff -Naur --no-dereference a/include/dt-bindings/pinctrl/omap.h b/include/dt-bindings/pinctrl/omap.h
--- a/include/dt-bindings/pinctrl/omap.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/dt-bindings/pinctrl/omap.h	2020-06-19 21:00:31.432227716 -0400
@@ -65,6 +65,7 @@
 #define DM814X_IOPAD(pa, val)		OMAP_IOPAD_OFFSET((pa), 0x0800) (val)
 #define DM816X_IOPAD(pa, val)		OMAP_IOPAD_OFFSET((pa), 0x0800) (val)
 #define AM33XX_IOPAD(pa, val)		OMAP_IOPAD_OFFSET((pa), 0x0800) (val)
+#define AM33XX_PADCONF(pa, dir, mux)	OMAP_IOPAD_OFFSET((pa), 0x0800) ((dir) | (mux))
 
 /*
  * Macros to allow using the offset from the padconf physical address
diff -Naur --no-dereference a/include/dt-bindings/soc/ti,sci_pm_domain.h b/include/dt-bindings/soc/ti,sci_pm_domain.h
--- a/include/dt-bindings/soc/ti,sci_pm_domain.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/dt-bindings/soc/ti,sci_pm_domain.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef __DT_BINDINGS_TI_SCI_PM_DOMAIN_H
+#define __DT_BINDINGS_TI_SCI_PM_DOMAIN_H
+
+#define TI_SCI_PD_EXCLUSIVE	1
+#define TI_SCI_PD_SHARED	0
+
+#endif /* __DT_BINDINGS_TI_SCI_PM_DOMAIN_H */
diff -Naur --no-dereference a/include/dt-bindings/sound/sii902x-audio.h b/include/dt-bindings/sound/sii902x-audio.h
--- a/include/dt-bindings/sound/sii902x-audio.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/dt-bindings/sound/sii902x-audio.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,11 @@
+#ifndef __DT_SII9022_AUDIO_H
+#define __DT_SII9022_AUDIO_H
+
+#define ENABLE_BIT			0x80
+#define CONNECT_SD0			0x00
+#define CONNECT_SD1			0x10
+#define CONNECT_SD2			0x20
+#define CONNECT_SD3			0x30
+#define LEFT_RIGHT_SWAP_BIT		0x04
+
+#endif /* __DT_SII9022_AUDIO_H */
diff -Naur --no-dereference a/include/dt-bindings/sound/ti-mcasp.h b/include/dt-bindings/sound/ti-mcasp.h
--- a/include/dt-bindings/sound/ti-mcasp.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/dt-bindings/sound/ti-mcasp.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,13 @@
+#ifndef _DT_BINDINGS_TI_MCASP_H
+#define _DT_BINDINGS_TI_MCASP_H
+
+/* Source of High-frequency transmit/receive clock */
+#define MCASP_CLK_HCLK_AHCLK		0 /* AHCLKX/R */
+#define MCASP_CLK_HCLK_AUXCLK		1 /* Internal functional clock */
+
+/* clock divider IDs */
+#define MCASP_CLKDIV_AUXCLK		0 /* HCLK divider from AUXCLK */
+#define MCASP_CLKDIV_BCLK		1 /* BCLK divider from HCLK */
+#define MCASP_CLKDIV_BCLK_FS_RATIO	2 /* to set BCLK FS ration */
+
+#endif /* _DT_BINDINGS_TI_MCASP_H */
diff -Naur --no-dereference a/include/dt-bindings/thermal/thermal.h b/include/dt-bindings/thermal/thermal.h
--- a/include/dt-bindings/thermal/thermal.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/dt-bindings/thermal/thermal.h	2020-06-19 21:00:31.432227716 -0400
@@ -1,10 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
 /*
  * This header provides constants for most thermal bindings.
  *
  * Copyright (C) 2013 Texas Instruments
  *	Eduardo Valentin <eduardo.valentin@ti.com>
- *
- * GPLv2 only
  */
 
 #ifndef _DT_BINDINGS_THERMAL_THERMAL_H
diff -Naur --no-dereference a/include/linux/bsg-lib.h b/include/linux/bsg-lib.h
--- a/include/linux/bsg-lib.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/bsg-lib.h	2020-06-19 21:00:31.432227716 -0400
@@ -72,7 +72,8 @@
 void bsg_job_done(struct bsg_job *job, int result,
 		  unsigned int reply_payload_rcv_len);
 struct request_queue *bsg_setup_queue(struct device *dev, const char *name,
-		bsg_job_fn *job_fn, int dd_job_size);
+		bsg_job_fn *job_fn, rq_timed_out_fn *timeout, int dd_job_size);
+void bsg_remove_queue(struct request_queue *q);
 void bsg_job_put(struct bsg_job *job);
 int __must_check bsg_job_get(struct bsg_job *job);
 
diff -Naur --no-dereference a/include/linux/clk/ti.h b/include/linux/clk/ti.h
--- a/include/linux/clk/ti.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/clk/ti.h	2020-06-19 21:00:31.432227716 -0400
@@ -159,6 +159,7 @@
 	const char		*clkdm_name;
 	struct clockdomain	*clkdm;
 	const struct clk_hw_omap_ops	*ops;
+	u32			context;
 };
 
 /*
@@ -290,9 +291,15 @@
 #define TI_CLK_DPLL4_DENY_REPROGRAM		BIT(1)
 #define TI_CLK_DISABLE_CLKDM_CONTROL		BIT(2)
 #define TI_CLK_ERRATA_I810			BIT(3)
+#define TI_CLK_DEVICE_TYPE_GP			BIT(4)
 
 void ti_clk_setup_features(struct ti_clk_features *features);
 const struct ti_clk_features *ti_clk_get_features(void);
+int omap3_noncore_dpll_save_context(struct clk_hw *hw);
+void omap3_noncore_dpll_restore_context(struct clk_hw *hw);
+
+int omap3_core_dpll_save_context(struct clk_hw *hw);
+void omap3_core_dpll_restore_context(struct clk_hw *hw);
 
 extern const struct clk_hw_omap_ops clkhwops_omap2xxx_dpll;
 
diff -Naur --no-dereference a/include/linux/clk.h b/include/linux/clk.h
--- a/include/linux/clk.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/clk.h	2020-06-19 21:00:31.432227716 -0400
@@ -349,6 +349,17 @@
 struct clk *devm_clk_get(struct device *dev, const char *id);
 
 /**
+ * devm_clk_get_optional - lookup and obtain a managed reference to an optional
+ *			   clock producer.
+ * @dev: device for clock "consumer"
+ * @id: clock consumer ID
+ *
+ * Behaves the same as devm_clk_get() except where there is no clock producer.
+ * In this case, instead of returning -ENOENT, the function returns NULL.
+ */
+struct clk *devm_clk_get_optional(struct device *dev, const char *id);
+
+/**
  * devm_get_clk_from_child - lookup and obtain a managed reference to a
  *			     clock producer from child node.
  * @dev: device for clock "consumer"
@@ -629,6 +640,23 @@
  */
 struct clk *clk_get_sys(const char *dev_id, const char *con_id);
 
+/**
+ * clk_save_context - save clock context for poweroff
+ *
+ * Saves the context of the clock register for powerstates in which the
+ * contents of the registers will be lost. Occurs deep within the suspend
+ * code so locking is not necessary.
+ */
+int clk_save_context(void);
+
+/**
+ * clk_restore_context - restore clock context after poweroff
+ *
+ * This occurs with all clocks enabled. Occurs deep within the resume code
+ * so locking is not necessary.
+ */
+void clk_restore_context(void);
+
 #else /* !CONFIG_HAVE_CLK */
 
 static inline struct clk *clk_get(struct device *dev, const char *id)
@@ -647,6 +675,12 @@
 	return NULL;
 }
 
+static inline struct clk *devm_clk_get_optional(struct device *dev,
+						const char *id)
+{
+	return NULL;
+}
+
 static inline int __must_check devm_clk_bulk_get(struct device *dev, int num_clks,
 						 struct clk_bulk_data *clks)
 {
@@ -728,6 +762,14 @@
 {
 	return NULL;
 }
+
+static inline int clk_save_context(void)
+{
+	return 0;
+}
+
+static inline void clk_restore_context(void) {}
+
 #endif
 
 /* clk_prepare_enable helps cases using clk_enable in non-atomic context. */
@@ -774,6 +816,25 @@
 	clk_bulk_unprepare(num_clks, clks);
 }
 
+/**
+ * clk_get_optional - lookup and obtain a reference to an optional clock
+ *		      producer.
+ * @dev: device for clock "consumer"
+ * @id: clock consumer ID
+ *
+ * Behaves the same as clk_get() except where there is no clock producer. In
+ * this case, instead of returning -ENOENT, the function returns NULL.
+ */
+static inline struct clk *clk_get_optional(struct device *dev, const char *id)
+{
+	struct clk *clk = clk_get(dev, id);
+
+	if (clk == ERR_PTR(-ENOENT))
+		return NULL;
+
+	return clk;
+}
+
 #if defined(CONFIG_OF) && defined(CONFIG_COMMON_CLK)
 struct clk *of_clk_get(struct device_node *np, int index);
 struct clk *of_clk_get_by_name(struct device_node *np, const char *name);
diff -Naur --no-dereference a/include/linux/clk-provider.h b/include/linux/clk-provider.h
--- a/include/linux/clk-provider.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/clk-provider.h	2020-06-19 21:00:31.432227716 -0400
@@ -119,6 +119,11 @@
  *		Called with enable_lock held.  This function must not
  *		sleep.
  *
+ * @save_context: Save the context of the clock in prepration for poweroff.
+ *
+ * @restore_context: Restore the context of the clock after a restoration
+ *		of power.
+ *
  * @recalc_rate	Recalculate the rate of this clock, by querying hardware. The
  *		parent rate is an input parameter.  It is up to the caller to
  *		ensure that the prepare_mutex is held across this call.
@@ -223,6 +228,8 @@
 	void		(*disable)(struct clk_hw *hw);
 	int		(*is_enabled)(struct clk_hw *hw);
 	void		(*disable_unused)(struct clk_hw *hw);
+	int		(*save_context)(struct clk_hw *hw);
+	void		(*restore_context)(struct clk_hw *hw);
 	unsigned long	(*recalc_rate)(struct clk_hw *hw,
 					unsigned long parent_rate);
 	long		(*round_rate)(struct clk_hw *hw, unsigned long rate,
@@ -1014,5 +1021,7 @@
 
 #endif	/* platform dependent I/O accessors */
 
+void clk_gate_restore_context(struct clk_hw *hw);
+
 #endif /* CONFIG_COMMON_CLK */
 #endif /* CLK_PROVIDER_H */
diff -Naur --no-dereference a/include/linux/compiler.h b/include/linux/compiler.h
--- a/include/linux/compiler.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/compiler.h	2020-06-19 21:00:31.432227716 -0400
@@ -228,21 +228,21 @@
 /*
  * Prevent the compiler from merging or refetching reads or writes. The
  * compiler is also forbidden from reordering successive instances of
- * READ_ONCE and WRITE_ONCE, but only when the compiler is aware of some
- * particular ordering. One way to make the compiler aware of ordering is to
- * put the two invocations of READ_ONCE or WRITE_ONCE in different C
- * statements.
- *
- * These two macros will also work on aggregate data types like structs or
- * unions. If the size of the accessed data type exceeds the word size of
- * the machine (e.g., 32 bits or 64 bits) READ_ONCE() and WRITE_ONCE() will
- * fall back to memcpy(). There's at least two memcpy()s: one for the
- * __builtin_memcpy() and then one for the macro doing the copy of variable
- * - '__u' allocated on the stack.
+ * READ_ONCE, WRITE_ONCE and ACCESS_ONCE (see below), but only when the
+ * compiler is aware of some particular ordering.  One way to make the
+ * compiler aware of ordering is to put the two invocations of READ_ONCE,
+ * WRITE_ONCE or ACCESS_ONCE() in different C statements.
+ *
+ * In contrast to ACCESS_ONCE these two macros will also work on aggregate
+ * data types like structs or unions. If the size of the accessed data
+ * type exceeds the word size of the machine (e.g., 32 bits or 64 bits)
+ * READ_ONCE() and WRITE_ONCE() will fall back to memcpy(). There's at
+ * least two memcpy()s: one for the __builtin_memcpy() and then one for
+ * the macro doing the copy of variable - '__u' allocated on the stack.
  *
  * Their two major use cases are: (1) Mediating communication between
  * process-level code and irq/NMI handlers, all running on the same CPU,
- * and (2) Ensuring that the compiler does not fold, spindle, or otherwise
+ * and (2) Ensuring that the compiler does not  fold, spindle, or otherwise
  * mutilate accesses that either do not require ordering or that interact
  * with an explicit memory barrier or atomic instruction that provides the
  * required ordering.
@@ -351,4 +351,29 @@
 	compiletime_assert(__native_word(t),				\
 		"Need native word sized stores/loads for atomicity.")
 
+/*
+ * Prevent the compiler from merging or refetching accesses.  The compiler
+ * is also forbidden from reordering successive instances of ACCESS_ONCE(),
+ * but only when the compiler is aware of some particular ordering.  One way
+ * to make the compiler aware of ordering is to put the two invocations of
+ * ACCESS_ONCE() in different C statements.
+ *
+ * ACCESS_ONCE will only work on scalar types. For union types, ACCESS_ONCE
+ * on a union member will work as long as the size of the member matches the
+ * size of the union and the size is smaller than word size.
+ *
+ * The major use cases of ACCESS_ONCE used to be (1) Mediating communication
+ * between process-level code and irq/NMI handlers, all running on the same CPU,
+ * and (2) Ensuring that the compiler does not  fold, spindle, or otherwise
+ * mutilate accesses that either do not require ordering or that interact
+ * with an explicit memory barrier or atomic instruction that provides the
+ * required ordering.
+ *
+ * If possible use READ_ONCE()/WRITE_ONCE() instead.
+ */
+#define __ACCESS_ONCE(x) ({ \
+	 __maybe_unused typeof(x) __var = (__force typeof(x)) 0; \
+	(volatile typeof(x) *)&(x); })
+#define ACCESS_ONCE(x) (*__ACCESS_ONCE(x))
+
 #endif /* __LINUX_COMPILER_H */
diff -Naur --no-dereference a/include/linux/dma/k3-navss-udma.h b/include/linux/dma/k3-navss-udma.h
--- a/include/linux/dma/k3-navss-udma.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/linux/dma/k3-navss-udma.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,148 @@
+/*
+ * K3 NAVSS DMA glue interface
+ *
+ * Copyright (C) 2017 Texas Instruments Incorporated - http://www.ti.com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+
+#ifndef K3_NAVSS_UDMA_H_
+#define K3_NAVSS_UDMA_H_
+
+#include <linux/soc/ti/k3-ringacc.h>
+
+struct k3_nav_udmax_tx_channel_cfg {
+	struct k3_ring_cfg tx_cfg;
+	struct k3_ring_cfg txcq_cfg;
+
+	bool tx_pause_on_err;
+	bool tx_filt_einfo;
+	bool tx_filt_pswords;
+	bool tx_supr_tdpkt;
+	u32  swdata_size;
+};
+
+struct k3_nav_udmax_tx_channel;
+
+struct k3_nav_udmax_tx_channel *k3_nav_udmax_request_tx_chn(struct device *dev,
+				       const char *name,
+				       struct k3_nav_udmax_tx_channel_cfg *cfg);
+
+void k3_nav_udmax_release_tx_chn(struct k3_nav_udmax_tx_channel *tx_chn);
+int k3_nav_udmax_push_tx_chn(struct k3_nav_udmax_tx_channel *tx_chn,
+			     struct cppi5_host_desc_t *desc_tx,
+			     dma_addr_t desc_dma);
+int k3_nav_udmax_pop_tx_chn(struct k3_nav_udmax_tx_channel *tx_chn,
+			    dma_addr_t *desc_dma);
+int k3_nav_udmax_enable_tx_chn(struct k3_nav_udmax_tx_channel *tx_chn);
+void k3_nav_udmax_disable_tx_chn(struct k3_nav_udmax_tx_channel *tx_chn);
+void k3_nav_udmax_tdown_tx_chn(struct k3_nav_udmax_tx_channel *tx_chn,
+			       bool sync);
+void k3_nav_udmax_reset_tx_chn(struct k3_nav_udmax_tx_channel *tx_chn,
+			void *data,
+			void (*cleanup)(void *data, dma_addr_t desc_dma));
+u32 k3_nav_udmax_tx_get_hdesc_size(struct k3_nav_udmax_tx_channel *tx_chn);
+u32 k3_nav_udmax_tx_get_txcq_id(struct k3_nav_udmax_tx_channel *tx_chn);
+int k3_nav_udmax_tx_get_irq(struct k3_nav_udmax_tx_channel *tx_chn,
+			    unsigned int *irq, u32 flags, bool share,
+			    struct k3_nav_udmax_tx_channel *tx_chn_share);
+void k3_nav_udmax_tx_put_irq(struct k3_nav_udmax_tx_channel *tx_chn);
+
+enum {
+	K3_NAV_UDMAX_SRC_TAG_LO_KEEP = 0,
+	K3_NAV_UDMAX_SRC_TAG_LO_USE_FLOW_REG = 1,
+	K3_NAV_UDMAX_SRC_TAG_LO_USE_REMOTE_FLOW_ID = 2,
+	K3_NAV_UDMAX_SRC_TAG_LO_USE_REMOTE_SRC_TAG = 4,
+};
+
+/**
+ * k3_nav_udmax_rx_flow_cfg - UDMA RX flow cfg
+ *
+ * @rx_cfg:		RX ring configuration
+ * @rxfdq_cfg:		RX free Host PD ring configuration
+ * @ring_rxq_id:	RX ring id (or -1 for any)
+ * @ring_rxfdq0_id:	RX free Host PD ring (FDQ) if (or -1 for any)
+ * @rx_error_handling:	Rx Error Handling Mode (0 - drop, 1 - re-try)
+ * @src_tag_lo_sel:	Rx Source Tag Low Byte Selector in Host PD
+ */
+struct k3_nav_udmax_rx_flow_cfg {
+	struct k3_ring_cfg rx_cfg;
+	struct k3_ring_cfg rxfdq_cfg;
+	int ring_rxq_id;
+	int ring_rxfdq0_id;
+	bool rx_error_handling;
+	int src_tag_lo_sel;
+};
+
+/**
+ * k3_nav_udmax_rx_channel_cfg - UDMA RX channel cfg
+ *
+ * @psdata_size:	SW Data is present in Host PD of @swdata_size bytes
+ * @flow_id_base:	first flow_id used by channel.
+ *			if @flow_id_base = -1 - range of GP rflows will be
+ *			allocated dynamically.
+ * @flow_id_num:	number of RX flows used by channel
+ * @flow_id_use_rxchan_id:	use RX channel id as flow id,
+ *				used only if @flow_id_num = 1
+ * @remote		indication that RX channel is remote - some remote CPU
+ *			core owns and control the RX channel. Linux Host only
+ *			allowed to attach and configure RX Flow within RX
+ *			channel. if set - not RX channel operation will be
+ *			performed by K3 NAVSS DMA glue interface.
+ * @def_flow_cfg	default RX flow configuration,
+ *			used only if @flow_id_num = 1
+ */
+struct k3_nav_udmax_rx_channel_cfg {
+	u32  swdata_size;
+	int  flow_id_base;
+	int  flow_id_num;
+	bool flow_id_use_rxchan_id;
+	bool remote;
+
+	struct k3_nav_udmax_rx_flow_cfg *def_flow_cfg;
+};
+
+struct k3_nav_udmax_rx_channel;
+
+struct k3_nav_udmax_rx_channel *k3_nav_udmax_request_rx_chn(struct device *dev,
+				       const char *name,
+				       struct k3_nav_udmax_rx_channel_cfg *cfg);
+
+void k3_nav_udmax_release_rx_chn(struct k3_nav_udmax_rx_channel *rx_chn);
+int k3_nav_udmax_enable_rx_chn(struct k3_nav_udmax_rx_channel *rx_chn);
+void k3_nav_udmax_disable_rx_chn(struct k3_nav_udmax_rx_channel *rx_chn);
+void k3_nav_udmax_tdown_rx_chn(struct k3_nav_udmax_rx_channel *rx_chn,
+			       bool sync);
+int k3_nav_udmax_push_rx_chn(struct k3_nav_udmax_rx_channel *rx_chn,
+			     u32 flow_num, struct cppi5_host_desc_t *desc_tx,
+			     dma_addr_t desc_dma);
+int k3_nav_udmax_pop_rx_chn(struct k3_nav_udmax_rx_channel *rx_chn,
+			    u32 flow_num, dma_addr_t *desc_dma);
+int k3_nav_udmax_rx_flow_init(struct k3_nav_udmax_rx_channel *rx_chn,
+			      u32 flow_idx,
+			      struct k3_nav_udmax_rx_flow_cfg *flow_cfg);
+u32 k3_nav_udmax_rx_flow_get_fdq_id(struct k3_nav_udmax_rx_channel *rx_chn,
+				    u32 flow_idx);
+u32 k3_nav_udmax_rx_get_flow_id_base(struct k3_nav_udmax_rx_channel *rx_chn);
+int k3_nav_udmax_rx_get_irq(struct k3_nav_udmax_rx_channel *rx_chn,
+			    u32 flow_num, unsigned int *irq, u32 flags,
+			    bool share, u32 flow_num_share);
+void k3_nav_udmax_rx_put_irq(struct k3_nav_udmax_rx_channel *rx_chn,
+			     u32 flow_num);
+void k3_nav_udmax_reset_rx_chn(struct k3_nav_udmax_rx_channel *rx_chn,
+			       u32 flow_num, void *data,
+			       void (*cleanup)(void *data, dma_addr_t desc_dma),
+			       bool skip_fdq);
+int k3_nav_udmax_rx_flow_enable(struct k3_nav_udmax_rx_channel *rx_chn,
+				u32 flow_idx);
+int k3_nav_udmax_rx_flow_disable(struct k3_nav_udmax_rx_channel *rx_chn,
+				 u32 flow_idx);
+
+#endif /* K3_NAVSS_UDMA_H_ */
diff -Naur --no-dereference a/include/linux/dma/ti-cppi5.h b/include/linux/dma/ti-cppi5.h
--- a/include/linux/dma/ti-cppi5.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/linux/dma/ti-cppi5.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,995 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * CPPI5 descriptors interface
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
+ */
+
+#ifndef __TI_CPPI5_H__
+#define __TI_CPPI5_H__
+
+#include <linux/bitops.h>
+
+/**
+ * Descriptor header, present in all types of descriptors
+ */
+struct cppi5_desc_hdr_t {
+	u32 pkt_info0;	/* Packet info word 0 (n/a in Buffer desc) */
+	u32 pkt_info1;	/* Packet info word 1 (n/a in Buffer desc) */
+	u32 pkt_info2;	/* Packet info word 2 Buffer reclamation info */
+	u32 src_dst_tag; /* Packet info word 3 (n/a in Buffer desc) */
+} __packed;
+
+/**
+ * Host-mode packet and buffer descriptor definition
+ */
+struct cppi5_host_desc_t {
+	struct cppi5_desc_hdr_t hdr;
+	u64 next_desc;	/* w4/5: Linking word */
+	u64 buf_ptr;	/* w6/7: Buffer pointer */
+	u32 buf_info1;	/* w8: Buffer valid data length */
+	u32 org_buf_len; /* w9: Original buffer length */
+	u64 org_buf_ptr; /* w10/11: Original buffer pointer */
+	u32 epib[0];	/* Extended Packet Info Data (optional, 4 words) */
+	/*
+	 * Protocol Specific Data (optional, 0-128 bytes in multiples of 4),
+	 * and/or Other Software Data (0-N bytes, optional)
+	 */
+} __packed;
+
+#define CPPI5_DESC_MIN_ALIGN			(16U)
+
+#define CPPI5_INFO0_HDESC_EPIB_SIZE		(16U)
+#define CPPI5_INFO0_HDESC_PSDATA_MAX_SIZE	(128U)
+
+#define CPPI5_INFO0_HDESC_TYPE_SHIFT		(30U)
+#define CPPI5_INFO0_HDESC_TYPE_MASK		GENMASK(31, 30)
+#define   CPPI5_INFO0_DESC_TYPE_VAL_HOST	(1U)
+#define   CPPI5_INFO0_DESC_TYPE_VAL_MONO	(2U)
+#define   CPPI5_INFO0_DESC_TYPE_VAL_TR		(3U)
+#define CPPI5_INFO0_HDESC_EPIB_PRESENT		BIT(29)
+/*
+ * Protocol Specific Words location:
+ * 0 - located in the descriptor,
+ * 1 = located in the SOP Buffer immediately prior to the data.
+ */
+#define CPPI5_INFO0_HDESC_PSINFO_LOCATION	BIT(28)
+#define CPPI5_INFO0_HDESC_PSINFO_SIZE_SHIFT	(22U)
+#define CPPI5_INFO0_HDESC_PSINFO_SIZE_MASK	GENMASK(27, 22)
+#define CPPI5_INFO0_HDESC_PKTLEN_SHIFT		(0)
+#define CPPI5_INFO0_HDESC_PKTLEN_MASK		GENMASK(21, 0)
+
+#define CPPI5_INFO1_DESC_PKTERROR_SHIFT		(28U)
+#define CPPI5_INFO1_DESC_PKTERROR_MASK		GENMASK(31, 28)
+#define CPPI5_INFO1_HDESC_PSFLGS_SHIFT		(24U)
+#define CPPI5_INFO1_HDESC_PSFLGS_MASK		GENMASK(27, 24)
+#define CPPI5_INFO1_DESC_PKTID_SHIFT		(14U)
+#define CPPI5_INFO1_DESC_PKTID_MASK		GENMASK(23, 14)
+#define CPPI5_INFO1_DESC_FLOWID_SHIFT		(0)
+#define CPPI5_INFO1_DESC_FLOWID_MASK		GENMASK(13, 0)
+
+#define CPPI5_INFO2_HDESC_PKTTYPE_SHIFT		(27U)
+#define CPPI5_INFO2_HDESC_PKTTYPE_MASK		GENMASK(31, 27)
+/* Return Policy: 0 - Entire packet 1 - Each buffer */
+#define CPPI5_INFO2_HDESC_RETPOLICY		BIT(18)
+/*
+ * Early Return:
+ * 0 = desc pointers should be returned after all reads have been completed
+ * 1 = desc pointers should be returned immediately upon fetching
+ * the descriptor and beginning to transfer data.
+ */
+#define CPPI5_INFO2_HDESC_EARLYRET		BIT(17)
+/*
+ * Return Push Policy:
+ * 0 = Descriptor must be returned to tail of queue
+ * 1 = Descriptor must be returned to head of queue
+ */
+#define CPPI5_INFO2_DESC_RETPUSHPOLICY		BIT(16)
+#define CPPI5_INFO2_DESC_RETQ_SHIFT		(0)
+#define CPPI5_INFO2_DESC_RETQ_MASK		GENMASK(15, 0)
+
+#define CPPI5_INFO3_DESC_SRCTAG_SHIFT		(16U)
+#define CPPI5_INFO3_DESC_SRCTAG_MASK		GENMASK(31, 16)
+#define CPPI5_INFO3_DESC_DSTTAG_SHIFT		(0)
+#define CPPI5_INFO3_DESC_DSTTAG_MASK		GENMASK(15, 0)
+
+#define CPPI5_BUFINFO1_HDESC_DATA_LEN_SHIFT	(0)
+#define CPPI5_BUFINFO1_HDESC_DATA_LEN_MASK	GENMASK(27, 0)
+
+#define CPPI5_OBUFINFO0_HDESC_BUF_LEN_SHIFT	(0)
+#define CPPI5_OBUFINFO0_HDESC_BUF_LEN_MASK	GENMASK(27, 0)
+
+/*
+ * Host Packet Descriptor Extended Packet Info Block
+ */
+struct cppi5_desc_epib_t {
+	u32 timestamp;	/* w0: application specific timestamp */
+	u32 sw_info0;	/* w1: Software Info 0 */
+	u32 sw_info1;	/* w2: Software Info 1 */
+	u32 sw_info2;	/* w3: Software Info 2 */
+};
+
+/**
+ * Monolithic-mode packet descriptor
+ */
+struct cppi5_monolithic_desc_t {
+	struct cppi5_desc_hdr_t hdr;
+	u32 epib[0];	/* Extended Packet Info Data (optional, 4 words) */
+	/*
+	 * Protocol Specific Data (optional, 0-128 bytes in multiples of 4),
+	 *  and/or Other Software Data (0-N bytes, optional)
+	 */
+};
+
+#define CPPI5_INFO2_MDESC_DATA_OFFSET_SHIFT	(18U)
+#define CPPI5_INFO2_MDESC_DATA_OFFSET_MASK	GENMASK(26, 18)
+
+/*
+ * Reload Enable:
+ * 0 = Finish the packet and place the descriptor back on the return queue
+ * 1 = Vector to the Reload Index and resume processing
+ */
+#define CPPI5_INFO0_TRDESC_RLDCNT_SHIFT		(20U)
+#define CPPI5_INFO0_TRDESC_RLDCNT_MASK		GENMASK(28, 20)
+#define CPPI5_INFO0_TRDESC_RLDCNT_MAX		(0x1ff)
+#define CPPI5_INFO0_TRDESC_RLDCNT_INFINITE	CPPI5_INFO0_TRDESC_RLDCNT_MAX
+#define CPPI5_INFO0_TRDESC_RLDIDX_SHIFT		(14U)
+#define CPPI5_INFO0_TRDESC_RLDIDX_MASK		GENMASK(19, 14)
+#define CPPI5_INFO0_TRDESC_RLDIDX_MAX		(0x3f)
+#define CPPI5_INFO0_TRDESC_LASTIDX_SHIFT	(0)
+#define CPPI5_INFO0_TRDESC_LASTIDX_MASK		GENMASK(13, 0)
+
+#define CPPI5_INFO1_TRDESC_RECSIZE_SHIFT	(24U)
+#define CPPI5_INFO1_TRDESC_RECSIZE_MASK		GENMASK(26, 24)
+#define   CPPI5_INFO1_TRDESC_RECSIZE_VAL_16B	(0)
+#define   CPPI5_INFO1_TRDESC_RECSIZE_VAL_32B	(1U)
+#define   CPPI5_INFO1_TRDESC_RECSIZE_VAL_64B	(2U)
+#define   CPPI5_INFO1_TRDESC_RECSIZE_VAL_128B	(3U)
+
+static inline void cppi5_desc_dump(void *desc, u32 size)
+{
+	print_hex_dump(KERN_ERR, "dump udmap_desc: ", DUMP_PREFIX_NONE,
+		       32, 4, desc, size, false);
+}
+
+/**
+ * cppi5_desc_get_type - get descriptor type
+ * @desc_hdr: packet descriptor/TR header
+ *
+ * Returns descriptor type:
+ * CPPI5_INFO0_DESC_TYPE_VAL_HOST
+ * CPPI5_INFO0_DESC_TYPE_VAL_MONO
+ * CPPI5_INFO0_DESC_TYPE_VAL_TR
+ */
+static inline u32 cppi5_desc_get_type(struct cppi5_desc_hdr_t *desc_hdr)
+{
+	WARN_ON(!desc_hdr);
+
+	return (desc_hdr->pkt_info0 & CPPI5_INFO0_HDESC_TYPE_MASK) >>
+		CPPI5_INFO0_HDESC_TYPE_SHIFT;
+}
+
+/**
+ * cppi5_desc_get_errflags - get Error Flags from Desc
+ * @desc_hdr: packet/TR descriptor header
+ *
+ * Returns Error Flags from Packet/TR Descriptor
+ */
+static inline u32 cppi5_desc_get_errflags(struct cppi5_desc_hdr_t *desc_hdr)
+{
+	WARN_ON(!desc_hdr);
+
+	return (desc_hdr->pkt_info1 & CPPI5_INFO1_DESC_PKTERROR_MASK) >>
+		CPPI5_INFO1_DESC_PKTERROR_SHIFT;
+}
+
+/**
+ * cppi5_desc_get_pktids - get Packet and Flow ids from Desc
+ * @desc_hdr: packet/TR descriptor header
+ * @pkt_id: Packet ID
+ * @flow_id: Flow ID
+ *
+ * Returns Packet and Flow ids from packet/TR descriptor
+ */
+static inline void cppi5_desc_get_pktids(struct cppi5_desc_hdr_t *desc_hdr,
+					 u32 *pkt_id, u32 *flow_id)
+{
+	WARN_ON(!desc_hdr);
+
+	*pkt_id = (desc_hdr->pkt_info1 & CPPI5_INFO1_DESC_PKTID_MASK) >>
+		   CPPI5_INFO1_DESC_PKTID_SHIFT;
+	*flow_id = (desc_hdr->pkt_info1 & CPPI5_INFO1_DESC_FLOWID_MASK) >>
+		    CPPI5_INFO1_DESC_FLOWID_SHIFT;
+}
+
+/**
+ * cppi5_desc_set_pktids - set Packet and Flow ids in Desc
+ * @desc_hdr: packet/TR descriptor header
+ * @pkt_id: Packet ID
+ * @flow_id: Flow ID
+ */
+static inline void cppi5_desc_set_pktids(struct cppi5_desc_hdr_t *desc_hdr,
+					 u32 pkt_id, u32 flow_id)
+{
+	WARN_ON(!desc_hdr);
+
+	desc_hdr->pkt_info1 |= (pkt_id << CPPI5_INFO1_DESC_PKTID_SHIFT) &
+				CPPI5_INFO1_DESC_PKTID_MASK;
+	desc_hdr->pkt_info1 |= (flow_id << CPPI5_INFO1_DESC_FLOWID_SHIFT) &
+				CPPI5_INFO1_DESC_FLOWID_MASK;
+}
+
+/**
+ * cppi5_desc_set_retpolicy - set Packet Return Policy in Desc
+ * @desc_hdr: packet/TR descriptor header
+ * @flags: fags, supported values
+ *  CPPI5_INFO2_HDESC_RETPOLICY
+ *  CPPI5_INFO2_HDESC_EARLYRET
+ *  CPPI5_INFO2_DESC_RETPUSHPOLICY
+ * @return_ring_id: Packet Return Queue/Ring id, value 0xFFFF reserved
+ */
+static inline void cppi5_desc_set_retpolicy(struct cppi5_desc_hdr_t *desc_hdr,
+					    u32 flags, u32 return_ring_id)
+{
+	WARN_ON(!desc_hdr);
+
+	desc_hdr->pkt_info2 |= flags;
+	desc_hdr->pkt_info2 |= return_ring_id & CPPI5_INFO2_DESC_RETQ_MASK;
+}
+
+/**
+ * cppi5_desc_get_tags_ids - get Packet Src/Dst Tags from Desc
+ * @desc_hdr: packet/TR descriptor header
+ * @src_tag_id: Source Tag
+ * @dst_tag_id: Dest Tag
+ *
+ * Returns Packet Src/Dst Tags from packet/TR descriptor
+ */
+static inline void cppi5_desc_get_tags_ids(struct cppi5_desc_hdr_t *desc_hdr,
+					   u32 *src_tag_id, u32 *dst_tag_id)
+{
+	WARN_ON(!desc_hdr);
+
+	if (src_tag_id)
+		*src_tag_id = (desc_hdr->src_dst_tag &
+			      CPPI5_INFO3_DESC_SRCTAG_MASK) >>
+			      CPPI5_INFO3_DESC_SRCTAG_SHIFT;
+	if (dst_tag_id)
+		*dst_tag_id = desc_hdr->src_dst_tag &
+			      CPPI5_INFO3_DESC_DSTTAG_MASK;
+}
+
+/**
+ * cppi5_desc_set_tags_ids - set Packet Src/Dst Tags in HDesc
+ * @desc_hdr: packet/TR descriptor header
+ * @src_tag_id: Source Tag
+ * @dst_tag_id: Dest Tag
+ *
+ * Returns Packet Src/Dst Tags from packet/TR descriptor
+ */
+static inline void cppi5_desc_set_tags_ids(struct cppi5_desc_hdr_t *desc_hdr,
+					   u32 src_tag_id, u32 dst_tag_id)
+{
+	WARN_ON(!desc_hdr);
+
+	desc_hdr->src_dst_tag = (src_tag_id << CPPI5_INFO3_DESC_SRCTAG_SHIFT) &
+				CPPI5_INFO3_DESC_SRCTAG_MASK;
+	desc_hdr->src_dst_tag |= dst_tag_id & CPPI5_INFO3_DESC_DSTTAG_MASK;
+}
+
+/**
+ * cppi5_hdesc_calc_size - Calculate Host Packet Descriptor size
+ * @epib: is EPIB present
+ * @psdata_size: PSDATA size
+ * @sw_data_size: SWDATA size
+ *
+ * Returns required Host Packet Descriptor size
+ * 0 - if PSDATA > CPPI5_INFO0_HDESC_PSDATA_MAX_SIZE
+ */
+static inline u32 cppi5_hdesc_calc_size(bool epib, u32 psdata_size,
+					u32 sw_data_size)
+{
+	u32 desc_size;
+
+	if (psdata_size > CPPI5_INFO0_HDESC_PSDATA_MAX_SIZE)
+		return 0;
+	//TODO_GS: align
+	desc_size = sizeof(struct cppi5_host_desc_t) + psdata_size +
+		    sw_data_size;
+
+	if (epib)
+		desc_size += CPPI5_INFO0_HDESC_EPIB_SIZE;
+
+	return ALIGN(desc_size, CPPI5_DESC_MIN_ALIGN);
+}
+
+/**
+ * cppi5_hdesc_init - Init Host Packet Descriptor size
+ * @desc: Host packet descriptor
+ * @flags: supported values
+ *	CPPI5_INFO0_HDESC_EPIB_PRESENT
+ *	CPPI5_INFO0_HDESC_PSINFO_LOCATION
+ * @psdata_size: PSDATA size
+ *
+ * Returns required Host Packet Descriptor size
+ * 0 - if PSDATA > CPPI5_INFO0_HDESC_PSDATA_MAX_SIZE
+ */
+static inline void cppi5_hdesc_init(struct cppi5_host_desc_t *desc, u32 flags,
+				    u32 psdata_size)
+{
+	WARN_ON(!desc);
+	WARN_ON(psdata_size > CPPI5_INFO0_HDESC_PSDATA_MAX_SIZE);
+	WARN_ON(flags & ~(CPPI5_INFO0_HDESC_EPIB_PRESENT |
+			  CPPI5_INFO0_HDESC_PSINFO_LOCATION));
+
+	desc->hdr.pkt_info0 = (CPPI5_INFO0_DESC_TYPE_VAL_HOST <<
+			       CPPI5_INFO0_HDESC_TYPE_SHIFT) | (flags);
+	desc->hdr.pkt_info0 |= ((psdata_size >> 2) <<
+				CPPI5_INFO0_HDESC_PSINFO_SIZE_SHIFT) &
+				CPPI5_INFO0_HDESC_PSINFO_SIZE_MASK;
+	desc->next_desc = 0;
+}
+
+/**
+ * cppi5_hdesc_update_flags - Replace descriptor flags
+ * @desc: Host packet descriptor
+ * @flags: supported values
+ *	CPPI5_INFO0_HDESC_EPIB_PRESENT
+ *	CPPI5_INFO0_HDESC_PSINFO_LOCATION
+ */
+static inline void cppi5_hdesc_update_flags(struct cppi5_host_desc_t *desc,
+					    u32 flags)
+{
+	WARN_ON(!desc);
+	WARN_ON(flags & ~(CPPI5_INFO0_HDESC_EPIB_PRESENT |
+			  CPPI5_INFO0_HDESC_PSINFO_LOCATION));
+
+	desc->hdr.pkt_info0 &= ~(CPPI5_INFO0_HDESC_EPIB_PRESENT |
+				 CPPI5_INFO0_HDESC_PSINFO_LOCATION);
+	desc->hdr.pkt_info0 |= flags;
+}
+
+/**
+ * cppi5_hdesc_update_psdata_size - Replace PSdata size
+ * @desc: Host packet descriptor
+ * @psdata_size: PSDATA size
+ */
+static inline void cppi5_hdesc_update_psdata_size(
+				struct cppi5_host_desc_t *desc, u32 psdata_size)
+{
+	WARN_ON(!desc);
+	WARN_ON(psdata_size > CPPI5_INFO0_HDESC_PSDATA_MAX_SIZE);
+
+	desc->hdr.pkt_info0 &= ~CPPI5_INFO0_HDESC_PSINFO_SIZE_MASK;
+	desc->hdr.pkt_info0 |= ((psdata_size >> 2) <<
+				CPPI5_INFO0_HDESC_PSINFO_SIZE_SHIFT) &
+				CPPI5_INFO0_HDESC_PSINFO_SIZE_MASK;
+}
+
+/**
+ * cppi5_hdesc_get_psdata_size - get PSdata size in bytes
+ * @desc: Host packet descriptor
+ */
+static inline u32 cppi5_hdesc_get_psdata_size(struct cppi5_host_desc_t *desc)
+{
+	u32 psdata_size = 0;
+
+	WARN_ON(!desc);
+
+	if (!(desc->hdr.pkt_info0 & CPPI5_INFO0_HDESC_PSINFO_LOCATION))
+		psdata_size = (desc->hdr.pkt_info0 &
+			       CPPI5_INFO0_HDESC_PSINFO_SIZE_MASK) >>
+			       CPPI5_INFO0_HDESC_PSINFO_SIZE_SHIFT;
+
+	return (psdata_size << 2);
+}
+
+/**
+ * cppi5_hdesc_get_pktlen - get Packet Length from HDesc
+ * @desc: Host packet descriptor
+ *
+ * Returns Packet Length from Host Packet Descriptor
+ */
+static inline u32 cppi5_hdesc_get_pktlen(struct cppi5_host_desc_t *desc)
+{
+	WARN_ON(!desc);
+
+	return (desc->hdr.pkt_info0 & CPPI5_INFO0_HDESC_PKTLEN_MASK);
+}
+
+/**
+ * cppi5_hdesc_set_pktlen - set Packet Length in HDesc
+ * @desc: Host packet descriptor
+ */
+static inline void cppi5_hdesc_set_pktlen(struct cppi5_host_desc_t *desc,
+					  u32 pkt_len)
+{
+	WARN_ON(!desc);
+
+	desc->hdr.pkt_info0 |= (pkt_len & CPPI5_INFO0_HDESC_PKTLEN_MASK);
+}
+
+/**
+ * cppi5_hdesc_get_psflags - get Protocol Specific Flags from HDesc
+ * @desc: Host packet descriptor
+ *
+ * Returns Protocol Specific Flags from Host Packet Descriptor
+ */
+static inline u32 cppi5_hdesc_get_psflags(struct cppi5_host_desc_t *desc)
+{
+	WARN_ON(!desc);
+
+	return (desc->hdr.pkt_info1 & CPPI5_INFO1_HDESC_PSFLGS_MASK) >>
+		CPPI5_INFO1_HDESC_PSFLGS_SHIFT;
+}
+
+/**
+ * cppi5_hdesc_set_psflags - set Protocol Specific Flags in HDesc
+ * @desc: Host packet descriptor
+ */
+static inline void cppi5_hdesc_set_psflags(struct cppi5_host_desc_t *desc,
+					   u32 ps_flags)
+{
+	WARN_ON(!desc);
+
+	desc->hdr.pkt_info1 |= (ps_flags <<
+				CPPI5_INFO1_HDESC_PSFLGS_SHIFT) &
+				CPPI5_INFO1_HDESC_PSFLGS_MASK;
+}
+
+/**
+ * cppi5_hdesc_get_errflags - get Packet Type from HDesc
+ * @desc: Host packet descriptor
+ */
+static inline u32 cppi5_hdesc_get_pkttype(struct cppi5_host_desc_t *desc)
+{
+	WARN_ON(!desc);
+
+	return (desc->hdr.pkt_info2 & CPPI5_INFO2_HDESC_PKTTYPE_MASK) >>
+		CPPI5_INFO2_HDESC_PKTTYPE_SHIFT;
+}
+
+/**
+ * cppi5_hdesc_get_errflags - set Packet Type in HDesc
+ * @desc: Host packet descriptor
+ * @pkt_type: Packet Type
+ */
+static inline void cppi5_hdesc_set_pkttype(struct cppi5_host_desc_t *desc,
+					   u32 pkt_type)
+{
+	WARN_ON(!desc);
+	desc->hdr.pkt_info2 &= ~CPPI5_INFO2_HDESC_PKTTYPE_MASK;
+	desc->hdr.pkt_info2 |=
+			(pkt_type << CPPI5_INFO2_HDESC_PKTTYPE_SHIFT) &
+			 CPPI5_INFO2_HDESC_PKTTYPE_MASK;
+}
+
+/**
+ * cppi5_hdesc_attach_buf - attach buffer to HDesc
+ * @desc: Host packet descriptor
+ * @buf: Buffer physical address
+ * @buf_data_len: Buffer length
+ * @obuf: Original Buffer physical address
+ * @obuf_len: Original Buffer length
+ *
+ * Attaches buffer to Host Packet Descriptor
+ */
+static inline void cppi5_hdesc_attach_buf(struct cppi5_host_desc_t *desc,
+					  dma_addr_t buf, u32 buf_data_len,
+					  dma_addr_t obuf, u32 obuf_len)
+{
+	WARN_ON(!desc);
+	WARN_ON(!buf && !obuf);
+
+	desc->buf_ptr = buf;
+	desc->buf_info1 = buf_data_len & CPPI5_BUFINFO1_HDESC_DATA_LEN_MASK;
+	desc->org_buf_ptr = obuf;
+	desc->org_buf_len = obuf_len & CPPI5_OBUFINFO0_HDESC_BUF_LEN_MASK;
+}
+
+static inline void cppi5_hdesc_get_obuf(struct cppi5_host_desc_t *desc,
+					dma_addr_t *obuf, u32 *obuf_len)
+{
+	WARN_ON(!desc);
+	WARN_ON(!obuf);
+	WARN_ON(!obuf_len);
+
+	*obuf = desc->org_buf_ptr;
+	*obuf_len = desc->org_buf_len & CPPI5_OBUFINFO0_HDESC_BUF_LEN_MASK;
+}
+
+static inline void cppi5_hdesc_reset_to_original(struct cppi5_host_desc_t *desc)
+{
+	WARN_ON(!desc);
+
+	desc->buf_ptr = desc->org_buf_ptr;
+	desc->buf_info1 = desc->org_buf_len;
+}
+
+/**
+ * cppi5_hdesc_link_hbdesc - link Host Buffer Descriptor to HDesc
+ * @desc: Host Packet Descriptor
+ * @buf_desc: Host Buffer Descriptor physical address
+ *
+ * add and link Host Buffer Descriptor to HDesc
+ */
+static inline void cppi5_hdesc_link_hbdesc(struct cppi5_host_desc_t *desc,
+					   dma_addr_t hbuf_desc)
+{
+	WARN_ON(!desc);
+	WARN_ON(!hbuf_desc);
+
+	desc->next_desc = hbuf_desc;
+}
+
+static inline dma_addr_t cppi5_hdesc_get_next_hbdesc(
+				struct cppi5_host_desc_t *desc)
+{
+	WARN_ON(!desc);
+
+	return (dma_addr_t)desc->next_desc;
+}
+
+static inline void cppi5_hdesc_reset_hbdesc(struct cppi5_host_desc_t *desc)
+{
+	WARN_ON(!desc);
+
+	desc->hdr = (struct cppi5_desc_hdr_t) { 0 };
+	desc->next_desc = 0;
+}
+
+/**
+ * cppi5_hdesc_epib_present -  check if EPIB present
+ * @desc_hdr: packet descriptor/TR header
+ *
+ * Returns true if EPIB present in the packet
+ */
+static inline bool cppi5_hdesc_epib_present(struct cppi5_desc_hdr_t *desc_hdr)
+{
+	WARN_ON(!desc_hdr);
+	return !!(desc_hdr->pkt_info0 & CPPI5_INFO0_HDESC_EPIB_PRESENT);
+}
+
+/**
+ * cppi5_hdesc_get_psdata -  Get pointer on PSDATA
+ * @desc: Host packet descriptor
+ *
+ * Returns pointer on PSDATA in HDesc.
+ * NULL - if ps_data placed at the start of data buffer.
+ */
+static inline void *cppi5_hdesc_get_psdata(struct cppi5_host_desc_t *desc)
+{
+	u32 psdata_size;
+	void *psdata;
+
+	WARN_ON(!desc);
+
+	if (desc->hdr.pkt_info0 & CPPI5_INFO0_HDESC_PSINFO_LOCATION)
+		return NULL;
+
+	psdata_size = (desc->hdr.pkt_info0 &
+		       CPPI5_INFO0_HDESC_PSINFO_SIZE_MASK) >>
+		       CPPI5_INFO0_HDESC_PSINFO_SIZE_SHIFT;
+
+	if (!psdata_size)
+		return NULL;
+
+	psdata = &desc->epib;
+
+	if (cppi5_hdesc_epib_present(&desc->hdr))
+		psdata += CPPI5_INFO0_HDESC_EPIB_SIZE;
+
+	return psdata;
+}
+
+static inline u32 *cppi5_hdesc_get_psdata32(struct cppi5_host_desc_t *desc)
+{
+	return (u32 *)cppi5_hdesc_get_psdata(desc);
+}
+
+/**
+ * cppi5_hdesc_get_swdata -  Get pointer on swdata
+ * @desc: Host packet descriptor
+ *
+ * Returns pointer on SWDATA in HDesc.
+ * NOTE. It's caller responsibility to be sure hdesc actually has swdata.
+ */
+static inline void *cppi5_hdesc_get_swdata(struct cppi5_host_desc_t *desc)
+{
+	u32 psdata_size = 0;
+	void *swdata;
+
+	WARN_ON(!desc);
+
+	if (!(desc->hdr.pkt_info0 & CPPI5_INFO0_HDESC_PSINFO_LOCATION))
+		psdata_size = (desc->hdr.pkt_info0 &
+			       CPPI5_INFO0_HDESC_PSINFO_SIZE_MASK) >>
+			       CPPI5_INFO0_HDESC_PSINFO_SIZE_SHIFT;
+
+	swdata = &desc->epib;
+
+	if (cppi5_hdesc_epib_present(&desc->hdr))
+		swdata += CPPI5_INFO0_HDESC_EPIB_SIZE;
+
+	swdata += (psdata_size << 2);
+
+	return swdata;
+}
+
+/* ================================== TR ================================== */
+
+#define CPPI5_TR_TYPE_SHIFT			(0U)
+#define CPPI5_TR_TYPE_MASK			GENMASK(3, 0)
+#define CPPI5_TR_STATIC				BIT(4)
+#define CPPI5_TR_WAIT				BIT(5)
+#define CPPI5_TR_EVENT_SIZE_SHIFT		(6U)
+#define CPPI5_TR_EVENT_SIZE_MASK		GENMASK(7, 6)
+#define CPPI5_TR_TRIGGER0_SHIFT			(8U)
+#define CPPI5_TR_TRIGGER0_MASK			GENMASK(9, 8)
+#define CPPI5_TR_TRIGGER0_TYPE_SHIFT		(10U)
+#define CPPI5_TR_TRIGGER0_TYPE_MASK		GENMASK(11, 10)
+#define CPPI5_TR_TRIGGER1_SHIFT			(12U)
+#define CPPI5_TR_TRIGGER1_MASK			GENMASK(13, 12)
+#define CPPI5_TR_TRIGGER1_TYPE_SHIFT		(14U)
+#define CPPI5_TR_TRIGGER1_TYPE_MASK		GENMASK(15, 14)
+#define CPPI5_TR_CMD_ID_SHIFT			(16U)
+#define CPPI5_TR_CMD_ID_MASK			GENMASK(23, 16)
+#define CPPI5_TR_CSF_FLAGS_SHIFT		(24U)
+#define CPPI5_TR_CSF_FLAGS_MASK			GENMASK(31, 24)
+#define   CPPI5_TR_CSF_SA_INDIRECT		BIT(0)
+#define   CPPI5_TR_CSF_DA_INDIRECT		BIT(1)
+#define   CPPI5_TR_CSF_SUPR_EVT			BIT(2)
+#define   CPPI5_TR_CSF_EOL_ADV_SHIFT		(4U)
+#define   CPPI5_TR_CSF_EOL_ADV_MASK		GENMASK(6, 4)
+#define   CPPI5_TR_CSF_EOP			BIT(7)
+
+/* Udmap TR flags Type field specifies the type of TR. */
+enum cppi5_tr_types {
+	/* type0: One dimensional data move */
+	CPPI5_TR_TYPE0 = 0,
+	/* type1: Two dimensional data move */
+	CPPI5_TR_TYPE1,
+	/* type2: Three dimensional data move */
+	CPPI5_TR_TYPE2,
+	/* type3: Four dimensional data move */
+	CPPI5_TR_TYPE3,
+	/* type4: Four dimensional data move with data formatting */
+	CPPI5_TR_TYPE4,
+	/* type5: Four dimensional Cache Warm */
+	CPPI5_TR_TYPE5,
+	/* type6-7: Reserved */
+	/* type8: Four Dimensional Block Move */
+	CPPI5_TR_TYPE8 = 8,
+	/* type9: Four Dimensional Block Move with Repacking */
+	CPPI5_TR_TYPE9,
+	/* type10: Two Dimensional Block Move */
+	CPPI5_TR_TYPE10,
+	/* type11: Two Dimensional Block Move with Repacking */
+	CPPI5_TR_TYPE11,
+	/* type12-14: Reserved */
+	/* type15 Four Dimensional Block Move with Repacking and Indirection */
+	CPPI5_TR_TYPE15 = 15,
+	CPPI5_TR_TYPE_MAX
+};
+
+/*
+ * Udmap TR Flags EVENT_SIZE field specifies when an event is generated
+ * for each TR.
+ */
+enum cppi5_tr_event_size {
+	/* When TR is complete and all status for the TR has been received */
+	CPPI5_TR_EVENT_SIZE_COMPLETION,
+	/*
+	 * Type 0: when the last data transaction is sent for the TR;
+	 * Type 1-11: when ICNT1 is decremented
+	 */
+	CPPI5_TR_EVENT_SIZE_ICNT1_DEC,
+	/*
+	 * Type 0-1,10-11: when the last data transaction is sent for the TR;
+	 * All other types: when ICNT2 is decremented
+	 */
+	CPPI5_TR_EVENT_SIZE_ICNT2_DEC,
+	/*
+	 * Type 0-2,10-11: when the last data transaction is sent for the TR;
+	 * All other types: when ICNT3 is decremented
+	 */
+	CPPI5_TR_EVENT_SIZE_ICNT3_DEC,
+	CPPI5_TR_EVENT_SIZE_MAX
+};
+
+/*
+ * Udmap TR Flags TRIGGERx field specifies the type of trigger used to
+ * enable the TR to transfer data as specified by TRIGGERx_TYPE field.
+ */
+enum cppi5_tr_trigger {
+	CPPI5_TR_TRIGGER_NONE,		/* No Trigger */
+	CPPI5_TR_TRIGGER_GLOBAL0,		/* Global Trigger 0 */
+	CPPI5_TR_TRIGGER_GLOBAL1,		/* Global Trigger 1 */
+	CPPI5_TR_TRIGGER_LOCAL_EVENT,	/* Local Event */
+	CPPI5_TR_TRIGGER_MAX
+};
+
+/*
+ * Udmap TR Flags TRIGGERx_TYPE field specifies the type of data transfer
+ * that will be enabled by receiving a trigger as specified by TRIGGERx.
+ */
+enum cppi5_tr_trigger_type {
+	/* The second inner most loop (ICNT1) will be decremented by 1 */
+	CPPI5_TR_TRIGGER_TYPE_ICNT1_DEC,
+	/* The third inner most loop (ICNT2) will be decremented by 1 */
+	CPPI5_TR_TRIGGER_TYPE_ICNT2_DEC,
+	/* The outer most loop (ICNT3) will be decremented by 1 */
+	CPPI5_TR_TRIGGER_TYPE_ICNT3_DEC,
+	/* The entire TR will be allowed to complete */
+	CPPI5_TR_TRIGGER_TYPE_ALL,
+	CPPI5_TR_TRIGGER_TYPE_MAX
+};
+
+typedef u32 cppi5_tr_flags_t;
+
+/* Type 0 (One dimensional data move) TR (16 byte) */
+struct cppi5_tr_type0_t {
+	cppi5_tr_flags_t flags;
+	u16 icnt0;
+	u16 unused;
+	u64 addr;
+} __aligned(16) __packed;
+
+/* Type 1 (Two dimensional data move) TR (32 byte) */
+struct cppi5_tr_type1_t {
+	cppi5_tr_flags_t flags;
+	u16 icnt0;
+	u16 icnt1;
+	u64 addr;
+	s32 dim1;
+} __aligned(32) __packed;
+
+/* Type 2 (Three dimensional data move) TR (32 byte) */
+struct cppi5_tr_type2_t {
+	cppi5_tr_flags_t flags;
+	u16 icnt0;
+	u16 icnt1;
+	u64 addr;
+	s32 dim1;
+	u16 icnt2;
+	u16 unused;
+	s32 dim2;
+} __aligned(32) __packed;
+
+/* Type 3 (Four dimensional data move) TR (32 byte) */
+struct cppi5_tr_type3_t {
+	cppi5_tr_flags_t flags;
+	u16 icnt0;
+	u16 icnt1;
+	u64 addr;
+	s32 dim1;
+	u16 icnt2;
+	u16 icnt3;
+	s32 dim2;
+	s32 dim3;
+} __aligned(32) __packed;
+
+/*
+ * Type 15 (Four Dimensional Block Copy with Repacking and
+ * Indirection Support) TR (64 byte).
+ */
+struct cppi5_tr_type15_t {
+	cppi5_tr_flags_t flags;
+	u16 icnt0;
+	u16 icnt1;
+	u64 addr;
+	s32 dim1;
+	u16 icnt2;
+	u16 icnt3;
+	s32 dim2;
+	s32 dim3;
+	u32 _reserved;
+	s32 ddim1;
+	u64 daddr;
+	s32 ddim2;
+	s32 ddim3;
+	u16 dicnt0;
+	u16 dicnt1;
+	u16 dicnt2;
+	u16 dicnt3;
+} __aligned(64) __packed;
+
+struct cppi5_tr_resp_t {
+	u8 status;
+	u8 reserved;
+	u8 cmd_id;
+	u8 flags;
+} __packed;
+
+#define CPPI5_TR_RESPONSE_STATUS_TYPE_SHIFT	(0U)
+#define CPPI5_TR_RESPONSE_STATUS_TYPE_MASK	GENMASK(3, 0)
+#define CPPI5_TR_RESPONSE_STATUS_INFO_SHIFT	(4U)
+#define CPPI5_TR_RESPONSE_STATUS_INFO_MASK	GENMASK(7, 4)
+#define CPPI5_TR_RESPONSE_CMDID_SHIFT		(16U)
+#define CPPI5_TR_RESPONSE_CMDID_MASK		GENMASK(23, 16)
+#define CPPI5_TR_RESPONSE_CFG_SPECIFIC_SHIFT	(24U)
+#define CPPI5_TR_RESPONSE_CFG_SPECIFIC_MASK	GENMASK(31, 24)
+
+/*
+ * Udmap TR Response Status Type field is used to determine
+ * what type of status is being returned.
+ */
+enum cppi5_tr_resp_status_type {
+	CPPI5_TR_RESPONSE_STATUS_COMPLETE,		/* None */
+	CPPI5_TR_RESPONSE_STATUS_TRANSFER_ERR,		/* Transfer Error */
+	CPPI5_TR_RESPONSE_STATUS_ABORTED_ERR,		/* Aborted Error */
+	CPPI5_TR_RESPONSE_STATUS_SUBMISSION_ERR,	/* Submission Error */
+	CPPI5_TR_RESPONSE_STATUS_UNSUPPORTED_ERR,	/* Unsup. Feature */
+	CPPI5_TR_RESPONSE_STATUS_MAX
+};
+
+/*
+ * Udmap TR Response Status field values which corresponds
+ * CPPI5_TR_RESPONSE_STATUS_SUBMISSION_ERR
+ */
+enum cppi5_tr_resp_status_submission {
+	/* ICNT0 was 0 */
+	CPPI5_TR_RESPONSE_STATUS_SUBMISSION_ICNT0,
+	/* Channel FIFO was full when TR received */
+	CPPI5_TR_RESPONSE_STATUS_SUBMISSION_FIFO_FULL,
+	/* Channel is not owned by the submitter */
+	CPPI5_TR_RESPONSE_STATUS_SUBMISSION_OWN,
+	CPPI5_TR_RESPONSE_STATUS_SUBMISSION_MAX
+};
+
+/*
+ * Udmap TR Response Status field values which corresponds
+ * CPPI5_TR_RESPONSE_STATUS_UNSUPPORTED_ERR
+ */
+enum cppi5_tr_resp_status_unsupported {
+	/* TR Type not supported */
+	CPPI5_TR_RESPONSE_STATUS_UNSUPPORTED_TR_TYPE,
+	/* STATIC not supported */
+	CPPI5_TR_RESPONSE_STATUS_UNSUPPORTED_STATIC,
+	/* EOL not supported */
+	CPPI5_TR_RESPONSE_STATUS_UNSUPPORTED_EOL,
+	/* CONFIGURATION SPECIFIC not supported */
+	CPPI5_TR_RESPONSE_STATUS_UNSUPPORTED_CFG_SPECIFIC,
+	/* AMODE not supported */
+	CPPI5_TR_RESPONSE_STATUS_UNSUPPORTED_AMODE,
+	/* ELTYPE not supported */
+	CPPI5_TR_RESPONSE_STATUS_UNSUPPORTED_ELTYPE,
+	/* DFMT not supported */
+	CPPI5_TR_RESPONSE_STATUS_UNSUPPORTED_DFMT,
+	/* SECTR not supported */
+	CPPI5_TR_RESPONSE_STATUS_UNSUPPORTED_SECTR,
+	/* AMODE SPECIFIC field not supported */
+	CPPI5_TR_RESPONSE_STATUS_UNSUPPORTED_AMODE_SPECIFIC,
+	CPPI5_TR_RESPONSE_STATUS_UNSUPPORTED_MAX
+};
+
+/**
+ * cppi5_trdesc_calc_size - Calculate TR Descriptor size
+ * @tr_count: number of TR records
+ * @tr_size: Nominal size of TR record (max) [16, 32, 64, 128]
+ *
+ * Returns required TR Descriptor size
+ */
+static inline size_t cppi5_trdesc_calc_size(u32 tr_count, u32 tr_size)
+{
+	/*
+	 * The Size of a TR descriptor is:
+	 * 1 x tr_size : the first 16 bytes is used by the packet info block +
+	 * tr_count x tr_size : Transfer Request Records +
+	 * tr_count x sizeof(struct cppi5_tr_resp_t) : Transfer Response Records
+	 */
+	return tr_size * (tr_count + 1) +
+		sizeof(struct cppi5_tr_resp_t) * tr_count;
+}
+
+/**
+ * cppi5_trdesc_init - Init TR Descriptor
+ * @desc: TR Descriptor
+ * @tr_count: number of TR records
+ * @tr_size: Nominal size of TR record (max) [16, 32, 64, 128]
+ * @reload_idx: Absolute index to jump to on the 2nd and following passes
+ *		through the TR packet.
+ * @reload_count: Number of times to jump from last entry to reload_idx. 0x1ff
+ *		  indicates infinite looping.
+ *
+ * Init TR Descriptor
+ */
+static inline void cppi5_trdesc_init(struct cppi5_desc_hdr_t *desc_hdr,
+				     u32 tr_count, u32 tr_size, u32 reload_idx,
+				     u32 reload_count)
+{
+	WARN_ON(!desc_hdr);
+	WARN_ON(tr_count & ~CPPI5_INFO0_TRDESC_LASTIDX_MASK);
+	WARN_ON(reload_idx > CPPI5_INFO0_TRDESC_RLDIDX_MAX);
+	WARN_ON(reload_count > CPPI5_INFO0_TRDESC_RLDCNT_MAX);
+
+	desc_hdr->pkt_info0 = CPPI5_INFO0_DESC_TYPE_VAL_TR <<
+			      CPPI5_INFO0_HDESC_TYPE_SHIFT;
+	desc_hdr->pkt_info0 |= (reload_count << CPPI5_INFO0_TRDESC_RLDCNT_SHIFT) &
+			       CPPI5_INFO0_TRDESC_RLDCNT_MASK;
+	desc_hdr->pkt_info0 |= (reload_idx << CPPI5_INFO0_TRDESC_RLDIDX_SHIFT) &
+			       CPPI5_INFO0_TRDESC_RLDIDX_MASK;
+	desc_hdr->pkt_info0 |= (tr_count - 1) & CPPI5_INFO0_TRDESC_LASTIDX_MASK;
+
+	desc_hdr->pkt_info1 |= ((ffs(tr_size >> 4) - 1) <<
+				CPPI5_INFO1_TRDESC_RECSIZE_SHIFT) &
+				CPPI5_INFO1_TRDESC_RECSIZE_MASK;
+}
+
+/**
+ * cppi5_tr_init - Init TR record
+ * @flags: Pointer to the TR's flags
+ * @type: TR type
+ * @static_tr: TR is static
+ * @wait: Wait for TR completion before allow the next TR to start
+ * @event_size: output event generation cfg
+ * @cmd_id: TR identifier (application specifics)
+ *
+ * Init TR record
+ */
+static inline void cppi5_tr_init(cppi5_tr_flags_t *flags,
+				 enum cppi5_tr_types type, bool static_tr,
+				 bool wait, enum cppi5_tr_event_size event_size,
+				 u32 cmd_id)
+{
+	WARN_ON(!flags);
+
+	*flags = type;
+	*flags |= (event_size << CPPI5_TR_EVENT_SIZE_SHIFT) &
+		  CPPI5_TR_EVENT_SIZE_MASK;
+
+	*flags |= (cmd_id << CPPI5_TR_CMD_ID_SHIFT) &
+		  CPPI5_TR_CMD_ID_MASK;
+
+	if (static_tr && (type == CPPI5_TR_TYPE8 || type == CPPI5_TR_TYPE9))
+		*flags |= CPPI5_TR_STATIC;
+
+	if (wait)
+		*flags |= CPPI5_TR_WAIT;
+}
+
+/**
+ * cppi5_tr_set_trigger - Configure trigger0/1 and trigger0/1_type
+ * @flags: Pointer to the TR's flags
+ * @trigger0: trigger0 selection
+ * @trigger0_type: type of data transfer that will be enabled by trigger0
+ * @trigger1: trigger1 selection
+ * @trigger1_type: type of data transfer that will be enabled by trigger1
+ *
+ * Configure the triggers for the TR
+ */
+static inline void cppi5_tr_set_trigger(cppi5_tr_flags_t *flags,
+				enum cppi5_tr_trigger trigger0,
+				enum cppi5_tr_trigger_type trigger0_type,
+				enum cppi5_tr_trigger trigger1,
+				enum cppi5_tr_trigger_type trigger1_type)
+{
+	WARN_ON(!flags);
+
+	*flags |= (trigger0 << CPPI5_TR_TRIGGER0_SHIFT) &
+		  CPPI5_TR_TRIGGER0_MASK;
+	*flags |= (trigger0_type << CPPI5_TR_TRIGGER0_TYPE_SHIFT) &
+		  CPPI5_TR_TRIGGER0_TYPE_MASK;
+
+	*flags |= (trigger1 << CPPI5_TR_TRIGGER1_SHIFT) &
+		  CPPI5_TR_TRIGGER1_MASK;
+	*flags |= (trigger1_type << CPPI5_TR_TRIGGER1_TYPE_SHIFT) &
+		  CPPI5_TR_TRIGGER1_TYPE_MASK;
+}
+
+/**
+ * cppi5_tr_cflag_set - Update the Configuration specific flags
+ * @flags: Pointer to the TR's flags
+ * @csf: Configuration specific flags
+ *
+ * Set a bit in Configuration Specific Flags section of the TR flags.
+ */
+static inline void cppi5_tr_csf_set(cppi5_tr_flags_t *flags, u32 csf)
+{
+	WARN_ON(!flags);
+
+	*flags |= (csf << CPPI5_TR_CSF_FLAGS_SHIFT) &
+		  CPPI5_TR_CSF_FLAGS_MASK;
+}
+
+#endif /* __TI_CPPI5_H__ */
diff -Naur --no-dereference a/include/linux/dmaengine.h b/include/linux/dmaengine.h
--- a/include/linux/dmaengine.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/dmaengine.h	2020-06-19 21:00:31.432227716 -0400
@@ -231,6 +231,58 @@
  * @bytes_transferred: byte counter
  */
 
+/**
+ * enum dma_desc_metadata_mode - per descriptor metadata mode types supported
+ * @DESC_METADATA_CLIENT - the metadata buffer is allocated/provided by the
+ *  client driver and it is attached (via the dmaengine_desc_attach_metadata()
+ *  helper) to the descriptor.
+ *
+ * Client drivers interested to use this mode can follow:
+ * - DMA_MEM_TO_DEV / DEV_MEM_TO_MEM:
+ *   1. prepare the descriptor (dmaengine_prep_*)
+ *	construct the metadata in the client's buffer
+ *   2. use dmaengine_desc_attach_metadata() to attach the buffer to the
+ *	descriptor
+ *   3. submit the transfer
+ * - DMA_DEV_TO_MEM:
+ *   1. prepare the descriptor (dmaengine_prep_*)
+ *   2. use dmaengine_desc_attach_metadata() to attach the buffer to the
+ *	descriptor
+ *   3. submit the transfer
+ *   4. when the transfer is completed, the metadata should be available in the
+ *	attached buffer
+ *
+ * @DESC_METADATA_ENGINE - the metadata buffer is allocated/managed by the DMA
+ *  driver. The client driver can ask for the pointer, maximum size and the
+ *  currently used size of the metadata and can directly update or read it.
+ *  dmaengine_desc_get_metadata_ptr() and dmaengine_desc_set_metadata_len() is
+ *  provided as helper functions.
+ *
+ * Client drivers interested to use this mode can follow:
+ * - DMA_MEM_TO_DEV / DEV_MEM_TO_MEM:
+ *   1. prepare the descriptor (dmaengine_prep_*)
+ *   2. use dmaengine_desc_get_metadata_ptr() to get the pointer to the engine's
+ *	metadata area
+ *   3. update the metadata at the pointer
+ *   4. use dmaengine_desc_set_metadata_len()  to tell the DMA engine the amount
+ *	of data the client has placed into the metadata buffer
+ *   5. submit the transfer
+ * - DMA_DEV_TO_MEM:
+ *   1. prepare the descriptor (dmaengine_prep_*)
+ *   2. submit the transfer
+ *   3. on transfer completion, use dmaengine_desc_get_metadata_ptr() to get the
+ *	pointer to the engine's metadata are
+ *   4. Read out the metadate from the pointer
+ *
+ * Note: the two mode is not compatible and clients must use one mode for a
+ * descriptor.
+ */
+enum dma_desc_metadata_mode {
+	DESC_METADATA_NONE = 0,
+	DESC_METADATA_CLIENT = BIT(0),
+	DESC_METADATA_ENGINE = BIT(1),
+};
+
 struct dma_chan_percpu {
 	/* stats */
 	unsigned long memcpy_count;
@@ -487,6 +539,18 @@
 	dma_addr_t addr[0];
 };
 
+struct dma_async_tx_descriptor;
+
+struct dma_descriptor_metadata_ops {
+	int (*attach)(struct dma_async_tx_descriptor *desc, void *data,
+		      size_t len);
+
+	void *(*get_ptr)(struct dma_async_tx_descriptor *desc,
+			 size_t *payload_len, size_t *max_len);
+	int (*set_len)(struct dma_async_tx_descriptor *desc,
+		       size_t payload_len);
+};
+
 /**
  * struct dma_async_tx_descriptor - async transaction descriptor
  * ---dma generic offload fields---
@@ -500,6 +564,11 @@
  * descriptor pending. To be pushed on .issue_pending() call
  * @callback: routine to call after this operation is complete
  * @callback_param: general parameter to pass to the callback routine
+ * @desc_metadata_mode: core managed metadata mode to protect mixed use of
+ *	DESC_METADATA_CLIENT or DESC_METADATA_ENGINE. Otherwise
+ *	DESC_METADATA_NONE
+ * @metadata_ops: DMA driver provided metadata mode ops, need to be set by the
+ *	DMA driver if metadata mode is supported with the descriptor
  * ---async_tx api specific fields---
  * @next: at completion submit this descriptor
  * @parent: pointer to the next level up in the dependency chain
@@ -516,6 +585,8 @@
 	dma_async_tx_callback_result callback_result;
 	void *callback_param;
 	struct dmaengine_unmap_data *unmap;
+	enum dma_desc_metadata_mode desc_metadata_mode;
+	struct dma_descriptor_metadata_ops *metadata_ops;
 #ifdef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH
 	struct dma_async_tx_descriptor *next;
 	struct dma_async_tx_descriptor *parent;
@@ -623,11 +694,13 @@
  * @residue: the remaining number of bytes left to transmit
  *	on the selected transfer for states DMA_IN_PROGRESS and
  *	DMA_PAUSED if this is implemented in the driver, else 0
+ * @in_flight_bytes: amount of data in bytes cached by the DMA.
  */
 struct dma_tx_state {
 	dma_cookie_t last;
 	dma_cookie_t used;
 	u32 residue;
+	u32 in_flight_bytes;
 };
 
 /**
@@ -678,6 +751,7 @@
  * @global_node: list_head for global dma_device_list
  * @filter: information for device/slave to filter function/param mapping
  * @cap_mask: one or more dma_capability flags
+ * @desc_metadata_modes: supported metadata modes by the DMA device
  * @max_xor: maximum number of xor sources, 0 if no capability
  * @max_pq: maximum number of PQ sources and PQ-continue capability
  * @copy_align: alignment shift for memcpy operations
@@ -739,6 +813,7 @@
 	struct list_head global_node;
 	struct dma_filter filter;
 	dma_cap_mask_t  cap_mask;
+	enum dma_desc_metadata_mode desc_metadata_modes;
 	unsigned short max_xor;
 	unsigned short max_pq;
 	enum dmaengine_alignment copy_align;
@@ -914,6 +989,41 @@
 						    len, flags);
 }
 
+static inline bool dmaengine_is_metadata_mode_supported(struct dma_chan *chan,
+		enum dma_desc_metadata_mode mode)
+{
+	if (!chan)
+		return false;
+
+	return !!(chan->device->desc_metadata_modes & mode);
+}
+
+#ifdef CONFIG_DMA_ENGINE
+int dmaengine_desc_attach_metadata(struct dma_async_tx_descriptor *desc,
+				   void *data, size_t len);
+void *dmaengine_desc_get_metadata_ptr(struct dma_async_tx_descriptor *desc,
+				      size_t *payload_len, size_t *max_len);
+int dmaengine_desc_set_metadata_len(struct dma_async_tx_descriptor *desc,
+				    size_t payload_len);
+#else /* CONFIG_DMA_ENGINE */
+static inline int dmaengine_desc_attach_metadata(
+		struct dma_async_tx_descriptor *desc, void *data, size_t len)
+{
+	return -EINVAL;
+}
+static inline void *dmaengine_desc_get_metadata_ptr(
+		struct dma_async_tx_descriptor *desc, size_t *payload_len,
+		size_t *max_len)
+{
+	return NULL;
+}
+static inline int dmaengine_desc_set_metadata_len(
+		struct dma_async_tx_descriptor *desc, size_t payload_len)
+{
+	return -EINVAL;
+}
+#endif /* CONFIG_DMA_ENGINE */
+
 /**
  * dmaengine_terminate_all() - Terminate all active DMA transfers
  * @chan: The channel for which to terminate the transfers
@@ -1413,7 +1523,10 @@
 void dma_async_device_unregister(struct dma_device *device);
 void dma_run_dependencies(struct dma_async_tx_descriptor *tx);
 struct dma_chan *dma_get_slave_channel(struct dma_chan *chan);
-struct dma_chan *dma_get_any_slave_channel(struct dma_device *device);
+struct dma_chan *dmadev_get_slave_channel(struct dma_device *device,
+					  dma_filter_fn fn, void *fn_param);
+#define dma_get_any_slave_channel(device) \
+	dmadev_get_slave_channel(device, NULL, NULL)
 #define dma_request_channel(mask, x, y) __dma_request_channel(&(mask), x, y)
 #define dma_request_slave_channel_compat(mask, x, y, dev, name) \
 	__dma_request_slave_channel_compat(&(mask), x, y, dev, name)
diff -Naur --no-dereference a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h
--- a/include/linux/dma-mapping.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/dma-mapping.h	2020-06-19 21:00:31.432227716 -0400
@@ -161,7 +161,8 @@
  * Don't use them in device drivers.
  */
 int dma_alloc_from_dev_coherent(struct device *dev, ssize_t size,
-				       dma_addr_t *dma_handle, void **ret);
+				dma_addr_t *dma_handle, void **ret,
+				bool zero);
 int dma_release_from_dev_coherent(struct device *dev, int order, void *vaddr);
 
 int dma_mmap_from_dev_coherent(struct device *dev, struct vm_area_struct *vma,
@@ -173,7 +174,7 @@
 				  size_t size, int *ret);
 
 #else
-#define dma_alloc_from_dev_coherent(dev, size, handle, ret) (0)
+#define dma_alloc_from_dev_coherent(dev, size, handle, ret, zero) (0)
 #define dma_release_from_dev_coherent(dev, order, vaddr) (0)
 #define dma_mmap_from_dev_coherent(dev, vma, vaddr, order, ret) (0)
 
@@ -505,9 +506,9 @@
 #define arch_dma_alloc_attrs(dev)	(true)
 #endif
 
-static inline void *dma_alloc_attrs(struct device *dev, size_t size,
-				       dma_addr_t *dma_handle, gfp_t flag,
-				       unsigned long attrs)
+static inline void *dma_malloc_attrs(struct device *dev, size_t size,
+				     dma_addr_t *dma_handle, gfp_t flag,
+				     unsigned long attrs, bool zero)
 {
 	const struct dma_map_ops *ops = get_dma_ops(dev);
 	void *cpu_addr;
@@ -515,7 +516,7 @@
 	BUG_ON(!ops);
 	WARN_ON_ONCE(dev && !dev->coherent_dma_mask);
 
-	if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr))
+	if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr, zero))
 		return cpu_addr;
 
 	/* let the implementation decide on the zone to allocate from: */
@@ -531,6 +532,13 @@
 	return cpu_addr;
 }
 
+static inline void *dma_alloc_attrs(struct device *dev, size_t size,
+				    dma_addr_t *dma_handle, gfp_t flag,
+				    unsigned long attrs)
+{
+	return dma_malloc_attrs(dev, size, dma_handle, flag, attrs, true);
+}
+
 static inline void dma_free_attrs(struct device *dev, size_t size,
 				     void *cpu_addr, dma_addr_t dma_handle,
 				     unsigned long attrs)
@@ -563,6 +571,12 @@
 	return dma_alloc_attrs(dev, size, dma_handle, flag, 0);
 }
 
+static inline void *dma_malloc_coherent(struct device *dev, size_t size,
+					dma_addr_t *dma_handle, gfp_t flag)
+{
+	return dma_malloc_attrs(dev, size, dma_handle, flag, 0, false);
+}
+
 static inline void dma_free_coherent(struct device *dev, size_t size,
 		void *cpu_addr, dma_addr_t dma_handle)
 {
diff -Naur --no-dereference a/include/linux/ethtool.h b/include/linux/ethtool.h
--- a/include/linux/ethtool.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/ethtool.h	2020-06-19 21:00:31.432227716 -0400
@@ -181,6 +181,14 @@
 bool ethtool_convert_link_mode_to_legacy_u32(u32 *legacy_u32,
 				     const unsigned long *src);
 
+bool convert_legacy_settings_to_link_ksettings(
+	struct ethtool_link_ksettings *link_ksettings,
+	const struct ethtool_cmd *legacy_settings);
+
+bool convert_link_ksettings_to_legacy_settings(
+	struct ethtool_cmd *legacy_settings,
+	const struct ethtool_link_ksettings *link_ksettings);
+
 /**
  * struct ethtool_ops - optional netdev operations
  * @get_settings: DEPRECATED, use %get_link_ksettings/%set_link_ksettings
diff -Naur --no-dereference a/include/linux/fs.h b/include/linux/fs.h
--- a/include/linux/fs.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/fs.h	2020-06-19 21:00:31.432227716 -0400
@@ -1295,6 +1295,7 @@
 /* can be called from interrupts */
 extern void kill_fasync(struct fasync_struct **, int, int);
 
+extern int setfl(int fd, struct file *filp, unsigned long arg);
 extern void __f_setown(struct file *filp, struct pid *, enum pid_type, int force);
 extern int f_setown(struct file *filp, unsigned long arg, int force);
 extern void f_delown(struct file *filp);
@@ -1759,6 +1760,7 @@
 	ssize_t (*sendpage) (struct file *, struct page *, int, size_t, loff_t *, int);
 	unsigned long (*get_unmapped_area)(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
 	int (*check_flags)(int);
+	int (*setfl)(struct file *, unsigned long);
 	int (*flock) (struct file *, int, struct file_lock *);
 	ssize_t (*splice_write)(struct pipe_inode_info *, struct file *, loff_t *, size_t, unsigned int);
 	ssize_t (*splice_read)(struct file *, loff_t *, struct pipe_inode_info *, size_t, unsigned int);
@@ -1830,6 +1832,12 @@
 			      struct iovec *fast_pointer,
 			      struct iovec **ret_pointer);
 
+typedef ssize_t (*vfs_readf_t)(struct file *, char __user *, size_t, loff_t *);
+typedef ssize_t (*vfs_writef_t)(struct file *, const char __user *, size_t,
+				loff_t *);
+vfs_readf_t vfs_readf(struct file *file);
+vfs_writef_t vfs_writef(struct file *file);
+
 extern ssize_t __vfs_read(struct file *, char __user *, size_t, loff_t *);
 extern ssize_t vfs_read(struct file *, char __user *, size_t, loff_t *);
 extern ssize_t vfs_write(struct file *, const char __user *, size_t, loff_t *);
@@ -2255,6 +2263,7 @@
 extern void ihold(struct inode * inode);
 extern void iput(struct inode *);
 extern int generic_update_time(struct inode *, struct timespec64 *, int);
+extern int update_time(struct inode *, struct timespec64 *, int);
 
 /* /sys/fs */
 extern struct kobject *fs_kobj;
@@ -2542,6 +2551,7 @@
 	return false;
 }
 #endif
+extern int __sync_filesystem(struct super_block *, int);
 extern int sync_filesystem(struct super_block *);
 extern const struct file_operations def_blk_fops;
 extern const struct file_operations def_chr_fops;
diff -Naur --no-dereference a/include/linux/i2c-algo-bit.h b/include/linux/i2c-algo-bit.h
--- a/include/linux/i2c-algo-bit.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/i2c-algo-bit.h	2020-06-19 21:00:31.432227716 -0400
@@ -46,6 +46,7 @@
 				   minimum 5 us for standard-mode I2C and SMBus,
 				   maximum 50 us for SMBus */
 	int timeout;		/* in jiffies */
+	bool can_do_atomic;	/* callbacks don't sleep, we can be atomic */
 };
 
 int i2c_bit_add_bus(struct i2c_adapter *);
diff -Naur --no-dereference a/include/linux/i2c.h b/include/linux/i2c.h
--- a/include/linux/i2c.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/i2c.h	2020-06-19 21:00:31.432227716 -0400
@@ -499,9 +499,13 @@
  * @master_xfer: Issue a set of i2c transactions to the given I2C adapter
  *   defined by the msgs array, with num messages available to transfer via
  *   the adapter specified by adap.
+ * @master_xfer_atomic: same as @master_xfer. Yet, only using atomic context
+ *   so e.g. PMICs can be accessed very late before shutdown. Optional.
  * @smbus_xfer: Issue smbus transactions to the given I2C adapter. If this
  *   is not present, then the bus layer will try and convert the SMBus calls
  *   into I2C transfers instead.
+ * @smbus_xfer_atomic: same as @smbus_xfer. Yet, only using atomic context
+ *   so e.g. PMICs can be accessed very late before shutdown. Optional.
  * @functionality: Return the flags that this algorithm/adapter pair supports
  *   from the I2C_FUNC_* flags.
  * @reg_slave: Register given client to I2C slave mode of this adapter
@@ -512,9 +516,9 @@
  * be addressed using the same bus algorithms - i.e. bit-banging or the PCF8584
  * to name two of the most common.
  *
- * The return codes from the @master_xfer field should indicate the type of
- * error code that occurred during the transfer, as documented in the kernel
- * Documentation file Documentation/i2c/fault-codes.
+ * The return codes from the @master_xfer{_atomic} fields should indicate the
+ * type of error code that occurred during the transfer, as documented in the
+ * Kernel Documentation file Documentation/i2c/fault-codes.
  */
 struct i2c_algorithm {
 	/* If an adapter algorithm can't do I2C-level access, set master_xfer
@@ -525,9 +529,14 @@
 	   processed, or a negative value on error */
 	int (*master_xfer)(struct i2c_adapter *adap, struct i2c_msg *msgs,
 			   int num);
+	int (*master_xfer_atomic)(struct i2c_adapter *adap,
+				   struct i2c_msg *msgs, int num);
 	int (*smbus_xfer) (struct i2c_adapter *adap, u16 addr,
 			   unsigned short flags, char read_write,
 			   u8 command, int size, union i2c_smbus_data *data);
+	int (*smbus_xfer_atomic)(struct i2c_adapter *adap, u16 addr,
+				 unsigned short flags, char read_write,
+				 u8 command, int size, union i2c_smbus_data *data);
 
 	/* To determine what the adapter supports */
 	u32 (*functionality) (struct i2c_adapter *);
diff -Naur --no-dereference a/include/linux/ieee80211.h b/include/linux/ieee80211.h
--- a/include/linux/ieee80211.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/ieee80211.h	2020-06-19 21:00:31.432227716 -0400
@@ -2542,6 +2542,7 @@
 #define FILS_ERP_MAX_RRK_LEN		64
 
 #define PMK_MAX_LEN			64
+#define SAE_PASSWORD_MAX_LEN		128
 
 /* Public action codes (IEEE Std 802.11-2016, 9.6.8.1, Table 9-307) */
 enum ieee80211_pub_actioncode {
diff -Naur --no-dereference a/include/linux/irqchip/irq-ti-sci-inta.h b/include/linux/irqchip/irq-ti-sci-inta.h
--- a/include/linux/irqchip/irq-ti-sci-inta.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/linux/irqchip/irq-ti-sci-inta.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Texas Instruments' System Control Interface (TI-SCI) irqchip
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ *	Lokesh Vutla <lokeshvutla@ti.com>
+ */
+
+#ifndef __INCLUDE_LINUX_IRQCHIP_TI_SCI_INTA_H
+#define __INCLUDE_LINUX_IRQCHIP_TI_SCI_INTA_H
+
+#if IS_ENABLED(CONFIG_TI_SCI_INTA_IRQCHIP)
+int ti_sci_inta_register_event(struct device *dev, u16 src_id, u16 src_index,
+			       unsigned int virq, bool ack_needed, u32 flags);
+int ti_sci_inta_unregister_event(struct device *dev, u16 src_id, u16 src_index,
+				 unsigned int virq);
+
+#else /* CONFIG_TI_SCI_INTA_IRQCHIP */
+
+static inline int ti_sci_inta_register_event(struct device *dev, u16 src_id,
+					     u16 src_index, unsigned int virq,
+					     bool ack_needed, u32 flags)
+{
+	return -EINVAL;
+}
+
+static inline int ti_sci_inta_unregister_event(struct device *dev, u16 src_id,
+					       u16 src_index, unsigned int virq)
+{
+	return -EINVAL;
+}
+
+#endif /* CONFIG_TI_SCI_INTA_IRQCHIP */
+
+#endif /* __INCLUDE_LINUX_IRQCHIP_TI_SCI_INTA_H */
diff -Naur --no-dereference a/include/linux/k3_sa2ul.h b/include/linux/k3_sa2ul.h
--- a/include/linux/k3_sa2ul.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/linux/k3_sa2ul.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef	__K3_SA2UL_H
+#define	__K3_SA2UL_H
+
+int sa2ul_trng_enable(struct device *dev);
+
+#endif /* __K3_SA2UL_H */
diff -Naur --no-dereference a/include/linux/kernel.h b/include/linux/kernel.h
--- a/include/linux/kernel.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/kernel.h	2020-06-19 21:00:31.432227716 -0400
@@ -59,6 +59,7 @@
 #define ALIGN_DOWN(x, a)	__ALIGN_KERNEL((x) - ((a) - 1), (a))
 #define __ALIGN_MASK(x, mask)	__ALIGN_KERNEL_MASK((x), (mask))
 #define PTR_ALIGN(p, a)		((typeof(p))ALIGN((unsigned long)(p), (a)))
+#define PTR_ALIGN_DOWN(p, a)	((typeof(p))ALIGN_DOWN((unsigned long)(p), (a)))
 #define IS_ALIGNED(x, a)		(((x) & ((typeof(x))(a) - 1)) == 0)
 
 /* generic data direction definitions */
diff -Naur --no-dereference a/include/linux/leds.h b/include/linux/leds.h
--- a/include/linux/leds.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/leds.h	2020-06-19 21:00:31.432227716 -0400
@@ -22,6 +22,7 @@
 #include <linux/workqueue.h>
 
 struct device;
+struct device_node;
 /*
  * LED Core
  */
@@ -139,6 +140,9 @@
 extern void led_classdev_suspend(struct led_classdev *led_cdev);
 extern void led_classdev_resume(struct led_classdev *led_cdev);
 
+extern struct led_classdev *of_led_get(struct device_node *np);
+extern void led_put(struct led_classdev *led_cdev);
+
 /**
  * led_blink_set - set blinking with software fallback
  * @led_cdev: the LED to start blinking
diff -Naur --no-dereference a/include/linux/lockdep.h b/include/linux/lockdep.h
--- a/include/linux/lockdep.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/lockdep.h	2020-06-19 21:00:31.432227716 -0400
@@ -313,6 +313,8 @@
 	return lock->key == key;
 }
 
+struct lock_class *lockdep_hlock_class(struct held_lock *hlock);
+
 /*
  * Acquire a lock.
  *
@@ -439,6 +441,7 @@
 
 #define lockdep_depth(tsk)	(0)
 
+#define lockdep_is_held(lock)			(1)
 #define lockdep_is_held_type(l, r)		(1)
 
 #define lockdep_assert_held(l)			do { (void)(l); } while (0)
diff -Naur --no-dereference a/include/linux/mfd/stmpe.h b/include/linux/mfd/stmpe.h
--- a/include/linux/mfd/stmpe.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/mfd/stmpe.h	2020-06-19 21:00:31.432227716 -0400
@@ -1,7 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
 /*
  * Copyright (C) ST-Ericsson SA 2010
  *
- * License Terms: GNU General Public License, version 2
  * Author: Rabin Vincent <rabin.vincent@stericsson.com> for ST-Ericsson
  */
 
@@ -10,6 +10,20 @@
 
 #include <linux/mutex.h>
 
+#define STMPE_SAMPLE_TIME(x)	((x & 0xf) << 4)
+#define STMPE_MOD_12B(x)	((x & 0x1) << 3)
+#define STMPE_REF_SEL(x)	((x & 0x1) << 1)
+#define STMPE_ADC_FREQ(x)	(x & 0x3)
+#define STMPE_AVE_CTRL(x)	((x & 0x3) << 6)
+#define STMPE_DET_DELAY(x)	((x & 0x7) << 3)
+#define STMPE_SETTLING(x)	(x & 0x7)
+#define STMPE_FRACTION_Z(x)	(x & 0x7)
+#define STMPE_I_DRIVE(x)	(x & 0x1)
+#define STMPE_OP_MODE(x)	((x & 0x7) << 1)
+
+#define STMPE811_REG_ADC_CTRL1	0x20
+#define STMPE811_REG_ADC_CTRL2	0x21
+
 struct device;
 struct regulator;
 
@@ -123,6 +137,12 @@
 	u8 ier[2];
 	u8 oldier[2];
 	struct stmpe_platform_data *pdata;
+
+	/* For devices that use an ADC */
+	u8 sample_time;
+	u8 mod_12b;
+	u8 ref_sel;
+	u8 adc_freq;
 };
 
 extern int stmpe_reg_write(struct stmpe *stmpe, u8 reg, u8 data);
@@ -136,6 +156,7 @@
 			     enum stmpe_block block);
 extern int stmpe_enable(struct stmpe *stmpe, unsigned int blocks);
 extern int stmpe_disable(struct stmpe *stmpe, unsigned int blocks);
+extern int stmpe811_adc_common_init(struct stmpe *stmpe);
 
 #define STMPE_GPIO_NOREQ_811_TOUCH	(0xf0)
 
diff -Naur --no-dereference a/include/linux/mfd/ti_am335x_tscadc.h b/include/linux/mfd/ti_am335x_tscadc.h
--- a/include/linux/mfd/ti_am335x_tscadc.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/mfd/ti_am335x_tscadc.h	2020-06-19 21:00:31.432227716 -0400
@@ -78,6 +78,8 @@
 #define STEPCONFIG_YNN		BIT(8)
 #define STEPCONFIG_XNP		BIT(9)
 #define STEPCONFIG_YPN		BIT(10)
+#define STEPCONFIG_RFP(val)	((val) << 12)
+#define STEPCONFIG_RFP_VREFP	(0x3 << 12)
 #define STEPCONFIG_INM_MASK	(0xF << 15)
 #define STEPCONFIG_INM(val)	((val) << 15)
 #define STEPCONFIG_INM_ADCREFM	STEPCONFIG_INM(8)
@@ -86,6 +88,8 @@
 #define STEPCONFIG_INP_AN4	STEPCONFIG_INP(4)
 #define STEPCONFIG_INP_ADCREFM	STEPCONFIG_INP(8)
 #define STEPCONFIG_FIFO1	BIT(26)
+#define STEPCONFIG_RFM(val)	((val) << 23)
+#define STEPCONFIG_RFM_VREFN	(0x3 << 23)
 
 /* Delay register */
 #define STEPDELAY_OPEN_MASK	(0x3FFFF << 0)
diff -Naur --no-dereference a/include/linux/mmc/host.h b/include/linux/mmc/host.h
--- a/include/linux/mmc/host.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/mmc/host.h	2020-06-19 21:00:31.432227716 -0400
@@ -321,6 +321,8 @@
 #define MMC_CAP_3_3V_DDR	(1 << 11)	/* Host supports eMMC DDR 3.3V */
 #define MMC_CAP_1_8V_DDR	(1 << 12)	/* Host supports eMMC DDR 1.8V */
 #define MMC_CAP_1_2V_DDR	(1 << 13)	/* Host supports eMMC DDR 1.2V */
+#define MMC_CAP_DDR		(MMC_CAP_3_3V_DDR | MMC_CAP_1_8V_DDR | \
+				 MMC_CAP_1_2V_DDR)
 #define MMC_CAP_POWER_OFF_CARD	(1 << 14)	/* Can power off after boot */
 #define MMC_CAP_BUS_WIDTH_TEST	(1 << 15)	/* CMD14/CMD19 bus width ok */
 #define MMC_CAP_UHS_SDR12	(1 << 16)	/* Host supports UHS SDR12 mode */
diff -Naur --no-dereference a/include/linux/mmc/sdio_ids.h b/include/linux/mmc/sdio_ids.h
--- a/include/linux/mmc/sdio_ids.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/mmc/sdio_ids.h	2020-06-19 21:00:31.432227716 -0400
@@ -41,7 +41,10 @@
 #define SDIO_DEVICE_ID_BROADCOM_43455		0xa9bf
 #define SDIO_DEVICE_ID_BROADCOM_4354		0x4354
 #define SDIO_DEVICE_ID_BROADCOM_4356		0x4356
+#define SDIO_DEVICE_ID_BROADCOM_4359		0x4355
 #define SDIO_DEVICE_ID_CYPRESS_4373		0x4373
+#define SDIO_DEVICE_ID_CYPRESS_43012		43012
+#define SDIO_DEVICE_ID_CYPRESS_89359		0x4359
 
 #define SDIO_VENDOR_ID_INTEL			0x0089
 #define SDIO_DEVICE_ID_INTEL_IWMC3200WIMAX	0x1402
diff -Naur --no-dereference a/include/linux/mm.h b/include/linux/mm.h
--- a/include/linux/mm.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/mm.h	2020-06-19 21:00:31.432227716 -0400
@@ -1448,6 +1448,28 @@
 	unmap_mapping_range(mapping, holebegin, holelen, 0);
 }
 
+extern void vma_do_file_update_time(struct vm_area_struct *, const char[], int);
+extern struct file *vma_do_pr_or_file(struct vm_area_struct *, const char[],
+				      int);
+extern void vma_do_get_file(struct vm_area_struct *, const char[], int);
+extern void vma_do_fput(struct vm_area_struct *, const char[], int);
+
+#define vma_file_update_time(vma)	vma_do_file_update_time(vma, __func__, \
+								__LINE__)
+#define vma_pr_or_file(vma)		vma_do_pr_or_file(vma, __func__, \
+							  __LINE__)
+#define vma_get_file(vma)		vma_do_get_file(vma, __func__, __LINE__)
+#define vma_fput(vma)			vma_do_fput(vma, __func__, __LINE__)
+
+#ifndef CONFIG_MMU
+extern struct file *vmr_do_pr_or_file(struct vm_region *, const char[], int);
+extern void vmr_do_fput(struct vm_region *, const char[], int);
+
+#define vmr_pr_or_file(region)		vmr_do_pr_or_file(region, __func__, \
+							  __LINE__)
+#define vmr_fput(region)		vmr_do_fput(region, __func__, __LINE__)
+#endif /* !CONFIG_MMU */
+
 extern int access_process_vm(struct task_struct *tsk, unsigned long addr,
 		void *buf, int len, unsigned int gup_flags);
 extern int access_remote_vm(struct mm_struct *mm, unsigned long addr,
diff -Naur --no-dereference a/include/linux/mm_types.h b/include/linux/mm_types.h
--- a/include/linux/mm_types.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/mm_types.h	2020-06-19 21:00:31.432227716 -0400
@@ -244,6 +244,7 @@
 	unsigned long	vm_top;		/* region allocated to here */
 	unsigned long	vm_pgoff;	/* the offset in vm_file corresponding to vm_start */
 	struct file	*vm_file;	/* the backing file or NULL */
+	struct file	*vm_prfile;	/* the virtual backing file or NULL */
 
 	int		vm_usage;	/* region usage count (access under nommu_region_sem) */
 	bool		vm_icache_flushed : 1; /* true if the icache has been flushed for
@@ -318,6 +319,7 @@
 	unsigned long vm_pgoff;		/* Offset (within vm_file) in PAGE_SIZE
 					   units */
 	struct file * vm_file;		/* File we map to (can be NULL). */
+	struct file *vm_prfile;		/* shadow of vm_file */
 	void * vm_private_data;		/* was vm_pte (shared mem) */
 
 	atomic_long_t swap_readahead_info;
diff -Naur --no-dereference a/include/linux/mnt_namespace.h b/include/linux/mnt_namespace.h
--- a/include/linux/mnt_namespace.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/mnt_namespace.h	2020-06-19 21:00:31.432227716 -0400
@@ -6,11 +6,14 @@
 struct mnt_namespace;
 struct fs_struct;
 struct user_namespace;
+struct vfsmount;
 
 extern struct mnt_namespace *copy_mnt_ns(unsigned long, struct mnt_namespace *,
 		struct user_namespace *, struct fs_struct *);
 extern void put_mnt_ns(struct mnt_namespace *ns);
 
+extern int is_current_mnt_ns(struct vfsmount *mnt);
+
 extern const struct file_operations proc_mounts_operations;
 extern const struct file_operations proc_mountinfo_operations;
 extern const struct file_operations proc_mountstats_operations;
diff -Naur --no-dereference a/include/linux/mtd/cfi.h b/include/linux/mtd/cfi.h
--- a/include/linux/mtd/cfi.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/mtd/cfi.h	2020-06-19 21:00:31.432227716 -0400
@@ -233,6 +233,11 @@
 	uint8_t  VppMin;
 	uint8_t  VppMax;
 	uint8_t  TopBottom;
+	/* Below field are added from version 1.5 */
+	uint8_t  ProgramSuspend;
+	uint8_t  UnlockBypass;
+	uint8_t  SecureSiliconSector;
+	uint8_t  SoftwareFeatures;
 } __packed;
 
 /* Vendor-Specific PRI for Atmel chips (command set 0x0002) */
@@ -377,6 +382,7 @@
 #define CFI_MFR_SHARP		0x00B0
 #define CFI_MFR_SST		0x00BF
 #define CFI_MFR_ST		0x0020 /* STMicroelectronics */
+#define CFI_MFR_MICRON		0x002C /* Micron */
 #define CFI_MFR_TOSHIBA		0x0098
 #define CFI_MFR_WINBOND		0x00DA
 
diff -Naur --no-dereference a/include/linux/mtd/hyperbus.h b/include/linux/mtd/hyperbus.h
--- a/include/linux/mtd/hyperbus.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/linux/mtd/hyperbus.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,86 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+#ifndef __LINUX_MTD_HYPERBUS_H__
+#define __LINUX_MTD_HYPERBUS_H__
+
+#include <linux/mtd/map.h>
+
+enum hyperbus_memtype {
+	HYPERFLASH,
+	HYPERRAM,
+};
+
+/**
+ * struct hyperbus_device - struct representing HyperBus slave device
+ * @map: map_info struct for accessing MMIO HyperBus flash memory
+ * @np: pointer to HyperBus slave device node
+ * @mtd: pointer to MTD struct
+ * @ctlr: pointer to HyperBus controller struct
+ * @memtype: type of memory device: HyperFlash or HyperRAM
+ * @registered: flag to indicate whether device is registered with MTD core
+ */
+
+struct hyperbus_device {
+	struct map_info map;
+	struct device_node *np;
+	struct mtd_info *mtd;
+	struct hyperbus_ctlr *ctlr;
+	enum hyperbus_memtype memtype;
+	bool registered;
+};
+
+/**
+ * struct hyperbus_ops - struct representing custom HyperBus operations
+ * @read16: read 16 bit of data to flash in a single burst. Used to read
+ *          from non default address space, such as ID/CFI space
+ * @write16: write 16 bit of data to flash in a single burst. Used to
+ *           send cmd to flash or write single word at a time.
+ * @copy_from: copy data from flash memory
+ * @copy_to: copy data to flash memory
+ * @calibrate: calibrate HyperBus controller
+ */
+
+struct hyperbus_ops {
+	u16 (*read16)(struct hyperbus_device *hbdev, unsigned long addr);
+	void (*write16)(struct hyperbus_device *hbdev,
+			unsigned long addr, u16 val);
+	void (*copy_from)(struct hyperbus_device *hbdev, void *to,
+			  unsigned long from, ssize_t len);
+	void (*copy_to)(struct hyperbus_device *dev, unsigned long to,
+			const void *from, ssize_t len);
+	int (*calibrate)(struct hyperbus_device *dev);
+};
+
+/**
+ * struct hyperbus_ctlr - struct representing HyperBus controller
+ * @dev: pointer to HyperBus controller device
+ * @calibrated: flag to indicate ctlr calibration sequence is complete
+ * @ops: HyperBus controller ops
+ */
+struct hyperbus_ctlr {
+	struct device *dev;
+	bool calibrated;
+
+	const struct hyperbus_ops *ops;
+};
+
+/**
+ * hyperbus_register_device - probe and register a HyperBus slave memory device
+ * @hbdev: hyperbus_device struct with dev, np and ctlr field populated
+ *
+ * Return: 0 for success, others for failure.
+ */
+int hyperbus_register_device(struct hyperbus_device *hbdev);
+
+/**
+ * hyperbus_unregister_device - deregister HyperBus slave memory device
+ * @hbdev: hyperbus_device to be unregistered
+ *
+ * Return: 0 for success, others for failure.
+ */
+int hyperbus_unregister_device(struct hyperbus_device *hbdev);
+
+#endif /* __LINUX_MTD_HYPERBUS_H__ */
diff -Naur --no-dereference a/include/linux/mtd/spi-nor.h b/include/linux/mtd/spi-nor.h
--- a/include/linux/mtd/spi-nor.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/mtd/spi-nor.h	2020-06-19 21:00:31.432227716 -0400
@@ -23,7 +23,8 @@
 #define SNOR_MFR_ATMEL		CFI_MFR_ATMEL
 #define SNOR_MFR_GIGADEVICE	0xc8
 #define SNOR_MFR_INTEL		CFI_MFR_INTEL
-#define SNOR_MFR_MICRON		CFI_MFR_ST /* ST Micro <--> Micron */
+#define SNOR_MFR_ST		CFI_MFR_ST	/* ST Micro */
+#define SNOR_MFR_MICRON		CFI_MFR_MICRON	/* Micron */
 #define SNOR_MFR_MACRONIX	CFI_MFR_MACRONIX
 #define SNOR_MFR_SPANSION	CFI_MFR_AMD
 #define SNOR_MFR_SST		CFI_MFR_SST
@@ -49,9 +50,13 @@
 #define SPINOR_OP_READ_1_2_2	0xbb	/* Read data bytes (Dual I/O SPI) */
 #define SPINOR_OP_READ_1_1_4	0x6b	/* Read data bytes (Quad Output SPI) */
 #define SPINOR_OP_READ_1_4_4	0xeb	/* Read data bytes (Quad I/O SPI) */
+#define SPINOR_OP_READ_1_1_8	0x8b	/* Read data bytes (Octal Output SPI) */
+#define SPINOR_OP_READ_1_8_8	0xcb	/* Read data bytes (Octal I/O SPI) */
 #define SPINOR_OP_PP		0x02	/* Page program (up to 256 bytes) */
 #define SPINOR_OP_PP_1_1_4	0x32	/* Quad page program */
 #define SPINOR_OP_PP_1_4_4	0x38	/* Quad page program */
+#define SPINOR_OP_PP_1_1_8	0x82	/* Octal page program */
+#define SPINOR_OP_PP_1_8_8	0xc2	/* Octal page program */
 #define SPINOR_OP_BE_4K		0x20	/* Erase 4KiB block */
 #define SPINOR_OP_BE_4K_PMC	0xd7	/* Erase 4KiB block on PMC chips */
 #define SPINOR_OP_BE_32K	0x52	/* Erase 32KiB block */
@@ -72,9 +77,13 @@
 #define SPINOR_OP_READ_1_2_2_4B	0xbc	/* Read data bytes (Dual I/O SPI) */
 #define SPINOR_OP_READ_1_1_4_4B	0x6c	/* Read data bytes (Quad Output SPI) */
 #define SPINOR_OP_READ_1_4_4_4B	0xec	/* Read data bytes (Quad I/O SPI) */
+#define SPINOR_OP_READ_1_1_8_4B	0x7c	/* Read data bytes (Octal Output SPI) */
+#define SPINOR_OP_READ_1_8_8_4B	0xcc	/* Read data bytes (Octal I/O SPI) */
 #define SPINOR_OP_PP_4B		0x12	/* Page program (up to 256 bytes) */
 #define SPINOR_OP_PP_1_1_4_4B	0x34	/* Quad page program */
 #define SPINOR_OP_PP_1_4_4_4B	0x3e	/* Quad page program */
+#define SPINOR_OP_PP_1_1_8_4B	0x84	/* Octal page program */
+#define SPINOR_OP_PP_1_8_8_4B	0x8e	/* Octal page program */
 #define SPINOR_OP_BE_4K_4B	0x21	/* Erase 4KiB block */
 #define SPINOR_OP_BE_32K_4B	0x5c	/* Erase 32KiB block */
 #define SPINOR_OP_SE_4B		0xdc	/* Sector erase (usually 64KiB) */
@@ -83,10 +92,12 @@
 #define SPINOR_OP_READ_1_1_1_DTR	0x0d
 #define SPINOR_OP_READ_1_2_2_DTR	0xbd
 #define SPINOR_OP_READ_1_4_4_DTR	0xed
+#define SPINOR_OP_READ_8_8_8_DTR	0xfd
 
 #define SPINOR_OP_READ_1_1_1_DTR_4B	0x0e
 #define SPINOR_OP_READ_1_2_2_DTR_4B	0xbe
 #define SPINOR_OP_READ_1_4_4_DTR_4B	0xee
+#define SPINOR_OP_READ_8_8_8_DTR_4B	0xfd
 
 /* Used for SST flashes only. */
 #define SPINOR_OP_BP		0x02	/* Byte program */
@@ -113,6 +124,9 @@
 /* Used for Micron flashes only. */
 #define SPINOR_OP_RD_EVCR      0x65    /* Read EVCR register */
 #define SPINOR_OP_WD_EVCR      0x61    /* Write EVCR register */
+#define SPINOR_OP_WR_VCR       0x81	/* Write VCR register */
+#define SPINOR_OP_RESET_EN     0x66	/* Enable Soft reset */
+#define SPINOR_OP_RESET_MEM    0x99	/* Reset memory */
 
 /* Status Register bits. */
 #define SR_WIP			BIT(0)	/* Write in progress */
@@ -132,6 +146,9 @@
 /* Enhanced Volatile Configuration Register bits */
 #define EVCR_QUAD_EN_MICRON	BIT(7)	/* Micron Quad I/O */
 
+/* Micron VCR Register Settings */
+#define VCR_OCTAL_DDR_EN_MICRON	0xe7	/* Octal DDR with DQS mode */
+
 /* Flag Status Register bits */
 #define FSR_READY		BIT(7)	/* Device status, 0 = Busy, 1 = Ready */
 #define FSR_E_ERR		BIT(5)	/* Erase operation status */
@@ -189,6 +206,7 @@
 	SNOR_PROTO_1_2_2_DTR = SNOR_PROTO_DTR(1, 2, 2),
 	SNOR_PROTO_1_4_4_DTR = SNOR_PROTO_DTR(1, 4, 4),
 	SNOR_PROTO_1_8_8_DTR = SNOR_PROTO_DTR(1, 8, 8),
+	SNOR_PROTO_8_8_8_DTR = SNOR_PROTO_DTR(8, 8, 8),
 };
 
 static inline bool spi_nor_protocol_is_dtr(enum spi_nor_protocol proto)
@@ -238,6 +256,15 @@
 	SNOR_F_BROKEN_RESET	= BIT(6),
 };
 
+enum spi_nor_mode {
+	SPI_NOR_MODE_SPI = 0,
+	SPI_NOR_MODE_DPI,
+	SPI_NOR_MODE_QPI,
+	SPI_NOR_MODE_OPI,
+	SPI_NOR_MODE_OPI_DTR,
+	SPI_NOR_NUM_MODES,
+};
+
 /**
  * struct flash_info - Forward declaration of a structure used internally by
  *		       spi_nor_scan()
@@ -295,6 +322,8 @@
 	enum spi_nor_protocol	write_proto;
 	enum spi_nor_protocol	reg_proto;
 	bool			sst_write_second;
+	enum spi_nor_mode	mode;
+	enum spi_nor_mode	preferred_mode;
 	u32			flags;
 	u8			cmd_buf[SPI_NOR_MAX_CMD_SIZE];
 
@@ -307,12 +336,15 @@
 			size_t len, u_char *read_buf);
 	ssize_t (*write)(struct spi_nor *nor, loff_t to,
 			size_t len, const u_char *write_buf);
+	int (*calibrate)(struct spi_nor *nor, void *calib_data, size_t len);
 	int (*erase)(struct spi_nor *nor, loff_t offs);
 
 	int (*flash_lock)(struct spi_nor *nor, loff_t ofs, uint64_t len);
 	int (*flash_unlock)(struct spi_nor *nor, loff_t ofs, uint64_t len);
 	int (*flash_is_locked)(struct spi_nor *nor, loff_t ofs, uint64_t len);
 	int (*quad_enable)(struct spi_nor *nor);
+	int (*change_mode)(struct spi_nor *nor, enum spi_nor_mode newmode);
+	void (*adjust_op)(struct spi_nor *nor, enum spi_nor_mode mode);
 
 	void *priv;
 };
@@ -340,7 +372,7 @@
 /*
  *(Fast) Read capabilities.
  * MUST be ordered by priority: the higher bit position, the higher priority.
- * As a matter of performances, it is relevant to use Octo SPI protocols first,
+ * As a matter of performances, it is relevant to use Octal SPI protocols first,
  * then Quad SPI protocols before Dual SPI protocols, Fast Read and lastly
  * (Slow) Read.
  */
@@ -361,16 +393,17 @@
 #define SNOR_HWCAPS_READ_4_4_4		BIT(9)
 #define SNOR_HWCAPS_READ_1_4_4_DTR	BIT(10)
 
-#define SNOR_HWCPAS_READ_OCTO		GENMASK(14, 11)
+#define SNOR_HWCPAS_READ_OCTAL		GENMASK(15, 11)
 #define SNOR_HWCAPS_READ_1_1_8		BIT(11)
 #define SNOR_HWCAPS_READ_1_8_8		BIT(12)
 #define SNOR_HWCAPS_READ_8_8_8		BIT(13)
 #define SNOR_HWCAPS_READ_1_8_8_DTR	BIT(14)
+#define SNOR_HWCAPS_READ_8_8_8_DTR	BIT(15)
 
 /*
  * Page Program capabilities.
  * MUST be ordered by priority: the higher bit position, the higher priority.
- * Like (Fast) Read capabilities, Octo/Quad SPI protocols are preferred to the
+ * Like (Fast) Read capabilities, Octal/Quad SPI protocols are preferred to the
  * legacy SPI 1-1-1 protocol.
  * Note that Dual Page Programs are not supported because there is no existing
  * JEDEC/SFDP standard to define them. Also at this moment no SPI flash memory
@@ -384,7 +417,7 @@
 #define SNOR_HWCAPS_PP_1_4_4	BIT(18)
 #define SNOR_HWCAPS_PP_4_4_4	BIT(19)
 
-#define SNOR_HWCAPS_PP_OCTO	GENMASK(22, 20)
+#define SNOR_HWCAPS_PP_OCTAL	GENMASK(22, 20)
 #define SNOR_HWCAPS_PP_1_1_8	BIT(20)
 #define SNOR_HWCAPS_PP_1_8_8	BIT(21)
 #define SNOR_HWCAPS_PP_8_8_8	BIT(22)
diff -Naur --no-dereference a/include/linux/of_platform.h b/include/linux/of_platform.h
--- a/include/linux/of_platform.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/of_platform.h	2020-06-19 21:00:31.432227716 -0400
@@ -70,6 +70,9 @@
 extern int of_platform_bus_probe(struct device_node *root,
 				 const struct of_device_id *matches,
 				 struct device *parent);
+extern struct platform_device *
+of_platform_device_create_pdata(struct device_node *np, const char *bus_id,
+				void *platform_data, struct device *parent);
 #ifdef CONFIG_OF_ADDRESS
 extern int of_platform_populate(struct device_node *root,
 				const struct of_device_id *matches,
diff -Naur --no-dereference a/include/linux/omap-iommu.h b/include/linux/omap-iommu.h
--- a/include/linux/omap-iommu.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/omap-iommu.h	2020-06-19 21:00:31.432227716 -0400
@@ -13,12 +13,29 @@
 #ifndef _OMAP_IOMMU_H_
 #define _OMAP_IOMMU_H_
 
+#include <linux/errno.h>
+
+struct iommu_domain;
+
 #ifdef CONFIG_OMAP_IOMMU
 extern void omap_iommu_save_ctx(struct device *dev);
 extern void omap_iommu_restore_ctx(struct device *dev);
+
+int omap_iommu_domain_deactivate(struct iommu_domain *domain);
+int omap_iommu_domain_activate(struct iommu_domain *domain);
 #else
 static inline void omap_iommu_save_ctx(struct device *dev) {}
 static inline void omap_iommu_restore_ctx(struct device *dev) {}
+
+static inline int omap_iommu_domain_deactivate(struct iommu_domain *domain)
+{
+	return -ENOTSUPP;
+}
+
+static inline int omap_iommu_domain_activate(struct iommu_domain *domain)
+{
+	return -ENOTSUPP;
+}
 #endif
 
 #endif
diff -Naur --no-dereference a/include/linux/omap-mailbox.h b/include/linux/omap-mailbox.h
--- a/include/linux/omap-mailbox.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/omap-mailbox.h	2020-06-19 21:00:31.432227716 -0400
@@ -6,7 +6,9 @@
 #ifndef OMAP_MAILBOX_H
 #define OMAP_MAILBOX_H
 
-typedef u32 mbox_msg_t;
+typedef uintptr_t mbox_msg_t;
+
+#define to_omap_mbox_msg(data) (u32)(mbox_msg_t)(data)
 
 typedef int __bitwise omap_mbox_irq_t;
 #define IRQ_TX ((__force omap_mbox_irq_t) 1)
diff -Naur --no-dereference a/include/linux/pci-epc.h b/include/linux/pci-epc.h
--- a/include/linux/pci-epc.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/pci-epc.h	2020-06-19 21:00:31.432227716 -0400
@@ -13,6 +13,11 @@
 
 struct pci_epc;
 
+enum pci_epc_interface_type {
+	PRIMARY_INTERFACE,
+	SECONDARY_INTERFACE,
+};
+
 enum pci_epc_irq_type {
 	PCI_EPC_IRQ_UNKNOWN,
 	PCI_EPC_IRQ_LEGACY,
@@ -20,8 +25,24 @@
 	PCI_EPC_IRQ_MSIX,
 };
 
+static inline const char *
+pci_epc_interface_string(enum pci_epc_interface_type type)
+{
+	switch (type) {
+	case PRIMARY_INTERFACE:
+		return "primary";
+	case SECONDARY_INTERFACE:
+		return "secondary";
+	default:
+		return "UNKNOWN interface";
+	}
+}
+
 /**
  * struct pci_epc_ops - set of function pointers for performing EPC operations
+ * @epf_init: ops to perform EPC specific initialization
+ * @epf_exit: ops to cleanup EPF
+ * @data_transfer: ops to transfer data with the remote RC
  * @write_header: ops to populate configuration space header
  * @set_bar: ops to configure the BAR
  * @clear_bar: ops to reset the BAR
@@ -36,29 +57,42 @@
  * @get_msix: ops to get the number of MSI-X interrupts allocated by the RC
  *	     from the MSI-X capability register
  * @raise_irq: ops to raise a legacy, MSI or MSI-X interrupt
+ * @map_msi_irq: ops to map physical address to MSI address and return MSI data
  * @start: ops to start the PCI link
  * @stop: ops to stop the PCI link
  * @owner: the module owner containing the ops
  */
 struct pci_epc_ops {
-	int	(*write_header)(struct pci_epc *epc, u8 func_no,
+	int	(*epf_init)(struct pci_epc *epc, struct pci_epf *epf);
+	void	(*epf_exit)(struct pci_epc *epc, struct pci_epf *epf);
+	int	(*data_transfer)(struct pci_epc *epc, struct pci_epf *epf,
+				 dma_addr_t dma_dst, dma_addr_t dma_src,
+				 size_t len);
+	int	(*write_header)(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
 				struct pci_epf_header *hdr);
-	int	(*set_bar)(struct pci_epc *epc, u8 func_no,
+	int	(*set_bar)(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
 			   struct pci_epf_bar *epf_bar);
-	void	(*clear_bar)(struct pci_epc *epc, u8 func_no,
+	void	(*clear_bar)(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
 			     struct pci_epf_bar *epf_bar);
-	int	(*map_addr)(struct pci_epc *epc, u8 func_no,
+	int	(*map_addr)(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
 			    phys_addr_t addr, u64 pci_addr, size_t size);
-	void	(*unmap_addr)(struct pci_epc *epc, u8 func_no,
+	void	(*unmap_addr)(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
 			      phys_addr_t addr);
-	int	(*set_msi)(struct pci_epc *epc, u8 func_no, u8 interrupts);
-	int	(*get_msi)(struct pci_epc *epc, u8 func_no);
-	int	(*set_msix)(struct pci_epc *epc, u8 func_no, u16 interrupts);
-	int	(*get_msix)(struct pci_epc *epc, u8 func_no);
-	int	(*raise_irq)(struct pci_epc *epc, u8 func_no,
+	int	(*set_msi)(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
+			   u8 interrupts);
+	int	(*get_msi)(struct pci_epc *epc, u8 func_no, u8 vfunc_no);
+	int	(*set_msix)(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
+			    u16 interrupts, enum pci_barno, u32 offset);
+	int	(*get_msix)(struct pci_epc *epc, u8 func_no, u8 vfunc_no);
+	int	(*raise_irq)(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
 			     enum pci_epc_irq_type type, u16 interrupt_num);
+	int	(*map_msi_irq)(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
+			       phys_addr_t phys_addr, u8 interrupt_num,
+			       u32 entry_size, u32 *msi_data);
 	int	(*start)(struct pci_epc *epc);
 	void	(*stop)(struct pci_epc *epc);
+	const struct pci_epc_features* (*get_features)(struct pci_epc *epc,
+						       u8 func_no, u8 vfunc_no);
 	struct module *owner;
 };
 
@@ -69,6 +103,7 @@
  * @bitmap: bitmap to manage the PCI address space
  * @pages: number of bits representing the address region
  * @page_size: size of each page
+ * @lock: mutex to protect bitmap
  */
 struct pci_epc_mem {
 	phys_addr_t	phys_base;
@@ -76,6 +111,8 @@
 	unsigned long	*bitmap;
 	size_t		page_size;
 	int		pages;
+	/* mutex to protect against concurrent access for memory allocation*/
+	struct mutex	lock;
 };
 
 /**
@@ -86,7 +123,9 @@
  * @mem: address space of the endpoint controller
  * @max_functions: max number of functions that can be configured in this EPC
  * @group: configfs group representing the PCI EPC device
- * @lock: spinlock to protect pci_epc ops
+ * @lock: mutex to protect pci_epc ops
+ * @function_num_map: bitmap to manage physical function number
+ * @notifier: used to notify EPF of any EPC events (like linkup)
  */
 struct pci_epc {
 	struct device			dev;
@@ -95,18 +134,31 @@
 	struct pci_epc_mem		*mem;
 	u8				max_functions;
 	struct config_group		*group;
-	/* spinlock to protect against concurrent access of EP controller */
-	spinlock_t			lock;
-	unsigned int			features;
+	/* mutex to protect against concurrent access of EP controller */
+	struct mutex			lock;
+	unsigned long			function_num_map;
+	struct atomic_notifier_head	notifier;
 };
 
-#define EPC_FEATURE_NO_LINKUP_NOTIFIER		BIT(0)
-#define EPC_FEATURE_BAR_MASK			(BIT(1) | BIT(2) | BIT(3))
-#define EPC_FEATURE_MSIX_AVAILABLE		BIT(4)
-#define EPC_FEATURE_SET_BAR(features, bar)	\
-		(features |= (EPC_FEATURE_BAR_MASK & (bar << 1)))
-#define EPC_FEATURE_GET_BAR(features)		\
-		((features & EPC_FEATURE_BAR_MASK) >> 1)
+/**
+ * struct pci_epc_features - features supported by a EPC device per function
+ * @linkup_notifier: indicate if the EPC device can notify EPF driver on link up
+ * @msi_capable: indicate if the endpoint function has MSI capability
+ * @msix_capable: indicate if the endpoint function has MSI-X capability
+ * @reserved_bar: bitmap to indicate reserved BAR unavailable to function driver
+ * @bar_fixed_64bit: bitmap to indicate fixed 64bit BARs
+ * @bar_fixed_size: Array specifying the size supported by each BAR
+ * @align: alignment size required for BAR buffer allocation
+ */
+struct pci_epc_features {
+	unsigned int	linkup_notifier : 1;
+	unsigned int	msi_capable : 1;
+	unsigned int	msix_capable : 1;
+	u8	reserved_bar;
+	u8	bar_fixed_64bit;
+	u64	bar_fixed_size[BAR_5 + 1];
+	size_t	align;
+};
 
 #define to_pci_epc(device) container_of((device), struct pci_epc, dev)
 
@@ -128,6 +180,12 @@
 	return dev_get_drvdata(&epc->dev);
 }
 
+static inline int
+pci_epc_register_notifier(struct pci_epc *epc, struct notifier_block *nb)
+{
+	return atomic_notifier_chain_register(&epc->notifier, nb);
+}
+
 struct pci_epc *
 __devm_pci_epc_create(struct device *dev, const struct pci_epc_ops *ops,
 		      struct module *owner);
@@ -136,31 +194,47 @@
 		 struct module *owner);
 void devm_pci_epc_destroy(struct device *dev, struct pci_epc *epc);
 void pci_epc_destroy(struct pci_epc *epc);
-int pci_epc_add_epf(struct pci_epc *epc, struct pci_epf *epf);
+int pci_epc_add_epf(struct pci_epc *epc, struct pci_epf *epf,
+		    enum pci_epc_interface_type type);
 void pci_epc_linkup(struct pci_epc *epc);
-void pci_epc_remove_epf(struct pci_epc *epc, struct pci_epf *epf);
-int pci_epc_write_header(struct pci_epc *epc, u8 func_no,
+void pci_epc_remove_epf(struct pci_epc *epc, struct pci_epf *epf,
+			enum pci_epc_interface_type type);
+int pci_epc_write_header(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
 			 struct pci_epf_header *hdr);
-int pci_epc_set_bar(struct pci_epc *epc, u8 func_no,
+int pci_epc_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
 		    struct pci_epf_bar *epf_bar);
-void pci_epc_clear_bar(struct pci_epc *epc, u8 func_no,
+void pci_epc_clear_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
 		       struct pci_epf_bar *epf_bar);
-int pci_epc_map_addr(struct pci_epc *epc, u8 func_no,
+int pci_epc_map_addr(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
 		     phys_addr_t phys_addr,
 		     u64 pci_addr, size_t size);
-void pci_epc_unmap_addr(struct pci_epc *epc, u8 func_no,
+void pci_epc_unmap_addr(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
 			phys_addr_t phys_addr);
-int pci_epc_set_msi(struct pci_epc *epc, u8 func_no, u8 interrupts);
-int pci_epc_get_msi(struct pci_epc *epc, u8 func_no);
-int pci_epc_set_msix(struct pci_epc *epc, u8 func_no, u16 interrupts);
-int pci_epc_get_msix(struct pci_epc *epc, u8 func_no);
-int pci_epc_raise_irq(struct pci_epc *epc, u8 func_no,
+int pci_epc_set_msi(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
+		    u8 interrupts);
+int pci_epc_get_msi(struct pci_epc *epc, u8 func_no, u8 vfunc_no);
+int pci_epc_set_msix(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
+		     u16 interrupts,  enum pci_barno, u32 offset);
+int pci_epc_get_msix(struct pci_epc *epc, u8 func_no, u8 vfunc_no);
+int pci_epc_map_msi_irq(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
+			phys_addr_t phys_addr, u8 interrupt_num,
+			u32 entry_size, u32 *msi_data);
+int pci_epc_raise_irq(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
 		      enum pci_epc_irq_type type, u16 interrupt_num);
 int pci_epc_start(struct pci_epc *epc);
 void pci_epc_stop(struct pci_epc *epc);
+void pci_epc_of_parse_header(struct device_node *node,
+			     struct pci_epf_header *header);
+const struct pci_epc_features *pci_epc_get_features(struct pci_epc *epc,
+						    u8 func_no, u8 vfunc_no);
+int pci_epc_get_first_free_bar(const struct pci_epc_features *epc_features);
+int pci_epc_get_next_free_bar(const struct pci_epc_features
+			      *epc_features, enum pci_barno bar);
 struct pci_epc *pci_epc_get(const char *epc_name);
 void pci_epc_put(struct pci_epc *epc);
-
+struct pci_epc *of_pci_epc_get(struct device_node *node, int index);
+struct pci_epc *of_pci_epc_get_by_name(struct device_node *node,
+				       const char *epc_name);
 int __pci_epc_mem_init(struct pci_epc *epc, phys_addr_t phys_addr, size_t size,
 		       size_t page_size);
 void pci_epc_mem_exit(struct pci_epc *epc);
@@ -168,4 +242,6 @@
 				     phys_addr_t *phys_addr, size_t size);
 void pci_epc_mem_free_addr(struct pci_epc *epc, phys_addr_t phys_addr,
 			   void __iomem *virt_addr, size_t size);
+int pci_epc_epf_init(struct pci_epc *epc, struct pci_epf *epf);
+void pci_epc_epf_exit(struct pci_epc *epc, struct pci_epf *epf);
 #endif /* __LINUX_PCI_EPC_H */
diff -Naur --no-dereference a/include/linux/pci-epf.h b/include/linux/pci-epf.h
--- a/include/linux/pci-epf.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/pci-epf.h	2020-06-19 21:00:31.432227716 -0400
@@ -11,9 +11,11 @@
 
 #include <linux/device.h>
 #include <linux/mod_devicetable.h>
+#include <linux/of.h>
 #include <linux/pci.h>
 
 struct pci_epf;
+enum pci_epc_interface_type;
 
 enum pci_barno {
 	BAR_0,
@@ -55,13 +57,10 @@
  * @bind: ops to perform when a EPC device has been bound to EPF device
  * @unbind: ops to perform when a binding has been lost between a EPC device
  *	    and EPF device
- * @linkup: ops to perform when the EPC device has established a connection with
- *	    a host system
  */
 struct pci_epf_ops {
 	int	(*bind)(struct pci_epf *epf);
 	void	(*unbind)(struct pci_epf *epf);
-	void	(*linkup)(struct pci_epf *epf);
 };
 
 /**
@@ -92,10 +91,12 @@
 /**
  * struct pci_epf_bar - represents the BAR of EPF device
  * @phys_addr: physical address that should be mapped to the BAR
+ * @addr: virtual address corresponding to the @phys_addr
  * @size: the size of the address space present in BAR
  */
 struct pci_epf_bar {
 	dma_addr_t	phys_addr;
+	void		*addr;
 	size_t		size;
 	enum pci_barno	barno;
 	int		flags;
@@ -104,27 +105,88 @@
 /**
  * struct pci_epf - represents the PCI EPF device
  * @dev: the PCI EPF device
+ * @node: the device tree node of the PCI EPF device
  * @name: the name of the PCI EPF device
  * @header: represents standard configuration header
  * @bar: represents the BAR of EPF device
  * @msi_interrupts: number of MSI interrupts required by this function
+ * @func_no: unique (physical) function number within this endpoint device
+ * @vfunc_no: unique virtual function number within a physical function
  * @func_no: unique function number within this endpoint device
+ * @dma_chan: allocated DMA memcpy channel
+ * @transfer_complete: completion variable to handle completion of data transfer
  * @epc: the EPC device to which this EPF device is bound
+ * @epf_pf: the physical EPF device to which this virtual EPF device is bound
+ *   An EPF cannot be associated with both EPC and physical EPF device at the
+ *   same time.
  * @driver: the EPF driver to which this EPF device is bound
  * @list: to add pci_epf as a list of PCI endpoint functions to pci_epc
+ * @nb: notifier block to notify EPF of any EPC events (like linkup)
+ * @lock: mutex to protect pci_epf_ops
+ * @is_bound: indicates if bind notification to function driver has been invoked
+ * @is_vf: true - virtual function, false - physical function
+ * @vfunction_num_map: bitmap to manage virtual function number
+ * @pci_vepf: list of virtual endpoint function associated with this function
+ * @sec_epc: the secondary EPC device to which this EPF device is bound
+ * @sec_epc_list: to add pci_epf as list of PCI endpoint functions to secondary
+ *   EPC device
+ * @sec_epc_bar: represents the BAR of EPF device associated with secondary EPC
+ * @sec_epc_func_no: unique (physical) function number within the secondary EPC
+ * @sec_epc_vfunc_no: unique virtual function number within a physical function
+ *   associated with secondary EPC
+ * @sec_epc_vfunction_num_map: bitmap to manage virtual function number
+ *   associated with the physical function of the
+ *   secondary EPC
+ * @sec_epc_pci_vepf: list of virtual endpoint function associated with the
+ *   physical function of the secondary EPC
  */
 struct pci_epf {
 	struct device		dev;
+	struct device_node	*node;
 	const char		*name;
 	struct pci_epf_header	*header;
 	struct pci_epf_bar	bar[6];
 	u8			msi_interrupts;
 	u16			msix_interrupts;
 	u8			func_no;
+	u8			vfunc_no;
+
+	struct dma_chan         *dma_chan;
+	struct completion       transfer_complete;
 
 	struct pci_epc		*epc;
+	struct pci_epf		*epf_pf;
 	struct pci_epf_driver	*driver;
 	struct list_head	list;
+	struct notifier_block   nb;
+	/* mutex to protect against concurrent access of pci_epf_ops */
+	struct mutex		lock;
+	unsigned int		is_bound;
+	unsigned int		is_vf;
+	unsigned long		vfunction_num_map;
+	struct list_head	pci_vepf;
+
+	/* Below members are to attach secondary EPC to an endpoint function */
+	struct pci_epc		*sec_epc;
+	struct list_head	sec_epc_list;
+	struct pci_epf_bar	sec_epc_bar[6];
+	u8			sec_epc_func_no;
+	u8			sec_epc_vfunc_no;
+	unsigned long		sec_epc_vfunction_num_map;
+	struct list_head	sec_epc_pci_vepf;
+};
+
+/**
+ * struct pci_epf_msix_tbl - represents the MSI-X table entry structure
+ * @msg_addr: Writes to this address will trigger MSI-X interrupt in host
+ * @msg_data: Data that should be written to @msg_addr to trigger MSI-X interrupt
+ * @vector_ctrl: Identifies if the function is prohibited from sending a message
+ * using this MSI-X table entry
+ */
+struct pci_epf_msix_tbl {
+	u64 msg_addr;
+	u32 msg_data;
+	u32 vector_ctrl;
 };
 
 #define to_pci_epf(epf_dev) container_of((epf_dev), struct pci_epf, dev)
@@ -142,16 +204,26 @@
 	return dev_get_drvdata(&epf->dev);
 }
 
-const struct pci_epf_device_id *
-pci_epf_match_device(const struct pci_epf_device_id *id, struct pci_epf *epf);
 struct pci_epf *pci_epf_create(const char *name);
+struct pci_epf *pci_epf_of_create(struct device_node *node);
+struct pci_epf *devm_pci_epf_of_create(struct device *dev,
+				       struct device_node *node);
 void pci_epf_destroy(struct pci_epf *epf);
 int __pci_epf_register_driver(struct pci_epf_driver *driver,
 			      struct module *owner);
 void pci_epf_unregister_driver(struct pci_epf_driver *driver);
-void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar);
-void pci_epf_free_space(struct pci_epf *epf, void *addr, enum pci_barno bar);
+void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar,
+			  size_t align, enum pci_epc_interface_type type);
+void pci_epf_free_space(struct pci_epf *epf, void *addr, enum pci_barno bar,
+			enum pci_epc_interface_type type);
 int pci_epf_bind(struct pci_epf *epf);
 void pci_epf_unbind(struct pci_epf *epf);
-void pci_epf_linkup(struct pci_epf *epf);
+int pci_epf_init_dma_chan(struct pci_epf *epf);
+void pci_epf_clean_dma_chan(struct pci_epf *epf);
+int pci_epf_data_transfer(struct pci_epf *epf, dma_addr_t dma_dst,
+			  dma_addr_t dma_src, size_t len);
+int pci_epf_tx(struct pci_epf *epf, dma_addr_t dma_dst, dma_addr_t dma_src,
+	       size_t len);
+int pci_epf_add_vepf(struct pci_epf *epf_pf, struct pci_epf *epf_vf);
+void pci_epf_remove_vepf(struct pci_epf *epf_pf, struct pci_epf *epf_vf);
 #endif /* __LINUX_PCI_EPF_H */
diff -Naur --no-dereference a/include/linux/pci.h b/include/linux/pci.h
--- a/include/linux/pci.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/pci.h	2020-06-19 21:00:31.432227716 -0400
@@ -1707,6 +1707,12 @@
 				      unsigned long *out_hwirq,
 				      unsigned int *out_type)
 { return -EINVAL; }
+
+static inline void __iomem *devm_pci_remap_cfg_resource(struct device *dev,
+							struct resource *res)
+{
+	return NULL;
+}
 #endif /* CONFIG_PCI */
 
 /* Include architecture-dependent settings and functions */
diff -Naur --no-dereference a/include/linux/pci_ids.h b/include/linux/pci_ids.h
--- a/include/linux/pci_ids.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/pci_ids.h	2020-06-19 21:00:31.432227716 -0400
@@ -868,6 +868,7 @@
 #define PCI_DEVICE_ID_TI_X620		0xac8d
 #define PCI_DEVICE_ID_TI_X420		0xac8e
 #define PCI_DEVICE_ID_TI_XX20_FM	0xac8f
+#define PCI_DEVICE_ID_TI_J721E		0xb00d
 #define PCI_DEVICE_ID_TI_DRA74x		0xb500
 #define PCI_DEVICE_ID_TI_DRA72x		0xb501
 
diff -Naur --no-dereference a/include/linux/phy/omap_usb.h b/include/linux/phy/omap_usb.h
--- a/include/linux/phy/omap_usb.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/phy/omap_usb.h	2020-06-19 21:00:31.432227716 -0400
@@ -66,6 +66,7 @@
 #define OMAP_USB2_HAS_START_SRP (1 << 0)
 #define OMAP_USB2_HAS_SET_VBUS (1 << 1)
 #define OMAP_USB2_CALIBRATE_FALSE_DISCONNECT (1 << 2)
+#define OMAP_USB2_DISABLE_CHG_DET (1 << 3)
 
 #define OMAP_DEV_PHY_PD		BIT(0)
 #define OMAP_USB2_PHY_PD	BIT(28)
diff -Naur --no-dereference a/include/linux/phy/phy-dp.h b/include/linux/phy/phy-dp.h
--- a/include/linux/phy/phy-dp.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/linux/phy/phy-dp.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,92 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2019 Cadence Design Systems Inc.
+ */
+
+#ifndef __PHY_DP_H_
+#define __PHY_DP_H_
+
+/**
+ * struct phy_configure_opts_dp - DisplayPort PHY configuration set
+ *
+ * This structure is used to represent the configuration state of a
+ * DisplayPort phy.
+ */
+struct phy_configure_opts_dp {
+	/**
+	 * @link_rate:
+	 *
+	 * Link Rate, in Mb/s, of the main link.
+	 *
+	 * Allowed values: 1620, 2160, 2430, 2700, 3240, 4320, 5400, 8100 Mb/s
+	 */
+	unsigned int		link_rate;
+
+	/**
+	 * @lanes:
+	 *
+	 * Number of active, consecutive, data lanes, starting from
+	 * lane 0, used for the transmissions on main link.
+	 *
+	 * Allowed values: 1, 2, 4
+	 */
+	unsigned int		lanes;
+
+	/**
+	 * @voltage:
+	 *
+	 * Voltage swing levels, as specified by DisplayPort specification, to be
+	 * used by particular lanes. One value per lane.
+	 * voltage[0] is for lane 0, voltage[1] is for lane 1, etc.
+	 *
+	 * Maximum value: 3
+	 */
+	unsigned int		voltage[4];
+
+	/**
+	 * @pre:
+	 *
+	 * Pre-emphasis levels, as specified by DisplayPort specification, to be
+	 * used by particular lanes. One value per lane.
+	 *
+	 * Maximum value: 3
+	 */
+	unsigned int		pre[4];
+
+	/**
+	 * @ssc:
+	 *
+	 * Flag indicating, whether or not to enable spread-spectrum clocking.
+	 *
+	 */
+	bool		ssc;
+
+	/**
+	 * @set_rate:
+	 *
+	 * Flag indicating, whether or not reconfigure link rate and SSC to
+	 * requested values.
+	 *
+	 */
+	bool		set_rate;
+
+	/**
+	 * @set_lanes:
+	 *
+	 * Flag indicating, whether or not reconfigure lane count to requested value.
+	 *
+	 */
+	bool		set_lanes;
+
+	/**
+	 * @set_voltages:
+	 *
+	 * Flag indicating, whether or not reconfigure voltage swing and pre-emphasis
+	 * to requested values. Only lanes specified by "lanes" parameter will be
+	 * affected.
+	 *
+	 */
+	bool		set_voltages;
+};
+
+#endif /* __PHY_DP_H_ */
diff -Naur --no-dereference a/include/linux/phy/phy.h b/include/linux/phy/phy.h
--- a/include/linux/phy/phy.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/phy/phy.h	2020-06-19 21:00:31.432227716 -0400
@@ -20,6 +20,9 @@
 #include <linux/pm_runtime.h>
 #include <linux/regulator/consumer.h>
 
+#include <linux/phy/phy-mipi-dphy.h>
+#include <linux/phy/phy-dp.h>
+
 struct phy;
 
 enum phy_mode {
@@ -35,11 +38,24 @@
 	PHY_MODE_USB_DEVICE_HS,
 	PHY_MODE_USB_DEVICE_SS,
 	PHY_MODE_USB_OTG,
-	PHY_MODE_SGMII,
-	PHY_MODE_2500SGMII,
-	PHY_MODE_10GKR,
 	PHY_MODE_UFS_HS_A,
 	PHY_MODE_UFS_HS_B,
+	PHY_MODE_PCIE,
+	PHY_MODE_ETHERNET,
+	PHY_MODE_MIPI_DPHY,
+};
+
+/**
+ * union phy_configure_opts - Opaque generic phy configuration
+ *
+ * @mipi_dphy:	Configuration set applicable for phys supporting
+ *		the MIPI_DPHY phy mode.
+ * @dp:	Configuration set applicable for phys supporting
+ *		the DisplayPort protocol.
+ */
+union phy_configure_opts {
+	struct phy_configure_opts_mipi_dphy	mipi_dphy;
+	struct phy_configure_opts_dp	dp;
 };
 
 /**
@@ -51,6 +67,7 @@
  * @set_mode: set the mode of the phy
  * @reset: resetting the phy
  * @calibrate: calibrate the phy
+ * @release: ops to be performed while the consumer reliquishes the PHY
  * @owner: the module owner containing the ops
  */
 struct phy_ops {
@@ -58,9 +75,41 @@
 	int	(*exit)(struct phy *phy);
 	int	(*power_on)(struct phy *phy);
 	int	(*power_off)(struct phy *phy);
-	int	(*set_mode)(struct phy *phy, enum phy_mode mode);
+	int	(*set_mode)(struct phy *phy, enum phy_mode mode, int submode);
+
+	/**
+	 * @configure:
+	 *
+	 * Optional.
+	 *
+	 * Used to change the PHY parameters. phy_init() must have
+	 * been called on the phy.
+	 *
+	 * Returns: 0 if successful, an negative error code otherwise
+	 */
+	int	(*configure)(struct phy *phy, union phy_configure_opts *opts);
+
+	/**
+	 * @validate:
+	 *
+	 * Optional.
+	 *
+	 * Used to check that the current set of parameters can be
+	 * handled by the phy. Implementations are free to tune the
+	 * parameters passed as arguments if needed by some
+	 * implementation detail or constraints. It must not change
+	 * any actual configuration of the PHY, so calling it as many
+	 * times as deemed fit by the consumer must have no side
+	 * effect.
+	 *
+	 * Returns: 0 if the configuration can be applied, an negative
+	 * error code otherwise
+	 */
+	int	(*validate)(struct phy *phy, enum phy_mode mode, int submode,
+			    union phy_configure_opts *opts);
 	int	(*reset)(struct phy *phy);
 	int	(*calibrate)(struct phy *phy);
+	void	(*release)(struct phy *phy);
 	struct module *owner;
 };
 
@@ -162,7 +211,13 @@
 int phy_exit(struct phy *phy);
 int phy_power_on(struct phy *phy);
 int phy_power_off(struct phy *phy);
-int phy_set_mode(struct phy *phy, enum phy_mode mode);
+int phy_set_mode_ext(struct phy *phy, enum phy_mode mode, int submode);
+#define phy_set_mode(phy, mode) \
+	phy_set_mode_ext(phy, mode, 0)
+int phy_configure(struct phy *phy, union phy_configure_opts *opts);
+int phy_validate(struct phy *phy, enum phy_mode mode, int submode,
+		 union phy_configure_opts *opts);
+
 static inline enum phy_mode phy_get_mode(struct phy *phy)
 {
 	return phy->attrs.mode;
@@ -276,13 +331,17 @@
 	return -ENOSYS;
 }
 
-static inline int phy_set_mode(struct phy *phy, enum phy_mode mode)
+static inline int phy_set_mode_ext(struct phy *phy, enum phy_mode mode,
+				   int submode)
 {
 	if (!phy)
 		return 0;
 	return -ENOSYS;
 }
 
+#define phy_set_mode(phy, mode) \
+	phy_set_mode_ext(phy, mode, 0)
+
 static inline enum phy_mode phy_get_mode(struct phy *phy)
 {
 	return PHY_MODE_INVALID;
@@ -302,6 +361,24 @@
 	return -ENOSYS;
 }
 
+static inline int phy_configure(struct phy *phy,
+				union phy_configure_opts *opts)
+{
+	if (!phy)
+		return 0;
+
+	return -ENOSYS;
+}
+
+static inline int phy_validate(struct phy *phy, enum phy_mode mode, int submode,
+			       union phy_configure_opts *opts)
+{
+	if (!phy)
+		return 0;
+
+	return -ENOSYS;
+}
+
 static inline int phy_get_bus_width(struct phy *phy)
 {
 	return -ENOSYS;
diff -Naur --no-dereference a/include/linux/phy/phy-mipi-dphy.h b/include/linux/phy/phy-mipi-dphy.h
--- a/include/linux/phy/phy-mipi-dphy.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/linux/phy/phy-mipi-dphy.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,279 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2018 Cadence Design Systems Inc.
+ */
+
+#ifndef __PHY_MIPI_DPHY_H_
+#define __PHY_MIPI_DPHY_H_
+
+#include <video/videomode.h>
+
+/**
+ * struct phy_configure_opts_mipi_dphy - MIPI D-PHY configuration set
+ *
+ * This structure is used to represent the configuration state of a
+ * MIPI D-PHY phy.
+ */
+struct phy_configure_opts_mipi_dphy {
+	/**
+	 * @clk_miss:
+	 *
+	 * Timeout, in picoseconds, for receiver to detect absence of
+	 * Clock transitions and disable the Clock Lane HS-RX.
+	 *
+	 * Maximum value: 60000 ps
+	 */
+	unsigned int		clk_miss;
+
+	/**
+	 * @clk_post:
+	 *
+	 * Time, in picoseconds, that the transmitter continues to
+	 * send HS clock after the last associated Data Lane has
+	 * transitioned to LP Mode. Interval is defined as the period
+	 * from the end of @hs_trail to the beginning of @clk_trail.
+	 *
+	 * Minimum value: 60000 ps + 52 * @hs_clk_rate period in ps
+	 */
+	unsigned int		clk_post;
+
+	/**
+	 * @clk_pre:
+	 *
+	 * Time, in UI, that the HS clock shall be driven by
+	 * the transmitter prior to any associated Data Lane beginning
+	 * the transition from LP to HS mode.
+	 *
+	 * Minimum value: 8 UI
+	 */
+	unsigned int		clk_pre;
+
+	/**
+	 * @clk_prepare:
+	 *
+	 * Time, in picoseconds, that the transmitter drives the Clock
+	 * Lane LP-00 Line state immediately before the HS-0 Line
+	 * state starting the HS transmission.
+	 *
+	 * Minimum value: 38000 ps
+	 * Maximum value: 95000 ps
+	 */
+	unsigned int		clk_prepare;
+
+	/**
+	 * @clk_settle:
+	 *
+	 * Time interval, in picoseconds, during which the HS receiver
+	 * should ignore any Clock Lane HS transitions, starting from
+	 * the beginning of @clk_prepare.
+	 *
+	 * Minimum value: 95000 ps
+	 * Maximum value: 300000 ps
+	 */
+	unsigned int		clk_settle;
+
+	/**
+	 * @clk_term_en:
+	 *
+	 * Time, in picoseconds, for the Clock Lane receiver to enable
+	 * the HS line termination.
+	 *
+	 * Maximum value: 38000 ps
+	 */
+	unsigned int		clk_term_en;
+
+	/**
+	 * @clk_trail:
+	 *
+	 * Time, in picoseconds, that the transmitter drives the HS-0
+	 * state after the last payload clock bit of a HS transmission
+	 * burst.
+	 *
+	 * Minimum value: 60000 ps
+	 */
+	unsigned int		clk_trail;
+
+	/**
+	 * @clk_zero:
+	 *
+	 * Time, in picoseconds, that the transmitter drives the HS-0
+	 * state prior to starting the Clock.
+	 */
+	unsigned int		clk_zero;
+
+	/**
+	 * @d_term_en:
+	 *
+	 * Time, in picoseconds, for the Data Lane receiver to enable
+	 * the HS line termination.
+	 *
+	 * Maximum value: 35000 ps + 4 * @hs_clk_rate period in ps
+	 */
+	unsigned int		d_term_en;
+
+	/**
+	 * @eot:
+	 *
+	 * Transmitted time interval, in picoseconds, from the start
+	 * of @hs_trail or @clk_trail, to the start of the LP- 11
+	 * state following a HS burst.
+	 *
+	 * Maximum value: 105000 ps + 12 * @hs_clk_rate period in ps
+	 */
+	unsigned int		eot;
+
+	/**
+	 * @hs_exit:
+	 *
+	 * Time, in picoseconds, that the transmitter drives LP-11
+	 * following a HS burst.
+	 *
+	 * Minimum value: 100000 ps
+	 */
+	unsigned int		hs_exit;
+
+	/**
+	 * @hs_prepare:
+	 *
+	 * Time, in picoseconds, that the transmitter drives the Data
+	 * Lane LP-00 Line state immediately before the HS-0 Line
+	 * state starting the HS transmission.
+	 *
+	 * Minimum value: 40000 ps + 4 * @hs_clk_rate period in ps
+	 * Maximum value: 85000 ps + 6 * @hs_clk_rate period in ps
+	 */
+	unsigned int		hs_prepare;
+
+	/**
+	 * @hs_settle:
+	 *
+	 * Time interval, in picoseconds, during which the HS receiver
+	 * shall ignore any Data Lane HS transitions, starting from
+	 * the beginning of @hs_prepare.
+	 *
+	 * Minimum value: 85000 ps + 6 * @hs_clk_rate period in ps
+	 * Maximum value: 145000 ps + 10 * @hs_clk_rate period in ps
+	 */
+	unsigned int		hs_settle;
+
+	/**
+	 * @hs_skip:
+	 *
+	 * Time interval, in picoseconds, during which the HS-RX
+	 * should ignore any transitions on the Data Lane, following a
+	 * HS burst. The end point of the interval is defined as the
+	 * beginning of the LP-11 state following the HS burst.
+	 *
+	 * Minimum value: 40000 ps
+	 * Maximum value: 55000 ps + 4 * @hs_clk_rate period in ps
+	 */
+	unsigned int		hs_skip;
+
+	/**
+	 * @hs_trail:
+	 *
+	 * Time, in picoseconds, that the transmitter drives the
+	 * flipped differential state after last payload data bit of a
+	 * HS transmission burst
+	 *
+	 * Minimum value: max(8 * @hs_clk_rate period in ps,
+	 *		      60000 ps + 4 * @hs_clk_rate period in ps)
+	 */
+	unsigned int		hs_trail;
+
+	/**
+	 * @hs_zero:
+	 *
+	 * Time, in picoseconds, that the transmitter drives the HS-0
+	 * state prior to transmitting the Sync sequence.
+	 */
+	unsigned int		hs_zero;
+
+	/**
+	 * @init:
+	 *
+	 * Time, in picoseconds for the initialization period to
+	 * complete.
+	 *
+	 * Minimum value: 100000000 ps
+	 */
+	unsigned int		init;
+
+	/**
+	 * @lpx:
+	 *
+	 * Transmitted length, in picoseconds, of any Low-Power state
+	 * period.
+	 *
+	 * Minimum value: 50000 ps
+	 */
+	unsigned int		lpx;
+
+	/**
+	 * @ta_get:
+	 *
+	 * Time, in picoseconds, that the new transmitter drives the
+	 * Bridge state (LP-00) after accepting control during a Link
+	 * Turnaround.
+	 *
+	 * Value: 5 * @lpx
+	 */
+	unsigned int		ta_get;
+
+	/**
+	 * @ta_go:
+	 *
+	 * Time, in picoseconds, that the transmitter drives the
+	 * Bridge state (LP-00) before releasing control during a Link
+	 * Turnaround.
+	 *
+	 * Value: 4 * @lpx
+	 */
+	unsigned int		ta_go;
+
+	/**
+	 * @ta_sure:
+	 *
+	 * Time, in picoseconds, that the new transmitter waits after
+	 * the LP-10 state before transmitting the Bridge state
+	 * (LP-00) during a Link Turnaround.
+	 *
+	 * Minimum value: @lpx
+	 * Maximum value: 2 * @lpx
+	 */
+	unsigned int		ta_sure;
+
+	/**
+	 * @wakeup:
+	 *
+	 * Time, in picoseconds, that a transmitter drives a Mark-1
+	 * state prior to a Stop state in order to initiate an exit
+	 * from ULPS.
+	 *
+	 * Minimum value: 1000000000 ps
+	 */
+	unsigned int		wakeup;
+
+	/**
+	 * @hs_clk_rate:
+	 *
+	 * Clock rate, in Hertz, of the high-speed clock.
+	 */
+	unsigned long		hs_clk_rate;
+
+	/**
+	 * @lp_clk_rate:
+	 *
+	 * Clock rate, in Hertz, of the low-power clock.
+	 */
+	unsigned long		lp_clk_rate;
+
+	/**
+	 * @lanes:
+	 *
+	 * Number of active data lanes used for the transmissions.
+	 */
+	unsigned char		lanes;
+};
+
+#endif /* __PHY_MIPI_DPHY_H_ */
diff -Naur --no-dereference a/include/linux/phy.h b/include/linux/phy.h
--- a/include/linux/phy.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/phy.h	2020-06-19 21:00:31.432227716 -0400
@@ -232,7 +232,8 @@
 	/* GPIO reset pulse width in microseconds */
 	int reset_delay_us;
 	/* RESET GPIO descriptor pointer */
-	struct gpio_desc *reset_gpiod;
+	int num_resets;
+	struct gpio_desc *reset_gpiod[PHY_MAX_ADDR];
 };
 #define to_mii_bus(d) container_of(d, struct mii_bus, dev)
 
diff -Naur --no-dereference a/include/linux/platform_data/davinci_asp.h b/include/linux/platform_data/davinci_asp.h
--- a/include/linux/platform_data/davinci_asp.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/platform_data/davinci_asp.h	2020-06-19 21:00:31.432227716 -0400
@@ -79,6 +79,7 @@
 	/* McASP specific fields */
 	int tdm_slots;
 	u8 op_mode;
+	u8 dismod;
 	u8 num_serializer;
 	u8 *serial_dir;
 	u8 version;
diff -Naur --no-dereference a/include/linux/platform_data/dmtimer-omap.h b/include/linux/platform_data/dmtimer-omap.h
--- a/include/linux/platform_data/dmtimer-omap.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/platform_data/dmtimer-omap.h	2020-06-19 21:00:31.432227716 -0400
@@ -28,6 +28,7 @@
 	int	(*free)(struct omap_dm_timer *timer);
 
 	void	(*enable)(struct omap_dm_timer *timer);
+	int	(*is_enabled)(struct omap_dm_timer *timer);
 	void	(*disable)(struct omap_dm_timer *timer);
 
 	int	(*get_irq)(struct omap_dm_timer *timer);
diff -Naur --no-dereference a/include/linux/platform_data/iommu-omap.h b/include/linux/platform_data/iommu-omap.h
--- a/include/linux/platform_data/iommu-omap.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/platform_data/iommu-omap.h	2020-06-19 21:00:31.432227716 -0400
@@ -16,4 +16,9 @@
 	const char *reset_name;
 	int (*assert_reset)(struct platform_device *pdev, const char *name);
 	int (*deassert_reset)(struct platform_device *pdev, const char *name);
+	int (*device_enable)(struct platform_device *pdev);
+	bool (*device_is_enabled)(struct platform_device *pdev);
+	int (*device_idle)(struct platform_device *pdev);
+	int (*set_pwrdm_constraint)(struct platform_device *pdev, bool request,
+				    u8 *pwrst);
 };
diff -Naur --no-dereference a/include/linux/platform_data/pm33xx.h b/include/linux/platform_data/pm33xx.h
--- a/include/linux/platform_data/pm33xx.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/platform_data/pm33xx.h	2020-06-19 21:00:31.432227716 -0400
@@ -46,11 +46,18 @@
 };
 
 struct am33xx_pm_platform_data {
-	int	(*init)(void);
+	int     (*init)(int (*idle)(u32 wfi_flags));
+	int     (*deinit)(void);
 	int	(*soc_suspend)(unsigned int state, int (*fn)(unsigned long),
 			       unsigned long args);
+	int	(*cpu_suspend)(int (*fn)(unsigned long), unsigned long args);
 	struct  am33xx_pm_sram_addr *(*get_sram_addrs)(void);
 	void __iomem *(*get_rtc_base_addr)(void);
+	void (*save_context)(void);
+	void (*restore_context)(void);
+	void (*prepare_rtc_suspend)(void);
+	void (*prepare_rtc_resume)(void);
+	int (*check_off_mode_enable)(void);
 };
 
 struct am33xx_pm_sram_data {
diff -Naur --no-dereference a/include/linux/platform_data/remoteproc-omap.h b/include/linux/platform_data/remoteproc-omap.h
--- a/include/linux/platform_data/remoteproc-omap.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/platform_data/remoteproc-omap.h	2020-06-19 21:00:31.432227716 -0400
@@ -1,59 +1,26 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Remote Processor - omap-specific bits
  *
- * Copyright (C) 2011 Texas Instruments, Inc.
+ * Copyright (C) 2011-2019 Texas Instruments Incorporated - http://www.ti.com/
  * Copyright (C) 2011 Google, Inc.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #ifndef _PLAT_REMOTEPROC_H
 #define _PLAT_REMOTEPROC_H
 
-struct rproc_ops;
 struct platform_device;
 
 /*
  * struct omap_rproc_pdata - omap remoteproc's platform data
- * @name: the remoteproc's name
- * @oh_name: omap hwmod device
- * @oh_name_opt: optional, secondary omap hwmod device
- * @firmware: name of firmware file to load
- * @mbox_name: name of omap mailbox device to use with this rproc
- * @ops: start/stop rproc handlers
  * @device_enable: omap-specific handler for enabling a device
  * @device_shutdown: omap-specific handler for shutting down a device
- * @set_bootaddr: omap-specific handler for setting the rproc boot address
+ * @device_is_enabled: omap-specific handler to check if device is booted
  */
 struct omap_rproc_pdata {
-	const char *name;
-	const char *oh_name;
-	const char *oh_name_opt;
-	const char *firmware;
-	const char *mbox_name;
-	const struct rproc_ops *ops;
 	int (*device_enable)(struct platform_device *pdev);
 	int (*device_shutdown)(struct platform_device *pdev);
-	void (*set_bootaddr)(u32);
+	bool (*device_is_enabled)(struct platform_device *pdev);
 };
 
-#if defined(CONFIG_OMAP_REMOTEPROC) || defined(CONFIG_OMAP_REMOTEPROC_MODULE)
-
-void __init omap_rproc_reserve_cma(void);
-
-#else
-
-static inline void __init omap_rproc_reserve_cma(void)
-{
-}
-
-#endif
-
 #endif /* _PLAT_REMOTEPROC_H */
diff -Naur --no-dereference a/include/linux/platform_data/sgx-omap.h b/include/linux/platform_data/sgx-omap.h
--- a/include/linux/platform_data/sgx-omap.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/linux/platform_data/sgx-omap.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * SGX Graphics Driver Platform Data
+ *
+ * Copyright (C) 2014-2019 Texas Instruments Incorporated - http://www.ti.com/
+ *	Darren Etheridge <detheridge@ti.com>
+ *
+ */
+
+#ifndef __SGX_OMAP_H__
+#define __SGX_OMAP_H__
+
+#include <linux/platform_device.h>
+
+struct gfx_sgx_platform_data {
+	const char *reset_name;
+
+	int (*assert_reset)(struct platform_device *pdev, const char *name);
+	int (*deassert_reset)(struct platform_device *pdev, const char *name);
+};
+
+#endif
diff -Naur --no-dereference a/include/linux/platform_data/spi-omap2-mcspi.h b/include/linux/platform_data/spi-omap2-mcspi.h
--- a/include/linux/platform_data/spi-omap2-mcspi.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/platform_data/spi-omap2-mcspi.h	2020-06-19 21:00:31.432227716 -0400
@@ -11,6 +11,7 @@
 	unsigned short	num_cs;
 	unsigned int regs_offset;
 	unsigned int pin_dir:1;
+	size_t max_xfer_len;
 };
 
 struct omap2_mcspi_device_config {
diff -Naur --no-dereference a/include/linux/platform_data/ti-pruss.h b/include/linux/platform_data/ti-pruss.h
--- a/include/linux/platform_data/ti-pruss.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/linux/platform_data/ti-pruss.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Platform data for PRUSS on TI SoCs
+ *
+ * Copyright (C) 2014-2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+#ifndef _PLAT_TI_PRUSS_H
+#define _PLAT_TI_PRUSS_H
+
+struct platform_device;
+
+/**
+ * struct pruss_platform_data - PRUSS platform data
+ * @reset_name: name of the reset
+ * @assert_reset: PRU-specific handler for putting the device in reset
+ * @deassert_reset: PRU-specific handler for releasing the device from reset
+ */
+struct pruss_platform_data {
+	const char *reset_name;
+	int (*assert_reset)(struct platform_device *pdev, const char *name);
+	int (*deassert_reset)(struct platform_device *pdev, const char *name);
+};
+
+#endif /* _PLAT_TI_PRUSS_H */
diff -Naur --no-dereference a/include/linux/platform_device.h b/include/linux/platform_device.h
--- a/include/linux/platform_device.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/platform_device.h	2020-06-19 21:00:31.432227716 -0400
@@ -51,6 +51,9 @@
 extern void arch_setup_pdev_archdata(struct platform_device *);
 extern struct resource *platform_get_resource(struct platform_device *,
 					      unsigned int, unsigned int);
+extern void __iomem *
+devm_platform_ioremap_resource(struct platform_device *pdev,
+			       unsigned int index);
 extern int platform_get_irq(struct platform_device *, unsigned int);
 extern int platform_irq_count(struct platform_device *);
 extern struct resource *platform_get_resource_byname(struct platform_device *,
diff -Naur --no-dereference a/include/linux/pruss_driver.h b/include/linux/pruss_driver.h
--- a/include/linux/pruss_driver.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/linux/pruss_driver.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,100 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * PRU-ICSS sub-system specific definitions
+ *
+ * Copyright (C) 2014-2019 Texas Instruments Incorporated - http://www.ti.com/
+ *	Suman Anna <s-anna@ti.com>
+ */
+
+#ifndef _PRUSS_DRIVER_H_
+#define _PRUSS_DRIVER_H_
+
+#include <linux/pruss.h>
+
+/* maximum number of system events */
+#define MAX_PRU_SYS_EVENTS	64
+#define MAX_PRU_SYS_EVENTS_K3	160
+
+/* maximum number of interrupt channels */
+#define MAX_PRU_CHANNELS	10
+#define MAX_PRU_CHANNELS_K3	20
+
+/* minimum starting host interrupt number for MPU */
+#define MIN_PRU_HOST_INT	2
+
+/* maximum number of host interrupts */
+#define MAX_PRU_HOST_INT	10
+#define MAX_PRU_HOST_INT_K3	20
+
+/**
+ * struct pruss_intc_config - INTC configuration info
+ * @sysev_to_ch: system events to channel mapping information
+ * @ch_to_host: interrupt channel to host interrupt information
+ */
+struct pruss_intc_config {
+	s8 sysev_to_ch[MAX_PRU_SYS_EVENTS_K3];
+	s8 ch_to_host[MAX_PRU_CHANNELS_K3];
+};
+
+/**
+ * struct pruss - PRUSS parent structure
+ * @dev: pruss device pointer
+ * @cfg_base: base iomap for CFG region
+ * @cfg: regmap for config region
+ * @mem_regions: data for each of the PRUSS memory regions
+ * @mem_in_use: to indicate if memory resource is in use
+ * @lock: mutex to serialize access to resources
+ */
+struct pruss {
+	struct device *dev;
+	void __iomem *cfg_base;
+	struct regmap *cfg;
+	struct pruss_mem_region mem_regions[PRUSS_MEM_MAX];
+	struct pruss_mem_region *mem_in_use[PRUSS_MEM_MAX];
+	struct mutex lock; /* PRU resource lock */
+	struct clk *core_clk_mux;
+	struct clk *iep_clk_mux;
+};
+
+int pruss_intc_configure(struct pruss *pruss,
+			 struct pruss_intc_config *intc_config);
+int pruss_intc_unconfigure(struct pruss *pruss,
+			   struct pruss_intc_config *intc_config);
+
+/**
+ * pruss_cfg_get_gpmux() - get the current GPMUX value for a PRU device
+ * @pruss: pruss instance
+ * @id: PRU identifier (0-1)
+ * @mux: pointer to store the current mux value into
+ */
+static inline int pruss_cfg_get_gpmux(struct pruss *pruss,
+				      enum pruss_pru_id id, u8 *mux)
+{
+	int ret = 0;
+	u32 val;
+
+	ret = pruss_cfg_read(pruss, PRUSS_CFG_GPCFG(id), &val);
+	if (!ret)
+		*mux = (u8)((val & PRUSS_GPCFG_PRU_MUX_SEL_MASK) >>
+			    PRUSS_GPCFG_PRU_MUX_SEL_SHIFT);
+	return ret;
+}
+
+/**
+ * pruss_cfg_set_gpmux() - set the GPMUX value for a PRU device
+ * @pruss: pruss instance
+ * @pru_id: PRU identifier (0-1)
+ * @mux: new mux value for PRU
+ */
+static inline int pruss_cfg_set_gpmux(struct pruss *pruss,
+				      enum pruss_pru_id id, u8 mux)
+{
+	if (mux >= PRUSS_GP_MUX_SEL_MAX)
+		return -EINVAL;
+
+	return pruss_cfg_update(pruss, PRUSS_CFG_GPCFG(id),
+				PRUSS_GPCFG_PRU_MUX_SEL_MASK,
+				(u32)mux << PRUSS_GPCFG_PRU_MUX_SEL_SHIFT);
+}
+
+#endif	/* _PRUSS_DRIVER_H_ */
diff -Naur --no-dereference a/include/linux/pruss.h b/include/linux/pruss.h
--- a/include/linux/pruss.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/linux/pruss.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,290 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/**
+ * PRU-ICSS Subsystem user interfaces
+ *
+ * Copyright (C) 2015-2019 Texas Instruments Incorporated - http://www.ti.com
+ *	Suman Anna <s-anna@ti.com>
+ *	Tero Kristo <t-kristo@ti.com>
+ */
+
+#ifndef __LINUX_PRUSS_H
+#define __LINUX_PRUSS_H
+
+/*
+ * PRU_ICSS_CFG registers
+ * SYSCFG, ISRP, ISP, IESP, IECP, SCRP applicable on AMxxxx devices only
+ */
+#define PRUSS_CFG_REVID		0x00
+#define PRUSS_CFG_SYSCFG	0x04
+#define PRUSS_CFG_GPCFG(x)	(0x08 + (x) * 4)
+#define PRUSS_CFG_CGR		0x10
+#define PRUSS_CFG_ISRP		0x14
+#define PRUSS_CFG_ISP		0x18
+#define PRUSS_CFG_IESP		0x1C
+#define PRUSS_CFG_IECP		0x20
+#define PRUSS_CFG_SCRP		0x24
+#define PRUSS_CFG_PMAO		0x28
+#define PRUSS_CFG_MII_RT	0x2C
+#define PRUSS_CFG_IEPCLK	0x30
+#define PRUSS_CFG_SPP		0x34
+#define PRUSS_CFG_PIN_MX	0x40
+
+#define ICSSG_CFG_CORE_SYNC	0x3c
+
+/* PRUSS_GPCFG register bits */
+#define PRUSS_GPCFG_PRU_GPO_SH_SEL		BIT(25)
+
+#define PRUSS_GPCFG_PRU_DIV1_SHIFT		20
+#define PRUSS_GPCFG_PRU_DIV1_MASK		GENMASK(24, 20)
+
+#define PRUSS_GPCFG_PRU_DIV0_SHIFT		15
+#define PRUSS_GPCFG_PRU_DIV0_MASK		GENMASK(15, 19)
+
+#define PRUSS_GPCFG_PRU_GPO_MODE		BIT(14)
+#define PRUSS_GPCFG_PRU_GPO_MODE_DIRECT		0
+#define PRUSS_GPCFG_PRU_GPO_MODE_SERIAL		BIT(14)
+
+#define PRUSS_GPCFG_PRU_GPI_SB			BIT(13)
+
+#define PRUSS_GPCFG_PRU_GPI_DIV1_SHIFT		8
+#define PRUSS_GPCFG_PRU_GPI_DIV1_MASK		GENMASK(12, 8)
+
+#define PRUSS_GPCFG_PRU_GPI_DIV0_SHIFT		3
+#define PRUSS_GPCFG_PRU_GPI_DIV0_MASK		GENMASK(7, 3)
+
+#define PRUSS_GPCFG_PRU_GPI_CLK_MODE_POSITIVE	0
+#define PRUSS_GPCFG_PRU_GPI_CLK_MODE_NEGATIVE	BIT(2)
+#define PRUSS_GPCFG_PRU_GPI_CLK_MODE		BIT(2)
+
+#define PRUSS_GPCFG_PRU_GPI_MODE_MASK		GENMASK(1, 0)
+#define PRUSS_GPCFG_PRU_GPI_MODE_SHIFT		0
+
+#define PRUSS_GPCFG_PRU_MUX_SEL_SHIFT		26
+#define PRUSS_GPCFG_PRU_MUX_SEL_MASK		GENMASK(29, 26)
+
+/* PRUSS_MII_RT register bits */
+#define PRUSS_MII_RT_EVENT_EN			BIT(0)
+
+/* PRUSS_SPP register bits */
+#define PRUSS_SPP_XFER_SHIFT_EN			BIT(1)
+#define PRUSS_SPP_PRU1_PAD_HP_EN		BIT(0)
+
+/* PRUSS_IEPCLK register bits */
+#define PRUSS_IEPCLK_IEP_OCP_CLK_EN		BIT(0)
+
+/* ICSSG CORE_SYNC register bits */
+#define ICSSG_CORE_VBUSP_SYNC_EN		BIT(0)
+
+/**
+ * enum pruss_gp_mux_sel - PRUSS GPI/O Mux modes for the
+ * PRUSS_GPCFG0/1 registers
+ *
+ * NOTE: The below defines are the most common values, but there
+ * are some exceptions like on 66AK2G, where the RESERVED and MII2
+ * values are interchanged. Also, this bit-field does not exist on
+ * AM335x SoCs
+ */
+enum pruss_gp_mux_sel {
+	PRUSS_GP_MUX_SEL_GP = 0,
+	PRUSS_GP_MUX_SEL_ENDAT,
+	PRUSS_GP_MUX_SEL_RESERVED,
+	PRUSS_GP_MUX_SEL_SD,
+	PRUSS_GP_MUX_SEL_MII2,
+	PRUSS_GP_MUX_SEL_MAX,
+};
+
+/**
+ * enum pruss_gpi_mode - PRUSS GPI configuration modes, used
+ *			 to program the PRUSS_GPCFG0/1 registers
+ */
+enum pruss_gpi_mode {
+	PRUSS_GPI_MODE_DIRECT = 0,
+	PRUSS_GPI_MODE_PARALLEL,
+	PRUSS_GPI_MODE_28BIT_SHIFT,
+	PRUSS_GPI_MODE_MII,
+};
+
+/**
+ * enum pruss_pru_id - PRU core identifiers
+ */
+enum pruss_pru_id {
+	PRUSS_PRU0 = 0,
+	PRUSS_PRU1,
+	PRUSS_NUM_PRUS,
+};
+
+/**
+ * enum pru_ctable_idx - Configurable Constant table index identifiers
+ */
+enum pru_ctable_idx {
+	PRU_C24 = 0,
+	PRU_C25,
+	PRU_C26,
+	PRU_C27,
+	PRU_C28,
+	PRU_C29,
+	PRU_C30,
+	PRU_C31,
+};
+
+/**
+ * enum pruss_mem - PRUSS memory range identifiers
+ */
+enum pruss_mem {
+	PRUSS_MEM_DRAM0 = 0,
+	PRUSS_MEM_DRAM1,
+	PRUSS_MEM_SHRD_RAM2,
+	PRUSS_MEM_MAX,
+};
+
+/**
+ * struct pruss_mem_region - PRUSS memory region structure
+ * @va: kernel virtual address of the PRUSS memory region
+ * @pa: physical (bus) address of the PRUSS memory region
+ * @size: size of the PRUSS memory region
+ */
+struct pruss_mem_region {
+	void __iomem *va;
+	phys_addr_t pa;
+	size_t size;
+};
+
+struct rproc;
+struct pruss;
+
+#if IS_ENABLED(CONFIG_TI_PRUSS)
+
+struct pruss *pruss_get(struct rproc *rproc);
+void pruss_put(struct pruss *pruss);
+int pruss_request_mem_region(struct pruss *pruss, enum pruss_mem mem_id,
+			     struct pruss_mem_region *region);
+int pruss_release_mem_region(struct pruss *pruss,
+			     struct pruss_mem_region *region);
+int pruss_cfg_read(struct pruss *pruss, unsigned int reg, unsigned int *val);
+int pruss_cfg_update(struct pruss *pruss, unsigned int reg,
+		     unsigned int mask, unsigned int val);
+int pruss_intc_trigger(unsigned int irq);
+
+#else
+
+static inline struct pruss *pruss_get(struct rproc *rproc)
+{
+	return ERR_PTR(-ENOTSUPP);
+}
+
+static inline void pruss_put(struct pruss *pruss) { }
+
+static inline int pruss_request_mem_region(struct pruss *pruss,
+					   enum pruss_mem mem_id,
+					   struct pruss_mem_region *region)
+{
+	return -ENOTSUPP;
+}
+
+static inline int pruss_release_mem_region(struct pruss *pruss,
+					   struct pruss_mem_region *region)
+{
+	return -ENOTSUPP;
+}
+
+static inline int pruss_cfg_read(struct pruss *pruss, unsigned int reg,
+				 unsigned int *val)
+{
+	return -ENOTSUPP;
+}
+
+static inline int pruss_cfg_update(struct pruss *pruss, unsigned int reg,
+				   unsigned int mask, unsigned int val)
+{
+	return -ENOTSUPP;
+}
+
+static inline int pruss_intc_trigger(unsigned int irq)
+{
+	return -ENOTSUPP;
+}
+
+#endif /* CONFIG_TI_PRUSS */
+
+#if IS_ENABLED(CONFIG_PRU_REMOTEPROC)
+
+struct rproc *pru_rproc_get(struct device_node *node, int index);
+void pru_rproc_put(struct rproc *rproc);
+enum pruss_pru_id pru_rproc_get_id(struct rproc *rproc);
+int pru_rproc_set_ctable(struct rproc *rproc, enum pru_ctable_idx c, u32 addr);
+
+#else
+
+static inline struct rproc *pru_rproc_get(struct device_node *node, int index)
+{
+	return ERR_PTR(-ENOTSUPP);
+}
+
+static inline void pru_rproc_put(struct rproc *rproc) { }
+
+static inline enum pruss_pru_id pru_rproc_get_id(struct rproc *rproc)
+{
+	return -ENOTSUPP;
+}
+
+static inline int pru_rproc_set_ctable(struct rproc *rproc,
+				       enum pru_ctable_idx c, u32 addr)
+{
+	return -ENOTSUPP;
+}
+
+#endif /* CONFIG_PRU_REMOTEPROC */
+
+/**
+ * pruss_cfg_gpimode() - set the GPI mode of the PRU
+ * @pruss: the pruss instance handle
+ * @pru: the rproc instance handle of the PRU
+ * @mode: GPI mode to set
+ *
+ * Sets the GPI mode for a given PRU by programming the
+ * corresponding PRUSS_CFG_GPCFGx register
+ *
+ * Returns 0 on success, or an error code otherwise
+ */
+static inline int pruss_cfg_gpimode(struct pruss *pruss, struct rproc *pru,
+				    enum pruss_gpi_mode mode)
+{
+	enum pruss_pru_id id = pru_rproc_get_id(pru);
+
+	if (id < 0)
+		return -EINVAL;
+
+	return pruss_cfg_update(pruss, PRUSS_CFG_GPCFG(id),
+				PRUSS_GPCFG_PRU_GPI_MODE_MASK,
+				mode << PRUSS_GPCFG_PRU_GPI_MODE_SHIFT);
+}
+
+/**
+ * pruss_cfg_miirt_enable() - Enable/disable MII RT Events
+ * @pruss: the pruss instance
+ * @enable: enable/disable
+ *
+ * Enable/disable the MII RT Events for the PRUSS.
+ */
+static inline int pruss_cfg_miirt_enable(struct pruss *pruss, bool enable)
+{
+	u32 set = enable ? PRUSS_MII_RT_EVENT_EN : 0;
+
+	return pruss_cfg_update(pruss, PRUSS_CFG_MII_RT,
+				PRUSS_MII_RT_EVENT_EN, set);
+}
+
+/**
+ * pruss_cfg_xfr_enable() - Enable/disable XIN XOUT shift functionality
+ * @pruss: the pruss instance
+ * @enable: enable/disable
+ */
+static inline int pruss_cfg_xfr_enable(struct pruss *pruss, bool enable)
+{
+	u32 set = enable ? PRUSS_SPP_XFER_SHIFT_EN : 0;
+
+	return pruss_cfg_update(pruss, PRUSS_CFG_SPP,
+				PRUSS_SPP_XFER_SHIFT_EN, set);
+}
+
+#endif /* __LINUX_PRUSS_H */
diff -Naur --no-dereference a/include/linux/pwm.h b/include/linux/pwm.h
--- a/include/linux/pwm.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/pwm.h	2020-06-19 21:00:31.432227716 -0400
@@ -291,6 +291,7 @@
  */
 struct pwm_chip {
 	struct device *dev;
+	struct device *sysfs_dev;
 	struct list_head list;
 	const struct pwm_ops *ops;
 	int base;
diff -Naur --no-dereference a/include/linux/remoteproc.h b/include/linux/remoteproc.h
--- a/include/linux/remoteproc.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/remoteproc.h	2020-06-19 21:00:31.432227716 -0400
@@ -41,6 +41,7 @@
 #include <linux/completion.h>
 #include <linux/idr.h>
 #include <linux/of.h>
+#include <linux/bitops.h>
 
 /**
  * struct resource_table - firmware resource table header
@@ -86,7 +87,13 @@
  * this header, and it should be parsed according to the resource type.
  */
 struct fw_rsc_hdr {
-	u32 type;
+	union {
+		u32 type;
+		struct {
+			u16 t;
+			u16 v;
+		} st;
+	};
 	u8 data[0];
 } __packed;
 
@@ -100,6 +107,10 @@
  *		    the remote processor will be writing logs.
  * @RSC_VDEV:       declare support for a virtio device, and serve as its
  *		    virtio header.
+ * @RSC_PRELOAD_VENDOR: a vendor resource type that needs to be handled by
+ *		    remoteproc implementations before loading
+ * @RSC_POSTLOAD_VENDOR: a vendor resource type that needs to be handled by
+ *		    remoteproc implementations after loading
  * @RSC_LAST:       just keep this one at the end
  *
  * For more details regarding a specific resource type, please see its
@@ -111,11 +122,13 @@
  * please update it as needed.
  */
 enum fw_resource_type {
-	RSC_CARVEOUT	= 0,
-	RSC_DEVMEM	= 1,
-	RSC_TRACE	= 2,
-	RSC_VDEV	= 3,
-	RSC_LAST	= 4,
+	RSC_CARVEOUT		= 0,
+	RSC_DEVMEM		= 1,
+	RSC_TRACE		= 2,
+	RSC_VDEV		= 3,
+	RSC_PRELOAD_VENDOR	= 4,
+	RSC_POSTLOAD_VENDOR	= 5,
+	RSC_LAST		= 6,
 };
 
 #define FW_RSC_ADDR_ANY (-1)
@@ -234,6 +247,32 @@
 } __packed;
 
 /**
+ * struct fw_rsc_trace2 - trace buffer declaration supporting 64-bits
+ * @padding: initial padding after type field for aligned 64-bit access
+ * @da: device address (64-bit)
+ * @len: length (in bytes)
+ * @reserved: reserved (must be zero)
+ * @name: human-readable name of the trace buffer
+ *
+ * This resource entry is an enhanced version of the fw_rsc_trace resourec entry
+ * and the provides equivalent functionality but designed for 64-bit remote
+ * processors.
+ *
+ * @da specifies the device address of the buffer, @len specifies
+ * its size, and @name may contain a human readable name of the trace buffer.
+ *
+ * After booting the remote processor, the trace buffers are exposed to the
+ * user via debugfs entries (called trace0, trace1, etc..).
+ */
+struct fw_rsc_trace2 {
+	u32 padding;
+	u64 da;
+	u32 len;
+	u32 reserved;
+	u8 name[32];
+} __packed;
+
+/**
  * struct fw_rsc_vdev_vring - vring descriptor entry
  * @da: device address
  * @align: the alignment between the consumer and producer parts of the vring
@@ -306,12 +345,31 @@
 } __packed;
 
 /**
+ * struct fw_rsc_vendor - vendor resource definition
+ * @sub_type: implementation specific type including version field
+ * @size: size of the vendor custom resource
+ * @data: label for the start of the resource
+ */
+struct fw_rsc_vendor {
+	union {
+		u32 sub_type;
+		struct {
+			u16 st_type;
+			u16 st_ver;
+		} st;
+	} u;
+	u32 size;
+	u8 data[0];
+} __packed;
+
+/**
  * struct rproc_mem_entry - memory entry descriptor
  * @va:	virtual address
  * @dma: dma address
  * @len: length, in bytes
  * @da: device address
  * @priv: associated data
+ * @name: associated memory region name (optional)
  * @node: list node
  */
 struct rproc_mem_entry {
@@ -320,33 +378,54 @@
 	int len;
 	u32 da;
 	void *priv;
+	char name[32];
 	struct list_head node;
 };
 
 struct rproc;
 struct firmware;
 
+/*
+ * Macros to use with flags field in rproc_da_to_va API. Use
+ * the upper 16 bits to dictate the flags type and the lower
+ * 16 bits to pass on the value of the flags pertinent to that
+ * type.
+ *
+ * Add any new flags type at a new bit-field position
+ */
+#define RPROC_FLAGS_SHIFT	16
+#define RPROC_FLAGS_NONE	0
+#define RPROC_FLAGS_ELF_PHDR	BIT(0 + RPROC_FLAGS_SHIFT)
+#define RPROC_FLAGS_ELF_SHDR	BIT(1 + RPROC_FLAGS_SHIFT)
+
 /**
  * struct rproc_ops - platform-specific device handlers
+ * @prepare:	prepare device for code loading
+ * @unprepare:	unprepare device after stop
  * @start:	power on the device and boot it
  * @stop:	power off the device
  * @kick:	kick a virtqueue (virtqueue id given as a parameter)
  * @da_to_va:	optional platform hook to perform address translations
  * @load_rsc_table:	load resource table from firmware image
  * @find_loaded_rsc_table: find the loaded resouce table
+ * @handle_vendor_rsc:	hook to handle device specific resource table entries
  * @load:		load firmeware to memory, where the remote processor
  *			expects to find it
  * @sanity_check:	sanity check the fw image
  * @get_boot_addr:	get boot address to entry point specified in firmware
  */
 struct rproc_ops {
+	int (*prepare)(struct rproc *rproc);
+	int (*unprepare)(struct rproc *rproc);
 	int (*start)(struct rproc *rproc);
 	int (*stop)(struct rproc *rproc);
 	void (*kick)(struct rproc *rproc, int vqid);
-	void * (*da_to_va)(struct rproc *rproc, u64 da, int len);
+	void * (*da_to_va)(struct rproc *rproc, u64 da, int len, u32 flags);
 	int (*parse_fw)(struct rproc *rproc, const struct firmware *fw);
 	struct resource_table *(*find_loaded_rsc_table)(
 				struct rproc *rproc, const struct firmware *fw);
+	int (*handle_vendor_rsc)(struct rproc *rproc,
+				 struct fw_rsc_vendor *rsc);
 	int (*load)(struct rproc *rproc, const struct firmware *fw);
 	int (*sanity_check)(struct rproc *rproc, const struct firmware *fw);
 	u32 (*get_boot_addr)(struct rproc *rproc, const struct firmware *fw);
@@ -424,6 +503,8 @@
  * @dbg_dir: debugfs directory of this rproc device
  * @traces: list of trace buffers
  * @num_traces: number of trace buffers
+ * @last_traces: list of last trace buffers
+ * @num_last_traces: number of last trace buffers
  * @carveouts: list of physically contiguous memory allocations
  * @mappings: list of iommu mappings we initiated, needed on shutdown
  * @bootaddr: address of first instruction to boot rproc with (optional)
@@ -433,18 +514,24 @@
  * @index: index of this rproc device
  * @crash_handler: workqueue for handling a crash
  * @crash_cnt: crash counter
+ * @crash_comp: completion used to sync crash handler and the rproc reload
  * @recovery_disabled: flag that state if recovery was disabled
  * @max_notifyid: largest allocated notify id.
  * @table_ptr: pointer to the resource table in effect
  * @cached_table: copy of the resource table
  * @table_sz: size of @cached_table
  * @has_iommu: flag to indicate if remote processor is behind an MMU
+ * @auto_boot: flag to indicate if remote processor should be auto-started
+ * @deny_sysfs_ops: flag to not permit sysfs operations on state and firmware
+ * @skip_firmware_request: flag to skip requesting the firmware
+ * @skip_load: flag to skip the loading of firmware segments
+ * @late_attach: flag indicating remote core has been externally pre-booted
  * @dump_segments: list of segments in the firmware
  */
 struct rproc {
 	struct list_head node;
 	struct iommu_domain *domain;
-	const char *name;
+	char *name;
 	char *firmware;
 	void *priv;
 	struct rproc_ops *ops;
@@ -455,6 +542,8 @@
 	struct dentry *dbg_dir;
 	struct list_head traces;
 	int num_traces;
+	struct list_head last_traces;
+	int num_last_traces;
 	struct list_head carveouts;
 	struct list_head mappings;
 	u32 bootaddr;
@@ -464,6 +553,7 @@
 	int index;
 	struct work_struct crash_handler;
 	unsigned int crash_cnt;
+	struct completion crash_comp;
 	bool recovery_disabled;
 	int max_notifyid;
 	struct resource_table *table_ptr;
@@ -471,6 +561,10 @@
 	size_t table_sz;
 	bool has_iommu;
 	bool auto_boot;
+	unsigned int deny_sysfs_ops		: 1;
+	unsigned int skip_firmware_request	: 1;
+	unsigned int skip_load			: 1;
+	unsigned int late_attach		: 1;
 	struct list_head dump_segments;
 };
 
@@ -555,8 +649,12 @@
 
 int rproc_boot(struct rproc *rproc);
 void rproc_shutdown(struct rproc *rproc);
+int rproc_set_firmware(struct rproc *rproc, const char *fw_name);
 void rproc_report_crash(struct rproc *rproc, enum rproc_crash_type type);
+void *rproc_da_to_va(struct rproc *rproc, u64 da, int len, u32 flags);
 int rproc_coredump_add_segment(struct rproc *rproc, dma_addr_t da, size_t size);
+int rproc_get_id(struct rproc *rproc);
+int rproc_pa_to_da(struct rproc *rproc, phys_addr_t pa, u64 *da);
 
 static inline struct rproc_vdev *vdev_to_rvdev(struct virtio_device *vdev)
 {
diff -Naur --no-dereference a/include/linux/rpmsg/virtio_rpmsg.h b/include/linux/rpmsg/virtio_rpmsg.h
--- a/include/linux/rpmsg/virtio_rpmsg.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/linux/rpmsg/virtio_rpmsg.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: BSD-3-Clause */
+
+#ifndef _LINUX_RPMSG_VIRTIO_RPMSG_H
+#define _LINUX_RPMSG_VIRTIO_RPMSG_H
+
+/**
+ * struct rpmsg_hdr - common header for all virtio rpmsg messages
+ * @src: source address
+ * @dst: destination address
+ * @reserved: reserved for future use
+ * @len: length of payload (in bytes)
+ * @flags: message flags
+ * @data: @len bytes of message payload data
+ *
+ * Every message sent(/received) on the rpmsg bus begins with this header.
+ */
+struct rpmsg_hdr {
+	u32 src;
+	u32 dst;
+	u32 reserved;
+	u16 len;
+	u16 flags;
+	u8 data[0];
+} __packed;
+
+#endif
diff -Naur --no-dereference a/include/linux/rpmsg.h b/include/linux/rpmsg.h
--- a/include/linux/rpmsg.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/rpmsg.h	2020-06-19 21:00:31.432227716 -0400
@@ -25,14 +25,20 @@
 struct rpmsg_device_ops;
 struct rpmsg_endpoint_ops;
 
+/* lockdep subclasses for use with ept cb_lock mutex nested calls */
+#define RPMSG_LOCKDEP_SUBCLASS_NORMAL   0 /* regular ept cb_lock */
+#define RPMSG_LOCKDEP_SUBCLASS_NS       1 /* name service ept cb_lock */
+
 /**
  * struct rpmsg_channel_info - channel info representation
  * @name: name of service
+ * @desc: description of service
  * @src: local address
  * @dst: destination address
  */
 struct rpmsg_channel_info {
 	char name[RPMSG_NAME_SIZE];
+	char desc[RPMSG_NAME_SIZE];
 	u32 src;
 	u32 dst;
 };
@@ -42,6 +48,7 @@
  * @dev: the device struct
  * @id: device id (used to match between rpmsg drivers and devices)
  * @driver_override: driver name to force a match
+ * @desc: description of remote service
  * @src: local address
  * @dst: destination address
  * @ept: the rpmsg endpoint of this channel
@@ -51,6 +58,7 @@
 	struct device dev;
 	struct rpmsg_device_id id;
 	char *driver_override;
+	char desc[RPMSG_NAME_SIZE];
 	u32 src;
 	u32 dst;
 	struct rpmsg_endpoint *ept;
@@ -67,6 +75,7 @@
  * @refcount: when this drops to zero, the ept is deallocated
  * @cb: rx callback handler
  * @cb_lock: must be taken before accessing/changing @cb
+ * @cb_lockdep_class: mutex lockdep class to be used with @cb_lock
  * @addr: local rpmsg address
  * @priv: private data for the driver's use
  *
@@ -89,6 +98,7 @@
 	struct kref refcount;
 	rpmsg_rx_cb_t cb;
 	struct mutex cb_lock;
+	int cb_lockdep_class;
 	u32 addr;
 	void *priv;
 
diff -Naur --no-dereference a/include/linux/rpmsg-remotedev/rpmsg-remotedev.h b/include/linux/rpmsg-remotedev/rpmsg-remotedev.h
--- a/include/linux/rpmsg-remotedev/rpmsg-remotedev.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/linux/rpmsg-remotedev/rpmsg-remotedev.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,185 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Subhajit Paul <subhajit_paul@ti.com>
+ */
+
+#ifndef __RPMSG_REMOTEDEV_H__
+#define __RPMSG_REMOTEDEV_H__
+
+#include <linux/etherdevice.h>
+
+#define RPMSG_REMOTEDEV_DISPLAY_MAX_PLANES		(3)
+#define RPMSG_REMOTEDEV_DISPLAY_MAX_DISPS		(8)
+#define RPMSG_REMOTEDEV_DISPLAY_MAX_PIPES		(8)
+#define RPMSG_REMOTEDEV_DISPLAY_MAX_FORMATS		(32)
+#define RPMSG_REMOTEDEV_DISPLAY_MAX_ZORDERS		(8)
+
+struct rpmsg_remotedev;
+
+struct rpmsg_remotedev_display_buffer {
+	uint32_t width;
+	uint32_t height;
+	uint32_t format;
+	uint32_t num_planes;
+	dma_addr_t planes[RPMSG_REMOTEDEV_DISPLAY_MAX_PLANES];
+	uint32_t pitches[RPMSG_REMOTEDEV_DISPLAY_MAX_PLANES];
+	struct rpmsg_remotedev *rdev;
+	void *priv;
+};
+
+struct rpmsg_remotedev_display_pipe_update {
+	uint32_t pipe_id;
+	bool enabled;
+	uint32_t dst_w;
+	uint32_t dst_h;
+	uint32_t dst_x;
+	uint32_t dst_y;
+	struct rpmsg_remotedev_display_buffer *buffer;
+};
+
+struct rpmsg_remotedev_display_commit {
+	uint32_t disp_id;
+	uint32_t num_pipe_updates;
+	struct rpmsg_remotedev_display_pipe_update pipes[RPMSG_REMOTEDEV_DISPLAY_MAX_PIPES];
+	struct rpmsg_remotedev *rdev;
+	void *priv;
+};
+
+struct rpmsg_remotedev_display_pipe {
+	uint32_t pipe_id;
+	bool can_scale;
+	bool can_mod_win;
+	uint32_t fixed_win_x;
+	uint32_t fixed_win_y;
+	uint32_t fixed_win_w;
+	uint32_t fixed_win_h;
+	uint32_t initial_zorder;
+	uint32_t num_formats;
+	uint32_t formats[RPMSG_REMOTEDEV_DISPLAY_MAX_FORMATS];
+	uint32_t num_allowed_zorders;
+	uint32_t allowed_zorders[RPMSG_REMOTEDEV_DISPLAY_MAX_ZORDERS];
+};
+
+struct rpmsg_remotedev_display_disp {
+	uint32_t disp_id;
+	uint32_t width;
+	uint32_t height;
+	uint32_t refresh;
+	uint32_t num_pipes;
+	struct rpmsg_remotedev_display_pipe pipes[RPMSG_REMOTEDEV_DISPLAY_MAX_PIPES];
+};
+
+struct rpmsg_remotedev_display_resinfo {
+	uint32_t num_disps;
+	struct rpmsg_remotedev_display_disp disps[RPMSG_REMOTEDEV_DISPLAY_MAX_DISPS];
+};
+
+struct rpmsg_remotedev_display_cb {
+	void (*commit_done)(struct rpmsg_remotedev_display_commit *commit, void *cb_data);
+	void (*buffer_done)(struct rpmsg_remotedev_display_buffer *buffer, void *cb_data);
+};
+
+struct rpmsg_remotedev_display_ops {
+	bool (*ready)(struct rpmsg_remotedev *rdev);
+	int (*get_res_info)(struct rpmsg_remotedev *rdev, struct rpmsg_remotedev_display_resinfo *res);
+	int (*commit)(struct rpmsg_remotedev *rdev, struct rpmsg_remotedev_display_commit *commit);
+};
+
+#define RPMSG_RDEV_ETHSWITCH_CPSW_PRIORITY_NUM   (8)
+
+struct rpmsg_rdev_eth_switch_attach_info {
+	/* MTU of rx packet */
+	u32 rx_mtu;
+	/* MTU of tx packet per priority */
+	u32 tx_mtu[RPMSG_RDEV_ETHSWITCH_CPSW_PRIORITY_NUM];
+	/* Supported Features mask */
+	u32 features;
+#define RPMSG_KDRV_ETHSWITCH_FEATURE_TXCSUM BIT(0)
+#define RPMSG_KDRV_ETHSWITCH_FEATURE_DUMP_STATS BIT(1)
+};
+
+struct rpmsg_rdev_eth_switch_attach_ext_info {
+	/* MTU of rx packet */
+	u32 rx_mtu;
+	/* MTU of tx packet per priority */
+	u32 tx_mtu[RPMSG_RDEV_ETHSWITCH_CPSW_PRIORITY_NUM];
+	/* Supported Features mask */
+	u32 features;
+#define RPMSG_KDRV_ETHSWITCH_FEATURE_TXCSUM BIT(0)
+#define RPMSG_KDRV_ETHSWITCH_FEATURE_DUMP_STATS BIT(1)
+	u32 flow_idx;
+	u32 tx_cpsw_psil_dst_id;
+	u8 mac_addr[ETH_ALEN];
+};
+
+struct rpmsg_rdev_eth_switch_tx_info {
+	/* Tx PSIL Peer destination thread id */
+	u32 tx_cpsw_psil_dst_id;
+};
+
+struct rpmsg_rdev_eth_switch_rx_info {
+	/* Allocated flow's index */
+	u32 flow_idx;
+};
+
+struct rpmsg_remotedev_eth_switch_ops {
+	void (*get_fw_ver)(struct rpmsg_remotedev *rdev,
+			   char *buf, size_t size);
+	int (*attach)(struct rpmsg_remotedev *rdev,
+		      struct rpmsg_rdev_eth_switch_attach_info *attach_info);
+	int (*attach_ext)(struct rpmsg_remotedev *rdev,
+			  struct rpmsg_rdev_eth_switch_attach_ext_info *attach_ext_info);
+	int (*detach)(struct rpmsg_remotedev *rdev);
+	int (*get_tx_info)(struct rpmsg_remotedev *rdev,
+			   struct rpmsg_rdev_eth_switch_tx_info *info);
+	int (*get_rx_info)(struct rpmsg_remotedev *rdev,
+			   struct rpmsg_rdev_eth_switch_rx_info *info);
+	int (*get_mac)(struct rpmsg_remotedev *rdev, void *mac_addr);
+	int (*register_mac)(struct rpmsg_remotedev *rdev,
+			    void *mac_addr, u32 flow_idx_offset);
+	int (*unregister_mac)(struct rpmsg_remotedev *rdev,
+			      void *mac_addr, u32 flow_idx_offset);
+	int (*register_ipv4)(struct rpmsg_remotedev *rdev,
+			     void *mac_addr, __be32 ipv4);
+	int (*unregister_ipv4)(struct rpmsg_remotedev *rdev, __be32 ipv4);
+	int (*ping)(struct rpmsg_remotedev *rdev, const u8 *data, int size);
+	int (*read_reg)(struct rpmsg_remotedev *rdev, u32 reg_addr, u32 *val);
+	int (*dbg_dump_stats)(struct rpmsg_remotedev *rdev);
+};
+
+enum rpmsg_remotedev_type {
+	RPMSG_REMOTEDEV_DISPLAY_DEVICE,
+	RPMSG_REMOTEDEV_ETH_SWITCH_DEVICE,
+};
+
+struct rpmsg_remotedev {
+	enum rpmsg_remotedev_type type;
+	union {
+		struct {
+			const struct rpmsg_remotedev_display_ops *ops;
+			const struct rpmsg_remotedev_display_cb *cb_ops;
+		} display;
+
+		struct {
+			struct rpmsg_remotedev_eth_switch_ops *ops;
+		} eth_switch;
+	} device;
+	void *cb_data;
+};
+
+#if IS_REACHABLE(CONFIG_RPMSG_KDRV)
+extern struct rpmsg_remotedev *rpmsg_remotedev_get_named_device(const char *device_name);
+extern void rpmsg_remotedev_put_device(struct rpmsg_remotedev *rdev);
+#else
+static inline struct rpmsg_remotedev * __maybe_unused rpmsg_remotedev_get_named_device(const char *device_name)
+{
+	return NULL;
+}
+
+static inline void __maybe_unused rpmsg_remotedev_put_device(struct rpmsg_remotedev *rdev)
+{
+}
+#endif
+
+#endif
diff -Naur --no-dereference a/include/linux/rpmsg_rpc.h b/include/linux/rpmsg_rpc.h
--- a/include/linux/rpmsg_rpc.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/linux/rpmsg_rpc.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,80 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */
+/*
+ * Remote Processor Procedure Call Driver
+ *
+ * Copyright (C) 2012-2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+#ifndef _LINUX_RPMSG_RPC_H_
+#define _LINUX_RPMSG_RPC_H_
+
+#include <uapi/linux/rpmsg_rpc.h>
+
+#define RPPC_MAX_NUM_FUNCS		(1024)
+#define RPPC_MAX_CHANNEL_NAMELEN	(64)
+#define RPPC_MAX_FUNC_NAMELEN		(64)
+#define RPPC_MAX_NUM_PARAMS		(10)
+
+/**
+ * enum rppc_param_direction - direction of the function parameter
+ * @RPPC_PARAMDIR_IN: input argument
+ * @RPPC_PARAMDIR_OUT: output argument
+ * @RPPC_PARAMDIR_BI: an in and out argument
+ * @RPPC_PARAMDIR_MAX: limit value for the direction type
+ *
+ * The parameter direction is described as relative to the function.
+ */
+enum rppc_param_direction {
+	RPPC_PARAMDIR_IN = 0,
+	RPPC_PARAMDIR_OUT,
+	RPPC_PARAMDIR_BI,
+	RPPC_PARAMDIR_MAX
+};
+
+/**
+ * enum rppc_param_datatype - parameter data type and descriptor flags
+ * @RPPC_PARAM_VOID: parameter is of type 'void'
+ * @RPPC_PARAM_S08: parameter is of type 's8'
+ * @RPPC_PARAM_U08: parameter is of type 'u8'
+ * @RPPC_PARAM_S16: parameter is of type 's16'
+ * @RPPC_PARAM_U16: parameter is of type 'u16'
+ * @RPPC_PARAM_S32: parameter is of type 's32'
+ * @RPPC_PARAM_U32: parameter is of type 'u32'
+ * @RPPC_PARAM_S64: parameter is of type 's64'
+ * @RPPC_PARAM_U64: parameter is of type 'u64'
+ * @RPPC_PARAM_ATOMIC_MAX: limit value for scalar data types
+ * @RPPC_PARAM_MASK: mask field for retrieving the scalar data type
+ * @RPPC_PARAM_PTR: flag to indicate the data type is a pointer
+ * @RPPC_PARAM_MAX: max limit value used as a marker
+ *
+ * This enum is used to describe the data type for the parameters.
+ * A pointer of a data type is reflected by using an additional bit
+ * mask field.
+ */
+enum rppc_param_datatype {
+	RPPC_PARAM_VOID = 0,
+	RPPC_PARAM_S08,
+	RPPC_PARAM_U08,
+	RPPC_PARAM_S16,
+	RPPC_PARAM_U16,
+	RPPC_PARAM_S32,
+	RPPC_PARAM_U32,
+	RPPC_PARAM_S64,
+	RPPC_PARAM_U64,
+	RPPC_PARAM_ATOMIC_MAX,
+
+	RPPC_PARAM_MASK = 0x7F,
+	RPPC_PARAM_PTR = 0x80,
+
+	RPPC_PARAM_MAX
+};
+
+/*
+ * helper macros to deal with parameter types
+ */
+#define RPPC_PTR_TYPE(type)	((type) | RPPC_PARAM_PTR)
+#define RPPC_IS_PTR(type)	((type) & RPPC_PARAM_PTR)
+#define RPPC_IS_ATOMIC(type)	(((type) > RPPC_PARAM_VOID) && \
+				 ((type) < RPPC_PARAM_ATOMIC_MAX))
+
+#endif /* _LINUX_RPMSG_RPC_H_ */
diff -Naur --no-dereference a/include/linux/rtc.h b/include/linux/rtc.h
--- a/include/linux/rtc.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/rtc.h	2020-06-19 21:00:31.432227716 -0400
@@ -85,6 +85,7 @@
 	int (*alarm_irq_enable)(struct device *, unsigned int enabled);
 	int (*read_offset)(struct device *, long *offset);
 	int (*set_offset)(struct device *, long offset);
+	int (*power_off_program)(struct device *dev);
 };
 
 struct rtc_timer {
@@ -216,6 +217,7 @@
 int rtc_read_offset(struct rtc_device *rtc, long *offset);
 int rtc_set_offset(struct rtc_device *rtc, long offset);
 void rtc_timer_do_work(struct work_struct *work);
+int rtc_power_off_program(struct rtc_device *rtc);
 
 static inline bool is_leap_year(unsigned int year)
 {
diff -Naur --no-dereference a/include/linux/soc/ti/k3-navss-desc-pool.h b/include/linux/soc/ti/k3-navss-desc-pool.h
--- a/include/linux/soc/ti/k3-navss-desc-pool.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/linux/soc/ti/k3-navss-desc-pool.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * CPPI5 descriptors pool
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
+ */
+
+#ifndef K3_NAVSS_DESC_POOL_H_
+#define K3_NAVSS_DESC_POOL_H_
+
+struct k3_knav_desc_pool;
+
+void k3_knav_pool_destroy(struct k3_knav_desc_pool *pool);
+struct k3_knav_desc_pool *k3_knav_pool_create_name(struct device *dev,
+						   size_t size,
+						   size_t desc_size,
+						   const char *name);
+#define k3_knav_pool_create(dev, size, desc_size) \
+		k3_knav_pool_create_name(dev, size, desc_size, NULL)
+dma_addr_t k3_knav_pool_virt2dma(struct k3_knav_desc_pool *pool, void *addr);
+void *k3_knav_pool_dma2virt(struct k3_knav_desc_pool *pool, dma_addr_t dma);
+void *k3_knav_pool_alloc(struct k3_knav_desc_pool *pool);
+void k3_knav_pool_free(struct k3_knav_desc_pool *pool, void *addr);
+size_t k3_knav_pool_avail(struct k3_knav_desc_pool *pool);
+
+#endif /* K3_NAVSS_DESC_POOL_H_ */
diff -Naur --no-dereference a/include/linux/soc/ti/k3-ringacc.h b/include/linux/soc/ti/k3-ringacc.h
--- a/include/linux/soc/ti/k3-ringacc.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/linux/soc/ti/k3-ringacc.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,258 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * K3 Ring Accelerator (RA) subsystem interface
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
+ */
+
+#ifndef __SOC_TI_K3_RINGACC_API_H_
+#define __SOC_TI_K3_RINGACC_API_H_
+
+/**
+ * enum k3_ring_mode - &struct k3_ring_cfg mode
+ *
+ * RA ring operational modes
+ *
+ * @K3_RINGACC_RING_MODE_RING: Exposed Ring mode for SW direct access
+ * @K3_RINGACC_RING_MODE_MESSAGE: Messaging mode. Messaging mode requires
+ *	that all accesses to the queue must go through this IP so that all
+ *	accesses to the memory are controlled and ordered. This IP then
+ *	controls the entire state of the queue, and SW has no directly control,
+ *	such as through doorbells and cannot access the storage memory directly.
+ *	This is particularly useful when more than one SW or HW entity can be
+ *	the producer and/or consumer at the same time
+ * @K3_RINGACC_RING_MODE_CREDENTIALS: Credentials mode is message mode plus
+ *	stores credentials with each message, requiring the element size to be
+ *	doubled to fit the credentials. Any exposed memory should be protected
+ *	by a firewall from unwanted access
+ * @K3_RINGACC_RING_MODE_QM:  Queue manager mode. This takes the credentials
+ *	mode and adds packet length per element, along with additional read only
+ *	fields for element count and accumulated queue length. The QM mode only
+ *	operates with an 8 byte element size (any other element size is
+ *	illegal), and like in credentials mode each operation uses 2 element
+ *	slots to store the credentials and length fields
+ */
+enum k3_ring_mode {
+	K3_RINGACC_RING_MODE_RING = 0,
+	K3_RINGACC_RING_MODE_MESSAGE,
+	K3_RINGACC_RING_MODE_CREDENTIALS,
+	K3_RINGACC_RING_MODE_QM,
+	K3_RINGACC_RING_MODE_INVALID
+};
+
+/**
+ * enum k3_ring_size - &struct k3_ring_cfg elm_size
+ *
+ * RA ring element's sizes in bytes.
+ */
+enum k3_ring_size {
+	K3_RINGACC_RING_ELSIZE_4 = 0,
+	K3_RINGACC_RING_ELSIZE_8,
+	K3_RINGACC_RING_ELSIZE_16,
+	K3_RINGACC_RING_ELSIZE_32,
+	K3_RINGACC_RING_ELSIZE_64,
+	K3_RINGACC_RING_ELSIZE_128,
+	K3_RINGACC_RING_ELSIZE_256,
+	K3_RINGACC_RING_ELSIZE_INVALID
+};
+
+struct k3_ringacc;
+struct k3_ring;
+
+/**
+ * enum k3_ring_cfg - RA ring configuration structure
+ *
+ * @size: Ring size, number of elements
+ * @elm_size: Ring element size
+ * @mode: Ring operational mode
+ * @flags: Ring configuration flags. Possible values:
+ *	 @K3_RINGACC_RING_SHARED: when set allows to request the same ring
+ *	 few times. It's usable when the same ring is used as Free Host PD ring
+ *	 for different flows, for example.
+ *	 Note: Locking should be done by consumer if required
+ */
+struct k3_ring_cfg {
+	u32 size;
+	enum k3_ring_size elm_size;
+	enum k3_ring_mode mode;
+#define K3_RINGACC_RING_SHARED BIT(1)
+	u32 flags;
+};
+
+#define K3_RINGACC_RING_ID_ANY (-1)
+
+/**
+ * of_k3_ringacc_get_by_phandle - find a RA by phandle property
+ * @np: device node
+ * @propname: property name containing phandle on RA node
+ *
+ * Returns pointer on the RA - struct k3_ringacc
+ * or -ENODEV if not found,
+ * or -EPROBE_DEFER if not yet registered
+ */
+struct k3_ringacc *of_k3_ringacc_get_by_phandle(struct device_node *np,
+						const char *property);
+
+#define K3_RINGACC_RING_USE_PROXY BIT(1)
+
+/**
+ * k3_ringacc_request_ring - request ring from ringacc
+ * @ringacc: pointer on ringacc
+ * @id: ring id or K3_RINGACC_RING_ID_ANY for any general purpose ring
+ * @flags:
+ *	@K3_RINGACC_RING_USE_PROXY: if set - proxy will be allocated and
+ *		used to access ring memory. Sopported only for rings in
+ *		Message/Credentials/Queue mode.
+ *
+ * Returns pointer on the Ring - struct k3_ring
+ * or NULL in case of failure.
+ */
+struct k3_ring *k3_ringacc_request_ring(struct k3_ringacc *ringacc,
+					int id, u32 flags);
+
+/**
+ * k3_ringacc_get_dev - get pointer on RA device
+ * @ringacc: pointer on RA
+ *
+ * Returns device pointer
+ */
+struct device *k3_ringacc_get_dev(struct k3_ringacc *ringacc);
+
+/**
+ * k3_ringacc_ring_reset - ring reset
+ * @ring: pointer on Ring
+ *
+ * Resets ring internal state ((hw)occ, (hw)idx).
+ * TODO_GS: ? Ring can be reused without reconfiguration
+ */
+void k3_ringacc_ring_reset(struct k3_ring *ring);
+/**
+ * k3_ringacc_ring_reset - ring reset for DMA rings
+ * @ring: pointer on Ring
+ *
+ * Resets ring internal state ((hw)occ, (hw)idx). Should be used for rings
+ * which are read by K3 UDMA, like TX or Free Host PD rings.
+ */
+void k3_ringacc_ring_reset_dma(struct k3_ring *ring, u32 occ);
+
+/**
+ * k3_ringacc_ring_free - ring free
+ * @ring: pointer on Ring
+ *
+ * Resets ring and free all alocated resources.
+ */
+int k3_ringacc_ring_free(struct k3_ring *ring);
+
+/**
+ * k3_ringacc_get_ring_id - Get the Ring ID
+ * @ring: pointer on ring
+ *
+ * Returns the Ring ID
+ */
+u32 k3_ringacc_get_ring_id(struct k3_ring *ring);
+
+/**
+ * k3_ringacc_ring_cfg - ring configure
+ * @ring: pointer on ring
+ * @cfg: Ring configuration parameters (see &struct k3_ring_cfg)
+ *
+ * Configures ring, including ring memory allocation.
+ * Returns 0 on success, errno otherwise.
+ */
+int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg);
+
+/**
+ * k3_ringacc_ring_get_size - get ring size
+ * @ring: pointer on ring
+ *
+ * Returns ring size in number of elements.
+ */
+u32 k3_ringacc_ring_get_size(struct k3_ring *ring);
+
+/**
+ * k3_ringacc_ring_get_free - get free elements
+ * @ring: pointer on ring
+ *
+ * Returns number of free elements in the ring.
+ */
+u32 k3_ringacc_ring_get_free(struct k3_ring *ring);
+
+/**
+ * k3_ringacc_ring_get_occ - get ring occupancy
+ * @ring: pointer on ring
+ *
+ * Returns total number of valid entries on the ring
+ */
+u32 k3_ringacc_ring_get_occ(struct k3_ring *ring);
+
+/**
+ * k3_ringacc_ring_is_full - checks if ring is full
+ * @ring: pointer on ring
+ *
+ * Returns true if the ring is full
+ */
+u32 k3_ringacc_ring_is_full(struct k3_ring *ring);
+
+/**
+ * k3_ringacc_ring_push - push element to the ring tail
+ * @ring: pointer on ring
+ * @elem: pointer on ring element buffer
+ *
+ * Push one ring element to the ring tail. Size of the ring element is
+ * determined by ring configuration &struct k3_ring_cfg elm_size.
+ *
+ * Returns 0 on success, errno otherwise.
+ */
+int k3_ringacc_ring_push(struct k3_ring *ring, void *elem);
+
+/**
+ * k3_ringacc_ring_pop - pop element from the ring head
+ * @ring: pointer on ring
+ * @elem: pointer on ring element buffer
+ *
+ * Push one ring element from the ring head. Size of the ring element is
+ * determined by ring configuration &struct k3_ring_cfg elm_size..
+ *
+ * Returns 0 on success, errno otherwise.
+ */
+int k3_ringacc_ring_pop(struct k3_ring *ring, void *elem);
+
+/**
+ * k3_ringacc_ring_push_head - push element to the ring head
+ * @ring: pointer on ring
+ * @elem: pointer on ring element buffer
+ *
+ * Push one ring element to the ring head. Size of the ring element is
+ * determined by ring configuration &struct k3_ring_cfg elm_size.
+ *
+ * Returns 0 on success, errno otherwise.
+ * Not Supported by ring modes: K3_RINGACC_RING_MODE_RING
+ */
+int k3_ringacc_ring_push_head(struct k3_ring *ring, void *elem);
+
+/**
+ * k3_ringacc_ring_pop_tail - pop element from the ring tail
+ * @ring: pointer on ring
+ * @elem: pointer on ring element buffer
+ *
+ * Push one ring element from the ring tail. Size of the ring element is
+ * determined by ring configuration &struct k3_ring_cfg elm_size.
+ *
+ * Returns 0 on success, errno otherwise.
+ * Not Supported by ring modes: K3_RINGACC_RING_MODE_RING
+ */
+int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem);
+
+u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring);
+
+/**
+ * Debugging definitions
+ * TODO: might be removed
+ */
+#ifdef CONFIG_TI_K3_RINGACC_DEBUG
+void k3_ringacc_ring_dump(struct k3_ring *ring);
+#else
+static inline void k3_ringacc_ring_dump(struct k3_ring *ring) {};
+#endif
+
+#endif /* __SOC_TI_K3_RINGACC_API_H_ */
diff -Naur --no-dereference a/include/linux/soc/ti/ti_sci_protocol.h b/include/linux/soc/ti/ti_sci_protocol.h
--- a/include/linux/soc/ti/ti_sci_protocol.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/soc/ti/ti_sci_protocol.h	2020-06-19 21:00:31.432227716 -0400
@@ -97,7 +97,10 @@
  */
 struct ti_sci_dev_ops {
 	int (*get_device)(const struct ti_sci_handle *handle, u32 id);
+	int (*get_device_exclusive)(const struct ti_sci_handle *handle, u32 id);
 	int (*idle_device)(const struct ti_sci_handle *handle, u32 id);
+	int (*idle_device_exclusive)(const struct ti_sci_handle *handle,
+				     u32 id);
 	int (*put_device)(const struct ti_sci_handle *handle, u32 id);
 	int (*is_valid)(const struct ti_sci_handle *handle, u32 id);
 	int (*get_context_loss_count)(const struct ti_sci_handle *handle,
@@ -166,41 +169,391 @@
  * managed by driver for that purpose.
  */
 struct ti_sci_clk_ops {
-	int (*get_clock)(const struct ti_sci_handle *handle, u32 did, u8 cid,
+	int (*get_clock)(const struct ti_sci_handle *handle, u32 did, u32 cid,
 			 bool needs_ssc, bool can_change_freq,
 			 bool enable_input_term);
-	int (*idle_clock)(const struct ti_sci_handle *handle, u32 did, u8 cid);
-	int (*put_clock)(const struct ti_sci_handle *handle, u32 did, u8 cid);
-	int (*is_auto)(const struct ti_sci_handle *handle, u32 did, u8 cid,
+	int (*idle_clock)(const struct ti_sci_handle *handle, u32 did, u32 cid);
+	int (*put_clock)(const struct ti_sci_handle *handle, u32 did, u32 cid);
+	int (*is_auto)(const struct ti_sci_handle *handle, u32 did, u32 cid,
 		       bool *req_state);
-	int (*is_on)(const struct ti_sci_handle *handle, u32 did, u8 cid,
+	int (*is_on)(const struct ti_sci_handle *handle, u32 did, u32 cid,
 		     bool *req_state, bool *current_state);
-	int (*is_off)(const struct ti_sci_handle *handle, u32 did, u8 cid,
+	int (*is_off)(const struct ti_sci_handle *handle, u32 did, u32 cid,
 		      bool *req_state, bool *current_state);
-	int (*set_parent)(const struct ti_sci_handle *handle, u32 did, u8 cid,
-			  u8 parent_id);
-	int (*get_parent)(const struct ti_sci_handle *handle, u32 did, u8 cid,
-			  u8 *parent_id);
+	int (*set_parent)(const struct ti_sci_handle *handle, u32 did, u32 cid,
+			  u32 parent_id);
+	int (*get_parent)(const struct ti_sci_handle *handle, u32 did, u32 cid,
+			  u32 *parent_id);
 	int (*get_num_parents)(const struct ti_sci_handle *handle, u32 did,
-			       u8 cid, u8 *num_parents);
+			       u32 cid, u32 *num_parents);
 	int (*get_best_match_freq)(const struct ti_sci_handle *handle, u32 did,
-				   u8 cid, u64 min_freq, u64 target_freq,
+				   u32 cid, u64 min_freq, u64 target_freq,
 				   u64 max_freq, u64 *match_freq);
-	int (*set_freq)(const struct ti_sci_handle *handle, u32 did, u8 cid,
+	int (*set_freq)(const struct ti_sci_handle *handle, u32 did, u32 cid,
 			u64 min_freq, u64 target_freq, u64 max_freq);
-	int (*get_freq)(const struct ti_sci_handle *handle, u32 did, u8 cid,
+	int (*get_freq)(const struct ti_sci_handle *handle, u32 did, u32 cid,
 			u64 *current_freq);
 };
 
 /**
+ * struct ti_sci_rm_core_ops - Resource management core operations
+ * @get_range:		Get a range of resources belonging to ti sci host.
+ * @get_rage_from_shost:	Get a range of resources belonging to
+ *				specified host id.
+ *			- s_host: Host processing entity to which the
+ *				  resources are allocated
+ *
+ * NOTE: for these functions, all the parameters are consolidated and defined
+ * as below:
+ * - handle:	Pointer to TISCI handle as retrieved by *ti_sci_get_handle
+ * - dev_id:	TISCI device ID.
+ * - subtype:	Resource assignment subtype that is being requested
+ *		from the given device.
+ * - range_start:	Start index of the resource range
+ * - range_end:		Number of resources in the range
+ */
+struct ti_sci_rm_core_ops {
+	int (*get_range)(const struct ti_sci_handle *handle, u32 dev_id,
+			 u8 subtype, u16 *range_start, u16 *range_num);
+	int (*get_range_from_shost)(const struct ti_sci_handle *handle,
+				    u32 dev_id, u8 subtype, u8 s_host,
+				    u16 *range_start, u16 *range_num);
+};
+
+/**
+ * struct ti_sci_rm_irq_ops: IRQ management operations
+ * @set_direct_irq:		Set Non-event Sourced direct irq to destination
+ *				host(same host as ti sci interface id).
+ * @set_event_irq:		Set Event based peripheral irq to destination
+ *				host(same host as ti sci interface id).
+ * @set_direct_irq_from_shost:	Set Non-event Sourced direct irq to a
+ *				specified destination host.
+ * @set_event_irq_from_shost:	Set Event based peripheral irq to a
+ *				specified destination host.
+ * @set_event_irq_to_poll:	Set Event based peripheral irq to polling mode.
+ *				vint_status_bit is used for polling.
+ * @free_direct_irq:		Free a non-event sourced direct irq to
+ *				destination host(same as ti sci interface id)
+ * @free_event_irq:		Free an event based peripheral irq to
+ *				destination host(same as ti sci interface id)
+ * @free_direct_irq_from_shost:	Free non-event based direct irq from a
+ *				specified destination host.
+ * @free_event_irq_from_shost:	Free event based peripheral irq from a
+ *				specified destination host.
+ * @free_event_irq_to_poll:	Free an event based peripheral irq that is
+ *				configured in polling mode.
+ *
+ * NOTE: for these functions, all the parameters are consolidated and defined
+ * as below:
+ * - handle:	Pointer to TISCI handle as retrieved by *ti_sci_get_handle
+ * - src_id:	Device ID of the IRQ source
+ * - src_index:	IRQ source index within the source device
+ * - dst_id:	Device ID of the IRQ destination.
+ * - dst_host_irq:	IRQ number of the destination device.
+ * - ia_id:	Device ID of the IA, if the IRQ flows through this IA
+ * - vint:	Virtual interrupt to be used within the IA
+ * - global_event:	Global event number to be used for the requesting event.
+ * - vint_status_bit:	Virtual interrupt status bit to be used for the event.
+ * - s_host:	Secondary host ID to which the irq/event is being requested.
+ */
+struct ti_sci_rm_irq_ops {
+	int (*set_direct_irq)(const struct ti_sci_handle *handle, u16 src_id,
+			      u16 src_index, u16 dst_id, u16 dst_host_irq);
+	int (*set_event_irq)(const struct ti_sci_handle *handle, u16 src_id,
+			     u16 src_index, u16 dst_id, u16 dst_host_irq,
+			     u16 ia_id, u16 vint, u16 global_event,
+			     u8 vint_status_bit);
+	int (*set_direct_irq_from_shost)(const struct ti_sci_handle *handle,
+					 u16 src_id, u16 src_index, u16 dst_id,
+					 u16 dst_host_irq, u8 s_host);
+	int (*set_event_irq_from_shost)(const struct ti_sci_handle *handle,
+					u16 src_id, u16 src_index, u16 dst_id,
+					u16 dst_host_irq, u16 ia_id, u16 vint,
+					u16 global_event, u8 vint_status_bit,
+					u8 s_host);
+	int (*set_event_irq_to_poll)(const struct ti_sci_handle *handle,
+				     u16 src_id, u16 src_index, u16 ia_id,
+				     u16 vint, u16 global_event,
+				     u8 vint_status_bit);
+	int (*free_direct_irq)(const struct ti_sci_handle *handle, u16 src_id,
+			       u16 src_index, u16 dst_id, u16 dst_host_irq);
+	int (*free_event_irq)(const struct ti_sci_handle *handle, u16 src_id,
+			      u16 src_index, u16 dst_id, u16 dst_host_irq,
+			      u16 ia_id, u16 vint, u16 global_event,
+			      u8 vint_status_bit);
+	int (*free_direct_irq_from_shost)(const struct ti_sci_handle *handle,
+					  u16 src_id, u16 src_index, u16 dst_id,
+					  u16 dst_host_irq, u8 s_host);
+	int (*free_event_irq_from_shost)(const struct ti_sci_handle *handle,
+					 u16 src_id, u16 src_index, u16 dst_id,
+					 u16 dst_host_irq, u16 ia_id, u16 vint,
+					 u16 global_event, u8 vint_status_bit,
+					 u8 s_host);
+	int (*free_event_irq_to_poll)(const struct ti_sci_handle *handle,
+				      u16 src_id, u16 src_index, u16 ia_id,
+				      u16 vint, u16 global_event,
+				      u8 vint_status_bit);
+};
+
+/**
+ * struct ti_sci_proc_ops - Processor Control operations
+ * @request:	Request to control a physical processor. The requesting host
+ *		should be in the processor access list
+ * @release:	Relinquish a physical processor control
+ * @handover:	Handover a physical processor control to another host
+ *		in the permitted list
+ * @set_config:	Set base configuration of a processor
+ * @set_control: Setup limited control flags in specific cases
+ * @get_status: Get the state of physical processor
+ *
+ * NOTE: The following paramteres are generic in nature for all these ops,
+ * -handle:	Pointer to TI SCI handle as retrieved by *ti_sci_get_handle
+ * -pid:	Processor ID
+ * -hid:	Host ID
+ */
+struct ti_sci_proc_ops {
+	int (*request)(const struct ti_sci_handle *handle, u8 pid);
+	int (*release)(const struct ti_sci_handle *handle, u8 pid);
+	int (*handover)(const struct ti_sci_handle *handle, u8 pid, u8 hid);
+	int (*set_config)(const struct ti_sci_handle *handle, u8 pid,
+			  u64 boot_vector, u32 cfg_set, u32 cfg_clr);
+	int (*set_control)(const struct ti_sci_handle *handle, u8 pid,
+			   u32 ctrl_set, u32 ctrl_clr);
+	int (*get_status)(const struct ti_sci_handle *handle, u8 pid,
+			  u64 *boot_vector, u32 *cfg_flags, u32 *ctrl_flags,
+			  u32 *status_flags);
+};
+
+/* RA config.addr_lo parameter is valid for RM ring configure TI_SCI message */
+#define TI_SCI_MSG_VALUE_RM_RING_ADDR_LO_VALID	BIT(0)
+/* RA config.addr_hi parameter is valid for RM ring configure TI_SCI message */
+#define TI_SCI_MSG_VALUE_RM_RING_ADDR_HI_VALID	BIT(1)
+ /* RA config.count parameter is valid for RM ring configure TI_SCI message */
+#define TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID	BIT(2)
+/* RA config.mode parameter is valid for RM ring configure TI_SCI message */
+#define TI_SCI_MSG_VALUE_RM_RING_MODE_VALID	BIT(3)
+/* RA config.size parameter is valid for RM ring configure TI_SCI message */
+#define TI_SCI_MSG_VALUE_RM_RING_SIZE_VALID	BIT(4)
+/* RA config.order_id parameter is valid for RM ring configure TISCI message */
+#define TI_SCI_MSG_VALUE_RM_RING_ORDER_ID_VALID	BIT(5)
+
+#define TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER \
+	(TI_SCI_MSG_VALUE_RM_RING_ADDR_LO_VALID | \
+	TI_SCI_MSG_VALUE_RM_RING_ADDR_HI_VALID | \
+	TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID | \
+	TI_SCI_MSG_VALUE_RM_RING_MODE_VALID | \
+	TI_SCI_MSG_VALUE_RM_RING_SIZE_VALID)
+
+/**
+ * struct ti_sci_rm_ringacc_ops - Ring Accelerator Management operations
+ * @config: configure the SoC Navigator Subsystem Ring Accelerator ring
+ * @get_config: get the SoC Navigator Subsystem Ring Accelerator ring
+ *		configuration
+ */
+struct ti_sci_rm_ringacc_ops {
+	int (*config)(const struct ti_sci_handle *handle,
+		      u32 valid_params, u16 nav_id, u16 index,
+		      u32 addr_lo, u32 addr_hi, u32 count, u8 mode,
+		      u8 size, u8 order_id
+	);
+	int (*get_config)(const struct ti_sci_handle *handle,
+			  u32 nav_id, u32 index, u8 *mode,
+			  u32 *addr_lo, u32 *addr_hi, u32 *count,
+			  u8 *size, u8 *order_id);
+};
+
+/**
+ * struct ti_sci_rm_psil_ops - PSI-L thread operations
+ * @pair: pair PSI-L source thread to a destination thread.
+ *	If the src_thread is mapped to UDMA tchan, the corresponding channel's
+ *	TCHAN_THRD_ID register is updated.
+ *	If the dst_thread is mapped to UDMA rchan, the corresponding channel's
+ *	RCHAN_THRD_ID register is updated.
+ * @unpair: unpair PSI-L source thread from a destination thread.
+ *	If the src_thread is mapped to UDMA tchan, the corresponding channel's
+ *	TCHAN_THRD_ID register is cleared.
+ *	If the dst_thread is mapped to UDMA rchan, the corresponding channel's
+ *	RCHAN_THRD_ID register is cleared.
+ */
+struct ti_sci_rm_psil_ops {
+	int (*pair)(const struct ti_sci_handle *handle, u32 nav_id,
+		    u32 src_thread, u32 dst_thread);
+	int (*unpair)(const struct ti_sci_handle *handle, u32 nav_id,
+		      u32 src_thread, u32 dst_thread);
+};
+
+/* UDMAP channel types */
+#define TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR		2
+#define TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR_SB		3	/* RX only */
+#define TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBRR		10
+#define TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBVR		11
+#define TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR	12
+#define TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBVR	13
+
+#define TI_SCI_RM_UDMAP_RX_FLOW_DESC_HOST		0
+#define TI_SCI_RM_UDMAP_RX_FLOW_DESC_MONO		2
+
+#define TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES	1
+#define TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_128_BYTES	2
+#define TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_256_BYTES	3
+
+/* UDMAP TX/RX channel valid_params common declarations */
+#define TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID		BIT(0)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID                BIT(1)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID            BIT(2)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID           BIT(3)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID              BIT(4)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_CH_PRIORITY_VALID             BIT(5)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_CH_QOS_VALID                  BIT(6)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_CH_ORDER_ID_VALID             BIT(7)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_CH_SCHED_PRIORITY_VALID       BIT(8)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_CH_BURST_SIZE_VALID		BIT(14)
+
+/**
+ * Configures a Navigator Subsystem UDMAP transmit channel
+ *
+ * Configures a Navigator Subsystem UDMAP transmit channel registers.
+ * See @ti_sci_msg_rm_udmap_tx_ch_cfg_req
+ */
+struct ti_sci_msg_rm_udmap_tx_ch_cfg {
+	u32 valid_params;
+#define TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID        BIT(9)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID      BIT(10)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID        BIT(11)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_CREDIT_COUNT_VALID      BIT(12)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FDEPTH_VALID            BIT(13)
+	u16 nav_id;
+	u16 index;
+	u8 tx_pause_on_err;
+	u8 tx_filt_einfo;
+	u8 tx_filt_pswords;
+	u8 tx_atype;
+	u8 tx_chan_type;
+	u8 tx_supr_tdpkt;
+	u16 tx_fetch_size;
+	u8 tx_credit_count;
+	u16 txcq_qnum;
+	u8 tx_priority;
+	u8 tx_qos;
+	u8 tx_orderid;
+	u16 fdepth;
+	u8 tx_sched_priority;
+	u8 tx_burst_size;
+};
+
+/**
+ * Configures a Navigator Subsystem UDMAP receive channel
+ *
+ * Configures a Navigator Subsystem UDMAP receive channel registers.
+ * See @ti_sci_msg_rm_udmap_rx_ch_cfg_req
+ */
+struct ti_sci_msg_rm_udmap_rx_ch_cfg {
+	u32 valid_params;
+#define TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID      BIT(9)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID        BIT(10)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_SHORT_VALID      BIT(11)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_LONG_VALID       BIT(12)
+	u16 nav_id;
+	u16 index;
+	u16 rx_fetch_size;
+	u16 rxcq_qnum;
+	u8 rx_priority;
+	u8 rx_qos;
+	u8 rx_orderid;
+	u8 rx_sched_priority;
+	u16 flowid_start;
+	u16 flowid_cnt;
+	u8 rx_pause_on_err;
+	u8 rx_atype;
+	u8 rx_chan_type;
+	u8 rx_ignore_short;
+	u8 rx_ignore_long;
+	u8 rx_burst_size;
+};
+
+/**
+ * Configures a Navigator Subsystem UDMAP receive flow
+ *
+ * Configures a Navigator Subsystem UDMAP receive flow's registers.
+ * See @tis_ci_msg_rm_udmap_flow_cfg_req
+ */
+struct ti_sci_msg_rm_udmap_flow_cfg {
+	u32 valid_params;
+#define TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID	BIT(0)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID     BIT(1)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID     BIT(2)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID          BIT(3)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SOP_OFFSET_VALID         BIT(4)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID          BIT(5)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_VALID         BIT(6)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_VALID         BIT(7)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_VALID        BIT(8)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_VALID        BIT(9)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID     BIT(10)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID     BIT(11)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID    BIT(12)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID    BIT(13)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID      BIT(14)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID          BIT(15)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID          BIT(16)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID          BIT(17)
+#define TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PS_LOCATION_VALID        BIT(18)
+	u16 nav_id;
+	u16 flow_index;
+	u8 rx_einfo_present;
+	u8 rx_psinfo_present;
+	u8 rx_error_handling;
+	u8 rx_desc_type;
+	u16 rx_sop_offset;
+	u16 rx_dest_qnum;
+	u8 rx_src_tag_hi;
+	u8 rx_src_tag_lo;
+	u8 rx_dest_tag_hi;
+	u8 rx_dest_tag_lo;
+	u8 rx_src_tag_hi_sel;
+	u8 rx_src_tag_lo_sel;
+	u8 rx_dest_tag_hi_sel;
+	u8 rx_dest_tag_lo_sel;
+	u16 rx_fdq0_sz0_qnum;
+	u16 rx_fdq1_qnum;
+	u16 rx_fdq2_qnum;
+	u16 rx_fdq3_qnum;
+	u8 rx_ps_location;
+};
+
+/**
+ * struct ti_sci_rm_udmap_ops - UDMA Management operations
+ * @tx_ch_cfg: configure SoC Navigator Subsystem UDMA transmit channel.
+ * @rx_ch_cfg: configure SoC Navigator Subsystem UDMA receive channel.
+ * @rx_flow_cfg1: configure SoC Navigator Subsystem UDMA receive flow.
+ */
+struct ti_sci_rm_udmap_ops {
+	int (*tx_ch_cfg)(const struct ti_sci_handle *handle,
+			 const struct ti_sci_msg_rm_udmap_tx_ch_cfg *params);
+	int (*rx_ch_cfg)(const struct ti_sci_handle *handle,
+			 const struct ti_sci_msg_rm_udmap_rx_ch_cfg *params);
+	int (*rx_flow_cfg)(const struct ti_sci_handle *handle,
+			   const struct ti_sci_msg_rm_udmap_flow_cfg *params);
+};
+
+/**
  * struct ti_sci_ops - Function support for TI SCI
  * @dev_ops:	Device specific operations
  * @clk_ops:	Clock specific operations
+ * @rm_core_ops:	Resource management core operations.
+ * @rm_irq_ops:		IRQ management specific operations
+ * @proc_ops:	Processor Control specific operations
  */
 struct ti_sci_ops {
 	struct ti_sci_core_ops core_ops;
 	struct ti_sci_dev_ops dev_ops;
 	struct ti_sci_clk_ops clk_ops;
+	struct ti_sci_rm_core_ops rm_core_ops;
+	struct ti_sci_rm_irq_ops rm_irq_ops;
+	struct ti_sci_proc_ops proc_ops;
+	struct ti_sci_rm_ringacc_ops rm_ring_ops;
+	struct ti_sci_rm_psil_ops rm_psil_ops;
+	struct ti_sci_rm_udmap_ops rm_udmap_ops;
 };
 
 /**
@@ -213,10 +566,46 @@
 	struct ti_sci_ops ops;
 };
 
+#define TI_SCI_RESOURCE_NULL	0xffff
+
+/**
+ * struct ti_sci_resource_desc - Description of TI SCI resource instance range.
+ * @start:	Start index of the resource.
+ * @num:	Number of resources.
+ * @res_map:	Bitmap to manage the allocation of these resources.
+ */
+struct ti_sci_resource_desc {
+	u16 start;
+	u16 num;
+	unsigned long *res_map;
+};
+
+/**
+ * struct ti_sci_resource - Structure representing a resource assigned
+ *			    to a device.
+ * @sets:	Number of sets available from this resource type
+ * @lock:	Lock to guard the res map in each set.
+ * @desc:	Array of resource descriptors.
+ */
+struct ti_sci_resource {
+	u16 sets;
+	raw_spinlock_t lock;
+	struct ti_sci_resource_desc *desc;
+};
+
 #if IS_ENABLED(CONFIG_TI_SCI_PROTOCOL)
 const struct ti_sci_handle *ti_sci_get_handle(struct device *dev);
 int ti_sci_put_handle(const struct ti_sci_handle *handle);
 const struct ti_sci_handle *devm_ti_sci_get_handle(struct device *dev);
+const struct ti_sci_handle *ti_sci_get_by_phandle(struct device_node *np,
+						  const char *property);
+const struct ti_sci_handle *devm_ti_sci_get_by_phandle(struct device *dev,
+						       const char *property);
+u16 ti_sci_get_free_resource(struct ti_sci_resource *res);
+void ti_sci_release_resource(struct ti_sci_resource *res, u16 id);
+struct ti_sci_resource *
+devm_ti_sci_get_of_resource(const struct ti_sci_handle *handle,
+			    struct device *dev, u32 dev_id, char *of_prop);
 
 #else	/* CONFIG_TI_SCI_PROTOCOL */
 
@@ -236,6 +625,35 @@
 	return ERR_PTR(-EINVAL);
 }
 
+static inline
+const struct ti_sci_handle *ti_sci_get_by_phandle(struct device_node *np,
+						  const char *property)
+{
+	return ERR_PTR(-EINVAL);
+}
+
+static inline
+const struct ti_sci_handle *devm_ti_sci_get_by_phandle(struct device *dev,
+						       const char *property)
+{
+	return ERR_PTR(-EINVAL);
+}
+
+static inline u16 ti_sci_get_free_resource(struct ti_sci_resource *res)
+{
+	return TI_SCI_RESOURCE_NULL;
+}
+
+static inline void ti_sci_release_resource(struct ti_sci_resource *res, u16 id)
+{
+}
+
+static inline struct ti_sci_resource *
+devm_ti_sci_get_of_resource(const struct ti_sci_handle *handle,
+			    struct device *dev, u32 dev_id, char *of_prop)
+{
+	return ERR_PTR(-EINVAL);
+}
 #endif	/* CONFIG_TI_SCI_PROTOCOL */
 
 #endif	/* __TISCI_PROTOCOL_H */
diff -Naur --no-dereference a/include/linux/socket.h b/include/linux/socket.h
--- a/include/linux/socket.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/socket.h	2020-06-19 21:00:31.432227716 -0400
@@ -208,8 +208,9 @@
 				 * reuses AF_INET address family
 				 */
 #define AF_XDP		44	/* XDP sockets			*/
+#define AF_RPMSG	45	/* Remote-processor messaging   */
 
-#define AF_MAX		45	/* For now.. */
+#define AF_MAX		46	/* For now.. */
 
 /* Protocol families, same as address families. */
 #define PF_UNSPEC	AF_UNSPEC
@@ -259,6 +260,7 @@
 #define PF_QIPCRTR	AF_QIPCRTR
 #define PF_SMC		AF_SMC
 #define PF_XDP		AF_XDP
+#define PF_RPMSG	AF_RPMSG
 #define PF_MAX		AF_MAX
 
 /* Maximum queue length specifiable by listen.  */
@@ -341,6 +343,7 @@
 #define SOL_KCM		281
 #define SOL_TLS		282
 #define SOL_XDP		283
+#define SOL_RPMSG	284
 
 /* IPX options */
 #define IPX_TYPE	1
diff -Naur --no-dereference a/include/linux/spi/spi.h b/include/linux/spi/spi.h
--- a/include/linux/spi/spi.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/spi/spi.h	2020-06-19 21:00:31.432227716 -0400
@@ -163,6 +163,9 @@
 #define	SPI_TX_QUAD	0x200			/* transmit with 4 wires */
 #define	SPI_RX_DUAL	0x400			/* receive with 2 wires */
 #define	SPI_RX_QUAD	0x800			/* receive with 4 wires */
+#define	SPI_CS_WORD	0x1000			/* toggle cs after each word */
+#define	SPI_TX_OCTAL	0x2000			/* transmit with 8 wires */
+#define	SPI_RX_OCTAL	0x4000			/* receive with 8 wires */
 	int			irq;
 	void			*controller_state;
 	void			*controller_data;
diff -Naur --no-dereference a/include/linux/splice.h b/include/linux/splice.h
--- a/include/linux/splice.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/splice.h	2020-06-19 21:00:31.432227716 -0400
@@ -87,4 +87,10 @@
 
 extern const struct pipe_buf_operations page_cache_pipe_buf_ops;
 extern const struct pipe_buf_operations default_pipe_buf_ops;
+
+extern long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
+			   loff_t *ppos, size_t len, unsigned int flags);
+extern long do_splice_to(struct file *in, loff_t *ppos,
+			 struct pipe_inode_info *pipe, size_t len,
+			 unsigned int flags);
 #endif
diff -Naur --no-dereference a/include/linux/ti-emif-sram.h b/include/linux/ti-emif-sram.h
--- a/include/linux/ti-emif-sram.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/ti-emif-sram.h	2020-06-19 21:00:31.432227716 -0400
@@ -55,6 +55,7 @@
 struct ti_emif_pm_functions {
 	u32 save_context;
 	u32 restore_context;
+	u32 run_hw_leveling;
 	u32 enter_sr;
 	u32 exit_sr;
 	u32 abort_sr;
@@ -126,6 +127,8 @@
 	       offsetof(struct ti_emif_pm_functions, save_context));
 	DEFINE(EMIF_PM_RESTORE_CONTEXT_OFFSET,
 	       offsetof(struct ti_emif_pm_functions, restore_context));
+	DEFINE(EMIF_PM_RUN_HW_LEVELING,
+	       offsetof(struct ti_emif_pm_functions, run_hw_leveling));
 	DEFINE(EMIF_PM_ENTER_SR_OFFSET,
 	       offsetof(struct ti_emif_pm_functions, enter_sr));
 	DEFINE(EMIF_PM_EXIT_SR_OFFSET,
diff -Naur --no-dereference a/include/linux/usb/ch9.h b/include/linux/usb/ch9.h
--- a/include/linux/usb/ch9.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/usb/ch9.h	2020-06-19 21:00:31.432227716 -0400
@@ -62,4 +62,31 @@
  */
 extern const char *usb_state_string(enum usb_device_state state);
 
+#ifdef CONFIG_TRACING
+/**
+ * usb_decode_ctrl - Returns human readable representation of control request.
+ * @str: buffer to return a human-readable representation of control request.
+ *       This buffer should have about 200 bytes.
+ * @size: size of str buffer.
+ * @bRequestType: matches the USB bmRequestType field
+ * @bRequest: matches the USB bRequest field
+ * @wValue: matches the USB wValue field (CPU byte order)
+ * @wIndex: matches the USB wIndex field (CPU byte order)
+ * @wLength: matches the USB wLength field (CPU byte order)
+ *
+ * Function returns decoded, formatted and human-readable description of
+ * control request packet.
+ *
+ * The usage scenario for this is for tracepoints, so function as a return
+ * use the same value as in parameters. This approach allows to use this
+ * function in TP_printk
+ *
+ * Important: wValue, wIndex, wLength parameters before invoking this function
+ * should be processed by le16_to_cpu macro.
+ */
+extern const char *usb_decode_ctrl(char *str, size_t size, __u8 bRequestType,
+				   __u8 bRequest, __u16 wValue, __u16 wIndex,
+				   __u16 wLength);
+#endif
+
 #endif /* __LINUX_USB_CH9_H */
diff -Naur --no-dereference a/include/linux/watchdog.h b/include/linux/watchdog.h
--- a/include/linux/watchdog.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/watchdog.h	2020-06-19 21:00:31.432227716 -0400
@@ -119,6 +119,7 @@
 #define WDOG_STOP_ON_REBOOT	2	/* Should be stopped on reboot */
 #define WDOG_HW_RUNNING		3	/* True if HW watchdog running */
 #define WDOG_STOP_ON_UNREGISTER	4	/* Should be stopped on unregister */
+#define WDOG_RESET_KEEPALIVE	5	/* Reset keepalive timers at start */
 	struct list_head deferred;
 };
 
diff -Naur --no-dereference a/include/linux/wkup_m3_ipc.h b/include/linux/wkup_m3_ipc.h
--- a/include/linux/wkup_m3_ipc.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/linux/wkup_m3_ipc.h	2020-06-19 21:00:31.432227716 -0400
@@ -33,7 +33,13 @@
 
 	int mem_type;
 	unsigned long resume_addr;
+	int vtt_conf;
+	int isolation_conf;
 	int state;
+	u32 halt;
+
+	unsigned long volt_scale_offsets;
+	const char *sd_fw_name;
 
 	struct completion sync_complete;
 	struct mbox_client mbox_client;
@@ -41,6 +47,7 @@
 
 	struct wkup_m3_ipc_ops *ops;
 	int is_rtc_only;
+	struct dentry *dbg_path;
 };
 
 struct wkup_m3_wakeup_src {
@@ -48,6 +55,12 @@
 	char src[10];
 };
 
+struct wkup_m3_scale_data_header {
+	u16 magic;
+	u8 sleep_offset;
+	u8 wake_offset;
+} __packed;
+
 struct wkup_m3_ipc_ops {
 	void (*set_mem_type)(struct wkup_m3_ipc *m3_ipc, int mem_type);
 	void (*set_resume_address)(struct wkup_m3_ipc *m3_ipc, void *addr);
diff -Naur --no-dereference a/include/media/v4l2-fwnode.h b/include/media/v4l2-fwnode.h
--- a/include/media/v4l2-fwnode.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/media/v4l2-fwnode.h	2020-06-19 21:00:31.432227716 -0400
@@ -57,6 +57,9 @@
 	unsigned int flags;
 	unsigned char bus_width;
 	unsigned char data_shift;
+	unsigned char num_channels;
+	unsigned char pixmux;
+	unsigned char channels[16];
 };
 
 /**
diff -Naur --no-dereference a/include/media/videobuf2-core.h b/include/media/videobuf2-core.h
--- a/include/media/videobuf2-core.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/media/videobuf2-core.h	2020-06-19 21:00:31.432227716 -0400
@@ -18,7 +18,7 @@
 #include <linux/dma-buf.h>
 #include <linux/bitops.h>
 
-#define VB2_MAX_FRAME	(32)
+#define VB2_MAX_FRAME	(128)
 #define VB2_MAX_PLANES	(8)
 
 /**
diff -Naur --no-dereference a/include/net/cfg80211.h b/include/net/cfg80211.h
--- a/include/net/cfg80211.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/net/cfg80211.h	2020-06-19 21:00:31.432227716 -0400
@@ -740,6 +740,9 @@
  *	CFG80211_MAX_WEP_KEYS WEP keys
  * @wep_tx_key: key index (0..3) of the default TX static WEP key
  * @psk: PSK (for devices supporting 4-way-handshake offload)
+ * @sae_pwd: password for SAE authentication (for devices supporting SAE
+ *	offload)
+ * @sae_pwd_len: length of SAE password (for devices supporting SAE offload)
  */
 struct cfg80211_crypto_settings {
 	u32 wpa_versions;
@@ -755,6 +758,8 @@
 	struct key_params *wep_keys;
 	int wep_tx_key;
 	const u8 *psk;
+	const u8 *sae_pwd;
+	u8 sae_pwd_len;
 };
 
 /**
@@ -5575,6 +5580,8 @@
  *	not known. This value is used only if @status < 0 to indicate that the
  *	failure is due to a timeout and not due to explicit rejection by the AP.
  *	This value is ignored in other cases (@status >= 0).
+ * @authorized: Indicates whether the connection is ready to transport
+ *	data packets.
  */
 struct cfg80211_connect_resp_params {
 	int status;
@@ -5586,6 +5593,7 @@
 	size_t resp_ie_len;
 	struct cfg80211_fils_resp_params fils;
 	enum nl80211_timeout_reason timeout_reason;
+	bool authorized;
 };
 
 /**
@@ -5730,6 +5738,9 @@
  * @req_ie_len: association request IEs length
  * @resp_ie: association response IEs (may be %NULL)
  * @resp_ie_len: assoc response IEs length
+ * @authorized: true if the 802.1X authentication was done by the driver or is
+ *	not needed (e.g., when Fast Transition protocol was used), false
+ *	otherwise. Ignored for networks that don't use 802.1X authentication.
  * @fils: FILS related roaming information.
  */
 struct cfg80211_roam_info {
@@ -5740,6 +5751,7 @@
 	size_t req_ie_len;
 	const u8 *resp_ie;
 	size_t resp_ie_len;
+	bool authorized;
 	struct cfg80211_fils_resp_params fils;
 };
 
diff -Naur --no-dereference a/include/sound/simple_card_utils.h b/include/sound/simple_card_utils.h
--- a/include/sound/simple_card_utils.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/sound/simple_card_utils.h	2020-06-19 21:00:31.432227716 -0400
@@ -19,6 +19,7 @@
 	const char *name;
 	unsigned int sysclk;
 	int clk_direction;
+	int sysclk_id;
 	int slots;
 	int slot_width;
 	unsigned int tx_slot_mask;
diff -Naur --no-dereference a/include/sound/soc.h b/include/sound/soc.h
--- a/include/sound/soc.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/sound/soc.h	2020-06-19 21:00:31.432227716 -0400
@@ -1021,6 +1021,11 @@
 	struct mutex mutex;
 	struct mutex dapm_mutex;
 
+	/* Mutex for PCM operations */
+	struct mutex pcm_mutex;
+	enum snd_soc_pcm_subclass pcm_subclass;
+
+	int id_hint;
 	bool instantiated;
 	bool topology_shortname_created;
 
@@ -1120,8 +1125,6 @@
 	struct device *dev;
 	struct snd_soc_card *card;
 	struct snd_soc_dai_link *dai_link;
-	struct mutex pcm_mutex;
-	enum snd_soc_pcm_subclass pcm_subclass;
 	struct snd_pcm_ops ops;
 
 	/* Dynamic PCM BE runtime data */
diff -Naur --no-dereference a/include/uapi/drm/omap_drm.h b/include/uapi/drm/omap_drm.h
--- a/include/uapi/drm/omap_drm.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/uapi/drm/omap_drm.h	2020-06-19 21:00:31.432227716 -0400
@@ -38,20 +38,29 @@
 	__u64 value;			/* in (set_param), out (get_param) */
 };
 
-#define OMAP_BO_SCANOUT		0x00000001	/* scanout capable (phys contiguous) */
-#define OMAP_BO_CACHE_MASK	0x00000006	/* cache type mask, see cache modes */
-#define OMAP_BO_TILED_MASK	0x00000f00	/* tiled mapping mask, see tiled modes */
-
-/* cache modes */
-#define OMAP_BO_CACHED		0x00000000	/* default */
-#define OMAP_BO_WC		0x00000002	/* write-combine */
-#define OMAP_BO_UNCACHED	0x00000004	/* strongly-ordered (uncached) */
+/* Scanout buffer, consumable by DSS */
+#define OMAP_BO_SCANOUT		0x00000001
 
-/* tiled modes */
+/* Buffer CPU caching mode: cached, write-combining or uncached. */
+#define OMAP_BO_CACHED		0x00000000
+#define OMAP_BO_WC		0x00000002
+#define OMAP_BO_UNCACHED	0x00000004
+#define OMAP_BO_CACHE_MASK	0x00000006
+
+/* Force allocation from contiguous DMA memory */
+#define OMAP_BO_MEM_CONTIG	0x00000008
+
+/* Force allocation via DMM */
+#define OMAP_BO_MEM_DMM		0x00000010
+
+/* Pin the buffer when allocating and keep pinned */
+#define OMAP_BO_MEM_PIN		0x00000020
+
+/* Use TILER for the buffer. The TILER container unit can be 8, 16 or 32 bits. */
 #define OMAP_BO_TILED_8		0x00000100
 #define OMAP_BO_TILED_16	0x00000200
 #define OMAP_BO_TILED_32	0x00000300
-#define OMAP_BO_TILED		(OMAP_BO_TILED_8 | OMAP_BO_TILED_16 | OMAP_BO_TILED_32)
+#define OMAP_BO_TILED_MASK	0x00000f00
 
 union omap_gem_size {
 	__u32 bytes;		/* (for non-tiled formats) */
diff -Naur --no-dereference a/include/uapi/linux/aufs_type.h b/include/uapi/linux/aufs_type.h
--- a/include/uapi/linux/aufs_type.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/uapi/linux/aufs_type.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,452 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2005-2020 Junjiro R. Okajima
+ *
+ * This program, aufs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __AUFS_TYPE_H__
+#define __AUFS_TYPE_H__
+
+#define AUFS_NAME	"aufs"
+
+#ifdef __KERNEL__
+/*
+ * define it before including all other headers.
+ * sched.h may use pr_* macros before defining "current", so define the
+ * no-current version first, and re-define later.
+ */
+#define pr_fmt(fmt)	AUFS_NAME " %s:%d: " fmt, __func__, __LINE__
+#include <linux/sched.h>
+#undef pr_fmt
+#define pr_fmt(fmt) \
+		AUFS_NAME " %s:%d:%.*s[%d]: " fmt, __func__, __LINE__, \
+		(int)sizeof(current->comm), current->comm, current->pid
+#include <linux/limits.h>
+#else
+#include <stdint.h>
+#include <sys/types.h>
+#include <limits.h>
+#endif /* __KERNEL__ */
+
+#define AUFS_VERSION	"4.19.63+-20200518"
+
+/* todo? move this to linux-2.6.19/include/magic.h */
+#define AUFS_SUPER_MAGIC	('a' << 24 | 'u' << 16 | 'f' << 8 | 's')
+
+/* ---------------------------------------------------------------------- */
+
+#ifdef __KERNEL__
+#ifdef CONFIG_AUFS_BRANCH_MAX_127
+typedef int8_t aufs_bindex_t;
+#define AUFS_BRANCH_MAX 127
+#else
+typedef int16_t aufs_bindex_t;
+#ifdef CONFIG_AUFS_BRANCH_MAX_511
+#define AUFS_BRANCH_MAX 511
+#elif defined(CONFIG_AUFS_BRANCH_MAX_1023)
+#define AUFS_BRANCH_MAX 1023
+#elif defined(CONFIG_AUFS_BRANCH_MAX_32767)
+#define AUFS_BRANCH_MAX 32767
+#endif
+#endif
+
+#ifndef AUFS_BRANCH_MAX
+#error unknown CONFIG_AUFS_BRANCH_MAX value
+#endif
+#endif /* __KERNEL__ */
+
+/* ---------------------------------------------------------------------- */
+
+#define AUFS_FSTYPE		AUFS_NAME
+
+#define AUFS_ROOT_INO		2
+#define AUFS_FIRST_INO		11
+
+#define AUFS_WH_PFX		".wh."
+#define AUFS_WH_PFX_LEN		((int)sizeof(AUFS_WH_PFX) - 1)
+#define AUFS_WH_TMP_LEN		4
+/* a limit for rmdir/rename a dir and copyup */
+#define AUFS_MAX_NAMELEN	(NAME_MAX \
+				- AUFS_WH_PFX_LEN * 2	/* doubly whiteouted */\
+				- 1			/* dot */\
+				- AUFS_WH_TMP_LEN)	/* hex */
+#define AUFS_XINO_FNAME		"." AUFS_NAME ".xino"
+#define AUFS_XINO_DEFPATH	"/tmp/" AUFS_XINO_FNAME
+#define AUFS_XINO_DEF_SEC	30 /* seconds */
+#define AUFS_XINO_DEF_TRUNC	45 /* percentage */
+#define AUFS_DIRWH_DEF		3
+#define AUFS_RDCACHE_DEF	10 /* seconds */
+#define AUFS_RDCACHE_MAX	3600 /* seconds */
+#define AUFS_RDBLK_DEF		512 /* bytes */
+#define AUFS_RDHASH_DEF		32
+#define AUFS_WKQ_NAME		AUFS_NAME "d"
+#define AUFS_MFS_DEF_SEC	30 /* seconds */
+#define AUFS_MFS_MAX_SEC	3600 /* seconds */
+#define AUFS_FHSM_CACHE_DEF_SEC	30 /* seconds */
+#define AUFS_PLINK_WARN		50 /* number of plinks in a single bucket */
+
+/* pseudo-link maintenace under /proc */
+#define AUFS_PLINK_MAINT_NAME	"plink_maint"
+#define AUFS_PLINK_MAINT_DIR	"fs/" AUFS_NAME
+#define AUFS_PLINK_MAINT_PATH	AUFS_PLINK_MAINT_DIR "/" AUFS_PLINK_MAINT_NAME
+
+/* dirren, renamed dir */
+#define AUFS_DR_INFO_PFX	AUFS_WH_PFX ".dr."
+#define AUFS_DR_BRHINO_NAME	AUFS_WH_PFX "hino"
+/* whiteouted doubly */
+#define AUFS_WH_DR_INFO_PFX	AUFS_WH_PFX AUFS_DR_INFO_PFX
+#define AUFS_WH_DR_BRHINO	AUFS_WH_PFX AUFS_DR_BRHINO_NAME
+
+#define AUFS_DIROPQ_NAME	AUFS_WH_PFX ".opq" /* whiteouted doubly */
+#define AUFS_WH_DIROPQ		AUFS_WH_PFX AUFS_DIROPQ_NAME
+
+#define AUFS_BASE_NAME		AUFS_WH_PFX AUFS_NAME
+#define AUFS_PLINKDIR_NAME	AUFS_WH_PFX "plnk"
+#define AUFS_ORPHDIR_NAME	AUFS_WH_PFX "orph"
+
+/* doubly whiteouted */
+#define AUFS_WH_BASE		AUFS_WH_PFX AUFS_BASE_NAME
+#define AUFS_WH_PLINKDIR	AUFS_WH_PFX AUFS_PLINKDIR_NAME
+#define AUFS_WH_ORPHDIR		AUFS_WH_PFX AUFS_ORPHDIR_NAME
+
+/* branch permissions and attributes */
+#define AUFS_BRPERM_RW		"rw"
+#define AUFS_BRPERM_RO		"ro"
+#define AUFS_BRPERM_RR		"rr"
+#define AUFS_BRATTR_COO_REG	"coo_reg"
+#define AUFS_BRATTR_COO_ALL	"coo_all"
+#define AUFS_BRATTR_FHSM	"fhsm"
+#define AUFS_BRATTR_UNPIN	"unpin"
+#define AUFS_BRATTR_ICEX	"icex"
+#define AUFS_BRATTR_ICEX_SEC	"icexsec"
+#define AUFS_BRATTR_ICEX_SYS	"icexsys"
+#define AUFS_BRATTR_ICEX_TR	"icextr"
+#define AUFS_BRATTR_ICEX_USR	"icexusr"
+#define AUFS_BRATTR_ICEX_OTH	"icexoth"
+#define AUFS_BRRATTR_WH		"wh"
+#define AUFS_BRWATTR_NLWH	"nolwh"
+#define AUFS_BRWATTR_MOO	"moo"
+
+#define AuBrPerm_RW		1		/* writable, hardlinkable wh */
+#define AuBrPerm_RO		(1 << 1)	/* readonly */
+#define AuBrPerm_RR		(1 << 2)	/* natively readonly */
+#define AuBrPerm_Mask		(AuBrPerm_RW | AuBrPerm_RO | AuBrPerm_RR)
+
+#define AuBrAttr_COO_REG	(1 << 3)	/* copy-up on open */
+#define AuBrAttr_COO_ALL	(1 << 4)
+#define AuBrAttr_COO_Mask	(AuBrAttr_COO_REG | AuBrAttr_COO_ALL)
+
+#define AuBrAttr_FHSM		(1 << 5)	/* file-based hsm */
+#define AuBrAttr_UNPIN		(1 << 6)	/* rename-able top dir of
+						   branch. meaningless since
+						   linux-3.18-rc1 */
+
+/* ignore error in copying XATTR */
+#define AuBrAttr_ICEX_SEC	(1 << 7)
+#define AuBrAttr_ICEX_SYS	(1 << 8)
+#define AuBrAttr_ICEX_TR	(1 << 9)
+#define AuBrAttr_ICEX_USR	(1 << 10)
+#define AuBrAttr_ICEX_OTH	(1 << 11)
+#define AuBrAttr_ICEX		(AuBrAttr_ICEX_SEC	\
+				 | AuBrAttr_ICEX_SYS	\
+				 | AuBrAttr_ICEX_TR	\
+				 | AuBrAttr_ICEX_USR	\
+				 | AuBrAttr_ICEX_OTH)
+
+#define AuBrRAttr_WH		(1 << 12)	/* whiteout-able */
+#define AuBrRAttr_Mask		AuBrRAttr_WH
+
+#define AuBrWAttr_NoLinkWH	(1 << 13)	/* un-hardlinkable whiteouts */
+#define AuBrWAttr_MOO		(1 << 14)	/* move-up on open */
+#define AuBrWAttr_Mask		(AuBrWAttr_NoLinkWH | AuBrWAttr_MOO)
+
+#define AuBrAttr_CMOO_Mask	(AuBrAttr_COO_Mask | AuBrWAttr_MOO)
+
+/* #warning test userspace */
+#ifdef __KERNEL__
+#ifndef CONFIG_AUFS_FHSM
+#undef AuBrAttr_FHSM
+#define AuBrAttr_FHSM		0
+#endif
+#ifndef CONFIG_AUFS_XATTR
+#undef	AuBrAttr_ICEX
+#define AuBrAttr_ICEX		0
+#undef	AuBrAttr_ICEX_SEC
+#define AuBrAttr_ICEX_SEC	0
+#undef	AuBrAttr_ICEX_SYS
+#define AuBrAttr_ICEX_SYS	0
+#undef	AuBrAttr_ICEX_TR
+#define AuBrAttr_ICEX_TR	0
+#undef	AuBrAttr_ICEX_USR
+#define AuBrAttr_ICEX_USR	0
+#undef	AuBrAttr_ICEX_OTH
+#define AuBrAttr_ICEX_OTH	0
+#endif
+#endif
+
+/* the longest combination */
+/* AUFS_BRATTR_ICEX and AUFS_BRATTR_ICEX_TR don't affect here */
+#define AuBrPermStrSz	sizeof(AUFS_BRPERM_RW			\
+			       "+" AUFS_BRATTR_COO_REG		\
+			       "+" AUFS_BRATTR_FHSM		\
+			       "+" AUFS_BRATTR_UNPIN		\
+			       "+" AUFS_BRATTR_ICEX_SEC		\
+			       "+" AUFS_BRATTR_ICEX_SYS		\
+			       "+" AUFS_BRATTR_ICEX_USR		\
+			       "+" AUFS_BRATTR_ICEX_OTH		\
+			       "+" AUFS_BRWATTR_NLWH)
+
+typedef struct {
+	char a[AuBrPermStrSz];
+} au_br_perm_str_t;
+
+static inline int au_br_writable(int brperm)
+{
+	return brperm & AuBrPerm_RW;
+}
+
+static inline int au_br_whable(int brperm)
+{
+	return brperm & (AuBrPerm_RW | AuBrRAttr_WH);
+}
+
+static inline int au_br_wh_linkable(int brperm)
+{
+	return !(brperm & AuBrWAttr_NoLinkWH);
+}
+
+static inline int au_br_cmoo(int brperm)
+{
+	return brperm & AuBrAttr_CMOO_Mask;
+}
+
+static inline int au_br_fhsm(int brperm)
+{
+	return brperm & AuBrAttr_FHSM;
+}
+
+/* ---------------------------------------------------------------------- */
+
+/* ioctl */
+enum {
+	/* readdir in userspace */
+	AuCtl_RDU,
+	AuCtl_RDU_INO,
+
+	AuCtl_WBR_FD,	/* pathconf wrapper */
+	AuCtl_IBUSY,	/* busy inode */
+	AuCtl_MVDOWN,	/* move-down */
+	AuCtl_BR,	/* info about branches */
+	AuCtl_FHSM_FD	/* connection for fhsm */
+};
+
+/* borrowed from linux/include/linux/kernel.h */
+#ifndef ALIGN
+#ifdef _GNU_SOURCE
+#define ALIGN(x, a)		__ALIGN_MASK(x, (typeof(x))(a)-1)
+#else
+#define ALIGN(x, a)		(((x) + (a) - 1) & ~((a) - 1))
+#endif
+#define __ALIGN_MASK(x, mask)	(((x)+(mask))&~(mask))
+#endif
+
+/* borrowed from linux/include/linux/compiler-gcc3.h */
+#ifndef __aligned
+#define __aligned(x)			__attribute__((aligned(x)))
+#endif
+
+#ifdef __KERNEL__
+#ifndef __packed
+#define __packed			__attribute__((packed))
+#endif
+#endif
+
+struct au_rdu_cookie {
+	uint64_t	h_pos;
+	int16_t		bindex;
+	uint8_t		flags;
+	uint8_t		pad;
+	uint32_t	generation;
+} __aligned(8);
+
+struct au_rdu_ent {
+	uint64_t	ino;
+	int16_t		bindex;
+	uint8_t		type;
+	uint8_t		nlen;
+	uint8_t		wh;
+	char		name[];
+} __aligned(8);
+
+static inline int au_rdu_len(int nlen)
+{
+	/* include the terminating NULL */
+	return ALIGN(sizeof(struct au_rdu_ent) + nlen + 1,
+		     sizeof(uint64_t));
+}
+
+union au_rdu_ent_ul {
+	struct au_rdu_ent __user	*e;
+	uint64_t			ul;
+};
+
+enum {
+	AufsCtlRduV_SZ,
+	AufsCtlRduV_End
+};
+
+struct aufs_rdu {
+	/* input */
+	union {
+		uint64_t	sz;	/* AuCtl_RDU */
+		uint64_t	nent;	/* AuCtl_RDU_INO */
+	};
+	union au_rdu_ent_ul	ent;
+	uint16_t		verify[AufsCtlRduV_End];
+
+	/* input/output */
+	uint32_t		blk;
+
+	/* output */
+	union au_rdu_ent_ul	tail;
+	/* number of entries which were added in a single call */
+	uint64_t		rent;
+	uint8_t			full;
+	uint8_t			shwh;
+
+	struct au_rdu_cookie	cookie;
+} __aligned(8);
+
+/* ---------------------------------------------------------------------- */
+
+/* dirren. the branch is identified by the filename who contains this */
+struct au_drinfo {
+	uint64_t ino;
+	union {
+		uint8_t oldnamelen;
+		uint64_t _padding;
+	};
+	uint8_t oldname[];
+} __aligned(8);
+
+struct au_drinfo_fdata {
+	uint32_t magic;
+	struct au_drinfo drinfo;
+} __aligned(8);
+
+#define AUFS_DRINFO_MAGIC_V1	('a' << 24 | 'd' << 16 | 'r' << 8 | 0x01)
+/* future */
+#define AUFS_DRINFO_MAGIC_V2	('a' << 24 | 'd' << 16 | 'r' << 8 | 0x02)
+
+/* ---------------------------------------------------------------------- */
+
+struct aufs_wbr_fd {
+	uint32_t	oflags;
+	int16_t		brid;
+} __aligned(8);
+
+/* ---------------------------------------------------------------------- */
+
+struct aufs_ibusy {
+	uint64_t	ino, h_ino;
+	int16_t		bindex;
+} __aligned(8);
+
+/* ---------------------------------------------------------------------- */
+
+/* error code for move-down */
+/* the actual message strings are implemented in aufs-util.git */
+enum {
+	EAU_MVDOWN_OPAQUE = 1,
+	EAU_MVDOWN_WHITEOUT,
+	EAU_MVDOWN_UPPER,
+	EAU_MVDOWN_BOTTOM,
+	EAU_MVDOWN_NOUPPER,
+	EAU_MVDOWN_NOLOWERBR,
+	EAU_Last
+};
+
+/* flags for move-down */
+#define AUFS_MVDOWN_DMSG	1
+#define AUFS_MVDOWN_OWLOWER	(1 << 1)	/* overwrite lower */
+#define AUFS_MVDOWN_KUPPER	(1 << 2)	/* keep upper */
+#define AUFS_MVDOWN_ROLOWER	(1 << 3)	/* do even if lower is RO */
+#define AUFS_MVDOWN_ROLOWER_R	(1 << 4)	/* did on lower RO */
+#define AUFS_MVDOWN_ROUPPER	(1 << 5)	/* do even if upper is RO */
+#define AUFS_MVDOWN_ROUPPER_R	(1 << 6)	/* did on upper RO */
+#define AUFS_MVDOWN_BRID_UPPER	(1 << 7)	/* upper brid */
+#define AUFS_MVDOWN_BRID_LOWER	(1 << 8)	/* lower brid */
+#define AUFS_MVDOWN_FHSM_LOWER	(1 << 9)	/* find fhsm attr for lower */
+#define AUFS_MVDOWN_STFS	(1 << 10)	/* req. stfs */
+#define AUFS_MVDOWN_STFS_FAILED	(1 << 11)	/* output: stfs is unusable */
+#define AUFS_MVDOWN_BOTTOM	(1 << 12)	/* output: no more lowers */
+
+/* index for move-down */
+enum {
+	AUFS_MVDOWN_UPPER,
+	AUFS_MVDOWN_LOWER,
+	AUFS_MVDOWN_NARRAY
+};
+
+/*
+ * additional info of move-down
+ * number of free blocks and inodes.
+ * subset of struct kstatfs, but smaller and always 64bit.
+ */
+struct aufs_stfs {
+	uint64_t	f_blocks;
+	uint64_t	f_bavail;
+	uint64_t	f_files;
+	uint64_t	f_ffree;
+};
+
+struct aufs_stbr {
+	int16_t			brid;	/* optional input */
+	int16_t			bindex;	/* output */
+	struct aufs_stfs	stfs;	/* output when AUFS_MVDOWN_STFS set */
+} __aligned(8);
+
+struct aufs_mvdown {
+	uint32_t		flags;			/* input/output */
+	struct aufs_stbr	stbr[AUFS_MVDOWN_NARRAY]; /* input/output */
+	int8_t			au_errno;		/* output */
+} __aligned(8);
+
+/* ---------------------------------------------------------------------- */
+
+union aufs_brinfo {
+	/* PATH_MAX may differ between kernel-space and user-space */
+	char	_spacer[4096];
+	struct {
+		int16_t	id;
+		int	perm;
+		char	path[];
+	};
+} __aligned(8);
+
+/* ---------------------------------------------------------------------- */
+
+#define AuCtlType		'A'
+#define AUFS_CTL_RDU		_IOWR(AuCtlType, AuCtl_RDU, struct aufs_rdu)
+#define AUFS_CTL_RDU_INO	_IOWR(AuCtlType, AuCtl_RDU_INO, struct aufs_rdu)
+#define AUFS_CTL_WBR_FD		_IOW(AuCtlType, AuCtl_WBR_FD, \
+				     struct aufs_wbr_fd)
+#define AUFS_CTL_IBUSY		_IOWR(AuCtlType, AuCtl_IBUSY, struct aufs_ibusy)
+#define AUFS_CTL_MVDOWN		_IOWR(AuCtlType, AuCtl_MVDOWN, \
+				      struct aufs_mvdown)
+#define AUFS_CTL_BRINFO		_IOW(AuCtlType, AuCtl_BR, union aufs_brinfo)
+#define AUFS_CTL_FHSM_FD	_IOW(AuCtlType, AuCtl_FHSM_FD, int)
+
+#endif /* __AUFS_TYPE_H__ */
diff -Naur --no-dereference a/include/uapi/linux/can/isotp.h b/include/uapi/linux/can/isotp.h
--- a/include/uapi/linux/can/isotp.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/uapi/linux/can/isotp.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,135 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/*
+ * socketcan/can/isotp.h
+ *
+ * Definitions for isotp CAN sockets
+ *
+ * Author: Oliver Hartkopp <oliver.hartkopp@volkswagen.de>
+ * Copyright (c) 2008 Volkswagen Group Electronic Research
+ * All rights reserved.
+ *
+ * Send feedback to <socketcan-users@lists.berlios.de>
+ *
+ */
+
+#ifndef CAN_ISOTP_H
+#define CAN_ISOTP_H
+
+#include <linux/can.h>
+
+#define SOL_CAN_ISOTP (SOL_CAN_BASE + CAN_ISOTP)
+
+/* for socket options affecting the socket (not the global system) */
+
+#define CAN_ISOTP_OPTS		1	/* pass struct can_isotp_options */
+
+#define CAN_ISOTP_RECV_FC	2	/* pass struct can_isotp_fc_options */
+
+/* sockopts to force stmin timer values for protocol regression tests */
+
+#define CAN_ISOTP_TX_STMIN	3	/* pass __u32 value in nano secs    */
+					/* use this time instead of value   */
+					/* provided in FC from the receiver */
+
+#define CAN_ISOTP_RX_STMIN	4	/* pass __u32 value in nano secs   */
+					/* ignore received CF frames which */
+					/* timestamps differ less than val */
+
+#define CAN_ISOTP_LL_OPTS	5	/* pass struct can_isotp_ll_options */
+
+struct can_isotp_options {
+
+	__u32 flags;		/* set flags for isotp behaviour.	*/
+				/* __u32 value : flags see below	*/
+
+	__u32 frame_txtime;	/* frame transmission time (N_As/N_Ar)	*/
+				/* __u32 value : time in nano secs	*/
+
+	__u8  ext_address;	/* set address for extended addressing	*/
+				/* __u8 value : extended address	*/
+
+	__u8  txpad_content;	/* set content of padding byte (tx)	*/
+				/* __u8 value : content	on tx path	*/
+
+	__u8  rxpad_content;	/* set content of padding byte (rx)	*/
+				/* __u8 value : content	on rx path	*/
+
+	__u8  rx_ext_address;	/* set address for extended addressing	*/
+				/* __u8 value : extended address (rx)	*/
+};
+
+struct can_isotp_fc_options {
+
+	__u8  bs;		/* blocksize provided in FC frame	*/
+				/* __u8 value : blocksize. 0 = off	*/
+
+	__u8  stmin;		/* separation time provided in FC frame	*/
+				/* __u8 value :				*/
+				/* 0x00 - 0x7F : 0 - 127 ms		*/
+				/* 0x80 - 0xF0 : reserved		*/
+				/* 0xF1 - 0xF9 : 100 us - 900 us	*/
+				/* 0xFA - 0xFF : reserved		*/
+
+	__u8  wftmax;		/* max. number of wait frame transmiss.	*/
+				/* __u8 value : 0 = omit FC N_PDU WT	*/
+};
+
+struct can_isotp_ll_options {
+
+	__u8  mtu;		/* generated & accepted CAN frame type	*/
+				/* __u8 value :				*/
+				/* CAN_MTU   (16) -> standard CAN 2.0	*/
+				/* CANFD_MTU (72) -> CAN FD frame	*/
+
+	__u8  tx_dl;		/* tx link layer data length in bytes	*/
+				/* (configured maximum payload length)	*/
+				/* __u8 value : 8,12,16,20,24,32,48,64	*/
+				/* => rx path supports all LL_DL values */
+
+	__u8  tx_flags;		/* set into struct canfd_frame.flags	*/
+				/* at frame creation: e.g. CANFD_BRS	*/
+				/* Obsolete when the BRS flag is fixed	*/
+				/* by the CAN netdriver configuration	*/
+};
+
+/* flags for isotp behaviour */
+
+#define CAN_ISOTP_LISTEN_MODE	0x001	/* listen only (do not send FC) */
+#define CAN_ISOTP_EXTEND_ADDR	0x002	/* enable extended addressing */
+#define CAN_ISOTP_TX_PADDING	0x004	/* enable CAN frame padding tx path */
+#define CAN_ISOTP_RX_PADDING	0x008	/* enable CAN frame padding rx path */
+#define CAN_ISOTP_CHK_PAD_LEN	0x010	/* check received CAN frame padding */
+#define CAN_ISOTP_CHK_PAD_DATA	0x020	/* check received CAN frame padding */
+#define CAN_ISOTP_HALF_DUPLEX	0x040	/* half duplex error state handling */
+#define CAN_ISOTP_FORCE_TXSTMIN	0x080	/* ignore stmin from received FC */
+#define CAN_ISOTP_FORCE_RXSTMIN	0x100	/* ignore CFs depending on rx stmin */
+#define CAN_ISOTP_RX_EXT_ADDR	0x200	/* different rx extended addressing */
+
+
+/* default values */
+
+#define CAN_ISOTP_DEFAULT_FLAGS		0
+#define CAN_ISOTP_DEFAULT_EXT_ADDRESS	0x00
+#define CAN_ISOTP_DEFAULT_PAD_CONTENT	0xCC /* prevent bit-stuffing */
+#define CAN_ISOTP_DEFAULT_FRAME_TXTIME	0
+#define CAN_ISOTP_DEFAULT_RECV_BS	0
+#define CAN_ISOTP_DEFAULT_RECV_STMIN	0x00
+#define CAN_ISOTP_DEFAULT_RECV_WFTMAX	0
+
+#define CAN_ISOTP_DEFAULT_LL_MTU	CAN_MTU
+#define CAN_ISOTP_DEFAULT_LL_TX_DL	CAN_MAX_DLEN
+#define CAN_ISOTP_DEFAULT_LL_TX_FLAGS	0
+
+/*
+ * Remark on CAN_ISOTP_DEFAULT_RECV_* values:
+ *
+ * We can strongly assume, that the Linux Kernel implementation of
+ * CAN_ISOTP is capable to run with BS=0, STmin=0 and WFTmax=0.
+ * But as we like to be able to behave as a commonly available ECU,
+ * these default settings can be changed via sockopts.
+ * For that reason the STmin value is intentionally _not_ checked for
+ * consistency and copied directly into the flow control (FC) frame.
+ *
+ */
+
+#endif
diff -Naur --no-dereference a/include/uapi/linux/dma_buf_phys.h b/include/uapi/linux/dma_buf_phys.h
--- a/include/uapi/linux/dma_buf_phys.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/uapi/linux/dma_buf_phys.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+/*
+ * DMA-BUF contiguous buffer physical address user-space exporter
+ *
+ * Copyright (C) 2018-2019 Texas Instruments Incorporated - http://www.ti.com/
+ *	Andrew F. Davis <afd@ti.com>
+ */
+
+#ifndef DMA_BUF_PHYS_H
+#define DMA_BUF_PHYS_H
+
+#include <linux/ioctl.h>
+#include <linux/types.h>
+
+/**
+ * struct dma_buf_phys_data - metadata passed from userspace for conversion
+ * @fd:		DMA-BUF fd for conversion
+ * @phys:	populated with CPU physical address of DMA-BUF
+ */
+struct dma_buf_phys_data {
+	__u32 fd;
+	__u64 phys;
+};
+
+#define DMA_BUF_PHYS_IOC_MAGIC 'D'
+
+/**
+ * DOC: DMA_BUF_PHYS_IOC_CONVERT - Convert DMA-BUF to physical address
+ *
+ * Takes a dma_buf_phys_data struct containing a fd for a physicaly contigous
+ * buffer. Pins this buffer and populates phys field with the CPU physical address.
+ */
+#define DMA_BUF_PHYS_IOC_CONVERT _IOWR(DMA_BUF_PHYS_IOC_MAGIC, 0, struct dma_buf_phys_data)
+
+#endif /* DMA_BUF_PHYS_H */
diff -Naur --no-dereference a/include/uapi/linux/input-event-codes.h b/include/uapi/linux/input-event-codes.h
--- a/include/uapi/linux/input-event-codes.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/uapi/linux/input-event-codes.h	2020-06-19 21:00:31.432227716 -0400
@@ -439,10 +439,12 @@
 #define KEY_TITLE		0x171
 #define KEY_SUBTITLE		0x172
 #define KEY_ANGLE		0x173
-#define KEY_ZOOM		0x174
+#define KEY_FULL_SCREEN		0x174	/* AC View Toggle */
+#define KEY_ZOOM		KEY_FULL_SCREEN
 #define KEY_MODE		0x175
 #define KEY_KEYBOARD		0x176
-#define KEY_SCREEN		0x177
+#define KEY_ASPECT_RATIO	0x177	/* HUTRR37: Aspect */
+#define KEY_SCREEN		KEY_ASPECT_RATIO
 #define KEY_PC			0x178	/* Media Select Computer */
 #define KEY_TV			0x179	/* Media Select TV */
 #define KEY_TV2			0x17a	/* Media Select Cable */
@@ -604,6 +606,7 @@
 #define KEY_SCREENSAVER		0x245	/* AL Screen Saver */
 #define KEY_VOICECOMMAND		0x246	/* Listening Voice Command */
 #define KEY_ASSISTANT		0x247	/* AL Context-aware desktop assistant */
+#define KEY_KBD_LAYOUT_NEXT	0x248	/* AC Next Keyboard Layout Select */
 
 #define KEY_BRIGHTNESS_MIN		0x250	/* Set Brightness to Minimum */
 #define KEY_BRIGHTNESS_MAX		0x251	/* Set Brightness to Maximum */
@@ -708,6 +711,16 @@
 #define REL_DIAL		0x07
 #define REL_WHEEL		0x08
 #define REL_MISC		0x09
+/*
+ * 0x0a is reserved and should not be used in input drivers.
+ * It was used by HID as REL_MISC+1 and userspace needs to detect if
+ * the next REL_* event is correct or is just REL_MISC + n.
+ * We define here REL_RESERVED so userspace can rely on it and detect
+ * the situation described above.
+ */
+#define REL_RESERVED		0x0a
+#define REL_WHEEL_HI_RES	0x0b
+#define REL_HWHEEL_HI_RES	0x0c
 #define REL_MAX			0x0f
 #define REL_CNT			(REL_MAX+1)
 
diff -Naur --no-dereference a/include/uapi/linux/keystone_dsp_mem.h b/include/uapi/linux/keystone_dsp_mem.h
--- a/include/uapi/linux/keystone_dsp_mem.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/uapi/linux/keystone_dsp_mem.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+/*
+ * Copyright (C) 2015-2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+#ifndef _UAPI_KEYSTONE_DSP_MEM_H_
+#define _UAPI_KEYSTONE_DSP_MEM_H_
+
+#define KEYSTONE_DSP_MEM_MAP_INDEX_MASK		(0x7)
+#define KEYSTONE_DSP_MEM_MAP_OFFSET_SHIFT	(3)
+
+#endif /* _UAPI_KEYSTONE_DSP_MEM_H_ */
diff -Naur --no-dereference a/include/uapi/linux/keystone_remoteproc.h b/include/uapi/linux/keystone_remoteproc.h
--- a/include/uapi/linux/keystone_remoteproc.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/uapi/linux/keystone_remoteproc.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,56 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+/*
+ * Copyright (C) 2015-2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+#ifndef _UAPI_LINUX_KEYSTONE_REMOTEPROC_H_
+#define _UAPI_LINUX_KEYSTONE_REMOTEPROC_H_
+
+#include <linux/types.h>
+#include <linux/ioctl.h>
+
+/**
+ * enum keystone_rproc_state - keystone remoteproc state setting values
+ *
+ * @KEYSTONE_RPROC_OFFLINE: request to configure the remoteproc into an offline
+ *			    state
+ * @KEYSTONE_RPROC_RUNNING: request to configure the remoteproc into a ready
+ *			    state
+ */
+enum keystone_rproc_state {
+	KEYSTONE_RPROC_OFFLINE,
+	KEYSTONE_RPROC_RUNNING,
+};
+
+/**
+ * struct keystone_rproc_set_state_params - keystone remoteproc set state
+ *					    parameters structure
+ *
+ * @state: enumerated state value to set
+ * @boot_addr: boot address/entry point for the remote processor
+ */
+struct keystone_rproc_set_state_params {
+	enum keystone_rproc_state state;
+	uint32_t boot_addr;
+};
+
+/* Macros used within mmap function */
+#define KEYSTONE_RPROC_UIO_MAP_INDEX_MASK	(0x7)
+#define KEYSTONE_RPROC_UIO_MAP_OFFSET_SHIFT	(3)
+
+/* IOCTL definitions */
+#define KEYSTONE_RPROC_IOC_MAGIC		'I'
+#define KEYSTONE_RPROC_IOC_SET_RSC_TABLE	_IOW(KEYSTONE_RPROC_IOC_MAGIC, \
+						0, void *)
+#define KEYSTONE_RPROC_IOC_SET_STATE	_IOW(KEYSTONE_RPROC_IOC_MAGIC, \
+					1, \
+					struct keystone_rproc_set_state_params)
+#define KEYSTONE_RPROC_IOC_SET_LOADED_RSC_TABLE _IOW(KEYSTONE_RPROC_IOC_MAGIC, \
+						2, uint32_t)
+#define KEYSTONE_RPROC_IOC_DSP_RESET		_IO(KEYSTONE_RPROC_IOC_MAGIC, 3)
+#define KEYSTONE_RPROC_IOC_DSP_BOOT		_IOW(KEYSTONE_RPROC_IOC_MAGIC, \
+						4, uint32_t)
+
+#define KEYSTONE_RPROC_IOC_MAXNR		(5)
+
+#endif /* _UAPI_LINUX_KEYSTONE_REMOTEPROC_H_ */
diff -Naur --no-dereference a/include/uapi/linux/net_switch_config.h b/include/uapi/linux/net_switch_config.h
--- a/include/uapi/linux/net_switch_config.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/uapi/linux/net_switch_config.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,64 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Texas Instruments Ethernet Switch Driver
+ *
+ * Copyright (C) 2014-2018 Texas Instruments, Inc
+ *
+ * Userspace API for Switch Configuration
+ */
+
+#ifndef __NET_CONFIG_SWITCH_H__
+#define __NET_CONFIG_SWITCH_H__
+
+enum {
+	CONFIG_SWITCH_INVALID,
+	CONFIG_SWITCH_ADD_MULTICAST,
+	CONFIG_SWITCH_DEL_MULTICAST,
+	CONFIG_SWITCH_ADD_VLAN,
+	CONFIG_SWITCH_DEL_VLAN,
+	CONFIG_SWITCH_SET_PORT_CONFIG,
+	CONFIG_SWITCH_GET_PORT_CONFIG,
+	CONFIG_SWITCH_ADD_UNKNOWN_VLAN_INFO,
+	CONFIG_SWITCH_GET_PORT_STATE,
+	CONFIG_SWITCH_SET_PORT_STATE,
+	CONFIG_SWITCH_GET_PORT_VLAN_CONFIG,
+	CONFIG_SWITCH_SET_PORT_VLAN_CONFIG,
+	CONFIG_SWITCH_RATELIMIT,
+};
+
+enum {
+	PORT_STATE_DISABLED = 0,
+	PORT_STATE_BLOCKED,
+	PORT_STATE_LEARN,
+	PORT_STATE_FORWARD,
+};
+
+/*
+ * Pass all unused parameters as zero is recomented.
+ */
+struct net_switch_config {
+	unsigned int cmd;	/* API to be invoked by the kernel driver */
+
+	unsigned int	port;
+	unsigned int	vid;		/* VLAN identifier */
+	unsigned char	unreg_multi;	/* unreg multicast Egress Ports */
+	unsigned char	reg_multi;	/* register multicast Egress ports */
+	unsigned char	untag_port;	/* Untag ports */
+	unsigned char	addr[6];
+	unsigned int	super;
+	struct ethtool_cmd ecmd;
+	unsigned char	unknown_vlan_member;
+	unsigned char	unknown_vlan_untag;
+	unsigned int	unknown_vlan_unreg_multi;
+	unsigned int	unknown_vlan_reg_multi;
+	unsigned int	port_state;
+	unsigned int	prio;
+	bool		vlan_cfi;
+	unsigned int	bcast_rate_limit;
+	unsigned int	mcast_rate_limit;
+	bool		direction;
+
+	unsigned int ret_type;   /* Return  Success/Failure */
+};
+
+#endif /* __NET_CONFIG_SWITCH_H__*/
diff -Naur --no-dereference a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h
--- a/include/uapi/linux/nl80211.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/uapi/linux/nl80211.h	2020-06-19 21:00:31.432227716 -0400
@@ -178,18 +178,27 @@
  *
  * By setting @NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK flag drivers
  * can indicate they support offloading EAPOL handshakes for WPA/WPA2
- * preshared key authentication. In %NL80211_CMD_CONNECT the preshared
- * key should be specified using %NL80211_ATTR_PMK. Drivers supporting
- * this offload may reject the %NL80211_CMD_CONNECT when no preshared
- * key material is provided, for example when that driver does not
- * support setting the temporal keys through %CMD_NEW_KEY.
+ * preshared key authentication in station mode. In %NL80211_CMD_CONNECT
+ * the preshared key should be specified using %NL80211_ATTR_PMK. Drivers
+ * supporting this offload may reject the %NL80211_CMD_CONNECT when no
+ * preshared key material is provided, for example when that driver does
+ * not support setting the temporal keys through %NL80211_CMD_NEW_KEY.
  *
  * Similarly @NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_1X flag can be
  * set by drivers indicating offload support of the PTK/GTK EAPOL
- * handshakes during 802.1X authentication. In order to use the offload
- * the %NL80211_CMD_CONNECT should have %NL80211_ATTR_WANT_1X_4WAY_HS
- * attribute flag. Drivers supporting this offload may reject the
- * %NL80211_CMD_CONNECT when the attribute flag is not present.
+ * handshakes during 802.1X authentication in station mode. In order to
+ * use the offload the %NL80211_CMD_CONNECT should have
+ * %NL80211_ATTR_WANT_1X_4WAY_HS attribute flag. Drivers supporting this
+ * offload may reject the %NL80211_CMD_CONNECT when the attribute flag is
+ * not present.
+ *
+ * By setting @NL80211_EXT_FEATURE_4WAY_HANDSHAKE_AP_PSK flag drivers
+ * can indicate they support offloading EAPOL handshakes for WPA/WPA2
+ * preshared key authentication in AP mode. In %NL80211_CMD_START_AP
+ * the preshared key should be specified using %NL80211_ATTR_PMK. Drivers
+ * supporting this offload may reject the %NL80211_CMD_START_AP when no
+ * preshared key material is provided, for example when that driver does
+ * not support setting the temporal keys through %NL80211_CMD_NEW_KEY.
  *
  * For 802.1X the PMK or PMK-R0 are set by providing %NL80211_ATTR_PMK
  * using %NL80211_CMD_SET_PMK. For offloaded FT support also
@@ -235,6 +244,16 @@
  */
 
 /**
+ * DOC: SAE authentication offload
+ *
+ * By setting @NL80211_EXT_FEATURE_SAE_OFFLOAD flag drivers can indicate they
+ * support offloading SAE authentication for WPA3-Personal networks. The
+ * password for SAE should be specified using %NL80211_ATTR_SAE_PASSWORD in
+ * %NL80211_CMD_CONNECT and %NL80211_CMD_START_AP for station and AP mode
+ * respectively.
+ */
+
+/**
  * enum nl80211_commands - supported nl80211 commands
  *
  * @NL80211_CMD_UNSPEC: unspecified command to catch errors
@@ -2199,10 +2218,11 @@
  *
  * @NL80211_ATTR_PMK: attribute for passing PMK key material. Used with
  *	%NL80211_CMD_SET_PMKSA for the PMKSA identified by %NL80211_ATTR_PMKID.
- *	For %NL80211_CMD_CONNECT it is used to provide PSK for offloading 4-way
- *	handshake for WPA/WPA2-PSK networks. For 802.1X authentication it is
- *	used with %NL80211_CMD_SET_PMK. For offloaded FT support this attribute
- *	specifies the PMK-R0 if NL80211_ATTR_PMKR0_NAME is included as well.
+ *	For %NL80211_CMD_CONNECT and %NL80211_CMD_START_AP it is used to provide
+ *	PSK for offloading 4-way handshake for WPA/WPA2-PSK networks. For 802.1X
+ *	authentication it is used with %NL80211_CMD_SET_PMK. For offloaded FT
+ *	support this attribute specifies the PMK-R0 if NL80211_ATTR_PMKR0_NAME
+ *	is included as well.
  *
  * @NL80211_ATTR_SCHED_SCAN_MULTI: flag attribute which user-space shall use to
  *	indicate that it supports multiple active scheduled scan requests.
@@ -2213,7 +2233,10 @@
  *	in %NL80211_CMD_CONNECT to indicate that for 802.1X authentication it
  *	wants to use the supported offload of the 4-way handshake.
  * @NL80211_ATTR_PMKR0_NAME: PMK-R0 Name for offloaded FT.
- * @NL80211_ATTR_PORT_AUTHORIZED: (reserved)
+ * @NL80211_ATTR_PORT_AUTHORIZED: flag attribute used in %NL80211_CMD_ROAMED
+ *	notification indicating that that 802.1X authentication was done by
+ *	the driver or is not needed (because roaming used the Fast Transition
+ *	protocol).
  *
  * @NL80211_ATTR_EXTERNAL_AUTH_ACTION: Identify the requested external
  *     authentication operation (u32 attribute with an
@@ -2241,6 +2264,40 @@
  *	association request when used with NL80211_CMD_NEW_STATION). Can be set
  *	only if %NL80211_STA_FLAG_WME is set.
  *
+ * @NL80211_ATTR_FTM_RESPONDER: nested attribute which user-space can include
+ *	in %NL80211_CMD_START_AP or %NL80211_CMD_SET_BEACON for fine timing
+ *	measurement (FTM) responder functionality and containing parameters as
+ *	possible, see &enum nl80211_ftm_responder_attr
+ *
+ * @NL80211_ATTR_FTM_RESPONDER_STATS: Nested attribute with FTM responder
+ *	statistics, see &enum nl80211_ftm_responder_stats.
+ *
+ * @NL80211_ATTR_TIMEOUT: Timeout for the given operation in milliseconds (u32),
+ *	if the attribute is not given no timeout is requested. Note that 0 is an
+ *	invalid value.
+ *
+ * @NL80211_ATTR_PEER_MEASUREMENTS: peer measurements request (and result)
+ *	data, uses nested attributes specified in
+ *	&enum nl80211_peer_measurement_attrs.
+ *	This is also used for capability advertisement in the wiphy information,
+ *	with the appropriate sub-attributes.
+ *
+ * @NL80211_ATTR_AIRTIME_WEIGHT: Station's weight when scheduled by the airtime
+ *	scheduler.
+ *
+ * @NL80211_ATTR_STA_TX_POWER_SETTING: Transmit power setting type (u8) for
+ *	station associated with the AP. See &enum nl80211_tx_power_setting for
+ *	possible values.
+ * @NL80211_ATTR_STA_TX_POWER: Transmit power level (s16) in dBm units. This
+ *	allows to set Tx power for a station. If this attribute is not included,
+ *	the default per-interface tx power setting will be overriding. Driver
+ *	should be picking up the lowest tx power, either tx power per-interface
+ *	or per-station.
+ *
+ * @NL80211_ATTR_SAE_PASSWORD: attribute for passing SAE password material. It
+ *	is used with %NL80211_CMD_CONNECT to provide password for offloading
+ *	SAE authentication for WPA3-Personal networks.
+ *
  * @NUM_NL80211_ATTR: total number of nl80211_attrs available
  * @NL80211_ATTR_MAX: highest attribute number currently defined
  * @__NL80211_ATTR_AFTER_LAST: internal use
@@ -2682,6 +2739,20 @@
 
 	NL80211_ATTR_HE_CAPABILITY,
 
+	NL80211_ATTR_FTM_RESPONDER,
+
+	NL80211_ATTR_FTM_RESPONDER_STATS,
+
+	NL80211_ATTR_TIMEOUT,
+
+	NL80211_ATTR_PEER_MEASUREMENTS,
+
+	NL80211_ATTR_AIRTIME_WEIGHT,
+	NL80211_ATTR_STA_TX_POWER_SETTING,
+	NL80211_ATTR_STA_TX_POWER,
+
+	NL80211_ATTR_SAE_PASSWORD,
+
 	/* add attributes here, update the policy in nl80211.c */
 
 	__NL80211_ATTR_AFTER_LAST,
@@ -4233,6 +4304,7 @@
 enum nl80211_wpa_versions {
 	NL80211_WPA_VERSION_1 = 1 << 0,
 	NL80211_WPA_VERSION_2 = 1 << 1,
+	NL80211_WPA_VERSION_3 = 1 << 2,
 };
 
 /**
@@ -5223,6 +5295,36 @@
  * @NL80211_EXT_FEATURE_SCAN_MIN_PREQ_CONTENT: Driver/device can omit all data
  *	except for supported rates from the probe request content if requested
  *	by the %NL80211_SCAN_FLAG_MIN_PREQ_CONTENT flag.
+ * @NL80211_EXT_FEATURE_ENABLE_FTM_RESPONDER: Driver supports enabling fine
+ *	timing measurement responder role.
+ *
+ * @NL80211_EXT_FEATURE_CAN_REPLACE_PTK0: Driver/device confirm that they are
+ *	able to rekey an in-use key correctly. Userspace must not rekey PTK keys
+ *	if this flag is not set. Ignoring this can leak clear text packets and/or
+ *	freeze the connection.
+ * @NL80211_EXT_FEATURE_EXT_KEY_ID: Driver supports "Extended Key ID for
+ *	Individually Addressed Frames" from IEEE802.11-2016.
+ *
+ * @NL80211_EXT_FEATURE_AIRTIME_FAIRNESS: Driver supports getting airtime
+ *	fairness for transmitted packets and has enabled airtime fairness
+ *	scheduling.
+ *
+ * @NL80211_EXT_FEATURE_AP_PMKSA_CACHING: Driver/device supports PMKSA caching
+ *	(set/del PMKSA operations) in AP mode.
+ *
+ * @NL80211_EXT_FEATURE_SCHED_SCAN_BAND_SPECIFIC_RSSI_THOLD: Driver supports
+ *	filtering of sched scan results using band specific RSSI thresholds.
+ *
+ * @NL80211_EXT_FEATURE_STA_TX_PWR: This driver supports controlling tx power
+ *	to a station.
+ *
+ * @NL80211_EXT_FEATURE_SAE_OFFLOAD: Device wants to do SAE authentication in
+ *	station mode (SAE password is passed as part of the connect command)
+ *	or AP mode (SAE password is passed as part of the start AP command).
+ *
+ * @NL80211_EXT_FEATURE_4WAY_HANDSHAKE_AP_PSK: Device wants to do 4-way
+ *	handshake with PSK in AP mode (PSK is passed as part of the start AP
+ *	command).
  *
  * @NUM_NL80211_EXT_FEATURES: number of extended features.
  * @MAX_NL80211_EXT_FEATURES: highest extended feature index.
@@ -5259,6 +5361,15 @@
 	NL80211_EXT_FEATURE_TXQS,
 	NL80211_EXT_FEATURE_SCAN_RANDOM_SN,
 	NL80211_EXT_FEATURE_SCAN_MIN_PREQ_CONTENT,
+	NL80211_EXT_FEATURE_CAN_REPLACE_PTK0,
+	NL80211_EXT_FEATURE_ENABLE_FTM_RESPONDER,
+	NL80211_EXT_FEATURE_AIRTIME_FAIRNESS,
+	NL80211_EXT_FEATURE_AP_PMKSA_CACHING,
+	NL80211_EXT_FEATURE_SCHED_SCAN_BAND_SPECIFIC_RSSI_THOLD,
+	NL80211_EXT_FEATURE_EXT_KEY_ID,
+	NL80211_EXT_FEATURE_STA_TX_PWR,
+	NL80211_EXT_FEATURE_SAE_OFFLOAD,
+	NL80211_EXT_FEATURE_4WAY_HANDSHAKE_AP_PSK,
 
 	/* add new features before the definition below */
 	NUM_NL80211_EXT_FEATURES,
diff -Naur --no-dereference a/include/uapi/linux/pcitest.h b/include/uapi/linux/pcitest.h
--- a/include/uapi/linux/pcitest.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/uapi/linux/pcitest.h	2020-06-19 21:00:31.432227716 -0400
@@ -19,5 +19,6 @@
 #define PCITEST_MSIX		_IOW('P', 0x7, int)
 #define PCITEST_SET_IRQTYPE	_IOW('P', 0x8, int)
 #define PCITEST_GET_IRQTYPE	_IO('P', 0x9)
+#define PCITEST_CLEAR_IRQ	_IO('P', 0x10)
 
 #endif /* __UAPI_LINUX_PCITEST_H */
diff -Naur --no-dereference a/include/uapi/linux/rpmsg_rpc.h b/include/uapi/linux/rpmsg_rpc.h
--- a/include/uapi/linux/rpmsg_rpc.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/uapi/linux/rpmsg_rpc.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,183 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/*
+ * Remote Processor Procedure Call Driver
+ *
+ * Copyright (C) 2012-2019 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+#ifndef _UAPI_LINUX_RPMSG_RPC_H_
+#define _UAPI_LINUX_RPMSG_RPC_H_
+
+#include <linux/ioctl.h>
+
+/**
+ * struct rppc_buf_fds - rppc buffer registration/deregistration
+ * @num: number of file descriptors
+ * @fds: pointer to the array holding the file descriptors
+ */
+struct rppc_buf_fds {
+	uint32_t num;
+	int32_t *fds;
+};
+
+/*
+ * ioctl definitions
+ */
+#define RPPC_IOC_MAGIC		'r'
+#define RPPC_IOC_CREATE		_IOW(RPPC_IOC_MAGIC, 1, char *)
+#define RPPC_IOC_BUFREGISTER    _IOW(RPPC_IOC_MAGIC, 2, struct rppc_buf_fds)
+#define RPPC_IOC_BUFUNREGISTER  _IOW(RPPC_IOC_MAGIC, 3, struct rppc_buf_fds)
+#define RPPC_IOC_MAXNR		(4)
+
+#define RPPC_MAX_PARAMETERS	(10)
+#define RPPC_MAX_TRANSLATIONS	(1024)
+#define RPPC_MAX_INST_NAMELEN	(48)
+
+/**
+ * enum rppc_param_type - RPC function parameter type
+ * @RPPC_PARAM_TYPE_UNKNOWN: unrecognized parameter
+ * @RPPC_PARAM_TYPE_ATOMIC: an atomic data type, 1 byte to architecture limit
+ *			    sized bytes
+ * @RPPC_PARAM_TYPE_PTR: a pointer to shared memory. The fd field in the
+ *			 structures rppc_param and rppc_param_translation must
+ *			 contain the file descriptor of the associated dma_buf
+ * @RPPC_PARAM_TYPE_STRUCT: (unsupported) a structure type. Will be architecture
+ *			    width aligned in memory
+ *
+ * These enum values are used to identify the parameter type for every
+ * parameter argument of the remote function.
+ */
+enum rppc_param_type {
+	RPPC_PARAM_TYPE_UNKNOWN = 0,
+	RPPC_PARAM_TYPE_ATOMIC,
+	RPPC_PARAM_TYPE_PTR,
+	RPPC_PARAM_TYPE_STRUCT,
+};
+
+/**
+ * struct rppc_param_translation - pointer translation helper structure
+ * @index: index of the parameter where the translation needs to be done in.
+ *	   used for computing the primary offset and mapping into kernel
+ *	   the page from the buffer referred to in the corresponding parameter
+ * @offset: offset from the primary base pointer to the pointer to translate.
+ *	    This is the secondary offset, and used either for mentioning the
+ *	    offset from an structure array element base, or within a single
+ *	    structure which itself is at an offset in an allocated buffer
+ * @base: the base user virtual address of the pointer to translate (used to
+ *	  calculate translated pointer offset)
+ * @fd: dma_buf file descriptor of the allocated buffer pointer within which
+ *	the translated pointer is present
+ */
+struct rppc_param_translation {
+	uint32_t index;
+	ptrdiff_t offset;
+	size_t base;
+	int32_t fd;
+};
+
+/**
+ * struct rppc_param - descriptor structure for each parameter
+ * @type: type of the parameter, as dictated by enum rppc_param_type
+ * @size: size of the data (for atomic types) or size of the containing
+ *	  structure in which translations are performed
+ * @data: either the parameter value itself (for atomic type) or
+ *	  the actual user space pointer address to the data (for pointer type)
+ * @base: the base user space pointer address of the original allocated buffer,
+ *	  providing a reference if data has the pointer that is at an offset
+ *	  from the original pointer
+ * @fd: file descriptor of the exported allocation (will be used to
+ *	import the associated dma_buf within the driver).
+ */
+struct rppc_param {
+	uint32_t type;
+	size_t size;
+	size_t data;
+	size_t base;
+	int32_t fd;
+};
+
+/**
+ * struct rppc_function - descriptor structure for the remote function
+ * @fxn_id: index of the function to invoke on the opened rppc device
+ * @num_params: number of parameters filled in the params field
+ * @params: array of parameter descriptor structures
+ * @num_translations: number of in-place translations to be performed within
+ *		      the arguments.
+ * @translations: an open array of the translation descriptor structures, whose
+ *		  length is given in @num_translations. Used for translating
+ *		  the pointers within the function data.
+ *
+ * This is the primary descriptor structure passed down from the userspace,
+ * describing the function, its parameter arguments and the needed translations.
+ */
+struct rppc_function {
+	uint32_t fxn_id;
+	uint32_t num_params;
+	struct rppc_param params[RPPC_MAX_PARAMETERS];
+	uint32_t num_translations;
+	struct rppc_param_translation translations[0];
+};
+
+/**
+ * struct rppc_function_return - function return status descriptor structure
+ * @fxn_id: index of the function invoked on the opened rppc device
+ * @status: return value of the executed function
+ */
+struct rppc_function_return {
+	uint32_t fxn_id;
+	uint32_t status;
+};
+
+/**
+ * struct rppc_create_instance - rppc channel connector helper
+ * @name: Name of the rppc server device to establish a connection with
+ */
+struct rppc_create_instance {
+	char name[RPPC_MAX_INST_NAMELEN];
+};
+
+/*
+ * helper macros for manipulating the function index in the marshalled packet
+ */
+#define RPPC_DESC_EXEC_SYNC	(0x0100)
+#define RPPC_DESC_TYPE_MASK	(0x0F00)
+
+/*
+ * helper macros for manipulating the function index in the marshalled packet.
+ * The remote functions are offset by one relative to the client
+ * XXX: Remove the relative offset
+ */
+#define RPPC_SET_FXN_IDX(idx)	(((idx) + 1) | 0x80000000)
+#define RPPC_FXN_MASK(idx)	(((idx) - 1) & 0x7FFFFFFF)
+
+/**
+ * struct rppc_packet - the actual marshalled packet
+ * @desc: type of function execution, currently only synchronous function
+ *	  invocations are supported
+ * @msg_id: an incremental message index identifier
+ * @flags: a combination of job id and pool id of the worker threads
+ *	   of the server
+ * @fxn_id: id of the function to execute
+ * @result: result of the remotely executed function
+ * @data_size: size of the payload packet
+ * @data: variable payload, containing the marshalled function data.
+ *
+ * This is actually a condensed structure of the Remote Command Messaging
+ * (RCM) structure. The initial fields of the structure are used by the
+ * remote-side server to schedule the execution of the function. The actual
+ * variable payload data starts from the .data field. This marshalled packet
+ * is the payload for a rpmsg message.
+ *
+ * XXX: remove or mask unneeded fields, some fields can be stripped down
+ */
+struct rppc_packet {
+	uint16_t desc;
+	uint16_t msg_id;
+	uint32_t flags;
+	uint32_t fxn_id;
+	int32_t  result;
+	uint32_t data_size;
+	uint8_t  data[0];
+} __packed;
+
+#endif /* _UAPI_LINUX_RPMSG_RPC_H_ */
diff -Naur --no-dereference a/include/uapi/linux/rpmsg_socket.h b/include/uapi/linux/rpmsg_socket.h
--- a/include/uapi/linux/rpmsg_socket.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/uapi/linux/rpmsg_socket.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/*
+ * Remote processor messaging sockets
+ *
+ * Copyright (C) 2011-2018 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * Ohad Ben-Cohen <ohad@wizery.com>
+ * Suman Anna <s-anna@ti.com>
+ */
+
+#ifndef _UAPI_RPMSG_SOCKET_H
+#define _UAPI_RPMSG_SOCKET_H
+
+#include <linux/types.h>
+#include <linux/socket.h>
+
+/* user space needs this */
+#ifndef AF_RPMSG
+#define AF_RPMSG	45
+#define PF_RPMSG	AF_RPMSG
+#endif
+
+struct sockaddr_rpmsg {
+	__kernel_sa_family_t family;
+	__u32 vproc_id;
+	__u32 addr;
+};
+
+#define RPMSG_LOCALHOST ((__u32)~0UL)
+
+#endif /* _UAPI_RPMSG_SOCKET_H */
diff -Naur --no-dereference a/include/uapi/linux/serial_core.h b/include/uapi/linux/serial_core.h
--- a/include/uapi/linux/serial_core.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/uapi/linux/serial_core.h	2020-06-19 21:00:31.432227716 -0400
@@ -281,4 +281,7 @@
 /* MediaTek BTIF */
 #define PORT_MTK_BTIF	117
 
+/* PRU SW UART */
+#define PORT_PSUART     118
+
 #endif /* _UAPILINUX_SERIAL_CORE_H */
diff -Naur --no-dereference a/include/uapi/linux/sockios.h b/include/uapi/linux/sockios.h
--- a/include/uapi/linux/sockios.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/uapi/linux/sockios.h	2020-06-19 21:00:31.432227716 -0400
@@ -132,6 +132,9 @@
 #define SIOCSHWTSTAMP	0x89b0		/* set and get config		*/
 #define SIOCGHWTSTAMP	0x89b1		/* get config			*/
 
+/* Switch config calls: parameters in linux/net_switch_config.h */
+#define SIOCSWITCHCONFIG    0x89c0
+
 /* Device private ioctl calls */
 
 /*
diff -Naur --no-dereference a/include/uapi/linux/ti-pat.h b/include/uapi/linux/ti-pat.h
--- a/include/uapi/linux/ti-pat.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/uapi/linux/ti-pat.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+/*
+ * TI PAT mapped DMA-BUF memory exporter UAPI
+ *
+ * Copyright (C) 2018-2019 Texas Instruments Incorporated - http://www.ti.com/
+ *	Andrew F. Davis <afd@ti.com>
+ */
+
+#ifndef _UAPI_LINUX_TI_PAT_H
+#define _UAPI_LINUX_TI_PAT_H
+
+#include <linux/ioctl.h>
+#include <linux/types.h>
+
+/**
+ * DOC: TI PAT Userspace API
+ *
+ * create a client by opening /dev/ti-pat
+ * most operations handled via following ioctls
+ */
+
+/**
+ * struct ti_pat_allocation_data - metadata passed from userspace for allocations
+ * @fd:			populated with DMA-BUF FD for this allocation
+ * @flags:		flags for the allocation
+ *
+ * Provided by userspace as an argument to the ioctl
+ */
+struct ti_pat_export_data {
+	__u32 fd;
+	__u32 flags;
+};
+
+#define TI_PAT_IOC_MAGIC 'P'
+
+/**
+ * DOC: TI_PAT_IOC_EXPORT - Re-export DMA-BUF through TI PAT
+ *
+ * Takes an ti_pat_export_data struct and returns it with the fd field
+ * populated with the DMA-BUF handle for the new export.
+ */
+#define TI_PAT_IOC_EXPORT _IOWR(TI_PAT_IOC_MAGIC, 0, struct ti_pat_export_data)
+
+#endif /* _UAPI_LINUX_TI_PAT_H */
diff -Naur --no-dereference a/include/uapi/linux/videodev2.h b/include/uapi/linux/videodev2.h
--- a/include/uapi/linux/videodev2.h	2020-01-09 04:19:10.000000000 -0500
+++ b/include/uapi/linux/videodev2.h	2020-06-19 21:00:31.432227716 -0400
@@ -70,7 +70,7 @@
  * Common stuff for both V4L1 and V4L2
  * Moved from videodev.h
  */
-#define VIDEO_MAX_FRAME               32
+#define VIDEO_MAX_FRAME               128
 #define VIDEO_MAX_PLANES               8
 
 /*
@@ -687,6 +687,8 @@
 #define V4L2_PIX_FMT_Z16      v4l2_fourcc('Z', '1', '6', ' ') /* Depth data 16-bit */
 #define V4L2_PIX_FMT_MT21C    v4l2_fourcc('M', 'T', '2', '1') /* Mediatek compressed block mode  */
 #define V4L2_PIX_FMT_INZI     v4l2_fourcc('I', 'N', 'Z', 'I') /* Intel Planar Greyscale 10-bit and Depth 16-bit */
+#define V4L2_PIX_FMT_TI1210   v4l2_fourcc('T', 'I', '1', '2') /* TI NV12 10-bit, two bytes per channel */
+#define V4L2_PIX_FMT_TI1610   v4l2_fourcc('T', 'I', '1', '6') /* TI NV16 10-bit, two bytes per channel */
 
 /* 10bit raw bayer packed, 32 bytes for every 25 pixels, last LSB 6 bits unused */
 #define V4L2_PIX_FMT_IPU3_SBGGR10	v4l2_fourcc('i', 'p', '3', 'b') /* IPU3 packed 10-bit BGGR bayer */
diff -Naur --no-dereference a/include/uapi/scsi/scsi_bsg_ufs.h b/include/uapi/scsi/scsi_bsg_ufs.h
--- a/include/uapi/scsi/scsi_bsg_ufs.h	1969-12-31 19:00:00.000000000 -0500
+++ b/include/uapi/scsi/scsi_bsg_ufs.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,105 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * UFS Transport SGIO v4 BSG Message Support
+ *
+ * Copyright (C) 2011-2013 Samsung India Software Operations
+ * Copyright (C) 2018 Western Digital Corporation
+ */
+#ifndef SCSI_BSG_UFS_H
+#define SCSI_BSG_UFS_H
+
+/*
+ * This file intended to be included by both kernel and user space
+ */
+
+#define UFS_CDB_SIZE	16
+#define UPIU_TRANSACTION_UIC_CMD 0x1F
+/* uic commands are 4DW long, per UFSHCI V2.1 paragraph 5.6.1 */
+#define UIC_CMD_SIZE (sizeof(u32) * 4)
+
+/**
+ * struct utp_upiu_header - UPIU header structure
+ * @dword_0: UPIU header DW-0
+ * @dword_1: UPIU header DW-1
+ * @dword_2: UPIU header DW-2
+ */
+struct utp_upiu_header {
+	__be32 dword_0;
+	__be32 dword_1;
+	__be32 dword_2;
+};
+
+/**
+ * struct utp_upiu_query - upiu request buffer structure for
+ * query request.
+ * @opcode: command to perform B-0
+ * @idn: a value that indicates the particular type of data B-1
+ * @index: Index to further identify data B-2
+ * @selector: Index to further identify data B-3
+ * @reserved_osf: spec reserved field B-4,5
+ * @length: number of descriptor bytes to read/write B-6,7
+ * @value: Attribute value to be written DW-5
+ * @reserved: spec reserved DW-6,7
+ */
+struct utp_upiu_query {
+	__u8 opcode;
+	__u8 idn;
+	__u8 index;
+	__u8 selector;
+	__be16 reserved_osf;
+	__be16 length;
+	__be32 value;
+	__be32 reserved[2];
+};
+
+/**
+ * struct utp_upiu_cmd - Command UPIU structure
+ * @data_transfer_len: Data Transfer Length DW-3
+ * @cdb: Command Descriptor Block CDB DW-4 to DW-7
+ */
+struct utp_upiu_cmd {
+	__be32 exp_data_transfer_len;
+	u8 cdb[UFS_CDB_SIZE];
+};
+
+/**
+ * struct utp_upiu_req - general upiu request structure
+ * @header:UPIU header structure DW-0 to DW-2
+ * @sc: fields structure for scsi command DW-3 to DW-7
+ * @qr: fields structure for query request DW-3 to DW-7
+ */
+struct utp_upiu_req {
+	struct utp_upiu_header header;
+	union {
+		struct utp_upiu_cmd		sc;
+		struct utp_upiu_query		qr;
+		struct utp_upiu_query		tr;
+		/* use utp_upiu_query to host the 4 dwords of uic command */
+		struct utp_upiu_query		uc;
+	};
+};
+
+/* request (CDB) structure of the sg_io_v4 */
+struct ufs_bsg_request {
+	uint32_t msgcode;
+	struct utp_upiu_req upiu_req;
+};
+
+/* response (request sense data) structure of the sg_io_v4 */
+struct ufs_bsg_reply {
+	/*
+	 * The completion result. Result exists in two forms:
+	 * if negative, it is an -Exxx system errno value. There will
+	 * be no further reply information supplied.
+	 * else, it's the 4-byte scsi error result, with driver, host,
+	 * msg and status fields. The per-msgcode reply structure
+	 * will contain valid data.
+	 */
+	uint32_t result;
+
+	/* If there was reply_payload, how much was received? */
+	uint32_t reply_payload_rcv_len;
+
+	struct utp_upiu_req upiu_rsp;
+};
+#endif /* UFS_BSG_H */
diff -Naur --no-dereference a/jenkins_build.sh b/jenkins_build.sh
--- a/jenkins_build.sh	1969-12-31 19:00:00.000000000 -0500
+++ b/jenkins_build.sh	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,62 @@
+#!/bin/bash
+
+#gcc:
+
+#
+#https://releases.linaro.org/components/toolchain/binaries/7.1-2017.05/arm-linux-gnueabihf/gcc-linaro-7.1.1-2017.05-x86_64_arm-linux-gnueabihf.tar.xz
+#https://releases.linaro.org/components/toolchain/binaries/7.1-2017.08/arm-linux-gnueabihf/gcc-linaro-7.1.1-2017.08-x86_64_arm-linux-gnueabihf.tar.xz
+#https://releases.linaro.org/components/toolchain/binaries/7.2-2017.11/arm-linux-gnueabihf/gcc-linaro-7.2.1-2017.11-x86_64_arm-linux-gnueabihf.tar.xz
+#https://releases.linaro.org/components/toolchain/binaries/7.3-2018.05/arm-linux-gnueabihf/gcc-linaro-7.3.1-2018.05-x86_64_arm-linux-gnueabihf.tar.xz
+#https://releases.linaro.org/components/toolchain/binaries/7.4-2019.02/arm-linux-gnueabihf/gcc-linaro-7.4.1-2019.02-x86_64_arm-linux-gnueabihf.tar.xz
+#https://releases.linaro.org/components/toolchain/binaries/7.5-2019.12/arm-linux-gnueabihf/gcc-linaro-7.5.0-2019.12-x86_64_arm-linux-gnueabihf.tar.xz
+#
+
+gcc_html_path="https://releases.linaro.org/components/toolchain/binaries/7.5-2019.12/arm-linux-gnueabihf/"
+gcc_filename_prefix="gcc-linaro-7.5.0-2019.12-x86_64_arm-linux-gnueabihf"
+gcc_banner="arm-linux-gnueabihf-gcc (Linaro GCC 7.5-2019.12) 7.5.0"
+gcc_copyright="2017"
+datestamp="2019.12-gcc-arm-linux-gnueabihf"
+
+#
+
+if [ ! -d ${gcc_filename_prefix}/ ] ; then
+	rm -rf ./gcc-* || true
+	#wget -c ${site}/${version}/${filename}
+	wget -c http://gfnd.rcn-ee.org/farm/jenkins-dl/${gcc_filename_prefix}.tar.xz
+	tar xf ${gcc_filename_prefix}.tar.xz
+fi
+
+export CC=`pwd`/${gcc_filename_prefix}/bin/arm-linux-gnueabihf-
+
+make ARCH=arm clean
+make ARCH=arm bb.org_defconfig
+
+echo "[make ARCH=arm -j4 CROSS_COMPILE=\"${binary}\" zImage]"
+make ARCH=arm -j4 CROSS_COMPILE="ccache ${CC}" zImage
+if [ ! -f arch/arm/boot/zImage ] ; then
+	echo "failed: [arch/arm/boot/zImage]"
+	exit 1
+fi
+
+echo "[make ARCH=arm -j4 CROSS_COMPILE=\"${binary}\" modules]"
+make ARCH=arm -j4 CROSS_COMPILE="ccache ${CC}" modules
+if [ ! -f drivers/spi/spidev.ko ] ; then
+	echo "failed: [drivers/spi/spidev.ko]"
+	exit 1
+fi
+
+echo "[make ARCH=arm CROSS_COMPILE=\"${binary}\" dtbs]"
+make ARCH=arm CROSS_COMPILE="ccache ${CC}" dtbs
+if [ ! -f arch/arm/boot/dts/am335x-boneblack.dtb ] ; then
+	echo "failed: [arch/arm/boot/dts/am335x-boneblack.dtb]"
+	exit 1
+else
+	if [ -f arch/arm/boot/dts/am335x-pocketbeagle.dts ] ; then
+		if [ ! -f arch/arm/boot/dts/am335x-pocketbeagle.dtb ] ; then
+			echo "failed: [arch/arm/boot/dts/am335x-pocketbeagle.dtb]"
+			exit 1
+		fi
+	fi
+fi
+
+make ARCH=arm clean
diff -Naur --no-dereference a/Jenkinsfile b/Jenkinsfile
--- a/Jenkinsfile	1969-12-31 19:00:00.000000000 -0500
+++ b/Jenkinsfile	2020-06-19 21:00:31.400227715 -0400
@@ -0,0 +1,11 @@
+pipeline {
+    agent any
+
+    stages {
+        stage('Build') {
+            steps {
+                sh '/bin/bash ./jenkins_build.sh'
+            }
+        }
+    }
+}
diff -Naur --no-dereference a/kernel/dma/coherent.c b/kernel/dma/coherent.c
--- a/kernel/dma/coherent.c	2020-01-09 04:19:10.000000000 -0500
+++ b/kernel/dma/coherent.c	2020-06-19 21:00:31.432227716 -0400
@@ -161,7 +161,7 @@
 EXPORT_SYMBOL(dma_mark_declared_memory_occupied);
 
 static void *__dma_alloc_from_coherent(struct dma_coherent_mem *mem,
-		ssize_t size, dma_addr_t *dma_handle)
+		ssize_t size, dma_addr_t *dma_handle, bool zero)
 {
 	int order = get_order(size);
 	unsigned long flags;
@@ -183,7 +183,8 @@
 	*dma_handle = mem->device_base + (pageno << PAGE_SHIFT);
 	ret = mem->virt_base + (pageno << PAGE_SHIFT);
 	spin_unlock_irqrestore(&mem->spinlock, flags);
-	memset(ret, 0, size);
+	if (zero)
+		memset(ret, 0, size);
 	return ret;
 err:
 	spin_unlock_irqrestore(&mem->spinlock, flags);
@@ -205,14 +206,14 @@
  * generic memory areas, or !0 if dma_alloc_coherent should return @ret.
  */
 int dma_alloc_from_dev_coherent(struct device *dev, ssize_t size,
-		dma_addr_t *dma_handle, void **ret)
+		dma_addr_t *dma_handle, void **ret, bool zero)
 {
 	struct dma_coherent_mem *mem = dev_get_coherent_memory(dev);
 
 	if (!mem)
 		return 0;
 
-	*ret = __dma_alloc_from_coherent(mem, size, dma_handle);
+	*ret = __dma_alloc_from_coherent(mem, size, dma_handle, zero);
 	if (*ret)
 		return 1;
 
@@ -231,7 +232,7 @@
 		return NULL;
 
 	return __dma_alloc_from_coherent(dma_coherent_default_memory, size,
-			dma_handle);
+			dma_handle, true);
 }
 
 static int __dma_release_from_coherent(struct dma_coherent_mem *mem,
diff -Naur --no-dereference a/kernel/fork.c b/kernel/fork.c
--- a/kernel/fork.c	2020-01-09 04:19:10.000000000 -0500
+++ b/kernel/fork.c	2020-06-19 21:00:31.432227716 -0400
@@ -500,7 +500,7 @@
 			struct inode *inode = file_inode(file);
 			struct address_space *mapping = file->f_mapping;
 
-			get_file(file);
+			vma_get_file(tmp);
 			if (tmp->vm_flags & VM_DENYWRITE)
 				atomic_dec(&inode->i_writecount);
 			i_mmap_lock_write(mapping);
diff -Naur --no-dereference a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c
--- a/kernel/locking/lockdep.c	2020-01-09 04:19:10.000000000 -0500
+++ b/kernel/locking/lockdep.c	2020-06-19 21:00:31.432227716 -0400
@@ -140,7 +140,7 @@
 unsigned long nr_lock_classes;
 static struct lock_class lock_classes[MAX_LOCKDEP_KEYS];
 
-static inline struct lock_class *hlock_class(struct held_lock *hlock)
+inline struct lock_class *lockdep_hlock_class(struct held_lock *hlock)
 {
 	if (!hlock->class_idx) {
 		/*
@@ -151,6 +151,8 @@
 	}
 	return lock_classes + hlock->class_idx - 1;
 }
+EXPORT_SYMBOL_GPL(lockdep_hlock_class);
+#define hlock_class(hlock) lockdep_hlock_class(hlock)
 
 #ifdef CONFIG_LOCK_STAT
 static DEFINE_PER_CPU(struct lock_class_stats[MAX_LOCKDEP_KEYS], cpu_lock_stats);
diff -Naur --no-dereference a/kernel/sched/idle.c b/kernel/sched/idle.c
--- a/kernel/sched/idle.c	2020-01-09 04:19:10.000000000 -0500
+++ b/kernel/sched/idle.c	2020-06-19 21:00:31.432227716 -0400
@@ -32,6 +32,7 @@
 		WARN_ON_ONCE(cpu_idle_force_poll < 0);
 	}
 }
+EXPORT_SYMBOL(cpu_idle_poll_ctrl);
 
 #ifdef CONFIG_GENERIC_IDLE_POLL_SETUP
 static int __init cpu_idle_poll_setup(char *__unused)
diff -Naur --no-dereference a/kernel/task_work.c b/kernel/task_work.c
--- a/kernel/task_work.c	2020-01-09 04:19:10.000000000 -0500
+++ b/kernel/task_work.c	2020-06-19 21:00:31.432227716 -0400
@@ -116,3 +116,4 @@
 		} while (work);
 	}
 }
+EXPORT_SYMBOL_GPL(task_work_run);
diff -Naur --no-dereference a/lib/ioremap.c b/lib/ioremap.c
--- a/lib/ioremap.c	2020-01-09 04:19:10.000000000 -0500
+++ b/lib/ioremap.c	2020-06-19 21:00:31.432227716 -0400
@@ -181,3 +181,4 @@
 
 	return err;
 }
+EXPORT_SYMBOL_GPL(ioremap_page_range);
diff -Naur --no-dereference a/lib/Kconfig b/lib/Kconfig
--- a/lib/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/lib/Kconfig	2020-06-19 21:00:31.432227716 -0400
@@ -10,6 +10,14 @@
 config RAID6_PQ
 	tristate
 
+config RAID6_PQ_BENCHMARK
+	bool "Automatically choose fastest RAID6 PQ functions"
+	depends on RAID6_PQ
+	default y
+	help
+	  Benchmark all available RAID6 PQ functions on init and choose the
+	  fastest one.
+
 config BITREVERSE
 	tristate
 
diff -Naur --no-dereference a/lib/raid6/algos.c b/lib/raid6/algos.c
--- a/lib/raid6/algos.c	2020-01-09 04:19:10.000000000 -0500
+++ b/lib/raid6/algos.c	2020-06-19 21:00:31.432227716 -0400
@@ -34,64 +34,64 @@
 EXPORT_SYMBOL_GPL(raid6_call);
 
 const struct raid6_calls * const raid6_algos[] = {
-#if defined(__ia64__)
-	&raid6_intx16,
-	&raid6_intx32,
-#endif
 #if defined(__i386__) && !defined(__arch_um__)
-	&raid6_mmxx1,
-	&raid6_mmxx2,
-	&raid6_sse1x1,
-	&raid6_sse1x2,
-	&raid6_sse2x1,
-	&raid6_sse2x2,
-#ifdef CONFIG_AS_AVX2
-	&raid6_avx2x1,
-	&raid6_avx2x2,
-#endif
 #ifdef CONFIG_AS_AVX512
-	&raid6_avx512x1,
 	&raid6_avx512x2,
+	&raid6_avx512x1,
 #endif
-#endif
-#if defined(__x86_64__) && !defined(__arch_um__)
-	&raid6_sse2x1,
-	&raid6_sse2x2,
-	&raid6_sse2x4,
 #ifdef CONFIG_AS_AVX2
-	&raid6_avx2x1,
 	&raid6_avx2x2,
-	&raid6_avx2x4,
+	&raid6_avx2x1,
 #endif
+	&raid6_sse2x2,
+	&raid6_sse2x1,
+	&raid6_sse1x2,
+	&raid6_sse1x1,
+	&raid6_mmxx2,
+	&raid6_mmxx1,
+#endif
+#if defined(__x86_64__) && !defined(__arch_um__)
 #ifdef CONFIG_AS_AVX512
-	&raid6_avx512x1,
-	&raid6_avx512x2,
 	&raid6_avx512x4,
+	&raid6_avx512x2,
+	&raid6_avx512x1,
+#endif
+#ifdef CONFIG_AS_AVX2
+	&raid6_avx2x4,
+	&raid6_avx2x2,
+	&raid6_avx2x1,
 #endif
+	&raid6_sse2x4,
+	&raid6_sse2x2,
+	&raid6_sse2x1,
 #endif
 #ifdef CONFIG_ALTIVEC
-	&raid6_altivec1,
-	&raid6_altivec2,
-	&raid6_altivec4,
-	&raid6_altivec8,
-	&raid6_vpermxor1,
-	&raid6_vpermxor2,
-	&raid6_vpermxor4,
 	&raid6_vpermxor8,
+	&raid6_vpermxor4,
+	&raid6_vpermxor2,
+	&raid6_vpermxor1,
+	&raid6_altivec8,
+	&raid6_altivec4,
+	&raid6_altivec2,
+	&raid6_altivec1,
 #endif
 #if defined(CONFIG_S390)
 	&raid6_s390vx8,
 #endif
-	&raid6_intx1,
-	&raid6_intx2,
-	&raid6_intx4,
-	&raid6_intx8,
 #ifdef CONFIG_KERNEL_MODE_NEON
-	&raid6_neonx1,
-	&raid6_neonx2,
-	&raid6_neonx4,
 	&raid6_neonx8,
+	&raid6_neonx4,
+	&raid6_neonx2,
+	&raid6_neonx1,
 #endif
+#if defined(__ia64__)
+	&raid6_intx32,
+	&raid6_intx16,
+#endif
+	&raid6_intx8,
+	&raid6_intx4,
+	&raid6_intx2,
+	&raid6_intx1,
 	NULL
 };
 
@@ -163,6 +163,11 @@
 			if ((*algo)->valid && !(*algo)->valid())
 				continue;
 
+			if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) {
+				best = *algo;
+				break;
+			}
+
 			perf = 0;
 
 			preempt_disable();
diff -Naur --no-dereference a/MAINTAINERS b/MAINTAINERS
--- a/MAINTAINERS	2020-01-09 04:19:10.000000000 -0500
+++ b/MAINTAINERS	2020-06-19 21:00:31.400227715 -0400
@@ -2140,6 +2140,7 @@
 F:	Documentation/devicetree/bindings/arm/ti/k3.txt
 F:	arch/arm64/boot/dts/ti/Makefile
 F:	arch/arm64/boot/dts/ti/k3-*
+F:	include/dt-bindings/pinctrl/k3.h
 
 ARM/TEXAS INSTRUMENT KEYSTONE ARCHITECTURE
 M:	Santosh Shilimkar <ssantosh@kernel.org>
@@ -2605,6 +2606,19 @@
 F:	include/uapi/linux/audit.h
 F:	kernel/audit*
 
+AUFS (advanced multi layered unification filesystem) FILESYSTEM
+M:	"J. R. Okajima" <hooanon05g@gmail.com>
+L:	aufs-users@lists.sourceforge.net (members only)
+L:	linux-unionfs@vger.kernel.org
+W:	http://aufs.sourceforge.net
+T:	git://github.com/sfjro/aufs4-linux.git
+S:	Supported
+F:	Documentation/filesystems/aufs/
+F:	Documentation/ABI/testing/debugfs-aufs
+F:	Documentation/ABI/testing/sysfs-aufs
+F:	fs/aufs/
+F:	include/uapi/linux/aufs_type.h
+
 AUXILIARY DISPLAY DRIVERS
 M:	Miguel Ojeda Sandonis <miguel.ojeda.sandonis@gmail.com>
 S:	Maintained
@@ -4947,6 +4961,15 @@
 F:	Documentation/devicetree/bindings/gpu/brcm,bcm-v3d.txt
 T:	git git://anongit.freedesktop.org/drm/drm-misc
 
+DRM DRIVERS FOR TI KEYSTONE
+M:	Tomi Valkeinen <tomi.valkeinen@ti.com>
+R:	Jyri Sarha <jsarha@ti.com>
+L:	dri-devel@lists.freedesktop.org
+S:	Maintained
+F:	drivers/gpu/drm/tidss/
+F:	Documentation/devicetree/bindings/display/ti/ti,k2g-dss.txt
+F:	Documentation/devicetree/bindings/display/ti/ti,am6-dss.txt
+
 DRM DRIVERS FOR VC4
 M:	Eric Anholt <eric@anholt.net>
 T:	git git://github.com/anholt/linux
@@ -6802,6 +6825,14 @@
 F:	tools/hv/
 F:	Documentation/ABI/stable/sysfs-bus-vmbus
 
+HYPERBUS SUPPORT
+M:	Vignesh Raghavendra <vigneshr@ti.com>
+S:	Supported
+F:	drivers/mtd/hyperbus/
+F:	include/linux/mtd/hyperbus.h
+F:	Documentation/devicetree/bindings/mtd/cypress,hyperflash.txt
+F:	Documentation/devicetree/bindings/mtd/ti,am654-hbmc.txt
+
 HYPERVISOR VIRTUAL CONSOLE DRIVER
 L:	linuxppc-dev@lists.ozlabs.org
 S:	Odd Fixes
@@ -9839,6 +9870,14 @@
 F:	drivers/media/i2c/mt9t112.c
 F:	include/media/i2c/mt9t112.h
 
+MT9T11x APTINA CAMERA SENSOR
+M:	Benoit Parrot <bparrot@ti.com>
+L:	linux-media@vger.kernel.org
+T:	git git://linuxtv.org/media_tree.git
+S:	Odd Fixes
+F:	Documentation/devicetree/bindings/media/i2c/mt9t11x.txt
+F:	drivers/media/i2c/mt9t11x.c
+
 MT9V032 APTINA CAMERA SENSOR
 M:	Laurent Pinchart <laurent.pinchart@ideasonboard.com>
 L:	linux-media@vger.kernel.org
@@ -10477,7 +10516,10 @@
 L:	alsa-devel@alsa-project.org (moderated for non-subscribers)
 L:	linux-omap@vger.kernel.org
 S:	Maintained
-F:	sound/soc/omap/
+F:	sound/soc/ti/omap*
+F:	sound/soc/ti/rx51.c
+F:	sound/soc/ti/n810.c
+F:	sound/soc/ti/sdma-pcm.*
 
 OMAP CLOCK FRAMEWORK SUPPORT
 M:	Paul Walmsley <paul@pwsan.com>
@@ -10675,6 +10717,15 @@
 S:	Maintained
 F:	drivers/char/pcmcia/cm4040_cs.*
 
+OMNIVISION OV1063x SENSOR DRIVER
+M:	Benoit Parrot <bparrot@ti.com>
+L:	linux-media@vger.kernel.org
+T:	git git://linuxtv.org/media_tree.git
+S:	Maintained
+F:	Documentation/devicetree/bindings/media/i2c/ov1063x.txt
+F:	drivers/media/i2c/ov1063x.c
+F:	drivers/media/i2c/ov1063x_regs.h
+
 OMNIVISION OV13858 SENSOR DRIVER
 M:	Sakari Ailus <sakari.ailus@linux.intel.com>
 L:	linux-media@vger.kernel.org
@@ -10697,6 +10748,14 @@
 S:	Maintained
 F:	drivers/media/i2c/ov2685.c
 
+OMNIVISION OV490 SENSOR DRIVER
+M:	Benoit Parrot <bparrot@ti.com>
+L:	linux-media@vger.kernel.org
+T:	git git://linuxtv.org/media_tree.git
+S:	Maintained
+F:	Documentation/devicetree/bindings/media/i2c/ov490.txt
+F:	drivers/media/i2c/ov490.c
+
 OMNIVISION OV5640 SENSOR DRIVER
 M:	Steve Longerbeam <slongerbeam@gmail.com>
 L:	linux-media@vger.kernel.org
@@ -11195,20 +11254,21 @@
 F:	Documentation/devicetree/bindings/pci/designware-pcie.txt
 F:	drivers/pci/controller/dwc/*designware*
 
-PCI DRIVER FOR TI DRA7XX
+PCI DRIVER FOR TI DRA7XX/J721E
 M:	Kishon Vijay Abraham I <kishon@ti.com>
 L:	linux-omap@vger.kernel.org
 L:	linux-pci@vger.kernel.org
 S:	Supported
 F:	Documentation/devicetree/bindings/pci/ti-pci.txt
 F:	drivers/pci/controller/dwc/pci-dra7xx.c
+F:	drivers/pci/controller/pci-j721e.c
 
 PCI DRIVER FOR TI KEYSTONE
 M:	Murali Karicheri <m-karicheri2@ti.com>
 L:	linux-pci@vger.kernel.org
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 S:	Maintained
-F:	drivers/pci/controller/dwc/*keystone*
+F:	drivers/pci/controller/dwc/pci-keystone.c
 
 PCI ENDPOINT SUBSYSTEM
 M:	Kishon Vijay Abraham I <kishon@ti.com>
@@ -14435,12 +14495,22 @@
 F:	drivers/firmware/ti_sci*
 F:	include/linux/soc/ti/ti_sci_protocol.h
 F:	Documentation/devicetree/bindings/soc/ti/sci-pm-domain.txt
-F:	include/dt-bindings/genpd/k2g.h
 F:	drivers/soc/ti/ti_sci_pm_domains.c
+F:	include/dt-bindings/soc/ti,sci_pm_domain.h
 F:	Documentation/devicetree/bindings/reset/ti,sci-reset.txt
 F:	Documentation/devicetree/bindings/clock/ti,sci-clk.txt
 F:	drivers/clk/keystone/sci-clk.c
 F:	drivers/reset/reset-ti-sci.c
+F:	Documentation/devicetree/bindings/interrupt-controller/ti,sci-intr.txt
+F:	Documentation/devicetree/bindings/interrupt-controller/ti,sci-inta.txt
+F:	drivers/irqchip/irq-ti-sci-intr.c
+F:	drivers/irqchip/irq-ti-sci-inta.c
+
+Texas Instruments ASoC drivers
+M:	Peter Ujfalusi <peter.ujfalusi@ti.com>
+L:	alsa-devel@alsa-project.org (moderated for non-subscribers)
+S:	Maintained
+F:	sound/soc/ti/
 
 THANKO'S RAREMONO AM/FM/SW RADIO RECEIVER USB DRIVER
 M:	Hans Verkuil <hverkuil@xs4all.nl>
@@ -14666,6 +14736,8 @@
 Q:	http://patchwork.linuxtv.org/project/linux-media/list/
 S:	Maintained
 F:	drivers/media/platform/ti-vpe/
+F:	Documentation/devicetree/bindings/media/ti-vpe.txt
+F:	Documentation/devicetree/bindings/media/ti-vip.txt
 
 TI WILINK WIRELESS DRIVERS
 L:	linux-wireless@vger.kernel.org
diff -Naur --no-dereference a/mm/filemap.c b/mm/filemap.c
--- a/mm/filemap.c	2020-01-09 04:19:10.000000000 -0500
+++ b/mm/filemap.c	2020-06-19 21:00:31.432227716 -0400
@@ -2723,7 +2723,7 @@
 	vm_fault_t ret = VM_FAULT_LOCKED;
 
 	sb_start_pagefault(inode->i_sb);
-	file_update_time(vmf->vma->vm_file);
+	vma_file_update_time(vmf->vma);
 	lock_page(page);
 	if (page->mapping != inode->i_mapping) {
 		unlock_page(page);
diff -Naur --no-dereference a/mm/Makefile b/mm/Makefile
--- a/mm/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/mm/Makefile	2020-06-19 21:00:31.432227716 -0400
@@ -39,7 +39,7 @@
 			   mm_init.o mmu_context.o percpu.o slab_common.o \
 			   compaction.o vmacache.o \
 			   interval_tree.o list_lru.o workingset.o \
-			   debug.o $(mmu-y)
+			   prfile.o debug.o $(mmu-y)
 
 obj-y += init-mm.o
 
diff -Naur --no-dereference a/mm/mmap.c b/mm/mmap.c
--- a/mm/mmap.c	2020-01-09 04:19:10.000000000 -0500
+++ b/mm/mmap.c	2020-06-19 21:00:31.432227716 -0400
@@ -175,7 +175,7 @@
 	if (vma->vm_ops && vma->vm_ops->close)
 		vma->vm_ops->close(vma);
 	if (vma->vm_file)
-		fput(vma->vm_file);
+		vma_fput(vma);
 	mpol_put(vma_policy(vma));
 	vm_area_free(vma);
 	return next;
@@ -900,7 +900,7 @@
 	if (remove_next) {
 		if (file) {
 			uprobe_munmap(next, next->vm_start, next->vm_end);
-			fput(file);
+			vma_fput(vma);
 		}
 		if (next->anon_vma)
 			anon_vma_merge(vma, next);
@@ -1816,8 +1816,8 @@
 	return addr;
 
 unmap_and_free_vma:
+	vma_fput(vma);
 	vma->vm_file = NULL;
-	fput(file);
 
 	/* Undo any partial mapping done by a device driver. */
 	unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end);
@@ -2639,7 +2639,7 @@
 		goto out_free_mpol;
 
 	if (new->vm_file)
-		get_file(new->vm_file);
+		vma_get_file(new);
 
 	if (new->vm_ops && new->vm_ops->open)
 		new->vm_ops->open(new);
@@ -2658,7 +2658,7 @@
 	if (new->vm_ops && new->vm_ops->close)
 		new->vm_ops->close(new);
 	if (new->vm_file)
-		fput(new->vm_file);
+		vma_fput(new);
 	unlink_anon_vmas(new);
  out_free_mpol:
 	mpol_put(vma_policy(new));
@@ -2820,7 +2820,7 @@
 	struct vm_area_struct *vma;
 	unsigned long populate = 0;
 	unsigned long ret = -EINVAL;
-	struct file *file;
+	struct file *file, *prfile;
 
 	pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. See Documentation/vm/remap_file_pages.rst.\n",
 		     current->comm, current->pid);
@@ -2895,10 +2895,27 @@
 		}
 	}
 
-	file = get_file(vma->vm_file);
+	vma_get_file(vma);
+	file = vma->vm_file;
+	prfile = vma->vm_prfile;
 	ret = do_mmap_pgoff(vma->vm_file, start, size,
 			prot, flags, pgoff, &populate, NULL);
+	if (!IS_ERR_VALUE(ret) && file && prfile) {
+		struct vm_area_struct *new_vma;
+
+		new_vma = find_vma(mm, ret);
+		if (!new_vma->vm_prfile)
+			new_vma->vm_prfile = prfile;
+		if (new_vma != vma)
+			get_file(prfile);
+	}
+	/*
+	 * two fput()s instead of vma_fput(vma),
+	 * coz vma may not be available anymore.
+	 */
 	fput(file);
+	if (prfile)
+		fput(prfile);
 out:
 	up_write(&mm->mmap_sem);
 	if (populate)
@@ -3204,7 +3221,7 @@
 		if (anon_vma_clone(new_vma, vma))
 			goto out_free_mempol;
 		if (new_vma->vm_file)
-			get_file(new_vma->vm_file);
+			vma_get_file(new_vma);
 		if (new_vma->vm_ops && new_vma->vm_ops->open)
 			new_vma->vm_ops->open(new_vma);
 		vma_link(mm, new_vma, prev, rb_link, rb_parent);
diff -Naur --no-dereference a/mm/nommu.c b/mm/nommu.c
--- a/mm/nommu.c	2020-01-09 04:19:10.000000000 -0500
+++ b/mm/nommu.c	2020-06-19 21:00:31.432227716 -0400
@@ -625,7 +625,7 @@
 		up_write(&nommu_region_sem);
 
 		if (region->vm_file)
-			fput(region->vm_file);
+			vmr_fput(region);
 
 		/* IO memory and memory shared directly out of the pagecache
 		 * from ramfs/tmpfs mustn't be released here */
@@ -763,7 +763,7 @@
 	if (vma->vm_ops && vma->vm_ops->close)
 		vma->vm_ops->close(vma);
 	if (vma->vm_file)
-		fput(vma->vm_file);
+		vma_fput(vma);
 	put_nommu_region(vma->vm_region);
 	vm_area_free(vma);
 }
@@ -1286,7 +1286,7 @@
 					goto error_just_free;
 				}
 			}
-			fput(region->vm_file);
+			vmr_fput(region);
 			kmem_cache_free(vm_region_jar, region);
 			region = pregion;
 			result = start;
@@ -1361,10 +1361,10 @@
 	up_write(&nommu_region_sem);
 error:
 	if (region->vm_file)
-		fput(region->vm_file);
+		vmr_fput(region);
 	kmem_cache_free(vm_region_jar, region);
 	if (vma->vm_file)
-		fput(vma->vm_file);
+		vma_fput(vma);
 	vm_area_free(vma);
 	return ret;
 
diff -Naur --no-dereference a/mm/prfile.c b/mm/prfile.c
--- a/mm/prfile.c	1969-12-31 19:00:00.000000000 -0500
+++ b/mm/prfile.c	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,86 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Mainly for aufs which mmap(2) different file and wants to print different
+ * path in /proc/PID/maps.
+ * Call these functions via macros defined in linux/mm.h.
+ *
+ * See Documentation/filesystems/aufs/design/06mmap.txt
+ *
+ * Copyright (c) 2014-2020 Junjro R. Okajima
+ * Copyright (c) 2014 Ian Campbell
+ */
+
+#include <linux/mm.h>
+#include <linux/file.h>
+#include <linux/fs.h>
+
+/* #define PRFILE_TRACE */
+static inline void prfile_trace(struct file *f, struct file *pr,
+			      const char func[], int line, const char func2[])
+{
+#ifdef PRFILE_TRACE
+	if (pr)
+		pr_info("%s:%d: %s, %pD2\n", func, line, func2, f);
+#endif
+}
+
+void vma_do_file_update_time(struct vm_area_struct *vma, const char func[],
+			     int line)
+{
+	struct file *f = vma->vm_file, *pr = vma->vm_prfile;
+
+	prfile_trace(f, pr, func, line, __func__);
+	file_update_time(f);
+	if (f && pr)
+		file_update_time(pr);
+}
+
+struct file *vma_do_pr_or_file(struct vm_area_struct *vma, const char func[],
+			       int line)
+{
+	struct file *f = vma->vm_file, *pr = vma->vm_prfile;
+
+	prfile_trace(f, pr, func, line, __func__);
+	return (f && pr) ? pr : f;
+}
+
+void vma_do_get_file(struct vm_area_struct *vma, const char func[], int line)
+{
+	struct file *f = vma->vm_file, *pr = vma->vm_prfile;
+
+	prfile_trace(f, pr, func, line, __func__);
+	get_file(f);
+	if (f && pr)
+		get_file(pr);
+}
+
+void vma_do_fput(struct vm_area_struct *vma, const char func[], int line)
+{
+	struct file *f = vma->vm_file, *pr = vma->vm_prfile;
+
+	prfile_trace(f, pr, func, line, __func__);
+	fput(f);
+	if (f && pr)
+		fput(pr);
+}
+
+#ifndef CONFIG_MMU
+struct file *vmr_do_pr_or_file(struct vm_region *region, const char func[],
+			       int line)
+{
+	struct file *f = region->vm_file, *pr = region->vm_prfile;
+
+	prfile_trace(f, pr, func, line, __func__);
+	return (f && pr) ? pr : f;
+}
+
+void vmr_do_fput(struct vm_region *region, const char func[], int line)
+{
+	struct file *f = region->vm_file, *pr = region->vm_prfile;
+
+	prfile_trace(f, pr, func, line, __func__);
+	fput(f);
+	if (f && pr)
+		fput(pr);
+}
+#endif /* !CONFIG_MMU */
diff -Naur --no-dereference a/net/can/isotp.c b/net/can/isotp.c
--- a/net/can/isotp.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/can/isotp.c	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,1537 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/*
+ * isotp.c - ISO 15765-2 CAN transport protocol for protocol family CAN
+ *
+ * WARNING: This is ALPHA code for discussions and first tests that should
+ *          not be used in production environments.
+ *
+ * In the discussion the Socket-API to the userspace or the ISO-TP socket
+ * options or the return values we may change! Current behaviour:
+ *
+ * - no ISO-TP specific return values are provided to the userspace
+ * - when a transfer (tx) is on the run the next write() blocks until it's done
+ * - no support for sending wait frames to the data source in the rx path
+ *
+ * Copyright (c) 2017 Volkswagen Group Electronic Research
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of Volkswagen nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * Alternatively, provided that this notice is retained in full, this
+ * software may be distributed under the terms of the GNU General
+ * Public License ("GPL") version 2, in which case the provisions of the
+ * GPL apply INSTEAD OF those given above.
+ *
+ * The provided data structures and external interfaces from this code
+ * are not restricted to be used by modules with a GPL compatible license.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ *
+ * Send feedback to <linux-can@vger.kernel.org>
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/version.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/hrtimer.h>
+#include <linux/wait.h>
+#include <linux/uio.h>
+#include <linux/net.h>
+#include <linux/netdevice.h>
+#include <linux/socket.h>
+#include <linux/if_arp.h>
+#include <linux/skbuff.h>
+#include <linux/can.h>
+#include <linux/can/core.h>
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,9,0)
+#include <linux/can/skb.h>
+#define CAN_SKBRES sizeof(struct can_skb_priv)
+#else
+#define CAN_SKBRES 0
+#endif
+#include <linux/can/isotp.h>
+#include <net/sock.h>
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,24)
+#include <net/net_namespace.h>
+#endif
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,25)
+#include "compat.h"
+#endif
+
+#define CAN_ISOTP_VERSION "20181217"
+static __initdata const char banner[] =
+	KERN_INFO "can: isotp protocol (rev " CAN_ISOTP_VERSION " alpha)\n";
+
+MODULE_DESCRIPTION("PF_CAN isotp 15765-2:2016 protocol");
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_AUTHOR("Oliver Hartkopp <socketcan@hartkopp.net>");
+MODULE_ALIAS("can-proto-6");
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22)
+#error This modules needs hrtimers (available since Kernel 2.6.22)
+#endif
+
+#define DBG(fmt, args...) (printk( KERN_DEBUG "can-isotp: %s: " fmt, \
+				   __func__, ##args))
+#undef DBG
+#define DBG(fmt, args...)
+
+#define SINGLE_MASK(id) ((id & CAN_EFF_FLAG) ? \
+			 (CAN_EFF_MASK | CAN_EFF_FLAG | CAN_RTR_FLAG) : \
+			 (CAN_SFF_MASK | CAN_EFF_FLAG | CAN_RTR_FLAG))
+
+/*
+  ISO 15765-2:2016 supports more than 4095 byte per ISO PDU as the FF_DL can
+  take full 32 bit values (4 Gbyte). We would need some good concept to handle
+  this between user space and kernel space. For now increase the static buffer
+  to something about 8 kbyte to be able to test this new functionality.
+*/
+#define MAX_MSG_LENGTH 8200
+
+/* N_PCI type values in bits 7-4 of N_PCI bytes */
+#define N_PCI_SF	0x00 /* single frame */
+#define N_PCI_FF	0x10 /* first frame */
+#define N_PCI_CF	0x20 /* consecutive frame */
+#define N_PCI_FC	0x30 /* flow control */
+
+#define N_PCI_SZ 1	/* size of the PCI byte #1 */
+#define SF_PCI_SZ4 1	/* size of SingleFrame PCI including 4 bit SF_DL */
+#define SF_PCI_SZ8 2	/* size of SingleFrame PCI including 8 bit SF_DL */
+#define FF_PCI_SZ12 2	/* size of FirstFrame PCI including 12 bit FF_DL */
+#define FF_PCI_SZ32 6	/* size of FirstFrame PCI including 32 bit FF_DL */
+#define FC_CONTENT_SZ 3	/* flow control content size in byte (FS/BS/STmin) */
+
+#define ISOTP_CHECK_PADDING (CAN_ISOTP_CHK_PAD_LEN | CAN_ISOTP_CHK_PAD_DATA)
+
+/* Flow Status given in FC frame */
+#define ISOTP_FC_CTS	0	/* clear to send */
+#define ISOTP_FC_WT	1	/* wait */
+#define ISOTP_FC_OVFLW	2	/* overflow */
+
+enum {
+	ISOTP_IDLE = 0,
+	ISOTP_WAIT_FIRST_FC,
+	ISOTP_WAIT_FC,
+	ISOTP_WAIT_DATA,
+	ISOTP_SENDING
+};
+
+struct tpcon {
+	int idx;
+	int len;
+	u8  state;
+	u8  bs;
+	u8  sn;
+	u8  ll_dl;
+	u8  buf[MAX_MSG_LENGTH+1];
+};
+
+struct isotp_sock {
+	struct sock sk;
+	int bound;
+	int ifindex;
+	canid_t txid;
+	canid_t rxid;
+	ktime_t tx_gap;
+	ktime_t lastrxcf_tstamp;
+	struct hrtimer rxtimer, txtimer;
+	struct tasklet_struct txtsklet;
+	struct can_isotp_options opt;
+	struct can_isotp_fc_options rxfc, txfc;
+	struct can_isotp_ll_options ll;
+	__u32 force_tx_stmin;
+	__u32 force_rx_stmin;
+	struct tpcon rx, tx;
+	struct notifier_block notifier;
+	wait_queue_head_t wait;
+};
+
+static inline struct isotp_sock *isotp_sk(const struct sock *sk)
+{
+	return (struct isotp_sock *)sk;
+}
+
+static enum hrtimer_restart isotp_rx_timer_handler(struct hrtimer *hrtimer)
+{
+	struct isotp_sock *so = container_of(hrtimer, struct isotp_sock,
+					     rxtimer);
+	if (so->rx.state == ISOTP_WAIT_DATA) {
+		DBG("we did not get new data frames in time.\n");
+
+		/* reset tx state */
+		so->rx.state = ISOTP_IDLE;
+	}
+
+	return HRTIMER_NORESTART;
+}
+
+static void isotp_skb_reserve(struct sk_buff *skb, struct net_device *dev)
+{
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,9,0)
+	can_skb_reserve(skb);
+	can_skb_prv(skb)->ifindex = dev->ifindex;
+#endif
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,1,5)
+	can_skb_prv(skb)->skbcnt = 0;
+#endif
+}
+
+static void isotp_skb_destructor(struct sk_buff *skb)
+{
+	sock_put(skb->sk);
+}
+
+static inline void isotp_skb_set_owner(struct sk_buff *skb, struct sock *sk)
+{
+	if (sk) {
+		sock_hold(sk);
+		skb->destructor = isotp_skb_destructor;
+		skb->sk = sk;
+	}
+}
+
+static int isotp_send_fc(struct sock *sk, int ae, u8 flowstatus)
+{
+	struct net_device *dev;
+	struct sk_buff *nskb;
+	struct canfd_frame *ncf;
+	struct isotp_sock *so = isotp_sk(sk);
+
+	nskb = alloc_skb(so->ll.mtu + CAN_SKBRES, gfp_any());
+	if (!nskb)
+		return 1;
+
+	dev = dev_get_by_index(sock_net(sk), so->ifindex);
+	if (!dev) {
+		kfree_skb(nskb);
+		return 1;
+	}
+	isotp_skb_reserve(nskb, dev);
+	nskb->dev = dev;
+	isotp_skb_set_owner(nskb, sk);
+	ncf = (struct canfd_frame *) nskb->data;
+	skb_put(nskb, so->ll.mtu);
+
+	/* create & send flow control reply */
+	ncf->can_id = so->txid;
+
+	if (so->opt.flags & CAN_ISOTP_TX_PADDING) {
+		memset(ncf->data, so->opt.txpad_content, CAN_MAX_DLEN);
+		ncf->len = CAN_MAX_DLEN;
+	} else
+		ncf->len = ae + FC_CONTENT_SZ;
+
+	ncf->data[ae] = N_PCI_FC | flowstatus;
+	ncf->data[ae + 1] = so->rxfc.bs;
+	ncf->data[ae + 2] = so->rxfc.stmin;
+
+	if (ae)
+		ncf->data[0] = so->opt.ext_address;
+
+	if (so->ll.mtu == CANFD_MTU)
+		ncf->flags = so->ll.tx_flags;
+
+	can_send(nskb, 1);
+	dev_put(dev);
+
+	/* reset blocksize counter */
+	so->rx.bs = 0;
+
+	/* reset last CF frame rx timestamp for rx stmin enforcement */
+	so->lastrxcf_tstamp = ktime_set(0,0);
+
+	/* start rx timeout watchdog */
+	hrtimer_start(&so->rxtimer, ktime_set(1,0), HRTIMER_MODE_REL);
+	return 0;
+}
+
+static void isotp_rcv_skb(struct sk_buff *skb, struct sock *sk)
+{
+	struct sockaddr_can *addr = (struct sockaddr_can *)skb->cb;
+
+	BUILD_BUG_ON(sizeof(skb->cb) < sizeof(struct sockaddr_can));
+
+	memset(addr, 0, sizeof(*addr));
+	addr->can_family  = AF_CAN;
+	addr->can_ifindex = skb->dev->ifindex;
+
+	if (sock_queue_rcv_skb(sk, skb) < 0)
+		kfree_skb(skb);
+}
+
+static u8 padlen(u8 datalen)
+{
+	const u8 plen[] = {8,  8,  8,  8,  8,  8,  8,  8,  8,	/* 0 - 8 */
+			   12, 12, 12, 12,			/* 9 - 12 */
+			   16, 16, 16, 16,			/* 13 - 16 */
+			   20, 20, 20, 20,			/* 17 - 20 */
+			   24, 24, 24, 24,			/* 21 - 24 */
+			   32, 32, 32, 32, 32, 32, 32, 32,	/* 25 - 32 */
+			   48, 48, 48, 48, 48, 48, 48, 48,	/* 33 - 40 */
+			   48, 48, 48, 48, 48, 48, 48, 48};	/* 41 - 48 */
+
+	if (datalen > 48)
+		return 64;
+
+	return plen[datalen];
+}
+
+/* check for length optimization and return 1/true when the check fails */
+static int check_optimized(struct canfd_frame *cf, int start_index)
+{
+	/*
+	 * for CAN_DL <= 8 the start_index is equal to the CAN_DL as the
+	 * padding would start at this point. E.g. if the padding would
+	 * start at cf.data[7] cf->len has to be 7 to be optimal.
+	 * Note: The data[] index starts with zero.
+	 */
+	if (cf->len <= CAN_MAX_DLEN)
+		return (cf->len != start_index);
+
+	/*
+	 * This relation is also valid in the non-linear DLC range, where
+	 * we need to take care of the minimal next possible CAN_DL.
+	 * The correct check would be (padlen(cf->len) != padlen(start_index)).
+	 * But as cf->len can only take discrete values from 12, .., 64 at this
+	 * point the padlen(cf->len) is always equal to cf->len.
+	 */
+	return (cf->len != padlen(start_index));
+}
+
+/* check padding and return 1/true when the check fails */
+static int check_pad(struct isotp_sock *so, struct canfd_frame *cf,
+		     int start_index, __u8 content)
+{
+	int i;
+
+	/* no RX_PADDING value => check length of optimized frame length */
+	if (!(so->opt.flags & CAN_ISOTP_RX_PADDING)) {
+
+		if (so->opt.flags & CAN_ISOTP_CHK_PAD_LEN)
+			return check_optimized(cf, start_index);
+
+		/* no valid test against empty value => ignore frame */
+		return 1;
+	}
+
+	/* check datalength of correctly padded CAN frame */
+	if ((so->opt.flags & CAN_ISOTP_CHK_PAD_LEN) &&
+	    cf->len != padlen(cf->len))
+			return 1;
+
+	/* check padding content */
+	if (so->opt.flags & CAN_ISOTP_CHK_PAD_DATA) {
+		for (i = start_index; i < cf->len; i++)
+			if (cf->data[i] != content)
+				return 1;
+	}
+	return 0;
+}
+
+static int isotp_rcv_fc(struct isotp_sock *so, struct canfd_frame *cf, int ae)
+{
+	if (so->tx.state != ISOTP_WAIT_FC &&
+	    so->tx.state != ISOTP_WAIT_FIRST_FC)
+		return 0;
+
+	hrtimer_cancel(&so->txtimer);
+
+	if ((cf->len < ae + FC_CONTENT_SZ) ||
+	    ((so->opt.flags & ISOTP_CHECK_PADDING) &&
+	     check_pad(so, cf, ae + FC_CONTENT_SZ, so->opt.rxpad_content))) {
+		so->tx.state = ISOTP_IDLE;
+		wake_up_interruptible(&so->wait);
+		return 1;
+	}
+
+	/* get communication parameters only from the first FC frame */
+	if (so->tx.state == ISOTP_WAIT_FIRST_FC) {
+
+		so->txfc.bs = cf->data[ae + 1];
+		so->txfc.stmin = cf->data[ae + 2];
+
+		/* fix wrong STmin values according spec */
+		if ((so->txfc.stmin > 0x7F) &&
+		    ((so->txfc.stmin < 0xF1) || (so->txfc.stmin > 0xF9)))
+			so->txfc.stmin = 0x7F;
+
+		so->tx_gap = ktime_set(0,0);
+		/* add transmission time for CAN frame N_As */
+		so->tx_gap = ktime_add_ns(so->tx_gap, so->opt.frame_txtime);
+		/* add waiting time for consecutive frames N_Cs */
+		if (so->opt.flags & CAN_ISOTP_FORCE_TXSTMIN)
+			so->tx_gap = ktime_add_ns(so->tx_gap,
+						  so->force_tx_stmin);
+		else if (so->txfc.stmin < 0x80)
+			so->tx_gap = ktime_add_ns(so->tx_gap,
+						  so->txfc.stmin * 1000000);
+		else
+			so->tx_gap = ktime_add_ns(so->tx_gap,
+						  (so->txfc.stmin - 0xF0)
+						  * 100000);
+		so->tx.state = ISOTP_WAIT_FC;
+	}
+
+	DBG("FC frame: FS %d, BS %d, STmin 0x%02X, tx_gap %lld\n",
+	    cf->data[ae] & 0x0F & 0x0F, so->txfc.bs, so->txfc.stmin,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)
+	    (long long)so->tx_gap);
+#else
+	    (long long)so->tx_gap.tv64);
+#endif
+
+	switch (cf->data[ae] & 0x0F) {
+
+	case ISOTP_FC_CTS:
+		so->tx.bs = 0;
+		so->tx.state = ISOTP_SENDING;
+		DBG("starting txtimer for sending\n");
+		/* start cyclic timer for sending CF frame */
+		hrtimer_start(&so->txtimer, so->tx_gap,
+			      HRTIMER_MODE_REL);
+		break;
+
+	case ISOTP_FC_WT:
+		DBG("starting waiting for next FC\n");
+		/* start timer to wait for next FC frame */
+		hrtimer_start(&so->txtimer, ktime_set(1,0),
+			      HRTIMER_MODE_REL);
+		break;
+
+	case ISOTP_FC_OVFLW:
+		DBG("overflow in receiver side\n");
+
+	default:
+		/* stop this tx job. TODO: error reporting? */
+		so->tx.state = ISOTP_IDLE;
+		wake_up_interruptible(&so->wait);
+	}
+	return 0;
+}
+
+static int isotp_rcv_sf(struct sock *sk, struct canfd_frame *cf, int pcilen,
+			struct sk_buff *skb, int len)
+{
+	struct isotp_sock *so = isotp_sk(sk);
+	struct sk_buff *nskb;
+
+	hrtimer_cancel(&so->rxtimer);
+	so->rx.state = ISOTP_IDLE;
+
+	if (!len || len > cf->len - pcilen)
+		return 1;
+
+	if ((so->opt.flags & ISOTP_CHECK_PADDING) &&
+	    check_pad(so, cf, pcilen + len, so->opt.rxpad_content))
+		return 1;
+
+	nskb = alloc_skb(len, gfp_any());
+	if (!nskb)
+		return 1;
+
+	memcpy(skb_put(nskb, len), &cf->data[pcilen], len);
+
+	nskb->tstamp = skb->tstamp;
+	nskb->dev = skb->dev;
+	isotp_rcv_skb(nskb, sk);
+	return 0;
+}
+
+static int isotp_rcv_ff(struct sock *sk, struct canfd_frame *cf, int ae)
+{
+	struct isotp_sock *so = isotp_sk(sk);
+	int i;
+	int off;
+	int ff_pci_sz;
+
+	hrtimer_cancel(&so->rxtimer);
+	so->rx.state = ISOTP_IDLE;
+
+	/* get the used sender LL_DL from the (first) CAN frame data length */
+	so->rx.ll_dl = padlen(cf->len);
+
+	/* the first frame has to use the entire frame up to LL_DL length */
+	if (cf->len != so->rx.ll_dl)
+		return 1;
+
+	/* get the FF_DL */
+	so->rx.len = (cf->data[ae] & 0x0F) << 8;
+	so->rx.len += cf->data[ae + 1];
+
+	/* Check for FF_DL escape sequence supporting 32 bit PDU length */
+	if (so->rx.len)
+		ff_pci_sz = FF_PCI_SZ12;
+	else {
+		/* FF_DL = 0 => get real length from next 4 bytes */
+		so->rx.len = cf->data[ae + 2] << 24;
+		so->rx.len += cf->data[ae + 3] << 16;
+		so->rx.len += cf->data[ae + 4] << 8;
+		so->rx.len += cf->data[ae + 5];
+		ff_pci_sz = FF_PCI_SZ32;
+	}
+
+	/* take care of a potential SF_DL ESC offset for TX_DL > 8 */
+	off = (so->rx.ll_dl > CAN_MAX_DLEN)? 1:0;
+
+	if (so->rx.len + ae + off + ff_pci_sz < so->rx.ll_dl)
+		return 1;
+
+	if (so->rx.len > MAX_MSG_LENGTH) {
+		/* send FC frame with overflow status */
+		isotp_send_fc(sk, ae, ISOTP_FC_OVFLW);
+		return 1;
+	}
+
+	/* copy the first received data bytes */
+	so->rx.idx = 0;
+	for (i = ae + ff_pci_sz; i < so->rx.ll_dl; i++)
+		so->rx.buf[so->rx.idx++] = cf->data[i];
+
+	/* initial setup for this pdu receiption */
+	so->rx.sn = 1;
+	so->rx.state = ISOTP_WAIT_DATA;
+
+	/* no creation of flow control frames */
+	if (so->opt.flags & CAN_ISOTP_LISTEN_MODE)
+		return 0;
+
+	/* send our first FC frame */
+	isotp_send_fc(sk, ae, ISOTP_FC_CTS);
+	return 0;
+}
+
+static int isotp_rcv_cf(struct sock *sk, struct canfd_frame *cf, int ae,
+			struct sk_buff *skb)
+{
+	struct isotp_sock *so = isotp_sk(sk);
+	struct sk_buff *nskb;
+	int i;
+
+	if (so->rx.state != ISOTP_WAIT_DATA)
+		return 0;
+
+	/* drop if timestamp gap is less than force_rx_stmin nano secs */
+	if (so->opt.flags & CAN_ISOTP_FORCE_RXSTMIN) {
+
+		if (ktime_to_ns(ktime_sub(skb->tstamp, so->lastrxcf_tstamp)) <
+		    so->force_rx_stmin)
+			return 0;
+
+		so->lastrxcf_tstamp = skb->tstamp;
+	}
+
+	hrtimer_cancel(&so->rxtimer);
+
+	/* CFs are never longer than the FF */
+	if (cf->len > so->rx.ll_dl)
+		return 1;
+
+	/* CFs have usually the LL_DL length */
+	if (cf->len < so->rx.ll_dl) {
+		/* this is only allowed for the last CF */
+		if (so->rx.len - so->rx.idx > so->rx.ll_dl - ae - N_PCI_SZ)
+			return 1;
+	}
+
+	if ((cf->data[ae] & 0x0F) != so->rx.sn) {
+		DBG("wrong sn %d. expected %d.\n",
+		    cf->data[ae] & 0x0F, so->rx.sn);
+		/* some error reporting? */
+		so->rx.state = ISOTP_IDLE;
+		return 1;
+	}
+	so->rx.sn++;
+	so->rx.sn %= 16;
+
+	for (i = ae + N_PCI_SZ; i < cf->len; i++) {
+		so->rx.buf[so->rx.idx++] = cf->data[i];
+		if (so->rx.idx >= so->rx.len)
+			break;
+	}
+
+	if (so->rx.idx >= so->rx.len) {
+
+		/* we are done */
+		so->rx.state = ISOTP_IDLE;
+
+		if ((so->opt.flags & ISOTP_CHECK_PADDING) &&
+		    check_pad(so, cf, i+1, so->opt.rxpad_content))
+			return 1;
+
+		nskb = alloc_skb(so->rx.len, gfp_any());
+		if (!nskb)
+			return 1;
+
+		memcpy(skb_put(nskb, so->rx.len), so->rx.buf,
+		       so->rx.len);
+
+		nskb->tstamp = skb->tstamp;
+		nskb->dev = skb->dev;
+		isotp_rcv_skb(nskb, sk);
+		return 0;
+	}
+
+	/* no creation of flow control frames */
+	if (so->opt.flags & CAN_ISOTP_LISTEN_MODE)
+		return 0;
+
+	/* perform blocksize handling, if enabled */
+	if (!so->rxfc.bs || ++so->rx.bs < so->rxfc.bs) {
+
+		/* start rx timeout watchdog */
+		hrtimer_start(&so->rxtimer, ktime_set(1,0),
+			      HRTIMER_MODE_REL);
+		return 0;
+	}
+
+	/* we reached the specified blocksize so->rxfc.bs */
+	isotp_send_fc(sk, ae, ISOTP_FC_CTS);
+	return 0;
+}
+
+static void isotp_rcv(struct sk_buff *skb, void *data)
+{
+	struct sock *sk = (struct sock *)data;
+	struct isotp_sock *so = isotp_sk(sk);
+	struct canfd_frame *cf;
+	int ae = (so->opt.flags & CAN_ISOTP_EXTEND_ADDR)? 1:0;
+	u8 n_pci_type, sf_dl;
+
+	BUG_ON(skb->len != CAN_MTU && skb->len != CANFD_MTU);
+
+	/*
+	 * Strictly receive only frames with the configured MTU size
+	 * => clear separation of CAN2.0 / CAN FD transport channels
+	 */
+	if (skb->len != so->ll.mtu)
+		return;
+
+	cf = (struct canfd_frame *) skb->data;
+
+	/* if enabled: check receiption of my configured extended address */
+	if (ae && cf->data[0] != so->opt.rx_ext_address)
+		return;
+
+	n_pci_type = cf->data[ae] & 0xF0;
+
+	if (so->opt.flags & CAN_ISOTP_HALF_DUPLEX) {
+		/* check rx/tx path half duplex expectations */
+		if ((so->tx.state != ISOTP_IDLE && n_pci_type != N_PCI_FC) ||
+		    (so->rx.state != ISOTP_IDLE && n_pci_type == N_PCI_FC))
+			return;
+	}
+
+	switch (n_pci_type) {
+	case N_PCI_FC:
+		/* tx path: flow control frame containing the FC parameters */
+		isotp_rcv_fc(so, cf, ae);
+		break;
+
+	case N_PCI_SF:
+		/*
+		 * rx path: single frame
+		 *
+		 * As we do not have a rx.ll_dl configuration, we can only test
+		 * if the CAN frames payload length matches the LL_DL == 8
+		 * requirements - no matter if it's CAN 2.0 or CAN FD
+		 */
+
+		/* get the SF_DL from the N_PCI byte */
+		sf_dl = cf->data[ae] & 0x0F;
+
+		if (cf->len <= CAN_MAX_DLEN)
+			isotp_rcv_sf(sk, cf, SF_PCI_SZ4 + ae, skb, sf_dl);
+		else if (skb->len == CANFD_MTU) {
+			/*
+			 * We have a CAN FD frame and CAN_DL is greater than 8:
+			 * Only frames with the SF_DL == 0 ESC value are valid.
+			 *
+			 * If so take care of the increased SF PCI size
+			 * (SF_PCI_SZ8) to point to the message content behind
+			 * the extended SF PCI info and get the real SF_DL
+			 * length value from the formerly first data byte.
+			 */
+			if (sf_dl == 0)
+				isotp_rcv_sf(sk, cf, SF_PCI_SZ8 + ae, skb,
+					     cf->data[SF_PCI_SZ4 + ae]);
+		}
+		break;
+
+	case N_PCI_FF:
+		/* rx path: first frame */
+		isotp_rcv_ff(sk, cf, ae);
+		break;
+
+	case N_PCI_CF:
+		/* rx path: consecutive frame */
+		isotp_rcv_cf(sk, cf, ae, skb);
+		break;
+
+	}
+}
+
+static void isotp_fill_dataframe(struct canfd_frame *cf, struct isotp_sock *so,
+				 int ae, int off)
+{
+	int pcilen = N_PCI_SZ + ae + off;
+	int space = so->tx.ll_dl - pcilen;
+	int num = min_t(int, so->tx.len - so->tx.idx, space);
+	int i;
+
+	cf->can_id = so->txid;
+	cf->len = num + pcilen;
+
+	if (num < space) {
+		if (so->opt.flags & CAN_ISOTP_TX_PADDING) {
+			/* user requested padding */
+			cf->len = padlen(cf->len);
+			memset(cf->data, so->opt.txpad_content, cf->len);
+		} else if (cf->len > CAN_MAX_DLEN) {
+			/* mandatory padding for CAN FD frames */
+			cf->len = padlen(cf->len);
+			memset(cf->data, CAN_ISOTP_DEFAULT_PAD_CONTENT,
+			       cf->len);
+		}
+	}
+
+	for (i = 0; i < num; i++)
+		cf->data[pcilen + i] = so->tx.buf[so->tx.idx++];
+
+	if (ae)
+		cf->data[0] = so->opt.ext_address;
+}
+
+static void isotp_create_fframe(struct canfd_frame *cf, struct isotp_sock *so,
+				int ae)
+{
+	int i;
+	int ff_pci_sz;
+
+	cf->can_id = so->txid;
+	cf->len = so->tx.ll_dl;
+	if (ae)
+		cf->data[0] = so->opt.ext_address;
+
+	/* create N_PCI bytes with 12/32 bit FF_DL data length */
+	if (so->tx.len > 4095) {
+		/* use 32 bit FF_DL notation */
+		cf->data[ae] = N_PCI_FF;
+		cf->data[ae + 1] = 0;
+		cf->data[ae + 2] = (u8) (so->tx.len >> 24) & 0xFFU;
+		cf->data[ae + 3] = (u8) (so->tx.len >> 16) & 0xFFU;
+		cf->data[ae + 4] = (u8) (so->tx.len >> 8) & 0xFFU;
+		cf->data[ae + 5] = (u8) so->tx.len & 0xFFU;
+		ff_pci_sz = FF_PCI_SZ32;
+	} else {
+		/* use 12 bit FF_DL notation */
+		cf->data[ae] = (u8) (so->tx.len>>8) | N_PCI_FF;
+		cf->data[ae + 1] = (u8) so->tx.len & 0xFFU;
+		ff_pci_sz = FF_PCI_SZ12;
+	}
+
+	/* add first data bytes depending on ae */
+	for (i = ae + ff_pci_sz; i < so->tx.ll_dl; i++)
+		cf->data[i] = so->tx.buf[so->tx.idx++];
+
+	so->tx.sn = 1;
+	so->tx.state = ISOTP_WAIT_FIRST_FC;
+}
+
+static void isotp_tx_timer_tsklet(unsigned long data)
+{
+	struct isotp_sock *so = (struct isotp_sock *)data;
+	struct sock *sk = &so->sk;
+	struct sk_buff *skb;
+	struct net_device *dev;
+	struct canfd_frame *cf;
+	int ae = (so->opt.flags & CAN_ISOTP_EXTEND_ADDR)? 1:0;
+
+	switch (so->tx.state) {
+
+	case ISOTP_WAIT_FC:
+	case ISOTP_WAIT_FIRST_FC:
+
+		/* we did not get any flow control frame in time */
+
+		DBG("we did not get FC frame in time.\n");
+
+		/* reset tx state */
+		so->tx.state = ISOTP_IDLE;
+		wake_up_interruptible(&so->wait);
+		break;
+
+	case ISOTP_SENDING:
+
+		/* push out the next segmented pdu */
+
+		DBG("next pdu to send.\n");
+
+		dev = dev_get_by_index(sock_net(sk), so->ifindex);
+		if (!dev)
+			break;
+
+isotp_tx_burst:
+		skb = alloc_skb(so->ll.mtu + CAN_SKBRES, gfp_any());
+		if (!skb) {
+			dev_put(dev);
+			break;
+		}
+
+		isotp_skb_reserve(skb, dev);
+		cf = (struct canfd_frame *)skb->data;
+		skb_put(skb, so->ll.mtu);
+
+		/* create consecutive frame */
+		isotp_fill_dataframe(cf, so, ae, 0);
+
+		/* place consecutive frame N_PCI in appropriate index */
+		cf->data[ae] = N_PCI_CF | so->tx.sn++;
+		so->tx.sn %= 16;
+		so->tx.bs++;
+
+		if (so->ll.mtu == CANFD_MTU)
+			cf->flags = so->ll.tx_flags;
+
+		skb->dev = dev;
+		isotp_skb_set_owner(skb, sk);
+		can_send(skb, 1);
+
+		if (so->tx.idx >= so->tx.len) {
+			/* we are done */
+			DBG("we are done\n");
+			so->tx.state = ISOTP_IDLE;
+			dev_put(dev);
+			wake_up_interruptible(&so->wait);
+			break;
+		}
+
+		if (so->txfc.bs && so->tx.bs >= so->txfc.bs) {
+			/* stop and wait for FC */
+			DBG("BS stop and wait for FC\n");
+			so->tx.state = ISOTP_WAIT_FC;
+			dev_put(dev);
+			hrtimer_start(&so->txtimer,
+				      ktime_add(ktime_get(), ktime_set(1,0)),
+				      HRTIMER_MODE_ABS);
+			break;
+		}
+
+		/* no gap between data frames needed => use burst mode */
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)
+		if (!so->tx_gap)
+			goto isotp_tx_burst;
+#else
+		if (!so->tx_gap.tv64)
+			goto isotp_tx_burst;
+#endif
+
+		/* start timer to send next data frame with correct delay */
+		dev_put(dev);
+		hrtimer_start(&so->txtimer,
+			      ktime_add(ktime_get(), so->tx_gap),
+			      HRTIMER_MODE_ABS);
+		break;
+
+	default:
+		BUG_ON(1);
+	}
+}
+
+static enum hrtimer_restart isotp_tx_timer_handler(struct hrtimer *hrtimer)
+{
+	struct isotp_sock *so = container_of(hrtimer, struct isotp_sock,
+					     txtimer);
+	tasklet_schedule(&so->txtsklet);
+
+	return HRTIMER_NORESTART;
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,1,0)
+static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
+#else
+static int isotp_sendmsg(struct kiocb *iocb, struct socket *sock,
+		       struct msghdr *msg, size_t size)
+#endif
+{
+	struct sock *sk = sock->sk;
+	struct isotp_sock *so = isotp_sk(sk);
+	struct sk_buff *skb;
+	struct net_device *dev;
+	struct canfd_frame *cf;
+	int ae = (so->opt.flags & CAN_ISOTP_EXTEND_ADDR)? 1:0;
+	int off;
+	int err;
+
+	if (!so->bound)
+		return -EADDRNOTAVAIL;
+
+	/* we do not support multiple buffers - for now */
+	if (so->tx.state != ISOTP_IDLE) {
+		if (msg->msg_flags & MSG_DONTWAIT)
+			return -EAGAIN;
+
+		/* wait for complete transmission of current pdu */
+		wait_event_interruptible(so->wait, so->tx.state == ISOTP_IDLE);
+	}
+
+	if (!size || size > MAX_MSG_LENGTH)
+		return -EINVAL;
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,19,0)
+	err = memcpy_from_msg(so->tx.buf, msg, size);
+#else
+	err = memcpy_fromiovec(so->tx.buf, msg->msg_iov, size);
+#endif
+	if (err < 0)
+		return err;
+
+	dev = dev_get_by_index(sock_net(sk), so->ifindex);
+	if (!dev)
+		return -ENXIO;
+
+	skb = sock_alloc_send_skb(sk, so->ll.mtu + CAN_SKBRES,
+				  msg->msg_flags & MSG_DONTWAIT, &err);
+	if (!skb) {
+		dev_put(dev);
+		return err;
+	}
+
+	isotp_skb_reserve(skb, dev);
+
+	so->tx.state = ISOTP_SENDING;
+	so->tx.len = size;
+	so->tx.idx = 0;
+
+	cf = (struct canfd_frame *)skb->data;
+	skb_put(skb, so->ll.mtu);
+
+	/* take care of a potential SF_DL ESC offset for TX_DL > 8 */
+	off = (so->tx.ll_dl > CAN_MAX_DLEN)? 1:0;
+
+	/* check for single frame transmission depending on TX_DL */
+	if (size <= so->tx.ll_dl - SF_PCI_SZ4 - ae - off) {
+
+		/*
+		 * The message size generally fits into a SingleFrame - good.
+		 *
+		 * SF_DL ESC offset optimization:
+		 *
+		 * When TX_DL is greater 8 but the message would still fit
+		 * into a 8 byte CAN frame, we can omit the offset.
+		 * This prevents a protocol caused length extension from
+		 * CAN_DL = 8 to CAN_DL = 12 due to the SF_SL ESC handling.
+		 */
+		if (size <= CAN_MAX_DLEN - SF_PCI_SZ4 - ae)
+			off = 0;
+
+		isotp_fill_dataframe(cf, so, ae, off);
+
+		/* place single frame N_PCI w/o length in appropriate index */
+		cf->data[ae] = N_PCI_SF;
+
+		/* place SF_DL size value depending on the SF_DL ESC offset */
+		if (off)
+			cf->data[SF_PCI_SZ4 + ae] = size;
+		else
+			cf->data[ae] |= size;
+
+		so->tx.state = ISOTP_IDLE;
+		wake_up_interruptible(&so->wait);
+	} else {
+		/* send first frame and wait for FC */
+
+		isotp_create_fframe(cf, so, ae);
+
+		DBG("starting txtimer for fc\n");
+		/* start timeout for FC */
+		hrtimer_start(&so->txtimer, ktime_set(1,0), HRTIMER_MODE_REL);
+	}
+
+	/* send the first or only CAN frame */
+	if (so->ll.mtu == CANFD_MTU)
+		cf->flags = so->ll.tx_flags;
+
+	skb->dev = dev;
+	skb->sk  = sk;
+	err = can_send(skb, 1);
+	dev_put(dev);
+	if (err)
+		return err;
+
+	return size;
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,1,0)
+static int isotp_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
+			 int flags)
+#else
+static int isotp_recvmsg(struct kiocb *iocb, struct socket *sock,
+			 struct msghdr *msg, size_t size, int flags)
+#endif
+{
+	struct sock *sk = sock->sk;
+	struct sk_buff *skb;
+	int err = 0;
+	int noblock;
+
+	noblock =  flags & MSG_DONTWAIT;
+	flags   &= ~MSG_DONTWAIT;
+
+	skb = skb_recv_datagram(sk, flags, noblock, &err);
+	if (!skb)
+		return err;
+
+	if (size < skb->len)
+		msg->msg_flags |= MSG_TRUNC;
+	else
+		size = skb->len;
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,19,0)
+	err = memcpy_to_msg(msg, skb->data, size);
+#else
+	err = memcpy_toiovec(msg->msg_iov, skb->data, size);
+#endif
+	if (err < 0) {
+		skb_free_datagram(sk, skb);
+		return err;
+	}
+
+	sock_recv_timestamp(msg, sk, skb);
+
+	if (msg->msg_name) {
+		msg->msg_namelen = sizeof(struct sockaddr_can);
+		memcpy(msg->msg_name, skb->cb, msg->msg_namelen);
+	}
+
+	skb_free_datagram(sk, skb);
+
+	return size;
+}
+
+static int isotp_release(struct socket *sock)
+{
+	struct sock *sk = sock->sk;
+	struct isotp_sock *so;
+	struct net *net;
+
+	if (!sk)
+		return 0;
+
+	so = isotp_sk(sk);
+	net = sock_net(sk);
+
+	/* wait for complete transmission of current pdu */
+	wait_event_interruptible(so->wait, so->tx.state == ISOTP_IDLE);
+
+	unregister_netdevice_notifier(&so->notifier);
+
+	lock_sock(sk);
+
+	hrtimer_cancel(&so->txtimer);
+	hrtimer_cancel(&so->rxtimer);
+	tasklet_kill(&so->txtsklet);
+
+	/* remove current filters & unregister */
+	if (so->bound) {
+		if (so->ifindex) {
+			struct net_device *dev;
+
+			dev = dev_get_by_index(net, so->ifindex);
+			if (dev) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0)
+				can_rx_unregister(net, dev, so->rxid,
+#else
+				can_rx_unregister(dev, so->rxid,
+#endif
+						  SINGLE_MASK(so->rxid),
+						  isotp_rcv, sk);
+				dev_put(dev);
+			}
+		}
+	}
+
+	so->ifindex = 0;
+	so->bound   = 0;
+
+	sock_orphan(sk);
+	sock->sk = NULL;
+
+	release_sock(sk);
+	sock_put(sk);
+
+	return 0;
+}
+
+static int isotp_bind(struct socket *sock, struct sockaddr *uaddr, int len)
+{
+	struct sockaddr_can *addr = (struct sockaddr_can *)uaddr;
+	struct sock *sk = sock->sk;
+	struct isotp_sock *so = isotp_sk(sk);
+	struct net *net = sock_net(sk);
+	int ifindex;
+	struct net_device *dev;
+	int err = 0;
+	int notify_enetdown = 0;
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,4,0)
+	if (len < CAN_REQUIRED_SIZE(struct sockaddr_can, can_addr.tp))
+#else
+	if (len < sizeof(*addr))
+#endif
+		return -EINVAL;
+
+	if (addr->can_addr.tp.rx_id == addr->can_addr.tp.tx_id)
+		return -EADDRNOTAVAIL;
+
+	if ((addr->can_addr.tp.rx_id | addr->can_addr.tp.tx_id) &
+	    (CAN_ERR_FLAG | CAN_RTR_FLAG))
+		return -EADDRNOTAVAIL;
+
+	if (!addr->can_ifindex)
+		return -ENODEV;
+
+	lock_sock(sk);
+
+	if (so->bound && addr->can_ifindex == so->ifindex &&
+	    addr->can_addr.tp.rx_id == so->rxid &&
+	    addr->can_addr.tp.tx_id == so->txid)
+		goto out;
+
+	dev = dev_get_by_index(net, addr->can_ifindex);
+	if (!dev) {
+		err = -ENODEV;
+		goto out;
+	}
+	if (dev->type != ARPHRD_CAN) {
+		dev_put(dev);
+		err = -ENODEV;
+		goto out;
+	}
+	if (dev->mtu < so->ll.mtu) {
+		dev_put(dev);
+		err = -EINVAL;
+		goto out;
+	}
+	if (!(dev->flags & IFF_UP))
+		notify_enetdown = 1;
+
+	ifindex = dev->ifindex;
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0)
+	can_rx_register(net, dev, addr->can_addr.tp.rx_id,
+#else
+	can_rx_register(dev, addr->can_addr.tp.rx_id,
+#endif
+			SINGLE_MASK(addr->can_addr.tp.rx_id), isotp_rcv, sk,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,9,11)) || \
+	((LINUX_VERSION_CODE >= KERNEL_VERSION(4,4,50)) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0))) || \
+	((LINUX_VERSION_CODE >= KERNEL_VERSION(3,18,49)) && (LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0))) || \
+	((LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,42)) && (LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)))
+			"isotp", sk);
+#else
+			"isotp");
+#endif
+	dev_put(dev);
+
+	if (so->bound) {
+		/* unregister old filter */
+		if (so->ifindex) {
+			dev = dev_get_by_index(net, so->ifindex);
+			if (dev) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0)
+				can_rx_unregister(net, dev, so->rxid,
+#else
+				can_rx_unregister(dev, so->rxid,
+#endif
+						  SINGLE_MASK(so->rxid),
+						  isotp_rcv, sk);
+				dev_put(dev);
+			}
+		}
+	}
+
+	/* switch to new settings */
+	so->ifindex = ifindex;
+	so->rxid = addr->can_addr.tp.rx_id;
+	so->txid = addr->can_addr.tp.tx_id;
+	so->bound = 1;
+
+out:
+	release_sock(sk);
+
+	if (notify_enetdown) {
+		sk->sk_err = ENETDOWN;
+		if (!sock_flag(sk, SOCK_DEAD))
+			sk->sk_error_report(sk);
+	}
+
+	return err;
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,17,0)
+static int isotp_getname(struct socket *sock, struct sockaddr *uaddr, int peer)
+#else
+static int isotp_getname(struct socket *sock, struct sockaddr *uaddr,
+		       int *len, int peer)
+#endif
+{
+	struct sockaddr_can *addr = (struct sockaddr_can *)uaddr;
+	struct sock *sk = sock->sk;
+	struct isotp_sock *so = isotp_sk(sk);
+
+	if (peer)
+		return -EOPNOTSUPP;
+
+	addr->can_family  = AF_CAN;
+	addr->can_ifindex = so->ifindex;
+	addr->can_addr.tp.rx_id = so->rxid;
+	addr->can_addr.tp.tx_id = so->txid;
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,17,0)
+	return sizeof(*addr);
+#else
+	*len = sizeof(*addr);
+
+	return 0;
+#endif
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,32)
+static int isotp_setsockopt(struct socket *sock, int level, int optname,
+			    char __user *optval, unsigned int optlen)
+#else
+static int isotp_setsockopt(struct socket *sock, int level, int optname,
+			    char __user *optval, int optlen)
+#endif
+{
+	struct sock *sk = sock->sk;
+	struct isotp_sock *so = isotp_sk(sk);
+	int ret = 0;
+
+	if (level != SOL_CAN_ISOTP)
+		return -EINVAL;
+	if (optlen < 0)
+		return -EINVAL;
+
+	switch (optname) {
+
+	case CAN_ISOTP_OPTS:
+		if (optlen != sizeof(struct can_isotp_options))
+			return -EINVAL;
+
+		if (copy_from_user(&so->opt, optval, optlen))
+			return -EFAULT;
+
+		/* no separate rx_ext_address is given => use ext_address */
+		if (!(so->opt.flags & CAN_ISOTP_RX_EXT_ADDR))
+			so->opt.rx_ext_address = so->opt.ext_address;
+		break;
+
+	case CAN_ISOTP_RECV_FC:
+		if (optlen != sizeof(struct can_isotp_fc_options))
+			return -EINVAL;
+
+		if (copy_from_user(&so->rxfc, optval, optlen))
+			return -EFAULT;
+		break;
+
+	case CAN_ISOTP_TX_STMIN:
+		if (optlen != sizeof(__u32))
+			return -EINVAL;
+
+		if (copy_from_user(&so->force_tx_stmin, optval, optlen))
+			return -EFAULT;
+		break;
+
+	case CAN_ISOTP_RX_STMIN:
+		if (optlen != sizeof(__u32))
+			return -EINVAL;
+
+		if (copy_from_user(&so->force_rx_stmin, optval, optlen))
+			return -EFAULT;
+		break;
+
+	case CAN_ISOTP_LL_OPTS:
+		if (optlen != sizeof(struct can_isotp_ll_options))
+			return -EINVAL;
+		else {
+			struct can_isotp_ll_options ll;
+
+			if (copy_from_user(&ll, optval, optlen))
+				return -EFAULT;
+
+			/* check for correct ISO 11898-1 DLC data lentgh */
+			if (ll.tx_dl != padlen(ll.tx_dl))
+				return -EINVAL;
+
+			if (ll.mtu != CAN_MTU && ll.mtu != CANFD_MTU)
+				return -EINVAL;
+
+			if (ll.mtu == CAN_MTU && ll.tx_dl > CAN_MAX_DLEN)
+				return -EINVAL;
+
+			memcpy(&so->ll, &ll, sizeof(ll));
+
+			/* set ll_dl for tx path to similar place as for rx */
+			so->tx.ll_dl = ll.tx_dl;
+		}
+		break;
+
+	default:
+		ret = -ENOPROTOOPT;
+	}
+
+	return ret;
+}
+
+static int isotp_getsockopt(struct socket *sock, int level, int optname,
+			  char __user *optval, int __user *optlen)
+{
+	struct sock *sk = sock->sk;
+	struct isotp_sock *so = isotp_sk(sk);
+	int len;
+	void *val;
+
+	if (level != SOL_CAN_ISOTP)
+		return -EINVAL;
+	if (get_user(len, optlen))
+		return -EFAULT;
+	if (len < 0)
+		return -EINVAL;
+
+	switch (optname) {
+
+	case CAN_ISOTP_OPTS:
+		len = min_t(int, len, sizeof(struct can_isotp_options));
+		val = &so->opt;
+		break;
+
+	case CAN_ISOTP_RECV_FC:
+		len = min_t(int, len, sizeof(struct can_isotp_fc_options));
+		val = &so->rxfc;
+		break;
+
+	case CAN_ISOTP_TX_STMIN:
+		len = min_t(int, len, sizeof(__u32));
+		val = &so->force_tx_stmin;
+		break;
+
+	case CAN_ISOTP_RX_STMIN:
+		len = min_t(int, len, sizeof(__u32));
+		val = &so->force_rx_stmin;
+		break;
+
+	case CAN_ISOTP_LL_OPTS:
+		len = min_t(int, len, sizeof(struct can_isotp_ll_options));
+		val = &so->ll;
+		break;
+
+	default:
+		return -ENOPROTOOPT;
+	}
+
+	if (put_user(len, optlen))
+		return -EFAULT;
+	if (copy_to_user(optval, val, len))
+		return -EFAULT;
+	return 0;
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,11,0)
+static int isotp_notifier(struct notifier_block *nb, unsigned long msg,
+			  void *ptr)
+{
+	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
+#else
+static int isotp_notifier(struct notifier_block *nb,
+			unsigned long msg, void *data)
+{
+	struct net_device *dev = (struct net_device *)data;
+#endif
+	struct isotp_sock *so = container_of(nb, struct isotp_sock, notifier);
+	struct sock *sk = &so->sk;
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0)
+	if (!net_eq(dev_net(dev), sock_net(sk)))
+		return NOTIFY_DONE;
+#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26)
+	if (dev_net(dev) != &init_net)
+		return NOTIFY_DONE;
+#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,24)
+	if (dev->nd_net != &init_net)
+		return NOTIFY_DONE;
+#endif
+
+	if (dev->type != ARPHRD_CAN)
+		return NOTIFY_DONE;
+
+	if (so->ifindex != dev->ifindex)
+		return NOTIFY_DONE;
+
+	switch (msg) {
+
+	case NETDEV_UNREGISTER:
+		lock_sock(sk);
+		/* remove current filters & unregister */
+		if (so->bound)
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0)
+			can_rx_unregister(dev_net(dev), dev, so->rxid,
+#else
+			can_rx_unregister(dev, so->rxid,
+#endif
+					  SINGLE_MASK(so->rxid),
+					  isotp_rcv, sk);
+
+		so->ifindex = 0;
+		so->bound   = 0;
+		release_sock(sk);
+
+		sk->sk_err = ENODEV;
+		if (!sock_flag(sk, SOCK_DEAD))
+			sk->sk_error_report(sk);
+		break;
+
+	case NETDEV_DOWN:
+		sk->sk_err = ENETDOWN;
+		if (!sock_flag(sk, SOCK_DEAD))
+			sk->sk_error_report(sk);
+		break;
+	}
+
+	return NOTIFY_DONE;
+}
+
+static int isotp_init(struct sock *sk)
+{
+	struct isotp_sock *so = isotp_sk(sk);
+
+	so->ifindex = 0;
+	so->bound   = 0;
+
+	so->opt.flags		= CAN_ISOTP_DEFAULT_FLAGS;
+	so->opt.ext_address	= CAN_ISOTP_DEFAULT_EXT_ADDRESS;
+	so->opt.rx_ext_address	= CAN_ISOTP_DEFAULT_EXT_ADDRESS;
+	so->opt.rxpad_content	= CAN_ISOTP_DEFAULT_PAD_CONTENT;
+	so->opt.txpad_content	= CAN_ISOTP_DEFAULT_PAD_CONTENT;
+	so->opt.frame_txtime	= CAN_ISOTP_DEFAULT_FRAME_TXTIME;
+	so->rxfc.bs		= CAN_ISOTP_DEFAULT_RECV_BS;
+	so->rxfc.stmin		= CAN_ISOTP_DEFAULT_RECV_STMIN;
+	so->rxfc.wftmax		= CAN_ISOTP_DEFAULT_RECV_WFTMAX;
+	so->ll.mtu		= CAN_ISOTP_DEFAULT_LL_MTU;
+	so->ll.tx_dl		= CAN_ISOTP_DEFAULT_LL_TX_DL;
+	so->ll.tx_flags		= CAN_ISOTP_DEFAULT_LL_TX_FLAGS;
+
+	/* set ll_dl for tx path to similar place as for rx */
+	so->tx.ll_dl		= so->ll.tx_dl;
+
+	so->rx.state = ISOTP_IDLE;
+	so->tx.state = ISOTP_IDLE;
+
+	hrtimer_init(&so->rxtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+	so->rxtimer.function = isotp_rx_timer_handler;
+	hrtimer_init(&so->txtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+	so->txtimer.function = isotp_tx_timer_handler;
+
+	tasklet_init(&so->txtsklet, isotp_tx_timer_tsklet, (unsigned long)so);
+
+	init_waitqueue_head(&so->wait);
+
+	so->notifier.notifier_call = isotp_notifier;
+	register_netdevice_notifier(&so->notifier);
+
+	return 0;
+}
+
+int isotp_sock_no_ioctlcmd(struct socket *sock, unsigned int cmd,
+			 unsigned long arg)
+{
+	/* no ioctls for socket layer -> hand it down to NIC layer */
+	return -ENOIOCTLCMD;
+}
+
+static const struct proto_ops isotp_ops = {
+	.family		= PF_CAN,
+	.release	= isotp_release,
+	.bind		= isotp_bind,
+	.connect	= sock_no_connect,
+	.socketpair	= sock_no_socketpair,
+	.accept		= sock_no_accept,
+	.getname	= isotp_getname,
+	.poll		= datagram_poll,
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,2,0)
+	.ioctl		= isotp_sock_no_ioctlcmd,
+	.gettstamp	= sock_gettstamp,
+#else
+	.ioctl		= can_ioctl,	/* use can_ioctl() from af_can.c */
+#endif
+	.listen		= sock_no_listen,
+	.shutdown	= sock_no_shutdown,
+	.setsockopt	= isotp_setsockopt,
+	.getsockopt	= isotp_getsockopt,
+	.sendmsg	= isotp_sendmsg,
+	.recvmsg	= isotp_recvmsg,
+	.mmap		= sock_no_mmap,
+	.sendpage	= sock_no_sendpage,
+};
+
+static struct proto isotp_proto __read_mostly = {
+	.name		= "CAN_ISOTP",
+	.owner		= THIS_MODULE,
+	.obj_size	= sizeof(struct isotp_sock),
+	.init		= isotp_init,
+};
+
+static const struct can_proto isotp_can_proto = {
+	.type		= SOCK_DGRAM,
+	.protocol	= CAN_ISOTP,
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,33)
+	.capability	= -1,
+#endif
+	.ops		= &isotp_ops,
+	.prot		= &isotp_proto,
+};
+
+static __init int isotp_module_init(void)
+{
+	int err;
+
+	printk(banner);
+
+	err = can_proto_register(&isotp_can_proto);
+	if (err < 0)
+		printk(KERN_ERR "can: registration of isotp protocol failed\n");
+
+	return err;
+}
+
+static __exit void isotp_module_exit(void)
+{
+	can_proto_unregister(&isotp_can_proto);
+}
+
+module_init(isotp_module_init);
+module_exit(isotp_module_exit);
diff -Naur --no-dereference a/net/can/Kconfig b/net/can/Kconfig
--- a/net/can/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/net/can/Kconfig	2020-06-19 21:00:31.432227716 -0400
@@ -51,6 +51,16 @@
 	  They can be modified with AND/OR/XOR/SET operations as configured
 	  by the netlink configuration interface known e.g. from iptables.
 
+config CAN_ISOTP
+	tristate "ISO 15765-2:2016 CAN transport protocol"
+	default y
+	---help---
+	  ISO 15765-2 CAN transport protocol for protocol family CAN
+
+	  This implementation is already widely used in automotive use-cases, e.g.
+	  for UDS based OBD diagnosis. Although some small adaptions may be applied
+	  to make it ready for Linux Mainline. Feedback is welcome.
+
 source "drivers/net/can/Kconfig"
 
 endif
diff -Naur --no-dereference a/net/can/Makefile b/net/can/Makefile
--- a/net/can/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/net/can/Makefile	2020-06-19 21:00:31.432227716 -0400
@@ -15,3 +15,6 @@
 
 obj-$(CONFIG_CAN_GW)	+= can-gw.o
 can-gw-y		:= gw.o
+
+obj-$(CONFIG_CAN_ISOTP)	+= can-isotp.o
+can-isotp-y		:= isotp.o
diff -Naur --no-dereference a/net/core/dev_ioctl.c b/net/core/dev_ioctl.c
--- a/net/core/dev_ioctl.c	2020-01-09 04:19:10.000000000 -0500
+++ b/net/core/dev_ioctl.c	2020-06-19 21:00:31.432227716 -0400
@@ -315,6 +315,7 @@
 		    cmd == SIOCBRDELIF ||
 		    cmd == SIOCSHWTSTAMP ||
 		    cmd == SIOCGHWTSTAMP ||
+		    cmd == SIOCSWITCHCONFIG ||
 		    cmd == SIOCWANDEV) {
 			err = -EOPNOTSUPP;
 			if (ops->ndo_do_ioctl) {
@@ -500,6 +501,12 @@
 	case SIOCSIFLINK:
 		return -ENOTTY;
 
+	case SIOCSWITCHCONFIG:
+		rtnl_lock();
+		ret = dev_ifsioc(net, ifr, cmd);
+		rtnl_unlock();
+		return ret;
+
 	/*
 	 *	Unknown or private ioctl.
 	 */
diff -Naur --no-dereference a/net/core/ethtool.c b/net/core/ethtool.c
--- a/net/core/ethtool.c	2020-01-09 04:19:10.000000000 -0500
+++ b/net/core/ethtool.c	2020-06-19 21:00:31.432227716 -0400
@@ -434,7 +434,7 @@
 /* return false if legacy contained non-0 deprecated fields
  * maxtxpkt/maxrxpkt. rest of ksettings always updated
  */
-static bool
+bool
 convert_legacy_settings_to_link_ksettings(
 	struct ethtool_link_ksettings *link_ksettings,
 	const struct ethtool_cmd *legacy_settings)
@@ -478,11 +478,12 @@
 		= legacy_settings->eth_tp_mdix_ctrl;
 	return retval;
 }
+EXPORT_SYMBOL_GPL(convert_legacy_settings_to_link_ksettings);
 
 /* return false if ksettings link modes had higher bits
  * set. legacy_settings always updated (best effort)
  */
-static bool
+bool
 convert_link_ksettings_to_legacy_settings(
 	struct ethtool_cmd *legacy_settings,
 	const struct ethtool_link_ksettings *link_ksettings)
@@ -524,6 +525,7 @@
 		= link_ksettings->base.transceiver;
 	return retval;
 }
+EXPORT_SYMBOL_GPL(convert_link_ksettings_to_legacy_settings);
 
 /* number of 32-bit words to store the user's link mode bitmaps */
 #define __ETHTOOL_LINK_MODE_MASK_NU32			\
diff -Naur --no-dereference a/net/core/sock.c b/net/core/sock.c
--- a/net/core/sock.c	2020-01-09 04:19:10.000000000 -0500
+++ b/net/core/sock.c	2020-06-19 21:00:31.432227716 -0400
@@ -228,7 +228,7 @@
   x "AF_IEEE802154",	x "AF_CAIF"	,	x "AF_ALG"      , \
   x "AF_NFC"   ,	x "AF_VSOCK"    ,	x "AF_KCM"      , \
   x "AF_QIPCRTR",	x "AF_SMC"	,	x "AF_XDP"	, \
-  x "AF_MAX"
+  x "AF_RPMSG" ,	x "AF_MAX"
 
 static const char *const af_family_key_strings[AF_MAX+1] = {
 	_sock_locks("sk_lock-")
diff -Naur --no-dereference a/net/Kconfig b/net/Kconfig
--- a/net/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/net/Kconfig	2020-06-19 21:00:31.432227716 -0400
@@ -86,6 +86,7 @@
 	  Short answer: say Y.
 
 if INET
+source "net/wireguard/Kconfig"
 source "net/ipv4/Kconfig"
 source "net/ipv6/Kconfig"
 source "net/netlabel/Kconfig"
@@ -381,6 +382,7 @@
 source "net/rfkill/Kconfig"
 source "net/9p/Kconfig"
 source "net/caif/Kconfig"
+source "net/rpmsg/Kconfig"
 source "net/ceph/Kconfig"
 source "net/nfc/Kconfig"
 source "net/psample/Kconfig"
diff -Naur --no-dereference a/net/Makefile b/net/Makefile
--- a/net/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/net/Makefile	2020-06-19 21:00:31.432227716 -0400
@@ -15,6 +15,7 @@
 obj-$(CONFIG_LLC)		+= llc/
 obj-$(CONFIG_NET)		+= ethernet/ 802/ sched/ netlink/ bpf/
 obj-$(CONFIG_NETFILTER)		+= netfilter/
+obj-$(CONFIG_WIREGUARD)		+= wireguard/
 obj-$(CONFIG_INET)		+= ipv4/
 obj-$(CONFIG_TLS)		+= tls/
 obj-$(CONFIG_XFRM)		+= xfrm/
@@ -87,3 +88,4 @@
 obj-$(CONFIG_QRTR)		+= qrtr/
 obj-$(CONFIG_NET_NCSI)		+= ncsi/
 obj-$(CONFIG_XDP_SOCKETS)	+= xdp/
+obj-$(CONFIG_RPMSG_PROTO)	+= rpmsg/
diff -Naur --no-dereference a/net/rpmsg/Kconfig b/net/rpmsg/Kconfig
--- a/net/rpmsg/Kconfig	1969-12-31 19:00:00.000000000 -0500
+++ b/net/rpmsg/Kconfig	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,19 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# RPMsg Sockets
+#
+
+config RPMSG_PROTO
+	tristate "RPMsg Sockets over virtio-rpmsg transport"
+	default n
+	depends on RPMSG_VIRTIO
+	depends on REMOTEPROC
+	help
+	  An rpmsg driver that provides support for remote processor messaging
+	  sockets over the virtio rpmsg transport. This exposes a socket
+	  interface to user space to allow applications to communicate with
+	  various remote processors over this transport. This is currently
+	  designed to work with the TI IPC stack on various available TI SoCs,
+	  but can be generalized easily enough.
+
+	  If unsure, say N.
diff -Naur --no-dereference a/net/rpmsg/Makefile b/net/rpmsg/Makefile
--- a/net/rpmsg/Makefile	1969-12-31 19:00:00.000000000 -0500
+++ b/net/rpmsg/Makefile	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_RPMSG_PROTO)	+= rpmsg_proto.o
diff -Naur --no-dereference a/net/rpmsg/rpmsg_proto.c b/net/rpmsg/rpmsg_proto.c
--- a/net/rpmsg/rpmsg_proto.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/rpmsg/rpmsg_proto.c	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,739 @@
+// SPDX-License-Identifier: GPL-2.0
+/* AF_RPMSG: Remote processor messaging sockets
+ *
+ * Copyright (C) 2011-2018 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * Ohad Ben-Cohen <ohad@wizery.com>
+ * Robert Tivy <rtivy@ti.com>
+ * G Anthony <a0783926@ti.com>
+ * Suman Anna <s-anna@ti.com>
+ */
+
+#define pr_fmt(fmt)    "%s: " fmt, __func__
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/types.h>
+#include <linux/list.h>
+#include <linux/errno.h>
+#include <linux/skbuff.h>
+#include <linux/err.h>
+#include <linux/mutex.h>
+#include <linux/rpmsg.h>
+#include <linux/radix-tree.h>
+#include <linux/remoteproc.h>
+#include <linux/rpmsg/virtio_rpmsg.h>
+#include <net/sock.h>
+#include <uapi/linux/rpmsg_socket.h>
+
+#define RPMSG_CB(skb)	(*(struct sockaddr_rpmsg *)&((skb)->cb))
+
+/* Maximum buffer size supported by virtio rpmsg transport.
+ * Must match value as in drivers/rpmsg/virtio_rpmsg_bus.c
+ */
+#define RPMSG_BUF_SIZE               (512)
+
+struct rpmsg_socket {
+	struct sock sk;
+	struct rpmsg_device *rpdev;
+	struct rpmsg_endpoint *endpt;
+	int rproc_id;
+	struct list_head elem;
+};
+
+/* Connection and socket states */
+enum {
+	RPMSG_CONNECTED = 1,
+	RPMSG_OPEN,
+	RPMSG_LISTENING,
+	RPMSG_CLOSED,
+	RPMSG_ERROR,
+};
+
+/* A single-level radix-tree-based scheme is used to maintain the rpmsg
+ * channels we're exposing to userland. The radix tree maps a rproc index
+ * id to its published rpmsg-proto channel. Only a single rpmsg device is
+ * supported at the moment from each remote processor. This can be easily
+ * scaled to multiple devices using unique destination addresses but this
+ *_will_ require additional semantic changes on bind() and connect().
+ */
+static RADIX_TREE(rpmsg_channels, GFP_KERNEL);
+
+/* Synchronization of access to the tree is achieved using a mutex,
+ * because we're using non-atomic radix tree allocations.
+ */
+static DEFINE_MUTEX(rpmsg_channels_lock);
+
+static int rpmsg_sock_cb(struct rpmsg_device *rpdev, void *data, int len,
+			 void *priv, u32 src);
+
+static struct proto rpmsg_proto = {
+	.name		= "RPMSG",
+	.owner		= THIS_MODULE,
+	.obj_size	= sizeof(struct rpmsg_socket),
+};
+
+/* Retrieve the rproc instance so that it can be used for retrieving
+ * the processor id associated with the rpmsg channel.
+ */
+static inline struct rproc *rpdev_to_rproc(struct rpmsg_device *rpdev)
+{
+	return rproc_get_by_child(&rpdev->dev);
+}
+
+/* Retrieve the rproc id. The rproc id _relies_ on aliases being defined
+ * in the DT blob for each of the remoteproc devices, and is essentially
+ * the alias id. These are assumed to match to be fixed for a particular
+ * SoC, and this provides a means to have a fixed interface to identify
+ * a remote processor.
+ */
+static int rpmsg_sock_get_proc_id(struct rpmsg_device *rpdev)
+{
+	struct rproc *rproc = rpdev_to_rproc(rpdev);
+	int id;
+
+	if (!rproc) {
+		WARN_ON(1);
+		return -EINVAL;
+	}
+
+	id = rproc_get_id(rproc);
+	WARN_ON(id < 0);
+
+	return id;
+}
+
+static int rpmsg_sock_connect(struct socket *sock, struct sockaddr *addr,
+			      int alen, int flags)
+{
+	struct sock *sk = sock->sk;
+	struct rpmsg_socket *rpsk;
+	struct sockaddr_rpmsg *sa;
+	int err = 0;
+	struct rpmsg_device *rpdev;
+
+	if (sk->sk_state != RPMSG_OPEN)
+		return -EBADFD;
+
+	if (sk->sk_type != SOCK_SEQPACKET)
+		return -EINVAL;
+
+	if (!addr || addr->sa_family != AF_RPMSG)
+		return -EINVAL;
+
+	if (alen < sizeof(*sa))
+		return -EINVAL;
+
+	sa = (struct sockaddr_rpmsg *)addr;
+
+	mutex_lock(&rpmsg_channels_lock);
+	lock_sock(sk);
+
+	rpsk = container_of(sk, struct rpmsg_socket, sk);
+
+	/* find the set of channels exposed by this remote processor */
+	rpdev = radix_tree_lookup(&rpmsg_channels, sa->vproc_id);
+	if (!rpdev) {
+		err = -EINVAL;
+		goto out;
+	}
+
+	rpsk->rproc_id = sa->vproc_id;
+	rpsk->rpdev = rpdev;
+
+	/* bind this socket with its parent rpmsg device */
+	list_add_tail(&rpsk->elem, rpdev->ept->priv);
+
+	/* XXX take care of disconnection state too */
+	sk->sk_state = RPMSG_CONNECTED;
+
+out:
+	release_sock(sk);
+	mutex_unlock(&rpmsg_channels_lock);
+	return err;
+}
+
+static int rpmsg_sock_sendmsg(struct socket *sock, struct msghdr *msg,
+			      size_t len)
+{
+	struct sock *sk = sock->sk;
+	struct rpmsg_socket *rpsk;
+	char payload[RPMSG_BUF_SIZE];/* todo: sane payload length methodology */
+	int err;
+
+	/* XXX check for sock_error as well ? */
+	/* XXX handle noblock ? */
+	if (msg->msg_flags & MSG_OOB)
+		return -EOPNOTSUPP;
+
+	/* no payload ? */
+	if (!msg->msg_iter.iov->iov_base)
+		return -EINVAL;
+
+	/* make sure the length is valid for copying into kernel buffer */
+	if (len > RPMSG_BUF_SIZE - sizeof(struct rpmsg_hdr))
+		return -EMSGSIZE;
+
+	lock_sock(sk);
+
+	/* we don't support Tx on errored-out sockets */
+	if (sk->sk_state == RPMSG_ERROR) {
+		release_sock(sk);
+		return -ESHUTDOWN;
+	}
+
+	/* we don't support loopback at this point */
+	if (sk->sk_state != RPMSG_CONNECTED) {
+		release_sock(sk);
+		return -ENOTCONN;
+	}
+
+	rpsk = container_of(sk, struct rpmsg_socket, sk);
+
+	/* XXX for now, ignore the peer address. later use it
+	 * with rpmsg_sendto, but only if user is root
+	 */
+	err = memcpy_from_msg(payload, msg, len);
+	if (err)
+		goto out;
+
+	err = rpmsg_send(rpsk->rpdev->ept, payload, len);
+	if (err)
+		pr_err("rpmsg_send failed: %d\n", err);
+	else
+		err = len;
+
+out:
+	release_sock(sk);
+	return err;
+}
+
+static int rpmsg_sock_recvmsg(struct socket *sock, struct msghdr *msg,
+			      size_t len, int flags)
+{
+	struct sock *sk = sock->sk;
+	struct sockaddr_rpmsg *sa;
+	struct sk_buff *skb;
+	int noblock = flags & MSG_DONTWAIT;
+	int ret;
+
+	if (flags & MSG_OOB) {
+		pr_err("MSG_OOB: %d\n", EOPNOTSUPP);
+		return -EOPNOTSUPP;
+	}
+
+	/* return failure on errored-out Rx sockets */
+	lock_sock(sk);
+	if (sk->sk_state == RPMSG_ERROR) {
+		release_sock(sk);
+		return -ENOLINK;
+	}
+	release_sock(sk);
+
+	msg->msg_namelen = 0;
+
+	skb = skb_recv_datagram(sk, flags, noblock, &ret);
+	if (!skb) {
+		/* check for shutdown ? */
+		pr_err("skb_recv_datagram: %d\n", ret);
+		return ret;
+	}
+
+	if (msg->msg_name) {
+		msg->msg_namelen = sizeof(*sa);
+		sa = (struct sockaddr_rpmsg *)msg->msg_name;
+		sa->vproc_id = RPMSG_CB(skb).vproc_id;
+		sa->addr = RPMSG_CB(skb).addr;
+		sa->family = AF_RPMSG;
+	}
+
+	if (len > skb->len) {
+		len = skb->len;
+	} else if (len < skb->len) {
+		pr_warn("user buffer is too small\n");
+		/* XXX truncate or error ? */
+		msg->msg_flags |= MSG_TRUNC;
+	}
+
+	ret = skb_copy_datagram_msg(skb, 0, msg, len);
+	if (ret) {
+		pr_err("error copying skb data: %d\n", ret);
+		goto out_free;
+	}
+
+	ret = len;
+
+out_free:
+	skb_free_datagram(sk, skb);
+	return ret;
+}
+
+static __poll_t rpmsg_sock_poll(struct file *file, struct socket *sock,
+				poll_table *wait)
+{
+	struct sock *sk = sock->sk;
+	__poll_t mask = 0;
+
+	poll_wait(file, sk_sleep(sk), wait);
+
+	/* exceptional events? */
+	if (sk->sk_err || !skb_queue_empty(&sk->sk_error_queue))
+		mask |= EPOLLERR;
+	if (sk->sk_state == RPMSG_ERROR)
+		mask |= EPOLLERR;
+	if (sk->sk_shutdown & RCV_SHUTDOWN)
+		mask |= EPOLLRDHUP;
+	if (sk->sk_shutdown == SHUTDOWN_MASK)
+		mask |= EPOLLHUP;
+
+	/* readable? */
+	if (!skb_queue_empty(&sk->sk_receive_queue) ||
+	    (sk->sk_shutdown & RCV_SHUTDOWN))
+		mask |= EPOLLIN | EPOLLRDNORM;
+
+	if (sk->sk_state == RPMSG_CLOSED)
+		mask |= EPOLLHUP;
+
+	/* XXX is writable ?
+	 * this depends on the destination processor.
+	 * if loopback: we're writable unless no memory
+	 * if to remote: we need enabled rpmsg buffer or user supplied bufs
+	 * for now, let's always be writable.
+	 */
+	mask |= EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND;
+
+	return mask;
+}
+
+/* return bound socket address information, either local or remote */
+static int rpmsg_sock_getname(struct socket *sock, struct sockaddr *addr,
+			      int peer)
+{
+	struct sock *sk = sock->sk;
+	struct rpmsg_socket *rpsk;
+	struct rpmsg_device *rpdev;
+	struct sockaddr_rpmsg *sa;
+	int ret;
+
+	rpsk = container_of(sk, struct rpmsg_socket, sk);
+
+	lock_sock(sk);
+	rpdev = rpsk->rpdev;
+	if (!rpdev) {
+		ret = peer ? -ENOTCONN : -EINVAL;
+		goto out;
+	}
+
+	addr->sa_family = AF_RPMSG;
+	sa = (struct sockaddr_rpmsg *)addr;
+	ret = sizeof(*sa);
+
+	if (peer) {
+		sa->vproc_id = rpsk->rproc_id;
+		sa->addr = rpdev->dst;
+	} else {
+		sa->vproc_id = RPMSG_LOCALHOST;
+		sa->addr = rpsk->endpt ? rpsk->endpt->addr : rpsk->rpdev->src;
+	}
+
+out:
+	release_sock(sk);
+	return ret;
+}
+
+static int rpmsg_sock_release(struct socket *sock)
+{
+	struct sock *sk = sock->sk;
+	struct rpmsg_socket *rpsk = container_of(sk, struct rpmsg_socket, sk);
+	struct rpmsg_endpoint *endpt;
+
+	if (!sk)
+		return 0;
+
+	if (sk->sk_state == RPMSG_OPEN)
+		goto out;
+
+	lock_sock(sk);
+	if (sk->sk_state != RPMSG_ERROR) {
+		rpsk->rpdev = NULL;
+		list_del(&rpsk->elem);
+		endpt = rpsk->endpt;
+		rpsk->endpt = NULL;
+		release_sock(sk);
+		if (endpt)
+			rpmsg_destroy_ept(endpt);
+		goto out;
+	}
+	release_sock(sk);
+
+out:
+	sock_put(sock->sk);
+	return 0;
+}
+
+/* Notes:
+ * - calling connect after bind isn't currently supported (is it even needed?).
+ * - userspace arguments to bind aren't intuitive: one needs to provide
+ *   the vproc id of the remote processor that the channel needs to be shared
+ *   with, and the -local- source address the channel is to be bound with
+ */
+static int
+rpmsg_sock_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
+{
+	struct sock *sk = sock->sk;
+	struct rpmsg_socket *rpsk = container_of(sk, struct rpmsg_socket, sk);
+	struct rpmsg_device *rpdev;
+	struct rpmsg_endpoint *endpt;
+	struct rpmsg_channel_info chinfo = {};
+	struct sockaddr_rpmsg *sa = (struct sockaddr_rpmsg *)uaddr;
+	int ret = 0;
+
+	if (sock->state == SS_CONNECTED)
+		return -EINVAL;
+
+	if (addr_len != sizeof(*sa))
+		return -EINVAL;
+
+	if (sa->family != AF_RPMSG)
+		return -EINVAL;
+
+	if (rpsk->endpt)
+		return -EBUSY;
+
+	if (sk->sk_state != RPMSG_OPEN)
+		return -EINVAL;
+
+	mutex_lock(&rpmsg_channels_lock);
+
+	rpdev = radix_tree_lookup(&rpmsg_channels, sa->vproc_id);
+	if (!rpdev) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	/* bind this socket with a receiving endpoint */
+	chinfo.src = sa->addr;
+	chinfo.dst = RPMSG_ADDR_ANY;
+	endpt = rpmsg_create_ept(rpdev, rpmsg_sock_cb, sk, chinfo);
+	if (!endpt) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	lock_sock(sk);
+	rpsk->rpdev = rpdev;
+	rpsk->endpt = endpt;
+	rpsk->rproc_id = sa->vproc_id;
+
+	/* bind this socket with its parent rpmsg device */
+	list_add_tail(&rpsk->elem, rpdev->ept->priv);
+
+	sk->sk_state = RPMSG_LISTENING;
+	release_sock(sk);
+
+out:
+	mutex_unlock(&rpmsg_channels_lock);
+	return ret;
+}
+
+static const struct proto_ops rpmsg_sock_ops = {
+	.family		= PF_RPMSG,
+	.owner		= THIS_MODULE,
+
+	.release	= rpmsg_sock_release,
+	.connect	= rpmsg_sock_connect,
+	.getname	= rpmsg_sock_getname,
+	.sendmsg	= rpmsg_sock_sendmsg,
+	.recvmsg	= rpmsg_sock_recvmsg,
+	.poll		= rpmsg_sock_poll,
+	.bind		= rpmsg_sock_bind,
+
+	.listen		= sock_no_listen,
+	.accept		= sock_no_accept,
+	.ioctl		= sock_no_ioctl,
+	.mmap		= sock_no_mmap,
+	.socketpair	= sock_no_socketpair,
+	.shutdown	= sock_no_shutdown,
+	.setsockopt	= sock_no_setsockopt,
+	.getsockopt	= sock_no_getsockopt
+};
+
+static void rpmsg_sock_destruct(struct sock *sk)
+{
+}
+
+static int rpmsg_sock_create(struct net *net, struct socket *sock, int proto,
+			     int kern)
+{
+	struct sock *sk;
+	struct rpmsg_socket *rpsk;
+
+	if (sock->type != SOCK_SEQPACKET)
+		return -ESOCKTNOSUPPORT;
+	if (proto != 0)
+		return -EPROTONOSUPPORT;
+
+	sk = sk_alloc(net, PF_RPMSG, GFP_KERNEL, &rpmsg_proto, kern);
+	if (!sk)
+		return -ENOMEM;
+
+	sock->state = SS_UNCONNECTED;
+	sock->ops = &rpmsg_sock_ops;
+	sock_init_data(sock, sk);
+
+	sk->sk_destruct = rpmsg_sock_destruct;
+	sk->sk_protocol = proto;
+
+	sk->sk_state = RPMSG_OPEN;
+
+	rpsk = container_of(sk, struct rpmsg_socket, sk);
+	INIT_LIST_HEAD(&rpsk->elem);
+	/* use RPMSG_LOCALHOST to serve as an invalid value */
+	rpsk->rproc_id = RPMSG_LOCALHOST;
+
+	return 0;
+}
+
+static const struct net_proto_family rpmsg_proto_family = {
+	.family = PF_RPMSG,
+	.create	= rpmsg_sock_create,
+	.owner = THIS_MODULE,
+};
+
+static int __rpmsg_sock_cb(struct device *dev, int from_vproc_id, void *data,
+			   int len, struct sock *sk, u32 src)
+{
+	struct rpmsg_socket *rpsk = container_of(sk, struct rpmsg_socket, sk);
+	struct sk_buff *skb;
+	int ret;
+
+#if defined(CONFIG_DYNAMIC_DEBUG)
+	dynamic_hex_dump("rpmsg_proto Rx: ", DUMP_PREFIX_NONE, 16, 1, data,
+			 len, true);
+#endif
+
+	lock_sock(sk);
+
+	switch (sk->sk_state) {
+	case RPMSG_CONNECTED:
+		if (rpsk->rpdev->dst != src)
+			dev_warn(dev, "unexpected source address: %d\n", src);
+		break;
+	case RPMSG_LISTENING:
+		/* When an inbound message is received while we're listening,
+		 * we implicitly become connected
+		 */
+		sk->sk_state = RPMSG_CONNECTED;
+		rpsk->rpdev->dst = src;
+		break;
+	default:
+		dev_warn(dev, "unexpected inbound message (from %d)\n", src);
+		break;
+	}
+
+	skb = sock_alloc_send_skb(sk, len, 1, &ret);
+	if (!skb) {
+		dev_err(dev, "sock_alloc_send_skb failed: %d\n", ret);
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	RPMSG_CB(skb).vproc_id = from_vproc_id;
+	RPMSG_CB(skb).addr = src;
+	RPMSG_CB(skb).family = AF_RPMSG;
+
+	memcpy(skb_put(skb, len), data, len);
+
+	ret = sock_queue_rcv_skb(sk, skb);
+	if (ret) {
+		dev_err(dev, "sock_queue_rcv_skb failed: %d\n", ret);
+		kfree_skb(skb);
+	}
+
+out:
+	release_sock(sk);
+	return ret;
+}
+
+static int rpmsg_sock_cb(struct rpmsg_device *rpdev, void *data, int len,
+			 void *priv, u32 src)
+{
+	int id = rpmsg_sock_get_proc_id(rpdev);
+
+	return __rpmsg_sock_cb(&rpdev->dev, id, data, len, priv, src);
+}
+
+static int rpmsg_proto_cb(struct rpmsg_device *rpdev, void *data, int len,
+			  void *priv, u32 src)
+{
+	dev_err(&rpdev->dev, "rpmsg_proto device not designed to receive any messages\n");
+	return 0;
+}
+
+static int rpmsg_proto_probe(struct rpmsg_device *rpdev)
+{
+	struct device *dev = &rpdev->dev;
+	int ret, dst = rpdev->dst, id;
+	struct rpmsg_device *vrp_dev;
+	struct list_head *sock_list = NULL;
+
+	if (WARN_ON(dst == RPMSG_ADDR_ANY))
+		return -EINVAL;
+
+	id = rpmsg_sock_get_proc_id(rpdev);
+	if (id < 0)
+		return -EINVAL;
+
+	mutex_lock(&rpmsg_channels_lock);
+
+	/* are we exposing a rpmsg proto device for this remote processor yet?
+	 * If not, associate id/device for later lookup in rpmsg_sock_bind().
+	 * Multiple devices per remote processor are not supported.
+	 */
+	vrp_dev = radix_tree_lookup(&rpmsg_channels, id);
+	if (!vrp_dev) {
+		ret = radix_tree_insert(&rpmsg_channels, id, rpdev);
+		if (ret) {
+			dev_err(dev, "radix_tree_insert failed: %d\n", ret);
+			goto out;
+		}
+	} else {
+		ret = -ENODEV;
+		dev_err(dev, "multiple rpmsg-proto devices from the same rproc is not supported.\n");
+		goto out;
+	}
+
+	/* reuse the rpdev endpoint's private field for storing the list of
+	 * all connected and bound sockets on this rpmsg device.
+	 */
+	WARN_ON(!!rpdev->ept->priv);
+	sock_list = kzalloc(sizeof(*sock_list), GFP_KERNEL);
+	if (!sock_list) {
+		dev_err(dev, "failed to allocate list_head\n");
+		radix_tree_delete(&rpmsg_channels, id);
+		ret = -ENOMEM;
+		goto out;
+	}
+	INIT_LIST_HEAD(sock_list);
+	rpdev->ept->priv = sock_list;
+
+out:
+	mutex_unlock(&rpmsg_channels_lock);
+
+	return ret;
+}
+
+static void rpmsg_proto_remove(struct rpmsg_device *rpdev)
+{
+	struct device *dev = &rpdev->dev;
+	int id, dst = rpdev->dst;
+	struct rpmsg_device *vrp_dev;
+	struct list_head *sk_list;
+	struct rpmsg_socket *rpsk, *tmp;
+	struct rpmsg_endpoint *endpt = NULL;
+
+	if (dst == RPMSG_ADDR_ANY)
+		return;
+
+	id = rpmsg_sock_get_proc_id(rpdev);
+
+	mutex_lock(&rpmsg_channels_lock);
+
+	vrp_dev = radix_tree_lookup(&rpmsg_channels, id);
+	if (!vrp_dev) {
+		dev_err(dev, "can't find rpmsg device for rproc %d\n", id);
+		goto out;
+	}
+	if (vrp_dev != rpdev)
+		dev_err(dev, "can't match the stored rpdev for rproc %d\n", id);
+
+	if (!radix_tree_delete(&rpmsg_channels, id))
+		dev_err(dev, "failed to delete rpdev for rproc %d\n", id);
+
+	/* mark all associated sockets invalid and remove them from the
+	 * rpdev's list. Destroy the endpoints for bound sockets as the
+	 * parent rpdev will not exist until the socket's release()
+	 */
+	sk_list = rpdev->ept->priv;
+	list_for_each_entry_safe(rpsk, tmp, sk_list, elem) {
+		lock_sock(&rpsk->sk);
+		if (rpsk->rpdev) {
+			rpsk->rpdev = NULL;
+			rpsk->sk.sk_state = RPMSG_ERROR;
+			list_del(&rpsk->elem);
+			endpt = rpsk->endpt;
+			rpsk->endpt = NULL;
+		}
+		release_sock(&rpsk->sk);
+		if (endpt) {
+			rpmsg_destroy_ept(endpt);
+			rpsk->sk.sk_error_report(&rpsk->sk);
+		}
+	}
+	kfree(sk_list);
+	rpdev->ept->priv = NULL;
+
+out:
+	mutex_unlock(&rpmsg_channels_lock);
+}
+
+static struct rpmsg_device_id rpmsg_proto_id_table[] = {
+	{ .name	= "rpmsg-proto" },
+	{ },
+};
+MODULE_DEVICE_TABLE(rpmsg, rpmsg_proto_id_table);
+
+static struct rpmsg_driver rpmsg_proto_driver = {
+	.drv.name	= KBUILD_MODNAME,
+	.id_table	= rpmsg_proto_id_table,
+	.probe		= rpmsg_proto_probe,
+	.callback	= rpmsg_proto_cb,
+	.remove		= rpmsg_proto_remove,
+};
+
+static int __init rpmsg_proto_init(void)
+{
+	int ret;
+
+	ret = proto_register(&rpmsg_proto, 0);
+	if (ret) {
+		pr_err("proto_register failed: %d\n", ret);
+		return ret;
+	}
+
+	ret = sock_register(&rpmsg_proto_family);
+	if (ret) {
+		pr_err("sock_register failed: %d\n", ret);
+		goto proto_unreg;
+	}
+
+	ret = register_rpmsg_driver(&rpmsg_proto_driver);
+	if (ret) {
+		pr_err("register_rpmsg_driver failed: %d\n", ret);
+		goto sock_unreg;
+	}
+
+	return 0;
+
+sock_unreg:
+	sock_unregister(PF_RPMSG);
+proto_unreg:
+	proto_unregister(&rpmsg_proto);
+	return ret;
+}
+
+static void __exit rpmsg_proto_exit(void)
+{
+	unregister_rpmsg_driver(&rpmsg_proto_driver);
+	sock_unregister(PF_RPMSG);
+	proto_unregister(&rpmsg_proto);
+}
+
+module_init(rpmsg_proto_init);
+module_exit(rpmsg_proto_exit);
+
+MODULE_DESCRIPTION("Remote processor messaging protocol");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("rpmsg:rpmsg-proto");
+MODULE_ALIAS_NETPROTO(AF_RPMSG);
diff -Naur --no-dereference a/net/wireguard/allowedips.c b/net/wireguard/allowedips.c
--- a/net/wireguard/allowedips.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/allowedips.c	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,381 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#include "allowedips.h"
+#include "peer.h"
+
+static void swap_endian(u8 *dst, const u8 *src, u8 bits)
+{
+	if (bits == 32) {
+		*(u32 *)dst = be32_to_cpu(*(const __be32 *)src);
+	} else if (bits == 128) {
+		((u64 *)dst)[0] = be64_to_cpu(((const __be64 *)src)[0]);
+		((u64 *)dst)[1] = be64_to_cpu(((const __be64 *)src)[1]);
+	}
+}
+
+static void copy_and_assign_cidr(struct allowedips_node *node, const u8 *src,
+				 u8 cidr, u8 bits)
+{
+	node->cidr = cidr;
+	node->bit_at_a = cidr / 8U;
+#ifdef __LITTLE_ENDIAN
+	node->bit_at_a ^= (bits / 8U - 1U) % 8U;
+#endif
+	node->bit_at_b = 7U - (cidr % 8U);
+	node->bitlen = bits;
+	memcpy(node->bits, src, bits / 8U);
+}
+#define CHOOSE_NODE(parent, key) \
+	parent->bit[(key[parent->bit_at_a] >> parent->bit_at_b) & 1]
+
+static void node_free_rcu(struct rcu_head *rcu)
+{
+	kfree(container_of(rcu, struct allowedips_node, rcu));
+}
+
+static void push_rcu(struct allowedips_node **stack,
+		     struct allowedips_node __rcu *p, unsigned int *len)
+{
+	if (rcu_access_pointer(p)) {
+		WARN_ON(IS_ENABLED(DEBUG) && *len >= 128);
+		stack[(*len)++] = rcu_dereference_raw(p);
+	}
+}
+
+static void root_free_rcu(struct rcu_head *rcu)
+{
+	struct allowedips_node *node, *stack[128] = {
+		container_of(rcu, struct allowedips_node, rcu) };
+	unsigned int len = 1;
+
+	while (len > 0 && (node = stack[--len])) {
+		push_rcu(stack, node->bit[0], &len);
+		push_rcu(stack, node->bit[1], &len);
+		kfree(node);
+	}
+}
+
+static void root_remove_peer_lists(struct allowedips_node *root)
+{
+	struct allowedips_node *node, *stack[128] = { root };
+	unsigned int len = 1;
+
+	while (len > 0 && (node = stack[--len])) {
+		push_rcu(stack, node->bit[0], &len);
+		push_rcu(stack, node->bit[1], &len);
+		if (rcu_access_pointer(node->peer))
+			list_del(&node->peer_list);
+	}
+}
+
+static void walk_remove_by_peer(struct allowedips_node __rcu **top,
+				struct wg_peer *peer, struct mutex *lock)
+{
+#define REF(p) rcu_access_pointer(p)
+#define DEREF(p) rcu_dereference_protected(*(p), lockdep_is_held(lock))
+#define PUSH(p) ({                                                             \
+		WARN_ON(IS_ENABLED(DEBUG) && len >= 128);                      \
+		stack[len++] = p;                                              \
+	})
+
+	struct allowedips_node __rcu **stack[128], **nptr;
+	struct allowedips_node *node, *prev;
+	unsigned int len;
+
+	if (unlikely(!peer || !REF(*top)))
+		return;
+
+	for (prev = NULL, len = 0, PUSH(top); len > 0; prev = node) {
+		nptr = stack[len - 1];
+		node = DEREF(nptr);
+		if (!node) {
+			--len;
+			continue;
+		}
+		if (!prev || REF(prev->bit[0]) == node ||
+		    REF(prev->bit[1]) == node) {
+			if (REF(node->bit[0]))
+				PUSH(&node->bit[0]);
+			else if (REF(node->bit[1]))
+				PUSH(&node->bit[1]);
+		} else if (REF(node->bit[0]) == prev) {
+			if (REF(node->bit[1]))
+				PUSH(&node->bit[1]);
+		} else {
+			if (rcu_dereference_protected(node->peer,
+				lockdep_is_held(lock)) == peer) {
+				RCU_INIT_POINTER(node->peer, NULL);
+				list_del_init(&node->peer_list);
+				if (!node->bit[0] || !node->bit[1]) {
+					rcu_assign_pointer(*nptr, DEREF(
+					       &node->bit[!REF(node->bit[0])]));
+					call_rcu(&node->rcu, node_free_rcu);
+					node = DEREF(nptr);
+				}
+			}
+			--len;
+		}
+	}
+
+#undef REF
+#undef DEREF
+#undef PUSH
+}
+
+static unsigned int fls128(u64 a, u64 b)
+{
+	return a ? fls64(a) + 64U : fls64(b);
+}
+
+static u8 common_bits(const struct allowedips_node *node, const u8 *key,
+		      u8 bits)
+{
+	if (bits == 32)
+		return 32U - fls(*(const u32 *)node->bits ^ *(const u32 *)key);
+	else if (bits == 128)
+		return 128U - fls128(
+			*(const u64 *)&node->bits[0] ^ *(const u64 *)&key[0],
+			*(const u64 *)&node->bits[8] ^ *(const u64 *)&key[8]);
+	return 0;
+}
+
+static bool prefix_matches(const struct allowedips_node *node, const u8 *key,
+			   u8 bits)
+{
+	/* This could be much faster if it actually just compared the common
+	 * bits properly, by precomputing a mask bswap(~0 << (32 - cidr)), and
+	 * the rest, but it turns out that common_bits is already super fast on
+	 * modern processors, even taking into account the unfortunate bswap.
+	 * So, we just inline it like this instead.
+	 */
+	return common_bits(node, key, bits) >= node->cidr;
+}
+
+static struct allowedips_node *find_node(struct allowedips_node *trie, u8 bits,
+					 const u8 *key)
+{
+	struct allowedips_node *node = trie, *found = NULL;
+
+	while (node && prefix_matches(node, key, bits)) {
+		if (rcu_access_pointer(node->peer))
+			found = node;
+		if (node->cidr == bits)
+			break;
+		node = rcu_dereference_bh(CHOOSE_NODE(node, key));
+	}
+	return found;
+}
+
+/* Returns a strong reference to a peer */
+static struct wg_peer *lookup(struct allowedips_node __rcu *root, u8 bits,
+			      const void *be_ip)
+{
+	/* Aligned so it can be passed to fls/fls64 */
+	u8 ip[16] __aligned(__alignof(u64));
+	struct allowedips_node *node;
+	struct wg_peer *peer = NULL;
+
+	swap_endian(ip, be_ip, bits);
+
+	rcu_read_lock_bh();
+retry:
+	node = find_node(rcu_dereference_bh(root), bits, ip);
+	if (node) {
+		peer = wg_peer_get_maybe_zero(rcu_dereference_bh(node->peer));
+		if (!peer)
+			goto retry;
+	}
+	rcu_read_unlock_bh();
+	return peer;
+}
+
+static bool node_placement(struct allowedips_node __rcu *trie, const u8 *key,
+			   u8 cidr, u8 bits, struct allowedips_node **rnode,
+			   struct mutex *lock)
+{
+	struct allowedips_node *node = rcu_dereference_protected(trie,
+						lockdep_is_held(lock));
+	struct allowedips_node *parent = NULL;
+	bool exact = false;
+
+	while (node && node->cidr <= cidr && prefix_matches(node, key, bits)) {
+		parent = node;
+		if (parent->cidr == cidr) {
+			exact = true;
+			break;
+		}
+		node = rcu_dereference_protected(CHOOSE_NODE(parent, key),
+						 lockdep_is_held(lock));
+	}
+	*rnode = parent;
+	return exact;
+}
+
+static int add(struct allowedips_node __rcu **trie, u8 bits, const u8 *key,
+	       u8 cidr, struct wg_peer *peer, struct mutex *lock)
+{
+	struct allowedips_node *node, *parent, *down, *newnode;
+
+	if (unlikely(cidr > bits || !peer))
+		return -EINVAL;
+
+	if (!rcu_access_pointer(*trie)) {
+		node = kzalloc(sizeof(*node), GFP_KERNEL);
+		if (unlikely(!node))
+			return -ENOMEM;
+		RCU_INIT_POINTER(node->peer, peer);
+		list_add_tail(&node->peer_list, &peer->allowedips_list);
+		copy_and_assign_cidr(node, key, cidr, bits);
+		rcu_assign_pointer(*trie, node);
+		return 0;
+	}
+	if (node_placement(*trie, key, cidr, bits, &node, lock)) {
+		rcu_assign_pointer(node->peer, peer);
+		list_move_tail(&node->peer_list, &peer->allowedips_list);
+		return 0;
+	}
+
+	newnode = kzalloc(sizeof(*newnode), GFP_KERNEL);
+	if (unlikely(!newnode))
+		return -ENOMEM;
+	RCU_INIT_POINTER(newnode->peer, peer);
+	list_add_tail(&newnode->peer_list, &peer->allowedips_list);
+	copy_and_assign_cidr(newnode, key, cidr, bits);
+
+	if (!node) {
+		down = rcu_dereference_protected(*trie, lockdep_is_held(lock));
+	} else {
+		down = rcu_dereference_protected(CHOOSE_NODE(node, key),
+						 lockdep_is_held(lock));
+		if (!down) {
+			rcu_assign_pointer(CHOOSE_NODE(node, key), newnode);
+			return 0;
+		}
+	}
+	cidr = min(cidr, common_bits(down, key, bits));
+	parent = node;
+
+	if (newnode->cidr == cidr) {
+		rcu_assign_pointer(CHOOSE_NODE(newnode, down->bits), down);
+		if (!parent)
+			rcu_assign_pointer(*trie, newnode);
+		else
+			rcu_assign_pointer(CHOOSE_NODE(parent, newnode->bits),
+					   newnode);
+	} else {
+		node = kzalloc(sizeof(*node), GFP_KERNEL);
+		if (unlikely(!node)) {
+			kfree(newnode);
+			return -ENOMEM;
+		}
+		INIT_LIST_HEAD(&node->peer_list);
+		copy_and_assign_cidr(node, newnode->bits, cidr, bits);
+
+		rcu_assign_pointer(CHOOSE_NODE(node, down->bits), down);
+		rcu_assign_pointer(CHOOSE_NODE(node, newnode->bits), newnode);
+		if (!parent)
+			rcu_assign_pointer(*trie, node);
+		else
+			rcu_assign_pointer(CHOOSE_NODE(parent, node->bits),
+					   node);
+	}
+	return 0;
+}
+
+void wg_allowedips_init(struct allowedips *table)
+{
+	table->root4 = table->root6 = NULL;
+	table->seq = 1;
+}
+
+void wg_allowedips_free(struct allowedips *table, struct mutex *lock)
+{
+	struct allowedips_node __rcu *old4 = table->root4, *old6 = table->root6;
+
+	++table->seq;
+	RCU_INIT_POINTER(table->root4, NULL);
+	RCU_INIT_POINTER(table->root6, NULL);
+	if (rcu_access_pointer(old4)) {
+		struct allowedips_node *node = rcu_dereference_protected(old4,
+							lockdep_is_held(lock));
+
+		root_remove_peer_lists(node);
+		call_rcu(&node->rcu, root_free_rcu);
+	}
+	if (rcu_access_pointer(old6)) {
+		struct allowedips_node *node = rcu_dereference_protected(old6,
+							lockdep_is_held(lock));
+
+		root_remove_peer_lists(node);
+		call_rcu(&node->rcu, root_free_rcu);
+	}
+}
+
+int wg_allowedips_insert_v4(struct allowedips *table, const struct in_addr *ip,
+			    u8 cidr, struct wg_peer *peer, struct mutex *lock)
+{
+	/* Aligned so it can be passed to fls */
+	u8 key[4] __aligned(__alignof(u32));
+
+	++table->seq;
+	swap_endian(key, (const u8 *)ip, 32);
+	return add(&table->root4, 32, key, cidr, peer, lock);
+}
+
+int wg_allowedips_insert_v6(struct allowedips *table, const struct in6_addr *ip,
+			    u8 cidr, struct wg_peer *peer, struct mutex *lock)
+{
+	/* Aligned so it can be passed to fls64 */
+	u8 key[16] __aligned(__alignof(u64));
+
+	++table->seq;
+	swap_endian(key, (const u8 *)ip, 128);
+	return add(&table->root6, 128, key, cidr, peer, lock);
+}
+
+void wg_allowedips_remove_by_peer(struct allowedips *table,
+				  struct wg_peer *peer, struct mutex *lock)
+{
+	++table->seq;
+	walk_remove_by_peer(&table->root4, peer, lock);
+	walk_remove_by_peer(&table->root6, peer, lock);
+}
+
+int wg_allowedips_read_node(struct allowedips_node *node, u8 ip[16], u8 *cidr)
+{
+	const unsigned int cidr_bytes = DIV_ROUND_UP(node->cidr, 8U);
+	swap_endian(ip, node->bits, node->bitlen);
+	memset(ip + cidr_bytes, 0, node->bitlen / 8U - cidr_bytes);
+	if (node->cidr)
+		ip[cidr_bytes - 1U] &= ~0U << (-node->cidr % 8U);
+
+	*cidr = node->cidr;
+	return node->bitlen == 32 ? AF_INET : AF_INET6;
+}
+
+/* Returns a strong reference to a peer */
+struct wg_peer *wg_allowedips_lookup_dst(struct allowedips *table,
+					 struct sk_buff *skb)
+{
+	if (skb->protocol == htons(ETH_P_IP))
+		return lookup(table->root4, 32, &ip_hdr(skb)->daddr);
+	else if (skb->protocol == htons(ETH_P_IPV6))
+		return lookup(table->root6, 128, &ipv6_hdr(skb)->daddr);
+	return NULL;
+}
+
+/* Returns a strong reference to a peer */
+struct wg_peer *wg_allowedips_lookup_src(struct allowedips *table,
+					 struct sk_buff *skb)
+{
+	if (skb->protocol == htons(ETH_P_IP))
+		return lookup(table->root4, 32, &ip_hdr(skb)->saddr);
+	else if (skb->protocol == htons(ETH_P_IPV6))
+		return lookup(table->root6, 128, &ipv6_hdr(skb)->saddr);
+	return NULL;
+}
+
+#include "selftest/allowedips.c"
diff -Naur --no-dereference a/net/wireguard/allowedips.h b/net/wireguard/allowedips.h
--- a/net/wireguard/allowedips.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/allowedips.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#ifndef _WG_ALLOWEDIPS_H
+#define _WG_ALLOWEDIPS_H
+
+#include <linux/mutex.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+
+struct wg_peer;
+
+struct allowedips_node {
+	struct wg_peer __rcu *peer;
+	struct allowedips_node __rcu *bit[2];
+	/* While it may seem scandalous that we waste space for v4,
+	 * we're alloc'ing to the nearest power of 2 anyway, so this
+	 * doesn't actually make a difference.
+	 */
+	u8 bits[16] __aligned(__alignof(u64));
+	u8 cidr, bit_at_a, bit_at_b, bitlen;
+
+	/* Keep rarely used list at bottom to be beyond cache line. */
+	union {
+		struct list_head peer_list;
+		struct rcu_head rcu;
+	};
+};
+
+struct allowedips {
+	struct allowedips_node __rcu *root4;
+	struct allowedips_node __rcu *root6;
+	u64 seq;
+};
+
+void wg_allowedips_init(struct allowedips *table);
+void wg_allowedips_free(struct allowedips *table, struct mutex *mutex);
+int wg_allowedips_insert_v4(struct allowedips *table, const struct in_addr *ip,
+			    u8 cidr, struct wg_peer *peer, struct mutex *lock);
+int wg_allowedips_insert_v6(struct allowedips *table, const struct in6_addr *ip,
+			    u8 cidr, struct wg_peer *peer, struct mutex *lock);
+void wg_allowedips_remove_by_peer(struct allowedips *table,
+				  struct wg_peer *peer, struct mutex *lock);
+/* The ip input pointer should be __aligned(__alignof(u64))) */
+int wg_allowedips_read_node(struct allowedips_node *node, u8 ip[16], u8 *cidr);
+
+/* These return a strong reference to a peer: */
+struct wg_peer *wg_allowedips_lookup_dst(struct allowedips *table,
+					 struct sk_buff *skb);
+struct wg_peer *wg_allowedips_lookup_src(struct allowedips *table,
+					 struct sk_buff *skb);
+
+#ifdef DEBUG
+bool wg_allowedips_selftest(void);
+#endif
+
+#endif /* _WG_ALLOWEDIPS_H */
diff -Naur --no-dereference a/net/wireguard/compat/checksum/checksum_partial_compat.h b/net/wireguard/compat/checksum/checksum_partial_compat.h
--- a/net/wireguard/compat/checksum/checksum_partial_compat.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/compat/checksum/checksum_partial_compat.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,208 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#include <net/route.h>
+#include <net/esp.h>
+#include <net/ip.h>
+#include <net/ipv6.h>
+#include <net/ip6_checksum.h>
+
+#define IP6_MF          0x0001
+#define IP6_OFFSET      0xFFF8
+static inline int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len, unsigned int max)
+{
+	if (skb_headlen(skb) >= len)
+		return 0;
+	if (max > skb->len)
+		max = skb->len;
+	if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL)
+		return -ENOMEM;
+	if (skb_headlen(skb) < len)
+		return -EPROTO;
+	return 0;
+}
+#define MAX_IP_HDR_LEN 128
+static inline int skb_checksum_setup_ip(struct sk_buff *skb, bool recalculate)
+{
+	unsigned int off;
+	bool fragment;
+	int err;
+	fragment = false;
+	err = skb_maybe_pull_tail(skb, sizeof(struct iphdr), MAX_IP_HDR_LEN);
+	if (err < 0)
+		goto out;
+	if (ip_hdr(skb)->frag_off & htons(IP_OFFSET | IP_MF))
+		fragment = true;
+	off = ip_hdrlen(skb);
+	err = -EPROTO;
+	if (fragment)
+		goto out;
+	switch (ip_hdr(skb)->protocol) {
+	case IPPROTO_TCP:
+		err = skb_maybe_pull_tail(skb,
+					  off + sizeof(struct tcphdr),
+					  MAX_IP_HDR_LEN);
+		if (err < 0)
+			goto out;
+
+		if (!skb_partial_csum_set(skb, off,
+					  offsetof(struct tcphdr, check))) {
+			err = -EPROTO;
+			goto out;
+		}
+
+		if (recalculate)
+			tcp_hdr(skb)->check =
+				~csum_tcpudp_magic(ip_hdr(skb)->saddr,
+						   ip_hdr(skb)->daddr,
+						   skb->len - off,
+						   IPPROTO_TCP, 0);
+		break;
+	case IPPROTO_UDP:
+		err = skb_maybe_pull_tail(skb,
+					  off + sizeof(struct udphdr),
+					  MAX_IP_HDR_LEN);
+		if (err < 0)
+			goto out;
+
+		if (!skb_partial_csum_set(skb, off,
+					  offsetof(struct udphdr, check))) {
+			err = -EPROTO;
+			goto out;
+		}
+
+		if (recalculate)
+			udp_hdr(skb)->check =
+				~csum_tcpudp_magic(ip_hdr(skb)->saddr,
+						   ip_hdr(skb)->daddr,
+						   skb->len - off,
+						   IPPROTO_UDP, 0);
+		break;
+	default:
+		goto out;
+	}
+	err = 0;
+out:
+	return err;
+}
+#define MAX_IPV6_HDR_LEN 256
+#define OPT_HDR(type, skb, off) \
+	(type *)(skb_network_header(skb) + (off))
+static inline int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate)
+{
+	int err;
+	u8 nexthdr;
+	unsigned int off;
+	unsigned int len;
+	bool fragment;
+	bool done;
+	fragment = false;
+	done = false;
+	off = sizeof(struct ipv6hdr);
+	err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN);
+	if (err < 0)
+		goto out;
+	nexthdr = ipv6_hdr(skb)->nexthdr;
+	len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len);
+	while (off <= len && !done) {
+		switch (nexthdr) {
+		case IPPROTO_DSTOPTS:
+		case IPPROTO_HOPOPTS:
+		case IPPROTO_ROUTING: {
+			struct ipv6_opt_hdr *hp;
+
+			err = skb_maybe_pull_tail(skb, off + sizeof(struct ipv6_opt_hdr), MAX_IPV6_HDR_LEN);
+			if (err < 0)
+				goto out;
+			hp = OPT_HDR(struct ipv6_opt_hdr, skb, off);
+			nexthdr = hp->nexthdr;
+			off += ipv6_optlen(hp);
+			break;
+		}
+		case IPPROTO_FRAGMENT: {
+			struct frag_hdr *hp;
+			err = skb_maybe_pull_tail(skb, off + sizeof(struct frag_hdr), MAX_IPV6_HDR_LEN);
+			if (err < 0)
+				goto out;
+			hp = OPT_HDR(struct frag_hdr, skb, off);
+			if (hp->frag_off & htons(IP6_OFFSET | IP6_MF))
+				fragment = true;
+			nexthdr = hp->nexthdr;
+			off += sizeof(struct frag_hdr);
+			break;
+		}
+		default:
+			done = true;
+			break;
+		}
+	}
+	err = -EPROTO;
+	if (!done || fragment)
+		goto out;
+	switch (nexthdr) {
+		case IPPROTO_TCP:
+			err = skb_maybe_pull_tail(skb,
+						  off + sizeof(struct tcphdr),
+						  MAX_IPV6_HDR_LEN);
+			if (err < 0)
+				goto out;
+
+			if (!skb_partial_csum_set(skb, off,
+						  offsetof(struct tcphdr, check))) {
+				err = -EPROTO;
+				goto out;
+			}
+
+			if (recalculate)
+				tcp_hdr(skb)->check =
+					~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
+							 &ipv6_hdr(skb)->daddr,
+							 skb->len - off,
+							 IPPROTO_TCP, 0);
+			break;
+		case IPPROTO_UDP:
+			err = skb_maybe_pull_tail(skb,
+						  off + sizeof(struct udphdr),
+						  MAX_IPV6_HDR_LEN);
+			if (err < 0)
+				goto out;
+
+			if (!skb_partial_csum_set(skb, off,
+						  offsetof(struct udphdr, check))) {
+				err = -EPROTO;
+				goto out;
+			}
+
+			if (recalculate)
+				udp_hdr(skb)->check =
+					~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
+							 &ipv6_hdr(skb)->daddr,
+							 skb->len - off,
+							 IPPROTO_UDP, 0);
+			break;
+		default:
+			goto out;
+	}
+	err = 0;
+out:
+	return err;
+}
+static inline int skb_checksum_setup(struct sk_buff *skb, bool recalculate)
+{
+	int err;
+	switch (skb->protocol) {
+	case htons(ETH_P_IP):
+		err = skb_checksum_setup_ip(skb, recalculate);
+		break;
+
+	case htons(ETH_P_IPV6):
+		err = skb_checksum_setup_ipv6(skb, recalculate);
+		break;
+	default:
+		err = -EPROTO;
+		break;
+	}
+	return err;
+}
diff -Naur --no-dereference a/net/wireguard/compat/compat-asm.h b/net/wireguard/compat/compat-asm.h
--- a/net/wireguard/compat/compat-asm.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/compat/compat-asm.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#ifndef _WG_COMPATASM_H
+#define _WG_COMPATASM_H
+
+#include <linux/linkage.h>
+#include <linux/kconfig.h>
+#include <linux/version.h>
+
+/* PaX compatibility */
+#if defined(RAP_PLUGIN)
+#undef ENTRY
+#define ENTRY RAP_ENTRY
+#endif
+
+#if defined(__LINUX_ARM_ARCH__) && LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
+	.irp	c,,eq,ne,cs,cc,mi,pl,vs,vc,hi,ls,ge,lt,gt,le,hs,lo
+	.macro	ret\c, reg
+#if __LINUX_ARM_ARCH__ < 6
+	mov\c	pc, \reg
+#else
+	.ifeqs	"\reg", "lr"
+	bx\c	\reg
+	.else
+	mov\c	pc, \reg
+	.endif
+#endif
+	.endm
+	.endr
+#endif
+
+#if defined(__LINUX_ARM_ARCH__) && LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0)
+#include <asm/assembler.h>
+#define lspush push
+#define lspull pull
+#undef push
+#undef pull
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 5, 0)
+#define SYM_FUNC_START ENTRY
+#define SYM_FUNC_END ENDPROC
+#endif
+
+#endif /* _WG_COMPATASM_H */
diff -Naur --no-dereference a/net/wireguard/compat/compat.h b/net/wireguard/compat/compat.h
--- a/net/wireguard/compat/compat.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/compat/compat.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,945 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#ifndef _WG_COMPAT_H
+#define _WG_COMPAT_H
+
+#include <linux/kconfig.h>
+#include <linux/version.h>
+#include <linux/types.h>
+#include <generated/utsrelease.h>
+
+#ifdef RHEL_MAJOR
+#if RHEL_MAJOR == 7
+#define ISRHEL7
+#elif RHEL_MAJOR == 8
+#define ISRHEL8
+#endif
+#endif
+#ifdef UTS_UBUNTU_RELEASE_ABI
+#if LINUX_VERSION_CODE == KERNEL_VERSION(3, 13, 11)
+#define ISUBUNTU1404
+#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0)
+#define ISUBUNTU1604
+#endif
+#endif
+#ifdef CONFIG_SUSE_KERNEL
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)
+#define ISOPENSUSE42
+#endif
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)
+#define ISOPENSUSE15
+#endif
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 0)
+#error "WireGuard requires Linux >= 3.10"
+#endif
+
+#if defined(ISRHEL7)
+#include <linux/skbuff.h>
+#define headers_end headers_start
+#elif LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 0)
+#define headers_start data
+#define headers_end data
+#endif
+
+#include <linux/cache.h>
+#include <linux/init.h>
+#ifndef __ro_after_init
+#define __ro_after_init __read_mostly
+#endif
+
+#include <linux/compiler.h>
+#ifndef READ_ONCE
+#define READ_ONCE ACCESS_ONCE
+#endif
+#ifndef WRITE_ONCE
+#ifdef ACCESS_ONCE_RW
+#define WRITE_ONCE(p, v) (ACCESS_ONCE_RW(p) = (v))
+#else
+#define WRITE_ONCE(p, v) (ACCESS_ONCE(p) = (v))
+#endif
+#endif
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0)
+#include "udp_tunnel/udp_tunnel_partial_compat.h"
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) && !defined(DEBUG) && defined(net_dbg_ratelimited)
+#undef net_dbg_ratelimited
+#define net_dbg_ratelimited(fmt, ...) do { if (0) no_printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__); } while (0)
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0)
+#include <linux/rcupdate.h>
+#ifndef RCU_LOCKDEP_WARN
+#define RCU_LOCKDEP_WARN(cond, message) rcu_lockdep_assert(!(cond), message)
+#endif
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) && !defined(ISRHEL7)
+#define ipv6_dst_lookup(a, b, c, d) ipv6_dst_lookup(b, c, d)
+#endif
+
+#if (LINUX_VERSION_CODE == KERNEL_VERSION(4, 4, 0) || \
+    (LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 5) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0)) || \
+    (LINUX_VERSION_CODE < KERNEL_VERSION(4, 1, 17) && LINUX_VERSION_CODE > KERNEL_VERSION(3, 19, 0)) || \
+    (LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 27) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0)) || \
+    (LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 8) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 15, 0)) || \
+    (LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 40) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)) || \
+    (LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 54))) && !defined(ISUBUNTU1404) && (!defined(ISRHEL7) || RHEL_MINOR < 7) /* TODO: remove < 7 workaround once CentOS 7.7 comes out. */
+#include <linux/if.h>
+#include <net/ip_tunnels.h>
+#define IP6_ECN_set_ce(a, b) IP6_ECN_set_ce(b)
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) && IS_ENABLED(CONFIG_IPV6) && !defined(ISRHEL7)
+#include <net/ipv6.h>
+struct ipv6_stub_type {
+	void *udpv6_encap_enable;
+	int (*ipv6_dst_lookup)(struct sock *sk, struct dst_entry **dst, struct flowi6 *fl6);
+};
+static const struct ipv6_stub_type ipv6_stub_impl = {
+	.udpv6_encap_enable = (void *)1,
+	.ipv6_dst_lookup = ip6_dst_lookup
+};
+static const struct ipv6_stub_type *ipv6_stub = &ipv6_stub_impl;
+#endif
+
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0) && IS_ENABLED(CONFIG_IPV6) && !defined(ISOPENSUSE42) && !defined(ISRHEL7)
+#include <net/addrconf.h>
+static inline bool ipv6_mod_enabled(void)
+{
+	return ipv6_stub != NULL && ipv6_stub->udpv6_encap_enable != NULL;
+}
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) && !defined(ISRHEL7)
+#include <linux/skbuff.h>
+static inline void skb_reset_tc(struct sk_buff *skb)
+{
+#ifdef CONFIG_NET_CLS_ACT
+	skb->tc_verd = 0;
+#endif
+}
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)
+#include <linux/random.h>
+#include <linux/siphash.h>
+static inline u32 __compat_get_random_u32(void)
+{
+	static siphash_key_t key;
+	static u32 counter = 0;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0)
+	static bool has_seeded = false;
+	if (unlikely(!has_seeded)) {
+		get_random_bytes(&key, sizeof(key));
+		has_seeded = true;
+	}
+#else
+	get_random_once(&key, sizeof(key));
+#endif
+	return siphash_2u32(counter++, get_random_int(), &key);
+}
+#define get_random_u32 __compat_get_random_u32
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 0) && !defined(ISRHEL7)
+static inline void netif_keep_dst(struct net_device *dev)
+{
+	dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
+}
+#define COMPAT_CANNOT_USE_CSUM_LEVEL
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0) && !defined(ISRHEL7)
+#include <linux/netdevice.h>
+#ifndef netdev_alloc_pcpu_stats
+#define pcpu_sw_netstats pcpu_tstats
+#endif
+#ifndef netdev_alloc_pcpu_stats
+#define netdev_alloc_pcpu_stats alloc_percpu
+#endif
+#elif LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0) && !defined(ISRHEL7)
+#include <linux/netdevice.h>
+#ifndef netdev_alloc_pcpu_stats
+#define netdev_alloc_pcpu_stats(type)					\
+({									\
+	typeof(type) __percpu *pcpu_stats = alloc_percpu(type);		\
+	if (pcpu_stats)	{						\
+		int __cpu;						\
+		for_each_possible_cpu(__cpu) {				\
+			typeof(type) *stat;				\
+			stat = per_cpu_ptr(pcpu_stats, __cpu);		\
+			u64_stats_init(&stat->syncp);			\
+		}							\
+	}								\
+	pcpu_stats;							\
+})
+#endif
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0) && !defined(ISRHEL7)
+#include "checksum/checksum_partial_compat.h"
+static inline void *__compat_pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len)
+{
+	if (tail != skb) {
+		skb->data_len += len;
+		skb->len += len;
+	}
+	return skb_put(tail, len);
+}
+#define pskb_put __compat_pskb_put
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 11, 0) && !defined(ISRHEL7)
+#include <net/xfrm.h>
+static inline void skb_scrub_packet(struct sk_buff *skb, bool xnet)
+{
+#ifdef CONFIG_CAVIUM_OCTEON_IPFWD_OFFLOAD
+	memset(&skb->cvm_info, 0, sizeof(skb->cvm_info));
+	skb->cvm_reserved = 0;
+#endif
+	skb->tstamp.tv64 = 0;
+	skb->pkt_type = PACKET_HOST;
+	skb->skb_iif = 0;
+	skb_dst_drop(skb);
+	secpath_reset(skb);
+	nf_reset(skb);
+	nf_reset_trace(skb);
+	if (!xnet)
+		return;
+	skb_orphan(skb);
+	skb->mark = 0;
+}
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) || defined(ISUBUNTU1404)) && !defined(ISRHEL7)
+#include <linux/random.h>
+static inline u32 __compat_prandom_u32_max(u32 ep_ro)
+{
+	return (u32)(((u64)prandom_u32() * ep_ro) >> 32);
+}
+#define prandom_u32_max __compat_prandom_u32_max
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 75) && !defined(ISRHEL7)
+#ifndef U8_MAX
+#define U8_MAX ((u8)~0U)
+#endif
+#ifndef S8_MAX
+#define S8_MAX ((s8)(U8_MAX >> 1))
+#endif
+#ifndef S8_MIN
+#define S8_MIN ((s8)(-S8_MAX - 1))
+#endif
+#ifndef U16_MAX
+#define U16_MAX ((u16)~0U)
+#endif
+#ifndef S16_MAX
+#define S16_MAX ((s16)(U16_MAX >> 1))
+#endif
+#ifndef S16_MIN
+#define S16_MIN ((s16)(-S16_MAX - 1))
+#endif
+#ifndef U32_MAX
+#define U32_MAX ((u32)~0U)
+#endif
+#ifndef S32_MAX
+#define S32_MAX ((s32)(U32_MAX >> 1))
+#endif
+#ifndef S32_MIN
+#define S32_MIN ((s32)(-S32_MAX - 1))
+#endif
+#ifndef U64_MAX
+#define U64_MAX ((u64)~0ULL)
+#endif
+#ifndef S64_MAX
+#define S64_MAX ((s64)(U64_MAX >> 1))
+#endif
+#ifndef S64_MIN
+#define S64_MIN ((s64)(-S64_MAX - 1))
+#endif
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 60) && !defined(ISRHEL7)
+/* Making this static may very well invalidate its usefulness,
+ * but so it goes with compat code. */
+static inline void memzero_explicit(void *s, size_t count)
+{
+	memset(s, 0, count);
+	barrier();
+}
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) && !defined(ISRHEL7)
+static const struct in6_addr __compat_in6addr_any = IN6ADDR_ANY_INIT;
+#define in6addr_any __compat_in6addr_any
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0) && !defined(ISOPENSUSE15)
+#include <linux/completion.h>
+#include <linux/random.h>
+#include <linux/errno.h>
+struct rng_initializer {
+	struct completion done;
+	struct random_ready_callback cb;
+};
+static inline void rng_initialized_callback(struct random_ready_callback *cb)
+{
+	complete(&container_of(cb, struct rng_initializer, cb)->done);
+}
+static inline int wait_for_random_bytes(void)
+{
+	static bool rng_is_initialized = false;
+	int ret;
+	if (unlikely(!rng_is_initialized)) {
+		struct rng_initializer rng = {
+			.done = COMPLETION_INITIALIZER(rng.done),
+			.cb = { .owner = THIS_MODULE, .func = rng_initialized_callback }
+		};
+		ret = add_random_ready_callback(&rng.cb);
+		if (!ret) {
+			ret = wait_for_completion_interruptible(&rng.done);
+			if (ret) {
+				del_random_ready_callback(&rng.cb);
+				return ret;
+			}
+		} else if (ret != -EALREADY)
+			return ret;
+		rng_is_initialized = true;
+	}
+	return 0;
+}
+#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0)
+/* This is a disaster. Without this API, we really have no way of
+ * knowing if it's initialized. We just return that it has and hope
+ * for the best... */
+static inline int wait_for_random_bytes(void)
+{
+	return 0;
+}
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0) && !defined(ISRHEL8)
+#include <linux/random.h>
+#include <linux/slab.h>
+struct rng_is_initialized_callback {
+	struct random_ready_callback cb;
+	atomic_t *rng_state;
+};
+static inline void rng_is_initialized_callback(struct random_ready_callback *cb)
+{
+	struct rng_is_initialized_callback *rdy = container_of(cb, struct rng_is_initialized_callback, cb);
+	atomic_set(rdy->rng_state, 2);
+	kfree(rdy);
+}
+static inline bool rng_is_initialized(void)
+{
+	static atomic_t rng_state = ATOMIC_INIT(0);
+
+	if (atomic_read(&rng_state) == 2)
+		return true;
+
+	if (atomic_cmpxchg(&rng_state, 0, 1) == 0) {
+		int ret;
+		struct rng_is_initialized_callback *rdy = kmalloc(sizeof(*rdy), GFP_ATOMIC);
+		if (!rdy) {
+			atomic_set(&rng_state, 0);
+			return false;
+		}
+		rdy->cb.owner = THIS_MODULE;
+		rdy->cb.func = rng_is_initialized_callback;
+		rdy->rng_state = &rng_state;
+		ret = add_random_ready_callback(&rdy->cb);
+		if (ret)
+			kfree(rdy);
+		if (ret == -EALREADY) {
+			atomic_set(&rng_state, 2);
+			return true;
+		} else if (ret)
+			atomic_set(&rng_state, 0);
+		return false;
+	}
+	return false;
+}
+#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0)
+/* This is a disaster. Without this API, we really have no way of
+ * knowing if it's initialized. We just return that it has and hope
+ * for the best... */
+static inline bool rng_is_initialized(void)
+{
+	return true;
+}
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0) && !defined(ISOPENSUSE15)
+static inline int get_random_bytes_wait(void *buf, int nbytes)
+{
+	int ret = wait_for_random_bytes();
+	if (unlikely(ret))
+		return ret;
+	get_random_bytes(buf, nbytes);
+	return 0;
+}
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 11, 0) && !defined(ISRHEL7)
+#define system_power_efficient_wq system_unbound_wq
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 3, 0)
+#include <linux/ktime.h>
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
+#include <linux/hrtimer.h>
+#ifndef ktime_get_real_ts64
+#define timespec64 timespec
+#define ktime_get_real_ts64 ktime_get_real_ts
+#endif
+#else
+#include <linux/timekeeping.h>
+#endif
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)
+static inline u64 __compat_jiffies64_to_nsecs(u64 j)
+{
+#if !(NSEC_PER_SEC % HZ)
+	return (NSEC_PER_SEC / HZ) * j;
+#else
+	return div_u64(j * HZ_TO_USEC_NUM, HZ_TO_USEC_DEN) * 1000;
+#endif
+}
+#define jiffies64_to_nsecs __compat_jiffies64_to_nsecs
+#endif
+static inline u64 ktime_get_coarse_boottime_ns(void)
+{
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
+	return ktime_to_ns(ktime_get_boottime());
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 12) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0)) || LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 53)
+	return ktime_to_ns(ktime_mono_to_any(ns_to_ktime(jiffies64_to_nsecs(get_jiffies_64())), TK_OFFS_BOOT));
+#else
+	return ktime_to_ns(ktime_get_coarse_boottime());
+#endif
+}
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0)
+#include <linux/inetdevice.h>
+static inline __be32 __compat_confirm_addr_indev(struct in_device *in_dev, __be32 dst,  __be32 local, int scope)
+{
+	int same = 0;
+	__be32 addr = 0;
+	for_ifa(in_dev) {
+		if (!addr && (local == ifa->ifa_local || !local) && ifa->ifa_scope <= scope) {
+			addr = ifa->ifa_local;
+			if (same)
+				break;
+		}
+		if (!same) {
+			same = (!local || inet_ifa_match(local, ifa)) && (!dst || inet_ifa_match(dst, ifa));
+			if (same && addr) {
+				if (local || !dst)
+					break;
+				if (inet_ifa_match(addr, ifa))
+					break;
+				if (ifa->ifa_scope <= scope) {
+					addr = ifa->ifa_local;
+					break;
+				}
+				same = 0;
+			}
+		}
+	} endfor_ifa(in_dev);
+	return same ? addr : 0;
+}
+static inline __be32 __compat_inet_confirm_addr(struct net *net, struct in_device *in_dev, __be32 dst, __be32 local, int scope)
+{
+	__be32 addr = 0;
+	struct net_device *dev;
+	if (in_dev)
+		return __compat_confirm_addr_indev(in_dev, dst, local, scope);
+	rcu_read_lock();
+	for_each_netdev_rcu(net, dev) {
+		in_dev = __in_dev_get_rcu(dev);
+		if (in_dev) {
+			addr = __compat_confirm_addr_indev(in_dev, dst, local, scope);
+			if (addr)
+				break;
+		}
+	}
+	rcu_read_unlock();
+	return addr;
+}
+#define inet_confirm_addr __compat_inet_confirm_addr
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)
+#include <linux/vmalloc.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+static inline void *__compat_kvmalloc(size_t size, gfp_t flags)
+{
+	gfp_t kmalloc_flags = flags;
+	void *ret;
+	if (size > PAGE_SIZE) {
+		kmalloc_flags |= __GFP_NOWARN;
+		if (!(kmalloc_flags & __GFP_REPEAT) || (size <= PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER))
+			kmalloc_flags |= __GFP_NORETRY;
+	}
+	ret = kmalloc(size, kmalloc_flags);
+	if (ret || size <= PAGE_SIZE)
+		return ret;
+	return __vmalloc(size, flags, PAGE_KERNEL);
+}
+static inline void *__compat_kvzalloc(size_t size, gfp_t flags)
+{
+	return __compat_kvmalloc(size, flags | __GFP_ZERO);
+}
+#define kvmalloc __compat_kvmalloc
+#define kvzalloc __compat_kvzalloc
+#endif
+
+#if ((LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)) || LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 41)) && !defined(ISUBUNTU1404)
+#include <linux/vmalloc.h>
+#include <linux/mm.h>
+static inline void __compat_kvfree(const void *addr)
+{
+	if (is_vmalloc_addr(addr))
+		vfree(addr);
+	else
+		kfree(addr);
+}
+#define kvfree __compat_kvfree
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 9)
+#include <linux/netdevice.h>
+#define priv_destructor destructor
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0) && !defined(ISOPENSUSE15)
+#define wg_newlink(a,b,c,d,e) wg_newlink(a,b,c,d)
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)
+#include <net/netlink.h>
+#include <net/genetlink.h>
+#define nlmsg_parse(a, b, c, d, e, f) nlmsg_parse(a, b, c, d, e)
+#define nla_parse_nested(a, b, c, d, e) nla_parse_nested(a, b, c, d)
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) && !defined(ISRHEL7)
+static inline struct nlattr **genl_family_attrbuf(const struct genl_family *family)
+{
+	return family->attrbuf;
+}
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0)
+#define PTR_ERR_OR_ZERO(p) PTR_RET(p)
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0)
+#include <net/netlink.h>
+#define nla_put_u64_64bit(a, b, c, d) nla_put_u64(a, b, c)
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)
+#include <net/genetlink.h>
+#ifndef GENL_UNS_ADMIN_PERM
+#define GENL_UNS_ADMIN_PERM GENL_ADMIN_PERM
+#endif
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) && !defined(ISRHEL7)
+#include <net/genetlink.h>
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 13, 0)
+#define genl_register_family(a) genl_register_family_with_ops(a, genl_ops, ARRAY_SIZE(genl_ops))
+#define COMPAT_CANNOT_USE_CONST_GENL_OPS
+#else
+#define genl_register_family(a) genl_register_family_with_ops(a, genl_ops)
+#endif
+#define COMPAT_CANNOT_USE_GENL_NOPS
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 2) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)) || (LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 16) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)) || (LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 65) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0)) || (LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 101) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) || LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 84)
+#define __COMPAT_NETLINK_DUMP_BLOCK { \
+	int ret; \
+	skb->end -= nlmsg_total_size(sizeof(int)); \
+	ret = wg_get_device_dump_real(skb, cb); \
+	skb->end += nlmsg_total_size(sizeof(int)); \
+	return ret; \
+}
+#define __COMPAT_NETLINK_DUMP_OVERRIDE
+#else
+#define __COMPAT_NETLINK_DUMP_BLOCK return wg_get_device_dump_real(skb, cb);
+#endif
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 8) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 15, 0)) || (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 25) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)) || LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 87)
+#define wg_get_device_dump(a, b) wg_get_device_dump_real(a, b); \
+static int wg_get_device_dump(a, b) { \
+	struct wg_device *wg = (struct wg_device *)cb->args[0]; \
+	if (!wg) { \
+		int ret = wg_get_device_start(cb); \
+		if (ret) \
+			return ret; \
+	} \
+	__COMPAT_NETLINK_DUMP_BLOCK \
+} \
+static int wg_get_device_dump_real(a, b)
+#define COMPAT_CANNOT_USE_NETLINK_START
+#elif defined(__COMPAT_NETLINK_DUMP_OVERRIDE)
+#define wg_get_device_dump(a, b) wg_get_device_dump_real(a, b); \
+static int wg_get_device_dump(a, b) { \
+	__COMPAT_NETLINK_DUMP_BLOCK \
+} \
+static int wg_get_device_dump_real(a, b)
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
+#define COMPAT_CANNOT_USE_IN6_DEV_GET
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0)
+#define COMPAT_CANNOT_USE_IFF_NO_QUEUE
+#endif
+
+#if defined(CONFIG_X86_64) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0)
+#include <asm/user.h>
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0)
+#include <asm/xsave.h>
+#include <asm/xcr.h>
+static inline int cpu_has_xfeatures(u64 xfeatures_needed, const char **feature_name)
+{
+	return boot_cpu_has(X86_FEATURE_XSAVE) && xgetbv(XCR_XFEATURE_ENABLED_MASK) & xfeatures_needed;
+}
+#endif
+#ifndef XFEATURE_MASK_YMM
+#define XFEATURE_MASK_YMM XSTATE_YMM
+#endif
+#ifndef XFEATURE_MASK_SSE
+#define XFEATURE_MASK_SSE XSTATE_SSE
+#endif
+#ifndef XSTATE_AVX512
+#define XSTATE_AVX512 (XSTATE_OPMASK | XSTATE_ZMM_Hi256 | XSTATE_Hi16_ZMM)
+#endif
+#ifndef XFEATURE_MASK_AVX512
+#define XFEATURE_MASK_AVX512 XSTATE_AVX512
+#endif
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && defined(CONFIG_X86_64)
+/* This is incredibly dumb and reckless, but as it turns out, there's
+ * not really hardware Linux runs properly on that supports F but not BW
+ * and VL, so in practice this isn't so bad. Plus, this is compat layer,
+ * so the bar remains fairly low.
+ */
+#include <asm/cpufeature.h>
+#ifndef X86_FEATURE_AVX512BW
+#define X86_FEATURE_AVX512BW X86_FEATURE_AVX512F
+#endif
+#ifndef X86_FEATURE_AVX512VL
+#define X86_FEATURE_AVX512VL X86_FEATURE_AVX512F
+#endif
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 11, 0)
+struct __compat_dummy_container { char dev; };
+#define netdev_notifier_info net_device *)data); __attribute((unused)) char __compat_dummy_variable = ((struct __compat_dummy_container
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0)
+#define timer_setup(a, b, c) setup_timer(a, ((void (*)(unsigned long))b), ((unsigned long)a))
+#define from_timer(var, callback_timer, timer_fieldname) container_of(callback_timer, typeof(*var), timer_fieldname)
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 3)
+#define COMPAT_CANNOT_USE_AVX512
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0)
+#include <net/genetlink.h>
+#define genl_dump_check_consistent(a, b) genl_dump_check_consistent(a, b, &genl_family)
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0) && !defined(ISRHEL7) && !defined(ISOPENSUSE15)
+static inline void *skb_put_data(struct sk_buff *skb, const void *data, unsigned int len)
+{
+	void *tmp = skb_put(skb, len);
+	memcpy(tmp, data, len);
+	return tmp;
+}
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0) && !defined(ISRHEL7)
+#define napi_complete_done(n, work_done) napi_complete(n)
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0)
+#include <linux/netdevice.h>
+/* NAPI_STATE_SCHED gets set by netif_napi_add anyway, so this is safe.
+ * Also, kernels without NAPI_STATE_NO_BUSY_POLL don't have a call to
+ * napi_hash_add inside of netif_napi_add.
+ */
+#define NAPI_STATE_NO_BUSY_POLL NAPI_STATE_SCHED
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0)
+#include <linux/atomic.h>
+#ifndef atomic_read_acquire
+#define atomic_read_acquire(v) ({ int __compat_p1 = atomic_read(v); smp_rmb(); __compat_p1; })
+#endif
+#ifndef atomic_set_release
+#define atomic_set_release(v, i) ({ smp_wmb(); atomic_set(v, i); })
+#endif
+#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0)
+#include <linux/atomic.h>
+#ifndef atomic_read_acquire
+#define atomic_read_acquire(v) smp_load_acquire(&(v)->counter)
+#endif
+#ifndef atomic_set_release
+#define atomic_set_release(v, i) smp_store_release(&(v)->counter, (i))
+#endif
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 17, 0)
+static inline void le32_to_cpu_array(u32 *buf, unsigned int words)
+{
+	while (words--) {
+		__le32_to_cpus(buf);
+		buf++;
+	}
+}
+static inline void cpu_to_le32_array(u32 *buf, unsigned int words)
+{
+	while (words--) {
+		__cpu_to_le32s(buf);
+		buf++;
+	}
+}
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0)
+#include <crypto/algapi.h>
+static inline void crypto_xor_cpy(u8 *dst, const u8 *src1, const u8 *src2,
+				  unsigned int size)
+{
+	if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&
+	    __builtin_constant_p(size) &&
+	    (size % sizeof(unsigned long)) == 0) {
+		unsigned long *d = (unsigned long *)dst;
+		unsigned long *s1 = (unsigned long *)src1;
+		unsigned long *s2 = (unsigned long *)src2;
+
+		while (size > 0) {
+			*d++ = *s1++ ^ *s2++;
+			size -= sizeof(unsigned long);
+		}
+	} else {
+		if (unlikely(dst != src1))
+			memmove(dst, src1, size);
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)
+		crypto_xor(dst, src2, size);
+#else
+		__crypto_xor(dst, src2, size);
+#endif
+	}
+}
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
+#define read_cpuid_part() read_cpuid_part_number()
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) && !defined(ISRHEL7)
+#define hlist_add_behind(a, b) hlist_add_after(b, a)
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
+#define totalram_pages() totalram_pages
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 18, 0)
+struct __kernel_timespec {
+	int64_t tv_sec, tv_nsec;
+};
+#elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0)
+#include <linux/time64.h>
+#ifdef __kernel_timespec
+#undef __kernel_timespec
+struct __kernel_timespec {
+	int64_t tv_sec, tv_nsec;
+};
+#endif
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)
+#include <linux/kernel.h>
+#ifndef ALIGN_DOWN
+#define ALIGN_DOWN(x, a) __ALIGN_KERNEL((x) - ((a) - 1), (a))
+#endif
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0)
+#include <linux/skbuff.h>
+#define skb_probe_transport_header(a) skb_probe_transport_header(a, 0)
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0) && !defined(ISRHEL7)
+#define ignore_df local_df
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0)
+/* Note that all intentional uses of the non-_bh variety need to explicitly
+ * undef these, conditionalized on COMPAT_CANNOT_DEPRECIATE_BH_RCU.
+ */
+#include <linux/rcupdate.h>
+static __always_inline void old_synchronize_rcu(void)
+{
+	synchronize_rcu();
+}
+static __always_inline void old_call_rcu(void *a, void *b)
+{
+	call_rcu(a, b);
+}
+static __always_inline void old_rcu_barrier(void)
+{
+	rcu_barrier();
+}
+#ifdef synchronize_rcu
+#undef synchronize_rcu
+#endif
+#ifdef call_rcu
+#undef call_rcu
+#endif
+#ifdef rcu_barrier
+#undef rcu_barrier
+#endif
+#define synchronize_rcu synchronize_rcu_bh
+#define call_rcu call_rcu_bh
+#define rcu_barrier rcu_barrier_bh
+#define COMPAT_CANNOT_DEPRECIATE_BH_RCU
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 10) && !defined(ISRHEL8)
+static inline void skb_mark_not_on_list(struct sk_buff *skb)
+{
+	skb->next = NULL;
+}
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 20, 0) && !defined(ISRHEL8)
+#define NLA_EXACT_LEN NLA_UNSPEC
+#endif
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 2, 0)
+#define NLA_MIN_LEN NLA_UNSPEC
+#define COMPAT_CANNOT_INDIVIDUAL_NETLINK_OPS_POLICY
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 2, 0) && defined(__aarch64__)
+#define cpu_have_named_feature(name) (elf_hwcap & (HWCAP_ ## name))
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 1, 0)
+#include <linux/stddef.h>
+#ifndef offsetofend
+#define offsetofend(TYPE, MEMBER) (offsetof(TYPE, MEMBER) + sizeof(((TYPE *)0)->MEMBER))
+#endif
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 5, 0)
+#define genl_dumpit_info(cb) ({ \
+	struct { struct nlattr **attrs; } *a = (void *)((u8 *)cb->args + offsetofend(struct dump_ctx, next_allowedip)); \
+	BUILD_BUG_ON(sizeof(cb->args) < offsetofend(struct dump_ctx, next_allowedip) + sizeof(*a)); \
+	a->attrs = genl_family_attrbuf(&genl_family); \
+	if (nlmsg_parse(cb->nlh, GENL_HDRLEN + genl_family.hdrsize, a->attrs, genl_family.maxattr, device_policy, NULL) < 0) \
+		memset(a->attrs, 0, (genl_family.maxattr + 1) * sizeof(struct nlattr *)); \
+	a; \
+})
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 5) && LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0)) || LINUX_VERSION_CODE < KERNEL_VERSION(5, 3, 18)
+#define ipv6_dst_lookup_flow(a, b, c, d) ipv6_dst_lookup(a, b, &dst, c) + (void *)0 ?: dst
+#endif
+
+#if defined(ISUBUNTU1604)
+#include <linux/siphash.h>
+#ifndef _WG_LINUX_SIPHASH_H
+#define hsiphash_2u32 siphash_2u32
+#define hsiphash_3u32 siphash_3u32
+#define hsiphash_key_t siphash_key_t
+#endif
+#endif
+
+#ifdef CONFIG_VE
+#include <linux/netdev_features.h>
+#ifdef NETIF_F_VIRTUAL
+#undef NETIF_F_LLTX
+#define NETIF_F_LLTX (__NETIF_F(LLTX) | __NETIF_F(VIRTUAL))
+#endif
+#endif
+
+/* https://github.com/ClangBuiltLinux/linux/issues/7 */
+#if defined( __clang__) && (!defined(CONFIG_CLANG_VERSION) || CONFIG_CLANG_VERSION < 80000)
+#include <linux/bug.h>
+#undef BUILD_BUG_ON
+#define BUILD_BUG_ON(x)
+#endif
+
+/* https://lkml.kernel.org/r/20170624021727.17835-1-Jason@zx2c4.com */
+#if IS_ENABLED(CONFIG_NF_CONNTRACK)
+#include <linux/ip.h>
+#include <linux/icmpv6.h>
+#include <net/ipv6.h>
+#include <net/icmp.h>
+#include <net/netfilter/nf_conntrack.h>
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0)
+#include <net/netfilter/nf_nat_core.h>
+#endif
+static inline void new_icmp_send(struct sk_buff *skb_in, int type, int code, __be32 info)
+{
+	enum ip_conntrack_info ctinfo;
+	struct nf_conn *ct = nf_ct_get(skb_in, &ctinfo);
+	if (skb_network_header(skb_in) < skb_in->head || (skb_network_header(skb_in) + sizeof(struct iphdr)) > skb_tail_pointer(skb_in))
+		return;
+	if (ct)
+		ip_hdr(skb_in)->saddr = ct->tuplehash[0].tuple.src.u3.ip;
+	icmp_send(skb_in, type, code, info);
+}
+static inline void new_icmpv6_send(struct sk_buff *skb, u8 type, u8 code, __u32 info)
+{
+	enum ip_conntrack_info ctinfo;
+	struct nf_conn *ct = nf_ct_get(skb, &ctinfo);
+	if (skb_network_header(skb) < skb->head || (skb_network_header(skb) + sizeof(struct ipv6hdr)) > skb_tail_pointer(skb))
+		return;
+	if (ct)
+		ipv6_hdr(skb)->saddr = ct->tuplehash[0].tuple.src.u3.in6;
+	icmpv6_send(skb, type, code, info);
+}
+#define icmp_send(a,b,c,d) new_icmp_send(a,b,c,d)
+#define icmpv6_send(a,b,c,d) new_icmpv6_send(a,b,c,d)
+#endif
+
+/* PaX compatibility */
+#ifdef CONSTIFY_PLUGIN
+#include <linux/cache.h>
+#undef __read_mostly
+#define __read_mostly
+#endif
+#if (defined(RAP_PLUGIN) || defined(CONFIG_CFI_CLANG)) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0)
+#include <linux/timer.h>
+#define wg_expired_retransmit_handshake(a) wg_expired_retransmit_handshake(unsigned long timer)
+#define wg_expired_send_keepalive(a) wg_expired_send_keepalive(unsigned long timer)
+#define wg_expired_new_handshake(a) wg_expired_new_handshake(unsigned long timer)
+#define wg_expired_zero_key_material(a) wg_expired_zero_key_material(unsigned long timer)
+#define wg_expired_send_persistent_keepalive(a) wg_expired_send_persistent_keepalive(unsigned long timer)
+#undef timer_setup
+#define timer_setup(a, b, c) setup_timer(a, ((void (*)(unsigned long))b), ((unsigned long)a))
+#undef from_timer
+#define from_timer(var, callback_timer, timer_fieldname) container_of((struct timer_list *)callback_timer, typeof(*var), timer_fieldname)
+#endif
+
+#endif /* _WG_COMPAT_H */
diff -Naur --no-dereference a/net/wireguard/compat/dst_cache/dst_cache.c b/net/wireguard/compat/dst_cache/dst_cache.c
--- a/net/wireguard/compat/dst_cache/dst_cache.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/compat/dst_cache/dst_cache.c	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,175 @@
+/*
+ * net/core/dst_cache.c - dst entry cache
+ *
+ * Copyright (c) 2016 Paolo Abeni <pabeni@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/kernel.h>
+#include <linux/percpu.h>
+#include <net/dst_cache.h>
+#include <net/route.h>
+#if IS_ENABLED(CONFIG_IPV6)
+#include <net/ip6_fib.h>
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0)) || LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 50)
+static inline u32 rt6_get_cookie(const struct rt6_info *rt)
+{
+	if ((unlikely(rt->dst.flags & DST_NOCACHE) && rt->dst.from))
+		rt = (struct rt6_info *)(rt->dst.from);
+
+	return rt->rt6i_node ? rt->rt6i_node->fn_sernum : 0;
+}
+#endif
+#endif
+#include <uapi/linux/in.h>
+
+struct dst_cache_pcpu {
+	unsigned long refresh_ts;
+	struct dst_entry *dst;
+	u32 cookie;
+	union {
+		struct in_addr in_saddr;
+		struct in6_addr in6_saddr;
+	};
+};
+
+static void dst_cache_per_cpu_dst_set(struct dst_cache_pcpu *dst_cache,
+				      struct dst_entry *dst, u32 cookie)
+{
+	dst_release(dst_cache->dst);
+	if (dst)
+		dst_hold(dst);
+
+	dst_cache->cookie = cookie;
+	dst_cache->dst = dst;
+}
+
+static struct dst_entry *dst_cache_per_cpu_get(struct dst_cache *dst_cache,
+					       struct dst_cache_pcpu *idst)
+{
+	struct dst_entry *dst;
+
+	dst = idst->dst;
+	if (!dst)
+		goto fail;
+
+	/* the cache already hold a dst reference; it can't go away */
+	dst_hold(dst);
+
+	if (unlikely(!time_after(idst->refresh_ts, dst_cache->reset_ts) ||
+		     (dst->obsolete && !dst->ops->check(dst, idst->cookie)))) {
+		dst_cache_per_cpu_dst_set(idst, NULL, 0);
+		dst_release(dst);
+		goto fail;
+	}
+	return dst;
+
+fail:
+	idst->refresh_ts = jiffies;
+	return NULL;
+}
+
+struct dst_entry *dst_cache_get(struct dst_cache *dst_cache)
+{
+	if (!dst_cache->cache)
+		return NULL;
+
+	return dst_cache_per_cpu_get(dst_cache, this_cpu_ptr(dst_cache->cache));
+}
+
+struct rtable *dst_cache_get_ip4(struct dst_cache *dst_cache, __be32 *saddr)
+{
+	struct dst_cache_pcpu *idst;
+	struct dst_entry *dst;
+
+	if (!dst_cache->cache)
+		return NULL;
+
+	idst = this_cpu_ptr(dst_cache->cache);
+	dst = dst_cache_per_cpu_get(dst_cache, idst);
+	if (!dst)
+		return NULL;
+
+	*saddr = idst->in_saddr.s_addr;
+	return container_of(dst, struct rtable, dst);
+}
+
+void dst_cache_set_ip4(struct dst_cache *dst_cache, struct dst_entry *dst,
+		       __be32 saddr)
+{
+	struct dst_cache_pcpu *idst;
+
+	if (!dst_cache->cache)
+		return;
+
+	idst = this_cpu_ptr(dst_cache->cache);
+	dst_cache_per_cpu_dst_set(idst, dst, 0);
+	idst->in_saddr.s_addr = saddr;
+}
+
+#if IS_ENABLED(CONFIG_IPV6)
+void dst_cache_set_ip6(struct dst_cache *dst_cache, struct dst_entry *dst,
+		       const struct in6_addr *addr)
+{
+	struct dst_cache_pcpu *idst;
+
+	if (!dst_cache->cache)
+		return;
+
+	idst = this_cpu_ptr(dst_cache->cache);
+	dst_cache_per_cpu_dst_set(this_cpu_ptr(dst_cache->cache), dst,
+				  rt6_get_cookie((struct rt6_info *)dst));
+	idst->in6_saddr = *addr;
+}
+
+struct dst_entry *dst_cache_get_ip6(struct dst_cache *dst_cache,
+				    struct in6_addr *saddr)
+{
+	struct dst_cache_pcpu *idst;
+	struct dst_entry *dst;
+
+	if (!dst_cache->cache)
+		return NULL;
+
+	idst = this_cpu_ptr(dst_cache->cache);
+	dst = dst_cache_per_cpu_get(dst_cache, idst);
+	if (!dst)
+		return NULL;
+
+	*saddr = idst->in6_saddr;
+	return dst;
+}
+#endif
+
+int dst_cache_init(struct dst_cache *dst_cache, gfp_t gfp)
+{
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 0)
+	BUG_ON(gfp & GFP_ATOMIC);
+	dst_cache->cache = alloc_percpu(struct dst_cache_pcpu);
+#else
+	dst_cache->cache = alloc_percpu_gfp(struct dst_cache_pcpu,
+					    gfp | __GFP_ZERO);
+#endif
+	if (!dst_cache->cache)
+		return -ENOMEM;
+
+	dst_cache_reset(dst_cache);
+	return 0;
+}
+
+void dst_cache_destroy(struct dst_cache *dst_cache)
+{
+	int i;
+
+	if (!dst_cache->cache)
+		return;
+
+	for_each_possible_cpu(i)
+		dst_release(per_cpu_ptr(dst_cache->cache, i)->dst);
+
+	free_percpu(dst_cache->cache);
+}
diff -Naur --no-dereference a/net/wireguard/compat/dst_cache/include/net/dst_cache.h b/net/wireguard/compat/dst_cache/include/net/dst_cache.h
--- a/net/wireguard/compat/dst_cache/include/net/dst_cache.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/compat/dst_cache/include/net/dst_cache.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,97 @@
+#ifndef _WG_NET_DST_CACHE_H
+#define _WG_NET_DST_CACHE_H
+
+#include <linux/jiffies.h>
+#include <net/dst.h>
+#if IS_ENABLED(CONFIG_IPV6)
+#include <net/ip6_fib.h>
+#endif
+
+struct dst_cache {
+	struct dst_cache_pcpu __percpu *cache;
+	unsigned long reset_ts;
+};
+
+/**
+ *	dst_cache_get - perform cache lookup
+ *	@dst_cache: the cache
+ *
+ *	The caller should use dst_cache_get_ip4() if it need to retrieve the
+ *	source address to be used when xmitting to the cached dst.
+ *	local BH must be disabled.
+ */
+struct dst_entry *dst_cache_get(struct dst_cache *dst_cache);
+
+/**
+ *	dst_cache_get_ip4 - perform cache lookup and fetch ipv4 source address
+ *	@dst_cache: the cache
+ *	@saddr: return value for the retrieved source address
+ *
+ *	local BH must be disabled.
+ */
+struct rtable *dst_cache_get_ip4(struct dst_cache *dst_cache, __be32 *saddr);
+
+/**
+ *	dst_cache_set_ip4 - store the ipv4 dst into the cache
+ *	@dst_cache: the cache
+ *	@dst: the entry to be cached
+ *	@saddr: the source address to be stored inside the cache
+ *
+ *	local BH must be disabled.
+ */
+void dst_cache_set_ip4(struct dst_cache *dst_cache, struct dst_entry *dst,
+		       __be32 saddr);
+
+#if IS_ENABLED(CONFIG_IPV6)
+
+/**
+ *	dst_cache_set_ip6 - store the ipv6 dst into the cache
+ *	@dst_cache: the cache
+ *	@dst: the entry to be cached
+ *	@saddr: the source address to be stored inside the cache
+ *
+ *	local BH must be disabled.
+ */
+void dst_cache_set_ip6(struct dst_cache *dst_cache, struct dst_entry *dst,
+		       const struct in6_addr *addr);
+
+/**
+ *	dst_cache_get_ip6 - perform cache lookup and fetch ipv6 source address
+ *	@dst_cache: the cache
+ *	@saddr: return value for the retrieved source address
+ *
+ *	local BH must be disabled.
+ */
+struct dst_entry *dst_cache_get_ip6(struct dst_cache *dst_cache,
+				    struct in6_addr *saddr);
+#endif
+
+/**
+ *	dst_cache_reset - invalidate the cache contents
+ *	@dst_cache: the cache
+ *
+ *	This do not free the cached dst to avoid races and contentions.
+ *	the dst will be freed on later cache lookup.
+ */
+static inline void dst_cache_reset(struct dst_cache *dst_cache)
+{
+	dst_cache->reset_ts = jiffies;
+}
+
+/**
+ *	dst_cache_init - initialize the cache, allocating the required storage
+ *	@dst_cache: the cache
+ *	@gfp: allocation flags
+ */
+int dst_cache_init(struct dst_cache *dst_cache, gfp_t gfp);
+
+/**
+ *	dst_cache_destroy - empty the cache and free the allocated storage
+ *	@dst_cache: the cache
+ *
+ *	No synchronization is enforced: it must be called only when the cache
+ *	is unused.
+ */
+void dst_cache_destroy(struct dst_cache *dst_cache);
+
+#endif /* _WG_NET_DST_CACHE_H */
diff -Naur --no-dereference a/net/wireguard/compat/fpu-x86/include/asm/fpu/api.h b/net/wireguard/compat/fpu-x86/include/asm/fpu/api.h
--- a/net/wireguard/compat/fpu-x86/include/asm/fpu/api.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/compat/fpu-x86/include/asm/fpu/api.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1 @@
+#include <asm/i387.h>
diff -Naur --no-dereference a/net/wireguard/compat/intel-family-x86/include/asm/intel-family.h b/net/wireguard/compat/intel-family-x86/include/asm/intel-family.h
--- a/net/wireguard/compat/intel-family-x86/include/asm/intel-family.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/compat/intel-family-x86/include/asm/intel-family.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,73 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_X86_INTEL_FAMILY_H
+#define _ASM_X86_INTEL_FAMILY_H
+
+/*
+ * "Big Core" Processors (Branded as Core, Xeon, etc...)
+ *
+ * The "_X" parts are generally the EP and EX Xeons, or the
+ * "Extreme" ones, like Broadwell-E.
+ *
+ * Things ending in "2" are usually because we have no better
+ * name for them.  There's no processor called "SILVERMONT2".
+ */
+
+#define INTEL_FAM6_CORE_YONAH		0x0E
+
+#define INTEL_FAM6_CORE2_MEROM		0x0F
+#define INTEL_FAM6_CORE2_MEROM_L	0x16
+#define INTEL_FAM6_CORE2_PENRYN		0x17
+#define INTEL_FAM6_CORE2_DUNNINGTON	0x1D
+
+#define INTEL_FAM6_NEHALEM		0x1E
+#define INTEL_FAM6_NEHALEM_G		0x1F /* Auburndale / Havendale */
+#define INTEL_FAM6_NEHALEM_EP		0x1A
+#define INTEL_FAM6_NEHALEM_EX		0x2E
+
+#define INTEL_FAM6_WESTMERE		0x25
+#define INTEL_FAM6_WESTMERE_EP		0x2C
+#define INTEL_FAM6_WESTMERE_EX		0x2F
+
+#define INTEL_FAM6_SANDYBRIDGE		0x2A
+#define INTEL_FAM6_SANDYBRIDGE_X	0x2D
+#define INTEL_FAM6_IVYBRIDGE		0x3A
+#define INTEL_FAM6_IVYBRIDGE_X		0x3E
+
+#define INTEL_FAM6_HASWELL_CORE		0x3C
+#define INTEL_FAM6_HASWELL_X		0x3F
+#define INTEL_FAM6_HASWELL_ULT		0x45
+#define INTEL_FAM6_HASWELL_GT3E		0x46
+
+#define INTEL_FAM6_BROADWELL_CORE	0x3D
+#define INTEL_FAM6_BROADWELL_GT3E	0x47
+#define INTEL_FAM6_BROADWELL_X		0x4F
+#define INTEL_FAM6_BROADWELL_XEON_D	0x56
+
+#define INTEL_FAM6_SKYLAKE_MOBILE	0x4E
+#define INTEL_FAM6_SKYLAKE_DESKTOP	0x5E
+#define INTEL_FAM6_SKYLAKE_X		0x55
+#define INTEL_FAM6_KABYLAKE_MOBILE	0x8E
+#define INTEL_FAM6_KABYLAKE_DESKTOP	0x9E
+
+/* "Small Core" Processors (Atom) */
+
+#define INTEL_FAM6_ATOM_PINEVIEW	0x1C
+#define INTEL_FAM6_ATOM_LINCROFT	0x26
+#define INTEL_FAM6_ATOM_PENWELL		0x27
+#define INTEL_FAM6_ATOM_CLOVERVIEW	0x35
+#define INTEL_FAM6_ATOM_CEDARVIEW	0x36
+#define INTEL_FAM6_ATOM_SILVERMONT1	0x37 /* BayTrail/BYT / Valleyview */
+#define INTEL_FAM6_ATOM_SILVERMONT2	0x4D /* Avaton/Rangely */
+#define INTEL_FAM6_ATOM_AIRMONT		0x4C /* CherryTrail / Braswell */
+#define INTEL_FAM6_ATOM_MERRIFIELD	0x4A /* Tangier */
+#define INTEL_FAM6_ATOM_MOOREFIELD	0x5A /* Anniedale */
+#define INTEL_FAM6_ATOM_GOLDMONT	0x5C
+#define INTEL_FAM6_ATOM_DENVERTON	0x5F /* Goldmont Microserver */
+#define INTEL_FAM6_ATOM_GEMINI_LAKE	0x7A
+
+/* Xeon Phi */
+
+#define INTEL_FAM6_XEON_PHI_KNL		0x57 /* Knights Landing */
+#define INTEL_FAM6_XEON_PHI_KNM		0x85 /* Knights Mill */
+
+#endif /* _ASM_X86_INTEL_FAMILY_H */
diff -Naur --no-dereference a/net/wireguard/compat/Makefile.include b/net/wireguard/compat/Makefile.include
--- a/net/wireguard/compat/Makefile.include	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/compat/Makefile.include	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,98 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+
+kbuild-dir := $(if $(filter /%,$(src)),$(src),$(srctree)/$(src))
+
+ccflags-y += -include $(kbuild-dir)/compat/compat.h
+asflags-y += -include $(kbuild-dir)/compat/compat-asm.h
+
+ifeq ($(wildcard $(srctree)/include/linux/ptr_ring.h),)
+ccflags-y += -I$(kbuild-dir)/compat/ptr_ring/include
+endif
+
+ifeq ($(wildcard $(srctree)/include/linux/siphash.h),)
+ccflags-y += -I$(kbuild-dir)/compat/siphash/include
+wireguard-y += compat/siphash/siphash.o
+endif
+
+ifeq ($(wildcard $(srctree)/include/net/dst_cache.h),)
+ccflags-y += -I$(kbuild-dir)/compat/dst_cache/include
+wireguard-y += compat/dst_cache/dst_cache.o
+endif
+
+ifeq ($(wildcard $(srctree)/arch/x86/include/asm/intel-family.h)$(CONFIG_X86),y)
+ccflags-y += -I$(kbuild-dir)/compat/intel-family-x86/include
+endif
+
+ifeq ($(wildcard $(srctree)/arch/x86/include/asm/fpu/api.h)$(CONFIG_X86),y)
+ccflags-y += -I$(kbuild-dir)/compat/fpu-x86/include
+endif
+
+ifeq ($(wildcard $(srctree)/arch/$(SRCARCH)/include/asm/simd.h)$(shell grep -s -F "generic-y += simd.h" "$(srctree)/arch/$(SRCARCH)/Makefile" "$(srctree)/arch/$(SRCARCH)/Makefile"),)
+ccflags-y += -I$(kbuild-dir)/compat/simd-asm/include
+endif
+
+ifeq ($(wildcard $(srctree)/include/linux/simd.h),)
+ccflags-y += -I$(kbuild-dir)/compat/simd/include
+endif
+
+ifeq ($(wildcard $(srctree)/include/net/udp_tunnel.h),)
+ccflags-y += -I$(kbuild-dir)/compat/udp_tunnel/include
+wireguard-y += compat/udp_tunnel/udp_tunnel.o
+endif
+
+ifeq ($(shell grep -s -F "int crypto_memneq" "$(srctree)/include/crypto/algapi.h"),)
+ccflags-y += -include $(kbuild-dir)/compat/memneq/include.h
+wireguard-y += compat/memneq/memneq.o
+endif
+
+ifeq ($(shell grep -s -F "addr_gen_mode" "$(srctree)/include/linux/ipv6.h"),)
+ccflags-y += -DCOMPAT_CANNOT_USE_DEV_CNF
+endif
+
+ifdef CONFIG_HZ
+ifeq ($(wildcard $(srctree)/include/generated/timeconst.h),)
+ccflags-y += $(shell echo 'define gcd(a,b){auto t;while(b){t=b;b=a%b;a=t;};return a;};hz=$(CONFIG_HZ);cd=gcd(hz,1000000);print "-DHZ_TO_USEC_NUM=",1000000/cd," -DHZ_TO_USEC_DEN=",hz/cd;halt;' | bc -q)
+endif
+endif
+
+ifeq ($(wildcard $(srctree)/arch/arm/include/asm/neon.h)$(CONFIG_ARM),y)
+ccflags-y += -I$(kbuild-dir)/compat/neon-arm/include
+endif
+ifeq ($(wildcard $(srctree)/arch/arm64/include/asm/neon.h)$(CONFIG_ARM64),y)
+ccflags-y += -I$(kbuild-dir)/compat/neon-arm/include
+endif
+
+ifeq ($(CONFIG_X86_64),y)
+	ifeq ($(ssse3_instr),)
+		ssse3_instr := $(call as-instr,pshufb %xmm0$(comma)%xmm0,-DCONFIG_AS_SSSE3=1)
+		ccflags-y += $(ssse3_instr)
+		asflags-y += $(ssse3_instr)
+	endif
+	ifeq ($(avx_instr),)
+		avx_instr := $(call as-instr,vxorps %ymm0$(comma)%ymm1$(comma)%ymm2,-DCONFIG_AS_AVX=1)
+		ccflags-y += $(avx_instr)
+		asflags-y += $(avx_instr)
+	endif
+	ifeq ($(avx2_instr),)
+		avx2_instr := $(call as-instr,vpbroadcastb %xmm0$(comma)%ymm1,-DCONFIG_AS_AVX2=1)
+		ccflags-y += $(avx2_instr)
+		asflags-y += $(avx2_instr)
+	endif
+	ifeq ($(avx512_instr),)
+		avx512_instr := $(call as-instr,vpmovm2b %k1$(comma)%zmm5,-DCONFIG_AS_AVX512=1)
+		ccflags-y += $(avx512_instr)
+		asflags-y += $(avx512_instr)
+	endif
+	ifeq ($(bmi2_instr),)
+		bmi2_instr :=$(call as-instr,mulx %rax$(comma)%rax$(comma)%rax,-DCONFIG_AS_BMI2=1)
+		ccflags-y += $(bmi2_instr)
+		asflags-y += $(bmi2_instr)
+	endif
+	ifeq ($(adx_instr),)
+		adx_instr :=$(call as-instr,adcx %rax$(comma)%rax,-DCONFIG_AS_ADX=1)
+		ccflags-y += $(adx_instr)
+		asflags-y += $(adx_instr)
+	endif
+endif
diff -Naur --no-dereference a/net/wireguard/compat/memneq/include.h b/net/wireguard/compat/memneq/include.h
--- a/net/wireguard/compat/memneq/include.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/compat/memneq/include.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,5 @@
+extern noinline unsigned long __crypto_memneq(const void *a, const void *b, size_t size);
+static inline int crypto_memneq(const void *a, const void *b, size_t size)
+{
+	return __crypto_memneq(a, b, size) != 0UL ? 1 : 0;
+}
diff -Naur --no-dereference a/net/wireguard/compat/memneq/memneq.c b/net/wireguard/compat/memneq/memneq.c
--- a/net/wireguard/compat/memneq/memneq.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/compat/memneq/memneq.c	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,170 @@
+/*
+ * Constant-time equality testing of memory regions.
+ *
+ * Authors:
+ *
+ *   James Yonan <james@openvpn.net>
+ *   Daniel Borkmann <dborkman@redhat.com>
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2013 OpenVPN Technologies, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ * The full GNU General Public License is included in this distribution
+ * in the file called LICENSE.GPL.
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2013 OpenVPN Technologies, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *   * Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in
+ *     the documentation and/or other materials provided with the
+ *     distribution.
+ *   * Neither the name of OpenVPN Technologies nor the names of its
+ *     contributors may be used to endorse or promote products derived
+ *     from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <crypto/algapi.h>
+
+/* Make the optimizer believe the variable can be manipulated arbitrarily. */
+#define COMPILER_OPTIMIZER_HIDE_VAR(var) asm("" : "=r" (var) : "0" (var))
+
+#ifndef __HAVE_ARCH_CRYPTO_MEMNEQ
+
+/* Generic path for arbitrary size */
+static inline unsigned long
+__crypto_memneq_generic(const void *a, const void *b, size_t size)
+{
+	unsigned long neq = 0;
+
+#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)
+	while (size >= sizeof(unsigned long)) {
+		neq |= *(unsigned long *)a ^ *(unsigned long *)b;
+		COMPILER_OPTIMIZER_HIDE_VAR(neq);
+		a += sizeof(unsigned long);
+		b += sizeof(unsigned long);
+		size -= sizeof(unsigned long);
+	}
+#endif /* CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS */
+	while (size > 0) {
+		neq |= *(unsigned char *)a ^ *(unsigned char *)b;
+		COMPILER_OPTIMIZER_HIDE_VAR(neq);
+		a += 1;
+		b += 1;
+		size -= 1;
+	}
+	return neq;
+}
+
+/* Loop-free fast-path for frequently used 16-byte size */
+static inline unsigned long __crypto_memneq_16(const void *a, const void *b)
+{
+	unsigned long neq = 0;
+
+#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+	if (sizeof(unsigned long) == 8) {
+		neq |= *(unsigned long *)(a)   ^ *(unsigned long *)(b);
+		COMPILER_OPTIMIZER_HIDE_VAR(neq);
+		neq |= *(unsigned long *)(a+8) ^ *(unsigned long *)(b+8);
+		COMPILER_OPTIMIZER_HIDE_VAR(neq);
+	} else if (sizeof(unsigned int) == 4) {
+		neq |= *(unsigned int *)(a)    ^ *(unsigned int *)(b);
+		COMPILER_OPTIMIZER_HIDE_VAR(neq);
+		neq |= *(unsigned int *)(a+4)  ^ *(unsigned int *)(b+4);
+		COMPILER_OPTIMIZER_HIDE_VAR(neq);
+		neq |= *(unsigned int *)(a+8)  ^ *(unsigned int *)(b+8);
+		COMPILER_OPTIMIZER_HIDE_VAR(neq);
+		neq |= *(unsigned int *)(a+12) ^ *(unsigned int *)(b+12);
+		COMPILER_OPTIMIZER_HIDE_VAR(neq);
+	} else
+#endif /* CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS */
+	{
+		neq |= *(unsigned char *)(a)    ^ *(unsigned char *)(b);
+		COMPILER_OPTIMIZER_HIDE_VAR(neq);
+		neq |= *(unsigned char *)(a+1)  ^ *(unsigned char *)(b+1);
+		COMPILER_OPTIMIZER_HIDE_VAR(neq);
+		neq |= *(unsigned char *)(a+2)  ^ *(unsigned char *)(b+2);
+		COMPILER_OPTIMIZER_HIDE_VAR(neq);
+		neq |= *(unsigned char *)(a+3)  ^ *(unsigned char *)(b+3);
+		COMPILER_OPTIMIZER_HIDE_VAR(neq);
+		neq |= *(unsigned char *)(a+4)  ^ *(unsigned char *)(b+4);
+		COMPILER_OPTIMIZER_HIDE_VAR(neq);
+		neq |= *(unsigned char *)(a+5)  ^ *(unsigned char *)(b+5);
+		COMPILER_OPTIMIZER_HIDE_VAR(neq);
+		neq |= *(unsigned char *)(a+6)  ^ *(unsigned char *)(b+6);
+		COMPILER_OPTIMIZER_HIDE_VAR(neq);
+		neq |= *(unsigned char *)(a+7)  ^ *(unsigned char *)(b+7);
+		COMPILER_OPTIMIZER_HIDE_VAR(neq);
+		neq |= *(unsigned char *)(a+8)  ^ *(unsigned char *)(b+8);
+		COMPILER_OPTIMIZER_HIDE_VAR(neq);
+		neq |= *(unsigned char *)(a+9)  ^ *(unsigned char *)(b+9);
+		COMPILER_OPTIMIZER_HIDE_VAR(neq);
+		neq |= *(unsigned char *)(a+10) ^ *(unsigned char *)(b+10);
+		COMPILER_OPTIMIZER_HIDE_VAR(neq);
+		neq |= *(unsigned char *)(a+11) ^ *(unsigned char *)(b+11);
+		COMPILER_OPTIMIZER_HIDE_VAR(neq);
+		neq |= *(unsigned char *)(a+12) ^ *(unsigned char *)(b+12);
+		COMPILER_OPTIMIZER_HIDE_VAR(neq);
+		neq |= *(unsigned char *)(a+13) ^ *(unsigned char *)(b+13);
+		COMPILER_OPTIMIZER_HIDE_VAR(neq);
+		neq |= *(unsigned char *)(a+14) ^ *(unsigned char *)(b+14);
+		COMPILER_OPTIMIZER_HIDE_VAR(neq);
+		neq |= *(unsigned char *)(a+15) ^ *(unsigned char *)(b+15);
+		COMPILER_OPTIMIZER_HIDE_VAR(neq);
+	}
+
+	return neq;
+}
+
+/* Compare two areas of memory without leaking timing information,
+ * and with special optimizations for common sizes.  Users should
+ * not call this function directly, but should instead use
+ * crypto_memneq defined in crypto/algapi.h.
+ */
+noinline unsigned long __crypto_memneq(const void *a, const void *b,
+				       size_t size)
+{
+	switch (size) {
+	case 16:
+		return __crypto_memneq_16(a, b);
+	default:
+		return __crypto_memneq_generic(a, b, size);
+	}
+}
+
+#endif /* __HAVE_ARCH_CRYPTO_MEMNEQ */
diff -Naur --no-dereference a/net/wireguard/compat/neon-arm/include/asm/neon.h b/net/wireguard/compat/neon-arm/include/asm/neon.h
--- a/net/wireguard/compat/neon-arm/include/asm/neon.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/compat/neon-arm/include/asm/neon.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,7 @@
+#ifndef _ARCH_ARM_ASM_NEON
+#define _ARCH_ARM_ASM_NEON
+#define kernel_neon_begin() \
+	BUILD_BUG_ON_MSG(1, "This kernel does not support ARM NEON")
+#define kernel_neon_end() \
+	BUILD_BUG_ON_MSG(1, "This kernel does not support ARM NEON")
+#endif
diff -Naur --no-dereference a/net/wireguard/compat/ptr_ring/include/linux/ptr_ring.h b/net/wireguard/compat/ptr_ring/include/linux/ptr_ring.h
--- a/net/wireguard/compat/ptr_ring/include/linux/ptr_ring.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/compat/ptr_ring/include/linux/ptr_ring.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,640 @@
+/*
+ *	Definitions for the 'struct ptr_ring' datastructure.
+ *
+ *	Author:
+ *		Michael S. Tsirkin <mst@redhat.com>
+ *
+ *	Copyright (C) 2016 Red Hat, Inc.
+ *
+ *	This program is free software; you can redistribute it and/or modify it
+ *	under the terms of the GNU General Public License as published by the
+ *	Free Software Foundation; either version 2 of the License, or (at your
+ *	option) any later version.
+ *
+ *	This is a limited-size FIFO maintaining pointers in FIFO order, with
+ *	one CPU producing entries and another consuming entries from a FIFO.
+ *
+ *	This implementation tries to minimize cache-contention when there is a
+ *	single producer and a single consumer CPU.
+ */
+
+#ifndef _LINUX_PTR_RING_H
+#define _LINUX_PTR_RING_H 1
+
+#ifdef __KERNEL__
+#include <linux/spinlock.h>
+#include <linux/cache.h>
+#include <linux/types.h>
+#include <linux/compiler.h>
+#include <linux/cache.h>
+#include <linux/slab.h>
+#include <asm/errno.h>
+#endif
+
+struct ptr_ring {
+	int producer ____cacheline_aligned_in_smp;
+	spinlock_t producer_lock;
+	int consumer_head ____cacheline_aligned_in_smp; /* next valid entry */
+	int consumer_tail; /* next entry to invalidate */
+	spinlock_t consumer_lock;
+	/* Shared consumer/producer data */
+	/* Read-only by both the producer and the consumer */
+	int size ____cacheline_aligned_in_smp; /* max entries in queue */
+	int batch; /* number of entries to consume in a batch */
+	void **queue;
+};
+
+/* Note: callers invoking this in a loop must use a compiler barrier,
+ * for example cpu_relax().  If ring is ever resized, callers must hold
+ * producer_lock - see e.g. ptr_ring_full.  Otherwise, if callers don't hold
+ * producer_lock, the next call to __ptr_ring_produce may fail.
+ */
+static inline bool __ptr_ring_full(struct ptr_ring *r)
+{
+	return r->queue[r->producer];
+}
+
+static inline bool ptr_ring_full(struct ptr_ring *r)
+{
+	bool ret;
+
+	spin_lock(&r->producer_lock);
+	ret = __ptr_ring_full(r);
+	spin_unlock(&r->producer_lock);
+
+	return ret;
+}
+
+static inline bool ptr_ring_full_irq(struct ptr_ring *r)
+{
+	bool ret;
+
+	spin_lock_irq(&r->producer_lock);
+	ret = __ptr_ring_full(r);
+	spin_unlock_irq(&r->producer_lock);
+
+	return ret;
+}
+
+static inline bool ptr_ring_full_any(struct ptr_ring *r)
+{
+	unsigned long flags;
+	bool ret;
+
+	spin_lock_irqsave(&r->producer_lock, flags);
+	ret = __ptr_ring_full(r);
+	spin_unlock_irqrestore(&r->producer_lock, flags);
+
+	return ret;
+}
+
+static inline bool ptr_ring_full_bh(struct ptr_ring *r)
+{
+	bool ret;
+
+	spin_lock_bh(&r->producer_lock);
+	ret = __ptr_ring_full(r);
+	spin_unlock_bh(&r->producer_lock);
+
+	return ret;
+}
+
+/* Note: callers invoking this in a loop must use a compiler barrier,
+ * for example cpu_relax(). Callers must hold producer_lock.
+ */
+static inline int __ptr_ring_produce(struct ptr_ring *r, void *ptr)
+{
+	if (unlikely(!r->size) || r->queue[r->producer])
+		return -ENOSPC;
+
+	r->queue[r->producer++] = ptr;
+	if (unlikely(r->producer >= r->size))
+		r->producer = 0;
+	return 0;
+}
+
+/*
+ * Note: resize (below) nests producer lock within consumer lock, so if you
+ * consume in interrupt or BH context, you must disable interrupts/BH when
+ * calling this.
+ */
+static inline int ptr_ring_produce(struct ptr_ring *r, void *ptr)
+{
+	int ret;
+
+	spin_lock(&r->producer_lock);
+	ret = __ptr_ring_produce(r, ptr);
+	spin_unlock(&r->producer_lock);
+
+	return ret;
+}
+
+static inline int ptr_ring_produce_irq(struct ptr_ring *r, void *ptr)
+{
+	int ret;
+
+	spin_lock_irq(&r->producer_lock);
+	ret = __ptr_ring_produce(r, ptr);
+	spin_unlock_irq(&r->producer_lock);
+
+	return ret;
+}
+
+static inline int ptr_ring_produce_any(struct ptr_ring *r, void *ptr)
+{
+	unsigned long flags;
+	int ret;
+
+	spin_lock_irqsave(&r->producer_lock, flags);
+	ret = __ptr_ring_produce(r, ptr);
+	spin_unlock_irqrestore(&r->producer_lock, flags);
+
+	return ret;
+}
+
+static inline int ptr_ring_produce_bh(struct ptr_ring *r, void *ptr)
+{
+	int ret;
+
+	spin_lock_bh(&r->producer_lock);
+	ret = __ptr_ring_produce(r, ptr);
+	spin_unlock_bh(&r->producer_lock);
+
+	return ret;
+}
+
+/* Note: callers invoking this in a loop must use a compiler barrier,
+ * for example cpu_relax(). Callers must take consumer_lock
+ * if they dereference the pointer - see e.g. PTR_RING_PEEK_CALL.
+ * If ring is never resized, and if the pointer is merely
+ * tested, there's no need to take the lock - see e.g.  __ptr_ring_empty.
+ */
+static inline void *__ptr_ring_peek(struct ptr_ring *r)
+{
+	if (likely(r->size))
+		return r->queue[r->consumer_head];
+	return NULL;
+}
+
+/* Note: callers invoking this in a loop must use a compiler barrier,
+ * for example cpu_relax(). Callers must take consumer_lock
+ * if the ring is ever resized - see e.g. ptr_ring_empty.
+ */
+static inline bool __ptr_ring_empty(struct ptr_ring *r)
+{
+	return !__ptr_ring_peek(r);
+}
+
+static inline bool ptr_ring_empty(struct ptr_ring *r)
+{
+	bool ret;
+
+	spin_lock(&r->consumer_lock);
+	ret = __ptr_ring_empty(r);
+	spin_unlock(&r->consumer_lock);
+
+	return ret;
+}
+
+static inline bool ptr_ring_empty_irq(struct ptr_ring *r)
+{
+	bool ret;
+
+	spin_lock_irq(&r->consumer_lock);
+	ret = __ptr_ring_empty(r);
+	spin_unlock_irq(&r->consumer_lock);
+
+	return ret;
+}
+
+static inline bool ptr_ring_empty_any(struct ptr_ring *r)
+{
+	unsigned long flags;
+	bool ret;
+
+	spin_lock_irqsave(&r->consumer_lock, flags);
+	ret = __ptr_ring_empty(r);
+	spin_unlock_irqrestore(&r->consumer_lock, flags);
+
+	return ret;
+}
+
+static inline bool ptr_ring_empty_bh(struct ptr_ring *r)
+{
+	bool ret;
+
+	spin_lock_bh(&r->consumer_lock);
+	ret = __ptr_ring_empty(r);
+	spin_unlock_bh(&r->consumer_lock);
+
+	return ret;
+}
+
+/* Must only be called after __ptr_ring_peek returned !NULL */
+static inline void __ptr_ring_discard_one(struct ptr_ring *r)
+{
+	/* Fundamentally, what we want to do is update consumer
+	 * index and zero out the entry so producer can reuse it.
+	 * Doing it naively at each consume would be as simple as:
+	 *       r->queue[r->consumer++] = NULL;
+	 *       if (unlikely(r->consumer >= r->size))
+	 *               r->consumer = 0;
+	 * but that is suboptimal when the ring is full as producer is writing
+	 * out new entries in the same cache line.  Defer these updates until a
+	 * batch of entries has been consumed.
+	 */
+	int head = r->consumer_head++;
+
+	/* Once we have processed enough entries invalidate them in
+	 * the ring all at once so producer can reuse their space in the ring.
+	 * We also do this when we reach end of the ring - not mandatory
+	 * but helps keep the implementation simple.
+	 */
+	if (unlikely(r->consumer_head - r->consumer_tail >= r->batch ||
+		     r->consumer_head >= r->size)) {
+		/* Zero out entries in the reverse order: this way we touch the
+		 * cache line that producer might currently be reading the last;
+		 * producer won't make progress and touch other cache lines
+		 * besides the first one until we write out all entries.
+		 */
+		while (likely(head >= r->consumer_tail))
+			r->queue[head--] = NULL;
+		r->consumer_tail = r->consumer_head;
+	}
+	if (unlikely(r->consumer_head >= r->size)) {
+		r->consumer_head = 0;
+		r->consumer_tail = 0;
+	}
+}
+
+static inline void *__ptr_ring_consume(struct ptr_ring *r)
+{
+	void *ptr;
+
+	ptr = __ptr_ring_peek(r);
+	if (ptr)
+		__ptr_ring_discard_one(r);
+
+	return ptr;
+}
+
+static inline int __ptr_ring_consume_batched(struct ptr_ring *r,
+					     void **array, int n)
+{
+	void *ptr;
+	int i;
+
+	for (i = 0; i < n; i++) {
+		ptr = __ptr_ring_consume(r);
+		if (!ptr)
+			break;
+		array[i] = ptr;
+	}
+
+	return i;
+}
+
+/*
+ * Note: resize (below) nests producer lock within consumer lock, so if you
+ * call this in interrupt or BH context, you must disable interrupts/BH when
+ * producing.
+ */
+static inline void *ptr_ring_consume(struct ptr_ring *r)
+{
+	void *ptr;
+
+	spin_lock(&r->consumer_lock);
+	ptr = __ptr_ring_consume(r);
+	spin_unlock(&r->consumer_lock);
+
+	return ptr;
+}
+
+static inline void *ptr_ring_consume_irq(struct ptr_ring *r)
+{
+	void *ptr;
+
+	spin_lock_irq(&r->consumer_lock);
+	ptr = __ptr_ring_consume(r);
+	spin_unlock_irq(&r->consumer_lock);
+
+	return ptr;
+}
+
+static inline void *ptr_ring_consume_any(struct ptr_ring *r)
+{
+	unsigned long flags;
+	void *ptr;
+
+	spin_lock_irqsave(&r->consumer_lock, flags);
+	ptr = __ptr_ring_consume(r);
+	spin_unlock_irqrestore(&r->consumer_lock, flags);
+
+	return ptr;
+}
+
+static inline void *ptr_ring_consume_bh(struct ptr_ring *r)
+{
+	void *ptr;
+
+	spin_lock_bh(&r->consumer_lock);
+	ptr = __ptr_ring_consume(r);
+	spin_unlock_bh(&r->consumer_lock);
+
+	return ptr;
+}
+
+static inline int ptr_ring_consume_batched(struct ptr_ring *r,
+					   void **array, int n)
+{
+	int ret;
+
+	spin_lock(&r->consumer_lock);
+	ret = __ptr_ring_consume_batched(r, array, n);
+	spin_unlock(&r->consumer_lock);
+
+	return ret;
+}
+
+static inline int ptr_ring_consume_batched_irq(struct ptr_ring *r,
+					       void **array, int n)
+{
+	int ret;
+
+	spin_lock_irq(&r->consumer_lock);
+	ret = __ptr_ring_consume_batched(r, array, n);
+	spin_unlock_irq(&r->consumer_lock);
+
+	return ret;
+}
+
+static inline int ptr_ring_consume_batched_any(struct ptr_ring *r,
+					       void **array, int n)
+{
+	unsigned long flags;
+	int ret;
+
+	spin_lock_irqsave(&r->consumer_lock, flags);
+	ret = __ptr_ring_consume_batched(r, array, n);
+	spin_unlock_irqrestore(&r->consumer_lock, flags);
+
+	return ret;
+}
+
+static inline int ptr_ring_consume_batched_bh(struct ptr_ring *r,
+					      void **array, int n)
+{
+	int ret;
+
+	spin_lock_bh(&r->consumer_lock);
+	ret = __ptr_ring_consume_batched(r, array, n);
+	spin_unlock_bh(&r->consumer_lock);
+
+	return ret;
+}
+
+/* Cast to structure type and call a function without discarding from FIFO.
+ * Function must return a value.
+ * Callers must take consumer_lock.
+ */
+#define __PTR_RING_PEEK_CALL(r, f) ((f)(__ptr_ring_peek(r)))
+
+#define PTR_RING_PEEK_CALL(r, f) ({ \
+	typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \
+	\
+	spin_lock(&(r)->consumer_lock); \
+	__PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \
+	spin_unlock(&(r)->consumer_lock); \
+	__PTR_RING_PEEK_CALL_v; \
+})
+
+#define PTR_RING_PEEK_CALL_IRQ(r, f) ({ \
+	typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \
+	\
+	spin_lock_irq(&(r)->consumer_lock); \
+	__PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \
+	spin_unlock_irq(&(r)->consumer_lock); \
+	__PTR_RING_PEEK_CALL_v; \
+})
+
+#define PTR_RING_PEEK_CALL_BH(r, f) ({ \
+	typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \
+	\
+	spin_lock_bh(&(r)->consumer_lock); \
+	__PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \
+	spin_unlock_bh(&(r)->consumer_lock); \
+	__PTR_RING_PEEK_CALL_v; \
+})
+
+#define PTR_RING_PEEK_CALL_ANY(r, f) ({ \
+	typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \
+	unsigned long __PTR_RING_PEEK_CALL_f;\
+	\
+	spin_lock_irqsave(&(r)->consumer_lock, __PTR_RING_PEEK_CALL_f); \
+	__PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \
+	spin_unlock_irqrestore(&(r)->consumer_lock, __PTR_RING_PEEK_CALL_f); \
+	__PTR_RING_PEEK_CALL_v; \
+})
+
+static inline void **__ptr_ring_init_queue_alloc(unsigned int size, gfp_t gfp)
+{
+	return kcalloc(size, sizeof(void *), gfp);
+}
+
+static inline void __ptr_ring_set_size(struct ptr_ring *r, int size)
+{
+	r->size = size;
+	r->batch = SMP_CACHE_BYTES * 2 / sizeof(*(r->queue));
+	/* We need to set batch at least to 1 to make logic
+	 * in __ptr_ring_discard_one work correctly.
+	 * Batching too much (because ring is small) would cause a lot of
+	 * burstiness. Needs tuning, for now disable batching.
+	 */
+	if (r->batch > r->size / 2 || !r->batch)
+		r->batch = 1;
+}
+
+static inline int ptr_ring_init(struct ptr_ring *r, int size, gfp_t gfp)
+{
+	r->queue = __ptr_ring_init_queue_alloc(size, gfp);
+	if (!r->queue)
+		return -ENOMEM;
+
+	__ptr_ring_set_size(r, size);
+	r->producer = r->consumer_head = r->consumer_tail = 0;
+	spin_lock_init(&r->producer_lock);
+	spin_lock_init(&r->consumer_lock);
+
+	return 0;
+}
+
+/*
+ * Return entries into ring. Destroy entries that don't fit.
+ *
+ * Note: this is expected to be a rare slow path operation.
+ *
+ * Note: producer lock is nested within consumer lock, so if you
+ * resize you must make sure all uses nest correctly.
+ * In particular if you consume ring in interrupt or BH context, you must
+ * disable interrupts/BH when doing so.
+ */
+static inline void ptr_ring_unconsume(struct ptr_ring *r, void **batch, int n,
+				      void (*destroy)(void *))
+{
+	unsigned long flags;
+	int head;
+
+	spin_lock_irqsave(&r->consumer_lock, flags);
+	spin_lock(&r->producer_lock);
+
+	if (!r->size)
+		goto done;
+
+	/*
+	 * Clean out buffered entries (for simplicity). This way following code
+	 * can test entries for NULL and if not assume they are valid.
+	 */
+	head = r->consumer_head - 1;
+	while (likely(head >= r->consumer_tail))
+		r->queue[head--] = NULL;
+	r->consumer_tail = r->consumer_head;
+
+	/*
+	 * Go over entries in batch, start moving head back and copy entries.
+	 * Stop when we run into previously unconsumed entries.
+	 */
+	while (n) {
+		head = r->consumer_head - 1;
+		if (head < 0)
+			head = r->size - 1;
+		if (r->queue[head]) {
+			/* This batch entry will have to be destroyed. */
+			goto done;
+		}
+		r->queue[head] = batch[--n];
+		r->consumer_tail = r->consumer_head = head;
+	}
+
+done:
+	/* Destroy all entries left in the batch. */
+	while (n)
+		destroy(batch[--n]);
+	spin_unlock(&r->producer_lock);
+	spin_unlock_irqrestore(&r->consumer_lock, flags);
+}
+
+static inline void **__ptr_ring_swap_queue(struct ptr_ring *r, void **queue,
+					   int size, gfp_t gfp,
+					   void (*destroy)(void *))
+{
+	int producer = 0;
+	void **old;
+	void *ptr;
+
+	while ((ptr = __ptr_ring_consume(r)))
+		if (producer < size)
+			queue[producer++] = ptr;
+		else if (destroy)
+			destroy(ptr);
+
+	__ptr_ring_set_size(r, size);
+	r->producer = producer;
+	r->consumer_head = 0;
+	r->consumer_tail = 0;
+	old = r->queue;
+	r->queue = queue;
+
+	return old;
+}
+
+/*
+ * Note: producer lock is nested within consumer lock, so if you
+ * resize you must make sure all uses nest correctly.
+ * In particular if you consume ring in interrupt or BH context, you must
+ * disable interrupts/BH when doing so.
+ */
+static inline int ptr_ring_resize(struct ptr_ring *r, int size, gfp_t gfp,
+				  void (*destroy)(void *))
+{
+	unsigned long flags;
+	void **queue = __ptr_ring_init_queue_alloc(size, gfp);
+	void **old;
+
+	if (!queue)
+		return -ENOMEM;
+
+	spin_lock_irqsave(&(r)->consumer_lock, flags);
+	spin_lock(&(r)->producer_lock);
+
+	old = __ptr_ring_swap_queue(r, queue, size, gfp, destroy);
+
+	spin_unlock(&(r)->producer_lock);
+	spin_unlock_irqrestore(&(r)->consumer_lock, flags);
+
+	kfree(old);
+
+	return 0;
+}
+
+/*
+ * Note: producer lock is nested within consumer lock, so if you
+ * resize you must make sure all uses nest correctly.
+ * In particular if you consume ring in interrupt or BH context, you must
+ * disable interrupts/BH when doing so.
+ */
+static inline int ptr_ring_resize_multiple(struct ptr_ring **rings,
+					   unsigned int nrings,
+					   int size,
+					   gfp_t gfp, void (*destroy)(void *))
+{
+	unsigned long flags;
+	void ***queues;
+	int i;
+
+	queues = kmalloc_array(nrings, sizeof(*queues), gfp);
+	if (!queues)
+		goto noqueues;
+
+	for (i = 0; i < nrings; ++i) {
+		queues[i] = __ptr_ring_init_queue_alloc(size, gfp);
+		if (!queues[i])
+			goto nomem;
+	}
+
+	for (i = 0; i < nrings; ++i) {
+		spin_lock_irqsave(&(rings[i])->consumer_lock, flags);
+		spin_lock(&(rings[i])->producer_lock);
+		queues[i] = __ptr_ring_swap_queue(rings[i], queues[i],
+						  size, gfp, destroy);
+		spin_unlock(&(rings[i])->producer_lock);
+		spin_unlock_irqrestore(&(rings[i])->consumer_lock, flags);
+	}
+
+	for (i = 0; i < nrings; ++i)
+		kfree(queues[i]);
+
+	kfree(queues);
+
+	return 0;
+
+nomem:
+	while (--i >= 0)
+		kfree(queues[i]);
+
+	kfree(queues);
+
+noqueues:
+	return -ENOMEM;
+}
+
+static inline void ptr_ring_cleanup(struct ptr_ring *r, void (*destroy)(void *))
+{
+	void *ptr;
+
+	if (destroy)
+		while ((ptr = ptr_ring_consume(r)))
+			destroy(ptr);
+	kfree(r->queue);
+}
+
+#endif /* _LINUX_PTR_RING_H  */
diff -Naur --no-dereference a/net/wireguard/compat/simd/include/linux/simd.h b/net/wireguard/compat/simd/include/linux/simd.h
--- a/net/wireguard/compat/simd/include/linux/simd.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/compat/simd/include/linux/simd.h	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,70 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#ifndef _WG_SIMD_H
+#define _WG_SIMD_H
+
+#include <linux/sched.h>
+#include <asm/simd.h>
+#if defined(CONFIG_X86_64)
+#include <linux/version.h>
+#include <asm/fpu/api.h>
+#elif defined(CONFIG_KERNEL_MODE_NEON)
+#include <asm/neon.h>
+#endif
+
+typedef enum {
+	HAVE_NO_SIMD = 1 << 0,
+	HAVE_FULL_SIMD = 1 << 1,
+	HAVE_SIMD_IN_USE = 1 << 31
+} simd_context_t;
+
+#define DONT_USE_SIMD ((simd_context_t []){ HAVE_NO_SIMD })
+
+static inline void simd_get(simd_context_t *ctx)
+{
+	*ctx = !IS_ENABLED(CONFIG_PREEMPT_RT_BASE) && may_use_simd() ? HAVE_FULL_SIMD : HAVE_NO_SIMD;
+}
+
+static inline void simd_put(simd_context_t *ctx)
+{
+#if defined(CONFIG_X86_64)
+	if (*ctx & HAVE_SIMD_IN_USE)
+		kernel_fpu_end();
+#elif defined(CONFIG_KERNEL_MODE_NEON)
+	if (*ctx & HAVE_SIMD_IN_USE)
+		kernel_neon_end();
+#endif
+	*ctx = HAVE_NO_SIMD;
+}
+
+static inline bool simd_relax(simd_context_t *ctx)
+{
+#ifdef CONFIG_PREEMPT
+	if ((*ctx & HAVE_SIMD_IN_USE) && need_resched()) {
+		simd_put(ctx);
+		simd_get(ctx);
+		return true;
+	}
+#endif
+	return false;
+}
+
+static __must_check inline bool simd_use(simd_context_t *ctx)
+{
+	if (!(*ctx & HAVE_FULL_SIMD))
+		return false;
+	if (*ctx & HAVE_SIMD_IN_USE)
+		return true;
+#if defined(CONFIG_X86_64)
+	kernel_fpu_begin();
+#elif defined(CONFIG_KERNEL_MODE_NEON)
+	kernel_neon_begin();
+#endif
+	*ctx |= HAVE_SIMD_IN_USE;
+	return true;
+}
+
+#endif /* _WG_SIMD_H */
diff -Naur --no-dereference a/net/wireguard/compat/simd-asm/include/asm/simd.h b/net/wireguard/compat/simd-asm/include/asm/simd.h
--- a/net/wireguard/compat/simd-asm/include/asm/simd.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/compat/simd-asm/include/asm/simd.h	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,21 @@
+#ifndef _COMPAT_ASM_SIMD_H
+#define _COMPAT_ASM_SIMD_H
+
+#if defined(CONFIG_X86_64)
+#include <asm/fpu/api.h>
+#endif
+
+static __must_check inline bool may_use_simd(void)
+{
+#if defined(CONFIG_X86_64)
+	return irq_fpu_usable();
+#elif defined(CONFIG_ARM64) && defined(CONFIG_KERNEL_MODE_NEON)
+	return true;
+#elif defined(CONFIG_ARM) && defined(CONFIG_KERNEL_MODE_NEON)
+	return !in_nmi() && !in_irq() && !in_serving_softirq();
+#else
+	return false;
+#endif
+}
+
+#endif
diff -Naur --no-dereference a/net/wireguard/compat/siphash/include/linux/siphash.h b/net/wireguard/compat/siphash/include/linux/siphash.h
--- a/net/wireguard/compat/siphash/include/linux/siphash.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/compat/siphash/include/linux/siphash.h	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,140 @@
+/* Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ *
+ * This file is provided under a dual BSD/GPLv2 license.
+ *
+ * SipHash: a fast short-input PRF
+ * https://131002.net/siphash/
+ *
+ * This implementation is specifically for SipHash2-4 for a secure PRF
+ * and HalfSipHash1-3/SipHash1-3 for an insecure PRF only suitable for
+ * hashtables.
+ */
+
+#ifndef _WG_LINUX_SIPHASH_H
+#define _WG_LINUX_SIPHASH_H
+
+#include <linux/types.h>
+#include <linux/kernel.h>
+
+#define SIPHASH_ALIGNMENT __alignof__(u64)
+typedef struct {
+	u64 key[2];
+} siphash_key_t;
+
+u64 __siphash_aligned(const void *data, size_t len, const siphash_key_t *key);
+#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+u64 __siphash_unaligned(const void *data, size_t len, const siphash_key_t *key);
+#endif
+
+u64 siphash_1u64(const u64 a, const siphash_key_t *key);
+u64 siphash_2u64(const u64 a, const u64 b, const siphash_key_t *key);
+u64 siphash_3u64(const u64 a, const u64 b, const u64 c,
+		 const siphash_key_t *key);
+u64 siphash_4u64(const u64 a, const u64 b, const u64 c, const u64 d,
+		 const siphash_key_t *key);
+u64 siphash_1u32(const u32 a, const siphash_key_t *key);
+u64 siphash_3u32(const u32 a, const u32 b, const u32 c,
+		 const siphash_key_t *key);
+
+static inline u64 siphash_2u32(const u32 a, const u32 b,
+			       const siphash_key_t *key)
+{
+	return siphash_1u64((u64)b << 32 | a, key);
+}
+static inline u64 siphash_4u32(const u32 a, const u32 b, const u32 c,
+			       const u32 d, const siphash_key_t *key)
+{
+	return siphash_2u64((u64)b << 32 | a, (u64)d << 32 | c, key);
+}
+
+
+static inline u64 ___siphash_aligned(const __le64 *data, size_t len,
+				     const siphash_key_t *key)
+{
+	if (__builtin_constant_p(len) && len == 4)
+		return siphash_1u32(le32_to_cpup((const __le32 *)data), key);
+	if (__builtin_constant_p(len) && len == 8)
+		return siphash_1u64(le64_to_cpu(data[0]), key);
+	if (__builtin_constant_p(len) && len == 16)
+		return siphash_2u64(le64_to_cpu(data[0]), le64_to_cpu(data[1]),
+				    key);
+	if (__builtin_constant_p(len) && len == 24)
+		return siphash_3u64(le64_to_cpu(data[0]), le64_to_cpu(data[1]),
+				    le64_to_cpu(data[2]), key);
+	if (__builtin_constant_p(len) && len == 32)
+		return siphash_4u64(le64_to_cpu(data[0]), le64_to_cpu(data[1]),
+				    le64_to_cpu(data[2]), le64_to_cpu(data[3]),
+				    key);
+	return __siphash_aligned(data, len, key);
+}
+
+/**
+ * siphash - compute 64-bit siphash PRF value
+ * @data: buffer to hash
+ * @size: size of @data
+ * @key: the siphash key
+ */
+static inline u64 siphash(const void *data, size_t len,
+			  const siphash_key_t *key)
+{
+#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+	if (!IS_ALIGNED((unsigned long)data, SIPHASH_ALIGNMENT))
+		return __siphash_unaligned(data, len, key);
+#endif
+	return ___siphash_aligned(data, len, key);
+}
+
+#define HSIPHASH_ALIGNMENT __alignof__(unsigned long)
+typedef struct {
+	unsigned long key[2];
+} hsiphash_key_t;
+
+u32 __hsiphash_aligned(const void *data, size_t len,
+		       const hsiphash_key_t *key);
+#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+u32 __hsiphash_unaligned(const void *data, size_t len,
+			 const hsiphash_key_t *key);
+#endif
+
+u32 hsiphash_1u32(const u32 a, const hsiphash_key_t *key);
+u32 hsiphash_2u32(const u32 a, const u32 b, const hsiphash_key_t *key);
+u32 hsiphash_3u32(const u32 a, const u32 b, const u32 c,
+		  const hsiphash_key_t *key);
+u32 hsiphash_4u32(const u32 a, const u32 b, const u32 c, const u32 d,
+		  const hsiphash_key_t *key);
+
+static inline u32 ___hsiphash_aligned(const __le32 *data, size_t len,
+				      const hsiphash_key_t *key)
+{
+	if (__builtin_constant_p(len) && len == 4)
+		return hsiphash_1u32(le32_to_cpu(data[0]), key);
+	if (__builtin_constant_p(len) && len == 8)
+		return hsiphash_2u32(le32_to_cpu(data[0]), le32_to_cpu(data[1]),
+				     key);
+	if (__builtin_constant_p(len) && len == 12)
+		return hsiphash_3u32(le32_to_cpu(data[0]), le32_to_cpu(data[1]),
+				     le32_to_cpu(data[2]), key);
+	if (__builtin_constant_p(len) && len == 16)
+		return hsiphash_4u32(le32_to_cpu(data[0]), le32_to_cpu(data[1]),
+				     le32_to_cpu(data[2]), le32_to_cpu(data[3]),
+				     key);
+	return __hsiphash_aligned(data, len, key);
+}
+
+/**
+ * hsiphash - compute 32-bit hsiphash PRF value
+ * @data: buffer to hash
+ * @size: size of @data
+ * @key: the hsiphash key
+ */
+static inline u32 hsiphash(const void *data, size_t len,
+			   const hsiphash_key_t *key)
+{
+#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+	if (!IS_ALIGNED((unsigned long)data, HSIPHASH_ALIGNMENT))
+		return __hsiphash_unaligned(data, len, key);
+#endif
+	return ___hsiphash_aligned(data, len, key);
+}
+
+#endif /* _WG_LINUX_SIPHASH_H */
diff -Naur --no-dereference a/net/wireguard/compat/siphash/siphash.c b/net/wireguard/compat/siphash/siphash.c
--- a/net/wireguard/compat/siphash/siphash.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/compat/siphash/siphash.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,539 @@
+/* Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ *
+ * This file is provided under a dual BSD/GPLv2 license.
+ *
+ * SipHash: a fast short-input PRF
+ * https://131002.net/siphash/
+ *
+ * This implementation is specifically for SipHash2-4 for a secure PRF
+ * and HalfSipHash1-3/SipHash1-3 for an insecure PRF only suitable for
+ * hashtables.
+ */
+
+#include <linux/siphash.h>
+#include <asm/unaligned.h>
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0)
+#ifdef __LITTLE_ENDIAN
+#define bytemask_from_count(cnt)	(~(~0ul << (cnt)*8))
+#else
+#define bytemask_from_count(cnt)	(~(~0ul >> (cnt)*8))
+#endif
+#endif
+
+#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64
+#include <linux/dcache.h>
+#include <asm/word-at-a-time.h>
+#endif
+
+#define SIPROUND \
+	do { \
+	v0 += v1; v1 = rol64(v1, 13); v1 ^= v0; v0 = rol64(v0, 32); \
+	v2 += v3; v3 = rol64(v3, 16); v3 ^= v2; \
+	v0 += v3; v3 = rol64(v3, 21); v3 ^= v0; \
+	v2 += v1; v1 = rol64(v1, 17); v1 ^= v2; v2 = rol64(v2, 32); \
+	} while (0)
+
+#define PREAMBLE(len) \
+	u64 v0 = 0x736f6d6570736575ULL; \
+	u64 v1 = 0x646f72616e646f6dULL; \
+	u64 v2 = 0x6c7967656e657261ULL; \
+	u64 v3 = 0x7465646279746573ULL; \
+	u64 b = ((u64)(len)) << 56; \
+	v3 ^= key->key[1]; \
+	v2 ^= key->key[0]; \
+	v1 ^= key->key[1]; \
+	v0 ^= key->key[0];
+
+#define POSTAMBLE \
+	v3 ^= b; \
+	SIPROUND; \
+	SIPROUND; \
+	v0 ^= b; \
+	v2 ^= 0xff; \
+	SIPROUND; \
+	SIPROUND; \
+	SIPROUND; \
+	SIPROUND; \
+	return (v0 ^ v1) ^ (v2 ^ v3);
+
+u64 __siphash_aligned(const void *data, size_t len, const siphash_key_t *key)
+{
+	const u8 *end = data + len - (len % sizeof(u64));
+	const u8 left = len & (sizeof(u64) - 1);
+	u64 m;
+	PREAMBLE(len)
+	for (; data != end; data += sizeof(u64)) {
+		m = le64_to_cpup(data);
+		v3 ^= m;
+		SIPROUND;
+		SIPROUND;
+		v0 ^= m;
+	}
+#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64
+	if (left)
+		b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) &
+						  bytemask_from_count(left)));
+#else
+	switch (left) {
+	case 7: b |= ((u64)end[6]) << 48;
+	case 6: b |= ((u64)end[5]) << 40;
+	case 5: b |= ((u64)end[4]) << 32;
+	case 4: b |= le32_to_cpup(data); break;
+	case 3: b |= ((u64)end[2]) << 16;
+	case 2: b |= le16_to_cpup(data); break;
+	case 1: b |= end[0];
+	}
+#endif
+	POSTAMBLE
+}
+
+#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+u64 __siphash_unaligned(const void *data, size_t len, const siphash_key_t *key)
+{
+	const u8 *end = data + len - (len % sizeof(u64));
+	const u8 left = len & (sizeof(u64) - 1);
+	u64 m;
+	PREAMBLE(len)
+	for (; data != end; data += sizeof(u64)) {
+		m = get_unaligned_le64(data);
+		v3 ^= m;
+		SIPROUND;
+		SIPROUND;
+		v0 ^= m;
+	}
+#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64
+	if (left)
+		b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) &
+						  bytemask_from_count(left)));
+#else
+	switch (left) {
+	case 7: b |= ((u64)end[6]) << 48;
+	case 6: b |= ((u64)end[5]) << 40;
+	case 5: b |= ((u64)end[4]) << 32;
+	case 4: b |= get_unaligned_le32(end); break;
+	case 3: b |= ((u64)end[2]) << 16;
+	case 2: b |= get_unaligned_le16(end); break;
+	case 1: b |= end[0];
+	}
+#endif
+	POSTAMBLE
+}
+#endif
+
+/**
+ * siphash_1u64 - compute 64-bit siphash PRF value of a u64
+ * @first: first u64
+ * @key: the siphash key
+ */
+u64 siphash_1u64(const u64 first, const siphash_key_t *key)
+{
+	PREAMBLE(8)
+	v3 ^= first;
+	SIPROUND;
+	SIPROUND;
+	v0 ^= first;
+	POSTAMBLE
+}
+
+/**
+ * siphash_2u64 - compute 64-bit siphash PRF value of 2 u64
+ * @first: first u64
+ * @second: second u64
+ * @key: the siphash key
+ */
+u64 siphash_2u64(const u64 first, const u64 second, const siphash_key_t *key)
+{
+	PREAMBLE(16)
+	v3 ^= first;
+	SIPROUND;
+	SIPROUND;
+	v0 ^= first;
+	v3 ^= second;
+	SIPROUND;
+	SIPROUND;
+	v0 ^= second;
+	POSTAMBLE
+}
+
+/**
+ * siphash_3u64 - compute 64-bit siphash PRF value of 3 u64
+ * @first: first u64
+ * @second: second u64
+ * @third: third u64
+ * @key: the siphash key
+ */
+u64 siphash_3u64(const u64 first, const u64 second, const u64 third,
+		 const siphash_key_t *key)
+{
+	PREAMBLE(24)
+	v3 ^= first;
+	SIPROUND;
+	SIPROUND;
+	v0 ^= first;
+	v3 ^= second;
+	SIPROUND;
+	SIPROUND;
+	v0 ^= second;
+	v3 ^= third;
+	SIPROUND;
+	SIPROUND;
+	v0 ^= third;
+	POSTAMBLE
+}
+
+/**
+ * siphash_4u64 - compute 64-bit siphash PRF value of 4 u64
+ * @first: first u64
+ * @second: second u64
+ * @third: third u64
+ * @forth: forth u64
+ * @key: the siphash key
+ */
+u64 siphash_4u64(const u64 first, const u64 second, const u64 third,
+		 const u64 forth, const siphash_key_t *key)
+{
+	PREAMBLE(32)
+	v3 ^= first;
+	SIPROUND;
+	SIPROUND;
+	v0 ^= first;
+	v3 ^= second;
+	SIPROUND;
+	SIPROUND;
+	v0 ^= second;
+	v3 ^= third;
+	SIPROUND;
+	SIPROUND;
+	v0 ^= third;
+	v3 ^= forth;
+	SIPROUND;
+	SIPROUND;
+	v0 ^= forth;
+	POSTAMBLE
+}
+
+u64 siphash_1u32(const u32 first, const siphash_key_t *key)
+{
+	PREAMBLE(4)
+	b |= first;
+	POSTAMBLE
+}
+
+u64 siphash_3u32(const u32 first, const u32 second, const u32 third,
+		 const siphash_key_t *key)
+{
+	u64 combined = (u64)second << 32 | first;
+	PREAMBLE(12)
+	v3 ^= combined;
+	SIPROUND;
+	SIPROUND;
+	v0 ^= combined;
+	b |= third;
+	POSTAMBLE
+}
+
+#if BITS_PER_LONG == 64
+/* Note that on 64-bit, we make HalfSipHash1-3 actually be SipHash1-3, for
+ * performance reasons. On 32-bit, below, we actually implement HalfSipHash1-3.
+ */
+
+#define HSIPROUND SIPROUND
+#define HPREAMBLE(len) PREAMBLE(len)
+#define HPOSTAMBLE \
+	v3 ^= b; \
+	HSIPROUND; \
+	v0 ^= b; \
+	v2 ^= 0xff; \
+	HSIPROUND; \
+	HSIPROUND; \
+	HSIPROUND; \
+	return (v0 ^ v1) ^ (v2 ^ v3);
+
+u32 __hsiphash_aligned(const void *data, size_t len, const hsiphash_key_t *key)
+{
+	const u8 *end = data + len - (len % sizeof(u64));
+	const u8 left = len & (sizeof(u64) - 1);
+	u64 m;
+	HPREAMBLE(len)
+	for (; data != end; data += sizeof(u64)) {
+		m = le64_to_cpup(data);
+		v3 ^= m;
+		HSIPROUND;
+		v0 ^= m;
+	}
+#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64
+	if (left)
+		b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) &
+						  bytemask_from_count(left)));
+#else
+	switch (left) {
+	case 7: b |= ((u64)end[6]) << 48;
+	case 6: b |= ((u64)end[5]) << 40;
+	case 5: b |= ((u64)end[4]) << 32;
+	case 4: b |= le32_to_cpup(data); break;
+	case 3: b |= ((u64)end[2]) << 16;
+	case 2: b |= le16_to_cpup(data); break;
+	case 1: b |= end[0];
+	}
+#endif
+	HPOSTAMBLE
+}
+
+#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+u32 __hsiphash_unaligned(const void *data, size_t len,
+			 const hsiphash_key_t *key)
+{
+	const u8 *end = data + len - (len % sizeof(u64));
+	const u8 left = len & (sizeof(u64) - 1);
+	u64 m;
+	HPREAMBLE(len)
+	for (; data != end; data += sizeof(u64)) {
+		m = get_unaligned_le64(data);
+		v3 ^= m;
+		HSIPROUND;
+		v0 ^= m;
+	}
+#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64
+	if (left)
+		b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) &
+						  bytemask_from_count(left)));
+#else
+	switch (left) {
+	case 7: b |= ((u64)end[6]) << 48;
+	case 6: b |= ((u64)end[5]) << 40;
+	case 5: b |= ((u64)end[4]) << 32;
+	case 4: b |= get_unaligned_le32(end); break;
+	case 3: b |= ((u64)end[2]) << 16;
+	case 2: b |= get_unaligned_le16(end); break;
+	case 1: b |= end[0];
+	}
+#endif
+	HPOSTAMBLE
+}
+#endif
+
+/**
+ * hsiphash_1u32 - compute 64-bit hsiphash PRF value of a u32
+ * @first: first u32
+ * @key: the hsiphash key
+ */
+u32 hsiphash_1u32(const u32 first, const hsiphash_key_t *key)
+{
+	HPREAMBLE(4)
+	b |= first;
+	HPOSTAMBLE
+}
+
+/**
+ * hsiphash_2u32 - compute 32-bit hsiphash PRF value of 2 u32
+ * @first: first u32
+ * @second: second u32
+ * @key: the hsiphash key
+ */
+u32 hsiphash_2u32(const u32 first, const u32 second, const hsiphash_key_t *key)
+{
+	u64 combined = (u64)second << 32 | first;
+	HPREAMBLE(8)
+	v3 ^= combined;
+	HSIPROUND;
+	v0 ^= combined;
+	HPOSTAMBLE
+}
+
+/**
+ * hsiphash_3u32 - compute 32-bit hsiphash PRF value of 3 u32
+ * @first: first u32
+ * @second: second u32
+ * @third: third u32
+ * @key: the hsiphash key
+ */
+u32 hsiphash_3u32(const u32 first, const u32 second, const u32 third,
+		  const hsiphash_key_t *key)
+{
+	u64 combined = (u64)second << 32 | first;
+	HPREAMBLE(12)
+	v3 ^= combined;
+	HSIPROUND;
+	v0 ^= combined;
+	b |= third;
+	HPOSTAMBLE
+}
+
+/**
+ * hsiphash_4u32 - compute 32-bit hsiphash PRF value of 4 u32
+ * @first: first u32
+ * @second: second u32
+ * @third: third u32
+ * @forth: forth u32
+ * @key: the hsiphash key
+ */
+u32 hsiphash_4u32(const u32 first, const u32 second, const u32 third,
+		  const u32 forth, const hsiphash_key_t *key)
+{
+	u64 combined = (u64)second << 32 | first;
+	HPREAMBLE(16)
+	v3 ^= combined;
+	HSIPROUND;
+	v0 ^= combined;
+	combined = (u64)forth << 32 | third;
+	v3 ^= combined;
+	HSIPROUND;
+	v0 ^= combined;
+	HPOSTAMBLE
+}
+#else
+#define HSIPROUND \
+	do { \
+	v0 += v1; v1 = rol32(v1, 5); v1 ^= v0; v0 = rol32(v0, 16); \
+	v2 += v3; v3 = rol32(v3, 8); v3 ^= v2; \
+	v0 += v3; v3 = rol32(v3, 7); v3 ^= v0; \
+	v2 += v1; v1 = rol32(v1, 13); v1 ^= v2; v2 = rol32(v2, 16); \
+	} while (0)
+
+#define HPREAMBLE(len) \
+	u32 v0 = 0; \
+	u32 v1 = 0; \
+	u32 v2 = 0x6c796765U; \
+	u32 v3 = 0x74656462U; \
+	u32 b = ((u32)(len)) << 24; \
+	v3 ^= key->key[1]; \
+	v2 ^= key->key[0]; \
+	v1 ^= key->key[1]; \
+	v0 ^= key->key[0];
+
+#define HPOSTAMBLE \
+	v3 ^= b; \
+	HSIPROUND; \
+	v0 ^= b; \
+	v2 ^= 0xff; \
+	HSIPROUND; \
+	HSIPROUND; \
+	HSIPROUND; \
+	return v1 ^ v3;
+
+u32 __hsiphash_aligned(const void *data, size_t len, const hsiphash_key_t *key)
+{
+	const u8 *end = data + len - (len % sizeof(u32));
+	const u8 left = len & (sizeof(u32) - 1);
+	u32 m;
+	HPREAMBLE(len)
+	for (; data != end; data += sizeof(u32)) {
+		m = le32_to_cpup(data);
+		v3 ^= m;
+		HSIPROUND;
+		v0 ^= m;
+	}
+	switch (left) {
+	case 3: b |= ((u32)end[2]) << 16;
+	case 2: b |= le16_to_cpup(data); break;
+	case 1: b |= end[0];
+	}
+	HPOSTAMBLE
+}
+
+#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+u32 __hsiphash_unaligned(const void *data, size_t len,
+			 const hsiphash_key_t *key)
+{
+	const u8 *end = data + len - (len % sizeof(u32));
+	const u8 left = len & (sizeof(u32) - 1);
+	u32 m;
+	HPREAMBLE(len)
+	for (; data != end; data += sizeof(u32)) {
+		m = get_unaligned_le32(data);
+		v3 ^= m;
+		HSIPROUND;
+		v0 ^= m;
+	}
+	switch (left) {
+	case 3: b |= ((u32)end[2]) << 16;
+	case 2: b |= get_unaligned_le16(end); break;
+	case 1: b |= end[0];
+	}
+	HPOSTAMBLE
+}
+#endif
+
+/**
+ * hsiphash_1u32 - compute 32-bit hsiphash PRF value of a u32
+ * @first: first u32
+ * @key: the hsiphash key
+ */
+u32 hsiphash_1u32(const u32 first, const hsiphash_key_t *key)
+{
+	HPREAMBLE(4)
+	v3 ^= first;
+	HSIPROUND;
+	v0 ^= first;
+	HPOSTAMBLE
+}
+
+/**
+ * hsiphash_2u32 - compute 32-bit hsiphash PRF value of 2 u32
+ * @first: first u32
+ * @second: second u32
+ * @key: the hsiphash key
+ */
+u32 hsiphash_2u32(const u32 first, const u32 second, const hsiphash_key_t *key)
+{
+	HPREAMBLE(8)
+	v3 ^= first;
+	HSIPROUND;
+	v0 ^= first;
+	v3 ^= second;
+	HSIPROUND;
+	v0 ^= second;
+	HPOSTAMBLE
+}
+
+/**
+ * hsiphash_3u32 - compute 32-bit hsiphash PRF value of 3 u32
+ * @first: first u32
+ * @second: second u32
+ * @third: third u32
+ * @key: the hsiphash key
+ */
+u32 hsiphash_3u32(const u32 first, const u32 second, const u32 third,
+		  const hsiphash_key_t *key)
+{
+	HPREAMBLE(12)
+	v3 ^= first;
+	HSIPROUND;
+	v0 ^= first;
+	v3 ^= second;
+	HSIPROUND;
+	v0 ^= second;
+	v3 ^= third;
+	HSIPROUND;
+	v0 ^= third;
+	HPOSTAMBLE
+}
+
+/**
+ * hsiphash_4u32 - compute 32-bit hsiphash PRF value of 4 u32
+ * @first: first u32
+ * @second: second u32
+ * @third: third u32
+ * @forth: forth u32
+ * @key: the hsiphash key
+ */
+u32 hsiphash_4u32(const u32 first, const u32 second, const u32 third,
+		  const u32 forth, const hsiphash_key_t *key)
+{
+	HPREAMBLE(16)
+	v3 ^= first;
+	HSIPROUND;
+	v0 ^= first;
+	v3 ^= second;
+	HSIPROUND;
+	v0 ^= second;
+	v3 ^= third;
+	HSIPROUND;
+	v0 ^= third;
+	v3 ^= forth;
+	HSIPROUND;
+	v0 ^= forth;
+	HPOSTAMBLE
+}
+#endif
diff -Naur --no-dereference a/net/wireguard/compat/udp_tunnel/include/net/udp_tunnel.h b/net/wireguard/compat/udp_tunnel/include/net/udp_tunnel.h
--- a/net/wireguard/compat/udp_tunnel/include/net/udp_tunnel.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/compat/udp_tunnel/include/net/udp_tunnel.h	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,94 @@
+#ifndef _WG_NET_UDP_TUNNEL_H
+#define _WG_NET_UDP_TUNNEL_H
+
+#include <net/ip_tunnels.h>
+#include <net/udp.h>
+
+#if IS_ENABLED(CONFIG_IPV6)
+#include <net/ipv6.h>
+#include <net/addrconf.h>
+#endif
+
+struct udp_port_cfg {
+	u8			family;
+
+	/* Used only for kernel-created sockets */
+	union {
+		struct in_addr		local_ip;
+#if IS_ENABLED(CONFIG_IPV6)
+		struct in6_addr		local_ip6;
+#endif
+	};
+
+	union {
+		struct in_addr		peer_ip;
+#if IS_ENABLED(CONFIG_IPV6)
+		struct in6_addr		peer_ip6;
+#endif
+	};
+
+	__be16			local_udp_port;
+	__be16			peer_udp_port;
+	unsigned int		use_udp_checksums:1,
+				use_udp6_tx_checksums:1,
+				use_udp6_rx_checksums:1,
+				ipv6_v6only:1;
+};
+
+int udp_sock_create4(struct net *net, struct udp_port_cfg *cfg,
+		     struct socket **sockp);
+
+#if IS_ENABLED(CONFIG_IPV6)
+int udp_sock_create6(struct net *net, struct udp_port_cfg *cfg,
+		     struct socket **sockp);
+#else
+static inline int udp_sock_create6(struct net *net, struct udp_port_cfg *cfg,
+				   struct socket **sockp)
+{
+	return 0;
+}
+#endif
+
+static inline int udp_sock_create(struct net *net,
+				  struct udp_port_cfg *cfg,
+				  struct socket **sockp)
+{
+	if (cfg->family == AF_INET)
+		return udp_sock_create4(net, cfg, sockp);
+
+	if (cfg->family == AF_INET6)
+		return udp_sock_create6(net, cfg, sockp);
+
+	return -EPFNOSUPPORT;
+}
+
+typedef int (*udp_tunnel_encap_rcv_t)(struct sock *sk, struct sk_buff *skb);
+
+struct udp_tunnel_sock_cfg {
+	void *sk_user_data;
+	__u8  encap_type;
+	udp_tunnel_encap_rcv_t encap_rcv;
+};
+
+/* Setup the given (UDP) sock to receive UDP encapsulated packets */
+void setup_udp_tunnel_sock(struct net *net, struct socket *sock,
+			   struct udp_tunnel_sock_cfg *sock_cfg);
+
+/* Transmit the skb using UDP encapsulation. */
+void udp_tunnel_xmit_skb(struct rtable *rt, struct sock *sk, struct sk_buff *skb,
+			 __be32 src, __be32 dst, __u8 tos, __u8 ttl,
+			 __be16 df, __be16 src_port, __be16 dst_port,
+			 bool xnet, bool nocheck);
+
+#if IS_ENABLED(CONFIG_IPV6)
+int udp_tunnel6_xmit_skb(struct dst_entry *dst, struct sock *sk,
+			 struct sk_buff *skb,
+			 struct net_device *dev, struct in6_addr *saddr,
+			 struct in6_addr *daddr,
+			 __u8 prio, __u8 ttl, __be32 label,
+			 __be16 src_port, __be16 dst_port, bool nocheck);
+#endif
+
+void udp_tunnel_sock_release(struct socket *sock);
+
+#endif /* _WG_NET_UDP_TUNNEL_H */
diff -Naur --no-dereference a/net/wireguard/compat/udp_tunnel/udp_tunnel.c b/net/wireguard/compat/udp_tunnel/udp_tunnel.c
--- a/net/wireguard/compat/udp_tunnel/udp_tunnel.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/compat/udp_tunnel/udp_tunnel.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,387 @@
+#include <linux/module.h>
+#include <linux/errno.h>
+#include <linux/socket.h>
+#include <linux/udp.h>
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <net/net_namespace.h>
+#include <net/inet_common.h>
+#include <net/udp.h>
+#include <net/udp_tunnel.h>
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0)
+#define __sk_user_data(sk) ((*((void __rcu **)&(sk)->sk_user_data)))
+#define rcu_dereference_sk_user_data(sk) rcu_dereference(__sk_user_data((sk)))
+#define rcu_assign_sk_user_data(sk, ptr) rcu_assign_pointer(__sk_user_data((sk)), ptr)
+#endif
+
+/* This is global so, uh, only one real call site... This is the kind of horrific hack you'd expect to see in compat code. */
+static udp_tunnel_encap_rcv_t encap_rcv = NULL;
+static void __compat_sk_data_ready(struct sock *sk
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0)
+			      ,int unused_vulnerable_length_param
+#endif
+			      )
+{
+	struct sk_buff *skb;
+	while ((skb = skb_dequeue(&sk->sk_receive_queue)) != NULL) {
+		skb_orphan(skb);
+		sk_mem_reclaim(sk);
+		encap_rcv(sk, skb);
+	}
+}
+
+int udp_sock_create4(struct net *net, struct udp_port_cfg *cfg,
+		     struct socket **sockp)
+{
+	int err;
+	struct socket *sock = NULL;
+	struct sockaddr_in udp_addr;
+
+	err = __sock_create(net, AF_INET, SOCK_DGRAM, 0, &sock, 1);
+	if (err < 0)
+		goto error;
+
+	udp_addr.sin_family = AF_INET;
+	udp_addr.sin_addr = cfg->local_ip;
+	udp_addr.sin_port = cfg->local_udp_port;
+	err = kernel_bind(sock, (struct sockaddr *)&udp_addr,
+			  sizeof(udp_addr));
+	if (err < 0)
+		goto error;
+
+	if (cfg->peer_udp_port) {
+		udp_addr.sin_family = AF_INET;
+		udp_addr.sin_addr = cfg->peer_ip;
+		udp_addr.sin_port = cfg->peer_udp_port;
+		err = kernel_connect(sock, (struct sockaddr *)&udp_addr,
+				     sizeof(udp_addr), 0);
+		if (err < 0)
+			goto error;
+	}
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0)
+	sock->sk->sk_no_check = !cfg->use_udp_checksums;
+#else
+	sock->sk->sk_no_check_tx = !cfg->use_udp_checksums;
+#endif
+
+	*sockp = sock;
+	return 0;
+
+error:
+	if (sock) {
+		kernel_sock_shutdown(sock, SHUT_RDWR);
+		sock_release(sock);
+	}
+	*sockp = NULL;
+	return err;
+}
+
+void setup_udp_tunnel_sock(struct net *net, struct socket *sock,
+			   struct udp_tunnel_sock_cfg *cfg)
+{
+	inet_sk(sock->sk)->mc_loop = 0;
+	encap_rcv = cfg->encap_rcv;
+	rcu_assign_sk_user_data(sock->sk, cfg->sk_user_data);
+	/* We force the cast in this awful way, due to various Android kernels
+	 * backporting things stupidly. */
+	*(void **)&sock->sk->sk_data_ready = (void *)__compat_sk_data_ready;
+}
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0)
+static inline __sum16 udp_v4_check(int len, __be32 saddr,
+				   __be32 daddr, __wsum base)
+{
+	return csum_tcpudp_magic(saddr, daddr, len, IPPROTO_UDP, base);
+}
+
+static void udp_set_csum(bool nocheck, struct sk_buff *skb,
+		  __be32 saddr, __be32 daddr, int len)
+{
+	struct udphdr *uh = udp_hdr(skb);
+
+	if (nocheck)
+		uh->check = 0;
+	else if (skb_is_gso(skb))
+		uh->check = ~udp_v4_check(len, saddr, daddr, 0);
+	else if (skb_dst(skb) && skb_dst(skb)->dev &&
+		 (skb_dst(skb)->dev->features & NETIF_F_V4_CSUM)) {
+
+		BUG_ON(skb->ip_summed == CHECKSUM_PARTIAL);
+
+		skb->ip_summed = CHECKSUM_PARTIAL;
+		skb->csum_start = skb_transport_header(skb) - skb->head;
+		skb->csum_offset = offsetof(struct udphdr, check);
+		uh->check = ~udp_v4_check(len, saddr, daddr, 0);
+	} else {
+		__wsum csum;
+
+		BUG_ON(skb->ip_summed == CHECKSUM_PARTIAL);
+
+		uh->check = 0;
+		csum = skb_checksum(skb, 0, len, 0);
+		uh->check = udp_v4_check(len, saddr, daddr, csum);
+		if (uh->check == 0)
+			uh->check = CSUM_MANGLED_0;
+
+		skb->ip_summed = CHECKSUM_UNNECESSARY;
+	}
+}
+
+#endif
+
+static void __compat_fake_destructor(struct sk_buff *skb)
+{
+}
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0)
+static void __compat_iptunnel_xmit(struct rtable *rt, struct sk_buff *skb,
+		  __be32 src, __be32 dst, __u8 proto,
+		  __u8 tos, __u8 ttl, __be16 df, bool xnet)
+{
+	struct iphdr *iph;
+	struct pcpu_tstats *tstats = this_cpu_ptr(skb->dev->tstats);
+
+	skb_scrub_packet(skb, xnet);
+
+	skb->rxhash = 0;
+	skb_dst_set(skb, &rt->dst);
+	memset(IPCB(skb), 0, sizeof(*IPCB(skb)));
+
+	/* Push down and install the IP header. */
+	skb_push(skb, sizeof(struct iphdr));
+	skb_reset_network_header(skb);
+
+	iph = ip_hdr(skb);
+
+	iph->version	=	4;
+	iph->ihl	=	sizeof(struct iphdr) >> 2;
+	iph->frag_off	=	df;
+	iph->protocol	=	proto;
+	iph->tos	=	tos;
+	iph->daddr	=	dst;
+	iph->saddr	=	src;
+	iph->ttl	=	ttl;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 53)
+	__ip_select_ident(iph, &rt->dst, (skb_shinfo(skb)->gso_segs ?: 1) - 1);
+#else
+	__ip_select_ident(iph, skb_shinfo(skb)->gso_segs ?: 1);
+#endif
+
+	iptunnel_xmit(skb, skb->dev);
+	u64_stats_update_begin(&tstats->syncp);
+	tstats->tx_bytes -= 8;
+	u64_stats_update_end(&tstats->syncp);
+}
+#define iptunnel_xmit __compat_iptunnel_xmit
+#endif
+
+void udp_tunnel_xmit_skb(struct rtable *rt, struct sock *sk, struct sk_buff *skb,
+			 __be32 src, __be32 dst, __u8 tos, __u8 ttl,
+			 __be16 df, __be16 src_port, __be16 dst_port,
+			 bool xnet, bool nocheck)
+{
+	struct udphdr *uh;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)
+	struct net_device *dev = skb->dev;
+	int ret;
+#endif
+
+	__skb_push(skb, sizeof(*uh));
+	skb_reset_transport_header(skb);
+	uh = udp_hdr(skb);
+
+	uh->dest = dst_port;
+	uh->source = src_port;
+	uh->len = htons(skb->len);
+
+	memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt));
+
+	udp_set_csum(nocheck, skb, src, dst, skb->len);
+
+	if (!skb->sk)
+		skb->sk = sk;
+	if (!skb->destructor)
+		skb->destructor = __compat_fake_destructor;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)
+	ret =
+#endif
+	     iptunnel_xmit(
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 15, 0)
+			   sk,
+#endif
+			   rt, skb, src, dst, IPPROTO_UDP, tos, ttl, df, xnet);
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)
+	if (ret)
+		iptunnel_xmit_stats(ret - 8, &dev->stats, dev->tstats);
+#endif
+}
+
+void udp_tunnel_sock_release(struct socket *sock)
+{
+	rcu_assign_sk_user_data(sock->sk, NULL);
+	kernel_sock_shutdown(sock, SHUT_RDWR);
+	sock_release(sock);
+}
+
+#if IS_ENABLED(CONFIG_IPV6)
+#include <linux/module.h>
+#include <linux/errno.h>
+#include <linux/socket.h>
+#include <linux/udp.h>
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/in6.h>
+#include <net/udp.h>
+#include <net/udp_tunnel.h>
+#include <net/net_namespace.h>
+#include <net/netns/generic.h>
+#include <net/ip6_tunnel.h>
+#include <net/ip6_checksum.h>
+
+int udp_sock_create6(struct net *net, struct udp_port_cfg *cfg,
+		     struct socket **sockp)
+{
+	struct sockaddr_in6 udp6_addr;
+	int err;
+	struct socket *sock = NULL;
+
+	err = __sock_create(net, AF_INET6, SOCK_DGRAM, 0, &sock, 1);
+	if (err < 0)
+		goto error;
+
+	if (cfg->ipv6_v6only) {
+		int val = 1;
+
+		err = kernel_setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY,
+					(char *) &val, sizeof(val));
+		if (err < 0)
+			goto error;
+	}
+
+	udp6_addr.sin6_family = AF_INET6;
+	memcpy(&udp6_addr.sin6_addr, &cfg->local_ip6,
+	       sizeof(udp6_addr.sin6_addr));
+	udp6_addr.sin6_port = cfg->local_udp_port;
+	err = kernel_bind(sock, (struct sockaddr *)&udp6_addr,
+			  sizeof(udp6_addr));
+	if (err < 0)
+		goto error;
+
+	if (cfg->peer_udp_port) {
+		udp6_addr.sin6_family = AF_INET6;
+		memcpy(&udp6_addr.sin6_addr, &cfg->peer_ip6,
+		       sizeof(udp6_addr.sin6_addr));
+		udp6_addr.sin6_port = cfg->peer_udp_port;
+		err = kernel_connect(sock,
+				     (struct sockaddr *)&udp6_addr,
+				     sizeof(udp6_addr), 0);
+	}
+	if (err < 0)
+		goto error;
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0)
+	sock->sk->sk_no_check = !cfg->use_udp_checksums;
+#else
+	udp_set_no_check6_tx(sock->sk, !cfg->use_udp6_tx_checksums);
+	udp_set_no_check6_rx(sock->sk, !cfg->use_udp6_rx_checksums);
+#endif
+
+	*sockp = sock;
+	return 0;
+
+error:
+	if (sock) {
+		kernel_sock_shutdown(sock, SHUT_RDWR);
+		sock_release(sock);
+	}
+	*sockp = NULL;
+	return err;
+}
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0)
+static inline __sum16 udp_v6_check(int len,
+		const struct in6_addr *saddr,
+		const struct in6_addr *daddr,
+		__wsum base)
+{
+	return csum_ipv6_magic(saddr, daddr, len, IPPROTO_UDP, base);
+}
+static void udp6_set_csum(bool nocheck, struct sk_buff *skb,
+		   const struct in6_addr *saddr,
+		   const struct in6_addr *daddr, int len)
+{
+	struct udphdr *uh = udp_hdr(skb);
+
+	if (nocheck)
+		uh->check = 0;
+	else if (skb_is_gso(skb))
+		uh->check = ~udp_v6_check(len, saddr, daddr, 0);
+	else if (skb_dst(skb) && skb_dst(skb)->dev &&
+		 (skb_dst(skb)->dev->features & NETIF_F_IPV6_CSUM)) {
+
+		BUG_ON(skb->ip_summed == CHECKSUM_PARTIAL);
+
+		skb->ip_summed = CHECKSUM_PARTIAL;
+		skb->csum_start = skb_transport_header(skb) - skb->head;
+		skb->csum_offset = offsetof(struct udphdr, check);
+		uh->check = ~udp_v6_check(len, saddr, daddr, 0);
+	} else {
+		__wsum csum;
+
+		BUG_ON(skb->ip_summed == CHECKSUM_PARTIAL);
+
+		uh->check = 0;
+		csum = skb_checksum(skb, 0, len, 0);
+		uh->check = udp_v6_check(len, saddr, daddr, csum);
+		if (uh->check == 0)
+			uh->check = CSUM_MANGLED_0;
+
+		skb->ip_summed = CHECKSUM_UNNECESSARY;
+	}
+}
+#endif
+
+int udp_tunnel6_xmit_skb(struct dst_entry *dst, struct sock *sk,
+			 struct sk_buff *skb,
+			 struct net_device *dev, struct in6_addr *saddr,
+			 struct in6_addr *daddr,
+			 __u8 prio, __u8 ttl, __be32 label,
+			 __be16 src_port, __be16 dst_port, bool nocheck)
+{
+	struct udphdr *uh;
+	struct ipv6hdr *ip6h;
+
+	__skb_push(skb, sizeof(*uh));
+	skb_reset_transport_header(skb);
+	uh = udp_hdr(skb);
+
+	uh->dest = dst_port;
+	uh->source = src_port;
+
+	uh->len = htons(skb->len);
+
+	skb_dst_set(skb, dst);
+
+	udp6_set_csum(nocheck, skb, saddr, daddr, skb->len);
+
+	__skb_push(skb, sizeof(*ip6h));
+	skb_reset_network_header(skb);
+	ip6h		  = ipv6_hdr(skb);
+	ip6_flow_hdr(ip6h, prio, label);
+	ip6h->payload_len = htons(skb->len);
+	ip6h->nexthdr     = IPPROTO_UDP;
+	ip6h->hop_limit   = ttl;
+	ip6h->daddr	  = *daddr;
+	ip6h->saddr	  = *saddr;
+
+	if (!skb->sk)
+		skb->sk = sk;
+	if (!skb->destructor)
+		skb->destructor = __compat_fake_destructor;
+
+	ip6tunnel_xmit(skb, dev);
+	return 0;
+}
+#endif
diff -Naur --no-dereference a/net/wireguard/compat/udp_tunnel/udp_tunnel_partial_compat.h b/net/wireguard/compat/udp_tunnel/udp_tunnel_partial_compat.h
--- a/net/wireguard/compat/udp_tunnel/udp_tunnel_partial_compat.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/compat/udp_tunnel/udp_tunnel_partial_compat.h	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,226 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 0)
+#define udp_sock_create4 udp_sock_create
+#define udp_sock_create6 udp_sock_create
+#include <linux/socket.h>
+#include <linux/if.h>
+#include <linux/in.h>
+#include <net/ip_tunnels.h>
+#include <net/udp.h>
+#include <net/inet_common.h>
+#if IS_ENABLED(CONFIG_IPV6)
+#include <linux/in6.h>
+#include <net/ipv6.h>
+#include <net/addrconf.h>
+#include <net/ip6_checksum.h>
+#include <net/ip6_tunnel.h>
+#endif
+static inline void __compat_fake_destructor(struct sk_buff *skb)
+{
+}
+typedef int (*udp_tunnel_encap_rcv_t)(struct sock *sk, struct sk_buff *skb);
+struct udp_tunnel_sock_cfg {
+        void *sk_user_data;
+        __u8  encap_type;
+        udp_tunnel_encap_rcv_t encap_rcv;
+};
+/* This is global so, uh, only one real call site... This is the kind of horrific hack you'd expect to see in compat code. */
+static udp_tunnel_encap_rcv_t encap_rcv = NULL;
+static void __compat_sk_data_ready(struct sock *sk)
+{
+	struct sk_buff *skb;
+	while ((skb = skb_dequeue(&sk->sk_receive_queue)) != NULL) {
+		skb_orphan(skb);
+		sk_mem_reclaim(sk);
+		encap_rcv(sk, skb);
+	}
+}
+static inline void setup_udp_tunnel_sock(struct net *net, struct socket *sock,
+                           struct udp_tunnel_sock_cfg *cfg)
+{
+	struct sock *sk = sock->sk;
+	inet_sk(sk)->mc_loop = 0;
+	encap_rcv = cfg->encap_rcv;
+	rcu_assign_sk_user_data(sk, cfg->sk_user_data);
+	sk->sk_data_ready = __compat_sk_data_ready;
+}
+static inline void udp_tunnel_sock_release(struct socket *sock)
+{
+	rcu_assign_sk_user_data(sock->sk, NULL);
+	kernel_sock_shutdown(sock, SHUT_RDWR);
+	sk_release_kernel(sock->sk);
+}
+static inline int udp_tunnel_xmit_skb(struct socket *sock, struct rtable *rt,
+                        struct sk_buff *skb, __be32 src, __be32 dst,
+                        __u8 tos, __u8 ttl, __be16 df, __be16 src_port,
+                        __be16 dst_port, bool xnet)
+{
+	struct udphdr *uh;
+	__skb_push(skb, sizeof(*uh));
+	skb_reset_transport_header(skb);
+	uh = udp_hdr(skb);
+	uh->dest = dst_port;
+	uh->source = src_port;
+	uh->len = htons(skb->len);
+	udp_set_csum(sock->sk->sk_no_check_tx, skb, src, dst, skb->len);
+	return iptunnel_xmit(sock->sk, rt, skb, src, dst, IPPROTO_UDP,
+			     tos, ttl, df, xnet);
+}
+#if IS_ENABLED(CONFIG_IPV6)
+static inline int udp_tunnel6_xmit_skb(struct socket *sock, struct dst_entry *dst,
+                         struct sk_buff *skb, struct net_device *dev,
+                         struct in6_addr *saddr, struct in6_addr *daddr,
+                         __u8 prio, __u8 ttl, __be16 src_port,
+                         __be16 dst_port)
+{
+	struct udphdr *uh;
+	struct ipv6hdr *ip6h;
+	struct sock *sk = sock->sk;
+	__skb_push(skb, sizeof(*uh));
+	skb_reset_transport_header(skb);
+	uh = udp_hdr(skb);
+	uh->dest = dst_port;
+	uh->source = src_port;
+	uh->len = htons(skb->len);
+	memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt));
+	IPCB(skb)->flags &= ~(IPSKB_XFRM_TUNNEL_SIZE | IPSKB_XFRM_TRANSFORMED
+			    | IPSKB_REROUTED);
+	skb_dst_set(skb, dst);
+	udp6_set_csum(udp_get_no_check6_tx(sk), skb, &inet6_sk(sk)->saddr,
+	              &sk->sk_v6_daddr, skb->len);
+	__skb_push(skb, sizeof(*ip6h));
+	skb_reset_network_header(skb);
+	ip6h		  = ipv6_hdr(skb);
+	ip6_flow_hdr(ip6h, prio, htonl(0));
+	ip6h->payload_len = htons(skb->len);
+	ip6h->nexthdr     = IPPROTO_UDP;
+	ip6h->hop_limit   = ttl;
+	ip6h->daddr	  = *daddr;
+	ip6h->saddr	  = *saddr;
+	ip6tunnel_xmit(skb, dev);
+	return 0;
+}
+#endif
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0)
+#include <linux/in.h>
+#include <linux/in6.h>
+#include <linux/udp.h>
+#include <linux/skbuff.h>
+#include <linux/if.h>
+#include <net/udp_tunnel.h>
+#define udp_tunnel_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l) do { struct net_device *dev__ = (c)->dev; int ret__; ret__ = udp_tunnel_xmit_skb((b)->sk_socket, a, c, d, e, f, g, h, i, j, k); if (ret__) iptunnel_xmit_stats(ret__ - 8, &dev__->stats, dev__->tstats); } while (0)
+#if IS_ENABLED(CONFIG_IPV6)
+#define udp_tunnel6_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l) udp_tunnel6_xmit_skb((b)->sk_socket, a, c, d, e, f, g, h, j, k);
+#endif
+#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 1, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0)
+#include <linux/if.h>
+#include <net/udp_tunnel.h>
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 18, 0)
+static inline void __compat_fake_destructor(struct sk_buff *skb)
+{
+}
+#endif
+#define udp_tunnel_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l) do { struct net_device *dev__ = (c)->dev; int ret__; if (!(c)->destructor) (c)->destructor = __compat_fake_destructor; if (!(c)->sk) (c)->sk = (b); ret__ = udp_tunnel_xmit_skb(a, c, d, e, f, g, h, i, j, k, l); if (ret__) iptunnel_xmit_stats(ret__ - 8, &dev__->stats, dev__->tstats); } while (0)
+#if IS_ENABLED(CONFIG_IPV6)
+#define udp_tunnel6_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l) do { if (!(c)->destructor) (c)->destructor = __compat_fake_destructor; if (!(c)->sk) (c)->sk = (b); udp_tunnel6_xmit_skb(a, c, d, e, f, g, h, j, k, l); } while(0)
+#endif
+#else
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0)
+#include <linux/if.h>
+#include <net/udp_tunnel.h>
+#define udp_tunnel_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l) do { struct net_device *dev__ = (c)->dev; int ret__ = udp_tunnel_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l); if (ret__) iptunnel_xmit_stats(ret__ - 8, &dev__->stats, dev__->tstats); } while (0)
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 3, 0)
+#include <linux/if.h>
+#include <net/udp_tunnel.h>
+#define udp_tunnel_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l) do { struct net_device *dev__ = (c)->dev; int ret__ = udp_tunnel_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l); iptunnel_xmit_stats(ret__, &dev__->stats, dev__->tstats); } while (0)
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && IS_ENABLED(CONFIG_IPV6) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0)
+#include <linux/if.h>
+#include <net/udp_tunnel.h>
+#define udp_tunnel6_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l) udp_tunnel6_xmit_skb(a, b, c, d, e, f, g, h, j, k, l)
+#endif
+
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0)
+#include <linux/skbuff.h>
+#include <linux/if.h>
+#include <net/udp_tunnel.h>
+struct __compat_udp_port_cfg {
+	u8 family;
+	union {
+		struct in_addr local_ip;
+#if IS_ENABLED(CONFIG_IPV6)
+		struct in6_addr local_ip6;
+#endif
+	};
+	union {
+		struct in_addr peer_ip;
+#if IS_ENABLED(CONFIG_IPV6)
+		struct in6_addr peer_ip6;
+#endif
+	};
+	__be16 local_udp_port;
+	__be16 peer_udp_port;
+	unsigned int use_udp_checksums:1, use_udp6_tx_checksums:1, use_udp6_rx_checksums:1, ipv6_v6only:1;
+};
+static inline int __maybe_unused __compat_udp_sock_create(struct net *net, struct __compat_udp_port_cfg *cfg, struct socket **sockp)
+{
+	struct udp_port_cfg old_cfg = {
+		.family = cfg->family,
+		.local_ip = cfg->local_ip,
+#if IS_ENABLED(CONFIG_IPV6)
+		.local_ip6 = cfg->local_ip6,
+#endif
+		.peer_ip = cfg->peer_ip,
+#if IS_ENABLED(CONFIG_IPV6)
+		.peer_ip6 = cfg->peer_ip6,
+#endif
+		.local_udp_port = cfg->local_udp_port,
+		.peer_udp_port = cfg->peer_udp_port,
+		.use_udp_checksums = cfg->use_udp_checksums,
+		.use_udp6_tx_checksums = cfg->use_udp6_tx_checksums,
+		.use_udp6_rx_checksums = cfg->use_udp6_rx_checksums
+	};
+	if (cfg->family == AF_INET)
+		return udp_sock_create4(net, &old_cfg, sockp);
+
+#if IS_ENABLED(CONFIG_IPV6)
+	if (cfg->family == AF_INET6) {
+		int ret;
+		int old_bindv6only;
+		struct net *nobns;
+
+		if (cfg->ipv6_v6only) {
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0)
+			nobns = &init_net;
+#else
+			nobns = net;
+#endif
+			/* Since udp_port_cfg only learned of ipv6_v6only in 4.3, we do this horrible
+			 * hack here and set the sysctl variable temporarily to something that will
+			 * set the right option for us in sock_create. It's super racey! */
+			old_bindv6only = nobns->ipv6.sysctl.bindv6only;
+			nobns->ipv6.sysctl.bindv6only = 1;
+		}
+		ret = udp_sock_create6(net, &old_cfg, sockp);
+		if (cfg->ipv6_v6only)
+			nobns->ipv6.sysctl.bindv6only = old_bindv6only;
+		return ret;
+	}
+#endif
+	return -EPFNOSUPPORT;
+}
+#define udp_port_cfg __compat_udp_port_cfg
+#define udp_sock_create(a, b, c) __compat_udp_sock_create(a, b, c)
+#endif
diff -Naur --no-dereference a/net/wireguard/cookie.c b/net/wireguard/cookie.c
--- a/net/wireguard/cookie.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/cookie.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,236 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#include "cookie.h"
+#include "peer.h"
+#include "device.h"
+#include "messages.h"
+#include "ratelimiter.h"
+#include "timers.h"
+
+#include <zinc/blake2s.h>
+#include <zinc/chacha20poly1305.h>
+
+#include <net/ipv6.h>
+#include <crypto/algapi.h>
+
+void wg_cookie_checker_init(struct cookie_checker *checker,
+			    struct wg_device *wg)
+{
+	init_rwsem(&checker->secret_lock);
+	checker->secret_birthdate = ktime_get_coarse_boottime_ns();
+	get_random_bytes(checker->secret, NOISE_HASH_LEN);
+	checker->device = wg;
+}
+
+enum { COOKIE_KEY_LABEL_LEN = 8 };
+static const u8 mac1_key_label[COOKIE_KEY_LABEL_LEN] = "mac1----";
+static const u8 cookie_key_label[COOKIE_KEY_LABEL_LEN] = "cookie--";
+
+static void precompute_key(u8 key[NOISE_SYMMETRIC_KEY_LEN],
+			   const u8 pubkey[NOISE_PUBLIC_KEY_LEN],
+			   const u8 label[COOKIE_KEY_LABEL_LEN])
+{
+	struct blake2s_state blake;
+
+	blake2s_init(&blake, NOISE_SYMMETRIC_KEY_LEN);
+	blake2s_update(&blake, label, COOKIE_KEY_LABEL_LEN);
+	blake2s_update(&blake, pubkey, NOISE_PUBLIC_KEY_LEN);
+	blake2s_final(&blake, key);
+}
+
+/* Must hold peer->handshake.static_identity->lock */
+void wg_cookie_checker_precompute_device_keys(struct cookie_checker *checker)
+{
+	if (likely(checker->device->static_identity.has_identity)) {
+		precompute_key(checker->cookie_encryption_key,
+			       checker->device->static_identity.static_public,
+			       cookie_key_label);
+		precompute_key(checker->message_mac1_key,
+			       checker->device->static_identity.static_public,
+			       mac1_key_label);
+	} else {
+		memset(checker->cookie_encryption_key, 0,
+		       NOISE_SYMMETRIC_KEY_LEN);
+		memset(checker->message_mac1_key, 0, NOISE_SYMMETRIC_KEY_LEN);
+	}
+}
+
+void wg_cookie_checker_precompute_peer_keys(struct wg_peer *peer)
+{
+	precompute_key(peer->latest_cookie.cookie_decryption_key,
+		       peer->handshake.remote_static, cookie_key_label);
+	precompute_key(peer->latest_cookie.message_mac1_key,
+		       peer->handshake.remote_static, mac1_key_label);
+}
+
+void wg_cookie_init(struct cookie *cookie)
+{
+	memset(cookie, 0, sizeof(*cookie));
+	init_rwsem(&cookie->lock);
+}
+
+static void compute_mac1(u8 mac1[COOKIE_LEN], const void *message, size_t len,
+			 const u8 key[NOISE_SYMMETRIC_KEY_LEN])
+{
+	len = len - sizeof(struct message_macs) +
+	      offsetof(struct message_macs, mac1);
+	blake2s(mac1, message, key, COOKIE_LEN, len, NOISE_SYMMETRIC_KEY_LEN);
+}
+
+static void compute_mac2(u8 mac2[COOKIE_LEN], const void *message, size_t len,
+			 const u8 cookie[COOKIE_LEN])
+{
+	len = len - sizeof(struct message_macs) +
+	      offsetof(struct message_macs, mac2);
+	blake2s(mac2, message, cookie, COOKIE_LEN, len, COOKIE_LEN);
+}
+
+static void make_cookie(u8 cookie[COOKIE_LEN], struct sk_buff *skb,
+			struct cookie_checker *checker)
+{
+	struct blake2s_state state;
+
+	if (wg_birthdate_has_expired(checker->secret_birthdate,
+				     COOKIE_SECRET_MAX_AGE)) {
+		down_write(&checker->secret_lock);
+		checker->secret_birthdate = ktime_get_coarse_boottime_ns();
+		get_random_bytes(checker->secret, NOISE_HASH_LEN);
+		up_write(&checker->secret_lock);
+	}
+
+	down_read(&checker->secret_lock);
+
+	blake2s_init_key(&state, COOKIE_LEN, checker->secret, NOISE_HASH_LEN);
+	if (skb->protocol == htons(ETH_P_IP))
+		blake2s_update(&state, (u8 *)&ip_hdr(skb)->saddr,
+			       sizeof(struct in_addr));
+	else if (skb->protocol == htons(ETH_P_IPV6))
+		blake2s_update(&state, (u8 *)&ipv6_hdr(skb)->saddr,
+			       sizeof(struct in6_addr));
+	blake2s_update(&state, (u8 *)&udp_hdr(skb)->source, sizeof(__be16));
+	blake2s_final(&state, cookie);
+
+	up_read(&checker->secret_lock);
+}
+
+enum cookie_mac_state wg_cookie_validate_packet(struct cookie_checker *checker,
+						struct sk_buff *skb,
+						bool check_cookie)
+{
+	struct message_macs *macs = (struct message_macs *)
+		(skb->data + skb->len - sizeof(*macs));
+	enum cookie_mac_state ret;
+	u8 computed_mac[COOKIE_LEN];
+	u8 cookie[COOKIE_LEN];
+
+	ret = INVALID_MAC;
+	compute_mac1(computed_mac, skb->data, skb->len,
+		     checker->message_mac1_key);
+	if (crypto_memneq(computed_mac, macs->mac1, COOKIE_LEN))
+		goto out;
+
+	ret = VALID_MAC_BUT_NO_COOKIE;
+
+	if (!check_cookie)
+		goto out;
+
+	make_cookie(cookie, skb, checker);
+
+	compute_mac2(computed_mac, skb->data, skb->len, cookie);
+	if (crypto_memneq(computed_mac, macs->mac2, COOKIE_LEN))
+		goto out;
+
+	ret = VALID_MAC_WITH_COOKIE_BUT_RATELIMITED;
+	if (!wg_ratelimiter_allow(skb, dev_net(checker->device->dev)))
+		goto out;
+
+	ret = VALID_MAC_WITH_COOKIE;
+
+out:
+	return ret;
+}
+
+void wg_cookie_add_mac_to_packet(void *message, size_t len,
+				 struct wg_peer *peer)
+{
+	struct message_macs *macs = (struct message_macs *)
+		((u8 *)message + len - sizeof(*macs));
+
+	down_write(&peer->latest_cookie.lock);
+	compute_mac1(macs->mac1, message, len,
+		     peer->latest_cookie.message_mac1_key);
+	memcpy(peer->latest_cookie.last_mac1_sent, macs->mac1, COOKIE_LEN);
+	peer->latest_cookie.have_sent_mac1 = true;
+	up_write(&peer->latest_cookie.lock);
+
+	down_read(&peer->latest_cookie.lock);
+	if (peer->latest_cookie.is_valid &&
+	    !wg_birthdate_has_expired(peer->latest_cookie.birthdate,
+				COOKIE_SECRET_MAX_AGE - COOKIE_SECRET_LATENCY))
+		compute_mac2(macs->mac2, message, len,
+			     peer->latest_cookie.cookie);
+	else
+		memset(macs->mac2, 0, COOKIE_LEN);
+	up_read(&peer->latest_cookie.lock);
+}
+
+void wg_cookie_message_create(struct message_handshake_cookie *dst,
+			      struct sk_buff *skb, __le32 index,
+			      struct cookie_checker *checker)
+{
+	struct message_macs *macs = (struct message_macs *)
+		((u8 *)skb->data + skb->len - sizeof(*macs));
+	u8 cookie[COOKIE_LEN];
+
+	dst->header.type = cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE);
+	dst->receiver_index = index;
+	get_random_bytes_wait(dst->nonce, COOKIE_NONCE_LEN);
+
+	make_cookie(cookie, skb, checker);
+	xchacha20poly1305_encrypt(dst->encrypted_cookie, cookie, COOKIE_LEN,
+				  macs->mac1, COOKIE_LEN, dst->nonce,
+				  checker->cookie_encryption_key);
+}
+
+void wg_cookie_message_consume(struct message_handshake_cookie *src,
+			       struct wg_device *wg)
+{
+	struct wg_peer *peer = NULL;
+	u8 cookie[COOKIE_LEN];
+	bool ret;
+
+	if (unlikely(!wg_index_hashtable_lookup(wg->index_hashtable,
+						INDEX_HASHTABLE_HANDSHAKE |
+						INDEX_HASHTABLE_KEYPAIR,
+						src->receiver_index, &peer)))
+		return;
+
+	down_read(&peer->latest_cookie.lock);
+	if (unlikely(!peer->latest_cookie.have_sent_mac1)) {
+		up_read(&peer->latest_cookie.lock);
+		goto out;
+	}
+	ret = xchacha20poly1305_decrypt(
+		cookie, src->encrypted_cookie, sizeof(src->encrypted_cookie),
+		peer->latest_cookie.last_mac1_sent, COOKIE_LEN, src->nonce,
+		peer->latest_cookie.cookie_decryption_key);
+	up_read(&peer->latest_cookie.lock);
+
+	if (ret) {
+		down_write(&peer->latest_cookie.lock);
+		memcpy(peer->latest_cookie.cookie, cookie, COOKIE_LEN);
+		peer->latest_cookie.birthdate = ktime_get_coarse_boottime_ns();
+		peer->latest_cookie.is_valid = true;
+		peer->latest_cookie.have_sent_mac1 = false;
+		up_write(&peer->latest_cookie.lock);
+	} else {
+		net_dbg_ratelimited("%s: Could not decrypt invalid cookie response\n",
+				    wg->dev->name);
+	}
+
+out:
+	wg_peer_put(peer);
+}
diff -Naur --no-dereference a/net/wireguard/cookie.h b/net/wireguard/cookie.h
--- a/net/wireguard/cookie.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/cookie.h	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#ifndef _WG_COOKIE_H
+#define _WG_COOKIE_H
+
+#include "messages.h"
+#include <linux/rwsem.h>
+
+struct wg_peer;
+
+struct cookie_checker {
+	u8 secret[NOISE_HASH_LEN];
+	u8 cookie_encryption_key[NOISE_SYMMETRIC_KEY_LEN];
+	u8 message_mac1_key[NOISE_SYMMETRIC_KEY_LEN];
+	u64 secret_birthdate;
+	struct rw_semaphore secret_lock;
+	struct wg_device *device;
+};
+
+struct cookie {
+	u64 birthdate;
+	bool is_valid;
+	u8 cookie[COOKIE_LEN];
+	bool have_sent_mac1;
+	u8 last_mac1_sent[COOKIE_LEN];
+	u8 cookie_decryption_key[NOISE_SYMMETRIC_KEY_LEN];
+	u8 message_mac1_key[NOISE_SYMMETRIC_KEY_LEN];
+	struct rw_semaphore lock;
+};
+
+enum cookie_mac_state {
+	INVALID_MAC,
+	VALID_MAC_BUT_NO_COOKIE,
+	VALID_MAC_WITH_COOKIE_BUT_RATELIMITED,
+	VALID_MAC_WITH_COOKIE
+};
+
+void wg_cookie_checker_init(struct cookie_checker *checker,
+			    struct wg_device *wg);
+void wg_cookie_checker_precompute_device_keys(struct cookie_checker *checker);
+void wg_cookie_checker_precompute_peer_keys(struct wg_peer *peer);
+void wg_cookie_init(struct cookie *cookie);
+
+enum cookie_mac_state wg_cookie_validate_packet(struct cookie_checker *checker,
+						struct sk_buff *skb,
+						bool check_cookie);
+void wg_cookie_add_mac_to_packet(void *message, size_t len,
+				 struct wg_peer *peer);
+
+void wg_cookie_message_create(struct message_handshake_cookie *src,
+			      struct sk_buff *skb, __le32 index,
+			      struct cookie_checker *checker);
+void wg_cookie_message_consume(struct message_handshake_cookie *src,
+			       struct wg_device *wg);
+
+#endif /* _WG_COOKIE_H */
diff -Naur --no-dereference a/net/wireguard/crypto/include/zinc/blake2s.h b/net/wireguard/crypto/include/zinc/blake2s.h
--- a/net/wireguard/crypto/include/zinc/blake2s.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/include/zinc/blake2s.h	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,56 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#ifndef _ZINC_BLAKE2S_H
+#define _ZINC_BLAKE2S_H
+
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <asm/bug.h>
+
+enum blake2s_lengths {
+	BLAKE2S_BLOCK_SIZE = 64,
+	BLAKE2S_HASH_SIZE = 32,
+	BLAKE2S_KEY_SIZE = 32
+};
+
+struct blake2s_state {
+	u32 h[8];
+	u32 t[2];
+	u32 f[2];
+	u8 buf[BLAKE2S_BLOCK_SIZE];
+	unsigned int buflen;
+	unsigned int outlen;
+};
+
+void blake2s_init(struct blake2s_state *state, const size_t outlen);
+void blake2s_init_key(struct blake2s_state *state, const size_t outlen,
+		      const void *key, const size_t keylen);
+void blake2s_update(struct blake2s_state *state, const u8 *in, size_t inlen);
+void blake2s_final(struct blake2s_state *state, u8 *out);
+
+static inline void blake2s(u8 *out, const u8 *in, const u8 *key,
+			   const size_t outlen, const size_t inlen,
+			   const size_t keylen)
+{
+	struct blake2s_state state;
+
+	WARN_ON(IS_ENABLED(DEBUG) && ((!in && inlen > 0) || !out || !outlen ||
+		outlen > BLAKE2S_HASH_SIZE || keylen > BLAKE2S_KEY_SIZE ||
+		(!key && keylen)));
+
+	if (keylen)
+		blake2s_init_key(&state, outlen, key, keylen);
+	else
+		blake2s_init(&state, outlen);
+
+	blake2s_update(&state, in, inlen);
+	blake2s_final(&state, out);
+}
+
+void blake2s_hmac(u8 *out, const u8 *in, const u8 *key, const size_t outlen,
+		  const size_t inlen, const size_t keylen);
+
+#endif /* _ZINC_BLAKE2S_H */
diff -Naur --no-dereference a/net/wireguard/crypto/include/zinc/chacha20.h b/net/wireguard/crypto/include/zinc/chacha20.h
--- a/net/wireguard/crypto/include/zinc/chacha20.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/include/zinc/chacha20.h	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,70 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#ifndef _ZINC_CHACHA20_H
+#define _ZINC_CHACHA20_H
+
+#include <asm/unaligned.h>
+#include <linux/simd.h>
+#include <linux/kernel.h>
+#include <linux/types.h>
+
+enum chacha20_lengths {
+	CHACHA20_NONCE_SIZE = 16,
+	CHACHA20_KEY_SIZE = 32,
+	CHACHA20_KEY_WORDS = CHACHA20_KEY_SIZE / sizeof(u32),
+	CHACHA20_BLOCK_SIZE = 64,
+	CHACHA20_BLOCK_WORDS = CHACHA20_BLOCK_SIZE / sizeof(u32),
+	HCHACHA20_NONCE_SIZE = CHACHA20_NONCE_SIZE,
+	HCHACHA20_KEY_SIZE = CHACHA20_KEY_SIZE
+};
+
+enum chacha20_constants { /* expand 32-byte k */
+	CHACHA20_CONSTANT_EXPA = 0x61707865U,
+	CHACHA20_CONSTANT_ND_3 = 0x3320646eU,
+	CHACHA20_CONSTANT_2_BY = 0x79622d32U,
+	CHACHA20_CONSTANT_TE_K = 0x6b206574U
+};
+
+struct chacha20_ctx {
+	union {
+		u32 state[16];
+		struct {
+			u32 constant[4];
+			u32 key[8];
+			u32 counter[4];
+		};
+	};
+};
+
+static inline void chacha20_init(struct chacha20_ctx *ctx,
+				 const u8 key[CHACHA20_KEY_SIZE],
+				 const u64 nonce)
+{
+	ctx->constant[0] = CHACHA20_CONSTANT_EXPA;
+	ctx->constant[1] = CHACHA20_CONSTANT_ND_3;
+	ctx->constant[2] = CHACHA20_CONSTANT_2_BY;
+	ctx->constant[3] = CHACHA20_CONSTANT_TE_K;
+	ctx->key[0] = get_unaligned_le32(key + 0);
+	ctx->key[1] = get_unaligned_le32(key + 4);
+	ctx->key[2] = get_unaligned_le32(key + 8);
+	ctx->key[3] = get_unaligned_le32(key + 12);
+	ctx->key[4] = get_unaligned_le32(key + 16);
+	ctx->key[5] = get_unaligned_le32(key + 20);
+	ctx->key[6] = get_unaligned_le32(key + 24);
+	ctx->key[7] = get_unaligned_le32(key + 28);
+	ctx->counter[0] = 0;
+	ctx->counter[1] = 0;
+	ctx->counter[2] = nonce & U32_MAX;
+	ctx->counter[3] = nonce >> 32;
+}
+void chacha20(struct chacha20_ctx *ctx, u8 *dst, const u8 *src, u32 len,
+	      simd_context_t *simd_context);
+
+void hchacha20(u32 derived_key[CHACHA20_KEY_WORDS],
+	       const u8 nonce[HCHACHA20_NONCE_SIZE],
+	       const u8 key[HCHACHA20_KEY_SIZE], simd_context_t *simd_context);
+
+#endif /* _ZINC_CHACHA20_H */
diff -Naur --no-dereference a/net/wireguard/crypto/include/zinc/chacha20poly1305.h b/net/wireguard/crypto/include/zinc/chacha20poly1305.h
--- a/net/wireguard/crypto/include/zinc/chacha20poly1305.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/include/zinc/chacha20poly1305.h	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,50 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#ifndef _ZINC_CHACHA20POLY1305_H
+#define _ZINC_CHACHA20POLY1305_H
+
+#include <linux/simd.h>
+#include <linux/types.h>
+
+struct scatterlist;
+
+enum chacha20poly1305_lengths {
+	XCHACHA20POLY1305_NONCE_SIZE = 24,
+	CHACHA20POLY1305_KEY_SIZE = 32,
+	CHACHA20POLY1305_AUTHTAG_SIZE = 16
+};
+
+void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
+			      const u8 *ad, const size_t ad_len,
+			      const u64 nonce,
+			      const u8 key[CHACHA20POLY1305_KEY_SIZE]);
+
+bool __must_check chacha20poly1305_encrypt_sg_inplace(
+	struct scatterlist *src, const size_t src_len, const u8 *ad,
+	const size_t ad_len, const u64 nonce,
+	const u8 key[CHACHA20POLY1305_KEY_SIZE], simd_context_t *simd_context);
+
+bool __must_check
+chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
+			 const u8 *ad, const size_t ad_len, const u64 nonce,
+			 const u8 key[CHACHA20POLY1305_KEY_SIZE]);
+
+bool __must_check chacha20poly1305_decrypt_sg_inplace(
+	struct scatterlist *src, size_t src_len, const u8 *ad,
+	const size_t ad_len, const u64 nonce,
+	const u8 key[CHACHA20POLY1305_KEY_SIZE], simd_context_t *simd_context);
+
+void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
+			       const u8 *ad, const size_t ad_len,
+			       const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
+			       const u8 key[CHACHA20POLY1305_KEY_SIZE]);
+
+bool __must_check xchacha20poly1305_decrypt(
+	u8 *dst, const u8 *src, const size_t src_len, const u8 *ad,
+	const size_t ad_len, const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
+	const u8 key[CHACHA20POLY1305_KEY_SIZE]);
+
+#endif /* _ZINC_CHACHA20POLY1305_H */
diff -Naur --no-dereference a/net/wireguard/crypto/include/zinc/curve25519.h b/net/wireguard/crypto/include/zinc/curve25519.h
--- a/net/wireguard/crypto/include/zinc/curve25519.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/include/zinc/curve25519.h	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#ifndef _ZINC_CURVE25519_H
+#define _ZINC_CURVE25519_H
+
+#include <linux/types.h>
+
+enum curve25519_lengths {
+	CURVE25519_KEY_SIZE = 32
+};
+
+bool __must_check curve25519(u8 mypublic[CURVE25519_KEY_SIZE],
+			     const u8 secret[CURVE25519_KEY_SIZE],
+			     const u8 basepoint[CURVE25519_KEY_SIZE]);
+void curve25519_generate_secret(u8 secret[CURVE25519_KEY_SIZE]);
+bool __must_check curve25519_generate_public(
+	u8 pub[CURVE25519_KEY_SIZE], const u8 secret[CURVE25519_KEY_SIZE]);
+
+static inline void curve25519_clamp_secret(u8 secret[CURVE25519_KEY_SIZE])
+{
+	secret[0] &= 248;
+	secret[31] = (secret[31] & 127) | 64;
+}
+
+#endif /* _ZINC_CURVE25519_H */
diff -Naur --no-dereference a/net/wireguard/crypto/include/zinc/poly1305.h b/net/wireguard/crypto/include/zinc/poly1305.h
--- a/net/wireguard/crypto/include/zinc/poly1305.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/include/zinc/poly1305.h	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#ifndef _ZINC_POLY1305_H
+#define _ZINC_POLY1305_H
+
+#include <linux/simd.h>
+#include <linux/types.h>
+
+enum poly1305_lengths {
+	POLY1305_BLOCK_SIZE = 16,
+	POLY1305_KEY_SIZE = 32,
+	POLY1305_MAC_SIZE = 16
+};
+
+struct poly1305_ctx {
+	u8 opaque[24 * sizeof(u64)];
+	u32 nonce[4];
+	u8 data[POLY1305_BLOCK_SIZE];
+	size_t num;
+} __aligned(8);
+
+void poly1305_init(struct poly1305_ctx *ctx, const u8 key[POLY1305_KEY_SIZE]);
+void poly1305_update(struct poly1305_ctx *ctx, const u8 *input, size_t len,
+		     simd_context_t *simd_context);
+void poly1305_final(struct poly1305_ctx *ctx, u8 mac[POLY1305_MAC_SIZE],
+		    simd_context_t *simd_context);
+
+#endif /* _ZINC_POLY1305_H */
diff -Naur --no-dereference a/net/wireguard/crypto/Makefile.include b/net/wireguard/crypto/Makefile.include
--- a/net/wireguard/crypto/Makefile.include	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/Makefile.include	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,57 @@
+ifeq ($(CONFIG_X86_64)$(if $(CONFIG_UML),y,n),yn)
+CONFIG_ZINC_ARCH_X86_64 := y
+endif
+ifeq ($(CONFIG_ARM)$(if $(CONFIG_CPU_32v3),y,n),yn)
+CONFIG_ZINC_ARCH_ARM := y
+endif
+ifeq ($(CONFIG_ARM64),y)
+CONFIG_ZINC_ARCH_ARM64 := y
+endif
+ifeq ($(CONFIG_MIPS)$(CONFIG_CPU_MIPS32_R2),yy)
+CONFIG_ZINC_ARCH_MIPS := y
+endif
+ifeq ($(CONFIG_MIPS)$(CONFIG_64BIT),yy)
+CONFIG_ZINC_ARCH_MIPS64 := y
+endif
+
+zinc-y += chacha20/chacha20.o
+zinc-$(CONFIG_ZINC_ARCH_X86_64) += chacha20/chacha20-x86_64.o
+zinc-$(CONFIG_ZINC_ARCH_ARM) += chacha20/chacha20-arm.o chacha20/chacha20-unrolled-arm.o
+zinc-$(CONFIG_ZINC_ARCH_ARM64) += chacha20/chacha20-arm64.o
+zinc-$(CONFIG_ZINC_ARCH_MIPS) += chacha20/chacha20-mips.o
+AFLAGS_chacha20-mips.o += -O2 # This is required to fill the branch delay slots
+
+zinc-y += poly1305/poly1305.o
+zinc-$(CONFIG_ZINC_ARCH_X86_64) += poly1305/poly1305-x86_64.o
+zinc-$(CONFIG_ZINC_ARCH_ARM) += poly1305/poly1305-arm.o
+zinc-$(CONFIG_ZINC_ARCH_ARM64) += poly1305/poly1305-arm64.o
+zinc-$(CONFIG_ZINC_ARCH_MIPS) += poly1305/poly1305-mips.o
+AFLAGS_poly1305-mips.o += -O2 # This is required to fill the branch delay slots
+zinc-$(CONFIG_ZINC_ARCH_MIPS64) += poly1305/poly1305-mips64.o
+
+zinc-y += chacha20poly1305.o
+
+zinc-y += blake2s/blake2s.o
+zinc-$(CONFIG_ZINC_ARCH_X86_64) += blake2s/blake2s-x86_64.o
+
+zinc-y += curve25519/curve25519.o
+zinc-$(CONFIG_ZINC_ARCH_ARM) += curve25519/curve25519-arm.o
+
+quiet_cmd_perlasm = PERLASM $@
+      cmd_perlasm = $(PERL) $< > $@
+$(obj)/%.S: $(src)/%.pl FORCE
+	$(call if_changed,perlasm)
+kbuild-dir := $(if $(filter /%,$(src)),$(src),$(srctree)/$(src))
+targets := $(patsubst $(kbuild-dir)/%.pl,%.S,$(wildcard $(patsubst %.o,$(kbuild-dir)/crypto/zinc/%.pl,$(zinc-y) $(zinc-m) $(zinc-))))
+
+# Old kernels don't set this, which causes trouble.
+.SECONDARY:
+
+wireguard-y += $(addprefix crypto/zinc/,$(zinc-y))
+ccflags-y += -I$(kbuild-dir)/crypto/include
+ccflags-$(CONFIG_ZINC_ARCH_X86_64) += -DCONFIG_ZINC_ARCH_X86_64
+ccflags-$(CONFIG_ZINC_ARCH_ARM) += -DCONFIG_ZINC_ARCH_ARM
+ccflags-$(CONFIG_ZINC_ARCH_ARM64) += -DCONFIG_ZINC_ARCH_ARM64
+ccflags-$(CONFIG_ZINC_ARCH_MIPS) += -DCONFIG_ZINC_ARCH_MIPS
+ccflags-$(CONFIG_ZINC_ARCH_MIPS64) += -DCONFIG_ZINC_ARCH_MIPS64
+ccflags-$(CONFIG_WIREGUARD_DEBUG) += -DCONFIG_ZINC_SELFTEST
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/blake2s/blake2s.c b/net/wireguard/crypto/zinc/blake2s/blake2s.c
--- a/net/wireguard/crypto/zinc/blake2s/blake2s.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/blake2s/blake2s.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,276 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ *
+ * This is an implementation of the BLAKE2s hash and PRF functions.
+ *
+ * Information: https://blake2.net/
+ *
+ */
+
+#include <zinc/blake2s.h>
+#include "../selftest/run.h"
+
+#include <linux/types.h>
+#include <linux/string.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/bug.h>
+#include <asm/unaligned.h>
+
+static const u32 blake2s_iv[8] = {
+	0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL, 0xA54FF53AUL,
+	0x510E527FUL, 0x9B05688CUL, 0x1F83D9ABUL, 0x5BE0CD19UL
+};
+
+static const u8 blake2s_sigma[10][16] = {
+	{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 },
+	{ 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 },
+	{ 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 },
+	{ 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 },
+	{ 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 },
+	{ 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 },
+	{ 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 },
+	{ 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 },
+	{ 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 },
+	{ 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 },
+};
+
+static inline void blake2s_set_lastblock(struct blake2s_state *state)
+{
+	state->f[0] = -1;
+}
+
+static inline void blake2s_increment_counter(struct blake2s_state *state,
+					     const u32 inc)
+{
+	state->t[0] += inc;
+	state->t[1] += (state->t[0] < inc);
+}
+
+static inline void blake2s_init_param(struct blake2s_state *state,
+				      const u32 param)
+{
+	int i;
+
+	memset(state, 0, sizeof(*state));
+	for (i = 0; i < 8; ++i)
+		state->h[i] = blake2s_iv[i];
+	state->h[0] ^= param;
+}
+
+void blake2s_init(struct blake2s_state *state, const size_t outlen)
+{
+	WARN_ON(IS_ENABLED(DEBUG) && (!outlen || outlen > BLAKE2S_HASH_SIZE));
+	blake2s_init_param(state, 0x01010000 | outlen);
+	state->outlen = outlen;
+}
+EXPORT_SYMBOL(blake2s_init);
+
+void blake2s_init_key(struct blake2s_state *state, const size_t outlen,
+		      const void *key, const size_t keylen)
+{
+	u8 block[BLAKE2S_BLOCK_SIZE] = { 0 };
+
+	WARN_ON(IS_ENABLED(DEBUG) && (!outlen || outlen > BLAKE2S_HASH_SIZE ||
+		!key || !keylen || keylen > BLAKE2S_KEY_SIZE));
+	blake2s_init_param(state, 0x01010000 | keylen << 8 | outlen);
+	state->outlen = outlen;
+	memcpy(block, key, keylen);
+	blake2s_update(state, block, BLAKE2S_BLOCK_SIZE);
+	memzero_explicit(block, BLAKE2S_BLOCK_SIZE);
+}
+EXPORT_SYMBOL(blake2s_init_key);
+
+#if defined(CONFIG_ZINC_ARCH_X86_64)
+#include "blake2s-x86_64-glue.c"
+#else
+static bool *const blake2s_nobs[] __initconst = { };
+static void __init blake2s_fpu_init(void)
+{
+}
+static inline bool blake2s_compress_arch(struct blake2s_state *state,
+					 const u8 *block, size_t nblocks,
+					 const u32 inc)
+{
+	return false;
+}
+#endif
+
+static inline void blake2s_compress(struct blake2s_state *state,
+				    const u8 *block, size_t nblocks,
+				    const u32 inc)
+{
+	u32 m[16];
+	u32 v[16];
+	int i;
+
+	WARN_ON(IS_ENABLED(DEBUG) &&
+		(nblocks > 1 && inc != BLAKE2S_BLOCK_SIZE));
+
+	if (blake2s_compress_arch(state, block, nblocks, inc))
+		return;
+
+	while (nblocks > 0) {
+		blake2s_increment_counter(state, inc);
+		memcpy(m, block, BLAKE2S_BLOCK_SIZE);
+		le32_to_cpu_array(m, ARRAY_SIZE(m));
+		memcpy(v, state->h, 32);
+		v[ 8] = blake2s_iv[0];
+		v[ 9] = blake2s_iv[1];
+		v[10] = blake2s_iv[2];
+		v[11] = blake2s_iv[3];
+		v[12] = blake2s_iv[4] ^ state->t[0];
+		v[13] = blake2s_iv[5] ^ state->t[1];
+		v[14] = blake2s_iv[6] ^ state->f[0];
+		v[15] = blake2s_iv[7] ^ state->f[1];
+
+#define G(r, i, a, b, c, d) do { \
+	a += b + m[blake2s_sigma[r][2 * i + 0]]; \
+	d = ror32(d ^ a, 16); \
+	c += d; \
+	b = ror32(b ^ c, 12); \
+	a += b + m[blake2s_sigma[r][2 * i + 1]]; \
+	d = ror32(d ^ a, 8); \
+	c += d; \
+	b = ror32(b ^ c, 7); \
+} while (0)
+
+#define ROUND(r) do { \
+	G(r, 0, v[0], v[ 4], v[ 8], v[12]); \
+	G(r, 1, v[1], v[ 5], v[ 9], v[13]); \
+	G(r, 2, v[2], v[ 6], v[10], v[14]); \
+	G(r, 3, v[3], v[ 7], v[11], v[15]); \
+	G(r, 4, v[0], v[ 5], v[10], v[15]); \
+	G(r, 5, v[1], v[ 6], v[11], v[12]); \
+	G(r, 6, v[2], v[ 7], v[ 8], v[13]); \
+	G(r, 7, v[3], v[ 4], v[ 9], v[14]); \
+} while (0)
+		ROUND(0);
+		ROUND(1);
+		ROUND(2);
+		ROUND(3);
+		ROUND(4);
+		ROUND(5);
+		ROUND(6);
+		ROUND(7);
+		ROUND(8);
+		ROUND(9);
+
+#undef G
+#undef ROUND
+
+		for (i = 0; i < 8; ++i)
+			state->h[i] ^= v[i] ^ v[i + 8];
+
+		block += BLAKE2S_BLOCK_SIZE;
+		--nblocks;
+	}
+}
+
+void blake2s_update(struct blake2s_state *state, const u8 *in, size_t inlen)
+{
+	const size_t fill = BLAKE2S_BLOCK_SIZE - state->buflen;
+
+	if (unlikely(!inlen))
+		return;
+	if (inlen > fill) {
+		memcpy(state->buf + state->buflen, in, fill);
+		blake2s_compress(state, state->buf, 1, BLAKE2S_BLOCK_SIZE);
+		state->buflen = 0;
+		in += fill;
+		inlen -= fill;
+	}
+	if (inlen > BLAKE2S_BLOCK_SIZE) {
+		const size_t nblocks = DIV_ROUND_UP(inlen, BLAKE2S_BLOCK_SIZE);
+		/* Hash one less (full) block than strictly possible */
+		blake2s_compress(state, in, nblocks - 1, BLAKE2S_BLOCK_SIZE);
+		in += BLAKE2S_BLOCK_SIZE * (nblocks - 1);
+		inlen -= BLAKE2S_BLOCK_SIZE * (nblocks - 1);
+	}
+	memcpy(state->buf + state->buflen, in, inlen);
+	state->buflen += inlen;
+}
+EXPORT_SYMBOL(blake2s_update);
+
+void blake2s_final(struct blake2s_state *state, u8 *out)
+{
+	WARN_ON(IS_ENABLED(DEBUG) && !out);
+	blake2s_set_lastblock(state);
+	memset(state->buf + state->buflen, 0,
+	       BLAKE2S_BLOCK_SIZE - state->buflen); /* Padding */
+	blake2s_compress(state, state->buf, 1, state->buflen);
+	cpu_to_le32_array(state->h, ARRAY_SIZE(state->h));
+	memcpy(out, state->h, state->outlen);
+	memzero_explicit(state, sizeof(*state));
+}
+EXPORT_SYMBOL(blake2s_final);
+
+void blake2s_hmac(u8 *out, const u8 *in, const u8 *key, const size_t outlen,
+		  const size_t inlen, const size_t keylen)
+{
+	struct blake2s_state state;
+	u8 x_key[BLAKE2S_BLOCK_SIZE] __aligned(__alignof__(u32)) = { 0 };
+	u8 i_hash[BLAKE2S_HASH_SIZE] __aligned(__alignof__(u32));
+	int i;
+
+	if (keylen > BLAKE2S_BLOCK_SIZE) {
+		blake2s_init(&state, BLAKE2S_HASH_SIZE);
+		blake2s_update(&state, key, keylen);
+		blake2s_final(&state, x_key);
+	} else
+		memcpy(x_key, key, keylen);
+
+	for (i = 0; i < BLAKE2S_BLOCK_SIZE; ++i)
+		x_key[i] ^= 0x36;
+
+	blake2s_init(&state, BLAKE2S_HASH_SIZE);
+	blake2s_update(&state, x_key, BLAKE2S_BLOCK_SIZE);
+	blake2s_update(&state, in, inlen);
+	blake2s_final(&state, i_hash);
+
+	for (i = 0; i < BLAKE2S_BLOCK_SIZE; ++i)
+		x_key[i] ^= 0x5c ^ 0x36;
+
+	blake2s_init(&state, BLAKE2S_HASH_SIZE);
+	blake2s_update(&state, x_key, BLAKE2S_BLOCK_SIZE);
+	blake2s_update(&state, i_hash, BLAKE2S_HASH_SIZE);
+	blake2s_final(&state, i_hash);
+
+	memcpy(out, i_hash, outlen);
+	memzero_explicit(x_key, BLAKE2S_BLOCK_SIZE);
+	memzero_explicit(i_hash, BLAKE2S_HASH_SIZE);
+}
+EXPORT_SYMBOL(blake2s_hmac);
+
+#include "../selftest/blake2s.c"
+
+static bool nosimd __initdata = false;
+
+#ifndef COMPAT_ZINC_IS_A_MODULE
+int __init blake2s_mod_init(void)
+#else
+static int __init mod_init(void)
+#endif
+{
+	if (!nosimd)
+		blake2s_fpu_init();
+	if (!selftest_run("blake2s", blake2s_selftest, blake2s_nobs,
+			  ARRAY_SIZE(blake2s_nobs)))
+		return -ENOTRECOVERABLE;
+	return 0;
+}
+
+#ifdef COMPAT_ZINC_IS_A_MODULE
+static void __exit mod_exit(void)
+{
+}
+
+module_param(nosimd, bool, 0);
+module_init(mod_init);
+module_exit(mod_exit);
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("BLAKE2s hash function");
+MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");
+#endif
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/blake2s/blake2s-x86_64-glue.c b/net/wireguard/crypto/zinc/blake2s/blake2s-x86_64-glue.c
--- a/net/wireguard/crypto/zinc/blake2s/blake2s-x86_64-glue.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/blake2s/blake2s-x86_64-glue.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,72 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#include <linux/simd.h>
+#include <asm/cpufeature.h>
+#include <asm/processor.h>
+#include <asm/fpu/api.h>
+
+asmlinkage void blake2s_compress_ssse3(struct blake2s_state *state,
+				       const u8 *block, const size_t nblocks,
+				       const u32 inc);
+asmlinkage void blake2s_compress_avx512(struct blake2s_state *state,
+					const u8 *block, const size_t nblocks,
+					const u32 inc);
+
+static bool blake2s_use_ssse3 __ro_after_init;
+static bool blake2s_use_avx512 __ro_after_init;
+static bool *const blake2s_nobs[] __initconst = { &blake2s_use_ssse3,
+						  &blake2s_use_avx512 };
+
+static void __init blake2s_fpu_init(void)
+{
+	blake2s_use_ssse3 = boot_cpu_has(X86_FEATURE_SSSE3);
+#ifndef COMPAT_CANNOT_USE_AVX512
+	blake2s_use_avx512 =
+		boot_cpu_has(X86_FEATURE_AVX) &&
+		boot_cpu_has(X86_FEATURE_AVX2) &&
+		boot_cpu_has(X86_FEATURE_AVX512F) &&
+		boot_cpu_has(X86_FEATURE_AVX512VL) &&
+		cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM |
+				  XFEATURE_MASK_AVX512, NULL);
+#endif
+}
+
+static inline bool blake2s_compress_arch(struct blake2s_state *state,
+					 const u8 *block, size_t nblocks,
+					 const u32 inc)
+{
+	simd_context_t simd_context;
+	bool used_arch = false;
+
+	/* SIMD disables preemption, so relax after processing each page. */
+	BUILD_BUG_ON(PAGE_SIZE / BLAKE2S_BLOCK_SIZE < 8);
+
+	simd_get(&simd_context);
+
+	if (!IS_ENABLED(CONFIG_AS_SSSE3) || !blake2s_use_ssse3 ||
+	    !simd_use(&simd_context))
+		goto out;
+	used_arch = true;
+
+	for (;;) {
+		const size_t blocks = min_t(size_t, nblocks,
+					    PAGE_SIZE / BLAKE2S_BLOCK_SIZE);
+
+		if (IS_ENABLED(CONFIG_AS_AVX512) && blake2s_use_avx512)
+			blake2s_compress_avx512(state, block, blocks, inc);
+		else
+			blake2s_compress_ssse3(state, block, blocks, inc);
+
+		nblocks -= blocks;
+		if (!nblocks)
+			break;
+		block += blocks * BLAKE2S_BLOCK_SIZE;
+		simd_relax(&simd_context);
+	}
+out:
+	simd_put(&simd_context);
+	return used_arch;
+}
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/blake2s/blake2s-x86_64.S b/net/wireguard/crypto/zinc/blake2s/blake2s-x86_64.S
--- a/net/wireguard/crypto/zinc/blake2s/blake2s-x86_64.S	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/blake2s/blake2s-x86_64.S	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,258 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ * Copyright (C) 2017-2019 Samuel Neves <sneves@dei.uc.pt>. All Rights Reserved.
+ */
+
+#include <linux/linkage.h>
+
+.section .rodata.cst32.BLAKE2S_IV, "aM", @progbits, 32
+.align 32
+IV:	.octa 0xA54FF53A3C6EF372BB67AE856A09E667
+	.octa 0x5BE0CD191F83D9AB9B05688C510E527F
+.section .rodata.cst16.ROT16, "aM", @progbits, 16
+.align 16
+ROT16:	.octa 0x0D0C0F0E09080B0A0504070601000302
+.section .rodata.cst16.ROR328, "aM", @progbits, 16
+.align 16
+ROR328:	.octa 0x0C0F0E0D080B0A090407060500030201
+.section .rodata.cst64.BLAKE2S_SIGMA, "aM", @progbits, 160
+.align 64
+SIGMA:
+.byte  0,  2,  4,  6,  1,  3,  5,  7, 14,  8, 10, 12, 15,  9, 11, 13
+.byte 14,  4,  9, 13, 10,  8, 15,  6,  5,  1,  0, 11,  3, 12,  2,  7
+.byte 11, 12,  5, 15,  8,  0,  2, 13,  9, 10,  3,  7,  4, 14,  6,  1
+.byte  7,  3, 13, 11,  9,  1, 12, 14, 15,  2,  5,  4,  8,  6, 10,  0
+.byte  9,  5,  2, 10,  0,  7,  4, 15,  3, 14, 11,  6, 13,  1, 12,  8
+.byte  2,  6,  0,  8, 12, 10, 11,  3,  1,  4,  7, 15,  9, 13,  5, 14
+.byte 12,  1, 14,  4,  5, 15, 13, 10,  8,  0,  6,  9, 11,  7,  3,  2
+.byte 13,  7, 12,  3, 11, 14,  1,  9,  2,  5, 15,  8, 10,  0,  4,  6
+.byte  6, 14, 11,  0, 15,  9,  3,  8, 10, 12, 13,  1,  5,  2,  7,  4
+.byte 10,  8,  7,  1,  2,  4,  6,  5, 13, 15,  9,  3,  0, 11, 14, 12
+#ifdef CONFIG_AS_AVX512
+.section .rodata.cst64.BLAKE2S_SIGMA2, "aM", @progbits, 640
+.align 64
+SIGMA2:
+.long  0,  2,  4,  6,  1,  3,  5,  7, 14,  8, 10, 12, 15,  9, 11, 13
+.long  8,  2, 13, 15, 10,  9, 12,  3,  6,  4,  0, 14,  5, 11,  1,  7
+.long 11, 13,  8,  6,  5, 10, 14,  3,  2,  4, 12, 15,  1,  0,  7,  9
+.long 11, 10,  7,  0,  8, 15,  1, 13,  3,  6,  2, 12,  4, 14,  9,  5
+.long  4, 10,  9, 14, 15,  0, 11,  8,  1,  7,  3, 13,  2,  5,  6, 12
+.long  2, 11,  4, 15, 14,  3, 10,  8, 13,  6,  5,  7,  0, 12,  1,  9
+.long  4,  8, 15,  9, 14, 11, 13,  5,  3,  2,  1, 12,  6, 10,  7,  0
+.long  6, 13,  0, 14, 12,  2,  1, 11, 15,  4,  5,  8,  7,  9,  3, 10
+.long 15,  5,  4, 13, 10,  7,  3, 11, 12,  2,  0,  6,  9,  8,  1, 14
+.long  8,  7, 14, 11, 13, 15,  0, 12, 10,  4,  5,  6,  3,  2,  1,  9
+#endif /* CONFIG_AS_AVX512 */
+
+.text
+#ifdef CONFIG_AS_SSSE3
+SYM_FUNC_START(blake2s_compress_ssse3)
+	testq		%rdx,%rdx
+	je		.Lendofloop
+	movdqu		(%rdi),%xmm0
+	movdqu		0x10(%rdi),%xmm1
+	movdqa		ROT16(%rip),%xmm12
+	movdqa		ROR328(%rip),%xmm13
+	movdqu		0x20(%rdi),%xmm14
+	movq		%rcx,%xmm15
+	leaq		SIGMA+0xa0(%rip),%r8
+	jmp		.Lbeginofloop
+	.align		32
+.Lbeginofloop:
+	movdqa		%xmm0,%xmm10
+	movdqa		%xmm1,%xmm11
+	paddq		%xmm15,%xmm14
+	movdqa		IV(%rip),%xmm2
+	movdqa		%xmm14,%xmm3
+	pxor		IV+0x10(%rip),%xmm3
+	leaq		SIGMA(%rip),%rcx
+.Lroundloop:
+	movzbl		(%rcx),%eax
+	movd		(%rsi,%rax,4),%xmm4
+	movzbl		0x1(%rcx),%eax
+	movd		(%rsi,%rax,4),%xmm5
+	movzbl		0x2(%rcx),%eax
+	movd		(%rsi,%rax,4),%xmm6
+	movzbl		0x3(%rcx),%eax
+	movd		(%rsi,%rax,4),%xmm7
+	punpckldq	%xmm5,%xmm4
+	punpckldq	%xmm7,%xmm6
+	punpcklqdq	%xmm6,%xmm4
+	paddd		%xmm4,%xmm0
+	paddd		%xmm1,%xmm0
+	pxor		%xmm0,%xmm3
+	pshufb		%xmm12,%xmm3
+	paddd		%xmm3,%xmm2
+	pxor		%xmm2,%xmm1
+	movdqa		%xmm1,%xmm8
+	psrld		$0xc,%xmm1
+	pslld		$0x14,%xmm8
+	por		%xmm8,%xmm1
+	movzbl		0x4(%rcx),%eax
+	movd		(%rsi,%rax,4),%xmm5
+	movzbl		0x5(%rcx),%eax
+	movd		(%rsi,%rax,4),%xmm6
+	movzbl		0x6(%rcx),%eax
+	movd		(%rsi,%rax,4),%xmm7
+	movzbl		0x7(%rcx),%eax
+	movd		(%rsi,%rax,4),%xmm4
+	punpckldq	%xmm6,%xmm5
+	punpckldq	%xmm4,%xmm7
+	punpcklqdq	%xmm7,%xmm5
+	paddd		%xmm5,%xmm0
+	paddd		%xmm1,%xmm0
+	pxor		%xmm0,%xmm3
+	pshufb		%xmm13,%xmm3
+	paddd		%xmm3,%xmm2
+	pxor		%xmm2,%xmm1
+	movdqa		%xmm1,%xmm8
+	psrld		$0x7,%xmm1
+	pslld		$0x19,%xmm8
+	por		%xmm8,%xmm1
+	pshufd		$0x93,%xmm0,%xmm0
+	pshufd		$0x4e,%xmm3,%xmm3
+	pshufd		$0x39,%xmm2,%xmm2
+	movzbl		0x8(%rcx),%eax
+	movd		(%rsi,%rax,4),%xmm6
+	movzbl		0x9(%rcx),%eax
+	movd		(%rsi,%rax,4),%xmm7
+	movzbl		0xa(%rcx),%eax
+	movd		(%rsi,%rax,4),%xmm4
+	movzbl		0xb(%rcx),%eax
+	movd		(%rsi,%rax,4),%xmm5
+	punpckldq	%xmm7,%xmm6
+	punpckldq	%xmm5,%xmm4
+	punpcklqdq	%xmm4,%xmm6
+	paddd		%xmm6,%xmm0
+	paddd		%xmm1,%xmm0
+	pxor		%xmm0,%xmm3
+	pshufb		%xmm12,%xmm3
+	paddd		%xmm3,%xmm2
+	pxor		%xmm2,%xmm1
+	movdqa		%xmm1,%xmm8
+	psrld		$0xc,%xmm1
+	pslld		$0x14,%xmm8
+	por		%xmm8,%xmm1
+	movzbl		0xc(%rcx),%eax
+	movd		(%rsi,%rax,4),%xmm7
+	movzbl		0xd(%rcx),%eax
+	movd		(%rsi,%rax,4),%xmm4
+	movzbl		0xe(%rcx),%eax
+	movd		(%rsi,%rax,4),%xmm5
+	movzbl		0xf(%rcx),%eax
+	movd		(%rsi,%rax,4),%xmm6
+	punpckldq	%xmm4,%xmm7
+	punpckldq	%xmm6,%xmm5
+	punpcklqdq	%xmm5,%xmm7
+	paddd		%xmm7,%xmm0
+	paddd		%xmm1,%xmm0
+	pxor		%xmm0,%xmm3
+	pshufb		%xmm13,%xmm3
+	paddd		%xmm3,%xmm2
+	pxor		%xmm2,%xmm1
+	movdqa		%xmm1,%xmm8
+	psrld		$0x7,%xmm1
+	pslld		$0x19,%xmm8
+	por		%xmm8,%xmm1
+	pshufd		$0x39,%xmm0,%xmm0
+	pshufd		$0x4e,%xmm3,%xmm3
+	pshufd		$0x93,%xmm2,%xmm2
+	addq		$0x10,%rcx
+	cmpq		%r8,%rcx
+	jnz		.Lroundloop
+	pxor		%xmm2,%xmm0
+	pxor		%xmm3,%xmm1
+	pxor		%xmm10,%xmm0
+	pxor		%xmm11,%xmm1
+	addq		$0x40,%rsi
+	decq		%rdx
+	jnz		.Lbeginofloop
+	movdqu		%xmm0,(%rdi)
+	movdqu		%xmm1,0x10(%rdi)
+	movdqu		%xmm14,0x20(%rdi)
+.Lendofloop:
+	ret
+SYM_FUNC_END(blake2s_compress_ssse3)
+#endif /* CONFIG_AS_SSSE3 */
+
+#ifdef CONFIG_AS_AVX512
+SYM_FUNC_START(blake2s_compress_avx512)
+	vmovdqu		(%rdi),%xmm0
+	vmovdqu		0x10(%rdi),%xmm1
+	vmovdqu		0x20(%rdi),%xmm4
+	vmovq		%rcx,%xmm5
+	vmovdqa		IV(%rip),%xmm14
+	vmovdqa		IV+16(%rip),%xmm15
+	jmp		.Lblake2s_compress_avx512_mainloop
+.align 32
+.Lblake2s_compress_avx512_mainloop:
+	vmovdqa		%xmm0,%xmm10
+	vmovdqa		%xmm1,%xmm11
+	vpaddq		%xmm5,%xmm4,%xmm4
+	vmovdqa		%xmm14,%xmm2
+	vpxor		%xmm15,%xmm4,%xmm3
+	vmovdqu		(%rsi),%ymm6
+	vmovdqu		0x20(%rsi),%ymm7
+	addq		$0x40,%rsi
+	leaq		SIGMA2(%rip),%rax
+	movb		$0xa,%cl
+.Lblake2s_compress_avx512_roundloop:
+	addq		$0x40,%rax
+	vmovdqa		-0x40(%rax),%ymm8
+	vmovdqa		-0x20(%rax),%ymm9
+	vpermi2d	%ymm7,%ymm6,%ymm8
+	vpermi2d	%ymm7,%ymm6,%ymm9
+	vmovdqa		%ymm8,%ymm6
+	vmovdqa		%ymm9,%ymm7
+	vpaddd		%xmm8,%xmm0,%xmm0
+	vpaddd		%xmm1,%xmm0,%xmm0
+	vpxor		%xmm0,%xmm3,%xmm3
+	vprord		$0x10,%xmm3,%xmm3
+	vpaddd		%xmm3,%xmm2,%xmm2
+	vpxor		%xmm2,%xmm1,%xmm1
+	vprord		$0xc,%xmm1,%xmm1
+	vextracti128	$0x1,%ymm8,%xmm8
+	vpaddd		%xmm8,%xmm0,%xmm0
+	vpaddd		%xmm1,%xmm0,%xmm0
+	vpxor		%xmm0,%xmm3,%xmm3
+	vprord		$0x8,%xmm3,%xmm3
+	vpaddd		%xmm3,%xmm2,%xmm2
+	vpxor		%xmm2,%xmm1,%xmm1
+	vprord		$0x7,%xmm1,%xmm1
+	vpshufd		$0x93,%xmm0,%xmm0
+	vpshufd		$0x4e,%xmm3,%xmm3
+	vpshufd		$0x39,%xmm2,%xmm2
+	vpaddd		%xmm9,%xmm0,%xmm0
+	vpaddd		%xmm1,%xmm0,%xmm0
+	vpxor		%xmm0,%xmm3,%xmm3
+	vprord		$0x10,%xmm3,%xmm3
+	vpaddd		%xmm3,%xmm2,%xmm2
+	vpxor		%xmm2,%xmm1,%xmm1
+	vprord		$0xc,%xmm1,%xmm1
+	vextracti128	$0x1,%ymm9,%xmm9
+	vpaddd		%xmm9,%xmm0,%xmm0
+	vpaddd		%xmm1,%xmm0,%xmm0
+	vpxor		%xmm0,%xmm3,%xmm3
+	vprord		$0x8,%xmm3,%xmm3
+	vpaddd		%xmm3,%xmm2,%xmm2
+	vpxor		%xmm2,%xmm1,%xmm1
+	vprord		$0x7,%xmm1,%xmm1
+	vpshufd		$0x39,%xmm0,%xmm0
+	vpshufd		$0x4e,%xmm3,%xmm3
+	vpshufd		$0x93,%xmm2,%xmm2
+	decb		%cl
+	jne		.Lblake2s_compress_avx512_roundloop
+	vpxor		%xmm10,%xmm0,%xmm0
+	vpxor		%xmm11,%xmm1,%xmm1
+	vpxor		%xmm2,%xmm0,%xmm0
+	vpxor		%xmm3,%xmm1,%xmm1
+	decq		%rdx
+	jne		.Lblake2s_compress_avx512_mainloop
+	vmovdqu		%xmm0,(%rdi)
+	vmovdqu		%xmm1,0x10(%rdi)
+	vmovdqu		%xmm4,0x20(%rdi)
+	vzeroupper
+	retq
+SYM_FUNC_END(blake2s_compress_avx512)
+#endif /* CONFIG_AS_AVX512 */
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/chacha20/chacha20-arm64.pl b/net/wireguard/crypto/zinc/chacha20/chacha20-arm64.pl
--- a/net/wireguard/crypto/zinc/chacha20/chacha20-arm64.pl	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/chacha20/chacha20-arm64.pl	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,1163 @@
+#!/usr/bin/env perl
+# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+#
+# This code is taken from the OpenSSL project but the author, Andy Polyakov,
+# has relicensed it under the licenses specified in the SPDX header above.
+# The original headers, including the original license headers, are
+# included below for completeness.
+#
+# ====================================================================
+# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
+# project. The module is, however, dual licensed under OpenSSL and
+# CRYPTOGAMS licenses depending on where you obtain it. For further
+# details see http://www.openssl.org/~appro/cryptogams/.
+# ====================================================================
+#
+# June 2015
+#
+# ChaCha20 for ARMv8.
+#
+# Performance in cycles per byte out of large buffer.
+#
+#			IALU/gcc-4.9    3xNEON+1xIALU	6xNEON+2xIALU(*)
+#
+# Apple A7		5.50/+49%       3.33            1.70
+# Cortex-A53		8.40/+80%       4.72		4.72(**)
+# Cortex-A57		8.06/+43%       4.90            4.43(***)
+# Denver		4.50/+82%       2.63		2.67(**)
+# X-Gene		9.50/+46%       8.82		8.89(**)
+# Mongoose		8.00/+44%	3.64		3.25(***)
+# Kryo			8.17/+50%	4.83		4.65(***)
+#
+# (*)	since no non-Apple processor exhibits significantly better
+#	performance, the code path is #ifdef __APPLE__-ed;
+# (**)	it's expected that doubling interleave factor doesn't help
+#	all processors, only those with higher NEON latency and
+#	higher instruction issue rate;
+# (***)	expected improvement was actually higher;
+
+$flavour=shift;
+if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; }
+else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} }
+
+if ($flavour && $flavour ne "void") {
+    $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
+    ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
+    ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
+    die "can't locate arm-xlate.pl";
+
+    open STDOUT,"| \"$^X\" $xlate $flavour $output";
+} else {
+    open STDOUT,">$output";
+}
+
+sub AUTOLOAD()		# thunk [simplified] x86-style perlasm
+{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
+  my $arg = pop;
+    $arg = "#$arg" if ($arg*1 eq $arg);
+    $code .= "\t$opcode\t".join(',',@_,$arg)."\n";
+}
+
+my ($out,$inp,$len,$key,$ctr) = map("x$_",(0..4));
+
+my @x=map("x$_",(5..17,19..21));
+my @d=map("x$_",(22..28,30));
+
+sub ROUND {
+my ($a0,$b0,$c0,$d0)=@_;
+my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
+my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
+my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
+
+    (
+	"&add_32	(@x[$a0],@x[$a0],@x[$b0])",
+	 "&add_32	(@x[$a1],@x[$a1],@x[$b1])",
+	  "&add_32	(@x[$a2],@x[$a2],@x[$b2])",
+	   "&add_32	(@x[$a3],@x[$a3],@x[$b3])",
+	"&eor_32	(@x[$d0],@x[$d0],@x[$a0])",
+	 "&eor_32	(@x[$d1],@x[$d1],@x[$a1])",
+	  "&eor_32	(@x[$d2],@x[$d2],@x[$a2])",
+	   "&eor_32	(@x[$d3],@x[$d3],@x[$a3])",
+	"&ror_32	(@x[$d0],@x[$d0],16)",
+	 "&ror_32	(@x[$d1],@x[$d1],16)",
+	  "&ror_32	(@x[$d2],@x[$d2],16)",
+	   "&ror_32	(@x[$d3],@x[$d3],16)",
+
+	"&add_32	(@x[$c0],@x[$c0],@x[$d0])",
+	 "&add_32	(@x[$c1],@x[$c1],@x[$d1])",
+	  "&add_32	(@x[$c2],@x[$c2],@x[$d2])",
+	   "&add_32	(@x[$c3],@x[$c3],@x[$d3])",
+	"&eor_32	(@x[$b0],@x[$b0],@x[$c0])",
+	 "&eor_32	(@x[$b1],@x[$b1],@x[$c1])",
+	  "&eor_32	(@x[$b2],@x[$b2],@x[$c2])",
+	   "&eor_32	(@x[$b3],@x[$b3],@x[$c3])",
+	"&ror_32	(@x[$b0],@x[$b0],20)",
+	 "&ror_32	(@x[$b1],@x[$b1],20)",
+	  "&ror_32	(@x[$b2],@x[$b2],20)",
+	   "&ror_32	(@x[$b3],@x[$b3],20)",
+
+	"&add_32	(@x[$a0],@x[$a0],@x[$b0])",
+	 "&add_32	(@x[$a1],@x[$a1],@x[$b1])",
+	  "&add_32	(@x[$a2],@x[$a2],@x[$b2])",
+	   "&add_32	(@x[$a3],@x[$a3],@x[$b3])",
+	"&eor_32	(@x[$d0],@x[$d0],@x[$a0])",
+	 "&eor_32	(@x[$d1],@x[$d1],@x[$a1])",
+	  "&eor_32	(@x[$d2],@x[$d2],@x[$a2])",
+	   "&eor_32	(@x[$d3],@x[$d3],@x[$a3])",
+	"&ror_32	(@x[$d0],@x[$d0],24)",
+	 "&ror_32	(@x[$d1],@x[$d1],24)",
+	  "&ror_32	(@x[$d2],@x[$d2],24)",
+	   "&ror_32	(@x[$d3],@x[$d3],24)",
+
+	"&add_32	(@x[$c0],@x[$c0],@x[$d0])",
+	 "&add_32	(@x[$c1],@x[$c1],@x[$d1])",
+	  "&add_32	(@x[$c2],@x[$c2],@x[$d2])",
+	   "&add_32	(@x[$c3],@x[$c3],@x[$d3])",
+	"&eor_32	(@x[$b0],@x[$b0],@x[$c0])",
+	 "&eor_32	(@x[$b1],@x[$b1],@x[$c1])",
+	  "&eor_32	(@x[$b2],@x[$b2],@x[$c2])",
+	   "&eor_32	(@x[$b3],@x[$b3],@x[$c3])",
+	"&ror_32	(@x[$b0],@x[$b0],25)",
+	 "&ror_32	(@x[$b1],@x[$b1],25)",
+	  "&ror_32	(@x[$b2],@x[$b2],25)",
+	   "&ror_32	(@x[$b3],@x[$b3],25)"
+    );
+}
+
+$code.=<<___;
+#ifndef	__KERNEL__
+# include "arm_arch.h"
+.extern	OPENSSL_armcap_P
+#else
+# define ChaCha20_ctr32 chacha20_arm
+# define ChaCha20_neon  chacha20_neon
+#endif
+
+.text
+
+.align	5
+.Lsigma:
+.quad	0x3320646e61707865,0x6b20657479622d32		// endian-neutral
+.Lone:
+.long	1,0,0,0
+#ifndef	__KERNEL__
+.LOPENSSL_armcap_P:
+# ifdef	__ILP32__
+.long	OPENSSL_armcap_P-.
+# else
+.quad	OPENSSL_armcap_P-.
+# endif
+#endif
+
+.globl	ChaCha20_ctr32
+.type	ChaCha20_ctr32,%function
+.align	5
+ChaCha20_ctr32:
+	cbz	$len,.Labort
+#ifndef	__KERNEL__
+	adr	@x[0],.LOPENSSL_armcap_P
+	cmp	$len,#192
+	b.lo	.Lshort
+# ifdef	__ILP32__
+	ldrsw	@x[1],[@x[0]]
+# else
+	ldr	@x[1],[@x[0]]
+# endif
+	ldr	w17,[@x[1],@x[0]]
+	tst	w17,#ARMV7_NEON
+	b.ne	ChaCha20_neon
+
+.Lshort:
+#endif
+	stp	x29,x30,[sp,#-96]!
+	add	x29,sp,#0
+
+	adr	@x[0],.Lsigma
+	stp	x19,x20,[sp,#16]
+	stp	x21,x22,[sp,#32]
+	stp	x23,x24,[sp,#48]
+	stp	x25,x26,[sp,#64]
+	stp	x27,x28,[sp,#80]
+	sub	sp,sp,#64
+
+	ldp	@d[0],@d[1],[@x[0]]		// load sigma
+	ldp	@d[2],@d[3],[$key]		// load key
+	ldp	@d[4],@d[5],[$key,#16]
+	ldp	@d[6],@d[7],[$ctr]		// load counter
+#ifdef	__AARCH64EB__
+	ror	@d[2],@d[2],#32
+	ror	@d[3],@d[3],#32
+	ror	@d[4],@d[4],#32
+	ror	@d[5],@d[5],#32
+	ror	@d[6],@d[6],#32
+	ror	@d[7],@d[7],#32
+#endif
+
+.Loop_outer:
+	mov.32	@x[0],@d[0]			// unpack key block
+	lsr	@x[1],@d[0],#32
+	mov.32	@x[2],@d[1]
+	lsr	@x[3],@d[1],#32
+	mov.32	@x[4],@d[2]
+	lsr	@x[5],@d[2],#32
+	mov.32	@x[6],@d[3]
+	lsr	@x[7],@d[3],#32
+	mov.32	@x[8],@d[4]
+	lsr	@x[9],@d[4],#32
+	mov.32	@x[10],@d[5]
+	lsr	@x[11],@d[5],#32
+	mov.32	@x[12],@d[6]
+	lsr	@x[13],@d[6],#32
+	mov.32	@x[14],@d[7]
+	lsr	@x[15],@d[7],#32
+
+	mov	$ctr,#10
+	subs	$len,$len,#64
+.Loop:
+	sub	$ctr,$ctr,#1
+___
+	foreach (&ROUND(0, 4, 8,12)) { eval; }
+	foreach (&ROUND(0, 5,10,15)) { eval; }
+$code.=<<___;
+	cbnz	$ctr,.Loop
+
+	add.32	@x[0],@x[0],@d[0]		// accumulate key block
+	add	@x[1],@x[1],@d[0],lsr#32
+	add.32	@x[2],@x[2],@d[1]
+	add	@x[3],@x[3],@d[1],lsr#32
+	add.32	@x[4],@x[4],@d[2]
+	add	@x[5],@x[5],@d[2],lsr#32
+	add.32	@x[6],@x[6],@d[3]
+	add	@x[7],@x[7],@d[3],lsr#32
+	add.32	@x[8],@x[8],@d[4]
+	add	@x[9],@x[9],@d[4],lsr#32
+	add.32	@x[10],@x[10],@d[5]
+	add	@x[11],@x[11],@d[5],lsr#32
+	add.32	@x[12],@x[12],@d[6]
+	add	@x[13],@x[13],@d[6],lsr#32
+	add.32	@x[14],@x[14],@d[7]
+	add	@x[15],@x[15],@d[7],lsr#32
+
+	b.lo	.Ltail
+
+	add	@x[0],@x[0],@x[1],lsl#32	// pack
+	add	@x[2],@x[2],@x[3],lsl#32
+	ldp	@x[1],@x[3],[$inp,#0]		// load input
+	add	@x[4],@x[4],@x[5],lsl#32
+	add	@x[6],@x[6],@x[7],lsl#32
+	ldp	@x[5],@x[7],[$inp,#16]
+	add	@x[8],@x[8],@x[9],lsl#32
+	add	@x[10],@x[10],@x[11],lsl#32
+	ldp	@x[9],@x[11],[$inp,#32]
+	add	@x[12],@x[12],@x[13],lsl#32
+	add	@x[14],@x[14],@x[15],lsl#32
+	ldp	@x[13],@x[15],[$inp,#48]
+	add	$inp,$inp,#64
+#ifdef	__AARCH64EB__
+	rev	@x[0],@x[0]
+	rev	@x[2],@x[2]
+	rev	@x[4],@x[4]
+	rev	@x[6],@x[6]
+	rev	@x[8],@x[8]
+	rev	@x[10],@x[10]
+	rev	@x[12],@x[12]
+	rev	@x[14],@x[14]
+#endif
+	eor	@x[0],@x[0],@x[1]
+	eor	@x[2],@x[2],@x[3]
+	eor	@x[4],@x[4],@x[5]
+	eor	@x[6],@x[6],@x[7]
+	eor	@x[8],@x[8],@x[9]
+	eor	@x[10],@x[10],@x[11]
+	eor	@x[12],@x[12],@x[13]
+	eor	@x[14],@x[14],@x[15]
+
+	stp	@x[0],@x[2],[$out,#0]		// store output
+	 add	@d[6],@d[6],#1			// increment counter
+	stp	@x[4],@x[6],[$out,#16]
+	stp	@x[8],@x[10],[$out,#32]
+	stp	@x[12],@x[14],[$out,#48]
+	add	$out,$out,#64
+
+	b.hi	.Loop_outer
+
+	ldp	x19,x20,[x29,#16]
+	add	sp,sp,#64
+	ldp	x21,x22,[x29,#32]
+	ldp	x23,x24,[x29,#48]
+	ldp	x25,x26,[x29,#64]
+	ldp	x27,x28,[x29,#80]
+	ldp	x29,x30,[sp],#96
+.Labort:
+	ret
+
+.align	4
+.Ltail:
+	add	$len,$len,#64
+.Less_than_64:
+	sub	$out,$out,#1
+	add	$inp,$inp,$len
+	add	$out,$out,$len
+	add	$ctr,sp,$len
+	neg	$len,$len
+
+	add	@x[0],@x[0],@x[1],lsl#32	// pack
+	add	@x[2],@x[2],@x[3],lsl#32
+	add	@x[4],@x[4],@x[5],lsl#32
+	add	@x[6],@x[6],@x[7],lsl#32
+	add	@x[8],@x[8],@x[9],lsl#32
+	add	@x[10],@x[10],@x[11],lsl#32
+	add	@x[12],@x[12],@x[13],lsl#32
+	add	@x[14],@x[14],@x[15],lsl#32
+#ifdef	__AARCH64EB__
+	rev	@x[0],@x[0]
+	rev	@x[2],@x[2]
+	rev	@x[4],@x[4]
+	rev	@x[6],@x[6]
+	rev	@x[8],@x[8]
+	rev	@x[10],@x[10]
+	rev	@x[12],@x[12]
+	rev	@x[14],@x[14]
+#endif
+	stp	@x[0],@x[2],[sp,#0]
+	stp	@x[4],@x[6],[sp,#16]
+	stp	@x[8],@x[10],[sp,#32]
+	stp	@x[12],@x[14],[sp,#48]
+
+.Loop_tail:
+	ldrb	w10,[$inp,$len]
+	ldrb	w11,[$ctr,$len]
+	add	$len,$len,#1
+	eor	w10,w10,w11
+	strb	w10,[$out,$len]
+	cbnz	$len,.Loop_tail
+
+	stp	xzr,xzr,[sp,#0]
+	stp	xzr,xzr,[sp,#16]
+	stp	xzr,xzr,[sp,#32]
+	stp	xzr,xzr,[sp,#48]
+
+	ldp	x19,x20,[x29,#16]
+	add	sp,sp,#64
+	ldp	x21,x22,[x29,#32]
+	ldp	x23,x24,[x29,#48]
+	ldp	x25,x26,[x29,#64]
+	ldp	x27,x28,[x29,#80]
+	ldp	x29,x30,[sp],#96
+	ret
+.size	ChaCha20_ctr32,.-ChaCha20_ctr32
+___
+
+{{{
+my ($A0,$B0,$C0,$D0,$A1,$B1,$C1,$D1,$A2,$B2,$C2,$D2,$T0,$T1,$T2,$T3) =
+    map("v$_.4s",(0..7,16..23));
+my (@K)=map("v$_.4s",(24..30));
+my $ONE="v31.4s";
+
+sub NEONROUND {
+my $odd = pop;
+my ($a,$b,$c,$d,$t)=@_;
+
+	(
+	"&add		('$a','$a','$b')",
+	"&eor		('$d','$d','$a')",
+	"&rev32_16	('$d','$d')",		# vrot ($d,16)
+
+	"&add		('$c','$c','$d')",
+	"&eor		('$t','$b','$c')",
+	"&ushr		('$b','$t',20)",
+	"&sli		('$b','$t',12)",
+
+	"&add		('$a','$a','$b')",
+	"&eor		('$t','$d','$a')",
+	"&ushr		('$d','$t',24)",
+	"&sli		('$d','$t',8)",
+
+	"&add		('$c','$c','$d')",
+	"&eor		('$t','$b','$c')",
+	"&ushr		('$b','$t',25)",
+	"&sli		('$b','$t',7)",
+
+	"&ext		('$a','$a','$a',$odd?4:12)",
+	"&ext		('$d','$d','$d',8)",
+	"&ext		('$c','$c','$c',$odd?12:4)"
+	);
+}
+
+$code.=<<___;
+#if !defined(__KERNEL__) || defined(CONFIG_KERNEL_MODE_NEON)
+#ifdef __KERNEL__
+.globl  ChaCha20_neon
+.type   ChaCha20_neon,%function
+#endif
+.type	ChaCha20_neon,%function
+.align	5
+ChaCha20_neon:
+	stp	x29,x30,[sp,#-96]!
+	add	x29,sp,#0
+
+	adr	@x[0],.Lsigma
+	stp	x19,x20,[sp,#16]
+	stp	x21,x22,[sp,#32]
+	stp	x23,x24,[sp,#48]
+	stp	x25,x26,[sp,#64]
+	stp	x27,x28,[sp,#80]
+#ifdef	__APPLE__
+	cmp	$len,#512
+	b.hs	.L512_or_more_neon
+#endif
+
+	sub	sp,sp,#64
+
+	ldp	@d[0],@d[1],[@x[0]]		// load sigma
+	ld1	{@K[0]},[@x[0]],#16
+	ldp	@d[2],@d[3],[$key]		// load key
+	ldp	@d[4],@d[5],[$key,#16]
+	ld1	{@K[1],@K[2]},[$key]
+	ldp	@d[6],@d[7],[$ctr]		// load counter
+	ld1	{@K[3]},[$ctr]
+	ld1	{$ONE},[@x[0]]
+#ifdef	__AARCH64EB__
+	rev64	@K[0],@K[0]
+	ror	@d[2],@d[2],#32
+	ror	@d[3],@d[3],#32
+	ror	@d[4],@d[4],#32
+	ror	@d[5],@d[5],#32
+	ror	@d[6],@d[6],#32
+	ror	@d[7],@d[7],#32
+#endif
+	add	@K[3],@K[3],$ONE		// += 1
+	add	@K[4],@K[3],$ONE
+	add	@K[5],@K[4],$ONE
+	shl	$ONE,$ONE,#2			// 1 -> 4
+
+.Loop_outer_neon:
+	mov.32	@x[0],@d[0]			// unpack key block
+	lsr	@x[1],@d[0],#32
+	 mov	$A0,@K[0]
+	mov.32	@x[2],@d[1]
+	lsr	@x[3],@d[1],#32
+	 mov	$A1,@K[0]
+	mov.32	@x[4],@d[2]
+	lsr	@x[5],@d[2],#32
+	 mov	$A2,@K[0]
+	mov.32	@x[6],@d[3]
+	 mov	$B0,@K[1]
+	lsr	@x[7],@d[3],#32
+	 mov	$B1,@K[1]
+	mov.32	@x[8],@d[4]
+	 mov	$B2,@K[1]
+	lsr	@x[9],@d[4],#32
+	 mov	$D0,@K[3]
+	mov.32	@x[10],@d[5]
+	 mov	$D1,@K[4]
+	lsr	@x[11],@d[5],#32
+	 mov	$D2,@K[5]
+	mov.32	@x[12],@d[6]
+	 mov	$C0,@K[2]
+	lsr	@x[13],@d[6],#32
+	 mov	$C1,@K[2]
+	mov.32	@x[14],@d[7]
+	 mov	$C2,@K[2]
+	lsr	@x[15],@d[7],#32
+
+	mov	$ctr,#10
+	subs	$len,$len,#256
+.Loop_neon:
+	sub	$ctr,$ctr,#1
+___
+	my @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,0);
+	my @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,0);
+	my @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,0);
+	my @thread3=&ROUND(0,4,8,12);
+
+	foreach (@thread0) {
+		eval;			eval(shift(@thread3));
+		eval(shift(@thread1));	eval(shift(@thread3));
+		eval(shift(@thread2));	eval(shift(@thread3));
+	}
+
+	@thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,1);
+	@thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,1);
+	@thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,1);
+	@thread3=&ROUND(0,5,10,15);
+
+	foreach (@thread0) {
+		eval;			eval(shift(@thread3));
+		eval(shift(@thread1));	eval(shift(@thread3));
+		eval(shift(@thread2));	eval(shift(@thread3));
+	}
+$code.=<<___;
+	cbnz	$ctr,.Loop_neon
+
+	add.32	@x[0],@x[0],@d[0]		// accumulate key block
+	 add	$A0,$A0,@K[0]
+	add	@x[1],@x[1],@d[0],lsr#32
+	 add	$A1,$A1,@K[0]
+	add.32	@x[2],@x[2],@d[1]
+	 add	$A2,$A2,@K[0]
+	add	@x[3],@x[3],@d[1],lsr#32
+	 add	$C0,$C0,@K[2]
+	add.32	@x[4],@x[4],@d[2]
+	 add	$C1,$C1,@K[2]
+	add	@x[5],@x[5],@d[2],lsr#32
+	 add	$C2,$C2,@K[2]
+	add.32	@x[6],@x[6],@d[3]
+	 add	$D0,$D0,@K[3]
+	add	@x[7],@x[7],@d[3],lsr#32
+	add.32	@x[8],@x[8],@d[4]
+	 add	$D1,$D1,@K[4]
+	add	@x[9],@x[9],@d[4],lsr#32
+	add.32	@x[10],@x[10],@d[5]
+	 add	$D2,$D2,@K[5]
+	add	@x[11],@x[11],@d[5],lsr#32
+	add.32	@x[12],@x[12],@d[6]
+	 add	$B0,$B0,@K[1]
+	add	@x[13],@x[13],@d[6],lsr#32
+	add.32	@x[14],@x[14],@d[7]
+	 add	$B1,$B1,@K[1]
+	add	@x[15],@x[15],@d[7],lsr#32
+	 add	$B2,$B2,@K[1]
+
+	b.lo	.Ltail_neon
+
+	add	@x[0],@x[0],@x[1],lsl#32	// pack
+	add	@x[2],@x[2],@x[3],lsl#32
+	ldp	@x[1],@x[3],[$inp,#0]		// load input
+	add	@x[4],@x[4],@x[5],lsl#32
+	add	@x[6],@x[6],@x[7],lsl#32
+	ldp	@x[5],@x[7],[$inp,#16]
+	add	@x[8],@x[8],@x[9],lsl#32
+	add	@x[10],@x[10],@x[11],lsl#32
+	ldp	@x[9],@x[11],[$inp,#32]
+	add	@x[12],@x[12],@x[13],lsl#32
+	add	@x[14],@x[14],@x[15],lsl#32
+	ldp	@x[13],@x[15],[$inp,#48]
+	add	$inp,$inp,#64
+#ifdef	__AARCH64EB__
+	rev	@x[0],@x[0]
+	rev	@x[2],@x[2]
+	rev	@x[4],@x[4]
+	rev	@x[6],@x[6]
+	rev	@x[8],@x[8]
+	rev	@x[10],@x[10]
+	rev	@x[12],@x[12]
+	rev	@x[14],@x[14]
+#endif
+	ld1.8	{$T0-$T3},[$inp],#64
+	eor	@x[0],@x[0],@x[1]
+	eor	@x[2],@x[2],@x[3]
+	eor	@x[4],@x[4],@x[5]
+	eor	@x[6],@x[6],@x[7]
+	eor	@x[8],@x[8],@x[9]
+	 eor	$A0,$A0,$T0
+	eor	@x[10],@x[10],@x[11]
+	 eor	$B0,$B0,$T1
+	eor	@x[12],@x[12],@x[13]
+	 eor	$C0,$C0,$T2
+	eor	@x[14],@x[14],@x[15]
+	 eor	$D0,$D0,$T3
+	 ld1.8	{$T0-$T3},[$inp],#64
+
+	stp	@x[0],@x[2],[$out,#0]		// store output
+	 add	@d[6],@d[6],#4			// increment counter
+	stp	@x[4],@x[6],[$out,#16]
+	 add	@K[3],@K[3],$ONE		// += 4
+	stp	@x[8],@x[10],[$out,#32]
+	 add	@K[4],@K[4],$ONE
+	stp	@x[12],@x[14],[$out,#48]
+	 add	@K[5],@K[5],$ONE
+	add	$out,$out,#64
+
+	st1.8	{$A0-$D0},[$out],#64
+	ld1.8	{$A0-$D0},[$inp],#64
+
+	eor	$A1,$A1,$T0
+	eor	$B1,$B1,$T1
+	eor	$C1,$C1,$T2
+	eor	$D1,$D1,$T3
+	st1.8	{$A1-$D1},[$out],#64
+
+	eor	$A2,$A2,$A0
+	eor	$B2,$B2,$B0
+	eor	$C2,$C2,$C0
+	eor	$D2,$D2,$D0
+	st1.8	{$A2-$D2},[$out],#64
+
+	b.hi	.Loop_outer_neon
+
+	ldp	x19,x20,[x29,#16]
+	add	sp,sp,#64
+	ldp	x21,x22,[x29,#32]
+	ldp	x23,x24,[x29,#48]
+	ldp	x25,x26,[x29,#64]
+	ldp	x27,x28,[x29,#80]
+	ldp	x29,x30,[sp],#96
+	ret
+
+.Ltail_neon:
+	add	$len,$len,#256
+	cmp	$len,#64
+	b.lo	.Less_than_64
+
+	add	@x[0],@x[0],@x[1],lsl#32	// pack
+	add	@x[2],@x[2],@x[3],lsl#32
+	ldp	@x[1],@x[3],[$inp,#0]		// load input
+	add	@x[4],@x[4],@x[5],lsl#32
+	add	@x[6],@x[6],@x[7],lsl#32
+	ldp	@x[5],@x[7],[$inp,#16]
+	add	@x[8],@x[8],@x[9],lsl#32
+	add	@x[10],@x[10],@x[11],lsl#32
+	ldp	@x[9],@x[11],[$inp,#32]
+	add	@x[12],@x[12],@x[13],lsl#32
+	add	@x[14],@x[14],@x[15],lsl#32
+	ldp	@x[13],@x[15],[$inp,#48]
+	add	$inp,$inp,#64
+#ifdef	__AARCH64EB__
+	rev	@x[0],@x[0]
+	rev	@x[2],@x[2]
+	rev	@x[4],@x[4]
+	rev	@x[6],@x[6]
+	rev	@x[8],@x[8]
+	rev	@x[10],@x[10]
+	rev	@x[12],@x[12]
+	rev	@x[14],@x[14]
+#endif
+	eor	@x[0],@x[0],@x[1]
+	eor	@x[2],@x[2],@x[3]
+	eor	@x[4],@x[4],@x[5]
+	eor	@x[6],@x[6],@x[7]
+	eor	@x[8],@x[8],@x[9]
+	eor	@x[10],@x[10],@x[11]
+	eor	@x[12],@x[12],@x[13]
+	eor	@x[14],@x[14],@x[15]
+
+	stp	@x[0],@x[2],[$out,#0]		// store output
+	 add	@d[6],@d[6],#4			// increment counter
+	stp	@x[4],@x[6],[$out,#16]
+	stp	@x[8],@x[10],[$out,#32]
+	stp	@x[12],@x[14],[$out,#48]
+	add	$out,$out,#64
+	b.eq	.Ldone_neon
+	sub	$len,$len,#64
+	cmp	$len,#64
+	b.lo	.Less_than_128
+
+	ld1.8	{$T0-$T3},[$inp],#64
+	eor	$A0,$A0,$T0
+	eor	$B0,$B0,$T1
+	eor	$C0,$C0,$T2
+	eor	$D0,$D0,$T3
+	st1.8	{$A0-$D0},[$out],#64
+	b.eq	.Ldone_neon
+	sub	$len,$len,#64
+	cmp	$len,#64
+	b.lo	.Less_than_192
+
+	ld1.8	{$T0-$T3},[$inp],#64
+	eor	$A1,$A1,$T0
+	eor	$B1,$B1,$T1
+	eor	$C1,$C1,$T2
+	eor	$D1,$D1,$T3
+	st1.8	{$A1-$D1},[$out],#64
+	b.eq	.Ldone_neon
+	sub	$len,$len,#64
+
+	st1.8	{$A2-$D2},[sp]
+	b	.Last_neon
+
+.Less_than_128:
+	st1.8	{$A0-$D0},[sp]
+	b	.Last_neon
+.Less_than_192:
+	st1.8	{$A1-$D1},[sp]
+	b	.Last_neon
+
+.align	4
+.Last_neon:
+	sub	$out,$out,#1
+	add	$inp,$inp,$len
+	add	$out,$out,$len
+	add	$ctr,sp,$len
+	neg	$len,$len
+
+.Loop_tail_neon:
+	ldrb	w10,[$inp,$len]
+	ldrb	w11,[$ctr,$len]
+	add	$len,$len,#1
+	eor	w10,w10,w11
+	strb	w10,[$out,$len]
+	cbnz	$len,.Loop_tail_neon
+
+	stp	xzr,xzr,[sp,#0]
+	stp	xzr,xzr,[sp,#16]
+	stp	xzr,xzr,[sp,#32]
+	stp	xzr,xzr,[sp,#48]
+
+.Ldone_neon:
+	ldp	x19,x20,[x29,#16]
+	add	sp,sp,#64
+	ldp	x21,x22,[x29,#32]
+	ldp	x23,x24,[x29,#48]
+	ldp	x25,x26,[x29,#64]
+	ldp	x27,x28,[x29,#80]
+	ldp	x29,x30,[sp],#96
+	ret
+.size	ChaCha20_neon,.-ChaCha20_neon
+___
+{
+my ($T0,$T1,$T2,$T3,$T4,$T5)=@K;
+my ($A0,$B0,$C0,$D0,$A1,$B1,$C1,$D1,$A2,$B2,$C2,$D2,
+    $A3,$B3,$C3,$D3,$A4,$B4,$C4,$D4,$A5,$B5,$C5,$D5) = map("v$_.4s",(0..23));
+
+$code.=<<___;
+#ifdef	__APPLE__
+.type	ChaCha20_512_neon,%function
+.align	5
+ChaCha20_512_neon:
+	stp	x29,x30,[sp,#-96]!
+	add	x29,sp,#0
+
+	adr	@x[0],.Lsigma
+	stp	x19,x20,[sp,#16]
+	stp	x21,x22,[sp,#32]
+	stp	x23,x24,[sp,#48]
+	stp	x25,x26,[sp,#64]
+	stp	x27,x28,[sp,#80]
+
+.L512_or_more_neon:
+	sub	sp,sp,#128+64
+
+	ldp	@d[0],@d[1],[@x[0]]		// load sigma
+	ld1	{@K[0]},[@x[0]],#16
+	ldp	@d[2],@d[3],[$key]		// load key
+	ldp	@d[4],@d[5],[$key,#16]
+	ld1	{@K[1],@K[2]},[$key]
+	ldp	@d[6],@d[7],[$ctr]		// load counter
+	ld1	{@K[3]},[$ctr]
+	ld1	{$ONE},[@x[0]]
+# ifdef	__AARCH64EB__
+	rev64	@K[0],@K[0]
+	ror	@d[2],@d[2],#32
+	ror	@d[3],@d[3],#32
+	ror	@d[4],@d[4],#32
+	ror	@d[5],@d[5],#32
+	ror	@d[6],@d[6],#32
+	ror	@d[7],@d[7],#32
+# endif
+	add	@K[3],@K[3],$ONE		// += 1
+	stp	@K[0],@K[1],[sp,#0]		// off-load key block, invariant part
+	add	@K[3],@K[3],$ONE		// not typo
+	str	@K[2],[sp,#32]
+	add	@K[4],@K[3],$ONE
+	add	@K[5],@K[4],$ONE
+	add	@K[6],@K[5],$ONE
+	shl	$ONE,$ONE,#2			// 1 -> 4
+
+	stp	d8,d9,[sp,#128+0]		// meet ABI requirements
+	stp	d10,d11,[sp,#128+16]
+	stp	d12,d13,[sp,#128+32]
+	stp	d14,d15,[sp,#128+48]
+
+	sub	$len,$len,#512			// not typo
+
+.Loop_outer_512_neon:
+	 mov	$A0,@K[0]
+	 mov	$A1,@K[0]
+	 mov	$A2,@K[0]
+	 mov	$A3,@K[0]
+	 mov	$A4,@K[0]
+	 mov	$A5,@K[0]
+	 mov	$B0,@K[1]
+	mov.32	@x[0],@d[0]			// unpack key block
+	 mov	$B1,@K[1]
+	lsr	@x[1],@d[0],#32
+	 mov	$B2,@K[1]
+	mov.32	@x[2],@d[1]
+	 mov	$B3,@K[1]
+	lsr	@x[3],@d[1],#32
+	 mov	$B4,@K[1]
+	mov.32	@x[4],@d[2]
+	 mov	$B5,@K[1]
+	lsr	@x[5],@d[2],#32
+	 mov	$D0,@K[3]
+	mov.32	@x[6],@d[3]
+	 mov	$D1,@K[4]
+	lsr	@x[7],@d[3],#32
+	 mov	$D2,@K[5]
+	mov.32	@x[8],@d[4]
+	 mov	$D3,@K[6]
+	lsr	@x[9],@d[4],#32
+	 mov	$C0,@K[2]
+	mov.32	@x[10],@d[5]
+	 mov	$C1,@K[2]
+	lsr	@x[11],@d[5],#32
+	 add	$D4,$D0,$ONE			// +4
+	mov.32	@x[12],@d[6]
+	 add	$D5,$D1,$ONE			// +4
+	lsr	@x[13],@d[6],#32
+	 mov	$C2,@K[2]
+	mov.32	@x[14],@d[7]
+	 mov	$C3,@K[2]
+	lsr	@x[15],@d[7],#32
+	 mov	$C4,@K[2]
+	 stp	@K[3],@K[4],[sp,#48]		// off-load key block, variable part
+	 mov	$C5,@K[2]
+	 str	@K[5],[sp,#80]
+
+	mov	$ctr,#5
+	subs	$len,$len,#512
+.Loop_upper_neon:
+	sub	$ctr,$ctr,#1
+___
+	my @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,0);
+	my @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,0);
+	my @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,0);
+	my @thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,0);
+	my @thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,0);
+	my @thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,0);
+	my @thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15));
+	my $diff = ($#thread0+1)*6 - $#thread67 - 1;
+	my $i = 0;
+
+	foreach (@thread0) {
+		eval;			eval(shift(@thread67));
+		eval(shift(@thread1));	eval(shift(@thread67));
+		eval(shift(@thread2));	eval(shift(@thread67));
+		eval(shift(@thread3));	eval(shift(@thread67));
+		eval(shift(@thread4));	eval(shift(@thread67));
+		eval(shift(@thread5));	eval(shift(@thread67));
+	}
+
+	@thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,1);
+	@thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,1);
+	@thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,1);
+	@thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,1);
+	@thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,1);
+	@thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,1);
+	@thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15));
+
+	foreach (@thread0) {
+		eval;			eval(shift(@thread67));
+		eval(shift(@thread1));	eval(shift(@thread67));
+		eval(shift(@thread2));	eval(shift(@thread67));
+		eval(shift(@thread3));	eval(shift(@thread67));
+		eval(shift(@thread4));	eval(shift(@thread67));
+		eval(shift(@thread5));	eval(shift(@thread67));
+	}
+$code.=<<___;
+	cbnz	$ctr,.Loop_upper_neon
+
+	add.32	@x[0],@x[0],@d[0]		// accumulate key block
+	add	@x[1],@x[1],@d[0],lsr#32
+	add.32	@x[2],@x[2],@d[1]
+	add	@x[3],@x[3],@d[1],lsr#32
+	add.32	@x[4],@x[4],@d[2]
+	add	@x[5],@x[5],@d[2],lsr#32
+	add.32	@x[6],@x[6],@d[3]
+	add	@x[7],@x[7],@d[3],lsr#32
+	add.32	@x[8],@x[8],@d[4]
+	add	@x[9],@x[9],@d[4],lsr#32
+	add.32	@x[10],@x[10],@d[5]
+	add	@x[11],@x[11],@d[5],lsr#32
+	add.32	@x[12],@x[12],@d[6]
+	add	@x[13],@x[13],@d[6],lsr#32
+	add.32	@x[14],@x[14],@d[7]
+	add	@x[15],@x[15],@d[7],lsr#32
+
+	add	@x[0],@x[0],@x[1],lsl#32	// pack
+	add	@x[2],@x[2],@x[3],lsl#32
+	ldp	@x[1],@x[3],[$inp,#0]		// load input
+	add	@x[4],@x[4],@x[5],lsl#32
+	add	@x[6],@x[6],@x[7],lsl#32
+	ldp	@x[5],@x[7],[$inp,#16]
+	add	@x[8],@x[8],@x[9],lsl#32
+	add	@x[10],@x[10],@x[11],lsl#32
+	ldp	@x[9],@x[11],[$inp,#32]
+	add	@x[12],@x[12],@x[13],lsl#32
+	add	@x[14],@x[14],@x[15],lsl#32
+	ldp	@x[13],@x[15],[$inp,#48]
+	add	$inp,$inp,#64
+# ifdef	__AARCH64EB__
+	rev	@x[0],@x[0]
+	rev	@x[2],@x[2]
+	rev	@x[4],@x[4]
+	rev	@x[6],@x[6]
+	rev	@x[8],@x[8]
+	rev	@x[10],@x[10]
+	rev	@x[12],@x[12]
+	rev	@x[14],@x[14]
+# endif
+	eor	@x[0],@x[0],@x[1]
+	eor	@x[2],@x[2],@x[3]
+	eor	@x[4],@x[4],@x[5]
+	eor	@x[6],@x[6],@x[7]
+	eor	@x[8],@x[8],@x[9]
+	eor	@x[10],@x[10],@x[11]
+	eor	@x[12],@x[12],@x[13]
+	eor	@x[14],@x[14],@x[15]
+
+	 stp	@x[0],@x[2],[$out,#0]		// store output
+	 add	@d[6],@d[6],#1			// increment counter
+	mov.32	@x[0],@d[0]			// unpack key block
+	lsr	@x[1],@d[0],#32
+	 stp	@x[4],@x[6],[$out,#16]
+	mov.32	@x[2],@d[1]
+	lsr	@x[3],@d[1],#32
+	 stp	@x[8],@x[10],[$out,#32]
+	mov.32	@x[4],@d[2]
+	lsr	@x[5],@d[2],#32
+	 stp	@x[12],@x[14],[$out,#48]
+	 add	$out,$out,#64
+	mov.32	@x[6],@d[3]
+	lsr	@x[7],@d[3],#32
+	mov.32	@x[8],@d[4]
+	lsr	@x[9],@d[4],#32
+	mov.32	@x[10],@d[5]
+	lsr	@x[11],@d[5],#32
+	mov.32	@x[12],@d[6]
+	lsr	@x[13],@d[6],#32
+	mov.32	@x[14],@d[7]
+	lsr	@x[15],@d[7],#32
+
+	mov	$ctr,#5
+.Loop_lower_neon:
+	sub	$ctr,$ctr,#1
+___
+	@thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,0);
+	@thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,0);
+	@thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,0);
+	@thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,0);
+	@thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,0);
+	@thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,0);
+	@thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15));
+
+	foreach (@thread0) {
+		eval;			eval(shift(@thread67));
+		eval(shift(@thread1));	eval(shift(@thread67));
+		eval(shift(@thread2));	eval(shift(@thread67));
+		eval(shift(@thread3));	eval(shift(@thread67));
+		eval(shift(@thread4));	eval(shift(@thread67));
+		eval(shift(@thread5));	eval(shift(@thread67));
+	}
+
+	@thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,1);
+	@thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,1);
+	@thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,1);
+	@thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,1);
+	@thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,1);
+	@thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,1);
+	@thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15));
+
+	foreach (@thread0) {
+		eval;			eval(shift(@thread67));
+		eval(shift(@thread1));	eval(shift(@thread67));
+		eval(shift(@thread2));	eval(shift(@thread67));
+		eval(shift(@thread3));	eval(shift(@thread67));
+		eval(shift(@thread4));	eval(shift(@thread67));
+		eval(shift(@thread5));	eval(shift(@thread67));
+	}
+$code.=<<___;
+	cbnz	$ctr,.Loop_lower_neon
+
+	add.32	@x[0],@x[0],@d[0]		// accumulate key block
+	 ldp	@K[0],@K[1],[sp,#0]
+	add	@x[1],@x[1],@d[0],lsr#32
+	 ldp	@K[2],@K[3],[sp,#32]
+	add.32	@x[2],@x[2],@d[1]
+	 ldp	@K[4],@K[5],[sp,#64]
+	add	@x[3],@x[3],@d[1],lsr#32
+	 add	$A0,$A0,@K[0]
+	add.32	@x[4],@x[4],@d[2]
+	 add	$A1,$A1,@K[0]
+	add	@x[5],@x[5],@d[2],lsr#32
+	 add	$A2,$A2,@K[0]
+	add.32	@x[6],@x[6],@d[3]
+	 add	$A3,$A3,@K[0]
+	add	@x[7],@x[7],@d[3],lsr#32
+	 add	$A4,$A4,@K[0]
+	add.32	@x[8],@x[8],@d[4]
+	 add	$A5,$A5,@K[0]
+	add	@x[9],@x[9],@d[4],lsr#32
+	 add	$C0,$C0,@K[2]
+	add.32	@x[10],@x[10],@d[5]
+	 add	$C1,$C1,@K[2]
+	add	@x[11],@x[11],@d[5],lsr#32
+	 add	$C2,$C2,@K[2]
+	add.32	@x[12],@x[12],@d[6]
+	 add	$C3,$C3,@K[2]
+	add	@x[13],@x[13],@d[6],lsr#32
+	 add	$C4,$C4,@K[2]
+	add.32	@x[14],@x[14],@d[7]
+	 add	$C5,$C5,@K[2]
+	add	@x[15],@x[15],@d[7],lsr#32
+	 add	$D4,$D4,$ONE			// +4
+	add	@x[0],@x[0],@x[1],lsl#32	// pack
+	 add	$D5,$D5,$ONE			// +4
+	add	@x[2],@x[2],@x[3],lsl#32
+	 add	$D0,$D0,@K[3]
+	ldp	@x[1],@x[3],[$inp,#0]		// load input
+	 add	$D1,$D1,@K[4]
+	add	@x[4],@x[4],@x[5],lsl#32
+	 add	$D2,$D2,@K[5]
+	add	@x[6],@x[6],@x[7],lsl#32
+	 add	$D3,$D3,@K[6]
+	ldp	@x[5],@x[7],[$inp,#16]
+	 add	$D4,$D4,@K[3]
+	add	@x[8],@x[8],@x[9],lsl#32
+	 add	$D5,$D5,@K[4]
+	add	@x[10],@x[10],@x[11],lsl#32
+	 add	$B0,$B0,@K[1]
+	ldp	@x[9],@x[11],[$inp,#32]
+	 add	$B1,$B1,@K[1]
+	add	@x[12],@x[12],@x[13],lsl#32
+	 add	$B2,$B2,@K[1]
+	add	@x[14],@x[14],@x[15],lsl#32
+	 add	$B3,$B3,@K[1]
+	ldp	@x[13],@x[15],[$inp,#48]
+	 add	$B4,$B4,@K[1]
+	add	$inp,$inp,#64
+	 add	$B5,$B5,@K[1]
+
+# ifdef	__AARCH64EB__
+	rev	@x[0],@x[0]
+	rev	@x[2],@x[2]
+	rev	@x[4],@x[4]
+	rev	@x[6],@x[6]
+	rev	@x[8],@x[8]
+	rev	@x[10],@x[10]
+	rev	@x[12],@x[12]
+	rev	@x[14],@x[14]
+# endif
+	ld1.8	{$T0-$T3},[$inp],#64
+	eor	@x[0],@x[0],@x[1]
+	eor	@x[2],@x[2],@x[3]
+	eor	@x[4],@x[4],@x[5]
+	eor	@x[6],@x[6],@x[7]
+	eor	@x[8],@x[8],@x[9]
+	 eor	$A0,$A0,$T0
+	eor	@x[10],@x[10],@x[11]
+	 eor	$B0,$B0,$T1
+	eor	@x[12],@x[12],@x[13]
+	 eor	$C0,$C0,$T2
+	eor	@x[14],@x[14],@x[15]
+	 eor	$D0,$D0,$T3
+	 ld1.8	{$T0-$T3},[$inp],#64
+
+	stp	@x[0],@x[2],[$out,#0]		// store output
+	 add	@d[6],@d[6],#7			// increment counter
+	stp	@x[4],@x[6],[$out,#16]
+	stp	@x[8],@x[10],[$out,#32]
+	stp	@x[12],@x[14],[$out,#48]
+	add	$out,$out,#64
+	st1.8	{$A0-$D0},[$out],#64
+
+	ld1.8	{$A0-$D0},[$inp],#64
+	eor	$A1,$A1,$T0
+	eor	$B1,$B1,$T1
+	eor	$C1,$C1,$T2
+	eor	$D1,$D1,$T3
+	st1.8	{$A1-$D1},[$out],#64
+
+	ld1.8	{$A1-$D1},[$inp],#64
+	eor	$A2,$A2,$A0
+	 ldp	@K[0],@K[1],[sp,#0]
+	eor	$B2,$B2,$B0
+	 ldp	@K[2],@K[3],[sp,#32]
+	eor	$C2,$C2,$C0
+	eor	$D2,$D2,$D0
+	st1.8	{$A2-$D2},[$out],#64
+
+	ld1.8	{$A2-$D2},[$inp],#64
+	eor	$A3,$A3,$A1
+	eor	$B3,$B3,$B1
+	eor	$C3,$C3,$C1
+	eor	$D3,$D3,$D1
+	st1.8	{$A3-$D3},[$out],#64
+
+	ld1.8	{$A3-$D3},[$inp],#64
+	eor	$A4,$A4,$A2
+	eor	$B4,$B4,$B2
+	eor	$C4,$C4,$C2
+	eor	$D4,$D4,$D2
+	st1.8	{$A4-$D4},[$out],#64
+
+	shl	$A0,$ONE,#1			// 4 -> 8
+	eor	$A5,$A5,$A3
+	eor	$B5,$B5,$B3
+	eor	$C5,$C5,$C3
+	eor	$D5,$D5,$D3
+	st1.8	{$A5-$D5},[$out],#64
+
+	add	@K[3],@K[3],$A0			// += 8
+	add	@K[4],@K[4],$A0
+	add	@K[5],@K[5],$A0
+	add	@K[6],@K[6],$A0
+
+	b.hs	.Loop_outer_512_neon
+
+	adds	$len,$len,#512
+	ushr	$A0,$ONE,#2			// 4 -> 1
+
+	ldp	d8,d9,[sp,#128+0]		// meet ABI requirements
+	ldp	d10,d11,[sp,#128+16]
+	ldp	d12,d13,[sp,#128+32]
+	ldp	d14,d15,[sp,#128+48]
+
+	stp	@K[0],$ONE,[sp,#0]		// wipe off-load area
+	stp	@K[0],$ONE,[sp,#32]
+	stp	@K[0],$ONE,[sp,#64]
+
+	b.eq	.Ldone_512_neon
+
+	cmp	$len,#192
+	sub	@K[3],@K[3],$A0			// -= 1
+	sub	@K[4],@K[4],$A0
+	sub	@K[5],@K[5],$A0
+	add	sp,sp,#128
+	b.hs	.Loop_outer_neon
+
+	eor	@K[1],@K[1],@K[1]
+	eor	@K[2],@K[2],@K[2]
+	eor	@K[3],@K[3],@K[3]
+	eor	@K[4],@K[4],@K[4]
+	eor	@K[5],@K[5],@K[5]
+	eor	@K[6],@K[6],@K[6]
+	b	.Loop_outer
+
+.Ldone_512_neon:
+	ldp	x19,x20,[x29,#16]
+	add	sp,sp,#128+64
+	ldp	x21,x22,[x29,#32]
+	ldp	x23,x24,[x29,#48]
+	ldp	x25,x26,[x29,#64]
+	ldp	x27,x28,[x29,#80]
+	ldp	x29,x30,[sp],#96
+	ret
+.size	ChaCha20_512_neon,.-ChaCha20_512_neon
+#endif
+#endif
+___
+}
+}}}
+
+open SELF,$0;
+while(<SELF>) {
+	next if (/^#!/);
+	last if (!s/^#/\/\// and !/^$/);
+	print;
+}
+close SELF;
+
+foreach (split("\n",$code)) {
+	s/\`([^\`]*)\`/eval $1/geo;
+
+	(s/\b([a-z]+)\.32\b/$1/ and (s/x([0-9]+)/w$1/g or 1))	or
+	(m/\b(eor|ext|mov)\b/ and (s/\.4s/\.16b/g or 1))	or
+	(s/\b((?:ld|st)1)\.8\b/$1/ and (s/\.4s/\.16b/g or 1))	or
+	(m/\b(ld|st)[rp]\b/ and (s/v([0-9]+)\.4s/q$1/g or 1))	or
+	(s/\brev32\.16\b/rev32/ and (s/\.4s/\.8h/g or 1));
+
+	print $_,"\n";
+}
+close STDOUT;	# flush
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/chacha20/chacha20-arm-glue.c b/net/wireguard/crypto/zinc/chacha20/chacha20-arm-glue.c
--- a/net/wireguard/crypto/zinc/chacha20/chacha20-arm-glue.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/chacha20/chacha20-arm-glue.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,98 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#include <asm/hwcap.h>
+#include <asm/neon.h>
+#if defined(CONFIG_ZINC_ARCH_ARM)
+#include <asm/system_info.h>
+#include <asm/cputype.h>
+#endif
+
+asmlinkage void chacha20_arm(u8 *out, const u8 *in, const size_t len,
+			     const u32 key[8], const u32 counter[4]);
+asmlinkage void hchacha20_arm(const u32 state[16], u32 out[8]);
+asmlinkage void chacha20_neon(u8 *out, const u8 *in, const size_t len,
+			      const u32 key[8], const u32 counter[4]);
+
+static bool chacha20_use_neon __ro_after_init;
+static bool *const chacha20_nobs[] __initconst = { &chacha20_use_neon };
+static void __init chacha20_fpu_init(void)
+{
+#if defined(CONFIG_ZINC_ARCH_ARM64)
+	chacha20_use_neon = cpu_have_named_feature(ASIMD);
+#elif defined(CONFIG_ZINC_ARCH_ARM)
+	switch (read_cpuid_part()) {
+	case ARM_CPU_PART_CORTEX_A7:
+	case ARM_CPU_PART_CORTEX_A5:
+		/* The Cortex-A7 and Cortex-A5 do not perform well with the NEON
+		 * implementation but do incredibly with the scalar one and use
+		 * less power.
+		 */
+		break;
+	default:
+		chacha20_use_neon = elf_hwcap & HWCAP_NEON;
+	}
+#endif
+}
+
+static inline bool chacha20_arch(struct chacha20_ctx *ctx, u8 *dst,
+				 const u8 *src, size_t len,
+				 simd_context_t *simd_context)
+{
+	/* SIMD disables preemption, so relax after processing each page. */
+	BUILD_BUG_ON(PAGE_SIZE < CHACHA20_BLOCK_SIZE ||
+		     PAGE_SIZE % CHACHA20_BLOCK_SIZE);
+
+	for (;;) {
+		if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && chacha20_use_neon &&
+		    len >= CHACHA20_BLOCK_SIZE * 3 && simd_use(simd_context)) {
+			const size_t bytes = min_t(size_t, len, PAGE_SIZE);
+
+			chacha20_neon(dst, src, bytes, ctx->key, ctx->counter);
+			ctx->counter[0] += (bytes + 63) / 64;
+			len -= bytes;
+			if (!len)
+				break;
+			dst += bytes;
+			src += bytes;
+			simd_relax(simd_context);
+		} else {
+			chacha20_arm(dst, src, len, ctx->key, ctx->counter);
+			ctx->counter[0] += (len + 63) / 64;
+			break;
+		}
+	}
+
+	return true;
+}
+
+static inline bool hchacha20_arch(u32 derived_key[CHACHA20_KEY_WORDS],
+				  const u8 nonce[HCHACHA20_NONCE_SIZE],
+				  const u8 key[HCHACHA20_KEY_SIZE],
+				  simd_context_t *simd_context)
+{
+	if (IS_ENABLED(CONFIG_ZINC_ARCH_ARM)) {
+		u32 x[] = { CHACHA20_CONSTANT_EXPA,
+			    CHACHA20_CONSTANT_ND_3,
+			    CHACHA20_CONSTANT_2_BY,
+			    CHACHA20_CONSTANT_TE_K,
+			    get_unaligned_le32(key + 0),
+			    get_unaligned_le32(key + 4),
+			    get_unaligned_le32(key + 8),
+			    get_unaligned_le32(key + 12),
+			    get_unaligned_le32(key + 16),
+			    get_unaligned_le32(key + 20),
+			    get_unaligned_le32(key + 24),
+			    get_unaligned_le32(key + 28),
+			    get_unaligned_le32(nonce + 0),
+			    get_unaligned_le32(nonce + 4),
+			    get_unaligned_le32(nonce + 8),
+			    get_unaligned_le32(nonce + 12)
+			  };
+		hchacha20_arm(x, derived_key);
+		return true;
+	}
+	return false;
+}
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/chacha20/chacha20-arm.pl b/net/wireguard/crypto/zinc/chacha20/chacha20-arm.pl
--- a/net/wireguard/crypto/zinc/chacha20/chacha20-arm.pl	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/chacha20/chacha20-arm.pl	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,1227 @@
+#!/usr/bin/env perl
+# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+#
+# This code is taken from the OpenSSL project but the author, Andy Polyakov,
+# has relicensed it under the licenses specified in the SPDX header above.
+# The original headers, including the original license headers, are
+# included below for completeness.
+#
+# ====================================================================
+# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
+# project. The module is, however, dual licensed under OpenSSL and
+# CRYPTOGAMS licenses depending on where you obtain it. For further
+# details see http://www.openssl.org/~appro/cryptogams/.
+# ====================================================================
+#
+# December 2014
+#
+# ChaCha20 for ARMv4.
+#
+# September 2018
+#
+# Improve scalar performance per Eric Biggers' suggestion to eliminate
+# separate rotates. This requires b[0..3] and d[0..3] to be maintained
+# pre-rotated, hence odd twists prior inner loop and when accumulating
+# key material. Since amount of instructions is reduced as result, even
+# NEON performance is improved somewhat, most notably by ~9% on low-end
+# Cortex-A5/A7. Full unroll was shown to provide even better scalar
+# performance on Cortex-A5/A7, naturally at the cost of manyfold size
+# increase. We let it be. Oversized code works in benchmarks, but is not
+# necessarily optimal in real life, when it's likely to be out-of-cache
+# upon entry and evict significant part of cache upon completion.
+#
+# Performance in cycles per byte out of large buffer.
+#
+#			IALU/gcc-4.4    1xNEON      3xNEON+1xIALU
+#
+# Cortex-A5		14.2(*)/+160%   21.8        12.9(**)
+# Cortex-A8		10.2(*)/+190%   13.9        6.10
+# Cortex-A9		10.8(*)/+150%   14.3        6.50
+# Cortex-A15		11.0/+40%       16.0        4.90
+# Snapdragon S4		13.9(***)/+90%  13.6        4.90
+#
+# (*)	most "favourable" result for aligned data on little-endian
+#	processor, result for misaligned data is 10-15% lower;
+# (**)	pure 4xNEON [with "vertical" layout] was shown to provide ~8%
+#	better performance on Cortex-A5/A7, but not on others;
+# (***)	it's 17% slower than original, trade-off is considered
+#	acceptable, because of improvement on others, specifically
+#	+36% on Cortex-A5/A7 and +20% on Cortex-A9;
+
+$flavour = shift;
+if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; }
+else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} }
+
+if ($flavour && $flavour ne "void") {
+    $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
+    ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
+    ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
+    die "can't locate arm-xlate.pl";
+
+    open STDOUT,"| \"$^X\" $xlate $flavour $output";
+} else {
+    open STDOUT,">$output";
+}
+
+sub AUTOLOAD()		# thunk [simplified] x86-style perlasm
+{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
+  my $arg = pop;
+    $arg = "#$arg" if ($arg*1 eq $arg);
+    $code .= "\t$opcode\t".join(',',@_,$arg)."\n";
+}
+
+my @x=map("r$_",(0..7,"x","x","x","x",12,"x",14,"x"));
+my @t=map("r$_",(8..11));
+
+sub ROUND {
+my ($a0,$b0,$c0,$d0)=@_;
+my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
+my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
+my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
+my $odd = $d0&1;
+my ($xc,$xc_) = (@t[0..1]);
+my ($xd,$xd_) = $odd ? (@t[2],@x[$d1]) : (@x[$d0],@t[2]);
+my @ret;
+
+	# Consider order in which variables are addressed by their
+	# index:
+	#
+	#       a   b   c   d
+	#
+	#       0   4   8  12 < even round
+	#       1   5   9  13
+	#       2   6  10  14
+	#       3   7  11  15
+	#       0   5  10  15 < odd round
+	#       1   6  11  12
+	#       2   7   8  13
+	#       3   4   9  14
+	#
+	# 'a', 'b' are permanently allocated in registers, @x[0..7],
+	# while 'c's and pair of 'd's are maintained in memory. If
+	# you observe 'c' column, you'll notice that pair of 'c's is
+	# invariant between rounds. This means that we have to reload
+	# them once per round, in the middle. This is why you'll see
+	# bunch of 'c' stores and loads in the middle, but none in
+	# the beginning or end. If you observe 'd' column, you'll
+	# notice that 15 and 13 are reused in next pair of rounds.
+	# This is why these two are chosen for offloading to memory,
+	# to make loads count more.
+							push @ret,(
+	"&add	(@x[$a0],@x[$a0],@x[$b0],'ror#13')",
+	 "&add	(@x[$a1],@x[$a1],@x[$b1],'ror#13')",
+	"&eor	($xd,@x[$a0],$xd,'ror#24')",
+	 "&eor	($xd_,@x[$a1],$xd_,'ror#24')",
+
+	"&add	($xc,$xc,$xd,'ror#16')",
+	 "&add	($xc_,$xc_,$xd_,'ror#16')",
+	"&eor	(@x[$b0],$xc, @x[$b0],'ror#13')",
+	 "&eor	(@x[$b1],$xc_,@x[$b1],'ror#13')",
+
+	"&add	(@x[$a0],@x[$a0],@x[$b0],'ror#20')",
+	 "&add	(@x[$a1],@x[$a1],@x[$b1],'ror#20')",
+	"&eor	($xd,@x[$a0],$xd,'ror#16')",
+	 "&eor	($xd_,@x[$a1],$xd_,'ror#16')"		);
+							push @ret,(
+	"&str	($xd,'[sp,#4*(16+$d0)]')"		) if ($odd);
+							push @ret,(
+	"&add	($xc,$xc,$xd,'ror#24')"			);
+							push @ret,(
+	"&ldr	($xd,'[sp,#4*(16+$d2)]')"		) if ($odd);
+							push @ret,(
+	 "&str	($xd_,'[sp,#4*(16+$d1)]')"		) if (!$odd);
+							push @ret,(
+	 "&add	($xc_,$xc_,$xd_,'ror#24')"		);
+							push @ret,(
+	 "&ldr	($xd_,'[sp,#4*(16+$d3)]')"		) if (!$odd);
+							push @ret,(
+	"&str	($xc,'[sp,#4*(16+$c0)]')",
+	"&eor	(@x[$b0],@x[$b0],$xc,'ror#12')",
+	 "&str	($xc_,'[sp,#4*(16+$c1)]')",
+	 "&eor	(@x[$b1],@x[$b1],$xc_,'ror#12')"	);
+
+	$xd=@x[$d2]					if (!$odd);
+	$xd_=@x[$d3]					if ($odd);
+							push @ret,(
+	"&ldr	($xc,'[sp,#4*(16+$c2)]')",
+	"&add	(@x[$a2],@x[$a2],@x[$b2],'ror#13')",
+	 "&ldr	($xc_,'[sp,#4*(16+$c3)]')",
+	 "&add	(@x[$a3],@x[$a3],@x[$b3],'ror#13')",
+	"&eor	($xd,@x[$a2],$xd,'ror#24')",
+	 "&eor	($xd_,@x[$a3],$xd_,'ror#24')",
+
+	"&add	($xc,$xc,$xd,'ror#16')",
+	 "&add	($xc_,$xc_,$xd_,'ror#16')",
+	"&eor	(@x[$b2],$xc, @x[$b2],'ror#13')",
+	 "&eor	(@x[$b3],$xc_,@x[$b3],'ror#13')",
+
+	"&add	(@x[$a2],@x[$a2],@x[$b2],'ror#20')",
+	 "&add	(@x[$a3],@x[$a3],@x[$b3],'ror#20')",
+	"&eor	($xd,@x[$a2],$xd,'ror#16')",
+	 "&eor	($xd_,@x[$a3],$xd_,'ror#16')",
+
+	"&add	($xc,$xc,$xd,'ror#24')",
+	 "&add	($xc_,$xc_,$xd_,'ror#24')",
+	"&eor	(@x[$b2],@x[$b2],$xc,'ror#12')",
+	 "&eor	(@x[$b3],@x[$b3],$xc_,'ror#12')"	);
+
+	@ret;
+}
+
+$code.=<<___;
+#ifndef __KERNEL__
+# include "arm_arch.h"
+#else
+# define __ARM_ARCH__ __LINUX_ARM_ARCH__
+# define __ARM_MAX_ARCH__ __LINUX_ARM_ARCH__
+# define ChaCha20_ctr32 chacha20_arm_cryptogams
+# define ChaCha20_neon  chacha20_neon
+#endif
+
+.text
+#if defined(__thumb2__) || defined(__clang__)
+.syntax	unified
+# define ldrhsb	ldrbhs
+#endif
+#if defined(__thumb2__)
+.thumb
+#else
+.code	32
+#endif
+
+.align	5
+.Lsigma:
+.long	0x61707865,0x3320646e,0x79622d32,0x6b206574	@ endian-neutral
+.Lone:
+.long	1,0,0,0
+.Lrot8:
+.long	0x02010003,0x06050407
+#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
+.LOPENSSL_armcap:
+.word   OPENSSL_armcap_P-.LChaCha20_ctr32
+#else
+.word	-1
+#endif
+
+.globl	ChaCha20_ctr32
+.type	ChaCha20_ctr32,%function
+.align	5
+ChaCha20_ctr32:
+.LChaCha20_ctr32:
+	ldr	r12,[sp,#0]		@ pull pointer to counter and nonce
+	stmdb	sp!,{r0-r2,r4-r11,lr}
+#if __ARM_ARCH__<7 && !defined(__thumb2__)
+	sub	r14,pc,#16		@ ChaCha20_ctr32
+#else
+	adr	r14,.LChaCha20_ctr32
+#endif
+	cmp	r2,#0			@ len==0?
+#ifdef	__thumb2__
+	itt	eq
+#endif
+	addeq	sp,sp,#4*3
+	beq	.Lno_data
+#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
+	cmp	r2,#192			@ test len
+	bls	.Lshort
+	ldr	r4,[r14,#-24]
+	ldr	r4,[r14,r4]
+# ifdef	__APPLE__
+	ldr	r4,[r4]
+# endif
+	tst	r4,#ARMV7_NEON
+	bne	.LChaCha20_neon
+.Lshort:
+#endif
+	ldmia	r12,{r4-r7}		@ load counter and nonce
+	sub	sp,sp,#4*(16)		@ off-load area
+	sub	r14,r14,#64		@ .Lsigma
+	stmdb	sp!,{r4-r7}		@ copy counter and nonce
+	ldmia	r3,{r4-r11}		@ load key
+	ldmia	r14,{r0-r3}		@ load sigma
+	stmdb	sp!,{r4-r11}		@ copy key
+	stmdb	sp!,{r0-r3}		@ copy sigma
+	str	r10,[sp,#4*(16+10)]	@ off-load "@x[10]"
+	str	r11,[sp,#4*(16+11)]	@ off-load "@x[11]"
+	b	.Loop_outer_enter
+
+.align	4
+.Loop_outer:
+	ldmia	sp,{r0-r9}		@ load key material
+	str	@t[3],[sp,#4*(32+2)]	@ save len
+	str	r12,  [sp,#4*(32+1)]	@ save inp
+	str	r14,  [sp,#4*(32+0)]	@ save out
+.Loop_outer_enter:
+	ldr	@t[3], [sp,#4*(15)]
+	 mov	@x[4],@x[4],ror#19	@ twist b[0..3]
+	ldr	@x[12],[sp,#4*(12)]	@ modulo-scheduled load
+	 mov	@x[5],@x[5],ror#19
+	ldr	@t[2], [sp,#4*(13)]
+	 mov	@x[6],@x[6],ror#19
+	ldr	@x[14],[sp,#4*(14)]
+	 mov	@x[7],@x[7],ror#19
+	mov	@t[3],@t[3],ror#8	@ twist d[0..3]
+	mov	@x[12],@x[12],ror#8
+	mov	@t[2],@t[2],ror#8
+	mov	@x[14],@x[14],ror#8
+	str	@t[3], [sp,#4*(16+15)]
+	mov	@t[3],#10
+	b	.Loop
+
+.align	4
+.Loop:
+	subs	@t[3],@t[3],#1
+___
+	foreach (&ROUND(0, 4, 8,12)) { eval; }
+	foreach (&ROUND(0, 5,10,15)) { eval; }
+$code.=<<___;
+	bne	.Loop
+
+	ldr	@t[3],[sp,#4*(32+2)]	@ load len
+
+	str	@t[0], [sp,#4*(16+8)]	@ modulo-scheduled store
+	str	@t[1], [sp,#4*(16+9)]
+	str	@x[12],[sp,#4*(16+12)]
+	str	@t[2], [sp,#4*(16+13)]
+	str	@x[14],[sp,#4*(16+14)]
+
+	@ at this point we have first half of 512-bit result in
+	@ @x[0-7] and second half at sp+4*(16+8)
+
+	cmp	@t[3],#64		@ done yet?
+#ifdef	__thumb2__
+	itete	lo
+#endif
+	addlo	r12,sp,#4*(0)		@ shortcut or ...
+	ldrhs	r12,[sp,#4*(32+1)]	@ ... load inp
+	addlo	r14,sp,#4*(0)		@ shortcut or ...
+	ldrhs	r14,[sp,#4*(32+0)]	@ ... load out
+
+	ldr	@t[0],[sp,#4*(0)]	@ load key material
+	ldr	@t[1],[sp,#4*(1)]
+
+#if __ARM_ARCH__>=6 || !defined(__ARMEB__)
+# if __ARM_ARCH__<7
+	orr	@t[2],r12,r14
+	tst	@t[2],#3		@ are input and output aligned?
+	ldr	@t[2],[sp,#4*(2)]
+	bne	.Lunaligned
+	cmp	@t[3],#64		@ restore flags
+# else
+	ldr	@t[2],[sp,#4*(2)]
+# endif
+	ldr	@t[3],[sp,#4*(3)]
+
+	add	@x[0],@x[0],@t[0]	@ accumulate key material
+	add	@x[1],@x[1],@t[1]
+# ifdef	__thumb2__
+	itt	hs
+# endif
+	ldrhs	@t[0],[r12],#16		@ load input
+	ldrhs	@t[1],[r12,#-12]
+
+	add	@x[2],@x[2],@t[2]
+	add	@x[3],@x[3],@t[3]
+# ifdef	__thumb2__
+	itt	hs
+# endif
+	ldrhs	@t[2],[r12,#-8]
+	ldrhs	@t[3],[r12,#-4]
+# if __ARM_ARCH__>=6 && defined(__ARMEB__)
+	rev	@x[0],@x[0]
+	rev	@x[1],@x[1]
+	rev	@x[2],@x[2]
+	rev	@x[3],@x[3]
+# endif
+# ifdef	__thumb2__
+	itt	hs
+# endif
+	eorhs	@x[0],@x[0],@t[0]	@ xor with input
+	eorhs	@x[1],@x[1],@t[1]
+	 add	@t[0],sp,#4*(4)
+	str	@x[0],[r14],#16		@ store output
+# ifdef	__thumb2__
+	itt	hs
+# endif
+	eorhs	@x[2],@x[2],@t[2]
+	eorhs	@x[3],@x[3],@t[3]
+	 ldmia	@t[0],{@t[0]-@t[3]}	@ load key material
+	str	@x[1],[r14,#-12]
+	str	@x[2],[r14,#-8]
+	str	@x[3],[r14,#-4]
+
+	add	@x[4],@t[0],@x[4],ror#13 @ accumulate key material
+	add	@x[5],@t[1],@x[5],ror#13
+# ifdef	__thumb2__
+	itt	hs
+# endif
+	ldrhs	@t[0],[r12],#16		@ load input
+	ldrhs	@t[1],[r12,#-12]
+	add	@x[6],@t[2],@x[6],ror#13
+	add	@x[7],@t[3],@x[7],ror#13
+# ifdef	__thumb2__
+	itt	hs
+# endif
+	ldrhs	@t[2],[r12,#-8]
+	ldrhs	@t[3],[r12,#-4]
+# if __ARM_ARCH__>=6 && defined(__ARMEB__)
+	rev	@x[4],@x[4]
+	rev	@x[5],@x[5]
+	rev	@x[6],@x[6]
+	rev	@x[7],@x[7]
+# endif
+# ifdef	__thumb2__
+	itt	hs
+# endif
+	eorhs	@x[4],@x[4],@t[0]
+	eorhs	@x[5],@x[5],@t[1]
+	 add	@t[0],sp,#4*(8)
+	str	@x[4],[r14],#16		@ store output
+# ifdef	__thumb2__
+	itt	hs
+# endif
+	eorhs	@x[6],@x[6],@t[2]
+	eorhs	@x[7],@x[7],@t[3]
+	str	@x[5],[r14,#-12]
+	 ldmia	@t[0],{@t[0]-@t[3]}	@ load key material
+	str	@x[6],[r14,#-8]
+	 add	@x[0],sp,#4*(16+8)
+	str	@x[7],[r14,#-4]
+
+	ldmia	@x[0],{@x[0]-@x[7]}	@ load second half
+
+	add	@x[0],@x[0],@t[0]	@ accumulate key material
+	add	@x[1],@x[1],@t[1]
+# ifdef	__thumb2__
+	itt	hs
+# endif
+	ldrhs	@t[0],[r12],#16		@ load input
+	ldrhs	@t[1],[r12,#-12]
+# ifdef	__thumb2__
+	itt	hi
+# endif
+	 strhi	@t[2],[sp,#4*(16+10)]	@ copy "@x[10]" while at it
+	 strhi	@t[3],[sp,#4*(16+11)]	@ copy "@x[11]" while at it
+	add	@x[2],@x[2],@t[2]
+	add	@x[3],@x[3],@t[3]
+# ifdef	__thumb2__
+	itt	hs
+# endif
+	ldrhs	@t[2],[r12,#-8]
+	ldrhs	@t[3],[r12,#-4]
+# if __ARM_ARCH__>=6 && defined(__ARMEB__)
+	rev	@x[0],@x[0]
+	rev	@x[1],@x[1]
+	rev	@x[2],@x[2]
+	rev	@x[3],@x[3]
+# endif
+# ifdef	__thumb2__
+	itt	hs
+# endif
+	eorhs	@x[0],@x[0],@t[0]
+	eorhs	@x[1],@x[1],@t[1]
+	 add	@t[0],sp,#4*(12)
+	str	@x[0],[r14],#16		@ store output
+# ifdef	__thumb2__
+	itt	hs
+# endif
+	eorhs	@x[2],@x[2],@t[2]
+	eorhs	@x[3],@x[3],@t[3]
+	str	@x[1],[r14,#-12]
+	 ldmia	@t[0],{@t[0]-@t[3]}	@ load key material
+	str	@x[2],[r14,#-8]
+	str	@x[3],[r14,#-4]
+
+	add	@x[4],@t[0],@x[4],ror#24 @ accumulate key material
+	add	@x[5],@t[1],@x[5],ror#24
+# ifdef	__thumb2__
+	itt	hi
+# endif
+	 addhi	@t[0],@t[0],#1		@ next counter value
+	 strhi	@t[0],[sp,#4*(12)]	@ save next counter value
+# ifdef	__thumb2__
+	itt	hs
+# endif
+	ldrhs	@t[0],[r12],#16		@ load input
+	ldrhs	@t[1],[r12,#-12]
+	add	@x[6],@t[2],@x[6],ror#24
+	add	@x[7],@t[3],@x[7],ror#24
+# ifdef	__thumb2__
+	itt	hs
+# endif
+	ldrhs	@t[2],[r12,#-8]
+	ldrhs	@t[3],[r12,#-4]
+# if __ARM_ARCH__>=6 && defined(__ARMEB__)
+	rev	@x[4],@x[4]
+	rev	@x[5],@x[5]
+	rev	@x[6],@x[6]
+	rev	@x[7],@x[7]
+# endif
+# ifdef	__thumb2__
+	itt	hs
+# endif
+	eorhs	@x[4],@x[4],@t[0]
+	eorhs	@x[5],@x[5],@t[1]
+# ifdef	__thumb2__
+	 it	ne
+# endif
+	 ldrne	@t[0],[sp,#4*(32+2)]	@ re-load len
+# ifdef	__thumb2__
+	itt	hs
+# endif
+	eorhs	@x[6],@x[6],@t[2]
+	eorhs	@x[7],@x[7],@t[3]
+	str	@x[4],[r14],#16		@ store output
+	str	@x[5],[r14,#-12]
+# ifdef	__thumb2__
+	it	hs
+# endif
+	 subhs	@t[3],@t[0],#64		@ len-=64
+	str	@x[6],[r14,#-8]
+	str	@x[7],[r14,#-4]
+	bhi	.Loop_outer
+
+	beq	.Ldone
+# if __ARM_ARCH__<7
+	b	.Ltail
+
+.align	4
+.Lunaligned:				@ unaligned endian-neutral path
+	cmp	@t[3],#64		@ restore flags
+# endif
+#endif
+#if __ARM_ARCH__<7
+	ldr	@t[3],[sp,#4*(3)]
+___
+for ($i=0;$i<16;$i+=4) {
+my $j=$i&0x7;
+my $twist="";
+if ($i==4)     { $twist = ",ror#13"; }
+elsif ($i==12) { $twist = ",ror#24"; }
+
+$code.=<<___	if ($i==4);
+	add	@x[0],sp,#4*(16+8)
+___
+$code.=<<___	if ($i==8);
+	ldmia	@x[0],{@x[0]-@x[7]}		@ load second half
+# ifdef	__thumb2__
+	itt	hi
+# endif
+	strhi	@t[2],[sp,#4*(16+10)]		@ copy "@x[10]"
+	strhi	@t[3],[sp,#4*(16+11)]		@ copy "@x[11]"
+___
+$code.=<<___;
+	add	@x[$j+0],@t[0],@x[$j+0]$twist	@ accumulate key material
+___
+$code.=<<___	if ($i==12);
+# ifdef	__thumb2__
+	itt	hi
+# endif
+	addhi	@t[0],@t[0],#1			@ next counter value
+	strhi	@t[0],[sp,#4*(12)]		@ save next counter value
+___
+$code.=<<___;
+	add	@x[$j+1],@t[1],@x[$j+1]$twist
+	add	@x[$j+2],@t[2],@x[$j+2]$twist
+# ifdef	__thumb2__
+	itete	lo
+# endif
+	eorlo	@t[0],@t[0],@t[0]		@ zero or ...
+	ldrhsb	@t[0],[r12],#16			@ ... load input
+	eorlo	@t[1],@t[1],@t[1]
+	ldrhsb	@t[1],[r12,#-12]
+
+	add	@x[$j+3],@t[3],@x[$j+3]$twist
+# ifdef	__thumb2__
+	itete	lo
+# endif
+	eorlo	@t[2],@t[2],@t[2]
+	ldrhsb	@t[2],[r12,#-8]
+	eorlo	@t[3],@t[3],@t[3]
+	ldrhsb	@t[3],[r12,#-4]
+
+	eor	@x[$j+0],@t[0],@x[$j+0]		@ xor with input (or zero)
+	eor	@x[$j+1],@t[1],@x[$j+1]
+# ifdef	__thumb2__
+	itt	hs
+# endif
+	ldrhsb	@t[0],[r12,#-15]		@ load more input
+	ldrhsb	@t[1],[r12,#-11]
+	eor	@x[$j+2],@t[2],@x[$j+2]
+	 strb	@x[$j+0],[r14],#16		@ store output
+	eor	@x[$j+3],@t[3],@x[$j+3]
+# ifdef	__thumb2__
+	itt	hs
+# endif
+	ldrhsb	@t[2],[r12,#-7]
+	ldrhsb	@t[3],[r12,#-3]
+	 strb	@x[$j+1],[r14,#-12]
+	eor	@x[$j+0],@t[0],@x[$j+0],lsr#8
+	 strb	@x[$j+2],[r14,#-8]
+	eor	@x[$j+1],@t[1],@x[$j+1],lsr#8
+# ifdef	__thumb2__
+	itt	hs
+# endif
+	ldrhsb	@t[0],[r12,#-14]		@ load more input
+	ldrhsb	@t[1],[r12,#-10]
+	 strb	@x[$j+3],[r14,#-4]
+	eor	@x[$j+2],@t[2],@x[$j+2],lsr#8
+	 strb	@x[$j+0],[r14,#-15]
+	eor	@x[$j+3],@t[3],@x[$j+3],lsr#8
+# ifdef	__thumb2__
+	itt	hs
+# endif
+	ldrhsb	@t[2],[r12,#-6]
+	ldrhsb	@t[3],[r12,#-2]
+	 strb	@x[$j+1],[r14,#-11]
+	eor	@x[$j+0],@t[0],@x[$j+0],lsr#8
+	 strb	@x[$j+2],[r14,#-7]
+	eor	@x[$j+1],@t[1],@x[$j+1],lsr#8
+# ifdef	__thumb2__
+	itt	hs
+# endif
+	ldrhsb	@t[0],[r12,#-13]		@ load more input
+	ldrhsb	@t[1],[r12,#-9]
+	 strb	@x[$j+3],[r14,#-3]
+	eor	@x[$j+2],@t[2],@x[$j+2],lsr#8
+	 strb	@x[$j+0],[r14,#-14]
+	eor	@x[$j+3],@t[3],@x[$j+3],lsr#8
+# ifdef	__thumb2__
+	itt	hs
+# endif
+	ldrhsb	@t[2],[r12,#-5]
+	ldrhsb	@t[3],[r12,#-1]
+	 strb	@x[$j+1],[r14,#-10]
+	 strb	@x[$j+2],[r14,#-6]
+	eor	@x[$j+0],@t[0],@x[$j+0],lsr#8
+	 strb	@x[$j+3],[r14,#-2]
+	eor	@x[$j+1],@t[1],@x[$j+1],lsr#8
+	 strb	@x[$j+0],[r14,#-13]
+	eor	@x[$j+2],@t[2],@x[$j+2],lsr#8
+	 strb	@x[$j+1],[r14,#-9]
+	eor	@x[$j+3],@t[3],@x[$j+3],lsr#8
+	 strb	@x[$j+2],[r14,#-5]
+	 strb	@x[$j+3],[r14,#-1]
+___
+$code.=<<___	if ($i<12);
+	add	@t[0],sp,#4*(4+$i)
+	ldmia	@t[0],{@t[0]-@t[3]}		@ load key material
+___
+}
+$code.=<<___;
+# ifdef	__thumb2__
+	it	ne
+# endif
+	ldrne	@t[0],[sp,#4*(32+2)]		@ re-load len
+# ifdef	__thumb2__
+	it	hs
+# endif
+	subhs	@t[3],@t[0],#64			@ len-=64
+	bhi	.Loop_outer
+
+	beq	.Ldone
+#endif
+
+.Ltail:
+	ldr	r12,[sp,#4*(32+1)]	@ load inp
+	add	@t[1],sp,#4*(0)
+	ldr	r14,[sp,#4*(32+0)]	@ load out
+
+.Loop_tail:
+	ldrb	@t[2],[@t[1]],#1	@ read buffer on stack
+	ldrb	@t[3],[r12],#1		@ read input
+	subs	@t[0],@t[0],#1
+	eor	@t[3],@t[3],@t[2]
+	strb	@t[3],[r14],#1		@ store output
+	bne	.Loop_tail
+
+.Ldone:
+	add	sp,sp,#4*(32+3)
+.Lno_data:
+#if __ARM_ARCH__>=5
+	ldmia	sp!,{r4-r11,pc}
+#else
+	ldmia	sp!,{r4-r12,lr}
+	tst	lr,#1
+	moveq	pc,lr			@ be binary compatible with V4, yet
+	.long	0xe12fff1e		@ interoperable with Thumb ISA:-)
+#endif
+.size	ChaCha20_ctr32,.-ChaCha20_ctr32
+___
+
+{{{
+my ($a0,$b0,$c0,$d0,$a1,$b1,$c1,$d1,$a2,$b2,$c2,$d2,$t0,$t1,$t2,$t3) =
+    map("q$_",(0..15));
+
+# This can replace vshr-by-24+vsli-by-8. It gives ~3% improvement on
+# Cortex-A5/A7, but hurts Cortex-A9 by 5% and Snapdragon S4 by 14%!
+sub vperm()
+{ my ($dst,$src,$tbl) = @_;
+    $code .= "	vtbl.8	$dst#lo,{$src#lo},$tbl#lo\n";
+    $code .= "	vtbl.8	$dst#hi,{$src#hi},$tbl#lo\n";
+}
+
+sub NEONROUND {
+my $odd = pop;
+my ($a,$b,$c,$d,$t)=@_;
+
+	(
+	"&vadd_i32	($a,$a,$b)",
+	"&veor		($d,$d,$a)",
+	"&vrev32_16	($d,$d)",	# vrot ($d,16)
+
+	"&vadd_i32	($c,$c,$d)",
+	"&veor		($t,$b,$c)",
+	"&vshr_u32	($b,$t,20)",
+	"&vsli_32	($b,$t,12)",
+
+	"&vadd_i32	($a,$a,$b)",
+	"&veor		($t,$d,$a)",
+	"&vshr_u32	($d,$t,24)",
+	"&vsli_32	($d,$t,8)",
+	#"&vperm	($d,$t,$t3)",
+
+	"&vadd_i32	($c,$c,$d)",
+	"&veor		($t,$b,$c)",
+	"&vshr_u32	($b,$t,25)",
+	"&vsli_32	($b,$t,7)",
+
+	"&vext_8	($a,$a,$a,$odd?4:12)",
+	"&vext_8	($d,$d,$d,8)",
+	"&vext_8	($c,$c,$c,$odd?12:4)"
+	);
+}
+
+$code.=<<___;
+#if (defined(__KERNEL__) && defined(CONFIG_KERNEL_MODE_NEON)) || (!defined(__KERNEL__) && __ARM_MAX_ARCH__>=7)
+.arch	armv7-a
+.fpu	neon
+
+# ifdef __KERNEL__
+.globl	ChaCha20_neon
+@ For optimal performance it's appropriate for caller to enforce
+@ minimum input length, 193 bytes is suggested.
+# endif
+.type	ChaCha20_neon,%function
+.align	5
+ChaCha20_neon:
+	ldr		r12,[sp,#0]		@ pull pointer to counter and nonce
+	stmdb		sp!,{r0-r2,r4-r11,lr}
+.LChaCha20_neon:
+	adr		r14,.Lsigma
+	vstmdb		sp!,{d8-d15}		@ ABI spec says so
+	stmdb		sp!,{r0-r3}
+
+	vld1.32		{$b0-$c0},[r3]		@ load key
+	ldmia		r3,{r4-r11}		@ load key
+
+	sub		sp,sp,#4*(16+16)
+	vld1.32		{$d0},[r12]		@ load counter and nonce
+	add		r12,sp,#4*8
+	ldmia		r14,{r0-r3}		@ load sigma
+	vld1.32		{$a0},[r14]!		@ load sigma
+	vld1.32		{$t0},[r14]!		@ one
+	@ vld1.32	{$t3#lo},[r14]		@ rot8
+	vst1.32		{$c0-$d0},[r12]		@ copy 1/2key|counter|nonce
+	vst1.32		{$a0-$b0},[sp]		@ copy sigma|1/2key
+
+	str		r10,[sp,#4*(16+10)]	@ off-load "@x[10]"
+	str		r11,[sp,#4*(16+11)]	@ off-load "@x[11]"
+	vshl.i32	$t1#lo,$t0#lo,#1	@ two
+	vstr		$t0#lo,[sp,#4*(16+0)]
+	vshl.i32	$t2#lo,$t0#lo,#2	@ four
+	vstr		$t1#lo,[sp,#4*(16+2)]
+	vmov		$a1,$a0
+	vstr		$t2#lo,[sp,#4*(16+4)]
+	vmov		$a2,$a0
+	@ vstr		$t3#lo,[sp,#4*(16+6)]
+	vmov		$b1,$b0
+	vmov		$b2,$b0
+	b		.Loop_neon_enter
+
+.align	4
+.Loop_neon_outer:
+	ldmia		sp,{r0-r9}		@ load key material
+	cmp		@t[3],#64*2		@ if len<=64*2
+	bls		.Lbreak_neon		@ switch to integer-only
+	@ vldr		$t3#lo,[sp,#4*(16+6)]	@ rot8
+	vmov		$a1,$a0
+	str		@t[3],[sp,#4*(32+2)]	@ save len
+	vmov		$a2,$a0
+	str		r12,  [sp,#4*(32+1)]	@ save inp
+	vmov		$b1,$b0
+	str		r14,  [sp,#4*(32+0)]	@ save out
+	vmov		$b2,$b0
+.Loop_neon_enter:
+	ldr		@t[3], [sp,#4*(15)]
+	 mov		@x[4],@x[4],ror#19	@ twist b[0..3]
+	vadd.i32	$d1,$d0,$t0		@ counter+1
+	ldr		@x[12],[sp,#4*(12)]	@ modulo-scheduled load
+	 mov		@x[5],@x[5],ror#19
+	vmov		$c1,$c0
+	ldr		@t[2], [sp,#4*(13)]
+	 mov		@x[6],@x[6],ror#19
+	vmov		$c2,$c0
+	ldr		@x[14],[sp,#4*(14)]
+	 mov		@x[7],@x[7],ror#19
+	vadd.i32	$d2,$d1,$t0		@ counter+2
+	add		@x[12],@x[12],#3	@ counter+3
+	mov		@t[3],@t[3],ror#8	@ twist d[0..3]
+	mov		@x[12],@x[12],ror#8
+	mov		@t[2],@t[2],ror#8
+	mov		@x[14],@x[14],ror#8
+	str		@t[3], [sp,#4*(16+15)]
+	mov		@t[3],#10
+	b		.Loop_neon
+
+.align	4
+.Loop_neon:
+	subs		@t[3],@t[3],#1
+___
+	my @thread0=&NEONROUND($a0,$b0,$c0,$d0,$t0,0);
+	my @thread1=&NEONROUND($a1,$b1,$c1,$d1,$t1,0);
+	my @thread2=&NEONROUND($a2,$b2,$c2,$d2,$t2,0);
+	my @thread3=&ROUND(0,4,8,12);
+
+	foreach (@thread0) {
+		eval;			eval(shift(@thread3));
+		eval(shift(@thread1));	eval(shift(@thread3));
+		eval(shift(@thread2));	eval(shift(@thread3));
+	}
+
+	@thread0=&NEONROUND($a0,$b0,$c0,$d0,$t0,1);
+	@thread1=&NEONROUND($a1,$b1,$c1,$d1,$t1,1);
+	@thread2=&NEONROUND($a2,$b2,$c2,$d2,$t2,1);
+	@thread3=&ROUND(0,5,10,15);
+
+	foreach (@thread0) {
+		eval;			eval(shift(@thread3));
+		eval(shift(@thread1));	eval(shift(@thread3));
+		eval(shift(@thread2));	eval(shift(@thread3));
+	}
+$code.=<<___;
+	bne		.Loop_neon
+
+	add		@t[3],sp,#32
+	vld1.32		{$t0-$t1},[sp]		@ load key material
+	vld1.32		{$t2-$t3},[@t[3]]
+
+	ldr		@t[3],[sp,#4*(32+2)]	@ load len
+
+	str		@t[0], [sp,#4*(16+8)]	@ modulo-scheduled store
+	str		@t[1], [sp,#4*(16+9)]
+	str		@x[12],[sp,#4*(16+12)]
+	str		@t[2], [sp,#4*(16+13)]
+	str		@x[14],[sp,#4*(16+14)]
+
+	@ at this point we have first half of 512-bit result in
+	@ @x[0-7] and second half at sp+4*(16+8)
+
+	ldr		r12,[sp,#4*(32+1)]	@ load inp
+	ldr		r14,[sp,#4*(32+0)]	@ load out
+
+	vadd.i32	$a0,$a0,$t0		@ accumulate key material
+	vadd.i32	$a1,$a1,$t0
+	vadd.i32	$a2,$a2,$t0
+	vldr		$t0#lo,[sp,#4*(16+0)]	@ one
+
+	vadd.i32	$b0,$b0,$t1
+	vadd.i32	$b1,$b1,$t1
+	vadd.i32	$b2,$b2,$t1
+	vldr		$t1#lo,[sp,#4*(16+2)]	@ two
+
+	vadd.i32	$c0,$c0,$t2
+	vadd.i32	$c1,$c1,$t2
+	vadd.i32	$c2,$c2,$t2
+	vadd.i32	$d1#lo,$d1#lo,$t0#lo	@ counter+1
+	vadd.i32	$d2#lo,$d2#lo,$t1#lo	@ counter+2
+
+	vadd.i32	$d0,$d0,$t3
+	vadd.i32	$d1,$d1,$t3
+	vadd.i32	$d2,$d2,$t3
+
+	cmp		@t[3],#64*4
+	blo		.Ltail_neon
+
+	vld1.8		{$t0-$t1},[r12]!	@ load input
+	 mov		@t[3],sp
+	vld1.8		{$t2-$t3},[r12]!
+	veor		$a0,$a0,$t0		@ xor with input
+	veor		$b0,$b0,$t1
+	vld1.8		{$t0-$t1},[r12]!
+	veor		$c0,$c0,$t2
+	veor		$d0,$d0,$t3
+	vld1.8		{$t2-$t3},[r12]!
+
+	veor		$a1,$a1,$t0
+	 vst1.8		{$a0-$b0},[r14]!	@ store output
+	veor		$b1,$b1,$t1
+	vld1.8		{$t0-$t1},[r12]!
+	veor		$c1,$c1,$t2
+	 vst1.8		{$c0-$d0},[r14]!
+	veor		$d1,$d1,$t3
+	vld1.8		{$t2-$t3},[r12]!
+
+	veor		$a2,$a2,$t0
+	 vld1.32	{$a0-$b0},[@t[3]]!	@ load for next iteration
+	 veor		$t0#hi,$t0#hi,$t0#hi
+	 vldr		$t0#lo,[sp,#4*(16+4)]	@ four
+	veor		$b2,$b2,$t1
+	 vld1.32	{$c0-$d0},[@t[3]]
+	veor		$c2,$c2,$t2
+	 vst1.8		{$a1-$b1},[r14]!
+	veor		$d2,$d2,$t3
+	 vst1.8		{$c1-$d1},[r14]!
+
+	vadd.i32	$d0#lo,$d0#lo,$t0#lo	@ next counter value
+	vldr		$t0#lo,[sp,#4*(16+0)]	@ one
+
+	ldmia		sp,{@t[0]-@t[3]}	@ load key material
+	add		@x[0],@x[0],@t[0]	@ accumulate key material
+	ldr		@t[0],[r12],#16		@ load input
+	 vst1.8		{$a2-$b2},[r14]!
+	add		@x[1],@x[1],@t[1]
+	ldr		@t[1],[r12,#-12]
+	 vst1.8		{$c2-$d2},[r14]!
+	add		@x[2],@x[2],@t[2]
+	ldr		@t[2],[r12,#-8]
+	add		@x[3],@x[3],@t[3]
+	ldr		@t[3],[r12,#-4]
+# ifdef	__ARMEB__
+	rev		@x[0],@x[0]
+	rev		@x[1],@x[1]
+	rev		@x[2],@x[2]
+	rev		@x[3],@x[3]
+# endif
+	eor		@x[0],@x[0],@t[0]	@ xor with input
+	 add		@t[0],sp,#4*(4)
+	eor		@x[1],@x[1],@t[1]
+	str		@x[0],[r14],#16		@ store output
+	eor		@x[2],@x[2],@t[2]
+	str		@x[1],[r14,#-12]
+	eor		@x[3],@x[3],@t[3]
+	 ldmia		@t[0],{@t[0]-@t[3]}	@ load key material
+	str		@x[2],[r14,#-8]
+	str		@x[3],[r14,#-4]
+
+	add		@x[4],@t[0],@x[4],ror#13 @ accumulate key material
+	ldr		@t[0],[r12],#16		@ load input
+	add		@x[5],@t[1],@x[5],ror#13
+	ldr		@t[1],[r12,#-12]
+	add		@x[6],@t[2],@x[6],ror#13
+	ldr		@t[2],[r12,#-8]
+	add		@x[7],@t[3],@x[7],ror#13
+	ldr		@t[3],[r12,#-4]
+# ifdef	__ARMEB__
+	rev		@x[4],@x[4]
+	rev		@x[5],@x[5]
+	rev		@x[6],@x[6]
+	rev		@x[7],@x[7]
+# endif
+	eor		@x[4],@x[4],@t[0]
+	 add		@t[0],sp,#4*(8)
+	eor		@x[5],@x[5],@t[1]
+	str		@x[4],[r14],#16		@ store output
+	eor		@x[6],@x[6],@t[2]
+	str		@x[5],[r14,#-12]
+	eor		@x[7],@x[7],@t[3]
+	 ldmia		@t[0],{@t[0]-@t[3]}	@ load key material
+	str		@x[6],[r14,#-8]
+	 add		@x[0],sp,#4*(16+8)
+	str		@x[7],[r14,#-4]
+
+	ldmia		@x[0],{@x[0]-@x[7]}	@ load second half
+
+	add		@x[0],@x[0],@t[0]	@ accumulate key material
+	ldr		@t[0],[r12],#16		@ load input
+	add		@x[1],@x[1],@t[1]
+	ldr		@t[1],[r12,#-12]
+# ifdef	__thumb2__
+	it	hi
+# endif
+	 strhi		@t[2],[sp,#4*(16+10)]	@ copy "@x[10]" while at it
+	add		@x[2],@x[2],@t[2]
+	ldr		@t[2],[r12,#-8]
+# ifdef	__thumb2__
+	it	hi
+# endif
+	 strhi		@t[3],[sp,#4*(16+11)]	@ copy "@x[11]" while at it
+	add		@x[3],@x[3],@t[3]
+	ldr		@t[3],[r12,#-4]
+# ifdef	__ARMEB__
+	rev		@x[0],@x[0]
+	rev		@x[1],@x[1]
+	rev		@x[2],@x[2]
+	rev		@x[3],@x[3]
+# endif
+	eor		@x[0],@x[0],@t[0]
+	 add		@t[0],sp,#4*(12)
+	eor		@x[1],@x[1],@t[1]
+	str		@x[0],[r14],#16		@ store output
+	eor		@x[2],@x[2],@t[2]
+	str		@x[1],[r14,#-12]
+	eor		@x[3],@x[3],@t[3]
+	 ldmia		@t[0],{@t[0]-@t[3]}	@ load key material
+	str		@x[2],[r14,#-8]
+	str		@x[3],[r14,#-4]
+
+	add		@x[4],@t[0],@x[4],ror#24 @ accumulate key material
+	 add		@t[0],@t[0],#4		@ next counter value
+	add		@x[5],@t[1],@x[5],ror#24
+	 str		@t[0],[sp,#4*(12)]	@ save next counter value
+	ldr		@t[0],[r12],#16		@ load input
+	add		@x[6],@t[2],@x[6],ror#24
+	 add		@x[4],@x[4],#3		@ counter+3
+	ldr		@t[1],[r12,#-12]
+	add		@x[7],@t[3],@x[7],ror#24
+	ldr		@t[2],[r12,#-8]
+	ldr		@t[3],[r12,#-4]
+# ifdef	__ARMEB__
+	rev		@x[4],@x[4]
+	rev		@x[5],@x[5]
+	rev		@x[6],@x[6]
+	rev		@x[7],@x[7]
+# endif
+	eor		@x[4],@x[4],@t[0]
+# ifdef	__thumb2__
+	it	hi
+# endif
+	 ldrhi		@t[0],[sp,#4*(32+2)]	@ re-load len
+	eor		@x[5],@x[5],@t[1]
+	eor		@x[6],@x[6],@t[2]
+	str		@x[4],[r14],#16		@ store output
+	eor		@x[7],@x[7],@t[3]
+	str		@x[5],[r14,#-12]
+	 sub		@t[3],@t[0],#64*4	@ len-=64*4
+	str		@x[6],[r14,#-8]
+	str		@x[7],[r14,#-4]
+	bhi		.Loop_neon_outer
+
+	b		.Ldone_neon
+
+.align	4
+.Lbreak_neon:
+	@ harmonize NEON and integer-only stack frames: load data
+	@ from NEON frame, but save to integer-only one; distance
+	@ between the two is 4*(32+4+16-32)=4*(20).
+
+	str		@t[3], [sp,#4*(20+32+2)]	@ save len
+	 add		@t[3],sp,#4*(32+4)
+	str		r12,   [sp,#4*(20+32+1)]	@ save inp
+	str		r14,   [sp,#4*(20+32+0)]	@ save out
+
+	ldr		@x[12],[sp,#4*(16+10)]
+	ldr		@x[14],[sp,#4*(16+11)]
+	 vldmia		@t[3],{d8-d15}			@ fulfill ABI requirement
+	str		@x[12],[sp,#4*(20+16+10)]	@ copy "@x[10]"
+	str		@x[14],[sp,#4*(20+16+11)]	@ copy "@x[11]"
+
+	ldr		@t[3], [sp,#4*(15)]
+	 mov		@x[4],@x[4],ror#19		@ twist b[0..3]
+	ldr		@x[12],[sp,#4*(12)]		@ modulo-scheduled load
+	 mov		@x[5],@x[5],ror#19
+	ldr		@t[2], [sp,#4*(13)]
+	 mov		@x[6],@x[6],ror#19
+	ldr		@x[14],[sp,#4*(14)]
+	 mov		@x[7],@x[7],ror#19
+	mov		@t[3],@t[3],ror#8		@ twist d[0..3]
+	mov		@x[12],@x[12],ror#8
+	mov		@t[2],@t[2],ror#8
+	mov		@x[14],@x[14],ror#8
+	str		@t[3], [sp,#4*(20+16+15)]
+	add		@t[3],sp,#4*(20)
+	vst1.32		{$a0-$b0},[@t[3]]!		@ copy key
+	add		sp,sp,#4*(20)			@ switch frame
+	vst1.32		{$c0-$d0},[@t[3]]
+	mov		@t[3],#10
+	b		.Loop				@ go integer-only
+
+.align	4
+.Ltail_neon:
+	cmp		@t[3],#64*3
+	bhs		.L192_or_more_neon
+	cmp		@t[3],#64*2
+	bhs		.L128_or_more_neon
+	cmp		@t[3],#64*1
+	bhs		.L64_or_more_neon
+
+	add		@t[0],sp,#4*(8)
+	vst1.8		{$a0-$b0},[sp]
+	add		@t[2],sp,#4*(0)
+	vst1.8		{$c0-$d0},[@t[0]]
+	b		.Loop_tail_neon
+
+.align	4
+.L64_or_more_neon:
+	vld1.8		{$t0-$t1},[r12]!
+	vld1.8		{$t2-$t3},[r12]!
+	veor		$a0,$a0,$t0
+	veor		$b0,$b0,$t1
+	veor		$c0,$c0,$t2
+	veor		$d0,$d0,$t3
+	vst1.8		{$a0-$b0},[r14]!
+	vst1.8		{$c0-$d0},[r14]!
+
+	beq		.Ldone_neon
+
+	add		@t[0],sp,#4*(8)
+	vst1.8		{$a1-$b1},[sp]
+	add		@t[2],sp,#4*(0)
+	vst1.8		{$c1-$d1},[@t[0]]
+	sub		@t[3],@t[3],#64*1	@ len-=64*1
+	b		.Loop_tail_neon
+
+.align	4
+.L128_or_more_neon:
+	vld1.8		{$t0-$t1},[r12]!
+	vld1.8		{$t2-$t3},[r12]!
+	veor		$a0,$a0,$t0
+	veor		$b0,$b0,$t1
+	vld1.8		{$t0-$t1},[r12]!
+	veor		$c0,$c0,$t2
+	veor		$d0,$d0,$t3
+	vld1.8		{$t2-$t3},[r12]!
+
+	veor		$a1,$a1,$t0
+	veor		$b1,$b1,$t1
+	 vst1.8		{$a0-$b0},[r14]!
+	veor		$c1,$c1,$t2
+	 vst1.8		{$c0-$d0},[r14]!
+	veor		$d1,$d1,$t3
+	vst1.8		{$a1-$b1},[r14]!
+	vst1.8		{$c1-$d1},[r14]!
+
+	beq		.Ldone_neon
+
+	add		@t[0],sp,#4*(8)
+	vst1.8		{$a2-$b2},[sp]
+	add		@t[2],sp,#4*(0)
+	vst1.8		{$c2-$d2},[@t[0]]
+	sub		@t[3],@t[3],#64*2	@ len-=64*2
+	b		.Loop_tail_neon
+
+.align	4
+.L192_or_more_neon:
+	vld1.8		{$t0-$t1},[r12]!
+	vld1.8		{$t2-$t3},[r12]!
+	veor		$a0,$a0,$t0
+	veor		$b0,$b0,$t1
+	vld1.8		{$t0-$t1},[r12]!
+	veor		$c0,$c0,$t2
+	veor		$d0,$d0,$t3
+	vld1.8		{$t2-$t3},[r12]!
+
+	veor		$a1,$a1,$t0
+	veor		$b1,$b1,$t1
+	vld1.8		{$t0-$t1},[r12]!
+	veor		$c1,$c1,$t2
+	 vst1.8		{$a0-$b0},[r14]!
+	veor		$d1,$d1,$t3
+	vld1.8		{$t2-$t3},[r12]!
+
+	veor		$a2,$a2,$t0
+	 vst1.8		{$c0-$d0},[r14]!
+	veor		$b2,$b2,$t1
+	 vst1.8		{$a1-$b1},[r14]!
+	veor		$c2,$c2,$t2
+	 vst1.8		{$c1-$d1},[r14]!
+	veor		$d2,$d2,$t3
+	vst1.8		{$a2-$b2},[r14]!
+	vst1.8		{$c2-$d2},[r14]!
+
+	beq		.Ldone_neon
+
+	ldmia		sp,{@t[0]-@t[3]}	@ load key material
+	add		@x[0],@x[0],@t[0]	@ accumulate key material
+	 add		@t[0],sp,#4*(4)
+	add		@x[1],@x[1],@t[1]
+	add		@x[2],@x[2],@t[2]
+	add		@x[3],@x[3],@t[3]
+	 ldmia		@t[0],{@t[0]-@t[3]}	@ load key material
+
+	add		@x[4],@t[0],@x[4],ror#13 @ accumulate key material
+	 add		@t[0],sp,#4*(8)
+	add		@x[5],@t[1],@x[5],ror#13
+	add		@x[6],@t[2],@x[6],ror#13
+	add		@x[7],@t[3],@x[7],ror#13
+	 ldmia		@t[0],{@t[0]-@t[3]}	@ load key material
+# ifdef	__ARMEB__
+	rev		@x[0],@x[0]
+	rev		@x[1],@x[1]
+	rev		@x[2],@x[2]
+	rev		@x[3],@x[3]
+	rev		@x[4],@x[4]
+	rev		@x[5],@x[5]
+	rev		@x[6],@x[6]
+	rev		@x[7],@x[7]
+# endif
+	stmia		sp,{@x[0]-@x[7]}
+	 add		@x[0],sp,#4*(16+8)
+
+	ldmia		@x[0],{@x[0]-@x[7]}	@ load second half
+
+	add		@x[0],@x[0],@t[0]	@ accumulate key material
+	 add		@t[0],sp,#4*(12)
+	add		@x[1],@x[1],@t[1]
+	add		@x[2],@x[2],@t[2]
+	add		@x[3],@x[3],@t[3]
+	 ldmia		@t[0],{@t[0]-@t[3]}	@ load key material
+
+	add		@x[4],@t[0],@x[4],ror#24 @ accumulate key material
+	 add		@t[0],sp,#4*(8)
+	add		@x[5],@t[1],@x[5],ror#24
+	 add		@x[4],@x[4],#3		@ counter+3
+	add		@x[6],@t[2],@x[6],ror#24
+	add		@x[7],@t[3],@x[7],ror#24
+	 ldr		@t[3],[sp,#4*(32+2)]	@ re-load len
+# ifdef	__ARMEB__
+	rev		@x[0],@x[0]
+	rev		@x[1],@x[1]
+	rev		@x[2],@x[2]
+	rev		@x[3],@x[3]
+	rev		@x[4],@x[4]
+	rev		@x[5],@x[5]
+	rev		@x[6],@x[6]
+	rev		@x[7],@x[7]
+# endif
+	stmia		@t[0],{@x[0]-@x[7]}
+	 add		@t[2],sp,#4*(0)
+	 sub		@t[3],@t[3],#64*3	@ len-=64*3
+
+.Loop_tail_neon:
+	ldrb		@t[0],[@t[2]],#1	@ read buffer on stack
+	ldrb		@t[1],[r12],#1		@ read input
+	subs		@t[3],@t[3],#1
+	eor		@t[0],@t[0],@t[1]
+	strb		@t[0],[r14],#1		@ store output
+	bne		.Loop_tail_neon
+
+.Ldone_neon:
+	add		sp,sp,#4*(32+4)
+	vldmia		sp,{d8-d15}
+	add		sp,sp,#4*(16+3)
+	ldmia		sp!,{r4-r11,pc}
+.size	ChaCha20_neon,.-ChaCha20_neon
+# ifndef __KERNEL__
+.comm	OPENSSL_armcap_P,4,4
+# endif
+#endif
+___
+}}}
+
+open SELF,$0;
+while(<SELF>) {
+	next if (/^#!/);
+	last if (!s/^#/@/ and !/^$/);
+	print;
+}
+close SELF;
+
+foreach (split("\n",$code)) {
+	s/\`([^\`]*)\`/eval $1/geo;
+
+	s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo;
+
+	print $_,"\n";
+}
+close STDOUT;
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/chacha20/chacha20.c b/net/wireguard/crypto/zinc/chacha20/chacha20.c
--- a/net/wireguard/crypto/zinc/chacha20/chacha20.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/chacha20/chacha20.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,193 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ *
+ * Implementation of the ChaCha20 stream cipher.
+ *
+ * Information: https://cr.yp.to/chacha.html
+ */
+
+#include <zinc/chacha20.h>
+#include "../selftest/run.h"
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/vmalloc.h>
+#include <crypto/algapi.h> // For crypto_xor_cpy.
+
+#if defined(CONFIG_ZINC_ARCH_X86_64)
+#include "chacha20-x86_64-glue.c"
+#elif defined(CONFIG_ZINC_ARCH_ARM) || defined(CONFIG_ZINC_ARCH_ARM64)
+#include "chacha20-arm-glue.c"
+#elif defined(CONFIG_ZINC_ARCH_MIPS)
+#include "chacha20-mips-glue.c"
+#else
+static bool *const chacha20_nobs[] __initconst = { };
+static void __init chacha20_fpu_init(void)
+{
+}
+static inline bool chacha20_arch(struct chacha20_ctx *ctx, u8 *dst,
+				 const u8 *src, size_t len,
+				 simd_context_t *simd_context)
+{
+	return false;
+}
+static inline bool hchacha20_arch(u32 derived_key[CHACHA20_KEY_WORDS],
+				  const u8 nonce[HCHACHA20_NONCE_SIZE],
+				  const u8 key[HCHACHA20_KEY_SIZE],
+				  simd_context_t *simd_context)
+{
+	return false;
+}
+#endif
+
+#define QUARTER_ROUND(x, a, b, c, d) ( \
+	x[a] += x[b], \
+	x[d] = rol32((x[d] ^ x[a]), 16), \
+	x[c] += x[d], \
+	x[b] = rol32((x[b] ^ x[c]), 12), \
+	x[a] += x[b], \
+	x[d] = rol32((x[d] ^ x[a]), 8), \
+	x[c] += x[d], \
+	x[b] = rol32((x[b] ^ x[c]), 7) \
+)
+
+#define C(i, j) (i * 4 + j)
+
+#define DOUBLE_ROUND(x) ( \
+	/* Column Round */ \
+	QUARTER_ROUND(x, C(0, 0), C(1, 0), C(2, 0), C(3, 0)), \
+	QUARTER_ROUND(x, C(0, 1), C(1, 1), C(2, 1), C(3, 1)), \
+	QUARTER_ROUND(x, C(0, 2), C(1, 2), C(2, 2), C(3, 2)), \
+	QUARTER_ROUND(x, C(0, 3), C(1, 3), C(2, 3), C(3, 3)), \
+	/* Diagonal Round */ \
+	QUARTER_ROUND(x, C(0, 0), C(1, 1), C(2, 2), C(3, 3)), \
+	QUARTER_ROUND(x, C(0, 1), C(1, 2), C(2, 3), C(3, 0)), \
+	QUARTER_ROUND(x, C(0, 2), C(1, 3), C(2, 0), C(3, 1)), \
+	QUARTER_ROUND(x, C(0, 3), C(1, 0), C(2, 1), C(3, 2)) \
+)
+
+#define TWENTY_ROUNDS(x) ( \
+	DOUBLE_ROUND(x), \
+	DOUBLE_ROUND(x), \
+	DOUBLE_ROUND(x), \
+	DOUBLE_ROUND(x), \
+	DOUBLE_ROUND(x), \
+	DOUBLE_ROUND(x), \
+	DOUBLE_ROUND(x), \
+	DOUBLE_ROUND(x), \
+	DOUBLE_ROUND(x), \
+	DOUBLE_ROUND(x) \
+)
+
+static void chacha20_block_generic(struct chacha20_ctx *ctx, __le32 *stream)
+{
+	u32 x[CHACHA20_BLOCK_WORDS];
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(x); ++i)
+		x[i] = ctx->state[i];
+
+	TWENTY_ROUNDS(x);
+
+	for (i = 0; i < ARRAY_SIZE(x); ++i)
+		stream[i] = cpu_to_le32(x[i] + ctx->state[i]);
+
+	ctx->counter[0] += 1;
+}
+
+static void chacha20_generic(struct chacha20_ctx *ctx, u8 *out, const u8 *in,
+			     u32 len)
+{
+	__le32 buf[CHACHA20_BLOCK_WORDS];
+
+	while (len >= CHACHA20_BLOCK_SIZE) {
+		chacha20_block_generic(ctx, buf);
+		crypto_xor_cpy(out, in, (u8 *)buf, CHACHA20_BLOCK_SIZE);
+		len -= CHACHA20_BLOCK_SIZE;
+		out += CHACHA20_BLOCK_SIZE;
+		in += CHACHA20_BLOCK_SIZE;
+	}
+	if (len) {
+		chacha20_block_generic(ctx, buf);
+		crypto_xor_cpy(out, in, (u8 *)buf, len);
+	}
+}
+
+void chacha20(struct chacha20_ctx *ctx, u8 *dst, const u8 *src, u32 len,
+	      simd_context_t *simd_context)
+{
+	if (!chacha20_arch(ctx, dst, src, len, simd_context))
+		chacha20_generic(ctx, dst, src, len);
+}
+EXPORT_SYMBOL(chacha20);
+
+static void hchacha20_generic(u32 derived_key[CHACHA20_KEY_WORDS],
+			      const u8 nonce[HCHACHA20_NONCE_SIZE],
+			      const u8 key[HCHACHA20_KEY_SIZE])
+{
+	u32 x[] = { CHACHA20_CONSTANT_EXPA,
+		    CHACHA20_CONSTANT_ND_3,
+		    CHACHA20_CONSTANT_2_BY,
+		    CHACHA20_CONSTANT_TE_K,
+		    get_unaligned_le32(key +  0),
+		    get_unaligned_le32(key +  4),
+		    get_unaligned_le32(key +  8),
+		    get_unaligned_le32(key + 12),
+		    get_unaligned_le32(key + 16),
+		    get_unaligned_le32(key + 20),
+		    get_unaligned_le32(key + 24),
+		    get_unaligned_le32(key + 28),
+		    get_unaligned_le32(nonce +  0),
+		    get_unaligned_le32(nonce +  4),
+		    get_unaligned_le32(nonce +  8),
+		    get_unaligned_le32(nonce + 12)
+	};
+
+	TWENTY_ROUNDS(x);
+
+	memcpy(derived_key + 0, x +  0, sizeof(u32) * 4);
+	memcpy(derived_key + 4, x + 12, sizeof(u32) * 4);
+}
+
+/* Derived key should be 32-bit aligned */
+void hchacha20(u32 derived_key[CHACHA20_KEY_WORDS],
+	       const u8 nonce[HCHACHA20_NONCE_SIZE],
+	       const u8 key[HCHACHA20_KEY_SIZE], simd_context_t *simd_context)
+{
+	if (!hchacha20_arch(derived_key, nonce, key, simd_context))
+		hchacha20_generic(derived_key, nonce, key);
+}
+EXPORT_SYMBOL(hchacha20);
+
+#include "../selftest/chacha20.c"
+
+static bool nosimd __initdata = false;
+
+#ifndef COMPAT_ZINC_IS_A_MODULE
+int __init chacha20_mod_init(void)
+#else
+static int __init mod_init(void)
+#endif
+{
+	if (!nosimd)
+		chacha20_fpu_init();
+	if (!selftest_run("chacha20", chacha20_selftest, chacha20_nobs,
+			  ARRAY_SIZE(chacha20_nobs)))
+		return -ENOTRECOVERABLE;
+	return 0;
+}
+
+#ifdef COMPAT_ZINC_IS_A_MODULE
+static void __exit mod_exit(void)
+{
+}
+
+module_param(nosimd, bool, 0);
+module_init(mod_init);
+module_exit(mod_exit);
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("ChaCha20 stream cipher");
+MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");
+#endif
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/chacha20/chacha20-mips-glue.c b/net/wireguard/crypto/zinc/chacha20/chacha20-mips-glue.c
--- a/net/wireguard/crypto/zinc/chacha20/chacha20-mips-glue.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/chacha20/chacha20-mips-glue.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+asmlinkage void chacha20_mips(u32 state[16], u8 *out, const u8 *in,
+			      const size_t len);
+static bool *const chacha20_nobs[] __initconst = { };
+static void __init chacha20_fpu_init(void)
+{
+}
+
+static inline bool chacha20_arch(struct chacha20_ctx *ctx, u8 *dst,
+				 const u8 *src, size_t len,
+				 simd_context_t *simd_context)
+{
+	chacha20_mips(ctx->state, dst, src, len);
+	return true;
+}
+
+static inline bool hchacha20_arch(u32 derived_key[CHACHA20_KEY_WORDS],
+				  const u8 nonce[HCHACHA20_NONCE_SIZE],
+				  const u8 key[HCHACHA20_KEY_SIZE],
+				  simd_context_t *simd_context)
+{
+	return false;
+}
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/chacha20/chacha20-mips.S b/net/wireguard/crypto/zinc/chacha20/chacha20-mips.S
--- a/net/wireguard/crypto/zinc/chacha20/chacha20-mips.S	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/chacha20/chacha20-mips.S	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,424 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/*
+ * Copyright (C) 2016-2018 René van Dorst <opensource@vdorst.com>. All Rights Reserved.
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#define MASK_U32		0x3c
+#define CHACHA20_BLOCK_SIZE	64
+#define STACK_SIZE		32
+
+#define X0	$t0
+#define X1	$t1
+#define X2	$t2
+#define X3	$t3
+#define X4	$t4
+#define X5	$t5
+#define X6	$t6
+#define X7	$t7
+#define X8	$t8
+#define X9	$t9
+#define X10	$v1
+#define X11	$s6
+#define X12	$s5
+#define X13	$s4
+#define X14	$s3
+#define X15	$s2
+/* Use regs which are overwritten on exit for Tx so we don't leak clear data. */
+#define T0	$s1
+#define T1	$s0
+#define T(n)	T ## n
+#define X(n)	X ## n
+
+/* Input arguments */
+#define STATE		$a0
+#define OUT		$a1
+#define IN		$a2
+#define BYTES		$a3
+
+/* Output argument */
+/* NONCE[0] is kept in a register and not in memory.
+ * We don't want to touch original value in memory.
+ * Must be incremented every loop iteration.
+ */
+#define NONCE_0		$v0
+
+/* SAVED_X and SAVED_CA are set in the jump table.
+ * Use regs which are overwritten on exit else we don't leak clear data.
+ * They are used to handling the last bytes which are not multiple of 4.
+ */
+#define SAVED_X		X15
+#define SAVED_CA	$s7
+
+#define IS_UNALIGNED	$s7
+
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+#define MSB 0
+#define LSB 3
+#define ROTx rotl
+#define ROTR(n) rotr n, 24
+#define	CPU_TO_LE32(n) \
+	wsbh	n; \
+	rotr	n, 16;
+#else
+#define MSB 3
+#define LSB 0
+#define ROTx rotr
+#define CPU_TO_LE32(n)
+#define ROTR(n)
+#endif
+
+#define FOR_EACH_WORD(x) \
+	x( 0); \
+	x( 1); \
+	x( 2); \
+	x( 3); \
+	x( 4); \
+	x( 5); \
+	x( 6); \
+	x( 7); \
+	x( 8); \
+	x( 9); \
+	x(10); \
+	x(11); \
+	x(12); \
+	x(13); \
+	x(14); \
+	x(15);
+
+#define FOR_EACH_WORD_REV(x) \
+	x(15); \
+	x(14); \
+	x(13); \
+	x(12); \
+	x(11); \
+	x(10); \
+	x( 9); \
+	x( 8); \
+	x( 7); \
+	x( 6); \
+	x( 5); \
+	x( 4); \
+	x( 3); \
+	x( 2); \
+	x( 1); \
+	x( 0);
+
+#define PLUS_ONE_0	 1
+#define PLUS_ONE_1	 2
+#define PLUS_ONE_2	 3
+#define PLUS_ONE_3	 4
+#define PLUS_ONE_4	 5
+#define PLUS_ONE_5	 6
+#define PLUS_ONE_6	 7
+#define PLUS_ONE_7	 8
+#define PLUS_ONE_8	 9
+#define PLUS_ONE_9	10
+#define PLUS_ONE_10	11
+#define PLUS_ONE_11	12
+#define PLUS_ONE_12	13
+#define PLUS_ONE_13	14
+#define PLUS_ONE_14	15
+#define PLUS_ONE_15	16
+#define PLUS_ONE(x)	PLUS_ONE_ ## x
+#define _CONCAT3(a,b,c)	a ## b ## c
+#define CONCAT3(a,b,c)	_CONCAT3(a,b,c)
+
+#define STORE_UNALIGNED(x) \
+CONCAT3(.Lchacha20_mips_xor_unaligned_, PLUS_ONE(x), _b: ;) \
+	.if (x != 12); \
+		lw	T0, (x*4)(STATE); \
+	.endif; \
+	lwl	T1, (x*4)+MSB ## (IN); \
+	lwr	T1, (x*4)+LSB ## (IN); \
+	.if (x == 12); \
+		addu	X ## x, NONCE_0; \
+	.else; \
+		addu	X ## x, T0; \
+	.endif; \
+	CPU_TO_LE32(X ## x); \
+	xor	X ## x, T1; \
+	swl	X ## x, (x*4)+MSB ## (OUT); \
+	swr	X ## x, (x*4)+LSB ## (OUT);
+
+#define STORE_ALIGNED(x) \
+CONCAT3(.Lchacha20_mips_xor_aligned_, PLUS_ONE(x), _b: ;) \
+	.if (x != 12); \
+		lw	T0, (x*4)(STATE); \
+	.endif; \
+	lw	T1, (x*4) ## (IN); \
+	.if (x == 12); \
+		addu	X ## x, NONCE_0; \
+	.else; \
+		addu	X ## x, T0; \
+	.endif; \
+	CPU_TO_LE32(X ## x); \
+	xor	X ## x, T1; \
+	sw	X ## x, (x*4) ## (OUT);
+
+/* Jump table macro.
+ * Used for setup and handling the last bytes, which are not multiple of 4.
+ * X15 is free to store Xn
+ * Every jumptable entry must be equal in size.
+ */
+#define JMPTBL_ALIGNED(x) \
+.Lchacha20_mips_jmptbl_aligned_ ## x: ; \
+	.set	noreorder; \
+	b	.Lchacha20_mips_xor_aligned_ ## x ## _b; \
+	.if (x == 12); \
+		addu	SAVED_X, X ## x, NONCE_0; \
+	.else; \
+		addu	SAVED_X, X ## x, SAVED_CA; \
+	.endif; \
+	.set	reorder
+
+#define JMPTBL_UNALIGNED(x) \
+.Lchacha20_mips_jmptbl_unaligned_ ## x: ; \
+	.set	noreorder; \
+	b	.Lchacha20_mips_xor_unaligned_ ## x ## _b; \
+	.if (x == 12); \
+		addu	SAVED_X, X ## x, NONCE_0; \
+	.else; \
+		addu	SAVED_X, X ## x, SAVED_CA; \
+	.endif; \
+	.set	reorder
+
+#define AXR(A, B, C, D,  K, L, M, N,  V, W, Y, Z,  S) \
+	addu	X(A), X(K); \
+	addu	X(B), X(L); \
+	addu	X(C), X(M); \
+	addu	X(D), X(N); \
+	xor	X(V), X(A); \
+	xor	X(W), X(B); \
+	xor	X(Y), X(C); \
+	xor	X(Z), X(D); \
+	rotl	X(V), S;    \
+	rotl	X(W), S;    \
+	rotl	X(Y), S;    \
+	rotl	X(Z), S;
+
+.text
+.set	reorder
+.set	noat
+.globl	chacha20_mips
+.ent	chacha20_mips
+chacha20_mips:
+	.frame	$sp, STACK_SIZE, $ra
+
+	addiu	$sp, -STACK_SIZE
+
+	/* Return bytes = 0. */
+	beqz	BYTES, .Lchacha20_mips_end
+
+	lw	NONCE_0, 48(STATE)
+
+	/* Save s0-s7 */
+	sw	$s0,  0($sp)
+	sw	$s1,  4($sp)
+	sw	$s2,  8($sp)
+	sw	$s3, 12($sp)
+	sw	$s4, 16($sp)
+	sw	$s5, 20($sp)
+	sw	$s6, 24($sp)
+	sw	$s7, 28($sp)
+
+	/* Test IN or OUT is unaligned.
+	 * IS_UNALIGNED = ( IN | OUT ) & 0x00000003
+	 */
+	or	IS_UNALIGNED, IN, OUT
+	andi	IS_UNALIGNED, 0x3
+
+	/* Set number of rounds */
+	li	$at, 20
+
+	b	.Lchacha20_rounds_start
+
+.align 4
+.Loop_chacha20_rounds:
+	addiu	IN,  CHACHA20_BLOCK_SIZE
+	addiu	OUT, CHACHA20_BLOCK_SIZE
+	addiu	NONCE_0, 1
+
+.Lchacha20_rounds_start:
+	lw	X0,  0(STATE)
+	lw	X1,  4(STATE)
+	lw	X2,  8(STATE)
+	lw	X3,  12(STATE)
+
+	lw	X4,  16(STATE)
+	lw	X5,  20(STATE)
+	lw	X6,  24(STATE)
+	lw	X7,  28(STATE)
+	lw	X8,  32(STATE)
+	lw	X9,  36(STATE)
+	lw	X10, 40(STATE)
+	lw	X11, 44(STATE)
+
+	move	X12, NONCE_0
+	lw	X13, 52(STATE)
+	lw	X14, 56(STATE)
+	lw	X15, 60(STATE)
+
+.Loop_chacha20_xor_rounds:
+	addiu	$at, -2
+	AXR( 0, 1, 2, 3,  4, 5, 6, 7, 12,13,14,15, 16);
+	AXR( 8, 9,10,11, 12,13,14,15,  4, 5, 6, 7, 12);
+	AXR( 0, 1, 2, 3,  4, 5, 6, 7, 12,13,14,15,  8);
+	AXR( 8, 9,10,11, 12,13,14,15,  4, 5, 6, 7,  7);
+	AXR( 0, 1, 2, 3,  5, 6, 7, 4, 15,12,13,14, 16);
+	AXR(10,11, 8, 9, 15,12,13,14,  5, 6, 7, 4, 12);
+	AXR( 0, 1, 2, 3,  5, 6, 7, 4, 15,12,13,14,  8);
+	AXR(10,11, 8, 9, 15,12,13,14,  5, 6, 7, 4,  7);
+	bnez	$at, .Loop_chacha20_xor_rounds
+
+	addiu	BYTES, -(CHACHA20_BLOCK_SIZE)
+
+	/* Is data src/dst unaligned? Jump */
+	bnez	IS_UNALIGNED, .Loop_chacha20_unaligned
+
+	/* Set number rounds here to fill delayslot. */
+	li	$at, 20
+
+	/* BYTES < 0, it has no full block. */
+	bltz	BYTES, .Lchacha20_mips_no_full_block_aligned
+
+	FOR_EACH_WORD_REV(STORE_ALIGNED)
+
+	/* BYTES > 0? Loop again. */
+	bgtz	BYTES, .Loop_chacha20_rounds
+
+	/* Place this here to fill delay slot */
+	addiu	NONCE_0, 1
+
+	/* BYTES < 0? Handle last bytes */
+	bltz	BYTES, .Lchacha20_mips_xor_bytes
+
+.Lchacha20_mips_xor_done:
+	/* Restore used registers */
+	lw	$s0,  0($sp)
+	lw	$s1,  4($sp)
+	lw	$s2,  8($sp)
+	lw	$s3, 12($sp)
+	lw	$s4, 16($sp)
+	lw	$s5, 20($sp)
+	lw	$s6, 24($sp)
+	lw	$s7, 28($sp)
+
+	/* Write NONCE_0 back to right location in state */
+	sw	NONCE_0, 48(STATE)
+
+.Lchacha20_mips_end:
+	addiu	$sp, STACK_SIZE
+	jr	$ra
+
+.Lchacha20_mips_no_full_block_aligned:
+	/* Restore the offset on BYTES */
+	addiu	BYTES, CHACHA20_BLOCK_SIZE
+
+	/* Get number of full WORDS */
+	andi	$at, BYTES, MASK_U32
+
+	/* Load upper half of jump table addr */
+	lui	T0, %hi(.Lchacha20_mips_jmptbl_aligned_0)
+
+	/* Calculate lower half jump table offset */
+	ins	T0, $at, 1, 6
+
+	/* Add offset to STATE */
+	addu	T1, STATE, $at
+
+	/* Add lower half jump table addr */
+	addiu	T0, %lo(.Lchacha20_mips_jmptbl_aligned_0)
+
+	/* Read value from STATE */
+	lw	SAVED_CA, 0(T1)
+
+	/* Store remaining bytecounter as negative value */
+	subu	BYTES, $at, BYTES
+
+	jr	T0
+
+	/* Jump table */
+	FOR_EACH_WORD(JMPTBL_ALIGNED)
+
+
+.Loop_chacha20_unaligned:
+	/* Set number rounds here to fill delayslot. */
+	li	$at, 20
+
+	/* BYTES > 0, it has no full block. */
+	bltz	BYTES, .Lchacha20_mips_no_full_block_unaligned
+
+	FOR_EACH_WORD_REV(STORE_UNALIGNED)
+
+	/* BYTES > 0? Loop again. */
+	bgtz	BYTES, .Loop_chacha20_rounds
+
+	/* Write NONCE_0 back to right location in state */
+	sw	NONCE_0, 48(STATE)
+
+	.set noreorder
+	/* Fall through to byte handling */
+	bgez	BYTES, .Lchacha20_mips_xor_done
+.Lchacha20_mips_xor_unaligned_0_b:
+.Lchacha20_mips_xor_aligned_0_b:
+	/* Place this here to fill delay slot */
+	addiu	NONCE_0, 1
+	.set reorder
+
+.Lchacha20_mips_xor_bytes:
+	addu	IN, $at
+	addu	OUT, $at
+	/* First byte */
+	lbu	T1, 0(IN)
+	addiu	$at, BYTES, 1
+	CPU_TO_LE32(SAVED_X)
+	ROTR(SAVED_X)
+	xor	T1, SAVED_X
+	sb	T1, 0(OUT)
+	beqz	$at, .Lchacha20_mips_xor_done
+	/* Second byte */
+	lbu	T1, 1(IN)
+	addiu	$at, BYTES, 2
+	ROTx	SAVED_X, 8
+	xor	T1, SAVED_X
+	sb	T1, 1(OUT)
+	beqz	$at, .Lchacha20_mips_xor_done
+	/* Third byte */
+	lbu	T1, 2(IN)
+	ROTx	SAVED_X, 8
+	xor	T1, SAVED_X
+	sb	T1, 2(OUT)
+	b	.Lchacha20_mips_xor_done
+
+.Lchacha20_mips_no_full_block_unaligned:
+	/* Restore the offset on BYTES */
+	addiu	BYTES, CHACHA20_BLOCK_SIZE
+
+	/* Get number of full WORDS */
+	andi	$at, BYTES, MASK_U32
+
+	/* Load upper half of jump table addr */
+	lui	T0, %hi(.Lchacha20_mips_jmptbl_unaligned_0)
+
+	/* Calculate lower half jump table offset */
+	ins	T0, $at, 1, 6
+
+	/* Add offset to STATE */
+	addu	T1, STATE, $at
+
+	/* Add lower half jump table addr */
+	addiu	T0, %lo(.Lchacha20_mips_jmptbl_unaligned_0)
+
+	/* Read value from STATE */
+	lw	SAVED_CA, 0(T1)
+
+	/* Store remaining bytecounter as negative value */
+	subu	BYTES, $at, BYTES
+
+	jr	T0
+
+	/* Jump table */
+	FOR_EACH_WORD(JMPTBL_UNALIGNED)
+.end chacha20_mips
+.set at
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/chacha20/chacha20-unrolled-arm.S b/net/wireguard/crypto/zinc/chacha20/chacha20-unrolled-arm.S
--- a/net/wireguard/crypto/zinc/chacha20/chacha20-unrolled-arm.S	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/chacha20/chacha20-unrolled-arm.S	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,461 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2018 Google, Inc.
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+/*
+ * Design notes:
+ *
+ * 16 registers would be needed to hold the state matrix, but only 14 are
+ * available because 'sp' and 'pc' cannot be used.  So we spill the elements
+ * (x8, x9) to the stack and swap them out with (x10, x11).  This adds one
+ * 'ldrd' and one 'strd' instruction per round.
+ *
+ * All rotates are performed using the implicit rotate operand accepted by the
+ * 'add' and 'eor' instructions.  This is faster than using explicit rotate
+ * instructions.  To make this work, we allow the values in the second and last
+ * rows of the ChaCha state matrix (rows 'b' and 'd') to temporarily have the
+ * wrong rotation amount.  The rotation amount is then fixed up just in time
+ * when the values are used.  'brot' is the number of bits the values in row 'b'
+ * need to be rotated right to arrive at the correct values, and 'drot'
+ * similarly for row 'd'.  (brot, drot) start out as (0, 0) but we make it such
+ * that they end up as (25, 24) after every round.
+ */
+
+	// ChaCha state registers
+	X0	.req	r0
+	X1	.req	r1
+	X2	.req	r2
+	X3	.req	r3
+	X4	.req	r4
+	X5	.req	r5
+	X6	.req	r6
+	X7	.req	r7
+	X8_X10	.req	r8	// shared by x8 and x10
+	X9_X11	.req	r9	// shared by x9 and x11
+	X12	.req	r10
+	X13	.req	r11
+	X14	.req	r12
+	X15	.req	r14
+
+.Lexpand_32byte_k:
+	// "expand 32-byte k"
+	.word	0x61707865, 0x3320646e, 0x79622d32, 0x6b206574
+
+#ifdef __thumb2__
+#  define adrl adr
+#endif
+
+.macro __rev		out, in,  t0, t1, t2
+.if __LINUX_ARM_ARCH__ >= 6
+	rev		\out, \in
+.else
+	lsl		\t0, \in, #24
+	and		\t1, \in, #0xff00
+	and		\t2, \in, #0xff0000
+	orr		\out, \t0, \in, lsr #24
+	orr		\out, \out, \t1, lsl #8
+	orr		\out, \out, \t2, lsr #8
+.endif
+.endm
+
+.macro _le32_bswap	x,  t0, t1, t2
+#ifdef __ARMEB__
+	__rev		\x, \x,  \t0, \t1, \t2
+#endif
+.endm
+
+.macro _le32_bswap_4x	a, b, c, d,  t0, t1, t2
+	_le32_bswap	\a,  \t0, \t1, \t2
+	_le32_bswap	\b,  \t0, \t1, \t2
+	_le32_bswap	\c,  \t0, \t1, \t2
+	_le32_bswap	\d,  \t0, \t1, \t2
+.endm
+
+.macro __ldrd		a, b, src, offset
+#if __LINUX_ARM_ARCH__ >= 6
+	ldrd		\a, \b, [\src, #\offset]
+#else
+	ldr		\a, [\src, #\offset]
+	ldr		\b, [\src, #\offset + 4]
+#endif
+.endm
+
+.macro __strd		a, b, dst, offset
+#if __LINUX_ARM_ARCH__ >= 6
+	strd		\a, \b, [\dst, #\offset]
+#else
+	str		\a, [\dst, #\offset]
+	str		\b, [\dst, #\offset + 4]
+#endif
+.endm
+
+.macro _halfround	a1, b1, c1, d1,  a2, b2, c2, d2
+
+	// a += b; d ^= a; d = rol(d, 16);
+	add		\a1, \a1, \b1, ror #brot
+	add		\a2, \a2, \b2, ror #brot
+	eor		\d1, \a1, \d1, ror #drot
+	eor		\d2, \a2, \d2, ror #drot
+	// drot == 32 - 16 == 16
+
+	// c += d; b ^= c; b = rol(b, 12);
+	add		\c1, \c1, \d1, ror #16
+	add		\c2, \c2, \d2, ror #16
+	eor		\b1, \c1, \b1, ror #brot
+	eor		\b2, \c2, \b2, ror #brot
+	// brot == 32 - 12 == 20
+
+	// a += b; d ^= a; d = rol(d, 8);
+	add		\a1, \a1, \b1, ror #20
+	add		\a2, \a2, \b2, ror #20
+	eor		\d1, \a1, \d1, ror #16
+	eor		\d2, \a2, \d2, ror #16
+	// drot == 32 - 8 == 24
+
+	// c += d; b ^= c; b = rol(b, 7);
+	add		\c1, \c1, \d1, ror #24
+	add		\c2, \c2, \d2, ror #24
+	eor		\b1, \c1, \b1, ror #20
+	eor		\b2, \c2, \b2, ror #20
+	// brot == 32 - 7 == 25
+.endm
+
+.macro _doubleround
+
+	// column round
+
+	// quarterrounds: (x0, x4, x8, x12) and (x1, x5, x9, x13)
+	_halfround	X0, X4, X8_X10, X12,  X1, X5, X9_X11, X13
+
+	// save (x8, x9); restore (x10, x11)
+	__strd		X8_X10, X9_X11, sp, 0
+	__ldrd		X8_X10, X9_X11, sp, 8
+
+	// quarterrounds: (x2, x6, x10, x14) and (x3, x7, x11, x15)
+	_halfround	X2, X6, X8_X10, X14,  X3, X7, X9_X11, X15
+
+	.set brot, 25
+	.set drot, 24
+
+	// diagonal round
+
+	// quarterrounds: (x0, x5, x10, x15) and (x1, x6, x11, x12)
+	_halfround	X0, X5, X8_X10, X15,  X1, X6, X9_X11, X12
+
+	// save (x10, x11); restore (x8, x9)
+	__strd		X8_X10, X9_X11, sp, 8
+	__ldrd		X8_X10, X9_X11, sp, 0
+
+	// quarterrounds: (x2, x7, x8, x13) and (x3, x4, x9, x14)
+	_halfround	X2, X7, X8_X10, X13,  X3, X4, X9_X11, X14
+.endm
+
+.macro _chacha_permute	nrounds
+	.set brot, 0
+	.set drot, 0
+	.rept \nrounds / 2
+	 _doubleround
+	.endr
+.endm
+
+.macro _chacha		nrounds
+
+.Lnext_block\@:
+	// Stack: unused0-unused1 x10-x11 x0-x15 OUT IN LEN
+	// Registers contain x0-x9,x12-x15.
+
+	// Do the core ChaCha permutation to update x0-x15.
+	_chacha_permute	\nrounds
+
+	add		sp, #8
+	// Stack: x10-x11 orig_x0-orig_x15 OUT IN LEN
+	// Registers contain x0-x9,x12-x15.
+	// x4-x7 are rotated by 'brot'; x12-x15 are rotated by 'drot'.
+
+	// Free up some registers (r8-r12,r14) by pushing (x8-x9,x12-x15).
+	push		{X8_X10, X9_X11, X12, X13, X14, X15}
+
+	// Load (OUT, IN, LEN).
+	ldr		r14, [sp, #96]
+	ldr		r12, [sp, #100]
+	ldr		r11, [sp, #104]
+
+	orr		r10, r14, r12
+
+	// Use slow path if fewer than 64 bytes remain.
+	cmp		r11, #64
+	blt		.Lxor_slowpath\@
+
+	// Use slow path if IN and/or OUT isn't 4-byte aligned.  Needed even on
+	// ARMv6+, since ldmia and stmia (used below) still require alignment.
+	tst		r10, #3
+	bne		.Lxor_slowpath\@
+
+	// Fast path: XOR 64 bytes of aligned data.
+
+	// Stack: x8-x9 x12-x15 x10-x11 orig_x0-orig_x15 OUT IN LEN
+	// Registers: r0-r7 are x0-x7; r8-r11 are free; r12 is IN; r14 is OUT.
+	// x4-x7 are rotated by 'brot'; x12-x15 are rotated by 'drot'.
+
+	// x0-x3
+	__ldrd		r8, r9, sp, 32
+	__ldrd		r10, r11, sp, 40
+	add		X0, X0, r8
+	add		X1, X1, r9
+	add		X2, X2, r10
+	add		X3, X3, r11
+	_le32_bswap_4x	X0, X1, X2, X3,  r8, r9, r10
+	ldmia		r12!, {r8-r11}
+	eor		X0, X0, r8
+	eor		X1, X1, r9
+	eor		X2, X2, r10
+	eor		X3, X3, r11
+	stmia		r14!, {X0-X3}
+
+	// x4-x7
+	__ldrd		r8, r9, sp, 48
+	__ldrd		r10, r11, sp, 56
+	add		X4, r8, X4, ror #brot
+	add		X5, r9, X5, ror #brot
+	ldmia		r12!, {X0-X3}
+	add		X6, r10, X6, ror #brot
+	add		X7, r11, X7, ror #brot
+	_le32_bswap_4x	X4, X5, X6, X7,  r8, r9, r10
+	eor		X4, X4, X0
+	eor		X5, X5, X1
+	eor		X6, X6, X2
+	eor		X7, X7, X3
+	stmia		r14!, {X4-X7}
+
+	// x8-x15
+	pop		{r0-r7}			// (x8-x9,x12-x15,x10-x11)
+	__ldrd		r8, r9, sp, 32
+	__ldrd		r10, r11, sp, 40
+	add		r0, r0, r8		// x8
+	add		r1, r1, r9		// x9
+	add		r6, r6, r10		// x10
+	add		r7, r7, r11		// x11
+	_le32_bswap_4x	r0, r1, r6, r7,  r8, r9, r10
+	ldmia		r12!, {r8-r11}
+	eor		r0, r0, r8		// x8
+	eor		r1, r1, r9		// x9
+	eor		r6, r6, r10		// x10
+	eor		r7, r7, r11		// x11
+	stmia		r14!, {r0,r1,r6,r7}
+	ldmia		r12!, {r0,r1,r6,r7}
+	__ldrd		r8, r9, sp, 48
+	__ldrd		r10, r11, sp, 56
+	add		r2, r8, r2, ror #drot	// x12
+	add		r3, r9, r3, ror #drot	// x13
+	add		r4, r10, r4, ror #drot	// x14
+	add		r5, r11, r5, ror #drot	// x15
+	_le32_bswap_4x	r2, r3, r4, r5,  r9, r10, r11
+	  ldr		r9, [sp, #72]		// load LEN
+	eor		r2, r2, r0		// x12
+	eor		r3, r3, r1		// x13
+	eor		r4, r4, r6		// x14
+	eor		r5, r5, r7		// x15
+	  subs		r9, #64			// decrement and check LEN
+	stmia		r14!, {r2-r5}
+
+	beq		.Ldone\@
+
+.Lprepare_for_next_block\@:
+
+	// Stack: x0-x15 OUT IN LEN
+
+	// Increment block counter (x12)
+	add		r8, #1
+
+	// Store updated (OUT, IN, LEN)
+	str		r14, [sp, #64]
+	str		r12, [sp, #68]
+	str		r9, [sp, #72]
+
+	  mov		r14, sp
+
+	// Store updated block counter (x12)
+	str		r8, [sp, #48]
+
+	  sub		sp, #16
+
+	// Reload state and do next block
+	ldmia		r14!, {r0-r11}		// load x0-x11
+	__strd		r10, r11, sp, 8		// store x10-x11 before state
+	ldmia		r14, {r10-r12,r14}	// load x12-x15
+	b		.Lnext_block\@
+
+.Lxor_slowpath\@:
+	// Slow path: < 64 bytes remaining, or unaligned input or output buffer.
+	// We handle it by storing the 64 bytes of keystream to the stack, then
+	// XOR-ing the needed portion with the data.
+
+	// Allocate keystream buffer
+	sub		sp, #64
+	mov		r14, sp
+
+	// Stack: ks0-ks15 x8-x9 x12-x15 x10-x11 orig_x0-orig_x15 OUT IN LEN
+	// Registers: r0-r7 are x0-x7; r8-r11 are free; r12 is IN; r14 is &ks0.
+	// x4-x7 are rotated by 'brot'; x12-x15 are rotated by 'drot'.
+
+	// Save keystream for x0-x3
+	__ldrd		r8, r9, sp, 96
+	__ldrd		r10, r11, sp, 104
+	add		X0, X0, r8
+	add		X1, X1, r9
+	add		X2, X2, r10
+	add		X3, X3, r11
+	_le32_bswap_4x	X0, X1, X2, X3,  r8, r9, r10
+	stmia		r14!, {X0-X3}
+
+	// Save keystream for x4-x7
+	__ldrd		r8, r9, sp, 112
+	__ldrd		r10, r11, sp, 120
+	add		X4, r8, X4, ror #brot
+	add		X5, r9, X5, ror #brot
+	add		X6, r10, X6, ror #brot
+	add		X7, r11, X7, ror #brot
+	_le32_bswap_4x	X4, X5, X6, X7,  r8, r9, r10
+	  add		r8, sp, #64
+	stmia		r14!, {X4-X7}
+
+	// Save keystream for x8-x15
+	ldm		r8, {r0-r7}		// (x8-x9,x12-x15,x10-x11)
+	__ldrd		r8, r9, sp, 128
+	__ldrd		r10, r11, sp, 136
+	add		r0, r0, r8		// x8
+	add		r1, r1, r9		// x9
+	add		r6, r6, r10		// x10
+	add		r7, r7, r11		// x11
+	_le32_bswap_4x	r0, r1, r6, r7,  r8, r9, r10
+	stmia		r14!, {r0,r1,r6,r7}
+	__ldrd		r8, r9, sp, 144
+	__ldrd		r10, r11, sp, 152
+	add		r2, r8, r2, ror #drot	// x12
+	add		r3, r9, r3, ror #drot	// x13
+	add		r4, r10, r4, ror #drot	// x14
+	add		r5, r11, r5, ror #drot	// x15
+	_le32_bswap_4x	r2, r3, r4, r5,  r9, r10, r11
+	stmia		r14, {r2-r5}
+
+	// Stack: ks0-ks15 unused0-unused7 x0-x15 OUT IN LEN
+	// Registers: r8 is block counter, r12 is IN.
+
+	ldr		r9, [sp, #168]		// LEN
+	ldr		r14, [sp, #160]		// OUT
+	cmp		r9, #64
+	  mov		r0, sp
+	movle		r1, r9
+	movgt		r1, #64
+	// r1 is number of bytes to XOR, in range [1, 64]
+
+.if __LINUX_ARM_ARCH__ < 6
+	orr		r2, r12, r14
+	tst		r2, #3			// IN or OUT misaligned?
+	bne		.Lxor_next_byte\@
+.endif
+
+	// XOR a word at a time
+.rept 16
+	subs		r1, #4
+	blt		.Lxor_words_done\@
+	ldr		r2, [r12], #4
+	ldr		r3, [r0], #4
+	eor		r2, r2, r3
+	str		r2, [r14], #4
+.endr
+	b		.Lxor_slowpath_done\@
+.Lxor_words_done\@:
+	ands		r1, r1, #3
+	beq		.Lxor_slowpath_done\@
+
+	// XOR a byte at a time
+.Lxor_next_byte\@:
+	ldrb		r2, [r12], #1
+	ldrb		r3, [r0], #1
+	eor		r2, r2, r3
+	strb		r2, [r14], #1
+	subs		r1, #1
+	bne		.Lxor_next_byte\@
+
+.Lxor_slowpath_done\@:
+	subs		r9, #64
+	add		sp, #96
+	bgt		.Lprepare_for_next_block\@
+
+.Ldone\@:
+.endm	// _chacha
+
+/*
+ * void chacha20_arm(u8 *out, const u8 *in, size_t len, const u32 key[8],
+ *		     const u32 iv[4]);
+ */
+SYM_FUNC_START(chacha20_arm)
+	cmp		r2, #0			// len == 0?
+	reteq		lr
+
+	push		{r0-r2,r4-r11,lr}
+
+	// Push state x0-x15 onto stack.
+	// Also store an extra copy of x10-x11 just before the state.
+
+	ldr		r4, [sp, #48]		// iv
+	mov		r0, sp
+	sub		sp, #80
+
+	// iv: x12-x15
+	ldm		r4, {X12,X13,X14,X15}
+	stmdb		r0!, {X12,X13,X14,X15}
+
+	// key: x4-x11
+	__ldrd		X8_X10, X9_X11, r3, 24
+	__strd		X8_X10, X9_X11, sp, 8
+	stmdb		r0!, {X8_X10, X9_X11}
+	ldm		r3, {X4-X9_X11}
+	stmdb		r0!, {X4-X9_X11}
+
+	// constants: x0-x3
+	adrl		X3, .Lexpand_32byte_k
+	ldm		X3, {X0-X3}
+	__strd		X0, X1, sp, 16
+	__strd		X2, X3, sp, 24
+
+	_chacha		20
+
+	add		sp, #76
+	pop		{r4-r11, pc}
+SYM_FUNC_END(chacha20_arm)
+
+/*
+ * void hchacha20_arm(const u32 state[16], u32 out[8]);
+ */
+SYM_FUNC_START(hchacha20_arm)
+	push		{r1,r4-r11,lr}
+
+	mov		r14, r0
+	ldmia		r14!, {r0-r11}		// load x0-x11
+	push		{r10-r11}		// store x10-x11 to stack
+	ldm		r14, {r10-r12,r14}	// load x12-x15
+	sub		sp, #8
+
+	_chacha_permute	20
+
+	// Skip over (unused0-unused1, x10-x11)
+	add		sp, #16
+
+	// Fix up rotations of x12-x15
+	ror		X12, X12, #drot
+	ror		X13, X13, #drot
+	  pop		{r4}			// load 'out'
+	ror		X14, X14, #drot
+	ror		X15, X15, #drot
+
+	// Store (x0-x3,x12-x15) to 'out'
+	stm		r4, {X0,X1,X2,X3,X12,X13,X14,X15}
+
+	pop		{r4-r11,pc}
+SYM_FUNC_END(hchacha20_arm)
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/chacha20/chacha20-x86_64-glue.c b/net/wireguard/crypto/zinc/chacha20/chacha20-x86_64-glue.c
--- a/net/wireguard/crypto/zinc/chacha20/chacha20-x86_64-glue.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/chacha20/chacha20-x86_64-glue.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,105 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#include <asm/fpu/api.h>
+#include <asm/cpufeature.h>
+#include <asm/processor.h>
+#include <asm/intel-family.h>
+
+asmlinkage void hchacha20_ssse3(u32 *derived_key, const u8 *nonce,
+				const u8 *key);
+asmlinkage void chacha20_ssse3(u8 *out, const u8 *in, const size_t len,
+			       const u32 key[8], const u32 counter[4]);
+asmlinkage void chacha20_avx2(u8 *out, const u8 *in, const size_t len,
+			      const u32 key[8], const u32 counter[4]);
+asmlinkage void chacha20_avx512(u8 *out, const u8 *in, const size_t len,
+				const u32 key[8], const u32 counter[4]);
+asmlinkage void chacha20_avx512vl(u8 *out, const u8 *in, const size_t len,
+				  const u32 key[8], const u32 counter[4]);
+
+static bool chacha20_use_ssse3 __ro_after_init;
+static bool chacha20_use_avx2 __ro_after_init;
+static bool chacha20_use_avx512 __ro_after_init;
+static bool chacha20_use_avx512vl __ro_after_init;
+static bool *const chacha20_nobs[] __initconst = {
+	&chacha20_use_ssse3, &chacha20_use_avx2, &chacha20_use_avx512,
+	&chacha20_use_avx512vl };
+
+static void __init chacha20_fpu_init(void)
+{
+	chacha20_use_ssse3 = boot_cpu_has(X86_FEATURE_SSSE3);
+	chacha20_use_avx2 =
+		boot_cpu_has(X86_FEATURE_AVX) &&
+		boot_cpu_has(X86_FEATURE_AVX2) &&
+		cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL);
+#ifndef COMPAT_CANNOT_USE_AVX512
+	chacha20_use_avx512 =
+		boot_cpu_has(X86_FEATURE_AVX) &&
+		boot_cpu_has(X86_FEATURE_AVX2) &&
+		boot_cpu_has(X86_FEATURE_AVX512F) &&
+		cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM |
+				  XFEATURE_MASK_AVX512, NULL) &&
+		/* Skylake downclocks unacceptably much when using zmm. */
+		boot_cpu_data.x86_model != INTEL_FAM6_SKYLAKE_X;
+	chacha20_use_avx512vl =
+		boot_cpu_has(X86_FEATURE_AVX) &&
+		boot_cpu_has(X86_FEATURE_AVX2) &&
+		boot_cpu_has(X86_FEATURE_AVX512F) &&
+		boot_cpu_has(X86_FEATURE_AVX512VL) &&
+		cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM |
+				  XFEATURE_MASK_AVX512, NULL);
+#endif
+}
+
+static inline bool chacha20_arch(struct chacha20_ctx *ctx, u8 *dst,
+				 const u8 *src, size_t len,
+				 simd_context_t *simd_context)
+{
+	/* SIMD disables preemption, so relax after processing each page. */
+	BUILD_BUG_ON(PAGE_SIZE < CHACHA20_BLOCK_SIZE ||
+		     PAGE_SIZE % CHACHA20_BLOCK_SIZE);
+
+	if (!IS_ENABLED(CONFIG_AS_SSSE3) || !chacha20_use_ssse3 ||
+	    len <= CHACHA20_BLOCK_SIZE || !simd_use(simd_context))
+		return false;
+
+	for (;;) {
+		const size_t bytes = min_t(size_t, len, PAGE_SIZE);
+
+		if (IS_ENABLED(CONFIG_AS_AVX512) && chacha20_use_avx512 &&
+		    len >= CHACHA20_BLOCK_SIZE * 8)
+			chacha20_avx512(dst, src, bytes, ctx->key, ctx->counter);
+		else if (IS_ENABLED(CONFIG_AS_AVX512) && chacha20_use_avx512vl &&
+			 len >= CHACHA20_BLOCK_SIZE * 4)
+			chacha20_avx512vl(dst, src, bytes, ctx->key, ctx->counter);
+		else if (IS_ENABLED(CONFIG_AS_AVX2) && chacha20_use_avx2 &&
+			 len >= CHACHA20_BLOCK_SIZE * 4)
+			chacha20_avx2(dst, src, bytes, ctx->key, ctx->counter);
+		else
+			chacha20_ssse3(dst, src, bytes, ctx->key, ctx->counter);
+		ctx->counter[0] += (bytes + 63) / 64;
+		len -= bytes;
+		if (!len)
+			break;
+		dst += bytes;
+		src += bytes;
+		simd_relax(simd_context);
+	}
+
+	return true;
+}
+
+static inline bool hchacha20_arch(u32 derived_key[CHACHA20_KEY_WORDS],
+				  const u8 nonce[HCHACHA20_NONCE_SIZE],
+				  const u8 key[HCHACHA20_KEY_SIZE],
+				  simd_context_t *simd_context)
+{
+	if (IS_ENABLED(CONFIG_AS_SSSE3) && chacha20_use_ssse3 &&
+	    simd_use(simd_context)) {
+		hchacha20_ssse3(derived_key, nonce, key);
+		return true;
+	}
+	return false;
+}
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/chacha20/chacha20-x86_64.pl b/net/wireguard/crypto/zinc/chacha20/chacha20-x86_64.pl
--- a/net/wireguard/crypto/zinc/chacha20/chacha20-x86_64.pl	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/chacha20/chacha20-x86_64.pl	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,4106 @@
+#!/usr/bin/env perl
+# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+#
+# Copyright (C) 2017-2019 Samuel Neves <sneves@dei.uc.pt>. All Rights Reserved.
+# Copyright (C) 2017-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+# Copyright (C) 2006-2017 CRYPTOGAMS by <appro@openssl.org>. All Rights Reserved.
+#
+# This code is taken from the OpenSSL project but the author, Andy Polyakov,
+# has relicensed it under the licenses specified in the SPDX header above.
+# The original headers, including the original license headers, are
+# included below for completeness.
+#
+# ====================================================================
+# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
+# project. The module is, however, dual licensed under OpenSSL and
+# CRYPTOGAMS licenses depending on where you obtain it. For further
+# details see http://www.openssl.org/~appro/cryptogams/.
+# ====================================================================
+#
+# November 2014
+#
+# ChaCha20 for x86_64.
+#
+# December 2016
+#
+# Add AVX512F code path.
+#
+# December 2017
+#
+# Add AVX512VL code path.
+#
+# Performance in cycles per byte out of large buffer.
+#
+#		IALU/gcc 4.8(i)	1x/2xSSSE3(ii)	4xSSSE3	    NxAVX(v)
+#
+# P4		9.48/+99%	-		-
+# Core2		7.83/+55%	7.90/5.76	4.35
+# Westmere	7.19/+50%	5.60/4.50	3.00
+# Sandy Bridge	8.31/+42%	5.45/4.00	2.72
+# Ivy Bridge	6.71/+46%	5.40/?		2.41
+# Haswell	5.92/+43%	5.20/3.45	2.42        1.23
+# Skylake[-X]	5.87/+39%	4.70/3.22	2.31        1.19[0.80(vi)]
+# Silvermont	12.0/+33%	7.75/6.90	7.03(iii)
+# Knights L	11.7/-		?		9.60(iii)   0.80
+# Goldmont	10.6/+17%	5.10/3.52	3.28
+# Sledgehammer	7.28/+52%	-		-
+# Bulldozer	9.66/+28%	9.85/5.35(iv)	3.06(iv)
+# Ryzen		5.96/+50%	5.19/3.00	2.40        2.09
+# VIA Nano	10.5/+46%	6.72/6.88	6.05
+#
+# (i)	compared to older gcc 3.x one can observe >2x improvement on
+#	most platforms;
+# (ii)	2xSSSE3 is code path optimized specifically for 128 bytes used
+#	by chacha20_poly1305_tls_cipher, results are EVP-free;
+# (iii)	this is not optimal result for Atom because of MSROM
+#	limitations, SSE2 can do better, but gain is considered too
+#	low to justify the [maintenance] effort;
+# (iv)	Bulldozer actually executes 4xXOP code path that delivers 2.20
+#	and 4.85 for 128-byte inputs;
+# (v)	8xAVX2, 8xAVX512VL or 16xAVX512F, whichever best applicable;
+# (vi)	even though Skylake-X can execute AVX512F code and deliver 0.57
+#	cpb in single thread, the corresponding capability is suppressed;
+
+$flavour = shift;
+$output  = shift;
+if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
+
+$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
+$kernel=0; $kernel=1 if (!$flavour && !$output);
+
+if (!$kernel) {
+	$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
+	( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
+	( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
+	die "can't locate x86_64-xlate.pl";
+
+	open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
+	*STDOUT=*OUT;
+
+	if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
+	    =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
+		$avx = ($1>=2.19) + ($1>=2.22) + ($1>=2.25);
+	}
+
+	if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
+	    `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) {
+		$avx = ($1>=2.09) + ($1>=2.10) + ($1>=2.12);
+		$avx += 1 if ($1==2.11 && $2>=8);
+	}
+
+	if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
+	    `ml64 2>&1` =~ /Version ([0-9]+)\./) {
+		$avx = ($1>=10) + ($1>=11);
+	}
+
+	if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9]\.[0-9]+)/) {
+		$avx = ($2>=3.0) + ($2>3.0);
+	}
+} else {
+	$avx = 4; # The kernel uses ifdefs for this.
+}
+
+# input parameter block
+($out,$inp,$len,$key,$counter)=("%rdi","%rsi","%rdx","%rcx","%r8");
+
+$code.=<<___ if $kernel;
+#include <linux/linkage.h>
+___
+
+sub declare_variable() {
+	my ($name, $size, $type, $payload) = @_;
+	if($kernel) {
+		$code.=".section .rodata.cst$size.L$name, \"aM\", \@progbits, $size\n";
+		$code.=".align $size\n";
+		$code.=".L$name:\n";
+		$code.=".$type $payload\n";
+	} else {
+		$code.=".L$name:\n";
+		$code.=".$type $payload\n";
+	}
+}
+
+sub declare_function() {
+	my ($name, $align, $nargs) = @_;
+	if($kernel) {
+		$code .= ".align $align\n";
+		$code .= "SYM_FUNC_START($name)\n";
+		$code .= ".L$name:\n";
+	} else {
+		$code .= ".globl	$name\n";
+		$code .= ".type	$name,\@function,$nargs\n";
+		$code .= ".align	$align\n";
+		$code .= "$name:\n";
+	}
+}
+
+sub end_function() {
+	my ($name) = @_;
+	if($kernel) {
+		$code .= "SYM_FUNC_END($name)\n";
+	} else {
+		$code .= ".size   $name,.-$name\n";
+	}
+}
+
+if(!$kernel) {
+	$code .= ".text\n";
+}
+&declare_variable('zero', 16, 'long', '0,0,0,0');
+&declare_variable('one', 16, 'long', '1,0,0,0');
+&declare_variable('inc', 16, 'long', '0,1,2,3');
+&declare_variable('four', 16, 'long', '4,4,4,4');
+&declare_variable('incy', 32, 'long', '0,2,4,6,1,3,5,7');
+&declare_variable('eight', 32, 'long', '8,8,8,8,8,8,8,8');
+&declare_variable('rot16', 16, 'byte', '0x2,0x3,0x0,0x1, 0x6,0x7,0x4,0x5, 0xa,0xb,0x8,0x9, 0xe,0xf,0xc,0xd');
+&declare_variable('rot24', 16, 'byte', '0x3,0x0,0x1,0x2, 0x7,0x4,0x5,0x6, 0xb,0x8,0x9,0xa, 0xf,0xc,0xd,0xe');
+&declare_variable('twoy', 32, 'long', '2,0,0,0, 2,0,0,0');
+&declare_variable('zeroz', 64, 'long', '0,0,0,0, 1,0,0,0, 2,0,0,0, 3,0,0,0');
+&declare_variable('fourz', 64, 'long', '4,0,0,0, 4,0,0,0, 4,0,0,0, 4,0,0,0');
+&declare_variable('incz', 64, 'long', '0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15');
+&declare_variable('sixteen', 64, 'long', '16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16');
+&declare_variable('sigma', 16, 'ascii', '"expand 32-byte k"');
+
+$code.=<<___ if !$kernel;
+.asciz "ChaCha20 for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
+___
+$code.=".text\n";
+
+sub AUTOLOAD()          # thunk [simplified] 32-bit style perlasm
+{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://;
+  my $arg = pop;
+    $arg = "\$$arg" if ($arg*1 eq $arg);
+    $code .= "\t$opcode\t".join(',',$arg,reverse @_)."\n";
+}
+
+@x=("%eax","%ebx","%ecx","%edx",map("%r${_}d",(8..11)),
+    "%nox","%nox","%nox","%nox",map("%r${_}d",(12..15)));
+@t=("%esi","%edi");
+
+sub ROUND {			# critical path is 24 cycles per round
+my ($a0,$b0,$c0,$d0)=@_;
+my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
+my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
+my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
+my ($xc,$xc_)=map("\"$_\"",@t);
+my @x=map("\"$_\"",@x);
+
+	# Consider order in which variables are addressed by their
+	# index:
+	#
+	#	a   b   c   d
+	#
+	#	0   4   8  12 < even round
+	#	1   5   9  13
+	#	2   6  10  14
+	#	3   7  11  15
+	#	0   5  10  15 < odd round
+	#	1   6  11  12
+	#	2   7   8  13
+	#	3   4   9  14
+	#
+	# 'a', 'b' and 'd's are permanently allocated in registers,
+	# @x[0..7,12..15], while 'c's are maintained in memory. If
+	# you observe 'c' column, you'll notice that pair of 'c's is
+	# invariant between rounds. This means that we have to reload
+	# them once per round, in the middle. This is why you'll see
+	# bunch of 'c' stores and loads in the middle, but none in
+	# the beginning or end.
+
+	# Normally instructions would be interleaved to favour in-order
+	# execution. Generally out-of-order cores manage it gracefully,
+	# but not this time for some reason. As in-order execution
+	# cores are dying breed, old Atom is the only one around,
+	# instructions are left uninterleaved. Besides, Atom is better
+	# off executing 1xSSSE3 code anyway...
+
+	(
+	"&add	(@x[$a0],@x[$b0])",	# Q1
+	"&xor	(@x[$d0],@x[$a0])",
+	"&rol	(@x[$d0],16)",
+	 "&add	(@x[$a1],@x[$b1])",	# Q2
+	 "&xor	(@x[$d1],@x[$a1])",
+	 "&rol	(@x[$d1],16)",
+
+	"&add	($xc,@x[$d0])",
+	"&xor	(@x[$b0],$xc)",
+	"&rol	(@x[$b0],12)",
+	 "&add	($xc_,@x[$d1])",
+	 "&xor	(@x[$b1],$xc_)",
+	 "&rol	(@x[$b1],12)",
+
+	"&add	(@x[$a0],@x[$b0])",
+	"&xor	(@x[$d0],@x[$a0])",
+	"&rol	(@x[$d0],8)",
+	 "&add	(@x[$a1],@x[$b1])",
+	 "&xor	(@x[$d1],@x[$a1])",
+	 "&rol	(@x[$d1],8)",
+
+	"&add	($xc,@x[$d0])",
+	"&xor	(@x[$b0],$xc)",
+	"&rol	(@x[$b0],7)",
+	 "&add	($xc_,@x[$d1])",
+	 "&xor	(@x[$b1],$xc_)",
+	 "&rol	(@x[$b1],7)",
+
+	"&mov	(\"4*$c0(%rsp)\",$xc)",	# reload pair of 'c's
+	 "&mov	(\"4*$c1(%rsp)\",$xc_)",
+	"&mov	($xc,\"4*$c2(%rsp)\")",
+	 "&mov	($xc_,\"4*$c3(%rsp)\")",
+
+	"&add	(@x[$a2],@x[$b2])",	# Q3
+	"&xor	(@x[$d2],@x[$a2])",
+	"&rol	(@x[$d2],16)",
+	 "&add	(@x[$a3],@x[$b3])",	# Q4
+	 "&xor	(@x[$d3],@x[$a3])",
+	 "&rol	(@x[$d3],16)",
+
+	"&add	($xc,@x[$d2])",
+	"&xor	(@x[$b2],$xc)",
+	"&rol	(@x[$b2],12)",
+	 "&add	($xc_,@x[$d3])",
+	 "&xor	(@x[$b3],$xc_)",
+	 "&rol	(@x[$b3],12)",
+
+	"&add	(@x[$a2],@x[$b2])",
+	"&xor	(@x[$d2],@x[$a2])",
+	"&rol	(@x[$d2],8)",
+	 "&add	(@x[$a3],@x[$b3])",
+	 "&xor	(@x[$d3],@x[$a3])",
+	 "&rol	(@x[$d3],8)",
+
+	"&add	($xc,@x[$d2])",
+	"&xor	(@x[$b2],$xc)",
+	"&rol	(@x[$b2],7)",
+	 "&add	($xc_,@x[$d3])",
+	 "&xor	(@x[$b3],$xc_)",
+	 "&rol	(@x[$b3],7)"
+	);
+}
+
+########################################################################
+# Generic code path that handles all lengths on pre-SSSE3 processors.
+if(!$kernel) {
+&declare_function("chacha20_ctr32", 64, 5);
+$code.=<<___;
+.cfi_startproc
+	cmp	\$0,$len
+	je	.Lno_data
+	mov	OPENSSL_ia32cap_P+4(%rip),%r9
+___
+$code.=<<___	if ($avx>2);
+	bt	\$48,%r9		# check for AVX512F
+	jc	.Lchacha20_avx512
+	test	%r9,%r9		# check for AVX512VL
+	js	.Lchacha20_avx512vl
+___
+$code.=<<___;
+	test	\$`1<<(41-32)`,%r9d
+	jnz	.Lchacha20_ssse3
+___
+$code.=<<___;
+	push	%rbx
+.cfi_push	%rbx
+	push	%rbp
+.cfi_push	%rbp
+	push	%r12
+.cfi_push	%r12
+	push	%r13
+.cfi_push	%r13
+	push	%r14
+.cfi_push	%r14
+	push	%r15
+.cfi_push	%r15
+	sub	\$64+24,%rsp
+.cfi_adjust_cfa_offset	64+24
+.Lctr32_body:
+
+	#movdqa	.Lsigma(%rip),%xmm0
+	movdqu	($key),%xmm1
+	movdqu	16($key),%xmm2
+	movdqu	($counter),%xmm3
+	movdqa	.Lone(%rip),%xmm4
+
+	#movdqa	%xmm0,4*0(%rsp)		# key[0]
+	movdqa	%xmm1,4*4(%rsp)		# key[1]
+	movdqa	%xmm2,4*8(%rsp)		# key[2]
+	movdqa	%xmm3,4*12(%rsp)	# key[3]
+	mov	$len,%rbp		# reassign $len
+	jmp	.Loop_outer
+
+.align	32
+.Loop_outer:
+	mov	\$0x61707865,@x[0]      # 'expa'
+	mov	\$0x3320646e,@x[1]      # 'nd 3'
+	mov	\$0x79622d32,@x[2]      # '2-by'
+	mov	\$0x6b206574,@x[3]      # 'te k'
+	mov	4*4(%rsp),@x[4]
+	mov	4*5(%rsp),@x[5]
+	mov	4*6(%rsp),@x[6]
+	mov	4*7(%rsp),@x[7]
+	movd	%xmm3,@x[12]
+	mov	4*13(%rsp),@x[13]
+	mov	4*14(%rsp),@x[14]
+	mov	4*15(%rsp),@x[15]
+
+	mov	%rbp,64+0(%rsp)		# save len
+	mov	\$10,%ebp
+	mov	$inp,64+8(%rsp)		# save inp
+	movq	%xmm2,%rsi		# "@x[8]"
+	mov	$out,64+16(%rsp)	# save out
+	mov	%rsi,%rdi
+	shr	\$32,%rdi		# "@x[9]"
+	jmp	.Loop
+
+.align	32
+.Loop:
+___
+	foreach (&ROUND (0, 4, 8,12)) { eval; }
+	foreach (&ROUND	(0, 5,10,15)) { eval; }
+	&dec	("%ebp");
+	&jnz	(".Loop");
+
+$code.=<<___;
+	mov	@t[1],4*9(%rsp)		# modulo-scheduled
+	mov	@t[0],4*8(%rsp)
+	mov	64(%rsp),%rbp		# load len
+	movdqa	%xmm2,%xmm1
+	mov	64+8(%rsp),$inp		# load inp
+	paddd	%xmm4,%xmm3		# increment counter
+	mov	64+16(%rsp),$out	# load out
+
+	add	\$0x61707865,@x[0]      # 'expa'
+	add	\$0x3320646e,@x[1]      # 'nd 3'
+	add	\$0x79622d32,@x[2]      # '2-by'
+	add	\$0x6b206574,@x[3]      # 'te k'
+	add	4*4(%rsp),@x[4]
+	add	4*5(%rsp),@x[5]
+	add	4*6(%rsp),@x[6]
+	add	4*7(%rsp),@x[7]
+	add	4*12(%rsp),@x[12]
+	add	4*13(%rsp),@x[13]
+	add	4*14(%rsp),@x[14]
+	add	4*15(%rsp),@x[15]
+	paddd	4*8(%rsp),%xmm1
+
+	cmp	\$64,%rbp
+	jb	.Ltail
+
+	xor	4*0($inp),@x[0]		# xor with input
+	xor	4*1($inp),@x[1]
+	xor	4*2($inp),@x[2]
+	xor	4*3($inp),@x[3]
+	xor	4*4($inp),@x[4]
+	xor	4*5($inp),@x[5]
+	xor	4*6($inp),@x[6]
+	xor	4*7($inp),@x[7]
+	movdqu	4*8($inp),%xmm0
+	xor	4*12($inp),@x[12]
+	xor	4*13($inp),@x[13]
+	xor	4*14($inp),@x[14]
+	xor	4*15($inp),@x[15]
+	lea	4*16($inp),$inp		# inp+=64
+	pxor	%xmm1,%xmm0
+
+	movdqa	%xmm2,4*8(%rsp)
+	movd	%xmm3,4*12(%rsp)
+
+	mov	@x[0],4*0($out)		# write output
+	mov	@x[1],4*1($out)
+	mov	@x[2],4*2($out)
+	mov	@x[3],4*3($out)
+	mov	@x[4],4*4($out)
+	mov	@x[5],4*5($out)
+	mov	@x[6],4*6($out)
+	mov	@x[7],4*7($out)
+	movdqu	%xmm0,4*8($out)
+	mov	@x[12],4*12($out)
+	mov	@x[13],4*13($out)
+	mov	@x[14],4*14($out)
+	mov	@x[15],4*15($out)
+	lea	4*16($out),$out		# out+=64
+
+	sub	\$64,%rbp
+	jnz	.Loop_outer
+
+	jmp	.Ldone
+
+.align	16
+.Ltail:
+	mov	@x[0],4*0(%rsp)
+	mov	@x[1],4*1(%rsp)
+	xor	%rbx,%rbx
+	mov	@x[2],4*2(%rsp)
+	mov	@x[3],4*3(%rsp)
+	mov	@x[4],4*4(%rsp)
+	mov	@x[5],4*5(%rsp)
+	mov	@x[6],4*6(%rsp)
+	mov	@x[7],4*7(%rsp)
+	movdqa	%xmm1,4*8(%rsp)
+	mov	@x[12],4*12(%rsp)
+	mov	@x[13],4*13(%rsp)
+	mov	@x[14],4*14(%rsp)
+	mov	@x[15],4*15(%rsp)
+
+.Loop_tail:
+	movzb	($inp,%rbx),%eax
+	movzb	(%rsp,%rbx),%edx
+	lea	1(%rbx),%rbx
+	xor	%edx,%eax
+	mov	%al,-1($out,%rbx)
+	dec	%rbp
+	jnz	.Loop_tail
+
+.Ldone:
+	add	\$64+24,%rsp
+.cfi_adjust_cfa_offset	-64-24
+	pop			%r15
+.cfi_restore	%r15
+	pop			%r14
+.cfi_restore	%r14
+	pop			%r13
+.cfi_restore	%r13
+	pop			%r12
+.cfi_restore	%r12
+	pop			%rbp
+.cfi_restore	%rbp
+	pop			%rbx
+.cfi_restore	%rbx
+.Lno_data:
+	ret
+.cfi_endproc
+___
+&end_function("chacha20_ctr32");
+}
+
+########################################################################
+# SSSE3 code path that handles shorter lengths
+{
+my ($a,$b,$c,$d,$t,$t1,$rot16,$rot24)=map("%xmm$_",(0..7));
+
+sub SSSE3ROUND {	# critical path is 20 "SIMD ticks" per round
+	&paddd	($a,$b);
+	&pxor	($d,$a);
+	&pshufb	($d,$rot16);
+
+	&paddd	($c,$d);
+	&pxor	($b,$c);
+	&movdqa	($t,$b);
+	&psrld	($b,20);
+	&pslld	($t,12);
+	&por	($b,$t);
+
+	&paddd	($a,$b);
+	&pxor	($d,$a);
+	&pshufb	($d,$rot24);
+
+	&paddd	($c,$d);
+	&pxor	($b,$c);
+	&movdqa	($t,$b);
+	&psrld	($b,25);
+	&pslld	($t,7);
+	&por	($b,$t);
+}
+
+my $xframe = $win64 ? 32+8 : 8;
+
+if($kernel) {
+	$code .= "#ifdef CONFIG_AS_SSSE3\n";
+}
+
+if($kernel) {
+&declare_function("hchacha20_ssse3", 32, 5);
+$code.=<<___;
+	movdqa	.Lsigma(%rip),$a
+	movdqu	($len),$b
+	movdqu	16($len),$c
+	movdqu	($inp),$d
+	# This code is only used when targeting kernel.
+	# If targeting win64, xmm{6,7} preserving needs to be added.
+	movdqa	.Lrot16(%rip),$rot16
+	movdqa	.Lrot24(%rip),$rot24
+	mov	\$10,$counter		# reuse $counter
+	jmp	1f
+.align	32
+1:
+___
+	&SSSE3ROUND();
+	&pshufd	($a,$a,0b10010011);
+	&pshufd	($d,$d,0b01001110);
+	&pshufd	($c,$c,0b00111001);
+	&nop	();
+
+	&SSSE3ROUND();
+	&pshufd	($a,$a,0b00111001);
+	&pshufd	($d,$d,0b01001110);
+	&pshufd	($c,$c,0b10010011);
+
+	&dec	($counter);
+	&jnz	("1b");
+
+$code.=<<___;
+	movdqu $a, ($out)
+	movdqu $d, 16($out)
+	ret
+___
+&end_function("hchacha20_ssse3");
+}
+
+&declare_function("chacha20_ssse3", 32, 5);
+$code.=<<___;
+.cfi_startproc
+	lea	8(%rsp),%r10		# frame pointer
+.cfi_def_cfa_register	%r10
+___
+$code.=<<___	if ($avx && !$kernel);
+	test	\$`1<<(43-32)`,%r10d
+	jnz	.Lchacha20_4xop		# XOP is fastest even if we use 1/4
+___
+$code.=<<___;
+	cmp	\$128,$len		# we might throw away some data,
+	je	.Lchacha20_128
+	ja	.Lchacha20_4x		# but overall it won't be slower
+
+.Ldo_ssse3_after_all:
+	sub	\$64+$xframe,%rsp
+	and \$-16,%rsp
+___
+$code.=<<___	if ($win64);
+	movaps	%xmm6,-0x30(%r10)
+	movaps	%xmm7,-0x20(%r10)
+.Lssse3_body:
+___
+$code.=<<___;
+	movdqa	.Lsigma(%rip),$a
+	movdqu	($key),$b
+	movdqu	16($key),$c
+	movdqu	($counter),$d
+	movdqa	.Lrot16(%rip),$rot16
+	movdqa	.Lrot24(%rip),$rot24
+
+	movdqa	$a,0x00(%rsp)
+	movdqa	$b,0x10(%rsp)
+	movdqa	$c,0x20(%rsp)
+	movdqa	$d,0x30(%rsp)
+	mov	\$10,$counter		# reuse $counter
+	jmp	.Loop_ssse3
+
+.align	32
+.Loop_outer_ssse3:
+	movdqa	.Lone(%rip),$d
+	movdqa	0x00(%rsp),$a
+	movdqa	0x10(%rsp),$b
+	movdqa	0x20(%rsp),$c
+	paddd	0x30(%rsp),$d
+	mov	\$10,$counter
+	movdqa	$d,0x30(%rsp)
+	jmp	.Loop_ssse3
+
+.align	32
+.Loop_ssse3:
+___
+	&SSSE3ROUND();
+	&pshufd	($a,$a,0b10010011);
+	&pshufd	($d,$d,0b01001110);
+	&pshufd	($c,$c,0b00111001);
+	&nop	();
+
+	&SSSE3ROUND();
+	&pshufd	($a,$a,0b00111001);
+	&pshufd	($d,$d,0b01001110);
+	&pshufd	($c,$c,0b10010011);
+
+	&dec	($counter);
+	&jnz	(".Loop_ssse3");
+
+$code.=<<___;
+	paddd	0x00(%rsp),$a
+	paddd	0x10(%rsp),$b
+	paddd	0x20(%rsp),$c
+	paddd	0x30(%rsp),$d
+
+	cmp	\$64,$len
+	jb	.Ltail_ssse3
+
+	movdqu	0x00($inp),$t
+	movdqu	0x10($inp),$t1
+	pxor	$t,$a			# xor with input
+	movdqu	0x20($inp),$t
+	pxor	$t1,$b
+	movdqu	0x30($inp),$t1
+	lea	0x40($inp),$inp		# inp+=64
+	pxor	$t,$c
+	pxor	$t1,$d
+
+	movdqu	$a,0x00($out)		# write output
+	movdqu	$b,0x10($out)
+	movdqu	$c,0x20($out)
+	movdqu	$d,0x30($out)
+	lea	0x40($out),$out		# out+=64
+
+	sub	\$64,$len
+	jnz	.Loop_outer_ssse3
+
+	jmp	.Ldone_ssse3
+
+.align	16
+.Ltail_ssse3:
+	movdqa	$a,0x00(%rsp)
+	movdqa	$b,0x10(%rsp)
+	movdqa	$c,0x20(%rsp)
+	movdqa	$d,0x30(%rsp)
+	xor	$counter,$counter
+
+.Loop_tail_ssse3:
+	movzb	($inp,$counter),%eax
+	movzb	(%rsp,$counter),%ecx
+	lea	1($counter),$counter
+	xor	%ecx,%eax
+	mov	%al,-1($out,$counter)
+	dec	$len
+	jnz	.Loop_tail_ssse3
+
+.Ldone_ssse3:
+___
+$code.=<<___	if ($win64);
+	movaps	-0x30(%r10),%xmm6
+	movaps	-0x20(%r10),%xmm7
+___
+$code.=<<___;
+	lea	-8(%r10),%rsp
+.cfi_def_cfa_register	%rsp
+.Lssse3_epilogue:
+	ret
+.cfi_endproc
+___
+}
+&end_function("chacha20_ssse3");
+
+########################################################################
+# SSSE3 code path that handles 128-byte inputs
+{
+my ($a,$b,$c,$d,$t,$t1,$rot16,$rot24)=map("%xmm$_",(8,9,2..7));
+my ($a1,$b1,$c1,$d1)=map("%xmm$_",(10,11,0,1));
+
+sub SSSE3ROUND_2x {
+	&paddd	($a,$b);
+	&pxor	($d,$a);
+	 &paddd	($a1,$b1);
+	 &pxor	($d1,$a1);
+	&pshufb	($d,$rot16);
+	 &pshufb($d1,$rot16);
+
+	&paddd	($c,$d);
+	 &paddd	($c1,$d1);
+	&pxor	($b,$c);
+	 &pxor	($b1,$c1);
+	&movdqa	($t,$b);
+	&psrld	($b,20);
+	 &movdqa($t1,$b1);
+	&pslld	($t,12);
+	 &psrld	($b1,20);
+	&por	($b,$t);
+	 &pslld	($t1,12);
+	 &por	($b1,$t1);
+
+	&paddd	($a,$b);
+	&pxor	($d,$a);
+	 &paddd	($a1,$b1);
+	 &pxor	($d1,$a1);
+	&pshufb	($d,$rot24);
+	 &pshufb($d1,$rot24);
+
+	&paddd	($c,$d);
+	 &paddd	($c1,$d1);
+	&pxor	($b,$c);
+	 &pxor	($b1,$c1);
+	&movdqa	($t,$b);
+	&psrld	($b,25);
+	 &movdqa($t1,$b1);
+	&pslld	($t,7);
+	 &psrld	($b1,25);
+	&por	($b,$t);
+	 &pslld	($t1,7);
+	 &por	($b1,$t1);
+}
+
+my $xframe = $win64 ? 0x68 : 8;
+
+$code.=<<___;
+.type	chacha20_128,\@function,5
+.align	32
+chacha20_128:
+.cfi_startproc
+.Lchacha20_128:
+	lea	8(%rsp),%r10		# frame pointer
+.cfi_def_cfa_register	%r10
+	sub	\$64+$xframe,%rsp
+	and \$-16,%rsp
+___
+$code.=<<___	if ($win64);
+	movaps	%xmm6,-0x70(%r10)
+	movaps	%xmm7,-0x60(%r10)
+	movaps	%xmm8,-0x50(%r10)
+	movaps	%xmm9,-0x40(%r10)
+	movaps	%xmm10,-0x30(%r10)
+	movaps	%xmm11,-0x20(%r10)
+.L128_body:
+___
+$code.=<<___;
+	movdqa	.Lsigma(%rip),$a
+	movdqu	($key),$b
+	movdqu	16($key),$c
+	movdqu	($counter),$d
+	movdqa	.Lone(%rip),$d1
+	movdqa	.Lrot16(%rip),$rot16
+	movdqa	.Lrot24(%rip),$rot24
+
+	movdqa	$a,$a1
+	movdqa	$a,0x00(%rsp)
+	movdqa	$b,$b1
+	movdqa	$b,0x10(%rsp)
+	movdqa	$c,$c1
+	movdqa	$c,0x20(%rsp)
+	paddd	$d,$d1
+	movdqa	$d,0x30(%rsp)
+	mov	\$10,$counter		# reuse $counter
+	jmp	.Loop_128
+
+.align	32
+.Loop_128:
+___
+	&SSSE3ROUND_2x();
+	&pshufd	($a,$a,0b10010011);
+	&pshufd	($d,$d,0b01001110);
+	&pshufd	($c,$c,0b00111001);
+	&pshufd	($a1,$a1,0b10010011);
+	&pshufd	($d1,$d1,0b01001110);
+	&pshufd	($c1,$c1,0b00111001);
+
+	&SSSE3ROUND_2x();
+	&pshufd	($a,$a,0b00111001);
+	&pshufd	($d,$d,0b01001110);
+	&pshufd	($c,$c,0b10010011);
+	&pshufd	($a1,$a1,0b00111001);
+	&pshufd	($d1,$d1,0b01001110);
+	&pshufd	($c1,$c1,0b10010011);
+
+	&dec	($counter);
+	&jnz	(".Loop_128");
+
+$code.=<<___;
+	paddd	0x00(%rsp),$a
+	paddd	0x10(%rsp),$b
+	paddd	0x20(%rsp),$c
+	paddd	0x30(%rsp),$d
+	paddd	.Lone(%rip),$d1
+	paddd	0x00(%rsp),$a1
+	paddd	0x10(%rsp),$b1
+	paddd	0x20(%rsp),$c1
+	paddd	0x30(%rsp),$d1
+
+	movdqu	0x00($inp),$t
+	movdqu	0x10($inp),$t1
+	pxor	$t,$a			# xor with input
+	movdqu	0x20($inp),$t
+	pxor	$t1,$b
+	movdqu	0x30($inp),$t1
+	pxor	$t,$c
+	movdqu	0x40($inp),$t
+	pxor	$t1,$d
+	movdqu	0x50($inp),$t1
+	pxor	$t,$a1
+	movdqu	0x60($inp),$t
+	pxor	$t1,$b1
+	movdqu	0x70($inp),$t1
+	pxor	$t,$c1
+	pxor	$t1,$d1
+
+	movdqu	$a,0x00($out)		# write output
+	movdqu	$b,0x10($out)
+	movdqu	$c,0x20($out)
+	movdqu	$d,0x30($out)
+	movdqu	$a1,0x40($out)
+	movdqu	$b1,0x50($out)
+	movdqu	$c1,0x60($out)
+	movdqu	$d1,0x70($out)
+___
+$code.=<<___	if ($win64);
+	movaps	-0x70(%r10),%xmm6
+	movaps	-0x60(%r10),%xmm7
+	movaps	-0x50(%r10),%xmm8
+	movaps	-0x40(%r10),%xmm9
+	movaps	-0x30(%r10),%xmm10
+	movaps	-0x20(%r10),%xmm11
+___
+$code.=<<___;
+	lea	-8(%r10),%rsp
+.cfi_def_cfa_register	%rsp
+.L128_epilogue:
+	ret
+.cfi_endproc
+.size	chacha20_128,.-chacha20_128
+___
+}
+
+########################################################################
+# SSSE3 code path that handles longer messages.
+{
+# assign variables to favor Atom front-end
+my ($xd0,$xd1,$xd2,$xd3, $xt0,$xt1,$xt2,$xt3,
+    $xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3)=map("%xmm$_",(0..15));
+my  @xx=($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3,
+	"%nox","%nox","%nox","%nox", $xd0,$xd1,$xd2,$xd3);
+
+sub SSSE3_lane_ROUND {
+my ($a0,$b0,$c0,$d0)=@_;
+my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
+my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
+my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
+my ($xc,$xc_,$t0,$t1)=map("\"$_\"",$xt0,$xt1,$xt2,$xt3);
+my @x=map("\"$_\"",@xx);
+
+	# Consider order in which variables are addressed by their
+	# index:
+	#
+	#	a   b   c   d
+	#
+	#	0   4   8  12 < even round
+	#	1   5   9  13
+	#	2   6  10  14
+	#	3   7  11  15
+	#	0   5  10  15 < odd round
+	#	1   6  11  12
+	#	2   7   8  13
+	#	3   4   9  14
+	#
+	# 'a', 'b' and 'd's are permanently allocated in registers,
+	# @x[0..7,12..15], while 'c's are maintained in memory. If
+	# you observe 'c' column, you'll notice that pair of 'c's is
+	# invariant between rounds. This means that we have to reload
+	# them once per round, in the middle. This is why you'll see
+	# bunch of 'c' stores and loads in the middle, but none in
+	# the beginning or end.
+
+	(
+	"&paddd		(@x[$a0],@x[$b0])",	# Q1
+	 "&paddd	(@x[$a1],@x[$b1])",	# Q2
+	"&pxor		(@x[$d0],@x[$a0])",
+	 "&pxor		(@x[$d1],@x[$a1])",
+	"&pshufb	(@x[$d0],$t1)",
+	 "&pshufb	(@x[$d1],$t1)",
+
+	"&paddd		($xc,@x[$d0])",
+	 "&paddd	($xc_,@x[$d1])",
+	"&pxor		(@x[$b0],$xc)",
+	 "&pxor		(@x[$b1],$xc_)",
+	"&movdqa	($t0,@x[$b0])",
+	"&pslld		(@x[$b0],12)",
+	"&psrld		($t0,20)",
+	 "&movdqa	($t1,@x[$b1])",
+	 "&pslld	(@x[$b1],12)",
+	"&por		(@x[$b0],$t0)",
+	 "&psrld	($t1,20)",
+	"&movdqa	($t0,'(%r11)')",	# .Lrot24(%rip)
+	 "&por		(@x[$b1],$t1)",
+
+	"&paddd		(@x[$a0],@x[$b0])",
+	 "&paddd	(@x[$a1],@x[$b1])",
+	"&pxor		(@x[$d0],@x[$a0])",
+	 "&pxor		(@x[$d1],@x[$a1])",
+	"&pshufb	(@x[$d0],$t0)",
+	 "&pshufb	(@x[$d1],$t0)",
+
+	"&paddd		($xc,@x[$d0])",
+	 "&paddd	($xc_,@x[$d1])",
+	"&pxor		(@x[$b0],$xc)",
+	 "&pxor		(@x[$b1],$xc_)",
+	"&movdqa	($t1,@x[$b0])",
+	"&pslld		(@x[$b0],7)",
+	"&psrld		($t1,25)",
+	 "&movdqa	($t0,@x[$b1])",
+	 "&pslld	(@x[$b1],7)",
+	"&por		(@x[$b0],$t1)",
+	 "&psrld	($t0,25)",
+	"&movdqa	($t1,'(%r9)')",	# .Lrot16(%rip)
+	 "&por		(@x[$b1],$t0)",
+
+	"&movdqa	(\"`16*($c0-8)`(%rsp)\",$xc)",	# reload pair of 'c's
+	 "&movdqa	(\"`16*($c1-8)`(%rsp)\",$xc_)",
+	"&movdqa	($xc,\"`16*($c2-8)`(%rsp)\")",
+	 "&movdqa	($xc_,\"`16*($c3-8)`(%rsp)\")",
+
+	"&paddd		(@x[$a2],@x[$b2])",	# Q3
+	 "&paddd	(@x[$a3],@x[$b3])",	# Q4
+	"&pxor		(@x[$d2],@x[$a2])",
+	 "&pxor		(@x[$d3],@x[$a3])",
+	"&pshufb	(@x[$d2],$t1)",
+	 "&pshufb	(@x[$d3],$t1)",
+
+	"&paddd		($xc,@x[$d2])",
+	 "&paddd	($xc_,@x[$d3])",
+	"&pxor		(@x[$b2],$xc)",
+	 "&pxor		(@x[$b3],$xc_)",
+	"&movdqa	($t0,@x[$b2])",
+	"&pslld		(@x[$b2],12)",
+	"&psrld		($t0,20)",
+	 "&movdqa	($t1,@x[$b3])",
+	 "&pslld	(@x[$b3],12)",
+	"&por		(@x[$b2],$t0)",
+	 "&psrld	($t1,20)",
+	"&movdqa	($t0,'(%r11)')",	# .Lrot24(%rip)
+	 "&por		(@x[$b3],$t1)",
+
+	"&paddd		(@x[$a2],@x[$b2])",
+	 "&paddd	(@x[$a3],@x[$b3])",
+	"&pxor		(@x[$d2],@x[$a2])",
+	 "&pxor		(@x[$d3],@x[$a3])",
+	"&pshufb	(@x[$d2],$t0)",
+	 "&pshufb	(@x[$d3],$t0)",
+
+	"&paddd		($xc,@x[$d2])",
+	 "&paddd	($xc_,@x[$d3])",
+	"&pxor		(@x[$b2],$xc)",
+	 "&pxor		(@x[$b3],$xc_)",
+	"&movdqa	($t1,@x[$b2])",
+	"&pslld		(@x[$b2],7)",
+	"&psrld		($t1,25)",
+	 "&movdqa	($t0,@x[$b3])",
+	 "&pslld	(@x[$b3],7)",
+	"&por		(@x[$b2],$t1)",
+	 "&psrld	($t0,25)",
+	"&movdqa	($t1,'(%r9)')",	# .Lrot16(%rip)
+	 "&por		(@x[$b3],$t0)"
+	);
+}
+
+my $xframe = $win64 ? 0xa8 : 8;
+
+$code.=<<___;
+.type	chacha20_4x,\@function,5
+.align	32
+chacha20_4x:
+.cfi_startproc
+.Lchacha20_4x:
+	lea		8(%rsp),%r10		# frame pointer
+.cfi_def_cfa_register	%r10
+___
+$code.=<<___ if (!$kernel);
+	mov		%r9,%r11
+___
+$code.=<<___	if ($avx>1 && !$kernel);
+	shr		\$32,%r9		# OPENSSL_ia32cap_P+8
+	test		\$`1<<5`,%r9		# test AVX2
+	jnz		.Lchacha20_8x
+___
+$code.=<<___;
+	cmp		\$192,$len
+	ja		.Lproceed4x
+___
+$code.=<<___ if (!$kernel);
+	and		\$`1<<26|1<<22`,%r11	# isolate XSAVE+MOVBE
+	cmp		\$`1<<22`,%r11		# check for MOVBE without XSAVE
+	je		.Ldo_ssse3_after_all	# to detect Atom
+___
+$code.=<<___;
+.Lproceed4x:
+	sub		\$0x140+$xframe,%rsp
+	and		\$-16,%rsp
+___
+	################ stack layout
+	# +0x00		SIMD equivalent of @x[8-12]
+	# ...
+	# +0x40		constant copy of key[0-2] smashed by lanes
+	# ...
+	# +0x100	SIMD counters (with nonce smashed by lanes)
+	# ...
+	# +0x140
+$code.=<<___	if ($win64);
+	movaps		%xmm6,-0xb0(%r10)
+	movaps		%xmm7,-0xa0(%r10)
+	movaps		%xmm8,-0x90(%r10)
+	movaps		%xmm9,-0x80(%r10)
+	movaps		%xmm10,-0x70(%r10)
+	movaps		%xmm11,-0x60(%r10)
+	movaps		%xmm12,-0x50(%r10)
+	movaps		%xmm13,-0x40(%r10)
+	movaps		%xmm14,-0x30(%r10)
+	movaps		%xmm15,-0x20(%r10)
+.L4x_body:
+___
+$code.=<<___;
+	movdqa		.Lsigma(%rip),$xa3	# key[0]
+	movdqu		($key),$xb3		# key[1]
+	movdqu		16($key),$xt3		# key[2]
+	movdqu		($counter),$xd3		# key[3]
+	lea		0x100(%rsp),%rcx	# size optimization
+	lea		.Lrot16(%rip),%r9
+	lea		.Lrot24(%rip),%r11
+
+	pshufd		\$0x00,$xa3,$xa0	# smash key by lanes...
+	pshufd		\$0x55,$xa3,$xa1
+	movdqa		$xa0,0x40(%rsp)		# ... and offload
+	pshufd		\$0xaa,$xa3,$xa2
+	movdqa		$xa1,0x50(%rsp)
+	pshufd		\$0xff,$xa3,$xa3
+	movdqa		$xa2,0x60(%rsp)
+	movdqa		$xa3,0x70(%rsp)
+
+	pshufd		\$0x00,$xb3,$xb0
+	pshufd		\$0x55,$xb3,$xb1
+	movdqa		$xb0,0x80-0x100(%rcx)
+	pshufd		\$0xaa,$xb3,$xb2
+	movdqa		$xb1,0x90-0x100(%rcx)
+	pshufd		\$0xff,$xb3,$xb3
+	movdqa		$xb2,0xa0-0x100(%rcx)
+	movdqa		$xb3,0xb0-0x100(%rcx)
+
+	pshufd		\$0x00,$xt3,$xt0	# "$xc0"
+	pshufd		\$0x55,$xt3,$xt1	# "$xc1"
+	movdqa		$xt0,0xc0-0x100(%rcx)
+	pshufd		\$0xaa,$xt3,$xt2	# "$xc2"
+	movdqa		$xt1,0xd0-0x100(%rcx)
+	pshufd		\$0xff,$xt3,$xt3	# "$xc3"
+	movdqa		$xt2,0xe0-0x100(%rcx)
+	movdqa		$xt3,0xf0-0x100(%rcx)
+
+	pshufd		\$0x00,$xd3,$xd0
+	pshufd		\$0x55,$xd3,$xd1
+	paddd		.Linc(%rip),$xd0	# don't save counters yet
+	pshufd		\$0xaa,$xd3,$xd2
+	movdqa		$xd1,0x110-0x100(%rcx)
+	pshufd		\$0xff,$xd3,$xd3
+	movdqa		$xd2,0x120-0x100(%rcx)
+	movdqa		$xd3,0x130-0x100(%rcx)
+
+	jmp		.Loop_enter4x
+
+.align	32
+.Loop_outer4x:
+	movdqa		0x40(%rsp),$xa0		# re-load smashed key
+	movdqa		0x50(%rsp),$xa1
+	movdqa		0x60(%rsp),$xa2
+	movdqa		0x70(%rsp),$xa3
+	movdqa		0x80-0x100(%rcx),$xb0
+	movdqa		0x90-0x100(%rcx),$xb1
+	movdqa		0xa0-0x100(%rcx),$xb2
+	movdqa		0xb0-0x100(%rcx),$xb3
+	movdqa		0xc0-0x100(%rcx),$xt0	# "$xc0"
+	movdqa		0xd0-0x100(%rcx),$xt1	# "$xc1"
+	movdqa		0xe0-0x100(%rcx),$xt2	# "$xc2"
+	movdqa		0xf0-0x100(%rcx),$xt3	# "$xc3"
+	movdqa		0x100-0x100(%rcx),$xd0
+	movdqa		0x110-0x100(%rcx),$xd1
+	movdqa		0x120-0x100(%rcx),$xd2
+	movdqa		0x130-0x100(%rcx),$xd3
+	paddd		.Lfour(%rip),$xd0	# next SIMD counters
+
+.Loop_enter4x:
+	movdqa		$xt2,0x20(%rsp)		# SIMD equivalent of "@x[10]"
+	movdqa		$xt3,0x30(%rsp)		# SIMD equivalent of "@x[11]"
+	movdqa		(%r9),$xt3		# .Lrot16(%rip)
+	mov		\$10,%eax
+	movdqa		$xd0,0x100-0x100(%rcx)	# save SIMD counters
+	jmp		.Loop4x
+
+.align	32
+.Loop4x:
+___
+	foreach (&SSSE3_lane_ROUND(0, 4, 8,12)) { eval; }
+	foreach (&SSSE3_lane_ROUND(0, 5,10,15)) { eval; }
+$code.=<<___;
+	dec		%eax
+	jnz		.Loop4x
+
+	paddd		0x40(%rsp),$xa0		# accumulate key material
+	paddd		0x50(%rsp),$xa1
+	paddd		0x60(%rsp),$xa2
+	paddd		0x70(%rsp),$xa3
+
+	movdqa		$xa0,$xt2		# "de-interlace" data
+	punpckldq	$xa1,$xa0
+	movdqa		$xa2,$xt3
+	punpckldq	$xa3,$xa2
+	punpckhdq	$xa1,$xt2
+	punpckhdq	$xa3,$xt3
+	movdqa		$xa0,$xa1
+	punpcklqdq	$xa2,$xa0		# "a0"
+	movdqa		$xt2,$xa3
+	punpcklqdq	$xt3,$xt2		# "a2"
+	punpckhqdq	$xa2,$xa1		# "a1"
+	punpckhqdq	$xt3,$xa3		# "a3"
+___
+	($xa2,$xt2)=($xt2,$xa2);
+$code.=<<___;
+	paddd		0x80-0x100(%rcx),$xb0
+	paddd		0x90-0x100(%rcx),$xb1
+	paddd		0xa0-0x100(%rcx),$xb2
+	paddd		0xb0-0x100(%rcx),$xb3
+
+	movdqa		$xa0,0x00(%rsp)		# offload $xaN
+	movdqa		$xa1,0x10(%rsp)
+	movdqa		0x20(%rsp),$xa0		# "xc2"
+	movdqa		0x30(%rsp),$xa1		# "xc3"
+
+	movdqa		$xb0,$xt2
+	punpckldq	$xb1,$xb0
+	movdqa		$xb2,$xt3
+	punpckldq	$xb3,$xb2
+	punpckhdq	$xb1,$xt2
+	punpckhdq	$xb3,$xt3
+	movdqa		$xb0,$xb1
+	punpcklqdq	$xb2,$xb0		# "b0"
+	movdqa		$xt2,$xb3
+	punpcklqdq	$xt3,$xt2		# "b2"
+	punpckhqdq	$xb2,$xb1		# "b1"
+	punpckhqdq	$xt3,$xb3		# "b3"
+___
+	($xb2,$xt2)=($xt2,$xb2);
+	my ($xc0,$xc1,$xc2,$xc3)=($xt0,$xt1,$xa0,$xa1);
+$code.=<<___;
+	paddd		0xc0-0x100(%rcx),$xc0
+	paddd		0xd0-0x100(%rcx),$xc1
+	paddd		0xe0-0x100(%rcx),$xc2
+	paddd		0xf0-0x100(%rcx),$xc3
+
+	movdqa		$xa2,0x20(%rsp)		# keep offloading $xaN
+	movdqa		$xa3,0x30(%rsp)
+
+	movdqa		$xc0,$xt2
+	punpckldq	$xc1,$xc0
+	movdqa		$xc2,$xt3
+	punpckldq	$xc3,$xc2
+	punpckhdq	$xc1,$xt2
+	punpckhdq	$xc3,$xt3
+	movdqa		$xc0,$xc1
+	punpcklqdq	$xc2,$xc0		# "c0"
+	movdqa		$xt2,$xc3
+	punpcklqdq	$xt3,$xt2		# "c2"
+	punpckhqdq	$xc2,$xc1		# "c1"
+	punpckhqdq	$xt3,$xc3		# "c3"
+___
+	($xc2,$xt2)=($xt2,$xc2);
+	($xt0,$xt1)=($xa2,$xa3);		# use $xaN as temporary
+$code.=<<___;
+	paddd		0x100-0x100(%rcx),$xd0
+	paddd		0x110-0x100(%rcx),$xd1
+	paddd		0x120-0x100(%rcx),$xd2
+	paddd		0x130-0x100(%rcx),$xd3
+
+	movdqa		$xd0,$xt2
+	punpckldq	$xd1,$xd0
+	movdqa		$xd2,$xt3
+	punpckldq	$xd3,$xd2
+	punpckhdq	$xd1,$xt2
+	punpckhdq	$xd3,$xt3
+	movdqa		$xd0,$xd1
+	punpcklqdq	$xd2,$xd0		# "d0"
+	movdqa		$xt2,$xd3
+	punpcklqdq	$xt3,$xt2		# "d2"
+	punpckhqdq	$xd2,$xd1		# "d1"
+	punpckhqdq	$xt3,$xd3		# "d3"
+___
+	($xd2,$xt2)=($xt2,$xd2);
+$code.=<<___;
+	cmp		\$64*4,$len
+	jb		.Ltail4x
+
+	movdqu		0x00($inp),$xt0		# xor with input
+	movdqu		0x10($inp),$xt1
+	movdqu		0x20($inp),$xt2
+	movdqu		0x30($inp),$xt3
+	pxor		0x00(%rsp),$xt0		# $xaN is offloaded, remember?
+	pxor		$xb0,$xt1
+	pxor		$xc0,$xt2
+	pxor		$xd0,$xt3
+
+	 movdqu		$xt0,0x00($out)
+	movdqu		0x40($inp),$xt0
+	 movdqu		$xt1,0x10($out)
+	movdqu		0x50($inp),$xt1
+	 movdqu		$xt2,0x20($out)
+	movdqu		0x60($inp),$xt2
+	 movdqu		$xt3,0x30($out)
+	movdqu		0x70($inp),$xt3
+	lea		0x80($inp),$inp		# size optimization
+	pxor		0x10(%rsp),$xt0
+	pxor		$xb1,$xt1
+	pxor		$xc1,$xt2
+	pxor		$xd1,$xt3
+
+	 movdqu		$xt0,0x40($out)
+	movdqu		0x00($inp),$xt0
+	 movdqu		$xt1,0x50($out)
+	movdqu		0x10($inp),$xt1
+	 movdqu		$xt2,0x60($out)
+	movdqu		0x20($inp),$xt2
+	 movdqu		$xt3,0x70($out)
+	 lea		0x80($out),$out		# size optimization
+	movdqu		0x30($inp),$xt3
+	pxor		0x20(%rsp),$xt0
+	pxor		$xb2,$xt1
+	pxor		$xc2,$xt2
+	pxor		$xd2,$xt3
+
+	 movdqu		$xt0,0x00($out)
+	movdqu		0x40($inp),$xt0
+	 movdqu		$xt1,0x10($out)
+	movdqu		0x50($inp),$xt1
+	 movdqu		$xt2,0x20($out)
+	movdqu		0x60($inp),$xt2
+	 movdqu		$xt3,0x30($out)
+	movdqu		0x70($inp),$xt3
+	lea		0x80($inp),$inp		# inp+=64*4
+	pxor		0x30(%rsp),$xt0
+	pxor		$xb3,$xt1
+	pxor		$xc3,$xt2
+	pxor		$xd3,$xt3
+	movdqu		$xt0,0x40($out)
+	movdqu		$xt1,0x50($out)
+	movdqu		$xt2,0x60($out)
+	movdqu		$xt3,0x70($out)
+	lea		0x80($out),$out		# out+=64*4
+
+	sub		\$64*4,$len
+	jnz		.Loop_outer4x
+
+	jmp		.Ldone4x
+
+.Ltail4x:
+	cmp		\$192,$len
+	jae		.L192_or_more4x
+	cmp		\$128,$len
+	jae		.L128_or_more4x
+	cmp		\$64,$len
+	jae		.L64_or_more4x
+
+	#movdqa		0x00(%rsp),$xt0		# $xaN is offloaded, remember?
+	xor		%r9,%r9
+	#movdqa		$xt0,0x00(%rsp)
+	movdqa		$xb0,0x10(%rsp)
+	movdqa		$xc0,0x20(%rsp)
+	movdqa		$xd0,0x30(%rsp)
+	jmp		.Loop_tail4x
+
+.align	32
+.L64_or_more4x:
+	movdqu		0x00($inp),$xt0		# xor with input
+	movdqu		0x10($inp),$xt1
+	movdqu		0x20($inp),$xt2
+	movdqu		0x30($inp),$xt3
+	pxor		0x00(%rsp),$xt0		# $xaxN is offloaded, remember?
+	pxor		$xb0,$xt1
+	pxor		$xc0,$xt2
+	pxor		$xd0,$xt3
+	movdqu		$xt0,0x00($out)
+	movdqu		$xt1,0x10($out)
+	movdqu		$xt2,0x20($out)
+	movdqu		$xt3,0x30($out)
+	je		.Ldone4x
+
+	movdqa		0x10(%rsp),$xt0		# $xaN is offloaded, remember?
+	lea		0x40($inp),$inp		# inp+=64*1
+	xor		%r9,%r9
+	movdqa		$xt0,0x00(%rsp)
+	movdqa		$xb1,0x10(%rsp)
+	lea		0x40($out),$out		# out+=64*1
+	movdqa		$xc1,0x20(%rsp)
+	sub		\$64,$len		# len-=64*1
+	movdqa		$xd1,0x30(%rsp)
+	jmp		.Loop_tail4x
+
+.align	32
+.L128_or_more4x:
+	movdqu		0x00($inp),$xt0		# xor with input
+	movdqu		0x10($inp),$xt1
+	movdqu		0x20($inp),$xt2
+	movdqu		0x30($inp),$xt3
+	pxor		0x00(%rsp),$xt0		# $xaN is offloaded, remember?
+	pxor		$xb0,$xt1
+	pxor		$xc0,$xt2
+	pxor		$xd0,$xt3
+
+	 movdqu		$xt0,0x00($out)
+	movdqu		0x40($inp),$xt0
+	 movdqu		$xt1,0x10($out)
+	movdqu		0x50($inp),$xt1
+	 movdqu		$xt2,0x20($out)
+	movdqu		0x60($inp),$xt2
+	 movdqu		$xt3,0x30($out)
+	movdqu		0x70($inp),$xt3
+	pxor		0x10(%rsp),$xt0
+	pxor		$xb1,$xt1
+	pxor		$xc1,$xt2
+	pxor		$xd1,$xt3
+	movdqu		$xt0,0x40($out)
+	movdqu		$xt1,0x50($out)
+	movdqu		$xt2,0x60($out)
+	movdqu		$xt3,0x70($out)
+	je		.Ldone4x
+
+	movdqa		0x20(%rsp),$xt0		# $xaN is offloaded, remember?
+	lea		0x80($inp),$inp		# inp+=64*2
+	xor		%r9,%r9
+	movdqa		$xt0,0x00(%rsp)
+	movdqa		$xb2,0x10(%rsp)
+	lea		0x80($out),$out		# out+=64*2
+	movdqa		$xc2,0x20(%rsp)
+	sub		\$128,$len		# len-=64*2
+	movdqa		$xd2,0x30(%rsp)
+	jmp		.Loop_tail4x
+
+.align	32
+.L192_or_more4x:
+	movdqu		0x00($inp),$xt0		# xor with input
+	movdqu		0x10($inp),$xt1
+	movdqu		0x20($inp),$xt2
+	movdqu		0x30($inp),$xt3
+	pxor		0x00(%rsp),$xt0		# $xaN is offloaded, remember?
+	pxor		$xb0,$xt1
+	pxor		$xc0,$xt2
+	pxor		$xd0,$xt3
+
+	 movdqu		$xt0,0x00($out)
+	movdqu		0x40($inp),$xt0
+	 movdqu		$xt1,0x10($out)
+	movdqu		0x50($inp),$xt1
+	 movdqu		$xt2,0x20($out)
+	movdqu		0x60($inp),$xt2
+	 movdqu		$xt3,0x30($out)
+	movdqu		0x70($inp),$xt3
+	lea		0x80($inp),$inp		# size optimization
+	pxor		0x10(%rsp),$xt0
+	pxor		$xb1,$xt1
+	pxor		$xc1,$xt2
+	pxor		$xd1,$xt3
+
+	 movdqu		$xt0,0x40($out)
+	movdqu		0x00($inp),$xt0
+	 movdqu		$xt1,0x50($out)
+	movdqu		0x10($inp),$xt1
+	 movdqu		$xt2,0x60($out)
+	movdqu		0x20($inp),$xt2
+	 movdqu		$xt3,0x70($out)
+	 lea		0x80($out),$out		# size optimization
+	movdqu		0x30($inp),$xt3
+	pxor		0x20(%rsp),$xt0
+	pxor		$xb2,$xt1
+	pxor		$xc2,$xt2
+	pxor		$xd2,$xt3
+	movdqu		$xt0,0x00($out)
+	movdqu		$xt1,0x10($out)
+	movdqu		$xt2,0x20($out)
+	movdqu		$xt3,0x30($out)
+	je		.Ldone4x
+
+	movdqa		0x30(%rsp),$xt0		# $xaN is offloaded, remember?
+	lea		0x40($inp),$inp		# inp+=64*3
+	xor		%r9,%r9
+	movdqa		$xt0,0x00(%rsp)
+	movdqa		$xb3,0x10(%rsp)
+	lea		0x40($out),$out		# out+=64*3
+	movdqa		$xc3,0x20(%rsp)
+	sub		\$192,$len		# len-=64*3
+	movdqa		$xd3,0x30(%rsp)
+
+.Loop_tail4x:
+	movzb		($inp,%r9),%eax
+	movzb		(%rsp,%r9),%ecx
+	lea		1(%r9),%r9
+	xor		%ecx,%eax
+	mov		%al,-1($out,%r9)
+	dec		$len
+	jnz		.Loop_tail4x
+
+.Ldone4x:
+___
+$code.=<<___	if ($win64);
+	movaps		-0xb0(%r10),%xmm6
+	movaps		-0xa0(%r10),%xmm7
+	movaps		-0x90(%r10),%xmm8
+	movaps		-0x80(%r10),%xmm9
+	movaps		-0x70(%r10),%xmm10
+	movaps		-0x60(%r10),%xmm11
+	movaps		-0x50(%r10),%xmm12
+	movaps		-0x40(%r10),%xmm13
+	movaps		-0x30(%r10),%xmm14
+	movaps		-0x20(%r10),%xmm15
+___
+$code.=<<___;
+	lea		-8(%r10),%rsp
+.cfi_def_cfa_register	%rsp
+.L4x_epilogue:
+	ret
+.cfi_endproc
+.size	chacha20_4x,.-chacha20_4x
+___
+}
+if($kernel) {
+	$code .= "#endif\n";
+}
+
+########################################################################
+# XOP code path that handles all lengths.
+if ($avx && !$kernel) {
+# There is some "anomaly" observed depending on instructions' size or
+# alignment. If you look closely at below code you'll notice that
+# sometimes argument order varies. The order affects instruction
+# encoding by making it larger, and such fiddling gives 5% performance
+# improvement. This is on FX-4100...
+
+my ($xb0,$xb1,$xb2,$xb3, $xd0,$xd1,$xd2,$xd3,
+    $xa0,$xa1,$xa2,$xa3, $xt0,$xt1,$xt2,$xt3)=map("%xmm$_",(0..15));
+my  @xx=($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3,
+	 $xt0,$xt1,$xt2,$xt3, $xd0,$xd1,$xd2,$xd3);
+
+sub XOP_lane_ROUND {
+my ($a0,$b0,$c0,$d0)=@_;
+my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
+my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
+my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
+my @x=map("\"$_\"",@xx);
+
+	(
+	"&vpaddd	(@x[$a0],@x[$a0],@x[$b0])",	# Q1
+	 "&vpaddd	(@x[$a1],@x[$a1],@x[$b1])",	# Q2
+	  "&vpaddd	(@x[$a2],@x[$a2],@x[$b2])",	# Q3
+	   "&vpaddd	(@x[$a3],@x[$a3],@x[$b3])",	# Q4
+	"&vpxor		(@x[$d0],@x[$a0],@x[$d0])",
+	 "&vpxor	(@x[$d1],@x[$a1],@x[$d1])",
+	  "&vpxor	(@x[$d2],@x[$a2],@x[$d2])",
+	   "&vpxor	(@x[$d3],@x[$a3],@x[$d3])",
+	"&vprotd	(@x[$d0],@x[$d0],16)",
+	 "&vprotd	(@x[$d1],@x[$d1],16)",
+	  "&vprotd	(@x[$d2],@x[$d2],16)",
+	   "&vprotd	(@x[$d3],@x[$d3],16)",
+
+	"&vpaddd	(@x[$c0],@x[$c0],@x[$d0])",
+	 "&vpaddd	(@x[$c1],@x[$c1],@x[$d1])",
+	  "&vpaddd	(@x[$c2],@x[$c2],@x[$d2])",
+	   "&vpaddd	(@x[$c3],@x[$c3],@x[$d3])",
+	"&vpxor		(@x[$b0],@x[$c0],@x[$b0])",
+	 "&vpxor	(@x[$b1],@x[$c1],@x[$b1])",
+	  "&vpxor	(@x[$b2],@x[$b2],@x[$c2])",	# flip
+	   "&vpxor	(@x[$b3],@x[$b3],@x[$c3])",	# flip
+	"&vprotd	(@x[$b0],@x[$b0],12)",
+	 "&vprotd	(@x[$b1],@x[$b1],12)",
+	  "&vprotd	(@x[$b2],@x[$b2],12)",
+	   "&vprotd	(@x[$b3],@x[$b3],12)",
+
+	"&vpaddd	(@x[$a0],@x[$b0],@x[$a0])",	# flip
+	 "&vpaddd	(@x[$a1],@x[$b1],@x[$a1])",	# flip
+	  "&vpaddd	(@x[$a2],@x[$a2],@x[$b2])",
+	   "&vpaddd	(@x[$a3],@x[$a3],@x[$b3])",
+	"&vpxor		(@x[$d0],@x[$a0],@x[$d0])",
+	 "&vpxor	(@x[$d1],@x[$a1],@x[$d1])",
+	  "&vpxor	(@x[$d2],@x[$a2],@x[$d2])",
+	   "&vpxor	(@x[$d3],@x[$a3],@x[$d3])",
+	"&vprotd	(@x[$d0],@x[$d0],8)",
+	 "&vprotd	(@x[$d1],@x[$d1],8)",
+	  "&vprotd	(@x[$d2],@x[$d2],8)",
+	   "&vprotd	(@x[$d3],@x[$d3],8)",
+
+	"&vpaddd	(@x[$c0],@x[$c0],@x[$d0])",
+	 "&vpaddd	(@x[$c1],@x[$c1],@x[$d1])",
+	  "&vpaddd	(@x[$c2],@x[$c2],@x[$d2])",
+	   "&vpaddd	(@x[$c3],@x[$c3],@x[$d3])",
+	"&vpxor		(@x[$b0],@x[$c0],@x[$b0])",
+	 "&vpxor	(@x[$b1],@x[$c1],@x[$b1])",
+	  "&vpxor	(@x[$b2],@x[$b2],@x[$c2])",	# flip
+	   "&vpxor	(@x[$b3],@x[$b3],@x[$c3])",	# flip
+	"&vprotd	(@x[$b0],@x[$b0],7)",
+	 "&vprotd	(@x[$b1],@x[$b1],7)",
+	  "&vprotd	(@x[$b2],@x[$b2],7)",
+	   "&vprotd	(@x[$b3],@x[$b3],7)"
+	);
+}
+
+my $xframe = $win64 ? 0xa8 : 8;
+
+&declare_function("chacha20_xop", 32, 5);
+$code.=<<___;
+.cfi_startproc
+.Lchacha20_4xop:
+	lea		8(%rsp),%r10		# frame pointer
+.cfi_def_cfa_register	%r10
+	sub		\$0x140+$xframe,%rsp
+	and 	\$-16,%rsp
+___
+	################ stack layout
+	# +0x00		SIMD equivalent of @x[8-12]
+	# ...
+	# +0x40		constant copy of key[0-2] smashed by lanes
+	# ...
+	# +0x100	SIMD counters (with nonce smashed by lanes)
+	# ...
+	# +0x140
+$code.=<<___	if ($win64);
+	movaps		%xmm6,-0xb0(%r10)
+	movaps		%xmm7,-0xa0(%r10)
+	movaps		%xmm8,-0x90(%r10)
+	movaps		%xmm9,-0x80(%r10)
+	movaps		%xmm10,-0x70(%r10)
+	movaps		%xmm11,-0x60(%r10)
+	movaps		%xmm12,-0x50(%r10)
+	movaps		%xmm13,-0x40(%r10)
+	movaps		%xmm14,-0x30(%r10)
+	movaps		%xmm15,-0x20(%r10)
+.L4xop_body:
+___
+$code.=<<___;
+	vzeroupper
+
+	vmovdqa		.Lsigma(%rip),$xa3	# key[0]
+	vmovdqu		($key),$xb3		# key[1]
+	vmovdqu		16($key),$xt3		# key[2]
+	vmovdqu		($counter),$xd3		# key[3]
+	lea		0x100(%rsp),%rcx	# size optimization
+
+	vpshufd		\$0x00,$xa3,$xa0	# smash key by lanes...
+	vpshufd		\$0x55,$xa3,$xa1
+	vmovdqa		$xa0,0x40(%rsp)		# ... and offload
+	vpshufd		\$0xaa,$xa3,$xa2
+	vmovdqa		$xa1,0x50(%rsp)
+	vpshufd		\$0xff,$xa3,$xa3
+	vmovdqa		$xa2,0x60(%rsp)
+	vmovdqa		$xa3,0x70(%rsp)
+
+	vpshufd		\$0x00,$xb3,$xb0
+	vpshufd		\$0x55,$xb3,$xb1
+	vmovdqa		$xb0,0x80-0x100(%rcx)
+	vpshufd		\$0xaa,$xb3,$xb2
+	vmovdqa		$xb1,0x90-0x100(%rcx)
+	vpshufd		\$0xff,$xb3,$xb3
+	vmovdqa		$xb2,0xa0-0x100(%rcx)
+	vmovdqa		$xb3,0xb0-0x100(%rcx)
+
+	vpshufd		\$0x00,$xt3,$xt0	# "$xc0"
+	vpshufd		\$0x55,$xt3,$xt1	# "$xc1"
+	vmovdqa		$xt0,0xc0-0x100(%rcx)
+	vpshufd		\$0xaa,$xt3,$xt2	# "$xc2"
+	vmovdqa		$xt1,0xd0-0x100(%rcx)
+	vpshufd		\$0xff,$xt3,$xt3	# "$xc3"
+	vmovdqa		$xt2,0xe0-0x100(%rcx)
+	vmovdqa		$xt3,0xf0-0x100(%rcx)
+
+	vpshufd		\$0x00,$xd3,$xd0
+	vpshufd		\$0x55,$xd3,$xd1
+	vpaddd		.Linc(%rip),$xd0,$xd0	# don't save counters yet
+	vpshufd		\$0xaa,$xd3,$xd2
+	vmovdqa		$xd1,0x110-0x100(%rcx)
+	vpshufd		\$0xff,$xd3,$xd3
+	vmovdqa		$xd2,0x120-0x100(%rcx)
+	vmovdqa		$xd3,0x130-0x100(%rcx)
+
+	jmp		.Loop_enter4xop
+
+.align	32
+.Loop_outer4xop:
+	vmovdqa		0x40(%rsp),$xa0		# re-load smashed key
+	vmovdqa		0x50(%rsp),$xa1
+	vmovdqa		0x60(%rsp),$xa2
+	vmovdqa		0x70(%rsp),$xa3
+	vmovdqa		0x80-0x100(%rcx),$xb0
+	vmovdqa		0x90-0x100(%rcx),$xb1
+	vmovdqa		0xa0-0x100(%rcx),$xb2
+	vmovdqa		0xb0-0x100(%rcx),$xb3
+	vmovdqa		0xc0-0x100(%rcx),$xt0	# "$xc0"
+	vmovdqa		0xd0-0x100(%rcx),$xt1	# "$xc1"
+	vmovdqa		0xe0-0x100(%rcx),$xt2	# "$xc2"
+	vmovdqa		0xf0-0x100(%rcx),$xt3	# "$xc3"
+	vmovdqa		0x100-0x100(%rcx),$xd0
+	vmovdqa		0x110-0x100(%rcx),$xd1
+	vmovdqa		0x120-0x100(%rcx),$xd2
+	vmovdqa		0x130-0x100(%rcx),$xd3
+	vpaddd		.Lfour(%rip),$xd0,$xd0	# next SIMD counters
+
+.Loop_enter4xop:
+	mov		\$10,%eax
+	vmovdqa		$xd0,0x100-0x100(%rcx)	# save SIMD counters
+	jmp		.Loop4xop
+
+.align	32
+.Loop4xop:
+___
+	foreach (&XOP_lane_ROUND(0, 4, 8,12)) { eval; }
+	foreach (&XOP_lane_ROUND(0, 5,10,15)) { eval; }
+$code.=<<___;
+	dec		%eax
+	jnz		.Loop4xop
+
+	vpaddd		0x40(%rsp),$xa0,$xa0	# accumulate key material
+	vpaddd		0x50(%rsp),$xa1,$xa1
+	vpaddd		0x60(%rsp),$xa2,$xa2
+	vpaddd		0x70(%rsp),$xa3,$xa3
+
+	vmovdqa		$xt2,0x20(%rsp)		# offload $xc2,3
+	vmovdqa		$xt3,0x30(%rsp)
+
+	vpunpckldq	$xa1,$xa0,$xt2		# "de-interlace" data
+	vpunpckldq	$xa3,$xa2,$xt3
+	vpunpckhdq	$xa1,$xa0,$xa0
+	vpunpckhdq	$xa3,$xa2,$xa2
+	vpunpcklqdq	$xt3,$xt2,$xa1		# "a0"
+	vpunpckhqdq	$xt3,$xt2,$xt2		# "a1"
+	vpunpcklqdq	$xa2,$xa0,$xa3		# "a2"
+	vpunpckhqdq	$xa2,$xa0,$xa0		# "a3"
+___
+        ($xa0,$xa1,$xa2,$xa3,$xt2)=($xa1,$xt2,$xa3,$xa0,$xa2);
+$code.=<<___;
+	vpaddd		0x80-0x100(%rcx),$xb0,$xb0
+	vpaddd		0x90-0x100(%rcx),$xb1,$xb1
+	vpaddd		0xa0-0x100(%rcx),$xb2,$xb2
+	vpaddd		0xb0-0x100(%rcx),$xb3,$xb3
+
+	vmovdqa		$xa0,0x00(%rsp)		# offload $xa0,1
+	vmovdqa		$xa1,0x10(%rsp)
+	vmovdqa		0x20(%rsp),$xa0		# "xc2"
+	vmovdqa		0x30(%rsp),$xa1		# "xc3"
+
+	vpunpckldq	$xb1,$xb0,$xt2
+	vpunpckldq	$xb3,$xb2,$xt3
+	vpunpckhdq	$xb1,$xb0,$xb0
+	vpunpckhdq	$xb3,$xb2,$xb2
+	vpunpcklqdq	$xt3,$xt2,$xb1		# "b0"
+	vpunpckhqdq	$xt3,$xt2,$xt2		# "b1"
+	vpunpcklqdq	$xb2,$xb0,$xb3		# "b2"
+	vpunpckhqdq	$xb2,$xb0,$xb0		# "b3"
+___
+	($xb0,$xb1,$xb2,$xb3,$xt2)=($xb1,$xt2,$xb3,$xb0,$xb2);
+	my ($xc0,$xc1,$xc2,$xc3)=($xt0,$xt1,$xa0,$xa1);
+$code.=<<___;
+	vpaddd		0xc0-0x100(%rcx),$xc0,$xc0
+	vpaddd		0xd0-0x100(%rcx),$xc1,$xc1
+	vpaddd		0xe0-0x100(%rcx),$xc2,$xc2
+	vpaddd		0xf0-0x100(%rcx),$xc3,$xc3
+
+	vpunpckldq	$xc1,$xc0,$xt2
+	vpunpckldq	$xc3,$xc2,$xt3
+	vpunpckhdq	$xc1,$xc0,$xc0
+	vpunpckhdq	$xc3,$xc2,$xc2
+	vpunpcklqdq	$xt3,$xt2,$xc1		# "c0"
+	vpunpckhqdq	$xt3,$xt2,$xt2		# "c1"
+	vpunpcklqdq	$xc2,$xc0,$xc3		# "c2"
+	vpunpckhqdq	$xc2,$xc0,$xc0		# "c3"
+___
+	($xc0,$xc1,$xc2,$xc3,$xt2)=($xc1,$xt2,$xc3,$xc0,$xc2);
+$code.=<<___;
+	vpaddd		0x100-0x100(%rcx),$xd0,$xd0
+	vpaddd		0x110-0x100(%rcx),$xd1,$xd1
+	vpaddd		0x120-0x100(%rcx),$xd2,$xd2
+	vpaddd		0x130-0x100(%rcx),$xd3,$xd3
+
+	vpunpckldq	$xd1,$xd0,$xt2
+	vpunpckldq	$xd3,$xd2,$xt3
+	vpunpckhdq	$xd1,$xd0,$xd0
+	vpunpckhdq	$xd3,$xd2,$xd2
+	vpunpcklqdq	$xt3,$xt2,$xd1		# "d0"
+	vpunpckhqdq	$xt3,$xt2,$xt2		# "d1"
+	vpunpcklqdq	$xd2,$xd0,$xd3		# "d2"
+	vpunpckhqdq	$xd2,$xd0,$xd0		# "d3"
+___
+	($xd0,$xd1,$xd2,$xd3,$xt2)=($xd1,$xt2,$xd3,$xd0,$xd2);
+	($xa0,$xa1)=($xt2,$xt3);
+$code.=<<___;
+	vmovdqa		0x00(%rsp),$xa0		# restore $xa0,1
+	vmovdqa		0x10(%rsp),$xa1
+
+	cmp		\$64*4,$len
+	jb		.Ltail4xop
+
+	vpxor		0x00($inp),$xa0,$xa0	# xor with input
+	vpxor		0x10($inp),$xb0,$xb0
+	vpxor		0x20($inp),$xc0,$xc0
+	vpxor		0x30($inp),$xd0,$xd0
+	vpxor		0x40($inp),$xa1,$xa1
+	vpxor		0x50($inp),$xb1,$xb1
+	vpxor		0x60($inp),$xc1,$xc1
+	vpxor		0x70($inp),$xd1,$xd1
+	lea		0x80($inp),$inp		# size optimization
+	vpxor		0x00($inp),$xa2,$xa2
+	vpxor		0x10($inp),$xb2,$xb2
+	vpxor		0x20($inp),$xc2,$xc2
+	vpxor		0x30($inp),$xd2,$xd2
+	vpxor		0x40($inp),$xa3,$xa3
+	vpxor		0x50($inp),$xb3,$xb3
+	vpxor		0x60($inp),$xc3,$xc3
+	vpxor		0x70($inp),$xd3,$xd3
+	lea		0x80($inp),$inp		# inp+=64*4
+
+	vmovdqu		$xa0,0x00($out)
+	vmovdqu		$xb0,0x10($out)
+	vmovdqu		$xc0,0x20($out)
+	vmovdqu		$xd0,0x30($out)
+	vmovdqu		$xa1,0x40($out)
+	vmovdqu		$xb1,0x50($out)
+	vmovdqu		$xc1,0x60($out)
+	vmovdqu		$xd1,0x70($out)
+	lea		0x80($out),$out		# size optimization
+	vmovdqu		$xa2,0x00($out)
+	vmovdqu		$xb2,0x10($out)
+	vmovdqu		$xc2,0x20($out)
+	vmovdqu		$xd2,0x30($out)
+	vmovdqu		$xa3,0x40($out)
+	vmovdqu		$xb3,0x50($out)
+	vmovdqu		$xc3,0x60($out)
+	vmovdqu		$xd3,0x70($out)
+	lea		0x80($out),$out		# out+=64*4
+
+	sub		\$64*4,$len
+	jnz		.Loop_outer4xop
+
+	jmp		.Ldone4xop
+
+.align	32
+.Ltail4xop:
+	cmp		\$192,$len
+	jae		.L192_or_more4xop
+	cmp		\$128,$len
+	jae		.L128_or_more4xop
+	cmp		\$64,$len
+	jae		.L64_or_more4xop
+
+	xor		%r9,%r9
+	vmovdqa		$xa0,0x00(%rsp)
+	vmovdqa		$xb0,0x10(%rsp)
+	vmovdqa		$xc0,0x20(%rsp)
+	vmovdqa		$xd0,0x30(%rsp)
+	jmp		.Loop_tail4xop
+
+.align	32
+.L64_or_more4xop:
+	vpxor		0x00($inp),$xa0,$xa0	# xor with input
+	vpxor		0x10($inp),$xb0,$xb0
+	vpxor		0x20($inp),$xc0,$xc0
+	vpxor		0x30($inp),$xd0,$xd0
+	vmovdqu		$xa0,0x00($out)
+	vmovdqu		$xb0,0x10($out)
+	vmovdqu		$xc0,0x20($out)
+	vmovdqu		$xd0,0x30($out)
+	je		.Ldone4xop
+
+	lea		0x40($inp),$inp		# inp+=64*1
+	vmovdqa		$xa1,0x00(%rsp)
+	xor		%r9,%r9
+	vmovdqa		$xb1,0x10(%rsp)
+	lea		0x40($out),$out		# out+=64*1
+	vmovdqa		$xc1,0x20(%rsp)
+	sub		\$64,$len		# len-=64*1
+	vmovdqa		$xd1,0x30(%rsp)
+	jmp		.Loop_tail4xop
+
+.align	32
+.L128_or_more4xop:
+	vpxor		0x00($inp),$xa0,$xa0	# xor with input
+	vpxor		0x10($inp),$xb0,$xb0
+	vpxor		0x20($inp),$xc0,$xc0
+	vpxor		0x30($inp),$xd0,$xd0
+	vpxor		0x40($inp),$xa1,$xa1
+	vpxor		0x50($inp),$xb1,$xb1
+	vpxor		0x60($inp),$xc1,$xc1
+	vpxor		0x70($inp),$xd1,$xd1
+
+	vmovdqu		$xa0,0x00($out)
+	vmovdqu		$xb0,0x10($out)
+	vmovdqu		$xc0,0x20($out)
+	vmovdqu		$xd0,0x30($out)
+	vmovdqu		$xa1,0x40($out)
+	vmovdqu		$xb1,0x50($out)
+	vmovdqu		$xc1,0x60($out)
+	vmovdqu		$xd1,0x70($out)
+	je		.Ldone4xop
+
+	lea		0x80($inp),$inp		# inp+=64*2
+	vmovdqa		$xa2,0x00(%rsp)
+	xor		%r9,%r9
+	vmovdqa		$xb2,0x10(%rsp)
+	lea		0x80($out),$out		# out+=64*2
+	vmovdqa		$xc2,0x20(%rsp)
+	sub		\$128,$len		# len-=64*2
+	vmovdqa		$xd2,0x30(%rsp)
+	jmp		.Loop_tail4xop
+
+.align	32
+.L192_or_more4xop:
+	vpxor		0x00($inp),$xa0,$xa0	# xor with input
+	vpxor		0x10($inp),$xb0,$xb0
+	vpxor		0x20($inp),$xc0,$xc0
+	vpxor		0x30($inp),$xd0,$xd0
+	vpxor		0x40($inp),$xa1,$xa1
+	vpxor		0x50($inp),$xb1,$xb1
+	vpxor		0x60($inp),$xc1,$xc1
+	vpxor		0x70($inp),$xd1,$xd1
+	lea		0x80($inp),$inp		# size optimization
+	vpxor		0x00($inp),$xa2,$xa2
+	vpxor		0x10($inp),$xb2,$xb2
+	vpxor		0x20($inp),$xc2,$xc2
+	vpxor		0x30($inp),$xd2,$xd2
+
+	vmovdqu		$xa0,0x00($out)
+	vmovdqu		$xb0,0x10($out)
+	vmovdqu		$xc0,0x20($out)
+	vmovdqu		$xd0,0x30($out)
+	vmovdqu		$xa1,0x40($out)
+	vmovdqu		$xb1,0x50($out)
+	vmovdqu		$xc1,0x60($out)
+	vmovdqu		$xd1,0x70($out)
+	lea		0x80($out),$out		# size optimization
+	vmovdqu		$xa2,0x00($out)
+	vmovdqu		$xb2,0x10($out)
+	vmovdqu		$xc2,0x20($out)
+	vmovdqu		$xd2,0x30($out)
+	je		.Ldone4xop
+
+	lea		0x40($inp),$inp		# inp+=64*3
+	vmovdqa		$xa3,0x00(%rsp)
+	xor		%r9,%r9
+	vmovdqa		$xb3,0x10(%rsp)
+	lea		0x40($out),$out		# out+=64*3
+	vmovdqa		$xc3,0x20(%rsp)
+	sub		\$192,$len		# len-=64*3
+	vmovdqa		$xd3,0x30(%rsp)
+
+.Loop_tail4xop:
+	movzb		($inp,%r9),%eax
+	movzb		(%rsp,%r9),%ecx
+	lea		1(%r9),%r9
+	xor		%ecx,%eax
+	mov		%al,-1($out,%r9)
+	dec		$len
+	jnz		.Loop_tail4xop
+
+.Ldone4xop:
+	vzeroupper
+___
+$code.=<<___	if ($win64);
+	movaps		-0xb0(%r10),%xmm6
+	movaps		-0xa0(%r10),%xmm7
+	movaps		-0x90(%r10),%xmm8
+	movaps		-0x80(%r10),%xmm9
+	movaps		-0x70(%r10),%xmm10
+	movaps		-0x60(%r10),%xmm11
+	movaps		-0x50(%r10),%xmm12
+	movaps		-0x40(%r10),%xmm13
+	movaps		-0x30(%r10),%xmm14
+	movaps		-0x20(%r10),%xmm15
+___
+$code.=<<___;
+	lea		-8(%r10),%rsp
+.cfi_def_cfa_register	%rsp
+.L4xop_epilogue:
+	ret
+.cfi_endproc
+___
+&end_function("chacha20_xop");
+}
+
+########################################################################
+# AVX2 code path
+if ($avx>1) {
+
+if($kernel) {
+	$code .= "#ifdef CONFIG_AS_AVX2\n";
+}
+
+my ($xb0,$xb1,$xb2,$xb3, $xd0,$xd1,$xd2,$xd3,
+    $xa0,$xa1,$xa2,$xa3, $xt0,$xt1,$xt2,$xt3)=map("%ymm$_",(0..15));
+my @xx=($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3,
+	"%nox","%nox","%nox","%nox", $xd0,$xd1,$xd2,$xd3);
+
+sub AVX2_lane_ROUND {
+my ($a0,$b0,$c0,$d0)=@_;
+my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
+my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
+my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
+my ($xc,$xc_,$t0,$t1)=map("\"$_\"",$xt0,$xt1,$xt2,$xt3);
+my @x=map("\"$_\"",@xx);
+
+	# Consider order in which variables are addressed by their
+	# index:
+	#
+	#	a   b   c   d
+	#
+	#	0   4   8  12 < even round
+	#	1   5   9  13
+	#	2   6  10  14
+	#	3   7  11  15
+	#	0   5  10  15 < odd round
+	#	1   6  11  12
+	#	2   7   8  13
+	#	3   4   9  14
+	#
+	# 'a', 'b' and 'd's are permanently allocated in registers,
+	# @x[0..7,12..15], while 'c's are maintained in memory. If
+	# you observe 'c' column, you'll notice that pair of 'c's is
+	# invariant between rounds. This means that we have to reload
+	# them once per round, in the middle. This is why you'll see
+	# bunch of 'c' stores and loads in the middle, but none in
+	# the beginning or end.
+
+	(
+	"&vpaddd	(@x[$a0],@x[$a0],@x[$b0])",	# Q1
+	"&vpxor		(@x[$d0],@x[$a0],@x[$d0])",
+	"&vpshufb	(@x[$d0],@x[$d0],$t1)",
+	 "&vpaddd	(@x[$a1],@x[$a1],@x[$b1])",	# Q2
+	 "&vpxor	(@x[$d1],@x[$a1],@x[$d1])",
+	 "&vpshufb	(@x[$d1],@x[$d1],$t1)",
+
+	"&vpaddd	($xc,$xc,@x[$d0])",
+	"&vpxor		(@x[$b0],$xc,@x[$b0])",
+	"&vpslld	($t0,@x[$b0],12)",
+	"&vpsrld	(@x[$b0],@x[$b0],20)",
+	"&vpor		(@x[$b0],$t0,@x[$b0])",
+	"&vbroadcasti128($t0,'(%r11)')",		# .Lrot24(%rip)
+	 "&vpaddd	($xc_,$xc_,@x[$d1])",
+	 "&vpxor	(@x[$b1],$xc_,@x[$b1])",
+	 "&vpslld	($t1,@x[$b1],12)",
+	 "&vpsrld	(@x[$b1],@x[$b1],20)",
+	 "&vpor		(@x[$b1],$t1,@x[$b1])",
+
+	"&vpaddd	(@x[$a0],@x[$a0],@x[$b0])",
+	"&vpxor		(@x[$d0],@x[$a0],@x[$d0])",
+	"&vpshufb	(@x[$d0],@x[$d0],$t0)",
+	 "&vpaddd	(@x[$a1],@x[$a1],@x[$b1])",
+	 "&vpxor	(@x[$d1],@x[$a1],@x[$d1])",
+	 "&vpshufb	(@x[$d1],@x[$d1],$t0)",
+
+	"&vpaddd	($xc,$xc,@x[$d0])",
+	"&vpxor		(@x[$b0],$xc,@x[$b0])",
+	"&vpslld	($t1,@x[$b0],7)",
+	"&vpsrld	(@x[$b0],@x[$b0],25)",
+	"&vpor		(@x[$b0],$t1,@x[$b0])",
+	"&vbroadcasti128($t1,'(%r9)')",		# .Lrot16(%rip)
+	 "&vpaddd	($xc_,$xc_,@x[$d1])",
+	 "&vpxor	(@x[$b1],$xc_,@x[$b1])",
+	 "&vpslld	($t0,@x[$b1],7)",
+	 "&vpsrld	(@x[$b1],@x[$b1],25)",
+	 "&vpor		(@x[$b1],$t0,@x[$b1])",
+
+	"&vmovdqa	(\"`32*($c0-8)`(%rsp)\",$xc)",	# reload pair of 'c's
+	 "&vmovdqa	(\"`32*($c1-8)`(%rsp)\",$xc_)",
+	"&vmovdqa	($xc,\"`32*($c2-8)`(%rsp)\")",
+	 "&vmovdqa	($xc_,\"`32*($c3-8)`(%rsp)\")",
+
+	"&vpaddd	(@x[$a2],@x[$a2],@x[$b2])",	# Q3
+	"&vpxor		(@x[$d2],@x[$a2],@x[$d2])",
+	"&vpshufb	(@x[$d2],@x[$d2],$t1)",
+	 "&vpaddd	(@x[$a3],@x[$a3],@x[$b3])",	# Q4
+	 "&vpxor	(@x[$d3],@x[$a3],@x[$d3])",
+	 "&vpshufb	(@x[$d3],@x[$d3],$t1)",
+
+	"&vpaddd	($xc,$xc,@x[$d2])",
+	"&vpxor		(@x[$b2],$xc,@x[$b2])",
+	"&vpslld	($t0,@x[$b2],12)",
+	"&vpsrld	(@x[$b2],@x[$b2],20)",
+	"&vpor		(@x[$b2],$t0,@x[$b2])",
+	"&vbroadcasti128($t0,'(%r11)')",		# .Lrot24(%rip)
+	 "&vpaddd	($xc_,$xc_,@x[$d3])",
+	 "&vpxor	(@x[$b3],$xc_,@x[$b3])",
+	 "&vpslld	($t1,@x[$b3],12)",
+	 "&vpsrld	(@x[$b3],@x[$b3],20)",
+	 "&vpor		(@x[$b3],$t1,@x[$b3])",
+
+	"&vpaddd	(@x[$a2],@x[$a2],@x[$b2])",
+	"&vpxor		(@x[$d2],@x[$a2],@x[$d2])",
+	"&vpshufb	(@x[$d2],@x[$d2],$t0)",
+	 "&vpaddd	(@x[$a3],@x[$a3],@x[$b3])",
+	 "&vpxor	(@x[$d3],@x[$a3],@x[$d3])",
+	 "&vpshufb	(@x[$d3],@x[$d3],$t0)",
+
+	"&vpaddd	($xc,$xc,@x[$d2])",
+	"&vpxor		(@x[$b2],$xc,@x[$b2])",
+	"&vpslld	($t1,@x[$b2],7)",
+	"&vpsrld	(@x[$b2],@x[$b2],25)",
+	"&vpor		(@x[$b2],$t1,@x[$b2])",
+	"&vbroadcasti128($t1,'(%r9)')",		# .Lrot16(%rip)
+	 "&vpaddd	($xc_,$xc_,@x[$d3])",
+	 "&vpxor	(@x[$b3],$xc_,@x[$b3])",
+	 "&vpslld	($t0,@x[$b3],7)",
+	 "&vpsrld	(@x[$b3],@x[$b3],25)",
+	 "&vpor		(@x[$b3],$t0,@x[$b3])"
+	);
+}
+
+my $xframe = $win64 ? 0xa8 : 8;
+
+&declare_function("chacha20_avx2", 32, 5);
+$code.=<<___;
+.cfi_startproc
+.Lchacha20_8x:
+	lea		8(%rsp),%r10		# frame register
+.cfi_def_cfa_register	%r10
+	sub		\$0x280+$xframe,%rsp
+	and		\$-32,%rsp
+___
+$code.=<<___	if ($win64);
+	movaps		%xmm6,-0xb0(%r10)
+	movaps		%xmm7,-0xa0(%r10)
+	movaps		%xmm8,-0x90(%r10)
+	movaps		%xmm9,-0x80(%r10)
+	movaps		%xmm10,-0x70(%r10)
+	movaps		%xmm11,-0x60(%r10)
+	movaps		%xmm12,-0x50(%r10)
+	movaps		%xmm13,-0x40(%r10)
+	movaps		%xmm14,-0x30(%r10)
+	movaps		%xmm15,-0x20(%r10)
+.L8x_body:
+___
+$code.=<<___;
+	vzeroupper
+
+	################ stack layout
+	# +0x00		SIMD equivalent of @x[8-12]
+	# ...
+	# +0x80		constant copy of key[0-2] smashed by lanes
+	# ...
+	# +0x200	SIMD counters (with nonce smashed by lanes)
+	# ...
+	# +0x280
+
+	vbroadcasti128	.Lsigma(%rip),$xa3	# key[0]
+	vbroadcasti128	($key),$xb3		# key[1]
+	vbroadcasti128	16($key),$xt3		# key[2]
+	vbroadcasti128	($counter),$xd3		# key[3]
+	lea		0x100(%rsp),%rcx	# size optimization
+	lea		0x200(%rsp),%rax	# size optimization
+	lea		.Lrot16(%rip),%r9
+	lea		.Lrot24(%rip),%r11
+
+	vpshufd		\$0x00,$xa3,$xa0	# smash key by lanes...
+	vpshufd		\$0x55,$xa3,$xa1
+	vmovdqa		$xa0,0x80-0x100(%rcx)	# ... and offload
+	vpshufd		\$0xaa,$xa3,$xa2
+	vmovdqa		$xa1,0xa0-0x100(%rcx)
+	vpshufd		\$0xff,$xa3,$xa3
+	vmovdqa		$xa2,0xc0-0x100(%rcx)
+	vmovdqa		$xa3,0xe0-0x100(%rcx)
+
+	vpshufd		\$0x00,$xb3,$xb0
+	vpshufd		\$0x55,$xb3,$xb1
+	vmovdqa		$xb0,0x100-0x100(%rcx)
+	vpshufd		\$0xaa,$xb3,$xb2
+	vmovdqa		$xb1,0x120-0x100(%rcx)
+	vpshufd		\$0xff,$xb3,$xb3
+	vmovdqa		$xb2,0x140-0x100(%rcx)
+	vmovdqa		$xb3,0x160-0x100(%rcx)
+
+	vpshufd		\$0x00,$xt3,$xt0	# "xc0"
+	vpshufd		\$0x55,$xt3,$xt1	# "xc1"
+	vmovdqa		$xt0,0x180-0x200(%rax)
+	vpshufd		\$0xaa,$xt3,$xt2	# "xc2"
+	vmovdqa		$xt1,0x1a0-0x200(%rax)
+	vpshufd		\$0xff,$xt3,$xt3	# "xc3"
+	vmovdqa		$xt2,0x1c0-0x200(%rax)
+	vmovdqa		$xt3,0x1e0-0x200(%rax)
+
+	vpshufd		\$0x00,$xd3,$xd0
+	vpshufd		\$0x55,$xd3,$xd1
+	vpaddd		.Lincy(%rip),$xd0,$xd0	# don't save counters yet
+	vpshufd		\$0xaa,$xd3,$xd2
+	vmovdqa		$xd1,0x220-0x200(%rax)
+	vpshufd		\$0xff,$xd3,$xd3
+	vmovdqa		$xd2,0x240-0x200(%rax)
+	vmovdqa		$xd3,0x260-0x200(%rax)
+
+	jmp		.Loop_enter8x
+
+.align	32
+.Loop_outer8x:
+	vmovdqa		0x80-0x100(%rcx),$xa0	# re-load smashed key
+	vmovdqa		0xa0-0x100(%rcx),$xa1
+	vmovdqa		0xc0-0x100(%rcx),$xa2
+	vmovdqa		0xe0-0x100(%rcx),$xa3
+	vmovdqa		0x100-0x100(%rcx),$xb0
+	vmovdqa		0x120-0x100(%rcx),$xb1
+	vmovdqa		0x140-0x100(%rcx),$xb2
+	vmovdqa		0x160-0x100(%rcx),$xb3
+	vmovdqa		0x180-0x200(%rax),$xt0	# "xc0"
+	vmovdqa		0x1a0-0x200(%rax),$xt1	# "xc1"
+	vmovdqa		0x1c0-0x200(%rax),$xt2	# "xc2"
+	vmovdqa		0x1e0-0x200(%rax),$xt3	# "xc3"
+	vmovdqa		0x200-0x200(%rax),$xd0
+	vmovdqa		0x220-0x200(%rax),$xd1
+	vmovdqa		0x240-0x200(%rax),$xd2
+	vmovdqa		0x260-0x200(%rax),$xd3
+	vpaddd		.Leight(%rip),$xd0,$xd0	# next SIMD counters
+
+.Loop_enter8x:
+	vmovdqa		$xt2,0x40(%rsp)		# SIMD equivalent of "@x[10]"
+	vmovdqa		$xt3,0x60(%rsp)		# SIMD equivalent of "@x[11]"
+	vbroadcasti128	(%r9),$xt3
+	vmovdqa		$xd0,0x200-0x200(%rax)	# save SIMD counters
+	mov		\$10,%eax
+	jmp		.Loop8x
+
+.align	32
+.Loop8x:
+___
+	foreach (&AVX2_lane_ROUND(0, 4, 8,12)) { eval; }
+	foreach (&AVX2_lane_ROUND(0, 5,10,15)) { eval; }
+$code.=<<___;
+	dec		%eax
+	jnz		.Loop8x
+
+	lea		0x200(%rsp),%rax	# size optimization
+	vpaddd		0x80-0x100(%rcx),$xa0,$xa0	# accumulate key
+	vpaddd		0xa0-0x100(%rcx),$xa1,$xa1
+	vpaddd		0xc0-0x100(%rcx),$xa2,$xa2
+	vpaddd		0xe0-0x100(%rcx),$xa3,$xa3
+
+	vpunpckldq	$xa1,$xa0,$xt2		# "de-interlace" data
+	vpunpckldq	$xa3,$xa2,$xt3
+	vpunpckhdq	$xa1,$xa0,$xa0
+	vpunpckhdq	$xa3,$xa2,$xa2
+	vpunpcklqdq	$xt3,$xt2,$xa1		# "a0"
+	vpunpckhqdq	$xt3,$xt2,$xt2		# "a1"
+	vpunpcklqdq	$xa2,$xa0,$xa3		# "a2"
+	vpunpckhqdq	$xa2,$xa0,$xa0		# "a3"
+___
+	($xa0,$xa1,$xa2,$xa3,$xt2)=($xa1,$xt2,$xa3,$xa0,$xa2);
+$code.=<<___;
+	vpaddd		0x100-0x100(%rcx),$xb0,$xb0
+	vpaddd		0x120-0x100(%rcx),$xb1,$xb1
+	vpaddd		0x140-0x100(%rcx),$xb2,$xb2
+	vpaddd		0x160-0x100(%rcx),$xb3,$xb3
+
+	vpunpckldq	$xb1,$xb0,$xt2
+	vpunpckldq	$xb3,$xb2,$xt3
+	vpunpckhdq	$xb1,$xb0,$xb0
+	vpunpckhdq	$xb3,$xb2,$xb2
+	vpunpcklqdq	$xt3,$xt2,$xb1		# "b0"
+	vpunpckhqdq	$xt3,$xt2,$xt2		# "b1"
+	vpunpcklqdq	$xb2,$xb0,$xb3		# "b2"
+	vpunpckhqdq	$xb2,$xb0,$xb0		# "b3"
+___
+	($xb0,$xb1,$xb2,$xb3,$xt2)=($xb1,$xt2,$xb3,$xb0,$xb2);
+$code.=<<___;
+	vperm2i128	\$0x20,$xb0,$xa0,$xt3	# "de-interlace" further
+	vperm2i128	\$0x31,$xb0,$xa0,$xb0
+	vperm2i128	\$0x20,$xb1,$xa1,$xa0
+	vperm2i128	\$0x31,$xb1,$xa1,$xb1
+	vperm2i128	\$0x20,$xb2,$xa2,$xa1
+	vperm2i128	\$0x31,$xb2,$xa2,$xb2
+	vperm2i128	\$0x20,$xb3,$xa3,$xa2
+	vperm2i128	\$0x31,$xb3,$xa3,$xb3
+___
+	($xa0,$xa1,$xa2,$xa3,$xt3)=($xt3,$xa0,$xa1,$xa2,$xa3);
+	my ($xc0,$xc1,$xc2,$xc3)=($xt0,$xt1,$xa0,$xa1);
+$code.=<<___;
+	vmovdqa		$xa0,0x00(%rsp)		# offload $xaN
+	vmovdqa		$xa1,0x20(%rsp)
+	vmovdqa		0x40(%rsp),$xc2		# $xa0
+	vmovdqa		0x60(%rsp),$xc3		# $xa1
+
+	vpaddd		0x180-0x200(%rax),$xc0,$xc0
+	vpaddd		0x1a0-0x200(%rax),$xc1,$xc1
+	vpaddd		0x1c0-0x200(%rax),$xc2,$xc2
+	vpaddd		0x1e0-0x200(%rax),$xc3,$xc3
+
+	vpunpckldq	$xc1,$xc0,$xt2
+	vpunpckldq	$xc3,$xc2,$xt3
+	vpunpckhdq	$xc1,$xc0,$xc0
+	vpunpckhdq	$xc3,$xc2,$xc2
+	vpunpcklqdq	$xt3,$xt2,$xc1		# "c0"
+	vpunpckhqdq	$xt3,$xt2,$xt2		# "c1"
+	vpunpcklqdq	$xc2,$xc0,$xc3		# "c2"
+	vpunpckhqdq	$xc2,$xc0,$xc0		# "c3"
+___
+	($xc0,$xc1,$xc2,$xc3,$xt2)=($xc1,$xt2,$xc3,$xc0,$xc2);
+$code.=<<___;
+	vpaddd		0x200-0x200(%rax),$xd0,$xd0
+	vpaddd		0x220-0x200(%rax),$xd1,$xd1
+	vpaddd		0x240-0x200(%rax),$xd2,$xd2
+	vpaddd		0x260-0x200(%rax),$xd3,$xd3
+
+	vpunpckldq	$xd1,$xd0,$xt2
+	vpunpckldq	$xd3,$xd2,$xt3
+	vpunpckhdq	$xd1,$xd0,$xd0
+	vpunpckhdq	$xd3,$xd2,$xd2
+	vpunpcklqdq	$xt3,$xt2,$xd1		# "d0"
+	vpunpckhqdq	$xt3,$xt2,$xt2		# "d1"
+	vpunpcklqdq	$xd2,$xd0,$xd3		# "d2"
+	vpunpckhqdq	$xd2,$xd0,$xd0		# "d3"
+___
+	($xd0,$xd1,$xd2,$xd3,$xt2)=($xd1,$xt2,$xd3,$xd0,$xd2);
+$code.=<<___;
+	vperm2i128	\$0x20,$xd0,$xc0,$xt3	# "de-interlace" further
+	vperm2i128	\$0x31,$xd0,$xc0,$xd0
+	vperm2i128	\$0x20,$xd1,$xc1,$xc0
+	vperm2i128	\$0x31,$xd1,$xc1,$xd1
+	vperm2i128	\$0x20,$xd2,$xc2,$xc1
+	vperm2i128	\$0x31,$xd2,$xc2,$xd2
+	vperm2i128	\$0x20,$xd3,$xc3,$xc2
+	vperm2i128	\$0x31,$xd3,$xc3,$xd3
+___
+	($xc0,$xc1,$xc2,$xc3,$xt3)=($xt3,$xc0,$xc1,$xc2,$xc3);
+	($xb0,$xb1,$xb2,$xb3,$xc0,$xc1,$xc2,$xc3)=
+	($xc0,$xc1,$xc2,$xc3,$xb0,$xb1,$xb2,$xb3);
+	($xa0,$xa1)=($xt2,$xt3);
+$code.=<<___;
+	vmovdqa		0x00(%rsp),$xa0		# $xaN was offloaded, remember?
+	vmovdqa		0x20(%rsp),$xa1
+
+	cmp		\$64*8,$len
+	jb		.Ltail8x
+
+	vpxor		0x00($inp),$xa0,$xa0	# xor with input
+	vpxor		0x20($inp),$xb0,$xb0
+	vpxor		0x40($inp),$xc0,$xc0
+	vpxor		0x60($inp),$xd0,$xd0
+	lea		0x80($inp),$inp		# size optimization
+	vmovdqu		$xa0,0x00($out)
+	vmovdqu		$xb0,0x20($out)
+	vmovdqu		$xc0,0x40($out)
+	vmovdqu		$xd0,0x60($out)
+	lea		0x80($out),$out		# size optimization
+
+	vpxor		0x00($inp),$xa1,$xa1
+	vpxor		0x20($inp),$xb1,$xb1
+	vpxor		0x40($inp),$xc1,$xc1
+	vpxor		0x60($inp),$xd1,$xd1
+	lea		0x80($inp),$inp		# size optimization
+	vmovdqu		$xa1,0x00($out)
+	vmovdqu		$xb1,0x20($out)
+	vmovdqu		$xc1,0x40($out)
+	vmovdqu		$xd1,0x60($out)
+	lea		0x80($out),$out		# size optimization
+
+	vpxor		0x00($inp),$xa2,$xa2
+	vpxor		0x20($inp),$xb2,$xb2
+	vpxor		0x40($inp),$xc2,$xc2
+	vpxor		0x60($inp),$xd2,$xd2
+	lea		0x80($inp),$inp		# size optimization
+	vmovdqu		$xa2,0x00($out)
+	vmovdqu		$xb2,0x20($out)
+	vmovdqu		$xc2,0x40($out)
+	vmovdqu		$xd2,0x60($out)
+	lea		0x80($out),$out		# size optimization
+
+	vpxor		0x00($inp),$xa3,$xa3
+	vpxor		0x20($inp),$xb3,$xb3
+	vpxor		0x40($inp),$xc3,$xc3
+	vpxor		0x60($inp),$xd3,$xd3
+	lea		0x80($inp),$inp		# size optimization
+	vmovdqu		$xa3,0x00($out)
+	vmovdqu		$xb3,0x20($out)
+	vmovdqu		$xc3,0x40($out)
+	vmovdqu		$xd3,0x60($out)
+	lea		0x80($out),$out		# size optimization
+
+	sub		\$64*8,$len
+	jnz		.Loop_outer8x
+
+	jmp		.Ldone8x
+
+.Ltail8x:
+	cmp		\$448,$len
+	jae		.L448_or_more8x
+	cmp		\$384,$len
+	jae		.L384_or_more8x
+	cmp		\$320,$len
+	jae		.L320_or_more8x
+	cmp		\$256,$len
+	jae		.L256_or_more8x
+	cmp		\$192,$len
+	jae		.L192_or_more8x
+	cmp		\$128,$len
+	jae		.L128_or_more8x
+	cmp		\$64,$len
+	jae		.L64_or_more8x
+
+	xor		%r9,%r9
+	vmovdqa		$xa0,0x00(%rsp)
+	vmovdqa		$xb0,0x20(%rsp)
+	jmp		.Loop_tail8x
+
+.align	32
+.L64_or_more8x:
+	vpxor		0x00($inp),$xa0,$xa0	# xor with input
+	vpxor		0x20($inp),$xb0,$xb0
+	vmovdqu		$xa0,0x00($out)
+	vmovdqu		$xb0,0x20($out)
+	je		.Ldone8x
+
+	lea		0x40($inp),$inp		# inp+=64*1
+	xor		%r9,%r9
+	vmovdqa		$xc0,0x00(%rsp)
+	lea		0x40($out),$out		# out+=64*1
+	sub		\$64,$len		# len-=64*1
+	vmovdqa		$xd0,0x20(%rsp)
+	jmp		.Loop_tail8x
+
+.align	32
+.L128_or_more8x:
+	vpxor		0x00($inp),$xa0,$xa0	# xor with input
+	vpxor		0x20($inp),$xb0,$xb0
+	vpxor		0x40($inp),$xc0,$xc0
+	vpxor		0x60($inp),$xd0,$xd0
+	vmovdqu		$xa0,0x00($out)
+	vmovdqu		$xb0,0x20($out)
+	vmovdqu		$xc0,0x40($out)
+	vmovdqu		$xd0,0x60($out)
+	je		.Ldone8x
+
+	lea		0x80($inp),$inp		# inp+=64*2
+	xor		%r9,%r9
+	vmovdqa		$xa1,0x00(%rsp)
+	lea		0x80($out),$out		# out+=64*2
+	sub		\$128,$len		# len-=64*2
+	vmovdqa		$xb1,0x20(%rsp)
+	jmp		.Loop_tail8x
+
+.align	32
+.L192_or_more8x:
+	vpxor		0x00($inp),$xa0,$xa0	# xor with input
+	vpxor		0x20($inp),$xb0,$xb0
+	vpxor		0x40($inp),$xc0,$xc0
+	vpxor		0x60($inp),$xd0,$xd0
+	vpxor		0x80($inp),$xa1,$xa1
+	vpxor		0xa0($inp),$xb1,$xb1
+	vmovdqu		$xa0,0x00($out)
+	vmovdqu		$xb0,0x20($out)
+	vmovdqu		$xc0,0x40($out)
+	vmovdqu		$xd0,0x60($out)
+	vmovdqu		$xa1,0x80($out)
+	vmovdqu		$xb1,0xa0($out)
+	je		.Ldone8x
+
+	lea		0xc0($inp),$inp		# inp+=64*3
+	xor		%r9,%r9
+	vmovdqa		$xc1,0x00(%rsp)
+	lea		0xc0($out),$out		# out+=64*3
+	sub		\$192,$len		# len-=64*3
+	vmovdqa		$xd1,0x20(%rsp)
+	jmp		.Loop_tail8x
+
+.align	32
+.L256_or_more8x:
+	vpxor		0x00($inp),$xa0,$xa0	# xor with input
+	vpxor		0x20($inp),$xb0,$xb0
+	vpxor		0x40($inp),$xc0,$xc0
+	vpxor		0x60($inp),$xd0,$xd0
+	vpxor		0x80($inp),$xa1,$xa1
+	vpxor		0xa0($inp),$xb1,$xb1
+	vpxor		0xc0($inp),$xc1,$xc1
+	vpxor		0xe0($inp),$xd1,$xd1
+	vmovdqu		$xa0,0x00($out)
+	vmovdqu		$xb0,0x20($out)
+	vmovdqu		$xc0,0x40($out)
+	vmovdqu		$xd0,0x60($out)
+	vmovdqu		$xa1,0x80($out)
+	vmovdqu		$xb1,0xa0($out)
+	vmovdqu		$xc1,0xc0($out)
+	vmovdqu		$xd1,0xe0($out)
+	je		.Ldone8x
+
+	lea		0x100($inp),$inp	# inp+=64*4
+	xor		%r9,%r9
+	vmovdqa		$xa2,0x00(%rsp)
+	lea		0x100($out),$out	# out+=64*4
+	sub		\$256,$len		# len-=64*4
+	vmovdqa		$xb2,0x20(%rsp)
+	jmp		.Loop_tail8x
+
+.align	32
+.L320_or_more8x:
+	vpxor		0x00($inp),$xa0,$xa0	# xor with input
+	vpxor		0x20($inp),$xb0,$xb0
+	vpxor		0x40($inp),$xc0,$xc0
+	vpxor		0x60($inp),$xd0,$xd0
+	vpxor		0x80($inp),$xa1,$xa1
+	vpxor		0xa0($inp),$xb1,$xb1
+	vpxor		0xc0($inp),$xc1,$xc1
+	vpxor		0xe0($inp),$xd1,$xd1
+	vpxor		0x100($inp),$xa2,$xa2
+	vpxor		0x120($inp),$xb2,$xb2
+	vmovdqu		$xa0,0x00($out)
+	vmovdqu		$xb0,0x20($out)
+	vmovdqu		$xc0,0x40($out)
+	vmovdqu		$xd0,0x60($out)
+	vmovdqu		$xa1,0x80($out)
+	vmovdqu		$xb1,0xa0($out)
+	vmovdqu		$xc1,0xc0($out)
+	vmovdqu		$xd1,0xe0($out)
+	vmovdqu		$xa2,0x100($out)
+	vmovdqu		$xb2,0x120($out)
+	je		.Ldone8x
+
+	lea		0x140($inp),$inp	# inp+=64*5
+	xor		%r9,%r9
+	vmovdqa		$xc2,0x00(%rsp)
+	lea		0x140($out),$out	# out+=64*5
+	sub		\$320,$len		# len-=64*5
+	vmovdqa		$xd2,0x20(%rsp)
+	jmp		.Loop_tail8x
+
+.align	32
+.L384_or_more8x:
+	vpxor		0x00($inp),$xa0,$xa0	# xor with input
+	vpxor		0x20($inp),$xb0,$xb0
+	vpxor		0x40($inp),$xc0,$xc0
+	vpxor		0x60($inp),$xd0,$xd0
+	vpxor		0x80($inp),$xa1,$xa1
+	vpxor		0xa0($inp),$xb1,$xb1
+	vpxor		0xc0($inp),$xc1,$xc1
+	vpxor		0xe0($inp),$xd1,$xd1
+	vpxor		0x100($inp),$xa2,$xa2
+	vpxor		0x120($inp),$xb2,$xb2
+	vpxor		0x140($inp),$xc2,$xc2
+	vpxor		0x160($inp),$xd2,$xd2
+	vmovdqu		$xa0,0x00($out)
+	vmovdqu		$xb0,0x20($out)
+	vmovdqu		$xc0,0x40($out)
+	vmovdqu		$xd0,0x60($out)
+	vmovdqu		$xa1,0x80($out)
+	vmovdqu		$xb1,0xa0($out)
+	vmovdqu		$xc1,0xc0($out)
+	vmovdqu		$xd1,0xe0($out)
+	vmovdqu		$xa2,0x100($out)
+	vmovdqu		$xb2,0x120($out)
+	vmovdqu		$xc2,0x140($out)
+	vmovdqu		$xd2,0x160($out)
+	je		.Ldone8x
+
+	lea		0x180($inp),$inp	# inp+=64*6
+	xor		%r9,%r9
+	vmovdqa		$xa3,0x00(%rsp)
+	lea		0x180($out),$out	# out+=64*6
+	sub		\$384,$len		# len-=64*6
+	vmovdqa		$xb3,0x20(%rsp)
+	jmp		.Loop_tail8x
+
+.align	32
+.L448_or_more8x:
+	vpxor		0x00($inp),$xa0,$xa0	# xor with input
+	vpxor		0x20($inp),$xb0,$xb0
+	vpxor		0x40($inp),$xc0,$xc0
+	vpxor		0x60($inp),$xd0,$xd0
+	vpxor		0x80($inp),$xa1,$xa1
+	vpxor		0xa0($inp),$xb1,$xb1
+	vpxor		0xc0($inp),$xc1,$xc1
+	vpxor		0xe0($inp),$xd1,$xd1
+	vpxor		0x100($inp),$xa2,$xa2
+	vpxor		0x120($inp),$xb2,$xb2
+	vpxor		0x140($inp),$xc2,$xc2
+	vpxor		0x160($inp),$xd2,$xd2
+	vpxor		0x180($inp),$xa3,$xa3
+	vpxor		0x1a0($inp),$xb3,$xb3
+	vmovdqu		$xa0,0x00($out)
+	vmovdqu		$xb0,0x20($out)
+	vmovdqu		$xc0,0x40($out)
+	vmovdqu		$xd0,0x60($out)
+	vmovdqu		$xa1,0x80($out)
+	vmovdqu		$xb1,0xa0($out)
+	vmovdqu		$xc1,0xc0($out)
+	vmovdqu		$xd1,0xe0($out)
+	vmovdqu		$xa2,0x100($out)
+	vmovdqu		$xb2,0x120($out)
+	vmovdqu		$xc2,0x140($out)
+	vmovdqu		$xd2,0x160($out)
+	vmovdqu		$xa3,0x180($out)
+	vmovdqu		$xb3,0x1a0($out)
+	je		.Ldone8x
+
+	lea		0x1c0($inp),$inp	# inp+=64*7
+	xor		%r9,%r9
+	vmovdqa		$xc3,0x00(%rsp)
+	lea		0x1c0($out),$out	# out+=64*7
+	sub		\$448,$len		# len-=64*7
+	vmovdqa		$xd3,0x20(%rsp)
+
+.Loop_tail8x:
+	movzb		($inp,%r9),%eax
+	movzb		(%rsp,%r9),%ecx
+	lea		1(%r9),%r9
+	xor		%ecx,%eax
+	mov		%al,-1($out,%r9)
+	dec		$len
+	jnz		.Loop_tail8x
+
+.Ldone8x:
+	vzeroall
+___
+$code.=<<___	if ($win64);
+	movaps		-0xb0(%r10),%xmm6
+	movaps		-0xa0(%r10),%xmm7
+	movaps		-0x90(%r10),%xmm8
+	movaps		-0x80(%r10),%xmm9
+	movaps		-0x70(%r10),%xmm10
+	movaps		-0x60(%r10),%xmm11
+	movaps		-0x50(%r10),%xmm12
+	movaps		-0x40(%r10),%xmm13
+	movaps		-0x30(%r10),%xmm14
+	movaps		-0x20(%r10),%xmm15
+___
+$code.=<<___;
+	lea		-8(%r10),%rsp
+.cfi_def_cfa_register	%rsp
+.L8x_epilogue:
+	ret
+.cfi_endproc
+___
+&end_function("chacha20_avx2");
+if($kernel) {
+	$code .= "#endif\n";
+}
+}
+
+########################################################################
+# AVX512 code paths
+if ($avx>2) {
+# This one handles shorter inputs...
+if($kernel) {
+	$code .= "#ifdef CONFIG_AS_AVX512\n";
+}
+
+my ($a,$b,$c,$d, $a_,$b_,$c_,$d_,$fourz) = map("%zmm$_",(0..3,16..20));
+my ($t0,$t1,$t2,$t3) = map("%xmm$_",(4..7));
+
+sub vpxord()		# size optimization
+{ my $opcode = "vpxor";	# adhere to vpxor when possible
+
+    foreach (@_) {
+	if (/%([zy])mm([0-9]+)/ && ($1 eq "z" || $2>=16)) {
+	    $opcode = "vpxord";
+	    last;
+	}
+    }
+
+    $code .= "\t$opcode\t".join(',',reverse @_)."\n";
+}
+
+sub AVX512ROUND {	# critical path is 14 "SIMD ticks" per round
+	&vpaddd	($a,$a,$b);
+	&vpxord	($d,$d,$a);
+	&vprold	($d,$d,16);
+
+	&vpaddd	($c,$c,$d);
+	&vpxord	($b,$b,$c);
+	&vprold	($b,$b,12);
+
+	&vpaddd	($a,$a,$b);
+	&vpxord	($d,$d,$a);
+	&vprold	($d,$d,8);
+
+	&vpaddd	($c,$c,$d);
+	&vpxord	($b,$b,$c);
+	&vprold	($b,$b,7);
+}
+
+my $xframe = $win64 ? 32+8 : 8;
+
+&declare_function("chacha20_avx512", 32, 5);
+$code.=<<___;
+.cfi_startproc
+.Lchacha20_avx512:
+	lea	8(%rsp),%r10		# frame pointer
+.cfi_def_cfa_register	%r10
+	cmp	\$512,$len
+	ja	.Lchacha20_16x
+
+	sub	\$64+$xframe,%rsp
+	and \$-64,%rsp
+___
+$code.=<<___	if ($win64);
+	movaps	%xmm6,-0x30(%r10)
+	movaps	%xmm7,-0x20(%r10)
+.Lavx512_body:
+___
+$code.=<<___;
+	vbroadcasti32x4	.Lsigma(%rip),$a
+	vbroadcasti32x4	($key),$b
+	vbroadcasti32x4	16($key),$c
+	vbroadcasti32x4	($counter),$d
+
+	vmovdqa32	$a,$a_
+	vmovdqa32	$b,$b_
+	vmovdqa32	$c,$c_
+	vpaddd		.Lzeroz(%rip),$d,$d
+	vmovdqa32	.Lfourz(%rip),$fourz
+	mov		\$10,$counter	# reuse $counter
+	vmovdqa32	$d,$d_
+	jmp		.Loop_avx512
+
+.align	16
+.Loop_outer_avx512:
+	vmovdqa32	$a_,$a
+	vmovdqa32	$b_,$b
+	vmovdqa32	$c_,$c
+	vpaddd		$fourz,$d_,$d
+	mov		\$10,$counter
+	vmovdqa32	$d,$d_
+	jmp		.Loop_avx512
+
+.align	32
+.Loop_avx512:
+___
+	&AVX512ROUND();
+	&vpshufd	($c,$c,0b01001110);
+	&vpshufd	($b,$b,0b00111001);
+	&vpshufd	($d,$d,0b10010011);
+
+	&AVX512ROUND();
+	&vpshufd	($c,$c,0b01001110);
+	&vpshufd	($b,$b,0b10010011);
+	&vpshufd	($d,$d,0b00111001);
+
+	&dec		($counter);
+	&jnz		(".Loop_avx512");
+
+$code.=<<___;
+	vpaddd		$a_,$a,$a
+	vpaddd		$b_,$b,$b
+	vpaddd		$c_,$c,$c
+	vpaddd		$d_,$d,$d
+
+	sub		\$64,$len
+	jb		.Ltail64_avx512
+
+	vpxor		0x00($inp),%x#$a,$t0	# xor with input
+	vpxor		0x10($inp),%x#$b,$t1
+	vpxor		0x20($inp),%x#$c,$t2
+	vpxor		0x30($inp),%x#$d,$t3
+	lea		0x40($inp),$inp		# inp+=64
+
+	vmovdqu		$t0,0x00($out)		# write output
+	vmovdqu		$t1,0x10($out)
+	vmovdqu		$t2,0x20($out)
+	vmovdqu		$t3,0x30($out)
+	lea		0x40($out),$out		# out+=64
+
+	jz		.Ldone_avx512
+
+	vextracti32x4	\$1,$a,$t0
+	vextracti32x4	\$1,$b,$t1
+	vextracti32x4	\$1,$c,$t2
+	vextracti32x4	\$1,$d,$t3
+
+	sub		\$64,$len
+	jb		.Ltail_avx512
+
+	vpxor		0x00($inp),$t0,$t0	# xor with input
+	vpxor		0x10($inp),$t1,$t1
+	vpxor		0x20($inp),$t2,$t2
+	vpxor		0x30($inp),$t3,$t3
+	lea		0x40($inp),$inp		# inp+=64
+
+	vmovdqu		$t0,0x00($out)		# write output
+	vmovdqu		$t1,0x10($out)
+	vmovdqu		$t2,0x20($out)
+	vmovdqu		$t3,0x30($out)
+	lea		0x40($out),$out		# out+=64
+
+	jz		.Ldone_avx512
+
+	vextracti32x4	\$2,$a,$t0
+	vextracti32x4	\$2,$b,$t1
+	vextracti32x4	\$2,$c,$t2
+	vextracti32x4	\$2,$d,$t3
+
+	sub		\$64,$len
+	jb		.Ltail_avx512
+
+	vpxor		0x00($inp),$t0,$t0	# xor with input
+	vpxor		0x10($inp),$t1,$t1
+	vpxor		0x20($inp),$t2,$t2
+	vpxor		0x30($inp),$t3,$t3
+	lea		0x40($inp),$inp		# inp+=64
+
+	vmovdqu		$t0,0x00($out)		# write output
+	vmovdqu		$t1,0x10($out)
+	vmovdqu		$t2,0x20($out)
+	vmovdqu		$t3,0x30($out)
+	lea		0x40($out),$out		# out+=64
+
+	jz		.Ldone_avx512
+
+	vextracti32x4	\$3,$a,$t0
+	vextracti32x4	\$3,$b,$t1
+	vextracti32x4	\$3,$c,$t2
+	vextracti32x4	\$3,$d,$t3
+
+	sub		\$64,$len
+	jb		.Ltail_avx512
+
+	vpxor		0x00($inp),$t0,$t0	# xor with input
+	vpxor		0x10($inp),$t1,$t1
+	vpxor		0x20($inp),$t2,$t2
+	vpxor		0x30($inp),$t3,$t3
+	lea		0x40($inp),$inp		# inp+=64
+
+	vmovdqu		$t0,0x00($out)		# write output
+	vmovdqu		$t1,0x10($out)
+	vmovdqu		$t2,0x20($out)
+	vmovdqu		$t3,0x30($out)
+	lea		0x40($out),$out		# out+=64
+
+	jnz		.Loop_outer_avx512
+
+	jmp		.Ldone_avx512
+
+.align	16
+.Ltail64_avx512:
+	vmovdqa		%x#$a,0x00(%rsp)
+	vmovdqa		%x#$b,0x10(%rsp)
+	vmovdqa		%x#$c,0x20(%rsp)
+	vmovdqa		%x#$d,0x30(%rsp)
+	add		\$64,$len
+	jmp		.Loop_tail_avx512
+
+.align	16
+.Ltail_avx512:
+	vmovdqa		$t0,0x00(%rsp)
+	vmovdqa		$t1,0x10(%rsp)
+	vmovdqa		$t2,0x20(%rsp)
+	vmovdqa		$t3,0x30(%rsp)
+	add		\$64,$len
+
+.Loop_tail_avx512:
+	movzb		($inp,$counter),%eax
+	movzb		(%rsp,$counter),%ecx
+	lea		1($counter),$counter
+	xor		%ecx,%eax
+	mov		%al,-1($out,$counter)
+	dec		$len
+	jnz		.Loop_tail_avx512
+
+	vmovdqu32	$a_,0x00(%rsp)
+
+.Ldone_avx512:
+	vzeroall
+___
+$code.=<<___	if ($win64);
+	movaps	-0x30(%r10),%xmm6
+	movaps	-0x20(%r10),%xmm7
+___
+$code.=<<___;
+	lea	-8(%r10),%rsp
+.cfi_def_cfa_register	%rsp
+.Lavx512_epilogue:
+	ret
+.cfi_endproc
+___
+&end_function("chacha20_avx512");
+
+map(s/%z/%y/, $a,$b,$c,$d, $a_,$b_,$c_,$d_,$fourz);
+
+&declare_function("chacha20_avx512vl", 32, 5);
+$code.=<<___;
+.cfi_startproc
+.Lchacha20_avx512vl:
+	lea	8(%rsp),%r10		# frame pointer
+.cfi_def_cfa_register	%r10
+	cmp	\$128,$len
+	ja	.Lchacha20_8xvl
+
+	sub	\$64+$xframe,%rsp
+	and \$-32,%rsp
+___
+$code.=<<___	if ($win64);
+	movaps	%xmm6,-0x30(%r10)
+	movaps	%xmm7,-0x20(%r10)
+.Lavx512vl_body:
+___
+$code.=<<___;
+	vbroadcasti128	.Lsigma(%rip),$a
+	vbroadcasti128	($key),$b
+	vbroadcasti128	16($key),$c
+	vbroadcasti128	($counter),$d
+
+	vmovdqa32	$a,$a_
+	vmovdqa32	$b,$b_
+	vmovdqa32	$c,$c_
+	vpaddd		.Lzeroz(%rip),$d,$d
+	vmovdqa32	.Ltwoy(%rip),$fourz
+	mov		\$10,$counter	# reuse $counter
+	vmovdqa32	$d,$d_
+	jmp		.Loop_avx512vl
+
+.align	16
+.Loop_outer_avx512vl:
+	vmovdqa32	$c_,$c
+	vpaddd		$fourz,$d_,$d
+	mov		\$10,$counter
+	vmovdqa32	$d,$d_
+	jmp		.Loop_avx512vl
+
+.align	32
+.Loop_avx512vl:
+___
+	&AVX512ROUND();
+	&vpshufd	($c,$c,0b01001110);
+	&vpshufd	($b,$b,0b00111001);
+	&vpshufd	($d,$d,0b10010011);
+
+	&AVX512ROUND();
+	&vpshufd	($c,$c,0b01001110);
+	&vpshufd	($b,$b,0b10010011);
+	&vpshufd	($d,$d,0b00111001);
+
+	&dec		($counter);
+	&jnz		(".Loop_avx512vl");
+
+$code.=<<___;
+	vpaddd		$a_,$a,$a
+	vpaddd		$b_,$b,$b
+	vpaddd		$c_,$c,$c
+	vpaddd		$d_,$d,$d
+
+	sub		\$64,$len
+	jb		.Ltail64_avx512vl
+
+	vpxor		0x00($inp),%x#$a,$t0	# xor with input
+	vpxor		0x10($inp),%x#$b,$t1
+	vpxor		0x20($inp),%x#$c,$t2
+	vpxor		0x30($inp),%x#$d,$t3
+	lea		0x40($inp),$inp		# inp+=64
+
+	vmovdqu		$t0,0x00($out)		# write output
+	vmovdqu		$t1,0x10($out)
+	vmovdqu		$t2,0x20($out)
+	vmovdqu		$t3,0x30($out)
+	lea		0x40($out),$out		# out+=64
+
+	jz		.Ldone_avx512vl
+
+	vextracti128	\$1,$a,$t0
+	vextracti128	\$1,$b,$t1
+	vextracti128	\$1,$c,$t2
+	vextracti128	\$1,$d,$t3
+
+	sub		\$64,$len
+	jb		.Ltail_avx512vl
+
+	vpxor		0x00($inp),$t0,$t0	# xor with input
+	vpxor		0x10($inp),$t1,$t1
+	vpxor		0x20($inp),$t2,$t2
+	vpxor		0x30($inp),$t3,$t3
+	lea		0x40($inp),$inp		# inp+=64
+
+	vmovdqu		$t0,0x00($out)		# write output
+	vmovdqu		$t1,0x10($out)
+	vmovdqu		$t2,0x20($out)
+	vmovdqu		$t3,0x30($out)
+	lea		0x40($out),$out		# out+=64
+
+	vmovdqa32	$a_,$a
+	vmovdqa32	$b_,$b
+	jnz		.Loop_outer_avx512vl
+
+	jmp		.Ldone_avx512vl
+
+.align	16
+.Ltail64_avx512vl:
+	vmovdqa		%x#$a,0x00(%rsp)
+	vmovdqa		%x#$b,0x10(%rsp)
+	vmovdqa		%x#$c,0x20(%rsp)
+	vmovdqa		%x#$d,0x30(%rsp)
+	add		\$64,$len
+	jmp		.Loop_tail_avx512vl
+
+.align	16
+.Ltail_avx512vl:
+	vmovdqa		$t0,0x00(%rsp)
+	vmovdqa		$t1,0x10(%rsp)
+	vmovdqa		$t2,0x20(%rsp)
+	vmovdqa		$t3,0x30(%rsp)
+	add		\$64,$len
+
+.Loop_tail_avx512vl:
+	movzb		($inp,$counter),%eax
+	movzb		(%rsp,$counter),%ecx
+	lea		1($counter),$counter
+	xor		%ecx,%eax
+	mov		%al,-1($out,$counter)
+	dec		$len
+	jnz		.Loop_tail_avx512vl
+
+	vmovdqu32	$a_,0x00(%rsp)
+	vmovdqu32	$a_,0x20(%rsp)
+
+.Ldone_avx512vl:
+	vzeroall
+___
+$code.=<<___	if ($win64);
+	movaps	-0x30(%r10),%xmm6
+	movaps	-0x20(%r10),%xmm7
+___
+$code.=<<___;
+	lea	-8(%r10),%rsp
+.cfi_def_cfa_register	%rsp
+.Lavx512vl_epilogue:
+	ret
+.cfi_endproc
+___
+&end_function("chacha20_avx512vl");
+
+# This one handles longer inputs...
+
+my ($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3,
+    $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3)=map("%zmm$_",(0..15));
+my  @xx=($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3,
+	 $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3);
+my @key=map("%zmm$_",(16..31));
+my ($xt0,$xt1,$xt2,$xt3)=@key[0..3];
+
+sub AVX512_lane_ROUND {
+my ($a0,$b0,$c0,$d0)=@_;
+my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
+my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
+my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
+my @x=map("\"$_\"",@xx);
+
+	(
+	"&vpaddd	(@x[$a0],@x[$a0],@x[$b0])",	# Q1
+	 "&vpaddd	(@x[$a1],@x[$a1],@x[$b1])",	# Q2
+	  "&vpaddd	(@x[$a2],@x[$a2],@x[$b2])",	# Q3
+	   "&vpaddd	(@x[$a3],@x[$a3],@x[$b3])",	# Q4
+	"&vpxord	(@x[$d0],@x[$d0],@x[$a0])",
+	 "&vpxord	(@x[$d1],@x[$d1],@x[$a1])",
+	  "&vpxord	(@x[$d2],@x[$d2],@x[$a2])",
+	   "&vpxord	(@x[$d3],@x[$d3],@x[$a3])",
+	"&vprold	(@x[$d0],@x[$d0],16)",
+	 "&vprold	(@x[$d1],@x[$d1],16)",
+	  "&vprold	(@x[$d2],@x[$d2],16)",
+	   "&vprold	(@x[$d3],@x[$d3],16)",
+
+	"&vpaddd	(@x[$c0],@x[$c0],@x[$d0])",
+	 "&vpaddd	(@x[$c1],@x[$c1],@x[$d1])",
+	  "&vpaddd	(@x[$c2],@x[$c2],@x[$d2])",
+	   "&vpaddd	(@x[$c3],@x[$c3],@x[$d3])",
+	"&vpxord	(@x[$b0],@x[$b0],@x[$c0])",
+	 "&vpxord	(@x[$b1],@x[$b1],@x[$c1])",
+	  "&vpxord	(@x[$b2],@x[$b2],@x[$c2])",
+	   "&vpxord	(@x[$b3],@x[$b3],@x[$c3])",
+	"&vprold	(@x[$b0],@x[$b0],12)",
+	 "&vprold	(@x[$b1],@x[$b1],12)",
+	  "&vprold	(@x[$b2],@x[$b2],12)",
+	   "&vprold	(@x[$b3],@x[$b3],12)",
+
+	"&vpaddd	(@x[$a0],@x[$a0],@x[$b0])",
+	 "&vpaddd	(@x[$a1],@x[$a1],@x[$b1])",
+	  "&vpaddd	(@x[$a2],@x[$a2],@x[$b2])",
+	   "&vpaddd	(@x[$a3],@x[$a3],@x[$b3])",
+	"&vpxord	(@x[$d0],@x[$d0],@x[$a0])",
+	 "&vpxord	(@x[$d1],@x[$d1],@x[$a1])",
+	  "&vpxord	(@x[$d2],@x[$d2],@x[$a2])",
+	   "&vpxord	(@x[$d3],@x[$d3],@x[$a3])",
+	"&vprold	(@x[$d0],@x[$d0],8)",
+	 "&vprold	(@x[$d1],@x[$d1],8)",
+	  "&vprold	(@x[$d2],@x[$d2],8)",
+	   "&vprold	(@x[$d3],@x[$d3],8)",
+
+	"&vpaddd	(@x[$c0],@x[$c0],@x[$d0])",
+	 "&vpaddd	(@x[$c1],@x[$c1],@x[$d1])",
+	  "&vpaddd	(@x[$c2],@x[$c2],@x[$d2])",
+	   "&vpaddd	(@x[$c3],@x[$c3],@x[$d3])",
+	"&vpxord	(@x[$b0],@x[$b0],@x[$c0])",
+	 "&vpxord	(@x[$b1],@x[$b1],@x[$c1])",
+	  "&vpxord	(@x[$b2],@x[$b2],@x[$c2])",
+	   "&vpxord	(@x[$b3],@x[$b3],@x[$c3])",
+	"&vprold	(@x[$b0],@x[$b0],7)",
+	 "&vprold	(@x[$b1],@x[$b1],7)",
+	  "&vprold	(@x[$b2],@x[$b2],7)",
+	   "&vprold	(@x[$b3],@x[$b3],7)"
+	);
+}
+
+my $xframe = $win64 ? 0xa8 : 8;
+
+$code.=<<___;
+.type	chacha20_16x,\@function,5
+.align	32
+chacha20_16x:
+.cfi_startproc
+.Lchacha20_16x:
+	lea		8(%rsp),%r10		# frame register
+.cfi_def_cfa_register	%r10
+	sub		\$64+$xframe,%rsp
+	and		\$-64,%rsp
+___
+$code.=<<___	if ($win64);
+	movaps		%xmm6,-0xb0(%r10)
+	movaps		%xmm7,-0xa0(%r10)
+	movaps		%xmm8,-0x90(%r10)
+	movaps		%xmm9,-0x80(%r10)
+	movaps		%xmm10,-0x70(%r10)
+	movaps		%xmm11,-0x60(%r10)
+	movaps		%xmm12,-0x50(%r10)
+	movaps		%xmm13,-0x40(%r10)
+	movaps		%xmm14,-0x30(%r10)
+	movaps		%xmm15,-0x20(%r10)
+.L16x_body:
+___
+$code.=<<___;
+	vzeroupper
+
+	lea		.Lsigma(%rip),%r9
+	vbroadcasti32x4	(%r9),$xa3		# key[0]
+	vbroadcasti32x4	($key),$xb3		# key[1]
+	vbroadcasti32x4	16($key),$xc3		# key[2]
+	vbroadcasti32x4	($counter),$xd3		# key[3]
+
+	vpshufd		\$0x00,$xa3,$xa0	# smash key by lanes...
+	vpshufd		\$0x55,$xa3,$xa1
+	vpshufd		\$0xaa,$xa3,$xa2
+	vpshufd		\$0xff,$xa3,$xa3
+	vmovdqa64	$xa0,@key[0]
+	vmovdqa64	$xa1,@key[1]
+	vmovdqa64	$xa2,@key[2]
+	vmovdqa64	$xa3,@key[3]
+
+	vpshufd		\$0x00,$xb3,$xb0
+	vpshufd		\$0x55,$xb3,$xb1
+	vpshufd		\$0xaa,$xb3,$xb2
+	vpshufd		\$0xff,$xb3,$xb3
+	vmovdqa64	$xb0,@key[4]
+	vmovdqa64	$xb1,@key[5]
+	vmovdqa64	$xb2,@key[6]
+	vmovdqa64	$xb3,@key[7]
+
+	vpshufd		\$0x00,$xc3,$xc0
+	vpshufd		\$0x55,$xc3,$xc1
+	vpshufd		\$0xaa,$xc3,$xc2
+	vpshufd		\$0xff,$xc3,$xc3
+	vmovdqa64	$xc0,@key[8]
+	vmovdqa64	$xc1,@key[9]
+	vmovdqa64	$xc2,@key[10]
+	vmovdqa64	$xc3,@key[11]
+
+	vpshufd		\$0x00,$xd3,$xd0
+	vpshufd		\$0x55,$xd3,$xd1
+	vpshufd		\$0xaa,$xd3,$xd2
+	vpshufd		\$0xff,$xd3,$xd3
+	vpaddd		.Lincz(%rip),$xd0,$xd0	# don't save counters yet
+	vmovdqa64	$xd0,@key[12]
+	vmovdqa64	$xd1,@key[13]
+	vmovdqa64	$xd2,@key[14]
+	vmovdqa64	$xd3,@key[15]
+
+	mov		\$10,%eax
+	jmp		.Loop16x
+
+.align	32
+.Loop_outer16x:
+	vpbroadcastd	0(%r9),$xa0		# reload key
+	vpbroadcastd	4(%r9),$xa1
+	vpbroadcastd	8(%r9),$xa2
+	vpbroadcastd	12(%r9),$xa3
+	vpaddd		.Lsixteen(%rip),@key[12],@key[12]	# next SIMD counters
+	vmovdqa64	@key[4],$xb0
+	vmovdqa64	@key[5],$xb1
+	vmovdqa64	@key[6],$xb2
+	vmovdqa64	@key[7],$xb3
+	vmovdqa64	@key[8],$xc0
+	vmovdqa64	@key[9],$xc1
+	vmovdqa64	@key[10],$xc2
+	vmovdqa64	@key[11],$xc3
+	vmovdqa64	@key[12],$xd0
+	vmovdqa64	@key[13],$xd1
+	vmovdqa64	@key[14],$xd2
+	vmovdqa64	@key[15],$xd3
+
+	vmovdqa64	$xa0,@key[0]
+	vmovdqa64	$xa1,@key[1]
+	vmovdqa64	$xa2,@key[2]
+	vmovdqa64	$xa3,@key[3]
+
+	mov		\$10,%eax
+	jmp		.Loop16x
+
+.align	32
+.Loop16x:
+___
+	foreach (&AVX512_lane_ROUND(0, 4, 8,12)) { eval; }
+	foreach (&AVX512_lane_ROUND(0, 5,10,15)) { eval; }
+$code.=<<___;
+	dec		%eax
+	jnz		.Loop16x
+
+	vpaddd		@key[0],$xa0,$xa0	# accumulate key
+	vpaddd		@key[1],$xa1,$xa1
+	vpaddd		@key[2],$xa2,$xa2
+	vpaddd		@key[3],$xa3,$xa3
+
+	vpunpckldq	$xa1,$xa0,$xt2		# "de-interlace" data
+	vpunpckldq	$xa3,$xa2,$xt3
+	vpunpckhdq	$xa1,$xa0,$xa0
+	vpunpckhdq	$xa3,$xa2,$xa2
+	vpunpcklqdq	$xt3,$xt2,$xa1		# "a0"
+	vpunpckhqdq	$xt3,$xt2,$xt2		# "a1"
+	vpunpcklqdq	$xa2,$xa0,$xa3		# "a2"
+	vpunpckhqdq	$xa2,$xa0,$xa0		# "a3"
+___
+	($xa0,$xa1,$xa2,$xa3,$xt2)=($xa1,$xt2,$xa3,$xa0,$xa2);
+$code.=<<___;
+	vpaddd		@key[4],$xb0,$xb0
+	vpaddd		@key[5],$xb1,$xb1
+	vpaddd		@key[6],$xb2,$xb2
+	vpaddd		@key[7],$xb3,$xb3
+
+	vpunpckldq	$xb1,$xb0,$xt2
+	vpunpckldq	$xb3,$xb2,$xt3
+	vpunpckhdq	$xb1,$xb0,$xb0
+	vpunpckhdq	$xb3,$xb2,$xb2
+	vpunpcklqdq	$xt3,$xt2,$xb1		# "b0"
+	vpunpckhqdq	$xt3,$xt2,$xt2		# "b1"
+	vpunpcklqdq	$xb2,$xb0,$xb3		# "b2"
+	vpunpckhqdq	$xb2,$xb0,$xb0		# "b3"
+___
+	($xb0,$xb1,$xb2,$xb3,$xt2)=($xb1,$xt2,$xb3,$xb0,$xb2);
+$code.=<<___;
+	vshufi32x4	\$0x44,$xb0,$xa0,$xt3	# "de-interlace" further
+	vshufi32x4	\$0xee,$xb0,$xa0,$xb0
+	vshufi32x4	\$0x44,$xb1,$xa1,$xa0
+	vshufi32x4	\$0xee,$xb1,$xa1,$xb1
+	vshufi32x4	\$0x44,$xb2,$xa2,$xa1
+	vshufi32x4	\$0xee,$xb2,$xa2,$xb2
+	vshufi32x4	\$0x44,$xb3,$xa3,$xa2
+	vshufi32x4	\$0xee,$xb3,$xa3,$xb3
+___
+	($xa0,$xa1,$xa2,$xa3,$xt3)=($xt3,$xa0,$xa1,$xa2,$xa3);
+$code.=<<___;
+	vpaddd		@key[8],$xc0,$xc0
+	vpaddd		@key[9],$xc1,$xc1
+	vpaddd		@key[10],$xc2,$xc2
+	vpaddd		@key[11],$xc3,$xc3
+
+	vpunpckldq	$xc1,$xc0,$xt2
+	vpunpckldq	$xc3,$xc2,$xt3
+	vpunpckhdq	$xc1,$xc0,$xc0
+	vpunpckhdq	$xc3,$xc2,$xc2
+	vpunpcklqdq	$xt3,$xt2,$xc1		# "c0"
+	vpunpckhqdq	$xt3,$xt2,$xt2		# "c1"
+	vpunpcklqdq	$xc2,$xc0,$xc3		# "c2"
+	vpunpckhqdq	$xc2,$xc0,$xc0		# "c3"
+___
+	($xc0,$xc1,$xc2,$xc3,$xt2)=($xc1,$xt2,$xc3,$xc0,$xc2);
+$code.=<<___;
+	vpaddd		@key[12],$xd0,$xd0
+	vpaddd		@key[13],$xd1,$xd1
+	vpaddd		@key[14],$xd2,$xd2
+	vpaddd		@key[15],$xd3,$xd3
+
+	vpunpckldq	$xd1,$xd0,$xt2
+	vpunpckldq	$xd3,$xd2,$xt3
+	vpunpckhdq	$xd1,$xd0,$xd0
+	vpunpckhdq	$xd3,$xd2,$xd2
+	vpunpcklqdq	$xt3,$xt2,$xd1		# "d0"
+	vpunpckhqdq	$xt3,$xt2,$xt2		# "d1"
+	vpunpcklqdq	$xd2,$xd0,$xd3		# "d2"
+	vpunpckhqdq	$xd2,$xd0,$xd0		# "d3"
+___
+	($xd0,$xd1,$xd2,$xd3,$xt2)=($xd1,$xt2,$xd3,$xd0,$xd2);
+$code.=<<___;
+	vshufi32x4	\$0x44,$xd0,$xc0,$xt3	# "de-interlace" further
+	vshufi32x4	\$0xee,$xd0,$xc0,$xd0
+	vshufi32x4	\$0x44,$xd1,$xc1,$xc0
+	vshufi32x4	\$0xee,$xd1,$xc1,$xd1
+	vshufi32x4	\$0x44,$xd2,$xc2,$xc1
+	vshufi32x4	\$0xee,$xd2,$xc2,$xd2
+	vshufi32x4	\$0x44,$xd3,$xc3,$xc2
+	vshufi32x4	\$0xee,$xd3,$xc3,$xd3
+___
+	($xc0,$xc1,$xc2,$xc3,$xt3)=($xt3,$xc0,$xc1,$xc2,$xc3);
+$code.=<<___;
+	vshufi32x4	\$0x88,$xc0,$xa0,$xt0	# "de-interlace" further
+	vshufi32x4	\$0xdd,$xc0,$xa0,$xa0
+	 vshufi32x4	\$0x88,$xd0,$xb0,$xc0
+	 vshufi32x4	\$0xdd,$xd0,$xb0,$xd0
+	vshufi32x4	\$0x88,$xc1,$xa1,$xt1
+	vshufi32x4	\$0xdd,$xc1,$xa1,$xa1
+	 vshufi32x4	\$0x88,$xd1,$xb1,$xc1
+	 vshufi32x4	\$0xdd,$xd1,$xb1,$xd1
+	vshufi32x4	\$0x88,$xc2,$xa2,$xt2
+	vshufi32x4	\$0xdd,$xc2,$xa2,$xa2
+	 vshufi32x4	\$0x88,$xd2,$xb2,$xc2
+	 vshufi32x4	\$0xdd,$xd2,$xb2,$xd2
+	vshufi32x4	\$0x88,$xc3,$xa3,$xt3
+	vshufi32x4	\$0xdd,$xc3,$xa3,$xa3
+	 vshufi32x4	\$0x88,$xd3,$xb3,$xc3
+	 vshufi32x4	\$0xdd,$xd3,$xb3,$xd3
+___
+	($xa0,$xa1,$xa2,$xa3,$xb0,$xb1,$xb2,$xb3)=
+	($xt0,$xt1,$xt2,$xt3,$xa0,$xa1,$xa2,$xa3);
+
+	($xa0,$xb0,$xc0,$xd0, $xa1,$xb1,$xc1,$xd1,
+	 $xa2,$xb2,$xc2,$xd2, $xa3,$xb3,$xc3,$xd3) =
+	($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3,
+	 $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3);
+$code.=<<___;
+	cmp		\$64*16,$len
+	jb		.Ltail16x
+
+	vpxord		0x00($inp),$xa0,$xa0	# xor with input
+	vpxord		0x40($inp),$xb0,$xb0
+	vpxord		0x80($inp),$xc0,$xc0
+	vpxord		0xc0($inp),$xd0,$xd0
+	vmovdqu32	$xa0,0x00($out)
+	vmovdqu32	$xb0,0x40($out)
+	vmovdqu32	$xc0,0x80($out)
+	vmovdqu32	$xd0,0xc0($out)
+
+	vpxord		0x100($inp),$xa1,$xa1
+	vpxord		0x140($inp),$xb1,$xb1
+	vpxord		0x180($inp),$xc1,$xc1
+	vpxord		0x1c0($inp),$xd1,$xd1
+	vmovdqu32	$xa1,0x100($out)
+	vmovdqu32	$xb1,0x140($out)
+	vmovdqu32	$xc1,0x180($out)
+	vmovdqu32	$xd1,0x1c0($out)
+
+	vpxord		0x200($inp),$xa2,$xa2
+	vpxord		0x240($inp),$xb2,$xb2
+	vpxord		0x280($inp),$xc2,$xc2
+	vpxord		0x2c0($inp),$xd2,$xd2
+	vmovdqu32	$xa2,0x200($out)
+	vmovdqu32	$xb2,0x240($out)
+	vmovdqu32	$xc2,0x280($out)
+	vmovdqu32	$xd2,0x2c0($out)
+
+	vpxord		0x300($inp),$xa3,$xa3
+	vpxord		0x340($inp),$xb3,$xb3
+	vpxord		0x380($inp),$xc3,$xc3
+	vpxord		0x3c0($inp),$xd3,$xd3
+	lea		0x400($inp),$inp
+	vmovdqu32	$xa3,0x300($out)
+	vmovdqu32	$xb3,0x340($out)
+	vmovdqu32	$xc3,0x380($out)
+	vmovdqu32	$xd3,0x3c0($out)
+	lea		0x400($out),$out
+
+	sub		\$64*16,$len
+	jnz		.Loop_outer16x
+
+	jmp		.Ldone16x
+
+.align	32
+.Ltail16x:
+	xor		%r9,%r9
+	sub		$inp,$out
+	cmp		\$64*1,$len
+	jb		.Less_than_64_16x
+	vpxord		($inp),$xa0,$xa0	# xor with input
+	vmovdqu32	$xa0,($out,$inp)
+	je		.Ldone16x
+	vmovdqa32	$xb0,$xa0
+	lea		64($inp),$inp
+
+	cmp		\$64*2,$len
+	jb		.Less_than_64_16x
+	vpxord		($inp),$xb0,$xb0
+	vmovdqu32	$xb0,($out,$inp)
+	je		.Ldone16x
+	vmovdqa32	$xc0,$xa0
+	lea		64($inp),$inp
+
+	cmp		\$64*3,$len
+	jb		.Less_than_64_16x
+	vpxord		($inp),$xc0,$xc0
+	vmovdqu32	$xc0,($out,$inp)
+	je		.Ldone16x
+	vmovdqa32	$xd0,$xa0
+	lea		64($inp),$inp
+
+	cmp		\$64*4,$len
+	jb		.Less_than_64_16x
+	vpxord		($inp),$xd0,$xd0
+	vmovdqu32	$xd0,($out,$inp)
+	je		.Ldone16x
+	vmovdqa32	$xa1,$xa0
+	lea		64($inp),$inp
+
+	cmp		\$64*5,$len
+	jb		.Less_than_64_16x
+	vpxord		($inp),$xa1,$xa1
+	vmovdqu32	$xa1,($out,$inp)
+	je		.Ldone16x
+	vmovdqa32	$xb1,$xa0
+	lea		64($inp),$inp
+
+	cmp		\$64*6,$len
+	jb		.Less_than_64_16x
+	vpxord		($inp),$xb1,$xb1
+	vmovdqu32	$xb1,($out,$inp)
+	je		.Ldone16x
+	vmovdqa32	$xc1,$xa0
+	lea		64($inp),$inp
+
+	cmp		\$64*7,$len
+	jb		.Less_than_64_16x
+	vpxord		($inp),$xc1,$xc1
+	vmovdqu32	$xc1,($out,$inp)
+	je		.Ldone16x
+	vmovdqa32	$xd1,$xa0
+	lea		64($inp),$inp
+
+	cmp		\$64*8,$len
+	jb		.Less_than_64_16x
+	vpxord		($inp),$xd1,$xd1
+	vmovdqu32	$xd1,($out,$inp)
+	je		.Ldone16x
+	vmovdqa32	$xa2,$xa0
+	lea		64($inp),$inp
+
+	cmp		\$64*9,$len
+	jb		.Less_than_64_16x
+	vpxord		($inp),$xa2,$xa2
+	vmovdqu32	$xa2,($out,$inp)
+	je		.Ldone16x
+	vmovdqa32	$xb2,$xa0
+	lea		64($inp),$inp
+
+	cmp		\$64*10,$len
+	jb		.Less_than_64_16x
+	vpxord		($inp),$xb2,$xb2
+	vmovdqu32	$xb2,($out,$inp)
+	je		.Ldone16x
+	vmovdqa32	$xc2,$xa0
+	lea		64($inp),$inp
+
+	cmp		\$64*11,$len
+	jb		.Less_than_64_16x
+	vpxord		($inp),$xc2,$xc2
+	vmovdqu32	$xc2,($out,$inp)
+	je		.Ldone16x
+	vmovdqa32	$xd2,$xa0
+	lea		64($inp),$inp
+
+	cmp		\$64*12,$len
+	jb		.Less_than_64_16x
+	vpxord		($inp),$xd2,$xd2
+	vmovdqu32	$xd2,($out,$inp)
+	je		.Ldone16x
+	vmovdqa32	$xa3,$xa0
+	lea		64($inp),$inp
+
+	cmp		\$64*13,$len
+	jb		.Less_than_64_16x
+	vpxord		($inp),$xa3,$xa3
+	vmovdqu32	$xa3,($out,$inp)
+	je		.Ldone16x
+	vmovdqa32	$xb3,$xa0
+	lea		64($inp),$inp
+
+	cmp		\$64*14,$len
+	jb		.Less_than_64_16x
+	vpxord		($inp),$xb3,$xb3
+	vmovdqu32	$xb3,($out,$inp)
+	je		.Ldone16x
+	vmovdqa32	$xc3,$xa0
+	lea		64($inp),$inp
+
+	cmp		\$64*15,$len
+	jb		.Less_than_64_16x
+	vpxord		($inp),$xc3,$xc3
+	vmovdqu32	$xc3,($out,$inp)
+	je		.Ldone16x
+	vmovdqa32	$xd3,$xa0
+	lea		64($inp),$inp
+
+.Less_than_64_16x:
+	vmovdqa32	$xa0,0x00(%rsp)
+	lea		($out,$inp),$out
+	and		\$63,$len
+
+.Loop_tail16x:
+	movzb		($inp,%r9),%eax
+	movzb		(%rsp,%r9),%ecx
+	lea		1(%r9),%r9
+	xor		%ecx,%eax
+	mov		%al,-1($out,%r9)
+	dec		$len
+	jnz		.Loop_tail16x
+
+	vpxord		$xa0,$xa0,$xa0
+	vmovdqa32	$xa0,0(%rsp)
+
+.Ldone16x:
+	vzeroall
+___
+$code.=<<___	if ($win64);
+	movaps		-0xb0(%r10),%xmm6
+	movaps		-0xa0(%r10),%xmm7
+	movaps		-0x90(%r10),%xmm8
+	movaps		-0x80(%r10),%xmm9
+	movaps		-0x70(%r10),%xmm10
+	movaps		-0x60(%r10),%xmm11
+	movaps		-0x50(%r10),%xmm12
+	movaps		-0x40(%r10),%xmm13
+	movaps		-0x30(%r10),%xmm14
+	movaps		-0x20(%r10),%xmm15
+___
+$code.=<<___;
+	lea		-8(%r10),%rsp
+.cfi_def_cfa_register	%rsp
+.L16x_epilogue:
+	ret
+.cfi_endproc
+.size	chacha20_16x,.-chacha20_16x
+___
+
+# switch to %ymm domain
+($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3,
+ $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3)=map("%ymm$_",(0..15));
+@xx=($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3,
+     $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3);
+@key=map("%ymm$_",(16..31));
+($xt0,$xt1,$xt2,$xt3)=@key[0..3];
+
+$code.=<<___;
+.type	chacha20_8xvl,\@function,5
+.align	32
+chacha20_8xvl:
+.cfi_startproc
+.Lchacha20_8xvl:
+	lea		8(%rsp),%r10		# frame register
+.cfi_def_cfa_register	%r10
+	sub		\$64+$xframe,%rsp
+	and		\$-64,%rsp
+___
+$code.=<<___	if ($win64);
+	movaps		%xmm6,-0xb0(%r10)
+	movaps		%xmm7,-0xa0(%r10)
+	movaps		%xmm8,-0x90(%r10)
+	movaps		%xmm9,-0x80(%r10)
+	movaps		%xmm10,-0x70(%r10)
+	movaps		%xmm11,-0x60(%r10)
+	movaps		%xmm12,-0x50(%r10)
+	movaps		%xmm13,-0x40(%r10)
+	movaps		%xmm14,-0x30(%r10)
+	movaps		%xmm15,-0x20(%r10)
+.L8xvl_body:
+___
+$code.=<<___;
+	vzeroupper
+
+	lea		.Lsigma(%rip),%r9
+	vbroadcasti128	(%r9),$xa3		# key[0]
+	vbroadcasti128	($key),$xb3		# key[1]
+	vbroadcasti128	16($key),$xc3		# key[2]
+	vbroadcasti128	($counter),$xd3		# key[3]
+
+	vpshufd		\$0x00,$xa3,$xa0	# smash key by lanes...
+	vpshufd		\$0x55,$xa3,$xa1
+	vpshufd		\$0xaa,$xa3,$xa2
+	vpshufd		\$0xff,$xa3,$xa3
+	vmovdqa64	$xa0,@key[0]
+	vmovdqa64	$xa1,@key[1]
+	vmovdqa64	$xa2,@key[2]
+	vmovdqa64	$xa3,@key[3]
+
+	vpshufd		\$0x00,$xb3,$xb0
+	vpshufd		\$0x55,$xb3,$xb1
+	vpshufd		\$0xaa,$xb3,$xb2
+	vpshufd		\$0xff,$xb3,$xb3
+	vmovdqa64	$xb0,@key[4]
+	vmovdqa64	$xb1,@key[5]
+	vmovdqa64	$xb2,@key[6]
+	vmovdqa64	$xb3,@key[7]
+
+	vpshufd		\$0x00,$xc3,$xc0
+	vpshufd		\$0x55,$xc3,$xc1
+	vpshufd		\$0xaa,$xc3,$xc2
+	vpshufd		\$0xff,$xc3,$xc3
+	vmovdqa64	$xc0,@key[8]
+	vmovdqa64	$xc1,@key[9]
+	vmovdqa64	$xc2,@key[10]
+	vmovdqa64	$xc3,@key[11]
+
+	vpshufd		\$0x00,$xd3,$xd0
+	vpshufd		\$0x55,$xd3,$xd1
+	vpshufd		\$0xaa,$xd3,$xd2
+	vpshufd		\$0xff,$xd3,$xd3
+	vpaddd		.Lincy(%rip),$xd0,$xd0	# don't save counters yet
+	vmovdqa64	$xd0,@key[12]
+	vmovdqa64	$xd1,@key[13]
+	vmovdqa64	$xd2,@key[14]
+	vmovdqa64	$xd3,@key[15]
+
+	mov		\$10,%eax
+	jmp		.Loop8xvl
+
+.align	32
+.Loop_outer8xvl:
+	#vpbroadcastd	0(%r9),$xa0		# reload key
+	#vpbroadcastd	4(%r9),$xa1
+	vpbroadcastd	8(%r9),$xa2
+	vpbroadcastd	12(%r9),$xa3
+	vpaddd		.Leight(%rip),@key[12],@key[12]	# next SIMD counters
+	vmovdqa64	@key[4],$xb0
+	vmovdqa64	@key[5],$xb1
+	vmovdqa64	@key[6],$xb2
+	vmovdqa64	@key[7],$xb3
+	vmovdqa64	@key[8],$xc0
+	vmovdqa64	@key[9],$xc1
+	vmovdqa64	@key[10],$xc2
+	vmovdqa64	@key[11],$xc3
+	vmovdqa64	@key[12],$xd0
+	vmovdqa64	@key[13],$xd1
+	vmovdqa64	@key[14],$xd2
+	vmovdqa64	@key[15],$xd3
+
+	vmovdqa64	$xa0,@key[0]
+	vmovdqa64	$xa1,@key[1]
+	vmovdqa64	$xa2,@key[2]
+	vmovdqa64	$xa3,@key[3]
+
+	mov		\$10,%eax
+	jmp		.Loop8xvl
+
+.align	32
+.Loop8xvl:
+___
+	foreach (&AVX512_lane_ROUND(0, 4, 8,12)) { eval; }
+	foreach (&AVX512_lane_ROUND(0, 5,10,15)) { eval; }
+$code.=<<___;
+	dec		%eax
+	jnz		.Loop8xvl
+
+	vpaddd		@key[0],$xa0,$xa0	# accumulate key
+	vpaddd		@key[1],$xa1,$xa1
+	vpaddd		@key[2],$xa2,$xa2
+	vpaddd		@key[3],$xa3,$xa3
+
+	vpunpckldq	$xa1,$xa0,$xt2		# "de-interlace" data
+	vpunpckldq	$xa3,$xa2,$xt3
+	vpunpckhdq	$xa1,$xa0,$xa0
+	vpunpckhdq	$xa3,$xa2,$xa2
+	vpunpcklqdq	$xt3,$xt2,$xa1		# "a0"
+	vpunpckhqdq	$xt3,$xt2,$xt2		# "a1"
+	vpunpcklqdq	$xa2,$xa0,$xa3		# "a2"
+	vpunpckhqdq	$xa2,$xa0,$xa0		# "a3"
+___
+	($xa0,$xa1,$xa2,$xa3,$xt2)=($xa1,$xt2,$xa3,$xa0,$xa2);
+$code.=<<___;
+	vpaddd		@key[4],$xb0,$xb0
+	vpaddd		@key[5],$xb1,$xb1
+	vpaddd		@key[6],$xb2,$xb2
+	vpaddd		@key[7],$xb3,$xb3
+
+	vpunpckldq	$xb1,$xb0,$xt2
+	vpunpckldq	$xb3,$xb2,$xt3
+	vpunpckhdq	$xb1,$xb0,$xb0
+	vpunpckhdq	$xb3,$xb2,$xb2
+	vpunpcklqdq	$xt3,$xt2,$xb1		# "b0"
+	vpunpckhqdq	$xt3,$xt2,$xt2		# "b1"
+	vpunpcklqdq	$xb2,$xb0,$xb3		# "b2"
+	vpunpckhqdq	$xb2,$xb0,$xb0		# "b3"
+___
+	($xb0,$xb1,$xb2,$xb3,$xt2)=($xb1,$xt2,$xb3,$xb0,$xb2);
+$code.=<<___;
+	vshufi32x4	\$0,$xb0,$xa0,$xt3	# "de-interlace" further
+	vshufi32x4	\$3,$xb0,$xa0,$xb0
+	vshufi32x4	\$0,$xb1,$xa1,$xa0
+	vshufi32x4	\$3,$xb1,$xa1,$xb1
+	vshufi32x4	\$0,$xb2,$xa2,$xa1
+	vshufi32x4	\$3,$xb2,$xa2,$xb2
+	vshufi32x4	\$0,$xb3,$xa3,$xa2
+	vshufi32x4	\$3,$xb3,$xa3,$xb3
+___
+	($xa0,$xa1,$xa2,$xa3,$xt3)=($xt3,$xa0,$xa1,$xa2,$xa3);
+$code.=<<___;
+	vpaddd		@key[8],$xc0,$xc0
+	vpaddd		@key[9],$xc1,$xc1
+	vpaddd		@key[10],$xc2,$xc2
+	vpaddd		@key[11],$xc3,$xc3
+
+	vpunpckldq	$xc1,$xc0,$xt2
+	vpunpckldq	$xc3,$xc2,$xt3
+	vpunpckhdq	$xc1,$xc0,$xc0
+	vpunpckhdq	$xc3,$xc2,$xc2
+	vpunpcklqdq	$xt3,$xt2,$xc1		# "c0"
+	vpunpckhqdq	$xt3,$xt2,$xt2		# "c1"
+	vpunpcklqdq	$xc2,$xc0,$xc3		# "c2"
+	vpunpckhqdq	$xc2,$xc0,$xc0		# "c3"
+___
+	($xc0,$xc1,$xc2,$xc3,$xt2)=($xc1,$xt2,$xc3,$xc0,$xc2);
+$code.=<<___;
+	vpaddd		@key[12],$xd0,$xd0
+	vpaddd		@key[13],$xd1,$xd1
+	vpaddd		@key[14],$xd2,$xd2
+	vpaddd		@key[15],$xd3,$xd3
+
+	vpunpckldq	$xd1,$xd0,$xt2
+	vpunpckldq	$xd3,$xd2,$xt3
+	vpunpckhdq	$xd1,$xd0,$xd0
+	vpunpckhdq	$xd3,$xd2,$xd2
+	vpunpcklqdq	$xt3,$xt2,$xd1		# "d0"
+	vpunpckhqdq	$xt3,$xt2,$xt2		# "d1"
+	vpunpcklqdq	$xd2,$xd0,$xd3		# "d2"
+	vpunpckhqdq	$xd2,$xd0,$xd0		# "d3"
+___
+	($xd0,$xd1,$xd2,$xd3,$xt2)=($xd1,$xt2,$xd3,$xd0,$xd2);
+$code.=<<___;
+	vperm2i128	\$0x20,$xd0,$xc0,$xt3	# "de-interlace" further
+	vperm2i128	\$0x31,$xd0,$xc0,$xd0
+	vperm2i128	\$0x20,$xd1,$xc1,$xc0
+	vperm2i128	\$0x31,$xd1,$xc1,$xd1
+	vperm2i128	\$0x20,$xd2,$xc2,$xc1
+	vperm2i128	\$0x31,$xd2,$xc2,$xd2
+	vperm2i128	\$0x20,$xd3,$xc3,$xc2
+	vperm2i128	\$0x31,$xd3,$xc3,$xd3
+___
+	($xc0,$xc1,$xc2,$xc3,$xt3)=($xt3,$xc0,$xc1,$xc2,$xc3);
+	($xb0,$xb1,$xb2,$xb3,$xc0,$xc1,$xc2,$xc3)=
+	($xc0,$xc1,$xc2,$xc3,$xb0,$xb1,$xb2,$xb3);
+$code.=<<___;
+	cmp		\$64*8,$len
+	jb		.Ltail8xvl
+
+	mov		\$0x80,%eax		# size optimization
+	vpxord		0x00($inp),$xa0,$xa0	# xor with input
+	vpxor		0x20($inp),$xb0,$xb0
+	vpxor		0x40($inp),$xc0,$xc0
+	vpxor		0x60($inp),$xd0,$xd0
+	lea		($inp,%rax),$inp	# size optimization
+	vmovdqu32	$xa0,0x00($out)
+	vmovdqu		$xb0,0x20($out)
+	vmovdqu		$xc0,0x40($out)
+	vmovdqu		$xd0,0x60($out)
+	lea		($out,%rax),$out	# size optimization
+
+	vpxor		0x00($inp),$xa1,$xa1
+	vpxor		0x20($inp),$xb1,$xb1
+	vpxor		0x40($inp),$xc1,$xc1
+	vpxor		0x60($inp),$xd1,$xd1
+	lea		($inp,%rax),$inp	# size optimization
+	vmovdqu		$xa1,0x00($out)
+	vmovdqu		$xb1,0x20($out)
+	vmovdqu		$xc1,0x40($out)
+	vmovdqu		$xd1,0x60($out)
+	lea		($out,%rax),$out	# size optimization
+
+	vpxord		0x00($inp),$xa2,$xa2
+	vpxor		0x20($inp),$xb2,$xb2
+	vpxor		0x40($inp),$xc2,$xc2
+	vpxor		0x60($inp),$xd2,$xd2
+	lea		($inp,%rax),$inp	# size optimization
+	vmovdqu32	$xa2,0x00($out)
+	vmovdqu		$xb2,0x20($out)
+	vmovdqu		$xc2,0x40($out)
+	vmovdqu		$xd2,0x60($out)
+	lea		($out,%rax),$out	# size optimization
+
+	vpxor		0x00($inp),$xa3,$xa3
+	vpxor		0x20($inp),$xb3,$xb3
+	vpxor		0x40($inp),$xc3,$xc3
+	vpxor		0x60($inp),$xd3,$xd3
+	lea		($inp,%rax),$inp	# size optimization
+	vmovdqu		$xa3,0x00($out)
+	vmovdqu		$xb3,0x20($out)
+	vmovdqu		$xc3,0x40($out)
+	vmovdqu		$xd3,0x60($out)
+	lea		($out,%rax),$out	# size optimization
+
+	vpbroadcastd	0(%r9),%ymm0		# reload key
+	vpbroadcastd	4(%r9),%ymm1
+
+	sub		\$64*8,$len
+	jnz		.Loop_outer8xvl
+
+	jmp		.Ldone8xvl
+
+.align	32
+.Ltail8xvl:
+	vmovdqa64	$xa0,%ymm8		# size optimization
+___
+$xa0 = "%ymm8";
+$code.=<<___;
+	xor		%r9,%r9
+	sub		$inp,$out
+	cmp		\$64*1,$len
+	jb		.Less_than_64_8xvl
+	vpxor		0x00($inp),$xa0,$xa0	# xor with input
+	vpxor		0x20($inp),$xb0,$xb0
+	vmovdqu		$xa0,0x00($out,$inp)
+	vmovdqu		$xb0,0x20($out,$inp)
+	je		.Ldone8xvl
+	vmovdqa		$xc0,$xa0
+	vmovdqa		$xd0,$xb0
+	lea		64($inp),$inp
+
+	cmp		\$64*2,$len
+	jb		.Less_than_64_8xvl
+	vpxor		0x00($inp),$xc0,$xc0
+	vpxor		0x20($inp),$xd0,$xd0
+	vmovdqu		$xc0,0x00($out,$inp)
+	vmovdqu		$xd0,0x20($out,$inp)
+	je		.Ldone8xvl
+	vmovdqa		$xa1,$xa0
+	vmovdqa		$xb1,$xb0
+	lea		64($inp),$inp
+
+	cmp		\$64*3,$len
+	jb		.Less_than_64_8xvl
+	vpxor		0x00($inp),$xa1,$xa1
+	vpxor		0x20($inp),$xb1,$xb1
+	vmovdqu		$xa1,0x00($out,$inp)
+	vmovdqu		$xb1,0x20($out,$inp)
+	je		.Ldone8xvl
+	vmovdqa		$xc1,$xa0
+	vmovdqa		$xd1,$xb0
+	lea		64($inp),$inp
+
+	cmp		\$64*4,$len
+	jb		.Less_than_64_8xvl
+	vpxor		0x00($inp),$xc1,$xc1
+	vpxor		0x20($inp),$xd1,$xd1
+	vmovdqu		$xc1,0x00($out,$inp)
+	vmovdqu		$xd1,0x20($out,$inp)
+	je		.Ldone8xvl
+	vmovdqa32	$xa2,$xa0
+	vmovdqa		$xb2,$xb0
+	lea		64($inp),$inp
+
+	cmp		\$64*5,$len
+	jb		.Less_than_64_8xvl
+	vpxord		0x00($inp),$xa2,$xa2
+	vpxor		0x20($inp),$xb2,$xb2
+	vmovdqu32	$xa2,0x00($out,$inp)
+	vmovdqu		$xb2,0x20($out,$inp)
+	je		.Ldone8xvl
+	vmovdqa		$xc2,$xa0
+	vmovdqa		$xd2,$xb0
+	lea		64($inp),$inp
+
+	cmp		\$64*6,$len
+	jb		.Less_than_64_8xvl
+	vpxor		0x00($inp),$xc2,$xc2
+	vpxor		0x20($inp),$xd2,$xd2
+	vmovdqu		$xc2,0x00($out,$inp)
+	vmovdqu		$xd2,0x20($out,$inp)
+	je		.Ldone8xvl
+	vmovdqa		$xa3,$xa0
+	vmovdqa		$xb3,$xb0
+	lea		64($inp),$inp
+
+	cmp		\$64*7,$len
+	jb		.Less_than_64_8xvl
+	vpxor		0x00($inp),$xa3,$xa3
+	vpxor		0x20($inp),$xb3,$xb3
+	vmovdqu		$xa3,0x00($out,$inp)
+	vmovdqu		$xb3,0x20($out,$inp)
+	je		.Ldone8xvl
+	vmovdqa		$xc3,$xa0
+	vmovdqa		$xd3,$xb0
+	lea		64($inp),$inp
+
+.Less_than_64_8xvl:
+	vmovdqa		$xa0,0x00(%rsp)
+	vmovdqa		$xb0,0x20(%rsp)
+	lea		($out,$inp),$out
+	and		\$63,$len
+
+.Loop_tail8xvl:
+	movzb		($inp,%r9),%eax
+	movzb		(%rsp,%r9),%ecx
+	lea		1(%r9),%r9
+	xor		%ecx,%eax
+	mov		%al,-1($out,%r9)
+	dec		$len
+	jnz		.Loop_tail8xvl
+
+	vpxor		$xa0,$xa0,$xa0
+	vmovdqa		$xa0,0x00(%rsp)
+	vmovdqa		$xa0,0x20(%rsp)
+
+.Ldone8xvl:
+	vzeroall
+___
+$code.=<<___	if ($win64);
+	movaps		-0xb0(%r10),%xmm6
+	movaps		-0xa0(%r10),%xmm7
+	movaps		-0x90(%r10),%xmm8
+	movaps		-0x80(%r10),%xmm9
+	movaps		-0x70(%r10),%xmm10
+	movaps		-0x60(%r10),%xmm11
+	movaps		-0x50(%r10),%xmm12
+	movaps		-0x40(%r10),%xmm13
+	movaps		-0x30(%r10),%xmm14
+	movaps		-0x20(%r10),%xmm15
+___
+$code.=<<___;
+	lea		-8(%r10),%rsp
+.cfi_def_cfa_register	%rsp
+.L8xvl_epilogue:
+	ret
+.cfi_endproc
+.size	chacha20_8xvl,.-chacha20_8xvl
+___
+if($kernel) {
+	$code .= "#endif\n";
+}
+}
+
+# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
+#		CONTEXT *context,DISPATCHER_CONTEXT *disp)
+if ($win64) {
+$rec="%rcx";
+$frame="%rdx";
+$context="%r8";
+$disp="%r9";
+
+$code.=<<___;
+.extern	__imp_RtlVirtualUnwind
+.type	se_handler,\@abi-omnipotent
+.align	16
+se_handler:
+	push	%rsi
+	push	%rdi
+	push	%rbx
+	push	%rbp
+	push	%r12
+	push	%r13
+	push	%r14
+	push	%r15
+	pushfq
+	sub	\$64,%rsp
+
+	mov	120($context),%rax	# pull context->Rax
+	mov	248($context),%rbx	# pull context->Rip
+
+	mov	8($disp),%rsi		# disp->ImageBase
+	mov	56($disp),%r11		# disp->HandlerData
+
+	lea	.Lctr32_body(%rip),%r10
+	cmp	%r10,%rbx		# context->Rip<.Lprologue
+	jb	.Lcommon_seh_tail
+
+	mov	152($context),%rax	# pull context->Rsp
+
+	lea	.Lno_data(%rip),%r10	# epilogue label
+	cmp	%r10,%rbx		# context->Rip>=.Lepilogue
+	jae	.Lcommon_seh_tail
+
+	lea	64+24+48(%rax),%rax
+
+	mov	-8(%rax),%rbx
+	mov	-16(%rax),%rbp
+	mov	-24(%rax),%r12
+	mov	-32(%rax),%r13
+	mov	-40(%rax),%r14
+	mov	-48(%rax),%r15
+	mov	%rbx,144($context)	# restore context->Rbx
+	mov	%rbp,160($context)	# restore context->Rbp
+	mov	%r12,216($context)	# restore context->R12
+	mov	%r13,224($context)	# restore context->R13
+	mov	%r14,232($context)	# restore context->R14
+	mov	%r15,240($context)	# restore context->R14
+
+.Lcommon_seh_tail:
+	mov	8(%rax),%rdi
+	mov	16(%rax),%rsi
+	mov	%rax,152($context)	# restore context->Rsp
+	mov	%rsi,168($context)	# restore context->Rsi
+	mov	%rdi,176($context)	# restore context->Rdi
+
+	mov	40($disp),%rdi		# disp->ContextRecord
+	mov	$context,%rsi		# context
+	mov	\$154,%ecx		# sizeof(CONTEXT)
+	.long	0xa548f3fc		# cld; rep movsq
+
+	mov	$disp,%rsi
+	xor	%rcx,%rcx		# arg1, UNW_FLAG_NHANDLER
+	mov	8(%rsi),%rdx		# arg2, disp->ImageBase
+	mov	0(%rsi),%r8		# arg3, disp->ControlPc
+	mov	16(%rsi),%r9		# arg4, disp->FunctionEntry
+	mov	40(%rsi),%r10		# disp->ContextRecord
+	lea	56(%rsi),%r11		# &disp->HandlerData
+	lea	24(%rsi),%r12		# &disp->EstablisherFrame
+	mov	%r10,32(%rsp)		# arg5
+	mov	%r11,40(%rsp)		# arg6
+	mov	%r12,48(%rsp)		# arg7
+	mov	%rcx,56(%rsp)		# arg8, (NULL)
+	call	*__imp_RtlVirtualUnwind(%rip)
+
+	mov	\$1,%eax		# ExceptionContinueSearch
+	add	\$64,%rsp
+	popfq
+	pop	%r15
+	pop	%r14
+	pop	%r13
+	pop	%r12
+	pop	%rbp
+	pop	%rbx
+	pop	%rdi
+	pop	%rsi
+	ret
+.size	se_handler,.-se_handler
+
+.type	simd_handler,\@abi-omnipotent
+.align	16
+simd_handler:
+	push	%rsi
+	push	%rdi
+	push	%rbx
+	push	%rbp
+	push	%r12
+	push	%r13
+	push	%r14
+	push	%r15
+	pushfq
+	sub	\$64,%rsp
+
+	mov	120($context),%rax	# pull context->Rax
+	mov	248($context),%rbx	# pull context->Rip
+
+	mov	8($disp),%rsi		# disp->ImageBase
+	mov	56($disp),%r11		# disp->HandlerData
+
+	mov	0(%r11),%r10d		# HandlerData[0]
+	lea	(%rsi,%r10),%r10	# prologue label
+	cmp	%r10,%rbx		# context->Rip<prologue label
+	jb	.Lcommon_seh_tail
+
+	mov	192($context),%rax	# pull context->R9
+
+	mov	4(%r11),%r10d		# HandlerData[1]
+	mov	8(%r11),%ecx		# HandlerData[2]
+	lea	(%rsi,%r10),%r10	# epilogue label
+	cmp	%r10,%rbx		# context->Rip>=epilogue label
+	jae	.Lcommon_seh_tail
+
+	neg	%rcx
+	lea	-8(%rax,%rcx),%rsi
+	lea	512($context),%rdi	# &context.Xmm6
+	neg	%ecx
+	shr	\$3,%ecx
+	.long	0xa548f3fc		# cld; rep movsq
+
+	jmp	.Lcommon_seh_tail
+.size	simd_handler,.-simd_handler
+
+.section	.pdata
+.align	4
+	.rva	.LSEH_begin_chacha20_ctr32
+	.rva	.LSEH_end_chacha20_ctr32
+	.rva	.LSEH_info_chacha20_ctr32
+
+	.rva	.LSEH_begin_chacha20_ssse3
+	.rva	.LSEH_end_chacha20_ssse3
+	.rva	.LSEH_info_chacha20_ssse3
+
+	.rva	.LSEH_begin_chacha20_128
+	.rva	.LSEH_end_chacha20_128
+	.rva	.LSEH_info_chacha20_128
+
+	.rva	.LSEH_begin_chacha20_4x
+	.rva	.LSEH_end_chacha20_4x
+	.rva	.LSEH_info_chacha20_4x
+___
+$code.=<<___ if ($avx);
+	.rva	.LSEH_begin_chacha20_xop
+	.rva	.LSEH_end_chacha20_xop
+	.rva	.LSEH_info_chacha20_xop
+___
+$code.=<<___ if ($avx>1);
+	.rva	.LSEH_begin_chacha20_avx2
+	.rva	.LSEH_end_chacha20_avx2
+	.rva	.LSEH_info_chacha20_avx2
+___
+$code.=<<___ if ($avx>2);
+	.rva	.LSEH_begin_chacha20_avx512
+	.rva	.LSEH_end_chacha20_avx512
+	.rva	.LSEH_info_chacha20_avx512
+
+	.rva	.LSEH_begin_chacha20_avx512vl
+	.rva	.LSEH_end_chacha20_avx512vl
+	.rva	.LSEH_info_chacha20_avx512vl
+
+	.rva	.LSEH_begin_chacha20_16x
+	.rva	.LSEH_end_chacha20_16x
+	.rva	.LSEH_info_chacha20_16x
+
+	.rva	.LSEH_begin_chacha20_8xvl
+	.rva	.LSEH_end_chacha20_8xvl
+	.rva	.LSEH_info_chacha20_8xvl
+___
+$code.=<<___;
+.section	.xdata
+.align	8
+.LSEH_info_chacha20_ctr32:
+	.byte	9,0,0,0
+	.rva	se_handler
+
+.LSEH_info_chacha20_ssse3:
+	.byte	9,0,0,0
+	.rva	simd_handler
+	.rva	.Lssse3_body,.Lssse3_epilogue
+	.long	0x20,0
+
+.LSEH_info_chacha20_128:
+	.byte	9,0,0,0
+	.rva	simd_handler
+	.rva	.L128_body,.L128_epilogue
+	.long	0x60,0
+
+.LSEH_info_chacha20_4x:
+	.byte	9,0,0,0
+	.rva	simd_handler
+	.rva	.L4x_body,.L4x_epilogue
+	.long	0xa0,0
+___
+$code.=<<___ if ($avx);
+.LSEH_info_chacha20_xop:
+	.byte	9,0,0,0
+	.rva	simd_handler
+	.rva	.L4xop_body,.L4xop_epilogue		# HandlerData[]
+	.long	0xa0,0
+___
+$code.=<<___ if ($avx>1);
+.LSEH_info_chacha20_avx2:
+	.byte	9,0,0,0
+	.rva	simd_handler
+	.rva	.L8x_body,.L8x_epilogue			# HandlerData[]
+	.long	0xa0,0
+___
+$code.=<<___ if ($avx>2);
+.LSEH_info_chacha20_avx512:
+	.byte	9,0,0,0
+	.rva	simd_handler
+	.rva	.Lavx512_body,.Lavx512_epilogue		# HandlerData[]
+	.long	0x20,0
+
+.LSEH_info_chacha20_avx512vl:
+	.byte	9,0,0,0
+	.rva	simd_handler
+	.rva	.Lavx512vl_body,.Lavx512vl_epilogue	# HandlerData[]
+	.long	0x20,0
+
+.LSEH_info_chacha20_16x:
+	.byte	9,0,0,0
+	.rva	simd_handler
+	.rva	.L16x_body,.L16x_epilogue		# HandlerData[]
+	.long	0xa0,0
+
+.LSEH_info_chacha20_8xvl:
+	.byte	9,0,0,0
+	.rva	simd_handler
+	.rva	.L8xvl_body,.L8xvl_epilogue		# HandlerData[]
+	.long	0xa0,0
+___
+}
+
+open SELF,$0;
+while(<SELF>) {
+	next if (/^#!/);
+	last if (!s/^#/\/\// and !/^$/);
+	print;
+}
+close SELF;
+
+foreach (split("\n",$code)) {
+	s/\`([^\`]*)\`/eval $1/ge;
+
+	s/%x#%[yz]/%x/g;	# "down-shift"
+
+	if ($kernel) {
+		s/(^\.type.*),[0-9]+$/\1/;
+		next if /^\.cfi.*/;
+	}
+
+	print $_,"\n";
+}
+
+close STDOUT;
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/chacha20poly1305.c b/net/wireguard/crypto/zinc/chacha20poly1305.c
--- a/net/wireguard/crypto/zinc/chacha20poly1305.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/chacha20poly1305.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,402 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ *
+ * This is an implementation of the ChaCha20Poly1305 AEAD construction.
+ *
+ * Information: https://tools.ietf.org/html/rfc8439
+ */
+
+#include <zinc/chacha20poly1305.h>
+#include <zinc/chacha20.h>
+#include <zinc/poly1305.h>
+#include "selftest/run.h"
+
+#include <asm/unaligned.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <crypto/scatterwalk.h> // For blkcipher_walk.
+
+static const u8 pad0[CHACHA20_BLOCK_SIZE] = { 0 };
+
+static inline void
+__chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
+			   const u8 *ad, const size_t ad_len, const u64 nonce,
+			   const u8 key[CHACHA20POLY1305_KEY_SIZE],
+			   simd_context_t *simd_context)
+{
+	struct poly1305_ctx poly1305_state;
+	struct chacha20_ctx chacha20_state;
+	union {
+		u8 block0[POLY1305_KEY_SIZE];
+		__le64 lens[2];
+	} b = { { 0 } };
+
+	chacha20_init(&chacha20_state, key, nonce);
+	chacha20(&chacha20_state, b.block0, b.block0, sizeof(b.block0),
+		 simd_context);
+	poly1305_init(&poly1305_state, b.block0);
+
+	poly1305_update(&poly1305_state, ad, ad_len, simd_context);
+	poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf,
+			simd_context);
+
+	chacha20(&chacha20_state, dst, src, src_len, simd_context);
+
+	poly1305_update(&poly1305_state, dst, src_len, simd_context);
+	poly1305_update(&poly1305_state, pad0, (0x10 - src_len) & 0xf,
+			simd_context);
+
+	b.lens[0] = cpu_to_le64(ad_len);
+	b.lens[1] = cpu_to_le64(src_len);
+	poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens),
+			simd_context);
+
+	poly1305_final(&poly1305_state, dst + src_len, simd_context);
+
+	memzero_explicit(&chacha20_state, sizeof(chacha20_state));
+	memzero_explicit(&b, sizeof(b));
+}
+
+void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
+			      const u8 *ad, const size_t ad_len,
+			      const u64 nonce,
+			      const u8 key[CHACHA20POLY1305_KEY_SIZE])
+{
+	simd_context_t simd_context;
+
+	simd_get(&simd_context);
+	__chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, nonce, key,
+				   &simd_context);
+	simd_put(&simd_context);
+}
+EXPORT_SYMBOL(chacha20poly1305_encrypt);
+
+bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src,
+					 const size_t src_len,
+					 const u8 *ad, const size_t ad_len,
+					 const u64 nonce,
+					 const u8 key[CHACHA20POLY1305_KEY_SIZE],
+					 simd_context_t *simd_context)
+{
+	struct poly1305_ctx poly1305_state;
+	struct chacha20_ctx chacha20_state;
+	struct sg_mapping_iter miter;
+	size_t partial = 0;
+	ssize_t sl;
+	union {
+		u8 chacha20_stream[CHACHA20_BLOCK_SIZE];
+		u8 block0[POLY1305_KEY_SIZE];
+		u8 mac[POLY1305_MAC_SIZE];
+		__le64 lens[2];
+	} b __aligned(16) = { { 0 } };
+
+
+	chacha20_init(&chacha20_state, key, nonce);
+	chacha20(&chacha20_state, b.block0, b.block0, sizeof(b.block0),
+		 simd_context);
+	poly1305_init(&poly1305_state, b.block0);
+
+	poly1305_update(&poly1305_state, ad, ad_len, simd_context);
+	poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf,
+			simd_context);
+
+	sg_miter_start(&miter, src, sg_nents(src), SG_MITER_TO_SG | SG_MITER_ATOMIC);
+	for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) {
+		u8 *addr = miter.addr;
+		size_t length = min_t(size_t, sl, miter.length);
+
+		if (unlikely(partial)) {
+			size_t l = min(length, CHACHA20_BLOCK_SIZE - partial);
+
+			crypto_xor(addr, b.chacha20_stream + partial, l);
+			partial = (partial + l) & (CHACHA20_BLOCK_SIZE - 1);
+
+			addr += l;
+			length -= l;
+		}
+
+		if (likely(length >= CHACHA20_BLOCK_SIZE || length == sl)) {
+			size_t l = length;
+
+			if (unlikely(length < sl))
+				l &= ~(CHACHA20_BLOCK_SIZE - 1);
+			chacha20(&chacha20_state, addr, addr, l, simd_context);
+			addr += l;
+			length -= l;
+		}
+
+		if (unlikely(length > 0)) {
+			chacha20(&chacha20_state, b.chacha20_stream, pad0,
+				 CHACHA20_BLOCK_SIZE, simd_context);
+			crypto_xor(addr, b.chacha20_stream, length);
+			partial = length;
+		}
+
+		poly1305_update(&poly1305_state, miter.addr,
+				min_t(size_t, sl, miter.length), simd_context);
+
+		simd_relax(simd_context);
+	}
+
+	poly1305_update(&poly1305_state, pad0, (0x10 - src_len) & 0xf,
+			simd_context);
+
+	b.lens[0] = cpu_to_le64(ad_len);
+	b.lens[1] = cpu_to_le64(src_len);
+	poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens),
+			simd_context);
+
+	if (likely(sl <= -POLY1305_MAC_SIZE))
+		poly1305_final(&poly1305_state, miter.addr + miter.length + sl,
+			       simd_context);
+
+	sg_miter_stop(&miter);
+
+	if (unlikely(sl > -POLY1305_MAC_SIZE)) {
+		poly1305_final(&poly1305_state, b.mac, simd_context);
+		scatterwalk_map_and_copy(b.mac, src, src_len, sizeof(b.mac), 1);
+	}
+
+	memzero_explicit(&chacha20_state, sizeof(chacha20_state));
+	memzero_explicit(&b, sizeof(b));
+	return true;
+}
+EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace);
+
+static inline bool
+__chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
+			   const u8 *ad, const size_t ad_len, const u64 nonce,
+			   const u8 key[CHACHA20POLY1305_KEY_SIZE],
+			   simd_context_t *simd_context)
+{
+	struct poly1305_ctx poly1305_state;
+	struct chacha20_ctx chacha20_state;
+	int ret;
+	size_t dst_len;
+	union {
+		u8 block0[POLY1305_KEY_SIZE];
+		u8 mac[POLY1305_MAC_SIZE];
+		__le64 lens[2];
+	} b = { { 0 } };
+
+	if (unlikely(src_len < POLY1305_MAC_SIZE))
+		return false;
+
+	chacha20_init(&chacha20_state, key, nonce);
+	chacha20(&chacha20_state, b.block0, b.block0, sizeof(b.block0),
+		 simd_context);
+	poly1305_init(&poly1305_state, b.block0);
+
+	poly1305_update(&poly1305_state, ad, ad_len, simd_context);
+	poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf,
+			simd_context);
+
+	dst_len = src_len - POLY1305_MAC_SIZE;
+	poly1305_update(&poly1305_state, src, dst_len, simd_context);
+	poly1305_update(&poly1305_state, pad0, (0x10 - dst_len) & 0xf,
+			simd_context);
+
+	b.lens[0] = cpu_to_le64(ad_len);
+	b.lens[1] = cpu_to_le64(dst_len);
+	poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens),
+			simd_context);
+
+	poly1305_final(&poly1305_state, b.mac, simd_context);
+
+	ret = crypto_memneq(b.mac, src + dst_len, POLY1305_MAC_SIZE);
+	if (likely(!ret))
+		chacha20(&chacha20_state, dst, src, dst_len, simd_context);
+
+	memzero_explicit(&chacha20_state, sizeof(chacha20_state));
+	memzero_explicit(&b, sizeof(b));
+
+	return !ret;
+}
+
+bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
+			      const u8 *ad, const size_t ad_len,
+			      const u64 nonce,
+			      const u8 key[CHACHA20POLY1305_KEY_SIZE])
+{
+	simd_context_t simd_context, ret;
+
+	simd_get(&simd_context);
+	ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, nonce,
+					 key, &simd_context);
+	simd_put(&simd_context);
+	return ret;
+}
+EXPORT_SYMBOL(chacha20poly1305_decrypt);
+
+bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src,
+					 size_t src_len,
+					 const u8 *ad, const size_t ad_len,
+					 const u64 nonce,
+					 const u8 key[CHACHA20POLY1305_KEY_SIZE],
+					 simd_context_t *simd_context)
+{
+	struct poly1305_ctx poly1305_state;
+	struct chacha20_ctx chacha20_state;
+	struct sg_mapping_iter miter;
+	size_t partial = 0;
+	ssize_t sl;
+	union {
+		u8 chacha20_stream[CHACHA20_BLOCK_SIZE];
+		u8 block0[POLY1305_KEY_SIZE];
+		struct {
+			u8 read_mac[POLY1305_MAC_SIZE];
+			u8 computed_mac[POLY1305_MAC_SIZE];
+		};
+		__le64 lens[2];
+	} b __aligned(16) = { { 0 } };
+	bool ret = false;
+
+	if (unlikely(src_len < POLY1305_MAC_SIZE))
+		return ret;
+	src_len -= POLY1305_MAC_SIZE;
+
+	chacha20_init(&chacha20_state, key, nonce);
+	chacha20(&chacha20_state, b.block0, b.block0, sizeof(b.block0),
+		 simd_context);
+	poly1305_init(&poly1305_state, b.block0);
+
+	poly1305_update(&poly1305_state, ad, ad_len, simd_context);
+	poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf,
+			simd_context);
+
+	sg_miter_start(&miter, src, sg_nents(src), SG_MITER_TO_SG | SG_MITER_ATOMIC);
+	for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) {
+		u8 *addr = miter.addr;
+		size_t length = min_t(size_t, sl, miter.length);
+
+		poly1305_update(&poly1305_state, addr, length, simd_context);
+
+		if (unlikely(partial)) {
+			size_t l = min(length, CHACHA20_BLOCK_SIZE - partial);
+
+			crypto_xor(addr, b.chacha20_stream + partial, l);
+			partial = (partial + l) & (CHACHA20_BLOCK_SIZE - 1);
+
+			addr += l;
+			length -= l;
+		}
+
+		if (likely(length >= CHACHA20_BLOCK_SIZE || length == sl)) {
+			size_t l = length;
+
+			if (unlikely(length < sl))
+				l &= ~(CHACHA20_BLOCK_SIZE - 1);
+			chacha20(&chacha20_state, addr, addr, l, simd_context);
+			addr += l;
+			length -= l;
+		}
+
+		if (unlikely(length > 0)) {
+			chacha20(&chacha20_state, b.chacha20_stream, pad0,
+				 CHACHA20_BLOCK_SIZE, simd_context);
+			crypto_xor(addr, b.chacha20_stream, length);
+			partial = length;
+		}
+
+		simd_relax(simd_context);
+	}
+
+	poly1305_update(&poly1305_state, pad0, (0x10 - src_len) & 0xf,
+			simd_context);
+
+	b.lens[0] = cpu_to_le64(ad_len);
+	b.lens[1] = cpu_to_le64(src_len);
+	poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens),
+			simd_context);
+
+	if (likely(sl <= -POLY1305_MAC_SIZE)) {
+		poly1305_final(&poly1305_state, b.computed_mac, simd_context);
+		ret = !crypto_memneq(b.computed_mac,
+				     miter.addr + miter.length + sl,
+				     POLY1305_MAC_SIZE);
+	}
+
+	sg_miter_stop(&miter);
+
+	if (unlikely(sl > -POLY1305_MAC_SIZE)) {
+		poly1305_final(&poly1305_state, b.computed_mac, simd_context);
+		scatterwalk_map_and_copy(b.read_mac, src, src_len,
+					 sizeof(b.read_mac), 0);
+		ret = !crypto_memneq(b.read_mac, b.computed_mac,
+				     POLY1305_MAC_SIZE);
+
+	}
+
+	memzero_explicit(&chacha20_state, sizeof(chacha20_state));
+	memzero_explicit(&b, sizeof(b));
+	return ret;
+}
+EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace);
+
+void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
+			       const u8 *ad, const size_t ad_len,
+			       const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
+			       const u8 key[CHACHA20POLY1305_KEY_SIZE])
+{
+	simd_context_t simd_context;
+	u32 derived_key[CHACHA20_KEY_WORDS] __aligned(16);
+
+	simd_get(&simd_context);
+	hchacha20(derived_key, nonce, key, &simd_context);
+	cpu_to_le32_array(derived_key, ARRAY_SIZE(derived_key));
+	__chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len,
+				   get_unaligned_le64(nonce + 16),
+				   (u8 *)derived_key, &simd_context);
+	memzero_explicit(derived_key, CHACHA20POLY1305_KEY_SIZE);
+	simd_put(&simd_context);
+}
+EXPORT_SYMBOL(xchacha20poly1305_encrypt);
+
+bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
+			       const u8 *ad, const size_t ad_len,
+			       const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
+			       const u8 key[CHACHA20POLY1305_KEY_SIZE])
+{
+	bool ret;
+	simd_context_t simd_context;
+	u32 derived_key[CHACHA20_KEY_WORDS] __aligned(16);
+
+	simd_get(&simd_context);
+	hchacha20(derived_key, nonce, key, &simd_context);
+	cpu_to_le32_array(derived_key, ARRAY_SIZE(derived_key));
+	ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
+					 get_unaligned_le64(nonce + 16),
+					 (u8 *)derived_key, &simd_context);
+	memzero_explicit(derived_key, CHACHA20POLY1305_KEY_SIZE);
+	simd_put(&simd_context);
+	return ret;
+}
+EXPORT_SYMBOL(xchacha20poly1305_decrypt);
+
+#include "selftest/chacha20poly1305.c"
+
+#ifndef COMPAT_ZINC_IS_A_MODULE
+int __init chacha20poly1305_mod_init(void)
+#else
+static int __init mod_init(void)
+#endif
+{
+	if (!selftest_run("chacha20poly1305", chacha20poly1305_selftest,
+			  NULL, 0))
+		return -ENOTRECOVERABLE;
+	return 0;
+}
+
+#ifdef COMPAT_ZINC_IS_A_MODULE
+static void __exit mod_exit(void)
+{
+}
+
+module_init(mod_init);
+module_exit(mod_exit);
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction");
+MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");
+#endif
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/curve25519/curve25519-arm-glue.c b/net/wireguard/crypto/zinc/curve25519/curve25519-arm-glue.c
--- a/net/wireguard/crypto/zinc/curve25519/curve25519-arm-glue.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/curve25519/curve25519-arm-glue.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,43 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#include <linux/simd.h>
+#include <asm/hwcap.h>
+#include <asm/neon.h>
+
+asmlinkage void curve25519_neon(u8 mypublic[CURVE25519_KEY_SIZE],
+				const u8 secret[CURVE25519_KEY_SIZE],
+				const u8 basepoint[CURVE25519_KEY_SIZE]);
+
+static bool curve25519_use_neon __ro_after_init;
+static bool *const curve25519_nobs[] __initconst = { &curve25519_use_neon };
+static void __init curve25519_fpu_init(void)
+{
+	curve25519_use_neon = elf_hwcap & HWCAP_NEON;
+}
+
+static inline bool curve25519_arch(u8 mypublic[CURVE25519_KEY_SIZE],
+				   const u8 secret[CURVE25519_KEY_SIZE],
+				   const u8 basepoint[CURVE25519_KEY_SIZE])
+{
+	simd_context_t simd_context;
+	bool used_arch = false;
+
+	simd_get(&simd_context);
+	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
+	    !IS_ENABLED(CONFIG_CPU_BIG_ENDIAN) && curve25519_use_neon &&
+	    simd_use(&simd_context)) {
+		curve25519_neon(mypublic, secret, basepoint);
+		used_arch = true;
+	}
+	simd_put(&simd_context);
+	return used_arch;
+}
+
+static inline bool curve25519_base_arch(u8 pub[CURVE25519_KEY_SIZE],
+					const u8 secret[CURVE25519_KEY_SIZE])
+{
+	return false;
+}
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/curve25519/curve25519-arm.S b/net/wireguard/crypto/zinc/curve25519/curve25519-arm.S
--- a/net/wireguard/crypto/zinc/curve25519/curve25519-arm.S	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/curve25519/curve25519-arm.S	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,2064 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ *
+ * Based on public domain code from Daniel J. Bernstein and Peter Schwabe. This
+ * began from SUPERCOP's curve25519/neon2/scalarmult.s, but has subsequently been
+ * manually reworked for use in kernel space.
+ */
+
+#if defined(CONFIG_KERNEL_MODE_NEON) && !defined(__ARMEB__)
+#include <linux/linkage.h>
+
+.text
+.fpu neon
+.arch armv7-a
+.align 4
+
+SYM_FUNC_START(curve25519_neon)
+	push		{r4-r11, lr}
+	mov		ip, sp
+	sub		r3, sp, #704
+	and		r3, r3, #0xfffffff0
+	mov		sp, r3
+	movw		r4, #0
+	movw		r5, #254
+	vmov.i32	q0, #1
+	vshr.u64	q1, q0, #7
+	vshr.u64	q0, q0, #8
+	vmov.i32	d4, #19
+	vmov.i32	d5, #38
+	add		r6, sp, #480
+	vst1.8		{d2-d3}, [r6, : 128]!
+	vst1.8		{d0-d1}, [r6, : 128]!
+	vst1.8		{d4-d5}, [r6, : 128]
+	add		r6, r3, #0
+	vmov.i32	q2, #0
+	vst1.8		{d4-d5}, [r6, : 128]!
+	vst1.8		{d4-d5}, [r6, : 128]!
+	vst1.8		d4, [r6, : 64]
+	add		r6, r3, #0
+	movw		r7, #960
+	sub		r7, r7, #2
+	neg		r7, r7
+	sub		r7, r7, r7, LSL #7
+	str		r7, [r6]
+	add		r6, sp, #672
+	vld1.8		{d4-d5}, [r1]!
+	vld1.8		{d6-d7}, [r1]
+	vst1.8		{d4-d5}, [r6, : 128]!
+	vst1.8		{d6-d7}, [r6, : 128]
+	sub		r1, r6, #16
+	ldrb		r6, [r1]
+	and		r6, r6, #248
+	strb		r6, [r1]
+	ldrb		r6, [r1, #31]
+	and		r6, r6, #127
+	orr		r6, r6, #64
+	strb		r6, [r1, #31]
+	vmov.i64	q2, #0xffffffff
+	vshr.u64	q3, q2, #7
+	vshr.u64	q2, q2, #6
+	vld1.8		{d8}, [r2]
+	vld1.8		{d10}, [r2]
+	add		r2, r2, #6
+	vld1.8		{d12}, [r2]
+	vld1.8		{d14}, [r2]
+	add		r2, r2, #6
+	vld1.8		{d16}, [r2]
+	add		r2, r2, #4
+	vld1.8		{d18}, [r2]
+	vld1.8		{d20}, [r2]
+	add		r2, r2, #6
+	vld1.8		{d22}, [r2]
+	add		r2, r2, #2
+	vld1.8		{d24}, [r2]
+	vld1.8		{d26}, [r2]
+	vshr.u64	q5, q5, #26
+	vshr.u64	q6, q6, #3
+	vshr.u64	q7, q7, #29
+	vshr.u64	q8, q8, #6
+	vshr.u64	q10, q10, #25
+	vshr.u64	q11, q11, #3
+	vshr.u64	q12, q12, #12
+	vshr.u64	q13, q13, #38
+	vand		q4, q4, q2
+	vand		q6, q6, q2
+	vand		q8, q8, q2
+	vand		q10, q10, q2
+	vand		q2, q12, q2
+	vand		q5, q5, q3
+	vand		q7, q7, q3
+	vand		q9, q9, q3
+	vand		q11, q11, q3
+	vand		q3, q13, q3
+	add		r2, r3, #48
+	vadd.i64	q12, q4, q1
+	vadd.i64	q13, q10, q1
+	vshr.s64	q12, q12, #26
+	vshr.s64	q13, q13, #26
+	vadd.i64	q5, q5, q12
+	vshl.i64	q12, q12, #26
+	vadd.i64	q14, q5, q0
+	vadd.i64	q11, q11, q13
+	vshl.i64	q13, q13, #26
+	vadd.i64	q15, q11, q0
+	vsub.i64	q4, q4, q12
+	vshr.s64	q12, q14, #25
+	vsub.i64	q10, q10, q13
+	vshr.s64	q13, q15, #25
+	vadd.i64	q6, q6, q12
+	vshl.i64	q12, q12, #25
+	vadd.i64	q14, q6, q1
+	vadd.i64	q2, q2, q13
+	vsub.i64	q5, q5, q12
+	vshr.s64	q12, q14, #26
+	vshl.i64	q13, q13, #25
+	vadd.i64	q14, q2, q1
+	vadd.i64	q7, q7, q12
+	vshl.i64	q12, q12, #26
+	vadd.i64	q15, q7, q0
+	vsub.i64	q11, q11, q13
+	vshr.s64	q13, q14, #26
+	vsub.i64	q6, q6, q12
+	vshr.s64	q12, q15, #25
+	vadd.i64	q3, q3, q13
+	vshl.i64	q13, q13, #26
+	vadd.i64	q14, q3, q0
+	vadd.i64	q8, q8, q12
+	vshl.i64	q12, q12, #25
+	vadd.i64	q15, q8, q1
+	add		r2, r2, #8
+	vsub.i64	q2, q2, q13
+	vshr.s64	q13, q14, #25
+	vsub.i64	q7, q7, q12
+	vshr.s64	q12, q15, #26
+	vadd.i64	q14, q13, q13
+	vadd.i64	q9, q9, q12
+	vtrn.32		d12, d14
+	vshl.i64	q12, q12, #26
+	vtrn.32		d13, d15
+	vadd.i64	q0, q9, q0
+	vadd.i64	q4, q4, q14
+	vst1.8		d12, [r2, : 64]!
+	vshl.i64	q6, q13, #4
+	vsub.i64	q7, q8, q12
+	vshr.s64	q0, q0, #25
+	vadd.i64	q4, q4, q6
+	vadd.i64	q6, q10, q0
+	vshl.i64	q0, q0, #25
+	vadd.i64	q8, q6, q1
+	vadd.i64	q4, q4, q13
+	vshl.i64	q10, q13, #25
+	vadd.i64	q1, q4, q1
+	vsub.i64	q0, q9, q0
+	vshr.s64	q8, q8, #26
+	vsub.i64	q3, q3, q10
+	vtrn.32		d14, d0
+	vshr.s64	q1, q1, #26
+	vtrn.32		d15, d1
+	vadd.i64	q0, q11, q8
+	vst1.8		d14, [r2, : 64]
+	vshl.i64	q7, q8, #26
+	vadd.i64	q5, q5, q1
+	vtrn.32		d4, d6
+	vshl.i64	q1, q1, #26
+	vtrn.32		d5, d7
+	vsub.i64	q3, q6, q7
+	add		r2, r2, #16
+	vsub.i64	q1, q4, q1
+	vst1.8		d4, [r2, : 64]
+	vtrn.32		d6, d0
+	vtrn.32		d7, d1
+	sub		r2, r2, #8
+	vtrn.32		d2, d10
+	vtrn.32		d3, d11
+	vst1.8		d6, [r2, : 64]
+	sub		r2, r2, #24
+	vst1.8		d2, [r2, : 64]
+	add		r2, r3, #96
+	vmov.i32	q0, #0
+	vmov.i64	d2, #0xff
+	vmov.i64	d3, #0
+	vshr.u32	q1, q1, #7
+	vst1.8		{d2-d3}, [r2, : 128]!
+	vst1.8		{d0-d1}, [r2, : 128]!
+	vst1.8		d0, [r2, : 64]
+	add		r2, r3, #144
+	vmov.i32	q0, #0
+	vst1.8		{d0-d1}, [r2, : 128]!
+	vst1.8		{d0-d1}, [r2, : 128]!
+	vst1.8		d0, [r2, : 64]
+	add		r2, r3, #240
+	vmov.i32	q0, #0
+	vmov.i64	d2, #0xff
+	vmov.i64	d3, #0
+	vshr.u32	q1, q1, #7
+	vst1.8		{d2-d3}, [r2, : 128]!
+	vst1.8		{d0-d1}, [r2, : 128]!
+	vst1.8		d0, [r2, : 64]
+	add		r2, r3, #48
+	add		r6, r3, #192
+	vld1.8		{d0-d1}, [r2, : 128]!
+	vld1.8		{d2-d3}, [r2, : 128]!
+	vld1.8		{d4}, [r2, : 64]
+	vst1.8		{d0-d1}, [r6, : 128]!
+	vst1.8		{d2-d3}, [r6, : 128]!
+	vst1.8		d4, [r6, : 64]
+.Lmainloop:
+	mov		r2, r5, LSR #3
+	and		r6, r5, #7
+	ldrb		r2, [r1, r2]
+	mov		r2, r2, LSR r6
+	and		r2, r2, #1
+	str		r5, [sp, #456]
+	eor		r4, r4, r2
+	str		r2, [sp, #460]
+	neg		r2, r4
+	add		r4, r3, #96
+	add		r5, r3, #192
+	add		r6, r3, #144
+	vld1.8		{d8-d9}, [r4, : 128]!
+	add		r7, r3, #240
+	vld1.8		{d10-d11}, [r5, : 128]!
+	veor		q6, q4, q5
+	vld1.8		{d14-d15}, [r6, : 128]!
+	vdup.i32	q8, r2
+	vld1.8		{d18-d19}, [r7, : 128]!
+	veor		q10, q7, q9
+	vld1.8		{d22-d23}, [r4, : 128]!
+	vand		q6, q6, q8
+	vld1.8		{d24-d25}, [r5, : 128]!
+	vand		q10, q10, q8
+	vld1.8		{d26-d27}, [r6, : 128]!
+	veor		q4, q4, q6
+	vld1.8		{d28-d29}, [r7, : 128]!
+	veor		q5, q5, q6
+	vld1.8		{d0}, [r4, : 64]
+	veor		q6, q7, q10
+	vld1.8		{d2}, [r5, : 64]
+	veor		q7, q9, q10
+	vld1.8		{d4}, [r6, : 64]
+	veor		q9, q11, q12
+	vld1.8		{d6}, [r7, : 64]
+	veor		q10, q0, q1
+	sub		r2, r4, #32
+	vand		q9, q9, q8
+	sub		r4, r5, #32
+	vand		q10, q10, q8
+	sub		r5, r6, #32
+	veor		q11, q11, q9
+	sub		r6, r7, #32
+	veor		q0, q0, q10
+	veor		q9, q12, q9
+	veor		q1, q1, q10
+	veor		q10, q13, q14
+	veor		q12, q2, q3
+	vand		q10, q10, q8
+	vand		q8, q12, q8
+	veor		q12, q13, q10
+	veor		q2, q2, q8
+	veor		q10, q14, q10
+	veor		q3, q3, q8
+	vadd.i32	q8, q4, q6
+	vsub.i32	q4, q4, q6
+	vst1.8		{d16-d17}, [r2, : 128]!
+	vadd.i32	q6, q11, q12
+	vst1.8		{d8-d9}, [r5, : 128]!
+	vsub.i32	q4, q11, q12
+	vst1.8		{d12-d13}, [r2, : 128]!
+	vadd.i32	q6, q0, q2
+	vst1.8		{d8-d9}, [r5, : 128]!
+	vsub.i32	q0, q0, q2
+	vst1.8		d12, [r2, : 64]
+	vadd.i32	q2, q5, q7
+	vst1.8		d0, [r5, : 64]
+	vsub.i32	q0, q5, q7
+	vst1.8		{d4-d5}, [r4, : 128]!
+	vadd.i32	q2, q9, q10
+	vst1.8		{d0-d1}, [r6, : 128]!
+	vsub.i32	q0, q9, q10
+	vst1.8		{d4-d5}, [r4, : 128]!
+	vadd.i32	q2, q1, q3
+	vst1.8		{d0-d1}, [r6, : 128]!
+	vsub.i32	q0, q1, q3
+	vst1.8		d4, [r4, : 64]
+	vst1.8		d0, [r6, : 64]
+	add		r2, sp, #512
+	add		r4, r3, #96
+	add		r5, r3, #144
+	vld1.8		{d0-d1}, [r2, : 128]
+	vld1.8		{d2-d3}, [r4, : 128]!
+	vld1.8		{d4-d5}, [r5, : 128]!
+	vzip.i32	q1, q2
+	vld1.8		{d6-d7}, [r4, : 128]!
+	vld1.8		{d8-d9}, [r5, : 128]!
+	vshl.i32	q5, q1, #1
+	vzip.i32	q3, q4
+	vshl.i32	q6, q2, #1
+	vld1.8		{d14}, [r4, : 64]
+	vshl.i32	q8, q3, #1
+	vld1.8		{d15}, [r5, : 64]
+	vshl.i32	q9, q4, #1
+	vmul.i32	d21, d7, d1
+	vtrn.32		d14, d15
+	vmul.i32	q11, q4, q0
+	vmul.i32	q0, q7, q0
+	vmull.s32	q12, d2, d2
+	vmlal.s32	q12, d11, d1
+	vmlal.s32	q12, d12, d0
+	vmlal.s32	q12, d13, d23
+	vmlal.s32	q12, d16, d22
+	vmlal.s32	q12, d7, d21
+	vmull.s32	q10, d2, d11
+	vmlal.s32	q10, d4, d1
+	vmlal.s32	q10, d13, d0
+	vmlal.s32	q10, d6, d23
+	vmlal.s32	q10, d17, d22
+	vmull.s32	q13, d10, d4
+	vmlal.s32	q13, d11, d3
+	vmlal.s32	q13, d13, d1
+	vmlal.s32	q13, d16, d0
+	vmlal.s32	q13, d17, d23
+	vmlal.s32	q13, d8, d22
+	vmull.s32	q1, d10, d5
+	vmlal.s32	q1, d11, d4
+	vmlal.s32	q1, d6, d1
+	vmlal.s32	q1, d17, d0
+	vmlal.s32	q1, d8, d23
+	vmull.s32	q14, d10, d6
+	vmlal.s32	q14, d11, d13
+	vmlal.s32	q14, d4, d4
+	vmlal.s32	q14, d17, d1
+	vmlal.s32	q14, d18, d0
+	vmlal.s32	q14, d9, d23
+	vmull.s32	q11, d10, d7
+	vmlal.s32	q11, d11, d6
+	vmlal.s32	q11, d12, d5
+	vmlal.s32	q11, d8, d1
+	vmlal.s32	q11, d19, d0
+	vmull.s32	q15, d10, d8
+	vmlal.s32	q15, d11, d17
+	vmlal.s32	q15, d12, d6
+	vmlal.s32	q15, d13, d5
+	vmlal.s32	q15, d19, d1
+	vmlal.s32	q15, d14, d0
+	vmull.s32	q2, d10, d9
+	vmlal.s32	q2, d11, d8
+	vmlal.s32	q2, d12, d7
+	vmlal.s32	q2, d13, d6
+	vmlal.s32	q2, d14, d1
+	vmull.s32	q0, d15, d1
+	vmlal.s32	q0, d10, d14
+	vmlal.s32	q0, d11, d19
+	vmlal.s32	q0, d12, d8
+	vmlal.s32	q0, d13, d17
+	vmlal.s32	q0, d6, d6
+	add		r2, sp, #480
+	vld1.8		{d18-d19}, [r2, : 128]!
+	vmull.s32	q3, d16, d7
+	vmlal.s32	q3, d10, d15
+	vmlal.s32	q3, d11, d14
+	vmlal.s32	q3, d12, d9
+	vmlal.s32	q3, d13, d8
+	vld1.8		{d8-d9}, [r2, : 128]
+	vadd.i64	q5, q12, q9
+	vadd.i64	q6, q15, q9
+	vshr.s64	q5, q5, #26
+	vshr.s64	q6, q6, #26
+	vadd.i64	q7, q10, q5
+	vshl.i64	q5, q5, #26
+	vadd.i64	q8, q7, q4
+	vadd.i64	q2, q2, q6
+	vshl.i64	q6, q6, #26
+	vadd.i64	q10, q2, q4
+	vsub.i64	q5, q12, q5
+	vshr.s64	q8, q8, #25
+	vsub.i64	q6, q15, q6
+	vshr.s64	q10, q10, #25
+	vadd.i64	q12, q13, q8
+	vshl.i64	q8, q8, #25
+	vadd.i64	q13, q12, q9
+	vadd.i64	q0, q0, q10
+	vsub.i64	q7, q7, q8
+	vshr.s64	q8, q13, #26
+	vshl.i64	q10, q10, #25
+	vadd.i64	q13, q0, q9
+	vadd.i64	q1, q1, q8
+	vshl.i64	q8, q8, #26
+	vadd.i64	q15, q1, q4
+	vsub.i64	q2, q2, q10
+	vshr.s64	q10, q13, #26
+	vsub.i64	q8, q12, q8
+	vshr.s64	q12, q15, #25
+	vadd.i64	q3, q3, q10
+	vshl.i64	q10, q10, #26
+	vadd.i64	q13, q3, q4
+	vadd.i64	q14, q14, q12
+	add		r2, r3, #288
+	vshl.i64	q12, q12, #25
+	add		r4, r3, #336
+	vadd.i64	q15, q14, q9
+	add		r2, r2, #8
+	vsub.i64	q0, q0, q10
+	add		r4, r4, #8
+	vshr.s64	q10, q13, #25
+	vsub.i64	q1, q1, q12
+	vshr.s64	q12, q15, #26
+	vadd.i64	q13, q10, q10
+	vadd.i64	q11, q11, q12
+	vtrn.32		d16, d2
+	vshl.i64	q12, q12, #26
+	vtrn.32		d17, d3
+	vadd.i64	q1, q11, q4
+	vadd.i64	q4, q5, q13
+	vst1.8		d16, [r2, : 64]!
+	vshl.i64	q5, q10, #4
+	vst1.8		d17, [r4, : 64]!
+	vsub.i64	q8, q14, q12
+	vshr.s64	q1, q1, #25
+	vadd.i64	q4, q4, q5
+	vadd.i64	q5, q6, q1
+	vshl.i64	q1, q1, #25
+	vadd.i64	q6, q5, q9
+	vadd.i64	q4, q4, q10
+	vshl.i64	q10, q10, #25
+	vadd.i64	q9, q4, q9
+	vsub.i64	q1, q11, q1
+	vshr.s64	q6, q6, #26
+	vsub.i64	q3, q3, q10
+	vtrn.32		d16, d2
+	vshr.s64	q9, q9, #26
+	vtrn.32		d17, d3
+	vadd.i64	q1, q2, q6
+	vst1.8		d16, [r2, : 64]
+	vshl.i64	q2, q6, #26
+	vst1.8		d17, [r4, : 64]
+	vadd.i64	q6, q7, q9
+	vtrn.32		d0, d6
+	vshl.i64	q7, q9, #26
+	vtrn.32		d1, d7
+	vsub.i64	q2, q5, q2
+	add		r2, r2, #16
+	vsub.i64	q3, q4, q7
+	vst1.8		d0, [r2, : 64]
+	add		r4, r4, #16
+	vst1.8		d1, [r4, : 64]
+	vtrn.32		d4, d2
+	vtrn.32		d5, d3
+	sub		r2, r2, #8
+	sub		r4, r4, #8
+	vtrn.32		d6, d12
+	vtrn.32		d7, d13
+	vst1.8		d4, [r2, : 64]
+	vst1.8		d5, [r4, : 64]
+	sub		r2, r2, #24
+	sub		r4, r4, #24
+	vst1.8		d6, [r2, : 64]
+	vst1.8		d7, [r4, : 64]
+	add		r2, r3, #240
+	add		r4, r3, #96
+	vld1.8		{d0-d1}, [r4, : 128]!
+	vld1.8		{d2-d3}, [r4, : 128]!
+	vld1.8		{d4}, [r4, : 64]
+	add		r4, r3, #144
+	vld1.8		{d6-d7}, [r4, : 128]!
+	vtrn.32		q0, q3
+	vld1.8		{d8-d9}, [r4, : 128]!
+	vshl.i32	q5, q0, #4
+	vtrn.32		q1, q4
+	vshl.i32	q6, q3, #4
+	vadd.i32	q5, q5, q0
+	vadd.i32	q6, q6, q3
+	vshl.i32	q7, q1, #4
+	vld1.8		{d5}, [r4, : 64]
+	vshl.i32	q8, q4, #4
+	vtrn.32		d4, d5
+	vadd.i32	q7, q7, q1
+	vadd.i32	q8, q8, q4
+	vld1.8		{d18-d19}, [r2, : 128]!
+	vshl.i32	q10, q2, #4
+	vld1.8		{d22-d23}, [r2, : 128]!
+	vadd.i32	q10, q10, q2
+	vld1.8		{d24}, [r2, : 64]
+	vadd.i32	q5, q5, q0
+	add		r2, r3, #192
+	vld1.8		{d26-d27}, [r2, : 128]!
+	vadd.i32	q6, q6, q3
+	vld1.8		{d28-d29}, [r2, : 128]!
+	vadd.i32	q8, q8, q4
+	vld1.8		{d25}, [r2, : 64]
+	vadd.i32	q10, q10, q2
+	vtrn.32		q9, q13
+	vadd.i32	q7, q7, q1
+	vadd.i32	q5, q5, q0
+	vtrn.32		q11, q14
+	vadd.i32	q6, q6, q3
+	add		r2, sp, #528
+	vadd.i32	q10, q10, q2
+	vtrn.32		d24, d25
+	vst1.8		{d12-d13}, [r2, : 128]!
+	vshl.i32	q6, q13, #1
+	vst1.8		{d20-d21}, [r2, : 128]!
+	vshl.i32	q10, q14, #1
+	vst1.8		{d12-d13}, [r2, : 128]!
+	vshl.i32	q15, q12, #1
+	vadd.i32	q8, q8, q4
+	vext.32		d10, d31, d30, #0
+	vadd.i32	q7, q7, q1
+	vst1.8		{d16-d17}, [r2, : 128]!
+	vmull.s32	q8, d18, d5
+	vmlal.s32	q8, d26, d4
+	vmlal.s32	q8, d19, d9
+	vmlal.s32	q8, d27, d3
+	vmlal.s32	q8, d22, d8
+	vmlal.s32	q8, d28, d2
+	vmlal.s32	q8, d23, d7
+	vmlal.s32	q8, d29, d1
+	vmlal.s32	q8, d24, d6
+	vmlal.s32	q8, d25, d0
+	vst1.8		{d14-d15}, [r2, : 128]!
+	vmull.s32	q2, d18, d4
+	vmlal.s32	q2, d12, d9
+	vmlal.s32	q2, d13, d8
+	vmlal.s32	q2, d19, d3
+	vmlal.s32	q2, d22, d2
+	vmlal.s32	q2, d23, d1
+	vmlal.s32	q2, d24, d0
+	vst1.8		{d20-d21}, [r2, : 128]!
+	vmull.s32	q7, d18, d9
+	vmlal.s32	q7, d26, d3
+	vmlal.s32	q7, d19, d8
+	vmlal.s32	q7, d27, d2
+	vmlal.s32	q7, d22, d7
+	vmlal.s32	q7, d28, d1
+	vmlal.s32	q7, d23, d6
+	vmlal.s32	q7, d29, d0
+	vst1.8		{d10-d11}, [r2, : 128]!
+	vmull.s32	q5, d18, d3
+	vmlal.s32	q5, d19, d2
+	vmlal.s32	q5, d22, d1
+	vmlal.s32	q5, d23, d0
+	vmlal.s32	q5, d12, d8
+	vst1.8		{d16-d17}, [r2, : 128]
+	vmull.s32	q4, d18, d8
+	vmlal.s32	q4, d26, d2
+	vmlal.s32	q4, d19, d7
+	vmlal.s32	q4, d27, d1
+	vmlal.s32	q4, d22, d6
+	vmlal.s32	q4, d28, d0
+	vmull.s32	q8, d18, d7
+	vmlal.s32	q8, d26, d1
+	vmlal.s32	q8, d19, d6
+	vmlal.s32	q8, d27, d0
+	add		r2, sp, #544
+	vld1.8		{d20-d21}, [r2, : 128]
+	vmlal.s32	q7, d24, d21
+	vmlal.s32	q7, d25, d20
+	vmlal.s32	q4, d23, d21
+	vmlal.s32	q4, d29, d20
+	vmlal.s32	q8, d22, d21
+	vmlal.s32	q8, d28, d20
+	vmlal.s32	q5, d24, d20
+	vst1.8		{d14-d15}, [r2, : 128]
+	vmull.s32	q7, d18, d6
+	vmlal.s32	q7, d26, d0
+	add		r2, sp, #624
+	vld1.8		{d30-d31}, [r2, : 128]
+	vmlal.s32	q2, d30, d21
+	vmlal.s32	q7, d19, d21
+	vmlal.s32	q7, d27, d20
+	add		r2, sp, #592
+	vld1.8		{d26-d27}, [r2, : 128]
+	vmlal.s32	q4, d25, d27
+	vmlal.s32	q8, d29, d27
+	vmlal.s32	q8, d25, d26
+	vmlal.s32	q7, d28, d27
+	vmlal.s32	q7, d29, d26
+	add		r2, sp, #576
+	vld1.8		{d28-d29}, [r2, : 128]
+	vmlal.s32	q4, d24, d29
+	vmlal.s32	q8, d23, d29
+	vmlal.s32	q8, d24, d28
+	vmlal.s32	q7, d22, d29
+	vmlal.s32	q7, d23, d28
+	vst1.8		{d8-d9}, [r2, : 128]
+	add		r2, sp, #528
+	vld1.8		{d8-d9}, [r2, : 128]
+	vmlal.s32	q7, d24, d9
+	vmlal.s32	q7, d25, d31
+	vmull.s32	q1, d18, d2
+	vmlal.s32	q1, d19, d1
+	vmlal.s32	q1, d22, d0
+	vmlal.s32	q1, d24, d27
+	vmlal.s32	q1, d23, d20
+	vmlal.s32	q1, d12, d7
+	vmlal.s32	q1, d13, d6
+	vmull.s32	q6, d18, d1
+	vmlal.s32	q6, d19, d0
+	vmlal.s32	q6, d23, d27
+	vmlal.s32	q6, d22, d20
+	vmlal.s32	q6, d24, d26
+	vmull.s32	q0, d18, d0
+	vmlal.s32	q0, d22, d27
+	vmlal.s32	q0, d23, d26
+	vmlal.s32	q0, d24, d31
+	vmlal.s32	q0, d19, d20
+	add		r2, sp, #608
+	vld1.8		{d18-d19}, [r2, : 128]
+	vmlal.s32	q2, d18, d7
+	vmlal.s32	q5, d18, d6
+	vmlal.s32	q1, d18, d21
+	vmlal.s32	q0, d18, d28
+	vmlal.s32	q6, d18, d29
+	vmlal.s32	q2, d19, d6
+	vmlal.s32	q5, d19, d21
+	vmlal.s32	q1, d19, d29
+	vmlal.s32	q0, d19, d9
+	vmlal.s32	q6, d19, d28
+	add		r2, sp, #560
+	vld1.8		{d18-d19}, [r2, : 128]
+	add		r2, sp, #480
+	vld1.8		{d22-d23}, [r2, : 128]
+	vmlal.s32	q5, d19, d7
+	vmlal.s32	q0, d18, d21
+	vmlal.s32	q0, d19, d29
+	vmlal.s32	q6, d18, d6
+	add		r2, sp, #496
+	vld1.8		{d6-d7}, [r2, : 128]
+	vmlal.s32	q6, d19, d21
+	add		r2, sp, #544
+	vld1.8		{d18-d19}, [r2, : 128]
+	vmlal.s32	q0, d30, d8
+	add		r2, sp, #640
+	vld1.8		{d20-d21}, [r2, : 128]
+	vmlal.s32	q5, d30, d29
+	add		r2, sp, #576
+	vld1.8		{d24-d25}, [r2, : 128]
+	vmlal.s32	q1, d30, d28
+	vadd.i64	q13, q0, q11
+	vadd.i64	q14, q5, q11
+	vmlal.s32	q6, d30, d9
+	vshr.s64	q4, q13, #26
+	vshr.s64	q13, q14, #26
+	vadd.i64	q7, q7, q4
+	vshl.i64	q4, q4, #26
+	vadd.i64	q14, q7, q3
+	vadd.i64	q9, q9, q13
+	vshl.i64	q13, q13, #26
+	vadd.i64	q15, q9, q3
+	vsub.i64	q0, q0, q4
+	vshr.s64	q4, q14, #25
+	vsub.i64	q5, q5, q13
+	vshr.s64	q13, q15, #25
+	vadd.i64	q6, q6, q4
+	vshl.i64	q4, q4, #25
+	vadd.i64	q14, q6, q11
+	vadd.i64	q2, q2, q13
+	vsub.i64	q4, q7, q4
+	vshr.s64	q7, q14, #26
+	vshl.i64	q13, q13, #25
+	vadd.i64	q14, q2, q11
+	vadd.i64	q8, q8, q7
+	vshl.i64	q7, q7, #26
+	vadd.i64	q15, q8, q3
+	vsub.i64	q9, q9, q13
+	vshr.s64	q13, q14, #26
+	vsub.i64	q6, q6, q7
+	vshr.s64	q7, q15, #25
+	vadd.i64	q10, q10, q13
+	vshl.i64	q13, q13, #26
+	vadd.i64	q14, q10, q3
+	vadd.i64	q1, q1, q7
+	add		r2, r3, #144
+	vshl.i64	q7, q7, #25
+	add		r4, r3, #96
+	vadd.i64	q15, q1, q11
+	add		r2, r2, #8
+	vsub.i64	q2, q2, q13
+	add		r4, r4, #8
+	vshr.s64	q13, q14, #25
+	vsub.i64	q7, q8, q7
+	vshr.s64	q8, q15, #26
+	vadd.i64	q14, q13, q13
+	vadd.i64	q12, q12, q8
+	vtrn.32		d12, d14
+	vshl.i64	q8, q8, #26
+	vtrn.32		d13, d15
+	vadd.i64	q3, q12, q3
+	vadd.i64	q0, q0, q14
+	vst1.8		d12, [r2, : 64]!
+	vshl.i64	q7, q13, #4
+	vst1.8		d13, [r4, : 64]!
+	vsub.i64	q1, q1, q8
+	vshr.s64	q3, q3, #25
+	vadd.i64	q0, q0, q7
+	vadd.i64	q5, q5, q3
+	vshl.i64	q3, q3, #25
+	vadd.i64	q6, q5, q11
+	vadd.i64	q0, q0, q13
+	vshl.i64	q7, q13, #25
+	vadd.i64	q8, q0, q11
+	vsub.i64	q3, q12, q3
+	vshr.s64	q6, q6, #26
+	vsub.i64	q7, q10, q7
+	vtrn.32		d2, d6
+	vshr.s64	q8, q8, #26
+	vtrn.32		d3, d7
+	vadd.i64	q3, q9, q6
+	vst1.8		d2, [r2, : 64]
+	vshl.i64	q6, q6, #26
+	vst1.8		d3, [r4, : 64]
+	vadd.i64	q1, q4, q8
+	vtrn.32		d4, d14
+	vshl.i64	q4, q8, #26
+	vtrn.32		d5, d15
+	vsub.i64	q5, q5, q6
+	add		r2, r2, #16
+	vsub.i64	q0, q0, q4
+	vst1.8		d4, [r2, : 64]
+	add		r4, r4, #16
+	vst1.8		d5, [r4, : 64]
+	vtrn.32		d10, d6
+	vtrn.32		d11, d7
+	sub		r2, r2, #8
+	sub		r4, r4, #8
+	vtrn.32		d0, d2
+	vtrn.32		d1, d3
+	vst1.8		d10, [r2, : 64]
+	vst1.8		d11, [r4, : 64]
+	sub		r2, r2, #24
+	sub		r4, r4, #24
+	vst1.8		d0, [r2, : 64]
+	vst1.8		d1, [r4, : 64]
+	add		r2, r3, #288
+	add		r4, r3, #336
+	vld1.8		{d0-d1}, [r2, : 128]!
+	vld1.8		{d2-d3}, [r4, : 128]!
+	vsub.i32	q0, q0, q1
+	vld1.8		{d2-d3}, [r2, : 128]!
+	vld1.8		{d4-d5}, [r4, : 128]!
+	vsub.i32	q1, q1, q2
+	add		r5, r3, #240
+	vld1.8		{d4}, [r2, : 64]
+	vld1.8		{d6}, [r4, : 64]
+	vsub.i32	q2, q2, q3
+	vst1.8		{d0-d1}, [r5, : 128]!
+	vst1.8		{d2-d3}, [r5, : 128]!
+	vst1.8		d4, [r5, : 64]
+	add		r2, r3, #144
+	add		r4, r3, #96
+	add		r5, r3, #144
+	add		r6, r3, #192
+	vld1.8		{d0-d1}, [r2, : 128]!
+	vld1.8		{d2-d3}, [r4, : 128]!
+	vsub.i32	q2, q0, q1
+	vadd.i32	q0, q0, q1
+	vld1.8		{d2-d3}, [r2, : 128]!
+	vld1.8		{d6-d7}, [r4, : 128]!
+	vsub.i32	q4, q1, q3
+	vadd.i32	q1, q1, q3
+	vld1.8		{d6}, [r2, : 64]
+	vld1.8		{d10}, [r4, : 64]
+	vsub.i32	q6, q3, q5
+	vadd.i32	q3, q3, q5
+	vst1.8		{d4-d5}, [r5, : 128]!
+	vst1.8		{d0-d1}, [r6, : 128]!
+	vst1.8		{d8-d9}, [r5, : 128]!
+	vst1.8		{d2-d3}, [r6, : 128]!
+	vst1.8		d12, [r5, : 64]
+	vst1.8		d6, [r6, : 64]
+	add		r2, r3, #0
+	add		r4, r3, #240
+	vld1.8		{d0-d1}, [r4, : 128]!
+	vld1.8		{d2-d3}, [r4, : 128]!
+	vld1.8		{d4}, [r4, : 64]
+	add		r4, r3, #336
+	vld1.8		{d6-d7}, [r4, : 128]!
+	vtrn.32		q0, q3
+	vld1.8		{d8-d9}, [r4, : 128]!
+	vshl.i32	q5, q0, #4
+	vtrn.32		q1, q4
+	vshl.i32	q6, q3, #4
+	vadd.i32	q5, q5, q0
+	vadd.i32	q6, q6, q3
+	vshl.i32	q7, q1, #4
+	vld1.8		{d5}, [r4, : 64]
+	vshl.i32	q8, q4, #4
+	vtrn.32		d4, d5
+	vadd.i32	q7, q7, q1
+	vadd.i32	q8, q8, q4
+	vld1.8		{d18-d19}, [r2, : 128]!
+	vshl.i32	q10, q2, #4
+	vld1.8		{d22-d23}, [r2, : 128]!
+	vadd.i32	q10, q10, q2
+	vld1.8		{d24}, [r2, : 64]
+	vadd.i32	q5, q5, q0
+	add		r2, r3, #288
+	vld1.8		{d26-d27}, [r2, : 128]!
+	vadd.i32	q6, q6, q3
+	vld1.8		{d28-d29}, [r2, : 128]!
+	vadd.i32	q8, q8, q4
+	vld1.8		{d25}, [r2, : 64]
+	vadd.i32	q10, q10, q2
+	vtrn.32		q9, q13
+	vadd.i32	q7, q7, q1
+	vadd.i32	q5, q5, q0
+	vtrn.32		q11, q14
+	vadd.i32	q6, q6, q3
+	add		r2, sp, #528
+	vadd.i32	q10, q10, q2
+	vtrn.32		d24, d25
+	vst1.8		{d12-d13}, [r2, : 128]!
+	vshl.i32	q6, q13, #1
+	vst1.8		{d20-d21}, [r2, : 128]!
+	vshl.i32	q10, q14, #1
+	vst1.8		{d12-d13}, [r2, : 128]!
+	vshl.i32	q15, q12, #1
+	vadd.i32	q8, q8, q4
+	vext.32		d10, d31, d30, #0
+	vadd.i32	q7, q7, q1
+	vst1.8		{d16-d17}, [r2, : 128]!
+	vmull.s32	q8, d18, d5
+	vmlal.s32	q8, d26, d4
+	vmlal.s32	q8, d19, d9
+	vmlal.s32	q8, d27, d3
+	vmlal.s32	q8, d22, d8
+	vmlal.s32	q8, d28, d2
+	vmlal.s32	q8, d23, d7
+	vmlal.s32	q8, d29, d1
+	vmlal.s32	q8, d24, d6
+	vmlal.s32	q8, d25, d0
+	vst1.8		{d14-d15}, [r2, : 128]!
+	vmull.s32	q2, d18, d4
+	vmlal.s32	q2, d12, d9
+	vmlal.s32	q2, d13, d8
+	vmlal.s32	q2, d19, d3
+	vmlal.s32	q2, d22, d2
+	vmlal.s32	q2, d23, d1
+	vmlal.s32	q2, d24, d0
+	vst1.8		{d20-d21}, [r2, : 128]!
+	vmull.s32	q7, d18, d9
+	vmlal.s32	q7, d26, d3
+	vmlal.s32	q7, d19, d8
+	vmlal.s32	q7, d27, d2
+	vmlal.s32	q7, d22, d7
+	vmlal.s32	q7, d28, d1
+	vmlal.s32	q7, d23, d6
+	vmlal.s32	q7, d29, d0
+	vst1.8		{d10-d11}, [r2, : 128]!
+	vmull.s32	q5, d18, d3
+	vmlal.s32	q5, d19, d2
+	vmlal.s32	q5, d22, d1
+	vmlal.s32	q5, d23, d0
+	vmlal.s32	q5, d12, d8
+	vst1.8		{d16-d17}, [r2, : 128]!
+	vmull.s32	q4, d18, d8
+	vmlal.s32	q4, d26, d2
+	vmlal.s32	q4, d19, d7
+	vmlal.s32	q4, d27, d1
+	vmlal.s32	q4, d22, d6
+	vmlal.s32	q4, d28, d0
+	vmull.s32	q8, d18, d7
+	vmlal.s32	q8, d26, d1
+	vmlal.s32	q8, d19, d6
+	vmlal.s32	q8, d27, d0
+	add		r2, sp, #544
+	vld1.8		{d20-d21}, [r2, : 128]
+	vmlal.s32	q7, d24, d21
+	vmlal.s32	q7, d25, d20
+	vmlal.s32	q4, d23, d21
+	vmlal.s32	q4, d29, d20
+	vmlal.s32	q8, d22, d21
+	vmlal.s32	q8, d28, d20
+	vmlal.s32	q5, d24, d20
+	vst1.8		{d14-d15}, [r2, : 128]
+	vmull.s32	q7, d18, d6
+	vmlal.s32	q7, d26, d0
+	add		r2, sp, #624
+	vld1.8		{d30-d31}, [r2, : 128]
+	vmlal.s32	q2, d30, d21
+	vmlal.s32	q7, d19, d21
+	vmlal.s32	q7, d27, d20
+	add		r2, sp, #592
+	vld1.8		{d26-d27}, [r2, : 128]
+	vmlal.s32	q4, d25, d27
+	vmlal.s32	q8, d29, d27
+	vmlal.s32	q8, d25, d26
+	vmlal.s32	q7, d28, d27
+	vmlal.s32	q7, d29, d26
+	add		r2, sp, #576
+	vld1.8		{d28-d29}, [r2, : 128]
+	vmlal.s32	q4, d24, d29
+	vmlal.s32	q8, d23, d29
+	vmlal.s32	q8, d24, d28
+	vmlal.s32	q7, d22, d29
+	vmlal.s32	q7, d23, d28
+	vst1.8		{d8-d9}, [r2, : 128]
+	add		r2, sp, #528
+	vld1.8		{d8-d9}, [r2, : 128]
+	vmlal.s32	q7, d24, d9
+	vmlal.s32	q7, d25, d31
+	vmull.s32	q1, d18, d2
+	vmlal.s32	q1, d19, d1
+	vmlal.s32	q1, d22, d0
+	vmlal.s32	q1, d24, d27
+	vmlal.s32	q1, d23, d20
+	vmlal.s32	q1, d12, d7
+	vmlal.s32	q1, d13, d6
+	vmull.s32	q6, d18, d1
+	vmlal.s32	q6, d19, d0
+	vmlal.s32	q6, d23, d27
+	vmlal.s32	q6, d22, d20
+	vmlal.s32	q6, d24, d26
+	vmull.s32	q0, d18, d0
+	vmlal.s32	q0, d22, d27
+	vmlal.s32	q0, d23, d26
+	vmlal.s32	q0, d24, d31
+	vmlal.s32	q0, d19, d20
+	add		r2, sp, #608
+	vld1.8		{d18-d19}, [r2, : 128]
+	vmlal.s32	q2, d18, d7
+	vmlal.s32	q5, d18, d6
+	vmlal.s32	q1, d18, d21
+	vmlal.s32	q0, d18, d28
+	vmlal.s32	q6, d18, d29
+	vmlal.s32	q2, d19, d6
+	vmlal.s32	q5, d19, d21
+	vmlal.s32	q1, d19, d29
+	vmlal.s32	q0, d19, d9
+	vmlal.s32	q6, d19, d28
+	add		r2, sp, #560
+	vld1.8		{d18-d19}, [r2, : 128]
+	add		r2, sp, #480
+	vld1.8		{d22-d23}, [r2, : 128]
+	vmlal.s32	q5, d19, d7
+	vmlal.s32	q0, d18, d21
+	vmlal.s32	q0, d19, d29
+	vmlal.s32	q6, d18, d6
+	add		r2, sp, #496
+	vld1.8		{d6-d7}, [r2, : 128]
+	vmlal.s32	q6, d19, d21
+	add		r2, sp, #544
+	vld1.8		{d18-d19}, [r2, : 128]
+	vmlal.s32	q0, d30, d8
+	add		r2, sp, #640
+	vld1.8		{d20-d21}, [r2, : 128]
+	vmlal.s32	q5, d30, d29
+	add		r2, sp, #576
+	vld1.8		{d24-d25}, [r2, : 128]
+	vmlal.s32	q1, d30, d28
+	vadd.i64	q13, q0, q11
+	vadd.i64	q14, q5, q11
+	vmlal.s32	q6, d30, d9
+	vshr.s64	q4, q13, #26
+	vshr.s64	q13, q14, #26
+	vadd.i64	q7, q7, q4
+	vshl.i64	q4, q4, #26
+	vadd.i64	q14, q7, q3
+	vadd.i64	q9, q9, q13
+	vshl.i64	q13, q13, #26
+	vadd.i64	q15, q9, q3
+	vsub.i64	q0, q0, q4
+	vshr.s64	q4, q14, #25
+	vsub.i64	q5, q5, q13
+	vshr.s64	q13, q15, #25
+	vadd.i64	q6, q6, q4
+	vshl.i64	q4, q4, #25
+	vadd.i64	q14, q6, q11
+	vadd.i64	q2, q2, q13
+	vsub.i64	q4, q7, q4
+	vshr.s64	q7, q14, #26
+	vshl.i64	q13, q13, #25
+	vadd.i64	q14, q2, q11
+	vadd.i64	q8, q8, q7
+	vshl.i64	q7, q7, #26
+	vadd.i64	q15, q8, q3
+	vsub.i64	q9, q9, q13
+	vshr.s64	q13, q14, #26
+	vsub.i64	q6, q6, q7
+	vshr.s64	q7, q15, #25
+	vadd.i64	q10, q10, q13
+	vshl.i64	q13, q13, #26
+	vadd.i64	q14, q10, q3
+	vadd.i64	q1, q1, q7
+	add		r2, r3, #288
+	vshl.i64	q7, q7, #25
+	add		r4, r3, #96
+	vadd.i64	q15, q1, q11
+	add		r2, r2, #8
+	vsub.i64	q2, q2, q13
+	add		r4, r4, #8
+	vshr.s64	q13, q14, #25
+	vsub.i64	q7, q8, q7
+	vshr.s64	q8, q15, #26
+	vadd.i64	q14, q13, q13
+	vadd.i64	q12, q12, q8
+	vtrn.32		d12, d14
+	vshl.i64	q8, q8, #26
+	vtrn.32		d13, d15
+	vadd.i64	q3, q12, q3
+	vadd.i64	q0, q0, q14
+	vst1.8		d12, [r2, : 64]!
+	vshl.i64	q7, q13, #4
+	vst1.8		d13, [r4, : 64]!
+	vsub.i64	q1, q1, q8
+	vshr.s64	q3, q3, #25
+	vadd.i64	q0, q0, q7
+	vadd.i64	q5, q5, q3
+	vshl.i64	q3, q3, #25
+	vadd.i64	q6, q5, q11
+	vadd.i64	q0, q0, q13
+	vshl.i64	q7, q13, #25
+	vadd.i64	q8, q0, q11
+	vsub.i64	q3, q12, q3
+	vshr.s64	q6, q6, #26
+	vsub.i64	q7, q10, q7
+	vtrn.32		d2, d6
+	vshr.s64	q8, q8, #26
+	vtrn.32		d3, d7
+	vadd.i64	q3, q9, q6
+	vst1.8		d2, [r2, : 64]
+	vshl.i64	q6, q6, #26
+	vst1.8		d3, [r4, : 64]
+	vadd.i64	q1, q4, q8
+	vtrn.32		d4, d14
+	vshl.i64	q4, q8, #26
+	vtrn.32		d5, d15
+	vsub.i64	q5, q5, q6
+	add		r2, r2, #16
+	vsub.i64	q0, q0, q4
+	vst1.8		d4, [r2, : 64]
+	add		r4, r4, #16
+	vst1.8		d5, [r4, : 64]
+	vtrn.32		d10, d6
+	vtrn.32		d11, d7
+	sub		r2, r2, #8
+	sub		r4, r4, #8
+	vtrn.32		d0, d2
+	vtrn.32		d1, d3
+	vst1.8		d10, [r2, : 64]
+	vst1.8		d11, [r4, : 64]
+	sub		r2, r2, #24
+	sub		r4, r4, #24
+	vst1.8		d0, [r2, : 64]
+	vst1.8		d1, [r4, : 64]
+	add		r2, sp, #512
+	add		r4, r3, #144
+	add		r5, r3, #192
+	vld1.8		{d0-d1}, [r2, : 128]
+	vld1.8		{d2-d3}, [r4, : 128]!
+	vld1.8		{d4-d5}, [r5, : 128]!
+	vzip.i32	q1, q2
+	vld1.8		{d6-d7}, [r4, : 128]!
+	vld1.8		{d8-d9}, [r5, : 128]!
+	vshl.i32	q5, q1, #1
+	vzip.i32	q3, q4
+	vshl.i32	q6, q2, #1
+	vld1.8		{d14}, [r4, : 64]
+	vshl.i32	q8, q3, #1
+	vld1.8		{d15}, [r5, : 64]
+	vshl.i32	q9, q4, #1
+	vmul.i32	d21, d7, d1
+	vtrn.32		d14, d15
+	vmul.i32	q11, q4, q0
+	vmul.i32	q0, q7, q0
+	vmull.s32	q12, d2, d2
+	vmlal.s32	q12, d11, d1
+	vmlal.s32	q12, d12, d0
+	vmlal.s32	q12, d13, d23
+	vmlal.s32	q12, d16, d22
+	vmlal.s32	q12, d7, d21
+	vmull.s32	q10, d2, d11
+	vmlal.s32	q10, d4, d1
+	vmlal.s32	q10, d13, d0
+	vmlal.s32	q10, d6, d23
+	vmlal.s32	q10, d17, d22
+	vmull.s32	q13, d10, d4
+	vmlal.s32	q13, d11, d3
+	vmlal.s32	q13, d13, d1
+	vmlal.s32	q13, d16, d0
+	vmlal.s32	q13, d17, d23
+	vmlal.s32	q13, d8, d22
+	vmull.s32	q1, d10, d5
+	vmlal.s32	q1, d11, d4
+	vmlal.s32	q1, d6, d1
+	vmlal.s32	q1, d17, d0
+	vmlal.s32	q1, d8, d23
+	vmull.s32	q14, d10, d6
+	vmlal.s32	q14, d11, d13
+	vmlal.s32	q14, d4, d4
+	vmlal.s32	q14, d17, d1
+	vmlal.s32	q14, d18, d0
+	vmlal.s32	q14, d9, d23
+	vmull.s32	q11, d10, d7
+	vmlal.s32	q11, d11, d6
+	vmlal.s32	q11, d12, d5
+	vmlal.s32	q11, d8, d1
+	vmlal.s32	q11, d19, d0
+	vmull.s32	q15, d10, d8
+	vmlal.s32	q15, d11, d17
+	vmlal.s32	q15, d12, d6
+	vmlal.s32	q15, d13, d5
+	vmlal.s32	q15, d19, d1
+	vmlal.s32	q15, d14, d0
+	vmull.s32	q2, d10, d9
+	vmlal.s32	q2, d11, d8
+	vmlal.s32	q2, d12, d7
+	vmlal.s32	q2, d13, d6
+	vmlal.s32	q2, d14, d1
+	vmull.s32	q0, d15, d1
+	vmlal.s32	q0, d10, d14
+	vmlal.s32	q0, d11, d19
+	vmlal.s32	q0, d12, d8
+	vmlal.s32	q0, d13, d17
+	vmlal.s32	q0, d6, d6
+	add		r2, sp, #480
+	vld1.8		{d18-d19}, [r2, : 128]!
+	vmull.s32	q3, d16, d7
+	vmlal.s32	q3, d10, d15
+	vmlal.s32	q3, d11, d14
+	vmlal.s32	q3, d12, d9
+	vmlal.s32	q3, d13, d8
+	vld1.8		{d8-d9}, [r2, : 128]
+	vadd.i64	q5, q12, q9
+	vadd.i64	q6, q15, q9
+	vshr.s64	q5, q5, #26
+	vshr.s64	q6, q6, #26
+	vadd.i64	q7, q10, q5
+	vshl.i64	q5, q5, #26
+	vadd.i64	q8, q7, q4
+	vadd.i64	q2, q2, q6
+	vshl.i64	q6, q6, #26
+	vadd.i64	q10, q2, q4
+	vsub.i64	q5, q12, q5
+	vshr.s64	q8, q8, #25
+	vsub.i64	q6, q15, q6
+	vshr.s64	q10, q10, #25
+	vadd.i64	q12, q13, q8
+	vshl.i64	q8, q8, #25
+	vadd.i64	q13, q12, q9
+	vadd.i64	q0, q0, q10
+	vsub.i64	q7, q7, q8
+	vshr.s64	q8, q13, #26
+	vshl.i64	q10, q10, #25
+	vadd.i64	q13, q0, q9
+	vadd.i64	q1, q1, q8
+	vshl.i64	q8, q8, #26
+	vadd.i64	q15, q1, q4
+	vsub.i64	q2, q2, q10
+	vshr.s64	q10, q13, #26
+	vsub.i64	q8, q12, q8
+	vshr.s64	q12, q15, #25
+	vadd.i64	q3, q3, q10
+	vshl.i64	q10, q10, #26
+	vadd.i64	q13, q3, q4
+	vadd.i64	q14, q14, q12
+	add		r2, r3, #144
+	vshl.i64	q12, q12, #25
+	add		r4, r3, #192
+	vadd.i64	q15, q14, q9
+	add		r2, r2, #8
+	vsub.i64	q0, q0, q10
+	add		r4, r4, #8
+	vshr.s64	q10, q13, #25
+	vsub.i64	q1, q1, q12
+	vshr.s64	q12, q15, #26
+	vadd.i64	q13, q10, q10
+	vadd.i64	q11, q11, q12
+	vtrn.32		d16, d2
+	vshl.i64	q12, q12, #26
+	vtrn.32		d17, d3
+	vadd.i64	q1, q11, q4
+	vadd.i64	q4, q5, q13
+	vst1.8		d16, [r2, : 64]!
+	vshl.i64	q5, q10, #4
+	vst1.8		d17, [r4, : 64]!
+	vsub.i64	q8, q14, q12
+	vshr.s64	q1, q1, #25
+	vadd.i64	q4, q4, q5
+	vadd.i64	q5, q6, q1
+	vshl.i64	q1, q1, #25
+	vadd.i64	q6, q5, q9
+	vadd.i64	q4, q4, q10
+	vshl.i64	q10, q10, #25
+	vadd.i64	q9, q4, q9
+	vsub.i64	q1, q11, q1
+	vshr.s64	q6, q6, #26
+	vsub.i64	q3, q3, q10
+	vtrn.32		d16, d2
+	vshr.s64	q9, q9, #26
+	vtrn.32		d17, d3
+	vadd.i64	q1, q2, q6
+	vst1.8		d16, [r2, : 64]
+	vshl.i64	q2, q6, #26
+	vst1.8		d17, [r4, : 64]
+	vadd.i64	q6, q7, q9
+	vtrn.32		d0, d6
+	vshl.i64	q7, q9, #26
+	vtrn.32		d1, d7
+	vsub.i64	q2, q5, q2
+	add		r2, r2, #16
+	vsub.i64	q3, q4, q7
+	vst1.8		d0, [r2, : 64]
+	add		r4, r4, #16
+	vst1.8		d1, [r4, : 64]
+	vtrn.32		d4, d2
+	vtrn.32		d5, d3
+	sub		r2, r2, #8
+	sub		r4, r4, #8
+	vtrn.32		d6, d12
+	vtrn.32		d7, d13
+	vst1.8		d4, [r2, : 64]
+	vst1.8		d5, [r4, : 64]
+	sub		r2, r2, #24
+	sub		r4, r4, #24
+	vst1.8		d6, [r2, : 64]
+	vst1.8		d7, [r4, : 64]
+	add		r2, r3, #336
+	add		r4, r3, #288
+	vld1.8		{d0-d1}, [r2, : 128]!
+	vld1.8		{d2-d3}, [r4, : 128]!
+	vadd.i32	q0, q0, q1
+	vld1.8		{d2-d3}, [r2, : 128]!
+	vld1.8		{d4-d5}, [r4, : 128]!
+	vadd.i32	q1, q1, q2
+	add		r5, r3, #288
+	vld1.8		{d4}, [r2, : 64]
+	vld1.8		{d6}, [r4, : 64]
+	vadd.i32	q2, q2, q3
+	vst1.8		{d0-d1}, [r5, : 128]!
+	vst1.8		{d2-d3}, [r5, : 128]!
+	vst1.8		d4, [r5, : 64]
+	add		r2, r3, #48
+	add		r4, r3, #144
+	vld1.8		{d0-d1}, [r4, : 128]!
+	vld1.8		{d2-d3}, [r4, : 128]!
+	vld1.8		{d4}, [r4, : 64]
+	add		r4, r3, #288
+	vld1.8		{d6-d7}, [r4, : 128]!
+	vtrn.32		q0, q3
+	vld1.8		{d8-d9}, [r4, : 128]!
+	vshl.i32	q5, q0, #4
+	vtrn.32		q1, q4
+	vshl.i32	q6, q3, #4
+	vadd.i32	q5, q5, q0
+	vadd.i32	q6, q6, q3
+	vshl.i32	q7, q1, #4
+	vld1.8		{d5}, [r4, : 64]
+	vshl.i32	q8, q4, #4
+	vtrn.32		d4, d5
+	vadd.i32	q7, q7, q1
+	vadd.i32	q8, q8, q4
+	vld1.8		{d18-d19}, [r2, : 128]!
+	vshl.i32	q10, q2, #4
+	vld1.8		{d22-d23}, [r2, : 128]!
+	vadd.i32	q10, q10, q2
+	vld1.8		{d24}, [r2, : 64]
+	vadd.i32	q5, q5, q0
+	add		r2, r3, #240
+	vld1.8		{d26-d27}, [r2, : 128]!
+	vadd.i32	q6, q6, q3
+	vld1.8		{d28-d29}, [r2, : 128]!
+	vadd.i32	q8, q8, q4
+	vld1.8		{d25}, [r2, : 64]
+	vadd.i32	q10, q10, q2
+	vtrn.32		q9, q13
+	vadd.i32	q7, q7, q1
+	vadd.i32	q5, q5, q0
+	vtrn.32		q11, q14
+	vadd.i32	q6, q6, q3
+	add		r2, sp, #528
+	vadd.i32	q10, q10, q2
+	vtrn.32		d24, d25
+	vst1.8		{d12-d13}, [r2, : 128]!
+	vshl.i32	q6, q13, #1
+	vst1.8		{d20-d21}, [r2, : 128]!
+	vshl.i32	q10, q14, #1
+	vst1.8		{d12-d13}, [r2, : 128]!
+	vshl.i32	q15, q12, #1
+	vadd.i32	q8, q8, q4
+	vext.32		d10, d31, d30, #0
+	vadd.i32	q7, q7, q1
+	vst1.8		{d16-d17}, [r2, : 128]!
+	vmull.s32	q8, d18, d5
+	vmlal.s32	q8, d26, d4
+	vmlal.s32	q8, d19, d9
+	vmlal.s32	q8, d27, d3
+	vmlal.s32	q8, d22, d8
+	vmlal.s32	q8, d28, d2
+	vmlal.s32	q8, d23, d7
+	vmlal.s32	q8, d29, d1
+	vmlal.s32	q8, d24, d6
+	vmlal.s32	q8, d25, d0
+	vst1.8		{d14-d15}, [r2, : 128]!
+	vmull.s32	q2, d18, d4
+	vmlal.s32	q2, d12, d9
+	vmlal.s32	q2, d13, d8
+	vmlal.s32	q2, d19, d3
+	vmlal.s32	q2, d22, d2
+	vmlal.s32	q2, d23, d1
+	vmlal.s32	q2, d24, d0
+	vst1.8		{d20-d21}, [r2, : 128]!
+	vmull.s32	q7, d18, d9
+	vmlal.s32	q7, d26, d3
+	vmlal.s32	q7, d19, d8
+	vmlal.s32	q7, d27, d2
+	vmlal.s32	q7, d22, d7
+	vmlal.s32	q7, d28, d1
+	vmlal.s32	q7, d23, d6
+	vmlal.s32	q7, d29, d0
+	vst1.8		{d10-d11}, [r2, : 128]!
+	vmull.s32	q5, d18, d3
+	vmlal.s32	q5, d19, d2
+	vmlal.s32	q5, d22, d1
+	vmlal.s32	q5, d23, d0
+	vmlal.s32	q5, d12, d8
+	vst1.8		{d16-d17}, [r2, : 128]!
+	vmull.s32	q4, d18, d8
+	vmlal.s32	q4, d26, d2
+	vmlal.s32	q4, d19, d7
+	vmlal.s32	q4, d27, d1
+	vmlal.s32	q4, d22, d6
+	vmlal.s32	q4, d28, d0
+	vmull.s32	q8, d18, d7
+	vmlal.s32	q8, d26, d1
+	vmlal.s32	q8, d19, d6
+	vmlal.s32	q8, d27, d0
+	add		r2, sp, #544
+	vld1.8		{d20-d21}, [r2, : 128]
+	vmlal.s32	q7, d24, d21
+	vmlal.s32	q7, d25, d20
+	vmlal.s32	q4, d23, d21
+	vmlal.s32	q4, d29, d20
+	vmlal.s32	q8, d22, d21
+	vmlal.s32	q8, d28, d20
+	vmlal.s32	q5, d24, d20
+	vst1.8		{d14-d15}, [r2, : 128]
+	vmull.s32	q7, d18, d6
+	vmlal.s32	q7, d26, d0
+	add		r2, sp, #624
+	vld1.8		{d30-d31}, [r2, : 128]
+	vmlal.s32	q2, d30, d21
+	vmlal.s32	q7, d19, d21
+	vmlal.s32	q7, d27, d20
+	add		r2, sp, #592
+	vld1.8		{d26-d27}, [r2, : 128]
+	vmlal.s32	q4, d25, d27
+	vmlal.s32	q8, d29, d27
+	vmlal.s32	q8, d25, d26
+	vmlal.s32	q7, d28, d27
+	vmlal.s32	q7, d29, d26
+	add		r2, sp, #576
+	vld1.8		{d28-d29}, [r2, : 128]
+	vmlal.s32	q4, d24, d29
+	vmlal.s32	q8, d23, d29
+	vmlal.s32	q8, d24, d28
+	vmlal.s32	q7, d22, d29
+	vmlal.s32	q7, d23, d28
+	vst1.8		{d8-d9}, [r2, : 128]
+	add		r2, sp, #528
+	vld1.8		{d8-d9}, [r2, : 128]
+	vmlal.s32	q7, d24, d9
+	vmlal.s32	q7, d25, d31
+	vmull.s32	q1, d18, d2
+	vmlal.s32	q1, d19, d1
+	vmlal.s32	q1, d22, d0
+	vmlal.s32	q1, d24, d27
+	vmlal.s32	q1, d23, d20
+	vmlal.s32	q1, d12, d7
+	vmlal.s32	q1, d13, d6
+	vmull.s32	q6, d18, d1
+	vmlal.s32	q6, d19, d0
+	vmlal.s32	q6, d23, d27
+	vmlal.s32	q6, d22, d20
+	vmlal.s32	q6, d24, d26
+	vmull.s32	q0, d18, d0
+	vmlal.s32	q0, d22, d27
+	vmlal.s32	q0, d23, d26
+	vmlal.s32	q0, d24, d31
+	vmlal.s32	q0, d19, d20
+	add		r2, sp, #608
+	vld1.8		{d18-d19}, [r2, : 128]
+	vmlal.s32	q2, d18, d7
+	vmlal.s32	q5, d18, d6
+	vmlal.s32	q1, d18, d21
+	vmlal.s32	q0, d18, d28
+	vmlal.s32	q6, d18, d29
+	vmlal.s32	q2, d19, d6
+	vmlal.s32	q5, d19, d21
+	vmlal.s32	q1, d19, d29
+	vmlal.s32	q0, d19, d9
+	vmlal.s32	q6, d19, d28
+	add		r2, sp, #560
+	vld1.8		{d18-d19}, [r2, : 128]
+	add		r2, sp, #480
+	vld1.8		{d22-d23}, [r2, : 128]
+	vmlal.s32	q5, d19, d7
+	vmlal.s32	q0, d18, d21
+	vmlal.s32	q0, d19, d29
+	vmlal.s32	q6, d18, d6
+	add		r2, sp, #496
+	vld1.8		{d6-d7}, [r2, : 128]
+	vmlal.s32	q6, d19, d21
+	add		r2, sp, #544
+	vld1.8		{d18-d19}, [r2, : 128]
+	vmlal.s32	q0, d30, d8
+	add		r2, sp, #640
+	vld1.8		{d20-d21}, [r2, : 128]
+	vmlal.s32	q5, d30, d29
+	add		r2, sp, #576
+	vld1.8		{d24-d25}, [r2, : 128]
+	vmlal.s32	q1, d30, d28
+	vadd.i64	q13, q0, q11
+	vadd.i64	q14, q5, q11
+	vmlal.s32	q6, d30, d9
+	vshr.s64	q4, q13, #26
+	vshr.s64	q13, q14, #26
+	vadd.i64	q7, q7, q4
+	vshl.i64	q4, q4, #26
+	vadd.i64	q14, q7, q3
+	vadd.i64	q9, q9, q13
+	vshl.i64	q13, q13, #26
+	vadd.i64	q15, q9, q3
+	vsub.i64	q0, q0, q4
+	vshr.s64	q4, q14, #25
+	vsub.i64	q5, q5, q13
+	vshr.s64	q13, q15, #25
+	vadd.i64	q6, q6, q4
+	vshl.i64	q4, q4, #25
+	vadd.i64	q14, q6, q11
+	vadd.i64	q2, q2, q13
+	vsub.i64	q4, q7, q4
+	vshr.s64	q7, q14, #26
+	vshl.i64	q13, q13, #25
+	vadd.i64	q14, q2, q11
+	vadd.i64	q8, q8, q7
+	vshl.i64	q7, q7, #26
+	vadd.i64	q15, q8, q3
+	vsub.i64	q9, q9, q13
+	vshr.s64	q13, q14, #26
+	vsub.i64	q6, q6, q7
+	vshr.s64	q7, q15, #25
+	vadd.i64	q10, q10, q13
+	vshl.i64	q13, q13, #26
+	vadd.i64	q14, q10, q3
+	vadd.i64	q1, q1, q7
+	add		r2, r3, #240
+	vshl.i64	q7, q7, #25
+	add		r4, r3, #144
+	vadd.i64	q15, q1, q11
+	add		r2, r2, #8
+	vsub.i64	q2, q2, q13
+	add		r4, r4, #8
+	vshr.s64	q13, q14, #25
+	vsub.i64	q7, q8, q7
+	vshr.s64	q8, q15, #26
+	vadd.i64	q14, q13, q13
+	vadd.i64	q12, q12, q8
+	vtrn.32		d12, d14
+	vshl.i64	q8, q8, #26
+	vtrn.32		d13, d15
+	vadd.i64	q3, q12, q3
+	vadd.i64	q0, q0, q14
+	vst1.8		d12, [r2, : 64]!
+	vshl.i64	q7, q13, #4
+	vst1.8		d13, [r4, : 64]!
+	vsub.i64	q1, q1, q8
+	vshr.s64	q3, q3, #25
+	vadd.i64	q0, q0, q7
+	vadd.i64	q5, q5, q3
+	vshl.i64	q3, q3, #25
+	vadd.i64	q6, q5, q11
+	vadd.i64	q0, q0, q13
+	vshl.i64	q7, q13, #25
+	vadd.i64	q8, q0, q11
+	vsub.i64	q3, q12, q3
+	vshr.s64	q6, q6, #26
+	vsub.i64	q7, q10, q7
+	vtrn.32		d2, d6
+	vshr.s64	q8, q8, #26
+	vtrn.32		d3, d7
+	vadd.i64	q3, q9, q6
+	vst1.8		d2, [r2, : 64]
+	vshl.i64	q6, q6, #26
+	vst1.8		d3, [r4, : 64]
+	vadd.i64	q1, q4, q8
+	vtrn.32		d4, d14
+	vshl.i64	q4, q8, #26
+	vtrn.32		d5, d15
+	vsub.i64	q5, q5, q6
+	add		r2, r2, #16
+	vsub.i64	q0, q0, q4
+	vst1.8		d4, [r2, : 64]
+	add		r4, r4, #16
+	vst1.8		d5, [r4, : 64]
+	vtrn.32		d10, d6
+	vtrn.32		d11, d7
+	sub		r2, r2, #8
+	sub		r4, r4, #8
+	vtrn.32		d0, d2
+	vtrn.32		d1, d3
+	vst1.8		d10, [r2, : 64]
+	vst1.8		d11, [r4, : 64]
+	sub		r2, r2, #24
+	sub		r4, r4, #24
+	vst1.8		d0, [r2, : 64]
+	vst1.8		d1, [r4, : 64]
+	ldr		r2, [sp, #456]
+	ldr		r4, [sp, #460]
+	subs		r5, r2, #1
+	bge		.Lmainloop
+	add		r1, r3, #144
+	add		r2, r3, #336
+	vld1.8		{d0-d1}, [r1, : 128]!
+	vld1.8		{d2-d3}, [r1, : 128]!
+	vld1.8		{d4}, [r1, : 64]
+	vst1.8		{d0-d1}, [r2, : 128]!
+	vst1.8		{d2-d3}, [r2, : 128]!
+	vst1.8		d4, [r2, : 64]
+	movw		r1, #0
+.Linvertloop:
+	add		r2, r3, #144
+	movw		r4, #0
+	movw		r5, #2
+	cmp		r1, #1
+	moveq		r5, #1
+	addeq		r2, r3, #336
+	addeq		r4, r3, #48
+	cmp		r1, #2
+	moveq		r5, #1
+	addeq		r2, r3, #48
+	cmp		r1, #3
+	moveq		r5, #5
+	addeq		r4, r3, #336
+	cmp		r1, #4
+	moveq		r5, #10
+	cmp		r1, #5
+	moveq		r5, #20
+	cmp		r1, #6
+	moveq		r5, #10
+	addeq		r2, r3, #336
+	addeq		r4, r3, #336
+	cmp		r1, #7
+	moveq		r5, #50
+	cmp		r1, #8
+	moveq		r5, #100
+	cmp		r1, #9
+	moveq		r5, #50
+	addeq		r2, r3, #336
+	cmp		r1, #10
+	moveq		r5, #5
+	addeq		r2, r3, #48
+	cmp		r1, #11
+	moveq		r5, #0
+	addeq		r2, r3, #96
+	add		r6, r3, #144
+	add		r7, r3, #288
+	vld1.8		{d0-d1}, [r6, : 128]!
+	vld1.8		{d2-d3}, [r6, : 128]!
+	vld1.8		{d4}, [r6, : 64]
+	vst1.8		{d0-d1}, [r7, : 128]!
+	vst1.8		{d2-d3}, [r7, : 128]!
+	vst1.8		d4, [r7, : 64]
+	cmp		r5, #0
+	beq		.Lskipsquaringloop
+.Lsquaringloop:
+	add		r6, r3, #288
+	add		r7, r3, #288
+	add		r8, r3, #288
+	vmov.i32	q0, #19
+	vmov.i32	q1, #0
+	vmov.i32	q2, #1
+	vzip.i32	q1, q2
+	vld1.8		{d4-d5}, [r7, : 128]!
+	vld1.8		{d6-d7}, [r7, : 128]!
+	vld1.8		{d9}, [r7, : 64]
+	vld1.8		{d10-d11}, [r6, : 128]!
+	add		r7, sp, #384
+	vld1.8		{d12-d13}, [r6, : 128]!
+	vmul.i32	q7, q2, q0
+	vld1.8		{d8}, [r6, : 64]
+	vext.32		d17, d11, d10, #1
+	vmul.i32	q9, q3, q0
+	vext.32		d16, d10, d8, #1
+	vshl.u32	q10, q5, q1
+	vext.32		d22, d14, d4, #1
+	vext.32		d24, d18, d6, #1
+	vshl.u32	q13, q6, q1
+	vshl.u32	d28, d8, d2
+	vrev64.i32	d22, d22
+	vmul.i32	d1, d9, d1
+	vrev64.i32	d24, d24
+	vext.32		d29, d8, d13, #1
+	vext.32		d0, d1, d9, #1
+	vrev64.i32	d0, d0
+	vext.32		d2, d9, d1, #1
+	vext.32		d23, d15, d5, #1
+	vmull.s32	q4, d20, d4
+	vrev64.i32	d23, d23
+	vmlal.s32	q4, d21, d1
+	vrev64.i32	d2, d2
+	vmlal.s32	q4, d26, d19
+	vext.32		d3, d5, d15, #1
+	vmlal.s32	q4, d27, d18
+	vrev64.i32	d3, d3
+	vmlal.s32	q4, d28, d15
+	vext.32		d14, d12, d11, #1
+	vmull.s32	q5, d16, d23
+	vext.32		d15, d13, d12, #1
+	vmlal.s32	q5, d17, d4
+	vst1.8		d8, [r7, : 64]!
+	vmlal.s32	q5, d14, d1
+	vext.32		d12, d9, d8, #0
+	vmlal.s32	q5, d15, d19
+	vmov.i64	d13, #0
+	vmlal.s32	q5, d29, d18
+	vext.32		d25, d19, d7, #1
+	vmlal.s32	q6, d20, d5
+	vrev64.i32	d25, d25
+	vmlal.s32	q6, d21, d4
+	vst1.8		d11, [r7, : 64]!
+	vmlal.s32	q6, d26, d1
+	vext.32		d9, d10, d10, #0
+	vmlal.s32	q6, d27, d19
+	vmov.i64	d8, #0
+	vmlal.s32	q6, d28, d18
+	vmlal.s32	q4, d16, d24
+	vmlal.s32	q4, d17, d5
+	vmlal.s32	q4, d14, d4
+	vst1.8		d12, [r7, : 64]!
+	vmlal.s32	q4, d15, d1
+	vext.32		d10, d13, d12, #0
+	vmlal.s32	q4, d29, d19
+	vmov.i64	d11, #0
+	vmlal.s32	q5, d20, d6
+	vmlal.s32	q5, d21, d5
+	vmlal.s32	q5, d26, d4
+	vext.32		d13, d8, d8, #0
+	vmlal.s32	q5, d27, d1
+	vmov.i64	d12, #0
+	vmlal.s32	q5, d28, d19
+	vst1.8		d9, [r7, : 64]!
+	vmlal.s32	q6, d16, d25
+	vmlal.s32	q6, d17, d6
+	vst1.8		d10, [r7, : 64]
+	vmlal.s32	q6, d14, d5
+	vext.32		d8, d11, d10, #0
+	vmlal.s32	q6, d15, d4
+	vmov.i64	d9, #0
+	vmlal.s32	q6, d29, d1
+	vmlal.s32	q4, d20, d7
+	vmlal.s32	q4, d21, d6
+	vmlal.s32	q4, d26, d5
+	vext.32		d11, d12, d12, #0
+	vmlal.s32	q4, d27, d4
+	vmov.i64	d10, #0
+	vmlal.s32	q4, d28, d1
+	vmlal.s32	q5, d16, d0
+	sub		r6, r7, #32
+	vmlal.s32	q5, d17, d7
+	vmlal.s32	q5, d14, d6
+	vext.32		d30, d9, d8, #0
+	vmlal.s32	q5, d15, d5
+	vld1.8		{d31}, [r6, : 64]!
+	vmlal.s32	q5, d29, d4
+	vmlal.s32	q15, d20, d0
+	vext.32		d0, d6, d18, #1
+	vmlal.s32	q15, d21, d25
+	vrev64.i32	d0, d0
+	vmlal.s32	q15, d26, d24
+	vext.32		d1, d7, d19, #1
+	vext.32		d7, d10, d10, #0
+	vmlal.s32	q15, d27, d23
+	vrev64.i32	d1, d1
+	vld1.8		{d6}, [r6, : 64]
+	vmlal.s32	q15, d28, d22
+	vmlal.s32	q3, d16, d4
+	add		r6, r6, #24
+	vmlal.s32	q3, d17, d2
+	vext.32		d4, d31, d30, #0
+	vmov		d17, d11
+	vmlal.s32	q3, d14, d1
+	vext.32		d11, d13, d13, #0
+	vext.32		d13, d30, d30, #0
+	vmlal.s32	q3, d15, d0
+	vext.32		d1, d8, d8, #0
+	vmlal.s32	q3, d29, d3
+	vld1.8		{d5}, [r6, : 64]
+	sub		r6, r6, #16
+	vext.32		d10, d6, d6, #0
+	vmov.i32	q1, #0xffffffff
+	vshl.i64	q4, q1, #25
+	add		r7, sp, #480
+	vld1.8		{d14-d15}, [r7, : 128]
+	vadd.i64	q9, q2, q7
+	vshl.i64	q1, q1, #26
+	vshr.s64	q10, q9, #26
+	vld1.8		{d0}, [r6, : 64]!
+	vadd.i64	q5, q5, q10
+	vand		q9, q9, q1
+	vld1.8		{d16}, [r6, : 64]!
+	add		r6, sp, #496
+	vld1.8		{d20-d21}, [r6, : 128]
+	vadd.i64	q11, q5, q10
+	vsub.i64	q2, q2, q9
+	vshr.s64	q9, q11, #25
+	vext.32		d12, d5, d4, #0
+	vand		q11, q11, q4
+	vadd.i64	q0, q0, q9
+	vmov		d19, d7
+	vadd.i64	q3, q0, q7
+	vsub.i64	q5, q5, q11
+	vshr.s64	q11, q3, #26
+	vext.32		d18, d11, d10, #0
+	vand		q3, q3, q1
+	vadd.i64	q8, q8, q11
+	vadd.i64	q11, q8, q10
+	vsub.i64	q0, q0, q3
+	vshr.s64	q3, q11, #25
+	vand		q11, q11, q4
+	vadd.i64	q3, q6, q3
+	vadd.i64	q6, q3, q7
+	vsub.i64	q8, q8, q11
+	vshr.s64	q11, q6, #26
+	vand		q6, q6, q1
+	vadd.i64	q9, q9, q11
+	vadd.i64	d25, d19, d21
+	vsub.i64	q3, q3, q6
+	vshr.s64	d23, d25, #25
+	vand		q4, q12, q4
+	vadd.i64	d21, d23, d23
+	vshl.i64	d25, d23, #4
+	vadd.i64	d21, d21, d23
+	vadd.i64	d25, d25, d21
+	vadd.i64	d4, d4, d25
+	vzip.i32	q0, q8
+	vadd.i64	d12, d4, d14
+	add		r6, r8, #8
+	vst1.8		d0, [r6, : 64]
+	vsub.i64	d19, d19, d9
+	add		r6, r6, #16
+	vst1.8		d16, [r6, : 64]
+	vshr.s64	d22, d12, #26
+	vand		q0, q6, q1
+	vadd.i64	d10, d10, d22
+	vzip.i32	q3, q9
+	vsub.i64	d4, d4, d0
+	sub		r6, r6, #8
+	vst1.8		d6, [r6, : 64]
+	add		r6, r6, #16
+	vst1.8		d18, [r6, : 64]
+	vzip.i32	q2, q5
+	sub		r6, r6, #32
+	vst1.8		d4, [r6, : 64]
+	subs		r5, r5, #1
+	bhi		.Lsquaringloop
+.Lskipsquaringloop:
+	mov		r2, r2
+	add		r5, r3, #288
+	add		r6, r3, #144
+	vmov.i32	q0, #19
+	vmov.i32	q1, #0
+	vmov.i32	q2, #1
+	vzip.i32	q1, q2
+	vld1.8		{d4-d5}, [r5, : 128]!
+	vld1.8		{d6-d7}, [r5, : 128]!
+	vld1.8		{d9}, [r5, : 64]
+	vld1.8		{d10-d11}, [r2, : 128]!
+	add		r5, sp, #384
+	vld1.8		{d12-d13}, [r2, : 128]!
+	vmul.i32	q7, q2, q0
+	vld1.8		{d8}, [r2, : 64]
+	vext.32		d17, d11, d10, #1
+	vmul.i32	q9, q3, q0
+	vext.32		d16, d10, d8, #1
+	vshl.u32	q10, q5, q1
+	vext.32		d22, d14, d4, #1
+	vext.32		d24, d18, d6, #1
+	vshl.u32	q13, q6, q1
+	vshl.u32	d28, d8, d2
+	vrev64.i32	d22, d22
+	vmul.i32	d1, d9, d1
+	vrev64.i32	d24, d24
+	vext.32		d29, d8, d13, #1
+	vext.32		d0, d1, d9, #1
+	vrev64.i32	d0, d0
+	vext.32		d2, d9, d1, #1
+	vext.32		d23, d15, d5, #1
+	vmull.s32	q4, d20, d4
+	vrev64.i32	d23, d23
+	vmlal.s32	q4, d21, d1
+	vrev64.i32	d2, d2
+	vmlal.s32	q4, d26, d19
+	vext.32		d3, d5, d15, #1
+	vmlal.s32	q4, d27, d18
+	vrev64.i32	d3, d3
+	vmlal.s32	q4, d28, d15
+	vext.32		d14, d12, d11, #1
+	vmull.s32	q5, d16, d23
+	vext.32		d15, d13, d12, #1
+	vmlal.s32	q5, d17, d4
+	vst1.8		d8, [r5, : 64]!
+	vmlal.s32	q5, d14, d1
+	vext.32		d12, d9, d8, #0
+	vmlal.s32	q5, d15, d19
+	vmov.i64	d13, #0
+	vmlal.s32	q5, d29, d18
+	vext.32		d25, d19, d7, #1
+	vmlal.s32	q6, d20, d5
+	vrev64.i32	d25, d25
+	vmlal.s32	q6, d21, d4
+	vst1.8		d11, [r5, : 64]!
+	vmlal.s32	q6, d26, d1
+	vext.32		d9, d10, d10, #0
+	vmlal.s32	q6, d27, d19
+	vmov.i64	d8, #0
+	vmlal.s32	q6, d28, d18
+	vmlal.s32	q4, d16, d24
+	vmlal.s32	q4, d17, d5
+	vmlal.s32	q4, d14, d4
+	vst1.8		d12, [r5, : 64]!
+	vmlal.s32	q4, d15, d1
+	vext.32		d10, d13, d12, #0
+	vmlal.s32	q4, d29, d19
+	vmov.i64	d11, #0
+	vmlal.s32	q5, d20, d6
+	vmlal.s32	q5, d21, d5
+	vmlal.s32	q5, d26, d4
+	vext.32		d13, d8, d8, #0
+	vmlal.s32	q5, d27, d1
+	vmov.i64	d12, #0
+	vmlal.s32	q5, d28, d19
+	vst1.8		d9, [r5, : 64]!
+	vmlal.s32	q6, d16, d25
+	vmlal.s32	q6, d17, d6
+	vst1.8		d10, [r5, : 64]
+	vmlal.s32	q6, d14, d5
+	vext.32		d8, d11, d10, #0
+	vmlal.s32	q6, d15, d4
+	vmov.i64	d9, #0
+	vmlal.s32	q6, d29, d1
+	vmlal.s32	q4, d20, d7
+	vmlal.s32	q4, d21, d6
+	vmlal.s32	q4, d26, d5
+	vext.32		d11, d12, d12, #0
+	vmlal.s32	q4, d27, d4
+	vmov.i64	d10, #0
+	vmlal.s32	q4, d28, d1
+	vmlal.s32	q5, d16, d0
+	sub		r2, r5, #32
+	vmlal.s32	q5, d17, d7
+	vmlal.s32	q5, d14, d6
+	vext.32		d30, d9, d8, #0
+	vmlal.s32	q5, d15, d5
+	vld1.8		{d31}, [r2, : 64]!
+	vmlal.s32	q5, d29, d4
+	vmlal.s32	q15, d20, d0
+	vext.32		d0, d6, d18, #1
+	vmlal.s32	q15, d21, d25
+	vrev64.i32	d0, d0
+	vmlal.s32	q15, d26, d24
+	vext.32		d1, d7, d19, #1
+	vext.32		d7, d10, d10, #0
+	vmlal.s32	q15, d27, d23
+	vrev64.i32	d1, d1
+	vld1.8		{d6}, [r2, : 64]
+	vmlal.s32	q15, d28, d22
+	vmlal.s32	q3, d16, d4
+	add		r2, r2, #24
+	vmlal.s32	q3, d17, d2
+	vext.32		d4, d31, d30, #0
+	vmov		d17, d11
+	vmlal.s32	q3, d14, d1
+	vext.32		d11, d13, d13, #0
+	vext.32		d13, d30, d30, #0
+	vmlal.s32	q3, d15, d0
+	vext.32		d1, d8, d8, #0
+	vmlal.s32	q3, d29, d3
+	vld1.8		{d5}, [r2, : 64]
+	sub		r2, r2, #16
+	vext.32		d10, d6, d6, #0
+	vmov.i32	q1, #0xffffffff
+	vshl.i64	q4, q1, #25
+	add		r5, sp, #480
+	vld1.8		{d14-d15}, [r5, : 128]
+	vadd.i64	q9, q2, q7
+	vshl.i64	q1, q1, #26
+	vshr.s64	q10, q9, #26
+	vld1.8		{d0}, [r2, : 64]!
+	vadd.i64	q5, q5, q10
+	vand		q9, q9, q1
+	vld1.8		{d16}, [r2, : 64]!
+	add		r2, sp, #496
+	vld1.8		{d20-d21}, [r2, : 128]
+	vadd.i64	q11, q5, q10
+	vsub.i64	q2, q2, q9
+	vshr.s64	q9, q11, #25
+	vext.32		d12, d5, d4, #0
+	vand		q11, q11, q4
+	vadd.i64	q0, q0, q9
+	vmov		d19, d7
+	vadd.i64	q3, q0, q7
+	vsub.i64	q5, q5, q11
+	vshr.s64	q11, q3, #26
+	vext.32		d18, d11, d10, #0
+	vand		q3, q3, q1
+	vadd.i64	q8, q8, q11
+	vadd.i64	q11, q8, q10
+	vsub.i64	q0, q0, q3
+	vshr.s64	q3, q11, #25
+	vand		q11, q11, q4
+	vadd.i64	q3, q6, q3
+	vadd.i64	q6, q3, q7
+	vsub.i64	q8, q8, q11
+	vshr.s64	q11, q6, #26
+	vand		q6, q6, q1
+	vadd.i64	q9, q9, q11
+	vadd.i64	d25, d19, d21
+	vsub.i64	q3, q3, q6
+	vshr.s64	d23, d25, #25
+	vand		q4, q12, q4
+	vadd.i64	d21, d23, d23
+	vshl.i64	d25, d23, #4
+	vadd.i64	d21, d21, d23
+	vadd.i64	d25, d25, d21
+	vadd.i64	d4, d4, d25
+	vzip.i32	q0, q8
+	vadd.i64	d12, d4, d14
+	add		r2, r6, #8
+	vst1.8		d0, [r2, : 64]
+	vsub.i64	d19, d19, d9
+	add		r2, r2, #16
+	vst1.8		d16, [r2, : 64]
+	vshr.s64	d22, d12, #26
+	vand		q0, q6, q1
+	vadd.i64	d10, d10, d22
+	vzip.i32	q3, q9
+	vsub.i64	d4, d4, d0
+	sub		r2, r2, #8
+	vst1.8		d6, [r2, : 64]
+	add		r2, r2, #16
+	vst1.8		d18, [r2, : 64]
+	vzip.i32	q2, q5
+	sub		r2, r2, #32
+	vst1.8		d4, [r2, : 64]
+	cmp		r4, #0
+	beq		.Lskippostcopy
+	add		r2, r3, #144
+	mov		r4, r4
+	vld1.8		{d0-d1}, [r2, : 128]!
+	vld1.8		{d2-d3}, [r2, : 128]!
+	vld1.8		{d4}, [r2, : 64]
+	vst1.8		{d0-d1}, [r4, : 128]!
+	vst1.8		{d2-d3}, [r4, : 128]!
+	vst1.8		d4, [r4, : 64]
+.Lskippostcopy:
+	cmp		r1, #1
+	bne		.Lskipfinalcopy
+	add		r2, r3, #288
+	add		r4, r3, #144
+	vld1.8		{d0-d1}, [r2, : 128]!
+	vld1.8		{d2-d3}, [r2, : 128]!
+	vld1.8		{d4}, [r2, : 64]
+	vst1.8		{d0-d1}, [r4, : 128]!
+	vst1.8		{d2-d3}, [r4, : 128]!
+	vst1.8		d4, [r4, : 64]
+.Lskipfinalcopy:
+	add		r1, r1, #1
+	cmp		r1, #12
+	blo		.Linvertloop
+	add		r1, r3, #144
+	ldr		r2, [r1], #4
+	ldr		r3, [r1], #4
+	ldr		r4, [r1], #4
+	ldr		r5, [r1], #4
+	ldr		r6, [r1], #4
+	ldr		r7, [r1], #4
+	ldr		r8, [r1], #4
+	ldr		r9, [r1], #4
+	ldr		r10, [r1], #4
+	ldr		r1, [r1]
+	add		r11, r1, r1, LSL #4
+	add		r11, r11, r1, LSL #1
+	add		r11, r11, #16777216
+	mov		r11, r11, ASR #25
+	add		r11, r11, r2
+	mov		r11, r11, ASR #26
+	add		r11, r11, r3
+	mov		r11, r11, ASR #25
+	add		r11, r11, r4
+	mov		r11, r11, ASR #26
+	add		r11, r11, r5
+	mov		r11, r11, ASR #25
+	add		r11, r11, r6
+	mov		r11, r11, ASR #26
+	add		r11, r11, r7
+	mov		r11, r11, ASR #25
+	add		r11, r11, r8
+	mov		r11, r11, ASR #26
+	add		r11, r11, r9
+	mov		r11, r11, ASR #25
+	add		r11, r11, r10
+	mov		r11, r11, ASR #26
+	add		r11, r11, r1
+	mov		r11, r11, ASR #25
+	add		r2, r2, r11
+	add		r2, r2, r11, LSL #1
+	add		r2, r2, r11, LSL #4
+	mov		r11, r2, ASR #26
+	add		r3, r3, r11
+	sub		r2, r2, r11, LSL #26
+	mov		r11, r3, ASR #25
+	add		r4, r4, r11
+	sub		r3, r3, r11, LSL #25
+	mov		r11, r4, ASR #26
+	add		r5, r5, r11
+	sub		r4, r4, r11, LSL #26
+	mov		r11, r5, ASR #25
+	add		r6, r6, r11
+	sub		r5, r5, r11, LSL #25
+	mov		r11, r6, ASR #26
+	add		r7, r7, r11
+	sub		r6, r6, r11, LSL #26
+	mov		r11, r7, ASR #25
+	add		r8, r8, r11
+	sub		r7, r7, r11, LSL #25
+	mov		r11, r8, ASR #26
+	add		r9, r9, r11
+	sub		r8, r8, r11, LSL #26
+	mov		r11, r9, ASR #25
+	add		r10, r10, r11
+	sub		r9, r9, r11, LSL #25
+	mov		r11, r10, ASR #26
+	add		r1, r1, r11
+	sub		r10, r10, r11, LSL #26
+	mov		r11, r1, ASR #25
+	sub		r1, r1, r11, LSL #25
+	add		r2, r2, r3, LSL #26
+	mov		r3, r3, LSR #6
+	add		r3, r3, r4, LSL #19
+	mov		r4, r4, LSR #13
+	add		r4, r4, r5, LSL #13
+	mov		r5, r5, LSR #19
+	add		r5, r5, r6, LSL #6
+	add		r6, r7, r8, LSL #25
+	mov		r7, r8, LSR #7
+	add		r7, r7, r9, LSL #19
+	mov		r8, r9, LSR #13
+	add		r8, r8, r10, LSL #12
+	mov		r9, r10, LSR #20
+	add		r1, r9, r1, LSL #6
+	str		r2, [r0]
+	str		r3, [r0, #4]
+	str		r4, [r0, #8]
+	str		r5, [r0, #12]
+	str		r6, [r0, #16]
+	str		r7, [r0, #20]
+	str		r8, [r0, #24]
+	str		r1, [r0, #28]
+	movw		r0, #0
+	mov		sp, ip
+	pop		{r4-r11, pc}
+SYM_FUNC_END(curve25519_neon)
+#endif
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/curve25519/curve25519.c b/net/wireguard/crypto/zinc/curve25519/curve25519.c
--- a/net/wireguard/crypto/zinc/curve25519/curve25519.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/curve25519/curve25519.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ *
+ * This is an implementation of the Curve25519 ECDH algorithm, using either
+ * a 32-bit implementation or a 64-bit implementation with 128-bit integers,
+ * depending on what is supported by the target compiler.
+ *
+ * Information: https://cr.yp.to/ecdh.html
+ */
+
+#include <zinc/curve25519.h>
+#include "../selftest/run.h"
+
+#include <asm/unaligned.h>
+#include <linux/version.h>
+#include <linux/string.h>
+#include <linux/random.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <crypto/algapi.h> // For crypto_memneq.
+
+#if defined(CONFIG_ZINC_ARCH_X86_64)
+#include "curve25519-x86_64-glue.c"
+#elif defined(CONFIG_ZINC_ARCH_ARM)
+#include "curve25519-arm-glue.c"
+#else
+static bool *const curve25519_nobs[] __initconst = { };
+static void __init curve25519_fpu_init(void)
+{
+}
+static inline bool curve25519_arch(u8 mypublic[CURVE25519_KEY_SIZE],
+				   const u8 secret[CURVE25519_KEY_SIZE],
+				   const u8 basepoint[CURVE25519_KEY_SIZE])
+{
+	return false;
+}
+static inline bool curve25519_base_arch(u8 pub[CURVE25519_KEY_SIZE],
+					const u8 secret[CURVE25519_KEY_SIZE])
+{
+	return false;
+}
+#endif
+
+#if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__)
+#include "curve25519-hacl64.c"
+#else
+#include "curve25519-fiat32.c"
+#endif
+
+static const u8 null_point[CURVE25519_KEY_SIZE] = { 0 };
+
+bool curve25519(u8 mypublic[CURVE25519_KEY_SIZE],
+		const u8 secret[CURVE25519_KEY_SIZE],
+		const u8 basepoint[CURVE25519_KEY_SIZE])
+{
+	if (!curve25519_arch(mypublic, secret, basepoint))
+		curve25519_generic(mypublic, secret, basepoint);
+	return crypto_memneq(mypublic, null_point, CURVE25519_KEY_SIZE);
+}
+EXPORT_SYMBOL(curve25519);
+
+bool curve25519_generate_public(u8 pub[CURVE25519_KEY_SIZE],
+				const u8 secret[CURVE25519_KEY_SIZE])
+{
+	static const u8 basepoint[CURVE25519_KEY_SIZE] __aligned(32) = { 9 };
+
+	if (unlikely(!crypto_memneq(secret, null_point, CURVE25519_KEY_SIZE)))
+		return false;
+
+	if (curve25519_base_arch(pub, secret))
+		return crypto_memneq(pub, null_point, CURVE25519_KEY_SIZE);
+	return curve25519(pub, secret, basepoint);
+}
+EXPORT_SYMBOL(curve25519_generate_public);
+
+void curve25519_generate_secret(u8 secret[CURVE25519_KEY_SIZE])
+{
+	get_random_bytes_wait(secret, CURVE25519_KEY_SIZE);
+	curve25519_clamp_secret(secret);
+}
+EXPORT_SYMBOL(curve25519_generate_secret);
+
+#include "../selftest/curve25519.c"
+
+static bool nosimd __initdata = false;
+
+#ifndef COMPAT_ZINC_IS_A_MODULE
+int __init curve25519_mod_init(void)
+#else
+static int __init mod_init(void)
+#endif
+{
+	if (!nosimd)
+		curve25519_fpu_init();
+	if (!selftest_run("curve25519", curve25519_selftest, curve25519_nobs,
+			  ARRAY_SIZE(curve25519_nobs)))
+		return -ENOTRECOVERABLE;
+	return 0;
+}
+
+#ifdef COMPAT_ZINC_IS_A_MODULE
+static void __exit mod_exit(void)
+{
+}
+
+module_param(nosimd, bool, 0);
+module_init(mod_init);
+module_exit(mod_exit);
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("Curve25519 scalar multiplication");
+MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");
+#endif
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/curve25519/curve25519-fiat32.c b/net/wireguard/crypto/zinc/curve25519/curve25519-fiat32.c
--- a/net/wireguard/crypto/zinc/curve25519/curve25519-fiat32.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/curve25519/curve25519-fiat32.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,860 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2015-2016 The fiat-crypto Authors.
+ * Copyright (C) 2018-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ *
+ * This is a machine-generated formally verified implementation of Curve25519
+ * ECDH from: <https://github.com/mit-plv/fiat-crypto>. Though originally
+ * machine generated, it has been tweaked to be suitable for use in the kernel.
+ * It is optimized for 32-bit machines and machines that cannot work efficiently
+ * with 128-bit integer types.
+ */
+
+/* fe means field element. Here the field is \Z/(2^255-19). An element t,
+ * entries t[0]...t[9], represents the integer t[0]+2^26 t[1]+2^51 t[2]+2^77
+ * t[3]+2^102 t[4]+...+2^230 t[9].
+ * fe limbs are bounded by 1.125*2^26,1.125*2^25,1.125*2^26,1.125*2^25,etc.
+ * Multiplication and carrying produce fe from fe_loose.
+ */
+typedef struct fe { u32 v[10]; } fe;
+
+/* fe_loose limbs are bounded by 3.375*2^26,3.375*2^25,3.375*2^26,3.375*2^25,etc
+ * Addition and subtraction produce fe_loose from (fe, fe).
+ */
+typedef struct fe_loose { u32 v[10]; } fe_loose;
+
+static __always_inline void fe_frombytes_impl(u32 h[10], const u8 *s)
+{
+	/* Ignores top bit of s. */
+	u32 a0 = get_unaligned_le32(s);
+	u32 a1 = get_unaligned_le32(s+4);
+	u32 a2 = get_unaligned_le32(s+8);
+	u32 a3 = get_unaligned_le32(s+12);
+	u32 a4 = get_unaligned_le32(s+16);
+	u32 a5 = get_unaligned_le32(s+20);
+	u32 a6 = get_unaligned_le32(s+24);
+	u32 a7 = get_unaligned_le32(s+28);
+	h[0] = a0&((1<<26)-1);                    /* 26 used, 32-26 left.   26 */
+	h[1] = (a0>>26) | ((a1&((1<<19)-1))<< 6); /* (32-26) + 19 =  6+19 = 25 */
+	h[2] = (a1>>19) | ((a2&((1<<13)-1))<<13); /* (32-19) + 13 = 13+13 = 26 */
+	h[3] = (a2>>13) | ((a3&((1<< 6)-1))<<19); /* (32-13) +  6 = 19+ 6 = 25 */
+	h[4] = (a3>> 6);                          /* (32- 6)              = 26 */
+	h[5] = a4&((1<<25)-1);                    /*                        25 */
+	h[6] = (a4>>25) | ((a5&((1<<19)-1))<< 7); /* (32-25) + 19 =  7+19 = 26 */
+	h[7] = (a5>>19) | ((a6&((1<<12)-1))<<13); /* (32-19) + 12 = 13+12 = 25 */
+	h[8] = (a6>>12) | ((a7&((1<< 6)-1))<<20); /* (32-12) +  6 = 20+ 6 = 26 */
+	h[9] = (a7>> 6)&((1<<25)-1); /*                                     25 */
+}
+
+static __always_inline void fe_frombytes(fe *h, const u8 *s)
+{
+	fe_frombytes_impl(h->v, s);
+}
+
+static __always_inline u8 /*bool*/
+addcarryx_u25(u8 /*bool*/ c, u32 a, u32 b, u32 *low)
+{
+	/* This function extracts 25 bits of result and 1 bit of carry
+	 * (26 total), so a 32-bit intermediate is sufficient.
+	 */
+	u32 x = a + b + c;
+	*low = x & ((1 << 25) - 1);
+	return (x >> 25) & 1;
+}
+
+static __always_inline u8 /*bool*/
+addcarryx_u26(u8 /*bool*/ c, u32 a, u32 b, u32 *low)
+{
+	/* This function extracts 26 bits of result and 1 bit of carry
+	 * (27 total), so a 32-bit intermediate is sufficient.
+	 */
+	u32 x = a + b + c;
+	*low = x & ((1 << 26) - 1);
+	return (x >> 26) & 1;
+}
+
+static __always_inline u8 /*bool*/
+subborrow_u25(u8 /*bool*/ c, u32 a, u32 b, u32 *low)
+{
+	/* This function extracts 25 bits of result and 1 bit of borrow
+	 * (26 total), so a 32-bit intermediate is sufficient.
+	 */
+	u32 x = a - b - c;
+	*low = x & ((1 << 25) - 1);
+	return x >> 31;
+}
+
+static __always_inline u8 /*bool*/
+subborrow_u26(u8 /*bool*/ c, u32 a, u32 b, u32 *low)
+{
+	/* This function extracts 26 bits of result and 1 bit of borrow
+	 *(27 total), so a 32-bit intermediate is sufficient.
+	 */
+	u32 x = a - b - c;
+	*low = x & ((1 << 26) - 1);
+	return x >> 31;
+}
+
+static __always_inline u32 cmovznz32(u32 t, u32 z, u32 nz)
+{
+	t = -!!t; /* all set if nonzero, 0 if 0 */
+	return (t&nz) | ((~t)&z);
+}
+
+static __always_inline void fe_freeze(u32 out[10], const u32 in1[10])
+{
+	{ const u32 x17 = in1[9];
+	{ const u32 x18 = in1[8];
+	{ const u32 x16 = in1[7];
+	{ const u32 x14 = in1[6];
+	{ const u32 x12 = in1[5];
+	{ const u32 x10 = in1[4];
+	{ const u32 x8 = in1[3];
+	{ const u32 x6 = in1[2];
+	{ const u32 x4 = in1[1];
+	{ const u32 x2 = in1[0];
+	{ u32 x20; u8/*bool*/ x21 = subborrow_u26(0x0, x2, 0x3ffffed, &x20);
+	{ u32 x23; u8/*bool*/ x24 = subborrow_u25(x21, x4, 0x1ffffff, &x23);
+	{ u32 x26; u8/*bool*/ x27 = subborrow_u26(x24, x6, 0x3ffffff, &x26);
+	{ u32 x29; u8/*bool*/ x30 = subborrow_u25(x27, x8, 0x1ffffff, &x29);
+	{ u32 x32; u8/*bool*/ x33 = subborrow_u26(x30, x10, 0x3ffffff, &x32);
+	{ u32 x35; u8/*bool*/ x36 = subborrow_u25(x33, x12, 0x1ffffff, &x35);
+	{ u32 x38; u8/*bool*/ x39 = subborrow_u26(x36, x14, 0x3ffffff, &x38);
+	{ u32 x41; u8/*bool*/ x42 = subborrow_u25(x39, x16, 0x1ffffff, &x41);
+	{ u32 x44; u8/*bool*/ x45 = subborrow_u26(x42, x18, 0x3ffffff, &x44);
+	{ u32 x47; u8/*bool*/ x48 = subborrow_u25(x45, x17, 0x1ffffff, &x47);
+	{ u32 x49 = cmovznz32(x48, 0x0, 0xffffffff);
+	{ u32 x50 = (x49 & 0x3ffffed);
+	{ u32 x52; u8/*bool*/ x53 = addcarryx_u26(0x0, x20, x50, &x52);
+	{ u32 x54 = (x49 & 0x1ffffff);
+	{ u32 x56; u8/*bool*/ x57 = addcarryx_u25(x53, x23, x54, &x56);
+	{ u32 x58 = (x49 & 0x3ffffff);
+	{ u32 x60; u8/*bool*/ x61 = addcarryx_u26(x57, x26, x58, &x60);
+	{ u32 x62 = (x49 & 0x1ffffff);
+	{ u32 x64; u8/*bool*/ x65 = addcarryx_u25(x61, x29, x62, &x64);
+	{ u32 x66 = (x49 & 0x3ffffff);
+	{ u32 x68; u8/*bool*/ x69 = addcarryx_u26(x65, x32, x66, &x68);
+	{ u32 x70 = (x49 & 0x1ffffff);
+	{ u32 x72; u8/*bool*/ x73 = addcarryx_u25(x69, x35, x70, &x72);
+	{ u32 x74 = (x49 & 0x3ffffff);
+	{ u32 x76; u8/*bool*/ x77 = addcarryx_u26(x73, x38, x74, &x76);
+	{ u32 x78 = (x49 & 0x1ffffff);
+	{ u32 x80; u8/*bool*/ x81 = addcarryx_u25(x77, x41, x78, &x80);
+	{ u32 x82 = (x49 & 0x3ffffff);
+	{ u32 x84; u8/*bool*/ x85 = addcarryx_u26(x81, x44, x82, &x84);
+	{ u32 x86 = (x49 & 0x1ffffff);
+	{ u32 x88; addcarryx_u25(x85, x47, x86, &x88);
+	out[0] = x52;
+	out[1] = x56;
+	out[2] = x60;
+	out[3] = x64;
+	out[4] = x68;
+	out[5] = x72;
+	out[6] = x76;
+	out[7] = x80;
+	out[8] = x84;
+	out[9] = x88;
+	}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}
+}
+
+static __always_inline void fe_tobytes(u8 s[32], const fe *f)
+{
+	u32 h[10];
+	fe_freeze(h, f->v);
+	s[0] = h[0] >> 0;
+	s[1] = h[0] >> 8;
+	s[2] = h[0] >> 16;
+	s[3] = (h[0] >> 24) | (h[1] << 2);
+	s[4] = h[1] >> 6;
+	s[5] = h[1] >> 14;
+	s[6] = (h[1] >> 22) | (h[2] << 3);
+	s[7] = h[2] >> 5;
+	s[8] = h[2] >> 13;
+	s[9] = (h[2] >> 21) | (h[3] << 5);
+	s[10] = h[3] >> 3;
+	s[11] = h[3] >> 11;
+	s[12] = (h[3] >> 19) | (h[4] << 6);
+	s[13] = h[4] >> 2;
+	s[14] = h[4] >> 10;
+	s[15] = h[4] >> 18;
+	s[16] = h[5] >> 0;
+	s[17] = h[5] >> 8;
+	s[18] = h[5] >> 16;
+	s[19] = (h[5] >> 24) | (h[6] << 1);
+	s[20] = h[6] >> 7;
+	s[21] = h[6] >> 15;
+	s[22] = (h[6] >> 23) | (h[7] << 3);
+	s[23] = h[7] >> 5;
+	s[24] = h[7] >> 13;
+	s[25] = (h[7] >> 21) | (h[8] << 4);
+	s[26] = h[8] >> 4;
+	s[27] = h[8] >> 12;
+	s[28] = (h[8] >> 20) | (h[9] << 6);
+	s[29] = h[9] >> 2;
+	s[30] = h[9] >> 10;
+	s[31] = h[9] >> 18;
+}
+
+/* h = f */
+static __always_inline void fe_copy(fe *h, const fe *f)
+{
+	memmove(h, f, sizeof(u32) * 10);
+}
+
+static __always_inline void fe_copy_lt(fe_loose *h, const fe *f)
+{
+	memmove(h, f, sizeof(u32) * 10);
+}
+
+/* h = 0 */
+static __always_inline void fe_0(fe *h)
+{
+	memset(h, 0, sizeof(u32) * 10);
+}
+
+/* h = 1 */
+static __always_inline void fe_1(fe *h)
+{
+	memset(h, 0, sizeof(u32) * 10);
+	h->v[0] = 1;
+}
+
+static void fe_add_impl(u32 out[10], const u32 in1[10], const u32 in2[10])
+{
+	{ const u32 x20 = in1[9];
+	{ const u32 x21 = in1[8];
+	{ const u32 x19 = in1[7];
+	{ const u32 x17 = in1[6];
+	{ const u32 x15 = in1[5];
+	{ const u32 x13 = in1[4];
+	{ const u32 x11 = in1[3];
+	{ const u32 x9 = in1[2];
+	{ const u32 x7 = in1[1];
+	{ const u32 x5 = in1[0];
+	{ const u32 x38 = in2[9];
+	{ const u32 x39 = in2[8];
+	{ const u32 x37 = in2[7];
+	{ const u32 x35 = in2[6];
+	{ const u32 x33 = in2[5];
+	{ const u32 x31 = in2[4];
+	{ const u32 x29 = in2[3];
+	{ const u32 x27 = in2[2];
+	{ const u32 x25 = in2[1];
+	{ const u32 x23 = in2[0];
+	out[0] = (x5 + x23);
+	out[1] = (x7 + x25);
+	out[2] = (x9 + x27);
+	out[3] = (x11 + x29);
+	out[4] = (x13 + x31);
+	out[5] = (x15 + x33);
+	out[6] = (x17 + x35);
+	out[7] = (x19 + x37);
+	out[8] = (x21 + x39);
+	out[9] = (x20 + x38);
+	}}}}}}}}}}}}}}}}}}}}
+}
+
+/* h = f + g
+ * Can overlap h with f or g.
+ */
+static __always_inline void fe_add(fe_loose *h, const fe *f, const fe *g)
+{
+	fe_add_impl(h->v, f->v, g->v);
+}
+
+static void fe_sub_impl(u32 out[10], const u32 in1[10], const u32 in2[10])
+{
+	{ const u32 x20 = in1[9];
+	{ const u32 x21 = in1[8];
+	{ const u32 x19 = in1[7];
+	{ const u32 x17 = in1[6];
+	{ const u32 x15 = in1[5];
+	{ const u32 x13 = in1[4];
+	{ const u32 x11 = in1[3];
+	{ const u32 x9 = in1[2];
+	{ const u32 x7 = in1[1];
+	{ const u32 x5 = in1[0];
+	{ const u32 x38 = in2[9];
+	{ const u32 x39 = in2[8];
+	{ const u32 x37 = in2[7];
+	{ const u32 x35 = in2[6];
+	{ const u32 x33 = in2[5];
+	{ const u32 x31 = in2[4];
+	{ const u32 x29 = in2[3];
+	{ const u32 x27 = in2[2];
+	{ const u32 x25 = in2[1];
+	{ const u32 x23 = in2[0];
+	out[0] = ((0x7ffffda + x5) - x23);
+	out[1] = ((0x3fffffe + x7) - x25);
+	out[2] = ((0x7fffffe + x9) - x27);
+	out[3] = ((0x3fffffe + x11) - x29);
+	out[4] = ((0x7fffffe + x13) - x31);
+	out[5] = ((0x3fffffe + x15) - x33);
+	out[6] = ((0x7fffffe + x17) - x35);
+	out[7] = ((0x3fffffe + x19) - x37);
+	out[8] = ((0x7fffffe + x21) - x39);
+	out[9] = ((0x3fffffe + x20) - x38);
+	}}}}}}}}}}}}}}}}}}}}
+}
+
+/* h = f - g
+ * Can overlap h with f or g.
+ */
+static __always_inline void fe_sub(fe_loose *h, const fe *f, const fe *g)
+{
+	fe_sub_impl(h->v, f->v, g->v);
+}
+
+static void fe_mul_impl(u32 out[10], const u32 in1[10], const u32 in2[10])
+{
+	{ const u32 x20 = in1[9];
+	{ const u32 x21 = in1[8];
+	{ const u32 x19 = in1[7];
+	{ const u32 x17 = in1[6];
+	{ const u32 x15 = in1[5];
+	{ const u32 x13 = in1[4];
+	{ const u32 x11 = in1[3];
+	{ const u32 x9 = in1[2];
+	{ const u32 x7 = in1[1];
+	{ const u32 x5 = in1[0];
+	{ const u32 x38 = in2[9];
+	{ const u32 x39 = in2[8];
+	{ const u32 x37 = in2[7];
+	{ const u32 x35 = in2[6];
+	{ const u32 x33 = in2[5];
+	{ const u32 x31 = in2[4];
+	{ const u32 x29 = in2[3];
+	{ const u32 x27 = in2[2];
+	{ const u32 x25 = in2[1];
+	{ const u32 x23 = in2[0];
+	{ u64 x40 = ((u64)x23 * x5);
+	{ u64 x41 = (((u64)x23 * x7) + ((u64)x25 * x5));
+	{ u64 x42 = ((((u64)(0x2 * x25) * x7) + ((u64)x23 * x9)) + ((u64)x27 * x5));
+	{ u64 x43 = (((((u64)x25 * x9) + ((u64)x27 * x7)) + ((u64)x23 * x11)) + ((u64)x29 * x5));
+	{ u64 x44 = (((((u64)x27 * x9) + (0x2 * (((u64)x25 * x11) + ((u64)x29 * x7)))) + ((u64)x23 * x13)) + ((u64)x31 * x5));
+	{ u64 x45 = (((((((u64)x27 * x11) + ((u64)x29 * x9)) + ((u64)x25 * x13)) + ((u64)x31 * x7)) + ((u64)x23 * x15)) + ((u64)x33 * x5));
+	{ u64 x46 = (((((0x2 * ((((u64)x29 * x11) + ((u64)x25 * x15)) + ((u64)x33 * x7))) + ((u64)x27 * x13)) + ((u64)x31 * x9)) + ((u64)x23 * x17)) + ((u64)x35 * x5));
+	{ u64 x47 = (((((((((u64)x29 * x13) + ((u64)x31 * x11)) + ((u64)x27 * x15)) + ((u64)x33 * x9)) + ((u64)x25 * x17)) + ((u64)x35 * x7)) + ((u64)x23 * x19)) + ((u64)x37 * x5));
+	{ u64 x48 = (((((((u64)x31 * x13) + (0x2 * (((((u64)x29 * x15) + ((u64)x33 * x11)) + ((u64)x25 * x19)) + ((u64)x37 * x7)))) + ((u64)x27 * x17)) + ((u64)x35 * x9)) + ((u64)x23 * x21)) + ((u64)x39 * x5));
+	{ u64 x49 = (((((((((((u64)x31 * x15) + ((u64)x33 * x13)) + ((u64)x29 * x17)) + ((u64)x35 * x11)) + ((u64)x27 * x19)) + ((u64)x37 * x9)) + ((u64)x25 * x21)) + ((u64)x39 * x7)) + ((u64)x23 * x20)) + ((u64)x38 * x5));
+	{ u64 x50 = (((((0x2 * ((((((u64)x33 * x15) + ((u64)x29 * x19)) + ((u64)x37 * x11)) + ((u64)x25 * x20)) + ((u64)x38 * x7))) + ((u64)x31 * x17)) + ((u64)x35 * x13)) + ((u64)x27 * x21)) + ((u64)x39 * x9));
+	{ u64 x51 = (((((((((u64)x33 * x17) + ((u64)x35 * x15)) + ((u64)x31 * x19)) + ((u64)x37 * x13)) + ((u64)x29 * x21)) + ((u64)x39 * x11)) + ((u64)x27 * x20)) + ((u64)x38 * x9));
+	{ u64 x52 = (((((u64)x35 * x17) + (0x2 * (((((u64)x33 * x19) + ((u64)x37 * x15)) + ((u64)x29 * x20)) + ((u64)x38 * x11)))) + ((u64)x31 * x21)) + ((u64)x39 * x13));
+	{ u64 x53 = (((((((u64)x35 * x19) + ((u64)x37 * x17)) + ((u64)x33 * x21)) + ((u64)x39 * x15)) + ((u64)x31 * x20)) + ((u64)x38 * x13));
+	{ u64 x54 = (((0x2 * ((((u64)x37 * x19) + ((u64)x33 * x20)) + ((u64)x38 * x15))) + ((u64)x35 * x21)) + ((u64)x39 * x17));
+	{ u64 x55 = (((((u64)x37 * x21) + ((u64)x39 * x19)) + ((u64)x35 * x20)) + ((u64)x38 * x17));
+	{ u64 x56 = (((u64)x39 * x21) + (0x2 * (((u64)x37 * x20) + ((u64)x38 * x19))));
+	{ u64 x57 = (((u64)x39 * x20) + ((u64)x38 * x21));
+	{ u64 x58 = ((u64)(0x2 * x38) * x20);
+	{ u64 x59 = (x48 + (x58 << 0x4));
+	{ u64 x60 = (x59 + (x58 << 0x1));
+	{ u64 x61 = (x60 + x58);
+	{ u64 x62 = (x47 + (x57 << 0x4));
+	{ u64 x63 = (x62 + (x57 << 0x1));
+	{ u64 x64 = (x63 + x57);
+	{ u64 x65 = (x46 + (x56 << 0x4));
+	{ u64 x66 = (x65 + (x56 << 0x1));
+	{ u64 x67 = (x66 + x56);
+	{ u64 x68 = (x45 + (x55 << 0x4));
+	{ u64 x69 = (x68 + (x55 << 0x1));
+	{ u64 x70 = (x69 + x55);
+	{ u64 x71 = (x44 + (x54 << 0x4));
+	{ u64 x72 = (x71 + (x54 << 0x1));
+	{ u64 x73 = (x72 + x54);
+	{ u64 x74 = (x43 + (x53 << 0x4));
+	{ u64 x75 = (x74 + (x53 << 0x1));
+	{ u64 x76 = (x75 + x53);
+	{ u64 x77 = (x42 + (x52 << 0x4));
+	{ u64 x78 = (x77 + (x52 << 0x1));
+	{ u64 x79 = (x78 + x52);
+	{ u64 x80 = (x41 + (x51 << 0x4));
+	{ u64 x81 = (x80 + (x51 << 0x1));
+	{ u64 x82 = (x81 + x51);
+	{ u64 x83 = (x40 + (x50 << 0x4));
+	{ u64 x84 = (x83 + (x50 << 0x1));
+	{ u64 x85 = (x84 + x50);
+	{ u64 x86 = (x85 >> 0x1a);
+	{ u32 x87 = ((u32)x85 & 0x3ffffff);
+	{ u64 x88 = (x86 + x82);
+	{ u64 x89 = (x88 >> 0x19);
+	{ u32 x90 = ((u32)x88 & 0x1ffffff);
+	{ u64 x91 = (x89 + x79);
+	{ u64 x92 = (x91 >> 0x1a);
+	{ u32 x93 = ((u32)x91 & 0x3ffffff);
+	{ u64 x94 = (x92 + x76);
+	{ u64 x95 = (x94 >> 0x19);
+	{ u32 x96 = ((u32)x94 & 0x1ffffff);
+	{ u64 x97 = (x95 + x73);
+	{ u64 x98 = (x97 >> 0x1a);
+	{ u32 x99 = ((u32)x97 & 0x3ffffff);
+	{ u64 x100 = (x98 + x70);
+	{ u64 x101 = (x100 >> 0x19);
+	{ u32 x102 = ((u32)x100 & 0x1ffffff);
+	{ u64 x103 = (x101 + x67);
+	{ u64 x104 = (x103 >> 0x1a);
+	{ u32 x105 = ((u32)x103 & 0x3ffffff);
+	{ u64 x106 = (x104 + x64);
+	{ u64 x107 = (x106 >> 0x19);
+	{ u32 x108 = ((u32)x106 & 0x1ffffff);
+	{ u64 x109 = (x107 + x61);
+	{ u64 x110 = (x109 >> 0x1a);
+	{ u32 x111 = ((u32)x109 & 0x3ffffff);
+	{ u64 x112 = (x110 + x49);
+	{ u64 x113 = (x112 >> 0x19);
+	{ u32 x114 = ((u32)x112 & 0x1ffffff);
+	{ u64 x115 = (x87 + (0x13 * x113));
+	{ u32 x116 = (u32) (x115 >> 0x1a);
+	{ u32 x117 = ((u32)x115 & 0x3ffffff);
+	{ u32 x118 = (x116 + x90);
+	{ u32 x119 = (x118 >> 0x19);
+	{ u32 x120 = (x118 & 0x1ffffff);
+	out[0] = x117;
+	out[1] = x120;
+	out[2] = (x119 + x93);
+	out[3] = x96;
+	out[4] = x99;
+	out[5] = x102;
+	out[6] = x105;
+	out[7] = x108;
+	out[8] = x111;
+	out[9] = x114;
+	}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}
+}
+
+static __always_inline void fe_mul_ttt(fe *h, const fe *f, const fe *g)
+{
+	fe_mul_impl(h->v, f->v, g->v);
+}
+
+static __always_inline void fe_mul_tlt(fe *h, const fe_loose *f, const fe *g)
+{
+	fe_mul_impl(h->v, f->v, g->v);
+}
+
+static __always_inline void
+fe_mul_tll(fe *h, const fe_loose *f, const fe_loose *g)
+{
+	fe_mul_impl(h->v, f->v, g->v);
+}
+
+static void fe_sqr_impl(u32 out[10], const u32 in1[10])
+{
+	{ const u32 x17 = in1[9];
+	{ const u32 x18 = in1[8];
+	{ const u32 x16 = in1[7];
+	{ const u32 x14 = in1[6];
+	{ const u32 x12 = in1[5];
+	{ const u32 x10 = in1[4];
+	{ const u32 x8 = in1[3];
+	{ const u32 x6 = in1[2];
+	{ const u32 x4 = in1[1];
+	{ const u32 x2 = in1[0];
+	{ u64 x19 = ((u64)x2 * x2);
+	{ u64 x20 = ((u64)(0x2 * x2) * x4);
+	{ u64 x21 = (0x2 * (((u64)x4 * x4) + ((u64)x2 * x6)));
+	{ u64 x22 = (0x2 * (((u64)x4 * x6) + ((u64)x2 * x8)));
+	{ u64 x23 = ((((u64)x6 * x6) + ((u64)(0x4 * x4) * x8)) + ((u64)(0x2 * x2) * x10));
+	{ u64 x24 = (0x2 * ((((u64)x6 * x8) + ((u64)x4 * x10)) + ((u64)x2 * x12)));
+	{ u64 x25 = (0x2 * (((((u64)x8 * x8) + ((u64)x6 * x10)) + ((u64)x2 * x14)) + ((u64)(0x2 * x4) * x12)));
+	{ u64 x26 = (0x2 * (((((u64)x8 * x10) + ((u64)x6 * x12)) + ((u64)x4 * x14)) + ((u64)x2 * x16)));
+	{ u64 x27 = (((u64)x10 * x10) + (0x2 * ((((u64)x6 * x14) + ((u64)x2 * x18)) + (0x2 * (((u64)x4 * x16) + ((u64)x8 * x12))))));
+	{ u64 x28 = (0x2 * ((((((u64)x10 * x12) + ((u64)x8 * x14)) + ((u64)x6 * x16)) + ((u64)x4 * x18)) + ((u64)x2 * x17)));
+	{ u64 x29 = (0x2 * (((((u64)x12 * x12) + ((u64)x10 * x14)) + ((u64)x6 * x18)) + (0x2 * (((u64)x8 * x16) + ((u64)x4 * x17)))));
+	{ u64 x30 = (0x2 * (((((u64)x12 * x14) + ((u64)x10 * x16)) + ((u64)x8 * x18)) + ((u64)x6 * x17)));
+	{ u64 x31 = (((u64)x14 * x14) + (0x2 * (((u64)x10 * x18) + (0x2 * (((u64)x12 * x16) + ((u64)x8 * x17))))));
+	{ u64 x32 = (0x2 * ((((u64)x14 * x16) + ((u64)x12 * x18)) + ((u64)x10 * x17)));
+	{ u64 x33 = (0x2 * ((((u64)x16 * x16) + ((u64)x14 * x18)) + ((u64)(0x2 * x12) * x17)));
+	{ u64 x34 = (0x2 * (((u64)x16 * x18) + ((u64)x14 * x17)));
+	{ u64 x35 = (((u64)x18 * x18) + ((u64)(0x4 * x16) * x17));
+	{ u64 x36 = ((u64)(0x2 * x18) * x17);
+	{ u64 x37 = ((u64)(0x2 * x17) * x17);
+	{ u64 x38 = (x27 + (x37 << 0x4));
+	{ u64 x39 = (x38 + (x37 << 0x1));
+	{ u64 x40 = (x39 + x37);
+	{ u64 x41 = (x26 + (x36 << 0x4));
+	{ u64 x42 = (x41 + (x36 << 0x1));
+	{ u64 x43 = (x42 + x36);
+	{ u64 x44 = (x25 + (x35 << 0x4));
+	{ u64 x45 = (x44 + (x35 << 0x1));
+	{ u64 x46 = (x45 + x35);
+	{ u64 x47 = (x24 + (x34 << 0x4));
+	{ u64 x48 = (x47 + (x34 << 0x1));
+	{ u64 x49 = (x48 + x34);
+	{ u64 x50 = (x23 + (x33 << 0x4));
+	{ u64 x51 = (x50 + (x33 << 0x1));
+	{ u64 x52 = (x51 + x33);
+	{ u64 x53 = (x22 + (x32 << 0x4));
+	{ u64 x54 = (x53 + (x32 << 0x1));
+	{ u64 x55 = (x54 + x32);
+	{ u64 x56 = (x21 + (x31 << 0x4));
+	{ u64 x57 = (x56 + (x31 << 0x1));
+	{ u64 x58 = (x57 + x31);
+	{ u64 x59 = (x20 + (x30 << 0x4));
+	{ u64 x60 = (x59 + (x30 << 0x1));
+	{ u64 x61 = (x60 + x30);
+	{ u64 x62 = (x19 + (x29 << 0x4));
+	{ u64 x63 = (x62 + (x29 << 0x1));
+	{ u64 x64 = (x63 + x29);
+	{ u64 x65 = (x64 >> 0x1a);
+	{ u32 x66 = ((u32)x64 & 0x3ffffff);
+	{ u64 x67 = (x65 + x61);
+	{ u64 x68 = (x67 >> 0x19);
+	{ u32 x69 = ((u32)x67 & 0x1ffffff);
+	{ u64 x70 = (x68 + x58);
+	{ u64 x71 = (x70 >> 0x1a);
+	{ u32 x72 = ((u32)x70 & 0x3ffffff);
+	{ u64 x73 = (x71 + x55);
+	{ u64 x74 = (x73 >> 0x19);
+	{ u32 x75 = ((u32)x73 & 0x1ffffff);
+	{ u64 x76 = (x74 + x52);
+	{ u64 x77 = (x76 >> 0x1a);
+	{ u32 x78 = ((u32)x76 & 0x3ffffff);
+	{ u64 x79 = (x77 + x49);
+	{ u64 x80 = (x79 >> 0x19);
+	{ u32 x81 = ((u32)x79 & 0x1ffffff);
+	{ u64 x82 = (x80 + x46);
+	{ u64 x83 = (x82 >> 0x1a);
+	{ u32 x84 = ((u32)x82 & 0x3ffffff);
+	{ u64 x85 = (x83 + x43);
+	{ u64 x86 = (x85 >> 0x19);
+	{ u32 x87 = ((u32)x85 & 0x1ffffff);
+	{ u64 x88 = (x86 + x40);
+	{ u64 x89 = (x88 >> 0x1a);
+	{ u32 x90 = ((u32)x88 & 0x3ffffff);
+	{ u64 x91 = (x89 + x28);
+	{ u64 x92 = (x91 >> 0x19);
+	{ u32 x93 = ((u32)x91 & 0x1ffffff);
+	{ u64 x94 = (x66 + (0x13 * x92));
+	{ u32 x95 = (u32) (x94 >> 0x1a);
+	{ u32 x96 = ((u32)x94 & 0x3ffffff);
+	{ u32 x97 = (x95 + x69);
+	{ u32 x98 = (x97 >> 0x19);
+	{ u32 x99 = (x97 & 0x1ffffff);
+	out[0] = x96;
+	out[1] = x99;
+	out[2] = (x98 + x72);
+	out[3] = x75;
+	out[4] = x78;
+	out[5] = x81;
+	out[6] = x84;
+	out[7] = x87;
+	out[8] = x90;
+	out[9] = x93;
+	}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}
+}
+
+static __always_inline void fe_sq_tl(fe *h, const fe_loose *f)
+{
+	fe_sqr_impl(h->v, f->v);
+}
+
+static __always_inline void fe_sq_tt(fe *h, const fe *f)
+{
+	fe_sqr_impl(h->v, f->v);
+}
+
+static __always_inline void fe_loose_invert(fe *out, const fe_loose *z)
+{
+	fe t0;
+	fe t1;
+	fe t2;
+	fe t3;
+	int i;
+
+	fe_sq_tl(&t0, z);
+	fe_sq_tt(&t1, &t0);
+	for (i = 1; i < 2; ++i)
+		fe_sq_tt(&t1, &t1);
+	fe_mul_tlt(&t1, z, &t1);
+	fe_mul_ttt(&t0, &t0, &t1);
+	fe_sq_tt(&t2, &t0);
+	fe_mul_ttt(&t1, &t1, &t2);
+	fe_sq_tt(&t2, &t1);
+	for (i = 1; i < 5; ++i)
+		fe_sq_tt(&t2, &t2);
+	fe_mul_ttt(&t1, &t2, &t1);
+	fe_sq_tt(&t2, &t1);
+	for (i = 1; i < 10; ++i)
+		fe_sq_tt(&t2, &t2);
+	fe_mul_ttt(&t2, &t2, &t1);
+	fe_sq_tt(&t3, &t2);
+	for (i = 1; i < 20; ++i)
+		fe_sq_tt(&t3, &t3);
+	fe_mul_ttt(&t2, &t3, &t2);
+	fe_sq_tt(&t2, &t2);
+	for (i = 1; i < 10; ++i)
+		fe_sq_tt(&t2, &t2);
+	fe_mul_ttt(&t1, &t2, &t1);
+	fe_sq_tt(&t2, &t1);
+	for (i = 1; i < 50; ++i)
+		fe_sq_tt(&t2, &t2);
+	fe_mul_ttt(&t2, &t2, &t1);
+	fe_sq_tt(&t3, &t2);
+	for (i = 1; i < 100; ++i)
+		fe_sq_tt(&t3, &t3);
+	fe_mul_ttt(&t2, &t3, &t2);
+	fe_sq_tt(&t2, &t2);
+	for (i = 1; i < 50; ++i)
+		fe_sq_tt(&t2, &t2);
+	fe_mul_ttt(&t1, &t2, &t1);
+	fe_sq_tt(&t1, &t1);
+	for (i = 1; i < 5; ++i)
+		fe_sq_tt(&t1, &t1);
+	fe_mul_ttt(out, &t1, &t0);
+}
+
+static __always_inline void fe_invert(fe *out, const fe *z)
+{
+	fe_loose l;
+	fe_copy_lt(&l, z);
+	fe_loose_invert(out, &l);
+}
+
+/* Replace (f,g) with (g,f) if b == 1;
+ * replace (f,g) with (f,g) if b == 0.
+ *
+ * Preconditions: b in {0,1}
+ */
+static __always_inline void fe_cswap(fe *f, fe *g, unsigned int b)
+{
+	unsigned i;
+	b = 0 - b;
+	for (i = 0; i < 10; i++) {
+		u32 x = f->v[i] ^ g->v[i];
+		x &= b;
+		f->v[i] ^= x;
+		g->v[i] ^= x;
+	}
+}
+
+/* NOTE: based on fiat-crypto fe_mul, edited for in2=121666, 0, 0.*/
+static __always_inline void fe_mul_121666_impl(u32 out[10], const u32 in1[10])
+{
+	{ const u32 x20 = in1[9];
+	{ const u32 x21 = in1[8];
+	{ const u32 x19 = in1[7];
+	{ const u32 x17 = in1[6];
+	{ const u32 x15 = in1[5];
+	{ const u32 x13 = in1[4];
+	{ const u32 x11 = in1[3];
+	{ const u32 x9 = in1[2];
+	{ const u32 x7 = in1[1];
+	{ const u32 x5 = in1[0];
+	{ const u32 x38 = 0;
+	{ const u32 x39 = 0;
+	{ const u32 x37 = 0;
+	{ const u32 x35 = 0;
+	{ const u32 x33 = 0;
+	{ const u32 x31 = 0;
+	{ const u32 x29 = 0;
+	{ const u32 x27 = 0;
+	{ const u32 x25 = 0;
+	{ const u32 x23 = 121666;
+	{ u64 x40 = ((u64)x23 * x5);
+	{ u64 x41 = (((u64)x23 * x7) + ((u64)x25 * x5));
+	{ u64 x42 = ((((u64)(0x2 * x25) * x7) + ((u64)x23 * x9)) + ((u64)x27 * x5));
+	{ u64 x43 = (((((u64)x25 * x9) + ((u64)x27 * x7)) + ((u64)x23 * x11)) + ((u64)x29 * x5));
+	{ u64 x44 = (((((u64)x27 * x9) + (0x2 * (((u64)x25 * x11) + ((u64)x29 * x7)))) + ((u64)x23 * x13)) + ((u64)x31 * x5));
+	{ u64 x45 = (((((((u64)x27 * x11) + ((u64)x29 * x9)) + ((u64)x25 * x13)) + ((u64)x31 * x7)) + ((u64)x23 * x15)) + ((u64)x33 * x5));
+	{ u64 x46 = (((((0x2 * ((((u64)x29 * x11) + ((u64)x25 * x15)) + ((u64)x33 * x7))) + ((u64)x27 * x13)) + ((u64)x31 * x9)) + ((u64)x23 * x17)) + ((u64)x35 * x5));
+	{ u64 x47 = (((((((((u64)x29 * x13) + ((u64)x31 * x11)) + ((u64)x27 * x15)) + ((u64)x33 * x9)) + ((u64)x25 * x17)) + ((u64)x35 * x7)) + ((u64)x23 * x19)) + ((u64)x37 * x5));
+	{ u64 x48 = (((((((u64)x31 * x13) + (0x2 * (((((u64)x29 * x15) + ((u64)x33 * x11)) + ((u64)x25 * x19)) + ((u64)x37 * x7)))) + ((u64)x27 * x17)) + ((u64)x35 * x9)) + ((u64)x23 * x21)) + ((u64)x39 * x5));
+	{ u64 x49 = (((((((((((u64)x31 * x15) + ((u64)x33 * x13)) + ((u64)x29 * x17)) + ((u64)x35 * x11)) + ((u64)x27 * x19)) + ((u64)x37 * x9)) + ((u64)x25 * x21)) + ((u64)x39 * x7)) + ((u64)x23 * x20)) + ((u64)x38 * x5));
+	{ u64 x50 = (((((0x2 * ((((((u64)x33 * x15) + ((u64)x29 * x19)) + ((u64)x37 * x11)) + ((u64)x25 * x20)) + ((u64)x38 * x7))) + ((u64)x31 * x17)) + ((u64)x35 * x13)) + ((u64)x27 * x21)) + ((u64)x39 * x9));
+	{ u64 x51 = (((((((((u64)x33 * x17) + ((u64)x35 * x15)) + ((u64)x31 * x19)) + ((u64)x37 * x13)) + ((u64)x29 * x21)) + ((u64)x39 * x11)) + ((u64)x27 * x20)) + ((u64)x38 * x9));
+	{ u64 x52 = (((((u64)x35 * x17) + (0x2 * (((((u64)x33 * x19) + ((u64)x37 * x15)) + ((u64)x29 * x20)) + ((u64)x38 * x11)))) + ((u64)x31 * x21)) + ((u64)x39 * x13));
+	{ u64 x53 = (((((((u64)x35 * x19) + ((u64)x37 * x17)) + ((u64)x33 * x21)) + ((u64)x39 * x15)) + ((u64)x31 * x20)) + ((u64)x38 * x13));
+	{ u64 x54 = (((0x2 * ((((u64)x37 * x19) + ((u64)x33 * x20)) + ((u64)x38 * x15))) + ((u64)x35 * x21)) + ((u64)x39 * x17));
+	{ u64 x55 = (((((u64)x37 * x21) + ((u64)x39 * x19)) + ((u64)x35 * x20)) + ((u64)x38 * x17));
+	{ u64 x56 = (((u64)x39 * x21) + (0x2 * (((u64)x37 * x20) + ((u64)x38 * x19))));
+	{ u64 x57 = (((u64)x39 * x20) + ((u64)x38 * x21));
+	{ u64 x58 = ((u64)(0x2 * x38) * x20);
+	{ u64 x59 = (x48 + (x58 << 0x4));
+	{ u64 x60 = (x59 + (x58 << 0x1));
+	{ u64 x61 = (x60 + x58);
+	{ u64 x62 = (x47 + (x57 << 0x4));
+	{ u64 x63 = (x62 + (x57 << 0x1));
+	{ u64 x64 = (x63 + x57);
+	{ u64 x65 = (x46 + (x56 << 0x4));
+	{ u64 x66 = (x65 + (x56 << 0x1));
+	{ u64 x67 = (x66 + x56);
+	{ u64 x68 = (x45 + (x55 << 0x4));
+	{ u64 x69 = (x68 + (x55 << 0x1));
+	{ u64 x70 = (x69 + x55);
+	{ u64 x71 = (x44 + (x54 << 0x4));
+	{ u64 x72 = (x71 + (x54 << 0x1));
+	{ u64 x73 = (x72 + x54);
+	{ u64 x74 = (x43 + (x53 << 0x4));
+	{ u64 x75 = (x74 + (x53 << 0x1));
+	{ u64 x76 = (x75 + x53);
+	{ u64 x77 = (x42 + (x52 << 0x4));
+	{ u64 x78 = (x77 + (x52 << 0x1));
+	{ u64 x79 = (x78 + x52);
+	{ u64 x80 = (x41 + (x51 << 0x4));
+	{ u64 x81 = (x80 + (x51 << 0x1));
+	{ u64 x82 = (x81 + x51);
+	{ u64 x83 = (x40 + (x50 << 0x4));
+	{ u64 x84 = (x83 + (x50 << 0x1));
+	{ u64 x85 = (x84 + x50);
+	{ u64 x86 = (x85 >> 0x1a);
+	{ u32 x87 = ((u32)x85 & 0x3ffffff);
+	{ u64 x88 = (x86 + x82);
+	{ u64 x89 = (x88 >> 0x19);
+	{ u32 x90 = ((u32)x88 & 0x1ffffff);
+	{ u64 x91 = (x89 + x79);
+	{ u64 x92 = (x91 >> 0x1a);
+	{ u32 x93 = ((u32)x91 & 0x3ffffff);
+	{ u64 x94 = (x92 + x76);
+	{ u64 x95 = (x94 >> 0x19);
+	{ u32 x96 = ((u32)x94 & 0x1ffffff);
+	{ u64 x97 = (x95 + x73);
+	{ u64 x98 = (x97 >> 0x1a);
+	{ u32 x99 = ((u32)x97 & 0x3ffffff);
+	{ u64 x100 = (x98 + x70);
+	{ u64 x101 = (x100 >> 0x19);
+	{ u32 x102 = ((u32)x100 & 0x1ffffff);
+	{ u64 x103 = (x101 + x67);
+	{ u64 x104 = (x103 >> 0x1a);
+	{ u32 x105 = ((u32)x103 & 0x3ffffff);
+	{ u64 x106 = (x104 + x64);
+	{ u64 x107 = (x106 >> 0x19);
+	{ u32 x108 = ((u32)x106 & 0x1ffffff);
+	{ u64 x109 = (x107 + x61);
+	{ u64 x110 = (x109 >> 0x1a);
+	{ u32 x111 = ((u32)x109 & 0x3ffffff);
+	{ u64 x112 = (x110 + x49);
+	{ u64 x113 = (x112 >> 0x19);
+	{ u32 x114 = ((u32)x112 & 0x1ffffff);
+	{ u64 x115 = (x87 + (0x13 * x113));
+	{ u32 x116 = (u32) (x115 >> 0x1a);
+	{ u32 x117 = ((u32)x115 & 0x3ffffff);
+	{ u32 x118 = (x116 + x90);
+	{ u32 x119 = (x118 >> 0x19);
+	{ u32 x120 = (x118 & 0x1ffffff);
+	out[0] = x117;
+	out[1] = x120;
+	out[2] = (x119 + x93);
+	out[3] = x96;
+	out[4] = x99;
+	out[5] = x102;
+	out[6] = x105;
+	out[7] = x108;
+	out[8] = x111;
+	out[9] = x114;
+	}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}
+}
+
+static __always_inline void fe_mul121666(fe *h, const fe_loose *f)
+{
+	fe_mul_121666_impl(h->v, f->v);
+}
+
+static void curve25519_generic(u8 out[CURVE25519_KEY_SIZE],
+			       const u8 scalar[CURVE25519_KEY_SIZE],
+			       const u8 point[CURVE25519_KEY_SIZE])
+{
+	fe x1, x2, z2, x3, z3;
+	fe_loose x2l, z2l, x3l;
+	unsigned swap = 0;
+	int pos;
+	u8 e[32];
+
+	memcpy(e, scalar, 32);
+	curve25519_clamp_secret(e);
+
+	/* The following implementation was transcribed to Coq and proven to
+	 * correspond to unary scalar multiplication in affine coordinates given
+	 * that x1 != 0 is the x coordinate of some point on the curve. It was
+	 * also checked in Coq that doing a ladderstep with x1 = x3 = 0 gives
+	 * z2' = z3' = 0, and z2 = z3 = 0 gives z2' = z3' = 0. The statement was
+	 * quantified over the underlying field, so it applies to Curve25519
+	 * itself and the quadratic twist of Curve25519. It was not proven in
+	 * Coq that prime-field arithmetic correctly simulates extension-field
+	 * arithmetic on prime-field values. The decoding of the byte array
+	 * representation of e was not considered.
+	 *
+	 * Specification of Montgomery curves in affine coordinates:
+	 * <https://github.com/mit-plv/fiat-crypto/blob/2456d821825521f7e03e65882cc3521795b0320f/src/Spec/MontgomeryCurve.v#L27>
+	 *
+	 * Proof that these form a group that is isomorphic to a Weierstrass
+	 * curve:
+	 * <https://github.com/mit-plv/fiat-crypto/blob/2456d821825521f7e03e65882cc3521795b0320f/src/Curves/Montgomery/AffineProofs.v#L35>
+	 *
+	 * Coq transcription and correctness proof of the loop
+	 * (where scalarbits=255):
+	 * <https://github.com/mit-plv/fiat-crypto/blob/2456d821825521f7e03e65882cc3521795b0320f/src/Curves/Montgomery/XZ.v#L118>
+	 * <https://github.com/mit-plv/fiat-crypto/blob/2456d821825521f7e03e65882cc3521795b0320f/src/Curves/Montgomery/XZProofs.v#L278>
+	 * preconditions: 0 <= e < 2^255 (not necessarily e < order),
+	 * fe_invert(0) = 0
+	 */
+	fe_frombytes(&x1, point);
+	fe_1(&x2);
+	fe_0(&z2);
+	fe_copy(&x3, &x1);
+	fe_1(&z3);
+
+	for (pos = 254; pos >= 0; --pos) {
+		fe tmp0, tmp1;
+		fe_loose tmp0l, tmp1l;
+		/* loop invariant as of right before the test, for the case
+		 * where x1 != 0:
+		 *   pos >= -1; if z2 = 0 then x2 is nonzero; if z3 = 0 then x3
+		 *   is nonzero
+		 *   let r := e >> (pos+1) in the following equalities of
+		 *   projective points:
+		 *   to_xz (r*P)     === if swap then (x3, z3) else (x2, z2)
+		 *   to_xz ((r+1)*P) === if swap then (x2, z2) else (x3, z3)
+		 *   x1 is the nonzero x coordinate of the nonzero
+		 *   point (r*P-(r+1)*P)
+		 */
+		unsigned b = 1 & (e[pos / 8] >> (pos & 7));
+		swap ^= b;
+		fe_cswap(&x2, &x3, swap);
+		fe_cswap(&z2, &z3, swap);
+		swap = b;
+		/* Coq transcription of ladderstep formula (called from
+		 * transcribed loop):
+		 * <https://github.com/mit-plv/fiat-crypto/blob/2456d821825521f7e03e65882cc3521795b0320f/src/Curves/Montgomery/XZ.v#L89>
+		 * <https://github.com/mit-plv/fiat-crypto/blob/2456d821825521f7e03e65882cc3521795b0320f/src/Curves/Montgomery/XZProofs.v#L131>
+		 * x1 != 0 <https://github.com/mit-plv/fiat-crypto/blob/2456d821825521f7e03e65882cc3521795b0320f/src/Curves/Montgomery/XZProofs.v#L217>
+		 * x1  = 0 <https://github.com/mit-plv/fiat-crypto/blob/2456d821825521f7e03e65882cc3521795b0320f/src/Curves/Montgomery/XZProofs.v#L147>
+		 */
+		fe_sub(&tmp0l, &x3, &z3);
+		fe_sub(&tmp1l, &x2, &z2);
+		fe_add(&x2l, &x2, &z2);
+		fe_add(&z2l, &x3, &z3);
+		fe_mul_tll(&z3, &tmp0l, &x2l);
+		fe_mul_tll(&z2, &z2l, &tmp1l);
+		fe_sq_tl(&tmp0, &tmp1l);
+		fe_sq_tl(&tmp1, &x2l);
+		fe_add(&x3l, &z3, &z2);
+		fe_sub(&z2l, &z3, &z2);
+		fe_mul_ttt(&x2, &tmp1, &tmp0);
+		fe_sub(&tmp1l, &tmp1, &tmp0);
+		fe_sq_tl(&z2, &z2l);
+		fe_mul121666(&z3, &tmp1l);
+		fe_sq_tl(&x3, &x3l);
+		fe_add(&tmp0l, &tmp0, &z3);
+		fe_mul_ttt(&z3, &x1, &z2);
+		fe_mul_tll(&z2, &tmp1l, &tmp0l);
+	}
+	/* here pos=-1, so r=e, so to_xz (e*P) === if swap then (x3, z3)
+	 * else (x2, z2)
+	 */
+	fe_cswap(&x2, &x3, swap);
+	fe_cswap(&z2, &z3, swap);
+
+	fe_invert(&z2, &z2);
+	fe_mul_ttt(&x2, &x2, &z2);
+	fe_tobytes(out, &x2);
+
+	memzero_explicit(&x1, sizeof(x1));
+	memzero_explicit(&x2, sizeof(x2));
+	memzero_explicit(&z2, sizeof(z2));
+	memzero_explicit(&x3, sizeof(x3));
+	memzero_explicit(&z3, sizeof(z3));
+	memzero_explicit(&x2l, sizeof(x2l));
+	memzero_explicit(&z2l, sizeof(z2l));
+	memzero_explicit(&x3l, sizeof(x3l));
+	memzero_explicit(&e, sizeof(e));
+}
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/curve25519/curve25519-hacl64.c b/net/wireguard/crypto/zinc/curve25519/curve25519-hacl64.c
--- a/net/wireguard/crypto/zinc/curve25519/curve25519-hacl64.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/curve25519/curve25519-hacl64.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,784 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2016-2017 INRIA and Microsoft Corporation.
+ * Copyright (C) 2018-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ *
+ * This is a machine-generated formally verified implementation of Curve25519
+ * ECDH from: <https://github.com/mitls/hacl-star>. Though originally machine
+ * generated, it has been tweaked to be suitable for use in the kernel. It is
+ * optimized for 64-bit machines that can efficiently work with 128-bit
+ * integer types.
+ */
+
+typedef __uint128_t u128;
+
+static __always_inline u64 u64_eq_mask(u64 a, u64 b)
+{
+	u64 x = a ^ b;
+	u64 minus_x = ~x + (u64)1U;
+	u64 x_or_minus_x = x | minus_x;
+	u64 xnx = x_or_minus_x >> (u32)63U;
+	u64 c = xnx - (u64)1U;
+	return c;
+}
+
+static __always_inline u64 u64_gte_mask(u64 a, u64 b)
+{
+	u64 x = a;
+	u64 y = b;
+	u64 x_xor_y = x ^ y;
+	u64 x_sub_y = x - y;
+	u64 x_sub_y_xor_y = x_sub_y ^ y;
+	u64 q = x_xor_y | x_sub_y_xor_y;
+	u64 x_xor_q = x ^ q;
+	u64 x_xor_q_ = x_xor_q >> (u32)63U;
+	u64 c = x_xor_q_ - (u64)1U;
+	return c;
+}
+
+static __always_inline void modulo_carry_top(u64 *b)
+{
+	u64 b4 = b[4];
+	u64 b0 = b[0];
+	u64 b4_ = b4 & 0x7ffffffffffffLLU;
+	u64 b0_ = b0 + 19 * (b4 >> 51);
+	b[4] = b4_;
+	b[0] = b0_;
+}
+
+static __always_inline void fproduct_copy_from_wide_(u64 *output, u128 *input)
+{
+	{
+		u128 xi = input[0];
+		output[0] = ((u64)(xi));
+	}
+	{
+		u128 xi = input[1];
+		output[1] = ((u64)(xi));
+	}
+	{
+		u128 xi = input[2];
+		output[2] = ((u64)(xi));
+	}
+	{
+		u128 xi = input[3];
+		output[3] = ((u64)(xi));
+	}
+	{
+		u128 xi = input[4];
+		output[4] = ((u64)(xi));
+	}
+}
+
+static __always_inline void
+fproduct_sum_scalar_multiplication_(u128 *output, u64 *input, u64 s)
+{
+	output[0] += (u128)input[0] * s;
+	output[1] += (u128)input[1] * s;
+	output[2] += (u128)input[2] * s;
+	output[3] += (u128)input[3] * s;
+	output[4] += (u128)input[4] * s;
+}
+
+static __always_inline void fproduct_carry_wide_(u128 *tmp)
+{
+	{
+		u32 ctr = 0;
+		u128 tctr = tmp[ctr];
+		u128 tctrp1 = tmp[ctr + 1];
+		u64 r0 = ((u64)(tctr)) & 0x7ffffffffffffLLU;
+		u128 c = ((tctr) >> (51));
+		tmp[ctr] = ((u128)(r0));
+		tmp[ctr + 1] = ((tctrp1) + (c));
+	}
+	{
+		u32 ctr = 1;
+		u128 tctr = tmp[ctr];
+		u128 tctrp1 = tmp[ctr + 1];
+		u64 r0 = ((u64)(tctr)) & 0x7ffffffffffffLLU;
+		u128 c = ((tctr) >> (51));
+		tmp[ctr] = ((u128)(r0));
+		tmp[ctr + 1] = ((tctrp1) + (c));
+	}
+
+	{
+		u32 ctr = 2;
+		u128 tctr = tmp[ctr];
+		u128 tctrp1 = tmp[ctr + 1];
+		u64 r0 = ((u64)(tctr)) & 0x7ffffffffffffLLU;
+		u128 c = ((tctr) >> (51));
+		tmp[ctr] = ((u128)(r0));
+		tmp[ctr + 1] = ((tctrp1) + (c));
+	}
+	{
+		u32 ctr = 3;
+		u128 tctr = tmp[ctr];
+		u128 tctrp1 = tmp[ctr + 1];
+		u64 r0 = ((u64)(tctr)) & 0x7ffffffffffffLLU;
+		u128 c = ((tctr) >> (51));
+		tmp[ctr] = ((u128)(r0));
+		tmp[ctr + 1] = ((tctrp1) + (c));
+	}
+}
+
+static __always_inline void fmul_shift_reduce(u64 *output)
+{
+	u64 tmp = output[4];
+	u64 b0;
+	{
+		u32 ctr = 5 - 0 - 1;
+		u64 z = output[ctr - 1];
+		output[ctr] = z;
+	}
+	{
+		u32 ctr = 5 - 1 - 1;
+		u64 z = output[ctr - 1];
+		output[ctr] = z;
+	}
+	{
+		u32 ctr = 5 - 2 - 1;
+		u64 z = output[ctr - 1];
+		output[ctr] = z;
+	}
+	{
+		u32 ctr = 5 - 3 - 1;
+		u64 z = output[ctr - 1];
+		output[ctr] = z;
+	}
+	output[0] = tmp;
+	b0 = output[0];
+	output[0] = 19 * b0;
+}
+
+static __always_inline void fmul_mul_shift_reduce_(u128 *output, u64 *input,
+						   u64 *input21)
+{
+	u32 i;
+	u64 input2i;
+	{
+		u64 input2i = input21[0];
+		fproduct_sum_scalar_multiplication_(output, input, input2i);
+		fmul_shift_reduce(input);
+	}
+	{
+		u64 input2i = input21[1];
+		fproduct_sum_scalar_multiplication_(output, input, input2i);
+		fmul_shift_reduce(input);
+	}
+	{
+		u64 input2i = input21[2];
+		fproduct_sum_scalar_multiplication_(output, input, input2i);
+		fmul_shift_reduce(input);
+	}
+	{
+		u64 input2i = input21[3];
+		fproduct_sum_scalar_multiplication_(output, input, input2i);
+		fmul_shift_reduce(input);
+	}
+	i = 4;
+	input2i = input21[i];
+	fproduct_sum_scalar_multiplication_(output, input, input2i);
+}
+
+static __always_inline void fmul_fmul(u64 *output, u64 *input, u64 *input21)
+{
+	u64 tmp[5] = { input[0], input[1], input[2], input[3], input[4] };
+	{
+		u128 b4;
+		u128 b0;
+		u128 b4_;
+		u128 b0_;
+		u64 i0;
+		u64 i1;
+		u64 i0_;
+		u64 i1_;
+		u128 t[5] = { 0 };
+		fmul_mul_shift_reduce_(t, tmp, input21);
+		fproduct_carry_wide_(t);
+		b4 = t[4];
+		b0 = t[0];
+		b4_ = ((b4) & (((u128)(0x7ffffffffffffLLU))));
+		b0_ = ((b0) + (((u128)(19) * (((u64)(((b4) >> (51))))))));
+		t[4] = b4_;
+		t[0] = b0_;
+		fproduct_copy_from_wide_(output, t);
+		i0 = output[0];
+		i1 = output[1];
+		i0_ = i0 & 0x7ffffffffffffLLU;
+		i1_ = i1 + (i0 >> 51);
+		output[0] = i0_;
+		output[1] = i1_;
+	}
+}
+
+static __always_inline void fsquare_fsquare__(u128 *tmp, u64 *output)
+{
+	u64 r0 = output[0];
+	u64 r1 = output[1];
+	u64 r2 = output[2];
+	u64 r3 = output[3];
+	u64 r4 = output[4];
+	u64 d0 = r0 * 2;
+	u64 d1 = r1 * 2;
+	u64 d2 = r2 * 2 * 19;
+	u64 d419 = r4 * 19;
+	u64 d4 = d419 * 2;
+	u128 s0 = ((((((u128)(r0) * (r0))) + (((u128)(d4) * (r1))))) +
+		   (((u128)(d2) * (r3))));
+	u128 s1 = ((((((u128)(d0) * (r1))) + (((u128)(d4) * (r2))))) +
+		   (((u128)(r3 * 19) * (r3))));
+	u128 s2 = ((((((u128)(d0) * (r2))) + (((u128)(r1) * (r1))))) +
+		   (((u128)(d4) * (r3))));
+	u128 s3 = ((((((u128)(d0) * (r3))) + (((u128)(d1) * (r2))))) +
+		   (((u128)(r4) * (d419))));
+	u128 s4 = ((((((u128)(d0) * (r4))) + (((u128)(d1) * (r3))))) +
+		   (((u128)(r2) * (r2))));
+	tmp[0] = s0;
+	tmp[1] = s1;
+	tmp[2] = s2;
+	tmp[3] = s3;
+	tmp[4] = s4;
+}
+
+static __always_inline void fsquare_fsquare_(u128 *tmp, u64 *output)
+{
+	u128 b4;
+	u128 b0;
+	u128 b4_;
+	u128 b0_;
+	u64 i0;
+	u64 i1;
+	u64 i0_;
+	u64 i1_;
+	fsquare_fsquare__(tmp, output);
+	fproduct_carry_wide_(tmp);
+	b4 = tmp[4];
+	b0 = tmp[0];
+	b4_ = ((b4) & (((u128)(0x7ffffffffffffLLU))));
+	b0_ = ((b0) + (((u128)(19) * (((u64)(((b4) >> (51))))))));
+	tmp[4] = b4_;
+	tmp[0] = b0_;
+	fproduct_copy_from_wide_(output, tmp);
+	i0 = output[0];
+	i1 = output[1];
+	i0_ = i0 & 0x7ffffffffffffLLU;
+	i1_ = i1 + (i0 >> 51);
+	output[0] = i0_;
+	output[1] = i1_;
+}
+
+static __always_inline void fsquare_fsquare_times_(u64 *output, u128 *tmp,
+						   u32 count1)
+{
+	u32 i;
+	fsquare_fsquare_(tmp, output);
+	for (i = 1; i < count1; ++i)
+		fsquare_fsquare_(tmp, output);
+}
+
+static __always_inline void fsquare_fsquare_times(u64 *output, u64 *input,
+						  u32 count1)
+{
+	u128 t[5];
+	memcpy(output, input, 5 * sizeof(*input));
+	fsquare_fsquare_times_(output, t, count1);
+}
+
+static __always_inline void fsquare_fsquare_times_inplace(u64 *output,
+							  u32 count1)
+{
+	u128 t[5];
+	fsquare_fsquare_times_(output, t, count1);
+}
+
+static __always_inline void crecip_crecip(u64 *out, u64 *z)
+{
+	u64 buf[20] = { 0 };
+	u64 *a0 = buf;
+	u64 *t00 = buf + 5;
+	u64 *b0 = buf + 10;
+	u64 *t01;
+	u64 *b1;
+	u64 *c0;
+	u64 *a;
+	u64 *t0;
+	u64 *b;
+	u64 *c;
+	fsquare_fsquare_times(a0, z, 1);
+	fsquare_fsquare_times(t00, a0, 2);
+	fmul_fmul(b0, t00, z);
+	fmul_fmul(a0, b0, a0);
+	fsquare_fsquare_times(t00, a0, 1);
+	fmul_fmul(b0, t00, b0);
+	fsquare_fsquare_times(t00, b0, 5);
+	t01 = buf + 5;
+	b1 = buf + 10;
+	c0 = buf + 15;
+	fmul_fmul(b1, t01, b1);
+	fsquare_fsquare_times(t01, b1, 10);
+	fmul_fmul(c0, t01, b1);
+	fsquare_fsquare_times(t01, c0, 20);
+	fmul_fmul(t01, t01, c0);
+	fsquare_fsquare_times_inplace(t01, 10);
+	fmul_fmul(b1, t01, b1);
+	fsquare_fsquare_times(t01, b1, 50);
+	a = buf;
+	t0 = buf + 5;
+	b = buf + 10;
+	c = buf + 15;
+	fmul_fmul(c, t0, b);
+	fsquare_fsquare_times(t0, c, 100);
+	fmul_fmul(t0, t0, c);
+	fsquare_fsquare_times_inplace(t0, 50);
+	fmul_fmul(t0, t0, b);
+	fsquare_fsquare_times_inplace(t0, 5);
+	fmul_fmul(out, t0, a);
+}
+
+static __always_inline void fsum(u64 *a, u64 *b)
+{
+	a[0] += b[0];
+	a[1] += b[1];
+	a[2] += b[2];
+	a[3] += b[3];
+	a[4] += b[4];
+}
+
+static __always_inline void fdifference(u64 *a, u64 *b)
+{
+	u64 tmp[5] = { 0 };
+	u64 b0;
+	u64 b1;
+	u64 b2;
+	u64 b3;
+	u64 b4;
+	memcpy(tmp, b, 5 * sizeof(*b));
+	b0 = tmp[0];
+	b1 = tmp[1];
+	b2 = tmp[2];
+	b3 = tmp[3];
+	b4 = tmp[4];
+	tmp[0] = b0 + 0x3fffffffffff68LLU;
+	tmp[1] = b1 + 0x3ffffffffffff8LLU;
+	tmp[2] = b2 + 0x3ffffffffffff8LLU;
+	tmp[3] = b3 + 0x3ffffffffffff8LLU;
+	tmp[4] = b4 + 0x3ffffffffffff8LLU;
+	{
+		u64 xi = a[0];
+		u64 yi = tmp[0];
+		a[0] = yi - xi;
+	}
+	{
+		u64 xi = a[1];
+		u64 yi = tmp[1];
+		a[1] = yi - xi;
+	}
+	{
+		u64 xi = a[2];
+		u64 yi = tmp[2];
+		a[2] = yi - xi;
+	}
+	{
+		u64 xi = a[3];
+		u64 yi = tmp[3];
+		a[3] = yi - xi;
+	}
+	{
+		u64 xi = a[4];
+		u64 yi = tmp[4];
+		a[4] = yi - xi;
+	}
+}
+
+static __always_inline void fscalar(u64 *output, u64 *b, u64 s)
+{
+	u128 tmp[5];
+	u128 b4;
+	u128 b0;
+	u128 b4_;
+	u128 b0_;
+	{
+		u64 xi = b[0];
+		tmp[0] = ((u128)(xi) * (s));
+	}
+	{
+		u64 xi = b[1];
+		tmp[1] = ((u128)(xi) * (s));
+	}
+	{
+		u64 xi = b[2];
+		tmp[2] = ((u128)(xi) * (s));
+	}
+	{
+		u64 xi = b[3];
+		tmp[3] = ((u128)(xi) * (s));
+	}
+	{
+		u64 xi = b[4];
+		tmp[4] = ((u128)(xi) * (s));
+	}
+	fproduct_carry_wide_(tmp);
+	b4 = tmp[4];
+	b0 = tmp[0];
+	b4_ = ((b4) & (((u128)(0x7ffffffffffffLLU))));
+	b0_ = ((b0) + (((u128)(19) * (((u64)(((b4) >> (51))))))));
+	tmp[4] = b4_;
+	tmp[0] = b0_;
+	fproduct_copy_from_wide_(output, tmp);
+}
+
+static __always_inline void fmul(u64 *output, u64 *a, u64 *b)
+{
+	fmul_fmul(output, a, b);
+}
+
+static __always_inline void crecip(u64 *output, u64 *input)
+{
+	crecip_crecip(output, input);
+}
+
+static __always_inline void point_swap_conditional_step(u64 *a, u64 *b,
+							u64 swap1, u32 ctr)
+{
+	u32 i = ctr - 1;
+	u64 ai = a[i];
+	u64 bi = b[i];
+	u64 x = swap1 & (ai ^ bi);
+	u64 ai1 = ai ^ x;
+	u64 bi1 = bi ^ x;
+	a[i] = ai1;
+	b[i] = bi1;
+}
+
+static __always_inline void point_swap_conditional5(u64 *a, u64 *b, u64 swap1)
+{
+	point_swap_conditional_step(a, b, swap1, 5);
+	point_swap_conditional_step(a, b, swap1, 4);
+	point_swap_conditional_step(a, b, swap1, 3);
+	point_swap_conditional_step(a, b, swap1, 2);
+	point_swap_conditional_step(a, b, swap1, 1);
+}
+
+static __always_inline void point_swap_conditional(u64 *a, u64 *b, u64 iswap)
+{
+	u64 swap1 = 0 - iswap;
+	point_swap_conditional5(a, b, swap1);
+	point_swap_conditional5(a + 5, b + 5, swap1);
+}
+
+static __always_inline void point_copy(u64 *output, u64 *input)
+{
+	memcpy(output, input, 5 * sizeof(*input));
+	memcpy(output + 5, input + 5, 5 * sizeof(*input));
+}
+
+static __always_inline void addanddouble_fmonty(u64 *pp, u64 *ppq, u64 *p,
+						u64 *pq, u64 *qmqp)
+{
+	u64 *qx = qmqp;
+	u64 *x2 = pp;
+	u64 *z2 = pp + 5;
+	u64 *x3 = ppq;
+	u64 *z3 = ppq + 5;
+	u64 *x = p;
+	u64 *z = p + 5;
+	u64 *xprime = pq;
+	u64 *zprime = pq + 5;
+	u64 buf[40] = { 0 };
+	u64 *origx = buf;
+	u64 *origxprime0 = buf + 5;
+	u64 *xxprime0;
+	u64 *zzprime0;
+	u64 *origxprime;
+	xxprime0 = buf + 25;
+	zzprime0 = buf + 30;
+	memcpy(origx, x, 5 * sizeof(*x));
+	fsum(x, z);
+	fdifference(z, origx);
+	memcpy(origxprime0, xprime, 5 * sizeof(*xprime));
+	fsum(xprime, zprime);
+	fdifference(zprime, origxprime0);
+	fmul(xxprime0, xprime, z);
+	fmul(zzprime0, x, zprime);
+	origxprime = buf + 5;
+	{
+		u64 *xx0;
+		u64 *zz0;
+		u64 *xxprime;
+		u64 *zzprime;
+		u64 *zzzprime;
+		xx0 = buf + 15;
+		zz0 = buf + 20;
+		xxprime = buf + 25;
+		zzprime = buf + 30;
+		zzzprime = buf + 35;
+		memcpy(origxprime, xxprime, 5 * sizeof(*xxprime));
+		fsum(xxprime, zzprime);
+		fdifference(zzprime, origxprime);
+		fsquare_fsquare_times(x3, xxprime, 1);
+		fsquare_fsquare_times(zzzprime, zzprime, 1);
+		fmul(z3, zzzprime, qx);
+		fsquare_fsquare_times(xx0, x, 1);
+		fsquare_fsquare_times(zz0, z, 1);
+		{
+			u64 *zzz;
+			u64 *xx;
+			u64 *zz;
+			u64 scalar;
+			zzz = buf + 10;
+			xx = buf + 15;
+			zz = buf + 20;
+			fmul(x2, xx, zz);
+			fdifference(zz, xx);
+			scalar = 121665;
+			fscalar(zzz, zz, scalar);
+			fsum(zzz, xx);
+			fmul(z2, zzz, zz);
+		}
+	}
+}
+
+static __always_inline void
+ladder_smallloop_cmult_small_loop_step(u64 *nq, u64 *nqpq, u64 *nq2, u64 *nqpq2,
+				       u64 *q, u8 byt)
+{
+	u64 bit0 = (u64)(byt >> 7);
+	u64 bit;
+	point_swap_conditional(nq, nqpq, bit0);
+	addanddouble_fmonty(nq2, nqpq2, nq, nqpq, q);
+	bit = (u64)(byt >> 7);
+	point_swap_conditional(nq2, nqpq2, bit);
+}
+
+static __always_inline void
+ladder_smallloop_cmult_small_loop_double_step(u64 *nq, u64 *nqpq, u64 *nq2,
+					      u64 *nqpq2, u64 *q, u8 byt)
+{
+	u8 byt1;
+	ladder_smallloop_cmult_small_loop_step(nq, nqpq, nq2, nqpq2, q, byt);
+	byt1 = byt << 1;
+	ladder_smallloop_cmult_small_loop_step(nq2, nqpq2, nq, nqpq, q, byt1);
+}
+
+static __always_inline void
+ladder_smallloop_cmult_small_loop(u64 *nq, u64 *nqpq, u64 *nq2, u64 *nqpq2,
+				  u64 *q, u8 byt, u32 i)
+{
+	while (i--) {
+		ladder_smallloop_cmult_small_loop_double_step(nq, nqpq, nq2,
+							      nqpq2, q, byt);
+		byt <<= 2;
+	}
+}
+
+static __always_inline void ladder_bigloop_cmult_big_loop(u8 *n1, u64 *nq,
+							  u64 *nqpq, u64 *nq2,
+							  u64 *nqpq2, u64 *q,
+							  u32 i)
+{
+	while (i--) {
+		u8 byte = n1[i];
+		ladder_smallloop_cmult_small_loop(nq, nqpq, nq2, nqpq2, q,
+						  byte, 4);
+	}
+}
+
+static void ladder_cmult(u64 *result, u8 *n1, u64 *q)
+{
+	u64 point_buf[40] = { 0 };
+	u64 *nq = point_buf;
+	u64 *nqpq = point_buf + 10;
+	u64 *nq2 = point_buf + 20;
+	u64 *nqpq2 = point_buf + 30;
+	point_copy(nqpq, q);
+	nq[0] = 1;
+	ladder_bigloop_cmult_big_loop(n1, nq, nqpq, nq2, nqpq2, q, 32);
+	point_copy(result, nq);
+}
+
+static __always_inline void format_fexpand(u64 *output, const u8 *input)
+{
+	const u8 *x00 = input + 6;
+	const u8 *x01 = input + 12;
+	const u8 *x02 = input + 19;
+	const u8 *x0 = input + 24;
+	u64 i0, i1, i2, i3, i4, output0, output1, output2, output3, output4;
+	i0 = get_unaligned_le64(input);
+	i1 = get_unaligned_le64(x00);
+	i2 = get_unaligned_le64(x01);
+	i3 = get_unaligned_le64(x02);
+	i4 = get_unaligned_le64(x0);
+	output0 = i0 & 0x7ffffffffffffLLU;
+	output1 = i1 >> 3 & 0x7ffffffffffffLLU;
+	output2 = i2 >> 6 & 0x7ffffffffffffLLU;
+	output3 = i3 >> 1 & 0x7ffffffffffffLLU;
+	output4 = i4 >> 12 & 0x7ffffffffffffLLU;
+	output[0] = output0;
+	output[1] = output1;
+	output[2] = output2;
+	output[3] = output3;
+	output[4] = output4;
+}
+
+static __always_inline void format_fcontract_first_carry_pass(u64 *input)
+{
+	u64 t0 = input[0];
+	u64 t1 = input[1];
+	u64 t2 = input[2];
+	u64 t3 = input[3];
+	u64 t4 = input[4];
+	u64 t1_ = t1 + (t0 >> 51);
+	u64 t0_ = t0 & 0x7ffffffffffffLLU;
+	u64 t2_ = t2 + (t1_ >> 51);
+	u64 t1__ = t1_ & 0x7ffffffffffffLLU;
+	u64 t3_ = t3 + (t2_ >> 51);
+	u64 t2__ = t2_ & 0x7ffffffffffffLLU;
+	u64 t4_ = t4 + (t3_ >> 51);
+	u64 t3__ = t3_ & 0x7ffffffffffffLLU;
+	input[0] = t0_;
+	input[1] = t1__;
+	input[2] = t2__;
+	input[3] = t3__;
+	input[4] = t4_;
+}
+
+static __always_inline void format_fcontract_first_carry_full(u64 *input)
+{
+	format_fcontract_first_carry_pass(input);
+	modulo_carry_top(input);
+}
+
+static __always_inline void format_fcontract_second_carry_pass(u64 *input)
+{
+	u64 t0 = input[0];
+	u64 t1 = input[1];
+	u64 t2 = input[2];
+	u64 t3 = input[3];
+	u64 t4 = input[4];
+	u64 t1_ = t1 + (t0 >> 51);
+	u64 t0_ = t0 & 0x7ffffffffffffLLU;
+	u64 t2_ = t2 + (t1_ >> 51);
+	u64 t1__ = t1_ & 0x7ffffffffffffLLU;
+	u64 t3_ = t3 + (t2_ >> 51);
+	u64 t2__ = t2_ & 0x7ffffffffffffLLU;
+	u64 t4_ = t4 + (t3_ >> 51);
+	u64 t3__ = t3_ & 0x7ffffffffffffLLU;
+	input[0] = t0_;
+	input[1] = t1__;
+	input[2] = t2__;
+	input[3] = t3__;
+	input[4] = t4_;
+}
+
+static __always_inline void format_fcontract_second_carry_full(u64 *input)
+{
+	u64 i0;
+	u64 i1;
+	u64 i0_;
+	u64 i1_;
+	format_fcontract_second_carry_pass(input);
+	modulo_carry_top(input);
+	i0 = input[0];
+	i1 = input[1];
+	i0_ = i0 & 0x7ffffffffffffLLU;
+	i1_ = i1 + (i0 >> 51);
+	input[0] = i0_;
+	input[1] = i1_;
+}
+
+static __always_inline void format_fcontract_trim(u64 *input)
+{
+	u64 a0 = input[0];
+	u64 a1 = input[1];
+	u64 a2 = input[2];
+	u64 a3 = input[3];
+	u64 a4 = input[4];
+	u64 mask0 = u64_gte_mask(a0, 0x7ffffffffffedLLU);
+	u64 mask1 = u64_eq_mask(a1, 0x7ffffffffffffLLU);
+	u64 mask2 = u64_eq_mask(a2, 0x7ffffffffffffLLU);
+	u64 mask3 = u64_eq_mask(a3, 0x7ffffffffffffLLU);
+	u64 mask4 = u64_eq_mask(a4, 0x7ffffffffffffLLU);
+	u64 mask = (((mask0 & mask1) & mask2) & mask3) & mask4;
+	u64 a0_ = a0 - (0x7ffffffffffedLLU & mask);
+	u64 a1_ = a1 - (0x7ffffffffffffLLU & mask);
+	u64 a2_ = a2 - (0x7ffffffffffffLLU & mask);
+	u64 a3_ = a3 - (0x7ffffffffffffLLU & mask);
+	u64 a4_ = a4 - (0x7ffffffffffffLLU & mask);
+	input[0] = a0_;
+	input[1] = a1_;
+	input[2] = a2_;
+	input[3] = a3_;
+	input[4] = a4_;
+}
+
+static __always_inline void format_fcontract_store(u8 *output, u64 *input)
+{
+	u64 t0 = input[0];
+	u64 t1 = input[1];
+	u64 t2 = input[2];
+	u64 t3 = input[3];
+	u64 t4 = input[4];
+	u64 o0 = t1 << 51 | t0;
+	u64 o1 = t2 << 38 | t1 >> 13;
+	u64 o2 = t3 << 25 | t2 >> 26;
+	u64 o3 = t4 << 12 | t3 >> 39;
+	u8 *b0 = output;
+	u8 *b1 = output + 8;
+	u8 *b2 = output + 16;
+	u8 *b3 = output + 24;
+	put_unaligned_le64(o0, b0);
+	put_unaligned_le64(o1, b1);
+	put_unaligned_le64(o2, b2);
+	put_unaligned_le64(o3, b3);
+}
+
+static __always_inline void format_fcontract(u8 *output, u64 *input)
+{
+	format_fcontract_first_carry_full(input);
+	format_fcontract_second_carry_full(input);
+	format_fcontract_trim(input);
+	format_fcontract_store(output, input);
+}
+
+static __always_inline void format_scalar_of_point(u8 *scalar, u64 *point)
+{
+	u64 *x = point;
+	u64 *z = point + 5;
+	u64 buf[10] __aligned(32) = { 0 };
+	u64 *zmone = buf;
+	u64 *sc = buf + 5;
+	crecip(zmone, z);
+	fmul(sc, x, zmone);
+	format_fcontract(scalar, sc);
+}
+
+static void curve25519_generic(u8 mypublic[CURVE25519_KEY_SIZE],
+			       const u8 secret[CURVE25519_KEY_SIZE],
+			       const u8 basepoint[CURVE25519_KEY_SIZE])
+{
+	u64 buf0[10] __aligned(32) = { 0 };
+	u64 *x0 = buf0;
+	u64 *z = buf0 + 5;
+	u64 *q;
+	format_fexpand(x0, basepoint);
+	z[0] = 1;
+	q = buf0;
+	{
+		u8 e[32] __aligned(32) = { 0 };
+		u8 *scalar;
+		memcpy(e, secret, 32);
+		curve25519_clamp_secret(e);
+		scalar = e;
+		{
+			u64 buf[15] = { 0 };
+			u64 *nq = buf;
+			u64 *x = nq;
+			x[0] = 1;
+			ladder_cmult(nq, scalar, q);
+			format_scalar_of_point(mypublic, nq);
+			memzero_explicit(buf, sizeof(buf));
+		}
+		memzero_explicit(e, sizeof(e));
+	}
+	memzero_explicit(buf0, sizeof(buf0));
+}
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/curve25519/curve25519-x86_64.c b/net/wireguard/crypto/zinc/curve25519/curve25519-x86_64.c
--- a/net/wireguard/crypto/zinc/curve25519/curve25519-x86_64.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/curve25519/curve25519-x86_64.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,2366 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (c) 2017 Armando Faz <armfazh@ic.unicamp.br>. All Rights Reserved.
+ * Copyright (C) 2018-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ * Copyright (C) 2018 Samuel Neves <sneves@dei.uc.pt>. All Rights Reserved.
+ */
+
+enum { NUM_WORDS_ELTFP25519 = 4 };
+typedef __aligned(32) u64 eltfp25519_1w[NUM_WORDS_ELTFP25519];
+typedef __aligned(32) u64 eltfp25519_1w_buffer[2 * NUM_WORDS_ELTFP25519];
+
+#define mul_eltfp25519_1w_adx(c, a, b) do { \
+	mul_256x256_integer_adx(m.buffer, a, b); \
+	red_eltfp25519_1w_adx(c, m.buffer); \
+} while (0)
+
+#define mul_eltfp25519_1w_bmi2(c, a, b) do { \
+	mul_256x256_integer_bmi2(m.buffer, a, b); \
+	red_eltfp25519_1w_bmi2(c, m.buffer); \
+} while (0)
+
+#define sqr_eltfp25519_1w_adx(a) do { \
+	sqr_256x256_integer_adx(m.buffer, a); \
+	red_eltfp25519_1w_adx(a, m.buffer); \
+} while (0)
+
+#define sqr_eltfp25519_1w_bmi2(a) do { \
+	sqr_256x256_integer_bmi2(m.buffer, a); \
+	red_eltfp25519_1w_bmi2(a, m.buffer); \
+} while (0)
+
+#define mul_eltfp25519_2w_adx(c, a, b) do { \
+	mul2_256x256_integer_adx(m.buffer, a, b); \
+	red_eltfp25519_2w_adx(c, m.buffer); \
+} while (0)
+
+#define mul_eltfp25519_2w_bmi2(c, a, b) do { \
+	mul2_256x256_integer_bmi2(m.buffer, a, b); \
+	red_eltfp25519_2w_bmi2(c, m.buffer); \
+} while (0)
+
+#define sqr_eltfp25519_2w_adx(a) do { \
+	sqr2_256x256_integer_adx(m.buffer, a); \
+	red_eltfp25519_2w_adx(a, m.buffer); \
+} while (0)
+
+#define sqr_eltfp25519_2w_bmi2(a) do { \
+	sqr2_256x256_integer_bmi2(m.buffer, a); \
+	red_eltfp25519_2w_bmi2(a, m.buffer); \
+} while (0)
+
+#define sqrn_eltfp25519_1w_adx(a, times) do { \
+	int ____counter = (times); \
+	while (____counter-- > 0) \
+		sqr_eltfp25519_1w_adx(a); \
+} while (0)
+
+#define sqrn_eltfp25519_1w_bmi2(a, times) do { \
+	int ____counter = (times); \
+	while (____counter-- > 0) \
+		sqr_eltfp25519_1w_bmi2(a); \
+} while (0)
+
+#define copy_eltfp25519_1w(C, A) do { \
+	(C)[0] = (A)[0]; \
+	(C)[1] = (A)[1]; \
+	(C)[2] = (A)[2]; \
+	(C)[3] = (A)[3]; \
+} while (0)
+
+#define setzero_eltfp25519_1w(C) do { \
+	(C)[0] = 0; \
+	(C)[1] = 0; \
+	(C)[2] = 0; \
+	(C)[3] = 0; \
+} while (0)
+
+__aligned(32) static const u64 table_ladder_8k[252 * NUM_WORDS_ELTFP25519] = {
+	/*   1 */ 0xfffffffffffffff3UL, 0xffffffffffffffffUL,
+		  0xffffffffffffffffUL, 0x5fffffffffffffffUL,
+	/*   2 */ 0x6b8220f416aafe96UL, 0x82ebeb2b4f566a34UL,
+		  0xd5a9a5b075a5950fUL, 0x5142b2cf4b2488f4UL,
+	/*   3 */ 0x6aaebc750069680cUL, 0x89cf7820a0f99c41UL,
+		  0x2a58d9183b56d0f4UL, 0x4b5aca80e36011a4UL,
+	/*   4 */ 0x329132348c29745dUL, 0xf4a2e616e1642fd7UL,
+		  0x1e45bb03ff67bc34UL, 0x306912d0f42a9b4aUL,
+	/*   5 */ 0xff886507e6af7154UL, 0x04f50e13dfeec82fUL,
+		  0xaa512fe82abab5ceUL, 0x174e251a68d5f222UL,
+	/*   6 */ 0xcf96700d82028898UL, 0x1743e3370a2c02c5UL,
+		  0x379eec98b4e86eaaUL, 0x0c59888a51e0482eUL,
+	/*   7 */ 0xfbcbf1d699b5d189UL, 0xacaef0d58e9fdc84UL,
+		  0xc1c20d06231f7614UL, 0x2938218da274f972UL,
+	/*   8 */ 0xf6af49beff1d7f18UL, 0xcc541c22387ac9c2UL,
+		  0x96fcc9ef4015c56bUL, 0x69c1627c690913a9UL,
+	/*   9 */ 0x7a86fd2f4733db0eUL, 0xfdb8c4f29e087de9UL,
+		  0x095e4b1a8ea2a229UL, 0x1ad7a7c829b37a79UL,
+	/*  10 */ 0x342d89cad17ea0c0UL, 0x67bedda6cced2051UL,
+		  0x19ca31bf2bb42f74UL, 0x3df7b4c84980acbbUL,
+	/*  11 */ 0xa8c6444dc80ad883UL, 0xb91e440366e3ab85UL,
+		  0xc215cda00164f6d8UL, 0x3d867c6ef247e668UL,
+	/*  12 */ 0xc7dd582bcc3e658cUL, 0xfd2c4748ee0e5528UL,
+		  0xa0fd9b95cc9f4f71UL, 0x7529d871b0675ddfUL,
+	/*  13 */ 0xb8f568b42d3cbd78UL, 0x1233011b91f3da82UL,
+		  0x2dce6ccd4a7c3b62UL, 0x75e7fc8e9e498603UL,
+	/*  14 */ 0x2f4f13f1fcd0b6ecUL, 0xf1a8ca1f29ff7a45UL,
+		  0xc249c1a72981e29bUL, 0x6ebe0dbb8c83b56aUL,
+	/*  15 */ 0x7114fa8d170bb222UL, 0x65a2dcd5bf93935fUL,
+		  0xbdc41f68b59c979aUL, 0x2f0eef79a2ce9289UL,
+	/*  16 */ 0x42ecbf0c083c37ceUL, 0x2930bc09ec496322UL,
+		  0xf294b0c19cfeac0dUL, 0x3780aa4bedfabb80UL,
+	/*  17 */ 0x56c17d3e7cead929UL, 0xe7cb4beb2e5722c5UL,
+		  0x0ce931732dbfe15aUL, 0x41b883c7621052f8UL,
+	/*  18 */ 0xdbf75ca0c3d25350UL, 0x2936be086eb1e351UL,
+		  0xc936e03cb4a9b212UL, 0x1d45bf82322225aaUL,
+	/*  19 */ 0xe81ab1036a024cc5UL, 0xe212201c304c9a72UL,
+		  0xc5d73fba6832b1fcUL, 0x20ffdb5a4d839581UL,
+	/*  20 */ 0xa283d367be5d0fadUL, 0x6c2b25ca8b164475UL,
+		  0x9d4935467caaf22eUL, 0x5166408eee85ff49UL,
+	/*  21 */ 0x3c67baa2fab4e361UL, 0xb3e433c67ef35cefUL,
+		  0x5259729241159b1cUL, 0x6a621892d5b0ab33UL,
+	/*  22 */ 0x20b74a387555cdcbUL, 0x532aa10e1208923fUL,
+		  0xeaa17b7762281dd1UL, 0x61ab3443f05c44bfUL,
+	/*  23 */ 0x257a6c422324def8UL, 0x131c6c1017e3cf7fUL,
+		  0x23758739f630a257UL, 0x295a407a01a78580UL,
+	/*  24 */ 0xf8c443246d5da8d9UL, 0x19d775450c52fa5dUL,
+		  0x2afcfc92731bf83dUL, 0x7d10c8e81b2b4700UL,
+	/*  25 */ 0xc8e0271f70baa20bUL, 0x993748867ca63957UL,
+		  0x5412efb3cb7ed4bbUL, 0x3196d36173e62975UL,
+	/*  26 */ 0xde5bcad141c7dffcUL, 0x47cc8cd2b395c848UL,
+		  0xa34cd942e11af3cbUL, 0x0256dbf2d04ecec2UL,
+	/*  27 */ 0x875ab7e94b0e667fUL, 0xcad4dd83c0850d10UL,
+		  0x47f12e8f4e72c79fUL, 0x5f1a87bb8c85b19bUL,
+	/*  28 */ 0x7ae9d0b6437f51b8UL, 0x12c7ce5518879065UL,
+		  0x2ade09fe5cf77aeeUL, 0x23a05a2f7d2c5627UL,
+	/*  29 */ 0x5908e128f17c169aUL, 0xf77498dd8ad0852dUL,
+		  0x74b4c4ceab102f64UL, 0x183abadd10139845UL,
+	/*  30 */ 0xb165ba8daa92aaacUL, 0xd5c5ef9599386705UL,
+		  0xbe2f8f0cf8fc40d1UL, 0x2701e635ee204514UL,
+	/*  31 */ 0x629fa80020156514UL, 0xf223868764a8c1ceUL,
+		  0x5b894fff0b3f060eUL, 0x60d9944cf708a3faUL,
+	/*  32 */ 0xaeea001a1c7a201fUL, 0xebf16a633ee2ce63UL,
+		  0x6f7709594c7a07e1UL, 0x79b958150d0208cbUL,
+	/*  33 */ 0x24b55e5301d410e7UL, 0xe3a34edff3fdc84dUL,
+		  0xd88768e4904032d8UL, 0x131384427b3aaeecUL,
+	/*  34 */ 0x8405e51286234f14UL, 0x14dc4739adb4c529UL,
+		  0xb8a2b5b250634ffdUL, 0x2fe2a94ad8a7ff93UL,
+	/*  35 */ 0xec5c57efe843faddUL, 0x2843ce40f0bb9918UL,
+		  0xa4b561d6cf3d6305UL, 0x743629bde8fb777eUL,
+	/*  36 */ 0x343edd46bbaf738fUL, 0xed981828b101a651UL,
+		  0xa401760b882c797aUL, 0x1fc223e28dc88730UL,
+	/*  37 */ 0x48604e91fc0fba0eUL, 0xb637f78f052c6fa4UL,
+		  0x91ccac3d09e9239cUL, 0x23f7eed4437a687cUL,
+	/*  38 */ 0x5173b1118d9bd800UL, 0x29d641b63189d4a7UL,
+		  0xfdbf177988bbc586UL, 0x2959894fcad81df5UL,
+	/*  39 */ 0xaebc8ef3b4bbc899UL, 0x4148995ab26992b9UL,
+		  0x24e20b0134f92cfbUL, 0x40d158894a05dee8UL,
+	/*  40 */ 0x46b00b1185af76f6UL, 0x26bac77873187a79UL,
+		  0x3dc0bf95ab8fff5fUL, 0x2a608bd8945524d7UL,
+	/*  41 */ 0x26449588bd446302UL, 0x7c4bc21c0388439cUL,
+		  0x8e98a4f383bd11b2UL, 0x26218d7bc9d876b9UL,
+	/*  42 */ 0xe3081542997c178aUL, 0x3c2d29a86fb6606fUL,
+		  0x5c217736fa279374UL, 0x7dde05734afeb1faUL,
+	/*  43 */ 0x3bf10e3906d42babUL, 0xe4f7803e1980649cUL,
+		  0xe6053bf89595bf7aUL, 0x394faf38da245530UL,
+	/*  44 */ 0x7a8efb58896928f4UL, 0xfbc778e9cc6a113cUL,
+		  0x72670ce330af596fUL, 0x48f222a81d3d6cf7UL,
+	/*  45 */ 0xf01fce410d72caa7UL, 0x5a20ecc7213b5595UL,
+		  0x7bc21165c1fa1483UL, 0x07f89ae31da8a741UL,
+	/*  46 */ 0x05d2c2b4c6830ff9UL, 0xd43e330fc6316293UL,
+		  0xa5a5590a96d3a904UL, 0x705edb91a65333b6UL,
+	/*  47 */ 0x048ee15e0bb9a5f7UL, 0x3240cfca9e0aaf5dUL,
+		  0x8f4b71ceedc4a40bUL, 0x621c0da3de544a6dUL,
+	/*  48 */ 0x92872836a08c4091UL, 0xce8375b010c91445UL,
+		  0x8a72eb524f276394UL, 0x2667fcfa7ec83635UL,
+	/*  49 */ 0x7f4c173345e8752aUL, 0x061b47feee7079a5UL,
+		  0x25dd9afa9f86ff34UL, 0x3780cef5425dc89cUL,
+	/*  50 */ 0x1a46035a513bb4e9UL, 0x3e1ef379ac575adaUL,
+		  0xc78c5f1c5fa24b50UL, 0x321a967634fd9f22UL,
+	/*  51 */ 0x946707b8826e27faUL, 0x3dca84d64c506fd0UL,
+		  0xc189218075e91436UL, 0x6d9284169b3b8484UL,
+	/*  52 */ 0x3a67e840383f2ddfUL, 0x33eec9a30c4f9b75UL,
+		  0x3ec7c86fa783ef47UL, 0x26ec449fbac9fbc4UL,
+	/*  53 */ 0x5c0f38cba09b9e7dUL, 0x81168cc762a3478cUL,
+		  0x3e23b0d306fc121cUL, 0x5a238aa0a5efdcddUL,
+	/*  54 */ 0x1ba26121c4ea43ffUL, 0x36f8c77f7c8832b5UL,
+		  0x88fbea0b0adcf99aUL, 0x5ca9938ec25bebf9UL,
+	/*  55 */ 0xd5436a5e51fccda0UL, 0x1dbc4797c2cd893bUL,
+		  0x19346a65d3224a08UL, 0x0f5034e49b9af466UL,
+	/*  56 */ 0xf23c3967a1e0b96eUL, 0xe58b08fa867a4d88UL,
+		  0xfb2fabc6a7341679UL, 0x2a75381eb6026946UL,
+	/*  57 */ 0xc80a3be4c19420acUL, 0x66b1f6c681f2b6dcUL,
+		  0x7cf7036761e93388UL, 0x25abbbd8a660a4c4UL,
+	/*  58 */ 0x91ea12ba14fd5198UL, 0x684950fc4a3cffa9UL,
+		  0xf826842130f5ad28UL, 0x3ea988f75301a441UL,
+	/*  59 */ 0xc978109a695f8c6fUL, 0x1746eb4a0530c3f3UL,
+		  0x444d6d77b4459995UL, 0x75952b8c054e5cc7UL,
+	/*  60 */ 0xa3703f7915f4d6aaUL, 0x66c346202f2647d8UL,
+		  0xd01469df811d644bUL, 0x77fea47d81a5d71fUL,
+	/*  61 */ 0xc5e9529ef57ca381UL, 0x6eeeb4b9ce2f881aUL,
+		  0xb6e91a28e8009bd6UL, 0x4b80be3e9afc3fecUL,
+	/*  62 */ 0x7e3773c526aed2c5UL, 0x1b4afcb453c9a49dUL,
+		  0xa920bdd7baffb24dUL, 0x7c54699f122d400eUL,
+	/*  63 */ 0xef46c8e14fa94bc8UL, 0xe0b074ce2952ed5eUL,
+		  0xbea450e1dbd885d5UL, 0x61b68649320f712cUL,
+	/*  64 */ 0x8a485f7309ccbdd1UL, 0xbd06320d7d4d1a2dUL,
+		  0x25232973322dbef4UL, 0x445dc4758c17f770UL,
+	/*  65 */ 0xdb0434177cc8933cUL, 0xed6fe82175ea059fUL,
+		  0x1efebefdc053db34UL, 0x4adbe867c65daf99UL,
+	/*  66 */ 0x3acd71a2a90609dfUL, 0xe5e991856dd04050UL,
+		  0x1ec69b688157c23cUL, 0x697427f6885cfe4dUL,
+	/*  67 */ 0xd7be7b9b65e1a851UL, 0xa03d28d522c536ddUL,
+		  0x28399d658fd2b645UL, 0x49e5b7e17c2641e1UL,
+	/*  68 */ 0x6f8c3a98700457a4UL, 0x5078f0a25ebb6778UL,
+		  0xd13c3ccbc382960fUL, 0x2e003258a7df84b1UL,
+	/*  69 */ 0x8ad1f39be6296a1cUL, 0xc1eeaa652a5fbfb2UL,
+		  0x33ee0673fd26f3cbUL, 0x59256173a69d2cccUL,
+	/*  70 */ 0x41ea07aa4e18fc41UL, 0xd9fc19527c87a51eUL,
+		  0xbdaacb805831ca6fUL, 0x445b652dc916694fUL,
+	/*  71 */ 0xce92a3a7f2172315UL, 0x1edc282de11b9964UL,
+		  0xa1823aafe04c314aUL, 0x790a2d94437cf586UL,
+	/*  72 */ 0x71c447fb93f6e009UL, 0x8922a56722845276UL,
+		  0xbf70903b204f5169UL, 0x2f7a89891ba319feUL,
+	/*  73 */ 0x02a08eb577e2140cUL, 0xed9a4ed4427bdcf4UL,
+		  0x5253ec44e4323cd1UL, 0x3e88363c14e9355bUL,
+	/*  74 */ 0xaa66c14277110b8cUL, 0x1ae0391610a23390UL,
+		  0x2030bd12c93fc2a2UL, 0x3ee141579555c7abUL,
+	/*  75 */ 0x9214de3a6d6e7d41UL, 0x3ccdd88607f17efeUL,
+		  0x674f1288f8e11217UL, 0x5682250f329f93d0UL,
+	/*  76 */ 0x6cf00b136d2e396eUL, 0x6e4cf86f1014debfUL,
+		  0x5930b1b5bfcc4e83UL, 0x047069b48aba16b6UL,
+	/*  77 */ 0x0d4ce4ab69b20793UL, 0xb24db91a97d0fb9eUL,
+		  0xcdfa50f54e00d01dUL, 0x221b1085368bddb5UL,
+	/*  78 */ 0xe7e59468b1e3d8d2UL, 0x53c56563bd122f93UL,
+		  0xeee8a903e0663f09UL, 0x61efa662cbbe3d42UL,
+	/*  79 */ 0x2cf8ddddde6eab2aUL, 0x9bf80ad51435f231UL,
+		  0x5deadacec9f04973UL, 0x29275b5d41d29b27UL,
+	/*  80 */ 0xcfde0f0895ebf14fUL, 0xb9aab96b054905a7UL,
+		  0xcae80dd9a1c420fdUL, 0x0a63bf2f1673bbc7UL,
+	/*  81 */ 0x092f6e11958fbc8cUL, 0x672a81e804822fadUL,
+		  0xcac8351560d52517UL, 0x6f3f7722c8f192f8UL,
+	/*  82 */ 0xf8ba90ccc2e894b7UL, 0x2c7557a438ff9f0dUL,
+		  0x894d1d855ae52359UL, 0x68e122157b743d69UL,
+	/*  83 */ 0xd87e5570cfb919f3UL, 0x3f2cdecd95798db9UL,
+		  0x2121154710c0a2ceUL, 0x3c66a115246dc5b2UL,
+	/*  84 */ 0xcbedc562294ecb72UL, 0xba7143c36a280b16UL,
+		  0x9610c2efd4078b67UL, 0x6144735d946a4b1eUL,
+	/*  85 */ 0x536f111ed75b3350UL, 0x0211db8c2041d81bUL,
+		  0xf93cb1000e10413cUL, 0x149dfd3c039e8876UL,
+	/*  86 */ 0xd479dde46b63155bUL, 0xb66e15e93c837976UL,
+		  0xdafde43b1f13e038UL, 0x5fafda1a2e4b0b35UL,
+	/*  87 */ 0x3600bbdf17197581UL, 0x3972050bbe3cd2c2UL,
+		  0x5938906dbdd5be86UL, 0x34fce5e43f9b860fUL,
+	/*  88 */ 0x75a8a4cd42d14d02UL, 0x828dabc53441df65UL,
+		  0x33dcabedd2e131d3UL, 0x3ebad76fb814d25fUL,
+	/*  89 */ 0xd4906f566f70e10fUL, 0x5d12f7aa51690f5aUL,
+		  0x45adb16e76cefcf2UL, 0x01f768aead232999UL,
+	/*  90 */ 0x2b6cc77b6248febdUL, 0x3cd30628ec3aaffdUL,
+		  0xce1c0b80d4ef486aUL, 0x4c3bff2ea6f66c23UL,
+	/*  91 */ 0x3f2ec4094aeaeb5fUL, 0x61b19b286e372ca7UL,
+		  0x5eefa966de2a701dUL, 0x23b20565de55e3efUL,
+	/*  92 */ 0xe301ca5279d58557UL, 0x07b2d4ce27c2874fUL,
+		  0xa532cd8a9dcf1d67UL, 0x2a52fee23f2bff56UL,
+	/*  93 */ 0x8624efb37cd8663dUL, 0xbbc7ac20ffbd7594UL,
+		  0x57b85e9c82d37445UL, 0x7b3052cb86a6ec66UL,
+	/*  94 */ 0x3482f0ad2525e91eUL, 0x2cb68043d28edca0UL,
+		  0xaf4f6d052e1b003aUL, 0x185f8c2529781b0aUL,
+	/*  95 */ 0xaa41de5bd80ce0d6UL, 0x9407b2416853e9d6UL,
+		  0x563ec36e357f4c3aUL, 0x4cc4b8dd0e297bceUL,
+	/*  96 */ 0xa2fc1a52ffb8730eUL, 0x1811f16e67058e37UL,
+		  0x10f9a366cddf4ee1UL, 0x72f4a0c4a0b9f099UL,
+	/*  97 */ 0x8c16c06f663f4ea7UL, 0x693b3af74e970fbaUL,
+		  0x2102e7f1d69ec345UL, 0x0ba53cbc968a8089UL,
+	/*  98 */ 0xca3d9dc7fea15537UL, 0x4c6824bb51536493UL,
+		  0xb9886314844006b1UL, 0x40d2a72ab454cc60UL,
+	/*  99 */ 0x5936a1b712570975UL, 0x91b9d648debda657UL,
+		  0x3344094bb64330eaUL, 0x006ba10d12ee51d0UL,
+	/* 100 */ 0x19228468f5de5d58UL, 0x0eb12f4c38cc05b0UL,
+		  0xa1039f9dd5601990UL, 0x4502d4ce4fff0e0bUL,
+	/* 101 */ 0xeb2054106837c189UL, 0xd0f6544c6dd3b93cUL,
+		  0x40727064c416d74fUL, 0x6e15c6114b502ef0UL,
+	/* 102 */ 0x4df2a398cfb1a76bUL, 0x11256c7419f2f6b1UL,
+		  0x4a497962066e6043UL, 0x705b3aab41355b44UL,
+	/* 103 */ 0x365ef536d797b1d8UL, 0x00076bd622ddf0dbUL,
+		  0x3bbf33b0e0575a88UL, 0x3777aa05c8e4ca4dUL,
+	/* 104 */ 0x392745c85578db5fUL, 0x6fda4149dbae5ae2UL,
+		  0xb1f0b00b8adc9867UL, 0x09963437d36f1da3UL,
+	/* 105 */ 0x7e824e90a5dc3853UL, 0xccb5f6641f135cbdUL,
+		  0x6736d86c87ce8fccUL, 0x625f3ce26604249fUL,
+	/* 106 */ 0xaf8ac8059502f63fUL, 0x0c05e70a2e351469UL,
+		  0x35292e9c764b6305UL, 0x1a394360c7e23ac3UL,
+	/* 107 */ 0xd5c6d53251183264UL, 0x62065abd43c2b74fUL,
+		  0xb5fbf5d03b973f9bUL, 0x13a3da3661206e5eUL,
+	/* 108 */ 0xc6bd5837725d94e5UL, 0x18e30912205016c5UL,
+		  0x2088ce1570033c68UL, 0x7fba1f495c837987UL,
+	/* 109 */ 0x5a8c7423f2f9079dUL, 0x1735157b34023fc5UL,
+		  0xe4f9b49ad2fab351UL, 0x6691ff72c878e33cUL,
+	/* 110 */ 0x122c2adedc5eff3eUL, 0xf8dd4bf1d8956cf4UL,
+		  0xeb86205d9e9e5bdaUL, 0x049b92b9d975c743UL,
+	/* 111 */ 0xa5379730b0f6c05aUL, 0x72a0ffacc6f3a553UL,
+		  0xb0032c34b20dcd6dUL, 0x470e9dbc88d5164aUL,
+	/* 112 */ 0xb19cf10ca237c047UL, 0xb65466711f6c81a2UL,
+		  0xb3321bd16dd80b43UL, 0x48c14f600c5fbe8eUL,
+	/* 113 */ 0x66451c264aa6c803UL, 0xb66e3904a4fa7da6UL,
+		  0xd45f19b0b3128395UL, 0x31602627c3c9bc10UL,
+	/* 114 */ 0x3120dc4832e4e10dUL, 0xeb20c46756c717f7UL,
+		  0x00f52e3f67280294UL, 0x566d4fc14730c509UL,
+	/* 115 */ 0x7e3a5d40fd837206UL, 0xc1e926dc7159547aUL,
+		  0x216730fba68d6095UL, 0x22e8c3843f69cea7UL,
+	/* 116 */ 0x33d074e8930e4b2bUL, 0xb6e4350e84d15816UL,
+		  0x5534c26ad6ba2365UL, 0x7773c12f89f1f3f3UL,
+	/* 117 */ 0x8cba404da57962aaUL, 0x5b9897a81999ce56UL,
+		  0x508e862f121692fcUL, 0x3a81907fa093c291UL,
+	/* 118 */ 0x0dded0ff4725a510UL, 0x10d8cc10673fc503UL,
+		  0x5b9d151c9f1f4e89UL, 0x32a5c1d5cb09a44cUL,
+	/* 119 */ 0x1e0aa442b90541fbUL, 0x5f85eb7cc1b485dbUL,
+		  0xbee595ce8a9df2e5UL, 0x25e496c722422236UL,
+	/* 120 */ 0x5edf3c46cd0fe5b9UL, 0x34e75a7ed2a43388UL,
+		  0xe488de11d761e352UL, 0x0e878a01a085545cUL,
+	/* 121 */ 0xba493c77e021bb04UL, 0x2b4d1843c7df899aUL,
+		  0x9ea37a487ae80d67UL, 0x67a9958011e41794UL,
+	/* 122 */ 0x4b58051a6697b065UL, 0x47e33f7d8d6ba6d4UL,
+		  0xbb4da8d483ca46c1UL, 0x68becaa181c2db0dUL,
+	/* 123 */ 0x8d8980e90b989aa5UL, 0xf95eb14a2c93c99bUL,
+		  0x51c6c7c4796e73a2UL, 0x6e228363b5efb569UL,
+	/* 124 */ 0xc6bbc0b02dd624c8UL, 0x777eb47dec8170eeUL,
+		  0x3cde15a004cfafa9UL, 0x1dc6bc087160bf9bUL,
+	/* 125 */ 0x2e07e043eec34002UL, 0x18e9fc677a68dc7fUL,
+		  0xd8da03188bd15b9aUL, 0x48fbc3bb00568253UL,
+	/* 126 */ 0x57547d4cfb654ce1UL, 0xd3565b82a058e2adUL,
+		  0xf63eaf0bbf154478UL, 0x47531ef114dfbb18UL,
+	/* 127 */ 0xe1ec630a4278c587UL, 0x5507d546ca8e83f3UL,
+		  0x85e135c63adc0c2bUL, 0x0aa7efa85682844eUL,
+	/* 128 */ 0x72691ba8b3e1f615UL, 0x32b4e9701fbe3ffaUL,
+		  0x97b6d92e39bb7868UL, 0x2cfe53dea02e39e8UL,
+	/* 129 */ 0x687392cd85cd52b0UL, 0x27ff66c910e29831UL,
+		  0x97134556a9832d06UL, 0x269bb0360a84f8a0UL,
+	/* 130 */ 0x706e55457643f85cUL, 0x3734a48c9b597d1bUL,
+		  0x7aee91e8c6efa472UL, 0x5cd6abc198a9d9e0UL,
+	/* 131 */ 0x0e04de06cb3ce41aUL, 0xd8c6eb893402e138UL,
+		  0x904659bb686e3772UL, 0x7215c371746ba8c8UL,
+	/* 132 */ 0xfd12a97eeae4a2d9UL, 0x9514b7516394f2c5UL,
+		  0x266fd5809208f294UL, 0x5c847085619a26b9UL,
+	/* 133 */ 0x52985410fed694eaUL, 0x3c905b934a2ed254UL,
+		  0x10bb47692d3be467UL, 0x063b3d2d69e5e9e1UL,
+	/* 134 */ 0x472726eedda57debUL, 0xefb6c4ae10f41891UL,
+		  0x2b1641917b307614UL, 0x117c554fc4f45b7cUL,
+	/* 135 */ 0xc07cf3118f9d8812UL, 0x01dbd82050017939UL,
+		  0xd7e803f4171b2827UL, 0x1015e87487d225eaUL,
+	/* 136 */ 0xc58de3fed23acc4dUL, 0x50db91c294a7be2dUL,
+		  0x0b94d43d1c9cf457UL, 0x6b1640fa6e37524aUL,
+	/* 137 */ 0x692f346c5fda0d09UL, 0x200b1c59fa4d3151UL,
+		  0xb8c46f760777a296UL, 0x4b38395f3ffdfbcfUL,
+	/* 138 */ 0x18d25e00be54d671UL, 0x60d50582bec8aba6UL,
+		  0x87ad8f263b78b982UL, 0x50fdf64e9cda0432UL,
+	/* 139 */ 0x90f567aac578dcf0UL, 0xef1e9b0ef2a3133bUL,
+		  0x0eebba9242d9de71UL, 0x15473c9bf03101c7UL,
+	/* 140 */ 0x7c77e8ae56b78095UL, 0xb678e7666e6f078eUL,
+		  0x2da0b9615348ba1fUL, 0x7cf931c1ff733f0bUL,
+	/* 141 */ 0x26b357f50a0a366cUL, 0xe9708cf42b87d732UL,
+		  0xc13aeea5f91cb2c0UL, 0x35d90c991143bb4cUL,
+	/* 142 */ 0x47c1c404a9a0d9dcUL, 0x659e58451972d251UL,
+		  0x3875a8c473b38c31UL, 0x1fbd9ed379561f24UL,
+	/* 143 */ 0x11fabc6fd41ec28dUL, 0x7ef8dfe3cd2a2dcaUL,
+		  0x72e73b5d8c404595UL, 0x6135fa4954b72f27UL,
+	/* 144 */ 0xccfc32a2de24b69cUL, 0x3f55698c1f095d88UL,
+		  0xbe3350ed5ac3f929UL, 0x5e9bf806ca477eebUL,
+	/* 145 */ 0xe9ce8fb63c309f68UL, 0x5376f63565e1f9f4UL,
+		  0xd1afcfb35a6393f1UL, 0x6632a1ede5623506UL,
+	/* 146 */ 0x0b7d6c390c2ded4cUL, 0x56cb3281df04cb1fUL,
+		  0x66305a1249ecc3c7UL, 0x5d588b60a38ca72aUL,
+	/* 147 */ 0xa6ecbf78e8e5f42dUL, 0x86eeb44b3c8a3eecUL,
+		  0xec219c48fbd21604UL, 0x1aaf1af517c36731UL,
+	/* 148 */ 0xc306a2836769bde7UL, 0x208280622b1e2adbUL,
+		  0x8027f51ffbff94a6UL, 0x76cfa1ce1124f26bUL,
+	/* 149 */ 0x18eb00562422abb6UL, 0xf377c4d58f8c29c3UL,
+		  0x4dbbc207f531561aUL, 0x0253b7f082128a27UL,
+	/* 150 */ 0x3d1f091cb62c17e0UL, 0x4860e1abd64628a9UL,
+		  0x52d17436309d4253UL, 0x356f97e13efae576UL,
+	/* 151 */ 0xd351e11aa150535bUL, 0x3e6b45bb1dd878ccUL,
+		  0x0c776128bed92c98UL, 0x1d34ae93032885b8UL,
+	/* 152 */ 0x4ba0488ca85ba4c3UL, 0x985348c33c9ce6ceUL,
+		  0x66124c6f97bda770UL, 0x0f81a0290654124aUL,
+	/* 153 */ 0x9ed09ca6569b86fdUL, 0x811009fd18af9a2dUL,
+		  0xff08d03f93d8c20aUL, 0x52a148199faef26bUL,
+	/* 154 */ 0x3e03f9dc2d8d1b73UL, 0x4205801873961a70UL,
+		  0xc0d987f041a35970UL, 0x07aa1f15a1c0d549UL,
+	/* 155 */ 0xdfd46ce08cd27224UL, 0x6d0a024f934e4239UL,
+		  0x808a7a6399897b59UL, 0x0a4556e9e13d95a2UL,
+	/* 156 */ 0xd21a991fe9c13045UL, 0x9b0e8548fe7751b8UL,
+		  0x5da643cb4bf30035UL, 0x77db28d63940f721UL,
+	/* 157 */ 0xfc5eeb614adc9011UL, 0x5229419ae8c411ebUL,
+		  0x9ec3e7787d1dcf74UL, 0x340d053e216e4cb5UL,
+	/* 158 */ 0xcac7af39b48df2b4UL, 0xc0faec2871a10a94UL,
+		  0x140a69245ca575edUL, 0x0cf1c37134273a4cUL,
+	/* 159 */ 0xc8ee306ac224b8a5UL, 0x57eaee7ccb4930b0UL,
+		  0xa1e806bdaacbe74fUL, 0x7d9a62742eeb657dUL,
+	/* 160 */ 0x9eb6b6ef546c4830UL, 0x885cca1fddb36e2eUL,
+		  0xe6b9f383ef0d7105UL, 0x58654fef9d2e0412UL,
+	/* 161 */ 0xa905c4ffbe0e8e26UL, 0x942de5df9b31816eUL,
+		  0x497d723f802e88e1UL, 0x30684dea602f408dUL,
+	/* 162 */ 0x21e5a278a3e6cb34UL, 0xaefb6e6f5b151dc4UL,
+		  0xb30b8e049d77ca15UL, 0x28c3c9cf53b98981UL,
+	/* 163 */ 0x287fb721556cdd2aUL, 0x0d317ca897022274UL,
+		  0x7468c7423a543258UL, 0x4a7f11464eb5642fUL,
+	/* 164 */ 0xa237a4774d193aa6UL, 0xd865986ea92129a1UL,
+		  0x24c515ecf87c1a88UL, 0x604003575f39f5ebUL,
+	/* 165 */ 0x47b9f189570a9b27UL, 0x2b98cede465e4b78UL,
+		  0x026df551dbb85c20UL, 0x74fcd91047e21901UL,
+	/* 166 */ 0x13e2a90a23c1bfa3UL, 0x0cb0074e478519f6UL,
+		  0x5ff1cbbe3af6cf44UL, 0x67fe5438be812dbeUL,
+	/* 167 */ 0xd13cf64fa40f05b0UL, 0x054dfb2f32283787UL,
+		  0x4173915b7f0d2aeaUL, 0x482f144f1f610d4eUL,
+	/* 168 */ 0xf6210201b47f8234UL, 0x5d0ae1929e70b990UL,
+		  0xdcd7f455b049567cUL, 0x7e93d0f1f0916f01UL,
+	/* 169 */ 0xdd79cbf18a7db4faUL, 0xbe8391bf6f74c62fUL,
+		  0x027145d14b8291bdUL, 0x585a73ea2cbf1705UL,
+	/* 170 */ 0x485ca03e928a0db2UL, 0x10fc01a5742857e7UL,
+		  0x2f482edbd6d551a7UL, 0x0f0433b5048fdb8aUL,
+	/* 171 */ 0x60da2e8dd7dc6247UL, 0x88b4c9d38cd4819aUL,
+		  0x13033ac001f66697UL, 0x273b24fe3b367d75UL,
+	/* 172 */ 0xc6e8f66a31b3b9d4UL, 0x281514a494df49d5UL,
+		  0xd1726fdfc8b23da7UL, 0x4b3ae7d103dee548UL,
+	/* 173 */ 0xc6256e19ce4b9d7eUL, 0xff5c5cf186e3c61cUL,
+		  0xacc63ca34b8ec145UL, 0x74621888fee66574UL,
+	/* 174 */ 0x956f409645290a1eUL, 0xef0bf8e3263a962eUL,
+		  0xed6a50eb5ec2647bUL, 0x0694283a9dca7502UL,
+	/* 175 */ 0x769b963643a2dcd1UL, 0x42b7c8ea09fc5353UL,
+		  0x4f002aee13397eabUL, 0x63005e2c19b7d63aUL,
+	/* 176 */ 0xca6736da63023beaUL, 0x966c7f6db12a99b7UL,
+		  0xace09390c537c5e1UL, 0x0b696063a1aa89eeUL,
+	/* 177 */ 0xebb03e97288c56e5UL, 0x432a9f9f938c8be8UL,
+		  0xa6a5a93d5b717f71UL, 0x1a5fb4c3e18f9d97UL,
+	/* 178 */ 0x1c94e7ad1c60cdceUL, 0xee202a43fc02c4a0UL,
+		  0x8dafe4d867c46a20UL, 0x0a10263c8ac27b58UL,
+	/* 179 */ 0xd0dea9dfe4432a4aUL, 0x856af87bbe9277c5UL,
+		  0xce8472acc212c71aUL, 0x6f151b6d9bbb1e91UL,
+	/* 180 */ 0x26776c527ceed56aUL, 0x7d211cb7fbf8faecUL,
+		  0x37ae66a6fd4609ccUL, 0x1f81b702d2770c42UL,
+	/* 181 */ 0x2fb0b057eac58392UL, 0xe1dd89fe29744e9dUL,
+		  0xc964f8eb17beb4f8UL, 0x29571073c9a2d41eUL,
+	/* 182 */ 0xa948a18981c0e254UL, 0x2df6369b65b22830UL,
+		  0xa33eb2d75fcfd3c6UL, 0x078cd6ec4199a01fUL,
+	/* 183 */ 0x4a584a41ad900d2fUL, 0x32142b78e2c74c52UL,
+		  0x68c4e8338431c978UL, 0x7f69ea9008689fc2UL,
+	/* 184 */ 0x52f2c81e46a38265UL, 0xfd78072d04a832fdUL,
+		  0x8cd7d5fa25359e94UL, 0x4de71b7454cc29d2UL,
+	/* 185 */ 0x42eb60ad1eda6ac9UL, 0x0aad37dfdbc09c3aUL,
+		  0x81004b71e33cc191UL, 0x44e6be345122803cUL,
+	/* 186 */ 0x03fe8388ba1920dbUL, 0xf5d57c32150db008UL,
+		  0x49c8c4281af60c29UL, 0x21edb518de701aeeUL,
+	/* 187 */ 0x7fb63e418f06dc99UL, 0xa4460d99c166d7b8UL,
+		  0x24dd5248ce520a83UL, 0x5ec3ad712b928358UL,
+	/* 188 */ 0x15022a5fbd17930fUL, 0xa4f64a77d82570e3UL,
+		  0x12bc8d6915783712UL, 0x498194c0fc620abbUL,
+	/* 189 */ 0x38a2d9d255686c82UL, 0x785c6bd9193e21f0UL,
+		  0xe4d5c81ab24a5484UL, 0x56307860b2e20989UL,
+	/* 190 */ 0x429d55f78b4d74c4UL, 0x22f1834643350131UL,
+		  0x1e60c24598c71fffUL, 0x59f2f014979983efUL,
+	/* 191 */ 0x46a47d56eb494a44UL, 0x3e22a854d636a18eUL,
+		  0xb346e15274491c3bUL, 0x2ceafd4e5390cde7UL,
+	/* 192 */ 0xba8a8538be0d6675UL, 0x4b9074bb50818e23UL,
+		  0xcbdab89085d304c3UL, 0x61a24fe0e56192c4UL,
+	/* 193 */ 0xcb7615e6db525bcbUL, 0xdd7d8c35a567e4caUL,
+		  0xe6b4153acafcdd69UL, 0x2d668e097f3c9766UL,
+	/* 194 */ 0xa57e7e265ce55ef0UL, 0x5d9f4e527cd4b967UL,
+		  0xfbc83606492fd1e5UL, 0x090d52beb7c3f7aeUL,
+	/* 195 */ 0x09b9515a1e7b4d7cUL, 0x1f266a2599da44c0UL,
+		  0xa1c49548e2c55504UL, 0x7ef04287126f15ccUL,
+	/* 196 */ 0xfed1659dbd30ef15UL, 0x8b4ab9eec4e0277bUL,
+		  0x884d6236a5df3291UL, 0x1fd96ea6bf5cf788UL,
+	/* 197 */ 0x42a161981f190d9aUL, 0x61d849507e6052c1UL,
+		  0x9fe113bf285a2cd5UL, 0x7c22d676dbad85d8UL,
+	/* 198 */ 0x82e770ed2bfbd27dUL, 0x4c05b2ece996f5a5UL,
+		  0xcd40a9c2b0900150UL, 0x5895319213d9bf64UL,
+	/* 199 */ 0xe7cc5d703fea2e08UL, 0xb50c491258e2188cUL,
+		  0xcce30baa48205bf0UL, 0x537c659ccfa32d62UL,
+	/* 200 */ 0x37b6623a98cfc088UL, 0xfe9bed1fa4d6aca4UL,
+		  0x04d29b8e56a8d1b0UL, 0x725f71c40b519575UL,
+	/* 201 */ 0x28c7f89cd0339ce6UL, 0x8367b14469ddc18bUL,
+		  0x883ada83a6a1652cUL, 0x585f1974034d6c17UL,
+	/* 202 */ 0x89cfb266f1b19188UL, 0xe63b4863e7c35217UL,
+		  0xd88c9da6b4c0526aUL, 0x3e035c9df0954635UL,
+	/* 203 */ 0xdd9d5412fb45de9dUL, 0xdd684532e4cff40dUL,
+		  0x4b5c999b151d671cUL, 0x2d8c2cc811e7f690UL,
+	/* 204 */ 0x7f54be1d90055d40UL, 0xa464c5df464aaf40UL,
+		  0x33979624f0e917beUL, 0x2c018dc527356b30UL,
+	/* 205 */ 0xa5415024e330b3d4UL, 0x73ff3d96691652d3UL,
+		  0x94ec42c4ef9b59f1UL, 0x0747201618d08e5aUL,
+	/* 206 */ 0x4d6ca48aca411c53UL, 0x66415f2fcfa66119UL,
+		  0x9c4dd40051e227ffUL, 0x59810bc09a02f7ebUL,
+	/* 207 */ 0x2a7eb171b3dc101dUL, 0x441c5ab99ffef68eUL,
+		  0x32025c9b93b359eaUL, 0x5e8ce0a71e9d112fUL,
+	/* 208 */ 0xbfcccb92429503fdUL, 0xd271ba752f095d55UL,
+		  0x345ead5e972d091eUL, 0x18c8df11a83103baUL,
+	/* 209 */ 0x90cd949a9aed0f4cUL, 0xc5d1f4cb6660e37eUL,
+		  0xb8cac52d56c52e0bUL, 0x6e42e400c5808e0dUL,
+	/* 210 */ 0xa3b46966eeaefd23UL, 0x0c4f1f0be39ecdcaUL,
+		  0x189dc8c9d683a51dUL, 0x51f27f054c09351bUL,
+	/* 211 */ 0x4c487ccd2a320682UL, 0x587ea95bb3df1c96UL,
+		  0xc8ccf79e555cb8e8UL, 0x547dc829a206d73dUL,
+	/* 212 */ 0xb822a6cd80c39b06UL, 0xe96d54732000d4c6UL,
+		  0x28535b6f91463b4dUL, 0x228f4660e2486e1dUL,
+	/* 213 */ 0x98799538de8d3abfUL, 0x8cd8330045ebca6eUL,
+		  0x79952a008221e738UL, 0x4322e1a7535cd2bbUL,
+	/* 214 */ 0xb114c11819d1801cUL, 0x2016e4d84f3f5ec7UL,
+		  0xdd0e2df409260f4cUL, 0x5ec362c0ae5f7266UL,
+	/* 215 */ 0xc0462b18b8b2b4eeUL, 0x7cc8d950274d1afbUL,
+		  0xf25f7105436b02d2UL, 0x43bbf8dcbff9ccd3UL,
+	/* 216 */ 0xb6ad1767a039e9dfUL, 0xb0714da8f69d3583UL,
+		  0x5e55fa18b42931f5UL, 0x4ed5558f33c60961UL,
+	/* 217 */ 0x1fe37901c647a5ddUL, 0x593ddf1f8081d357UL,
+		  0x0249a4fd813fd7a6UL, 0x69acca274e9caf61UL,
+	/* 218 */ 0x047ba3ea330721c9UL, 0x83423fc20e7e1ea0UL,
+		  0x1df4c0af01314a60UL, 0x09a62dab89289527UL,
+	/* 219 */ 0xa5b325a49cc6cb00UL, 0xe94b5dc654b56cb6UL,
+		  0x3be28779adc994a0UL, 0x4296e8f8ba3a4aadUL,
+	/* 220 */ 0x328689761e451eabUL, 0x2e4d598bff59594aUL,
+		  0x49b96853d7a7084aUL, 0x4980a319601420a8UL,
+	/* 221 */ 0x9565b9e12f552c42UL, 0x8a5318db7100fe96UL,
+		  0x05c90b4d43add0d7UL, 0x538b4cd66a5d4edaUL,
+	/* 222 */ 0xf4e94fc3e89f039fUL, 0x592c9af26f618045UL,
+		  0x08a36eb5fd4b9550UL, 0x25fffaf6c2ed1419UL,
+	/* 223 */ 0x34434459cc79d354UL, 0xeeecbfb4b1d5476bUL,
+		  0xddeb34a061615d99UL, 0x5129cecceb64b773UL,
+	/* 224 */ 0xee43215894993520UL, 0x772f9c7cf14c0b3bUL,
+		  0xd2e2fce306bedad5UL, 0x715f42b546f06a97UL,
+	/* 225 */ 0x434ecdceda5b5f1aUL, 0x0da17115a49741a9UL,
+		  0x680bd77c73edad2eUL, 0x487c02354edd9041UL,
+	/* 226 */ 0xb8efeff3a70ed9c4UL, 0x56a32aa3e857e302UL,
+		  0xdf3a68bd48a2a5a0UL, 0x07f650b73176c444UL,
+	/* 227 */ 0xe38b9b1626e0ccb1UL, 0x79e053c18b09fb36UL,
+		  0x56d90319c9f94964UL, 0x1ca941e7ac9ff5c4UL,
+	/* 228 */ 0x49c4df29162fa0bbUL, 0x8488cf3282b33305UL,
+		  0x95dfda14cabb437dUL, 0x3391f78264d5ad86UL,
+	/* 229 */ 0x729ae06ae2b5095dUL, 0xd58a58d73259a946UL,
+		  0xe9834262d13921edUL, 0x27fedafaa54bb592UL,
+	/* 230 */ 0xa99dc5b829ad48bbUL, 0x5f025742499ee260UL,
+		  0x802c8ecd5d7513fdUL, 0x78ceb3ef3f6dd938UL,
+	/* 231 */ 0xc342f44f8a135d94UL, 0x7b9edb44828cdda3UL,
+		  0x9436d11a0537cfe7UL, 0x5064b164ec1ab4c8UL,
+	/* 232 */ 0x7020eccfd37eb2fcUL, 0x1f31ea3ed90d25fcUL,
+		  0x1b930d7bdfa1bb34UL, 0x5344467a48113044UL,
+	/* 233 */ 0x70073170f25e6dfbUL, 0xe385dc1a50114cc8UL,
+		  0x2348698ac8fc4f00UL, 0x2a77a55284dd40d8UL,
+	/* 234 */ 0xfe06afe0c98c6ce4UL, 0xc235df96dddfd6e4UL,
+		  0x1428d01e33bf1ed3UL, 0x785768ec9300bdafUL,
+	/* 235 */ 0x9702e57a91deb63bUL, 0x61bdb8bfe5ce8b80UL,
+		  0x645b426f3d1d58acUL, 0x4804a82227a557bcUL,
+	/* 236 */ 0x8e57048ab44d2601UL, 0x68d6501a4b3a6935UL,
+		  0xc39c9ec3f9e1c293UL, 0x4172f257d4de63e2UL,
+	/* 237 */ 0xd368b450330c6401UL, 0x040d3017418f2391UL,
+		  0x2c34bb6090b7d90dUL, 0x16f649228fdfd51fUL,
+	/* 238 */ 0xbea6818e2b928ef5UL, 0xe28ccf91cdc11e72UL,
+		  0x594aaa68e77a36cdUL, 0x313034806c7ffd0fUL,
+	/* 239 */ 0x8a9d27ac2249bd65UL, 0x19a3b464018e9512UL,
+		  0xc26ccff352b37ec7UL, 0x056f68341d797b21UL,
+	/* 240 */ 0x5e79d6757efd2327UL, 0xfabdbcb6553afe15UL,
+		  0xd3e7222c6eaf5a60UL, 0x7046c76d4dae743bUL,
+	/* 241 */ 0x660be872b18d4a55UL, 0x19992518574e1496UL,
+		  0xc103053a302bdcbbUL, 0x3ed8e9800b218e8eUL,
+	/* 242 */ 0x7b0b9239fa75e03eUL, 0xefe9fb684633c083UL,
+		  0x98a35fbe391a7793UL, 0x6065510fe2d0fe34UL,
+	/* 243 */ 0x55cb668548abad0cUL, 0xb4584548da87e527UL,
+		  0x2c43ecea0107c1ddUL, 0x526028809372de35UL,
+	/* 244 */ 0x3415c56af9213b1fUL, 0x5bee1a4d017e98dbUL,
+		  0x13f6b105b5cf709bUL, 0x5ff20e3482b29ab6UL,
+	/* 245 */ 0x0aa29c75cc2e6c90UL, 0xfc7d73ca3a70e206UL,
+		  0x899fc38fc4b5c515UL, 0x250386b124ffc207UL,
+	/* 246 */ 0x54ea28d5ae3d2b56UL, 0x9913149dd6de60ceUL,
+		  0x16694fc58f06d6c1UL, 0x46b23975eb018fc7UL,
+	/* 247 */ 0x470a6a0fb4b7b4e2UL, 0x5d92475a8f7253deUL,
+		  0xabeee5b52fbd3adbUL, 0x7fa20801a0806968UL,
+	/* 248 */ 0x76f3faf19f7714d2UL, 0xb3e840c12f4660c3UL,
+		  0x0fb4cd8df212744eUL, 0x4b065a251d3a2dd2UL,
+	/* 249 */ 0x5cebde383d77cd4aUL, 0x6adf39df882c9cb1UL,
+		  0xa2dd242eb09af759UL, 0x3147c0e50e5f6422UL,
+	/* 250 */ 0x164ca5101d1350dbUL, 0xf8d13479c33fc962UL,
+		  0xe640ce4d13e5da08UL, 0x4bdee0c45061f8baUL,
+	/* 251 */ 0xd7c46dc1a4edb1c9UL, 0x5514d7b6437fd98aUL,
+		  0x58942f6bb2a1c00bUL, 0x2dffb2ab1d70710eUL,
+	/* 252 */ 0xccdfcf2fc18b6d68UL, 0xa8ebcba8b7806167UL,
+		  0x980697f95e2937e3UL, 0x02fbba1cd0126e8cUL
+};
+
+#ifdef CONFIG_AS_ADX
+/* c is two 512-bit products: c0[0:7]=a0[0:3]*b0[0:3] and c1[8:15]=a1[4:7]*b1[4:7]
+ * a is two 256-bit integers: a0[0:3] and a1[4:7]
+ * b is two 256-bit integers: b0[0:3] and b1[4:7]
+ */
+static void mul2_256x256_integer_adx(u64 *const c, const u64 *const a,
+				     const u64 *const b)
+{
+	asm volatile(
+		"xorl %%r14d, %%r14d ;"
+		"movq   (%1), %%rdx; "	/* A[0] */
+		"mulx   (%2),  %%r8, %%r15; " /* A[0]*B[0] */
+		"xorl %%r10d, %%r10d ;"
+		"movq %%r8, (%0) ;"
+		"mulx  8(%2), %%r10, %%rax; " /* A[0]*B[1] */
+		"adox %%r10, %%r15 ;"
+		"mulx 16(%2),  %%r8, %%rbx; " /* A[0]*B[2] */
+		"adox  %%r8, %%rax ;"
+		"mulx 24(%2), %%r10, %%rcx; " /* A[0]*B[3] */
+		"adox %%r10, %%rbx ;"
+		/******************************************/
+		"adox %%r14, %%rcx ;"
+
+		"movq  8(%1), %%rdx; "	/* A[1] */
+		"mulx   (%2),  %%r8,  %%r9; " /* A[1]*B[0] */
+		"adox %%r15,  %%r8 ;"
+		"movq  %%r8, 8(%0) ;"
+		"mulx  8(%2), %%r10, %%r11; " /* A[1]*B[1] */
+		"adox %%r10,  %%r9 ;"
+		"adcx  %%r9, %%rax ;"
+		"mulx 16(%2),  %%r8, %%r13; " /* A[1]*B[2] */
+		"adox  %%r8, %%r11 ;"
+		"adcx %%r11, %%rbx ;"
+		"mulx 24(%2), %%r10, %%r15; " /* A[1]*B[3] */
+		"adox %%r10, %%r13 ;"
+		"adcx %%r13, %%rcx ;"
+		/******************************************/
+		"adox %%r14, %%r15 ;"
+		"adcx %%r14, %%r15 ;"
+
+		"movq 16(%1), %%rdx; " /* A[2] */
+		"xorl %%r10d, %%r10d ;"
+		"mulx   (%2),  %%r8,  %%r9; " /* A[2]*B[0] */
+		"adox %%rax,  %%r8 ;"
+		"movq %%r8, 16(%0) ;"
+		"mulx  8(%2), %%r10, %%r11; " /* A[2]*B[1] */
+		"adox %%r10,  %%r9 ;"
+		"adcx  %%r9, %%rbx ;"
+		"mulx 16(%2),  %%r8, %%r13; " /* A[2]*B[2] */
+		"adox  %%r8, %%r11 ;"
+		"adcx %%r11, %%rcx ;"
+		"mulx 24(%2), %%r10, %%rax; " /* A[2]*B[3] */
+		"adox %%r10, %%r13 ;"
+		"adcx %%r13, %%r15 ;"
+		/******************************************/
+		"adox %%r14, %%rax ;"
+		"adcx %%r14, %%rax ;"
+
+		"movq 24(%1), %%rdx; " /* A[3] */
+		"xorl %%r10d, %%r10d ;"
+		"mulx   (%2),  %%r8,  %%r9; " /* A[3]*B[0] */
+		"adox %%rbx,  %%r8 ;"
+		"movq %%r8, 24(%0) ;"
+		"mulx  8(%2), %%r10, %%r11; " /* A[3]*B[1] */
+		"adox %%r10,  %%r9 ;"
+		"adcx  %%r9, %%rcx ;"
+		"movq %%rcx, 32(%0) ;"
+		"mulx 16(%2),  %%r8, %%r13; " /* A[3]*B[2] */
+		"adox  %%r8, %%r11 ;"
+		"adcx %%r11, %%r15 ;"
+		"movq %%r15, 40(%0) ;"
+		"mulx 24(%2), %%r10, %%rbx; " /* A[3]*B[3] */
+		"adox %%r10, %%r13 ;"
+		"adcx %%r13, %%rax ;"
+		"movq %%rax, 48(%0) ;"
+		/******************************************/
+		"adox %%r14, %%rbx ;"
+		"adcx %%r14, %%rbx ;"
+		"movq %%rbx, 56(%0) ;"
+
+		"movq 32(%1), %%rdx; "	/* C[0] */
+		"mulx 32(%2),  %%r8, %%r15; " /* C[0]*D[0] */
+		"xorl %%r10d, %%r10d ;"
+		"movq %%r8, 64(%0);"
+		"mulx 40(%2), %%r10, %%rax; " /* C[0]*D[1] */
+		"adox %%r10, %%r15 ;"
+		"mulx 48(%2),  %%r8, %%rbx; " /* C[0]*D[2] */
+		"adox  %%r8, %%rax ;"
+		"mulx 56(%2), %%r10, %%rcx; " /* C[0]*D[3] */
+		"adox %%r10, %%rbx ;"
+		/******************************************/
+		"adox %%r14, %%rcx ;"
+
+		"movq 40(%1), %%rdx; " /* C[1] */
+		"xorl %%r10d, %%r10d ;"
+		"mulx 32(%2),  %%r8,  %%r9; " /* C[1]*D[0] */
+		"adox %%r15,  %%r8 ;"
+		"movq  %%r8, 72(%0);"
+		"mulx 40(%2), %%r10, %%r11; " /* C[1]*D[1] */
+		"adox %%r10,  %%r9 ;"
+		"adcx  %%r9, %%rax ;"
+		"mulx 48(%2),  %%r8, %%r13; " /* C[1]*D[2] */
+		"adox  %%r8, %%r11 ;"
+		"adcx %%r11, %%rbx ;"
+		"mulx 56(%2), %%r10, %%r15; " /* C[1]*D[3] */
+		"adox %%r10, %%r13 ;"
+		"adcx %%r13, %%rcx ;"
+		/******************************************/
+		"adox %%r14, %%r15 ;"
+		"adcx %%r14, %%r15 ;"
+
+		"movq 48(%1), %%rdx; " /* C[2] */
+		"xorl %%r10d, %%r10d ;"
+		"mulx 32(%2),  %%r8,  %%r9; " /* C[2]*D[0] */
+		"adox %%rax,  %%r8 ;"
+		"movq  %%r8, 80(%0);"
+		"mulx 40(%2), %%r10, %%r11; " /* C[2]*D[1] */
+		"adox %%r10,  %%r9 ;"
+		"adcx  %%r9, %%rbx ;"
+		"mulx 48(%2),  %%r8, %%r13; " /* C[2]*D[2] */
+		"adox  %%r8, %%r11 ;"
+		"adcx %%r11, %%rcx ;"
+		"mulx 56(%2), %%r10, %%rax; " /* C[2]*D[3] */
+		"adox %%r10, %%r13 ;"
+		"adcx %%r13, %%r15 ;"
+		/******************************************/
+		"adox %%r14, %%rax ;"
+		"adcx %%r14, %%rax ;"
+
+		"movq 56(%1), %%rdx; " /* C[3] */
+		"xorl %%r10d, %%r10d ;"
+		"mulx 32(%2),  %%r8,  %%r9; " /* C[3]*D[0] */
+		"adox %%rbx,  %%r8 ;"
+		"movq  %%r8, 88(%0);"
+		"mulx 40(%2), %%r10, %%r11; " /* C[3]*D[1] */
+		"adox %%r10,  %%r9 ;"
+		"adcx  %%r9, %%rcx ;"
+		"movq %%rcx,  96(%0) ;"
+		"mulx 48(%2),  %%r8, %%r13; " /* C[3]*D[2] */
+		"adox  %%r8, %%r11 ;"
+		"adcx %%r11, %%r15 ;"
+		"movq %%r15, 104(%0) ;"
+		"mulx 56(%2), %%r10, %%rbx; " /* C[3]*D[3] */
+		"adox %%r10, %%r13 ;"
+		"adcx %%r13, %%rax ;"
+		"movq %%rax, 112(%0) ;"
+		/******************************************/
+		"adox %%r14, %%rbx ;"
+		"adcx %%r14, %%rbx ;"
+		"movq %%rbx, 120(%0) ;"
+		:
+		: "r"(c), "r"(a), "r"(b)
+		: "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9",
+		  "%r10", "%r11", "%r13", "%r14", "%r15");
+}
+#endif
+
+#ifdef CONFIG_AS_BMI2
+static void mul2_256x256_integer_bmi2(u64 *const c, const u64 *const a,
+				      const u64 *const b)
+{
+	asm volatile(
+		"movq   (%1), %%rdx; "	/* A[0] */
+		"mulx   (%2),  %%r8, %%r15; " /* A[0]*B[0] */
+		"movq %%r8,  (%0) ;"
+		"mulx  8(%2), %%r10, %%rax; " /* A[0]*B[1] */
+		"addq %%r10, %%r15 ;"
+		"mulx 16(%2),  %%r8, %%rbx; " /* A[0]*B[2] */
+		"adcq  %%r8, %%rax ;"
+		"mulx 24(%2), %%r10, %%rcx; " /* A[0]*B[3] */
+		"adcq %%r10, %%rbx ;"
+		/******************************************/
+		"adcq    $0, %%rcx ;"
+
+		"movq  8(%1), %%rdx; "	/* A[1] */
+		"mulx   (%2),  %%r8,  %%r9; " /* A[1]*B[0] */
+		"addq %%r15,  %%r8 ;"
+		"movq %%r8, 8(%0) ;"
+		"mulx  8(%2), %%r10, %%r11; " /* A[1]*B[1] */
+		"adcq %%r10,  %%r9 ;"
+		"mulx 16(%2),  %%r8, %%r13; " /* A[1]*B[2] */
+		"adcq  %%r8, %%r11 ;"
+		"mulx 24(%2), %%r10, %%r15; " /* A[1]*B[3] */
+		"adcq %%r10, %%r13 ;"
+		/******************************************/
+		"adcq    $0, %%r15 ;"
+
+		"addq  %%r9, %%rax ;"
+		"adcq %%r11, %%rbx ;"
+		"adcq %%r13, %%rcx ;"
+		"adcq    $0, %%r15 ;"
+
+		"movq 16(%1), %%rdx; "	/* A[2] */
+		"mulx   (%2),  %%r8,  %%r9; " /* A[2]*B[0] */
+		"addq %%rax,  %%r8 ;"
+		"movq %%r8, 16(%0) ;"
+		"mulx  8(%2), %%r10, %%r11; " /* A[2]*B[1] */
+		"adcq %%r10,  %%r9 ;"
+		"mulx 16(%2),  %%r8, %%r13; " /* A[2]*B[2] */
+		"adcq  %%r8, %%r11 ;"
+		"mulx 24(%2), %%r10, %%rax; " /* A[2]*B[3] */
+		"adcq %%r10, %%r13 ;"
+		/******************************************/
+		"adcq    $0, %%rax ;"
+
+		"addq  %%r9, %%rbx ;"
+		"adcq %%r11, %%rcx ;"
+		"adcq %%r13, %%r15 ;"
+		"adcq    $0, %%rax ;"
+
+		"movq 24(%1), %%rdx; "	/* A[3] */
+		"mulx   (%2),  %%r8,  %%r9; " /* A[3]*B[0] */
+		"addq %%rbx,  %%r8 ;"
+		"movq %%r8, 24(%0) ;"
+		"mulx  8(%2), %%r10, %%r11; " /* A[3]*B[1] */
+		"adcq %%r10,  %%r9 ;"
+		"mulx 16(%2),  %%r8, %%r13; " /* A[3]*B[2] */
+		"adcq  %%r8, %%r11 ;"
+		"mulx 24(%2), %%r10, %%rbx; " /* A[3]*B[3] */
+		"adcq %%r10, %%r13 ;"
+		/******************************************/
+		"adcq    $0, %%rbx ;"
+
+		"addq  %%r9, %%rcx ;"
+		"movq %%rcx, 32(%0) ;"
+		"adcq %%r11, %%r15 ;"
+		"movq %%r15, 40(%0) ;"
+		"adcq %%r13, %%rax ;"
+		"movq %%rax, 48(%0) ;"
+		"adcq    $0, %%rbx ;"
+		"movq %%rbx, 56(%0) ;"
+
+		"movq 32(%1), %%rdx; "	/* C[0] */
+		"mulx 32(%2),  %%r8, %%r15; " /* C[0]*D[0] */
+		"movq %%r8, 64(%0) ;"
+		"mulx 40(%2), %%r10, %%rax; " /* C[0]*D[1] */
+		"addq %%r10, %%r15 ;"
+		"mulx 48(%2),  %%r8, %%rbx; " /* C[0]*D[2] */
+		"adcq  %%r8, %%rax ;"
+		"mulx 56(%2), %%r10, %%rcx; " /* C[0]*D[3] */
+		"adcq %%r10, %%rbx ;"
+		/******************************************/
+		"adcq    $0, %%rcx ;"
+
+		"movq 40(%1), %%rdx; "	/* C[1] */
+		"mulx 32(%2),  %%r8,  %%r9; " /* C[1]*D[0] */
+		"addq %%r15,  %%r8 ;"
+		"movq %%r8, 72(%0) ;"
+		"mulx 40(%2), %%r10, %%r11; " /* C[1]*D[1] */
+		"adcq %%r10,  %%r9 ;"
+		"mulx 48(%2),  %%r8, %%r13; " /* C[1]*D[2] */
+		"adcq  %%r8, %%r11 ;"
+		"mulx 56(%2), %%r10, %%r15; " /* C[1]*D[3] */
+		"adcq %%r10, %%r13 ;"
+		/******************************************/
+		"adcq    $0, %%r15 ;"
+
+		"addq  %%r9, %%rax ;"
+		"adcq %%r11, %%rbx ;"
+		"adcq %%r13, %%rcx ;"
+		"adcq    $0, %%r15 ;"
+
+		"movq 48(%1), %%rdx; "	/* C[2] */
+		"mulx 32(%2),  %%r8,  %%r9; " /* C[2]*D[0] */
+		"addq %%rax,  %%r8 ;"
+		"movq %%r8, 80(%0) ;"
+		"mulx 40(%2), %%r10, %%r11; " /* C[2]*D[1] */
+		"adcq %%r10,  %%r9 ;"
+		"mulx 48(%2),  %%r8, %%r13; " /* C[2]*D[2] */
+		"adcq  %%r8, %%r11 ;"
+		"mulx 56(%2), %%r10, %%rax; " /* C[2]*D[3] */
+		"adcq %%r10, %%r13 ;"
+		/******************************************/
+		"adcq    $0, %%rax ;"
+
+		"addq  %%r9, %%rbx ;"
+		"adcq %%r11, %%rcx ;"
+		"adcq %%r13, %%r15 ;"
+		"adcq    $0, %%rax ;"
+
+		"movq 56(%1), %%rdx; "	/* C[3] */
+		"mulx 32(%2),  %%r8,  %%r9; " /* C[3]*D[0] */
+		"addq %%rbx,  %%r8 ;"
+		"movq %%r8, 88(%0) ;"
+		"mulx 40(%2), %%r10, %%r11; " /* C[3]*D[1] */
+		"adcq %%r10,  %%r9 ;"
+		"mulx 48(%2),  %%r8, %%r13; " /* C[3]*D[2] */
+		"adcq  %%r8, %%r11 ;"
+		"mulx 56(%2), %%r10, %%rbx; " /* C[3]*D[3] */
+		"adcq %%r10, %%r13 ;"
+		/******************************************/
+		"adcq    $0, %%rbx ;"
+
+		"addq  %%r9, %%rcx ;"
+		"movq %%rcx,  96(%0) ;"
+		"adcq %%r11, %%r15 ;"
+		"movq %%r15, 104(%0) ;"
+		"adcq %%r13, %%rax ;"
+		"movq %%rax, 112(%0) ;"
+		"adcq    $0, %%rbx ;"
+		"movq %%rbx, 120(%0) ;"
+		:
+		: "r"(c), "r"(a), "r"(b)
+		: "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9",
+		  "%r10", "%r11", "%r13", "%r15");
+}
+#endif
+
+#ifdef CONFIG_AS_ADX
+static void sqr2_256x256_integer_adx(u64 *const c, const u64 *const a)
+{
+	asm volatile(
+		"movq   (%1), %%rdx        ;" /* A[0]      */
+		"mulx  8(%1),  %%r8, %%r14 ;" /* A[1]*A[0] */
+		"xorl %%r15d, %%r15d;"
+		"mulx 16(%1),  %%r9, %%r10 ;" /* A[2]*A[0] */
+		"adcx %%r14,  %%r9 ;"
+		"mulx 24(%1), %%rax, %%rcx ;" /* A[3]*A[0] */
+		"adcx %%rax, %%r10 ;"
+		"movq 24(%1), %%rdx        ;" /* A[3]      */
+		"mulx  8(%1), %%r11, %%rbx ;" /* A[1]*A[3] */
+		"adcx %%rcx, %%r11 ;"
+		"mulx 16(%1), %%rax, %%r13 ;" /* A[2]*A[3] */
+		"adcx %%rax, %%rbx ;"
+		"movq  8(%1), %%rdx        ;" /* A[1]      */
+		"adcx %%r15, %%r13 ;"
+		"mulx 16(%1), %%rax, %%rcx ;" /* A[2]*A[1] */
+		"movq    $0, %%r14 ;"
+		/******************************************/
+		"adcx %%r15, %%r14 ;"
+
+		"xorl %%r15d, %%r15d;"
+		"adox %%rax, %%r10 ;"
+		"adcx  %%r8,  %%r8 ;"
+		"adox %%rcx, %%r11 ;"
+		"adcx  %%r9,  %%r9 ;"
+		"adox %%r15, %%rbx ;"
+		"adcx %%r10, %%r10 ;"
+		"adox %%r15, %%r13 ;"
+		"adcx %%r11, %%r11 ;"
+		"adox %%r15, %%r14 ;"
+		"adcx %%rbx, %%rbx ;"
+		"adcx %%r13, %%r13 ;"
+		"adcx %%r14, %%r14 ;"
+
+		"movq   (%1), %%rdx ;"
+		"mulx %%rdx, %%rax, %%rcx ;" /* A[0]^2 */
+		/*******************/
+		"movq %%rax,  0(%0) ;"
+		"addq %%rcx,  %%r8 ;"
+		"movq  %%r8,  8(%0) ;"
+		"movq  8(%1), %%rdx ;"
+		"mulx %%rdx, %%rax, %%rcx ;" /* A[1]^2 */
+		"adcq %%rax,  %%r9 ;"
+		"movq  %%r9, 16(%0) ;"
+		"adcq %%rcx, %%r10 ;"
+		"movq %%r10, 24(%0) ;"
+		"movq 16(%1), %%rdx ;"
+		"mulx %%rdx, %%rax, %%rcx ;" /* A[2]^2 */
+		"adcq %%rax, %%r11 ;"
+		"movq %%r11, 32(%0) ;"
+		"adcq %%rcx, %%rbx ;"
+		"movq %%rbx, 40(%0) ;"
+		"movq 24(%1), %%rdx ;"
+		"mulx %%rdx, %%rax, %%rcx ;" /* A[3]^2 */
+		"adcq %%rax, %%r13 ;"
+		"movq %%r13, 48(%0) ;"
+		"adcq %%rcx, %%r14 ;"
+		"movq %%r14, 56(%0) ;"
+
+
+		"movq 32(%1), %%rdx        ;" /* B[0]      */
+		"mulx 40(%1),  %%r8, %%r14 ;" /* B[1]*B[0] */
+		"xorl %%r15d, %%r15d;"
+		"mulx 48(%1),  %%r9, %%r10 ;" /* B[2]*B[0] */
+		"adcx %%r14,  %%r9 ;"
+		"mulx 56(%1), %%rax, %%rcx ;" /* B[3]*B[0] */
+		"adcx %%rax, %%r10 ;"
+		"movq 56(%1), %%rdx        ;" /* B[3]      */
+		"mulx 40(%1), %%r11, %%rbx ;" /* B[1]*B[3] */
+		"adcx %%rcx, %%r11 ;"
+		"mulx 48(%1), %%rax, %%r13 ;" /* B[2]*B[3] */
+		"adcx %%rax, %%rbx ;"
+		"movq 40(%1), %%rdx        ;" /* B[1]      */
+		"adcx %%r15, %%r13 ;"
+		"mulx 48(%1), %%rax, %%rcx ;" /* B[2]*B[1] */
+		"movq    $0, %%r14 ;"
+		/******************************************/
+		"adcx %%r15, %%r14 ;"
+
+		"xorl %%r15d, %%r15d;"
+		"adox %%rax, %%r10 ;"
+		"adcx  %%r8,  %%r8 ;"
+		"adox %%rcx, %%r11 ;"
+		"adcx  %%r9,  %%r9 ;"
+		"adox %%r15, %%rbx ;"
+		"adcx %%r10, %%r10 ;"
+		"adox %%r15, %%r13 ;"
+		"adcx %%r11, %%r11 ;"
+		"adox %%r15, %%r14 ;"
+		"adcx %%rbx, %%rbx ;"
+		"adcx %%r13, %%r13 ;"
+		"adcx %%r14, %%r14 ;"
+
+		"movq 32(%1), %%rdx ;"
+		"mulx %%rdx, %%rax, %%rcx ;" /* B[0]^2 */
+		/*******************/
+		"movq %%rax,  64(%0) ;"
+		"addq %%rcx,  %%r8 ;"
+		"movq  %%r8,  72(%0) ;"
+		"movq 40(%1), %%rdx ;"
+		"mulx %%rdx, %%rax, %%rcx ;" /* B[1]^2 */
+		"adcq %%rax,  %%r9 ;"
+		"movq  %%r9,  80(%0) ;"
+		"adcq %%rcx, %%r10 ;"
+		"movq %%r10,  88(%0) ;"
+		"movq 48(%1), %%rdx ;"
+		"mulx %%rdx, %%rax, %%rcx ;" /* B[2]^2 */
+		"adcq %%rax, %%r11 ;"
+		"movq %%r11,  96(%0) ;"
+		"adcq %%rcx, %%rbx ;"
+		"movq %%rbx, 104(%0) ;"
+		"movq 56(%1), %%rdx ;"
+		"mulx %%rdx, %%rax, %%rcx ;" /* B[3]^2 */
+		"adcq %%rax, %%r13 ;"
+		"movq %%r13, 112(%0) ;"
+		"adcq %%rcx, %%r14 ;"
+		"movq %%r14, 120(%0) ;"
+		:
+		: "r"(c), "r"(a)
+		: "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9",
+		  "%r10", "%r11", "%r13", "%r14", "%r15");
+}
+#endif
+
+#ifdef CONFIG_AS_BMI2
+static void sqr2_256x256_integer_bmi2(u64 *const c, const u64 *const a)
+{
+	asm volatile(
+		"movq  8(%1), %%rdx        ;" /* A[1]      */
+		"mulx   (%1),  %%r8,  %%r9 ;" /* A[0]*A[1] */
+		"mulx 16(%1), %%r10, %%r11 ;" /* A[2]*A[1] */
+		"mulx 24(%1), %%rcx, %%r14 ;" /* A[3]*A[1] */
+
+		"movq 16(%1), %%rdx        ;" /* A[2]      */
+		"mulx 24(%1), %%r15, %%r13 ;" /* A[3]*A[2] */
+		"mulx   (%1), %%rax, %%rdx ;" /* A[0]*A[2] */
+
+		"addq %%rax,  %%r9 ;"
+		"adcq %%rdx, %%r10 ;"
+		"adcq %%rcx, %%r11 ;"
+		"adcq %%r14, %%r15 ;"
+		"adcq    $0, %%r13 ;"
+		"movq    $0, %%r14 ;"
+		"adcq    $0, %%r14 ;"
+
+		"movq   (%1), %%rdx        ;" /* A[0]      */
+		"mulx 24(%1), %%rax, %%rcx ;" /* A[0]*A[3] */
+
+		"addq %%rax, %%r10 ;"
+		"adcq %%rcx, %%r11 ;"
+		"adcq    $0, %%r15 ;"
+		"adcq    $0, %%r13 ;"
+		"adcq    $0, %%r14 ;"
+
+		"shldq $1, %%r13, %%r14 ;"
+		"shldq $1, %%r15, %%r13 ;"
+		"shldq $1, %%r11, %%r15 ;"
+		"shldq $1, %%r10, %%r11 ;"
+		"shldq $1,  %%r9, %%r10 ;"
+		"shldq $1,  %%r8,  %%r9 ;"
+		"shlq  $1,  %%r8        ;"
+
+		/*******************/
+		"mulx %%rdx, %%rax, %%rcx ; " /* A[0]^2 */
+		/*******************/
+		"movq %%rax,  0(%0) ;"
+		"addq %%rcx,  %%r8 ;"
+		"movq  %%r8,  8(%0) ;"
+		"movq  8(%1), %%rdx ;"
+		"mulx %%rdx, %%rax, %%rcx ; " /* A[1]^2 */
+		"adcq %%rax,  %%r9 ;"
+		"movq  %%r9, 16(%0) ;"
+		"adcq %%rcx, %%r10 ;"
+		"movq %%r10, 24(%0) ;"
+		"movq 16(%1), %%rdx ;"
+		"mulx %%rdx, %%rax, %%rcx ; " /* A[2]^2 */
+		"adcq %%rax, %%r11 ;"
+		"movq %%r11, 32(%0) ;"
+		"adcq %%rcx, %%r15 ;"
+		"movq %%r15, 40(%0) ;"
+		"movq 24(%1), %%rdx ;"
+		"mulx %%rdx, %%rax, %%rcx ; " /* A[3]^2 */
+		"adcq %%rax, %%r13 ;"
+		"movq %%r13, 48(%0) ;"
+		"adcq %%rcx, %%r14 ;"
+		"movq %%r14, 56(%0) ;"
+
+		"movq 40(%1), %%rdx        ;" /* B[1]      */
+		"mulx 32(%1),  %%r8,  %%r9 ;" /* B[0]*B[1] */
+		"mulx 48(%1), %%r10, %%r11 ;" /* B[2]*B[1] */
+		"mulx 56(%1), %%rcx, %%r14 ;" /* B[3]*B[1] */
+
+		"movq 48(%1), %%rdx        ;" /* B[2]      */
+		"mulx 56(%1), %%r15, %%r13 ;" /* B[3]*B[2] */
+		"mulx 32(%1), %%rax, %%rdx ;" /* B[0]*B[2] */
+
+		"addq %%rax,  %%r9 ;"
+		"adcq %%rdx, %%r10 ;"
+		"adcq %%rcx, %%r11 ;"
+		"adcq %%r14, %%r15 ;"
+		"adcq    $0, %%r13 ;"
+		"movq    $0, %%r14 ;"
+		"adcq    $0, %%r14 ;"
+
+		"movq 32(%1), %%rdx        ;" /* B[0]      */
+		"mulx 56(%1), %%rax, %%rcx ;" /* B[0]*B[3] */
+
+		"addq %%rax, %%r10 ;"
+		"adcq %%rcx, %%r11 ;"
+		"adcq    $0, %%r15 ;"
+		"adcq    $0, %%r13 ;"
+		"adcq    $0, %%r14 ;"
+
+		"shldq $1, %%r13, %%r14 ;"
+		"shldq $1, %%r15, %%r13 ;"
+		"shldq $1, %%r11, %%r15 ;"
+		"shldq $1, %%r10, %%r11 ;"
+		"shldq $1,  %%r9, %%r10 ;"
+		"shldq $1,  %%r8,  %%r9 ;"
+		"shlq  $1,  %%r8        ;"
+
+		/*******************/
+		"mulx %%rdx, %%rax, %%rcx ; " /* B[0]^2 */
+		/*******************/
+		"movq %%rax,  64(%0) ;"
+		"addq %%rcx,  %%r8 ;"
+		"movq  %%r8,  72(%0) ;"
+		"movq 40(%1), %%rdx ;"
+		"mulx %%rdx, %%rax, %%rcx ; " /* B[1]^2 */
+		"adcq %%rax,  %%r9 ;"
+		"movq  %%r9,  80(%0) ;"
+		"adcq %%rcx, %%r10 ;"
+		"movq %%r10,  88(%0) ;"
+		"movq 48(%1), %%rdx ;"
+		"mulx %%rdx, %%rax, %%rcx ; " /* B[2]^2 */
+		"adcq %%rax, %%r11 ;"
+		"movq %%r11,  96(%0) ;"
+		"adcq %%rcx, %%r15 ;"
+		"movq %%r15, 104(%0) ;"
+		"movq 56(%1), %%rdx ;"
+		"mulx %%rdx, %%rax, %%rcx ; " /* B[3]^2 */
+		"adcq %%rax, %%r13 ;"
+		"movq %%r13, 112(%0) ;"
+		"adcq %%rcx, %%r14 ;"
+		"movq %%r14, 120(%0) ;"
+		:
+		: "r"(c), "r"(a)
+		: "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10",
+		  "%r11", "%r13", "%r14", "%r15");
+}
+#endif
+
+#ifdef CONFIG_AS_ADX
+static void red_eltfp25519_2w_adx(u64 *const c, const u64 *const a)
+{
+	asm volatile(
+		"movl    $38, %%edx; "	/* 2*c = 38 = 2^256 */
+		"mulx 32(%1),  %%r8, %%r10; " /* c*C[4] */
+		"xorl %%ebx, %%ebx ;"
+		"adox   (%1),  %%r8 ;"
+		"mulx 40(%1),  %%r9, %%r11; " /* c*C[5] */
+		"adcx %%r10,  %%r9 ;"
+		"adox  8(%1),  %%r9 ;"
+		"mulx 48(%1), %%r10, %%rax; " /* c*C[6] */
+		"adcx %%r11, %%r10 ;"
+		"adox 16(%1), %%r10 ;"
+		"mulx 56(%1), %%r11, %%rcx; " /* c*C[7] */
+		"adcx %%rax, %%r11 ;"
+		"adox 24(%1), %%r11 ;"
+		/***************************************/
+		"adcx %%rbx, %%rcx ;"
+		"adox  %%rbx, %%rcx ;"
+		"imul %%rdx, %%rcx ;" /* c*C[4], cf=0, of=0 */
+		"adcx %%rcx,  %%r8 ;"
+		"adcx %%rbx,  %%r9 ;"
+		"movq  %%r9,  8(%0) ;"
+		"adcx %%rbx, %%r10 ;"
+		"movq %%r10, 16(%0) ;"
+		"adcx %%rbx, %%r11 ;"
+		"movq %%r11, 24(%0) ;"
+		"mov     $0, %%ecx ;"
+		"cmovc %%edx, %%ecx ;"
+		"addq %%rcx,  %%r8 ;"
+		"movq  %%r8,   (%0) ;"
+
+		"mulx  96(%1),  %%r8, %%r10; " /* c*C[4] */
+		"xorl %%ebx, %%ebx ;"
+		"adox 64(%1),  %%r8 ;"
+		"mulx 104(%1),  %%r9, %%r11; " /* c*C[5] */
+		"adcx %%r10,  %%r9 ;"
+		"adox 72(%1),  %%r9 ;"
+		"mulx 112(%1), %%r10, %%rax; " /* c*C[6] */
+		"adcx %%r11, %%r10 ;"
+		"adox 80(%1), %%r10 ;"
+		"mulx 120(%1), %%r11, %%rcx; " /* c*C[7] */
+		"adcx %%rax, %%r11 ;"
+		"adox 88(%1), %%r11 ;"
+		/****************************************/
+		"adcx %%rbx, %%rcx ;"
+		"adox  %%rbx, %%rcx ;"
+		"imul %%rdx, %%rcx ;" /* c*C[4], cf=0, of=0 */
+		"adcx %%rcx,  %%r8 ;"
+		"adcx %%rbx,  %%r9 ;"
+		"movq  %%r9, 40(%0) ;"
+		"adcx %%rbx, %%r10 ;"
+		"movq %%r10, 48(%0) ;"
+		"adcx %%rbx, %%r11 ;"
+		"movq %%r11, 56(%0) ;"
+		"mov     $0, %%ecx ;"
+		"cmovc %%edx, %%ecx ;"
+		"addq %%rcx,  %%r8 ;"
+		"movq  %%r8, 32(%0) ;"
+		:
+		: "r"(c), "r"(a)
+		: "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9",
+		  "%r10", "%r11");
+}
+#endif
+
+#ifdef CONFIG_AS_BMI2
+static void red_eltfp25519_2w_bmi2(u64 *const c, const u64 *const a)
+{
+	asm volatile(
+		"movl    $38, %%edx ; "       /* 2*c = 38 = 2^256 */
+		"mulx 32(%1),  %%r8, %%r10 ;" /* c*C[4] */
+		"mulx 40(%1),  %%r9, %%r11 ;" /* c*C[5] */
+		"addq %%r10,  %%r9 ;"
+		"mulx 48(%1), %%r10, %%rax ;" /* c*C[6] */
+		"adcq %%r11, %%r10 ;"
+		"mulx 56(%1), %%r11, %%rcx ;" /* c*C[7] */
+		"adcq %%rax, %%r11 ;"
+		/***************************************/
+		"adcq    $0, %%rcx ;"
+		"addq   (%1),  %%r8 ;"
+		"adcq  8(%1),  %%r9 ;"
+		"adcq 16(%1), %%r10 ;"
+		"adcq 24(%1), %%r11 ;"
+		"adcq     $0, %%rcx ;"
+		"imul %%rdx, %%rcx ;" /* c*C[4], cf=0 */
+		"addq %%rcx,  %%r8 ;"
+		"adcq    $0,  %%r9 ;"
+		"movq  %%r9,  8(%0) ;"
+		"adcq    $0, %%r10 ;"
+		"movq %%r10, 16(%0) ;"
+		"adcq    $0, %%r11 ;"
+		"movq %%r11, 24(%0) ;"
+		"mov     $0, %%ecx ;"
+		"cmovc %%edx, %%ecx ;"
+		"addq %%rcx,  %%r8 ;"
+		"movq  %%r8,   (%0) ;"
+
+		"mulx  96(%1),  %%r8, %%r10 ;" /* c*C[4] */
+		"mulx 104(%1),  %%r9, %%r11 ;" /* c*C[5] */
+		"addq %%r10,  %%r9 ;"
+		"mulx 112(%1), %%r10, %%rax ;" /* c*C[6] */
+		"adcq %%r11, %%r10 ;"
+		"mulx 120(%1), %%r11, %%rcx ;" /* c*C[7] */
+		"adcq %%rax, %%r11 ;"
+		/****************************************/
+		"adcq    $0, %%rcx ;"
+		"addq 64(%1),  %%r8 ;"
+		"adcq 72(%1),  %%r9 ;"
+		"adcq 80(%1), %%r10 ;"
+		"adcq 88(%1), %%r11 ;"
+		"adcq     $0, %%rcx ;"
+		"imul %%rdx, %%rcx ;" /* c*C[4], cf=0 */
+		"addq %%rcx,  %%r8 ;"
+		"adcq    $0,  %%r9 ;"
+		"movq  %%r9, 40(%0) ;"
+		"adcq    $0, %%r10 ;"
+		"movq %%r10, 48(%0) ;"
+		"adcq    $0, %%r11 ;"
+		"movq %%r11, 56(%0) ;"
+		"mov     $0, %%ecx ;"
+		"cmovc %%edx, %%ecx ;"
+		"addq %%rcx,  %%r8 ;"
+		"movq  %%r8, 32(%0) ;"
+		:
+		: "r"(c), "r"(a)
+		: "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10",
+		  "%r11");
+}
+#endif
+
+#ifdef CONFIG_AS_ADX
+static void mul_256x256_integer_adx(u64 *const c, const u64 *const a,
+				    const u64 *const b)
+{
+	asm volatile(
+		"movq   (%1), %%rdx; "	/* A[0] */
+		"mulx   (%2),  %%r8,  %%r9; " /* A[0]*B[0] */
+		"xorl %%r10d, %%r10d ;"
+		"movq  %%r8,  (%0) ;"
+		"mulx  8(%2), %%r10, %%r11; " /* A[0]*B[1] */
+		"adox  %%r9, %%r10 ;"
+		"movq %%r10, 8(%0) ;"
+		"mulx 16(%2), %%r15, %%r13; " /* A[0]*B[2] */
+		"adox %%r11, %%r15 ;"
+		"mulx 24(%2), %%r14, %%rdx; " /* A[0]*B[3] */
+		"adox %%r13, %%r14 ;"
+		"movq $0, %%rax ;"
+		/******************************************/
+		"adox %%rdx, %%rax ;"
+
+		"movq  8(%1), %%rdx; "	/* A[1] */
+		"mulx   (%2),  %%r8,  %%r9; " /* A[1]*B[0] */
+		"xorl %%r10d, %%r10d ;"
+		"adcx 8(%0),  %%r8 ;"
+		"movq  %%r8,  8(%0) ;"
+		"mulx  8(%2), %%r10, %%r11; " /* A[1]*B[1] */
+		"adox  %%r9, %%r10 ;"
+		"adcx %%r15, %%r10 ;"
+		"movq %%r10, 16(%0) ;"
+		"mulx 16(%2), %%r15, %%r13; " /* A[1]*B[2] */
+		"adox %%r11, %%r15 ;"
+		"adcx %%r14, %%r15 ;"
+		"movq $0, %%r8  ;"
+		"mulx 24(%2), %%r14, %%rdx; " /* A[1]*B[3] */
+		"adox %%r13, %%r14 ;"
+		"adcx %%rax, %%r14 ;"
+		"movq $0, %%rax ;"
+		/******************************************/
+		"adox %%rdx, %%rax ;"
+		"adcx  %%r8, %%rax ;"
+
+		"movq 16(%1), %%rdx; "	/* A[2] */
+		"mulx   (%2),  %%r8,  %%r9; " /* A[2]*B[0] */
+		"xorl %%r10d, %%r10d ;"
+		"adcx 16(%0), %%r8 ;"
+		"movq  %%r8, 16(%0) ;"
+		"mulx  8(%2), %%r10, %%r11; " /* A[2]*B[1] */
+		"adox  %%r9, %%r10 ;"
+		"adcx %%r15, %%r10 ;"
+		"movq %%r10, 24(%0) ;"
+		"mulx 16(%2), %%r15, %%r13; " /* A[2]*B[2] */
+		"adox %%r11, %%r15 ;"
+		"adcx %%r14, %%r15 ;"
+		"movq $0, %%r8  ;"
+		"mulx 24(%2), %%r14, %%rdx; " /* A[2]*B[3] */
+		"adox %%r13, %%r14 ;"
+		"adcx %%rax, %%r14 ;"
+		"movq $0, %%rax ;"
+		/******************************************/
+		"adox %%rdx, %%rax ;"
+		"adcx  %%r8, %%rax ;"
+
+		"movq 24(%1), %%rdx; "	/* A[3] */
+		"mulx   (%2),  %%r8,  %%r9; " /* A[3]*B[0] */
+		"xorl %%r10d, %%r10d ;"
+		"adcx 24(%0), %%r8 ;"
+		"movq  %%r8, 24(%0) ;"
+		"mulx  8(%2), %%r10, %%r11; " /* A[3]*B[1] */
+		"adox  %%r9, %%r10 ;"
+		"adcx %%r15, %%r10 ;"
+		"movq %%r10, 32(%0) ;"
+		"mulx 16(%2), %%r15, %%r13; " /* A[3]*B[2] */
+		"adox %%r11, %%r15 ;"
+		"adcx %%r14, %%r15 ;"
+		"movq %%r15, 40(%0) ;"
+		"movq $0, %%r8  ;"
+		"mulx 24(%2), %%r14, %%rdx; " /* A[3]*B[3] */
+		"adox %%r13, %%r14 ;"
+		"adcx %%rax, %%r14 ;"
+		"movq %%r14, 48(%0) ;"
+		"movq $0, %%rax ;"
+		/******************************************/
+		"adox %%rdx, %%rax ;"
+		"adcx  %%r8, %%rax ;"
+		"movq %%rax, 56(%0) ;"
+		:
+		: "r"(c), "r"(a), "r"(b)
+		: "memory", "cc", "%rax", "%rdx", "%r8", "%r9", "%r10", "%r11",
+		  "%r13", "%r14", "%r15");
+}
+#endif
+
+#ifdef CONFIG_AS_BMI2
+static void mul_256x256_integer_bmi2(u64 *const c, const u64 *const a,
+				     const u64 *const b)
+{
+	asm volatile(
+		"movq   (%1), %%rdx; "	/* A[0] */
+		"mulx   (%2),  %%r8, %%r15; " /* A[0]*B[0] */
+		"movq %%r8,  (%0) ;"
+		"mulx  8(%2), %%r10, %%rax; " /* A[0]*B[1] */
+		"addq %%r10, %%r15 ;"
+		"mulx 16(%2),  %%r8, %%rbx; " /* A[0]*B[2] */
+		"adcq  %%r8, %%rax ;"
+		"mulx 24(%2), %%r10, %%rcx; " /* A[0]*B[3] */
+		"adcq %%r10, %%rbx ;"
+		/******************************************/
+		"adcq    $0, %%rcx ;"
+
+		"movq  8(%1), %%rdx; "	/* A[1] */
+		"mulx   (%2),  %%r8,  %%r9; " /* A[1]*B[0] */
+		"addq %%r15,  %%r8 ;"
+		"movq %%r8, 8(%0) ;"
+		"mulx  8(%2), %%r10, %%r11; " /* A[1]*B[1] */
+		"adcq %%r10,  %%r9 ;"
+		"mulx 16(%2),  %%r8, %%r13; " /* A[1]*B[2] */
+		"adcq  %%r8, %%r11 ;"
+		"mulx 24(%2), %%r10, %%r15; " /* A[1]*B[3] */
+		"adcq %%r10, %%r13 ;"
+		/******************************************/
+		"adcq    $0, %%r15 ;"
+
+		"addq  %%r9, %%rax ;"
+		"adcq %%r11, %%rbx ;"
+		"adcq %%r13, %%rcx ;"
+		"adcq    $0, %%r15 ;"
+
+		"movq 16(%1), %%rdx; "	/* A[2] */
+		"mulx   (%2),  %%r8,  %%r9; " /* A[2]*B[0] */
+		"addq %%rax,  %%r8 ;"
+		"movq %%r8, 16(%0) ;"
+		"mulx  8(%2), %%r10, %%r11; " /* A[2]*B[1] */
+		"adcq %%r10,  %%r9 ;"
+		"mulx 16(%2),  %%r8, %%r13; " /* A[2]*B[2] */
+		"adcq  %%r8, %%r11 ;"
+		"mulx 24(%2), %%r10, %%rax; " /* A[2]*B[3] */
+		"adcq %%r10, %%r13 ;"
+		/******************************************/
+		"adcq    $0, %%rax ;"
+
+		"addq  %%r9, %%rbx ;"
+		"adcq %%r11, %%rcx ;"
+		"adcq %%r13, %%r15 ;"
+		"adcq    $0, %%rax ;"
+
+		"movq 24(%1), %%rdx; "	/* A[3] */
+		"mulx   (%2),  %%r8,  %%r9; " /* A[3]*B[0] */
+		"addq %%rbx,  %%r8 ;"
+		"movq %%r8, 24(%0) ;"
+		"mulx  8(%2), %%r10, %%r11; " /* A[3]*B[1] */
+		"adcq %%r10,  %%r9 ;"
+		"mulx 16(%2),  %%r8, %%r13; " /* A[3]*B[2] */
+		"adcq  %%r8, %%r11 ;"
+		"mulx 24(%2), %%r10, %%rbx; " /* A[3]*B[3] */
+		"adcq %%r10, %%r13 ;"
+		/******************************************/
+		"adcq    $0, %%rbx ;"
+
+		"addq  %%r9, %%rcx ;"
+		"movq %%rcx, 32(%0) ;"
+		"adcq %%r11, %%r15 ;"
+		"movq %%r15, 40(%0) ;"
+		"adcq %%r13, %%rax ;"
+		"movq %%rax, 48(%0) ;"
+		"adcq    $0, %%rbx ;"
+		"movq %%rbx, 56(%0) ;"
+		:
+		: "r"(c), "r"(a), "r"(b)
+		: "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9",
+		  "%r10", "%r11", "%r13", "%r15");
+}
+#endif
+
+#ifdef CONFIG_AS_ADX
+static void sqr_256x256_integer_adx(u64 *const c, const u64 *const a)
+{
+	asm volatile(
+		"movq   (%1), %%rdx        ;" /* A[0]      */
+		"mulx  8(%1),  %%r8, %%r14 ;" /* A[1]*A[0] */
+		"xorl %%r15d, %%r15d;"
+		"mulx 16(%1),  %%r9, %%r10 ;" /* A[2]*A[0] */
+		"adcx %%r14,  %%r9 ;"
+		"mulx 24(%1), %%rax, %%rcx ;" /* A[3]*A[0] */
+		"adcx %%rax, %%r10 ;"
+		"movq 24(%1), %%rdx        ;" /* A[3]      */
+		"mulx  8(%1), %%r11, %%rbx ;" /* A[1]*A[3] */
+		"adcx %%rcx, %%r11 ;"
+		"mulx 16(%1), %%rax, %%r13 ;" /* A[2]*A[3] */
+		"adcx %%rax, %%rbx ;"
+		"movq  8(%1), %%rdx        ;" /* A[1]      */
+		"adcx %%r15, %%r13 ;"
+		"mulx 16(%1), %%rax, %%rcx ;" /* A[2]*A[1] */
+		"movq    $0, %%r14 ;"
+		/******************************************/
+		"adcx %%r15, %%r14 ;"
+
+		"xorl %%r15d, %%r15d;"
+		"adox %%rax, %%r10 ;"
+		"adcx  %%r8,  %%r8 ;"
+		"adox %%rcx, %%r11 ;"
+		"adcx  %%r9,  %%r9 ;"
+		"adox %%r15, %%rbx ;"
+		"adcx %%r10, %%r10 ;"
+		"adox %%r15, %%r13 ;"
+		"adcx %%r11, %%r11 ;"
+		"adox %%r15, %%r14 ;"
+		"adcx %%rbx, %%rbx ;"
+		"adcx %%r13, %%r13 ;"
+		"adcx %%r14, %%r14 ;"
+
+		"movq   (%1), %%rdx ;"
+		"mulx %%rdx, %%rax, %%rcx ;" /* A[0]^2 */
+		/*******************/
+		"movq %%rax,  0(%0) ;"
+		"addq %%rcx,  %%r8 ;"
+		"movq  %%r8,  8(%0) ;"
+		"movq  8(%1), %%rdx ;"
+		"mulx %%rdx, %%rax, %%rcx ;" /* A[1]^2 */
+		"adcq %%rax,  %%r9 ;"
+		"movq  %%r9, 16(%0) ;"
+		"adcq %%rcx, %%r10 ;"
+		"movq %%r10, 24(%0) ;"
+		"movq 16(%1), %%rdx ;"
+		"mulx %%rdx, %%rax, %%rcx ;" /* A[2]^2 */
+		"adcq %%rax, %%r11 ;"
+		"movq %%r11, 32(%0) ;"
+		"adcq %%rcx, %%rbx ;"
+		"movq %%rbx, 40(%0) ;"
+		"movq 24(%1), %%rdx ;"
+		"mulx %%rdx, %%rax, %%rcx ;" /* A[3]^2 */
+		"adcq %%rax, %%r13 ;"
+		"movq %%r13, 48(%0) ;"
+		"adcq %%rcx, %%r14 ;"
+		"movq %%r14, 56(%0) ;"
+		:
+		: "r"(c), "r"(a)
+		: "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9",
+		  "%r10", "%r11", "%r13", "%r14", "%r15");
+}
+#endif
+
+#ifdef CONFIG_AS_BMI2
+static void sqr_256x256_integer_bmi2(u64 *const c, const u64 *const a)
+{
+	asm volatile(
+		"movq  8(%1), %%rdx        ;" /* A[1]      */
+		"mulx   (%1),  %%r8,  %%r9 ;" /* A[0]*A[1] */
+		"mulx 16(%1), %%r10, %%r11 ;" /* A[2]*A[1] */
+		"mulx 24(%1), %%rcx, %%r14 ;" /* A[3]*A[1] */
+
+		"movq 16(%1), %%rdx        ;" /* A[2]      */
+		"mulx 24(%1), %%r15, %%r13 ;" /* A[3]*A[2] */
+		"mulx   (%1), %%rax, %%rdx ;" /* A[0]*A[2] */
+
+		"addq %%rax,  %%r9 ;"
+		"adcq %%rdx, %%r10 ;"
+		"adcq %%rcx, %%r11 ;"
+		"adcq %%r14, %%r15 ;"
+		"adcq    $0, %%r13 ;"
+		"movq    $0, %%r14 ;"
+		"adcq    $0, %%r14 ;"
+
+		"movq   (%1), %%rdx        ;" /* A[0]      */
+		"mulx 24(%1), %%rax, %%rcx ;" /* A[0]*A[3] */
+
+		"addq %%rax, %%r10 ;"
+		"adcq %%rcx, %%r11 ;"
+		"adcq    $0, %%r15 ;"
+		"adcq    $0, %%r13 ;"
+		"adcq    $0, %%r14 ;"
+
+		"shldq $1, %%r13, %%r14 ;"
+		"shldq $1, %%r15, %%r13 ;"
+		"shldq $1, %%r11, %%r15 ;"
+		"shldq $1, %%r10, %%r11 ;"
+		"shldq $1,  %%r9, %%r10 ;"
+		"shldq $1,  %%r8,  %%r9 ;"
+		"shlq  $1,  %%r8        ;"
+
+		/*******************/
+		"mulx %%rdx, %%rax, %%rcx ;" /* A[0]^2 */
+		/*******************/
+		"movq %%rax,  0(%0) ;"
+		"addq %%rcx,  %%r8 ;"
+		"movq  %%r8,  8(%0) ;"
+		"movq  8(%1), %%rdx ;"
+		"mulx %%rdx, %%rax, %%rcx ;" /* A[1]^2 */
+		"adcq %%rax,  %%r9 ;"
+		"movq  %%r9, 16(%0) ;"
+		"adcq %%rcx, %%r10 ;"
+		"movq %%r10, 24(%0) ;"
+		"movq 16(%1), %%rdx ;"
+		"mulx %%rdx, %%rax, %%rcx ;" /* A[2]^2 */
+		"adcq %%rax, %%r11 ;"
+		"movq %%r11, 32(%0) ;"
+		"adcq %%rcx, %%r15 ;"
+		"movq %%r15, 40(%0) ;"
+		"movq 24(%1), %%rdx ;"
+		"mulx %%rdx, %%rax, %%rcx ;" /* A[3]^2 */
+		"adcq %%rax, %%r13 ;"
+		"movq %%r13, 48(%0) ;"
+		"adcq %%rcx, %%r14 ;"
+		"movq %%r14, 56(%0) ;"
+		:
+		: "r"(c), "r"(a)
+		: "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10",
+		  "%r11", "%r13", "%r14", "%r15");
+}
+#endif
+
+#ifdef CONFIG_AS_ADX
+static void red_eltfp25519_1w_adx(u64 *const c, const u64 *const a)
+{
+	asm volatile(
+		"movl    $38, %%edx ;"	/* 2*c = 38 = 2^256 */
+		"mulx 32(%1),  %%r8, %%r10 ;" /* c*C[4] */
+		"xorl %%ebx, %%ebx ;"
+		"adox   (%1),  %%r8 ;"
+		"mulx 40(%1),  %%r9, %%r11 ;" /* c*C[5] */
+		"adcx %%r10,  %%r9 ;"
+		"adox  8(%1),  %%r9 ;"
+		"mulx 48(%1), %%r10, %%rax ;" /* c*C[6] */
+		"adcx %%r11, %%r10 ;"
+		"adox 16(%1), %%r10 ;"
+		"mulx 56(%1), %%r11, %%rcx ;" /* c*C[7] */
+		"adcx %%rax, %%r11 ;"
+		"adox 24(%1), %%r11 ;"
+		/***************************************/
+		"adcx %%rbx, %%rcx ;"
+		"adox  %%rbx, %%rcx ;"
+		"imul %%rdx, %%rcx ;" /* c*C[4], cf=0, of=0 */
+		"adcx %%rcx,  %%r8 ;"
+		"adcx %%rbx,  %%r9 ;"
+		"movq  %%r9,  8(%0) ;"
+		"adcx %%rbx, %%r10 ;"
+		"movq %%r10, 16(%0) ;"
+		"adcx %%rbx, %%r11 ;"
+		"movq %%r11, 24(%0) ;"
+		"mov     $0, %%ecx ;"
+		"cmovc %%edx, %%ecx ;"
+		"addq %%rcx,  %%r8 ;"
+		"movq  %%r8,   (%0) ;"
+		:
+		: "r"(c), "r"(a)
+		: "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9",
+		  "%r10", "%r11");
+}
+#endif
+
+#ifdef CONFIG_AS_BMI2
+static void red_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a)
+{
+	asm volatile(
+		"movl    $38, %%edx ;"	/* 2*c = 38 = 2^256 */
+		"mulx 32(%1),  %%r8, %%r10 ;" /* c*C[4] */
+		"mulx 40(%1),  %%r9, %%r11 ;" /* c*C[5] */
+		"addq %%r10,  %%r9 ;"
+		"mulx 48(%1), %%r10, %%rax ;" /* c*C[6] */
+		"adcq %%r11, %%r10 ;"
+		"mulx 56(%1), %%r11, %%rcx ;" /* c*C[7] */
+		"adcq %%rax, %%r11 ;"
+		/***************************************/
+		"adcq    $0, %%rcx ;"
+		"addq   (%1),  %%r8 ;"
+		"adcq  8(%1),  %%r9 ;"
+		"adcq 16(%1), %%r10 ;"
+		"adcq 24(%1), %%r11 ;"
+		"adcq     $0, %%rcx ;"
+		"imul %%rdx, %%rcx ;" /* c*C[4], cf=0 */
+		"addq %%rcx,  %%r8 ;"
+		"adcq    $0,  %%r9 ;"
+		"movq  %%r9,  8(%0) ;"
+		"adcq    $0, %%r10 ;"
+		"movq %%r10, 16(%0) ;"
+		"adcq    $0, %%r11 ;"
+		"movq %%r11, 24(%0) ;"
+		"mov     $0, %%ecx ;"
+		"cmovc %%edx, %%ecx ;"
+		"addq %%rcx,  %%r8 ;"
+		"movq  %%r8,   (%0) ;"
+		:
+		: "r"(c), "r"(a)
+		: "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10",
+		  "%r11");
+}
+#endif
+
+#ifdef CONFIG_AS_ADX
+static __always_inline void
+add_eltfp25519_1w_adx(u64 *const c, const u64 *const a, const u64 *const b)
+{
+	asm volatile(
+		"mov     $38, %%eax ;"
+		"xorl  %%ecx, %%ecx ;"
+		"movq   (%2),  %%r8 ;"
+		"adcx   (%1),  %%r8 ;"
+		"movq  8(%2),  %%r9 ;"
+		"adcx  8(%1),  %%r9 ;"
+		"movq 16(%2), %%r10 ;"
+		"adcx 16(%1), %%r10 ;"
+		"movq 24(%2), %%r11 ;"
+		"adcx 24(%1), %%r11 ;"
+		"cmovc %%eax, %%ecx ;"
+		"xorl %%eax, %%eax  ;"
+		"adcx %%rcx,  %%r8  ;"
+		"adcx %%rax,  %%r9  ;"
+		"movq  %%r9,  8(%0) ;"
+		"adcx %%rax, %%r10  ;"
+		"movq %%r10, 16(%0) ;"
+		"adcx %%rax, %%r11  ;"
+		"movq %%r11, 24(%0) ;"
+		"mov     $38, %%ecx ;"
+		"cmovc %%ecx, %%eax ;"
+		"addq %%rax,  %%r8  ;"
+		"movq  %%r8,   (%0) ;"
+		:
+		: "r"(c), "r"(a), "r"(b)
+		: "memory", "cc", "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11");
+}
+#endif
+
+#ifdef CONFIG_AS_BMI2
+static __always_inline void
+add_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a, const u64 *const b)
+{
+	asm volatile(
+		"mov     $38, %%eax ;"
+		"movq   (%2),  %%r8 ;"
+		"addq   (%1),  %%r8 ;"
+		"movq  8(%2),  %%r9 ;"
+		"adcq  8(%1),  %%r9 ;"
+		"movq 16(%2), %%r10 ;"
+		"adcq 16(%1), %%r10 ;"
+		"movq 24(%2), %%r11 ;"
+		"adcq 24(%1), %%r11 ;"
+		"mov      $0, %%ecx ;"
+		"cmovc %%eax, %%ecx ;"
+		"addq %%rcx,  %%r8  ;"
+		"adcq    $0,  %%r9  ;"
+		"movq  %%r9,  8(%0) ;"
+		"adcq    $0, %%r10  ;"
+		"movq %%r10, 16(%0) ;"
+		"adcq    $0, %%r11  ;"
+		"movq %%r11, 24(%0) ;"
+		"mov     $0, %%ecx  ;"
+		"cmovc %%eax, %%ecx ;"
+		"addq %%rcx,  %%r8  ;"
+		"movq  %%r8,   (%0) ;"
+		:
+		: "r"(c), "r"(a), "r"(b)
+		: "memory", "cc", "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11");
+}
+#endif
+
+static __always_inline void
+sub_eltfp25519_1w(u64 *const c, const u64 *const a, const u64 *const b)
+{
+	asm volatile(
+		"mov     $38, %%eax ;"
+		"movq   (%1),  %%r8 ;"
+		"subq   (%2),  %%r8 ;"
+		"movq  8(%1),  %%r9 ;"
+		"sbbq  8(%2),  %%r9 ;"
+		"movq 16(%1), %%r10 ;"
+		"sbbq 16(%2), %%r10 ;"
+		"movq 24(%1), %%r11 ;"
+		"sbbq 24(%2), %%r11 ;"
+		"mov      $0, %%ecx ;"
+		"cmovc %%eax, %%ecx ;"
+		"subq %%rcx,  %%r8  ;"
+		"sbbq    $0,  %%r9  ;"
+		"movq  %%r9,  8(%0) ;"
+		"sbbq    $0, %%r10  ;"
+		"movq %%r10, 16(%0) ;"
+		"sbbq    $0, %%r11  ;"
+		"movq %%r11, 24(%0) ;"
+		"mov     $0, %%ecx  ;"
+		"cmovc %%eax, %%ecx ;"
+		"subq %%rcx,  %%r8  ;"
+		"movq  %%r8,   (%0) ;"
+		:
+		: "r"(c), "r"(a), "r"(b)
+		: "memory", "cc", "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11");
+}
+
+/* Multiplication by a24 = (A+2)/4 = (486662+2)/4 = 121666 */
+static __always_inline void
+mul_a24_eltfp25519_1w(u64 *const c, const u64 *const a)
+{
+	const u64 a24 = 121666;
+	asm volatile(
+		"movq     %2, %%rdx ;"
+		"mulx   (%1),  %%r8, %%r10 ;"
+		"mulx  8(%1),  %%r9, %%r11 ;"
+		"addq %%r10,  %%r9 ;"
+		"mulx 16(%1), %%r10, %%rax ;"
+		"adcq %%r11, %%r10 ;"
+		"mulx 24(%1), %%r11, %%rcx ;"
+		"adcq %%rax, %%r11 ;"
+		/**************************/
+		"adcq    $0, %%rcx ;"
+		"movl   $38, %%edx ;" /* 2*c = 38 = 2^256 mod 2^255-19*/
+		"imul %%rdx, %%rcx ;"
+		"addq %%rcx,  %%r8 ;"
+		"adcq    $0,  %%r9 ;"
+		"movq  %%r9,  8(%0) ;"
+		"adcq    $0, %%r10 ;"
+		"movq %%r10, 16(%0) ;"
+		"adcq    $0, %%r11 ;"
+		"movq %%r11, 24(%0) ;"
+		"mov     $0, %%ecx ;"
+		"cmovc %%edx, %%ecx ;"
+		"addq %%rcx,  %%r8 ;"
+		"movq  %%r8,   (%0) ;"
+		:
+		: "r"(c), "r"(a), "r"(a24)
+		: "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10",
+		  "%r11");
+}
+
+#ifdef CONFIG_AS_ADX
+static void inv_eltfp25519_1w_adx(u64 *const c, const u64 *const a)
+{
+	struct {
+		eltfp25519_1w_buffer buffer;
+		eltfp25519_1w x0, x1, x2;
+	} __aligned(32) m;
+	u64 *T[4];
+
+	T[0] = m.x0;
+	T[1] = c; /* x^(-1) */
+	T[2] = m.x1;
+	T[3] = m.x2;
+
+	copy_eltfp25519_1w(T[1], a);
+	sqrn_eltfp25519_1w_adx(T[1], 1);
+	copy_eltfp25519_1w(T[2], T[1]);
+	sqrn_eltfp25519_1w_adx(T[2], 2);
+	mul_eltfp25519_1w_adx(T[0], a, T[2]);
+	mul_eltfp25519_1w_adx(T[1], T[1], T[0]);
+	copy_eltfp25519_1w(T[2], T[1]);
+	sqrn_eltfp25519_1w_adx(T[2], 1);
+	mul_eltfp25519_1w_adx(T[0], T[0], T[2]);
+	copy_eltfp25519_1w(T[2], T[0]);
+	sqrn_eltfp25519_1w_adx(T[2], 5);
+	mul_eltfp25519_1w_adx(T[0], T[0], T[2]);
+	copy_eltfp25519_1w(T[2], T[0]);
+	sqrn_eltfp25519_1w_adx(T[2], 10);
+	mul_eltfp25519_1w_adx(T[2], T[2], T[0]);
+	copy_eltfp25519_1w(T[3], T[2]);
+	sqrn_eltfp25519_1w_adx(T[3], 20);
+	mul_eltfp25519_1w_adx(T[3], T[3], T[2]);
+	sqrn_eltfp25519_1w_adx(T[3], 10);
+	mul_eltfp25519_1w_adx(T[3], T[3], T[0]);
+	copy_eltfp25519_1w(T[0], T[3]);
+	sqrn_eltfp25519_1w_adx(T[0], 50);
+	mul_eltfp25519_1w_adx(T[0], T[0], T[3]);
+	copy_eltfp25519_1w(T[2], T[0]);
+	sqrn_eltfp25519_1w_adx(T[2], 100);
+	mul_eltfp25519_1w_adx(T[2], T[2], T[0]);
+	sqrn_eltfp25519_1w_adx(T[2], 50);
+	mul_eltfp25519_1w_adx(T[2], T[2], T[3]);
+	sqrn_eltfp25519_1w_adx(T[2], 5);
+	mul_eltfp25519_1w_adx(T[1], T[1], T[2]);
+
+	memzero_explicit(&m, sizeof(m));
+}
+#endif
+
+#ifdef CONFIG_AS_BMI2
+static void inv_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a)
+{
+	struct {
+		eltfp25519_1w_buffer buffer;
+		eltfp25519_1w x0, x1, x2;
+	} __aligned(32) m;
+	u64 *T[5];
+
+	T[0] = m.x0;
+	T[1] = c; /* x^(-1) */
+	T[2] = m.x1;
+	T[3] = m.x2;
+
+	copy_eltfp25519_1w(T[1], a);
+	sqrn_eltfp25519_1w_bmi2(T[1], 1);
+	copy_eltfp25519_1w(T[2], T[1]);
+	sqrn_eltfp25519_1w_bmi2(T[2], 2);
+	mul_eltfp25519_1w_bmi2(T[0], a, T[2]);
+	mul_eltfp25519_1w_bmi2(T[1], T[1], T[0]);
+	copy_eltfp25519_1w(T[2], T[1]);
+	sqrn_eltfp25519_1w_bmi2(T[2], 1);
+	mul_eltfp25519_1w_bmi2(T[0], T[0], T[2]);
+	copy_eltfp25519_1w(T[2], T[0]);
+	sqrn_eltfp25519_1w_bmi2(T[2], 5);
+	mul_eltfp25519_1w_bmi2(T[0], T[0], T[2]);
+	copy_eltfp25519_1w(T[2], T[0]);
+	sqrn_eltfp25519_1w_bmi2(T[2], 10);
+	mul_eltfp25519_1w_bmi2(T[2], T[2], T[0]);
+	copy_eltfp25519_1w(T[3], T[2]);
+	sqrn_eltfp25519_1w_bmi2(T[3], 20);
+	mul_eltfp25519_1w_bmi2(T[3], T[3], T[2]);
+	sqrn_eltfp25519_1w_bmi2(T[3], 10);
+	mul_eltfp25519_1w_bmi2(T[3], T[3], T[0]);
+	copy_eltfp25519_1w(T[0], T[3]);
+	sqrn_eltfp25519_1w_bmi2(T[0], 50);
+	mul_eltfp25519_1w_bmi2(T[0], T[0], T[3]);
+	copy_eltfp25519_1w(T[2], T[0]);
+	sqrn_eltfp25519_1w_bmi2(T[2], 100);
+	mul_eltfp25519_1w_bmi2(T[2], T[2], T[0]);
+	sqrn_eltfp25519_1w_bmi2(T[2], 50);
+	mul_eltfp25519_1w_bmi2(T[2], T[2], T[3]);
+	sqrn_eltfp25519_1w_bmi2(T[2], 5);
+	mul_eltfp25519_1w_bmi2(T[1], T[1], T[2]);
+
+	memzero_explicit(&m, sizeof(m));
+}
+#endif
+
+/* Given c, a 256-bit number, fred_eltfp25519_1w updates c
+ * with a number such that 0 <= C < 2**255-19.
+ */
+static __always_inline void fred_eltfp25519_1w(u64 *const c)
+{
+	u64 tmp0 = 38, tmp1 = 19;
+	asm volatile(
+		"btrq   $63,    %3 ;" /* Put bit 255 in carry flag and clear */
+		"cmovncl %k5,   %k4 ;" /* c[255] ? 38 : 19 */
+
+		/* Add either 19 or 38 to c */
+		"addq    %4,   %0 ;"
+		"adcq    $0,   %1 ;"
+		"adcq    $0,   %2 ;"
+		"adcq    $0,   %3 ;"
+
+		/* Test for bit 255 again; only triggered on overflow modulo 2^255-19 */
+		"movl    $0,  %k4 ;"
+		"cmovnsl %k5,  %k4 ;" /* c[255] ? 0 : 19 */
+		"btrq   $63,   %3 ;" /* Clear bit 255 */
+
+		/* Subtract 19 if necessary */
+		"subq    %4,   %0 ;"
+		"sbbq    $0,   %1 ;"
+		"sbbq    $0,   %2 ;"
+		"sbbq    $0,   %3 ;"
+
+		: "+r"(c[0]), "+r"(c[1]), "+r"(c[2]), "+r"(c[3]), "+r"(tmp0),
+		  "+r"(tmp1)
+		:
+		: "memory", "cc");
+}
+
+static __always_inline void cswap(u8 bit, u64 *const px, u64 *const py)
+{
+	u64 temp;
+	asm volatile(
+		"test %9, %9 ;"
+		"movq %0, %8 ;"
+		"cmovnzq %4, %0 ;"
+		"cmovnzq %8, %4 ;"
+		"movq %1, %8 ;"
+		"cmovnzq %5, %1 ;"
+		"cmovnzq %8, %5 ;"
+		"movq %2, %8 ;"
+		"cmovnzq %6, %2 ;"
+		"cmovnzq %8, %6 ;"
+		"movq %3, %8 ;"
+		"cmovnzq %7, %3 ;"
+		"cmovnzq %8, %7 ;"
+		: "+r"(px[0]), "+r"(px[1]), "+r"(px[2]), "+r"(px[3]),
+		  "+r"(py[0]), "+r"(py[1]), "+r"(py[2]), "+r"(py[3]),
+		  "=r"(temp)
+		: "r"(bit)
+		: "cc"
+	);
+}
+
+static __always_inline void cselect(u8 bit, u64 *const px, const u64 *const py)
+{
+	asm volatile(
+		"test %4, %4 ;"
+		"cmovnzq %5, %0 ;"
+		"cmovnzq %6, %1 ;"
+		"cmovnzq %7, %2 ;"
+		"cmovnzq %8, %3 ;"
+		: "+r"(px[0]), "+r"(px[1]), "+r"(px[2]), "+r"(px[3])
+		: "r"(bit), "rm"(py[0]), "rm"(py[1]), "rm"(py[2]), "rm"(py[3])
+		: "cc"
+	);
+}
+
+static void curve25519_adx(u8 shared[CURVE25519_KEY_SIZE],
+			   const u8 private_key[CURVE25519_KEY_SIZE],
+			   const u8 session_key[CURVE25519_KEY_SIZE])
+{
+#ifdef CONFIG_AS_ADX
+	struct {
+		u64 buffer[4 * NUM_WORDS_ELTFP25519];
+		u64 coordinates[4 * NUM_WORDS_ELTFP25519];
+		u64 workspace[6 * NUM_WORDS_ELTFP25519];
+		u8 session[CURVE25519_KEY_SIZE];
+		u8 private[CURVE25519_KEY_SIZE];
+	} __aligned(32) m;
+
+	int i = 0, j = 0;
+	u64 prev = 0;
+	u64 *const X1 = (u64 *)m.session;
+	u64 *const key = (u64 *)m.private;
+	u64 *const Px = m.coordinates + 0;
+	u64 *const Pz = m.coordinates + 4;
+	u64 *const Qx = m.coordinates + 8;
+	u64 *const Qz = m.coordinates + 12;
+	u64 *const X2 = Qx;
+	u64 *const Z2 = Qz;
+	u64 *const X3 = Px;
+	u64 *const Z3 = Pz;
+	u64 *const X2Z2 = Qx;
+	u64 *const X3Z3 = Px;
+
+	u64 *const A = m.workspace + 0;
+	u64 *const B = m.workspace + 4;
+	u64 *const D = m.workspace + 8;
+	u64 *const C = m.workspace + 12;
+	u64 *const DA = m.workspace + 16;
+	u64 *const CB = m.workspace + 20;
+	u64 *const AB = A;
+	u64 *const DC = D;
+	u64 *const DACB = DA;
+
+	memcpy(m.private, private_key, sizeof(m.private));
+	memcpy(m.session, session_key, sizeof(m.session));
+
+	curve25519_clamp_secret(m.private);
+
+	/* As in the draft:
+	 * When receiving such an array, implementations of curve25519
+	 * MUST mask the most-significant bit in the final byte. This
+	 * is done to preserve compatibility with point formats which
+	 * reserve the sign bit for use in other protocols and to
+	 * increase resistance to implementation fingerprinting
+	 */
+	m.session[CURVE25519_KEY_SIZE - 1] &= (1 << (255 % 8)) - 1;
+
+	copy_eltfp25519_1w(Px, X1);
+	setzero_eltfp25519_1w(Pz);
+	setzero_eltfp25519_1w(Qx);
+	setzero_eltfp25519_1w(Qz);
+
+	Pz[0] = 1;
+	Qx[0] = 1;
+
+	/* main-loop */
+	prev = 0;
+	j = 62;
+	for (i = 3; i >= 0; --i) {
+		while (j >= 0) {
+			u64 bit = (key[i] >> j) & 0x1;
+			u64 swap = bit ^ prev;
+			prev = bit;
+
+			add_eltfp25519_1w_adx(A, X2, Z2);	/* A = (X2+Z2) */
+			sub_eltfp25519_1w(B, X2, Z2);		/* B = (X2-Z2) */
+			add_eltfp25519_1w_adx(C, X3, Z3);	/* C = (X3+Z3) */
+			sub_eltfp25519_1w(D, X3, Z3);		/* D = (X3-Z3) */
+			mul_eltfp25519_2w_adx(DACB, AB, DC);	/* [DA|CB] = [A|B]*[D|C] */
+
+			cselect(swap, A, C);
+			cselect(swap, B, D);
+
+			sqr_eltfp25519_2w_adx(AB);		/* [AA|BB] = [A^2|B^2] */
+			add_eltfp25519_1w_adx(X3, DA, CB);	/* X3 = (DA+CB) */
+			sub_eltfp25519_1w(Z3, DA, CB);		/* Z3 = (DA-CB) */
+			sqr_eltfp25519_2w_adx(X3Z3);		/* [X3|Z3] = [(DA+CB)|(DA+CB)]^2 */
+
+			copy_eltfp25519_1w(X2, B);		/* X2 = B^2 */
+			sub_eltfp25519_1w(Z2, A, B);		/* Z2 = E = AA-BB */
+
+			mul_a24_eltfp25519_1w(B, Z2);		/* B = a24*E */
+			add_eltfp25519_1w_adx(B, B, X2);	/* B = a24*E+B */
+			mul_eltfp25519_2w_adx(X2Z2, X2Z2, AB);	/* [X2|Z2] = [B|E]*[A|a24*E+B] */
+			mul_eltfp25519_1w_adx(Z3, Z3, X1);	/* Z3 = Z3*X1 */
+			--j;
+		}
+		j = 63;
+	}
+
+	inv_eltfp25519_1w_adx(A, Qz);
+	mul_eltfp25519_1w_adx((u64 *)shared, Qx, A);
+	fred_eltfp25519_1w((u64 *)shared);
+
+	memzero_explicit(&m, sizeof(m));
+#endif
+}
+
+static void curve25519_adx_base(u8 session_key[CURVE25519_KEY_SIZE],
+				const u8 private_key[CURVE25519_KEY_SIZE])
+{
+#ifdef CONFIG_AS_ADX
+	struct {
+		u64 buffer[4 * NUM_WORDS_ELTFP25519];
+		u64 coordinates[4 * NUM_WORDS_ELTFP25519];
+		u64 workspace[4 * NUM_WORDS_ELTFP25519];
+		u8 private[CURVE25519_KEY_SIZE];
+	} __aligned(32) m;
+
+	const int ite[4] = { 64, 64, 64, 63 };
+	const int q = 3;
+	u64 swap = 1;
+
+	int i = 0, j = 0, k = 0;
+	u64 *const key = (u64 *)m.private;
+	u64 *const Ur1 = m.coordinates + 0;
+	u64 *const Zr1 = m.coordinates + 4;
+	u64 *const Ur2 = m.coordinates + 8;
+	u64 *const Zr2 = m.coordinates + 12;
+
+	u64 *const UZr1 = m.coordinates + 0;
+	u64 *const ZUr2 = m.coordinates + 8;
+
+	u64 *const A = m.workspace + 0;
+	u64 *const B = m.workspace + 4;
+	u64 *const C = m.workspace + 8;
+	u64 *const D = m.workspace + 12;
+
+	u64 *const AB = m.workspace + 0;
+	u64 *const CD = m.workspace + 8;
+
+	const u64 *const P = table_ladder_8k;
+
+	memcpy(m.private, private_key, sizeof(m.private));
+
+	curve25519_clamp_secret(m.private);
+
+	setzero_eltfp25519_1w(Ur1);
+	setzero_eltfp25519_1w(Zr1);
+	setzero_eltfp25519_1w(Zr2);
+	Ur1[0] = 1;
+	Zr1[0] = 1;
+	Zr2[0] = 1;
+
+	/* G-S */
+	Ur2[3] = 0x1eaecdeee27cab34UL;
+	Ur2[2] = 0xadc7a0b9235d48e2UL;
+	Ur2[1] = 0xbbf095ae14b2edf8UL;
+	Ur2[0] = 0x7e94e1fec82faabdUL;
+
+	/* main-loop */
+	j = q;
+	for (i = 0; i < NUM_WORDS_ELTFP25519; ++i) {
+		while (j < ite[i]) {
+			u64 bit = (key[i] >> j) & 0x1;
+			k = (64 * i + j - q);
+			swap = swap ^ bit;
+			cswap(swap, Ur1, Ur2);
+			cswap(swap, Zr1, Zr2);
+			swap = bit;
+			/* Addition */
+			sub_eltfp25519_1w(B, Ur1, Zr1);		/* B = Ur1-Zr1 */
+			add_eltfp25519_1w_adx(A, Ur1, Zr1);	/* A = Ur1+Zr1 */
+			mul_eltfp25519_1w_adx(C, &P[4 * k], B);	/* C = M0-B */
+			sub_eltfp25519_1w(B, A, C);		/* B = (Ur1+Zr1) - M*(Ur1-Zr1) */
+			add_eltfp25519_1w_adx(A, A, C);		/* A = (Ur1+Zr1) + M*(Ur1-Zr1) */
+			sqr_eltfp25519_2w_adx(AB);		/* A = A^2      |  B = B^2 */
+			mul_eltfp25519_2w_adx(UZr1, ZUr2, AB);	/* Ur1 = Zr2*A  |  Zr1 = Ur2*B */
+			++j;
+		}
+		j = 0;
+	}
+
+	/* Doubling */
+	for (i = 0; i < q; ++i) {
+		add_eltfp25519_1w_adx(A, Ur1, Zr1);	/*  A = Ur1+Zr1 */
+		sub_eltfp25519_1w(B, Ur1, Zr1);		/*  B = Ur1-Zr1 */
+		sqr_eltfp25519_2w_adx(AB);		/*  A = A**2     B = B**2 */
+		copy_eltfp25519_1w(C, B);		/*  C = B */
+		sub_eltfp25519_1w(B, A, B);		/*  B = A-B */
+		mul_a24_eltfp25519_1w(D, B);		/*  D = my_a24*B */
+		add_eltfp25519_1w_adx(D, D, C);		/*  D = D+C */
+		mul_eltfp25519_2w_adx(UZr1, AB, CD);	/*  Ur1 = A*B   Zr1 = Zr1*A */
+	}
+
+	/* Convert to affine coordinates */
+	inv_eltfp25519_1w_adx(A, Zr1);
+	mul_eltfp25519_1w_adx((u64 *)session_key, Ur1, A);
+	fred_eltfp25519_1w((u64 *)session_key);
+
+	memzero_explicit(&m, sizeof(m));
+#endif
+}
+
+static void curve25519_bmi2(u8 shared[CURVE25519_KEY_SIZE],
+			    const u8 private_key[CURVE25519_KEY_SIZE],
+			    const u8 session_key[CURVE25519_KEY_SIZE])
+{
+#ifdef CONFIG_AS_BMI2
+	struct {
+		u64 buffer[4 * NUM_WORDS_ELTFP25519];
+		u64 coordinates[4 * NUM_WORDS_ELTFP25519];
+		u64 workspace[6 * NUM_WORDS_ELTFP25519];
+		u8 session[CURVE25519_KEY_SIZE];
+		u8 private[CURVE25519_KEY_SIZE];
+	} __aligned(32) m;
+
+	int i = 0, j = 0;
+	u64 prev = 0;
+	u64 *const X1 = (u64 *)m.session;
+	u64 *const key = (u64 *)m.private;
+	u64 *const Px = m.coordinates + 0;
+	u64 *const Pz = m.coordinates + 4;
+	u64 *const Qx = m.coordinates + 8;
+	u64 *const Qz = m.coordinates + 12;
+	u64 *const X2 = Qx;
+	u64 *const Z2 = Qz;
+	u64 *const X3 = Px;
+	u64 *const Z3 = Pz;
+	u64 *const X2Z2 = Qx;
+	u64 *const X3Z3 = Px;
+
+	u64 *const A = m.workspace + 0;
+	u64 *const B = m.workspace + 4;
+	u64 *const D = m.workspace + 8;
+	u64 *const C = m.workspace + 12;
+	u64 *const DA = m.workspace + 16;
+	u64 *const CB = m.workspace + 20;
+	u64 *const AB = A;
+	u64 *const DC = D;
+	u64 *const DACB = DA;
+
+	memcpy(m.private, private_key, sizeof(m.private));
+	memcpy(m.session, session_key, sizeof(m.session));
+
+	curve25519_clamp_secret(m.private);
+
+	/* As in the draft:
+	 * When receiving such an array, implementations of curve25519
+	 * MUST mask the most-significant bit in the final byte. This
+	 * is done to preserve compatibility with point formats which
+	 * reserve the sign bit for use in other protocols and to
+	 * increase resistance to implementation fingerprinting
+	 */
+	m.session[CURVE25519_KEY_SIZE - 1] &= (1 << (255 % 8)) - 1;
+
+	copy_eltfp25519_1w(Px, X1);
+	setzero_eltfp25519_1w(Pz);
+	setzero_eltfp25519_1w(Qx);
+	setzero_eltfp25519_1w(Qz);
+
+	Pz[0] = 1;
+	Qx[0] = 1;
+
+	/* main-loop */
+	prev = 0;
+	j = 62;
+	for (i = 3; i >= 0; --i) {
+		while (j >= 0) {
+			u64 bit = (key[i] >> j) & 0x1;
+			u64 swap = bit ^ prev;
+			prev = bit;
+
+			add_eltfp25519_1w_bmi2(A, X2, Z2);	/* A = (X2+Z2) */
+			sub_eltfp25519_1w(B, X2, Z2);		/* B = (X2-Z2) */
+			add_eltfp25519_1w_bmi2(C, X3, Z3);	/* C = (X3+Z3) */
+			sub_eltfp25519_1w(D, X3, Z3);		/* D = (X3-Z3) */
+			mul_eltfp25519_2w_bmi2(DACB, AB, DC);	/* [DA|CB] = [A|B]*[D|C] */
+
+			cselect(swap, A, C);
+			cselect(swap, B, D);
+
+			sqr_eltfp25519_2w_bmi2(AB);		/* [AA|BB] = [A^2|B^2] */
+			add_eltfp25519_1w_bmi2(X3, DA, CB);	/* X3 = (DA+CB) */
+			sub_eltfp25519_1w(Z3, DA, CB);		/* Z3 = (DA-CB) */
+			sqr_eltfp25519_2w_bmi2(X3Z3);		/* [X3|Z3] = [(DA+CB)|(DA+CB)]^2 */
+
+			copy_eltfp25519_1w(X2, B);		/* X2 = B^2 */
+			sub_eltfp25519_1w(Z2, A, B);		/* Z2 = E = AA-BB */
+
+			mul_a24_eltfp25519_1w(B, Z2);		/* B = a24*E */
+			add_eltfp25519_1w_bmi2(B, B, X2);	/* B = a24*E+B */
+			mul_eltfp25519_2w_bmi2(X2Z2, X2Z2, AB);	/* [X2|Z2] = [B|E]*[A|a24*E+B] */
+			mul_eltfp25519_1w_bmi2(Z3, Z3, X1);	/* Z3 = Z3*X1 */
+			--j;
+		}
+		j = 63;
+	}
+
+	inv_eltfp25519_1w_bmi2(A, Qz);
+	mul_eltfp25519_1w_bmi2((u64 *)shared, Qx, A);
+	fred_eltfp25519_1w((u64 *)shared);
+
+	memzero_explicit(&m, sizeof(m));
+#endif
+}
+
+static void curve25519_bmi2_base(u8 session_key[CURVE25519_KEY_SIZE],
+				 const u8 private_key[CURVE25519_KEY_SIZE])
+{
+#ifdef CONFIG_AS_BMI2
+	struct {
+		u64 buffer[4 * NUM_WORDS_ELTFP25519];
+		u64 coordinates[4 * NUM_WORDS_ELTFP25519];
+		u64 workspace[4 * NUM_WORDS_ELTFP25519];
+		u8 private[CURVE25519_KEY_SIZE];
+	} __aligned(32) m;
+
+	const int ite[4] = { 64, 64, 64, 63 };
+	const int q = 3;
+	u64 swap = 1;
+
+	int i = 0, j = 0, k = 0;
+	u64 *const key = (u64 *)m.private;
+	u64 *const Ur1 = m.coordinates + 0;
+	u64 *const Zr1 = m.coordinates + 4;
+	u64 *const Ur2 = m.coordinates + 8;
+	u64 *const Zr2 = m.coordinates + 12;
+
+	u64 *const UZr1 = m.coordinates + 0;
+	u64 *const ZUr2 = m.coordinates + 8;
+
+	u64 *const A = m.workspace + 0;
+	u64 *const B = m.workspace + 4;
+	u64 *const C = m.workspace + 8;
+	u64 *const D = m.workspace + 12;
+
+	u64 *const AB = m.workspace + 0;
+	u64 *const CD = m.workspace + 8;
+
+	const u64 *const P = table_ladder_8k;
+
+	memcpy(m.private, private_key, sizeof(m.private));
+
+	curve25519_clamp_secret(m.private);
+
+	setzero_eltfp25519_1w(Ur1);
+	setzero_eltfp25519_1w(Zr1);
+	setzero_eltfp25519_1w(Zr2);
+	Ur1[0] = 1;
+	Zr1[0] = 1;
+	Zr2[0] = 1;
+
+	/* G-S */
+	Ur2[3] = 0x1eaecdeee27cab34UL;
+	Ur2[2] = 0xadc7a0b9235d48e2UL;
+	Ur2[1] = 0xbbf095ae14b2edf8UL;
+	Ur2[0] = 0x7e94e1fec82faabdUL;
+
+	/* main-loop */
+	j = q;
+	for (i = 0; i < NUM_WORDS_ELTFP25519; ++i) {
+		while (j < ite[i]) {
+			u64 bit = (key[i] >> j) & 0x1;
+			k = (64 * i + j - q);
+			swap = swap ^ bit;
+			cswap(swap, Ur1, Ur2);
+			cswap(swap, Zr1, Zr2);
+			swap = bit;
+			/* Addition */
+			sub_eltfp25519_1w(B, Ur1, Zr1);		/* B = Ur1-Zr1 */
+			add_eltfp25519_1w_bmi2(A, Ur1, Zr1);	/* A = Ur1+Zr1 */
+			mul_eltfp25519_1w_bmi2(C, &P[4 * k], B);/* C = M0-B */
+			sub_eltfp25519_1w(B, A, C);		/* B = (Ur1+Zr1) - M*(Ur1-Zr1) */
+			add_eltfp25519_1w_bmi2(A, A, C);	/* A = (Ur1+Zr1) + M*(Ur1-Zr1) */
+			sqr_eltfp25519_2w_bmi2(AB);		/* A = A^2      |  B = B^2 */
+			mul_eltfp25519_2w_bmi2(UZr1, ZUr2, AB);	/* Ur1 = Zr2*A  |  Zr1 = Ur2*B */
+			++j;
+		}
+		j = 0;
+	}
+
+	/* Doubling */
+	for (i = 0; i < q; ++i) {
+		add_eltfp25519_1w_bmi2(A, Ur1, Zr1);	/*  A = Ur1+Zr1 */
+		sub_eltfp25519_1w(B, Ur1, Zr1);		/*  B = Ur1-Zr1 */
+		sqr_eltfp25519_2w_bmi2(AB);		/*  A = A**2     B = B**2 */
+		copy_eltfp25519_1w(C, B);		/*  C = B */
+		sub_eltfp25519_1w(B, A, B);		/*  B = A-B */
+		mul_a24_eltfp25519_1w(D, B);		/*  D = my_a24*B */
+		add_eltfp25519_1w_bmi2(D, D, C);	/*  D = D+C */
+		mul_eltfp25519_2w_bmi2(UZr1, AB, CD);	/*  Ur1 = A*B   Zr1 = Zr1*A */
+	}
+
+	/* Convert to affine coordinates */
+	inv_eltfp25519_1w_bmi2(A, Zr1);
+	mul_eltfp25519_1w_bmi2((u64 *)session_key, Ur1, A);
+	fred_eltfp25519_1w((u64 *)session_key);
+
+	memzero_explicit(&m, sizeof(m));
+#endif
+}
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/curve25519/curve25519-x86_64-glue.c b/net/wireguard/crypto/zinc/curve25519/curve25519-x86_64-glue.c
--- a/net/wireguard/crypto/zinc/curve25519/curve25519-x86_64-glue.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/curve25519/curve25519-x86_64-glue.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,50 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#include <asm/cpufeature.h>
+#include <asm/processor.h>
+
+#include "curve25519-x86_64.c"
+
+static bool curve25519_use_bmi2 __ro_after_init;
+static bool curve25519_use_adx __ro_after_init;
+static bool *const curve25519_nobs[] __initconst = {
+	&curve25519_use_bmi2, &curve25519_use_adx };
+
+static void __init curve25519_fpu_init(void)
+{
+	curve25519_use_bmi2 = IS_ENABLED(CONFIG_AS_BMI2) &&
+			      boot_cpu_has(X86_FEATURE_BMI2);
+	curve25519_use_adx = IS_ENABLED(CONFIG_AS_ADX) &&
+			     boot_cpu_has(X86_FEATURE_BMI2) &&
+			     boot_cpu_has(X86_FEATURE_ADX);
+}
+
+static inline bool curve25519_arch(u8 mypublic[CURVE25519_KEY_SIZE],
+				   const u8 secret[CURVE25519_KEY_SIZE],
+				   const u8 basepoint[CURVE25519_KEY_SIZE])
+{
+	if (IS_ENABLED(CONFIG_AS_ADX) && curve25519_use_adx) {
+		curve25519_adx(mypublic, secret, basepoint);
+		return true;
+	} else if (IS_ENABLED(CONFIG_AS_BMI2) && curve25519_use_bmi2) {
+		curve25519_bmi2(mypublic, secret, basepoint);
+		return true;
+	}
+	return false;
+}
+
+static inline bool curve25519_base_arch(u8 pub[CURVE25519_KEY_SIZE],
+					const u8 secret[CURVE25519_KEY_SIZE])
+{
+	if (IS_ENABLED(CONFIG_AS_ADX) && curve25519_use_adx) {
+		curve25519_adx_base(pub, secret);
+		return true;
+	} else if (IS_ENABLED(CONFIG_AS_BMI2) && curve25519_use_bmi2) {
+		curve25519_bmi2_base(pub, secret);
+		return true;
+	}
+	return false;
+}
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/poly1305/poly1305-arm64.pl b/net/wireguard/crypto/zinc/poly1305/poly1305-arm64.pl
--- a/net/wireguard/crypto/zinc/poly1305/poly1305-arm64.pl	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/poly1305/poly1305-arm64.pl	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,974 @@
+#!/usr/bin/env perl
+# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+#
+# This code is taken from the OpenSSL project but the author, Andy Polyakov,
+# has relicensed it under the licenses specified in the SPDX header above.
+# The original headers, including the original license headers, are
+# included below for completeness.
+#
+# ====================================================================
+# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
+# project. The module is, however, dual licensed under OpenSSL and
+# CRYPTOGAMS licenses depending on where you obtain it. For further
+# details see http://www.openssl.org/~appro/cryptogams/.
+# ====================================================================
+#
+# This module implements Poly1305 hash for ARMv8.
+#
+# June 2015
+#
+# Numbers are cycles per processed byte with poly1305_blocks alone.
+#
+#		IALU/gcc-4.9	NEON
+#
+# Apple A7	1.86/+5%	0.72
+# Cortex-A53	2.69/+58%	1.47
+# Cortex-A57	2.70/+7%	1.14
+# Denver	1.64/+50%	1.18(*)
+# X-Gene	2.13/+68%	2.27
+# Mongoose	1.77/+75%	1.12
+# Kryo		2.70/+55%	1.13
+#
+# (*)	estimate based on resources availability is less than 1.0,
+#	i.e. measured result is worse than expected, presumably binary
+#	translator is not almighty;
+
+$flavour=shift;
+if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; }
+else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} }
+
+if ($flavour && $flavour ne "void") {
+    $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
+    ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
+    ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
+    die "can't locate arm-xlate.pl";
+
+    open STDOUT,"| \"$^X\" $xlate $flavour $output";
+} else {
+    open STDOUT,">$output";
+}
+
+my ($ctx,$inp,$len,$padbit) = map("x$_",(0..3));
+my ($mac,$nonce)=($inp,$len);
+
+my ($h0,$h1,$h2,$r0,$r1,$s1,$t0,$t1,$d0,$d1,$d2) = map("x$_",(4..14));
+
+$code.=<<___;
+#ifndef	__KERNEL__
+# include "arm_arch.h"
+.extern	OPENSSL_armcap_P
+#else
+# define poly1305_init   poly1305_init_arm
+# define poly1305_blocks poly1305_blocks_arm
+# define poly1305_emit   poly1305_emit_arm
+#endif
+
+.text
+
+// forward "declarations" are required for Apple
+.globl	poly1305_blocks
+.globl	poly1305_emit
+.globl	poly1305_init
+.type	poly1305_init,%function
+.align	5
+poly1305_init:
+	cmp	$inp,xzr
+	stp	xzr,xzr,[$ctx]		// zero hash value
+	stp	xzr,xzr,[$ctx,#16]	// [along with is_base2_26]
+
+	csel	x0,xzr,x0,eq
+	b.eq	.Lno_key
+
+#ifndef	__KERNEL__
+# ifdef	__ILP32__
+	ldrsw	$t1,.LOPENSSL_armcap_P
+# else
+	ldr	$t1,.LOPENSSL_armcap_P
+# endif
+	adr	$t0,.LOPENSSL_armcap_P
+	ldr	w17,[$t0,$t1]
+#endif
+
+	ldp	$r0,$r1,[$inp]		// load key
+	mov	$s1,#0xfffffffc0fffffff
+	movk	$s1,#0x0fff,lsl#48
+#ifdef	__AARCH64EB__
+	rev	$r0,$r0			// flip bytes
+	rev	$r1,$r1
+#endif
+	and	$r0,$r0,$s1		// &=0ffffffc0fffffff
+	and	$s1,$s1,#-4
+	and	$r1,$r1,$s1		// &=0ffffffc0ffffffc
+	stp	$r0,$r1,[$ctx,#32]	// save key value
+
+#ifndef	__KERNEL__
+	tst	w17,#ARMV7_NEON
+
+	adr	$d0,poly1305_blocks
+	adr	$r0,poly1305_blocks_neon
+	adr	$d1,poly1305_emit
+	adr	$r1,poly1305_emit_neon
+
+	csel	$d0,$d0,$r0,eq
+	csel	$d1,$d1,$r1,eq
+
+# ifdef	__ILP32__
+	stp	w12,w13,[$len]
+# else
+	stp	$d0,$d1,[$len]
+# endif
+
+	mov	x0,#1
+#else
+	mov	x0,#0
+#endif
+.Lno_key:
+	ret
+.size	poly1305_init,.-poly1305_init
+
+.type	poly1305_blocks,%function
+.align	5
+poly1305_blocks:
+	ands	$len,$len,#-16
+	b.eq	.Lno_data
+
+	ldp	$h0,$h1,[$ctx]		// load hash value
+	ldp	$r0,$r1,[$ctx,#32]	// load key value
+	ldr	$h2,[$ctx,#16]
+	add	$s1,$r1,$r1,lsr#2	// s1 = r1 + (r1 >> 2)
+	b	.Loop
+
+.align	5
+.Loop:
+	ldp	$t0,$t1,[$inp],#16	// load input
+	sub	$len,$len,#16
+#ifdef	__AARCH64EB__
+	rev	$t0,$t0
+	rev	$t1,$t1
+#endif
+	adds	$h0,$h0,$t0		// accumulate input
+	adcs	$h1,$h1,$t1
+
+	mul	$d0,$h0,$r0		// h0*r0
+	adc	$h2,$h2,$padbit
+	umulh	$d1,$h0,$r0
+
+	mul	$t0,$h1,$s1		// h1*5*r1
+	umulh	$t1,$h1,$s1
+
+	adds	$d0,$d0,$t0
+	mul	$t0,$h0,$r1		// h0*r1
+	adc	$d1,$d1,$t1
+	umulh	$d2,$h0,$r1
+
+	adds	$d1,$d1,$t0
+	mul	$t0,$h1,$r0		// h1*r0
+	adc	$d2,$d2,xzr
+	umulh	$t1,$h1,$r0
+
+	adds	$d1,$d1,$t0
+	mul	$t0,$h2,$s1		// h2*5*r1
+	adc	$d2,$d2,$t1
+	mul	$t1,$h2,$r0		// h2*r0
+
+	adds	$d1,$d1,$t0
+	adc	$d2,$d2,$t1
+
+	and	$t0,$d2,#-4		// final reduction
+	and	$h2,$d2,#3
+	add	$t0,$t0,$d2,lsr#2
+	adds	$h0,$d0,$t0
+	adcs	$h1,$d1,xzr
+	adc	$h2,$h2,xzr
+
+	cbnz	$len,.Loop
+
+	stp	$h0,$h1,[$ctx]		// store hash value
+	str	$h2,[$ctx,#16]
+
+.Lno_data:
+	ret
+.size	poly1305_blocks,.-poly1305_blocks
+
+.type	poly1305_emit,%function
+.align	5
+poly1305_emit:
+	ldp	$h0,$h1,[$ctx]		// load hash base 2^64
+	ldr	$h2,[$ctx,#16]
+	ldp	$t0,$t1,[$nonce]	// load nonce
+
+	adds	$d0,$h0,#5		// compare to modulus
+	adcs	$d1,$h1,xzr
+	adc	$d2,$h2,xzr
+
+	tst	$d2,#-4			// see if it's carried/borrowed
+
+	csel	$h0,$h0,$d0,eq
+	csel	$h1,$h1,$d1,eq
+
+#ifdef	__AARCH64EB__
+	ror	$t0,$t0,#32		// flip nonce words
+	ror	$t1,$t1,#32
+#endif
+	adds	$h0,$h0,$t0		// accumulate nonce
+	adc	$h1,$h1,$t1
+#ifdef	__AARCH64EB__
+	rev	$h0,$h0			// flip output bytes
+	rev	$h1,$h1
+#endif
+	stp	$h0,$h1,[$mac]		// write result
+
+	ret
+.size	poly1305_emit,.-poly1305_emit
+___
+my ($R0,$R1,$S1,$R2,$S2,$R3,$S3,$R4,$S4) = map("v$_.4s",(0..8));
+my ($IN01_0,$IN01_1,$IN01_2,$IN01_3,$IN01_4) = map("v$_.2s",(9..13));
+my ($IN23_0,$IN23_1,$IN23_2,$IN23_3,$IN23_4) = map("v$_.2s",(14..18));
+my ($ACC0,$ACC1,$ACC2,$ACC3,$ACC4) = map("v$_.2d",(19..23));
+my ($H0,$H1,$H2,$H3,$H4) = map("v$_.2s",(24..28));
+my ($T0,$T1,$MASK) = map("v$_",(29..31));
+
+my ($in2,$zeros)=("x16","x17");
+my $is_base2_26 = $zeros;		# borrow
+
+$code.=<<___;
+.type	__poly1305_mult,%function
+.align	5
+__poly1305_mult:
+	mul	$d0,$h0,$r0		// h0*r0
+	umulh	$d1,$h0,$r0
+
+	mul	$t0,$h1,$s1		// h1*5*r1
+	umulh	$t1,$h1,$s1
+
+	adds	$d0,$d0,$t0
+	mul	$t0,$h0,$r1		// h0*r1
+	adc	$d1,$d1,$t1
+	umulh	$d2,$h0,$r1
+
+	adds	$d1,$d1,$t0
+	mul	$t0,$h1,$r0		// h1*r0
+	adc	$d2,$d2,xzr
+	umulh	$t1,$h1,$r0
+
+	adds	$d1,$d1,$t0
+	mul	$t0,$h2,$s1		// h2*5*r1
+	adc	$d2,$d2,$t1
+	mul	$t1,$h2,$r0		// h2*r0
+
+	adds	$d1,$d1,$t0
+	adc	$d2,$d2,$t1
+
+	and	$t0,$d2,#-4		// final reduction
+	and	$h2,$d2,#3
+	add	$t0,$t0,$d2,lsr#2
+	adds	$h0,$d0,$t0
+	adcs	$h1,$d1,xzr
+	adc	$h2,$h2,xzr
+
+	ret
+.size	__poly1305_mult,.-__poly1305_mult
+
+.type	__poly1305_splat,%function
+.align	5
+__poly1305_splat:
+	and	x12,$h0,#0x03ffffff	// base 2^64 -> base 2^26
+	ubfx	x13,$h0,#26,#26
+	extr	x14,$h1,$h0,#52
+	and	x14,x14,#0x03ffffff
+	ubfx	x15,$h1,#14,#26
+	extr	x16,$h2,$h1,#40
+
+	str	w12,[$ctx,#16*0]	// r0
+	add	w12,w13,w13,lsl#2	// r1*5
+	str	w13,[$ctx,#16*1]	// r1
+	add	w13,w14,w14,lsl#2	// r2*5
+	str	w12,[$ctx,#16*2]	// s1
+	str	w14,[$ctx,#16*3]	// r2
+	add	w14,w15,w15,lsl#2	// r3*5
+	str	w13,[$ctx,#16*4]	// s2
+	str	w15,[$ctx,#16*5]	// r3
+	add	w15,w16,w16,lsl#2	// r4*5
+	str	w14,[$ctx,#16*6]	// s3
+	str	w16,[$ctx,#16*7]	// r4
+	str	w15,[$ctx,#16*8]	// s4
+
+	ret
+.size	__poly1305_splat,.-__poly1305_splat
+
+#if !defined(__KERNEL__) || defined(CONFIG_KERNEL_MODE_NEON)
+#ifdef	__KERNEL__
+.globl	poly1305_blocks_neon
+.globl	poly1305_emit_neon
+#endif
+
+.type	poly1305_blocks_neon,%function
+.align	5
+poly1305_blocks_neon:
+	ldr	$is_base2_26,[$ctx,#24]
+	cmp	$len,#128
+	b.hs	.Lblocks_neon
+	cbz	$is_base2_26,poly1305_blocks
+
+.Lblocks_neon:
+	stp	x29,x30,[sp,#-80]!
+	add	x29,sp,#0
+
+	ands	$len,$len,#-16
+	b.eq	.Lno_data_neon
+
+	cbz	$is_base2_26,.Lbase2_64_neon
+
+	ldp	w10,w11,[$ctx]		// load hash value base 2^26
+	ldp	w12,w13,[$ctx,#8]
+	ldr	w14,[$ctx,#16]
+
+	tst	$len,#31
+	b.eq	.Leven_neon
+
+	ldp	$r0,$r1,[$ctx,#32]	// load key value
+
+	add	$h0,x10,x11,lsl#26	// base 2^26 -> base 2^64
+	lsr	$h1,x12,#12
+	adds	$h0,$h0,x12,lsl#52
+	add	$h1,$h1,x13,lsl#14
+	adc	$h1,$h1,xzr
+	lsr	$h2,x14,#24
+	adds	$h1,$h1,x14,lsl#40
+	adc	$d2,$h2,xzr		// can be partially reduced...
+
+	ldp	$d0,$d1,[$inp],#16	// load input
+	sub	$len,$len,#16
+	add	$s1,$r1,$r1,lsr#2	// s1 = r1 + (r1 >> 2)
+
+	and	$t0,$d2,#-4		// ... so reduce
+	and	$h2,$d2,#3
+	add	$t0,$t0,$d2,lsr#2
+	adds	$h0,$h0,$t0
+	adcs	$h1,$h1,xzr
+	adc	$h2,$h2,xzr
+
+#ifdef	__AARCH64EB__
+	rev	$d0,$d0
+	rev	$d1,$d1
+#endif
+	adds	$h0,$h0,$d0		// accumulate input
+	adcs	$h1,$h1,$d1
+	adc	$h2,$h2,$padbit
+
+	bl	__poly1305_mult
+	ldr	x30,[sp,#8]
+
+	cbz	$padbit,.Lstore_base2_64_neon
+
+	and	x10,$h0,#0x03ffffff	// base 2^64 -> base 2^26
+	ubfx	x11,$h0,#26,#26
+	extr	x12,$h1,$h0,#52
+	and	x12,x12,#0x03ffffff
+	ubfx	x13,$h1,#14,#26
+	extr	x14,$h2,$h1,#40
+
+	cbnz	$len,.Leven_neon
+
+	stp	w10,w11,[$ctx]		// store hash value base 2^26
+	stp	w12,w13,[$ctx,#8]
+	str	w14,[$ctx,#16]
+	b	.Lno_data_neon
+
+.align	4
+.Lstore_base2_64_neon:
+	stp	$h0,$h1,[$ctx]		// store hash value base 2^64
+	stp	$h2,xzr,[$ctx,#16]	// note that is_base2_26 is zeroed
+	b	.Lno_data_neon
+
+.align	4
+.Lbase2_64_neon:
+	ldp	$r0,$r1,[$ctx,#32]	// load key value
+
+	ldp	$h0,$h1,[$ctx]		// load hash value base 2^64
+	ldr	$h2,[$ctx,#16]
+
+	tst	$len,#31
+	b.eq	.Linit_neon
+
+	ldp	$d0,$d1,[$inp],#16	// load input
+	sub	$len,$len,#16
+	add	$s1,$r1,$r1,lsr#2	// s1 = r1 + (r1 >> 2)
+#ifdef	__AARCH64EB__
+	rev	$d0,$d0
+	rev	$d1,$d1
+#endif
+	adds	$h0,$h0,$d0		// accumulate input
+	adcs	$h1,$h1,$d1
+	adc	$h2,$h2,$padbit
+
+	bl	__poly1305_mult
+
+.Linit_neon:
+	and	x10,$h0,#0x03ffffff	// base 2^64 -> base 2^26
+	ubfx	x11,$h0,#26,#26
+	extr	x12,$h1,$h0,#52
+	and	x12,x12,#0x03ffffff
+	ubfx	x13,$h1,#14,#26
+	extr	x14,$h2,$h1,#40
+
+	stp	d8,d9,[sp,#16]		// meet ABI requirements
+	stp	d10,d11,[sp,#32]
+	stp	d12,d13,[sp,#48]
+	stp	d14,d15,[sp,#64]
+
+	fmov	${H0},x10
+	fmov	${H1},x11
+	fmov	${H2},x12
+	fmov	${H3},x13
+	fmov	${H4},x14
+
+	////////////////////////////////// initialize r^n table
+	mov	$h0,$r0			// r^1
+	add	$s1,$r1,$r1,lsr#2	// s1 = r1 + (r1 >> 2)
+	mov	$h1,$r1
+	mov	$h2,xzr
+	add	$ctx,$ctx,#48+12
+	bl	__poly1305_splat
+
+	bl	__poly1305_mult		// r^2
+	sub	$ctx,$ctx,#4
+	bl	__poly1305_splat
+
+	bl	__poly1305_mult		// r^3
+	sub	$ctx,$ctx,#4
+	bl	__poly1305_splat
+
+	bl	__poly1305_mult		// r^4
+	sub	$ctx,$ctx,#4
+	bl	__poly1305_splat
+	ldr	x30,[sp,#8]
+
+	add	$in2,$inp,#32
+	adr	$zeros,.Lzeros
+	subs	$len,$len,#64
+	csel	$in2,$zeros,$in2,lo
+
+	mov	x4,#1
+	str	x4,[$ctx,#-24]		// set is_base2_26
+	sub	$ctx,$ctx,#48		// restore original $ctx
+	b	.Ldo_neon
+
+.align	4
+.Leven_neon:
+	add	$in2,$inp,#32
+	adr	$zeros,.Lzeros
+	subs	$len,$len,#64
+	csel	$in2,$zeros,$in2,lo
+
+	stp	d8,d9,[sp,#16]		// meet ABI requirements
+	stp	d10,d11,[sp,#32]
+	stp	d12,d13,[sp,#48]
+	stp	d14,d15,[sp,#64]
+
+	fmov	${H0},x10
+	fmov	${H1},x11
+	fmov	${H2},x12
+	fmov	${H3},x13
+	fmov	${H4},x14
+
+.Ldo_neon:
+	ldp	x8,x12,[$in2],#16	// inp[2:3] (or zero)
+	ldp	x9,x13,[$in2],#48
+
+	lsl	$padbit,$padbit,#24
+	add	x15,$ctx,#48
+
+#ifdef	__AARCH64EB__
+	rev	x8,x8
+	rev	x12,x12
+	rev	x9,x9
+	rev	x13,x13
+#endif
+	and	x4,x8,#0x03ffffff	// base 2^64 -> base 2^26
+	and	x5,x9,#0x03ffffff
+	ubfx	x6,x8,#26,#26
+	ubfx	x7,x9,#26,#26
+	add	x4,x4,x5,lsl#32		// bfi	x4,x5,#32,#32
+	extr	x8,x12,x8,#52
+	extr	x9,x13,x9,#52
+	add	x6,x6,x7,lsl#32		// bfi	x6,x7,#32,#32
+	fmov	$IN23_0,x4
+	and	x8,x8,#0x03ffffff
+	and	x9,x9,#0x03ffffff
+	ubfx	x10,x12,#14,#26
+	ubfx	x11,x13,#14,#26
+	add	x12,$padbit,x12,lsr#40
+	add	x13,$padbit,x13,lsr#40
+	add	x8,x8,x9,lsl#32		// bfi	x8,x9,#32,#32
+	fmov	$IN23_1,x6
+	add	x10,x10,x11,lsl#32	// bfi	x10,x11,#32,#32
+	add	x12,x12,x13,lsl#32	// bfi	x12,x13,#32,#32
+	fmov	$IN23_2,x8
+	fmov	$IN23_3,x10
+	fmov	$IN23_4,x12
+
+	ldp	x8,x12,[$inp],#16	// inp[0:1]
+	ldp	x9,x13,[$inp],#48
+
+	ld1	{$R0,$R1,$S1,$R2},[x15],#64
+	ld1	{$S2,$R3,$S3,$R4},[x15],#64
+	ld1	{$S4},[x15]
+
+#ifdef	__AARCH64EB__
+	rev	x8,x8
+	rev	x12,x12
+	rev	x9,x9
+	rev	x13,x13
+#endif
+	and	x4,x8,#0x03ffffff	// base 2^64 -> base 2^26
+	and	x5,x9,#0x03ffffff
+	ubfx	x6,x8,#26,#26
+	ubfx	x7,x9,#26,#26
+	add	x4,x4,x5,lsl#32		// bfi	x4,x5,#32,#32
+	extr	x8,x12,x8,#52
+	extr	x9,x13,x9,#52
+	add	x6,x6,x7,lsl#32		// bfi	x6,x7,#32,#32
+	fmov	$IN01_0,x4
+	and	x8,x8,#0x03ffffff
+	and	x9,x9,#0x03ffffff
+	ubfx	x10,x12,#14,#26
+	ubfx	x11,x13,#14,#26
+	add	x12,$padbit,x12,lsr#40
+	add	x13,$padbit,x13,lsr#40
+	add	x8,x8,x9,lsl#32		// bfi	x8,x9,#32,#32
+	fmov	$IN01_1,x6
+	add	x10,x10,x11,lsl#32	// bfi	x10,x11,#32,#32
+	add	x12,x12,x13,lsl#32	// bfi	x12,x13,#32,#32
+	movi	$MASK.2d,#-1
+	fmov	$IN01_2,x8
+	fmov	$IN01_3,x10
+	fmov	$IN01_4,x12
+	ushr	$MASK.2d,$MASK.2d,#38
+
+	b.ls	.Lskip_loop
+
+.align	4
+.Loop_neon:
+	////////////////////////////////////////////////////////////////
+	// ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
+	// ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
+	//   \___________________/
+	// ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
+	// ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
+	//   \___________________/ \____________________/
+	//
+	// Note that we start with inp[2:3]*r^2. This is because it
+	// doesn't depend on reduction in previous iteration.
+	////////////////////////////////////////////////////////////////
+	// d4 = h0*r4 + h1*r3   + h2*r2   + h3*r1   + h4*r0
+	// d3 = h0*r3 + h1*r2   + h2*r1   + h3*r0   + h4*5*r4
+	// d2 = h0*r2 + h1*r1   + h2*r0   + h3*5*r4 + h4*5*r3
+	// d1 = h0*r1 + h1*r0   + h2*5*r4 + h3*5*r3 + h4*5*r2
+	// d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
+
+	subs	$len,$len,#64
+	umull	$ACC4,$IN23_0,${R4}[2]
+	csel	$in2,$zeros,$in2,lo
+	umull	$ACC3,$IN23_0,${R3}[2]
+	umull	$ACC2,$IN23_0,${R2}[2]
+	 ldp	x8,x12,[$in2],#16	// inp[2:3] (or zero)
+	umull	$ACC1,$IN23_0,${R1}[2]
+	 ldp	x9,x13,[$in2],#48
+	umull	$ACC0,$IN23_0,${R0}[2]
+#ifdef	__AARCH64EB__
+	 rev	x8,x8
+	 rev	x12,x12
+	 rev	x9,x9
+	 rev	x13,x13
+#endif
+
+	umlal	$ACC4,$IN23_1,${R3}[2]
+	 and	x4,x8,#0x03ffffff	// base 2^64 -> base 2^26
+	umlal	$ACC3,$IN23_1,${R2}[2]
+	 and	x5,x9,#0x03ffffff
+	umlal	$ACC2,$IN23_1,${R1}[2]
+	 ubfx	x6,x8,#26,#26
+	umlal	$ACC1,$IN23_1,${R0}[2]
+	 ubfx	x7,x9,#26,#26
+	umlal	$ACC0,$IN23_1,${S4}[2]
+	 add	x4,x4,x5,lsl#32		// bfi	x4,x5,#32,#32
+
+	umlal	$ACC4,$IN23_2,${R2}[2]
+	 extr	x8,x12,x8,#52
+	umlal	$ACC3,$IN23_2,${R1}[2]
+	 extr	x9,x13,x9,#52
+	umlal	$ACC2,$IN23_2,${R0}[2]
+	 add	x6,x6,x7,lsl#32		// bfi	x6,x7,#32,#32
+	umlal	$ACC1,$IN23_2,${S4}[2]
+	 fmov	$IN23_0,x4
+	umlal	$ACC0,$IN23_2,${S3}[2]
+	 and	x8,x8,#0x03ffffff
+
+	umlal	$ACC4,$IN23_3,${R1}[2]
+	 and	x9,x9,#0x03ffffff
+	umlal	$ACC3,$IN23_3,${R0}[2]
+	 ubfx	x10,x12,#14,#26
+	umlal	$ACC2,$IN23_3,${S4}[2]
+	 ubfx	x11,x13,#14,#26
+	umlal	$ACC1,$IN23_3,${S3}[2]
+	 add	x8,x8,x9,lsl#32		// bfi	x8,x9,#32,#32
+	umlal	$ACC0,$IN23_3,${S2}[2]
+	 fmov	$IN23_1,x6
+
+	add	$IN01_2,$IN01_2,$H2
+	 add	x12,$padbit,x12,lsr#40
+	umlal	$ACC4,$IN23_4,${R0}[2]
+	 add	x13,$padbit,x13,lsr#40
+	umlal	$ACC3,$IN23_4,${S4}[2]
+	 add	x10,x10,x11,lsl#32	// bfi	x10,x11,#32,#32
+	umlal	$ACC2,$IN23_4,${S3}[2]
+	 add	x12,x12,x13,lsl#32	// bfi	x12,x13,#32,#32
+	umlal	$ACC1,$IN23_4,${S2}[2]
+	 fmov	$IN23_2,x8
+	umlal	$ACC0,$IN23_4,${S1}[2]
+	 fmov	$IN23_3,x10
+
+	////////////////////////////////////////////////////////////////
+	// (hash+inp[0:1])*r^4 and accumulate
+
+	add	$IN01_0,$IN01_0,$H0
+	 fmov	$IN23_4,x12
+	umlal	$ACC3,$IN01_2,${R1}[0]
+	 ldp	x8,x12,[$inp],#16	// inp[0:1]
+	umlal	$ACC0,$IN01_2,${S3}[0]
+	 ldp	x9,x13,[$inp],#48
+	umlal	$ACC4,$IN01_2,${R2}[0]
+	umlal	$ACC1,$IN01_2,${S4}[0]
+	umlal	$ACC2,$IN01_2,${R0}[0]
+#ifdef	__AARCH64EB__
+	 rev	x8,x8
+	 rev	x12,x12
+	 rev	x9,x9
+	 rev	x13,x13
+#endif
+
+	add	$IN01_1,$IN01_1,$H1
+	umlal	$ACC3,$IN01_0,${R3}[0]
+	umlal	$ACC4,$IN01_0,${R4}[0]
+	 and	x4,x8,#0x03ffffff	// base 2^64 -> base 2^26
+	umlal	$ACC2,$IN01_0,${R2}[0]
+	 and	x5,x9,#0x03ffffff
+	umlal	$ACC0,$IN01_0,${R0}[0]
+	 ubfx	x6,x8,#26,#26
+	umlal	$ACC1,$IN01_0,${R1}[0]
+	 ubfx	x7,x9,#26,#26
+
+	add	$IN01_3,$IN01_3,$H3
+	 add	x4,x4,x5,lsl#32		// bfi	x4,x5,#32,#32
+	umlal	$ACC3,$IN01_1,${R2}[0]
+	 extr	x8,x12,x8,#52
+	umlal	$ACC4,$IN01_1,${R3}[0]
+	 extr	x9,x13,x9,#52
+	umlal	$ACC0,$IN01_1,${S4}[0]
+	 add	x6,x6,x7,lsl#32		// bfi	x6,x7,#32,#32
+	umlal	$ACC2,$IN01_1,${R1}[0]
+	 fmov	$IN01_0,x4
+	umlal	$ACC1,$IN01_1,${R0}[0]
+	 and	x8,x8,#0x03ffffff
+
+	add	$IN01_4,$IN01_4,$H4
+	 and	x9,x9,#0x03ffffff
+	umlal	$ACC3,$IN01_3,${R0}[0]
+	 ubfx	x10,x12,#14,#26
+	umlal	$ACC0,$IN01_3,${S2}[0]
+	 ubfx	x11,x13,#14,#26
+	umlal	$ACC4,$IN01_3,${R1}[0]
+	 add	x8,x8,x9,lsl#32		// bfi	x8,x9,#32,#32
+	umlal	$ACC1,$IN01_3,${S3}[0]
+	 fmov	$IN01_1,x6
+	umlal	$ACC2,$IN01_3,${S4}[0]
+	 add	x12,$padbit,x12,lsr#40
+
+	umlal	$ACC3,$IN01_4,${S4}[0]
+	 add	x13,$padbit,x13,lsr#40
+	umlal	$ACC0,$IN01_4,${S1}[0]
+	 add	x10,x10,x11,lsl#32	// bfi	x10,x11,#32,#32
+	umlal	$ACC4,$IN01_4,${R0}[0]
+	 add	x12,x12,x13,lsl#32	// bfi	x12,x13,#32,#32
+	umlal	$ACC1,$IN01_4,${S2}[0]
+	 fmov	$IN01_2,x8
+	umlal	$ACC2,$IN01_4,${S3}[0]
+	 fmov	$IN01_3,x10
+	 fmov	$IN01_4,x12
+
+	/////////////////////////////////////////////////////////////////
+	// lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
+	// and P. Schwabe
+	//
+	// [see discussion in poly1305-armv4 module]
+
+	ushr	$T0.2d,$ACC3,#26
+	xtn	$H3,$ACC3
+	 ushr	$T1.2d,$ACC0,#26
+	 and	$ACC0,$ACC0,$MASK.2d
+	add	$ACC4,$ACC4,$T0.2d	// h3 -> h4
+	bic	$H3,#0xfc,lsl#24	// &=0x03ffffff
+	 add	$ACC1,$ACC1,$T1.2d	// h0 -> h1
+
+	ushr	$T0.2d,$ACC4,#26
+	xtn	$H4,$ACC4
+	 ushr	$T1.2d,$ACC1,#26
+	 xtn	$H1,$ACC1
+	bic	$H4,#0xfc,lsl#24
+	 add	$ACC2,$ACC2,$T1.2d	// h1 -> h2
+
+	add	$ACC0,$ACC0,$T0.2d
+	shl	$T0.2d,$T0.2d,#2
+	 shrn	$T1.2s,$ACC2,#26
+	 xtn	$H2,$ACC2
+	add	$ACC0,$ACC0,$T0.2d	// h4 -> h0
+	 bic	$H1,#0xfc,lsl#24
+	 add	$H3,$H3,$T1.2s		// h2 -> h3
+	 bic	$H2,#0xfc,lsl#24
+
+	shrn	$T0.2s,$ACC0,#26
+	xtn	$H0,$ACC0
+	 ushr	$T1.2s,$H3,#26
+	 bic	$H3,#0xfc,lsl#24
+	 bic	$H0,#0xfc,lsl#24
+	add	$H1,$H1,$T0.2s		// h0 -> h1
+	 add	$H4,$H4,$T1.2s		// h3 -> h4
+
+	b.hi	.Loop_neon
+
+.Lskip_loop:
+	dup	$IN23_2,${IN23_2}[0]
+	add	$IN01_2,$IN01_2,$H2
+
+	////////////////////////////////////////////////////////////////
+	// multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
+
+	adds	$len,$len,#32
+	b.ne	.Long_tail
+
+	dup	$IN23_2,${IN01_2}[0]
+	add	$IN23_0,$IN01_0,$H0
+	add	$IN23_3,$IN01_3,$H3
+	add	$IN23_1,$IN01_1,$H1
+	add	$IN23_4,$IN01_4,$H4
+
+.Long_tail:
+	dup	$IN23_0,${IN23_0}[0]
+	umull2	$ACC0,$IN23_2,${S3}
+	umull2	$ACC3,$IN23_2,${R1}
+	umull2	$ACC4,$IN23_2,${R2}
+	umull2	$ACC2,$IN23_2,${R0}
+	umull2	$ACC1,$IN23_2,${S4}
+
+	dup	$IN23_1,${IN23_1}[0]
+	umlal2	$ACC0,$IN23_0,${R0}
+	umlal2	$ACC2,$IN23_0,${R2}
+	umlal2	$ACC3,$IN23_0,${R3}
+	umlal2	$ACC4,$IN23_0,${R4}
+	umlal2	$ACC1,$IN23_0,${R1}
+
+	dup	$IN23_3,${IN23_3}[0]
+	umlal2	$ACC0,$IN23_1,${S4}
+	umlal2	$ACC3,$IN23_1,${R2}
+	umlal2	$ACC2,$IN23_1,${R1}
+	umlal2	$ACC4,$IN23_1,${R3}
+	umlal2	$ACC1,$IN23_1,${R0}
+
+	dup	$IN23_4,${IN23_4}[0]
+	umlal2	$ACC3,$IN23_3,${R0}
+	umlal2	$ACC4,$IN23_3,${R1}
+	umlal2	$ACC0,$IN23_3,${S2}
+	umlal2	$ACC1,$IN23_3,${S3}
+	umlal2	$ACC2,$IN23_3,${S4}
+
+	umlal2	$ACC3,$IN23_4,${S4}
+	umlal2	$ACC0,$IN23_4,${S1}
+	umlal2	$ACC4,$IN23_4,${R0}
+	umlal2	$ACC1,$IN23_4,${S2}
+	umlal2	$ACC2,$IN23_4,${S3}
+
+	b.eq	.Lshort_tail
+
+	////////////////////////////////////////////////////////////////
+	// (hash+inp[0:1])*r^4:r^3 and accumulate
+
+	add	$IN01_0,$IN01_0,$H0
+	umlal	$ACC3,$IN01_2,${R1}
+	umlal	$ACC0,$IN01_2,${S3}
+	umlal	$ACC4,$IN01_2,${R2}
+	umlal	$ACC1,$IN01_2,${S4}
+	umlal	$ACC2,$IN01_2,${R0}
+
+	add	$IN01_1,$IN01_1,$H1
+	umlal	$ACC3,$IN01_0,${R3}
+	umlal	$ACC0,$IN01_0,${R0}
+	umlal	$ACC4,$IN01_0,${R4}
+	umlal	$ACC1,$IN01_0,${R1}
+	umlal	$ACC2,$IN01_0,${R2}
+
+	add	$IN01_3,$IN01_3,$H3
+	umlal	$ACC3,$IN01_1,${R2}
+	umlal	$ACC0,$IN01_1,${S4}
+	umlal	$ACC4,$IN01_1,${R3}
+	umlal	$ACC1,$IN01_1,${R0}
+	umlal	$ACC2,$IN01_1,${R1}
+
+	add	$IN01_4,$IN01_4,$H4
+	umlal	$ACC3,$IN01_3,${R0}
+	umlal	$ACC0,$IN01_3,${S2}
+	umlal	$ACC4,$IN01_3,${R1}
+	umlal	$ACC1,$IN01_3,${S3}
+	umlal	$ACC2,$IN01_3,${S4}
+
+	umlal	$ACC3,$IN01_4,${S4}
+	umlal	$ACC0,$IN01_4,${S1}
+	umlal	$ACC4,$IN01_4,${R0}
+	umlal	$ACC1,$IN01_4,${S2}
+	umlal	$ACC2,$IN01_4,${S3}
+
+.Lshort_tail:
+	////////////////////////////////////////////////////////////////
+	// horizontal add
+
+	addp	$ACC3,$ACC3,$ACC3
+	 ldp	d8,d9,[sp,#16]		// meet ABI requirements
+	addp	$ACC0,$ACC0,$ACC0
+	 ldp	d10,d11,[sp,#32]
+	addp	$ACC4,$ACC4,$ACC4
+	 ldp	d12,d13,[sp,#48]
+	addp	$ACC1,$ACC1,$ACC1
+	 ldp	d14,d15,[sp,#64]
+	addp	$ACC2,$ACC2,$ACC2
+
+	////////////////////////////////////////////////////////////////
+	// lazy reduction, but without narrowing
+
+	ushr	$T0.2d,$ACC3,#26
+	and	$ACC3,$ACC3,$MASK.2d
+	 ushr	$T1.2d,$ACC0,#26
+	 and	$ACC0,$ACC0,$MASK.2d
+
+	add	$ACC4,$ACC4,$T0.2d	// h3 -> h4
+	 add	$ACC1,$ACC1,$T1.2d	// h0 -> h1
+
+	ushr	$T0.2d,$ACC4,#26
+	and	$ACC4,$ACC4,$MASK.2d
+	 ushr	$T1.2d,$ACC1,#26
+	 and	$ACC1,$ACC1,$MASK.2d
+	 add	$ACC2,$ACC2,$T1.2d	// h1 -> h2
+
+	add	$ACC0,$ACC0,$T0.2d
+	shl	$T0.2d,$T0.2d,#2
+	 ushr	$T1.2d,$ACC2,#26
+	 and	$ACC2,$ACC2,$MASK.2d
+	add	$ACC0,$ACC0,$T0.2d	// h4 -> h0
+	 add	$ACC3,$ACC3,$T1.2d	// h2 -> h3
+
+	ushr	$T0.2d,$ACC0,#26
+	and	$ACC0,$ACC0,$MASK.2d
+	 ushr	$T1.2d,$ACC3,#26
+	 and	$ACC3,$ACC3,$MASK.2d
+	add	$ACC1,$ACC1,$T0.2d	// h0 -> h1
+	 add	$ACC4,$ACC4,$T1.2d	// h3 -> h4
+
+	////////////////////////////////////////////////////////////////
+	// write the result, can be partially reduced
+
+	st4	{$ACC0,$ACC1,$ACC2,$ACC3}[0],[$ctx],#16
+	st1	{$ACC4}[0],[$ctx]
+
+.Lno_data_neon:
+	ldr	x29,[sp],#80
+	ret
+.size	poly1305_blocks_neon,.-poly1305_blocks_neon
+
+.type	poly1305_emit_neon,%function
+.align	5
+poly1305_emit_neon:
+	ldr	$is_base2_26,[$ctx,#24]
+	cbz	$is_base2_26,poly1305_emit
+
+	ldp	w10,w11,[$ctx]		// load hash value base 2^26
+	ldp	w12,w13,[$ctx,#8]
+	ldr	w14,[$ctx,#16]
+
+	add	$h0,x10,x11,lsl#26	// base 2^26 -> base 2^64
+	lsr	$h1,x12,#12
+	adds	$h0,$h0,x12,lsl#52
+	add	$h1,$h1,x13,lsl#14
+	adc	$h1,$h1,xzr
+	lsr	$h2,x14,#24
+	adds	$h1,$h1,x14,lsl#40
+	adc	$h2,$h2,xzr		// can be partially reduced...
+
+	ldp	$t0,$t1,[$nonce]	// load nonce
+
+	and	$d0,$h2,#-4		// ... so reduce
+	add	$d0,$d0,$h2,lsr#2
+	and	$h2,$h2,#3
+	adds	$h0,$h0,$d0
+	adcs	$h1,$h1,xzr
+	adc	$h2,$h2,xzr
+
+	adds	$d0,$h0,#5		// compare to modulus
+	adcs	$d1,$h1,xzr
+	adc	$d2,$h2,xzr
+
+	tst	$d2,#-4			// see if it's carried/borrowed
+
+	csel	$h0,$h0,$d0,eq
+	csel	$h1,$h1,$d1,eq
+
+#ifdef	__AARCH64EB__
+	ror	$t0,$t0,#32		// flip nonce words
+	ror	$t1,$t1,#32
+#endif
+	adds	$h0,$h0,$t0		// accumulate nonce
+	adc	$h1,$h1,$t1
+#ifdef	__AARCH64EB__
+	rev	$h0,$h0			// flip output bytes
+	rev	$h1,$h1
+#endif
+	stp	$h0,$h1,[$mac]		// write result
+
+	ret
+.size	poly1305_emit_neon,.-poly1305_emit_neon
+#endif
+
+.align	5
+.Lzeros:
+.long	0,0,0,0,0,0,0,0
+#ifndef __KERNEL__
+.LOPENSSL_armcap_P:
+#ifdef	__ILP32__
+.long	OPENSSL_armcap_P-.
+#else
+.quad	OPENSSL_armcap_P-.
+#endif
+#endif
+.align	2
+___
+
+open SELF,$0;
+while(<SELF>) {
+	next if (/^#!/);
+	last if (!s/^#/\/\// and !/^$/);
+	print;
+}
+close SELF;
+
+foreach (split("\n",$code)) {
+	s/\b(shrn\s+v[0-9]+)\.[24]d/$1.2s/			or
+	s/\b(fmov\s+)v([0-9]+)[^,]*,\s*x([0-9]+)/$1d$2,x$3/	or
+	(m/\bdup\b/ and (s/\.[24]s/.2d/g or 1))			or
+	(m/\b(eor|and)/ and (s/\.[248][sdh]/.16b/g or 1))	or
+	(m/\bum(ul|la)l\b/ and (s/\.4s/.2s/g or 1))		or
+	(m/\bum(ul|la)l2\b/ and (s/\.2s/.4s/g or 1))		or
+	(m/\bst[1-4]\s+{[^}]+}\[/ and (s/\.[24]d/.s/g or 1));
+
+	s/\.[124]([sd])\[/.$1\[/;
+
+	print $_,"\n";
+}
+close STDOUT;
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/poly1305/poly1305-arm-glue.c b/net/wireguard/crypto/zinc/poly1305/poly1305-arm-glue.c
--- a/net/wireguard/crypto/zinc/poly1305/poly1305-arm-glue.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/poly1305/poly1305-arm-glue.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,140 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#include <asm/hwcap.h>
+#include <asm/neon.h>
+
+asmlinkage void poly1305_init_arm(void *ctx, const u8 key[16]);
+asmlinkage void poly1305_blocks_arm(void *ctx, const u8 *inp, const size_t len,
+				    const u32 padbit);
+asmlinkage void poly1305_emit_arm(void *ctx, u8 mac[16], const u32 nonce[4]);
+asmlinkage void poly1305_blocks_neon(void *ctx, const u8 *inp, const size_t len,
+				     const u32 padbit);
+asmlinkage void poly1305_emit_neon(void *ctx, u8 mac[16], const u32 nonce[4]);
+
+static bool poly1305_use_neon __ro_after_init;
+static bool *const poly1305_nobs[] __initconst = { &poly1305_use_neon };
+
+static void __init poly1305_fpu_init(void)
+{
+#if defined(CONFIG_ZINC_ARCH_ARM64)
+	poly1305_use_neon = cpu_have_named_feature(ASIMD);
+#elif defined(CONFIG_ZINC_ARCH_ARM)
+	poly1305_use_neon = elf_hwcap & HWCAP_NEON;
+#endif
+}
+
+#if defined(CONFIG_ZINC_ARCH_ARM64)
+struct poly1305_arch_internal {
+	union {
+		u32 h[5];
+		struct {
+			u64 h0, h1, h2;
+		};
+	};
+	u64 is_base2_26;
+	u64 r[2];
+};
+#elif defined(CONFIG_ZINC_ARCH_ARM)
+struct poly1305_arch_internal {
+	union {
+		u32 h[5];
+		struct {
+			u64 h0, h1;
+			u32 h2;
+		} __packed;
+	};
+	u32 r[4];
+	u32 is_base2_26;
+};
+#endif
+
+/* The NEON code uses base 2^26, while the scalar code uses base 2^64 on 64-bit
+ * and base 2^32 on 32-bit. If we hit the unfortunate situation of using NEON
+ * and then having to go back to scalar -- because the user is silly and has
+ * called the update function from two separate contexts -- then we need to
+ * convert back to the original base before proceeding. The below function is
+ * written for 64-bit integers, and so we have to swap words at the end on
+ * big-endian 32-bit. It is possible to reason that the initial reduction below
+ * is sufficient given the implementation invariants. However, for an avoidance
+ * of doubt and because this is not performance critical, we do the full
+ * reduction anyway.
+ */
+static void convert_to_base2_64(void *ctx)
+{
+	struct poly1305_arch_internal *state = ctx;
+	u32 cy;
+
+	if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !state->is_base2_26)
+		return;
+
+	cy = state->h[0] >> 26; state->h[0] &= 0x3ffffff; state->h[1] += cy;
+	cy = state->h[1] >> 26; state->h[1] &= 0x3ffffff; state->h[2] += cy;
+	cy = state->h[2] >> 26; state->h[2] &= 0x3ffffff; state->h[3] += cy;
+	cy = state->h[3] >> 26; state->h[3] &= 0x3ffffff; state->h[4] += cy;
+	state->h0 = ((u64)state->h[2] << 52) | ((u64)state->h[1] << 26) | state->h[0];
+	state->h1 = ((u64)state->h[4] << 40) | ((u64)state->h[3] << 14) | (state->h[2] >> 12);
+	state->h2 = state->h[4] >> 24;
+	if (IS_ENABLED(CONFIG_ZINC_ARCH_ARM) && IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) {
+		state->h0 = rol64(state->h0, 32);
+		state->h1 = rol64(state->h1, 32);
+	}
+#define ULT(a, b) ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1))
+	cy = (state->h2 >> 2) + (state->h2 & ~3ULL);
+	state->h2 &= 3;
+	state->h0 += cy;
+	state->h1 += (cy = ULT(state->h0, cy));
+	state->h2 += ULT(state->h1, cy);
+#undef ULT
+	state->is_base2_26 = 0;
+}
+
+static inline bool poly1305_init_arch(void *ctx,
+				      const u8 key[POLY1305_KEY_SIZE])
+{
+	poly1305_init_arm(ctx, key);
+	return true;
+}
+
+static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp,
+					size_t len, const u32 padbit,
+					simd_context_t *simd_context)
+{
+	/* SIMD disables preemption, so relax after processing each page. */
+	BUILD_BUG_ON(PAGE_SIZE < POLY1305_BLOCK_SIZE ||
+		     PAGE_SIZE % POLY1305_BLOCK_SIZE);
+
+	if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !poly1305_use_neon ||
+	    !simd_use(simd_context)) {
+		convert_to_base2_64(ctx);
+		poly1305_blocks_arm(ctx, inp, len, padbit);
+		return true;
+	}
+
+	for (;;) {
+		const size_t bytes = min_t(size_t, len, PAGE_SIZE);
+
+		poly1305_blocks_neon(ctx, inp, bytes, padbit);
+		len -= bytes;
+		if (!len)
+			break;
+		inp += bytes;
+		simd_relax(simd_context);
+	}
+	return true;
+}
+
+static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE],
+				      const u32 nonce[4],
+				      simd_context_t *simd_context)
+{
+	if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !poly1305_use_neon ||
+	    !simd_use(simd_context)) {
+		convert_to_base2_64(ctx);
+		poly1305_emit_arm(ctx, mac, nonce);
+	} else
+		poly1305_emit_neon(ctx, mac, nonce);
+	return true;
+}
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/poly1305/poly1305-arm.pl b/net/wireguard/crypto/zinc/poly1305/poly1305-arm.pl
--- a/net/wireguard/crypto/zinc/poly1305/poly1305-arm.pl	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/poly1305/poly1305-arm.pl	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,1276 @@
+#!/usr/bin/env perl
+# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+#
+# This code is taken from the OpenSSL project but the author, Andy Polyakov,
+# has relicensed it under the licenses specified in the SPDX header above.
+# The original headers, including the original license headers, are
+# included below for completeness.
+#
+# ====================================================================
+# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
+# project. The module is, however, dual licensed under OpenSSL and
+# CRYPTOGAMS licenses depending on where you obtain it. For further
+# details see http://www.openssl.org/~appro/cryptogams/.
+# ====================================================================
+#
+#			IALU(*)/gcc-4.4		NEON
+#
+# ARM11xx(ARMv6)	7.78/+100%		-
+# Cortex-A5		6.35/+130%		3.00
+# Cortex-A8		6.25/+115%		2.36
+# Cortex-A9		5.10/+95%		2.55
+# Cortex-A15		3.85/+85%		1.25(**)
+# Snapdragon S4		5.70/+100%		1.48(**)
+#
+# (*)	this is for -march=armv6, i.e. with bunch of ldrb loading data;
+# (**)	these are trade-off results, they can be improved by ~8% but at
+#	the cost of 15/12% regression on Cortex-A5/A7, it's even possible
+#	to improve Cortex-A9 result, but then A5/A7 loose more than 20%;
+
+$flavour = shift;
+if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; }
+else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} }
+
+if ($flavour && $flavour ne "void") {
+    $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
+    ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
+    ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
+    die "can't locate arm-xlate.pl";
+
+    open STDOUT,"| \"$^X\" $xlate $flavour $output";
+} else {
+    open STDOUT,">$output";
+}
+
+($ctx,$inp,$len,$padbit)=map("r$_",(0..3));
+
+$code.=<<___;
+#ifndef	__KERNEL__
+# include "arm_arch.h"
+#else
+# define __ARM_ARCH__ __LINUX_ARM_ARCH__
+# define __ARM_MAX_ARCH__ __LINUX_ARM_ARCH__
+# define poly1305_init   poly1305_init_arm
+# define poly1305_blocks poly1305_blocks_arm
+# define poly1305_emit   poly1305_emit_arm
+#endif
+
+.text
+#if defined(__thumb2__)
+.syntax	unified
+.thumb
+#else
+.code	32
+#endif
+
+.globl	poly1305_emit
+.globl	poly1305_blocks
+.globl	poly1305_init
+.type	poly1305_init,%function
+.align	5
+poly1305_init:
+.Lpoly1305_init:
+	stmdb	sp!,{r4-r11}
+
+	eor	r3,r3,r3
+	cmp	$inp,#0
+	str	r3,[$ctx,#0]		@ zero hash value
+	str	r3,[$ctx,#4]
+	str	r3,[$ctx,#8]
+	str	r3,[$ctx,#12]
+	str	r3,[$ctx,#16]
+	str	r3,[$ctx,#36]		@ is_base2_26
+	add	$ctx,$ctx,#20
+
+#ifdef	__thumb2__
+	it	eq
+#endif
+	moveq	r0,#0
+	beq	.Lno_key
+
+#if	__ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
+	adr	r11,.Lpoly1305_init
+	ldr	r12,.LOPENSSL_armcap
+#endif
+	ldrb	r4,[$inp,#0]
+	mov	r10,#0x0fffffff
+	ldrb	r5,[$inp,#1]
+	and	r3,r10,#-4		@ 0x0ffffffc
+	ldrb	r6,[$inp,#2]
+	ldrb	r7,[$inp,#3]
+	orr	r4,r4,r5,lsl#8
+	ldrb	r5,[$inp,#4]
+	orr	r4,r4,r6,lsl#16
+	ldrb	r6,[$inp,#5]
+	orr	r4,r4,r7,lsl#24
+	ldrb	r7,[$inp,#6]
+	and	r4,r4,r10
+
+#if	__ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
+	ldr	r12,[r11,r12]		@ OPENSSL_armcap_P
+# ifdef	__APPLE__
+	ldr	r12,[r12]
+# endif
+#endif
+	ldrb	r8,[$inp,#7]
+	orr	r5,r5,r6,lsl#8
+	ldrb	r6,[$inp,#8]
+	orr	r5,r5,r7,lsl#16
+	ldrb	r7,[$inp,#9]
+	orr	r5,r5,r8,lsl#24
+	ldrb	r8,[$inp,#10]
+	and	r5,r5,r3
+
+#if	__ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
+	tst	r12,#ARMV7_NEON		@ check for NEON
+# ifdef	__APPLE__
+	adr	r9,poly1305_blocks_neon
+	adr	r11,poly1305_blocks
+#  ifdef __thumb2__
+	it	ne
+#  endif
+	movne	r11,r9
+	adr	r12,poly1305_emit
+	adr	r10,poly1305_emit_neon
+#  ifdef __thumb2__
+	it	ne
+#  endif
+	movne	r12,r10
+# else
+#  ifdef __thumb2__
+	itete	eq
+#  endif
+	addeq	r12,r11,#(poly1305_emit-.Lpoly1305_init)
+	addne	r12,r11,#(poly1305_emit_neon-.Lpoly1305_init)
+	addeq	r11,r11,#(poly1305_blocks-.Lpoly1305_init)
+	addne	r11,r11,#(poly1305_blocks_neon-.Lpoly1305_init)
+# endif
+# ifdef	__thumb2__
+	orr	r12,r12,#1	@ thumb-ify address
+	orr	r11,r11,#1
+# endif
+#endif
+	ldrb	r9,[$inp,#11]
+	orr	r6,r6,r7,lsl#8
+	ldrb	r7,[$inp,#12]
+	orr	r6,r6,r8,lsl#16
+	ldrb	r8,[$inp,#13]
+	orr	r6,r6,r9,lsl#24
+	ldrb	r9,[$inp,#14]
+	and	r6,r6,r3
+
+	ldrb	r10,[$inp,#15]
+	orr	r7,r7,r8,lsl#8
+	str	r4,[$ctx,#0]
+	orr	r7,r7,r9,lsl#16
+	str	r5,[$ctx,#4]
+	orr	r7,r7,r10,lsl#24
+	str	r6,[$ctx,#8]
+	and	r7,r7,r3
+	str	r7,[$ctx,#12]
+#if	__ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
+	stmia	r2,{r11,r12}		@ fill functions table
+	mov	r0,#1
+#else
+	mov	r0,#0
+#endif
+.Lno_key:
+	ldmia	sp!,{r4-r11}
+#if	__ARM_ARCH__>=5
+	ret				@ bx	lr
+#else
+	tst	lr,#1
+	moveq	pc,lr			@ be binary compatible with V4, yet
+	bx	lr			@ interoperable with Thumb ISA:-)
+#endif
+.size	poly1305_init,.-poly1305_init
+___
+{
+my ($h0,$h1,$h2,$h3,$h4,$r0,$r1,$r2,$r3)=map("r$_",(4..12));
+my ($s1,$s2,$s3)=($r1,$r2,$r3);
+
+$code.=<<___;
+.type	poly1305_blocks,%function
+.align	5
+poly1305_blocks:
+.Lpoly1305_blocks:
+	stmdb	sp!,{r3-r11,lr}
+
+	ands	$len,$len,#-16
+	beq	.Lno_data
+
+	cmp	$padbit,#0
+	add	$len,$len,$inp		@ end pointer
+	sub	sp,sp,#32
+
+	ldmia	$ctx,{$h0-$r3}		@ load context
+
+	str	$ctx,[sp,#12]		@ offload stuff
+	mov	lr,$inp
+	str	$len,[sp,#16]
+	str	$r1,[sp,#20]
+	str	$r2,[sp,#24]
+	str	$r3,[sp,#28]
+	b	.Loop
+
+.Loop:
+#if __ARM_ARCH__<7
+	ldrb	r0,[lr],#16		@ load input
+# ifdef	__thumb2__
+	it	hi
+# endif
+	addhi	$h4,$h4,#1		@ 1<<128
+	ldrb	r1,[lr,#-15]
+	ldrb	r2,[lr,#-14]
+	ldrb	r3,[lr,#-13]
+	orr	r1,r0,r1,lsl#8
+	ldrb	r0,[lr,#-12]
+	orr	r2,r1,r2,lsl#16
+	ldrb	r1,[lr,#-11]
+	orr	r3,r2,r3,lsl#24
+	ldrb	r2,[lr,#-10]
+	adds	$h0,$h0,r3		@ accumulate input
+
+	ldrb	r3,[lr,#-9]
+	orr	r1,r0,r1,lsl#8
+	ldrb	r0,[lr,#-8]
+	orr	r2,r1,r2,lsl#16
+	ldrb	r1,[lr,#-7]
+	orr	r3,r2,r3,lsl#24
+	ldrb	r2,[lr,#-6]
+	adcs	$h1,$h1,r3
+
+	ldrb	r3,[lr,#-5]
+	orr	r1,r0,r1,lsl#8
+	ldrb	r0,[lr,#-4]
+	orr	r2,r1,r2,lsl#16
+	ldrb	r1,[lr,#-3]
+	orr	r3,r2,r3,lsl#24
+	ldrb	r2,[lr,#-2]
+	adcs	$h2,$h2,r3
+
+	ldrb	r3,[lr,#-1]
+	orr	r1,r0,r1,lsl#8
+	str	lr,[sp,#8]		@ offload input pointer
+	orr	r2,r1,r2,lsl#16
+	add	$s1,$r1,$r1,lsr#2
+	orr	r3,r2,r3,lsl#24
+#else
+	ldr	r0,[lr],#16		@ load input
+# ifdef	__thumb2__
+	it	hi
+# endif
+	addhi	$h4,$h4,#1		@ padbit
+	ldr	r1,[lr,#-12]
+	ldr	r2,[lr,#-8]
+	ldr	r3,[lr,#-4]
+# ifdef	__ARMEB__
+	rev	r0,r0
+	rev	r1,r1
+	rev	r2,r2
+	rev	r3,r3
+# endif
+	adds	$h0,$h0,r0		@ accumulate input
+	str	lr,[sp,#8]		@ offload input pointer
+	adcs	$h1,$h1,r1
+	add	$s1,$r1,$r1,lsr#2
+	adcs	$h2,$h2,r2
+#endif
+	add	$s2,$r2,$r2,lsr#2
+	adcs	$h3,$h3,r3
+	add	$s3,$r3,$r3,lsr#2
+
+	umull	r2,r3,$h1,$r0
+	 adc	$h4,$h4,#0
+	umull	r0,r1,$h0,$r0
+	umlal	r2,r3,$h4,$s1
+	umlal	r0,r1,$h3,$s1
+	ldr	$r1,[sp,#20]		@ reload $r1
+	umlal	r2,r3,$h2,$s3
+	umlal	r0,r1,$h1,$s3
+	umlal	r2,r3,$h3,$s2
+	umlal	r0,r1,$h2,$s2
+	umlal	r2,r3,$h0,$r1
+	str	r0,[sp,#0]		@ future $h0
+	 mul	r0,$s2,$h4
+	ldr	$r2,[sp,#24]		@ reload $r2
+	adds	r2,r2,r1		@ d1+=d0>>32
+	 eor	r1,r1,r1
+	adc	lr,r3,#0		@ future $h2
+	str	r2,[sp,#4]		@ future $h1
+
+	mul	r2,$s3,$h4
+	eor	r3,r3,r3
+	umlal	r0,r1,$h3,$s3
+	ldr	$r3,[sp,#28]		@ reload $r3
+	umlal	r2,r3,$h3,$r0
+	umlal	r0,r1,$h2,$r0
+	umlal	r2,r3,$h2,$r1
+	umlal	r0,r1,$h1,$r1
+	umlal	r2,r3,$h1,$r2
+	umlal	r0,r1,$h0,$r2
+	umlal	r2,r3,$h0,$r3
+	ldr	$h0,[sp,#0]
+	mul	$h4,$r0,$h4
+	ldr	$h1,[sp,#4]
+
+	adds	$h2,lr,r0		@ d2+=d1>>32
+	ldr	lr,[sp,#8]		@ reload input pointer
+	adc	r1,r1,#0
+	adds	$h3,r2,r1		@ d3+=d2>>32
+	ldr	r0,[sp,#16]		@ reload end pointer
+	adc	r3,r3,#0
+	add	$h4,$h4,r3		@ h4+=d3>>32
+
+	and	r1,$h4,#-4
+	and	$h4,$h4,#3
+	add	r1,r1,r1,lsr#2		@ *=5
+	adds	$h0,$h0,r1
+	adcs	$h1,$h1,#0
+	adcs	$h2,$h2,#0
+	adcs	$h3,$h3,#0
+	adc	$h4,$h4,#0
+
+	cmp	r0,lr			@ done yet?
+	bhi	.Loop
+
+	ldr	$ctx,[sp,#12]
+	add	sp,sp,#32
+	stmia	$ctx,{$h0-$h4}		@ store the result
+
+.Lno_data:
+#if	__ARM_ARCH__>=5
+	ldmia	sp!,{r3-r11,pc}
+#else
+	ldmia	sp!,{r3-r11,lr}
+	tst	lr,#1
+	moveq	pc,lr			@ be binary compatible with V4, yet
+	bx	lr			@ interoperable with Thumb ISA:-)
+#endif
+.size	poly1305_blocks,.-poly1305_blocks
+___
+}
+{
+my ($ctx,$mac,$nonce)=map("r$_",(0..2));
+my ($h0,$h1,$h2,$h3,$h4,$g0,$g1,$g2,$g3)=map("r$_",(3..11));
+my $g4=$h4;
+
+$code.=<<___;
+.type	poly1305_emit,%function
+.align	5
+poly1305_emit:
+	stmdb	sp!,{r4-r11}
+.Lpoly1305_emit_enter:
+
+	ldmia	$ctx,{$h0-$h4}
+	adds	$g0,$h0,#5		@ compare to modulus
+	adcs	$g1,$h1,#0
+	adcs	$g2,$h2,#0
+	adcs	$g3,$h3,#0
+	adc	$g4,$h4,#0
+	tst	$g4,#4			@ did it carry/borrow?
+
+#ifdef	__thumb2__
+	it	ne
+#endif
+	movne	$h0,$g0
+	ldr	$g0,[$nonce,#0]
+#ifdef	__thumb2__
+	it	ne
+#endif
+	movne	$h1,$g1
+	ldr	$g1,[$nonce,#4]
+#ifdef	__thumb2__
+	it	ne
+#endif
+	movne	$h2,$g2
+	ldr	$g2,[$nonce,#8]
+#ifdef	__thumb2__
+	it	ne
+#endif
+	movne	$h3,$g3
+	ldr	$g3,[$nonce,#12]
+
+	adds	$h0,$h0,$g0
+	adcs	$h1,$h1,$g1
+	adcs	$h2,$h2,$g2
+	adc	$h3,$h3,$g3
+
+#if __ARM_ARCH__>=7
+# ifdef __ARMEB__
+	rev	$h0,$h0
+	rev	$h1,$h1
+	rev	$h2,$h2
+	rev	$h3,$h3
+# endif
+	str	$h0,[$mac,#0]
+	str	$h1,[$mac,#4]
+	str	$h2,[$mac,#8]
+	str	$h3,[$mac,#12]
+#else
+	strb	$h0,[$mac,#0]
+	mov	$h0,$h0,lsr#8
+	strb	$h1,[$mac,#4]
+	mov	$h1,$h1,lsr#8
+	strb	$h2,[$mac,#8]
+	mov	$h2,$h2,lsr#8
+	strb	$h3,[$mac,#12]
+	mov	$h3,$h3,lsr#8
+
+	strb	$h0,[$mac,#1]
+	mov	$h0,$h0,lsr#8
+	strb	$h1,[$mac,#5]
+	mov	$h1,$h1,lsr#8
+	strb	$h2,[$mac,#9]
+	mov	$h2,$h2,lsr#8
+	strb	$h3,[$mac,#13]
+	mov	$h3,$h3,lsr#8
+
+	strb	$h0,[$mac,#2]
+	mov	$h0,$h0,lsr#8
+	strb	$h1,[$mac,#6]
+	mov	$h1,$h1,lsr#8
+	strb	$h2,[$mac,#10]
+	mov	$h2,$h2,lsr#8
+	strb	$h3,[$mac,#14]
+	mov	$h3,$h3,lsr#8
+
+	strb	$h0,[$mac,#3]
+	strb	$h1,[$mac,#7]
+	strb	$h2,[$mac,#11]
+	strb	$h3,[$mac,#15]
+#endif
+	ldmia	sp!,{r4-r11}
+#if	__ARM_ARCH__>=5
+	ret				@ bx	lr
+#else
+	tst	lr,#1
+	moveq	pc,lr			@ be binary compatible with V4, yet
+	bx	lr			@ interoperable with Thumb ISA:-)
+#endif
+.size	poly1305_emit,.-poly1305_emit
+___
+{
+my ($R0,$R1,$S1,$R2,$S2,$R3,$S3,$R4,$S4) = map("d$_",(0..9));
+my ($D0,$D1,$D2,$D3,$D4, $H0,$H1,$H2,$H3,$H4) = map("q$_",(5..14));
+my ($T0,$T1,$MASK) = map("q$_",(15,4,0));
+
+my ($in2,$zeros,$tbl0,$tbl1) = map("r$_",(4..7));
+
+$code.=<<___;
+#if (defined(__KERNEL__) && defined(CONFIG_KERNEL_MODE_NEON)) || (!defined(__KERNEL__) && __ARM_MAX_ARCH__>=7)
+.fpu	neon
+
+.type	poly1305_init_neon,%function
+.align	5
+poly1305_init_neon:
+.Lpoly1305_init_neon:
+	ldr	r4,[$ctx,#20]		@ load key base 2^32
+	ldr	r5,[$ctx,#24]
+	ldr	r6,[$ctx,#28]
+	ldr	r7,[$ctx,#32]
+
+	and	r2,r4,#0x03ffffff	@ base 2^32 -> base 2^26
+	mov	r3,r4,lsr#26
+	mov	r4,r5,lsr#20
+	orr	r3,r3,r5,lsl#6
+	mov	r5,r6,lsr#14
+	orr	r4,r4,r6,lsl#12
+	mov	r6,r7,lsr#8
+	orr	r5,r5,r7,lsl#18
+	and	r3,r3,#0x03ffffff
+	and	r4,r4,#0x03ffffff
+	and	r5,r5,#0x03ffffff
+
+	vdup.32	$R0,r2			@ r^1 in both lanes
+	add	r2,r3,r3,lsl#2		@ *5
+	vdup.32	$R1,r3
+	add	r3,r4,r4,lsl#2
+	vdup.32	$S1,r2
+	vdup.32	$R2,r4
+	add	r4,r5,r5,lsl#2
+	vdup.32	$S2,r3
+	vdup.32	$R3,r5
+	add	r5,r6,r6,lsl#2
+	vdup.32	$S3,r4
+	vdup.32	$R4,r6
+	vdup.32	$S4,r5
+
+	mov	$zeros,#2		@ counter
+
+.Lsquare_neon:
+	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+	@ d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
+	@ d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
+	@ d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
+	@ d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
+	@ d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
+
+	vmull.u32	$D0,$R0,${R0}[1]
+	vmull.u32	$D1,$R1,${R0}[1]
+	vmull.u32	$D2,$R2,${R0}[1]
+	vmull.u32	$D3,$R3,${R0}[1]
+	vmull.u32	$D4,$R4,${R0}[1]
+
+	vmlal.u32	$D0,$R4,${S1}[1]
+	vmlal.u32	$D1,$R0,${R1}[1]
+	vmlal.u32	$D2,$R1,${R1}[1]
+	vmlal.u32	$D3,$R2,${R1}[1]
+	vmlal.u32	$D4,$R3,${R1}[1]
+
+	vmlal.u32	$D0,$R3,${S2}[1]
+	vmlal.u32	$D1,$R4,${S2}[1]
+	vmlal.u32	$D3,$R1,${R2}[1]
+	vmlal.u32	$D2,$R0,${R2}[1]
+	vmlal.u32	$D4,$R2,${R2}[1]
+
+	vmlal.u32	$D0,$R2,${S3}[1]
+	vmlal.u32	$D3,$R0,${R3}[1]
+	vmlal.u32	$D1,$R3,${S3}[1]
+	vmlal.u32	$D2,$R4,${S3}[1]
+	vmlal.u32	$D4,$R1,${R3}[1]
+
+	vmlal.u32	$D3,$R4,${S4}[1]
+	vmlal.u32	$D0,$R1,${S4}[1]
+	vmlal.u32	$D1,$R2,${S4}[1]
+	vmlal.u32	$D2,$R3,${S4}[1]
+	vmlal.u32	$D4,$R0,${R4}[1]
+
+	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+	@ lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
+	@ and P. Schwabe
+	@
+	@ H0>>+H1>>+H2>>+H3>>+H4
+	@ H3>>+H4>>*5+H0>>+H1
+	@
+	@ Trivia.
+	@
+	@ Result of multiplication of n-bit number by m-bit number is
+	@ n+m bits wide. However! Even though 2^n is a n+1-bit number,
+	@ m-bit number multiplied by 2^n is still n+m bits wide.
+	@
+	@ Sum of two n-bit numbers is n+1 bits wide, sum of three - n+2,
+	@ and so is sum of four. Sum of 2^m n-m-bit numbers and n-bit
+	@ one is n+1 bits wide.
+	@
+	@ >>+ denotes Hnext += Hn>>26, Hn &= 0x3ffffff. This means that
+	@ H0, H2, H3 are guaranteed to be 26 bits wide, while H1 and H4
+	@ can be 27. However! In cases when their width exceeds 26 bits
+	@ they are limited by 2^26+2^6. This in turn means that *sum*
+	@ of the products with these values can still be viewed as sum
+	@ of 52-bit numbers as long as the amount of addends is not a
+	@ power of 2. For example,
+	@
+	@ H4 = H4*R0 + H3*R1 + H2*R2 + H1*R3 + H0 * R4,
+	@
+	@ which can't be larger than 5 * (2^26 + 2^6) * (2^26 + 2^6), or
+	@ 5 * (2^52 + 2*2^32 + 2^12), which in turn is smaller than
+	@ 8 * (2^52) or 2^55. However, the value is then multiplied by
+	@ by 5, so we should be looking at 5 * 5 * (2^52 + 2^33 + 2^12),
+	@ which is less than 32 * (2^52) or 2^57. And when processing
+	@ data we are looking at triple as many addends...
+	@
+	@ In key setup procedure pre-reduced H0 is limited by 5*4+1 and
+	@ 5*H4 - by 5*5 52-bit addends, or 57 bits. But when hashing the
+	@ input H0 is limited by (5*4+1)*3 addends, or 58 bits, while
+	@ 5*H4 by 5*5*3, or 59[!] bits. How is this relevant? vmlal.u32
+	@ instruction accepts 2x32-bit input and writes 2x64-bit result.
+	@ This means that result of reduction have to be compressed upon
+	@ loop wrap-around. This can be done in the process of reduction
+	@ to minimize amount of instructions [as well as amount of
+	@ 128-bit instructions, which benefits low-end processors], but
+	@ one has to watch for H2 (which is narrower than H0) and 5*H4
+	@ not being wider than 58 bits, so that result of right shift
+	@ by 26 bits fits in 32 bits. This is also useful on x86,
+	@ because it allows to use paddd in place for paddq, which
+	@ benefits Atom, where paddq is ridiculously slow.
+
+	vshr.u64	$T0,$D3,#26
+	vmovn.i64	$D3#lo,$D3
+	 vshr.u64	$T1,$D0,#26
+	 vmovn.i64	$D0#lo,$D0
+	vadd.i64	$D4,$D4,$T0		@ h3 -> h4
+	vbic.i32	$D3#lo,#0xfc000000	@ &=0x03ffffff
+	 vadd.i64	$D1,$D1,$T1		@ h0 -> h1
+	 vbic.i32	$D0#lo,#0xfc000000
+
+	vshrn.u64	$T0#lo,$D4,#26
+	vmovn.i64	$D4#lo,$D4
+	 vshr.u64	$T1,$D1,#26
+	 vmovn.i64	$D1#lo,$D1
+	 vadd.i64	$D2,$D2,$T1		@ h1 -> h2
+	vbic.i32	$D4#lo,#0xfc000000
+	 vbic.i32	$D1#lo,#0xfc000000
+
+	vadd.i32	$D0#lo,$D0#lo,$T0#lo
+	vshl.u32	$T0#lo,$T0#lo,#2
+	 vshrn.u64	$T1#lo,$D2,#26
+	 vmovn.i64	$D2#lo,$D2
+	vadd.i32	$D0#lo,$D0#lo,$T0#lo	@ h4 -> h0
+	 vadd.i32	$D3#lo,$D3#lo,$T1#lo	@ h2 -> h3
+	 vbic.i32	$D2#lo,#0xfc000000
+
+	vshr.u32	$T0#lo,$D0#lo,#26
+	vbic.i32	$D0#lo,#0xfc000000
+	 vshr.u32	$T1#lo,$D3#lo,#26
+	 vbic.i32	$D3#lo,#0xfc000000
+	vadd.i32	$D1#lo,$D1#lo,$T0#lo	@ h0 -> h1
+	 vadd.i32	$D4#lo,$D4#lo,$T1#lo	@ h3 -> h4
+
+	subs		$zeros,$zeros,#1
+	beq		.Lsquare_break_neon
+
+	add		$tbl0,$ctx,#(48+0*9*4)
+	add		$tbl1,$ctx,#(48+1*9*4)
+
+	vtrn.32		$R0,$D0#lo		@ r^2:r^1
+	vtrn.32		$R2,$D2#lo
+	vtrn.32		$R3,$D3#lo
+	vtrn.32		$R1,$D1#lo
+	vtrn.32		$R4,$D4#lo
+
+	vshl.u32	$S2,$R2,#2		@ *5
+	vshl.u32	$S3,$R3,#2
+	vshl.u32	$S1,$R1,#2
+	vshl.u32	$S4,$R4,#2
+	vadd.i32	$S2,$S2,$R2
+	vadd.i32	$S1,$S1,$R1
+	vadd.i32	$S3,$S3,$R3
+	vadd.i32	$S4,$S4,$R4
+
+	vst4.32		{${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!
+	vst4.32		{${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!
+	vst4.32		{${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
+	vst4.32		{${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
+	vst1.32		{${S4}[0]},[$tbl0,:32]
+	vst1.32		{${S4}[1]},[$tbl1,:32]
+
+	b		.Lsquare_neon
+
+.align	4
+.Lsquare_break_neon:
+	add		$tbl0,$ctx,#(48+2*4*9)
+	add		$tbl1,$ctx,#(48+3*4*9)
+
+	vmov		$R0,$D0#lo		@ r^4:r^3
+	vshl.u32	$S1,$D1#lo,#2		@ *5
+	vmov		$R1,$D1#lo
+	vshl.u32	$S2,$D2#lo,#2
+	vmov		$R2,$D2#lo
+	vshl.u32	$S3,$D3#lo,#2
+	vmov		$R3,$D3#lo
+	vshl.u32	$S4,$D4#lo,#2
+	vmov		$R4,$D4#lo
+	vadd.i32	$S1,$S1,$D1#lo
+	vadd.i32	$S2,$S2,$D2#lo
+	vadd.i32	$S3,$S3,$D3#lo
+	vadd.i32	$S4,$S4,$D4#lo
+
+	vst4.32		{${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!
+	vst4.32		{${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!
+	vst4.32		{${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
+	vst4.32		{${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
+	vst1.32		{${S4}[0]},[$tbl0]
+	vst1.32		{${S4}[1]},[$tbl1]
+
+	ret				@ bx	lr
+.size	poly1305_init_neon,.-poly1305_init_neon
+
+#ifdef __KERNEL__
+.globl	poly1305_blocks_neon
+#endif
+.type	poly1305_blocks_neon,%function
+.align	5
+poly1305_blocks_neon:
+	ldr	ip,[$ctx,#36]		@ is_base2_26
+	ands	$len,$len,#-16
+	beq	.Lno_data_neon
+
+	cmp	$len,#64
+	bhs	.Lenter_neon
+	tst	ip,ip			@ is_base2_26?
+	beq	.Lpoly1305_blocks
+
+.Lenter_neon:
+	stmdb	sp!,{r4-r7}
+	vstmdb	sp!,{d8-d15}		@ ABI specification says so
+
+	tst	ip,ip			@ is_base2_26?
+	bne	.Lbase2_26_neon
+
+	stmdb	sp!,{r1-r3,lr}
+	bl	.Lpoly1305_init_neon
+
+	ldr	r4,[$ctx,#0]		@ load hash value base 2^32
+	ldr	r5,[$ctx,#4]
+	ldr	r6,[$ctx,#8]
+	ldr	r7,[$ctx,#12]
+	ldr	ip,[$ctx,#16]
+
+	and	r2,r4,#0x03ffffff	@ base 2^32 -> base 2^26
+	mov	r3,r4,lsr#26
+	 veor	$D0#lo,$D0#lo,$D0#lo
+	mov	r4,r5,lsr#20
+	orr	r3,r3,r5,lsl#6
+	 veor	$D1#lo,$D1#lo,$D1#lo
+	mov	r5,r6,lsr#14
+	orr	r4,r4,r6,lsl#12
+	 veor	$D2#lo,$D2#lo,$D2#lo
+	mov	r6,r7,lsr#8
+	orr	r5,r5,r7,lsl#18
+	 veor	$D3#lo,$D3#lo,$D3#lo
+	and	r3,r3,#0x03ffffff
+	orr	r6,r6,ip,lsl#24
+	 veor	$D4#lo,$D4#lo,$D4#lo
+	and	r4,r4,#0x03ffffff
+	mov	r1,#1
+	and	r5,r5,#0x03ffffff
+	str	r1,[$ctx,#36]		@ is_base2_26
+
+	vmov.32	$D0#lo[0],r2
+	vmov.32	$D1#lo[0],r3
+	vmov.32	$D2#lo[0],r4
+	vmov.32	$D3#lo[0],r5
+	vmov.32	$D4#lo[0],r6
+	adr	$zeros,.Lzeros
+
+	ldmia	sp!,{r1-r3,lr}
+	b	.Lbase2_32_neon
+
+.align	4
+.Lbase2_26_neon:
+	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+	@ load hash value
+
+	veor		$D0#lo,$D0#lo,$D0#lo
+	veor		$D1#lo,$D1#lo,$D1#lo
+	veor		$D2#lo,$D2#lo,$D2#lo
+	veor		$D3#lo,$D3#lo,$D3#lo
+	veor		$D4#lo,$D4#lo,$D4#lo
+	vld4.32		{$D0#lo[0],$D1#lo[0],$D2#lo[0],$D3#lo[0]},[$ctx]!
+	adr		$zeros,.Lzeros
+	vld1.32		{$D4#lo[0]},[$ctx]
+	sub		$ctx,$ctx,#16		@ rewind
+
+.Lbase2_32_neon:
+	add		$in2,$inp,#32
+	mov		$padbit,$padbit,lsl#24
+	tst		$len,#31
+	beq		.Leven
+
+	vld4.32		{$H0#lo[0],$H1#lo[0],$H2#lo[0],$H3#lo[0]},[$inp]!
+	vmov.32		$H4#lo[0],$padbit
+	sub		$len,$len,#16
+	add		$in2,$inp,#32
+
+# ifdef	__ARMEB__
+	vrev32.8	$H0,$H0
+	vrev32.8	$H3,$H3
+	vrev32.8	$H1,$H1
+	vrev32.8	$H2,$H2
+# endif
+	vsri.u32	$H4#lo,$H3#lo,#8	@ base 2^32 -> base 2^26
+	vshl.u32	$H3#lo,$H3#lo,#18
+
+	vsri.u32	$H3#lo,$H2#lo,#14
+	vshl.u32	$H2#lo,$H2#lo,#12
+	vadd.i32	$H4#hi,$H4#lo,$D4#lo	@ add hash value and move to #hi
+
+	vbic.i32	$H3#lo,#0xfc000000
+	vsri.u32	$H2#lo,$H1#lo,#20
+	vshl.u32	$H1#lo,$H1#lo,#6
+
+	vbic.i32	$H2#lo,#0xfc000000
+	vsri.u32	$H1#lo,$H0#lo,#26
+	vadd.i32	$H3#hi,$H3#lo,$D3#lo
+
+	vbic.i32	$H0#lo,#0xfc000000
+	vbic.i32	$H1#lo,#0xfc000000
+	vadd.i32	$H2#hi,$H2#lo,$D2#lo
+
+	vadd.i32	$H0#hi,$H0#lo,$D0#lo
+	vadd.i32	$H1#hi,$H1#lo,$D1#lo
+
+	mov		$tbl1,$zeros
+	add		$tbl0,$ctx,#48
+
+	cmp		$len,$len
+	b		.Long_tail
+
+.align	4
+.Leven:
+	subs		$len,$len,#64
+	it		lo
+	movlo		$in2,$zeros
+
+	vmov.i32	$H4,#1<<24		@ padbit, yes, always
+	vld4.32		{$H0#lo,$H1#lo,$H2#lo,$H3#lo},[$inp]	@ inp[0:1]
+	add		$inp,$inp,#64
+	vld4.32		{$H0#hi,$H1#hi,$H2#hi,$H3#hi},[$in2]	@ inp[2:3] (or 0)
+	add		$in2,$in2,#64
+	itt		hi
+	addhi		$tbl1,$ctx,#(48+1*9*4)
+	addhi		$tbl0,$ctx,#(48+3*9*4)
+
+# ifdef	__ARMEB__
+	vrev32.8	$H0,$H0
+	vrev32.8	$H3,$H3
+	vrev32.8	$H1,$H1
+	vrev32.8	$H2,$H2
+# endif
+	vsri.u32	$H4,$H3,#8		@ base 2^32 -> base 2^26
+	vshl.u32	$H3,$H3,#18
+
+	vsri.u32	$H3,$H2,#14
+	vshl.u32	$H2,$H2,#12
+
+	vbic.i32	$H3,#0xfc000000
+	vsri.u32	$H2,$H1,#20
+	vshl.u32	$H1,$H1,#6
+
+	vbic.i32	$H2,#0xfc000000
+	vsri.u32	$H1,$H0,#26
+
+	vbic.i32	$H0,#0xfc000000
+	vbic.i32	$H1,#0xfc000000
+
+	bls		.Lskip_loop
+
+	vld4.32		{${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!	@ load r^2
+	vld4.32		{${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!	@ load r^4
+	vld4.32		{${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
+	vld4.32		{${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
+	b		.Loop_neon
+
+.align	5
+.Loop_neon:
+	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+	@ ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
+	@ ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
+	@   \___________________/
+	@ ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
+	@ ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
+	@   \___________________/ \____________________/
+	@
+	@ Note that we start with inp[2:3]*r^2. This is because it
+	@ doesn't depend on reduction in previous iteration.
+	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+	@ d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
+	@ d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
+	@ d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
+	@ d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
+	@ d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
+
+	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+	@ inp[2:3]*r^2
+
+	vadd.i32	$H2#lo,$H2#lo,$D2#lo	@ accumulate inp[0:1]
+	vmull.u32	$D2,$H2#hi,${R0}[1]
+	vadd.i32	$H0#lo,$H0#lo,$D0#lo
+	vmull.u32	$D0,$H0#hi,${R0}[1]
+	vadd.i32	$H3#lo,$H3#lo,$D3#lo
+	vmull.u32	$D3,$H3#hi,${R0}[1]
+	vmlal.u32	$D2,$H1#hi,${R1}[1]
+	vadd.i32	$H1#lo,$H1#lo,$D1#lo
+	vmull.u32	$D1,$H1#hi,${R0}[1]
+
+	vadd.i32	$H4#lo,$H4#lo,$D4#lo
+	vmull.u32	$D4,$H4#hi,${R0}[1]
+	subs		$len,$len,#64
+	vmlal.u32	$D0,$H4#hi,${S1}[1]
+	it		lo
+	movlo		$in2,$zeros
+	vmlal.u32	$D3,$H2#hi,${R1}[1]
+	vld1.32		${S4}[1],[$tbl1,:32]
+	vmlal.u32	$D1,$H0#hi,${R1}[1]
+	vmlal.u32	$D4,$H3#hi,${R1}[1]
+
+	vmlal.u32	$D0,$H3#hi,${S2}[1]
+	vmlal.u32	$D3,$H1#hi,${R2}[1]
+	vmlal.u32	$D4,$H2#hi,${R2}[1]
+	vmlal.u32	$D1,$H4#hi,${S2}[1]
+	vmlal.u32	$D2,$H0#hi,${R2}[1]
+
+	vmlal.u32	$D3,$H0#hi,${R3}[1]
+	vmlal.u32	$D0,$H2#hi,${S3}[1]
+	vmlal.u32	$D4,$H1#hi,${R3}[1]
+	vmlal.u32	$D1,$H3#hi,${S3}[1]
+	vmlal.u32	$D2,$H4#hi,${S3}[1]
+
+	vmlal.u32	$D3,$H4#hi,${S4}[1]
+	vmlal.u32	$D0,$H1#hi,${S4}[1]
+	vmlal.u32	$D4,$H0#hi,${R4}[1]
+	vmlal.u32	$D1,$H2#hi,${S4}[1]
+	vmlal.u32	$D2,$H3#hi,${S4}[1]
+
+	vld4.32		{$H0#hi,$H1#hi,$H2#hi,$H3#hi},[$in2]	@ inp[2:3] (or 0)
+	add		$in2,$in2,#64
+
+	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+	@ (hash+inp[0:1])*r^4 and accumulate
+
+	vmlal.u32	$D3,$H3#lo,${R0}[0]
+	vmlal.u32	$D0,$H0#lo,${R0}[0]
+	vmlal.u32	$D4,$H4#lo,${R0}[0]
+	vmlal.u32	$D1,$H1#lo,${R0}[0]
+	vmlal.u32	$D2,$H2#lo,${R0}[0]
+	vld1.32		${S4}[0],[$tbl0,:32]
+
+	vmlal.u32	$D3,$H2#lo,${R1}[0]
+	vmlal.u32	$D0,$H4#lo,${S1}[0]
+	vmlal.u32	$D4,$H3#lo,${R1}[0]
+	vmlal.u32	$D1,$H0#lo,${R1}[0]
+	vmlal.u32	$D2,$H1#lo,${R1}[0]
+
+	vmlal.u32	$D3,$H1#lo,${R2}[0]
+	vmlal.u32	$D0,$H3#lo,${S2}[0]
+	vmlal.u32	$D4,$H2#lo,${R2}[0]
+	vmlal.u32	$D1,$H4#lo,${S2}[0]
+	vmlal.u32	$D2,$H0#lo,${R2}[0]
+
+	vmlal.u32	$D3,$H0#lo,${R3}[0]
+	vmlal.u32	$D0,$H2#lo,${S3}[0]
+	vmlal.u32	$D4,$H1#lo,${R3}[0]
+	vmlal.u32	$D1,$H3#lo,${S3}[0]
+	vmlal.u32	$D3,$H4#lo,${S4}[0]
+
+	vmlal.u32	$D2,$H4#lo,${S3}[0]
+	vmlal.u32	$D0,$H1#lo,${S4}[0]
+	vmlal.u32	$D4,$H0#lo,${R4}[0]
+	vmov.i32	$H4,#1<<24		@ padbit, yes, always
+	vmlal.u32	$D1,$H2#lo,${S4}[0]
+	vmlal.u32	$D2,$H3#lo,${S4}[0]
+
+	vld4.32		{$H0#lo,$H1#lo,$H2#lo,$H3#lo},[$inp]	@ inp[0:1]
+	add		$inp,$inp,#64
+# ifdef	__ARMEB__
+	vrev32.8	$H0,$H0
+	vrev32.8	$H1,$H1
+	vrev32.8	$H2,$H2
+	vrev32.8	$H3,$H3
+# endif
+
+	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+	@ lazy reduction interleaved with base 2^32 -> base 2^26 of
+	@ inp[0:3] previously loaded to $H0-$H3 and smashed to $H0-$H4.
+
+	vshr.u64	$T0,$D3,#26
+	vmovn.i64	$D3#lo,$D3
+	 vshr.u64	$T1,$D0,#26
+	 vmovn.i64	$D0#lo,$D0
+	vadd.i64	$D4,$D4,$T0		@ h3 -> h4
+	vbic.i32	$D3#lo,#0xfc000000
+	  vsri.u32	$H4,$H3,#8		@ base 2^32 -> base 2^26
+	 vadd.i64	$D1,$D1,$T1		@ h0 -> h1
+	  vshl.u32	$H3,$H3,#18
+	 vbic.i32	$D0#lo,#0xfc000000
+
+	vshrn.u64	$T0#lo,$D4,#26
+	vmovn.i64	$D4#lo,$D4
+	 vshr.u64	$T1,$D1,#26
+	 vmovn.i64	$D1#lo,$D1
+	 vadd.i64	$D2,$D2,$T1		@ h1 -> h2
+	  vsri.u32	$H3,$H2,#14
+	vbic.i32	$D4#lo,#0xfc000000
+	  vshl.u32	$H2,$H2,#12
+	 vbic.i32	$D1#lo,#0xfc000000
+
+	vadd.i32	$D0#lo,$D0#lo,$T0#lo
+	vshl.u32	$T0#lo,$T0#lo,#2
+	  vbic.i32	$H3,#0xfc000000
+	 vshrn.u64	$T1#lo,$D2,#26
+	 vmovn.i64	$D2#lo,$D2
+	vaddl.u32	$D0,$D0#lo,$T0#lo	@ h4 -> h0 [widen for a sec]
+	  vsri.u32	$H2,$H1,#20
+	 vadd.i32	$D3#lo,$D3#lo,$T1#lo	@ h2 -> h3
+	  vshl.u32	$H1,$H1,#6
+	 vbic.i32	$D2#lo,#0xfc000000
+	  vbic.i32	$H2,#0xfc000000
+
+	vshrn.u64	$T0#lo,$D0,#26		@ re-narrow
+	vmovn.i64	$D0#lo,$D0
+	  vsri.u32	$H1,$H0,#26
+	  vbic.i32	$H0,#0xfc000000
+	 vshr.u32	$T1#lo,$D3#lo,#26
+	 vbic.i32	$D3#lo,#0xfc000000
+	vbic.i32	$D0#lo,#0xfc000000
+	vadd.i32	$D1#lo,$D1#lo,$T0#lo	@ h0 -> h1
+	 vadd.i32	$D4#lo,$D4#lo,$T1#lo	@ h3 -> h4
+	  vbic.i32	$H1,#0xfc000000
+
+	bhi		.Loop_neon
+
+.Lskip_loop:
+	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+	@ multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
+
+	add		$tbl1,$ctx,#(48+0*9*4)
+	add		$tbl0,$ctx,#(48+1*9*4)
+	adds		$len,$len,#32
+	it		ne
+	movne		$len,#0
+	bne		.Long_tail
+
+	vadd.i32	$H2#hi,$H2#lo,$D2#lo	@ add hash value and move to #hi
+	vadd.i32	$H0#hi,$H0#lo,$D0#lo
+	vadd.i32	$H3#hi,$H3#lo,$D3#lo
+	vadd.i32	$H1#hi,$H1#lo,$D1#lo
+	vadd.i32	$H4#hi,$H4#lo,$D4#lo
+
+.Long_tail:
+	vld4.32		{${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!	@ load r^1
+	vld4.32		{${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!	@ load r^2
+
+	vadd.i32	$H2#lo,$H2#lo,$D2#lo	@ can be redundant
+	vmull.u32	$D2,$H2#hi,$R0
+	vadd.i32	$H0#lo,$H0#lo,$D0#lo
+	vmull.u32	$D0,$H0#hi,$R0
+	vadd.i32	$H3#lo,$H3#lo,$D3#lo
+	vmull.u32	$D3,$H3#hi,$R0
+	vadd.i32	$H1#lo,$H1#lo,$D1#lo
+	vmull.u32	$D1,$H1#hi,$R0
+	vadd.i32	$H4#lo,$H4#lo,$D4#lo
+	vmull.u32	$D4,$H4#hi,$R0
+
+	vmlal.u32	$D0,$H4#hi,$S1
+	vld4.32		{${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
+	vmlal.u32	$D3,$H2#hi,$R1
+	vld4.32		{${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
+	vmlal.u32	$D1,$H0#hi,$R1
+	vmlal.u32	$D4,$H3#hi,$R1
+	vmlal.u32	$D2,$H1#hi,$R1
+
+	vmlal.u32	$D3,$H1#hi,$R2
+	vld1.32		${S4}[1],[$tbl1,:32]
+	vmlal.u32	$D0,$H3#hi,$S2
+	vld1.32		${S4}[0],[$tbl0,:32]
+	vmlal.u32	$D4,$H2#hi,$R2
+	vmlal.u32	$D1,$H4#hi,$S2
+	vmlal.u32	$D2,$H0#hi,$R2
+
+	vmlal.u32	$D3,$H0#hi,$R3
+	 it		ne
+	 addne		$tbl1,$ctx,#(48+2*9*4)
+	vmlal.u32	$D0,$H2#hi,$S3
+	 it		ne
+	 addne		$tbl0,$ctx,#(48+3*9*4)
+	vmlal.u32	$D4,$H1#hi,$R3
+	vmlal.u32	$D1,$H3#hi,$S3
+	vmlal.u32	$D2,$H4#hi,$S3
+
+	vmlal.u32	$D3,$H4#hi,$S4
+	 vorn		$MASK,$MASK,$MASK	@ all-ones, can be redundant
+	vmlal.u32	$D0,$H1#hi,$S4
+	 vshr.u64	$MASK,$MASK,#38
+	vmlal.u32	$D4,$H0#hi,$R4
+	vmlal.u32	$D1,$H2#hi,$S4
+	vmlal.u32	$D2,$H3#hi,$S4
+
+	beq		.Lshort_tail
+
+	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+	@ (hash+inp[0:1])*r^4:r^3 and accumulate
+
+	vld4.32		{${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!	@ load r^3
+	vld4.32		{${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!	@ load r^4
+
+	vmlal.u32	$D2,$H2#lo,$R0
+	vmlal.u32	$D0,$H0#lo,$R0
+	vmlal.u32	$D3,$H3#lo,$R0
+	vmlal.u32	$D1,$H1#lo,$R0
+	vmlal.u32	$D4,$H4#lo,$R0
+
+	vmlal.u32	$D0,$H4#lo,$S1
+	vld4.32		{${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
+	vmlal.u32	$D3,$H2#lo,$R1
+	vld4.32		{${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
+	vmlal.u32	$D1,$H0#lo,$R1
+	vmlal.u32	$D4,$H3#lo,$R1
+	vmlal.u32	$D2,$H1#lo,$R1
+
+	vmlal.u32	$D3,$H1#lo,$R2
+	vld1.32		${S4}[1],[$tbl1,:32]
+	vmlal.u32	$D0,$H3#lo,$S2
+	vld1.32		${S4}[0],[$tbl0,:32]
+	vmlal.u32	$D4,$H2#lo,$R2
+	vmlal.u32	$D1,$H4#lo,$S2
+	vmlal.u32	$D2,$H0#lo,$R2
+
+	vmlal.u32	$D3,$H0#lo,$R3
+	vmlal.u32	$D0,$H2#lo,$S3
+	vmlal.u32	$D4,$H1#lo,$R3
+	vmlal.u32	$D1,$H3#lo,$S3
+	vmlal.u32	$D2,$H4#lo,$S3
+
+	vmlal.u32	$D3,$H4#lo,$S4
+	 vorn		$MASK,$MASK,$MASK	@ all-ones
+	vmlal.u32	$D0,$H1#lo,$S4
+	 vshr.u64	$MASK,$MASK,#38
+	vmlal.u32	$D4,$H0#lo,$R4
+	vmlal.u32	$D1,$H2#lo,$S4
+	vmlal.u32	$D2,$H3#lo,$S4
+
+.Lshort_tail:
+	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+	@ horizontal addition
+
+	vadd.i64	$D3#lo,$D3#lo,$D3#hi
+	vadd.i64	$D0#lo,$D0#lo,$D0#hi
+	vadd.i64	$D4#lo,$D4#lo,$D4#hi
+	vadd.i64	$D1#lo,$D1#lo,$D1#hi
+	vadd.i64	$D2#lo,$D2#lo,$D2#hi
+
+	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+	@ lazy reduction, but without narrowing
+
+	vshr.u64	$T0,$D3,#26
+	vand.i64	$D3,$D3,$MASK
+	 vshr.u64	$T1,$D0,#26
+	 vand.i64	$D0,$D0,$MASK
+	vadd.i64	$D4,$D4,$T0		@ h3 -> h4
+	 vadd.i64	$D1,$D1,$T1		@ h0 -> h1
+
+	vshr.u64	$T0,$D4,#26
+	vand.i64	$D4,$D4,$MASK
+	 vshr.u64	$T1,$D1,#26
+	 vand.i64	$D1,$D1,$MASK
+	 vadd.i64	$D2,$D2,$T1		@ h1 -> h2
+
+	vadd.i64	$D0,$D0,$T0
+	vshl.u64	$T0,$T0,#2
+	 vshr.u64	$T1,$D2,#26
+	 vand.i64	$D2,$D2,$MASK
+	vadd.i64	$D0,$D0,$T0		@ h4 -> h0
+	 vadd.i64	$D3,$D3,$T1		@ h2 -> h3
+
+	vshr.u64	$T0,$D0,#26
+	vand.i64	$D0,$D0,$MASK
+	 vshr.u64	$T1,$D3,#26
+	 vand.i64	$D3,$D3,$MASK
+	vadd.i64	$D1,$D1,$T0		@ h0 -> h1
+	 vadd.i64	$D4,$D4,$T1		@ h3 -> h4
+
+	cmp		$len,#0
+	bne		.Leven
+
+	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+	@ store hash value
+
+	vst4.32		{$D0#lo[0],$D1#lo[0],$D2#lo[0],$D3#lo[0]},[$ctx]!
+	vst1.32		{$D4#lo[0]},[$ctx]
+
+	vldmia	sp!,{d8-d15}			@ epilogue
+	ldmia	sp!,{r4-r7}
+.Lno_data_neon:
+	ret					@ bx	lr
+.size	poly1305_blocks_neon,.-poly1305_blocks_neon
+
+#ifdef __KERNEL__
+.globl	poly1305_emit_neon
+#endif
+.type	poly1305_emit_neon,%function
+.align	5
+poly1305_emit_neon:
+	ldr	ip,[$ctx,#36]		@ is_base2_26
+
+	stmdb	sp!,{r4-r11}
+
+	tst	ip,ip
+	beq	.Lpoly1305_emit_enter
+
+	ldmia	$ctx,{$h0-$h4}
+	eor	$g0,$g0,$g0
+
+	adds	$h0,$h0,$h1,lsl#26	@ base 2^26 -> base 2^32
+	mov	$h1,$h1,lsr#6
+	adcs	$h1,$h1,$h2,lsl#20
+	mov	$h2,$h2,lsr#12
+	adcs	$h2,$h2,$h3,lsl#14
+	mov	$h3,$h3,lsr#18
+	adcs	$h3,$h3,$h4,lsl#8
+	adc	$h4,$g0,$h4,lsr#24	@ can be partially reduced ...
+
+	and	$g0,$h4,#-4		@ ... so reduce
+	and	$h4,$h3,#3
+	add	$g0,$g0,$g0,lsr#2	@ *= 5
+	adds	$h0,$h0,$g0
+	adcs	$h1,$h1,#0
+	adcs	$h2,$h2,#0
+	adcs	$h3,$h3,#0
+	adc	$h4,$h4,#0
+
+	adds	$g0,$h0,#5		@ compare to modulus
+	adcs	$g1,$h1,#0
+	adcs	$g2,$h2,#0
+	adcs	$g3,$h3,#0
+	adc	$g4,$h4,#0
+	tst	$g4,#4			@ did it carry/borrow?
+
+	it	ne
+	movne	$h0,$g0
+	ldr	$g0,[$nonce,#0]
+	it	ne
+	movne	$h1,$g1
+	ldr	$g1,[$nonce,#4]
+	it	ne
+	movne	$h2,$g2
+	ldr	$g2,[$nonce,#8]
+	it	ne
+	movne	$h3,$g3
+	ldr	$g3,[$nonce,#12]
+
+	adds	$h0,$h0,$g0		@ accumulate nonce
+	adcs	$h1,$h1,$g1
+	adcs	$h2,$h2,$g2
+	adc	$h3,$h3,$g3
+
+# ifdef __ARMEB__
+	rev	$h0,$h0
+	rev	$h1,$h1
+	rev	$h2,$h2
+	rev	$h3,$h3
+# endif
+	str	$h0,[$mac,#0]		@ store the result
+	str	$h1,[$mac,#4]
+	str	$h2,[$mac,#8]
+	str	$h3,[$mac,#12]
+
+	ldmia	sp!,{r4-r11}
+	ret				@ bx	lr
+.size	poly1305_emit_neon,.-poly1305_emit_neon
+
+.align	5
+.Lzeros:
+.long	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+# ifndef __KERNEL__
+.LOPENSSL_armcap:
+.word	OPENSSL_armcap_P-.Lpoly1305_init
+# endif
+#endif
+___
+}	}
+$code.=<<___;
+.align	2
+#if	__ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
+.comm   OPENSSL_armcap_P,4,4
+#endif
+___
+
+open SELF,$0;
+while(<SELF>) {
+	next if (/^#!/);
+	last if (!s/^#/@/ and !/^$/);
+	print;
+}
+close SELF;
+
+foreach (split("\n",$code)) {
+	s/\`([^\`]*)\`/eval $1/geo;
+
+	s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo	or
+	s/\bret\b/bx	lr/go						or
+	s/\bbx\s+lr\b/.word\t0xe12fff1e/go;	# make it possible to compile with -march=armv4
+
+	print $_,"\n";
+}
+close STDOUT; # enforce flush
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/poly1305/poly1305.c b/net/wireguard/crypto/zinc/poly1305/poly1305.c
--- a/net/wireguard/crypto/zinc/poly1305/poly1305.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/poly1305/poly1305.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,168 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ *
+ * Implementation of the Poly1305 message authenticator.
+ *
+ * Information: https://cr.yp.to/mac.html
+ */
+
+#include <zinc/poly1305.h>
+#include "../selftest/run.h"
+
+#include <asm/unaligned.h>
+#include <linux/kernel.h>
+#include <linux/string.h>
+#include <linux/module.h>
+#include <linux/init.h>
+
+#if defined(CONFIG_ZINC_ARCH_X86_64)
+#include "poly1305-x86_64-glue.c"
+#elif defined(CONFIG_ZINC_ARCH_ARM) || defined(CONFIG_ZINC_ARCH_ARM64)
+#include "poly1305-arm-glue.c"
+#elif defined(CONFIG_ZINC_ARCH_MIPS) || defined(CONFIG_ZINC_ARCH_MIPS64)
+#include "poly1305-mips-glue.c"
+#else
+static inline bool poly1305_init_arch(void *ctx,
+				      const u8 key[POLY1305_KEY_SIZE])
+{
+	return false;
+}
+static inline bool poly1305_blocks_arch(void *ctx, const u8 *input,
+					size_t len, const u32 padbit,
+					simd_context_t *simd_context)
+{
+	return false;
+}
+static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE],
+				      const u32 nonce[4],
+				      simd_context_t *simd_context)
+{
+	return false;
+}
+static bool *const poly1305_nobs[] __initconst = { };
+static void __init poly1305_fpu_init(void)
+{
+}
+#endif
+
+#if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__)
+#include "poly1305-donna64.c"
+#else
+#include "poly1305-donna32.c"
+#endif
+
+void poly1305_init(struct poly1305_ctx *ctx, const u8 key[POLY1305_KEY_SIZE])
+{
+	ctx->nonce[0] = get_unaligned_le32(&key[16]);
+	ctx->nonce[1] = get_unaligned_le32(&key[20]);
+	ctx->nonce[2] = get_unaligned_le32(&key[24]);
+	ctx->nonce[3] = get_unaligned_le32(&key[28]);
+
+	if (!poly1305_init_arch(ctx->opaque, key))
+		poly1305_init_generic(ctx->opaque, key);
+
+	ctx->num = 0;
+}
+EXPORT_SYMBOL(poly1305_init);
+
+static inline void poly1305_blocks(void *ctx, const u8 *input, const size_t len,
+				   const u32 padbit,
+				   simd_context_t *simd_context)
+{
+	if (!poly1305_blocks_arch(ctx, input, len, padbit, simd_context))
+		poly1305_blocks_generic(ctx, input, len, padbit);
+}
+
+static inline void poly1305_emit(void *ctx, u8 mac[POLY1305_KEY_SIZE],
+				 const u32 nonce[4],
+				 simd_context_t *simd_context)
+{
+	if (!poly1305_emit_arch(ctx, mac, nonce, simd_context))
+		poly1305_emit_generic(ctx, mac, nonce);
+}
+
+void poly1305_update(struct poly1305_ctx *ctx, const u8 *input, size_t len,
+		     simd_context_t *simd_context)
+{
+	const size_t num = ctx->num;
+	size_t rem;
+
+	if (num) {
+		rem = POLY1305_BLOCK_SIZE - num;
+		if (len < rem) {
+			memcpy(ctx->data + num, input, len);
+			ctx->num = num + len;
+			return;
+		}
+		memcpy(ctx->data + num, input, rem);
+		poly1305_blocks(ctx->opaque, ctx->data, POLY1305_BLOCK_SIZE, 1,
+				simd_context);
+		input += rem;
+		len -= rem;
+	}
+
+	rem = len % POLY1305_BLOCK_SIZE;
+	len -= rem;
+
+	if (len >= POLY1305_BLOCK_SIZE) {
+		poly1305_blocks(ctx->opaque, input, len, 1, simd_context);
+		input += len;
+	}
+
+	if (rem)
+		memcpy(ctx->data, input, rem);
+
+	ctx->num = rem;
+}
+EXPORT_SYMBOL(poly1305_update);
+
+void poly1305_final(struct poly1305_ctx *ctx, u8 mac[POLY1305_MAC_SIZE],
+		    simd_context_t *simd_context)
+{
+	size_t num = ctx->num;
+
+	if (num) {
+		ctx->data[num++] = 1;
+		while (num < POLY1305_BLOCK_SIZE)
+			ctx->data[num++] = 0;
+		poly1305_blocks(ctx->opaque, ctx->data, POLY1305_BLOCK_SIZE, 0,
+				simd_context);
+	}
+
+	poly1305_emit(ctx->opaque, mac, ctx->nonce, simd_context);
+
+	memzero_explicit(ctx, sizeof(*ctx));
+}
+EXPORT_SYMBOL(poly1305_final);
+
+#include "../selftest/poly1305.c"
+
+static bool nosimd __initdata = false;
+
+#ifndef COMPAT_ZINC_IS_A_MODULE
+int __init poly1305_mod_init(void)
+#else
+static int __init mod_init(void)
+#endif
+{
+	if (!nosimd)
+		poly1305_fpu_init();
+	if (!selftest_run("poly1305", poly1305_selftest, poly1305_nobs,
+			  ARRAY_SIZE(poly1305_nobs)))
+		return -ENOTRECOVERABLE;
+	return 0;
+}
+
+#ifdef COMPAT_ZINC_IS_A_MODULE
+static void __exit mod_exit(void)
+{
+}
+
+module_param(nosimd, bool, 0);
+module_init(mod_init);
+module_exit(mod_exit);
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("Poly1305 one-time authenticator");
+MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");
+#endif
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/poly1305/poly1305-donna32.c b/net/wireguard/crypto/zinc/poly1305/poly1305-donna32.c
--- a/net/wireguard/crypto/zinc/poly1305/poly1305-donna32.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/poly1305/poly1305-donna32.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,205 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ *
+ * This is based in part on Andrew Moon's poly1305-donna, which is in the
+ * public domain.
+ */
+
+struct poly1305_internal {
+	u32 h[5];
+	u32 r[5];
+	u32 s[4];
+};
+
+static void poly1305_init_generic(void *ctx, const u8 key[16])
+{
+	struct poly1305_internal *st = (struct poly1305_internal *)ctx;
+
+	/* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
+	st->r[0] = (get_unaligned_le32(&key[0])) & 0x3ffffff;
+	st->r[1] = (get_unaligned_le32(&key[3]) >> 2) & 0x3ffff03;
+	st->r[2] = (get_unaligned_le32(&key[6]) >> 4) & 0x3ffc0ff;
+	st->r[3] = (get_unaligned_le32(&key[9]) >> 6) & 0x3f03fff;
+	st->r[4] = (get_unaligned_le32(&key[12]) >> 8) & 0x00fffff;
+
+	/* s = 5*r */
+	st->s[0] = st->r[1] * 5;
+	st->s[1] = st->r[2] * 5;
+	st->s[2] = st->r[3] * 5;
+	st->s[3] = st->r[4] * 5;
+
+	/* h = 0 */
+	st->h[0] = 0;
+	st->h[1] = 0;
+	st->h[2] = 0;
+	st->h[3] = 0;
+	st->h[4] = 0;
+}
+
+static void poly1305_blocks_generic(void *ctx, const u8 *input, size_t len,
+				    const u32 padbit)
+{
+	struct poly1305_internal *st = (struct poly1305_internal *)ctx;
+	const u32 hibit = padbit << 24;
+	u32 r0, r1, r2, r3, r4;
+	u32 s1, s2, s3, s4;
+	u32 h0, h1, h2, h3, h4;
+	u64 d0, d1, d2, d3, d4;
+	u32 c;
+
+	r0 = st->r[0];
+	r1 = st->r[1];
+	r2 = st->r[2];
+	r3 = st->r[3];
+	r4 = st->r[4];
+
+	s1 = st->s[0];
+	s2 = st->s[1];
+	s3 = st->s[2];
+	s4 = st->s[3];
+
+	h0 = st->h[0];
+	h1 = st->h[1];
+	h2 = st->h[2];
+	h3 = st->h[3];
+	h4 = st->h[4];
+
+	while (len >= POLY1305_BLOCK_SIZE) {
+		/* h += m[i] */
+		h0 += (get_unaligned_le32(&input[0])) & 0x3ffffff;
+		h1 += (get_unaligned_le32(&input[3]) >> 2) & 0x3ffffff;
+		h2 += (get_unaligned_le32(&input[6]) >> 4) & 0x3ffffff;
+		h3 += (get_unaligned_le32(&input[9]) >> 6) & 0x3ffffff;
+		h4 += (get_unaligned_le32(&input[12]) >> 8) | hibit;
+
+		/* h *= r */
+		d0 = ((u64)h0 * r0) + ((u64)h1 * s4) +
+		     ((u64)h2 * s3) + ((u64)h3 * s2) +
+		     ((u64)h4 * s1);
+		d1 = ((u64)h0 * r1) + ((u64)h1 * r0) +
+		     ((u64)h2 * s4) + ((u64)h3 * s3) +
+		     ((u64)h4 * s2);
+		d2 = ((u64)h0 * r2) + ((u64)h1 * r1) +
+		     ((u64)h2 * r0) + ((u64)h3 * s4) +
+		     ((u64)h4 * s3);
+		d3 = ((u64)h0 * r3) + ((u64)h1 * r2) +
+		     ((u64)h2 * r1) + ((u64)h3 * r0) +
+		     ((u64)h4 * s4);
+		d4 = ((u64)h0 * r4) + ((u64)h1 * r3) +
+		     ((u64)h2 * r2) + ((u64)h3 * r1) +
+		     ((u64)h4 * r0);
+
+		/* (partial) h %= p */
+		c = (u32)(d0 >> 26);
+		h0 = (u32)d0 & 0x3ffffff;
+		d1 += c;
+		c = (u32)(d1 >> 26);
+		h1 = (u32)d1 & 0x3ffffff;
+		d2 += c;
+		c = (u32)(d2 >> 26);
+		h2 = (u32)d2 & 0x3ffffff;
+		d3 += c;
+		c = (u32)(d3 >> 26);
+		h3 = (u32)d3 & 0x3ffffff;
+		d4 += c;
+		c = (u32)(d4 >> 26);
+		h4 = (u32)d4 & 0x3ffffff;
+		h0 += c * 5;
+		c = (h0 >> 26);
+		h0 = h0 & 0x3ffffff;
+		h1 += c;
+
+		input += POLY1305_BLOCK_SIZE;
+		len -= POLY1305_BLOCK_SIZE;
+	}
+
+	st->h[0] = h0;
+	st->h[1] = h1;
+	st->h[2] = h2;
+	st->h[3] = h3;
+	st->h[4] = h4;
+}
+
+static void poly1305_emit_generic(void *ctx, u8 mac[16], const u32 nonce[4])
+{
+	struct poly1305_internal *st = (struct poly1305_internal *)ctx;
+	u32 h0, h1, h2, h3, h4, c;
+	u32 g0, g1, g2, g3, g4;
+	u64 f;
+	u32 mask;
+
+	/* fully carry h */
+	h0 = st->h[0];
+	h1 = st->h[1];
+	h2 = st->h[2];
+	h3 = st->h[3];
+	h4 = st->h[4];
+
+	c = h1 >> 26;
+	h1 = h1 & 0x3ffffff;
+	h2 += c;
+	c = h2 >> 26;
+	h2 = h2 & 0x3ffffff;
+	h3 += c;
+	c = h3 >> 26;
+	h3 = h3 & 0x3ffffff;
+	h4 += c;
+	c = h4 >> 26;
+	h4 = h4 & 0x3ffffff;
+	h0 += c * 5;
+	c = h0 >> 26;
+	h0 = h0 & 0x3ffffff;
+	h1 += c;
+
+	/* compute h + -p */
+	g0 = h0 + 5;
+	c = g0 >> 26;
+	g0 &= 0x3ffffff;
+	g1 = h1 + c;
+	c = g1 >> 26;
+	g1 &= 0x3ffffff;
+	g2 = h2 + c;
+	c = g2 >> 26;
+	g2 &= 0x3ffffff;
+	g3 = h3 + c;
+	c = g3 >> 26;
+	g3 &= 0x3ffffff;
+	g4 = h4 + c - (1UL << 26);
+
+	/* select h if h < p, or h + -p if h >= p */
+	mask = (g4 >> ((sizeof(u32) * 8) - 1)) - 1;
+	g0 &= mask;
+	g1 &= mask;
+	g2 &= mask;
+	g3 &= mask;
+	g4 &= mask;
+	mask = ~mask;
+
+	h0 = (h0 & mask) | g0;
+	h1 = (h1 & mask) | g1;
+	h2 = (h2 & mask) | g2;
+	h3 = (h3 & mask) | g3;
+	h4 = (h4 & mask) | g4;
+
+	/* h = h % (2^128) */
+	h0 = ((h0) | (h1 << 26)) & 0xffffffff;
+	h1 = ((h1 >> 6) | (h2 << 20)) & 0xffffffff;
+	h2 = ((h2 >> 12) | (h3 << 14)) & 0xffffffff;
+	h3 = ((h3 >> 18) | (h4 << 8)) & 0xffffffff;
+
+	/* mac = (h + nonce) % (2^128) */
+	f = (u64)h0 + nonce[0];
+	h0 = (u32)f;
+	f = (u64)h1 + nonce[1] + (f >> 32);
+	h1 = (u32)f;
+	f = (u64)h2 + nonce[2] + (f >> 32);
+	h2 = (u32)f;
+	f = (u64)h3 + nonce[3] + (f >> 32);
+	h3 = (u32)f;
+
+	put_unaligned_le32(h0, &mac[0]);
+	put_unaligned_le32(h1, &mac[4]);
+	put_unaligned_le32(h2, &mac[8]);
+	put_unaligned_le32(h3, &mac[12]);
+}
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/poly1305/poly1305-donna64.c b/net/wireguard/crypto/zinc/poly1305/poly1305-donna64.c
--- a/net/wireguard/crypto/zinc/poly1305/poly1305-donna64.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/poly1305/poly1305-donna64.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,182 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ *
+ * This is based in part on Andrew Moon's poly1305-donna, which is in the
+ * public domain.
+ */
+
+typedef __uint128_t u128;
+
+struct poly1305_internal {
+	u64 r[3];
+	u64 h[3];
+	u64 s[2];
+};
+
+static void poly1305_init_generic(void *ctx, const u8 key[16])
+{
+	struct poly1305_internal *st = (struct poly1305_internal *)ctx;
+	u64 t0, t1;
+
+	/* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
+	t0 = get_unaligned_le64(&key[0]);
+	t1 = get_unaligned_le64(&key[8]);
+
+	st->r[0] = t0 & 0xffc0fffffffULL;
+	st->r[1] = ((t0 >> 44) | (t1 << 20)) & 0xfffffc0ffffULL;
+	st->r[2] = ((t1 >> 24)) & 0x00ffffffc0fULL;
+
+	/* s = 20*r */
+	st->s[0] = st->r[1] * 20;
+	st->s[1] = st->r[2] * 20;
+
+	/* h = 0 */
+	st->h[0] = 0;
+	st->h[1] = 0;
+	st->h[2] = 0;
+}
+
+static void poly1305_blocks_generic(void *ctx, const u8 *input, size_t len,
+				    const u32 padbit)
+{
+	struct poly1305_internal *st = (struct poly1305_internal *)ctx;
+	const u64 hibit = ((u64)padbit) << 40;
+	u64 r0, r1, r2;
+	u64 s1, s2;
+	u64 h0, h1, h2;
+	u64 c;
+	u128 d0, d1, d2, d;
+
+	r0 = st->r[0];
+	r1 = st->r[1];
+	r2 = st->r[2];
+
+	h0 = st->h[0];
+	h1 = st->h[1];
+	h2 = st->h[2];
+
+	s1 = st->s[0];
+	s2 = st->s[1];
+
+	while (len >= POLY1305_BLOCK_SIZE) {
+		u64 t0, t1;
+
+		/* h += m[i] */
+		t0 = get_unaligned_le64(&input[0]);
+		t1 = get_unaligned_le64(&input[8]);
+
+		h0 += t0 & 0xfffffffffffULL;
+		h1 += ((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL;
+		h2 += (((t1 >> 24)) & 0x3ffffffffffULL) | hibit;
+
+		/* h *= r */
+		d0 = (u128)h0 * r0;
+		d = (u128)h1 * s2;
+		d0 += d;
+		d = (u128)h2 * s1;
+		d0 += d;
+		d1 = (u128)h0 * r1;
+		d = (u128)h1 * r0;
+		d1 += d;
+		d = (u128)h2 * s2;
+		d1 += d;
+		d2 = (u128)h0 * r2;
+		d = (u128)h1 * r1;
+		d2 += d;
+		d = (u128)h2 * r0;
+		d2 += d;
+
+		/* (partial) h %= p */
+		c = (u64)(d0 >> 44);
+		h0 = (u64)d0 & 0xfffffffffffULL;
+		d1 += c;
+		c = (u64)(d1 >> 44);
+		h1 = (u64)d1 & 0xfffffffffffULL;
+		d2 += c;
+		c = (u64)(d2 >> 42);
+		h2 = (u64)d2 & 0x3ffffffffffULL;
+		h0 += c * 5;
+		c = h0 >> 44;
+		h0 = h0 & 0xfffffffffffULL;
+		h1 += c;
+
+		input += POLY1305_BLOCK_SIZE;
+		len -= POLY1305_BLOCK_SIZE;
+	}
+
+	st->h[0] = h0;
+	st->h[1] = h1;
+	st->h[2] = h2;
+}
+
+static void poly1305_emit_generic(void *ctx, u8 mac[16], const u32 nonce[4])
+{
+	struct poly1305_internal *st = (struct poly1305_internal *)ctx;
+	u64 h0, h1, h2, c;
+	u64 g0, g1, g2;
+	u64 t0, t1;
+
+	/* fully carry h */
+	h0 = st->h[0];
+	h1 = st->h[1];
+	h2 = st->h[2];
+
+	c = h1 >> 44;
+	h1 &= 0xfffffffffffULL;
+	h2 += c;
+	c = h2 >> 42;
+	h2 &= 0x3ffffffffffULL;
+	h0 += c * 5;
+	c = h0 >> 44;
+	h0 &= 0xfffffffffffULL;
+	h1 += c;
+	c = h1 >> 44;
+	h1 &= 0xfffffffffffULL;
+	h2 += c;
+	c = h2 >> 42;
+	h2 &= 0x3ffffffffffULL;
+	h0 += c * 5;
+	c = h0 >> 44;
+	h0 &= 0xfffffffffffULL;
+	h1 += c;
+
+	/* compute h + -p */
+	g0 = h0 + 5;
+	c  = g0 >> 44;
+	g0 &= 0xfffffffffffULL;
+	g1 = h1 + c;
+	c  = g1 >> 44;
+	g1 &= 0xfffffffffffULL;
+	g2 = h2 + c - (1ULL << 42);
+
+	/* select h if h < p, or h + -p if h >= p */
+	c = (g2 >> ((sizeof(u64) * 8) - 1)) - 1;
+	g0 &= c;
+	g1 &= c;
+	g2 &= c;
+	c  = ~c;
+	h0 = (h0 & c) | g0;
+	h1 = (h1 & c) | g1;
+	h2 = (h2 & c) | g2;
+
+	/* h = (h + nonce) */
+	t0 = ((u64)nonce[1] << 32) | nonce[0];
+	t1 = ((u64)nonce[3] << 32) | nonce[2];
+
+	h0 += t0 & 0xfffffffffffULL;
+	c = h0 >> 44;
+	h0 &= 0xfffffffffffULL;
+	h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL) + c;
+	c = h1 >> 44;
+	h1 &= 0xfffffffffffULL;
+	h2 += (((t1 >> 24)) & 0x3ffffffffffULL) + c;
+	h2 &= 0x3ffffffffffULL;
+
+	/* mac = h % (2^128) */
+	h0 = h0 | (h1 << 44);
+	h1 = (h1 >> 20) | (h2 << 24);
+
+	put_unaligned_le64(h0, &mac[0]);
+	put_unaligned_le64(h1, &mac[8]);
+}
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/poly1305/poly1305-mips64.pl b/net/wireguard/crypto/zinc/poly1305/poly1305-mips64.pl
--- a/net/wireguard/crypto/zinc/poly1305/poly1305-mips64.pl	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/poly1305/poly1305-mips64.pl	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,466 @@
+#!/usr/bin/env perl
+# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+#
+# This code is taken from the OpenSSL project but the author, Andy Polyakov,
+# has relicensed it under the licenses specified in the SPDX header above.
+# The original headers, including the original license headers, are
+# included below for completeness.
+#
+# ====================================================================
+# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
+# project. The module is, however, dual licensed under OpenSSL and
+# CRYPTOGAMS licenses depending on where you obtain it. For further
+# details see http://www.openssl.org/~appro/cryptogams/.
+# ====================================================================
+#
+# Poly1305 hash for MIPS64.
+#
+# May 2016
+#
+# Numbers are cycles per processed byte with poly1305_blocks alone.
+#
+#		IALU/gcc
+# R1x000	5.64/+120%	(big-endian)
+# Octeon II	3.80/+280%	(little-endian)
+
+######################################################################
+# There is a number of MIPS ABI in use, O32 and N32/64 are most
+# widely used. Then there is a new contender: NUBI. It appears that if
+# one picks the latter, it's possible to arrange code in ABI neutral
+# manner. Therefore let's stick to NUBI register layout:
+#
+($zero,$at,$t0,$t1,$t2)=map("\$$_",(0..2,24,25));
+($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7)=map("\$$_",(4..11));
+($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7,$s8,$s9,$s10,$s11)=map("\$$_",(12..23));
+($gp,$tp,$sp,$fp,$ra)=map("\$$_",(3,28..31));
+#
+# The return value is placed in $a0. Following coding rules facilitate
+# interoperability:
+#
+# - never ever touch $tp, "thread pointer", former $gp [o32 can be
+#   excluded from the rule, because it's specified volatile];
+# - copy return value to $t0, former $v0 [or to $a0 if you're adapting
+#   old code];
+# - on O32 populate $a4-$a7 with 'lw $aN,4*N($sp)' if necessary;
+#
+# For reference here is register layout for N32/64 MIPS ABIs:
+#
+# ($zero,$at,$v0,$v1)=map("\$$_",(0..3));
+# ($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7)=map("\$$_",(4..11));
+# ($t0,$t1,$t2,$t3,$t8,$t9)=map("\$$_",(12..15,24,25));
+# ($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7)=map("\$$_",(16..23));
+# ($gp,$sp,$fp,$ra)=map("\$$_",(28..31));
+#
+# <appro@openssl.org>
+#
+######################################################################
+
+$flavour = shift || "64"; # supported flavours are o32,n32,64,nubi32,nubi64
+
+die "MIPS64 only" unless ($flavour =~ /64|n32/i);
+
+$v0 = ($flavour =~ /nubi/i) ? $a0 : $t0;
+$SAVED_REGS_MASK = ($flavour =~ /nubi/i) ? "0x0003f000" : "0x00030000";
+
+($ctx,$inp,$len,$padbit) = ($a0,$a1,$a2,$a3);
+($in0,$in1,$tmp0,$tmp1,$tmp2,$tmp3,$tmp4) = ($a4,$a5,$a6,$a7,$at,$t0,$t1);
+
+$code.=<<___;
+#if (defined(_MIPS_ARCH_MIPS64R3) || defined(_MIPS_ARCH_MIPS64R5) || \\
+     defined(_MIPS_ARCH_MIPS64R6)) \\
+     && !defined(_MIPS_ARCH_MIPS64R2)
+# define _MIPS_ARCH_MIPS64R2
+#endif
+
+#if defined(_MIPS_ARCH_MIPS64R6)
+# define dmultu(rs,rt)
+# define mflo(rd,rs,rt)	dmulu	rd,rs,rt
+# define mfhi(rd,rs,rt)	dmuhu	rd,rs,rt
+#else
+# define dmultu(rs,rt)		dmultu	rs,rt
+# define mflo(rd,rs,rt)	mflo	rd
+# define mfhi(rd,rs,rt)	mfhi	rd
+#endif
+
+#ifdef	__KERNEL__
+# define poly1305_init   poly1305_init_mips
+# define poly1305_blocks poly1305_blocks_mips
+# define poly1305_emit   poly1305_emit_mips
+#endif
+
+#if defined(__MIPSEB__) && !defined(MIPSEB)
+# define MIPSEB
+#endif
+
+#ifdef MIPSEB
+# define MSB 0
+# define LSB 7
+#else
+# define MSB 7
+# define LSB 0
+#endif
+
+.text
+.set	noat
+.set	noreorder
+
+.align	5
+.globl	poly1305_init
+.ent	poly1305_init
+poly1305_init:
+	.frame	$sp,0,$ra
+	.set	reorder
+
+	sd	$zero,0($ctx)
+	sd	$zero,8($ctx)
+	sd	$zero,16($ctx)
+
+	beqz	$inp,.Lno_key
+
+#if defined(_MIPS_ARCH_MIPS64R6)
+	ld	$in0,0($inp)
+	ld	$in1,8($inp)
+#else
+	ldl	$in0,0+MSB($inp)
+	ldl	$in1,8+MSB($inp)
+	ldr	$in0,0+LSB($inp)
+	ldr	$in1,8+LSB($inp)
+#endif
+#ifdef	MIPSEB
+# if defined(_MIPS_ARCH_MIPS64R2)
+	dsbh	$in0,$in0		# byte swap
+	 dsbh	$in1,$in1
+	dshd	$in0,$in0
+	 dshd	$in1,$in1
+# else
+	ori	$tmp0,$zero,0xFF
+	dsll	$tmp2,$tmp0,32
+	or	$tmp0,$tmp2		# 0x000000FF000000FF
+
+	and	$tmp1,$in0,$tmp0	# byte swap
+	 and	$tmp3,$in1,$tmp0
+	dsrl	$tmp2,$in0,24
+	 dsrl	$tmp4,$in1,24
+	dsll	$tmp1,24
+	 dsll	$tmp3,24
+	and	$tmp2,$tmp0
+	 and	$tmp4,$tmp0
+	dsll	$tmp0,8			# 0x0000FF000000FF00
+	or	$tmp1,$tmp2
+	 or	$tmp3,$tmp4
+	and	$tmp2,$in0,$tmp0
+	 and	$tmp4,$in1,$tmp0
+	dsrl	$in0,8
+	 dsrl	$in1,8
+	dsll	$tmp2,8
+	 dsll	$tmp4,8
+	and	$in0,$tmp0
+	 and	$in1,$tmp0
+	or	$tmp1,$tmp2
+	 or	$tmp3,$tmp4
+	or	$in0,$tmp1
+	 or	$in1,$tmp3
+	dsrl	$tmp1,$in0,32
+	 dsrl	$tmp3,$in1,32
+	dsll	$in0,32
+	 dsll	$in1,32
+	or	$in0,$tmp1
+	 or	$in1,$tmp3
+# endif
+#endif
+	li	$tmp0,1
+	dsll	$tmp0,32
+	daddiu	$tmp0,-63
+	dsll	$tmp0,28
+	daddiu	$tmp0,-1		# 0ffffffc0fffffff
+
+	and	$in0,$tmp0
+	daddiu	$tmp0,-3		# 0ffffffc0ffffffc
+	and	$in1,$tmp0
+
+	sd	$in0,24($ctx)
+	dsrl	$tmp0,$in1,2
+	sd	$in1,32($ctx)
+	daddu	$tmp0,$in1		# s1 = r1 + (r1 >> 2)
+	sd	$tmp0,40($ctx)
+
+.Lno_key:
+	li	$v0,0			# return 0
+	jr	$ra
+.end	poly1305_init
+___
+{
+my ($h0,$h1,$h2,$r0,$r1,$s1,$d0,$d1,$d2) =
+   ($s0,$s1,$s2,$s3,$s4,$s5,$in0,$in1,$t2);
+
+$code.=<<___;
+.align	5
+.globl	poly1305_blocks
+.ent	poly1305_blocks
+poly1305_blocks:
+	.set	noreorder
+	dsrl	$len,4			# number of complete blocks
+	bnez	$len,poly1305_blocks_internal
+	nop
+	jr	$ra
+	nop
+.end	poly1305_blocks
+
+.align	5
+.ent	poly1305_blocks_internal
+poly1305_blocks_internal:
+	.frame	$sp,6*8,$ra
+	.mask	$SAVED_REGS_MASK,-8
+	.set	noreorder
+	dsubu	$sp,6*8
+	sd	$s5,40($sp)
+	sd	$s4,32($sp)
+___
+$code.=<<___ if ($flavour =~ /nubi/i);	# optimize non-nubi prologue
+	sd	$s3,24($sp)
+	sd	$s2,16($sp)
+	sd	$s1,8($sp)
+	sd	$s0,0($sp)
+___
+$code.=<<___;
+	.set	reorder
+
+	ld	$h0,0($ctx)		# load hash value
+	ld	$h1,8($ctx)
+	ld	$h2,16($ctx)
+
+	ld	$r0,24($ctx)		# load key
+	ld	$r1,32($ctx)
+	ld	$s1,40($ctx)
+
+.Loop:
+#if defined(_MIPS_ARCH_MIPS64R6)
+	ld	$in0,0($inp)		# load input
+	ld	$in1,8($inp)
+#else
+	ldl	$in0,0+MSB($inp)	# load input
+	ldl	$in1,8+MSB($inp)
+	ldr	$in0,0+LSB($inp)
+	ldr	$in1,8+LSB($inp)
+#endif
+	daddiu	$len,-1
+	daddiu	$inp,16
+#ifdef	MIPSEB
+# if defined(_MIPS_ARCH_MIPS64R2)
+	dsbh	$in0,$in0		# byte swap
+	 dsbh	$in1,$in1
+	dshd	$in0,$in0
+	 dshd	$in1,$in1
+# else
+	ori	$tmp0,$zero,0xFF
+	dsll	$tmp2,$tmp0,32
+	or	$tmp0,$tmp2		# 0x000000FF000000FF
+
+	and	$tmp1,$in0,$tmp0	# byte swap
+	 and	$tmp3,$in1,$tmp0
+	dsrl	$tmp2,$in0,24
+	 dsrl	$tmp4,$in1,24
+	dsll	$tmp1,24
+	 dsll	$tmp3,24
+	and	$tmp2,$tmp0
+	 and	$tmp4,$tmp0
+	dsll	$tmp0,8			# 0x0000FF000000FF00
+	or	$tmp1,$tmp2
+	 or	$tmp3,$tmp4
+	and	$tmp2,$in0,$tmp0
+	 and	$tmp4,$in1,$tmp0
+	dsrl	$in0,8
+	 dsrl	$in1,8
+	dsll	$tmp2,8
+	 dsll	$tmp4,8
+	and	$in0,$tmp0
+	 and	$in1,$tmp0
+	or	$tmp1,$tmp2
+	 or	$tmp3,$tmp4
+	or	$in0,$tmp1
+	 or	$in1,$tmp3
+	dsrl	$tmp1,$in0,32
+	 dsrl	$tmp3,$in1,32
+	dsll	$in0,32
+	 dsll	$in1,32
+	or	$in0,$tmp1
+	 or	$in1,$tmp3
+# endif
+#endif
+	daddu	$h0,$in0		# accumulate input
+	daddu	$h1,$in1
+	sltu	$tmp0,$h0,$in0
+	sltu	$tmp1,$h1,$in1
+	daddu	$h1,$tmp0
+
+	dmultu	($r0,$h0)		# h0*r0
+	 daddu	$h2,$padbit
+	 sltu	$tmp0,$h1,$tmp0
+	mflo	($d0,$r0,$h0)
+	mfhi	($d1,$r0,$h0)
+
+	dmultu	($s1,$h1)		# h1*5*r1
+	 daddu	$tmp0,$tmp1
+	 daddu	$h2,$tmp0
+	mflo	($tmp0,$s1,$h1)
+	mfhi	($tmp1,$s1,$h1)
+
+	dmultu	($r1,$h0)		# h0*r1
+	 daddu	$d0,$tmp0
+	 daddu	$d1,$tmp1
+	mflo	($tmp2,$r1,$h0)
+	mfhi	($d2,$r1,$h0)
+	 sltu	$tmp0,$d0,$tmp0
+	 daddu	$d1,$tmp0
+
+	dmultu	($r0,$h1)		# h1*r0
+	 daddu	$d1,$tmp2
+	 sltu	$tmp2,$d1,$tmp2
+	mflo	($tmp0,$r0,$h1)
+	mfhi	($tmp1,$r0,$h1)
+	 daddu	$d2,$tmp2
+
+	dmultu	($s1,$h2)		# h2*5*r1
+	 daddu	$d1,$tmp0
+	 daddu	$d2,$tmp1
+	mflo	($tmp2,$s1,$h2)
+
+	dmultu	($r0,$h2)		# h2*r0
+	 sltu	$tmp0,$d1,$tmp0
+	 daddu	$d2,$tmp0
+	mflo	($tmp3,$r0,$h2)
+
+	daddu	$d1,$tmp2
+	daddu	$d2,$tmp3
+	sltu	$tmp2,$d1,$tmp2
+	daddu	$d2,$tmp2
+
+	li	$tmp0,-4		# final reduction
+	and	$tmp0,$d2
+	dsrl	$tmp1,$d2,2
+	andi	$h2,$d2,3
+	daddu	$tmp0,$tmp1
+	daddu	$h0,$d0,$tmp0
+	sltu	$tmp0,$h0,$tmp0
+	daddu	$h1,$d1,$tmp0
+	sltu	$tmp0,$h1,$tmp0
+	daddu	$h2,$h2,$tmp0
+
+	bnez	$len,.Loop
+
+	sd	$h0,0($ctx)		# store hash value
+	sd	$h1,8($ctx)
+	sd	$h2,16($ctx)
+
+	.set	noreorder
+	ld	$s5,40($sp)		# epilogue
+	ld	$s4,32($sp)
+___
+$code.=<<___ if ($flavour =~ /nubi/i);	# optimize non-nubi epilogue
+	ld	$s3,24($sp)
+	ld	$s2,16($sp)
+	ld	$s1,8($sp)
+	ld	$s0,0($sp)
+___
+$code.=<<___;
+	jr	$ra
+	daddu	$sp,6*8
+.end	poly1305_blocks_internal
+___
+}
+{
+my ($ctx,$mac,$nonce) = ($a0,$a1,$a2);
+
+$code.=<<___;
+.align	5
+.globl	poly1305_emit
+.ent	poly1305_emit
+poly1305_emit:
+	.frame	$sp,0,$ra
+	.set	reorder
+
+	ld	$tmp0,0($ctx)
+	ld	$tmp1,8($ctx)
+	ld	$tmp2,16($ctx)
+
+	daddiu	$in0,$tmp0,5		# compare to modulus
+	sltiu	$tmp3,$in0,5
+	daddu	$in1,$tmp1,$tmp3
+	sltu	$tmp3,$in1,$tmp3
+	daddu	$tmp2,$tmp2,$tmp3
+
+	dsrl	$tmp2,2			# see if it carried/borrowed
+	dsubu	$tmp2,$zero,$tmp2
+	nor	$tmp3,$zero,$tmp2
+
+	and	$in0,$tmp2
+	and	$tmp0,$tmp3
+	and	$in1,$tmp2
+	and	$tmp1,$tmp3
+	or	$in0,$tmp0
+	or	$in1,$tmp1
+
+	lwu	$tmp0,0($nonce)		# load nonce
+	lwu	$tmp1,4($nonce)
+	lwu	$tmp2,8($nonce)
+	lwu	$tmp3,12($nonce)
+	dsll	$tmp1,32
+	dsll	$tmp3,32
+	or	$tmp0,$tmp1
+	or	$tmp2,$tmp3
+
+	daddu	$in0,$tmp0		# accumulate nonce
+	daddu	$in1,$tmp2
+	sltu	$tmp0,$in0,$tmp0
+	daddu	$in1,$tmp0
+
+	dsrl	$tmp0,$in0,8		# write mac value
+	dsrl	$tmp1,$in0,16
+	dsrl	$tmp2,$in0,24
+	sb	$in0,0($mac)
+	dsrl	$tmp3,$in0,32
+	sb	$tmp0,1($mac)
+	dsrl	$tmp0,$in0,40
+	sb	$tmp1,2($mac)
+	dsrl	$tmp1,$in0,48
+	sb	$tmp2,3($mac)
+	dsrl	$tmp2,$in0,56
+	sb	$tmp3,4($mac)
+	dsrl	$tmp3,$in1,8
+	sb	$tmp0,5($mac)
+	dsrl	$tmp0,$in1,16
+	sb	$tmp1,6($mac)
+	dsrl	$tmp1,$in1,24
+	sb	$tmp2,7($mac)
+
+	sb	$in1,8($mac)
+	dsrl	$tmp2,$in1,32
+	sb	$tmp3,9($mac)
+	dsrl	$tmp3,$in1,40
+	sb	$tmp0,10($mac)
+	dsrl	$tmp0,$in1,48
+	sb	$tmp1,11($mac)
+	dsrl	$tmp1,$in1,56
+	sb	$tmp2,12($mac)
+	sb	$tmp3,13($mac)
+	sb	$tmp0,14($mac)
+	sb	$tmp1,15($mac)
+
+	jr	$ra
+.end	poly1305_emit
+.rdata
+.align	2
+___
+}
+
+open SELF,$0;
+while(<SELF>) {
+	next if (/^#!/);
+	last if (!s/^#/\/\// and !/^$/);
+	print;
+}
+close SELF;
+
+$output=pop and open STDOUT,">$output";
+print $code;
+close STDOUT;
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/poly1305/poly1305-mips-glue.c b/net/wireguard/crypto/zinc/poly1305/poly1305-mips-glue.c
--- a/net/wireguard/crypto/zinc/poly1305/poly1305-mips-glue.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/poly1305/poly1305-mips-glue.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+asmlinkage void poly1305_init_mips(void *ctx, const u8 key[16]);
+asmlinkage void poly1305_blocks_mips(void *ctx, const u8 *inp, const size_t len,
+				     const u32 padbit);
+asmlinkage void poly1305_emit_mips(void *ctx, u8 mac[16], const u32 nonce[4]);
+
+static bool *const poly1305_nobs[] __initconst = { };
+static void __init poly1305_fpu_init(void)
+{
+}
+
+static inline bool poly1305_init_arch(void *ctx,
+				      const u8 key[POLY1305_KEY_SIZE])
+{
+	poly1305_init_mips(ctx, key);
+	return true;
+}
+
+static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp,
+					size_t len, const u32 padbit,
+					simd_context_t *simd_context)
+{
+	poly1305_blocks_mips(ctx, inp, len, padbit);
+	return true;
+}
+
+static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE],
+				      const u32 nonce[4],
+				      simd_context_t *simd_context)
+{
+	poly1305_emit_mips(ctx, mac, nonce);
+	return true;
+}
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/poly1305/poly1305-mips.S b/net/wireguard/crypto/zinc/poly1305/poly1305-mips.S
--- a/net/wireguard/crypto/zinc/poly1305/poly1305-mips.S	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/poly1305/poly1305-mips.S	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,407 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/*
+ * Copyright (C) 2016-2018 René van Dorst <opensource@vdorst.com> All Rights Reserved.
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+#define MSB 0
+#define LSB 3
+#else
+#define MSB 3
+#define LSB 0
+#endif
+
+#define POLY1305_BLOCK_SIZE 16
+.text
+#define H0 $t0
+#define H1 $t1
+#define H2 $t2
+#define H3 $t3
+#define H4 $t4
+
+#define R0 $t5
+#define R1 $t6
+#define R2 $t7
+#define R3 $t8
+
+#define O0 $s0
+#define O1 $s4
+#define O2 $v1
+#define O3 $t9
+#define O4 $s5
+
+#define S1 $s1
+#define S2 $s2
+#define S3 $s3
+
+#define SC $at
+#define CA $v0
+
+/* Input arguments */
+#define poly	$a0
+#define src	$a1
+#define srclen	$a2
+#define hibit	$a3
+
+/* Location in the opaque buffer
+ * R[0..3], CA, H[0..4]
+ */
+#define PTR_POLY1305_R(n) ( 0 + (n*4)) ## ($a0)
+#define PTR_POLY1305_CA   (16        ) ## ($a0)
+#define PTR_POLY1305_H(n) (20 + (n*4)) ## ($a0)
+
+#define POLY1305_BLOCK_SIZE 16
+#define POLY1305_STACK_SIZE 32
+
+.set	noat
+.align	4
+.globl	poly1305_blocks_mips
+.ent	poly1305_blocks_mips
+poly1305_blocks_mips:
+	.frame	$sp, POLY1305_STACK_SIZE, $ra
+	/* srclen &= 0xFFFFFFF0 */
+	ins	srclen, $zero, 0, 4
+
+	addiu	$sp, -(POLY1305_STACK_SIZE)
+
+	/* check srclen >= 16 bytes */
+	beqz	srclen, .Lpoly1305_blocks_mips_end
+
+	/* Calculate last round based on src address pointer.
+	 * last round src ptr (srclen) = src + (srclen & 0xFFFFFFF0)
+	 */
+	addu	srclen, src
+
+	lw	R0, PTR_POLY1305_R(0)
+	lw	R1, PTR_POLY1305_R(1)
+	lw	R2, PTR_POLY1305_R(2)
+	lw	R3, PTR_POLY1305_R(3)
+
+	/* store the used save registers. */
+	sw	$s0, 0($sp)
+	sw	$s1, 4($sp)
+	sw	$s2, 8($sp)
+	sw	$s3, 12($sp)
+	sw	$s4, 16($sp)
+	sw	$s5, 20($sp)
+
+	/* load Hx and Carry */
+	lw	CA, PTR_POLY1305_CA
+	lw	H0, PTR_POLY1305_H(0)
+	lw	H1, PTR_POLY1305_H(1)
+	lw	H2, PTR_POLY1305_H(2)
+	lw	H3, PTR_POLY1305_H(3)
+	lw	H4, PTR_POLY1305_H(4)
+
+	/* Sx = Rx + (Rx >> 2) */
+	srl	S1, R1, 2
+	srl	S2, R2, 2
+	srl	S3, R3, 2
+	addu	S1, R1
+	addu	S2, R2
+	addu	S3, R3
+
+	addiu	SC, $zero, 1
+
+.Lpoly1305_loop:
+	lwl	O0, 0+MSB(src)
+	lwl	O1, 4+MSB(src)
+	lwl	O2, 8+MSB(src)
+	lwl	O3,12+MSB(src)
+	lwr	O0, 0+LSB(src)
+	lwr	O1, 4+LSB(src)
+	lwr	O2, 8+LSB(src)
+	lwr	O3,12+LSB(src)
+
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+	wsbh	O0
+	wsbh	O1
+	wsbh	O2
+	wsbh	O3
+	rotr	O0, 16
+	rotr	O1, 16
+	rotr	O2, 16
+	rotr	O3, 16
+#endif
+
+	/* h0 = (u32)(d0 = (u64)h0 + inp[0] + c 'Carry_previous cycle'); */
+	addu	H0, CA
+	sltu	CA, H0, CA
+	addu	O0, H0
+	sltu	H0, O0, H0
+	addu	CA, H0
+
+	/* h1 = (u32)(d1 = (u64)h1 + (d0 >> 32) + inp[4]); */
+	addu	H1, CA
+	sltu	CA, H1, CA
+	addu	O1, H1
+	sltu	H1, O1, H1
+	addu	CA, H1
+
+	/* h2 = (u32)(d2 = (u64)h2 + (d1 >> 32) + inp[8]); */
+	addu	H2, CA
+	sltu	CA, H2, CA
+	addu	O2, H2
+	sltu	H2, O2, H2
+	addu	CA, H2
+
+	/* h3 = (u32)(d3 = (u64)h3 + (d2 >> 32) + inp[12]); */
+	addu	H3, CA
+	sltu	CA, H3, CA
+	addu	O3, H3
+	sltu	H3, O3, H3
+	addu	CA, H3
+
+	/* h4 += (u32)(d3 >> 32) + padbit; */
+	addu	H4, hibit
+	addu	O4, H4, CA
+
+	/* D0 */
+	multu	O0, R0
+	maddu	O1, S3
+	maddu	O2, S2
+	maddu	O3, S1
+	mfhi	CA
+	mflo	H0
+
+	/* D1 */
+	multu	O0, R1
+	maddu	O1, R0
+	maddu	O2, S3
+	maddu	O3, S2
+	maddu	O4, S1
+	maddu	CA, SC
+	mfhi	CA
+	mflo	H1
+
+	/* D2 */
+	multu	O0, R2
+	maddu	O1, R1
+	maddu	O2, R0
+	maddu	O3, S3
+	maddu	O4, S2
+	maddu	CA, SC
+	mfhi	CA
+	mflo	H2
+
+	/* D4 */
+	mul	H4, O4, R0
+
+	/* D3 */
+	multu	O0, R3
+	maddu	O1, R2
+	maddu	O2, R1
+	maddu	O3, R0
+	maddu	O4, S3
+	maddu	CA, SC
+	mfhi	CA
+	mflo	H3
+
+	addiu	src, POLY1305_BLOCK_SIZE
+
+	/* h4 += (u32)(d3 >> 32); */
+	addu	O4, H4, CA
+	/* h4 &= 3 */
+	andi	H4, O4, 3
+	/* c = (h4 >> 2) + (h4 & ~3U); */
+	srl	CA, O4, 2
+	ins	O4, $zero, 0, 2
+
+	addu	CA, O4
+
+	/* able to do a 16 byte block. */
+	bne	src, srclen, .Lpoly1305_loop
+
+	/* restore the used save registers. */
+	lw	$s0, 0($sp)
+	lw	$s1, 4($sp)
+	lw	$s2, 8($sp)
+	lw	$s3, 12($sp)
+	lw	$s4, 16($sp)
+	lw	$s5, 20($sp)
+
+	/* store Hx and Carry */
+	sw	CA, PTR_POLY1305_CA
+	sw	H0, PTR_POLY1305_H(0)
+	sw	H1, PTR_POLY1305_H(1)
+	sw	H2, PTR_POLY1305_H(2)
+	sw	H3, PTR_POLY1305_H(3)
+	sw	H4, PTR_POLY1305_H(4)
+
+.Lpoly1305_blocks_mips_end:
+	addiu	$sp, POLY1305_STACK_SIZE
+
+	/* Jump Back */
+	jr	$ra
+.end poly1305_blocks_mips
+.set at
+
+/* Input arguments CTX=$a0, MAC=$a1, NONCE=$a2 */
+#define MAC	$a1
+#define NONCE	$a2
+
+#define G0	$t5
+#define G1	$t6
+#define G2	$t7
+#define G3	$t8
+#define G4	$t9
+
+.set	noat
+.align	4
+.globl	poly1305_emit_mips
+.ent	poly1305_emit_mips
+poly1305_emit_mips:
+	/* load Hx and Carry */
+	lw	CA, PTR_POLY1305_CA
+	lw	H0, PTR_POLY1305_H(0)
+	lw	H1, PTR_POLY1305_H(1)
+	lw	H2, PTR_POLY1305_H(2)
+	lw	H3, PTR_POLY1305_H(3)
+	lw	H4, PTR_POLY1305_H(4)
+
+	/* Add left over carry */
+	addu	H0, CA
+	sltu	CA, H0, CA
+	addu	H1, CA
+	sltu	CA, H1, CA
+	addu	H2, CA
+	sltu	CA, H2, CA
+	addu	H3, CA
+	sltu	CA, H3, CA
+	addu	H4, CA
+
+	/* compare to modulus by computing h + -p */
+	addiu	G0, H0, 5
+	sltu	CA, G0, H0
+	addu	G1, H1, CA
+	sltu	CA, G1, H1
+	addu	G2, H2, CA
+	sltu	CA, G2, H2
+	addu	G3, H3, CA
+	sltu	CA, G3, H3
+	addu	G4, H4, CA
+
+	srl	SC, G4, 2
+
+	/* if there was carry into 131st bit, h3:h0 = g3:g0 */
+	movn	H0, G0, SC
+	movn	H1, G1, SC
+	movn	H2, G2, SC
+	movn	H3, G3, SC
+
+	lwl	G0, 0+MSB(NONCE)
+	lwl	G1, 4+MSB(NONCE)
+	lwl	G2, 8+MSB(NONCE)
+	lwl	G3,12+MSB(NONCE)
+	lwr	G0, 0+LSB(NONCE)
+	lwr	G1, 4+LSB(NONCE)
+	lwr	G2, 8+LSB(NONCE)
+	lwr	G3,12+LSB(NONCE)
+
+	/* mac = (h + nonce) % (2^128) */
+	addu	H0, G0
+	sltu	CA, H0, G0
+
+	/* H1 */
+	addu	H1, CA
+	sltu	CA, H1, CA
+	addu	H1, G1
+	sltu	G1, H1, G1
+	addu	CA, G1
+
+	/* H2 */
+	addu	H2, CA
+	sltu	CA, H2, CA
+	addu	H2, G2
+	sltu	G2, H2, G2
+	addu	CA, G2
+
+	/* H3 */
+	addu	H3, CA
+	addu	H3, G3
+
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+	wsbh	H0
+	wsbh	H1
+	wsbh	H2
+	wsbh	H3
+	rotr	H0, 16
+	rotr	H1, 16
+	rotr	H2, 16
+	rotr	H3, 16
+#endif
+
+	/* store MAC */
+	swl	H0, 0+MSB(MAC)
+	swl	H1, 4+MSB(MAC)
+	swl	H2, 8+MSB(MAC)
+	swl	H3,12+MSB(MAC)
+	swr	H0, 0+LSB(MAC)
+	swr	H1, 4+LSB(MAC)
+	swr	H2, 8+LSB(MAC)
+	swr	H3,12+LSB(MAC)
+
+	jr	$ra
+.end poly1305_emit_mips
+
+#define PR0 $t0
+#define PR1 $t1
+#define PR2 $t2
+#define PR3 $t3
+#define PT0 $t4
+
+/* Input arguments CTX=$a0, KEY=$a1 */
+
+.align	4
+.globl	poly1305_init_mips
+.ent	poly1305_init_mips
+poly1305_init_mips:
+	lwl	PR0, 0+MSB($a1)
+	lwl	PR1, 4+MSB($a1)
+	lwl	PR2, 8+MSB($a1)
+	lwl	PR3,12+MSB($a1)
+	lwr	PR0, 0+LSB($a1)
+	lwr	PR1, 4+LSB($a1)
+	lwr	PR2, 8+LSB($a1)
+	lwr	PR3,12+LSB($a1)
+
+	/* store Hx and Carry */
+	sw	$zero, PTR_POLY1305_CA
+	sw	$zero, PTR_POLY1305_H(0)
+	sw	$zero, PTR_POLY1305_H(1)
+	sw	$zero, PTR_POLY1305_H(2)
+	sw	$zero, PTR_POLY1305_H(3)
+	sw	$zero, PTR_POLY1305_H(4)
+
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+	wsbh	PR0
+	wsbh	PR1
+	wsbh	PR2
+	wsbh	PR3
+	rotr	PR0, 16
+	rotr	PR1, 16
+	rotr	PR2, 16
+	rotr	PR3, 16
+#endif
+
+	lui	PT0, 0x0FFF
+	ori	PT0, 0xFFFC
+
+	/* AND 0x0fffffff; */
+	ext	PR0, PR0, 0, (32-4)
+
+	/* AND 0x0ffffffc; */
+	and	PR1, PT0
+	and	PR2, PT0
+	and	PR3, PT0
+
+	/* store Rx */
+	sw	PR0, PTR_POLY1305_R(0)
+	sw	PR1, PTR_POLY1305_R(1)
+	sw	PR2, PTR_POLY1305_R(2)
+	sw	PR3, PTR_POLY1305_R(3)
+
+	/* Jump Back  */
+	jr	$ra
+.end poly1305_init_mips
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/poly1305/poly1305-x86_64-glue.c b/net/wireguard/crypto/zinc/poly1305/poly1305-x86_64-glue.c
--- a/net/wireguard/crypto/zinc/poly1305/poly1305-x86_64-glue.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/poly1305/poly1305-x86_64-glue.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,156 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#include <asm/cpufeature.h>
+#include <asm/processor.h>
+#include <asm/intel-family.h>
+
+asmlinkage void poly1305_init_x86_64(void *ctx,
+				     const u8 key[POLY1305_KEY_SIZE]);
+asmlinkage void poly1305_blocks_x86_64(void *ctx, const u8 *inp,
+				       const size_t len, const u32 padbit);
+asmlinkage void poly1305_emit_x86_64(void *ctx, u8 mac[POLY1305_MAC_SIZE],
+				     const u32 nonce[4]);
+asmlinkage void poly1305_emit_avx(void *ctx, u8 mac[POLY1305_MAC_SIZE],
+				  const u32 nonce[4]);
+asmlinkage void poly1305_blocks_avx(void *ctx, const u8 *inp, const size_t len,
+				    const u32 padbit);
+asmlinkage void poly1305_blocks_avx2(void *ctx, const u8 *inp, const size_t len,
+				     const u32 padbit);
+asmlinkage void poly1305_blocks_avx512(void *ctx, const u8 *inp,
+				       const size_t len, const u32 padbit);
+
+static bool poly1305_use_avx __ro_after_init;
+static bool poly1305_use_avx2 __ro_after_init;
+static bool poly1305_use_avx512 __ro_after_init;
+static bool *const poly1305_nobs[] __initconst = {
+	&poly1305_use_avx, &poly1305_use_avx2, &poly1305_use_avx512 };
+
+static void __init poly1305_fpu_init(void)
+{
+	poly1305_use_avx =
+		boot_cpu_has(X86_FEATURE_AVX) &&
+		cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL);
+	poly1305_use_avx2 =
+		boot_cpu_has(X86_FEATURE_AVX) &&
+		boot_cpu_has(X86_FEATURE_AVX2) &&
+		cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL);
+#ifndef COMPAT_CANNOT_USE_AVX512
+	poly1305_use_avx512 =
+		boot_cpu_has(X86_FEATURE_AVX) &&
+		boot_cpu_has(X86_FEATURE_AVX2) &&
+		boot_cpu_has(X86_FEATURE_AVX512F) &&
+		cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM |
+				  XFEATURE_MASK_AVX512, NULL) &&
+		/* Skylake downclocks unacceptably much when using zmm. */
+		boot_cpu_data.x86_model != INTEL_FAM6_SKYLAKE_X;
+#endif
+}
+
+static inline bool poly1305_init_arch(void *ctx,
+				      const u8 key[POLY1305_KEY_SIZE])
+{
+	poly1305_init_x86_64(ctx, key);
+	return true;
+}
+
+struct poly1305_arch_internal {
+	union {
+		struct {
+			u32 h[5];
+			u32 is_base2_26;
+		};
+		u64 hs[3];
+	};
+	u64 r[2];
+	u64 pad;
+	struct { u32 r2, r1, r4, r3; } rn[9];
+};
+
+/* The AVX code uses base 2^26, while the scalar code uses base 2^64. If we hit
+ * the unfortunate situation of using AVX and then having to go back to scalar
+ * -- because the user is silly and has called the update function from two
+ * separate contexts -- then we need to convert back to the original base before
+ * proceeding. It is possible to reason that the initial reduction below is
+ * sufficient given the implementation invariants. However, for an avoidance of
+ * doubt and because this is not performance critical, we do the full reduction
+ * anyway.
+ */
+static void convert_to_base2_64(void *ctx)
+{
+	struct poly1305_arch_internal *state = ctx;
+	u32 cy;
+
+	if (!state->is_base2_26)
+		return;
+
+	cy = state->h[0] >> 26; state->h[0] &= 0x3ffffff; state->h[1] += cy;
+	cy = state->h[1] >> 26; state->h[1] &= 0x3ffffff; state->h[2] += cy;
+	cy = state->h[2] >> 26; state->h[2] &= 0x3ffffff; state->h[3] += cy;
+	cy = state->h[3] >> 26; state->h[3] &= 0x3ffffff; state->h[4] += cy;
+	state->hs[0] = ((u64)state->h[2] << 52) | ((u64)state->h[1] << 26) | state->h[0];
+	state->hs[1] = ((u64)state->h[4] << 40) | ((u64)state->h[3] << 14) | (state->h[2] >> 12);
+	state->hs[2] = state->h[4] >> 24;
+#define ULT(a, b) ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1))
+	cy = (state->hs[2] >> 2) + (state->hs[2] & ~3ULL);
+	state->hs[2] &= 3;
+	state->hs[0] += cy;
+	state->hs[1] += (cy = ULT(state->hs[0], cy));
+	state->hs[2] += ULT(state->hs[1], cy);
+#undef ULT
+	state->is_base2_26 = 0;
+}
+
+static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp,
+					size_t len, const u32 padbit,
+					simd_context_t *simd_context)
+{
+	struct poly1305_arch_internal *state = ctx;
+
+	/* SIMD disables preemption, so relax after processing each page. */
+	BUILD_BUG_ON(PAGE_SIZE < POLY1305_BLOCK_SIZE ||
+		     PAGE_SIZE % POLY1305_BLOCK_SIZE);
+
+	if (!IS_ENABLED(CONFIG_AS_AVX) || !poly1305_use_avx ||
+	    (len < (POLY1305_BLOCK_SIZE * 18) && !state->is_base2_26) ||
+	    !simd_use(simd_context)) {
+		convert_to_base2_64(ctx);
+		poly1305_blocks_x86_64(ctx, inp, len, padbit);
+		return true;
+	}
+
+	for (;;) {
+		const size_t bytes = min_t(size_t, len, PAGE_SIZE);
+
+		if (IS_ENABLED(CONFIG_AS_AVX512) && poly1305_use_avx512)
+			poly1305_blocks_avx512(ctx, inp, bytes, padbit);
+		else if (IS_ENABLED(CONFIG_AS_AVX2) && poly1305_use_avx2)
+			poly1305_blocks_avx2(ctx, inp, bytes, padbit);
+		else
+			poly1305_blocks_avx(ctx, inp, bytes, padbit);
+		len -= bytes;
+		if (!len)
+			break;
+		inp += bytes;
+		simd_relax(simd_context);
+	}
+
+	return true;
+}
+
+static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE],
+				      const u32 nonce[4],
+				      simd_context_t *simd_context)
+{
+	struct poly1305_arch_internal *state = ctx;
+
+	if (!IS_ENABLED(CONFIG_AS_AVX) || !poly1305_use_avx ||
+	    !state->is_base2_26 || !simd_use(simd_context)) {
+		convert_to_base2_64(ctx);
+		poly1305_emit_x86_64(ctx, mac, nonce);
+	} else
+		poly1305_emit_avx(ctx, mac, nonce);
+	return true;
+}
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/poly1305/poly1305-x86_64.pl b/net/wireguard/crypto/zinc/poly1305/poly1305-x86_64.pl
--- a/net/wireguard/crypto/zinc/poly1305/poly1305-x86_64.pl	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/poly1305/poly1305-x86_64.pl	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,4266 @@
+#!/usr/bin/env perl
+# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+#
+# Copyright (C) 2017-2018 Samuel Neves <sneves@dei.uc.pt>. All Rights Reserved.
+# Copyright (C) 2017-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+# Copyright (C) 2006-2017 CRYPTOGAMS by <appro@openssl.org>. All Rights Reserved.
+#
+# This code is taken from the OpenSSL project but the author, Andy Polyakov,
+# has relicensed it under the licenses specified in the SPDX header above.
+# The original headers, including the original license headers, are
+# included below for completeness.
+#
+# ====================================================================
+# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
+# project. The module is, however, dual licensed under OpenSSL and
+# CRYPTOGAMS licenses depending on where you obtain it. For further
+# details see http://www.openssl.org/~appro/cryptogams/.
+# ====================================================================
+#
+# This module implements Poly1305 hash for x86_64.
+#
+# March 2015
+#
+# Initial release.
+#
+# December 2016
+#
+# Add AVX512F+VL+BW code path.
+#
+# November 2017
+#
+# Convert AVX512F+VL+BW code path to pure AVX512F, so that it can be
+# executed even on Knights Landing. Trigger for modification was
+# observation that AVX512 code paths can negatively affect overall
+# Skylake-X system performance. Since we are likely to suppress
+# AVX512F capability flag [at least on Skylake-X], conversion serves
+# as kind of "investment protection". Note that next *lake processor,
+# Cannonlake, has AVX512IFMA code path to execute...
+#
+# Numbers are cycles per processed byte with poly1305_blocks alone,
+# measured with rdtsc at fixed clock frequency.
+#
+#		IALU/gcc-4.8(*)	AVX(**)		AVX2	AVX-512
+# P4		4.46/+120%	-
+# Core 2	2.41/+90%	-
+# Westmere	1.88/+120%	-
+# Sandy Bridge	1.39/+140%	1.10
+# Haswell	1.14/+175%	1.11		0.65
+# Skylake[-X]	1.13/+120%	0.96		0.51	[0.35]
+# Silvermont	2.83/+95%	-
+# Knights L	3.60/?		1.65		1.10	0.41(***)
+# Goldmont	1.70/+180%	-
+# VIA Nano	1.82/+150%	-
+# Sledgehammer	1.38/+160%	-
+# Bulldozer	2.30/+130%	0.97
+# Ryzen		1.15/+200%	1.08		1.18
+#
+# (*)	improvement coefficients relative to clang are more modest and
+#	are ~50% on most processors, in both cases we are comparing to
+#	__int128 code;
+# (**)	SSE2 implementation was attempted, but among non-AVX processors
+#	it was faster than integer-only code only on older Intel P4 and
+#	Core processors, 50-30%, less newer processor is, but slower on
+#	contemporary ones, for example almost 2x slower on Atom, and as
+#	former are naturally disappearing, SSE2 is deemed unnecessary;
+# (***)	strangely enough performance seems to vary from core to core,
+#	listed result is best case;
+
+$flavour = shift;
+$output  = shift;
+if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
+
+$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
+$kernel=0; $kernel=1 if (!$flavour && !$output);
+
+if (!$kernel) {
+	$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
+	( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
+	( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
+	die "can't locate x86_64-xlate.pl";
+
+	open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
+	*STDOUT=*OUT;
+
+	if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
+	    =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
+		$avx = ($1>=2.19) + ($1>=2.22) + ($1>=2.25);
+	}
+
+	if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
+	    `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) {
+		$avx = ($1>=2.09) + ($1>=2.10) + ($1>=2.12);
+		$avx += 1 if ($1==2.11 && $2>=8);
+	}
+
+	if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
+	    `ml64 2>&1` =~ /Version ([0-9]+)\./) {
+		$avx = ($1>=10) + ($1>=11);
+	}
+
+	if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9]\.[0-9]+)/) {
+		$avx = ($2>=3.0) + ($2>3.0);
+	}
+} else {
+	$avx = 4; # The kernel uses ifdefs for this.
+}
+
+sub declare_function() {
+	my ($name, $align, $nargs) = @_;
+	if($kernel) {
+		$code .= ".align $align\n";
+		$code .= "SYM_FUNC_START($name)\n";
+		$code .= ".L$name:\n";
+	} else {
+		$code .= ".globl	$name\n";
+		$code .= ".type	$name,\@function,$nargs\n";
+		$code .= ".align	$align\n";
+		$code .= "$name:\n";
+	}
+}
+
+sub end_function() {
+	my ($name) = @_;
+	if($kernel) {
+		$code .= "SYM_FUNC_END($name)\n";
+	} else {
+		$code .= ".size   $name,.-$name\n";
+	}
+}
+
+$code.=<<___ if $kernel;
+#include <linux/linkage.h>
+___
+
+if ($avx) {
+$code.=<<___ if $kernel;
+.section .rodata
+___
+$code.=<<___;
+.align	64
+.Lconst:
+.Lmask24:
+.long	0x0ffffff,0,0x0ffffff,0,0x0ffffff,0,0x0ffffff,0
+.L129:
+.long	`1<<24`,0,`1<<24`,0,`1<<24`,0,`1<<24`,0
+.Lmask26:
+.long	0x3ffffff,0,0x3ffffff,0,0x3ffffff,0,0x3ffffff,0
+.Lpermd_avx2:
+.long	2,2,2,3,2,0,2,1
+.Lpermd_avx512:
+.long	0,0,0,1, 0,2,0,3, 0,4,0,5, 0,6,0,7
+
+.L2_44_inp_permd:
+.long	0,1,1,2,2,3,7,7
+.L2_44_inp_shift:
+.quad	0,12,24,64
+.L2_44_mask:
+.quad	0xfffffffffff,0xfffffffffff,0x3ffffffffff,0xffffffffffffffff
+.L2_44_shift_rgt:
+.quad	44,44,42,64
+.L2_44_shift_lft:
+.quad	8,8,10,64
+
+.align	64
+.Lx_mask44:
+.quad	0xfffffffffff,0xfffffffffff,0xfffffffffff,0xfffffffffff
+.quad	0xfffffffffff,0xfffffffffff,0xfffffffffff,0xfffffffffff
+.Lx_mask42:
+.quad	0x3ffffffffff,0x3ffffffffff,0x3ffffffffff,0x3ffffffffff
+.quad	0x3ffffffffff,0x3ffffffffff,0x3ffffffffff,0x3ffffffffff
+___
+}
+$code.=<<___ if (!$kernel);
+.asciz	"Poly1305 for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
+.align	16
+___
+
+my ($ctx,$inp,$len,$padbit)=("%rdi","%rsi","%rdx","%rcx");
+my ($mac,$nonce)=($inp,$len);	# *_emit arguments
+my ($d1,$d2,$d3, $r0,$r1,$s1)=("%r8","%r9","%rdi","%r11","%r12","%r13");
+my ($h0,$h1,$h2)=("%r14","%rbx","%r10");
+
+sub poly1305_iteration {
+# input:	copy of $r1 in %rax, $h0-$h2, $r0-$r1
+# output:	$h0-$h2 *= $r0-$r1
+$code.=<<___;
+	mulq	$h0			# h0*r1
+	mov	%rax,$d2
+	 mov	$r0,%rax
+	mov	%rdx,$d3
+
+	mulq	$h0			# h0*r0
+	mov	%rax,$h0		# future $h0
+	 mov	$r0,%rax
+	mov	%rdx,$d1
+
+	mulq	$h1			# h1*r0
+	add	%rax,$d2
+	 mov	$s1,%rax
+	adc	%rdx,$d3
+
+	mulq	$h1			# h1*s1
+	 mov	$h2,$h1			# borrow $h1
+	add	%rax,$h0
+	adc	%rdx,$d1
+
+	imulq	$s1,$h1			# h2*s1
+	add	$h1,$d2
+	 mov	$d1,$h1
+	adc	\$0,$d3
+
+	imulq	$r0,$h2			# h2*r0
+	add	$d2,$h1
+	mov	\$-4,%rax		# mask value
+	adc	$h2,$d3
+
+	and	$d3,%rax		# last reduction step
+	mov	$d3,$h2
+	shr	\$2,$d3
+	and	\$3,$h2
+	add	$d3,%rax
+	add	%rax,$h0
+	adc	\$0,$h1
+	adc	\$0,$h2
+___
+}
+
+########################################################################
+# Layout of opaque area is following.
+#
+#	unsigned __int64 h[3];		# current hash value base 2^64
+#	unsigned __int64 r[2];		# key value base 2^64
+
+$code.=<<___;
+.text
+___
+$code.=<<___ if (!$kernel);
+.extern	OPENSSL_ia32cap_P
+
+.globl	poly1305_init_x86_64
+.hidden	poly1305_init_x86_64
+.globl	poly1305_blocks_x86_64
+.hidden	poly1305_blocks_x86_64
+.globl	poly1305_emit_x86_64
+.hidden	poly1305_emit_x86_64
+___
+&declare_function("poly1305_init_x86_64", 32, 3);
+$code.=<<___;
+	xor	%rax,%rax
+	mov	%rax,0($ctx)		# initialize hash value
+	mov	%rax,8($ctx)
+	mov	%rax,16($ctx)
+
+	cmp	\$0,$inp
+	je	.Lno_key
+___
+$code.=<<___ if (!$kernel);
+	lea	poly1305_blocks_x86_64(%rip),%r10
+	lea	poly1305_emit_x86_64(%rip),%r11
+___
+$code.=<<___	if (!$kernel && $avx);
+	mov	OPENSSL_ia32cap_P+4(%rip),%r9
+	lea	poly1305_blocks_avx(%rip),%rax
+	lea	poly1305_emit_avx(%rip),%rcx
+	bt	\$`60-32`,%r9		# AVX?
+	cmovc	%rax,%r10
+	cmovc	%rcx,%r11
+___
+$code.=<<___	if (!$kernel && $avx>1);
+	lea	poly1305_blocks_avx2(%rip),%rax
+	bt	\$`5+32`,%r9		# AVX2?
+	cmovc	%rax,%r10
+___
+$code.=<<___	if (!$kernel && $avx>3);
+	mov	\$`(1<<31|1<<21|1<<16)`,%rax
+	shr	\$32,%r9
+	and	%rax,%r9
+	cmp	%rax,%r9
+	je	.Linit_base2_44
+___
+$code.=<<___;
+	mov	\$0x0ffffffc0fffffff,%rax
+	mov	\$0x0ffffffc0ffffffc,%rcx
+	and	0($inp),%rax
+	and	8($inp),%rcx
+	mov	%rax,24($ctx)
+	mov	%rcx,32($ctx)
+___
+$code.=<<___	if (!$kernel && $flavour !~ /elf32/);
+	mov	%r10,0(%rdx)
+	mov	%r11,8(%rdx)
+___
+$code.=<<___	if (!$kernel && $flavour =~ /elf32/);
+	mov	%r10d,0(%rdx)
+	mov	%r11d,4(%rdx)
+___
+$code.=<<___;
+	mov	\$1,%eax
+.Lno_key:
+	ret
+___
+&end_function("poly1305_init_x86_64");
+
+&declare_function("poly1305_blocks_x86_64", 32, 4);
+$code.=<<___;
+.cfi_startproc
+.Lblocks:
+	shr	\$4,$len
+	jz	.Lno_data		# too short
+
+	push	%rbx
+.cfi_push	%rbx
+	push	%r12
+.cfi_push	%r12
+	push	%r13
+.cfi_push	%r13
+	push	%r14
+.cfi_push	%r14
+	push	%r15
+.cfi_push	%r15
+	push	$ctx
+.cfi_push	$ctx
+.Lblocks_body:
+
+	mov	$len,%r15		# reassign $len
+
+	mov	24($ctx),$r0		# load r
+	mov	32($ctx),$s1
+
+	mov	0($ctx),$h0		# load hash value
+	mov	8($ctx),$h1
+	mov	16($ctx),$h2
+
+	mov	$s1,$r1
+	shr	\$2,$s1
+	mov	$r1,%rax
+	add	$r1,$s1			# s1 = r1 + (r1 >> 2)
+	jmp	.Loop
+
+.align	32
+.Loop:
+	add	0($inp),$h0		# accumulate input
+	adc	8($inp),$h1
+	lea	16($inp),$inp
+	adc	$padbit,$h2
+___
+
+	&poly1305_iteration();
+
+$code.=<<___;
+	mov	$r1,%rax
+	dec	%r15			# len-=16
+	jnz	.Loop
+
+	mov	0(%rsp),$ctx
+.cfi_restore	$ctx
+
+	mov	$h0,0($ctx)		# store hash value
+	mov	$h1,8($ctx)
+	mov	$h2,16($ctx)
+
+	mov	8(%rsp),%r15
+.cfi_restore	%r15
+	mov	16(%rsp),%r14
+.cfi_restore	%r14
+	mov	24(%rsp),%r13
+.cfi_restore	%r13
+	mov	32(%rsp),%r12
+.cfi_restore	%r12
+	mov	40(%rsp),%rbx
+.cfi_restore	%rbx
+	lea	48(%rsp),%rsp
+.cfi_adjust_cfa_offset	-48
+.Lno_data:
+.Lblocks_epilogue:
+	ret
+.cfi_endproc
+___
+&end_function("poly1305_blocks_x86_64");
+
+&declare_function("poly1305_emit_x86_64", 32, 3);
+$code.=<<___;
+.Lemit:
+	mov	0($ctx),%r8	# load hash value
+	mov	8($ctx),%r9
+	mov	16($ctx),%r10
+
+	mov	%r8,%rax
+	add	\$5,%r8		# compare to modulus
+	mov	%r9,%rcx
+	adc	\$0,%r9
+	adc	\$0,%r10
+	shr	\$2,%r10	# did 130-bit value overflow?
+	cmovnz	%r8,%rax
+	cmovnz	%r9,%rcx
+
+	add	0($nonce),%rax	# accumulate nonce
+	adc	8($nonce),%rcx
+	mov	%rax,0($mac)	# write result
+	mov	%rcx,8($mac)
+
+	ret
+___
+&end_function("poly1305_emit_x86_64");
+if ($avx) {
+
+if($kernel) {
+	$code .= "#ifdef CONFIG_AS_AVX\n";
+}
+
+########################################################################
+# Layout of opaque area is following.
+#
+#	unsigned __int32 h[5];		# current hash value base 2^26
+#	unsigned __int32 is_base2_26;
+#	unsigned __int64 r[2];		# key value base 2^64
+#	unsigned __int64 pad;
+#	struct { unsigned __int32 r^2, r^1, r^4, r^3; } r[9];
+#
+# where r^n are base 2^26 digits of degrees of multiplier key. There are
+# 5 digits, but last four are interleaved with multiples of 5, totalling
+# in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4.
+
+my ($H0,$H1,$H2,$H3,$H4, $T0,$T1,$T2,$T3,$T4, $D0,$D1,$D2,$D3,$D4, $MASK) =
+    map("%xmm$_",(0..15));
+
+$code.=<<___;
+.type	__poly1305_block,\@abi-omnipotent
+.align	32
+__poly1305_block:
+	push $ctx
+___
+	&poly1305_iteration();
+$code.=<<___;
+	pop $ctx
+	ret
+.size	__poly1305_block,.-__poly1305_block
+
+.type	__poly1305_init_avx,\@abi-omnipotent
+.align	32
+__poly1305_init_avx:
+	push %rbp
+	mov %rsp,%rbp
+	mov	$r0,$h0
+	mov	$r1,$h1
+	xor	$h2,$h2
+
+	lea	48+64($ctx),$ctx	# size optimization
+
+	mov	$r1,%rax
+	call	__poly1305_block	# r^2
+
+	mov	\$0x3ffffff,%eax	# save interleaved r^2 and r base 2^26
+	mov	\$0x3ffffff,%edx
+	mov	$h0,$d1
+	and	$h0#d,%eax
+	mov	$r0,$d2
+	and	$r0#d,%edx
+	mov	%eax,`16*0+0-64`($ctx)
+	shr	\$26,$d1
+	mov	%edx,`16*0+4-64`($ctx)
+	shr	\$26,$d2
+
+	mov	\$0x3ffffff,%eax
+	mov	\$0x3ffffff,%edx
+	and	$d1#d,%eax
+	and	$d2#d,%edx
+	mov	%eax,`16*1+0-64`($ctx)
+	lea	(%rax,%rax,4),%eax	# *5
+	mov	%edx,`16*1+4-64`($ctx)
+	lea	(%rdx,%rdx,4),%edx	# *5
+	mov	%eax,`16*2+0-64`($ctx)
+	shr	\$26,$d1
+	mov	%edx,`16*2+4-64`($ctx)
+	shr	\$26,$d2
+
+	mov	$h1,%rax
+	mov	$r1,%rdx
+	shl	\$12,%rax
+	shl	\$12,%rdx
+	or	$d1,%rax
+	or	$d2,%rdx
+	and	\$0x3ffffff,%eax
+	and	\$0x3ffffff,%edx
+	mov	%eax,`16*3+0-64`($ctx)
+	lea	(%rax,%rax,4),%eax	# *5
+	mov	%edx,`16*3+4-64`($ctx)
+	lea	(%rdx,%rdx,4),%edx	# *5
+	mov	%eax,`16*4+0-64`($ctx)
+	mov	$h1,$d1
+	mov	%edx,`16*4+4-64`($ctx)
+	mov	$r1,$d2
+
+	mov	\$0x3ffffff,%eax
+	mov	\$0x3ffffff,%edx
+	shr	\$14,$d1
+	shr	\$14,$d2
+	and	$d1#d,%eax
+	and	$d2#d,%edx
+	mov	%eax,`16*5+0-64`($ctx)
+	lea	(%rax,%rax,4),%eax	# *5
+	mov	%edx,`16*5+4-64`($ctx)
+	lea	(%rdx,%rdx,4),%edx	# *5
+	mov	%eax,`16*6+0-64`($ctx)
+	shr	\$26,$d1
+	mov	%edx,`16*6+4-64`($ctx)
+	shr	\$26,$d2
+
+	mov	$h2,%rax
+	shl	\$24,%rax
+	or	%rax,$d1
+	mov	$d1#d,`16*7+0-64`($ctx)
+	lea	($d1,$d1,4),$d1		# *5
+	mov	$d2#d,`16*7+4-64`($ctx)
+	lea	($d2,$d2,4),$d2		# *5
+	mov	$d1#d,`16*8+0-64`($ctx)
+	mov	$d2#d,`16*8+4-64`($ctx)
+
+	mov	$r1,%rax
+	call	__poly1305_block	# r^3
+
+	mov	\$0x3ffffff,%eax	# save r^3 base 2^26
+	mov	$h0,$d1
+	and	$h0#d,%eax
+	shr	\$26,$d1
+	mov	%eax,`16*0+12-64`($ctx)
+
+	mov	\$0x3ffffff,%edx
+	and	$d1#d,%edx
+	mov	%edx,`16*1+12-64`($ctx)
+	lea	(%rdx,%rdx,4),%edx	# *5
+	shr	\$26,$d1
+	mov	%edx,`16*2+12-64`($ctx)
+
+	mov	$h1,%rax
+	shl	\$12,%rax
+	or	$d1,%rax
+	and	\$0x3ffffff,%eax
+	mov	%eax,`16*3+12-64`($ctx)
+	lea	(%rax,%rax,4),%eax	# *5
+	mov	$h1,$d1
+	mov	%eax,`16*4+12-64`($ctx)
+
+	mov	\$0x3ffffff,%edx
+	shr	\$14,$d1
+	and	$d1#d,%edx
+	mov	%edx,`16*5+12-64`($ctx)
+	lea	(%rdx,%rdx,4),%edx	# *5
+	shr	\$26,$d1
+	mov	%edx,`16*6+12-64`($ctx)
+
+	mov	$h2,%rax
+	shl	\$24,%rax
+	or	%rax,$d1
+	mov	$d1#d,`16*7+12-64`($ctx)
+	lea	($d1,$d1,4),$d1		# *5
+	mov	$d1#d,`16*8+12-64`($ctx)
+
+	mov	$r1,%rax
+	call	__poly1305_block	# r^4
+
+	mov	\$0x3ffffff,%eax	# save r^4 base 2^26
+	mov	$h0,$d1
+	and	$h0#d,%eax
+	shr	\$26,$d1
+	mov	%eax,`16*0+8-64`($ctx)
+
+	mov	\$0x3ffffff,%edx
+	and	$d1#d,%edx
+	mov	%edx,`16*1+8-64`($ctx)
+	lea	(%rdx,%rdx,4),%edx	# *5
+	shr	\$26,$d1
+	mov	%edx,`16*2+8-64`($ctx)
+
+	mov	$h1,%rax
+	shl	\$12,%rax
+	or	$d1,%rax
+	and	\$0x3ffffff,%eax
+	mov	%eax,`16*3+8-64`($ctx)
+	lea	(%rax,%rax,4),%eax	# *5
+	mov	$h1,$d1
+	mov	%eax,`16*4+8-64`($ctx)
+
+	mov	\$0x3ffffff,%edx
+	shr	\$14,$d1
+	and	$d1#d,%edx
+	mov	%edx,`16*5+8-64`($ctx)
+	lea	(%rdx,%rdx,4),%edx	# *5
+	shr	\$26,$d1
+	mov	%edx,`16*6+8-64`($ctx)
+
+	mov	$h2,%rax
+	shl	\$24,%rax
+	or	%rax,$d1
+	mov	$d1#d,`16*7+8-64`($ctx)
+	lea	($d1,$d1,4),$d1		# *5
+	mov	$d1#d,`16*8+8-64`($ctx)
+
+	lea	-48-64($ctx),$ctx	# size [de-]optimization
+	pop %rbp
+	ret
+.size	__poly1305_init_avx,.-__poly1305_init_avx
+___
+
+&declare_function("poly1305_blocks_avx", 32, 4);
+$code.=<<___;
+.cfi_startproc
+	mov	20($ctx),%r8d		# is_base2_26
+	cmp	\$128,$len
+	jae	.Lblocks_avx
+	test	%r8d,%r8d
+	jz	.Lblocks
+
+.Lblocks_avx:
+	and	\$-16,$len
+	jz	.Lno_data_avx
+
+	vzeroupper
+
+	test	%r8d,%r8d
+	jz	.Lbase2_64_avx
+
+	test	\$31,$len
+	jz	.Leven_avx
+
+	push	%rbp
+.cfi_push	%rbp
+	mov 	%rsp,%rbp
+	push	%rbx
+.cfi_push	%rbx
+	push	%r12
+.cfi_push	%r12
+	push	%r13
+.cfi_push	%r13
+	push	%r14
+.cfi_push	%r14
+	push	%r15
+.cfi_push	%r15
+.Lblocks_avx_body:
+
+	mov	$len,%r15		# reassign $len
+
+	mov	0($ctx),$d1		# load hash value
+	mov	8($ctx),$d2
+	mov	16($ctx),$h2#d
+
+	mov	24($ctx),$r0		# load r
+	mov	32($ctx),$s1
+
+	################################# base 2^26 -> base 2^64
+	mov	$d1#d,$h0#d
+	and	\$`-1*(1<<31)`,$d1
+	mov	$d2,$r1			# borrow $r1
+	mov	$d2#d,$h1#d
+	and	\$`-1*(1<<31)`,$d2
+
+	shr	\$6,$d1
+	shl	\$52,$r1
+	add	$d1,$h0
+	shr	\$12,$h1
+	shr	\$18,$d2
+	add	$r1,$h0
+	adc	$d2,$h1
+
+	mov	$h2,$d1
+	shl	\$40,$d1
+	shr	\$24,$h2
+	add	$d1,$h1
+	adc	\$0,$h2			# can be partially reduced...
+
+	mov	\$-4,$d2		# ... so reduce
+	mov	$h2,$d1
+	and	$h2,$d2
+	shr	\$2,$d1
+	and	\$3,$h2
+	add	$d2,$d1			# =*5
+	add	$d1,$h0
+	adc	\$0,$h1
+	adc	\$0,$h2
+
+	mov	$s1,$r1
+	mov	$s1,%rax
+	shr	\$2,$s1
+	add	$r1,$s1			# s1 = r1 + (r1 >> 2)
+
+	add	0($inp),$h0		# accumulate input
+	adc	8($inp),$h1
+	lea	16($inp),$inp
+	adc	$padbit,$h2
+
+	call	__poly1305_block
+
+	test	$padbit,$padbit		# if $padbit is zero,
+	jz	.Lstore_base2_64_avx	# store hash in base 2^64 format
+
+	################################# base 2^64 -> base 2^26
+	mov	$h0,%rax
+	mov	$h0,%rdx
+	shr	\$52,$h0
+	mov	$h1,$r0
+	mov	$h1,$r1
+	shr	\$26,%rdx
+	and	\$0x3ffffff,%rax	# h[0]
+	shl	\$12,$r0
+	and	\$0x3ffffff,%rdx	# h[1]
+	shr	\$14,$h1
+	or	$r0,$h0
+	shl	\$24,$h2
+	and	\$0x3ffffff,$h0		# h[2]
+	shr	\$40,$r1
+	and	\$0x3ffffff,$h1		# h[3]
+	or	$r1,$h2			# h[4]
+
+	sub	\$16,%r15
+	jz	.Lstore_base2_26_avx
+
+	vmovd	%rax#d,$H0
+	vmovd	%rdx#d,$H1
+	vmovd	$h0#d,$H2
+	vmovd	$h1#d,$H3
+	vmovd	$h2#d,$H4
+	jmp	.Lproceed_avx
+
+.align	32
+.Lstore_base2_64_avx:
+	mov	$h0,0($ctx)
+	mov	$h1,8($ctx)
+	mov	$h2,16($ctx)		# note that is_base2_26 is zeroed
+	jmp	.Ldone_avx
+
+.align	16
+.Lstore_base2_26_avx:
+	mov	%rax#d,0($ctx)		# store hash value base 2^26
+	mov	%rdx#d,4($ctx)
+	mov	$h0#d,8($ctx)
+	mov	$h1#d,12($ctx)
+	mov	$h2#d,16($ctx)
+.align	16
+.Ldone_avx:
+	pop 		%r15
+.cfi_restore	%r15
+	pop 		%r14
+.cfi_restore	%r14
+	pop 		%r13
+.cfi_restore	%r13
+	pop 		%r12
+.cfi_restore	%r12
+	pop 		%rbx
+.cfi_restore	%rbx
+	pop 		%rbp
+.cfi_restore	%rbp
+.Lno_data_avx:
+.Lblocks_avx_epilogue:
+	ret
+.cfi_endproc
+
+.align	32
+.Lbase2_64_avx:
+.cfi_startproc
+	push	%rbp
+.cfi_push	%rbp
+	mov 	%rsp,%rbp
+	push	%rbx
+.cfi_push	%rbx
+	push	%r12
+.cfi_push	%r12
+	push	%r13
+.cfi_push	%r13
+	push	%r14
+.cfi_push	%r14
+	push	%r15
+.cfi_push	%r15
+.Lbase2_64_avx_body:
+
+	mov	$len,%r15		# reassign $len
+
+	mov	24($ctx),$r0		# load r
+	mov	32($ctx),$s1
+
+	mov	0($ctx),$h0		# load hash value
+	mov	8($ctx),$h1
+	mov	16($ctx),$h2#d
+
+	mov	$s1,$r1
+	mov	$s1,%rax
+	shr	\$2,$s1
+	add	$r1,$s1			# s1 = r1 + (r1 >> 2)
+
+	test	\$31,$len
+	jz	.Linit_avx
+
+	add	0($inp),$h0		# accumulate input
+	adc	8($inp),$h1
+	lea	16($inp),$inp
+	adc	$padbit,$h2
+	sub	\$16,%r15
+
+	call	__poly1305_block
+
+.Linit_avx:
+	################################# base 2^64 -> base 2^26
+	mov	$h0,%rax
+	mov	$h0,%rdx
+	shr	\$52,$h0
+	mov	$h1,$d1
+	mov	$h1,$d2
+	shr	\$26,%rdx
+	and	\$0x3ffffff,%rax	# h[0]
+	shl	\$12,$d1
+	and	\$0x3ffffff,%rdx	# h[1]
+	shr	\$14,$h1
+	or	$d1,$h0
+	shl	\$24,$h2
+	and	\$0x3ffffff,$h0		# h[2]
+	shr	\$40,$d2
+	and	\$0x3ffffff,$h1		# h[3]
+	or	$d2,$h2			# h[4]
+
+	vmovd	%rax#d,$H0
+	vmovd	%rdx#d,$H1
+	vmovd	$h0#d,$H2
+	vmovd	$h1#d,$H3
+	vmovd	$h2#d,$H4
+	movl	\$1,20($ctx)		# set is_base2_26
+
+	call	__poly1305_init_avx
+
+.Lproceed_avx:
+	mov	%r15,$len
+	pop 		%r15
+.cfi_restore	%r15
+	pop 		%r14
+.cfi_restore	%r14
+	pop 		%r13
+.cfi_restore	%r13
+	pop 		%r12
+.cfi_restore	%r12
+	pop 		%rbx
+.cfi_restore	%rbx
+	pop 		%rbp
+.cfi_restore	%rbp
+.Lbase2_64_avx_epilogue:
+	jmp	.Ldo_avx
+.cfi_endproc
+
+.align	32
+.Leven_avx:
+.cfi_startproc
+	vmovd		4*0($ctx),$H0		# load hash value
+	vmovd		4*1($ctx),$H1
+	vmovd		4*2($ctx),$H2
+	vmovd		4*3($ctx),$H3
+	vmovd		4*4($ctx),$H4
+
+.Ldo_avx:
+___
+$code.=<<___	if (!$win64);
+	lea		8(%rsp),%r10
+.cfi_def_cfa_register	%r10
+	and		\$-32,%rsp
+	sub		\$-8,%rsp
+	lea		-0x58(%rsp),%r11
+	sub		\$0x178,%rsp
+
+___
+$code.=<<___	if ($win64);
+	lea		-0xf8(%rsp),%r11
+	sub		\$0x218,%rsp
+	vmovdqa		%xmm6,0x50(%r11)
+	vmovdqa		%xmm7,0x60(%r11)
+	vmovdqa		%xmm8,0x70(%r11)
+	vmovdqa		%xmm9,0x80(%r11)
+	vmovdqa		%xmm10,0x90(%r11)
+	vmovdqa		%xmm11,0xa0(%r11)
+	vmovdqa		%xmm12,0xb0(%r11)
+	vmovdqa		%xmm13,0xc0(%r11)
+	vmovdqa		%xmm14,0xd0(%r11)
+	vmovdqa		%xmm15,0xe0(%r11)
+.Ldo_avx_body:
+___
+$code.=<<___;
+	sub		\$64,$len
+	lea		-32($inp),%rax
+	cmovc		%rax,$inp
+
+	vmovdqu		`16*3`($ctx),$D4	# preload r0^2
+	lea		`16*3+64`($ctx),$ctx	# size optimization
+	lea		.Lconst(%rip),%rcx
+
+	################################################################
+	# load input
+	vmovdqu		16*2($inp),$T0
+	vmovdqu		16*3($inp),$T1
+	vmovdqa		64(%rcx),$MASK		# .Lmask26
+
+	vpsrldq		\$6,$T0,$T2		# splat input
+	vpsrldq		\$6,$T1,$T3
+	vpunpckhqdq	$T1,$T0,$T4		# 4
+	vpunpcklqdq	$T1,$T0,$T0		# 0:1
+	vpunpcklqdq	$T3,$T2,$T3		# 2:3
+
+	vpsrlq		\$40,$T4,$T4		# 4
+	vpsrlq		\$26,$T0,$T1
+	vpand		$MASK,$T0,$T0		# 0
+	vpsrlq		\$4,$T3,$T2
+	vpand		$MASK,$T1,$T1		# 1
+	vpsrlq		\$30,$T3,$T3
+	vpand		$MASK,$T2,$T2		# 2
+	vpand		$MASK,$T3,$T3		# 3
+	vpor		32(%rcx),$T4,$T4	# padbit, yes, always
+
+	jbe		.Lskip_loop_avx
+
+	# expand and copy pre-calculated table to stack
+	vmovdqu		`16*1-64`($ctx),$D1
+	vmovdqu		`16*2-64`($ctx),$D2
+	vpshufd		\$0xEE,$D4,$D3		# 34xx -> 3434
+	vpshufd		\$0x44,$D4,$D0		# xx12 -> 1212
+	vmovdqa		$D3,-0x90(%r11)
+	vmovdqa		$D0,0x00(%rsp)
+	vpshufd		\$0xEE,$D1,$D4
+	vmovdqu		`16*3-64`($ctx),$D0
+	vpshufd		\$0x44,$D1,$D1
+	vmovdqa		$D4,-0x80(%r11)
+	vmovdqa		$D1,0x10(%rsp)
+	vpshufd		\$0xEE,$D2,$D3
+	vmovdqu		`16*4-64`($ctx),$D1
+	vpshufd		\$0x44,$D2,$D2
+	vmovdqa		$D3,-0x70(%r11)
+	vmovdqa		$D2,0x20(%rsp)
+	vpshufd		\$0xEE,$D0,$D4
+	vmovdqu		`16*5-64`($ctx),$D2
+	vpshufd		\$0x44,$D0,$D0
+	vmovdqa		$D4,-0x60(%r11)
+	vmovdqa		$D0,0x30(%rsp)
+	vpshufd		\$0xEE,$D1,$D3
+	vmovdqu		`16*6-64`($ctx),$D0
+	vpshufd		\$0x44,$D1,$D1
+	vmovdqa		$D3,-0x50(%r11)
+	vmovdqa		$D1,0x40(%rsp)
+	vpshufd		\$0xEE,$D2,$D4
+	vmovdqu		`16*7-64`($ctx),$D1
+	vpshufd		\$0x44,$D2,$D2
+	vmovdqa		$D4,-0x40(%r11)
+	vmovdqa		$D2,0x50(%rsp)
+	vpshufd		\$0xEE,$D0,$D3
+	vmovdqu		`16*8-64`($ctx),$D2
+	vpshufd		\$0x44,$D0,$D0
+	vmovdqa		$D3,-0x30(%r11)
+	vmovdqa		$D0,0x60(%rsp)
+	vpshufd		\$0xEE,$D1,$D4
+	vpshufd		\$0x44,$D1,$D1
+	vmovdqa		$D4,-0x20(%r11)
+	vmovdqa		$D1,0x70(%rsp)
+	vpshufd		\$0xEE,$D2,$D3
+	 vmovdqa	0x00(%rsp),$D4		# preload r0^2
+	vpshufd		\$0x44,$D2,$D2
+	vmovdqa		$D3,-0x10(%r11)
+	vmovdqa		$D2,0x80(%rsp)
+
+	jmp		.Loop_avx
+
+.align	32
+.Loop_avx:
+	################################################################
+	# ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
+	# ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
+	#   \___________________/
+	# ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
+	# ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
+	#   \___________________/ \____________________/
+	#
+	# Note that we start with inp[2:3]*r^2. This is because it
+	# doesn't depend on reduction in previous iteration.
+	################################################################
+	# d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
+	# d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
+	# d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
+	# d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
+	# d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
+	#
+	# though note that $Tx and $Hx are "reversed" in this section,
+	# and $D4 is preloaded with r0^2...
+
+	vpmuludq	$T0,$D4,$D0		# d0 = h0*r0
+	vpmuludq	$T1,$D4,$D1		# d1 = h1*r0
+	  vmovdqa	$H2,0x20(%r11)				# offload hash
+	vpmuludq	$T2,$D4,$D2		# d3 = h2*r0
+	 vmovdqa	0x10(%rsp),$H2		# r1^2
+	vpmuludq	$T3,$D4,$D3		# d3 = h3*r0
+	vpmuludq	$T4,$D4,$D4		# d4 = h4*r0
+
+	  vmovdqa	$H0,0x00(%r11)				#
+	vpmuludq	0x20(%rsp),$T4,$H0	# h4*s1
+	  vmovdqa	$H1,0x10(%r11)				#
+	vpmuludq	$T3,$H2,$H1		# h3*r1
+	vpaddq		$H0,$D0,$D0		# d0 += h4*s1
+	vpaddq		$H1,$D4,$D4		# d4 += h3*r1
+	  vmovdqa	$H3,0x30(%r11)				#
+	vpmuludq	$T2,$H2,$H0		# h2*r1
+	vpmuludq	$T1,$H2,$H1		# h1*r1
+	vpaddq		$H0,$D3,$D3		# d3 += h2*r1
+	 vmovdqa	0x30(%rsp),$H3		# r2^2
+	vpaddq		$H1,$D2,$D2		# d2 += h1*r1
+	  vmovdqa	$H4,0x40(%r11)				#
+	vpmuludq	$T0,$H2,$H2		# h0*r1
+	 vpmuludq	$T2,$H3,$H0		# h2*r2
+	vpaddq		$H2,$D1,$D1		# d1 += h0*r1
+
+	 vmovdqa	0x40(%rsp),$H4		# s2^2
+	vpaddq		$H0,$D4,$D4		# d4 += h2*r2
+	vpmuludq	$T1,$H3,$H1		# h1*r2
+	vpmuludq	$T0,$H3,$H3		# h0*r2
+	vpaddq		$H1,$D3,$D3		# d3 += h1*r2
+	 vmovdqa	0x50(%rsp),$H2		# r3^2
+	vpaddq		$H3,$D2,$D2		# d2 += h0*r2
+	vpmuludq	$T4,$H4,$H0		# h4*s2
+	vpmuludq	$T3,$H4,$H4		# h3*s2
+	vpaddq		$H0,$D1,$D1		# d1 += h4*s2
+	 vmovdqa	0x60(%rsp),$H3		# s3^2
+	vpaddq		$H4,$D0,$D0		# d0 += h3*s2
+
+	 vmovdqa	0x80(%rsp),$H4		# s4^2
+	vpmuludq	$T1,$H2,$H1		# h1*r3
+	vpmuludq	$T0,$H2,$H2		# h0*r3
+	vpaddq		$H1,$D4,$D4		# d4 += h1*r3
+	vpaddq		$H2,$D3,$D3		# d3 += h0*r3
+	vpmuludq	$T4,$H3,$H0		# h4*s3
+	vpmuludq	$T3,$H3,$H1		# h3*s3
+	vpaddq		$H0,$D2,$D2		# d2 += h4*s3
+	 vmovdqu	16*0($inp),$H0				# load input
+	vpaddq		$H1,$D1,$D1		# d1 += h3*s3
+	vpmuludq	$T2,$H3,$H3		# h2*s3
+	 vpmuludq	$T2,$H4,$T2		# h2*s4
+	vpaddq		$H3,$D0,$D0		# d0 += h2*s3
+
+	 vmovdqu	16*1($inp),$H1				#
+	vpaddq		$T2,$D1,$D1		# d1 += h2*s4
+	vpmuludq	$T3,$H4,$T3		# h3*s4
+	vpmuludq	$T4,$H4,$T4		# h4*s4
+	 vpsrldq	\$6,$H0,$H2				# splat input
+	vpaddq		$T3,$D2,$D2		# d2 += h3*s4
+	vpaddq		$T4,$D3,$D3		# d3 += h4*s4
+	 vpsrldq	\$6,$H1,$H3				#
+	vpmuludq	0x70(%rsp),$T0,$T4	# h0*r4
+	vpmuludq	$T1,$H4,$T0		# h1*s4
+	 vpunpckhqdq	$H1,$H0,$H4		# 4
+	vpaddq		$T4,$D4,$D4		# d4 += h0*r4
+	 vmovdqa	-0x90(%r11),$T4		# r0^4
+	vpaddq		$T0,$D0,$D0		# d0 += h1*s4
+
+	vpunpcklqdq	$H1,$H0,$H0		# 0:1
+	vpunpcklqdq	$H3,$H2,$H3		# 2:3
+
+	#vpsrlq		\$40,$H4,$H4		# 4
+	vpsrldq		\$`40/8`,$H4,$H4	# 4
+	vpsrlq		\$26,$H0,$H1
+	vpand		$MASK,$H0,$H0		# 0
+	vpsrlq		\$4,$H3,$H2
+	vpand		$MASK,$H1,$H1		# 1
+	vpand		0(%rcx),$H4,$H4		# .Lmask24
+	vpsrlq		\$30,$H3,$H3
+	vpand		$MASK,$H2,$H2		# 2
+	vpand		$MASK,$H3,$H3		# 3
+	vpor		32(%rcx),$H4,$H4	# padbit, yes, always
+
+	vpaddq		0x00(%r11),$H0,$H0	# add hash value
+	vpaddq		0x10(%r11),$H1,$H1
+	vpaddq		0x20(%r11),$H2,$H2
+	vpaddq		0x30(%r11),$H3,$H3
+	vpaddq		0x40(%r11),$H4,$H4
+
+	lea		16*2($inp),%rax
+	lea		16*4($inp),$inp
+	sub		\$64,$len
+	cmovc		%rax,$inp
+
+	################################################################
+	# Now we accumulate (inp[0:1]+hash)*r^4
+	################################################################
+	# d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
+	# d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
+	# d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
+	# d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
+	# d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
+
+	vpmuludq	$H0,$T4,$T0		# h0*r0
+	vpmuludq	$H1,$T4,$T1		# h1*r0
+	vpaddq		$T0,$D0,$D0
+	vpaddq		$T1,$D1,$D1
+	 vmovdqa	-0x80(%r11),$T2		# r1^4
+	vpmuludq	$H2,$T4,$T0		# h2*r0
+	vpmuludq	$H3,$T4,$T1		# h3*r0
+	vpaddq		$T0,$D2,$D2
+	vpaddq		$T1,$D3,$D3
+	vpmuludq	$H4,$T4,$T4		# h4*r0
+	 vpmuludq	-0x70(%r11),$H4,$T0	# h4*s1
+	vpaddq		$T4,$D4,$D4
+
+	vpaddq		$T0,$D0,$D0		# d0 += h4*s1
+	vpmuludq	$H2,$T2,$T1		# h2*r1
+	vpmuludq	$H3,$T2,$T0		# h3*r1
+	vpaddq		$T1,$D3,$D3		# d3 += h2*r1
+	 vmovdqa	-0x60(%r11),$T3		# r2^4
+	vpaddq		$T0,$D4,$D4		# d4 += h3*r1
+	vpmuludq	$H1,$T2,$T1		# h1*r1
+	vpmuludq	$H0,$T2,$T2		# h0*r1
+	vpaddq		$T1,$D2,$D2		# d2 += h1*r1
+	vpaddq		$T2,$D1,$D1		# d1 += h0*r1
+
+	 vmovdqa	-0x50(%r11),$T4		# s2^4
+	vpmuludq	$H2,$T3,$T0		# h2*r2
+	vpmuludq	$H1,$T3,$T1		# h1*r2
+	vpaddq		$T0,$D4,$D4		# d4 += h2*r2
+	vpaddq		$T1,$D3,$D3		# d3 += h1*r2
+	 vmovdqa	-0x40(%r11),$T2		# r3^4
+	vpmuludq	$H0,$T3,$T3		# h0*r2
+	vpmuludq	$H4,$T4,$T0		# h4*s2
+	vpaddq		$T3,$D2,$D2		# d2 += h0*r2
+	vpaddq		$T0,$D1,$D1		# d1 += h4*s2
+	 vmovdqa	-0x30(%r11),$T3		# s3^4
+	vpmuludq	$H3,$T4,$T4		# h3*s2
+	 vpmuludq	$H1,$T2,$T1		# h1*r3
+	vpaddq		$T4,$D0,$D0		# d0 += h3*s2
+
+	 vmovdqa	-0x10(%r11),$T4		# s4^4
+	vpaddq		$T1,$D4,$D4		# d4 += h1*r3
+	vpmuludq	$H0,$T2,$T2		# h0*r3
+	vpmuludq	$H4,$T3,$T0		# h4*s3
+	vpaddq		$T2,$D3,$D3		# d3 += h0*r3
+	vpaddq		$T0,$D2,$D2		# d2 += h4*s3
+	 vmovdqu	16*2($inp),$T0				# load input
+	vpmuludq	$H3,$T3,$T2		# h3*s3
+	vpmuludq	$H2,$T3,$T3		# h2*s3
+	vpaddq		$T2,$D1,$D1		# d1 += h3*s3
+	 vmovdqu	16*3($inp),$T1				#
+	vpaddq		$T3,$D0,$D0		# d0 += h2*s3
+
+	vpmuludq	$H2,$T4,$H2		# h2*s4
+	vpmuludq	$H3,$T4,$H3		# h3*s4
+	 vpsrldq	\$6,$T0,$T2				# splat input
+	vpaddq		$H2,$D1,$D1		# d1 += h2*s4
+	vpmuludq	$H4,$T4,$H4		# h4*s4
+	 vpsrldq	\$6,$T1,$T3				#
+	vpaddq		$H3,$D2,$H2		# h2 = d2 + h3*s4
+	vpaddq		$H4,$D3,$H3		# h3 = d3 + h4*s4
+	vpmuludq	-0x20(%r11),$H0,$H4	# h0*r4
+	vpmuludq	$H1,$T4,$H0
+	 vpunpckhqdq	$T1,$T0,$T4		# 4
+	vpaddq		$H4,$D4,$H4		# h4 = d4 + h0*r4
+	vpaddq		$H0,$D0,$H0		# h0 = d0 + h1*s4
+
+	vpunpcklqdq	$T1,$T0,$T0		# 0:1
+	vpunpcklqdq	$T3,$T2,$T3		# 2:3
+
+	#vpsrlq		\$40,$T4,$T4		# 4
+	vpsrldq		\$`40/8`,$T4,$T4	# 4
+	vpsrlq		\$26,$T0,$T1
+	 vmovdqa	0x00(%rsp),$D4		# preload r0^2
+	vpand		$MASK,$T0,$T0		# 0
+	vpsrlq		\$4,$T3,$T2
+	vpand		$MASK,$T1,$T1		# 1
+	vpand		0(%rcx),$T4,$T4		# .Lmask24
+	vpsrlq		\$30,$T3,$T3
+	vpand		$MASK,$T2,$T2		# 2
+	vpand		$MASK,$T3,$T3		# 3
+	vpor		32(%rcx),$T4,$T4	# padbit, yes, always
+
+	################################################################
+	# lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
+	# and P. Schwabe
+
+	vpsrlq		\$26,$H3,$D3
+	vpand		$MASK,$H3,$H3
+	vpaddq		$D3,$H4,$H4		# h3 -> h4
+
+	vpsrlq		\$26,$H0,$D0
+	vpand		$MASK,$H0,$H0
+	vpaddq		$D0,$D1,$H1		# h0 -> h1
+
+	vpsrlq		\$26,$H4,$D0
+	vpand		$MASK,$H4,$H4
+
+	vpsrlq		\$26,$H1,$D1
+	vpand		$MASK,$H1,$H1
+	vpaddq		$D1,$H2,$H2		# h1 -> h2
+
+	vpaddq		$D0,$H0,$H0
+	vpsllq		\$2,$D0,$D0
+	vpaddq		$D0,$H0,$H0		# h4 -> h0
+
+	vpsrlq		\$26,$H2,$D2
+	vpand		$MASK,$H2,$H2
+	vpaddq		$D2,$H3,$H3		# h2 -> h3
+
+	vpsrlq		\$26,$H0,$D0
+	vpand		$MASK,$H0,$H0
+	vpaddq		$D0,$H1,$H1		# h0 -> h1
+
+	vpsrlq		\$26,$H3,$D3
+	vpand		$MASK,$H3,$H3
+	vpaddq		$D3,$H4,$H4		# h3 -> h4
+
+	ja		.Loop_avx
+
+.Lskip_loop_avx:
+	################################################################
+	# multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
+
+	vpshufd		\$0x10,$D4,$D4		# r0^n, xx12 -> x1x2
+	add		\$32,$len
+	jnz		.Long_tail_avx
+
+	vpaddq		$H2,$T2,$T2
+	vpaddq		$H0,$T0,$T0
+	vpaddq		$H1,$T1,$T1
+	vpaddq		$H3,$T3,$T3
+	vpaddq		$H4,$T4,$T4
+
+.Long_tail_avx:
+	vmovdqa		$H2,0x20(%r11)
+	vmovdqa		$H0,0x00(%r11)
+	vmovdqa		$H1,0x10(%r11)
+	vmovdqa		$H3,0x30(%r11)
+	vmovdqa		$H4,0x40(%r11)
+
+	# d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
+	# d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
+	# d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
+	# d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
+	# d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
+
+	vpmuludq	$T2,$D4,$D2		# d2 = h2*r0
+	vpmuludq	$T0,$D4,$D0		# d0 = h0*r0
+	 vpshufd	\$0x10,`16*1-64`($ctx),$H2		# r1^n
+	vpmuludq	$T1,$D4,$D1		# d1 = h1*r0
+	vpmuludq	$T3,$D4,$D3		# d3 = h3*r0
+	vpmuludq	$T4,$D4,$D4		# d4 = h4*r0
+
+	vpmuludq	$T3,$H2,$H0		# h3*r1
+	vpaddq		$H0,$D4,$D4		# d4 += h3*r1
+	 vpshufd	\$0x10,`16*2-64`($ctx),$H3		# s1^n
+	vpmuludq	$T2,$H2,$H1		# h2*r1
+	vpaddq		$H1,$D3,$D3		# d3 += h2*r1
+	 vpshufd	\$0x10,`16*3-64`($ctx),$H4		# r2^n
+	vpmuludq	$T1,$H2,$H0		# h1*r1
+	vpaddq		$H0,$D2,$D2		# d2 += h1*r1
+	vpmuludq	$T0,$H2,$H2		# h0*r1
+	vpaddq		$H2,$D1,$D1		# d1 += h0*r1
+	vpmuludq	$T4,$H3,$H3		# h4*s1
+	vpaddq		$H3,$D0,$D0		# d0 += h4*s1
+
+	 vpshufd	\$0x10,`16*4-64`($ctx),$H2		# s2^n
+	vpmuludq	$T2,$H4,$H1		# h2*r2
+	vpaddq		$H1,$D4,$D4		# d4 += h2*r2
+	vpmuludq	$T1,$H4,$H0		# h1*r2
+	vpaddq		$H0,$D3,$D3		# d3 += h1*r2
+	 vpshufd	\$0x10,`16*5-64`($ctx),$H3		# r3^n
+	vpmuludq	$T0,$H4,$H4		# h0*r2
+	vpaddq		$H4,$D2,$D2		# d2 += h0*r2
+	vpmuludq	$T4,$H2,$H1		# h4*s2
+	vpaddq		$H1,$D1,$D1		# d1 += h4*s2
+	 vpshufd	\$0x10,`16*6-64`($ctx),$H4		# s3^n
+	vpmuludq	$T3,$H2,$H2		# h3*s2
+	vpaddq		$H2,$D0,$D0		# d0 += h3*s2
+
+	vpmuludq	$T1,$H3,$H0		# h1*r3
+	vpaddq		$H0,$D4,$D4		# d4 += h1*r3
+	vpmuludq	$T0,$H3,$H3		# h0*r3
+	vpaddq		$H3,$D3,$D3		# d3 += h0*r3
+	 vpshufd	\$0x10,`16*7-64`($ctx),$H2		# r4^n
+	vpmuludq	$T4,$H4,$H1		# h4*s3
+	vpaddq		$H1,$D2,$D2		# d2 += h4*s3
+	 vpshufd	\$0x10,`16*8-64`($ctx),$H3		# s4^n
+	vpmuludq	$T3,$H4,$H0		# h3*s3
+	vpaddq		$H0,$D1,$D1		# d1 += h3*s3
+	vpmuludq	$T2,$H4,$H4		# h2*s3
+	vpaddq		$H4,$D0,$D0		# d0 += h2*s3
+
+	vpmuludq	$T0,$H2,$H2		# h0*r4
+	vpaddq		$H2,$D4,$D4		# h4 = d4 + h0*r4
+	vpmuludq	$T4,$H3,$H1		# h4*s4
+	vpaddq		$H1,$D3,$D3		# h3 = d3 + h4*s4
+	vpmuludq	$T3,$H3,$H0		# h3*s4
+	vpaddq		$H0,$D2,$D2		# h2 = d2 + h3*s4
+	vpmuludq	$T2,$H3,$H1		# h2*s4
+	vpaddq		$H1,$D1,$D1		# h1 = d1 + h2*s4
+	vpmuludq	$T1,$H3,$H3		# h1*s4
+	vpaddq		$H3,$D0,$D0		# h0 = d0 + h1*s4
+
+	jz		.Lshort_tail_avx
+
+	vmovdqu		16*0($inp),$H0		# load input
+	vmovdqu		16*1($inp),$H1
+
+	vpsrldq		\$6,$H0,$H2		# splat input
+	vpsrldq		\$6,$H1,$H3
+	vpunpckhqdq	$H1,$H0,$H4		# 4
+	vpunpcklqdq	$H1,$H0,$H0		# 0:1
+	vpunpcklqdq	$H3,$H2,$H3		# 2:3
+
+	vpsrlq		\$40,$H4,$H4		# 4
+	vpsrlq		\$26,$H0,$H1
+	vpand		$MASK,$H0,$H0		# 0
+	vpsrlq		\$4,$H3,$H2
+	vpand		$MASK,$H1,$H1		# 1
+	vpsrlq		\$30,$H3,$H3
+	vpand		$MASK,$H2,$H2		# 2
+	vpand		$MASK,$H3,$H3		# 3
+	vpor		32(%rcx),$H4,$H4	# padbit, yes, always
+
+	vpshufd		\$0x32,`16*0-64`($ctx),$T4	# r0^n, 34xx -> x3x4
+	vpaddq		0x00(%r11),$H0,$H0
+	vpaddq		0x10(%r11),$H1,$H1
+	vpaddq		0x20(%r11),$H2,$H2
+	vpaddq		0x30(%r11),$H3,$H3
+	vpaddq		0x40(%r11),$H4,$H4
+
+	################################################################
+	# multiply (inp[0:1]+hash) by r^4:r^3 and accumulate
+
+	vpmuludq	$H0,$T4,$T0		# h0*r0
+	vpaddq		$T0,$D0,$D0		# d0 += h0*r0
+	vpmuludq	$H1,$T4,$T1		# h1*r0
+	vpaddq		$T1,$D1,$D1		# d1 += h1*r0
+	vpmuludq	$H2,$T4,$T0		# h2*r0
+	vpaddq		$T0,$D2,$D2		# d2 += h2*r0
+	 vpshufd	\$0x32,`16*1-64`($ctx),$T2		# r1^n
+	vpmuludq	$H3,$T4,$T1		# h3*r0
+	vpaddq		$T1,$D3,$D3		# d3 += h3*r0
+	vpmuludq	$H4,$T4,$T4		# h4*r0
+	vpaddq		$T4,$D4,$D4		# d4 += h4*r0
+
+	vpmuludq	$H3,$T2,$T0		# h3*r1
+	vpaddq		$T0,$D4,$D4		# d4 += h3*r1
+	 vpshufd	\$0x32,`16*2-64`($ctx),$T3		# s1
+	vpmuludq	$H2,$T2,$T1		# h2*r1
+	vpaddq		$T1,$D3,$D3		# d3 += h2*r1
+	 vpshufd	\$0x32,`16*3-64`($ctx),$T4		# r2
+	vpmuludq	$H1,$T2,$T0		# h1*r1
+	vpaddq		$T0,$D2,$D2		# d2 += h1*r1
+	vpmuludq	$H0,$T2,$T2		# h0*r1
+	vpaddq		$T2,$D1,$D1		# d1 += h0*r1
+	vpmuludq	$H4,$T3,$T3		# h4*s1
+	vpaddq		$T3,$D0,$D0		# d0 += h4*s1
+
+	 vpshufd	\$0x32,`16*4-64`($ctx),$T2		# s2
+	vpmuludq	$H2,$T4,$T1		# h2*r2
+	vpaddq		$T1,$D4,$D4		# d4 += h2*r2
+	vpmuludq	$H1,$T4,$T0		# h1*r2
+	vpaddq		$T0,$D3,$D3		# d3 += h1*r2
+	 vpshufd	\$0x32,`16*5-64`($ctx),$T3		# r3
+	vpmuludq	$H0,$T4,$T4		# h0*r2
+	vpaddq		$T4,$D2,$D2		# d2 += h0*r2
+	vpmuludq	$H4,$T2,$T1		# h4*s2
+	vpaddq		$T1,$D1,$D1		# d1 += h4*s2
+	 vpshufd	\$0x32,`16*6-64`($ctx),$T4		# s3
+	vpmuludq	$H3,$T2,$T2		# h3*s2
+	vpaddq		$T2,$D0,$D0		# d0 += h3*s2
+
+	vpmuludq	$H1,$T3,$T0		# h1*r3
+	vpaddq		$T0,$D4,$D4		# d4 += h1*r3
+	vpmuludq	$H0,$T3,$T3		# h0*r3
+	vpaddq		$T3,$D3,$D3		# d3 += h0*r3
+	 vpshufd	\$0x32,`16*7-64`($ctx),$T2		# r4
+	vpmuludq	$H4,$T4,$T1		# h4*s3
+	vpaddq		$T1,$D2,$D2		# d2 += h4*s3
+	 vpshufd	\$0x32,`16*8-64`($ctx),$T3		# s4
+	vpmuludq	$H3,$T4,$T0		# h3*s3
+	vpaddq		$T0,$D1,$D1		# d1 += h3*s3
+	vpmuludq	$H2,$T4,$T4		# h2*s3
+	vpaddq		$T4,$D0,$D0		# d0 += h2*s3
+
+	vpmuludq	$H0,$T2,$T2		# h0*r4
+	vpaddq		$T2,$D4,$D4		# d4 += h0*r4
+	vpmuludq	$H4,$T3,$T1		# h4*s4
+	vpaddq		$T1,$D3,$D3		# d3 += h4*s4
+	vpmuludq	$H3,$T3,$T0		# h3*s4
+	vpaddq		$T0,$D2,$D2		# d2 += h3*s4
+	vpmuludq	$H2,$T3,$T1		# h2*s4
+	vpaddq		$T1,$D1,$D1		# d1 += h2*s4
+	vpmuludq	$H1,$T3,$T3		# h1*s4
+	vpaddq		$T3,$D0,$D0		# d0 += h1*s4
+
+.Lshort_tail_avx:
+	################################################################
+	# horizontal addition
+
+	vpsrldq		\$8,$D4,$T4
+	vpsrldq		\$8,$D3,$T3
+	vpsrldq		\$8,$D1,$T1
+	vpsrldq		\$8,$D0,$T0
+	vpsrldq		\$8,$D2,$T2
+	vpaddq		$T3,$D3,$D3
+	vpaddq		$T4,$D4,$D4
+	vpaddq		$T0,$D0,$D0
+	vpaddq		$T1,$D1,$D1
+	vpaddq		$T2,$D2,$D2
+
+	################################################################
+	# lazy reduction
+
+	vpsrlq		\$26,$D3,$H3
+	vpand		$MASK,$D3,$D3
+	vpaddq		$H3,$D4,$D4		# h3 -> h4
+
+	vpsrlq		\$26,$D0,$H0
+	vpand		$MASK,$D0,$D0
+	vpaddq		$H0,$D1,$D1		# h0 -> h1
+
+	vpsrlq		\$26,$D4,$H4
+	vpand		$MASK,$D4,$D4
+
+	vpsrlq		\$26,$D1,$H1
+	vpand		$MASK,$D1,$D1
+	vpaddq		$H1,$D2,$D2		# h1 -> h2
+
+	vpaddq		$H4,$D0,$D0
+	vpsllq		\$2,$H4,$H4
+	vpaddq		$H4,$D0,$D0		# h4 -> h0
+
+	vpsrlq		\$26,$D2,$H2
+	vpand		$MASK,$D2,$D2
+	vpaddq		$H2,$D3,$D3		# h2 -> h3
+
+	vpsrlq		\$26,$D0,$H0
+	vpand		$MASK,$D0,$D0
+	vpaddq		$H0,$D1,$D1		# h0 -> h1
+
+	vpsrlq		\$26,$D3,$H3
+	vpand		$MASK,$D3,$D3
+	vpaddq		$H3,$D4,$D4		# h3 -> h4
+
+	vmovd		$D0,`4*0-48-64`($ctx)	# save partially reduced
+	vmovd		$D1,`4*1-48-64`($ctx)
+	vmovd		$D2,`4*2-48-64`($ctx)
+	vmovd		$D3,`4*3-48-64`($ctx)
+	vmovd		$D4,`4*4-48-64`($ctx)
+___
+$code.=<<___	if ($win64);
+	vmovdqa		0x50(%r11),%xmm6
+	vmovdqa		0x60(%r11),%xmm7
+	vmovdqa		0x70(%r11),%xmm8
+	vmovdqa		0x80(%r11),%xmm9
+	vmovdqa		0x90(%r11),%xmm10
+	vmovdqa		0xa0(%r11),%xmm11
+	vmovdqa		0xb0(%r11),%xmm12
+	vmovdqa		0xc0(%r11),%xmm13
+	vmovdqa		0xd0(%r11),%xmm14
+	vmovdqa		0xe0(%r11),%xmm15
+	lea		0xf8(%r11),%rsp
+.Ldo_avx_epilogue:
+___
+$code.=<<___	if (!$win64);
+	lea		-8(%r10),%rsp
+.cfi_def_cfa_register	%rsp
+___
+$code.=<<___;
+	vzeroupper
+	ret
+.cfi_endproc
+___
+&end_function("poly1305_blocks_avx");
+
+&declare_function("poly1305_emit_avx", 32, 3);
+$code.=<<___;
+	cmpl	\$0,20($ctx)	# is_base2_26?
+	je	.Lemit
+
+	mov	0($ctx),%eax	# load hash value base 2^26
+	mov	4($ctx),%ecx
+	mov	8($ctx),%r8d
+	mov	12($ctx),%r11d
+	mov	16($ctx),%r10d
+
+	shl	\$26,%rcx	# base 2^26 -> base 2^64
+	mov	%r8,%r9
+	shl	\$52,%r8
+	add	%rcx,%rax
+	shr	\$12,%r9
+	add	%rax,%r8	# h0
+	adc	\$0,%r9
+
+	shl	\$14,%r11
+	mov	%r10,%rax
+	shr	\$24,%r10
+	add	%r11,%r9
+	shl	\$40,%rax
+	add	%rax,%r9	# h1
+	adc	\$0,%r10	# h2
+
+	mov	%r10,%rax	# could be partially reduced, so reduce
+	mov	%r10,%rcx
+	and	\$3,%r10
+	shr	\$2,%rax
+	and	\$-4,%rcx
+	add	%rcx,%rax
+	add	%rax,%r8
+	adc	\$0,%r9
+	adc	\$0,%r10
+
+	mov	%r8,%rax
+	add	\$5,%r8		# compare to modulus
+	mov	%r9,%rcx
+	adc	\$0,%r9
+	adc	\$0,%r10
+	shr	\$2,%r10	# did 130-bit value overflow?
+	cmovnz	%r8,%rax
+	cmovnz	%r9,%rcx
+
+	add	0($nonce),%rax	# accumulate nonce
+	adc	8($nonce),%rcx
+	mov	%rax,0($mac)	# write result
+	mov	%rcx,8($mac)
+
+	ret
+___
+&end_function("poly1305_emit_avx");
+
+if ($kernel) {
+	$code .= "#endif\n";
+}
+
+if ($avx>1) {
+
+if ($kernel) {
+	$code .= "#ifdef CONFIG_AS_AVX2\n";
+}
+
+my ($H0,$H1,$H2,$H3,$H4, $MASK, $T4,$T0,$T1,$T2,$T3, $D0,$D1,$D2,$D3,$D4) =
+    map("%ymm$_",(0..15));
+my $S4=$MASK;
+
+sub poly1305_blocks_avxN {
+	my ($avx512) = @_;
+	my $suffix = $avx512 ? "_avx512" : "";
+$code.=<<___;
+.cfi_startproc
+	mov	20($ctx),%r8d		# is_base2_26
+	cmp	\$128,$len
+	jae	.Lblocks_avx2$suffix
+	test	%r8d,%r8d
+	jz	.Lblocks
+
+.Lblocks_avx2$suffix:
+	and	\$-16,$len
+	jz	.Lno_data_avx2$suffix
+
+	vzeroupper
+
+	test	%r8d,%r8d
+	jz	.Lbase2_64_avx2$suffix
+
+	test	\$63,$len
+	jz	.Leven_avx2$suffix
+
+	push	%rbp
+.cfi_push	%rbp
+	mov 	%rsp,%rbp
+	push	%rbx
+.cfi_push	%rbx
+	push	%r12
+.cfi_push	%r12
+	push	%r13
+.cfi_push	%r13
+	push	%r14
+.cfi_push	%r14
+	push	%r15
+.cfi_push	%r15
+.Lblocks_avx2_body$suffix:
+
+	mov	$len,%r15		# reassign $len
+
+	mov	0($ctx),$d1		# load hash value
+	mov	8($ctx),$d2
+	mov	16($ctx),$h2#d
+
+	mov	24($ctx),$r0		# load r
+	mov	32($ctx),$s1
+
+	################################# base 2^26 -> base 2^64
+	mov	$d1#d,$h0#d
+	and	\$`-1*(1<<31)`,$d1
+	mov	$d2,$r1			# borrow $r1
+	mov	$d2#d,$h1#d
+	and	\$`-1*(1<<31)`,$d2
+
+	shr	\$6,$d1
+	shl	\$52,$r1
+	add	$d1,$h0
+	shr	\$12,$h1
+	shr	\$18,$d2
+	add	$r1,$h0
+	adc	$d2,$h1
+
+	mov	$h2,$d1
+	shl	\$40,$d1
+	shr	\$24,$h2
+	add	$d1,$h1
+	adc	\$0,$h2			# can be partially reduced...
+
+	mov	\$-4,$d2		# ... so reduce
+	mov	$h2,$d1
+	and	$h2,$d2
+	shr	\$2,$d1
+	and	\$3,$h2
+	add	$d2,$d1			# =*5
+	add	$d1,$h0
+	adc	\$0,$h1
+	adc	\$0,$h2
+
+	mov	$s1,$r1
+	mov	$s1,%rax
+	shr	\$2,$s1
+	add	$r1,$s1			# s1 = r1 + (r1 >> 2)
+
+.Lbase2_26_pre_avx2$suffix:
+	add	0($inp),$h0		# accumulate input
+	adc	8($inp),$h1
+	lea	16($inp),$inp
+	adc	$padbit,$h2
+	sub	\$16,%r15
+
+	call	__poly1305_block
+	mov	$r1,%rax
+
+	test	\$63,%r15
+	jnz	.Lbase2_26_pre_avx2$suffix
+
+	test	$padbit,$padbit		# if $padbit is zero,
+	jz	.Lstore_base2_64_avx2$suffix	# store hash in base 2^64 format
+
+	################################# base 2^64 -> base 2^26
+	mov	$h0,%rax
+	mov	$h0,%rdx
+	shr	\$52,$h0
+	mov	$h1,$r0
+	mov	$h1,$r1
+	shr	\$26,%rdx
+	and	\$0x3ffffff,%rax	# h[0]
+	shl	\$12,$r0
+	and	\$0x3ffffff,%rdx	# h[1]
+	shr	\$14,$h1
+	or	$r0,$h0
+	shl	\$24,$h2
+	and	\$0x3ffffff,$h0		# h[2]
+	shr	\$40,$r1
+	and	\$0x3ffffff,$h1		# h[3]
+	or	$r1,$h2			# h[4]
+
+	test	%r15,%r15
+	jz	.Lstore_base2_26_avx2$suffix
+
+	vmovd	%rax#d,%x#$H0
+	vmovd	%rdx#d,%x#$H1
+	vmovd	$h0#d,%x#$H2
+	vmovd	$h1#d,%x#$H3
+	vmovd	$h2#d,%x#$H4
+	jmp	.Lproceed_avx2$suffix
+
+.align	32
+.Lstore_base2_64_avx2$suffix:
+	mov	$h0,0($ctx)
+	mov	$h1,8($ctx)
+	mov	$h2,16($ctx)		# note that is_base2_26 is zeroed
+	jmp	.Ldone_avx2$suffix
+
+.align	16
+.Lstore_base2_26_avx2$suffix:
+	mov	%rax#d,0($ctx)		# store hash value base 2^26
+	mov	%rdx#d,4($ctx)
+	mov	$h0#d,8($ctx)
+	mov	$h1#d,12($ctx)
+	mov	$h2#d,16($ctx)
+.align	16
+.Ldone_avx2$suffix:
+	pop 		%r15
+.cfi_restore	%r15
+	pop 		%r14
+.cfi_restore	%r14
+	pop 		%r13
+.cfi_restore	%r13
+	pop 		%r12
+.cfi_restore	%r12
+	pop 		%rbx
+.cfi_restore	%rbx
+	pop 		%rbp
+.cfi_restore 	%rbp
+.Lno_data_avx2$suffix:
+.Lblocks_avx2_epilogue$suffix:
+	ret
+.cfi_endproc
+
+.align	32
+.Lbase2_64_avx2$suffix:
+.cfi_startproc
+	push	%rbp
+.cfi_push	%rbp
+	mov 	%rsp,%rbp
+	push	%rbx
+.cfi_push	%rbx
+	push	%r12
+.cfi_push	%r12
+	push	%r13
+.cfi_push	%r13
+	push	%r14
+.cfi_push	%r14
+	push	%r15
+.cfi_push	%r15
+.Lbase2_64_avx2_body$suffix:
+
+	mov	$len,%r15		# reassign $len
+
+	mov	24($ctx),$r0		# load r
+	mov	32($ctx),$s1
+
+	mov	0($ctx),$h0		# load hash value
+	mov	8($ctx),$h1
+	mov	16($ctx),$h2#d
+
+	mov	$s1,$r1
+	mov	$s1,%rax
+	shr	\$2,$s1
+	add	$r1,$s1			# s1 = r1 + (r1 >> 2)
+
+	test	\$63,$len
+	jz	.Linit_avx2$suffix
+
+.Lbase2_64_pre_avx2$suffix:
+	add	0($inp),$h0		# accumulate input
+	adc	8($inp),$h1
+	lea	16($inp),$inp
+	adc	$padbit,$h2
+	sub	\$16,%r15
+
+	call	__poly1305_block
+	mov	$r1,%rax
+
+	test	\$63,%r15
+	jnz	.Lbase2_64_pre_avx2$suffix
+
+.Linit_avx2$suffix:
+	################################# base 2^64 -> base 2^26
+	mov	$h0,%rax
+	mov	$h0,%rdx
+	shr	\$52,$h0
+	mov	$h1,$d1
+	mov	$h1,$d2
+	shr	\$26,%rdx
+	and	\$0x3ffffff,%rax	# h[0]
+	shl	\$12,$d1
+	and	\$0x3ffffff,%rdx	# h[1]
+	shr	\$14,$h1
+	or	$d1,$h0
+	shl	\$24,$h2
+	and	\$0x3ffffff,$h0		# h[2]
+	shr	\$40,$d2
+	and	\$0x3ffffff,$h1		# h[3]
+	or	$d2,$h2			# h[4]
+
+	vmovd	%rax#d,%x#$H0
+	vmovd	%rdx#d,%x#$H1
+	vmovd	$h0#d,%x#$H2
+	vmovd	$h1#d,%x#$H3
+	vmovd	$h2#d,%x#$H4
+	movl	\$1,20($ctx)		# set is_base2_26
+
+	call	__poly1305_init_avx
+
+.Lproceed_avx2$suffix:
+	mov	%r15,$len			# restore $len
+___
+$code.=<<___ if (!$kernel);
+	mov	OPENSSL_ia32cap_P+8(%rip),%r9d
+	mov	\$`(1<<31|1<<30|1<<16)`,%r11d
+___
+$code.=<<___;
+	pop 		%r15
+.cfi_restore	%r15
+	pop 		%r14
+.cfi_restore	%r14
+	pop 		%r13
+.cfi_restore	%r13
+	pop 		%r12
+.cfi_restore	%r12
+	pop 		%rbx
+.cfi_restore	%rbx
+	pop 		%rbp
+.cfi_restore 	%rbp
+.Lbase2_64_avx2_epilogue$suffix:
+	jmp	.Ldo_avx2$suffix
+.cfi_endproc
+
+.align	32
+.Leven_avx2$suffix:
+.cfi_startproc
+___
+$code.=<<___ if (!$kernel);
+	mov		OPENSSL_ia32cap_P+8(%rip),%r9d
+___
+$code.=<<___;
+	vmovd		4*0($ctx),%x#$H0	# load hash value base 2^26
+	vmovd		4*1($ctx),%x#$H1
+	vmovd		4*2($ctx),%x#$H2
+	vmovd		4*3($ctx),%x#$H3
+	vmovd		4*4($ctx),%x#$H4
+
+.Ldo_avx2$suffix:
+___
+$code.=<<___		if (!$kernel && $avx>2);
+	cmp		\$512,$len
+	jb		.Lskip_avx512
+	and		%r11d,%r9d
+	test		\$`1<<16`,%r9d		# check for AVX512F
+	jnz		.Lblocks_avx512
+.Lskip_avx512$suffix:
+___
+$code.=<<___ if ($avx > 2 && $avx512 && $kernel);
+	cmp		\$512,$len
+	jae		.Lblocks_avx512
+___
+$code.=<<___	if (!$win64);
+	lea		8(%rsp),%r10
+.cfi_def_cfa_register	%r10
+	sub		\$0x128,%rsp
+___
+$code.=<<___	if ($win64);
+	lea		8(%rsp),%r10
+	sub		\$0x1c8,%rsp
+	vmovdqa		%xmm6,-0xb0(%r10)
+	vmovdqa		%xmm7,-0xa0(%r10)
+	vmovdqa		%xmm8,-0x90(%r10)
+	vmovdqa		%xmm9,-0x80(%r10)
+	vmovdqa		%xmm10,-0x70(%r10)
+	vmovdqa		%xmm11,-0x60(%r10)
+	vmovdqa		%xmm12,-0x50(%r10)
+	vmovdqa		%xmm13,-0x40(%r10)
+	vmovdqa		%xmm14,-0x30(%r10)
+	vmovdqa		%xmm15,-0x20(%r10)
+.Ldo_avx2_body$suffix:
+___
+$code.=<<___;
+	lea		.Lconst(%rip),%rcx
+	lea		48+64($ctx),$ctx	# size optimization
+	vmovdqa		96(%rcx),$T0		# .Lpermd_avx2
+
+	# expand and copy pre-calculated table to stack
+	vmovdqu		`16*0-64`($ctx),%x#$T2
+	and		\$-512,%rsp
+	vmovdqu		`16*1-64`($ctx),%x#$T3
+	vmovdqu		`16*2-64`($ctx),%x#$T4
+	vmovdqu		`16*3-64`($ctx),%x#$D0
+	vmovdqu		`16*4-64`($ctx),%x#$D1
+	vmovdqu		`16*5-64`($ctx),%x#$D2
+	lea		0x90(%rsp),%rax		# size optimization
+	vmovdqu		`16*6-64`($ctx),%x#$D3
+	vpermd		$T2,$T0,$T2		# 00003412 -> 14243444
+	vmovdqu		`16*7-64`($ctx),%x#$D4
+	vpermd		$T3,$T0,$T3
+	vmovdqu		`16*8-64`($ctx),%x#$MASK
+	vpermd		$T4,$T0,$T4
+	vmovdqa		$T2,0x00(%rsp)
+	vpermd		$D0,$T0,$D0
+	vmovdqa		$T3,0x20-0x90(%rax)
+	vpermd		$D1,$T0,$D1
+	vmovdqa		$T4,0x40-0x90(%rax)
+	vpermd		$D2,$T0,$D2
+	vmovdqa		$D0,0x60-0x90(%rax)
+	vpermd		$D3,$T0,$D3
+	vmovdqa		$D1,0x80-0x90(%rax)
+	vpermd		$D4,$T0,$D4
+	vmovdqa		$D2,0xa0-0x90(%rax)
+	vpermd		$MASK,$T0,$MASK
+	vmovdqa		$D3,0xc0-0x90(%rax)
+	vmovdqa		$D4,0xe0-0x90(%rax)
+	vmovdqa		$MASK,0x100-0x90(%rax)
+	vmovdqa		64(%rcx),$MASK		# .Lmask26
+
+	################################################################
+	# load input
+	vmovdqu		16*0($inp),%x#$T0
+	vmovdqu		16*1($inp),%x#$T1
+	vinserti128	\$1,16*2($inp),$T0,$T0
+	vinserti128	\$1,16*3($inp),$T1,$T1
+	lea		16*4($inp),$inp
+
+	vpsrldq		\$6,$T0,$T2		# splat input
+	vpsrldq		\$6,$T1,$T3
+	vpunpckhqdq	$T1,$T0,$T4		# 4
+	vpunpcklqdq	$T3,$T2,$T2		# 2:3
+	vpunpcklqdq	$T1,$T0,$T0		# 0:1
+
+	vpsrlq		\$30,$T2,$T3
+	vpsrlq		\$4,$T2,$T2
+	vpsrlq		\$26,$T0,$T1
+	vpsrlq		\$40,$T4,$T4		# 4
+	vpand		$MASK,$T2,$T2		# 2
+	vpand		$MASK,$T0,$T0		# 0
+	vpand		$MASK,$T1,$T1		# 1
+	vpand		$MASK,$T3,$T3		# 3
+	vpor		32(%rcx),$T4,$T4	# padbit, yes, always
+
+	vpaddq		$H2,$T2,$H2		# accumulate input
+	sub		\$64,$len
+	jz		.Ltail_avx2$suffix
+	jmp		.Loop_avx2$suffix
+
+.align	32
+.Loop_avx2$suffix:
+	################################################################
+	# ((inp[0]*r^4+inp[4])*r^4+inp[ 8])*r^4
+	# ((inp[1]*r^4+inp[5])*r^4+inp[ 9])*r^3
+	# ((inp[2]*r^4+inp[6])*r^4+inp[10])*r^2
+	# ((inp[3]*r^4+inp[7])*r^4+inp[11])*r^1
+	#   \________/\__________/
+	################################################################
+	#vpaddq		$H2,$T2,$H2		# accumulate input
+	vpaddq		$H0,$T0,$H0
+	vmovdqa		`32*0`(%rsp),$T0	# r0^4
+	vpaddq		$H1,$T1,$H1
+	vmovdqa		`32*1`(%rsp),$T1	# r1^4
+	vpaddq		$H3,$T3,$H3
+	vmovdqa		`32*3`(%rsp),$T2	# r2^4
+	vpaddq		$H4,$T4,$H4
+	vmovdqa		`32*6-0x90`(%rax),$T3	# s3^4
+	vmovdqa		`32*8-0x90`(%rax),$S4	# s4^4
+
+	# d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
+	# d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
+	# d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
+	# d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
+	# d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
+	#
+	# however, as h2 is "chronologically" first one available pull
+	# corresponding operations up, so it's
+	#
+	# d4 = h2*r2   + h4*r0 + h3*r1             + h1*r3   + h0*r4
+	# d3 = h2*r1   + h3*r0           + h1*r2   + h0*r3   + h4*5*r4
+	# d2 = h2*r0           + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
+	# d1 = h2*5*r4 + h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3
+	# d0 = h2*5*r3 + h0*r0 + h4*5*r1 + h3*5*r2           + h1*5*r4
+
+	vpmuludq	$H2,$T0,$D2		# d2 = h2*r0
+	vpmuludq	$H2,$T1,$D3		# d3 = h2*r1
+	vpmuludq	$H2,$T2,$D4		# d4 = h2*r2
+	vpmuludq	$H2,$T3,$D0		# d0 = h2*s3
+	vpmuludq	$H2,$S4,$D1		# d1 = h2*s4
+
+	vpmuludq	$H0,$T1,$T4		# h0*r1
+	vpmuludq	$H1,$T1,$H2		# h1*r1, borrow $H2 as temp
+	vpaddq		$T4,$D1,$D1		# d1 += h0*r1
+	vpaddq		$H2,$D2,$D2		# d2 += h1*r1
+	vpmuludq	$H3,$T1,$T4		# h3*r1
+	vpmuludq	`32*2`(%rsp),$H4,$H2	# h4*s1
+	vpaddq		$T4,$D4,$D4		# d4 += h3*r1
+	vpaddq		$H2,$D0,$D0		# d0 += h4*s1
+	 vmovdqa	`32*4-0x90`(%rax),$T1	# s2
+
+	vpmuludq	$H0,$T0,$T4		# h0*r0
+	vpmuludq	$H1,$T0,$H2		# h1*r0
+	vpaddq		$T4,$D0,$D0		# d0 += h0*r0
+	vpaddq		$H2,$D1,$D1		# d1 += h1*r0
+	vpmuludq	$H3,$T0,$T4		# h3*r0
+	vpmuludq	$H4,$T0,$H2		# h4*r0
+	 vmovdqu	16*0($inp),%x#$T0	# load input
+	vpaddq		$T4,$D3,$D3		# d3 += h3*r0
+	vpaddq		$H2,$D4,$D4		# d4 += h4*r0
+	 vinserti128	\$1,16*2($inp),$T0,$T0
+
+	vpmuludq	$H3,$T1,$T4		# h3*s2
+	vpmuludq	$H4,$T1,$H2		# h4*s2
+	 vmovdqu	16*1($inp),%x#$T1
+	vpaddq		$T4,$D0,$D0		# d0 += h3*s2
+	vpaddq		$H2,$D1,$D1		# d1 += h4*s2
+	 vmovdqa	`32*5-0x90`(%rax),$H2	# r3
+	vpmuludq	$H1,$T2,$T4		# h1*r2
+	vpmuludq	$H0,$T2,$T2		# h0*r2
+	vpaddq		$T4,$D3,$D3		# d3 += h1*r2
+	vpaddq		$T2,$D2,$D2		# d2 += h0*r2
+	 vinserti128	\$1,16*3($inp),$T1,$T1
+	 lea		16*4($inp),$inp
+
+	vpmuludq	$H1,$H2,$T4		# h1*r3
+	vpmuludq	$H0,$H2,$H2		# h0*r3
+	 vpsrldq	\$6,$T0,$T2		# splat input
+	vpaddq		$T4,$D4,$D4		# d4 += h1*r3
+	vpaddq		$H2,$D3,$D3		# d3 += h0*r3
+	vpmuludq	$H3,$T3,$T4		# h3*s3
+	vpmuludq	$H4,$T3,$H2		# h4*s3
+	 vpsrldq	\$6,$T1,$T3
+	vpaddq		$T4,$D1,$D1		# d1 += h3*s3
+	vpaddq		$H2,$D2,$D2		# d2 += h4*s3
+	 vpunpckhqdq	$T1,$T0,$T4		# 4
+
+	vpmuludq	$H3,$S4,$H3		# h3*s4
+	vpmuludq	$H4,$S4,$H4		# h4*s4
+	 vpunpcklqdq	$T1,$T0,$T0		# 0:1
+	vpaddq		$H3,$D2,$H2		# h2 = d2 + h3*r4
+	vpaddq		$H4,$D3,$H3		# h3 = d3 + h4*r4
+	 vpunpcklqdq	$T3,$T2,$T3		# 2:3
+	vpmuludq	`32*7-0x90`(%rax),$H0,$H4	# h0*r4
+	vpmuludq	$H1,$S4,$H0		# h1*s4
+	vmovdqa		64(%rcx),$MASK		# .Lmask26
+	vpaddq		$H4,$D4,$H4		# h4 = d4 + h0*r4
+	vpaddq		$H0,$D0,$H0		# h0 = d0 + h1*s4
+
+	################################################################
+	# lazy reduction (interleaved with tail of input splat)
+
+	vpsrlq		\$26,$H3,$D3
+	vpand		$MASK,$H3,$H3
+	vpaddq		$D3,$H4,$H4		# h3 -> h4
+
+	vpsrlq		\$26,$H0,$D0
+	vpand		$MASK,$H0,$H0
+	vpaddq		$D0,$D1,$H1		# h0 -> h1
+
+	vpsrlq		\$26,$H4,$D4
+	vpand		$MASK,$H4,$H4
+
+	 vpsrlq		\$4,$T3,$T2
+
+	vpsrlq		\$26,$H1,$D1
+	vpand		$MASK,$H1,$H1
+	vpaddq		$D1,$H2,$H2		# h1 -> h2
+
+	vpaddq		$D4,$H0,$H0
+	vpsllq		\$2,$D4,$D4
+	vpaddq		$D4,$H0,$H0		# h4 -> h0
+
+	 vpand		$MASK,$T2,$T2		# 2
+	 vpsrlq		\$26,$T0,$T1
+
+	vpsrlq		\$26,$H2,$D2
+	vpand		$MASK,$H2,$H2
+	vpaddq		$D2,$H3,$H3		# h2 -> h3
+
+	 vpaddq		$T2,$H2,$H2		# modulo-scheduled
+	 vpsrlq		\$30,$T3,$T3
+
+	vpsrlq		\$26,$H0,$D0
+	vpand		$MASK,$H0,$H0
+	vpaddq		$D0,$H1,$H1		# h0 -> h1
+
+	 vpsrlq		\$40,$T4,$T4		# 4
+
+	vpsrlq		\$26,$H3,$D3
+	vpand		$MASK,$H3,$H3
+	vpaddq		$D3,$H4,$H4		# h3 -> h4
+
+	 vpand		$MASK,$T0,$T0		# 0
+	 vpand		$MASK,$T1,$T1		# 1
+	 vpand		$MASK,$T3,$T3		# 3
+	 vpor		32(%rcx),$T4,$T4	# padbit, yes, always
+
+	sub		\$64,$len
+	jnz		.Loop_avx2$suffix
+
+	.byte		0x66,0x90
+.Ltail_avx2$suffix:
+	################################################################
+	# while above multiplications were by r^4 in all lanes, in last
+	# iteration we multiply least significant lane by r^4 and most
+	# significant one by r, so copy of above except that references
+	# to the precomputed table are displaced by 4...
+
+	#vpaddq		$H2,$T2,$H2		# accumulate input
+	vpaddq		$H0,$T0,$H0
+	vmovdqu		`32*0+4`(%rsp),$T0	# r0^4
+	vpaddq		$H1,$T1,$H1
+	vmovdqu		`32*1+4`(%rsp),$T1	# r1^4
+	vpaddq		$H3,$T3,$H3
+	vmovdqu		`32*3+4`(%rsp),$T2	# r2^4
+	vpaddq		$H4,$T4,$H4
+	vmovdqu		`32*6+4-0x90`(%rax),$T3	# s3^4
+	vmovdqu		`32*8+4-0x90`(%rax),$S4	# s4^4
+
+	vpmuludq	$H2,$T0,$D2		# d2 = h2*r0
+	vpmuludq	$H2,$T1,$D3		# d3 = h2*r1
+	vpmuludq	$H2,$T2,$D4		# d4 = h2*r2
+	vpmuludq	$H2,$T3,$D0		# d0 = h2*s3
+	vpmuludq	$H2,$S4,$D1		# d1 = h2*s4
+
+	vpmuludq	$H0,$T1,$T4		# h0*r1
+	vpmuludq	$H1,$T1,$H2		# h1*r1
+	vpaddq		$T4,$D1,$D1		# d1 += h0*r1
+	vpaddq		$H2,$D2,$D2		# d2 += h1*r1
+	vpmuludq	$H3,$T1,$T4		# h3*r1
+	vpmuludq	`32*2+4`(%rsp),$H4,$H2	# h4*s1
+	vpaddq		$T4,$D4,$D4		# d4 += h3*r1
+	vpaddq		$H2,$D0,$D0		# d0 += h4*s1
+
+	vpmuludq	$H0,$T0,$T4		# h0*r0
+	vpmuludq	$H1,$T0,$H2		# h1*r0
+	vpaddq		$T4,$D0,$D0		# d0 += h0*r0
+	 vmovdqu	`32*4+4-0x90`(%rax),$T1	# s2
+	vpaddq		$H2,$D1,$D1		# d1 += h1*r0
+	vpmuludq	$H3,$T0,$T4		# h3*r0
+	vpmuludq	$H4,$T0,$H2		# h4*r0
+	vpaddq		$T4,$D3,$D3		# d3 += h3*r0
+	vpaddq		$H2,$D4,$D4		# d4 += h4*r0
+
+	vpmuludq	$H3,$T1,$T4		# h3*s2
+	vpmuludq	$H4,$T1,$H2		# h4*s2
+	vpaddq		$T4,$D0,$D0		# d0 += h3*s2
+	vpaddq		$H2,$D1,$D1		# d1 += h4*s2
+	 vmovdqu	`32*5+4-0x90`(%rax),$H2	# r3
+	vpmuludq	$H1,$T2,$T4		# h1*r2
+	vpmuludq	$H0,$T2,$T2		# h0*r2
+	vpaddq		$T4,$D3,$D3		# d3 += h1*r2
+	vpaddq		$T2,$D2,$D2		# d2 += h0*r2
+
+	vpmuludq	$H1,$H2,$T4		# h1*r3
+	vpmuludq	$H0,$H2,$H2		# h0*r3
+	vpaddq		$T4,$D4,$D4		# d4 += h1*r3
+	vpaddq		$H2,$D3,$D3		# d3 += h0*r3
+	vpmuludq	$H3,$T3,$T4		# h3*s3
+	vpmuludq	$H4,$T3,$H2		# h4*s3
+	vpaddq		$T4,$D1,$D1		# d1 += h3*s3
+	vpaddq		$H2,$D2,$D2		# d2 += h4*s3
+
+	vpmuludq	$H3,$S4,$H3		# h3*s4
+	vpmuludq	$H4,$S4,$H4		# h4*s4
+	vpaddq		$H3,$D2,$H2		# h2 = d2 + h3*r4
+	vpaddq		$H4,$D3,$H3		# h3 = d3 + h4*r4
+	vpmuludq	`32*7+4-0x90`(%rax),$H0,$H4		# h0*r4
+	vpmuludq	$H1,$S4,$H0		# h1*s4
+	vmovdqa		64(%rcx),$MASK		# .Lmask26
+	vpaddq		$H4,$D4,$H4		# h4 = d4 + h0*r4
+	vpaddq		$H0,$D0,$H0		# h0 = d0 + h1*s4
+
+	################################################################
+	# horizontal addition
+
+	vpsrldq		\$8,$D1,$T1
+	vpsrldq		\$8,$H2,$T2
+	vpsrldq		\$8,$H3,$T3
+	vpsrldq		\$8,$H4,$T4
+	vpsrldq		\$8,$H0,$T0
+	vpaddq		$T1,$D1,$D1
+	vpaddq		$T2,$H2,$H2
+	vpaddq		$T3,$H3,$H3
+	vpaddq		$T4,$H4,$H4
+	vpaddq		$T0,$H0,$H0
+
+	vpermq		\$0x2,$H3,$T3
+	vpermq		\$0x2,$H4,$T4
+	vpermq		\$0x2,$H0,$T0
+	vpermq		\$0x2,$D1,$T1
+	vpermq		\$0x2,$H2,$T2
+	vpaddq		$T3,$H3,$H3
+	vpaddq		$T4,$H4,$H4
+	vpaddq		$T0,$H0,$H0
+	vpaddq		$T1,$D1,$D1
+	vpaddq		$T2,$H2,$H2
+
+	################################################################
+	# lazy reduction
+
+	vpsrlq		\$26,$H3,$D3
+	vpand		$MASK,$H3,$H3
+	vpaddq		$D3,$H4,$H4		# h3 -> h4
+
+	vpsrlq		\$26,$H0,$D0
+	vpand		$MASK,$H0,$H0
+	vpaddq		$D0,$D1,$H1		# h0 -> h1
+
+	vpsrlq		\$26,$H4,$D4
+	vpand		$MASK,$H4,$H4
+
+	vpsrlq		\$26,$H1,$D1
+	vpand		$MASK,$H1,$H1
+	vpaddq		$D1,$H2,$H2		# h1 -> h2
+
+	vpaddq		$D4,$H0,$H0
+	vpsllq		\$2,$D4,$D4
+	vpaddq		$D4,$H0,$H0		# h4 -> h0
+
+	vpsrlq		\$26,$H2,$D2
+	vpand		$MASK,$H2,$H2
+	vpaddq		$D2,$H3,$H3		# h2 -> h3
+
+	vpsrlq		\$26,$H0,$D0
+	vpand		$MASK,$H0,$H0
+	vpaddq		$D0,$H1,$H1		# h0 -> h1
+
+	vpsrlq		\$26,$H3,$D3
+	vpand		$MASK,$H3,$H3
+	vpaddq		$D3,$H4,$H4		# h3 -> h4
+
+	vmovd		%x#$H0,`4*0-48-64`($ctx)# save partially reduced
+	vmovd		%x#$H1,`4*1-48-64`($ctx)
+	vmovd		%x#$H2,`4*2-48-64`($ctx)
+	vmovd		%x#$H3,`4*3-48-64`($ctx)
+	vmovd		%x#$H4,`4*4-48-64`($ctx)
+___
+$code.=<<___	if ($win64);
+	vmovdqa		-0xb0(%r10),%xmm6
+	vmovdqa		-0xa0(%r10),%xmm7
+	vmovdqa		-0x90(%r10),%xmm8
+	vmovdqa		-0x80(%r10),%xmm9
+	vmovdqa		-0x70(%r10),%xmm10
+	vmovdqa		-0x60(%r10),%xmm11
+	vmovdqa		-0x50(%r10),%xmm12
+	vmovdqa		-0x40(%r10),%xmm13
+	vmovdqa		-0x30(%r10),%xmm14
+	vmovdqa		-0x20(%r10),%xmm15
+	lea		-8(%r10),%rsp
+.Ldo_avx2_epilogue$suffix:
+___
+$code.=<<___	if (!$win64);
+	lea		-8(%r10),%rsp
+.cfi_def_cfa_register	%rsp
+___
+$code.=<<___;
+	vzeroupper
+	ret
+.cfi_endproc
+___
+if($avx > 2 && $avx512) {
+my ($R0,$R1,$R2,$R3,$R4, $S1,$S2,$S3,$S4) = map("%zmm$_",(16..24));
+my ($M0,$M1,$M2,$M3,$M4) = map("%zmm$_",(25..29));
+my $PADBIT="%zmm30";
+
+map(s/%y/%z/,($T4,$T0,$T1,$T2,$T3));		# switch to %zmm domain
+map(s/%y/%z/,($D0,$D1,$D2,$D3,$D4));
+map(s/%y/%z/,($H0,$H1,$H2,$H3,$H4));
+map(s/%y/%z/,($MASK));
+
+$code.=<<___;
+.cfi_startproc
+.Lblocks_avx512:
+	mov		\$15,%eax
+	kmovw		%eax,%k2
+___
+$code.=<<___	if (!$win64);
+	lea		8(%rsp),%r10
+.cfi_def_cfa_register	%r10
+	sub		\$0x128,%rsp
+___
+$code.=<<___	if ($win64);
+	lea		8(%rsp),%r10
+	sub		\$0x1c8,%rsp
+	vmovdqa		%xmm6,-0xb0(%r10)
+	vmovdqa		%xmm7,-0xa0(%r10)
+	vmovdqa		%xmm8,-0x90(%r10)
+	vmovdqa		%xmm9,-0x80(%r10)
+	vmovdqa		%xmm10,-0x70(%r10)
+	vmovdqa		%xmm11,-0x60(%r10)
+	vmovdqa		%xmm12,-0x50(%r10)
+	vmovdqa		%xmm13,-0x40(%r10)
+	vmovdqa		%xmm14,-0x30(%r10)
+	vmovdqa		%xmm15,-0x20(%r10)
+.Ldo_avx512_body:
+___
+$code.=<<___;
+	lea		.Lconst(%rip),%rcx
+	lea		48+64($ctx),$ctx	# size optimization
+	vmovdqa		96(%rcx),%y#$T2		# .Lpermd_avx2
+
+	# expand pre-calculated table
+	vmovdqu		`16*0-64`($ctx),%x#$D0	# will become expanded ${R0}
+	and		\$-512,%rsp
+	vmovdqu		`16*1-64`($ctx),%x#$D1	# will become ... ${R1}
+	mov		\$0x20,%rax
+	vmovdqu		`16*2-64`($ctx),%x#$T0	# ... ${S1}
+	vmovdqu		`16*3-64`($ctx),%x#$D2	# ... ${R2}
+	vmovdqu		`16*4-64`($ctx),%x#$T1	# ... ${S2}
+	vmovdqu		`16*5-64`($ctx),%x#$D3	# ... ${R3}
+	vmovdqu		`16*6-64`($ctx),%x#$T3	# ... ${S3}
+	vmovdqu		`16*7-64`($ctx),%x#$D4	# ... ${R4}
+	vmovdqu		`16*8-64`($ctx),%x#$T4	# ... ${S4}
+	vpermd		$D0,$T2,$R0		# 00003412 -> 14243444
+	vpbroadcastq	64(%rcx),$MASK		# .Lmask26
+	vpermd		$D1,$T2,$R1
+	vpermd		$T0,$T2,$S1
+	vpermd		$D2,$T2,$R2
+	vmovdqa64	$R0,0x00(%rsp){%k2}	# save in case $len%128 != 0
+	 vpsrlq		\$32,$R0,$T0		# 14243444 -> 01020304
+	vpermd		$T1,$T2,$S2
+	vmovdqu64	$R1,0x00(%rsp,%rax){%k2}
+	 vpsrlq		\$32,$R1,$T1
+	vpermd		$D3,$T2,$R3
+	vmovdqa64	$S1,0x40(%rsp){%k2}
+	vpermd		$T3,$T2,$S3
+	vpermd		$D4,$T2,$R4
+	vmovdqu64	$R2,0x40(%rsp,%rax){%k2}
+	vpermd		$T4,$T2,$S4
+	vmovdqa64	$S2,0x80(%rsp){%k2}
+	vmovdqu64	$R3,0x80(%rsp,%rax){%k2}
+	vmovdqa64	$S3,0xc0(%rsp){%k2}
+	vmovdqu64	$R4,0xc0(%rsp,%rax){%k2}
+	vmovdqa64	$S4,0x100(%rsp){%k2}
+
+	################################################################
+	# calculate 5th through 8th powers of the key
+	#
+	# d0 = r0'*r0 + r1'*5*r4 + r2'*5*r3 + r3'*5*r2 + r4'*5*r1
+	# d1 = r0'*r1 + r1'*r0   + r2'*5*r4 + r3'*5*r3 + r4'*5*r2
+	# d2 = r0'*r2 + r1'*r1   + r2'*r0   + r3'*5*r4 + r4'*5*r3
+	# d3 = r0'*r3 + r1'*r2   + r2'*r1   + r3'*r0   + r4'*5*r4
+	# d4 = r0'*r4 + r1'*r3   + r2'*r2   + r3'*r1   + r4'*r0
+
+	vpmuludq	$T0,$R0,$D0		# d0 = r0'*r0
+	vpmuludq	$T0,$R1,$D1		# d1 = r0'*r1
+	vpmuludq	$T0,$R2,$D2		# d2 = r0'*r2
+	vpmuludq	$T0,$R3,$D3		# d3 = r0'*r3
+	vpmuludq	$T0,$R4,$D4		# d4 = r0'*r4
+	 vpsrlq		\$32,$R2,$T2
+
+	vpmuludq	$T1,$S4,$M0
+	vpmuludq	$T1,$R0,$M1
+	vpmuludq	$T1,$R1,$M2
+	vpmuludq	$T1,$R2,$M3
+	vpmuludq	$T1,$R3,$M4
+	 vpsrlq		\$32,$R3,$T3
+	vpaddq		$M0,$D0,$D0		# d0 += r1'*5*r4
+	vpaddq		$M1,$D1,$D1		# d1 += r1'*r0
+	vpaddq		$M2,$D2,$D2		# d2 += r1'*r1
+	vpaddq		$M3,$D3,$D3		# d3 += r1'*r2
+	vpaddq		$M4,$D4,$D4		# d4 += r1'*r3
+
+	vpmuludq	$T2,$S3,$M0
+	vpmuludq	$T2,$S4,$M1
+	vpmuludq	$T2,$R1,$M3
+	vpmuludq	$T2,$R2,$M4
+	vpmuludq	$T2,$R0,$M2
+	 vpsrlq		\$32,$R4,$T4
+	vpaddq		$M0,$D0,$D0		# d0 += r2'*5*r3
+	vpaddq		$M1,$D1,$D1		# d1 += r2'*5*r4
+	vpaddq		$M3,$D3,$D3		# d3 += r2'*r1
+	vpaddq		$M4,$D4,$D4		# d4 += r2'*r2
+	vpaddq		$M2,$D2,$D2		# d2 += r2'*r0
+
+	vpmuludq	$T3,$S2,$M0
+	vpmuludq	$T3,$R0,$M3
+	vpmuludq	$T3,$R1,$M4
+	vpmuludq	$T3,$S3,$M1
+	vpmuludq	$T3,$S4,$M2
+	vpaddq		$M0,$D0,$D0		# d0 += r3'*5*r2
+	vpaddq		$M3,$D3,$D3		# d3 += r3'*r0
+	vpaddq		$M4,$D4,$D4		# d4 += r3'*r1
+	vpaddq		$M1,$D1,$D1		# d1 += r3'*5*r3
+	vpaddq		$M2,$D2,$D2		# d2 += r3'*5*r4
+
+	vpmuludq	$T4,$S4,$M3
+	vpmuludq	$T4,$R0,$M4
+	vpmuludq	$T4,$S1,$M0
+	vpmuludq	$T4,$S2,$M1
+	vpmuludq	$T4,$S3,$M2
+	vpaddq		$M3,$D3,$D3		# d3 += r2'*5*r4
+	vpaddq		$M4,$D4,$D4		# d4 += r2'*r0
+	vpaddq		$M0,$D0,$D0		# d0 += r2'*5*r1
+	vpaddq		$M1,$D1,$D1		# d1 += r2'*5*r2
+	vpaddq		$M2,$D2,$D2		# d2 += r2'*5*r3
+
+	################################################################
+	# load input
+	vmovdqu64	16*0($inp),%z#$T3
+	vmovdqu64	16*4($inp),%z#$T4
+	lea		16*8($inp),$inp
+
+	################################################################
+	# lazy reduction
+
+	vpsrlq		\$26,$D3,$M3
+	vpandq		$MASK,$D3,$D3
+	vpaddq		$M3,$D4,$D4		# d3 -> d4
+
+	vpsrlq		\$26,$D0,$M0
+	vpandq		$MASK,$D0,$D0
+	vpaddq		$M0,$D1,$D1		# d0 -> d1
+
+	vpsrlq		\$26,$D4,$M4
+	vpandq		$MASK,$D4,$D4
+
+	vpsrlq		\$26,$D1,$M1
+	vpandq		$MASK,$D1,$D1
+	vpaddq		$M1,$D2,$D2		# d1 -> d2
+
+	vpaddq		$M4,$D0,$D0
+	vpsllq		\$2,$M4,$M4
+	vpaddq		$M4,$D0,$D0		# d4 -> d0
+
+	vpsrlq		\$26,$D2,$M2
+	vpandq		$MASK,$D2,$D2
+	vpaddq		$M2,$D3,$D3		# d2 -> d3
+
+	vpsrlq		\$26,$D0,$M0
+	vpandq		$MASK,$D0,$D0
+	vpaddq		$M0,$D1,$D1		# d0 -> d1
+
+	vpsrlq		\$26,$D3,$M3
+	vpandq		$MASK,$D3,$D3
+	vpaddq		$M3,$D4,$D4		# d3 -> d4
+
+	################################################################
+	# at this point we have 14243444 in $R0-$S4 and 05060708 in
+	# $D0-$D4, ...
+
+	vpunpcklqdq	$T4,$T3,$T0	# transpose input
+	vpunpckhqdq	$T4,$T3,$T4
+
+	# ... since input 64-bit lanes are ordered as 73625140, we could
+	# "vperm" it to 76543210 (here and in each loop iteration), *or*
+	# we could just flow along, hence the goal for $R0-$S4 is
+	# 1858286838784888 ...
+
+	vmovdqa32	128(%rcx),$M0		# .Lpermd_avx512:
+	mov		\$0x7777,%eax
+	kmovw		%eax,%k1
+
+	vpermd		$R0,$M0,$R0		# 14243444 -> 1---2---3---4---
+	vpermd		$R1,$M0,$R1
+	vpermd		$R2,$M0,$R2
+	vpermd		$R3,$M0,$R3
+	vpermd		$R4,$M0,$R4
+
+	vpermd		$D0,$M0,${R0}{%k1}	# 05060708 -> 1858286838784888
+	vpermd		$D1,$M0,${R1}{%k1}
+	vpermd		$D2,$M0,${R2}{%k1}
+	vpermd		$D3,$M0,${R3}{%k1}
+	vpermd		$D4,$M0,${R4}{%k1}
+
+	vpslld		\$2,$R1,$S1		# *5
+	vpslld		\$2,$R2,$S2
+	vpslld		\$2,$R3,$S3
+	vpslld		\$2,$R4,$S4
+	vpaddd		$R1,$S1,$S1
+	vpaddd		$R2,$S2,$S2
+	vpaddd		$R3,$S3,$S3
+	vpaddd		$R4,$S4,$S4
+
+	vpbroadcastq	32(%rcx),$PADBIT	# .L129
+
+	vpsrlq		\$52,$T0,$T2		# splat input
+	vpsllq		\$12,$T4,$T3
+	vporq		$T3,$T2,$T2
+	vpsrlq		\$26,$T0,$T1
+	vpsrlq		\$14,$T4,$T3
+	vpsrlq		\$40,$T4,$T4		# 4
+	vpandq		$MASK,$T2,$T2		# 2
+	vpandq		$MASK,$T0,$T0		# 0
+	#vpandq		$MASK,$T1,$T1		# 1
+	#vpandq		$MASK,$T3,$T3		# 3
+	#vporq		$PADBIT,$T4,$T4		# padbit, yes, always
+
+	vpaddq		$H2,$T2,$H2		# accumulate input
+	sub		\$192,$len
+	jbe		.Ltail_avx512
+	jmp		.Loop_avx512
+
+.align	32
+.Loop_avx512:
+	################################################################
+	# ((inp[0]*r^8+inp[ 8])*r^8+inp[16])*r^8
+	# ((inp[1]*r^8+inp[ 9])*r^8+inp[17])*r^7
+	# ((inp[2]*r^8+inp[10])*r^8+inp[18])*r^6
+	# ((inp[3]*r^8+inp[11])*r^8+inp[19])*r^5
+	# ((inp[4]*r^8+inp[12])*r^8+inp[20])*r^4
+	# ((inp[5]*r^8+inp[13])*r^8+inp[21])*r^3
+	# ((inp[6]*r^8+inp[14])*r^8+inp[22])*r^2
+	# ((inp[7]*r^8+inp[15])*r^8+inp[23])*r^1
+	#   \________/\___________/
+	################################################################
+	#vpaddq		$H2,$T2,$H2		# accumulate input
+
+	# d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
+	# d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
+	# d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
+	# d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
+	# d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
+	#
+	# however, as h2 is "chronologically" first one available pull
+	# corresponding operations up, so it's
+	#
+	# d3 = h2*r1   + h0*r3 + h1*r2   + h3*r0 + h4*5*r4
+	# d4 = h2*r2   + h0*r4 + h1*r3   + h3*r1 + h4*r0
+	# d0 = h2*5*r3 + h0*r0 + h1*5*r4         + h3*5*r2 + h4*5*r1
+	# d1 = h2*5*r4 + h0*r1           + h1*r0 + h3*5*r3 + h4*5*r2
+	# d2 = h2*r0           + h0*r2   + h1*r1 + h3*5*r4 + h4*5*r3
+
+	vpmuludq	$H2,$R1,$D3		# d3 = h2*r1
+	 vpaddq		$H0,$T0,$H0
+	vpmuludq	$H2,$R2,$D4		# d4 = h2*r2
+	 vpandq		$MASK,$T1,$T1		# 1
+	vpmuludq	$H2,$S3,$D0		# d0 = h2*s3
+	 vpandq		$MASK,$T3,$T3		# 3
+	vpmuludq	$H2,$S4,$D1		# d1 = h2*s4
+	 vporq		$PADBIT,$T4,$T4		# padbit, yes, always
+	vpmuludq	$H2,$R0,$D2		# d2 = h2*r0
+	 vpaddq		$H1,$T1,$H1		# accumulate input
+	 vpaddq		$H3,$T3,$H3
+	 vpaddq		$H4,$T4,$H4
+
+	  vmovdqu64	16*0($inp),$T3		# load input
+	  vmovdqu64	16*4($inp),$T4
+	  lea		16*8($inp),$inp
+	vpmuludq	$H0,$R3,$M3
+	vpmuludq	$H0,$R4,$M4
+	vpmuludq	$H0,$R0,$M0
+	vpmuludq	$H0,$R1,$M1
+	vpaddq		$M3,$D3,$D3		# d3 += h0*r3
+	vpaddq		$M4,$D4,$D4		# d4 += h0*r4
+	vpaddq		$M0,$D0,$D0		# d0 += h0*r0
+	vpaddq		$M1,$D1,$D1		# d1 += h0*r1
+
+	vpmuludq	$H1,$R2,$M3
+	vpmuludq	$H1,$R3,$M4
+	vpmuludq	$H1,$S4,$M0
+	vpmuludq	$H0,$R2,$M2
+	vpaddq		$M3,$D3,$D3		# d3 += h1*r2
+	vpaddq		$M4,$D4,$D4		# d4 += h1*r3
+	vpaddq		$M0,$D0,$D0		# d0 += h1*s4
+	vpaddq		$M2,$D2,$D2		# d2 += h0*r2
+
+	  vpunpcklqdq	$T4,$T3,$T0		# transpose input
+	  vpunpckhqdq	$T4,$T3,$T4
+
+	vpmuludq	$H3,$R0,$M3
+	vpmuludq	$H3,$R1,$M4
+	vpmuludq	$H1,$R0,$M1
+	vpmuludq	$H1,$R1,$M2
+	vpaddq		$M3,$D3,$D3		# d3 += h3*r0
+	vpaddq		$M4,$D4,$D4		# d4 += h3*r1
+	vpaddq		$M1,$D1,$D1		# d1 += h1*r0
+	vpaddq		$M2,$D2,$D2		# d2 += h1*r1
+
+	vpmuludq	$H4,$S4,$M3
+	vpmuludq	$H4,$R0,$M4
+	vpmuludq	$H3,$S2,$M0
+	vpmuludq	$H3,$S3,$M1
+	vpaddq		$M3,$D3,$D3		# d3 += h4*s4
+	vpmuludq	$H3,$S4,$M2
+	vpaddq		$M4,$D4,$D4		# d4 += h4*r0
+	vpaddq		$M0,$D0,$D0		# d0 += h3*s2
+	vpaddq		$M1,$D1,$D1		# d1 += h3*s3
+	vpaddq		$M2,$D2,$D2		# d2 += h3*s4
+
+	vpmuludq	$H4,$S1,$M0
+	vpmuludq	$H4,$S2,$M1
+	vpmuludq	$H4,$S3,$M2
+	vpaddq		$M0,$D0,$H0		# h0 = d0 + h4*s1
+	vpaddq		$M1,$D1,$H1		# h1 = d2 + h4*s2
+	vpaddq		$M2,$D2,$H2		# h2 = d3 + h4*s3
+
+	################################################################
+	# lazy reduction (interleaved with input splat)
+
+	 vpsrlq		\$52,$T0,$T2		# splat input
+	 vpsllq		\$12,$T4,$T3
+
+	vpsrlq		\$26,$D3,$H3
+	vpandq		$MASK,$D3,$D3
+	vpaddq		$H3,$D4,$H4		# h3 -> h4
+
+	 vporq		$T3,$T2,$T2
+
+	vpsrlq		\$26,$H0,$D0
+	vpandq		$MASK,$H0,$H0
+	vpaddq		$D0,$H1,$H1		# h0 -> h1
+
+	 vpandq		$MASK,$T2,$T2		# 2
+
+	vpsrlq		\$26,$H4,$D4
+	vpandq		$MASK,$H4,$H4
+
+	vpsrlq		\$26,$H1,$D1
+	vpandq		$MASK,$H1,$H1
+	vpaddq		$D1,$H2,$H2		# h1 -> h2
+
+	vpaddq		$D4,$H0,$H0
+	vpsllq		\$2,$D4,$D4
+	vpaddq		$D4,$H0,$H0		# h4 -> h0
+
+	 vpaddq		$T2,$H2,$H2		# modulo-scheduled
+	 vpsrlq		\$26,$T0,$T1
+
+	vpsrlq		\$26,$H2,$D2
+	vpandq		$MASK,$H2,$H2
+	vpaddq		$D2,$D3,$H3		# h2 -> h3
+
+	 vpsrlq		\$14,$T4,$T3
+
+	vpsrlq		\$26,$H0,$D0
+	vpandq		$MASK,$H0,$H0
+	vpaddq		$D0,$H1,$H1		# h0 -> h1
+
+	 vpsrlq		\$40,$T4,$T4		# 4
+
+	vpsrlq		\$26,$H3,$D3
+	vpandq		$MASK,$H3,$H3
+	vpaddq		$D3,$H4,$H4		# h3 -> h4
+
+	 vpandq		$MASK,$T0,$T0		# 0
+	 #vpandq	$MASK,$T1,$T1		# 1
+	 #vpandq	$MASK,$T3,$T3		# 3
+	 #vporq		$PADBIT,$T4,$T4		# padbit, yes, always
+
+	sub		\$128,$len
+	ja		.Loop_avx512
+
+.Ltail_avx512:
+	################################################################
+	# while above multiplications were by r^8 in all lanes, in last
+	# iteration we multiply least significant lane by r^8 and most
+	# significant one by r, that's why table gets shifted...
+
+	vpsrlq		\$32,$R0,$R0		# 0105020603070408
+	vpsrlq		\$32,$R1,$R1
+	vpsrlq		\$32,$R2,$R2
+	vpsrlq		\$32,$S3,$S3
+	vpsrlq		\$32,$S4,$S4
+	vpsrlq		\$32,$R3,$R3
+	vpsrlq		\$32,$R4,$R4
+	vpsrlq		\$32,$S1,$S1
+	vpsrlq		\$32,$S2,$S2
+
+	################################################################
+	# load either next or last 64 byte of input
+	lea		($inp,$len),$inp
+
+	#vpaddq		$H2,$T2,$H2		# accumulate input
+	vpaddq		$H0,$T0,$H0
+
+	vpmuludq	$H2,$R1,$D3		# d3 = h2*r1
+	vpmuludq	$H2,$R2,$D4		# d4 = h2*r2
+	vpmuludq	$H2,$S3,$D0		# d0 = h2*s3
+	 vpandq		$MASK,$T1,$T1		# 1
+	vpmuludq	$H2,$S4,$D1		# d1 = h2*s4
+	 vpandq		$MASK,$T3,$T3		# 3
+	vpmuludq	$H2,$R0,$D2		# d2 = h2*r0
+	 vporq		$PADBIT,$T4,$T4		# padbit, yes, always
+	 vpaddq		$H1,$T1,$H1		# accumulate input
+	 vpaddq		$H3,$T3,$H3
+	 vpaddq		$H4,$T4,$H4
+
+	  vmovdqu	16*0($inp),%x#$T0
+	vpmuludq	$H0,$R3,$M3
+	vpmuludq	$H0,$R4,$M4
+	vpmuludq	$H0,$R0,$M0
+	vpmuludq	$H0,$R1,$M1
+	vpaddq		$M3,$D3,$D3		# d3 += h0*r3
+	vpaddq		$M4,$D4,$D4		# d4 += h0*r4
+	vpaddq		$M0,$D0,$D0		# d0 += h0*r0
+	vpaddq		$M1,$D1,$D1		# d1 += h0*r1
+
+	  vmovdqu	16*1($inp),%x#$T1
+	vpmuludq	$H1,$R2,$M3
+	vpmuludq	$H1,$R3,$M4
+	vpmuludq	$H1,$S4,$M0
+	vpmuludq	$H0,$R2,$M2
+	vpaddq		$M3,$D3,$D3		# d3 += h1*r2
+	vpaddq		$M4,$D4,$D4		# d4 += h1*r3
+	vpaddq		$M0,$D0,$D0		# d0 += h1*s4
+	vpaddq		$M2,$D2,$D2		# d2 += h0*r2
+
+	  vinserti128	\$1,16*2($inp),%y#$T0,%y#$T0
+	vpmuludq	$H3,$R0,$M3
+	vpmuludq	$H3,$R1,$M4
+	vpmuludq	$H1,$R0,$M1
+	vpmuludq	$H1,$R1,$M2
+	vpaddq		$M3,$D3,$D3		# d3 += h3*r0
+	vpaddq		$M4,$D4,$D4		# d4 += h3*r1
+	vpaddq		$M1,$D1,$D1		# d1 += h1*r0
+	vpaddq		$M2,$D2,$D2		# d2 += h1*r1
+
+	  vinserti128	\$1,16*3($inp),%y#$T1,%y#$T1
+	vpmuludq	$H4,$S4,$M3
+	vpmuludq	$H4,$R0,$M4
+	vpmuludq	$H3,$S2,$M0
+	vpmuludq	$H3,$S3,$M1
+	vpmuludq	$H3,$S4,$M2
+	vpaddq		$M3,$D3,$H3		# h3 = d3 + h4*s4
+	vpaddq		$M4,$D4,$D4		# d4 += h4*r0
+	vpaddq		$M0,$D0,$D0		# d0 += h3*s2
+	vpaddq		$M1,$D1,$D1		# d1 += h3*s3
+	vpaddq		$M2,$D2,$D2		# d2 += h3*s4
+
+	vpmuludq	$H4,$S1,$M0
+	vpmuludq	$H4,$S2,$M1
+	vpmuludq	$H4,$S3,$M2
+	vpaddq		$M0,$D0,$H0		# h0 = d0 + h4*s1
+	vpaddq		$M1,$D1,$H1		# h1 = d2 + h4*s2
+	vpaddq		$M2,$D2,$H2		# h2 = d3 + h4*s3
+
+	################################################################
+	# horizontal addition
+
+	mov		\$1,%eax
+	vpermq		\$0xb1,$H3,$D3
+	vpermq		\$0xb1,$D4,$H4
+	vpermq		\$0xb1,$H0,$D0
+	vpermq		\$0xb1,$H1,$D1
+	vpermq		\$0xb1,$H2,$D2
+	vpaddq		$D3,$H3,$H3
+	vpaddq		$D4,$H4,$H4
+	vpaddq		$D0,$H0,$H0
+	vpaddq		$D1,$H1,$H1
+	vpaddq		$D2,$H2,$H2
+
+	kmovw		%eax,%k3
+	vpermq		\$0x2,$H3,$D3
+	vpermq		\$0x2,$H4,$D4
+	vpermq		\$0x2,$H0,$D0
+	vpermq		\$0x2,$H1,$D1
+	vpermq		\$0x2,$H2,$D2
+	vpaddq		$D3,$H3,$H3
+	vpaddq		$D4,$H4,$H4
+	vpaddq		$D0,$H0,$H0
+	vpaddq		$D1,$H1,$H1
+	vpaddq		$D2,$H2,$H2
+
+	vextracti64x4	\$0x1,$H3,%y#$D3
+	vextracti64x4	\$0x1,$H4,%y#$D4
+	vextracti64x4	\$0x1,$H0,%y#$D0
+	vextracti64x4	\$0x1,$H1,%y#$D1
+	vextracti64x4	\$0x1,$H2,%y#$D2
+	vpaddq		$D3,$H3,${H3}{%k3}{z}	# keep single qword in case
+	vpaddq		$D4,$H4,${H4}{%k3}{z}	# it's passed to .Ltail_avx2
+	vpaddq		$D0,$H0,${H0}{%k3}{z}
+	vpaddq		$D1,$H1,${H1}{%k3}{z}
+	vpaddq		$D2,$H2,${H2}{%k3}{z}
+___
+map(s/%z/%y/,($T0,$T1,$T2,$T3,$T4, $PADBIT));
+map(s/%z/%y/,($H0,$H1,$H2,$H3,$H4, $D0,$D1,$D2,$D3,$D4, $MASK));
+$code.=<<___;
+	################################################################
+	# lazy reduction (interleaved with input splat)
+
+	vpsrlq		\$26,$H3,$D3
+	vpand		$MASK,$H3,$H3
+	 vpsrldq	\$6,$T0,$T2		# splat input
+	 vpsrldq	\$6,$T1,$T3
+	 vpunpckhqdq	$T1,$T0,$T4		# 4
+	vpaddq		$D3,$H4,$H4		# h3 -> h4
+
+	vpsrlq		\$26,$H0,$D0
+	vpand		$MASK,$H0,$H0
+	 vpunpcklqdq	$T3,$T2,$T2		# 2:3
+	 vpunpcklqdq	$T1,$T0,$T0		# 0:1
+	vpaddq		$D0,$H1,$H1		# h0 -> h1
+
+	vpsrlq		\$26,$H4,$D4
+	vpand		$MASK,$H4,$H4
+
+	vpsrlq		\$26,$H1,$D1
+	vpand		$MASK,$H1,$H1
+	 vpsrlq		\$30,$T2,$T3
+	 vpsrlq		\$4,$T2,$T2
+	vpaddq		$D1,$H2,$H2		# h1 -> h2
+
+	vpaddq		$D4,$H0,$H0
+	vpsllq		\$2,$D4,$D4
+	 vpsrlq		\$26,$T0,$T1
+	 vpsrlq		\$40,$T4,$T4		# 4
+	vpaddq		$D4,$H0,$H0		# h4 -> h0
+
+	vpsrlq		\$26,$H2,$D2
+	vpand		$MASK,$H2,$H2
+	 vpand		$MASK,$T2,$T2		# 2
+	 vpand		$MASK,$T0,$T0		# 0
+	vpaddq		$D2,$H3,$H3		# h2 -> h3
+
+	vpsrlq		\$26,$H0,$D0
+	vpand		$MASK,$H0,$H0
+	 vpaddq		$H2,$T2,$H2		# accumulate input for .Ltail_avx2
+	 vpand		$MASK,$T1,$T1		# 1
+	vpaddq		$D0,$H1,$H1		# h0 -> h1
+
+	vpsrlq		\$26,$H3,$D3
+	vpand		$MASK,$H3,$H3
+	 vpand		$MASK,$T3,$T3		# 3
+	 vpor		32(%rcx),$T4,$T4	# padbit, yes, always
+	vpaddq		$D3,$H4,$H4		# h3 -> h4
+
+	lea		0x90(%rsp),%rax		# size optimization for .Ltail_avx2
+	add		\$64,$len
+	jnz		.Ltail_avx2$suffix
+
+	vpsubq		$T2,$H2,$H2		# undo input accumulation
+	vmovd		%x#$H0,`4*0-48-64`($ctx)# save partially reduced
+	vmovd		%x#$H1,`4*1-48-64`($ctx)
+	vmovd		%x#$H2,`4*2-48-64`($ctx)
+	vmovd		%x#$H3,`4*3-48-64`($ctx)
+	vmovd		%x#$H4,`4*4-48-64`($ctx)
+	vzeroall
+___
+$code.=<<___	if ($win64);
+	movdqa		-0xb0(%r10),%xmm6
+	movdqa		-0xa0(%r10),%xmm7
+	movdqa		-0x90(%r10),%xmm8
+	movdqa		-0x80(%r10),%xmm9
+	movdqa		-0x70(%r10),%xmm10
+	movdqa		-0x60(%r10),%xmm11
+	movdqa		-0x50(%r10),%xmm12
+	movdqa		-0x40(%r10),%xmm13
+	movdqa		-0x30(%r10),%xmm14
+	movdqa		-0x20(%r10),%xmm15
+	lea		-8(%r10),%rsp
+.Ldo_avx512_epilogue:
+___
+$code.=<<___	if (!$win64);
+	lea		-8(%r10),%rsp
+.cfi_def_cfa_register	%rsp
+___
+$code.=<<___;
+	ret
+.cfi_endproc
+___
+
+}
+
+}
+
+&declare_function("poly1305_blocks_avx2", 32, 4);
+poly1305_blocks_avxN(0);
+&end_function("poly1305_blocks_avx2");
+
+if($kernel) {
+	$code .= "#endif\n";
+}
+
+#######################################################################
+if ($avx>2) {
+# On entry we have input length divisible by 64. But since inner loop
+# processes 128 bytes per iteration, cases when length is not divisible
+# by 128 are handled by passing tail 64 bytes to .Ltail_avx2. For this
+# reason stack layout is kept identical to poly1305_blocks_avx2. If not
+# for this tail, we wouldn't have to even allocate stack frame...
+
+if($kernel) {
+	$code .= "#ifdef CONFIG_AS_AVX512\n";
+}
+
+&declare_function("poly1305_blocks_avx512", 32, 4);
+poly1305_blocks_avxN(1);
+&end_function("poly1305_blocks_avx512");
+
+if ($kernel) {
+	$code .= "#endif\n";
+}
+
+if (!$kernel && $avx>3) {
+########################################################################
+# VPMADD52 version using 2^44 radix.
+#
+# One can argue that base 2^52 would be more natural. Well, even though
+# some operations would be more natural, one has to recognize couple of
+# things. Base 2^52 doesn't provide advantage over base 2^44 if you look
+# at amount of multiply-n-accumulate operations. Secondly, it makes it
+# impossible to pre-compute multiples of 5 [referred to as s[]/sN in
+# reference implementations], which means that more such operations
+# would have to be performed in inner loop, which in turn makes critical
+# path longer. In other words, even though base 2^44 reduction might
+# look less elegant, overall critical path is actually shorter...
+
+########################################################################
+# Layout of opaque area is following.
+#
+#	unsigned __int64 h[3];		# current hash value base 2^44
+#	unsigned __int64 s[2];		# key value*20 base 2^44
+#	unsigned __int64 r[3];		# key value base 2^44
+#	struct { unsigned __int64 r^1, r^3, r^2, r^4; } R[4];
+#					# r^n positions reflect
+#					# placement in register, not
+#					# memory, R[3] is R[1]*20
+
+$code.=<<___;
+.type	poly1305_init_base2_44,\@function,3
+.align	32
+poly1305_init_base2_44:
+	xor	%rax,%rax
+	mov	%rax,0($ctx)		# initialize hash value
+	mov	%rax,8($ctx)
+	mov	%rax,16($ctx)
+
+.Linit_base2_44:
+	lea	poly1305_blocks_vpmadd52(%rip),%r10
+	lea	poly1305_emit_base2_44(%rip),%r11
+
+	mov	\$0x0ffffffc0fffffff,%rax
+	mov	\$0x0ffffffc0ffffffc,%rcx
+	and	0($inp),%rax
+	mov	\$0x00000fffffffffff,%r8
+	and	8($inp),%rcx
+	mov	\$0x00000fffffffffff,%r9
+	and	%rax,%r8
+	shrd	\$44,%rcx,%rax
+	mov	%r8,40($ctx)		# r0
+	and	%r9,%rax
+	shr	\$24,%rcx
+	mov	%rax,48($ctx)		# r1
+	lea	(%rax,%rax,4),%rax	# *5
+	mov	%rcx,56($ctx)		# r2
+	shl	\$2,%rax		# magic <<2
+	lea	(%rcx,%rcx,4),%rcx	# *5
+	shl	\$2,%rcx		# magic <<2
+	mov	%rax,24($ctx)		# s1
+	mov	%rcx,32($ctx)		# s2
+	movq	\$-1,64($ctx)		# write impossible value
+___
+$code.=<<___	if ($flavour !~ /elf32/);
+	mov	%r10,0(%rdx)
+	mov	%r11,8(%rdx)
+___
+$code.=<<___	if ($flavour =~ /elf32/);
+	mov	%r10d,0(%rdx)
+	mov	%r11d,4(%rdx)
+___
+$code.=<<___;
+	mov	\$1,%eax
+	ret
+.size	poly1305_init_base2_44,.-poly1305_init_base2_44
+___
+{
+my ($H0,$H1,$H2,$r2r1r0,$r1r0s2,$r0s2s1,$Dlo,$Dhi) = map("%ymm$_",(0..5,16,17));
+my ($T0,$inp_permd,$inp_shift,$PAD) = map("%ymm$_",(18..21));
+my ($reduc_mask,$reduc_rght,$reduc_left) = map("%ymm$_",(22..25));
+
+$code.=<<___;
+.type	poly1305_blocks_vpmadd52,\@function,4
+.align	32
+poly1305_blocks_vpmadd52:
+	shr	\$4,$len
+	jz	.Lno_data_vpmadd52		# too short
+
+	shl	\$40,$padbit
+	mov	64($ctx),%r8			# peek on power of the key
+
+	# if powers of the key are not calculated yet, process up to 3
+	# blocks with this single-block subroutine, otherwise ensure that
+	# length is divisible by 2 blocks and pass the rest down to next
+	# subroutine...
+
+	mov	\$3,%rax
+	mov	\$1,%r10
+	cmp	\$4,$len			# is input long
+	cmovae	%r10,%rax
+	test	%r8,%r8				# is power value impossible?
+	cmovns	%r10,%rax
+
+	and	$len,%rax			# is input of favourable length?
+	jz	.Lblocks_vpmadd52_4x
+
+	sub		%rax,$len
+	mov		\$7,%r10d
+	mov		\$1,%r11d
+	kmovw		%r10d,%k7
+	lea		.L2_44_inp_permd(%rip),%r10
+	kmovw		%r11d,%k1
+
+	vmovq		$padbit,%x#$PAD
+	vmovdqa64	0(%r10),$inp_permd	# .L2_44_inp_permd
+	vmovdqa64	32(%r10),$inp_shift	# .L2_44_inp_shift
+	vpermq		\$0xcf,$PAD,$PAD
+	vmovdqa64	64(%r10),$reduc_mask	# .L2_44_mask
+
+	vmovdqu64	0($ctx),${Dlo}{%k7}{z}		# load hash value
+	vmovdqu64	40($ctx),${r2r1r0}{%k7}{z}	# load keys
+	vmovdqu64	32($ctx),${r1r0s2}{%k7}{z}
+	vmovdqu64	24($ctx),${r0s2s1}{%k7}{z}
+
+	vmovdqa64	96(%r10),$reduc_rght	# .L2_44_shift_rgt
+	vmovdqa64	128(%r10),$reduc_left	# .L2_44_shift_lft
+
+	jmp		.Loop_vpmadd52
+
+.align	32
+.Loop_vpmadd52:
+	vmovdqu32	0($inp),%x#$T0		# load input as ----3210
+	lea		16($inp),$inp
+
+	vpermd		$T0,$inp_permd,$T0	# ----3210 -> --322110
+	vpsrlvq		$inp_shift,$T0,$T0
+	vpandq		$reduc_mask,$T0,$T0
+	vporq		$PAD,$T0,$T0
+
+	vpaddq		$T0,$Dlo,$Dlo		# accumulate input
+
+	vpermq		\$0,$Dlo,${H0}{%k7}{z}	# smash hash value
+	vpermq		\$0b01010101,$Dlo,${H1}{%k7}{z}
+	vpermq		\$0b10101010,$Dlo,${H2}{%k7}{z}
+
+	vpxord		$Dlo,$Dlo,$Dlo
+	vpxord		$Dhi,$Dhi,$Dhi
+
+	vpmadd52luq	$r2r1r0,$H0,$Dlo
+	vpmadd52huq	$r2r1r0,$H0,$Dhi
+
+	vpmadd52luq	$r1r0s2,$H1,$Dlo
+	vpmadd52huq	$r1r0s2,$H1,$Dhi
+
+	vpmadd52luq	$r0s2s1,$H2,$Dlo
+	vpmadd52huq	$r0s2s1,$H2,$Dhi
+
+	vpsrlvq		$reduc_rght,$Dlo,$T0	# 0 in topmost qword
+	vpsllvq		$reduc_left,$Dhi,$Dhi	# 0 in topmost qword
+	vpandq		$reduc_mask,$Dlo,$Dlo
+
+	vpaddq		$T0,$Dhi,$Dhi
+
+	vpermq		\$0b10010011,$Dhi,$Dhi	# 0 in lowest qword
+
+	vpaddq		$Dhi,$Dlo,$Dlo		# note topmost qword :-)
+
+	vpsrlvq		$reduc_rght,$Dlo,$T0	# 0 in topmost word
+	vpandq		$reduc_mask,$Dlo,$Dlo
+
+	vpermq		\$0b10010011,$T0,$T0
+
+	vpaddq		$T0,$Dlo,$Dlo
+
+	vpermq		\$0b10010011,$Dlo,${T0}{%k1}{z}
+
+	vpaddq		$T0,$Dlo,$Dlo
+	vpsllq		\$2,$T0,$T0
+
+	vpaddq		$T0,$Dlo,$Dlo
+
+	dec		%rax			# len-=16
+	jnz		.Loop_vpmadd52
+
+	vmovdqu64	$Dlo,0($ctx){%k7}	# store hash value
+
+	test		$len,$len
+	jnz		.Lblocks_vpmadd52_4x
+
+.Lno_data_vpmadd52:
+	ret
+.size	poly1305_blocks_vpmadd52,.-poly1305_blocks_vpmadd52
+___
+}
+{
+########################################################################
+# As implied by its name 4x subroutine processes 4 blocks in parallel
+# (but handles even 4*n+2 blocks lengths). It takes up to 4th key power
+# and is handled in 256-bit %ymm registers.
+
+my ($H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2) = map("%ymm$_",(0..5,16,17));
+my ($D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi) = map("%ymm$_",(18..23));
+my ($T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD) = map("%ymm$_",(24..31));
+
+$code.=<<___;
+.type	poly1305_blocks_vpmadd52_4x,\@function,4
+.align	32
+poly1305_blocks_vpmadd52_4x:
+	shr	\$4,$len
+	jz	.Lno_data_vpmadd52_4x		# too short
+
+	shl	\$40,$padbit
+	mov	64($ctx),%r8			# peek on power of the key
+
+.Lblocks_vpmadd52_4x:
+	vpbroadcastq	$padbit,$PAD
+
+	vmovdqa64	.Lx_mask44(%rip),$mask44
+	mov		\$5,%eax
+	vmovdqa64	.Lx_mask42(%rip),$mask42
+	kmovw		%eax,%k1		# used in 2x path
+
+	test		%r8,%r8			# is power value impossible?
+	js		.Linit_vpmadd52		# if it is, then init R[4]
+
+	vmovq		0($ctx),%x#$H0		# load current hash value
+	vmovq		8($ctx),%x#$H1
+	vmovq		16($ctx),%x#$H2
+
+	test		\$3,$len		# is length 4*n+2?
+	jnz		.Lblocks_vpmadd52_2x_do
+
+.Lblocks_vpmadd52_4x_do:
+	vpbroadcastq	64($ctx),$R0		# load 4th power of the key
+	vpbroadcastq	96($ctx),$R1
+	vpbroadcastq	128($ctx),$R2
+	vpbroadcastq	160($ctx),$S1
+
+.Lblocks_vpmadd52_4x_key_loaded:
+	vpsllq		\$2,$R2,$S2		# S2 = R2*5*4
+	vpaddq		$R2,$S2,$S2
+	vpsllq		\$2,$S2,$S2
+
+	test		\$7,$len		# is len 8*n?
+	jz		.Lblocks_vpmadd52_8x
+
+	vmovdqu64	16*0($inp),$T2		# load data
+	vmovdqu64	16*2($inp),$T3
+	lea		16*4($inp),$inp
+
+	vpunpcklqdq	$T3,$T2,$T1		# transpose data
+	vpunpckhqdq	$T3,$T2,$T3
+
+	# at this point 64-bit lanes are ordered as 3-1-2-0
+
+	vpsrlq		\$24,$T3,$T2		# splat the data
+	vporq		$PAD,$T2,$T2
+	 vpaddq		$T2,$H2,$H2		# accumulate input
+	vpandq		$mask44,$T1,$T0
+	vpsrlq		\$44,$T1,$T1
+	vpsllq		\$20,$T3,$T3
+	vporq		$T3,$T1,$T1
+	vpandq		$mask44,$T1,$T1
+
+	sub		\$4,$len
+	jz		.Ltail_vpmadd52_4x
+	jmp		.Loop_vpmadd52_4x
+	ud2
+
+.align	32
+.Linit_vpmadd52:
+	vmovq		24($ctx),%x#$S1		# load key
+	vmovq		56($ctx),%x#$H2
+	vmovq		32($ctx),%x#$S2
+	vmovq		40($ctx),%x#$R0
+	vmovq		48($ctx),%x#$R1
+
+	vmovdqa		$R0,$H0
+	vmovdqa		$R1,$H1
+	vmovdqa		$H2,$R2
+
+	mov		\$2,%eax
+
+.Lmul_init_vpmadd52:
+	vpxorq		$D0lo,$D0lo,$D0lo
+	vpmadd52luq	$H2,$S1,$D0lo
+	vpxorq		$D0hi,$D0hi,$D0hi
+	vpmadd52huq	$H2,$S1,$D0hi
+	vpxorq		$D1lo,$D1lo,$D1lo
+	vpmadd52luq	$H2,$S2,$D1lo
+	vpxorq		$D1hi,$D1hi,$D1hi
+	vpmadd52huq	$H2,$S2,$D1hi
+	vpxorq		$D2lo,$D2lo,$D2lo
+	vpmadd52luq	$H2,$R0,$D2lo
+	vpxorq		$D2hi,$D2hi,$D2hi
+	vpmadd52huq	$H2,$R0,$D2hi
+
+	vpmadd52luq	$H0,$R0,$D0lo
+	vpmadd52huq	$H0,$R0,$D0hi
+	vpmadd52luq	$H0,$R1,$D1lo
+	vpmadd52huq	$H0,$R1,$D1hi
+	vpmadd52luq	$H0,$R2,$D2lo
+	vpmadd52huq	$H0,$R2,$D2hi
+
+	vpmadd52luq	$H1,$S2,$D0lo
+	vpmadd52huq	$H1,$S2,$D0hi
+	vpmadd52luq	$H1,$R0,$D1lo
+	vpmadd52huq	$H1,$R0,$D1hi
+	vpmadd52luq	$H1,$R1,$D2lo
+	vpmadd52huq	$H1,$R1,$D2hi
+
+	################################################################
+	# partial reduction
+	vpsrlq		\$44,$D0lo,$tmp
+	vpsllq		\$8,$D0hi,$D0hi
+	vpandq		$mask44,$D0lo,$H0
+	vpaddq		$tmp,$D0hi,$D0hi
+
+	vpaddq		$D0hi,$D1lo,$D1lo
+
+	vpsrlq		\$44,$D1lo,$tmp
+	vpsllq		\$8,$D1hi,$D1hi
+	vpandq		$mask44,$D1lo,$H1
+	vpaddq		$tmp,$D1hi,$D1hi
+
+	vpaddq		$D1hi,$D2lo,$D2lo
+
+	vpsrlq		\$42,$D2lo,$tmp
+	vpsllq		\$10,$D2hi,$D2hi
+	vpandq		$mask42,$D2lo,$H2
+	vpaddq		$tmp,$D2hi,$D2hi
+
+	vpaddq		$D2hi,$H0,$H0
+	vpsllq		\$2,$D2hi,$D2hi
+
+	vpaddq		$D2hi,$H0,$H0
+
+	vpsrlq		\$44,$H0,$tmp		# additional step
+	vpandq		$mask44,$H0,$H0
+
+	vpaddq		$tmp,$H1,$H1
+
+	dec		%eax
+	jz		.Ldone_init_vpmadd52
+
+	vpunpcklqdq	$R1,$H1,$R1		# 1,2
+	vpbroadcastq	%x#$H1,%x#$H1		# 2,2
+	vpunpcklqdq	$R2,$H2,$R2
+	vpbroadcastq	%x#$H2,%x#$H2
+	vpunpcklqdq	$R0,$H0,$R0
+	vpbroadcastq	%x#$H0,%x#$H0
+
+	vpsllq		\$2,$R1,$S1		# S1 = R1*5*4
+	vpsllq		\$2,$R2,$S2		# S2 = R2*5*4
+	vpaddq		$R1,$S1,$S1
+	vpaddq		$R2,$S2,$S2
+	vpsllq		\$2,$S1,$S1
+	vpsllq		\$2,$S2,$S2
+
+	jmp		.Lmul_init_vpmadd52
+	ud2
+
+.align	32
+.Ldone_init_vpmadd52:
+	vinserti128	\$1,%x#$R1,$H1,$R1	# 1,2,3,4
+	vinserti128	\$1,%x#$R2,$H2,$R2
+	vinserti128	\$1,%x#$R0,$H0,$R0
+
+	vpermq		\$0b11011000,$R1,$R1	# 1,3,2,4
+	vpermq		\$0b11011000,$R2,$R2
+	vpermq		\$0b11011000,$R0,$R0
+
+	vpsllq		\$2,$R1,$S1		# S1 = R1*5*4
+	vpaddq		$R1,$S1,$S1
+	vpsllq		\$2,$S1,$S1
+
+	vmovq		0($ctx),%x#$H0		# load current hash value
+	vmovq		8($ctx),%x#$H1
+	vmovq		16($ctx),%x#$H2
+
+	test		\$3,$len		# is length 4*n+2?
+	jnz		.Ldone_init_vpmadd52_2x
+
+	vmovdqu64	$R0,64($ctx)		# save key powers
+	vpbroadcastq	%x#$R0,$R0		# broadcast 4th power
+	vmovdqu64	$R1,96($ctx)
+	vpbroadcastq	%x#$R1,$R1
+	vmovdqu64	$R2,128($ctx)
+	vpbroadcastq	%x#$R2,$R2
+	vmovdqu64	$S1,160($ctx)
+	vpbroadcastq	%x#$S1,$S1
+
+	jmp		.Lblocks_vpmadd52_4x_key_loaded
+	ud2
+
+.align	32
+.Ldone_init_vpmadd52_2x:
+	vmovdqu64	$R0,64($ctx)		# save key powers
+	vpsrldq		\$8,$R0,$R0		# 0-1-0-2
+	vmovdqu64	$R1,96($ctx)
+	vpsrldq		\$8,$R1,$R1
+	vmovdqu64	$R2,128($ctx)
+	vpsrldq		\$8,$R2,$R2
+	vmovdqu64	$S1,160($ctx)
+	vpsrldq		\$8,$S1,$S1
+	jmp		.Lblocks_vpmadd52_2x_key_loaded
+	ud2
+
+.align	32
+.Lblocks_vpmadd52_2x_do:
+	vmovdqu64	128+8($ctx),${R2}{%k1}{z}# load 2nd and 1st key powers
+	vmovdqu64	160+8($ctx),${S1}{%k1}{z}
+	vmovdqu64	64+8($ctx),${R0}{%k1}{z}
+	vmovdqu64	96+8($ctx),${R1}{%k1}{z}
+
+.Lblocks_vpmadd52_2x_key_loaded:
+	vmovdqu64	16*0($inp),$T2		# load data
+	vpxorq		$T3,$T3,$T3
+	lea		16*2($inp),$inp
+
+	vpunpcklqdq	$T3,$T2,$T1		# transpose data
+	vpunpckhqdq	$T3,$T2,$T3
+
+	# at this point 64-bit lanes are ordered as x-1-x-0
+
+	vpsrlq		\$24,$T3,$T2		# splat the data
+	vporq		$PAD,$T2,$T2
+	 vpaddq		$T2,$H2,$H2		# accumulate input
+	vpandq		$mask44,$T1,$T0
+	vpsrlq		\$44,$T1,$T1
+	vpsllq		\$20,$T3,$T3
+	vporq		$T3,$T1,$T1
+	vpandq		$mask44,$T1,$T1
+
+	jmp		.Ltail_vpmadd52_2x
+	ud2
+
+.align	32
+.Loop_vpmadd52_4x:
+	#vpaddq		$T2,$H2,$H2		# accumulate input
+	vpaddq		$T0,$H0,$H0
+	vpaddq		$T1,$H1,$H1
+
+	vpxorq		$D0lo,$D0lo,$D0lo
+	vpmadd52luq	$H2,$S1,$D0lo
+	vpxorq		$D0hi,$D0hi,$D0hi
+	vpmadd52huq	$H2,$S1,$D0hi
+	vpxorq		$D1lo,$D1lo,$D1lo
+	vpmadd52luq	$H2,$S2,$D1lo
+	vpxorq		$D1hi,$D1hi,$D1hi
+	vpmadd52huq	$H2,$S2,$D1hi
+	vpxorq		$D2lo,$D2lo,$D2lo
+	vpmadd52luq	$H2,$R0,$D2lo
+	vpxorq		$D2hi,$D2hi,$D2hi
+	vpmadd52huq	$H2,$R0,$D2hi
+
+	 vmovdqu64	16*0($inp),$T2		# load data
+	 vmovdqu64	16*2($inp),$T3
+	 lea		16*4($inp),$inp
+	vpmadd52luq	$H0,$R0,$D0lo
+	vpmadd52huq	$H0,$R0,$D0hi
+	vpmadd52luq	$H0,$R1,$D1lo
+	vpmadd52huq	$H0,$R1,$D1hi
+	vpmadd52luq	$H0,$R2,$D2lo
+	vpmadd52huq	$H0,$R2,$D2hi
+
+	 vpunpcklqdq	$T3,$T2,$T1		# transpose data
+	 vpunpckhqdq	$T3,$T2,$T3
+	vpmadd52luq	$H1,$S2,$D0lo
+	vpmadd52huq	$H1,$S2,$D0hi
+	vpmadd52luq	$H1,$R0,$D1lo
+	vpmadd52huq	$H1,$R0,$D1hi
+	vpmadd52luq	$H1,$R1,$D2lo
+	vpmadd52huq	$H1,$R1,$D2hi
+
+	################################################################
+	# partial reduction (interleaved with data splat)
+	vpsrlq		\$44,$D0lo,$tmp
+	vpsllq		\$8,$D0hi,$D0hi
+	vpandq		$mask44,$D0lo,$H0
+	vpaddq		$tmp,$D0hi,$D0hi
+
+	 vpsrlq		\$24,$T3,$T2
+	 vporq		$PAD,$T2,$T2
+	vpaddq		$D0hi,$D1lo,$D1lo
+
+	vpsrlq		\$44,$D1lo,$tmp
+	vpsllq		\$8,$D1hi,$D1hi
+	vpandq		$mask44,$D1lo,$H1
+	vpaddq		$tmp,$D1hi,$D1hi
+
+	 vpandq		$mask44,$T1,$T0
+	 vpsrlq		\$44,$T1,$T1
+	 vpsllq		\$20,$T3,$T3
+	vpaddq		$D1hi,$D2lo,$D2lo
+
+	vpsrlq		\$42,$D2lo,$tmp
+	vpsllq		\$10,$D2hi,$D2hi
+	vpandq		$mask42,$D2lo,$H2
+	vpaddq		$tmp,$D2hi,$D2hi
+
+	  vpaddq	$T2,$H2,$H2		# accumulate input
+	vpaddq		$D2hi,$H0,$H0
+	vpsllq		\$2,$D2hi,$D2hi
+
+	vpaddq		$D2hi,$H0,$H0
+	 vporq		$T3,$T1,$T1
+	 vpandq		$mask44,$T1,$T1
+
+	vpsrlq		\$44,$H0,$tmp		# additional step
+	vpandq		$mask44,$H0,$H0
+
+	vpaddq		$tmp,$H1,$H1
+
+	sub		\$4,$len		# len-=64
+	jnz		.Loop_vpmadd52_4x
+
+.Ltail_vpmadd52_4x:
+	vmovdqu64	128($ctx),$R2		# load all key powers
+	vmovdqu64	160($ctx),$S1
+	vmovdqu64	64($ctx),$R0
+	vmovdqu64	96($ctx),$R1
+
+.Ltail_vpmadd52_2x:
+	vpsllq		\$2,$R2,$S2		# S2 = R2*5*4
+	vpaddq		$R2,$S2,$S2
+	vpsllq		\$2,$S2,$S2
+
+	#vpaddq		$T2,$H2,$H2		# accumulate input
+	vpaddq		$T0,$H0,$H0
+	vpaddq		$T1,$H1,$H1
+
+	vpxorq		$D0lo,$D0lo,$D0lo
+	vpmadd52luq	$H2,$S1,$D0lo
+	vpxorq		$D0hi,$D0hi,$D0hi
+	vpmadd52huq	$H2,$S1,$D0hi
+	vpxorq		$D1lo,$D1lo,$D1lo
+	vpmadd52luq	$H2,$S2,$D1lo
+	vpxorq		$D1hi,$D1hi,$D1hi
+	vpmadd52huq	$H2,$S2,$D1hi
+	vpxorq		$D2lo,$D2lo,$D2lo
+	vpmadd52luq	$H2,$R0,$D2lo
+	vpxorq		$D2hi,$D2hi,$D2hi
+	vpmadd52huq	$H2,$R0,$D2hi
+
+	vpmadd52luq	$H0,$R0,$D0lo
+	vpmadd52huq	$H0,$R0,$D0hi
+	vpmadd52luq	$H0,$R1,$D1lo
+	vpmadd52huq	$H0,$R1,$D1hi
+	vpmadd52luq	$H0,$R2,$D2lo
+	vpmadd52huq	$H0,$R2,$D2hi
+
+	vpmadd52luq	$H1,$S2,$D0lo
+	vpmadd52huq	$H1,$S2,$D0hi
+	vpmadd52luq	$H1,$R0,$D1lo
+	vpmadd52huq	$H1,$R0,$D1hi
+	vpmadd52luq	$H1,$R1,$D2lo
+	vpmadd52huq	$H1,$R1,$D2hi
+
+	################################################################
+	# horizontal addition
+
+	mov		\$1,%eax
+	kmovw		%eax,%k1
+	vpsrldq		\$8,$D0lo,$T0
+	vpsrldq		\$8,$D0hi,$H0
+	vpsrldq		\$8,$D1lo,$T1
+	vpsrldq		\$8,$D1hi,$H1
+	vpaddq		$T0,$D0lo,$D0lo
+	vpaddq		$H0,$D0hi,$D0hi
+	vpsrldq		\$8,$D2lo,$T2
+	vpsrldq		\$8,$D2hi,$H2
+	vpaddq		$T1,$D1lo,$D1lo
+	vpaddq		$H1,$D1hi,$D1hi
+	 vpermq		\$0x2,$D0lo,$T0
+	 vpermq		\$0x2,$D0hi,$H0
+	vpaddq		$T2,$D2lo,$D2lo
+	vpaddq		$H2,$D2hi,$D2hi
+
+	vpermq		\$0x2,$D1lo,$T1
+	vpermq		\$0x2,$D1hi,$H1
+	vpaddq		$T0,$D0lo,${D0lo}{%k1}{z}
+	vpaddq		$H0,$D0hi,${D0hi}{%k1}{z}
+	vpermq		\$0x2,$D2lo,$T2
+	vpermq		\$0x2,$D2hi,$H2
+	vpaddq		$T1,$D1lo,${D1lo}{%k1}{z}
+	vpaddq		$H1,$D1hi,${D1hi}{%k1}{z}
+	vpaddq		$T2,$D2lo,${D2lo}{%k1}{z}
+	vpaddq		$H2,$D2hi,${D2hi}{%k1}{z}
+
+	################################################################
+	# partial reduction
+	vpsrlq		\$44,$D0lo,$tmp
+	vpsllq		\$8,$D0hi,$D0hi
+	vpandq		$mask44,$D0lo,$H0
+	vpaddq		$tmp,$D0hi,$D0hi
+
+	vpaddq		$D0hi,$D1lo,$D1lo
+
+	vpsrlq		\$44,$D1lo,$tmp
+	vpsllq		\$8,$D1hi,$D1hi
+	vpandq		$mask44,$D1lo,$H1
+	vpaddq		$tmp,$D1hi,$D1hi
+
+	vpaddq		$D1hi,$D2lo,$D2lo
+
+	vpsrlq		\$42,$D2lo,$tmp
+	vpsllq		\$10,$D2hi,$D2hi
+	vpandq		$mask42,$D2lo,$H2
+	vpaddq		$tmp,$D2hi,$D2hi
+
+	vpaddq		$D2hi,$H0,$H0
+	vpsllq		\$2,$D2hi,$D2hi
+
+	vpaddq		$D2hi,$H0,$H0
+
+	vpsrlq		\$44,$H0,$tmp		# additional step
+	vpandq		$mask44,$H0,$H0
+
+	vpaddq		$tmp,$H1,$H1
+						# at this point $len is
+						# either 4*n+2 or 0...
+	sub		\$2,$len		# len-=32
+	ja		.Lblocks_vpmadd52_4x_do
+
+	vmovq		%x#$H0,0($ctx)
+	vmovq		%x#$H1,8($ctx)
+	vmovq		%x#$H2,16($ctx)
+	vzeroall
+
+.Lno_data_vpmadd52_4x:
+	ret
+.size	poly1305_blocks_vpmadd52_4x,.-poly1305_blocks_vpmadd52_4x
+___
+}
+{
+########################################################################
+# As implied by its name 8x subroutine processes 8 blocks in parallel...
+# This is intermediate version, as it's used only in cases when input
+# length is either 8*n, 8*n+1 or 8*n+2...
+
+my ($H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2) = map("%ymm$_",(0..5,16,17));
+my ($D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi) = map("%ymm$_",(18..23));
+my ($T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD) = map("%ymm$_",(24..31));
+my ($RR0,$RR1,$RR2,$SS1,$SS2) = map("%ymm$_",(6..10));
+
+$code.=<<___;
+.type	poly1305_blocks_vpmadd52_8x,\@function,4
+.align	32
+poly1305_blocks_vpmadd52_8x:
+	shr	\$4,$len
+	jz	.Lno_data_vpmadd52_8x		# too short
+
+	shl	\$40,$padbit
+	mov	64($ctx),%r8			# peek on power of the key
+
+	vmovdqa64	.Lx_mask44(%rip),$mask44
+	vmovdqa64	.Lx_mask42(%rip),$mask42
+
+	test	%r8,%r8				# is power value impossible?
+	js	.Linit_vpmadd52			# if it is, then init R[4]
+
+	vmovq	0($ctx),%x#$H0			# load current hash value
+	vmovq	8($ctx),%x#$H1
+	vmovq	16($ctx),%x#$H2
+
+.Lblocks_vpmadd52_8x:
+	################################################################
+	# fist we calculate more key powers
+
+	vmovdqu64	128($ctx),$R2		# load 1-3-2-4 powers
+	vmovdqu64	160($ctx),$S1
+	vmovdqu64	64($ctx),$R0
+	vmovdqu64	96($ctx),$R1
+
+	vpsllq		\$2,$R2,$S2		# S2 = R2*5*4
+	vpaddq		$R2,$S2,$S2
+	vpsllq		\$2,$S2,$S2
+
+	vpbroadcastq	%x#$R2,$RR2		# broadcast 4th power
+	vpbroadcastq	%x#$R0,$RR0
+	vpbroadcastq	%x#$R1,$RR1
+
+	vpxorq		$D0lo,$D0lo,$D0lo
+	vpmadd52luq	$RR2,$S1,$D0lo
+	vpxorq		$D0hi,$D0hi,$D0hi
+	vpmadd52huq	$RR2,$S1,$D0hi
+	vpxorq		$D1lo,$D1lo,$D1lo
+	vpmadd52luq	$RR2,$S2,$D1lo
+	vpxorq		$D1hi,$D1hi,$D1hi
+	vpmadd52huq	$RR2,$S2,$D1hi
+	vpxorq		$D2lo,$D2lo,$D2lo
+	vpmadd52luq	$RR2,$R0,$D2lo
+	vpxorq		$D2hi,$D2hi,$D2hi
+	vpmadd52huq	$RR2,$R0,$D2hi
+
+	vpmadd52luq	$RR0,$R0,$D0lo
+	vpmadd52huq	$RR0,$R0,$D0hi
+	vpmadd52luq	$RR0,$R1,$D1lo
+	vpmadd52huq	$RR0,$R1,$D1hi
+	vpmadd52luq	$RR0,$R2,$D2lo
+	vpmadd52huq	$RR0,$R2,$D2hi
+
+	vpmadd52luq	$RR1,$S2,$D0lo
+	vpmadd52huq	$RR1,$S2,$D0hi
+	vpmadd52luq	$RR1,$R0,$D1lo
+	vpmadd52huq	$RR1,$R0,$D1hi
+	vpmadd52luq	$RR1,$R1,$D2lo
+	vpmadd52huq	$RR1,$R1,$D2hi
+
+	################################################################
+	# partial reduction
+	vpsrlq		\$44,$D0lo,$tmp
+	vpsllq		\$8,$D0hi,$D0hi
+	vpandq		$mask44,$D0lo,$RR0
+	vpaddq		$tmp,$D0hi,$D0hi
+
+	vpaddq		$D0hi,$D1lo,$D1lo
+
+	vpsrlq		\$44,$D1lo,$tmp
+	vpsllq		\$8,$D1hi,$D1hi
+	vpandq		$mask44,$D1lo,$RR1
+	vpaddq		$tmp,$D1hi,$D1hi
+
+	vpaddq		$D1hi,$D2lo,$D2lo
+
+	vpsrlq		\$42,$D2lo,$tmp
+	vpsllq		\$10,$D2hi,$D2hi
+	vpandq		$mask42,$D2lo,$RR2
+	vpaddq		$tmp,$D2hi,$D2hi
+
+	vpaddq		$D2hi,$RR0,$RR0
+	vpsllq		\$2,$D2hi,$D2hi
+
+	vpaddq		$D2hi,$RR0,$RR0
+
+	vpsrlq		\$44,$RR0,$tmp		# additional step
+	vpandq		$mask44,$RR0,$RR0
+
+	vpaddq		$tmp,$RR1,$RR1
+
+	################################################################
+	# At this point Rx holds 1324 powers, RRx - 5768, and the goal
+	# is 15263748, which reflects how data is loaded...
+
+	vpunpcklqdq	$R2,$RR2,$T2		# 3748
+	vpunpckhqdq	$R2,$RR2,$R2		# 1526
+	vpunpcklqdq	$R0,$RR0,$T0
+	vpunpckhqdq	$R0,$RR0,$R0
+	vpunpcklqdq	$R1,$RR1,$T1
+	vpunpckhqdq	$R1,$RR1,$R1
+___
+######## switch to %zmm
+map(s/%y/%z/, $H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2);
+map(s/%y/%z/, $D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi);
+map(s/%y/%z/, $T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD);
+map(s/%y/%z/, $RR0,$RR1,$RR2,$SS1,$SS2);
+
+$code.=<<___;
+	vshufi64x2	\$0x44,$R2,$T2,$RR2	# 15263748
+	vshufi64x2	\$0x44,$R0,$T0,$RR0
+	vshufi64x2	\$0x44,$R1,$T1,$RR1
+
+	vmovdqu64	16*0($inp),$T2		# load data
+	vmovdqu64	16*4($inp),$T3
+	lea		16*8($inp),$inp
+
+	vpsllq		\$2,$RR2,$SS2		# S2 = R2*5*4
+	vpsllq		\$2,$RR1,$SS1		# S1 = R1*5*4
+	vpaddq		$RR2,$SS2,$SS2
+	vpaddq		$RR1,$SS1,$SS1
+	vpsllq		\$2,$SS2,$SS2
+	vpsllq		\$2,$SS1,$SS1
+
+	vpbroadcastq	$padbit,$PAD
+	vpbroadcastq	%x#$mask44,$mask44
+	vpbroadcastq	%x#$mask42,$mask42
+
+	vpbroadcastq	%x#$SS1,$S1		# broadcast 8th power
+	vpbroadcastq	%x#$SS2,$S2
+	vpbroadcastq	%x#$RR0,$R0
+	vpbroadcastq	%x#$RR1,$R1
+	vpbroadcastq	%x#$RR2,$R2
+
+	vpunpcklqdq	$T3,$T2,$T1		# transpose data
+	vpunpckhqdq	$T3,$T2,$T3
+
+	# at this point 64-bit lanes are ordered as 73625140
+
+	vpsrlq		\$24,$T3,$T2		# splat the data
+	vporq		$PAD,$T2,$T2
+	 vpaddq		$T2,$H2,$H2		# accumulate input
+	vpandq		$mask44,$T1,$T0
+	vpsrlq		\$44,$T1,$T1
+	vpsllq		\$20,$T3,$T3
+	vporq		$T3,$T1,$T1
+	vpandq		$mask44,$T1,$T1
+
+	sub		\$8,$len
+	jz		.Ltail_vpmadd52_8x
+	jmp		.Loop_vpmadd52_8x
+
+.align	32
+.Loop_vpmadd52_8x:
+	#vpaddq		$T2,$H2,$H2		# accumulate input
+	vpaddq		$T0,$H0,$H0
+	vpaddq		$T1,$H1,$H1
+
+	vpxorq		$D0lo,$D0lo,$D0lo
+	vpmadd52luq	$H2,$S1,$D0lo
+	vpxorq		$D0hi,$D0hi,$D0hi
+	vpmadd52huq	$H2,$S1,$D0hi
+	vpxorq		$D1lo,$D1lo,$D1lo
+	vpmadd52luq	$H2,$S2,$D1lo
+	vpxorq		$D1hi,$D1hi,$D1hi
+	vpmadd52huq	$H2,$S2,$D1hi
+	vpxorq		$D2lo,$D2lo,$D2lo
+	vpmadd52luq	$H2,$R0,$D2lo
+	vpxorq		$D2hi,$D2hi,$D2hi
+	vpmadd52huq	$H2,$R0,$D2hi
+
+	 vmovdqu64	16*0($inp),$T2		# load data
+	 vmovdqu64	16*4($inp),$T3
+	 lea		16*8($inp),$inp
+	vpmadd52luq	$H0,$R0,$D0lo
+	vpmadd52huq	$H0,$R0,$D0hi
+	vpmadd52luq	$H0,$R1,$D1lo
+	vpmadd52huq	$H0,$R1,$D1hi
+	vpmadd52luq	$H0,$R2,$D2lo
+	vpmadd52huq	$H0,$R2,$D2hi
+
+	 vpunpcklqdq	$T3,$T2,$T1		# transpose data
+	 vpunpckhqdq	$T3,$T2,$T3
+	vpmadd52luq	$H1,$S2,$D0lo
+	vpmadd52huq	$H1,$S2,$D0hi
+	vpmadd52luq	$H1,$R0,$D1lo
+	vpmadd52huq	$H1,$R0,$D1hi
+	vpmadd52luq	$H1,$R1,$D2lo
+	vpmadd52huq	$H1,$R1,$D2hi
+
+	################################################################
+	# partial reduction (interleaved with data splat)
+	vpsrlq		\$44,$D0lo,$tmp
+	vpsllq		\$8,$D0hi,$D0hi
+	vpandq		$mask44,$D0lo,$H0
+	vpaddq		$tmp,$D0hi,$D0hi
+
+	 vpsrlq		\$24,$T3,$T2
+	 vporq		$PAD,$T2,$T2
+	vpaddq		$D0hi,$D1lo,$D1lo
+
+	vpsrlq		\$44,$D1lo,$tmp
+	vpsllq		\$8,$D1hi,$D1hi
+	vpandq		$mask44,$D1lo,$H1
+	vpaddq		$tmp,$D1hi,$D1hi
+
+	 vpandq		$mask44,$T1,$T0
+	 vpsrlq		\$44,$T1,$T1
+	 vpsllq		\$20,$T3,$T3
+	vpaddq		$D1hi,$D2lo,$D2lo
+
+	vpsrlq		\$42,$D2lo,$tmp
+	vpsllq		\$10,$D2hi,$D2hi
+	vpandq		$mask42,$D2lo,$H2
+	vpaddq		$tmp,$D2hi,$D2hi
+
+	  vpaddq	$T2,$H2,$H2		# accumulate input
+	vpaddq		$D2hi,$H0,$H0
+	vpsllq		\$2,$D2hi,$D2hi
+
+	vpaddq		$D2hi,$H0,$H0
+	 vporq		$T3,$T1,$T1
+	 vpandq		$mask44,$T1,$T1
+
+	vpsrlq		\$44,$H0,$tmp		# additional step
+	vpandq		$mask44,$H0,$H0
+
+	vpaddq		$tmp,$H1,$H1
+
+	sub		\$8,$len		# len-=128
+	jnz		.Loop_vpmadd52_8x
+
+.Ltail_vpmadd52_8x:
+	#vpaddq		$T2,$H2,$H2		# accumulate input
+	vpaddq		$T0,$H0,$H0
+	vpaddq		$T1,$H1,$H1
+
+	vpxorq		$D0lo,$D0lo,$D0lo
+	vpmadd52luq	$H2,$SS1,$D0lo
+	vpxorq		$D0hi,$D0hi,$D0hi
+	vpmadd52huq	$H2,$SS1,$D0hi
+	vpxorq		$D1lo,$D1lo,$D1lo
+	vpmadd52luq	$H2,$SS2,$D1lo
+	vpxorq		$D1hi,$D1hi,$D1hi
+	vpmadd52huq	$H2,$SS2,$D1hi
+	vpxorq		$D2lo,$D2lo,$D2lo
+	vpmadd52luq	$H2,$RR0,$D2lo
+	vpxorq		$D2hi,$D2hi,$D2hi
+	vpmadd52huq	$H2,$RR0,$D2hi
+
+	vpmadd52luq	$H0,$RR0,$D0lo
+	vpmadd52huq	$H0,$RR0,$D0hi
+	vpmadd52luq	$H0,$RR1,$D1lo
+	vpmadd52huq	$H0,$RR1,$D1hi
+	vpmadd52luq	$H0,$RR2,$D2lo
+	vpmadd52huq	$H0,$RR2,$D2hi
+
+	vpmadd52luq	$H1,$SS2,$D0lo
+	vpmadd52huq	$H1,$SS2,$D0hi
+	vpmadd52luq	$H1,$RR0,$D1lo
+	vpmadd52huq	$H1,$RR0,$D1hi
+	vpmadd52luq	$H1,$RR1,$D2lo
+	vpmadd52huq	$H1,$RR1,$D2hi
+
+	################################################################
+	# horizontal addition
+
+	mov		\$1,%eax
+	kmovw		%eax,%k1
+	vpsrldq		\$8,$D0lo,$T0
+	vpsrldq		\$8,$D0hi,$H0
+	vpsrldq		\$8,$D1lo,$T1
+	vpsrldq		\$8,$D1hi,$H1
+	vpaddq		$T0,$D0lo,$D0lo
+	vpaddq		$H0,$D0hi,$D0hi
+	vpsrldq		\$8,$D2lo,$T2
+	vpsrldq		\$8,$D2hi,$H2
+	vpaddq		$T1,$D1lo,$D1lo
+	vpaddq		$H1,$D1hi,$D1hi
+	 vpermq		\$0x2,$D0lo,$T0
+	 vpermq		\$0x2,$D0hi,$H0
+	vpaddq		$T2,$D2lo,$D2lo
+	vpaddq		$H2,$D2hi,$D2hi
+
+	vpermq		\$0x2,$D1lo,$T1
+	vpermq		\$0x2,$D1hi,$H1
+	vpaddq		$T0,$D0lo,$D0lo
+	vpaddq		$H0,$D0hi,$D0hi
+	vpermq		\$0x2,$D2lo,$T2
+	vpermq		\$0x2,$D2hi,$H2
+	vpaddq		$T1,$D1lo,$D1lo
+	vpaddq		$H1,$D1hi,$D1hi
+	 vextracti64x4	\$1,$D0lo,%y#$T0
+	 vextracti64x4	\$1,$D0hi,%y#$H0
+	vpaddq		$T2,$D2lo,$D2lo
+	vpaddq		$H2,$D2hi,$D2hi
+
+	vextracti64x4	\$1,$D1lo,%y#$T1
+	vextracti64x4	\$1,$D1hi,%y#$H1
+	vextracti64x4	\$1,$D2lo,%y#$T2
+	vextracti64x4	\$1,$D2hi,%y#$H2
+___
+######## switch back to %ymm
+map(s/%z/%y/, $H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2);
+map(s/%z/%y/, $D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi);
+map(s/%z/%y/, $T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD);
+
+$code.=<<___;
+	vpaddq		$T0,$D0lo,${D0lo}{%k1}{z}
+	vpaddq		$H0,$D0hi,${D0hi}{%k1}{z}
+	vpaddq		$T1,$D1lo,${D1lo}{%k1}{z}
+	vpaddq		$H1,$D1hi,${D1hi}{%k1}{z}
+	vpaddq		$T2,$D2lo,${D2lo}{%k1}{z}
+	vpaddq		$H2,$D2hi,${D2hi}{%k1}{z}
+
+	################################################################
+	# partial reduction
+	vpsrlq		\$44,$D0lo,$tmp
+	vpsllq		\$8,$D0hi,$D0hi
+	vpandq		$mask44,$D0lo,$H0
+	vpaddq		$tmp,$D0hi,$D0hi
+
+	vpaddq		$D0hi,$D1lo,$D1lo
+
+	vpsrlq		\$44,$D1lo,$tmp
+	vpsllq		\$8,$D1hi,$D1hi
+	vpandq		$mask44,$D1lo,$H1
+	vpaddq		$tmp,$D1hi,$D1hi
+
+	vpaddq		$D1hi,$D2lo,$D2lo
+
+	vpsrlq		\$42,$D2lo,$tmp
+	vpsllq		\$10,$D2hi,$D2hi
+	vpandq		$mask42,$D2lo,$H2
+	vpaddq		$tmp,$D2hi,$D2hi
+
+	vpaddq		$D2hi,$H0,$H0
+	vpsllq		\$2,$D2hi,$D2hi
+
+	vpaddq		$D2hi,$H0,$H0
+
+	vpsrlq		\$44,$H0,$tmp		# additional step
+	vpandq		$mask44,$H0,$H0
+
+	vpaddq		$tmp,$H1,$H1
+
+	################################################################
+
+	vmovq		%x#$H0,0($ctx)
+	vmovq		%x#$H1,8($ctx)
+	vmovq		%x#$H2,16($ctx)
+	vzeroall
+
+.Lno_data_vpmadd52_8x:
+	ret
+.size	poly1305_blocks_vpmadd52_8x,.-poly1305_blocks_vpmadd52_8x
+___
+}
+$code.=<<___;
+.type	poly1305_emit_base2_44,\@function,3
+.align	32
+poly1305_emit_base2_44:
+	mov	0($ctx),%r8	# load hash value
+	mov	8($ctx),%r9
+	mov	16($ctx),%r10
+
+	mov	%r9,%rax
+	shr	\$20,%r9
+	shl	\$44,%rax
+	mov	%r10,%rcx
+	shr	\$40,%r10
+	shl	\$24,%rcx
+
+	add	%rax,%r8
+	adc	%rcx,%r9
+	adc	\$0,%r10
+
+	mov	%r8,%rax
+	add	\$5,%r8		# compare to modulus
+	mov	%r9,%rcx
+	adc	\$0,%r9
+	adc	\$0,%r10
+	shr	\$2,%r10	# did 130-bit value overflow?
+	cmovnz	%r8,%rax
+	cmovnz	%r9,%rcx
+
+	add	0($nonce),%rax	# accumulate nonce
+	adc	8($nonce),%rcx
+	mov	%rax,0($mac)	# write result
+	mov	%rcx,8($mac)
+
+	ret
+.size	poly1305_emit_base2_44,.-poly1305_emit_base2_44
+___
+}	}	}
+}
+
+if (!$kernel)
+{	# chacha20-poly1305 helpers
+my ($out,$inp,$otp,$len)=$win64 ? ("%rcx","%rdx","%r8", "%r9") :  # Win64 order
+                                  ("%rdi","%rsi","%rdx","%rcx");  # Unix order
+$code.=<<___;
+.globl	xor128_encrypt_n_pad
+.type	xor128_encrypt_n_pad,\@abi-omnipotent
+.align	16
+xor128_encrypt_n_pad:
+	sub	$otp,$inp
+	sub	$otp,$out
+	mov	$len,%r10		# put len aside
+	shr	\$4,$len		# len / 16
+	jz	.Ltail_enc
+	nop
+.Loop_enc_xmm:
+	movdqu	($inp,$otp),%xmm0
+	pxor	($otp),%xmm0
+	movdqu	%xmm0,($out,$otp)
+	movdqa	%xmm0,($otp)
+	lea	16($otp),$otp
+	dec	$len
+	jnz	.Loop_enc_xmm
+
+	and	\$15,%r10		# len % 16
+	jz	.Ldone_enc
+
+.Ltail_enc:
+	mov	\$16,$len
+	sub	%r10,$len
+	xor	%eax,%eax
+.Loop_enc_byte:
+	mov	($inp,$otp),%al
+	xor	($otp),%al
+	mov	%al,($out,$otp)
+	mov	%al,($otp)
+	lea	1($otp),$otp
+	dec	%r10
+	jnz	.Loop_enc_byte
+
+	xor	%eax,%eax
+.Loop_enc_pad:
+	mov	%al,($otp)
+	lea	1($otp),$otp
+	dec	$len
+	jnz	.Loop_enc_pad
+
+.Ldone_enc:
+	mov	$otp,%rax
+	ret
+.size	xor128_encrypt_n_pad,.-xor128_encrypt_n_pad
+
+.globl	xor128_decrypt_n_pad
+.type	xor128_decrypt_n_pad,\@abi-omnipotent
+.align	16
+xor128_decrypt_n_pad:
+	sub	$otp,$inp
+	sub	$otp,$out
+	mov	$len,%r10		# put len aside
+	shr	\$4,$len		# len / 16
+	jz	.Ltail_dec
+	nop
+.Loop_dec_xmm:
+	movdqu	($inp,$otp),%xmm0
+	movdqa	($otp),%xmm1
+	pxor	%xmm0,%xmm1
+	movdqu	%xmm1,($out,$otp)
+	movdqa	%xmm0,($otp)
+	lea	16($otp),$otp
+	dec	$len
+	jnz	.Loop_dec_xmm
+
+	pxor	%xmm1,%xmm1
+	and	\$15,%r10		# len % 16
+	jz	.Ldone_dec
+
+.Ltail_dec:
+	mov	\$16,$len
+	sub	%r10,$len
+	xor	%eax,%eax
+	xor	%r11,%r11
+.Loop_dec_byte:
+	mov	($inp,$otp),%r11b
+	mov	($otp),%al
+	xor	%r11b,%al
+	mov	%al,($out,$otp)
+	mov	%r11b,($otp)
+	lea	1($otp),$otp
+	dec	%r10
+	jnz	.Loop_dec_byte
+
+	xor	%eax,%eax
+.Loop_dec_pad:
+	mov	%al,($otp)
+	lea	1($otp),$otp
+	dec	$len
+	jnz	.Loop_dec_pad
+
+.Ldone_dec:
+	mov	$otp,%rax
+	ret
+.size	xor128_decrypt_n_pad,.-xor128_decrypt_n_pad
+___
+}
+
+# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
+#		CONTEXT *context,DISPATCHER_CONTEXT *disp)
+if ($win64) {
+$rec="%rcx";
+$frame="%rdx";
+$context="%r8";
+$disp="%r9";
+
+$code.=<<___;
+.extern	__imp_RtlVirtualUnwind
+.type	se_handler,\@abi-omnipotent
+.align	16
+se_handler:
+	push	%rsi
+	push	%rdi
+	push	%rbx
+	push	%rbp
+	push	%r12
+	push	%r13
+	push	%r14
+	push	%r15
+	pushfq
+	sub	\$64,%rsp
+
+	mov	120($context),%rax	# pull context->Rax
+	mov	248($context),%rbx	# pull context->Rip
+
+	mov	8($disp),%rsi		# disp->ImageBase
+	mov	56($disp),%r11		# disp->HandlerData
+
+	mov	0(%r11),%r10d		# HandlerData[0]
+	lea	(%rsi,%r10),%r10	# prologue label
+	cmp	%r10,%rbx		# context->Rip<.Lprologue
+	jb	.Lcommon_seh_tail
+
+	mov	152($context),%rax	# pull context->Rsp
+
+	mov	4(%r11),%r10d		# HandlerData[1]
+	lea	(%rsi,%r10),%r10	# epilogue label
+	cmp	%r10,%rbx		# context->Rip>=.Lepilogue
+	jae	.Lcommon_seh_tail
+
+	lea	48(%rax),%rax
+
+	mov	-8(%rax),%rbx
+	mov	-16(%rax),%rbp
+	mov	-24(%rax),%r12
+	mov	-32(%rax),%r13
+	mov	-40(%rax),%r14
+	mov	-48(%rax),%r15
+	mov	%rbx,144($context)	# restore context->Rbx
+	mov	%rbp,160($context)	# restore context->Rbp
+	mov	%r12,216($context)	# restore context->R12
+	mov	%r13,224($context)	# restore context->R13
+	mov	%r14,232($context)	# restore context->R14
+	mov	%r15,240($context)	# restore context->R14
+
+	jmp	.Lcommon_seh_tail
+.size	se_handler,.-se_handler
+
+.type	avx_handler,\@abi-omnipotent
+.align	16
+avx_handler:
+	push	%rsi
+	push	%rdi
+	push	%rbx
+	push	%rbp
+	push	%r12
+	push	%r13
+	push	%r14
+	push	%r15
+	pushfq
+	sub	\$64,%rsp
+
+	mov	120($context),%rax	# pull context->Rax
+	mov	248($context),%rbx	# pull context->Rip
+
+	mov	8($disp),%rsi		# disp->ImageBase
+	mov	56($disp),%r11		# disp->HandlerData
+
+	mov	0(%r11),%r10d		# HandlerData[0]
+	lea	(%rsi,%r10),%r10	# prologue label
+	cmp	%r10,%rbx		# context->Rip<prologue label
+	jb	.Lcommon_seh_tail
+
+	mov	152($context),%rax	# pull context->Rsp
+
+	mov	4(%r11),%r10d		# HandlerData[1]
+	lea	(%rsi,%r10),%r10	# epilogue label
+	cmp	%r10,%rbx		# context->Rip>=epilogue label
+	jae	.Lcommon_seh_tail
+
+	mov	208($context),%rax	# pull context->R11
+
+	lea	0x50(%rax),%rsi
+	lea	0xf8(%rax),%rax
+	lea	512($context),%rdi	# &context.Xmm6
+	mov	\$20,%ecx
+	.long	0xa548f3fc		# cld; rep movsq
+
+.Lcommon_seh_tail:
+	mov	8(%rax),%rdi
+	mov	16(%rax),%rsi
+	mov	%rax,152($context)	# restore context->Rsp
+	mov	%rsi,168($context)	# restore context->Rsi
+	mov	%rdi,176($context)	# restore context->Rdi
+
+	mov	40($disp),%rdi		# disp->ContextRecord
+	mov	$context,%rsi		# context
+	mov	\$154,%ecx		# sizeof(CONTEXT)
+	.long	0xa548f3fc		# cld; rep movsq
+
+	mov	$disp,%rsi
+	xor	%rcx,%rcx		# arg1, UNW_FLAG_NHANDLER
+	mov	8(%rsi),%rdx		# arg2, disp->ImageBase
+	mov	0(%rsi),%r8		# arg3, disp->ControlPc
+	mov	16(%rsi),%r9		# arg4, disp->FunctionEntry
+	mov	40(%rsi),%r10		# disp->ContextRecord
+	lea	56(%rsi),%r11		# &disp->HandlerData
+	lea	24(%rsi),%r12		# &disp->EstablisherFrame
+	mov	%r10,32(%rsp)		# arg5
+	mov	%r11,40(%rsp)		# arg6
+	mov	%r12,48(%rsp)		# arg7
+	mov	%rcx,56(%rsp)		# arg8, (NULL)
+	call	*__imp_RtlVirtualUnwind(%rip)
+
+	mov	\$1,%eax		# ExceptionContinueSearch
+	add	\$64,%rsp
+	popfq
+	pop	%r15
+	pop	%r14
+	pop	%r13
+	pop	%r12
+	pop	%rbp
+	pop	%rbx
+	pop	%rdi
+	pop	%rsi
+	ret
+.size	avx_handler,.-avx_handler
+
+.section	.pdata
+.align	4
+	.rva	.LSEH_begin_poly1305_init_x86_64
+	.rva	.LSEH_end_poly1305_init_x86_64
+	.rva	.LSEH_info_poly1305_init_x86_64
+
+	.rva	.LSEH_begin_poly1305_blocks_x86_64
+	.rva	.LSEH_end_poly1305_blocks_x86_64
+	.rva	.LSEH_info_poly1305_blocks_x86_64
+
+	.rva	.LSEH_begin_poly1305_emit_x86_64
+	.rva	.LSEH_end_poly1305_emit_x86_64
+	.rva	.LSEH_info_poly1305_emit_x86_64
+___
+$code.=<<___ if ($avx);
+	.rva	.LSEH_begin_poly1305_blocks_avx
+	.rva	.Lbase2_64_avx
+	.rva	.LSEH_info_poly1305_blocks_avx_1
+
+	.rva	.Lbase2_64_avx
+	.rva	.Leven_avx
+	.rva	.LSEH_info_poly1305_blocks_avx_2
+
+	.rva	.Leven_avx
+	.rva	.LSEH_end_poly1305_blocks_avx
+	.rva	.LSEH_info_poly1305_blocks_avx_3
+
+	.rva	.LSEH_begin_poly1305_emit_avx
+	.rva	.LSEH_end_poly1305_emit_avx
+	.rva	.LSEH_info_poly1305_emit_avx
+___
+$code.=<<___ if ($avx>1);
+	.rva	.LSEH_begin_poly1305_blocks_avx2
+	.rva	.Lbase2_64_avx2
+	.rva	.LSEH_info_poly1305_blocks_avx2_1
+
+	.rva	.Lbase2_64_avx2
+	.rva	.Leven_avx2
+	.rva	.LSEH_info_poly1305_blocks_avx2_2
+
+	.rva	.Leven_avx2
+	.rva	.LSEH_end_poly1305_blocks_avx2
+	.rva	.LSEH_info_poly1305_blocks_avx2_3
+___
+$code.=<<___ if ($avx>2);
+	.rva	.LSEH_begin_poly1305_blocks_avx512
+	.rva	.LSEH_end_poly1305_blocks_avx512
+	.rva	.LSEH_info_poly1305_blocks_avx512
+___
+$code.=<<___;
+.section	.xdata
+.align	8
+.LSEH_info_poly1305_init_x86_64:
+	.byte	9,0,0,0
+	.rva	se_handler
+	.rva	.LSEH_begin_poly1305_init_x86_64,.LSEH_begin_poly1305_init_x86_64
+
+.LSEH_info_poly1305_blocks_x86_64:
+	.byte	9,0,0,0
+	.rva	se_handler
+	.rva	.Lblocks_body,.Lblocks_epilogue
+
+.LSEH_info_poly1305_emit_x86_64:
+	.byte	9,0,0,0
+	.rva	se_handler
+	.rva	.LSEH_begin_poly1305_emit_x86_64,.LSEH_begin_poly1305_emit_x86_64
+___
+$code.=<<___ if ($avx);
+.LSEH_info_poly1305_blocks_avx_1:
+	.byte	9,0,0,0
+	.rva	se_handler
+	.rva	.Lblocks_avx_body,.Lblocks_avx_epilogue		# HandlerData[]
+
+.LSEH_info_poly1305_blocks_avx_2:
+	.byte	9,0,0,0
+	.rva	se_handler
+	.rva	.Lbase2_64_avx_body,.Lbase2_64_avx_epilogue	# HandlerData[]
+
+.LSEH_info_poly1305_blocks_avx_3:
+	.byte	9,0,0,0
+	.rva	avx_handler
+	.rva	.Ldo_avx_body,.Ldo_avx_epilogue			# HandlerData[]
+
+.LSEH_info_poly1305_emit_avx:
+	.byte	9,0,0,0
+	.rva	se_handler
+	.rva	.LSEH_begin_poly1305_emit_avx,.LSEH_begin_poly1305_emit_avx
+___
+$code.=<<___ if ($avx>1);
+.LSEH_info_poly1305_blocks_avx2_1:
+	.byte	9,0,0,0
+	.rva	se_handler
+	.rva	.Lblocks_avx2_body,.Lblocks_avx2_epilogue	# HandlerData[]
+
+.LSEH_info_poly1305_blocks_avx2_2:
+	.byte	9,0,0,0
+	.rva	se_handler
+	.rva	.Lbase2_64_avx2_body,.Lbase2_64_avx2_epilogue	# HandlerData[]
+
+.LSEH_info_poly1305_blocks_avx2_3:
+	.byte	9,0,0,0
+	.rva	avx_handler
+	.rva	.Ldo_avx2_body,.Ldo_avx2_epilogue		# HandlerData[]
+___
+$code.=<<___ if ($avx>2);
+.LSEH_info_poly1305_blocks_avx512:
+	.byte	9,0,0,0
+	.rva	avx_handler
+	.rva	.Ldo_avx512_body,.Ldo_avx512_epilogue		# HandlerData[]
+___
+}
+
+open SELF,$0;
+while(<SELF>) {
+	next if (/^#!/);
+	last if (!s/^#/\/\// and !/^$/);
+	print;
+}
+close SELF;
+
+foreach (split('\n',$code)) {
+	s/\`([^\`]*)\`/eval($1)/ge;
+	s/%r([a-z]+)#d/%e$1/g;
+	s/%r([0-9]+)#d/%r$1d/g;
+	s/%x#%[yz]/%x/g or s/%y#%z/%y/g or s/%z#%[yz]/%z/g;
+
+	if ($kernel) {
+		s/(^\.type.*),[0-9]+$/\1/;
+		s/(^\.type.*),\@abi-omnipotent+$/\1,\@function/;
+		next if /^\.cfi.*/;
+	}
+
+	print $_,"\n";
+}
+close STDOUT;
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/selftest/blake2s.c b/net/wireguard/crypto/zinc/selftest/blake2s.c
--- a/net/wireguard/crypto/zinc/selftest/blake2s.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/selftest/blake2s.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,2090 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+static const u8 blake2s_testvecs[][BLAKE2S_HASH_SIZE] __initconst = {
+	{ 0x69, 0x21, 0x7a, 0x30, 0x79, 0x90, 0x80, 0x94,
+	  0xe1, 0x11, 0x21, 0xd0, 0x42, 0x35, 0x4a, 0x7c,
+	  0x1f, 0x55, 0xb6, 0x48, 0x2c, 0xa1, 0xa5, 0x1e,
+	  0x1b, 0x25, 0x0d, 0xfd, 0x1e, 0xd0, 0xee, 0xf9 },
+	{ 0xe3, 0x4d, 0x74, 0xdb, 0xaf, 0x4f, 0xf4, 0xc6,
+	  0xab, 0xd8, 0x71, 0xcc, 0x22, 0x04, 0x51, 0xd2,
+	  0xea, 0x26, 0x48, 0x84, 0x6c, 0x77, 0x57, 0xfb,
+	  0xaa, 0xc8, 0x2f, 0xe5, 0x1a, 0xd6, 0x4b, 0xea },
+	{ 0xdd, 0xad, 0x9a, 0xb1, 0x5d, 0xac, 0x45, 0x49,
+	  0xba, 0x42, 0xf4, 0x9d, 0x26, 0x24, 0x96, 0xbe,
+	  0xf6, 0xc0, 0xba, 0xe1, 0xdd, 0x34, 0x2a, 0x88,
+	  0x08, 0xf8, 0xea, 0x26, 0x7c, 0x6e, 0x21, 0x0c },
+	{ 0xe8, 0xf9, 0x1c, 0x6e, 0xf2, 0x32, 0xa0, 0x41,
+	  0x45, 0x2a, 0xb0, 0xe1, 0x49, 0x07, 0x0c, 0xdd,
+	  0x7d, 0xd1, 0x76, 0x9e, 0x75, 0xb3, 0xa5, 0x92,
+	  0x1b, 0xe3, 0x78, 0x76, 0xc4, 0x5c, 0x99, 0x00 },
+	{ 0x0c, 0xc7, 0x0e, 0x00, 0x34, 0x8b, 0x86, 0xba,
+	  0x29, 0x44, 0xd0, 0xc3, 0x20, 0x38, 0xb2, 0x5c,
+	  0x55, 0x58, 0x4f, 0x90, 0xdf, 0x23, 0x04, 0xf5,
+	  0x5f, 0xa3, 0x32, 0xaf, 0x5f, 0xb0, 0x1e, 0x20 },
+	{ 0xec, 0x19, 0x64, 0x19, 0x10, 0x87, 0xa4, 0xfe,
+	  0x9d, 0xf1, 0xc7, 0x95, 0x34, 0x2a, 0x02, 0xff,
+	  0xc1, 0x91, 0xa5, 0xb2, 0x51, 0x76, 0x48, 0x56,
+	  0xae, 0x5b, 0x8b, 0x57, 0x69, 0xf0, 0xc6, 0xcd },
+	{ 0xe1, 0xfa, 0x51, 0x61, 0x8d, 0x7d, 0xf4, 0xeb,
+	  0x70, 0xcf, 0x0d, 0x5a, 0x9e, 0x90, 0x6f, 0x80,
+	  0x6e, 0x9d, 0x19, 0xf7, 0xf4, 0xf0, 0x1e, 0x3b,
+	  0x62, 0x12, 0x88, 0xe4, 0x12, 0x04, 0x05, 0xd6 },
+	{ 0x59, 0x80, 0x01, 0xfa, 0xfb, 0xe8, 0xf9, 0x4e,
+	  0xc6, 0x6d, 0xc8, 0x27, 0xd0, 0x12, 0xcf, 0xcb,
+	  0xba, 0x22, 0x28, 0x56, 0x9f, 0x44, 0x8e, 0x89,
+	  0xea, 0x22, 0x08, 0xc8, 0xbf, 0x76, 0x92, 0x93 },
+	{ 0xc7, 0xe8, 0x87, 0xb5, 0x46, 0x62, 0x36, 0x35,
+	  0xe9, 0x3e, 0x04, 0x95, 0x59, 0x8f, 0x17, 0x26,
+	  0x82, 0x19, 0x96, 0xc2, 0x37, 0x77, 0x05, 0xb9,
+	  0x3a, 0x1f, 0x63, 0x6f, 0x87, 0x2b, 0xfa, 0x2d },
+	{ 0xc3, 0x15, 0xa4, 0x37, 0xdd, 0x28, 0x06, 0x2a,
+	  0x77, 0x0d, 0x48, 0x19, 0x67, 0x13, 0x6b, 0x1b,
+	  0x5e, 0xb8, 0x8b, 0x21, 0xee, 0x53, 0xd0, 0x32,
+	  0x9c, 0x58, 0x97, 0x12, 0x6e, 0x9d, 0xb0, 0x2c },
+	{ 0xbb, 0x47, 0x3d, 0xed, 0xdc, 0x05, 0x5f, 0xea,
+	  0x62, 0x28, 0xf2, 0x07, 0xda, 0x57, 0x53, 0x47,
+	  0xbb, 0x00, 0x40, 0x4c, 0xd3, 0x49, 0xd3, 0x8c,
+	  0x18, 0x02, 0x63, 0x07, 0xa2, 0x24, 0xcb, 0xff },
+	{ 0x68, 0x7e, 0x18, 0x73, 0xa8, 0x27, 0x75, 0x91,
+	  0xbb, 0x33, 0xd9, 0xad, 0xf9, 0xa1, 0x39, 0x12,
+	  0xef, 0xef, 0xe5, 0x57, 0xca, 0xfc, 0x39, 0xa7,
+	  0x95, 0x26, 0x23, 0xe4, 0x72, 0x55, 0xf1, 0x6d },
+	{ 0x1a, 0xc7, 0xba, 0x75, 0x4d, 0x6e, 0x2f, 0x94,
+	  0xe0, 0xe8, 0x6c, 0x46, 0xbf, 0xb2, 0x62, 0xab,
+	  0xbb, 0x74, 0xf4, 0x50, 0xef, 0x45, 0x6d, 0x6b,
+	  0x4d, 0x97, 0xaa, 0x80, 0xce, 0x6d, 0xa7, 0x67 },
+	{ 0x01, 0x2c, 0x97, 0x80, 0x96, 0x14, 0x81, 0x6b,
+	  0x5d, 0x94, 0x94, 0x47, 0x7d, 0x4b, 0x68, 0x7d,
+	  0x15, 0xb9, 0x6e, 0xb6, 0x9c, 0x0e, 0x80, 0x74,
+	  0xa8, 0x51, 0x6f, 0x31, 0x22, 0x4b, 0x5c, 0x98 },
+	{ 0x91, 0xff, 0xd2, 0x6c, 0xfa, 0x4d, 0xa5, 0x13,
+	  0x4c, 0x7e, 0xa2, 0x62, 0xf7, 0x88, 0x9c, 0x32,
+	  0x9f, 0x61, 0xf6, 0xa6, 0x57, 0x22, 0x5c, 0xc2,
+	  0x12, 0xf4, 0x00, 0x56, 0xd9, 0x86, 0xb3, 0xf4 },
+	{ 0xd9, 0x7c, 0x82, 0x8d, 0x81, 0x82, 0xa7, 0x21,
+	  0x80, 0xa0, 0x6a, 0x78, 0x26, 0x83, 0x30, 0x67,
+	  0x3f, 0x7c, 0x4e, 0x06, 0x35, 0x94, 0x7c, 0x04,
+	  0xc0, 0x23, 0x23, 0xfd, 0x45, 0xc0, 0xa5, 0x2d },
+	{ 0xef, 0xc0, 0x4c, 0xdc, 0x39, 0x1c, 0x7e, 0x91,
+	  0x19, 0xbd, 0x38, 0x66, 0x8a, 0x53, 0x4e, 0x65,
+	  0xfe, 0x31, 0x03, 0x6d, 0x6a, 0x62, 0x11, 0x2e,
+	  0x44, 0xeb, 0xeb, 0x11, 0xf9, 0xc5, 0x70, 0x80 },
+	{ 0x99, 0x2c, 0xf5, 0xc0, 0x53, 0x44, 0x2a, 0x5f,
+	  0xbc, 0x4f, 0xaf, 0x58, 0x3e, 0x04, 0xe5, 0x0b,
+	  0xb7, 0x0d, 0x2f, 0x39, 0xfb, 0xb6, 0xa5, 0x03,
+	  0xf8, 0x9e, 0x56, 0xa6, 0x3e, 0x18, 0x57, 0x8a },
+	{ 0x38, 0x64, 0x0e, 0x9f, 0x21, 0x98, 0x3e, 0x67,
+	  0xb5, 0x39, 0xca, 0xcc, 0xae, 0x5e, 0xcf, 0x61,
+	  0x5a, 0xe2, 0x76, 0x4f, 0x75, 0xa0, 0x9c, 0x9c,
+	  0x59, 0xb7, 0x64, 0x83, 0xc1, 0xfb, 0xc7, 0x35 },
+	{ 0x21, 0x3d, 0xd3, 0x4c, 0x7e, 0xfe, 0x4f, 0xb2,
+	  0x7a, 0x6b, 0x35, 0xf6, 0xb4, 0x00, 0x0d, 0x1f,
+	  0xe0, 0x32, 0x81, 0xaf, 0x3c, 0x72, 0x3e, 0x5c,
+	  0x9f, 0x94, 0x74, 0x7a, 0x5f, 0x31, 0xcd, 0x3b },
+	{ 0xec, 0x24, 0x6e, 0xee, 0xb9, 0xce, 0xd3, 0xf7,
+	  0xad, 0x33, 0xed, 0x28, 0x66, 0x0d, 0xd9, 0xbb,
+	  0x07, 0x32, 0x51, 0x3d, 0xb4, 0xe2, 0xfa, 0x27,
+	  0x8b, 0x60, 0xcd, 0xe3, 0x68, 0x2a, 0x4c, 0xcd },
+	{ 0xac, 0x9b, 0x61, 0xd4, 0x46, 0x64, 0x8c, 0x30,
+	  0x05, 0xd7, 0x89, 0x2b, 0xf3, 0xa8, 0x71, 0x9f,
+	  0x4c, 0x81, 0x81, 0xcf, 0xdc, 0xbc, 0x2b, 0x79,
+	  0xfe, 0xf1, 0x0a, 0x27, 0x9b, 0x91, 0x10, 0x95 },
+	{ 0x7b, 0xf8, 0xb2, 0x29, 0x59, 0xe3, 0x4e, 0x3a,
+	  0x43, 0xf7, 0x07, 0x92, 0x23, 0xe8, 0x3a, 0x97,
+	  0x54, 0x61, 0x7d, 0x39, 0x1e, 0x21, 0x3d, 0xfd,
+	  0x80, 0x8e, 0x41, 0xb9, 0xbe, 0xad, 0x4c, 0xe7 },
+	{ 0x68, 0xd4, 0xb5, 0xd4, 0xfa, 0x0e, 0x30, 0x2b,
+	  0x64, 0xcc, 0xc5, 0xaf, 0x79, 0x29, 0x13, 0xac,
+	  0x4c, 0x88, 0xec, 0x95, 0xc0, 0x7d, 0xdf, 0x40,
+	  0x69, 0x42, 0x56, 0xeb, 0x88, 0xce, 0x9f, 0x3d },
+	{ 0xb2, 0xc2, 0x42, 0x0f, 0x05, 0xf9, 0xab, 0xe3,
+	  0x63, 0x15, 0x91, 0x93, 0x36, 0xb3, 0x7e, 0x4e,
+	  0x0f, 0xa3, 0x3f, 0xf7, 0xe7, 0x6a, 0x49, 0x27,
+	  0x67, 0x00, 0x6f, 0xdb, 0x5d, 0x93, 0x54, 0x62 },
+	{ 0x13, 0x4f, 0x61, 0xbb, 0xd0, 0xbb, 0xb6, 0x9a,
+	  0xed, 0x53, 0x43, 0x90, 0x45, 0x51, 0xa3, 0xe6,
+	  0xc1, 0xaa, 0x7d, 0xcd, 0xd7, 0x7e, 0x90, 0x3e,
+	  0x70, 0x23, 0xeb, 0x7c, 0x60, 0x32, 0x0a, 0xa7 },
+	{ 0x46, 0x93, 0xf9, 0xbf, 0xf7, 0xd4, 0xf3, 0x98,
+	  0x6a, 0x7d, 0x17, 0x6e, 0x6e, 0x06, 0xf7, 0x2a,
+	  0xd1, 0x49, 0x0d, 0x80, 0x5c, 0x99, 0xe2, 0x53,
+	  0x47, 0xb8, 0xde, 0x77, 0xb4, 0xdb, 0x6d, 0x9b },
+	{ 0x85, 0x3e, 0x26, 0xf7, 0x41, 0x95, 0x3b, 0x0f,
+	  0xd5, 0xbd, 0xb4, 0x24, 0xe8, 0xab, 0x9e, 0x8b,
+	  0x37, 0x50, 0xea, 0xa8, 0xef, 0x61, 0xe4, 0x79,
+	  0x02, 0xc9, 0x1e, 0x55, 0x4e, 0x9c, 0x73, 0xb9 },
+	{ 0xf7, 0xde, 0x53, 0x63, 0x61, 0xab, 0xaa, 0x0e,
+	  0x15, 0x81, 0x56, 0xcf, 0x0e, 0xa4, 0xf6, 0x3a,
+	  0x99, 0xb5, 0xe4, 0x05, 0x4f, 0x8f, 0xa4, 0xc9,
+	  0xd4, 0x5f, 0x62, 0x85, 0xca, 0xd5, 0x56, 0x94 },
+	{ 0x4c, 0x23, 0x06, 0x08, 0x86, 0x0a, 0x99, 0xae,
+	  0x8d, 0x7b, 0xd5, 0xc2, 0xcc, 0x17, 0xfa, 0x52,
+	  0x09, 0x6b, 0x9a, 0x61, 0xbe, 0xdb, 0x17, 0xcb,
+	  0x76, 0x17, 0x86, 0x4a, 0xd2, 0x9c, 0xa7, 0xa6 },
+	{ 0xae, 0xb9, 0x20, 0xea, 0x87, 0x95, 0x2d, 0xad,
+	  0xb1, 0xfb, 0x75, 0x92, 0x91, 0xe3, 0x38, 0x81,
+	  0x39, 0xa8, 0x72, 0x86, 0x50, 0x01, 0x88, 0x6e,
+	  0xd8, 0x47, 0x52, 0xe9, 0x3c, 0x25, 0x0c, 0x2a },
+	{ 0xab, 0xa4, 0xad, 0x9b, 0x48, 0x0b, 0x9d, 0xf3,
+	  0xd0, 0x8c, 0xa5, 0xe8, 0x7b, 0x0c, 0x24, 0x40,
+	  0xd4, 0xe4, 0xea, 0x21, 0x22, 0x4c, 0x2e, 0xb4,
+	  0x2c, 0xba, 0xe4, 0x69, 0xd0, 0x89, 0xb9, 0x31 },
+	{ 0x05, 0x82, 0x56, 0x07, 0xd7, 0xfd, 0xf2, 0xd8,
+	  0x2e, 0xf4, 0xc3, 0xc8, 0xc2, 0xae, 0xa9, 0x61,
+	  0xad, 0x98, 0xd6, 0x0e, 0xdf, 0xf7, 0xd0, 0x18,
+	  0x98, 0x3e, 0x21, 0x20, 0x4c, 0x0d, 0x93, 0xd1 },
+	{ 0xa7, 0x42, 0xf8, 0xb6, 0xaf, 0x82, 0xd8, 0xa6,
+	  0xca, 0x23, 0x57, 0xc5, 0xf1, 0xcf, 0x91, 0xde,
+	  0xfb, 0xd0, 0x66, 0x26, 0x7d, 0x75, 0xc0, 0x48,
+	  0xb3, 0x52, 0x36, 0x65, 0x85, 0x02, 0x59, 0x62 },
+	{ 0x2b, 0xca, 0xc8, 0x95, 0x99, 0x00, 0x0b, 0x42,
+	  0xc9, 0x5a, 0xe2, 0x38, 0x35, 0xa7, 0x13, 0x70,
+	  0x4e, 0xd7, 0x97, 0x89, 0xc8, 0x4f, 0xef, 0x14,
+	  0x9a, 0x87, 0x4f, 0xf7, 0x33, 0xf0, 0x17, 0xa2 },
+	{ 0xac, 0x1e, 0xd0, 0x7d, 0x04, 0x8f, 0x10, 0x5a,
+	  0x9e, 0x5b, 0x7a, 0xb8, 0x5b, 0x09, 0xa4, 0x92,
+	  0xd5, 0xba, 0xff, 0x14, 0xb8, 0xbf, 0xb0, 0xe9,
+	  0xfd, 0x78, 0x94, 0x86, 0xee, 0xa2, 0xb9, 0x74 },
+	{ 0xe4, 0x8d, 0x0e, 0xcf, 0xaf, 0x49, 0x7d, 0x5b,
+	  0x27, 0xc2, 0x5d, 0x99, 0xe1, 0x56, 0xcb, 0x05,
+	  0x79, 0xd4, 0x40, 0xd6, 0xe3, 0x1f, 0xb6, 0x24,
+	  0x73, 0x69, 0x6d, 0xbf, 0x95, 0xe0, 0x10, 0xe4 },
+	{ 0x12, 0xa9, 0x1f, 0xad, 0xf8, 0xb2, 0x16, 0x44,
+	  0xfd, 0x0f, 0x93, 0x4f, 0x3c, 0x4a, 0x8f, 0x62,
+	  0xba, 0x86, 0x2f, 0xfd, 0x20, 0xe8, 0xe9, 0x61,
+	  0x15, 0x4c, 0x15, 0xc1, 0x38, 0x84, 0xed, 0x3d },
+	{ 0x7c, 0xbe, 0xe9, 0x6e, 0x13, 0x98, 0x97, 0xdc,
+	  0x98, 0xfb, 0xef, 0x3b, 0xe8, 0x1a, 0xd4, 0xd9,
+	  0x64, 0xd2, 0x35, 0xcb, 0x12, 0x14, 0x1f, 0xb6,
+	  0x67, 0x27, 0xe6, 0xe5, 0xdf, 0x73, 0xa8, 0x78 },
+	{ 0xeb, 0xf6, 0x6a, 0xbb, 0x59, 0x7a, 0xe5, 0x72,
+	  0xa7, 0x29, 0x7c, 0xb0, 0x87, 0x1e, 0x35, 0x5a,
+	  0xcc, 0xaf, 0xad, 0x83, 0x77, 0xb8, 0xe7, 0x8b,
+	  0xf1, 0x64, 0xce, 0x2a, 0x18, 0xde, 0x4b, 0xaf },
+	{ 0x71, 0xb9, 0x33, 0xb0, 0x7e, 0x4f, 0xf7, 0x81,
+	  0x8c, 0xe0, 0x59, 0xd0, 0x08, 0x82, 0x9e, 0x45,
+	  0x3c, 0x6f, 0xf0, 0x2e, 0xc0, 0xa7, 0xdb, 0x39,
+	  0x3f, 0xc2, 0xd8, 0x70, 0xf3, 0x7a, 0x72, 0x86 },
+	{ 0x7c, 0xf7, 0xc5, 0x13, 0x31, 0x22, 0x0b, 0x8d,
+	  0x3e, 0xba, 0xed, 0x9c, 0x29, 0x39, 0x8a, 0x16,
+	  0xd9, 0x81, 0x56, 0xe2, 0x61, 0x3c, 0xb0, 0x88,
+	  0xf2, 0xb0, 0xe0, 0x8a, 0x1b, 0xe4, 0xcf, 0x4f },
+	{ 0x3e, 0x41, 0xa1, 0x08, 0xe0, 0xf6, 0x4a, 0xd2,
+	  0x76, 0xb9, 0x79, 0xe1, 0xce, 0x06, 0x82, 0x79,
+	  0xe1, 0x6f, 0x7b, 0xc7, 0xe4, 0xaa, 0x1d, 0x21,
+	  0x1e, 0x17, 0xb8, 0x11, 0x61, 0xdf, 0x16, 0x02 },
+	{ 0x88, 0x65, 0x02, 0xa8, 0x2a, 0xb4, 0x7b, 0xa8,
+	  0xd8, 0x67, 0x10, 0xaa, 0x9d, 0xe3, 0xd4, 0x6e,
+	  0xa6, 0x5c, 0x47, 0xaf, 0x6e, 0xe8, 0xde, 0x45,
+	  0x0c, 0xce, 0xb8, 0xb1, 0x1b, 0x04, 0x5f, 0x50 },
+	{ 0xc0, 0x21, 0xbc, 0x5f, 0x09, 0x54, 0xfe, 0xe9,
+	  0x4f, 0x46, 0xea, 0x09, 0x48, 0x7e, 0x10, 0xa8,
+	  0x48, 0x40, 0xd0, 0x2f, 0x64, 0x81, 0x0b, 0xc0,
+	  0x8d, 0x9e, 0x55, 0x1f, 0x7d, 0x41, 0x68, 0x14 },
+	{ 0x20, 0x30, 0x51, 0x6e, 0x8a, 0x5f, 0xe1, 0x9a,
+	  0xe7, 0x9c, 0x33, 0x6f, 0xce, 0x26, 0x38, 0x2a,
+	  0x74, 0x9d, 0x3f, 0xd0, 0xec, 0x91, 0xe5, 0x37,
+	  0xd4, 0xbd, 0x23, 0x58, 0xc1, 0x2d, 0xfb, 0x22 },
+	{ 0x55, 0x66, 0x98, 0xda, 0xc8, 0x31, 0x7f, 0xd3,
+	  0x6d, 0xfb, 0xdf, 0x25, 0xa7, 0x9c, 0xb1, 0x12,
+	  0xd5, 0x42, 0x58, 0x60, 0x60, 0x5c, 0xba, 0xf5,
+	  0x07, 0xf2, 0x3b, 0xf7, 0xe9, 0xf4, 0x2a, 0xfe },
+	{ 0x2f, 0x86, 0x7b, 0xa6, 0x77, 0x73, 0xfd, 0xc3,
+	  0xe9, 0x2f, 0xce, 0xd9, 0x9a, 0x64, 0x09, 0xad,
+	  0x39, 0xd0, 0xb8, 0x80, 0xfd, 0xe8, 0xf1, 0x09,
+	  0xa8, 0x17, 0x30, 0xc4, 0x45, 0x1d, 0x01, 0x78 },
+	{ 0x17, 0x2e, 0xc2, 0x18, 0xf1, 0x19, 0xdf, 0xae,
+	  0x98, 0x89, 0x6d, 0xff, 0x29, 0xdd, 0x98, 0x76,
+	  0xc9, 0x4a, 0xf8, 0x74, 0x17, 0xf9, 0xae, 0x4c,
+	  0x70, 0x14, 0xbb, 0x4e, 0x4b, 0x96, 0xaf, 0xc7 },
+	{ 0x3f, 0x85, 0x81, 0x4a, 0x18, 0x19, 0x5f, 0x87,
+	  0x9a, 0xa9, 0x62, 0xf9, 0x5d, 0x26, 0xbd, 0x82,
+	  0xa2, 0x78, 0xf2, 0xb8, 0x23, 0x20, 0x21, 0x8f,
+	  0x6b, 0x3b, 0xd6, 0xf7, 0xf6, 0x67, 0xa6, 0xd9 },
+	{ 0x1b, 0x61, 0x8f, 0xba, 0xa5, 0x66, 0xb3, 0xd4,
+	  0x98, 0xc1, 0x2e, 0x98, 0x2c, 0x9e, 0xc5, 0x2e,
+	  0x4d, 0xa8, 0x5a, 0x8c, 0x54, 0xf3, 0x8f, 0x34,
+	  0xc0, 0x90, 0x39, 0x4f, 0x23, 0xc1, 0x84, 0xc1 },
+	{ 0x0c, 0x75, 0x8f, 0xb5, 0x69, 0x2f, 0xfd, 0x41,
+	  0xa3, 0x57, 0x5d, 0x0a, 0xf0, 0x0c, 0xc7, 0xfb,
+	  0xf2, 0xcb, 0xe5, 0x90, 0x5a, 0x58, 0x32, 0x3a,
+	  0x88, 0xae, 0x42, 0x44, 0xf6, 0xe4, 0xc9, 0x93 },
+	{ 0xa9, 0x31, 0x36, 0x0c, 0xad, 0x62, 0x8c, 0x7f,
+	  0x12, 0xa6, 0xc1, 0xc4, 0xb7, 0x53, 0xb0, 0xf4,
+	  0x06, 0x2a, 0xef, 0x3c, 0xe6, 0x5a, 0x1a, 0xe3,
+	  0xf1, 0x93, 0x69, 0xda, 0xdf, 0x3a, 0xe2, 0x3d },
+	{ 0xcb, 0xac, 0x7d, 0x77, 0x3b, 0x1e, 0x3b, 0x3c,
+	  0x66, 0x91, 0xd7, 0xab, 0xb7, 0xe9, 0xdf, 0x04,
+	  0x5c, 0x8b, 0xa1, 0x92, 0x68, 0xde, 0xd1, 0x53,
+	  0x20, 0x7f, 0x5e, 0x80, 0x43, 0x52, 0xec, 0x5d },
+	{ 0x23, 0xa1, 0x96, 0xd3, 0x80, 0x2e, 0xd3, 0xc1,
+	  0xb3, 0x84, 0x01, 0x9a, 0x82, 0x32, 0x58, 0x40,
+	  0xd3, 0x2f, 0x71, 0x95, 0x0c, 0x45, 0x80, 0xb0,
+	  0x34, 0x45, 0xe0, 0x89, 0x8e, 0x14, 0x05, 0x3c },
+	{ 0xf4, 0x49, 0x54, 0x70, 0xf2, 0x26, 0xc8, 0xc2,
+	  0x14, 0xbe, 0x08, 0xfd, 0xfa, 0xd4, 0xbc, 0x4a,
+	  0x2a, 0x9d, 0xbe, 0xa9, 0x13, 0x6a, 0x21, 0x0d,
+	  0xf0, 0xd4, 0xb6, 0x49, 0x29, 0xe6, 0xfc, 0x14 },
+	{ 0xe2, 0x90, 0xdd, 0x27, 0x0b, 0x46, 0x7f, 0x34,
+	  0xab, 0x1c, 0x00, 0x2d, 0x34, 0x0f, 0xa0, 0x16,
+	  0x25, 0x7f, 0xf1, 0x9e, 0x58, 0x33, 0xfd, 0xbb,
+	  0xf2, 0xcb, 0x40, 0x1c, 0x3b, 0x28, 0x17, 0xde },
+	{ 0x9f, 0xc7, 0xb5, 0xde, 0xd3, 0xc1, 0x50, 0x42,
+	  0xb2, 0xa6, 0x58, 0x2d, 0xc3, 0x9b, 0xe0, 0x16,
+	  0xd2, 0x4a, 0x68, 0x2d, 0x5e, 0x61, 0xad, 0x1e,
+	  0xff, 0x9c, 0x63, 0x30, 0x98, 0x48, 0xf7, 0x06 },
+	{ 0x8c, 0xca, 0x67, 0xa3, 0x6d, 0x17, 0xd5, 0xe6,
+	  0x34, 0x1c, 0xb5, 0x92, 0xfd, 0x7b, 0xef, 0x99,
+	  0x26, 0xc9, 0xe3, 0xaa, 0x10, 0x27, 0xea, 0x11,
+	  0xa7, 0xd8, 0xbd, 0x26, 0x0b, 0x57, 0x6e, 0x04 },
+	{ 0x40, 0x93, 0x92, 0xf5, 0x60, 0xf8, 0x68, 0x31,
+	  0xda, 0x43, 0x73, 0xee, 0x5e, 0x00, 0x74, 0x26,
+	  0x05, 0x95, 0xd7, 0xbc, 0x24, 0x18, 0x3b, 0x60,
+	  0xed, 0x70, 0x0d, 0x45, 0x83, 0xd3, 0xf6, 0xf0 },
+	{ 0x28, 0x02, 0x16, 0x5d, 0xe0, 0x90, 0x91, 0x55,
+	  0x46, 0xf3, 0x39, 0x8c, 0xd8, 0x49, 0x16, 0x4a,
+	  0x19, 0xf9, 0x2a, 0xdb, 0xc3, 0x61, 0xad, 0xc9,
+	  0x9b, 0x0f, 0x20, 0xc8, 0xea, 0x07, 0x10, 0x54 },
+	{ 0xad, 0x83, 0x91, 0x68, 0xd9, 0xf8, 0xa4, 0xbe,
+	  0x95, 0xba, 0x9e, 0xf9, 0xa6, 0x92, 0xf0, 0x72,
+	  0x56, 0xae, 0x43, 0xfe, 0x6f, 0x98, 0x64, 0xe2,
+	  0x90, 0x69, 0x1b, 0x02, 0x56, 0xce, 0x50, 0xa9 },
+	{ 0x75, 0xfd, 0xaa, 0x50, 0x38, 0xc2, 0x84, 0xb8,
+	  0x6d, 0x6e, 0x8a, 0xff, 0xe8, 0xb2, 0x80, 0x7e,
+	  0x46, 0x7b, 0x86, 0x60, 0x0e, 0x79, 0xaf, 0x36,
+	  0x89, 0xfb, 0xc0, 0x63, 0x28, 0xcb, 0xf8, 0x94 },
+	{ 0xe5, 0x7c, 0xb7, 0x94, 0x87, 0xdd, 0x57, 0x90,
+	  0x24, 0x32, 0xb2, 0x50, 0x73, 0x38, 0x13, 0xbd,
+	  0x96, 0xa8, 0x4e, 0xfc, 0xe5, 0x9f, 0x65, 0x0f,
+	  0xac, 0x26, 0xe6, 0x69, 0x6a, 0xef, 0xaf, 0xc3 },
+	{ 0x56, 0xf3, 0x4e, 0x8b, 0x96, 0x55, 0x7e, 0x90,
+	  0xc1, 0xf2, 0x4b, 0x52, 0xd0, 0xc8, 0x9d, 0x51,
+	  0x08, 0x6a, 0xcf, 0x1b, 0x00, 0xf6, 0x34, 0xcf,
+	  0x1d, 0xde, 0x92, 0x33, 0xb8, 0xea, 0xaa, 0x3e },
+	{ 0x1b, 0x53, 0xee, 0x94, 0xaa, 0xf3, 0x4e, 0x4b,
+	  0x15, 0x9d, 0x48, 0xde, 0x35, 0x2c, 0x7f, 0x06,
+	  0x61, 0xd0, 0xa4, 0x0e, 0xdf, 0xf9, 0x5a, 0x0b,
+	  0x16, 0x39, 0xb4, 0x09, 0x0e, 0x97, 0x44, 0x72 },
+	{ 0x05, 0x70, 0x5e, 0x2a, 0x81, 0x75, 0x7c, 0x14,
+	  0xbd, 0x38, 0x3e, 0xa9, 0x8d, 0xda, 0x54, 0x4e,
+	  0xb1, 0x0e, 0x6b, 0xc0, 0x7b, 0xae, 0x43, 0x5e,
+	  0x25, 0x18, 0xdb, 0xe1, 0x33, 0x52, 0x53, 0x75 },
+	{ 0xd8, 0xb2, 0x86, 0x6e, 0x8a, 0x30, 0x9d, 0xb5,
+	  0x3e, 0x52, 0x9e, 0xc3, 0x29, 0x11, 0xd8, 0x2f,
+	  0x5c, 0xa1, 0x6c, 0xff, 0x76, 0x21, 0x68, 0x91,
+	  0xa9, 0x67, 0x6a, 0xa3, 0x1a, 0xaa, 0x6c, 0x42 },
+	{ 0xf5, 0x04, 0x1c, 0x24, 0x12, 0x70, 0xeb, 0x04,
+	  0xc7, 0x1e, 0xc2, 0xc9, 0x5d, 0x4c, 0x38, 0xd8,
+	  0x03, 0xb1, 0x23, 0x7b, 0x0f, 0x29, 0xfd, 0x4d,
+	  0xb3, 0xeb, 0x39, 0x76, 0x69, 0xe8, 0x86, 0x99 },
+	{ 0x9a, 0x4c, 0xe0, 0x77, 0xc3, 0x49, 0x32, 0x2f,
+	  0x59, 0x5e, 0x0e, 0xe7, 0x9e, 0xd0, 0xda, 0x5f,
+	  0xab, 0x66, 0x75, 0x2c, 0xbf, 0xef, 0x8f, 0x87,
+	  0xd0, 0xe9, 0xd0, 0x72, 0x3c, 0x75, 0x30, 0xdd },
+	{ 0x65, 0x7b, 0x09, 0xf3, 0xd0, 0xf5, 0x2b, 0x5b,
+	  0x8f, 0x2f, 0x97, 0x16, 0x3a, 0x0e, 0xdf, 0x0c,
+	  0x04, 0xf0, 0x75, 0x40, 0x8a, 0x07, 0xbb, 0xeb,
+	  0x3a, 0x41, 0x01, 0xa8, 0x91, 0x99, 0x0d, 0x62 },
+	{ 0x1e, 0x3f, 0x7b, 0xd5, 0xa5, 0x8f, 0xa5, 0x33,
+	  0x34, 0x4a, 0xa8, 0xed, 0x3a, 0xc1, 0x22, 0xbb,
+	  0x9e, 0x70, 0xd4, 0xef, 0x50, 0xd0, 0x04, 0x53,
+	  0x08, 0x21, 0x94, 0x8f, 0x5f, 0xe6, 0x31, 0x5a },
+	{ 0x80, 0xdc, 0xcf, 0x3f, 0xd8, 0x3d, 0xfd, 0x0d,
+	  0x35, 0xaa, 0x28, 0x58, 0x59, 0x22, 0xab, 0x89,
+	  0xd5, 0x31, 0x39, 0x97, 0x67, 0x3e, 0xaf, 0x90,
+	  0x5c, 0xea, 0x9c, 0x0b, 0x22, 0x5c, 0x7b, 0x5f },
+	{ 0x8a, 0x0d, 0x0f, 0xbf, 0x63, 0x77, 0xd8, 0x3b,
+	  0xb0, 0x8b, 0x51, 0x4b, 0x4b, 0x1c, 0x43, 0xac,
+	  0xc9, 0x5d, 0x75, 0x17, 0x14, 0xf8, 0x92, 0x56,
+	  0x45, 0xcb, 0x6b, 0xc8, 0x56, 0xca, 0x15, 0x0a },
+	{ 0x9f, 0xa5, 0xb4, 0x87, 0x73, 0x8a, 0xd2, 0x84,
+	  0x4c, 0xc6, 0x34, 0x8a, 0x90, 0x19, 0x18, 0xf6,
+	  0x59, 0xa3, 0xb8, 0x9e, 0x9c, 0x0d, 0xfe, 0xea,
+	  0xd3, 0x0d, 0xd9, 0x4b, 0xcf, 0x42, 0xef, 0x8e },
+	{ 0x80, 0x83, 0x2c, 0x4a, 0x16, 0x77, 0xf5, 0xea,
+	  0x25, 0x60, 0xf6, 0x68, 0xe9, 0x35, 0x4d, 0xd3,
+	  0x69, 0x97, 0xf0, 0x37, 0x28, 0xcf, 0xa5, 0x5e,
+	  0x1b, 0x38, 0x33, 0x7c, 0x0c, 0x9e, 0xf8, 0x18 },
+	{ 0xab, 0x37, 0xdd, 0xb6, 0x83, 0x13, 0x7e, 0x74,
+	  0x08, 0x0d, 0x02, 0x6b, 0x59, 0x0b, 0x96, 0xae,
+	  0x9b, 0xb4, 0x47, 0x72, 0x2f, 0x30, 0x5a, 0x5a,
+	  0xc5, 0x70, 0xec, 0x1d, 0xf9, 0xb1, 0x74, 0x3c },
+	{ 0x3e, 0xe7, 0x35, 0xa6, 0x94, 0xc2, 0x55, 0x9b,
+	  0x69, 0x3a, 0xa6, 0x86, 0x29, 0x36, 0x1e, 0x15,
+	  0xd1, 0x22, 0x65, 0xad, 0x6a, 0x3d, 0xed, 0xf4,
+	  0x88, 0xb0, 0xb0, 0x0f, 0xac, 0x97, 0x54, 0xba },
+	{ 0xd6, 0xfc, 0xd2, 0x32, 0x19, 0xb6, 0x47, 0xe4,
+	  0xcb, 0xd5, 0xeb, 0x2d, 0x0a, 0xd0, 0x1e, 0xc8,
+	  0x83, 0x8a, 0x4b, 0x29, 0x01, 0xfc, 0x32, 0x5c,
+	  0xc3, 0x70, 0x19, 0x81, 0xca, 0x6c, 0x88, 0x8b },
+	{ 0x05, 0x20, 0xec, 0x2f, 0x5b, 0xf7, 0xa7, 0x55,
+	  0xda, 0xcb, 0x50, 0xc6, 0xbf, 0x23, 0x3e, 0x35,
+	  0x15, 0x43, 0x47, 0x63, 0xdb, 0x01, 0x39, 0xcc,
+	  0xd9, 0xfa, 0xef, 0xbb, 0x82, 0x07, 0x61, 0x2d },
+	{ 0xaf, 0xf3, 0xb7, 0x5f, 0x3f, 0x58, 0x12, 0x64,
+	  0xd7, 0x66, 0x16, 0x62, 0xb9, 0x2f, 0x5a, 0xd3,
+	  0x7c, 0x1d, 0x32, 0xbd, 0x45, 0xff, 0x81, 0xa4,
+	  0xed, 0x8a, 0xdc, 0x9e, 0xf3, 0x0d, 0xd9, 0x89 },
+	{ 0xd0, 0xdd, 0x65, 0x0b, 0xef, 0xd3, 0xba, 0x63,
+	  0xdc, 0x25, 0x10, 0x2c, 0x62, 0x7c, 0x92, 0x1b,
+	  0x9c, 0xbe, 0xb0, 0xb1, 0x30, 0x68, 0x69, 0x35,
+	  0xb5, 0xc9, 0x27, 0xcb, 0x7c, 0xcd, 0x5e, 0x3b },
+	{ 0xe1, 0x14, 0x98, 0x16, 0xb1, 0x0a, 0x85, 0x14,
+	  0xfb, 0x3e, 0x2c, 0xab, 0x2c, 0x08, 0xbe, 0xe9,
+	  0xf7, 0x3c, 0xe7, 0x62, 0x21, 0x70, 0x12, 0x46,
+	  0xa5, 0x89, 0xbb, 0xb6, 0x73, 0x02, 0xd8, 0xa9 },
+	{ 0x7d, 0xa3, 0xf4, 0x41, 0xde, 0x90, 0x54, 0x31,
+	  0x7e, 0x72, 0xb5, 0xdb, 0xf9, 0x79, 0xda, 0x01,
+	  0xe6, 0xbc, 0xee, 0xbb, 0x84, 0x78, 0xea, 0xe6,
+	  0xa2, 0x28, 0x49, 0xd9, 0x02, 0x92, 0x63, 0x5c },
+	{ 0x12, 0x30, 0xb1, 0xfc, 0x8a, 0x7d, 0x92, 0x15,
+	  0xed, 0xc2, 0xd4, 0xa2, 0xde, 0xcb, 0xdd, 0x0a,
+	  0x6e, 0x21, 0x6c, 0x92, 0x42, 0x78, 0xc9, 0x1f,
+	  0xc5, 0xd1, 0x0e, 0x7d, 0x60, 0x19, 0x2d, 0x94 },
+	{ 0x57, 0x50, 0xd7, 0x16, 0xb4, 0x80, 0x8f, 0x75,
+	  0x1f, 0xeb, 0xc3, 0x88, 0x06, 0xba, 0x17, 0x0b,
+	  0xf6, 0xd5, 0x19, 0x9a, 0x78, 0x16, 0xbe, 0x51,
+	  0x4e, 0x3f, 0x93, 0x2f, 0xbe, 0x0c, 0xb8, 0x71 },
+	{ 0x6f, 0xc5, 0x9b, 0x2f, 0x10, 0xfe, 0xba, 0x95,
+	  0x4a, 0xa6, 0x82, 0x0b, 0x3c, 0xa9, 0x87, 0xee,
+	  0x81, 0xd5, 0xcc, 0x1d, 0xa3, 0xc6, 0x3c, 0xe8,
+	  0x27, 0x30, 0x1c, 0x56, 0x9d, 0xfb, 0x39, 0xce },
+	{ 0xc7, 0xc3, 0xfe, 0x1e, 0xeb, 0xdc, 0x7b, 0x5a,
+	  0x93, 0x93, 0x26, 0xe8, 0xdd, 0xb8, 0x3e, 0x8b,
+	  0xf2, 0xb7, 0x80, 0xb6, 0x56, 0x78, 0xcb, 0x62,
+	  0xf2, 0x08, 0xb0, 0x40, 0xab, 0xdd, 0x35, 0xe2 },
+	{ 0x0c, 0x75, 0xc1, 0xa1, 0x5c, 0xf3, 0x4a, 0x31,
+	  0x4e, 0xe4, 0x78, 0xf4, 0xa5, 0xce, 0x0b, 0x8a,
+	  0x6b, 0x36, 0x52, 0x8e, 0xf7, 0xa8, 0x20, 0x69,
+	  0x6c, 0x3e, 0x42, 0x46, 0xc5, 0xa1, 0x58, 0x64 },
+	{ 0x21, 0x6d, 0xc1, 0x2a, 0x10, 0x85, 0x69, 0xa3,
+	  0xc7, 0xcd, 0xde, 0x4a, 0xed, 0x43, 0xa6, 0xc3,
+	  0x30, 0x13, 0x9d, 0xda, 0x3c, 0xcc, 0x4a, 0x10,
+	  0x89, 0x05, 0xdb, 0x38, 0x61, 0x89, 0x90, 0x50 },
+	{ 0xa5, 0x7b, 0xe6, 0xae, 0x67, 0x56, 0xf2, 0x8b,
+	  0x02, 0xf5, 0x9d, 0xad, 0xf7, 0xe0, 0xd7, 0xd8,
+	  0x80, 0x7f, 0x10, 0xfa, 0x15, 0xce, 0xd1, 0xad,
+	  0x35, 0x85, 0x52, 0x1a, 0x1d, 0x99, 0x5a, 0x89 },
+	{ 0x81, 0x6a, 0xef, 0x87, 0x59, 0x53, 0x71, 0x6c,
+	  0xd7, 0xa5, 0x81, 0xf7, 0x32, 0xf5, 0x3d, 0xd4,
+	  0x35, 0xda, 0xb6, 0x6d, 0x09, 0xc3, 0x61, 0xd2,
+	  0xd6, 0x59, 0x2d, 0xe1, 0x77, 0x55, 0xd8, 0xa8 },
+	{ 0x9a, 0x76, 0x89, 0x32, 0x26, 0x69, 0x3b, 0x6e,
+	  0xa9, 0x7e, 0x6a, 0x73, 0x8f, 0x9d, 0x10, 0xfb,
+	  0x3d, 0x0b, 0x43, 0xae, 0x0e, 0x8b, 0x7d, 0x81,
+	  0x23, 0xea, 0x76, 0xce, 0x97, 0x98, 0x9c, 0x7e },
+	{ 0x8d, 0xae, 0xdb, 0x9a, 0x27, 0x15, 0x29, 0xdb,
+	  0xb7, 0xdc, 0x3b, 0x60, 0x7f, 0xe5, 0xeb, 0x2d,
+	  0x32, 0x11, 0x77, 0x07, 0x58, 0xdd, 0x3b, 0x0a,
+	  0x35, 0x93, 0xd2, 0xd7, 0x95, 0x4e, 0x2d, 0x5b },
+	{ 0x16, 0xdb, 0xc0, 0xaa, 0x5d, 0xd2, 0xc7, 0x74,
+	  0xf5, 0x05, 0x10, 0x0f, 0x73, 0x37, 0x86, 0xd8,
+	  0xa1, 0x75, 0xfc, 0xbb, 0xb5, 0x9c, 0x43, 0xe1,
+	  0xfb, 0xff, 0x3e, 0x1e, 0xaf, 0x31, 0xcb, 0x4a },
+	{ 0x86, 0x06, 0xcb, 0x89, 0x9c, 0x6a, 0xea, 0xf5,
+	  0x1b, 0x9d, 0xb0, 0xfe, 0x49, 0x24, 0xa9, 0xfd,
+	  0x5d, 0xab, 0xc1, 0x9f, 0x88, 0x26, 0xf2, 0xbc,
+	  0x1c, 0x1d, 0x7d, 0xa1, 0x4d, 0x2c, 0x2c, 0x99 },
+	{ 0x84, 0x79, 0x73, 0x1a, 0xed, 0xa5, 0x7b, 0xd3,
+	  0x7e, 0xad, 0xb5, 0x1a, 0x50, 0x7e, 0x30, 0x7f,
+	  0x3b, 0xd9, 0x5e, 0x69, 0xdb, 0xca, 0x94, 0xf3,
+	  0xbc, 0x21, 0x72, 0x60, 0x66, 0xad, 0x6d, 0xfd },
+	{ 0x58, 0x47, 0x3a, 0x9e, 0xa8, 0x2e, 0xfa, 0x3f,
+	  0x3b, 0x3d, 0x8f, 0xc8, 0x3e, 0xd8, 0x86, 0x31,
+	  0x27, 0xb3, 0x3a, 0xe8, 0xde, 0xae, 0x63, 0x07,
+	  0x20, 0x1e, 0xdb, 0x6d, 0xde, 0x61, 0xde, 0x29 },
+	{ 0x9a, 0x92, 0x55, 0xd5, 0x3a, 0xf1, 0x16, 0xde,
+	  0x8b, 0xa2, 0x7c, 0xe3, 0x5b, 0x4c, 0x7e, 0x15,
+	  0x64, 0x06, 0x57, 0xa0, 0xfc, 0xb8, 0x88, 0xc7,
+	  0x0d, 0x95, 0x43, 0x1d, 0xac, 0xd8, 0xf8, 0x30 },
+	{ 0x9e, 0xb0, 0x5f, 0xfb, 0xa3, 0x9f, 0xd8, 0x59,
+	  0x6a, 0x45, 0x49, 0x3e, 0x18, 0xd2, 0x51, 0x0b,
+	  0xf3, 0xef, 0x06, 0x5c, 0x51, 0xd6, 0xe1, 0x3a,
+	  0xbe, 0x66, 0xaa, 0x57, 0xe0, 0x5c, 0xfd, 0xb7 },
+	{ 0x81, 0xdc, 0xc3, 0xa5, 0x05, 0xea, 0xce, 0x3f,
+	  0x87, 0x9d, 0x8f, 0x70, 0x27, 0x76, 0x77, 0x0f,
+	  0x9d, 0xf5, 0x0e, 0x52, 0x1d, 0x14, 0x28, 0xa8,
+	  0x5d, 0xaf, 0x04, 0xf9, 0xad, 0x21, 0x50, 0xe0 },
+	{ 0xe3, 0xe3, 0xc4, 0xaa, 0x3a, 0xcb, 0xbc, 0x85,
+	  0x33, 0x2a, 0xf9, 0xd5, 0x64, 0xbc, 0x24, 0x16,
+	  0x5e, 0x16, 0x87, 0xf6, 0xb1, 0xad, 0xcb, 0xfa,
+	  0xe7, 0x7a, 0x8f, 0x03, 0xc7, 0x2a, 0xc2, 0x8c },
+	{ 0x67, 0x46, 0xc8, 0x0b, 0x4e, 0xb5, 0x6a, 0xea,
+	  0x45, 0xe6, 0x4e, 0x72, 0x89, 0xbb, 0xa3, 0xed,
+	  0xbf, 0x45, 0xec, 0xf8, 0x20, 0x64, 0x81, 0xff,
+	  0x63, 0x02, 0x12, 0x29, 0x84, 0xcd, 0x52, 0x6a },
+	{ 0x2b, 0x62, 0x8e, 0x52, 0x76, 0x4d, 0x7d, 0x62,
+	  0xc0, 0x86, 0x8b, 0x21, 0x23, 0x57, 0xcd, 0xd1,
+	  0x2d, 0x91, 0x49, 0x82, 0x2f, 0x4e, 0x98, 0x45,
+	  0xd9, 0x18, 0xa0, 0x8d, 0x1a, 0xe9, 0x90, 0xc0 },
+	{ 0xe4, 0xbf, 0xe8, 0x0d, 0x58, 0xc9, 0x19, 0x94,
+	  0x61, 0x39, 0x09, 0xdc, 0x4b, 0x1a, 0x12, 0x49,
+	  0x68, 0x96, 0xc0, 0x04, 0xaf, 0x7b, 0x57, 0x01,
+	  0x48, 0x3d, 0xe4, 0x5d, 0x28, 0x23, 0xd7, 0x8e },
+	{ 0xeb, 0xb4, 0xba, 0x15, 0x0c, 0xef, 0x27, 0x34,
+	  0x34, 0x5b, 0x5d, 0x64, 0x1b, 0xbe, 0xd0, 0x3a,
+	  0x21, 0xea, 0xfa, 0xe9, 0x33, 0xc9, 0x9e, 0x00,
+	  0x92, 0x12, 0xef, 0x04, 0x57, 0x4a, 0x85, 0x30 },
+	{ 0x39, 0x66, 0xec, 0x73, 0xb1, 0x54, 0xac, 0xc6,
+	  0x97, 0xac, 0x5c, 0xf5, 0xb2, 0x4b, 0x40, 0xbd,
+	  0xb0, 0xdb, 0x9e, 0x39, 0x88, 0x36, 0xd7, 0x6d,
+	  0x4b, 0x88, 0x0e, 0x3b, 0x2a, 0xf1, 0xaa, 0x27 },
+	{ 0xef, 0x7e, 0x48, 0x31, 0xb3, 0xa8, 0x46, 0x36,
+	  0x51, 0x8d, 0x6e, 0x4b, 0xfc, 0xe6, 0x4a, 0x43,
+	  0xdb, 0x2a, 0x5d, 0xda, 0x9c, 0xca, 0x2b, 0x44,
+	  0xf3, 0x90, 0x33, 0xbd, 0xc4, 0x0d, 0x62, 0x43 },
+	{ 0x7a, 0xbf, 0x6a, 0xcf, 0x5c, 0x8e, 0x54, 0x9d,
+	  0xdb, 0xb1, 0x5a, 0xe8, 0xd8, 0xb3, 0x88, 0xc1,
+	  0xc1, 0x97, 0xe6, 0x98, 0x73, 0x7c, 0x97, 0x85,
+	  0x50, 0x1e, 0xd1, 0xf9, 0x49, 0x30, 0xb7, 0xd9 },
+	{ 0x88, 0x01, 0x8d, 0xed, 0x66, 0x81, 0x3f, 0x0c,
+	  0xa9, 0x5d, 0xef, 0x47, 0x4c, 0x63, 0x06, 0x92,
+	  0x01, 0x99, 0x67, 0xb9, 0xe3, 0x68, 0x88, 0xda,
+	  0xdd, 0x94, 0x12, 0x47, 0x19, 0xb6, 0x82, 0xf6 },
+	{ 0x39, 0x30, 0x87, 0x6b, 0x9f, 0xc7, 0x52, 0x90,
+	  0x36, 0xb0, 0x08, 0xb1, 0xb8, 0xbb, 0x99, 0x75,
+	  0x22, 0xa4, 0x41, 0x63, 0x5a, 0x0c, 0x25, 0xec,
+	  0x02, 0xfb, 0x6d, 0x90, 0x26, 0xe5, 0x5a, 0x97 },
+	{ 0x0a, 0x40, 0x49, 0xd5, 0x7e, 0x83, 0x3b, 0x56,
+	  0x95, 0xfa, 0xc9, 0x3d, 0xd1, 0xfb, 0xef, 0x31,
+	  0x66, 0xb4, 0x4b, 0x12, 0xad, 0x11, 0x24, 0x86,
+	  0x62, 0x38, 0x3a, 0xe0, 0x51, 0xe1, 0x58, 0x27 },
+	{ 0x81, 0xdc, 0xc0, 0x67, 0x8b, 0xb6, 0xa7, 0x65,
+	  0xe4, 0x8c, 0x32, 0x09, 0x65, 0x4f, 0xe9, 0x00,
+	  0x89, 0xce, 0x44, 0xff, 0x56, 0x18, 0x47, 0x7e,
+	  0x39, 0xab, 0x28, 0x64, 0x76, 0xdf, 0x05, 0x2b },
+	{ 0xe6, 0x9b, 0x3a, 0x36, 0xa4, 0x46, 0x19, 0x12,
+	  0xdc, 0x08, 0x34, 0x6b, 0x11, 0xdd, 0xcb, 0x9d,
+	  0xb7, 0x96, 0xf8, 0x85, 0xfd, 0x01, 0x93, 0x6e,
+	  0x66, 0x2f, 0xe2, 0x92, 0x97, 0xb0, 0x99, 0xa4 },
+	{ 0x5a, 0xc6, 0x50, 0x3b, 0x0d, 0x8d, 0xa6, 0x91,
+	  0x76, 0x46, 0xe6, 0xdc, 0xc8, 0x7e, 0xdc, 0x58,
+	  0xe9, 0x42, 0x45, 0x32, 0x4c, 0xc2, 0x04, 0xf4,
+	  0xdd, 0x4a, 0xf0, 0x15, 0x63, 0xac, 0xd4, 0x27 },
+	{ 0xdf, 0x6d, 0xda, 0x21, 0x35, 0x9a, 0x30, 0xbc,
+	  0x27, 0x17, 0x80, 0x97, 0x1c, 0x1a, 0xbd, 0x56,
+	  0xa6, 0xef, 0x16, 0x7e, 0x48, 0x08, 0x87, 0x88,
+	  0x8e, 0x73, 0xa8, 0x6d, 0x3b, 0xf6, 0x05, 0xe9 },
+	{ 0xe8, 0xe6, 0xe4, 0x70, 0x71, 0xe7, 0xb7, 0xdf,
+	  0x25, 0x80, 0xf2, 0x25, 0xcf, 0xbb, 0xed, 0xf8,
+	  0x4c, 0xe6, 0x77, 0x46, 0x62, 0x66, 0x28, 0xd3,
+	  0x30, 0x97, 0xe4, 0xb7, 0xdc, 0x57, 0x11, 0x07 },
+	{ 0x53, 0xe4, 0x0e, 0xad, 0x62, 0x05, 0x1e, 0x19,
+	  0xcb, 0x9b, 0xa8, 0x13, 0x3e, 0x3e, 0x5c, 0x1c,
+	  0xe0, 0x0d, 0xdc, 0xad, 0x8a, 0xcf, 0x34, 0x2a,
+	  0x22, 0x43, 0x60, 0xb0, 0xac, 0xc1, 0x47, 0x77 },
+	{ 0x9c, 0xcd, 0x53, 0xfe, 0x80, 0xbe, 0x78, 0x6a,
+	  0xa9, 0x84, 0x63, 0x84, 0x62, 0xfb, 0x28, 0xaf,
+	  0xdf, 0x12, 0x2b, 0x34, 0xd7, 0x8f, 0x46, 0x87,
+	  0xec, 0x63, 0x2b, 0xb1, 0x9d, 0xe2, 0x37, 0x1a },
+	{ 0xcb, 0xd4, 0x80, 0x52, 0xc4, 0x8d, 0x78, 0x84,
+	  0x66, 0xa3, 0xe8, 0x11, 0x8c, 0x56, 0xc9, 0x7f,
+	  0xe1, 0x46, 0xe5, 0x54, 0x6f, 0xaa, 0xf9, 0x3e,
+	  0x2b, 0xc3, 0xc4, 0x7e, 0x45, 0x93, 0x97, 0x53 },
+	{ 0x25, 0x68, 0x83, 0xb1, 0x4e, 0x2a, 0xf4, 0x4d,
+	  0xad, 0xb2, 0x8e, 0x1b, 0x34, 0xb2, 0xac, 0x0f,
+	  0x0f, 0x4c, 0x91, 0xc3, 0x4e, 0xc9, 0x16, 0x9e,
+	  0x29, 0x03, 0x61, 0x58, 0xac, 0xaa, 0x95, 0xb9 },
+	{ 0x44, 0x71, 0xb9, 0x1a, 0xb4, 0x2d, 0xb7, 0xc4,
+	  0xdd, 0x84, 0x90, 0xab, 0x95, 0xa2, 0xee, 0x8d,
+	  0x04, 0xe3, 0xef, 0x5c, 0x3d, 0x6f, 0xc7, 0x1a,
+	  0xc7, 0x4b, 0x2b, 0x26, 0x91, 0x4d, 0x16, 0x41 },
+	{ 0xa5, 0xeb, 0x08, 0x03, 0x8f, 0x8f, 0x11, 0x55,
+	  0xed, 0x86, 0xe6, 0x31, 0x90, 0x6f, 0xc1, 0x30,
+	  0x95, 0xf6, 0xbb, 0xa4, 0x1d, 0xe5, 0xd4, 0xe7,
+	  0x95, 0x75, 0x8e, 0xc8, 0xc8, 0xdf, 0x8a, 0xf1 },
+	{ 0xdc, 0x1d, 0xb6, 0x4e, 0xd8, 0xb4, 0x8a, 0x91,
+	  0x0e, 0x06, 0x0a, 0x6b, 0x86, 0x63, 0x74, 0xc5,
+	  0x78, 0x78, 0x4e, 0x9a, 0xc4, 0x9a, 0xb2, 0x77,
+	  0x40, 0x92, 0xac, 0x71, 0x50, 0x19, 0x34, 0xac },
+	{ 0x28, 0x54, 0x13, 0xb2, 0xf2, 0xee, 0x87, 0x3d,
+	  0x34, 0x31, 0x9e, 0xe0, 0xbb, 0xfb, 0xb9, 0x0f,
+	  0x32, 0xda, 0x43, 0x4c, 0xc8, 0x7e, 0x3d, 0xb5,
+	  0xed, 0x12, 0x1b, 0xb3, 0x98, 0xed, 0x96, 0x4b },
+	{ 0x02, 0x16, 0xe0, 0xf8, 0x1f, 0x75, 0x0f, 0x26,
+	  0xf1, 0x99, 0x8b, 0xc3, 0x93, 0x4e, 0x3e, 0x12,
+	  0x4c, 0x99, 0x45, 0xe6, 0x85, 0xa6, 0x0b, 0x25,
+	  0xe8, 0xfb, 0xd9, 0x62, 0x5a, 0xb6, 0xb5, 0x99 },
+	{ 0x38, 0xc4, 0x10, 0xf5, 0xb9, 0xd4, 0x07, 0x20,
+	  0x50, 0x75, 0x5b, 0x31, 0xdc, 0xa8, 0x9f, 0xd5,
+	  0x39, 0x5c, 0x67, 0x85, 0xee, 0xb3, 0xd7, 0x90,
+	  0xf3, 0x20, 0xff, 0x94, 0x1c, 0x5a, 0x93, 0xbf },
+	{ 0xf1, 0x84, 0x17, 0xb3, 0x9d, 0x61, 0x7a, 0xb1,
+	  0xc1, 0x8f, 0xdf, 0x91, 0xeb, 0xd0, 0xfc, 0x6d,
+	  0x55, 0x16, 0xbb, 0x34, 0xcf, 0x39, 0x36, 0x40,
+	  0x37, 0xbc, 0xe8, 0x1f, 0xa0, 0x4c, 0xec, 0xb1 },
+	{ 0x1f, 0xa8, 0x77, 0xde, 0x67, 0x25, 0x9d, 0x19,
+	  0x86, 0x3a, 0x2a, 0x34, 0xbc, 0xc6, 0x96, 0x2a,
+	  0x2b, 0x25, 0xfc, 0xbf, 0x5c, 0xbe, 0xcd, 0x7e,
+	  0xde, 0x8f, 0x1f, 0xa3, 0x66, 0x88, 0xa7, 0x96 },
+	{ 0x5b, 0xd1, 0x69, 0xe6, 0x7c, 0x82, 0xc2, 0xc2,
+	  0xe9, 0x8e, 0xf7, 0x00, 0x8b, 0xdf, 0x26, 0x1f,
+	  0x2d, 0xdf, 0x30, 0xb1, 0xc0, 0x0f, 0x9e, 0x7f,
+	  0x27, 0x5b, 0xb3, 0xe8, 0xa2, 0x8d, 0xc9, 0xa2 },
+	{ 0xc8, 0x0a, 0xbe, 0xeb, 0xb6, 0x69, 0xad, 0x5d,
+	  0xee, 0xb5, 0xf5, 0xec, 0x8e, 0xa6, 0xb7, 0xa0,
+	  0x5d, 0xdf, 0x7d, 0x31, 0xec, 0x4c, 0x0a, 0x2e,
+	  0xe2, 0x0b, 0x0b, 0x98, 0xca, 0xec, 0x67, 0x46 },
+	{ 0xe7, 0x6d, 0x3f, 0xbd, 0xa5, 0xba, 0x37, 0x4e,
+	  0x6b, 0xf8, 0xe5, 0x0f, 0xad, 0xc3, 0xbb, 0xb9,
+	  0xba, 0x5c, 0x20, 0x6e, 0xbd, 0xec, 0x89, 0xa3,
+	  0xa5, 0x4c, 0xf3, 0xdd, 0x84, 0xa0, 0x70, 0x16 },
+	{ 0x7b, 0xba, 0x9d, 0xc5, 0xb5, 0xdb, 0x20, 0x71,
+	  0xd1, 0x77, 0x52, 0xb1, 0x04, 0x4c, 0x1e, 0xce,
+	  0xd9, 0x6a, 0xaf, 0x2d, 0xd4, 0x6e, 0x9b, 0x43,
+	  0x37, 0x50, 0xe8, 0xea, 0x0d, 0xcc, 0x18, 0x70 },
+	{ 0xf2, 0x9b, 0x1b, 0x1a, 0xb9, 0xba, 0xb1, 0x63,
+	  0x01, 0x8e, 0xe3, 0xda, 0x15, 0x23, 0x2c, 0xca,
+	  0x78, 0xec, 0x52, 0xdb, 0xc3, 0x4e, 0xda, 0x5b,
+	  0x82, 0x2e, 0xc1, 0xd8, 0x0f, 0xc2, 0x1b, 0xd0 },
+	{ 0x9e, 0xe3, 0xe3, 0xe7, 0xe9, 0x00, 0xf1, 0xe1,
+	  0x1d, 0x30, 0x8c, 0x4b, 0x2b, 0x30, 0x76, 0xd2,
+	  0x72, 0xcf, 0x70, 0x12, 0x4f, 0x9f, 0x51, 0xe1,
+	  0xda, 0x60, 0xf3, 0x78, 0x46, 0xcd, 0xd2, 0xf4 },
+	{ 0x70, 0xea, 0x3b, 0x01, 0x76, 0x92, 0x7d, 0x90,
+	  0x96, 0xa1, 0x85, 0x08, 0xcd, 0x12, 0x3a, 0x29,
+	  0x03, 0x25, 0x92, 0x0a, 0x9d, 0x00, 0xa8, 0x9b,
+	  0x5d, 0xe0, 0x42, 0x73, 0xfb, 0xc7, 0x6b, 0x85 },
+	{ 0x67, 0xde, 0x25, 0xc0, 0x2a, 0x4a, 0xab, 0xa2,
+	  0x3b, 0xdc, 0x97, 0x3c, 0x8b, 0xb0, 0xb5, 0x79,
+	  0x6d, 0x47, 0xcc, 0x06, 0x59, 0xd4, 0x3d, 0xff,
+	  0x1f, 0x97, 0xde, 0x17, 0x49, 0x63, 0xb6, 0x8e },
+	{ 0xb2, 0x16, 0x8e, 0x4e, 0x0f, 0x18, 0xb0, 0xe6,
+	  0x41, 0x00, 0xb5, 0x17, 0xed, 0x95, 0x25, 0x7d,
+	  0x73, 0xf0, 0x62, 0x0d, 0xf8, 0x85, 0xc1, 0x3d,
+	  0x2e, 0xcf, 0x79, 0x36, 0x7b, 0x38, 0x4c, 0xee },
+	{ 0x2e, 0x7d, 0xec, 0x24, 0x28, 0x85, 0x3b, 0x2c,
+	  0x71, 0x76, 0x07, 0x45, 0x54, 0x1f, 0x7a, 0xfe,
+	  0x98, 0x25, 0xb5, 0xdd, 0x77, 0xdf, 0x06, 0x51,
+	  0x1d, 0x84, 0x41, 0xa9, 0x4b, 0xac, 0xc9, 0x27 },
+	{ 0xca, 0x9f, 0xfa, 0xc4, 0xc4, 0x3f, 0x0b, 0x48,
+	  0x46, 0x1d, 0xc5, 0xc2, 0x63, 0xbe, 0xa3, 0xf6,
+	  0xf0, 0x06, 0x11, 0xce, 0xac, 0xab, 0xf6, 0xf8,
+	  0x95, 0xba, 0x2b, 0x01, 0x01, 0xdb, 0xb6, 0x8d },
+	{ 0x74, 0x10, 0xd4, 0x2d, 0x8f, 0xd1, 0xd5, 0xe9,
+	  0xd2, 0xf5, 0x81, 0x5c, 0xb9, 0x34, 0x17, 0x99,
+	  0x88, 0x28, 0xef, 0x3c, 0x42, 0x30, 0xbf, 0xbd,
+	  0x41, 0x2d, 0xf0, 0xa4, 0xa7, 0xa2, 0x50, 0x7a },
+	{ 0x50, 0x10, 0xf6, 0x84, 0x51, 0x6d, 0xcc, 0xd0,
+	  0xb6, 0xee, 0x08, 0x52, 0xc2, 0x51, 0x2b, 0x4d,
+	  0xc0, 0x06, 0x6c, 0xf0, 0xd5, 0x6f, 0x35, 0x30,
+	  0x29, 0x78, 0xdb, 0x8a, 0xe3, 0x2c, 0x6a, 0x81 },
+	{ 0xac, 0xaa, 0xb5, 0x85, 0xf7, 0xb7, 0x9b, 0x71,
+	  0x99, 0x35, 0xce, 0xb8, 0x95, 0x23, 0xdd, 0xc5,
+	  0x48, 0x27, 0xf7, 0x5c, 0x56, 0x88, 0x38, 0x56,
+	  0x15, 0x4a, 0x56, 0xcd, 0xcd, 0x5e, 0xe9, 0x88 },
+	{ 0x66, 0x6d, 0xe5, 0xd1, 0x44, 0x0f, 0xee, 0x73,
+	  0x31, 0xaa, 0xf0, 0x12, 0x3a, 0x62, 0xef, 0x2d,
+	  0x8b, 0xa5, 0x74, 0x53, 0xa0, 0x76, 0x96, 0x35,
+	  0xac, 0x6c, 0xd0, 0x1e, 0x63, 0x3f, 0x77, 0x12 },
+	{ 0xa6, 0xf9, 0x86, 0x58, 0xf6, 0xea, 0xba, 0xf9,
+	  0x02, 0xd8, 0xb3, 0x87, 0x1a, 0x4b, 0x10, 0x1d,
+	  0x16, 0x19, 0x6e, 0x8a, 0x4b, 0x24, 0x1e, 0x15,
+	  0x58, 0xfe, 0x29, 0x96, 0x6e, 0x10, 0x3e, 0x8d },
+	{ 0x89, 0x15, 0x46, 0xa8, 0xb2, 0x9f, 0x30, 0x47,
+	  0xdd, 0xcf, 0xe5, 0xb0, 0x0e, 0x45, 0xfd, 0x55,
+	  0x75, 0x63, 0x73, 0x10, 0x5e, 0xa8, 0x63, 0x7d,
+	  0xfc, 0xff, 0x54, 0x7b, 0x6e, 0xa9, 0x53, 0x5f },
+	{ 0x18, 0xdf, 0xbc, 0x1a, 0xc5, 0xd2, 0x5b, 0x07,
+	  0x61, 0x13, 0x7d, 0xbd, 0x22, 0xc1, 0x7c, 0x82,
+	  0x9d, 0x0f, 0x0e, 0xf1, 0xd8, 0x23, 0x44, 0xe9,
+	  0xc8, 0x9c, 0x28, 0x66, 0x94, 0xda, 0x24, 0xe8 },
+	{ 0xb5, 0x4b, 0x9b, 0x67, 0xf8, 0xfe, 0xd5, 0x4b,
+	  0xbf, 0x5a, 0x26, 0x66, 0xdb, 0xdf, 0x4b, 0x23,
+	  0xcf, 0xf1, 0xd1, 0xb6, 0xf4, 0xaf, 0xc9, 0x85,
+	  0xb2, 0xe6, 0xd3, 0x30, 0x5a, 0x9f, 0xf8, 0x0f },
+	{ 0x7d, 0xb4, 0x42, 0xe1, 0x32, 0xba, 0x59, 0xbc,
+	  0x12, 0x89, 0xaa, 0x98, 0xb0, 0xd3, 0xe8, 0x06,
+	  0x00, 0x4f, 0x8e, 0xc1, 0x28, 0x11, 0xaf, 0x1e,
+	  0x2e, 0x33, 0xc6, 0x9b, 0xfd, 0xe7, 0x29, 0xe1 },
+	{ 0x25, 0x0f, 0x37, 0xcd, 0xc1, 0x5e, 0x81, 0x7d,
+	  0x2f, 0x16, 0x0d, 0x99, 0x56, 0xc7, 0x1f, 0xe3,
+	  0xeb, 0x5d, 0xb7, 0x45, 0x56, 0xe4, 0xad, 0xf9,
+	  0xa4, 0xff, 0xaf, 0xba, 0x74, 0x01, 0x03, 0x96 },
+	{ 0x4a, 0xb8, 0xa3, 0xdd, 0x1d, 0xdf, 0x8a, 0xd4,
+	  0x3d, 0xab, 0x13, 0xa2, 0x7f, 0x66, 0xa6, 0x54,
+	  0x4f, 0x29, 0x05, 0x97, 0xfa, 0x96, 0x04, 0x0e,
+	  0x0e, 0x1d, 0xb9, 0x26, 0x3a, 0xa4, 0x79, 0xf8 },
+	{ 0xee, 0x61, 0x72, 0x7a, 0x07, 0x66, 0xdf, 0x93,
+	  0x9c, 0xcd, 0xc8, 0x60, 0x33, 0x40, 0x44, 0xc7,
+	  0x9a, 0x3c, 0x9b, 0x15, 0x62, 0x00, 0xbc, 0x3a,
+	  0xa3, 0x29, 0x73, 0x48, 0x3d, 0x83, 0x41, 0xae },
+	{ 0x3f, 0x68, 0xc7, 0xec, 0x63, 0xac, 0x11, 0xeb,
+	  0xb9, 0x8f, 0x94, 0xb3, 0x39, 0xb0, 0x5c, 0x10,
+	  0x49, 0x84, 0xfd, 0xa5, 0x01, 0x03, 0x06, 0x01,
+	  0x44, 0xe5, 0xa2, 0xbf, 0xcc, 0xc9, 0xda, 0x95 },
+	{ 0x05, 0x6f, 0x29, 0x81, 0x6b, 0x8a, 0xf8, 0xf5,
+	  0x66, 0x82, 0xbc, 0x4d, 0x7c, 0xf0, 0x94, 0x11,
+	  0x1d, 0xa7, 0x73, 0x3e, 0x72, 0x6c, 0xd1, 0x3d,
+	  0x6b, 0x3e, 0x8e, 0xa0, 0x3e, 0x92, 0xa0, 0xd5 },
+	{ 0xf5, 0xec, 0x43, 0xa2, 0x8a, 0xcb, 0xef, 0xf1,
+	  0xf3, 0x31, 0x8a, 0x5b, 0xca, 0xc7, 0xc6, 0x6d,
+	  0xdb, 0x52, 0x30, 0xb7, 0x9d, 0xb2, 0xd1, 0x05,
+	  0xbc, 0xbe, 0x15, 0xf3, 0xc1, 0x14, 0x8d, 0x69 },
+	{ 0x2a, 0x69, 0x60, 0xad, 0x1d, 0x8d, 0xd5, 0x47,
+	  0x55, 0x5c, 0xfb, 0xd5, 0xe4, 0x60, 0x0f, 0x1e,
+	  0xaa, 0x1c, 0x8e, 0xda, 0x34, 0xde, 0x03, 0x74,
+	  0xec, 0x4a, 0x26, 0xea, 0xaa, 0xa3, 0x3b, 0x4e },
+	{ 0xdc, 0xc1, 0xea, 0x7b, 0xaa, 0xb9, 0x33, 0x84,
+	  0xf7, 0x6b, 0x79, 0x68, 0x66, 0x19, 0x97, 0x54,
+	  0x74, 0x2f, 0x7b, 0x96, 0xd6, 0xb4, 0xc1, 0x20,
+	  0x16, 0x5c, 0x04, 0xa6, 0xc4, 0xf5, 0xce, 0x10 },
+	{ 0x13, 0xd5, 0xdf, 0x17, 0x92, 0x21, 0x37, 0x9c,
+	  0x6a, 0x78, 0xc0, 0x7c, 0x79, 0x3f, 0xf5, 0x34,
+	  0x87, 0xca, 0xe6, 0xbf, 0x9f, 0xe8, 0x82, 0x54,
+	  0x1a, 0xb0, 0xe7, 0x35, 0xe3, 0xea, 0xda, 0x3b },
+	{ 0x8c, 0x59, 0xe4, 0x40, 0x76, 0x41, 0xa0, 0x1e,
+	  0x8f, 0xf9, 0x1f, 0x99, 0x80, 0xdc, 0x23, 0x6f,
+	  0x4e, 0xcd, 0x6f, 0xcf, 0x52, 0x58, 0x9a, 0x09,
+	  0x9a, 0x96, 0x16, 0x33, 0x96, 0x77, 0x14, 0xe1 },
+	{ 0x83, 0x3b, 0x1a, 0xc6, 0xa2, 0x51, 0xfd, 0x08,
+	  0xfd, 0x6d, 0x90, 0x8f, 0xea, 0x2a, 0x4e, 0xe1,
+	  0xe0, 0x40, 0xbc, 0xa9, 0x3f, 0xc1, 0xa3, 0x8e,
+	  0xc3, 0x82, 0x0e, 0x0c, 0x10, 0xbd, 0x82, 0xea },
+	{ 0xa2, 0x44, 0xf9, 0x27, 0xf3, 0xb4, 0x0b, 0x8f,
+	  0x6c, 0x39, 0x15, 0x70, 0xc7, 0x65, 0x41, 0x8f,
+	  0x2f, 0x6e, 0x70, 0x8e, 0xac, 0x90, 0x06, 0xc5,
+	  0x1a, 0x7f, 0xef, 0xf4, 0xaf, 0x3b, 0x2b, 0x9e },
+	{ 0x3d, 0x99, 0xed, 0x95, 0x50, 0xcf, 0x11, 0x96,
+	  0xe6, 0xc4, 0xd2, 0x0c, 0x25, 0x96, 0x20, 0xf8,
+	  0x58, 0xc3, 0xd7, 0x03, 0x37, 0x4c, 0x12, 0x8c,
+	  0xe7, 0xb5, 0x90, 0x31, 0x0c, 0x83, 0x04, 0x6d },
+	{ 0x2b, 0x35, 0xc4, 0x7d, 0x7b, 0x87, 0x76, 0x1f,
+	  0x0a, 0xe4, 0x3a, 0xc5, 0x6a, 0xc2, 0x7b, 0x9f,
+	  0x25, 0x83, 0x03, 0x67, 0xb5, 0x95, 0xbe, 0x8c,
+	  0x24, 0x0e, 0x94, 0x60, 0x0c, 0x6e, 0x33, 0x12 },
+	{ 0x5d, 0x11, 0xed, 0x37, 0xd2, 0x4d, 0xc7, 0x67,
+	  0x30, 0x5c, 0xb7, 0xe1, 0x46, 0x7d, 0x87, 0xc0,
+	  0x65, 0xac, 0x4b, 0xc8, 0xa4, 0x26, 0xde, 0x38,
+	  0x99, 0x1f, 0xf5, 0x9a, 0xa8, 0x73, 0x5d, 0x02 },
+	{ 0xb8, 0x36, 0x47, 0x8e, 0x1c, 0xa0, 0x64, 0x0d,
+	  0xce, 0x6f, 0xd9, 0x10, 0xa5, 0x09, 0x62, 0x72,
+	  0xc8, 0x33, 0x09, 0x90, 0xcd, 0x97, 0x86, 0x4a,
+	  0xc2, 0xbf, 0x14, 0xef, 0x6b, 0x23, 0x91, 0x4a },
+	{ 0x91, 0x00, 0xf9, 0x46, 0xd6, 0xcc, 0xde, 0x3a,
+	  0x59, 0x7f, 0x90, 0xd3, 0x9f, 0xc1, 0x21, 0x5b,
+	  0xad, 0xdc, 0x74, 0x13, 0x64, 0x3d, 0x85, 0xc2,
+	  0x1c, 0x3e, 0xee, 0x5d, 0x2d, 0xd3, 0x28, 0x94 },
+	{ 0xda, 0x70, 0xee, 0xdd, 0x23, 0xe6, 0x63, 0xaa,
+	  0x1a, 0x74, 0xb9, 0x76, 0x69, 0x35, 0xb4, 0x79,
+	  0x22, 0x2a, 0x72, 0xaf, 0xba, 0x5c, 0x79, 0x51,
+	  0x58, 0xda, 0xd4, 0x1a, 0x3b, 0xd7, 0x7e, 0x40 },
+	{ 0xf0, 0x67, 0xed, 0x6a, 0x0d, 0xbd, 0x43, 0xaa,
+	  0x0a, 0x92, 0x54, 0xe6, 0x9f, 0xd6, 0x6b, 0xdd,
+	  0x8a, 0xcb, 0x87, 0xde, 0x93, 0x6c, 0x25, 0x8c,
+	  0xfb, 0x02, 0x28, 0x5f, 0x2c, 0x11, 0xfa, 0x79 },
+	{ 0x71, 0x5c, 0x99, 0xc7, 0xd5, 0x75, 0x80, 0xcf,
+	  0x97, 0x53, 0xb4, 0xc1, 0xd7, 0x95, 0xe4, 0x5a,
+	  0x83, 0xfb, 0xb2, 0x28, 0xc0, 0xd3, 0x6f, 0xbe,
+	  0x20, 0xfa, 0xf3, 0x9b, 0xdd, 0x6d, 0x4e, 0x85 },
+	{ 0xe4, 0x57, 0xd6, 0xad, 0x1e, 0x67, 0xcb, 0x9b,
+	  0xbd, 0x17, 0xcb, 0xd6, 0x98, 0xfa, 0x6d, 0x7d,
+	  0xae, 0x0c, 0x9b, 0x7a, 0xd6, 0xcb, 0xd6, 0x53,
+	  0x96, 0x34, 0xe3, 0x2a, 0x71, 0x9c, 0x84, 0x92 },
+	{ 0xec, 0xe3, 0xea, 0x81, 0x03, 0xe0, 0x24, 0x83,
+	  0xc6, 0x4a, 0x70, 0xa4, 0xbd, 0xce, 0xe8, 0xce,
+	  0xb6, 0x27, 0x8f, 0x25, 0x33, 0xf3, 0xf4, 0x8d,
+	  0xbe, 0xed, 0xfb, 0xa9, 0x45, 0x31, 0xd4, 0xae },
+	{ 0x38, 0x8a, 0xa5, 0xd3, 0x66, 0x7a, 0x97, 0xc6,
+	  0x8d, 0x3d, 0x56, 0xf8, 0xf3, 0xee, 0x8d, 0x3d,
+	  0x36, 0x09, 0x1f, 0x17, 0xfe, 0x5d, 0x1b, 0x0d,
+	  0x5d, 0x84, 0xc9, 0x3b, 0x2f, 0xfe, 0x40, 0xbd },
+	{ 0x8b, 0x6b, 0x31, 0xb9, 0xad, 0x7c, 0x3d, 0x5c,
+	  0xd8, 0x4b, 0xf9, 0x89, 0x47, 0xb9, 0xcd, 0xb5,
+	  0x9d, 0xf8, 0xa2, 0x5f, 0xf7, 0x38, 0x10, 0x10,
+	  0x13, 0xbe, 0x4f, 0xd6, 0x5e, 0x1d, 0xd1, 0xa3 },
+	{ 0x06, 0x62, 0x91, 0xf6, 0xbb, 0xd2, 0x5f, 0x3c,
+	  0x85, 0x3d, 0xb7, 0xd8, 0xb9, 0x5c, 0x9a, 0x1c,
+	  0xfb, 0x9b, 0xf1, 0xc1, 0xc9, 0x9f, 0xb9, 0x5a,
+	  0x9b, 0x78, 0x69, 0xd9, 0x0f, 0x1c, 0x29, 0x03 },
+	{ 0xa7, 0x07, 0xef, 0xbc, 0xcd, 0xce, 0xed, 0x42,
+	  0x96, 0x7a, 0x66, 0xf5, 0x53, 0x9b, 0x93, 0xed,
+	  0x75, 0x60, 0xd4, 0x67, 0x30, 0x40, 0x16, 0xc4,
+	  0x78, 0x0d, 0x77, 0x55, 0xa5, 0x65, 0xd4, 0xc4 },
+	{ 0x38, 0xc5, 0x3d, 0xfb, 0x70, 0xbe, 0x7e, 0x79,
+	  0x2b, 0x07, 0xa6, 0xa3, 0x5b, 0x8a, 0x6a, 0x0a,
+	  0xba, 0x02, 0xc5, 0xc5, 0xf3, 0x8b, 0xaf, 0x5c,
+	  0x82, 0x3f, 0xdf, 0xd9, 0xe4, 0x2d, 0x65, 0x7e },
+	{ 0xf2, 0x91, 0x13, 0x86, 0x50, 0x1d, 0x9a, 0xb9,
+	  0xd7, 0x20, 0xcf, 0x8a, 0xd1, 0x05, 0x03, 0xd5,
+	  0x63, 0x4b, 0xf4, 0xb7, 0xd1, 0x2b, 0x56, 0xdf,
+	  0xb7, 0x4f, 0xec, 0xc6, 0xe4, 0x09, 0x3f, 0x68 },
+	{ 0xc6, 0xf2, 0xbd, 0xd5, 0x2b, 0x81, 0xe6, 0xe4,
+	  0xf6, 0x59, 0x5a, 0xbd, 0x4d, 0x7f, 0xb3, 0x1f,
+	  0x65, 0x11, 0x69, 0xd0, 0x0f, 0xf3, 0x26, 0x92,
+	  0x6b, 0x34, 0x94, 0x7b, 0x28, 0xa8, 0x39, 0x59 },
+	{ 0x29, 0x3d, 0x94, 0xb1, 0x8c, 0x98, 0xbb, 0x32,
+	  0x23, 0x36, 0x6b, 0x8c, 0xe7, 0x4c, 0x28, 0xfb,
+	  0xdf, 0x28, 0xe1, 0xf8, 0x4a, 0x33, 0x50, 0xb0,
+	  0xeb, 0x2d, 0x18, 0x04, 0xa5, 0x77, 0x57, 0x9b },
+	{ 0x2c, 0x2f, 0xa5, 0xc0, 0xb5, 0x15, 0x33, 0x16,
+	  0x5b, 0xc3, 0x75, 0xc2, 0x2e, 0x27, 0x81, 0x76,
+	  0x82, 0x70, 0xa3, 0x83, 0x98, 0x5d, 0x13, 0xbd,
+	  0x6b, 0x67, 0xb6, 0xfd, 0x67, 0xf8, 0x89, 0xeb },
+	{ 0xca, 0xa0, 0x9b, 0x82, 0xb7, 0x25, 0x62, 0xe4,
+	  0x3f, 0x4b, 0x22, 0x75, 0xc0, 0x91, 0x91, 0x8e,
+	  0x62, 0x4d, 0x91, 0x16, 0x61, 0xcc, 0x81, 0x1b,
+	  0xb5, 0xfa, 0xec, 0x51, 0xf6, 0x08, 0x8e, 0xf7 },
+	{ 0x24, 0x76, 0x1e, 0x45, 0xe6, 0x74, 0x39, 0x53,
+	  0x79, 0xfb, 0x17, 0x72, 0x9c, 0x78, 0xcb, 0x93,
+	  0x9e, 0x6f, 0x74, 0xc5, 0xdf, 0xfb, 0x9c, 0x96,
+	  0x1f, 0x49, 0x59, 0x82, 0xc3, 0xed, 0x1f, 0xe3 },
+	{ 0x55, 0xb7, 0x0a, 0x82, 0x13, 0x1e, 0xc9, 0x48,
+	  0x88, 0xd7, 0xab, 0x54, 0xa7, 0xc5, 0x15, 0x25,
+	  0x5c, 0x39, 0x38, 0xbb, 0x10, 0xbc, 0x78, 0x4d,
+	  0xc9, 0xb6, 0x7f, 0x07, 0x6e, 0x34, 0x1a, 0x73 },
+	{ 0x6a, 0xb9, 0x05, 0x7b, 0x97, 0x7e, 0xbc, 0x3c,
+	  0xa4, 0xd4, 0xce, 0x74, 0x50, 0x6c, 0x25, 0xcc,
+	  0xcd, 0xc5, 0x66, 0x49, 0x7c, 0x45, 0x0b, 0x54,
+	  0x15, 0xa3, 0x94, 0x86, 0xf8, 0x65, 0x7a, 0x03 },
+	{ 0x24, 0x06, 0x6d, 0xee, 0xe0, 0xec, 0xee, 0x15,
+	  0xa4, 0x5f, 0x0a, 0x32, 0x6d, 0x0f, 0x8d, 0xbc,
+	  0x79, 0x76, 0x1e, 0xbb, 0x93, 0xcf, 0x8c, 0x03,
+	  0x77, 0xaf, 0x44, 0x09, 0x78, 0xfc, 0xf9, 0x94 },
+	{ 0x20, 0x00, 0x0d, 0x3f, 0x66, 0xba, 0x76, 0x86,
+	  0x0d, 0x5a, 0x95, 0x06, 0x88, 0xb9, 0xaa, 0x0d,
+	  0x76, 0xcf, 0xea, 0x59, 0xb0, 0x05, 0xd8, 0x59,
+	  0x91, 0x4b, 0x1a, 0x46, 0x65, 0x3a, 0x93, 0x9b },
+	{ 0xb9, 0x2d, 0xaa, 0x79, 0x60, 0x3e, 0x3b, 0xdb,
+	  0xc3, 0xbf, 0xe0, 0xf4, 0x19, 0xe4, 0x09, 0xb2,
+	  0xea, 0x10, 0xdc, 0x43, 0x5b, 0xee, 0xfe, 0x29,
+	  0x59, 0xda, 0x16, 0x89, 0x5d, 0x5d, 0xca, 0x1c },
+	{ 0xe9, 0x47, 0x94, 0x87, 0x05, 0xb2, 0x06, 0xd5,
+	  0x72, 0xb0, 0xe8, 0xf6, 0x2f, 0x66, 0xa6, 0x55,
+	  0x1c, 0xbd, 0x6b, 0xc3, 0x05, 0xd2, 0x6c, 0xe7,
+	  0x53, 0x9a, 0x12, 0xf9, 0xaa, 0xdf, 0x75, 0x71 },
+	{ 0x3d, 0x67, 0xc1, 0xb3, 0xf9, 0xb2, 0x39, 0x10,
+	  0xe3, 0xd3, 0x5e, 0x6b, 0x0f, 0x2c, 0xcf, 0x44,
+	  0xa0, 0xb5, 0x40, 0xa4, 0x5c, 0x18, 0xba, 0x3c,
+	  0x36, 0x26, 0x4d, 0xd4, 0x8e, 0x96, 0xaf, 0x6a },
+	{ 0xc7, 0x55, 0x8b, 0xab, 0xda, 0x04, 0xbc, 0xcb,
+	  0x76, 0x4d, 0x0b, 0xbf, 0x33, 0x58, 0x42, 0x51,
+	  0x41, 0x90, 0x2d, 0x22, 0x39, 0x1d, 0x9f, 0x8c,
+	  0x59, 0x15, 0x9f, 0xec, 0x9e, 0x49, 0xb1, 0x51 },
+	{ 0x0b, 0x73, 0x2b, 0xb0, 0x35, 0x67, 0x5a, 0x50,
+	  0xff, 0x58, 0xf2, 0xc2, 0x42, 0xe4, 0x71, 0x0a,
+	  0xec, 0xe6, 0x46, 0x70, 0x07, 0x9c, 0x13, 0x04,
+	  0x4c, 0x79, 0xc9, 0xb7, 0x49, 0x1f, 0x70, 0x00 },
+	{ 0xd1, 0x20, 0xb5, 0xef, 0x6d, 0x57, 0xeb, 0xf0,
+	  0x6e, 0xaf, 0x96, 0xbc, 0x93, 0x3c, 0x96, 0x7b,
+	  0x16, 0xcb, 0xe6, 0xe2, 0xbf, 0x00, 0x74, 0x1c,
+	  0x30, 0xaa, 0x1c, 0x54, 0xba, 0x64, 0x80, 0x1f },
+	{ 0x58, 0xd2, 0x12, 0xad, 0x6f, 0x58, 0xae, 0xf0,
+	  0xf8, 0x01, 0x16, 0xb4, 0x41, 0xe5, 0x7f, 0x61,
+	  0x95, 0xbf, 0xef, 0x26, 0xb6, 0x14, 0x63, 0xed,
+	  0xec, 0x11, 0x83, 0xcd, 0xb0, 0x4f, 0xe7, 0x6d },
+	{ 0xb8, 0x83, 0x6f, 0x51, 0xd1, 0xe2, 0x9b, 0xdf,
+	  0xdb, 0xa3, 0x25, 0x56, 0x53, 0x60, 0x26, 0x8b,
+	  0x8f, 0xad, 0x62, 0x74, 0x73, 0xed, 0xec, 0xef,
+	  0x7e, 0xae, 0xfe, 0xe8, 0x37, 0xc7, 0x40, 0x03 },
+	{ 0xc5, 0x47, 0xa3, 0xc1, 0x24, 0xae, 0x56, 0x85,
+	  0xff, 0xa7, 0xb8, 0xed, 0xaf, 0x96, 0xec, 0x86,
+	  0xf8, 0xb2, 0xd0, 0xd5, 0x0c, 0xee, 0x8b, 0xe3,
+	  0xb1, 0xf0, 0xc7, 0x67, 0x63, 0x06, 0x9d, 0x9c },
+	{ 0x5d, 0x16, 0x8b, 0x76, 0x9a, 0x2f, 0x67, 0x85,
+	  0x3d, 0x62, 0x95, 0xf7, 0x56, 0x8b, 0xe4, 0x0b,
+	  0xb7, 0xa1, 0x6b, 0x8d, 0x65, 0xba, 0x87, 0x63,
+	  0x5d, 0x19, 0x78, 0xd2, 0xab, 0x11, 0xba, 0x2a },
+	{ 0xa2, 0xf6, 0x75, 0xdc, 0x73, 0x02, 0x63, 0x8c,
+	  0xb6, 0x02, 0x01, 0x06, 0x4c, 0xa5, 0x50, 0x77,
+	  0x71, 0x4d, 0x71, 0xfe, 0x09, 0x6a, 0x31, 0x5f,
+	  0x2f, 0xe7, 0x40, 0x12, 0x77, 0xca, 0xa5, 0xaf },
+	{ 0xc8, 0xaa, 0xb5, 0xcd, 0x01, 0x60, 0xae, 0x78,
+	  0xcd, 0x2e, 0x8a, 0xc5, 0xfb, 0x0e, 0x09, 0x3c,
+	  0xdb, 0x5c, 0x4b, 0x60, 0x52, 0xa0, 0xa9, 0x7b,
+	  0xb0, 0x42, 0x16, 0x82, 0x6f, 0xa7, 0xa4, 0x37 },
+	{ 0xff, 0x68, 0xca, 0x40, 0x35, 0xbf, 0xeb, 0x43,
+	  0xfb, 0xf1, 0x45, 0xfd, 0xdd, 0x5e, 0x43, 0xf1,
+	  0xce, 0xa5, 0x4f, 0x11, 0xf7, 0xbe, 0xe1, 0x30,
+	  0x58, 0xf0, 0x27, 0x32, 0x9a, 0x4a, 0x5f, 0xa4 },
+	{ 0x1d, 0x4e, 0x54, 0x87, 0xae, 0x3c, 0x74, 0x0f,
+	  0x2b, 0xa6, 0xe5, 0x41, 0xac, 0x91, 0xbc, 0x2b,
+	  0xfc, 0xd2, 0x99, 0x9c, 0x51, 0x8d, 0x80, 0x7b,
+	  0x42, 0x67, 0x48, 0x80, 0x3a, 0x35, 0x0f, 0xd4 },
+	{ 0x6d, 0x24, 0x4e, 0x1a, 0x06, 0xce, 0x4e, 0xf5,
+	  0x78, 0xdd, 0x0f, 0x63, 0xaf, 0xf0, 0x93, 0x67,
+	  0x06, 0x73, 0x51, 0x19, 0xca, 0x9c, 0x8d, 0x22,
+	  0xd8, 0x6c, 0x80, 0x14, 0x14, 0xab, 0x97, 0x41 },
+	{ 0xde, 0xcf, 0x73, 0x29, 0xdb, 0xcc, 0x82, 0x7b,
+	  0x8f, 0xc5, 0x24, 0xc9, 0x43, 0x1e, 0x89, 0x98,
+	  0x02, 0x9e, 0xce, 0x12, 0xce, 0x93, 0xb7, 0xb2,
+	  0xf3, 0xe7, 0x69, 0xa9, 0x41, 0xfb, 0x8c, 0xea },
+	{ 0x2f, 0xaf, 0xcc, 0x0f, 0x2e, 0x63, 0xcb, 0xd0,
+	  0x77, 0x55, 0xbe, 0x7b, 0x75, 0xec, 0xea, 0x0a,
+	  0xdf, 0xf9, 0xaa, 0x5e, 0xde, 0x2a, 0x52, 0xfd,
+	  0xab, 0x4d, 0xfd, 0x03, 0x74, 0xcd, 0x48, 0x3f },
+	{ 0xaa, 0x85, 0x01, 0x0d, 0xd4, 0x6a, 0x54, 0x6b,
+	  0x53, 0x5e, 0xf4, 0xcf, 0x5f, 0x07, 0xd6, 0x51,
+	  0x61, 0xe8, 0x98, 0x28, 0xf3, 0xa7, 0x7d, 0xb7,
+	  0xb9, 0xb5, 0x6f, 0x0d, 0xf5, 0x9a, 0xae, 0x45 },
+	{ 0x07, 0xe8, 0xe1, 0xee, 0x73, 0x2c, 0xb0, 0xd3,
+	  0x56, 0xc9, 0xc0, 0xd1, 0x06, 0x9c, 0x89, 0xd1,
+	  0x7a, 0xdf, 0x6a, 0x9a, 0x33, 0x4f, 0x74, 0x5e,
+	  0xc7, 0x86, 0x73, 0x32, 0x54, 0x8c, 0xa8, 0xe9 },
+	{ 0x0e, 0x01, 0xe8, 0x1c, 0xad, 0xa8, 0x16, 0x2b,
+	  0xfd, 0x5f, 0x8a, 0x8c, 0x81, 0x8a, 0x6c, 0x69,
+	  0xfe, 0xdf, 0x02, 0xce, 0xb5, 0x20, 0x85, 0x23,
+	  0xcb, 0xe5, 0x31, 0x3b, 0x89, 0xca, 0x10, 0x53 },
+	{ 0x6b, 0xb6, 0xc6, 0x47, 0x26, 0x55, 0x08, 0x43,
+	  0x99, 0x85, 0x2e, 0x00, 0x24, 0x9f, 0x8c, 0xb2,
+	  0x47, 0x89, 0x6d, 0x39, 0x2b, 0x02, 0xd7, 0x3b,
+	  0x7f, 0x0d, 0xd8, 0x18, 0xe1, 0xe2, 0x9b, 0x07 },
+	{ 0x42, 0xd4, 0x63, 0x6e, 0x20, 0x60, 0xf0, 0x8f,
+	  0x41, 0xc8, 0x82, 0xe7, 0x6b, 0x39, 0x6b, 0x11,
+	  0x2e, 0xf6, 0x27, 0xcc, 0x24, 0xc4, 0x3d, 0xd5,
+	  0xf8, 0x3a, 0x1d, 0x1a, 0x7e, 0xad, 0x71, 0x1a },
+	{ 0x48, 0x58, 0xc9, 0xa1, 0x88, 0xb0, 0x23, 0x4f,
+	  0xb9, 0xa8, 0xd4, 0x7d, 0x0b, 0x41, 0x33, 0x65,
+	  0x0a, 0x03, 0x0b, 0xd0, 0x61, 0x1b, 0x87, 0xc3,
+	  0x89, 0x2e, 0x94, 0x95, 0x1f, 0x8d, 0xf8, 0x52 },
+	{ 0x3f, 0xab, 0x3e, 0x36, 0x98, 0x8d, 0x44, 0x5a,
+	  0x51, 0xc8, 0x78, 0x3e, 0x53, 0x1b, 0xe3, 0xa0,
+	  0x2b, 0xe4, 0x0c, 0xd0, 0x47, 0x96, 0xcf, 0xb6,
+	  0x1d, 0x40, 0x34, 0x74, 0x42, 0xd3, 0xf7, 0x94 },
+	{ 0xeb, 0xab, 0xc4, 0x96, 0x36, 0xbd, 0x43, 0x3d,
+	  0x2e, 0xc8, 0xf0, 0xe5, 0x18, 0x73, 0x2e, 0xf8,
+	  0xfa, 0x21, 0xd4, 0xd0, 0x71, 0xcc, 0x3b, 0xc4,
+	  0x6c, 0xd7, 0x9f, 0xa3, 0x8a, 0x28, 0xb8, 0x10 },
+	{ 0xa1, 0xd0, 0x34, 0x35, 0x23, 0xb8, 0x93, 0xfc,
+	  0xa8, 0x4f, 0x47, 0xfe, 0xb4, 0xa6, 0x4d, 0x35,
+	  0x0a, 0x17, 0xd8, 0xee, 0xf5, 0x49, 0x7e, 0xce,
+	  0x69, 0x7d, 0x02, 0xd7, 0x91, 0x78, 0xb5, 0x91 },
+	{ 0x26, 0x2e, 0xbf, 0xd9, 0x13, 0x0b, 0x7d, 0x28,
+	  0x76, 0x0d, 0x08, 0xef, 0x8b, 0xfd, 0x3b, 0x86,
+	  0xcd, 0xd3, 0xb2, 0x11, 0x3d, 0x2c, 0xae, 0xf7,
+	  0xea, 0x95, 0x1a, 0x30, 0x3d, 0xfa, 0x38, 0x46 },
+	{ 0xf7, 0x61, 0x58, 0xed, 0xd5, 0x0a, 0x15, 0x4f,
+	  0xa7, 0x82, 0x03, 0xed, 0x23, 0x62, 0x93, 0x2f,
+	  0xcb, 0x82, 0x53, 0xaa, 0xe3, 0x78, 0x90, 0x3e,
+	  0xde, 0xd1, 0xe0, 0x3f, 0x70, 0x21, 0xa2, 0x57 },
+	{ 0x26, 0x17, 0x8e, 0x95, 0x0a, 0xc7, 0x22, 0xf6,
+	  0x7a, 0xe5, 0x6e, 0x57, 0x1b, 0x28, 0x4c, 0x02,
+	  0x07, 0x68, 0x4a, 0x63, 0x34, 0xa1, 0x77, 0x48,
+	  0xa9, 0x4d, 0x26, 0x0b, 0xc5, 0xf5, 0x52, 0x74 },
+	{ 0xc3, 0x78, 0xd1, 0xe4, 0x93, 0xb4, 0x0e, 0xf1,
+	  0x1f, 0xe6, 0xa1, 0x5d, 0x9c, 0x27, 0x37, 0xa3,
+	  0x78, 0x09, 0x63, 0x4c, 0x5a, 0xba, 0xd5, 0xb3,
+	  0x3d, 0x7e, 0x39, 0x3b, 0x4a, 0xe0, 0x5d, 0x03 },
+	{ 0x98, 0x4b, 0xd8, 0x37, 0x91, 0x01, 0xbe, 0x8f,
+	  0xd8, 0x06, 0x12, 0xd8, 0xea, 0x29, 0x59, 0xa7,
+	  0x86, 0x5e, 0xc9, 0x71, 0x85, 0x23, 0x55, 0x01,
+	  0x07, 0xae, 0x39, 0x38, 0xdf, 0x32, 0x01, 0x1b },
+	{ 0xc6, 0xf2, 0x5a, 0x81, 0x2a, 0x14, 0x48, 0x58,
+	  0xac, 0x5c, 0xed, 0x37, 0xa9, 0x3a, 0x9f, 0x47,
+	  0x59, 0xba, 0x0b, 0x1c, 0x0f, 0xdc, 0x43, 0x1d,
+	  0xce, 0x35, 0xf9, 0xec, 0x1f, 0x1f, 0x4a, 0x99 },
+	{ 0x92, 0x4c, 0x75, 0xc9, 0x44, 0x24, 0xff, 0x75,
+	  0xe7, 0x4b, 0x8b, 0x4e, 0x94, 0x35, 0x89, 0x58,
+	  0xb0, 0x27, 0xb1, 0x71, 0xdf, 0x5e, 0x57, 0x89,
+	  0x9a, 0xd0, 0xd4, 0xda, 0xc3, 0x73, 0x53, 0xb6 },
+	{ 0x0a, 0xf3, 0x58, 0x92, 0xa6, 0x3f, 0x45, 0x93,
+	  0x1f, 0x68, 0x46, 0xed, 0x19, 0x03, 0x61, 0xcd,
+	  0x07, 0x30, 0x89, 0xe0, 0x77, 0x16, 0x57, 0x14,
+	  0xb5, 0x0b, 0x81, 0xa2, 0xe3, 0xdd, 0x9b, 0xa1 },
+	{ 0xcc, 0x80, 0xce, 0xfb, 0x26, 0xc3, 0xb2, 0xb0,
+	  0xda, 0xef, 0x23, 0x3e, 0x60, 0x6d, 0x5f, 0xfc,
+	  0x80, 0xfa, 0x17, 0x42, 0x7d, 0x18, 0xe3, 0x04,
+	  0x89, 0x67, 0x3e, 0x06, 0xef, 0x4b, 0x87, 0xf7 },
+	{ 0xc2, 0xf8, 0xc8, 0x11, 0x74, 0x47, 0xf3, 0x97,
+	  0x8b, 0x08, 0x18, 0xdc, 0xf6, 0xf7, 0x01, 0x16,
+	  0xac, 0x56, 0xfd, 0x18, 0x4d, 0xd1, 0x27, 0x84,
+	  0x94, 0xe1, 0x03, 0xfc, 0x6d, 0x74, 0xa8, 0x87 },
+	{ 0xbd, 0xec, 0xf6, 0xbf, 0xc1, 0xba, 0x0d, 0xf6,
+	  0xe8, 0x62, 0xc8, 0x31, 0x99, 0x22, 0x07, 0x79,
+	  0x6a, 0xcc, 0x79, 0x79, 0x68, 0x35, 0x88, 0x28,
+	  0xc0, 0x6e, 0x7a, 0x51, 0xe0, 0x90, 0x09, 0x8f },
+	{ 0x24, 0xd1, 0xa2, 0x6e, 0x3d, 0xab, 0x02, 0xfe,
+	  0x45, 0x72, 0xd2, 0xaa, 0x7d, 0xbd, 0x3e, 0xc3,
+	  0x0f, 0x06, 0x93, 0xdb, 0x26, 0xf2, 0x73, 0xd0,
+	  0xab, 0x2c, 0xb0, 0xc1, 0x3b, 0x5e, 0x64, 0x51 },
+	{ 0xec, 0x56, 0xf5, 0x8b, 0x09, 0x29, 0x9a, 0x30,
+	  0x0b, 0x14, 0x05, 0x65, 0xd7, 0xd3, 0xe6, 0x87,
+	  0x82, 0xb6, 0xe2, 0xfb, 0xeb, 0x4b, 0x7e, 0xa9,
+	  0x7a, 0xc0, 0x57, 0x98, 0x90, 0x61, 0xdd, 0x3f },
+	{ 0x11, 0xa4, 0x37, 0xc1, 0xab, 0xa3, 0xc1, 0x19,
+	  0xdd, 0xfa, 0xb3, 0x1b, 0x3e, 0x8c, 0x84, 0x1d,
+	  0xee, 0xeb, 0x91, 0x3e, 0xf5, 0x7f, 0x7e, 0x48,
+	  0xf2, 0xc9, 0xcf, 0x5a, 0x28, 0xfa, 0x42, 0xbc },
+	{ 0x53, 0xc7, 0xe6, 0x11, 0x4b, 0x85, 0x0a, 0x2c,
+	  0xb4, 0x96, 0xc9, 0xb3, 0xc6, 0x9a, 0x62, 0x3e,
+	  0xae, 0xa2, 0xcb, 0x1d, 0x33, 0xdd, 0x81, 0x7e,
+	  0x47, 0x65, 0xed, 0xaa, 0x68, 0x23, 0xc2, 0x28 },
+	{ 0x15, 0x4c, 0x3e, 0x96, 0xfe, 0xe5, 0xdb, 0x14,
+	  0xf8, 0x77, 0x3e, 0x18, 0xaf, 0x14, 0x85, 0x79,
+	  0x13, 0x50, 0x9d, 0xa9, 0x99, 0xb4, 0x6c, 0xdd,
+	  0x3d, 0x4c, 0x16, 0x97, 0x60, 0xc8, 0x3a, 0xd2 },
+	{ 0x40, 0xb9, 0x91, 0x6f, 0x09, 0x3e, 0x02, 0x7a,
+	  0x87, 0x86, 0x64, 0x18, 0x18, 0x92, 0x06, 0x20,
+	  0x47, 0x2f, 0xbc, 0xf6, 0x8f, 0x70, 0x1d, 0x1b,
+	  0x68, 0x06, 0x32, 0xe6, 0x99, 0x6b, 0xde, 0xd3 },
+	{ 0x24, 0xc4, 0xcb, 0xba, 0x07, 0x11, 0x98, 0x31,
+	  0xa7, 0x26, 0xb0, 0x53, 0x05, 0xd9, 0x6d, 0xa0,
+	  0x2f, 0xf8, 0xb1, 0x48, 0xf0, 0xda, 0x44, 0x0f,
+	  0xe2, 0x33, 0xbc, 0xaa, 0x32, 0xc7, 0x2f, 0x6f },
+	{ 0x5d, 0x20, 0x15, 0x10, 0x25, 0x00, 0x20, 0xb7,
+	  0x83, 0x68, 0x96, 0x88, 0xab, 0xbf, 0x8e, 0xcf,
+	  0x25, 0x94, 0xa9, 0x6a, 0x08, 0xf2, 0xbf, 0xec,
+	  0x6c, 0xe0, 0x57, 0x44, 0x65, 0xdd, 0xed, 0x71 },
+	{ 0x04, 0x3b, 0x97, 0xe3, 0x36, 0xee, 0x6f, 0xdb,
+	  0xbe, 0x2b, 0x50, 0xf2, 0x2a, 0xf8, 0x32, 0x75,
+	  0xa4, 0x08, 0x48, 0x05, 0xd2, 0xd5, 0x64, 0x59,
+	  0x62, 0x45, 0x4b, 0x6c, 0x9b, 0x80, 0x53, 0xa0 },
+	{ 0x56, 0x48, 0x35, 0xcb, 0xae, 0xa7, 0x74, 0x94,
+	  0x85, 0x68, 0xbe, 0x36, 0xcf, 0x52, 0xfc, 0xdd,
+	  0x83, 0x93, 0x4e, 0xb0, 0xa2, 0x75, 0x12, 0xdb,
+	  0xe3, 0xe2, 0xdb, 0x47, 0xb9, 0xe6, 0x63, 0x5a },
+	{ 0xf2, 0x1c, 0x33, 0xf4, 0x7b, 0xde, 0x40, 0xa2,
+	  0xa1, 0x01, 0xc9, 0xcd, 0xe8, 0x02, 0x7a, 0xaf,
+	  0x61, 0xa3, 0x13, 0x7d, 0xe2, 0x42, 0x2b, 0x30,
+	  0x03, 0x5a, 0x04, 0xc2, 0x70, 0x89, 0x41, 0x83 },
+	{ 0x9d, 0xb0, 0xef, 0x74, 0xe6, 0x6c, 0xbb, 0x84,
+	  0x2e, 0xb0, 0xe0, 0x73, 0x43, 0xa0, 0x3c, 0x5c,
+	  0x56, 0x7e, 0x37, 0x2b, 0x3f, 0x23, 0xb9, 0x43,
+	  0xc7, 0x88, 0xa4, 0xf2, 0x50, 0xf6, 0x78, 0x91 },
+	{ 0xab, 0x8d, 0x08, 0x65, 0x5f, 0xf1, 0xd3, 0xfe,
+	  0x87, 0x58, 0xd5, 0x62, 0x23, 0x5f, 0xd2, 0x3e,
+	  0x7c, 0xf9, 0xdc, 0xaa, 0xd6, 0x58, 0x87, 0x2a,
+	  0x49, 0xe5, 0xd3, 0x18, 0x3b, 0x6c, 0xce, 0xbd },
+	{ 0x6f, 0x27, 0xf7, 0x7e, 0x7b, 0xcf, 0x46, 0xa1,
+	  0xe9, 0x63, 0xad, 0xe0, 0x30, 0x97, 0x33, 0x54,
+	  0x30, 0x31, 0xdc, 0xcd, 0xd4, 0x7c, 0xaa, 0xc1,
+	  0x74, 0xd7, 0xd2, 0x7c, 0xe8, 0x07, 0x7e, 0x8b },
+	{ 0xe3, 0xcd, 0x54, 0xda, 0x7e, 0x44, 0x4c, 0xaa,
+	  0x62, 0x07, 0x56, 0x95, 0x25, 0xa6, 0x70, 0xeb,
+	  0xae, 0x12, 0x78, 0xde, 0x4e, 0x3f, 0xe2, 0x68,
+	  0x4b, 0x3e, 0x33, 0xf5, 0xef, 0x90, 0xcc, 0x1b },
+	{ 0xb2, 0xc3, 0xe3, 0x3a, 0x51, 0xd2, 0x2c, 0x4c,
+	  0x08, 0xfc, 0x09, 0x89, 0xc8, 0x73, 0xc9, 0xcc,
+	  0x41, 0x50, 0x57, 0x9b, 0x1e, 0x61, 0x63, 0xfa,
+	  0x69, 0x4a, 0xd5, 0x1d, 0x53, 0xd7, 0x12, 0xdc },
+	{ 0xbe, 0x7f, 0xda, 0x98, 0x3e, 0x13, 0x18, 0x9b,
+	  0x4c, 0x77, 0xe0, 0xa8, 0x09, 0x20, 0xb6, 0xe0,
+	  0xe0, 0xea, 0x80, 0xc3, 0xb8, 0x4d, 0xbe, 0x7e,
+	  0x71, 0x17, 0xd2, 0x53, 0xf4, 0x81, 0x12, 0xf4 },
+	{ 0xb6, 0x00, 0x8c, 0x28, 0xfa, 0xe0, 0x8a, 0xa4,
+	  0x27, 0xe5, 0xbd, 0x3a, 0xad, 0x36, 0xf1, 0x00,
+	  0x21, 0xf1, 0x6c, 0x77, 0xcf, 0xea, 0xbe, 0xd0,
+	  0x7f, 0x97, 0xcc, 0x7d, 0xc1, 0xf1, 0x28, 0x4a },
+	{ 0x6e, 0x4e, 0x67, 0x60, 0xc5, 0x38, 0xf2, 0xe9,
+	  0x7b, 0x3a, 0xdb, 0xfb, 0xbc, 0xde, 0x57, 0xf8,
+	  0x96, 0x6b, 0x7e, 0xa8, 0xfc, 0xb5, 0xbf, 0x7e,
+	  0xfe, 0xc9, 0x13, 0xfd, 0x2a, 0x2b, 0x0c, 0x55 },
+	{ 0x4a, 0xe5, 0x1f, 0xd1, 0x83, 0x4a, 0xa5, 0xbd,
+	  0x9a, 0x6f, 0x7e, 0xc3, 0x9f, 0xc6, 0x63, 0x33,
+	  0x8d, 0xc5, 0xd2, 0xe2, 0x07, 0x61, 0x56, 0x6d,
+	  0x90, 0xcc, 0x68, 0xb1, 0xcb, 0x87, 0x5e, 0xd8 },
+	{ 0xb6, 0x73, 0xaa, 0xd7, 0x5a, 0xb1, 0xfd, 0xb5,
+	  0x40, 0x1a, 0xbf, 0xa1, 0xbf, 0x89, 0xf3, 0xad,
+	  0xd2, 0xeb, 0xc4, 0x68, 0xdf, 0x36, 0x24, 0xa4,
+	  0x78, 0xf4, 0xfe, 0x85, 0x9d, 0x8d, 0x55, 0xe2 },
+	{ 0x13, 0xc9, 0x47, 0x1a, 0x98, 0x55, 0x91, 0x35,
+	  0x39, 0x83, 0x66, 0x60, 0x39, 0x8d, 0xa0, 0xf3,
+	  0xf9, 0x9a, 0xda, 0x08, 0x47, 0x9c, 0x69, 0xd1,
+	  0xb7, 0xfc, 0xaa, 0x34, 0x61, 0xdd, 0x7e, 0x59 },
+	{ 0x2c, 0x11, 0xf4, 0xa7, 0xf9, 0x9a, 0x1d, 0x23,
+	  0xa5, 0x8b, 0xb6, 0x36, 0x35, 0x0f, 0xe8, 0x49,
+	  0xf2, 0x9c, 0xba, 0xc1, 0xb2, 0xa1, 0x11, 0x2d,
+	  0x9f, 0x1e, 0xd5, 0xbc, 0x5b, 0x31, 0x3c, 0xcd },
+	{ 0xc7, 0xd3, 0xc0, 0x70, 0x6b, 0x11, 0xae, 0x74,
+	  0x1c, 0x05, 0xa1, 0xef, 0x15, 0x0d, 0xd6, 0x5b,
+	  0x54, 0x94, 0xd6, 0xd5, 0x4c, 0x9a, 0x86, 0xe2,
+	  0x61, 0x78, 0x54, 0xe6, 0xae, 0xee, 0xbb, 0xd9 },
+	{ 0x19, 0x4e, 0x10, 0xc9, 0x38, 0x93, 0xaf, 0xa0,
+	  0x64, 0xc3, 0xac, 0x04, 0xc0, 0xdd, 0x80, 0x8d,
+	  0x79, 0x1c, 0x3d, 0x4b, 0x75, 0x56, 0xe8, 0x9d,
+	  0x8d, 0x9c, 0xb2, 0x25, 0xc4, 0xb3, 0x33, 0x39 },
+	{ 0x6f, 0xc4, 0x98, 0x8b, 0x8f, 0x78, 0x54, 0x6b,
+	  0x16, 0x88, 0x99, 0x18, 0x45, 0x90, 0x8f, 0x13,
+	  0x4b, 0x6a, 0x48, 0x2e, 0x69, 0x94, 0xb3, 0xd4,
+	  0x83, 0x17, 0xbf, 0x08, 0xdb, 0x29, 0x21, 0x85 },
+	{ 0x56, 0x65, 0xbe, 0xb8, 0xb0, 0x95, 0x55, 0x25,
+	  0x81, 0x3b, 0x59, 0x81, 0xcd, 0x14, 0x2e, 0xd4,
+	  0xd0, 0x3f, 0xba, 0x38, 0xa6, 0xf3, 0xe5, 0xad,
+	  0x26, 0x8e, 0x0c, 0xc2, 0x70, 0xd1, 0xcd, 0x11 },
+	{ 0xb8, 0x83, 0xd6, 0x8f, 0x5f, 0xe5, 0x19, 0x36,
+	  0x43, 0x1b, 0xa4, 0x25, 0x67, 0x38, 0x05, 0x3b,
+	  0x1d, 0x04, 0x26, 0xd4, 0xcb, 0x64, 0xb1, 0x6e,
+	  0x83, 0xba, 0xdc, 0x5e, 0x9f, 0xbe, 0x3b, 0x81 },
+	{ 0x53, 0xe7, 0xb2, 0x7e, 0xa5, 0x9c, 0x2f, 0x6d,
+	  0xbb, 0x50, 0x76, 0x9e, 0x43, 0x55, 0x4d, 0xf3,
+	  0x5a, 0xf8, 0x9f, 0x48, 0x22, 0xd0, 0x46, 0x6b,
+	  0x00, 0x7d, 0xd6, 0xf6, 0xde, 0xaf, 0xff, 0x02 },
+	{ 0x1f, 0x1a, 0x02, 0x29, 0xd4, 0x64, 0x0f, 0x01,
+	  0x90, 0x15, 0x88, 0xd9, 0xde, 0xc2, 0x2d, 0x13,
+	  0xfc, 0x3e, 0xb3, 0x4a, 0x61, 0xb3, 0x29, 0x38,
+	  0xef, 0xbf, 0x53, 0x34, 0xb2, 0x80, 0x0a, 0xfa },
+	{ 0xc2, 0xb4, 0x05, 0xaf, 0xa0, 0xfa, 0x66, 0x68,
+	  0x85, 0x2a, 0xee, 0x4d, 0x88, 0x04, 0x08, 0x53,
+	  0xfa, 0xb8, 0x00, 0xe7, 0x2b, 0x57, 0x58, 0x14,
+	  0x18, 0xe5, 0x50, 0x6f, 0x21, 0x4c, 0x7d, 0x1f },
+	{ 0xc0, 0x8a, 0xa1, 0xc2, 0x86, 0xd7, 0x09, 0xfd,
+	  0xc7, 0x47, 0x37, 0x44, 0x97, 0x71, 0x88, 0xc8,
+	  0x95, 0xba, 0x01, 0x10, 0x14, 0x24, 0x7e, 0x4e,
+	  0xfa, 0x8d, 0x07, 0xe7, 0x8f, 0xec, 0x69, 0x5c },
+	{ 0xf0, 0x3f, 0x57, 0x89, 0xd3, 0x33, 0x6b, 0x80,
+	  0xd0, 0x02, 0xd5, 0x9f, 0xdf, 0x91, 0x8b, 0xdb,
+	  0x77, 0x5b, 0x00, 0x95, 0x6e, 0xd5, 0x52, 0x8e,
+	  0x86, 0xaa, 0x99, 0x4a, 0xcb, 0x38, 0xfe, 0x2d }
+};
+
+static const u8 blake2s_keyed_testvecs[][BLAKE2S_HASH_SIZE] __initconst = {
+	{ 0x48, 0xa8, 0x99, 0x7d, 0xa4, 0x07, 0x87, 0x6b,
+	  0x3d, 0x79, 0xc0, 0xd9, 0x23, 0x25, 0xad, 0x3b,
+	  0x89, 0xcb, 0xb7, 0x54, 0xd8, 0x6a, 0xb7, 0x1a,
+	  0xee, 0x04, 0x7a, 0xd3, 0x45, 0xfd, 0x2c, 0x49 },
+	{ 0x40, 0xd1, 0x5f, 0xee, 0x7c, 0x32, 0x88, 0x30,
+	  0x16, 0x6a, 0xc3, 0xf9, 0x18, 0x65, 0x0f, 0x80,
+	  0x7e, 0x7e, 0x01, 0xe1, 0x77, 0x25, 0x8c, 0xdc,
+	  0x0a, 0x39, 0xb1, 0x1f, 0x59, 0x80, 0x66, 0xf1 },
+	{ 0x6b, 0xb7, 0x13, 0x00, 0x64, 0x4c, 0xd3, 0x99,
+	  0x1b, 0x26, 0xcc, 0xd4, 0xd2, 0x74, 0xac, 0xd1,
+	  0xad, 0xea, 0xb8, 0xb1, 0xd7, 0x91, 0x45, 0x46,
+	  0xc1, 0x19, 0x8b, 0xbe, 0x9f, 0xc9, 0xd8, 0x03 },
+	{ 0x1d, 0x22, 0x0d, 0xbe, 0x2e, 0xe1, 0x34, 0x66,
+	  0x1f, 0xdf, 0x6d, 0x9e, 0x74, 0xb4, 0x17, 0x04,
+	  0x71, 0x05, 0x56, 0xf2, 0xf6, 0xe5, 0xa0, 0x91,
+	  0xb2, 0x27, 0x69, 0x74, 0x45, 0xdb, 0xea, 0x6b },
+	{ 0xf6, 0xc3, 0xfb, 0xad, 0xb4, 0xcc, 0x68, 0x7a,
+	  0x00, 0x64, 0xa5, 0xbe, 0x6e, 0x79, 0x1b, 0xec,
+	  0x63, 0xb8, 0x68, 0xad, 0x62, 0xfb, 0xa6, 0x1b,
+	  0x37, 0x57, 0xef, 0x9c, 0xa5, 0x2e, 0x05, 0xb2 },
+	{ 0x49, 0xc1, 0xf2, 0x11, 0x88, 0xdf, 0xd7, 0x69,
+	  0xae, 0xa0, 0xe9, 0x11, 0xdd, 0x6b, 0x41, 0xf1,
+	  0x4d, 0xab, 0x10, 0x9d, 0x2b, 0x85, 0x97, 0x7a,
+	  0xa3, 0x08, 0x8b, 0x5c, 0x70, 0x7e, 0x85, 0x98 },
+	{ 0xfd, 0xd8, 0x99, 0x3d, 0xcd, 0x43, 0xf6, 0x96,
+	  0xd4, 0x4f, 0x3c, 0xea, 0x0f, 0xf3, 0x53, 0x45,
+	  0x23, 0x4e, 0xc8, 0xee, 0x08, 0x3e, 0xb3, 0xca,
+	  0xda, 0x01, 0x7c, 0x7f, 0x78, 0xc1, 0x71, 0x43 },
+	{ 0xe6, 0xc8, 0x12, 0x56, 0x37, 0x43, 0x8d, 0x09,
+	  0x05, 0xb7, 0x49, 0xf4, 0x65, 0x60, 0xac, 0x89,
+	  0xfd, 0x47, 0x1c, 0xf8, 0x69, 0x2e, 0x28, 0xfa,
+	  0xb9, 0x82, 0xf7, 0x3f, 0x01, 0x9b, 0x83, 0xa9 },
+	{ 0x19, 0xfc, 0x8c, 0xa6, 0x97, 0x9d, 0x60, 0xe6,
+	  0xed, 0xd3, 0xb4, 0x54, 0x1e, 0x2f, 0x96, 0x7c,
+	  0xed, 0x74, 0x0d, 0xf6, 0xec, 0x1e, 0xae, 0xbb,
+	  0xfe, 0x81, 0x38, 0x32, 0xe9, 0x6b, 0x29, 0x74 },
+	{ 0xa6, 0xad, 0x77, 0x7c, 0xe8, 0x81, 0xb5, 0x2b,
+	  0xb5, 0xa4, 0x42, 0x1a, 0xb6, 0xcd, 0xd2, 0xdf,
+	  0xba, 0x13, 0xe9, 0x63, 0x65, 0x2d, 0x4d, 0x6d,
+	  0x12, 0x2a, 0xee, 0x46, 0x54, 0x8c, 0x14, 0xa7 },
+	{ 0xf5, 0xc4, 0xb2, 0xba, 0x1a, 0x00, 0x78, 0x1b,
+	  0x13, 0xab, 0xa0, 0x42, 0x52, 0x42, 0xc6, 0x9c,
+	  0xb1, 0x55, 0x2f, 0x3f, 0x71, 0xa9, 0xa3, 0xbb,
+	  0x22, 0xb4, 0xa6, 0xb4, 0x27, 0x7b, 0x46, 0xdd },
+	{ 0xe3, 0x3c, 0x4c, 0x9b, 0xd0, 0xcc, 0x7e, 0x45,
+	  0xc8, 0x0e, 0x65, 0xc7, 0x7f, 0xa5, 0x99, 0x7f,
+	  0xec, 0x70, 0x02, 0x73, 0x85, 0x41, 0x50, 0x9e,
+	  0x68, 0xa9, 0x42, 0x38, 0x91, 0xe8, 0x22, 0xa3 },
+	{ 0xfb, 0xa1, 0x61, 0x69, 0xb2, 0xc3, 0xee, 0x10,
+	  0x5b, 0xe6, 0xe1, 0xe6, 0x50, 0xe5, 0xcb, 0xf4,
+	  0x07, 0x46, 0xb6, 0x75, 0x3d, 0x03, 0x6a, 0xb5,
+	  0x51, 0x79, 0x01, 0x4a, 0xd7, 0xef, 0x66, 0x51 },
+	{ 0xf5, 0xc4, 0xbe, 0xc6, 0xd6, 0x2f, 0xc6, 0x08,
+	  0xbf, 0x41, 0xcc, 0x11, 0x5f, 0x16, 0xd6, 0x1c,
+	  0x7e, 0xfd, 0x3f, 0xf6, 0xc6, 0x56, 0x92, 0xbb,
+	  0xe0, 0xaf, 0xff, 0xb1, 0xfe, 0xde, 0x74, 0x75 },
+	{ 0xa4, 0x86, 0x2e, 0x76, 0xdb, 0x84, 0x7f, 0x05,
+	  0xba, 0x17, 0xed, 0xe5, 0xda, 0x4e, 0x7f, 0x91,
+	  0xb5, 0x92, 0x5c, 0xf1, 0xad, 0x4b, 0xa1, 0x27,
+	  0x32, 0xc3, 0x99, 0x57, 0x42, 0xa5, 0xcd, 0x6e },
+	{ 0x65, 0xf4, 0xb8, 0x60, 0xcd, 0x15, 0xb3, 0x8e,
+	  0xf8, 0x14, 0xa1, 0xa8, 0x04, 0x31, 0x4a, 0x55,
+	  0xbe, 0x95, 0x3c, 0xaa, 0x65, 0xfd, 0x75, 0x8a,
+	  0xd9, 0x89, 0xff, 0x34, 0xa4, 0x1c, 0x1e, 0xea },
+	{ 0x19, 0xba, 0x23, 0x4f, 0x0a, 0x4f, 0x38, 0x63,
+	  0x7d, 0x18, 0x39, 0xf9, 0xd9, 0xf7, 0x6a, 0xd9,
+	  0x1c, 0x85, 0x22, 0x30, 0x71, 0x43, 0xc9, 0x7d,
+	  0x5f, 0x93, 0xf6, 0x92, 0x74, 0xce, 0xc9, 0xa7 },
+	{ 0x1a, 0x67, 0x18, 0x6c, 0xa4, 0xa5, 0xcb, 0x8e,
+	  0x65, 0xfc, 0xa0, 0xe2, 0xec, 0xbc, 0x5d, 0xdc,
+	  0x14, 0xae, 0x38, 0x1b, 0xb8, 0xbf, 0xfe, 0xb9,
+	  0xe0, 0xa1, 0x03, 0x44, 0x9e, 0x3e, 0xf0, 0x3c },
+	{ 0xaf, 0xbe, 0xa3, 0x17, 0xb5, 0xa2, 0xe8, 0x9c,
+	  0x0b, 0xd9, 0x0c, 0xcf, 0x5d, 0x7f, 0xd0, 0xed,
+	  0x57, 0xfe, 0x58, 0x5e, 0x4b, 0xe3, 0x27, 0x1b,
+	  0x0a, 0x6b, 0xf0, 0xf5, 0x78, 0x6b, 0x0f, 0x26 },
+	{ 0xf1, 0xb0, 0x15, 0x58, 0xce, 0x54, 0x12, 0x62,
+	  0xf5, 0xec, 0x34, 0x29, 0x9d, 0x6f, 0xb4, 0x09,
+	  0x00, 0x09, 0xe3, 0x43, 0x4b, 0xe2, 0xf4, 0x91,
+	  0x05, 0xcf, 0x46, 0xaf, 0x4d, 0x2d, 0x41, 0x24 },
+	{ 0x13, 0xa0, 0xa0, 0xc8, 0x63, 0x35, 0x63, 0x5e,
+	  0xaa, 0x74, 0xca, 0x2d, 0x5d, 0x48, 0x8c, 0x79,
+	  0x7b, 0xbb, 0x4f, 0x47, 0xdc, 0x07, 0x10, 0x50,
+	  0x15, 0xed, 0x6a, 0x1f, 0x33, 0x09, 0xef, 0xce },
+	{ 0x15, 0x80, 0xaf, 0xee, 0xbe, 0xbb, 0x34, 0x6f,
+	  0x94, 0xd5, 0x9f, 0xe6, 0x2d, 0xa0, 0xb7, 0x92,
+	  0x37, 0xea, 0xd7, 0xb1, 0x49, 0x1f, 0x56, 0x67,
+	  0xa9, 0x0e, 0x45, 0xed, 0xf6, 0xca, 0x8b, 0x03 },
+	{ 0x20, 0xbe, 0x1a, 0x87, 0x5b, 0x38, 0xc5, 0x73,
+	  0xdd, 0x7f, 0xaa, 0xa0, 0xde, 0x48, 0x9d, 0x65,
+	  0x5c, 0x11, 0xef, 0xb6, 0xa5, 0x52, 0x69, 0x8e,
+	  0x07, 0xa2, 0xd3, 0x31, 0xb5, 0xf6, 0x55, 0xc3 },
+	{ 0xbe, 0x1f, 0xe3, 0xc4, 0xc0, 0x40, 0x18, 0xc5,
+	  0x4c, 0x4a, 0x0f, 0x6b, 0x9a, 0x2e, 0xd3, 0xc5,
+	  0x3a, 0xbe, 0x3a, 0x9f, 0x76, 0xb4, 0xd2, 0x6d,
+	  0xe5, 0x6f, 0xc9, 0xae, 0x95, 0x05, 0x9a, 0x99 },
+	{ 0xe3, 0xe3, 0xac, 0xe5, 0x37, 0xeb, 0x3e, 0xdd,
+	  0x84, 0x63, 0xd9, 0xad, 0x35, 0x82, 0xe1, 0x3c,
+	  0xf8, 0x65, 0x33, 0xff, 0xde, 0x43, 0xd6, 0x68,
+	  0xdd, 0x2e, 0x93, 0xbb, 0xdb, 0xd7, 0x19, 0x5a },
+	{ 0x11, 0x0c, 0x50, 0xc0, 0xbf, 0x2c, 0x6e, 0x7a,
+	  0xeb, 0x7e, 0x43, 0x5d, 0x92, 0xd1, 0x32, 0xab,
+	  0x66, 0x55, 0x16, 0x8e, 0x78, 0xa2, 0xde, 0xcd,
+	  0xec, 0x33, 0x30, 0x77, 0x76, 0x84, 0xd9, 0xc1 },
+	{ 0xe9, 0xba, 0x8f, 0x50, 0x5c, 0x9c, 0x80, 0xc0,
+	  0x86, 0x66, 0xa7, 0x01, 0xf3, 0x36, 0x7e, 0x6c,
+	  0xc6, 0x65, 0xf3, 0x4b, 0x22, 0xe7, 0x3c, 0x3c,
+	  0x04, 0x17, 0xeb, 0x1c, 0x22, 0x06, 0x08, 0x2f },
+	{ 0x26, 0xcd, 0x66, 0xfc, 0xa0, 0x23, 0x79, 0xc7,
+	  0x6d, 0xf1, 0x23, 0x17, 0x05, 0x2b, 0xca, 0xfd,
+	  0x6c, 0xd8, 0xc3, 0xa7, 0xb8, 0x90, 0xd8, 0x05,
+	  0xf3, 0x6c, 0x49, 0x98, 0x97, 0x82, 0x43, 0x3a },
+	{ 0x21, 0x3f, 0x35, 0x96, 0xd6, 0xe3, 0xa5, 0xd0,
+	  0xe9, 0x93, 0x2c, 0xd2, 0x15, 0x91, 0x46, 0x01,
+	  0x5e, 0x2a, 0xbc, 0x94, 0x9f, 0x47, 0x29, 0xee,
+	  0x26, 0x32, 0xfe, 0x1e, 0xdb, 0x78, 0xd3, 0x37 },
+	{ 0x10, 0x15, 0xd7, 0x01, 0x08, 0xe0, 0x3b, 0xe1,
+	  0xc7, 0x02, 0xfe, 0x97, 0x25, 0x36, 0x07, 0xd1,
+	  0x4a, 0xee, 0x59, 0x1f, 0x24, 0x13, 0xea, 0x67,
+	  0x87, 0x42, 0x7b, 0x64, 0x59, 0xff, 0x21, 0x9a },
+	{ 0x3c, 0xa9, 0x89, 0xde, 0x10, 0xcf, 0xe6, 0x09,
+	  0x90, 0x94, 0x72, 0xc8, 0xd3, 0x56, 0x10, 0x80,
+	  0x5b, 0x2f, 0x97, 0x77, 0x34, 0xcf, 0x65, 0x2c,
+	  0xc6, 0x4b, 0x3b, 0xfc, 0x88, 0x2d, 0x5d, 0x89 },
+	{ 0xb6, 0x15, 0x6f, 0x72, 0xd3, 0x80, 0xee, 0x9e,
+	  0xa6, 0xac, 0xd1, 0x90, 0x46, 0x4f, 0x23, 0x07,
+	  0xa5, 0xc1, 0x79, 0xef, 0x01, 0xfd, 0x71, 0xf9,
+	  0x9f, 0x2d, 0x0f, 0x7a, 0x57, 0x36, 0x0a, 0xea },
+	{ 0xc0, 0x3b, 0xc6, 0x42, 0xb2, 0x09, 0x59, 0xcb,
+	  0xe1, 0x33, 0xa0, 0x30, 0x3e, 0x0c, 0x1a, 0xbf,
+	  0xf3, 0xe3, 0x1e, 0xc8, 0xe1, 0xa3, 0x28, 0xec,
+	  0x85, 0x65, 0xc3, 0x6d, 0xec, 0xff, 0x52, 0x65 },
+	{ 0x2c, 0x3e, 0x08, 0x17, 0x6f, 0x76, 0x0c, 0x62,
+	  0x64, 0xc3, 0xa2, 0xcd, 0x66, 0xfe, 0xc6, 0xc3,
+	  0xd7, 0x8d, 0xe4, 0x3f, 0xc1, 0x92, 0x45, 0x7b,
+	  0x2a, 0x4a, 0x66, 0x0a, 0x1e, 0x0e, 0xb2, 0x2b },
+	{ 0xf7, 0x38, 0xc0, 0x2f, 0x3c, 0x1b, 0x19, 0x0c,
+	  0x51, 0x2b, 0x1a, 0x32, 0xde, 0xab, 0xf3, 0x53,
+	  0x72, 0x8e, 0x0e, 0x9a, 0xb0, 0x34, 0x49, 0x0e,
+	  0x3c, 0x34, 0x09, 0x94, 0x6a, 0x97, 0xae, 0xec },
+	{ 0x8b, 0x18, 0x80, 0xdf, 0x30, 0x1c, 0xc9, 0x63,
+	  0x41, 0x88, 0x11, 0x08, 0x89, 0x64, 0x83, 0x92,
+	  0x87, 0xff, 0x7f, 0xe3, 0x1c, 0x49, 0xea, 0x6e,
+	  0xbd, 0x9e, 0x48, 0xbd, 0xee, 0xe4, 0x97, 0xc5 },
+	{ 0x1e, 0x75, 0xcb, 0x21, 0xc6, 0x09, 0x89, 0x02,
+	  0x03, 0x75, 0xf1, 0xa7, 0xa2, 0x42, 0x83, 0x9f,
+	  0x0b, 0x0b, 0x68, 0x97, 0x3a, 0x4c, 0x2a, 0x05,
+	  0xcf, 0x75, 0x55, 0xed, 0x5a, 0xae, 0xc4, 0xc1 },
+	{ 0x62, 0xbf, 0x8a, 0x9c, 0x32, 0xa5, 0xbc, 0xcf,
+	  0x29, 0x0b, 0x6c, 0x47, 0x4d, 0x75, 0xb2, 0xa2,
+	  0xa4, 0x09, 0x3f, 0x1a, 0x9e, 0x27, 0x13, 0x94,
+	  0x33, 0xa8, 0xf2, 0xb3, 0xbc, 0xe7, 0xb8, 0xd7 },
+	{ 0x16, 0x6c, 0x83, 0x50, 0xd3, 0x17, 0x3b, 0x5e,
+	  0x70, 0x2b, 0x78, 0x3d, 0xfd, 0x33, 0xc6, 0x6e,
+	  0xe0, 0x43, 0x27, 0x42, 0xe9, 0xb9, 0x2b, 0x99,
+	  0x7f, 0xd2, 0x3c, 0x60, 0xdc, 0x67, 0x56, 0xca },
+	{ 0x04, 0x4a, 0x14, 0xd8, 0x22, 0xa9, 0x0c, 0xac,
+	  0xf2, 0xf5, 0xa1, 0x01, 0x42, 0x8a, 0xdc, 0x8f,
+	  0x41, 0x09, 0x38, 0x6c, 0xcb, 0x15, 0x8b, 0xf9,
+	  0x05, 0xc8, 0x61, 0x8b, 0x8e, 0xe2, 0x4e, 0xc3 },
+	{ 0x38, 0x7d, 0x39, 0x7e, 0xa4, 0x3a, 0x99, 0x4b,
+	  0xe8, 0x4d, 0x2d, 0x54, 0x4a, 0xfb, 0xe4, 0x81,
+	  0xa2, 0x00, 0x0f, 0x55, 0x25, 0x26, 0x96, 0xbb,
+	  0xa2, 0xc5, 0x0c, 0x8e, 0xbd, 0x10, 0x13, 0x47 },
+	{ 0x56, 0xf8, 0xcc, 0xf1, 0xf8, 0x64, 0x09, 0xb4,
+	  0x6c, 0xe3, 0x61, 0x66, 0xae, 0x91, 0x65, 0x13,
+	  0x84, 0x41, 0x57, 0x75, 0x89, 0xdb, 0x08, 0xcb,
+	  0xc5, 0xf6, 0x6c, 0xa2, 0x97, 0x43, 0xb9, 0xfd },
+	{ 0x97, 0x06, 0xc0, 0x92, 0xb0, 0x4d, 0x91, 0xf5,
+	  0x3d, 0xff, 0x91, 0xfa, 0x37, 0xb7, 0x49, 0x3d,
+	  0x28, 0xb5, 0x76, 0xb5, 0xd7, 0x10, 0x46, 0x9d,
+	  0xf7, 0x94, 0x01, 0x66, 0x22, 0x36, 0xfc, 0x03 },
+	{ 0x87, 0x79, 0x68, 0x68, 0x6c, 0x06, 0x8c, 0xe2,
+	  0xf7, 0xe2, 0xad, 0xcf, 0xf6, 0x8b, 0xf8, 0x74,
+	  0x8e, 0xdf, 0x3c, 0xf8, 0x62, 0xcf, 0xb4, 0xd3,
+	  0x94, 0x7a, 0x31, 0x06, 0x95, 0x80, 0x54, 0xe3 },
+	{ 0x88, 0x17, 0xe5, 0x71, 0x98, 0x79, 0xac, 0xf7,
+	  0x02, 0x47, 0x87, 0xec, 0xcd, 0xb2, 0x71, 0x03,
+	  0x55, 0x66, 0xcf, 0xa3, 0x33, 0xe0, 0x49, 0x40,
+	  0x7c, 0x01, 0x78, 0xcc, 0xc5, 0x7a, 0x5b, 0x9f },
+	{ 0x89, 0x38, 0x24, 0x9e, 0x4b, 0x50, 0xca, 0xda,
+	  0xcc, 0xdf, 0x5b, 0x18, 0x62, 0x13, 0x26, 0xcb,
+	  0xb1, 0x52, 0x53, 0xe3, 0x3a, 0x20, 0xf5, 0x63,
+	  0x6e, 0x99, 0x5d, 0x72, 0x47, 0x8d, 0xe4, 0x72 },
+	{ 0xf1, 0x64, 0xab, 0xba, 0x49, 0x63, 0xa4, 0x4d,
+	  0x10, 0x72, 0x57, 0xe3, 0x23, 0x2d, 0x90, 0xac,
+	  0xa5, 0xe6, 0x6a, 0x14, 0x08, 0x24, 0x8c, 0x51,
+	  0x74, 0x1e, 0x99, 0x1d, 0xb5, 0x22, 0x77, 0x56 },
+	{ 0xd0, 0x55, 0x63, 0xe2, 0xb1, 0xcb, 0xa0, 0xc4,
+	  0xa2, 0xa1, 0xe8, 0xbd, 0xe3, 0xa1, 0xa0, 0xd9,
+	  0xf5, 0xb4, 0x0c, 0x85, 0xa0, 0x70, 0xd6, 0xf5,
+	  0xfb, 0x21, 0x06, 0x6e, 0xad, 0x5d, 0x06, 0x01 },
+	{ 0x03, 0xfb, 0xb1, 0x63, 0x84, 0xf0, 0xa3, 0x86,
+	  0x6f, 0x4c, 0x31, 0x17, 0x87, 0x76, 0x66, 0xef,
+	  0xbf, 0x12, 0x45, 0x97, 0x56, 0x4b, 0x29, 0x3d,
+	  0x4a, 0xab, 0x0d, 0x26, 0x9f, 0xab, 0xdd, 0xfa },
+	{ 0x5f, 0xa8, 0x48, 0x6a, 0xc0, 0xe5, 0x29, 0x64,
+	  0xd1, 0x88, 0x1b, 0xbe, 0x33, 0x8e, 0xb5, 0x4b,
+	  0xe2, 0xf7, 0x19, 0x54, 0x92, 0x24, 0x89, 0x20,
+	  0x57, 0xb4, 0xda, 0x04, 0xba, 0x8b, 0x34, 0x75 },
+	{ 0xcd, 0xfa, 0xbc, 0xee, 0x46, 0x91, 0x11, 0x11,
+	  0x23, 0x6a, 0x31, 0x70, 0x8b, 0x25, 0x39, 0xd7,
+	  0x1f, 0xc2, 0x11, 0xd9, 0xb0, 0x9c, 0x0d, 0x85,
+	  0x30, 0xa1, 0x1e, 0x1d, 0xbf, 0x6e, 0xed, 0x01 },
+	{ 0x4f, 0x82, 0xde, 0x03, 0xb9, 0x50, 0x47, 0x93,
+	  0xb8, 0x2a, 0x07, 0xa0, 0xbd, 0xcd, 0xff, 0x31,
+	  0x4d, 0x75, 0x9e, 0x7b, 0x62, 0xd2, 0x6b, 0x78,
+	  0x49, 0x46, 0xb0, 0xd3, 0x6f, 0x91, 0x6f, 0x52 },
+	{ 0x25, 0x9e, 0xc7, 0xf1, 0x73, 0xbc, 0xc7, 0x6a,
+	  0x09, 0x94, 0xc9, 0x67, 0xb4, 0xf5, 0xf0, 0x24,
+	  0xc5, 0x60, 0x57, 0xfb, 0x79, 0xc9, 0x65, 0xc4,
+	  0xfa, 0xe4, 0x18, 0x75, 0xf0, 0x6a, 0x0e, 0x4c },
+	{ 0x19, 0x3c, 0xc8, 0xe7, 0xc3, 0xe0, 0x8b, 0xb3,
+	  0x0f, 0x54, 0x37, 0xaa, 0x27, 0xad, 0xe1, 0xf1,
+	  0x42, 0x36, 0x9b, 0x24, 0x6a, 0x67, 0x5b, 0x23,
+	  0x83, 0xe6, 0xda, 0x9b, 0x49, 0xa9, 0x80, 0x9e },
+	{ 0x5c, 0x10, 0x89, 0x6f, 0x0e, 0x28, 0x56, 0xb2,
+	  0xa2, 0xee, 0xe0, 0xfe, 0x4a, 0x2c, 0x16, 0x33,
+	  0x56, 0x5d, 0x18, 0xf0, 0xe9, 0x3e, 0x1f, 0xab,
+	  0x26, 0xc3, 0x73, 0xe8, 0xf8, 0x29, 0x65, 0x4d },
+	{ 0xf1, 0x60, 0x12, 0xd9, 0x3f, 0x28, 0x85, 0x1a,
+	  0x1e, 0xb9, 0x89, 0xf5, 0xd0, 0xb4, 0x3f, 0x3f,
+	  0x39, 0xca, 0x73, 0xc9, 0xa6, 0x2d, 0x51, 0x81,
+	  0xbf, 0xf2, 0x37, 0x53, 0x6b, 0xd3, 0x48, 0xc3 },
+	{ 0x29, 0x66, 0xb3, 0xcf, 0xae, 0x1e, 0x44, 0xea,
+	  0x99, 0x6d, 0xc5, 0xd6, 0x86, 0xcf, 0x25, 0xfa,
+	  0x05, 0x3f, 0xb6, 0xf6, 0x72, 0x01, 0xb9, 0xe4,
+	  0x6e, 0xad, 0xe8, 0x5d, 0x0a, 0xd6, 0xb8, 0x06 },
+	{ 0xdd, 0xb8, 0x78, 0x24, 0x85, 0xe9, 0x00, 0xbc,
+	  0x60, 0xbc, 0xf4, 0xc3, 0x3a, 0x6f, 0xd5, 0x85,
+	  0x68, 0x0c, 0xc6, 0x83, 0xd5, 0x16, 0xef, 0xa0,
+	  0x3e, 0xb9, 0x98, 0x5f, 0xad, 0x87, 0x15, 0xfb },
+	{ 0x4c, 0x4d, 0x6e, 0x71, 0xae, 0xa0, 0x57, 0x86,
+	  0x41, 0x31, 0x48, 0xfc, 0x7a, 0x78, 0x6b, 0x0e,
+	  0xca, 0xf5, 0x82, 0xcf, 0xf1, 0x20, 0x9f, 0x5a,
+	  0x80, 0x9f, 0xba, 0x85, 0x04, 0xce, 0x66, 0x2c },
+	{ 0xfb, 0x4c, 0x5e, 0x86, 0xd7, 0xb2, 0x22, 0x9b,
+	  0x99, 0xb8, 0xba, 0x6d, 0x94, 0xc2, 0x47, 0xef,
+	  0x96, 0x4a, 0xa3, 0xa2, 0xba, 0xe8, 0xed, 0xc7,
+	  0x75, 0x69, 0xf2, 0x8d, 0xbb, 0xff, 0x2d, 0x4e },
+	{ 0xe9, 0x4f, 0x52, 0x6d, 0xe9, 0x01, 0x96, 0x33,
+	  0xec, 0xd5, 0x4a, 0xc6, 0x12, 0x0f, 0x23, 0x95,
+	  0x8d, 0x77, 0x18, 0xf1, 0xe7, 0x71, 0x7b, 0xf3,
+	  0x29, 0x21, 0x1a, 0x4f, 0xae, 0xed, 0x4e, 0x6d },
+	{ 0xcb, 0xd6, 0x66, 0x0a, 0x10, 0xdb, 0x3f, 0x23,
+	  0xf7, 0xa0, 0x3d, 0x4b, 0x9d, 0x40, 0x44, 0xc7,
+	  0x93, 0x2b, 0x28, 0x01, 0xac, 0x89, 0xd6, 0x0b,
+	  0xc9, 0xeb, 0x92, 0xd6, 0x5a, 0x46, 0xc2, 0xa0 },
+	{ 0x88, 0x18, 0xbb, 0xd3, 0xdb, 0x4d, 0xc1, 0x23,
+	  0xb2, 0x5c, 0xbb, 0xa5, 0xf5, 0x4c, 0x2b, 0xc4,
+	  0xb3, 0xfc, 0xf9, 0xbf, 0x7d, 0x7a, 0x77, 0x09,
+	  0xf4, 0xae, 0x58, 0x8b, 0x26, 0x7c, 0x4e, 0xce },
+	{ 0xc6, 0x53, 0x82, 0x51, 0x3f, 0x07, 0x46, 0x0d,
+	  0xa3, 0x98, 0x33, 0xcb, 0x66, 0x6c, 0x5e, 0xd8,
+	  0x2e, 0x61, 0xb9, 0xe9, 0x98, 0xf4, 0xb0, 0xc4,
+	  0x28, 0x7c, 0xee, 0x56, 0xc3, 0xcc, 0x9b, 0xcd },
+	{ 0x89, 0x75, 0xb0, 0x57, 0x7f, 0xd3, 0x55, 0x66,
+	  0xd7, 0x50, 0xb3, 0x62, 0xb0, 0x89, 0x7a, 0x26,
+	  0xc3, 0x99, 0x13, 0x6d, 0xf0, 0x7b, 0xab, 0xab,
+	  0xbd, 0xe6, 0x20, 0x3f, 0xf2, 0x95, 0x4e, 0xd4 },
+	{ 0x21, 0xfe, 0x0c, 0xeb, 0x00, 0x52, 0xbe, 0x7f,
+	  0xb0, 0xf0, 0x04, 0x18, 0x7c, 0xac, 0xd7, 0xde,
+	  0x67, 0xfa, 0x6e, 0xb0, 0x93, 0x8d, 0x92, 0x76,
+	  0x77, 0xf2, 0x39, 0x8c, 0x13, 0x23, 0x17, 0xa8 },
+	{ 0x2e, 0xf7, 0x3f, 0x3c, 0x26, 0xf1, 0x2d, 0x93,
+	  0x88, 0x9f, 0x3c, 0x78, 0xb6, 0xa6, 0x6c, 0x1d,
+	  0x52, 0xb6, 0x49, 0xdc, 0x9e, 0x85, 0x6e, 0x2c,
+	  0x17, 0x2e, 0xa7, 0xc5, 0x8a, 0xc2, 0xb5, 0xe3 },
+	{ 0x38, 0x8a, 0x3c, 0xd5, 0x6d, 0x73, 0x86, 0x7a,
+	  0xbb, 0x5f, 0x84, 0x01, 0x49, 0x2b, 0x6e, 0x26,
+	  0x81, 0xeb, 0x69, 0x85, 0x1e, 0x76, 0x7f, 0xd8,
+	  0x42, 0x10, 0xa5, 0x60, 0x76, 0xfb, 0x3d, 0xd3 },
+	{ 0xaf, 0x53, 0x3e, 0x02, 0x2f, 0xc9, 0x43, 0x9e,
+	  0x4e, 0x3c, 0xb8, 0x38, 0xec, 0xd1, 0x86, 0x92,
+	  0x23, 0x2a, 0xdf, 0x6f, 0xe9, 0x83, 0x95, 0x26,
+	  0xd3, 0xc3, 0xdd, 0x1b, 0x71, 0x91, 0x0b, 0x1a },
+	{ 0x75, 0x1c, 0x09, 0xd4, 0x1a, 0x93, 0x43, 0x88,
+	  0x2a, 0x81, 0xcd, 0x13, 0xee, 0x40, 0x81, 0x8d,
+	  0x12, 0xeb, 0x44, 0xc6, 0xc7, 0xf4, 0x0d, 0xf1,
+	  0x6e, 0x4a, 0xea, 0x8f, 0xab, 0x91, 0x97, 0x2a },
+	{ 0x5b, 0x73, 0xdd, 0xb6, 0x8d, 0x9d, 0x2b, 0x0a,
+	  0xa2, 0x65, 0xa0, 0x79, 0x88, 0xd6, 0xb8, 0x8a,
+	  0xe9, 0xaa, 0xc5, 0x82, 0xaf, 0x83, 0x03, 0x2f,
+	  0x8a, 0x9b, 0x21, 0xa2, 0xe1, 0xb7, 0xbf, 0x18 },
+	{ 0x3d, 0xa2, 0x91, 0x26, 0xc7, 0xc5, 0xd7, 0xf4,
+	  0x3e, 0x64, 0x24, 0x2a, 0x79, 0xfe, 0xaa, 0x4e,
+	  0xf3, 0x45, 0x9c, 0xde, 0xcc, 0xc8, 0x98, 0xed,
+	  0x59, 0xa9, 0x7f, 0x6e, 0xc9, 0x3b, 0x9d, 0xab },
+	{ 0x56, 0x6d, 0xc9, 0x20, 0x29, 0x3d, 0xa5, 0xcb,
+	  0x4f, 0xe0, 0xaa, 0x8a, 0xbd, 0xa8, 0xbb, 0xf5,
+	  0x6f, 0x55, 0x23, 0x13, 0xbf, 0xf1, 0x90, 0x46,
+	  0x64, 0x1e, 0x36, 0x15, 0xc1, 0xe3, 0xed, 0x3f },
+	{ 0x41, 0x15, 0xbe, 0xa0, 0x2f, 0x73, 0xf9, 0x7f,
+	  0x62, 0x9e, 0x5c, 0x55, 0x90, 0x72, 0x0c, 0x01,
+	  0xe7, 0xe4, 0x49, 0xae, 0x2a, 0x66, 0x97, 0xd4,
+	  0xd2, 0x78, 0x33, 0x21, 0x30, 0x36, 0x92, 0xf9 },
+	{ 0x4c, 0xe0, 0x8f, 0x47, 0x62, 0x46, 0x8a, 0x76,
+	  0x70, 0x01, 0x21, 0x64, 0x87, 0x8d, 0x68, 0x34,
+	  0x0c, 0x52, 0xa3, 0x5e, 0x66, 0xc1, 0x88, 0x4d,
+	  0x5c, 0x86, 0x48, 0x89, 0xab, 0xc9, 0x66, 0x77 },
+	{ 0x81, 0xea, 0x0b, 0x78, 0x04, 0x12, 0x4e, 0x0c,
+	  0x22, 0xea, 0x5f, 0xc7, 0x11, 0x04, 0xa2, 0xaf,
+	  0xcb, 0x52, 0xa1, 0xfa, 0x81, 0x6f, 0x3e, 0xcb,
+	  0x7d, 0xcb, 0x5d, 0x9d, 0xea, 0x17, 0x86, 0xd0 },
+	{ 0xfe, 0x36, 0x27, 0x33, 0xb0, 0x5f, 0x6b, 0xed,
+	  0xaf, 0x93, 0x79, 0xd7, 0xf7, 0x93, 0x6e, 0xde,
+	  0x20, 0x9b, 0x1f, 0x83, 0x23, 0xc3, 0x92, 0x25,
+	  0x49, 0xd9, 0xe7, 0x36, 0x81, 0xb5, 0xdb, 0x7b },
+	{ 0xef, 0xf3, 0x7d, 0x30, 0xdf, 0xd2, 0x03, 0x59,
+	  0xbe, 0x4e, 0x73, 0xfd, 0xf4, 0x0d, 0x27, 0x73,
+	  0x4b, 0x3d, 0xf9, 0x0a, 0x97, 0xa5, 0x5e, 0xd7,
+	  0x45, 0x29, 0x72, 0x94, 0xca, 0x85, 0xd0, 0x9f },
+	{ 0x17, 0x2f, 0xfc, 0x67, 0x15, 0x3d, 0x12, 0xe0,
+	  0xca, 0x76, 0xa8, 0xb6, 0xcd, 0x5d, 0x47, 0x31,
+	  0x88, 0x5b, 0x39, 0xce, 0x0c, 0xac, 0x93, 0xa8,
+	  0x97, 0x2a, 0x18, 0x00, 0x6c, 0x8b, 0x8b, 0xaf },
+	{ 0xc4, 0x79, 0x57, 0xf1, 0xcc, 0x88, 0xe8, 0x3e,
+	  0xf9, 0x44, 0x58, 0x39, 0x70, 0x9a, 0x48, 0x0a,
+	  0x03, 0x6b, 0xed, 0x5f, 0x88, 0xac, 0x0f, 0xcc,
+	  0x8e, 0x1e, 0x70, 0x3f, 0xfa, 0xac, 0x13, 0x2c },
+	{ 0x30, 0xf3, 0x54, 0x83, 0x70, 0xcf, 0xdc, 0xed,
+	  0xa5, 0xc3, 0x7b, 0x56, 0x9b, 0x61, 0x75, 0xe7,
+	  0x99, 0xee, 0xf1, 0xa6, 0x2a, 0xaa, 0x94, 0x32,
+	  0x45, 0xae, 0x76, 0x69, 0xc2, 0x27, 0xa7, 0xb5 },
+	{ 0xc9, 0x5d, 0xcb, 0x3c, 0xf1, 0xf2, 0x7d, 0x0e,
+	  0xef, 0x2f, 0x25, 0xd2, 0x41, 0x38, 0x70, 0x90,
+	  0x4a, 0x87, 0x7c, 0x4a, 0x56, 0xc2, 0xde, 0x1e,
+	  0x83, 0xe2, 0xbc, 0x2a, 0xe2, 0xe4, 0x68, 0x21 },
+	{ 0xd5, 0xd0, 0xb5, 0xd7, 0x05, 0x43, 0x4c, 0xd4,
+	  0x6b, 0x18, 0x57, 0x49, 0xf6, 0x6b, 0xfb, 0x58,
+	  0x36, 0xdc, 0xdf, 0x6e, 0xe5, 0x49, 0xa2, 0xb7,
+	  0xa4, 0xae, 0xe7, 0xf5, 0x80, 0x07, 0xca, 0xaf },
+	{ 0xbb, 0xc1, 0x24, 0xa7, 0x12, 0xf1, 0x5d, 0x07,
+	  0xc3, 0x00, 0xe0, 0x5b, 0x66, 0x83, 0x89, 0xa4,
+	  0x39, 0xc9, 0x17, 0x77, 0xf7, 0x21, 0xf8, 0x32,
+	  0x0c, 0x1c, 0x90, 0x78, 0x06, 0x6d, 0x2c, 0x7e },
+	{ 0xa4, 0x51, 0xb4, 0x8c, 0x35, 0xa6, 0xc7, 0x85,
+	  0x4c, 0xfa, 0xae, 0x60, 0x26, 0x2e, 0x76, 0x99,
+	  0x08, 0x16, 0x38, 0x2a, 0xc0, 0x66, 0x7e, 0x5a,
+	  0x5c, 0x9e, 0x1b, 0x46, 0xc4, 0x34, 0x2d, 0xdf },
+	{ 0xb0, 0xd1, 0x50, 0xfb, 0x55, 0xe7, 0x78, 0xd0,
+	  0x11, 0x47, 0xf0, 0xb5, 0xd8, 0x9d, 0x99, 0xec,
+	  0xb2, 0x0f, 0xf0, 0x7e, 0x5e, 0x67, 0x60, 0xd6,
+	  0xb6, 0x45, 0xeb, 0x5b, 0x65, 0x4c, 0x62, 0x2b },
+	{ 0x34, 0xf7, 0x37, 0xc0, 0xab, 0x21, 0x99, 0x51,
+	  0xee, 0xe8, 0x9a, 0x9f, 0x8d, 0xac, 0x29, 0x9c,
+	  0x9d, 0x4c, 0x38, 0xf3, 0x3f, 0xa4, 0x94, 0xc5,
+	  0xc6, 0xee, 0xfc, 0x92, 0xb6, 0xdb, 0x08, 0xbc },
+	{ 0x1a, 0x62, 0xcc, 0x3a, 0x00, 0x80, 0x0d, 0xcb,
+	  0xd9, 0x98, 0x91, 0x08, 0x0c, 0x1e, 0x09, 0x84,
+	  0x58, 0x19, 0x3a, 0x8c, 0xc9, 0xf9, 0x70, 0xea,
+	  0x99, 0xfb, 0xef, 0xf0, 0x03, 0x18, 0xc2, 0x89 },
+	{ 0xcf, 0xce, 0x55, 0xeb, 0xaf, 0xc8, 0x40, 0xd7,
+	  0xae, 0x48, 0x28, 0x1c, 0x7f, 0xd5, 0x7e, 0xc8,
+	  0xb4, 0x82, 0xd4, 0xb7, 0x04, 0x43, 0x74, 0x95,
+	  0x49, 0x5a, 0xc4, 0x14, 0xcf, 0x4a, 0x37, 0x4b },
+	{ 0x67, 0x46, 0xfa, 0xcf, 0x71, 0x14, 0x6d, 0x99,
+	  0x9d, 0xab, 0xd0, 0x5d, 0x09, 0x3a, 0xe5, 0x86,
+	  0x64, 0x8d, 0x1e, 0xe2, 0x8e, 0x72, 0x61, 0x7b,
+	  0x99, 0xd0, 0xf0, 0x08, 0x6e, 0x1e, 0x45, 0xbf },
+	{ 0x57, 0x1c, 0xed, 0x28, 0x3b, 0x3f, 0x23, 0xb4,
+	  0xe7, 0x50, 0xbf, 0x12, 0xa2, 0xca, 0xf1, 0x78,
+	  0x18, 0x47, 0xbd, 0x89, 0x0e, 0x43, 0x60, 0x3c,
+	  0xdc, 0x59, 0x76, 0x10, 0x2b, 0x7b, 0xb1, 0x1b },
+	{ 0xcf, 0xcb, 0x76, 0x5b, 0x04, 0x8e, 0x35, 0x02,
+	  0x2c, 0x5d, 0x08, 0x9d, 0x26, 0xe8, 0x5a, 0x36,
+	  0xb0, 0x05, 0xa2, 0xb8, 0x04, 0x93, 0xd0, 0x3a,
+	  0x14, 0x4e, 0x09, 0xf4, 0x09, 0xb6, 0xaf, 0xd1 },
+	{ 0x40, 0x50, 0xc7, 0xa2, 0x77, 0x05, 0xbb, 0x27,
+	  0xf4, 0x20, 0x89, 0xb2, 0x99, 0xf3, 0xcb, 0xe5,
+	  0x05, 0x4e, 0xad, 0x68, 0x72, 0x7e, 0x8e, 0xf9,
+	  0x31, 0x8c, 0xe6, 0xf2, 0x5c, 0xd6, 0xf3, 0x1d },
+	{ 0x18, 0x40, 0x70, 0xbd, 0x5d, 0x26, 0x5f, 0xbd,
+	  0xc1, 0x42, 0xcd, 0x1c, 0x5c, 0xd0, 0xd7, 0xe4,
+	  0x14, 0xe7, 0x03, 0x69, 0xa2, 0x66, 0xd6, 0x27,
+	  0xc8, 0xfb, 0xa8, 0x4f, 0xa5, 0xe8, 0x4c, 0x34 },
+	{ 0x9e, 0xdd, 0xa9, 0xa4, 0x44, 0x39, 0x02, 0xa9,
+	  0x58, 0x8c, 0x0d, 0x0c, 0xcc, 0x62, 0xb9, 0x30,
+	  0x21, 0x84, 0x79, 0xa6, 0x84, 0x1e, 0x6f, 0xe7,
+	  0xd4, 0x30, 0x03, 0xf0, 0x4b, 0x1f, 0xd6, 0x43 },
+	{ 0xe4, 0x12, 0xfe, 0xef, 0x79, 0x08, 0x32, 0x4a,
+	  0x6d, 0xa1, 0x84, 0x16, 0x29, 0xf3, 0x5d, 0x3d,
+	  0x35, 0x86, 0x42, 0x01, 0x93, 0x10, 0xec, 0x57,
+	  0xc6, 0x14, 0x83, 0x6b, 0x63, 0xd3, 0x07, 0x63 },
+	{ 0x1a, 0x2b, 0x8e, 0xdf, 0xf3, 0xf9, 0xac, 0xc1,
+	  0x55, 0x4f, 0xcb, 0xae, 0x3c, 0xf1, 0xd6, 0x29,
+	  0x8c, 0x64, 0x62, 0xe2, 0x2e, 0x5e, 0xb0, 0x25,
+	  0x96, 0x84, 0xf8, 0x35, 0x01, 0x2b, 0xd1, 0x3f },
+	{ 0x28, 0x8c, 0x4a, 0xd9, 0xb9, 0x40, 0x97, 0x62,
+	  0xea, 0x07, 0xc2, 0x4a, 0x41, 0xf0, 0x4f, 0x69,
+	  0xa7, 0xd7, 0x4b, 0xee, 0x2d, 0x95, 0x43, 0x53,
+	  0x74, 0xbd, 0xe9, 0x46, 0xd7, 0x24, 0x1c, 0x7b },
+	{ 0x80, 0x56, 0x91, 0xbb, 0x28, 0x67, 0x48, 0xcf,
+	  0xb5, 0x91, 0xd3, 0xae, 0xbe, 0x7e, 0x6f, 0x4e,
+	  0x4d, 0xc6, 0xe2, 0x80, 0x8c, 0x65, 0x14, 0x3c,
+	  0xc0, 0x04, 0xe4, 0xeb, 0x6f, 0xd0, 0x9d, 0x43 },
+	{ 0xd4, 0xac, 0x8d, 0x3a, 0x0a, 0xfc, 0x6c, 0xfa,
+	  0x7b, 0x46, 0x0a, 0xe3, 0x00, 0x1b, 0xae, 0xb3,
+	  0x6d, 0xad, 0xb3, 0x7d, 0xa0, 0x7d, 0x2e, 0x8a,
+	  0xc9, 0x18, 0x22, 0xdf, 0x34, 0x8a, 0xed, 0x3d },
+	{ 0xc3, 0x76, 0x61, 0x70, 0x14, 0xd2, 0x01, 0x58,
+	  0xbc, 0xed, 0x3d, 0x3b, 0xa5, 0x52, 0xb6, 0xec,
+	  0xcf, 0x84, 0xe6, 0x2a, 0xa3, 0xeb, 0x65, 0x0e,
+	  0x90, 0x02, 0x9c, 0x84, 0xd1, 0x3e, 0xea, 0x69 },
+	{ 0xc4, 0x1f, 0x09, 0xf4, 0x3c, 0xec, 0xae, 0x72,
+	  0x93, 0xd6, 0x00, 0x7c, 0xa0, 0xa3, 0x57, 0x08,
+	  0x7d, 0x5a, 0xe5, 0x9b, 0xe5, 0x00, 0xc1, 0xcd,
+	  0x5b, 0x28, 0x9e, 0xe8, 0x10, 0xc7, 0xb0, 0x82 },
+	{ 0x03, 0xd1, 0xce, 0xd1, 0xfb, 0xa5, 0xc3, 0x91,
+	  0x55, 0xc4, 0x4b, 0x77, 0x65, 0xcb, 0x76, 0x0c,
+	  0x78, 0x70, 0x8d, 0xcf, 0xc8, 0x0b, 0x0b, 0xd8,
+	  0xad, 0xe3, 0xa5, 0x6d, 0xa8, 0x83, 0x0b, 0x29 },
+	{ 0x09, 0xbd, 0xe6, 0xf1, 0x52, 0x21, 0x8d, 0xc9,
+	  0x2c, 0x41, 0xd7, 0xf4, 0x53, 0x87, 0xe6, 0x3e,
+	  0x58, 0x69, 0xd8, 0x07, 0xec, 0x70, 0xb8, 0x21,
+	  0x40, 0x5d, 0xbd, 0x88, 0x4b, 0x7f, 0xcf, 0x4b },
+	{ 0x71, 0xc9, 0x03, 0x6e, 0x18, 0x17, 0x9b, 0x90,
+	  0xb3, 0x7d, 0x39, 0xe9, 0xf0, 0x5e, 0xb8, 0x9c,
+	  0xc5, 0xfc, 0x34, 0x1f, 0xd7, 0xc4, 0x77, 0xd0,
+	  0xd7, 0x49, 0x32, 0x85, 0xfa, 0xca, 0x08, 0xa4 },
+	{ 0x59, 0x16, 0x83, 0x3e, 0xbb, 0x05, 0xcd, 0x91,
+	  0x9c, 0xa7, 0xfe, 0x83, 0xb6, 0x92, 0xd3, 0x20,
+	  0x5b, 0xef, 0x72, 0x39, 0x2b, 0x2c, 0xf6, 0xbb,
+	  0x0a, 0x6d, 0x43, 0xf9, 0x94, 0xf9, 0x5f, 0x11 },
+	{ 0xf6, 0x3a, 0xab, 0x3e, 0xc6, 0x41, 0xb3, 0xb0,
+	  0x24, 0x96, 0x4c, 0x2b, 0x43, 0x7c, 0x04, 0xf6,
+	  0x04, 0x3c, 0x4c, 0x7e, 0x02, 0x79, 0x23, 0x99,
+	  0x95, 0x40, 0x19, 0x58, 0xf8, 0x6b, 0xbe, 0x54 },
+	{ 0xf1, 0x72, 0xb1, 0x80, 0xbf, 0xb0, 0x97, 0x40,
+	  0x49, 0x31, 0x20, 0xb6, 0x32, 0x6c, 0xbd, 0xc5,
+	  0x61, 0xe4, 0x77, 0xde, 0xf9, 0xbb, 0xcf, 0xd2,
+	  0x8c, 0xc8, 0xc1, 0xc5, 0xe3, 0x37, 0x9a, 0x31 },
+	{ 0xcb, 0x9b, 0x89, 0xcc, 0x18, 0x38, 0x1d, 0xd9,
+	  0x14, 0x1a, 0xde, 0x58, 0x86, 0x54, 0xd4, 0xe6,
+	  0xa2, 0x31, 0xd5, 0xbf, 0x49, 0xd4, 0xd5, 0x9a,
+	  0xc2, 0x7d, 0x86, 0x9c, 0xbe, 0x10, 0x0c, 0xf3 },
+	{ 0x7b, 0xd8, 0x81, 0x50, 0x46, 0xfd, 0xd8, 0x10,
+	  0xa9, 0x23, 0xe1, 0x98, 0x4a, 0xae, 0xbd, 0xcd,
+	  0xf8, 0x4d, 0x87, 0xc8, 0x99, 0x2d, 0x68, 0xb5,
+	  0xee, 0xb4, 0x60, 0xf9, 0x3e, 0xb3, 0xc8, 0xd7 },
+	{ 0x60, 0x7b, 0xe6, 0x68, 0x62, 0xfd, 0x08, 0xee,
+	  0x5b, 0x19, 0xfa, 0xca, 0xc0, 0x9d, 0xfd, 0xbc,
+	  0xd4, 0x0c, 0x31, 0x21, 0x01, 0xd6, 0x6e, 0x6e,
+	  0xbd, 0x2b, 0x84, 0x1f, 0x1b, 0x9a, 0x93, 0x25 },
+	{ 0x9f, 0xe0, 0x3b, 0xbe, 0x69, 0xab, 0x18, 0x34,
+	  0xf5, 0x21, 0x9b, 0x0d, 0xa8, 0x8a, 0x08, 0xb3,
+	  0x0a, 0x66, 0xc5, 0x91, 0x3f, 0x01, 0x51, 0x96,
+	  0x3c, 0x36, 0x05, 0x60, 0xdb, 0x03, 0x87, 0xb3 },
+	{ 0x90, 0xa8, 0x35, 0x85, 0x71, 0x7b, 0x75, 0xf0,
+	  0xe9, 0xb7, 0x25, 0xe0, 0x55, 0xee, 0xee, 0xb9,
+	  0xe7, 0xa0, 0x28, 0xea, 0x7e, 0x6c, 0xbc, 0x07,
+	  0xb2, 0x09, 0x17, 0xec, 0x03, 0x63, 0xe3, 0x8c },
+	{ 0x33, 0x6e, 0xa0, 0x53, 0x0f, 0x4a, 0x74, 0x69,
+	  0x12, 0x6e, 0x02, 0x18, 0x58, 0x7e, 0xbb, 0xde,
+	  0x33, 0x58, 0xa0, 0xb3, 0x1c, 0x29, 0xd2, 0x00,
+	  0xf7, 0xdc, 0x7e, 0xb1, 0x5c, 0x6a, 0xad, 0xd8 },
+	{ 0xa7, 0x9e, 0x76, 0xdc, 0x0a, 0xbc, 0xa4, 0x39,
+	  0x6f, 0x07, 0x47, 0xcd, 0x7b, 0x74, 0x8d, 0xf9,
+	  0x13, 0x00, 0x76, 0x26, 0xb1, 0xd6, 0x59, 0xda,
+	  0x0c, 0x1f, 0x78, 0xb9, 0x30, 0x3d, 0x01, 0xa3 },
+	{ 0x44, 0xe7, 0x8a, 0x77, 0x37, 0x56, 0xe0, 0x95,
+	  0x15, 0x19, 0x50, 0x4d, 0x70, 0x38, 0xd2, 0x8d,
+	  0x02, 0x13, 0xa3, 0x7e, 0x0c, 0xe3, 0x75, 0x37,
+	  0x17, 0x57, 0xbc, 0x99, 0x63, 0x11, 0xe3, 0xb8 },
+	{ 0x77, 0xac, 0x01, 0x2a, 0x3f, 0x75, 0x4d, 0xcf,
+	  0xea, 0xb5, 0xeb, 0x99, 0x6b, 0xe9, 0xcd, 0x2d,
+	  0x1f, 0x96, 0x11, 0x1b, 0x6e, 0x49, 0xf3, 0x99,
+	  0x4d, 0xf1, 0x81, 0xf2, 0x85, 0x69, 0xd8, 0x25 },
+	{ 0xce, 0x5a, 0x10, 0xdb, 0x6f, 0xcc, 0xda, 0xf1,
+	  0x40, 0xaa, 0xa4, 0xde, 0xd6, 0x25, 0x0a, 0x9c,
+	  0x06, 0xe9, 0x22, 0x2b, 0xc9, 0xf9, 0xf3, 0x65,
+	  0x8a, 0x4a, 0xff, 0x93, 0x5f, 0x2b, 0x9f, 0x3a },
+	{ 0xec, 0xc2, 0x03, 0xa7, 0xfe, 0x2b, 0xe4, 0xab,
+	  0xd5, 0x5b, 0xb5, 0x3e, 0x6e, 0x67, 0x35, 0x72,
+	  0xe0, 0x07, 0x8d, 0xa8, 0xcd, 0x37, 0x5e, 0xf4,
+	  0x30, 0xcc, 0x97, 0xf9, 0xf8, 0x00, 0x83, 0xaf },
+	{ 0x14, 0xa5, 0x18, 0x6d, 0xe9, 0xd7, 0xa1, 0x8b,
+	  0x04, 0x12, 0xb8, 0x56, 0x3e, 0x51, 0xcc, 0x54,
+	  0x33, 0x84, 0x0b, 0x4a, 0x12, 0x9a, 0x8f, 0xf9,
+	  0x63, 0xb3, 0x3a, 0x3c, 0x4a, 0xfe, 0x8e, 0xbb },
+	{ 0x13, 0xf8, 0xef, 0x95, 0xcb, 0x86, 0xe6, 0xa6,
+	  0x38, 0x93, 0x1c, 0x8e, 0x10, 0x76, 0x73, 0xeb,
+	  0x76, 0xba, 0x10, 0xd7, 0xc2, 0xcd, 0x70, 0xb9,
+	  0xd9, 0x92, 0x0b, 0xbe, 0xed, 0x92, 0x94, 0x09 },
+	{ 0x0b, 0x33, 0x8f, 0x4e, 0xe1, 0x2f, 0x2d, 0xfc,
+	  0xb7, 0x87, 0x13, 0x37, 0x79, 0x41, 0xe0, 0xb0,
+	  0x63, 0x21, 0x52, 0x58, 0x1d, 0x13, 0x32, 0x51,
+	  0x6e, 0x4a, 0x2c, 0xab, 0x19, 0x42, 0xcc, 0xa4 },
+	{ 0xea, 0xab, 0x0e, 0xc3, 0x7b, 0x3b, 0x8a, 0xb7,
+	  0x96, 0xe9, 0xf5, 0x72, 0x38, 0xde, 0x14, 0xa2,
+	  0x64, 0xa0, 0x76, 0xf3, 0x88, 0x7d, 0x86, 0xe2,
+	  0x9b, 0xb5, 0x90, 0x6d, 0xb5, 0xa0, 0x0e, 0x02 },
+	{ 0x23, 0xcb, 0x68, 0xb8, 0xc0, 0xe6, 0xdc, 0x26,
+	  0xdc, 0x27, 0x76, 0x6d, 0xdc, 0x0a, 0x13, 0xa9,
+	  0x94, 0x38, 0xfd, 0x55, 0x61, 0x7a, 0xa4, 0x09,
+	  0x5d, 0x8f, 0x96, 0x97, 0x20, 0xc8, 0x72, 0xdf },
+	{ 0x09, 0x1d, 0x8e, 0xe3, 0x0d, 0x6f, 0x29, 0x68,
+	  0xd4, 0x6b, 0x68, 0x7d, 0xd6, 0x52, 0x92, 0x66,
+	  0x57, 0x42, 0xde, 0x0b, 0xb8, 0x3d, 0xcc, 0x00,
+	  0x04, 0xc7, 0x2c, 0xe1, 0x00, 0x07, 0xa5, 0x49 },
+	{ 0x7f, 0x50, 0x7a, 0xbc, 0x6d, 0x19, 0xba, 0x00,
+	  0xc0, 0x65, 0xa8, 0x76, 0xec, 0x56, 0x57, 0x86,
+	  0x88, 0x82, 0xd1, 0x8a, 0x22, 0x1b, 0xc4, 0x6c,
+	  0x7a, 0x69, 0x12, 0x54, 0x1f, 0x5b, 0xc7, 0xba },
+	{ 0xa0, 0x60, 0x7c, 0x24, 0xe1, 0x4e, 0x8c, 0x22,
+	  0x3d, 0xb0, 0xd7, 0x0b, 0x4d, 0x30, 0xee, 0x88,
+	  0x01, 0x4d, 0x60, 0x3f, 0x43, 0x7e, 0x9e, 0x02,
+	  0xaa, 0x7d, 0xaf, 0xa3, 0xcd, 0xfb, 0xad, 0x94 },
+	{ 0xdd, 0xbf, 0xea, 0x75, 0xcc, 0x46, 0x78, 0x82,
+	  0xeb, 0x34, 0x83, 0xce, 0x5e, 0x2e, 0x75, 0x6a,
+	  0x4f, 0x47, 0x01, 0xb7, 0x6b, 0x44, 0x55, 0x19,
+	  0xe8, 0x9f, 0x22, 0xd6, 0x0f, 0xa8, 0x6e, 0x06 },
+	{ 0x0c, 0x31, 0x1f, 0x38, 0xc3, 0x5a, 0x4f, 0xb9,
+	  0x0d, 0x65, 0x1c, 0x28, 0x9d, 0x48, 0x68, 0x56,
+	  0xcd, 0x14, 0x13, 0xdf, 0x9b, 0x06, 0x77, 0xf5,
+	  0x3e, 0xce, 0x2c, 0xd9, 0xe4, 0x77, 0xc6, 0x0a },
+	{ 0x46, 0xa7, 0x3a, 0x8d, 0xd3, 0xe7, 0x0f, 0x59,
+	  0xd3, 0x94, 0x2c, 0x01, 0xdf, 0x59, 0x9d, 0xef,
+	  0x78, 0x3c, 0x9d, 0xa8, 0x2f, 0xd8, 0x32, 0x22,
+	  0xcd, 0x66, 0x2b, 0x53, 0xdc, 0xe7, 0xdb, 0xdf },
+	{ 0xad, 0x03, 0x8f, 0xf9, 0xb1, 0x4d, 0xe8, 0x4a,
+	  0x80, 0x1e, 0x4e, 0x62, 0x1c, 0xe5, 0xdf, 0x02,
+	  0x9d, 0xd9, 0x35, 0x20, 0xd0, 0xc2, 0xfa, 0x38,
+	  0xbf, 0xf1, 0x76, 0xa8, 0xb1, 0xd1, 0x69, 0x8c },
+	{ 0xab, 0x70, 0xc5, 0xdf, 0xbd, 0x1e, 0xa8, 0x17,
+	  0xfe, 0xd0, 0xcd, 0x06, 0x72, 0x93, 0xab, 0xf3,
+	  0x19, 0xe5, 0xd7, 0x90, 0x1c, 0x21, 0x41, 0xd5,
+	  0xd9, 0x9b, 0x23, 0xf0, 0x3a, 0x38, 0xe7, 0x48 },
+	{ 0x1f, 0xff, 0xda, 0x67, 0x93, 0x2b, 0x73, 0xc8,
+	  0xec, 0xaf, 0x00, 0x9a, 0x34, 0x91, 0xa0, 0x26,
+	  0x95, 0x3b, 0xab, 0xfe, 0x1f, 0x66, 0x3b, 0x06,
+	  0x97, 0xc3, 0xc4, 0xae, 0x8b, 0x2e, 0x7d, 0xcb },
+	{ 0xb0, 0xd2, 0xcc, 0x19, 0x47, 0x2d, 0xd5, 0x7f,
+	  0x2b, 0x17, 0xef, 0xc0, 0x3c, 0x8d, 0x58, 0xc2,
+	  0x28, 0x3d, 0xbb, 0x19, 0xda, 0x57, 0x2f, 0x77,
+	  0x55, 0x85, 0x5a, 0xa9, 0x79, 0x43, 0x17, 0xa0 },
+	{ 0xa0, 0xd1, 0x9a, 0x6e, 0xe3, 0x39, 0x79, 0xc3,
+	  0x25, 0x51, 0x0e, 0x27, 0x66, 0x22, 0xdf, 0x41,
+	  0xf7, 0x15, 0x83, 0xd0, 0x75, 0x01, 0xb8, 0x70,
+	  0x71, 0x12, 0x9a, 0x0a, 0xd9, 0x47, 0x32, 0xa5 },
+	{ 0x72, 0x46, 0x42, 0xa7, 0x03, 0x2d, 0x10, 0x62,
+	  0xb8, 0x9e, 0x52, 0xbe, 0xa3, 0x4b, 0x75, 0xdf,
+	  0x7d, 0x8f, 0xe7, 0x72, 0xd9, 0xfe, 0x3c, 0x93,
+	  0xdd, 0xf3, 0xc4, 0x54, 0x5a, 0xb5, 0xa9, 0x9b },
+	{ 0xad, 0xe5, 0xea, 0xa7, 0xe6, 0x1f, 0x67, 0x2d,
+	  0x58, 0x7e, 0xa0, 0x3d, 0xae, 0x7d, 0x7b, 0x55,
+	  0x22, 0x9c, 0x01, 0xd0, 0x6b, 0xc0, 0xa5, 0x70,
+	  0x14, 0x36, 0xcb, 0xd1, 0x83, 0x66, 0xa6, 0x26 },
+	{ 0x01, 0x3b, 0x31, 0xeb, 0xd2, 0x28, 0xfc, 0xdd,
+	  0xa5, 0x1f, 0xab, 0xb0, 0x3b, 0xb0, 0x2d, 0x60,
+	  0xac, 0x20, 0xca, 0x21, 0x5a, 0xaf, 0xa8, 0x3b,
+	  0xdd, 0x85, 0x5e, 0x37, 0x55, 0xa3, 0x5f, 0x0b },
+	{ 0x33, 0x2e, 0xd4, 0x0b, 0xb1, 0x0d, 0xde, 0x3c,
+	  0x95, 0x4a, 0x75, 0xd7, 0xb8, 0x99, 0x9d, 0x4b,
+	  0x26, 0xa1, 0xc0, 0x63, 0xc1, 0xdc, 0x6e, 0x32,
+	  0xc1, 0xd9, 0x1b, 0xab, 0x7b, 0xbb, 0x7d, 0x16 },
+	{ 0xc7, 0xa1, 0x97, 0xb3, 0xa0, 0x5b, 0x56, 0x6b,
+	  0xcc, 0x9f, 0xac, 0xd2, 0x0e, 0x44, 0x1d, 0x6f,
+	  0x6c, 0x28, 0x60, 0xac, 0x96, 0x51, 0xcd, 0x51,
+	  0xd6, 0xb9, 0xd2, 0xcd, 0xee, 0xea, 0x03, 0x90 },
+	{ 0xbd, 0x9c, 0xf6, 0x4e, 0xa8, 0x95, 0x3c, 0x03,
+	  0x71, 0x08, 0xe6, 0xf6, 0x54, 0x91, 0x4f, 0x39,
+	  0x58, 0xb6, 0x8e, 0x29, 0xc1, 0x67, 0x00, 0xdc,
+	  0x18, 0x4d, 0x94, 0xa2, 0x17, 0x08, 0xff, 0x60 },
+	{ 0x88, 0x35, 0xb0, 0xac, 0x02, 0x11, 0x51, 0xdf,
+	  0x71, 0x64, 0x74, 0xce, 0x27, 0xce, 0x4d, 0x3c,
+	  0x15, 0xf0, 0xb2, 0xda, 0xb4, 0x80, 0x03, 0xcf,
+	  0x3f, 0x3e, 0xfd, 0x09, 0x45, 0x10, 0x6b, 0x9a },
+	{ 0x3b, 0xfe, 0xfa, 0x33, 0x01, 0xaa, 0x55, 0xc0,
+	  0x80, 0x19, 0x0c, 0xff, 0xda, 0x8e, 0xae, 0x51,
+	  0xd9, 0xaf, 0x48, 0x8b, 0x4c, 0x1f, 0x24, 0xc3,
+	  0xd9, 0xa7, 0x52, 0x42, 0xfd, 0x8e, 0xa0, 0x1d },
+	{ 0x08, 0x28, 0x4d, 0x14, 0x99, 0x3c, 0xd4, 0x7d,
+	  0x53, 0xeb, 0xae, 0xcf, 0x0d, 0xf0, 0x47, 0x8c,
+	  0xc1, 0x82, 0xc8, 0x9c, 0x00, 0xe1, 0x85, 0x9c,
+	  0x84, 0x85, 0x16, 0x86, 0xdd, 0xf2, 0xc1, 0xb7 },
+	{ 0x1e, 0xd7, 0xef, 0x9f, 0x04, 0xc2, 0xac, 0x8d,
+	  0xb6, 0xa8, 0x64, 0xdb, 0x13, 0x10, 0x87, 0xf2,
+	  0x70, 0x65, 0x09, 0x8e, 0x69, 0xc3, 0xfe, 0x78,
+	  0x71, 0x8d, 0x9b, 0x94, 0x7f, 0x4a, 0x39, 0xd0 },
+	{ 0xc1, 0x61, 0xf2, 0xdc, 0xd5, 0x7e, 0x9c, 0x14,
+	  0x39, 0xb3, 0x1a, 0x9d, 0xd4, 0x3d, 0x8f, 0x3d,
+	  0x7d, 0xd8, 0xf0, 0xeb, 0x7c, 0xfa, 0xc6, 0xfb,
+	  0x25, 0xa0, 0xf2, 0x8e, 0x30, 0x6f, 0x06, 0x61 },
+	{ 0xc0, 0x19, 0x69, 0xad, 0x34, 0xc5, 0x2c, 0xaf,
+	  0x3d, 0xc4, 0xd8, 0x0d, 0x19, 0x73, 0x5c, 0x29,
+	  0x73, 0x1a, 0xc6, 0xe7, 0xa9, 0x20, 0x85, 0xab,
+	  0x92, 0x50, 0xc4, 0x8d, 0xea, 0x48, 0xa3, 0xfc },
+	{ 0x17, 0x20, 0xb3, 0x65, 0x56, 0x19, 0xd2, 0xa5,
+	  0x2b, 0x35, 0x21, 0xae, 0x0e, 0x49, 0xe3, 0x45,
+	  0xcb, 0x33, 0x89, 0xeb, 0xd6, 0x20, 0x8a, 0xca,
+	  0xf9, 0xf1, 0x3f, 0xda, 0xcc, 0xa8, 0xbe, 0x49 },
+	{ 0x75, 0x62, 0x88, 0x36, 0x1c, 0x83, 0xe2, 0x4c,
+	  0x61, 0x7c, 0xf9, 0x5c, 0x90, 0x5b, 0x22, 0xd0,
+	  0x17, 0xcd, 0xc8, 0x6f, 0x0b, 0xf1, 0xd6, 0x58,
+	  0xf4, 0x75, 0x6c, 0x73, 0x79, 0x87, 0x3b, 0x7f },
+	{ 0xe7, 0xd0, 0xed, 0xa3, 0x45, 0x26, 0x93, 0xb7,
+	  0x52, 0xab, 0xcd, 0xa1, 0xb5, 0x5e, 0x27, 0x6f,
+	  0x82, 0x69, 0x8f, 0x5f, 0x16, 0x05, 0x40, 0x3e,
+	  0xff, 0x83, 0x0b, 0xea, 0x00, 0x71, 0xa3, 0x94 },
+	{ 0x2c, 0x82, 0xec, 0xaa, 0x6b, 0x84, 0x80, 0x3e,
+	  0x04, 0x4a, 0xf6, 0x31, 0x18, 0xaf, 0xe5, 0x44,
+	  0x68, 0x7c, 0xb6, 0xe6, 0xc7, 0xdf, 0x49, 0xed,
+	  0x76, 0x2d, 0xfd, 0x7c, 0x86, 0x93, 0xa1, 0xbc },
+	{ 0x61, 0x36, 0xcb, 0xf4, 0xb4, 0x41, 0x05, 0x6f,
+	  0xa1, 0xe2, 0x72, 0x24, 0x98, 0x12, 0x5d, 0x6d,
+	  0xed, 0x45, 0xe1, 0x7b, 0x52, 0x14, 0x39, 0x59,
+	  0xc7, 0xf4, 0xd4, 0xe3, 0x95, 0x21, 0x8a, 0xc2 },
+	{ 0x72, 0x1d, 0x32, 0x45, 0xaa, 0xfe, 0xf2, 0x7f,
+	  0x6a, 0x62, 0x4f, 0x47, 0x95, 0x4b, 0x6c, 0x25,
+	  0x50, 0x79, 0x52, 0x6f, 0xfa, 0x25, 0xe9, 0xff,
+	  0x77, 0xe5, 0xdc, 0xff, 0x47, 0x3b, 0x15, 0x97 },
+	{ 0x9d, 0xd2, 0xfb, 0xd8, 0xce, 0xf1, 0x6c, 0x35,
+	  0x3c, 0x0a, 0xc2, 0x11, 0x91, 0xd5, 0x09, 0xeb,
+	  0x28, 0xdd, 0x9e, 0x3e, 0x0d, 0x8c, 0xea, 0x5d,
+	  0x26, 0xca, 0x83, 0x93, 0x93, 0x85, 0x1c, 0x3a },
+	{ 0xb2, 0x39, 0x4c, 0xea, 0xcd, 0xeb, 0xf2, 0x1b,
+	  0xf9, 0xdf, 0x2c, 0xed, 0x98, 0xe5, 0x8f, 0x1c,
+	  0x3a, 0x4b, 0xbb, 0xff, 0x66, 0x0d, 0xd9, 0x00,
+	  0xf6, 0x22, 0x02, 0xd6, 0x78, 0x5c, 0xc4, 0x6e },
+	{ 0x57, 0x08, 0x9f, 0x22, 0x27, 0x49, 0xad, 0x78,
+	  0x71, 0x76, 0x5f, 0x06, 0x2b, 0x11, 0x4f, 0x43,
+	  0xba, 0x20, 0xec, 0x56, 0x42, 0x2a, 0x8b, 0x1e,
+	  0x3f, 0x87, 0x19, 0x2c, 0x0e, 0xa7, 0x18, 0xc6 },
+	{ 0xe4, 0x9a, 0x94, 0x59, 0x96, 0x1c, 0xd3, 0x3c,
+	  0xdf, 0x4a, 0xae, 0x1b, 0x10, 0x78, 0xa5, 0xde,
+	  0xa7, 0xc0, 0x40, 0xe0, 0xfe, 0xa3, 0x40, 0xc9,
+	  0x3a, 0x72, 0x48, 0x72, 0xfc, 0x4a, 0xf8, 0x06 },
+	{ 0xed, 0xe6, 0x7f, 0x72, 0x0e, 0xff, 0xd2, 0xca,
+	  0x9c, 0x88, 0x99, 0x41, 0x52, 0xd0, 0x20, 0x1d,
+	  0xee, 0x6b, 0x0a, 0x2d, 0x2c, 0x07, 0x7a, 0xca,
+	  0x6d, 0xae, 0x29, 0xf7, 0x3f, 0x8b, 0x63, 0x09 },
+	{ 0xe0, 0xf4, 0x34, 0xbf, 0x22, 0xe3, 0x08, 0x80,
+	  0x39, 0xc2, 0x1f, 0x71, 0x9f, 0xfc, 0x67, 0xf0,
+	  0xf2, 0xcb, 0x5e, 0x98, 0xa7, 0xa0, 0x19, 0x4c,
+	  0x76, 0xe9, 0x6b, 0xf4, 0xe8, 0xe1, 0x7e, 0x61 },
+	{ 0x27, 0x7c, 0x04, 0xe2, 0x85, 0x34, 0x84, 0xa4,
+	  0xeb, 0xa9, 0x10, 0xad, 0x33, 0x6d, 0x01, 0xb4,
+	  0x77, 0xb6, 0x7c, 0xc2, 0x00, 0xc5, 0x9f, 0x3c,
+	  0x8d, 0x77, 0xee, 0xf8, 0x49, 0x4f, 0x29, 0xcd },
+	{ 0x15, 0x6d, 0x57, 0x47, 0xd0, 0xc9, 0x9c, 0x7f,
+	  0x27, 0x09, 0x7d, 0x7b, 0x7e, 0x00, 0x2b, 0x2e,
+	  0x18, 0x5c, 0xb7, 0x2d, 0x8d, 0xd7, 0xeb, 0x42,
+	  0x4a, 0x03, 0x21, 0x52, 0x81, 0x61, 0x21, 0x9f },
+	{ 0x20, 0xdd, 0xd1, 0xed, 0x9b, 0x1c, 0xa8, 0x03,
+	  0x94, 0x6d, 0x64, 0xa8, 0x3a, 0xe4, 0x65, 0x9d,
+	  0xa6, 0x7f, 0xba, 0x7a, 0x1a, 0x3e, 0xdd, 0xb1,
+	  0xe1, 0x03, 0xc0, 0xf5, 0xe0, 0x3e, 0x3a, 0x2c },
+	{ 0xf0, 0xaf, 0x60, 0x4d, 0x3d, 0xab, 0xbf, 0x9a,
+	  0x0f, 0x2a, 0x7d, 0x3d, 0xda, 0x6b, 0xd3, 0x8b,
+	  0xba, 0x72, 0xc6, 0xd0, 0x9b, 0xe4, 0x94, 0xfc,
+	  0xef, 0x71, 0x3f, 0xf1, 0x01, 0x89, 0xb6, 0xe6 },
+	{ 0x98, 0x02, 0xbb, 0x87, 0xde, 0xf4, 0xcc, 0x10,
+	  0xc4, 0xa5, 0xfd, 0x49, 0xaa, 0x58, 0xdf, 0xe2,
+	  0xf3, 0xfd, 0xdb, 0x46, 0xb4, 0x70, 0x88, 0x14,
+	  0xea, 0xd8, 0x1d, 0x23, 0xba, 0x95, 0x13, 0x9b },
+	{ 0x4f, 0x8c, 0xe1, 0xe5, 0x1d, 0x2f, 0xe7, 0xf2,
+	  0x40, 0x43, 0xa9, 0x04, 0xd8, 0x98, 0xeb, 0xfc,
+	  0x91, 0x97, 0x54, 0x18, 0x75, 0x34, 0x13, 0xaa,
+	  0x09, 0x9b, 0x79, 0x5e, 0xcb, 0x35, 0xce, 0xdb },
+	{ 0xbd, 0xdc, 0x65, 0x14, 0xd7, 0xee, 0x6a, 0xce,
+	  0x0a, 0x4a, 0xc1, 0xd0, 0xe0, 0x68, 0x11, 0x22,
+	  0x88, 0xcb, 0xcf, 0x56, 0x04, 0x54, 0x64, 0x27,
+	  0x05, 0x63, 0x01, 0x77, 0xcb, 0xa6, 0x08, 0xbd },
+	{ 0xd6, 0x35, 0x99, 0x4f, 0x62, 0x91, 0x51, 0x7b,
+	  0x02, 0x81, 0xff, 0xdd, 0x49, 0x6a, 0xfa, 0x86,
+	  0x27, 0x12, 0xe5, 0xb3, 0xc4, 0xe5, 0x2e, 0x4c,
+	  0xd5, 0xfd, 0xae, 0x8c, 0x0e, 0x72, 0xfb, 0x08 },
+	{ 0x87, 0x8d, 0x9c, 0xa6, 0x00, 0xcf, 0x87, 0xe7,
+	  0x69, 0xcc, 0x30, 0x5c, 0x1b, 0x35, 0x25, 0x51,
+	  0x86, 0x61, 0x5a, 0x73, 0xa0, 0xda, 0x61, 0x3b,
+	  0x5f, 0x1c, 0x98, 0xdb, 0xf8, 0x12, 0x83, 0xea },
+	{ 0xa6, 0x4e, 0xbe, 0x5d, 0xc1, 0x85, 0xde, 0x9f,
+	  0xdd, 0xe7, 0x60, 0x7b, 0x69, 0x98, 0x70, 0x2e,
+	  0xb2, 0x34, 0x56, 0x18, 0x49, 0x57, 0x30, 0x7d,
+	  0x2f, 0xa7, 0x2e, 0x87, 0xa4, 0x77, 0x02, 0xd6 },
+	{ 0xce, 0x50, 0xea, 0xb7, 0xb5, 0xeb, 0x52, 0xbd,
+	  0xc9, 0xad, 0x8e, 0x5a, 0x48, 0x0a, 0xb7, 0x80,
+	  0xca, 0x93, 0x20, 0xe4, 0x43, 0x60, 0xb1, 0xfe,
+	  0x37, 0xe0, 0x3f, 0x2f, 0x7a, 0xd7, 0xde, 0x01 },
+	{ 0xee, 0xdd, 0xb7, 0xc0, 0xdb, 0x6e, 0x30, 0xab,
+	  0xe6, 0x6d, 0x79, 0xe3, 0x27, 0x51, 0x1e, 0x61,
+	  0xfc, 0xeb, 0xbc, 0x29, 0xf1, 0x59, 0xb4, 0x0a,
+	  0x86, 0xb0, 0x46, 0xec, 0xf0, 0x51, 0x38, 0x23 },
+	{ 0x78, 0x7f, 0xc9, 0x34, 0x40, 0xc1, 0xec, 0x96,
+	  0xb5, 0xad, 0x01, 0xc1, 0x6c, 0xf7, 0x79, 0x16,
+	  0xa1, 0x40, 0x5f, 0x94, 0x26, 0x35, 0x6e, 0xc9,
+	  0x21, 0xd8, 0xdf, 0xf3, 0xea, 0x63, 0xb7, 0xe0 },
+	{ 0x7f, 0x0d, 0x5e, 0xab, 0x47, 0xee, 0xfd, 0xa6,
+	  0x96, 0xc0, 0xbf, 0x0f, 0xbf, 0x86, 0xab, 0x21,
+	  0x6f, 0xce, 0x46, 0x1e, 0x93, 0x03, 0xab, 0xa6,
+	  0xac, 0x37, 0x41, 0x20, 0xe8, 0x90, 0xe8, 0xdf },
+	{ 0xb6, 0x80, 0x04, 0xb4, 0x2f, 0x14, 0xad, 0x02,
+	  0x9f, 0x4c, 0x2e, 0x03, 0xb1, 0xd5, 0xeb, 0x76,
+	  0xd5, 0x71, 0x60, 0xe2, 0x64, 0x76, 0xd2, 0x11,
+	  0x31, 0xbe, 0xf2, 0x0a, 0xda, 0x7d, 0x27, 0xf4 },
+	{ 0xb0, 0xc4, 0xeb, 0x18, 0xae, 0x25, 0x0b, 0x51,
+	  0xa4, 0x13, 0x82, 0xea, 0xd9, 0x2d, 0x0d, 0xc7,
+	  0x45, 0x5f, 0x93, 0x79, 0xfc, 0x98, 0x84, 0x42,
+	  0x8e, 0x47, 0x70, 0x60, 0x8d, 0xb0, 0xfa, 0xec },
+	{ 0xf9, 0x2b, 0x7a, 0x87, 0x0c, 0x05, 0x9f, 0x4d,
+	  0x46, 0x46, 0x4c, 0x82, 0x4e, 0xc9, 0x63, 0x55,
+	  0x14, 0x0b, 0xdc, 0xe6, 0x81, 0x32, 0x2c, 0xc3,
+	  0xa9, 0x92, 0xff, 0x10, 0x3e, 0x3f, 0xea, 0x52 },
+	{ 0x53, 0x64, 0x31, 0x26, 0x14, 0x81, 0x33, 0x98,
+	  0xcc, 0x52, 0x5d, 0x4c, 0x4e, 0x14, 0x6e, 0xde,
+	  0xb3, 0x71, 0x26, 0x5f, 0xba, 0x19, 0x13, 0x3a,
+	  0x2c, 0x3d, 0x21, 0x59, 0x29, 0x8a, 0x17, 0x42 },
+	{ 0xf6, 0x62, 0x0e, 0x68, 0xd3, 0x7f, 0xb2, 0xaf,
+	  0x50, 0x00, 0xfc, 0x28, 0xe2, 0x3b, 0x83, 0x22,
+	  0x97, 0xec, 0xd8, 0xbc, 0xe9, 0x9e, 0x8b, 0xe4,
+	  0xd0, 0x4e, 0x85, 0x30, 0x9e, 0x3d, 0x33, 0x74 },
+	{ 0x53, 0x16, 0xa2, 0x79, 0x69, 0xd7, 0xfe, 0x04,
+	  0xff, 0x27, 0xb2, 0x83, 0x96, 0x1b, 0xff, 0xc3,
+	  0xbf, 0x5d, 0xfb, 0x32, 0xfb, 0x6a, 0x89, 0xd1,
+	  0x01, 0xc6, 0xc3, 0xb1, 0x93, 0x7c, 0x28, 0x71 },
+	{ 0x81, 0xd1, 0x66, 0x4f, 0xdf, 0x3c, 0xb3, 0x3c,
+	  0x24, 0xee, 0xba, 0xc0, 0xbd, 0x64, 0x24, 0x4b,
+	  0x77, 0xc4, 0xab, 0xea, 0x90, 0xbb, 0xe8, 0xb5,
+	  0xee, 0x0b, 0x2a, 0xaf, 0xcf, 0x2d, 0x6a, 0x53 },
+	{ 0x34, 0x57, 0x82, 0xf2, 0x95, 0xb0, 0x88, 0x03,
+	  0x52, 0xe9, 0x24, 0xa0, 0x46, 0x7b, 0x5f, 0xbc,
+	  0x3e, 0x8f, 0x3b, 0xfb, 0xc3, 0xc7, 0xe4, 0x8b,
+	  0x67, 0x09, 0x1f, 0xb5, 0xe8, 0x0a, 0x94, 0x42 },
+	{ 0x79, 0x41, 0x11, 0xea, 0x6c, 0xd6, 0x5e, 0x31,
+	  0x1f, 0x74, 0xee, 0x41, 0xd4, 0x76, 0xcb, 0x63,
+	  0x2c, 0xe1, 0xe4, 0xb0, 0x51, 0xdc, 0x1d, 0x9e,
+	  0x9d, 0x06, 0x1a, 0x19, 0xe1, 0xd0, 0xbb, 0x49 },
+	{ 0x2a, 0x85, 0xda, 0xf6, 0x13, 0x88, 0x16, 0xb9,
+	  0x9b, 0xf8, 0xd0, 0x8b, 0xa2, 0x11, 0x4b, 0x7a,
+	  0xb0, 0x79, 0x75, 0xa7, 0x84, 0x20, 0xc1, 0xa3,
+	  0xb0, 0x6a, 0x77, 0x7c, 0x22, 0xdd, 0x8b, 0xcb },
+	{ 0x89, 0xb0, 0xd5, 0xf2, 0x89, 0xec, 0x16, 0x40,
+	  0x1a, 0x06, 0x9a, 0x96, 0x0d, 0x0b, 0x09, 0x3e,
+	  0x62, 0x5d, 0xa3, 0xcf, 0x41, 0xee, 0x29, 0xb5,
+	  0x9b, 0x93, 0x0c, 0x58, 0x20, 0x14, 0x54, 0x55 },
+	{ 0xd0, 0xfd, 0xcb, 0x54, 0x39, 0x43, 0xfc, 0x27,
+	  0xd2, 0x08, 0x64, 0xf5, 0x21, 0x81, 0x47, 0x1b,
+	  0x94, 0x2c, 0xc7, 0x7c, 0xa6, 0x75, 0xbc, 0xb3,
+	  0x0d, 0xf3, 0x1d, 0x35, 0x8e, 0xf7, 0xb1, 0xeb },
+	{ 0xb1, 0x7e, 0xa8, 0xd7, 0x70, 0x63, 0xc7, 0x09,
+	  0xd4, 0xdc, 0x6b, 0x87, 0x94, 0x13, 0xc3, 0x43,
+	  0xe3, 0x79, 0x0e, 0x9e, 0x62, 0xca, 0x85, 0xb7,
+	  0x90, 0x0b, 0x08, 0x6f, 0x6b, 0x75, 0xc6, 0x72 },
+	{ 0xe7, 0x1a, 0x3e, 0x2c, 0x27, 0x4d, 0xb8, 0x42,
+	  0xd9, 0x21, 0x14, 0xf2, 0x17, 0xe2, 0xc0, 0xea,
+	  0xc8, 0xb4, 0x50, 0x93, 0xfd, 0xfd, 0x9d, 0xf4,
+	  0xca, 0x71, 0x62, 0x39, 0x48, 0x62, 0xd5, 0x01 },
+	{ 0xc0, 0x47, 0x67, 0x59, 0xab, 0x7a, 0xa3, 0x33,
+	  0x23, 0x4f, 0x6b, 0x44, 0xf5, 0xfd, 0x85, 0x83,
+	  0x90, 0xec, 0x23, 0x69, 0x4c, 0x62, 0x2c, 0xb9,
+	  0x86, 0xe7, 0x69, 0xc7, 0x8e, 0xdd, 0x73, 0x3e },
+	{ 0x9a, 0xb8, 0xea, 0xbb, 0x14, 0x16, 0x43, 0x4d,
+	  0x85, 0x39, 0x13, 0x41, 0xd5, 0x69, 0x93, 0xc5,
+	  0x54, 0x58, 0x16, 0x7d, 0x44, 0x18, 0xb1, 0x9a,
+	  0x0f, 0x2a, 0xd8, 0xb7, 0x9a, 0x83, 0xa7, 0x5b },
+	{ 0x79, 0x92, 0xd0, 0xbb, 0xb1, 0x5e, 0x23, 0x82,
+	  0x6f, 0x44, 0x3e, 0x00, 0x50, 0x5d, 0x68, 0xd3,
+	  0xed, 0x73, 0x72, 0x99, 0x5a, 0x5c, 0x3e, 0x49,
+	  0x86, 0x54, 0x10, 0x2f, 0xbc, 0xd0, 0x96, 0x4e },
+	{ 0xc0, 0x21, 0xb3, 0x00, 0x85, 0x15, 0x14, 0x35,
+	  0xdf, 0x33, 0xb0, 0x07, 0xcc, 0xec, 0xc6, 0x9d,
+	  0xf1, 0x26, 0x9f, 0x39, 0xba, 0x25, 0x09, 0x2b,
+	  0xed, 0x59, 0xd9, 0x32, 0xac, 0x0f, 0xdc, 0x28 },
+	{ 0x91, 0xa2, 0x5e, 0xc0, 0xec, 0x0d, 0x9a, 0x56,
+	  0x7f, 0x89, 0xc4, 0xbf, 0xe1, 0xa6, 0x5a, 0x0e,
+	  0x43, 0x2d, 0x07, 0x06, 0x4b, 0x41, 0x90, 0xe2,
+	  0x7d, 0xfb, 0x81, 0x90, 0x1f, 0xd3, 0x13, 0x9b },
+	{ 0x59, 0x50, 0xd3, 0x9a, 0x23, 0xe1, 0x54, 0x5f,
+	  0x30, 0x12, 0x70, 0xaa, 0x1a, 0x12, 0xf2, 0xe6,
+	  0xc4, 0x53, 0x77, 0x6e, 0x4d, 0x63, 0x55, 0xde,
+	  0x42, 0x5c, 0xc1, 0x53, 0xf9, 0x81, 0x88, 0x67 },
+	{ 0xd7, 0x9f, 0x14, 0x72, 0x0c, 0x61, 0x0a, 0xf1,
+	  0x79, 0xa3, 0x76, 0x5d, 0x4b, 0x7c, 0x09, 0x68,
+	  0xf9, 0x77, 0x96, 0x2d, 0xbf, 0x65, 0x5b, 0x52,
+	  0x12, 0x72, 0xb6, 0xf1, 0xe1, 0x94, 0x48, 0x8e },
+	{ 0xe9, 0x53, 0x1b, 0xfc, 0x8b, 0x02, 0x99, 0x5a,
+	  0xea, 0xa7, 0x5b, 0xa2, 0x70, 0x31, 0xfa, 0xdb,
+	  0xcb, 0xf4, 0xa0, 0xda, 0xb8, 0x96, 0x1d, 0x92,
+	  0x96, 0xcd, 0x7e, 0x84, 0xd2, 0x5d, 0x60, 0x06 },
+	{ 0x34, 0xe9, 0xc2, 0x6a, 0x01, 0xd7, 0xf1, 0x61,
+	  0x81, 0xb4, 0x54, 0xa9, 0xd1, 0x62, 0x3c, 0x23,
+	  0x3c, 0xb9, 0x9d, 0x31, 0xc6, 0x94, 0x65, 0x6e,
+	  0x94, 0x13, 0xac, 0xa3, 0xe9, 0x18, 0x69, 0x2f },
+	{ 0xd9, 0xd7, 0x42, 0x2f, 0x43, 0x7b, 0xd4, 0x39,
+	  0xdd, 0xd4, 0xd8, 0x83, 0xda, 0xe2, 0xa0, 0x83,
+	  0x50, 0x17, 0x34, 0x14, 0xbe, 0x78, 0x15, 0x51,
+	  0x33, 0xff, 0xf1, 0x96, 0x4c, 0x3d, 0x79, 0x72 },
+	{ 0x4a, 0xee, 0x0c, 0x7a, 0xaf, 0x07, 0x54, 0x14,
+	  0xff, 0x17, 0x93, 0xea, 0xd7, 0xea, 0xca, 0x60,
+	  0x17, 0x75, 0xc6, 0x15, 0xdb, 0xd6, 0x0b, 0x64,
+	  0x0b, 0x0a, 0x9f, 0x0c, 0xe5, 0x05, 0xd4, 0x35 },
+	{ 0x6b, 0xfd, 0xd1, 0x54, 0x59, 0xc8, 0x3b, 0x99,
+	  0xf0, 0x96, 0xbf, 0xb4, 0x9e, 0xe8, 0x7b, 0x06,
+	  0x3d, 0x69, 0xc1, 0x97, 0x4c, 0x69, 0x28, 0xac,
+	  0xfc, 0xfb, 0x40, 0x99, 0xf8, 0xc4, 0xef, 0x67 },
+	{ 0x9f, 0xd1, 0xc4, 0x08, 0xfd, 0x75, 0xc3, 0x36,
+	  0x19, 0x3a, 0x2a, 0x14, 0xd9, 0x4f, 0x6a, 0xf5,
+	  0xad, 0xf0, 0x50, 0xb8, 0x03, 0x87, 0xb4, 0xb0,
+	  0x10, 0xfb, 0x29, 0xf4, 0xcc, 0x72, 0x70, 0x7c },
+	{ 0x13, 0xc8, 0x84, 0x80, 0xa5, 0xd0, 0x0d, 0x6c,
+	  0x8c, 0x7a, 0xd2, 0x11, 0x0d, 0x76, 0xa8, 0x2d,
+	  0x9b, 0x70, 0xf4, 0xfa, 0x66, 0x96, 0xd4, 0xe5,
+	  0xdd, 0x42, 0xa0, 0x66, 0xdc, 0xaf, 0x99, 0x20 },
+	{ 0x82, 0x0e, 0x72, 0x5e, 0xe2, 0x5f, 0xe8, 0xfd,
+	  0x3a, 0x8d, 0x5a, 0xbe, 0x4c, 0x46, 0xc3, 0xba,
+	  0x88, 0x9d, 0xe6, 0xfa, 0x91, 0x91, 0xaa, 0x22,
+	  0xba, 0x67, 0xd5, 0x70, 0x54, 0x21, 0x54, 0x2b },
+	{ 0x32, 0xd9, 0x3a, 0x0e, 0xb0, 0x2f, 0x42, 0xfb,
+	  0xbc, 0xaf, 0x2b, 0xad, 0x00, 0x85, 0xb2, 0x82,
+	  0xe4, 0x60, 0x46, 0xa4, 0xdf, 0x7a, 0xd1, 0x06,
+	  0x57, 0xc9, 0xd6, 0x47, 0x63, 0x75, 0xb9, 0x3e },
+	{ 0xad, 0xc5, 0x18, 0x79, 0x05, 0xb1, 0x66, 0x9c,
+	  0xd8, 0xec, 0x9c, 0x72, 0x1e, 0x19, 0x53, 0x78,
+	  0x6b, 0x9d, 0x89, 0xa9, 0xba, 0xe3, 0x07, 0x80,
+	  0xf1, 0xe1, 0xea, 0xb2, 0x4a, 0x00, 0x52, 0x3c },
+	{ 0xe9, 0x07, 0x56, 0xff, 0x7f, 0x9a, 0xd8, 0x10,
+	  0xb2, 0x39, 0xa1, 0x0c, 0xed, 0x2c, 0xf9, 0xb2,
+	  0x28, 0x43, 0x54, 0xc1, 0xf8, 0xc7, 0xe0, 0xac,
+	  0xcc, 0x24, 0x61, 0xdc, 0x79, 0x6d, 0x6e, 0x89 },
+	{ 0x12, 0x51, 0xf7, 0x6e, 0x56, 0x97, 0x84, 0x81,
+	  0x87, 0x53, 0x59, 0x80, 0x1d, 0xb5, 0x89, 0xa0,
+	  0xb2, 0x2f, 0x86, 0xd8, 0xd6, 0x34, 0xdc, 0x04,
+	  0x50, 0x6f, 0x32, 0x2e, 0xd7, 0x8f, 0x17, 0xe8 },
+	{ 0x3a, 0xfa, 0x89, 0x9f, 0xd9, 0x80, 0xe7, 0x3e,
+	  0xcb, 0x7f, 0x4d, 0x8b, 0x8f, 0x29, 0x1d, 0xc9,
+	  0xaf, 0x79, 0x6b, 0xc6, 0x5d, 0x27, 0xf9, 0x74,
+	  0xc6, 0xf1, 0x93, 0xc9, 0x19, 0x1a, 0x09, 0xfd },
+	{ 0xaa, 0x30, 0x5b, 0xe2, 0x6e, 0x5d, 0xed, 0xdc,
+	  0x3c, 0x10, 0x10, 0xcb, 0xc2, 0x13, 0xf9, 0x5f,
+	  0x05, 0x1c, 0x78, 0x5c, 0x5b, 0x43, 0x1e, 0x6a,
+	  0x7c, 0xd0, 0x48, 0xf1, 0x61, 0x78, 0x75, 0x28 },
+	{ 0x8e, 0xa1, 0x88, 0x4f, 0xf3, 0x2e, 0x9d, 0x10,
+	  0xf0, 0x39, 0xb4, 0x07, 0xd0, 0xd4, 0x4e, 0x7e,
+	  0x67, 0x0a, 0xbd, 0x88, 0x4a, 0xee, 0xe0, 0xfb,
+	  0x75, 0x7a, 0xe9, 0x4e, 0xaa, 0x97, 0x37, 0x3d },
+	{ 0xd4, 0x82, 0xb2, 0x15, 0x5d, 0x4d, 0xec, 0x6b,
+	  0x47, 0x36, 0xa1, 0xf1, 0x61, 0x7b, 0x53, 0xaa,
+	  0xa3, 0x73, 0x10, 0x27, 0x7d, 0x3f, 0xef, 0x0c,
+	  0x37, 0xad, 0x41, 0x76, 0x8f, 0xc2, 0x35, 0xb4 },
+	{ 0x4d, 0x41, 0x39, 0x71, 0x38, 0x7e, 0x7a, 0x88,
+	  0x98, 0xa8, 0xdc, 0x2a, 0x27, 0x50, 0x07, 0x78,
+	  0x53, 0x9e, 0xa2, 0x14, 0xa2, 0xdf, 0xe9, 0xb3,
+	  0xd7, 0xe8, 0xeb, 0xdc, 0xe5, 0xcf, 0x3d, 0xb3 },
+	{ 0x69, 0x6e, 0x5d, 0x46, 0xe6, 0xc5, 0x7e, 0x87,
+	  0x96, 0xe4, 0x73, 0x5d, 0x08, 0x91, 0x6e, 0x0b,
+	  0x79, 0x29, 0xb3, 0xcf, 0x29, 0x8c, 0x29, 0x6d,
+	  0x22, 0xe9, 0xd3, 0x01, 0x96, 0x53, 0x37, 0x1c },
+	{ 0x1f, 0x56, 0x47, 0xc1, 0xd3, 0xb0, 0x88, 0x22,
+	  0x88, 0x85, 0x86, 0x5c, 0x89, 0x40, 0x90, 0x8b,
+	  0xf4, 0x0d, 0x1a, 0x82, 0x72, 0x82, 0x19, 0x73,
+	  0xb1, 0x60, 0x00, 0x8e, 0x7a, 0x3c, 0xe2, 0xeb },
+	{ 0xb6, 0xe7, 0x6c, 0x33, 0x0f, 0x02, 0x1a, 0x5b,
+	  0xda, 0x65, 0x87, 0x50, 0x10, 0xb0, 0xed, 0xf0,
+	  0x91, 0x26, 0xc0, 0xf5, 0x10, 0xea, 0x84, 0x90,
+	  0x48, 0x19, 0x20, 0x03, 0xae, 0xf4, 0xc6, 0x1c },
+	{ 0x3c, 0xd9, 0x52, 0xa0, 0xbe, 0xad, 0xa4, 0x1a,
+	  0xbb, 0x42, 0x4c, 0xe4, 0x7f, 0x94, 0xb4, 0x2b,
+	  0xe6, 0x4e, 0x1f, 0xfb, 0x0f, 0xd0, 0x78, 0x22,
+	  0x76, 0x80, 0x79, 0x46, 0xd0, 0xd0, 0xbc, 0x55 },
+	{ 0x98, 0xd9, 0x26, 0x77, 0x43, 0x9b, 0x41, 0xb7,
+	  0xbb, 0x51, 0x33, 0x12, 0xaf, 0xb9, 0x2b, 0xcc,
+	  0x8e, 0xe9, 0x68, 0xb2, 0xe3, 0xb2, 0x38, 0xce,
+	  0xcb, 0x9b, 0x0f, 0x34, 0xc9, 0xbb, 0x63, 0xd0 },
+	{ 0xec, 0xbc, 0xa2, 0xcf, 0x08, 0xae, 0x57, 0xd5,
+	  0x17, 0xad, 0x16, 0x15, 0x8a, 0x32, 0xbf, 0xa7,
+	  0xdc, 0x03, 0x82, 0xea, 0xed, 0xa1, 0x28, 0xe9,
+	  0x18, 0x86, 0x73, 0x4c, 0x24, 0xa0, 0xb2, 0x9d },
+	{ 0x94, 0x2c, 0xc7, 0xc0, 0xb5, 0x2e, 0x2b, 0x16,
+	  0xa4, 0xb8, 0x9f, 0xa4, 0xfc, 0x7e, 0x0b, 0xf6,
+	  0x09, 0xe2, 0x9a, 0x08, 0xc1, 0xa8, 0x54, 0x34,
+	  0x52, 0xb7, 0x7c, 0x7b, 0xfd, 0x11, 0xbb, 0x28 },
+	{ 0x8a, 0x06, 0x5d, 0x8b, 0x61, 0xa0, 0xdf, 0xfb,
+	  0x17, 0x0d, 0x56, 0x27, 0x73, 0x5a, 0x76, 0xb0,
+	  0xe9, 0x50, 0x60, 0x37, 0x80, 0x8c, 0xba, 0x16,
+	  0xc3, 0x45, 0x00, 0x7c, 0x9f, 0x79, 0xcf, 0x8f },
+	{ 0x1b, 0x9f, 0xa1, 0x97, 0x14, 0x65, 0x9c, 0x78,
+	  0xff, 0x41, 0x38, 0x71, 0x84, 0x92, 0x15, 0x36,
+	  0x10, 0x29, 0xac, 0x80, 0x2b, 0x1c, 0xbc, 0xd5,
+	  0x4e, 0x40, 0x8b, 0xd8, 0x72, 0x87, 0xf8, 0x1f },
+	{ 0x8d, 0xab, 0x07, 0x1b, 0xcd, 0x6c, 0x72, 0x92,
+	  0xa9, 0xef, 0x72, 0x7b, 0x4a, 0xe0, 0xd8, 0x67,
+	  0x13, 0x30, 0x1d, 0xa8, 0x61, 0x8d, 0x9a, 0x48,
+	  0xad, 0xce, 0x55, 0xf3, 0x03, 0xa8, 0x69, 0xa1 },
+	{ 0x82, 0x53, 0xe3, 0xe7, 0xc7, 0xb6, 0x84, 0xb9,
+	  0xcb, 0x2b, 0xeb, 0x01, 0x4c, 0xe3, 0x30, 0xff,
+	  0x3d, 0x99, 0xd1, 0x7a, 0xbb, 0xdb, 0xab, 0xe4,
+	  0xf4, 0xd6, 0x74, 0xde, 0xd5, 0x3f, 0xfc, 0x6b },
+	{ 0xf1, 0x95, 0xf3, 0x21, 0xe9, 0xe3, 0xd6, 0xbd,
+	  0x7d, 0x07, 0x45, 0x04, 0xdd, 0x2a, 0xb0, 0xe6,
+	  0x24, 0x1f, 0x92, 0xe7, 0x84, 0xb1, 0xaa, 0x27,
+	  0x1f, 0xf6, 0x48, 0xb1, 0xca, 0xb6, 0xd7, 0xf6 },
+	{ 0x27, 0xe4, 0xcc, 0x72, 0x09, 0x0f, 0x24, 0x12,
+	  0x66, 0x47, 0x6a, 0x7c, 0x09, 0x49, 0x5f, 0x2d,
+	  0xb1, 0x53, 0xd5, 0xbc, 0xbd, 0x76, 0x19, 0x03,
+	  0xef, 0x79, 0x27, 0x5e, 0xc5, 0x6b, 0x2e, 0xd8 },
+	{ 0x89, 0x9c, 0x24, 0x05, 0x78, 0x8e, 0x25, 0xb9,
+	  0x9a, 0x18, 0x46, 0x35, 0x5e, 0x64, 0x6d, 0x77,
+	  0xcf, 0x40, 0x00, 0x83, 0x41, 0x5f, 0x7d, 0xc5,
+	  0xaf, 0xe6, 0x9d, 0x6e, 0x17, 0xc0, 0x00, 0x23 },
+	{ 0xa5, 0x9b, 0x78, 0xc4, 0x90, 0x57, 0x44, 0x07,
+	  0x6b, 0xfe, 0xe8, 0x94, 0xde, 0x70, 0x7d, 0x4f,
+	  0x12, 0x0b, 0x5c, 0x68, 0x93, 0xea, 0x04, 0x00,
+	  0x29, 0x7d, 0x0b, 0xb8, 0x34, 0x72, 0x76, 0x32 },
+	{ 0x59, 0xdc, 0x78, 0xb1, 0x05, 0x64, 0x97, 0x07,
+	  0xa2, 0xbb, 0x44, 0x19, 0xc4, 0x8f, 0x00, 0x54,
+	  0x00, 0xd3, 0x97, 0x3d, 0xe3, 0x73, 0x66, 0x10,
+	  0x23, 0x04, 0x35, 0xb1, 0x04, 0x24, 0xb2, 0x4f },
+	{ 0xc0, 0x14, 0x9d, 0x1d, 0x7e, 0x7a, 0x63, 0x53,
+	  0xa6, 0xd9, 0x06, 0xef, 0xe7, 0x28, 0xf2, 0xf3,
+	  0x29, 0xfe, 0x14, 0xa4, 0x14, 0x9a, 0x3e, 0xa7,
+	  0x76, 0x09, 0xbc, 0x42, 0xb9, 0x75, 0xdd, 0xfa },
+	{ 0xa3, 0x2f, 0x24, 0x14, 0x74, 0xa6, 0xc1, 0x69,
+	  0x32, 0xe9, 0x24, 0x3b, 0xe0, 0xcf, 0x09, 0xbc,
+	  0xdc, 0x7e, 0x0c, 0xa0, 0xe7, 0xa6, 0xa1, 0xb9,
+	  0xb1, 0xa0, 0xf0, 0x1e, 0x41, 0x50, 0x23, 0x77 },
+	{ 0xb2, 0x39, 0xb2, 0xe4, 0xf8, 0x18, 0x41, 0x36,
+	  0x1c, 0x13, 0x39, 0xf6, 0x8e, 0x2c, 0x35, 0x9f,
+	  0x92, 0x9a, 0xf9, 0xad, 0x9f, 0x34, 0xe0, 0x1a,
+	  0xab, 0x46, 0x31, 0xad, 0x6d, 0x55, 0x00, 0xb0 },
+	{ 0x85, 0xfb, 0x41, 0x9c, 0x70, 0x02, 0xa3, 0xe0,
+	  0xb4, 0xb6, 0xea, 0x09, 0x3b, 0x4c, 0x1a, 0xc6,
+	  0x93, 0x66, 0x45, 0xb6, 0x5d, 0xac, 0x5a, 0xc1,
+	  0x5a, 0x85, 0x28, 0xb7, 0xb9, 0x4c, 0x17, 0x54 },
+	{ 0x96, 0x19, 0x72, 0x06, 0x25, 0xf1, 0x90, 0xb9,
+	  0x3a, 0x3f, 0xad, 0x18, 0x6a, 0xb3, 0x14, 0x18,
+	  0x96, 0x33, 0xc0, 0xd3, 0xa0, 0x1e, 0x6f, 0x9b,
+	  0xc8, 0xc4, 0xa8, 0xf8, 0x2f, 0x38, 0x3d, 0xbf },
+	{ 0x7d, 0x62, 0x0d, 0x90, 0xfe, 0x69, 0xfa, 0x46,
+	  0x9a, 0x65, 0x38, 0x38, 0x89, 0x70, 0xa1, 0xaa,
+	  0x09, 0xbb, 0x48, 0xa2, 0xd5, 0x9b, 0x34, 0x7b,
+	  0x97, 0xe8, 0xce, 0x71, 0xf4, 0x8c, 0x7f, 0x46 },
+	{ 0x29, 0x43, 0x83, 0x56, 0x85, 0x96, 0xfb, 0x37,
+	  0xc7, 0x5b, 0xba, 0xcd, 0x97, 0x9c, 0x5f, 0xf6,
+	  0xf2, 0x0a, 0x55, 0x6b, 0xf8, 0x87, 0x9c, 0xc7,
+	  0x29, 0x24, 0x85, 0x5d, 0xf9, 0xb8, 0x24, 0x0e },
+	{ 0x16, 0xb1, 0x8a, 0xb3, 0x14, 0x35, 0x9c, 0x2b,
+	  0x83, 0x3c, 0x1c, 0x69, 0x86, 0xd4, 0x8c, 0x55,
+	  0xa9, 0xfc, 0x97, 0xcd, 0xe9, 0xa3, 0xc1, 0xf1,
+	  0x0a, 0x31, 0x77, 0x14, 0x0f, 0x73, 0xf7, 0x38 },
+	{ 0x8c, 0xbb, 0xdd, 0x14, 0xbc, 0x33, 0xf0, 0x4c,
+	  0xf4, 0x58, 0x13, 0xe4, 0xa1, 0x53, 0xa2, 0x73,
+	  0xd3, 0x6a, 0xda, 0xd5, 0xce, 0x71, 0xf4, 0x99,
+	  0xee, 0xb8, 0x7f, 0xb8, 0xac, 0x63, 0xb7, 0x29 },
+	{ 0x69, 0xc9, 0xa4, 0x98, 0xdb, 0x17, 0x4e, 0xca,
+	  0xef, 0xcc, 0x5a, 0x3a, 0xc9, 0xfd, 0xed, 0xf0,
+	  0xf8, 0x13, 0xa5, 0xbe, 0xc7, 0x27, 0xf1, 0xe7,
+	  0x75, 0xba, 0xbd, 0xec, 0x77, 0x18, 0x81, 0x6e },
+	{ 0xb4, 0x62, 0xc3, 0xbe, 0x40, 0x44, 0x8f, 0x1d,
+	  0x4f, 0x80, 0x62, 0x62, 0x54, 0xe5, 0x35, 0xb0,
+	  0x8b, 0xc9, 0xcd, 0xcf, 0xf5, 0x99, 0xa7, 0x68,
+	  0x57, 0x8d, 0x4b, 0x28, 0x81, 0xa8, 0xe3, 0xf0 },
+	{ 0x55, 0x3e, 0x9d, 0x9c, 0x5f, 0x36, 0x0a, 0xc0,
+	  0xb7, 0x4a, 0x7d, 0x44, 0xe5, 0xa3, 0x91, 0xda,
+	  0xd4, 0xce, 0xd0, 0x3e, 0x0c, 0x24, 0x18, 0x3b,
+	  0x7e, 0x8e, 0xca, 0xbd, 0xf1, 0x71, 0x5a, 0x64 },
+	{ 0x7a, 0x7c, 0x55, 0xa5, 0x6f, 0xa9, 0xae, 0x51,
+	  0xe6, 0x55, 0xe0, 0x19, 0x75, 0xd8, 0xa6, 0xff,
+	  0x4a, 0xe9, 0xe4, 0xb4, 0x86, 0xfc, 0xbe, 0x4e,
+	  0xac, 0x04, 0x45, 0x88, 0xf2, 0x45, 0xeb, 0xea },
+	{ 0x2a, 0xfd, 0xf3, 0xc8, 0x2a, 0xbc, 0x48, 0x67,
+	  0xf5, 0xde, 0x11, 0x12, 0x86, 0xc2, 0xb3, 0xbe,
+	  0x7d, 0x6e, 0x48, 0x65, 0x7b, 0xa9, 0x23, 0xcf,
+	  0xbf, 0x10, 0x1a, 0x6d, 0xfc, 0xf9, 0xdb, 0x9a },
+	{ 0x41, 0x03, 0x7d, 0x2e, 0xdc, 0xdc, 0xe0, 0xc4,
+	  0x9b, 0x7f, 0xb4, 0xa6, 0xaa, 0x09, 0x99, 0xca,
+	  0x66, 0x97, 0x6c, 0x74, 0x83, 0xaf, 0xe6, 0x31,
+	  0xd4, 0xed, 0xa2, 0x83, 0x14, 0x4f, 0x6d, 0xfc },
+	{ 0xc4, 0x46, 0x6f, 0x84, 0x97, 0xca, 0x2e, 0xeb,
+	  0x45, 0x83, 0xa0, 0xb0, 0x8e, 0x9d, 0x9a, 0xc7,
+	  0x43, 0x95, 0x70, 0x9f, 0xda, 0x10, 0x9d, 0x24,
+	  0xf2, 0xe4, 0x46, 0x21, 0x96, 0x77, 0x9c, 0x5d },
+	{ 0x75, 0xf6, 0x09, 0x33, 0x8a, 0xa6, 0x7d, 0x96,
+	  0x9a, 0x2a, 0xe2, 0xa2, 0x36, 0x2b, 0x2d, 0xa9,
+	  0xd7, 0x7c, 0x69, 0x5d, 0xfd, 0x1d, 0xf7, 0x22,
+	  0x4a, 0x69, 0x01, 0xdb, 0x93, 0x2c, 0x33, 0x64 },
+	{ 0x68, 0x60, 0x6c, 0xeb, 0x98, 0x9d, 0x54, 0x88,
+	  0xfc, 0x7c, 0xf6, 0x49, 0xf3, 0xd7, 0xc2, 0x72,
+	  0xef, 0x05, 0x5d, 0xa1, 0xa9, 0x3f, 0xae, 0xcd,
+	  0x55, 0xfe, 0x06, 0xf6, 0x96, 0x70, 0x98, 0xca },
+	{ 0x44, 0x34, 0x6b, 0xde, 0xb7, 0xe0, 0x52, 0xf6,
+	  0x25, 0x50, 0x48, 0xf0, 0xd9, 0xb4, 0x2c, 0x42,
+	  0x5b, 0xab, 0x9c, 0x3d, 0xd2, 0x41, 0x68, 0x21,
+	  0x2c, 0x3e, 0xcf, 0x1e, 0xbf, 0x34, 0xe6, 0xae },
+	{ 0x8e, 0x9c, 0xf6, 0xe1, 0xf3, 0x66, 0x47, 0x1f,
+	  0x2a, 0xc7, 0xd2, 0xee, 0x9b, 0x5e, 0x62, 0x66,
+	  0xfd, 0xa7, 0x1f, 0x8f, 0x2e, 0x41, 0x09, 0xf2,
+	  0x23, 0x7e, 0xd5, 0xf8, 0x81, 0x3f, 0xc7, 0x18 },
+	{ 0x84, 0xbb, 0xeb, 0x84, 0x06, 0xd2, 0x50, 0x95,
+	  0x1f, 0x8c, 0x1b, 0x3e, 0x86, 0xa7, 0xc0, 0x10,
+	  0x08, 0x29, 0x21, 0x83, 0x3d, 0xfd, 0x95, 0x55,
+	  0xa2, 0xf9, 0x09, 0xb1, 0x08, 0x6e, 0xb4, 0xb8 },
+	{ 0xee, 0x66, 0x6f, 0x3e, 0xef, 0x0f, 0x7e, 0x2a,
+	  0x9c, 0x22, 0x29, 0x58, 0xc9, 0x7e, 0xaf, 0x35,
+	  0xf5, 0x1c, 0xed, 0x39, 0x3d, 0x71, 0x44, 0x85,
+	  0xab, 0x09, 0xa0, 0x69, 0x34, 0x0f, 0xdf, 0x88 },
+	{ 0xc1, 0x53, 0xd3, 0x4a, 0x65, 0xc4, 0x7b, 0x4a,
+	  0x62, 0xc5, 0xca, 0xcf, 0x24, 0x01, 0x09, 0x75,
+	  0xd0, 0x35, 0x6b, 0x2f, 0x32, 0xc8, 0xf5, 0xda,
+	  0x53, 0x0d, 0x33, 0x88, 0x16, 0xad, 0x5d, 0xe6 },
+	{ 0x9f, 0xc5, 0x45, 0x01, 0x09, 0xe1, 0xb7, 0x79,
+	  0xf6, 0xc7, 0xae, 0x79, 0xd5, 0x6c, 0x27, 0x63,
+	  0x5c, 0x8d, 0xd4, 0x26, 0xc5, 0xa9, 0xd5, 0x4e,
+	  0x25, 0x78, 0xdb, 0x98, 0x9b, 0x8c, 0x3b, 0x4e },
+	{ 0xd1, 0x2b, 0xf3, 0x73, 0x2e, 0xf4, 0xaf, 0x5c,
+	  0x22, 0xfa, 0x90, 0x35, 0x6a, 0xf8, 0xfc, 0x50,
+	  0xfc, 0xb4, 0x0f, 0x8f, 0x2e, 0xa5, 0xc8, 0x59,
+	  0x47, 0x37, 0xa3, 0xb3, 0xd5, 0xab, 0xdb, 0xd7 },
+	{ 0x11, 0x03, 0x0b, 0x92, 0x89, 0xbb, 0xa5, 0xaf,
+	  0x65, 0x26, 0x06, 0x72, 0xab, 0x6f, 0xee, 0x88,
+	  0xb8, 0x74, 0x20, 0xac, 0xef, 0x4a, 0x17, 0x89,
+	  0xa2, 0x07, 0x3b, 0x7e, 0xc2, 0xf2, 0xa0, 0x9e },
+	{ 0x69, 0xcb, 0x19, 0x2b, 0x84, 0x44, 0x00, 0x5c,
+	  0x8c, 0x0c, 0xeb, 0x12, 0xc8, 0x46, 0x86, 0x07,
+	  0x68, 0x18, 0x8c, 0xda, 0x0a, 0xec, 0x27, 0xa9,
+	  0xc8, 0xa5, 0x5c, 0xde, 0xe2, 0x12, 0x36, 0x32 },
+	{ 0xdb, 0x44, 0x4c, 0x15, 0x59, 0x7b, 0x5f, 0x1a,
+	  0x03, 0xd1, 0xf9, 0xed, 0xd1, 0x6e, 0x4a, 0x9f,
+	  0x43, 0xa6, 0x67, 0xcc, 0x27, 0x51, 0x75, 0xdf,
+	  0xa2, 0xb7, 0x04, 0xe3, 0xbb, 0x1a, 0x9b, 0x83 },
+	{ 0x3f, 0xb7, 0x35, 0x06, 0x1a, 0xbc, 0x51, 0x9d,
+	  0xfe, 0x97, 0x9e, 0x54, 0xc1, 0xee, 0x5b, 0xfa,
+	  0xd0, 0xa9, 0xd8, 0x58, 0xb3, 0x31, 0x5b, 0xad,
+	  0x34, 0xbd, 0xe9, 0x99, 0xef, 0xd7, 0x24, 0xdd }
+};
+
+static bool __init blake2s_selftest(void)
+{
+	u8 key[BLAKE2S_KEY_SIZE];
+	u8 buf[ARRAY_SIZE(blake2s_testvecs)];
+	u8 hash[BLAKE2S_HASH_SIZE];
+	size_t i;
+	bool success = true;
+
+	for (i = 0; i < BLAKE2S_KEY_SIZE; ++i)
+		key[i] = (u8)i;
+
+	for (i = 0; i < ARRAY_SIZE(blake2s_testvecs); ++i)
+		buf[i] = (u8)i;
+
+	for (i = 0; i < ARRAY_SIZE(blake2s_keyed_testvecs); ++i) {
+		blake2s(hash, buf, key, BLAKE2S_HASH_SIZE, i, BLAKE2S_KEY_SIZE);
+		if (memcmp(hash, blake2s_keyed_testvecs[i], BLAKE2S_HASH_SIZE)) {
+			pr_err("blake2s keyed self-test %zu: FAIL\n", i + 1);
+			success = false;
+		}
+	}
+
+	for (i = 0; i < ARRAY_SIZE(blake2s_testvecs); ++i) {
+		blake2s(hash, buf, NULL, BLAKE2S_HASH_SIZE, i, 0);
+		if (memcmp(hash, blake2s_testvecs[i], BLAKE2S_HASH_SIZE)) {
+			pr_err("blake2s unkeyed self-test %zu: FAIL\n", i + i);
+			success = false;
+		}
+	}
+	return success;
+}
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/selftest/chacha20.c b/net/wireguard/crypto/zinc/selftest/chacha20.c
--- a/net/wireguard/crypto/zinc/selftest/chacha20.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/selftest/chacha20.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,2698 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+struct chacha20_testvec {
+	const u8 *input, *output, *key;
+	u64 nonce;
+	size_t ilen;
+};
+
+struct hchacha20_testvec {
+	u8 key[HCHACHA20_KEY_SIZE];
+	u8 nonce[HCHACHA20_NONCE_SIZE];
+	u8 output[CHACHA20_KEY_SIZE];
+};
+
+/* These test vectors are generated by reference implementations and are
+ * designed to check chacha20 implementation block handling, as well as from
+ * the draft-arciszewski-xchacha-01 document.
+ */
+
+static const u8 input01[] __initconst = { };
+static const u8 output01[] __initconst = { };
+static const u8 key01[] __initconst = {
+	0x09, 0xf4, 0xe8, 0x57, 0x10, 0xf2, 0x12, 0xc3,
+	0xc6, 0x91, 0xc4, 0x09, 0x97, 0x46, 0xef, 0xfe,
+	0x02, 0x00, 0xe4, 0x5c, 0x82, 0xed, 0x16, 0xf3,
+	0x32, 0xbe, 0xec, 0x7a, 0xe6, 0x68, 0x12, 0x26
+};
+enum { nonce01 = 0x3834e2afca3c66d3ULL };
+
+static const u8 input02[] __initconst = {
+	0x9d
+};
+static const u8 output02[] __initconst = {
+	0x94
+};
+static const u8 key02[] __initconst = {
+	0x8c, 0x01, 0xac, 0xaf, 0x62, 0x63, 0x56, 0x7a,
+	0xad, 0x23, 0x4c, 0x58, 0x29, 0x29, 0xbe, 0xab,
+	0xe9, 0xf8, 0xdf, 0x6c, 0x8c, 0x74, 0x4d, 0x7d,
+	0x13, 0x94, 0x10, 0x02, 0x3d, 0x8e, 0x9f, 0x94
+};
+enum { nonce02 = 0x5d1b3bfdedd9f73aULL };
+
+static const u8 input03[] __initconst = {
+	0x04, 0x16
+};
+static const u8 output03[] __initconst = {
+	0x92, 0x07
+};
+static const u8 key03[] __initconst = {
+	0x22, 0x0c, 0x79, 0x2c, 0x38, 0x51, 0xbe, 0x99,
+	0xa9, 0x59, 0x24, 0x50, 0xef, 0x87, 0x38, 0xa6,
+	0xa0, 0x97, 0x20, 0xcb, 0xb4, 0x0c, 0x94, 0x67,
+	0x1f, 0x98, 0xdc, 0xc4, 0x83, 0xbc, 0x35, 0x4d
+};
+enum { nonce03 = 0x7a3353ad720a3e2eULL };
+
+static const u8 input04[] __initconst = {
+	0xc7, 0xcc, 0xd0
+};
+static const u8 output04[] __initconst = {
+	0xd8, 0x41, 0x80
+};
+static const u8 key04[] __initconst = {
+	0x81, 0x5e, 0x12, 0x01, 0xc4, 0x36, 0x15, 0x03,
+	0x11, 0xa0, 0xe9, 0x86, 0xbb, 0x5a, 0xdc, 0x45,
+	0x7d, 0x5e, 0x98, 0xf8, 0x06, 0x76, 0x1c, 0xec,
+	0xc0, 0xf7, 0xca, 0x4e, 0x99, 0xd9, 0x42, 0x38
+};
+enum { nonce04 = 0x6816e2fc66176da2ULL };
+
+static const u8 input05[] __initconst = {
+	0x48, 0xf1, 0x31, 0x5f
+};
+static const u8 output05[] __initconst = {
+	0x48, 0xf7, 0x13, 0x67
+};
+static const u8 key05[] __initconst = {
+	0x3f, 0xd6, 0xb6, 0x5e, 0x2f, 0xda, 0x82, 0x39,
+	0x97, 0x06, 0xd3, 0x62, 0x4f, 0xbd, 0xcb, 0x9b,
+	0x1d, 0xe6, 0x4a, 0x76, 0xab, 0xdd, 0x14, 0x50,
+	0x59, 0x21, 0xe3, 0xb2, 0xc7, 0x95, 0xbc, 0x45
+};
+enum { nonce05 = 0xc41a7490e228cc42ULL };
+
+static const u8 input06[] __initconst = {
+	0xae, 0xa2, 0x85, 0x1d, 0xc8
+};
+static const u8 output06[] __initconst = {
+	0xfa, 0xff, 0x45, 0x6b, 0x6f
+};
+static const u8 key06[] __initconst = {
+	0x04, 0x8d, 0xea, 0x67, 0x20, 0x78, 0xfb, 0x8f,
+	0x49, 0x80, 0x35, 0xb5, 0x7b, 0xe4, 0x31, 0x74,
+	0x57, 0x43, 0x3a, 0x64, 0x64, 0xb9, 0xe6, 0x23,
+	0x4d, 0xfe, 0xb8, 0x7b, 0x71, 0x4d, 0x9d, 0x21
+};
+enum { nonce06 = 0x251366db50b10903ULL };
+
+static const u8 input07[] __initconst = {
+	0x1a, 0x32, 0x85, 0xb6, 0xe8, 0x52
+};
+static const u8 output07[] __initconst = {
+	0xd3, 0x5f, 0xf0, 0x07, 0x69, 0xec
+};
+static const u8 key07[] __initconst = {
+	0xbf, 0x2d, 0x42, 0x99, 0x97, 0x76, 0x04, 0xad,
+	0xd3, 0x8f, 0x6e, 0x6a, 0x34, 0x85, 0xaf, 0x81,
+	0xef, 0x36, 0x33, 0xd5, 0x43, 0xa2, 0xaa, 0x08,
+	0x0f, 0x77, 0x42, 0x83, 0x58, 0xc5, 0x42, 0x2a
+};
+enum { nonce07 = 0xe0796da17dba9b58ULL };
+
+static const u8 input08[] __initconst = {
+	0x40, 0xae, 0xcd, 0xe4, 0x3d, 0x22, 0xe0
+};
+static const u8 output08[] __initconst = {
+	0xfd, 0x8a, 0x9f, 0x3d, 0x05, 0xc9, 0xd3
+};
+static const u8 key08[] __initconst = {
+	0xdc, 0x3f, 0x41, 0xe3, 0x23, 0x2a, 0x8d, 0xf6,
+	0x41, 0x2a, 0xa7, 0x66, 0x05, 0x68, 0xe4, 0x7b,
+	0xc4, 0x58, 0xd6, 0xcc, 0xdf, 0x0d, 0xc6, 0x25,
+	0x1b, 0x61, 0x32, 0x12, 0x4e, 0xf1, 0xe6, 0x29
+};
+enum { nonce08 = 0xb1d2536d9e159832ULL };
+
+static const u8 input09[] __initconst = {
+	0xba, 0x1d, 0x14, 0x16, 0x9f, 0x83, 0x67, 0x24
+};
+static const u8 output09[] __initconst = {
+	0x7c, 0xe3, 0x78, 0x1d, 0xa2, 0xe7, 0xe9, 0x39
+};
+static const u8 key09[] __initconst = {
+	0x17, 0x55, 0x90, 0x52, 0xa4, 0xce, 0x12, 0xae,
+	0xd4, 0xfd, 0xd4, 0xfb, 0xd5, 0x18, 0x59, 0x50,
+	0x4e, 0x51, 0x99, 0x32, 0x09, 0x31, 0xfc, 0xf7,
+	0x27, 0x10, 0x8e, 0xa2, 0x4b, 0xa5, 0xf5, 0x62
+};
+enum { nonce09 = 0x495fc269536d003ULL };
+
+static const u8 input10[] __initconst = {
+	0x09, 0xfd, 0x3c, 0x0b, 0x3d, 0x0e, 0xf3, 0x9d,
+	0x27
+};
+static const u8 output10[] __initconst = {
+	0xdc, 0xe4, 0x33, 0x60, 0x0c, 0x07, 0xcb, 0x51,
+	0x6b
+};
+static const u8 key10[] __initconst = {
+	0x4e, 0x00, 0x72, 0x37, 0x0f, 0x52, 0x4d, 0x6f,
+	0x37, 0x50, 0x3c, 0xb3, 0x51, 0x81, 0x49, 0x16,
+	0x7e, 0xfd, 0xb1, 0x51, 0x72, 0x2e, 0xe4, 0x16,
+	0x68, 0x5c, 0x5b, 0x8a, 0xc3, 0x90, 0x70, 0x04
+};
+enum { nonce10 = 0x1ad9d1114d88cbbdULL };
+
+static const u8 input11[] __initconst = {
+	0x70, 0x18, 0x52, 0x85, 0xba, 0x66, 0xff, 0x2c,
+	0x9a, 0x46
+};
+static const u8 output11[] __initconst = {
+	0xf5, 0x2a, 0x7a, 0xfd, 0x31, 0x7c, 0x91, 0x41,
+	0xb1, 0xcf
+};
+static const u8 key11[] __initconst = {
+	0x48, 0xb4, 0xd0, 0x7c, 0x88, 0xd1, 0x96, 0x0d,
+	0x80, 0x33, 0xb4, 0xd5, 0x31, 0x9a, 0x88, 0xca,
+	0x14, 0xdc, 0xf0, 0xa8, 0xf3, 0xac, 0xb8, 0x47,
+	0x75, 0x86, 0x7c, 0x88, 0x50, 0x11, 0x43, 0x40
+};
+enum { nonce11 = 0x47c35dd1f4f8aa4fULL };
+
+static const u8 input12[] __initconst = {
+	0x9e, 0x8e, 0x3d, 0x2a, 0x05, 0xfd, 0xe4, 0x90,
+	0x24, 0x1c, 0xd3
+};
+static const u8 output12[] __initconst = {
+	0x97, 0x72, 0x40, 0x9f, 0xc0, 0x6b, 0x05, 0x33,
+	0x42, 0x7e, 0x28
+};
+static const u8 key12[] __initconst = {
+	0xee, 0xff, 0x33, 0x33, 0xe0, 0x28, 0xdf, 0xa2,
+	0xb6, 0x5e, 0x25, 0x09, 0x52, 0xde, 0xa5, 0x9c,
+	0x8f, 0x95, 0xa9, 0x03, 0x77, 0x0f, 0xbe, 0xa1,
+	0xd0, 0x7d, 0x73, 0x2f, 0xf8, 0x7e, 0x51, 0x44
+};
+enum { nonce12 = 0xc22d044dc6ea4af3ULL };
+
+static const u8 input13[] __initconst = {
+	0x9c, 0x16, 0xa2, 0x22, 0x4d, 0xbe, 0x04, 0x9a,
+	0xb3, 0xb5, 0xc6, 0x58
+};
+static const u8 output13[] __initconst = {
+	0xf0, 0x81, 0xdb, 0x6d, 0xa3, 0xe9, 0xb2, 0xc6,
+	0x32, 0x50, 0x16, 0x9f
+};
+static const u8 key13[] __initconst = {
+	0x96, 0xb3, 0x01, 0xd2, 0x7a, 0x8c, 0x94, 0x09,
+	0x4f, 0x58, 0xbe, 0x80, 0xcc, 0xa9, 0x7e, 0x2d,
+	0xad, 0x58, 0x3b, 0x63, 0xb8, 0x5c, 0x17, 0xce,
+	0xbf, 0x43, 0x33, 0x7a, 0x7b, 0x82, 0x28, 0x2f
+};
+enum { nonce13 = 0x2a5d05d88cd7b0daULL };
+
+static const u8 input14[] __initconst = {
+	0x57, 0x4f, 0xaa, 0x30, 0xe6, 0x23, 0x50, 0x86,
+	0x91, 0xa5, 0x60, 0x96, 0x2b
+};
+static const u8 output14[] __initconst = {
+	0x6c, 0x1f, 0x3b, 0x42, 0xb6, 0x2f, 0xf0, 0xbd,
+	0x76, 0x60, 0xc7, 0x7e, 0x8d
+};
+static const u8 key14[] __initconst = {
+	0x22, 0x85, 0xaf, 0x8f, 0xa3, 0x53, 0xa0, 0xc4,
+	0xb5, 0x75, 0xc0, 0xba, 0x30, 0x92, 0xc3, 0x32,
+	0x20, 0x5a, 0x8f, 0x7e, 0x93, 0xda, 0x65, 0x18,
+	0xd1, 0xf6, 0x9a, 0x9b, 0x8f, 0x85, 0x30, 0xe6
+};
+enum { nonce14 = 0xf9946c166aa4475fULL };
+
+static const u8 input15[] __initconst = {
+	0x89, 0x81, 0xc7, 0xe2, 0x00, 0xac, 0x52, 0x70,
+	0xa4, 0x79, 0xab, 0xeb, 0x74, 0xf7
+};
+static const u8 output15[] __initconst = {
+	0xb4, 0xd0, 0xa9, 0x9d, 0x15, 0x5f, 0x48, 0xd6,
+	0x00, 0x7e, 0x4c, 0x77, 0x5a, 0x46
+};
+static const u8 key15[] __initconst = {
+	0x0a, 0x66, 0x36, 0xca, 0x5d, 0x82, 0x23, 0xb6,
+	0xe4, 0x9b, 0xad, 0x5e, 0xd0, 0x7f, 0xf6, 0x7a,
+	0x7b, 0x03, 0xa7, 0x4c, 0xfd, 0xec, 0xd5, 0xa1,
+	0xfc, 0x25, 0x54, 0xda, 0x5a, 0x5c, 0xf0, 0x2c
+};
+enum { nonce15 = 0x9ab2b87a35e772c8ULL };
+
+static const u8 input16[] __initconst = {
+	0x5f, 0x09, 0xc0, 0x8b, 0x1e, 0xde, 0xca, 0xd9,
+	0xb7, 0x5c, 0x23, 0xc9, 0x55, 0x1e, 0xcf
+};
+static const u8 output16[] __initconst = {
+	0x76, 0x9b, 0x53, 0xf3, 0x66, 0x88, 0x28, 0x60,
+	0x98, 0x80, 0x2c, 0xa8, 0x80, 0xa6, 0x48
+};
+static const u8 key16[] __initconst = {
+	0x80, 0xb5, 0x51, 0xdf, 0x17, 0x5b, 0xb0, 0xef,
+	0x8b, 0x5b, 0x2e, 0x3e, 0xc5, 0xe3, 0xa5, 0x86,
+	0xac, 0x0d, 0x8e, 0x32, 0x90, 0x9d, 0x82, 0x27,
+	0xf1, 0x23, 0x26, 0xc3, 0xea, 0x55, 0xb6, 0x63
+};
+enum { nonce16 = 0xa82e9d39e4d02ef5ULL };
+
+static const u8 input17[] __initconst = {
+	0x87, 0x0b, 0x36, 0x71, 0x7c, 0xb9, 0x0b, 0x80,
+	0x4d, 0x77, 0x5c, 0x4f, 0xf5, 0x51, 0x0e, 0x1a
+};
+static const u8 output17[] __initconst = {
+	0xf1, 0x12, 0x4a, 0x8a, 0xd9, 0xd0, 0x08, 0x67,
+	0x66, 0xd7, 0x34, 0xea, 0x32, 0x3b, 0x54, 0x0e
+};
+static const u8 key17[] __initconst = {
+	0xfb, 0x71, 0x5f, 0x3f, 0x7a, 0xc0, 0x9a, 0xc8,
+	0xc8, 0xcf, 0xe8, 0xbc, 0xfb, 0x09, 0xbf, 0x89,
+	0x6a, 0xef, 0xd5, 0xe5, 0x36, 0x87, 0x14, 0x76,
+	0x00, 0xb9, 0x32, 0x28, 0xb2, 0x00, 0x42, 0x53
+};
+enum { nonce17 = 0x229b87e73d557b96ULL };
+
+static const u8 input18[] __initconst = {
+	0x38, 0x42, 0xb5, 0x37, 0xb4, 0x3d, 0xfe, 0x59,
+	0x38, 0x68, 0x88, 0xfa, 0x89, 0x8a, 0x5f, 0x90,
+	0x3c
+};
+static const u8 output18[] __initconst = {
+	0xac, 0xad, 0x14, 0xe8, 0x7e, 0xd7, 0xce, 0x96,
+	0x3d, 0xb3, 0x78, 0x85, 0x22, 0x5a, 0xcb, 0x39,
+	0xd4
+};
+static const u8 key18[] __initconst = {
+	0xe1, 0xc1, 0xa8, 0xe0, 0x91, 0xe7, 0x38, 0x66,
+	0x80, 0x17, 0x12, 0x3c, 0x5e, 0x2d, 0xbb, 0xea,
+	0xeb, 0x6c, 0x8b, 0xc8, 0x1b, 0x6f, 0x7c, 0xea,
+	0x50, 0x57, 0x23, 0x1e, 0x65, 0x6f, 0x6d, 0x81
+};
+enum { nonce18 = 0xfaf5fcf8f30e57a9ULL };
+
+static const u8 input19[] __initconst = {
+	0x1c, 0x4a, 0x30, 0x26, 0xef, 0x9a, 0x32, 0xa7,
+	0x8f, 0xe5, 0xc0, 0x0f, 0x30, 0x3a, 0xbf, 0x38,
+	0x54, 0xba
+};
+static const u8 output19[] __initconst = {
+	0x57, 0x67, 0x54, 0x4f, 0x31, 0xd6, 0xef, 0x35,
+	0x0b, 0xd9, 0x52, 0xa7, 0x46, 0x7d, 0x12, 0x17,
+	0x1e, 0xe3
+};
+static const u8 key19[] __initconst = {
+	0x5a, 0x79, 0xc1, 0xea, 0x33, 0xb3, 0xc7, 0x21,
+	0xec, 0xf8, 0xcb, 0xd2, 0x58, 0x96, 0x23, 0xd6,
+	0x4d, 0xed, 0x2f, 0xdf, 0x8a, 0x79, 0xe6, 0x8b,
+	0x38, 0xa3, 0xc3, 0x7a, 0x33, 0xda, 0x02, 0xc7
+};
+enum { nonce19 = 0x2b23b61840429604ULL };
+
+static const u8 input20[] __initconst = {
+	0xab, 0xe9, 0x32, 0xbb, 0x35, 0x17, 0xe0, 0x60,
+	0x80, 0xb1, 0x27, 0xdc, 0xe6, 0x62, 0x9e, 0x0c,
+	0x77, 0xf4, 0x50
+};
+static const u8 output20[] __initconst = {
+	0x54, 0x6d, 0xaa, 0xfc, 0x08, 0xfb, 0x71, 0xa8,
+	0xd6, 0x1d, 0x7d, 0xf3, 0x45, 0x10, 0xb5, 0x4c,
+	0xcc, 0x4b, 0x45
+};
+static const u8 key20[] __initconst = {
+	0xa3, 0xfd, 0x3d, 0xa9, 0xeb, 0xea, 0x2c, 0x69,
+	0xcf, 0x59, 0x38, 0x13, 0x5b, 0xa7, 0x53, 0x8f,
+	0x5e, 0xa2, 0x33, 0x86, 0x4c, 0x75, 0x26, 0xaf,
+	0x35, 0x12, 0x09, 0x71, 0x81, 0xea, 0x88, 0x66
+};
+enum { nonce20 = 0x7459667a8fadff58ULL };
+
+static const u8 input21[] __initconst = {
+	0xa6, 0x82, 0x21, 0x23, 0xad, 0x27, 0x3f, 0xc6,
+	0xd7, 0x16, 0x0d, 0x6d, 0x24, 0x15, 0x54, 0xc5,
+	0x96, 0x72, 0x59, 0x8a
+};
+static const u8 output21[] __initconst = {
+	0x5f, 0x34, 0x32, 0xea, 0x06, 0xd4, 0x9e, 0x01,
+	0xdc, 0x32, 0x32, 0x40, 0x66, 0x73, 0x6d, 0x4a,
+	0x6b, 0x12, 0x20, 0xe8
+};
+static const u8 key21[] __initconst = {
+	0x96, 0xfd, 0x13, 0x23, 0xa9, 0x89, 0x04, 0xe6,
+	0x31, 0xa5, 0x2c, 0xc1, 0x40, 0xd5, 0x69, 0x5c,
+	0x32, 0x79, 0x56, 0xe0, 0x29, 0x93, 0x8f, 0xe8,
+	0x5f, 0x65, 0x53, 0x7f, 0xc1, 0xe9, 0xaf, 0xaf
+};
+enum { nonce21 = 0xba8defee9d8e13b5ULL };
+
+static const u8 input22[] __initconst = {
+	0xb8, 0x32, 0x1a, 0x81, 0xd8, 0x38, 0x89, 0x5a,
+	0xb0, 0x05, 0xbe, 0xf4, 0xd2, 0x08, 0xc6, 0xee,
+	0x79, 0x7b, 0x3a, 0x76, 0x59
+};
+static const u8 output22[] __initconst = {
+	0xb7, 0xba, 0xae, 0x80, 0xe4, 0x9f, 0x79, 0x84,
+	0x5a, 0x48, 0x50, 0x6d, 0xcb, 0xd0, 0x06, 0x0c,
+	0x15, 0x63, 0xa7, 0x5e, 0xbd
+};
+static const u8 key22[] __initconst = {
+	0x0f, 0x35, 0x3d, 0xeb, 0x5f, 0x0a, 0x82, 0x0d,
+	0x24, 0x59, 0x71, 0xd8, 0xe6, 0x2d, 0x5f, 0xe1,
+	0x7e, 0x0c, 0xae, 0xf6, 0xdc, 0x2c, 0xc5, 0x4a,
+	0x38, 0x88, 0xf2, 0xde, 0xd9, 0x5f, 0x76, 0x7c
+};
+enum { nonce22 = 0xe77f1760e9f5e192ULL };
+
+static const u8 input23[] __initconst = {
+	0x4b, 0x1e, 0x79, 0x99, 0xcf, 0xef, 0x64, 0x4b,
+	0xb0, 0x66, 0xae, 0x99, 0x2e, 0x68, 0x97, 0xf5,
+	0x5d, 0x9b, 0x3f, 0x7a, 0xa9, 0xd9
+};
+static const u8 output23[] __initconst = {
+	0x5f, 0xa4, 0x08, 0x39, 0xca, 0xfa, 0x2b, 0x83,
+	0x5d, 0x95, 0x70, 0x7c, 0x2e, 0xd4, 0xae, 0xfa,
+	0x45, 0x4a, 0x77, 0x7f, 0xa7, 0x65
+};
+static const u8 key23[] __initconst = {
+	0x4a, 0x06, 0x83, 0x64, 0xaa, 0xe3, 0x38, 0x32,
+	0x28, 0x5d, 0xa4, 0xb2, 0x5a, 0xee, 0xcf, 0x8e,
+	0x19, 0x67, 0xf1, 0x09, 0xe8, 0xc9, 0xf6, 0x40,
+	0x02, 0x6d, 0x0b, 0xde, 0xfa, 0x81, 0x03, 0xb1
+};
+enum { nonce23 = 0x9b3f349158709849ULL };
+
+static const u8 input24[] __initconst = {
+	0xc6, 0xfc, 0x47, 0x5e, 0xd8, 0xed, 0xa9, 0xe5,
+	0x4f, 0x82, 0x79, 0x35, 0xee, 0x3e, 0x7e, 0x3e,
+	0x35, 0x70, 0x6e, 0xfa, 0x6d, 0x08, 0xe8
+};
+static const u8 output24[] __initconst = {
+	0x3b, 0xc5, 0xf8, 0xc2, 0xbf, 0x2b, 0x90, 0x33,
+	0xa6, 0xae, 0xf5, 0x5a, 0x65, 0xb3, 0x3d, 0xe1,
+	0xcd, 0x5f, 0x55, 0xfa, 0xe7, 0xa5, 0x4a
+};
+static const u8 key24[] __initconst = {
+	0x00, 0x24, 0xc3, 0x65, 0x5f, 0xe6, 0x31, 0xbb,
+	0x6d, 0xfc, 0x20, 0x7b, 0x1b, 0xa8, 0x96, 0x26,
+	0x55, 0x21, 0x62, 0x25, 0x7e, 0xba, 0x23, 0x97,
+	0xc9, 0xb8, 0x53, 0xa8, 0xef, 0xab, 0xad, 0x61
+};
+enum { nonce24 = 0x13ee0b8f526177c3ULL };
+
+static const u8 input25[] __initconst = {
+	0x33, 0x07, 0x16, 0xb1, 0x34, 0x33, 0x67, 0x04,
+	0x9b, 0x0a, 0xce, 0x1b, 0xe9, 0xde, 0x1a, 0xec,
+	0xd0, 0x55, 0xfb, 0xc6, 0x33, 0xaf, 0x2d, 0xe3
+};
+static const u8 output25[] __initconst = {
+	0x05, 0x93, 0x10, 0xd1, 0x58, 0x6f, 0x68, 0x62,
+	0x45, 0xdb, 0x91, 0xae, 0x70, 0xcf, 0xd4, 0x5f,
+	0xee, 0xdf, 0xd5, 0xba, 0x9e, 0xde, 0x68, 0xe6
+};
+static const u8 key25[] __initconst = {
+	0x83, 0xa9, 0x4f, 0x5d, 0x74, 0xd5, 0x91, 0xb3,
+	0xc9, 0x97, 0x19, 0x15, 0xdb, 0x0d, 0x0b, 0x4a,
+	0x3d, 0x55, 0xcf, 0xab, 0xb2, 0x05, 0x21, 0x35,
+	0x45, 0x50, 0xeb, 0xf8, 0xf5, 0xbf, 0x36, 0x35
+};
+enum { nonce25 = 0x7c6f459e49ebfebcULL };
+
+static const u8 input26[] __initconst = {
+	0xc2, 0xd4, 0x7a, 0xa3, 0x92, 0xe1, 0xac, 0x46,
+	0x1a, 0x15, 0x38, 0xc9, 0xb5, 0xfd, 0xdf, 0x84,
+	0x38, 0xbc, 0x6b, 0x1d, 0xb0, 0x83, 0x43, 0x04,
+	0x39
+};
+static const u8 output26[] __initconst = {
+	0x7f, 0xde, 0xd6, 0x87, 0xcc, 0x34, 0xf4, 0x12,
+	0xae, 0x55, 0xa5, 0x89, 0x95, 0x29, 0xfc, 0x18,
+	0xd8, 0xc7, 0x7c, 0xd3, 0xcb, 0x85, 0x95, 0x21,
+	0xd2
+};
+static const u8 key26[] __initconst = {
+	0xe4, 0xd0, 0x54, 0x1d, 0x7d, 0x47, 0xa8, 0xc1,
+	0x08, 0xca, 0xe2, 0x42, 0x52, 0x95, 0x16, 0x43,
+	0xa3, 0x01, 0x23, 0x03, 0xcc, 0x3b, 0x81, 0x78,
+	0x23, 0xcc, 0xa7, 0x36, 0xd7, 0xa0, 0x97, 0x8d
+};
+enum { nonce26 = 0x524401012231683ULL };
+
+static const u8 input27[] __initconst = {
+	0x0d, 0xb0, 0xcf, 0xec, 0xfc, 0x38, 0x9d, 0x9d,
+	0x89, 0x00, 0x96, 0xf2, 0x79, 0x8a, 0xa1, 0x8d,
+	0x32, 0x5e, 0xc6, 0x12, 0x22, 0xec, 0xf6, 0x52,
+	0xc1, 0x0b
+};
+static const u8 output27[] __initconst = {
+	0xef, 0xe1, 0xf2, 0x67, 0x8e, 0x2c, 0x00, 0x9f,
+	0x1d, 0x4c, 0x66, 0x1f, 0x94, 0x58, 0xdc, 0xbb,
+	0xb9, 0x11, 0x8f, 0x74, 0xfd, 0x0e, 0x14, 0x01,
+	0xa8, 0x21
+};
+static const u8 key27[] __initconst = {
+	0x78, 0x71, 0xa4, 0xe6, 0xb2, 0x95, 0x44, 0x12,
+	0x81, 0xaa, 0x7e, 0x94, 0xa7, 0x8d, 0x44, 0xea,
+	0xc4, 0xbc, 0x01, 0xb7, 0x9e, 0xf7, 0x82, 0x9e,
+	0x3b, 0x23, 0x9f, 0x31, 0xdd, 0xb8, 0x0d, 0x18
+};
+enum { nonce27 = 0xd58fe0e58fb254d6ULL };
+
+static const u8 input28[] __initconst = {
+	0xaa, 0xb7, 0xaa, 0xd9, 0xa8, 0x91, 0xd7, 0x8a,
+	0x97, 0x9b, 0xdb, 0x7c, 0x47, 0x2b, 0xdb, 0xd2,
+	0xda, 0x77, 0xb1, 0xfa, 0x2d, 0x12, 0xe3, 0xe9,
+	0xc4, 0x7f, 0x54
+};
+static const u8 output28[] __initconst = {
+	0x87, 0x84, 0xa9, 0xa6, 0xad, 0x8f, 0xe6, 0x0f,
+	0x69, 0xf8, 0x21, 0xc3, 0x54, 0x95, 0x0f, 0xb0,
+	0x4e, 0xc7, 0x02, 0xe4, 0x04, 0xb0, 0x6c, 0x42,
+	0x8c, 0x63, 0xe3
+};
+static const u8 key28[] __initconst = {
+	0x12, 0x23, 0x37, 0x95, 0x04, 0xb4, 0x21, 0xe8,
+	0xbc, 0x65, 0x46, 0x7a, 0xf4, 0x01, 0x05, 0x3f,
+	0xb1, 0x34, 0x73, 0xd2, 0x49, 0xbf, 0x6f, 0x20,
+	0xbd, 0x23, 0x58, 0x5f, 0xd1, 0x73, 0x57, 0xa6
+};
+enum { nonce28 = 0x3a04d51491eb4e07ULL };
+
+static const u8 input29[] __initconst = {
+	0x55, 0xd0, 0xd4, 0x4b, 0x17, 0xc8, 0xc4, 0x2b,
+	0xc0, 0x28, 0xbd, 0x9d, 0x65, 0x4d, 0xaf, 0x77,
+	0x72, 0x7c, 0x36, 0x68, 0xa7, 0xb6, 0x87, 0x4d,
+	0xb9, 0x27, 0x25, 0x6c
+};
+static const u8 output29[] __initconst = {
+	0x0e, 0xac, 0x4c, 0xf5, 0x12, 0xb5, 0x56, 0xa5,
+	0x00, 0x9a, 0xd6, 0xe5, 0x1a, 0x59, 0x2c, 0xf6,
+	0x42, 0x22, 0xcf, 0x23, 0x98, 0x34, 0x29, 0xac,
+	0x6e, 0xe3, 0x37, 0x6d
+};
+static const u8 key29[] __initconst = {
+	0xda, 0x9d, 0x05, 0x0c, 0x0c, 0xba, 0x75, 0xb9,
+	0x9e, 0xb1, 0x8d, 0xd9, 0x73, 0x26, 0x2c, 0xa9,
+	0x3a, 0xb5, 0xcb, 0x19, 0x49, 0xa7, 0x4f, 0xf7,
+	0x64, 0x35, 0x23, 0x20, 0x2a, 0x45, 0x78, 0xc7
+};
+enum { nonce29 = 0xc25ac9982431cbfULL };
+
+static const u8 input30[] __initconst = {
+	0x4e, 0xd6, 0x85, 0xbb, 0xe7, 0x99, 0xfa, 0x04,
+	0x33, 0x24, 0xfd, 0x75, 0x18, 0xe3, 0xd3, 0x25,
+	0xcd, 0xca, 0xae, 0x00, 0xbe, 0x52, 0x56, 0x4a,
+	0x31, 0xe9, 0x4f, 0xae, 0x8a
+};
+static const u8 output30[] __initconst = {
+	0x30, 0x36, 0x32, 0xa2, 0x3c, 0xb6, 0xf9, 0xf9,
+	0x76, 0x70, 0xad, 0xa6, 0x10, 0x41, 0x00, 0x4a,
+	0xfa, 0xce, 0x1b, 0x86, 0x05, 0xdb, 0x77, 0x96,
+	0xb3, 0xb7, 0x8f, 0x61, 0x24
+};
+static const u8 key30[] __initconst = {
+	0x49, 0x35, 0x4c, 0x15, 0x98, 0xfb, 0xc6, 0x57,
+	0x62, 0x6d, 0x06, 0xc3, 0xd4, 0x79, 0x20, 0x96,
+	0x05, 0x2a, 0x31, 0x63, 0xc0, 0x44, 0x42, 0x09,
+	0x13, 0x13, 0xff, 0x1b, 0xc8, 0x63, 0x1f, 0x0b
+};
+enum { nonce30 = 0x4967f9c08e41568bULL };
+
+static const u8 input31[] __initconst = {
+	0x91, 0x04, 0x20, 0x47, 0x59, 0xee, 0xa6, 0x0f,
+	0x04, 0x75, 0xc8, 0x18, 0x95, 0x44, 0x01, 0x28,
+	0x20, 0x6f, 0x73, 0x68, 0x66, 0xb5, 0x03, 0xb3,
+	0x58, 0x27, 0x6e, 0x7a, 0x76, 0xb8
+};
+static const u8 output31[] __initconst = {
+	0xe8, 0x03, 0x78, 0x9d, 0x13, 0x15, 0x98, 0xef,
+	0x64, 0x68, 0x12, 0x41, 0xb0, 0x29, 0x94, 0x0c,
+	0x83, 0x35, 0x46, 0xa9, 0x74, 0xe1, 0x75, 0xf0,
+	0xb6, 0x96, 0xc3, 0x6f, 0xd7, 0x70
+};
+static const u8 key31[] __initconst = {
+	0xef, 0xcd, 0x5a, 0x4a, 0xf4, 0x7e, 0x6a, 0x3a,
+	0x11, 0x88, 0x72, 0x94, 0xb8, 0xae, 0x84, 0xc3,
+	0x66, 0xe0, 0xde, 0x4b, 0x00, 0xa5, 0xd6, 0x2d,
+	0x50, 0xb7, 0x28, 0xff, 0x76, 0x57, 0x18, 0x1f
+};
+enum { nonce31 = 0xcb6f428fa4192e19ULL };
+
+static const u8 input32[] __initconst = {
+	0x90, 0x06, 0x50, 0x4b, 0x98, 0x14, 0x30, 0xf1,
+	0xb8, 0xd7, 0xf0, 0xa4, 0x3e, 0x4e, 0xd8, 0x00,
+	0xea, 0xdb, 0x4f, 0x93, 0x05, 0xef, 0x02, 0x71,
+	0x1a, 0xcd, 0xa3, 0xb1, 0xae, 0xd3, 0x18
+};
+static const u8 output32[] __initconst = {
+	0xcb, 0x4a, 0x37, 0x3f, 0xea, 0x40, 0xab, 0x86,
+	0xfe, 0xcc, 0x07, 0xd5, 0xdc, 0xb2, 0x25, 0xb6,
+	0xfd, 0x2a, 0x72, 0xbc, 0x5e, 0xd4, 0x75, 0xff,
+	0x71, 0xfc, 0xce, 0x1e, 0x6f, 0x22, 0xc1
+};
+static const u8 key32[] __initconst = {
+	0xfc, 0x6d, 0xc3, 0x80, 0xce, 0xa4, 0x31, 0xa1,
+	0xcc, 0xfa, 0x9d, 0x10, 0x0b, 0xc9, 0x11, 0x77,
+	0x34, 0xdb, 0xad, 0x1b, 0xc4, 0xfc, 0xeb, 0x79,
+	0x91, 0xda, 0x59, 0x3b, 0x0d, 0xb1, 0x19, 0x3b
+};
+enum { nonce32 = 0x88551bf050059467ULL };
+
+static const u8 input33[] __initconst = {
+	0x88, 0x94, 0x71, 0x92, 0xe8, 0xd7, 0xf9, 0xbd,
+	0x55, 0xe3, 0x22, 0xdb, 0x99, 0x51, 0xfb, 0x50,
+	0xbf, 0x82, 0xb5, 0x70, 0x8b, 0x2b, 0x6a, 0x03,
+	0x37, 0xa0, 0xc6, 0x19, 0x5d, 0xc9, 0xbc, 0xcc
+};
+static const u8 output33[] __initconst = {
+	0xb6, 0x17, 0x51, 0xc8, 0xea, 0x8a, 0x14, 0xdc,
+	0x23, 0x1b, 0xd4, 0xed, 0xbf, 0x50, 0xb9, 0x38,
+	0x00, 0xc2, 0x3f, 0x78, 0x3d, 0xbf, 0xa0, 0x84,
+	0xef, 0x45, 0xb2, 0x7d, 0x48, 0x7b, 0x62, 0xa7
+};
+static const u8 key33[] __initconst = {
+	0xb9, 0x8f, 0x6a, 0xad, 0xb4, 0x6f, 0xb5, 0xdc,
+	0x48, 0xfa, 0x43, 0x57, 0x62, 0x97, 0xef, 0x89,
+	0x4c, 0x5a, 0x7b, 0x67, 0xb8, 0x9d, 0xf0, 0x42,
+	0x2b, 0x8f, 0xf3, 0x18, 0x05, 0x2e, 0x48, 0xd0
+};
+enum { nonce33 = 0x31f16488fe8447f5ULL };
+
+static const u8 input34[] __initconst = {
+	0xda, 0x2b, 0x3d, 0x63, 0x9e, 0x4f, 0xc2, 0xb8,
+	0x7f, 0xc2, 0x1a, 0x8b, 0x0d, 0x95, 0x65, 0x55,
+	0x52, 0xba, 0x51, 0x51, 0xc0, 0x61, 0x9f, 0x0a,
+	0x5d, 0xb0, 0x59, 0x8c, 0x64, 0x6a, 0xab, 0xf5,
+	0x57
+};
+static const u8 output34[] __initconst = {
+	0x5c, 0xf6, 0x62, 0x24, 0x8c, 0x45, 0xa3, 0x26,
+	0xd0, 0xe4, 0x88, 0x1c, 0xed, 0xc4, 0x26, 0x58,
+	0xb5, 0x5d, 0x92, 0xc4, 0x17, 0x44, 0x1c, 0xb8,
+	0x2c, 0xf3, 0x55, 0x7e, 0xd6, 0xe5, 0xb3, 0x65,
+	0xa8
+};
+static const u8 key34[] __initconst = {
+	0xde, 0xd1, 0x27, 0xb7, 0x7c, 0xfa, 0xa6, 0x78,
+	0x39, 0x80, 0xdf, 0xb7, 0x46, 0xac, 0x71, 0x26,
+	0xd0, 0x2a, 0x56, 0x79, 0x12, 0xeb, 0x26, 0x37,
+	0x01, 0x0d, 0x30, 0xe0, 0xe3, 0x66, 0xb2, 0xf4
+};
+enum { nonce34 = 0x92d0d9b252c24149ULL };
+
+static const u8 input35[] __initconst = {
+	0x3a, 0x15, 0x5b, 0x75, 0x6e, 0xd0, 0x52, 0x20,
+	0x6c, 0x82, 0xfa, 0xce, 0x5b, 0xea, 0xf5, 0x43,
+	0xc1, 0x81, 0x7c, 0xb2, 0xac, 0x16, 0x3f, 0xd3,
+	0x5a, 0xaf, 0x55, 0x98, 0xf4, 0xc6, 0xba, 0x71,
+	0x25, 0x8b
+};
+static const u8 output35[] __initconst = {
+	0xb3, 0xaf, 0xac, 0x6d, 0x4d, 0xc7, 0x68, 0x56,
+	0x50, 0x5b, 0x69, 0x2a, 0xe5, 0x90, 0xf9, 0x5f,
+	0x99, 0x88, 0xff, 0x0c, 0xa6, 0xb1, 0x83, 0xd6,
+	0x80, 0xa6, 0x1b, 0xde, 0x94, 0xa4, 0x2c, 0xc3,
+	0x74, 0xfa
+};
+static const u8 key35[] __initconst = {
+	0xd8, 0x24, 0xe2, 0x06, 0xd7, 0x7a, 0xce, 0x81,
+	0x52, 0x72, 0x02, 0x69, 0x89, 0xc4, 0xe9, 0x53,
+	0x3b, 0x08, 0x5f, 0x98, 0x1e, 0x1b, 0x99, 0x6e,
+	0x28, 0x17, 0x6d, 0xba, 0xc0, 0x96, 0xf9, 0x3c
+};
+enum { nonce35 = 0x7baf968c4c8e3a37ULL };
+
+static const u8 input36[] __initconst = {
+	0x31, 0x5d, 0x4f, 0xe3, 0xac, 0xad, 0x17, 0xa6,
+	0xb5, 0x01, 0xe2, 0xc6, 0xd4, 0x7e, 0xc4, 0x80,
+	0xc0, 0x59, 0x72, 0xbb, 0x4b, 0x74, 0x6a, 0x41,
+	0x0f, 0x9c, 0xf6, 0xca, 0x20, 0xb3, 0x73, 0x07,
+	0x6b, 0x02, 0x2a
+};
+static const u8 output36[] __initconst = {
+	0xf9, 0x09, 0x92, 0x94, 0x7e, 0x31, 0xf7, 0x53,
+	0xe8, 0x8a, 0x5b, 0x20, 0xef, 0x9b, 0x45, 0x81,
+	0xba, 0x5e, 0x45, 0x63, 0xc1, 0xc7, 0x9e, 0x06,
+	0x0e, 0xd9, 0x62, 0x8e, 0x96, 0xf9, 0xfa, 0x43,
+	0x4d, 0xd4, 0x28
+};
+static const u8 key36[] __initconst = {
+	0x13, 0x30, 0x4c, 0x06, 0xae, 0x18, 0xde, 0x03,
+	0x1d, 0x02, 0x40, 0xf5, 0xbb, 0x19, 0xe3, 0x88,
+	0x41, 0xb1, 0x29, 0x15, 0x97, 0xc2, 0x69, 0x3f,
+	0x32, 0x2a, 0x0c, 0x8b, 0xcf, 0x83, 0x8b, 0x6c
+};
+enum { nonce36 = 0x226d251d475075a0ULL };
+
+static const u8 input37[] __initconst = {
+	0x10, 0x18, 0xbe, 0xfd, 0x66, 0xc9, 0x77, 0xcc,
+	0x43, 0xe5, 0x46, 0x0b, 0x08, 0x8b, 0xae, 0x11,
+	0x86, 0x15, 0xc2, 0xf6, 0x45, 0xd4, 0x5f, 0xd6,
+	0xb6, 0x5f, 0x9f, 0x3e, 0x97, 0xb7, 0xd4, 0xad,
+	0x0b, 0xe8, 0x31, 0x94
+};
+static const u8 output37[] __initconst = {
+	0x03, 0x2c, 0x1c, 0xee, 0xc6, 0xdd, 0xed, 0x38,
+	0x80, 0x6d, 0x84, 0x16, 0xc3, 0xc2, 0x04, 0x63,
+	0xcd, 0xa7, 0x6e, 0x36, 0x8b, 0xed, 0x78, 0x63,
+	0x95, 0xfc, 0x69, 0x7a, 0x3f, 0x8d, 0x75, 0x6b,
+	0x6c, 0x26, 0x56, 0x4d
+};
+static const u8 key37[] __initconst = {
+	0xac, 0x84, 0x4d, 0xa9, 0x29, 0x49, 0x3c, 0x39,
+	0x7f, 0xd9, 0xa6, 0x01, 0xf3, 0x7e, 0xfa, 0x4a,
+	0x14, 0x80, 0x22, 0x74, 0xf0, 0x29, 0x30, 0x2d,
+	0x07, 0x21, 0xda, 0xc0, 0x4d, 0x70, 0x56, 0xa2
+};
+enum { nonce37 = 0x167823ce3b64925aULL };
+
+static const u8 input38[] __initconst = {
+	0x30, 0x8f, 0xfa, 0x24, 0x29, 0xb1, 0xfb, 0xce,
+	0x31, 0x62, 0xdc, 0xd0, 0x46, 0xab, 0xe1, 0x31,
+	0xd9, 0xae, 0x60, 0x0d, 0xca, 0x0a, 0x49, 0x12,
+	0x3d, 0x92, 0xe9, 0x91, 0x67, 0x12, 0x62, 0x18,
+	0x89, 0xe2, 0xf9, 0x1c, 0xcc
+};
+static const u8 output38[] __initconst = {
+	0x56, 0x9c, 0xc8, 0x7a, 0xc5, 0x98, 0xa3, 0x0f,
+	0xba, 0xd5, 0x3e, 0xe1, 0xc9, 0x33, 0x64, 0x33,
+	0xf0, 0xd5, 0xf7, 0x43, 0x66, 0x0e, 0x08, 0x9a,
+	0x6e, 0x09, 0xe4, 0x01, 0x0d, 0x1e, 0x2f, 0x4b,
+	0xed, 0x9c, 0x08, 0x8c, 0x03
+};
+static const u8 key38[] __initconst = {
+	0x77, 0x52, 0x2a, 0x23, 0xf1, 0xc5, 0x96, 0x2b,
+	0x89, 0x4f, 0x3e, 0xf3, 0xff, 0x0e, 0x94, 0xce,
+	0xf1, 0xbd, 0x53, 0xf5, 0x77, 0xd6, 0x9e, 0x47,
+	0x49, 0x3d, 0x16, 0x64, 0xff, 0x95, 0x42, 0x42
+};
+enum { nonce38 = 0xff629d7b82cef357ULL };
+
+static const u8 input39[] __initconst = {
+	0x38, 0x26, 0x27, 0xd0, 0xc2, 0xf5, 0x34, 0xba,
+	0xda, 0x0f, 0x1c, 0x1c, 0x9a, 0x70, 0xe5, 0x8a,
+	0x78, 0x2d, 0x8f, 0x9a, 0xbf, 0x89, 0x6a, 0xfd,
+	0xd4, 0x9c, 0x33, 0xf1, 0xb6, 0x89, 0x16, 0xe3,
+	0x6a, 0x00, 0xfa, 0x3a, 0x0f, 0x26
+};
+static const u8 output39[] __initconst = {
+	0x0f, 0xaf, 0x91, 0x6d, 0x9c, 0x99, 0xa4, 0xf7,
+	0x3b, 0x9d, 0x9a, 0x98, 0xca, 0xbb, 0x50, 0x48,
+	0xee, 0xcb, 0x5d, 0xa1, 0x37, 0x2d, 0x36, 0x09,
+	0x2a, 0xe2, 0x1c, 0x3d, 0x98, 0x40, 0x1c, 0x16,
+	0x56, 0xa7, 0x98, 0xe9, 0x7d, 0x2b
+};
+static const u8 key39[] __initconst = {
+	0x6e, 0x83, 0x15, 0x4d, 0xf8, 0x78, 0xa8, 0x0e,
+	0x71, 0x37, 0xd4, 0x6e, 0x28, 0x5c, 0x06, 0xa1,
+	0x2d, 0x6c, 0x72, 0x7a, 0xfd, 0xf8, 0x65, 0x1a,
+	0xb8, 0xe6, 0x29, 0x7b, 0xe5, 0xb3, 0x23, 0x79
+};
+enum { nonce39 = 0xa4d8c491cf093e9dULL };
+
+static const u8 input40[] __initconst = {
+	0x8f, 0x32, 0x7c, 0x40, 0x37, 0x95, 0x08, 0x00,
+	0x00, 0xfe, 0x2f, 0x95, 0x20, 0x12, 0x40, 0x18,
+	0x5e, 0x7e, 0x5e, 0x99, 0xee, 0x8d, 0x91, 0x7d,
+	0x50, 0x7d, 0x21, 0x45, 0x27, 0xe1, 0x7f, 0xd4,
+	0x73, 0x10, 0xe1, 0x33, 0xbc, 0xf8, 0xdd
+};
+static const u8 output40[] __initconst = {
+	0x78, 0x7c, 0xdc, 0x55, 0x2b, 0xd9, 0x2b, 0x3a,
+	0xdd, 0x56, 0x11, 0x52, 0xd3, 0x2e, 0xe0, 0x0d,
+	0x23, 0x20, 0x8a, 0xf1, 0x4f, 0xee, 0xf1, 0x68,
+	0xf6, 0xdc, 0x53, 0xcf, 0x17, 0xd4, 0xf0, 0x6c,
+	0xdc, 0x80, 0x5f, 0x1c, 0xa4, 0x91, 0x05
+};
+static const u8 key40[] __initconst = {
+	0x0d, 0x86, 0xbf, 0x8a, 0xba, 0x9e, 0x39, 0x91,
+	0xa8, 0xe7, 0x22, 0xf0, 0x0c, 0x43, 0x18, 0xe4,
+	0x1f, 0xb0, 0xaf, 0x8a, 0x34, 0x31, 0xf4, 0x41,
+	0xf0, 0x89, 0x85, 0xca, 0x5d, 0x05, 0x3b, 0x94
+};
+enum { nonce40 = 0xae7acc4f5986439eULL };
+
+static const u8 input41[] __initconst = {
+	0x20, 0x5f, 0xc1, 0x83, 0x36, 0x02, 0x76, 0x96,
+	0xf0, 0xbf, 0x8e, 0x0e, 0x1a, 0xd1, 0xc7, 0x88,
+	0x18, 0xc7, 0x09, 0xc4, 0x15, 0xd9, 0x4f, 0x5e,
+	0x1f, 0xb3, 0xb4, 0x6d, 0xcb, 0xa0, 0xd6, 0x8a,
+	0x3b, 0x40, 0x8e, 0x80, 0xf1, 0xe8, 0x8f, 0x5f
+};
+static const u8 output41[] __initconst = {
+	0x0b, 0xd1, 0x49, 0x9a, 0x9d, 0xe8, 0x97, 0xb8,
+	0xd1, 0xeb, 0x90, 0x62, 0x37, 0xd2, 0x99, 0x15,
+	0x67, 0x6d, 0x27, 0x93, 0xce, 0x37, 0x65, 0xa2,
+	0x94, 0x88, 0xd6, 0x17, 0xbc, 0x1c, 0x6e, 0xa2,
+	0xcc, 0xfb, 0x81, 0x0e, 0x30, 0x60, 0x5a, 0x6f
+};
+static const u8 key41[] __initconst = {
+	0x36, 0x27, 0x57, 0x01, 0x21, 0x68, 0x97, 0xc7,
+	0x00, 0x67, 0x7b, 0xe9, 0x0f, 0x55, 0x49, 0xbb,
+	0x92, 0x18, 0x98, 0xf5, 0x5e, 0xbc, 0xe7, 0x5a,
+	0x9d, 0x3d, 0xc7, 0xbd, 0x59, 0xec, 0x82, 0x8e
+};
+enum { nonce41 = 0x5da05e4c8dfab464ULL };
+
+static const u8 input42[] __initconst = {
+	0xca, 0x30, 0xcd, 0x63, 0xf0, 0x2d, 0xf1, 0x03,
+	0x4d, 0x0d, 0xf2, 0xf7, 0x6f, 0xae, 0xd6, 0x34,
+	0xea, 0xf6, 0x13, 0xcf, 0x1c, 0xa0, 0xd0, 0xe8,
+	0xa4, 0x78, 0x80, 0x3b, 0x1e, 0xa5, 0x32, 0x4c,
+	0x73, 0x12, 0xd4, 0x6a, 0x94, 0xbc, 0xba, 0x80,
+	0x5e
+};
+static const u8 output42[] __initconst = {
+	0xec, 0x3f, 0x18, 0x31, 0xc0, 0x7b, 0xb5, 0xe2,
+	0xad, 0xf3, 0xec, 0xa0, 0x16, 0x9d, 0xef, 0xce,
+	0x05, 0x65, 0x59, 0x9d, 0x5a, 0xca, 0x3e, 0x13,
+	0xb9, 0x5d, 0x5d, 0xb5, 0xeb, 0xae, 0xc0, 0x87,
+	0xbb, 0xfd, 0xe7, 0xe4, 0x89, 0x5b, 0xd2, 0x6c,
+	0x56
+};
+static const u8 key42[] __initconst = {
+	0x7c, 0x6b, 0x7e, 0x77, 0xcc, 0x8c, 0x1b, 0x03,
+	0x8b, 0x2a, 0xb3, 0x7c, 0x5a, 0x73, 0xcc, 0xac,
+	0xdd, 0x53, 0x54, 0x0c, 0x85, 0xed, 0xcd, 0x47,
+	0x24, 0xc1, 0xb8, 0x9b, 0x2e, 0x41, 0x92, 0x36
+};
+enum { nonce42 = 0xe4d7348b09682c9cULL };
+
+static const u8 input43[] __initconst = {
+	0x52, 0xf2, 0x4b, 0x7c, 0xe5, 0x58, 0xe8, 0xd2,
+	0xb7, 0xf3, 0xa1, 0x29, 0x68, 0xa2, 0x50, 0x50,
+	0xae, 0x9c, 0x1b, 0xe2, 0x67, 0x77, 0xe2, 0xdb,
+	0x85, 0x55, 0x7e, 0x84, 0x8a, 0x12, 0x3c, 0xb6,
+	0x2e, 0xed, 0xd3, 0xec, 0x47, 0x68, 0xfa, 0x52,
+	0x46, 0x9d
+};
+static const u8 output43[] __initconst = {
+	0x1b, 0xf0, 0x05, 0xe4, 0x1c, 0xd8, 0x74, 0x9a,
+	0xf0, 0xee, 0x00, 0x54, 0xce, 0x02, 0x83, 0x15,
+	0xfb, 0x23, 0x35, 0x78, 0xc3, 0xda, 0x98, 0xd8,
+	0x9d, 0x1b, 0xb2, 0x51, 0x82, 0xb0, 0xff, 0xbe,
+	0x05, 0xa9, 0xa4, 0x04, 0xba, 0xea, 0x4b, 0x73,
+	0x47, 0x6e
+};
+static const u8 key43[] __initconst = {
+	0xeb, 0xec, 0x0e, 0xa1, 0x65, 0xe2, 0x99, 0x46,
+	0xd8, 0x54, 0x8c, 0x4a, 0x93, 0xdf, 0x6d, 0xbf,
+	0x93, 0x34, 0x94, 0x57, 0xc9, 0x12, 0x9d, 0x68,
+	0x05, 0xc5, 0x05, 0xad, 0x5a, 0xc9, 0x2a, 0x3b
+};
+enum { nonce43 = 0xe14f6a902b7827fULL };
+
+static const u8 input44[] __initconst = {
+	0x3e, 0x22, 0x3e, 0x8e, 0xcd, 0x18, 0xe2, 0xa3,
+	0x8d, 0x8b, 0x38, 0xc3, 0x02, 0xa3, 0x31, 0x48,
+	0xc6, 0x0e, 0xec, 0x99, 0x51, 0x11, 0x6d, 0x8b,
+	0x32, 0x35, 0x3b, 0x08, 0x58, 0x76, 0x25, 0x30,
+	0xe2, 0xfc, 0xa2, 0x46, 0x7d, 0x6e, 0x34, 0x87,
+	0xac, 0x42, 0xbf
+};
+static const u8 output44[] __initconst = {
+	0x08, 0x92, 0x58, 0x02, 0x1a, 0xf4, 0x1f, 0x3d,
+	0x38, 0x7b, 0x6b, 0xf6, 0x84, 0x07, 0xa3, 0x19,
+	0x17, 0x2a, 0xed, 0x57, 0x1c, 0xf9, 0x55, 0x37,
+	0x4e, 0xf4, 0x68, 0x68, 0x82, 0x02, 0x4f, 0xca,
+	0x21, 0x00, 0xc6, 0x66, 0x79, 0x53, 0x19, 0xef,
+	0x7f, 0xdd, 0x74
+};
+static const u8 key44[] __initconst = {
+	0x73, 0xb6, 0x3e, 0xf4, 0x57, 0x52, 0xa6, 0x43,
+	0x51, 0xd8, 0x25, 0x00, 0xdb, 0xb4, 0x52, 0x69,
+	0xd6, 0x27, 0x49, 0xeb, 0x9b, 0xf1, 0x7b, 0xa0,
+	0xd6, 0x7c, 0x9c, 0xd8, 0x95, 0x03, 0x69, 0x26
+};
+enum { nonce44 = 0xf5e6dc4f35ce24e5ULL };
+
+static const u8 input45[] __initconst = {
+	0x55, 0x76, 0xc0, 0xf1, 0x74, 0x03, 0x7a, 0x6d,
+	0x14, 0xd8, 0x36, 0x2c, 0x9f, 0x9a, 0x59, 0x7a,
+	0x2a, 0xf5, 0x77, 0x84, 0x70, 0x7c, 0x1d, 0x04,
+	0x90, 0x45, 0xa4, 0xc1, 0x5e, 0xdd, 0x2e, 0x07,
+	0x18, 0x34, 0xa6, 0x85, 0x56, 0x4f, 0x09, 0xaf,
+	0x2f, 0x83, 0xe1, 0xc6
+};
+static const u8 output45[] __initconst = {
+	0x22, 0x46, 0xe4, 0x0b, 0x3a, 0x55, 0xcc, 0x9b,
+	0xf0, 0xc0, 0x53, 0xcd, 0x95, 0xc7, 0x57, 0x6c,
+	0x77, 0x46, 0x41, 0x72, 0x07, 0xbf, 0xa8, 0xe5,
+	0x68, 0x69, 0xd8, 0x1e, 0x45, 0xc1, 0xa2, 0x50,
+	0xa5, 0xd1, 0x62, 0xc9, 0x5a, 0x7d, 0x08, 0x14,
+	0xae, 0x44, 0x16, 0xb9
+};
+static const u8 key45[] __initconst = {
+	0x41, 0xf3, 0x88, 0xb2, 0x51, 0x25, 0x47, 0x02,
+	0x39, 0xe8, 0x15, 0x3a, 0x22, 0x78, 0x86, 0x0b,
+	0xf9, 0x1e, 0x8d, 0x98, 0xb2, 0x22, 0x82, 0xac,
+	0x42, 0x94, 0xde, 0x64, 0xf0, 0xfd, 0xb3, 0x6c
+};
+enum { nonce45 = 0xf51a582daf4aa01aULL };
+
+static const u8 input46[] __initconst = {
+	0xf6, 0xff, 0x20, 0xf9, 0x26, 0x7e, 0x0f, 0xa8,
+	0x6a, 0x45, 0x5a, 0x91, 0x73, 0xc4, 0x4c, 0x63,
+	0xe5, 0x61, 0x59, 0xca, 0xec, 0xc0, 0x20, 0x35,
+	0xbc, 0x9f, 0x58, 0x9c, 0x5e, 0xa1, 0x17, 0x46,
+	0xcc, 0xab, 0x6e, 0xd0, 0x4f, 0x24, 0xeb, 0x05,
+	0x4d, 0x40, 0x41, 0xe0, 0x9d
+};
+static const u8 output46[] __initconst = {
+	0x31, 0x6e, 0x63, 0x3f, 0x9c, 0xe6, 0xb1, 0xb7,
+	0xef, 0x47, 0x46, 0xd7, 0xb1, 0x53, 0x42, 0x2f,
+	0x2c, 0xc8, 0x01, 0xae, 0x8b, 0xec, 0x42, 0x2c,
+	0x6b, 0x2c, 0x9c, 0xb2, 0xf0, 0x29, 0x06, 0xa5,
+	0xcd, 0x7e, 0xc7, 0x3a, 0x38, 0x98, 0x8a, 0xde,
+	0x03, 0x29, 0x14, 0x8f, 0xf9
+};
+static const u8 key46[] __initconst = {
+	0xac, 0xa6, 0x44, 0x4a, 0x0d, 0x42, 0x10, 0xbc,
+	0xd3, 0xc9, 0x8e, 0x9e, 0x71, 0xa3, 0x1c, 0x14,
+	0x9d, 0x65, 0x0d, 0x49, 0x4d, 0x8c, 0xec, 0x46,
+	0xe1, 0x41, 0xcd, 0xf5, 0xfc, 0x82, 0x75, 0x34
+};
+enum { nonce46 = 0x25f85182df84dec5ULL };
+
+static const u8 input47[] __initconst = {
+	0xa1, 0xd2, 0xf2, 0x52, 0x2f, 0x79, 0x50, 0xb2,
+	0x42, 0x29, 0x5b, 0x44, 0x20, 0xf9, 0xbd, 0x85,
+	0xb7, 0x65, 0x77, 0x86, 0xce, 0x3e, 0x1c, 0xe4,
+	0x70, 0x80, 0xdd, 0x72, 0x07, 0x48, 0x0f, 0x84,
+	0x0d, 0xfd, 0x97, 0xc0, 0xb7, 0x48, 0x9b, 0xb4,
+	0xec, 0xff, 0x73, 0x14, 0x99, 0xe4
+};
+static const u8 output47[] __initconst = {
+	0xe5, 0x3c, 0x78, 0x66, 0x31, 0x1e, 0xd6, 0xc4,
+	0x9e, 0x71, 0xb3, 0xd7, 0xd5, 0xad, 0x84, 0xf2,
+	0x78, 0x61, 0x77, 0xf8, 0x31, 0xf0, 0x13, 0xad,
+	0x66, 0xf5, 0x31, 0x7d, 0xeb, 0xdf, 0xaf, 0xcb,
+	0xac, 0x28, 0x6c, 0xc2, 0x9e, 0xe7, 0x78, 0xa2,
+	0xa2, 0x58, 0xce, 0x84, 0x76, 0x70
+};
+static const u8 key47[] __initconst = {
+	0x05, 0x7f, 0xc0, 0x7f, 0x37, 0x20, 0x71, 0x02,
+	0x3a, 0xe7, 0x20, 0x5a, 0x0a, 0x8f, 0x79, 0x5a,
+	0xfe, 0xbb, 0x43, 0x4d, 0x2f, 0xcb, 0xf6, 0x9e,
+	0xa2, 0x97, 0x00, 0xad, 0x0d, 0x51, 0x7e, 0x17
+};
+enum { nonce47 = 0xae707c60f54de32bULL };
+
+static const u8 input48[] __initconst = {
+	0x80, 0x93, 0x77, 0x2e, 0x8d, 0xe8, 0xe6, 0xc1,
+	0x27, 0xe6, 0xf2, 0x89, 0x5b, 0x33, 0x62, 0x18,
+	0x80, 0x6e, 0x17, 0x22, 0x8e, 0x83, 0x31, 0x40,
+	0x8f, 0xc9, 0x5c, 0x52, 0x6c, 0x0e, 0xa5, 0xe9,
+	0x6c, 0x7f, 0xd4, 0x6a, 0x27, 0x56, 0x99, 0xce,
+	0x8d, 0x37, 0x59, 0xaf, 0xc0, 0x0e, 0xe1
+};
+static const u8 output48[] __initconst = {
+	0x02, 0xa4, 0x2e, 0x33, 0xb7, 0x7c, 0x2b, 0x9a,
+	0x18, 0x5a, 0xba, 0x53, 0x38, 0xaf, 0x00, 0xeb,
+	0xd8, 0x3d, 0x02, 0x77, 0x43, 0x45, 0x03, 0x91,
+	0xe2, 0x5e, 0x4e, 0xeb, 0x50, 0xd5, 0x5b, 0xe0,
+	0xf3, 0x33, 0xa7, 0xa2, 0xac, 0x07, 0x6f, 0xeb,
+	0x3f, 0x6c, 0xcd, 0xf2, 0x6c, 0x61, 0x64
+};
+static const u8 key48[] __initconst = {
+	0xf3, 0x79, 0xe7, 0xf8, 0x0e, 0x02, 0x05, 0x6b,
+	0x83, 0x1a, 0xe7, 0x86, 0x6b, 0xe6, 0x8f, 0x3f,
+	0xd3, 0xa3, 0xe4, 0x6e, 0x29, 0x06, 0xad, 0xbc,
+	0xe8, 0x33, 0x56, 0x39, 0xdf, 0xb0, 0xe2, 0xfe
+};
+enum { nonce48 = 0xd849b938c6569da0ULL };
+
+static const u8 input49[] __initconst = {
+	0x89, 0x3b, 0x88, 0x9e, 0x7b, 0x38, 0x16, 0x9f,
+	0xa1, 0x28, 0xf6, 0xf5, 0x23, 0x74, 0x28, 0xb0,
+	0xdf, 0x6c, 0x9e, 0x8a, 0x71, 0xaf, 0xed, 0x7a,
+	0x39, 0x21, 0x57, 0x7d, 0x31, 0x6c, 0xee, 0x0d,
+	0x11, 0x8d, 0x41, 0x9a, 0x5f, 0xb7, 0x27, 0x40,
+	0x08, 0xad, 0xc6, 0xe0, 0x00, 0x43, 0x9e, 0xae
+};
+static const u8 output49[] __initconst = {
+	0x4d, 0xfd, 0xdb, 0x4c, 0x77, 0xc1, 0x05, 0x07,
+	0x4d, 0x6d, 0x32, 0xcb, 0x2e, 0x0e, 0xff, 0x65,
+	0xc9, 0x27, 0xeb, 0xa9, 0x46, 0x5b, 0xab, 0x06,
+	0xe6, 0xb6, 0x5a, 0x1e, 0x00, 0xfb, 0xcf, 0xe4,
+	0xb9, 0x71, 0x40, 0x10, 0xef, 0x12, 0x39, 0xf0,
+	0xea, 0x40, 0xb8, 0x9a, 0xa2, 0x85, 0x38, 0x48
+};
+static const u8 key49[] __initconst = {
+	0xe7, 0x10, 0x40, 0xd9, 0x66, 0xc0, 0xa8, 0x6d,
+	0xa3, 0xcc, 0x8b, 0xdd, 0x93, 0xf2, 0x6e, 0xe0,
+	0x90, 0x7f, 0xd0, 0xf4, 0x37, 0x0c, 0x8b, 0x9b,
+	0x4c, 0x4d, 0xe6, 0xf2, 0x1f, 0xe9, 0x95, 0x24
+};
+enum { nonce49 = 0xf269817bdae01bc0ULL };
+
+static const u8 input50[] __initconst = {
+	0xda, 0x5b, 0x60, 0xcd, 0xed, 0x58, 0x8e, 0x7f,
+	0xae, 0xdd, 0xc8, 0x2e, 0x16, 0x90, 0xea, 0x4b,
+	0x0c, 0x74, 0x14, 0x35, 0xeb, 0xee, 0x2c, 0xff,
+	0x46, 0x99, 0x97, 0x6e, 0xae, 0xa7, 0x8e, 0x6e,
+	0x38, 0xfe, 0x63, 0xe7, 0x51, 0xd9, 0xaa, 0xce,
+	0x7b, 0x1e, 0x7e, 0x5d, 0xc0, 0xe8, 0x10, 0x06,
+	0x14
+};
+static const u8 output50[] __initconst = {
+	0xe4, 0xe5, 0x86, 0x1b, 0x66, 0x19, 0xac, 0x49,
+	0x1c, 0xbd, 0xee, 0x03, 0xaf, 0x11, 0xfc, 0x1f,
+	0x6a, 0xd2, 0x50, 0x5c, 0xea, 0x2c, 0xa5, 0x75,
+	0xfd, 0xb7, 0x0e, 0x80, 0x8f, 0xed, 0x3f, 0x31,
+	0x47, 0xac, 0x67, 0x43, 0xb8, 0x2e, 0xb4, 0x81,
+	0x6d, 0xe4, 0x1e, 0xb7, 0x8b, 0x0c, 0x53, 0xa9,
+	0x26
+};
+static const u8 key50[] __initconst = {
+	0xd7, 0xb2, 0x04, 0x76, 0x30, 0xcc, 0x38, 0x45,
+	0xef, 0xdb, 0xc5, 0x86, 0x08, 0x61, 0xf0, 0xee,
+	0x6d, 0xd8, 0x22, 0x04, 0x8c, 0xfb, 0xcb, 0x37,
+	0xa6, 0xfb, 0x95, 0x22, 0xe1, 0x87, 0xb7, 0x6f
+};
+enum { nonce50 = 0x3b44d09c45607d38ULL };
+
+static const u8 input51[] __initconst = {
+	0xa9, 0x41, 0x02, 0x4b, 0xd7, 0xd5, 0xd1, 0xf1,
+	0x21, 0x55, 0xb2, 0x75, 0x6d, 0x77, 0x1b, 0x86,
+	0xa9, 0xc8, 0x90, 0xfd, 0xed, 0x4a, 0x7b, 0x6c,
+	0xb2, 0x5f, 0x9b, 0x5f, 0x16, 0xa1, 0x54, 0xdb,
+	0xd6, 0x3f, 0x6a, 0x7f, 0x2e, 0x51, 0x9d, 0x49,
+	0x5b, 0xa5, 0x0e, 0xf9, 0xfb, 0x2a, 0x38, 0xff,
+	0x20, 0x8c
+};
+static const u8 output51[] __initconst = {
+	0x18, 0xf7, 0x88, 0xc1, 0x72, 0xfd, 0x90, 0x4b,
+	0xa9, 0x2d, 0xdb, 0x47, 0xb0, 0xa5, 0xc4, 0x37,
+	0x01, 0x95, 0xc4, 0xb1, 0xab, 0xc5, 0x5b, 0xcd,
+	0xe1, 0x97, 0x78, 0x13, 0xde, 0x6a, 0xff, 0x36,
+	0xce, 0xa4, 0x67, 0xc5, 0x4a, 0x45, 0x2b, 0xd9,
+	0xff, 0x8f, 0x06, 0x7c, 0x63, 0xbb, 0x83, 0x17,
+	0xb4, 0x6b
+};
+static const u8 key51[] __initconst = {
+	0x82, 0x1a, 0x79, 0xab, 0x9a, 0xb5, 0x49, 0x6a,
+	0x30, 0x6b, 0x99, 0x19, 0x11, 0xc7, 0xa2, 0xf4,
+	0xca, 0x55, 0xb9, 0xdd, 0xe7, 0x2f, 0xe7, 0xc1,
+	0xdd, 0x27, 0xad, 0x80, 0xf2, 0x56, 0xad, 0xf3
+};
+enum { nonce51 = 0xe93aff94ca71a4a6ULL };
+
+static const u8 input52[] __initconst = {
+	0x89, 0xdd, 0xf3, 0xfa, 0xb6, 0xc1, 0xaa, 0x9a,
+	0xc8, 0xad, 0x6b, 0x00, 0xa1, 0x65, 0xea, 0x14,
+	0x55, 0x54, 0x31, 0x8f, 0xf0, 0x03, 0x84, 0x51,
+	0x17, 0x1e, 0x0a, 0x93, 0x6e, 0x79, 0x96, 0xa3,
+	0x2a, 0x85, 0x9c, 0x89, 0xf8, 0xd1, 0xe2, 0x15,
+	0x95, 0x05, 0xf4, 0x43, 0x4d, 0x6b, 0xf0, 0x71,
+	0x3b, 0x3e, 0xba
+};
+static const u8 output52[] __initconst = {
+	0x0c, 0x42, 0x6a, 0xb3, 0x66, 0x63, 0x5d, 0x2c,
+	0x9f, 0x3d, 0xa6, 0x6e, 0xc7, 0x5f, 0x79, 0x2f,
+	0x50, 0xe3, 0xd6, 0x07, 0x56, 0xa4, 0x2b, 0x2d,
+	0x8d, 0x10, 0xc0, 0x6c, 0xa2, 0xfc, 0x97, 0xec,
+	0x3f, 0x5c, 0x8d, 0x59, 0xbe, 0x84, 0xf1, 0x3e,
+	0x38, 0x47, 0x4f, 0x75, 0x25, 0x66, 0x88, 0x14,
+	0x03, 0xdd, 0xde
+};
+static const u8 key52[] __initconst = {
+	0x4f, 0xb0, 0x27, 0xb6, 0xdd, 0x24, 0x0c, 0xdb,
+	0x6b, 0x71, 0x2e, 0xac, 0xfc, 0x3f, 0xa6, 0x48,
+	0x5d, 0xd5, 0xff, 0x53, 0xb5, 0x62, 0xf1, 0xe0,
+	0x93, 0xfe, 0x39, 0x4c, 0x9f, 0x03, 0x11, 0xa7
+};
+enum { nonce52 = 0xed8becec3bdf6f25ULL };
+
+static const u8 input53[] __initconst = {
+	0x68, 0xd1, 0xc7, 0x74, 0x44, 0x1c, 0x84, 0xde,
+	0x27, 0x27, 0x35, 0xf0, 0x18, 0x0b, 0x57, 0xaa,
+	0xd0, 0x1a, 0xd3, 0x3b, 0x5e, 0x5c, 0x62, 0x93,
+	0xd7, 0x6b, 0x84, 0x3b, 0x71, 0x83, 0x77, 0x01,
+	0x3e, 0x59, 0x45, 0xf4, 0x77, 0x6c, 0x6b, 0xcb,
+	0x88, 0x45, 0x09, 0x1d, 0xc6, 0x45, 0x6e, 0xdc,
+	0x6e, 0x51, 0xb8, 0x28
+};
+static const u8 output53[] __initconst = {
+	0xc5, 0x90, 0x96, 0x78, 0x02, 0xf5, 0xc4, 0x3c,
+	0xde, 0xd4, 0xd4, 0xc6, 0xa7, 0xad, 0x12, 0x47,
+	0x45, 0xce, 0xcd, 0x8c, 0x35, 0xcc, 0xa6, 0x9e,
+	0x5a, 0xc6, 0x60, 0xbb, 0xe3, 0xed, 0xec, 0x68,
+	0x3f, 0x64, 0xf7, 0x06, 0x63, 0x9c, 0x8c, 0xc8,
+	0x05, 0x3a, 0xad, 0x32, 0x79, 0x8b, 0x45, 0x96,
+	0x93, 0x73, 0x4c, 0xe0
+};
+static const u8 key53[] __initconst = {
+	0x42, 0x4b, 0x20, 0x81, 0x49, 0x50, 0xe9, 0xc2,
+	0x43, 0x69, 0x36, 0xe7, 0x68, 0xae, 0xd5, 0x7e,
+	0x42, 0x1a, 0x1b, 0xb4, 0x06, 0x4d, 0xa7, 0x17,
+	0xb5, 0x31, 0xd6, 0x0c, 0xb0, 0x5c, 0x41, 0x0b
+};
+enum { nonce53 = 0xf44ce1931fbda3d7ULL };
+
+static const u8 input54[] __initconst = {
+	0x7b, 0xf6, 0x8b, 0xae, 0xc0, 0xcb, 0x10, 0x8e,
+	0xe8, 0xd8, 0x2e, 0x3b, 0x14, 0xba, 0xb4, 0xd2,
+	0x58, 0x6b, 0x2c, 0xec, 0xc1, 0x81, 0x71, 0xb4,
+	0xc6, 0xea, 0x08, 0xc5, 0xc9, 0x78, 0xdb, 0xa2,
+	0xfa, 0x44, 0x50, 0x9b, 0xc8, 0x53, 0x8d, 0x45,
+	0x42, 0xe7, 0x09, 0xc4, 0x29, 0xd8, 0x75, 0x02,
+	0xbb, 0xb2, 0x78, 0xcf, 0xe7
+};
+static const u8 output54[] __initconst = {
+	0xaf, 0x2c, 0x83, 0x26, 0x6e, 0x7f, 0xa6, 0xe9,
+	0x03, 0x75, 0xfe, 0xfe, 0x87, 0x58, 0xcf, 0xb5,
+	0xbc, 0x3c, 0x9d, 0xa1, 0x6e, 0x13, 0xf1, 0x0f,
+	0x9e, 0xbc, 0xe0, 0x54, 0x24, 0x32, 0xce, 0x95,
+	0xe6, 0xa5, 0x59, 0x3d, 0x24, 0x1d, 0x8f, 0xb1,
+	0x74, 0x6c, 0x56, 0xe7, 0x96, 0xc1, 0x91, 0xc8,
+	0x2d, 0x0e, 0xb7, 0x51, 0x10
+};
+static const u8 key54[] __initconst = {
+	0x00, 0x68, 0x74, 0xdc, 0x30, 0x9e, 0xe3, 0x52,
+	0xa9, 0xae, 0xb6, 0x7c, 0xa1, 0xdc, 0x12, 0x2d,
+	0x98, 0x32, 0x7a, 0x77, 0xe1, 0xdd, 0xa3, 0x76,
+	0x72, 0x34, 0x83, 0xd8, 0xb7, 0x69, 0xba, 0x77
+};
+enum { nonce54 = 0xbea57d79b798b63aULL };
+
+static const u8 input55[] __initconst = {
+	0xb5, 0xf4, 0x2f, 0xc1, 0x5e, 0x10, 0xa7, 0x4e,
+	0x74, 0x3d, 0xa3, 0x96, 0xc0, 0x4d, 0x7b, 0x92,
+	0x8f, 0xdb, 0x2d, 0x15, 0x52, 0x6a, 0x95, 0x5e,
+	0x40, 0x81, 0x4f, 0x70, 0x73, 0xea, 0x84, 0x65,
+	0x3d, 0x9a, 0x4e, 0x03, 0x95, 0xf8, 0x5d, 0x2f,
+	0x07, 0x02, 0x13, 0x13, 0xdd, 0x82, 0xe6, 0x3b,
+	0xe1, 0x5f, 0xb3, 0x37, 0x9b, 0x88
+};
+static const u8 output55[] __initconst = {
+	0xc1, 0x88, 0xbd, 0x92, 0x77, 0xad, 0x7c, 0x5f,
+	0xaf, 0xa8, 0x57, 0x0e, 0x40, 0x0a, 0xdc, 0x70,
+	0xfb, 0xc6, 0x71, 0xfd, 0xc4, 0x74, 0x60, 0xcc,
+	0xa0, 0x89, 0x8e, 0x99, 0xf0, 0x06, 0xa6, 0x7c,
+	0x97, 0x42, 0x21, 0x81, 0x6a, 0x07, 0xe7, 0xb3,
+	0xf7, 0xa5, 0x03, 0x71, 0x50, 0x05, 0x63, 0x17,
+	0xa9, 0x46, 0x0b, 0xff, 0x30, 0x78
+};
+static const u8 key55[] __initconst = {
+	0x19, 0x8f, 0xe7, 0xd7, 0x6b, 0x7f, 0x6f, 0x69,
+	0x86, 0x91, 0x0f, 0xa7, 0x4a, 0x69, 0x8e, 0x34,
+	0xf3, 0xdb, 0xde, 0xaf, 0xf2, 0x66, 0x1d, 0x64,
+	0x97, 0x0c, 0xcf, 0xfa, 0x33, 0x84, 0xfd, 0x0c
+};
+enum { nonce55 = 0x80aa3d3e2c51ef06ULL };
+
+static const u8 input56[] __initconst = {
+	0x6b, 0xe9, 0x73, 0x42, 0x27, 0x5e, 0x12, 0xcd,
+	0xaa, 0x45, 0x12, 0x8b, 0xb3, 0xe6, 0x54, 0x33,
+	0x31, 0x7d, 0xe2, 0x25, 0xc6, 0x86, 0x47, 0x67,
+	0x86, 0x83, 0xe4, 0x46, 0xb5, 0x8f, 0x2c, 0xbb,
+	0xe4, 0xb8, 0x9f, 0xa2, 0xa4, 0xe8, 0x75, 0x96,
+	0x92, 0x51, 0x51, 0xac, 0x8e, 0x2e, 0x6f, 0xfc,
+	0xbd, 0x0d, 0xa3, 0x9f, 0x16, 0x55, 0x3e
+};
+static const u8 output56[] __initconst = {
+	0x42, 0x99, 0x73, 0x6c, 0xd9, 0x4b, 0x16, 0xe5,
+	0x18, 0x63, 0x1a, 0xd9, 0x0e, 0xf1, 0x15, 0x2e,
+	0x0f, 0x4b, 0xe4, 0x5f, 0xa0, 0x4d, 0xde, 0x9f,
+	0xa7, 0x18, 0xc1, 0x0c, 0x0b, 0xae, 0x55, 0xe4,
+	0x89, 0x18, 0xa4, 0x78, 0x9d, 0x25, 0x0d, 0xd5,
+	0x94, 0x0f, 0xf9, 0x78, 0xa3, 0xa6, 0xe9, 0x9e,
+	0x2c, 0x73, 0xf0, 0xf7, 0x35, 0xf3, 0x2b
+};
+static const u8 key56[] __initconst = {
+	0x7d, 0x12, 0xad, 0x51, 0xd5, 0x6f, 0x8f, 0x96,
+	0xc0, 0x5d, 0x9a, 0xd1, 0x7e, 0x20, 0x98, 0x0e,
+	0x3c, 0x0a, 0x67, 0x6b, 0x1b, 0x88, 0x69, 0xd4,
+	0x07, 0x8c, 0xaf, 0x0f, 0x3a, 0x28, 0xe4, 0x5d
+};
+enum { nonce56 = 0x70f4c372fb8b5984ULL };
+
+static const u8 input57[] __initconst = {
+	0x28, 0xa3, 0x06, 0xe8, 0xe7, 0x08, 0xb9, 0xef,
+	0x0d, 0x63, 0x15, 0x99, 0xb2, 0x78, 0x7e, 0xaf,
+	0x30, 0x50, 0xcf, 0xea, 0xc9, 0x91, 0x41, 0x2f,
+	0x3b, 0x38, 0x70, 0xc4, 0x87, 0xb0, 0x3a, 0xee,
+	0x4a, 0xea, 0xe3, 0x83, 0x68, 0x8b, 0xcf, 0xda,
+	0x04, 0xa5, 0xbd, 0xb2, 0xde, 0x3c, 0x55, 0x13,
+	0xfe, 0x96, 0xad, 0xc1, 0x61, 0x1b, 0x98, 0xde
+};
+static const u8 output57[] __initconst = {
+	0xf4, 0x44, 0xe9, 0xd2, 0x6d, 0xc2, 0x5a, 0xe9,
+	0xfd, 0x7e, 0x41, 0x54, 0x3f, 0xf4, 0x12, 0xd8,
+	0x55, 0x0d, 0x12, 0x9b, 0xd5, 0x2e, 0x95, 0xe5,
+	0x77, 0x42, 0x3f, 0x2c, 0xfb, 0x28, 0x9d, 0x72,
+	0x6d, 0x89, 0x82, 0x27, 0x64, 0x6f, 0x0d, 0x57,
+	0xa1, 0x25, 0xa3, 0x6b, 0x88, 0x9a, 0xac, 0x0c,
+	0x76, 0x19, 0x90, 0xe2, 0x50, 0x5a, 0xf8, 0x12
+};
+static const u8 key57[] __initconst = {
+	0x08, 0x26, 0xb8, 0xac, 0xf3, 0xa5, 0xc6, 0xa3,
+	0x7f, 0x09, 0x87, 0xf5, 0x6c, 0x5a, 0x85, 0x6c,
+	0x3d, 0xbd, 0xde, 0xd5, 0x87, 0xa3, 0x98, 0x7a,
+	0xaa, 0x40, 0x3e, 0xf7, 0xff, 0x44, 0x5d, 0xee
+};
+enum { nonce57 = 0xc03a6130bf06b089ULL };
+
+static const u8 input58[] __initconst = {
+	0x82, 0xa5, 0x38, 0x6f, 0xaa, 0xb4, 0xaf, 0xb2,
+	0x42, 0x01, 0xa8, 0x39, 0x3f, 0x15, 0x51, 0xa8,
+	0x11, 0x1b, 0x93, 0xca, 0x9c, 0xa0, 0x57, 0x68,
+	0x8f, 0xdb, 0x68, 0x53, 0x51, 0x6d, 0x13, 0x22,
+	0x12, 0x9b, 0xbd, 0x33, 0xa8, 0x52, 0x40, 0x57,
+	0x80, 0x9b, 0x98, 0xef, 0x56, 0x70, 0x11, 0xfa,
+	0x36, 0x69, 0x7d, 0x15, 0x48, 0xf9, 0x3b, 0xeb,
+	0x42
+};
+static const u8 output58[] __initconst = {
+	0xff, 0x3a, 0x74, 0xc3, 0x3e, 0x44, 0x64, 0x4d,
+	0x0e, 0x5f, 0x9d, 0xa8, 0xdb, 0xbe, 0x12, 0xef,
+	0xba, 0x56, 0x65, 0x50, 0x76, 0xaf, 0xa4, 0x4e,
+	0x01, 0xc1, 0xd3, 0x31, 0x14, 0xe2, 0xbe, 0x7b,
+	0xa5, 0x67, 0xb4, 0xe3, 0x68, 0x40, 0x9c, 0xb0,
+	0xb1, 0x78, 0xef, 0x49, 0x03, 0x0f, 0x2d, 0x56,
+	0xb4, 0x37, 0xdb, 0xbc, 0x2d, 0x68, 0x1c, 0x3c,
+	0xf1
+};
+static const u8 key58[] __initconst = {
+	0x7e, 0xf1, 0x7c, 0x20, 0x65, 0xed, 0xcd, 0xd7,
+	0x57, 0xe8, 0xdb, 0x90, 0x87, 0xdb, 0x5f, 0x63,
+	0x3d, 0xdd, 0xb8, 0x2b, 0x75, 0x8e, 0x04, 0xb5,
+	0xf4, 0x12, 0x79, 0xa9, 0x4d, 0x42, 0x16, 0x7f
+};
+enum { nonce58 = 0x92838183f80d2f7fULL };
+
+static const u8 input59[] __initconst = {
+	0x37, 0xf1, 0x9d, 0xdd, 0xd7, 0x08, 0x9f, 0x13,
+	0xc5, 0x21, 0x82, 0x75, 0x08, 0x9e, 0x25, 0x16,
+	0xb1, 0xd1, 0x71, 0x42, 0x28, 0x63, 0xac, 0x47,
+	0x71, 0x54, 0xb1, 0xfc, 0x39, 0xf0, 0x61, 0x4f,
+	0x7c, 0x6d, 0x4f, 0xc8, 0x33, 0xef, 0x7e, 0xc8,
+	0xc0, 0x97, 0xfc, 0x1a, 0x61, 0xb4, 0x87, 0x6f,
+	0xdd, 0x5a, 0x15, 0x7b, 0x1b, 0x95, 0x50, 0x94,
+	0x1d, 0xba
+};
+static const u8 output59[] __initconst = {
+	0x73, 0x67, 0xc5, 0x07, 0xbb, 0x57, 0x79, 0xd5,
+	0xc9, 0x04, 0xdd, 0x88, 0xf3, 0x86, 0xe5, 0x70,
+	0x49, 0x31, 0xe0, 0xcc, 0x3b, 0x1d, 0xdf, 0xb0,
+	0xaf, 0xf4, 0x2d, 0xe0, 0x06, 0x10, 0x91, 0x8d,
+	0x1c, 0xcf, 0x31, 0x0b, 0xf6, 0x73, 0xda, 0x1c,
+	0xf0, 0x17, 0x52, 0x9e, 0x20, 0x2e, 0x9f, 0x8c,
+	0xb3, 0x59, 0xce, 0xd4, 0xd3, 0xc1, 0x81, 0xe9,
+	0x11, 0x36
+};
+static const u8 key59[] __initconst = {
+	0xbd, 0x07, 0xd0, 0x53, 0x2c, 0xb3, 0xcc, 0x3f,
+	0xc4, 0x95, 0xfd, 0xe7, 0x81, 0xb3, 0x29, 0x99,
+	0x05, 0x45, 0xd6, 0x95, 0x25, 0x0b, 0x72, 0xd3,
+	0xcd, 0xbb, 0x73, 0xf8, 0xfa, 0xc0, 0x9b, 0x7a
+};
+enum { nonce59 = 0x4a0db819b0d519e2ULL };
+
+static const u8 input60[] __initconst = {
+	0x58, 0x4e, 0xdf, 0x94, 0x3c, 0x76, 0x0a, 0x79,
+	0x47, 0xf1, 0xbe, 0x88, 0xd3, 0xba, 0x94, 0xd8,
+	0xe2, 0x8f, 0xe3, 0x2f, 0x2f, 0x74, 0x82, 0x55,
+	0xc3, 0xda, 0xe2, 0x4e, 0x2c, 0x8c, 0x45, 0x1d,
+	0x72, 0x8f, 0x54, 0x41, 0xb5, 0xb7, 0x69, 0xe4,
+	0xdc, 0xd2, 0x36, 0x21, 0x5c, 0x28, 0x52, 0xf7,
+	0x98, 0x8e, 0x72, 0xa7, 0x6d, 0x57, 0xed, 0xdc,
+	0x3c, 0xe6, 0x6a
+};
+static const u8 output60[] __initconst = {
+	0xda, 0xaf, 0xb5, 0xe3, 0x30, 0x65, 0x5c, 0xb1,
+	0x48, 0x08, 0x43, 0x7b, 0x9e, 0xd2, 0x6a, 0x62,
+	0x56, 0x7c, 0xad, 0xd9, 0xe5, 0xf6, 0x09, 0x71,
+	0xcd, 0xe6, 0x05, 0x6b, 0x3f, 0x44, 0x3a, 0x5c,
+	0xf6, 0xf8, 0xd7, 0xce, 0x7d, 0xd1, 0xe0, 0x4f,
+	0x88, 0x15, 0x04, 0xd8, 0x20, 0xf0, 0x3e, 0xef,
+	0xae, 0xa6, 0x27, 0xa3, 0x0e, 0xfc, 0x18, 0x90,
+	0x33, 0xcd, 0xd3
+};
+static const u8 key60[] __initconst = {
+	0xbf, 0xfd, 0x25, 0xb5, 0xb2, 0xfc, 0x78, 0x0c,
+	0x8e, 0xb9, 0x57, 0x2f, 0x26, 0x4a, 0x7e, 0x71,
+	0xcc, 0xf2, 0xe0, 0xfd, 0x24, 0x11, 0x20, 0x23,
+	0x57, 0x00, 0xff, 0x80, 0x11, 0x0c, 0x1e, 0xff
+};
+enum { nonce60 = 0xf18df56fdb7954adULL };
+
+static const u8 input61[] __initconst = {
+	0xb0, 0xf3, 0x06, 0xbc, 0x22, 0xae, 0x49, 0x40,
+	0xae, 0xff, 0x1b, 0x31, 0xa7, 0x98, 0xab, 0x1d,
+	0xe7, 0x40, 0x23, 0x18, 0x4f, 0xab, 0x8e, 0x93,
+	0x82, 0xf4, 0x56, 0x61, 0xfd, 0x2b, 0xcf, 0xa7,
+	0xc4, 0xb4, 0x0a, 0xf4, 0xcb, 0xc7, 0x8c, 0x40,
+	0x57, 0xac, 0x0b, 0x3e, 0x2a, 0x0a, 0x67, 0x83,
+	0x50, 0xbf, 0xec, 0xb0, 0xc7, 0xf1, 0x32, 0x26,
+	0x98, 0x80, 0x33, 0xb4
+};
+static const u8 output61[] __initconst = {
+	0x9d, 0x23, 0x0e, 0xff, 0xcc, 0x7c, 0xd5, 0xcf,
+	0x1a, 0xb8, 0x59, 0x1e, 0x92, 0xfd, 0x7f, 0xca,
+	0xca, 0x3c, 0x18, 0x81, 0xde, 0xfa, 0x59, 0xc8,
+	0x6f, 0x9c, 0x24, 0x3f, 0x3a, 0xe6, 0x0b, 0xb4,
+	0x34, 0x48, 0x69, 0xfc, 0xb6, 0xea, 0xb2, 0xde,
+	0x9f, 0xfd, 0x92, 0x36, 0x18, 0x98, 0x99, 0xaa,
+	0x65, 0xe2, 0xea, 0xf4, 0xb1, 0x47, 0x8e, 0xb0,
+	0xe7, 0xd4, 0x7a, 0x2c
+};
+static const u8 key61[] __initconst = {
+	0xd7, 0xfd, 0x9b, 0xbd, 0x8f, 0x65, 0x0d, 0x00,
+	0xca, 0xa1, 0x6c, 0x85, 0x85, 0xa4, 0x6d, 0xf1,
+	0xb1, 0x68, 0x0c, 0x8b, 0x5d, 0x37, 0x72, 0xd0,
+	0xd8, 0xd2, 0x25, 0xab, 0x9f, 0x7b, 0x7d, 0x95
+};
+enum { nonce61 = 0xd82caf72a9c4864fULL };
+
+static const u8 input62[] __initconst = {
+	0x10, 0x77, 0xf3, 0x2f, 0xc2, 0x50, 0xd6, 0x0c,
+	0xba, 0xa8, 0x8d, 0xce, 0x0d, 0x58, 0x9e, 0x87,
+	0xb1, 0x59, 0x66, 0x0a, 0x4a, 0xb3, 0xd8, 0xca,
+	0x0a, 0x6b, 0xf8, 0xc6, 0x2b, 0x3f, 0x8e, 0x09,
+	0xe0, 0x0a, 0x15, 0x85, 0xfe, 0xaa, 0xc6, 0xbd,
+	0x30, 0xef, 0xe4, 0x10, 0x78, 0x03, 0xc1, 0xc7,
+	0x8a, 0xd9, 0xde, 0x0b, 0x51, 0x07, 0xc4, 0x7b,
+	0xe2, 0x2e, 0x36, 0x3a, 0xc2
+};
+static const u8 output62[] __initconst = {
+	0xa0, 0x0c, 0xfc, 0xc1, 0xf6, 0xaf, 0xc2, 0xb8,
+	0x5c, 0xef, 0x6e, 0xf3, 0xce, 0x15, 0x48, 0x05,
+	0xb5, 0x78, 0x49, 0x51, 0x1f, 0x9d, 0xf4, 0xbf,
+	0x2f, 0x53, 0xa2, 0xd1, 0x15, 0x20, 0x82, 0x6b,
+	0xd2, 0x22, 0x6c, 0x4e, 0x14, 0x87, 0xe3, 0xd7,
+	0x49, 0x45, 0x84, 0xdb, 0x5f, 0x68, 0x60, 0xc4,
+	0xb3, 0xe6, 0x3f, 0xd1, 0xfc, 0xa5, 0x73, 0xf3,
+	0xfc, 0xbb, 0xbe, 0xc8, 0x9d
+};
+static const u8 key62[] __initconst = {
+	0x6e, 0xc9, 0xaf, 0xce, 0x35, 0xb9, 0x86, 0xd1,
+	0xce, 0x5f, 0xd9, 0xbb, 0xd5, 0x1f, 0x7c, 0xcd,
+	0xfe, 0x19, 0xaa, 0x3d, 0xea, 0x64, 0xc1, 0x28,
+	0x40, 0xba, 0xa1, 0x28, 0xcd, 0x40, 0xb6, 0xf2
+};
+enum { nonce62 = 0xa1c0c265f900cde8ULL };
+
+static const u8 input63[] __initconst = {
+	0x7a, 0x70, 0x21, 0x2c, 0xef, 0xa6, 0x36, 0xd4,
+	0xe0, 0xab, 0x8c, 0x25, 0x73, 0x34, 0xc8, 0x94,
+	0x6c, 0x81, 0xcb, 0x19, 0x8d, 0x5a, 0x49, 0xaa,
+	0x6f, 0xba, 0x83, 0x72, 0x02, 0x5e, 0xf5, 0x89,
+	0xce, 0x79, 0x7e, 0x13, 0x3d, 0x5b, 0x98, 0x60,
+	0x5d, 0xd9, 0xfb, 0x15, 0x93, 0x4c, 0xf3, 0x51,
+	0x49, 0x55, 0xd1, 0x58, 0xdd, 0x7e, 0x6d, 0xfe,
+	0xdd, 0x84, 0x23, 0x05, 0xba, 0xe9
+};
+static const u8 output63[] __initconst = {
+	0x20, 0xb3, 0x5c, 0x03, 0x03, 0x78, 0x17, 0xfc,
+	0x3b, 0x35, 0x30, 0x9a, 0x00, 0x18, 0xf5, 0xc5,
+	0x06, 0x53, 0xf5, 0x04, 0x24, 0x9d, 0xd1, 0xb2,
+	0xac, 0x5a, 0xb6, 0x2a, 0xa5, 0xda, 0x50, 0x00,
+	0xec, 0xff, 0xa0, 0x7a, 0x14, 0x7b, 0xe4, 0x6b,
+	0x63, 0xe8, 0x66, 0x86, 0x34, 0xfd, 0x74, 0x44,
+	0xa2, 0x50, 0x97, 0x0d, 0xdc, 0xc3, 0x84, 0xf8,
+	0x71, 0x02, 0x31, 0x95, 0xed, 0x54
+};
+static const u8 key63[] __initconst = {
+	0x7d, 0x64, 0xb4, 0x12, 0x81, 0xe4, 0xe6, 0x8f,
+	0xcc, 0xe7, 0xd1, 0x1f, 0x70, 0x20, 0xfd, 0xb8,
+	0x3a, 0x7d, 0xa6, 0x53, 0x65, 0x30, 0x5d, 0xe3,
+	0x1a, 0x44, 0xbe, 0x62, 0xed, 0x90, 0xc4, 0xd1
+};
+enum { nonce63 = 0xe8e849596c942276ULL };
+
+static const u8 input64[] __initconst = {
+	0x84, 0xf8, 0xda, 0x87, 0x23, 0x39, 0x60, 0xcf,
+	0xc5, 0x50, 0x7e, 0xc5, 0x47, 0x29, 0x7c, 0x05,
+	0xc2, 0xb4, 0xf4, 0xb2, 0xec, 0x5d, 0x48, 0x36,
+	0xbf, 0xfc, 0x06, 0x8c, 0xf2, 0x0e, 0x88, 0xe7,
+	0xc9, 0xc5, 0xa4, 0xa2, 0x83, 0x20, 0xa1, 0x6f,
+	0x37, 0xe5, 0x2d, 0xa1, 0x72, 0xa1, 0x19, 0xef,
+	0x05, 0x42, 0x08, 0xf2, 0x57, 0x47, 0x31, 0x1e,
+	0x17, 0x76, 0x13, 0xd3, 0xcc, 0x75, 0x2c
+};
+static const u8 output64[] __initconst = {
+	0xcb, 0xec, 0x90, 0x88, 0xeb, 0x31, 0x69, 0x20,
+	0xa6, 0xdc, 0xff, 0x76, 0x98, 0xb0, 0x24, 0x49,
+	0x7b, 0x20, 0xd9, 0xd1, 0x1b, 0xe3, 0x61, 0xdc,
+	0xcf, 0x51, 0xf6, 0x70, 0x72, 0x33, 0x28, 0x94,
+	0xac, 0x73, 0x18, 0xcf, 0x93, 0xfd, 0xca, 0x08,
+	0x0d, 0xa2, 0xb9, 0x57, 0x1e, 0x51, 0xb6, 0x07,
+	0x5c, 0xc1, 0x13, 0x64, 0x1d, 0x18, 0x6f, 0xe6,
+	0x0b, 0xb7, 0x14, 0x03, 0x43, 0xb6, 0xaf
+};
+static const u8 key64[] __initconst = {
+	0xbf, 0x82, 0x65, 0xe4, 0x50, 0xf9, 0x5e, 0xea,
+	0x28, 0x91, 0xd1, 0xd2, 0x17, 0x7c, 0x13, 0x7e,
+	0xf5, 0xd5, 0x6b, 0x06, 0x1c, 0x20, 0xc2, 0x82,
+	0xa1, 0x7a, 0xa2, 0x14, 0xa1, 0xb0, 0x54, 0x58
+};
+enum { nonce64 = 0xe57c5095aa5723c9ULL };
+
+static const u8 input65[] __initconst = {
+	0x1c, 0xfb, 0xd3, 0x3f, 0x85, 0xd7, 0xba, 0x7b,
+	0xae, 0xb1, 0xa5, 0xd2, 0xe5, 0x40, 0xce, 0x4d,
+	0x3e, 0xab, 0x17, 0x9d, 0x7d, 0x9f, 0x03, 0x98,
+	0x3f, 0x9f, 0xc8, 0xdd, 0x36, 0x17, 0x43, 0x5c,
+	0x34, 0xd1, 0x23, 0xe0, 0x77, 0xbf, 0x35, 0x5d,
+	0x8f, 0xb1, 0xcb, 0x82, 0xbb, 0x39, 0x69, 0xd8,
+	0x90, 0x45, 0x37, 0xfd, 0x98, 0x25, 0xf7, 0x5b,
+	0xce, 0x06, 0x43, 0xba, 0x61, 0xa8, 0x47, 0xb9
+};
+static const u8 output65[] __initconst = {
+	0x73, 0xa5, 0x68, 0xab, 0x8b, 0xa5, 0xc3, 0x7e,
+	0x74, 0xf8, 0x9d, 0xf5, 0x93, 0x6e, 0xf2, 0x71,
+	0x6d, 0xde, 0x82, 0xc5, 0x40, 0xa0, 0x46, 0xb3,
+	0x9a, 0x78, 0xa8, 0xf7, 0xdf, 0xb1, 0xc3, 0xdd,
+	0x8d, 0x90, 0x00, 0x68, 0x21, 0x48, 0xe8, 0xba,
+	0x56, 0x9f, 0x8f, 0xe7, 0xa4, 0x4d, 0x36, 0x55,
+	0xd0, 0x34, 0x99, 0xa6, 0x1c, 0x4c, 0xc1, 0xe2,
+	0x65, 0x98, 0x14, 0x8e, 0x6a, 0x05, 0xb1, 0x2b
+};
+static const u8 key65[] __initconst = {
+	0xbd, 0x5c, 0x8a, 0xb0, 0x11, 0x29, 0xf3, 0x00,
+	0x7a, 0x78, 0x32, 0x63, 0x34, 0x00, 0xe6, 0x7d,
+	0x30, 0x54, 0xde, 0x37, 0xda, 0xc2, 0xc4, 0x3d,
+	0x92, 0x6b, 0x4c, 0xc2, 0x92, 0xe9, 0x9e, 0x2a
+};
+enum { nonce65 = 0xf654a3031de746f2ULL };
+
+static const u8 input66[] __initconst = {
+	0x4b, 0x27, 0x30, 0x8f, 0x28, 0xd8, 0x60, 0x46,
+	0x39, 0x06, 0x49, 0xea, 0x1b, 0x71, 0x26, 0xe0,
+	0x99, 0x2b, 0xd4, 0x8f, 0x64, 0x64, 0xcd, 0xac,
+	0x1d, 0x78, 0x88, 0x90, 0xe1, 0x5c, 0x24, 0x4b,
+	0xdc, 0x2d, 0xb7, 0xee, 0x3a, 0xe6, 0x86, 0x2c,
+	0x21, 0xe4, 0x2b, 0xfc, 0xe8, 0x19, 0xca, 0x65,
+	0xe7, 0xdd, 0x6f, 0x52, 0xb3, 0x11, 0xe1, 0xe2,
+	0xbf, 0xe8, 0x70, 0xe3, 0x0d, 0x45, 0xb8, 0xa5,
+	0x20, 0xb7, 0xb5, 0xaf, 0xff, 0x08, 0xcf, 0x23,
+	0x65, 0xdf, 0x8d, 0xc3, 0x31, 0xf3, 0x1e, 0x6a,
+	0x58, 0x8d, 0xcc, 0x45, 0x16, 0x86, 0x1f, 0x31,
+	0x5c, 0x27, 0xcd, 0xc8, 0x6b, 0x19, 0x1e, 0xec,
+	0x44, 0x75, 0x63, 0x97, 0xfd, 0x79, 0xf6, 0x62,
+	0xc5, 0xba, 0x17, 0xc7, 0xab, 0x8f, 0xbb, 0xed,
+	0x85, 0x2a, 0x98, 0x79, 0x21, 0xec, 0x6e, 0x4d,
+	0xdc, 0xfa, 0x72, 0x52, 0xba, 0xc8, 0x4c
+};
+static const u8 output66[] __initconst = {
+	0x76, 0x5b, 0x2c, 0xa7, 0x62, 0xb9, 0x08, 0x4a,
+	0xc6, 0x4a, 0x92, 0xc3, 0xbb, 0x10, 0xb3, 0xee,
+	0xff, 0xb9, 0x07, 0xc7, 0x27, 0xcb, 0x1e, 0xcf,
+	0x58, 0x6f, 0xa1, 0x64, 0xe8, 0xf1, 0x4e, 0xe1,
+	0xef, 0x18, 0x96, 0xab, 0x97, 0x28, 0xd1, 0x7c,
+	0x71, 0x6c, 0xd1, 0xe2, 0xfa, 0xd9, 0x75, 0xcb,
+	0xeb, 0xea, 0x0c, 0x86, 0x82, 0xd8, 0xf4, 0xcc,
+	0xea, 0xa3, 0x00, 0xfa, 0x82, 0xd2, 0xcd, 0xcb,
+	0xdb, 0x63, 0x28, 0xe2, 0x82, 0xe9, 0x01, 0xed,
+	0x31, 0xe6, 0x71, 0x45, 0x08, 0x89, 0x8a, 0x23,
+	0xa8, 0xb5, 0xc2, 0xe2, 0x9f, 0xe9, 0xb8, 0x9a,
+	0xc4, 0x79, 0x6d, 0x71, 0x52, 0x61, 0x74, 0x6c,
+	0x1b, 0xd7, 0x65, 0x6d, 0x03, 0xc4, 0x1a, 0xc0,
+	0x50, 0xba, 0xd6, 0xc9, 0x43, 0x50, 0xbe, 0x09,
+	0x09, 0x8a, 0xdb, 0xaa, 0x76, 0x4e, 0x3b, 0x61,
+	0x3c, 0x7c, 0x44, 0xe7, 0xdb, 0x10, 0xa7
+};
+static const u8 key66[] __initconst = {
+	0x88, 0xdf, 0xca, 0x68, 0xaf, 0x4f, 0xb3, 0xfd,
+	0x6e, 0xa7, 0x95, 0x35, 0x8a, 0xe8, 0x37, 0xe8,
+	0xc8, 0x55, 0xa2, 0x2a, 0x6d, 0x77, 0xf8, 0x93,
+	0x7a, 0x41, 0xf3, 0x7b, 0x95, 0xdf, 0x89, 0xf5
+};
+enum { nonce66 = 0x1024b4fdd415cf82ULL };
+
+static const u8 input67[] __initconst = {
+	0xd4, 0x2e, 0xfa, 0x92, 0xe9, 0x29, 0x68, 0xb7,
+	0x54, 0x2c, 0xf7, 0xa4, 0x2d, 0xb7, 0x50, 0xb5,
+	0xc5, 0xb2, 0x9d, 0x17, 0x5e, 0x0a, 0xca, 0x37,
+	0xbf, 0x60, 0xae, 0xd2, 0x98, 0xe9, 0xfa, 0x59,
+	0x67, 0x62, 0xe6, 0x43, 0x0c, 0x77, 0x80, 0x82,
+	0x33, 0x61, 0xa3, 0xff, 0xc1, 0xa0, 0x8f, 0x56,
+	0xbc, 0xec, 0x65, 0x43, 0x88, 0xa5, 0xff, 0x51,
+	0x64, 0x30, 0xee, 0x34, 0xb7, 0x5c, 0x28, 0x68,
+	0xc3, 0x52, 0xd2, 0xac, 0x78, 0x2a, 0xa6, 0x10,
+	0xb8, 0xb2, 0x4c, 0x80, 0x4f, 0x99, 0xb2, 0x36,
+	0x94, 0x8f, 0x66, 0xcb, 0xa1, 0x91, 0xed, 0x06,
+	0x42, 0x6d, 0xc1, 0xae, 0x55, 0x93, 0xdd, 0x93,
+	0x9e, 0x88, 0x34, 0x7f, 0x98, 0xeb, 0xbe, 0x61,
+	0xf9, 0xa9, 0x0f, 0xd9, 0xc4, 0x87, 0xd5, 0xef,
+	0xcc, 0x71, 0x8c, 0x0e, 0xce, 0xad, 0x02, 0xcf,
+	0xa2, 0x61, 0xdf, 0xb1, 0xfe, 0x3b, 0xdc, 0xc0,
+	0x58, 0xb5, 0x71, 0xa1, 0x83, 0xc9, 0xb4, 0xaf,
+	0x9d, 0x54, 0x12, 0xcd, 0xea, 0x06, 0xd6, 0x4e,
+	0xe5, 0x27, 0x0c, 0xc3, 0xbb, 0xa8, 0x0a, 0x81,
+	0x75, 0xc3, 0xc9, 0xd4, 0x35, 0x3e, 0x53, 0x9f,
+	0xaa, 0x20, 0xc0, 0x68, 0x39, 0x2c, 0x96, 0x39,
+	0x53, 0x81, 0xda, 0x07, 0x0f, 0x44, 0xa5, 0x47,
+	0x0e, 0xb3, 0x87, 0x0d, 0x1b, 0xc1, 0xe5, 0x41,
+	0x35, 0x12, 0x58, 0x96, 0x69, 0x8a, 0x1a, 0xa3,
+	0x9d, 0x3d, 0xd4, 0xb1, 0x8e, 0x1f, 0x96, 0x87,
+	0xda, 0xd3, 0x19, 0xe2, 0xb1, 0x3a, 0x19, 0x74,
+	0xa0, 0x00, 0x9f, 0x4d, 0xbc, 0xcb, 0x0c, 0xe9,
+	0xec, 0x10, 0xdf, 0x2a, 0x88, 0xdc, 0x30, 0x51,
+	0x46, 0x56, 0x53, 0x98, 0x6a, 0x26, 0x14, 0x05,
+	0x54, 0x81, 0x55, 0x0b, 0x3c, 0x85, 0xdd, 0x33,
+	0x81, 0x11, 0x29, 0x82, 0x46, 0x35, 0xe1, 0xdb,
+	0x59, 0x7b
+};
+static const u8 output67[] __initconst = {
+	0x64, 0x6c, 0xda, 0x7f, 0xd4, 0xa9, 0x2a, 0x5e,
+	0x22, 0xae, 0x8d, 0x67, 0xdb, 0xee, 0xfd, 0xd0,
+	0x44, 0x80, 0x17, 0xb2, 0xe3, 0x87, 0xad, 0x57,
+	0x15, 0xcb, 0x88, 0x64, 0xc0, 0xf1, 0x49, 0x3d,
+	0xfa, 0xbe, 0xa8, 0x9f, 0x12, 0xc3, 0x57, 0x56,
+	0x70, 0xa5, 0xc5, 0x6b, 0xf1, 0xab, 0xd5, 0xde,
+	0x77, 0x92, 0x6a, 0x56, 0x03, 0xf5, 0x21, 0x0d,
+	0xb6, 0xc4, 0xcc, 0x62, 0x44, 0x3f, 0xb1, 0xc1,
+	0x61, 0x41, 0x90, 0xb2, 0xd5, 0xb8, 0xf3, 0x57,
+	0xfb, 0xc2, 0x6b, 0x25, 0x58, 0xc8, 0x45, 0x20,
+	0x72, 0x29, 0x6f, 0x9d, 0xb5, 0x81, 0x4d, 0x2b,
+	0xb2, 0x89, 0x9e, 0x91, 0x53, 0x97, 0x1c, 0xd9,
+	0x3d, 0x79, 0xdc, 0x14, 0xae, 0x01, 0x73, 0x75,
+	0xf0, 0xca, 0xd5, 0xab, 0x62, 0x5c, 0x7a, 0x7d,
+	0x3f, 0xfe, 0x22, 0x7d, 0xee, 0xe2, 0xcb, 0x76,
+	0x55, 0xec, 0x06, 0xdd, 0x41, 0x47, 0x18, 0x62,
+	0x1d, 0x57, 0xd0, 0xd6, 0xb6, 0x0f, 0x4b, 0xfc,
+	0x79, 0x19, 0xf4, 0xd6, 0x37, 0x86, 0x18, 0x1f,
+	0x98, 0x0d, 0x9e, 0x15, 0x2d, 0xb6, 0x9a, 0x8a,
+	0x8c, 0x80, 0x22, 0x2f, 0x82, 0xc4, 0xc7, 0x36,
+	0xfa, 0xfa, 0x07, 0xbd, 0xc2, 0x2a, 0xe2, 0xea,
+	0x93, 0xc8, 0xb2, 0x90, 0x33, 0xf2, 0xee, 0x4b,
+	0x1b, 0xf4, 0x37, 0x92, 0x13, 0xbb, 0xe2, 0xce,
+	0xe3, 0x03, 0xcf, 0x07, 0x94, 0xab, 0x9a, 0xc9,
+	0xff, 0x83, 0x69, 0x3a, 0xda, 0x2c, 0xd0, 0x47,
+	0x3d, 0x6c, 0x1a, 0x60, 0x68, 0x47, 0xb9, 0x36,
+	0x52, 0xdd, 0x16, 0xef, 0x6c, 0xbf, 0x54, 0x11,
+	0x72, 0x62, 0xce, 0x8c, 0x9d, 0x90, 0xa0, 0x25,
+	0x06, 0x92, 0x3e, 0x12, 0x7e, 0x1a, 0x1d, 0xe5,
+	0xa2, 0x71, 0xce, 0x1c, 0x4c, 0x6a, 0x7c, 0xdc,
+	0x3d, 0xe3, 0x6e, 0x48, 0x9d, 0xb3, 0x64, 0x7d,
+	0x78, 0x40
+};
+static const u8 key67[] __initconst = {
+	0xa9, 0x20, 0x75, 0x89, 0x7e, 0x37, 0x85, 0x48,
+	0xa3, 0xfb, 0x7b, 0xe8, 0x30, 0xa7, 0xe3, 0x6e,
+	0xa6, 0xc1, 0x71, 0x17, 0xc1, 0x6c, 0x9b, 0xc2,
+	0xde, 0xf0, 0xa7, 0x19, 0xec, 0xce, 0xc6, 0x53
+};
+enum { nonce67 = 0x4adc4d1f968c8a10ULL };
+
+static const u8 input68[] __initconst = {
+	0x99, 0xae, 0x72, 0xfb, 0x16, 0xe1, 0xf1, 0x59,
+	0x43, 0x15, 0x4e, 0x33, 0xa0, 0x95, 0xe7, 0x6c,
+	0x74, 0x24, 0x31, 0xca, 0x3b, 0x2e, 0xeb, 0xd7,
+	0x11, 0xd8, 0xe0, 0x56, 0x92, 0x91, 0x61, 0x57,
+	0xe2, 0x82, 0x9f, 0x8f, 0x37, 0xf5, 0x3d, 0x24,
+	0x92, 0x9d, 0x87, 0x00, 0x8d, 0x89, 0xe0, 0x25,
+	0x8b, 0xe4, 0x20, 0x5b, 0x8a, 0x26, 0x2c, 0x61,
+	0x78, 0xb0, 0xa6, 0x3e, 0x82, 0x18, 0xcf, 0xdc,
+	0x2d, 0x24, 0xdd, 0x81, 0x42, 0xc4, 0x95, 0xf0,
+	0x48, 0x60, 0x71, 0xe3, 0xe3, 0xac, 0xec, 0xbe,
+	0x98, 0x6b, 0x0c, 0xb5, 0x6a, 0xa9, 0xc8, 0x79,
+	0x23, 0x2e, 0x38, 0x0b, 0x72, 0x88, 0x8c, 0xe7,
+	0x71, 0x8b, 0x36, 0xe3, 0x58, 0x3d, 0x9c, 0xa0,
+	0xa2, 0xea, 0xcf, 0x0c, 0x6a, 0x6c, 0x64, 0xdf,
+	0x97, 0x21, 0x8f, 0x93, 0xfb, 0xba, 0xf3, 0x5a,
+	0xd7, 0x8f, 0xa6, 0x37, 0x15, 0x50, 0x43, 0x02,
+	0x46, 0x7f, 0x93, 0x46, 0x86, 0x31, 0xe2, 0xaa,
+	0x24, 0xa8, 0x26, 0xae, 0xe6, 0xc0, 0x05, 0x73,
+	0x0b, 0x4f, 0x7e, 0xed, 0x65, 0xeb, 0x56, 0x1e,
+	0xb6, 0xb3, 0x0b, 0xc3, 0x0e, 0x31, 0x95, 0xa9,
+	0x18, 0x4d, 0xaf, 0x38, 0xd7, 0xec, 0xc6, 0x44,
+	0x72, 0x77, 0x4e, 0x25, 0x4b, 0x25, 0xdd, 0x1e,
+	0x8c, 0xa2, 0xdf, 0xf6, 0x2a, 0x97, 0x1a, 0x88,
+	0x2c, 0x8a, 0x5d, 0xfe, 0xe8, 0xfb, 0x35, 0xe8,
+	0x0f, 0x2b, 0x7a, 0x18, 0x69, 0x43, 0x31, 0x1d,
+	0x38, 0x6a, 0x62, 0x95, 0x0f, 0x20, 0x4b, 0xbb,
+	0x97, 0x3c, 0xe0, 0x64, 0x2f, 0x52, 0xc9, 0x2d,
+	0x4d, 0x9d, 0x54, 0x04, 0x3d, 0xc9, 0xea, 0xeb,
+	0xd0, 0x86, 0x52, 0xff, 0x42, 0xe1, 0x0d, 0x7a,
+	0xad, 0x88, 0xf9, 0x9b, 0x1e, 0x5e, 0x12, 0x27,
+	0x95, 0x3e, 0x0c, 0x2c, 0x13, 0x00, 0x6f, 0x8e,
+	0x93, 0x69, 0x0e, 0x01, 0x8c, 0xc1, 0xfd, 0xb3
+};
+static const u8 output68[] __initconst = {
+	0x26, 0x3e, 0xf2, 0xb1, 0xf5, 0xef, 0x81, 0xa4,
+	0xb7, 0x42, 0xd4, 0x26, 0x18, 0x4b, 0xdd, 0x6a,
+	0x47, 0x15, 0xcb, 0x0e, 0x57, 0xdb, 0xa7, 0x29,
+	0x7e, 0x7b, 0x3f, 0x47, 0x89, 0x57, 0xab, 0xea,
+	0x14, 0x7b, 0xcf, 0x37, 0xdb, 0x1c, 0xe1, 0x11,
+	0x77, 0xae, 0x2e, 0x4c, 0xd2, 0x08, 0x3f, 0xa6,
+	0x62, 0x86, 0xa6, 0xb2, 0x07, 0xd5, 0x3f, 0x9b,
+	0xdc, 0xc8, 0x50, 0x4b, 0x7b, 0xb9, 0x06, 0xe6,
+	0xeb, 0xac, 0x98, 0x8c, 0x36, 0x0c, 0x1e, 0xb2,
+	0xc8, 0xfb, 0x24, 0x60, 0x2c, 0x08, 0x17, 0x26,
+	0x5b, 0xc8, 0xc2, 0xdf, 0x9c, 0x73, 0x67, 0x4a,
+	0xdb, 0xcf, 0xd5, 0x2c, 0x2b, 0xca, 0x24, 0xcc,
+	0xdb, 0xc9, 0xa8, 0xf2, 0x5d, 0x67, 0xdf, 0x5c,
+	0x62, 0x0b, 0x58, 0xc0, 0x83, 0xde, 0x8b, 0xf6,
+	0x15, 0x0a, 0xd6, 0x32, 0xd8, 0xf5, 0xf2, 0x5f,
+	0x33, 0xce, 0x7e, 0xab, 0x76, 0xcd, 0x14, 0x91,
+	0xd8, 0x41, 0x90, 0x93, 0xa1, 0xaf, 0xf3, 0x45,
+	0x6c, 0x1b, 0x25, 0xbd, 0x48, 0x51, 0x6d, 0x15,
+	0x47, 0xe6, 0x23, 0x50, 0x32, 0x69, 0x1e, 0xb5,
+	0x94, 0xd3, 0x97, 0xba, 0xd7, 0x37, 0x4a, 0xba,
+	0xb9, 0xcd, 0xfb, 0x96, 0x9a, 0x90, 0xe0, 0x37,
+	0xf8, 0xdf, 0x91, 0x6c, 0x62, 0x13, 0x19, 0x21,
+	0x4b, 0xa9, 0xf1, 0x12, 0x66, 0xe2, 0x74, 0xd7,
+	0x81, 0xa0, 0x74, 0x8d, 0x7e, 0x7e, 0xc9, 0xb1,
+	0x69, 0x8f, 0xed, 0xb3, 0xf6, 0x97, 0xcd, 0x72,
+	0x78, 0x93, 0xd3, 0x54, 0x6b, 0x43, 0xac, 0x29,
+	0xb4, 0xbc, 0x7d, 0xa4, 0x26, 0x4b, 0x7b, 0xab,
+	0xd6, 0x67, 0x22, 0xff, 0x03, 0x92, 0xb6, 0xd4,
+	0x96, 0x94, 0x5a, 0xe5, 0x02, 0x35, 0x77, 0xfa,
+	0x3f, 0x54, 0x1d, 0xdd, 0x35, 0x39, 0xfe, 0x03,
+	0xdd, 0x8e, 0x3c, 0x8c, 0xc2, 0x69, 0x2a, 0xb1,
+	0xb7, 0xb3, 0xa1, 0x89, 0x84, 0xea, 0x16, 0xe2
+};
+static const u8 key68[] __initconst = {
+	0xd2, 0x49, 0x7f, 0xd7, 0x49, 0x66, 0x0d, 0xb3,
+	0x5a, 0x7e, 0x3c, 0xfc, 0x37, 0x83, 0x0e, 0xf7,
+	0x96, 0xd8, 0xd6, 0x33, 0x79, 0x2b, 0x84, 0x53,
+	0x06, 0xbc, 0x6c, 0x0a, 0x55, 0x84, 0xfe, 0xab
+};
+enum { nonce68 = 0x6a6df7ff0a20de06ULL };
+
+static const u8 input69[] __initconst = {
+	0xf9, 0x18, 0x4c, 0xd2, 0x3f, 0xf7, 0x22, 0xd9,
+	0x58, 0xb6, 0x3b, 0x38, 0x69, 0x79, 0xf4, 0x71,
+	0x5f, 0x38, 0x52, 0x1f, 0x17, 0x6f, 0x6f, 0xd9,
+	0x09, 0x2b, 0xfb, 0x67, 0xdc, 0xc9, 0xe8, 0x4a,
+	0x70, 0x9f, 0x2e, 0x3c, 0x06, 0xe5, 0x12, 0x20,
+	0x25, 0x29, 0xd0, 0xdc, 0x81, 0xc5, 0xc6, 0x0f,
+	0xd2, 0xa8, 0x81, 0x15, 0x98, 0xb2, 0x71, 0x5a,
+	0x9a, 0xe9, 0xfb, 0xaf, 0x0e, 0x5f, 0x8a, 0xf3,
+	0x16, 0x4a, 0x47, 0xf2, 0x5c, 0xbf, 0xda, 0x52,
+	0x9a, 0xa6, 0x36, 0xfd, 0xc6, 0xf7, 0x66, 0x00,
+	0xcc, 0x6c, 0xd4, 0xb3, 0x07, 0x6d, 0xeb, 0xfe,
+	0x92, 0x71, 0x25, 0xd0, 0xcf, 0x9c, 0xe8, 0x65,
+	0x45, 0x10, 0xcf, 0x62, 0x74, 0x7d, 0xf2, 0x1b,
+	0x57, 0xa0, 0xf1, 0x6b, 0xa4, 0xd5, 0xfa, 0x12,
+	0x27, 0x5a, 0xf7, 0x99, 0xfc, 0xca, 0xf3, 0xb8,
+	0x2c, 0x8b, 0xba, 0x28, 0x74, 0xde, 0x8f, 0x78,
+	0xa2, 0x8c, 0xaf, 0x89, 0x4b, 0x05, 0xe2, 0xf3,
+	0xf8, 0xd2, 0xef, 0xac, 0xa4, 0xc4, 0xe2, 0xe2,
+	0x36, 0xbb, 0x5e, 0xae, 0xe6, 0x87, 0x3d, 0x88,
+	0x9f, 0xb8, 0x11, 0xbb, 0xcf, 0x57, 0xce, 0xd0,
+	0xba, 0x62, 0xf4, 0xf8, 0x9b, 0x95, 0x04, 0xc9,
+	0xcf, 0x01, 0xe9, 0xf1, 0xc8, 0xc6, 0x22, 0xa4,
+	0xf2, 0x8b, 0x2f, 0x24, 0x0a, 0xf5, 0x6e, 0xb7,
+	0xd4, 0x2c, 0xb6, 0xf7, 0x5c, 0x97, 0x61, 0x0b,
+	0xd9, 0xb5, 0x06, 0xcd, 0xed, 0x3e, 0x1f, 0xc5,
+	0xb2, 0x6c, 0xa3, 0xea, 0xb8, 0xad, 0xa6, 0x42,
+	0x88, 0x7a, 0x52, 0xd5, 0x64, 0xba, 0xb5, 0x20,
+	0x10, 0xa0, 0x0f, 0x0d, 0xea, 0xef, 0x5a, 0x9b,
+	0x27, 0xb8, 0xca, 0x20, 0x19, 0x6d, 0xa8, 0xc4,
+	0x46, 0x04, 0xb3, 0xe8, 0xf8, 0x66, 0x1b, 0x0a,
+	0xce, 0x76, 0x5d, 0x59, 0x58, 0x05, 0xee, 0x3e,
+	0x3c, 0x86, 0x5b, 0x49, 0x1c, 0x72, 0x18, 0x01,
+	0x62, 0x92, 0x0f, 0x3e, 0xd1, 0x57, 0x5e, 0x20,
+	0x7b, 0xfb, 0x4d, 0x3c, 0xc5, 0x35, 0x43, 0x2f,
+	0xb0, 0xc5, 0x7c, 0xe4, 0xa2, 0x84, 0x13, 0x77
+};
+static const u8 output69[] __initconst = {
+	0xbb, 0x4a, 0x7f, 0x7c, 0xd5, 0x2f, 0x89, 0x06,
+	0xec, 0x20, 0xf1, 0x9a, 0x11, 0x09, 0x14, 0x2e,
+	0x17, 0x50, 0xf9, 0xd5, 0xf5, 0x48, 0x7c, 0x7a,
+	0x55, 0xc0, 0x57, 0x03, 0xe3, 0xc4, 0xb2, 0xb7,
+	0x18, 0x47, 0x95, 0xde, 0xaf, 0x80, 0x06, 0x3c,
+	0x5a, 0xf2, 0xc3, 0x53, 0xe3, 0x29, 0x92, 0xf8,
+	0xff, 0x64, 0x85, 0xb9, 0xf7, 0xd3, 0x80, 0xd2,
+	0x0c, 0x5d, 0x7b, 0x57, 0x0c, 0x51, 0x79, 0x86,
+	0xf3, 0x20, 0xd2, 0xb8, 0x6e, 0x0c, 0x5a, 0xce,
+	0xeb, 0x88, 0x02, 0x8b, 0x82, 0x1b, 0x7f, 0xf5,
+	0xde, 0x7f, 0x48, 0x48, 0xdf, 0xa0, 0x55, 0xc6,
+	0x0c, 0x22, 0xa1, 0x80, 0x8d, 0x3b, 0xcb, 0x40,
+	0x2d, 0x3d, 0x0b, 0xf2, 0xe0, 0x22, 0x13, 0x99,
+	0xe1, 0xa7, 0x27, 0x68, 0x31, 0xe1, 0x24, 0x5d,
+	0xd2, 0xee, 0x16, 0xc1, 0xd7, 0xa8, 0x14, 0x19,
+	0x23, 0x72, 0x67, 0x27, 0xdc, 0x5e, 0xb9, 0xc7,
+	0xd8, 0xe3, 0x55, 0x50, 0x40, 0x98, 0x7b, 0xe7,
+	0x34, 0x1c, 0x3b, 0x18, 0x14, 0xd8, 0x62, 0xc1,
+	0x93, 0x84, 0xf3, 0x5b, 0xdd, 0x9e, 0x1f, 0x3b,
+	0x0b, 0xbc, 0x4e, 0x5b, 0x79, 0xa3, 0xca, 0x74,
+	0x2a, 0x98, 0xe8, 0x04, 0x39, 0xef, 0xc6, 0x76,
+	0x6d, 0xee, 0x9f, 0x67, 0x5b, 0x59, 0x3a, 0xe5,
+	0xf2, 0x3b, 0xca, 0x89, 0xe8, 0x9b, 0x03, 0x3d,
+	0x11, 0xd2, 0x4a, 0x70, 0xaf, 0x88, 0xb0, 0x94,
+	0x96, 0x26, 0xab, 0x3c, 0xc1, 0xb8, 0xe4, 0xe7,
+	0x14, 0x61, 0x64, 0x3a, 0x61, 0x08, 0x0f, 0xa9,
+	0xce, 0x64, 0xb2, 0x40, 0xf8, 0x20, 0x3a, 0xa9,
+	0x31, 0xbd, 0x7e, 0x16, 0xca, 0xf5, 0x62, 0x0f,
+	0x91, 0x9f, 0x8e, 0x1d, 0xa4, 0x77, 0xf3, 0x87,
+	0x61, 0xe8, 0x14, 0xde, 0x18, 0x68, 0x4e, 0x9d,
+	0x73, 0xcd, 0x8a, 0xe4, 0x80, 0x84, 0x23, 0xaa,
+	0x9d, 0x64, 0x1c, 0x80, 0x41, 0xca, 0x82, 0x40,
+	0x94, 0x55, 0xe3, 0x28, 0xa1, 0x97, 0x71, 0xba,
+	0xf2, 0x2c, 0x39, 0x62, 0x29, 0x56, 0xd0, 0xff,
+	0xb2, 0x82, 0x20, 0x59, 0x1f, 0xc3, 0x64, 0x57
+};
+static const u8 key69[] __initconst = {
+	0x19, 0x09, 0xe9, 0x7c, 0xd9, 0x02, 0x4a, 0x0c,
+	0x52, 0x25, 0xad, 0x5c, 0x2e, 0x8d, 0x86, 0x10,
+	0x85, 0x2b, 0xba, 0xa4, 0x44, 0x5b, 0x39, 0x3e,
+	0x18, 0xaa, 0xce, 0x0e, 0xe2, 0x69, 0x3c, 0xcf
+};
+enum { nonce69 = 0xdb925a1948f0f060ULL };
+
+static const u8 input70[] __initconst = {
+	0x10, 0xe7, 0x83, 0xcf, 0x42, 0x9f, 0xf2, 0x41,
+	0xc7, 0xe4, 0xdb, 0xf9, 0xa3, 0x02, 0x1d, 0x8d,
+	0x50, 0x81, 0x2c, 0x6b, 0x92, 0xe0, 0x4e, 0xea,
+	0x26, 0x83, 0x2a, 0xd0, 0x31, 0xf1, 0x23, 0xf3,
+	0x0e, 0x88, 0x14, 0x31, 0xf9, 0x01, 0x63, 0x59,
+	0x21, 0xd1, 0x8b, 0xdd, 0x06, 0xd0, 0xc6, 0xab,
+	0x91, 0x71, 0x82, 0x4d, 0xd4, 0x62, 0x37, 0x17,
+	0xf9, 0x50, 0xf9, 0xb5, 0x74, 0xce, 0x39, 0x80,
+	0x80, 0x78, 0xf8, 0xdc, 0x1c, 0xdb, 0x7c, 0x3d,
+	0xd4, 0x86, 0x31, 0x00, 0x75, 0x7b, 0xd1, 0x42,
+	0x9f, 0x1b, 0x97, 0x88, 0x0e, 0x14, 0x0e, 0x1e,
+	0x7d, 0x7b, 0xc4, 0xd2, 0xf3, 0xc1, 0x6d, 0x17,
+	0x5d, 0xc4, 0x75, 0x54, 0x0f, 0x38, 0x65, 0x89,
+	0xd8, 0x7d, 0xab, 0xc9, 0xa7, 0x0a, 0x21, 0x0b,
+	0x37, 0x12, 0x05, 0x07, 0xb5, 0x68, 0x32, 0x32,
+	0xb9, 0xf8, 0x97, 0x17, 0x03, 0xed, 0x51, 0x8f,
+	0x3d, 0x5a, 0xd0, 0x12, 0x01, 0x6e, 0x2e, 0x91,
+	0x1c, 0xbe, 0x6b, 0xa3, 0xcc, 0x75, 0x62, 0x06,
+	0x8e, 0x65, 0xbb, 0xe2, 0x29, 0x71, 0x4b, 0x89,
+	0x6a, 0x9d, 0x85, 0x8c, 0x8c, 0xdf, 0x94, 0x95,
+	0x23, 0x66, 0xf8, 0x92, 0xee, 0x56, 0xeb, 0xb3,
+	0xeb, 0xd2, 0x4a, 0x3b, 0x77, 0x8a, 0x6e, 0xf6,
+	0xca, 0xd2, 0x34, 0x00, 0xde, 0xbe, 0x1d, 0x7a,
+	0x73, 0xef, 0x2b, 0x80, 0x56, 0x16, 0x29, 0xbf,
+	0x6e, 0x33, 0xed, 0x0d, 0xe2, 0x02, 0x60, 0x74,
+	0xe9, 0x0a, 0xbc, 0xd1, 0xc5, 0xe8, 0x53, 0x02,
+	0x79, 0x0f, 0x25, 0x0c, 0xef, 0xab, 0xd3, 0xbc,
+	0xb7, 0xfc, 0xf3, 0xb0, 0x34, 0xd1, 0x07, 0xd2,
+	0x5a, 0x31, 0x1f, 0xec, 0x1f, 0x87, 0xed, 0xdd,
+	0x6a, 0xc1, 0xe8, 0xb3, 0x25, 0x4c, 0xc6, 0x9b,
+	0x91, 0x73, 0xec, 0x06, 0x73, 0x9e, 0x57, 0x65,
+	0x32, 0x75, 0x11, 0x74, 0x6e, 0xa4, 0x7d, 0x0d,
+	0x74, 0x9f, 0x51, 0x10, 0x10, 0x47, 0xc9, 0x71,
+	0x6e, 0x97, 0xae, 0x44, 0x41, 0xef, 0x98, 0x78,
+	0xf4, 0xc5, 0xbd, 0x5e, 0x00, 0xe5, 0xfd, 0xe2,
+	0xbe, 0x8c, 0xc2, 0xae, 0xc2, 0xee, 0x59, 0xf6,
+	0xcb, 0x20, 0x54, 0x84, 0xc3, 0x31, 0x7e, 0x67,
+	0x71, 0xb6, 0x76, 0xbe, 0x81, 0x8f, 0x82, 0xad,
+	0x01, 0x8f, 0xc4, 0x00, 0x04, 0x3d, 0x8d, 0x34,
+	0xaa, 0xea, 0xc0, 0xea, 0x91, 0x42, 0xb6, 0xb8,
+	0x43, 0xf3, 0x17, 0xb2, 0x73, 0x64, 0x82, 0x97,
+	0xd5, 0xc9, 0x07, 0x77, 0xb1, 0x26, 0xe2, 0x00,
+	0x6a, 0xae, 0x70, 0x0b, 0xbe, 0xe6, 0xb8, 0x42,
+	0x81, 0x55, 0xf7, 0xb8, 0x96, 0x41, 0x9d, 0xd4,
+	0x2c, 0x27, 0x00, 0xcc, 0x91, 0x28, 0x22, 0xa4,
+	0x7b, 0x42, 0x51, 0x9e, 0xd6, 0xec, 0xf3, 0x6b,
+	0x00, 0xff, 0x5c, 0xa2, 0xac, 0x47, 0x33, 0x2d,
+	0xf8, 0x11, 0x65, 0x5f, 0x4d, 0x79, 0x8b, 0x4f,
+	0xad, 0xf0, 0x9d, 0xcd, 0xb9, 0x7b, 0x08, 0xf7,
+	0x32, 0x51, 0xfa, 0x39, 0xaa, 0x78, 0x05, 0xb1,
+	0xf3, 0x5d, 0xe8, 0x7c, 0x8e, 0x4f, 0xa2, 0xe0,
+	0x98, 0x0c, 0xb2, 0xa7, 0xf0, 0x35, 0x8e, 0x70,
+	0x7c, 0x82, 0xf3, 0x1b, 0x26, 0x28, 0x12, 0xe5,
+	0x23, 0x57, 0xe4, 0xb4, 0x9b, 0x00, 0x39, 0x97,
+	0xef, 0x7c, 0x46, 0x9b, 0x34, 0x6b, 0xe7, 0x0e,
+	0xa3, 0x2a, 0x18, 0x11, 0x64, 0xc6, 0x7c, 0x8b,
+	0x06, 0x02, 0xf5, 0x69, 0x76, 0xf9, 0xaa, 0x09,
+	0x5f, 0x68, 0xf8, 0x4a, 0x79, 0x58, 0xec, 0x37,
+	0xcf, 0x3a, 0xcc, 0x97, 0x70, 0x1d, 0x3e, 0x52,
+	0x18, 0x0a, 0xad, 0x28, 0x5b, 0x3b, 0xe9, 0x03,
+	0x84, 0xe9, 0x68, 0x50, 0xce, 0xc4, 0xbc, 0x3e,
+	0x21, 0xad, 0x63, 0xfe, 0xc6, 0xfd, 0x6e, 0x69,
+	0x84, 0xa9, 0x30, 0xb1, 0x7a, 0xc4, 0x31, 0x10,
+	0xc1, 0x1f, 0x6e, 0xeb, 0xa5, 0xa6, 0x01
+};
+static const u8 output70[] __initconst = {
+	0x0f, 0x93, 0x2a, 0x20, 0xb3, 0x87, 0x2d, 0xce,
+	0xd1, 0x3b, 0x30, 0xfd, 0x06, 0x6d, 0x0a, 0xaa,
+	0x3e, 0xc4, 0x29, 0x02, 0x8a, 0xde, 0xa6, 0x4b,
+	0x45, 0x1b, 0x4f, 0x25, 0x59, 0xd5, 0x56, 0x6a,
+	0x3b, 0x37, 0xbd, 0x3e, 0x47, 0x12, 0x2c, 0x4e,
+	0x60, 0x5f, 0x05, 0x75, 0x61, 0x23, 0x05, 0x74,
+	0xcb, 0xfc, 0x5a, 0xb3, 0xac, 0x5c, 0x3d, 0xab,
+	0x52, 0x5f, 0x05, 0xbc, 0x57, 0xc0, 0x7e, 0xcf,
+	0x34, 0x5d, 0x7f, 0x41, 0xa3, 0x17, 0x78, 0xd5,
+	0x9f, 0xec, 0x0f, 0x1e, 0xf9, 0xfe, 0xa3, 0xbd,
+	0x28, 0xb0, 0xba, 0x4d, 0x84, 0xdb, 0xae, 0x8f,
+	0x1d, 0x98, 0xb7, 0xdc, 0xf9, 0xad, 0x55, 0x9c,
+	0x89, 0xfe, 0x9b, 0x9c, 0xa9, 0x89, 0xf6, 0x97,
+	0x9c, 0x3f, 0x09, 0x3e, 0xc6, 0x02, 0xc2, 0x55,
+	0x58, 0x09, 0x54, 0x66, 0xe4, 0x36, 0x81, 0x35,
+	0xca, 0x88, 0x17, 0x89, 0x80, 0x24, 0x2b, 0x21,
+	0x89, 0xee, 0x45, 0x5a, 0xe7, 0x1f, 0xd5, 0xa5,
+	0x16, 0xa4, 0xda, 0x70, 0x7e, 0xe9, 0x4f, 0x24,
+	0x61, 0x97, 0xab, 0xa0, 0xe0, 0xe7, 0xb8, 0x5c,
+	0x0f, 0x25, 0x17, 0x37, 0x75, 0x12, 0xb5, 0x40,
+	0xde, 0x1c, 0x0d, 0x8a, 0x77, 0x62, 0x3c, 0x86,
+	0xd9, 0x70, 0x2e, 0x96, 0x30, 0xd2, 0x55, 0xb3,
+	0x6b, 0xc3, 0xf2, 0x9c, 0x47, 0xf3, 0x3a, 0x24,
+	0x52, 0xc6, 0x38, 0xd8, 0x22, 0xb3, 0x0c, 0xfd,
+	0x2f, 0xa3, 0x3c, 0xb5, 0xe8, 0x26, 0xe1, 0xa3,
+	0xad, 0xb0, 0x82, 0x17, 0xc1, 0x53, 0xb8, 0x34,
+	0x48, 0xee, 0x39, 0xae, 0x51, 0x43, 0xec, 0x82,
+	0xce, 0x87, 0xc6, 0x76, 0xb9, 0x76, 0xd3, 0x53,
+	0xfe, 0x49, 0x24, 0x7d, 0x02, 0x42, 0x2b, 0x72,
+	0xfb, 0xcb, 0xd8, 0x96, 0x02, 0xc6, 0x9a, 0x20,
+	0xf3, 0x5a, 0x67, 0xe8, 0x13, 0xf8, 0xb2, 0xcb,
+	0xa2, 0xec, 0x18, 0x20, 0x4a, 0xb0, 0x73, 0x53,
+	0x21, 0xb0, 0x77, 0x53, 0xd8, 0x76, 0xa1, 0x30,
+	0x17, 0x72, 0x2e, 0x33, 0x5f, 0x33, 0x6b, 0x28,
+	0xfb, 0xb0, 0xf4, 0xec, 0x8e, 0xed, 0x20, 0x7d,
+	0x57, 0x8c, 0x74, 0x28, 0x64, 0x8b, 0xeb, 0x59,
+	0x38, 0x3f, 0xe7, 0x83, 0x2e, 0xe5, 0x64, 0x4d,
+	0x5c, 0x1f, 0xe1, 0x3b, 0xd9, 0x84, 0xdb, 0xc9,
+	0xec, 0xd8, 0xc1, 0x7c, 0x1f, 0x1b, 0x68, 0x35,
+	0xc6, 0x34, 0x10, 0xef, 0x19, 0xc9, 0x0a, 0xd6,
+	0x43, 0x7f, 0xa6, 0xcb, 0x9d, 0xf4, 0xf0, 0x16,
+	0xb1, 0xb1, 0x96, 0x64, 0xec, 0x8d, 0x22, 0x4c,
+	0x4b, 0xe8, 0x1a, 0xba, 0x6f, 0xb7, 0xfc, 0xa5,
+	0x69, 0x3e, 0xad, 0x78, 0x79, 0x19, 0xb5, 0x04,
+	0x69, 0xe5, 0x3f, 0xff, 0x60, 0x8c, 0xda, 0x0b,
+	0x7b, 0xf7, 0xe7, 0xe6, 0x29, 0x3a, 0x85, 0xba,
+	0xb5, 0xb0, 0x35, 0xbd, 0x38, 0xce, 0x34, 0x5e,
+	0xf2, 0xdc, 0xd1, 0x8f, 0xc3, 0x03, 0x24, 0xa2,
+	0x03, 0xf7, 0x4e, 0x49, 0x5b, 0xcf, 0x6d, 0xb0,
+	0xeb, 0xe3, 0x30, 0x28, 0xd5, 0x5b, 0x82, 0x5f,
+	0xe4, 0x7c, 0x1e, 0xec, 0xd2, 0x39, 0xf9, 0x6f,
+	0x2e, 0xb3, 0xcd, 0x01, 0xb1, 0x67, 0xaa, 0xea,
+	0xaa, 0xb3, 0x63, 0xaf, 0xd9, 0xb2, 0x1f, 0xba,
+	0x05, 0x20, 0xeb, 0x19, 0x32, 0xf0, 0x6c, 0x3f,
+	0x40, 0xcc, 0x93, 0xb3, 0xd8, 0x25, 0xa6, 0xe4,
+	0xce, 0xd7, 0x7e, 0x48, 0x99, 0x65, 0x7f, 0x86,
+	0xc5, 0xd4, 0x79, 0x6b, 0xab, 0x43, 0xb8, 0x6b,
+	0xf1, 0x2f, 0xea, 0x4c, 0x5e, 0xf0, 0x3b, 0xb4,
+	0xb8, 0xb0, 0x94, 0x0c, 0x6b, 0xe7, 0x22, 0x93,
+	0xaa, 0x01, 0xcb, 0xf1, 0x11, 0x60, 0xf6, 0x69,
+	0xcf, 0x14, 0xde, 0xfb, 0x90, 0x05, 0x27, 0x0c,
+	0x1a, 0x9e, 0xf0, 0xb4, 0xc6, 0xa1, 0xe8, 0xdd,
+	0xd0, 0x4c, 0x25, 0x4f, 0x9c, 0xb7, 0xb1, 0xb0,
+	0x21, 0xdb, 0x87, 0x09, 0x03, 0xf2, 0xb3
+};
+static const u8 key70[] __initconst = {
+	0x3b, 0x5b, 0x59, 0x36, 0x44, 0xd1, 0xba, 0x71,
+	0x55, 0x87, 0x4d, 0x62, 0x3d, 0xc2, 0xfc, 0xaa,
+	0x3f, 0x4e, 0x1a, 0xe4, 0xca, 0x09, 0xfc, 0x6a,
+	0xb2, 0xd6, 0x5d, 0x79, 0xf9, 0x1a, 0x91, 0xa7
+};
+enum { nonce70 = 0x3fd6786dd147a85ULL };
+
+static const u8 input71[] __initconst = {
+	0x18, 0x78, 0xd6, 0x79, 0xe4, 0x9a, 0x6c, 0x73,
+	0x17, 0xd4, 0x05, 0x0f, 0x1e, 0x9f, 0xd9, 0x2b,
+	0x86, 0x48, 0x7d, 0xf4, 0xd9, 0x1c, 0x76, 0xfc,
+	0x8e, 0x22, 0x34, 0xe1, 0x48, 0x4a, 0x8d, 0x79,
+	0xb7, 0xbb, 0x88, 0xab, 0x90, 0xde, 0xc5, 0xb4,
+	0xb4, 0xe7, 0x85, 0x49, 0xda, 0x57, 0xeb, 0xc9,
+	0xcd, 0x21, 0xfc, 0x45, 0x6e, 0x32, 0x67, 0xf2,
+	0x4f, 0xa6, 0x54, 0xe5, 0x20, 0xed, 0xcf, 0xc6,
+	0x62, 0x25, 0x8e, 0x00, 0xf8, 0x6b, 0xa2, 0x80,
+	0xac, 0x88, 0xa6, 0x59, 0x27, 0x83, 0x95, 0x11,
+	0x3f, 0x70, 0x5e, 0x3f, 0x11, 0xfb, 0x26, 0xbf,
+	0xe1, 0x48, 0x75, 0xf9, 0x86, 0xbf, 0xa6, 0x5d,
+	0x15, 0x61, 0x66, 0xbf, 0x78, 0x8f, 0x6b, 0x9b,
+	0xda, 0x98, 0xb7, 0x19, 0xe2, 0xf2, 0xa3, 0x9c,
+	0x7c, 0x6a, 0x9a, 0xd8, 0x3d, 0x4c, 0x2c, 0xe1,
+	0x09, 0xb4, 0x28, 0x82, 0x4e, 0xab, 0x0c, 0x75,
+	0x63, 0xeb, 0xbc, 0xd0, 0x71, 0xa2, 0x73, 0x85,
+	0xed, 0x53, 0x7a, 0x3f, 0x68, 0x9f, 0xd0, 0xa9,
+	0x00, 0x5a, 0x9e, 0x80, 0x55, 0x00, 0xe6, 0xae,
+	0x0c, 0x03, 0x40, 0xed, 0xfc, 0x68, 0x4a, 0xb7,
+	0x1e, 0x09, 0x65, 0x30, 0x5a, 0x3d, 0x97, 0x4d,
+	0x5e, 0x51, 0x8e, 0xda, 0xc3, 0x55, 0x8c, 0xfb,
+	0xcf, 0x83, 0x05, 0x35, 0x0d, 0x08, 0x1b, 0xf3,
+	0x3a, 0x57, 0x96, 0xac, 0x58, 0x8b, 0xfa, 0x00,
+	0x49, 0x15, 0x78, 0xd2, 0x4b, 0xed, 0xb8, 0x59,
+	0x78, 0x9b, 0x7f, 0xaa, 0xfc, 0xe7, 0x46, 0xdc,
+	0x7b, 0x34, 0xd0, 0x34, 0xe5, 0x10, 0xff, 0x4d,
+	0x5a, 0x4d, 0x60, 0xa7, 0x16, 0x54, 0xc4, 0xfd,
+	0xca, 0x5d, 0x68, 0xc7, 0x4a, 0x01, 0x8d, 0x7f,
+	0x74, 0x5d, 0xff, 0xb8, 0x37, 0x15, 0x62, 0xfa,
+	0x44, 0x45, 0xcf, 0x77, 0x3b, 0x1d, 0xb2, 0xd2,
+	0x0d, 0x42, 0x00, 0x39, 0x68, 0x1f, 0xcc, 0x89,
+	0x73, 0x5d, 0xa9, 0x2e, 0xfd, 0x58, 0x62, 0xca,
+	0x35, 0x8e, 0x70, 0x70, 0xaa, 0x6e, 0x14, 0xe9,
+	0xa4, 0xe2, 0x10, 0x66, 0x71, 0xdc, 0x4c, 0xfc,
+	0xa9, 0xdc, 0x8f, 0x57, 0x4d, 0xc5, 0xac, 0xd7,
+	0xa9, 0xf3, 0xf3, 0xa1, 0xff, 0x62, 0xa0, 0x8f,
+	0xe4, 0x96, 0x3e, 0xcb, 0x9f, 0x76, 0x42, 0x39,
+	0x1f, 0x24, 0xfd, 0xfd, 0x79, 0xe8, 0x27, 0xdf,
+	0xa8, 0xf6, 0x33, 0x8b, 0x31, 0x59, 0x69, 0xcf,
+	0x6a, 0xef, 0x89, 0x4d, 0xa7, 0xf6, 0x7e, 0x97,
+	0x14, 0xbd, 0xda, 0xdd, 0xb4, 0x84, 0x04, 0x24,
+	0xe0, 0x17, 0xe1, 0x0f, 0x1f, 0x8a, 0x6a, 0x71,
+	0x74, 0x41, 0xdc, 0x59, 0x5c, 0x8f, 0x01, 0x25,
+	0x92, 0xf0, 0x2e, 0x15, 0x62, 0x71, 0x9a, 0x9f,
+	0x87, 0xdf, 0x62, 0x49, 0x7f, 0x86, 0x62, 0xfc,
+	0x20, 0x84, 0xd7, 0xe3, 0x3a, 0xd9, 0x37, 0x85,
+	0xb7, 0x84, 0x5a, 0xf9, 0xed, 0x21, 0x32, 0x94,
+	0x3e, 0x04, 0xe7, 0x8c, 0x46, 0x76, 0x21, 0x67,
+	0xf6, 0x95, 0x64, 0x92, 0xb7, 0x15, 0xf6, 0xe3,
+	0x41, 0x27, 0x9d, 0xd7, 0xe3, 0x79, 0x75, 0x92,
+	0xd0, 0xc1, 0xf3, 0x40, 0x92, 0x08, 0xde, 0x90,
+	0x22, 0x82, 0xb2, 0x69, 0xae, 0x1a, 0x35, 0x11,
+	0x89, 0xc8, 0x06, 0x82, 0x95, 0x23, 0x44, 0x08,
+	0x22, 0xf2, 0x71, 0x73, 0x1b, 0x88, 0x11, 0xcf,
+	0x1c, 0x7e, 0x8a, 0x2e, 0xdc, 0x79, 0x57, 0xce,
+	0x1f, 0xe7, 0x6c, 0x07, 0xd8, 0x06, 0xbe, 0xec,
+	0xa3, 0xcf, 0xf9, 0x68, 0xa5, 0xb8, 0xf0, 0xe3,
+	0x3f, 0x01, 0x92, 0xda, 0xf1, 0xa0, 0x2d, 0x7b,
+	0xab, 0x57, 0x58, 0x2a, 0xaf, 0xab, 0xbd, 0xf2,
+	0xe5, 0xaf, 0x7e, 0x1f, 0x46, 0x24, 0x9e, 0x20,
+	0x22, 0x0f, 0x84, 0x4c, 0xb7, 0xd8, 0x03, 0xe8,
+	0x09, 0x73, 0x6c, 0xc6, 0x9b, 0x90, 0xe0, 0xdb,
+	0xf2, 0x71, 0xba, 0xad, 0xb3, 0xec, 0xda, 0x7a
+};
+static const u8 output71[] __initconst = {
+	0x28, 0xc5, 0x9b, 0x92, 0xf9, 0x21, 0x4f, 0xbb,
+	0xef, 0x3b, 0xf0, 0xf5, 0x3a, 0x6d, 0x7f, 0xd6,
+	0x6a, 0x8d, 0xa1, 0x01, 0x5c, 0x62, 0x20, 0x8b,
+	0x5b, 0x39, 0xd5, 0xd3, 0xc2, 0xf6, 0x9d, 0x5e,
+	0xcc, 0xe1, 0xa2, 0x61, 0x16, 0xe2, 0xce, 0xe9,
+	0x86, 0xd0, 0xfc, 0xce, 0x9a, 0x28, 0x27, 0xc4,
+	0x0c, 0xb9, 0xaa, 0x8d, 0x48, 0xdb, 0xbf, 0x82,
+	0x7d, 0xd0, 0x35, 0xc4, 0x06, 0x34, 0xb4, 0x19,
+	0x51, 0x73, 0xf4, 0x7a, 0xf4, 0xfd, 0xe9, 0x1d,
+	0xdc, 0x0f, 0x7e, 0xf7, 0x96, 0x03, 0xe3, 0xb1,
+	0x2e, 0x22, 0x59, 0xb7, 0x6d, 0x1c, 0x97, 0x8c,
+	0xd7, 0x31, 0x08, 0x26, 0x4c, 0x6d, 0xc6, 0x14,
+	0xa5, 0xeb, 0x45, 0x6a, 0x88, 0xa3, 0xa2, 0x36,
+	0xc4, 0x35, 0xb1, 0x5a, 0xa0, 0xad, 0xf7, 0x06,
+	0x9b, 0x5d, 0xc1, 0x15, 0xc1, 0xce, 0x0a, 0xb0,
+	0x57, 0x2e, 0x3f, 0x6f, 0x0d, 0x10, 0xd9, 0x11,
+	0x2c, 0x9c, 0xad, 0x2d, 0xa5, 0x81, 0xfb, 0x4e,
+	0x8f, 0xd5, 0x32, 0x4e, 0xaf, 0x5c, 0xc1, 0x86,
+	0xde, 0x56, 0x5a, 0x33, 0x29, 0xf7, 0x67, 0xc6,
+	0x37, 0x6f, 0xb2, 0x37, 0x4e, 0xd4, 0x69, 0x79,
+	0xaf, 0xd5, 0x17, 0x79, 0xe0, 0xba, 0x62, 0xa3,
+	0x68, 0xa4, 0x87, 0x93, 0x8d, 0x7e, 0x8f, 0xa3,
+	0x9c, 0xef, 0xda, 0xe3, 0xa5, 0x1f, 0xcd, 0x30,
+	0xa6, 0x55, 0xac, 0x4c, 0x69, 0x74, 0x02, 0xc7,
+	0x5d, 0x95, 0x81, 0x4a, 0x68, 0x11, 0xd3, 0xa9,
+	0x98, 0xb1, 0x0b, 0x0d, 0xae, 0x40, 0x86, 0x65,
+	0xbf, 0xcc, 0x2d, 0xef, 0x57, 0xca, 0x1f, 0xe4,
+	0x34, 0x4e, 0xa6, 0x5e, 0x82, 0x6e, 0x61, 0xad,
+	0x0b, 0x3c, 0xf8, 0xeb, 0x01, 0x43, 0x7f, 0x87,
+	0xa2, 0xa7, 0x6a, 0xe9, 0x62, 0x23, 0x24, 0x61,
+	0xf1, 0xf7, 0x36, 0xdb, 0x10, 0xe5, 0x57, 0x72,
+	0x3a, 0xc2, 0xae, 0xcc, 0x75, 0xc7, 0x80, 0x05,
+	0x0a, 0x5c, 0x4c, 0x95, 0xda, 0x02, 0x01, 0x14,
+	0x06, 0x6b, 0x5c, 0x65, 0xc2, 0xb8, 0x4a, 0xd6,
+	0xd3, 0xb4, 0xd8, 0x12, 0x52, 0xb5, 0x60, 0xd3,
+	0x8e, 0x5f, 0x5c, 0x76, 0x33, 0x7a, 0x05, 0xe5,
+	0xcb, 0xef, 0x4f, 0x89, 0xf1, 0xba, 0x32, 0x6f,
+	0x33, 0xcd, 0x15, 0x8d, 0xa3, 0x0c, 0x3f, 0x63,
+	0x11, 0xe7, 0x0e, 0xe0, 0x00, 0x01, 0xe9, 0xe8,
+	0x8e, 0x36, 0x34, 0x8d, 0x96, 0xb5, 0x03, 0xcf,
+	0x55, 0x62, 0x49, 0x7a, 0x34, 0x44, 0xa5, 0xee,
+	0x8c, 0x46, 0x06, 0x22, 0xab, 0x1d, 0x53, 0x9c,
+	0xa1, 0xf9, 0x67, 0x18, 0x57, 0x89, 0xf9, 0xc2,
+	0xd1, 0x7e, 0xbe, 0x36, 0x40, 0xcb, 0xe9, 0x04,
+	0xde, 0xb1, 0x3b, 0x29, 0x52, 0xc5, 0x9a, 0xb5,
+	0xa2, 0x7c, 0x7b, 0xfe, 0xe5, 0x92, 0x73, 0xea,
+	0xea, 0x7b, 0xba, 0x0a, 0x8c, 0x88, 0x15, 0xe6,
+	0x53, 0xbf, 0x1c, 0x33, 0xf4, 0x9b, 0x9a, 0x5e,
+	0x8d, 0xae, 0x60, 0xdc, 0xcb, 0x5d, 0xfa, 0xbe,
+	0x06, 0xc3, 0x3f, 0x06, 0xe7, 0x00, 0x40, 0x7b,
+	0xaa, 0x94, 0xfa, 0x6d, 0x1f, 0xe4, 0xc5, 0xa9,
+	0x1b, 0x5f, 0x36, 0xea, 0x5a, 0xdd, 0xa5, 0x48,
+	0x6a, 0x55, 0xd2, 0x47, 0x28, 0xbf, 0x96, 0xf1,
+	0x9f, 0xb6, 0x11, 0x4b, 0xd3, 0x44, 0x7d, 0x48,
+	0x41, 0x61, 0xdb, 0x12, 0xd4, 0xc2, 0x59, 0x82,
+	0x4c, 0x47, 0x5c, 0x04, 0xf6, 0x7b, 0xd3, 0x92,
+	0x2e, 0xe8, 0x40, 0xef, 0x15, 0x32, 0x97, 0xdc,
+	0x35, 0x4c, 0x6e, 0xa4, 0x97, 0xe9, 0x24, 0xde,
+	0x63, 0x8b, 0xb1, 0x6b, 0x48, 0xbb, 0x46, 0x1f,
+	0x84, 0xd6, 0x17, 0xb0, 0x5a, 0x4a, 0x4e, 0xd5,
+	0x31, 0xd7, 0xcf, 0xa0, 0x39, 0xc6, 0x2e, 0xfc,
+	0xa6, 0xa3, 0xd3, 0x0f, 0xa4, 0x28, 0xac, 0xb2,
+	0xf4, 0x48, 0x8d, 0x50, 0xa5, 0x1c, 0x44, 0x5d,
+	0x6e, 0x38, 0xb7, 0x2b, 0x8a, 0x45, 0xa7, 0x3d
+};
+static const u8 key71[] __initconst = {
+	0x8b, 0x68, 0xc4, 0xb7, 0x0d, 0x81, 0xef, 0x52,
+	0x1e, 0x05, 0x96, 0x72, 0x62, 0x89, 0x27, 0x83,
+	0xd0, 0xc7, 0x33, 0x6d, 0xf2, 0xcc, 0x69, 0xf9,
+	0x23, 0xae, 0x99, 0xb1, 0xd1, 0x05, 0x4e, 0x54
+};
+enum { nonce71 = 0x983f03656d64b5f6ULL };
+
+static const u8 input72[] __initconst = {
+	0x6b, 0x09, 0xc9, 0x57, 0x3d, 0x79, 0x04, 0x8c,
+	0x65, 0xad, 0x4a, 0x0f, 0xa1, 0x31, 0x3a, 0xdd,
+	0x14, 0x8e, 0xe8, 0xfe, 0xbf, 0x42, 0x87, 0x98,
+	0x2e, 0x8d, 0x83, 0xa3, 0xf8, 0x55, 0x3d, 0x84,
+	0x1e, 0x0e, 0x05, 0x4a, 0x38, 0x9e, 0xe7, 0xfe,
+	0xd0, 0x4d, 0x79, 0x74, 0x3a, 0x0b, 0x9b, 0xe1,
+	0xfd, 0x51, 0x84, 0x4e, 0xb2, 0x25, 0xe4, 0x64,
+	0x4c, 0xda, 0xcf, 0x46, 0xec, 0xba, 0x12, 0xeb,
+	0x5a, 0x33, 0x09, 0x6e, 0x78, 0x77, 0x8f, 0x30,
+	0xb1, 0x7d, 0x3f, 0x60, 0x8c, 0xf2, 0x1d, 0x8e,
+	0xb4, 0x70, 0xa2, 0x90, 0x7c, 0x79, 0x1a, 0x2c,
+	0xf6, 0x28, 0x79, 0x7c, 0x53, 0xc5, 0xfa, 0xcc,
+	0x65, 0x9b, 0xe1, 0x51, 0xd1, 0x7f, 0x1d, 0xc4,
+	0xdb, 0xd4, 0xd9, 0x04, 0x61, 0x7d, 0xbe, 0x12,
+	0xfc, 0xcd, 0xaf, 0xe4, 0x0f, 0x9c, 0x20, 0xb5,
+	0x22, 0x40, 0x18, 0xda, 0xe4, 0xda, 0x8c, 0x2d,
+	0x84, 0xe3, 0x5f, 0x53, 0x17, 0xed, 0x78, 0xdc,
+	0x2f, 0xe8, 0x31, 0xc7, 0xe6, 0x39, 0x71, 0x40,
+	0xb4, 0x0f, 0xc9, 0xa9, 0x7e, 0x78, 0x87, 0xc1,
+	0x05, 0x78, 0xbb, 0x01, 0xf2, 0x8f, 0x33, 0xb0,
+	0x6e, 0x84, 0xcd, 0x36, 0x33, 0x5c, 0x5b, 0x8e,
+	0xf1, 0xac, 0x30, 0xfe, 0x33, 0xec, 0x08, 0xf3,
+	0x7e, 0xf2, 0xf0, 0x4c, 0xf2, 0xad, 0xd8, 0xc1,
+	0xd4, 0x4e, 0x87, 0x06, 0xd4, 0x75, 0xe7, 0xe3,
+	0x09, 0xd3, 0x4d, 0xe3, 0x21, 0x32, 0xba, 0xb4,
+	0x68, 0x68, 0xcb, 0x4c, 0xa3, 0x1e, 0xb3, 0x87,
+	0x7b, 0xd3, 0x0c, 0x63, 0x37, 0x71, 0x79, 0xfb,
+	0x58, 0x36, 0x57, 0x0f, 0x34, 0x1d, 0xc1, 0x42,
+	0x02, 0x17, 0xe7, 0xed, 0xe8, 0xe7, 0x76, 0xcb,
+	0x42, 0xc4, 0x4b, 0xe2, 0xb2, 0x5e, 0x42, 0xd5,
+	0xec, 0x9d, 0xc1, 0x32, 0x71, 0xe4, 0xeb, 0x10,
+	0x68, 0x1a, 0x6e, 0x99, 0x8e, 0x73, 0x12, 0x1f,
+	0x97, 0x0c, 0x9e, 0xcd, 0x02, 0x3e, 0x4c, 0xa0,
+	0xf2, 0x8d, 0xe5, 0x44, 0xca, 0x6d, 0xfe, 0x07,
+	0xe3, 0xe8, 0x9b, 0x76, 0xc1, 0x6d, 0xb7, 0x6e,
+	0x0d, 0x14, 0x00, 0x6f, 0x8a, 0xfd, 0x43, 0xc6,
+	0x43, 0xa5, 0x9c, 0x02, 0x47, 0x10, 0xd4, 0xb4,
+	0x9b, 0x55, 0x67, 0xc8, 0x7f, 0xc1, 0x8a, 0x1f,
+	0x1e, 0xd1, 0xbc, 0x99, 0x5d, 0x50, 0x4f, 0x89,
+	0xf1, 0xe6, 0x5d, 0x91, 0x40, 0xdc, 0x20, 0x67,
+	0x56, 0xc2, 0xef, 0xbd, 0x2c, 0xa2, 0x99, 0x38,
+	0xe0, 0x45, 0xec, 0x44, 0x05, 0x52, 0x65, 0x11,
+	0xfc, 0x3b, 0x19, 0xcb, 0x71, 0xc2, 0x8e, 0x0e,
+	0x03, 0x2a, 0x03, 0x3b, 0x63, 0x06, 0x31, 0x9a,
+	0xac, 0x53, 0x04, 0x14, 0xd4, 0x80, 0x9d, 0x6b,
+	0x42, 0x7e, 0x7e, 0x4e, 0xdc, 0xc7, 0x01, 0x49,
+	0x9f, 0xf5, 0x19, 0x86, 0x13, 0x28, 0x2b, 0xa6,
+	0xa6, 0xbe, 0xa1, 0x7e, 0x71, 0x05, 0x00, 0xff,
+	0x59, 0x2d, 0xb6, 0x63, 0xf0, 0x1e, 0x2e, 0x69,
+	0x9b, 0x85, 0xf1, 0x1e, 0x8a, 0x64, 0x39, 0xab,
+	0x00, 0x12, 0xe4, 0x33, 0x4b, 0xb5, 0xd8, 0xb3,
+	0x6b, 0x5b, 0x8b, 0x5c, 0xd7, 0x6f, 0x23, 0xcf,
+	0x3f, 0x2e, 0x5e, 0x47, 0xb9, 0xb8, 0x1f, 0xf0,
+	0x1d, 0xda, 0xe7, 0x4f, 0x6e, 0xab, 0xc3, 0x36,
+	0xb4, 0x74, 0x6b, 0xeb, 0xc7, 0x5d, 0x91, 0xe5,
+	0xda, 0xf2, 0xc2, 0x11, 0x17, 0x48, 0xf8, 0x9c,
+	0xc9, 0x8b, 0xc1, 0xa2, 0xf4, 0xcd, 0x16, 0xf8,
+	0x27, 0xd9, 0x6c, 0x6f, 0xb5, 0x8f, 0x77, 0xca,
+	0x1b, 0xd8, 0xef, 0x84, 0x68, 0x71, 0x53, 0xc1,
+	0x43, 0x0f, 0x9f, 0x98, 0xae, 0x7e, 0x31, 0xd2,
+	0x98, 0xfb, 0x20, 0xa2, 0xad, 0x00, 0x10, 0x83,
+	0x00, 0x8b, 0xeb, 0x56, 0xd2, 0xc4, 0xcc, 0x7f,
+	0x2f, 0x4e, 0xfa, 0x88, 0x13, 0xa4, 0x2c, 0xde,
+	0x6b, 0x77, 0x86, 0x10, 0x6a, 0xab, 0x43, 0x0a,
+	0x02
+};
+static const u8 output72[] __initconst = {
+	0x42, 0x89, 0xa4, 0x80, 0xd2, 0xcb, 0x5f, 0x7f,
+	0x2a, 0x1a, 0x23, 0x00, 0xa5, 0x6a, 0x95, 0xa3,
+	0x9a, 0x41, 0xa1, 0xd0, 0x2d, 0x1e, 0xd6, 0x13,
+	0x34, 0x40, 0x4e, 0x7f, 0x1a, 0xbe, 0xa0, 0x3d,
+	0x33, 0x9c, 0x56, 0x2e, 0x89, 0x25, 0x45, 0xf9,
+	0xf0, 0xba, 0x9c, 0x6d, 0xd1, 0xd1, 0xde, 0x51,
+	0x47, 0x63, 0xc9, 0xbd, 0xfa, 0xa2, 0x9e, 0xad,
+	0x6a, 0x7b, 0x21, 0x1a, 0x6c, 0x3e, 0xff, 0x46,
+	0xbe, 0xf3, 0x35, 0x7a, 0x6e, 0xb3, 0xb9, 0xf7,
+	0xda, 0x5e, 0xf0, 0x14, 0xb5, 0x70, 0xa4, 0x2b,
+	0xdb, 0xbb, 0xc7, 0x31, 0x4b, 0x69, 0x5a, 0x83,
+	0x70, 0xd9, 0x58, 0xd4, 0x33, 0x84, 0x23, 0xf0,
+	0xae, 0xbb, 0x6d, 0x26, 0x7c, 0xc8, 0x30, 0xf7,
+	0x24, 0xad, 0xbd, 0xe4, 0x2c, 0x38, 0x38, 0xac,
+	0xe1, 0x4a, 0x9b, 0xac, 0x33, 0x0e, 0x4a, 0xf4,
+	0x93, 0xed, 0x07, 0x82, 0x81, 0x4f, 0x8f, 0xb1,
+	0xdd, 0x73, 0xd5, 0x50, 0x6d, 0x44, 0x1e, 0xbe,
+	0xa7, 0xcd, 0x17, 0x57, 0xd5, 0x3b, 0x62, 0x36,
+	0xcf, 0x7d, 0xc8, 0xd8, 0xd1, 0x78, 0xd7, 0x85,
+	0x46, 0x76, 0x5d, 0xcc, 0xfe, 0xe8, 0x94, 0xc5,
+	0xad, 0xbc, 0x5e, 0xbc, 0x8d, 0x1d, 0xdf, 0x03,
+	0xc9, 0x6b, 0x1b, 0x81, 0xd1, 0xb6, 0x5a, 0x24,
+	0xe3, 0xdc, 0x3f, 0x20, 0xc9, 0x07, 0x73, 0x4c,
+	0x43, 0x13, 0x87, 0x58, 0x34, 0x0d, 0x14, 0x63,
+	0x0f, 0x6f, 0xad, 0x8d, 0xac, 0x7c, 0x67, 0x68,
+	0xa3, 0x9d, 0x7f, 0x00, 0xdf, 0x28, 0xee, 0x67,
+	0xf4, 0x5c, 0x26, 0xcb, 0xef, 0x56, 0x71, 0xc8,
+	0xc6, 0x67, 0x5f, 0x38, 0xbb, 0xa0, 0xb1, 0x5c,
+	0x1f, 0xb3, 0x08, 0xd9, 0x38, 0xcf, 0x74, 0x54,
+	0xc6, 0xa4, 0xc4, 0xc0, 0x9f, 0xb3, 0xd0, 0xda,
+	0x62, 0x67, 0x8b, 0x81, 0x33, 0xf0, 0xa9, 0x73,
+	0xa4, 0xd1, 0x46, 0x88, 0x8d, 0x85, 0x12, 0x40,
+	0xba, 0x1a, 0xcd, 0x82, 0xd8, 0x8d, 0xc4, 0x52,
+	0xe7, 0x01, 0x94, 0x2e, 0x0e, 0xd0, 0xaf, 0xe7,
+	0x2d, 0x3f, 0x3c, 0xaa, 0xf4, 0xf5, 0xa7, 0x01,
+	0x4c, 0x14, 0xe2, 0xc2, 0x96, 0x76, 0xbe, 0x05,
+	0xaa, 0x19, 0xb1, 0xbd, 0x95, 0xbb, 0x5a, 0xf9,
+	0xa5, 0xa7, 0xe6, 0x16, 0x38, 0x34, 0xf7, 0x9d,
+	0x19, 0x66, 0x16, 0x8e, 0x7f, 0x2b, 0x5a, 0xfb,
+	0xb5, 0x29, 0x79, 0xbf, 0x52, 0xae, 0x30, 0x95,
+	0x3f, 0x31, 0x33, 0x28, 0xde, 0xc5, 0x0d, 0x55,
+	0x89, 0xec, 0x21, 0x11, 0x0f, 0x8b, 0xfe, 0x63,
+	0x3a, 0xf1, 0x95, 0x5c, 0xcd, 0x50, 0xe4, 0x5d,
+	0x8f, 0xa7, 0xc8, 0xca, 0x93, 0xa0, 0x67, 0x82,
+	0x63, 0x5c, 0xd0, 0xed, 0xe7, 0x08, 0xc5, 0x60,
+	0xf8, 0xb4, 0x47, 0xf0, 0x1a, 0x65, 0x4e, 0xa3,
+	0x51, 0x68, 0xc7, 0x14, 0xa1, 0xd9, 0x39, 0x72,
+	0xa8, 0x6f, 0x7c, 0x7e, 0xf6, 0x03, 0x0b, 0x25,
+	0x9b, 0xf2, 0xca, 0x49, 0xae, 0x5b, 0xf8, 0x0f,
+	0x71, 0x51, 0x01, 0xa6, 0x23, 0xa9, 0xdf, 0xd0,
+	0x7a, 0x39, 0x19, 0xf5, 0xc5, 0x26, 0x44, 0x7b,
+	0x0a, 0x4a, 0x41, 0xbf, 0xf2, 0x8e, 0x83, 0x50,
+	0x91, 0x96, 0x72, 0x02, 0xf6, 0x80, 0xbf, 0x95,
+	0x41, 0xac, 0xda, 0xb0, 0xba, 0xe3, 0x76, 0xb1,
+	0x9d, 0xff, 0x1f, 0x33, 0x02, 0x85, 0xfc, 0x2a,
+	0x29, 0xe6, 0xe3, 0x9d, 0xd0, 0xef, 0xc2, 0xd6,
+	0x9c, 0x4a, 0x62, 0xac, 0xcb, 0xea, 0x8b, 0xc3,
+	0x08, 0x6e, 0x49, 0x09, 0x26, 0x19, 0xc1, 0x30,
+	0xcc, 0x27, 0xaa, 0xc6, 0x45, 0x88, 0xbd, 0xae,
+	0xd6, 0x79, 0xff, 0x4e, 0xfc, 0x66, 0x4d, 0x02,
+	0xa5, 0xee, 0x8e, 0xa5, 0xb6, 0x15, 0x72, 0x24,
+	0xb1, 0xbf, 0xbf, 0x64, 0xcf, 0xcc, 0x93, 0xe9,
+	0xb6, 0xfd, 0xb4, 0xb6, 0x21, 0xb5, 0x48, 0x08,
+	0x0f, 0x11, 0x65, 0xe1, 0x47, 0xee, 0x93, 0x29,
+	0xad
+};
+static const u8 key72[] __initconst = {
+	0xb9, 0xa2, 0xfc, 0x59, 0x06, 0x3f, 0x77, 0xa5,
+	0x66, 0xd0, 0x2b, 0x22, 0x74, 0x22, 0x4c, 0x1e,
+	0x6a, 0x39, 0xdf, 0xe1, 0x0d, 0x4c, 0x64, 0x99,
+	0x54, 0x8a, 0xba, 0x1d, 0x2c, 0x21, 0x5f, 0xc3
+};
+enum { nonce72 = 0x3d069308fa3db04bULL };
+
+static const u8 input73[] __initconst = {
+	0xe4, 0xdd, 0x36, 0xd4, 0xf5, 0x70, 0x51, 0x73,
+	0x97, 0x1d, 0x45, 0x05, 0x92, 0xe7, 0xeb, 0xb7,
+	0x09, 0x82, 0x6e, 0x25, 0x6c, 0x50, 0xf5, 0x40,
+	0x19, 0xba, 0xbc, 0xf4, 0x39, 0x14, 0xc5, 0x15,
+	0x83, 0x40, 0xbd, 0x26, 0xe0, 0xff, 0x3b, 0x22,
+	0x7c, 0x7c, 0xd7, 0x0b, 0xe9, 0x25, 0x0c, 0x3d,
+	0x92, 0x38, 0xbe, 0xe4, 0x22, 0x75, 0x65, 0xf1,
+	0x03, 0x85, 0x34, 0x09, 0xb8, 0x77, 0xfb, 0x48,
+	0xb1, 0x2e, 0x21, 0x67, 0x9b, 0x9d, 0xad, 0x18,
+	0x82, 0x0d, 0x6b, 0xc3, 0xcf, 0x00, 0x61, 0x6e,
+	0xda, 0xdc, 0xa7, 0x0b, 0x5c, 0x02, 0x1d, 0xa6,
+	0x4e, 0x0d, 0x7f, 0x37, 0x01, 0x5a, 0x37, 0xf3,
+	0x2b, 0xbf, 0xba, 0xe2, 0x1c, 0xb3, 0xa3, 0xbc,
+	0x1c, 0x93, 0x1a, 0xb1, 0x71, 0xaf, 0xe2, 0xdd,
+	0x17, 0xee, 0x53, 0xfa, 0xfb, 0x02, 0x40, 0x3e,
+	0x03, 0xca, 0xe7, 0xc3, 0x51, 0x81, 0xcc, 0x8c,
+	0xca, 0xcf, 0x4e, 0xc5, 0x78, 0x99, 0xfd, 0xbf,
+	0xea, 0xab, 0x38, 0x81, 0xfc, 0xd1, 0x9e, 0x41,
+	0x0b, 0x84, 0x25, 0xf1, 0x6b, 0x3c, 0xf5, 0x40,
+	0x0d, 0xc4, 0x3e, 0xb3, 0x6a, 0xec, 0x6e, 0x75,
+	0xdc, 0x9b, 0xdf, 0x08, 0x21, 0x16, 0xfb, 0x7a,
+	0x8e, 0x19, 0x13, 0x02, 0xa7, 0xfc, 0x58, 0x21,
+	0xc3, 0xb3, 0x59, 0x5a, 0x9c, 0xef, 0x38, 0xbd,
+	0x87, 0x55, 0xd7, 0x0d, 0x1f, 0x84, 0xdc, 0x98,
+	0x22, 0xca, 0x87, 0x96, 0x71, 0x6d, 0x68, 0x00,
+	0xcb, 0x4f, 0x2f, 0xc4, 0x64, 0x0c, 0xc1, 0x53,
+	0x0c, 0x90, 0xe7, 0x3c, 0x88, 0xca, 0xc5, 0x85,
+	0xa3, 0x2a, 0x96, 0x7c, 0x82, 0x6d, 0x45, 0xf5,
+	0xb7, 0x8d, 0x17, 0x69, 0xd6, 0xcd, 0x3c, 0xd3,
+	0xe7, 0x1c, 0xce, 0x93, 0x50, 0xd4, 0x59, 0xa2,
+	0xd8, 0x8b, 0x72, 0x60, 0x5b, 0x25, 0x14, 0xcd,
+	0x5a, 0xe8, 0x8c, 0xdb, 0x23, 0x8d, 0x2b, 0x59,
+	0x12, 0x13, 0x10, 0x47, 0xa4, 0xc8, 0x3c, 0xc1,
+	0x81, 0x89, 0x6c, 0x98, 0xec, 0x8f, 0x7b, 0x32,
+	0xf2, 0x87, 0xd9, 0xa2, 0x0d, 0xc2, 0x08, 0xf9,
+	0xd5, 0xf3, 0x91, 0xe7, 0xb3, 0x87, 0xa7, 0x0b,
+	0x64, 0x8f, 0xb9, 0x55, 0x1c, 0x81, 0x96, 0x6c,
+	0xa1, 0xc9, 0x6e, 0x3b, 0xcd, 0x17, 0x1b, 0xfc,
+	0xa6, 0x05, 0xba, 0x4a, 0x7d, 0x03, 0x3c, 0x59,
+	0xc8, 0xee, 0x50, 0xb2, 0x5b, 0xe1, 0x4d, 0x6a,
+	0x1f, 0x09, 0xdc, 0xa2, 0x51, 0xd1, 0x93, 0x3a,
+	0x5f, 0x72, 0x1d, 0x26, 0x14, 0x62, 0xa2, 0x41,
+	0x3d, 0x08, 0x70, 0x7b, 0x27, 0x3d, 0xbc, 0xdf,
+	0x15, 0xfa, 0xb9, 0x5f, 0xb5, 0x38, 0x84, 0x0b,
+	0x58, 0x3d, 0xee, 0x3f, 0x32, 0x65, 0x6d, 0xd7,
+	0xce, 0x97, 0x3c, 0x8d, 0xfb, 0x63, 0xb9, 0xb0,
+	0xa8, 0x4a, 0x72, 0x99, 0x97, 0x58, 0xc8, 0xa7,
+	0xf9, 0x4c, 0xae, 0xc1, 0x63, 0xb9, 0x57, 0x18,
+	0x8a, 0xfa, 0xab, 0xe9, 0xf3, 0x67, 0xe6, 0xfd,
+	0xd2, 0x9d, 0x5c, 0xa9, 0x8e, 0x11, 0x0a, 0xf4,
+	0x4b, 0xf1, 0xec, 0x1a, 0xaf, 0x50, 0x5d, 0x16,
+	0x13, 0x69, 0x2e, 0xbd, 0x0d, 0xe6, 0xf0, 0xb2,
+	0xed, 0xb4, 0x4c, 0x59, 0x77, 0x37, 0x00, 0x0b,
+	0xc7, 0xa7, 0x9e, 0x37, 0xf3, 0x60, 0x70, 0xef,
+	0xf3, 0xc1, 0x74, 0x52, 0x87, 0xc6, 0xa1, 0x81,
+	0xbd, 0x0a, 0x2c, 0x5d, 0x2c, 0x0c, 0x6a, 0x81,
+	0xa1, 0xfe, 0x26, 0x78, 0x6c, 0x03, 0x06, 0x07,
+	0x34, 0xaa, 0xd1, 0x1b, 0x40, 0x03, 0x39, 0x56,
+	0xcf, 0x2a, 0x92, 0xc1, 0x4e, 0xdf, 0x29, 0x24,
+	0x83, 0x22, 0x7a, 0xea, 0x67, 0x1e, 0xe7, 0x54,
+	0x64, 0xd3, 0xbd, 0x3a, 0x5d, 0xae, 0xca, 0xf0,
+	0x9c, 0xd6, 0x5a, 0x9a, 0x62, 0xc8, 0xc7, 0x83,
+	0xf9, 0x89, 0xde, 0x2d, 0x53, 0x64, 0x61, 0xf7,
+	0xa3, 0xa7, 0x31, 0x38, 0xc6, 0x22, 0x9c, 0xb4,
+	0x87, 0xe0
+};
+static const u8 output73[] __initconst = {
+	0x34, 0xed, 0x05, 0xb0, 0x14, 0xbc, 0x8c, 0xcc,
+	0x95, 0xbd, 0x99, 0x0f, 0xb1, 0x98, 0x17, 0x10,
+	0xae, 0xe0, 0x08, 0x53, 0xa3, 0x69, 0xd2, 0xed,
+	0x66, 0xdb, 0x2a, 0x34, 0x8d, 0x0c, 0x6e, 0xce,
+	0x63, 0x69, 0xc9, 0xe4, 0x57, 0xc3, 0x0c, 0x8b,
+	0xa6, 0x2c, 0xa7, 0xd2, 0x08, 0xff, 0x4f, 0xec,
+	0x61, 0x8c, 0xee, 0x0d, 0xfa, 0x6b, 0xe0, 0xe8,
+	0x71, 0xbc, 0x41, 0x46, 0xd7, 0x33, 0x1d, 0xc0,
+	0xfd, 0xad, 0xca, 0x8b, 0x34, 0x56, 0xa4, 0x86,
+	0x71, 0x62, 0xae, 0x5e, 0x3d, 0x2b, 0x66, 0x3e,
+	0xae, 0xd8, 0xc0, 0xe1, 0x21, 0x3b, 0xca, 0xd2,
+	0x6b, 0xa2, 0xb8, 0xc7, 0x98, 0x4a, 0xf3, 0xcf,
+	0xb8, 0x62, 0xd8, 0x33, 0xe6, 0x80, 0xdb, 0x2f,
+	0x0a, 0xaf, 0x90, 0x3c, 0xe1, 0xec, 0xe9, 0x21,
+	0x29, 0x42, 0x9e, 0xa5, 0x50, 0xe9, 0x93, 0xd3,
+	0x53, 0x1f, 0xac, 0x2a, 0x24, 0x07, 0xb8, 0xed,
+	0xed, 0x38, 0x2c, 0xc4, 0xa1, 0x2b, 0x31, 0x5d,
+	0x9c, 0x24, 0x7b, 0xbf, 0xd9, 0xbb, 0x4e, 0x87,
+	0x8f, 0x32, 0x30, 0xf1, 0x11, 0x29, 0x54, 0x94,
+	0x00, 0x95, 0x1d, 0x1d, 0x24, 0xc0, 0xd4, 0x34,
+	0x49, 0x1d, 0xd5, 0xe3, 0xa6, 0xde, 0x8b, 0xbf,
+	0x5a, 0x9f, 0x58, 0x5a, 0x9b, 0x70, 0xe5, 0x9b,
+	0xb3, 0xdb, 0xe8, 0xb8, 0xca, 0x1b, 0x43, 0xe3,
+	0xc6, 0x6f, 0x0a, 0xd6, 0x32, 0x11, 0xd4, 0x04,
+	0xef, 0xa3, 0xe4, 0x3f, 0x12, 0xd8, 0xc1, 0x73,
+	0x51, 0x87, 0x03, 0xbd, 0xba, 0x60, 0x79, 0xee,
+	0x08, 0xcc, 0xf7, 0xc0, 0xaa, 0x4c, 0x33, 0xc4,
+	0xc7, 0x09, 0xf5, 0x91, 0xcb, 0x74, 0x57, 0x08,
+	0x1b, 0x90, 0xa9, 0x1b, 0x60, 0x02, 0xd2, 0x3f,
+	0x7a, 0xbb, 0xfd, 0x78, 0xf0, 0x15, 0xf9, 0x29,
+	0x82, 0x8f, 0xc4, 0xb2, 0x88, 0x1f, 0xbc, 0xcc,
+	0x53, 0x27, 0x8b, 0x07, 0x5f, 0xfc, 0x91, 0x29,
+	0x82, 0x80, 0x59, 0x0a, 0x3c, 0xea, 0xc4, 0x7e,
+	0xad, 0xd2, 0x70, 0x46, 0xbd, 0x9e, 0x3b, 0x1c,
+	0x8a, 0x62, 0xea, 0x69, 0xbd, 0xf6, 0x96, 0x15,
+	0xb5, 0x57, 0xe8, 0x63, 0x5f, 0x65, 0x46, 0x84,
+	0x58, 0x50, 0x87, 0x4b, 0x0e, 0x5b, 0x52, 0x90,
+	0xb0, 0xae, 0x37, 0x0f, 0xdd, 0x7e, 0xa2, 0xa0,
+	0x8b, 0x78, 0xc8, 0x5a, 0x1f, 0x53, 0xdb, 0xc5,
+	0xbf, 0x73, 0x20, 0xa9, 0x44, 0xfb, 0x1e, 0xc7,
+	0x97, 0xb2, 0x3a, 0x5a, 0x17, 0xe6, 0x8b, 0x9b,
+	0xe8, 0xf8, 0x2a, 0x01, 0x27, 0xa3, 0x71, 0x28,
+	0xe3, 0x19, 0xc6, 0xaf, 0xf5, 0x3a, 0x26, 0xc0,
+	0x5c, 0x69, 0x30, 0x78, 0x75, 0x27, 0xf2, 0x0c,
+	0x22, 0x71, 0x65, 0xc6, 0x8e, 0x7b, 0x47, 0xe3,
+	0x31, 0xaf, 0x7b, 0xc6, 0xc2, 0x55, 0x68, 0x81,
+	0xaa, 0x1b, 0x21, 0x65, 0xfb, 0x18, 0x35, 0x45,
+	0x36, 0x9a, 0x44, 0xba, 0x5c, 0xff, 0x06, 0xde,
+	0x3a, 0xc8, 0x44, 0x0b, 0xaa, 0x8e, 0x34, 0xe2,
+	0x84, 0xac, 0x18, 0xfe, 0x9b, 0xe1, 0x4f, 0xaa,
+	0xb6, 0x90, 0x0b, 0x1c, 0x2c, 0xd9, 0x9a, 0x10,
+	0x18, 0xf9, 0x49, 0x41, 0x42, 0x1b, 0xb5, 0xe1,
+	0x26, 0xac, 0x2d, 0x38, 0x00, 0x00, 0xe4, 0xb4,
+	0x50, 0x6f, 0x14, 0x18, 0xd6, 0x3d, 0x00, 0x59,
+	0x3c, 0x45, 0xf3, 0x42, 0x13, 0x44, 0xb8, 0x57,
+	0xd4, 0x43, 0x5c, 0x8a, 0x2a, 0xb4, 0xfc, 0x0a,
+	0x25, 0x5a, 0xdc, 0x8f, 0x11, 0x0b, 0x11, 0x44,
+	0xc7, 0x0e, 0x54, 0x8b, 0x22, 0x01, 0x7e, 0x67,
+	0x2e, 0x15, 0x3a, 0xb9, 0xee, 0x84, 0x10, 0xd4,
+	0x80, 0x57, 0xd7, 0x75, 0xcf, 0x8b, 0xcb, 0x03,
+	0xc9, 0x92, 0x2b, 0x69, 0xd8, 0x5a, 0x9b, 0x06,
+	0x85, 0x47, 0xaa, 0x4c, 0x28, 0xde, 0x49, 0x58,
+	0xe6, 0x11, 0x1e, 0x5e, 0x64, 0x8e, 0x3b, 0xe0,
+	0x40, 0x2e, 0xac, 0x96, 0x97, 0x15, 0x37, 0x1e,
+	0x30, 0xdd
+};
+static const u8 key73[] __initconst = {
+	0x96, 0x06, 0x1e, 0xc1, 0x6d, 0xba, 0x49, 0x5b,
+	0x65, 0x80, 0x79, 0xdd, 0xf3, 0x67, 0xa8, 0x6e,
+	0x2d, 0x9c, 0x54, 0x46, 0xd8, 0x4a, 0xeb, 0x7e,
+	0x23, 0x86, 0x51, 0xd8, 0x49, 0x49, 0x56, 0xe0
+};
+enum { nonce73 = 0xbefb83cb67e11ffdULL };
+
+static const u8 input74[] __initconst = {
+	0x47, 0x22, 0x70, 0xe5, 0x2f, 0x41, 0x18, 0x45,
+	0x07, 0xd3, 0x6d, 0x32, 0x0d, 0x43, 0x92, 0x2b,
+	0x9b, 0x65, 0x73, 0x13, 0x1a, 0x4f, 0x49, 0x8f,
+	0xff, 0xf8, 0xcc, 0xae, 0x15, 0xab, 0x9d, 0x7d,
+	0xee, 0x22, 0x5d, 0x8b, 0xde, 0x81, 0x5b, 0x81,
+	0x83, 0x49, 0x35, 0x9b, 0xb4, 0xbc, 0x4e, 0x01,
+	0xc2, 0x29, 0xa7, 0xf1, 0xca, 0x3a, 0xce, 0x3f,
+	0xf5, 0x31, 0x93, 0xa8, 0xe2, 0xc9, 0x7d, 0x03,
+	0x26, 0xa4, 0xbc, 0xa8, 0x9c, 0xb9, 0x68, 0xf3,
+	0xb3, 0x91, 0xe8, 0xe6, 0xc7, 0x2b, 0x1a, 0xce,
+	0xd2, 0x41, 0x53, 0xbd, 0xa3, 0x2c, 0x54, 0x94,
+	0x21, 0xa1, 0x40, 0xae, 0xc9, 0x0c, 0x11, 0x92,
+	0xfd, 0x91, 0xa9, 0x40, 0xca, 0xde, 0x21, 0x4e,
+	0x1e, 0x3d, 0xcc, 0x2c, 0x87, 0x11, 0xef, 0x46,
+	0xed, 0x52, 0x03, 0x11, 0x19, 0x43, 0x25, 0xc7,
+	0x0d, 0xc3, 0x37, 0x5f, 0xd3, 0x6f, 0x0c, 0x6a,
+	0x45, 0x30, 0x88, 0xec, 0xf0, 0x21, 0xef, 0x1d,
+	0x7b, 0x38, 0x63, 0x4b, 0x49, 0x0c, 0x72, 0xf6,
+	0x4c, 0x40, 0xc3, 0xcc, 0x03, 0xa7, 0xae, 0xa8,
+	0x8c, 0x37, 0x03, 0x1c, 0x11, 0xae, 0x0d, 0x1b,
+	0x62, 0x97, 0x27, 0xfc, 0x56, 0x4b, 0xb7, 0xfd,
+	0xbc, 0xfb, 0x0e, 0xfc, 0x61, 0xad, 0xc6, 0xb5,
+	0x9c, 0x8c, 0xc6, 0x38, 0x27, 0x91, 0x29, 0x3d,
+	0x29, 0xc8, 0x37, 0xc9, 0x96, 0x69, 0xe3, 0xdc,
+	0x3e, 0x61, 0x35, 0x9b, 0x99, 0x4f, 0xb9, 0x4e,
+	0x5a, 0x29, 0x1c, 0x2e, 0xcf, 0x16, 0xcb, 0x69,
+	0x87, 0xe4, 0x1a, 0xc4, 0x6e, 0x78, 0x43, 0x00,
+	0x03, 0xb2, 0x8b, 0x03, 0xd0, 0xb4, 0xf1, 0xd2,
+	0x7d, 0x2d, 0x7e, 0xfc, 0x19, 0x66, 0x5b, 0xa3,
+	0x60, 0x3f, 0x9d, 0xbd, 0xfa, 0x3e, 0xca, 0x7b,
+	0x26, 0x08, 0x19, 0x16, 0x93, 0x5d, 0x83, 0xfd,
+	0xf9, 0x21, 0xc6, 0x31, 0x34, 0x6f, 0x0c, 0xaa,
+	0x28, 0xf9, 0x18, 0xa2, 0xc4, 0x78, 0x3b, 0x56,
+	0xc0, 0x88, 0x16, 0xba, 0x22, 0x2c, 0x07, 0x2f,
+	0x70, 0xd0, 0xb0, 0x46, 0x35, 0xc7, 0x14, 0xdc,
+	0xbb, 0x56, 0x23, 0x1e, 0x36, 0x36, 0x2d, 0x73,
+	0x78, 0xc7, 0xce, 0xf3, 0x58, 0xf7, 0x58, 0xb5,
+	0x51, 0xff, 0x33, 0x86, 0x0e, 0x3b, 0x39, 0xfb,
+	0x1a, 0xfd, 0xf8, 0x8b, 0x09, 0x33, 0x1b, 0x83,
+	0xf2, 0xe6, 0x38, 0x37, 0xef, 0x47, 0x84, 0xd9,
+	0x82, 0x77, 0x2b, 0x82, 0xcc, 0xf9, 0xee, 0x94,
+	0x71, 0x78, 0x81, 0xc8, 0x4d, 0x91, 0xd7, 0x35,
+	0x29, 0x31, 0x30, 0x5c, 0x4a, 0x23, 0x23, 0xb1,
+	0x38, 0x6b, 0xac, 0x22, 0x3f, 0x80, 0xc7, 0xe0,
+	0x7d, 0xfa, 0x76, 0x47, 0xd4, 0x6f, 0x93, 0xa0,
+	0xa0, 0x93, 0x5d, 0x68, 0xf7, 0x43, 0x25, 0x8f,
+	0x1b, 0xc7, 0x87, 0xea, 0x59, 0x0c, 0xa2, 0xfa,
+	0xdb, 0x2f, 0x72, 0x43, 0xcf, 0x90, 0xf1, 0xd6,
+	0x58, 0xf3, 0x17, 0x6a, 0xdf, 0xb3, 0x4e, 0x0e,
+	0x38, 0x24, 0x48, 0x1f, 0xb7, 0x01, 0xec, 0x81,
+	0xb1, 0x87, 0x5b, 0xec, 0x9c, 0x11, 0x1a, 0xff,
+	0xa5, 0xca, 0x5a, 0x63, 0x31, 0xb2, 0xe4, 0xc6,
+	0x3c, 0x1d, 0xaf, 0x27, 0xb2, 0xd4, 0x19, 0xa2,
+	0xcc, 0x04, 0x92, 0x42, 0xd2, 0xc1, 0x8c, 0x3b,
+	0xce, 0xf5, 0x74, 0xc1, 0x81, 0xf8, 0x20, 0x23,
+	0x6f, 0x20, 0x6d, 0x78, 0x36, 0x72, 0x2c, 0x52,
+	0xdf, 0x5e, 0xe8, 0x75, 0xce, 0x1c, 0x49, 0x9d,
+	0x93, 0x6f, 0x65, 0xeb, 0xb1, 0xbd, 0x8e, 0x5e,
+	0xe5, 0x89, 0xc4, 0x8a, 0x81, 0x3d, 0x9a, 0xa7,
+	0x11, 0x82, 0x8e, 0x38, 0x5b, 0x5b, 0xca, 0x7d,
+	0x4b, 0x72, 0xc2, 0x9c, 0x30, 0x5e, 0x7f, 0xc0,
+	0x6f, 0x91, 0xd5, 0x67, 0x8c, 0x3e, 0xae, 0xda,
+	0x2b, 0x3c, 0x53, 0xcc, 0x50, 0x97, 0x36, 0x0b,
+	0x79, 0xd6, 0x73, 0x6e, 0x7d, 0x42, 0x56, 0xe1,
+	0xaa, 0xfc, 0xb3, 0xa7, 0xc8, 0x01, 0xaa, 0xc1,
+	0xfc, 0x5c, 0x72, 0x8e, 0x63, 0xa8, 0x46, 0x18,
+	0xee, 0x11, 0xe7, 0x30, 0x09, 0x83, 0x6c, 0xd9,
+	0xf4, 0x7a, 0x7b, 0xb5, 0x1f, 0x6d, 0xc7, 0xbc,
+	0xcb, 0x55, 0xea, 0x40, 0x58, 0x7a, 0x00, 0x00,
+	0x90, 0x60, 0xc5, 0x64, 0x69, 0x05, 0x99, 0xd2,
+	0x49, 0x62, 0x4f, 0xcb, 0x97, 0xdf, 0xdd, 0x6b,
+	0x60, 0x75, 0xe2, 0xe0, 0x6f, 0x76, 0xd0, 0x37,
+	0x67, 0x0a, 0xcf, 0xff, 0xc8, 0x61, 0x84, 0x14,
+	0x80, 0x7c, 0x1d, 0x31, 0x8d, 0x90, 0xde, 0x0b,
+	0x1c, 0x74, 0x9f, 0x82, 0x96, 0x80, 0xda, 0xaf,
+	0x8d, 0x99, 0x86, 0x9f, 0x24, 0x99, 0x28, 0x3e,
+	0xe0, 0xa3, 0xc3, 0x90, 0x2d, 0x14, 0x65, 0x1e,
+	0x3b, 0xb9, 0xba, 0x13, 0xa5, 0x77, 0x73, 0x63,
+	0x9a, 0x06, 0x3d, 0xa9, 0x28, 0x9b, 0xba, 0x25,
+	0x61, 0xc9, 0xcd, 0xcf, 0x7a, 0x4d, 0x96, 0x09,
+	0xcb, 0xca, 0x03, 0x9c, 0x54, 0x34, 0x31, 0x85,
+	0xa0, 0x3d, 0xe5, 0xbc, 0xa5, 0x5f, 0x1b, 0xd3,
+	0x10, 0x63, 0x74, 0x9d, 0x01, 0x92, 0x88, 0xf0,
+	0x27, 0x9c, 0x28, 0xd9, 0xfd, 0xe2, 0x4e, 0x01,
+	0x8d, 0x61, 0x79, 0x60, 0x61, 0x5b, 0x76, 0xab,
+	0x06, 0xd3, 0x44, 0x87, 0x43, 0x52, 0xcd, 0x06,
+	0x68, 0x1e, 0x2d, 0xc5, 0xb0, 0x07, 0x25, 0xdf,
+	0x0a, 0x50, 0xd7, 0xd9, 0x08, 0x53, 0x65, 0xf1,
+	0x0c, 0x2c, 0xde, 0x3f, 0x9d, 0x03, 0x1f, 0xe1,
+	0x49, 0x43, 0x3c, 0x83, 0x81, 0x37, 0xf8, 0xa2,
+	0x0b, 0xf9, 0x61, 0x1c, 0xc1, 0xdb, 0x79, 0xbc,
+	0x64, 0xce, 0x06, 0x4e, 0x87, 0x89, 0x62, 0x73,
+	0x51, 0xbc, 0xa4, 0x32, 0xd4, 0x18, 0x62, 0xab,
+	0x65, 0x7e, 0xad, 0x1e, 0x91, 0xa3, 0xfa, 0x2d,
+	0x58, 0x9e, 0x2a, 0xe9, 0x74, 0x44, 0x64, 0x11,
+	0xe6, 0xb6, 0xb3, 0x00, 0x7e, 0xa3, 0x16, 0xef,
+	0x72
+};
+static const u8 output74[] __initconst = {
+	0xf5, 0xca, 0x45, 0x65, 0x50, 0x35, 0x47, 0x67,
+	0x6f, 0x4f, 0x67, 0xff, 0x34, 0xd9, 0xc3, 0x37,
+	0x2a, 0x26, 0xb0, 0x4f, 0x08, 0x1e, 0x45, 0x13,
+	0xc7, 0x2c, 0x14, 0x75, 0x33, 0xd8, 0x8e, 0x1e,
+	0x1b, 0x11, 0x0d, 0x97, 0x04, 0x33, 0x8a, 0xe4,
+	0xd8, 0x8d, 0x0e, 0x12, 0x8d, 0xdb, 0x6e, 0x02,
+	0xfa, 0xe5, 0xbd, 0x3a, 0xb5, 0x28, 0x07, 0x7d,
+	0x20, 0xf0, 0x12, 0x64, 0x83, 0x2f, 0x59, 0x79,
+	0x17, 0x88, 0x3c, 0x2d, 0x08, 0x2f, 0x55, 0xda,
+	0xcc, 0x02, 0x3a, 0x82, 0xcd, 0x03, 0x94, 0xdf,
+	0xdf, 0xab, 0x8a, 0x13, 0xf5, 0xe6, 0x74, 0xdf,
+	0x7b, 0xe2, 0xab, 0x34, 0xbc, 0x00, 0x85, 0xbf,
+	0x5a, 0x48, 0xc8, 0xff, 0x8d, 0x6c, 0x27, 0x48,
+	0x19, 0x2d, 0x08, 0xfa, 0x82, 0x62, 0x39, 0x55,
+	0x32, 0x11, 0xa8, 0xd7, 0xb9, 0x08, 0x2c, 0xd6,
+	0x7a, 0xd9, 0x83, 0x9f, 0x9b, 0xfb, 0xec, 0x3a,
+	0xd1, 0x08, 0xc7, 0xad, 0xdc, 0x98, 0x4c, 0xbc,
+	0x98, 0xeb, 0x36, 0xb0, 0x39, 0xf4, 0x3a, 0xd6,
+	0x53, 0x02, 0xa0, 0xa9, 0x73, 0xa1, 0xca, 0xef,
+	0xd8, 0xd2, 0xec, 0x0e, 0xf8, 0xf5, 0xac, 0x8d,
+	0x34, 0x41, 0x06, 0xa8, 0xc6, 0xc3, 0x31, 0xbc,
+	0xe5, 0xcc, 0x7e, 0x72, 0x63, 0x59, 0x3e, 0x63,
+	0xc2, 0x8d, 0x2b, 0xd5, 0xb9, 0xfd, 0x1e, 0x31,
+	0x69, 0x32, 0x05, 0xd6, 0xde, 0xc9, 0xe6, 0x4c,
+	0xac, 0x68, 0xf7, 0x1f, 0x9d, 0xcd, 0x0e, 0xa2,
+	0x15, 0x3d, 0xd6, 0x47, 0x99, 0xab, 0x08, 0x5f,
+	0x28, 0xc3, 0x4c, 0xc2, 0xd5, 0xdd, 0x10, 0xb7,
+	0xbd, 0xdb, 0x9b, 0xcf, 0x85, 0x27, 0x29, 0x76,
+	0x98, 0xeb, 0xad, 0x31, 0x64, 0xe7, 0xfb, 0x61,
+	0xe0, 0xd8, 0x1a, 0xa6, 0xe2, 0xe7, 0x43, 0x42,
+	0x77, 0xc9, 0x82, 0x00, 0xac, 0x85, 0xe0, 0xa2,
+	0xd4, 0x62, 0xe3, 0xb7, 0x17, 0x6e, 0xb2, 0x9e,
+	0x21, 0x58, 0x73, 0xa9, 0x53, 0x2d, 0x3c, 0xe1,
+	0xdd, 0xd6, 0x6e, 0x92, 0xf2, 0x1d, 0xc2, 0x22,
+	0x5f, 0x9a, 0x7e, 0xd0, 0x52, 0xbf, 0x54, 0x19,
+	0xd7, 0x80, 0x63, 0x3e, 0xd0, 0x08, 0x2d, 0x37,
+	0x0c, 0x15, 0xf7, 0xde, 0xab, 0x2b, 0xe3, 0x16,
+	0x21, 0x3a, 0xee, 0xa5, 0xdc, 0xdf, 0xde, 0xa3,
+	0x69, 0xcb, 0xfd, 0x92, 0x89, 0x75, 0xcf, 0xc9,
+	0x8a, 0xa4, 0xc8, 0xdd, 0xcc, 0x21, 0xe6, 0xfe,
+	0x9e, 0x43, 0x76, 0xb2, 0x45, 0x22, 0xb9, 0xb5,
+	0xac, 0x7e, 0x3d, 0x26, 0xb0, 0x53, 0xc8, 0xab,
+	0xfd, 0xea, 0x2c, 0xd1, 0x44, 0xc5, 0x60, 0x1b,
+	0x8a, 0x99, 0x0d, 0xa5, 0x0e, 0x67, 0x6e, 0x3a,
+	0x96, 0x55, 0xec, 0xe8, 0xcc, 0xbe, 0x49, 0xd9,
+	0xf2, 0x72, 0x9f, 0x30, 0x21, 0x97, 0x57, 0x19,
+	0xbe, 0x5e, 0x33, 0x0c, 0xee, 0xc0, 0x72, 0x0d,
+	0x2e, 0xd1, 0xe1, 0x52, 0xc2, 0xea, 0x41, 0xbb,
+	0xe1, 0x6d, 0xd4, 0x17, 0xa9, 0x8d, 0x89, 0xa9,
+	0xd6, 0x4b, 0xc6, 0x4c, 0xf2, 0x88, 0x97, 0x54,
+	0x3f, 0x4f, 0x57, 0xb7, 0x37, 0xf0, 0x2c, 0x11,
+	0x15, 0x56, 0xdb, 0x28, 0xb5, 0x16, 0x84, 0x66,
+	0xce, 0x45, 0x3f, 0x61, 0x75, 0xb6, 0xbe, 0x00,
+	0xd1, 0xe4, 0xf5, 0x27, 0x54, 0x7f, 0xc2, 0xf1,
+	0xb3, 0x32, 0x9a, 0xe8, 0x07, 0x02, 0xf3, 0xdb,
+	0xa9, 0xd1, 0xc2, 0xdf, 0xee, 0xad, 0xe5, 0x8a,
+	0x3c, 0xfa, 0x67, 0xec, 0x6b, 0xa4, 0x08, 0xfe,
+	0xba, 0x5a, 0x58, 0x0b, 0x78, 0x11, 0x91, 0x76,
+	0xe3, 0x1a, 0x28, 0x54, 0x5e, 0xbd, 0x71, 0x1b,
+	0x8b, 0xdc, 0x6c, 0xf4, 0x6f, 0xd7, 0xf4, 0xf3,
+	0xe1, 0x03, 0xa4, 0x3c, 0x8d, 0x91, 0x2e, 0xba,
+	0x5f, 0x7f, 0x8c, 0xaf, 0x69, 0x89, 0x29, 0x0a,
+	0x5b, 0x25, 0x13, 0xc4, 0x2e, 0x16, 0xc2, 0x15,
+	0x07, 0x5d, 0x58, 0x33, 0x7c, 0xe0, 0xf0, 0x55,
+	0x5f, 0xbf, 0x5e, 0xf0, 0x71, 0x48, 0x8f, 0xf7,
+	0x48, 0xb3, 0xf7, 0x0d, 0xa1, 0xd0, 0x63, 0xb1,
+	0xad, 0xae, 0xb5, 0xb0, 0x5f, 0x71, 0xaf, 0x24,
+	0x8b, 0xb9, 0x1c, 0x44, 0xd2, 0x1a, 0x53, 0xd1,
+	0xd5, 0xb4, 0xa9, 0xff, 0x88, 0x73, 0xb5, 0xaa,
+	0x15, 0x32, 0x5f, 0x59, 0x9d, 0x2e, 0xb5, 0xcb,
+	0xde, 0x21, 0x2e, 0xe9, 0x35, 0xed, 0xfd, 0x0f,
+	0xb6, 0xbb, 0xe6, 0x4b, 0x16, 0xf1, 0x45, 0x1e,
+	0xb4, 0x84, 0xe9, 0x58, 0x1c, 0x0c, 0x95, 0xc0,
+	0xcf, 0x49, 0x8b, 0x59, 0xa1, 0x78, 0xe6, 0x80,
+	0x12, 0x49, 0x7a, 0xd4, 0x66, 0x62, 0xdf, 0x9c,
+	0x18, 0xc8, 0x8c, 0xda, 0xc1, 0xa6, 0xbc, 0x65,
+	0x28, 0xd2, 0xa4, 0xe8, 0xf1, 0x35, 0xdb, 0x5a,
+	0x75, 0x1f, 0x73, 0x60, 0xec, 0xa8, 0xda, 0x5a,
+	0x43, 0x15, 0x83, 0x9b, 0xe7, 0xb1, 0xa6, 0x81,
+	0xbb, 0xef, 0xf3, 0x8f, 0x0f, 0xd3, 0x79, 0xa2,
+	0xe5, 0xaa, 0x42, 0xef, 0xa0, 0x13, 0x4e, 0x91,
+	0x2d, 0xcb, 0x61, 0x7a, 0x9a, 0x33, 0x14, 0x50,
+	0x77, 0x4a, 0xd0, 0x91, 0x48, 0xe0, 0x0c, 0xe0,
+	0x11, 0xcb, 0xdf, 0xb0, 0xce, 0x06, 0xd2, 0x79,
+	0x4d, 0x69, 0xb9, 0xc9, 0x36, 0x74, 0x8f, 0x81,
+	0x72, 0x73, 0xf3, 0x17, 0xb7, 0x13, 0xcb, 0x5b,
+	0xd2, 0x5c, 0x33, 0x61, 0xb7, 0x61, 0x79, 0xb0,
+	0xc0, 0x4d, 0xa1, 0xc7, 0x5d, 0x98, 0xc9, 0xe1,
+	0x98, 0xbd, 0x78, 0x5a, 0x2c, 0x64, 0x53, 0xaf,
+	0xaf, 0x66, 0x51, 0x47, 0xe4, 0x48, 0x66, 0x8b,
+	0x07, 0x52, 0xa3, 0x03, 0x93, 0x28, 0xad, 0xcc,
+	0xa3, 0x86, 0xad, 0x63, 0x04, 0x35, 0x6c, 0x49,
+	0xd5, 0x28, 0x0e, 0x00, 0x47, 0xf4, 0xd4, 0x32,
+	0x27, 0x19, 0xb3, 0x29, 0xe7, 0xbc, 0xbb, 0xce,
+	0x3e, 0x3e, 0xd5, 0x67, 0x20, 0xe4, 0x0b, 0x75,
+	0x95, 0x24, 0xe0, 0x6c, 0xb6, 0x29, 0x0c, 0x14,
+	0xfd
+};
+static const u8 key74[] __initconst = {
+	0xf0, 0x41, 0x5b, 0x00, 0x56, 0xc4, 0xac, 0xf6,
+	0xa2, 0x4c, 0x33, 0x41, 0x16, 0x09, 0x1b, 0x8e,
+	0x4d, 0xe8, 0x8c, 0xd9, 0x48, 0xab, 0x3e, 0x60,
+	0xcb, 0x49, 0x3e, 0xaf, 0x2b, 0x8b, 0xc8, 0xf0
+};
+enum { nonce74 = 0xcbdb0ffd0e923384ULL };
+
+static const struct chacha20_testvec chacha20_testvecs[] __initconst = {
+	{ input01, output01, key01, nonce01, sizeof(input01) },
+	{ input02, output02, key02, nonce02, sizeof(input02) },
+	{ input03, output03, key03, nonce03, sizeof(input03) },
+	{ input04, output04, key04, nonce04, sizeof(input04) },
+	{ input05, output05, key05, nonce05, sizeof(input05) },
+	{ input06, output06, key06, nonce06, sizeof(input06) },
+	{ input07, output07, key07, nonce07, sizeof(input07) },
+	{ input08, output08, key08, nonce08, sizeof(input08) },
+	{ input09, output09, key09, nonce09, sizeof(input09) },
+	{ input10, output10, key10, nonce10, sizeof(input10) },
+	{ input11, output11, key11, nonce11, sizeof(input11) },
+	{ input12, output12, key12, nonce12, sizeof(input12) },
+	{ input13, output13, key13, nonce13, sizeof(input13) },
+	{ input14, output14, key14, nonce14, sizeof(input14) },
+	{ input15, output15, key15, nonce15, sizeof(input15) },
+	{ input16, output16, key16, nonce16, sizeof(input16) },
+	{ input17, output17, key17, nonce17, sizeof(input17) },
+	{ input18, output18, key18, nonce18, sizeof(input18) },
+	{ input19, output19, key19, nonce19, sizeof(input19) },
+	{ input20, output20, key20, nonce20, sizeof(input20) },
+	{ input21, output21, key21, nonce21, sizeof(input21) },
+	{ input22, output22, key22, nonce22, sizeof(input22) },
+	{ input23, output23, key23, nonce23, sizeof(input23) },
+	{ input24, output24, key24, nonce24, sizeof(input24) },
+	{ input25, output25, key25, nonce25, sizeof(input25) },
+	{ input26, output26, key26, nonce26, sizeof(input26) },
+	{ input27, output27, key27, nonce27, sizeof(input27) },
+	{ input28, output28, key28, nonce28, sizeof(input28) },
+	{ input29, output29, key29, nonce29, sizeof(input29) },
+	{ input30, output30, key30, nonce30, sizeof(input30) },
+	{ input31, output31, key31, nonce31, sizeof(input31) },
+	{ input32, output32, key32, nonce32, sizeof(input32) },
+	{ input33, output33, key33, nonce33, sizeof(input33) },
+	{ input34, output34, key34, nonce34, sizeof(input34) },
+	{ input35, output35, key35, nonce35, sizeof(input35) },
+	{ input36, output36, key36, nonce36, sizeof(input36) },
+	{ input37, output37, key37, nonce37, sizeof(input37) },
+	{ input38, output38, key38, nonce38, sizeof(input38) },
+	{ input39, output39, key39, nonce39, sizeof(input39) },
+	{ input40, output40, key40, nonce40, sizeof(input40) },
+	{ input41, output41, key41, nonce41, sizeof(input41) },
+	{ input42, output42, key42, nonce42, sizeof(input42) },
+	{ input43, output43, key43, nonce43, sizeof(input43) },
+	{ input44, output44, key44, nonce44, sizeof(input44) },
+	{ input45, output45, key45, nonce45, sizeof(input45) },
+	{ input46, output46, key46, nonce46, sizeof(input46) },
+	{ input47, output47, key47, nonce47, sizeof(input47) },
+	{ input48, output48, key48, nonce48, sizeof(input48) },
+	{ input49, output49, key49, nonce49, sizeof(input49) },
+	{ input50, output50, key50, nonce50, sizeof(input50) },
+	{ input51, output51, key51, nonce51, sizeof(input51) },
+	{ input52, output52, key52, nonce52, sizeof(input52) },
+	{ input53, output53, key53, nonce53, sizeof(input53) },
+	{ input54, output54, key54, nonce54, sizeof(input54) },
+	{ input55, output55, key55, nonce55, sizeof(input55) },
+	{ input56, output56, key56, nonce56, sizeof(input56) },
+	{ input57, output57, key57, nonce57, sizeof(input57) },
+	{ input58, output58, key58, nonce58, sizeof(input58) },
+	{ input59, output59, key59, nonce59, sizeof(input59) },
+	{ input60, output60, key60, nonce60, sizeof(input60) },
+	{ input61, output61, key61, nonce61, sizeof(input61) },
+	{ input62, output62, key62, nonce62, sizeof(input62) },
+	{ input63, output63, key63, nonce63, sizeof(input63) },
+	{ input64, output64, key64, nonce64, sizeof(input64) },
+	{ input65, output65, key65, nonce65, sizeof(input65) },
+	{ input66, output66, key66, nonce66, sizeof(input66) },
+	{ input67, output67, key67, nonce67, sizeof(input67) },
+	{ input68, output68, key68, nonce68, sizeof(input68) },
+	{ input69, output69, key69, nonce69, sizeof(input69) },
+	{ input70, output70, key70, nonce70, sizeof(input70) },
+	{ input71, output71, key71, nonce71, sizeof(input71) },
+	{ input72, output72, key72, nonce72, sizeof(input72) },
+	{ input73, output73, key73, nonce73, sizeof(input73) },
+	{ input74, output74, key74, nonce74, sizeof(input74) }
+};
+
+static const struct hchacha20_testvec hchacha20_testvecs[] __initconst = {{
+	.key	= { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+		    0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+		    0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+		    0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f },
+	.nonce	= { 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x4a,
+		    0x00, 0x00, 0x00, 0x00, 0x31, 0x41, 0x59, 0x27 },
+	.output	= { 0x82, 0x41, 0x3b, 0x42, 0x27, 0xb2, 0x7b, 0xfe,
+		    0xd3, 0x0e, 0x42, 0x50, 0x8a, 0x87, 0x7d, 0x73,
+		    0xa0, 0xf9, 0xe4, 0xd5, 0x8a, 0x74, 0xa8, 0x53,
+		    0xc1, 0x2e, 0xc4, 0x13, 0x26, 0xd3, 0xec, 0xdc }
+}};
+
+static bool __init chacha20_selftest(void)
+{
+	enum {
+		MAXIMUM_TEST_BUFFER_LEN = 1UL << 10,
+		OUTRAGEOUSLY_HUGE_BUFFER_LEN = PAGE_SIZE * 35 + 17 /* 143k */
+	};
+	size_t i, j, k;
+	u32 derived_key[CHACHA20_KEY_WORDS];
+	u8 *offset_input = NULL, *computed_output = NULL, *massive_input = NULL;
+	u8 offset_key[CHACHA20_KEY_SIZE + 1]
+			__aligned(__alignof__(unsigned long));
+	struct chacha20_ctx state;
+	bool success = true;
+	simd_context_t simd_context;
+
+	offset_input = kmalloc(MAXIMUM_TEST_BUFFER_LEN + 1, GFP_KERNEL);
+	computed_output = kmalloc(MAXIMUM_TEST_BUFFER_LEN + 1, GFP_KERNEL);
+	massive_input = vzalloc(OUTRAGEOUSLY_HUGE_BUFFER_LEN);
+	if (!computed_output || !offset_input || !massive_input) {
+		pr_err("chacha20 self-test malloc: FAIL\n");
+		success = false;
+		goto out;
+	}
+
+	simd_get(&simd_context);
+	for (i = 0; i < ARRAY_SIZE(chacha20_testvecs); ++i) {
+		/* Boring case */
+		memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN + 1);
+		memset(&state, 0, sizeof(state));
+		chacha20_init(&state, chacha20_testvecs[i].key,
+			      chacha20_testvecs[i].nonce);
+		chacha20(&state, computed_output, chacha20_testvecs[i].input,
+			 chacha20_testvecs[i].ilen, &simd_context);
+		if (memcmp(computed_output, chacha20_testvecs[i].output,
+			   chacha20_testvecs[i].ilen)) {
+			pr_err("chacha20 self-test %zu: FAIL\n", i + 1);
+			success = false;
+		}
+		for (k = chacha20_testvecs[i].ilen;
+		     k < MAXIMUM_TEST_BUFFER_LEN + 1; ++k) {
+			if (computed_output[k]) {
+				pr_err("chacha20 self-test %zu (zero check): FAIL\n",
+				       i + 1);
+				success = false;
+				break;
+			}
+		}
+
+		/* Unaligned case */
+		memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN + 1);
+		memset(&state, 0, sizeof(state));
+		memcpy(offset_input + 1, chacha20_testvecs[i].input,
+		       chacha20_testvecs[i].ilen);
+		memcpy(offset_key + 1, chacha20_testvecs[i].key,
+		       CHACHA20_KEY_SIZE);
+		chacha20_init(&state, offset_key + 1, chacha20_testvecs[i].nonce);
+		chacha20(&state, computed_output + 1, offset_input + 1,
+			 chacha20_testvecs[i].ilen, &simd_context);
+		if (memcmp(computed_output + 1, chacha20_testvecs[i].output,
+			   chacha20_testvecs[i].ilen)) {
+			pr_err("chacha20 self-test %zu (unaligned): FAIL\n",
+			       i + 1);
+			success = false;
+		}
+		if (computed_output[0]) {
+			pr_err("chacha20 self-test %zu (unaligned, zero check): FAIL\n",
+			       i + 1);
+			success = false;
+		}
+		for (k = chacha20_testvecs[i].ilen + 1;
+		     k < MAXIMUM_TEST_BUFFER_LEN + 1; ++k) {
+			if (computed_output[k]) {
+				pr_err("chacha20 self-test %zu (unaligned, zero check): FAIL\n",
+				       i + 1);
+				success = false;
+				break;
+			}
+		}
+
+		/* Chunked case */
+		if (chacha20_testvecs[i].ilen <= CHACHA20_BLOCK_SIZE)
+			goto next_test;
+		memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN + 1);
+		memset(&state, 0, sizeof(state));
+		chacha20_init(&state, chacha20_testvecs[i].key,
+			      chacha20_testvecs[i].nonce);
+		chacha20(&state, computed_output, chacha20_testvecs[i].input,
+			 CHACHA20_BLOCK_SIZE, &simd_context);
+		chacha20(&state, computed_output + CHACHA20_BLOCK_SIZE,
+			 chacha20_testvecs[i].input + CHACHA20_BLOCK_SIZE,
+			 chacha20_testvecs[i].ilen - CHACHA20_BLOCK_SIZE,
+			 &simd_context);
+		if (memcmp(computed_output, chacha20_testvecs[i].output,
+			   chacha20_testvecs[i].ilen)) {
+			pr_err("chacha20 self-test %zu (chunked): FAIL\n",
+			       i + 1);
+			success = false;
+		}
+		for (k = chacha20_testvecs[i].ilen;
+		     k < MAXIMUM_TEST_BUFFER_LEN + 1; ++k) {
+			if (computed_output[k]) {
+				pr_err("chacha20 self-test %zu (chunked, zero check): FAIL\n",
+				       i + 1);
+				success = false;
+				break;
+			}
+		}
+
+next_test:
+		/* Sliding unaligned case */
+		if (chacha20_testvecs[i].ilen > CHACHA20_BLOCK_SIZE + 1 ||
+		    !chacha20_testvecs[i].ilen)
+			continue;
+		for (j = 1; j < CHACHA20_BLOCK_SIZE; ++j) {
+			memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN + 1);
+			memset(&state, 0, sizeof(state));
+			memcpy(offset_input + j, chacha20_testvecs[i].input,
+			       chacha20_testvecs[i].ilen);
+			chacha20_init(&state, chacha20_testvecs[i].key,
+				      chacha20_testvecs[i].nonce);
+			chacha20(&state, computed_output + j, offset_input + j,
+				 chacha20_testvecs[i].ilen, &simd_context);
+			if (memcmp(computed_output + j,
+				   chacha20_testvecs[i].output,
+				   chacha20_testvecs[i].ilen)) {
+				pr_err("chacha20 self-test %zu (unaligned, slide %zu): FAIL\n",
+				       i + 1, j);
+				success = false;
+			}
+			for (k = j; k < j; ++k) {
+				if (computed_output[k]) {
+					pr_err("chacha20 self-test %zu (unaligned, slide %zu, zero check): FAIL\n",
+					       i + 1, j);
+					success = false;
+					break;
+				}
+			}
+			for (k = chacha20_testvecs[i].ilen + j;
+			     k < MAXIMUM_TEST_BUFFER_LEN + 1; ++k) {
+				if (computed_output[k]) {
+					pr_err("chacha20 self-test %zu (unaligned, slide %zu, zero check): FAIL\n",
+					       i + 1, j);
+					success = false;
+					break;
+				}
+			}
+		}
+	}
+	for (i = 0; i < ARRAY_SIZE(hchacha20_testvecs); ++i) {
+		memset(&derived_key, 0, sizeof(derived_key));
+		hchacha20(derived_key, hchacha20_testvecs[i].nonce,
+			  hchacha20_testvecs[i].key, &simd_context);
+		cpu_to_le32_array(derived_key, ARRAY_SIZE(derived_key));
+		if (memcmp(derived_key, hchacha20_testvecs[i].output,
+			   CHACHA20_KEY_SIZE)) {
+			pr_err("hchacha20 self-test %zu: FAIL\n", i + 1);
+			success = false;
+		}
+	}
+	memset(&state, 0, sizeof(state));
+	chacha20_init(&state, chacha20_testvecs[0].key,
+		      chacha20_testvecs[0].nonce);
+	chacha20(&state, massive_input, massive_input,
+		 OUTRAGEOUSLY_HUGE_BUFFER_LEN, &simd_context);
+	chacha20_init(&state, chacha20_testvecs[0].key,
+		      chacha20_testvecs[0].nonce);
+	chacha20(&state, massive_input, massive_input,
+		 OUTRAGEOUSLY_HUGE_BUFFER_LEN, DONT_USE_SIMD);
+	for (k = 0; k < OUTRAGEOUSLY_HUGE_BUFFER_LEN; ++k) {
+		if (massive_input[k]) {
+			pr_err("chacha20 self-test massive: FAIL\n");
+			success = false;
+			break;
+		}
+	}
+
+	simd_put(&simd_context);
+
+out:
+	kfree(offset_input);
+	kfree(computed_output);
+	vfree(massive_input);
+	return success;
+}
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/selftest/chacha20poly1305.c b/net/wireguard/crypto/zinc/selftest/chacha20poly1305.c
--- a/net/wireguard/crypto/zinc/selftest/chacha20poly1305.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/selftest/chacha20poly1305.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,9076 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+struct chacha20poly1305_testvec {
+	const u8 *input, *output, *assoc, *nonce, *key;
+	size_t ilen, alen, nlen;
+	bool failure;
+};
+
+/* The first of these are the ChaCha20-Poly1305 AEAD test vectors from RFC7539
+ * 2.8.2. After they are generated by reference implementations. And the final
+ * marked ones are taken from wycheproof, but we only do these for the encrypt
+ * side, because mostly we're stressing the primitives rather than the actual
+ * chapoly construction. This also requires adding a 96-bit nonce construction,
+ * just for the purpose of the tests.
+ */
+
+static const u8 enc_input001[] __initconst = {
+	0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74,
+	0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20,
+	0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66,
+	0x74, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65,
+	0x6e, 0x74, 0x73, 0x20, 0x76, 0x61, 0x6c, 0x69,
+	0x64, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x20,
+	0x6d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x20,
+	0x6f, 0x66, 0x20, 0x73, 0x69, 0x78, 0x20, 0x6d,
+	0x6f, 0x6e, 0x74, 0x68, 0x73, 0x20, 0x61, 0x6e,
+	0x64, 0x20, 0x6d, 0x61, 0x79, 0x20, 0x62, 0x65,
+	0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64,
+	0x2c, 0x20, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63,
+	0x65, 0x64, 0x2c, 0x20, 0x6f, 0x72, 0x20, 0x6f,
+	0x62, 0x73, 0x6f, 0x6c, 0x65, 0x74, 0x65, 0x64,
+	0x20, 0x62, 0x79, 0x20, 0x6f, 0x74, 0x68, 0x65,
+	0x72, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65,
+	0x6e, 0x74, 0x73, 0x20, 0x61, 0x74, 0x20, 0x61,
+	0x6e, 0x79, 0x20, 0x74, 0x69, 0x6d, 0x65, 0x2e,
+	0x20, 0x49, 0x74, 0x20, 0x69, 0x73, 0x20, 0x69,
+	0x6e, 0x61, 0x70, 0x70, 0x72, 0x6f, 0x70, 0x72,
+	0x69, 0x61, 0x74, 0x65, 0x20, 0x74, 0x6f, 0x20,
+	0x75, 0x73, 0x65, 0x20, 0x49, 0x6e, 0x74, 0x65,
+	0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61,
+	0x66, 0x74, 0x73, 0x20, 0x61, 0x73, 0x20, 0x72,
+	0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65,
+	0x20, 0x6d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61,
+	0x6c, 0x20, 0x6f, 0x72, 0x20, 0x74, 0x6f, 0x20,
+	0x63, 0x69, 0x74, 0x65, 0x20, 0x74, 0x68, 0x65,
+	0x6d, 0x20, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x20,
+	0x74, 0x68, 0x61, 0x6e, 0x20, 0x61, 0x73, 0x20,
+	0x2f, 0xe2, 0x80, 0x9c, 0x77, 0x6f, 0x72, 0x6b,
+	0x20, 0x69, 0x6e, 0x20, 0x70, 0x72, 0x6f, 0x67,
+	0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80,
+	0x9d
+};
+static const u8 enc_output001[] __initconst = {
+	0x64, 0xa0, 0x86, 0x15, 0x75, 0x86, 0x1a, 0xf4,
+	0x60, 0xf0, 0x62, 0xc7, 0x9b, 0xe6, 0x43, 0xbd,
+	0x5e, 0x80, 0x5c, 0xfd, 0x34, 0x5c, 0xf3, 0x89,
+	0xf1, 0x08, 0x67, 0x0a, 0xc7, 0x6c, 0x8c, 0xb2,
+	0x4c, 0x6c, 0xfc, 0x18, 0x75, 0x5d, 0x43, 0xee,
+	0xa0, 0x9e, 0xe9, 0x4e, 0x38, 0x2d, 0x26, 0xb0,
+	0xbd, 0xb7, 0xb7, 0x3c, 0x32, 0x1b, 0x01, 0x00,
+	0xd4, 0xf0, 0x3b, 0x7f, 0x35, 0x58, 0x94, 0xcf,
+	0x33, 0x2f, 0x83, 0x0e, 0x71, 0x0b, 0x97, 0xce,
+	0x98, 0xc8, 0xa8, 0x4a, 0xbd, 0x0b, 0x94, 0x81,
+	0x14, 0xad, 0x17, 0x6e, 0x00, 0x8d, 0x33, 0xbd,
+	0x60, 0xf9, 0x82, 0xb1, 0xff, 0x37, 0xc8, 0x55,
+	0x97, 0x97, 0xa0, 0x6e, 0xf4, 0xf0, 0xef, 0x61,
+	0xc1, 0x86, 0x32, 0x4e, 0x2b, 0x35, 0x06, 0x38,
+	0x36, 0x06, 0x90, 0x7b, 0x6a, 0x7c, 0x02, 0xb0,
+	0xf9, 0xf6, 0x15, 0x7b, 0x53, 0xc8, 0x67, 0xe4,
+	0xb9, 0x16, 0x6c, 0x76, 0x7b, 0x80, 0x4d, 0x46,
+	0xa5, 0x9b, 0x52, 0x16, 0xcd, 0xe7, 0xa4, 0xe9,
+	0x90, 0x40, 0xc5, 0xa4, 0x04, 0x33, 0x22, 0x5e,
+	0xe2, 0x82, 0xa1, 0xb0, 0xa0, 0x6c, 0x52, 0x3e,
+	0xaf, 0x45, 0x34, 0xd7, 0xf8, 0x3f, 0xa1, 0x15,
+	0x5b, 0x00, 0x47, 0x71, 0x8c, 0xbc, 0x54, 0x6a,
+	0x0d, 0x07, 0x2b, 0x04, 0xb3, 0x56, 0x4e, 0xea,
+	0x1b, 0x42, 0x22, 0x73, 0xf5, 0x48, 0x27, 0x1a,
+	0x0b, 0xb2, 0x31, 0x60, 0x53, 0xfa, 0x76, 0x99,
+	0x19, 0x55, 0xeb, 0xd6, 0x31, 0x59, 0x43, 0x4e,
+	0xce, 0xbb, 0x4e, 0x46, 0x6d, 0xae, 0x5a, 0x10,
+	0x73, 0xa6, 0x72, 0x76, 0x27, 0x09, 0x7a, 0x10,
+	0x49, 0xe6, 0x17, 0xd9, 0x1d, 0x36, 0x10, 0x94,
+	0xfa, 0x68, 0xf0, 0xff, 0x77, 0x98, 0x71, 0x30,
+	0x30, 0x5b, 0xea, 0xba, 0x2e, 0xda, 0x04, 0xdf,
+	0x99, 0x7b, 0x71, 0x4d, 0x6c, 0x6f, 0x2c, 0x29,
+	0xa6, 0xad, 0x5c, 0xb4, 0x02, 0x2b, 0x02, 0x70,
+	0x9b, 0xee, 0xad, 0x9d, 0x67, 0x89, 0x0c, 0xbb,
+	0x22, 0x39, 0x23, 0x36, 0xfe, 0xa1, 0x85, 0x1f,
+	0x38
+};
+static const u8 enc_assoc001[] __initconst = {
+	0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x4e, 0x91
+};
+static const u8 enc_nonce001[] __initconst = {
+	0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08
+};
+static const u8 enc_key001[] __initconst = {
+	0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a,
+	0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0,
+	0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09,
+	0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0
+};
+
+static const u8 enc_input002[] __initconst = { };
+static const u8 enc_output002[] __initconst = {
+	0xea, 0xe0, 0x1e, 0x9e, 0x2c, 0x91, 0xaa, 0xe1,
+	0xdb, 0x5d, 0x99, 0x3f, 0x8a, 0xf7, 0x69, 0x92
+};
+static const u8 enc_assoc002[] __initconst = { };
+static const u8 enc_nonce002[] __initconst = {
+	0xca, 0xbf, 0x33, 0x71, 0x32, 0x45, 0x77, 0x8e
+};
+static const u8 enc_key002[] __initconst = {
+	0x4c, 0xf5, 0x96, 0x83, 0x38, 0xe6, 0xae, 0x7f,
+	0x2d, 0x29, 0x25, 0x76, 0xd5, 0x75, 0x27, 0x86,
+	0x91, 0x9a, 0x27, 0x7a, 0xfb, 0x46, 0xc5, 0xef,
+	0x94, 0x81, 0x79, 0x57, 0x14, 0x59, 0x40, 0x68
+};
+
+static const u8 enc_input003[] __initconst = { };
+static const u8 enc_output003[] __initconst = {
+	0xdd, 0x6b, 0x3b, 0x82, 0xce, 0x5a, 0xbd, 0xd6,
+	0xa9, 0x35, 0x83, 0xd8, 0x8c, 0x3d, 0x85, 0x77
+};
+static const u8 enc_assoc003[] __initconst = {
+	0x33, 0x10, 0x41, 0x12, 0x1f, 0xf3, 0xd2, 0x6b
+};
+static const u8 enc_nonce003[] __initconst = {
+	0x3d, 0x86, 0xb5, 0x6b, 0xc8, 0xa3, 0x1f, 0x1d
+};
+static const u8 enc_key003[] __initconst = {
+	0x2d, 0xb0, 0x5d, 0x40, 0xc8, 0xed, 0x44, 0x88,
+	0x34, 0xd1, 0x13, 0xaf, 0x57, 0xa1, 0xeb, 0x3a,
+	0x2a, 0x80, 0x51, 0x36, 0xec, 0x5b, 0xbc, 0x08,
+	0x93, 0x84, 0x21, 0xb5, 0x13, 0x88, 0x3c, 0x0d
+};
+
+static const u8 enc_input004[] __initconst = {
+	0xa4
+};
+static const u8 enc_output004[] __initconst = {
+	0xb7, 0x1b, 0xb0, 0x73, 0x59, 0xb0, 0x84, 0xb2,
+	0x6d, 0x8e, 0xab, 0x94, 0x31, 0xa1, 0xae, 0xac,
+	0x89
+};
+static const u8 enc_assoc004[] __initconst = {
+	0x6a, 0xe2, 0xad, 0x3f, 0x88, 0x39, 0x5a, 0x40
+};
+static const u8 enc_nonce004[] __initconst = {
+	0xd2, 0x32, 0x1f, 0x29, 0x28, 0xc6, 0xc4, 0xc4
+};
+static const u8 enc_key004[] __initconst = {
+	0x4b, 0x28, 0x4b, 0xa3, 0x7b, 0xbe, 0xe9, 0xf8,
+	0x31, 0x80, 0x82, 0xd7, 0xd8, 0xe8, 0xb5, 0xa1,
+	0xe2, 0x18, 0x18, 0x8a, 0x9c, 0xfa, 0xa3, 0x3d,
+	0x25, 0x71, 0x3e, 0x40, 0xbc, 0x54, 0x7a, 0x3e
+};
+
+static const u8 enc_input005[] __initconst = {
+	0x2d
+};
+static const u8 enc_output005[] __initconst = {
+	0xbf, 0xe1, 0x5b, 0x0b, 0xdb, 0x6b, 0xf5, 0x5e,
+	0x6c, 0x5d, 0x84, 0x44, 0x39, 0x81, 0xc1, 0x9c,
+	0xac
+};
+static const u8 enc_assoc005[] __initconst = { };
+static const u8 enc_nonce005[] __initconst = {
+	0x20, 0x1c, 0xaa, 0x5f, 0x9c, 0xbf, 0x92, 0x30
+};
+static const u8 enc_key005[] __initconst = {
+	0x66, 0xca, 0x9c, 0x23, 0x2a, 0x4b, 0x4b, 0x31,
+	0x0e, 0x92, 0x89, 0x8b, 0xf4, 0x93, 0xc7, 0x87,
+	0x98, 0xa3, 0xd8, 0x39, 0xf8, 0xf4, 0xa7, 0x01,
+	0xc0, 0x2e, 0x0a, 0xa6, 0x7e, 0x5a, 0x78, 0x87
+};
+
+static const u8 enc_input006[] __initconst = {
+	0x33, 0x2f, 0x94, 0xc1, 0xa4, 0xef, 0xcc, 0x2a,
+	0x5b, 0xa6, 0xe5, 0x8f, 0x1d, 0x40, 0xf0, 0x92,
+	0x3c, 0xd9, 0x24, 0x11, 0xa9, 0x71, 0xf9, 0x37,
+	0x14, 0x99, 0xfa, 0xbe, 0xe6, 0x80, 0xde, 0x50,
+	0xc9, 0x96, 0xd4, 0xb0, 0xec, 0x9e, 0x17, 0xec,
+	0xd2, 0x5e, 0x72, 0x99, 0xfc, 0x0a, 0xe1, 0xcb,
+	0x48, 0xd2, 0x85, 0xdd, 0x2f, 0x90, 0xe0, 0x66,
+	0x3b, 0xe6, 0x20, 0x74, 0xbe, 0x23, 0x8f, 0xcb,
+	0xb4, 0xe4, 0xda, 0x48, 0x40, 0xa6, 0xd1, 0x1b,
+	0xc7, 0x42, 0xce, 0x2f, 0x0c, 0xa6, 0x85, 0x6e,
+	0x87, 0x37, 0x03, 0xb1, 0x7c, 0x25, 0x96, 0xa3,
+	0x05, 0xd8, 0xb0, 0xf4, 0xed, 0xea, 0xc2, 0xf0,
+	0x31, 0x98, 0x6c, 0xd1, 0x14, 0x25, 0xc0, 0xcb,
+	0x01, 0x74, 0xd0, 0x82, 0xf4, 0x36, 0xf5, 0x41,
+	0xd5, 0xdc, 0xca, 0xc5, 0xbb, 0x98, 0xfe, 0xfc,
+	0x69, 0x21, 0x70, 0xd8, 0xa4, 0x4b, 0xc8, 0xde,
+	0x8f
+};
+static const u8 enc_output006[] __initconst = {
+	0x8b, 0x06, 0xd3, 0x31, 0xb0, 0x93, 0x45, 0xb1,
+	0x75, 0x6e, 0x26, 0xf9, 0x67, 0xbc, 0x90, 0x15,
+	0x81, 0x2c, 0xb5, 0xf0, 0xc6, 0x2b, 0xc7, 0x8c,
+	0x56, 0xd1, 0xbf, 0x69, 0x6c, 0x07, 0xa0, 0xda,
+	0x65, 0x27, 0xc9, 0x90, 0x3d, 0xef, 0x4b, 0x11,
+	0x0f, 0x19, 0x07, 0xfd, 0x29, 0x92, 0xd9, 0xc8,
+	0xf7, 0x99, 0x2e, 0x4a, 0xd0, 0xb8, 0x2c, 0xdc,
+	0x93, 0xf5, 0x9e, 0x33, 0x78, 0xd1, 0x37, 0xc3,
+	0x66, 0xd7, 0x5e, 0xbc, 0x44, 0xbf, 0x53, 0xa5,
+	0xbc, 0xc4, 0xcb, 0x7b, 0x3a, 0x8e, 0x7f, 0x02,
+	0xbd, 0xbb, 0xe7, 0xca, 0xa6, 0x6c, 0x6b, 0x93,
+	0x21, 0x93, 0x10, 0x61, 0xe7, 0x69, 0xd0, 0x78,
+	0xf3, 0x07, 0x5a, 0x1a, 0x8f, 0x73, 0xaa, 0xb1,
+	0x4e, 0xd3, 0xda, 0x4f, 0xf3, 0x32, 0xe1, 0x66,
+	0x3e, 0x6c, 0xc6, 0x13, 0xba, 0x06, 0x5b, 0xfc,
+	0x6a, 0xe5, 0x6f, 0x60, 0xfb, 0x07, 0x40, 0xb0,
+	0x8c, 0x9d, 0x84, 0x43, 0x6b, 0xc1, 0xf7, 0x8d,
+	0x8d, 0x31, 0xf7, 0x7a, 0x39, 0x4d, 0x8f, 0x9a,
+	0xeb
+};
+static const u8 enc_assoc006[] __initconst = {
+	0x70, 0xd3, 0x33, 0xf3, 0x8b, 0x18, 0x0b
+};
+static const u8 enc_nonce006[] __initconst = {
+	0xdf, 0x51, 0x84, 0x82, 0x42, 0x0c, 0x75, 0x9c
+};
+static const u8 enc_key006[] __initconst = {
+	0x68, 0x7b, 0x8d, 0x8e, 0xe3, 0xc4, 0xdd, 0xae,
+	0xdf, 0x72, 0x7f, 0x53, 0x72, 0x25, 0x1e, 0x78,
+	0x91, 0xcb, 0x69, 0x76, 0x1f, 0x49, 0x93, 0xf9,
+	0x6f, 0x21, 0xcc, 0x39, 0x9c, 0xad, 0xb1, 0x01
+};
+
+static const u8 enc_input007[] __initconst = {
+	0x9b, 0x18, 0xdb, 0xdd, 0x9a, 0x0f, 0x3e, 0xa5,
+	0x15, 0x17, 0xde, 0xdf, 0x08, 0x9d, 0x65, 0x0a,
+	0x67, 0x30, 0x12, 0xe2, 0x34, 0x77, 0x4b, 0xc1,
+	0xd9, 0xc6, 0x1f, 0xab, 0xc6, 0x18, 0x50, 0x17,
+	0xa7, 0x9d, 0x3c, 0xa6, 0xc5, 0x35, 0x8c, 0x1c,
+	0xc0, 0xa1, 0x7c, 0x9f, 0x03, 0x89, 0xca, 0xe1,
+	0xe6, 0xe9, 0xd4, 0xd3, 0x88, 0xdb, 0xb4, 0x51,
+	0x9d, 0xec, 0xb4, 0xfc, 0x52, 0xee, 0x6d, 0xf1,
+	0x75, 0x42, 0xc6, 0xfd, 0xbd, 0x7a, 0x8e, 0x86,
+	0xfc, 0x44, 0xb3, 0x4f, 0xf3, 0xea, 0x67, 0x5a,
+	0x41, 0x13, 0xba, 0xb0, 0xdc, 0xe1, 0xd3, 0x2a,
+	0x7c, 0x22, 0xb3, 0xca, 0xac, 0x6a, 0x37, 0x98,
+	0x3e, 0x1d, 0x40, 0x97, 0xf7, 0x9b, 0x1d, 0x36,
+	0x6b, 0xb3, 0x28, 0xbd, 0x60, 0x82, 0x47, 0x34,
+	0xaa, 0x2f, 0x7d, 0xe9, 0xa8, 0x70, 0x81, 0x57,
+	0xd4, 0xb9, 0x77, 0x0a, 0x9d, 0x29, 0xa7, 0x84,
+	0x52, 0x4f, 0xc2, 0x4a, 0x40, 0x3b, 0x3c, 0xd4,
+	0xc9, 0x2a, 0xdb, 0x4a, 0x53, 0xc4, 0xbe, 0x80,
+	0xe9, 0x51, 0x7f, 0x8f, 0xc7, 0xa2, 0xce, 0x82,
+	0x5c, 0x91, 0x1e, 0x74, 0xd9, 0xd0, 0xbd, 0xd5,
+	0xf3, 0xfd, 0xda, 0x4d, 0x25, 0xb4, 0xbb, 0x2d,
+	0xac, 0x2f, 0x3d, 0x71, 0x85, 0x7b, 0xcf, 0x3c,
+	0x7b, 0x3e, 0x0e, 0x22, 0x78, 0x0c, 0x29, 0xbf,
+	0xe4, 0xf4, 0x57, 0xb3, 0xcb, 0x49, 0xa0, 0xfc,
+	0x1e, 0x05, 0x4e, 0x16, 0xbc, 0xd5, 0xa8, 0xa3,
+	0xee, 0x05, 0x35, 0xc6, 0x7c, 0xab, 0x60, 0x14,
+	0x55, 0x1a, 0x8e, 0xc5, 0x88, 0x5d, 0xd5, 0x81,
+	0xc2, 0x81, 0xa5, 0xc4, 0x60, 0xdb, 0xaf, 0x77,
+	0x91, 0xe1, 0xce, 0xa2, 0x7e, 0x7f, 0x42, 0xe3,
+	0xb0, 0x13, 0x1c, 0x1f, 0x25, 0x60, 0x21, 0xe2,
+	0x40, 0x5f, 0x99, 0xb7, 0x73, 0xec, 0x9b, 0x2b,
+	0xf0, 0x65, 0x11, 0xc8, 0xd0, 0x0a, 0x9f, 0xd3
+};
+static const u8 enc_output007[] __initconst = {
+	0x85, 0x04, 0xc2, 0xed, 0x8d, 0xfd, 0x97, 0x5c,
+	0xd2, 0xb7, 0xe2, 0xc1, 0x6b, 0xa3, 0xba, 0xf8,
+	0xc9, 0x50, 0xc3, 0xc6, 0xa5, 0xe3, 0xa4, 0x7c,
+	0xc3, 0x23, 0x49, 0x5e, 0xa9, 0xb9, 0x32, 0xeb,
+	0x8a, 0x7c, 0xca, 0xe5, 0xec, 0xfb, 0x7c, 0xc0,
+	0xcb, 0x7d, 0xdc, 0x2c, 0x9d, 0x92, 0x55, 0x21,
+	0x0a, 0xc8, 0x43, 0x63, 0x59, 0x0a, 0x31, 0x70,
+	0x82, 0x67, 0x41, 0x03, 0xf8, 0xdf, 0xf2, 0xac,
+	0xa7, 0x02, 0xd4, 0xd5, 0x8a, 0x2d, 0xc8, 0x99,
+	0x19, 0x66, 0xd0, 0xf6, 0x88, 0x2c, 0x77, 0xd9,
+	0xd4, 0x0d, 0x6c, 0xbd, 0x98, 0xde, 0xe7, 0x7f,
+	0xad, 0x7e, 0x8a, 0xfb, 0xe9, 0x4b, 0xe5, 0xf7,
+	0xe5, 0x50, 0xa0, 0x90, 0x3f, 0xd6, 0x22, 0x53,
+	0xe3, 0xfe, 0x1b, 0xcc, 0x79, 0x3b, 0xec, 0x12,
+	0x47, 0x52, 0xa7, 0xd6, 0x04, 0xe3, 0x52, 0xe6,
+	0x93, 0x90, 0x91, 0x32, 0x73, 0x79, 0xb8, 0xd0,
+	0x31, 0xde, 0x1f, 0x9f, 0x2f, 0x05, 0x38, 0x54,
+	0x2f, 0x35, 0x04, 0x39, 0xe0, 0xa7, 0xba, 0xc6,
+	0x52, 0xf6, 0x37, 0x65, 0x4c, 0x07, 0xa9, 0x7e,
+	0xb3, 0x21, 0x6f, 0x74, 0x8c, 0xc9, 0xde, 0xdb,
+	0x65, 0x1b, 0x9b, 0xaa, 0x60, 0xb1, 0x03, 0x30,
+	0x6b, 0xb2, 0x03, 0xc4, 0x1c, 0x04, 0xf8, 0x0f,
+	0x64, 0xaf, 0x46, 0xe4, 0x65, 0x99, 0x49, 0xe2,
+	0xea, 0xce, 0x78, 0x00, 0xd8, 0x8b, 0xd5, 0x2e,
+	0xcf, 0xfc, 0x40, 0x49, 0xe8, 0x58, 0xdc, 0x34,
+	0x9c, 0x8c, 0x61, 0xbf, 0x0a, 0x8e, 0xec, 0x39,
+	0xa9, 0x30, 0x05, 0x5a, 0xd2, 0x56, 0x01, 0xc7,
+	0xda, 0x8f, 0x4e, 0xbb, 0x43, 0xa3, 0x3a, 0xf9,
+	0x15, 0x2a, 0xd0, 0xa0, 0x7a, 0x87, 0x34, 0x82,
+	0xfe, 0x8a, 0xd1, 0x2d, 0x5e, 0xc7, 0xbf, 0x04,
+	0x53, 0x5f, 0x3b, 0x36, 0xd4, 0x25, 0x5c, 0x34,
+	0x7a, 0x8d, 0xd5, 0x05, 0xce, 0x72, 0xca, 0xef,
+	0x7a, 0x4b, 0xbc, 0xb0, 0x10, 0x5c, 0x96, 0x42,
+	0x3a, 0x00, 0x98, 0xcd, 0x15, 0xe8, 0xb7, 0x53
+};
+static const u8 enc_assoc007[] __initconst = { };
+static const u8 enc_nonce007[] __initconst = {
+	0xde, 0x7b, 0xef, 0xc3, 0x65, 0x1b, 0x68, 0xb0
+};
+static const u8 enc_key007[] __initconst = {
+	0x8d, 0xb8, 0x91, 0x48, 0xf0, 0xe7, 0x0a, 0xbd,
+	0xf9, 0x3f, 0xcd, 0xd9, 0xa0, 0x1e, 0x42, 0x4c,
+	0xe7, 0xde, 0x25, 0x3d, 0xa3, 0xd7, 0x05, 0x80,
+	0x8d, 0xf2, 0x82, 0xac, 0x44, 0x16, 0x51, 0x01
+};
+
+static const u8 enc_input008[] __initconst = {
+	0xc3, 0x09, 0x94, 0x62, 0xe6, 0x46, 0x2e, 0x10,
+	0xbe, 0x00, 0xe4, 0xfc, 0xf3, 0x40, 0xa3, 0xe2,
+	0x0f, 0xc2, 0x8b, 0x28, 0xdc, 0xba, 0xb4, 0x3c,
+	0xe4, 0x21, 0x58, 0x61, 0xcd, 0x8b, 0xcd, 0xfb,
+	0xac, 0x94, 0xa1, 0x45, 0xf5, 0x1c, 0xe1, 0x12,
+	0xe0, 0x3b, 0x67, 0x21, 0x54, 0x5e, 0x8c, 0xaa,
+	0xcf, 0xdb, 0xb4, 0x51, 0xd4, 0x13, 0xda, 0xe6,
+	0x83, 0x89, 0xb6, 0x92, 0xe9, 0x21, 0x76, 0xa4,
+	0x93, 0x7d, 0x0e, 0xfd, 0x96, 0x36, 0x03, 0x91,
+	0x43, 0x5c, 0x92, 0x49, 0x62, 0x61, 0x7b, 0xeb,
+	0x43, 0x89, 0xb8, 0x12, 0x20, 0x43, 0xd4, 0x47,
+	0x06, 0x84, 0xee, 0x47, 0xe9, 0x8a, 0x73, 0x15,
+	0x0f, 0x72, 0xcf, 0xed, 0xce, 0x96, 0xb2, 0x7f,
+	0x21, 0x45, 0x76, 0xeb, 0x26, 0x28, 0x83, 0x6a,
+	0xad, 0xaa, 0xa6, 0x81, 0xd8, 0x55, 0xb1, 0xa3,
+	0x85, 0xb3, 0x0c, 0xdf, 0xf1, 0x69, 0x2d, 0x97,
+	0x05, 0x2a, 0xbc, 0x7c, 0x7b, 0x25, 0xf8, 0x80,
+	0x9d, 0x39, 0x25, 0xf3, 0x62, 0xf0, 0x66, 0x5e,
+	0xf4, 0xa0, 0xcf, 0xd8, 0xfd, 0x4f, 0xb1, 0x1f,
+	0x60, 0x3a, 0x08, 0x47, 0xaf, 0xe1, 0xf6, 0x10,
+	0x77, 0x09, 0xa7, 0x27, 0x8f, 0x9a, 0x97, 0x5a,
+	0x26, 0xfa, 0xfe, 0x41, 0x32, 0x83, 0x10, 0xe0,
+	0x1d, 0xbf, 0x64, 0x0d, 0xf4, 0x1c, 0x32, 0x35,
+	0xe5, 0x1b, 0x36, 0xef, 0xd4, 0x4a, 0x93, 0x4d,
+	0x00, 0x7c, 0xec, 0x02, 0x07, 0x8b, 0x5d, 0x7d,
+	0x1b, 0x0e, 0xd1, 0xa6, 0xa5, 0x5d, 0x7d, 0x57,
+	0x88, 0xa8, 0xcc, 0x81, 0xb4, 0x86, 0x4e, 0xb4,
+	0x40, 0xe9, 0x1d, 0xc3, 0xb1, 0x24, 0x3e, 0x7f,
+	0xcc, 0x8a, 0x24, 0x9b, 0xdf, 0x6d, 0xf0, 0x39,
+	0x69, 0x3e, 0x4c, 0xc0, 0x96, 0xe4, 0x13, 0xda,
+	0x90, 0xda, 0xf4, 0x95, 0x66, 0x8b, 0x17, 0x17,
+	0xfe, 0x39, 0x43, 0x25, 0xaa, 0xda, 0xa0, 0x43,
+	0x3c, 0xb1, 0x41, 0x02, 0xa3, 0xf0, 0xa7, 0x19,
+	0x59, 0xbc, 0x1d, 0x7d, 0x6c, 0x6d, 0x91, 0x09,
+	0x5c, 0xb7, 0x5b, 0x01, 0xd1, 0x6f, 0x17, 0x21,
+	0x97, 0xbf, 0x89, 0x71, 0xa5, 0xb0, 0x6e, 0x07,
+	0x45, 0xfd, 0x9d, 0xea, 0x07, 0xf6, 0x7a, 0x9f,
+	0x10, 0x18, 0x22, 0x30, 0x73, 0xac, 0xd4, 0x6b,
+	0x72, 0x44, 0xed, 0xd9, 0x19, 0x9b, 0x2d, 0x4a,
+	0x41, 0xdd, 0xd1, 0x85, 0x5e, 0x37, 0x19, 0xed,
+	0xd2, 0x15, 0x8f, 0x5e, 0x91, 0xdb, 0x33, 0xf2,
+	0xe4, 0xdb, 0xff, 0x98, 0xfb, 0xa3, 0xb5, 0xca,
+	0x21, 0x69, 0x08, 0xe7, 0x8a, 0xdf, 0x90, 0xff,
+	0x3e, 0xe9, 0x20, 0x86, 0x3c, 0xe9, 0xfc, 0x0b,
+	0xfe, 0x5c, 0x61, 0xaa, 0x13, 0x92, 0x7f, 0x7b,
+	0xec, 0xe0, 0x6d, 0xa8, 0x23, 0x22, 0xf6, 0x6b,
+	0x77, 0xc4, 0xfe, 0x40, 0x07, 0x3b, 0xb6, 0xf6,
+	0x8e, 0x5f, 0xd4, 0xb9, 0xb7, 0x0f, 0x21, 0x04,
+	0xef, 0x83, 0x63, 0x91, 0x69, 0x40, 0xa3, 0x48,
+	0x5c, 0xd2, 0x60, 0xf9, 0x4f, 0x6c, 0x47, 0x8b,
+	0x3b, 0xb1, 0x9f, 0x8e, 0xee, 0x16, 0x8a, 0x13,
+	0xfc, 0x46, 0x17, 0xc3, 0xc3, 0x32, 0x56, 0xf8,
+	0x3c, 0x85, 0x3a, 0xb6, 0x3e, 0xaa, 0x89, 0x4f,
+	0xb3, 0xdf, 0x38, 0xfd, 0xf1, 0xe4, 0x3a, 0xc0,
+	0xe6, 0x58, 0xb5, 0x8f, 0xc5, 0x29, 0xa2, 0x92,
+	0x4a, 0xb6, 0xa0, 0x34, 0x7f, 0xab, 0xb5, 0x8a,
+	0x90, 0xa1, 0xdb, 0x4d, 0xca, 0xb6, 0x2c, 0x41,
+	0x3c, 0xf7, 0x2b, 0x21, 0xc3, 0xfd, 0xf4, 0x17,
+	0x5c, 0xb5, 0x33, 0x17, 0x68, 0x2b, 0x08, 0x30,
+	0xf3, 0xf7, 0x30, 0x3c, 0x96, 0xe6, 0x6a, 0x20,
+	0x97, 0xe7, 0x4d, 0x10, 0x5f, 0x47, 0x5f, 0x49,
+	0x96, 0x09, 0xf0, 0x27, 0x91, 0xc8, 0xf8, 0x5a,
+	0x2e, 0x79, 0xb5, 0xe2, 0xb8, 0xe8, 0xb9, 0x7b,
+	0xd5, 0x10, 0xcb, 0xff, 0x5d, 0x14, 0x73, 0xf3
+};
+static const u8 enc_output008[] __initconst = {
+	0x14, 0xf6, 0x41, 0x37, 0xa6, 0xd4, 0x27, 0xcd,
+	0xdb, 0x06, 0x3e, 0x9a, 0x4e, 0xab, 0xd5, 0xb1,
+	0x1e, 0x6b, 0xd2, 0xbc, 0x11, 0xf4, 0x28, 0x93,
+	0x63, 0x54, 0xef, 0xbb, 0x5e, 0x1d, 0x3a, 0x1d,
+	0x37, 0x3c, 0x0a, 0x6c, 0x1e, 0xc2, 0xd1, 0x2c,
+	0xb5, 0xa3, 0xb5, 0x7b, 0xb8, 0x8f, 0x25, 0xa6,
+	0x1b, 0x61, 0x1c, 0xec, 0x28, 0x58, 0x26, 0xa4,
+	0xa8, 0x33, 0x28, 0x25, 0x5c, 0x45, 0x05, 0xe5,
+	0x6c, 0x99, 0xe5, 0x45, 0xc4, 0xa2, 0x03, 0x84,
+	0x03, 0x73, 0x1e, 0x8c, 0x49, 0xac, 0x20, 0xdd,
+	0x8d, 0xb3, 0xc4, 0xf5, 0xe7, 0x4f, 0xf1, 0xed,
+	0xa1, 0x98, 0xde, 0xa4, 0x96, 0xdd, 0x2f, 0xab,
+	0xab, 0x97, 0xcf, 0x3e, 0xd2, 0x9e, 0xb8, 0x13,
+	0x07, 0x28, 0x29, 0x19, 0xaf, 0xfd, 0xf2, 0x49,
+	0x43, 0xea, 0x49, 0x26, 0x91, 0xc1, 0x07, 0xd6,
+	0xbb, 0x81, 0x75, 0x35, 0x0d, 0x24, 0x7f, 0xc8,
+	0xda, 0xd4, 0xb7, 0xeb, 0xe8, 0x5c, 0x09, 0xa2,
+	0x2f, 0xdc, 0x28, 0x7d, 0x3a, 0x03, 0xfa, 0x94,
+	0xb5, 0x1d, 0x17, 0x99, 0x36, 0xc3, 0x1c, 0x18,
+	0x34, 0xe3, 0x9f, 0xf5, 0x55, 0x7c, 0xb0, 0x60,
+	0x9d, 0xff, 0xac, 0xd4, 0x61, 0xf2, 0xad, 0xf8,
+	0xce, 0xc7, 0xbe, 0x5c, 0xd2, 0x95, 0xa8, 0x4b,
+	0x77, 0x13, 0x19, 0x59, 0x26, 0xc9, 0xb7, 0x8f,
+	0x6a, 0xcb, 0x2d, 0x37, 0x91, 0xea, 0x92, 0x9c,
+	0x94, 0x5b, 0xda, 0x0b, 0xce, 0xfe, 0x30, 0x20,
+	0xf8, 0x51, 0xad, 0xf2, 0xbe, 0xe7, 0xc7, 0xff,
+	0xb3, 0x33, 0x91, 0x6a, 0xc9, 0x1a, 0x41, 0xc9,
+	0x0f, 0xf3, 0x10, 0x0e, 0xfd, 0x53, 0xff, 0x6c,
+	0x16, 0x52, 0xd9, 0xf3, 0xf7, 0x98, 0x2e, 0xc9,
+	0x07, 0x31, 0x2c, 0x0c, 0x72, 0xd7, 0xc5, 0xc6,
+	0x08, 0x2a, 0x7b, 0xda, 0xbd, 0x7e, 0x02, 0xea,
+	0x1a, 0xbb, 0xf2, 0x04, 0x27, 0x61, 0x28, 0x8e,
+	0xf5, 0x04, 0x03, 0x1f, 0x4c, 0x07, 0x55, 0x82,
+	0xec, 0x1e, 0xd7, 0x8b, 0x2f, 0x65, 0x56, 0xd1,
+	0xd9, 0x1e, 0x3c, 0xe9, 0x1f, 0x5e, 0x98, 0x70,
+	0x38, 0x4a, 0x8c, 0x49, 0xc5, 0x43, 0xa0, 0xa1,
+	0x8b, 0x74, 0x9d, 0x4c, 0x62, 0x0d, 0x10, 0x0c,
+	0xf4, 0x6c, 0x8f, 0xe0, 0xaa, 0x9a, 0x8d, 0xb7,
+	0xe0, 0xbe, 0x4c, 0x87, 0xf1, 0x98, 0x2f, 0xcc,
+	0xed, 0xc0, 0x52, 0x29, 0xdc, 0x83, 0xf8, 0xfc,
+	0x2c, 0x0e, 0xa8, 0x51, 0x4d, 0x80, 0x0d, 0xa3,
+	0xfe, 0xd8, 0x37, 0xe7, 0x41, 0x24, 0xfc, 0xfb,
+	0x75, 0xe3, 0x71, 0x7b, 0x57, 0x45, 0xf5, 0x97,
+	0x73, 0x65, 0x63, 0x14, 0x74, 0xb8, 0x82, 0x9f,
+	0xf8, 0x60, 0x2f, 0x8a, 0xf2, 0x4e, 0xf1, 0x39,
+	0xda, 0x33, 0x91, 0xf8, 0x36, 0xe0, 0x8d, 0x3f,
+	0x1f, 0x3b, 0x56, 0xdc, 0xa0, 0x8f, 0x3c, 0x9d,
+	0x71, 0x52, 0xa7, 0xb8, 0xc0, 0xa5, 0xc6, 0xa2,
+	0x73, 0xda, 0xf4, 0x4b, 0x74, 0x5b, 0x00, 0x3d,
+	0x99, 0xd7, 0x96, 0xba, 0xe6, 0xe1, 0xa6, 0x96,
+	0x38, 0xad, 0xb3, 0xc0, 0xd2, 0xba, 0x91, 0x6b,
+	0xf9, 0x19, 0xdd, 0x3b, 0xbe, 0xbe, 0x9c, 0x20,
+	0x50, 0xba, 0xa1, 0xd0, 0xce, 0x11, 0xbd, 0x95,
+	0xd8, 0xd1, 0xdd, 0x33, 0x85, 0x74, 0xdc, 0xdb,
+	0x66, 0x76, 0x44, 0xdc, 0x03, 0x74, 0x48, 0x35,
+	0x98, 0xb1, 0x18, 0x47, 0x94, 0x7d, 0xff, 0x62,
+	0xe4, 0x58, 0x78, 0xab, 0xed, 0x95, 0x36, 0xd9,
+	0x84, 0x91, 0x82, 0x64, 0x41, 0xbb, 0x58, 0xe6,
+	0x1c, 0x20, 0x6d, 0x15, 0x6b, 0x13, 0x96, 0xe8,
+	0x35, 0x7f, 0xdc, 0x40, 0x2c, 0xe9, 0xbc, 0x8a,
+	0x4f, 0x92, 0xec, 0x06, 0x2d, 0x50, 0xdf, 0x93,
+	0x5d, 0x65, 0x5a, 0xa8, 0xfc, 0x20, 0x50, 0x14,
+	0xa9, 0x8a, 0x7e, 0x1d, 0x08, 0x1f, 0xe2, 0x99,
+	0xd0, 0xbe, 0xfb, 0x3a, 0x21, 0x9d, 0xad, 0x86,
+	0x54, 0xfd, 0x0d, 0x98, 0x1c, 0x5a, 0x6f, 0x1f,
+	0x9a, 0x40, 0xcd, 0xa2, 0xff, 0x6a, 0xf1, 0x54
+};
+static const u8 enc_assoc008[] __initconst = { };
+static const u8 enc_nonce008[] __initconst = {
+	0x0e, 0x0d, 0x57, 0xbb, 0x7b, 0x40, 0x54, 0x02
+};
+static const u8 enc_key008[] __initconst = {
+	0xf2, 0xaa, 0x4f, 0x99, 0xfd, 0x3e, 0xa8, 0x53,
+	0xc1, 0x44, 0xe9, 0x81, 0x18, 0xdc, 0xf5, 0xf0,
+	0x3e, 0x44, 0x15, 0x59, 0xe0, 0xc5, 0x44, 0x86,
+	0xc3, 0x91, 0xa8, 0x75, 0xc0, 0x12, 0x46, 0xba
+};
+
+static const u8 enc_input009[] __initconst = {
+	0xe6, 0xc3, 0xdb, 0x63, 0x55, 0x15, 0xe3, 0x5b,
+	0xb7, 0x4b, 0x27, 0x8b, 0x5a, 0xdd, 0xc2, 0xe8,
+	0x3a, 0x6b, 0xd7, 0x81, 0x96, 0x35, 0x97, 0xca,
+	0xd7, 0x68, 0xe8, 0xef, 0xce, 0xab, 0xda, 0x09,
+	0x6e, 0xd6, 0x8e, 0xcb, 0x55, 0xb5, 0xe1, 0xe5,
+	0x57, 0xfd, 0xc4, 0xe3, 0xe0, 0x18, 0x4f, 0x85,
+	0xf5, 0x3f, 0x7e, 0x4b, 0x88, 0xc9, 0x52, 0x44,
+	0x0f, 0xea, 0xaf, 0x1f, 0x71, 0x48, 0x9f, 0x97,
+	0x6d, 0xb9, 0x6f, 0x00, 0xa6, 0xde, 0x2b, 0x77,
+	0x8b, 0x15, 0xad, 0x10, 0xa0, 0x2b, 0x7b, 0x41,
+	0x90, 0x03, 0x2d, 0x69, 0xae, 0xcc, 0x77, 0x7c,
+	0xa5, 0x9d, 0x29, 0x22, 0xc2, 0xea, 0xb4, 0x00,
+	0x1a, 0xd2, 0x7a, 0x98, 0x8a, 0xf9, 0xf7, 0x82,
+	0xb0, 0xab, 0xd8, 0xa6, 0x94, 0x8d, 0x58, 0x2f,
+	0x01, 0x9e, 0x00, 0x20, 0xfc, 0x49, 0xdc, 0x0e,
+	0x03, 0xe8, 0x45, 0x10, 0xd6, 0xa8, 0xda, 0x55,
+	0x10, 0x9a, 0xdf, 0x67, 0x22, 0x8b, 0x43, 0xab,
+	0x00, 0xbb, 0x02, 0xc8, 0xdd, 0x7b, 0x97, 0x17,
+	0xd7, 0x1d, 0x9e, 0x02, 0x5e, 0x48, 0xde, 0x8e,
+	0xcf, 0x99, 0x07, 0x95, 0x92, 0x3c, 0x5f, 0x9f,
+	0xc5, 0x8a, 0xc0, 0x23, 0xaa, 0xd5, 0x8c, 0x82,
+	0x6e, 0x16, 0x92, 0xb1, 0x12, 0x17, 0x07, 0xc3,
+	0xfb, 0x36, 0xf5, 0x6c, 0x35, 0xd6, 0x06, 0x1f,
+	0x9f, 0xa7, 0x94, 0xa2, 0x38, 0x63, 0x9c, 0xb0,
+	0x71, 0xb3, 0xa5, 0xd2, 0xd8, 0xba, 0x9f, 0x08,
+	0x01, 0xb3, 0xff, 0x04, 0x97, 0x73, 0x45, 0x1b,
+	0xd5, 0xa9, 0x9c, 0x80, 0xaf, 0x04, 0x9a, 0x85,
+	0xdb, 0x32, 0x5b, 0x5d, 0x1a, 0xc1, 0x36, 0x28,
+	0x10, 0x79, 0xf1, 0x3c, 0xbf, 0x1a, 0x41, 0x5c,
+	0x4e, 0xdf, 0xb2, 0x7c, 0x79, 0x3b, 0x7a, 0x62,
+	0x3d, 0x4b, 0xc9, 0x9b, 0x2a, 0x2e, 0x7c, 0xa2,
+	0xb1, 0x11, 0x98, 0xa7, 0x34, 0x1a, 0x00, 0xf3,
+	0xd1, 0xbc, 0x18, 0x22, 0xba, 0x02, 0x56, 0x62,
+	0x31, 0x10, 0x11, 0x6d, 0xe0, 0x54, 0x9d, 0x40,
+	0x1f, 0x26, 0x80, 0x41, 0xca, 0x3f, 0x68, 0x0f,
+	0x32, 0x1d, 0x0a, 0x8e, 0x79, 0xd8, 0xa4, 0x1b,
+	0x29, 0x1c, 0x90, 0x8e, 0xc5, 0xe3, 0xb4, 0x91,
+	0x37, 0x9a, 0x97, 0x86, 0x99, 0xd5, 0x09, 0xc5,
+	0xbb, 0xa3, 0x3f, 0x21, 0x29, 0x82, 0x14, 0x5c,
+	0xab, 0x25, 0xfb, 0xf2, 0x4f, 0x58, 0x26, 0xd4,
+	0x83, 0xaa, 0x66, 0x89, 0x67, 0x7e, 0xc0, 0x49,
+	0xe1, 0x11, 0x10, 0x7f, 0x7a, 0xda, 0x29, 0x04,
+	0xff, 0xf0, 0xcb, 0x09, 0x7c, 0x9d, 0xfa, 0x03,
+	0x6f, 0x81, 0x09, 0x31, 0x60, 0xfb, 0x08, 0xfa,
+	0x74, 0xd3, 0x64, 0x44, 0x7c, 0x55, 0x85, 0xec,
+	0x9c, 0x6e, 0x25, 0xb7, 0x6c, 0xc5, 0x37, 0xb6,
+	0x83, 0x87, 0x72, 0x95, 0x8b, 0x9d, 0xe1, 0x69,
+	0x5c, 0x31, 0x95, 0x42, 0xa6, 0x2c, 0xd1, 0x36,
+	0x47, 0x1f, 0xec, 0x54, 0xab, 0xa2, 0x1c, 0xd8,
+	0x00, 0xcc, 0xbc, 0x0d, 0x65, 0xe2, 0x67, 0xbf,
+	0xbc, 0xea, 0xee, 0x9e, 0xe4, 0x36, 0x95, 0xbe,
+	0x73, 0xd9, 0xa6, 0xd9, 0x0f, 0xa0, 0xcc, 0x82,
+	0x76, 0x26, 0xad, 0x5b, 0x58, 0x6c, 0x4e, 0xab,
+	0x29, 0x64, 0xd3, 0xd9, 0xa9, 0x08, 0x8c, 0x1d,
+	0xa1, 0x4f, 0x80, 0xd8, 0x3f, 0x94, 0xfb, 0xd3,
+	0x7b, 0xfc, 0xd1, 0x2b, 0xc3, 0x21, 0xeb, 0xe5,
+	0x1c, 0x84, 0x23, 0x7f, 0x4b, 0xfa, 0xdb, 0x34,
+	0x18, 0xa2, 0xc2, 0xe5, 0x13, 0xfe, 0x6c, 0x49,
+	0x81, 0xd2, 0x73, 0xe7, 0xe2, 0xd7, 0xe4, 0x4f,
+	0x4b, 0x08, 0x6e, 0xb1, 0x12, 0x22, 0x10, 0x9d,
+	0xac, 0x51, 0x1e, 0x17, 0xd9, 0x8a, 0x0b, 0x42,
+	0x88, 0x16, 0x81, 0x37, 0x7c, 0x6a, 0xf7, 0xef,
+	0x2d, 0xe3, 0xd9, 0xf8, 0x5f, 0xe0, 0x53, 0x27,
+	0x74, 0xb9, 0xe2, 0xd6, 0x1c, 0x80, 0x2c, 0x52,
+	0x65
+};
+static const u8 enc_output009[] __initconst = {
+	0xfd, 0x81, 0x8d, 0xd0, 0x3d, 0xb4, 0xd5, 0xdf,
+	0xd3, 0x42, 0x47, 0x5a, 0x6d, 0x19, 0x27, 0x66,
+	0x4b, 0x2e, 0x0c, 0x27, 0x9c, 0x96, 0x4c, 0x72,
+	0x02, 0xa3, 0x65, 0xc3, 0xb3, 0x6f, 0x2e, 0xbd,
+	0x63, 0x8a, 0x4a, 0x5d, 0x29, 0xa2, 0xd0, 0x28,
+	0x48, 0xc5, 0x3d, 0x98, 0xa3, 0xbc, 0xe0, 0xbe,
+	0x3b, 0x3f, 0xe6, 0x8a, 0xa4, 0x7f, 0x53, 0x06,
+	0xfa, 0x7f, 0x27, 0x76, 0x72, 0x31, 0xa1, 0xf5,
+	0xd6, 0x0c, 0x52, 0x47, 0xba, 0xcd, 0x4f, 0xd7,
+	0xeb, 0x05, 0x48, 0x0d, 0x7c, 0x35, 0x4a, 0x09,
+	0xc9, 0x76, 0x71, 0x02, 0xa3, 0xfb, 0xb7, 0x1a,
+	0x65, 0xb7, 0xed, 0x98, 0xc6, 0x30, 0x8a, 0x00,
+	0xae, 0xa1, 0x31, 0xe5, 0xb5, 0x9e, 0x6d, 0x62,
+	0xda, 0xda, 0x07, 0x0f, 0x38, 0x38, 0xd3, 0xcb,
+	0xc1, 0xb0, 0xad, 0xec, 0x72, 0xec, 0xb1, 0xa2,
+	0x7b, 0x59, 0xf3, 0x3d, 0x2b, 0xef, 0xcd, 0x28,
+	0x5b, 0x83, 0xcc, 0x18, 0x91, 0x88, 0xb0, 0x2e,
+	0xf9, 0x29, 0x31, 0x18, 0xf9, 0x4e, 0xe9, 0x0a,
+	0x91, 0x92, 0x9f, 0xae, 0x2d, 0xad, 0xf4, 0xe6,
+	0x1a, 0xe2, 0xa4, 0xee, 0x47, 0x15, 0xbf, 0x83,
+	0x6e, 0xd7, 0x72, 0x12, 0x3b, 0x2d, 0x24, 0xe9,
+	0xb2, 0x55, 0xcb, 0x3c, 0x10, 0xf0, 0x24, 0x8a,
+	0x4a, 0x02, 0xea, 0x90, 0x25, 0xf0, 0xb4, 0x79,
+	0x3a, 0xef, 0x6e, 0xf5, 0x52, 0xdf, 0xb0, 0x0a,
+	0xcd, 0x24, 0x1c, 0xd3, 0x2e, 0x22, 0x74, 0xea,
+	0x21, 0x6f, 0xe9, 0xbd, 0xc8, 0x3e, 0x36, 0x5b,
+	0x19, 0xf1, 0xca, 0x99, 0x0a, 0xb4, 0xa7, 0x52,
+	0x1a, 0x4e, 0xf2, 0xad, 0x8d, 0x56, 0x85, 0xbb,
+	0x64, 0x89, 0xba, 0x26, 0xf9, 0xc7, 0xe1, 0x89,
+	0x19, 0x22, 0x77, 0xc3, 0xa8, 0xfc, 0xff, 0xad,
+	0xfe, 0xb9, 0x48, 0xae, 0x12, 0x30, 0x9f, 0x19,
+	0xfb, 0x1b, 0xef, 0x14, 0x87, 0x8a, 0x78, 0x71,
+	0xf3, 0xf4, 0xb7, 0x00, 0x9c, 0x1d, 0xb5, 0x3d,
+	0x49, 0x00, 0x0c, 0x06, 0xd4, 0x50, 0xf9, 0x54,
+	0x45, 0xb2, 0x5b, 0x43, 0xdb, 0x6d, 0xcf, 0x1a,
+	0xe9, 0x7a, 0x7a, 0xcf, 0xfc, 0x8a, 0x4e, 0x4d,
+	0x0b, 0x07, 0x63, 0x28, 0xd8, 0xe7, 0x08, 0x95,
+	0xdf, 0xa6, 0x72, 0x93, 0x2e, 0xbb, 0xa0, 0x42,
+	0x89, 0x16, 0xf1, 0xd9, 0x0c, 0xf9, 0xa1, 0x16,
+	0xfd, 0xd9, 0x03, 0xb4, 0x3b, 0x8a, 0xf5, 0xf6,
+	0xe7, 0x6b, 0x2e, 0x8e, 0x4c, 0x3d, 0xe2, 0xaf,
+	0x08, 0x45, 0x03, 0xff, 0x09, 0xb6, 0xeb, 0x2d,
+	0xc6, 0x1b, 0x88, 0x94, 0xac, 0x3e, 0xf1, 0x9f,
+	0x0e, 0x0e, 0x2b, 0xd5, 0x00, 0x4d, 0x3f, 0x3b,
+	0x53, 0xae, 0xaf, 0x1c, 0x33, 0x5f, 0x55, 0x6e,
+	0x8d, 0xaf, 0x05, 0x7a, 0x10, 0x34, 0xc9, 0xf4,
+	0x66, 0xcb, 0x62, 0x12, 0xa6, 0xee, 0xe8, 0x1c,
+	0x5d, 0x12, 0x86, 0xdb, 0x6f, 0x1c, 0x33, 0xc4,
+	0x1c, 0xda, 0x82, 0x2d, 0x3b, 0x59, 0xfe, 0xb1,
+	0xa4, 0x59, 0x41, 0x86, 0xd0, 0xef, 0xae, 0xfb,
+	0xda, 0x6d, 0x11, 0xb8, 0xca, 0xe9, 0x6e, 0xff,
+	0xf7, 0xa9, 0xd9, 0x70, 0x30, 0xfc, 0x53, 0xe2,
+	0xd7, 0xa2, 0x4e, 0xc7, 0x91, 0xd9, 0x07, 0x06,
+	0xaa, 0xdd, 0xb0, 0x59, 0x28, 0x1d, 0x00, 0x66,
+	0xc5, 0x54, 0xc2, 0xfc, 0x06, 0xda, 0x05, 0x90,
+	0x52, 0x1d, 0x37, 0x66, 0xee, 0xf0, 0xb2, 0x55,
+	0x8a, 0x5d, 0xd2, 0x38, 0x86, 0x94, 0x9b, 0xfc,
+	0x10, 0x4c, 0xa1, 0xb9, 0x64, 0x3e, 0x44, 0xb8,
+	0x5f, 0xb0, 0x0c, 0xec, 0xe0, 0xc9, 0xe5, 0x62,
+	0x75, 0x3f, 0x09, 0xd5, 0xf5, 0xd9, 0x26, 0xba,
+	0x9e, 0xd2, 0xf4, 0xb9, 0x48, 0x0a, 0xbc, 0xa2,
+	0xd6, 0x7c, 0x36, 0x11, 0x7d, 0x26, 0x81, 0x89,
+	0xcf, 0xa4, 0xad, 0x73, 0x0e, 0xee, 0xcc, 0x06,
+	0xa9, 0xdb, 0xb1, 0xfd, 0xfb, 0x09, 0x7f, 0x90,
+	0x42, 0x37, 0x2f, 0xe1, 0x9c, 0x0f, 0x6f, 0xcf,
+	0x43, 0xb5, 0xd9, 0x90, 0xe1, 0x85, 0xf5, 0xa8,
+	0xae
+};
+static const u8 enc_assoc009[] __initconst = {
+	0x5a, 0x27, 0xff, 0xeb, 0xdf, 0x84, 0xb2, 0x9e,
+	0xef
+};
+static const u8 enc_nonce009[] __initconst = {
+	0xef, 0x2d, 0x63, 0xee, 0x6b, 0x80, 0x8b, 0x78
+};
+static const u8 enc_key009[] __initconst = {
+	0xea, 0xbc, 0x56, 0x99, 0xe3, 0x50, 0xff, 0xc5,
+	0xcc, 0x1a, 0xd7, 0xc1, 0x57, 0x72, 0xea, 0x86,
+	0x5b, 0x89, 0x88, 0x61, 0x3d, 0x2f, 0x9b, 0xb2,
+	0xe7, 0x9c, 0xec, 0x74, 0x6e, 0x3e, 0xf4, 0x3b
+};
+
+static const u8 enc_input010[] __initconst = {
+	0x42, 0x93, 0xe4, 0xeb, 0x97, 0xb0, 0x57, 0xbf,
+	0x1a, 0x8b, 0x1f, 0xe4, 0x5f, 0x36, 0x20, 0x3c,
+	0xef, 0x0a, 0xa9, 0x48, 0x5f, 0x5f, 0x37, 0x22,
+	0x3a, 0xde, 0xe3, 0xae, 0xbe, 0xad, 0x07, 0xcc,
+	0xb1, 0xf6, 0xf5, 0xf9, 0x56, 0xdd, 0xe7, 0x16,
+	0x1e, 0x7f, 0xdf, 0x7a, 0x9e, 0x75, 0xb7, 0xc7,
+	0xbe, 0xbe, 0x8a, 0x36, 0x04, 0xc0, 0x10, 0xf4,
+	0x95, 0x20, 0x03, 0xec, 0xdc, 0x05, 0xa1, 0x7d,
+	0xc4, 0xa9, 0x2c, 0x82, 0xd0, 0xbc, 0x8b, 0xc5,
+	0xc7, 0x45, 0x50, 0xf6, 0xa2, 0x1a, 0xb5, 0x46,
+	0x3b, 0x73, 0x02, 0xa6, 0x83, 0x4b, 0x73, 0x82,
+	0x58, 0x5e, 0x3b, 0x65, 0x2f, 0x0e, 0xfd, 0x2b,
+	0x59, 0x16, 0xce, 0xa1, 0x60, 0x9c, 0xe8, 0x3a,
+	0x99, 0xed, 0x8d, 0x5a, 0xcf, 0xf6, 0x83, 0xaf,
+	0xba, 0xd7, 0x73, 0x73, 0x40, 0x97, 0x3d, 0xca,
+	0xef, 0x07, 0x57, 0xe6, 0xd9, 0x70, 0x0e, 0x95,
+	0xae, 0xa6, 0x8d, 0x04, 0xcc, 0xee, 0xf7, 0x09,
+	0x31, 0x77, 0x12, 0xa3, 0x23, 0x97, 0x62, 0xb3,
+	0x7b, 0x32, 0xfb, 0x80, 0x14, 0x48, 0x81, 0xc3,
+	0xe5, 0xea, 0x91, 0x39, 0x52, 0x81, 0xa2, 0x4f,
+	0xe4, 0xb3, 0x09, 0xff, 0xde, 0x5e, 0xe9, 0x58,
+	0x84, 0x6e, 0xf9, 0x3d, 0xdf, 0x25, 0xea, 0xad,
+	0xae, 0xe6, 0x9a, 0xd1, 0x89, 0x55, 0xd3, 0xde,
+	0x6c, 0x52, 0xdb, 0x70, 0xfe, 0x37, 0xce, 0x44,
+	0x0a, 0xa8, 0x25, 0x5f, 0x92, 0xc1, 0x33, 0x4a,
+	0x4f, 0x9b, 0x62, 0x35, 0xff, 0xce, 0xc0, 0xa9,
+	0x60, 0xce, 0x52, 0x00, 0x97, 0x51, 0x35, 0x26,
+	0x2e, 0xb9, 0x36, 0xa9, 0x87, 0x6e, 0x1e, 0xcc,
+	0x91, 0x78, 0x53, 0x98, 0x86, 0x5b, 0x9c, 0x74,
+	0x7d, 0x88, 0x33, 0xe1, 0xdf, 0x37, 0x69, 0x2b,
+	0xbb, 0xf1, 0x4d, 0xf4, 0xd1, 0xf1, 0x39, 0x93,
+	0x17, 0x51, 0x19, 0xe3, 0x19, 0x1e, 0x76, 0x37,
+	0x25, 0xfb, 0x09, 0x27, 0x6a, 0xab, 0x67, 0x6f,
+	0x14, 0x12, 0x64, 0xe7, 0xc4, 0x07, 0xdf, 0x4d,
+	0x17, 0xbb, 0x6d, 0xe0, 0xe9, 0xb9, 0xab, 0xca,
+	0x10, 0x68, 0xaf, 0x7e, 0xb7, 0x33, 0x54, 0x73,
+	0x07, 0x6e, 0xf7, 0x81, 0x97, 0x9c, 0x05, 0x6f,
+	0x84, 0x5f, 0xd2, 0x42, 0xfb, 0x38, 0xcf, 0xd1,
+	0x2f, 0x14, 0x30, 0x88, 0x98, 0x4d, 0x5a, 0xa9,
+	0x76, 0xd5, 0x4f, 0x3e, 0x70, 0x6c, 0x85, 0x76,
+	0xd7, 0x01, 0xa0, 0x1a, 0xc8, 0x4e, 0xaa, 0xac,
+	0x78, 0xfe, 0x46, 0xde, 0x6a, 0x05, 0x46, 0xa7,
+	0x43, 0x0c, 0xb9, 0xde, 0xb9, 0x68, 0xfb, 0xce,
+	0x42, 0x99, 0x07, 0x4d, 0x0b, 0x3b, 0x5a, 0x30,
+	0x35, 0xa8, 0xf9, 0x3a, 0x73, 0xef, 0x0f, 0xdb,
+	0x1e, 0x16, 0x42, 0xc4, 0xba, 0xae, 0x58, 0xaa,
+	0xf8, 0xe5, 0x75, 0x2f, 0x1b, 0x15, 0x5c, 0xfd,
+	0x0a, 0x97, 0xd0, 0xe4, 0x37, 0x83, 0x61, 0x5f,
+	0x43, 0xa6, 0xc7, 0x3f, 0x38, 0x59, 0xe6, 0xeb,
+	0xa3, 0x90, 0xc3, 0xaa, 0xaa, 0x5a, 0xd3, 0x34,
+	0xd4, 0x17, 0xc8, 0x65, 0x3e, 0x57, 0xbc, 0x5e,
+	0xdd, 0x9e, 0xb7, 0xf0, 0x2e, 0x5b, 0xb2, 0x1f,
+	0x8a, 0x08, 0x0d, 0x45, 0x91, 0x0b, 0x29, 0x53,
+	0x4f, 0x4c, 0x5a, 0x73, 0x56, 0xfe, 0xaf, 0x41,
+	0x01, 0x39, 0x0a, 0x24, 0x3c, 0x7e, 0xbe, 0x4e,
+	0x53, 0xf3, 0xeb, 0x06, 0x66, 0x51, 0x28, 0x1d,
+	0xbd, 0x41, 0x0a, 0x01, 0xab, 0x16, 0x47, 0x27,
+	0x47, 0x47, 0xf7, 0xcb, 0x46, 0x0a, 0x70, 0x9e,
+	0x01, 0x9c, 0x09, 0xe1, 0x2a, 0x00, 0x1a, 0xd8,
+	0xd4, 0x79, 0x9d, 0x80, 0x15, 0x8e, 0x53, 0x2a,
+	0x65, 0x83, 0x78, 0x3e, 0x03, 0x00, 0x07, 0x12,
+	0x1f, 0x33, 0x3e, 0x7b, 0x13, 0x37, 0xf1, 0xc3,
+	0xef, 0xb7, 0xc1, 0x20, 0x3c, 0x3e, 0x67, 0x66,
+	0x5d, 0x88, 0xa7, 0x7d, 0x33, 0x50, 0x77, 0xb0,
+	0x28, 0x8e, 0xe7, 0x2c, 0x2e, 0x7a, 0xf4, 0x3c,
+	0x8d, 0x74, 0x83, 0xaf, 0x8e, 0x87, 0x0f, 0xe4,
+	0x50, 0xff, 0x84, 0x5c, 0x47, 0x0c, 0x6a, 0x49,
+	0xbf, 0x42, 0x86, 0x77, 0x15, 0x48, 0xa5, 0x90,
+	0x5d, 0x93, 0xd6, 0x2a, 0x11, 0xd5, 0xd5, 0x11,
+	0xaa, 0xce, 0xe7, 0x6f, 0xa5, 0xb0, 0x09, 0x2c,
+	0x8d, 0xd3, 0x92, 0xf0, 0x5a, 0x2a, 0xda, 0x5b,
+	0x1e, 0xd5, 0x9a, 0xc4, 0xc4, 0xf3, 0x49, 0x74,
+	0x41, 0xca, 0xe8, 0xc1, 0xf8, 0x44, 0xd6, 0x3c,
+	0xae, 0x6c, 0x1d, 0x9a, 0x30, 0x04, 0x4d, 0x27,
+	0x0e, 0xb1, 0x5f, 0x59, 0xa2, 0x24, 0xe8, 0xe1,
+	0x98, 0xc5, 0x6a, 0x4c, 0xfe, 0x41, 0xd2, 0x27,
+	0x42, 0x52, 0xe1, 0xe9, 0x7d, 0x62, 0xe4, 0x88,
+	0x0f, 0xad, 0xb2, 0x70, 0xcb, 0x9d, 0x4c, 0x27,
+	0x2e, 0x76, 0x1e, 0x1a, 0x63, 0x65, 0xf5, 0x3b,
+	0xf8, 0x57, 0x69, 0xeb, 0x5b, 0x38, 0x26, 0x39,
+	0x33, 0x25, 0x45, 0x3e, 0x91, 0xb8, 0xd8, 0xc7,
+	0xd5, 0x42, 0xc0, 0x22, 0x31, 0x74, 0xf4, 0xbc,
+	0x0c, 0x23, 0xf1, 0xca, 0xc1, 0x8d, 0xd7, 0xbe,
+	0xc9, 0x62, 0xe4, 0x08, 0x1a, 0xcf, 0x36, 0xd5,
+	0xfe, 0x55, 0x21, 0x59, 0x91, 0x87, 0x87, 0xdf,
+	0x06, 0xdb, 0xdf, 0x96, 0x45, 0x58, 0xda, 0x05,
+	0xcd, 0x50, 0x4d, 0xd2, 0x7d, 0x05, 0x18, 0x73,
+	0x6a, 0x8d, 0x11, 0x85, 0xa6, 0x88, 0xe8, 0xda,
+	0xe6, 0x30, 0x33, 0xa4, 0x89, 0x31, 0x75, 0xbe,
+	0x69, 0x43, 0x84, 0x43, 0x50, 0x87, 0xdd, 0x71,
+	0x36, 0x83, 0xc3, 0x78, 0x74, 0x24, 0x0a, 0xed,
+	0x7b, 0xdb, 0xa4, 0x24, 0x0b, 0xb9, 0x7e, 0x5d,
+	0xff, 0xde, 0xb1, 0xef, 0x61, 0x5a, 0x45, 0x33,
+	0xf6, 0x17, 0x07, 0x08, 0x98, 0x83, 0x92, 0x0f,
+	0x23, 0x6d, 0xe6, 0xaa, 0x17, 0x54, 0xad, 0x6a,
+	0xc8, 0xdb, 0x26, 0xbe, 0xb8, 0xb6, 0x08, 0xfa,
+	0x68, 0xf1, 0xd7, 0x79, 0x6f, 0x18, 0xb4, 0x9e,
+	0x2d, 0x3f, 0x1b, 0x64, 0xaf, 0x8d, 0x06, 0x0e,
+	0x49, 0x28, 0xe0, 0x5d, 0x45, 0x68, 0x13, 0x87,
+	0xfa, 0xde, 0x40, 0x7b, 0xd2, 0xc3, 0x94, 0xd5,
+	0xe1, 0xd9, 0xc2, 0xaf, 0x55, 0x89, 0xeb, 0xb4,
+	0x12, 0x59, 0xa8, 0xd4, 0xc5, 0x29, 0x66, 0x38,
+	0xe6, 0xac, 0x22, 0x22, 0xd9, 0x64, 0x9b, 0x34,
+	0x0a, 0x32, 0x9f, 0xc2, 0xbf, 0x17, 0x6c, 0x3f,
+	0x71, 0x7a, 0x38, 0x6b, 0x98, 0xfb, 0x49, 0x36,
+	0x89, 0xc9, 0xe2, 0xd6, 0xc7, 0x5d, 0xd0, 0x69,
+	0x5f, 0x23, 0x35, 0xc9, 0x30, 0xe2, 0xfd, 0x44,
+	0x58, 0x39, 0xd7, 0x97, 0xfb, 0x5c, 0x00, 0xd5,
+	0x4f, 0x7a, 0x1a, 0x95, 0x8b, 0x62, 0x4b, 0xce,
+	0xe5, 0x91, 0x21, 0x7b, 0x30, 0x00, 0xd6, 0xdd,
+	0x6d, 0x02, 0x86, 0x49, 0x0f, 0x3c, 0x1a, 0x27,
+	0x3c, 0xd3, 0x0e, 0x71, 0xf2, 0xff, 0xf5, 0x2f,
+	0x87, 0xac, 0x67, 0x59, 0x81, 0xa3, 0xf7, 0xf8,
+	0xd6, 0x11, 0x0c, 0x84, 0xa9, 0x03, 0xee, 0x2a,
+	0xc4, 0xf3, 0x22, 0xab, 0x7c, 0xe2, 0x25, 0xf5,
+	0x67, 0xa3, 0xe4, 0x11, 0xe0, 0x59, 0xb3, 0xca,
+	0x87, 0xa0, 0xae, 0xc9, 0xa6, 0x62, 0x1b, 0x6e,
+	0x4d, 0x02, 0x6b, 0x07, 0x9d, 0xfd, 0xd0, 0x92,
+	0x06, 0xe1, 0xb2, 0x9a, 0x4a, 0x1f, 0x1f, 0x13,
+	0x49, 0x99, 0x97, 0x08, 0xde, 0x7f, 0x98, 0xaf,
+	0x51, 0x98, 0xee, 0x2c, 0xcb, 0xf0, 0x0b, 0xc6,
+	0xb6, 0xb7, 0x2d, 0x9a, 0xb1, 0xac, 0xa6, 0xe3,
+	0x15, 0x77, 0x9d, 0x6b, 0x1a, 0xe4, 0xfc, 0x8b,
+	0xf2, 0x17, 0x59, 0x08, 0x04, 0x58, 0x81, 0x9d,
+	0x1b, 0x1b, 0x69, 0x55, 0xc2, 0xb4, 0x3c, 0x1f,
+	0x50, 0xf1, 0x7f, 0x77, 0x90, 0x4c, 0x66, 0x40,
+	0x5a, 0xc0, 0x33, 0x1f, 0xcb, 0x05, 0x6d, 0x5c,
+	0x06, 0x87, 0x52, 0xa2, 0x8f, 0x26, 0xd5, 0x4f
+};
+static const u8 enc_output010[] __initconst = {
+	0xe5, 0x26, 0xa4, 0x3d, 0xbd, 0x33, 0xd0, 0x4b,
+	0x6f, 0x05, 0xa7, 0x6e, 0x12, 0x7a, 0xd2, 0x74,
+	0xa6, 0xdd, 0xbd, 0x95, 0xeb, 0xf9, 0xa4, 0xf1,
+	0x59, 0x93, 0x91, 0x70, 0xd9, 0xfe, 0x9a, 0xcd,
+	0x53, 0x1f, 0x3a, 0xab, 0xa6, 0x7c, 0x9f, 0xa6,
+	0x9e, 0xbd, 0x99, 0xd9, 0xb5, 0x97, 0x44, 0xd5,
+	0x14, 0x48, 0x4d, 0x9d, 0xc0, 0xd0, 0x05, 0x96,
+	0xeb, 0x4c, 0x78, 0x55, 0x09, 0x08, 0x01, 0x02,
+	0x30, 0x90, 0x7b, 0x96, 0x7a, 0x7b, 0x5f, 0x30,
+	0x41, 0x24, 0xce, 0x68, 0x61, 0x49, 0x86, 0x57,
+	0x82, 0xdd, 0x53, 0x1c, 0x51, 0x28, 0x2b, 0x53,
+	0x6e, 0x2d, 0xc2, 0x20, 0x4c, 0xdd, 0x8f, 0x65,
+	0x10, 0x20, 0x50, 0xdd, 0x9d, 0x50, 0xe5, 0x71,
+	0x40, 0x53, 0x69, 0xfc, 0x77, 0x48, 0x11, 0xb9,
+	0xde, 0xa4, 0x8d, 0x58, 0xe4, 0xa6, 0x1a, 0x18,
+	0x47, 0x81, 0x7e, 0xfc, 0xdd, 0xf6, 0xef, 0xce,
+	0x2f, 0x43, 0x68, 0xd6, 0x06, 0xe2, 0x74, 0x6a,
+	0xad, 0x90, 0xf5, 0x37, 0xf3, 0x3d, 0x82, 0x69,
+	0x40, 0xe9, 0x6b, 0xa7, 0x3d, 0xa8, 0x1e, 0xd2,
+	0x02, 0x7c, 0xb7, 0x9b, 0xe4, 0xda, 0x8f, 0x95,
+	0x06, 0xc5, 0xdf, 0x73, 0xa3, 0x20, 0x9a, 0x49,
+	0xde, 0x9c, 0xbc, 0xee, 0x14, 0x3f, 0x81, 0x5e,
+	0xf8, 0x3b, 0x59, 0x3c, 0xe1, 0x68, 0x12, 0x5a,
+	0x3a, 0x76, 0x3a, 0x3f, 0xf7, 0x87, 0x33, 0x0a,
+	0x01, 0xb8, 0xd4, 0xed, 0xb6, 0xbe, 0x94, 0x5e,
+	0x70, 0x40, 0x56, 0x67, 0x1f, 0x50, 0x44, 0x19,
+	0xce, 0x82, 0x70, 0x10, 0x87, 0x13, 0x20, 0x0b,
+	0x4c, 0x5a, 0xb6, 0xf6, 0xa7, 0xae, 0x81, 0x75,
+	0x01, 0x81, 0xe6, 0x4b, 0x57, 0x7c, 0xdd, 0x6d,
+	0xf8, 0x1c, 0x29, 0x32, 0xf7, 0xda, 0x3c, 0x2d,
+	0xf8, 0x9b, 0x25, 0x6e, 0x00, 0xb4, 0xf7, 0x2f,
+	0xf7, 0x04, 0xf7, 0xa1, 0x56, 0xac, 0x4f, 0x1a,
+	0x64, 0xb8, 0x47, 0x55, 0x18, 0x7b, 0x07, 0x4d,
+	0xbd, 0x47, 0x24, 0x80, 0x5d, 0xa2, 0x70, 0xc5,
+	0xdd, 0x8e, 0x82, 0xd4, 0xeb, 0xec, 0xb2, 0x0c,
+	0x39, 0xd2, 0x97, 0xc1, 0xcb, 0xeb, 0xf4, 0x77,
+	0x59, 0xb4, 0x87, 0xef, 0xcb, 0x43, 0x2d, 0x46,
+	0x54, 0xd1, 0xa7, 0xd7, 0x15, 0x99, 0x0a, 0x43,
+	0xa1, 0xe0, 0x99, 0x33, 0x71, 0xc1, 0xed, 0xfe,
+	0x72, 0x46, 0x33, 0x8e, 0x91, 0x08, 0x9f, 0xc8,
+	0x2e, 0xca, 0xfa, 0xdc, 0x59, 0xd5, 0xc3, 0x76,
+	0x84, 0x9f, 0xa3, 0x37, 0x68, 0xc3, 0xf0, 0x47,
+	0x2c, 0x68, 0xdb, 0x5e, 0xc3, 0x49, 0x4c, 0xe8,
+	0x92, 0x85, 0xe2, 0x23, 0xd3, 0x3f, 0xad, 0x32,
+	0xe5, 0x2b, 0x82, 0xd7, 0x8f, 0x99, 0x0a, 0x59,
+	0x5c, 0x45, 0xd9, 0xb4, 0x51, 0x52, 0xc2, 0xae,
+	0xbf, 0x80, 0xcf, 0xc9, 0xc9, 0x51, 0x24, 0x2a,
+	0x3b, 0x3a, 0x4d, 0xae, 0xeb, 0xbd, 0x22, 0xc3,
+	0x0e, 0x0f, 0x59, 0x25, 0x92, 0x17, 0xe9, 0x74,
+	0xc7, 0x8b, 0x70, 0x70, 0x36, 0x55, 0x95, 0x75,
+	0x4b, 0xad, 0x61, 0x2b, 0x09, 0xbc, 0x82, 0xf2,
+	0x6e, 0x94, 0x43, 0xae, 0xc3, 0xd5, 0xcd, 0x8e,
+	0xfe, 0x5b, 0x9a, 0x88, 0x43, 0x01, 0x75, 0xb2,
+	0x23, 0x09, 0xf7, 0x89, 0x83, 0xe7, 0xfa, 0xf9,
+	0xb4, 0x9b, 0xf8, 0xef, 0xbd, 0x1c, 0x92, 0xc1,
+	0xda, 0x7e, 0xfe, 0x05, 0xba, 0x5a, 0xcd, 0x07,
+	0x6a, 0x78, 0x9e, 0x5d, 0xfb, 0x11, 0x2f, 0x79,
+	0x38, 0xb6, 0xc2, 0x5b, 0x6b, 0x51, 0xb4, 0x71,
+	0xdd, 0xf7, 0x2a, 0xe4, 0xf4, 0x72, 0x76, 0xad,
+	0xc2, 0xdd, 0x64, 0x5d, 0x79, 0xb6, 0xf5, 0x7a,
+	0x77, 0x20, 0x05, 0x3d, 0x30, 0x06, 0xd4, 0x4c,
+	0x0a, 0x2c, 0x98, 0x5a, 0xb9, 0xd4, 0x98, 0xa9,
+	0x3f, 0xc6, 0x12, 0xea, 0x3b, 0x4b, 0xc5, 0x79,
+	0x64, 0x63, 0x6b, 0x09, 0x54, 0x3b, 0x14, 0x27,
+	0xba, 0x99, 0x80, 0xc8, 0x72, 0xa8, 0x12, 0x90,
+	0x29, 0xba, 0x40, 0x54, 0x97, 0x2b, 0x7b, 0xfe,
+	0xeb, 0xcd, 0x01, 0x05, 0x44, 0x72, 0xdb, 0x99,
+	0xe4, 0x61, 0xc9, 0x69, 0xd6, 0xb9, 0x28, 0xd1,
+	0x05, 0x3e, 0xf9, 0x0b, 0x49, 0x0a, 0x49, 0xe9,
+	0x8d, 0x0e, 0xa7, 0x4a, 0x0f, 0xaf, 0x32, 0xd0,
+	0xe0, 0xb2, 0x3a, 0x55, 0x58, 0xfe, 0x5c, 0x28,
+	0x70, 0x51, 0x23, 0xb0, 0x7b, 0x6a, 0x5f, 0x1e,
+	0xb8, 0x17, 0xd7, 0x94, 0x15, 0x8f, 0xee, 0x20,
+	0xc7, 0x42, 0x25, 0x3e, 0x9a, 0x14, 0xd7, 0x60,
+	0x72, 0x39, 0x47, 0x48, 0xa9, 0xfe, 0xdd, 0x47,
+	0x0a, 0xb1, 0xe6, 0x60, 0x28, 0x8c, 0x11, 0x68,
+	0xe1, 0xff, 0xd7, 0xce, 0xc8, 0xbe, 0xb3, 0xfe,
+	0x27, 0x30, 0x09, 0x70, 0xd7, 0xfa, 0x02, 0x33,
+	0x3a, 0x61, 0x2e, 0xc7, 0xff, 0xa4, 0x2a, 0xa8,
+	0x6e, 0xb4, 0x79, 0x35, 0x6d, 0x4c, 0x1e, 0x38,
+	0xf8, 0xee, 0xd4, 0x84, 0x4e, 0x6e, 0x28, 0xa7,
+	0xce, 0xc8, 0xc1, 0xcf, 0x80, 0x05, 0xf3, 0x04,
+	0xef, 0xc8, 0x18, 0x28, 0x2e, 0x8d, 0x5e, 0x0c,
+	0xdf, 0xb8, 0x5f, 0x96, 0xe8, 0xc6, 0x9c, 0x2f,
+	0xe5, 0xa6, 0x44, 0xd7, 0xe7, 0x99, 0x44, 0x0c,
+	0xec, 0xd7, 0x05, 0x60, 0x97, 0xbb, 0x74, 0x77,
+	0x58, 0xd5, 0xbb, 0x48, 0xde, 0x5a, 0xb2, 0x54,
+	0x7f, 0x0e, 0x46, 0x70, 0x6a, 0x6f, 0x78, 0xa5,
+	0x08, 0x89, 0x05, 0x4e, 0x7e, 0xa0, 0x69, 0xb4,
+	0x40, 0x60, 0x55, 0x77, 0x75, 0x9b, 0x19, 0xf2,
+	0xd5, 0x13, 0x80, 0x77, 0xf9, 0x4b, 0x3f, 0x1e,
+	0xee, 0xe6, 0x76, 0x84, 0x7b, 0x8c, 0xe5, 0x27,
+	0xa8, 0x0a, 0x91, 0x01, 0x68, 0x71, 0x8a, 0x3f,
+	0x06, 0xab, 0xf6, 0xa9, 0xa5, 0xe6, 0x72, 0x92,
+	0xe4, 0x67, 0xe2, 0xa2, 0x46, 0x35, 0x84, 0x55,
+	0x7d, 0xca, 0xa8, 0x85, 0xd0, 0xf1, 0x3f, 0xbe,
+	0xd7, 0x34, 0x64, 0xfc, 0xae, 0xe3, 0xe4, 0x04,
+	0x9f, 0x66, 0x02, 0xb9, 0x88, 0x10, 0xd9, 0xc4,
+	0x4c, 0x31, 0x43, 0x7a, 0x93, 0xe2, 0x9b, 0x56,
+	0x43, 0x84, 0xdc, 0xdc, 0xde, 0x1d, 0xa4, 0x02,
+	0x0e, 0xc2, 0xef, 0xc3, 0xf8, 0x78, 0xd1, 0xb2,
+	0x6b, 0x63, 0x18, 0xc9, 0xa9, 0xe5, 0x72, 0xd8,
+	0xf3, 0xb9, 0xd1, 0x8a, 0xc7, 0x1a, 0x02, 0x27,
+	0x20, 0x77, 0x10, 0xe5, 0xc8, 0xd4, 0x4a, 0x47,
+	0xe5, 0xdf, 0x5f, 0x01, 0xaa, 0xb0, 0xd4, 0x10,
+	0xbb, 0x69, 0xe3, 0x36, 0xc8, 0xe1, 0x3d, 0x43,
+	0xfb, 0x86, 0xcd, 0xcc, 0xbf, 0xf4, 0x88, 0xe0,
+	0x20, 0xca, 0xb7, 0x1b, 0xf1, 0x2f, 0x5c, 0xee,
+	0xd4, 0xd3, 0xa3, 0xcc, 0xa4, 0x1e, 0x1c, 0x47,
+	0xfb, 0xbf, 0xfc, 0xa2, 0x41, 0x55, 0x9d, 0xf6,
+	0x5a, 0x5e, 0x65, 0x32, 0x34, 0x7b, 0x52, 0x8d,
+	0xd5, 0xd0, 0x20, 0x60, 0x03, 0xab, 0x3f, 0x8c,
+	0xd4, 0x21, 0xea, 0x2a, 0xd9, 0xc4, 0xd0, 0xd3,
+	0x65, 0xd8, 0x7a, 0x13, 0x28, 0x62, 0x32, 0x4b,
+	0x2c, 0x87, 0x93, 0xa8, 0xb4, 0x52, 0x45, 0x09,
+	0x44, 0xec, 0xec, 0xc3, 0x17, 0xdb, 0x9a, 0x4d,
+	0x5c, 0xa9, 0x11, 0xd4, 0x7d, 0xaf, 0x9e, 0xf1,
+	0x2d, 0xb2, 0x66, 0xc5, 0x1d, 0xed, 0xb7, 0xcd,
+	0x0b, 0x25, 0x5e, 0x30, 0x47, 0x3f, 0x40, 0xf4,
+	0xa1, 0xa0, 0x00, 0x94, 0x10, 0xc5, 0x6a, 0x63,
+	0x1a, 0xd5, 0x88, 0x92, 0x8e, 0x82, 0x39, 0x87,
+	0x3c, 0x78, 0x65, 0x58, 0x42, 0x75, 0x5b, 0xdd,
+	0x77, 0x3e, 0x09, 0x4e, 0x76, 0x5b, 0xe6, 0x0e,
+	0x4d, 0x38, 0xb2, 0xc0, 0xb8, 0x95, 0x01, 0x7a,
+	0x10, 0xe0, 0xfb, 0x07, 0xf2, 0xab, 0x2d, 0x8c,
+	0x32, 0xed, 0x2b, 0xc0, 0x46, 0xc2, 0xf5, 0x38,
+	0x83, 0xf0, 0x17, 0xec, 0xc1, 0x20, 0x6a, 0x9a,
+	0x0b, 0x00, 0xa0, 0x98, 0x22, 0x50, 0x23, 0xd5,
+	0x80, 0x6b, 0xf6, 0x1f, 0xc3, 0xcc, 0x97, 0xc9,
+	0x24, 0x9f, 0xf3, 0xaf, 0x43, 0x14, 0xd5, 0xa0
+};
+static const u8 enc_assoc010[] __initconst = {
+	0xd2, 0xa1, 0x70, 0xdb, 0x7a, 0xf8, 0xfa, 0x27,
+	0xba, 0x73, 0x0f, 0xbf, 0x3d, 0x1e, 0x82, 0xb2
+};
+static const u8 enc_nonce010[] __initconst = {
+	0xdb, 0x92, 0x0f, 0x7f, 0x17, 0x54, 0x0c, 0x30
+};
+static const u8 enc_key010[] __initconst = {
+	0x47, 0x11, 0xeb, 0x86, 0x2b, 0x2c, 0xab, 0x44,
+	0x34, 0xda, 0x7f, 0x57, 0x03, 0x39, 0x0c, 0xaf,
+	0x2c, 0x14, 0xfd, 0x65, 0x23, 0xe9, 0x8e, 0x74,
+	0xd5, 0x08, 0x68, 0x08, 0xe7, 0xb4, 0x72, 0xd7
+};
+
+static const u8 enc_input011[] __initconst = {
+	0x7a, 0x57, 0xf2, 0xc7, 0x06, 0x3f, 0x50, 0x7b,
+	0x36, 0x1a, 0x66, 0x5c, 0xb9, 0x0e, 0x5e, 0x3b,
+	0x45, 0x60, 0xbe, 0x9a, 0x31, 0x9f, 0xff, 0x5d,
+	0x66, 0x34, 0xb4, 0xdc, 0xfb, 0x9d, 0x8e, 0xee,
+	0x6a, 0x33, 0xa4, 0x07, 0x3c, 0xf9, 0x4c, 0x30,
+	0xa1, 0x24, 0x52, 0xf9, 0x50, 0x46, 0x88, 0x20,
+	0x02, 0x32, 0x3a, 0x0e, 0x99, 0x63, 0xaf, 0x1f,
+	0x15, 0x28, 0x2a, 0x05, 0xff, 0x57, 0x59, 0x5e,
+	0x18, 0xa1, 0x1f, 0xd0, 0x92, 0x5c, 0x88, 0x66,
+	0x1b, 0x00, 0x64, 0xa5, 0x93, 0x8d, 0x06, 0x46,
+	0xb0, 0x64, 0x8b, 0x8b, 0xef, 0x99, 0x05, 0x35,
+	0x85, 0xb3, 0xf3, 0x33, 0xbb, 0xec, 0x66, 0xb6,
+	0x3d, 0x57, 0x42, 0xe3, 0xb4, 0xc6, 0xaa, 0xb0,
+	0x41, 0x2a, 0xb9, 0x59, 0xa9, 0xf6, 0x3e, 0x15,
+	0x26, 0x12, 0x03, 0x21, 0x4c, 0x74, 0x43, 0x13,
+	0x2a, 0x03, 0x27, 0x09, 0xb4, 0xfb, 0xe7, 0xb7,
+	0x40, 0xff, 0x5e, 0xce, 0x48, 0x9a, 0x60, 0xe3,
+	0x8b, 0x80, 0x8c, 0x38, 0x2d, 0xcb, 0x93, 0x37,
+	0x74, 0x05, 0x52, 0x6f, 0x73, 0x3e, 0xc3, 0xbc,
+	0xca, 0x72, 0x0a, 0xeb, 0xf1, 0x3b, 0xa0, 0x95,
+	0xdc, 0x8a, 0xc4, 0xa9, 0xdc, 0xca, 0x44, 0xd8,
+	0x08, 0x63, 0x6a, 0x36, 0xd3, 0x3c, 0xb8, 0xac,
+	0x46, 0x7d, 0xfd, 0xaa, 0xeb, 0x3e, 0x0f, 0x45,
+	0x8f, 0x49, 0xda, 0x2b, 0xf2, 0x12, 0xbd, 0xaf,
+	0x67, 0x8a, 0x63, 0x48, 0x4b, 0x55, 0x5f, 0x6d,
+	0x8c, 0xb9, 0x76, 0x34, 0x84, 0xae, 0xc2, 0xfc,
+	0x52, 0x64, 0x82, 0xf7, 0xb0, 0x06, 0xf0, 0x45,
+	0x73, 0x12, 0x50, 0x30, 0x72, 0xea, 0x78, 0x9a,
+	0xa8, 0xaf, 0xb5, 0xe3, 0xbb, 0x77, 0x52, 0xec,
+	0x59, 0x84, 0xbf, 0x6b, 0x8f, 0xce, 0x86, 0x5e,
+	0x1f, 0x23, 0xe9, 0xfb, 0x08, 0x86, 0xf7, 0x10,
+	0xb9, 0xf2, 0x44, 0x96, 0x44, 0x63, 0xa9, 0xa8,
+	0x78, 0x00, 0x23, 0xd6, 0xc7, 0xe7, 0x6e, 0x66,
+	0x4f, 0xcc, 0xee, 0x15, 0xb3, 0xbd, 0x1d, 0xa0,
+	0xe5, 0x9c, 0x1b, 0x24, 0x2c, 0x4d, 0x3c, 0x62,
+	0x35, 0x9c, 0x88, 0x59, 0x09, 0xdd, 0x82, 0x1b,
+	0xcf, 0x0a, 0x83, 0x6b, 0x3f, 0xae, 0x03, 0xc4,
+	0xb4, 0xdd, 0x7e, 0x5b, 0x28, 0x76, 0x25, 0x96,
+	0xd9, 0xc9, 0x9d, 0x5f, 0x86, 0xfa, 0xf6, 0xd7,
+	0xd2, 0xe6, 0x76, 0x1d, 0x0f, 0xa1, 0xdc, 0x74,
+	0x05, 0x1b, 0x1d, 0xe0, 0xcd, 0x16, 0xb0, 0xa8,
+	0x8a, 0x34, 0x7b, 0x15, 0x11, 0x77, 0xe5, 0x7b,
+	0x7e, 0x20, 0xf7, 0xda, 0x38, 0xda, 0xce, 0x70,
+	0xe9, 0xf5, 0x6c, 0xd9, 0xbe, 0x0c, 0x4c, 0x95,
+	0x4c, 0xc2, 0x9b, 0x34, 0x55, 0x55, 0xe1, 0xf3,
+	0x46, 0x8e, 0x48, 0x74, 0x14, 0x4f, 0x9d, 0xc9,
+	0xf5, 0xe8, 0x1a, 0xf0, 0x11, 0x4a, 0xc1, 0x8d,
+	0xe0, 0x93, 0xa0, 0xbe, 0x09, 0x1c, 0x2b, 0x4e,
+	0x0f, 0xb2, 0x87, 0x8b, 0x84, 0xfe, 0x92, 0x32,
+	0x14, 0xd7, 0x93, 0xdf, 0xe7, 0x44, 0xbc, 0xc5,
+	0xae, 0x53, 0x69, 0xd8, 0xb3, 0x79, 0x37, 0x80,
+	0xe3, 0x17, 0x5c, 0xec, 0x53, 0x00, 0x9a, 0xe3,
+	0x8e, 0xdc, 0x38, 0xb8, 0x66, 0xf0, 0xd3, 0xad,
+	0x1d, 0x02, 0x96, 0x86, 0x3e, 0x9d, 0x3b, 0x5d,
+	0xa5, 0x7f, 0x21, 0x10, 0xf1, 0x1f, 0x13, 0x20,
+	0xf9, 0x57, 0x87, 0x20, 0xf5, 0x5f, 0xf1, 0x17,
+	0x48, 0x0a, 0x51, 0x5a, 0xcd, 0x19, 0x03, 0xa6,
+	0x5a, 0xd1, 0x12, 0x97, 0xe9, 0x48, 0xe2, 0x1d,
+	0x83, 0x75, 0x50, 0xd9, 0x75, 0x7d, 0x6a, 0x82,
+	0xa1, 0xf9, 0x4e, 0x54, 0x87, 0x89, 0xc9, 0x0c,
+	0xb7, 0x5b, 0x6a, 0x91, 0xc1, 0x9c, 0xb2, 0xa9,
+	0xdc, 0x9a, 0xa4, 0x49, 0x0a, 0x6d, 0x0d, 0xbb,
+	0xde, 0x86, 0x44, 0xdd, 0x5d, 0x89, 0x2b, 0x96,
+	0x0f, 0x23, 0x95, 0xad, 0xcc, 0xa2, 0xb3, 0xb9,
+	0x7e, 0x74, 0x38, 0xba, 0x9f, 0x73, 0xae, 0x5f,
+	0xf8, 0x68, 0xa2, 0xe0, 0xa9, 0xce, 0xbd, 0x40,
+	0xd4, 0x4c, 0x6b, 0xd2, 0x56, 0x62, 0xb0, 0xcc,
+	0x63, 0x7e, 0x5b, 0xd3, 0xae, 0xd1, 0x75, 0xce,
+	0xbb, 0xb4, 0x5b, 0xa8, 0xf8, 0xb4, 0xac, 0x71,
+	0x75, 0xaa, 0xc9, 0x9f, 0xbb, 0x6c, 0xad, 0x0f,
+	0x55, 0x5d, 0xe8, 0x85, 0x7d, 0xf9, 0x21, 0x35,
+	0xea, 0x92, 0x85, 0x2b, 0x00, 0xec, 0x84, 0x90,
+	0x0a, 0x63, 0x96, 0xe4, 0x6b, 0xa9, 0x77, 0xb8,
+	0x91, 0xf8, 0x46, 0x15, 0x72, 0x63, 0x70, 0x01,
+	0x40, 0xa3, 0xa5, 0x76, 0x62, 0x2b, 0xbf, 0xf1,
+	0xe5, 0x8d, 0x9f, 0xa3, 0xfa, 0x9b, 0x03, 0xbe,
+	0xfe, 0x65, 0x6f, 0xa2, 0x29, 0x0d, 0x54, 0xb4,
+	0x71, 0xce, 0xa9, 0xd6, 0x3d, 0x88, 0xf9, 0xaf,
+	0x6b, 0xa8, 0x9e, 0xf4, 0x16, 0x96, 0x36, 0xb9,
+	0x00, 0xdc, 0x10, 0xab, 0xb5, 0x08, 0x31, 0x1f,
+	0x00, 0xb1, 0x3c, 0xd9, 0x38, 0x3e, 0xc6, 0x04,
+	0xa7, 0x4e, 0xe8, 0xae, 0xed, 0x98, 0xc2, 0xf7,
+	0xb9, 0x00, 0x5f, 0x8c, 0x60, 0xd1, 0xe5, 0x15,
+	0xf7, 0xae, 0x1e, 0x84, 0x88, 0xd1, 0xf6, 0xbc,
+	0x3a, 0x89, 0x35, 0x22, 0x83, 0x7c, 0xca, 0xf0,
+	0x33, 0x82, 0x4c, 0x79, 0x3c, 0xfd, 0xb1, 0xae,
+	0x52, 0x62, 0x55, 0xd2, 0x41, 0x60, 0xc6, 0xbb,
+	0xfa, 0x0e, 0x59, 0xd6, 0xa8, 0xfe, 0x5d, 0xed,
+	0x47, 0x3d, 0xe0, 0xea, 0x1f, 0x6e, 0x43, 0x51,
+	0xec, 0x10, 0x52, 0x56, 0x77, 0x42, 0x6b, 0x52,
+	0x87, 0xd8, 0xec, 0xe0, 0xaa, 0x76, 0xa5, 0x84,
+	0x2a, 0x22, 0x24, 0xfd, 0x92, 0x40, 0x88, 0xd5,
+	0x85, 0x1c, 0x1f, 0x6b, 0x47, 0xa0, 0xc4, 0xe4,
+	0xef, 0xf4, 0xea, 0xd7, 0x59, 0xac, 0x2a, 0x9e,
+	0x8c, 0xfa, 0x1f, 0x42, 0x08, 0xfe, 0x4f, 0x74,
+	0xa0, 0x26, 0xf5, 0xb3, 0x84, 0xf6, 0x58, 0x5f,
+	0x26, 0x66, 0x3e, 0xd7, 0xe4, 0x22, 0x91, 0x13,
+	0xc8, 0xac, 0x25, 0x96, 0x23, 0xd8, 0x09, 0xea,
+	0x45, 0x75, 0x23, 0xb8, 0x5f, 0xc2, 0x90, 0x8b,
+	0x09, 0xc4, 0xfc, 0x47, 0x6c, 0x6d, 0x0a, 0xef,
+	0x69, 0xa4, 0x38, 0x19, 0xcf, 0x7d, 0xf9, 0x09,
+	0x73, 0x9b, 0x60, 0x5a, 0xf7, 0x37, 0xb5, 0xfe,
+	0x9f, 0xe3, 0x2b, 0x4c, 0x0d, 0x6e, 0x19, 0xf1,
+	0xd6, 0xc0, 0x70, 0xf3, 0x9d, 0x22, 0x3c, 0xf9,
+	0x49, 0xce, 0x30, 0x8e, 0x44, 0xb5, 0x76, 0x15,
+	0x8f, 0x52, 0xfd, 0xa5, 0x04, 0xb8, 0x55, 0x6a,
+	0x36, 0x59, 0x7c, 0xc4, 0x48, 0xb8, 0xd7, 0xab,
+	0x05, 0x66, 0xe9, 0x5e, 0x21, 0x6f, 0x6b, 0x36,
+	0x29, 0xbb, 0xe9, 0xe3, 0xa2, 0x9a, 0xa8, 0xcd,
+	0x55, 0x25, 0x11, 0xba, 0x5a, 0x58, 0xa0, 0xde,
+	0xae, 0x19, 0x2a, 0x48, 0x5a, 0xff, 0x36, 0xcd,
+	0x6d, 0x16, 0x7a, 0x73, 0x38, 0x46, 0xe5, 0x47,
+	0x59, 0xc8, 0xa2, 0xf6, 0xe2, 0x6c, 0x83, 0xc5,
+	0x36, 0x2c, 0x83, 0x7d, 0xb4, 0x01, 0x05, 0x69,
+	0xe7, 0xaf, 0x5c, 0xc4, 0x64, 0x82, 0x12, 0x21,
+	0xef, 0xf7, 0xd1, 0x7d, 0xb8, 0x8d, 0x8c, 0x98,
+	0x7c, 0x5f, 0x7d, 0x92, 0x88, 0xb9, 0x94, 0x07,
+	0x9c, 0xd8, 0xe9, 0x9c, 0x17, 0x38, 0xe3, 0x57,
+	0x6c, 0xe0, 0xdc, 0xa5, 0x92, 0x42, 0xb3, 0xbd,
+	0x50, 0xa2, 0x7e, 0xb5, 0xb1, 0x52, 0x72, 0x03,
+	0x97, 0xd8, 0xaa, 0x9a, 0x1e, 0x75, 0x41, 0x11,
+	0xa3, 0x4f, 0xcc, 0xd4, 0xe3, 0x73, 0xad, 0x96,
+	0xdc, 0x47, 0x41, 0x9f, 0xb0, 0xbe, 0x79, 0x91,
+	0xf5, 0xb6, 0x18, 0xfe, 0xc2, 0x83, 0x18, 0x7d,
+	0x73, 0xd9, 0x4f, 0x83, 0x84, 0x03, 0xb3, 0xf0,
+	0x77, 0x66, 0x3d, 0x83, 0x63, 0x2e, 0x2c, 0xf9,
+	0xdd, 0xa6, 0x1f, 0x89, 0x82, 0xb8, 0x23, 0x42,
+	0xeb, 0xe2, 0xca, 0x70, 0x82, 0x61, 0x41, 0x0a,
+	0x6d, 0x5f, 0x75, 0xc5, 0xe2, 0xc4, 0x91, 0x18,
+	0x44, 0x22, 0xfa, 0x34, 0x10, 0xf5, 0x20, 0xdc,
+	0xb7, 0xdd, 0x2a, 0x20, 0x77, 0xf5, 0xf9, 0xce,
+	0xdb, 0xa0, 0x0a, 0x52, 0x2a, 0x4e, 0xdd, 0xcc,
+	0x97, 0xdf, 0x05, 0xe4, 0x5e, 0xb7, 0xaa, 0xf0,
+	0xe2, 0x80, 0xff, 0xba, 0x1a, 0x0f, 0xac, 0xdf,
+	0x02, 0x32, 0xe6, 0xf7, 0xc7, 0x17, 0x13, 0xb7,
+	0xfc, 0x98, 0x48, 0x8c, 0x0d, 0x82, 0xc9, 0x80,
+	0x7a, 0xe2, 0x0a, 0xc5, 0xb4, 0xde, 0x7c, 0x3c,
+	0x79, 0x81, 0x0e, 0x28, 0x65, 0x79, 0x67, 0x82,
+	0x69, 0x44, 0x66, 0x09, 0xf7, 0x16, 0x1a, 0xf9,
+	0x7d, 0x80, 0xa1, 0x79, 0x14, 0xa9, 0xc8, 0x20,
+	0xfb, 0xa2, 0x46, 0xbe, 0x08, 0x35, 0x17, 0x58,
+	0xc1, 0x1a, 0xda, 0x2a, 0x6b, 0x2e, 0x1e, 0xe6,
+	0x27, 0x55, 0x7b, 0x19, 0xe2, 0xfb, 0x64, 0xfc,
+	0x5e, 0x15, 0x54, 0x3c, 0xe7, 0xc2, 0x11, 0x50,
+	0x30, 0xb8, 0x72, 0x03, 0x0b, 0x1a, 0x9f, 0x86,
+	0x27, 0x11, 0x5c, 0x06, 0x2b, 0xbd, 0x75, 0x1a,
+	0x0a, 0xda, 0x01, 0xfa, 0x5c, 0x4a, 0xc1, 0x80,
+	0x3a, 0x6e, 0x30, 0xc8, 0x2c, 0xeb, 0x56, 0xec,
+	0x89, 0xfa, 0x35, 0x7b, 0xb2, 0xf0, 0x97, 0x08,
+	0x86, 0x53, 0xbe, 0xbd, 0x40, 0x41, 0x38, 0x1c,
+	0xb4, 0x8b, 0x79, 0x2e, 0x18, 0x96, 0x94, 0xde,
+	0xe8, 0xca, 0xe5, 0x9f, 0x92, 0x9f, 0x15, 0x5d,
+	0x56, 0x60, 0x5c, 0x09, 0xf9, 0x16, 0xf4, 0x17,
+	0x0f, 0xf6, 0x4c, 0xda, 0xe6, 0x67, 0x89, 0x9f,
+	0xca, 0x6c, 0xe7, 0x9b, 0x04, 0x62, 0x0e, 0x26,
+	0xa6, 0x52, 0xbd, 0x29, 0xff, 0xc7, 0xa4, 0x96,
+	0xe6, 0x6a, 0x02, 0xa5, 0x2e, 0x7b, 0xfe, 0x97,
+	0x68, 0x3e, 0x2e, 0x5f, 0x3b, 0x0f, 0x36, 0xd6,
+	0x98, 0x19, 0x59, 0x48, 0xd2, 0xc6, 0xe1, 0x55,
+	0x1a, 0x6e, 0xd6, 0xed, 0x2c, 0xba, 0xc3, 0x9e,
+	0x64, 0xc9, 0x95, 0x86, 0x35, 0x5e, 0x3e, 0x88,
+	0x69, 0x99, 0x4b, 0xee, 0xbe, 0x9a, 0x99, 0xb5,
+	0x6e, 0x58, 0xae, 0xdd, 0x22, 0xdb, 0xdd, 0x6b,
+	0xfc, 0xaf, 0x90, 0xa3, 0x3d, 0xa4, 0xc1, 0x15,
+	0x92, 0x18, 0x8d, 0xd2, 0x4b, 0x7b, 0x06, 0xd1,
+	0x37, 0xb5, 0xe2, 0x7c, 0x2c, 0xf0, 0x25, 0xe4,
+	0x94, 0x2a, 0xbd, 0xe3, 0x82, 0x70, 0x78, 0xa3,
+	0x82, 0x10, 0x5a, 0x90, 0xd7, 0xa4, 0xfa, 0xaf,
+	0x1a, 0x88, 0x59, 0xdc, 0x74, 0x12, 0xb4, 0x8e,
+	0xd7, 0x19, 0x46, 0xf4, 0x84, 0x69, 0x9f, 0xbb,
+	0x70, 0xa8, 0x4c, 0x52, 0x81, 0xa9, 0xff, 0x76,
+	0x1c, 0xae, 0xd8, 0x11, 0x3d, 0x7f, 0x7d, 0xc5,
+	0x12, 0x59, 0x28, 0x18, 0xc2, 0xa2, 0xb7, 0x1c,
+	0x88, 0xf8, 0xd6, 0x1b, 0xa6, 0x7d, 0x9e, 0xde,
+	0x29, 0xf8, 0xed, 0xff, 0xeb, 0x92, 0x24, 0x4f,
+	0x05, 0xaa, 0xd9, 0x49, 0xba, 0x87, 0x59, 0x51,
+	0xc9, 0x20, 0x5c, 0x9b, 0x74, 0xcf, 0x03, 0xd9,
+	0x2d, 0x34, 0xc7, 0x5b, 0xa5, 0x40, 0xb2, 0x99,
+	0xf5, 0xcb, 0xb4, 0xf6, 0xb7, 0x72, 0x4a, 0xd6,
+	0xbd, 0xb0, 0xf3, 0x93, 0xe0, 0x1b, 0xa8, 0x04,
+	0x1e, 0x35, 0xd4, 0x80, 0x20, 0xf4, 0x9c, 0x31,
+	0x6b, 0x45, 0xb9, 0x15, 0xb0, 0x5e, 0xdd, 0x0a,
+	0x33, 0x9c, 0x83, 0xcd, 0x58, 0x89, 0x50, 0x56,
+	0xbb, 0x81, 0x00, 0x91, 0x32, 0xf3, 0x1b, 0x3e,
+	0xcf, 0x45, 0xe1, 0xf9, 0xe1, 0x2c, 0x26, 0x78,
+	0x93, 0x9a, 0x60, 0x46, 0xc9, 0xb5, 0x5e, 0x6a,
+	0x28, 0x92, 0x87, 0x3f, 0x63, 0x7b, 0xdb, 0xf7,
+	0xd0, 0x13, 0x9d, 0x32, 0x40, 0x5e, 0xcf, 0xfb,
+	0x79, 0x68, 0x47, 0x4c, 0xfd, 0x01, 0x17, 0xe6,
+	0x97, 0x93, 0x78, 0xbb, 0xa6, 0x27, 0xa3, 0xe8,
+	0x1a, 0xe8, 0x94, 0x55, 0x7d, 0x08, 0xe5, 0xdc,
+	0x66, 0xa3, 0x69, 0xc8, 0xca, 0xc5, 0xa1, 0x84,
+	0x55, 0xde, 0x08, 0x91, 0x16, 0x3a, 0x0c, 0x86,
+	0xab, 0x27, 0x2b, 0x64, 0x34, 0x02, 0x6c, 0x76,
+	0x8b, 0xc6, 0xaf, 0xcc, 0xe1, 0xd6, 0x8c, 0x2a,
+	0x18, 0x3d, 0xa6, 0x1b, 0x37, 0x75, 0x45, 0x73,
+	0xc2, 0x75, 0xd7, 0x53, 0x78, 0x3a, 0xd6, 0xe8,
+	0x29, 0xd2, 0x4a, 0xa8, 0x1e, 0x82, 0xf6, 0xb6,
+	0x81, 0xde, 0x21, 0xed, 0x2b, 0x56, 0xbb, 0xf2,
+	0xd0, 0x57, 0xc1, 0x7c, 0xd2, 0x6a, 0xd2, 0x56,
+	0xf5, 0x13, 0x5f, 0x1c, 0x6a, 0x0b, 0x74, 0xfb,
+	0xe9, 0xfe, 0x9e, 0xea, 0x95, 0xb2, 0x46, 0xab,
+	0x0a, 0xfc, 0xfd, 0xf3, 0xbb, 0x04, 0x2b, 0x76,
+	0x1b, 0xa4, 0x74, 0xb0, 0xc1, 0x78, 0xc3, 0x69,
+	0xe2, 0xb0, 0x01, 0xe1, 0xde, 0x32, 0x4c, 0x8d,
+	0x1a, 0xb3, 0x38, 0x08, 0xd5, 0xfc, 0x1f, 0xdc,
+	0x0e, 0x2c, 0x9c, 0xb1, 0xa1, 0x63, 0x17, 0x22,
+	0xf5, 0x6c, 0x93, 0x70, 0x74, 0x00, 0xf8, 0x39,
+	0x01, 0x94, 0xd1, 0x32, 0x23, 0x56, 0x5d, 0xa6,
+	0x02, 0x76, 0x76, 0x93, 0xce, 0x2f, 0x19, 0xe9,
+	0x17, 0x52, 0xae, 0x6e, 0x2c, 0x6d, 0x61, 0x7f,
+	0x3b, 0xaa, 0xe0, 0x52, 0x85, 0xc5, 0x65, 0xc1,
+	0xbb, 0x8e, 0x5b, 0x21, 0xd5, 0xc9, 0x78, 0x83,
+	0x07, 0x97, 0x4c, 0x62, 0x61, 0x41, 0xd4, 0xfc,
+	0xc9, 0x39, 0xe3, 0x9b, 0xd0, 0xcc, 0x75, 0xc4,
+	0x97, 0xe6, 0xdd, 0x2a, 0x5f, 0xa6, 0xe8, 0x59,
+	0x6c, 0x98, 0xb9, 0x02, 0xe2, 0xa2, 0xd6, 0x68,
+	0xee, 0x3b, 0x1d, 0xe3, 0x4d, 0x5b, 0x30, 0xef,
+	0x03, 0xf2, 0xeb, 0x18, 0x57, 0x36, 0xe8, 0xa1,
+	0xf4, 0x47, 0xfb, 0xcb, 0x8f, 0xcb, 0xc8, 0xf3,
+	0x4f, 0x74, 0x9d, 0x9d, 0xb1, 0x8d, 0x14, 0x44,
+	0xd9, 0x19, 0xb4, 0x54, 0x4f, 0x75, 0x19, 0x09,
+	0xa0, 0x75, 0xbc, 0x3b, 0x82, 0xc6, 0x3f, 0xb8,
+	0x83, 0x19, 0x6e, 0xd6, 0x37, 0xfe, 0x6e, 0x8a,
+	0x4e, 0xe0, 0x4a, 0xab, 0x7b, 0xc8, 0xb4, 0x1d,
+	0xf4, 0xed, 0x27, 0x03, 0x65, 0xa2, 0xa1, 0xae,
+	0x11, 0xe7, 0x98, 0x78, 0x48, 0x91, 0xd2, 0xd2,
+	0xd4, 0x23, 0x78, 0x50, 0xb1, 0x5b, 0x85, 0x10,
+	0x8d, 0xca, 0x5f, 0x0f, 0x71, 0xae, 0x72, 0x9a,
+	0xf6, 0x25, 0x19, 0x60, 0x06, 0xf7, 0x10, 0x34,
+	0x18, 0x0d, 0xc9, 0x9f, 0x7b, 0x0c, 0x9b, 0x8f,
+	0x91, 0x1b, 0x9f, 0xcd, 0x10, 0xee, 0x75, 0xf9,
+	0x97, 0x66, 0xfc, 0x4d, 0x33, 0x6e, 0x28, 0x2b,
+	0x92, 0x85, 0x4f, 0xab, 0x43, 0x8d, 0x8f, 0x7d,
+	0x86, 0xa7, 0xc7, 0xd8, 0xd3, 0x0b, 0x8b, 0x57,
+	0xb6, 0x1d, 0x95, 0x0d, 0xe9, 0xbc, 0xd9, 0x03,
+	0xd9, 0x10, 0x19, 0xc3, 0x46, 0x63, 0x55, 0x87,
+	0x61, 0x79, 0x6c, 0x95, 0x0e, 0x9c, 0xdd, 0xca,
+	0xc3, 0xf3, 0x64, 0xf0, 0x7d, 0x76, 0xb7, 0x53,
+	0x67, 0x2b, 0x1e, 0x44, 0x56, 0x81, 0xea, 0x8f,
+	0x5c, 0x42, 0x16, 0xb8, 0x28, 0xeb, 0x1b, 0x61,
+	0x10, 0x1e, 0xbf, 0xec, 0xa8
+};
+static const u8 enc_output011[] __initconst = {
+	0x6a, 0xfc, 0x4b, 0x25, 0xdf, 0xc0, 0xe4, 0xe8,
+	0x17, 0x4d, 0x4c, 0xc9, 0x7e, 0xde, 0x3a, 0xcc,
+	0x3c, 0xba, 0x6a, 0x77, 0x47, 0xdb, 0xe3, 0x74,
+	0x7a, 0x4d, 0x5f, 0x8d, 0x37, 0x55, 0x80, 0x73,
+	0x90, 0x66, 0x5d, 0x3a, 0x7d, 0x5d, 0x86, 0x5e,
+	0x8d, 0xfd, 0x83, 0xff, 0x4e, 0x74, 0x6f, 0xf9,
+	0xe6, 0x70, 0x17, 0x70, 0x3e, 0x96, 0xa7, 0x7e,
+	0xcb, 0xab, 0x8f, 0x58, 0x24, 0x9b, 0x01, 0xfd,
+	0xcb, 0xe6, 0x4d, 0x9b, 0xf0, 0x88, 0x94, 0x57,
+	0x66, 0xef, 0x72, 0x4c, 0x42, 0x6e, 0x16, 0x19,
+	0x15, 0xea, 0x70, 0x5b, 0xac, 0x13, 0xdb, 0x9f,
+	0x18, 0xe2, 0x3c, 0x26, 0x97, 0xbc, 0xdc, 0x45,
+	0x8c, 0x6c, 0x24, 0x69, 0x9c, 0xf7, 0x65, 0x1e,
+	0x18, 0x59, 0x31, 0x7c, 0xe4, 0x73, 0xbc, 0x39,
+	0x62, 0xc6, 0x5c, 0x9f, 0xbf, 0xfa, 0x90, 0x03,
+	0xc9, 0x72, 0x26, 0xb6, 0x1b, 0xc2, 0xb7, 0x3f,
+	0xf2, 0x13, 0x77, 0xf2, 0x8d, 0xb9, 0x47, 0xd0,
+	0x53, 0xdd, 0xc8, 0x91, 0x83, 0x8b, 0xb1, 0xce,
+	0xa3, 0xfe, 0xcd, 0xd9, 0xdd, 0x92, 0x7b, 0xdb,
+	0xb8, 0xfb, 0xc9, 0x2d, 0x01, 0x59, 0x39, 0x52,
+	0xad, 0x1b, 0xec, 0xcf, 0xd7, 0x70, 0x13, 0x21,
+	0xf5, 0x47, 0xaa, 0x18, 0x21, 0x5c, 0xc9, 0x9a,
+	0xd2, 0x6b, 0x05, 0x9c, 0x01, 0xa1, 0xda, 0x35,
+	0x5d, 0xb3, 0x70, 0xe6, 0xa9, 0x80, 0x8b, 0x91,
+	0xb7, 0xb3, 0x5f, 0x24, 0x9a, 0xb7, 0xd1, 0x6b,
+	0xa1, 0x1c, 0x50, 0xba, 0x49, 0xe0, 0xee, 0x2e,
+	0x75, 0xac, 0x69, 0xc0, 0xeb, 0x03, 0xdd, 0x19,
+	0xe5, 0xf6, 0x06, 0xdd, 0xc3, 0xd7, 0x2b, 0x07,
+	0x07, 0x30, 0xa7, 0x19, 0x0c, 0xbf, 0xe6, 0x18,
+	0xcc, 0xb1, 0x01, 0x11, 0x85, 0x77, 0x1d, 0x96,
+	0xa7, 0xa3, 0x00, 0x84, 0x02, 0xa2, 0x83, 0x68,
+	0xda, 0x17, 0x27, 0xc8, 0x7f, 0x23, 0xb7, 0xf4,
+	0x13, 0x85, 0xcf, 0xdd, 0x7a, 0x7d, 0x24, 0x57,
+	0xfe, 0x05, 0x93, 0xf5, 0x74, 0xce, 0xed, 0x0c,
+	0x20, 0x98, 0x8d, 0x92, 0x30, 0xa1, 0x29, 0x23,
+	0x1a, 0xa0, 0x4f, 0x69, 0x56, 0x4c, 0xe1, 0xc8,
+	0xce, 0xf6, 0x9a, 0x0c, 0xa4, 0xfa, 0x04, 0xf6,
+	0x62, 0x95, 0xf2, 0xfa, 0xc7, 0x40, 0x68, 0x40,
+	0x8f, 0x41, 0xda, 0xb4, 0x26, 0x6f, 0x70, 0xab,
+	0x40, 0x61, 0xa4, 0x0e, 0x75, 0xfb, 0x86, 0xeb,
+	0x9d, 0x9a, 0x1f, 0xec, 0x76, 0x99, 0xe7, 0xea,
+	0xaa, 0x1e, 0x2d, 0xb5, 0xd4, 0xa6, 0x1a, 0xb8,
+	0x61, 0x0a, 0x1d, 0x16, 0x5b, 0x98, 0xc2, 0x31,
+	0x40, 0xe7, 0x23, 0x1d, 0x66, 0x99, 0xc8, 0xc0,
+	0xd7, 0xce, 0xf3, 0x57, 0x40, 0x04, 0x3f, 0xfc,
+	0xea, 0xb3, 0xfc, 0xd2, 0xd3, 0x99, 0xa4, 0x94,
+	0x69, 0xa0, 0xef, 0xd1, 0x85, 0xb3, 0xa6, 0xb1,
+	0x28, 0xbf, 0x94, 0x67, 0x22, 0xc3, 0x36, 0x46,
+	0xf8, 0xd2, 0x0f, 0x5f, 0xf4, 0x59, 0x80, 0xe6,
+	0x2d, 0x43, 0x08, 0x7d, 0x19, 0x09, 0x97, 0xa7,
+	0x4c, 0x3d, 0x8d, 0xba, 0x65, 0x62, 0xa3, 0x71,
+	0x33, 0x29, 0x62, 0xdb, 0xc1, 0x33, 0x34, 0x1a,
+	0x63, 0x33, 0x16, 0xb6, 0x64, 0x7e, 0xab, 0x33,
+	0xf0, 0xe6, 0x26, 0x68, 0xba, 0x1d, 0x2e, 0x38,
+	0x08, 0xe6, 0x02, 0xd3, 0x25, 0x2c, 0x47, 0x23,
+	0x58, 0x34, 0x0f, 0x9d, 0x63, 0x4f, 0x63, 0xbb,
+	0x7f, 0x3b, 0x34, 0x38, 0xa7, 0xb5, 0x8d, 0x65,
+	0xd9, 0x9f, 0x79, 0x55, 0x3e, 0x4d, 0xe7, 0x73,
+	0xd8, 0xf6, 0x98, 0x97, 0x84, 0x60, 0x9c, 0xc8,
+	0xa9, 0x3c, 0xf6, 0xdc, 0x12, 0x5c, 0xe1, 0xbb,
+	0x0b, 0x8b, 0x98, 0x9c, 0x9d, 0x26, 0x7c, 0x4a,
+	0xe6, 0x46, 0x36, 0x58, 0x21, 0x4a, 0xee, 0xca,
+	0xd7, 0x3b, 0xc2, 0x6c, 0x49, 0x2f, 0xe5, 0xd5,
+	0x03, 0x59, 0x84, 0x53, 0xcb, 0xfe, 0x92, 0x71,
+	0x2e, 0x7c, 0x21, 0xcc, 0x99, 0x85, 0x7f, 0xb8,
+	0x74, 0x90, 0x13, 0x42, 0x3f, 0xe0, 0x6b, 0x1d,
+	0xf2, 0x4d, 0x54, 0xd4, 0xfc, 0x3a, 0x05, 0xe6,
+	0x74, 0xaf, 0xa6, 0xa0, 0x2a, 0x20, 0x23, 0x5d,
+	0x34, 0x5c, 0xd9, 0x3e, 0x4e, 0xfa, 0x93, 0xe7,
+	0xaa, 0xe9, 0x6f, 0x08, 0x43, 0x67, 0x41, 0xc5,
+	0xad, 0xfb, 0x31, 0x95, 0x82, 0x73, 0x32, 0xd8,
+	0xa6, 0xa3, 0xed, 0x0e, 0x2d, 0xf6, 0x5f, 0xfd,
+	0x80, 0xa6, 0x7a, 0xe0, 0xdf, 0x78, 0x15, 0x29,
+	0x74, 0x33, 0xd0, 0x9e, 0x83, 0x86, 0x72, 0x22,
+	0x57, 0x29, 0xb9, 0x9e, 0x5d, 0xd3, 0x1a, 0xb5,
+	0x96, 0x72, 0x41, 0x3d, 0xf1, 0x64, 0x43, 0x67,
+	0xee, 0xaa, 0x5c, 0xd3, 0x9a, 0x96, 0x13, 0x11,
+	0x5d, 0xf3, 0x0c, 0x87, 0x82, 0x1e, 0x41, 0x9e,
+	0xd0, 0x27, 0xd7, 0x54, 0x3b, 0x67, 0x73, 0x09,
+	0x91, 0xe9, 0xd5, 0x36, 0xa7, 0xb5, 0x55, 0xe4,
+	0xf3, 0x21, 0x51, 0x49, 0x22, 0x07, 0x55, 0x4f,
+	0x44, 0x4b, 0xd2, 0x15, 0x93, 0x17, 0x2a, 0xfa,
+	0x4d, 0x4a, 0x57, 0xdb, 0x4c, 0xa6, 0xeb, 0xec,
+	0x53, 0x25, 0x6c, 0x21, 0xed, 0x00, 0x4c, 0x3b,
+	0xca, 0x14, 0x57, 0xa9, 0xd6, 0x6a, 0xcd, 0x8d,
+	0x5e, 0x74, 0xac, 0x72, 0xc1, 0x97, 0xe5, 0x1b,
+	0x45, 0x4e, 0xda, 0xfc, 0xcc, 0x40, 0xe8, 0x48,
+	0x88, 0x0b, 0xa3, 0xe3, 0x8d, 0x83, 0x42, 0xc3,
+	0x23, 0xfd, 0x68, 0xb5, 0x8e, 0xf1, 0x9d, 0x63,
+	0x77, 0xe9, 0xa3, 0x8e, 0x8c, 0x26, 0x6b, 0xbd,
+	0x72, 0x73, 0x35, 0x0c, 0x03, 0xf8, 0x43, 0x78,
+	0x52, 0x71, 0x15, 0x1f, 0x71, 0x5d, 0x6e, 0xed,
+	0xb9, 0xcc, 0x86, 0x30, 0xdb, 0x2b, 0xd3, 0x82,
+	0x88, 0x23, 0x71, 0x90, 0x53, 0x5c, 0xa9, 0x2f,
+	0x76, 0x01, 0xb7, 0x9a, 0xfe, 0x43, 0x55, 0xa3,
+	0x04, 0x9b, 0x0e, 0xe4, 0x59, 0xdf, 0xc9, 0xe9,
+	0xb1, 0xea, 0x29, 0x28, 0x3c, 0x5c, 0xae, 0x72,
+	0x84, 0xb6, 0xc6, 0xeb, 0x0c, 0x27, 0x07, 0x74,
+	0x90, 0x0d, 0x31, 0xb0, 0x00, 0x77, 0xe9, 0x40,
+	0x70, 0x6f, 0x68, 0xa7, 0xfd, 0x06, 0xec, 0x4b,
+	0xc0, 0xb7, 0xac, 0xbc, 0x33, 0xb7, 0x6d, 0x0a,
+	0xbd, 0x12, 0x1b, 0x59, 0xcb, 0xdd, 0x32, 0xf5,
+	0x1d, 0x94, 0x57, 0x76, 0x9e, 0x0c, 0x18, 0x98,
+	0x71, 0xd7, 0x2a, 0xdb, 0x0b, 0x7b, 0xa7, 0x71,
+	0xb7, 0x67, 0x81, 0x23, 0x96, 0xae, 0xb9, 0x7e,
+	0x32, 0x43, 0x92, 0x8a, 0x19, 0xa0, 0xc4, 0xd4,
+	0x3b, 0x57, 0xf9, 0x4a, 0x2c, 0xfb, 0x51, 0x46,
+	0xbb, 0xcb, 0x5d, 0xb3, 0xef, 0x13, 0x93, 0x6e,
+	0x68, 0x42, 0x54, 0x57, 0xd3, 0x6a, 0x3a, 0x8f,
+	0x9d, 0x66, 0xbf, 0xbd, 0x36, 0x23, 0xf5, 0x93,
+	0x83, 0x7b, 0x9c, 0xc0, 0xdd, 0xc5, 0x49, 0xc0,
+	0x64, 0xed, 0x07, 0x12, 0xb3, 0xe6, 0xe4, 0xe5,
+	0x38, 0x95, 0x23, 0xb1, 0xa0, 0x3b, 0x1a, 0x61,
+	0xda, 0x17, 0xac, 0xc3, 0x58, 0xdd, 0x74, 0x64,
+	0x22, 0x11, 0xe8, 0x32, 0x1d, 0x16, 0x93, 0x85,
+	0x99, 0xa5, 0x9c, 0x34, 0x55, 0xb1, 0xe9, 0x20,
+	0x72, 0xc9, 0x28, 0x7b, 0x79, 0x00, 0xa1, 0xa6,
+	0xa3, 0x27, 0x40, 0x18, 0x8a, 0x54, 0xe0, 0xcc,
+	0xe8, 0x4e, 0x8e, 0x43, 0x96, 0xe7, 0x3f, 0xc8,
+	0xe9, 0xb2, 0xf9, 0xc9, 0xda, 0x04, 0x71, 0x50,
+	0x47, 0xe4, 0xaa, 0xce, 0xa2, 0x30, 0xc8, 0xe4,
+	0xac, 0xc7, 0x0d, 0x06, 0x2e, 0xe6, 0xe8, 0x80,
+	0x36, 0x29, 0x9e, 0x01, 0xb8, 0xc3, 0xf0, 0xa0,
+	0x5d, 0x7a, 0xca, 0x4d, 0xa0, 0x57, 0xbd, 0x2a,
+	0x45, 0xa7, 0x7f, 0x9c, 0x93, 0x07, 0x8f, 0x35,
+	0x67, 0x92, 0xe3, 0xe9, 0x7f, 0xa8, 0x61, 0x43,
+	0x9e, 0x25, 0x4f, 0x33, 0x76, 0x13, 0x6e, 0x12,
+	0xb9, 0xdd, 0xa4, 0x7c, 0x08, 0x9f, 0x7c, 0xe7,
+	0x0a, 0x8d, 0x84, 0x06, 0xa4, 0x33, 0x17, 0x34,
+	0x5e, 0x10, 0x7c, 0xc0, 0xa8, 0x3d, 0x1f, 0x42,
+	0x20, 0x51, 0x65, 0x5d, 0x09, 0xc3, 0xaa, 0xc0,
+	0xc8, 0x0d, 0xf0, 0x79, 0xbc, 0x20, 0x1b, 0x95,
+	0xe7, 0x06, 0x7d, 0x47, 0x20, 0x03, 0x1a, 0x74,
+	0xdd, 0xe2, 0xd4, 0xae, 0x38, 0x71, 0x9b, 0xf5,
+	0x80, 0xec, 0x08, 0x4e, 0x56, 0xba, 0x76, 0x12,
+	0x1a, 0xdf, 0x48, 0xf3, 0xae, 0xb3, 0xe6, 0xe6,
+	0xbe, 0xc0, 0x91, 0x2e, 0x01, 0xb3, 0x01, 0x86,
+	0xa2, 0xb9, 0x52, 0xd1, 0x21, 0xae, 0xd4, 0x97,
+	0x1d, 0xef, 0x41, 0x12, 0x95, 0x3d, 0x48, 0x45,
+	0x1c, 0x56, 0x32, 0x8f, 0xb8, 0x43, 0xbb, 0x19,
+	0xf3, 0xca, 0xe9, 0xeb, 0x6d, 0x84, 0xbe, 0x86,
+	0x06, 0xe2, 0x36, 0xb2, 0x62, 0x9d, 0xd3, 0x4c,
+	0x48, 0x18, 0x54, 0x13, 0x4e, 0xcf, 0xfd, 0xba,
+	0x84, 0xb9, 0x30, 0x53, 0xcf, 0xfb, 0xb9, 0x29,
+	0x8f, 0xdc, 0x9f, 0xef, 0x60, 0x0b, 0x64, 0xf6,
+	0x8b, 0xee, 0xa6, 0x91, 0xc2, 0x41, 0x6c, 0xf6,
+	0xfa, 0x79, 0x67, 0x4b, 0xc1, 0x3f, 0xaf, 0x09,
+	0x81, 0xd4, 0x5d, 0xcb, 0x09, 0xdf, 0x36, 0x31,
+	0xc0, 0x14, 0x3c, 0x7c, 0x0e, 0x65, 0x95, 0x99,
+	0x6d, 0xa3, 0xf4, 0xd7, 0x38, 0xee, 0x1a, 0x2b,
+	0x37, 0xe2, 0xa4, 0x3b, 0x4b, 0xd0, 0x65, 0xca,
+	0xf8, 0xc3, 0xe8, 0x15, 0x20, 0xef, 0xf2, 0x00,
+	0xfd, 0x01, 0x09, 0xc5, 0xc8, 0x17, 0x04, 0x93,
+	0xd0, 0x93, 0x03, 0x55, 0xc5, 0xfe, 0x32, 0xa3,
+	0x3e, 0x28, 0x2d, 0x3b, 0x93, 0x8a, 0xcc, 0x07,
+	0x72, 0x80, 0x8b, 0x74, 0x16, 0x24, 0xbb, 0xda,
+	0x94, 0x39, 0x30, 0x8f, 0xb1, 0xcd, 0x4a, 0x90,
+	0x92, 0x7c, 0x14, 0x8f, 0x95, 0x4e, 0xac, 0x9b,
+	0xd8, 0x8f, 0x1a, 0x87, 0xa4, 0x32, 0x27, 0x8a,
+	0xba, 0xf7, 0x41, 0xcf, 0x84, 0x37, 0x19, 0xe6,
+	0x06, 0xf5, 0x0e, 0xcf, 0x36, 0xf5, 0x9e, 0x6c,
+	0xde, 0xbc, 0xff, 0x64, 0x7e, 0x4e, 0x59, 0x57,
+	0x48, 0xfe, 0x14, 0xf7, 0x9c, 0x93, 0x5d, 0x15,
+	0xad, 0xcc, 0x11, 0xb1, 0x17, 0x18, 0xb2, 0x7e,
+	0xcc, 0xab, 0xe9, 0xce, 0x7d, 0x77, 0x5b, 0x51,
+	0x1b, 0x1e, 0x20, 0xa8, 0x32, 0x06, 0x0e, 0x75,
+	0x93, 0xac, 0xdb, 0x35, 0x37, 0x1f, 0xe9, 0x19,
+	0x1d, 0xb4, 0x71, 0x97, 0xd6, 0x4e, 0x2c, 0x08,
+	0xa5, 0x13, 0xf9, 0x0e, 0x7e, 0x78, 0x6e, 0x14,
+	0xe0, 0xa9, 0xb9, 0x96, 0x4c, 0x80, 0x82, 0xba,
+	0x17, 0xb3, 0x9d, 0x69, 0xb0, 0x84, 0x46, 0xff,
+	0xf9, 0x52, 0x79, 0x94, 0x58, 0x3a, 0x62, 0x90,
+	0x15, 0x35, 0x71, 0x10, 0x37, 0xed, 0xa1, 0x8e,
+	0x53, 0x6e, 0xf4, 0x26, 0x57, 0x93, 0x15, 0x93,
+	0xf6, 0x81, 0x2c, 0x5a, 0x10, 0xda, 0x92, 0xad,
+	0x2f, 0xdb, 0x28, 0x31, 0x2d, 0x55, 0x04, 0xd2,
+	0x06, 0x28, 0x8c, 0x1e, 0xdc, 0xea, 0x54, 0xac,
+	0xff, 0xb7, 0x6c, 0x30, 0x15, 0xd4, 0xb4, 0x0d,
+	0x00, 0x93, 0x57, 0xdd, 0xd2, 0x07, 0x07, 0x06,
+	0xd9, 0x43, 0x9b, 0xcd, 0x3a, 0xf4, 0x7d, 0x4c,
+	0x36, 0x5d, 0x23, 0xa2, 0xcc, 0x57, 0x40, 0x91,
+	0xe9, 0x2c, 0x2f, 0x2c, 0xd5, 0x30, 0x9b, 0x17,
+	0xb0, 0xc9, 0xf7, 0xa7, 0x2f, 0xd1, 0x93, 0x20,
+	0x6b, 0xc6, 0xc1, 0xe4, 0x6f, 0xcb, 0xd1, 0xe7,
+	0x09, 0x0f, 0x9e, 0xdc, 0xaa, 0x9f, 0x2f, 0xdf,
+	0x56, 0x9f, 0xd4, 0x33, 0x04, 0xaf, 0xd3, 0x6c,
+	0x58, 0x61, 0xf0, 0x30, 0xec, 0xf2, 0x7f, 0xf2,
+	0x9c, 0xdf, 0x39, 0xbb, 0x6f, 0xa2, 0x8c, 0x7e,
+	0xc4, 0x22, 0x51, 0x71, 0xc0, 0x4d, 0x14, 0x1a,
+	0xc4, 0xcd, 0x04, 0xd9, 0x87, 0x08, 0x50, 0x05,
+	0xcc, 0xaf, 0xf6, 0xf0, 0x8f, 0x92, 0x54, 0x58,
+	0xc2, 0xc7, 0x09, 0x7a, 0x59, 0x02, 0x05, 0xe8,
+	0xb0, 0x86, 0xd9, 0xbf, 0x7b, 0x35, 0x51, 0x4d,
+	0xaf, 0x08, 0x97, 0x2c, 0x65, 0xda, 0x2a, 0x71,
+	0x3a, 0xa8, 0x51, 0xcc, 0xf2, 0x73, 0x27, 0xc3,
+	0xfd, 0x62, 0xcf, 0xe3, 0xb2, 0xca, 0xcb, 0xbe,
+	0x1a, 0x0a, 0xa1, 0x34, 0x7b, 0x77, 0xc4, 0x62,
+	0x68, 0x78, 0x5f, 0x94, 0x07, 0x04, 0x65, 0x16,
+	0x4b, 0x61, 0xcb, 0xff, 0x75, 0x26, 0x50, 0x66,
+	0x1f, 0x6e, 0x93, 0xf8, 0xc5, 0x51, 0xeb, 0xa4,
+	0x4a, 0x48, 0x68, 0x6b, 0xe2, 0x5e, 0x44, 0xb2,
+	0x50, 0x2c, 0x6c, 0xae, 0x79, 0x4e, 0x66, 0x35,
+	0x81, 0x50, 0xac, 0xbc, 0x3f, 0xb1, 0x0c, 0xf3,
+	0x05, 0x3c, 0x4a, 0xa3, 0x6c, 0x2a, 0x79, 0xb4,
+	0xb7, 0xab, 0xca, 0xc7, 0x9b, 0x8e, 0xcd, 0x5f,
+	0x11, 0x03, 0xcb, 0x30, 0xa3, 0xab, 0xda, 0xfe,
+	0x64, 0xb9, 0xbb, 0xd8, 0x5e, 0x3a, 0x1a, 0x56,
+	0xe5, 0x05, 0x48, 0x90, 0x1e, 0x61, 0x69, 0x1b,
+	0x22, 0xe6, 0x1a, 0x3c, 0x75, 0xad, 0x1f, 0x37,
+	0x28, 0xdc, 0xe4, 0x6d, 0xbd, 0x42, 0xdc, 0xd3,
+	0xc8, 0xb6, 0x1c, 0x48, 0xfe, 0x94, 0x77, 0x7f,
+	0xbd, 0x62, 0xac, 0xa3, 0x47, 0x27, 0xcf, 0x5f,
+	0xd9, 0xdb, 0xaf, 0xec, 0xf7, 0x5e, 0xc1, 0xb0,
+	0x9d, 0x01, 0x26, 0x99, 0x7e, 0x8f, 0x03, 0x70,
+	0xb5, 0x42, 0xbe, 0x67, 0x28, 0x1b, 0x7c, 0xbd,
+	0x61, 0x21, 0x97, 0xcc, 0x5c, 0xe1, 0x97, 0x8f,
+	0x8d, 0xde, 0x2b, 0xaa, 0xa7, 0x71, 0x1d, 0x1e,
+	0x02, 0x73, 0x70, 0x58, 0x32, 0x5b, 0x1d, 0x67,
+	0x3d, 0xe0, 0x74, 0x4f, 0x03, 0xf2, 0x70, 0x51,
+	0x79, 0xf1, 0x61, 0x70, 0x15, 0x74, 0x9d, 0x23,
+	0x89, 0xde, 0xac, 0xfd, 0xde, 0xd0, 0x1f, 0xc3,
+	0x87, 0x44, 0x35, 0x4b, 0xe5, 0xb0, 0x60, 0xc5,
+	0x22, 0xe4, 0x9e, 0xca, 0xeb, 0xd5, 0x3a, 0x09,
+	0x45, 0xa4, 0xdb, 0xfa, 0x3f, 0xeb, 0x1b, 0xc7,
+	0xc8, 0x14, 0x99, 0x51, 0x92, 0x10, 0xed, 0xed,
+	0x28, 0xe0, 0xa1, 0xf8, 0x26, 0xcf, 0xcd, 0xcb,
+	0x63, 0xa1, 0x3b, 0xe3, 0xdf, 0x7e, 0xfe, 0xa6,
+	0xf0, 0x81, 0x9a, 0xbf, 0x55, 0xde, 0x54, 0xd5,
+	0x56, 0x60, 0x98, 0x10, 0x68, 0xf4, 0x38, 0x96,
+	0x8e, 0x6f, 0x1d, 0x44, 0x7f, 0xd6, 0x2f, 0xfe,
+	0x55, 0xfb, 0x0c, 0x7e, 0x67, 0xe2, 0x61, 0x44,
+	0xed, 0xf2, 0x35, 0x30, 0x5d, 0xe9, 0xc7, 0xd6,
+	0x6d, 0xe0, 0xa0, 0xed, 0xf3, 0xfc, 0xd8, 0x3e,
+	0x0a, 0x7b, 0xcd, 0xaf, 0x65, 0x68, 0x18, 0xc0,
+	0xec, 0x04, 0x1c, 0x74, 0x6d, 0xe2, 0x6e, 0x79,
+	0xd4, 0x11, 0x2b, 0x62, 0xd5, 0x27, 0xad, 0x4f,
+	0x01, 0x59, 0x73, 0xcc, 0x6a, 0x53, 0xfb, 0x2d,
+	0xd5, 0x4e, 0x99, 0x21, 0x65, 0x4d, 0xf5, 0x82,
+	0xf7, 0xd8, 0x42, 0xce, 0x6f, 0x3d, 0x36, 0x47,
+	0xf1, 0x05, 0x16, 0xe8, 0x1b, 0x6a, 0x8f, 0x93,
+	0xf2, 0x8f, 0x37, 0x40, 0x12, 0x28, 0xa3, 0xe6,
+	0xb9, 0x17, 0x4a, 0x1f, 0xb1, 0xd1, 0x66, 0x69,
+	0x86, 0xc4, 0xfc, 0x97, 0xae, 0x3f, 0x8f, 0x1e,
+	0x2b, 0xdf, 0xcd, 0xf9, 0x3c
+};
+static const u8 enc_assoc011[] __initconst = {
+	0xd6, 0x31, 0xda, 0x5d, 0x42, 0x5e, 0xd7
+};
+static const u8 enc_nonce011[] __initconst = {
+	0xfd, 0x87, 0xd4, 0xd8, 0x62, 0xfd, 0xec, 0xaa
+};
+static const u8 enc_key011[] __initconst = {
+	0x35, 0x4e, 0xb5, 0x70, 0x50, 0x42, 0x8a, 0x85,
+	0xf2, 0xfb, 0xed, 0x7b, 0xd0, 0x9e, 0x97, 0xca,
+	0xfa, 0x98, 0x66, 0x63, 0xee, 0x37, 0xcc, 0x52,
+	0xfe, 0xd1, 0xdf, 0x95, 0x15, 0x34, 0x29, 0x38
+};
+
+static const u8 enc_input012[] __initconst = {
+	0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0,
+	0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5,
+	0x42, 0x34, 0x8a, 0xa1, 0x03, 0xb7, 0xe9, 0x57,
+	0x46, 0xbe, 0x20, 0xe4, 0x6e, 0xb0, 0xeb, 0xff,
+	0xea, 0x07, 0x7e, 0xef, 0xe2, 0x55, 0x9f, 0xe5,
+	0x78, 0x3a, 0xb7, 0x83, 0xc2, 0x18, 0x40, 0x7b,
+	0xeb, 0xcd, 0x81, 0xfb, 0x90, 0x12, 0x9e, 0x46,
+	0xa9, 0xd6, 0x4a, 0xba, 0xb0, 0x62, 0xdb, 0x6b,
+	0x99, 0xc4, 0xdb, 0x54, 0x4b, 0xb8, 0xa5, 0x71,
+	0xcb, 0xcd, 0x63, 0x32, 0x55, 0xfb, 0x31, 0xf0,
+	0x38, 0xf5, 0xbe, 0x78, 0xe4, 0x45, 0xce, 0x1b,
+	0x6a, 0x5b, 0x0e, 0xf4, 0x16, 0xe4, 0xb1, 0x3d,
+	0xf6, 0x63, 0x7b, 0xa7, 0x0c, 0xde, 0x6f, 0x8f,
+	0x74, 0xdf, 0xe0, 0x1e, 0x9d, 0xce, 0x8f, 0x24,
+	0xef, 0x23, 0x35, 0x33, 0x7b, 0x83, 0x34, 0x23,
+	0x58, 0x74, 0x14, 0x77, 0x1f, 0xc2, 0x4f, 0x4e,
+	0xc6, 0x89, 0xf9, 0x52, 0x09, 0x37, 0x64, 0x14,
+	0xc4, 0x01, 0x6b, 0x9d, 0x77, 0xe8, 0x90, 0x5d,
+	0xa8, 0x4a, 0x2a, 0xef, 0x5c, 0x7f, 0xeb, 0xbb,
+	0xb2, 0xc6, 0x93, 0x99, 0x66, 0xdc, 0x7f, 0xd4,
+	0x9e, 0x2a, 0xca, 0x8d, 0xdb, 0xe7, 0x20, 0xcf,
+	0xe4, 0x73, 0xae, 0x49, 0x7d, 0x64, 0x0f, 0x0e,
+	0x28, 0x46, 0xa9, 0xa8, 0x32, 0xe4, 0x0e, 0xf6,
+	0x51, 0x53, 0xb8, 0x3c, 0xb1, 0xff, 0xa3, 0x33,
+	0x41, 0x75, 0xff, 0xf1, 0x6f, 0xf1, 0xfb, 0xbb,
+	0x83, 0x7f, 0x06, 0x9b, 0xe7, 0x1b, 0x0a, 0xe0,
+	0x5c, 0x33, 0x60, 0x5b, 0xdb, 0x5b, 0xed, 0xfe,
+	0xa5, 0x16, 0x19, 0x72, 0xa3, 0x64, 0x23, 0x00,
+	0x02, 0xc7, 0xf3, 0x6a, 0x81, 0x3e, 0x44, 0x1d,
+	0x79, 0x15, 0x5f, 0x9a, 0xde, 0xe2, 0xfd, 0x1b,
+	0x73, 0xc1, 0xbc, 0x23, 0xba, 0x31, 0xd2, 0x50,
+	0xd5, 0xad, 0x7f, 0x74, 0xa7, 0xc9, 0xf8, 0x3e,
+	0x2b, 0x26, 0x10, 0xf6, 0x03, 0x36, 0x74, 0xe4,
+	0x0e, 0x6a, 0x72, 0xb7, 0x73, 0x0a, 0x42, 0x28,
+	0xc2, 0xad, 0x5e, 0x03, 0xbe, 0xb8, 0x0b, 0xa8,
+	0x5b, 0xd4, 0xb8, 0xba, 0x52, 0x89, 0xb1, 0x9b,
+	0xc1, 0xc3, 0x65, 0x87, 0xed, 0xa5, 0xf4, 0x86,
+	0xfd, 0x41, 0x80, 0x91, 0x27, 0x59, 0x53, 0x67,
+	0x15, 0x78, 0x54, 0x8b, 0x2d, 0x3d, 0xc7, 0xff,
+	0x02, 0x92, 0x07, 0x5f, 0x7a, 0x4b, 0x60, 0x59,
+	0x3c, 0x6f, 0x5c, 0xd8, 0xec, 0x95, 0xd2, 0xfe,
+	0xa0, 0x3b, 0xd8, 0x3f, 0xd1, 0x69, 0xa6, 0xd6,
+	0x41, 0xb2, 0xf4, 0x4d, 0x12, 0xf4, 0x58, 0x3e,
+	0x66, 0x64, 0x80, 0x31, 0x9b, 0xa8, 0x4c, 0x8b,
+	0x07, 0xb2, 0xec, 0x66, 0x94, 0x66, 0x47, 0x50,
+	0x50, 0x5f, 0x18, 0x0b, 0x0e, 0xd6, 0xc0, 0x39,
+	0x21, 0x13, 0x9e, 0x33, 0xbc, 0x79, 0x36, 0x02,
+	0x96, 0x70, 0xf0, 0x48, 0x67, 0x2f, 0x26, 0xe9,
+	0x6d, 0x10, 0xbb, 0xd6, 0x3f, 0xd1, 0x64, 0x7a,
+	0x2e, 0xbe, 0x0c, 0x61, 0xf0, 0x75, 0x42, 0x38,
+	0x23, 0xb1, 0x9e, 0x9f, 0x7c, 0x67, 0x66, 0xd9,
+	0x58, 0x9a, 0xf1, 0xbb, 0x41, 0x2a, 0x8d, 0x65,
+	0x84, 0x94, 0xfc, 0xdc, 0x6a, 0x50, 0x64, 0xdb,
+	0x56, 0x33, 0x76, 0x00, 0x10, 0xed, 0xbe, 0xd2,
+	0x12, 0xf6, 0xf6, 0x1b, 0xa2, 0x16, 0xde, 0xae,
+	0x31, 0x95, 0xdd, 0xb1, 0x08, 0x7e, 0x4e, 0xee,
+	0xe7, 0xf9, 0xa5, 0xfb, 0x5b, 0x61, 0x43, 0x00,
+	0x40, 0xf6, 0x7e, 0x02, 0x04, 0x32, 0x4e, 0x0c,
+	0xe2, 0x66, 0x0d, 0xd7, 0x07, 0x98, 0x0e, 0xf8,
+	0x72, 0x34, 0x6d, 0x95, 0x86, 0xd7, 0xcb, 0x31,
+	0x54, 0x47, 0xd0, 0x38, 0x29, 0x9c, 0x5a, 0x68,
+	0xd4, 0x87, 0x76, 0xc9, 0xe7, 0x7e, 0xe3, 0xf4,
+	0x81, 0x6d, 0x18, 0xcb, 0xc9, 0x05, 0xaf, 0xa0,
+	0xfb, 0x66, 0xf7, 0xf1, 0x1c, 0xc6, 0x14, 0x11,
+	0x4f, 0x2b, 0x79, 0x42, 0x8b, 0xbc, 0xac, 0xe7,
+	0x6c, 0xfe, 0x0f, 0x58, 0xe7, 0x7c, 0x78, 0x39,
+	0x30, 0xb0, 0x66, 0x2c, 0x9b, 0x6d, 0x3a, 0xe1,
+	0xcf, 0xc9, 0xa4, 0x0e, 0x6d, 0x6d, 0x8a, 0xa1,
+	0x3a, 0xe7, 0x28, 0xd4, 0x78, 0x4c, 0xa6, 0xa2,
+	0x2a, 0xa6, 0x03, 0x30, 0xd7, 0xa8, 0x25, 0x66,
+	0x87, 0x2f, 0x69, 0x5c, 0x4e, 0xdd, 0xa5, 0x49,
+	0x5d, 0x37, 0x4a, 0x59, 0xc4, 0xaf, 0x1f, 0xa2,
+	0xe4, 0xf8, 0xa6, 0x12, 0x97, 0xd5, 0x79, 0xf5,
+	0xe2, 0x4a, 0x2b, 0x5f, 0x61, 0xe4, 0x9e, 0xe3,
+	0xee, 0xb8, 0xa7, 0x5b, 0x2f, 0xf4, 0x9e, 0x6c,
+	0xfb, 0xd1, 0xc6, 0x56, 0x77, 0xba, 0x75, 0xaa,
+	0x3d, 0x1a, 0xa8, 0x0b, 0xb3, 0x68, 0x24, 0x00,
+	0x10, 0x7f, 0xfd, 0xd7, 0xa1, 0x8d, 0x83, 0x54,
+	0x4f, 0x1f, 0xd8, 0x2a, 0xbe, 0x8a, 0x0c, 0x87,
+	0xab, 0xa2, 0xde, 0xc3, 0x39, 0xbf, 0x09, 0x03,
+	0xa5, 0xf3, 0x05, 0x28, 0xe1, 0xe1, 0xee, 0x39,
+	0x70, 0x9c, 0xd8, 0x81, 0x12, 0x1e, 0x02, 0x40,
+	0xd2, 0x6e, 0xf0, 0xeb, 0x1b, 0x3d, 0x22, 0xc6,
+	0xe5, 0xe3, 0xb4, 0x5a, 0x98, 0xbb, 0xf0, 0x22,
+	0x28, 0x8d, 0xe5, 0xd3, 0x16, 0x48, 0x24, 0xa5,
+	0xe6, 0x66, 0x0c, 0xf9, 0x08, 0xf9, 0x7e, 0x1e,
+	0xe1, 0x28, 0x26, 0x22, 0xc7, 0xc7, 0x0a, 0x32,
+	0x47, 0xfa, 0xa3, 0xbe, 0x3c, 0xc4, 0xc5, 0x53,
+	0x0a, 0xd5, 0x94, 0x4a, 0xd7, 0x93, 0xd8, 0x42,
+	0x99, 0xb9, 0x0a, 0xdb, 0x56, 0xf7, 0xb9, 0x1c,
+	0x53, 0x4f, 0xfa, 0xd3, 0x74, 0xad, 0xd9, 0x68,
+	0xf1, 0x1b, 0xdf, 0x61, 0xc6, 0x5e, 0xa8, 0x48,
+	0xfc, 0xd4, 0x4a, 0x4c, 0x3c, 0x32, 0xf7, 0x1c,
+	0x96, 0x21, 0x9b, 0xf9, 0xa3, 0xcc, 0x5a, 0xce,
+	0xd5, 0xd7, 0x08, 0x24, 0xf6, 0x1c, 0xfd, 0xdd,
+	0x38, 0xc2, 0x32, 0xe9, 0xb8, 0xe7, 0xb6, 0xfa,
+	0x9d, 0x45, 0x13, 0x2c, 0x83, 0xfd, 0x4a, 0x69,
+	0x82, 0xcd, 0xdc, 0xb3, 0x76, 0x0c, 0x9e, 0xd8,
+	0xf4, 0x1b, 0x45, 0x15, 0xb4, 0x97, 0xe7, 0x58,
+	0x34, 0xe2, 0x03, 0x29, 0x5a, 0xbf, 0xb6, 0xe0,
+	0x5d, 0x13, 0xd9, 0x2b, 0xb4, 0x80, 0xb2, 0x45,
+	0x81, 0x6a, 0x2e, 0x6c, 0x89, 0x7d, 0xee, 0xbb,
+	0x52, 0xdd, 0x1f, 0x18, 0xe7, 0x13, 0x6b, 0x33,
+	0x0e, 0xea, 0x36, 0x92, 0x77, 0x7b, 0x6d, 0x9c,
+	0x5a, 0x5f, 0x45, 0x7b, 0x7b, 0x35, 0x62, 0x23,
+	0xd1, 0xbf, 0x0f, 0xd0, 0x08, 0x1b, 0x2b, 0x80,
+	0x6b, 0x7e, 0xf1, 0x21, 0x47, 0xb0, 0x57, 0xd1,
+	0x98, 0x72, 0x90, 0x34, 0x1c, 0x20, 0x04, 0xff,
+	0x3d, 0x5c, 0xee, 0x0e, 0x57, 0x5f, 0x6f, 0x24,
+	0x4e, 0x3c, 0xea, 0xfc, 0xa5, 0xa9, 0x83, 0xc9,
+	0x61, 0xb4, 0x51, 0x24, 0xf8, 0x27, 0x5e, 0x46,
+	0x8c, 0xb1, 0x53, 0x02, 0x96, 0x35, 0xba, 0xb8,
+	0x4c, 0x71, 0xd3, 0x15, 0x59, 0x35, 0x22, 0x20,
+	0xad, 0x03, 0x9f, 0x66, 0x44, 0x3b, 0x9c, 0x35,
+	0x37, 0x1f, 0x9b, 0xbb, 0xf3, 0xdb, 0x35, 0x63,
+	0x30, 0x64, 0xaa, 0xa2, 0x06, 0xa8, 0x5d, 0xbb,
+	0xe1, 0x9f, 0x70, 0xec, 0x82, 0x11, 0x06, 0x36,
+	0xec, 0x8b, 0x69, 0x66, 0x24, 0x44, 0xc9, 0x4a,
+	0x57, 0xbb, 0x9b, 0x78, 0x13, 0xce, 0x9c, 0x0c,
+	0xba, 0x92, 0x93, 0x63, 0xb8, 0xe2, 0x95, 0x0f,
+	0x0f, 0x16, 0x39, 0x52, 0xfd, 0x3a, 0x6d, 0x02,
+	0x4b, 0xdf, 0x13, 0xd3, 0x2a, 0x22, 0xb4, 0x03,
+	0x7c, 0x54, 0x49, 0x96, 0x68, 0x54, 0x10, 0xfa,
+	0xef, 0xaa, 0x6c, 0xe8, 0x22, 0xdc, 0x71, 0x16,
+	0x13, 0x1a, 0xf6, 0x28, 0xe5, 0x6d, 0x77, 0x3d,
+	0xcd, 0x30, 0x63, 0xb1, 0x70, 0x52, 0xa1, 0xc5,
+	0x94, 0x5f, 0xcf, 0xe8, 0xb8, 0x26, 0x98, 0xf7,
+	0x06, 0xa0, 0x0a, 0x70, 0xfa, 0x03, 0x80, 0xac,
+	0xc1, 0xec, 0xd6, 0x4c, 0x54, 0xd7, 0xfe, 0x47,
+	0xb6, 0x88, 0x4a, 0xf7, 0x71, 0x24, 0xee, 0xf3,
+	0xd2, 0xc2, 0x4a, 0x7f, 0xfe, 0x61, 0xc7, 0x35,
+	0xc9, 0x37, 0x67, 0xcb, 0x24, 0x35, 0xda, 0x7e,
+	0xca, 0x5f, 0xf3, 0x8d, 0xd4, 0x13, 0x8e, 0xd6,
+	0xcb, 0x4d, 0x53, 0x8f, 0x53, 0x1f, 0xc0, 0x74,
+	0xf7, 0x53, 0xb9, 0x5e, 0x23, 0x37, 0xba, 0x6e,
+	0xe3, 0x9d, 0x07, 0x55, 0x25, 0x7b, 0xe6, 0x2a,
+	0x64, 0xd1, 0x32, 0xdd, 0x54, 0x1b, 0x4b, 0xc0,
+	0xe1, 0xd7, 0x69, 0x58, 0xf8, 0x93, 0x29, 0xc4,
+	0xdd, 0x23, 0x2f, 0xa5, 0xfc, 0x9d, 0x7e, 0xf8,
+	0xd4, 0x90, 0xcd, 0x82, 0x55, 0xdc, 0x16, 0x16,
+	0x9f, 0x07, 0x52, 0x9b, 0x9d, 0x25, 0xed, 0x32,
+	0xc5, 0x7b, 0xdf, 0xf6, 0x83, 0x46, 0x3d, 0x65,
+	0xb7, 0xef, 0x87, 0x7a, 0x12, 0x69, 0x8f, 0x06,
+	0x7c, 0x51, 0x15, 0x4a, 0x08, 0xe8, 0xac, 0x9a,
+	0x0c, 0x24, 0xa7, 0x27, 0xd8, 0x46, 0x2f, 0xe7,
+	0x01, 0x0e, 0x1c, 0xc6, 0x91, 0xb0, 0x6e, 0x85,
+	0x65, 0xf0, 0x29, 0x0d, 0x2e, 0x6b, 0x3b, 0xfb,
+	0x4b, 0xdf, 0xe4, 0x80, 0x93, 0x03, 0x66, 0x46,
+	0x3e, 0x8a, 0x6e, 0xf3, 0x5e, 0x4d, 0x62, 0x0e,
+	0x49, 0x05, 0xaf, 0xd4, 0xf8, 0x21, 0x20, 0x61,
+	0x1d, 0x39, 0x17, 0xf4, 0x61, 0x47, 0x95, 0xfb,
+	0x15, 0x2e, 0xb3, 0x4f, 0xd0, 0x5d, 0xf5, 0x7d,
+	0x40, 0xda, 0x90, 0x3c, 0x6b, 0xcb, 0x17, 0x00,
+	0x13, 0x3b, 0x64, 0x34, 0x1b, 0xf0, 0xf2, 0xe5,
+	0x3b, 0xb2, 0xc7, 0xd3, 0x5f, 0x3a, 0x44, 0xa6,
+	0x9b, 0xb7, 0x78, 0x0e, 0x42, 0x5d, 0x4c, 0xc1,
+	0xe9, 0xd2, 0xcb, 0xb7, 0x78, 0xd1, 0xfe, 0x9a,
+	0xb5, 0x07, 0xe9, 0xe0, 0xbe, 0xe2, 0x8a, 0xa7,
+	0x01, 0x83, 0x00, 0x8c, 0x5c, 0x08, 0xe6, 0x63,
+	0x12, 0x92, 0xb7, 0xb7, 0xa6, 0x19, 0x7d, 0x38,
+	0x13, 0x38, 0x92, 0x87, 0x24, 0xf9, 0x48, 0xb3,
+	0x5e, 0x87, 0x6a, 0x40, 0x39, 0x5c, 0x3f, 0xed,
+	0x8f, 0xee, 0xdb, 0x15, 0x82, 0x06, 0xda, 0x49,
+	0x21, 0x2b, 0xb5, 0xbf, 0x32, 0x7c, 0x9f, 0x42,
+	0x28, 0x63, 0xcf, 0xaf, 0x1e, 0xf8, 0xc6, 0xa0,
+	0xd1, 0x02, 0x43, 0x57, 0x62, 0xec, 0x9b, 0x0f,
+	0x01, 0x9e, 0x71, 0xd8, 0x87, 0x9d, 0x01, 0xc1,
+	0x58, 0x77, 0xd9, 0xaf, 0xb1, 0x10, 0x7e, 0xdd,
+	0xa6, 0x50, 0x96, 0xe5, 0xf0, 0x72, 0x00, 0x6d,
+	0x4b, 0xf8, 0x2a, 0x8f, 0x19, 0xf3, 0x22, 0x88,
+	0x11, 0x4a, 0x8b, 0x7c, 0xfd, 0xb7, 0xed, 0xe1,
+	0xf6, 0x40, 0x39, 0xe0, 0xe9, 0xf6, 0x3d, 0x25,
+	0xe6, 0x74, 0x3c, 0x58, 0x57, 0x7f, 0xe1, 0x22,
+	0x96, 0x47, 0x31, 0x91, 0xba, 0x70, 0x85, 0x28,
+	0x6b, 0x9f, 0x6e, 0x25, 0xac, 0x23, 0x66, 0x2f,
+	0x29, 0x88, 0x28, 0xce, 0x8c, 0x5c, 0x88, 0x53,
+	0xd1, 0x3b, 0xcc, 0x6a, 0x51, 0xb2, 0xe1, 0x28,
+	0x3f, 0x91, 0xb4, 0x0d, 0x00, 0x3a, 0xe3, 0xf8,
+	0xc3, 0x8f, 0xd7, 0x96, 0x62, 0x0e, 0x2e, 0xfc,
+	0xc8, 0x6c, 0x77, 0xa6, 0x1d, 0x22, 0xc1, 0xb8,
+	0xe6, 0x61, 0xd7, 0x67, 0x36, 0x13, 0x7b, 0xbb,
+	0x9b, 0x59, 0x09, 0xa6, 0xdf, 0xf7, 0x6b, 0xa3,
+	0x40, 0x1a, 0xf5, 0x4f, 0xb4, 0xda, 0xd3, 0xf3,
+	0x81, 0x93, 0xc6, 0x18, 0xd9, 0x26, 0xee, 0xac,
+	0xf0, 0xaa, 0xdf, 0xc5, 0x9c, 0xca, 0xc2, 0xa2,
+	0xcc, 0x7b, 0x5c, 0x24, 0xb0, 0xbc, 0xd0, 0x6a,
+	0x4d, 0x89, 0x09, 0xb8, 0x07, 0xfe, 0x87, 0xad,
+	0x0a, 0xea, 0xb8, 0x42, 0xf9, 0x5e, 0xb3, 0x3e,
+	0x36, 0x4c, 0xaf, 0x75, 0x9e, 0x1c, 0xeb, 0xbd,
+	0xbc, 0xbb, 0x80, 0x40, 0xa7, 0x3a, 0x30, 0xbf,
+	0xa8, 0x44, 0xf4, 0xeb, 0x38, 0xad, 0x29, 0xba,
+	0x23, 0xed, 0x41, 0x0c, 0xea, 0xd2, 0xbb, 0x41,
+	0x18, 0xd6, 0xb9, 0xba, 0x65, 0x2b, 0xa3, 0x91,
+	0x6d, 0x1f, 0xa9, 0xf4, 0xd1, 0x25, 0x8d, 0x4d,
+	0x38, 0xff, 0x64, 0xa0, 0xec, 0xde, 0xa6, 0xb6,
+	0x79, 0xab, 0x8e, 0x33, 0x6c, 0x47, 0xde, 0xaf,
+	0x94, 0xa4, 0xa5, 0x86, 0x77, 0x55, 0x09, 0x92,
+	0x81, 0x31, 0x76, 0xc7, 0x34, 0x22, 0x89, 0x8e,
+	0x3d, 0x26, 0x26, 0xd7, 0xfc, 0x1e, 0x16, 0x72,
+	0x13, 0x33, 0x63, 0xd5, 0x22, 0xbe, 0xb8, 0x04,
+	0x34, 0x84, 0x41, 0xbb, 0x80, 0xd0, 0x9f, 0x46,
+	0x48, 0x07, 0xa7, 0xfc, 0x2b, 0x3a, 0x75, 0x55,
+	0x8c, 0xc7, 0x6a, 0xbd, 0x7e, 0x46, 0x08, 0x84,
+	0x0f, 0xd5, 0x74, 0xc0, 0x82, 0x8e, 0xaa, 0x61,
+	0x05, 0x01, 0xb2, 0x47, 0x6e, 0x20, 0x6a, 0x2d,
+	0x58, 0x70, 0x48, 0x32, 0xa7, 0x37, 0xd2, 0xb8,
+	0x82, 0x1a, 0x51, 0xb9, 0x61, 0xdd, 0xfd, 0x9d,
+	0x6b, 0x0e, 0x18, 0x97, 0xf8, 0x45, 0x5f, 0x87,
+	0x10, 0xcf, 0x34, 0x72, 0x45, 0x26, 0x49, 0x70,
+	0xe7, 0xa3, 0x78, 0xe0, 0x52, 0x89, 0x84, 0x94,
+	0x83, 0x82, 0xc2, 0x69, 0x8f, 0xe3, 0xe1, 0x3f,
+	0x60, 0x74, 0x88, 0xc4, 0xf7, 0x75, 0x2c, 0xfb,
+	0xbd, 0xb6, 0xc4, 0x7e, 0x10, 0x0a, 0x6c, 0x90,
+	0x04, 0x9e, 0xc3, 0x3f, 0x59, 0x7c, 0xce, 0x31,
+	0x18, 0x60, 0x57, 0x73, 0x46, 0x94, 0x7d, 0x06,
+	0xa0, 0x6d, 0x44, 0xec, 0xa2, 0x0a, 0x9e, 0x05,
+	0x15, 0xef, 0xca, 0x5c, 0xbf, 0x00, 0xeb, 0xf7,
+	0x3d, 0x32, 0xd4, 0xa5, 0xef, 0x49, 0x89, 0x5e,
+	0x46, 0xb0, 0xa6, 0x63, 0x5b, 0x8a, 0x73, 0xae,
+	0x6f, 0xd5, 0x9d, 0xf8, 0x4f, 0x40, 0xb5, 0xb2,
+	0x6e, 0xd3, 0xb6, 0x01, 0xa9, 0x26, 0xa2, 0x21,
+	0xcf, 0x33, 0x7a, 0x3a, 0xa4, 0x23, 0x13, 0xb0,
+	0x69, 0x6a, 0xee, 0xce, 0xd8, 0x9d, 0x01, 0x1d,
+	0x50, 0xc1, 0x30, 0x6c, 0xb1, 0xcd, 0xa0, 0xf0,
+	0xf0, 0xa2, 0x64, 0x6f, 0xbb, 0xbf, 0x5e, 0xe6,
+	0xab, 0x87, 0xb4, 0x0f, 0x4f, 0x15, 0xaf, 0xb5,
+	0x25, 0xa1, 0xb2, 0xd0, 0x80, 0x2c, 0xfb, 0xf9,
+	0xfe, 0xd2, 0x33, 0xbb, 0x76, 0xfe, 0x7c, 0xa8,
+	0x66, 0xf7, 0xe7, 0x85, 0x9f, 0x1f, 0x85, 0x57,
+	0x88, 0xe1, 0xe9, 0x63, 0xe4, 0xd8, 0x1c, 0xa1,
+	0xfb, 0xda, 0x44, 0x05, 0x2e, 0x1d, 0x3a, 0x1c,
+	0xff, 0xc8, 0x3b, 0xc0, 0xfe, 0xda, 0x22, 0x0b,
+	0x43, 0xd6, 0x88, 0x39, 0x4c, 0x4a, 0xa6, 0x69,
+	0x18, 0x93, 0x42, 0x4e, 0xb5, 0xcc, 0x66, 0x0d,
+	0x09, 0xf8, 0x1e, 0x7c, 0xd3, 0x3c, 0x99, 0x0d,
+	0x50, 0x1d, 0x62, 0xe9, 0x57, 0x06, 0xbf, 0x19,
+	0x88, 0xdd, 0xad, 0x7b, 0x4f, 0xf9, 0xc7, 0x82,
+	0x6d, 0x8d, 0xc8, 0xc4, 0xc5, 0x78, 0x17, 0x20,
+	0x15, 0xc5, 0x52, 0x41, 0xcf, 0x5b, 0xd6, 0x7f,
+	0x94, 0x02, 0x41, 0xe0, 0x40, 0x22, 0x03, 0x5e,
+	0xd1, 0x53, 0xd4, 0x86, 0xd3, 0x2c, 0x9f, 0x0f,
+	0x96, 0xe3, 0x6b, 0x9a, 0x76, 0x32, 0x06, 0x47,
+	0x4b, 0x11, 0xb3, 0xdd, 0x03, 0x65, 0xbd, 0x9b,
+	0x01, 0xda, 0x9c, 0xb9, 0x7e, 0x3f, 0x6a, 0xc4,
+	0x7b, 0xea, 0xd4, 0x3c, 0xb9, 0xfb, 0x5c, 0x6b,
+	0x64, 0x33, 0x52, 0xba, 0x64, 0x78, 0x8f, 0xa4,
+	0xaf, 0x7a, 0x61, 0x8d, 0xbc, 0xc5, 0x73, 0xe9,
+	0x6b, 0x58, 0x97, 0x4b, 0xbf, 0x63, 0x22, 0xd3,
+	0x37, 0x02, 0x54, 0xc5, 0xb9, 0x16, 0x4a, 0xf0,
+	0x19, 0xd8, 0x94, 0x57, 0xb8, 0x8a, 0xb3, 0x16,
+	0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d,
+	0x78, 0xec, 0x00
+};
+static const u8 enc_output012[] __initconst = {
+	0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3,
+	0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf,
+	0xdf, 0x34, 0x56, 0x82, 0xe2, 0xbe, 0xe5, 0xe1,
+	0x28, 0xd1, 0x4e, 0x5f, 0x4f, 0x01, 0x7d, 0x3f,
+	0x99, 0x6b, 0x30, 0x6e, 0x1a, 0x7c, 0x4c, 0x8e,
+	0x62, 0x81, 0xae, 0x86, 0x3f, 0x6b, 0xd0, 0xb5,
+	0xa9, 0xcf, 0x50, 0xf1, 0x02, 0x12, 0xa0, 0x0b,
+	0x24, 0xe9, 0xe6, 0x72, 0x89, 0x2c, 0x52, 0x1b,
+	0x34, 0x38, 0xf8, 0x75, 0x5f, 0xa0, 0x74, 0xe2,
+	0x99, 0xdd, 0xa6, 0x4b, 0x14, 0x50, 0x4e, 0xf1,
+	0xbe, 0xd6, 0x9e, 0xdb, 0xb2, 0x24, 0x27, 0x74,
+	0x12, 0x4a, 0x78, 0x78, 0x17, 0xa5, 0x58, 0x8e,
+	0x2f, 0xf9, 0xf4, 0x8d, 0xee, 0x03, 0x88, 0xae,
+	0xb8, 0x29, 0xa1, 0x2f, 0x4b, 0xee, 0x92, 0xbd,
+	0x87, 0xb3, 0xce, 0x34, 0x21, 0x57, 0x46, 0x04,
+	0x49, 0x0c, 0x80, 0xf2, 0x01, 0x13, 0xa1, 0x55,
+	0xb3, 0xff, 0x44, 0x30, 0x3c, 0x1c, 0xd0, 0xef,
+	0xbc, 0x18, 0x74, 0x26, 0xad, 0x41, 0x5b, 0x5b,
+	0x3e, 0x9a, 0x7a, 0x46, 0x4f, 0x16, 0xd6, 0x74,
+	0x5a, 0xb7, 0x3a, 0x28, 0x31, 0xd8, 0xae, 0x26,
+	0xac, 0x50, 0x53, 0x86, 0xf2, 0x56, 0xd7, 0x3f,
+	0x29, 0xbc, 0x45, 0x68, 0x8e, 0xcb, 0x98, 0x64,
+	0xdd, 0xc9, 0xba, 0xb8, 0x4b, 0x7b, 0x82, 0xdd,
+	0x14, 0xa7, 0xcb, 0x71, 0x72, 0x00, 0x5c, 0xad,
+	0x7b, 0x6a, 0x89, 0xa4, 0x3d, 0xbf, 0xb5, 0x4b,
+	0x3e, 0x7c, 0x5a, 0xcf, 0xb8, 0xa1, 0xc5, 0x6e,
+	0xc8, 0xb6, 0x31, 0x57, 0x7b, 0xdf, 0xa5, 0x7e,
+	0xb1, 0xd6, 0x42, 0x2a, 0x31, 0x36, 0xd1, 0xd0,
+	0x3f, 0x7a, 0xe5, 0x94, 0xd6, 0x36, 0xa0, 0x6f,
+	0xb7, 0x40, 0x7d, 0x37, 0xc6, 0x55, 0x7c, 0x50,
+	0x40, 0x6d, 0x29, 0x89, 0xe3, 0x5a, 0xae, 0x97,
+	0xe7, 0x44, 0x49, 0x6e, 0xbd, 0x81, 0x3d, 0x03,
+	0x93, 0x06, 0x12, 0x06, 0xe2, 0x41, 0x12, 0x4a,
+	0xf1, 0x6a, 0xa4, 0x58, 0xa2, 0xfb, 0xd2, 0x15,
+	0xba, 0xc9, 0x79, 0xc9, 0xce, 0x5e, 0x13, 0xbb,
+	0xf1, 0x09, 0x04, 0xcc, 0xfd, 0xe8, 0x51, 0x34,
+	0x6a, 0xe8, 0x61, 0x88, 0xda, 0xed, 0x01, 0x47,
+	0x84, 0xf5, 0x73, 0x25, 0xf9, 0x1c, 0x42, 0x86,
+	0x07, 0xf3, 0x5b, 0x1a, 0x01, 0xb3, 0xeb, 0x24,
+	0x32, 0x8d, 0xf6, 0xed, 0x7c, 0x4b, 0xeb, 0x3c,
+	0x36, 0x42, 0x28, 0xdf, 0xdf, 0xb6, 0xbe, 0xd9,
+	0x8c, 0x52, 0xd3, 0x2b, 0x08, 0x90, 0x8c, 0xe7,
+	0x98, 0x31, 0xe2, 0x32, 0x8e, 0xfc, 0x11, 0x48,
+	0x00, 0xa8, 0x6a, 0x42, 0x4a, 0x02, 0xc6, 0x4b,
+	0x09, 0xf1, 0xe3, 0x49, 0xf3, 0x45, 0x1f, 0x0e,
+	0xbc, 0x56, 0xe2, 0xe4, 0xdf, 0xfb, 0xeb, 0x61,
+	0xfa, 0x24, 0xc1, 0x63, 0x75, 0xbb, 0x47, 0x75,
+	0xaf, 0xe1, 0x53, 0x16, 0x96, 0x21, 0x85, 0x26,
+	0x11, 0xb3, 0x76, 0xe3, 0x23, 0xa1, 0x6b, 0x74,
+	0x37, 0xd0, 0xde, 0x06, 0x90, 0x71, 0x5d, 0x43,
+	0x88, 0x9b, 0x00, 0x54, 0xa6, 0x75, 0x2f, 0xa1,
+	0xc2, 0x0b, 0x73, 0x20, 0x1d, 0xb6, 0x21, 0x79,
+	0x57, 0x3f, 0xfa, 0x09, 0xbe, 0x8a, 0x33, 0xc3,
+	0x52, 0xf0, 0x1d, 0x82, 0x31, 0xd1, 0x55, 0xb5,
+	0x6c, 0x99, 0x25, 0xcf, 0x5c, 0x32, 0xce, 0xe9,
+	0x0d, 0xfa, 0x69, 0x2c, 0xd5, 0x0d, 0xc5, 0x6d,
+	0x86, 0xd0, 0x0c, 0x3b, 0x06, 0x50, 0x79, 0xe8,
+	0xc3, 0xae, 0x04, 0xe6, 0xcd, 0x51, 0xe4, 0x26,
+	0x9b, 0x4f, 0x7e, 0xa6, 0x0f, 0xab, 0xd8, 0xe5,
+	0xde, 0xa9, 0x00, 0x95, 0xbe, 0xa3, 0x9d, 0x5d,
+	0xb2, 0x09, 0x70, 0x18, 0x1c, 0xf0, 0xac, 0x29,
+	0x23, 0x02, 0x29, 0x28, 0xd2, 0x74, 0x35, 0x57,
+	0x62, 0x0f, 0x24, 0xea, 0x5e, 0x33, 0xc2, 0x92,
+	0xf3, 0x78, 0x4d, 0x30, 0x1e, 0xa1, 0x99, 0xa9,
+	0x82, 0xb0, 0x42, 0x31, 0x8d, 0xad, 0x8a, 0xbc,
+	0xfc, 0xd4, 0x57, 0x47, 0x3e, 0xb4, 0x50, 0xdd,
+	0x6e, 0x2c, 0x80, 0x4d, 0x22, 0xf1, 0xfb, 0x57,
+	0xc4, 0xdd, 0x17, 0xe1, 0x8a, 0x36, 0x4a, 0xb3,
+	0x37, 0xca, 0xc9, 0x4e, 0xab, 0xd5, 0x69, 0xc4,
+	0xf4, 0xbc, 0x0b, 0x3b, 0x44, 0x4b, 0x29, 0x9c,
+	0xee, 0xd4, 0x35, 0x22, 0x21, 0xb0, 0x1f, 0x27,
+	0x64, 0xa8, 0x51, 0x1b, 0xf0, 0x9f, 0x19, 0x5c,
+	0xfb, 0x5a, 0x64, 0x74, 0x70, 0x45, 0x09, 0xf5,
+	0x64, 0xfe, 0x1a, 0x2d, 0xc9, 0x14, 0x04, 0x14,
+	0xcf, 0xd5, 0x7d, 0x60, 0xaf, 0x94, 0x39, 0x94,
+	0xe2, 0x7d, 0x79, 0x82, 0xd0, 0x65, 0x3b, 0x6b,
+	0x9c, 0x19, 0x84, 0xb4, 0x6d, 0xb3, 0x0c, 0x99,
+	0xc0, 0x56, 0xa8, 0xbd, 0x73, 0xce, 0x05, 0x84,
+	0x3e, 0x30, 0xaa, 0xc4, 0x9b, 0x1b, 0x04, 0x2a,
+	0x9f, 0xd7, 0x43, 0x2b, 0x23, 0xdf, 0xbf, 0xaa,
+	0xd5, 0xc2, 0x43, 0x2d, 0x70, 0xab, 0xdc, 0x75,
+	0xad, 0xac, 0xf7, 0xc0, 0xbe, 0x67, 0xb2, 0x74,
+	0xed, 0x67, 0x10, 0x4a, 0x92, 0x60, 0xc1, 0x40,
+	0x50, 0x19, 0x8a, 0x8a, 0x8c, 0x09, 0x0e, 0x72,
+	0xe1, 0x73, 0x5e, 0xe8, 0x41, 0x85, 0x63, 0x9f,
+	0x3f, 0xd7, 0x7d, 0xc4, 0xfb, 0x22, 0x5d, 0x92,
+	0x6c, 0xb3, 0x1e, 0xe2, 0x50, 0x2f, 0x82, 0xa8,
+	0x28, 0xc0, 0xb5, 0xd7, 0x5f, 0x68, 0x0d, 0x2c,
+	0x2d, 0xaf, 0x7e, 0xfa, 0x2e, 0x08, 0x0f, 0x1f,
+	0x70, 0x9f, 0xe9, 0x19, 0x72, 0x55, 0xf8, 0xfb,
+	0x51, 0xd2, 0x33, 0x5d, 0xa0, 0xd3, 0x2b, 0x0a,
+	0x6c, 0xbc, 0x4e, 0xcf, 0x36, 0x4d, 0xdc, 0x3b,
+	0xe9, 0x3e, 0x81, 0x7c, 0x61, 0xdb, 0x20, 0x2d,
+	0x3a, 0xc3, 0xb3, 0x0c, 0x1e, 0x00, 0xb9, 0x7c,
+	0xf5, 0xca, 0x10, 0x5f, 0x3a, 0x71, 0xb3, 0xe4,
+	0x20, 0xdb, 0x0c, 0x2a, 0x98, 0x63, 0x45, 0x00,
+	0x58, 0xf6, 0x68, 0xe4, 0x0b, 0xda, 0x13, 0x3b,
+	0x60, 0x5c, 0x76, 0xdb, 0xb9, 0x97, 0x71, 0xe4,
+	0xd9, 0xb7, 0xdb, 0xbd, 0x68, 0xc7, 0x84, 0x84,
+	0xaa, 0x7c, 0x68, 0x62, 0x5e, 0x16, 0xfc, 0xba,
+	0x72, 0xaa, 0x9a, 0xa9, 0xeb, 0x7c, 0x75, 0x47,
+	0x97, 0x7e, 0xad, 0xe2, 0xd9, 0x91, 0xe8, 0xe4,
+	0xa5, 0x31, 0xd7, 0x01, 0x8e, 0xa2, 0x11, 0x88,
+	0x95, 0xb9, 0xf2, 0x9b, 0xd3, 0x7f, 0x1b, 0x81,
+	0x22, 0xf7, 0x98, 0x60, 0x0a, 0x64, 0xa6, 0xc1,
+	0xf6, 0x49, 0xc7, 0xe3, 0x07, 0x4d, 0x94, 0x7a,
+	0xcf, 0x6e, 0x68, 0x0c, 0x1b, 0x3f, 0x6e, 0x2e,
+	0xee, 0x92, 0xfa, 0x52, 0xb3, 0x59, 0xf8, 0xf1,
+	0x8f, 0x6a, 0x66, 0xa3, 0x82, 0x76, 0x4a, 0x07,
+	0x1a, 0xc7, 0xdd, 0xf5, 0xda, 0x9c, 0x3c, 0x24,
+	0xbf, 0xfd, 0x42, 0xa1, 0x10, 0x64, 0x6a, 0x0f,
+	0x89, 0xee, 0x36, 0xa5, 0xce, 0x99, 0x48, 0x6a,
+	0xf0, 0x9f, 0x9e, 0x69, 0xa4, 0x40, 0x20, 0xe9,
+	0x16, 0x15, 0xf7, 0xdb, 0x75, 0x02, 0xcb, 0xe9,
+	0x73, 0x8b, 0x3b, 0x49, 0x2f, 0xf0, 0xaf, 0x51,
+	0x06, 0x5c, 0xdf, 0x27, 0x27, 0x49, 0x6a, 0xd1,
+	0xcc, 0xc7, 0xb5, 0x63, 0xb5, 0xfc, 0xb8, 0x5c,
+	0x87, 0x7f, 0x84, 0xb4, 0xcc, 0x14, 0xa9, 0x53,
+	0xda, 0xa4, 0x56, 0xf8, 0xb6, 0x1b, 0xcc, 0x40,
+	0x27, 0x52, 0x06, 0x5a, 0x13, 0x81, 0xd7, 0x3a,
+	0xd4, 0x3b, 0xfb, 0x49, 0x65, 0x31, 0x33, 0xb2,
+	0xfa, 0xcd, 0xad, 0x58, 0x4e, 0x2b, 0xae, 0xd2,
+	0x20, 0xfb, 0x1a, 0x48, 0xb4, 0x3f, 0x9a, 0xd8,
+	0x7a, 0x35, 0x4a, 0xc8, 0xee, 0x88, 0x5e, 0x07,
+	0x66, 0x54, 0xb9, 0xec, 0x9f, 0xa3, 0xe3, 0xb9,
+	0x37, 0xaa, 0x49, 0x76, 0x31, 0xda, 0x74, 0x2d,
+	0x3c, 0xa4, 0x65, 0x10, 0x32, 0x38, 0xf0, 0xde,
+	0xd3, 0x99, 0x17, 0xaa, 0x71, 0xaa, 0x8f, 0x0f,
+	0x8c, 0xaf, 0xa2, 0xf8, 0x5d, 0x64, 0xba, 0x1d,
+	0xa3, 0xef, 0x96, 0x73, 0xe8, 0xa1, 0x02, 0x8d,
+	0x0c, 0x6d, 0xb8, 0x06, 0x90, 0xb8, 0x08, 0x56,
+	0x2c, 0xa7, 0x06, 0xc9, 0xc2, 0x38, 0xdb, 0x7c,
+	0x63, 0xb1, 0x57, 0x8e, 0xea, 0x7c, 0x79, 0xf3,
+	0x49, 0x1d, 0xfe, 0x9f, 0xf3, 0x6e, 0xb1, 0x1d,
+	0xba, 0x19, 0x80, 0x1a, 0x0a, 0xd3, 0xb0, 0x26,
+	0x21, 0x40, 0xb1, 0x7c, 0xf9, 0x4d, 0x8d, 0x10,
+	0xc1, 0x7e, 0xf4, 0xf6, 0x3c, 0xa8, 0xfd, 0x7c,
+	0xa3, 0x92, 0xb2, 0x0f, 0xaa, 0xcc, 0xa6, 0x11,
+	0xfe, 0x04, 0xe3, 0xd1, 0x7a, 0x32, 0x89, 0xdf,
+	0x0d, 0xc4, 0x8f, 0x79, 0x6b, 0xca, 0x16, 0x7c,
+	0x6e, 0xf9, 0xad, 0x0f, 0xf6, 0xfe, 0x27, 0xdb,
+	0xc4, 0x13, 0x70, 0xf1, 0x62, 0x1a, 0x4f, 0x79,
+	0x40, 0xc9, 0x9b, 0x8b, 0x21, 0xea, 0x84, 0xfa,
+	0xf5, 0xf1, 0x89, 0xce, 0xb7, 0x55, 0x0a, 0x80,
+	0x39, 0x2f, 0x55, 0x36, 0x16, 0x9c, 0x7b, 0x08,
+	0xbd, 0x87, 0x0d, 0xa5, 0x32, 0xf1, 0x52, 0x7c,
+	0xe8, 0x55, 0x60, 0x5b, 0xd7, 0x69, 0xe4, 0xfc,
+	0xfa, 0x12, 0x85, 0x96, 0xea, 0x50, 0x28, 0xab,
+	0x8a, 0xf7, 0xbb, 0x0e, 0x53, 0x74, 0xca, 0xa6,
+	0x27, 0x09, 0xc2, 0xb5, 0xde, 0x18, 0x14, 0xd9,
+	0xea, 0xe5, 0x29, 0x1c, 0x40, 0x56, 0xcf, 0xd7,
+	0xae, 0x05, 0x3f, 0x65, 0xaf, 0x05, 0x73, 0xe2,
+	0x35, 0x96, 0x27, 0x07, 0x14, 0xc0, 0xad, 0x33,
+	0xf1, 0xdc, 0x44, 0x7a, 0x89, 0x17, 0x77, 0xd2,
+	0x9c, 0x58, 0x60, 0xf0, 0x3f, 0x7b, 0x2d, 0x2e,
+	0x57, 0x95, 0x54, 0x87, 0xed, 0xf2, 0xc7, 0x4c,
+	0xf0, 0xae, 0x56, 0x29, 0x19, 0x7d, 0x66, 0x4b,
+	0x9b, 0x83, 0x84, 0x42, 0x3b, 0x01, 0x25, 0x66,
+	0x8e, 0x02, 0xde, 0xb9, 0x83, 0x54, 0x19, 0xf6,
+	0x9f, 0x79, 0x0d, 0x67, 0xc5, 0x1d, 0x7a, 0x44,
+	0x02, 0x98, 0xa7, 0x16, 0x1c, 0x29, 0x0d, 0x74,
+	0xff, 0x85, 0x40, 0x06, 0xef, 0x2c, 0xa9, 0xc6,
+	0xf5, 0x53, 0x07, 0x06, 0xae, 0xe4, 0xfa, 0x5f,
+	0xd8, 0x39, 0x4d, 0xf1, 0x9b, 0x6b, 0xd9, 0x24,
+	0x84, 0xfe, 0x03, 0x4c, 0xb2, 0x3f, 0xdf, 0xa1,
+	0x05, 0x9e, 0x50, 0x14, 0x5a, 0xd9, 0x1a, 0xa2,
+	0xa7, 0xfa, 0xfa, 0x17, 0xf7, 0x78, 0xd6, 0xb5,
+	0x92, 0x61, 0x91, 0xac, 0x36, 0xfa, 0x56, 0x0d,
+	0x38, 0x32, 0x18, 0x85, 0x08, 0x58, 0x37, 0xf0,
+	0x4b, 0xdb, 0x59, 0xe7, 0xa4, 0x34, 0xc0, 0x1b,
+	0x01, 0xaf, 0x2d, 0xde, 0xa1, 0xaa, 0x5d, 0xd3,
+	0xec, 0xe1, 0xd4, 0xf7, 0xe6, 0x54, 0x68, 0xf0,
+	0x51, 0x97, 0xa7, 0x89, 0xea, 0x24, 0xad, 0xd3,
+	0x6e, 0x47, 0x93, 0x8b, 0x4b, 0xb4, 0xf7, 0x1c,
+	0x42, 0x06, 0x67, 0xe8, 0x99, 0xf6, 0xf5, 0x7b,
+	0x85, 0xb5, 0x65, 0xb5, 0xb5, 0xd2, 0x37, 0xf5,
+	0xf3, 0x02, 0xa6, 0x4d, 0x11, 0xa7, 0xdc, 0x51,
+	0x09, 0x7f, 0xa0, 0xd8, 0x88, 0x1c, 0x13, 0x71,
+	0xae, 0x9c, 0xb7, 0x7b, 0x34, 0xd6, 0x4e, 0x68,
+	0x26, 0x83, 0x51, 0xaf, 0x1d, 0xee, 0x8b, 0xbb,
+	0x69, 0x43, 0x2b, 0x9e, 0x8a, 0xbc, 0x02, 0x0e,
+	0xa0, 0x1b, 0xe0, 0xa8, 0x5f, 0x6f, 0xaf, 0x1b,
+	0x8f, 0xe7, 0x64, 0x71, 0x74, 0x11, 0x7e, 0xa8,
+	0xd8, 0xf9, 0x97, 0x06, 0xc3, 0xb6, 0xfb, 0xfb,
+	0xb7, 0x3d, 0x35, 0x9d, 0x3b, 0x52, 0xed, 0x54,
+	0xca, 0xf4, 0x81, 0x01, 0x2d, 0x1b, 0xc3, 0xa7,
+	0x00, 0x3d, 0x1a, 0x39, 0x54, 0xe1, 0xf6, 0xff,
+	0xed, 0x6f, 0x0b, 0x5a, 0x68, 0xda, 0x58, 0xdd,
+	0xa9, 0xcf, 0x5c, 0x4a, 0xe5, 0x09, 0x4e, 0xde,
+	0x9d, 0xbc, 0x3e, 0xee, 0x5a, 0x00, 0x3b, 0x2c,
+	0x87, 0x10, 0x65, 0x60, 0xdd, 0xd7, 0x56, 0xd1,
+	0x4c, 0x64, 0x45, 0xe4, 0x21, 0xec, 0x78, 0xf8,
+	0x25, 0x7a, 0x3e, 0x16, 0x5d, 0x09, 0x53, 0x14,
+	0xbe, 0x4f, 0xae, 0x87, 0xd8, 0xd1, 0xaa, 0x3c,
+	0xf6, 0x3e, 0xa4, 0x70, 0x8c, 0x5e, 0x70, 0xa4,
+	0xb3, 0x6b, 0x66, 0x73, 0xd3, 0xbf, 0x31, 0x06,
+	0x19, 0x62, 0x93, 0x15, 0xf2, 0x86, 0xe4, 0x52,
+	0x7e, 0x53, 0x4c, 0x12, 0x38, 0xcc, 0x34, 0x7d,
+	0x57, 0xf6, 0x42, 0x93, 0x8a, 0xc4, 0xee, 0x5c,
+	0x8a, 0xe1, 0x52, 0x8f, 0x56, 0x64, 0xf6, 0xa6,
+	0xd1, 0x91, 0x57, 0x70, 0xcd, 0x11, 0x76, 0xf5,
+	0x59, 0x60, 0x60, 0x3c, 0xc1, 0xc3, 0x0b, 0x7f,
+	0x58, 0x1a, 0x50, 0x91, 0xf1, 0x68, 0x8f, 0x6e,
+	0x74, 0x74, 0xa8, 0x51, 0x0b, 0xf7, 0x7a, 0x98,
+	0x37, 0xf2, 0x0a, 0x0e, 0xa4, 0x97, 0x04, 0xb8,
+	0x9b, 0xfd, 0xa0, 0xea, 0xf7, 0x0d, 0xe1, 0xdb,
+	0x03, 0xf0, 0x31, 0x29, 0xf8, 0xdd, 0x6b, 0x8b,
+	0x5d, 0xd8, 0x59, 0xa9, 0x29, 0xcf, 0x9a, 0x79,
+	0x89, 0x19, 0x63, 0x46, 0x09, 0x79, 0x6a, 0x11,
+	0xda, 0x63, 0x68, 0x48, 0x77, 0x23, 0xfb, 0x7d,
+	0x3a, 0x43, 0xcb, 0x02, 0x3b, 0x7a, 0x6d, 0x10,
+	0x2a, 0x9e, 0xac, 0xf1, 0xd4, 0x19, 0xf8, 0x23,
+	0x64, 0x1d, 0x2c, 0x5f, 0xf2, 0xb0, 0x5c, 0x23,
+	0x27, 0xf7, 0x27, 0x30, 0x16, 0x37, 0xb1, 0x90,
+	0xab, 0x38, 0xfb, 0x55, 0xcd, 0x78, 0x58, 0xd4,
+	0x7d, 0x43, 0xf6, 0x45, 0x5e, 0x55, 0x8d, 0xb1,
+	0x02, 0x65, 0x58, 0xb4, 0x13, 0x4b, 0x36, 0xf7,
+	0xcc, 0xfe, 0x3d, 0x0b, 0x82, 0xe2, 0x12, 0x11,
+	0xbb, 0xe6, 0xb8, 0x3a, 0x48, 0x71, 0xc7, 0x50,
+	0x06, 0x16, 0x3a, 0xe6, 0x7c, 0x05, 0xc7, 0xc8,
+	0x4d, 0x2f, 0x08, 0x6a, 0x17, 0x9a, 0x95, 0x97,
+	0x50, 0x68, 0xdc, 0x28, 0x18, 0xc4, 0x61, 0x38,
+	0xb9, 0xe0, 0x3e, 0x78, 0xdb, 0x29, 0xe0, 0x9f,
+	0x52, 0xdd, 0xf8, 0x4f, 0x91, 0xc1, 0xd0, 0x33,
+	0xa1, 0x7a, 0x8e, 0x30, 0x13, 0x82, 0x07, 0x9f,
+	0xd3, 0x31, 0x0f, 0x23, 0xbe, 0x32, 0x5a, 0x75,
+	0xcf, 0x96, 0xb2, 0xec, 0xb5, 0x32, 0xac, 0x21,
+	0xd1, 0x82, 0x33, 0xd3, 0x15, 0x74, 0xbd, 0x90,
+	0xf1, 0x2c, 0xe6, 0x5f, 0x8d, 0xe3, 0x02, 0xe8,
+	0xe9, 0xc4, 0xca, 0x96, 0xeb, 0x0e, 0xbc, 0x91,
+	0xf4, 0xb9, 0xea, 0xd9, 0x1b, 0x75, 0xbd, 0xe1,
+	0xac, 0x2a, 0x05, 0x37, 0x52, 0x9b, 0x1b, 0x3f,
+	0x5a, 0xdc, 0x21, 0xc3, 0x98, 0xbb, 0xaf, 0xa3,
+	0xf2, 0x00, 0xbf, 0x0d, 0x30, 0x89, 0x05, 0xcc,
+	0xa5, 0x76, 0xf5, 0x06, 0xf0, 0xc6, 0x54, 0x8a,
+	0x5d, 0xd4, 0x1e, 0xc1, 0xf2, 0xce, 0xb0, 0x62,
+	0xc8, 0xfc, 0x59, 0x42, 0x9a, 0x90, 0x60, 0x55,
+	0xfe, 0x88, 0xa5, 0x8b, 0xb8, 0x33, 0x0c, 0x23,
+	0x24, 0x0d, 0x15, 0x70, 0x37, 0x1e, 0x3d, 0xf6,
+	0xd2, 0xea, 0x92, 0x10, 0xb2, 0xc4, 0x51, 0xac,
+	0xf2, 0xac, 0xf3, 0x6b, 0x6c, 0xaa, 0xcf, 0x12,
+	0xc5, 0x6c, 0x90, 0x50, 0xb5, 0x0c, 0xfc, 0x1a,
+	0x15, 0x52, 0xe9, 0x26, 0xc6, 0x52, 0xa4, 0xe7,
+	0x81, 0x69, 0xe1, 0xe7, 0x9e, 0x30, 0x01, 0xec,
+	0x84, 0x89, 0xb2, 0x0d, 0x66, 0xdd, 0xce, 0x28,
+	0x5c, 0xec, 0x98, 0x46, 0x68, 0x21, 0x9f, 0x88,
+	0x3f, 0x1f, 0x42, 0x77, 0xce, 0xd0, 0x61, 0xd4,
+	0x20, 0xa7, 0xff, 0x53, 0xad, 0x37, 0xd0, 0x17,
+	0x35, 0xc9, 0xfc, 0xba, 0x0a, 0x78, 0x3f, 0xf2,
+	0xcc, 0x86, 0x89, 0xe8, 0x4b, 0x3c, 0x48, 0x33,
+	0x09, 0x7f, 0xc6, 0xc0, 0xdd, 0xb8, 0xfd, 0x7a,
+	0x66, 0x66, 0x65, 0xeb, 0x47, 0xa7, 0x04, 0x28,
+	0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62,
+	0x70, 0xcf, 0xd6
+};
+static const u8 enc_assoc012[] __initconst = {
+	0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8,
+	0x77, 0xe8, 0x21, 0xff, 0x06, 0x59, 0x35, 0xce,
+	0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c,
+	0xcf, 0x70, 0xdd, 0x0a, 0x68, 0xbf, 0xd4, 0xbc,
+	0x16, 0x76, 0x99, 0x36, 0x1e, 0x58, 0x79, 0x5e,
+	0xd4, 0x29, 0xf7, 0x33, 0x93, 0x48, 0xdb, 0x5f,
+	0x01, 0xae, 0x9c, 0xb6, 0xe4, 0x88, 0x6d, 0x2b,
+	0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9
+};
+static const u8 enc_nonce012[] __initconst = {
+	0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06
+};
+static const u8 enc_key012[] __initconst = {
+	0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e,
+	0x8f, 0x59, 0x8e, 0xc5, 0x90, 0xd5, 0x27, 0x2d,
+	0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, 0x44, 0x1e,
+	0x65, 0x91, 0x6e, 0x2a, 0x79, 0x22, 0xda, 0x64
+};
+
+/* wycheproof - rfc7539 */
+static const u8 enc_input013[] __initconst = {
+	0x4c, 0x61, 0x64, 0x69, 0x65, 0x73, 0x20, 0x61,
+	0x6e, 0x64, 0x20, 0x47, 0x65, 0x6e, 0x74, 0x6c,
+	0x65, 0x6d, 0x65, 0x6e, 0x20, 0x6f, 0x66, 0x20,
+	0x74, 0x68, 0x65, 0x20, 0x63, 0x6c, 0x61, 0x73,
+	0x73, 0x20, 0x6f, 0x66, 0x20, 0x27, 0x39, 0x39,
+	0x3a, 0x20, 0x49, 0x66, 0x20, 0x49, 0x20, 0x63,
+	0x6f, 0x75, 0x6c, 0x64, 0x20, 0x6f, 0x66, 0x66,
+	0x65, 0x72, 0x20, 0x79, 0x6f, 0x75, 0x20, 0x6f,
+	0x6e, 0x6c, 0x79, 0x20, 0x6f, 0x6e, 0x65, 0x20,
+	0x74, 0x69, 0x70, 0x20, 0x66, 0x6f, 0x72, 0x20,
+	0x74, 0x68, 0x65, 0x20, 0x66, 0x75, 0x74, 0x75,
+	0x72, 0x65, 0x2c, 0x20, 0x73, 0x75, 0x6e, 0x73,
+	0x63, 0x72, 0x65, 0x65, 0x6e, 0x20, 0x77, 0x6f,
+	0x75, 0x6c, 0x64, 0x20, 0x62, 0x65, 0x20, 0x69,
+	0x74, 0x2e
+};
+static const u8 enc_output013[] __initconst = {
+	0xd3, 0x1a, 0x8d, 0x34, 0x64, 0x8e, 0x60, 0xdb,
+	0x7b, 0x86, 0xaf, 0xbc, 0x53, 0xef, 0x7e, 0xc2,
+	0xa4, 0xad, 0xed, 0x51, 0x29, 0x6e, 0x08, 0xfe,
+	0xa9, 0xe2, 0xb5, 0xa7, 0x36, 0xee, 0x62, 0xd6,
+	0x3d, 0xbe, 0xa4, 0x5e, 0x8c, 0xa9, 0x67, 0x12,
+	0x82, 0xfa, 0xfb, 0x69, 0xda, 0x92, 0x72, 0x8b,
+	0x1a, 0x71, 0xde, 0x0a, 0x9e, 0x06, 0x0b, 0x29,
+	0x05, 0xd6, 0xa5, 0xb6, 0x7e, 0xcd, 0x3b, 0x36,
+	0x92, 0xdd, 0xbd, 0x7f, 0x2d, 0x77, 0x8b, 0x8c,
+	0x98, 0x03, 0xae, 0xe3, 0x28, 0x09, 0x1b, 0x58,
+	0xfa, 0xb3, 0x24, 0xe4, 0xfa, 0xd6, 0x75, 0x94,
+	0x55, 0x85, 0x80, 0x8b, 0x48, 0x31, 0xd7, 0xbc,
+	0x3f, 0xf4, 0xde, 0xf0, 0x8e, 0x4b, 0x7a, 0x9d,
+	0xe5, 0x76, 0xd2, 0x65, 0x86, 0xce, 0xc6, 0x4b,
+	0x61, 0x16, 0x1a, 0xe1, 0x0b, 0x59, 0x4f, 0x09,
+	0xe2, 0x6a, 0x7e, 0x90, 0x2e, 0xcb, 0xd0, 0x60,
+	0x06, 0x91
+};
+static const u8 enc_assoc013[] __initconst = {
+	0x50, 0x51, 0x52, 0x53, 0xc0, 0xc1, 0xc2, 0xc3,
+	0xc4, 0xc5, 0xc6, 0xc7
+};
+static const u8 enc_nonce013[] __initconst = {
+	0x07, 0x00, 0x00, 0x00, 0x40, 0x41, 0x42, 0x43,
+	0x44, 0x45, 0x46, 0x47
+};
+static const u8 enc_key013[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - misc */
+static const u8 enc_input014[] __initconst = { };
+static const u8 enc_output014[] __initconst = {
+	0x76, 0xac, 0xb3, 0x42, 0xcf, 0x31, 0x66, 0xa5,
+	0xb6, 0x3c, 0x0c, 0x0e, 0xa1, 0x38, 0x3c, 0x8d
+};
+static const u8 enc_assoc014[] __initconst = { };
+static const u8 enc_nonce014[] __initconst = {
+	0x4d, 0xa5, 0xbf, 0x8d, 0xfd, 0x58, 0x52, 0xc1,
+	0xea, 0x12, 0x37, 0x9d
+};
+static const u8 enc_key014[] __initconst = {
+	0x80, 0xba, 0x31, 0x92, 0xc8, 0x03, 0xce, 0x96,
+	0x5e, 0xa3, 0x71, 0xd5, 0xff, 0x07, 0x3c, 0xf0,
+	0xf4, 0x3b, 0x6a, 0x2a, 0xb5, 0x76, 0xb2, 0x08,
+	0x42, 0x6e, 0x11, 0x40, 0x9c, 0x09, 0xb9, 0xb0
+};
+
+/* wycheproof - misc */
+static const u8 enc_input015[] __initconst = { };
+static const u8 enc_output015[] __initconst = {
+	0x90, 0x6f, 0xa6, 0x28, 0x4b, 0x52, 0xf8, 0x7b,
+	0x73, 0x59, 0xcb, 0xaa, 0x75, 0x63, 0xc7, 0x09
+};
+static const u8 enc_assoc015[] __initconst = {
+	0xbd, 0x50, 0x67, 0x64, 0xf2, 0xd2, 0xc4, 0x10
+};
+static const u8 enc_nonce015[] __initconst = {
+	0xa9, 0x2e, 0xf0, 0xac, 0x99, 0x1d, 0xd5, 0x16,
+	0xa3, 0xc6, 0xf6, 0x89
+};
+static const u8 enc_key015[] __initconst = {
+	0x7a, 0x4c, 0xd7, 0x59, 0x17, 0x2e, 0x02, 0xeb,
+	0x20, 0x4d, 0xb2, 0xc3, 0xf5, 0xc7, 0x46, 0x22,
+	0x7d, 0xf5, 0x84, 0xfc, 0x13, 0x45, 0x19, 0x63,
+	0x91, 0xdb, 0xb9, 0x57, 0x7a, 0x25, 0x07, 0x42
+};
+
+/* wycheproof - misc */
+static const u8 enc_input016[] __initconst = {
+	0x2a
+};
+static const u8 enc_output016[] __initconst = {
+	0x3a, 0xca, 0xc2, 0x7d, 0xec, 0x09, 0x68, 0x80,
+	0x1e, 0x9f, 0x6e, 0xde, 0xd6, 0x9d, 0x80, 0x75,
+	0x22
+};
+static const u8 enc_assoc016[] __initconst = { };
+static const u8 enc_nonce016[] __initconst = {
+	0x99, 0xe2, 0x3e, 0xc4, 0x89, 0x85, 0xbc, 0xcd,
+	0xee, 0xab, 0x60, 0xf1
+};
+static const u8 enc_key016[] __initconst = {
+	0xcc, 0x56, 0xb6, 0x80, 0x55, 0x2e, 0xb7, 0x50,
+	0x08, 0xf5, 0x48, 0x4b, 0x4c, 0xb8, 0x03, 0xfa,
+	0x50, 0x63, 0xeb, 0xd6, 0xea, 0xb9, 0x1f, 0x6a,
+	0xb6, 0xae, 0xf4, 0x91, 0x6a, 0x76, 0x62, 0x73
+};
+
+/* wycheproof - misc */
+static const u8 enc_input017[] __initconst = {
+	0x51
+};
+static const u8 enc_output017[] __initconst = {
+	0xc4, 0x16, 0x83, 0x10, 0xca, 0x45, 0xb1, 0xf7,
+	0xc6, 0x6c, 0xad, 0x4e, 0x99, 0xe4, 0x3f, 0x72,
+	0xb9
+};
+static const u8 enc_assoc017[] __initconst = {
+	0x91, 0xca, 0x6c, 0x59, 0x2c, 0xbc, 0xca, 0x53
+};
+static const u8 enc_nonce017[] __initconst = {
+	0xab, 0x0d, 0xca, 0x71, 0x6e, 0xe0, 0x51, 0xd2,
+	0x78, 0x2f, 0x44, 0x03
+};
+static const u8 enc_key017[] __initconst = {
+	0x46, 0xf0, 0x25, 0x49, 0x65, 0xf7, 0x69, 0xd5,
+	0x2b, 0xdb, 0x4a, 0x70, 0xb4, 0x43, 0x19, 0x9f,
+	0x8e, 0xf2, 0x07, 0x52, 0x0d, 0x12, 0x20, 0xc5,
+	0x5e, 0x4b, 0x70, 0xf0, 0xfd, 0xa6, 0x20, 0xee
+};
+
+/* wycheproof - misc */
+static const u8 enc_input018[] __initconst = {
+	0x5c, 0x60
+};
+static const u8 enc_output018[] __initconst = {
+	0x4d, 0x13, 0x91, 0xe8, 0xb6, 0x1e, 0xfb, 0x39,
+	0xc1, 0x22, 0x19, 0x54, 0x53, 0x07, 0x7b, 0x22,
+	0xe5, 0xe2
+};
+static const u8 enc_assoc018[] __initconst = { };
+static const u8 enc_nonce018[] __initconst = {
+	0x46, 0x1a, 0xf1, 0x22, 0xe9, 0xf2, 0xe0, 0x34,
+	0x7e, 0x03, 0xf2, 0xdb
+};
+static const u8 enc_key018[] __initconst = {
+	0x2f, 0x7f, 0x7e, 0x4f, 0x59, 0x2b, 0xb3, 0x89,
+	0x19, 0x49, 0x89, 0x74, 0x35, 0x07, 0xbf, 0x3e,
+	0xe9, 0xcb, 0xde, 0x17, 0x86, 0xb6, 0x69, 0x5f,
+	0xe6, 0xc0, 0x25, 0xfd, 0x9b, 0xa4, 0xc1, 0x00
+};
+
+/* wycheproof - misc */
+static const u8 enc_input019[] __initconst = {
+	0xdd, 0xf2
+};
+static const u8 enc_output019[] __initconst = {
+	0xb6, 0x0d, 0xea, 0xd0, 0xfd, 0x46, 0x97, 0xec,
+	0x2e, 0x55, 0x58, 0x23, 0x77, 0x19, 0xd0, 0x24,
+	0x37, 0xa2
+};
+static const u8 enc_assoc019[] __initconst = {
+	0x88, 0x36, 0x4f, 0xc8, 0x06, 0x05, 0x18, 0xbf
+};
+static const u8 enc_nonce019[] __initconst = {
+	0x61, 0x54, 0x6b, 0xa5, 0xf1, 0x72, 0x05, 0x90,
+	0xb6, 0x04, 0x0a, 0xc6
+};
+static const u8 enc_key019[] __initconst = {
+	0xc8, 0x83, 0x3d, 0xce, 0x5e, 0xa9, 0xf2, 0x48,
+	0xaa, 0x20, 0x30, 0xea, 0xcf, 0xe7, 0x2b, 0xff,
+	0xe6, 0x9a, 0x62, 0x0c, 0xaf, 0x79, 0x33, 0x44,
+	0xe5, 0x71, 0x8f, 0xe0, 0xd7, 0xab, 0x1a, 0x58
+};
+
+/* wycheproof - misc */
+static const u8 enc_input020[] __initconst = {
+	0xab, 0x85, 0xe9, 0xc1, 0x57, 0x17, 0x31
+};
+static const u8 enc_output020[] __initconst = {
+	0x5d, 0xfe, 0x34, 0x40, 0xdb, 0xb3, 0xc3, 0xed,
+	0x7a, 0x43, 0x4e, 0x26, 0x02, 0xd3, 0x94, 0x28,
+	0x1e, 0x0a, 0xfa, 0x9f, 0xb7, 0xaa, 0x42
+};
+static const u8 enc_assoc020[] __initconst = { };
+static const u8 enc_nonce020[] __initconst = {
+	0x3c, 0x4e, 0x65, 0x4d, 0x66, 0x3f, 0xa4, 0x59,
+	0x6d, 0xc5, 0x5b, 0xb7
+};
+static const u8 enc_key020[] __initconst = {
+	0x55, 0x56, 0x81, 0x58, 0xd3, 0xa6, 0x48, 0x3f,
+	0x1f, 0x70, 0x21, 0xea, 0xb6, 0x9b, 0x70, 0x3f,
+	0x61, 0x42, 0x51, 0xca, 0xdc, 0x1a, 0xf5, 0xd3,
+	0x4a, 0x37, 0x4f, 0xdb, 0xfc, 0x5a, 0xda, 0xc7
+};
+
+/* wycheproof - misc */
+static const u8 enc_input021[] __initconst = {
+	0x4e, 0xe5, 0xcd, 0xa2, 0x0d, 0x42, 0x90
+};
+static const u8 enc_output021[] __initconst = {
+	0x4b, 0xd4, 0x72, 0x12, 0x94, 0x1c, 0xe3, 0x18,
+	0x5f, 0x14, 0x08, 0xee, 0x7f, 0xbf, 0x18, 0xf5,
+	0xab, 0xad, 0x6e, 0x22, 0x53, 0xa1, 0xba
+};
+static const u8 enc_assoc021[] __initconst = {
+	0x84, 0xe4, 0x6b, 0xe8, 0xc0, 0x91, 0x90, 0x53
+};
+static const u8 enc_nonce021[] __initconst = {
+	0x58, 0x38, 0x93, 0x75, 0xc6, 0x9e, 0xe3, 0x98,
+	0xde, 0x94, 0x83, 0x96
+};
+static const u8 enc_key021[] __initconst = {
+	0xe3, 0xc0, 0x9e, 0x7f, 0xab, 0x1a, 0xef, 0xb5,
+	0x16, 0xda, 0x6a, 0x33, 0x02, 0x2a, 0x1d, 0xd4,
+	0xeb, 0x27, 0x2c, 0x80, 0xd5, 0x40, 0xc5, 0xda,
+	0x52, 0xa7, 0x30, 0xf3, 0x4d, 0x84, 0x0d, 0x7f
+};
+
+/* wycheproof - misc */
+static const u8 enc_input022[] __initconst = {
+	0xbe, 0x33, 0x08, 0xf7, 0x2a, 0x2c, 0x6a, 0xed
+};
+static const u8 enc_output022[] __initconst = {
+	0x8e, 0x94, 0x39, 0xa5, 0x6e, 0xee, 0xc8, 0x17,
+	0xfb, 0xe8, 0xa6, 0xed, 0x8f, 0xab, 0xb1, 0x93,
+	0x75, 0x39, 0xdd, 0x6c, 0x00, 0xe9, 0x00, 0x21
+};
+static const u8 enc_assoc022[] __initconst = { };
+static const u8 enc_nonce022[] __initconst = {
+	0x4f, 0x07, 0xaf, 0xed, 0xfd, 0xc3, 0xb6, 0xc2,
+	0x36, 0x18, 0x23, 0xd3
+};
+static const u8 enc_key022[] __initconst = {
+	0x51, 0xe4, 0xbf, 0x2b, 0xad, 0x92, 0xb7, 0xaf,
+	0xf1, 0xa4, 0xbc, 0x05, 0x55, 0x0b, 0xa8, 0x1d,
+	0xf4, 0xb9, 0x6f, 0xab, 0xf4, 0x1c, 0x12, 0xc7,
+	0xb0, 0x0e, 0x60, 0xe4, 0x8d, 0xb7, 0xe1, 0x52
+};
+
+/* wycheproof - misc */
+static const u8 enc_input023[] __initconst = {
+	0xa4, 0xc9, 0xc2, 0x80, 0x1b, 0x71, 0xf7, 0xdf
+};
+static const u8 enc_output023[] __initconst = {
+	0xb9, 0xb9, 0x10, 0x43, 0x3a, 0xf0, 0x52, 0xb0,
+	0x45, 0x30, 0xf5, 0x1a, 0xee, 0xe0, 0x24, 0xe0,
+	0xa4, 0x45, 0xa6, 0x32, 0x8f, 0xa6, 0x7a, 0x18
+};
+static const u8 enc_assoc023[] __initconst = {
+	0x66, 0xc0, 0xae, 0x70, 0x07, 0x6c, 0xb1, 0x4d
+};
+static const u8 enc_nonce023[] __initconst = {
+	0xb4, 0xea, 0x66, 0x6e, 0xe1, 0x19, 0x56, 0x33,
+	0x66, 0x48, 0x4a, 0x78
+};
+static const u8 enc_key023[] __initconst = {
+	0x11, 0x31, 0xc1, 0x41, 0x85, 0x77, 0xa0, 0x54,
+	0xde, 0x7a, 0x4a, 0xc5, 0x51, 0x95, 0x0f, 0x1a,
+	0x05, 0x3f, 0x9a, 0xe4, 0x6e, 0x5b, 0x75, 0xfe,
+	0x4a, 0xbd, 0x56, 0x08, 0xd7, 0xcd, 0xda, 0xdd
+};
+
+/* wycheproof - misc */
+static const u8 enc_input024[] __initconst = {
+	0x42, 0xba, 0xae, 0x59, 0x78, 0xfe, 0xaf, 0x5c,
+	0x36, 0x8d, 0x14, 0xe0
+};
+static const u8 enc_output024[] __initconst = {
+	0xff, 0x7d, 0xc2, 0x03, 0xb2, 0x6c, 0x46, 0x7a,
+	0x6b, 0x50, 0xdb, 0x33, 0x57, 0x8c, 0x0f, 0x27,
+	0x58, 0xc2, 0xe1, 0x4e, 0x36, 0xd4, 0xfc, 0x10,
+	0x6d, 0xcb, 0x29, 0xb4
+};
+static const u8 enc_assoc024[] __initconst = { };
+static const u8 enc_nonce024[] __initconst = {
+	0x9a, 0x59, 0xfc, 0xe2, 0x6d, 0xf0, 0x00, 0x5e,
+	0x07, 0x53, 0x86, 0x56
+};
+static const u8 enc_key024[] __initconst = {
+	0x99, 0xb6, 0x2b, 0xd5, 0xaf, 0xbe, 0x3f, 0xb0,
+	0x15, 0xbd, 0xe9, 0x3f, 0x0a, 0xbf, 0x48, 0x39,
+	0x57, 0xa1, 0xc3, 0xeb, 0x3c, 0xa5, 0x9c, 0xb5,
+	0x0b, 0x39, 0xf7, 0xf8, 0xa9, 0xcc, 0x51, 0xbe
+};
+
+/* wycheproof - misc */
+static const u8 enc_input025[] __initconst = {
+	0xfd, 0xc8, 0x5b, 0x94, 0xa4, 0xb2, 0xa6, 0xb7,
+	0x59, 0xb1, 0xa0, 0xda
+};
+static const u8 enc_output025[] __initconst = {
+	0x9f, 0x88, 0x16, 0xde, 0x09, 0x94, 0xe9, 0x38,
+	0xd9, 0xe5, 0x3f, 0x95, 0xd0, 0x86, 0xfc, 0x6c,
+	0x9d, 0x8f, 0xa9, 0x15, 0xfd, 0x84, 0x23, 0xa7,
+	0xcf, 0x05, 0x07, 0x2f
+};
+static const u8 enc_assoc025[] __initconst = {
+	0xa5, 0x06, 0xe1, 0xa5, 0xc6, 0x90, 0x93, 0xf9
+};
+static const u8 enc_nonce025[] __initconst = {
+	0x58, 0xdb, 0xd4, 0xad, 0x2c, 0x4a, 0xd3, 0x5d,
+	0xd9, 0x06, 0xe9, 0xce
+};
+static const u8 enc_key025[] __initconst = {
+	0x85, 0xf3, 0x5b, 0x62, 0x82, 0xcf, 0xf4, 0x40,
+	0xbc, 0x10, 0x20, 0xc8, 0x13, 0x6f, 0xf2, 0x70,
+	0x31, 0x11, 0x0f, 0xa6, 0x3e, 0xc1, 0x6f, 0x1e,
+	0x82, 0x51, 0x18, 0xb0, 0x06, 0xb9, 0x12, 0x57
+};
+
+/* wycheproof - misc */
+static const u8 enc_input026[] __initconst = {
+	0x51, 0xf8, 0xc1, 0xf7, 0x31, 0xea, 0x14, 0xac,
+	0xdb, 0x21, 0x0a, 0x6d, 0x97, 0x3e, 0x07
+};
+static const u8 enc_output026[] __initconst = {
+	0x0b, 0x29, 0x63, 0x8e, 0x1f, 0xbd, 0xd6, 0xdf,
+	0x53, 0x97, 0x0b, 0xe2, 0x21, 0x00, 0x42, 0x2a,
+	0x91, 0x34, 0x08, 0x7d, 0x67, 0xa4, 0x6e, 0x79,
+	0x17, 0x8d, 0x0a, 0x93, 0xf5, 0xe1, 0xd2
+};
+static const u8 enc_assoc026[] __initconst = { };
+static const u8 enc_nonce026[] __initconst = {
+	0x68, 0xab, 0x7f, 0xdb, 0xf6, 0x19, 0x01, 0xda,
+	0xd4, 0x61, 0xd2, 0x3c
+};
+static const u8 enc_key026[] __initconst = {
+	0x67, 0x11, 0x96, 0x27, 0xbd, 0x98, 0x8e, 0xda,
+	0x90, 0x62, 0x19, 0xe0, 0x8c, 0x0d, 0x0d, 0x77,
+	0x9a, 0x07, 0xd2, 0x08, 0xce, 0x8a, 0x4f, 0xe0,
+	0x70, 0x9a, 0xf7, 0x55, 0xee, 0xec, 0x6d, 0xcb
+};
+
+/* wycheproof - misc */
+static const u8 enc_input027[] __initconst = {
+	0x97, 0x46, 0x9d, 0xa6, 0x67, 0xd6, 0x11, 0x0f,
+	0x9c, 0xbd, 0xa1, 0xd1, 0xa2, 0x06, 0x73
+};
+static const u8 enc_output027[] __initconst = {
+	0x32, 0xdb, 0x66, 0xc4, 0xa3, 0x81, 0x9d, 0x81,
+	0x55, 0x74, 0x55, 0xe5, 0x98, 0x0f, 0xed, 0xfe,
+	0xae, 0x30, 0xde, 0xc9, 0x4e, 0x6a, 0xd3, 0xa9,
+	0xee, 0xa0, 0x6a, 0x0d, 0x70, 0x39, 0x17
+};
+static const u8 enc_assoc027[] __initconst = {
+	0x64, 0x53, 0xa5, 0x33, 0x84, 0x63, 0x22, 0x12
+};
+static const u8 enc_nonce027[] __initconst = {
+	0xd9, 0x5b, 0x32, 0x43, 0xaf, 0xae, 0xf7, 0x14,
+	0xc5, 0x03, 0x5b, 0x6a
+};
+static const u8 enc_key027[] __initconst = {
+	0xe6, 0xf1, 0x11, 0x8d, 0x41, 0xe4, 0xb4, 0x3f,
+	0xb5, 0x82, 0x21, 0xb7, 0xed, 0x79, 0x67, 0x38,
+	0x34, 0xe0, 0xd8, 0xac, 0x5c, 0x4f, 0xa6, 0x0b,
+	0xbc, 0x8b, 0xc4, 0x89, 0x3a, 0x58, 0x89, 0x4d
+};
+
+/* wycheproof - misc */
+static const u8 enc_input028[] __initconst = {
+	0x54, 0x9b, 0x36, 0x5a, 0xf9, 0x13, 0xf3, 0xb0,
+	0x81, 0x13, 0x1c, 0xcb, 0x6b, 0x82, 0x55, 0x88
+};
+static const u8 enc_output028[] __initconst = {
+	0xe9, 0x11, 0x0e, 0x9f, 0x56, 0xab, 0x3c, 0xa4,
+	0x83, 0x50, 0x0c, 0xea, 0xba, 0xb6, 0x7a, 0x13,
+	0x83, 0x6c, 0xca, 0xbf, 0x15, 0xa6, 0xa2, 0x2a,
+	0x51, 0xc1, 0x07, 0x1c, 0xfa, 0x68, 0xfa, 0x0c
+};
+static const u8 enc_assoc028[] __initconst = { };
+static const u8 enc_nonce028[] __initconst = {
+	0x2f, 0xcb, 0x1b, 0x38, 0xa9, 0x9e, 0x71, 0xb8,
+	0x47, 0x40, 0xad, 0x9b
+};
+static const u8 enc_key028[] __initconst = {
+	0x59, 0xd4, 0xea, 0xfb, 0x4d, 0xe0, 0xcf, 0xc7,
+	0xd3, 0xdb, 0x99, 0xa8, 0xf5, 0x4b, 0x15, 0xd7,
+	0xb3, 0x9f, 0x0a, 0xcc, 0x8d, 0xa6, 0x97, 0x63,
+	0xb0, 0x19, 0xc1, 0x69, 0x9f, 0x87, 0x67, 0x4a
+};
+
+/* wycheproof - misc */
+static const u8 enc_input029[] __initconst = {
+	0x55, 0xa4, 0x65, 0x64, 0x4f, 0x5b, 0x65, 0x09,
+	0x28, 0xcb, 0xee, 0x7c, 0x06, 0x32, 0x14, 0xd6
+};
+static const u8 enc_output029[] __initconst = {
+	0xe4, 0xb1, 0x13, 0xcb, 0x77, 0x59, 0x45, 0xf3,
+	0xd3, 0xa8, 0xae, 0x9e, 0xc1, 0x41, 0xc0, 0x0c,
+	0x7c, 0x43, 0xf1, 0x6c, 0xe0, 0x96, 0xd0, 0xdc,
+	0x27, 0xc9, 0x58, 0x49, 0xdc, 0x38, 0x3b, 0x7d
+};
+static const u8 enc_assoc029[] __initconst = {
+	0x03, 0x45, 0x85, 0x62, 0x1a, 0xf8, 0xd7, 0xff
+};
+static const u8 enc_nonce029[] __initconst = {
+	0x11, 0x8a, 0x69, 0x64, 0xc2, 0xd3, 0xe3, 0x80,
+	0x07, 0x1f, 0x52, 0x66
+};
+static const u8 enc_key029[] __initconst = {
+	0xb9, 0x07, 0xa4, 0x50, 0x75, 0x51, 0x3f, 0xe8,
+	0xa8, 0x01, 0x9e, 0xde, 0xe3, 0xf2, 0x59, 0x14,
+	0x87, 0xb2, 0xa0, 0x30, 0xb0, 0x3c, 0x6e, 0x1d,
+	0x77, 0x1c, 0x86, 0x25, 0x71, 0xd2, 0xea, 0x1e
+};
+
+/* wycheproof - misc */
+static const u8 enc_input030[] __initconst = {
+	0x3f, 0xf1, 0x51, 0x4b, 0x1c, 0x50, 0x39, 0x15,
+	0x91, 0x8f, 0x0c, 0x0c, 0x31, 0x09, 0x4a, 0x6e,
+	0x1f
+};
+static const u8 enc_output030[] __initconst = {
+	0x02, 0xcc, 0x3a, 0xcb, 0x5e, 0xe1, 0xfc, 0xdd,
+	0x12, 0xa0, 0x3b, 0xb8, 0x57, 0x97, 0x64, 0x74,
+	0xd3, 0xd8, 0x3b, 0x74, 0x63, 0xa2, 0xc3, 0x80,
+	0x0f, 0xe9, 0x58, 0xc2, 0x8e, 0xaa, 0x29, 0x08,
+	0x13
+};
+static const u8 enc_assoc030[] __initconst = { };
+static const u8 enc_nonce030[] __initconst = {
+	0x45, 0xaa, 0xa3, 0xe5, 0xd1, 0x6d, 0x2d, 0x42,
+	0xdc, 0x03, 0x44, 0x5d
+};
+static const u8 enc_key030[] __initconst = {
+	0x3b, 0x24, 0x58, 0xd8, 0x17, 0x6e, 0x16, 0x21,
+	0xc0, 0xcc, 0x24, 0xc0, 0xc0, 0xe2, 0x4c, 0x1e,
+	0x80, 0xd7, 0x2f, 0x7e, 0xe9, 0x14, 0x9a, 0x4b,
+	0x16, 0x61, 0x76, 0x62, 0x96, 0x16, 0xd0, 0x11
+};
+
+/* wycheproof - misc */
+static const u8 enc_input031[] __initconst = {
+	0x63, 0x85, 0x8c, 0xa3, 0xe2, 0xce, 0x69, 0x88,
+	0x7b, 0x57, 0x8a, 0x3c, 0x16, 0x7b, 0x42, 0x1c,
+	0x9c
+};
+static const u8 enc_output031[] __initconst = {
+	0x35, 0x76, 0x64, 0x88, 0xd2, 0xbc, 0x7c, 0x2b,
+	0x8d, 0x17, 0xcb, 0xbb, 0x9a, 0xbf, 0xad, 0x9e,
+	0x6d, 0x1f, 0x39, 0x1e, 0x65, 0x7b, 0x27, 0x38,
+	0xdd, 0xa0, 0x84, 0x48, 0xcb, 0xa2, 0x81, 0x1c,
+	0xeb
+};
+static const u8 enc_assoc031[] __initconst = {
+	0x9a, 0xaf, 0x29, 0x9e, 0xee, 0xa7, 0x8f, 0x79
+};
+static const u8 enc_nonce031[] __initconst = {
+	0xf0, 0x38, 0x4f, 0xb8, 0x76, 0x12, 0x14, 0x10,
+	0x63, 0x3d, 0x99, 0x3d
+};
+static const u8 enc_key031[] __initconst = {
+	0xf6, 0x0c, 0x6a, 0x1b, 0x62, 0x57, 0x25, 0xf7,
+	0x6c, 0x70, 0x37, 0xb4, 0x8f, 0xe3, 0x57, 0x7f,
+	0xa7, 0xf7, 0xb8, 0x7b, 0x1b, 0xd5, 0xa9, 0x82,
+	0x17, 0x6d, 0x18, 0x23, 0x06, 0xff, 0xb8, 0x70
+};
+
+/* wycheproof - misc */
+static const u8 enc_input032[] __initconst = {
+	0x10, 0xf1, 0xec, 0xf9, 0xc6, 0x05, 0x84, 0x66,
+	0x5d, 0x9a, 0xe5, 0xef, 0xe2, 0x79, 0xe7, 0xf7,
+	0x37, 0x7e, 0xea, 0x69, 0x16, 0xd2, 0xb1, 0x11
+};
+static const u8 enc_output032[] __initconst = {
+	0x42, 0xf2, 0x6c, 0x56, 0xcb, 0x4b, 0xe2, 0x1d,
+	0x9d, 0x8d, 0x0c, 0x80, 0xfc, 0x99, 0xdd, 0xe0,
+	0x0d, 0x75, 0xf3, 0x80, 0x74, 0xbf, 0xe7, 0x64,
+	0x54, 0xaa, 0x7e, 0x13, 0xd4, 0x8f, 0xff, 0x7d,
+	0x75, 0x57, 0x03, 0x94, 0x57, 0x04, 0x0a, 0x3a
+};
+static const u8 enc_assoc032[] __initconst = { };
+static const u8 enc_nonce032[] __initconst = {
+	0xe6, 0xb1, 0xad, 0xf2, 0xfd, 0x58, 0xa8, 0x76,
+	0x2c, 0x65, 0xf3, 0x1b
+};
+static const u8 enc_key032[] __initconst = {
+	0x02, 0x12, 0xa8, 0xde, 0x50, 0x07, 0xed, 0x87,
+	0xb3, 0x3f, 0x1a, 0x70, 0x90, 0xb6, 0x11, 0x4f,
+	0x9e, 0x08, 0xce, 0xfd, 0x96, 0x07, 0xf2, 0xc2,
+	0x76, 0xbd, 0xcf, 0xdb, 0xc5, 0xce, 0x9c, 0xd7
+};
+
+/* wycheproof - misc */
+static const u8 enc_input033[] __initconst = {
+	0x92, 0x22, 0xf9, 0x01, 0x8e, 0x54, 0xfd, 0x6d,
+	0xe1, 0x20, 0x08, 0x06, 0xa9, 0xee, 0x8e, 0x4c,
+	0xc9, 0x04, 0xd2, 0x9f, 0x25, 0xcb, 0xa1, 0x93
+};
+static const u8 enc_output033[] __initconst = {
+	0x12, 0x30, 0x32, 0x43, 0x7b, 0x4b, 0xfd, 0x69,
+	0x20, 0xe8, 0xf7, 0xe7, 0xe0, 0x08, 0x7a, 0xe4,
+	0x88, 0x9e, 0xbe, 0x7a, 0x0a, 0xd0, 0xe9, 0x00,
+	0x3c, 0xf6, 0x8f, 0x17, 0x95, 0x50, 0xda, 0x63,
+	0xd3, 0xb9, 0x6c, 0x2d, 0x55, 0x41, 0x18, 0x65
+};
+static const u8 enc_assoc033[] __initconst = {
+	0x3e, 0x8b, 0xc5, 0xad, 0xe1, 0x82, 0xff, 0x08
+};
+static const u8 enc_nonce033[] __initconst = {
+	0x6b, 0x28, 0x2e, 0xbe, 0xcc, 0x54, 0x1b, 0xcd,
+	0x78, 0x34, 0xed, 0x55
+};
+static const u8 enc_key033[] __initconst = {
+	0xc5, 0xbc, 0x09, 0x56, 0x56, 0x46, 0xe7, 0xed,
+	0xda, 0x95, 0x4f, 0x1f, 0x73, 0x92, 0x23, 0xda,
+	0xda, 0x20, 0xb9, 0x5c, 0x44, 0xab, 0x03, 0x3d,
+	0x0f, 0xae, 0x4b, 0x02, 0x83, 0xd1, 0x8b, 0xe3
+};
+
+/* wycheproof - misc */
+static const u8 enc_input034[] __initconst = {
+	0xb0, 0x53, 0x99, 0x92, 0x86, 0xa2, 0x82, 0x4f,
+	0x42, 0xcc, 0x8c, 0x20, 0x3a, 0xb2, 0x4e, 0x2c,
+	0x97, 0xa6, 0x85, 0xad, 0xcc, 0x2a, 0xd3, 0x26,
+	0x62, 0x55, 0x8e, 0x55, 0xa5, 0xc7, 0x29
+};
+static const u8 enc_output034[] __initconst = {
+	0x45, 0xc7, 0xd6, 0xb5, 0x3a, 0xca, 0xd4, 0xab,
+	0xb6, 0x88, 0x76, 0xa6, 0xe9, 0x6a, 0x48, 0xfb,
+	0x59, 0x52, 0x4d, 0x2c, 0x92, 0xc9, 0xd8, 0xa1,
+	0x89, 0xc9, 0xfd, 0x2d, 0xb9, 0x17, 0x46, 0x56,
+	0x6d, 0x3c, 0xa1, 0x0e, 0x31, 0x1b, 0x69, 0x5f,
+	0x3e, 0xae, 0x15, 0x51, 0x65, 0x24, 0x93
+};
+static const u8 enc_assoc034[] __initconst = { };
+static const u8 enc_nonce034[] __initconst = {
+	0x04, 0xa9, 0xbe, 0x03, 0x50, 0x8a, 0x5f, 0x31,
+	0x37, 0x1a, 0x6f, 0xd2
+};
+static const u8 enc_key034[] __initconst = {
+	0x2e, 0xb5, 0x1c, 0x46, 0x9a, 0xa8, 0xeb, 0x9e,
+	0x6c, 0x54, 0xa8, 0x34, 0x9b, 0xae, 0x50, 0xa2,
+	0x0f, 0x0e, 0x38, 0x27, 0x11, 0xbb, 0xa1, 0x15,
+	0x2c, 0x42, 0x4f, 0x03, 0xb6, 0x67, 0x1d, 0x71
+};
+
+/* wycheproof - misc */
+static const u8 enc_input035[] __initconst = {
+	0xf4, 0x52, 0x06, 0xab, 0xc2, 0x55, 0x52, 0xb2,
+	0xab, 0xc9, 0xab, 0x7f, 0xa2, 0x43, 0x03, 0x5f,
+	0xed, 0xaa, 0xdd, 0xc3, 0xb2, 0x29, 0x39, 0x56,
+	0xf1, 0xea, 0x6e, 0x71, 0x56, 0xe7, 0xeb
+};
+static const u8 enc_output035[] __initconst = {
+	0x46, 0xa8, 0x0c, 0x41, 0x87, 0x02, 0x47, 0x20,
+	0x08, 0x46, 0x27, 0x58, 0x00, 0x80, 0xdd, 0xe5,
+	0xa3, 0xf4, 0xa1, 0x10, 0x93, 0xa7, 0x07, 0x6e,
+	0xd6, 0xf3, 0xd3, 0x26, 0xbc, 0x7b, 0x70, 0x53,
+	0x4d, 0x4a, 0xa2, 0x83, 0x5a, 0x52, 0xe7, 0x2d,
+	0x14, 0xdf, 0x0e, 0x4f, 0x47, 0xf2, 0x5f
+};
+static const u8 enc_assoc035[] __initconst = {
+	0x37, 0x46, 0x18, 0xa0, 0x6e, 0xa9, 0x8a, 0x48
+};
+static const u8 enc_nonce035[] __initconst = {
+	0x47, 0x0a, 0x33, 0x9e, 0xcb, 0x32, 0x19, 0xb8,
+	0xb8, 0x1a, 0x1f, 0x8b
+};
+static const u8 enc_key035[] __initconst = {
+	0x7f, 0x5b, 0x74, 0xc0, 0x7e, 0xd1, 0xb4, 0x0f,
+	0xd1, 0x43, 0x58, 0xfe, 0x2f, 0xf2, 0xa7, 0x40,
+	0xc1, 0x16, 0xc7, 0x70, 0x65, 0x10, 0xe6, 0xa4,
+	0x37, 0xf1, 0x9e, 0xa4, 0x99, 0x11, 0xce, 0xc4
+};
+
+/* wycheproof - misc */
+static const u8 enc_input036[] __initconst = {
+	0xb9, 0xc5, 0x54, 0xcb, 0xc3, 0x6a, 0xc1, 0x8a,
+	0xe8, 0x97, 0xdf, 0x7b, 0xee, 0xca, 0xc1, 0xdb,
+	0xeb, 0x4e, 0xaf, 0xa1, 0x56, 0xbb, 0x60, 0xce,
+	0x2e, 0x5d, 0x48, 0xf0, 0x57, 0x15, 0xe6, 0x78
+};
+static const u8 enc_output036[] __initconst = {
+	0xea, 0x29, 0xaf, 0xa4, 0x9d, 0x36, 0xe8, 0x76,
+	0x0f, 0x5f, 0xe1, 0x97, 0x23, 0xb9, 0x81, 0x1e,
+	0xd5, 0xd5, 0x19, 0x93, 0x4a, 0x44, 0x0f, 0x50,
+	0x81, 0xac, 0x43, 0x0b, 0x95, 0x3b, 0x0e, 0x21,
+	0x22, 0x25, 0x41, 0xaf, 0x46, 0xb8, 0x65, 0x33,
+	0xc6, 0xb6, 0x8d, 0x2f, 0xf1, 0x08, 0xa7, 0xea
+};
+static const u8 enc_assoc036[] __initconst = { };
+static const u8 enc_nonce036[] __initconst = {
+	0x72, 0xcf, 0xd9, 0x0e, 0xf3, 0x02, 0x6c, 0xa2,
+	0x2b, 0x7e, 0x6e, 0x6a
+};
+static const u8 enc_key036[] __initconst = {
+	0xe1, 0x73, 0x1d, 0x58, 0x54, 0xe1, 0xb7, 0x0c,
+	0xb3, 0xff, 0xe8, 0xb7, 0x86, 0xa2, 0xb3, 0xeb,
+	0xf0, 0x99, 0x43, 0x70, 0x95, 0x47, 0x57, 0xb9,
+	0xdc, 0x8c, 0x7b, 0xc5, 0x35, 0x46, 0x34, 0xa3
+};
+
+/* wycheproof - misc */
+static const u8 enc_input037[] __initconst = {
+	0x6b, 0x26, 0x04, 0x99, 0x6c, 0xd3, 0x0c, 0x14,
+	0xa1, 0x3a, 0x52, 0x57, 0xed, 0x6c, 0xff, 0xd3,
+	0xbc, 0x5e, 0x29, 0xd6, 0xb9, 0x7e, 0xb1, 0x79,
+	0x9e, 0xb3, 0x35, 0xe2, 0x81, 0xea, 0x45, 0x1e
+};
+static const u8 enc_output037[] __initconst = {
+	0x6d, 0xad, 0x63, 0x78, 0x97, 0x54, 0x4d, 0x8b,
+	0xf6, 0xbe, 0x95, 0x07, 0xed, 0x4d, 0x1b, 0xb2,
+	0xe9, 0x54, 0xbc, 0x42, 0x7e, 0x5d, 0xe7, 0x29,
+	0xda, 0xf5, 0x07, 0x62, 0x84, 0x6f, 0xf2, 0xf4,
+	0x7b, 0x99, 0x7d, 0x93, 0xc9, 0x82, 0x18, 0x9d,
+	0x70, 0x95, 0xdc, 0x79, 0x4c, 0x74, 0x62, 0x32
+};
+static const u8 enc_assoc037[] __initconst = {
+	0x23, 0x33, 0xe5, 0xce, 0x0f, 0x93, 0xb0, 0x59
+};
+static const u8 enc_nonce037[] __initconst = {
+	0x26, 0x28, 0x80, 0xd4, 0x75, 0xf3, 0xda, 0xc5,
+	0x34, 0x0d, 0xd1, 0xb8
+};
+static const u8 enc_key037[] __initconst = {
+	0x27, 0xd8, 0x60, 0x63, 0x1b, 0x04, 0x85, 0xa4,
+	0x10, 0x70, 0x2f, 0xea, 0x61, 0xbc, 0x87, 0x3f,
+	0x34, 0x42, 0x26, 0x0c, 0xad, 0xed, 0x4a, 0xbd,
+	0xe2, 0x5b, 0x78, 0x6a, 0x2d, 0x97, 0xf1, 0x45
+};
+
+/* wycheproof - misc */
+static const u8 enc_input038[] __initconst = {
+	0x97, 0x3d, 0x0c, 0x75, 0x38, 0x26, 0xba, 0xe4,
+	0x66, 0xcf, 0x9a, 0xbb, 0x34, 0x93, 0x15, 0x2e,
+	0x9d, 0xe7, 0x81, 0x9e, 0x2b, 0xd0, 0xc7, 0x11,
+	0x71, 0x34, 0x6b, 0x4d, 0x2c, 0xeb, 0xf8, 0x04,
+	0x1a, 0xa3, 0xce, 0xdc, 0x0d, 0xfd, 0x7b, 0x46,
+	0x7e, 0x26, 0x22, 0x8b, 0xc8, 0x6c, 0x9a
+};
+static const u8 enc_output038[] __initconst = {
+	0xfb, 0xa7, 0x8a, 0xe4, 0xf9, 0xd8, 0x08, 0xa6,
+	0x2e, 0x3d, 0xa4, 0x0b, 0xe2, 0xcb, 0x77, 0x00,
+	0xc3, 0x61, 0x3d, 0x9e, 0xb2, 0xc5, 0x29, 0xc6,
+	0x52, 0xe7, 0x6a, 0x43, 0x2c, 0x65, 0x8d, 0x27,
+	0x09, 0x5f, 0x0e, 0xb8, 0xf9, 0x40, 0xc3, 0x24,
+	0x98, 0x1e, 0xa9, 0x35, 0xe5, 0x07, 0xf9, 0x8f,
+	0x04, 0x69, 0x56, 0xdb, 0x3a, 0x51, 0x29, 0x08,
+	0xbd, 0x7a, 0xfc, 0x8f, 0x2a, 0xb0, 0xa9
+};
+static const u8 enc_assoc038[] __initconst = { };
+static const u8 enc_nonce038[] __initconst = {
+	0xe7, 0x4a, 0x51, 0x5e, 0x7e, 0x21, 0x02, 0xb9,
+	0x0b, 0xef, 0x55, 0xd2
+};
+static const u8 enc_key038[] __initconst = {
+	0xcf, 0x0d, 0x40, 0xa4, 0x64, 0x4e, 0x5f, 0x51,
+	0x81, 0x51, 0x65, 0xd5, 0x30, 0x1b, 0x22, 0x63,
+	0x1f, 0x45, 0x44, 0xc4, 0x9a, 0x18, 0x78, 0xe3,
+	0xa0, 0xa5, 0xe8, 0xe1, 0xaa, 0xe0, 0xf2, 0x64
+};
+
+/* wycheproof - misc */
+static const u8 enc_input039[] __initconst = {
+	0xa9, 0x89, 0x95, 0x50, 0x4d, 0xf1, 0x6f, 0x74,
+	0x8b, 0xfb, 0x77, 0x85, 0xff, 0x91, 0xee, 0xb3,
+	0xb6, 0x60, 0xea, 0x9e, 0xd3, 0x45, 0x0c, 0x3d,
+	0x5e, 0x7b, 0x0e, 0x79, 0xef, 0x65, 0x36, 0x59,
+	0xa9, 0x97, 0x8d, 0x75, 0x54, 0x2e, 0xf9, 0x1c,
+	0x45, 0x67, 0x62, 0x21, 0x56, 0x40, 0xb9
+};
+static const u8 enc_output039[] __initconst = {
+	0xa1, 0xff, 0xed, 0x80, 0x76, 0x18, 0x29, 0xec,
+	0xce, 0x24, 0x2e, 0x0e, 0x88, 0xb1, 0x38, 0x04,
+	0x90, 0x16, 0xbc, 0xa0, 0x18, 0xda, 0x2b, 0x6e,
+	0x19, 0x98, 0x6b, 0x3e, 0x31, 0x8c, 0xae, 0x8d,
+	0x80, 0x61, 0x98, 0xfb, 0x4c, 0x52, 0x7c, 0xc3,
+	0x93, 0x50, 0xeb, 0xdd, 0xea, 0xc5, 0x73, 0xc4,
+	0xcb, 0xf0, 0xbe, 0xfd, 0xa0, 0xb7, 0x02, 0x42,
+	0xc6, 0x40, 0xd7, 0xcd, 0x02, 0xd7, 0xa3
+};
+static const u8 enc_assoc039[] __initconst = {
+	0xb3, 0xe4, 0x06, 0x46, 0x83, 0xb0, 0x2d, 0x84
+};
+static const u8 enc_nonce039[] __initconst = {
+	0xd4, 0xd8, 0x07, 0x34, 0x16, 0x83, 0x82, 0x5b,
+	0x31, 0xcd, 0x4d, 0x95
+};
+static const u8 enc_key039[] __initconst = {
+	0x6c, 0xbf, 0xd7, 0x1c, 0x64, 0x5d, 0x18, 0x4c,
+	0xf5, 0xd2, 0x3c, 0x40, 0x2b, 0xdb, 0x0d, 0x25,
+	0xec, 0x54, 0x89, 0x8c, 0x8a, 0x02, 0x73, 0xd4,
+	0x2e, 0xb5, 0xbe, 0x10, 0x9f, 0xdc, 0xb2, 0xac
+};
+
+/* wycheproof - misc */
+static const u8 enc_input040[] __initconst = {
+	0xd0, 0x96, 0x80, 0x31, 0x81, 0xbe, 0xef, 0x9e,
+	0x00, 0x8f, 0xf8, 0x5d, 0x5d, 0xdc, 0x38, 0xdd,
+	0xac, 0xf0, 0xf0, 0x9e, 0xe5, 0xf7, 0xe0, 0x7f,
+	0x1e, 0x40, 0x79, 0xcb, 0x64, 0xd0, 0xdc, 0x8f,
+	0x5e, 0x67, 0x11, 0xcd, 0x49, 0x21, 0xa7, 0x88,
+	0x7d, 0xe7, 0x6e, 0x26, 0x78, 0xfd, 0xc6, 0x76,
+	0x18, 0xf1, 0x18, 0x55, 0x86, 0xbf, 0xea, 0x9d,
+	0x4c, 0x68, 0x5d, 0x50, 0xe4, 0xbb, 0x9a, 0x82
+};
+static const u8 enc_output040[] __initconst = {
+	0x9a, 0x4e, 0xf2, 0x2b, 0x18, 0x16, 0x77, 0xb5,
+	0x75, 0x5c, 0x08, 0xf7, 0x47, 0xc0, 0xf8, 0xd8,
+	0xe8, 0xd4, 0xc1, 0x8a, 0x9c, 0xc2, 0x40, 0x5c,
+	0x12, 0xbb, 0x51, 0xbb, 0x18, 0x72, 0xc8, 0xe8,
+	0xb8, 0x77, 0x67, 0x8b, 0xec, 0x44, 0x2c, 0xfc,
+	0xbb, 0x0f, 0xf4, 0x64, 0xa6, 0x4b, 0x74, 0x33,
+	0x2c, 0xf0, 0x72, 0x89, 0x8c, 0x7e, 0x0e, 0xdd,
+	0xf6, 0x23, 0x2e, 0xa6, 0xe2, 0x7e, 0xfe, 0x50,
+	0x9f, 0xf3, 0x42, 0x7a, 0x0f, 0x32, 0xfa, 0x56,
+	0x6d, 0x9c, 0xa0, 0xa7, 0x8a, 0xef, 0xc0, 0x13
+};
+static const u8 enc_assoc040[] __initconst = { };
+static const u8 enc_nonce040[] __initconst = {
+	0xd6, 0x10, 0x40, 0xa3, 0x13, 0xed, 0x49, 0x28,
+	0x23, 0xcc, 0x06, 0x5b
+};
+static const u8 enc_key040[] __initconst = {
+	0x5b, 0x1d, 0x10, 0x35, 0xc0, 0xb1, 0x7e, 0xe0,
+	0xb0, 0x44, 0x47, 0x67, 0xf8, 0x0a, 0x25, 0xb8,
+	0xc1, 0xb7, 0x41, 0xf4, 0xb5, 0x0a, 0x4d, 0x30,
+	0x52, 0x22, 0x6b, 0xaa, 0x1c, 0x6f, 0xb7, 0x01
+};
+
+/* wycheproof - misc */
+static const u8 enc_input041[] __initconst = {
+	0x94, 0xee, 0x16, 0x6d, 0x6d, 0x6e, 0xcf, 0x88,
+	0x32, 0x43, 0x71, 0x36, 0xb4, 0xae, 0x80, 0x5d,
+	0x42, 0x88, 0x64, 0x35, 0x95, 0x86, 0xd9, 0x19,
+	0x3a, 0x25, 0x01, 0x62, 0x93, 0xed, 0xba, 0x44,
+	0x3c, 0x58, 0xe0, 0x7e, 0x7b, 0x71, 0x95, 0xec,
+	0x5b, 0xd8, 0x45, 0x82, 0xa9, 0xd5, 0x6c, 0x8d,
+	0x4a, 0x10, 0x8c, 0x7d, 0x7c, 0xe3, 0x4e, 0x6c,
+	0x6f, 0x8e, 0xa1, 0xbe, 0xc0, 0x56, 0x73, 0x17
+};
+static const u8 enc_output041[] __initconst = {
+	0x5f, 0xbb, 0xde, 0xcc, 0x34, 0xbe, 0x20, 0x16,
+	0x14, 0xf6, 0x36, 0x03, 0x1e, 0xeb, 0x42, 0xf1,
+	0xca, 0xce, 0x3c, 0x79, 0xa1, 0x2c, 0xff, 0xd8,
+	0x71, 0xee, 0x8e, 0x73, 0x82, 0x0c, 0x82, 0x97,
+	0x49, 0xf1, 0xab, 0xb4, 0x29, 0x43, 0x67, 0x84,
+	0x9f, 0xb6, 0xc2, 0xaa, 0x56, 0xbd, 0xa8, 0xa3,
+	0x07, 0x8f, 0x72, 0x3d, 0x7c, 0x1c, 0x85, 0x20,
+	0x24, 0xb0, 0x17, 0xb5, 0x89, 0x73, 0xfb, 0x1e,
+	0x09, 0x26, 0x3d, 0xa7, 0xb4, 0xcb, 0x92, 0x14,
+	0x52, 0xf9, 0x7d, 0xca, 0x40, 0xf5, 0x80, 0xec
+};
+static const u8 enc_assoc041[] __initconst = {
+	0x71, 0x93, 0xf6, 0x23, 0x66, 0x33, 0x21, 0xa2
+};
+static const u8 enc_nonce041[] __initconst = {
+	0xd3, 0x1c, 0x21, 0xab, 0xa1, 0x75, 0xb7, 0x0d,
+	0xe4, 0xeb, 0xb1, 0x9c
+};
+static const u8 enc_key041[] __initconst = {
+	0x97, 0xd6, 0x35, 0xc4, 0xf4, 0x75, 0x74, 0xd9,
+	0x99, 0x8a, 0x90, 0x87, 0x5d, 0xa1, 0xd3, 0xa2,
+	0x84, 0xb7, 0x55, 0xb2, 0xd3, 0x92, 0x97, 0xa5,
+	0x72, 0x52, 0x35, 0x19, 0x0e, 0x10, 0xa9, 0x7e
+};
+
+/* wycheproof - misc */
+static const u8 enc_input042[] __initconst = {
+	0xb4, 0x29, 0xeb, 0x80, 0xfb, 0x8f, 0xe8, 0xba,
+	0xed, 0xa0, 0xc8, 0x5b, 0x9c, 0x33, 0x34, 0x58,
+	0xe7, 0xc2, 0x99, 0x2e, 0x55, 0x84, 0x75, 0x06,
+	0x9d, 0x12, 0xd4, 0x5c, 0x22, 0x21, 0x75, 0x64,
+	0x12, 0x15, 0x88, 0x03, 0x22, 0x97, 0xef, 0xf5,
+	0x67, 0x83, 0x74, 0x2a, 0x5f, 0xc2, 0x2d, 0x74,
+	0x10, 0xff, 0xb2, 0x9d, 0x66, 0x09, 0x86, 0x61,
+	0xd7, 0x6f, 0x12, 0x6c, 0x3c, 0x27, 0x68, 0x9e,
+	0x43, 0xb3, 0x72, 0x67, 0xca, 0xc5, 0xa3, 0xa6,
+	0xd3, 0xab, 0x49, 0xe3, 0x91, 0xda, 0x29, 0xcd,
+	0x30, 0x54, 0xa5, 0x69, 0x2e, 0x28, 0x07, 0xe4,
+	0xc3, 0xea, 0x46, 0xc8, 0x76, 0x1d, 0x50, 0xf5,
+	0x92
+};
+static const u8 enc_output042[] __initconst = {
+	0xd0, 0x10, 0x2f, 0x6c, 0x25, 0x8b, 0xf4, 0x97,
+	0x42, 0xce, 0xc3, 0x4c, 0xf2, 0xd0, 0xfe, 0xdf,
+	0x23, 0xd1, 0x05, 0xfb, 0x4c, 0x84, 0xcf, 0x98,
+	0x51, 0x5e, 0x1b, 0xc9, 0xa6, 0x4f, 0x8a, 0xd5,
+	0xbe, 0x8f, 0x07, 0x21, 0xbd, 0xe5, 0x06, 0x45,
+	0xd0, 0x00, 0x83, 0xc3, 0xa2, 0x63, 0xa3, 0x10,
+	0x53, 0xb7, 0x60, 0x24, 0x5f, 0x52, 0xae, 0x28,
+	0x66, 0xa5, 0xec, 0x83, 0xb1, 0x9f, 0x61, 0xbe,
+	0x1d, 0x30, 0xd5, 0xc5, 0xd9, 0xfe, 0xcc, 0x4c,
+	0xbb, 0xe0, 0x8f, 0xd3, 0x85, 0x81, 0x3a, 0x2a,
+	0xa3, 0x9a, 0x00, 0xff, 0x9c, 0x10, 0xf7, 0xf2,
+	0x37, 0x02, 0xad, 0xd1, 0xe4, 0xb2, 0xff, 0xa3,
+	0x1c, 0x41, 0x86, 0x5f, 0xc7, 0x1d, 0xe1, 0x2b,
+	0x19, 0x61, 0x21, 0x27, 0xce, 0x49, 0x99, 0x3b,
+	0xb0
+};
+static const u8 enc_assoc042[] __initconst = { };
+static const u8 enc_nonce042[] __initconst = {
+	0x17, 0xc8, 0x6a, 0x8a, 0xbb, 0xb7, 0xe0, 0x03,
+	0xac, 0xde, 0x27, 0x99
+};
+static const u8 enc_key042[] __initconst = {
+	0xfe, 0x6e, 0x55, 0xbd, 0xae, 0xd1, 0xf7, 0x28,
+	0x4c, 0xa5, 0xfc, 0x0f, 0x8c, 0x5f, 0x2b, 0x8d,
+	0xf5, 0x6d, 0xc0, 0xf4, 0x9e, 0x8c, 0xa6, 0x6a,
+	0x41, 0x99, 0x5e, 0x78, 0x33, 0x51, 0xf9, 0x01
+};
+
+/* wycheproof - misc */
+static const u8 enc_input043[] __initconst = {
+	0xce, 0xb5, 0x34, 0xce, 0x50, 0xdc, 0x23, 0xff,
+	0x63, 0x8a, 0xce, 0x3e, 0xf6, 0x3a, 0xb2, 0xcc,
+	0x29, 0x73, 0xee, 0xad, 0xa8, 0x07, 0x85, 0xfc,
+	0x16, 0x5d, 0x06, 0xc2, 0xf5, 0x10, 0x0f, 0xf5,
+	0xe8, 0xab, 0x28, 0x82, 0xc4, 0x75, 0xaf, 0xcd,
+	0x05, 0xcc, 0xd4, 0x9f, 0x2e, 0x7d, 0x8f, 0x55,
+	0xef, 0x3a, 0x72, 0xe3, 0xdc, 0x51, 0xd6, 0x85,
+	0x2b, 0x8e, 0x6b, 0x9e, 0x7a, 0xec, 0xe5, 0x7b,
+	0xe6, 0x55, 0x6b, 0x0b, 0x6d, 0x94, 0x13, 0xe3,
+	0x3f, 0xc5, 0xfc, 0x24, 0xa9, 0xa2, 0x05, 0xad,
+	0x59, 0x57, 0x4b, 0xb3, 0x9d, 0x94, 0x4a, 0x92,
+	0xdc, 0x47, 0x97, 0x0d, 0x84, 0xa6, 0xad, 0x31,
+	0x76
+};
+static const u8 enc_output043[] __initconst = {
+	0x75, 0x45, 0x39, 0x1b, 0x51, 0xde, 0x01, 0xd5,
+	0xc5, 0x3d, 0xfa, 0xca, 0x77, 0x79, 0x09, 0x06,
+	0x3e, 0x58, 0xed, 0xee, 0x4b, 0xb1, 0x22, 0x7e,
+	0x71, 0x10, 0xac, 0x4d, 0x26, 0x20, 0xc2, 0xae,
+	0xc2, 0xf8, 0x48, 0xf5, 0x6d, 0xee, 0xb0, 0x37,
+	0xa8, 0xdc, 0xed, 0x75, 0xaf, 0xa8, 0xa6, 0xc8,
+	0x90, 0xe2, 0xde, 0xe4, 0x2f, 0x95, 0x0b, 0xb3,
+	0x3d, 0x9e, 0x24, 0x24, 0xd0, 0x8a, 0x50, 0x5d,
+	0x89, 0x95, 0x63, 0x97, 0x3e, 0xd3, 0x88, 0x70,
+	0xf3, 0xde, 0x6e, 0xe2, 0xad, 0xc7, 0xfe, 0x07,
+	0x2c, 0x36, 0x6c, 0x14, 0xe2, 0xcf, 0x7c, 0xa6,
+	0x2f, 0xb3, 0xd3, 0x6b, 0xee, 0x11, 0x68, 0x54,
+	0x61, 0xb7, 0x0d, 0x44, 0xef, 0x8c, 0x66, 0xc5,
+	0xc7, 0xbb, 0xf1, 0x0d, 0xca, 0xdd, 0x7f, 0xac,
+	0xf6
+};
+static const u8 enc_assoc043[] __initconst = {
+	0xa1, 0x1c, 0x40, 0xb6, 0x03, 0x76, 0x73, 0x30
+};
+static const u8 enc_nonce043[] __initconst = {
+	0x46, 0x36, 0x2f, 0x45, 0xd6, 0x37, 0x9e, 0x63,
+	0xe5, 0x22, 0x94, 0x60
+};
+static const u8 enc_key043[] __initconst = {
+	0xaa, 0xbc, 0x06, 0x34, 0x74, 0xe6, 0x5c, 0x4c,
+	0x3e, 0x9b, 0xdc, 0x48, 0x0d, 0xea, 0x97, 0xb4,
+	0x51, 0x10, 0xc8, 0x61, 0x88, 0x46, 0xff, 0x6b,
+	0x15, 0xbd, 0xd2, 0xa4, 0xa5, 0x68, 0x2c, 0x4e
+};
+
+/* wycheproof - misc */
+static const u8 enc_input044[] __initconst = {
+	0xe5, 0xcc, 0xaa, 0x44, 0x1b, 0xc8, 0x14, 0x68,
+	0x8f, 0x8f, 0x6e, 0x8f, 0x28, 0xb5, 0x00, 0xb2
+};
+static const u8 enc_output044[] __initconst = {
+	0x7e, 0x72, 0xf5, 0xa1, 0x85, 0xaf, 0x16, 0xa6,
+	0x11, 0x92, 0x1b, 0x43, 0x8f, 0x74, 0x9f, 0x0b,
+	0x12, 0x42, 0xc6, 0x70, 0x73, 0x23, 0x34, 0x02,
+	0x9a, 0xdf, 0xe1, 0xc5, 0x00, 0x16, 0x51, 0xe4
+};
+static const u8 enc_assoc044[] __initconst = {
+	0x02
+};
+static const u8 enc_nonce044[] __initconst = {
+	0x87, 0x34, 0x5f, 0x10, 0x55, 0xfd, 0x9e, 0x21,
+	0x02, 0xd5, 0x06, 0x56
+};
+static const u8 enc_key044[] __initconst = {
+	0x7d, 0x00, 0xb4, 0x80, 0x95, 0xad, 0xfa, 0x32,
+	0x72, 0x05, 0x06, 0x07, 0xb2, 0x64, 0x18, 0x50,
+	0x02, 0xba, 0x99, 0x95, 0x7c, 0x49, 0x8b, 0xe0,
+	0x22, 0x77, 0x0f, 0x2c, 0xe2, 0xf3, 0x14, 0x3c
+};
+
+/* wycheproof - misc */
+static const u8 enc_input045[] __initconst = {
+	0x02, 0xcd, 0xe1, 0x68, 0xfb, 0xa3, 0xf5, 0x44,
+	0xbb, 0xd0, 0x33, 0x2f, 0x7a, 0xde, 0xad, 0xa8
+};
+static const u8 enc_output045[] __initconst = {
+	0x85, 0xf2, 0x9a, 0x71, 0x95, 0x57, 0xcd, 0xd1,
+	0x4d, 0x1f, 0x8f, 0xff, 0xab, 0x6d, 0x9e, 0x60,
+	0x73, 0x2c, 0xa3, 0x2b, 0xec, 0xd5, 0x15, 0xa1,
+	0xed, 0x35, 0x3f, 0x54, 0x2e, 0x99, 0x98, 0x58
+};
+static const u8 enc_assoc045[] __initconst = {
+	0xb6, 0x48
+};
+static const u8 enc_nonce045[] __initconst = {
+	0x87, 0xa3, 0x16, 0x3e, 0xc0, 0x59, 0x8a, 0xd9,
+	0x5b, 0x3a, 0xa7, 0x13
+};
+static const u8 enc_key045[] __initconst = {
+	0x64, 0x32, 0x71, 0x7f, 0x1d, 0xb8, 0x5e, 0x41,
+	0xac, 0x78, 0x36, 0xbc, 0xe2, 0x51, 0x85, 0xa0,
+	0x80, 0xd5, 0x76, 0x2b, 0x9e, 0x2b, 0x18, 0x44,
+	0x4b, 0x6e, 0xc7, 0x2c, 0x3b, 0xd8, 0xe4, 0xdc
+};
+
+/* wycheproof - misc */
+static const u8 enc_input046[] __initconst = {
+	0x16, 0xdd, 0xd2, 0x3f, 0xf5, 0x3f, 0x3d, 0x23,
+	0xc0, 0x63, 0x34, 0x48, 0x70, 0x40, 0xeb, 0x47
+};
+static const u8 enc_output046[] __initconst = {
+	0xc1, 0xb2, 0x95, 0x93, 0x6d, 0x56, 0xfa, 0xda,
+	0xc0, 0x3e, 0x5f, 0x74, 0x2b, 0xff, 0x73, 0xa1,
+	0x39, 0xc4, 0x57, 0xdb, 0xab, 0x66, 0x38, 0x2b,
+	0xab, 0xb3, 0xb5, 0x58, 0x00, 0xcd, 0xa5, 0xb8
+};
+static const u8 enc_assoc046[] __initconst = {
+	0xbd, 0x4c, 0xd0, 0x2f, 0xc7, 0x50, 0x2b, 0xbd,
+	0xbd, 0xf6, 0xc9, 0xa3, 0xcb, 0xe8, 0xf0
+};
+static const u8 enc_nonce046[] __initconst = {
+	0x6f, 0x57, 0x3a, 0xa8, 0x6b, 0xaa, 0x49, 0x2b,
+	0xa4, 0x65, 0x96, 0xdf
+};
+static const u8 enc_key046[] __initconst = {
+	0x8e, 0x34, 0xcf, 0x73, 0xd2, 0x45, 0xa1, 0x08,
+	0x2a, 0x92, 0x0b, 0x86, 0x36, 0x4e, 0xb8, 0x96,
+	0xc4, 0x94, 0x64, 0x67, 0xbc, 0xb3, 0xd5, 0x89,
+	0x29, 0xfc, 0xb3, 0x66, 0x90, 0xe6, 0x39, 0x4f
+};
+
+/* wycheproof - misc */
+static const u8 enc_input047[] __initconst = {
+	0x62, 0x3b, 0x78, 0x50, 0xc3, 0x21, 0xe2, 0xcf,
+	0x0c, 0x6f, 0xbc, 0xc8, 0xdf, 0xd1, 0xaf, 0xf2
+};
+static const u8 enc_output047[] __initconst = {
+	0xc8, 0x4c, 0x9b, 0xb7, 0xc6, 0x1c, 0x1b, 0xcb,
+	0x17, 0x77, 0x2a, 0x1c, 0x50, 0x0c, 0x50, 0x95,
+	0xdb, 0xad, 0xf7, 0xa5, 0x13, 0x8c, 0xa0, 0x34,
+	0x59, 0xa2, 0xcd, 0x65, 0x83, 0x1e, 0x09, 0x2f
+};
+static const u8 enc_assoc047[] __initconst = {
+	0x89, 0xcc, 0xe9, 0xfb, 0x47, 0x44, 0x1d, 0x07,
+	0xe0, 0x24, 0x5a, 0x66, 0xfe, 0x8b, 0x77, 0x8b
+};
+static const u8 enc_nonce047[] __initconst = {
+	0x1a, 0x65, 0x18, 0xf0, 0x2e, 0xde, 0x1d, 0xa6,
+	0x80, 0x92, 0x66, 0xd9
+};
+static const u8 enc_key047[] __initconst = {
+	0xcb, 0x55, 0x75, 0xf5, 0xc7, 0xc4, 0x5c, 0x91,
+	0xcf, 0x32, 0x0b, 0x13, 0x9f, 0xb5, 0x94, 0x23,
+	0x75, 0x60, 0xd0, 0xa3, 0xe6, 0xf8, 0x65, 0xa6,
+	0x7d, 0x4f, 0x63, 0x3f, 0x2c, 0x08, 0xf0, 0x16
+};
+
+/* wycheproof - misc */
+static const u8 enc_input048[] __initconst = {
+	0x87, 0xb3, 0xa4, 0xd7, 0xb2, 0x6d, 0x8d, 0x32,
+	0x03, 0xa0, 0xde, 0x1d, 0x64, 0xef, 0x82, 0xe3
+};
+static const u8 enc_output048[] __initconst = {
+	0x94, 0xbc, 0x80, 0x62, 0x1e, 0xd1, 0xe7, 0x1b,
+	0x1f, 0xd2, 0xb5, 0xc3, 0xa1, 0x5e, 0x35, 0x68,
+	0x33, 0x35, 0x11, 0x86, 0x17, 0x96, 0x97, 0x84,
+	0x01, 0x59, 0x8b, 0x96, 0x37, 0x22, 0xf5, 0xb3
+};
+static const u8 enc_assoc048[] __initconst = {
+	0xd1, 0x9f, 0x2d, 0x98, 0x90, 0x95, 0xf7, 0xab,
+	0x03, 0xa5, 0xfd, 0xe8, 0x44, 0x16, 0xe0, 0x0c,
+	0x0e
+};
+static const u8 enc_nonce048[] __initconst = {
+	0x56, 0x4d, 0xee, 0x49, 0xab, 0x00, 0xd2, 0x40,
+	0xfc, 0x10, 0x68, 0xc3
+};
+static const u8 enc_key048[] __initconst = {
+	0xa5, 0x56, 0x9e, 0x72, 0x9a, 0x69, 0xb2, 0x4b,
+	0xa6, 0xe0, 0xff, 0x15, 0xc4, 0x62, 0x78, 0x97,
+	0x43, 0x68, 0x24, 0xc9, 0x41, 0xe9, 0xd0, 0x0b,
+	0x2e, 0x93, 0xfd, 0xdc, 0x4b, 0xa7, 0x76, 0x57
+};
+
+/* wycheproof - misc */
+static const u8 enc_input049[] __initconst = {
+	0xe6, 0x01, 0xb3, 0x85, 0x57, 0x79, 0x7d, 0xa2,
+	0xf8, 0xa4, 0x10, 0x6a, 0x08, 0x9d, 0x1d, 0xa6
+};
+static const u8 enc_output049[] __initconst = {
+	0x29, 0x9b, 0x5d, 0x3f, 0x3d, 0x03, 0xc0, 0x87,
+	0x20, 0x9a, 0x16, 0xe2, 0x85, 0x14, 0x31, 0x11,
+	0x4b, 0x45, 0x4e, 0xd1, 0x98, 0xde, 0x11, 0x7e,
+	0x83, 0xec, 0x49, 0xfa, 0x8d, 0x85, 0x08, 0xd6
+};
+static const u8 enc_assoc049[] __initconst = {
+	0x5e, 0x64, 0x70, 0xfa, 0xcd, 0x99, 0xc1, 0xd8,
+	0x1e, 0x37, 0xcd, 0x44, 0x01, 0x5f, 0xe1, 0x94,
+	0x80, 0xa2, 0xa4, 0xd3, 0x35, 0x2a, 0x4f, 0xf5,
+	0x60, 0xc0, 0x64, 0x0f, 0xdb, 0xda
+};
+static const u8 enc_nonce049[] __initconst = {
+	0xdf, 0x87, 0x13, 0xe8, 0x7e, 0xc3, 0xdb, 0xcf,
+	0xad, 0x14, 0xd5, 0x3e
+};
+static const u8 enc_key049[] __initconst = {
+	0x56, 0x20, 0x74, 0x65, 0xb4, 0xe4, 0x8e, 0x6d,
+	0x04, 0x63, 0x0f, 0x4a, 0x42, 0xf3, 0x5c, 0xfc,
+	0x16, 0x3a, 0xb2, 0x89, 0xc2, 0x2a, 0x2b, 0x47,
+	0x84, 0xf6, 0xf9, 0x29, 0x03, 0x30, 0xbe, 0xe0
+};
+
+/* wycheproof - misc */
+static const u8 enc_input050[] __initconst = {
+	0xdc, 0x9e, 0x9e, 0xaf, 0x11, 0xe3, 0x14, 0x18,
+	0x2d, 0xf6, 0xa4, 0xeb, 0xa1, 0x7a, 0xec, 0x9c
+};
+static const u8 enc_output050[] __initconst = {
+	0x60, 0x5b, 0xbf, 0x90, 0xae, 0xb9, 0x74, 0xf6,
+	0x60, 0x2b, 0xc7, 0x78, 0x05, 0x6f, 0x0d, 0xca,
+	0x38, 0xea, 0x23, 0xd9, 0x90, 0x54, 0xb4, 0x6b,
+	0x42, 0xff, 0xe0, 0x04, 0x12, 0x9d, 0x22, 0x04
+};
+static const u8 enc_assoc050[] __initconst = {
+	0xba, 0x44, 0x6f, 0x6f, 0x9a, 0x0c, 0xed, 0x22,
+	0x45, 0x0f, 0xeb, 0x10, 0x73, 0x7d, 0x90, 0x07,
+	0xfd, 0x69, 0xab, 0xc1, 0x9b, 0x1d, 0x4d, 0x90,
+	0x49, 0xa5, 0x55, 0x1e, 0x86, 0xec, 0x2b, 0x37
+};
+static const u8 enc_nonce050[] __initconst = {
+	0x8d, 0xf4, 0xb1, 0x5a, 0x88, 0x8c, 0x33, 0x28,
+	0x6a, 0x7b, 0x76, 0x51
+};
+static const u8 enc_key050[] __initconst = {
+	0x39, 0x37, 0x98, 0x6a, 0xf8, 0x6d, 0xaf, 0xc1,
+	0xba, 0x0c, 0x46, 0x72, 0xd8, 0xab, 0xc4, 0x6c,
+	0x20, 0x70, 0x62, 0x68, 0x2d, 0x9c, 0x26, 0x4a,
+	0xb0, 0x6d, 0x6c, 0x58, 0x07, 0x20, 0x51, 0x30
+};
+
+/* wycheproof - misc */
+static const u8 enc_input051[] __initconst = {
+	0x81, 0xce, 0x84, 0xed, 0xe9, 0xb3, 0x58, 0x59,
+	0xcc, 0x8c, 0x49, 0xa8, 0xf6, 0xbe, 0x7d, 0xc6
+};
+static const u8 enc_output051[] __initconst = {
+	0x7b, 0x7c, 0xe0, 0xd8, 0x24, 0x80, 0x9a, 0x70,
+	0xde, 0x32, 0x56, 0x2c, 0xcf, 0x2c, 0x2b, 0xbd,
+	0x15, 0xd4, 0x4a, 0x00, 0xce, 0x0d, 0x19, 0xb4,
+	0x23, 0x1f, 0x92, 0x1e, 0x22, 0xbc, 0x0a, 0x43
+};
+static const u8 enc_assoc051[] __initconst = {
+	0xd4, 0x1a, 0x82, 0x8d, 0x5e, 0x71, 0x82, 0x92,
+	0x47, 0x02, 0x19, 0x05, 0x40, 0x2e, 0xa2, 0x57,
+	0xdc, 0xcb, 0xc3, 0xb8, 0x0f, 0xcd, 0x56, 0x75,
+	0x05, 0x6b, 0x68, 0xbb, 0x59, 0xe6, 0x2e, 0x88,
+	0x73
+};
+static const u8 enc_nonce051[] __initconst = {
+	0xbe, 0x40, 0xe5, 0xf1, 0xa1, 0x18, 0x17, 0xa0,
+	0xa8, 0xfa, 0x89, 0x49
+};
+static const u8 enc_key051[] __initconst = {
+	0x36, 0x37, 0x2a, 0xbc, 0xdb, 0x78, 0xe0, 0x27,
+	0x96, 0x46, 0xac, 0x3d, 0x17, 0x6b, 0x96, 0x74,
+	0xe9, 0x15, 0x4e, 0xec, 0xf0, 0xd5, 0x46, 0x9c,
+	0x65, 0x1e, 0xc7, 0xe1, 0x6b, 0x4c, 0x11, 0x99
+};
+
+/* wycheproof - misc */
+static const u8 enc_input052[] __initconst = {
+	0xa6, 0x67, 0x47, 0xc8, 0x9e, 0x85, 0x7a, 0xf3,
+	0xa1, 0x8e, 0x2c, 0x79, 0x50, 0x00, 0x87, 0xed
+};
+static const u8 enc_output052[] __initconst = {
+	0xca, 0x82, 0xbf, 0xf3, 0xe2, 0xf3, 0x10, 0xcc,
+	0xc9, 0x76, 0x67, 0x2c, 0x44, 0x15, 0xe6, 0x9b,
+	0x57, 0x63, 0x8c, 0x62, 0xa5, 0xd8, 0x5d, 0xed,
+	0x77, 0x4f, 0x91, 0x3c, 0x81, 0x3e, 0xa0, 0x32
+};
+static const u8 enc_assoc052[] __initconst = {
+	0x3f, 0x2d, 0xd4, 0x9b, 0xbf, 0x09, 0xd6, 0x9a,
+	0x78, 0xa3, 0xd8, 0x0e, 0xa2, 0x56, 0x66, 0x14,
+	0xfc, 0x37, 0x94, 0x74, 0x19, 0x6c, 0x1a, 0xae,
+	0x84, 0x58, 0x3d, 0xa7, 0x3d, 0x7f, 0xf8, 0x5c,
+	0x6f, 0x42, 0xca, 0x42, 0x05, 0x6a, 0x97, 0x92,
+	0xcc, 0x1b, 0x9f, 0xb3, 0xc7, 0xd2, 0x61
+};
+static const u8 enc_nonce052[] __initconst = {
+	0x84, 0xc8, 0x7d, 0xae, 0x4e, 0xee, 0x27, 0x73,
+	0x0e, 0xc3, 0x5d, 0x12
+};
+static const u8 enc_key052[] __initconst = {
+	0x9f, 0x14, 0x79, 0xed, 0x09, 0x7d, 0x7f, 0xe5,
+	0x29, 0xc1, 0x1f, 0x2f, 0x5a, 0xdd, 0x9a, 0xaf,
+	0xf4, 0xa1, 0xca, 0x0b, 0x68, 0x99, 0x7a, 0x2c,
+	0xb7, 0xf7, 0x97, 0x49, 0xbd, 0x90, 0xaa, 0xf4
+};
+
+/* wycheproof - misc */
+static const u8 enc_input053[] __initconst = {
+	0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83,
+	0x55, 0xd3, 0x04, 0x84, 0x64, 0x43, 0xfe, 0xe8,
+	0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, 0x3b, 0x7b,
+	0x80, 0xe0, 0x30, 0xbe, 0xeb, 0xd3, 0x29, 0xbe
+};
+static const u8 enc_output053[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0xe6, 0xd3, 0xd7, 0x32, 0x4a, 0x1c, 0xbb, 0xa7,
+	0x77, 0xbb, 0xb0, 0xec, 0xdd, 0xa3, 0x78, 0x07
+};
+static const u8 enc_assoc053[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+static const u8 enc_nonce053[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
+};
+static const u8 enc_key053[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - misc */
+static const u8 enc_input054[] __initconst = {
+	0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83,
+	0x55, 0xd3, 0x04, 0x84, 0x64, 0x43, 0xfe, 0xe8,
+	0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, 0x3b, 0x7b,
+	0x80, 0xe0, 0x30, 0xbe, 0xeb, 0xd3, 0x29, 0xbe,
+	0xe3, 0xbc, 0xdb, 0x5b, 0x1e, 0xde, 0xfc, 0xfe,
+	0x8b, 0xcd, 0xa1, 0xb6, 0xa1, 0x5c, 0x8c, 0x2b,
+	0x08, 0x69, 0xff, 0xd2, 0xec, 0x5e, 0x26, 0xe5,
+	0x53, 0xb7, 0xb2, 0x27, 0xfe, 0x87, 0xfd, 0xbd
+};
+static const u8 enc_output054[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x06, 0x2d, 0xe6, 0x79, 0x5f, 0x27, 0x4f, 0xd2,
+	0xa3, 0x05, 0xd7, 0x69, 0x80, 0xbc, 0x9c, 0xce
+};
+static const u8 enc_assoc054[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+static const u8 enc_nonce054[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
+};
+static const u8 enc_key054[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - misc */
+static const u8 enc_input055[] __initconst = {
+	0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83,
+	0x55, 0xd3, 0x04, 0x84, 0x64, 0x43, 0xfe, 0xe8,
+	0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, 0x3b, 0x7b,
+	0x80, 0xe0, 0x30, 0xbe, 0xeb, 0xd3, 0x29, 0xbe,
+	0xe3, 0xbc, 0xdb, 0x5b, 0x1e, 0xde, 0xfc, 0xfe,
+	0x8b, 0xcd, 0xa1, 0xb6, 0xa1, 0x5c, 0x8c, 0x2b,
+	0x08, 0x69, 0xff, 0xd2, 0xec, 0x5e, 0x26, 0xe5,
+	0x53, 0xb7, 0xb2, 0x27, 0xfe, 0x87, 0xfd, 0xbd,
+	0x7a, 0xda, 0x44, 0x42, 0x42, 0x69, 0xbf, 0xfa,
+	0x55, 0x27, 0xf2, 0x70, 0xac, 0xf6, 0x85, 0x02,
+	0xb7, 0x4c, 0x5a, 0xe2, 0xe6, 0x0c, 0x05, 0x80,
+	0x98, 0x1a, 0x49, 0x38, 0x45, 0x93, 0x92, 0xc4,
+	0x9b, 0xb2, 0xf2, 0x84, 0xb6, 0x46, 0xef, 0xc7,
+	0xf3, 0xf0, 0xb1, 0x36, 0x1d, 0xc3, 0x48, 0xed,
+	0x77, 0xd3, 0x0b, 0xc5, 0x76, 0x92, 0xed, 0x38,
+	0xfb, 0xac, 0x01, 0x88, 0x38, 0x04, 0x88, 0xc7
+};
+static const u8 enc_output055[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0xd8, 0xb4, 0x79, 0x02, 0xba, 0xae, 0xaf, 0xb3,
+	0x42, 0x03, 0x05, 0x15, 0x29, 0xaf, 0x28, 0x2e
+};
+static const u8 enc_assoc055[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+static const u8 enc_nonce055[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
+};
+static const u8 enc_key055[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - misc */
+static const u8 enc_input056[] __initconst = {
+	0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c,
+	0xaa, 0x2c, 0xfb, 0x7b, 0x9b, 0xbc, 0x01, 0x17,
+	0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, 0xc4, 0x84,
+	0x7f, 0x1f, 0xcf, 0x41, 0x14, 0x2c, 0xd6, 0x41
+};
+static const u8 enc_output056[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xb3, 0x89, 0x1c, 0x84, 0x9c, 0xb5, 0x2c, 0x27,
+	0x74, 0x7e, 0xdf, 0xcf, 0x31, 0x21, 0x3b, 0xb6
+};
+static const u8 enc_assoc056[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce056[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
+};
+static const u8 enc_key056[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - misc */
+static const u8 enc_input057[] __initconst = {
+	0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c,
+	0xaa, 0x2c, 0xfb, 0x7b, 0x9b, 0xbc, 0x01, 0x17,
+	0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, 0xc4, 0x84,
+	0x7f, 0x1f, 0xcf, 0x41, 0x14, 0x2c, 0xd6, 0x41,
+	0x1c, 0x43, 0x24, 0xa4, 0xe1, 0x21, 0x03, 0x01,
+	0x74, 0x32, 0x5e, 0x49, 0x5e, 0xa3, 0x73, 0xd4,
+	0xf7, 0x96, 0x00, 0x2d, 0x13, 0xa1, 0xd9, 0x1a,
+	0xac, 0x48, 0x4d, 0xd8, 0x01, 0x78, 0x02, 0x42
+};
+static const u8 enc_output057[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xf0, 0xc1, 0x2d, 0x26, 0xef, 0x03, 0x02, 0x9b,
+	0x62, 0xc0, 0x08, 0xda, 0x27, 0xc5, 0xdc, 0x68
+};
+static const u8 enc_assoc057[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce057[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
+};
+static const u8 enc_key057[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - misc */
+static const u8 enc_input058[] __initconst = {
+	0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c,
+	0xaa, 0x2c, 0xfb, 0x7b, 0x9b, 0xbc, 0x01, 0x17,
+	0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, 0xc4, 0x84,
+	0x7f, 0x1f, 0xcf, 0x41, 0x14, 0x2c, 0xd6, 0x41,
+	0x1c, 0x43, 0x24, 0xa4, 0xe1, 0x21, 0x03, 0x01,
+	0x74, 0x32, 0x5e, 0x49, 0x5e, 0xa3, 0x73, 0xd4,
+	0xf7, 0x96, 0x00, 0x2d, 0x13, 0xa1, 0xd9, 0x1a,
+	0xac, 0x48, 0x4d, 0xd8, 0x01, 0x78, 0x02, 0x42,
+	0x85, 0x25, 0xbb, 0xbd, 0xbd, 0x96, 0x40, 0x05,
+	0xaa, 0xd8, 0x0d, 0x8f, 0x53, 0x09, 0x7a, 0xfd,
+	0x48, 0xb3, 0xa5, 0x1d, 0x19, 0xf3, 0xfa, 0x7f,
+	0x67, 0xe5, 0xb6, 0xc7, 0xba, 0x6c, 0x6d, 0x3b,
+	0x64, 0x4d, 0x0d, 0x7b, 0x49, 0xb9, 0x10, 0x38,
+	0x0c, 0x0f, 0x4e, 0xc9, 0xe2, 0x3c, 0xb7, 0x12,
+	0x88, 0x2c, 0xf4, 0x3a, 0x89, 0x6d, 0x12, 0xc7,
+	0x04, 0x53, 0xfe, 0x77, 0xc7, 0xfb, 0x77, 0x38
+};
+static const u8 enc_output058[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xee, 0x65, 0x78, 0x30, 0x01, 0xc2, 0x56, 0x91,
+	0xfa, 0x28, 0xd0, 0xf5, 0xf1, 0xc1, 0xd7, 0x62
+};
+static const u8 enc_assoc058[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce058[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
+};
+static const u8 enc_key058[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - misc */
+static const u8 enc_input059[] __initconst = {
+	0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03,
+	0x55, 0xd3, 0x04, 0x04, 0x64, 0x43, 0xfe, 0x68,
+	0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, 0x3b, 0xfb,
+	0x80, 0xe0, 0x30, 0x3e, 0xeb, 0xd3, 0x29, 0x3e
+};
+static const u8 enc_output059[] __initconst = {
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x79, 0xba, 0x7a, 0x29, 0xf5, 0xa7, 0xbb, 0x75,
+	0x79, 0x7a, 0xf8, 0x7a, 0x61, 0x01, 0x29, 0xa4
+};
+static const u8 enc_assoc059[] __initconst = {
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80
+};
+static const u8 enc_nonce059[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
+};
+static const u8 enc_key059[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - misc */
+static const u8 enc_input060[] __initconst = {
+	0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03,
+	0x55, 0xd3, 0x04, 0x04, 0x64, 0x43, 0xfe, 0x68,
+	0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, 0x3b, 0xfb,
+	0x80, 0xe0, 0x30, 0x3e, 0xeb, 0xd3, 0x29, 0x3e,
+	0xe3, 0xbc, 0xdb, 0xdb, 0x1e, 0xde, 0xfc, 0x7e,
+	0x8b, 0xcd, 0xa1, 0x36, 0xa1, 0x5c, 0x8c, 0xab,
+	0x08, 0x69, 0xff, 0x52, 0xec, 0x5e, 0x26, 0x65,
+	0x53, 0xb7, 0xb2, 0xa7, 0xfe, 0x87, 0xfd, 0x3d
+};
+static const u8 enc_output060[] __initconst = {
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x36, 0xb1, 0x74, 0x38, 0x19, 0xe1, 0xb9, 0xba,
+	0x15, 0x51, 0xe8, 0xed, 0x92, 0x2a, 0x95, 0x9a
+};
+static const u8 enc_assoc060[] __initconst = {
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80
+};
+static const u8 enc_nonce060[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
+};
+static const u8 enc_key060[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - misc */
+static const u8 enc_input061[] __initconst = {
+	0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03,
+	0x55, 0xd3, 0x04, 0x04, 0x64, 0x43, 0xfe, 0x68,
+	0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, 0x3b, 0xfb,
+	0x80, 0xe0, 0x30, 0x3e, 0xeb, 0xd3, 0x29, 0x3e,
+	0xe3, 0xbc, 0xdb, 0xdb, 0x1e, 0xde, 0xfc, 0x7e,
+	0x8b, 0xcd, 0xa1, 0x36, 0xa1, 0x5c, 0x8c, 0xab,
+	0x08, 0x69, 0xff, 0x52, 0xec, 0x5e, 0x26, 0x65,
+	0x53, 0xb7, 0xb2, 0xa7, 0xfe, 0x87, 0xfd, 0x3d,
+	0x7a, 0xda, 0x44, 0xc2, 0x42, 0x69, 0xbf, 0x7a,
+	0x55, 0x27, 0xf2, 0xf0, 0xac, 0xf6, 0x85, 0x82,
+	0xb7, 0x4c, 0x5a, 0x62, 0xe6, 0x0c, 0x05, 0x00,
+	0x98, 0x1a, 0x49, 0xb8, 0x45, 0x93, 0x92, 0x44,
+	0x9b, 0xb2, 0xf2, 0x04, 0xb6, 0x46, 0xef, 0x47,
+	0xf3, 0xf0, 0xb1, 0xb6, 0x1d, 0xc3, 0x48, 0x6d,
+	0x77, 0xd3, 0x0b, 0x45, 0x76, 0x92, 0xed, 0xb8,
+	0xfb, 0xac, 0x01, 0x08, 0x38, 0x04, 0x88, 0x47
+};
+static const u8 enc_output061[] __initconst = {
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0xfe, 0xac, 0x49, 0x55, 0x55, 0x4e, 0x80, 0x6f,
+	0x3a, 0x19, 0x02, 0xe2, 0x44, 0x32, 0xc0, 0x8a
+};
+static const u8 enc_assoc061[] __initconst = {
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80
+};
+static const u8 enc_nonce061[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
+};
+static const u8 enc_key061[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - misc */
+static const u8 enc_input062[] __initconst = {
+	0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc,
+	0xaa, 0x2c, 0xfb, 0xfb, 0x9b, 0xbc, 0x01, 0x97,
+	0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, 0xc4, 0x04,
+	0x7f, 0x1f, 0xcf, 0xc1, 0x14, 0x2c, 0xd6, 0xc1
+};
+static const u8 enc_output062[] __initconst = {
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0x20, 0xa3, 0x79, 0x8d, 0xf1, 0x29, 0x2c, 0x59,
+	0x72, 0xbf, 0x97, 0x41, 0xae, 0xc3, 0x8a, 0x19
+};
+static const u8 enc_assoc062[] __initconst = {
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f
+};
+static const u8 enc_nonce062[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
+};
+static const u8 enc_key062[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - misc */
+static const u8 enc_input063[] __initconst = {
+	0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc,
+	0xaa, 0x2c, 0xfb, 0xfb, 0x9b, 0xbc, 0x01, 0x97,
+	0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, 0xc4, 0x04,
+	0x7f, 0x1f, 0xcf, 0xc1, 0x14, 0x2c, 0xd6, 0xc1,
+	0x1c, 0x43, 0x24, 0x24, 0xe1, 0x21, 0x03, 0x81,
+	0x74, 0x32, 0x5e, 0xc9, 0x5e, 0xa3, 0x73, 0x54,
+	0xf7, 0x96, 0x00, 0xad, 0x13, 0xa1, 0xd9, 0x9a,
+	0xac, 0x48, 0x4d, 0x58, 0x01, 0x78, 0x02, 0xc2
+};
+static const u8 enc_output063[] __initconst = {
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xc0, 0x3d, 0x9f, 0x67, 0x35, 0x4a, 0x97, 0xb2,
+	0xf0, 0x74, 0xf7, 0x55, 0x15, 0x57, 0xe4, 0x9c
+};
+static const u8 enc_assoc063[] __initconst = {
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f
+};
+static const u8 enc_nonce063[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
+};
+static const u8 enc_key063[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - misc */
+static const u8 enc_input064[] __initconst = {
+	0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc,
+	0xaa, 0x2c, 0xfb, 0xfb, 0x9b, 0xbc, 0x01, 0x97,
+	0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, 0xc4, 0x04,
+	0x7f, 0x1f, 0xcf, 0xc1, 0x14, 0x2c, 0xd6, 0xc1,
+	0x1c, 0x43, 0x24, 0x24, 0xe1, 0x21, 0x03, 0x81,
+	0x74, 0x32, 0x5e, 0xc9, 0x5e, 0xa3, 0x73, 0x54,
+	0xf7, 0x96, 0x00, 0xad, 0x13, 0xa1, 0xd9, 0x9a,
+	0xac, 0x48, 0x4d, 0x58, 0x01, 0x78, 0x02, 0xc2,
+	0x85, 0x25, 0xbb, 0x3d, 0xbd, 0x96, 0x40, 0x85,
+	0xaa, 0xd8, 0x0d, 0x0f, 0x53, 0x09, 0x7a, 0x7d,
+	0x48, 0xb3, 0xa5, 0x9d, 0x19, 0xf3, 0xfa, 0xff,
+	0x67, 0xe5, 0xb6, 0x47, 0xba, 0x6c, 0x6d, 0xbb,
+	0x64, 0x4d, 0x0d, 0xfb, 0x49, 0xb9, 0x10, 0xb8,
+	0x0c, 0x0f, 0x4e, 0x49, 0xe2, 0x3c, 0xb7, 0x92,
+	0x88, 0x2c, 0xf4, 0xba, 0x89, 0x6d, 0x12, 0x47,
+	0x04, 0x53, 0xfe, 0xf7, 0xc7, 0xfb, 0x77, 0xb8
+};
+static const u8 enc_output064[] __initconst = {
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xc8, 0x6d, 0xa8, 0xdd, 0x65, 0x22, 0x86, 0xd5,
+	0x02, 0x13, 0xd3, 0x28, 0xd6, 0x3e, 0x40, 0x06
+};
+static const u8 enc_assoc064[] __initconst = {
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f
+};
+static const u8 enc_nonce064[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
+};
+static const u8 enc_key064[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - misc */
+static const u8 enc_input065[] __initconst = {
+	0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c,
+	0x2a, 0x2c, 0xfb, 0x7b, 0x1b, 0xbc, 0x01, 0x17,
+	0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, 0xc4, 0x84,
+	0xff, 0x1f, 0xcf, 0x41, 0x94, 0x2c, 0xd6, 0x41
+};
+static const u8 enc_output065[] __initconst = {
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0xbe, 0xde, 0x90, 0x83, 0xce, 0xb3, 0x6d, 0xdf,
+	0xe5, 0xfa, 0x81, 0x1f, 0x95, 0x47, 0x1c, 0x67
+};
+static const u8 enc_assoc065[] __initconst = {
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce065[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
+};
+static const u8 enc_key065[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - misc */
+static const u8 enc_input066[] __initconst = {
+	0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c,
+	0x2a, 0x2c, 0xfb, 0x7b, 0x1b, 0xbc, 0x01, 0x17,
+	0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, 0xc4, 0x84,
+	0xff, 0x1f, 0xcf, 0x41, 0x94, 0x2c, 0xd6, 0x41,
+	0x9c, 0x43, 0x24, 0xa4, 0x61, 0x21, 0x03, 0x01,
+	0xf4, 0x32, 0x5e, 0x49, 0xde, 0xa3, 0x73, 0xd4,
+	0x77, 0x96, 0x00, 0x2d, 0x93, 0xa1, 0xd9, 0x1a,
+	0x2c, 0x48, 0x4d, 0xd8, 0x81, 0x78, 0x02, 0x42
+};
+static const u8 enc_output066[] __initconst = {
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x30, 0x08, 0x74, 0xbb, 0x06, 0x92, 0xb6, 0x89,
+	0xde, 0xad, 0x9a, 0xe1, 0x5b, 0x06, 0x73, 0x90
+};
+static const u8 enc_assoc066[] __initconst = {
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce066[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
+};
+static const u8 enc_key066[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - misc */
+static const u8 enc_input067[] __initconst = {
+	0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c,
+	0x2a, 0x2c, 0xfb, 0x7b, 0x1b, 0xbc, 0x01, 0x17,
+	0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, 0xc4, 0x84,
+	0xff, 0x1f, 0xcf, 0x41, 0x94, 0x2c, 0xd6, 0x41,
+	0x9c, 0x43, 0x24, 0xa4, 0x61, 0x21, 0x03, 0x01,
+	0xf4, 0x32, 0x5e, 0x49, 0xde, 0xa3, 0x73, 0xd4,
+	0x77, 0x96, 0x00, 0x2d, 0x93, 0xa1, 0xd9, 0x1a,
+	0x2c, 0x48, 0x4d, 0xd8, 0x81, 0x78, 0x02, 0x42,
+	0x05, 0x25, 0xbb, 0xbd, 0x3d, 0x96, 0x40, 0x05,
+	0x2a, 0xd8, 0x0d, 0x8f, 0xd3, 0x09, 0x7a, 0xfd,
+	0xc8, 0xb3, 0xa5, 0x1d, 0x99, 0xf3, 0xfa, 0x7f,
+	0xe7, 0xe5, 0xb6, 0xc7, 0x3a, 0x6c, 0x6d, 0x3b,
+	0xe4, 0x4d, 0x0d, 0x7b, 0xc9, 0xb9, 0x10, 0x38,
+	0x8c, 0x0f, 0x4e, 0xc9, 0x62, 0x3c, 0xb7, 0x12,
+	0x08, 0x2c, 0xf4, 0x3a, 0x09, 0x6d, 0x12, 0xc7,
+	0x84, 0x53, 0xfe, 0x77, 0x47, 0xfb, 0x77, 0x38
+};
+static const u8 enc_output067[] __initconst = {
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x99, 0xca, 0xd8, 0x5f, 0x45, 0xca, 0x40, 0x94,
+	0x2d, 0x0d, 0x4d, 0x5e, 0x95, 0x0a, 0xde, 0x22
+};
+static const u8 enc_assoc067[] __initconst = {
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
+	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce067[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
+};
+static const u8 enc_key067[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - misc */
+static const u8 enc_input068[] __initconst = {
+	0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c,
+	0x55, 0xd3, 0x04, 0x84, 0x9b, 0xbc, 0x01, 0x17,
+	0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, 0xc4, 0x84,
+	0x80, 0xe0, 0x30, 0xbe, 0x14, 0x2c, 0xd6, 0x41
+};
+static const u8 enc_output068[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x8b, 0xbe, 0x14, 0x52, 0x72, 0xe7, 0xc2, 0xd9,
+	0xa1, 0x89, 0x1a, 0x3a, 0xb0, 0x98, 0x3d, 0x9d
+};
+static const u8 enc_assoc068[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce068[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
+};
+static const u8 enc_key068[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - misc */
+static const u8 enc_input069[] __initconst = {
+	0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c,
+	0x55, 0xd3, 0x04, 0x84, 0x9b, 0xbc, 0x01, 0x17,
+	0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, 0xc4, 0x84,
+	0x80, 0xe0, 0x30, 0xbe, 0x14, 0x2c, 0xd6, 0x41,
+	0xe3, 0xbc, 0xdb, 0x5b, 0xe1, 0x21, 0x03, 0x01,
+	0x8b, 0xcd, 0xa1, 0xb6, 0x5e, 0xa3, 0x73, 0xd4,
+	0x08, 0x69, 0xff, 0xd2, 0x13, 0xa1, 0xd9, 0x1a,
+	0x53, 0xb7, 0xb2, 0x27, 0x01, 0x78, 0x02, 0x42
+};
+static const u8 enc_output069[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x3b, 0x41, 0x86, 0x19, 0x13, 0xa8, 0xf6, 0xde,
+	0x7f, 0x61, 0xe2, 0x25, 0x63, 0x1b, 0xc3, 0x82
+};
+static const u8 enc_assoc069[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce069[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
+};
+static const u8 enc_key069[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - misc */
+static const u8 enc_input070[] __initconst = {
+	0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c,
+	0x55, 0xd3, 0x04, 0x84, 0x9b, 0xbc, 0x01, 0x17,
+	0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, 0xc4, 0x84,
+	0x80, 0xe0, 0x30, 0xbe, 0x14, 0x2c, 0xd6, 0x41,
+	0xe3, 0xbc, 0xdb, 0x5b, 0xe1, 0x21, 0x03, 0x01,
+	0x8b, 0xcd, 0xa1, 0xb6, 0x5e, 0xa3, 0x73, 0xd4,
+	0x08, 0x69, 0xff, 0xd2, 0x13, 0xa1, 0xd9, 0x1a,
+	0x53, 0xb7, 0xb2, 0x27, 0x01, 0x78, 0x02, 0x42,
+	0x7a, 0xda, 0x44, 0x42, 0xbd, 0x96, 0x40, 0x05,
+	0x55, 0x27, 0xf2, 0x70, 0x53, 0x09, 0x7a, 0xfd,
+	0xb7, 0x4c, 0x5a, 0xe2, 0x19, 0xf3, 0xfa, 0x7f,
+	0x98, 0x1a, 0x49, 0x38, 0xba, 0x6c, 0x6d, 0x3b,
+	0x9b, 0xb2, 0xf2, 0x84, 0x49, 0xb9, 0x10, 0x38,
+	0xf3, 0xf0, 0xb1, 0x36, 0xe2, 0x3c, 0xb7, 0x12,
+	0x77, 0xd3, 0x0b, 0xc5, 0x89, 0x6d, 0x12, 0xc7,
+	0xfb, 0xac, 0x01, 0x88, 0xc7, 0xfb, 0x77, 0x38
+};
+static const u8 enc_output070[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x84, 0x28, 0xbc, 0xf0, 0x23, 0xec, 0x6b, 0xf3,
+	0x1f, 0xd9, 0xef, 0xb2, 0x03, 0xff, 0x08, 0x71
+};
+static const u8 enc_assoc070[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce070[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
+};
+static const u8 enc_key070[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - misc */
+static const u8 enc_input071[] __initconst = {
+	0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83,
+	0xaa, 0x2c, 0xfb, 0x7b, 0x64, 0x43, 0xfe, 0xe8,
+	0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, 0x3b, 0x7b,
+	0x7f, 0x1f, 0xcf, 0x41, 0xeb, 0xd3, 0x29, 0xbe
+};
+static const u8 enc_output071[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0x13, 0x9f, 0xdf, 0x64, 0x74, 0xea, 0x24, 0xf5,
+	0x49, 0xb0, 0x75, 0x82, 0x5f, 0x2c, 0x76, 0x20
+};
+static const u8 enc_assoc071[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00
+};
+static const u8 enc_nonce071[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
+};
+static const u8 enc_key071[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - misc */
+static const u8 enc_input072[] __initconst = {
+	0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83,
+	0xaa, 0x2c, 0xfb, 0x7b, 0x64, 0x43, 0xfe, 0xe8,
+	0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, 0x3b, 0x7b,
+	0x7f, 0x1f, 0xcf, 0x41, 0xeb, 0xd3, 0x29, 0xbe,
+	0x1c, 0x43, 0x24, 0xa4, 0x1e, 0xde, 0xfc, 0xfe,
+	0x74, 0x32, 0x5e, 0x49, 0xa1, 0x5c, 0x8c, 0x2b,
+	0xf7, 0x96, 0x00, 0x2d, 0xec, 0x5e, 0x26, 0xe5,
+	0xac, 0x48, 0x4d, 0xd8, 0xfe, 0x87, 0xfd, 0xbd
+};
+static const u8 enc_output072[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xbb, 0xad, 0x8d, 0x86, 0x3b, 0x83, 0x5a, 0x8e,
+	0x86, 0x64, 0xfd, 0x1d, 0x45, 0x66, 0xb6, 0xb4
+};
+static const u8 enc_assoc072[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00
+};
+static const u8 enc_nonce072[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
+};
+static const u8 enc_key072[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - misc */
+static const u8 enc_input073[] __initconst = {
+	0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83,
+	0xaa, 0x2c, 0xfb, 0x7b, 0x64, 0x43, 0xfe, 0xe8,
+	0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, 0x3b, 0x7b,
+	0x7f, 0x1f, 0xcf, 0x41, 0xeb, 0xd3, 0x29, 0xbe,
+	0x1c, 0x43, 0x24, 0xa4, 0x1e, 0xde, 0xfc, 0xfe,
+	0x74, 0x32, 0x5e, 0x49, 0xa1, 0x5c, 0x8c, 0x2b,
+	0xf7, 0x96, 0x00, 0x2d, 0xec, 0x5e, 0x26, 0xe5,
+	0xac, 0x48, 0x4d, 0xd8, 0xfe, 0x87, 0xfd, 0xbd,
+	0x85, 0x25, 0xbb, 0xbd, 0x42, 0x69, 0xbf, 0xfa,
+	0xaa, 0xd8, 0x0d, 0x8f, 0xac, 0xf6, 0x85, 0x02,
+	0x48, 0xb3, 0xa5, 0x1d, 0xe6, 0x0c, 0x05, 0x80,
+	0x67, 0xe5, 0xb6, 0xc7, 0x45, 0x93, 0x92, 0xc4,
+	0x64, 0x4d, 0x0d, 0x7b, 0xb6, 0x46, 0xef, 0xc7,
+	0x0c, 0x0f, 0x4e, 0xc9, 0x1d, 0xc3, 0x48, 0xed,
+	0x88, 0x2c, 0xf4, 0x3a, 0x76, 0x92, 0xed, 0x38,
+	0x04, 0x53, 0xfe, 0x77, 0x38, 0x04, 0x88, 0xc7
+};
+static const u8 enc_output073[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0x42, 0xf2, 0x35, 0x42, 0x97, 0x84, 0x9a, 0x51,
+	0x1d, 0x53, 0xe5, 0x57, 0x17, 0x72, 0xf7, 0x1f
+};
+static const u8 enc_assoc073[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00
+};
+static const u8 enc_nonce073[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
+};
+static const u8 enc_key073[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - checking for int overflows */
+static const u8 enc_input074[] __initconst = {
+	0xd4, 0x50, 0x0b, 0xf0, 0x09, 0x49, 0x35, 0x51,
+	0xc3, 0x80, 0xad, 0xf5, 0x2c, 0x57, 0x3a, 0x69,
+	0xdf, 0x7e, 0x8b, 0x76, 0x24, 0x63, 0x33, 0x0f,
+	0xac, 0xc1, 0x6a, 0x57, 0x26, 0xbe, 0x71, 0x90,
+	0xc6, 0x3c, 0x5a, 0x1c, 0x92, 0x65, 0x84, 0xa0,
+	0x96, 0x75, 0x68, 0x28, 0xdc, 0xdc, 0x64, 0xac,
+	0xdf, 0x96, 0x3d, 0x93, 0x1b, 0xf1, 0xda, 0xe2,
+	0x38, 0xf3, 0xf1, 0x57, 0x22, 0x4a, 0xc4, 0xb5,
+	0x42, 0xd7, 0x85, 0xb0, 0xdd, 0x84, 0xdb, 0x6b,
+	0xe3, 0xbc, 0x5a, 0x36, 0x63, 0xe8, 0x41, 0x49,
+	0xff, 0xbe, 0xd0, 0x9e, 0x54, 0xf7, 0x8f, 0x16,
+	0xa8, 0x22, 0x3b, 0x24, 0xcb, 0x01, 0x9f, 0x58,
+	0xb2, 0x1b, 0x0e, 0x55, 0x1e, 0x7a, 0xa0, 0x73,
+	0x27, 0x62, 0x95, 0x51, 0x37, 0x6c, 0xcb, 0xc3,
+	0x93, 0x76, 0x71, 0xa0, 0x62, 0x9b, 0xd9, 0x5c,
+	0x99, 0x15, 0xc7, 0x85, 0x55, 0x77, 0x1e, 0x7a
+};
+static const u8 enc_output074[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x0b, 0x30, 0x0d, 0x8d, 0xa5, 0x6c, 0x21, 0x85,
+	0x75, 0x52, 0x79, 0x55, 0x3c, 0x4c, 0x82, 0xca
+};
+static const u8 enc_assoc074[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce074[] __initconst = {
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+	0x00, 0x02, 0x50, 0x6e
+};
+static const u8 enc_key074[] __initconst = {
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30
+};
+
+/* wycheproof - checking for int overflows */
+static const u8 enc_input075[] __initconst = {
+	0x7d, 0xe8, 0x7f, 0x67, 0x29, 0x94, 0x52, 0x75,
+	0xd0, 0x65, 0x5d, 0xa4, 0xc7, 0xfd, 0xe4, 0x56,
+	0x9e, 0x16, 0xf1, 0x11, 0xb5, 0xeb, 0x26, 0xc2,
+	0x2d, 0x85, 0x9e, 0x3f, 0xf8, 0x22, 0xec, 0xed,
+	0x3a, 0x6d, 0xd9, 0xa6, 0x0f, 0x22, 0x95, 0x7f,
+	0x7b, 0x7c, 0x85, 0x7e, 0x88, 0x22, 0xeb, 0x9f,
+	0xe0, 0xb8, 0xd7, 0x02, 0x21, 0x41, 0xf2, 0xd0,
+	0xb4, 0x8f, 0x4b, 0x56, 0x12, 0xd3, 0x22, 0xa8,
+	0x8d, 0xd0, 0xfe, 0x0b, 0x4d, 0x91, 0x79, 0x32,
+	0x4f, 0x7c, 0x6c, 0x9e, 0x99, 0x0e, 0xfb, 0xd8,
+	0x0e, 0x5e, 0xd6, 0x77, 0x58, 0x26, 0x49, 0x8b,
+	0x1e, 0xfe, 0x0f, 0x71, 0xa0, 0xf3, 0xec, 0x5b,
+	0x29, 0xcb, 0x28, 0xc2, 0x54, 0x0a, 0x7d, 0xcd,
+	0x51, 0xb7, 0xda, 0xae, 0xe0, 0xff, 0x4a, 0x7f,
+	0x3a, 0xc1, 0xee, 0x54, 0xc2, 0x9e, 0xe4, 0xc1,
+	0x70, 0xde, 0x40, 0x8f, 0x66, 0x69, 0x21, 0x94
+};
+static const u8 enc_output075[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xc5, 0x78, 0xe2, 0xaa, 0x44, 0xd3, 0x09, 0xb7,
+	0xb6, 0xa5, 0x19, 0x3b, 0xdc, 0x61, 0x18, 0xf5
+};
+static const u8 enc_assoc075[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce075[] __initconst = {
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+	0x00, 0x03, 0x18, 0xa5
+};
+static const u8 enc_key075[] __initconst = {
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30
+};
+
+/* wycheproof - checking for int overflows */
+static const u8 enc_input076[] __initconst = {
+	0x1b, 0x99, 0x6f, 0x9a, 0x3c, 0xcc, 0x67, 0x85,
+	0xde, 0x22, 0xff, 0x5b, 0x8a, 0xdd, 0x95, 0x02,
+	0xce, 0x03, 0xa0, 0xfa, 0xf5, 0x99, 0x2a, 0x09,
+	0x52, 0x2c, 0xdd, 0x12, 0x06, 0xd2, 0x20, 0xb8,
+	0xf8, 0xbd, 0x07, 0xd1, 0xf1, 0xf5, 0xa1, 0xbd,
+	0x9a, 0x71, 0xd1, 0x1c, 0x7f, 0x57, 0x9b, 0x85,
+	0x58, 0x18, 0xc0, 0x8d, 0x4d, 0xe0, 0x36, 0x39,
+	0x31, 0x83, 0xb7, 0xf5, 0x90, 0xb3, 0x35, 0xae,
+	0xd8, 0xde, 0x5b, 0x57, 0xb1, 0x3c, 0x5f, 0xed,
+	0xe2, 0x44, 0x1c, 0x3e, 0x18, 0x4a, 0xa9, 0xd4,
+	0x6e, 0x61, 0x59, 0x85, 0x06, 0xb3, 0xe1, 0x1c,
+	0x43, 0xc6, 0x2c, 0xbc, 0xac, 0xec, 0xed, 0x33,
+	0x19, 0x08, 0x75, 0xb0, 0x12, 0x21, 0x8b, 0x19,
+	0x30, 0xfb, 0x7c, 0x38, 0xec, 0x45, 0xac, 0x11,
+	0xc3, 0x53, 0xd0, 0xcf, 0x93, 0x8d, 0xcc, 0xb9,
+	0xef, 0xad, 0x8f, 0xed, 0xbe, 0x46, 0xda, 0xa5
+};
+static const u8 enc_output076[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x4b, 0x0b, 0xda, 0x8a, 0xd0, 0x43, 0x83, 0x0d,
+	0x83, 0x19, 0xab, 0x82, 0xc5, 0x0c, 0x76, 0x63
+};
+static const u8 enc_assoc076[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce076[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xb4, 0xf0
+};
+static const u8 enc_key076[] __initconst = {
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30
+};
+
+/* wycheproof - checking for int overflows */
+static const u8 enc_input077[] __initconst = {
+	0x86, 0xcb, 0xac, 0xae, 0x4d, 0x3f, 0x74, 0xae,
+	0x01, 0x21, 0x3e, 0x05, 0x51, 0xcc, 0x15, 0x16,
+	0x0e, 0xa1, 0xbe, 0x84, 0x08, 0xe3, 0xd5, 0xd7,
+	0x4f, 0x01, 0x46, 0x49, 0x95, 0xa6, 0x9e, 0x61,
+	0x76, 0xcb, 0x9e, 0x02, 0xb2, 0x24, 0x7e, 0xd2,
+	0x99, 0x89, 0x2f, 0x91, 0x82, 0xa4, 0x5c, 0xaf,
+	0x4c, 0x69, 0x40, 0x56, 0x11, 0x76, 0x6e, 0xdf,
+	0xaf, 0xdc, 0x28, 0x55, 0x19, 0xea, 0x30, 0x48,
+	0x0c, 0x44, 0xf0, 0x5e, 0x78, 0x1e, 0xac, 0xf8,
+	0xfc, 0xec, 0xc7, 0x09, 0x0a, 0xbb, 0x28, 0xfa,
+	0x5f, 0xd5, 0x85, 0xac, 0x8c, 0xda, 0x7e, 0x87,
+	0x72, 0xe5, 0x94, 0xe4, 0xce, 0x6c, 0x88, 0x32,
+	0x81, 0x93, 0x2e, 0x0f, 0x89, 0xf8, 0x77, 0xa1,
+	0xf0, 0x4d, 0x9c, 0x32, 0xb0, 0x6c, 0xf9, 0x0b,
+	0x0e, 0x76, 0x2b, 0x43, 0x0c, 0x4d, 0x51, 0x7c,
+	0x97, 0x10, 0x70, 0x68, 0xf4, 0x98, 0xef, 0x7f
+};
+static const u8 enc_output077[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x4b, 0xc9, 0x8f, 0x72, 0xc4, 0x94, 0xc2, 0xa4,
+	0x3c, 0x2b, 0x15, 0xa1, 0x04, 0x3f, 0x1c, 0xfa
+};
+static const u8 enc_assoc077[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce077[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0xfb, 0x66
+};
+static const u8 enc_key077[] __initconst = {
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30
+};
+
+/* wycheproof - checking for int overflows */
+static const u8 enc_input078[] __initconst = {
+	0xfa, 0xb1, 0xcd, 0xdf, 0x4f, 0xe1, 0x98, 0xef,
+	0x63, 0xad, 0xd8, 0x81, 0xd6, 0xea, 0xd6, 0xc5,
+	0x76, 0x37, 0xbb, 0xe9, 0x20, 0x18, 0xca, 0x7c,
+	0x0b, 0x96, 0xfb, 0xa0, 0x87, 0x1e, 0x93, 0x2d,
+	0xb1, 0xfb, 0xf9, 0x07, 0x61, 0xbe, 0x25, 0xdf,
+	0x8d, 0xfa, 0xf9, 0x31, 0xce, 0x57, 0x57, 0xe6,
+	0x17, 0xb3, 0xd7, 0xa9, 0xf0, 0xbf, 0x0f, 0xfe,
+	0x5d, 0x59, 0x1a, 0x33, 0xc1, 0x43, 0xb8, 0xf5,
+	0x3f, 0xd0, 0xb5, 0xa1, 0x96, 0x09, 0xfd, 0x62,
+	0xe5, 0xc2, 0x51, 0xa4, 0x28, 0x1a, 0x20, 0x0c,
+	0xfd, 0xc3, 0x4f, 0x28, 0x17, 0x10, 0x40, 0x6f,
+	0x4e, 0x37, 0x62, 0x54, 0x46, 0xff, 0x6e, 0xf2,
+	0x24, 0x91, 0x3d, 0xeb, 0x0d, 0x89, 0xaf, 0x33,
+	0x71, 0x28, 0xe3, 0xd1, 0x55, 0xd1, 0x6d, 0x3e,
+	0xc3, 0x24, 0x60, 0x41, 0x43, 0x21, 0x43, 0xe9,
+	0xab, 0x3a, 0x6d, 0x2c, 0xcc, 0x2f, 0x4d, 0x62
+};
+static const u8 enc_output078[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xf7, 0xe9, 0xe1, 0x51, 0xb0, 0x25, 0x33, 0xc7,
+	0x46, 0x58, 0xbf, 0xc7, 0x73, 0x7c, 0x68, 0x0d
+};
+static const u8 enc_assoc078[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce078[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0xbb, 0x90
+};
+static const u8 enc_key078[] __initconst = {
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30
+};
+
+/* wycheproof - checking for int overflows */
+static const u8 enc_input079[] __initconst = {
+	0x22, 0x72, 0x02, 0xbe, 0x7f, 0x35, 0x15, 0xe9,
+	0xd1, 0xc0, 0x2e, 0xea, 0x2f, 0x19, 0x50, 0xb6,
+	0x48, 0x1b, 0x04, 0x8a, 0x4c, 0x91, 0x50, 0x6c,
+	0xb4, 0x0d, 0x50, 0x4e, 0x6c, 0x94, 0x9f, 0x82,
+	0xd1, 0x97, 0xc2, 0x5a, 0xd1, 0x7d, 0xc7, 0x21,
+	0x65, 0x11, 0x25, 0x78, 0x2a, 0xc7, 0xa7, 0x12,
+	0x47, 0xfe, 0xae, 0xf3, 0x2f, 0x1f, 0x25, 0x0c,
+	0xe4, 0xbb, 0x8f, 0x79, 0xac, 0xaa, 0x17, 0x9d,
+	0x45, 0xa7, 0xb0, 0x54, 0x5f, 0x09, 0x24, 0x32,
+	0x5e, 0xfa, 0x87, 0xd5, 0xe4, 0x41, 0xd2, 0x84,
+	0x78, 0xc6, 0x1f, 0x22, 0x23, 0xee, 0x67, 0xc3,
+	0xb4, 0x1f, 0x43, 0x94, 0x53, 0x5e, 0x2a, 0x24,
+	0x36, 0x9a, 0x2e, 0x16, 0x61, 0x3c, 0x45, 0x94,
+	0x90, 0xc1, 0x4f, 0xb1, 0xd7, 0x55, 0xfe, 0x53,
+	0xfb, 0xe1, 0xee, 0x45, 0xb1, 0xb2, 0x1f, 0x71,
+	0x62, 0xe2, 0xfc, 0xaa, 0x74, 0x2a, 0xbe, 0xfd
+};
+static const u8 enc_output079[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x79, 0x5b, 0xcf, 0xf6, 0x47, 0xc5, 0x53, 0xc2,
+	0xe4, 0xeb, 0x6e, 0x0e, 0xaf, 0xd9, 0xe0, 0x4e
+};
+static const u8 enc_assoc079[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce079[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x48, 0x4a
+};
+static const u8 enc_key079[] __initconst = {
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30
+};
+
+/* wycheproof - checking for int overflows */
+static const u8 enc_input080[] __initconst = {
+	0xfa, 0xe5, 0x83, 0x45, 0xc1, 0x6c, 0xb0, 0xf5,
+	0xcc, 0x53, 0x7f, 0x2b, 0x1b, 0x34, 0x69, 0xc9,
+	0x69, 0x46, 0x3b, 0x3e, 0xa7, 0x1b, 0xcf, 0x6b,
+	0x98, 0xd6, 0x69, 0xa8, 0xe6, 0x0e, 0x04, 0xfc,
+	0x08, 0xd5, 0xfd, 0x06, 0x9c, 0x36, 0x26, 0x38,
+	0xe3, 0x40, 0x0e, 0xf4, 0xcb, 0x24, 0x2e, 0x27,
+	0xe2, 0x24, 0x5e, 0x68, 0xcb, 0x9e, 0xc5, 0x83,
+	0xda, 0x53, 0x40, 0xb1, 0x2e, 0xdf, 0x42, 0x3b,
+	0x73, 0x26, 0xad, 0x20, 0xfe, 0xeb, 0x57, 0xda,
+	0xca, 0x2e, 0x04, 0x67, 0xa3, 0x28, 0x99, 0xb4,
+	0x2d, 0xf8, 0xe5, 0x6d, 0x84, 0xe0, 0x06, 0xbc,
+	0x8a, 0x7a, 0xcc, 0x73, 0x1e, 0x7c, 0x1f, 0x6b,
+	0xec, 0xb5, 0x71, 0x9f, 0x70, 0x77, 0xf0, 0xd4,
+	0xf4, 0xc6, 0x1a, 0xb1, 0x1e, 0xba, 0xc1, 0x00,
+	0x18, 0x01, 0xce, 0x33, 0xc4, 0xe4, 0xa7, 0x7d,
+	0x83, 0x1d, 0x3c, 0xe3, 0x4e, 0x84, 0x10, 0xe1
+};
+static const u8 enc_output080[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x19, 0x46, 0xd6, 0x53, 0x96, 0x0f, 0x94, 0x7a,
+	0x74, 0xd3, 0xe8, 0x09, 0x3c, 0xf4, 0x85, 0x02
+};
+static const u8 enc_assoc080[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce080[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0x2f, 0x40
+};
+static const u8 enc_key080[] __initconst = {
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30
+};
+
+/* wycheproof - checking for int overflows */
+static const u8 enc_input081[] __initconst = {
+	0xeb, 0xb2, 0x16, 0xdd, 0xd7, 0xca, 0x70, 0x92,
+	0x15, 0xf5, 0x03, 0xdf, 0x9c, 0xe6, 0x3c, 0x5c,
+	0xd2, 0x19, 0x4e, 0x7d, 0x90, 0x99, 0xe8, 0xa9,
+	0x0b, 0x2a, 0xfa, 0xad, 0x5e, 0xba, 0x35, 0x06,
+	0x99, 0x25, 0xa6, 0x03, 0xfd, 0xbc, 0x34, 0x1a,
+	0xae, 0xd4, 0x15, 0x05, 0xb1, 0x09, 0x41, 0xfa,
+	0x38, 0x56, 0xa7, 0xe2, 0x47, 0xb1, 0x04, 0x07,
+	0x09, 0x74, 0x6c, 0xfc, 0x20, 0x96, 0xca, 0xa6,
+	0x31, 0xb2, 0xff, 0xf4, 0x1c, 0x25, 0x05, 0x06,
+	0xd8, 0x89, 0xc1, 0xc9, 0x06, 0x71, 0xad, 0xe8,
+	0x53, 0xee, 0x63, 0x94, 0xc1, 0x91, 0x92, 0xa5,
+	0xcf, 0x37, 0x10, 0xd1, 0x07, 0x30, 0x99, 0xe5,
+	0xbc, 0x94, 0x65, 0x82, 0xfc, 0x0f, 0xab, 0x9f,
+	0x54, 0x3c, 0x71, 0x6a, 0xe2, 0x48, 0x6a, 0x86,
+	0x83, 0xfd, 0xca, 0x39, 0xd2, 0xe1, 0x4f, 0x23,
+	0xd0, 0x0a, 0x58, 0x26, 0x64, 0xf4, 0xec, 0xb1
+};
+static const u8 enc_output081[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x36, 0xc3, 0x00, 0x29, 0x85, 0xdd, 0x21, 0xba,
+	0xf8, 0x95, 0xd6, 0x33, 0x57, 0x3f, 0x12, 0xc0
+};
+static const u8 enc_assoc081[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce081[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x00, 0xe2, 0x93, 0x35
+};
+static const u8 enc_key081[] __initconst = {
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30
+};
+
+/* wycheproof - checking for int overflows */
+static const u8 enc_input082[] __initconst = {
+	0x40, 0x8a, 0xe6, 0xef, 0x1c, 0x7e, 0xf0, 0xfb,
+	0x2c, 0x2d, 0x61, 0x08, 0x16, 0xfc, 0x78, 0x49,
+	0xef, 0xa5, 0x8f, 0x78, 0x27, 0x3f, 0x5f, 0x16,
+	0x6e, 0xa6, 0x5f, 0x81, 0xb5, 0x75, 0x74, 0x7d,
+	0x03, 0x5b, 0x30, 0x40, 0xfe, 0xde, 0x1e, 0xb9,
+	0x45, 0x97, 0x88, 0x66, 0x97, 0x88, 0x40, 0x8e,
+	0x00, 0x41, 0x3b, 0x3e, 0x37, 0x6d, 0x15, 0x2d,
+	0x20, 0x4a, 0xa2, 0xb7, 0xa8, 0x35, 0x58, 0xfc,
+	0xd4, 0x8a, 0x0e, 0xf7, 0xa2, 0x6b, 0x1c, 0xd6,
+	0xd3, 0x5d, 0x23, 0xb3, 0xf5, 0xdf, 0xe0, 0xca,
+	0x77, 0xa4, 0xce, 0x32, 0xb9, 0x4a, 0xbf, 0x83,
+	0xda, 0x2a, 0xef, 0xca, 0xf0, 0x68, 0x38, 0x08,
+	0x79, 0xe8, 0x9f, 0xb0, 0xa3, 0x82, 0x95, 0x95,
+	0xcf, 0x44, 0xc3, 0x85, 0x2a, 0xe2, 0xcc, 0x66,
+	0x2b, 0x68, 0x9f, 0x93, 0x55, 0xd9, 0xc1, 0x83,
+	0x80, 0x1f, 0x6a, 0xcc, 0x31, 0x3f, 0x89, 0x07
+};
+static const u8 enc_output082[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x65, 0x14, 0x51, 0x8e, 0x0a, 0x26, 0x41, 0x42,
+	0xe0, 0xb7, 0x35, 0x1f, 0x96, 0x7f, 0xc2, 0xae
+};
+static const u8 enc_assoc082[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce082[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0xf7, 0xd5
+};
+static const u8 enc_key082[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - checking for int overflows */
+static const u8 enc_input083[] __initconst = {
+	0x0a, 0x0a, 0x24, 0x49, 0x9b, 0xca, 0xde, 0x58,
+	0xcf, 0x15, 0x76, 0xc3, 0x12, 0xac, 0xa9, 0x84,
+	0x71, 0x8c, 0xb4, 0xcc, 0x7e, 0x01, 0x53, 0xf5,
+	0xa9, 0x01, 0x58, 0x10, 0x85, 0x96, 0x44, 0xdf,
+	0xc0, 0x21, 0x17, 0x4e, 0x0b, 0x06, 0x0a, 0x39,
+	0x74, 0x48, 0xde, 0x8b, 0x48, 0x4a, 0x86, 0x03,
+	0xbe, 0x68, 0x0a, 0x69, 0x34, 0xc0, 0x90, 0x6f,
+	0x30, 0xdd, 0x17, 0xea, 0xe2, 0xd4, 0xc5, 0xfa,
+	0xa7, 0x77, 0xf8, 0xca, 0x53, 0x37, 0x0e, 0x08,
+	0x33, 0x1b, 0x88, 0xc3, 0x42, 0xba, 0xc9, 0x59,
+	0x78, 0x7b, 0xbb, 0x33, 0x93, 0x0e, 0x3b, 0x56,
+	0xbe, 0x86, 0xda, 0x7f, 0x2a, 0x6e, 0xb1, 0xf9,
+	0x40, 0x89, 0xd1, 0xd1, 0x81, 0x07, 0x4d, 0x43,
+	0x02, 0xf8, 0xe0, 0x55, 0x2d, 0x0d, 0xe1, 0xfa,
+	0xb3, 0x06, 0xa2, 0x1b, 0x42, 0xd4, 0xc3, 0xba,
+	0x6e, 0x6f, 0x0c, 0xbc, 0xc8, 0x1e, 0x87, 0x7a
+};
+static const u8 enc_output083[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x4c, 0x19, 0x4d, 0xa6, 0xa9, 0x9f, 0xd6, 0x5b,
+	0x40, 0xe9, 0xca, 0xd7, 0x98, 0xf4, 0x4b, 0x19
+};
+static const u8 enc_assoc083[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce083[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x3d, 0xfc, 0xe4
+};
+static const u8 enc_key083[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - checking for int overflows */
+static const u8 enc_input084[] __initconst = {
+	0x4a, 0x0a, 0xaf, 0xf8, 0x49, 0x47, 0x29, 0x18,
+	0x86, 0x91, 0x70, 0x13, 0x40, 0xf3, 0xce, 0x2b,
+	0x8a, 0x78, 0xee, 0xd3, 0xa0, 0xf0, 0x65, 0x99,
+	0x4b, 0x72, 0x48, 0x4e, 0x79, 0x91, 0xd2, 0x5c,
+	0x29, 0xaa, 0x07, 0x5e, 0xb1, 0xfc, 0x16, 0xde,
+	0x93, 0xfe, 0x06, 0x90, 0x58, 0x11, 0x2a, 0xb2,
+	0x84, 0xa3, 0xed, 0x18, 0x78, 0x03, 0x26, 0xd1,
+	0x25, 0x8a, 0x47, 0x22, 0x2f, 0xa6, 0x33, 0xd8,
+	0xb2, 0x9f, 0x3b, 0xd9, 0x15, 0x0b, 0x23, 0x9b,
+	0x15, 0x46, 0xc2, 0xbb, 0x9b, 0x9f, 0x41, 0x0f,
+	0xeb, 0xea, 0xd3, 0x96, 0x00, 0x0e, 0xe4, 0x77,
+	0x70, 0x15, 0x32, 0xc3, 0xd0, 0xf5, 0xfb, 0xf8,
+	0x95, 0xd2, 0x80, 0x19, 0x6d, 0x2f, 0x73, 0x7c,
+	0x5e, 0x9f, 0xec, 0x50, 0xd9, 0x2b, 0xb0, 0xdf,
+	0x5d, 0x7e, 0x51, 0x3b, 0xe5, 0xb8, 0xea, 0x97,
+	0x13, 0x10, 0xd5, 0xbf, 0x16, 0xba, 0x7a, 0xee
+};
+static const u8 enc_output084[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xc8, 0xae, 0x77, 0x88, 0xcd, 0x28, 0x74, 0xab,
+	0xc1, 0x38, 0x54, 0x1e, 0x11, 0xfd, 0x05, 0x87
+};
+static const u8 enc_assoc084[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce084[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x01, 0x84, 0x86, 0xa8
+};
+static const u8 enc_key084[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - checking for int overflows */
+static const u8 enc_input085[] __initconst = {
+	0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
+	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
+	0x78, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09,
+	0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8,
+	0x9f, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca,
+	0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64,
+	0x9c, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39,
+	0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4,
+	0x47, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2,
+	0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73,
+	0xd4, 0xd2, 0x06, 0x61, 0x6f, 0x92, 0x93, 0xf6,
+	0x5b, 0x45, 0xdb, 0xbc, 0x74, 0xe7, 0xc2, 0xed,
+	0xfb, 0xcb, 0xbf, 0x1c, 0xfb, 0x67, 0x9b, 0xb7,
+	0x39, 0xa5, 0x86, 0x2d, 0xe2, 0xbc, 0xb9, 0x37,
+	0xf7, 0x4d, 0x5b, 0xf8, 0x67, 0x1c, 0x5a, 0x8a,
+	0x50, 0x92, 0xf6, 0x1d, 0x54, 0xc9, 0xaa, 0x5b
+};
+static const u8 enc_output085[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x93, 0x3a, 0x51, 0x63, 0xc7, 0xf6, 0x23, 0x68,
+	0x32, 0x7b, 0x3f, 0xbc, 0x10, 0x36, 0xc9, 0x43
+};
+static const u8 enc_assoc085[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce085[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
+};
+static const u8 enc_key085[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - special case tag */
+static const u8 enc_input086[] __initconst = {
+	0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6,
+	0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd,
+	0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b,
+	0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2,
+	0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19,
+	0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4,
+	0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63,
+	0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d
+};
+static const u8 enc_output086[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+	0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
+};
+static const u8 enc_assoc086[] __initconst = {
+	0x85, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xa6, 0x90, 0x2f, 0xcb, 0xc8, 0x83, 0xbb, 0xc1,
+	0x80, 0xb2, 0x56, 0xae, 0x34, 0xad, 0x7f, 0x00
+};
+static const u8 enc_nonce086[] __initconst = {
+	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+	0x08, 0x09, 0x0a, 0x0b
+};
+static const u8 enc_key086[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - special case tag */
+static const u8 enc_input087[] __initconst = {
+	0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6,
+	0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd,
+	0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b,
+	0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2,
+	0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19,
+	0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4,
+	0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63,
+	0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d
+};
+static const u8 enc_output087[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+static const u8 enc_assoc087[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x24, 0x7e, 0x50, 0x64, 0x2a, 0x1c, 0x0a, 0x2f,
+	0x8f, 0x77, 0x21, 0x96, 0x09, 0xdb, 0xa9, 0x58
+};
+static const u8 enc_nonce087[] __initconst = {
+	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+	0x08, 0x09, 0x0a, 0x0b
+};
+static const u8 enc_key087[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - special case tag */
+static const u8 enc_input088[] __initconst = {
+	0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6,
+	0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd,
+	0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b,
+	0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2,
+	0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19,
+	0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4,
+	0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63,
+	0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d
+};
+static const u8 enc_output088[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_assoc088[] __initconst = {
+	0x7c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xd9, 0xe7, 0x2c, 0x06, 0x4a, 0xc8, 0x96, 0x1f,
+	0x3f, 0xa5, 0x85, 0xe0, 0xe2, 0xab, 0xd6, 0x00
+};
+static const u8 enc_nonce088[] __initconst = {
+	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+	0x08, 0x09, 0x0a, 0x0b
+};
+static const u8 enc_key088[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - special case tag */
+static const u8 enc_input089[] __initconst = {
+	0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6,
+	0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd,
+	0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b,
+	0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2,
+	0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19,
+	0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4,
+	0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63,
+	0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d
+};
+static const u8 enc_output089[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80
+};
+static const u8 enc_assoc089[] __initconst = {
+	0x65, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x95, 0xaf, 0x0f, 0x4d, 0x0b, 0x68, 0x6e, 0xae,
+	0xcc, 0xca, 0x43, 0x07, 0xd5, 0x96, 0xf5, 0x02
+};
+static const u8 enc_nonce089[] __initconst = {
+	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+	0x08, 0x09, 0x0a, 0x0b
+};
+static const u8 enc_key089[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - special case tag */
+static const u8 enc_input090[] __initconst = {
+	0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6,
+	0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd,
+	0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b,
+	0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2,
+	0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19,
+	0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4,
+	0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63,
+	0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d
+};
+static const u8 enc_output090[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f
+};
+static const u8 enc_assoc090[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x85, 0x40, 0xb4, 0x64, 0x35, 0x77, 0x07, 0xbe,
+	0x3a, 0x39, 0xd5, 0x5c, 0x34, 0xf8, 0xbc, 0xb3
+};
+static const u8 enc_nonce090[] __initconst = {
+	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+	0x08, 0x09, 0x0a, 0x0b
+};
+static const u8 enc_key090[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - special case tag */
+static const u8 enc_input091[] __initconst = {
+	0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6,
+	0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd,
+	0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b,
+	0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2,
+	0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19,
+	0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4,
+	0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63,
+	0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d
+};
+static const u8 enc_output091[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+	0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00
+};
+static const u8 enc_assoc091[] __initconst = {
+	0x4f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x66, 0x23, 0xd9, 0x90, 0xb8, 0x98, 0xd8, 0x30,
+	0xd2, 0x12, 0xaf, 0x23, 0x83, 0x33, 0x07, 0x01
+};
+static const u8 enc_nonce091[] __initconst = {
+	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+	0x08, 0x09, 0x0a, 0x0b
+};
+static const u8 enc_key091[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - special case tag */
+static const u8 enc_input092[] __initconst = {
+	0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6,
+	0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd,
+	0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b,
+	0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2,
+	0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19,
+	0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4,
+	0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63,
+	0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d
+};
+static const u8 enc_output092[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+static const u8 enc_assoc092[] __initconst = {
+	0x83, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x5f, 0x16, 0xd0, 0x9f, 0x17, 0x78, 0x72, 0x11,
+	0xb7, 0xd4, 0x84, 0xe0, 0x24, 0xf8, 0x97, 0x01
+};
+static const u8 enc_nonce092[] __initconst = {
+	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+	0x08, 0x09, 0x0a, 0x0b
+};
+static const u8 enc_key092[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - edge case intermediate sums in poly1305 */
+static const u8 enc_input093[] __initconst = {
+	0x00, 0x52, 0x35, 0xd2, 0xa9, 0x19, 0xf2, 0x8d,
+	0x3d, 0xb7, 0x66, 0x4a, 0x34, 0xae, 0x6b, 0x44,
+	0x4d, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09,
+	0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8,
+	0x5b, 0x8b, 0x94, 0x50, 0x9e, 0x2b, 0x74, 0xa3,
+	0x6d, 0x34, 0x6e, 0x33, 0xd5, 0x72, 0x65, 0x9b,
+	0xa9, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39,
+	0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4,
+	0x83, 0xdc, 0xe9, 0xf3, 0x07, 0x3e, 0xfa, 0xdb,
+	0x7d, 0x23, 0xb8, 0x7a, 0xce, 0x35, 0x16, 0x8c
+};
+static const u8 enc_output093[] __initconst = {
+	0x00, 0x39, 0xe2, 0xfd, 0x2f, 0xd3, 0x12, 0x14,
+	0x9e, 0x98, 0x98, 0x80, 0x88, 0x48, 0x13, 0xe7,
+	0xca, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x3b, 0x0e, 0x86, 0x9a, 0xaa, 0x8e, 0xa4, 0x96,
+	0x32, 0xff, 0xff, 0x37, 0xb9, 0xe8, 0xce, 0x00,
+	0xca, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x3b, 0x0e, 0x86, 0x9a, 0xaa, 0x8e, 0xa4, 0x96,
+	0x32, 0xff, 0xff, 0x37, 0xb9, 0xe8, 0xce, 0x00,
+	0xa5, 0x19, 0xac, 0x1a, 0x35, 0xb4, 0xa5, 0x77,
+	0x87, 0x51, 0x0a, 0xf7, 0x8d, 0x8d, 0x20, 0x0a
+};
+static const u8 enc_assoc093[] __initconst = {
+	0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce093[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
+};
+static const u8 enc_key093[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - edge case intermediate sums in poly1305 */
+static const u8 enc_input094[] __initconst = {
+	0xd3, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
+	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
+	0xe5, 0xda, 0x78, 0x76, 0x6f, 0xa1, 0x92, 0x90,
+	0xc0, 0x31, 0xf7, 0x52, 0x08, 0x50, 0x67, 0x45,
+	0xae, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca,
+	0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64,
+	0x49, 0x6d, 0xde, 0xb0, 0x55, 0x09, 0xc6, 0xef,
+	0xff, 0xab, 0x75, 0xeb, 0x2d, 0xf4, 0xab, 0x09,
+	0x76, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2,
+	0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73,
+	0x01, 0x49, 0xef, 0x50, 0x4b, 0x71, 0xb1, 0x20,
+	0xca, 0x4f, 0xf3, 0x95, 0x19, 0xc2, 0xc2, 0x10
+};
+static const u8 enc_output094[] __initconst = {
+	0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x62, 0x18, 0xb2, 0x7f, 0x83, 0xb8, 0xb4, 0x66,
+	0x02, 0xf6, 0xe1, 0xd8, 0x34, 0x20, 0x7b, 0x02,
+	0xce, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x2a, 0x64, 0x16, 0xce, 0xdb, 0x1c, 0xdd, 0x29,
+	0x6e, 0xf5, 0xd7, 0xd6, 0x92, 0xda, 0xff, 0x02,
+	0xce, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x2a, 0x64, 0x16, 0xce, 0xdb, 0x1c, 0xdd, 0x29,
+	0x6e, 0xf5, 0xd7, 0xd6, 0x92, 0xda, 0xff, 0x02,
+	0x30, 0x2f, 0xe8, 0x2a, 0xb0, 0xa0, 0x9a, 0xf6,
+	0x44, 0x00, 0xd0, 0x15, 0xae, 0x83, 0xd9, 0xcc
+};
+static const u8 enc_assoc094[] __initconst = {
+	0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce094[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
+};
+static const u8 enc_key094[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - edge case intermediate sums in poly1305 */
+static const u8 enc_input095[] __initconst = {
+	0xe9, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
+	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
+	0x6d, 0xf1, 0x39, 0x4e, 0xdc, 0x53, 0x9b, 0x5b,
+	0x3a, 0x09, 0x57, 0xbe, 0x0f, 0xb8, 0x59, 0x46,
+	0x80, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca,
+	0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64,
+	0xd1, 0x76, 0x9f, 0xe8, 0x06, 0xbb, 0xfe, 0xb6,
+	0xf5, 0x90, 0x95, 0x0f, 0x2e, 0xac, 0x9e, 0x0a,
+	0x58, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2,
+	0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73,
+	0x99, 0x52, 0xae, 0x08, 0x18, 0xc3, 0x89, 0x79,
+	0xc0, 0x74, 0x13, 0x71, 0x1a, 0x9a, 0xf7, 0x13
+};
+static const u8 enc_output095[] __initconst = {
+	0xe9, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xea, 0x33, 0xf3, 0x47, 0x30, 0x4a, 0xbd, 0xad,
+	0xf8, 0xce, 0x41, 0x34, 0x33, 0xc8, 0x45, 0x01,
+	0xe0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xb2, 0x7f, 0x57, 0x96, 0x88, 0xae, 0xe5, 0x70,
+	0x64, 0xce, 0x37, 0x32, 0x91, 0x82, 0xca, 0x01,
+	0xe0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xb2, 0x7f, 0x57, 0x96, 0x88, 0xae, 0xe5, 0x70,
+	0x64, 0xce, 0x37, 0x32, 0x91, 0x82, 0xca, 0x01,
+	0x98, 0xa7, 0xe8, 0x36, 0xe0, 0xee, 0x4d, 0x02,
+	0x35, 0x00, 0xd0, 0x55, 0x7e, 0xc2, 0xcb, 0xe0
+};
+static const u8 enc_assoc095[] __initconst = {
+	0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce095[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
+};
+static const u8 enc_key095[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - edge case intermediate sums in poly1305 */
+static const u8 enc_input096[] __initconst = {
+	0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
+	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
+	0x64, 0xf9, 0x0f, 0x5b, 0x26, 0x92, 0xb8, 0x60,
+	0xd4, 0x59, 0x6f, 0xf4, 0xb3, 0x40, 0x2c, 0x5c,
+	0x00, 0xb9, 0xbb, 0x53, 0x70, 0x7a, 0xa6, 0x67,
+	0xd3, 0x56, 0xfe, 0x50, 0xc7, 0x19, 0x96, 0x94,
+	0x03, 0x35, 0x61, 0xe7, 0xca, 0xca, 0x6d, 0x94,
+	0x1d, 0xc3, 0xcd, 0x69, 0x14, 0xad, 0x69, 0x04
+};
+static const u8 enc_output096[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xe3, 0x3b, 0xc5, 0x52, 0xca, 0x8b, 0x9e, 0x96,
+	0x16, 0x9e, 0x79, 0x7e, 0x8f, 0x30, 0x30, 0x1b,
+	0x60, 0x3c, 0xa9, 0x99, 0x44, 0xdf, 0x76, 0x52,
+	0x8c, 0x9d, 0x6f, 0x54, 0xab, 0x83, 0x3d, 0x0f,
+	0x60, 0x3c, 0xa9, 0x99, 0x44, 0xdf, 0x76, 0x52,
+	0x8c, 0x9d, 0x6f, 0x54, 0xab, 0x83, 0x3d, 0x0f,
+	0x6a, 0xb8, 0xdc, 0xe2, 0xc5, 0x9d, 0xa4, 0x73,
+	0x71, 0x30, 0xb0, 0x25, 0x2f, 0x68, 0xa8, 0xd8
+};
+static const u8 enc_assoc096[] __initconst = {
+	0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce096[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
+};
+static const u8 enc_key096[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - edge case intermediate sums in poly1305 */
+static const u8 enc_input097[] __initconst = {
+	0x68, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
+	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
+	0xb0, 0x8f, 0x25, 0x67, 0x5b, 0x9b, 0xcb, 0xf6,
+	0xe3, 0x84, 0x07, 0xde, 0x2e, 0xc7, 0x5a, 0x47,
+	0x9f, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca,
+	0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64,
+	0x2d, 0x2a, 0xf7, 0xcd, 0x6b, 0x08, 0x05, 0x01,
+	0xd3, 0x1b, 0xa5, 0x4f, 0xb2, 0xeb, 0x75, 0x96,
+	0x47, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2,
+	0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73,
+	0x65, 0x0e, 0xc6, 0x2d, 0x75, 0x70, 0x72, 0xce,
+	0xe6, 0xff, 0x23, 0x31, 0x86, 0xdd, 0x1c, 0x8f
+};
+static const u8 enc_output097[] __initconst = {
+	0x68, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x37, 0x4d, 0xef, 0x6e, 0xb7, 0x82, 0xed, 0x00,
+	0x21, 0x43, 0x11, 0x54, 0x12, 0xb7, 0x46, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x4e, 0x23, 0x3f, 0xb3, 0xe5, 0x1d, 0x1e, 0xc7,
+	0x42, 0x45, 0x07, 0x72, 0x0d, 0xc5, 0x21, 0x9d,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x4e, 0x23, 0x3f, 0xb3, 0xe5, 0x1d, 0x1e, 0xc7,
+	0x42, 0x45, 0x07, 0x72, 0x0d, 0xc5, 0x21, 0x9d,
+	0x04, 0x4d, 0xea, 0x60, 0x88, 0x80, 0x41, 0x2b,
+	0xfd, 0xff, 0xcf, 0x35, 0x57, 0x9e, 0x9b, 0x26
+};
+static const u8 enc_assoc097[] __initconst = {
+	0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce097[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
+};
+static const u8 enc_key097[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - edge case intermediate sums in poly1305 */
+static const u8 enc_input098[] __initconst = {
+	0x6d, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
+	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
+	0xa1, 0x61, 0xb5, 0xab, 0x04, 0x09, 0x00, 0x62,
+	0x9e, 0xfe, 0xff, 0x78, 0xd7, 0xd8, 0x6b, 0x45,
+	0x9f, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca,
+	0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64,
+	0xc6, 0xf8, 0x07, 0x8c, 0xc8, 0xef, 0x12, 0xa0,
+	0xff, 0x65, 0x7d, 0x6d, 0x08, 0xdb, 0x10, 0xb8,
+	0x47, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2,
+	0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73,
+	0x8e, 0xdc, 0x36, 0x6c, 0xd6, 0x97, 0x65, 0x6f,
+	0xca, 0x81, 0xfb, 0x13, 0x3c, 0xed, 0x79, 0xa1
+};
+static const u8 enc_output098[] __initconst = {
+	0x6d, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x26, 0xa3, 0x7f, 0xa2, 0xe8, 0x10, 0x26, 0x94,
+	0x5c, 0x39, 0xe9, 0xf2, 0xeb, 0xa8, 0x77, 0x02,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xa5, 0xf1, 0xcf, 0xf2, 0x46, 0xfa, 0x09, 0x66,
+	0x6e, 0x3b, 0xdf, 0x50, 0xb7, 0xf5, 0x44, 0xb3,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xa5, 0xf1, 0xcf, 0xf2, 0x46, 0xfa, 0x09, 0x66,
+	0x6e, 0x3b, 0xdf, 0x50, 0xb7, 0xf5, 0x44, 0xb3,
+	0x1e, 0x6b, 0xea, 0x63, 0x14, 0x54, 0x2e, 0x2e,
+	0xf9, 0xff, 0xcf, 0x45, 0x0b, 0x2e, 0x98, 0x2b
+};
+static const u8 enc_assoc098[] __initconst = {
+	0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce098[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
+};
+static const u8 enc_key098[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - edge case intermediate sums in poly1305 */
+static const u8 enc_input099[] __initconst = {
+	0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
+	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
+	0xfc, 0x01, 0xb8, 0x91, 0xe5, 0xf0, 0xf9, 0x12,
+	0x8d, 0x7d, 0x1c, 0x57, 0x91, 0x92, 0xb6, 0x98,
+	0x63, 0x41, 0x44, 0x15, 0xb6, 0x99, 0x68, 0x95,
+	0x9a, 0x72, 0x91, 0xb7, 0xa5, 0xaf, 0x13, 0x48,
+	0x60, 0xcd, 0x9e, 0xa1, 0x0c, 0x29, 0xa3, 0x66,
+	0x54, 0xe7, 0xa2, 0x8e, 0x76, 0x1b, 0xec, 0xd8
+};
+static const u8 enc_output099[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x7b, 0xc3, 0x72, 0x98, 0x09, 0xe9, 0xdf, 0xe4,
+	0x4f, 0xba, 0x0a, 0xdd, 0xad, 0xe2, 0xaa, 0xdf,
+	0x03, 0xc4, 0x56, 0xdf, 0x82, 0x3c, 0xb8, 0xa0,
+	0xc5, 0xb9, 0x00, 0xb3, 0xc9, 0x35, 0xb8, 0xd3,
+	0x03, 0xc4, 0x56, 0xdf, 0x82, 0x3c, 0xb8, 0xa0,
+	0xc5, 0xb9, 0x00, 0xb3, 0xc9, 0x35, 0xb8, 0xd3,
+	0xed, 0x20, 0x17, 0xc8, 0xdb, 0xa4, 0x77, 0x56,
+	0x29, 0x04, 0x9d, 0x78, 0x6e, 0x3b, 0xce, 0xb1
+};
+static const u8 enc_assoc099[] __initconst = {
+	0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce099[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
+};
+static const u8 enc_key099[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - edge case intermediate sums in poly1305 */
+static const u8 enc_input100[] __initconst = {
+	0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
+	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
+	0x6b, 0x6d, 0xc9, 0xd2, 0x1a, 0x81, 0x9e, 0x70,
+	0xb5, 0x77, 0xf4, 0x41, 0x37, 0xd3, 0xd6, 0xbd,
+	0x13, 0x35, 0xf5, 0xeb, 0x44, 0x49, 0x40, 0x77,
+	0xb2, 0x64, 0x49, 0xa5, 0x4b, 0x6c, 0x7c, 0x75,
+	0x10, 0xb9, 0x2f, 0x5f, 0xfe, 0xf9, 0x8b, 0x84,
+	0x7c, 0xf1, 0x7a, 0x9c, 0x98, 0xd8, 0x83, 0xe5
+};
+static const u8 enc_output100[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xec, 0xaf, 0x03, 0xdb, 0xf6, 0x98, 0xb8, 0x86,
+	0x77, 0xb0, 0xe2, 0xcb, 0x0b, 0xa3, 0xca, 0xfa,
+	0x73, 0xb0, 0xe7, 0x21, 0x70, 0xec, 0x90, 0x42,
+	0xed, 0xaf, 0xd8, 0xa1, 0x27, 0xf6, 0xd7, 0xee,
+	0x73, 0xb0, 0xe7, 0x21, 0x70, 0xec, 0x90, 0x42,
+	0xed, 0xaf, 0xd8, 0xa1, 0x27, 0xf6, 0xd7, 0xee,
+	0x07, 0x3f, 0x17, 0xcb, 0x67, 0x78, 0x64, 0x59,
+	0x25, 0x04, 0x9d, 0x88, 0x22, 0xcb, 0xca, 0xb6
+};
+static const u8 enc_assoc100[] __initconst = {
+	0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce100[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
+};
+static const u8 enc_key100[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - edge case intermediate sums in poly1305 */
+static const u8 enc_input101[] __initconst = {
+	0xff, 0xcb, 0x2b, 0x11, 0x06, 0xf8, 0x23, 0x4c,
+	0x5e, 0x99, 0xd4, 0xdb, 0x4c, 0x70, 0x48, 0xde,
+	0x32, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09,
+	0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8,
+	0x16, 0xe9, 0x88, 0x4a, 0x11, 0x4f, 0x0e, 0x92,
+	0x66, 0xce, 0xa3, 0x88, 0x5f, 0xe3, 0x6b, 0x9f,
+	0xd6, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39,
+	0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4,
+	0xce, 0xbe, 0xf5, 0xe9, 0x88, 0x5a, 0x80, 0xea,
+	0x76, 0xd9, 0x75, 0xc1, 0x44, 0xa4, 0x18, 0x88
+};
+static const u8 enc_output101[] __initconst = {
+	0xff, 0xa0, 0xfc, 0x3e, 0x80, 0x32, 0xc3, 0xd5,
+	0xfd, 0xb6, 0x2a, 0x11, 0xf0, 0x96, 0x30, 0x7d,
+	0xb5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x76, 0x6c, 0x9a, 0x80, 0x25, 0xea, 0xde, 0xa7,
+	0x39, 0x05, 0x32, 0x8c, 0x33, 0x79, 0xc0, 0x04,
+	0xb5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x76, 0x6c, 0x9a, 0x80, 0x25, 0xea, 0xde, 0xa7,
+	0x39, 0x05, 0x32, 0x8c, 0x33, 0x79, 0xc0, 0x04,
+	0x8b, 0x9b, 0xb4, 0xb4, 0x86, 0x12, 0x89, 0x65,
+	0x8c, 0x69, 0x6a, 0x83, 0x40, 0x15, 0x04, 0x05
+};
+static const u8 enc_assoc101[] __initconst = {
+	0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce101[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
+};
+static const u8 enc_key101[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - edge case intermediate sums in poly1305 */
+static const u8 enc_input102[] __initconst = {
+	0x6f, 0x9e, 0x70, 0xed, 0x3b, 0x8b, 0xac, 0xa0,
+	0x26, 0xe4, 0x6a, 0x5a, 0x09, 0x43, 0x15, 0x8d,
+	0x21, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09,
+	0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8,
+	0x0c, 0x61, 0x2c, 0x5e, 0x8d, 0x89, 0xa8, 0x73,
+	0xdb, 0xca, 0xad, 0x5b, 0x73, 0x46, 0x42, 0x9b,
+	0xc5, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39,
+	0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4,
+	0xd4, 0x36, 0x51, 0xfd, 0x14, 0x9c, 0x26, 0x0b,
+	0xcb, 0xdd, 0x7b, 0x12, 0x68, 0x01, 0x31, 0x8c
+};
+static const u8 enc_output102[] __initconst = {
+	0x6f, 0xf5, 0xa7, 0xc2, 0xbd, 0x41, 0x4c, 0x39,
+	0x85, 0xcb, 0x94, 0x90, 0xb5, 0xa5, 0x6d, 0x2e,
+	0xa6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x6c, 0xe4, 0x3e, 0x94, 0xb9, 0x2c, 0x78, 0x46,
+	0x84, 0x01, 0x3c, 0x5f, 0x1f, 0xdc, 0xe9, 0x00,
+	0xa6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x6c, 0xe4, 0x3e, 0x94, 0xb9, 0x2c, 0x78, 0x46,
+	0x84, 0x01, 0x3c, 0x5f, 0x1f, 0xdc, 0xe9, 0x00,
+	0x8b, 0x3b, 0xbd, 0x51, 0x64, 0x44, 0x59, 0x56,
+	0x8d, 0x81, 0xca, 0x1f, 0xa7, 0x2c, 0xe4, 0x04
+};
+static const u8 enc_assoc102[] __initconst = {
+	0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce102[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
+};
+static const u8 enc_key102[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - edge case intermediate sums in poly1305 */
+static const u8 enc_input103[] __initconst = {
+	0x41, 0x2b, 0x08, 0x0a, 0x3e, 0x19, 0xc1, 0x0d,
+	0x44, 0xa1, 0xaf, 0x1e, 0xab, 0xde, 0xb4, 0xce,
+	0x35, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09,
+	0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8,
+	0x6b, 0x83, 0x94, 0x33, 0x09, 0x21, 0x48, 0x6c,
+	0xa1, 0x1d, 0x29, 0x1c, 0x3e, 0x97, 0xee, 0x9a,
+	0xd1, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39,
+	0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4,
+	0xb3, 0xd4, 0xe9, 0x90, 0x90, 0x34, 0xc6, 0x14,
+	0xb1, 0x0a, 0xff, 0x55, 0x25, 0xd0, 0x9d, 0x8d
+};
+static const u8 enc_output103[] __initconst = {
+	0x41, 0x40, 0xdf, 0x25, 0xb8, 0xd3, 0x21, 0x94,
+	0xe7, 0x8e, 0x51, 0xd4, 0x17, 0x38, 0xcc, 0x6d,
+	0xb2, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x0b, 0x06, 0x86, 0xf9, 0x3d, 0x84, 0x98, 0x59,
+	0xfe, 0xd6, 0xb8, 0x18, 0x52, 0x0d, 0x45, 0x01,
+	0xb2, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x0b, 0x06, 0x86, 0xf9, 0x3d, 0x84, 0x98, 0x59,
+	0xfe, 0xd6, 0xb8, 0x18, 0x52, 0x0d, 0x45, 0x01,
+	0x86, 0xfb, 0xab, 0x2b, 0x4a, 0x94, 0xf4, 0x7a,
+	0xa5, 0x6f, 0x0a, 0xea, 0x65, 0xd1, 0x10, 0x08
+};
+static const u8 enc_assoc103[] __initconst = {
+	0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce103[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
+};
+static const u8 enc_key103[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - edge case intermediate sums in poly1305 */
+static const u8 enc_input104[] __initconst = {
+	0xb2, 0x47, 0xa7, 0x47, 0x23, 0x49, 0x1a, 0xac,
+	0xac, 0xaa, 0xd7, 0x09, 0xc9, 0x1e, 0x93, 0x2b,
+	0x31, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09,
+	0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8,
+	0x9a, 0xde, 0x04, 0xe7, 0x5b, 0xb7, 0x01, 0xd9,
+	0x66, 0x06, 0x01, 0xb3, 0x47, 0x65, 0xde, 0x98,
+	0xd5, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39,
+	0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4,
+	0x42, 0x89, 0x79, 0x44, 0xc2, 0xa2, 0x8f, 0xa1,
+	0x76, 0x11, 0xd7, 0xfa, 0x5c, 0x22, 0xad, 0x8f
+};
+static const u8 enc_output104[] __initconst = {
+	0xb2, 0x2c, 0x70, 0x68, 0xa5, 0x83, 0xfa, 0x35,
+	0x0f, 0x85, 0x29, 0xc3, 0x75, 0xf8, 0xeb, 0x88,
+	0xb6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xfa, 0x5b, 0x16, 0x2d, 0x6f, 0x12, 0xd1, 0xec,
+	0x39, 0xcd, 0x90, 0xb7, 0x2b, 0xff, 0x75, 0x03,
+	0xb6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xfa, 0x5b, 0x16, 0x2d, 0x6f, 0x12, 0xd1, 0xec,
+	0x39, 0xcd, 0x90, 0xb7, 0x2b, 0xff, 0x75, 0x03,
+	0xa0, 0x19, 0xac, 0x2e, 0xd6, 0x67, 0xe1, 0x7d,
+	0xa1, 0x6f, 0x0a, 0xfa, 0x19, 0x61, 0x0d, 0x0d
+};
+static const u8 enc_assoc104[] __initconst = {
+	0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce104[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
+};
+static const u8 enc_key104[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - edge case intermediate sums in poly1305 */
+static const u8 enc_input105[] __initconst = {
+	0x74, 0x0f, 0x9e, 0x49, 0xf6, 0x10, 0xef, 0xa5,
+	0x85, 0xb6, 0x59, 0xca, 0x6e, 0xd8, 0xb4, 0x99,
+	0x2d, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09,
+	0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8,
+	0x41, 0x2d, 0x96, 0xaf, 0xbe, 0x80, 0xec, 0x3e,
+	0x79, 0xd4, 0x51, 0xb0, 0x0a, 0x2d, 0xb2, 0x9a,
+	0xc9, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39,
+	0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4,
+	0x99, 0x7a, 0xeb, 0x0c, 0x27, 0x95, 0x62, 0x46,
+	0x69, 0xc3, 0x87, 0xf9, 0x11, 0x6a, 0xc1, 0x8d
+};
+static const u8 enc_output105[] __initconst = {
+	0x74, 0x64, 0x49, 0x66, 0x70, 0xda, 0x0f, 0x3c,
+	0x26, 0x99, 0xa7, 0x00, 0xd2, 0x3e, 0xcc, 0x3a,
+	0xaa, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x21, 0xa8, 0x84, 0x65, 0x8a, 0x25, 0x3c, 0x0b,
+	0x26, 0x1f, 0xc0, 0xb4, 0x66, 0xb7, 0x19, 0x01,
+	0xaa, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x21, 0xa8, 0x84, 0x65, 0x8a, 0x25, 0x3c, 0x0b,
+	0x26, 0x1f, 0xc0, 0xb4, 0x66, 0xb7, 0x19, 0x01,
+	0x73, 0x6e, 0x18, 0x18, 0x16, 0x96, 0xa5, 0x88,
+	0x9c, 0x31, 0x59, 0xfa, 0xab, 0xab, 0x20, 0xfd
+};
+static const u8 enc_assoc105[] __initconst = {
+	0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce105[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
+};
+static const u8 enc_key105[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - edge case intermediate sums in poly1305 */
+static const u8 enc_input106[] __initconst = {
+	0xad, 0xba, 0x5d, 0x10, 0x5b, 0xc8, 0xaa, 0x06,
+	0x2c, 0x23, 0x36, 0xcb, 0x88, 0x9d, 0xdb, 0xd5,
+	0x37, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09,
+	0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8,
+	0x17, 0x7c, 0x5f, 0xfe, 0x28, 0x75, 0xf4, 0x68,
+	0xf6, 0xc2, 0x96, 0x57, 0x48, 0xf3, 0x59, 0x9a,
+	0xd3, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39,
+	0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4,
+	0xcf, 0x2b, 0x22, 0x5d, 0xb1, 0x60, 0x7a, 0x10,
+	0xe6, 0xd5, 0x40, 0x1e, 0x53, 0xb4, 0x2a, 0x8d
+};
+static const u8 enc_output106[] __initconst = {
+	0xad, 0xd1, 0x8a, 0x3f, 0xdd, 0x02, 0x4a, 0x9f,
+	0x8f, 0x0c, 0xc8, 0x01, 0x34, 0x7b, 0xa3, 0x76,
+	0xb0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x77, 0xf9, 0x4d, 0x34, 0x1c, 0xd0, 0x24, 0x5d,
+	0xa9, 0x09, 0x07, 0x53, 0x24, 0x69, 0xf2, 0x01,
+	0xb0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x77, 0xf9, 0x4d, 0x34, 0x1c, 0xd0, 0x24, 0x5d,
+	0xa9, 0x09, 0x07, 0x53, 0x24, 0x69, 0xf2, 0x01,
+	0xba, 0xd5, 0x8f, 0x10, 0xa9, 0x1e, 0x6a, 0x88,
+	0x9a, 0xba, 0x32, 0xfd, 0x17, 0xd8, 0x33, 0x1a
+};
+static const u8 enc_assoc106[] __initconst = {
+	0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce106[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
+};
+static const u8 enc_key106[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - edge case intermediate sums in poly1305 */
+static const u8 enc_input107[] __initconst = {
+	0xfe, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
+	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
+	0xc0, 0x01, 0xed, 0xc5, 0xda, 0x44, 0x2e, 0x71,
+	0x9b, 0xce, 0x9a, 0xbe, 0x27, 0x3a, 0xf1, 0x44,
+	0xb4, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca,
+	0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64,
+	0x48, 0x02, 0x5f, 0x41, 0xfa, 0x4e, 0x33, 0x6c,
+	0x78, 0x69, 0x57, 0xa2, 0xa7, 0xc4, 0x93, 0x0a,
+	0x6c, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2,
+	0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73,
+	0x00, 0x26, 0x6e, 0xa1, 0xe4, 0x36, 0x44, 0xa3,
+	0x4d, 0x8d, 0xd1, 0xdc, 0x93, 0xf2, 0xfa, 0x13
+};
+static const u8 enc_output107[] __initconst = {
+	0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x47, 0xc3, 0x27, 0xcc, 0x36, 0x5d, 0x08, 0x87,
+	0x59, 0x09, 0x8c, 0x34, 0x1b, 0x4a, 0xed, 0x03,
+	0xd4, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x2b, 0x0b, 0x97, 0x3f, 0x74, 0x5b, 0x28, 0xaa,
+	0xe9, 0x37, 0xf5, 0x9f, 0x18, 0xea, 0xc7, 0x01,
+	0xd4, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x2b, 0x0b, 0x97, 0x3f, 0x74, 0x5b, 0x28, 0xaa,
+	0xe9, 0x37, 0xf5, 0x9f, 0x18, 0xea, 0xc7, 0x01,
+	0xd6, 0x8c, 0xe1, 0x74, 0x07, 0x9a, 0xdd, 0x02,
+	0x8d, 0xd0, 0x5c, 0xf8, 0x14, 0x63, 0x04, 0x88
+};
+static const u8 enc_assoc107[] __initconst = {
+	0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce107[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
+};
+static const u8 enc_key107[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - edge case intermediate sums in poly1305 */
+static const u8 enc_input108[] __initconst = {
+	0xb5, 0x13, 0xb0, 0x6a, 0xb9, 0xac, 0x14, 0x43,
+	0x5a, 0xcb, 0x8a, 0xa3, 0xa3, 0x7a, 0xfd, 0xb6,
+	0x54, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09,
+	0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8,
+	0x61, 0x95, 0x01, 0x93, 0xb1, 0xbf, 0x03, 0x11,
+	0xff, 0x11, 0x79, 0x89, 0xae, 0xd9, 0xa9, 0x99,
+	0xb0, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39,
+	0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4,
+	0xb9, 0xc2, 0x7c, 0x30, 0x28, 0xaa, 0x8d, 0x69,
+	0xef, 0x06, 0xaf, 0xc0, 0xb5, 0x9e, 0xda, 0x8e
+};
+static const u8 enc_output108[] __initconst = {
+	0xb5, 0x78, 0x67, 0x45, 0x3f, 0x66, 0xf4, 0xda,
+	0xf9, 0xe4, 0x74, 0x69, 0x1f, 0x9c, 0x85, 0x15,
+	0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x01, 0x10, 0x13, 0x59, 0x85, 0x1a, 0xd3, 0x24,
+	0xa0, 0xda, 0xe8, 0x8d, 0xc2, 0x43, 0x02, 0x02,
+	0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x01, 0x10, 0x13, 0x59, 0x85, 0x1a, 0xd3, 0x24,
+	0xa0, 0xda, 0xe8, 0x8d, 0xc2, 0x43, 0x02, 0x02,
+	0xaa, 0x48, 0xa3, 0x88, 0x7d, 0x4b, 0x05, 0x96,
+	0x99, 0xc2, 0xfd, 0xf9, 0xc6, 0x78, 0x7e, 0x0a
+};
+static const u8 enc_assoc108[] __initconst = {
+	0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce108[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
+};
+static const u8 enc_key108[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - edge case intermediate sums in poly1305 */
+static const u8 enc_input109[] __initconst = {
+	0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
+	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
+	0xd4, 0xf1, 0x09, 0xe8, 0x14, 0xce, 0xa8, 0x5a,
+	0x08, 0xc0, 0x11, 0xd8, 0x50, 0xdd, 0x1d, 0xcb,
+	0xcf, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca,
+	0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64,
+	0x53, 0x40, 0xb8, 0x5a, 0x9a, 0xa0, 0x82, 0x96,
+	0xb7, 0x7a, 0x5f, 0xc3, 0x96, 0x1f, 0x66, 0x0f,
+	0x17, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2,
+	0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73,
+	0x1b, 0x64, 0x89, 0xba, 0x84, 0xd8, 0xf5, 0x59,
+	0x82, 0x9e, 0xd9, 0xbd, 0xa2, 0x29, 0x0f, 0x16
+};
+static const u8 enc_output109[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x53, 0x33, 0xc3, 0xe1, 0xf8, 0xd7, 0x8e, 0xac,
+	0xca, 0x07, 0x07, 0x52, 0x6c, 0xad, 0x01, 0x8c,
+	0xaf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x30, 0x49, 0x70, 0x24, 0x14, 0xb5, 0x99, 0x50,
+	0x26, 0x24, 0xfd, 0xfe, 0x29, 0x31, 0x32, 0x04,
+	0xaf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x30, 0x49, 0x70, 0x24, 0x14, 0xb5, 0x99, 0x50,
+	0x26, 0x24, 0xfd, 0xfe, 0x29, 0x31, 0x32, 0x04,
+	0xb9, 0x36, 0xa8, 0x17, 0xf2, 0x21, 0x1a, 0xf1,
+	0x29, 0xe2, 0xcf, 0x16, 0x0f, 0xd4, 0x2b, 0xcb
+};
+static const u8 enc_assoc109[] __initconst = {
+	0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce109[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
+};
+static const u8 enc_key109[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - edge case intermediate sums in poly1305 */
+static const u8 enc_input110[] __initconst = {
+	0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
+	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
+	0xdf, 0x4c, 0x62, 0x03, 0x2d, 0x41, 0x19, 0xb5,
+	0x88, 0x47, 0x7e, 0x99, 0x92, 0x5a, 0x56, 0xd9,
+	0xd6, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca,
+	0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64,
+	0xfa, 0x84, 0xf0, 0x64, 0x55, 0x36, 0x42, 0x1b,
+	0x2b, 0xb9, 0x24, 0x6e, 0xc2, 0x19, 0xed, 0x0b,
+	0x0e, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2,
+	0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73,
+	0xb2, 0xa0, 0xc1, 0x84, 0x4b, 0x4e, 0x35, 0xd4,
+	0x1e, 0x5d, 0xa2, 0x10, 0xf6, 0x2f, 0x84, 0x12
+};
+static const u8 enc_output110[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x58, 0x8e, 0xa8, 0x0a, 0xc1, 0x58, 0x3f, 0x43,
+	0x4a, 0x80, 0x68, 0x13, 0xae, 0x2a, 0x4a, 0x9e,
+	0xb6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x99, 0x8d, 0x38, 0x1a, 0xdb, 0x23, 0x59, 0xdd,
+	0xba, 0xe7, 0x86, 0x53, 0x7d, 0x37, 0xb9, 0x00,
+	0xb6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x99, 0x8d, 0x38, 0x1a, 0xdb, 0x23, 0x59, 0xdd,
+	0xba, 0xe7, 0x86, 0x53, 0x7d, 0x37, 0xb9, 0x00,
+	0x9f, 0x7a, 0xc4, 0x35, 0x1f, 0x6b, 0x91, 0xe6,
+	0x30, 0x97, 0xa7, 0x13, 0x11, 0x5d, 0x05, 0xbe
+};
+static const u8 enc_assoc110[] __initconst = {
+	0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce110[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
+};
+static const u8 enc_key110[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - edge case intermediate sums in poly1305 */
+static const u8 enc_input111[] __initconst = {
+	0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
+	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
+	0x13, 0xf8, 0x0a, 0x00, 0x6d, 0xc1, 0xbb, 0xda,
+	0xd6, 0x39, 0xa9, 0x2f, 0xc7, 0xec, 0xa6, 0x55,
+	0xf7, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca,
+	0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64,
+	0x63, 0x48, 0xb8, 0xfd, 0x29, 0xbf, 0x96, 0xd5,
+	0x63, 0xa5, 0x17, 0xe2, 0x7d, 0x7b, 0xfc, 0x0f,
+	0x2f, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2,
+	0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73,
+	0x2b, 0x6c, 0x89, 0x1d, 0x37, 0xc7, 0xe1, 0x1a,
+	0x56, 0x41, 0x91, 0x9c, 0x49, 0x4d, 0x95, 0x16
+};
+static const u8 enc_output111[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x94, 0x3a, 0xc0, 0x09, 0x81, 0xd8, 0x9d, 0x2c,
+	0x14, 0xfe, 0xbf, 0xa5, 0xfb, 0x9c, 0xba, 0x12,
+	0x97, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x41, 0x70, 0x83, 0xa7, 0xaa, 0x8d, 0x13,
+	0xf2, 0xfb, 0xb5, 0xdf, 0xc2, 0x55, 0xa8, 0x04,
+	0x97, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x41, 0x70, 0x83, 0xa7, 0xaa, 0x8d, 0x13,
+	0xf2, 0xfb, 0xb5, 0xdf, 0xc2, 0x55, 0xa8, 0x04,
+	0x9a, 0x18, 0xa8, 0x28, 0x07, 0x02, 0x69, 0xf4,
+	0x47, 0x00, 0xd0, 0x09, 0xe7, 0x17, 0x1c, 0xc9
+};
+static const u8 enc_assoc111[] __initconst = {
+	0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce111[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
+};
+static const u8 enc_key111[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - edge case intermediate sums in poly1305 */
+static const u8 enc_input112[] __initconst = {
+	0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
+	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
+	0x82, 0xe5, 0x9b, 0x45, 0x82, 0x91, 0x50, 0x38,
+	0xf9, 0x33, 0x81, 0x1e, 0x65, 0x2d, 0xc6, 0x6a,
+	0xfc, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca,
+	0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64,
+	0xb6, 0x71, 0xc8, 0xca, 0xc2, 0x70, 0xc2, 0x65,
+	0xa0, 0xac, 0x2f, 0x53, 0x57, 0x99, 0x88, 0x0a,
+	0x24, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2,
+	0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73,
+	0xfe, 0x55, 0xf9, 0x2a, 0xdc, 0x08, 0xb5, 0xaa,
+	0x95, 0x48, 0xa9, 0x2d, 0x63, 0xaf, 0xe1, 0x13
+};
+static const u8 enc_output112[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x05, 0x27, 0x51, 0x4c, 0x6e, 0x88, 0x76, 0xce,
+	0x3b, 0xf4, 0x97, 0x94, 0x59, 0x5d, 0xda, 0x2d,
+	0x9c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xd5, 0x78, 0x00, 0xb4, 0x4c, 0x65, 0xd9, 0xa3,
+	0x31, 0xf2, 0x8d, 0x6e, 0xe8, 0xb7, 0xdc, 0x01,
+	0x9c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xd5, 0x78, 0x00, 0xb4, 0x4c, 0x65, 0xd9, 0xa3,
+	0x31, 0xf2, 0x8d, 0x6e, 0xe8, 0xb7, 0xdc, 0x01,
+	0xb4, 0x36, 0xa8, 0x2b, 0x93, 0xd5, 0x55, 0xf7,
+	0x43, 0x00, 0xd0, 0x19, 0x9b, 0xa7, 0x18, 0xce
+};
+static const u8 enc_assoc112[] __initconst = {
+	0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce112[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
+};
+static const u8 enc_key112[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - edge case intermediate sums in poly1305 */
+static const u8 enc_input113[] __initconst = {
+	0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
+	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
+	0xf1, 0xd1, 0x28, 0x87, 0xb7, 0x21, 0x69, 0x86,
+	0xa1, 0x2d, 0x79, 0x09, 0x8b, 0x6d, 0xe6, 0x0f,
+	0xc0, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca,
+	0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64,
+	0xa7, 0xc7, 0x58, 0x99, 0xf3, 0xe6, 0x0a, 0xf1,
+	0xfc, 0xb6, 0xc7, 0x30, 0x7d, 0x87, 0x59, 0x0f,
+	0x18, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2,
+	0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73,
+	0xef, 0xe3, 0x69, 0x79, 0xed, 0x9e, 0x7d, 0x3e,
+	0xc9, 0x52, 0x41, 0x4e, 0x49, 0xb1, 0x30, 0x16
+};
+static const u8 enc_output113[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x76, 0x13, 0xe2, 0x8e, 0x5b, 0x38, 0x4f, 0x70,
+	0x63, 0xea, 0x6f, 0x83, 0xb7, 0x1d, 0xfa, 0x48,
+	0xa0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xc4, 0xce, 0x90, 0xe7, 0x7d, 0xf3, 0x11, 0x37,
+	0x6d, 0xe8, 0x65, 0x0d, 0xc2, 0xa9, 0x0d, 0x04,
+	0xa0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xc4, 0xce, 0x90, 0xe7, 0x7d, 0xf3, 0x11, 0x37,
+	0x6d, 0xe8, 0x65, 0x0d, 0xc2, 0xa9, 0x0d, 0x04,
+	0xce, 0x54, 0xa8, 0x2e, 0x1f, 0xa9, 0x42, 0xfa,
+	0x3f, 0x00, 0xd0, 0x29, 0x4f, 0x37, 0x15, 0xd3
+};
+static const u8 enc_assoc113[] __initconst = {
+	0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce113[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
+};
+static const u8 enc_key113[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - edge case intermediate sums in poly1305 */
+static const u8 enc_input114[] __initconst = {
+	0xcb, 0xf1, 0xda, 0x9e, 0x0b, 0xa9, 0x37, 0x73,
+	0x74, 0xe6, 0x9e, 0x1c, 0x0e, 0x60, 0x0c, 0xfc,
+	0x34, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09,
+	0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8,
+	0xbe, 0x3f, 0xa6, 0x6b, 0x6c, 0xe7, 0x80, 0x8a,
+	0xa3, 0xe4, 0x59, 0x49, 0xf9, 0x44, 0x64, 0x9f,
+	0xd0, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39,
+	0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4,
+	0x66, 0x68, 0xdb, 0xc8, 0xf5, 0xf2, 0x0e, 0xf2,
+	0xb3, 0xf3, 0x8f, 0x00, 0xe2, 0x03, 0x17, 0x88
+};
+static const u8 enc_output114[] __initconst = {
+	0xcb, 0x9a, 0x0d, 0xb1, 0x8d, 0x63, 0xd7, 0xea,
+	0xd7, 0xc9, 0x60, 0xd6, 0xb2, 0x86, 0x74, 0x5f,
+	0xb3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xde, 0xba, 0xb4, 0xa1, 0x58, 0x42, 0x50, 0xbf,
+	0xfc, 0x2f, 0xc8, 0x4d, 0x95, 0xde, 0xcf, 0x04,
+	0xb3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xde, 0xba, 0xb4, 0xa1, 0x58, 0x42, 0x50, 0xbf,
+	0xfc, 0x2f, 0xc8, 0x4d, 0x95, 0xde, 0xcf, 0x04,
+	0x23, 0x83, 0xab, 0x0b, 0x79, 0x92, 0x05, 0x69,
+	0x9b, 0x51, 0x0a, 0xa7, 0x09, 0xbf, 0x31, 0xf1
+};
+static const u8 enc_assoc114[] __initconst = {
+	0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce114[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
+};
+static const u8 enc_key114[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - edge case intermediate sums in poly1305 */
+static const u8 enc_input115[] __initconst = {
+	0x8f, 0x27, 0x86, 0x94, 0xc4, 0xe9, 0xda, 0xeb,
+	0xd5, 0x8d, 0x3e, 0x5b, 0x96, 0x6e, 0x8b, 0x68,
+	0x42, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09,
+	0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8,
+	0x06, 0x53, 0xe7, 0xa3, 0x31, 0x71, 0x88, 0x33,
+	0xac, 0xc3, 0xb9, 0xad, 0xff, 0x1c, 0x31, 0x98,
+	0xa6, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39,
+	0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4,
+	0xde, 0x04, 0x9a, 0x00, 0xa8, 0x64, 0x06, 0x4b,
+	0xbc, 0xd4, 0x6f, 0xe4, 0xe4, 0x5b, 0x42, 0x8f
+};
+static const u8 enc_output115[] __initconst = {
+	0x8f, 0x4c, 0x51, 0xbb, 0x42, 0x23, 0x3a, 0x72,
+	0x76, 0xa2, 0xc0, 0x91, 0x2a, 0x88, 0xf3, 0xcb,
+	0xc5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x66, 0xd6, 0xf5, 0x69, 0x05, 0xd4, 0x58, 0x06,
+	0xf3, 0x08, 0x28, 0xa9, 0x93, 0x86, 0x9a, 0x03,
+	0xc5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x66, 0xd6, 0xf5, 0x69, 0x05, 0xd4, 0x58, 0x06,
+	0xf3, 0x08, 0x28, 0xa9, 0x93, 0x86, 0x9a, 0x03,
+	0x8b, 0xfb, 0xab, 0x17, 0xa9, 0xe0, 0xb8, 0x74,
+	0x8b, 0x51, 0x0a, 0xe7, 0xd9, 0xfd, 0x23, 0x05
+};
+static const u8 enc_assoc115[] __initconst = {
+	0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce115[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
+};
+static const u8 enc_key115[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - edge case intermediate sums in poly1305 */
+static const u8 enc_input116[] __initconst = {
+	0xd5, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
+	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
+	0x9a, 0x22, 0xd7, 0x0a, 0x48, 0xe2, 0x4f, 0xdd,
+	0xcd, 0xd4, 0x41, 0x9d, 0xe6, 0x4c, 0x8f, 0x44,
+	0xfc, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca,
+	0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64,
+	0x77, 0xb5, 0xc9, 0x07, 0xd9, 0xc9, 0xe1, 0xea,
+	0x51, 0x85, 0x1a, 0x20, 0x4a, 0xad, 0x9f, 0x0a,
+	0x24, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2,
+	0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73,
+	0x3f, 0x91, 0xf8, 0xe7, 0xc7, 0xb1, 0x96, 0x25,
+	0x64, 0x61, 0x9c, 0x5e, 0x7e, 0x9b, 0xf6, 0x13
+};
+static const u8 enc_output116[] __initconst = {
+	0xd5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x1d, 0xe0, 0x1d, 0x03, 0xa4, 0xfb, 0x69, 0x2b,
+	0x0f, 0x13, 0x57, 0x17, 0xda, 0x3c, 0x93, 0x03,
+	0x9c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x14, 0xbc, 0x01, 0x79, 0x57, 0xdc, 0xfa, 0x2c,
+	0xc0, 0xdb, 0xb8, 0x1d, 0xf5, 0x83, 0xcb, 0x01,
+	0x9c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x14, 0xbc, 0x01, 0x79, 0x57, 0xdc, 0xfa, 0x2c,
+	0xc0, 0xdb, 0xb8, 0x1d, 0xf5, 0x83, 0xcb, 0x01,
+	0x49, 0xbc, 0x6e, 0x9f, 0xc5, 0x1c, 0x4d, 0x50,
+	0x30, 0x36, 0x64, 0x4d, 0x84, 0x27, 0x73, 0xd2
+};
+static const u8 enc_assoc116[] __initconst = {
+	0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce116[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
+};
+static const u8 enc_key116[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - edge case intermediate sums in poly1305 */
+static const u8 enc_input117[] __initconst = {
+	0xdb, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
+	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
+	0x75, 0xd5, 0x64, 0x3a, 0xa5, 0xaf, 0x93, 0x4d,
+	0x8c, 0xce, 0x39, 0x2c, 0xc3, 0xee, 0xdb, 0x47,
+	0xc0, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca,
+	0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64,
+	0x60, 0x1b, 0x5a, 0xd2, 0x06, 0x7f, 0x28, 0x06,
+	0x6a, 0x8f, 0x32, 0x81, 0x71, 0x5b, 0xa8, 0x08,
+	0x18, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2,
+	0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73,
+	0x28, 0x3f, 0x6b, 0x32, 0x18, 0x07, 0x5f, 0xc9,
+	0x5f, 0x6b, 0xb4, 0xff, 0x45, 0x6d, 0xc1, 0x11
+};
+static const u8 enc_output117[] __initconst = {
+	0xdb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xf2, 0x17, 0xae, 0x33, 0x49, 0xb6, 0xb5, 0xbb,
+	0x4e, 0x09, 0x2f, 0xa6, 0xff, 0x9e, 0xc7, 0x00,
+	0xa0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x03, 0x12, 0x92, 0xac, 0x88, 0x6a, 0x33, 0xc0,
+	0xfb, 0xd1, 0x90, 0xbc, 0xce, 0x75, 0xfc, 0x03,
+	0xa0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x03, 0x12, 0x92, 0xac, 0x88, 0x6a, 0x33, 0xc0,
+	0xfb, 0xd1, 0x90, 0xbc, 0xce, 0x75, 0xfc, 0x03,
+	0x63, 0xda, 0x6e, 0xa2, 0x51, 0xf0, 0x39, 0x53,
+	0x2c, 0x36, 0x64, 0x5d, 0x38, 0xb7, 0x6f, 0xd7
+};
+static const u8 enc_assoc117[] __initconst = {
+	0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce117[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
+};
+static const u8 enc_key117[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+/* wycheproof - edge case intermediate sums in poly1305 */
+static const u8 enc_input118[] __initconst = {
+	0x93, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
+	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
+	0x62, 0x48, 0x39, 0x60, 0x42, 0x16, 0xe4, 0x03,
+	0xeb, 0xcc, 0x6a, 0xf5, 0x59, 0xec, 0x8b, 0x43,
+	0x97, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca,
+	0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64,
+	0xd8, 0xc8, 0xc3, 0xfa, 0x1a, 0x9e, 0x47, 0x4a,
+	0xbe, 0x52, 0xd0, 0x2c, 0x81, 0x87, 0xe9, 0x0f,
+	0x4f, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2,
+	0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73,
+	0x90, 0xec, 0xf2, 0x1a, 0x04, 0xe6, 0x30, 0x85,
+	0x8b, 0xb6, 0x56, 0x52, 0xb5, 0xb1, 0x80, 0x16
+};
+static const u8 enc_output118[] __initconst = {
+	0x93, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xe5, 0x8a, 0xf3, 0x69, 0xae, 0x0f, 0xc2, 0xf5,
+	0x29, 0x0b, 0x7c, 0x7f, 0x65, 0x9c, 0x97, 0x04,
+	0xf7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xbb, 0xc1, 0x0b, 0x84, 0x94, 0x8b, 0x5c, 0x8c,
+	0x2f, 0x0c, 0x72, 0x11, 0x3e, 0xa9, 0xbd, 0x04,
+	0xf7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xbb, 0xc1, 0x0b, 0x84, 0x94, 0x8b, 0x5c, 0x8c,
+	0x2f, 0x0c, 0x72, 0x11, 0x3e, 0xa9, 0xbd, 0x04,
+	0x73, 0xeb, 0x27, 0x24, 0xb5, 0xc4, 0x05, 0xf0,
+	0x4d, 0x00, 0xd0, 0xf1, 0x58, 0x40, 0xa1, 0xc1
+};
+static const u8 enc_assoc118[] __initconst = {
+	0xff, 0xff, 0xff, 0xff
+};
+static const u8 enc_nonce118[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
+};
+static const u8 enc_key118[] __initconst = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f
+};
+
+static const struct chacha20poly1305_testvec
+chacha20poly1305_enc_vectors[] __initconst = {
+	{ enc_input001, enc_output001, enc_assoc001, enc_nonce001, enc_key001,
+	  sizeof(enc_input001), sizeof(enc_assoc001), sizeof(enc_nonce001) },
+	{ enc_input002, enc_output002, enc_assoc002, enc_nonce002, enc_key002,
+	  sizeof(enc_input002), sizeof(enc_assoc002), sizeof(enc_nonce002) },
+	{ enc_input003, enc_output003, enc_assoc003, enc_nonce003, enc_key003,
+	  sizeof(enc_input003), sizeof(enc_assoc003), sizeof(enc_nonce003) },
+	{ enc_input004, enc_output004, enc_assoc004, enc_nonce004, enc_key004,
+	  sizeof(enc_input004), sizeof(enc_assoc004), sizeof(enc_nonce004) },
+	{ enc_input005, enc_output005, enc_assoc005, enc_nonce005, enc_key005,
+	  sizeof(enc_input005), sizeof(enc_assoc005), sizeof(enc_nonce005) },
+	{ enc_input006, enc_output006, enc_assoc006, enc_nonce006, enc_key006,
+	  sizeof(enc_input006), sizeof(enc_assoc006), sizeof(enc_nonce006) },
+	{ enc_input007, enc_output007, enc_assoc007, enc_nonce007, enc_key007,
+	  sizeof(enc_input007), sizeof(enc_assoc007), sizeof(enc_nonce007) },
+	{ enc_input008, enc_output008, enc_assoc008, enc_nonce008, enc_key008,
+	  sizeof(enc_input008), sizeof(enc_assoc008), sizeof(enc_nonce008) },
+	{ enc_input009, enc_output009, enc_assoc009, enc_nonce009, enc_key009,
+	  sizeof(enc_input009), sizeof(enc_assoc009), sizeof(enc_nonce009) },
+	{ enc_input010, enc_output010, enc_assoc010, enc_nonce010, enc_key010,
+	  sizeof(enc_input010), sizeof(enc_assoc010), sizeof(enc_nonce010) },
+	{ enc_input011, enc_output011, enc_assoc011, enc_nonce011, enc_key011,
+	  sizeof(enc_input011), sizeof(enc_assoc011), sizeof(enc_nonce011) },
+	{ enc_input012, enc_output012, enc_assoc012, enc_nonce012, enc_key012,
+	  sizeof(enc_input012), sizeof(enc_assoc012), sizeof(enc_nonce012) },
+	{ enc_input013, enc_output013, enc_assoc013, enc_nonce013, enc_key013,
+	  sizeof(enc_input013), sizeof(enc_assoc013), sizeof(enc_nonce013) },
+	{ enc_input014, enc_output014, enc_assoc014, enc_nonce014, enc_key014,
+	  sizeof(enc_input014), sizeof(enc_assoc014), sizeof(enc_nonce014) },
+	{ enc_input015, enc_output015, enc_assoc015, enc_nonce015, enc_key015,
+	  sizeof(enc_input015), sizeof(enc_assoc015), sizeof(enc_nonce015) },
+	{ enc_input016, enc_output016, enc_assoc016, enc_nonce016, enc_key016,
+	  sizeof(enc_input016), sizeof(enc_assoc016), sizeof(enc_nonce016) },
+	{ enc_input017, enc_output017, enc_assoc017, enc_nonce017, enc_key017,
+	  sizeof(enc_input017), sizeof(enc_assoc017), sizeof(enc_nonce017) },
+	{ enc_input018, enc_output018, enc_assoc018, enc_nonce018, enc_key018,
+	  sizeof(enc_input018), sizeof(enc_assoc018), sizeof(enc_nonce018) },
+	{ enc_input019, enc_output019, enc_assoc019, enc_nonce019, enc_key019,
+	  sizeof(enc_input019), sizeof(enc_assoc019), sizeof(enc_nonce019) },
+	{ enc_input020, enc_output020, enc_assoc020, enc_nonce020, enc_key020,
+	  sizeof(enc_input020), sizeof(enc_assoc020), sizeof(enc_nonce020) },
+	{ enc_input021, enc_output021, enc_assoc021, enc_nonce021, enc_key021,
+	  sizeof(enc_input021), sizeof(enc_assoc021), sizeof(enc_nonce021) },
+	{ enc_input022, enc_output022, enc_assoc022, enc_nonce022, enc_key022,
+	  sizeof(enc_input022), sizeof(enc_assoc022), sizeof(enc_nonce022) },
+	{ enc_input023, enc_output023, enc_assoc023, enc_nonce023, enc_key023,
+	  sizeof(enc_input023), sizeof(enc_assoc023), sizeof(enc_nonce023) },
+	{ enc_input024, enc_output024, enc_assoc024, enc_nonce024, enc_key024,
+	  sizeof(enc_input024), sizeof(enc_assoc024), sizeof(enc_nonce024) },
+	{ enc_input025, enc_output025, enc_assoc025, enc_nonce025, enc_key025,
+	  sizeof(enc_input025), sizeof(enc_assoc025), sizeof(enc_nonce025) },
+	{ enc_input026, enc_output026, enc_assoc026, enc_nonce026, enc_key026,
+	  sizeof(enc_input026), sizeof(enc_assoc026), sizeof(enc_nonce026) },
+	{ enc_input027, enc_output027, enc_assoc027, enc_nonce027, enc_key027,
+	  sizeof(enc_input027), sizeof(enc_assoc027), sizeof(enc_nonce027) },
+	{ enc_input028, enc_output028, enc_assoc028, enc_nonce028, enc_key028,
+	  sizeof(enc_input028), sizeof(enc_assoc028), sizeof(enc_nonce028) },
+	{ enc_input029, enc_output029, enc_assoc029, enc_nonce029, enc_key029,
+	  sizeof(enc_input029), sizeof(enc_assoc029), sizeof(enc_nonce029) },
+	{ enc_input030, enc_output030, enc_assoc030, enc_nonce030, enc_key030,
+	  sizeof(enc_input030), sizeof(enc_assoc030), sizeof(enc_nonce030) },
+	{ enc_input031, enc_output031, enc_assoc031, enc_nonce031, enc_key031,
+	  sizeof(enc_input031), sizeof(enc_assoc031), sizeof(enc_nonce031) },
+	{ enc_input032, enc_output032, enc_assoc032, enc_nonce032, enc_key032,
+	  sizeof(enc_input032), sizeof(enc_assoc032), sizeof(enc_nonce032) },
+	{ enc_input033, enc_output033, enc_assoc033, enc_nonce033, enc_key033,
+	  sizeof(enc_input033), sizeof(enc_assoc033), sizeof(enc_nonce033) },
+	{ enc_input034, enc_output034, enc_assoc034, enc_nonce034, enc_key034,
+	  sizeof(enc_input034), sizeof(enc_assoc034), sizeof(enc_nonce034) },
+	{ enc_input035, enc_output035, enc_assoc035, enc_nonce035, enc_key035,
+	  sizeof(enc_input035), sizeof(enc_assoc035), sizeof(enc_nonce035) },
+	{ enc_input036, enc_output036, enc_assoc036, enc_nonce036, enc_key036,
+	  sizeof(enc_input036), sizeof(enc_assoc036), sizeof(enc_nonce036) },
+	{ enc_input037, enc_output037, enc_assoc037, enc_nonce037, enc_key037,
+	  sizeof(enc_input037), sizeof(enc_assoc037), sizeof(enc_nonce037) },
+	{ enc_input038, enc_output038, enc_assoc038, enc_nonce038, enc_key038,
+	  sizeof(enc_input038), sizeof(enc_assoc038), sizeof(enc_nonce038) },
+	{ enc_input039, enc_output039, enc_assoc039, enc_nonce039, enc_key039,
+	  sizeof(enc_input039), sizeof(enc_assoc039), sizeof(enc_nonce039) },
+	{ enc_input040, enc_output040, enc_assoc040, enc_nonce040, enc_key040,
+	  sizeof(enc_input040), sizeof(enc_assoc040), sizeof(enc_nonce040) },
+	{ enc_input041, enc_output041, enc_assoc041, enc_nonce041, enc_key041,
+	  sizeof(enc_input041), sizeof(enc_assoc041), sizeof(enc_nonce041) },
+	{ enc_input042, enc_output042, enc_assoc042, enc_nonce042, enc_key042,
+	  sizeof(enc_input042), sizeof(enc_assoc042), sizeof(enc_nonce042) },
+	{ enc_input043, enc_output043, enc_assoc043, enc_nonce043, enc_key043,
+	  sizeof(enc_input043), sizeof(enc_assoc043), sizeof(enc_nonce043) },
+	{ enc_input044, enc_output044, enc_assoc044, enc_nonce044, enc_key044,
+	  sizeof(enc_input044), sizeof(enc_assoc044), sizeof(enc_nonce044) },
+	{ enc_input045, enc_output045, enc_assoc045, enc_nonce045, enc_key045,
+	  sizeof(enc_input045), sizeof(enc_assoc045), sizeof(enc_nonce045) },
+	{ enc_input046, enc_output046, enc_assoc046, enc_nonce046, enc_key046,
+	  sizeof(enc_input046), sizeof(enc_assoc046), sizeof(enc_nonce046) },
+	{ enc_input047, enc_output047, enc_assoc047, enc_nonce047, enc_key047,
+	  sizeof(enc_input047), sizeof(enc_assoc047), sizeof(enc_nonce047) },
+	{ enc_input048, enc_output048, enc_assoc048, enc_nonce048, enc_key048,
+	  sizeof(enc_input048), sizeof(enc_assoc048), sizeof(enc_nonce048) },
+	{ enc_input049, enc_output049, enc_assoc049, enc_nonce049, enc_key049,
+	  sizeof(enc_input049), sizeof(enc_assoc049), sizeof(enc_nonce049) },
+	{ enc_input050, enc_output050, enc_assoc050, enc_nonce050, enc_key050,
+	  sizeof(enc_input050), sizeof(enc_assoc050), sizeof(enc_nonce050) },
+	{ enc_input051, enc_output051, enc_assoc051, enc_nonce051, enc_key051,
+	  sizeof(enc_input051), sizeof(enc_assoc051), sizeof(enc_nonce051) },
+	{ enc_input052, enc_output052, enc_assoc052, enc_nonce052, enc_key052,
+	  sizeof(enc_input052), sizeof(enc_assoc052), sizeof(enc_nonce052) },
+	{ enc_input053, enc_output053, enc_assoc053, enc_nonce053, enc_key053,
+	  sizeof(enc_input053), sizeof(enc_assoc053), sizeof(enc_nonce053) },
+	{ enc_input054, enc_output054, enc_assoc054, enc_nonce054, enc_key054,
+	  sizeof(enc_input054), sizeof(enc_assoc054), sizeof(enc_nonce054) },
+	{ enc_input055, enc_output055, enc_assoc055, enc_nonce055, enc_key055,
+	  sizeof(enc_input055), sizeof(enc_assoc055), sizeof(enc_nonce055) },
+	{ enc_input056, enc_output056, enc_assoc056, enc_nonce056, enc_key056,
+	  sizeof(enc_input056), sizeof(enc_assoc056), sizeof(enc_nonce056) },
+	{ enc_input057, enc_output057, enc_assoc057, enc_nonce057, enc_key057,
+	  sizeof(enc_input057), sizeof(enc_assoc057), sizeof(enc_nonce057) },
+	{ enc_input058, enc_output058, enc_assoc058, enc_nonce058, enc_key058,
+	  sizeof(enc_input058), sizeof(enc_assoc058), sizeof(enc_nonce058) },
+	{ enc_input059, enc_output059, enc_assoc059, enc_nonce059, enc_key059,
+	  sizeof(enc_input059), sizeof(enc_assoc059), sizeof(enc_nonce059) },
+	{ enc_input060, enc_output060, enc_assoc060, enc_nonce060, enc_key060,
+	  sizeof(enc_input060), sizeof(enc_assoc060), sizeof(enc_nonce060) },
+	{ enc_input061, enc_output061, enc_assoc061, enc_nonce061, enc_key061,
+	  sizeof(enc_input061), sizeof(enc_assoc061), sizeof(enc_nonce061) },
+	{ enc_input062, enc_output062, enc_assoc062, enc_nonce062, enc_key062,
+	  sizeof(enc_input062), sizeof(enc_assoc062), sizeof(enc_nonce062) },
+	{ enc_input063, enc_output063, enc_assoc063, enc_nonce063, enc_key063,
+	  sizeof(enc_input063), sizeof(enc_assoc063), sizeof(enc_nonce063) },
+	{ enc_input064, enc_output064, enc_assoc064, enc_nonce064, enc_key064,
+	  sizeof(enc_input064), sizeof(enc_assoc064), sizeof(enc_nonce064) },
+	{ enc_input065, enc_output065, enc_assoc065, enc_nonce065, enc_key065,
+	  sizeof(enc_input065), sizeof(enc_assoc065), sizeof(enc_nonce065) },
+	{ enc_input066, enc_output066, enc_assoc066, enc_nonce066, enc_key066,
+	  sizeof(enc_input066), sizeof(enc_assoc066), sizeof(enc_nonce066) },
+	{ enc_input067, enc_output067, enc_assoc067, enc_nonce067, enc_key067,
+	  sizeof(enc_input067), sizeof(enc_assoc067), sizeof(enc_nonce067) },
+	{ enc_input068, enc_output068, enc_assoc068, enc_nonce068, enc_key068,
+	  sizeof(enc_input068), sizeof(enc_assoc068), sizeof(enc_nonce068) },
+	{ enc_input069, enc_output069, enc_assoc069, enc_nonce069, enc_key069,
+	  sizeof(enc_input069), sizeof(enc_assoc069), sizeof(enc_nonce069) },
+	{ enc_input070, enc_output070, enc_assoc070, enc_nonce070, enc_key070,
+	  sizeof(enc_input070), sizeof(enc_assoc070), sizeof(enc_nonce070) },
+	{ enc_input071, enc_output071, enc_assoc071, enc_nonce071, enc_key071,
+	  sizeof(enc_input071), sizeof(enc_assoc071), sizeof(enc_nonce071) },
+	{ enc_input072, enc_output072, enc_assoc072, enc_nonce072, enc_key072,
+	  sizeof(enc_input072), sizeof(enc_assoc072), sizeof(enc_nonce072) },
+	{ enc_input073, enc_output073, enc_assoc073, enc_nonce073, enc_key073,
+	  sizeof(enc_input073), sizeof(enc_assoc073), sizeof(enc_nonce073) },
+	{ enc_input074, enc_output074, enc_assoc074, enc_nonce074, enc_key074,
+	  sizeof(enc_input074), sizeof(enc_assoc074), sizeof(enc_nonce074) },
+	{ enc_input075, enc_output075, enc_assoc075, enc_nonce075, enc_key075,
+	  sizeof(enc_input075), sizeof(enc_assoc075), sizeof(enc_nonce075) },
+	{ enc_input076, enc_output076, enc_assoc076, enc_nonce076, enc_key076,
+	  sizeof(enc_input076), sizeof(enc_assoc076), sizeof(enc_nonce076) },
+	{ enc_input077, enc_output077, enc_assoc077, enc_nonce077, enc_key077,
+	  sizeof(enc_input077), sizeof(enc_assoc077), sizeof(enc_nonce077) },
+	{ enc_input078, enc_output078, enc_assoc078, enc_nonce078, enc_key078,
+	  sizeof(enc_input078), sizeof(enc_assoc078), sizeof(enc_nonce078) },
+	{ enc_input079, enc_output079, enc_assoc079, enc_nonce079, enc_key079,
+	  sizeof(enc_input079), sizeof(enc_assoc079), sizeof(enc_nonce079) },
+	{ enc_input080, enc_output080, enc_assoc080, enc_nonce080, enc_key080,
+	  sizeof(enc_input080), sizeof(enc_assoc080), sizeof(enc_nonce080) },
+	{ enc_input081, enc_output081, enc_assoc081, enc_nonce081, enc_key081,
+	  sizeof(enc_input081), sizeof(enc_assoc081), sizeof(enc_nonce081) },
+	{ enc_input082, enc_output082, enc_assoc082, enc_nonce082, enc_key082,
+	  sizeof(enc_input082), sizeof(enc_assoc082), sizeof(enc_nonce082) },
+	{ enc_input083, enc_output083, enc_assoc083, enc_nonce083, enc_key083,
+	  sizeof(enc_input083), sizeof(enc_assoc083), sizeof(enc_nonce083) },
+	{ enc_input084, enc_output084, enc_assoc084, enc_nonce084, enc_key084,
+	  sizeof(enc_input084), sizeof(enc_assoc084), sizeof(enc_nonce084) },
+	{ enc_input085, enc_output085, enc_assoc085, enc_nonce085, enc_key085,
+	  sizeof(enc_input085), sizeof(enc_assoc085), sizeof(enc_nonce085) },
+	{ enc_input086, enc_output086, enc_assoc086, enc_nonce086, enc_key086,
+	  sizeof(enc_input086), sizeof(enc_assoc086), sizeof(enc_nonce086) },
+	{ enc_input087, enc_output087, enc_assoc087, enc_nonce087, enc_key087,
+	  sizeof(enc_input087), sizeof(enc_assoc087), sizeof(enc_nonce087) },
+	{ enc_input088, enc_output088, enc_assoc088, enc_nonce088, enc_key088,
+	  sizeof(enc_input088), sizeof(enc_assoc088), sizeof(enc_nonce088) },
+	{ enc_input089, enc_output089, enc_assoc089, enc_nonce089, enc_key089,
+	  sizeof(enc_input089), sizeof(enc_assoc089), sizeof(enc_nonce089) },
+	{ enc_input090, enc_output090, enc_assoc090, enc_nonce090, enc_key090,
+	  sizeof(enc_input090), sizeof(enc_assoc090), sizeof(enc_nonce090) },
+	{ enc_input091, enc_output091, enc_assoc091, enc_nonce091, enc_key091,
+	  sizeof(enc_input091), sizeof(enc_assoc091), sizeof(enc_nonce091) },
+	{ enc_input092, enc_output092, enc_assoc092, enc_nonce092, enc_key092,
+	  sizeof(enc_input092), sizeof(enc_assoc092), sizeof(enc_nonce092) },
+	{ enc_input093, enc_output093, enc_assoc093, enc_nonce093, enc_key093,
+	  sizeof(enc_input093), sizeof(enc_assoc093), sizeof(enc_nonce093) },
+	{ enc_input094, enc_output094, enc_assoc094, enc_nonce094, enc_key094,
+	  sizeof(enc_input094), sizeof(enc_assoc094), sizeof(enc_nonce094) },
+	{ enc_input095, enc_output095, enc_assoc095, enc_nonce095, enc_key095,
+	  sizeof(enc_input095), sizeof(enc_assoc095), sizeof(enc_nonce095) },
+	{ enc_input096, enc_output096, enc_assoc096, enc_nonce096, enc_key096,
+	  sizeof(enc_input096), sizeof(enc_assoc096), sizeof(enc_nonce096) },
+	{ enc_input097, enc_output097, enc_assoc097, enc_nonce097, enc_key097,
+	  sizeof(enc_input097), sizeof(enc_assoc097), sizeof(enc_nonce097) },
+	{ enc_input098, enc_output098, enc_assoc098, enc_nonce098, enc_key098,
+	  sizeof(enc_input098), sizeof(enc_assoc098), sizeof(enc_nonce098) },
+	{ enc_input099, enc_output099, enc_assoc099, enc_nonce099, enc_key099,
+	  sizeof(enc_input099), sizeof(enc_assoc099), sizeof(enc_nonce099) },
+	{ enc_input100, enc_output100, enc_assoc100, enc_nonce100, enc_key100,
+	  sizeof(enc_input100), sizeof(enc_assoc100), sizeof(enc_nonce100) },
+	{ enc_input101, enc_output101, enc_assoc101, enc_nonce101, enc_key101,
+	  sizeof(enc_input101), sizeof(enc_assoc101), sizeof(enc_nonce101) },
+	{ enc_input102, enc_output102, enc_assoc102, enc_nonce102, enc_key102,
+	  sizeof(enc_input102), sizeof(enc_assoc102), sizeof(enc_nonce102) },
+	{ enc_input103, enc_output103, enc_assoc103, enc_nonce103, enc_key103,
+	  sizeof(enc_input103), sizeof(enc_assoc103), sizeof(enc_nonce103) },
+	{ enc_input104, enc_output104, enc_assoc104, enc_nonce104, enc_key104,
+	  sizeof(enc_input104), sizeof(enc_assoc104), sizeof(enc_nonce104) },
+	{ enc_input105, enc_output105, enc_assoc105, enc_nonce105, enc_key105,
+	  sizeof(enc_input105), sizeof(enc_assoc105), sizeof(enc_nonce105) },
+	{ enc_input106, enc_output106, enc_assoc106, enc_nonce106, enc_key106,
+	  sizeof(enc_input106), sizeof(enc_assoc106), sizeof(enc_nonce106) },
+	{ enc_input107, enc_output107, enc_assoc107, enc_nonce107, enc_key107,
+	  sizeof(enc_input107), sizeof(enc_assoc107), sizeof(enc_nonce107) },
+	{ enc_input108, enc_output108, enc_assoc108, enc_nonce108, enc_key108,
+	  sizeof(enc_input108), sizeof(enc_assoc108), sizeof(enc_nonce108) },
+	{ enc_input109, enc_output109, enc_assoc109, enc_nonce109, enc_key109,
+	  sizeof(enc_input109), sizeof(enc_assoc109), sizeof(enc_nonce109) },
+	{ enc_input110, enc_output110, enc_assoc110, enc_nonce110, enc_key110,
+	  sizeof(enc_input110), sizeof(enc_assoc110), sizeof(enc_nonce110) },
+	{ enc_input111, enc_output111, enc_assoc111, enc_nonce111, enc_key111,
+	  sizeof(enc_input111), sizeof(enc_assoc111), sizeof(enc_nonce111) },
+	{ enc_input112, enc_output112, enc_assoc112, enc_nonce112, enc_key112,
+	  sizeof(enc_input112), sizeof(enc_assoc112), sizeof(enc_nonce112) },
+	{ enc_input113, enc_output113, enc_assoc113, enc_nonce113, enc_key113,
+	  sizeof(enc_input113), sizeof(enc_assoc113), sizeof(enc_nonce113) },
+	{ enc_input114, enc_output114, enc_assoc114, enc_nonce114, enc_key114,
+	  sizeof(enc_input114), sizeof(enc_assoc114), sizeof(enc_nonce114) },
+	{ enc_input115, enc_output115, enc_assoc115, enc_nonce115, enc_key115,
+	  sizeof(enc_input115), sizeof(enc_assoc115), sizeof(enc_nonce115) },
+	{ enc_input116, enc_output116, enc_assoc116, enc_nonce116, enc_key116,
+	  sizeof(enc_input116), sizeof(enc_assoc116), sizeof(enc_nonce116) },
+	{ enc_input117, enc_output117, enc_assoc117, enc_nonce117, enc_key117,
+	  sizeof(enc_input117), sizeof(enc_assoc117), sizeof(enc_nonce117) },
+	{ enc_input118, enc_output118, enc_assoc118, enc_nonce118, enc_key118,
+	  sizeof(enc_input118), sizeof(enc_assoc118), sizeof(enc_nonce118) }
+};
+
+static const u8 dec_input001[] __initconst = {
+	0x64, 0xa0, 0x86, 0x15, 0x75, 0x86, 0x1a, 0xf4,
+	0x60, 0xf0, 0x62, 0xc7, 0x9b, 0xe6, 0x43, 0xbd,
+	0x5e, 0x80, 0x5c, 0xfd, 0x34, 0x5c, 0xf3, 0x89,
+	0xf1, 0x08, 0x67, 0x0a, 0xc7, 0x6c, 0x8c, 0xb2,
+	0x4c, 0x6c, 0xfc, 0x18, 0x75, 0x5d, 0x43, 0xee,
+	0xa0, 0x9e, 0xe9, 0x4e, 0x38, 0x2d, 0x26, 0xb0,
+	0xbd, 0xb7, 0xb7, 0x3c, 0x32, 0x1b, 0x01, 0x00,
+	0xd4, 0xf0, 0x3b, 0x7f, 0x35, 0x58, 0x94, 0xcf,
+	0x33, 0x2f, 0x83, 0x0e, 0x71, 0x0b, 0x97, 0xce,
+	0x98, 0xc8, 0xa8, 0x4a, 0xbd, 0x0b, 0x94, 0x81,
+	0x14, 0xad, 0x17, 0x6e, 0x00, 0x8d, 0x33, 0xbd,
+	0x60, 0xf9, 0x82, 0xb1, 0xff, 0x37, 0xc8, 0x55,
+	0x97, 0x97, 0xa0, 0x6e, 0xf4, 0xf0, 0xef, 0x61,
+	0xc1, 0x86, 0x32, 0x4e, 0x2b, 0x35, 0x06, 0x38,
+	0x36, 0x06, 0x90, 0x7b, 0x6a, 0x7c, 0x02, 0xb0,
+	0xf9, 0xf6, 0x15, 0x7b, 0x53, 0xc8, 0x67, 0xe4,
+	0xb9, 0x16, 0x6c, 0x76, 0x7b, 0x80, 0x4d, 0x46,
+	0xa5, 0x9b, 0x52, 0x16, 0xcd, 0xe7, 0xa4, 0xe9,
+	0x90, 0x40, 0xc5, 0xa4, 0x04, 0x33, 0x22, 0x5e,
+	0xe2, 0x82, 0xa1, 0xb0, 0xa0, 0x6c, 0x52, 0x3e,
+	0xaf, 0x45, 0x34, 0xd7, 0xf8, 0x3f, 0xa1, 0x15,
+	0x5b, 0x00, 0x47, 0x71, 0x8c, 0xbc, 0x54, 0x6a,
+	0x0d, 0x07, 0x2b, 0x04, 0xb3, 0x56, 0x4e, 0xea,
+	0x1b, 0x42, 0x22, 0x73, 0xf5, 0x48, 0x27, 0x1a,
+	0x0b, 0xb2, 0x31, 0x60, 0x53, 0xfa, 0x76, 0x99,
+	0x19, 0x55, 0xeb, 0xd6, 0x31, 0x59, 0x43, 0x4e,
+	0xce, 0xbb, 0x4e, 0x46, 0x6d, 0xae, 0x5a, 0x10,
+	0x73, 0xa6, 0x72, 0x76, 0x27, 0x09, 0x7a, 0x10,
+	0x49, 0xe6, 0x17, 0xd9, 0x1d, 0x36, 0x10, 0x94,
+	0xfa, 0x68, 0xf0, 0xff, 0x77, 0x98, 0x71, 0x30,
+	0x30, 0x5b, 0xea, 0xba, 0x2e, 0xda, 0x04, 0xdf,
+	0x99, 0x7b, 0x71, 0x4d, 0x6c, 0x6f, 0x2c, 0x29,
+	0xa6, 0xad, 0x5c, 0xb4, 0x02, 0x2b, 0x02, 0x70,
+	0x9b, 0xee, 0xad, 0x9d, 0x67, 0x89, 0x0c, 0xbb,
+	0x22, 0x39, 0x23, 0x36, 0xfe, 0xa1, 0x85, 0x1f,
+	0x38
+};
+static const u8 dec_output001[] __initconst = {
+	0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74,
+	0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20,
+	0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66,
+	0x74, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65,
+	0x6e, 0x74, 0x73, 0x20, 0x76, 0x61, 0x6c, 0x69,
+	0x64, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x20,
+	0x6d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x20,
+	0x6f, 0x66, 0x20, 0x73, 0x69, 0x78, 0x20, 0x6d,
+	0x6f, 0x6e, 0x74, 0x68, 0x73, 0x20, 0x61, 0x6e,
+	0x64, 0x20, 0x6d, 0x61, 0x79, 0x20, 0x62, 0x65,
+	0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64,
+	0x2c, 0x20, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63,
+	0x65, 0x64, 0x2c, 0x20, 0x6f, 0x72, 0x20, 0x6f,
+	0x62, 0x73, 0x6f, 0x6c, 0x65, 0x74, 0x65, 0x64,
+	0x20, 0x62, 0x79, 0x20, 0x6f, 0x74, 0x68, 0x65,
+	0x72, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65,
+	0x6e, 0x74, 0x73, 0x20, 0x61, 0x74, 0x20, 0x61,
+	0x6e, 0x79, 0x20, 0x74, 0x69, 0x6d, 0x65, 0x2e,
+	0x20, 0x49, 0x74, 0x20, 0x69, 0x73, 0x20, 0x69,
+	0x6e, 0x61, 0x70, 0x70, 0x72, 0x6f, 0x70, 0x72,
+	0x69, 0x61, 0x74, 0x65, 0x20, 0x74, 0x6f, 0x20,
+	0x75, 0x73, 0x65, 0x20, 0x49, 0x6e, 0x74, 0x65,
+	0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61,
+	0x66, 0x74, 0x73, 0x20, 0x61, 0x73, 0x20, 0x72,
+	0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65,
+	0x20, 0x6d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61,
+	0x6c, 0x20, 0x6f, 0x72, 0x20, 0x74, 0x6f, 0x20,
+	0x63, 0x69, 0x74, 0x65, 0x20, 0x74, 0x68, 0x65,
+	0x6d, 0x20, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x20,
+	0x74, 0x68, 0x61, 0x6e, 0x20, 0x61, 0x73, 0x20,
+	0x2f, 0xe2, 0x80, 0x9c, 0x77, 0x6f, 0x72, 0x6b,
+	0x20, 0x69, 0x6e, 0x20, 0x70, 0x72, 0x6f, 0x67,
+	0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80,
+	0x9d
+};
+static const u8 dec_assoc001[] __initconst = {
+	0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x4e, 0x91
+};
+static const u8 dec_nonce001[] __initconst = {
+	0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08
+};
+static const u8 dec_key001[] __initconst = {
+	0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a,
+	0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0,
+	0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09,
+	0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0
+};
+
+static const u8 dec_input002[] __initconst = {
+	0xea, 0xe0, 0x1e, 0x9e, 0x2c, 0x91, 0xaa, 0xe1,
+	0xdb, 0x5d, 0x99, 0x3f, 0x8a, 0xf7, 0x69, 0x92
+};
+static const u8 dec_output002[] __initconst = { };
+static const u8 dec_assoc002[] __initconst = { };
+static const u8 dec_nonce002[] __initconst = {
+	0xca, 0xbf, 0x33, 0x71, 0x32, 0x45, 0x77, 0x8e
+};
+static const u8 dec_key002[] __initconst = {
+	0x4c, 0xf5, 0x96, 0x83, 0x38, 0xe6, 0xae, 0x7f,
+	0x2d, 0x29, 0x25, 0x76, 0xd5, 0x75, 0x27, 0x86,
+	0x91, 0x9a, 0x27, 0x7a, 0xfb, 0x46, 0xc5, 0xef,
+	0x94, 0x81, 0x79, 0x57, 0x14, 0x59, 0x40, 0x68
+};
+
+static const u8 dec_input003[] __initconst = {
+	0xdd, 0x6b, 0x3b, 0x82, 0xce, 0x5a, 0xbd, 0xd6,
+	0xa9, 0x35, 0x83, 0xd8, 0x8c, 0x3d, 0x85, 0x77
+};
+static const u8 dec_output003[] __initconst = { };
+static const u8 dec_assoc003[] __initconst = {
+	0x33, 0x10, 0x41, 0x12, 0x1f, 0xf3, 0xd2, 0x6b
+};
+static const u8 dec_nonce003[] __initconst = {
+	0x3d, 0x86, 0xb5, 0x6b, 0xc8, 0xa3, 0x1f, 0x1d
+};
+static const u8 dec_key003[] __initconst = {
+	0x2d, 0xb0, 0x5d, 0x40, 0xc8, 0xed, 0x44, 0x88,
+	0x34, 0xd1, 0x13, 0xaf, 0x57, 0xa1, 0xeb, 0x3a,
+	0x2a, 0x80, 0x51, 0x36, 0xec, 0x5b, 0xbc, 0x08,
+	0x93, 0x84, 0x21, 0xb5, 0x13, 0x88, 0x3c, 0x0d
+};
+
+static const u8 dec_input004[] __initconst = {
+	0xb7, 0x1b, 0xb0, 0x73, 0x59, 0xb0, 0x84, 0xb2,
+	0x6d, 0x8e, 0xab, 0x94, 0x31, 0xa1, 0xae, 0xac,
+	0x89
+};
+static const u8 dec_output004[] __initconst = {
+	0xa4
+};
+static const u8 dec_assoc004[] __initconst = {
+	0x6a, 0xe2, 0xad, 0x3f, 0x88, 0x39, 0x5a, 0x40
+};
+static const u8 dec_nonce004[] __initconst = {
+	0xd2, 0x32, 0x1f, 0x29, 0x28, 0xc6, 0xc4, 0xc4
+};
+static const u8 dec_key004[] __initconst = {
+	0x4b, 0x28, 0x4b, 0xa3, 0x7b, 0xbe, 0xe9, 0xf8,
+	0x31, 0x80, 0x82, 0xd7, 0xd8, 0xe8, 0xb5, 0xa1,
+	0xe2, 0x18, 0x18, 0x8a, 0x9c, 0xfa, 0xa3, 0x3d,
+	0x25, 0x71, 0x3e, 0x40, 0xbc, 0x54, 0x7a, 0x3e
+};
+
+static const u8 dec_input005[] __initconst = {
+	0xbf, 0xe1, 0x5b, 0x0b, 0xdb, 0x6b, 0xf5, 0x5e,
+	0x6c, 0x5d, 0x84, 0x44, 0x39, 0x81, 0xc1, 0x9c,
+	0xac
+};
+static const u8 dec_output005[] __initconst = {
+	0x2d
+};
+static const u8 dec_assoc005[] __initconst = { };
+static const u8 dec_nonce005[] __initconst = {
+	0x20, 0x1c, 0xaa, 0x5f, 0x9c, 0xbf, 0x92, 0x30
+};
+static const u8 dec_key005[] __initconst = {
+	0x66, 0xca, 0x9c, 0x23, 0x2a, 0x4b, 0x4b, 0x31,
+	0x0e, 0x92, 0x89, 0x8b, 0xf4, 0x93, 0xc7, 0x87,
+	0x98, 0xa3, 0xd8, 0x39, 0xf8, 0xf4, 0xa7, 0x01,
+	0xc0, 0x2e, 0x0a, 0xa6, 0x7e, 0x5a, 0x78, 0x87
+};
+
+static const u8 dec_input006[] __initconst = {
+	0x8b, 0x06, 0xd3, 0x31, 0xb0, 0x93, 0x45, 0xb1,
+	0x75, 0x6e, 0x26, 0xf9, 0x67, 0xbc, 0x90, 0x15,
+	0x81, 0x2c, 0xb5, 0xf0, 0xc6, 0x2b, 0xc7, 0x8c,
+	0x56, 0xd1, 0xbf, 0x69, 0x6c, 0x07, 0xa0, 0xda,
+	0x65, 0x27, 0xc9, 0x90, 0x3d, 0xef, 0x4b, 0x11,
+	0x0f, 0x19, 0x07, 0xfd, 0x29, 0x92, 0xd9, 0xc8,
+	0xf7, 0x99, 0x2e, 0x4a, 0xd0, 0xb8, 0x2c, 0xdc,
+	0x93, 0xf5, 0x9e, 0x33, 0x78, 0xd1, 0x37, 0xc3,
+	0x66, 0xd7, 0x5e, 0xbc, 0x44, 0xbf, 0x53, 0xa5,
+	0xbc, 0xc4, 0xcb, 0x7b, 0x3a, 0x8e, 0x7f, 0x02,
+	0xbd, 0xbb, 0xe7, 0xca, 0xa6, 0x6c, 0x6b, 0x93,
+	0x21, 0x93, 0x10, 0x61, 0xe7, 0x69, 0xd0, 0x78,
+	0xf3, 0x07, 0x5a, 0x1a, 0x8f, 0x73, 0xaa, 0xb1,
+	0x4e, 0xd3, 0xda, 0x4f, 0xf3, 0x32, 0xe1, 0x66,
+	0x3e, 0x6c, 0xc6, 0x13, 0xba, 0x06, 0x5b, 0xfc,
+	0x6a, 0xe5, 0x6f, 0x60, 0xfb, 0x07, 0x40, 0xb0,
+	0x8c, 0x9d, 0x84, 0x43, 0x6b, 0xc1, 0xf7, 0x8d,
+	0x8d, 0x31, 0xf7, 0x7a, 0x39, 0x4d, 0x8f, 0x9a,
+	0xeb
+};
+static const u8 dec_output006[] __initconst = {
+	0x33, 0x2f, 0x94, 0xc1, 0xa4, 0xef, 0xcc, 0x2a,
+	0x5b, 0xa6, 0xe5, 0x8f, 0x1d, 0x40, 0xf0, 0x92,
+	0x3c, 0xd9, 0x24, 0x11, 0xa9, 0x71, 0xf9, 0x37,
+	0x14, 0x99, 0xfa, 0xbe, 0xe6, 0x80, 0xde, 0x50,
+	0xc9, 0x96, 0xd4, 0xb0, 0xec, 0x9e, 0x17, 0xec,
+	0xd2, 0x5e, 0x72, 0x99, 0xfc, 0x0a, 0xe1, 0xcb,
+	0x48, 0xd2, 0x85, 0xdd, 0x2f, 0x90, 0xe0, 0x66,
+	0x3b, 0xe6, 0x20, 0x74, 0xbe, 0x23, 0x8f, 0xcb,
+	0xb4, 0xe4, 0xda, 0x48, 0x40, 0xa6, 0xd1, 0x1b,
+	0xc7, 0x42, 0xce, 0x2f, 0x0c, 0xa6, 0x85, 0x6e,
+	0x87, 0x37, 0x03, 0xb1, 0x7c, 0x25, 0x96, 0xa3,
+	0x05, 0xd8, 0xb0, 0xf4, 0xed, 0xea, 0xc2, 0xf0,
+	0x31, 0x98, 0x6c, 0xd1, 0x14, 0x25, 0xc0, 0xcb,
+	0x01, 0x74, 0xd0, 0x82, 0xf4, 0x36, 0xf5, 0x41,
+	0xd5, 0xdc, 0xca, 0xc5, 0xbb, 0x98, 0xfe, 0xfc,
+	0x69, 0x21, 0x70, 0xd8, 0xa4, 0x4b, 0xc8, 0xde,
+	0x8f
+};
+static const u8 dec_assoc006[] __initconst = {
+	0x70, 0xd3, 0x33, 0xf3, 0x8b, 0x18, 0x0b
+};
+static const u8 dec_nonce006[] __initconst = {
+	0xdf, 0x51, 0x84, 0x82, 0x42, 0x0c, 0x75, 0x9c
+};
+static const u8 dec_key006[] __initconst = {
+	0x68, 0x7b, 0x8d, 0x8e, 0xe3, 0xc4, 0xdd, 0xae,
+	0xdf, 0x72, 0x7f, 0x53, 0x72, 0x25, 0x1e, 0x78,
+	0x91, 0xcb, 0x69, 0x76, 0x1f, 0x49, 0x93, 0xf9,
+	0x6f, 0x21, 0xcc, 0x39, 0x9c, 0xad, 0xb1, 0x01
+};
+
+static const u8 dec_input007[] __initconst = {
+	0x85, 0x04, 0xc2, 0xed, 0x8d, 0xfd, 0x97, 0x5c,
+	0xd2, 0xb7, 0xe2, 0xc1, 0x6b, 0xa3, 0xba, 0xf8,
+	0xc9, 0x50, 0xc3, 0xc6, 0xa5, 0xe3, 0xa4, 0x7c,
+	0xc3, 0x23, 0x49, 0x5e, 0xa9, 0xb9, 0x32, 0xeb,
+	0x8a, 0x7c, 0xca, 0xe5, 0xec, 0xfb, 0x7c, 0xc0,
+	0xcb, 0x7d, 0xdc, 0x2c, 0x9d, 0x92, 0x55, 0x21,
+	0x0a, 0xc8, 0x43, 0x63, 0x59, 0x0a, 0x31, 0x70,
+	0x82, 0x67, 0x41, 0x03, 0xf8, 0xdf, 0xf2, 0xac,
+	0xa7, 0x02, 0xd4, 0xd5, 0x8a, 0x2d, 0xc8, 0x99,
+	0x19, 0x66, 0xd0, 0xf6, 0x88, 0x2c, 0x77, 0xd9,
+	0xd4, 0x0d, 0x6c, 0xbd, 0x98, 0xde, 0xe7, 0x7f,
+	0xad, 0x7e, 0x8a, 0xfb, 0xe9, 0x4b, 0xe5, 0xf7,
+	0xe5, 0x50, 0xa0, 0x90, 0x3f, 0xd6, 0x22, 0x53,
+	0xe3, 0xfe, 0x1b, 0xcc, 0x79, 0x3b, 0xec, 0x12,
+	0x47, 0x52, 0xa7, 0xd6, 0x04, 0xe3, 0x52, 0xe6,
+	0x93, 0x90, 0x91, 0x32, 0x73, 0x79, 0xb8, 0xd0,
+	0x31, 0xde, 0x1f, 0x9f, 0x2f, 0x05, 0x38, 0x54,
+	0x2f, 0x35, 0x04, 0x39, 0xe0, 0xa7, 0xba, 0xc6,
+	0x52, 0xf6, 0x37, 0x65, 0x4c, 0x07, 0xa9, 0x7e,
+	0xb3, 0x21, 0x6f, 0x74, 0x8c, 0xc9, 0xde, 0xdb,
+	0x65, 0x1b, 0x9b, 0xaa, 0x60, 0xb1, 0x03, 0x30,
+	0x6b, 0xb2, 0x03, 0xc4, 0x1c, 0x04, 0xf8, 0x0f,
+	0x64, 0xaf, 0x46, 0xe4, 0x65, 0x99, 0x49, 0xe2,
+	0xea, 0xce, 0x78, 0x00, 0xd8, 0x8b, 0xd5, 0x2e,
+	0xcf, 0xfc, 0x40, 0x49, 0xe8, 0x58, 0xdc, 0x34,
+	0x9c, 0x8c, 0x61, 0xbf, 0x0a, 0x8e, 0xec, 0x39,
+	0xa9, 0x30, 0x05, 0x5a, 0xd2, 0x56, 0x01, 0xc7,
+	0xda, 0x8f, 0x4e, 0xbb, 0x43, 0xa3, 0x3a, 0xf9,
+	0x15, 0x2a, 0xd0, 0xa0, 0x7a, 0x87, 0x34, 0x82,
+	0xfe, 0x8a, 0xd1, 0x2d, 0x5e, 0xc7, 0xbf, 0x04,
+	0x53, 0x5f, 0x3b, 0x36, 0xd4, 0x25, 0x5c, 0x34,
+	0x7a, 0x8d, 0xd5, 0x05, 0xce, 0x72, 0xca, 0xef,
+	0x7a, 0x4b, 0xbc, 0xb0, 0x10, 0x5c, 0x96, 0x42,
+	0x3a, 0x00, 0x98, 0xcd, 0x15, 0xe8, 0xb7, 0x53
+};
+static const u8 dec_output007[] __initconst = {
+	0x9b, 0x18, 0xdb, 0xdd, 0x9a, 0x0f, 0x3e, 0xa5,
+	0x15, 0x17, 0xde, 0xdf, 0x08, 0x9d, 0x65, 0x0a,
+	0x67, 0x30, 0x12, 0xe2, 0x34, 0x77, 0x4b, 0xc1,
+	0xd9, 0xc6, 0x1f, 0xab, 0xc6, 0x18, 0x50, 0x17,
+	0xa7, 0x9d, 0x3c, 0xa6, 0xc5, 0x35, 0x8c, 0x1c,
+	0xc0, 0xa1, 0x7c, 0x9f, 0x03, 0x89, 0xca, 0xe1,
+	0xe6, 0xe9, 0xd4, 0xd3, 0x88, 0xdb, 0xb4, 0x51,
+	0x9d, 0xec, 0xb4, 0xfc, 0x52, 0xee, 0x6d, 0xf1,
+	0x75, 0x42, 0xc6, 0xfd, 0xbd, 0x7a, 0x8e, 0x86,
+	0xfc, 0x44, 0xb3, 0x4f, 0xf3, 0xea, 0x67, 0x5a,
+	0x41, 0x13, 0xba, 0xb0, 0xdc, 0xe1, 0xd3, 0x2a,
+	0x7c, 0x22, 0xb3, 0xca, 0xac, 0x6a, 0x37, 0x98,
+	0x3e, 0x1d, 0x40, 0x97, 0xf7, 0x9b, 0x1d, 0x36,
+	0x6b, 0xb3, 0x28, 0xbd, 0x60, 0x82, 0x47, 0x34,
+	0xaa, 0x2f, 0x7d, 0xe9, 0xa8, 0x70, 0x81, 0x57,
+	0xd4, 0xb9, 0x77, 0x0a, 0x9d, 0x29, 0xa7, 0x84,
+	0x52, 0x4f, 0xc2, 0x4a, 0x40, 0x3b, 0x3c, 0xd4,
+	0xc9, 0x2a, 0xdb, 0x4a, 0x53, 0xc4, 0xbe, 0x80,
+	0xe9, 0x51, 0x7f, 0x8f, 0xc7, 0xa2, 0xce, 0x82,
+	0x5c, 0x91, 0x1e, 0x74, 0xd9, 0xd0, 0xbd, 0xd5,
+	0xf3, 0xfd, 0xda, 0x4d, 0x25, 0xb4, 0xbb, 0x2d,
+	0xac, 0x2f, 0x3d, 0x71, 0x85, 0x7b, 0xcf, 0x3c,
+	0x7b, 0x3e, 0x0e, 0x22, 0x78, 0x0c, 0x29, 0xbf,
+	0xe4, 0xf4, 0x57, 0xb3, 0xcb, 0x49, 0xa0, 0xfc,
+	0x1e, 0x05, 0x4e, 0x16, 0xbc, 0xd5, 0xa8, 0xa3,
+	0xee, 0x05, 0x35, 0xc6, 0x7c, 0xab, 0x60, 0x14,
+	0x55, 0x1a, 0x8e, 0xc5, 0x88, 0x5d, 0xd5, 0x81,
+	0xc2, 0x81, 0xa5, 0xc4, 0x60, 0xdb, 0xaf, 0x77,
+	0x91, 0xe1, 0xce, 0xa2, 0x7e, 0x7f, 0x42, 0xe3,
+	0xb0, 0x13, 0x1c, 0x1f, 0x25, 0x60, 0x21, 0xe2,
+	0x40, 0x5f, 0x99, 0xb7, 0x73, 0xec, 0x9b, 0x2b,
+	0xf0, 0x65, 0x11, 0xc8, 0xd0, 0x0a, 0x9f, 0xd3
+};
+static const u8 dec_assoc007[] __initconst = { };
+static const u8 dec_nonce007[] __initconst = {
+	0xde, 0x7b, 0xef, 0xc3, 0x65, 0x1b, 0x68, 0xb0
+};
+static const u8 dec_key007[] __initconst = {
+	0x8d, 0xb8, 0x91, 0x48, 0xf0, 0xe7, 0x0a, 0xbd,
+	0xf9, 0x3f, 0xcd, 0xd9, 0xa0, 0x1e, 0x42, 0x4c,
+	0xe7, 0xde, 0x25, 0x3d, 0xa3, 0xd7, 0x05, 0x80,
+	0x8d, 0xf2, 0x82, 0xac, 0x44, 0x16, 0x51, 0x01
+};
+
+static const u8 dec_input008[] __initconst = {
+	0x14, 0xf6, 0x41, 0x37, 0xa6, 0xd4, 0x27, 0xcd,
+	0xdb, 0x06, 0x3e, 0x9a, 0x4e, 0xab, 0xd5, 0xb1,
+	0x1e, 0x6b, 0xd2, 0xbc, 0x11, 0xf4, 0x28, 0x93,
+	0x63, 0x54, 0xef, 0xbb, 0x5e, 0x1d, 0x3a, 0x1d,
+	0x37, 0x3c, 0x0a, 0x6c, 0x1e, 0xc2, 0xd1, 0x2c,
+	0xb5, 0xa3, 0xb5, 0x7b, 0xb8, 0x8f, 0x25, 0xa6,
+	0x1b, 0x61, 0x1c, 0xec, 0x28, 0x58, 0x26, 0xa4,
+	0xa8, 0x33, 0x28, 0x25, 0x5c, 0x45, 0x05, 0xe5,
+	0x6c, 0x99, 0xe5, 0x45, 0xc4, 0xa2, 0x03, 0x84,
+	0x03, 0x73, 0x1e, 0x8c, 0x49, 0xac, 0x20, 0xdd,
+	0x8d, 0xb3, 0xc4, 0xf5, 0xe7, 0x4f, 0xf1, 0xed,
+	0xa1, 0x98, 0xde, 0xa4, 0x96, 0xdd, 0x2f, 0xab,
+	0xab, 0x97, 0xcf, 0x3e, 0xd2, 0x9e, 0xb8, 0x13,
+	0x07, 0x28, 0x29, 0x19, 0xaf, 0xfd, 0xf2, 0x49,
+	0x43, 0xea, 0x49, 0x26, 0x91, 0xc1, 0x07, 0xd6,
+	0xbb, 0x81, 0x75, 0x35, 0x0d, 0x24, 0x7f, 0xc8,
+	0xda, 0xd4, 0xb7, 0xeb, 0xe8, 0x5c, 0x09, 0xa2,
+	0x2f, 0xdc, 0x28, 0x7d, 0x3a, 0x03, 0xfa, 0x94,
+	0xb5, 0x1d, 0x17, 0x99, 0x36, 0xc3, 0x1c, 0x18,
+	0x34, 0xe3, 0x9f, 0xf5, 0x55, 0x7c, 0xb0, 0x60,
+	0x9d, 0xff, 0xac, 0xd4, 0x61, 0xf2, 0xad, 0xf8,
+	0xce, 0xc7, 0xbe, 0x5c, 0xd2, 0x95, 0xa8, 0x4b,
+	0x77, 0x13, 0x19, 0x59, 0x26, 0xc9, 0xb7, 0x8f,
+	0x6a, 0xcb, 0x2d, 0x37, 0x91, 0xea, 0x92, 0x9c,
+	0x94, 0x5b, 0xda, 0x0b, 0xce, 0xfe, 0x30, 0x20,
+	0xf8, 0x51, 0xad, 0xf2, 0xbe, 0xe7, 0xc7, 0xff,
+	0xb3, 0x33, 0x91, 0x6a, 0xc9, 0x1a, 0x41, 0xc9,
+	0x0f, 0xf3, 0x10, 0x0e, 0xfd, 0x53, 0xff, 0x6c,
+	0x16, 0x52, 0xd9, 0xf3, 0xf7, 0x98, 0x2e, 0xc9,
+	0x07, 0x31, 0x2c, 0x0c, 0x72, 0xd7, 0xc5, 0xc6,
+	0x08, 0x2a, 0x7b, 0xda, 0xbd, 0x7e, 0x02, 0xea,
+	0x1a, 0xbb, 0xf2, 0x04, 0x27, 0x61, 0x28, 0x8e,
+	0xf5, 0x04, 0x03, 0x1f, 0x4c, 0x07, 0x55, 0x82,
+	0xec, 0x1e, 0xd7, 0x8b, 0x2f, 0x65, 0x56, 0xd1,
+	0xd9, 0x1e, 0x3c, 0xe9, 0x1f, 0x5e, 0x98, 0x70,
+	0x38, 0x4a, 0x8c, 0x49, 0xc5, 0x43, 0xa0, 0xa1,
+	0x8b, 0x74, 0x9d, 0x4c, 0x62, 0x0d, 0x10, 0x0c,
+	0xf4, 0x6c, 0x8f, 0xe0, 0xaa, 0x9a, 0x8d, 0xb7,
+	0xe0, 0xbe, 0x4c, 0x87, 0xf1, 0x98, 0x2f, 0xcc,
+	0xed, 0xc0, 0x52, 0x29, 0xdc, 0x83, 0xf8, 0xfc,
+	0x2c, 0x0e, 0xa8, 0x51, 0x4d, 0x80, 0x0d, 0xa3,
+	0xfe, 0xd8, 0x37, 0xe7, 0x41, 0x24, 0xfc, 0xfb,
+	0x75, 0xe3, 0x71, 0x7b, 0x57, 0x45, 0xf5, 0x97,
+	0x73, 0x65, 0x63, 0x14, 0x74, 0xb8, 0x82, 0x9f,
+	0xf8, 0x60, 0x2f, 0x8a, 0xf2, 0x4e, 0xf1, 0x39,
+	0xda, 0x33, 0x91, 0xf8, 0x36, 0xe0, 0x8d, 0x3f,
+	0x1f, 0x3b, 0x56, 0xdc, 0xa0, 0x8f, 0x3c, 0x9d,
+	0x71, 0x52, 0xa7, 0xb8, 0xc0, 0xa5, 0xc6, 0xa2,
+	0x73, 0xda, 0xf4, 0x4b, 0x74, 0x5b, 0x00, 0x3d,
+	0x99, 0xd7, 0x96, 0xba, 0xe6, 0xe1, 0xa6, 0x96,
+	0x38, 0xad, 0xb3, 0xc0, 0xd2, 0xba, 0x91, 0x6b,
+	0xf9, 0x19, 0xdd, 0x3b, 0xbe, 0xbe, 0x9c, 0x20,
+	0x50, 0xba, 0xa1, 0xd0, 0xce, 0x11, 0xbd, 0x95,
+	0xd8, 0xd1, 0xdd, 0x33, 0x85, 0x74, 0xdc, 0xdb,
+	0x66, 0x76, 0x44, 0xdc, 0x03, 0x74, 0x48, 0x35,
+	0x98, 0xb1, 0x18, 0x47, 0x94, 0x7d, 0xff, 0x62,
+	0xe4, 0x58, 0x78, 0xab, 0xed, 0x95, 0x36, 0xd9,
+	0x84, 0x91, 0x82, 0x64, 0x41, 0xbb, 0x58, 0xe6,
+	0x1c, 0x20, 0x6d, 0x15, 0x6b, 0x13, 0x96, 0xe8,
+	0x35, 0x7f, 0xdc, 0x40, 0x2c, 0xe9, 0xbc, 0x8a,
+	0x4f, 0x92, 0xec, 0x06, 0x2d, 0x50, 0xdf, 0x93,
+	0x5d, 0x65, 0x5a, 0xa8, 0xfc, 0x20, 0x50, 0x14,
+	0xa9, 0x8a, 0x7e, 0x1d, 0x08, 0x1f, 0xe2, 0x99,
+	0xd0, 0xbe, 0xfb, 0x3a, 0x21, 0x9d, 0xad, 0x86,
+	0x54, 0xfd, 0x0d, 0x98, 0x1c, 0x5a, 0x6f, 0x1f,
+	0x9a, 0x40, 0xcd, 0xa2, 0xff, 0x6a, 0xf1, 0x54
+};
+static const u8 dec_output008[] __initconst = {
+	0xc3, 0x09, 0x94, 0x62, 0xe6, 0x46, 0x2e, 0x10,
+	0xbe, 0x00, 0xe4, 0xfc, 0xf3, 0x40, 0xa3, 0xe2,
+	0x0f, 0xc2, 0x8b, 0x28, 0xdc, 0xba, 0xb4, 0x3c,
+	0xe4, 0x21, 0x58, 0x61, 0xcd, 0x8b, 0xcd, 0xfb,
+	0xac, 0x94, 0xa1, 0x45, 0xf5, 0x1c, 0xe1, 0x12,
+	0xe0, 0x3b, 0x67, 0x21, 0x54, 0x5e, 0x8c, 0xaa,
+	0xcf, 0xdb, 0xb4, 0x51, 0xd4, 0x13, 0xda, 0xe6,
+	0x83, 0x89, 0xb6, 0x92, 0xe9, 0x21, 0x76, 0xa4,
+	0x93, 0x7d, 0x0e, 0xfd, 0x96, 0x36, 0x03, 0x91,
+	0x43, 0x5c, 0x92, 0x49, 0x62, 0x61, 0x7b, 0xeb,
+	0x43, 0x89, 0xb8, 0x12, 0x20, 0x43, 0xd4, 0x47,
+	0x06, 0x84, 0xee, 0x47, 0xe9, 0x8a, 0x73, 0x15,
+	0x0f, 0x72, 0xcf, 0xed, 0xce, 0x96, 0xb2, 0x7f,
+	0x21, 0x45, 0x76, 0xeb, 0x26, 0x28, 0x83, 0x6a,
+	0xad, 0xaa, 0xa6, 0x81, 0xd8, 0x55, 0xb1, 0xa3,
+	0x85, 0xb3, 0x0c, 0xdf, 0xf1, 0x69, 0x2d, 0x97,
+	0x05, 0x2a, 0xbc, 0x7c, 0x7b, 0x25, 0xf8, 0x80,
+	0x9d, 0x39, 0x25, 0xf3, 0x62, 0xf0, 0x66, 0x5e,
+	0xf4, 0xa0, 0xcf, 0xd8, 0xfd, 0x4f, 0xb1, 0x1f,
+	0x60, 0x3a, 0x08, 0x47, 0xaf, 0xe1, 0xf6, 0x10,
+	0x77, 0x09, 0xa7, 0x27, 0x8f, 0x9a, 0x97, 0x5a,
+	0x26, 0xfa, 0xfe, 0x41, 0x32, 0x83, 0x10, 0xe0,
+	0x1d, 0xbf, 0x64, 0x0d, 0xf4, 0x1c, 0x32, 0x35,
+	0xe5, 0x1b, 0x36, 0xef, 0xd4, 0x4a, 0x93, 0x4d,
+	0x00, 0x7c, 0xec, 0x02, 0x07, 0x8b, 0x5d, 0x7d,
+	0x1b, 0x0e, 0xd1, 0xa6, 0xa5, 0x5d, 0x7d, 0x57,
+	0x88, 0xa8, 0xcc, 0x81, 0xb4, 0x86, 0x4e, 0xb4,
+	0x40, 0xe9, 0x1d, 0xc3, 0xb1, 0x24, 0x3e, 0x7f,
+	0xcc, 0x8a, 0x24, 0x9b, 0xdf, 0x6d, 0xf0, 0x39,
+	0x69, 0x3e, 0x4c, 0xc0, 0x96, 0xe4, 0x13, 0xda,
+	0x90, 0xda, 0xf4, 0x95, 0x66, 0x8b, 0x17, 0x17,
+	0xfe, 0x39, 0x43, 0x25, 0xaa, 0xda, 0xa0, 0x43,
+	0x3c, 0xb1, 0x41, 0x02, 0xa3, 0xf0, 0xa7, 0x19,
+	0x59, 0xbc, 0x1d, 0x7d, 0x6c, 0x6d, 0x91, 0x09,
+	0x5c, 0xb7, 0x5b, 0x01, 0xd1, 0x6f, 0x17, 0x21,
+	0x97, 0xbf, 0x89, 0x71, 0xa5, 0xb0, 0x6e, 0x07,
+	0x45, 0xfd, 0x9d, 0xea, 0x07, 0xf6, 0x7a, 0x9f,
+	0x10, 0x18, 0x22, 0x30, 0x73, 0xac, 0xd4, 0x6b,
+	0x72, 0x44, 0xed, 0xd9, 0x19, 0x9b, 0x2d, 0x4a,
+	0x41, 0xdd, 0xd1, 0x85, 0x5e, 0x37, 0x19, 0xed,
+	0xd2, 0x15, 0x8f, 0x5e, 0x91, 0xdb, 0x33, 0xf2,
+	0xe4, 0xdb, 0xff, 0x98, 0xfb, 0xa3, 0xb5, 0xca,
+	0x21, 0x69, 0x08, 0xe7, 0x8a, 0xdf, 0x90, 0xff,
+	0x3e, 0xe9, 0x20, 0x86, 0x3c, 0xe9, 0xfc, 0x0b,
+	0xfe, 0x5c, 0x61, 0xaa, 0x13, 0x92, 0x7f, 0x7b,
+	0xec, 0xe0, 0x6d, 0xa8, 0x23, 0x22, 0xf6, 0x6b,
+	0x77, 0xc4, 0xfe, 0x40, 0x07, 0x3b, 0xb6, 0xf6,
+	0x8e, 0x5f, 0xd4, 0xb9, 0xb7, 0x0f, 0x21, 0x04,
+	0xef, 0x83, 0x63, 0x91, 0x69, 0x40, 0xa3, 0x48,
+	0x5c, 0xd2, 0x60, 0xf9, 0x4f, 0x6c, 0x47, 0x8b,
+	0x3b, 0xb1, 0x9f, 0x8e, 0xee, 0x16, 0x8a, 0x13,
+	0xfc, 0x46, 0x17, 0xc3, 0xc3, 0x32, 0x56, 0xf8,
+	0x3c, 0x85, 0x3a, 0xb6, 0x3e, 0xaa, 0x89, 0x4f,
+	0xb3, 0xdf, 0x38, 0xfd, 0xf1, 0xe4, 0x3a, 0xc0,
+	0xe6, 0x58, 0xb5, 0x8f, 0xc5, 0x29, 0xa2, 0x92,
+	0x4a, 0xb6, 0xa0, 0x34, 0x7f, 0xab, 0xb5, 0x8a,
+	0x90, 0xa1, 0xdb, 0x4d, 0xca, 0xb6, 0x2c, 0x41,
+	0x3c, 0xf7, 0x2b, 0x21, 0xc3, 0xfd, 0xf4, 0x17,
+	0x5c, 0xb5, 0x33, 0x17, 0x68, 0x2b, 0x08, 0x30,
+	0xf3, 0xf7, 0x30, 0x3c, 0x96, 0xe6, 0x6a, 0x20,
+	0x97, 0xe7, 0x4d, 0x10, 0x5f, 0x47, 0x5f, 0x49,
+	0x96, 0x09, 0xf0, 0x27, 0x91, 0xc8, 0xf8, 0x5a,
+	0x2e, 0x79, 0xb5, 0xe2, 0xb8, 0xe8, 0xb9, 0x7b,
+	0xd5, 0x10, 0xcb, 0xff, 0x5d, 0x14, 0x73, 0xf3
+};
+static const u8 dec_assoc008[] __initconst = { };
+static const u8 dec_nonce008[] __initconst = {
+	0x0e, 0x0d, 0x57, 0xbb, 0x7b, 0x40, 0x54, 0x02
+};
+static const u8 dec_key008[] __initconst = {
+	0xf2, 0xaa, 0x4f, 0x99, 0xfd, 0x3e, 0xa8, 0x53,
+	0xc1, 0x44, 0xe9, 0x81, 0x18, 0xdc, 0xf5, 0xf0,
+	0x3e, 0x44, 0x15, 0x59, 0xe0, 0xc5, 0x44, 0x86,
+	0xc3, 0x91, 0xa8, 0x75, 0xc0, 0x12, 0x46, 0xba
+};
+
+static const u8 dec_input009[] __initconst = {
+	0xfd, 0x81, 0x8d, 0xd0, 0x3d, 0xb4, 0xd5, 0xdf,
+	0xd3, 0x42, 0x47, 0x5a, 0x6d, 0x19, 0x27, 0x66,
+	0x4b, 0x2e, 0x0c, 0x27, 0x9c, 0x96, 0x4c, 0x72,
+	0x02, 0xa3, 0x65, 0xc3, 0xb3, 0x6f, 0x2e, 0xbd,
+	0x63, 0x8a, 0x4a, 0x5d, 0x29, 0xa2, 0xd0, 0x28,
+	0x48, 0xc5, 0x3d, 0x98, 0xa3, 0xbc, 0xe0, 0xbe,
+	0x3b, 0x3f, 0xe6, 0x8a, 0xa4, 0x7f, 0x53, 0x06,
+	0xfa, 0x7f, 0x27, 0x76, 0x72, 0x31, 0xa1, 0xf5,
+	0xd6, 0x0c, 0x52, 0x47, 0xba, 0xcd, 0x4f, 0xd7,
+	0xeb, 0x05, 0x48, 0x0d, 0x7c, 0x35, 0x4a, 0x09,
+	0xc9, 0x76, 0x71, 0x02, 0xa3, 0xfb, 0xb7, 0x1a,
+	0x65, 0xb7, 0xed, 0x98, 0xc6, 0x30, 0x8a, 0x00,
+	0xae, 0xa1, 0x31, 0xe5, 0xb5, 0x9e, 0x6d, 0x62,
+	0xda, 0xda, 0x07, 0x0f, 0x38, 0x38, 0xd3, 0xcb,
+	0xc1, 0xb0, 0xad, 0xec, 0x72, 0xec, 0xb1, 0xa2,
+	0x7b, 0x59, 0xf3, 0x3d, 0x2b, 0xef, 0xcd, 0x28,
+	0x5b, 0x83, 0xcc, 0x18, 0x91, 0x88, 0xb0, 0x2e,
+	0xf9, 0x29, 0x31, 0x18, 0xf9, 0x4e, 0xe9, 0x0a,
+	0x91, 0x92, 0x9f, 0xae, 0x2d, 0xad, 0xf4, 0xe6,
+	0x1a, 0xe2, 0xa4, 0xee, 0x47, 0x15, 0xbf, 0x83,
+	0x6e, 0xd7, 0x72, 0x12, 0x3b, 0x2d, 0x24, 0xe9,
+	0xb2, 0x55, 0xcb, 0x3c, 0x10, 0xf0, 0x24, 0x8a,
+	0x4a, 0x02, 0xea, 0x90, 0x25, 0xf0, 0xb4, 0x79,
+	0x3a, 0xef, 0x6e, 0xf5, 0x52, 0xdf, 0xb0, 0x0a,
+	0xcd, 0x24, 0x1c, 0xd3, 0x2e, 0x22, 0x74, 0xea,
+	0x21, 0x6f, 0xe9, 0xbd, 0xc8, 0x3e, 0x36, 0x5b,
+	0x19, 0xf1, 0xca, 0x99, 0x0a, 0xb4, 0xa7, 0x52,
+	0x1a, 0x4e, 0xf2, 0xad, 0x8d, 0x56, 0x85, 0xbb,
+	0x64, 0x89, 0xba, 0x26, 0xf9, 0xc7, 0xe1, 0x89,
+	0x19, 0x22, 0x77, 0xc3, 0xa8, 0xfc, 0xff, 0xad,
+	0xfe, 0xb9, 0x48, 0xae, 0x12, 0x30, 0x9f, 0x19,
+	0xfb, 0x1b, 0xef, 0x14, 0x87, 0x8a, 0x78, 0x71,
+	0xf3, 0xf4, 0xb7, 0x00, 0x9c, 0x1d, 0xb5, 0x3d,
+	0x49, 0x00, 0x0c, 0x06, 0xd4, 0x50, 0xf9, 0x54,
+	0x45, 0xb2, 0x5b, 0x43, 0xdb, 0x6d, 0xcf, 0x1a,
+	0xe9, 0x7a, 0x7a, 0xcf, 0xfc, 0x8a, 0x4e, 0x4d,
+	0x0b, 0x07, 0x63, 0x28, 0xd8, 0xe7, 0x08, 0x95,
+	0xdf, 0xa6, 0x72, 0x93, 0x2e, 0xbb, 0xa0, 0x42,
+	0x89, 0x16, 0xf1, 0xd9, 0x0c, 0xf9, 0xa1, 0x16,
+	0xfd, 0xd9, 0x03, 0xb4, 0x3b, 0x8a, 0xf5, 0xf6,
+	0xe7, 0x6b, 0x2e, 0x8e, 0x4c, 0x3d, 0xe2, 0xaf,
+	0x08, 0x45, 0x03, 0xff, 0x09, 0xb6, 0xeb, 0x2d,
+	0xc6, 0x1b, 0x88, 0x94, 0xac, 0x3e, 0xf1, 0x9f,
+	0x0e, 0x0e, 0x2b, 0xd5, 0x00, 0x4d, 0x3f, 0x3b,
+	0x53, 0xae, 0xaf, 0x1c, 0x33, 0x5f, 0x55, 0x6e,
+	0x8d, 0xaf, 0x05, 0x7a, 0x10, 0x34, 0xc9, 0xf4,
+	0x66, 0xcb, 0x62, 0x12, 0xa6, 0xee, 0xe8, 0x1c,
+	0x5d, 0x12, 0x86, 0xdb, 0x6f, 0x1c, 0x33, 0xc4,
+	0x1c, 0xda, 0x82, 0x2d, 0x3b, 0x59, 0xfe, 0xb1,
+	0xa4, 0x59, 0x41, 0x86, 0xd0, 0xef, 0xae, 0xfb,
+	0xda, 0x6d, 0x11, 0xb8, 0xca, 0xe9, 0x6e, 0xff,
+	0xf7, 0xa9, 0xd9, 0x70, 0x30, 0xfc, 0x53, 0xe2,
+	0xd7, 0xa2, 0x4e, 0xc7, 0x91, 0xd9, 0x07, 0x06,
+	0xaa, 0xdd, 0xb0, 0x59, 0x28, 0x1d, 0x00, 0x66,
+	0xc5, 0x54, 0xc2, 0xfc, 0x06, 0xda, 0x05, 0x90,
+	0x52, 0x1d, 0x37, 0x66, 0xee, 0xf0, 0xb2, 0x55,
+	0x8a, 0x5d, 0xd2, 0x38, 0x86, 0x94, 0x9b, 0xfc,
+	0x10, 0x4c, 0xa1, 0xb9, 0x64, 0x3e, 0x44, 0xb8,
+	0x5f, 0xb0, 0x0c, 0xec, 0xe0, 0xc9, 0xe5, 0x62,
+	0x75, 0x3f, 0x09, 0xd5, 0xf5, 0xd9, 0x26, 0xba,
+	0x9e, 0xd2, 0xf4, 0xb9, 0x48, 0x0a, 0xbc, 0xa2,
+	0xd6, 0x7c, 0x36, 0x11, 0x7d, 0x26, 0x81, 0x89,
+	0xcf, 0xa4, 0xad, 0x73, 0x0e, 0xee, 0xcc, 0x06,
+	0xa9, 0xdb, 0xb1, 0xfd, 0xfb, 0x09, 0x7f, 0x90,
+	0x42, 0x37, 0x2f, 0xe1, 0x9c, 0x0f, 0x6f, 0xcf,
+	0x43, 0xb5, 0xd9, 0x90, 0xe1, 0x85, 0xf5, 0xa8,
+	0xae
+};
+static const u8 dec_output009[] __initconst = {
+	0xe6, 0xc3, 0xdb, 0x63, 0x55, 0x15, 0xe3, 0x5b,
+	0xb7, 0x4b, 0x27, 0x8b, 0x5a, 0xdd, 0xc2, 0xe8,
+	0x3a, 0x6b, 0xd7, 0x81, 0x96, 0x35, 0x97, 0xca,
+	0xd7, 0x68, 0xe8, 0xef, 0xce, 0xab, 0xda, 0x09,
+	0x6e, 0xd6, 0x8e, 0xcb, 0x55, 0xb5, 0xe1, 0xe5,
+	0x57, 0xfd, 0xc4, 0xe3, 0xe0, 0x18, 0x4f, 0x85,
+	0xf5, 0x3f, 0x7e, 0x4b, 0x88, 0xc9, 0x52, 0x44,
+	0x0f, 0xea, 0xaf, 0x1f, 0x71, 0x48, 0x9f, 0x97,
+	0x6d, 0xb9, 0x6f, 0x00, 0xa6, 0xde, 0x2b, 0x77,
+	0x8b, 0x15, 0xad, 0x10, 0xa0, 0x2b, 0x7b, 0x41,
+	0x90, 0x03, 0x2d, 0x69, 0xae, 0xcc, 0x77, 0x7c,
+	0xa5, 0x9d, 0x29, 0x22, 0xc2, 0xea, 0xb4, 0x00,
+	0x1a, 0xd2, 0x7a, 0x98, 0x8a, 0xf9, 0xf7, 0x82,
+	0xb0, 0xab, 0xd8, 0xa6, 0x94, 0x8d, 0x58, 0x2f,
+	0x01, 0x9e, 0x00, 0x20, 0xfc, 0x49, 0xdc, 0x0e,
+	0x03, 0xe8, 0x45, 0x10, 0xd6, 0xa8, 0xda, 0x55,
+	0x10, 0x9a, 0xdf, 0x67, 0x22, 0x8b, 0x43, 0xab,
+	0x00, 0xbb, 0x02, 0xc8, 0xdd, 0x7b, 0x97, 0x17,
+	0xd7, 0x1d, 0x9e, 0x02, 0x5e, 0x48, 0xde, 0x8e,
+	0xcf, 0x99, 0x07, 0x95, 0x92, 0x3c, 0x5f, 0x9f,
+	0xc5, 0x8a, 0xc0, 0x23, 0xaa, 0xd5, 0x8c, 0x82,
+	0x6e, 0x16, 0x92, 0xb1, 0x12, 0x17, 0x07, 0xc3,
+	0xfb, 0x36, 0xf5, 0x6c, 0x35, 0xd6, 0x06, 0x1f,
+	0x9f, 0xa7, 0x94, 0xa2, 0x38, 0x63, 0x9c, 0xb0,
+	0x71, 0xb3, 0xa5, 0xd2, 0xd8, 0xba, 0x9f, 0x08,
+	0x01, 0xb3, 0xff, 0x04, 0x97, 0x73, 0x45, 0x1b,
+	0xd5, 0xa9, 0x9c, 0x80, 0xaf, 0x04, 0x9a, 0x85,
+	0xdb, 0x32, 0x5b, 0x5d, 0x1a, 0xc1, 0x36, 0x28,
+	0x10, 0x79, 0xf1, 0x3c, 0xbf, 0x1a, 0x41, 0x5c,
+	0x4e, 0xdf, 0xb2, 0x7c, 0x79, 0x3b, 0x7a, 0x62,
+	0x3d, 0x4b, 0xc9, 0x9b, 0x2a, 0x2e, 0x7c, 0xa2,
+	0xb1, 0x11, 0x98, 0xa7, 0x34, 0x1a, 0x00, 0xf3,
+	0xd1, 0xbc, 0x18, 0x22, 0xba, 0x02, 0x56, 0x62,
+	0x31, 0x10, 0x11, 0x6d, 0xe0, 0x54, 0x9d, 0x40,
+	0x1f, 0x26, 0x80, 0x41, 0xca, 0x3f, 0x68, 0x0f,
+	0x32, 0x1d, 0x0a, 0x8e, 0x79, 0xd8, 0xa4, 0x1b,
+	0x29, 0x1c, 0x90, 0x8e, 0xc5, 0xe3, 0xb4, 0x91,
+	0x37, 0x9a, 0x97, 0x86, 0x99, 0xd5, 0x09, 0xc5,
+	0xbb, 0xa3, 0x3f, 0x21, 0x29, 0x82, 0x14, 0x5c,
+	0xab, 0x25, 0xfb, 0xf2, 0x4f, 0x58, 0x26, 0xd4,
+	0x83, 0xaa, 0x66, 0x89, 0x67, 0x7e, 0xc0, 0x49,
+	0xe1, 0x11, 0x10, 0x7f, 0x7a, 0xda, 0x29, 0x04,
+	0xff, 0xf0, 0xcb, 0x09, 0x7c, 0x9d, 0xfa, 0x03,
+	0x6f, 0x81, 0x09, 0x31, 0x60, 0xfb, 0x08, 0xfa,
+	0x74, 0xd3, 0x64, 0x44, 0x7c, 0x55, 0x85, 0xec,
+	0x9c, 0x6e, 0x25, 0xb7, 0x6c, 0xc5, 0x37, 0xb6,
+	0x83, 0x87, 0x72, 0x95, 0x8b, 0x9d, 0xe1, 0x69,
+	0x5c, 0x31, 0x95, 0x42, 0xa6, 0x2c, 0xd1, 0x36,
+	0x47, 0x1f, 0xec, 0x54, 0xab, 0xa2, 0x1c, 0xd8,
+	0x00, 0xcc, 0xbc, 0x0d, 0x65, 0xe2, 0x67, 0xbf,
+	0xbc, 0xea, 0xee, 0x9e, 0xe4, 0x36, 0x95, 0xbe,
+	0x73, 0xd9, 0xa6, 0xd9, 0x0f, 0xa0, 0xcc, 0x82,
+	0x76, 0x26, 0xad, 0x5b, 0x58, 0x6c, 0x4e, 0xab,
+	0x29, 0x64, 0xd3, 0xd9, 0xa9, 0x08, 0x8c, 0x1d,
+	0xa1, 0x4f, 0x80, 0xd8, 0x3f, 0x94, 0xfb, 0xd3,
+	0x7b, 0xfc, 0xd1, 0x2b, 0xc3, 0x21, 0xeb, 0xe5,
+	0x1c, 0x84, 0x23, 0x7f, 0x4b, 0xfa, 0xdb, 0x34,
+	0x18, 0xa2, 0xc2, 0xe5, 0x13, 0xfe, 0x6c, 0x49,
+	0x81, 0xd2, 0x73, 0xe7, 0xe2, 0xd7, 0xe4, 0x4f,
+	0x4b, 0x08, 0x6e, 0xb1, 0x12, 0x22, 0x10, 0x9d,
+	0xac, 0x51, 0x1e, 0x17, 0xd9, 0x8a, 0x0b, 0x42,
+	0x88, 0x16, 0x81, 0x37, 0x7c, 0x6a, 0xf7, 0xef,
+	0x2d, 0xe3, 0xd9, 0xf8, 0x5f, 0xe0, 0x53, 0x27,
+	0x74, 0xb9, 0xe2, 0xd6, 0x1c, 0x80, 0x2c, 0x52,
+	0x65
+};
+static const u8 dec_assoc009[] __initconst = {
+	0x5a, 0x27, 0xff, 0xeb, 0xdf, 0x84, 0xb2, 0x9e,
+	0xef
+};
+static const u8 dec_nonce009[] __initconst = {
+	0xef, 0x2d, 0x63, 0xee, 0x6b, 0x80, 0x8b, 0x78
+};
+static const u8 dec_key009[] __initconst = {
+	0xea, 0xbc, 0x56, 0x99, 0xe3, 0x50, 0xff, 0xc5,
+	0xcc, 0x1a, 0xd7, 0xc1, 0x57, 0x72, 0xea, 0x86,
+	0x5b, 0x89, 0x88, 0x61, 0x3d, 0x2f, 0x9b, 0xb2,
+	0xe7, 0x9c, 0xec, 0x74, 0x6e, 0x3e, 0xf4, 0x3b
+};
+
+static const u8 dec_input010[] __initconst = {
+	0xe5, 0x26, 0xa4, 0x3d, 0xbd, 0x33, 0xd0, 0x4b,
+	0x6f, 0x05, 0xa7, 0x6e, 0x12, 0x7a, 0xd2, 0x74,
+	0xa6, 0xdd, 0xbd, 0x95, 0xeb, 0xf9, 0xa4, 0xf1,
+	0x59, 0x93, 0x91, 0x70, 0xd9, 0xfe, 0x9a, 0xcd,
+	0x53, 0x1f, 0x3a, 0xab, 0xa6, 0x7c, 0x9f, 0xa6,
+	0x9e, 0xbd, 0x99, 0xd9, 0xb5, 0x97, 0x44, 0xd5,
+	0x14, 0x48, 0x4d, 0x9d, 0xc0, 0xd0, 0x05, 0x96,
+	0xeb, 0x4c, 0x78, 0x55, 0x09, 0x08, 0x01, 0x02,
+	0x30, 0x90, 0x7b, 0x96, 0x7a, 0x7b, 0x5f, 0x30,
+	0x41, 0x24, 0xce, 0x68, 0x61, 0x49, 0x86, 0x57,
+	0x82, 0xdd, 0x53, 0x1c, 0x51, 0x28, 0x2b, 0x53,
+	0x6e, 0x2d, 0xc2, 0x20, 0x4c, 0xdd, 0x8f, 0x65,
+	0x10, 0x20, 0x50, 0xdd, 0x9d, 0x50, 0xe5, 0x71,
+	0x40, 0x53, 0x69, 0xfc, 0x77, 0x48, 0x11, 0xb9,
+	0xde, 0xa4, 0x8d, 0x58, 0xe4, 0xa6, 0x1a, 0x18,
+	0x47, 0x81, 0x7e, 0xfc, 0xdd, 0xf6, 0xef, 0xce,
+	0x2f, 0x43, 0x68, 0xd6, 0x06, 0xe2, 0x74, 0x6a,
+	0xad, 0x90, 0xf5, 0x37, 0xf3, 0x3d, 0x82, 0x69,
+	0x40, 0xe9, 0x6b, 0xa7, 0x3d, 0xa8, 0x1e, 0xd2,
+	0x02, 0x7c, 0xb7, 0x9b, 0xe4, 0xda, 0x8f, 0x95,
+	0x06, 0xc5, 0xdf, 0x73, 0xa3, 0x20, 0x9a, 0x49,
+	0xde, 0x9c, 0xbc, 0xee, 0x14, 0x3f, 0x81, 0x5e,
+	0xf8, 0x3b, 0x59, 0x3c, 0xe1, 0x68, 0x12, 0x5a,
+	0x3a, 0x76, 0x3a, 0x3f, 0xf7, 0x87, 0x33, 0x0a,
+	0x01, 0xb8, 0xd4, 0xed, 0xb6, 0xbe, 0x94, 0x5e,
+	0x70, 0x40, 0x56, 0x67, 0x1f, 0x50, 0x44, 0x19,
+	0xce, 0x82, 0x70, 0x10, 0x87, 0x13, 0x20, 0x0b,
+	0x4c, 0x5a, 0xb6, 0xf6, 0xa7, 0xae, 0x81, 0x75,
+	0x01, 0x81, 0xe6, 0x4b, 0x57, 0x7c, 0xdd, 0x6d,
+	0xf8, 0x1c, 0x29, 0x32, 0xf7, 0xda, 0x3c, 0x2d,
+	0xf8, 0x9b, 0x25, 0x6e, 0x00, 0xb4, 0xf7, 0x2f,
+	0xf7, 0x04, 0xf7, 0xa1, 0x56, 0xac, 0x4f, 0x1a,
+	0x64, 0xb8, 0x47, 0x55, 0x18, 0x7b, 0x07, 0x4d,
+	0xbd, 0x47, 0x24, 0x80, 0x5d, 0xa2, 0x70, 0xc5,
+	0xdd, 0x8e, 0x82, 0xd4, 0xeb, 0xec, 0xb2, 0x0c,
+	0x39, 0xd2, 0x97, 0xc1, 0xcb, 0xeb, 0xf4, 0x77,
+	0x59, 0xb4, 0x87, 0xef, 0xcb, 0x43, 0x2d, 0x46,
+	0x54, 0xd1, 0xa7, 0xd7, 0x15, 0x99, 0x0a, 0x43,
+	0xa1, 0xe0, 0x99, 0x33, 0x71, 0xc1, 0xed, 0xfe,
+	0x72, 0x46, 0x33, 0x8e, 0x91, 0x08, 0x9f, 0xc8,
+	0x2e, 0xca, 0xfa, 0xdc, 0x59, 0xd5, 0xc3, 0x76,
+	0x84, 0x9f, 0xa3, 0x37, 0x68, 0xc3, 0xf0, 0x47,
+	0x2c, 0x68, 0xdb, 0x5e, 0xc3, 0x49, 0x4c, 0xe8,
+	0x92, 0x85, 0xe2, 0x23, 0xd3, 0x3f, 0xad, 0x32,
+	0xe5, 0x2b, 0x82, 0xd7, 0x8f, 0x99, 0x0a, 0x59,
+	0x5c, 0x45, 0xd9, 0xb4, 0x51, 0x52, 0xc2, 0xae,
+	0xbf, 0x80, 0xcf, 0xc9, 0xc9, 0x51, 0x24, 0x2a,
+	0x3b, 0x3a, 0x4d, 0xae, 0xeb, 0xbd, 0x22, 0xc3,
+	0x0e, 0x0f, 0x59, 0x25, 0x92, 0x17, 0xe9, 0x74,
+	0xc7, 0x8b, 0x70, 0x70, 0x36, 0x55, 0x95, 0x75,
+	0x4b, 0xad, 0x61, 0x2b, 0x09, 0xbc, 0x82, 0xf2,
+	0x6e, 0x94, 0x43, 0xae, 0xc3, 0xd5, 0xcd, 0x8e,
+	0xfe, 0x5b, 0x9a, 0x88, 0x43, 0x01, 0x75, 0xb2,
+	0x23, 0x09, 0xf7, 0x89, 0x83, 0xe7, 0xfa, 0xf9,
+	0xb4, 0x9b, 0xf8, 0xef, 0xbd, 0x1c, 0x92, 0xc1,
+	0xda, 0x7e, 0xfe, 0x05, 0xba, 0x5a, 0xcd, 0x07,
+	0x6a, 0x78, 0x9e, 0x5d, 0xfb, 0x11, 0x2f, 0x79,
+	0x38, 0xb6, 0xc2, 0x5b, 0x6b, 0x51, 0xb4, 0x71,
+	0xdd, 0xf7, 0x2a, 0xe4, 0xf4, 0x72, 0x76, 0xad,
+	0xc2, 0xdd, 0x64, 0x5d, 0x79, 0xb6, 0xf5, 0x7a,
+	0x77, 0x20, 0x05, 0x3d, 0x30, 0x06, 0xd4, 0x4c,
+	0x0a, 0x2c, 0x98, 0x5a, 0xb9, 0xd4, 0x98, 0xa9,
+	0x3f, 0xc6, 0x12, 0xea, 0x3b, 0x4b, 0xc5, 0x79,
+	0x64, 0x63, 0x6b, 0x09, 0x54, 0x3b, 0x14, 0x27,
+	0xba, 0x99, 0x80, 0xc8, 0x72, 0xa8, 0x12, 0x90,
+	0x29, 0xba, 0x40, 0x54, 0x97, 0x2b, 0x7b, 0xfe,
+	0xeb, 0xcd, 0x01, 0x05, 0x44, 0x72, 0xdb, 0x99,
+	0xe4, 0x61, 0xc9, 0x69, 0xd6, 0xb9, 0x28, 0xd1,
+	0x05, 0x3e, 0xf9, 0x0b, 0x49, 0x0a, 0x49, 0xe9,
+	0x8d, 0x0e, 0xa7, 0x4a, 0x0f, 0xaf, 0x32, 0xd0,
+	0xe0, 0xb2, 0x3a, 0x55, 0x58, 0xfe, 0x5c, 0x28,
+	0x70, 0x51, 0x23, 0xb0, 0x7b, 0x6a, 0x5f, 0x1e,
+	0xb8, 0x17, 0xd7, 0x94, 0x15, 0x8f, 0xee, 0x20,
+	0xc7, 0x42, 0x25, 0x3e, 0x9a, 0x14, 0xd7, 0x60,
+	0x72, 0x39, 0x47, 0x48, 0xa9, 0xfe, 0xdd, 0x47,
+	0x0a, 0xb1, 0xe6, 0x60, 0x28, 0x8c, 0x11, 0x68,
+	0xe1, 0xff, 0xd7, 0xce, 0xc8, 0xbe, 0xb3, 0xfe,
+	0x27, 0x30, 0x09, 0x70, 0xd7, 0xfa, 0x02, 0x33,
+	0x3a, 0x61, 0x2e, 0xc7, 0xff, 0xa4, 0x2a, 0xa8,
+	0x6e, 0xb4, 0x79, 0x35, 0x6d, 0x4c, 0x1e, 0x38,
+	0xf8, 0xee, 0xd4, 0x84, 0x4e, 0x6e, 0x28, 0xa7,
+	0xce, 0xc8, 0xc1, 0xcf, 0x80, 0x05, 0xf3, 0x04,
+	0xef, 0xc8, 0x18, 0x28, 0x2e, 0x8d, 0x5e, 0x0c,
+	0xdf, 0xb8, 0x5f, 0x96, 0xe8, 0xc6, 0x9c, 0x2f,
+	0xe5, 0xa6, 0x44, 0xd7, 0xe7, 0x99, 0x44, 0x0c,
+	0xec, 0xd7, 0x05, 0x60, 0x97, 0xbb, 0x74, 0x77,
+	0x58, 0xd5, 0xbb, 0x48, 0xde, 0x5a, 0xb2, 0x54,
+	0x7f, 0x0e, 0x46, 0x70, 0x6a, 0x6f, 0x78, 0xa5,
+	0x08, 0x89, 0x05, 0x4e, 0x7e, 0xa0, 0x69, 0xb4,
+	0x40, 0x60, 0x55, 0x77, 0x75, 0x9b, 0x19, 0xf2,
+	0xd5, 0x13, 0x80, 0x77, 0xf9, 0x4b, 0x3f, 0x1e,
+	0xee, 0xe6, 0x76, 0x84, 0x7b, 0x8c, 0xe5, 0x27,
+	0xa8, 0x0a, 0x91, 0x01, 0x68, 0x71, 0x8a, 0x3f,
+	0x06, 0xab, 0xf6, 0xa9, 0xa5, 0xe6, 0x72, 0x92,
+	0xe4, 0x67, 0xe2, 0xa2, 0x46, 0x35, 0x84, 0x55,
+	0x7d, 0xca, 0xa8, 0x85, 0xd0, 0xf1, 0x3f, 0xbe,
+	0xd7, 0x34, 0x64, 0xfc, 0xae, 0xe3, 0xe4, 0x04,
+	0x9f, 0x66, 0x02, 0xb9, 0x88, 0x10, 0xd9, 0xc4,
+	0x4c, 0x31, 0x43, 0x7a, 0x93, 0xe2, 0x9b, 0x56,
+	0x43, 0x84, 0xdc, 0xdc, 0xde, 0x1d, 0xa4, 0x02,
+	0x0e, 0xc2, 0xef, 0xc3, 0xf8, 0x78, 0xd1, 0xb2,
+	0x6b, 0x63, 0x18, 0xc9, 0xa9, 0xe5, 0x72, 0xd8,
+	0xf3, 0xb9, 0xd1, 0x8a, 0xc7, 0x1a, 0x02, 0x27,
+	0x20, 0x77, 0x10, 0xe5, 0xc8, 0xd4, 0x4a, 0x47,
+	0xe5, 0xdf, 0x5f, 0x01, 0xaa, 0xb0, 0xd4, 0x10,
+	0xbb, 0x69, 0xe3, 0x36, 0xc8, 0xe1, 0x3d, 0x43,
+	0xfb, 0x86, 0xcd, 0xcc, 0xbf, 0xf4, 0x88, 0xe0,
+	0x20, 0xca, 0xb7, 0x1b, 0xf1, 0x2f, 0x5c, 0xee,
+	0xd4, 0xd3, 0xa3, 0xcc, 0xa4, 0x1e, 0x1c, 0x47,
+	0xfb, 0xbf, 0xfc, 0xa2, 0x41, 0x55, 0x9d, 0xf6,
+	0x5a, 0x5e, 0x65, 0x32, 0x34, 0x7b, 0x52, 0x8d,
+	0xd5, 0xd0, 0x20, 0x60, 0x03, 0xab, 0x3f, 0x8c,
+	0xd4, 0x21, 0xea, 0x2a, 0xd9, 0xc4, 0xd0, 0xd3,
+	0x65, 0xd8, 0x7a, 0x13, 0x28, 0x62, 0x32, 0x4b,
+	0x2c, 0x87, 0x93, 0xa8, 0xb4, 0x52, 0x45, 0x09,
+	0x44, 0xec, 0xec, 0xc3, 0x17, 0xdb, 0x9a, 0x4d,
+	0x5c, 0xa9, 0x11, 0xd4, 0x7d, 0xaf, 0x9e, 0xf1,
+	0x2d, 0xb2, 0x66, 0xc5, 0x1d, 0xed, 0xb7, 0xcd,
+	0x0b, 0x25, 0x5e, 0x30, 0x47, 0x3f, 0x40, 0xf4,
+	0xa1, 0xa0, 0x00, 0x94, 0x10, 0xc5, 0x6a, 0x63,
+	0x1a, 0xd5, 0x88, 0x92, 0x8e, 0x82, 0x39, 0x87,
+	0x3c, 0x78, 0x65, 0x58, 0x42, 0x75, 0x5b, 0xdd,
+	0x77, 0x3e, 0x09, 0x4e, 0x76, 0x5b, 0xe6, 0x0e,
+	0x4d, 0x38, 0xb2, 0xc0, 0xb8, 0x95, 0x01, 0x7a,
+	0x10, 0xe0, 0xfb, 0x07, 0xf2, 0xab, 0x2d, 0x8c,
+	0x32, 0xed, 0x2b, 0xc0, 0x46, 0xc2, 0xf5, 0x38,
+	0x83, 0xf0, 0x17, 0xec, 0xc1, 0x20, 0x6a, 0x9a,
+	0x0b, 0x00, 0xa0, 0x98, 0x22, 0x50, 0x23, 0xd5,
+	0x80, 0x6b, 0xf6, 0x1f, 0xc3, 0xcc, 0x97, 0xc9,
+	0x24, 0x9f, 0xf3, 0xaf, 0x43, 0x14, 0xd5, 0xa0
+};
+static const u8 dec_output010[] __initconst = {
+	0x42, 0x93, 0xe4, 0xeb, 0x97, 0xb0, 0x57, 0xbf,
+	0x1a, 0x8b, 0x1f, 0xe4, 0x5f, 0x36, 0x20, 0x3c,
+	0xef, 0x0a, 0xa9, 0x48, 0x5f, 0x5f, 0x37, 0x22,
+	0x3a, 0xde, 0xe3, 0xae, 0xbe, 0xad, 0x07, 0xcc,
+	0xb1, 0xf6, 0xf5, 0xf9, 0x56, 0xdd, 0xe7, 0x16,
+	0x1e, 0x7f, 0xdf, 0x7a, 0x9e, 0x75, 0xb7, 0xc7,
+	0xbe, 0xbe, 0x8a, 0x36, 0x04, 0xc0, 0x10, 0xf4,
+	0x95, 0x20, 0x03, 0xec, 0xdc, 0x05, 0xa1, 0x7d,
+	0xc4, 0xa9, 0x2c, 0x82, 0xd0, 0xbc, 0x8b, 0xc5,
+	0xc7, 0x45, 0x50, 0xf6, 0xa2, 0x1a, 0xb5, 0x46,
+	0x3b, 0x73, 0x02, 0xa6, 0x83, 0x4b, 0x73, 0x82,
+	0x58, 0x5e, 0x3b, 0x65, 0x2f, 0x0e, 0xfd, 0x2b,
+	0x59, 0x16, 0xce, 0xa1, 0x60, 0x9c, 0xe8, 0x3a,
+	0x99, 0xed, 0x8d, 0x5a, 0xcf, 0xf6, 0x83, 0xaf,
+	0xba, 0xd7, 0x73, 0x73, 0x40, 0x97, 0x3d, 0xca,
+	0xef, 0x07, 0x57, 0xe6, 0xd9, 0x70, 0x0e, 0x95,
+	0xae, 0xa6, 0x8d, 0x04, 0xcc, 0xee, 0xf7, 0x09,
+	0x31, 0x77, 0x12, 0xa3, 0x23, 0x97, 0x62, 0xb3,
+	0x7b, 0x32, 0xfb, 0x80, 0x14, 0x48, 0x81, 0xc3,
+	0xe5, 0xea, 0x91, 0x39, 0x52, 0x81, 0xa2, 0x4f,
+	0xe4, 0xb3, 0x09, 0xff, 0xde, 0x5e, 0xe9, 0x58,
+	0x84, 0x6e, 0xf9, 0x3d, 0xdf, 0x25, 0xea, 0xad,
+	0xae, 0xe6, 0x9a, 0xd1, 0x89, 0x55, 0xd3, 0xde,
+	0x6c, 0x52, 0xdb, 0x70, 0xfe, 0x37, 0xce, 0x44,
+	0x0a, 0xa8, 0x25, 0x5f, 0x92, 0xc1, 0x33, 0x4a,
+	0x4f, 0x9b, 0x62, 0x35, 0xff, 0xce, 0xc0, 0xa9,
+	0x60, 0xce, 0x52, 0x00, 0x97, 0x51, 0x35, 0x26,
+	0x2e, 0xb9, 0x36, 0xa9, 0x87, 0x6e, 0x1e, 0xcc,
+	0x91, 0x78, 0x53, 0x98, 0x86, 0x5b, 0x9c, 0x74,
+	0x7d, 0x88, 0x33, 0xe1, 0xdf, 0x37, 0x69, 0x2b,
+	0xbb, 0xf1, 0x4d, 0xf4, 0xd1, 0xf1, 0x39, 0x93,
+	0x17, 0x51, 0x19, 0xe3, 0x19, 0x1e, 0x76, 0x37,
+	0x25, 0xfb, 0x09, 0x27, 0x6a, 0xab, 0x67, 0x6f,
+	0x14, 0x12, 0x64, 0xe7, 0xc4, 0x07, 0xdf, 0x4d,
+	0x17, 0xbb, 0x6d, 0xe0, 0xe9, 0xb9, 0xab, 0xca,
+	0x10, 0x68, 0xaf, 0x7e, 0xb7, 0x33, 0x54, 0x73,
+	0x07, 0x6e, 0xf7, 0x81, 0x97, 0x9c, 0x05, 0x6f,
+	0x84, 0x5f, 0xd2, 0x42, 0xfb, 0x38, 0xcf, 0xd1,
+	0x2f, 0x14, 0x30, 0x88, 0x98, 0x4d, 0x5a, 0xa9,
+	0x76, 0xd5, 0x4f, 0x3e, 0x70, 0x6c, 0x85, 0x76,
+	0xd7, 0x01, 0xa0, 0x1a, 0xc8, 0x4e, 0xaa, 0xac,
+	0x78, 0xfe, 0x46, 0xde, 0x6a, 0x05, 0x46, 0xa7,
+	0x43, 0x0c, 0xb9, 0xde, 0xb9, 0x68, 0xfb, 0xce,
+	0x42, 0x99, 0x07, 0x4d, 0x0b, 0x3b, 0x5a, 0x30,
+	0x35, 0xa8, 0xf9, 0x3a, 0x73, 0xef, 0x0f, 0xdb,
+	0x1e, 0x16, 0x42, 0xc4, 0xba, 0xae, 0x58, 0xaa,
+	0xf8, 0xe5, 0x75, 0x2f, 0x1b, 0x15, 0x5c, 0xfd,
+	0x0a, 0x97, 0xd0, 0xe4, 0x37, 0x83, 0x61, 0x5f,
+	0x43, 0xa6, 0xc7, 0x3f, 0x38, 0x59, 0xe6, 0xeb,
+	0xa3, 0x90, 0xc3, 0xaa, 0xaa, 0x5a, 0xd3, 0x34,
+	0xd4, 0x17, 0xc8, 0x65, 0x3e, 0x57, 0xbc, 0x5e,
+	0xdd, 0x9e, 0xb7, 0xf0, 0x2e, 0x5b, 0xb2, 0x1f,
+	0x8a, 0x08, 0x0d, 0x45, 0x91, 0x0b, 0x29, 0x53,
+	0x4f, 0x4c, 0x5a, 0x73, 0x56, 0xfe, 0xaf, 0x41,
+	0x01, 0x39, 0x0a, 0x24, 0x3c, 0x7e, 0xbe, 0x4e,
+	0x53, 0xf3, 0xeb, 0x06, 0x66, 0x51, 0x28, 0x1d,
+	0xbd, 0x41, 0x0a, 0x01, 0xab, 0x16, 0x47, 0x27,
+	0x47, 0x47, 0xf7, 0xcb, 0x46, 0x0a, 0x70, 0x9e,
+	0x01, 0x9c, 0x09, 0xe1, 0x2a, 0x00, 0x1a, 0xd8,
+	0xd4, 0x79, 0x9d, 0x80, 0x15, 0x8e, 0x53, 0x2a,
+	0x65, 0x83, 0x78, 0x3e, 0x03, 0x00, 0x07, 0x12,
+	0x1f, 0x33, 0x3e, 0x7b, 0x13, 0x37, 0xf1, 0xc3,
+	0xef, 0xb7, 0xc1, 0x20, 0x3c, 0x3e, 0x67, 0x66,
+	0x5d, 0x88, 0xa7, 0x7d, 0x33, 0x50, 0x77, 0xb0,
+	0x28, 0x8e, 0xe7, 0x2c, 0x2e, 0x7a, 0xf4, 0x3c,
+	0x8d, 0x74, 0x83, 0xaf, 0x8e, 0x87, 0x0f, 0xe4,
+	0x50, 0xff, 0x84, 0x5c, 0x47, 0x0c, 0x6a, 0x49,
+	0xbf, 0x42, 0x86, 0x77, 0x15, 0x48, 0xa5, 0x90,
+	0x5d, 0x93, 0xd6, 0x2a, 0x11, 0xd5, 0xd5, 0x11,
+	0xaa, 0xce, 0xe7, 0x6f, 0xa5, 0xb0, 0x09, 0x2c,
+	0x8d, 0xd3, 0x92, 0xf0, 0x5a, 0x2a, 0xda, 0x5b,
+	0x1e, 0xd5, 0x9a, 0xc4, 0xc4, 0xf3, 0x49, 0x74,
+	0x41, 0xca, 0xe8, 0xc1, 0xf8, 0x44, 0xd6, 0x3c,
+	0xae, 0x6c, 0x1d, 0x9a, 0x30, 0x04, 0x4d, 0x27,
+	0x0e, 0xb1, 0x5f, 0x59, 0xa2, 0x24, 0xe8, 0xe1,
+	0x98, 0xc5, 0x6a, 0x4c, 0xfe, 0x41, 0xd2, 0x27,
+	0x42, 0x52, 0xe1, 0xe9, 0x7d, 0x62, 0xe4, 0x88,
+	0x0f, 0xad, 0xb2, 0x70, 0xcb, 0x9d, 0x4c, 0x27,
+	0x2e, 0x76, 0x1e, 0x1a, 0x63, 0x65, 0xf5, 0x3b,
+	0xf8, 0x57, 0x69, 0xeb, 0x5b, 0x38, 0x26, 0x39,
+	0x33, 0x25, 0x45, 0x3e, 0x91, 0xb8, 0xd8, 0xc7,
+	0xd5, 0x42, 0xc0, 0x22, 0x31, 0x74, 0xf4, 0xbc,
+	0x0c, 0x23, 0xf1, 0xca, 0xc1, 0x8d, 0xd7, 0xbe,
+	0xc9, 0x62, 0xe4, 0x08, 0x1a, 0xcf, 0x36, 0xd5,
+	0xfe, 0x55, 0x21, 0x59, 0x91, 0x87, 0x87, 0xdf,
+	0x06, 0xdb, 0xdf, 0x96, 0x45, 0x58, 0xda, 0x05,
+	0xcd, 0x50, 0x4d, 0xd2, 0x7d, 0x05, 0x18, 0x73,
+	0x6a, 0x8d, 0x11, 0x85, 0xa6, 0x88, 0xe8, 0xda,
+	0xe6, 0x30, 0x33, 0xa4, 0x89, 0x31, 0x75, 0xbe,
+	0x69, 0x43, 0x84, 0x43, 0x50, 0x87, 0xdd, 0x71,
+	0x36, 0x83, 0xc3, 0x78, 0x74, 0x24, 0x0a, 0xed,
+	0x7b, 0xdb, 0xa4, 0x24, 0x0b, 0xb9, 0x7e, 0x5d,
+	0xff, 0xde, 0xb1, 0xef, 0x61, 0x5a, 0x45, 0x33,
+	0xf6, 0x17, 0x07, 0x08, 0x98, 0x83, 0x92, 0x0f,
+	0x23, 0x6d, 0xe6, 0xaa, 0x17, 0x54, 0xad, 0x6a,
+	0xc8, 0xdb, 0x26, 0xbe, 0xb8, 0xb6, 0x08, 0xfa,
+	0x68, 0xf1, 0xd7, 0x79, 0x6f, 0x18, 0xb4, 0x9e,
+	0x2d, 0x3f, 0x1b, 0x64, 0xaf, 0x8d, 0x06, 0x0e,
+	0x49, 0x28, 0xe0, 0x5d, 0x45, 0x68, 0x13, 0x87,
+	0xfa, 0xde, 0x40, 0x7b, 0xd2, 0xc3, 0x94, 0xd5,
+	0xe1, 0xd9, 0xc2, 0xaf, 0x55, 0x89, 0xeb, 0xb4,
+	0x12, 0x59, 0xa8, 0xd4, 0xc5, 0x29, 0x66, 0x38,
+	0xe6, 0xac, 0x22, 0x22, 0xd9, 0x64, 0x9b, 0x34,
+	0x0a, 0x32, 0x9f, 0xc2, 0xbf, 0x17, 0x6c, 0x3f,
+	0x71, 0x7a, 0x38, 0x6b, 0x98, 0xfb, 0x49, 0x36,
+	0x89, 0xc9, 0xe2, 0xd6, 0xc7, 0x5d, 0xd0, 0x69,
+	0x5f, 0x23, 0x35, 0xc9, 0x30, 0xe2, 0xfd, 0x44,
+	0x58, 0x39, 0xd7, 0x97, 0xfb, 0x5c, 0x00, 0xd5,
+	0x4f, 0x7a, 0x1a, 0x95, 0x8b, 0x62, 0x4b, 0xce,
+	0xe5, 0x91, 0x21, 0x7b, 0x30, 0x00, 0xd6, 0xdd,
+	0x6d, 0x02, 0x86, 0x49, 0x0f, 0x3c, 0x1a, 0x27,
+	0x3c, 0xd3, 0x0e, 0x71, 0xf2, 0xff, 0xf5, 0x2f,
+	0x87, 0xac, 0x67, 0x59, 0x81, 0xa3, 0xf7, 0xf8,
+	0xd6, 0x11, 0x0c, 0x84, 0xa9, 0x03, 0xee, 0x2a,
+	0xc4, 0xf3, 0x22, 0xab, 0x7c, 0xe2, 0x25, 0xf5,
+	0x67, 0xa3, 0xe4, 0x11, 0xe0, 0x59, 0xb3, 0xca,
+	0x87, 0xa0, 0xae, 0xc9, 0xa6, 0x62, 0x1b, 0x6e,
+	0x4d, 0x02, 0x6b, 0x07, 0x9d, 0xfd, 0xd0, 0x92,
+	0x06, 0xe1, 0xb2, 0x9a, 0x4a, 0x1f, 0x1f, 0x13,
+	0x49, 0x99, 0x97, 0x08, 0xde, 0x7f, 0x98, 0xaf,
+	0x51, 0x98, 0xee, 0x2c, 0xcb, 0xf0, 0x0b, 0xc6,
+	0xb6, 0xb7, 0x2d, 0x9a, 0xb1, 0xac, 0xa6, 0xe3,
+	0x15, 0x77, 0x9d, 0x6b, 0x1a, 0xe4, 0xfc, 0x8b,
+	0xf2, 0x17, 0x59, 0x08, 0x04, 0x58, 0x81, 0x9d,
+	0x1b, 0x1b, 0x69, 0x55, 0xc2, 0xb4, 0x3c, 0x1f,
+	0x50, 0xf1, 0x7f, 0x77, 0x90, 0x4c, 0x66, 0x40,
+	0x5a, 0xc0, 0x33, 0x1f, 0xcb, 0x05, 0x6d, 0x5c,
+	0x06, 0x87, 0x52, 0xa2, 0x8f, 0x26, 0xd5, 0x4f
+};
+static const u8 dec_assoc010[] __initconst = {
+	0xd2, 0xa1, 0x70, 0xdb, 0x7a, 0xf8, 0xfa, 0x27,
+	0xba, 0x73, 0x0f, 0xbf, 0x3d, 0x1e, 0x82, 0xb2
+};
+static const u8 dec_nonce010[] __initconst = {
+	0xdb, 0x92, 0x0f, 0x7f, 0x17, 0x54, 0x0c, 0x30
+};
+static const u8 dec_key010[] __initconst = {
+	0x47, 0x11, 0xeb, 0x86, 0x2b, 0x2c, 0xab, 0x44,
+	0x34, 0xda, 0x7f, 0x57, 0x03, 0x39, 0x0c, 0xaf,
+	0x2c, 0x14, 0xfd, 0x65, 0x23, 0xe9, 0x8e, 0x74,
+	0xd5, 0x08, 0x68, 0x08, 0xe7, 0xb4, 0x72, 0xd7
+};
+
+static const u8 dec_input011[] __initconst = {
+	0x6a, 0xfc, 0x4b, 0x25, 0xdf, 0xc0, 0xe4, 0xe8,
+	0x17, 0x4d, 0x4c, 0xc9, 0x7e, 0xde, 0x3a, 0xcc,
+	0x3c, 0xba, 0x6a, 0x77, 0x47, 0xdb, 0xe3, 0x74,
+	0x7a, 0x4d, 0x5f, 0x8d, 0x37, 0x55, 0x80, 0x73,
+	0x90, 0x66, 0x5d, 0x3a, 0x7d, 0x5d, 0x86, 0x5e,
+	0x8d, 0xfd, 0x83, 0xff, 0x4e, 0x74, 0x6f, 0xf9,
+	0xe6, 0x70, 0x17, 0x70, 0x3e, 0x96, 0xa7, 0x7e,
+	0xcb, 0xab, 0x8f, 0x58, 0x24, 0x9b, 0x01, 0xfd,
+	0xcb, 0xe6, 0x4d, 0x9b, 0xf0, 0x88, 0x94, 0x57,
+	0x66, 0xef, 0x72, 0x4c, 0x42, 0x6e, 0x16, 0x19,
+	0x15, 0xea, 0x70, 0x5b, 0xac, 0x13, 0xdb, 0x9f,
+	0x18, 0xe2, 0x3c, 0x26, 0x97, 0xbc, 0xdc, 0x45,
+	0x8c, 0x6c, 0x24, 0x69, 0x9c, 0xf7, 0x65, 0x1e,
+	0x18, 0x59, 0x31, 0x7c, 0xe4, 0x73, 0xbc, 0x39,
+	0x62, 0xc6, 0x5c, 0x9f, 0xbf, 0xfa, 0x90, 0x03,
+	0xc9, 0x72, 0x26, 0xb6, 0x1b, 0xc2, 0xb7, 0x3f,
+	0xf2, 0x13, 0x77, 0xf2, 0x8d, 0xb9, 0x47, 0xd0,
+	0x53, 0xdd, 0xc8, 0x91, 0x83, 0x8b, 0xb1, 0xce,
+	0xa3, 0xfe, 0xcd, 0xd9, 0xdd, 0x92, 0x7b, 0xdb,
+	0xb8, 0xfb, 0xc9, 0x2d, 0x01, 0x59, 0x39, 0x52,
+	0xad, 0x1b, 0xec, 0xcf, 0xd7, 0x70, 0x13, 0x21,
+	0xf5, 0x47, 0xaa, 0x18, 0x21, 0x5c, 0xc9, 0x9a,
+	0xd2, 0x6b, 0x05, 0x9c, 0x01, 0xa1, 0xda, 0x35,
+	0x5d, 0xb3, 0x70, 0xe6, 0xa9, 0x80, 0x8b, 0x91,
+	0xb7, 0xb3, 0x5f, 0x24, 0x9a, 0xb7, 0xd1, 0x6b,
+	0xa1, 0x1c, 0x50, 0xba, 0x49, 0xe0, 0xee, 0x2e,
+	0x75, 0xac, 0x69, 0xc0, 0xeb, 0x03, 0xdd, 0x19,
+	0xe5, 0xf6, 0x06, 0xdd, 0xc3, 0xd7, 0x2b, 0x07,
+	0x07, 0x30, 0xa7, 0x19, 0x0c, 0xbf, 0xe6, 0x18,
+	0xcc, 0xb1, 0x01, 0x11, 0x85, 0x77, 0x1d, 0x96,
+	0xa7, 0xa3, 0x00, 0x84, 0x02, 0xa2, 0x83, 0x68,
+	0xda, 0x17, 0x27, 0xc8, 0x7f, 0x23, 0xb7, 0xf4,
+	0x13, 0x85, 0xcf, 0xdd, 0x7a, 0x7d, 0x24, 0x57,
+	0xfe, 0x05, 0x93, 0xf5, 0x74, 0xce, 0xed, 0x0c,
+	0x20, 0x98, 0x8d, 0x92, 0x30, 0xa1, 0x29, 0x23,
+	0x1a, 0xa0, 0x4f, 0x69, 0x56, 0x4c, 0xe1, 0xc8,
+	0xce, 0xf6, 0x9a, 0x0c, 0xa4, 0xfa, 0x04, 0xf6,
+	0x62, 0x95, 0xf2, 0xfa, 0xc7, 0x40, 0x68, 0x40,
+	0x8f, 0x41, 0xda, 0xb4, 0x26, 0x6f, 0x70, 0xab,
+	0x40, 0x61, 0xa4, 0x0e, 0x75, 0xfb, 0x86, 0xeb,
+	0x9d, 0x9a, 0x1f, 0xec, 0x76, 0x99, 0xe7, 0xea,
+	0xaa, 0x1e, 0x2d, 0xb5, 0xd4, 0xa6, 0x1a, 0xb8,
+	0x61, 0x0a, 0x1d, 0x16, 0x5b, 0x98, 0xc2, 0x31,
+	0x40, 0xe7, 0x23, 0x1d, 0x66, 0x99, 0xc8, 0xc0,
+	0xd7, 0xce, 0xf3, 0x57, 0x40, 0x04, 0x3f, 0xfc,
+	0xea, 0xb3, 0xfc, 0xd2, 0xd3, 0x99, 0xa4, 0x94,
+	0x69, 0xa0, 0xef, 0xd1, 0x85, 0xb3, 0xa6, 0xb1,
+	0x28, 0xbf, 0x94, 0x67, 0x22, 0xc3, 0x36, 0x46,
+	0xf8, 0xd2, 0x0f, 0x5f, 0xf4, 0x59, 0x80, 0xe6,
+	0x2d, 0x43, 0x08, 0x7d, 0x19, 0x09, 0x97, 0xa7,
+	0x4c, 0x3d, 0x8d, 0xba, 0x65, 0x62, 0xa3, 0x71,
+	0x33, 0x29, 0x62, 0xdb, 0xc1, 0x33, 0x34, 0x1a,
+	0x63, 0x33, 0x16, 0xb6, 0x64, 0x7e, 0xab, 0x33,
+	0xf0, 0xe6, 0x26, 0x68, 0xba, 0x1d, 0x2e, 0x38,
+	0x08, 0xe6, 0x02, 0xd3, 0x25, 0x2c, 0x47, 0x23,
+	0x58, 0x34, 0x0f, 0x9d, 0x63, 0x4f, 0x63, 0xbb,
+	0x7f, 0x3b, 0x34, 0x38, 0xa7, 0xb5, 0x8d, 0x65,
+	0xd9, 0x9f, 0x79, 0x55, 0x3e, 0x4d, 0xe7, 0x73,
+	0xd8, 0xf6, 0x98, 0x97, 0x84, 0x60, 0x9c, 0xc8,
+	0xa9, 0x3c, 0xf6, 0xdc, 0x12, 0x5c, 0xe1, 0xbb,
+	0x0b, 0x8b, 0x98, 0x9c, 0x9d, 0x26, 0x7c, 0x4a,
+	0xe6, 0x46, 0x36, 0x58, 0x21, 0x4a, 0xee, 0xca,
+	0xd7, 0x3b, 0xc2, 0x6c, 0x49, 0x2f, 0xe5, 0xd5,
+	0x03, 0x59, 0x84, 0x53, 0xcb, 0xfe, 0x92, 0x71,
+	0x2e, 0x7c, 0x21, 0xcc, 0x99, 0x85, 0x7f, 0xb8,
+	0x74, 0x90, 0x13, 0x42, 0x3f, 0xe0, 0x6b, 0x1d,
+	0xf2, 0x4d, 0x54, 0xd4, 0xfc, 0x3a, 0x05, 0xe6,
+	0x74, 0xaf, 0xa6, 0xa0, 0x2a, 0x20, 0x23, 0x5d,
+	0x34, 0x5c, 0xd9, 0x3e, 0x4e, 0xfa, 0x93, 0xe7,
+	0xaa, 0xe9, 0x6f, 0x08, 0x43, 0x67, 0x41, 0xc5,
+	0xad, 0xfb, 0x31, 0x95, 0x82, 0x73, 0x32, 0xd8,
+	0xa6, 0xa3, 0xed, 0x0e, 0x2d, 0xf6, 0x5f, 0xfd,
+	0x80, 0xa6, 0x7a, 0xe0, 0xdf, 0x78, 0x15, 0x29,
+	0x74, 0x33, 0xd0, 0x9e, 0x83, 0x86, 0x72, 0x22,
+	0x57, 0x29, 0xb9, 0x9e, 0x5d, 0xd3, 0x1a, 0xb5,
+	0x96, 0x72, 0x41, 0x3d, 0xf1, 0x64, 0x43, 0x67,
+	0xee, 0xaa, 0x5c, 0xd3, 0x9a, 0x96, 0x13, 0x11,
+	0x5d, 0xf3, 0x0c, 0x87, 0x82, 0x1e, 0x41, 0x9e,
+	0xd0, 0x27, 0xd7, 0x54, 0x3b, 0x67, 0x73, 0x09,
+	0x91, 0xe9, 0xd5, 0x36, 0xa7, 0xb5, 0x55, 0xe4,
+	0xf3, 0x21, 0x51, 0x49, 0x22, 0x07, 0x55, 0x4f,
+	0x44, 0x4b, 0xd2, 0x15, 0x93, 0x17, 0x2a, 0xfa,
+	0x4d, 0x4a, 0x57, 0xdb, 0x4c, 0xa6, 0xeb, 0xec,
+	0x53, 0x25, 0x6c, 0x21, 0xed, 0x00, 0x4c, 0x3b,
+	0xca, 0x14, 0x57, 0xa9, 0xd6, 0x6a, 0xcd, 0x8d,
+	0x5e, 0x74, 0xac, 0x72, 0xc1, 0x97, 0xe5, 0x1b,
+	0x45, 0x4e, 0xda, 0xfc, 0xcc, 0x40, 0xe8, 0x48,
+	0x88, 0x0b, 0xa3, 0xe3, 0x8d, 0x83, 0x42, 0xc3,
+	0x23, 0xfd, 0x68, 0xb5, 0x8e, 0xf1, 0x9d, 0x63,
+	0x77, 0xe9, 0xa3, 0x8e, 0x8c, 0x26, 0x6b, 0xbd,
+	0x72, 0x73, 0x35, 0x0c, 0x03, 0xf8, 0x43, 0x78,
+	0x52, 0x71, 0x15, 0x1f, 0x71, 0x5d, 0x6e, 0xed,
+	0xb9, 0xcc, 0x86, 0x30, 0xdb, 0x2b, 0xd3, 0x82,
+	0x88, 0x23, 0x71, 0x90, 0x53, 0x5c, 0xa9, 0x2f,
+	0x76, 0x01, 0xb7, 0x9a, 0xfe, 0x43, 0x55, 0xa3,
+	0x04, 0x9b, 0x0e, 0xe4, 0x59, 0xdf, 0xc9, 0xe9,
+	0xb1, 0xea, 0x29, 0x28, 0x3c, 0x5c, 0xae, 0x72,
+	0x84, 0xb6, 0xc6, 0xeb, 0x0c, 0x27, 0x07, 0x74,
+	0x90, 0x0d, 0x31, 0xb0, 0x00, 0x77, 0xe9, 0x40,
+	0x70, 0x6f, 0x68, 0xa7, 0xfd, 0x06, 0xec, 0x4b,
+	0xc0, 0xb7, 0xac, 0xbc, 0x33, 0xb7, 0x6d, 0x0a,
+	0xbd, 0x12, 0x1b, 0x59, 0xcb, 0xdd, 0x32, 0xf5,
+	0x1d, 0x94, 0x57, 0x76, 0x9e, 0x0c, 0x18, 0x98,
+	0x71, 0xd7, 0x2a, 0xdb, 0x0b, 0x7b, 0xa7, 0x71,
+	0xb7, 0x67, 0x81, 0x23, 0x96, 0xae, 0xb9, 0x7e,
+	0x32, 0x43, 0x92, 0x8a, 0x19, 0xa0, 0xc4, 0xd4,
+	0x3b, 0x57, 0xf9, 0x4a, 0x2c, 0xfb, 0x51, 0x46,
+	0xbb, 0xcb, 0x5d, 0xb3, 0xef, 0x13, 0x93, 0x6e,
+	0x68, 0x42, 0x54, 0x57, 0xd3, 0x6a, 0x3a, 0x8f,
+	0x9d, 0x66, 0xbf, 0xbd, 0x36, 0x23, 0xf5, 0x93,
+	0x83, 0x7b, 0x9c, 0xc0, 0xdd, 0xc5, 0x49, 0xc0,
+	0x64, 0xed, 0x07, 0x12, 0xb3, 0xe6, 0xe4, 0xe5,
+	0x38, 0x95, 0x23, 0xb1, 0xa0, 0x3b, 0x1a, 0x61,
+	0xda, 0x17, 0xac, 0xc3, 0x58, 0xdd, 0x74, 0x64,
+	0x22, 0x11, 0xe8, 0x32, 0x1d, 0x16, 0x93, 0x85,
+	0x99, 0xa5, 0x9c, 0x34, 0x55, 0xb1, 0xe9, 0x20,
+	0x72, 0xc9, 0x28, 0x7b, 0x79, 0x00, 0xa1, 0xa6,
+	0xa3, 0x27, 0x40, 0x18, 0x8a, 0x54, 0xe0, 0xcc,
+	0xe8, 0x4e, 0x8e, 0x43, 0x96, 0xe7, 0x3f, 0xc8,
+	0xe9, 0xb2, 0xf9, 0xc9, 0xda, 0x04, 0x71, 0x50,
+	0x47, 0xe4, 0xaa, 0xce, 0xa2, 0x30, 0xc8, 0xe4,
+	0xac, 0xc7, 0x0d, 0x06, 0x2e, 0xe6, 0xe8, 0x80,
+	0x36, 0x29, 0x9e, 0x01, 0xb8, 0xc3, 0xf0, 0xa0,
+	0x5d, 0x7a, 0xca, 0x4d, 0xa0, 0x57, 0xbd, 0x2a,
+	0x45, 0xa7, 0x7f, 0x9c, 0x93, 0x07, 0x8f, 0x35,
+	0x67, 0x92, 0xe3, 0xe9, 0x7f, 0xa8, 0x61, 0x43,
+	0x9e, 0x25, 0x4f, 0x33, 0x76, 0x13, 0x6e, 0x12,
+	0xb9, 0xdd, 0xa4, 0x7c, 0x08, 0x9f, 0x7c, 0xe7,
+	0x0a, 0x8d, 0x84, 0x06, 0xa4, 0x33, 0x17, 0x34,
+	0x5e, 0x10, 0x7c, 0xc0, 0xa8, 0x3d, 0x1f, 0x42,
+	0x20, 0x51, 0x65, 0x5d, 0x09, 0xc3, 0xaa, 0xc0,
+	0xc8, 0x0d, 0xf0, 0x79, 0xbc, 0x20, 0x1b, 0x95,
+	0xe7, 0x06, 0x7d, 0x47, 0x20, 0x03, 0x1a, 0x74,
+	0xdd, 0xe2, 0xd4, 0xae, 0x38, 0x71, 0x9b, 0xf5,
+	0x80, 0xec, 0x08, 0x4e, 0x56, 0xba, 0x76, 0x12,
+	0x1a, 0xdf, 0x48, 0xf3, 0xae, 0xb3, 0xe6, 0xe6,
+	0xbe, 0xc0, 0x91, 0x2e, 0x01, 0xb3, 0x01, 0x86,
+	0xa2, 0xb9, 0x52, 0xd1, 0x21, 0xae, 0xd4, 0x97,
+	0x1d, 0xef, 0x41, 0x12, 0x95, 0x3d, 0x48, 0x45,
+	0x1c, 0x56, 0x32, 0x8f, 0xb8, 0x43, 0xbb, 0x19,
+	0xf3, 0xca, 0xe9, 0xeb, 0x6d, 0x84, 0xbe, 0x86,
+	0x06, 0xe2, 0x36, 0xb2, 0x62, 0x9d, 0xd3, 0x4c,
+	0x48, 0x18, 0x54, 0x13, 0x4e, 0xcf, 0xfd, 0xba,
+	0x84, 0xb9, 0x30, 0x53, 0xcf, 0xfb, 0xb9, 0x29,
+	0x8f, 0xdc, 0x9f, 0xef, 0x60, 0x0b, 0x64, 0xf6,
+	0x8b, 0xee, 0xa6, 0x91, 0xc2, 0x41, 0x6c, 0xf6,
+	0xfa, 0x79, 0x67, 0x4b, 0xc1, 0x3f, 0xaf, 0x09,
+	0x81, 0xd4, 0x5d, 0xcb, 0x09, 0xdf, 0x36, 0x31,
+	0xc0, 0x14, 0x3c, 0x7c, 0x0e, 0x65, 0x95, 0x99,
+	0x6d, 0xa3, 0xf4, 0xd7, 0x38, 0xee, 0x1a, 0x2b,
+	0x37, 0xe2, 0xa4, 0x3b, 0x4b, 0xd0, 0x65, 0xca,
+	0xf8, 0xc3, 0xe8, 0x15, 0x20, 0xef, 0xf2, 0x00,
+	0xfd, 0x01, 0x09, 0xc5, 0xc8, 0x17, 0x04, 0x93,
+	0xd0, 0x93, 0x03, 0x55, 0xc5, 0xfe, 0x32, 0xa3,
+	0x3e, 0x28, 0x2d, 0x3b, 0x93, 0x8a, 0xcc, 0x07,
+	0x72, 0x80, 0x8b, 0x74, 0x16, 0x24, 0xbb, 0xda,
+	0x94, 0x39, 0x30, 0x8f, 0xb1, 0xcd, 0x4a, 0x90,
+	0x92, 0x7c, 0x14, 0x8f, 0x95, 0x4e, 0xac, 0x9b,
+	0xd8, 0x8f, 0x1a, 0x87, 0xa4, 0x32, 0x27, 0x8a,
+	0xba, 0xf7, 0x41, 0xcf, 0x84, 0x37, 0x19, 0xe6,
+	0x06, 0xf5, 0x0e, 0xcf, 0x36, 0xf5, 0x9e, 0x6c,
+	0xde, 0xbc, 0xff, 0x64, 0x7e, 0x4e, 0x59, 0x57,
+	0x48, 0xfe, 0x14, 0xf7, 0x9c, 0x93, 0x5d, 0x15,
+	0xad, 0xcc, 0x11, 0xb1, 0x17, 0x18, 0xb2, 0x7e,
+	0xcc, 0xab, 0xe9, 0xce, 0x7d, 0x77, 0x5b, 0x51,
+	0x1b, 0x1e, 0x20, 0xa8, 0x32, 0x06, 0x0e, 0x75,
+	0x93, 0xac, 0xdb, 0x35, 0x37, 0x1f, 0xe9, 0x19,
+	0x1d, 0xb4, 0x71, 0x97, 0xd6, 0x4e, 0x2c, 0x08,
+	0xa5, 0x13, 0xf9, 0x0e, 0x7e, 0x78, 0x6e, 0x14,
+	0xe0, 0xa9, 0xb9, 0x96, 0x4c, 0x80, 0x82, 0xba,
+	0x17, 0xb3, 0x9d, 0x69, 0xb0, 0x84, 0x46, 0xff,
+	0xf9, 0x52, 0x79, 0x94, 0x58, 0x3a, 0x62, 0x90,
+	0x15, 0x35, 0x71, 0x10, 0x37, 0xed, 0xa1, 0x8e,
+	0x53, 0x6e, 0xf4, 0x26, 0x57, 0x93, 0x15, 0x93,
+	0xf6, 0x81, 0x2c, 0x5a, 0x10, 0xda, 0x92, 0xad,
+	0x2f, 0xdb, 0x28, 0x31, 0x2d, 0x55, 0x04, 0xd2,
+	0x06, 0x28, 0x8c, 0x1e, 0xdc, 0xea, 0x54, 0xac,
+	0xff, 0xb7, 0x6c, 0x30, 0x15, 0xd4, 0xb4, 0x0d,
+	0x00, 0x93, 0x57, 0xdd, 0xd2, 0x07, 0x07, 0x06,
+	0xd9, 0x43, 0x9b, 0xcd, 0x3a, 0xf4, 0x7d, 0x4c,
+	0x36, 0x5d, 0x23, 0xa2, 0xcc, 0x57, 0x40, 0x91,
+	0xe9, 0x2c, 0x2f, 0x2c, 0xd5, 0x30, 0x9b, 0x17,
+	0xb0, 0xc9, 0xf7, 0xa7, 0x2f, 0xd1, 0x93, 0x20,
+	0x6b, 0xc6, 0xc1, 0xe4, 0x6f, 0xcb, 0xd1, 0xe7,
+	0x09, 0x0f, 0x9e, 0xdc, 0xaa, 0x9f, 0x2f, 0xdf,
+	0x56, 0x9f, 0xd4, 0x33, 0x04, 0xaf, 0xd3, 0x6c,
+	0x58, 0x61, 0xf0, 0x30, 0xec, 0xf2, 0x7f, 0xf2,
+	0x9c, 0xdf, 0x39, 0xbb, 0x6f, 0xa2, 0x8c, 0x7e,
+	0xc4, 0x22, 0x51, 0x71, 0xc0, 0x4d, 0x14, 0x1a,
+	0xc4, 0xcd, 0x04, 0xd9, 0x87, 0x08, 0x50, 0x05,
+	0xcc, 0xaf, 0xf6, 0xf0, 0x8f, 0x92, 0x54, 0x58,
+	0xc2, 0xc7, 0x09, 0x7a, 0x59, 0x02, 0x05, 0xe8,
+	0xb0, 0x86, 0xd9, 0xbf, 0x7b, 0x35, 0x51, 0x4d,
+	0xaf, 0x08, 0x97, 0x2c, 0x65, 0xda, 0x2a, 0x71,
+	0x3a, 0xa8, 0x51, 0xcc, 0xf2, 0x73, 0x27, 0xc3,
+	0xfd, 0x62, 0xcf, 0xe3, 0xb2, 0xca, 0xcb, 0xbe,
+	0x1a, 0x0a, 0xa1, 0x34, 0x7b, 0x77, 0xc4, 0x62,
+	0x68, 0x78, 0x5f, 0x94, 0x07, 0x04, 0x65, 0x16,
+	0x4b, 0x61, 0xcb, 0xff, 0x75, 0x26, 0x50, 0x66,
+	0x1f, 0x6e, 0x93, 0xf8, 0xc5, 0x51, 0xeb, 0xa4,
+	0x4a, 0x48, 0x68, 0x6b, 0xe2, 0x5e, 0x44, 0xb2,
+	0x50, 0x2c, 0x6c, 0xae, 0x79, 0x4e, 0x66, 0x35,
+	0x81, 0x50, 0xac, 0xbc, 0x3f, 0xb1, 0x0c, 0xf3,
+	0x05, 0x3c, 0x4a, 0xa3, 0x6c, 0x2a, 0x79, 0xb4,
+	0xb7, 0xab, 0xca, 0xc7, 0x9b, 0x8e, 0xcd, 0x5f,
+	0x11, 0x03, 0xcb, 0x30, 0xa3, 0xab, 0xda, 0xfe,
+	0x64, 0xb9, 0xbb, 0xd8, 0x5e, 0x3a, 0x1a, 0x56,
+	0xe5, 0x05, 0x48, 0x90, 0x1e, 0x61, 0x69, 0x1b,
+	0x22, 0xe6, 0x1a, 0x3c, 0x75, 0xad, 0x1f, 0x37,
+	0x28, 0xdc, 0xe4, 0x6d, 0xbd, 0x42, 0xdc, 0xd3,
+	0xc8, 0xb6, 0x1c, 0x48, 0xfe, 0x94, 0x77, 0x7f,
+	0xbd, 0x62, 0xac, 0xa3, 0x47, 0x27, 0xcf, 0x5f,
+	0xd9, 0xdb, 0xaf, 0xec, 0xf7, 0x5e, 0xc1, 0xb0,
+	0x9d, 0x01, 0x26, 0x99, 0x7e, 0x8f, 0x03, 0x70,
+	0xb5, 0x42, 0xbe, 0x67, 0x28, 0x1b, 0x7c, 0xbd,
+	0x61, 0x21, 0x97, 0xcc, 0x5c, 0xe1, 0x97, 0x8f,
+	0x8d, 0xde, 0x2b, 0xaa, 0xa7, 0x71, 0x1d, 0x1e,
+	0x02, 0x73, 0x70, 0x58, 0x32, 0x5b, 0x1d, 0x67,
+	0x3d, 0xe0, 0x74, 0x4f, 0x03, 0xf2, 0x70, 0x51,
+	0x79, 0xf1, 0x61, 0x70, 0x15, 0x74, 0x9d, 0x23,
+	0x89, 0xde, 0xac, 0xfd, 0xde, 0xd0, 0x1f, 0xc3,
+	0x87, 0x44, 0x35, 0x4b, 0xe5, 0xb0, 0x60, 0xc5,
+	0x22, 0xe4, 0x9e, 0xca, 0xeb, 0xd5, 0x3a, 0x09,
+	0x45, 0xa4, 0xdb, 0xfa, 0x3f, 0xeb, 0x1b, 0xc7,
+	0xc8, 0x14, 0x99, 0x51, 0x92, 0x10, 0xed, 0xed,
+	0x28, 0xe0, 0xa1, 0xf8, 0x26, 0xcf, 0xcd, 0xcb,
+	0x63, 0xa1, 0x3b, 0xe3, 0xdf, 0x7e, 0xfe, 0xa6,
+	0xf0, 0x81, 0x9a, 0xbf, 0x55, 0xde, 0x54, 0xd5,
+	0x56, 0x60, 0x98, 0x10, 0x68, 0xf4, 0x38, 0x96,
+	0x8e, 0x6f, 0x1d, 0x44, 0x7f, 0xd6, 0x2f, 0xfe,
+	0x55, 0xfb, 0x0c, 0x7e, 0x67, 0xe2, 0x61, 0x44,
+	0xed, 0xf2, 0x35, 0x30, 0x5d, 0xe9, 0xc7, 0xd6,
+	0x6d, 0xe0, 0xa0, 0xed, 0xf3, 0xfc, 0xd8, 0x3e,
+	0x0a, 0x7b, 0xcd, 0xaf, 0x65, 0x68, 0x18, 0xc0,
+	0xec, 0x04, 0x1c, 0x74, 0x6d, 0xe2, 0x6e, 0x79,
+	0xd4, 0x11, 0x2b, 0x62, 0xd5, 0x27, 0xad, 0x4f,
+	0x01, 0x59, 0x73, 0xcc, 0x6a, 0x53, 0xfb, 0x2d,
+	0xd5, 0x4e, 0x99, 0x21, 0x65, 0x4d, 0xf5, 0x82,
+	0xf7, 0xd8, 0x42, 0xce, 0x6f, 0x3d, 0x36, 0x47,
+	0xf1, 0x05, 0x16, 0xe8, 0x1b, 0x6a, 0x8f, 0x93,
+	0xf2, 0x8f, 0x37, 0x40, 0x12, 0x28, 0xa3, 0xe6,
+	0xb9, 0x17, 0x4a, 0x1f, 0xb1, 0xd1, 0x66, 0x69,
+	0x86, 0xc4, 0xfc, 0x97, 0xae, 0x3f, 0x8f, 0x1e,
+	0x2b, 0xdf, 0xcd, 0xf9, 0x3c
+};
+static const u8 dec_output011[] __initconst = {
+	0x7a, 0x57, 0xf2, 0xc7, 0x06, 0x3f, 0x50, 0x7b,
+	0x36, 0x1a, 0x66, 0x5c, 0xb9, 0x0e, 0x5e, 0x3b,
+	0x45, 0x60, 0xbe, 0x9a, 0x31, 0x9f, 0xff, 0x5d,
+	0x66, 0x34, 0xb4, 0xdc, 0xfb, 0x9d, 0x8e, 0xee,
+	0x6a, 0x33, 0xa4, 0x07, 0x3c, 0xf9, 0x4c, 0x30,
+	0xa1, 0x24, 0x52, 0xf9, 0x50, 0x46, 0x88, 0x20,
+	0x02, 0x32, 0x3a, 0x0e, 0x99, 0x63, 0xaf, 0x1f,
+	0x15, 0x28, 0x2a, 0x05, 0xff, 0x57, 0x59, 0x5e,
+	0x18, 0xa1, 0x1f, 0xd0, 0x92, 0x5c, 0x88, 0x66,
+	0x1b, 0x00, 0x64, 0xa5, 0x93, 0x8d, 0x06, 0x46,
+	0xb0, 0x64, 0x8b, 0x8b, 0xef, 0x99, 0x05, 0x35,
+	0x85, 0xb3, 0xf3, 0x33, 0xbb, 0xec, 0x66, 0xb6,
+	0x3d, 0x57, 0x42, 0xe3, 0xb4, 0xc6, 0xaa, 0xb0,
+	0x41, 0x2a, 0xb9, 0x59, 0xa9, 0xf6, 0x3e, 0x15,
+	0x26, 0x12, 0x03, 0x21, 0x4c, 0x74, 0x43, 0x13,
+	0x2a, 0x03, 0x27, 0x09, 0xb4, 0xfb, 0xe7, 0xb7,
+	0x40, 0xff, 0x5e, 0xce, 0x48, 0x9a, 0x60, 0xe3,
+	0x8b, 0x80, 0x8c, 0x38, 0x2d, 0xcb, 0x93, 0x37,
+	0x74, 0x05, 0x52, 0x6f, 0x73, 0x3e, 0xc3, 0xbc,
+	0xca, 0x72, 0x0a, 0xeb, 0xf1, 0x3b, 0xa0, 0x95,
+	0xdc, 0x8a, 0xc4, 0xa9, 0xdc, 0xca, 0x44, 0xd8,
+	0x08, 0x63, 0x6a, 0x36, 0xd3, 0x3c, 0xb8, 0xac,
+	0x46, 0x7d, 0xfd, 0xaa, 0xeb, 0x3e, 0x0f, 0x45,
+	0x8f, 0x49, 0xda, 0x2b, 0xf2, 0x12, 0xbd, 0xaf,
+	0x67, 0x8a, 0x63, 0x48, 0x4b, 0x55, 0x5f, 0x6d,
+	0x8c, 0xb9, 0x76, 0x34, 0x84, 0xae, 0xc2, 0xfc,
+	0x52, 0x64, 0x82, 0xf7, 0xb0, 0x06, 0xf0, 0x45,
+	0x73, 0x12, 0x50, 0x30, 0x72, 0xea, 0x78, 0x9a,
+	0xa8, 0xaf, 0xb5, 0xe3, 0xbb, 0x77, 0x52, 0xec,
+	0x59, 0x84, 0xbf, 0x6b, 0x8f, 0xce, 0x86, 0x5e,
+	0x1f, 0x23, 0xe9, 0xfb, 0x08, 0x86, 0xf7, 0x10,
+	0xb9, 0xf2, 0x44, 0x96, 0x44, 0x63, 0xa9, 0xa8,
+	0x78, 0x00, 0x23, 0xd6, 0xc7, 0xe7, 0x6e, 0x66,
+	0x4f, 0xcc, 0xee, 0x15, 0xb3, 0xbd, 0x1d, 0xa0,
+	0xe5, 0x9c, 0x1b, 0x24, 0x2c, 0x4d, 0x3c, 0x62,
+	0x35, 0x9c, 0x88, 0x59, 0x09, 0xdd, 0x82, 0x1b,
+	0xcf, 0x0a, 0x83, 0x6b, 0x3f, 0xae, 0x03, 0xc4,
+	0xb4, 0xdd, 0x7e, 0x5b, 0x28, 0x76, 0x25, 0x96,
+	0xd9, 0xc9, 0x9d, 0x5f, 0x86, 0xfa, 0xf6, 0xd7,
+	0xd2, 0xe6, 0x76, 0x1d, 0x0f, 0xa1, 0xdc, 0x74,
+	0x05, 0x1b, 0x1d, 0xe0, 0xcd, 0x16, 0xb0, 0xa8,
+	0x8a, 0x34, 0x7b, 0x15, 0x11, 0x77, 0xe5, 0x7b,
+	0x7e, 0x20, 0xf7, 0xda, 0x38, 0xda, 0xce, 0x70,
+	0xe9, 0xf5, 0x6c, 0xd9, 0xbe, 0x0c, 0x4c, 0x95,
+	0x4c, 0xc2, 0x9b, 0x34, 0x55, 0x55, 0xe1, 0xf3,
+	0x46, 0x8e, 0x48, 0x74, 0x14, 0x4f, 0x9d, 0xc9,
+	0xf5, 0xe8, 0x1a, 0xf0, 0x11, 0x4a, 0xc1, 0x8d,
+	0xe0, 0x93, 0xa0, 0xbe, 0x09, 0x1c, 0x2b, 0x4e,
+	0x0f, 0xb2, 0x87, 0x8b, 0x84, 0xfe, 0x92, 0x32,
+	0x14, 0xd7, 0x93, 0xdf, 0xe7, 0x44, 0xbc, 0xc5,
+	0xae, 0x53, 0x69, 0xd8, 0xb3, 0x79, 0x37, 0x80,
+	0xe3, 0x17, 0x5c, 0xec, 0x53, 0x00, 0x9a, 0xe3,
+	0x8e, 0xdc, 0x38, 0xb8, 0x66, 0xf0, 0xd3, 0xad,
+	0x1d, 0x02, 0x96, 0x86, 0x3e, 0x9d, 0x3b, 0x5d,
+	0xa5, 0x7f, 0x21, 0x10, 0xf1, 0x1f, 0x13, 0x20,
+	0xf9, 0x57, 0x87, 0x20, 0xf5, 0x5f, 0xf1, 0x17,
+	0x48, 0x0a, 0x51, 0x5a, 0xcd, 0x19, 0x03, 0xa6,
+	0x5a, 0xd1, 0x12, 0x97, 0xe9, 0x48, 0xe2, 0x1d,
+	0x83, 0x75, 0x50, 0xd9, 0x75, 0x7d, 0x6a, 0x82,
+	0xa1, 0xf9, 0x4e, 0x54, 0x87, 0x89, 0xc9, 0x0c,
+	0xb7, 0x5b, 0x6a, 0x91, 0xc1, 0x9c, 0xb2, 0xa9,
+	0xdc, 0x9a, 0xa4, 0x49, 0x0a, 0x6d, 0x0d, 0xbb,
+	0xde, 0x86, 0x44, 0xdd, 0x5d, 0x89, 0x2b, 0x96,
+	0x0f, 0x23, 0x95, 0xad, 0xcc, 0xa2, 0xb3, 0xb9,
+	0x7e, 0x74, 0x38, 0xba, 0x9f, 0x73, 0xae, 0x5f,
+	0xf8, 0x68, 0xa2, 0xe0, 0xa9, 0xce, 0xbd, 0x40,
+	0xd4, 0x4c, 0x6b, 0xd2, 0x56, 0x62, 0xb0, 0xcc,
+	0x63, 0x7e, 0x5b, 0xd3, 0xae, 0xd1, 0x75, 0xce,
+	0xbb, 0xb4, 0x5b, 0xa8, 0xf8, 0xb4, 0xac, 0x71,
+	0x75, 0xaa, 0xc9, 0x9f, 0xbb, 0x6c, 0xad, 0x0f,
+	0x55, 0x5d, 0xe8, 0x85, 0x7d, 0xf9, 0x21, 0x35,
+	0xea, 0x92, 0x85, 0x2b, 0x00, 0xec, 0x84, 0x90,
+	0x0a, 0x63, 0x96, 0xe4, 0x6b, 0xa9, 0x77, 0xb8,
+	0x91, 0xf8, 0x46, 0x15, 0x72, 0x63, 0x70, 0x01,
+	0x40, 0xa3, 0xa5, 0x76, 0x62, 0x2b, 0xbf, 0xf1,
+	0xe5, 0x8d, 0x9f, 0xa3, 0xfa, 0x9b, 0x03, 0xbe,
+	0xfe, 0x65, 0x6f, 0xa2, 0x29, 0x0d, 0x54, 0xb4,
+	0x71, 0xce, 0xa9, 0xd6, 0x3d, 0x88, 0xf9, 0xaf,
+	0x6b, 0xa8, 0x9e, 0xf4, 0x16, 0x96, 0x36, 0xb9,
+	0x00, 0xdc, 0x10, 0xab, 0xb5, 0x08, 0x31, 0x1f,
+	0x00, 0xb1, 0x3c, 0xd9, 0x38, 0x3e, 0xc6, 0x04,
+	0xa7, 0x4e, 0xe8, 0xae, 0xed, 0x98, 0xc2, 0xf7,
+	0xb9, 0x00, 0x5f, 0x8c, 0x60, 0xd1, 0xe5, 0x15,
+	0xf7, 0xae, 0x1e, 0x84, 0x88, 0xd1, 0xf6, 0xbc,
+	0x3a, 0x89, 0x35, 0x22, 0x83, 0x7c, 0xca, 0xf0,
+	0x33, 0x82, 0x4c, 0x79, 0x3c, 0xfd, 0xb1, 0xae,
+	0x52, 0x62, 0x55, 0xd2, 0x41, 0x60, 0xc6, 0xbb,
+	0xfa, 0x0e, 0x59, 0xd6, 0xa8, 0xfe, 0x5d, 0xed,
+	0x47, 0x3d, 0xe0, 0xea, 0x1f, 0x6e, 0x43, 0x51,
+	0xec, 0x10, 0x52, 0x56, 0x77, 0x42, 0x6b, 0x52,
+	0x87, 0xd8, 0xec, 0xe0, 0xaa, 0x76, 0xa5, 0x84,
+	0x2a, 0x22, 0x24, 0xfd, 0x92, 0x40, 0x88, 0xd5,
+	0x85, 0x1c, 0x1f, 0x6b, 0x47, 0xa0, 0xc4, 0xe4,
+	0xef, 0xf4, 0xea, 0xd7, 0x59, 0xac, 0x2a, 0x9e,
+	0x8c, 0xfa, 0x1f, 0x42, 0x08, 0xfe, 0x4f, 0x74,
+	0xa0, 0x26, 0xf5, 0xb3, 0x84, 0xf6, 0x58, 0x5f,
+	0x26, 0x66, 0x3e, 0xd7, 0xe4, 0x22, 0x91, 0x13,
+	0xc8, 0xac, 0x25, 0x96, 0x23, 0xd8, 0x09, 0xea,
+	0x45, 0x75, 0x23, 0xb8, 0x5f, 0xc2, 0x90, 0x8b,
+	0x09, 0xc4, 0xfc, 0x47, 0x6c, 0x6d, 0x0a, 0xef,
+	0x69, 0xa4, 0x38, 0x19, 0xcf, 0x7d, 0xf9, 0x09,
+	0x73, 0x9b, 0x60, 0x5a, 0xf7, 0x37, 0xb5, 0xfe,
+	0x9f, 0xe3, 0x2b, 0x4c, 0x0d, 0x6e, 0x19, 0xf1,
+	0xd6, 0xc0, 0x70, 0xf3, 0x9d, 0x22, 0x3c, 0xf9,
+	0x49, 0xce, 0x30, 0x8e, 0x44, 0xb5, 0x76, 0x15,
+	0x8f, 0x52, 0xfd, 0xa5, 0x04, 0xb8, 0x55, 0x6a,
+	0x36, 0x59, 0x7c, 0xc4, 0x48, 0xb8, 0xd7, 0xab,
+	0x05, 0x66, 0xe9, 0x5e, 0x21, 0x6f, 0x6b, 0x36,
+	0x29, 0xbb, 0xe9, 0xe3, 0xa2, 0x9a, 0xa8, 0xcd,
+	0x55, 0x25, 0x11, 0xba, 0x5a, 0x58, 0xa0, 0xde,
+	0xae, 0x19, 0x2a, 0x48, 0x5a, 0xff, 0x36, 0xcd,
+	0x6d, 0x16, 0x7a, 0x73, 0x38, 0x46, 0xe5, 0x47,
+	0x59, 0xc8, 0xa2, 0xf6, 0xe2, 0x6c, 0x83, 0xc5,
+	0x36, 0x2c, 0x83, 0x7d, 0xb4, 0x01, 0x05, 0x69,
+	0xe7, 0xaf, 0x5c, 0xc4, 0x64, 0x82, 0x12, 0x21,
+	0xef, 0xf7, 0xd1, 0x7d, 0xb8, 0x8d, 0x8c, 0x98,
+	0x7c, 0x5f, 0x7d, 0x92, 0x88, 0xb9, 0x94, 0x07,
+	0x9c, 0xd8, 0xe9, 0x9c, 0x17, 0x38, 0xe3, 0x57,
+	0x6c, 0xe0, 0xdc, 0xa5, 0x92, 0x42, 0xb3, 0xbd,
+	0x50, 0xa2, 0x7e, 0xb5, 0xb1, 0x52, 0x72, 0x03,
+	0x97, 0xd8, 0xaa, 0x9a, 0x1e, 0x75, 0x41, 0x11,
+	0xa3, 0x4f, 0xcc, 0xd4, 0xe3, 0x73, 0xad, 0x96,
+	0xdc, 0x47, 0x41, 0x9f, 0xb0, 0xbe, 0x79, 0x91,
+	0xf5, 0xb6, 0x18, 0xfe, 0xc2, 0x83, 0x18, 0x7d,
+	0x73, 0xd9, 0x4f, 0x83, 0x84, 0x03, 0xb3, 0xf0,
+	0x77, 0x66, 0x3d, 0x83, 0x63, 0x2e, 0x2c, 0xf9,
+	0xdd, 0xa6, 0x1f, 0x89, 0x82, 0xb8, 0x23, 0x42,
+	0xeb, 0xe2, 0xca, 0x70, 0x82, 0x61, 0x41, 0x0a,
+	0x6d, 0x5f, 0x75, 0xc5, 0xe2, 0xc4, 0x91, 0x18,
+	0x44, 0x22, 0xfa, 0x34, 0x10, 0xf5, 0x20, 0xdc,
+	0xb7, 0xdd, 0x2a, 0x20, 0x77, 0xf5, 0xf9, 0xce,
+	0xdb, 0xa0, 0x0a, 0x52, 0x2a, 0x4e, 0xdd, 0xcc,
+	0x97, 0xdf, 0x05, 0xe4, 0x5e, 0xb7, 0xaa, 0xf0,
+	0xe2, 0x80, 0xff, 0xba, 0x1a, 0x0f, 0xac, 0xdf,
+	0x02, 0x32, 0xe6, 0xf7, 0xc7, 0x17, 0x13, 0xb7,
+	0xfc, 0x98, 0x48, 0x8c, 0x0d, 0x82, 0xc9, 0x80,
+	0x7a, 0xe2, 0x0a, 0xc5, 0xb4, 0xde, 0x7c, 0x3c,
+	0x79, 0x81, 0x0e, 0x28, 0x65, 0x79, 0x67, 0x82,
+	0x69, 0x44, 0x66, 0x09, 0xf7, 0x16, 0x1a, 0xf9,
+	0x7d, 0x80, 0xa1, 0x79, 0x14, 0xa9, 0xc8, 0x20,
+	0xfb, 0xa2, 0x46, 0xbe, 0x08, 0x35, 0x17, 0x58,
+	0xc1, 0x1a, 0xda, 0x2a, 0x6b, 0x2e, 0x1e, 0xe6,
+	0x27, 0x55, 0x7b, 0x19, 0xe2, 0xfb, 0x64, 0xfc,
+	0x5e, 0x15, 0x54, 0x3c, 0xe7, 0xc2, 0x11, 0x50,
+	0x30, 0xb8, 0x72, 0x03, 0x0b, 0x1a, 0x9f, 0x86,
+	0x27, 0x11, 0x5c, 0x06, 0x2b, 0xbd, 0x75, 0x1a,
+	0x0a, 0xda, 0x01, 0xfa, 0x5c, 0x4a, 0xc1, 0x80,
+	0x3a, 0x6e, 0x30, 0xc8, 0x2c, 0xeb, 0x56, 0xec,
+	0x89, 0xfa, 0x35, 0x7b, 0xb2, 0xf0, 0x97, 0x08,
+	0x86, 0x53, 0xbe, 0xbd, 0x40, 0x41, 0x38, 0x1c,
+	0xb4, 0x8b, 0x79, 0x2e, 0x18, 0x96, 0x94, 0xde,
+	0xe8, 0xca, 0xe5, 0x9f, 0x92, 0x9f, 0x15, 0x5d,
+	0x56, 0x60, 0x5c, 0x09, 0xf9, 0x16, 0xf4, 0x17,
+	0x0f, 0xf6, 0x4c, 0xda, 0xe6, 0x67, 0x89, 0x9f,
+	0xca, 0x6c, 0xe7, 0x9b, 0x04, 0x62, 0x0e, 0x26,
+	0xa6, 0x52, 0xbd, 0x29, 0xff, 0xc7, 0xa4, 0x96,
+	0xe6, 0x6a, 0x02, 0xa5, 0x2e, 0x7b, 0xfe, 0x97,
+	0x68, 0x3e, 0x2e, 0x5f, 0x3b, 0x0f, 0x36, 0xd6,
+	0x98, 0x19, 0x59, 0x48, 0xd2, 0xc6, 0xe1, 0x55,
+	0x1a, 0x6e, 0xd6, 0xed, 0x2c, 0xba, 0xc3, 0x9e,
+	0x64, 0xc9, 0x95, 0x86, 0x35, 0x5e, 0x3e, 0x88,
+	0x69, 0x99, 0x4b, 0xee, 0xbe, 0x9a, 0x99, 0xb5,
+	0x6e, 0x58, 0xae, 0xdd, 0x22, 0xdb, 0xdd, 0x6b,
+	0xfc, 0xaf, 0x90, 0xa3, 0x3d, 0xa4, 0xc1, 0x15,
+	0x92, 0x18, 0x8d, 0xd2, 0x4b, 0x7b, 0x06, 0xd1,
+	0x37, 0xb5, 0xe2, 0x7c, 0x2c, 0xf0, 0x25, 0xe4,
+	0x94, 0x2a, 0xbd, 0xe3, 0x82, 0x70, 0x78, 0xa3,
+	0x82, 0x10, 0x5a, 0x90, 0xd7, 0xa4, 0xfa, 0xaf,
+	0x1a, 0x88, 0x59, 0xdc, 0x74, 0x12, 0xb4, 0x8e,
+	0xd7, 0x19, 0x46, 0xf4, 0x84, 0x69, 0x9f, 0xbb,
+	0x70, 0xa8, 0x4c, 0x52, 0x81, 0xa9, 0xff, 0x76,
+	0x1c, 0xae, 0xd8, 0x11, 0x3d, 0x7f, 0x7d, 0xc5,
+	0x12, 0x59, 0x28, 0x18, 0xc2, 0xa2, 0xb7, 0x1c,
+	0x88, 0xf8, 0xd6, 0x1b, 0xa6, 0x7d, 0x9e, 0xde,
+	0x29, 0xf8, 0xed, 0xff, 0xeb, 0x92, 0x24, 0x4f,
+	0x05, 0xaa, 0xd9, 0x49, 0xba, 0x87, 0x59, 0x51,
+	0xc9, 0x20, 0x5c, 0x9b, 0x74, 0xcf, 0x03, 0xd9,
+	0x2d, 0x34, 0xc7, 0x5b, 0xa5, 0x40, 0xb2, 0x99,
+	0xf5, 0xcb, 0xb4, 0xf6, 0xb7, 0x72, 0x4a, 0xd6,
+	0xbd, 0xb0, 0xf3, 0x93, 0xe0, 0x1b, 0xa8, 0x04,
+	0x1e, 0x35, 0xd4, 0x80, 0x20, 0xf4, 0x9c, 0x31,
+	0x6b, 0x45, 0xb9, 0x15, 0xb0, 0x5e, 0xdd, 0x0a,
+	0x33, 0x9c, 0x83, 0xcd, 0x58, 0x89, 0x50, 0x56,
+	0xbb, 0x81, 0x00, 0x91, 0x32, 0xf3, 0x1b, 0x3e,
+	0xcf, 0x45, 0xe1, 0xf9, 0xe1, 0x2c, 0x26, 0x78,
+	0x93, 0x9a, 0x60, 0x46, 0xc9, 0xb5, 0x5e, 0x6a,
+	0x28, 0x92, 0x87, 0x3f, 0x63, 0x7b, 0xdb, 0xf7,
+	0xd0, 0x13, 0x9d, 0x32, 0x40, 0x5e, 0xcf, 0xfb,
+	0x79, 0x68, 0x47, 0x4c, 0xfd, 0x01, 0x17, 0xe6,
+	0x97, 0x93, 0x78, 0xbb, 0xa6, 0x27, 0xa3, 0xe8,
+	0x1a, 0xe8, 0x94, 0x55, 0x7d, 0x08, 0xe5, 0xdc,
+	0x66, 0xa3, 0x69, 0xc8, 0xca, 0xc5, 0xa1, 0x84,
+	0x55, 0xde, 0x08, 0x91, 0x16, 0x3a, 0x0c, 0x86,
+	0xab, 0x27, 0x2b, 0x64, 0x34, 0x02, 0x6c, 0x76,
+	0x8b, 0xc6, 0xaf, 0xcc, 0xe1, 0xd6, 0x8c, 0x2a,
+	0x18, 0x3d, 0xa6, 0x1b, 0x37, 0x75, 0x45, 0x73,
+	0xc2, 0x75, 0xd7, 0x53, 0x78, 0x3a, 0xd6, 0xe8,
+	0x29, 0xd2, 0x4a, 0xa8, 0x1e, 0x82, 0xf6, 0xb6,
+	0x81, 0xde, 0x21, 0xed, 0x2b, 0x56, 0xbb, 0xf2,
+	0xd0, 0x57, 0xc1, 0x7c, 0xd2, 0x6a, 0xd2, 0x56,
+	0xf5, 0x13, 0x5f, 0x1c, 0x6a, 0x0b, 0x74, 0xfb,
+	0xe9, 0xfe, 0x9e, 0xea, 0x95, 0xb2, 0x46, 0xab,
+	0x0a, 0xfc, 0xfd, 0xf3, 0xbb, 0x04, 0x2b, 0x76,
+	0x1b, 0xa4, 0x74, 0xb0, 0xc1, 0x78, 0xc3, 0x69,
+	0xe2, 0xb0, 0x01, 0xe1, 0xde, 0x32, 0x4c, 0x8d,
+	0x1a, 0xb3, 0x38, 0x08, 0xd5, 0xfc, 0x1f, 0xdc,
+	0x0e, 0x2c, 0x9c, 0xb1, 0xa1, 0x63, 0x17, 0x22,
+	0xf5, 0x6c, 0x93, 0x70, 0x74, 0x00, 0xf8, 0x39,
+	0x01, 0x94, 0xd1, 0x32, 0x23, 0x56, 0x5d, 0xa6,
+	0x02, 0x76, 0x76, 0x93, 0xce, 0x2f, 0x19, 0xe9,
+	0x17, 0x52, 0xae, 0x6e, 0x2c, 0x6d, 0x61, 0x7f,
+	0x3b, 0xaa, 0xe0, 0x52, 0x85, 0xc5, 0x65, 0xc1,
+	0xbb, 0x8e, 0x5b, 0x21, 0xd5, 0xc9, 0x78, 0x83,
+	0x07, 0x97, 0x4c, 0x62, 0x61, 0x41, 0xd4, 0xfc,
+	0xc9, 0x39, 0xe3, 0x9b, 0xd0, 0xcc, 0x75, 0xc4,
+	0x97, 0xe6, 0xdd, 0x2a, 0x5f, 0xa6, 0xe8, 0x59,
+	0x6c, 0x98, 0xb9, 0x02, 0xe2, 0xa2, 0xd6, 0x68,
+	0xee, 0x3b, 0x1d, 0xe3, 0x4d, 0x5b, 0x30, 0xef,
+	0x03, 0xf2, 0xeb, 0x18, 0x57, 0x36, 0xe8, 0xa1,
+	0xf4, 0x47, 0xfb, 0xcb, 0x8f, 0xcb, 0xc8, 0xf3,
+	0x4f, 0x74, 0x9d, 0x9d, 0xb1, 0x8d, 0x14, 0x44,
+	0xd9, 0x19, 0xb4, 0x54, 0x4f, 0x75, 0x19, 0x09,
+	0xa0, 0x75, 0xbc, 0x3b, 0x82, 0xc6, 0x3f, 0xb8,
+	0x83, 0x19, 0x6e, 0xd6, 0x37, 0xfe, 0x6e, 0x8a,
+	0x4e, 0xe0, 0x4a, 0xab, 0x7b, 0xc8, 0xb4, 0x1d,
+	0xf4, 0xed, 0x27, 0x03, 0x65, 0xa2, 0xa1, 0xae,
+	0x11, 0xe7, 0x98, 0x78, 0x48, 0x91, 0xd2, 0xd2,
+	0xd4, 0x23, 0x78, 0x50, 0xb1, 0x5b, 0x85, 0x10,
+	0x8d, 0xca, 0x5f, 0x0f, 0x71, 0xae, 0x72, 0x9a,
+	0xf6, 0x25, 0x19, 0x60, 0x06, 0xf7, 0x10, 0x34,
+	0x18, 0x0d, 0xc9, 0x9f, 0x7b, 0x0c, 0x9b, 0x8f,
+	0x91, 0x1b, 0x9f, 0xcd, 0x10, 0xee, 0x75, 0xf9,
+	0x97, 0x66, 0xfc, 0x4d, 0x33, 0x6e, 0x28, 0x2b,
+	0x92, 0x85, 0x4f, 0xab, 0x43, 0x8d, 0x8f, 0x7d,
+	0x86, 0xa7, 0xc7, 0xd8, 0xd3, 0x0b, 0x8b, 0x57,
+	0xb6, 0x1d, 0x95, 0x0d, 0xe9, 0xbc, 0xd9, 0x03,
+	0xd9, 0x10, 0x19, 0xc3, 0x46, 0x63, 0x55, 0x87,
+	0x61, 0x79, 0x6c, 0x95, 0x0e, 0x9c, 0xdd, 0xca,
+	0xc3, 0xf3, 0x64, 0xf0, 0x7d, 0x76, 0xb7, 0x53,
+	0x67, 0x2b, 0x1e, 0x44, 0x56, 0x81, 0xea, 0x8f,
+	0x5c, 0x42, 0x16, 0xb8, 0x28, 0xeb, 0x1b, 0x61,
+	0x10, 0x1e, 0xbf, 0xec, 0xa8
+};
+static const u8 dec_assoc011[] __initconst = {
+	0xd6, 0x31, 0xda, 0x5d, 0x42, 0x5e, 0xd7
+};
+static const u8 dec_nonce011[] __initconst = {
+	0xfd, 0x87, 0xd4, 0xd8, 0x62, 0xfd, 0xec, 0xaa
+};
+static const u8 dec_key011[] __initconst = {
+	0x35, 0x4e, 0xb5, 0x70, 0x50, 0x42, 0x8a, 0x85,
+	0xf2, 0xfb, 0xed, 0x7b, 0xd0, 0x9e, 0x97, 0xca,
+	0xfa, 0x98, 0x66, 0x63, 0xee, 0x37, 0xcc, 0x52,
+	0xfe, 0xd1, 0xdf, 0x95, 0x15, 0x34, 0x29, 0x38
+};
+
+static const u8 dec_input012[] __initconst = {
+	0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3,
+	0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf,
+	0xdf, 0x34, 0x56, 0x82, 0xe2, 0xbe, 0xe5, 0xe1,
+	0x28, 0xd1, 0x4e, 0x5f, 0x4f, 0x01, 0x7d, 0x3f,
+	0x99, 0x6b, 0x30, 0x6e, 0x1a, 0x7c, 0x4c, 0x8e,
+	0x62, 0x81, 0xae, 0x86, 0x3f, 0x6b, 0xd0, 0xb5,
+	0xa9, 0xcf, 0x50, 0xf1, 0x02, 0x12, 0xa0, 0x0b,
+	0x24, 0xe9, 0xe6, 0x72, 0x89, 0x2c, 0x52, 0x1b,
+	0x34, 0x38, 0xf8, 0x75, 0x5f, 0xa0, 0x74, 0xe2,
+	0x99, 0xdd, 0xa6, 0x4b, 0x14, 0x50, 0x4e, 0xf1,
+	0xbe, 0xd6, 0x9e, 0xdb, 0xb2, 0x24, 0x27, 0x74,
+	0x12, 0x4a, 0x78, 0x78, 0x17, 0xa5, 0x58, 0x8e,
+	0x2f, 0xf9, 0xf4, 0x8d, 0xee, 0x03, 0x88, 0xae,
+	0xb8, 0x29, 0xa1, 0x2f, 0x4b, 0xee, 0x92, 0xbd,
+	0x87, 0xb3, 0xce, 0x34, 0x21, 0x57, 0x46, 0x04,
+	0x49, 0x0c, 0x80, 0xf2, 0x01, 0x13, 0xa1, 0x55,
+	0xb3, 0xff, 0x44, 0x30, 0x3c, 0x1c, 0xd0, 0xef,
+	0xbc, 0x18, 0x74, 0x26, 0xad, 0x41, 0x5b, 0x5b,
+	0x3e, 0x9a, 0x7a, 0x46, 0x4f, 0x16, 0xd6, 0x74,
+	0x5a, 0xb7, 0x3a, 0x28, 0x31, 0xd8, 0xae, 0x26,
+	0xac, 0x50, 0x53, 0x86, 0xf2, 0x56, 0xd7, 0x3f,
+	0x29, 0xbc, 0x45, 0x68, 0x8e, 0xcb, 0x98, 0x64,
+	0xdd, 0xc9, 0xba, 0xb8, 0x4b, 0x7b, 0x82, 0xdd,
+	0x14, 0xa7, 0xcb, 0x71, 0x72, 0x00, 0x5c, 0xad,
+	0x7b, 0x6a, 0x89, 0xa4, 0x3d, 0xbf, 0xb5, 0x4b,
+	0x3e, 0x7c, 0x5a, 0xcf, 0xb8, 0xa1, 0xc5, 0x6e,
+	0xc8, 0xb6, 0x31, 0x57, 0x7b, 0xdf, 0xa5, 0x7e,
+	0xb1, 0xd6, 0x42, 0x2a, 0x31, 0x36, 0xd1, 0xd0,
+	0x3f, 0x7a, 0xe5, 0x94, 0xd6, 0x36, 0xa0, 0x6f,
+	0xb7, 0x40, 0x7d, 0x37, 0xc6, 0x55, 0x7c, 0x50,
+	0x40, 0x6d, 0x29, 0x89, 0xe3, 0x5a, 0xae, 0x97,
+	0xe7, 0x44, 0x49, 0x6e, 0xbd, 0x81, 0x3d, 0x03,
+	0x93, 0x06, 0x12, 0x06, 0xe2, 0x41, 0x12, 0x4a,
+	0xf1, 0x6a, 0xa4, 0x58, 0xa2, 0xfb, 0xd2, 0x15,
+	0xba, 0xc9, 0x79, 0xc9, 0xce, 0x5e, 0x13, 0xbb,
+	0xf1, 0x09, 0x04, 0xcc, 0xfd, 0xe8, 0x51, 0x34,
+	0x6a, 0xe8, 0x61, 0x88, 0xda, 0xed, 0x01, 0x47,
+	0x84, 0xf5, 0x73, 0x25, 0xf9, 0x1c, 0x42, 0x86,
+	0x07, 0xf3, 0x5b, 0x1a, 0x01, 0xb3, 0xeb, 0x24,
+	0x32, 0x8d, 0xf6, 0xed, 0x7c, 0x4b, 0xeb, 0x3c,
+	0x36, 0x42, 0x28, 0xdf, 0xdf, 0xb6, 0xbe, 0xd9,
+	0x8c, 0x52, 0xd3, 0x2b, 0x08, 0x90, 0x8c, 0xe7,
+	0x98, 0x31, 0xe2, 0x32, 0x8e, 0xfc, 0x11, 0x48,
+	0x00, 0xa8, 0x6a, 0x42, 0x4a, 0x02, 0xc6, 0x4b,
+	0x09, 0xf1, 0xe3, 0x49, 0xf3, 0x45, 0x1f, 0x0e,
+	0xbc, 0x56, 0xe2, 0xe4, 0xdf, 0xfb, 0xeb, 0x61,
+	0xfa, 0x24, 0xc1, 0x63, 0x75, 0xbb, 0x47, 0x75,
+	0xaf, 0xe1, 0x53, 0x16, 0x96, 0x21, 0x85, 0x26,
+	0x11, 0xb3, 0x76, 0xe3, 0x23, 0xa1, 0x6b, 0x74,
+	0x37, 0xd0, 0xde, 0x06, 0x90, 0x71, 0x5d, 0x43,
+	0x88, 0x9b, 0x00, 0x54, 0xa6, 0x75, 0x2f, 0xa1,
+	0xc2, 0x0b, 0x73, 0x20, 0x1d, 0xb6, 0x21, 0x79,
+	0x57, 0x3f, 0xfa, 0x09, 0xbe, 0x8a, 0x33, 0xc3,
+	0x52, 0xf0, 0x1d, 0x82, 0x31, 0xd1, 0x55, 0xb5,
+	0x6c, 0x99, 0x25, 0xcf, 0x5c, 0x32, 0xce, 0xe9,
+	0x0d, 0xfa, 0x69, 0x2c, 0xd5, 0x0d, 0xc5, 0x6d,
+	0x86, 0xd0, 0x0c, 0x3b, 0x06, 0x50, 0x79, 0xe8,
+	0xc3, 0xae, 0x04, 0xe6, 0xcd, 0x51, 0xe4, 0x26,
+	0x9b, 0x4f, 0x7e, 0xa6, 0x0f, 0xab, 0xd8, 0xe5,
+	0xde, 0xa9, 0x00, 0x95, 0xbe, 0xa3, 0x9d, 0x5d,
+	0xb2, 0x09, 0x70, 0x18, 0x1c, 0xf0, 0xac, 0x29,
+	0x23, 0x02, 0x29, 0x28, 0xd2, 0x74, 0x35, 0x57,
+	0x62, 0x0f, 0x24, 0xea, 0x5e, 0x33, 0xc2, 0x92,
+	0xf3, 0x78, 0x4d, 0x30, 0x1e, 0xa1, 0x99, 0xa9,
+	0x82, 0xb0, 0x42, 0x31, 0x8d, 0xad, 0x8a, 0xbc,
+	0xfc, 0xd4, 0x57, 0x47, 0x3e, 0xb4, 0x50, 0xdd,
+	0x6e, 0x2c, 0x80, 0x4d, 0x22, 0xf1, 0xfb, 0x57,
+	0xc4, 0xdd, 0x17, 0xe1, 0x8a, 0x36, 0x4a, 0xb3,
+	0x37, 0xca, 0xc9, 0x4e, 0xab, 0xd5, 0x69, 0xc4,
+	0xf4, 0xbc, 0x0b, 0x3b, 0x44, 0x4b, 0x29, 0x9c,
+	0xee, 0xd4, 0x35, 0x22, 0x21, 0xb0, 0x1f, 0x27,
+	0x64, 0xa8, 0x51, 0x1b, 0xf0, 0x9f, 0x19, 0x5c,
+	0xfb, 0x5a, 0x64, 0x74, 0x70, 0x45, 0x09, 0xf5,
+	0x64, 0xfe, 0x1a, 0x2d, 0xc9, 0x14, 0x04, 0x14,
+	0xcf, 0xd5, 0x7d, 0x60, 0xaf, 0x94, 0x39, 0x94,
+	0xe2, 0x7d, 0x79, 0x82, 0xd0, 0x65, 0x3b, 0x6b,
+	0x9c, 0x19, 0x84, 0xb4, 0x6d, 0xb3, 0x0c, 0x99,
+	0xc0, 0x56, 0xa8, 0xbd, 0x73, 0xce, 0x05, 0x84,
+	0x3e, 0x30, 0xaa, 0xc4, 0x9b, 0x1b, 0x04, 0x2a,
+	0x9f, 0xd7, 0x43, 0x2b, 0x23, 0xdf, 0xbf, 0xaa,
+	0xd5, 0xc2, 0x43, 0x2d, 0x70, 0xab, 0xdc, 0x75,
+	0xad, 0xac, 0xf7, 0xc0, 0xbe, 0x67, 0xb2, 0x74,
+	0xed, 0x67, 0x10, 0x4a, 0x92, 0x60, 0xc1, 0x40,
+	0x50, 0x19, 0x8a, 0x8a, 0x8c, 0x09, 0x0e, 0x72,
+	0xe1, 0x73, 0x5e, 0xe8, 0x41, 0x85, 0x63, 0x9f,
+	0x3f, 0xd7, 0x7d, 0xc4, 0xfb, 0x22, 0x5d, 0x92,
+	0x6c, 0xb3, 0x1e, 0xe2, 0x50, 0x2f, 0x82, 0xa8,
+	0x28, 0xc0, 0xb5, 0xd7, 0x5f, 0x68, 0x0d, 0x2c,
+	0x2d, 0xaf, 0x7e, 0xfa, 0x2e, 0x08, 0x0f, 0x1f,
+	0x70, 0x9f, 0xe9, 0x19, 0x72, 0x55, 0xf8, 0xfb,
+	0x51, 0xd2, 0x33, 0x5d, 0xa0, 0xd3, 0x2b, 0x0a,
+	0x6c, 0xbc, 0x4e, 0xcf, 0x36, 0x4d, 0xdc, 0x3b,
+	0xe9, 0x3e, 0x81, 0x7c, 0x61, 0xdb, 0x20, 0x2d,
+	0x3a, 0xc3, 0xb3, 0x0c, 0x1e, 0x00, 0xb9, 0x7c,
+	0xf5, 0xca, 0x10, 0x5f, 0x3a, 0x71, 0xb3, 0xe4,
+	0x20, 0xdb, 0x0c, 0x2a, 0x98, 0x63, 0x45, 0x00,
+	0x58, 0xf6, 0x68, 0xe4, 0x0b, 0xda, 0x13, 0x3b,
+	0x60, 0x5c, 0x76, 0xdb, 0xb9, 0x97, 0x71, 0xe4,
+	0xd9, 0xb7, 0xdb, 0xbd, 0x68, 0xc7, 0x84, 0x84,
+	0xaa, 0x7c, 0x68, 0x62, 0x5e, 0x16, 0xfc, 0xba,
+	0x72, 0xaa, 0x9a, 0xa9, 0xeb, 0x7c, 0x75, 0x47,
+	0x97, 0x7e, 0xad, 0xe2, 0xd9, 0x91, 0xe8, 0xe4,
+	0xa5, 0x31, 0xd7, 0x01, 0x8e, 0xa2, 0x11, 0x88,
+	0x95, 0xb9, 0xf2, 0x9b, 0xd3, 0x7f, 0x1b, 0x81,
+	0x22, 0xf7, 0x98, 0x60, 0x0a, 0x64, 0xa6, 0xc1,
+	0xf6, 0x49, 0xc7, 0xe3, 0x07, 0x4d, 0x94, 0x7a,
+	0xcf, 0x6e, 0x68, 0x0c, 0x1b, 0x3f, 0x6e, 0x2e,
+	0xee, 0x92, 0xfa, 0x52, 0xb3, 0x59, 0xf8, 0xf1,
+	0x8f, 0x6a, 0x66, 0xa3, 0x82, 0x76, 0x4a, 0x07,
+	0x1a, 0xc7, 0xdd, 0xf5, 0xda, 0x9c, 0x3c, 0x24,
+	0xbf, 0xfd, 0x42, 0xa1, 0x10, 0x64, 0x6a, 0x0f,
+	0x89, 0xee, 0x36, 0xa5, 0xce, 0x99, 0x48, 0x6a,
+	0xf0, 0x9f, 0x9e, 0x69, 0xa4, 0x40, 0x20, 0xe9,
+	0x16, 0x15, 0xf7, 0xdb, 0x75, 0x02, 0xcb, 0xe9,
+	0x73, 0x8b, 0x3b, 0x49, 0x2f, 0xf0, 0xaf, 0x51,
+	0x06, 0x5c, 0xdf, 0x27, 0x27, 0x49, 0x6a, 0xd1,
+	0xcc, 0xc7, 0xb5, 0x63, 0xb5, 0xfc, 0xb8, 0x5c,
+	0x87, 0x7f, 0x84, 0xb4, 0xcc, 0x14, 0xa9, 0x53,
+	0xda, 0xa4, 0x56, 0xf8, 0xb6, 0x1b, 0xcc, 0x40,
+	0x27, 0x52, 0x06, 0x5a, 0x13, 0x81, 0xd7, 0x3a,
+	0xd4, 0x3b, 0xfb, 0x49, 0x65, 0x31, 0x33, 0xb2,
+	0xfa, 0xcd, 0xad, 0x58, 0x4e, 0x2b, 0xae, 0xd2,
+	0x20, 0xfb, 0x1a, 0x48, 0xb4, 0x3f, 0x9a, 0xd8,
+	0x7a, 0x35, 0x4a, 0xc8, 0xee, 0x88, 0x5e, 0x07,
+	0x66, 0x54, 0xb9, 0xec, 0x9f, 0xa3, 0xe3, 0xb9,
+	0x37, 0xaa, 0x49, 0x76, 0x31, 0xda, 0x74, 0x2d,
+	0x3c, 0xa4, 0x65, 0x10, 0x32, 0x38, 0xf0, 0xde,
+	0xd3, 0x99, 0x17, 0xaa, 0x71, 0xaa, 0x8f, 0x0f,
+	0x8c, 0xaf, 0xa2, 0xf8, 0x5d, 0x64, 0xba, 0x1d,
+	0xa3, 0xef, 0x96, 0x73, 0xe8, 0xa1, 0x02, 0x8d,
+	0x0c, 0x6d, 0xb8, 0x06, 0x90, 0xb8, 0x08, 0x56,
+	0x2c, 0xa7, 0x06, 0xc9, 0xc2, 0x38, 0xdb, 0x7c,
+	0x63, 0xb1, 0x57, 0x8e, 0xea, 0x7c, 0x79, 0xf3,
+	0x49, 0x1d, 0xfe, 0x9f, 0xf3, 0x6e, 0xb1, 0x1d,
+	0xba, 0x19, 0x80, 0x1a, 0x0a, 0xd3, 0xb0, 0x26,
+	0x21, 0x40, 0xb1, 0x7c, 0xf9, 0x4d, 0x8d, 0x10,
+	0xc1, 0x7e, 0xf4, 0xf6, 0x3c, 0xa8, 0xfd, 0x7c,
+	0xa3, 0x92, 0xb2, 0x0f, 0xaa, 0xcc, 0xa6, 0x11,
+	0xfe, 0x04, 0xe3, 0xd1, 0x7a, 0x32, 0x89, 0xdf,
+	0x0d, 0xc4, 0x8f, 0x79, 0x6b, 0xca, 0x16, 0x7c,
+	0x6e, 0xf9, 0xad, 0x0f, 0xf6, 0xfe, 0x27, 0xdb,
+	0xc4, 0x13, 0x70, 0xf1, 0x62, 0x1a, 0x4f, 0x79,
+	0x40, 0xc9, 0x9b, 0x8b, 0x21, 0xea, 0x84, 0xfa,
+	0xf5, 0xf1, 0x89, 0xce, 0xb7, 0x55, 0x0a, 0x80,
+	0x39, 0x2f, 0x55, 0x36, 0x16, 0x9c, 0x7b, 0x08,
+	0xbd, 0x87, 0x0d, 0xa5, 0x32, 0xf1, 0x52, 0x7c,
+	0xe8, 0x55, 0x60, 0x5b, 0xd7, 0x69, 0xe4, 0xfc,
+	0xfa, 0x12, 0x85, 0x96, 0xea, 0x50, 0x28, 0xab,
+	0x8a, 0xf7, 0xbb, 0x0e, 0x53, 0x74, 0xca, 0xa6,
+	0x27, 0x09, 0xc2, 0xb5, 0xde, 0x18, 0x14, 0xd9,
+	0xea, 0xe5, 0x29, 0x1c, 0x40, 0x56, 0xcf, 0xd7,
+	0xae, 0x05, 0x3f, 0x65, 0xaf, 0x05, 0x73, 0xe2,
+	0x35, 0x96, 0x27, 0x07, 0x14, 0xc0, 0xad, 0x33,
+	0xf1, 0xdc, 0x44, 0x7a, 0x89, 0x17, 0x77, 0xd2,
+	0x9c, 0x58, 0x60, 0xf0, 0x3f, 0x7b, 0x2d, 0x2e,
+	0x57, 0x95, 0x54, 0x87, 0xed, 0xf2, 0xc7, 0x4c,
+	0xf0, 0xae, 0x56, 0x29, 0x19, 0x7d, 0x66, 0x4b,
+	0x9b, 0x83, 0x84, 0x42, 0x3b, 0x01, 0x25, 0x66,
+	0x8e, 0x02, 0xde, 0xb9, 0x83, 0x54, 0x19, 0xf6,
+	0x9f, 0x79, 0x0d, 0x67, 0xc5, 0x1d, 0x7a, 0x44,
+	0x02, 0x98, 0xa7, 0x16, 0x1c, 0x29, 0x0d, 0x74,
+	0xff, 0x85, 0x40, 0x06, 0xef, 0x2c, 0xa9, 0xc6,
+	0xf5, 0x53, 0x07, 0x06, 0xae, 0xe4, 0xfa, 0x5f,
+	0xd8, 0x39, 0x4d, 0xf1, 0x9b, 0x6b, 0xd9, 0x24,
+	0x84, 0xfe, 0x03, 0x4c, 0xb2, 0x3f, 0xdf, 0xa1,
+	0x05, 0x9e, 0x50, 0x14, 0x5a, 0xd9, 0x1a, 0xa2,
+	0xa7, 0xfa, 0xfa, 0x17, 0xf7, 0x78, 0xd6, 0xb5,
+	0x92, 0x61, 0x91, 0xac, 0x36, 0xfa, 0x56, 0x0d,
+	0x38, 0x32, 0x18, 0x85, 0x08, 0x58, 0x37, 0xf0,
+	0x4b, 0xdb, 0x59, 0xe7, 0xa4, 0x34, 0xc0, 0x1b,
+	0x01, 0xaf, 0x2d, 0xde, 0xa1, 0xaa, 0x5d, 0xd3,
+	0xec, 0xe1, 0xd4, 0xf7, 0xe6, 0x54, 0x68, 0xf0,
+	0x51, 0x97, 0xa7, 0x89, 0xea, 0x24, 0xad, 0xd3,
+	0x6e, 0x47, 0x93, 0x8b, 0x4b, 0xb4, 0xf7, 0x1c,
+	0x42, 0x06, 0x67, 0xe8, 0x99, 0xf6, 0xf5, 0x7b,
+	0x85, 0xb5, 0x65, 0xb5, 0xb5, 0xd2, 0x37, 0xf5,
+	0xf3, 0x02, 0xa6, 0x4d, 0x11, 0xa7, 0xdc, 0x51,
+	0x09, 0x7f, 0xa0, 0xd8, 0x88, 0x1c, 0x13, 0x71,
+	0xae, 0x9c, 0xb7, 0x7b, 0x34, 0xd6, 0x4e, 0x68,
+	0x26, 0x83, 0x51, 0xaf, 0x1d, 0xee, 0x8b, 0xbb,
+	0x69, 0x43, 0x2b, 0x9e, 0x8a, 0xbc, 0x02, 0x0e,
+	0xa0, 0x1b, 0xe0, 0xa8, 0x5f, 0x6f, 0xaf, 0x1b,
+	0x8f, 0xe7, 0x64, 0x71, 0x74, 0x11, 0x7e, 0xa8,
+	0xd8, 0xf9, 0x97, 0x06, 0xc3, 0xb6, 0xfb, 0xfb,
+	0xb7, 0x3d, 0x35, 0x9d, 0x3b, 0x52, 0xed, 0x54,
+	0xca, 0xf4, 0x81, 0x01, 0x2d, 0x1b, 0xc3, 0xa7,
+	0x00, 0x3d, 0x1a, 0x39, 0x54, 0xe1, 0xf6, 0xff,
+	0xed, 0x6f, 0x0b, 0x5a, 0x68, 0xda, 0x58, 0xdd,
+	0xa9, 0xcf, 0x5c, 0x4a, 0xe5, 0x09, 0x4e, 0xde,
+	0x9d, 0xbc, 0x3e, 0xee, 0x5a, 0x00, 0x3b, 0x2c,
+	0x87, 0x10, 0x65, 0x60, 0xdd, 0xd7, 0x56, 0xd1,
+	0x4c, 0x64, 0x45, 0xe4, 0x21, 0xec, 0x78, 0xf8,
+	0x25, 0x7a, 0x3e, 0x16, 0x5d, 0x09, 0x53, 0x14,
+	0xbe, 0x4f, 0xae, 0x87, 0xd8, 0xd1, 0xaa, 0x3c,
+	0xf6, 0x3e, 0xa4, 0x70, 0x8c, 0x5e, 0x70, 0xa4,
+	0xb3, 0x6b, 0x66, 0x73, 0xd3, 0xbf, 0x31, 0x06,
+	0x19, 0x62, 0x93, 0x15, 0xf2, 0x86, 0xe4, 0x52,
+	0x7e, 0x53, 0x4c, 0x12, 0x38, 0xcc, 0x34, 0x7d,
+	0x57, 0xf6, 0x42, 0x93, 0x8a, 0xc4, 0xee, 0x5c,
+	0x8a, 0xe1, 0x52, 0x8f, 0x56, 0x64, 0xf6, 0xa6,
+	0xd1, 0x91, 0x57, 0x70, 0xcd, 0x11, 0x76, 0xf5,
+	0x59, 0x60, 0x60, 0x3c, 0xc1, 0xc3, 0x0b, 0x7f,
+	0x58, 0x1a, 0x50, 0x91, 0xf1, 0x68, 0x8f, 0x6e,
+	0x74, 0x74, 0xa8, 0x51, 0x0b, 0xf7, 0x7a, 0x98,
+	0x37, 0xf2, 0x0a, 0x0e, 0xa4, 0x97, 0x04, 0xb8,
+	0x9b, 0xfd, 0xa0, 0xea, 0xf7, 0x0d, 0xe1, 0xdb,
+	0x03, 0xf0, 0x31, 0x29, 0xf8, 0xdd, 0x6b, 0x8b,
+	0x5d, 0xd8, 0x59, 0xa9, 0x29, 0xcf, 0x9a, 0x79,
+	0x89, 0x19, 0x63, 0x46, 0x09, 0x79, 0x6a, 0x11,
+	0xda, 0x63, 0x68, 0x48, 0x77, 0x23, 0xfb, 0x7d,
+	0x3a, 0x43, 0xcb, 0x02, 0x3b, 0x7a, 0x6d, 0x10,
+	0x2a, 0x9e, 0xac, 0xf1, 0xd4, 0x19, 0xf8, 0x23,
+	0x64, 0x1d, 0x2c, 0x5f, 0xf2, 0xb0, 0x5c, 0x23,
+	0x27, 0xf7, 0x27, 0x30, 0x16, 0x37, 0xb1, 0x90,
+	0xab, 0x38, 0xfb, 0x55, 0xcd, 0x78, 0x58, 0xd4,
+	0x7d, 0x43, 0xf6, 0x45, 0x5e, 0x55, 0x8d, 0xb1,
+	0x02, 0x65, 0x58, 0xb4, 0x13, 0x4b, 0x36, 0xf7,
+	0xcc, 0xfe, 0x3d, 0x0b, 0x82, 0xe2, 0x12, 0x11,
+	0xbb, 0xe6, 0xb8, 0x3a, 0x48, 0x71, 0xc7, 0x50,
+	0x06, 0x16, 0x3a, 0xe6, 0x7c, 0x05, 0xc7, 0xc8,
+	0x4d, 0x2f, 0x08, 0x6a, 0x17, 0x9a, 0x95, 0x97,
+	0x50, 0x68, 0xdc, 0x28, 0x18, 0xc4, 0x61, 0x38,
+	0xb9, 0xe0, 0x3e, 0x78, 0xdb, 0x29, 0xe0, 0x9f,
+	0x52, 0xdd, 0xf8, 0x4f, 0x91, 0xc1, 0xd0, 0x33,
+	0xa1, 0x7a, 0x8e, 0x30, 0x13, 0x82, 0x07, 0x9f,
+	0xd3, 0x31, 0x0f, 0x23, 0xbe, 0x32, 0x5a, 0x75,
+	0xcf, 0x96, 0xb2, 0xec, 0xb5, 0x32, 0xac, 0x21,
+	0xd1, 0x82, 0x33, 0xd3, 0x15, 0x74, 0xbd, 0x90,
+	0xf1, 0x2c, 0xe6, 0x5f, 0x8d, 0xe3, 0x02, 0xe8,
+	0xe9, 0xc4, 0xca, 0x96, 0xeb, 0x0e, 0xbc, 0x91,
+	0xf4, 0xb9, 0xea, 0xd9, 0x1b, 0x75, 0xbd, 0xe1,
+	0xac, 0x2a, 0x05, 0x37, 0x52, 0x9b, 0x1b, 0x3f,
+	0x5a, 0xdc, 0x21, 0xc3, 0x98, 0xbb, 0xaf, 0xa3,
+	0xf2, 0x00, 0xbf, 0x0d, 0x30, 0x89, 0x05, 0xcc,
+	0xa5, 0x76, 0xf5, 0x06, 0xf0, 0xc6, 0x54, 0x8a,
+	0x5d, 0xd4, 0x1e, 0xc1, 0xf2, 0xce, 0xb0, 0x62,
+	0xc8, 0xfc, 0x59, 0x42, 0x9a, 0x90, 0x60, 0x55,
+	0xfe, 0x88, 0xa5, 0x8b, 0xb8, 0x33, 0x0c, 0x23,
+	0x24, 0x0d, 0x15, 0x70, 0x37, 0x1e, 0x3d, 0xf6,
+	0xd2, 0xea, 0x92, 0x10, 0xb2, 0xc4, 0x51, 0xac,
+	0xf2, 0xac, 0xf3, 0x6b, 0x6c, 0xaa, 0xcf, 0x12,
+	0xc5, 0x6c, 0x90, 0x50, 0xb5, 0x0c, 0xfc, 0x1a,
+	0x15, 0x52, 0xe9, 0x26, 0xc6, 0x52, 0xa4, 0xe7,
+	0x81, 0x69, 0xe1, 0xe7, 0x9e, 0x30, 0x01, 0xec,
+	0x84, 0x89, 0xb2, 0x0d, 0x66, 0xdd, 0xce, 0x28,
+	0x5c, 0xec, 0x98, 0x46, 0x68, 0x21, 0x9f, 0x88,
+	0x3f, 0x1f, 0x42, 0x77, 0xce, 0xd0, 0x61, 0xd4,
+	0x20, 0xa7, 0xff, 0x53, 0xad, 0x37, 0xd0, 0x17,
+	0x35, 0xc9, 0xfc, 0xba, 0x0a, 0x78, 0x3f, 0xf2,
+	0xcc, 0x86, 0x89, 0xe8, 0x4b, 0x3c, 0x48, 0x33,
+	0x09, 0x7f, 0xc6, 0xc0, 0xdd, 0xb8, 0xfd, 0x7a,
+	0x66, 0x66, 0x65, 0xeb, 0x47, 0xa7, 0x04, 0x28,
+	0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62,
+	0x70, 0xcf, 0xd6
+};
+static const u8 dec_output012[] __initconst = {
+	0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0,
+	0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5,
+	0x42, 0x34, 0x8a, 0xa1, 0x03, 0xb7, 0xe9, 0x57,
+	0x46, 0xbe, 0x20, 0xe4, 0x6e, 0xb0, 0xeb, 0xff,
+	0xea, 0x07, 0x7e, 0xef, 0xe2, 0x55, 0x9f, 0xe5,
+	0x78, 0x3a, 0xb7, 0x83, 0xc2, 0x18, 0x40, 0x7b,
+	0xeb, 0xcd, 0x81, 0xfb, 0x90, 0x12, 0x9e, 0x46,
+	0xa9, 0xd6, 0x4a, 0xba, 0xb0, 0x62, 0xdb, 0x6b,
+	0x99, 0xc4, 0xdb, 0x54, 0x4b, 0xb8, 0xa5, 0x71,
+	0xcb, 0xcd, 0x63, 0x32, 0x55, 0xfb, 0x31, 0xf0,
+	0x38, 0xf5, 0xbe, 0x78, 0xe4, 0x45, 0xce, 0x1b,
+	0x6a, 0x5b, 0x0e, 0xf4, 0x16, 0xe4, 0xb1, 0x3d,
+	0xf6, 0x63, 0x7b, 0xa7, 0x0c, 0xde, 0x6f, 0x8f,
+	0x74, 0xdf, 0xe0, 0x1e, 0x9d, 0xce, 0x8f, 0x24,
+	0xef, 0x23, 0x35, 0x33, 0x7b, 0x83, 0x34, 0x23,
+	0x58, 0x74, 0x14, 0x77, 0x1f, 0xc2, 0x4f, 0x4e,
+	0xc6, 0x89, 0xf9, 0x52, 0x09, 0x37, 0x64, 0x14,
+	0xc4, 0x01, 0x6b, 0x9d, 0x77, 0xe8, 0x90, 0x5d,
+	0xa8, 0x4a, 0x2a, 0xef, 0x5c, 0x7f, 0xeb, 0xbb,
+	0xb2, 0xc6, 0x93, 0x99, 0x66, 0xdc, 0x7f, 0xd4,
+	0x9e, 0x2a, 0xca, 0x8d, 0xdb, 0xe7, 0x20, 0xcf,
+	0xe4, 0x73, 0xae, 0x49, 0x7d, 0x64, 0x0f, 0x0e,
+	0x28, 0x46, 0xa9, 0xa8, 0x32, 0xe4, 0x0e, 0xf6,
+	0x51, 0x53, 0xb8, 0x3c, 0xb1, 0xff, 0xa3, 0x33,
+	0x41, 0x75, 0xff, 0xf1, 0x6f, 0xf1, 0xfb, 0xbb,
+	0x83, 0x7f, 0x06, 0x9b, 0xe7, 0x1b, 0x0a, 0xe0,
+	0x5c, 0x33, 0x60, 0x5b, 0xdb, 0x5b, 0xed, 0xfe,
+	0xa5, 0x16, 0x19, 0x72, 0xa3, 0x64, 0x23, 0x00,
+	0x02, 0xc7, 0xf3, 0x6a, 0x81, 0x3e, 0x44, 0x1d,
+	0x79, 0x15, 0x5f, 0x9a, 0xde, 0xe2, 0xfd, 0x1b,
+	0x73, 0xc1, 0xbc, 0x23, 0xba, 0x31, 0xd2, 0x50,
+	0xd5, 0xad, 0x7f, 0x74, 0xa7, 0xc9, 0xf8, 0x3e,
+	0x2b, 0x26, 0x10, 0xf6, 0x03, 0x36, 0x74, 0xe4,
+	0x0e, 0x6a, 0x72, 0xb7, 0x73, 0x0a, 0x42, 0x28,
+	0xc2, 0xad, 0x5e, 0x03, 0xbe, 0xb8, 0x0b, 0xa8,
+	0x5b, 0xd4, 0xb8, 0xba, 0x52, 0x89, 0xb1, 0x9b,
+	0xc1, 0xc3, 0x65, 0x87, 0xed, 0xa5, 0xf4, 0x86,
+	0xfd, 0x41, 0x80, 0x91, 0x27, 0x59, 0x53, 0x67,
+	0x15, 0x78, 0x54, 0x8b, 0x2d, 0x3d, 0xc7, 0xff,
+	0x02, 0x92, 0x07, 0x5f, 0x7a, 0x4b, 0x60, 0x59,
+	0x3c, 0x6f, 0x5c, 0xd8, 0xec, 0x95, 0xd2, 0xfe,
+	0xa0, 0x3b, 0xd8, 0x3f, 0xd1, 0x69, 0xa6, 0xd6,
+	0x41, 0xb2, 0xf4, 0x4d, 0x12, 0xf4, 0x58, 0x3e,
+	0x66, 0x64, 0x80, 0x31, 0x9b, 0xa8, 0x4c, 0x8b,
+	0x07, 0xb2, 0xec, 0x66, 0x94, 0x66, 0x47, 0x50,
+	0x50, 0x5f, 0x18, 0x0b, 0x0e, 0xd6, 0xc0, 0x39,
+	0x21, 0x13, 0x9e, 0x33, 0xbc, 0x79, 0x36, 0x02,
+	0x96, 0x70, 0xf0, 0x48, 0x67, 0x2f, 0x26, 0xe9,
+	0x6d, 0x10, 0xbb, 0xd6, 0x3f, 0xd1, 0x64, 0x7a,
+	0x2e, 0xbe, 0x0c, 0x61, 0xf0, 0x75, 0x42, 0x38,
+	0x23, 0xb1, 0x9e, 0x9f, 0x7c, 0x67, 0x66, 0xd9,
+	0x58, 0x9a, 0xf1, 0xbb, 0x41, 0x2a, 0x8d, 0x65,
+	0x84, 0x94, 0xfc, 0xdc, 0x6a, 0x50, 0x64, 0xdb,
+	0x56, 0x33, 0x76, 0x00, 0x10, 0xed, 0xbe, 0xd2,
+	0x12, 0xf6, 0xf6, 0x1b, 0xa2, 0x16, 0xde, 0xae,
+	0x31, 0x95, 0xdd, 0xb1, 0x08, 0x7e, 0x4e, 0xee,
+	0xe7, 0xf9, 0xa5, 0xfb, 0x5b, 0x61, 0x43, 0x00,
+	0x40, 0xf6, 0x7e, 0x02, 0x04, 0x32, 0x4e, 0x0c,
+	0xe2, 0x66, 0x0d, 0xd7, 0x07, 0x98, 0x0e, 0xf8,
+	0x72, 0x34, 0x6d, 0x95, 0x86, 0xd7, 0xcb, 0x31,
+	0x54, 0x47, 0xd0, 0x38, 0x29, 0x9c, 0x5a, 0x68,
+	0xd4, 0x87, 0x76, 0xc9, 0xe7, 0x7e, 0xe3, 0xf4,
+	0x81, 0x6d, 0x18, 0xcb, 0xc9, 0x05, 0xaf, 0xa0,
+	0xfb, 0x66, 0xf7, 0xf1, 0x1c, 0xc6, 0x14, 0x11,
+	0x4f, 0x2b, 0x79, 0x42, 0x8b, 0xbc, 0xac, 0xe7,
+	0x6c, 0xfe, 0x0f, 0x58, 0xe7, 0x7c, 0x78, 0x39,
+	0x30, 0xb0, 0x66, 0x2c, 0x9b, 0x6d, 0x3a, 0xe1,
+	0xcf, 0xc9, 0xa4, 0x0e, 0x6d, 0x6d, 0x8a, 0xa1,
+	0x3a, 0xe7, 0x28, 0xd4, 0x78, 0x4c, 0xa6, 0xa2,
+	0x2a, 0xa6, 0x03, 0x30, 0xd7, 0xa8, 0x25, 0x66,
+	0x87, 0x2f, 0x69, 0x5c, 0x4e, 0xdd, 0xa5, 0x49,
+	0x5d, 0x37, 0x4a, 0x59, 0xc4, 0xaf, 0x1f, 0xa2,
+	0xe4, 0xf8, 0xa6, 0x12, 0x97, 0xd5, 0x79, 0xf5,
+	0xe2, 0x4a, 0x2b, 0x5f, 0x61, 0xe4, 0x9e, 0xe3,
+	0xee, 0xb8, 0xa7, 0x5b, 0x2f, 0xf4, 0x9e, 0x6c,
+	0xfb, 0xd1, 0xc6, 0x56, 0x77, 0xba, 0x75, 0xaa,
+	0x3d, 0x1a, 0xa8, 0x0b, 0xb3, 0x68, 0x24, 0x00,
+	0x10, 0x7f, 0xfd, 0xd7, 0xa1, 0x8d, 0x83, 0x54,
+	0x4f, 0x1f, 0xd8, 0x2a, 0xbe, 0x8a, 0x0c, 0x87,
+	0xab, 0xa2, 0xde, 0xc3, 0x39, 0xbf, 0x09, 0x03,
+	0xa5, 0xf3, 0x05, 0x28, 0xe1, 0xe1, 0xee, 0x39,
+	0x70, 0x9c, 0xd8, 0x81, 0x12, 0x1e, 0x02, 0x40,
+	0xd2, 0x6e, 0xf0, 0xeb, 0x1b, 0x3d, 0x22, 0xc6,
+	0xe5, 0xe3, 0xb4, 0x5a, 0x98, 0xbb, 0xf0, 0x22,
+	0x28, 0x8d, 0xe5, 0xd3, 0x16, 0x48, 0x24, 0xa5,
+	0xe6, 0x66, 0x0c, 0xf9, 0x08, 0xf9, 0x7e, 0x1e,
+	0xe1, 0x28, 0x26, 0x22, 0xc7, 0xc7, 0x0a, 0x32,
+	0x47, 0xfa, 0xa3, 0xbe, 0x3c, 0xc4, 0xc5, 0x53,
+	0x0a, 0xd5, 0x94, 0x4a, 0xd7, 0x93, 0xd8, 0x42,
+	0x99, 0xb9, 0x0a, 0xdb, 0x56, 0xf7, 0xb9, 0x1c,
+	0x53, 0x4f, 0xfa, 0xd3, 0x74, 0xad, 0xd9, 0x68,
+	0xf1, 0x1b, 0xdf, 0x61, 0xc6, 0x5e, 0xa8, 0x48,
+	0xfc, 0xd4, 0x4a, 0x4c, 0x3c, 0x32, 0xf7, 0x1c,
+	0x96, 0x21, 0x9b, 0xf9, 0xa3, 0xcc, 0x5a, 0xce,
+	0xd5, 0xd7, 0x08, 0x24, 0xf6, 0x1c, 0xfd, 0xdd,
+	0x38, 0xc2, 0x32, 0xe9, 0xb8, 0xe7, 0xb6, 0xfa,
+	0x9d, 0x45, 0x13, 0x2c, 0x83, 0xfd, 0x4a, 0x69,
+	0x82, 0xcd, 0xdc, 0xb3, 0x76, 0x0c, 0x9e, 0xd8,
+	0xf4, 0x1b, 0x45, 0x15, 0xb4, 0x97, 0xe7, 0x58,
+	0x34, 0xe2, 0x03, 0x29, 0x5a, 0xbf, 0xb6, 0xe0,
+	0x5d, 0x13, 0xd9, 0x2b, 0xb4, 0x80, 0xb2, 0x45,
+	0x81, 0x6a, 0x2e, 0x6c, 0x89, 0x7d, 0xee, 0xbb,
+	0x52, 0xdd, 0x1f, 0x18, 0xe7, 0x13, 0x6b, 0x33,
+	0x0e, 0xea, 0x36, 0x92, 0x77, 0x7b, 0x6d, 0x9c,
+	0x5a, 0x5f, 0x45, 0x7b, 0x7b, 0x35, 0x62, 0x23,
+	0xd1, 0xbf, 0x0f, 0xd0, 0x08, 0x1b, 0x2b, 0x80,
+	0x6b, 0x7e, 0xf1, 0x21, 0x47, 0xb0, 0x57, 0xd1,
+	0x98, 0x72, 0x90, 0x34, 0x1c, 0x20, 0x04, 0xff,
+	0x3d, 0x5c, 0xee, 0x0e, 0x57, 0x5f, 0x6f, 0x24,
+	0x4e, 0x3c, 0xea, 0xfc, 0xa5, 0xa9, 0x83, 0xc9,
+	0x61, 0xb4, 0x51, 0x24, 0xf8, 0x27, 0x5e, 0x46,
+	0x8c, 0xb1, 0x53, 0x02, 0x96, 0x35, 0xba, 0xb8,
+	0x4c, 0x71, 0xd3, 0x15, 0x59, 0x35, 0x22, 0x20,
+	0xad, 0x03, 0x9f, 0x66, 0x44, 0x3b, 0x9c, 0x35,
+	0x37, 0x1f, 0x9b, 0xbb, 0xf3, 0xdb, 0x35, 0x63,
+	0x30, 0x64, 0xaa, 0xa2, 0x06, 0xa8, 0x5d, 0xbb,
+	0xe1, 0x9f, 0x70, 0xec, 0x82, 0x11, 0x06, 0x36,
+	0xec, 0x8b, 0x69, 0x66, 0x24, 0x44, 0xc9, 0x4a,
+	0x57, 0xbb, 0x9b, 0x78, 0x13, 0xce, 0x9c, 0x0c,
+	0xba, 0x92, 0x93, 0x63, 0xb8, 0xe2, 0x95, 0x0f,
+	0x0f, 0x16, 0x39, 0x52, 0xfd, 0x3a, 0x6d, 0x02,
+	0x4b, 0xdf, 0x13, 0xd3, 0x2a, 0x22, 0xb4, 0x03,
+	0x7c, 0x54, 0x49, 0x96, 0x68, 0x54, 0x10, 0xfa,
+	0xef, 0xaa, 0x6c, 0xe8, 0x22, 0xdc, 0x71, 0x16,
+	0x13, 0x1a, 0xf6, 0x28, 0xe5, 0x6d, 0x77, 0x3d,
+	0xcd, 0x30, 0x63, 0xb1, 0x70, 0x52, 0xa1, 0xc5,
+	0x94, 0x5f, 0xcf, 0xe8, 0xb8, 0x26, 0x98, 0xf7,
+	0x06, 0xa0, 0x0a, 0x70, 0xfa, 0x03, 0x80, 0xac,
+	0xc1, 0xec, 0xd6, 0x4c, 0x54, 0xd7, 0xfe, 0x47,
+	0xb6, 0x88, 0x4a, 0xf7, 0x71, 0x24, 0xee, 0xf3,
+	0xd2, 0xc2, 0x4a, 0x7f, 0xfe, 0x61, 0xc7, 0x35,
+	0xc9, 0x37, 0x67, 0xcb, 0x24, 0x35, 0xda, 0x7e,
+	0xca, 0x5f, 0xf3, 0x8d, 0xd4, 0x13, 0x8e, 0xd6,
+	0xcb, 0x4d, 0x53, 0x8f, 0x53, 0x1f, 0xc0, 0x74,
+	0xf7, 0x53, 0xb9, 0x5e, 0x23, 0x37, 0xba, 0x6e,
+	0xe3, 0x9d, 0x07, 0x55, 0x25, 0x7b, 0xe6, 0x2a,
+	0x64, 0xd1, 0x32, 0xdd, 0x54, 0x1b, 0x4b, 0xc0,
+	0xe1, 0xd7, 0x69, 0x58, 0xf8, 0x93, 0x29, 0xc4,
+	0xdd, 0x23, 0x2f, 0xa5, 0xfc, 0x9d, 0x7e, 0xf8,
+	0xd4, 0x90, 0xcd, 0x82, 0x55, 0xdc, 0x16, 0x16,
+	0x9f, 0x07, 0x52, 0x9b, 0x9d, 0x25, 0xed, 0x32,
+	0xc5, 0x7b, 0xdf, 0xf6, 0x83, 0x46, 0x3d, 0x65,
+	0xb7, 0xef, 0x87, 0x7a, 0x12, 0x69, 0x8f, 0x06,
+	0x7c, 0x51, 0x15, 0x4a, 0x08, 0xe8, 0xac, 0x9a,
+	0x0c, 0x24, 0xa7, 0x27, 0xd8, 0x46, 0x2f, 0xe7,
+	0x01, 0x0e, 0x1c, 0xc6, 0x91, 0xb0, 0x6e, 0x85,
+	0x65, 0xf0, 0x29, 0x0d, 0x2e, 0x6b, 0x3b, 0xfb,
+	0x4b, 0xdf, 0xe4, 0x80, 0x93, 0x03, 0x66, 0x46,
+	0x3e, 0x8a, 0x6e, 0xf3, 0x5e, 0x4d, 0x62, 0x0e,
+	0x49, 0x05, 0xaf, 0xd4, 0xf8, 0x21, 0x20, 0x61,
+	0x1d, 0x39, 0x17, 0xf4, 0x61, 0x47, 0x95, 0xfb,
+	0x15, 0x2e, 0xb3, 0x4f, 0xd0, 0x5d, 0xf5, 0x7d,
+	0x40, 0xda, 0x90, 0x3c, 0x6b, 0xcb, 0x17, 0x00,
+	0x13, 0x3b, 0x64, 0x34, 0x1b, 0xf0, 0xf2, 0xe5,
+	0x3b, 0xb2, 0xc7, 0xd3, 0x5f, 0x3a, 0x44, 0xa6,
+	0x9b, 0xb7, 0x78, 0x0e, 0x42, 0x5d, 0x4c, 0xc1,
+	0xe9, 0xd2, 0xcb, 0xb7, 0x78, 0xd1, 0xfe, 0x9a,
+	0xb5, 0x07, 0xe9, 0xe0, 0xbe, 0xe2, 0x8a, 0xa7,
+	0x01, 0x83, 0x00, 0x8c, 0x5c, 0x08, 0xe6, 0x63,
+	0x12, 0x92, 0xb7, 0xb7, 0xa6, 0x19, 0x7d, 0x38,
+	0x13, 0x38, 0x92, 0x87, 0x24, 0xf9, 0x48, 0xb3,
+	0x5e, 0x87, 0x6a, 0x40, 0x39, 0x5c, 0x3f, 0xed,
+	0x8f, 0xee, 0xdb, 0x15, 0x82, 0x06, 0xda, 0x49,
+	0x21, 0x2b, 0xb5, 0xbf, 0x32, 0x7c, 0x9f, 0x42,
+	0x28, 0x63, 0xcf, 0xaf, 0x1e, 0xf8, 0xc6, 0xa0,
+	0xd1, 0x02, 0x43, 0x57, 0x62, 0xec, 0x9b, 0x0f,
+	0x01, 0x9e, 0x71, 0xd8, 0x87, 0x9d, 0x01, 0xc1,
+	0x58, 0x77, 0xd9, 0xaf, 0xb1, 0x10, 0x7e, 0xdd,
+	0xa6, 0x50, 0x96, 0xe5, 0xf0, 0x72, 0x00, 0x6d,
+	0x4b, 0xf8, 0x2a, 0x8f, 0x19, 0xf3, 0x22, 0x88,
+	0x11, 0x4a, 0x8b, 0x7c, 0xfd, 0xb7, 0xed, 0xe1,
+	0xf6, 0x40, 0x39, 0xe0, 0xe9, 0xf6, 0x3d, 0x25,
+	0xe6, 0x74, 0x3c, 0x58, 0x57, 0x7f, 0xe1, 0x22,
+	0x96, 0x47, 0x31, 0x91, 0xba, 0x70, 0x85, 0x28,
+	0x6b, 0x9f, 0x6e, 0x25, 0xac, 0x23, 0x66, 0x2f,
+	0x29, 0x88, 0x28, 0xce, 0x8c, 0x5c, 0x88, 0x53,
+	0xd1, 0x3b, 0xcc, 0x6a, 0x51, 0xb2, 0xe1, 0x28,
+	0x3f, 0x91, 0xb4, 0x0d, 0x00, 0x3a, 0xe3, 0xf8,
+	0xc3, 0x8f, 0xd7, 0x96, 0x62, 0x0e, 0x2e, 0xfc,
+	0xc8, 0x6c, 0x77, 0xa6, 0x1d, 0x22, 0xc1, 0xb8,
+	0xe6, 0x61, 0xd7, 0x67, 0x36, 0x13, 0x7b, 0xbb,
+	0x9b, 0x59, 0x09, 0xa6, 0xdf, 0xf7, 0x6b, 0xa3,
+	0x40, 0x1a, 0xf5, 0x4f, 0xb4, 0xda, 0xd3, 0xf3,
+	0x81, 0x93, 0xc6, 0x18, 0xd9, 0x26, 0xee, 0xac,
+	0xf0, 0xaa, 0xdf, 0xc5, 0x9c, 0xca, 0xc2, 0xa2,
+	0xcc, 0x7b, 0x5c, 0x24, 0xb0, 0xbc, 0xd0, 0x6a,
+	0x4d, 0x89, 0x09, 0xb8, 0x07, 0xfe, 0x87, 0xad,
+	0x0a, 0xea, 0xb8, 0x42, 0xf9, 0x5e, 0xb3, 0x3e,
+	0x36, 0x4c, 0xaf, 0x75, 0x9e, 0x1c, 0xeb, 0xbd,
+	0xbc, 0xbb, 0x80, 0x40, 0xa7, 0x3a, 0x30, 0xbf,
+	0xa8, 0x44, 0xf4, 0xeb, 0x38, 0xad, 0x29, 0xba,
+	0x23, 0xed, 0x41, 0x0c, 0xea, 0xd2, 0xbb, 0x41,
+	0x18, 0xd6, 0xb9, 0xba, 0x65, 0x2b, 0xa3, 0x91,
+	0x6d, 0x1f, 0xa9, 0xf4, 0xd1, 0x25, 0x8d, 0x4d,
+	0x38, 0xff, 0x64, 0xa0, 0xec, 0xde, 0xa6, 0xb6,
+	0x79, 0xab, 0x8e, 0x33, 0x6c, 0x47, 0xde, 0xaf,
+	0x94, 0xa4, 0xa5, 0x86, 0x77, 0x55, 0x09, 0x92,
+	0x81, 0x31, 0x76, 0xc7, 0x34, 0x22, 0x89, 0x8e,
+	0x3d, 0x26, 0x26, 0xd7, 0xfc, 0x1e, 0x16, 0x72,
+	0x13, 0x33, 0x63, 0xd5, 0x22, 0xbe, 0xb8, 0x04,
+	0x34, 0x84, 0x41, 0xbb, 0x80, 0xd0, 0x9f, 0x46,
+	0x48, 0x07, 0xa7, 0xfc, 0x2b, 0x3a, 0x75, 0x55,
+	0x8c, 0xc7, 0x6a, 0xbd, 0x7e, 0x46, 0x08, 0x84,
+	0x0f, 0xd5, 0x74, 0xc0, 0x82, 0x8e, 0xaa, 0x61,
+	0x05, 0x01, 0xb2, 0x47, 0x6e, 0x20, 0x6a, 0x2d,
+	0x58, 0x70, 0x48, 0x32, 0xa7, 0x37, 0xd2, 0xb8,
+	0x82, 0x1a, 0x51, 0xb9, 0x61, 0xdd, 0xfd, 0x9d,
+	0x6b, 0x0e, 0x18, 0x97, 0xf8, 0x45, 0x5f, 0x87,
+	0x10, 0xcf, 0x34, 0x72, 0x45, 0x26, 0x49, 0x70,
+	0xe7, 0xa3, 0x78, 0xe0, 0x52, 0x89, 0x84, 0x94,
+	0x83, 0x82, 0xc2, 0x69, 0x8f, 0xe3, 0xe1, 0x3f,
+	0x60, 0x74, 0x88, 0xc4, 0xf7, 0x75, 0x2c, 0xfb,
+	0xbd, 0xb6, 0xc4, 0x7e, 0x10, 0x0a, 0x6c, 0x90,
+	0x04, 0x9e, 0xc3, 0x3f, 0x59, 0x7c, 0xce, 0x31,
+	0x18, 0x60, 0x57, 0x73, 0x46, 0x94, 0x7d, 0x06,
+	0xa0, 0x6d, 0x44, 0xec, 0xa2, 0x0a, 0x9e, 0x05,
+	0x15, 0xef, 0xca, 0x5c, 0xbf, 0x00, 0xeb, 0xf7,
+	0x3d, 0x32, 0xd4, 0xa5, 0xef, 0x49, 0x89, 0x5e,
+	0x46, 0xb0, 0xa6, 0x63, 0x5b, 0x8a, 0x73, 0xae,
+	0x6f, 0xd5, 0x9d, 0xf8, 0x4f, 0x40, 0xb5, 0xb2,
+	0x6e, 0xd3, 0xb6, 0x01, 0xa9, 0x26, 0xa2, 0x21,
+	0xcf, 0x33, 0x7a, 0x3a, 0xa4, 0x23, 0x13, 0xb0,
+	0x69, 0x6a, 0xee, 0xce, 0xd8, 0x9d, 0x01, 0x1d,
+	0x50, 0xc1, 0x30, 0x6c, 0xb1, 0xcd, 0xa0, 0xf0,
+	0xf0, 0xa2, 0x64, 0x6f, 0xbb, 0xbf, 0x5e, 0xe6,
+	0xab, 0x87, 0xb4, 0x0f, 0x4f, 0x15, 0xaf, 0xb5,
+	0x25, 0xa1, 0xb2, 0xd0, 0x80, 0x2c, 0xfb, 0xf9,
+	0xfe, 0xd2, 0x33, 0xbb, 0x76, 0xfe, 0x7c, 0xa8,
+	0x66, 0xf7, 0xe7, 0x85, 0x9f, 0x1f, 0x85, 0x57,
+	0x88, 0xe1, 0xe9, 0x63, 0xe4, 0xd8, 0x1c, 0xa1,
+	0xfb, 0xda, 0x44, 0x05, 0x2e, 0x1d, 0x3a, 0x1c,
+	0xff, 0xc8, 0x3b, 0xc0, 0xfe, 0xda, 0x22, 0x0b,
+	0x43, 0xd6, 0x88, 0x39, 0x4c, 0x4a, 0xa6, 0x69,
+	0x18, 0x93, 0x42, 0x4e, 0xb5, 0xcc, 0x66, 0x0d,
+	0x09, 0xf8, 0x1e, 0x7c, 0xd3, 0x3c, 0x99, 0x0d,
+	0x50, 0x1d, 0x62, 0xe9, 0x57, 0x06, 0xbf, 0x19,
+	0x88, 0xdd, 0xad, 0x7b, 0x4f, 0xf9, 0xc7, 0x82,
+	0x6d, 0x8d, 0xc8, 0xc4, 0xc5, 0x78, 0x17, 0x20,
+	0x15, 0xc5, 0x52, 0x41, 0xcf, 0x5b, 0xd6, 0x7f,
+	0x94, 0x02, 0x41, 0xe0, 0x40, 0x22, 0x03, 0x5e,
+	0xd1, 0x53, 0xd4, 0x86, 0xd3, 0x2c, 0x9f, 0x0f,
+	0x96, 0xe3, 0x6b, 0x9a, 0x76, 0x32, 0x06, 0x47,
+	0x4b, 0x11, 0xb3, 0xdd, 0x03, 0x65, 0xbd, 0x9b,
+	0x01, 0xda, 0x9c, 0xb9, 0x7e, 0x3f, 0x6a, 0xc4,
+	0x7b, 0xea, 0xd4, 0x3c, 0xb9, 0xfb, 0x5c, 0x6b,
+	0x64, 0x33, 0x52, 0xba, 0x64, 0x78, 0x8f, 0xa4,
+	0xaf, 0x7a, 0x61, 0x8d, 0xbc, 0xc5, 0x73, 0xe9,
+	0x6b, 0x58, 0x97, 0x4b, 0xbf, 0x63, 0x22, 0xd3,
+	0x37, 0x02, 0x54, 0xc5, 0xb9, 0x16, 0x4a, 0xf0,
+	0x19, 0xd8, 0x94, 0x57, 0xb8, 0x8a, 0xb3, 0x16,
+	0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d,
+	0x78, 0xec, 0x00
+};
+static const u8 dec_assoc012[] __initconst = {
+	0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8,
+	0x77, 0xe8, 0x21, 0xff, 0x06, 0x59, 0x35, 0xce,
+	0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c,
+	0xcf, 0x70, 0xdd, 0x0a, 0x68, 0xbf, 0xd4, 0xbc,
+	0x16, 0x76, 0x99, 0x36, 0x1e, 0x58, 0x79, 0x5e,
+	0xd4, 0x29, 0xf7, 0x33, 0x93, 0x48, 0xdb, 0x5f,
+	0x01, 0xae, 0x9c, 0xb6, 0xe4, 0x88, 0x6d, 0x2b,
+	0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9
+};
+static const u8 dec_nonce012[] __initconst = {
+	0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06
+};
+static const u8 dec_key012[] __initconst = {
+	0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e,
+	0x8f, 0x59, 0x8e, 0xc5, 0x90, 0xd5, 0x27, 0x2d,
+	0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, 0x44, 0x1e,
+	0x65, 0x91, 0x6e, 0x2a, 0x79, 0x22, 0xda, 0x64
+};
+
+static const u8 dec_input013[] __initconst = {
+	0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3,
+	0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf,
+	0xdf, 0x34, 0x56, 0x82, 0xe2, 0xbe, 0xe5, 0xe1,
+	0x28, 0xd1, 0x4e, 0x5f, 0x4f, 0x01, 0x7d, 0x3f,
+	0x99, 0x6b, 0x30, 0x6e, 0x1a, 0x7c, 0x4c, 0x8e,
+	0x62, 0x81, 0xae, 0x86, 0x3f, 0x6b, 0xd0, 0xb5,
+	0xa9, 0xcf, 0x50, 0xf1, 0x02, 0x12, 0xa0, 0x0b,
+	0x24, 0xe9, 0xe6, 0x72, 0x89, 0x2c, 0x52, 0x1b,
+	0x34, 0x38, 0xf8, 0x75, 0x5f, 0xa0, 0x74, 0xe2,
+	0x99, 0xdd, 0xa6, 0x4b, 0x14, 0x50, 0x4e, 0xf1,
+	0xbe, 0xd6, 0x9e, 0xdb, 0xb2, 0x24, 0x27, 0x74,
+	0x12, 0x4a, 0x78, 0x78, 0x17, 0xa5, 0x58, 0x8e,
+	0x2f, 0xf9, 0xf4, 0x8d, 0xee, 0x03, 0x88, 0xae,
+	0xb8, 0x29, 0xa1, 0x2f, 0x4b, 0xee, 0x92, 0xbd,
+	0x87, 0xb3, 0xce, 0x34, 0x21, 0x57, 0x46, 0x04,
+	0x49, 0x0c, 0x80, 0xf2, 0x01, 0x13, 0xa1, 0x55,
+	0xb3, 0xff, 0x44, 0x30, 0x3c, 0x1c, 0xd0, 0xef,
+	0xbc, 0x18, 0x74, 0x26, 0xad, 0x41, 0x5b, 0x5b,
+	0x3e, 0x9a, 0x7a, 0x46, 0x4f, 0x16, 0xd6, 0x74,
+	0x5a, 0xb7, 0x3a, 0x28, 0x31, 0xd8, 0xae, 0x26,
+	0xac, 0x50, 0x53, 0x86, 0xf2, 0x56, 0xd7, 0x3f,
+	0x29, 0xbc, 0x45, 0x68, 0x8e, 0xcb, 0x98, 0x64,
+	0xdd, 0xc9, 0xba, 0xb8, 0x4b, 0x7b, 0x82, 0xdd,
+	0x14, 0xa7, 0xcb, 0x71, 0x72, 0x00, 0x5c, 0xad,
+	0x7b, 0x6a, 0x89, 0xa4, 0x3d, 0xbf, 0xb5, 0x4b,
+	0x3e, 0x7c, 0x5a, 0xcf, 0xb8, 0xa1, 0xc5, 0x6e,
+	0xc8, 0xb6, 0x31, 0x57, 0x7b, 0xdf, 0xa5, 0x7e,
+	0xb1, 0xd6, 0x42, 0x2a, 0x31, 0x36, 0xd1, 0xd0,
+	0x3f, 0x7a, 0xe5, 0x94, 0xd6, 0x36, 0xa0, 0x6f,
+	0xb7, 0x40, 0x7d, 0x37, 0xc6, 0x55, 0x7c, 0x50,
+	0x40, 0x6d, 0x29, 0x89, 0xe3, 0x5a, 0xae, 0x97,
+	0xe7, 0x44, 0x49, 0x6e, 0xbd, 0x81, 0x3d, 0x03,
+	0x93, 0x06, 0x12, 0x06, 0xe2, 0x41, 0x12, 0x4a,
+	0xf1, 0x6a, 0xa4, 0x58, 0xa2, 0xfb, 0xd2, 0x15,
+	0xba, 0xc9, 0x79, 0xc9, 0xce, 0x5e, 0x13, 0xbb,
+	0xf1, 0x09, 0x04, 0xcc, 0xfd, 0xe8, 0x51, 0x34,
+	0x6a, 0xe8, 0x61, 0x88, 0xda, 0xed, 0x01, 0x47,
+	0x84, 0xf5, 0x73, 0x25, 0xf9, 0x1c, 0x42, 0x86,
+	0x07, 0xf3, 0x5b, 0x1a, 0x01, 0xb3, 0xeb, 0x24,
+	0x32, 0x8d, 0xf6, 0xed, 0x7c, 0x4b, 0xeb, 0x3c,
+	0x36, 0x42, 0x28, 0xdf, 0xdf, 0xb6, 0xbe, 0xd9,
+	0x8c, 0x52, 0xd3, 0x2b, 0x08, 0x90, 0x8c, 0xe7,
+	0x98, 0x31, 0xe2, 0x32, 0x8e, 0xfc, 0x11, 0x48,
+	0x00, 0xa8, 0x6a, 0x42, 0x4a, 0x02, 0xc6, 0x4b,
+	0x09, 0xf1, 0xe3, 0x49, 0xf3, 0x45, 0x1f, 0x0e,
+	0xbc, 0x56, 0xe2, 0xe4, 0xdf, 0xfb, 0xeb, 0x61,
+	0xfa, 0x24, 0xc1, 0x63, 0x75, 0xbb, 0x47, 0x75,
+	0xaf, 0xe1, 0x53, 0x16, 0x96, 0x21, 0x85, 0x26,
+	0x11, 0xb3, 0x76, 0xe3, 0x23, 0xa1, 0x6b, 0x74,
+	0x37, 0xd0, 0xde, 0x06, 0x90, 0x71, 0x5d, 0x43,
+	0x88, 0x9b, 0x00, 0x54, 0xa6, 0x75, 0x2f, 0xa1,
+	0xc2, 0x0b, 0x73, 0x20, 0x1d, 0xb6, 0x21, 0x79,
+	0x57, 0x3f, 0xfa, 0x09, 0xbe, 0x8a, 0x33, 0xc3,
+	0x52, 0xf0, 0x1d, 0x82, 0x31, 0xd1, 0x55, 0xb5,
+	0x6c, 0x99, 0x25, 0xcf, 0x5c, 0x32, 0xce, 0xe9,
+	0x0d, 0xfa, 0x69, 0x2c, 0xd5, 0x0d, 0xc5, 0x6d,
+	0x86, 0xd0, 0x0c, 0x3b, 0x06, 0x50, 0x79, 0xe8,
+	0xc3, 0xae, 0x04, 0xe6, 0xcd, 0x51, 0xe4, 0x26,
+	0x9b, 0x4f, 0x7e, 0xa6, 0x0f, 0xab, 0xd8, 0xe5,
+	0xde, 0xa9, 0x00, 0x95, 0xbe, 0xa3, 0x9d, 0x5d,
+	0xb2, 0x09, 0x70, 0x18, 0x1c, 0xf0, 0xac, 0x29,
+	0x23, 0x02, 0x29, 0x28, 0xd2, 0x74, 0x35, 0x57,
+	0x62, 0x0f, 0x24, 0xea, 0x5e, 0x33, 0xc2, 0x92,
+	0xf3, 0x78, 0x4d, 0x30, 0x1e, 0xa1, 0x99, 0xa9,
+	0x82, 0xb0, 0x42, 0x31, 0x8d, 0xad, 0x8a, 0xbc,
+	0xfc, 0xd4, 0x57, 0x47, 0x3e, 0xb4, 0x50, 0xdd,
+	0x6e, 0x2c, 0x80, 0x4d, 0x22, 0xf1, 0xfb, 0x57,
+	0xc4, 0xdd, 0x17, 0xe1, 0x8a, 0x36, 0x4a, 0xb3,
+	0x37, 0xca, 0xc9, 0x4e, 0xab, 0xd5, 0x69, 0xc4,
+	0xf4, 0xbc, 0x0b, 0x3b, 0x44, 0x4b, 0x29, 0x9c,
+	0xee, 0xd4, 0x35, 0x22, 0x21, 0xb0, 0x1f, 0x27,
+	0x64, 0xa8, 0x51, 0x1b, 0xf0, 0x9f, 0x19, 0x5c,
+	0xfb, 0x5a, 0x64, 0x74, 0x70, 0x45, 0x09, 0xf5,
+	0x64, 0xfe, 0x1a, 0x2d, 0xc9, 0x14, 0x04, 0x14,
+	0xcf, 0xd5, 0x7d, 0x60, 0xaf, 0x94, 0x39, 0x94,
+	0xe2, 0x7d, 0x79, 0x82, 0xd0, 0x65, 0x3b, 0x6b,
+	0x9c, 0x19, 0x84, 0xb4, 0x6d, 0xb3, 0x0c, 0x99,
+	0xc0, 0x56, 0xa8, 0xbd, 0x73, 0xce, 0x05, 0x84,
+	0x3e, 0x30, 0xaa, 0xc4, 0x9b, 0x1b, 0x04, 0x2a,
+	0x9f, 0xd7, 0x43, 0x2b, 0x23, 0xdf, 0xbf, 0xaa,
+	0xd5, 0xc2, 0x43, 0x2d, 0x70, 0xab, 0xdc, 0x75,
+	0xad, 0xac, 0xf7, 0xc0, 0xbe, 0x67, 0xb2, 0x74,
+	0xed, 0x67, 0x10, 0x4a, 0x92, 0x60, 0xc1, 0x40,
+	0x50, 0x19, 0x8a, 0x8a, 0x8c, 0x09, 0x0e, 0x72,
+	0xe1, 0x73, 0x5e, 0xe8, 0x41, 0x85, 0x63, 0x9f,
+	0x3f, 0xd7, 0x7d, 0xc4, 0xfb, 0x22, 0x5d, 0x92,
+	0x6c, 0xb3, 0x1e, 0xe2, 0x50, 0x2f, 0x82, 0xa8,
+	0x28, 0xc0, 0xb5, 0xd7, 0x5f, 0x68, 0x0d, 0x2c,
+	0x2d, 0xaf, 0x7e, 0xfa, 0x2e, 0x08, 0x0f, 0x1f,
+	0x70, 0x9f, 0xe9, 0x19, 0x72, 0x55, 0xf8, 0xfb,
+	0x51, 0xd2, 0x33, 0x5d, 0xa0, 0xd3, 0x2b, 0x0a,
+	0x6c, 0xbc, 0x4e, 0xcf, 0x36, 0x4d, 0xdc, 0x3b,
+	0xe9, 0x3e, 0x81, 0x7c, 0x61, 0xdb, 0x20, 0x2d,
+	0x3a, 0xc3, 0xb3, 0x0c, 0x1e, 0x00, 0xb9, 0x7c,
+	0xf5, 0xca, 0x10, 0x5f, 0x3a, 0x71, 0xb3, 0xe4,
+	0x20, 0xdb, 0x0c, 0x2a, 0x98, 0x63, 0x45, 0x00,
+	0x58, 0xf6, 0x68, 0xe4, 0x0b, 0xda, 0x13, 0x3b,
+	0x60, 0x5c, 0x76, 0xdb, 0xb9, 0x97, 0x71, 0xe4,
+	0xd9, 0xb7, 0xdb, 0xbd, 0x68, 0xc7, 0x84, 0x84,
+	0xaa, 0x7c, 0x68, 0x62, 0x5e, 0x16, 0xfc, 0xba,
+	0x72, 0xaa, 0x9a, 0xa9, 0xeb, 0x7c, 0x75, 0x47,
+	0x97, 0x7e, 0xad, 0xe2, 0xd9, 0x91, 0xe8, 0xe4,
+	0xa5, 0x31, 0xd7, 0x01, 0x8e, 0xa2, 0x11, 0x88,
+	0x95, 0xb9, 0xf2, 0x9b, 0xd3, 0x7f, 0x1b, 0x81,
+	0x22, 0xf7, 0x98, 0x60, 0x0a, 0x64, 0xa6, 0xc1,
+	0xf6, 0x49, 0xc7, 0xe3, 0x07, 0x4d, 0x94, 0x7a,
+	0xcf, 0x6e, 0x68, 0x0c, 0x1b, 0x3f, 0x6e, 0x2e,
+	0xee, 0x92, 0xfa, 0x52, 0xb3, 0x59, 0xf8, 0xf1,
+	0x8f, 0x6a, 0x66, 0xa3, 0x82, 0x76, 0x4a, 0x07,
+	0x1a, 0xc7, 0xdd, 0xf5, 0xda, 0x9c, 0x3c, 0x24,
+	0xbf, 0xfd, 0x42, 0xa1, 0x10, 0x64, 0x6a, 0x0f,
+	0x89, 0xee, 0x36, 0xa5, 0xce, 0x99, 0x48, 0x6a,
+	0xf0, 0x9f, 0x9e, 0x69, 0xa4, 0x40, 0x20, 0xe9,
+	0x16, 0x15, 0xf7, 0xdb, 0x75, 0x02, 0xcb, 0xe9,
+	0x73, 0x8b, 0x3b, 0x49, 0x2f, 0xf0, 0xaf, 0x51,
+	0x06, 0x5c, 0xdf, 0x27, 0x27, 0x49, 0x6a, 0xd1,
+	0xcc, 0xc7, 0xb5, 0x63, 0xb5, 0xfc, 0xb8, 0x5c,
+	0x87, 0x7f, 0x84, 0xb4, 0xcc, 0x14, 0xa9, 0x53,
+	0xda, 0xa4, 0x56, 0xf8, 0xb6, 0x1b, 0xcc, 0x40,
+	0x27, 0x52, 0x06, 0x5a, 0x13, 0x81, 0xd7, 0x3a,
+	0xd4, 0x3b, 0xfb, 0x49, 0x65, 0x31, 0x33, 0xb2,
+	0xfa, 0xcd, 0xad, 0x58, 0x4e, 0x2b, 0xae, 0xd2,
+	0x20, 0xfb, 0x1a, 0x48, 0xb4, 0x3f, 0x9a, 0xd8,
+	0x7a, 0x35, 0x4a, 0xc8, 0xee, 0x88, 0x5e, 0x07,
+	0x66, 0x54, 0xb9, 0xec, 0x9f, 0xa3, 0xe3, 0xb9,
+	0x37, 0xaa, 0x49, 0x76, 0x31, 0xda, 0x74, 0x2d,
+	0x3c, 0xa4, 0x65, 0x10, 0x32, 0x38, 0xf0, 0xde,
+	0xd3, 0x99, 0x17, 0xaa, 0x71, 0xaa, 0x8f, 0x0f,
+	0x8c, 0xaf, 0xa2, 0xf8, 0x5d, 0x64, 0xba, 0x1d,
+	0xa3, 0xef, 0x96, 0x73, 0xe8, 0xa1, 0x02, 0x8d,
+	0x0c, 0x6d, 0xb8, 0x06, 0x90, 0xb8, 0x08, 0x56,
+	0x2c, 0xa7, 0x06, 0xc9, 0xc2, 0x38, 0xdb, 0x7c,
+	0x63, 0xb1, 0x57, 0x8e, 0xea, 0x7c, 0x79, 0xf3,
+	0x49, 0x1d, 0xfe, 0x9f, 0xf3, 0x6e, 0xb1, 0x1d,
+	0xba, 0x19, 0x80, 0x1a, 0x0a, 0xd3, 0xb0, 0x26,
+	0x21, 0x40, 0xb1, 0x7c, 0xf9, 0x4d, 0x8d, 0x10,
+	0xc1, 0x7e, 0xf4, 0xf6, 0x3c, 0xa8, 0xfd, 0x7c,
+	0xa3, 0x92, 0xb2, 0x0f, 0xaa, 0xcc, 0xa6, 0x11,
+	0xfe, 0x04, 0xe3, 0xd1, 0x7a, 0x32, 0x89, 0xdf,
+	0x0d, 0xc4, 0x8f, 0x79, 0x6b, 0xca, 0x16, 0x7c,
+	0x6e, 0xf9, 0xad, 0x0f, 0xf6, 0xfe, 0x27, 0xdb,
+	0xc4, 0x13, 0x70, 0xf1, 0x62, 0x1a, 0x4f, 0x79,
+	0x40, 0xc9, 0x9b, 0x8b, 0x21, 0xea, 0x84, 0xfa,
+	0xf5, 0xf1, 0x89, 0xce, 0xb7, 0x55, 0x0a, 0x80,
+	0x39, 0x2f, 0x55, 0x36, 0x16, 0x9c, 0x7b, 0x08,
+	0xbd, 0x87, 0x0d, 0xa5, 0x32, 0xf1, 0x52, 0x7c,
+	0xe8, 0x55, 0x60, 0x5b, 0xd7, 0x69, 0xe4, 0xfc,
+	0xfa, 0x12, 0x85, 0x96, 0xea, 0x50, 0x28, 0xab,
+	0x8a, 0xf7, 0xbb, 0x0e, 0x53, 0x74, 0xca, 0xa6,
+	0x27, 0x09, 0xc2, 0xb5, 0xde, 0x18, 0x14, 0xd9,
+	0xea, 0xe5, 0x29, 0x1c, 0x40, 0x56, 0xcf, 0xd7,
+	0xae, 0x05, 0x3f, 0x65, 0xaf, 0x05, 0x73, 0xe2,
+	0x35, 0x96, 0x27, 0x07, 0x14, 0xc0, 0xad, 0x33,
+	0xf1, 0xdc, 0x44, 0x7a, 0x89, 0x17, 0x77, 0xd2,
+	0x9c, 0x58, 0x60, 0xf0, 0x3f, 0x7b, 0x2d, 0x2e,
+	0x57, 0x95, 0x54, 0x87, 0xed, 0xf2, 0xc7, 0x4c,
+	0xf0, 0xae, 0x56, 0x29, 0x19, 0x7d, 0x66, 0x4b,
+	0x9b, 0x83, 0x84, 0x42, 0x3b, 0x01, 0x25, 0x66,
+	0x8e, 0x02, 0xde, 0xb9, 0x83, 0x54, 0x19, 0xf6,
+	0x9f, 0x79, 0x0d, 0x67, 0xc5, 0x1d, 0x7a, 0x44,
+	0x02, 0x98, 0xa7, 0x16, 0x1c, 0x29, 0x0d, 0x74,
+	0xff, 0x85, 0x40, 0x06, 0xef, 0x2c, 0xa9, 0xc6,
+	0xf5, 0x53, 0x07, 0x06, 0xae, 0xe4, 0xfa, 0x5f,
+	0xd8, 0x39, 0x4d, 0xf1, 0x9b, 0x6b, 0xd9, 0x24,
+	0x84, 0xfe, 0x03, 0x4c, 0xb2, 0x3f, 0xdf, 0xa1,
+	0x05, 0x9e, 0x50, 0x14, 0x5a, 0xd9, 0x1a, 0xa2,
+	0xa7, 0xfa, 0xfa, 0x17, 0xf7, 0x78, 0xd6, 0xb5,
+	0x92, 0x61, 0x91, 0xac, 0x36, 0xfa, 0x56, 0x0d,
+	0x38, 0x32, 0x18, 0x85, 0x08, 0x58, 0x37, 0xf0,
+	0x4b, 0xdb, 0x59, 0xe7, 0xa4, 0x34, 0xc0, 0x1b,
+	0x01, 0xaf, 0x2d, 0xde, 0xa1, 0xaa, 0x5d, 0xd3,
+	0xec, 0xe1, 0xd4, 0xf7, 0xe6, 0x54, 0x68, 0xf0,
+	0x51, 0x97, 0xa7, 0x89, 0xea, 0x24, 0xad, 0xd3,
+	0x6e, 0x47, 0x93, 0x8b, 0x4b, 0xb4, 0xf7, 0x1c,
+	0x42, 0x06, 0x67, 0xe8, 0x99, 0xf6, 0xf5, 0x7b,
+	0x85, 0xb5, 0x65, 0xb5, 0xb5, 0xd2, 0x37, 0xf5,
+	0xf3, 0x02, 0xa6, 0x4d, 0x11, 0xa7, 0xdc, 0x51,
+	0x09, 0x7f, 0xa0, 0xd8, 0x88, 0x1c, 0x13, 0x71,
+	0xae, 0x9c, 0xb7, 0x7b, 0x34, 0xd6, 0x4e, 0x68,
+	0x26, 0x83, 0x51, 0xaf, 0x1d, 0xee, 0x8b, 0xbb,
+	0x69, 0x43, 0x2b, 0x9e, 0x8a, 0xbc, 0x02, 0x0e,
+	0xa0, 0x1b, 0xe0, 0xa8, 0x5f, 0x6f, 0xaf, 0x1b,
+	0x8f, 0xe7, 0x64, 0x71, 0x74, 0x11, 0x7e, 0xa8,
+	0xd8, 0xf9, 0x97, 0x06, 0xc3, 0xb6, 0xfb, 0xfb,
+	0xb7, 0x3d, 0x35, 0x9d, 0x3b, 0x52, 0xed, 0x54,
+	0xca, 0xf4, 0x81, 0x01, 0x2d, 0x1b, 0xc3, 0xa7,
+	0x00, 0x3d, 0x1a, 0x39, 0x54, 0xe1, 0xf6, 0xff,
+	0xed, 0x6f, 0x0b, 0x5a, 0x68, 0xda, 0x58, 0xdd,
+	0xa9, 0xcf, 0x5c, 0x4a, 0xe5, 0x09, 0x4e, 0xde,
+	0x9d, 0xbc, 0x3e, 0xee, 0x5a, 0x00, 0x3b, 0x2c,
+	0x87, 0x10, 0x65, 0x60, 0xdd, 0xd7, 0x56, 0xd1,
+	0x4c, 0x64, 0x45, 0xe4, 0x21, 0xec, 0x78, 0xf8,
+	0x25, 0x7a, 0x3e, 0x16, 0x5d, 0x09, 0x53, 0x14,
+	0xbe, 0x4f, 0xae, 0x87, 0xd8, 0xd1, 0xaa, 0x3c,
+	0xf6, 0x3e, 0xa4, 0x70, 0x8c, 0x5e, 0x70, 0xa4,
+	0xb3, 0x6b, 0x66, 0x73, 0xd3, 0xbf, 0x31, 0x06,
+	0x19, 0x62, 0x93, 0x15, 0xf2, 0x86, 0xe4, 0x52,
+	0x7e, 0x53, 0x4c, 0x12, 0x38, 0xcc, 0x34, 0x7d,
+	0x57, 0xf6, 0x42, 0x93, 0x8a, 0xc4, 0xee, 0x5c,
+	0x8a, 0xe1, 0x52, 0x8f, 0x56, 0x64, 0xf6, 0xa6,
+	0xd1, 0x91, 0x57, 0x70, 0xcd, 0x11, 0x76, 0xf5,
+	0x59, 0x60, 0x60, 0x3c, 0xc1, 0xc3, 0x0b, 0x7f,
+	0x58, 0x1a, 0x50, 0x91, 0xf1, 0x68, 0x8f, 0x6e,
+	0x74, 0x74, 0xa8, 0x51, 0x0b, 0xf7, 0x7a, 0x98,
+	0x37, 0xf2, 0x0a, 0x0e, 0xa4, 0x97, 0x04, 0xb8,
+	0x9b, 0xfd, 0xa0, 0xea, 0xf7, 0x0d, 0xe1, 0xdb,
+	0x03, 0xf0, 0x31, 0x29, 0xf8, 0xdd, 0x6b, 0x8b,
+	0x5d, 0xd8, 0x59, 0xa9, 0x29, 0xcf, 0x9a, 0x79,
+	0x89, 0x19, 0x63, 0x46, 0x09, 0x79, 0x6a, 0x11,
+	0xda, 0x63, 0x68, 0x48, 0x77, 0x23, 0xfb, 0x7d,
+	0x3a, 0x43, 0xcb, 0x02, 0x3b, 0x7a, 0x6d, 0x10,
+	0x2a, 0x9e, 0xac, 0xf1, 0xd4, 0x19, 0xf8, 0x23,
+	0x64, 0x1d, 0x2c, 0x5f, 0xf2, 0xb0, 0x5c, 0x23,
+	0x27, 0xf7, 0x27, 0x30, 0x16, 0x37, 0xb1, 0x90,
+	0xab, 0x38, 0xfb, 0x55, 0xcd, 0x78, 0x58, 0xd4,
+	0x7d, 0x43, 0xf6, 0x45, 0x5e, 0x55, 0x8d, 0xb1,
+	0x02, 0x65, 0x58, 0xb4, 0x13, 0x4b, 0x36, 0xf7,
+	0xcc, 0xfe, 0x3d, 0x0b, 0x82, 0xe2, 0x12, 0x11,
+	0xbb, 0xe6, 0xb8, 0x3a, 0x48, 0x71, 0xc7, 0x50,
+	0x06, 0x16, 0x3a, 0xe6, 0x7c, 0x05, 0xc7, 0xc8,
+	0x4d, 0x2f, 0x08, 0x6a, 0x17, 0x9a, 0x95, 0x97,
+	0x50, 0x68, 0xdc, 0x28, 0x18, 0xc4, 0x61, 0x38,
+	0xb9, 0xe0, 0x3e, 0x78, 0xdb, 0x29, 0xe0, 0x9f,
+	0x52, 0xdd, 0xf8, 0x4f, 0x91, 0xc1, 0xd0, 0x33,
+	0xa1, 0x7a, 0x8e, 0x30, 0x13, 0x82, 0x07, 0x9f,
+	0xd3, 0x31, 0x0f, 0x23, 0xbe, 0x32, 0x5a, 0x75,
+	0xcf, 0x96, 0xb2, 0xec, 0xb5, 0x32, 0xac, 0x21,
+	0xd1, 0x82, 0x33, 0xd3, 0x15, 0x74, 0xbd, 0x90,
+	0xf1, 0x2c, 0xe6, 0x5f, 0x8d, 0xe3, 0x02, 0xe8,
+	0xe9, 0xc4, 0xca, 0x96, 0xeb, 0x0e, 0xbc, 0x91,
+	0xf4, 0xb9, 0xea, 0xd9, 0x1b, 0x75, 0xbd, 0xe1,
+	0xac, 0x2a, 0x05, 0x37, 0x52, 0x9b, 0x1b, 0x3f,
+	0x5a, 0xdc, 0x21, 0xc3, 0x98, 0xbb, 0xaf, 0xa3,
+	0xf2, 0x00, 0xbf, 0x0d, 0x30, 0x89, 0x05, 0xcc,
+	0xa5, 0x76, 0xf5, 0x06, 0xf0, 0xc6, 0x54, 0x8a,
+	0x5d, 0xd4, 0x1e, 0xc1, 0xf2, 0xce, 0xb0, 0x62,
+	0xc8, 0xfc, 0x59, 0x42, 0x9a, 0x90, 0x60, 0x55,
+	0xfe, 0x88, 0xa5, 0x8b, 0xb8, 0x33, 0x0c, 0x23,
+	0x24, 0x0d, 0x15, 0x70, 0x37, 0x1e, 0x3d, 0xf6,
+	0xd2, 0xea, 0x92, 0x10, 0xb2, 0xc4, 0x51, 0xac,
+	0xf2, 0xac, 0xf3, 0x6b, 0x6c, 0xaa, 0xcf, 0x12,
+	0xc5, 0x6c, 0x90, 0x50, 0xb5, 0x0c, 0xfc, 0x1a,
+	0x15, 0x52, 0xe9, 0x26, 0xc6, 0x52, 0xa4, 0xe7,
+	0x81, 0x69, 0xe1, 0xe7, 0x9e, 0x30, 0x01, 0xec,
+	0x84, 0x89, 0xb2, 0x0d, 0x66, 0xdd, 0xce, 0x28,
+	0x5c, 0xec, 0x98, 0x46, 0x68, 0x21, 0x9f, 0x88,
+	0x3f, 0x1f, 0x42, 0x77, 0xce, 0xd0, 0x61, 0xd4,
+	0x20, 0xa7, 0xff, 0x53, 0xad, 0x37, 0xd0, 0x17,
+	0x35, 0xc9, 0xfc, 0xba, 0x0a, 0x78, 0x3f, 0xf2,
+	0xcc, 0x86, 0x89, 0xe8, 0x4b, 0x3c, 0x48, 0x33,
+	0x09, 0x7f, 0xc6, 0xc0, 0xdd, 0xb8, 0xfd, 0x7a,
+	0x66, 0x66, 0x65, 0xeb, 0x47, 0xa7, 0x04, 0x28,
+	0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62,
+	0x70, 0xcf, 0xd7
+};
+static const u8 dec_output013[] __initconst = {
+	0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0,
+	0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5,
+	0x42, 0x34, 0x8a, 0xa1, 0x03, 0xb7, 0xe9, 0x57,
+	0x46, 0xbe, 0x20, 0xe4, 0x6e, 0xb0, 0xeb, 0xff,
+	0xea, 0x07, 0x7e, 0xef, 0xe2, 0x55, 0x9f, 0xe5,
+	0x78, 0x3a, 0xb7, 0x83, 0xc2, 0x18, 0x40, 0x7b,
+	0xeb, 0xcd, 0x81, 0xfb, 0x90, 0x12, 0x9e, 0x46,
+	0xa9, 0xd6, 0x4a, 0xba, 0xb0, 0x62, 0xdb, 0x6b,
+	0x99, 0xc4, 0xdb, 0x54, 0x4b, 0xb8, 0xa5, 0x71,
+	0xcb, 0xcd, 0x63, 0x32, 0x55, 0xfb, 0x31, 0xf0,
+	0x38, 0xf5, 0xbe, 0x78, 0xe4, 0x45, 0xce, 0x1b,
+	0x6a, 0x5b, 0x0e, 0xf4, 0x16, 0xe4, 0xb1, 0x3d,
+	0xf6, 0x63, 0x7b, 0xa7, 0x0c, 0xde, 0x6f, 0x8f,
+	0x74, 0xdf, 0xe0, 0x1e, 0x9d, 0xce, 0x8f, 0x24,
+	0xef, 0x23, 0x35, 0x33, 0x7b, 0x83, 0x34, 0x23,
+	0x58, 0x74, 0x14, 0x77, 0x1f, 0xc2, 0x4f, 0x4e,
+	0xc6, 0x89, 0xf9, 0x52, 0x09, 0x37, 0x64, 0x14,
+	0xc4, 0x01, 0x6b, 0x9d, 0x77, 0xe8, 0x90, 0x5d,
+	0xa8, 0x4a, 0x2a, 0xef, 0x5c, 0x7f, 0xeb, 0xbb,
+	0xb2, 0xc6, 0x93, 0x99, 0x66, 0xdc, 0x7f, 0xd4,
+	0x9e, 0x2a, 0xca, 0x8d, 0xdb, 0xe7, 0x20, 0xcf,
+	0xe4, 0x73, 0xae, 0x49, 0x7d, 0x64, 0x0f, 0x0e,
+	0x28, 0x46, 0xa9, 0xa8, 0x32, 0xe4, 0x0e, 0xf6,
+	0x51, 0x53, 0xb8, 0x3c, 0xb1, 0xff, 0xa3, 0x33,
+	0x41, 0x75, 0xff, 0xf1, 0x6f, 0xf1, 0xfb, 0xbb,
+	0x83, 0x7f, 0x06, 0x9b, 0xe7, 0x1b, 0x0a, 0xe0,
+	0x5c, 0x33, 0x60, 0x5b, 0xdb, 0x5b, 0xed, 0xfe,
+	0xa5, 0x16, 0x19, 0x72, 0xa3, 0x64, 0x23, 0x00,
+	0x02, 0xc7, 0xf3, 0x6a, 0x81, 0x3e, 0x44, 0x1d,
+	0x79, 0x15, 0x5f, 0x9a, 0xde, 0xe2, 0xfd, 0x1b,
+	0x73, 0xc1, 0xbc, 0x23, 0xba, 0x31, 0xd2, 0x50,
+	0xd5, 0xad, 0x7f, 0x74, 0xa7, 0xc9, 0xf8, 0x3e,
+	0x2b, 0x26, 0x10, 0xf6, 0x03, 0x36, 0x74, 0xe4,
+	0x0e, 0x6a, 0x72, 0xb7, 0x73, 0x0a, 0x42, 0x28,
+	0xc2, 0xad, 0x5e, 0x03, 0xbe, 0xb8, 0x0b, 0xa8,
+	0x5b, 0xd4, 0xb8, 0xba, 0x52, 0x89, 0xb1, 0x9b,
+	0xc1, 0xc3, 0x65, 0x87, 0xed, 0xa5, 0xf4, 0x86,
+	0xfd, 0x41, 0x80, 0x91, 0x27, 0x59, 0x53, 0x67,
+	0x15, 0x78, 0x54, 0x8b, 0x2d, 0x3d, 0xc7, 0xff,
+	0x02, 0x92, 0x07, 0x5f, 0x7a, 0x4b, 0x60, 0x59,
+	0x3c, 0x6f, 0x5c, 0xd8, 0xec, 0x95, 0xd2, 0xfe,
+	0xa0, 0x3b, 0xd8, 0x3f, 0xd1, 0x69, 0xa6, 0xd6,
+	0x41, 0xb2, 0xf4, 0x4d, 0x12, 0xf4, 0x58, 0x3e,
+	0x66, 0x64, 0x80, 0x31, 0x9b, 0xa8, 0x4c, 0x8b,
+	0x07, 0xb2, 0xec, 0x66, 0x94, 0x66, 0x47, 0x50,
+	0x50, 0x5f, 0x18, 0x0b, 0x0e, 0xd6, 0xc0, 0x39,
+	0x21, 0x13, 0x9e, 0x33, 0xbc, 0x79, 0x36, 0x02,
+	0x96, 0x70, 0xf0, 0x48, 0x67, 0x2f, 0x26, 0xe9,
+	0x6d, 0x10, 0xbb, 0xd6, 0x3f, 0xd1, 0x64, 0x7a,
+	0x2e, 0xbe, 0x0c, 0x61, 0xf0, 0x75, 0x42, 0x38,
+	0x23, 0xb1, 0x9e, 0x9f, 0x7c, 0x67, 0x66, 0xd9,
+	0x58, 0x9a, 0xf1, 0xbb, 0x41, 0x2a, 0x8d, 0x65,
+	0x84, 0x94, 0xfc, 0xdc, 0x6a, 0x50, 0x64, 0xdb,
+	0x56, 0x33, 0x76, 0x00, 0x10, 0xed, 0xbe, 0xd2,
+	0x12, 0xf6, 0xf6, 0x1b, 0xa2, 0x16, 0xde, 0xae,
+	0x31, 0x95, 0xdd, 0xb1, 0x08, 0x7e, 0x4e, 0xee,
+	0xe7, 0xf9, 0xa5, 0xfb, 0x5b, 0x61, 0x43, 0x00,
+	0x40, 0xf6, 0x7e, 0x02, 0x04, 0x32, 0x4e, 0x0c,
+	0xe2, 0x66, 0x0d, 0xd7, 0x07, 0x98, 0x0e, 0xf8,
+	0x72, 0x34, 0x6d, 0x95, 0x86, 0xd7, 0xcb, 0x31,
+	0x54, 0x47, 0xd0, 0x38, 0x29, 0x9c, 0x5a, 0x68,
+	0xd4, 0x87, 0x76, 0xc9, 0xe7, 0x7e, 0xe3, 0xf4,
+	0x81, 0x6d, 0x18, 0xcb, 0xc9, 0x05, 0xaf, 0xa0,
+	0xfb, 0x66, 0xf7, 0xf1, 0x1c, 0xc6, 0x14, 0x11,
+	0x4f, 0x2b, 0x79, 0x42, 0x8b, 0xbc, 0xac, 0xe7,
+	0x6c, 0xfe, 0x0f, 0x58, 0xe7, 0x7c, 0x78, 0x39,
+	0x30, 0xb0, 0x66, 0x2c, 0x9b, 0x6d, 0x3a, 0xe1,
+	0xcf, 0xc9, 0xa4, 0x0e, 0x6d, 0x6d, 0x8a, 0xa1,
+	0x3a, 0xe7, 0x28, 0xd4, 0x78, 0x4c, 0xa6, 0xa2,
+	0x2a, 0xa6, 0x03, 0x30, 0xd7, 0xa8, 0x25, 0x66,
+	0x87, 0x2f, 0x69, 0x5c, 0x4e, 0xdd, 0xa5, 0x49,
+	0x5d, 0x37, 0x4a, 0x59, 0xc4, 0xaf, 0x1f, 0xa2,
+	0xe4, 0xf8, 0xa6, 0x12, 0x97, 0xd5, 0x79, 0xf5,
+	0xe2, 0x4a, 0x2b, 0x5f, 0x61, 0xe4, 0x9e, 0xe3,
+	0xee, 0xb8, 0xa7, 0x5b, 0x2f, 0xf4, 0x9e, 0x6c,
+	0xfb, 0xd1, 0xc6, 0x56, 0x77, 0xba, 0x75, 0xaa,
+	0x3d, 0x1a, 0xa8, 0x0b, 0xb3, 0x68, 0x24, 0x00,
+	0x10, 0x7f, 0xfd, 0xd7, 0xa1, 0x8d, 0x83, 0x54,
+	0x4f, 0x1f, 0xd8, 0x2a, 0xbe, 0x8a, 0x0c, 0x87,
+	0xab, 0xa2, 0xde, 0xc3, 0x39, 0xbf, 0x09, 0x03,
+	0xa5, 0xf3, 0x05, 0x28, 0xe1, 0xe1, 0xee, 0x39,
+	0x70, 0x9c, 0xd8, 0x81, 0x12, 0x1e, 0x02, 0x40,
+	0xd2, 0x6e, 0xf0, 0xeb, 0x1b, 0x3d, 0x22, 0xc6,
+	0xe5, 0xe3, 0xb4, 0x5a, 0x98, 0xbb, 0xf0, 0x22,
+	0x28, 0x8d, 0xe5, 0xd3, 0x16, 0x48, 0x24, 0xa5,
+	0xe6, 0x66, 0x0c, 0xf9, 0x08, 0xf9, 0x7e, 0x1e,
+	0xe1, 0x28, 0x26, 0x22, 0xc7, 0xc7, 0x0a, 0x32,
+	0x47, 0xfa, 0xa3, 0xbe, 0x3c, 0xc4, 0xc5, 0x53,
+	0x0a, 0xd5, 0x94, 0x4a, 0xd7, 0x93, 0xd8, 0x42,
+	0x99, 0xb9, 0x0a, 0xdb, 0x56, 0xf7, 0xb9, 0x1c,
+	0x53, 0x4f, 0xfa, 0xd3, 0x74, 0xad, 0xd9, 0x68,
+	0xf1, 0x1b, 0xdf, 0x61, 0xc6, 0x5e, 0xa8, 0x48,
+	0xfc, 0xd4, 0x4a, 0x4c, 0x3c, 0x32, 0xf7, 0x1c,
+	0x96, 0x21, 0x9b, 0xf9, 0xa3, 0xcc, 0x5a, 0xce,
+	0xd5, 0xd7, 0x08, 0x24, 0xf6, 0x1c, 0xfd, 0xdd,
+	0x38, 0xc2, 0x32, 0xe9, 0xb8, 0xe7, 0xb6, 0xfa,
+	0x9d, 0x45, 0x13, 0x2c, 0x83, 0xfd, 0x4a, 0x69,
+	0x82, 0xcd, 0xdc, 0xb3, 0x76, 0x0c, 0x9e, 0xd8,
+	0xf4, 0x1b, 0x45, 0x15, 0xb4, 0x97, 0xe7, 0x58,
+	0x34, 0xe2, 0x03, 0x29, 0x5a, 0xbf, 0xb6, 0xe0,
+	0x5d, 0x13, 0xd9, 0x2b, 0xb4, 0x80, 0xb2, 0x45,
+	0x81, 0x6a, 0x2e, 0x6c, 0x89, 0x7d, 0xee, 0xbb,
+	0x52, 0xdd, 0x1f, 0x18, 0xe7, 0x13, 0x6b, 0x33,
+	0x0e, 0xea, 0x36, 0x92, 0x77, 0x7b, 0x6d, 0x9c,
+	0x5a, 0x5f, 0x45, 0x7b, 0x7b, 0x35, 0x62, 0x23,
+	0xd1, 0xbf, 0x0f, 0xd0, 0x08, 0x1b, 0x2b, 0x80,
+	0x6b, 0x7e, 0xf1, 0x21, 0x47, 0xb0, 0x57, 0xd1,
+	0x98, 0x72, 0x90, 0x34, 0x1c, 0x20, 0x04, 0xff,
+	0x3d, 0x5c, 0xee, 0x0e, 0x57, 0x5f, 0x6f, 0x24,
+	0x4e, 0x3c, 0xea, 0xfc, 0xa5, 0xa9, 0x83, 0xc9,
+	0x61, 0xb4, 0x51, 0x24, 0xf8, 0x27, 0x5e, 0x46,
+	0x8c, 0xb1, 0x53, 0x02, 0x96, 0x35, 0xba, 0xb8,
+	0x4c, 0x71, 0xd3, 0x15, 0x59, 0x35, 0x22, 0x20,
+	0xad, 0x03, 0x9f, 0x66, 0x44, 0x3b, 0x9c, 0x35,
+	0x37, 0x1f, 0x9b, 0xbb, 0xf3, 0xdb, 0x35, 0x63,
+	0x30, 0x64, 0xaa, 0xa2, 0x06, 0xa8, 0x5d, 0xbb,
+	0xe1, 0x9f, 0x70, 0xec, 0x82, 0x11, 0x06, 0x36,
+	0xec, 0x8b, 0x69, 0x66, 0x24, 0x44, 0xc9, 0x4a,
+	0x57, 0xbb, 0x9b, 0x78, 0x13, 0xce, 0x9c, 0x0c,
+	0xba, 0x92, 0x93, 0x63, 0xb8, 0xe2, 0x95, 0x0f,
+	0x0f, 0x16, 0x39, 0x52, 0xfd, 0x3a, 0x6d, 0x02,
+	0x4b, 0xdf, 0x13, 0xd3, 0x2a, 0x22, 0xb4, 0x03,
+	0x7c, 0x54, 0x49, 0x96, 0x68, 0x54, 0x10, 0xfa,
+	0xef, 0xaa, 0x6c, 0xe8, 0x22, 0xdc, 0x71, 0x16,
+	0x13, 0x1a, 0xf6, 0x28, 0xe5, 0x6d, 0x77, 0x3d,
+	0xcd, 0x30, 0x63, 0xb1, 0x70, 0x52, 0xa1, 0xc5,
+	0x94, 0x5f, 0xcf, 0xe8, 0xb8, 0x26, 0x98, 0xf7,
+	0x06, 0xa0, 0x0a, 0x70, 0xfa, 0x03, 0x80, 0xac,
+	0xc1, 0xec, 0xd6, 0x4c, 0x54, 0xd7, 0xfe, 0x47,
+	0xb6, 0x88, 0x4a, 0xf7, 0x71, 0x24, 0xee, 0xf3,
+	0xd2, 0xc2, 0x4a, 0x7f, 0xfe, 0x61, 0xc7, 0x35,
+	0xc9, 0x37, 0x67, 0xcb, 0x24, 0x35, 0xda, 0x7e,
+	0xca, 0x5f, 0xf3, 0x8d, 0xd4, 0x13, 0x8e, 0xd6,
+	0xcb, 0x4d, 0x53, 0x8f, 0x53, 0x1f, 0xc0, 0x74,
+	0xf7, 0x53, 0xb9, 0x5e, 0x23, 0x37, 0xba, 0x6e,
+	0xe3, 0x9d, 0x07, 0x55, 0x25, 0x7b, 0xe6, 0x2a,
+	0x64, 0xd1, 0x32, 0xdd, 0x54, 0x1b, 0x4b, 0xc0,
+	0xe1, 0xd7, 0x69, 0x58, 0xf8, 0x93, 0x29, 0xc4,
+	0xdd, 0x23, 0x2f, 0xa5, 0xfc, 0x9d, 0x7e, 0xf8,
+	0xd4, 0x90, 0xcd, 0x82, 0x55, 0xdc, 0x16, 0x16,
+	0x9f, 0x07, 0x52, 0x9b, 0x9d, 0x25, 0xed, 0x32,
+	0xc5, 0x7b, 0xdf, 0xf6, 0x83, 0x46, 0x3d, 0x65,
+	0xb7, 0xef, 0x87, 0x7a, 0x12, 0x69, 0x8f, 0x06,
+	0x7c, 0x51, 0x15, 0x4a, 0x08, 0xe8, 0xac, 0x9a,
+	0x0c, 0x24, 0xa7, 0x27, 0xd8, 0x46, 0x2f, 0xe7,
+	0x01, 0x0e, 0x1c, 0xc6, 0x91, 0xb0, 0x6e, 0x85,
+	0x65, 0xf0, 0x29, 0x0d, 0x2e, 0x6b, 0x3b, 0xfb,
+	0x4b, 0xdf, 0xe4, 0x80, 0x93, 0x03, 0x66, 0x46,
+	0x3e, 0x8a, 0x6e, 0xf3, 0x5e, 0x4d, 0x62, 0x0e,
+	0x49, 0x05, 0xaf, 0xd4, 0xf8, 0x21, 0x20, 0x61,
+	0x1d, 0x39, 0x17, 0xf4, 0x61, 0x47, 0x95, 0xfb,
+	0x15, 0x2e, 0xb3, 0x4f, 0xd0, 0x5d, 0xf5, 0x7d,
+	0x40, 0xda, 0x90, 0x3c, 0x6b, 0xcb, 0x17, 0x00,
+	0x13, 0x3b, 0x64, 0x34, 0x1b, 0xf0, 0xf2, 0xe5,
+	0x3b, 0xb2, 0xc7, 0xd3, 0x5f, 0x3a, 0x44, 0xa6,
+	0x9b, 0xb7, 0x78, 0x0e, 0x42, 0x5d, 0x4c, 0xc1,
+	0xe9, 0xd2, 0xcb, 0xb7, 0x78, 0xd1, 0xfe, 0x9a,
+	0xb5, 0x07, 0xe9, 0xe0, 0xbe, 0xe2, 0x8a, 0xa7,
+	0x01, 0x83, 0x00, 0x8c, 0x5c, 0x08, 0xe6, 0x63,
+	0x12, 0x92, 0xb7, 0xb7, 0xa6, 0x19, 0x7d, 0x38,
+	0x13, 0x38, 0x92, 0x87, 0x24, 0xf9, 0x48, 0xb3,
+	0x5e, 0x87, 0x6a, 0x40, 0x39, 0x5c, 0x3f, 0xed,
+	0x8f, 0xee, 0xdb, 0x15, 0x82, 0x06, 0xda, 0x49,
+	0x21, 0x2b, 0xb5, 0xbf, 0x32, 0x7c, 0x9f, 0x42,
+	0x28, 0x63, 0xcf, 0xaf, 0x1e, 0xf8, 0xc6, 0xa0,
+	0xd1, 0x02, 0x43, 0x57, 0x62, 0xec, 0x9b, 0x0f,
+	0x01, 0x9e, 0x71, 0xd8, 0x87, 0x9d, 0x01, 0xc1,
+	0x58, 0x77, 0xd9, 0xaf, 0xb1, 0x10, 0x7e, 0xdd,
+	0xa6, 0x50, 0x96, 0xe5, 0xf0, 0x72, 0x00, 0x6d,
+	0x4b, 0xf8, 0x2a, 0x8f, 0x19, 0xf3, 0x22, 0x88,
+	0x11, 0x4a, 0x8b, 0x7c, 0xfd, 0xb7, 0xed, 0xe1,
+	0xf6, 0x40, 0x39, 0xe0, 0xe9, 0xf6, 0x3d, 0x25,
+	0xe6, 0x74, 0x3c, 0x58, 0x57, 0x7f, 0xe1, 0x22,
+	0x96, 0x47, 0x31, 0x91, 0xba, 0x70, 0x85, 0x28,
+	0x6b, 0x9f, 0x6e, 0x25, 0xac, 0x23, 0x66, 0x2f,
+	0x29, 0x88, 0x28, 0xce, 0x8c, 0x5c, 0x88, 0x53,
+	0xd1, 0x3b, 0xcc, 0x6a, 0x51, 0xb2, 0xe1, 0x28,
+	0x3f, 0x91, 0xb4, 0x0d, 0x00, 0x3a, 0xe3, 0xf8,
+	0xc3, 0x8f, 0xd7, 0x96, 0x62, 0x0e, 0x2e, 0xfc,
+	0xc8, 0x6c, 0x77, 0xa6, 0x1d, 0x22, 0xc1, 0xb8,
+	0xe6, 0x61, 0xd7, 0x67, 0x36, 0x13, 0x7b, 0xbb,
+	0x9b, 0x59, 0x09, 0xa6, 0xdf, 0xf7, 0x6b, 0xa3,
+	0x40, 0x1a, 0xf5, 0x4f, 0xb4, 0xda, 0xd3, 0xf3,
+	0x81, 0x93, 0xc6, 0x18, 0xd9, 0x26, 0xee, 0xac,
+	0xf0, 0xaa, 0xdf, 0xc5, 0x9c, 0xca, 0xc2, 0xa2,
+	0xcc, 0x7b, 0x5c, 0x24, 0xb0, 0xbc, 0xd0, 0x6a,
+	0x4d, 0x89, 0x09, 0xb8, 0x07, 0xfe, 0x87, 0xad,
+	0x0a, 0xea, 0xb8, 0x42, 0xf9, 0x5e, 0xb3, 0x3e,
+	0x36, 0x4c, 0xaf, 0x75, 0x9e, 0x1c, 0xeb, 0xbd,
+	0xbc, 0xbb, 0x80, 0x40, 0xa7, 0x3a, 0x30, 0xbf,
+	0xa8, 0x44, 0xf4, 0xeb, 0x38, 0xad, 0x29, 0xba,
+	0x23, 0xed, 0x41, 0x0c, 0xea, 0xd2, 0xbb, 0x41,
+	0x18, 0xd6, 0xb9, 0xba, 0x65, 0x2b, 0xa3, 0x91,
+	0x6d, 0x1f, 0xa9, 0xf4, 0xd1, 0x25, 0x8d, 0x4d,
+	0x38, 0xff, 0x64, 0xa0, 0xec, 0xde, 0xa6, 0xb6,
+	0x79, 0xab, 0x8e, 0x33, 0x6c, 0x47, 0xde, 0xaf,
+	0x94, 0xa4, 0xa5, 0x86, 0x77, 0x55, 0x09, 0x92,
+	0x81, 0x31, 0x76, 0xc7, 0x34, 0x22, 0x89, 0x8e,
+	0x3d, 0x26, 0x26, 0xd7, 0xfc, 0x1e, 0x16, 0x72,
+	0x13, 0x33, 0x63, 0xd5, 0x22, 0xbe, 0xb8, 0x04,
+	0x34, 0x84, 0x41, 0xbb, 0x80, 0xd0, 0x9f, 0x46,
+	0x48, 0x07, 0xa7, 0xfc, 0x2b, 0x3a, 0x75, 0x55,
+	0x8c, 0xc7, 0x6a, 0xbd, 0x7e, 0x46, 0x08, 0x84,
+	0x0f, 0xd5, 0x74, 0xc0, 0x82, 0x8e, 0xaa, 0x61,
+	0x05, 0x01, 0xb2, 0x47, 0x6e, 0x20, 0x6a, 0x2d,
+	0x58, 0x70, 0x48, 0x32, 0xa7, 0x37, 0xd2, 0xb8,
+	0x82, 0x1a, 0x51, 0xb9, 0x61, 0xdd, 0xfd, 0x9d,
+	0x6b, 0x0e, 0x18, 0x97, 0xf8, 0x45, 0x5f, 0x87,
+	0x10, 0xcf, 0x34, 0x72, 0x45, 0x26, 0x49, 0x70,
+	0xe7, 0xa3, 0x78, 0xe0, 0x52, 0x89, 0x84, 0x94,
+	0x83, 0x82, 0xc2, 0x69, 0x8f, 0xe3, 0xe1, 0x3f,
+	0x60, 0x74, 0x88, 0xc4, 0xf7, 0x75, 0x2c, 0xfb,
+	0xbd, 0xb6, 0xc4, 0x7e, 0x10, 0x0a, 0x6c, 0x90,
+	0x04, 0x9e, 0xc3, 0x3f, 0x59, 0x7c, 0xce, 0x31,
+	0x18, 0x60, 0x57, 0x73, 0x46, 0x94, 0x7d, 0x06,
+	0xa0, 0x6d, 0x44, 0xec, 0xa2, 0x0a, 0x9e, 0x05,
+	0x15, 0xef, 0xca, 0x5c, 0xbf, 0x00, 0xeb, 0xf7,
+	0x3d, 0x32, 0xd4, 0xa5, 0xef, 0x49, 0x89, 0x5e,
+	0x46, 0xb0, 0xa6, 0x63, 0x5b, 0x8a, 0x73, 0xae,
+	0x6f, 0xd5, 0x9d, 0xf8, 0x4f, 0x40, 0xb5, 0xb2,
+	0x6e, 0xd3, 0xb6, 0x01, 0xa9, 0x26, 0xa2, 0x21,
+	0xcf, 0x33, 0x7a, 0x3a, 0xa4, 0x23, 0x13, 0xb0,
+	0x69, 0x6a, 0xee, 0xce, 0xd8, 0x9d, 0x01, 0x1d,
+	0x50, 0xc1, 0x30, 0x6c, 0xb1, 0xcd, 0xa0, 0xf0,
+	0xf0, 0xa2, 0x64, 0x6f, 0xbb, 0xbf, 0x5e, 0xe6,
+	0xab, 0x87, 0xb4, 0x0f, 0x4f, 0x15, 0xaf, 0xb5,
+	0x25, 0xa1, 0xb2, 0xd0, 0x80, 0x2c, 0xfb, 0xf9,
+	0xfe, 0xd2, 0x33, 0xbb, 0x76, 0xfe, 0x7c, 0xa8,
+	0x66, 0xf7, 0xe7, 0x85, 0x9f, 0x1f, 0x85, 0x57,
+	0x88, 0xe1, 0xe9, 0x63, 0xe4, 0xd8, 0x1c, 0xa1,
+	0xfb, 0xda, 0x44, 0x05, 0x2e, 0x1d, 0x3a, 0x1c,
+	0xff, 0xc8, 0x3b, 0xc0, 0xfe, 0xda, 0x22, 0x0b,
+	0x43, 0xd6, 0x88, 0x39, 0x4c, 0x4a, 0xa6, 0x69,
+	0x18, 0x93, 0x42, 0x4e, 0xb5, 0xcc, 0x66, 0x0d,
+	0x09, 0xf8, 0x1e, 0x7c, 0xd3, 0x3c, 0x99, 0x0d,
+	0x50, 0x1d, 0x62, 0xe9, 0x57, 0x06, 0xbf, 0x19,
+	0x88, 0xdd, 0xad, 0x7b, 0x4f, 0xf9, 0xc7, 0x82,
+	0x6d, 0x8d, 0xc8, 0xc4, 0xc5, 0x78, 0x17, 0x20,
+	0x15, 0xc5, 0x52, 0x41, 0xcf, 0x5b, 0xd6, 0x7f,
+	0x94, 0x02, 0x41, 0xe0, 0x40, 0x22, 0x03, 0x5e,
+	0xd1, 0x53, 0xd4, 0x86, 0xd3, 0x2c, 0x9f, 0x0f,
+	0x96, 0xe3, 0x6b, 0x9a, 0x76, 0x32, 0x06, 0x47,
+	0x4b, 0x11, 0xb3, 0xdd, 0x03, 0x65, 0xbd, 0x9b,
+	0x01, 0xda, 0x9c, 0xb9, 0x7e, 0x3f, 0x6a, 0xc4,
+	0x7b, 0xea, 0xd4, 0x3c, 0xb9, 0xfb, 0x5c, 0x6b,
+	0x64, 0x33, 0x52, 0xba, 0x64, 0x78, 0x8f, 0xa4,
+	0xaf, 0x7a, 0x61, 0x8d, 0xbc, 0xc5, 0x73, 0xe9,
+	0x6b, 0x58, 0x97, 0x4b, 0xbf, 0x63, 0x22, 0xd3,
+	0x37, 0x02, 0x54, 0xc5, 0xb9, 0x16, 0x4a, 0xf0,
+	0x19, 0xd8, 0x94, 0x57, 0xb8, 0x8a, 0xb3, 0x16,
+	0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d,
+	0x78, 0xec, 0x00
+};
+static const u8 dec_assoc013[] __initconst = {
+	0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8,
+	0x77, 0xe8, 0x21, 0xff, 0x06, 0x59, 0x35, 0xce,
+	0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c,
+	0xcf, 0x70, 0xdd, 0x0a, 0x68, 0xbf, 0xd4, 0xbc,
+	0x16, 0x76, 0x99, 0x36, 0x1e, 0x58, 0x79, 0x5e,
+	0xd4, 0x29, 0xf7, 0x33, 0x93, 0x48, 0xdb, 0x5f,
+	0x01, 0xae, 0x9c, 0xb6, 0xe4, 0x88, 0x6d, 0x2b,
+	0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9
+};
+static const u8 dec_nonce013[] __initconst = {
+	0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06
+};
+static const u8 dec_key013[] __initconst = {
+	0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e,
+	0x8f, 0x59, 0x8e, 0xc5, 0x90, 0xd5, 0x27, 0x2d,
+	0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, 0x44, 0x1e,
+	0x65, 0x91, 0x6e, 0x2a, 0x79, 0x22, 0xda, 0x64
+};
+
+static const struct chacha20poly1305_testvec
+chacha20poly1305_dec_vectors[] __initconst = {
+	{ dec_input001, dec_output001, dec_assoc001, dec_nonce001, dec_key001,
+	  sizeof(dec_input001), sizeof(dec_assoc001), sizeof(dec_nonce001) },
+	{ dec_input002, dec_output002, dec_assoc002, dec_nonce002, dec_key002,
+	  sizeof(dec_input002), sizeof(dec_assoc002), sizeof(dec_nonce002) },
+	{ dec_input003, dec_output003, dec_assoc003, dec_nonce003, dec_key003,
+	  sizeof(dec_input003), sizeof(dec_assoc003), sizeof(dec_nonce003) },
+	{ dec_input004, dec_output004, dec_assoc004, dec_nonce004, dec_key004,
+	  sizeof(dec_input004), sizeof(dec_assoc004), sizeof(dec_nonce004) },
+	{ dec_input005, dec_output005, dec_assoc005, dec_nonce005, dec_key005,
+	  sizeof(dec_input005), sizeof(dec_assoc005), sizeof(dec_nonce005) },
+	{ dec_input006, dec_output006, dec_assoc006, dec_nonce006, dec_key006,
+	  sizeof(dec_input006), sizeof(dec_assoc006), sizeof(dec_nonce006) },
+	{ dec_input007, dec_output007, dec_assoc007, dec_nonce007, dec_key007,
+	  sizeof(dec_input007), sizeof(dec_assoc007), sizeof(dec_nonce007) },
+	{ dec_input008, dec_output008, dec_assoc008, dec_nonce008, dec_key008,
+	  sizeof(dec_input008), sizeof(dec_assoc008), sizeof(dec_nonce008) },
+	{ dec_input009, dec_output009, dec_assoc009, dec_nonce009, dec_key009,
+	  sizeof(dec_input009), sizeof(dec_assoc009), sizeof(dec_nonce009) },
+	{ dec_input010, dec_output010, dec_assoc010, dec_nonce010, dec_key010,
+	  sizeof(dec_input010), sizeof(dec_assoc010), sizeof(dec_nonce010) },
+	{ dec_input011, dec_output011, dec_assoc011, dec_nonce011, dec_key011,
+	  sizeof(dec_input011), sizeof(dec_assoc011), sizeof(dec_nonce011) },
+	{ dec_input012, dec_output012, dec_assoc012, dec_nonce012, dec_key012,
+	  sizeof(dec_input012), sizeof(dec_assoc012), sizeof(dec_nonce012) },
+	{ dec_input013, dec_output013, dec_assoc013, dec_nonce013, dec_key013,
+	  sizeof(dec_input013), sizeof(dec_assoc013), sizeof(dec_nonce013),
+	  true }
+};
+
+static const u8 xenc_input001[] __initconst = {
+	0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74,
+	0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20,
+	0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66,
+	0x74, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65,
+	0x6e, 0x74, 0x73, 0x20, 0x76, 0x61, 0x6c, 0x69,
+	0x64, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x20,
+	0x6d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x20,
+	0x6f, 0x66, 0x20, 0x73, 0x69, 0x78, 0x20, 0x6d,
+	0x6f, 0x6e, 0x74, 0x68, 0x73, 0x20, 0x61, 0x6e,
+	0x64, 0x20, 0x6d, 0x61, 0x79, 0x20, 0x62, 0x65,
+	0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64,
+	0x2c, 0x20, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63,
+	0x65, 0x64, 0x2c, 0x20, 0x6f, 0x72, 0x20, 0x6f,
+	0x62, 0x73, 0x6f, 0x6c, 0x65, 0x74, 0x65, 0x64,
+	0x20, 0x62, 0x79, 0x20, 0x6f, 0x74, 0x68, 0x65,
+	0x72, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65,
+	0x6e, 0x74, 0x73, 0x20, 0x61, 0x74, 0x20, 0x61,
+	0x6e, 0x79, 0x20, 0x74, 0x69, 0x6d, 0x65, 0x2e,
+	0x20, 0x49, 0x74, 0x20, 0x69, 0x73, 0x20, 0x69,
+	0x6e, 0x61, 0x70, 0x70, 0x72, 0x6f, 0x70, 0x72,
+	0x69, 0x61, 0x74, 0x65, 0x20, 0x74, 0x6f, 0x20,
+	0x75, 0x73, 0x65, 0x20, 0x49, 0x6e, 0x74, 0x65,
+	0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61,
+	0x66, 0x74, 0x73, 0x20, 0x61, 0x73, 0x20, 0x72,
+	0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65,
+	0x20, 0x6d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61,
+	0x6c, 0x20, 0x6f, 0x72, 0x20, 0x74, 0x6f, 0x20,
+	0x63, 0x69, 0x74, 0x65, 0x20, 0x74, 0x68, 0x65,
+	0x6d, 0x20, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x20,
+	0x74, 0x68, 0x61, 0x6e, 0x20, 0x61, 0x73, 0x20,
+	0x2f, 0xe2, 0x80, 0x9c, 0x77, 0x6f, 0x72, 0x6b,
+	0x20, 0x69, 0x6e, 0x20, 0x70, 0x72, 0x6f, 0x67,
+	0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80,
+	0x9d
+};
+static const u8 xenc_output001[] __initconst = {
+	0x1a, 0x6e, 0x3a, 0xd9, 0xfd, 0x41, 0x3f, 0x77,
+	0x54, 0x72, 0x0a, 0x70, 0x9a, 0xa0, 0x29, 0x92,
+	0x2e, 0xed, 0x93, 0xcf, 0x0f, 0x71, 0x88, 0x18,
+	0x7a, 0x9d, 0x2d, 0x24, 0xe0, 0xf5, 0xea, 0x3d,
+	0x55, 0x64, 0xd7, 0xad, 0x2a, 0x1a, 0x1f, 0x7e,
+	0x86, 0x6d, 0xb0, 0xce, 0x80, 0x41, 0x72, 0x86,
+	0x26, 0xee, 0x84, 0xd7, 0xef, 0x82, 0x9e, 0xe2,
+	0x60, 0x9d, 0x5a, 0xfc, 0xf0, 0xe4, 0x19, 0x85,
+	0xea, 0x09, 0xc6, 0xfb, 0xb3, 0xa9, 0x50, 0x09,
+	0xec, 0x5e, 0x11, 0x90, 0xa1, 0xc5, 0x4e, 0x49,
+	0xef, 0x50, 0xd8, 0x8f, 0xe0, 0x78, 0xd7, 0xfd,
+	0xb9, 0x3b, 0xc9, 0xf2, 0x91, 0xc8, 0x25, 0xc8,
+	0xa7, 0x63, 0x60, 0xce, 0x10, 0xcd, 0xc6, 0x7f,
+	0xf8, 0x16, 0xf8, 0xe1, 0x0a, 0xd9, 0xde, 0x79,
+	0x50, 0x33, 0xf2, 0x16, 0x0f, 0x17, 0xba, 0xb8,
+	0x5d, 0xd8, 0xdf, 0x4e, 0x51, 0xa8, 0x39, 0xd0,
+	0x85, 0xca, 0x46, 0x6a, 0x10, 0xa7, 0xa3, 0x88,
+	0xef, 0x79, 0xb9, 0xf8, 0x24, 0xf3, 0xe0, 0x71,
+	0x7b, 0x76, 0x28, 0x46, 0x3a, 0x3a, 0x1b, 0x91,
+	0xb6, 0xd4, 0x3e, 0x23, 0xe5, 0x44, 0x15, 0xbf,
+	0x60, 0x43, 0x9d, 0xa4, 0xbb, 0xd5, 0x5f, 0x89,
+	0xeb, 0xef, 0x8e, 0xfd, 0xdd, 0xb4, 0x0d, 0x46,
+	0xf0, 0x69, 0x23, 0x63, 0xae, 0x94, 0xf5, 0x5e,
+	0xa5, 0xad, 0x13, 0x1c, 0x41, 0x76, 0xe6, 0x90,
+	0xd6, 0x6d, 0xa2, 0x8f, 0x97, 0x4c, 0xa8, 0x0b,
+	0xcf, 0x8d, 0x43, 0x2b, 0x9c, 0x9b, 0xc5, 0x58,
+	0xa5, 0xb6, 0x95, 0x9a, 0xbf, 0x81, 0xc6, 0x54,
+	0xc9, 0x66, 0x0c, 0xe5, 0x4f, 0x6a, 0x53, 0xa1,
+	0xe5, 0x0c, 0xba, 0x31, 0xde, 0x34, 0x64, 0x73,
+	0x8a, 0x3b, 0xbd, 0x92, 0x01, 0xdb, 0x71, 0x69,
+	0xf3, 0x58, 0x99, 0xbc, 0xd1, 0xcb, 0x4a, 0x05,
+	0xe2, 0x58, 0x9c, 0x25, 0x17, 0xcd, 0xdc, 0x83,
+	0xb7, 0xff, 0xfb, 0x09, 0x61, 0xad, 0xbf, 0x13,
+	0x5b, 0x5e, 0xed, 0x46, 0x82, 0x6f, 0x22, 0xd8,
+	0x93, 0xa6, 0x85, 0x5b, 0x40, 0x39, 0x5c, 0xc5,
+	0x9c
+};
+static const u8 xenc_assoc001[] __initconst = {
+	0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x4e, 0x91
+};
+static const u8 xenc_nonce001[] __initconst = {
+	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+	0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+	0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17
+};
+static const u8 xenc_key001[] __initconst = {
+	0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a,
+	0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0,
+	0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09,
+	0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0
+};
+
+static const struct chacha20poly1305_testvec
+xchacha20poly1305_enc_vectors[] __initconst = {
+	{ xenc_input001, xenc_output001, xenc_assoc001, xenc_nonce001, xenc_key001,
+	  sizeof(xenc_input001), sizeof(xenc_assoc001), sizeof(xenc_nonce001) }
+};
+
+static const u8 xdec_input001[] __initconst = {
+	0x1a, 0x6e, 0x3a, 0xd9, 0xfd, 0x41, 0x3f, 0x77,
+	0x54, 0x72, 0x0a, 0x70, 0x9a, 0xa0, 0x29, 0x92,
+	0x2e, 0xed, 0x93, 0xcf, 0x0f, 0x71, 0x88, 0x18,
+	0x7a, 0x9d, 0x2d, 0x24, 0xe0, 0xf5, 0xea, 0x3d,
+	0x55, 0x64, 0xd7, 0xad, 0x2a, 0x1a, 0x1f, 0x7e,
+	0x86, 0x6d, 0xb0, 0xce, 0x80, 0x41, 0x72, 0x86,
+	0x26, 0xee, 0x84, 0xd7, 0xef, 0x82, 0x9e, 0xe2,
+	0x60, 0x9d, 0x5a, 0xfc, 0xf0, 0xe4, 0x19, 0x85,
+	0xea, 0x09, 0xc6, 0xfb, 0xb3, 0xa9, 0x50, 0x09,
+	0xec, 0x5e, 0x11, 0x90, 0xa1, 0xc5, 0x4e, 0x49,
+	0xef, 0x50, 0xd8, 0x8f, 0xe0, 0x78, 0xd7, 0xfd,
+	0xb9, 0x3b, 0xc9, 0xf2, 0x91, 0xc8, 0x25, 0xc8,
+	0xa7, 0x63, 0x60, 0xce, 0x10, 0xcd, 0xc6, 0x7f,
+	0xf8, 0x16, 0xf8, 0xe1, 0x0a, 0xd9, 0xde, 0x79,
+	0x50, 0x33, 0xf2, 0x16, 0x0f, 0x17, 0xba, 0xb8,
+	0x5d, 0xd8, 0xdf, 0x4e, 0x51, 0xa8, 0x39, 0xd0,
+	0x85, 0xca, 0x46, 0x6a, 0x10, 0xa7, 0xa3, 0x88,
+	0xef, 0x79, 0xb9, 0xf8, 0x24, 0xf3, 0xe0, 0x71,
+	0x7b, 0x76, 0x28, 0x46, 0x3a, 0x3a, 0x1b, 0x91,
+	0xb6, 0xd4, 0x3e, 0x23, 0xe5, 0x44, 0x15, 0xbf,
+	0x60, 0x43, 0x9d, 0xa4, 0xbb, 0xd5, 0x5f, 0x89,
+	0xeb, 0xef, 0x8e, 0xfd, 0xdd, 0xb4, 0x0d, 0x46,
+	0xf0, 0x69, 0x23, 0x63, 0xae, 0x94, 0xf5, 0x5e,
+	0xa5, 0xad, 0x13, 0x1c, 0x41, 0x76, 0xe6, 0x90,
+	0xd6, 0x6d, 0xa2, 0x8f, 0x97, 0x4c, 0xa8, 0x0b,
+	0xcf, 0x8d, 0x43, 0x2b, 0x9c, 0x9b, 0xc5, 0x58,
+	0xa5, 0xb6, 0x95, 0x9a, 0xbf, 0x81, 0xc6, 0x54,
+	0xc9, 0x66, 0x0c, 0xe5, 0x4f, 0x6a, 0x53, 0xa1,
+	0xe5, 0x0c, 0xba, 0x31, 0xde, 0x34, 0x64, 0x73,
+	0x8a, 0x3b, 0xbd, 0x92, 0x01, 0xdb, 0x71, 0x69,
+	0xf3, 0x58, 0x99, 0xbc, 0xd1, 0xcb, 0x4a, 0x05,
+	0xe2, 0x58, 0x9c, 0x25, 0x17, 0xcd, 0xdc, 0x83,
+	0xb7, 0xff, 0xfb, 0x09, 0x61, 0xad, 0xbf, 0x13,
+	0x5b, 0x5e, 0xed, 0x46, 0x82, 0x6f, 0x22, 0xd8,
+	0x93, 0xa6, 0x85, 0x5b, 0x40, 0x39, 0x5c, 0xc5,
+	0x9c
+};
+static const u8 xdec_output001[] __initconst = {
+	0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74,
+	0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20,
+	0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66,
+	0x74, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65,
+	0x6e, 0x74, 0x73, 0x20, 0x76, 0x61, 0x6c, 0x69,
+	0x64, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x20,
+	0x6d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x20,
+	0x6f, 0x66, 0x20, 0x73, 0x69, 0x78, 0x20, 0x6d,
+	0x6f, 0x6e, 0x74, 0x68, 0x73, 0x20, 0x61, 0x6e,
+	0x64, 0x20, 0x6d, 0x61, 0x79, 0x20, 0x62, 0x65,
+	0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64,
+	0x2c, 0x20, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63,
+	0x65, 0x64, 0x2c, 0x20, 0x6f, 0x72, 0x20, 0x6f,
+	0x62, 0x73, 0x6f, 0x6c, 0x65, 0x74, 0x65, 0x64,
+	0x20, 0x62, 0x79, 0x20, 0x6f, 0x74, 0x68, 0x65,
+	0x72, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65,
+	0x6e, 0x74, 0x73, 0x20, 0x61, 0x74, 0x20, 0x61,
+	0x6e, 0x79, 0x20, 0x74, 0x69, 0x6d, 0x65, 0x2e,
+	0x20, 0x49, 0x74, 0x20, 0x69, 0x73, 0x20, 0x69,
+	0x6e, 0x61, 0x70, 0x70, 0x72, 0x6f, 0x70, 0x72,
+	0x69, 0x61, 0x74, 0x65, 0x20, 0x74, 0x6f, 0x20,
+	0x75, 0x73, 0x65, 0x20, 0x49, 0x6e, 0x74, 0x65,
+	0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61,
+	0x66, 0x74, 0x73, 0x20, 0x61, 0x73, 0x20, 0x72,
+	0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65,
+	0x20, 0x6d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61,
+	0x6c, 0x20, 0x6f, 0x72, 0x20, 0x74, 0x6f, 0x20,
+	0x63, 0x69, 0x74, 0x65, 0x20, 0x74, 0x68, 0x65,
+	0x6d, 0x20, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x20,
+	0x74, 0x68, 0x61, 0x6e, 0x20, 0x61, 0x73, 0x20,
+	0x2f, 0xe2, 0x80, 0x9c, 0x77, 0x6f, 0x72, 0x6b,
+	0x20, 0x69, 0x6e, 0x20, 0x70, 0x72, 0x6f, 0x67,
+	0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80,
+	0x9d
+};
+static const u8 xdec_assoc001[] __initconst = {
+	0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x4e, 0x91
+};
+static const u8 xdec_nonce001[] __initconst = {
+	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+	0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+	0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17
+};
+static const u8 xdec_key001[] __initconst = {
+	0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a,
+	0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0,
+	0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09,
+	0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0
+};
+
+static const struct chacha20poly1305_testvec
+xchacha20poly1305_dec_vectors[] __initconst = {
+	{ xdec_input001, xdec_output001, xdec_assoc001, xdec_nonce001, xdec_key001,
+	  sizeof(xdec_input001), sizeof(xdec_assoc001), sizeof(xdec_nonce001) }
+};
+
+static void __init
+chacha20poly1305_selftest_encrypt_bignonce(u8 *dst, const u8 *src,
+					   const size_t src_len, const u8 *ad,
+					   const size_t ad_len,
+					   const u8 nonce[12],
+					   const u8 key[CHACHA20POLY1305_KEY_SIZE])
+{
+	simd_context_t simd_context;
+	struct poly1305_ctx poly1305_state;
+	struct chacha20_ctx chacha20_state;
+	union {
+		u8 block0[POLY1305_KEY_SIZE];
+		__le64 lens[2];
+	} b = {{ 0 }};
+
+	simd_get(&simd_context);
+	chacha20_init(&chacha20_state, key, 0);
+	chacha20_state.counter[1] = get_unaligned_le32(nonce + 0);
+	chacha20_state.counter[2] = get_unaligned_le32(nonce + 4);
+	chacha20_state.counter[3] = get_unaligned_le32(nonce + 8);
+	chacha20(&chacha20_state, b.block0, b.block0, sizeof(b.block0),
+		 &simd_context);
+	poly1305_init(&poly1305_state, b.block0);
+	poly1305_update(&poly1305_state, ad, ad_len, &simd_context);
+	poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf,
+			&simd_context);
+	chacha20(&chacha20_state, dst, src, src_len, &simd_context);
+	poly1305_update(&poly1305_state, dst, src_len, &simd_context);
+	poly1305_update(&poly1305_state, pad0, (0x10 - src_len) & 0xf,
+			&simd_context);
+	b.lens[0] = cpu_to_le64(ad_len);
+	b.lens[1] = cpu_to_le64(src_len);
+	poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens),
+			&simd_context);
+	poly1305_final(&poly1305_state, dst + src_len, &simd_context);
+	simd_put(&simd_context);
+	memzero_explicit(&chacha20_state, sizeof(chacha20_state));
+	memzero_explicit(&b, sizeof(b));
+}
+
+static void __init
+chacha20poly1305_selftest_encrypt(u8 *dst, const u8 *src, const size_t src_len,
+				  const u8 *ad, const size_t ad_len,
+				  const u8 *nonce, const size_t nonce_len,
+				  const u8 key[CHACHA20POLY1305_KEY_SIZE])
+{
+	if (nonce_len == 8)
+		chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len,
+					 get_unaligned_le64(nonce), key);
+	else if (nonce_len == 12)
+		chacha20poly1305_selftest_encrypt_bignonce(dst, src, src_len,
+							   ad, ad_len, nonce,
+							   key);
+	else
+		BUG();
+}
+
+static bool __init
+decryption_success(bool func_ret, bool expect_failure, int memcmp_result)
+{
+	if (expect_failure)
+		return !func_ret;
+	return func_ret && !memcmp_result;
+}
+
+static bool __init chacha20poly1305_selftest(void)
+{
+	enum { MAXIMUM_TEST_BUFFER_LEN = 1UL << 12 };
+	size_t i, j, k, total_len;
+	u8 *computed_output = NULL, *input = NULL;
+	bool success = true, ret;
+	simd_context_t simd_context;
+	struct scatterlist sg_src[3];
+
+	computed_output = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL);
+	input = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL);
+	if (!computed_output || !input) {
+		pr_err("chacha20poly1305 self-test malloc: FAIL\n");
+		success = false;
+		goto out;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(chacha20poly1305_enc_vectors); ++i) {
+		memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN);
+		chacha20poly1305_selftest_encrypt(computed_output,
+					chacha20poly1305_enc_vectors[i].input,
+					chacha20poly1305_enc_vectors[i].ilen,
+					chacha20poly1305_enc_vectors[i].assoc,
+					chacha20poly1305_enc_vectors[i].alen,
+					chacha20poly1305_enc_vectors[i].nonce,
+					chacha20poly1305_enc_vectors[i].nlen,
+					chacha20poly1305_enc_vectors[i].key);
+		if (memcmp(computed_output,
+			   chacha20poly1305_enc_vectors[i].output,
+			   chacha20poly1305_enc_vectors[i].ilen +
+							POLY1305_MAC_SIZE)) {
+			pr_err("chacha20poly1305 encryption self-test %zu: FAIL\n",
+			       i + 1);
+			success = false;
+		}
+	}
+	simd_get(&simd_context);
+	for (i = 0; i < ARRAY_SIZE(chacha20poly1305_enc_vectors); ++i) {
+		if (chacha20poly1305_enc_vectors[i].nlen != 8)
+			continue;
+		memcpy(computed_output, chacha20poly1305_enc_vectors[i].input,
+		       chacha20poly1305_enc_vectors[i].ilen);
+		sg_init_one(sg_src, computed_output,
+			    chacha20poly1305_enc_vectors[i].ilen +
+				POLY1305_MAC_SIZE);
+		ret = chacha20poly1305_encrypt_sg_inplace(sg_src,
+			chacha20poly1305_enc_vectors[i].ilen,
+			chacha20poly1305_enc_vectors[i].assoc,
+			chacha20poly1305_enc_vectors[i].alen,
+			get_unaligned_le64(chacha20poly1305_enc_vectors[i].nonce),
+			chacha20poly1305_enc_vectors[i].key,
+			&simd_context);
+		if (!ret || memcmp(computed_output,
+				   chacha20poly1305_enc_vectors[i].output,
+				   chacha20poly1305_enc_vectors[i].ilen +
+							POLY1305_MAC_SIZE)) {
+			pr_err("chacha20poly1305 sg encryption self-test %zu: FAIL\n",
+			       i + 1);
+			success = false;
+		}
+	}
+	simd_put(&simd_context);
+	for (i = 0; i < ARRAY_SIZE(chacha20poly1305_dec_vectors); ++i) {
+		memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN);
+		ret = chacha20poly1305_decrypt(computed_output,
+			chacha20poly1305_dec_vectors[i].input,
+			chacha20poly1305_dec_vectors[i].ilen,
+			chacha20poly1305_dec_vectors[i].assoc,
+			chacha20poly1305_dec_vectors[i].alen,
+			get_unaligned_le64(chacha20poly1305_dec_vectors[i].nonce),
+			chacha20poly1305_dec_vectors[i].key);
+		if (!decryption_success(ret,
+				chacha20poly1305_dec_vectors[i].failure,
+				memcmp(computed_output,
+				       chacha20poly1305_dec_vectors[i].output,
+				       chacha20poly1305_dec_vectors[i].ilen -
+							POLY1305_MAC_SIZE))) {
+			pr_err("chacha20poly1305 decryption self-test %zu: FAIL\n",
+			       i + 1);
+			success = false;
+		}
+	}
+	simd_get(&simd_context);
+	for (i = 0; i < ARRAY_SIZE(chacha20poly1305_dec_vectors); ++i) {
+		memcpy(computed_output, chacha20poly1305_dec_vectors[i].input,
+		       chacha20poly1305_dec_vectors[i].ilen);
+		sg_init_one(sg_src, computed_output,
+			    chacha20poly1305_dec_vectors[i].ilen);
+		ret = chacha20poly1305_decrypt_sg_inplace(sg_src,
+			chacha20poly1305_dec_vectors[i].ilen,
+			chacha20poly1305_dec_vectors[i].assoc,
+			chacha20poly1305_dec_vectors[i].alen,
+			get_unaligned_le64(chacha20poly1305_dec_vectors[i].nonce),
+			chacha20poly1305_dec_vectors[i].key, &simd_context);
+		if (!decryption_success(ret,
+			chacha20poly1305_dec_vectors[i].failure,
+			memcmp(computed_output, chacha20poly1305_dec_vectors[i].output,
+			       chacha20poly1305_dec_vectors[i].ilen -
+							POLY1305_MAC_SIZE))) {
+			pr_err("chacha20poly1305 sg decryption self-test %zu: FAIL\n",
+			       i + 1);
+			success = false;
+		}
+	}
+	simd_put(&simd_context);
+	for (i = 0; i < ARRAY_SIZE(xchacha20poly1305_enc_vectors); ++i) {
+		memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN);
+		xchacha20poly1305_encrypt(computed_output,
+					xchacha20poly1305_enc_vectors[i].input,
+					xchacha20poly1305_enc_vectors[i].ilen,
+					xchacha20poly1305_enc_vectors[i].assoc,
+					xchacha20poly1305_enc_vectors[i].alen,
+					xchacha20poly1305_enc_vectors[i].nonce,
+					xchacha20poly1305_enc_vectors[i].key);
+		if (memcmp(computed_output,
+			   xchacha20poly1305_enc_vectors[i].output,
+			   xchacha20poly1305_enc_vectors[i].ilen +
+							POLY1305_MAC_SIZE)) {
+			pr_err("xchacha20poly1305 encryption self-test %zu: FAIL\n",
+			       i + 1);
+			success = false;
+		}
+	}
+	for (i = 0; i < ARRAY_SIZE(xchacha20poly1305_dec_vectors); ++i) {
+		memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN);
+		ret = xchacha20poly1305_decrypt(computed_output,
+					xchacha20poly1305_dec_vectors[i].input,
+					xchacha20poly1305_dec_vectors[i].ilen,
+					xchacha20poly1305_dec_vectors[i].assoc,
+					xchacha20poly1305_dec_vectors[i].alen,
+					xchacha20poly1305_dec_vectors[i].nonce,
+					xchacha20poly1305_dec_vectors[i].key);
+		if (!decryption_success(ret,
+				xchacha20poly1305_dec_vectors[i].failure,
+				memcmp(computed_output,
+				       xchacha20poly1305_dec_vectors[i].output,
+				       xchacha20poly1305_dec_vectors[i].ilen -
+							POLY1305_MAC_SIZE))) {
+			pr_err("xchacha20poly1305 decryption self-test %zu: FAIL\n",
+			       i + 1);
+			success = false;
+		}
+	}
+
+	simd_get(&simd_context);
+	for (total_len = POLY1305_MAC_SIZE; IS_ENABLED(DEBUG_CHACHA20POLY1305_SLOW_CHUNK_TEST)
+	     && total_len <= 1 << 10; ++total_len) {
+		for (i = 0; i <= total_len; ++i) {
+			for (j = i; j <= total_len; ++j) {
+				sg_init_table(sg_src, 3);
+				sg_set_buf(&sg_src[0], input, i);
+				sg_set_buf(&sg_src[1], input + i, j - i);
+				sg_set_buf(&sg_src[2], input + j, total_len - j);
+				memset(computed_output, 0, total_len);
+				memset(input, 0, total_len);
+
+				if (!chacha20poly1305_encrypt_sg_inplace(sg_src,
+					total_len - POLY1305_MAC_SIZE, NULL, 0,
+					0, enc_key001, &simd_context))
+					goto chunkfail;
+				chacha20poly1305_encrypt(computed_output,
+					computed_output,
+					total_len - POLY1305_MAC_SIZE, NULL, 0, 0,
+					enc_key001);
+				if (memcmp(computed_output, input, total_len))
+					goto chunkfail;;
+				if (!chacha20poly1305_decrypt(computed_output,
+					input, total_len, NULL, 0, 0, enc_key001))
+					goto chunkfail;
+				for (k = 0; k < total_len - POLY1305_MAC_SIZE; ++k) {
+					if (computed_output[k])
+						goto chunkfail;
+				}
+				if (!chacha20poly1305_decrypt_sg_inplace(sg_src,
+					total_len, NULL, 0, 0, enc_key001,
+					&simd_context))
+					goto chunkfail;
+				for (k = 0; k < total_len - POLY1305_MAC_SIZE; ++k) {
+					if (input[k])
+						goto chunkfail;
+				}
+				continue;
+
+			chunkfail:
+				pr_err("chacha20poly1305 chunked self-test %zu/%zu/%zu: FAIL\n",
+				       total_len, i, j);
+				success = false;
+			}
+
+		}
+	}
+	simd_put(&simd_context);
+
+out:
+	kfree(computed_output);
+	kfree(input);
+	return success;
+}
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/selftest/curve25519.c b/net/wireguard/crypto/zinc/selftest/curve25519.c
--- a/net/wireguard/crypto/zinc/selftest/curve25519.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/selftest/curve25519.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,1315 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+struct curve25519_test_vector {
+	u8 private[CURVE25519_KEY_SIZE];
+	u8 public[CURVE25519_KEY_SIZE];
+	u8 result[CURVE25519_KEY_SIZE];
+	bool valid;
+};
+static const struct curve25519_test_vector curve25519_test_vectors[] __initconst = {
+	{
+		.private = { 0x77, 0x07, 0x6d, 0x0a, 0x73, 0x18, 0xa5, 0x7d,
+			     0x3c, 0x16, 0xc1, 0x72, 0x51, 0xb2, 0x66, 0x45,
+			     0xdf, 0x4c, 0x2f, 0x87, 0xeb, 0xc0, 0x99, 0x2a,
+			     0xb1, 0x77, 0xfb, 0xa5, 0x1d, 0xb9, 0x2c, 0x2a },
+		.public = { 0xde, 0x9e, 0xdb, 0x7d, 0x7b, 0x7d, 0xc1, 0xb4,
+			    0xd3, 0x5b, 0x61, 0xc2, 0xec, 0xe4, 0x35, 0x37,
+			    0x3f, 0x83, 0x43, 0xc8, 0x5b, 0x78, 0x67, 0x4d,
+			    0xad, 0xfc, 0x7e, 0x14, 0x6f, 0x88, 0x2b, 0x4f },
+		.result = { 0x4a, 0x5d, 0x9d, 0x5b, 0xa4, 0xce, 0x2d, 0xe1,
+			    0x72, 0x8e, 0x3b, 0xf4, 0x80, 0x35, 0x0f, 0x25,
+			    0xe0, 0x7e, 0x21, 0xc9, 0x47, 0xd1, 0x9e, 0x33,
+			    0x76, 0xf0, 0x9b, 0x3c, 0x1e, 0x16, 0x17, 0x42 },
+		.valid = true
+	},
+	{
+		.private = { 0x5d, 0xab, 0x08, 0x7e, 0x62, 0x4a, 0x8a, 0x4b,
+			     0x79, 0xe1, 0x7f, 0x8b, 0x83, 0x80, 0x0e, 0xe6,
+			     0x6f, 0x3b, 0xb1, 0x29, 0x26, 0x18, 0xb6, 0xfd,
+			     0x1c, 0x2f, 0x8b, 0x27, 0xff, 0x88, 0xe0, 0xeb },
+		.public = { 0x85, 0x20, 0xf0, 0x09, 0x89, 0x30, 0xa7, 0x54,
+			    0x74, 0x8b, 0x7d, 0xdc, 0xb4, 0x3e, 0xf7, 0x5a,
+			    0x0d, 0xbf, 0x3a, 0x0d, 0x26, 0x38, 0x1a, 0xf4,
+			    0xeb, 0xa4, 0xa9, 0x8e, 0xaa, 0x9b, 0x4e, 0x6a },
+		.result = { 0x4a, 0x5d, 0x9d, 0x5b, 0xa4, 0xce, 0x2d, 0xe1,
+			    0x72, 0x8e, 0x3b, 0xf4, 0x80, 0x35, 0x0f, 0x25,
+			    0xe0, 0x7e, 0x21, 0xc9, 0x47, 0xd1, 0x9e, 0x33,
+			    0x76, 0xf0, 0x9b, 0x3c, 0x1e, 0x16, 0x17, 0x42 },
+		.valid = true
+	},
+	{
+		.private = { 1 },
+		.public = { 0x25, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+		.result = { 0x3c, 0x77, 0x77, 0xca, 0xf9, 0x97, 0xb2, 0x64,
+			    0x41, 0x60, 0x77, 0x66, 0x5b, 0x4e, 0x22, 0x9d,
+			    0x0b, 0x95, 0x48, 0xdc, 0x0c, 0xd8, 0x19, 0x98,
+			    0xdd, 0xcd, 0xc5, 0xc8, 0x53, 0x3c, 0x79, 0x7f },
+		.valid = true
+	},
+	{
+		.private = { 1 },
+		.public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
+		.result = { 0xb3, 0x2d, 0x13, 0x62, 0xc2, 0x48, 0xd6, 0x2f,
+			    0xe6, 0x26, 0x19, 0xcf, 0xf0, 0x4d, 0xd4, 0x3d,
+			    0xb7, 0x3f, 0xfc, 0x1b, 0x63, 0x08, 0xed, 0xe3,
+			    0x0b, 0x78, 0xd8, 0x73, 0x80, 0xf1, 0xe8, 0x34 },
+		.valid = true
+	},
+	{
+		.private = { 0xa5, 0x46, 0xe3, 0x6b, 0xf0, 0x52, 0x7c, 0x9d,
+			     0x3b, 0x16, 0x15, 0x4b, 0x82, 0x46, 0x5e, 0xdd,
+			     0x62, 0x14, 0x4c, 0x0a, 0xc1, 0xfc, 0x5a, 0x18,
+			     0x50, 0x6a, 0x22, 0x44, 0xba, 0x44, 0x9a, 0xc4 },
+		.public = { 0xe6, 0xdb, 0x68, 0x67, 0x58, 0x30, 0x30, 0xdb,
+			    0x35, 0x94, 0xc1, 0xa4, 0x24, 0xb1, 0x5f, 0x7c,
+			    0x72, 0x66, 0x24, 0xec, 0x26, 0xb3, 0x35, 0x3b,
+			    0x10, 0xa9, 0x03, 0xa6, 0xd0, 0xab, 0x1c, 0x4c },
+		.result = { 0xc3, 0xda, 0x55, 0x37, 0x9d, 0xe9, 0xc6, 0x90,
+			    0x8e, 0x94, 0xea, 0x4d, 0xf2, 0x8d, 0x08, 0x4f,
+			    0x32, 0xec, 0xcf, 0x03, 0x49, 0x1c, 0x71, 0xf7,
+			    0x54, 0xb4, 0x07, 0x55, 0x77, 0xa2, 0x85, 0x52 },
+		.valid = true
+	},
+	{
+		.private = { 1, 2, 3, 4 },
+		.public = { 0 },
+		.result = { 0 },
+		.valid = false
+	},
+	{
+		.private = { 2, 4, 6, 8 },
+		.public = { 0xe0, 0xeb, 0x7a, 0x7c, 0x3b, 0x41, 0xb8, 0xae,
+			    0x16, 0x56, 0xe3, 0xfa, 0xf1, 0x9f, 0xc4, 0x6a,
+			    0xda, 0x09, 0x8d, 0xeb, 0x9c, 0x32, 0xb1, 0xfd,
+			    0x86, 0x62, 0x05, 0x16, 0x5f, 0x49, 0xb8 },
+		.result = { 0 },
+		.valid = false
+	},
+	{
+		.private = { 0xff, 0xff, 0xff, 0xff, 0x0a, 0xff, 0xff, 0xff,
+			     0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			     0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			     0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
+		.public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0x0a, 0x00, 0xfb, 0x9f },
+		.result = { 0x77, 0x52, 0xb6, 0x18, 0xc1, 0x2d, 0x48, 0xd2,
+			    0xc6, 0x93, 0x46, 0x83, 0x81, 0x7c, 0xc6, 0x57,
+			    0xf3, 0x31, 0x03, 0x19, 0x49, 0x48, 0x20, 0x05,
+			    0x42, 0x2b, 0x4e, 0xae, 0x8d, 0x1d, 0x43, 0x23 },
+		.valid = true
+	},
+	{
+		.private = { 0x8e, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+		.public = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8e, 0x06 },
+		.result = { 0x5a, 0xdf, 0xaa, 0x25, 0x86, 0x8e, 0x32, 0x3d,
+			    0xae, 0x49, 0x62, 0xc1, 0x01, 0x5c, 0xb3, 0x12,
+			    0xe1, 0xc5, 0xc7, 0x9e, 0x95, 0x3f, 0x03, 0x99,
+			    0xb0, 0xba, 0x16, 0x22, 0xf3, 0xb6, 0xf7, 0x0c },
+		.valid = true
+	},
+	/* wycheproof - normal case */
+	{
+		.private = { 0x48, 0x52, 0x83, 0x4d, 0x9d, 0x6b, 0x77, 0xda,
+			     0xde, 0xab, 0xaa, 0xf2, 0xe1, 0x1d, 0xca, 0x66,
+			     0xd1, 0x9f, 0xe7, 0x49, 0x93, 0xa7, 0xbe, 0xc3,
+			     0x6c, 0x6e, 0x16, 0xa0, 0x98, 0x3f, 0xea, 0xba },
+		.public = { 0x9c, 0x64, 0x7d, 0x9a, 0xe5, 0x89, 0xb9, 0xf5,
+			    0x8f, 0xdc, 0x3c, 0xa4, 0x94, 0x7e, 0xfb, 0xc9,
+			    0x15, 0xc4, 0xb2, 0xe0, 0x8e, 0x74, 0x4a, 0x0e,
+			    0xdf, 0x46, 0x9d, 0xac, 0x59, 0xc8, 0xf8, 0x5a },
+		.result = { 0x87, 0xb7, 0xf2, 0x12, 0xb6, 0x27, 0xf7, 0xa5,
+			    0x4c, 0xa5, 0xe0, 0xbc, 0xda, 0xdd, 0xd5, 0x38,
+			    0x9d, 0x9d, 0xe6, 0x15, 0x6c, 0xdb, 0xcf, 0x8e,
+			    0xbe, 0x14, 0xff, 0xbc, 0xfb, 0x43, 0x65, 0x51 },
+		.valid = true
+	},
+	/* wycheproof - public key on twist */
+	{
+		.private = { 0x58, 0x8c, 0x06, 0x1a, 0x50, 0x80, 0x4a, 0xc4,
+			     0x88, 0xad, 0x77, 0x4a, 0xc7, 0x16, 0xc3, 0xf5,
+			     0xba, 0x71, 0x4b, 0x27, 0x12, 0xe0, 0x48, 0x49,
+			     0x13, 0x79, 0xa5, 0x00, 0x21, 0x19, 0x98, 0xa8 },
+		.public = { 0x63, 0xaa, 0x40, 0xc6, 0xe3, 0x83, 0x46, 0xc5,
+			    0xca, 0xf2, 0x3a, 0x6d, 0xf0, 0xa5, 0xe6, 0xc8,
+			    0x08, 0x89, 0xa0, 0x86, 0x47, 0xe5, 0x51, 0xb3,
+			    0x56, 0x34, 0x49, 0xbe, 0xfc, 0xfc, 0x97, 0x33 },
+		.result = { 0xb1, 0xa7, 0x07, 0x51, 0x94, 0x95, 0xff, 0xff,
+			    0xb2, 0x98, 0xff, 0x94, 0x17, 0x16, 0xb0, 0x6d,
+			    0xfa, 0xb8, 0x7c, 0xf8, 0xd9, 0x11, 0x23, 0xfe,
+			    0x2b, 0xe9, 0xa2, 0x33, 0xdd, 0xa2, 0x22, 0x12 },
+		.valid = true
+	},
+	/* wycheproof - public key on twist */
+	{
+		.private = { 0xb0, 0x5b, 0xfd, 0x32, 0xe5, 0x53, 0x25, 0xd9,
+			     0xfd, 0x64, 0x8c, 0xb3, 0x02, 0x84, 0x80, 0x39,
+			     0x00, 0x0b, 0x39, 0x0e, 0x44, 0xd5, 0x21, 0xe5,
+			     0x8a, 0xab, 0x3b, 0x29, 0xa6, 0x96, 0x0b, 0xa8 },
+		.public = { 0x0f, 0x83, 0xc3, 0x6f, 0xde, 0xd9, 0xd3, 0x2f,
+			    0xad, 0xf4, 0xef, 0xa3, 0xae, 0x93, 0xa9, 0x0b,
+			    0xb5, 0xcf, 0xa6, 0x68, 0x93, 0xbc, 0x41, 0x2c,
+			    0x43, 0xfa, 0x72, 0x87, 0xdb, 0xb9, 0x97, 0x79 },
+		.result = { 0x67, 0xdd, 0x4a, 0x6e, 0x16, 0x55, 0x33, 0x53,
+			    0x4c, 0x0e, 0x3f, 0x17, 0x2e, 0x4a, 0xb8, 0x57,
+			    0x6b, 0xca, 0x92, 0x3a, 0x5f, 0x07, 0xb2, 0xc0,
+			    0x69, 0xb4, 0xc3, 0x10, 0xff, 0x2e, 0x93, 0x5b },
+		.valid = true
+	},
+	/* wycheproof - public key on twist */
+	{
+		.private = { 0x70, 0xe3, 0x4b, 0xcb, 0xe1, 0xf4, 0x7f, 0xbc,
+			     0x0f, 0xdd, 0xfd, 0x7c, 0x1e, 0x1a, 0xa5, 0x3d,
+			     0x57, 0xbf, 0xe0, 0xf6, 0x6d, 0x24, 0x30, 0x67,
+			     0xb4, 0x24, 0xbb, 0x62, 0x10, 0xbe, 0xd1, 0x9c },
+		.public = { 0x0b, 0x82, 0x11, 0xa2, 0xb6, 0x04, 0x90, 0x97,
+			    0xf6, 0x87, 0x1c, 0x6c, 0x05, 0x2d, 0x3c, 0x5f,
+			    0xc1, 0xba, 0x17, 0xda, 0x9e, 0x32, 0xae, 0x45,
+			    0x84, 0x03, 0xb0, 0x5b, 0xb2, 0x83, 0x09, 0x2a },
+		.result = { 0x4a, 0x06, 0x38, 0xcf, 0xaa, 0x9e, 0xf1, 0x93,
+			    0x3b, 0x47, 0xf8, 0x93, 0x92, 0x96, 0xa6, 0xb2,
+			    0x5b, 0xe5, 0x41, 0xef, 0x7f, 0x70, 0xe8, 0x44,
+			    0xc0, 0xbc, 0xc0, 0x0b, 0x13, 0x4d, 0xe6, 0x4a },
+		.valid = true
+	},
+	/* wycheproof - public key on twist */
+	{
+		.private = { 0x68, 0xc1, 0xf3, 0xa6, 0x53, 0xa4, 0xcd, 0xb1,
+			     0xd3, 0x7b, 0xba, 0x94, 0x73, 0x8f, 0x8b, 0x95,
+			     0x7a, 0x57, 0xbe, 0xb2, 0x4d, 0x64, 0x6e, 0x99,
+			     0x4d, 0xc2, 0x9a, 0x27, 0x6a, 0xad, 0x45, 0x8d },
+		.public = { 0x34, 0x3a, 0xc2, 0x0a, 0x3b, 0x9c, 0x6a, 0x27,
+			    0xb1, 0x00, 0x81, 0x76, 0x50, 0x9a, 0xd3, 0x07,
+			    0x35, 0x85, 0x6e, 0xc1, 0xc8, 0xd8, 0xfc, 0xae,
+			    0x13, 0x91, 0x2d, 0x08, 0xd1, 0x52, 0xf4, 0x6c },
+		.result = { 0x39, 0x94, 0x91, 0xfc, 0xe8, 0xdf, 0xab, 0x73,
+			    0xb4, 0xf9, 0xf6, 0x11, 0xde, 0x8e, 0xa0, 0xb2,
+			    0x7b, 0x28, 0xf8, 0x59, 0x94, 0x25, 0x0b, 0x0f,
+			    0x47, 0x5d, 0x58, 0x5d, 0x04, 0x2a, 0xc2, 0x07 },
+		.valid = true
+	},
+	/* wycheproof - public key on twist */
+	{
+		.private = { 0xd8, 0x77, 0xb2, 0x6d, 0x06, 0xdf, 0xf9, 0xd9,
+			     0xf7, 0xfd, 0x4c, 0x5b, 0x37, 0x69, 0xf8, 0xcd,
+			     0xd5, 0xb3, 0x05, 0x16, 0xa5, 0xab, 0x80, 0x6b,
+			     0xe3, 0x24, 0xff, 0x3e, 0xb6, 0x9e, 0xa0, 0xb2 },
+		.public = { 0xfa, 0x69, 0x5f, 0xc7, 0xbe, 0x8d, 0x1b, 0xe5,
+			    0xbf, 0x70, 0x48, 0x98, 0xf3, 0x88, 0xc4, 0x52,
+			    0xba, 0xfd, 0xd3, 0xb8, 0xea, 0xe8, 0x05, 0xf8,
+			    0x68, 0x1a, 0x8d, 0x15, 0xc2, 0xd4, 0xe1, 0x42 },
+		.result = { 0x2c, 0x4f, 0xe1, 0x1d, 0x49, 0x0a, 0x53, 0x86,
+			    0x17, 0x76, 0xb1, 0x3b, 0x43, 0x54, 0xab, 0xd4,
+			    0xcf, 0x5a, 0x97, 0x69, 0x9d, 0xb6, 0xe6, 0xc6,
+			    0x8c, 0x16, 0x26, 0xd0, 0x76, 0x62, 0xf7, 0x58 },
+		.valid = true
+	},
+	/* wycheproof - public key = 0 */
+	{
+		.private = { 0x20, 0x74, 0x94, 0x03, 0x8f, 0x2b, 0xb8, 0x11,
+			     0xd4, 0x78, 0x05, 0xbc, 0xdf, 0x04, 0xa2, 0xac,
+			     0x58, 0x5a, 0xda, 0x7f, 0x2f, 0x23, 0x38, 0x9b,
+			     0xfd, 0x46, 0x58, 0xf9, 0xdd, 0xd4, 0xde, 0xbc },
+		.public = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+		.result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+		.valid = false
+	},
+	/* wycheproof - public key = 1 */
+	{
+		.private = { 0x20, 0x2e, 0x89, 0x72, 0xb6, 0x1c, 0x7e, 0x61,
+			     0x93, 0x0e, 0xb9, 0x45, 0x0b, 0x50, 0x70, 0xea,
+			     0xe1, 0xc6, 0x70, 0x47, 0x56, 0x85, 0x54, 0x1f,
+			     0x04, 0x76, 0x21, 0x7e, 0x48, 0x18, 0xcf, 0xab },
+		.public = { 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+		.result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+		.valid = false
+	},
+	/* wycheproof - edge case on twist */
+	{
+		.private = { 0x38, 0xdd, 0xe9, 0xf3, 0xe7, 0xb7, 0x99, 0x04,
+			     0x5f, 0x9a, 0xc3, 0x79, 0x3d, 0x4a, 0x92, 0x77,
+			     0xda, 0xde, 0xad, 0xc4, 0x1b, 0xec, 0x02, 0x90,
+			     0xf8, 0x1f, 0x74, 0x4f, 0x73, 0x77, 0x5f, 0x84 },
+		.public = { 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+		.result = { 0x9a, 0x2c, 0xfe, 0x84, 0xff, 0x9c, 0x4a, 0x97,
+			    0x39, 0x62, 0x5c, 0xae, 0x4a, 0x3b, 0x82, 0xa9,
+			    0x06, 0x87, 0x7a, 0x44, 0x19, 0x46, 0xf8, 0xd7,
+			    0xb3, 0xd7, 0x95, 0xfe, 0x8f, 0x5d, 0x16, 0x39 },
+		.valid = true
+	},
+	/* wycheproof - edge case on twist */
+	{
+		.private = { 0x98, 0x57, 0xa9, 0x14, 0xe3, 0xc2, 0x90, 0x36,
+			     0xfd, 0x9a, 0x44, 0x2b, 0xa5, 0x26, 0xb5, 0xcd,
+			     0xcd, 0xf2, 0x82, 0x16, 0x15, 0x3e, 0x63, 0x6c,
+			     0x10, 0x67, 0x7a, 0xca, 0xb6, 0xbd, 0x6a, 0xa5 },
+		.public = { 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+		.result = { 0x4d, 0xa4, 0xe0, 0xaa, 0x07, 0x2c, 0x23, 0x2e,
+			    0xe2, 0xf0, 0xfa, 0x4e, 0x51, 0x9a, 0xe5, 0x0b,
+			    0x52, 0xc1, 0xed, 0xd0, 0x8a, 0x53, 0x4d, 0x4e,
+			    0xf3, 0x46, 0xc2, 0xe1, 0x06, 0xd2, 0x1d, 0x60 },
+		.valid = true
+	},
+	/* wycheproof - edge case on twist */
+	{
+		.private = { 0x48, 0xe2, 0x13, 0x0d, 0x72, 0x33, 0x05, 0xed,
+			     0x05, 0xe6, 0xe5, 0x89, 0x4d, 0x39, 0x8a, 0x5e,
+			     0x33, 0x36, 0x7a, 0x8c, 0x6a, 0xac, 0x8f, 0xcd,
+			     0xf0, 0xa8, 0x8e, 0x4b, 0x42, 0x82, 0x0d, 0xb7 },
+		.public = { 0xff, 0xff, 0xff, 0x03, 0x00, 0x00, 0xf8, 0xff,
+			    0xff, 0x1f, 0x00, 0x00, 0xc0, 0xff, 0xff, 0xff,
+			    0x00, 0x00, 0x00, 0xfe, 0xff, 0xff, 0x07, 0x00,
+			    0x00, 0xf0, 0xff, 0xff, 0x3f, 0x00, 0x00, 0x00 },
+		.result = { 0x9e, 0xd1, 0x0c, 0x53, 0x74, 0x7f, 0x64, 0x7f,
+			    0x82, 0xf4, 0x51, 0x25, 0xd3, 0xde, 0x15, 0xa1,
+			    0xe6, 0xb8, 0x24, 0x49, 0x6a, 0xb4, 0x04, 0x10,
+			    0xff, 0xcc, 0x3c, 0xfe, 0x95, 0x76, 0x0f, 0x3b },
+		.valid = true
+	},
+	/* wycheproof - edge case on twist */
+	{
+		.private = { 0x28, 0xf4, 0x10, 0x11, 0x69, 0x18, 0x51, 0xb3,
+			     0xa6, 0x2b, 0x64, 0x15, 0x53, 0xb3, 0x0d, 0x0d,
+			     0xfd, 0xdc, 0xb8, 0xff, 0xfc, 0xf5, 0x37, 0x00,
+			     0xa7, 0xbe, 0x2f, 0x6a, 0x87, 0x2e, 0x9f, 0xb0 },
+		.public = { 0x00, 0x00, 0x00, 0xfc, 0xff, 0xff, 0x07, 0x00,
+			    0x00, 0xe0, 0xff, 0xff, 0x3f, 0x00, 0x00, 0x00,
+			    0xff, 0xff, 0xff, 0x01, 0x00, 0x00, 0xf8, 0xff,
+			    0xff, 0x0f, 0x00, 0x00, 0xc0, 0xff, 0xff, 0x7f },
+		.result = { 0xcf, 0x72, 0xb4, 0xaa, 0x6a, 0xa1, 0xc9, 0xf8,
+			    0x94, 0xf4, 0x16, 0x5b, 0x86, 0x10, 0x9a, 0xa4,
+			    0x68, 0x51, 0x76, 0x48, 0xe1, 0xf0, 0xcc, 0x70,
+			    0xe1, 0xab, 0x08, 0x46, 0x01, 0x76, 0x50, 0x6b },
+		.valid = true
+	},
+	/* wycheproof - edge case on twist */
+	{
+		.private = { 0x18, 0xa9, 0x3b, 0x64, 0x99, 0xb9, 0xf6, 0xb3,
+			     0x22, 0x5c, 0xa0, 0x2f, 0xef, 0x41, 0x0e, 0x0a,
+			     0xde, 0xc2, 0x35, 0x32, 0x32, 0x1d, 0x2d, 0x8e,
+			     0xf1, 0xa6, 0xd6, 0x02, 0xa8, 0xc6, 0x5b, 0x83 },
+		.public = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+			    0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+			    0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+			    0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x7f },
+		.result = { 0x5d, 0x50, 0xb6, 0x28, 0x36, 0xbb, 0x69, 0x57,
+			    0x94, 0x10, 0x38, 0x6c, 0xf7, 0xbb, 0x81, 0x1c,
+			    0x14, 0xbf, 0x85, 0xb1, 0xc7, 0xb1, 0x7e, 0x59,
+			    0x24, 0xc7, 0xff, 0xea, 0x91, 0xef, 0x9e, 0x12 },
+		.valid = true
+	},
+	/* wycheproof - edge case on twist */
+	{
+		.private = { 0xc0, 0x1d, 0x13, 0x05, 0xa1, 0x33, 0x8a, 0x1f,
+			     0xca, 0xc2, 0xba, 0x7e, 0x2e, 0x03, 0x2b, 0x42,
+			     0x7e, 0x0b, 0x04, 0x90, 0x31, 0x65, 0xac, 0xa9,
+			     0x57, 0xd8, 0xd0, 0x55, 0x3d, 0x87, 0x17, 0xb0 },
+		.public = { 0xea, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f },
+		.result = { 0x19, 0x23, 0x0e, 0xb1, 0x48, 0xd5, 0xd6, 0x7c,
+			    0x3c, 0x22, 0xab, 0x1d, 0xae, 0xff, 0x80, 0xa5,
+			    0x7e, 0xae, 0x42, 0x65, 0xce, 0x28, 0x72, 0x65,
+			    0x7b, 0x2c, 0x80, 0x99, 0xfc, 0x69, 0x8e, 0x50 },
+		.valid = true
+	},
+	/* wycheproof - edge case for public key */
+	{
+		.private = { 0x38, 0x6f, 0x7f, 0x16, 0xc5, 0x07, 0x31, 0xd6,
+			     0x4f, 0x82, 0xe6, 0xa1, 0x70, 0xb1, 0x42, 0xa4,
+			     0xe3, 0x4f, 0x31, 0xfd, 0x77, 0x68, 0xfc, 0xb8,
+			     0x90, 0x29, 0x25, 0xe7, 0xd1, 0xe2, 0x1a, 0xbe },
+		.public = { 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+		.result = { 0x0f, 0xca, 0xb5, 0xd8, 0x42, 0xa0, 0x78, 0xd7,
+			    0xa7, 0x1f, 0xc5, 0x9b, 0x57, 0xbf, 0xb4, 0xca,
+			    0x0b, 0xe6, 0x87, 0x3b, 0x49, 0xdc, 0xdb, 0x9f,
+			    0x44, 0xe1, 0x4a, 0xe8, 0xfb, 0xdf, 0xa5, 0x42 },
+		.valid = true
+	},
+	/* wycheproof - edge case for public key */
+	{
+		.private = { 0xe0, 0x23, 0xa2, 0x89, 0xbd, 0x5e, 0x90, 0xfa,
+			     0x28, 0x04, 0xdd, 0xc0, 0x19, 0xa0, 0x5e, 0xf3,
+			     0xe7, 0x9d, 0x43, 0x4b, 0xb6, 0xea, 0x2f, 0x52,
+			     0x2e, 0xcb, 0x64, 0x3a, 0x75, 0x29, 0x6e, 0x95 },
+		.public = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+			    0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+			    0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+			    0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 },
+		.result = { 0x54, 0xce, 0x8f, 0x22, 0x75, 0xc0, 0x77, 0xe3,
+			    0xb1, 0x30, 0x6a, 0x39, 0x39, 0xc5, 0xe0, 0x3e,
+			    0xef, 0x6b, 0xbb, 0x88, 0x06, 0x05, 0x44, 0x75,
+			    0x8d, 0x9f, 0xef, 0x59, 0xb0, 0xbc, 0x3e, 0x4f },
+		.valid = true
+	},
+	/* wycheproof - edge case for public key */
+	{
+		.private = { 0x68, 0xf0, 0x10, 0xd6, 0x2e, 0xe8, 0xd9, 0x26,
+			     0x05, 0x3a, 0x36, 0x1c, 0x3a, 0x75, 0xc6, 0xea,
+			     0x4e, 0xbd, 0xc8, 0x60, 0x6a, 0xb2, 0x85, 0x00,
+			     0x3a, 0x6f, 0x8f, 0x40, 0x76, 0xb0, 0x1e, 0x83 },
+		.public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x03 },
+		.result = { 0xf1, 0x36, 0x77, 0x5c, 0x5b, 0xeb, 0x0a, 0xf8,
+			    0x11, 0x0a, 0xf1, 0x0b, 0x20, 0x37, 0x23, 0x32,
+			    0x04, 0x3c, 0xab, 0x75, 0x24, 0x19, 0x67, 0x87,
+			    0x75, 0xa2, 0x23, 0xdf, 0x57, 0xc9, 0xd3, 0x0d },
+		.valid = true
+	},
+	/* wycheproof - edge case for public key */
+	{
+		.private = { 0x58, 0xeb, 0xcb, 0x35, 0xb0, 0xf8, 0x84, 0x5c,
+			     0xaf, 0x1e, 0xc6, 0x30, 0xf9, 0x65, 0x76, 0xb6,
+			     0x2c, 0x4b, 0x7b, 0x6c, 0x36, 0xb2, 0x9d, 0xeb,
+			     0x2c, 0xb0, 0x08, 0x46, 0x51, 0x75, 0x5c, 0x96 },
+		.public = { 0xff, 0xff, 0xff, 0xfb, 0xff, 0xff, 0xfb, 0xff,
+			    0xff, 0xdf, 0xff, 0xff, 0xdf, 0xff, 0xff, 0xff,
+			    0xfe, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xf7, 0xff,
+			    0xff, 0xf7, 0xff, 0xff, 0xbf, 0xff, 0xff, 0x3f },
+		.result = { 0xbf, 0x9a, 0xff, 0xd0, 0x6b, 0x84, 0x40, 0x85,
+			    0x58, 0x64, 0x60, 0x96, 0x2e, 0xf2, 0x14, 0x6f,
+			    0xf3, 0xd4, 0x53, 0x3d, 0x94, 0x44, 0xaa, 0xb0,
+			    0x06, 0xeb, 0x88, 0xcc, 0x30, 0x54, 0x40, 0x7d },
+		.valid = true
+	},
+	/* wycheproof - edge case for public key */
+	{
+		.private = { 0x18, 0x8c, 0x4b, 0xc5, 0xb9, 0xc4, 0x4b, 0x38,
+			     0xbb, 0x65, 0x8b, 0x9b, 0x2a, 0xe8, 0x2d, 0x5b,
+			     0x01, 0x01, 0x5e, 0x09, 0x31, 0x84, 0xb1, 0x7c,
+			     0xb7, 0x86, 0x35, 0x03, 0xa7, 0x83, 0xe1, 0xbb },
+		.public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f },
+		.result = { 0xd4, 0x80, 0xde, 0x04, 0xf6, 0x99, 0xcb, 0x3b,
+			    0xe0, 0x68, 0x4a, 0x9c, 0xc2, 0xe3, 0x12, 0x81,
+			    0xea, 0x0b, 0xc5, 0xa9, 0xdc, 0xc1, 0x57, 0xd3,
+			    0xd2, 0x01, 0x58, 0xd4, 0x6c, 0xa5, 0x24, 0x6d },
+		.valid = true
+	},
+	/* wycheproof - edge case for public key */
+	{
+		.private = { 0xe0, 0x6c, 0x11, 0xbb, 0x2e, 0x13, 0xce, 0x3d,
+			     0xc7, 0x67, 0x3f, 0x67, 0xf5, 0x48, 0x22, 0x42,
+			     0x90, 0x94, 0x23, 0xa9, 0xae, 0x95, 0xee, 0x98,
+			     0x6a, 0x98, 0x8d, 0x98, 0xfa, 0xee, 0x23, 0xa2 },
+		.public = { 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0x7f,
+			    0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0x7f,
+			    0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0x7f,
+			    0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0x7f },
+		.result = { 0x4c, 0x44, 0x01, 0xcc, 0xe6, 0xb5, 0x1e, 0x4c,
+			    0xb1, 0x8f, 0x27, 0x90, 0x24, 0x6c, 0x9b, 0xf9,
+			    0x14, 0xdb, 0x66, 0x77, 0x50, 0xa1, 0xcb, 0x89,
+			    0x06, 0x90, 0x92, 0xaf, 0x07, 0x29, 0x22, 0x76 },
+		.valid = true
+	},
+	/* wycheproof - edge case for public key */
+	{
+		.private = { 0xc0, 0x65, 0x8c, 0x46, 0xdd, 0xe1, 0x81, 0x29,
+			     0x29, 0x38, 0x77, 0x53, 0x5b, 0x11, 0x62, 0xb6,
+			     0xf9, 0xf5, 0x41, 0x4a, 0x23, 0xcf, 0x4d, 0x2c,
+			     0xbc, 0x14, 0x0a, 0x4d, 0x99, 0xda, 0x2b, 0x8f },
+		.public = { 0xeb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f },
+		.result = { 0x57, 0x8b, 0xa8, 0xcc, 0x2d, 0xbd, 0xc5, 0x75,
+			    0xaf, 0xcf, 0x9d, 0xf2, 0xb3, 0xee, 0x61, 0x89,
+			    0xf5, 0x33, 0x7d, 0x68, 0x54, 0xc7, 0x9b, 0x4c,
+			    0xe1, 0x65, 0xea, 0x12, 0x29, 0x3b, 0x3a, 0x0f },
+		.valid = true
+	},
+	/* wycheproof - public key with low order */
+	{
+		.private = { 0x10, 0x25, 0x5c, 0x92, 0x30, 0xa9, 0x7a, 0x30,
+			     0xa4, 0x58, 0xca, 0x28, 0x4a, 0x62, 0x96, 0x69,
+			     0x29, 0x3a, 0x31, 0x89, 0x0c, 0xda, 0x9d, 0x14,
+			     0x7f, 0xeb, 0xc7, 0xd1, 0xe2, 0x2d, 0x6b, 0xb1 },
+		.public = { 0xe0, 0xeb, 0x7a, 0x7c, 0x3b, 0x41, 0xb8, 0xae,
+			    0x16, 0x56, 0xe3, 0xfa, 0xf1, 0x9f, 0xc4, 0x6a,
+			    0xda, 0x09, 0x8d, 0xeb, 0x9c, 0x32, 0xb1, 0xfd,
+			    0x86, 0x62, 0x05, 0x16, 0x5f, 0x49, 0xb8, 0x00 },
+		.result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+		.valid = false
+	},
+	/* wycheproof - public key with low order */
+	{
+		.private = { 0x78, 0xf1, 0xe8, 0xed, 0xf1, 0x44, 0x81, 0xb3,
+			     0x89, 0x44, 0x8d, 0xac, 0x8f, 0x59, 0xc7, 0x0b,
+			     0x03, 0x8e, 0x7c, 0xf9, 0x2e, 0xf2, 0xc7, 0xef,
+			     0xf5, 0x7a, 0x72, 0x46, 0x6e, 0x11, 0x52, 0x96 },
+		.public = { 0x5f, 0x9c, 0x95, 0xbc, 0xa3, 0x50, 0x8c, 0x24,
+			    0xb1, 0xd0, 0xb1, 0x55, 0x9c, 0x83, 0xef, 0x5b,
+			    0x04, 0x44, 0x5c, 0xc4, 0x58, 0x1c, 0x8e, 0x86,
+			    0xd8, 0x22, 0x4e, 0xdd, 0xd0, 0x9f, 0x11, 0x57 },
+		.result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+		.valid = false
+	},
+	/* wycheproof - public key with low order */
+	{
+		.private = { 0xa0, 0xa0, 0x5a, 0x3e, 0x8f, 0x9f, 0x44, 0x20,
+			     0x4d, 0x5f, 0x80, 0x59, 0xa9, 0x4a, 0xc7, 0xdf,
+			     0xc3, 0x9a, 0x49, 0xac, 0x01, 0x6d, 0xd7, 0x43,
+			     0xdb, 0xfa, 0x43, 0xc5, 0xd6, 0x71, 0xfd, 0x88 },
+		.public = { 0xec, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f },
+		.result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+		.valid = false
+	},
+	/* wycheproof - public key with low order */
+	{
+		.private = { 0xd0, 0xdb, 0xb3, 0xed, 0x19, 0x06, 0x66, 0x3f,
+			     0x15, 0x42, 0x0a, 0xf3, 0x1f, 0x4e, 0xaf, 0x65,
+			     0x09, 0xd9, 0xa9, 0x94, 0x97, 0x23, 0x50, 0x06,
+			     0x05, 0xad, 0x7c, 0x1c, 0x6e, 0x74, 0x50, 0xa9 },
+		.public = { 0xed, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f },
+		.result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+		.valid = false
+	},
+	/* wycheproof - public key with low order */
+	{
+		.private = { 0xc0, 0xb1, 0xd0, 0xeb, 0x22, 0xb2, 0x44, 0xfe,
+			     0x32, 0x91, 0x14, 0x00, 0x72, 0xcd, 0xd9, 0xd9,
+			     0x89, 0xb5, 0xf0, 0xec, 0xd9, 0x6c, 0x10, 0x0f,
+			     0xeb, 0x5b, 0xca, 0x24, 0x1c, 0x1d, 0x9f, 0x8f },
+		.public = { 0xee, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f },
+		.result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+		.valid = false
+	},
+	/* wycheproof - public key with low order */
+	{
+		.private = { 0x48, 0x0b, 0xf4, 0x5f, 0x59, 0x49, 0x42, 0xa8,
+			     0xbc, 0x0f, 0x33, 0x53, 0xc6, 0xe8, 0xb8, 0x85,
+			     0x3d, 0x77, 0xf3, 0x51, 0xf1, 0xc2, 0xca, 0x6c,
+			     0x2d, 0x1a, 0xbf, 0x8a, 0x00, 0xb4, 0x22, 0x9c },
+		.public = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 },
+		.result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+		.valid = false
+	},
+	/* wycheproof - public key with low order */
+	{
+		.private = { 0x30, 0xf9, 0x93, 0xfc, 0xf8, 0x51, 0x4f, 0xc8,
+			     0x9b, 0xd8, 0xdb, 0x14, 0xcd, 0x43, 0xba, 0x0d,
+			     0x4b, 0x25, 0x30, 0xe7, 0x3c, 0x42, 0x76, 0xa0,
+			     0x5e, 0x1b, 0x14, 0x5d, 0x42, 0x0c, 0xed, 0xb4 },
+		.public = { 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 },
+		.result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+		.valid = false
+	},
+	/* wycheproof - public key with low order */
+	{
+		.private = { 0xc0, 0x49, 0x74, 0xb7, 0x58, 0x38, 0x0e, 0x2a,
+			     0x5b, 0x5d, 0xf6, 0xeb, 0x09, 0xbb, 0x2f, 0x6b,
+			     0x34, 0x34, 0xf9, 0x82, 0x72, 0x2a, 0x8e, 0x67,
+			     0x6d, 0x3d, 0xa2, 0x51, 0xd1, 0xb3, 0xde, 0x83 },
+		.public = { 0xe0, 0xeb, 0x7a, 0x7c, 0x3b, 0x41, 0xb8, 0xae,
+			    0x16, 0x56, 0xe3, 0xfa, 0xf1, 0x9f, 0xc4, 0x6a,
+			    0xda, 0x09, 0x8d, 0xeb, 0x9c, 0x32, 0xb1, 0xfd,
+			    0x86, 0x62, 0x05, 0x16, 0x5f, 0x49, 0xb8, 0x80 },
+		.result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+		.valid = false
+	},
+	/* wycheproof - public key with low order */
+	{
+		.private = { 0x50, 0x2a, 0x31, 0x37, 0x3d, 0xb3, 0x24, 0x46,
+			     0x84, 0x2f, 0xe5, 0xad, 0xd3, 0xe0, 0x24, 0x02,
+			     0x2e, 0xa5, 0x4f, 0x27, 0x41, 0x82, 0xaf, 0xc3,
+			     0xd9, 0xf1, 0xbb, 0x3d, 0x39, 0x53, 0x4e, 0xb5 },
+		.public = { 0x5f, 0x9c, 0x95, 0xbc, 0xa3, 0x50, 0x8c, 0x24,
+			    0xb1, 0xd0, 0xb1, 0x55, 0x9c, 0x83, 0xef, 0x5b,
+			    0x04, 0x44, 0x5c, 0xc4, 0x58, 0x1c, 0x8e, 0x86,
+			    0xd8, 0x22, 0x4e, 0xdd, 0xd0, 0x9f, 0x11, 0xd7 },
+		.result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+		.valid = false
+	},
+	/* wycheproof - public key with low order */
+	{
+		.private = { 0x90, 0xfa, 0x64, 0x17, 0xb0, 0xe3, 0x70, 0x30,
+			     0xfd, 0x6e, 0x43, 0xef, 0xf2, 0xab, 0xae, 0xf1,
+			     0x4c, 0x67, 0x93, 0x11, 0x7a, 0x03, 0x9c, 0xf6,
+			     0x21, 0x31, 0x8b, 0xa9, 0x0f, 0x4e, 0x98, 0xbe },
+		.public = { 0xec, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
+		.result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+		.valid = false
+	},
+	/* wycheproof - public key with low order */
+	{
+		.private = { 0x78, 0xad, 0x3f, 0x26, 0x02, 0x7f, 0x1c, 0x9f,
+			     0xdd, 0x97, 0x5a, 0x16, 0x13, 0xb9, 0x47, 0x77,
+			     0x9b, 0xad, 0x2c, 0xf2, 0xb7, 0x41, 0xad, 0xe0,
+			     0x18, 0x40, 0x88, 0x5a, 0x30, 0xbb, 0x97, 0x9c },
+		.public = { 0xed, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
+		.result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+		.valid = false
+	},
+	/* wycheproof - public key with low order */
+	{
+		.private = { 0x98, 0xe2, 0x3d, 0xe7, 0xb1, 0xe0, 0x92, 0x6e,
+			     0xd9, 0xc8, 0x7e, 0x7b, 0x14, 0xba, 0xf5, 0x5f,
+			     0x49, 0x7a, 0x1d, 0x70, 0x96, 0xf9, 0x39, 0x77,
+			     0x68, 0x0e, 0x44, 0xdc, 0x1c, 0x7b, 0x7b, 0x8b },
+		.public = { 0xee, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
+		.result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+		.valid = false
+	},
+	/* wycheproof - public key >= p */
+	{
+		.private = { 0xf0, 0x1e, 0x48, 0xda, 0xfa, 0xc9, 0xd7, 0xbc,
+			     0xf5, 0x89, 0xcb, 0xc3, 0x82, 0xc8, 0x78, 0xd1,
+			     0x8b, 0xda, 0x35, 0x50, 0x58, 0x9f, 0xfb, 0x5d,
+			     0x50, 0xb5, 0x23, 0xbe, 0xbe, 0x32, 0x9d, 0xae },
+		.public = { 0xef, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f },
+		.result = { 0xbd, 0x36, 0xa0, 0x79, 0x0e, 0xb8, 0x83, 0x09,
+			    0x8c, 0x98, 0x8b, 0x21, 0x78, 0x67, 0x73, 0xde,
+			    0x0b, 0x3a, 0x4d, 0xf1, 0x62, 0x28, 0x2c, 0xf1,
+			    0x10, 0xde, 0x18, 0xdd, 0x48, 0x4c, 0xe7, 0x4b },
+		.valid = true
+	},
+	/* wycheproof - public key >= p */
+	{
+		.private = { 0x28, 0x87, 0x96, 0xbc, 0x5a, 0xff, 0x4b, 0x81,
+			     0xa3, 0x75, 0x01, 0x75, 0x7b, 0xc0, 0x75, 0x3a,
+			     0x3c, 0x21, 0x96, 0x47, 0x90, 0xd3, 0x86, 0x99,
+			     0x30, 0x8d, 0xeb, 0xc1, 0x7a, 0x6e, 0xaf, 0x8d },
+		.public = { 0xf0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f },
+		.result = { 0xb4, 0xe0, 0xdd, 0x76, 0xda, 0x7b, 0x07, 0x17,
+			    0x28, 0xb6, 0x1f, 0x85, 0x67, 0x71, 0xaa, 0x35,
+			    0x6e, 0x57, 0xed, 0xa7, 0x8a, 0x5b, 0x16, 0x55,
+			    0xcc, 0x38, 0x20, 0xfb, 0x5f, 0x85, 0x4c, 0x5c },
+		.valid = true
+	},
+	/* wycheproof - public key >= p */
+	{
+		.private = { 0x98, 0xdf, 0x84, 0x5f, 0x66, 0x51, 0xbf, 0x11,
+			     0x38, 0x22, 0x1f, 0x11, 0x90, 0x41, 0xf7, 0x2b,
+			     0x6d, 0xbc, 0x3c, 0x4a, 0xce, 0x71, 0x43, 0xd9,
+			     0x9f, 0xd5, 0x5a, 0xd8, 0x67, 0x48, 0x0d, 0xa8 },
+		.public = { 0xf1, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f },
+		.result = { 0x6f, 0xdf, 0x6c, 0x37, 0x61, 0x1d, 0xbd, 0x53,
+			    0x04, 0xdc, 0x0f, 0x2e, 0xb7, 0xc9, 0x51, 0x7e,
+			    0xb3, 0xc5, 0x0e, 0x12, 0xfd, 0x05, 0x0a, 0xc6,
+			    0xde, 0xc2, 0x70, 0x71, 0xd4, 0xbf, 0xc0, 0x34 },
+		.valid = true
+	},
+	/* wycheproof - public key >= p */
+	{
+		.private = { 0xf0, 0x94, 0x98, 0xe4, 0x6f, 0x02, 0xf8, 0x78,
+			     0x82, 0x9e, 0x78, 0xb8, 0x03, 0xd3, 0x16, 0xa2,
+			     0xed, 0x69, 0x5d, 0x04, 0x98, 0xa0, 0x8a, 0xbd,
+			     0xf8, 0x27, 0x69, 0x30, 0xe2, 0x4e, 0xdc, 0xb0 },
+		.public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f },
+		.result = { 0x4c, 0x8f, 0xc4, 0xb1, 0xc6, 0xab, 0x88, 0xfb,
+			    0x21, 0xf1, 0x8f, 0x6d, 0x4c, 0x81, 0x02, 0x40,
+			    0xd4, 0xe9, 0x46, 0x51, 0xba, 0x44, 0xf7, 0xa2,
+			    0xc8, 0x63, 0xce, 0xc7, 0xdc, 0x56, 0x60, 0x2d },
+		.valid = true
+	},
+	/* wycheproof - public key >= p */
+	{
+		.private = { 0x18, 0x13, 0xc1, 0x0a, 0x5c, 0x7f, 0x21, 0xf9,
+			     0x6e, 0x17, 0xf2, 0x88, 0xc0, 0xcc, 0x37, 0x60,
+			     0x7c, 0x04, 0xc5, 0xf5, 0xae, 0xa2, 0xdb, 0x13,
+			     0x4f, 0x9e, 0x2f, 0xfc, 0x66, 0xbd, 0x9d, 0xb8 },
+		.public = { 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 },
+		.result = { 0x1c, 0xd0, 0xb2, 0x82, 0x67, 0xdc, 0x54, 0x1c,
+			    0x64, 0x2d, 0x6d, 0x7d, 0xca, 0x44, 0xa8, 0xb3,
+			    0x8a, 0x63, 0x73, 0x6e, 0xef, 0x5c, 0x4e, 0x65,
+			    0x01, 0xff, 0xbb, 0xb1, 0x78, 0x0c, 0x03, 0x3c },
+		.valid = true
+	},
+	/* wycheproof - public key >= p */
+	{
+		.private = { 0x78, 0x57, 0xfb, 0x80, 0x86, 0x53, 0x64, 0x5a,
+			     0x0b, 0xeb, 0x13, 0x8a, 0x64, 0xf5, 0xf4, 0xd7,
+			     0x33, 0xa4, 0x5e, 0xa8, 0x4c, 0x3c, 0xda, 0x11,
+			     0xa9, 0xc0, 0x6f, 0x7e, 0x71, 0x39, 0x14, 0x9e },
+		.public = { 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 },
+		.result = { 0x87, 0x55, 0xbe, 0x01, 0xc6, 0x0a, 0x7e, 0x82,
+			    0x5c, 0xff, 0x3e, 0x0e, 0x78, 0xcb, 0x3a, 0xa4,
+			    0x33, 0x38, 0x61, 0x51, 0x6a, 0xa5, 0x9b, 0x1c,
+			    0x51, 0xa8, 0xb2, 0xa5, 0x43, 0xdf, 0xa8, 0x22 },
+		.valid = true
+	},
+	/* wycheproof - public key >= p */
+	{
+		.private = { 0xe0, 0x3a, 0xa8, 0x42, 0xe2, 0xab, 0xc5, 0x6e,
+			     0x81, 0xe8, 0x7b, 0x8b, 0x9f, 0x41, 0x7b, 0x2a,
+			     0x1e, 0x59, 0x13, 0xc7, 0x23, 0xee, 0xd2, 0x8d,
+			     0x75, 0x2f, 0x8d, 0x47, 0xa5, 0x9f, 0x49, 0x8f },
+		.public = { 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 },
+		.result = { 0x54, 0xc9, 0xa1, 0xed, 0x95, 0xe5, 0x46, 0xd2,
+			    0x78, 0x22, 0xa3, 0x60, 0x93, 0x1d, 0xda, 0x60,
+			    0xa1, 0xdf, 0x04, 0x9d, 0xa6, 0xf9, 0x04, 0x25,
+			    0x3c, 0x06, 0x12, 0xbb, 0xdc, 0x08, 0x74, 0x76 },
+		.valid = true
+	},
+	/* wycheproof - public key >= p */
+	{
+		.private = { 0xf8, 0xf7, 0x07, 0xb7, 0x99, 0x9b, 0x18, 0xcb,
+			     0x0d, 0x6b, 0x96, 0x12, 0x4f, 0x20, 0x45, 0x97,
+			     0x2c, 0xa2, 0x74, 0xbf, 0xc1, 0x54, 0xad, 0x0c,
+			     0x87, 0x03, 0x8c, 0x24, 0xc6, 0xd0, 0xd4, 0xb2 },
+		.public = { 0xda, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
+		.result = { 0xcc, 0x1f, 0x40, 0xd7, 0x43, 0xcd, 0xc2, 0x23,
+			    0x0e, 0x10, 0x43, 0xda, 0xba, 0x8b, 0x75, 0xe8,
+			    0x10, 0xf1, 0xfb, 0xab, 0x7f, 0x25, 0x52, 0x69,
+			    0xbd, 0x9e, 0xbb, 0x29, 0xe6, 0xbf, 0x49, 0x4f },
+		.valid = true
+	},
+	/* wycheproof - public key >= p */
+	{
+		.private = { 0xa0, 0x34, 0xf6, 0x84, 0xfa, 0x63, 0x1e, 0x1a,
+			     0x34, 0x81, 0x18, 0xc1, 0xce, 0x4c, 0x98, 0x23,
+			     0x1f, 0x2d, 0x9e, 0xec, 0x9b, 0xa5, 0x36, 0x5b,
+			     0x4a, 0x05, 0xd6, 0x9a, 0x78, 0x5b, 0x07, 0x96 },
+		.public = { 0xdb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
+		.result = { 0x54, 0x99, 0x8e, 0xe4, 0x3a, 0x5b, 0x00, 0x7b,
+			    0xf4, 0x99, 0xf0, 0x78, 0xe7, 0x36, 0x52, 0x44,
+			    0x00, 0xa8, 0xb5, 0xc7, 0xe9, 0xb9, 0xb4, 0x37,
+			    0x71, 0x74, 0x8c, 0x7c, 0xdf, 0x88, 0x04, 0x12 },
+		.valid = true
+	},
+	/* wycheproof - public key >= p */
+	{
+		.private = { 0x30, 0xb6, 0xc6, 0xa0, 0xf2, 0xff, 0xa6, 0x80,
+			     0x76, 0x8f, 0x99, 0x2b, 0xa8, 0x9e, 0x15, 0x2d,
+			     0x5b, 0xc9, 0x89, 0x3d, 0x38, 0xc9, 0x11, 0x9b,
+			     0xe4, 0xf7, 0x67, 0xbf, 0xab, 0x6e, 0x0c, 0xa5 },
+		.public = { 0xdc, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
+		.result = { 0xea, 0xd9, 0xb3, 0x8e, 0xfd, 0xd7, 0x23, 0x63,
+			    0x79, 0x34, 0xe5, 0x5a, 0xb7, 0x17, 0xa7, 0xae,
+			    0x09, 0xeb, 0x86, 0xa2, 0x1d, 0xc3, 0x6a, 0x3f,
+			    0xee, 0xb8, 0x8b, 0x75, 0x9e, 0x39, 0x1e, 0x09 },
+		.valid = true
+	},
+	/* wycheproof - public key >= p */
+	{
+		.private = { 0x90, 0x1b, 0x9d, 0xcf, 0x88, 0x1e, 0x01, 0xe0,
+			     0x27, 0x57, 0x50, 0x35, 0xd4, 0x0b, 0x43, 0xbd,
+			     0xc1, 0xc5, 0x24, 0x2e, 0x03, 0x08, 0x47, 0x49,
+			     0x5b, 0x0c, 0x72, 0x86, 0x46, 0x9b, 0x65, 0x91 },
+		.public = { 0xea, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
+		.result = { 0x60, 0x2f, 0xf4, 0x07, 0x89, 0xb5, 0x4b, 0x41,
+			    0x80, 0x59, 0x15, 0xfe, 0x2a, 0x62, 0x21, 0xf0,
+			    0x7a, 0x50, 0xff, 0xc2, 0xc3, 0xfc, 0x94, 0xcf,
+			    0x61, 0xf1, 0x3d, 0x79, 0x04, 0xe8, 0x8e, 0x0e },
+		.valid = true
+	},
+	/* wycheproof - public key >= p */
+	{
+		.private = { 0x80, 0x46, 0x67, 0x7c, 0x28, 0xfd, 0x82, 0xc9,
+			     0xa1, 0xbd, 0xb7, 0x1a, 0x1a, 0x1a, 0x34, 0xfa,
+			     0xba, 0x12, 0x25, 0xe2, 0x50, 0x7f, 0xe3, 0xf5,
+			     0x4d, 0x10, 0xbd, 0x5b, 0x0d, 0x86, 0x5f, 0x8e },
+		.public = { 0xeb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
+		.result = { 0xe0, 0x0a, 0xe8, 0xb1, 0x43, 0x47, 0x12, 0x47,
+			    0xba, 0x24, 0xf1, 0x2c, 0x88, 0x55, 0x36, 0xc3,
+			    0xcb, 0x98, 0x1b, 0x58, 0xe1, 0xe5, 0x6b, 0x2b,
+			    0xaf, 0x35, 0xc1, 0x2a, 0xe1, 0xf7, 0x9c, 0x26 },
+		.valid = true
+	},
+	/* wycheproof - public key >= p */
+	{
+		.private = { 0x60, 0x2f, 0x7e, 0x2f, 0x68, 0xa8, 0x46, 0xb8,
+			     0x2c, 0xc2, 0x69, 0xb1, 0xd4, 0x8e, 0x93, 0x98,
+			     0x86, 0xae, 0x54, 0xfd, 0x63, 0x6c, 0x1f, 0xe0,
+			     0x74, 0xd7, 0x10, 0x12, 0x7d, 0x47, 0x24, 0x91 },
+		.public = { 0xef, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
+		.result = { 0x98, 0xcb, 0x9b, 0x50, 0xdd, 0x3f, 0xc2, 0xb0,
+			    0xd4, 0xf2, 0xd2, 0xbf, 0x7c, 0x5c, 0xfd, 0xd1,
+			    0x0c, 0x8f, 0xcd, 0x31, 0xfc, 0x40, 0xaf, 0x1a,
+			    0xd4, 0x4f, 0x47, 0xc1, 0x31, 0x37, 0x63, 0x62 },
+		.valid = true
+	},
+	/* wycheproof - public key >= p */
+	{
+		.private = { 0x60, 0x88, 0x7b, 0x3d, 0xc7, 0x24, 0x43, 0x02,
+			     0x6e, 0xbe, 0xdb, 0xbb, 0xb7, 0x06, 0x65, 0xf4,
+			     0x2b, 0x87, 0xad, 0xd1, 0x44, 0x0e, 0x77, 0x68,
+			     0xfb, 0xd7, 0xe8, 0xe2, 0xce, 0x5f, 0x63, 0x9d },
+		.public = { 0xf0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
+		.result = { 0x38, 0xd6, 0x30, 0x4c, 0x4a, 0x7e, 0x6d, 0x9f,
+			    0x79, 0x59, 0x33, 0x4f, 0xb5, 0x24, 0x5b, 0xd2,
+			    0xc7, 0x54, 0x52, 0x5d, 0x4c, 0x91, 0xdb, 0x95,
+			    0x02, 0x06, 0x92, 0x62, 0x34, 0xc1, 0xf6, 0x33 },
+		.valid = true
+	},
+	/* wycheproof - public key >= p */
+	{
+		.private = { 0x78, 0xd3, 0x1d, 0xfa, 0x85, 0x44, 0x97, 0xd7,
+			     0x2d, 0x8d, 0xef, 0x8a, 0x1b, 0x7f, 0xb0, 0x06,
+			     0xce, 0xc2, 0xd8, 0xc4, 0x92, 0x46, 0x47, 0xc9,
+			     0x38, 0x14, 0xae, 0x56, 0xfa, 0xed, 0xa4, 0x95 },
+		.public = { 0xf1, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
+		.result = { 0x78, 0x6c, 0xd5, 0x49, 0x96, 0xf0, 0x14, 0xa5,
+			    0xa0, 0x31, 0xec, 0x14, 0xdb, 0x81, 0x2e, 0xd0,
+			    0x83, 0x55, 0x06, 0x1f, 0xdb, 0x5d, 0xe6, 0x80,
+			    0xa8, 0x00, 0xac, 0x52, 0x1f, 0x31, 0x8e, 0x23 },
+		.valid = true
+	},
+	/* wycheproof - public key >= p */
+	{
+		.private = { 0xc0, 0x4c, 0x5b, 0xae, 0xfa, 0x83, 0x02, 0xdd,
+			     0xde, 0xd6, 0xa4, 0xbb, 0x95, 0x77, 0x61, 0xb4,
+			     0xeb, 0x97, 0xae, 0xfa, 0x4f, 0xc3, 0xb8, 0x04,
+			     0x30, 0x85, 0xf9, 0x6a, 0x56, 0x59, 0xb3, 0xa5 },
+		.public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
+		.result = { 0x29, 0xae, 0x8b, 0xc7, 0x3e, 0x9b, 0x10, 0xa0,
+			    0x8b, 0x4f, 0x68, 0x1c, 0x43, 0xc3, 0xe0, 0xac,
+			    0x1a, 0x17, 0x1d, 0x31, 0xb3, 0x8f, 0x1a, 0x48,
+			    0xef, 0xba, 0x29, 0xae, 0x63, 0x9e, 0xa1, 0x34 },
+		.valid = true
+	},
+	/* wycheproof - RFC 7748 */
+	{
+		.private = { 0xa0, 0x46, 0xe3, 0x6b, 0xf0, 0x52, 0x7c, 0x9d,
+			     0x3b, 0x16, 0x15, 0x4b, 0x82, 0x46, 0x5e, 0xdd,
+			     0x62, 0x14, 0x4c, 0x0a, 0xc1, 0xfc, 0x5a, 0x18,
+			     0x50, 0x6a, 0x22, 0x44, 0xba, 0x44, 0x9a, 0x44 },
+		.public = { 0xe6, 0xdb, 0x68, 0x67, 0x58, 0x30, 0x30, 0xdb,
+			    0x35, 0x94, 0xc1, 0xa4, 0x24, 0xb1, 0x5f, 0x7c,
+			    0x72, 0x66, 0x24, 0xec, 0x26, 0xb3, 0x35, 0x3b,
+			    0x10, 0xa9, 0x03, 0xa6, 0xd0, 0xab, 0x1c, 0x4c },
+		.result = { 0xc3, 0xda, 0x55, 0x37, 0x9d, 0xe9, 0xc6, 0x90,
+			    0x8e, 0x94, 0xea, 0x4d, 0xf2, 0x8d, 0x08, 0x4f,
+			    0x32, 0xec, 0xcf, 0x03, 0x49, 0x1c, 0x71, 0xf7,
+			    0x54, 0xb4, 0x07, 0x55, 0x77, 0xa2, 0x85, 0x52 },
+		.valid = true
+	},
+	/* wycheproof - RFC 7748 */
+	{
+		.private = { 0x48, 0x66, 0xe9, 0xd4, 0xd1, 0xb4, 0x67, 0x3c,
+			     0x5a, 0xd2, 0x26, 0x91, 0x95, 0x7d, 0x6a, 0xf5,
+			     0xc1, 0x1b, 0x64, 0x21, 0xe0, 0xea, 0x01, 0xd4,
+			     0x2c, 0xa4, 0x16, 0x9e, 0x79, 0x18, 0xba, 0x4d },
+		.public = { 0xe5, 0x21, 0x0f, 0x12, 0x78, 0x68, 0x11, 0xd3,
+			    0xf4, 0xb7, 0x95, 0x9d, 0x05, 0x38, 0xae, 0x2c,
+			    0x31, 0xdb, 0xe7, 0x10, 0x6f, 0xc0, 0x3c, 0x3e,
+			    0xfc, 0x4c, 0xd5, 0x49, 0xc7, 0x15, 0xa4, 0x13 },
+		.result = { 0x95, 0xcb, 0xde, 0x94, 0x76, 0xe8, 0x90, 0x7d,
+			    0x7a, 0xad, 0xe4, 0x5c, 0xb4, 0xb8, 0x73, 0xf8,
+			    0x8b, 0x59, 0x5a, 0x68, 0x79, 0x9f, 0xa1, 0x52,
+			    0xe6, 0xf8, 0xf7, 0x64, 0x7a, 0xac, 0x79, 0x57 },
+		.valid = true
+	},
+	/* wycheproof - edge case for shared secret */
+	{
+		.private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4,
+			     0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3,
+			     0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc,
+			     0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 },
+		.public = { 0x0a, 0xb4, 0xe7, 0x63, 0x80, 0xd8, 0x4d, 0xde,
+			    0x4f, 0x68, 0x33, 0xc5, 0x8f, 0x2a, 0x9f, 0xb8,
+			    0xf8, 0x3b, 0xb0, 0x16, 0x9b, 0x17, 0x2b, 0xe4,
+			    0xb6, 0xe0, 0x59, 0x28, 0x87, 0x74, 0x1a, 0x36 },
+		.result = { 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+		.valid = true
+	},
+	/* wycheproof - edge case for shared secret */
+	{
+		.private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4,
+			     0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3,
+			     0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc,
+			     0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 },
+		.public = { 0x89, 0xe1, 0x0d, 0x57, 0x01, 0xb4, 0x33, 0x7d,
+			    0x2d, 0x03, 0x21, 0x81, 0x53, 0x8b, 0x10, 0x64,
+			    0xbd, 0x40, 0x84, 0x40, 0x1c, 0xec, 0xa1, 0xfd,
+			    0x12, 0x66, 0x3a, 0x19, 0x59, 0x38, 0x80, 0x00 },
+		.result = { 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+		.valid = true
+	},
+	/* wycheproof - edge case for shared secret */
+	{
+		.private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4,
+			     0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3,
+			     0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc,
+			     0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 },
+		.public = { 0x2b, 0x55, 0xd3, 0xaa, 0x4a, 0x8f, 0x80, 0xc8,
+			    0xc0, 0xb2, 0xae, 0x5f, 0x93, 0x3e, 0x85, 0xaf,
+			    0x49, 0xbe, 0xac, 0x36, 0xc2, 0xfa, 0x73, 0x94,
+			    0xba, 0xb7, 0x6c, 0x89, 0x33, 0xf8, 0xf8, 0x1d },
+		.result = { 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+		.valid = true
+	},
+	/* wycheproof - edge case for shared secret */
+	{
+		.private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4,
+			     0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3,
+			     0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc,
+			     0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 },
+		.public = { 0x63, 0xe5, 0xb1, 0xfe, 0x96, 0x01, 0xfe, 0x84,
+			    0x38, 0x5d, 0x88, 0x66, 0xb0, 0x42, 0x12, 0x62,
+			    0xf7, 0x8f, 0xbf, 0xa5, 0xaf, 0xf9, 0x58, 0x5e,
+			    0x62, 0x66, 0x79, 0xb1, 0x85, 0x47, 0xd9, 0x59 },
+		.result = { 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f },
+		.valid = true
+	},
+	/* wycheproof - edge case for shared secret */
+	{
+		.private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4,
+			     0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3,
+			     0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc,
+			     0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 },
+		.public = { 0xe4, 0x28, 0xf3, 0xda, 0xc1, 0x78, 0x09, 0xf8,
+			    0x27, 0xa5, 0x22, 0xce, 0x32, 0x35, 0x50, 0x58,
+			    0xd0, 0x73, 0x69, 0x36, 0x4a, 0xa7, 0x89, 0x02,
+			    0xee, 0x10, 0x13, 0x9b, 0x9f, 0x9d, 0xd6, 0x53 },
+		.result = { 0xfc, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f },
+		.valid = true
+	},
+	/* wycheproof - edge case for shared secret */
+	{
+		.private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4,
+			     0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3,
+			     0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc,
+			     0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 },
+		.public = { 0xb3, 0xb5, 0x0e, 0x3e, 0xd3, 0xa4, 0x07, 0xb9,
+			    0x5d, 0xe9, 0x42, 0xef, 0x74, 0x57, 0x5b, 0x5a,
+			    0xb8, 0xa1, 0x0c, 0x09, 0xee, 0x10, 0x35, 0x44,
+			    0xd6, 0x0b, 0xdf, 0xed, 0x81, 0x38, 0xab, 0x2b },
+		.result = { 0xf9, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f },
+		.valid = true
+	},
+	/* wycheproof - edge case for shared secret */
+	{
+		.private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4,
+			     0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3,
+			     0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc,
+			     0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 },
+		.public = { 0x21, 0x3f, 0xff, 0xe9, 0x3d, 0x5e, 0xa8, 0xcd,
+			    0x24, 0x2e, 0x46, 0x28, 0x44, 0x02, 0x99, 0x22,
+			    0xc4, 0x3c, 0x77, 0xc9, 0xe3, 0xe4, 0x2f, 0x56,
+			    0x2f, 0x48, 0x5d, 0x24, 0xc5, 0x01, 0xa2, 0x0b },
+		.result = { 0xf3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f },
+		.valid = true
+	},
+	/* wycheproof - edge case for shared secret */
+	{
+		.private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4,
+			     0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3,
+			     0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc,
+			     0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 },
+		.public = { 0x91, 0xb2, 0x32, 0xa1, 0x78, 0xb3, 0xcd, 0x53,
+			    0x09, 0x32, 0x44, 0x1e, 0x61, 0x39, 0x41, 0x8f,
+			    0x72, 0x17, 0x22, 0x92, 0xf1, 0xda, 0x4c, 0x18,
+			    0x34, 0xfc, 0x5e, 0xbf, 0xef, 0xb5, 0x1e, 0x3f },
+		.result = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x03 },
+		.valid = true
+	},
+	/* wycheproof - edge case for shared secret */
+	{
+		.private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4,
+			     0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3,
+			     0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc,
+			     0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 },
+		.public = { 0x04, 0x5c, 0x6e, 0x11, 0xc5, 0xd3, 0x32, 0x55,
+			    0x6c, 0x78, 0x22, 0xfe, 0x94, 0xeb, 0xf8, 0x9b,
+			    0x56, 0xa3, 0x87, 0x8d, 0xc2, 0x7c, 0xa0, 0x79,
+			    0x10, 0x30, 0x58, 0x84, 0x9f, 0xab, 0xcb, 0x4f },
+		.result = { 0xe5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f },
+		.valid = true
+	},
+	/* wycheproof - edge case for shared secret */
+	{
+		.private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4,
+			     0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3,
+			     0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc,
+			     0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 },
+		.public = { 0x1c, 0xa2, 0x19, 0x0b, 0x71, 0x16, 0x35, 0x39,
+			    0x06, 0x3c, 0x35, 0x77, 0x3b, 0xda, 0x0c, 0x9c,
+			    0x92, 0x8e, 0x91, 0x36, 0xf0, 0x62, 0x0a, 0xeb,
+			    0x09, 0x3f, 0x09, 0x91, 0x97, 0xb7, 0xf7, 0x4e },
+		.result = { 0xe3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f },
+		.valid = true
+	},
+	/* wycheproof - edge case for shared secret */
+	{
+		.private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4,
+			     0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3,
+			     0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc,
+			     0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 },
+		.public = { 0xf7, 0x6e, 0x90, 0x10, 0xac, 0x33, 0xc5, 0x04,
+			    0x3b, 0x2d, 0x3b, 0x76, 0xa8, 0x42, 0x17, 0x10,
+			    0x00, 0xc4, 0x91, 0x62, 0x22, 0xe9, 0xe8, 0x58,
+			    0x97, 0xa0, 0xae, 0xc7, 0xf6, 0x35, 0x0b, 0x3c },
+		.result = { 0xdd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f },
+		.valid = true
+	},
+	/* wycheproof - edge case for shared secret */
+	{
+		.private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4,
+			     0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3,
+			     0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc,
+			     0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 },
+		.public = { 0xbb, 0x72, 0x68, 0x8d, 0x8f, 0x8a, 0xa7, 0xa3,
+			    0x9c, 0xd6, 0x06, 0x0c, 0xd5, 0xc8, 0x09, 0x3c,
+			    0xde, 0xc6, 0xfe, 0x34, 0x19, 0x37, 0xc3, 0x88,
+			    0x6a, 0x99, 0x34, 0x6c, 0xd0, 0x7f, 0xaa, 0x55 },
+		.result = { 0xdb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f },
+		.valid = true
+	},
+	/* wycheproof - edge case for shared secret */
+	{
+		.private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4,
+			     0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3,
+			     0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc,
+			     0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 },
+		.public = { 0x88, 0xfd, 0xde, 0xa1, 0x93, 0x39, 0x1c, 0x6a,
+			    0x59, 0x33, 0xef, 0x9b, 0x71, 0x90, 0x15, 0x49,
+			    0x44, 0x72, 0x05, 0xaa, 0xe9, 0xda, 0x92, 0x8a,
+			    0x6b, 0x91, 0xa3, 0x52, 0xba, 0x10, 0xf4, 0x1f },
+		.result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02 },
+		.valid = true
+	},
+	/* wycheproof - edge case for shared secret */
+	{
+		.private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4,
+			     0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3,
+			     0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc,
+			     0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 },
+		.public = { 0x30, 0x3b, 0x39, 0x2f, 0x15, 0x31, 0x16, 0xca,
+			    0xd9, 0xcc, 0x68, 0x2a, 0x00, 0xcc, 0xc4, 0x4c,
+			    0x95, 0xff, 0x0d, 0x3b, 0xbe, 0x56, 0x8b, 0xeb,
+			    0x6c, 0x4e, 0x73, 0x9b, 0xaf, 0xdc, 0x2c, 0x68 },
+		.result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00 },
+		.valid = true
+	},
+	/* wycheproof - checking for overflow */
+	{
+		.private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d,
+			     0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d,
+			     0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c,
+			     0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 },
+		.public = { 0xfd, 0x30, 0x0a, 0xeb, 0x40, 0xe1, 0xfa, 0x58,
+			    0x25, 0x18, 0x41, 0x2b, 0x49, 0xb2, 0x08, 0xa7,
+			    0x84, 0x2b, 0x1e, 0x1f, 0x05, 0x6a, 0x04, 0x01,
+			    0x78, 0xea, 0x41, 0x41, 0x53, 0x4f, 0x65, 0x2d },
+		.result = { 0xb7, 0x34, 0x10, 0x5d, 0xc2, 0x57, 0x58, 0x5d,
+			    0x73, 0xb5, 0x66, 0xcc, 0xb7, 0x6f, 0x06, 0x27,
+			    0x95, 0xcc, 0xbe, 0xc8, 0x91, 0x28, 0xe5, 0x2b,
+			    0x02, 0xf3, 0xe5, 0x96, 0x39, 0xf1, 0x3c, 0x46 },
+		.valid = true
+	},
+	/* wycheproof - checking for overflow */
+	{
+		.private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d,
+			     0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d,
+			     0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c,
+			     0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 },
+		.public = { 0xc8, 0xef, 0x79, 0xb5, 0x14, 0xd7, 0x68, 0x26,
+			    0x77, 0xbc, 0x79, 0x31, 0xe0, 0x6e, 0xe5, 0xc2,
+			    0x7c, 0x9b, 0x39, 0x2b, 0x4a, 0xe9, 0x48, 0x44,
+			    0x73, 0xf5, 0x54, 0xe6, 0x67, 0x8e, 0xcc, 0x2e },
+		.result = { 0x64, 0x7a, 0x46, 0xb6, 0xfc, 0x3f, 0x40, 0xd6,
+			    0x21, 0x41, 0xee, 0x3c, 0xee, 0x70, 0x6b, 0x4d,
+			    0x7a, 0x92, 0x71, 0x59, 0x3a, 0x7b, 0x14, 0x3e,
+			    0x8e, 0x2e, 0x22, 0x79, 0x88, 0x3e, 0x45, 0x50 },
+		.valid = true
+	},
+	/* wycheproof - checking for overflow */
+	{
+		.private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d,
+			     0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d,
+			     0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c,
+			     0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 },
+		.public = { 0x64, 0xae, 0xac, 0x25, 0x04, 0x14, 0x48, 0x61,
+			    0x53, 0x2b, 0x7b, 0xbc, 0xb6, 0xc8, 0x7d, 0x67,
+			    0xdd, 0x4c, 0x1f, 0x07, 0xeb, 0xc2, 0xe0, 0x6e,
+			    0xff, 0xb9, 0x5a, 0xec, 0xc6, 0x17, 0x0b, 0x2c },
+		.result = { 0x4f, 0xf0, 0x3d, 0x5f, 0xb4, 0x3c, 0xd8, 0x65,
+			    0x7a, 0x3c, 0xf3, 0x7c, 0x13, 0x8c, 0xad, 0xce,
+			    0xcc, 0xe5, 0x09, 0xe4, 0xeb, 0xa0, 0x89, 0xd0,
+			    0xef, 0x40, 0xb4, 0xe4, 0xfb, 0x94, 0x61, 0x55 },
+		.valid = true
+	},
+	/* wycheproof - checking for overflow */
+	{
+		.private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d,
+			     0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d,
+			     0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c,
+			     0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 },
+		.public = { 0xbf, 0x68, 0xe3, 0x5e, 0x9b, 0xdb, 0x7e, 0xee,
+			    0x1b, 0x50, 0x57, 0x02, 0x21, 0x86, 0x0f, 0x5d,
+			    0xcd, 0xad, 0x8a, 0xcb, 0xab, 0x03, 0x1b, 0x14,
+			    0x97, 0x4c, 0xc4, 0x90, 0x13, 0xc4, 0x98, 0x31 },
+		.result = { 0x21, 0xce, 0xe5, 0x2e, 0xfd, 0xbc, 0x81, 0x2e,
+			    0x1d, 0x02, 0x1a, 0x4a, 0xf1, 0xe1, 0xd8, 0xbc,
+			    0x4d, 0xb3, 0xc4, 0x00, 0xe4, 0xd2, 0xa2, 0xc5,
+			    0x6a, 0x39, 0x26, 0xdb, 0x4d, 0x99, 0xc6, 0x5b },
+		.valid = true
+	},
+	/* wycheproof - checking for overflow */
+	{
+		.private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d,
+			     0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d,
+			     0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c,
+			     0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 },
+		.public = { 0x53, 0x47, 0xc4, 0x91, 0x33, 0x1a, 0x64, 0xb4,
+			    0x3d, 0xdc, 0x68, 0x30, 0x34, 0xe6, 0x77, 0xf5,
+			    0x3d, 0xc3, 0x2b, 0x52, 0xa5, 0x2a, 0x57, 0x7c,
+			    0x15, 0xa8, 0x3b, 0xf2, 0x98, 0xe9, 0x9f, 0x19 },
+		.result = { 0x18, 0xcb, 0x89, 0xe4, 0xe2, 0x0c, 0x0c, 0x2b,
+			    0xd3, 0x24, 0x30, 0x52, 0x45, 0x26, 0x6c, 0x93,
+			    0x27, 0x69, 0x0b, 0xbe, 0x79, 0xac, 0xb8, 0x8f,
+			    0x5b, 0x8f, 0xb3, 0xf7, 0x4e, 0xca, 0x3e, 0x52 },
+		.valid = true
+	},
+	/* wycheproof - private key == -1 (mod order) */
+	{
+		.private = { 0xa0, 0x23, 0xcd, 0xd0, 0x83, 0xef, 0x5b, 0xb8,
+			     0x2f, 0x10, 0xd6, 0x2e, 0x59, 0xe1, 0x5a, 0x68,
+			     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50 },
+		.public = { 0x25, 0x8e, 0x04, 0x52, 0x3b, 0x8d, 0x25, 0x3e,
+			    0xe6, 0x57, 0x19, 0xfc, 0x69, 0x06, 0xc6, 0x57,
+			    0x19, 0x2d, 0x80, 0x71, 0x7e, 0xdc, 0x82, 0x8f,
+			    0xa0, 0xaf, 0x21, 0x68, 0x6e, 0x2f, 0xaa, 0x75 },
+		.result = { 0x25, 0x8e, 0x04, 0x52, 0x3b, 0x8d, 0x25, 0x3e,
+			    0xe6, 0x57, 0x19, 0xfc, 0x69, 0x06, 0xc6, 0x57,
+			    0x19, 0x2d, 0x80, 0x71, 0x7e, 0xdc, 0x82, 0x8f,
+			    0xa0, 0xaf, 0x21, 0x68, 0x6e, 0x2f, 0xaa, 0x75 },
+		.valid = true
+	},
+	/* wycheproof - private key == 1 (mod order) on twist */
+	{
+		.private = { 0x58, 0x08, 0x3d, 0xd2, 0x61, 0xad, 0x91, 0xef,
+			     0xf9, 0x52, 0x32, 0x2e, 0xc8, 0x24, 0xc6, 0x82,
+			     0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			     0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x5f },
+		.public = { 0x2e, 0xae, 0x5e, 0xc3, 0xdd, 0x49, 0x4e, 0x9f,
+			    0x2d, 0x37, 0xd2, 0x58, 0xf8, 0x73, 0xa8, 0xe6,
+			    0xe9, 0xd0, 0xdb, 0xd1, 0xe3, 0x83, 0xef, 0x64,
+			    0xd9, 0x8b, 0xb9, 0x1b, 0x3e, 0x0b, 0xe0, 0x35 },
+		.result = { 0x2e, 0xae, 0x5e, 0xc3, 0xdd, 0x49, 0x4e, 0x9f,
+			    0x2d, 0x37, 0xd2, 0x58, 0xf8, 0x73, 0xa8, 0xe6,
+			    0xe9, 0xd0, 0xdb, 0xd1, 0xe3, 0x83, 0xef, 0x64,
+			    0xd9, 0x8b, 0xb9, 0x1b, 0x3e, 0x0b, 0xe0, 0x35 },
+		.valid = true
+	}
+};
+
+static bool __init curve25519_selftest(void)
+{
+	bool success = true, ret, ret2;
+	size_t i = 0, j;
+	u8 in[CURVE25519_KEY_SIZE];
+	u8 out[CURVE25519_KEY_SIZE], out2[CURVE25519_KEY_SIZE];
+
+	for (i = 0; i < ARRAY_SIZE(curve25519_test_vectors); ++i) {
+		memset(out, 0, CURVE25519_KEY_SIZE);
+		ret = curve25519(out, curve25519_test_vectors[i].private,
+				 curve25519_test_vectors[i].public);
+		if (ret != curve25519_test_vectors[i].valid ||
+		    memcmp(out, curve25519_test_vectors[i].result,
+			   CURVE25519_KEY_SIZE)) {
+			pr_err("curve25519 self-test %zu: FAIL\n", i + 1);
+			success = false;
+		}
+	}
+
+	for (i = 0; i < 5; ++i) {
+		get_random_bytes(in, sizeof(in));
+		ret = curve25519_generate_public(out, in);
+		ret2 = curve25519(out2, in, (u8[CURVE25519_KEY_SIZE]){ 9 });
+		if (ret != ret2 || memcmp(out, out2, CURVE25519_KEY_SIZE)) {
+			pr_err("curve25519 basepoint self-test %zu: FAIL: input - 0x",
+			       i + 1);
+			for (j = CURVE25519_KEY_SIZE; j-- > 0;)
+				printk(KERN_CONT "%02x", in[j]);
+			printk(KERN_CONT "\n");
+			success = false;
+		}
+	}
+
+	return success;
+}
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/selftest/poly1305.c b/net/wireguard/crypto/zinc/selftest/poly1305.c
--- a/net/wireguard/crypto/zinc/selftest/poly1305.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/selftest/poly1305.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,1107 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+struct poly1305_testvec {
+	const u8 *input, *output, *key;
+	size_t ilen;
+};
+
+/* RFC7539 */
+static const u8 input01[] __initconst = {
+	0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x67, 0x72,
+	0x61, 0x70, 0x68, 0x69, 0x63, 0x20, 0x46, 0x6f,
+	0x72, 0x75, 0x6d, 0x20, 0x52, 0x65, 0x73, 0x65,
+	0x61, 0x72, 0x63, 0x68, 0x20, 0x47, 0x72, 0x6f,
+	0x75, 0x70
+};
+static const u8 output01[] __initconst = {
+	0xa8, 0x06, 0x1d, 0xc1, 0x30, 0x51, 0x36, 0xc6,
+	0xc2, 0x2b, 0x8b, 0xaf, 0x0c, 0x01, 0x27, 0xa9
+};
+static const u8 key01[] __initconst = {
+	0x85, 0xd6, 0xbe, 0x78, 0x57, 0x55, 0x6d, 0x33,
+	0x7f, 0x44, 0x52, 0xfe, 0x42, 0xd5, 0x06, 0xa8,
+	0x01, 0x03, 0x80, 0x8a, 0xfb, 0x0d, 0xb2, 0xfd,
+	0x4a, 0xbf, 0xf6, 0xaf, 0x41, 0x49, 0xf5, 0x1b
+};
+
+/* "The Poly1305-AES message-authentication code" */
+static const u8 input02[] __initconst = {
+	0xf3, 0xf6
+};
+static const u8 output02[] __initconst = {
+	0xf4, 0xc6, 0x33, 0xc3, 0x04, 0x4f, 0xc1, 0x45,
+	0xf8, 0x4f, 0x33, 0x5c, 0xb8, 0x19, 0x53, 0xde
+};
+static const u8 key02[] __initconst = {
+	0x85, 0x1f, 0xc4, 0x0c, 0x34, 0x67, 0xac, 0x0b,
+	0xe0, 0x5c, 0xc2, 0x04, 0x04, 0xf3, 0xf7, 0x00,
+	0x58, 0x0b, 0x3b, 0x0f, 0x94, 0x47, 0xbb, 0x1e,
+	0x69, 0xd0, 0x95, 0xb5, 0x92, 0x8b, 0x6d, 0xbc
+};
+
+static const u8 input03[] __initconst = { };
+static const u8 output03[] __initconst = {
+	0xdd, 0x3f, 0xab, 0x22, 0x51, 0xf1, 0x1a, 0xc7,
+	0x59, 0xf0, 0x88, 0x71, 0x29, 0xcc, 0x2e, 0xe7
+};
+static const u8 key03[] __initconst = {
+	0xa0, 0xf3, 0x08, 0x00, 0x00, 0xf4, 0x64, 0x00,
+	0xd0, 0xc7, 0xe9, 0x07, 0x6c, 0x83, 0x44, 0x03,
+	0xdd, 0x3f, 0xab, 0x22, 0x51, 0xf1, 0x1a, 0xc7,
+	0x59, 0xf0, 0x88, 0x71, 0x29, 0xcc, 0x2e, 0xe7
+};
+
+static const u8 input04[] __initconst = {
+	0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8,
+	0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24,
+	0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb,
+	0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36
+};
+static const u8 output04[] __initconst = {
+	0x0e, 0xe1, 0xc1, 0x6b, 0xb7, 0x3f, 0x0f, 0x4f,
+	0xd1, 0x98, 0x81, 0x75, 0x3c, 0x01, 0xcd, 0xbe
+};
+static const u8 key04[] __initconst = {
+	0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09,
+	0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08,
+	0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88,
+	0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef
+};
+
+static const u8 input05[] __initconst = {
+	0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34,
+	0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1,
+	0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81,
+	0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0,
+	0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2,
+	0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67,
+	0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61,
+	0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9
+};
+static const u8 output05[] __initconst = {
+	0x51, 0x54, 0xad, 0x0d, 0x2c, 0xb2, 0x6e, 0x01,
+	0x27, 0x4f, 0xc5, 0x11, 0x48, 0x49, 0x1f, 0x1b
+};
+static const u8 key05[] __initconst = {
+	0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c,
+	0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07,
+	0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1,
+	0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57
+};
+
+/* self-generated vectors exercise "significant" lengths, such that they
+ * are handled by different code paths */
+static const u8 input06[] __initconst = {
+	0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34,
+	0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1,
+	0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81,
+	0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0,
+	0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2,
+	0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67,
+	0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61,
+	0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf
+};
+static const u8 output06[] __initconst = {
+	0x81, 0x20, 0x59, 0xa5, 0xda, 0x19, 0x86, 0x37,
+	0xca, 0xc7, 0xc4, 0xa6, 0x31, 0xbe, 0xe4, 0x66
+};
+static const u8 key06[] __initconst = {
+	0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c,
+	0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07,
+	0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1,
+	0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57
+};
+
+static const u8 input07[] __initconst = {
+	0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34,
+	0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1,
+	0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81,
+	0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0,
+	0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2,
+	0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67
+};
+static const u8 output07[] __initconst = {
+	0x5b, 0x88, 0xd7, 0xf6, 0x22, 0x8b, 0x11, 0xe2,
+	0xe2, 0x85, 0x79, 0xa5, 0xc0, 0xc1, 0xf7, 0x61
+};
+static const u8 key07[] __initconst = {
+	0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c,
+	0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07,
+	0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1,
+	0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57
+};
+
+static const u8 input08[] __initconst = {
+	0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34,
+	0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1,
+	0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81,
+	0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0,
+	0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2,
+	0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67,
+	0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61,
+	0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf,
+	0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8,
+	0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24,
+	0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb,
+	0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36
+};
+static const u8 output08[] __initconst = {
+	0xbb, 0xb6, 0x13, 0xb2, 0xb6, 0xd7, 0x53, 0xba,
+	0x07, 0x39, 0x5b, 0x91, 0x6a, 0xae, 0xce, 0x15
+};
+static const u8 key08[] __initconst = {
+	0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c,
+	0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07,
+	0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1,
+	0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57
+};
+
+static const u8 input09[] __initconst = {
+	0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34,
+	0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1,
+	0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81,
+	0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0,
+	0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2,
+	0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67,
+	0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61,
+	0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf,
+	0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09,
+	0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08,
+	0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88,
+	0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef,
+	0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8,
+	0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24
+};
+static const u8 output09[] __initconst = {
+	0xc7, 0x94, 0xd7, 0x05, 0x7d, 0x17, 0x78, 0xc4,
+	0xbb, 0xee, 0x0a, 0x39, 0xb3, 0xd9, 0x73, 0x42
+};
+static const u8 key09[] __initconst = {
+	0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c,
+	0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07,
+	0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1,
+	0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57
+};
+
+static const u8 input10[] __initconst = {
+	0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34,
+	0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1,
+	0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81,
+	0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0,
+	0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2,
+	0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67,
+	0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61,
+	0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf,
+	0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09,
+	0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08,
+	0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88,
+	0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef,
+	0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8,
+	0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24,
+	0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb,
+	0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36
+};
+static const u8 output10[] __initconst = {
+	0xff, 0xbc, 0xb9, 0xb3, 0x71, 0x42, 0x31, 0x52,
+	0xd7, 0xfc, 0xa5, 0xad, 0x04, 0x2f, 0xba, 0xa9
+};
+static const u8 key10[] __initconst = {
+	0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c,
+	0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07,
+	0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1,
+	0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57
+};
+
+static const u8 input11[] __initconst = {
+	0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34,
+	0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1,
+	0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81,
+	0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0,
+	0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2,
+	0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67,
+	0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61,
+	0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf,
+	0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09,
+	0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08,
+	0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88,
+	0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef,
+	0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8,
+	0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24,
+	0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb,
+	0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36,
+	0x81, 0x20, 0x59, 0xa5, 0xda, 0x19, 0x86, 0x37,
+	0xca, 0xc7, 0xc4, 0xa6, 0x31, 0xbe, 0xe4, 0x66
+};
+static const u8 output11[] __initconst = {
+	0x06, 0x9e, 0xd6, 0xb8, 0xef, 0x0f, 0x20, 0x7b,
+	0x3e, 0x24, 0x3b, 0xb1, 0x01, 0x9f, 0xe6, 0x32
+};
+static const u8 key11[] __initconst = {
+	0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c,
+	0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07,
+	0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1,
+	0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57
+};
+
+static const u8 input12[] __initconst = {
+	0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34,
+	0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1,
+	0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81,
+	0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0,
+	0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2,
+	0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67,
+	0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61,
+	0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf,
+	0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09,
+	0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08,
+	0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88,
+	0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef,
+	0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8,
+	0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24,
+	0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb,
+	0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36,
+	0x81, 0x20, 0x59, 0xa5, 0xda, 0x19, 0x86, 0x37,
+	0xca, 0xc7, 0xc4, 0xa6, 0x31, 0xbe, 0xe4, 0x66,
+	0x5b, 0x88, 0xd7, 0xf6, 0x22, 0x8b, 0x11, 0xe2,
+	0xe2, 0x85, 0x79, 0xa5, 0xc0, 0xc1, 0xf7, 0x61
+};
+static const u8 output12[] __initconst = {
+	0xcc, 0xa3, 0x39, 0xd9, 0xa4, 0x5f, 0xa2, 0x36,
+	0x8c, 0x2c, 0x68, 0xb3, 0xa4, 0x17, 0x91, 0x33
+};
+static const u8 key12[] __initconst = {
+	0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c,
+	0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07,
+	0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1,
+	0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57
+};
+
+static const u8 input13[] __initconst = {
+	0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34,
+	0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1,
+	0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81,
+	0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0,
+	0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2,
+	0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67,
+	0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61,
+	0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf,
+	0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09,
+	0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08,
+	0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88,
+	0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef,
+	0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8,
+	0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24,
+	0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb,
+	0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36,
+	0x81, 0x20, 0x59, 0xa5, 0xda, 0x19, 0x86, 0x37,
+	0xca, 0xc7, 0xc4, 0xa6, 0x31, 0xbe, 0xe4, 0x66,
+	0x5b, 0x88, 0xd7, 0xf6, 0x22, 0x8b, 0x11, 0xe2,
+	0xe2, 0x85, 0x79, 0xa5, 0xc0, 0xc1, 0xf7, 0x61,
+	0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34,
+	0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1,
+	0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81,
+	0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0,
+	0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2,
+	0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67,
+	0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61,
+	0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf,
+	0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09,
+	0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08,
+	0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88,
+	0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef,
+	0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8,
+	0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24,
+	0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb,
+	0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36
+};
+static const u8 output13[] __initconst = {
+	0x53, 0xf6, 0xe8, 0x28, 0xa2, 0xf0, 0xfe, 0x0e,
+	0xe8, 0x15, 0xbf, 0x0b, 0xd5, 0x84, 0x1a, 0x34
+};
+static const u8 key13[] __initconst = {
+	0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c,
+	0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07,
+	0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1,
+	0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57
+};
+
+static const u8 input14[] __initconst = {
+	0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34,
+	0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1,
+	0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81,
+	0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0,
+	0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2,
+	0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67,
+	0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61,
+	0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf,
+	0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09,
+	0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08,
+	0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88,
+	0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef,
+	0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8,
+	0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24,
+	0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb,
+	0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36,
+	0x81, 0x20, 0x59, 0xa5, 0xda, 0x19, 0x86, 0x37,
+	0xca, 0xc7, 0xc4, 0xa6, 0x31, 0xbe, 0xe4, 0x66,
+	0x5b, 0x88, 0xd7, 0xf6, 0x22, 0x8b, 0x11, 0xe2,
+	0xe2, 0x85, 0x79, 0xa5, 0xc0, 0xc1, 0xf7, 0x61,
+	0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34,
+	0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1,
+	0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81,
+	0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0,
+	0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2,
+	0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67,
+	0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61,
+	0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf,
+	0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09,
+	0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08,
+	0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88,
+	0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef,
+	0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8,
+	0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24,
+	0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb,
+	0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36,
+	0x81, 0x20, 0x59, 0xa5, 0xda, 0x19, 0x86, 0x37,
+	0xca, 0xc7, 0xc4, 0xa6, 0x31, 0xbe, 0xe4, 0x66,
+	0x5b, 0x88, 0xd7, 0xf6, 0x22, 0x8b, 0x11, 0xe2,
+	0xe2, 0x85, 0x79, 0xa5, 0xc0, 0xc1, 0xf7, 0x61
+};
+static const u8 output14[] __initconst = {
+	0xb8, 0x46, 0xd4, 0x4e, 0x9b, 0xbd, 0x53, 0xce,
+	0xdf, 0xfb, 0xfb, 0xb6, 0xb7, 0xfa, 0x49, 0x33
+};
+static const u8 key14[] __initconst = {
+	0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c,
+	0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07,
+	0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1,
+	0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57
+};
+
+/* 4th power of the key spills to 131th bit in SIMD key setup */
+static const u8 input15[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+};
+static const u8 output15[] __initconst = {
+	0x07, 0x14, 0x5a, 0x4c, 0x02, 0xfe, 0x5f, 0xa3,
+	0x20, 0x36, 0xde, 0x68, 0xfa, 0xbe, 0x90, 0x66
+};
+static const u8 key15[] __initconst = {
+	0xad, 0x62, 0x81, 0x07, 0xe8, 0x35, 0x1d, 0x0f,
+	0x2c, 0x23, 0x1a, 0x05, 0xdc, 0x4a, 0x41, 0x06,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+
+/* OpenSSL's poly1305_ieee754.c failed this in final stage */
+static const u8 input16[] __initconst = {
+	0x84, 0x23, 0x64, 0xe1, 0x56, 0x33, 0x6c, 0x09,
+	0x98, 0xb9, 0x33, 0xa6, 0x23, 0x77, 0x26, 0x18,
+	0x0d, 0x9e, 0x3f, 0xdc, 0xbd, 0xe4, 0xcd, 0x5d,
+	0x17, 0x08, 0x0f, 0xc3, 0xbe, 0xb4, 0x96, 0x14,
+	0xd7, 0x12, 0x2c, 0x03, 0x74, 0x63, 0xff, 0x10,
+	0x4d, 0x73, 0xf1, 0x9c, 0x12, 0x70, 0x46, 0x28,
+	0xd4, 0x17, 0xc4, 0xc5, 0x4a, 0x3f, 0xe3, 0x0d,
+	0x3c, 0x3d, 0x77, 0x14, 0x38, 0x2d, 0x43, 0xb0,
+	0x38, 0x2a, 0x50, 0xa5, 0xde, 0xe5, 0x4b, 0xe8,
+	0x44, 0xb0, 0x76, 0xe8, 0xdf, 0x88, 0x20, 0x1a,
+	0x1c, 0xd4, 0x3b, 0x90, 0xeb, 0x21, 0x64, 0x3f,
+	0xa9, 0x6f, 0x39, 0xb5, 0x18, 0xaa, 0x83, 0x40,
+	0xc9, 0x42, 0xff, 0x3c, 0x31, 0xba, 0xf7, 0xc9,
+	0xbd, 0xbf, 0x0f, 0x31, 0xae, 0x3f, 0xa0, 0x96,
+	0xbf, 0x8c, 0x63, 0x03, 0x06, 0x09, 0x82, 0x9f,
+	0xe7, 0x2e, 0x17, 0x98, 0x24, 0x89, 0x0b, 0xc8,
+	0xe0, 0x8c, 0x31, 0x5c, 0x1c, 0xce, 0x2a, 0x83,
+	0x14, 0x4d, 0xbb, 0xff, 0x09, 0xf7, 0x4e, 0x3e,
+	0xfc, 0x77, 0x0b, 0x54, 0xd0, 0x98, 0x4a, 0x8f,
+	0x19, 0xb1, 0x47, 0x19, 0xe6, 0x36, 0x35, 0x64,
+	0x1d, 0x6b, 0x1e, 0xed, 0xf6, 0x3e, 0xfb, 0xf0,
+	0x80, 0xe1, 0x78, 0x3d, 0x32, 0x44, 0x54, 0x12,
+	0x11, 0x4c, 0x20, 0xde, 0x0b, 0x83, 0x7a, 0x0d,
+	0xfa, 0x33, 0xd6, 0xb8, 0x28, 0x25, 0xff, 0xf4,
+	0x4c, 0x9a, 0x70, 0xea, 0x54, 0xce, 0x47, 0xf0,
+	0x7d, 0xf6, 0x98, 0xe6, 0xb0, 0x33, 0x23, 0xb5,
+	0x30, 0x79, 0x36, 0x4a, 0x5f, 0xc3, 0xe9, 0xdd,
+	0x03, 0x43, 0x92, 0xbd, 0xde, 0x86, 0xdc, 0xcd,
+	0xda, 0x94, 0x32, 0x1c, 0x5e, 0x44, 0x06, 0x04,
+	0x89, 0x33, 0x6c, 0xb6, 0x5b, 0xf3, 0x98, 0x9c,
+	0x36, 0xf7, 0x28, 0x2c, 0x2f, 0x5d, 0x2b, 0x88,
+	0x2c, 0x17, 0x1e, 0x74
+};
+static const u8 output16[] __initconst = {
+	0xf2, 0x48, 0x31, 0x2e, 0x57, 0x8d, 0x9d, 0x58,
+	0xf8, 0xb7, 0xbb, 0x4d, 0x19, 0x10, 0x54, 0x31
+};
+static const u8 key16[] __initconst = {
+	0x95, 0xd5, 0xc0, 0x05, 0x50, 0x3e, 0x51, 0x0d,
+	0x8c, 0xd0, 0xaa, 0x07, 0x2c, 0x4a, 0x4d, 0x06,
+	0x6e, 0xab, 0xc5, 0x2d, 0x11, 0x65, 0x3d, 0xf4,
+	0x7f, 0xbf, 0x63, 0xab, 0x19, 0x8b, 0xcc, 0x26
+};
+
+/* AVX2 in OpenSSL's poly1305-x86.pl failed this with 176+32 split */
+static const u8 input17[] __initconst = {
+	0x24, 0x8a, 0xc3, 0x10, 0x85, 0xb6, 0xc2, 0xad,
+	0xaa, 0xa3, 0x82, 0x59, 0xa0, 0xd7, 0x19, 0x2c,
+	0x5c, 0x35, 0xd1, 0xbb, 0x4e, 0xf3, 0x9a, 0xd9,
+	0x4c, 0x38, 0xd1, 0xc8, 0x24, 0x79, 0xe2, 0xdd,
+	0x21, 0x59, 0xa0, 0x77, 0x02, 0x4b, 0x05, 0x89,
+	0xbc, 0x8a, 0x20, 0x10, 0x1b, 0x50, 0x6f, 0x0a,
+	0x1a, 0xd0, 0xbb, 0xab, 0x76, 0xe8, 0x3a, 0x83,
+	0xf1, 0xb9, 0x4b, 0xe6, 0xbe, 0xae, 0x74, 0xe8,
+	0x74, 0xca, 0xb6, 0x92, 0xc5, 0x96, 0x3a, 0x75,
+	0x43, 0x6b, 0x77, 0x61, 0x21, 0xec, 0x9f, 0x62,
+	0x39, 0x9a, 0x3e, 0x66, 0xb2, 0xd2, 0x27, 0x07,
+	0xda, 0xe8, 0x19, 0x33, 0xb6, 0x27, 0x7f, 0x3c,
+	0x85, 0x16, 0xbc, 0xbe, 0x26, 0xdb, 0xbd, 0x86,
+	0xf3, 0x73, 0x10, 0x3d, 0x7c, 0xf4, 0xca, 0xd1,
+	0x88, 0x8c, 0x95, 0x21, 0x18, 0xfb, 0xfb, 0xd0,
+	0xd7, 0xb4, 0xbe, 0xdc, 0x4a, 0xe4, 0x93, 0x6a,
+	0xff, 0x91, 0x15, 0x7e, 0x7a, 0xa4, 0x7c, 0x54,
+	0x44, 0x2e, 0xa7, 0x8d, 0x6a, 0xc2, 0x51, 0xd3,
+	0x24, 0xa0, 0xfb, 0xe4, 0x9d, 0x89, 0xcc, 0x35,
+	0x21, 0xb6, 0x6d, 0x16, 0xe9, 0xc6, 0x6a, 0x37,
+	0x09, 0x89, 0x4e, 0x4e, 0xb0, 0xa4, 0xee, 0xdc,
+	0x4a, 0xe1, 0x94, 0x68, 0xe6, 0x6b, 0x81, 0xf2,
+	0x71, 0x35, 0x1b, 0x1d, 0x92, 0x1e, 0xa5, 0x51,
+	0x04, 0x7a, 0xbc, 0xc6, 0xb8, 0x7a, 0x90, 0x1f,
+	0xde, 0x7d, 0xb7, 0x9f, 0xa1, 0x81, 0x8c, 0x11,
+	0x33, 0x6d, 0xbc, 0x07, 0x24, 0x4a, 0x40, 0xeb
+};
+static const u8 output17[] __initconst = {
+	0xbc, 0x93, 0x9b, 0xc5, 0x28, 0x14, 0x80, 0xfa,
+	0x99, 0xc6, 0xd6, 0x8c, 0x25, 0x8e, 0xc4, 0x2f
+};
+static const u8 key17[] __initconst = {
+	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+	0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+
+/* test vectors from Google */
+static const u8 input18[] __initconst = { };
+static const u8 output18[] __initconst = {
+	0x47, 0x10, 0x13, 0x0e, 0x9f, 0x6f, 0xea, 0x8d,
+	0x72, 0x29, 0x38, 0x50, 0xa6, 0x67, 0xd8, 0x6c
+};
+static const u8 key18[] __initconst = {
+	0xc8, 0xaf, 0xaa, 0xc3, 0x31, 0xee, 0x37, 0x2c,
+	0xd6, 0x08, 0x2d, 0xe1, 0x34, 0x94, 0x3b, 0x17,
+	0x47, 0x10, 0x13, 0x0e, 0x9f, 0x6f, 0xea, 0x8d,
+	0x72, 0x29, 0x38, 0x50, 0xa6, 0x67, 0xd8, 0x6c
+};
+
+static const u8 input19[] __initconst = {
+	0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f,
+	0x72, 0x6c, 0x64, 0x21
+};
+static const u8 output19[] __initconst = {
+	0xa6, 0xf7, 0x45, 0x00, 0x8f, 0x81, 0xc9, 0x16,
+	0xa2, 0x0d, 0xcc, 0x74, 0xee, 0xf2, 0xb2, 0xf0
+};
+static const u8 key19[] __initconst = {
+	0x74, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73, 0x20,
+	0x33, 0x32, 0x2d, 0x62, 0x79, 0x74, 0x65, 0x20,
+	0x6b, 0x65, 0x79, 0x20, 0x66, 0x6f, 0x72, 0x20,
+	0x50, 0x6f, 0x6c, 0x79, 0x31, 0x33, 0x30, 0x35
+};
+
+static const u8 input20[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+static const u8 output20[] __initconst = {
+	0x49, 0xec, 0x78, 0x09, 0x0e, 0x48, 0x1e, 0xc6,
+	0xc2, 0x6b, 0x33, 0xb9, 0x1c, 0xcc, 0x03, 0x07
+};
+static const u8 key20[] __initconst = {
+	0x74, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73, 0x20,
+	0x33, 0x32, 0x2d, 0x62, 0x79, 0x74, 0x65, 0x20,
+	0x6b, 0x65, 0x79, 0x20, 0x66, 0x6f, 0x72, 0x20,
+	0x50, 0x6f, 0x6c, 0x79, 0x31, 0x33, 0x30, 0x35
+};
+
+static const u8 input21[] __initconst = {
+	0x89, 0xda, 0xb8, 0x0b, 0x77, 0x17, 0xc1, 0xdb,
+	0x5d, 0xb4, 0x37, 0x86, 0x0a, 0x3f, 0x70, 0x21,
+	0x8e, 0x93, 0xe1, 0xb8, 0xf4, 0x61, 0xfb, 0x67,
+	0x7f, 0x16, 0xf3, 0x5f, 0x6f, 0x87, 0xe2, 0xa9,
+	0x1c, 0x99, 0xbc, 0x3a, 0x47, 0xac, 0xe4, 0x76,
+	0x40, 0xcc, 0x95, 0xc3, 0x45, 0xbe, 0x5e, 0xcc,
+	0xa5, 0xa3, 0x52, 0x3c, 0x35, 0xcc, 0x01, 0x89,
+	0x3a, 0xf0, 0xb6, 0x4a, 0x62, 0x03, 0x34, 0x27,
+	0x03, 0x72, 0xec, 0x12, 0x48, 0x2d, 0x1b, 0x1e,
+	0x36, 0x35, 0x61, 0x69, 0x8a, 0x57, 0x8b, 0x35,
+	0x98, 0x03, 0x49, 0x5b, 0xb4, 0xe2, 0xef, 0x19,
+	0x30, 0xb1, 0x7a, 0x51, 0x90, 0xb5, 0x80, 0xf1,
+	0x41, 0x30, 0x0d, 0xf3, 0x0a, 0xdb, 0xec, 0xa2,
+	0x8f, 0x64, 0x27, 0xa8, 0xbc, 0x1a, 0x99, 0x9f,
+	0xd5, 0x1c, 0x55, 0x4a, 0x01, 0x7d, 0x09, 0x5d,
+	0x8c, 0x3e, 0x31, 0x27, 0xda, 0xf9, 0xf5, 0x95
+};
+static const u8 output21[] __initconst = {
+	0xc8, 0x5d, 0x15, 0xed, 0x44, 0xc3, 0x78, 0xd6,
+	0xb0, 0x0e, 0x23, 0x06, 0x4c, 0x7b, 0xcd, 0x51
+};
+static const u8 key21[] __initconst = {
+	0x2d, 0x77, 0x3b, 0xe3, 0x7a, 0xdb, 0x1e, 0x4d,
+	0x68, 0x3b, 0xf0, 0x07, 0x5e, 0x79, 0xc4, 0xee,
+	0x03, 0x79, 0x18, 0x53, 0x5a, 0x7f, 0x99, 0xcc,
+	0xb7, 0x04, 0x0f, 0xb5, 0xf5, 0xf4, 0x3a, 0xea
+};
+
+static const u8 input22[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0b,
+	0x17, 0x03, 0x03, 0x02, 0x00, 0x00, 0x00, 0x00,
+	0x06, 0xdb, 0x1f, 0x1f, 0x36, 0x8d, 0x69, 0x6a,
+	0x81, 0x0a, 0x34, 0x9c, 0x0c, 0x71, 0x4c, 0x9a,
+	0x5e, 0x78, 0x50, 0xc2, 0x40, 0x7d, 0x72, 0x1a,
+	0xcd, 0xed, 0x95, 0xe0, 0x18, 0xd7, 0xa8, 0x52,
+	0x66, 0xa6, 0xe1, 0x28, 0x9c, 0xdb, 0x4a, 0xeb,
+	0x18, 0xda, 0x5a, 0xc8, 0xa2, 0xb0, 0x02, 0x6d,
+	0x24, 0xa5, 0x9a, 0xd4, 0x85, 0x22, 0x7f, 0x3e,
+	0xae, 0xdb, 0xb2, 0xe7, 0xe3, 0x5e, 0x1c, 0x66,
+	0xcd, 0x60, 0xf9, 0xab, 0xf7, 0x16, 0xdc, 0xc9,
+	0xac, 0x42, 0x68, 0x2d, 0xd7, 0xda, 0xb2, 0x87,
+	0xa7, 0x02, 0x4c, 0x4e, 0xef, 0xc3, 0x21, 0xcc,
+	0x05, 0x74, 0xe1, 0x67, 0x93, 0xe3, 0x7c, 0xec,
+	0x03, 0xc5, 0xbd, 0xa4, 0x2b, 0x54, 0xc1, 0x14,
+	0xa8, 0x0b, 0x57, 0xaf, 0x26, 0x41, 0x6c, 0x7b,
+	0xe7, 0x42, 0x00, 0x5e, 0x20, 0x85, 0x5c, 0x73,
+	0xe2, 0x1d, 0xc8, 0xe2, 0xed, 0xc9, 0xd4, 0x35,
+	0xcb, 0x6f, 0x60, 0x59, 0x28, 0x00, 0x11, 0xc2,
+	0x70, 0xb7, 0x15, 0x70, 0x05, 0x1c, 0x1c, 0x9b,
+	0x30, 0x52, 0x12, 0x66, 0x20, 0xbc, 0x1e, 0x27,
+	0x30, 0xfa, 0x06, 0x6c, 0x7a, 0x50, 0x9d, 0x53,
+	0xc6, 0x0e, 0x5a, 0xe1, 0xb4, 0x0a, 0xa6, 0xe3,
+	0x9e, 0x49, 0x66, 0x92, 0x28, 0xc9, 0x0e, 0xec,
+	0xb4, 0xa5, 0x0d, 0xb3, 0x2a, 0x50, 0xbc, 0x49,
+	0xe9, 0x0b, 0x4f, 0x4b, 0x35, 0x9a, 0x1d, 0xfd,
+	0x11, 0x74, 0x9c, 0xd3, 0x86, 0x7f, 0xcf, 0x2f,
+	0xb7, 0xbb, 0x6c, 0xd4, 0x73, 0x8f, 0x6a, 0x4a,
+	0xd6, 0xf7, 0xca, 0x50, 0x58, 0xf7, 0x61, 0x88,
+	0x45, 0xaf, 0x9f, 0x02, 0x0f, 0x6c, 0x3b, 0x96,
+	0x7b, 0x8f, 0x4c, 0xd4, 0xa9, 0x1e, 0x28, 0x13,
+	0xb5, 0x07, 0xae, 0x66, 0xf2, 0xd3, 0x5c, 0x18,
+	0x28, 0x4f, 0x72, 0x92, 0x18, 0x60, 0x62, 0xe1,
+	0x0f, 0xd5, 0x51, 0x0d, 0x18, 0x77, 0x53, 0x51,
+	0xef, 0x33, 0x4e, 0x76, 0x34, 0xab, 0x47, 0x43,
+	0xf5, 0xb6, 0x8f, 0x49, 0xad, 0xca, 0xb3, 0x84,
+	0xd3, 0xfd, 0x75, 0xf7, 0x39, 0x0f, 0x40, 0x06,
+	0xef, 0x2a, 0x29, 0x5c, 0x8c, 0x7a, 0x07, 0x6a,
+	0xd5, 0x45, 0x46, 0xcd, 0x25, 0xd2, 0x10, 0x7f,
+	0xbe, 0x14, 0x36, 0xc8, 0x40, 0x92, 0x4a, 0xae,
+	0xbe, 0x5b, 0x37, 0x08, 0x93, 0xcd, 0x63, 0xd1,
+	0x32, 0x5b, 0x86, 0x16, 0xfc, 0x48, 0x10, 0x88,
+	0x6b, 0xc1, 0x52, 0xc5, 0x32, 0x21, 0xb6, 0xdf,
+	0x37, 0x31, 0x19, 0x39, 0x32, 0x55, 0xee, 0x72,
+	0xbc, 0xaa, 0x88, 0x01, 0x74, 0xf1, 0x71, 0x7f,
+	0x91, 0x84, 0xfa, 0x91, 0x64, 0x6f, 0x17, 0xa2,
+	0x4a, 0xc5, 0x5d, 0x16, 0xbf, 0xdd, 0xca, 0x95,
+	0x81, 0xa9, 0x2e, 0xda, 0x47, 0x92, 0x01, 0xf0,
+	0xed, 0xbf, 0x63, 0x36, 0x00, 0xd6, 0x06, 0x6d,
+	0x1a, 0xb3, 0x6d, 0x5d, 0x24, 0x15, 0xd7, 0x13,
+	0x51, 0xbb, 0xcd, 0x60, 0x8a, 0x25, 0x10, 0x8d,
+	0x25, 0x64, 0x19, 0x92, 0xc1, 0xf2, 0x6c, 0x53,
+	0x1c, 0xf9, 0xf9, 0x02, 0x03, 0xbc, 0x4c, 0xc1,
+	0x9f, 0x59, 0x27, 0xd8, 0x34, 0xb0, 0xa4, 0x71,
+	0x16, 0xd3, 0x88, 0x4b, 0xbb, 0x16, 0x4b, 0x8e,
+	0xc8, 0x83, 0xd1, 0xac, 0x83, 0x2e, 0x56, 0xb3,
+	0x91, 0x8a, 0x98, 0x60, 0x1a, 0x08, 0xd1, 0x71,
+	0x88, 0x15, 0x41, 0xd5, 0x94, 0xdb, 0x39, 0x9c,
+	0x6a, 0xe6, 0x15, 0x12, 0x21, 0x74, 0x5a, 0xec,
+	0x81, 0x4c, 0x45, 0xb0, 0xb0, 0x5b, 0x56, 0x54,
+	0x36, 0xfd, 0x6f, 0x13, 0x7a, 0xa1, 0x0a, 0x0c,
+	0x0b, 0x64, 0x37, 0x61, 0xdb, 0xd6, 0xf9, 0xa9,
+	0xdc, 0xb9, 0x9b, 0x1a, 0x6e, 0x69, 0x08, 0x54,
+	0xce, 0x07, 0x69, 0xcd, 0xe3, 0x97, 0x61, 0xd8,
+	0x2f, 0xcd, 0xec, 0x15, 0xf0, 0xd9, 0x2d, 0x7d,
+	0x8e, 0x94, 0xad, 0xe8, 0xeb, 0x83, 0xfb, 0xe0
+};
+static const u8 output22[] __initconst = {
+	0x26, 0x37, 0x40, 0x8f, 0xe1, 0x30, 0x86, 0xea,
+	0x73, 0xf9, 0x71, 0xe3, 0x42, 0x5e, 0x28, 0x20
+};
+static const u8 key22[] __initconst = {
+	0x99, 0xe5, 0x82, 0x2d, 0xd4, 0x17, 0x3c, 0x99,
+	0x5e, 0x3d, 0xae, 0x0d, 0xde, 0xfb, 0x97, 0x74,
+	0x3f, 0xde, 0x3b, 0x08, 0x01, 0x34, 0xb3, 0x9f,
+	0x76, 0xe9, 0xbf, 0x8d, 0x0e, 0x88, 0xd5, 0x46
+};
+
+/* test vectors from Hanno Böck */
+static const u8 input23[] __initconst = {
+	0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc,
+	0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc,
+	0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc,
+	0xcc, 0x80, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc,
+	0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc,
+	0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc,
+	0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc,
+	0xcc, 0xcc, 0xcc, 0xcc, 0xce, 0xcc, 0xcc, 0xcc,
+	0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc,
+	0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xc5,
+	0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc,
+	0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc,
+	0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xe3, 0xcc, 0xcc,
+	0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc,
+	0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc,
+	0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc,
+	0xcc, 0xcc, 0xcc, 0xcc, 0xac, 0xcc, 0xcc, 0xcc,
+	0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xe6,
+	0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0x00, 0x00, 0x00,
+	0xaf, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc,
+	0xcc, 0xcc, 0xff, 0xff, 0xff, 0xf5, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0xff, 0xff, 0xff, 0xe7, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x71, 0x92, 0x05, 0xa8, 0x52, 0x1d,
+	0xfc
+};
+static const u8 output23[] __initconst = {
+	0x85, 0x59, 0xb8, 0x76, 0xec, 0xee, 0xd6, 0x6e,
+	0xb3, 0x77, 0x98, 0xc0, 0x45, 0x7b, 0xaf, 0xf9
+};
+static const u8 key23[] __initconst = {
+	0x7f, 0x1b, 0x02, 0x64, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc
+};
+
+static const u8 input24[] __initconst = {
+	0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa,
+	0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa,
+	0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa,
+	0xaa, 0xaa, 0xaa, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x80, 0x02, 0x64
+};
+static const u8 output24[] __initconst = {
+	0x00, 0xbd, 0x12, 0x58, 0x97, 0x8e, 0x20, 0x54,
+	0x44, 0xc9, 0xaa, 0xaa, 0x82, 0x00, 0x6f, 0xed
+};
+static const u8 key24[] __initconst = {
+	0xe0, 0x00, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa,
+	0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa
+};
+
+static const u8 input25[] __initconst = {
+	0x02, 0xfc
+};
+static const u8 output25[] __initconst = {
+	0x06, 0x12, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c,
+	0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c
+};
+static const u8 key25[] __initconst = {
+	0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c,
+	0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c,
+	0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c,
+	0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c
+};
+
+static const u8 input26[] __initconst = {
+	0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b,
+	0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b,
+	0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b,
+	0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b,
+	0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7a, 0x7b,
+	0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b,
+	0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b,
+	0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b,
+	0x7b, 0x7b, 0x5c, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b,
+	0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b,
+	0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b,
+	0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b,
+	0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b,
+	0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b,
+	0x7b, 0x7b, 0x7b, 0x7b, 0x6e, 0x7b, 0x00, 0x7b,
+	0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b,
+	0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b,
+	0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b,
+	0x7b, 0x7b, 0x7b, 0x7a, 0x7b, 0x7b, 0x7b, 0x7b,
+	0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b,
+	0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b,
+	0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x5c,
+	0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b,
+	0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b,
+	0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b,
+	0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b,
+	0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b,
+	0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b,
+	0x7b, 0x6e, 0x7b, 0x00, 0x13, 0x00, 0x00, 0x00,
+	0x00, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0xf2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x20, 0x00, 0xef, 0xff, 0x00,
+	0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00,
+	0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x64, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x00,
+	0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf2,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x20, 0x00, 0xef, 0xff, 0x00, 0x09,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x7a, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00,
+	0x00, 0x09, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfc
+};
+static const u8 output26[] __initconst = {
+	0x33, 0x20, 0x5b, 0xbf, 0x9e, 0x9f, 0x8f, 0x72,
+	0x12, 0xab, 0x9e, 0x2a, 0xb9, 0xb7, 0xe4, 0xa5
+};
+static const u8 key26[] __initconst = {
+	0x00, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7b, 0x7b
+};
+
+static const u8 input27[] __initconst = {
+	0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77,
+	0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77,
+	0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77,
+	0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77,
+	0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77,
+	0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77,
+	0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77,
+	0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77,
+	0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77,
+	0x77, 0x77, 0x77, 0x77, 0xff, 0xff, 0xff, 0xe9,
+	0xe9, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac,
+	0xac, 0xac, 0xac, 0xac, 0x00, 0x00, 0xac, 0xac,
+	0xec, 0x01, 0x00, 0xac, 0xac, 0xac, 0x2c, 0xac,
+	0xa2, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac,
+	0xac, 0xac, 0xac, 0xac, 0x64, 0xf2
+};
+static const u8 output27[] __initconst = {
+	0x02, 0xee, 0x7c, 0x8c, 0x54, 0x6d, 0xde, 0xb1,
+	0xa4, 0x67, 0xe4, 0xc3, 0x98, 0x11, 0x58, 0xb9
+};
+static const u8 key27[] __initconst = {
+	0x00, 0x00, 0x00, 0x7f, 0x00, 0x00, 0x00, 0x7f,
+	0x01, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0xcf, 0x77, 0x77, 0x77, 0x77, 0x77,
+	0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77
+};
+
+/* nacl */
+static const u8 input28[] __initconst = {
+	0x8e, 0x99, 0x3b, 0x9f, 0x48, 0x68, 0x12, 0x73,
+	0xc2, 0x96, 0x50, 0xba, 0x32, 0xfc, 0x76, 0xce,
+	0x48, 0x33, 0x2e, 0xa7, 0x16, 0x4d, 0x96, 0xa4,
+	0x47, 0x6f, 0xb8, 0xc5, 0x31, 0xa1, 0x18, 0x6a,
+	0xc0, 0xdf, 0xc1, 0x7c, 0x98, 0xdc, 0xe8, 0x7b,
+	0x4d, 0xa7, 0xf0, 0x11, 0xec, 0x48, 0xc9, 0x72,
+	0x71, 0xd2, 0xc2, 0x0f, 0x9b, 0x92, 0x8f, 0xe2,
+	0x27, 0x0d, 0x6f, 0xb8, 0x63, 0xd5, 0x17, 0x38,
+	0xb4, 0x8e, 0xee, 0xe3, 0x14, 0xa7, 0xcc, 0x8a,
+	0xb9, 0x32, 0x16, 0x45, 0x48, 0xe5, 0x26, 0xae,
+	0x90, 0x22, 0x43, 0x68, 0x51, 0x7a, 0xcf, 0xea,
+	0xbd, 0x6b, 0xb3, 0x73, 0x2b, 0xc0, 0xe9, 0xda,
+	0x99, 0x83, 0x2b, 0x61, 0xca, 0x01, 0xb6, 0xde,
+	0x56, 0x24, 0x4a, 0x9e, 0x88, 0xd5, 0xf9, 0xb3,
+	0x79, 0x73, 0xf6, 0x22, 0xa4, 0x3d, 0x14, 0xa6,
+	0x59, 0x9b, 0x1f, 0x65, 0x4c, 0xb4, 0x5a, 0x74,
+	0xe3, 0x55, 0xa5
+};
+static const u8 output28[] __initconst = {
+	0xf3, 0xff, 0xc7, 0x70, 0x3f, 0x94, 0x00, 0xe5,
+	0x2a, 0x7d, 0xfb, 0x4b, 0x3d, 0x33, 0x05, 0xd9
+};
+static const u8 key28[] __initconst = {
+	0xee, 0xa6, 0xa7, 0x25, 0x1c, 0x1e, 0x72, 0x91,
+	0x6d, 0x11, 0xc2, 0xcb, 0x21, 0x4d, 0x3c, 0x25,
+	0x25, 0x39, 0x12, 0x1d, 0x8e, 0x23, 0x4e, 0x65,
+	0x2d, 0x65, 0x1f, 0xa4, 0xc8, 0xcf, 0xf8, 0x80
+};
+
+/* wrap 2^130-5 */
+static const u8 input29[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+};
+static const u8 output29[] __initconst = {
+	0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+static const u8 key29[] __initconst = {
+	0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+
+/* wrap 2^128 */
+static const u8 input30[] __initconst = {
+	0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+static const u8 output30[] __initconst = {
+	0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+static const u8 key30[] __initconst = {
+	0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+};
+
+/* limb carry */
+static const u8 input31[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xf0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+static const u8 output31[] __initconst = {
+	0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+static const u8 key31[] __initconst = {
+	0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+
+/* 2^130-5 */
+static const u8 input32[] __initconst = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xfb, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe,
+	0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe,
+	0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+	0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01
+};
+static const u8 output32[] __initconst = {
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+static const u8 key32[] __initconst = {
+	0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+
+/* 2^130-6 */
+static const u8 input33[] __initconst = {
+	0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+};
+static const u8 output33[] __initconst = {
+	0xfa, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+};
+static const u8 key33[] __initconst = {
+	0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+
+/* 5*H+L reduction intermediate */
+static const u8 input34[] __initconst = {
+	0xe3, 0x35, 0x94, 0xd7, 0x50, 0x5e, 0x43, 0xb9,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x33, 0x94, 0xd7, 0x50, 0x5e, 0x43, 0x79, 0xcd,
+	0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+static const u8 output34[] __initconst = {
+	0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x55, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+static const u8 key34[] __initconst = {
+	0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+
+/* 5*H+L reduction final */
+static const u8 input35[] __initconst = {
+	0xe3, 0x35, 0x94, 0xd7, 0x50, 0x5e, 0x43, 0xb9,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x33, 0x94, 0xd7, 0x50, 0x5e, 0x43, 0x79, 0xcd,
+	0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+static const u8 output35[] __initconst = {
+	0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+static const u8 key35[] __initconst = {
+	0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+
+static const struct poly1305_testvec poly1305_testvecs[] __initconst = {
+	{ input01, output01, key01, sizeof(input01) },
+	{ input02, output02, key02, sizeof(input02) },
+	{ input03, output03, key03, sizeof(input03) },
+	{ input04, output04, key04, sizeof(input04) },
+	{ input05, output05, key05, sizeof(input05) },
+	{ input06, output06, key06, sizeof(input06) },
+	{ input07, output07, key07, sizeof(input07) },
+	{ input08, output08, key08, sizeof(input08) },
+	{ input09, output09, key09, sizeof(input09) },
+	{ input10, output10, key10, sizeof(input10) },
+	{ input11, output11, key11, sizeof(input11) },
+	{ input12, output12, key12, sizeof(input12) },
+	{ input13, output13, key13, sizeof(input13) },
+	{ input14, output14, key14, sizeof(input14) },
+	{ input15, output15, key15, sizeof(input15) },
+	{ input16, output16, key16, sizeof(input16) },
+	{ input17, output17, key17, sizeof(input17) },
+	{ input18, output18, key18, sizeof(input18) },
+	{ input19, output19, key19, sizeof(input19) },
+	{ input20, output20, key20, sizeof(input20) },
+	{ input21, output21, key21, sizeof(input21) },
+	{ input22, output22, key22, sizeof(input22) },
+	{ input23, output23, key23, sizeof(input23) },
+	{ input24, output24, key24, sizeof(input24) },
+	{ input25, output25, key25, sizeof(input25) },
+	{ input26, output26, key26, sizeof(input26) },
+	{ input27, output27, key27, sizeof(input27) },
+	{ input28, output28, key28, sizeof(input28) },
+	{ input29, output29, key29, sizeof(input29) },
+	{ input30, output30, key30, sizeof(input30) },
+	{ input31, output31, key31, sizeof(input31) },
+	{ input32, output32, key32, sizeof(input32) },
+	{ input33, output33, key33, sizeof(input33) },
+	{ input34, output34, key34, sizeof(input34) },
+	{ input35, output35, key35, sizeof(input35) }
+};
+
+static bool __init poly1305_selftest(void)
+{
+	simd_context_t simd_context;
+	bool success = true;
+	size_t i, j;
+
+	simd_get(&simd_context);
+	for (i = 0; i < ARRAY_SIZE(poly1305_testvecs); ++i) {
+		struct poly1305_ctx poly1305;
+		u8 out[POLY1305_MAC_SIZE];
+
+		memset(out, 0, sizeof(out));
+		memset(&poly1305, 0, sizeof(poly1305));
+		poly1305_init(&poly1305, poly1305_testvecs[i].key);
+		poly1305_update(&poly1305, poly1305_testvecs[i].input,
+				poly1305_testvecs[i].ilen, &simd_context);
+		poly1305_final(&poly1305, out, &simd_context);
+		if (memcmp(out, poly1305_testvecs[i].output,
+			   POLY1305_MAC_SIZE)) {
+			pr_err("poly1305 self-test %zu: FAIL\n", i + 1);
+			success = false;
+		}
+		simd_relax(&simd_context);
+
+		if (poly1305_testvecs[i].ilen <= 1)
+			continue;
+
+		for (j = 1; j < poly1305_testvecs[i].ilen - 1; ++j) {
+			memset(out, 0, sizeof(out));
+			memset(&poly1305, 0, sizeof(poly1305));
+			poly1305_init(&poly1305, poly1305_testvecs[i].key);
+			poly1305_update(&poly1305, poly1305_testvecs[i].input,
+					j, &simd_context);
+			poly1305_update(&poly1305,
+					poly1305_testvecs[i].input + j,
+					poly1305_testvecs[i].ilen - j,
+					&simd_context);
+			poly1305_final(&poly1305, out, &simd_context);
+			if (memcmp(out, poly1305_testvecs[i].output,
+				   POLY1305_MAC_SIZE)) {
+				pr_err("poly1305 self-test %zu (split %zu): FAIL\n",
+				       i + 1, j);
+				success = false;
+			}
+
+			memset(out, 0, sizeof(out));
+			memset(&poly1305, 0, sizeof(poly1305));
+			poly1305_init(&poly1305, poly1305_testvecs[i].key);
+			poly1305_update(&poly1305, poly1305_testvecs[i].input,
+					j, &simd_context);
+			poly1305_update(&poly1305,
+					poly1305_testvecs[i].input + j,
+					poly1305_testvecs[i].ilen - j,
+					DONT_USE_SIMD);
+			poly1305_final(&poly1305, out, &simd_context);
+			if (memcmp(out, poly1305_testvecs[i].output,
+				   POLY1305_MAC_SIZE)) {
+				pr_err("poly1305 self-test %zu (split %zu, mixed simd): FAIL\n",
+				       i + 1, j);
+				success = false;
+			}
+			simd_relax(&simd_context);
+		}
+	}
+	simd_put(&simd_context);
+
+	return success;
+}
diff -Naur --no-dereference a/net/wireguard/crypto/zinc/selftest/run.h b/net/wireguard/crypto/zinc/selftest/run.h
--- a/net/wireguard/crypto/zinc/selftest/run.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc/selftest/run.h	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#ifndef _ZINC_SELFTEST_RUN_H
+#define _ZINC_SELFTEST_RUN_H
+
+#include <linux/kernel.h>
+#include <linux/printk.h>
+#include <linux/bug.h>
+
+static inline bool selftest_run(const char *name, bool (*selftest)(void),
+				bool *const nobs[], unsigned int nobs_len)
+{
+	unsigned long set = 0, subset = 0, largest_subset = 0;
+	unsigned int i;
+
+	BUILD_BUG_ON(!__builtin_constant_p(nobs_len) ||
+		     nobs_len >= BITS_PER_LONG);
+
+	if (!IS_ENABLED(CONFIG_ZINC_SELFTEST))
+		return true;
+
+	for (i = 0; i < nobs_len; ++i)
+		set |= ((unsigned long)*nobs[i]) << i;
+
+	do {
+		for (i = 0; i < nobs_len; ++i)
+			*nobs[i] = BIT(i) & subset;
+		if (selftest())
+			largest_subset = max(subset, largest_subset);
+		else
+			pr_err("%s self-test combination 0x%lx: FAIL\n", name,
+			       subset);
+		subset = (subset - set) & set;
+	} while (subset);
+
+	for (i = 0; i < nobs_len; ++i)
+		*nobs[i] = BIT(i) & largest_subset;
+
+	if (largest_subset == set)
+		pr_info("%s self-tests: pass\n", name);
+
+	return !WARN_ON(largest_subset != set);
+}
+
+#endif
diff -Naur --no-dereference a/net/wireguard/crypto/zinc.h b/net/wireguard/crypto/zinc.h
--- a/net/wireguard/crypto/zinc.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/crypto/zinc.h	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#ifndef _WG_ZINC_H
+#define _WG_ZINC_H
+
+int chacha20_mod_init(void);
+int poly1305_mod_init(void);
+int chacha20poly1305_mod_init(void);
+int blake2s_mod_init(void);
+int curve25519_mod_init(void);
+
+#endif
diff -Naur --no-dereference a/net/wireguard/device.c b/net/wireguard/device.c
--- a/net/wireguard/device.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/device.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,470 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#include "queueing.h"
+#include "socket.h"
+#include "timers.h"
+#include "device.h"
+#include "ratelimiter.h"
+#include "peer.h"
+#include "messages.h"
+
+#include <linux/module.h>
+#include <linux/rtnetlink.h>
+#include <linux/inet.h>
+#include <linux/netdevice.h>
+#include <linux/inetdevice.h>
+#include <linux/if_arp.h>
+#include <linux/icmp.h>
+#include <linux/suspend.h>
+#include <net/icmp.h>
+#include <net/rtnetlink.h>
+#include <net/ip_tunnels.h>
+#include <net/addrconf.h>
+
+static LIST_HEAD(device_list);
+
+static int wg_open(struct net_device *dev)
+{
+	struct in_device *dev_v4 = __in_dev_get_rtnl(dev);
+#ifndef COMPAT_CANNOT_USE_IN6_DEV_GET
+	struct inet6_dev *dev_v6 = __in6_dev_get(dev);
+#endif
+	struct wg_device *wg = netdev_priv(dev);
+	struct wg_peer *peer;
+	int ret;
+
+	if (dev_v4) {
+		/* At some point we might put this check near the ip_rt_send_
+		 * redirect call of ip_forward in net/ipv4/ip_forward.c, similar
+		 * to the current secpath check.
+		 */
+		IN_DEV_CONF_SET(dev_v4, SEND_REDIRECTS, false);
+		IPV4_DEVCONF_ALL(dev_net(dev), SEND_REDIRECTS) = false;
+	}
+#ifndef COMPAT_CANNOT_USE_IN6_DEV_GET
+	if (dev_v6)
+#ifndef COMPAT_CANNOT_USE_DEV_CNF
+		dev_v6->cnf.addr_gen_mode = IN6_ADDR_GEN_MODE_NONE;
+#else
+		dev_v6->addr_gen_mode = IN6_ADDR_GEN_MODE_NONE;
+#endif
+#endif
+
+	ret = wg_socket_init(wg, wg->incoming_port);
+	if (ret < 0)
+		return ret;
+	mutex_lock(&wg->device_update_lock);
+	list_for_each_entry(peer, &wg->peer_list, peer_list) {
+		wg_packet_send_staged_packets(peer);
+		if (peer->persistent_keepalive_interval)
+			wg_packet_send_keepalive(peer);
+	}
+	mutex_unlock(&wg->device_update_lock);
+	return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int wg_pm_notification(struct notifier_block *nb, unsigned long action,
+			      void *data)
+{
+	struct wg_device *wg;
+	struct wg_peer *peer;
+
+	/* If the machine is constantly suspending and resuming, as part of
+	 * its normal operation rather than as a somewhat rare event, then we
+	 * don't actually want to clear keys.
+	 */
+	if (IS_ENABLED(CONFIG_PM_AUTOSLEEP) || IS_ENABLED(CONFIG_ANDROID))
+		return 0;
+
+	if (action != PM_HIBERNATION_PREPARE && action != PM_SUSPEND_PREPARE)
+		return 0;
+
+	rtnl_lock();
+	list_for_each_entry(wg, &device_list, device_list) {
+		mutex_lock(&wg->device_update_lock);
+		list_for_each_entry(peer, &wg->peer_list, peer_list) {
+			del_timer(&peer->timer_zero_key_material);
+			wg_noise_handshake_clear(&peer->handshake);
+			wg_noise_keypairs_clear(&peer->keypairs);
+		}
+		mutex_unlock(&wg->device_update_lock);
+	}
+	rtnl_unlock();
+	rcu_barrier();
+	return 0;
+}
+
+static struct notifier_block pm_notifier = { .notifier_call = wg_pm_notification };
+#endif
+
+static int wg_stop(struct net_device *dev)
+{
+	struct wg_device *wg = netdev_priv(dev);
+	struct wg_peer *peer;
+
+	mutex_lock(&wg->device_update_lock);
+	list_for_each_entry(peer, &wg->peer_list, peer_list) {
+		wg_packet_purge_staged_packets(peer);
+		wg_timers_stop(peer);
+		wg_noise_handshake_clear(&peer->handshake);
+		wg_noise_keypairs_clear(&peer->keypairs);
+		wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake);
+	}
+	mutex_unlock(&wg->device_update_lock);
+	skb_queue_purge(&wg->incoming_handshakes);
+	wg_socket_reinit(wg, NULL, NULL);
+	return 0;
+}
+
+static netdev_tx_t wg_xmit(struct sk_buff *skb, struct net_device *dev)
+{
+	struct wg_device *wg = netdev_priv(dev);
+	struct sk_buff_head packets;
+	struct wg_peer *peer;
+	struct sk_buff *next;
+	sa_family_t family;
+	u32 mtu;
+	int ret;
+
+	if (unlikely(wg_skb_examine_untrusted_ip_hdr(skb) != skb->protocol)) {
+		ret = -EPROTONOSUPPORT;
+		net_dbg_ratelimited("%s: Invalid IP packet\n", dev->name);
+		goto err;
+	}
+
+	peer = wg_allowedips_lookup_dst(&wg->peer_allowedips, skb);
+	if (unlikely(!peer)) {
+		ret = -ENOKEY;
+		if (skb->protocol == htons(ETH_P_IP))
+			net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI4\n",
+					    dev->name, &ip_hdr(skb)->daddr);
+		else if (skb->protocol == htons(ETH_P_IPV6))
+			net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI6\n",
+					    dev->name, &ipv6_hdr(skb)->daddr);
+		goto err;
+	}
+
+	family = READ_ONCE(peer->endpoint.addr.sa_family);
+	if (unlikely(family != AF_INET && family != AF_INET6)) {
+		ret = -EDESTADDRREQ;
+		net_dbg_ratelimited("%s: No valid endpoint has been configured or discovered for peer %llu\n",
+				    dev->name, peer->internal_id);
+		goto err_peer;
+	}
+
+	mtu = skb_dst(skb) ? dst_mtu(skb_dst(skb)) : dev->mtu;
+
+	__skb_queue_head_init(&packets);
+	if (!skb_is_gso(skb)) {
+		skb_mark_not_on_list(skb);
+	} else {
+		struct sk_buff *segs = skb_gso_segment(skb, 0);
+
+		if (unlikely(IS_ERR(segs))) {
+			ret = PTR_ERR(segs);
+			goto err_peer;
+		}
+		dev_kfree_skb(skb);
+		skb = segs;
+	}
+
+	skb_list_walk_safe(skb, skb, next) {
+		skb_mark_not_on_list(skb);
+
+		skb = skb_share_check(skb, GFP_ATOMIC);
+		if (unlikely(!skb))
+			continue;
+
+		/* We only need to keep the original dst around for icmp,
+		 * so at this point we're in a position to drop it.
+		 */
+		skb_dst_drop(skb);
+
+		PACKET_CB(skb)->mtu = mtu;
+
+		__skb_queue_tail(&packets, skb);
+	}
+
+	spin_lock_bh(&peer->staged_packet_queue.lock);
+	/* If the queue is getting too big, we start removing the oldest packets
+	 * until it's small again. We do this before adding the new packet, so
+	 * we don't remove GSO segments that are in excess.
+	 */
+	while (skb_queue_len(&peer->staged_packet_queue) > MAX_STAGED_PACKETS) {
+		dev_kfree_skb(__skb_dequeue(&peer->staged_packet_queue));
+		++dev->stats.tx_dropped;
+	}
+	skb_queue_splice_tail(&packets, &peer->staged_packet_queue);
+	spin_unlock_bh(&peer->staged_packet_queue.lock);
+
+	wg_packet_send_staged_packets(peer);
+
+	wg_peer_put(peer);
+	return NETDEV_TX_OK;
+
+err_peer:
+	wg_peer_put(peer);
+err:
+	++dev->stats.tx_errors;
+	if (skb->protocol == htons(ETH_P_IP))
+		icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
+	else if (skb->protocol == htons(ETH_P_IPV6))
+		icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0);
+	kfree_skb(skb);
+	return ret;
+}
+
+static const struct net_device_ops netdev_ops = {
+	.ndo_open		= wg_open,
+	.ndo_stop		= wg_stop,
+	.ndo_start_xmit		= wg_xmit,
+	.ndo_get_stats64	= ip_tunnel_get_stats64
+};
+
+static void wg_destruct(struct net_device *dev)
+{
+	struct wg_device *wg = netdev_priv(dev);
+
+	rtnl_lock();
+	list_del(&wg->device_list);
+	rtnl_unlock();
+	mutex_lock(&wg->device_update_lock);
+	wg->incoming_port = 0;
+	wg_socket_reinit(wg, NULL, NULL);
+	/* The final references are cleared in the below calls to destroy_workqueue. */
+	wg_peer_remove_all(wg);
+	destroy_workqueue(wg->handshake_receive_wq);
+	destroy_workqueue(wg->handshake_send_wq);
+	destroy_workqueue(wg->packet_crypt_wq);
+	wg_packet_queue_free(&wg->decrypt_queue, true);
+	wg_packet_queue_free(&wg->encrypt_queue, true);
+	rcu_barrier(); /* Wait for all the peers to be actually freed. */
+	wg_ratelimiter_uninit();
+	memzero_explicit(&wg->static_identity, sizeof(wg->static_identity));
+	skb_queue_purge(&wg->incoming_handshakes);
+	free_percpu(dev->tstats);
+	free_percpu(wg->incoming_handshakes_worker);
+	if (wg->have_creating_net_ref)
+		put_net(wg->creating_net);
+	kvfree(wg->index_hashtable);
+	kvfree(wg->peer_hashtable);
+	mutex_unlock(&wg->device_update_lock);
+
+	pr_debug("%s: Interface deleted\n", dev->name);
+	free_netdev(dev);
+}
+
+static const struct device_type device_type = { .name = KBUILD_MODNAME };
+
+static void wg_setup(struct net_device *dev)
+{
+	struct wg_device *wg = netdev_priv(dev);
+	enum { WG_NETDEV_FEATURES = NETIF_F_HW_CSUM | NETIF_F_RXCSUM |
+				    NETIF_F_SG | NETIF_F_GSO |
+				    NETIF_F_GSO_SOFTWARE | NETIF_F_HIGHDMA };
+
+	dev->netdev_ops = &netdev_ops;
+	dev->hard_header_len = 0;
+	dev->addr_len = 0;
+	dev->needed_headroom = DATA_PACKET_HEAD_ROOM;
+	dev->needed_tailroom = noise_encrypted_len(MESSAGE_PADDING_MULTIPLE);
+	dev->type = ARPHRD_NONE;
+	dev->flags = IFF_POINTOPOINT | IFF_NOARP;
+#ifndef COMPAT_CANNOT_USE_IFF_NO_QUEUE
+	dev->priv_flags |= IFF_NO_QUEUE;
+#else
+	dev->tx_queue_len = 0;
+#endif
+	dev->features |= NETIF_F_LLTX;
+	dev->features |= WG_NETDEV_FEATURES;
+	dev->hw_features |= WG_NETDEV_FEATURES;
+	dev->hw_enc_features |= WG_NETDEV_FEATURES;
+	dev->mtu = ETH_DATA_LEN - MESSAGE_MINIMUM_LENGTH -
+		   sizeof(struct udphdr) -
+		   max(sizeof(struct ipv6hdr), sizeof(struct iphdr));
+
+	SET_NETDEV_DEVTYPE(dev, &device_type);
+
+	/* We need to keep the dst around in case of icmp replies. */
+	netif_keep_dst(dev);
+
+	memset(wg, 0, sizeof(*wg));
+	wg->dev = dev;
+}
+
+static int wg_newlink(struct net *src_net, struct net_device *dev,
+		      struct nlattr *tb[], struct nlattr *data[],
+		      struct netlink_ext_ack *extack)
+{
+	struct wg_device *wg = netdev_priv(dev);
+	int ret = -ENOMEM;
+
+	wg->creating_net = src_net;
+	init_rwsem(&wg->static_identity.lock);
+	mutex_init(&wg->socket_update_lock);
+	mutex_init(&wg->device_update_lock);
+	skb_queue_head_init(&wg->incoming_handshakes);
+	wg_allowedips_init(&wg->peer_allowedips);
+	wg_cookie_checker_init(&wg->cookie_checker, wg);
+	INIT_LIST_HEAD(&wg->peer_list);
+	wg->device_update_gen = 1;
+
+	wg->peer_hashtable = wg_pubkey_hashtable_alloc();
+	if (!wg->peer_hashtable)
+		return ret;
+
+	wg->index_hashtable = wg_index_hashtable_alloc();
+	if (!wg->index_hashtable)
+		goto err_free_peer_hashtable;
+
+	dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
+	if (!dev->tstats)
+		goto err_free_index_hashtable;
+
+	wg->incoming_handshakes_worker =
+		wg_packet_percpu_multicore_worker_alloc(
+				wg_packet_handshake_receive_worker, wg);
+	if (!wg->incoming_handshakes_worker)
+		goto err_free_tstats;
+
+	wg->handshake_receive_wq = alloc_workqueue("wg-kex-%s",
+			WQ_CPU_INTENSIVE | WQ_FREEZABLE, 0, dev->name);
+	if (!wg->handshake_receive_wq)
+		goto err_free_incoming_handshakes;
+
+	wg->handshake_send_wq = alloc_workqueue("wg-kex-%s",
+			WQ_UNBOUND | WQ_FREEZABLE, 0, dev->name);
+	if (!wg->handshake_send_wq)
+		goto err_destroy_handshake_receive;
+
+	wg->packet_crypt_wq = alloc_workqueue("wg-crypt-%s",
+			WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 0, dev->name);
+	if (!wg->packet_crypt_wq)
+		goto err_destroy_handshake_send;
+
+	ret = wg_packet_queue_init(&wg->encrypt_queue, wg_packet_encrypt_worker,
+				   true, MAX_QUEUED_PACKETS);
+	if (ret < 0)
+		goto err_destroy_packet_crypt;
+
+	ret = wg_packet_queue_init(&wg->decrypt_queue, wg_packet_decrypt_worker,
+				   true, MAX_QUEUED_PACKETS);
+	if (ret < 0)
+		goto err_free_encrypt_queue;
+
+	ret = wg_ratelimiter_init();
+	if (ret < 0)
+		goto err_free_decrypt_queue;
+
+	ret = register_netdevice(dev);
+	if (ret < 0)
+		goto err_uninit_ratelimiter;
+
+	list_add(&wg->device_list, &device_list);
+
+	/* We wait until the end to assign priv_destructor, so that
+	 * register_netdevice doesn't call it for us if it fails.
+	 */
+	dev->priv_destructor = wg_destruct;
+
+	pr_debug("%s: Interface created\n", dev->name);
+	return ret;
+
+err_uninit_ratelimiter:
+	wg_ratelimiter_uninit();
+err_free_decrypt_queue:
+	wg_packet_queue_free(&wg->decrypt_queue, true);
+err_free_encrypt_queue:
+	wg_packet_queue_free(&wg->encrypt_queue, true);
+err_destroy_packet_crypt:
+	destroy_workqueue(wg->packet_crypt_wq);
+err_destroy_handshake_send:
+	destroy_workqueue(wg->handshake_send_wq);
+err_destroy_handshake_receive:
+	destroy_workqueue(wg->handshake_receive_wq);
+err_free_incoming_handshakes:
+	free_percpu(wg->incoming_handshakes_worker);
+err_free_tstats:
+	free_percpu(dev->tstats);
+err_free_index_hashtable:
+	kvfree(wg->index_hashtable);
+err_free_peer_hashtable:
+	kvfree(wg->peer_hashtable);
+	return ret;
+}
+
+static struct rtnl_link_ops link_ops __read_mostly = {
+	.kind			= KBUILD_MODNAME,
+	.priv_size		= sizeof(struct wg_device),
+	.setup			= wg_setup,
+	.newlink		= wg_newlink,
+};
+
+static int wg_netdevice_notification(struct notifier_block *nb,
+				     unsigned long action, void *data)
+{
+	struct net_device *dev = ((struct netdev_notifier_info *)data)->dev;
+	struct wg_device *wg = netdev_priv(dev);
+
+	ASSERT_RTNL();
+
+	if (action != NETDEV_REGISTER || dev->netdev_ops != &netdev_ops)
+		return 0;
+
+	if (dev_net(dev) == wg->creating_net && wg->have_creating_net_ref) {
+		put_net(wg->creating_net);
+		wg->have_creating_net_ref = false;
+	} else if (dev_net(dev) != wg->creating_net &&
+		   !wg->have_creating_net_ref) {
+		wg->have_creating_net_ref = true;
+		get_net(wg->creating_net);
+	}
+	return 0;
+}
+
+static struct notifier_block netdevice_notifier = {
+	.notifier_call = wg_netdevice_notification
+};
+
+int __init wg_device_init(void)
+{
+	int ret;
+
+#ifdef CONFIG_PM_SLEEP
+	ret = register_pm_notifier(&pm_notifier);
+	if (ret)
+		return ret;
+#endif
+
+	ret = register_netdevice_notifier(&netdevice_notifier);
+	if (ret)
+		goto error_pm;
+
+	ret = rtnl_link_register(&link_ops);
+	if (ret)
+		goto error_netdevice;
+
+	return 0;
+
+error_netdevice:
+	unregister_netdevice_notifier(&netdevice_notifier);
+error_pm:
+#ifdef CONFIG_PM_SLEEP
+	unregister_pm_notifier(&pm_notifier);
+#endif
+	return ret;
+}
+
+void wg_device_uninit(void)
+{
+	rtnl_link_unregister(&link_ops);
+	unregister_netdevice_notifier(&netdevice_notifier);
+#ifdef CONFIG_PM_SLEEP
+	unregister_pm_notifier(&pm_notifier);
+#endif
+	rcu_barrier();
+}
diff -Naur --no-dereference a/net/wireguard/device.h b/net/wireguard/device.h
--- a/net/wireguard/device.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/device.h	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,73 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#ifndef _WG_DEVICE_H
+#define _WG_DEVICE_H
+
+#include "noise.h"
+#include "allowedips.h"
+#include "peerlookup.h"
+#include "cookie.h"
+
+#include <linux/types.h>
+#include <linux/netdevice.h>
+#include <linux/workqueue.h>
+#include <linux/mutex.h>
+#include <linux/net.h>
+#include <linux/ptr_ring.h>
+
+struct wg_device;
+
+struct multicore_worker {
+	void *ptr;
+	struct work_struct work;
+};
+
+struct crypt_queue {
+	struct ptr_ring ring;
+	union {
+		struct {
+			struct multicore_worker __percpu *worker;
+			int last_cpu;
+		};
+		struct work_struct work;
+	};
+};
+
+struct wg_device {
+	struct net_device *dev;
+	struct crypt_queue encrypt_queue, decrypt_queue;
+	struct sock __rcu *sock4, *sock6;
+	struct net *creating_net;
+	struct noise_static_identity static_identity;
+	struct workqueue_struct *handshake_receive_wq, *handshake_send_wq;
+	struct workqueue_struct *packet_crypt_wq;
+	struct sk_buff_head incoming_handshakes;
+	int incoming_handshake_cpu;
+	struct multicore_worker __percpu *incoming_handshakes_worker;
+	struct cookie_checker cookie_checker;
+	struct pubkey_hashtable *peer_hashtable;
+	struct index_hashtable *index_hashtable;
+	struct allowedips peer_allowedips;
+	struct mutex device_update_lock, socket_update_lock;
+	struct list_head device_list, peer_list;
+	unsigned int num_peers, device_update_gen;
+	u32 fwmark;
+	u16 incoming_port;
+	bool have_creating_net_ref;
+};
+
+int wg_device_init(void);
+void wg_device_uninit(void);
+
+/* Later after the dust settles, this can be moved into include/linux/skbuff.h,
+ * where virtually all code that deals with GSO segs can benefit, around ~30
+ * drivers as of writing.
+ */
+#define skb_list_walk_safe(first, skb, next)                                   \
+	for (skb = first, next = skb->next; skb;                               \
+	     skb = next, next = skb ? skb->next : NULL)
+
+#endif /* _WG_DEVICE_H */
diff -Naur --no-dereference a/net/wireguard/Kconfig b/net/wireguard/Kconfig
--- a/net/wireguard/Kconfig	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/Kconfig	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,33 @@
+config WIREGUARD
+	tristate "IP: WireGuard secure network tunnel"
+	depends on NET && INET
+	depends on IPV6 || !IPV6
+	select NET_UDP_TUNNEL
+	select DST_CACHE
+	select CRYPTO
+	select CRYPTO_ALGAPI
+	select VFP
+	select VFPv3 if CPU_V7
+	select NEON if CPU_V7
+	select KERNEL_MODE_NEON if CPU_V7
+	default m
+	help
+	  WireGuard is a secure, fast, and easy to use replacement for IPsec
+	  that uses modern cryptography and clever networking tricks. It's
+	  designed to be fairly general purpose and abstract enough to fit most
+	  use cases, while at the same time remaining extremely simple to
+	  configure. See www.wireguard.com for more info.
+
+	  It's safe to say Y or M here, as the driver is very lightweight and
+	  is only in use when an administrator chooses to add an interface.
+
+config WIREGUARD_DEBUG
+	bool "Debugging checks and verbose messages"
+	depends on WIREGUARD
+	help
+	  This will write log messages for handshake and other events
+	  that occur for a WireGuard interface. It will also perform some
+	  extra validation checks and unit tests at various points. This is
+	  only useful for debugging.
+
+	  Say N here unless you know what you're doing.
diff -Naur --no-dereference a/net/wireguard/main.c b/net/wireguard/main.c
--- a/net/wireguard/main.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/main.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#include "version.h"
+#include "device.h"
+#include "noise.h"
+#include "queueing.h"
+#include "ratelimiter.h"
+#include "netlink.h"
+#include "uapi/wireguard.h"
+#include "crypto/zinc.h"
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/genetlink.h>
+#include <net/rtnetlink.h>
+
+static int __init mod_init(void)
+{
+	int ret;
+
+	if ((ret = chacha20_mod_init()) || (ret = poly1305_mod_init()) ||
+	    (ret = chacha20poly1305_mod_init()) || (ret = blake2s_mod_init()) ||
+	    (ret = curve25519_mod_init()))
+		return ret;
+
+#ifdef DEBUG
+	if (!wg_allowedips_selftest() || !wg_packet_counter_selftest() ||
+	    !wg_ratelimiter_selftest())
+		return -ENOTRECOVERABLE;
+#endif
+	wg_noise_init();
+
+	ret = wg_device_init();
+	if (ret < 0)
+		goto err_device;
+
+	ret = wg_genetlink_init();
+	if (ret < 0)
+		goto err_netlink;
+
+	pr_info("WireGuard " WIREGUARD_VERSION " loaded. See www.wireguard.com for information.\n");
+	pr_info("Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.\n");
+
+	return 0;
+
+err_netlink:
+	wg_device_uninit();
+err_device:
+	return ret;
+}
+
+static void __exit mod_exit(void)
+{
+	wg_genetlink_uninit();
+	wg_device_uninit();
+}
+
+module_init(mod_init);
+module_exit(mod_exit);
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("WireGuard secure network tunnel");
+MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");
+MODULE_VERSION(WIREGUARD_VERSION);
+MODULE_ALIAS_RTNL_LINK(KBUILD_MODNAME);
+MODULE_ALIAS_GENL_FAMILY(WG_GENL_NAME);
diff -Naur --no-dereference a/net/wireguard/Makefile b/net/wireguard/Makefile
--- a/net/wireguard/Makefile	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/Makefile	2020-06-19 21:00:31.432227716 -0400
@@ -0,0 +1,15 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+
+ccflags-y := -O3 -fvisibility=hidden
+ccflags-$(CONFIG_WIREGUARD_DEBUG) += -DDEBUG -g
+ccflags-y += -D'pr_fmt(fmt)=KBUILD_MODNAME ": " fmt'
+ccflags-y += -Wframe-larger-than=2048
+
+wireguard-y := main.o noise.o device.o peer.o timers.o queueing.o send.o receive.o socket.o peerlookup.o allowedips.o ratelimiter.o cookie.o netlink.o
+
+include $(src)/crypto/Makefile.include
+include $(src)/compat/Makefile.include
+
+obj-$(if $(KBUILD_EXTMOD),m,$(CONFIG_WIREGUARD)) := wireguard.o
diff -Naur --no-dereference a/net/wireguard/messages.h b/net/wireguard/messages.h
--- a/net/wireguard/messages.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/messages.h	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,128 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#ifndef _WG_MESSAGES_H
+#define _WG_MESSAGES_H
+
+#include <zinc/curve25519.h>
+#include <zinc/chacha20poly1305.h>
+#include <zinc/blake2s.h>
+
+#include <linux/kernel.h>
+#include <linux/param.h>
+#include <linux/skbuff.h>
+
+enum noise_lengths {
+	NOISE_PUBLIC_KEY_LEN = CURVE25519_KEY_SIZE,
+	NOISE_SYMMETRIC_KEY_LEN = CHACHA20POLY1305_KEY_SIZE,
+	NOISE_TIMESTAMP_LEN = sizeof(u64) + sizeof(u32),
+	NOISE_AUTHTAG_LEN = CHACHA20POLY1305_AUTHTAG_SIZE,
+	NOISE_HASH_LEN = BLAKE2S_HASH_SIZE
+};
+
+#define noise_encrypted_len(plain_len) ((plain_len) + NOISE_AUTHTAG_LEN)
+
+enum cookie_values {
+	COOKIE_SECRET_MAX_AGE = 2 * 60,
+	COOKIE_SECRET_LATENCY = 5,
+	COOKIE_NONCE_LEN = XCHACHA20POLY1305_NONCE_SIZE,
+	COOKIE_LEN = 16
+};
+
+enum counter_values {
+	COUNTER_BITS_TOTAL = 2048,
+	COUNTER_REDUNDANT_BITS = BITS_PER_LONG,
+	COUNTER_WINDOW_SIZE = COUNTER_BITS_TOTAL - COUNTER_REDUNDANT_BITS
+};
+
+enum limits {
+	REKEY_AFTER_MESSAGES = 1ULL << 60,
+	REJECT_AFTER_MESSAGES = U64_MAX - COUNTER_WINDOW_SIZE - 1,
+	REKEY_TIMEOUT = 5,
+	REKEY_TIMEOUT_JITTER_MAX_JIFFIES = HZ / 3,
+	REKEY_AFTER_TIME = 120,
+	REJECT_AFTER_TIME = 180,
+	INITIATIONS_PER_SECOND = 50,
+	MAX_PEERS_PER_DEVICE = 1U << 20,
+	KEEPALIVE_TIMEOUT = 10,
+	MAX_TIMER_HANDSHAKES = 90 / REKEY_TIMEOUT,
+	MAX_QUEUED_INCOMING_HANDSHAKES = 4096, /* TODO: replace this with DQL */
+	MAX_STAGED_PACKETS = 128,
+	MAX_QUEUED_PACKETS = 1024 /* TODO: replace this with DQL */
+};
+
+enum message_type {
+	MESSAGE_INVALID = 0,
+	MESSAGE_HANDSHAKE_INITIATION = 1,
+	MESSAGE_HANDSHAKE_RESPONSE = 2,
+	MESSAGE_HANDSHAKE_COOKIE = 3,
+	MESSAGE_DATA = 4
+};
+
+struct message_header {
+	/* The actual layout of this that we want is:
+	 * u8 type
+	 * u8 reserved_zero[3]
+	 *
+	 * But it turns out that by encoding this as little endian,
+	 * we achieve the same thing, and it makes checking faster.
+	 */
+	__le32 type;
+};
+
+struct message_macs {
+	u8 mac1[COOKIE_LEN];
+	u8 mac2[COOKIE_LEN];
+};
+
+struct message_handshake_initiation {
+	struct message_header header;
+	__le32 sender_index;
+	u8 unencrypted_ephemeral[NOISE_PUBLIC_KEY_LEN];
+	u8 encrypted_static[noise_encrypted_len(NOISE_PUBLIC_KEY_LEN)];
+	u8 encrypted_timestamp[noise_encrypted_len(NOISE_TIMESTAMP_LEN)];
+	struct message_macs macs;
+};
+
+struct message_handshake_response {
+	struct message_header header;
+	__le32 sender_index;
+	__le32 receiver_index;
+	u8 unencrypted_ephemeral[NOISE_PUBLIC_KEY_LEN];
+	u8 encrypted_nothing[noise_encrypted_len(0)];
+	struct message_macs macs;
+};
+
+struct message_handshake_cookie {
+	struct message_header header;
+	__le32 receiver_index;
+	u8 nonce[COOKIE_NONCE_LEN];
+	u8 encrypted_cookie[noise_encrypted_len(COOKIE_LEN)];
+};
+
+struct message_data {
+	struct message_header header;
+	__le32 key_idx;
+	__le64 counter;
+	u8 encrypted_data[];
+};
+
+#define message_data_len(plain_len) \
+	(noise_encrypted_len(plain_len) + sizeof(struct message_data))
+
+enum message_alignments {
+	MESSAGE_PADDING_MULTIPLE = 16,
+	MESSAGE_MINIMUM_LENGTH = message_data_len(0)
+};
+
+#define SKB_HEADER_LEN                                       \
+	(max(sizeof(struct iphdr), sizeof(struct ipv6hdr)) + \
+	 sizeof(struct udphdr) + NET_SKB_PAD)
+#define DATA_PACKET_HEAD_ROOM \
+	ALIGN(sizeof(struct message_data) + SKB_HEADER_LEN, 4)
+
+enum { HANDSHAKE_DSCP = 0x88 /* AF41, plus 00 ECN */ };
+
+#endif /* _WG_MESSAGES_H */
diff -Naur --no-dereference a/net/wireguard/netlink.c b/net/wireguard/netlink.c
--- a/net/wireguard/netlink.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/netlink.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,660 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#include "netlink.h"
+#include "device.h"
+#include "peer.h"
+#include "socket.h"
+#include "queueing.h"
+#include "messages.h"
+#include "uapi/wireguard.h"
+#include <linux/if.h>
+#include <net/genetlink.h>
+#include <net/sock.h>
+#include <crypto/algapi.h>
+
+static struct genl_family genl_family;
+
+static const struct nla_policy device_policy[WGDEVICE_A_MAX + 1] = {
+	[WGDEVICE_A_IFINDEX]		= { .type = NLA_U32 },
+	[WGDEVICE_A_IFNAME]		= { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
+	[WGDEVICE_A_PRIVATE_KEY]	= { .type = NLA_EXACT_LEN, .len = NOISE_PUBLIC_KEY_LEN },
+	[WGDEVICE_A_PUBLIC_KEY]		= { .type = NLA_EXACT_LEN, .len = NOISE_PUBLIC_KEY_LEN },
+	[WGDEVICE_A_FLAGS]		= { .type = NLA_U32 },
+	[WGDEVICE_A_LISTEN_PORT]	= { .type = NLA_U16 },
+	[WGDEVICE_A_FWMARK]		= { .type = NLA_U32 },
+	[WGDEVICE_A_PEERS]		= { .type = NLA_NESTED }
+};
+
+static const struct nla_policy peer_policy[WGPEER_A_MAX + 1] = {
+	[WGPEER_A_PUBLIC_KEY]				= { .type = NLA_EXACT_LEN, .len = NOISE_PUBLIC_KEY_LEN },
+	[WGPEER_A_PRESHARED_KEY]			= { .type = NLA_EXACT_LEN, .len = NOISE_SYMMETRIC_KEY_LEN },
+	[WGPEER_A_FLAGS]				= { .type = NLA_U32 },
+	[WGPEER_A_ENDPOINT]				= { .type = NLA_MIN_LEN, .len = sizeof(struct sockaddr) },
+	[WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL]	= { .type = NLA_U16 },
+	[WGPEER_A_LAST_HANDSHAKE_TIME]			= { .type = NLA_EXACT_LEN, .len = sizeof(struct __kernel_timespec) },
+	[WGPEER_A_RX_BYTES]				= { .type = NLA_U64 },
+	[WGPEER_A_TX_BYTES]				= { .type = NLA_U64 },
+	[WGPEER_A_ALLOWEDIPS]				= { .type = NLA_NESTED },
+	[WGPEER_A_PROTOCOL_VERSION]			= { .type = NLA_U32 }
+};
+
+static const struct nla_policy allowedip_policy[WGALLOWEDIP_A_MAX + 1] = {
+	[WGALLOWEDIP_A_FAMILY]		= { .type = NLA_U16 },
+	[WGALLOWEDIP_A_IPADDR]		= { .type = NLA_MIN_LEN, .len = sizeof(struct in_addr) },
+	[WGALLOWEDIP_A_CIDR_MASK]	= { .type = NLA_U8 }
+};
+
+static struct wg_device *lookup_interface(struct nlattr **attrs,
+					  struct sk_buff *skb)
+{
+	struct net_device *dev = NULL;
+
+	if (!attrs[WGDEVICE_A_IFINDEX] == !attrs[WGDEVICE_A_IFNAME])
+		return ERR_PTR(-EBADR);
+	if (attrs[WGDEVICE_A_IFINDEX])
+		dev = dev_get_by_index(sock_net(skb->sk),
+				       nla_get_u32(attrs[WGDEVICE_A_IFINDEX]));
+	else if (attrs[WGDEVICE_A_IFNAME])
+		dev = dev_get_by_name(sock_net(skb->sk),
+				      nla_data(attrs[WGDEVICE_A_IFNAME]));
+	if (!dev)
+		return ERR_PTR(-ENODEV);
+	if (!dev->rtnl_link_ops || !dev->rtnl_link_ops->kind ||
+	    strcmp(dev->rtnl_link_ops->kind, KBUILD_MODNAME)) {
+		dev_put(dev);
+		return ERR_PTR(-EOPNOTSUPP);
+	}
+	return netdev_priv(dev);
+}
+
+static int get_allowedips(struct sk_buff *skb, const u8 *ip, u8 cidr,
+			  int family)
+{
+	struct nlattr *allowedip_nest;
+
+	allowedip_nest = nla_nest_start(skb, 0);
+	if (!allowedip_nest)
+		return -EMSGSIZE;
+
+	if (nla_put_u8(skb, WGALLOWEDIP_A_CIDR_MASK, cidr) ||
+	    nla_put_u16(skb, WGALLOWEDIP_A_FAMILY, family) ||
+	    nla_put(skb, WGALLOWEDIP_A_IPADDR, family == AF_INET6 ?
+		    sizeof(struct in6_addr) : sizeof(struct in_addr), ip)) {
+		nla_nest_cancel(skb, allowedip_nest);
+		return -EMSGSIZE;
+	}
+
+	nla_nest_end(skb, allowedip_nest);
+	return 0;
+}
+
+struct dump_ctx {
+	struct wg_device *wg;
+	struct wg_peer *next_peer;
+	u64 allowedips_seq;
+	struct allowedips_node *next_allowedip;
+};
+
+#define DUMP_CTX(cb) ((struct dump_ctx *)(cb)->args)
+
+static int
+get_peer(struct wg_peer *peer, struct sk_buff *skb, struct dump_ctx *ctx)
+{
+
+	struct nlattr *allowedips_nest, *peer_nest = nla_nest_start(skb, 0);
+	struct allowedips_node *allowedips_node = ctx->next_allowedip;
+	bool fail;
+
+	if (!peer_nest)
+		return -EMSGSIZE;
+
+	down_read(&peer->handshake.lock);
+	fail = nla_put(skb, WGPEER_A_PUBLIC_KEY, NOISE_PUBLIC_KEY_LEN,
+		       peer->handshake.remote_static);
+	up_read(&peer->handshake.lock);
+	if (fail)
+		goto err;
+
+	if (!allowedips_node) {
+		const struct __kernel_timespec last_handshake = {
+			.tv_sec = peer->walltime_last_handshake.tv_sec,
+			.tv_nsec = peer->walltime_last_handshake.tv_nsec
+		};
+
+		down_read(&peer->handshake.lock);
+		fail = nla_put(skb, WGPEER_A_PRESHARED_KEY,
+			       NOISE_SYMMETRIC_KEY_LEN,
+			       peer->handshake.preshared_key);
+		up_read(&peer->handshake.lock);
+		if (fail)
+			goto err;
+
+		if (nla_put(skb, WGPEER_A_LAST_HANDSHAKE_TIME,
+			    sizeof(last_handshake), &last_handshake) ||
+		    nla_put_u16(skb, WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL,
+				peer->persistent_keepalive_interval) ||
+		    nla_put_u64_64bit(skb, WGPEER_A_TX_BYTES, peer->tx_bytes,
+				      WGPEER_A_UNSPEC) ||
+		    nla_put_u64_64bit(skb, WGPEER_A_RX_BYTES, peer->rx_bytes,
+				      WGPEER_A_UNSPEC) ||
+		    nla_put_u32(skb, WGPEER_A_PROTOCOL_VERSION, 1))
+			goto err;
+
+		read_lock_bh(&peer->endpoint_lock);
+		if (peer->endpoint.addr.sa_family == AF_INET)
+			fail = nla_put(skb, WGPEER_A_ENDPOINT,
+				       sizeof(peer->endpoint.addr4),
+				       &peer->endpoint.addr4);
+		else if (peer->endpoint.addr.sa_family == AF_INET6)
+			fail = nla_put(skb, WGPEER_A_ENDPOINT,
+				       sizeof(peer->endpoint.addr6),
+				       &peer->endpoint.addr6);
+		read_unlock_bh(&peer->endpoint_lock);
+		if (fail)
+			goto err;
+		allowedips_node =
+			list_first_entry_or_null(&peer->allowedips_list,
+					struct allowedips_node, peer_list);
+	}
+	if (!allowedips_node)
+		goto no_allowedips;
+	if (!ctx->allowedips_seq)
+		ctx->allowedips_seq = peer->device->peer_allowedips.seq;
+	else if (ctx->allowedips_seq != peer->device->peer_allowedips.seq)
+		goto no_allowedips;
+
+	allowedips_nest = nla_nest_start(skb, WGPEER_A_ALLOWEDIPS);
+	if (!allowedips_nest)
+		goto err;
+
+	list_for_each_entry_from(allowedips_node, &peer->allowedips_list,
+				 peer_list) {
+		u8 cidr, ip[16] __aligned(__alignof(u64));
+		int family;
+
+		family = wg_allowedips_read_node(allowedips_node, ip, &cidr);
+		if (get_allowedips(skb, ip, cidr, family)) {
+			nla_nest_end(skb, allowedips_nest);
+			nla_nest_end(skb, peer_nest);
+			ctx->next_allowedip = allowedips_node;
+			return -EMSGSIZE;
+		}
+	}
+	nla_nest_end(skb, allowedips_nest);
+no_allowedips:
+	nla_nest_end(skb, peer_nest);
+	ctx->next_allowedip = NULL;
+	ctx->allowedips_seq = 0;
+	return 0;
+err:
+	nla_nest_cancel(skb, peer_nest);
+	return -EMSGSIZE;
+}
+
+static int wg_get_device_start(struct netlink_callback *cb)
+{
+	struct wg_device *wg;
+
+	wg = lookup_interface(genl_dumpit_info(cb)->attrs, cb->skb);
+	if (IS_ERR(wg))
+		return PTR_ERR(wg);
+	DUMP_CTX(cb)->wg = wg;
+	return 0;
+}
+
+static int wg_get_device_dump(struct sk_buff *skb, struct netlink_callback *cb)
+{
+	struct wg_peer *peer, *next_peer_cursor;
+	struct dump_ctx *ctx = DUMP_CTX(cb);
+	struct wg_device *wg = ctx->wg;
+	struct nlattr *peers_nest;
+	int ret = -EMSGSIZE;
+	bool done = true;
+	void *hdr;
+
+	rtnl_lock();
+	mutex_lock(&wg->device_update_lock);
+	cb->seq = wg->device_update_gen;
+	next_peer_cursor = ctx->next_peer;
+
+	hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
+			  &genl_family, NLM_F_MULTI, WG_CMD_GET_DEVICE);
+	if (!hdr)
+		goto out;
+	genl_dump_check_consistent(cb, hdr);
+
+	if (!ctx->next_peer) {
+		if (nla_put_u16(skb, WGDEVICE_A_LISTEN_PORT,
+				wg->incoming_port) ||
+		    nla_put_u32(skb, WGDEVICE_A_FWMARK, wg->fwmark) ||
+		    nla_put_u32(skb, WGDEVICE_A_IFINDEX, wg->dev->ifindex) ||
+		    nla_put_string(skb, WGDEVICE_A_IFNAME, wg->dev->name))
+			goto out;
+
+		down_read(&wg->static_identity.lock);
+		if (wg->static_identity.has_identity) {
+			if (nla_put(skb, WGDEVICE_A_PRIVATE_KEY,
+				    NOISE_PUBLIC_KEY_LEN,
+				    wg->static_identity.static_private) ||
+			    nla_put(skb, WGDEVICE_A_PUBLIC_KEY,
+				    NOISE_PUBLIC_KEY_LEN,
+				    wg->static_identity.static_public)) {
+				up_read(&wg->static_identity.lock);
+				goto out;
+			}
+		}
+		up_read(&wg->static_identity.lock);
+	}
+
+	peers_nest = nla_nest_start(skb, WGDEVICE_A_PEERS);
+	if (!peers_nest)
+		goto out;
+	ret = 0;
+	/* If the last cursor was removed via list_del_init in peer_remove, then
+	 * we just treat this the same as there being no more peers left. The
+	 * reason is that seq_nr should indicate to userspace that this isn't a
+	 * coherent dump anyway, so they'll try again.
+	 */
+	if (list_empty(&wg->peer_list) ||
+	    (ctx->next_peer && list_empty(&ctx->next_peer->peer_list))) {
+		nla_nest_cancel(skb, peers_nest);
+		goto out;
+	}
+	lockdep_assert_held(&wg->device_update_lock);
+	peer = list_prepare_entry(ctx->next_peer, &wg->peer_list, peer_list);
+	list_for_each_entry_continue(peer, &wg->peer_list, peer_list) {
+		if (get_peer(peer, skb, ctx)) {
+			done = false;
+			break;
+		}
+		next_peer_cursor = peer;
+	}
+	nla_nest_end(skb, peers_nest);
+
+out:
+	if (!ret && !done && next_peer_cursor)
+		wg_peer_get(next_peer_cursor);
+	wg_peer_put(ctx->next_peer);
+	mutex_unlock(&wg->device_update_lock);
+	rtnl_unlock();
+
+	if (ret) {
+		genlmsg_cancel(skb, hdr);
+		return ret;
+	}
+	genlmsg_end(skb, hdr);
+	if (done) {
+		ctx->next_peer = NULL;
+		return 0;
+	}
+	ctx->next_peer = next_peer_cursor;
+	return skb->len;
+
+	/* At this point, we can't really deal ourselves with safely zeroing out
+	 * the private key material after usage. This will need an additional API
+	 * in the kernel for marking skbs as zero_on_free.
+	 */
+}
+
+static int wg_get_device_done(struct netlink_callback *cb)
+{
+	struct dump_ctx *ctx = DUMP_CTX(cb);
+
+	if (ctx->wg)
+		dev_put(ctx->wg->dev);
+	wg_peer_put(ctx->next_peer);
+	return 0;
+}
+
+static int set_port(struct wg_device *wg, u16 port)
+{
+	struct wg_peer *peer;
+
+	if (wg->incoming_port == port)
+		return 0;
+	list_for_each_entry(peer, &wg->peer_list, peer_list)
+		wg_socket_clear_peer_endpoint_src(peer);
+	if (!netif_running(wg->dev)) {
+		wg->incoming_port = port;
+		return 0;
+	}
+	return wg_socket_init(wg, port);
+}
+
+static int set_allowedip(struct wg_peer *peer, struct nlattr **attrs)
+{
+	int ret = -EINVAL;
+	u16 family;
+	u8 cidr;
+
+	if (!attrs[WGALLOWEDIP_A_FAMILY] || !attrs[WGALLOWEDIP_A_IPADDR] ||
+	    !attrs[WGALLOWEDIP_A_CIDR_MASK])
+		return ret;
+	family = nla_get_u16(attrs[WGALLOWEDIP_A_FAMILY]);
+	cidr = nla_get_u8(attrs[WGALLOWEDIP_A_CIDR_MASK]);
+
+	if (family == AF_INET && cidr <= 32 &&
+	    nla_len(attrs[WGALLOWEDIP_A_IPADDR]) == sizeof(struct in_addr))
+		ret = wg_allowedips_insert_v4(
+			&peer->device->peer_allowedips,
+			nla_data(attrs[WGALLOWEDIP_A_IPADDR]), cidr, peer,
+			&peer->device->device_update_lock);
+	else if (family == AF_INET6 && cidr <= 128 &&
+		 nla_len(attrs[WGALLOWEDIP_A_IPADDR]) == sizeof(struct in6_addr))
+		ret = wg_allowedips_insert_v6(
+			&peer->device->peer_allowedips,
+			nla_data(attrs[WGALLOWEDIP_A_IPADDR]), cidr, peer,
+			&peer->device->device_update_lock);
+
+	return ret;
+}
+
+static int set_peer(struct wg_device *wg, struct nlattr **attrs)
+{
+	u8 *public_key = NULL, *preshared_key = NULL;
+	struct wg_peer *peer = NULL;
+	u32 flags = 0;
+	int ret;
+
+	ret = -EINVAL;
+	if (attrs[WGPEER_A_PUBLIC_KEY] &&
+	    nla_len(attrs[WGPEER_A_PUBLIC_KEY]) == NOISE_PUBLIC_KEY_LEN)
+		public_key = nla_data(attrs[WGPEER_A_PUBLIC_KEY]);
+	else
+		goto out;
+	if (attrs[WGPEER_A_PRESHARED_KEY] &&
+	    nla_len(attrs[WGPEER_A_PRESHARED_KEY]) == NOISE_SYMMETRIC_KEY_LEN)
+		preshared_key = nla_data(attrs[WGPEER_A_PRESHARED_KEY]);
+
+	if (attrs[WGPEER_A_FLAGS])
+		flags = nla_get_u32(attrs[WGPEER_A_FLAGS]);
+	ret = -EOPNOTSUPP;
+	if (flags & ~__WGPEER_F_ALL)
+		goto out;
+
+	ret = -EPFNOSUPPORT;
+	if (attrs[WGPEER_A_PROTOCOL_VERSION]) {
+		if (nla_get_u32(attrs[WGPEER_A_PROTOCOL_VERSION]) != 1)
+			goto out;
+	}
+
+	peer = wg_pubkey_hashtable_lookup(wg->peer_hashtable,
+					  nla_data(attrs[WGPEER_A_PUBLIC_KEY]));
+	ret = 0;
+	if (!peer) { /* Peer doesn't exist yet. Add a new one. */
+		if (flags & (WGPEER_F_REMOVE_ME | WGPEER_F_UPDATE_ONLY))
+			goto out;
+
+		/* The peer is new, so there aren't allowed IPs to remove. */
+		flags &= ~WGPEER_F_REPLACE_ALLOWEDIPS;
+
+		down_read(&wg->static_identity.lock);
+		if (wg->static_identity.has_identity &&
+		    !memcmp(nla_data(attrs[WGPEER_A_PUBLIC_KEY]),
+			    wg->static_identity.static_public,
+			    NOISE_PUBLIC_KEY_LEN)) {
+			/* We silently ignore peers that have the same public
+			 * key as the device. The reason we do it silently is
+			 * that we'd like for people to be able to reuse the
+			 * same set of API calls across peers.
+			 */
+			up_read(&wg->static_identity.lock);
+			ret = 0;
+			goto out;
+		}
+		up_read(&wg->static_identity.lock);
+
+		peer = wg_peer_create(wg, public_key, preshared_key);
+		if (IS_ERR(peer)) {
+			/* Similar to the above, if the key is invalid, we skip
+			 * it without fanfare, so that services don't need to
+			 * worry about doing key validation themselves.
+			 */
+			ret = PTR_ERR(peer) == -EKEYREJECTED ? 0 : PTR_ERR(peer);
+			peer = NULL;
+			goto out;
+		}
+		/* Take additional reference, as though we've just been
+		 * looked up.
+		 */
+		wg_peer_get(peer);
+	}
+
+	if (flags & WGPEER_F_REMOVE_ME) {
+		wg_peer_remove(peer);
+		goto out;
+	}
+
+	if (preshared_key) {
+		down_write(&peer->handshake.lock);
+		memcpy(&peer->handshake.preshared_key, preshared_key,
+		       NOISE_SYMMETRIC_KEY_LEN);
+		up_write(&peer->handshake.lock);
+	}
+
+	if (attrs[WGPEER_A_ENDPOINT]) {
+		struct sockaddr *addr = nla_data(attrs[WGPEER_A_ENDPOINT]);
+		size_t len = nla_len(attrs[WGPEER_A_ENDPOINT]);
+
+		if ((len == sizeof(struct sockaddr_in) &&
+		     addr->sa_family == AF_INET) ||
+		    (len == sizeof(struct sockaddr_in6) &&
+		     addr->sa_family == AF_INET6)) {
+			struct endpoint endpoint = { { { 0 } } };
+
+			memcpy(&endpoint.addr, addr, len);
+			wg_socket_set_peer_endpoint(peer, &endpoint);
+		}
+	}
+
+	if (flags & WGPEER_F_REPLACE_ALLOWEDIPS)
+		wg_allowedips_remove_by_peer(&wg->peer_allowedips, peer,
+					     &wg->device_update_lock);
+
+	if (attrs[WGPEER_A_ALLOWEDIPS]) {
+		struct nlattr *attr, *allowedip[WGALLOWEDIP_A_MAX + 1];
+		int rem;
+
+		nla_for_each_nested(attr, attrs[WGPEER_A_ALLOWEDIPS], rem) {
+			ret = nla_parse_nested(allowedip, WGALLOWEDIP_A_MAX,
+					       attr, allowedip_policy, NULL);
+			if (ret < 0)
+				goto out;
+			ret = set_allowedip(peer, allowedip);
+			if (ret < 0)
+				goto out;
+		}
+	}
+
+	if (attrs[WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL]) {
+		const u16 persistent_keepalive_interval = nla_get_u16(
+				attrs[WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL]);
+		const bool send_keepalive =
+			!peer->persistent_keepalive_interval &&
+			persistent_keepalive_interval &&
+			netif_running(wg->dev);
+
+		peer->persistent_keepalive_interval = persistent_keepalive_interval;
+		if (send_keepalive)
+			wg_packet_send_keepalive(peer);
+	}
+
+	if (netif_running(wg->dev))
+		wg_packet_send_staged_packets(peer);
+
+out:
+	wg_peer_put(peer);
+	if (attrs[WGPEER_A_PRESHARED_KEY])
+		memzero_explicit(nla_data(attrs[WGPEER_A_PRESHARED_KEY]),
+				 nla_len(attrs[WGPEER_A_PRESHARED_KEY]));
+	return ret;
+}
+
+static int wg_set_device(struct sk_buff *skb, struct genl_info *info)
+{
+	struct wg_device *wg = lookup_interface(info->attrs, skb);
+	u32 flags = 0;
+	int ret;
+
+	if (IS_ERR(wg)) {
+		ret = PTR_ERR(wg);
+		goto out_nodev;
+	}
+
+	rtnl_lock();
+	mutex_lock(&wg->device_update_lock);
+
+	if (info->attrs[WGDEVICE_A_FLAGS])
+		flags = nla_get_u32(info->attrs[WGDEVICE_A_FLAGS]);
+	ret = -EOPNOTSUPP;
+	if (flags & ~__WGDEVICE_F_ALL)
+		goto out;
+
+	ret = -EPERM;
+	if ((info->attrs[WGDEVICE_A_LISTEN_PORT] ||
+	     info->attrs[WGDEVICE_A_FWMARK]) &&
+	    !ns_capable(wg->creating_net->user_ns, CAP_NET_ADMIN))
+		goto out;
+
+	++wg->device_update_gen;
+
+	if (info->attrs[WGDEVICE_A_FWMARK]) {
+		struct wg_peer *peer;
+
+		wg->fwmark = nla_get_u32(info->attrs[WGDEVICE_A_FWMARK]);
+		list_for_each_entry(peer, &wg->peer_list, peer_list)
+			wg_socket_clear_peer_endpoint_src(peer);
+	}
+
+	if (info->attrs[WGDEVICE_A_LISTEN_PORT]) {
+		ret = set_port(wg,
+			nla_get_u16(info->attrs[WGDEVICE_A_LISTEN_PORT]));
+		if (ret)
+			goto out;
+	}
+
+	if (flags & WGDEVICE_F_REPLACE_PEERS)
+		wg_peer_remove_all(wg);
+
+	if (info->attrs[WGDEVICE_A_PRIVATE_KEY] &&
+	    nla_len(info->attrs[WGDEVICE_A_PRIVATE_KEY]) ==
+		    NOISE_PUBLIC_KEY_LEN) {
+		u8 *private_key = nla_data(info->attrs[WGDEVICE_A_PRIVATE_KEY]);
+		u8 public_key[NOISE_PUBLIC_KEY_LEN];
+		struct wg_peer *peer, *temp;
+
+		if (!crypto_memneq(wg->static_identity.static_private,
+				   private_key, NOISE_PUBLIC_KEY_LEN))
+			goto skip_set_private_key;
+
+		/* We remove before setting, to prevent race, which means doing
+		 * two 25519-genpub ops.
+		 */
+		if (curve25519_generate_public(public_key, private_key)) {
+			peer = wg_pubkey_hashtable_lookup(wg->peer_hashtable,
+							  public_key);
+			if (peer) {
+				wg_peer_put(peer);
+				wg_peer_remove(peer);
+			}
+		}
+
+		down_write(&wg->static_identity.lock);
+		wg_noise_set_static_identity_private_key(&wg->static_identity,
+							 private_key);
+		list_for_each_entry_safe(peer, temp, &wg->peer_list,
+					 peer_list) {
+			if (wg_noise_precompute_static_static(peer))
+				wg_noise_expire_current_peer_keypairs(peer);
+			else
+				wg_peer_remove(peer);
+		}
+		wg_cookie_checker_precompute_device_keys(&wg->cookie_checker);
+		up_write(&wg->static_identity.lock);
+	}
+skip_set_private_key:
+
+	if (info->attrs[WGDEVICE_A_PEERS]) {
+		struct nlattr *attr, *peer[WGPEER_A_MAX + 1];
+		int rem;
+
+		nla_for_each_nested(attr, info->attrs[WGDEVICE_A_PEERS], rem) {
+			ret = nla_parse_nested(peer, WGPEER_A_MAX, attr,
+					       peer_policy, NULL);
+			if (ret < 0)
+				goto out;
+			ret = set_peer(wg, peer);
+			if (ret < 0)
+				goto out;
+		}
+	}
+	ret = 0;
+
+out:
+	mutex_unlock(&wg->device_update_lock);
+	rtnl_unlock();
+	dev_put(wg->dev);
+out_nodev:
+	if (info->attrs[WGDEVICE_A_PRIVATE_KEY])
+		memzero_explicit(nla_data(info->attrs[WGDEVICE_A_PRIVATE_KEY]),
+				 nla_len(info->attrs[WGDEVICE_A_PRIVATE_KEY]));
+	return ret;
+}
+
+#ifndef COMPAT_CANNOT_USE_CONST_GENL_OPS
+static const
+#else
+static
+#endif
+struct genl_ops genl_ops[] = {
+	{
+		.cmd = WG_CMD_GET_DEVICE,
+#ifndef COMPAT_CANNOT_USE_NETLINK_START
+		.start = wg_get_device_start,
+#endif
+		.dumpit = wg_get_device_dump,
+		.done = wg_get_device_done,
+#ifdef COMPAT_CANNOT_INDIVIDUAL_NETLINK_OPS_POLICY
+		.policy = device_policy,
+#endif
+		.flags = GENL_UNS_ADMIN_PERM
+	}, {
+		.cmd = WG_CMD_SET_DEVICE,
+		.doit = wg_set_device,
+#ifdef COMPAT_CANNOT_INDIVIDUAL_NETLINK_OPS_POLICY
+		.policy = device_policy,
+#endif
+		.flags = GENL_UNS_ADMIN_PERM
+	}
+};
+
+static struct genl_family genl_family
+#ifndef COMPAT_CANNOT_USE_GENL_NOPS
+__ro_after_init = {
+	.ops = genl_ops,
+	.n_ops = ARRAY_SIZE(genl_ops),
+#else
+= {
+#endif
+	.name = WG_GENL_NAME,
+	.version = WG_GENL_VERSION,
+	.maxattr = WGDEVICE_A_MAX,
+	.module = THIS_MODULE,
+#ifndef COMPAT_CANNOT_INDIVIDUAL_NETLINK_OPS_POLICY
+	.policy = device_policy,
+#endif
+	.netnsok = true
+};
+
+int __init wg_genetlink_init(void)
+{
+	return genl_register_family(&genl_family);
+}
+
+void __exit wg_genetlink_uninit(void)
+{
+	genl_unregister_family(&genl_family);
+}
diff -Naur --no-dereference a/net/wireguard/netlink.h b/net/wireguard/netlink.h
--- a/net/wireguard/netlink.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/netlink.h	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#ifndef _WG_NETLINK_H
+#define _WG_NETLINK_H
+
+int wg_genetlink_init(void);
+void wg_genetlink_uninit(void);
+
+#endif /* _WG_NETLINK_H */
diff -Naur --no-dereference a/net/wireguard/noise.c b/net/wireguard/noise.c
--- a/net/wireguard/noise.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/noise.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,830 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#include "noise.h"
+#include "device.h"
+#include "peer.h"
+#include "messages.h"
+#include "queueing.h"
+#include "peerlookup.h"
+
+#include <linux/rcupdate.h>
+#include <linux/slab.h>
+#include <linux/bitmap.h>
+#include <linux/scatterlist.h>
+#include <linux/highmem.h>
+#include <crypto/algapi.h>
+
+/* This implements Noise_IKpsk2:
+ *
+ * <- s
+ * ******
+ * -> e, es, s, ss, {t}
+ * <- e, ee, se, psk, {}
+ */
+
+static const u8 handshake_name[37] = "Noise_IKpsk2_25519_ChaChaPoly_BLAKE2s";
+static const u8 identifier_name[34] = "WireGuard v1 zx2c4 Jason@zx2c4.com";
+static u8 handshake_init_hash[NOISE_HASH_LEN] __ro_after_init;
+static u8 handshake_init_chaining_key[NOISE_HASH_LEN] __ro_after_init;
+static atomic64_t keypair_counter = ATOMIC64_INIT(0);
+
+void __init wg_noise_init(void)
+{
+	struct blake2s_state blake;
+
+	blake2s(handshake_init_chaining_key, handshake_name, NULL,
+		NOISE_HASH_LEN, sizeof(handshake_name), 0);
+	blake2s_init(&blake, NOISE_HASH_LEN);
+	blake2s_update(&blake, handshake_init_chaining_key, NOISE_HASH_LEN);
+	blake2s_update(&blake, identifier_name, sizeof(identifier_name));
+	blake2s_final(&blake, handshake_init_hash);
+}
+
+/* Must hold peer->handshake.static_identity->lock */
+bool wg_noise_precompute_static_static(struct wg_peer *peer)
+{
+	bool ret = true;
+
+	down_write(&peer->handshake.lock);
+	if (peer->handshake.static_identity->has_identity)
+		ret = curve25519(
+			peer->handshake.precomputed_static_static,
+			peer->handshake.static_identity->static_private,
+			peer->handshake.remote_static);
+	else
+		memset(peer->handshake.precomputed_static_static, 0,
+		       NOISE_PUBLIC_KEY_LEN);
+	up_write(&peer->handshake.lock);
+	return ret;
+}
+
+bool wg_noise_handshake_init(struct noise_handshake *handshake,
+			   struct noise_static_identity *static_identity,
+			   const u8 peer_public_key[NOISE_PUBLIC_KEY_LEN],
+			   const u8 peer_preshared_key[NOISE_SYMMETRIC_KEY_LEN],
+			   struct wg_peer *peer)
+{
+	memset(handshake, 0, sizeof(*handshake));
+	init_rwsem(&handshake->lock);
+	handshake->entry.type = INDEX_HASHTABLE_HANDSHAKE;
+	handshake->entry.peer = peer;
+	memcpy(handshake->remote_static, peer_public_key, NOISE_PUBLIC_KEY_LEN);
+	if (peer_preshared_key)
+		memcpy(handshake->preshared_key, peer_preshared_key,
+		       NOISE_SYMMETRIC_KEY_LEN);
+	handshake->static_identity = static_identity;
+	handshake->state = HANDSHAKE_ZEROED;
+	return wg_noise_precompute_static_static(peer);
+}
+
+static void handshake_zero(struct noise_handshake *handshake)
+{
+	memset(&handshake->ephemeral_private, 0, NOISE_PUBLIC_KEY_LEN);
+	memset(&handshake->remote_ephemeral, 0, NOISE_PUBLIC_KEY_LEN);
+	memset(&handshake->hash, 0, NOISE_HASH_LEN);
+	memset(&handshake->chaining_key, 0, NOISE_HASH_LEN);
+	handshake->remote_index = 0;
+	handshake->state = HANDSHAKE_ZEROED;
+}
+
+void wg_noise_handshake_clear(struct noise_handshake *handshake)
+{
+	wg_index_hashtable_remove(
+			handshake->entry.peer->device->index_hashtable,
+			&handshake->entry);
+	down_write(&handshake->lock);
+	handshake_zero(handshake);
+	up_write(&handshake->lock);
+	wg_index_hashtable_remove(
+			handshake->entry.peer->device->index_hashtable,
+			&handshake->entry);
+}
+
+static struct noise_keypair *keypair_create(struct wg_peer *peer)
+{
+	struct noise_keypair *keypair = kzalloc(sizeof(*keypair), GFP_KERNEL);
+
+	if (unlikely(!keypair))
+		return NULL;
+	keypair->internal_id = atomic64_inc_return(&keypair_counter);
+	keypair->entry.type = INDEX_HASHTABLE_KEYPAIR;
+	keypair->entry.peer = peer;
+	kref_init(&keypair->refcount);
+	return keypair;
+}
+
+static void keypair_free_rcu(struct rcu_head *rcu)
+{
+	kzfree(container_of(rcu, struct noise_keypair, rcu));
+}
+
+static void keypair_free_kref(struct kref *kref)
+{
+	struct noise_keypair *keypair =
+		container_of(kref, struct noise_keypair, refcount);
+
+	net_dbg_ratelimited("%s: Keypair %llu destroyed for peer %llu\n",
+			    keypair->entry.peer->device->dev->name,
+			    keypair->internal_id,
+			    keypair->entry.peer->internal_id);
+	wg_index_hashtable_remove(keypair->entry.peer->device->index_hashtable,
+				  &keypair->entry);
+	call_rcu(&keypair->rcu, keypair_free_rcu);
+}
+
+void wg_noise_keypair_put(struct noise_keypair *keypair, bool unreference_now)
+{
+	if (unlikely(!keypair))
+		return;
+	if (unlikely(unreference_now))
+		wg_index_hashtable_remove(
+			keypair->entry.peer->device->index_hashtable,
+			&keypair->entry);
+	kref_put(&keypair->refcount, keypair_free_kref);
+}
+
+struct noise_keypair *wg_noise_keypair_get(struct noise_keypair *keypair)
+{
+	RCU_LOCKDEP_WARN(!rcu_read_lock_bh_held(),
+		"Taking noise keypair reference without holding the RCU BH read lock");
+	if (unlikely(!keypair || !kref_get_unless_zero(&keypair->refcount)))
+		return NULL;
+	return keypair;
+}
+
+void wg_noise_keypairs_clear(struct noise_keypairs *keypairs)
+{
+	struct noise_keypair *old;
+
+	spin_lock_bh(&keypairs->keypair_update_lock);
+
+	/* We zero the next_keypair before zeroing the others, so that
+	 * wg_noise_received_with_keypair returns early before subsequent ones
+	 * are zeroed.
+	 */
+	old = rcu_dereference_protected(keypairs->next_keypair,
+		lockdep_is_held(&keypairs->keypair_update_lock));
+	RCU_INIT_POINTER(keypairs->next_keypair, NULL);
+	wg_noise_keypair_put(old, true);
+
+	old = rcu_dereference_protected(keypairs->previous_keypair,
+		lockdep_is_held(&keypairs->keypair_update_lock));
+	RCU_INIT_POINTER(keypairs->previous_keypair, NULL);
+	wg_noise_keypair_put(old, true);
+
+	old = rcu_dereference_protected(keypairs->current_keypair,
+		lockdep_is_held(&keypairs->keypair_update_lock));
+	RCU_INIT_POINTER(keypairs->current_keypair, NULL);
+	wg_noise_keypair_put(old, true);
+
+	spin_unlock_bh(&keypairs->keypair_update_lock);
+}
+
+void wg_noise_expire_current_peer_keypairs(struct wg_peer *peer)
+{
+	struct noise_keypair *keypair;
+
+	wg_noise_handshake_clear(&peer->handshake);
+	wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake);
+
+	spin_lock_bh(&peer->keypairs.keypair_update_lock);
+	keypair = rcu_dereference_protected(peer->keypairs.next_keypair,
+			lockdep_is_held(&peer->keypairs.keypair_update_lock));
+	if (keypair)
+		keypair->sending.is_valid = false;
+	keypair = rcu_dereference_protected(peer->keypairs.current_keypair,
+			lockdep_is_held(&peer->keypairs.keypair_update_lock));
+	if (keypair)
+		keypair->sending.is_valid = false;
+	spin_unlock_bh(&peer->keypairs.keypair_update_lock);
+}
+
+static void add_new_keypair(struct noise_keypairs *keypairs,
+			    struct noise_keypair *new_keypair)
+{
+	struct noise_keypair *previous_keypair, *next_keypair, *current_keypair;
+
+	spin_lock_bh(&keypairs->keypair_update_lock);
+	previous_keypair = rcu_dereference_protected(keypairs->previous_keypair,
+		lockdep_is_held(&keypairs->keypair_update_lock));
+	next_keypair = rcu_dereference_protected(keypairs->next_keypair,
+		lockdep_is_held(&keypairs->keypair_update_lock));
+	current_keypair = rcu_dereference_protected(keypairs->current_keypair,
+		lockdep_is_held(&keypairs->keypair_update_lock));
+	if (new_keypair->i_am_the_initiator) {
+		/* If we're the initiator, it means we've sent a handshake, and
+		 * received a confirmation response, which means this new
+		 * keypair can now be used.
+		 */
+		if (next_keypair) {
+			/* If there already was a next keypair pending, we
+			 * demote it to be the previous keypair, and free the
+			 * existing current. Note that this means KCI can result
+			 * in this transition. It would perhaps be more sound to
+			 * always just get rid of the unused next keypair
+			 * instead of putting it in the previous slot, but this
+			 * might be a bit less robust. Something to think about
+			 * for the future.
+			 */
+			RCU_INIT_POINTER(keypairs->next_keypair, NULL);
+			rcu_assign_pointer(keypairs->previous_keypair,
+					   next_keypair);
+			wg_noise_keypair_put(current_keypair, true);
+		} else /* If there wasn't an existing next keypair, we replace
+			* the previous with the current one.
+			*/
+			rcu_assign_pointer(keypairs->previous_keypair,
+					   current_keypair);
+		/* At this point we can get rid of the old previous keypair, and
+		 * set up the new keypair.
+		 */
+		wg_noise_keypair_put(previous_keypair, true);
+		rcu_assign_pointer(keypairs->current_keypair, new_keypair);
+	} else {
+		/* If we're the responder, it means we can't use the new keypair
+		 * until we receive confirmation via the first data packet, so
+		 * we get rid of the existing previous one, the possibly
+		 * existing next one, and slide in the new next one.
+		 */
+		rcu_assign_pointer(keypairs->next_keypair, new_keypair);
+		wg_noise_keypair_put(next_keypair, true);
+		RCU_INIT_POINTER(keypairs->previous_keypair, NULL);
+		wg_noise_keypair_put(previous_keypair, true);
+	}
+	spin_unlock_bh(&keypairs->keypair_update_lock);
+}
+
+bool wg_noise_received_with_keypair(struct noise_keypairs *keypairs,
+				    struct noise_keypair *received_keypair)
+{
+	struct noise_keypair *old_keypair;
+	bool key_is_new;
+
+	/* We first check without taking the spinlock. */
+	key_is_new = received_keypair ==
+		     rcu_access_pointer(keypairs->next_keypair);
+	if (likely(!key_is_new))
+		return false;
+
+	spin_lock_bh(&keypairs->keypair_update_lock);
+	/* After locking, we double check that things didn't change from
+	 * beneath us.
+	 */
+	if (unlikely(received_keypair !=
+		    rcu_dereference_protected(keypairs->next_keypair,
+			    lockdep_is_held(&keypairs->keypair_update_lock)))) {
+		spin_unlock_bh(&keypairs->keypair_update_lock);
+		return false;
+	}
+
+	/* When we've finally received the confirmation, we slide the next
+	 * into the current, the current into the previous, and get rid of
+	 * the old previous.
+	 */
+	old_keypair = rcu_dereference_protected(keypairs->previous_keypair,
+		lockdep_is_held(&keypairs->keypair_update_lock));
+	rcu_assign_pointer(keypairs->previous_keypair,
+		rcu_dereference_protected(keypairs->current_keypair,
+			lockdep_is_held(&keypairs->keypair_update_lock)));
+	wg_noise_keypair_put(old_keypair, true);
+	rcu_assign_pointer(keypairs->current_keypair, received_keypair);
+	RCU_INIT_POINTER(keypairs->next_keypair, NULL);
+
+	spin_unlock_bh(&keypairs->keypair_update_lock);
+	return true;
+}
+
+/* Must hold static_identity->lock */
+void wg_noise_set_static_identity_private_key(
+	struct noise_static_identity *static_identity,
+	const u8 private_key[NOISE_PUBLIC_KEY_LEN])
+{
+	memcpy(static_identity->static_private, private_key,
+	       NOISE_PUBLIC_KEY_LEN);
+	curve25519_clamp_secret(static_identity->static_private);
+	static_identity->has_identity = curve25519_generate_public(
+		static_identity->static_public, private_key);
+}
+
+/* This is Hugo Krawczyk's HKDF:
+ *  - https://eprint.iacr.org/2010/264.pdf
+ *  - https://tools.ietf.org/html/rfc5869
+ */
+static void kdf(u8 *first_dst, u8 *second_dst, u8 *third_dst, const u8 *data,
+		size_t first_len, size_t second_len, size_t third_len,
+		size_t data_len, const u8 chaining_key[NOISE_HASH_LEN])
+{
+	u8 output[BLAKE2S_HASH_SIZE + 1];
+	u8 secret[BLAKE2S_HASH_SIZE];
+
+	WARN_ON(IS_ENABLED(DEBUG) &&
+		(first_len > BLAKE2S_HASH_SIZE ||
+		 second_len > BLAKE2S_HASH_SIZE ||
+		 third_len > BLAKE2S_HASH_SIZE ||
+		 ((second_len || second_dst || third_len || third_dst) &&
+		  (!first_len || !first_dst)) ||
+		 ((third_len || third_dst) && (!second_len || !second_dst))));
+
+	/* Extract entropy from data into secret */
+	blake2s_hmac(secret, data, chaining_key, BLAKE2S_HASH_SIZE, data_len,
+		     NOISE_HASH_LEN);
+
+	if (!first_dst || !first_len)
+		goto out;
+
+	/* Expand first key: key = secret, data = 0x1 */
+	output[0] = 1;
+	blake2s_hmac(output, output, secret, BLAKE2S_HASH_SIZE, 1,
+		     BLAKE2S_HASH_SIZE);
+	memcpy(first_dst, output, first_len);
+
+	if (!second_dst || !second_len)
+		goto out;
+
+	/* Expand second key: key = secret, data = first-key || 0x2 */
+	output[BLAKE2S_HASH_SIZE] = 2;
+	blake2s_hmac(output, output, secret, BLAKE2S_HASH_SIZE,
+		     BLAKE2S_HASH_SIZE + 1, BLAKE2S_HASH_SIZE);
+	memcpy(second_dst, output, second_len);
+
+	if (!third_dst || !third_len)
+		goto out;
+
+	/* Expand third key: key = secret, data = second-key || 0x3 */
+	output[BLAKE2S_HASH_SIZE] = 3;
+	blake2s_hmac(output, output, secret, BLAKE2S_HASH_SIZE,
+		     BLAKE2S_HASH_SIZE + 1, BLAKE2S_HASH_SIZE);
+	memcpy(third_dst, output, third_len);
+
+out:
+	/* Clear sensitive data from stack */
+	memzero_explicit(secret, BLAKE2S_HASH_SIZE);
+	memzero_explicit(output, BLAKE2S_HASH_SIZE + 1);
+}
+
+static void symmetric_key_init(struct noise_symmetric_key *key)
+{
+	spin_lock_init(&key->counter.receive.lock);
+	atomic64_set(&key->counter.counter, 0);
+	memset(key->counter.receive.backtrack, 0,
+	       sizeof(key->counter.receive.backtrack));
+	key->birthdate = ktime_get_coarse_boottime_ns();
+	key->is_valid = true;
+}
+
+static void derive_keys(struct noise_symmetric_key *first_dst,
+			struct noise_symmetric_key *second_dst,
+			const u8 chaining_key[NOISE_HASH_LEN])
+{
+	kdf(first_dst->key, second_dst->key, NULL, NULL,
+	    NOISE_SYMMETRIC_KEY_LEN, NOISE_SYMMETRIC_KEY_LEN, 0, 0,
+	    chaining_key);
+	symmetric_key_init(first_dst);
+	symmetric_key_init(second_dst);
+}
+
+static bool __must_check mix_dh(u8 chaining_key[NOISE_HASH_LEN],
+				u8 key[NOISE_SYMMETRIC_KEY_LEN],
+				const u8 private[NOISE_PUBLIC_KEY_LEN],
+				const u8 public[NOISE_PUBLIC_KEY_LEN])
+{
+	u8 dh_calculation[NOISE_PUBLIC_KEY_LEN];
+
+	if (unlikely(!curve25519(dh_calculation, private, public)))
+		return false;
+	kdf(chaining_key, key, NULL, dh_calculation, NOISE_HASH_LEN,
+	    NOISE_SYMMETRIC_KEY_LEN, 0, NOISE_PUBLIC_KEY_LEN, chaining_key);
+	memzero_explicit(dh_calculation, NOISE_PUBLIC_KEY_LEN);
+	return true;
+}
+
+static void mix_hash(u8 hash[NOISE_HASH_LEN], const u8 *src, size_t src_len)
+{
+	struct blake2s_state blake;
+
+	blake2s_init(&blake, NOISE_HASH_LEN);
+	blake2s_update(&blake, hash, NOISE_HASH_LEN);
+	blake2s_update(&blake, src, src_len);
+	blake2s_final(&blake, hash);
+}
+
+static void mix_psk(u8 chaining_key[NOISE_HASH_LEN], u8 hash[NOISE_HASH_LEN],
+		    u8 key[NOISE_SYMMETRIC_KEY_LEN],
+		    const u8 psk[NOISE_SYMMETRIC_KEY_LEN])
+{
+	u8 temp_hash[NOISE_HASH_LEN];
+
+	kdf(chaining_key, temp_hash, key, psk, NOISE_HASH_LEN, NOISE_HASH_LEN,
+	    NOISE_SYMMETRIC_KEY_LEN, NOISE_SYMMETRIC_KEY_LEN, chaining_key);
+	mix_hash(hash, temp_hash, NOISE_HASH_LEN);
+	memzero_explicit(temp_hash, NOISE_HASH_LEN);
+}
+
+static void handshake_init(u8 chaining_key[NOISE_HASH_LEN],
+			   u8 hash[NOISE_HASH_LEN],
+			   const u8 remote_static[NOISE_PUBLIC_KEY_LEN])
+{
+	memcpy(hash, handshake_init_hash, NOISE_HASH_LEN);
+	memcpy(chaining_key, handshake_init_chaining_key, NOISE_HASH_LEN);
+	mix_hash(hash, remote_static, NOISE_PUBLIC_KEY_LEN);
+}
+
+static void message_encrypt(u8 *dst_ciphertext, const u8 *src_plaintext,
+			    size_t src_len, u8 key[NOISE_SYMMETRIC_KEY_LEN],
+			    u8 hash[NOISE_HASH_LEN])
+{
+	chacha20poly1305_encrypt(dst_ciphertext, src_plaintext, src_len, hash,
+				 NOISE_HASH_LEN,
+				 0 /* Always zero for Noise_IK */, key);
+	mix_hash(hash, dst_ciphertext, noise_encrypted_len(src_len));
+}
+
+static bool message_decrypt(u8 *dst_plaintext, const u8 *src_ciphertext,
+			    size_t src_len, u8 key[NOISE_SYMMETRIC_KEY_LEN],
+			    u8 hash[NOISE_HASH_LEN])
+{
+	if (!chacha20poly1305_decrypt(dst_plaintext, src_ciphertext, src_len,
+				      hash, NOISE_HASH_LEN,
+				      0 /* Always zero for Noise_IK */, key))
+		return false;
+	mix_hash(hash, src_ciphertext, src_len);
+	return true;
+}
+
+static void message_ephemeral(u8 ephemeral_dst[NOISE_PUBLIC_KEY_LEN],
+			      const u8 ephemeral_src[NOISE_PUBLIC_KEY_LEN],
+			      u8 chaining_key[NOISE_HASH_LEN],
+			      u8 hash[NOISE_HASH_LEN])
+{
+	if (ephemeral_dst != ephemeral_src)
+		memcpy(ephemeral_dst, ephemeral_src, NOISE_PUBLIC_KEY_LEN);
+	mix_hash(hash, ephemeral_src, NOISE_PUBLIC_KEY_LEN);
+	kdf(chaining_key, NULL, NULL, ephemeral_src, NOISE_HASH_LEN, 0, 0,
+	    NOISE_PUBLIC_KEY_LEN, chaining_key);
+}
+
+static void tai64n_now(u8 output[NOISE_TIMESTAMP_LEN])
+{
+	struct timespec64 now;
+
+	ktime_get_real_ts64(&now);
+
+	/* In order to prevent some sort of infoleak from precise timers, we
+	 * round down the nanoseconds part to the closest rounded-down power of
+	 * two to the maximum initiations per second allowed anyway by the
+	 * implementation.
+	 */
+	now.tv_nsec = ALIGN_DOWN(now.tv_nsec,
+		rounddown_pow_of_two(NSEC_PER_SEC / INITIATIONS_PER_SECOND));
+
+	/* https://cr.yp.to/libtai/tai64.html */
+	*(__be64 *)output = cpu_to_be64(0x400000000000000aULL + now.tv_sec);
+	*(__be32 *)(output + sizeof(__be64)) = cpu_to_be32(now.tv_nsec);
+}
+
+bool
+wg_noise_handshake_create_initiation(struct message_handshake_initiation *dst,
+				     struct noise_handshake *handshake)
+{
+	u8 timestamp[NOISE_TIMESTAMP_LEN];
+	u8 key[NOISE_SYMMETRIC_KEY_LEN];
+	bool ret = false;
+
+	/* We need to wait for crng _before_ taking any locks, since
+	 * curve25519_generate_secret uses get_random_bytes_wait.
+	 */
+	wait_for_random_bytes();
+
+	down_read(&handshake->static_identity->lock);
+	down_write(&handshake->lock);
+
+	if (unlikely(!handshake->static_identity->has_identity))
+		goto out;
+
+	dst->header.type = cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION);
+
+	handshake_init(handshake->chaining_key, handshake->hash,
+		       handshake->remote_static);
+
+	/* e */
+	curve25519_generate_secret(handshake->ephemeral_private);
+	if (!curve25519_generate_public(dst->unencrypted_ephemeral,
+					handshake->ephemeral_private))
+		goto out;
+	message_ephemeral(dst->unencrypted_ephemeral,
+			  dst->unencrypted_ephemeral, handshake->chaining_key,
+			  handshake->hash);
+
+	/* es */
+	if (!mix_dh(handshake->chaining_key, key, handshake->ephemeral_private,
+		    handshake->remote_static))
+		goto out;
+
+	/* s */
+	message_encrypt(dst->encrypted_static,
+			handshake->static_identity->static_public,
+			NOISE_PUBLIC_KEY_LEN, key, handshake->hash);
+
+	/* ss */
+	kdf(handshake->chaining_key, key, NULL,
+	    handshake->precomputed_static_static, NOISE_HASH_LEN,
+	    NOISE_SYMMETRIC_KEY_LEN, 0, NOISE_PUBLIC_KEY_LEN,
+	    handshake->chaining_key);
+
+	/* {t} */
+	tai64n_now(timestamp);
+	message_encrypt(dst->encrypted_timestamp, timestamp,
+			NOISE_TIMESTAMP_LEN, key, handshake->hash);
+
+	dst->sender_index = wg_index_hashtable_insert(
+		handshake->entry.peer->device->index_hashtable,
+		&handshake->entry);
+
+	handshake->state = HANDSHAKE_CREATED_INITIATION;
+	ret = true;
+
+out:
+	up_write(&handshake->lock);
+	up_read(&handshake->static_identity->lock);
+	memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN);
+	return ret;
+}
+
+struct wg_peer *
+wg_noise_handshake_consume_initiation(struct message_handshake_initiation *src,
+				      struct wg_device *wg)
+{
+	struct wg_peer *peer = NULL, *ret_peer = NULL;
+	struct noise_handshake *handshake;
+	bool replay_attack, flood_attack;
+	u8 key[NOISE_SYMMETRIC_KEY_LEN];
+	u8 chaining_key[NOISE_HASH_LEN];
+	u8 hash[NOISE_HASH_LEN];
+	u8 s[NOISE_PUBLIC_KEY_LEN];
+	u8 e[NOISE_PUBLIC_KEY_LEN];
+	u8 t[NOISE_TIMESTAMP_LEN];
+	u64 initiation_consumption;
+
+	down_read(&wg->static_identity.lock);
+	if (unlikely(!wg->static_identity.has_identity))
+		goto out;
+
+	handshake_init(chaining_key, hash, wg->static_identity.static_public);
+
+	/* e */
+	message_ephemeral(e, src->unencrypted_ephemeral, chaining_key, hash);
+
+	/* es */
+	if (!mix_dh(chaining_key, key, wg->static_identity.static_private, e))
+		goto out;
+
+	/* s */
+	if (!message_decrypt(s, src->encrypted_static,
+			     sizeof(src->encrypted_static), key, hash))
+		goto out;
+
+	/* Lookup which peer we're actually talking to */
+	peer = wg_pubkey_hashtable_lookup(wg->peer_hashtable, s);
+	if (!peer)
+		goto out;
+	handshake = &peer->handshake;
+
+	/* ss */
+	kdf(chaining_key, key, NULL, handshake->precomputed_static_static,
+	    NOISE_HASH_LEN, NOISE_SYMMETRIC_KEY_LEN, 0, NOISE_PUBLIC_KEY_LEN,
+	    chaining_key);
+
+	/* {t} */
+	if (!message_decrypt(t, src->encrypted_timestamp,
+			     sizeof(src->encrypted_timestamp), key, hash))
+		goto out;
+
+	down_read(&handshake->lock);
+	replay_attack = memcmp(t, handshake->latest_timestamp,
+			       NOISE_TIMESTAMP_LEN) <= 0;
+	flood_attack = (s64)handshake->last_initiation_consumption +
+			       NSEC_PER_SEC / INITIATIONS_PER_SECOND >
+		       (s64)ktime_get_coarse_boottime_ns();
+	up_read(&handshake->lock);
+	if (replay_attack || flood_attack)
+		goto out;
+
+	/* Success! Copy everything to peer */
+	down_write(&handshake->lock);
+	memcpy(handshake->remote_ephemeral, e, NOISE_PUBLIC_KEY_LEN);
+	if (memcmp(t, handshake->latest_timestamp, NOISE_TIMESTAMP_LEN) > 0)
+		memcpy(handshake->latest_timestamp, t, NOISE_TIMESTAMP_LEN);
+	memcpy(handshake->hash, hash, NOISE_HASH_LEN);
+	memcpy(handshake->chaining_key, chaining_key, NOISE_HASH_LEN);
+	handshake->remote_index = src->sender_index;
+	if ((s64)(handshake->last_initiation_consumption -
+	    (initiation_consumption = ktime_get_coarse_boottime_ns())) < 0)
+		handshake->last_initiation_consumption = initiation_consumption;
+	handshake->state = HANDSHAKE_CONSUMED_INITIATION;
+	up_write(&handshake->lock);
+	ret_peer = peer;
+
+out:
+	memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN);
+	memzero_explicit(hash, NOISE_HASH_LEN);
+	memzero_explicit(chaining_key, NOISE_HASH_LEN);
+	up_read(&wg->static_identity.lock);
+	if (!ret_peer)
+		wg_peer_put(peer);
+	return ret_peer;
+}
+
+bool wg_noise_handshake_create_response(struct message_handshake_response *dst,
+					struct noise_handshake *handshake)
+{
+	u8 key[NOISE_SYMMETRIC_KEY_LEN];
+	bool ret = false;
+
+	/* We need to wait for crng _before_ taking any locks, since
+	 * curve25519_generate_secret uses get_random_bytes_wait.
+	 */
+	wait_for_random_bytes();
+
+	down_read(&handshake->static_identity->lock);
+	down_write(&handshake->lock);
+
+	if (handshake->state != HANDSHAKE_CONSUMED_INITIATION)
+		goto out;
+
+	dst->header.type = cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE);
+	dst->receiver_index = handshake->remote_index;
+
+	/* e */
+	curve25519_generate_secret(handshake->ephemeral_private);
+	if (!curve25519_generate_public(dst->unencrypted_ephemeral,
+					handshake->ephemeral_private))
+		goto out;
+	message_ephemeral(dst->unencrypted_ephemeral,
+			  dst->unencrypted_ephemeral, handshake->chaining_key,
+			  handshake->hash);
+
+	/* ee */
+	if (!mix_dh(handshake->chaining_key, NULL, handshake->ephemeral_private,
+		    handshake->remote_ephemeral))
+		goto out;
+
+	/* se */
+	if (!mix_dh(handshake->chaining_key, NULL, handshake->ephemeral_private,
+		    handshake->remote_static))
+		goto out;
+
+	/* psk */
+	mix_psk(handshake->chaining_key, handshake->hash, key,
+		handshake->preshared_key);
+
+	/* {} */
+	message_encrypt(dst->encrypted_nothing, NULL, 0, key, handshake->hash);
+
+	dst->sender_index = wg_index_hashtable_insert(
+		handshake->entry.peer->device->index_hashtable,
+		&handshake->entry);
+
+	handshake->state = HANDSHAKE_CREATED_RESPONSE;
+	ret = true;
+
+out:
+	up_write(&handshake->lock);
+	up_read(&handshake->static_identity->lock);
+	memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN);
+	return ret;
+}
+
+struct wg_peer *
+wg_noise_handshake_consume_response(struct message_handshake_response *src,
+				    struct wg_device *wg)
+{
+	enum noise_handshake_state state = HANDSHAKE_ZEROED;
+	struct wg_peer *peer = NULL, *ret_peer = NULL;
+	struct noise_handshake *handshake;
+	u8 key[NOISE_SYMMETRIC_KEY_LEN];
+	u8 hash[NOISE_HASH_LEN];
+	u8 chaining_key[NOISE_HASH_LEN];
+	u8 e[NOISE_PUBLIC_KEY_LEN];
+	u8 ephemeral_private[NOISE_PUBLIC_KEY_LEN];
+	u8 static_private[NOISE_PUBLIC_KEY_LEN];
+
+	down_read(&wg->static_identity.lock);
+
+	if (unlikely(!wg->static_identity.has_identity))
+		goto out;
+
+	handshake = (struct noise_handshake *)wg_index_hashtable_lookup(
+		wg->index_hashtable, INDEX_HASHTABLE_HANDSHAKE,
+		src->receiver_index, &peer);
+	if (unlikely(!handshake))
+		goto out;
+
+	down_read(&handshake->lock);
+	state = handshake->state;
+	memcpy(hash, handshake->hash, NOISE_HASH_LEN);
+	memcpy(chaining_key, handshake->chaining_key, NOISE_HASH_LEN);
+	memcpy(ephemeral_private, handshake->ephemeral_private,
+	       NOISE_PUBLIC_KEY_LEN);
+	up_read(&handshake->lock);
+
+	if (state != HANDSHAKE_CREATED_INITIATION)
+		goto fail;
+
+	/* e */
+	message_ephemeral(e, src->unencrypted_ephemeral, chaining_key, hash);
+
+	/* ee */
+	if (!mix_dh(chaining_key, NULL, ephemeral_private, e))
+		goto fail;
+
+	/* se */
+	if (!mix_dh(chaining_key, NULL, wg->static_identity.static_private, e))
+		goto fail;
+
+	/* psk */
+	mix_psk(chaining_key, hash, key, handshake->preshared_key);
+
+	/* {} */
+	if (!message_decrypt(NULL, src->encrypted_nothing,
+			     sizeof(src->encrypted_nothing), key, hash))
+		goto fail;
+
+	/* Success! Copy everything to peer */
+	down_write(&handshake->lock);
+	/* It's important to check that the state is still the same, while we
+	 * have an exclusive lock.
+	 */
+	if (handshake->state != state) {
+		up_write(&handshake->lock);
+		goto fail;
+	}
+	memcpy(handshake->remote_ephemeral, e, NOISE_PUBLIC_KEY_LEN);
+	memcpy(handshake->hash, hash, NOISE_HASH_LEN);
+	memcpy(handshake->chaining_key, chaining_key, NOISE_HASH_LEN);
+	handshake->remote_index = src->sender_index;
+	handshake->state = HANDSHAKE_CONSUMED_RESPONSE;
+	up_write(&handshake->lock);
+	ret_peer = peer;
+	goto out;
+
+fail:
+	wg_peer_put(peer);
+out:
+	memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN);
+	memzero_explicit(hash, NOISE_HASH_LEN);
+	memzero_explicit(chaining_key, NOISE_HASH_LEN);
+	memzero_explicit(ephemeral_private, NOISE_PUBLIC_KEY_LEN);
+	memzero_explicit(static_private, NOISE_PUBLIC_KEY_LEN);
+	up_read(&wg->static_identity.lock);
+	return ret_peer;
+}
+
+bool wg_noise_handshake_begin_session(struct noise_handshake *handshake,
+				      struct noise_keypairs *keypairs)
+{
+	struct noise_keypair *new_keypair;
+	bool ret = false;
+
+	down_write(&handshake->lock);
+	if (handshake->state != HANDSHAKE_CREATED_RESPONSE &&
+	    handshake->state != HANDSHAKE_CONSUMED_RESPONSE)
+		goto out;
+
+	new_keypair = keypair_create(handshake->entry.peer);
+	if (!new_keypair)
+		goto out;
+	new_keypair->i_am_the_initiator = handshake->state ==
+					  HANDSHAKE_CONSUMED_RESPONSE;
+	new_keypair->remote_index = handshake->remote_index;
+
+	if (new_keypair->i_am_the_initiator)
+		derive_keys(&new_keypair->sending, &new_keypair->receiving,
+			    handshake->chaining_key);
+	else
+		derive_keys(&new_keypair->receiving, &new_keypair->sending,
+			    handshake->chaining_key);
+
+	handshake_zero(handshake);
+	rcu_read_lock_bh();
+	if (likely(!READ_ONCE(container_of(handshake, struct wg_peer,
+					   handshake)->is_dead))) {
+		add_new_keypair(keypairs, new_keypair);
+		net_dbg_ratelimited("%s: Keypair %llu created for peer %llu\n",
+				    handshake->entry.peer->device->dev->name,
+				    new_keypair->internal_id,
+				    handshake->entry.peer->internal_id);
+		ret = wg_index_hashtable_replace(
+			handshake->entry.peer->device->index_hashtable,
+			&handshake->entry, &new_keypair->entry);
+	} else {
+		kzfree(new_keypair);
+	}
+	rcu_read_unlock_bh();
+
+out:
+	up_write(&handshake->lock);
+	return ret;
+}
diff -Naur --no-dereference a/net/wireguard/noise.h b/net/wireguard/noise.h
--- a/net/wireguard/noise.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/noise.h	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,137 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+#ifndef _WG_NOISE_H
+#define _WG_NOISE_H
+
+#include "messages.h"
+#include "peerlookup.h"
+
+#include <linux/types.h>
+#include <linux/spinlock.h>
+#include <linux/atomic.h>
+#include <linux/rwsem.h>
+#include <linux/mutex.h>
+#include <linux/kref.h>
+
+union noise_counter {
+	struct {
+		u64 counter;
+		unsigned long backtrack[COUNTER_BITS_TOTAL / BITS_PER_LONG];
+		spinlock_t lock;
+	} receive;
+	atomic64_t counter;
+};
+
+struct noise_symmetric_key {
+	u8 key[NOISE_SYMMETRIC_KEY_LEN];
+	union noise_counter counter;
+	u64 birthdate;
+	bool is_valid;
+};
+
+struct noise_keypair {
+	struct index_hashtable_entry entry;
+	struct noise_symmetric_key sending;
+	struct noise_symmetric_key receiving;
+	__le32 remote_index;
+	bool i_am_the_initiator;
+	struct kref refcount;
+	struct rcu_head rcu;
+	u64 internal_id;
+};
+
+struct noise_keypairs {
+	struct noise_keypair __rcu *current_keypair;
+	struct noise_keypair __rcu *previous_keypair;
+	struct noise_keypair __rcu *next_keypair;
+	spinlock_t keypair_update_lock;
+};
+
+struct noise_static_identity {
+	u8 static_public[NOISE_PUBLIC_KEY_LEN];
+	u8 static_private[NOISE_PUBLIC_KEY_LEN];
+	struct rw_semaphore lock;
+	bool has_identity;
+};
+
+enum noise_handshake_state {
+	HANDSHAKE_ZEROED,
+	HANDSHAKE_CREATED_INITIATION,
+	HANDSHAKE_CONSUMED_INITIATION,
+	HANDSHAKE_CREATED_RESPONSE,
+	HANDSHAKE_CONSUMED_RESPONSE
+};
+
+struct noise_handshake {
+	struct index_hashtable_entry entry;
+
+	enum noise_handshake_state state;
+	u64 last_initiation_consumption;
+
+	struct noise_static_identity *static_identity;
+
+	u8 ephemeral_private[NOISE_PUBLIC_KEY_LEN];
+	u8 remote_static[NOISE_PUBLIC_KEY_LEN];
+	u8 remote_ephemeral[NOISE_PUBLIC_KEY_LEN];
+	u8 precomputed_static_static[NOISE_PUBLIC_KEY_LEN];
+
+	u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN];
+
+	u8 hash[NOISE_HASH_LEN];
+	u8 chaining_key[NOISE_HASH_LEN];
+
+	u8 latest_timestamp[NOISE_TIMESTAMP_LEN];
+	__le32 remote_index;
+
+	/* Protects all members except the immutable (after noise_handshake_
+	 * init): remote_static, precomputed_static_static, static_identity.
+	 */
+	struct rw_semaphore lock;
+};
+
+struct wg_device;
+
+void wg_noise_init(void);
+bool wg_noise_handshake_init(struct noise_handshake *handshake,
+			   struct noise_static_identity *static_identity,
+			   const u8 peer_public_key[NOISE_PUBLIC_KEY_LEN],
+			   const u8 peer_preshared_key[NOISE_SYMMETRIC_KEY_LEN],
+			   struct wg_peer *peer);
+void wg_noise_handshake_clear(struct noise_handshake *handshake);
+static inline void wg_noise_reset_last_sent_handshake(atomic64_t *handshake_ns)
+{
+	atomic64_set(handshake_ns, ktime_get_coarse_boottime_ns() -
+				       (u64)(REKEY_TIMEOUT + 1) * NSEC_PER_SEC);
+}
+
+void wg_noise_keypair_put(struct noise_keypair *keypair, bool unreference_now);
+struct noise_keypair *wg_noise_keypair_get(struct noise_keypair *keypair);
+void wg_noise_keypairs_clear(struct noise_keypairs *keypairs);
+bool wg_noise_received_with_keypair(struct noise_keypairs *keypairs,
+				    struct noise_keypair *received_keypair);
+void wg_noise_expire_current_peer_keypairs(struct wg_peer *peer);
+
+void wg_noise_set_static_identity_private_key(
+	struct noise_static_identity *static_identity,
+	const u8 private_key[NOISE_PUBLIC_KEY_LEN]);
+bool wg_noise_precompute_static_static(struct wg_peer *peer);
+
+bool
+wg_noise_handshake_create_initiation(struct message_handshake_initiation *dst,
+				     struct noise_handshake *handshake);
+struct wg_peer *
+wg_noise_handshake_consume_initiation(struct message_handshake_initiation *src,
+				      struct wg_device *wg);
+
+bool wg_noise_handshake_create_response(struct message_handshake_response *dst,
+					struct noise_handshake *handshake);
+struct wg_peer *
+wg_noise_handshake_consume_response(struct message_handshake_response *src,
+				    struct wg_device *wg);
+
+bool wg_noise_handshake_begin_session(struct noise_handshake *handshake,
+				      struct noise_keypairs *keypairs);
+
+#endif /* _WG_NOISE_H */
diff -Naur --no-dereference a/net/wireguard/peer.c b/net/wireguard/peer.c
--- a/net/wireguard/peer.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/peer.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,240 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#include "peer.h"
+#include "device.h"
+#include "queueing.h"
+#include "timers.h"
+#include "peerlookup.h"
+#include "noise.h"
+
+#include <linux/kref.h>
+#include <linux/lockdep.h>
+#include <linux/rcupdate.h>
+#include <linux/list.h>
+
+static atomic64_t peer_counter = ATOMIC64_INIT(0);
+
+struct wg_peer *wg_peer_create(struct wg_device *wg,
+			       const u8 public_key[NOISE_PUBLIC_KEY_LEN],
+			       const u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN])
+{
+	struct wg_peer *peer;
+	int ret = -ENOMEM;
+
+	lockdep_assert_held(&wg->device_update_lock);
+
+	if (wg->num_peers >= MAX_PEERS_PER_DEVICE)
+		return ERR_PTR(ret);
+
+	peer = kzalloc(sizeof(*peer), GFP_KERNEL);
+	if (unlikely(!peer))
+		return ERR_PTR(ret);
+	peer->device = wg;
+
+	if (!wg_noise_handshake_init(&peer->handshake, &wg->static_identity,
+				     public_key, preshared_key, peer)) {
+		ret = -EKEYREJECTED;
+		goto err_1;
+	}
+	if (dst_cache_init(&peer->endpoint_cache, GFP_KERNEL))
+		goto err_1;
+	if (wg_packet_queue_init(&peer->tx_queue, wg_packet_tx_worker, false,
+				 MAX_QUEUED_PACKETS))
+		goto err_2;
+	if (wg_packet_queue_init(&peer->rx_queue, NULL, false,
+				 MAX_QUEUED_PACKETS))
+		goto err_3;
+
+	peer->internal_id = atomic64_inc_return(&peer_counter);
+	peer->serial_work_cpu = nr_cpumask_bits;
+	wg_cookie_init(&peer->latest_cookie);
+	wg_timers_init(peer);
+	wg_cookie_checker_precompute_peer_keys(peer);
+	spin_lock_init(&peer->keypairs.keypair_update_lock);
+	INIT_WORK(&peer->transmit_handshake_work,
+		  wg_packet_handshake_send_worker);
+	rwlock_init(&peer->endpoint_lock);
+	kref_init(&peer->refcount);
+	skb_queue_head_init(&peer->staged_packet_queue);
+	wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake);
+	set_bit(NAPI_STATE_NO_BUSY_POLL, &peer->napi.state);
+	netif_napi_add(wg->dev, &peer->napi, wg_packet_rx_poll,
+		       NAPI_POLL_WEIGHT);
+	napi_enable(&peer->napi);
+	list_add_tail(&peer->peer_list, &wg->peer_list);
+	INIT_LIST_HEAD(&peer->allowedips_list);
+	wg_pubkey_hashtable_add(wg->peer_hashtable, peer);
+	++wg->num_peers;
+	pr_debug("%s: Peer %llu created\n", wg->dev->name, peer->internal_id);
+	return peer;
+
+err_3:
+	wg_packet_queue_free(&peer->tx_queue, false);
+err_2:
+	dst_cache_destroy(&peer->endpoint_cache);
+err_1:
+	kfree(peer);
+	return ERR_PTR(ret);
+}
+
+struct wg_peer *wg_peer_get_maybe_zero(struct wg_peer *peer)
+{
+	RCU_LOCKDEP_WARN(!rcu_read_lock_bh_held(),
+			 "Taking peer reference without holding the RCU read lock");
+	if (unlikely(!peer || !kref_get_unless_zero(&peer->refcount)))
+		return NULL;
+	return peer;
+}
+
+static void peer_make_dead(struct wg_peer *peer)
+{
+	/* Remove from configuration-time lookup structures. */
+	list_del_init(&peer->peer_list);
+	wg_allowedips_remove_by_peer(&peer->device->peer_allowedips, peer,
+				     &peer->device->device_update_lock);
+	wg_pubkey_hashtable_remove(peer->device->peer_hashtable, peer);
+
+	/* Mark as dead, so that we don't allow jumping contexts after. */
+	WRITE_ONCE(peer->is_dead, true);
+
+	/* The caller must now synchronize_rcu() for this to take effect. */
+}
+
+static void peer_remove_after_dead(struct wg_peer *peer)
+{
+	WARN_ON(!peer->is_dead);
+
+	/* No more keypairs can be created for this peer, since is_dead protects
+	 * add_new_keypair, so we can now destroy existing ones.
+	 */
+	wg_noise_keypairs_clear(&peer->keypairs);
+
+	/* Destroy all ongoing timers that were in-flight at the beginning of
+	 * this function.
+	 */
+	wg_timers_stop(peer);
+
+	/* The transition between packet encryption/decryption queues isn't
+	 * guarded by is_dead, but each reference's life is strictly bounded by
+	 * two generations: once for parallel crypto and once for serial
+	 * ingestion, so we can simply flush twice, and be sure that we no
+	 * longer have references inside these queues.
+	 */
+
+	/* a) For encrypt/decrypt. */
+	flush_workqueue(peer->device->packet_crypt_wq);
+	/* b.1) For send (but not receive, since that's napi). */
+	flush_workqueue(peer->device->packet_crypt_wq);
+	/* b.2.1) For receive (but not send, since that's wq). */
+	napi_disable(&peer->napi);
+	/* b.2.1) It's now safe to remove the napi struct, which must be done
+	 * here from process context.
+	 */
+	netif_napi_del(&peer->napi);
+
+	/* Ensure any workstructs we own (like transmit_handshake_work or
+	 * clear_peer_work) no longer are in use.
+	 */
+	flush_workqueue(peer->device->handshake_send_wq);
+
+	/* After the above flushes, a peer might still be active in a few
+	 * different contexts: 1) from xmit(), before hitting is_dead and
+	 * returning, 2) from wg_packet_consume_data(), before hitting is_dead
+	 * and returning, 3) from wg_receive_handshake_packet() after a point
+	 * where it has processed an incoming handshake packet, but where
+	 * all calls to pass it off to timers fails because of is_dead. We won't
+	 * have new references in (1) eventually, because we're removed from
+	 * allowedips; we won't have new references in (2) eventually, because
+	 * wg_index_hashtable_lookup will always return NULL, since we removed
+	 * all existing keypairs and no more can be created; we won't have new
+	 * references in (3) eventually, because we're removed from the pubkey
+	 * hash table, which allows for a maximum of one handshake response,
+	 * via the still-uncleared index hashtable entry, but not more than one,
+	 * and in wg_cookie_message_consume, the lookup eventually gets a peer
+	 * with a refcount of zero, so no new reference is taken.
+	 */
+
+	--peer->device->num_peers;
+	wg_peer_put(peer);
+}
+
+/* We have a separate "remove" function make sure that all active places where
+ * a peer is currently operating will eventually come to an end and not pass
+ * their reference onto another context.
+ */
+void wg_peer_remove(struct wg_peer *peer)
+{
+	if (unlikely(!peer))
+		return;
+	lockdep_assert_held(&peer->device->device_update_lock);
+
+	peer_make_dead(peer);
+	synchronize_rcu();
+	peer_remove_after_dead(peer);
+}
+
+void wg_peer_remove_all(struct wg_device *wg)
+{
+	struct wg_peer *peer, *temp;
+	LIST_HEAD(dead_peers);
+
+	lockdep_assert_held(&wg->device_update_lock);
+
+	/* Avoid having to traverse individually for each one. */
+	wg_allowedips_free(&wg->peer_allowedips, &wg->device_update_lock);
+
+	list_for_each_entry_safe(peer, temp, &wg->peer_list, peer_list) {
+		peer_make_dead(peer);
+		list_add_tail(&peer->peer_list, &dead_peers);
+	}
+	synchronize_rcu();
+	list_for_each_entry_safe(peer, temp, &dead_peers, peer_list)
+		peer_remove_after_dead(peer);
+}
+
+static void rcu_release(struct rcu_head *rcu)
+{
+	struct wg_peer *peer = container_of(rcu, struct wg_peer, rcu);
+
+	dst_cache_destroy(&peer->endpoint_cache);
+	wg_packet_queue_free(&peer->rx_queue, false);
+	wg_packet_queue_free(&peer->tx_queue, false);
+
+	/* The final zeroing takes care of clearing any remaining handshake key
+	 * material and other potentially sensitive information.
+	 */
+	kzfree(peer);
+}
+
+static void kref_release(struct kref *refcount)
+{
+	struct wg_peer *peer = container_of(refcount, struct wg_peer, refcount);
+
+	pr_debug("%s: Peer %llu (%pISpfsc) destroyed\n",
+		 peer->device->dev->name, peer->internal_id,
+		 &peer->endpoint.addr);
+
+	/* Remove ourself from dynamic runtime lookup structures, now that the
+	 * last reference is gone.
+	 */
+	wg_index_hashtable_remove(peer->device->index_hashtable,
+				  &peer->handshake.entry);
+
+	/* Remove any lingering packets that didn't have a chance to be
+	 * transmitted.
+	 */
+	wg_packet_purge_staged_packets(peer);
+
+	/* Free the memory used. */
+	call_rcu(&peer->rcu, rcu_release);
+}
+
+void wg_peer_put(struct wg_peer *peer)
+{
+	if (unlikely(!peer))
+		return;
+	kref_put(&peer->refcount, kref_release);
+}
diff -Naur --no-dereference a/net/wireguard/peer.h b/net/wireguard/peer.h
--- a/net/wireguard/peer.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/peer.h	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,83 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#ifndef _WG_PEER_H
+#define _WG_PEER_H
+
+#include "device.h"
+#include "noise.h"
+#include "cookie.h"
+
+#include <linux/types.h>
+#include <linux/netfilter.h>
+#include <linux/spinlock.h>
+#include <linux/kref.h>
+#include <net/dst_cache.h>
+
+struct wg_device;
+
+struct endpoint {
+	union {
+		struct sockaddr addr;
+		struct sockaddr_in addr4;
+		struct sockaddr_in6 addr6;
+	};
+	union {
+		struct {
+			struct in_addr src4;
+			/* Essentially the same as addr6->scope_id */
+			int src_if4;
+		};
+		struct in6_addr src6;
+	};
+};
+
+struct wg_peer {
+	struct wg_device *device;
+	struct crypt_queue tx_queue, rx_queue;
+	struct sk_buff_head staged_packet_queue;
+	int serial_work_cpu;
+	struct noise_keypairs keypairs;
+	struct endpoint endpoint;
+	struct dst_cache endpoint_cache;
+	rwlock_t endpoint_lock;
+	struct noise_handshake handshake;
+	atomic64_t last_sent_handshake;
+	struct work_struct transmit_handshake_work, clear_peer_work;
+	struct cookie latest_cookie;
+	struct hlist_node pubkey_hash;
+	u64 rx_bytes, tx_bytes;
+	struct timer_list timer_retransmit_handshake, timer_send_keepalive;
+	struct timer_list timer_new_handshake, timer_zero_key_material;
+	struct timer_list timer_persistent_keepalive;
+	unsigned int timer_handshake_attempts;
+	u16 persistent_keepalive_interval;
+	bool timer_need_another_keepalive;
+	bool sent_lastminute_handshake;
+	struct timespec64 walltime_last_handshake;
+	struct kref refcount;
+	struct rcu_head rcu;
+	struct list_head peer_list;
+	struct list_head allowedips_list;
+	u64 internal_id;
+	struct napi_struct napi;
+	bool is_dead;
+};
+
+struct wg_peer *wg_peer_create(struct wg_device *wg,
+			       const u8 public_key[NOISE_PUBLIC_KEY_LEN],
+			       const u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN]);
+
+struct wg_peer *__must_check wg_peer_get_maybe_zero(struct wg_peer *peer);
+static inline struct wg_peer *wg_peer_get(struct wg_peer *peer)
+{
+	kref_get(&peer->refcount);
+	return peer;
+}
+void wg_peer_put(struct wg_peer *peer);
+void wg_peer_remove(struct wg_peer *peer);
+void wg_peer_remove_all(struct wg_device *wg);
+
+#endif /* _WG_PEER_H */
diff -Naur --no-dereference a/net/wireguard/peerlookup.c b/net/wireguard/peerlookup.c
--- a/net/wireguard/peerlookup.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/peerlookup.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,221 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#include "peerlookup.h"
+#include "peer.h"
+#include "noise.h"
+
+static struct hlist_head *pubkey_bucket(struct pubkey_hashtable *table,
+					const u8 pubkey[NOISE_PUBLIC_KEY_LEN])
+{
+	/* siphash gives us a secure 64bit number based on a random key. Since
+	 * the bits are uniformly distributed, we can then mask off to get the
+	 * bits we need.
+	 */
+	const u64 hash = siphash(pubkey, NOISE_PUBLIC_KEY_LEN, &table->key);
+
+	return &table->hashtable[hash & (HASH_SIZE(table->hashtable) - 1)];
+}
+
+struct pubkey_hashtable *wg_pubkey_hashtable_alloc(void)
+{
+	struct pubkey_hashtable *table = kvmalloc(sizeof(*table), GFP_KERNEL);
+
+	if (!table)
+		return NULL;
+
+	get_random_bytes(&table->key, sizeof(table->key));
+	hash_init(table->hashtable);
+	mutex_init(&table->lock);
+	return table;
+}
+
+void wg_pubkey_hashtable_add(struct pubkey_hashtable *table,
+			     struct wg_peer *peer)
+{
+	mutex_lock(&table->lock);
+	hlist_add_head_rcu(&peer->pubkey_hash,
+			   pubkey_bucket(table, peer->handshake.remote_static));
+	mutex_unlock(&table->lock);
+}
+
+void wg_pubkey_hashtable_remove(struct pubkey_hashtable *table,
+				struct wg_peer *peer)
+{
+	mutex_lock(&table->lock);
+	hlist_del_init_rcu(&peer->pubkey_hash);
+	mutex_unlock(&table->lock);
+}
+
+/* Returns a strong reference to a peer */
+struct wg_peer *
+wg_pubkey_hashtable_lookup(struct pubkey_hashtable *table,
+			   const u8 pubkey[NOISE_PUBLIC_KEY_LEN])
+{
+	struct wg_peer *iter_peer, *peer = NULL;
+
+	rcu_read_lock_bh();
+	hlist_for_each_entry_rcu_bh(iter_peer, pubkey_bucket(table, pubkey),
+				    pubkey_hash) {
+		if (!memcmp(pubkey, iter_peer->handshake.remote_static,
+			    NOISE_PUBLIC_KEY_LEN)) {
+			peer = iter_peer;
+			break;
+		}
+	}
+	peer = wg_peer_get_maybe_zero(peer);
+	rcu_read_unlock_bh();
+	return peer;
+}
+
+static struct hlist_head *index_bucket(struct index_hashtable *table,
+				       const __le32 index)
+{
+	/* Since the indices are random and thus all bits are uniformly
+	 * distributed, we can find its bucket simply by masking.
+	 */
+	return &table->hashtable[(__force u32)index &
+				 (HASH_SIZE(table->hashtable) - 1)];
+}
+
+struct index_hashtable *wg_index_hashtable_alloc(void)
+{
+	struct index_hashtable *table = kvmalloc(sizeof(*table), GFP_KERNEL);
+
+	if (!table)
+		return NULL;
+
+	hash_init(table->hashtable);
+	spin_lock_init(&table->lock);
+	return table;
+}
+
+/* At the moment, we limit ourselves to 2^20 total peers, which generally might
+ * amount to 2^20*3 items in this hashtable. The algorithm below works by
+ * picking a random number and testing it. We can see that these limits mean we
+ * usually succeed pretty quickly:
+ *
+ * >>> def calculation(tries, size):
+ * ...     return (size / 2**32)**(tries - 1) *  (1 - (size / 2**32))
+ * ...
+ * >>> calculation(1, 2**20 * 3)
+ * 0.999267578125
+ * >>> calculation(2, 2**20 * 3)
+ * 0.0007318854331970215
+ * >>> calculation(3, 2**20 * 3)
+ * 5.360489012673497e-07
+ * >>> calculation(4, 2**20 * 3)
+ * 3.9261394135792216e-10
+ *
+ * At the moment, we don't do any masking, so this algorithm isn't exactly
+ * constant time in either the random guessing or in the hash list lookup. We
+ * could require a minimum of 3 tries, which would successfully mask the
+ * guessing. this would not, however, help with the growing hash lengths, which
+ * is another thing to consider moving forward.
+ */
+
+__le32 wg_index_hashtable_insert(struct index_hashtable *table,
+				 struct index_hashtable_entry *entry)
+{
+	struct index_hashtable_entry *existing_entry;
+
+	spin_lock_bh(&table->lock);
+	hlist_del_init_rcu(&entry->index_hash);
+	spin_unlock_bh(&table->lock);
+
+	rcu_read_lock_bh();
+
+search_unused_slot:
+	/* First we try to find an unused slot, randomly, while unlocked. */
+	entry->index = (__force __le32)get_random_u32();
+	hlist_for_each_entry_rcu_bh(existing_entry,
+				    index_bucket(table, entry->index),
+				    index_hash) {
+		if (existing_entry->index == entry->index)
+			/* If it's already in use, we continue searching. */
+			goto search_unused_slot;
+	}
+
+	/* Once we've found an unused slot, we lock it, and then double-check
+	 * that nobody else stole it from us.
+	 */
+	spin_lock_bh(&table->lock);
+	hlist_for_each_entry_rcu_bh(existing_entry,
+				    index_bucket(table, entry->index),
+				    index_hash) {
+		if (existing_entry->index == entry->index) {
+			spin_unlock_bh(&table->lock);
+			/* If it was stolen, we start over. */
+			goto search_unused_slot;
+		}
+	}
+	/* Otherwise, we know we have it exclusively (since we're locked),
+	 * so we insert.
+	 */
+	hlist_add_head_rcu(&entry->index_hash,
+			   index_bucket(table, entry->index));
+	spin_unlock_bh(&table->lock);
+
+	rcu_read_unlock_bh();
+
+	return entry->index;
+}
+
+bool wg_index_hashtable_replace(struct index_hashtable *table,
+				struct index_hashtable_entry *old,
+				struct index_hashtable_entry *new)
+{
+	if (unlikely(hlist_unhashed(&old->index_hash)))
+		return false;
+	spin_lock_bh(&table->lock);
+	new->index = old->index;
+	hlist_replace_rcu(&old->index_hash, &new->index_hash);
+
+	/* Calling init here NULLs out index_hash, and in fact after this
+	 * function returns, it's theoretically possible for this to get
+	 * reinserted elsewhere. That means the RCU lookup below might either
+	 * terminate early or jump between buckets, in which case the packet
+	 * simply gets dropped, which isn't terrible.
+	 */
+	INIT_HLIST_NODE(&old->index_hash);
+	spin_unlock_bh(&table->lock);
+	return true;
+}
+
+void wg_index_hashtable_remove(struct index_hashtable *table,
+			       struct index_hashtable_entry *entry)
+{
+	spin_lock_bh(&table->lock);
+	hlist_del_init_rcu(&entry->index_hash);
+	spin_unlock_bh(&table->lock);
+}
+
+/* Returns a strong reference to a entry->peer */
+struct index_hashtable_entry *
+wg_index_hashtable_lookup(struct index_hashtable *table,
+			  const enum index_hashtable_type type_mask,
+			  const __le32 index, struct wg_peer **peer)
+{
+	struct index_hashtable_entry *iter_entry, *entry = NULL;
+
+	rcu_read_lock_bh();
+	hlist_for_each_entry_rcu_bh(iter_entry, index_bucket(table, index),
+				    index_hash) {
+		if (iter_entry->index == index) {
+			if (likely(iter_entry->type & type_mask))
+				entry = iter_entry;
+			break;
+		}
+	}
+	if (likely(entry)) {
+		entry->peer = wg_peer_get_maybe_zero(entry->peer);
+		if (likely(entry->peer))
+			*peer = entry->peer;
+		else
+			entry = NULL;
+	}
+	rcu_read_unlock_bh();
+	return entry;
+}
diff -Naur --no-dereference a/net/wireguard/peerlookup.h b/net/wireguard/peerlookup.h
--- a/net/wireguard/peerlookup.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/peerlookup.h	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,64 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#ifndef _WG_PEERLOOKUP_H
+#define _WG_PEERLOOKUP_H
+
+#include "messages.h"
+
+#include <linux/hashtable.h>
+#include <linux/mutex.h>
+#include <linux/siphash.h>
+
+struct wg_peer;
+
+struct pubkey_hashtable {
+	/* TODO: move to rhashtable */
+	DECLARE_HASHTABLE(hashtable, 11);
+	siphash_key_t key;
+	struct mutex lock;
+};
+
+struct pubkey_hashtable *wg_pubkey_hashtable_alloc(void);
+void wg_pubkey_hashtable_add(struct pubkey_hashtable *table,
+			     struct wg_peer *peer);
+void wg_pubkey_hashtable_remove(struct pubkey_hashtable *table,
+				struct wg_peer *peer);
+struct wg_peer *
+wg_pubkey_hashtable_lookup(struct pubkey_hashtable *table,
+			   const u8 pubkey[NOISE_PUBLIC_KEY_LEN]);
+
+struct index_hashtable {
+	/* TODO: move to rhashtable */
+	DECLARE_HASHTABLE(hashtable, 13);
+	spinlock_t lock;
+};
+
+enum index_hashtable_type {
+	INDEX_HASHTABLE_HANDSHAKE = 1U << 0,
+	INDEX_HASHTABLE_KEYPAIR = 1U << 1
+};
+
+struct index_hashtable_entry {
+	struct wg_peer *peer;
+	struct hlist_node index_hash;
+	enum index_hashtable_type type;
+	__le32 index;
+};
+
+struct index_hashtable *wg_index_hashtable_alloc(void);
+__le32 wg_index_hashtable_insert(struct index_hashtable *table,
+				 struct index_hashtable_entry *entry);
+bool wg_index_hashtable_replace(struct index_hashtable *table,
+				struct index_hashtable_entry *old,
+				struct index_hashtable_entry *new);
+void wg_index_hashtable_remove(struct index_hashtable *table,
+			       struct index_hashtable_entry *entry);
+struct index_hashtable_entry *
+wg_index_hashtable_lookup(struct index_hashtable *table,
+			  const enum index_hashtable_type type_mask,
+			  const __le32 index, struct wg_peer **peer);
+
+#endif /* _WG_PEERLOOKUP_H */
diff -Naur --no-dereference a/net/wireguard/queueing.c b/net/wireguard/queueing.c
--- a/net/wireguard/queueing.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/queueing.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#include "queueing.h"
+
+struct multicore_worker __percpu *
+wg_packet_percpu_multicore_worker_alloc(work_func_t function, void *ptr)
+{
+	int cpu;
+	struct multicore_worker __percpu *worker =
+		alloc_percpu(struct multicore_worker);
+
+	if (!worker)
+		return NULL;
+
+	for_each_possible_cpu(cpu) {
+		per_cpu_ptr(worker, cpu)->ptr = ptr;
+		INIT_WORK(&per_cpu_ptr(worker, cpu)->work, function);
+	}
+	return worker;
+}
+
+int wg_packet_queue_init(struct crypt_queue *queue, work_func_t function,
+			 bool multicore, unsigned int len)
+{
+	int ret;
+
+	memset(queue, 0, sizeof(*queue));
+	ret = ptr_ring_init(&queue->ring, len, GFP_KERNEL);
+	if (ret)
+		return ret;
+	if (function) {
+		if (multicore) {
+			queue->worker = wg_packet_percpu_multicore_worker_alloc(
+				function, queue);
+			if (!queue->worker)
+				return -ENOMEM;
+		} else {
+			INIT_WORK(&queue->work, function);
+		}
+	}
+	return 0;
+}
+
+void wg_packet_queue_free(struct crypt_queue *queue, bool multicore)
+{
+	if (multicore)
+		free_percpu(queue->worker);
+	WARN_ON(!__ptr_ring_empty(&queue->ring));
+	ptr_ring_cleanup(&queue->ring, NULL);
+}
diff -Naur --no-dereference a/net/wireguard/queueing.h b/net/wireguard/queueing.h
--- a/net/wireguard/queueing.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/queueing.h	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,197 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#ifndef _WG_QUEUEING_H
+#define _WG_QUEUEING_H
+
+#include "peer.h"
+#include <linux/types.h>
+#include <linux/skbuff.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+
+struct wg_device;
+struct wg_peer;
+struct multicore_worker;
+struct crypt_queue;
+struct sk_buff;
+
+/* queueing.c APIs: */
+int wg_packet_queue_init(struct crypt_queue *queue, work_func_t function,
+			 bool multicore, unsigned int len);
+void wg_packet_queue_free(struct crypt_queue *queue, bool multicore);
+struct multicore_worker __percpu *
+wg_packet_percpu_multicore_worker_alloc(work_func_t function, void *ptr);
+
+/* receive.c APIs: */
+void wg_packet_receive(struct wg_device *wg, struct sk_buff *skb);
+void wg_packet_handshake_receive_worker(struct work_struct *work);
+/* NAPI poll function: */
+int wg_packet_rx_poll(struct napi_struct *napi, int budget);
+/* Workqueue worker: */
+void wg_packet_decrypt_worker(struct work_struct *work);
+
+/* send.c APIs: */
+void wg_packet_send_queued_handshake_initiation(struct wg_peer *peer,
+						bool is_retry);
+void wg_packet_send_handshake_response(struct wg_peer *peer);
+void wg_packet_send_handshake_cookie(struct wg_device *wg,
+				     struct sk_buff *initiating_skb,
+				     __le32 sender_index);
+void wg_packet_send_keepalive(struct wg_peer *peer);
+void wg_packet_purge_staged_packets(struct wg_peer *peer);
+void wg_packet_send_staged_packets(struct wg_peer *peer);
+/* Workqueue workers: */
+void wg_packet_handshake_send_worker(struct work_struct *work);
+void wg_packet_tx_worker(struct work_struct *work);
+void wg_packet_encrypt_worker(struct work_struct *work);
+
+enum packet_state {
+	PACKET_STATE_UNCRYPTED,
+	PACKET_STATE_CRYPTED,
+	PACKET_STATE_DEAD
+};
+
+struct packet_cb {
+	u64 nonce;
+	struct noise_keypair *keypair;
+	atomic_t state;
+	u32 mtu;
+	u8 ds;
+};
+
+#define PACKET_CB(skb) ((struct packet_cb *)((skb)->cb))
+#define PACKET_PEER(skb) (PACKET_CB(skb)->keypair->entry.peer)
+
+/* Returns either the correct skb->protocol value, or 0 if invalid. */
+static inline __be16 wg_skb_examine_untrusted_ip_hdr(struct sk_buff *skb)
+{
+	if (skb_network_header(skb) >= skb->head &&
+	    (skb_network_header(skb) + sizeof(struct iphdr)) <=
+		    skb_tail_pointer(skb) &&
+	    ip_hdr(skb)->version == 4)
+		return htons(ETH_P_IP);
+	if (skb_network_header(skb) >= skb->head &&
+	    (skb_network_header(skb) + sizeof(struct ipv6hdr)) <=
+		    skb_tail_pointer(skb) &&
+	    ipv6_hdr(skb)->version == 6)
+		return htons(ETH_P_IPV6);
+	return 0;
+}
+
+static inline void wg_reset_packet(struct sk_buff *skb)
+{
+	const int pfmemalloc = skb->pfmemalloc;
+
+	skb_scrub_packet(skb, true);
+	memset(&skb->headers_start, 0,
+	       offsetof(struct sk_buff, headers_end) -
+		       offsetof(struct sk_buff, headers_start));
+	skb->pfmemalloc = pfmemalloc;
+	skb->queue_mapping = 0;
+	skb->nohdr = 0;
+	skb->peeked = 0;
+	skb->mac_len = 0;
+	skb->dev = NULL;
+#ifdef CONFIG_NET_SCHED
+	skb->tc_index = 0;
+	skb_reset_tc(skb);
+#endif
+	skb->hdr_len = skb_headroom(skb);
+	skb_reset_mac_header(skb);
+	skb_reset_network_header(skb);
+	skb_reset_transport_header(skb);
+	skb_probe_transport_header(skb);
+	skb_reset_inner_headers(skb);
+}
+
+static inline int wg_cpumask_choose_online(int *stored_cpu, unsigned int id)
+{
+	unsigned int cpu = *stored_cpu, cpu_index, i;
+
+	if (unlikely(cpu == nr_cpumask_bits ||
+		     !cpumask_test_cpu(cpu, cpu_online_mask))) {
+		cpu_index = id % cpumask_weight(cpu_online_mask);
+		cpu = cpumask_first(cpu_online_mask);
+		for (i = 0; i < cpu_index; ++i)
+			cpu = cpumask_next(cpu, cpu_online_mask);
+		*stored_cpu = cpu;
+	}
+	return cpu;
+}
+
+/* This function is racy, in the sense that next is unlocked, so it could return
+ * the same CPU twice. A race-free version of this would be to instead store an
+ * atomic sequence number, do an increment-and-return, and then iterate through
+ * every possible CPU until we get to that index -- choose_cpu. However that's
+ * a bit slower, and it doesn't seem like this potential race actually
+ * introduces any performance loss, so we live with it.
+ */
+static inline int wg_cpumask_next_online(int *next)
+{
+	int cpu = *next;
+
+	while (unlikely(!cpumask_test_cpu(cpu, cpu_online_mask)))
+		cpu = cpumask_next(cpu, cpu_online_mask) % nr_cpumask_bits;
+	*next = cpumask_next(cpu, cpu_online_mask) % nr_cpumask_bits;
+	return cpu;
+}
+
+static inline int wg_queue_enqueue_per_device_and_peer(
+	struct crypt_queue *device_queue, struct crypt_queue *peer_queue,
+	struct sk_buff *skb, struct workqueue_struct *wq, int *next_cpu)
+{
+	int cpu;
+
+	atomic_set_release(&PACKET_CB(skb)->state, PACKET_STATE_UNCRYPTED);
+	/* We first queue this up for the peer ingestion, but the consumer
+	 * will wait for the state to change to CRYPTED or DEAD before.
+	 */
+	if (unlikely(ptr_ring_produce_bh(&peer_queue->ring, skb)))
+		return -ENOSPC;
+	/* Then we queue it up in the device queue, which consumes the
+	 * packet as soon as it can.
+	 */
+	cpu = wg_cpumask_next_online(next_cpu);
+	if (unlikely(ptr_ring_produce_bh(&device_queue->ring, skb)))
+		return -EPIPE;
+	queue_work_on(cpu, wq, &per_cpu_ptr(device_queue->worker, cpu)->work);
+	return 0;
+}
+
+static inline void wg_queue_enqueue_per_peer(struct crypt_queue *queue,
+					     struct sk_buff *skb,
+					     enum packet_state state)
+{
+	/* We take a reference, because as soon as we call atomic_set, the
+	 * peer can be freed from below us.
+	 */
+	struct wg_peer *peer = wg_peer_get(PACKET_PEER(skb));
+
+	atomic_set_release(&PACKET_CB(skb)->state, state);
+	queue_work_on(wg_cpumask_choose_online(&peer->serial_work_cpu,
+					       peer->internal_id),
+		      peer->device->packet_crypt_wq, &queue->work);
+	wg_peer_put(peer);
+}
+
+static inline void wg_queue_enqueue_per_peer_napi(struct sk_buff *skb,
+						  enum packet_state state)
+{
+	/* We take a reference, because as soon as we call atomic_set, the
+	 * peer can be freed from below us.
+	 */
+	struct wg_peer *peer = wg_peer_get(PACKET_PEER(skb));
+
+	atomic_set_release(&PACKET_CB(skb)->state, state);
+	napi_schedule(&peer->napi);
+	wg_peer_put(peer);
+}
+
+#ifdef DEBUG
+bool wg_packet_counter_selftest(void);
+#endif
+
+#endif /* _WG_QUEUEING_H */
diff -Naur --no-dereference a/net/wireguard/ratelimiter.c b/net/wireguard/ratelimiter.c
--- a/net/wireguard/ratelimiter.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/ratelimiter.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,235 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#ifdef COMPAT_CANNOT_DEPRECIATE_BH_RCU
+/* We normally alias all non-_bh functions to the _bh ones in the compat layer,
+ * but that's not appropriate here, where we actually do want non-_bh ones.
+ */
+#undef synchronize_rcu
+#define synchronize_rcu old_synchronize_rcu
+#undef call_rcu
+#define call_rcu old_call_rcu
+#undef rcu_barrier
+#define rcu_barrier old_rcu_barrier
+#endif
+
+#include "ratelimiter.h"
+#include <linux/siphash.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <net/ip.h>
+
+static struct kmem_cache *entry_cache;
+static hsiphash_key_t key;
+static spinlock_t table_lock = __SPIN_LOCK_UNLOCKED("ratelimiter_table_lock");
+static DEFINE_MUTEX(init_lock);
+static u64 init_refcnt; /* Protected by init_lock, hence not atomic. */
+static atomic_t total_entries = ATOMIC_INIT(0);
+static unsigned int max_entries, table_size;
+static void wg_ratelimiter_gc_entries(struct work_struct *);
+static DECLARE_DEFERRABLE_WORK(gc_work, wg_ratelimiter_gc_entries);
+static struct hlist_head *table_v4;
+#if IS_ENABLED(CONFIG_IPV6)
+static struct hlist_head *table_v6;
+#endif
+
+struct ratelimiter_entry {
+	u64 last_time_ns, tokens, ip;
+	void *net;
+	spinlock_t lock;
+	struct hlist_node hash;
+	struct rcu_head rcu;
+};
+
+enum {
+	PACKETS_PER_SECOND = 20,
+	PACKETS_BURSTABLE = 5,
+	PACKET_COST = NSEC_PER_SEC / PACKETS_PER_SECOND,
+	TOKEN_MAX = PACKET_COST * PACKETS_BURSTABLE
+};
+
+static void entry_free(struct rcu_head *rcu)
+{
+	kmem_cache_free(entry_cache,
+			container_of(rcu, struct ratelimiter_entry, rcu));
+	atomic_dec(&total_entries);
+}
+
+static void entry_uninit(struct ratelimiter_entry *entry)
+{
+	hlist_del_rcu(&entry->hash);
+	call_rcu(&entry->rcu, entry_free);
+}
+
+/* Calling this function with a NULL work uninits all entries. */
+static void wg_ratelimiter_gc_entries(struct work_struct *work)
+{
+	const u64 now = ktime_get_coarse_boottime_ns();
+	struct ratelimiter_entry *entry;
+	struct hlist_node *temp;
+	unsigned int i;
+
+	for (i = 0; i < table_size; ++i) {
+		spin_lock(&table_lock);
+		hlist_for_each_entry_safe(entry, temp, &table_v4[i], hash) {
+			if (unlikely(!work) ||
+			    now - entry->last_time_ns > NSEC_PER_SEC)
+				entry_uninit(entry);
+		}
+#if IS_ENABLED(CONFIG_IPV6)
+		hlist_for_each_entry_safe(entry, temp, &table_v6[i], hash) {
+			if (unlikely(!work) ||
+			    now - entry->last_time_ns > NSEC_PER_SEC)
+				entry_uninit(entry);
+		}
+#endif
+		spin_unlock(&table_lock);
+		if (likely(work))
+			cond_resched();
+	}
+	if (likely(work))
+		queue_delayed_work(system_power_efficient_wq, &gc_work, HZ);
+}
+
+bool wg_ratelimiter_allow(struct sk_buff *skb, struct net *net)
+{
+	/* We only take the bottom half of the net pointer, so that we can hash
+	 * 3 words in the end. This way, siphash's len param fits into the final
+	 * u32, and we don't incur an extra round.
+	 */
+	const u32 net_word = (unsigned long)net;
+	struct ratelimiter_entry *entry;
+	struct hlist_head *bucket;
+	u64 ip;
+
+	if (skb->protocol == htons(ETH_P_IP)) {
+		ip = (u64 __force)ip_hdr(skb)->saddr;
+		bucket = &table_v4[hsiphash_2u32(net_word, ip, &key) &
+				   (table_size - 1)];
+	}
+#if IS_ENABLED(CONFIG_IPV6)
+	else if (skb->protocol == htons(ETH_P_IPV6)) {
+		/* Only use 64 bits, so as to ratelimit the whole /64. */
+		memcpy(&ip, &ipv6_hdr(skb)->saddr, sizeof(ip));
+		bucket = &table_v6[hsiphash_3u32(net_word, ip >> 32, ip, &key) &
+				   (table_size - 1)];
+	}
+#endif
+	else
+		return false;
+	rcu_read_lock();
+	hlist_for_each_entry_rcu(entry, bucket, hash) {
+		if (entry->net == net && entry->ip == ip) {
+			u64 now, tokens;
+			bool ret;
+			/* Quasi-inspired by nft_limit.c, but this is actually a
+			 * slightly different algorithm. Namely, we incorporate
+			 * the burst as part of the maximum tokens, rather than
+			 * as part of the rate.
+			 */
+			spin_lock(&entry->lock);
+			now = ktime_get_coarse_boottime_ns();
+			tokens = min_t(u64, TOKEN_MAX,
+				       entry->tokens + now -
+					       entry->last_time_ns);
+			entry->last_time_ns = now;
+			ret = tokens >= PACKET_COST;
+			entry->tokens = ret ? tokens - PACKET_COST : tokens;
+			spin_unlock(&entry->lock);
+			rcu_read_unlock();
+			return ret;
+		}
+	}
+	rcu_read_unlock();
+
+	if (atomic_inc_return(&total_entries) > max_entries)
+		goto err_oom;
+
+	entry = kmem_cache_alloc(entry_cache, GFP_KERNEL);
+	if (unlikely(!entry))
+		goto err_oom;
+
+	entry->net = net;
+	entry->ip = ip;
+	INIT_HLIST_NODE(&entry->hash);
+	spin_lock_init(&entry->lock);
+	entry->last_time_ns = ktime_get_coarse_boottime_ns();
+	entry->tokens = TOKEN_MAX - PACKET_COST;
+	spin_lock(&table_lock);
+	hlist_add_head_rcu(&entry->hash, bucket);
+	spin_unlock(&table_lock);
+	return true;
+
+err_oom:
+	atomic_dec(&total_entries);
+	return false;
+}
+
+int wg_ratelimiter_init(void)
+{
+	mutex_lock(&init_lock);
+	if (++init_refcnt != 1)
+		goto out;
+
+	entry_cache = KMEM_CACHE(ratelimiter_entry, 0);
+	if (!entry_cache)
+		goto err;
+
+	/* xt_hashlimit.c uses a slightly different algorithm for ratelimiting,
+	 * but what it shares in common is that it uses a massive hashtable. So,
+	 * we borrow their wisdom about good table sizes on different systems
+	 * dependent on RAM. This calculation here comes from there.
+	 */
+	table_size = (totalram_pages() > (1U << 30) / PAGE_SIZE) ? 8192 :
+		max_t(unsigned long, 16, roundup_pow_of_two(
+			(totalram_pages() << PAGE_SHIFT) /
+			(1U << 14) / sizeof(struct hlist_head)));
+	max_entries = table_size * 8;
+
+	table_v4 = kvzalloc(table_size * sizeof(*table_v4), GFP_KERNEL);
+	if (unlikely(!table_v4))
+		goto err_kmemcache;
+
+#if IS_ENABLED(CONFIG_IPV6)
+	table_v6 = kvzalloc(table_size * sizeof(*table_v6), GFP_KERNEL);
+	if (unlikely(!table_v6)) {
+		kvfree(table_v4);
+		goto err_kmemcache;
+	}
+#endif
+
+	queue_delayed_work(system_power_efficient_wq, &gc_work, HZ);
+	get_random_bytes(&key, sizeof(key));
+out:
+	mutex_unlock(&init_lock);
+	return 0;
+
+err_kmemcache:
+	kmem_cache_destroy(entry_cache);
+err:
+	--init_refcnt;
+	mutex_unlock(&init_lock);
+	return -ENOMEM;
+}
+
+void wg_ratelimiter_uninit(void)
+{
+	mutex_lock(&init_lock);
+	if (!init_refcnt || --init_refcnt)
+		goto out;
+
+	cancel_delayed_work_sync(&gc_work);
+	wg_ratelimiter_gc_entries(NULL);
+	rcu_barrier();
+	kvfree(table_v4);
+#if IS_ENABLED(CONFIG_IPV6)
+	kvfree(table_v6);
+#endif
+	kmem_cache_destroy(entry_cache);
+out:
+	mutex_unlock(&init_lock);
+}
+
+#include "selftest/ratelimiter.c"
diff -Naur --no-dereference a/net/wireguard/ratelimiter.h b/net/wireguard/ratelimiter.h
--- a/net/wireguard/ratelimiter.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/ratelimiter.h	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#ifndef _WG_RATELIMITER_H
+#define _WG_RATELIMITER_H
+
+#include <linux/skbuff.h>
+
+int wg_ratelimiter_init(void);
+void wg_ratelimiter_uninit(void);
+bool wg_ratelimiter_allow(struct sk_buff *skb, struct net *net);
+
+#ifdef DEBUG
+bool wg_ratelimiter_selftest(void);
+#endif
+
+#endif /* _WG_RATELIMITER_H */
diff -Naur --no-dereference a/net/wireguard/receive.c b/net/wireguard/receive.c
--- a/net/wireguard/receive.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/receive.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,605 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#include "queueing.h"
+#include "device.h"
+#include "peer.h"
+#include "timers.h"
+#include "messages.h"
+#include "cookie.h"
+#include "socket.h"
+
+#include <linux/simd.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+#include <linux/udp.h>
+#include <net/ip_tunnels.h>
+
+/* Must be called with bh disabled. */
+static void update_rx_stats(struct wg_peer *peer, size_t len)
+{
+	struct pcpu_sw_netstats *tstats =
+		get_cpu_ptr(peer->device->dev->tstats);
+
+	u64_stats_update_begin(&tstats->syncp);
+	++tstats->rx_packets;
+	tstats->rx_bytes += len;
+	peer->rx_bytes += len;
+	u64_stats_update_end(&tstats->syncp);
+	put_cpu_ptr(tstats);
+}
+
+#define SKB_TYPE_LE32(skb) (((struct message_header *)(skb)->data)->type)
+
+static size_t validate_header_len(struct sk_buff *skb)
+{
+	if (unlikely(skb->len < sizeof(struct message_header)))
+		return 0;
+	if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_DATA) &&
+	    skb->len >= MESSAGE_MINIMUM_LENGTH)
+		return sizeof(struct message_data);
+	if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION) &&
+	    skb->len == sizeof(struct message_handshake_initiation))
+		return sizeof(struct message_handshake_initiation);
+	if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE) &&
+	    skb->len == sizeof(struct message_handshake_response))
+		return sizeof(struct message_handshake_response);
+	if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE) &&
+	    skb->len == sizeof(struct message_handshake_cookie))
+		return sizeof(struct message_handshake_cookie);
+	return 0;
+}
+
+static int prepare_skb_header(struct sk_buff *skb, struct wg_device *wg)
+{
+	size_t data_offset, data_len, header_len;
+	struct udphdr *udp;
+
+	if (unlikely(wg_skb_examine_untrusted_ip_hdr(skb) != skb->protocol ||
+		     skb_transport_header(skb) < skb->head ||
+		     (skb_transport_header(skb) + sizeof(struct udphdr)) >
+			     skb_tail_pointer(skb)))
+		return -EINVAL; /* Bogus IP header */
+	udp = udp_hdr(skb);
+	data_offset = (u8 *)udp - skb->data;
+	if (unlikely(data_offset > U16_MAX ||
+		     data_offset + sizeof(struct udphdr) > skb->len))
+		/* Packet has offset at impossible location or isn't big enough
+		 * to have UDP fields.
+		 */
+		return -EINVAL;
+	data_len = ntohs(udp->len);
+	if (unlikely(data_len < sizeof(struct udphdr) ||
+		     data_len > skb->len - data_offset))
+		/* UDP packet is reporting too small of a size or lying about
+		 * its size.
+		 */
+		return -EINVAL;
+	data_len -= sizeof(struct udphdr);
+	data_offset = (u8 *)udp + sizeof(struct udphdr) - skb->data;
+	if (unlikely(!pskb_may_pull(skb,
+				data_offset + sizeof(struct message_header)) ||
+		     pskb_trim(skb, data_len + data_offset) < 0))
+		return -EINVAL;
+	skb_pull(skb, data_offset);
+	if (unlikely(skb->len != data_len))
+		/* Final len does not agree with calculated len */
+		return -EINVAL;
+	header_len = validate_header_len(skb);
+	if (unlikely(!header_len))
+		return -EINVAL;
+	__skb_push(skb, data_offset);
+	if (unlikely(!pskb_may_pull(skb, data_offset + header_len)))
+		return -EINVAL;
+	__skb_pull(skb, data_offset);
+	return 0;
+}
+
+static void wg_receive_handshake_packet(struct wg_device *wg,
+					struct sk_buff *skb)
+{
+	enum cookie_mac_state mac_state;
+	struct wg_peer *peer = NULL;
+	/* This is global, so that our load calculation applies to the whole
+	 * system. We don't care about races with it at all.
+	 */
+	static u64 last_under_load;
+	bool packet_needs_cookie;
+	bool under_load;
+
+	if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE)) {
+		net_dbg_skb_ratelimited("%s: Receiving cookie response from %pISpfsc\n",
+					wg->dev->name, skb);
+		wg_cookie_message_consume(
+			(struct message_handshake_cookie *)skb->data, wg);
+		return;
+	}
+
+	under_load = skb_queue_len(&wg->incoming_handshakes) >=
+		     MAX_QUEUED_INCOMING_HANDSHAKES / 8;
+	if (under_load)
+		last_under_load = ktime_get_coarse_boottime_ns();
+	else if (last_under_load)
+		under_load = !wg_birthdate_has_expired(last_under_load, 1);
+	mac_state = wg_cookie_validate_packet(&wg->cookie_checker, skb,
+					      under_load);
+	if ((under_load && mac_state == VALID_MAC_WITH_COOKIE) ||
+	    (!under_load && mac_state == VALID_MAC_BUT_NO_COOKIE)) {
+		packet_needs_cookie = false;
+	} else if (under_load && mac_state == VALID_MAC_BUT_NO_COOKIE) {
+		packet_needs_cookie = true;
+	} else {
+		net_dbg_skb_ratelimited("%s: Invalid MAC of handshake, dropping packet from %pISpfsc\n",
+					wg->dev->name, skb);
+		return;
+	}
+
+	switch (SKB_TYPE_LE32(skb)) {
+	case cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION): {
+		struct message_handshake_initiation *message =
+			(struct message_handshake_initiation *)skb->data;
+
+		if (packet_needs_cookie) {
+			wg_packet_send_handshake_cookie(wg, skb,
+							message->sender_index);
+			return;
+		}
+		peer = wg_noise_handshake_consume_initiation(message, wg);
+		if (unlikely(!peer)) {
+			net_dbg_skb_ratelimited("%s: Invalid handshake initiation from %pISpfsc\n",
+						wg->dev->name, skb);
+			return;
+		}
+		wg_socket_set_peer_endpoint_from_skb(peer, skb);
+		net_dbg_ratelimited("%s: Receiving handshake initiation from peer %llu (%pISpfsc)\n",
+				    wg->dev->name, peer->internal_id,
+				    &peer->endpoint.addr);
+		wg_packet_send_handshake_response(peer);
+		break;
+	}
+	case cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE): {
+		struct message_handshake_response *message =
+			(struct message_handshake_response *)skb->data;
+
+		if (packet_needs_cookie) {
+			wg_packet_send_handshake_cookie(wg, skb,
+							message->sender_index);
+			return;
+		}
+		peer = wg_noise_handshake_consume_response(message, wg);
+		if (unlikely(!peer)) {
+			net_dbg_skb_ratelimited("%s: Invalid handshake response from %pISpfsc\n",
+						wg->dev->name, skb);
+			return;
+		}
+		wg_socket_set_peer_endpoint_from_skb(peer, skb);
+		net_dbg_ratelimited("%s: Receiving handshake response from peer %llu (%pISpfsc)\n",
+				    wg->dev->name, peer->internal_id,
+				    &peer->endpoint.addr);
+		if (wg_noise_handshake_begin_session(&peer->handshake,
+						     &peer->keypairs)) {
+			wg_timers_session_derived(peer);
+			wg_timers_handshake_complete(peer);
+			/* Calling this function will either send any existing
+			 * packets in the queue and not send a keepalive, which
+			 * is the best case, Or, if there's nothing in the
+			 * queue, it will send a keepalive, in order to give
+			 * immediate confirmation of the session.
+			 */
+			wg_packet_send_keepalive(peer);
+		}
+		break;
+	}
+	}
+
+	if (unlikely(!peer)) {
+		WARN(1, "Somehow a wrong type of packet wound up in the handshake queue!\n");
+		return;
+	}
+
+	local_bh_disable();
+	update_rx_stats(peer, skb->len);
+	local_bh_enable();
+
+	wg_timers_any_authenticated_packet_received(peer);
+	wg_timers_any_authenticated_packet_traversal(peer);
+	wg_peer_put(peer);
+}
+
+void wg_packet_handshake_receive_worker(struct work_struct *work)
+{
+	struct wg_device *wg = container_of(work, struct multicore_worker,
+					    work)->ptr;
+	struct sk_buff *skb;
+
+	while ((skb = skb_dequeue(&wg->incoming_handshakes)) != NULL) {
+		wg_receive_handshake_packet(wg, skb);
+		dev_kfree_skb(skb);
+		cond_resched();
+	}
+}
+
+static void keep_key_fresh(struct wg_peer *peer)
+{
+	struct noise_keypair *keypair;
+	bool send = false;
+
+	if (peer->sent_lastminute_handshake)
+		return;
+
+	rcu_read_lock_bh();
+	keypair = rcu_dereference_bh(peer->keypairs.current_keypair);
+	if (likely(keypair && READ_ONCE(keypair->sending.is_valid)) &&
+	    keypair->i_am_the_initiator &&
+	    unlikely(wg_birthdate_has_expired(keypair->sending.birthdate,
+			REJECT_AFTER_TIME - KEEPALIVE_TIMEOUT - REKEY_TIMEOUT)))
+		send = true;
+	rcu_read_unlock_bh();
+
+	if (send) {
+		peer->sent_lastminute_handshake = true;
+		wg_packet_send_queued_handshake_initiation(peer, false);
+	}
+}
+
+static bool decrypt_packet(struct sk_buff *skb, struct noise_symmetric_key *key,
+			   simd_context_t *simd_context)
+{
+	struct scatterlist sg[MAX_SKB_FRAGS + 8];
+	struct sk_buff *trailer;
+	unsigned int offset;
+	int num_frags;
+
+	if (unlikely(!key))
+		return false;
+
+	if (unlikely(!READ_ONCE(key->is_valid) ||
+		  wg_birthdate_has_expired(key->birthdate, REJECT_AFTER_TIME) ||
+		  key->counter.receive.counter >= REJECT_AFTER_MESSAGES)) {
+		WRITE_ONCE(key->is_valid, false);
+		return false;
+	}
+
+	PACKET_CB(skb)->nonce =
+		le64_to_cpu(((struct message_data *)skb->data)->counter);
+
+	/* We ensure that the network header is part of the packet before we
+	 * call skb_cow_data, so that there's no chance that data is removed
+	 * from the skb, so that later we can extract the original endpoint.
+	 */
+	offset = skb->data - skb_network_header(skb);
+	skb_push(skb, offset);
+	num_frags = skb_cow_data(skb, 0, &trailer);
+	offset += sizeof(struct message_data);
+	skb_pull(skb, offset);
+	if (unlikely(num_frags < 0 || num_frags > ARRAY_SIZE(sg)))
+		return false;
+
+	sg_init_table(sg, num_frags);
+	if (skb_to_sgvec(skb, sg, 0, skb->len) <= 0)
+		return false;
+
+	if (!chacha20poly1305_decrypt_sg_inplace(sg, skb->len, NULL, 0,
+						 PACKET_CB(skb)->nonce, key->key,
+						 simd_context))
+		return false;
+
+	/* Another ugly situation of pushing and pulling the header so as to
+	 * keep endpoint information intact.
+	 */
+	skb_push(skb, offset);
+	if (pskb_trim(skb, skb->len - noise_encrypted_len(0)))
+		return false;
+	skb_pull(skb, offset);
+
+	return true;
+}
+
+/* This is RFC6479, a replay detection bitmap algorithm that avoids bitshifts */
+static bool counter_validate(union noise_counter *counter, u64 their_counter)
+{
+	unsigned long index, index_current, top, i;
+	bool ret = false;
+
+	spin_lock_bh(&counter->receive.lock);
+
+	if (unlikely(counter->receive.counter >= REJECT_AFTER_MESSAGES + 1 ||
+		     their_counter >= REJECT_AFTER_MESSAGES))
+		goto out;
+
+	++their_counter;
+
+	if (unlikely((COUNTER_WINDOW_SIZE + their_counter) <
+		     counter->receive.counter))
+		goto out;
+
+	index = their_counter >> ilog2(BITS_PER_LONG);
+
+	if (likely(their_counter > counter->receive.counter)) {
+		index_current = counter->receive.counter >> ilog2(BITS_PER_LONG);
+		top = min_t(unsigned long, index - index_current,
+			    COUNTER_BITS_TOTAL / BITS_PER_LONG);
+		for (i = 1; i <= top; ++i)
+			counter->receive.backtrack[(i + index_current) &
+				((COUNTER_BITS_TOTAL / BITS_PER_LONG) - 1)] = 0;
+		counter->receive.counter = their_counter;
+	}
+
+	index &= (COUNTER_BITS_TOTAL / BITS_PER_LONG) - 1;
+	ret = !test_and_set_bit(their_counter & (BITS_PER_LONG - 1),
+				&counter->receive.backtrack[index]);
+
+out:
+	spin_unlock_bh(&counter->receive.lock);
+	return ret;
+}
+
+#include "selftest/counter.c"
+
+static void wg_packet_consume_data_done(struct wg_peer *peer,
+					struct sk_buff *skb,
+					struct endpoint *endpoint)
+{
+	struct net_device *dev = peer->device->dev;
+	unsigned int len, len_before_trim;
+	struct wg_peer *routed_peer;
+
+	wg_socket_set_peer_endpoint(peer, endpoint);
+
+	if (unlikely(wg_noise_received_with_keypair(&peer->keypairs,
+						    PACKET_CB(skb)->keypair))) {
+		wg_timers_handshake_complete(peer);
+		wg_packet_send_staged_packets(peer);
+	}
+
+	keep_key_fresh(peer);
+
+	wg_timers_any_authenticated_packet_received(peer);
+	wg_timers_any_authenticated_packet_traversal(peer);
+
+	/* A packet with length 0 is a keepalive packet */
+	if (unlikely(!skb->len)) {
+		update_rx_stats(peer, message_data_len(0));
+		net_dbg_ratelimited("%s: Receiving keepalive packet from peer %llu (%pISpfsc)\n",
+				    dev->name, peer->internal_id,
+				    &peer->endpoint.addr);
+		goto packet_processed;
+	}
+
+	wg_timers_data_received(peer);
+
+	if (unlikely(skb_network_header(skb) < skb->head))
+		goto dishonest_packet_size;
+	if (unlikely(!(pskb_network_may_pull(skb, sizeof(struct iphdr)) &&
+		       (ip_hdr(skb)->version == 4 ||
+			(ip_hdr(skb)->version == 6 &&
+			 pskb_network_may_pull(skb, sizeof(struct ipv6hdr)))))))
+		goto dishonest_packet_type;
+
+	skb->dev = dev;
+	/* We've already verified the Poly1305 auth tag, which means this packet
+	 * was not modified in transit. We can therefore tell the networking
+	 * stack that all checksums of every layer of encapsulation have already
+	 * been checked "by the hardware" and therefore is unnecessary to check
+	 * again in software.
+	 */
+	skb->ip_summed = CHECKSUM_UNNECESSARY;
+#ifndef COMPAT_CANNOT_USE_CSUM_LEVEL
+	skb->csum_level = ~0; /* All levels */
+#endif
+	skb->protocol = wg_skb_examine_untrusted_ip_hdr(skb);
+	if (skb->protocol == htons(ETH_P_IP)) {
+		len = ntohs(ip_hdr(skb)->tot_len);
+		if (unlikely(len < sizeof(struct iphdr)))
+			goto dishonest_packet_size;
+		if (INET_ECN_is_ce(PACKET_CB(skb)->ds))
+			IP_ECN_set_ce(ip_hdr(skb));
+	} else if (skb->protocol == htons(ETH_P_IPV6)) {
+		len = ntohs(ipv6_hdr(skb)->payload_len) +
+		      sizeof(struct ipv6hdr);
+		if (INET_ECN_is_ce(PACKET_CB(skb)->ds))
+			IP6_ECN_set_ce(skb, ipv6_hdr(skb));
+	} else {
+		goto dishonest_packet_type;
+	}
+
+	if (unlikely(len > skb->len))
+		goto dishonest_packet_size;
+	len_before_trim = skb->len;
+	if (unlikely(pskb_trim(skb, len)))
+		goto packet_processed;
+
+	routed_peer = wg_allowedips_lookup_src(&peer->device->peer_allowedips,
+					       skb);
+	wg_peer_put(routed_peer); /* We don't need the extra reference. */
+
+	if (unlikely(routed_peer != peer))
+		goto dishonest_packet_peer;
+
+	if (unlikely(napi_gro_receive(&peer->napi, skb) == GRO_DROP)) {
+		++dev->stats.rx_dropped;
+		net_dbg_ratelimited("%s: Failed to give packet to userspace from peer %llu (%pISpfsc)\n",
+				    dev->name, peer->internal_id,
+				    &peer->endpoint.addr);
+	} else {
+		update_rx_stats(peer, message_data_len(len_before_trim));
+	}
+	return;
+
+dishonest_packet_peer:
+	net_dbg_skb_ratelimited("%s: Packet has unallowed src IP (%pISc) from peer %llu (%pISpfsc)\n",
+				dev->name, skb, peer->internal_id,
+				&peer->endpoint.addr);
+	++dev->stats.rx_errors;
+	++dev->stats.rx_frame_errors;
+	goto packet_processed;
+dishonest_packet_type:
+	net_dbg_ratelimited("%s: Packet is neither ipv4 nor ipv6 from peer %llu (%pISpfsc)\n",
+			    dev->name, peer->internal_id, &peer->endpoint.addr);
+	++dev->stats.rx_errors;
+	++dev->stats.rx_frame_errors;
+	goto packet_processed;
+dishonest_packet_size:
+	net_dbg_ratelimited("%s: Packet has incorrect size from peer %llu (%pISpfsc)\n",
+			    dev->name, peer->internal_id, &peer->endpoint.addr);
+	++dev->stats.rx_errors;
+	++dev->stats.rx_length_errors;
+	goto packet_processed;
+packet_processed:
+	dev_kfree_skb(skb);
+}
+
+int wg_packet_rx_poll(struct napi_struct *napi, int budget)
+{
+	struct wg_peer *peer = container_of(napi, struct wg_peer, napi);
+	struct crypt_queue *queue = &peer->rx_queue;
+	struct noise_keypair *keypair;
+	struct endpoint endpoint;
+	enum packet_state state;
+	struct sk_buff *skb;
+	int work_done = 0;
+	bool free;
+
+	if (unlikely(budget <= 0))
+		return 0;
+
+	while ((skb = __ptr_ring_peek(&queue->ring)) != NULL &&
+	       (state = atomic_read_acquire(&PACKET_CB(skb)->state)) !=
+		       PACKET_STATE_UNCRYPTED) {
+		__ptr_ring_discard_one(&queue->ring);
+		peer = PACKET_PEER(skb);
+		keypair = PACKET_CB(skb)->keypair;
+		free = true;
+
+		if (unlikely(state != PACKET_STATE_CRYPTED))
+			goto next;
+
+		if (unlikely(!counter_validate(&keypair->receiving.counter,
+					       PACKET_CB(skb)->nonce))) {
+			net_dbg_ratelimited("%s: Packet has invalid nonce %llu (max %llu)\n",
+					    peer->device->dev->name,
+					    PACKET_CB(skb)->nonce,
+					    keypair->receiving.counter.receive.counter);
+			goto next;
+		}
+
+		if (unlikely(wg_socket_endpoint_from_skb(&endpoint, skb)))
+			goto next;
+
+		wg_reset_packet(skb);
+		wg_packet_consume_data_done(peer, skb, &endpoint);
+		free = false;
+
+next:
+		wg_noise_keypair_put(keypair, false);
+		wg_peer_put(peer);
+		if (unlikely(free))
+			dev_kfree_skb(skb);
+
+		if (++work_done >= budget)
+			break;
+	}
+
+	if (work_done < budget)
+		napi_complete_done(napi, work_done);
+
+	return work_done;
+}
+
+void wg_packet_decrypt_worker(struct work_struct *work)
+{
+	struct crypt_queue *queue = container_of(work, struct multicore_worker,
+						 work)->ptr;
+	simd_context_t simd_context;
+	struct sk_buff *skb;
+
+	simd_get(&simd_context);
+	while ((skb = ptr_ring_consume_bh(&queue->ring)) != NULL) {
+		enum packet_state state = likely(decrypt_packet(skb,
+					   &PACKET_CB(skb)->keypair->receiving,
+					   &simd_context)) ?
+				PACKET_STATE_CRYPTED : PACKET_STATE_DEAD;
+		wg_queue_enqueue_per_peer_napi(skb, state);
+		simd_relax(&simd_context);
+	}
+
+	simd_put(&simd_context);
+}
+
+static void wg_packet_consume_data(struct wg_device *wg, struct sk_buff *skb)
+{
+	__le32 idx = ((struct message_data *)skb->data)->key_idx;
+	struct wg_peer *peer = NULL;
+	int ret;
+
+	rcu_read_lock_bh();
+	PACKET_CB(skb)->keypair =
+		(struct noise_keypair *)wg_index_hashtable_lookup(
+			wg->index_hashtable, INDEX_HASHTABLE_KEYPAIR, idx,
+			&peer);
+	if (unlikely(!wg_noise_keypair_get(PACKET_CB(skb)->keypair)))
+		goto err_keypair;
+
+	if (unlikely(READ_ONCE(peer->is_dead)))
+		goto err;
+
+	ret = wg_queue_enqueue_per_device_and_peer(&wg->decrypt_queue,
+						   &peer->rx_queue, skb,
+						   wg->packet_crypt_wq,
+						   &wg->decrypt_queue.last_cpu);
+	if (unlikely(ret == -EPIPE))
+		wg_queue_enqueue_per_peer_napi(skb, PACKET_STATE_DEAD);
+	if (likely(!ret || ret == -EPIPE)) {
+		rcu_read_unlock_bh();
+		return;
+	}
+err:
+	wg_noise_keypair_put(PACKET_CB(skb)->keypair, false);
+err_keypair:
+	rcu_read_unlock_bh();
+	wg_peer_put(peer);
+	dev_kfree_skb(skb);
+}
+
+void wg_packet_receive(struct wg_device *wg, struct sk_buff *skb)
+{
+	if (unlikely(prepare_skb_header(skb, wg) < 0))
+		goto err;
+	switch (SKB_TYPE_LE32(skb)) {
+	case cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION):
+	case cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE):
+	case cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE): {
+		int cpu;
+
+		if (skb_queue_len(&wg->incoming_handshakes) >
+			    MAX_QUEUED_INCOMING_HANDSHAKES ||
+		    unlikely(!rng_is_initialized())) {
+			net_dbg_skb_ratelimited("%s: Dropping handshake packet from %pISpfsc\n",
+						wg->dev->name, skb);
+			goto err;
+		}
+		skb_queue_tail(&wg->incoming_handshakes, skb);
+		/* Queues up a call to packet_process_queued_handshake_
+		 * packets(skb):
+		 */
+		cpu = wg_cpumask_next_online(&wg->incoming_handshake_cpu);
+		queue_work_on(cpu, wg->handshake_receive_wq,
+			&per_cpu_ptr(wg->incoming_handshakes_worker, cpu)->work);
+		break;
+	}
+	case cpu_to_le32(MESSAGE_DATA):
+		PACKET_CB(skb)->ds = ip_tunnel_get_dsfield(ip_hdr(skb), skb);
+		wg_packet_consume_data(wg, skb);
+		break;
+	default:
+		net_dbg_skb_ratelimited("%s: Invalid packet from %pISpfsc\n",
+					wg->dev->name, skb);
+		goto err;
+	}
+	return;
+
+err:
+	dev_kfree_skb(skb);
+}
diff -Naur --no-dereference a/net/wireguard/selftest/allowedips.c b/net/wireguard/selftest/allowedips.c
--- a/net/wireguard/selftest/allowedips.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/selftest/allowedips.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,683 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ *
+ * This contains some basic static unit tests for the allowedips data structure.
+ * It also has two additional modes that are disabled and meant to be used by
+ * folks directly playing with this file. If you define the macro
+ * DEBUG_PRINT_TRIE_GRAPHVIZ to be 1, then every time there's a full tree in
+ * memory, it will be printed out as KERN_DEBUG in a format that can be passed
+ * to graphviz (the dot command) to visualize it. If you define the macro
+ * DEBUG_RANDOM_TRIE to be 1, then there will be an extremely costly set of
+ * randomized tests done against a trivial implementation, which may take
+ * upwards of a half-hour to complete. There's no set of users who should be
+ * enabling these, and the only developers that should go anywhere near these
+ * nobs are the ones who are reading this comment.
+ */
+
+#ifdef DEBUG
+
+#include <linux/siphash.h>
+
+static __init void swap_endian_and_apply_cidr(u8 *dst, const u8 *src, u8 bits,
+					      u8 cidr)
+{
+	swap_endian(dst, src, bits);
+	memset(dst + (cidr + 7) / 8, 0, bits / 8 - (cidr + 7) / 8);
+	if (cidr)
+		dst[(cidr + 7) / 8 - 1] &= ~0U << ((8 - (cidr % 8)) % 8);
+}
+
+static __init void print_node(struct allowedips_node *node, u8 bits)
+{
+	char *fmt_connection = KERN_DEBUG "\t\"%p/%d\" -> \"%p/%d\";\n";
+	char *fmt_declaration = KERN_DEBUG
+		"\t\"%p/%d\"[style=%s, color=\"#%06x\"];\n";
+	char *style = "dotted";
+	u8 ip1[16], ip2[16];
+	u32 color = 0;
+
+	if (bits == 32) {
+		fmt_connection = KERN_DEBUG "\t\"%pI4/%d\" -> \"%pI4/%d\";\n";
+		fmt_declaration = KERN_DEBUG
+			"\t\"%pI4/%d\"[style=%s, color=\"#%06x\"];\n";
+	} else if (bits == 128) {
+		fmt_connection = KERN_DEBUG "\t\"%pI6/%d\" -> \"%pI6/%d\";\n";
+		fmt_declaration = KERN_DEBUG
+			"\t\"%pI6/%d\"[style=%s, color=\"#%06x\"];\n";
+	}
+	if (node->peer) {
+		hsiphash_key_t key = { { 0 } };
+
+		memcpy(&key, &node->peer, sizeof(node->peer));
+		color = hsiphash_1u32(0xdeadbeef, &key) % 200 << 16 |
+			hsiphash_1u32(0xbabecafe, &key) % 200 << 8 |
+			hsiphash_1u32(0xabad1dea, &key) % 200;
+		style = "bold";
+	}
+	swap_endian_and_apply_cidr(ip1, node->bits, bits, node->cidr);
+	printk(fmt_declaration, ip1, node->cidr, style, color);
+	if (node->bit[0]) {
+		swap_endian_and_apply_cidr(ip2,
+				rcu_dereference_raw(node->bit[0])->bits, bits,
+				node->cidr);
+		printk(fmt_connection, ip1, node->cidr, ip2,
+		       rcu_dereference_raw(node->bit[0])->cidr);
+		print_node(rcu_dereference_raw(node->bit[0]), bits);
+	}
+	if (node->bit[1]) {
+		swap_endian_and_apply_cidr(ip2,
+				rcu_dereference_raw(node->bit[1])->bits,
+				bits, node->cidr);
+		printk(fmt_connection, ip1, node->cidr, ip2,
+		       rcu_dereference_raw(node->bit[1])->cidr);
+		print_node(rcu_dereference_raw(node->bit[1]), bits);
+	}
+}
+
+static __init void print_tree(struct allowedips_node __rcu *top, u8 bits)
+{
+	printk(KERN_DEBUG "digraph trie {\n");
+	print_node(rcu_dereference_raw(top), bits);
+	printk(KERN_DEBUG "}\n");
+}
+
+enum {
+	NUM_PEERS = 2000,
+	NUM_RAND_ROUTES = 400,
+	NUM_MUTATED_ROUTES = 100,
+	NUM_QUERIES = NUM_RAND_ROUTES * NUM_MUTATED_ROUTES * 30
+};
+
+struct horrible_allowedips {
+	struct hlist_head head;
+};
+
+struct horrible_allowedips_node {
+	struct hlist_node table;
+	union nf_inet_addr ip;
+	union nf_inet_addr mask;
+	u8 ip_version;
+	void *value;
+};
+
+static __init void horrible_allowedips_init(struct horrible_allowedips *table)
+{
+	INIT_HLIST_HEAD(&table->head);
+}
+
+static __init void horrible_allowedips_free(struct horrible_allowedips *table)
+{
+	struct horrible_allowedips_node *node;
+	struct hlist_node *h;
+
+	hlist_for_each_entry_safe(node, h, &table->head, table) {
+		hlist_del(&node->table);
+		kfree(node);
+	}
+}
+
+static __init inline union nf_inet_addr horrible_cidr_to_mask(u8 cidr)
+{
+	union nf_inet_addr mask;
+
+	memset(&mask, 0x00, 128 / 8);
+	memset(&mask, 0xff, cidr / 8);
+	if (cidr % 32)
+		mask.all[cidr / 32] = (__force u32)htonl(
+			(0xFFFFFFFFUL << (32 - (cidr % 32))) & 0xFFFFFFFFUL);
+	return mask;
+}
+
+static __init inline u8 horrible_mask_to_cidr(union nf_inet_addr subnet)
+{
+	return hweight32(subnet.all[0]) + hweight32(subnet.all[1]) +
+	       hweight32(subnet.all[2]) + hweight32(subnet.all[3]);
+}
+
+static __init inline void
+horrible_mask_self(struct horrible_allowedips_node *node)
+{
+	if (node->ip_version == 4) {
+		node->ip.ip &= node->mask.ip;
+	} else if (node->ip_version == 6) {
+		node->ip.ip6[0] &= node->mask.ip6[0];
+		node->ip.ip6[1] &= node->mask.ip6[1];
+		node->ip.ip6[2] &= node->mask.ip6[2];
+		node->ip.ip6[3] &= node->mask.ip6[3];
+	}
+}
+
+static __init inline bool
+horrible_match_v4(const struct horrible_allowedips_node *node,
+		  struct in_addr *ip)
+{
+	return (ip->s_addr & node->mask.ip) == node->ip.ip;
+}
+
+static __init inline bool
+horrible_match_v6(const struct horrible_allowedips_node *node,
+		  struct in6_addr *ip)
+{
+	return (ip->in6_u.u6_addr32[0] & node->mask.ip6[0]) ==
+		       node->ip.ip6[0] &&
+	       (ip->in6_u.u6_addr32[1] & node->mask.ip6[1]) ==
+		       node->ip.ip6[1] &&
+	       (ip->in6_u.u6_addr32[2] & node->mask.ip6[2]) ==
+		       node->ip.ip6[2] &&
+	       (ip->in6_u.u6_addr32[3] & node->mask.ip6[3]) == node->ip.ip6[3];
+}
+
+static __init void
+horrible_insert_ordered(struct horrible_allowedips *table,
+			struct horrible_allowedips_node *node)
+{
+	struct horrible_allowedips_node *other = NULL, *where = NULL;
+	u8 my_cidr = horrible_mask_to_cidr(node->mask);
+
+	hlist_for_each_entry(other, &table->head, table) {
+		if (!memcmp(&other->mask, &node->mask,
+			    sizeof(union nf_inet_addr)) &&
+		    !memcmp(&other->ip, &node->ip,
+			    sizeof(union nf_inet_addr)) &&
+		    other->ip_version == node->ip_version) {
+			other->value = node->value;
+			kfree(node);
+			return;
+		}
+		where = other;
+		if (horrible_mask_to_cidr(other->mask) <= my_cidr)
+			break;
+	}
+	if (!other && !where)
+		hlist_add_head(&node->table, &table->head);
+	else if (!other)
+		hlist_add_behind(&node->table, &where->table);
+	else
+		hlist_add_before(&node->table, &where->table);
+}
+
+static __init int
+horrible_allowedips_insert_v4(struct horrible_allowedips *table,
+			      struct in_addr *ip, u8 cidr, void *value)
+{
+	struct horrible_allowedips_node *node = kzalloc(sizeof(*node),
+							GFP_KERNEL);
+
+	if (unlikely(!node))
+		return -ENOMEM;
+	node->ip.in = *ip;
+	node->mask = horrible_cidr_to_mask(cidr);
+	node->ip_version = 4;
+	node->value = value;
+	horrible_mask_self(node);
+	horrible_insert_ordered(table, node);
+	return 0;
+}
+
+static __init int
+horrible_allowedips_insert_v6(struct horrible_allowedips *table,
+			      struct in6_addr *ip, u8 cidr, void *value)
+{
+	struct horrible_allowedips_node *node = kzalloc(sizeof(*node),
+							GFP_KERNEL);
+
+	if (unlikely(!node))
+		return -ENOMEM;
+	node->ip.in6 = *ip;
+	node->mask = horrible_cidr_to_mask(cidr);
+	node->ip_version = 6;
+	node->value = value;
+	horrible_mask_self(node);
+	horrible_insert_ordered(table, node);
+	return 0;
+}
+
+static __init void *
+horrible_allowedips_lookup_v4(struct horrible_allowedips *table,
+			      struct in_addr *ip)
+{
+	struct horrible_allowedips_node *node;
+	void *ret = NULL;
+
+	hlist_for_each_entry(node, &table->head, table) {
+		if (node->ip_version != 4)
+			continue;
+		if (horrible_match_v4(node, ip)) {
+			ret = node->value;
+			break;
+		}
+	}
+	return ret;
+}
+
+static __init void *
+horrible_allowedips_lookup_v6(struct horrible_allowedips *table,
+			      struct in6_addr *ip)
+{
+	struct horrible_allowedips_node *node;
+	void *ret = NULL;
+
+	hlist_for_each_entry(node, &table->head, table) {
+		if (node->ip_version != 6)
+			continue;
+		if (horrible_match_v6(node, ip)) {
+			ret = node->value;
+			break;
+		}
+	}
+	return ret;
+}
+
+static __init bool randomized_test(void)
+{
+	unsigned int i, j, k, mutate_amount, cidr;
+	u8 ip[16], mutate_mask[16], mutated[16];
+	struct wg_peer **peers, *peer;
+	struct horrible_allowedips h;
+	DEFINE_MUTEX(mutex);
+	struct allowedips t;
+	bool ret = false;
+
+	mutex_init(&mutex);
+
+	wg_allowedips_init(&t);
+	horrible_allowedips_init(&h);
+
+	peers = kcalloc(NUM_PEERS, sizeof(*peers), GFP_KERNEL);
+	if (unlikely(!peers)) {
+		pr_err("allowedips random self-test malloc: FAIL\n");
+		goto free;
+	}
+	for (i = 0; i < NUM_PEERS; ++i) {
+		peers[i] = kzalloc(sizeof(*peers[i]), GFP_KERNEL);
+		if (unlikely(!peers[i])) {
+			pr_err("allowedips random self-test malloc: FAIL\n");
+			goto free;
+		}
+		kref_init(&peers[i]->refcount);
+	}
+
+	mutex_lock(&mutex);
+
+	for (i = 0; i < NUM_RAND_ROUTES; ++i) {
+		prandom_bytes(ip, 4);
+		cidr = prandom_u32_max(32) + 1;
+		peer = peers[prandom_u32_max(NUM_PEERS)];
+		if (wg_allowedips_insert_v4(&t, (struct in_addr *)ip, cidr,
+					    peer, &mutex) < 0) {
+			pr_err("allowedips random self-test malloc: FAIL\n");
+			goto free_locked;
+		}
+		if (horrible_allowedips_insert_v4(&h, (struct in_addr *)ip,
+						  cidr, peer) < 0) {
+			pr_err("allowedips random self-test malloc: FAIL\n");
+			goto free_locked;
+		}
+		for (j = 0; j < NUM_MUTATED_ROUTES; ++j) {
+			memcpy(mutated, ip, 4);
+			prandom_bytes(mutate_mask, 4);
+			mutate_amount = prandom_u32_max(32);
+			for (k = 0; k < mutate_amount / 8; ++k)
+				mutate_mask[k] = 0xff;
+			mutate_mask[k] = 0xff
+					 << ((8 - (mutate_amount % 8)) % 8);
+			for (; k < 4; ++k)
+				mutate_mask[k] = 0;
+			for (k = 0; k < 4; ++k)
+				mutated[k] = (mutated[k] & mutate_mask[k]) |
+					     (~mutate_mask[k] &
+					      prandom_u32_max(256));
+			cidr = prandom_u32_max(32) + 1;
+			peer = peers[prandom_u32_max(NUM_PEERS)];
+			if (wg_allowedips_insert_v4(&t,
+						    (struct in_addr *)mutated,
+						    cidr, peer, &mutex) < 0) {
+				pr_err("allowedips random malloc: FAIL\n");
+				goto free_locked;
+			}
+			if (horrible_allowedips_insert_v4(&h,
+				(struct in_addr *)mutated, cidr, peer)) {
+				pr_err("allowedips random self-test malloc: FAIL\n");
+				goto free_locked;
+			}
+		}
+	}
+
+	for (i = 0; i < NUM_RAND_ROUTES; ++i) {
+		prandom_bytes(ip, 16);
+		cidr = prandom_u32_max(128) + 1;
+		peer = peers[prandom_u32_max(NUM_PEERS)];
+		if (wg_allowedips_insert_v6(&t, (struct in6_addr *)ip, cidr,
+					    peer, &mutex) < 0) {
+			pr_err("allowedips random self-test malloc: FAIL\n");
+			goto free_locked;
+		}
+		if (horrible_allowedips_insert_v6(&h, (struct in6_addr *)ip,
+						  cidr, peer) < 0) {
+			pr_err("allowedips random self-test malloc: FAIL\n");
+			goto free_locked;
+		}
+		for (j = 0; j < NUM_MUTATED_ROUTES; ++j) {
+			memcpy(mutated, ip, 16);
+			prandom_bytes(mutate_mask, 16);
+			mutate_amount = prandom_u32_max(128);
+			for (k = 0; k < mutate_amount / 8; ++k)
+				mutate_mask[k] = 0xff;
+			mutate_mask[k] = 0xff
+					 << ((8 - (mutate_amount % 8)) % 8);
+			for (; k < 4; ++k)
+				mutate_mask[k] = 0;
+			for (k = 0; k < 4; ++k)
+				mutated[k] = (mutated[k] & mutate_mask[k]) |
+					     (~mutate_mask[k] &
+					      prandom_u32_max(256));
+			cidr = prandom_u32_max(128) + 1;
+			peer = peers[prandom_u32_max(NUM_PEERS)];
+			if (wg_allowedips_insert_v6(&t,
+						    (struct in6_addr *)mutated,
+						    cidr, peer, &mutex) < 0) {
+				pr_err("allowedips random self-test malloc: FAIL\n");
+				goto free_locked;
+			}
+			if (horrible_allowedips_insert_v6(
+				    &h, (struct in6_addr *)mutated, cidr,
+				    peer)) {
+				pr_err("allowedips random self-test malloc: FAIL\n");
+				goto free_locked;
+			}
+		}
+	}
+
+	mutex_unlock(&mutex);
+
+	if (IS_ENABLED(DEBUG_PRINT_TRIE_GRAPHVIZ)) {
+		print_tree(t.root4, 32);
+		print_tree(t.root6, 128);
+	}
+
+	for (i = 0; i < NUM_QUERIES; ++i) {
+		prandom_bytes(ip, 4);
+		if (lookup(t.root4, 32, ip) !=
+		    horrible_allowedips_lookup_v4(&h, (struct in_addr *)ip)) {
+			pr_err("allowedips random self-test: FAIL\n");
+			goto free;
+		}
+	}
+
+	for (i = 0; i < NUM_QUERIES; ++i) {
+		prandom_bytes(ip, 16);
+		if (lookup(t.root6, 128, ip) !=
+		    horrible_allowedips_lookup_v6(&h, (struct in6_addr *)ip)) {
+			pr_err("allowedips random self-test: FAIL\n");
+			goto free;
+		}
+	}
+	ret = true;
+
+free:
+	mutex_lock(&mutex);
+free_locked:
+	wg_allowedips_free(&t, &mutex);
+	mutex_unlock(&mutex);
+	horrible_allowedips_free(&h);
+	if (peers) {
+		for (i = 0; i < NUM_PEERS; ++i)
+			kfree(peers[i]);
+	}
+	kfree(peers);
+	return ret;
+}
+
+static __init inline struct in_addr *ip4(u8 a, u8 b, u8 c, u8 d)
+{
+	static struct in_addr ip;
+	u8 *split = (u8 *)&ip;
+
+	split[0] = a;
+	split[1] = b;
+	split[2] = c;
+	split[3] = d;
+	return &ip;
+}
+
+static __init inline struct in6_addr *ip6(u32 a, u32 b, u32 c, u32 d)
+{
+	static struct in6_addr ip;
+	__be32 *split = (__be32 *)&ip;
+
+	split[0] = cpu_to_be32(a);
+	split[1] = cpu_to_be32(b);
+	split[2] = cpu_to_be32(c);
+	split[3] = cpu_to_be32(d);
+	return &ip;
+}
+
+static __init struct wg_peer *init_peer(void)
+{
+	struct wg_peer *peer = kzalloc(sizeof(*peer), GFP_KERNEL);
+
+	if (!peer)
+		return NULL;
+	kref_init(&peer->refcount);
+	INIT_LIST_HEAD(&peer->allowedips_list);
+	return peer;
+}
+
+#define insert(version, mem, ipa, ipb, ipc, ipd, cidr)                       \
+	wg_allowedips_insert_v##version(&t, ip##version(ipa, ipb, ipc, ipd), \
+					cidr, mem, &mutex)
+
+#define maybe_fail() do {                                               \
+		++i;                                                    \
+		if (!_s) {                                              \
+			pr_info("allowedips self-test %zu: FAIL\n", i); \
+			success = false;                                \
+		}                                                       \
+	} while (0)
+
+#define test(version, mem, ipa, ipb, ipc, ipd) do {                          \
+		bool _s = lookup(t.root##version, (version) == 4 ? 32 : 128, \
+				 ip##version(ipa, ipb, ipc, ipd)) == (mem);  \
+		maybe_fail();                                                \
+	} while (0)
+
+#define test_negative(version, mem, ipa, ipb, ipc, ipd) do {                 \
+		bool _s = lookup(t.root##version, (version) == 4 ? 32 : 128, \
+				 ip##version(ipa, ipb, ipc, ipd)) != (mem);  \
+		maybe_fail();                                                \
+	} while (0)
+
+#define test_boolean(cond) do {   \
+		bool _s = (cond); \
+		maybe_fail();     \
+	} while (0)
+
+bool __init wg_allowedips_selftest(void)
+{
+	bool found_a = false, found_b = false, found_c = false, found_d = false,
+	     found_e = false, found_other = false;
+	struct wg_peer *a = init_peer(), *b = init_peer(), *c = init_peer(),
+		       *d = init_peer(), *e = init_peer(), *f = init_peer(),
+		       *g = init_peer(), *h = init_peer();
+	struct allowedips_node *iter_node;
+	bool success = false;
+	struct allowedips t;
+	DEFINE_MUTEX(mutex);
+	struct in6_addr ip;
+	size_t i = 0, count = 0;
+	__be64 part;
+
+	mutex_init(&mutex);
+	mutex_lock(&mutex);
+	wg_allowedips_init(&t);
+
+	if (!a || !b || !c || !d || !e || !f || !g || !h) {
+		pr_err("allowedips self-test malloc: FAIL\n");
+		goto free;
+	}
+
+	insert(4, a, 192, 168, 4, 0, 24);
+	insert(4, b, 192, 168, 4, 4, 32);
+	insert(4, c, 192, 168, 0, 0, 16);
+	insert(4, d, 192, 95, 5, 64, 27);
+	/* replaces previous entry, and maskself is required */
+	insert(4, c, 192, 95, 5, 65, 27);
+	insert(6, d, 0x26075300, 0x60006b00, 0, 0xc05f0543, 128);
+	insert(6, c, 0x26075300, 0x60006b00, 0, 0, 64);
+	insert(4, e, 0, 0, 0, 0, 0);
+	insert(6, e, 0, 0, 0, 0, 0);
+	/* replaces previous entry */
+	insert(6, f, 0, 0, 0, 0, 0);
+	insert(6, g, 0x24046800, 0, 0, 0, 32);
+	/* maskself is required */
+	insert(6, h, 0x24046800, 0x40040800, 0xdeadbeef, 0xdeadbeef, 64);
+	insert(6, a, 0x24046800, 0x40040800, 0xdeadbeef, 0xdeadbeef, 128);
+	insert(6, c, 0x24446800, 0x40e40800, 0xdeaebeef, 0xdefbeef, 128);
+	insert(6, b, 0x24446800, 0xf0e40800, 0xeeaebeef, 0, 98);
+	insert(4, g, 64, 15, 112, 0, 20);
+	/* maskself is required */
+	insert(4, h, 64, 15, 123, 211, 25);
+	insert(4, a, 10, 0, 0, 0, 25);
+	insert(4, b, 10, 0, 0, 128, 25);
+	insert(4, a, 10, 1, 0, 0, 30);
+	insert(4, b, 10, 1, 0, 4, 30);
+	insert(4, c, 10, 1, 0, 8, 29);
+	insert(4, d, 10, 1, 0, 16, 29);
+
+	if (IS_ENABLED(DEBUG_PRINT_TRIE_GRAPHVIZ)) {
+		print_tree(t.root4, 32);
+		print_tree(t.root6, 128);
+	}
+
+	success = true;
+
+	test(4, a, 192, 168, 4, 20);
+	test(4, a, 192, 168, 4, 0);
+	test(4, b, 192, 168, 4, 4);
+	test(4, c, 192, 168, 200, 182);
+	test(4, c, 192, 95, 5, 68);
+	test(4, e, 192, 95, 5, 96);
+	test(6, d, 0x26075300, 0x60006b00, 0, 0xc05f0543);
+	test(6, c, 0x26075300, 0x60006b00, 0, 0xc02e01ee);
+	test(6, f, 0x26075300, 0x60006b01, 0, 0);
+	test(6, g, 0x24046800, 0x40040806, 0, 0x1006);
+	test(6, g, 0x24046800, 0x40040806, 0x1234, 0x5678);
+	test(6, f, 0x240467ff, 0x40040806, 0x1234, 0x5678);
+	test(6, f, 0x24046801, 0x40040806, 0x1234, 0x5678);
+	test(6, h, 0x24046800, 0x40040800, 0x1234, 0x5678);
+	test(6, h, 0x24046800, 0x40040800, 0, 0);
+	test(6, h, 0x24046800, 0x40040800, 0x10101010, 0x10101010);
+	test(6, a, 0x24046800, 0x40040800, 0xdeadbeef, 0xdeadbeef);
+	test(4, g, 64, 15, 116, 26);
+	test(4, g, 64, 15, 127, 3);
+	test(4, g, 64, 15, 123, 1);
+	test(4, h, 64, 15, 123, 128);
+	test(4, h, 64, 15, 123, 129);
+	test(4, a, 10, 0, 0, 52);
+	test(4, b, 10, 0, 0, 220);
+	test(4, a, 10, 1, 0, 2);
+	test(4, b, 10, 1, 0, 6);
+	test(4, c, 10, 1, 0, 10);
+	test(4, d, 10, 1, 0, 20);
+
+	insert(4, a, 1, 0, 0, 0, 32);
+	insert(4, a, 64, 0, 0, 0, 32);
+	insert(4, a, 128, 0, 0, 0, 32);
+	insert(4, a, 192, 0, 0, 0, 32);
+	insert(4, a, 255, 0, 0, 0, 32);
+	wg_allowedips_remove_by_peer(&t, a, &mutex);
+	test_negative(4, a, 1, 0, 0, 0);
+	test_negative(4, a, 64, 0, 0, 0);
+	test_negative(4, a, 128, 0, 0, 0);
+	test_negative(4, a, 192, 0, 0, 0);
+	test_negative(4, a, 255, 0, 0, 0);
+
+	wg_allowedips_free(&t, &mutex);
+	wg_allowedips_init(&t);
+	insert(4, a, 192, 168, 0, 0, 16);
+	insert(4, a, 192, 168, 0, 0, 24);
+	wg_allowedips_remove_by_peer(&t, a, &mutex);
+	test_negative(4, a, 192, 168, 0, 1);
+
+	/* These will hit the WARN_ON(len >= 128) in free_node if something
+	 * goes wrong.
+	 */
+	for (i = 0; i < 128; ++i) {
+		part = cpu_to_be64(~(1LLU << (i % 64)));
+		memset(&ip, 0xff, 16);
+		memcpy((u8 *)&ip + (i < 64) * 8, &part, 8);
+		wg_allowedips_insert_v6(&t, &ip, 128, a, &mutex);
+	}
+
+	wg_allowedips_free(&t, &mutex);
+
+	wg_allowedips_init(&t);
+	insert(4, a, 192, 95, 5, 93, 27);
+	insert(6, a, 0x26075300, 0x60006b00, 0, 0xc05f0543, 128);
+	insert(4, a, 10, 1, 0, 20, 29);
+	insert(6, a, 0x26075300, 0x6d8a6bf8, 0xdab1f1df, 0xc05f1523, 83);
+	insert(6, a, 0x26075300, 0x6d8a6bf8, 0xdab1f1df, 0xc05f1523, 21);
+	list_for_each_entry(iter_node, &a->allowedips_list, peer_list) {
+		u8 cidr, ip[16] __aligned(__alignof(u64));
+		int family = wg_allowedips_read_node(iter_node, ip, &cidr);
+
+		count++;
+
+		if (cidr == 27 && family == AF_INET &&
+		    !memcmp(ip, ip4(192, 95, 5, 64), sizeof(struct in_addr)))
+			found_a = true;
+		else if (cidr == 128 && family == AF_INET6 &&
+			 !memcmp(ip, ip6(0x26075300, 0x60006b00, 0, 0xc05f0543),
+				 sizeof(struct in6_addr)))
+			found_b = true;
+		else if (cidr == 29 && family == AF_INET &&
+			 !memcmp(ip, ip4(10, 1, 0, 16), sizeof(struct in_addr)))
+			found_c = true;
+		else if (cidr == 83 && family == AF_INET6 &&
+			 !memcmp(ip, ip6(0x26075300, 0x6d8a6bf8, 0xdab1e000, 0),
+				 sizeof(struct in6_addr)))
+			found_d = true;
+		else if (cidr == 21 && family == AF_INET6 &&
+			 !memcmp(ip, ip6(0x26075000, 0, 0, 0),
+				 sizeof(struct in6_addr)))
+			found_e = true;
+		else
+			found_other = true;
+	}
+	test_boolean(count == 5);
+	test_boolean(found_a);
+	test_boolean(found_b);
+	test_boolean(found_c);
+	test_boolean(found_d);
+	test_boolean(found_e);
+	test_boolean(!found_other);
+
+	if (IS_ENABLED(DEBUG_RANDOM_TRIE) && success)
+		success = randomized_test();
+
+	if (success)
+		pr_info("allowedips self-tests: pass\n");
+
+free:
+	wg_allowedips_free(&t, &mutex);
+	kfree(a);
+	kfree(b);
+	kfree(c);
+	kfree(d);
+	kfree(e);
+	kfree(f);
+	kfree(g);
+	kfree(h);
+	mutex_unlock(&mutex);
+
+	return success;
+}
+
+#undef test_negative
+#undef test
+#undef remove
+#undef insert
+#undef init_peer
+
+#endif
diff -Naur --no-dereference a/net/wireguard/selftest/counter.c b/net/wireguard/selftest/counter.c
--- a/net/wireguard/selftest/counter.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/selftest/counter.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,104 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#ifdef DEBUG
+bool __init wg_packet_counter_selftest(void)
+{
+	unsigned int test_num = 0, i;
+	union noise_counter counter;
+	bool success = true;
+
+#define T_INIT do {                                               \
+		memset(&counter, 0, sizeof(union noise_counter)); \
+		spin_lock_init(&counter.receive.lock);            \
+	} while (0)
+#define T_LIM (COUNTER_WINDOW_SIZE + 1)
+#define T(n, v) do {                                                  \
+		++test_num;                                           \
+		if (counter_validate(&counter, n) != (v)) {           \
+			pr_err("nonce counter self-test %u: FAIL\n",  \
+			       test_num);                             \
+			success = false;                              \
+		}                                                     \
+	} while (0)
+
+	T_INIT;
+	/*  1 */ T(0, true);
+	/*  2 */ T(1, true);
+	/*  3 */ T(1, false);
+	/*  4 */ T(9, true);
+	/*  5 */ T(8, true);
+	/*  6 */ T(7, true);
+	/*  7 */ T(7, false);
+	/*  8 */ T(T_LIM, true);
+	/*  9 */ T(T_LIM - 1, true);
+	/* 10 */ T(T_LIM - 1, false);
+	/* 11 */ T(T_LIM - 2, true);
+	/* 12 */ T(2, true);
+	/* 13 */ T(2, false);
+	/* 14 */ T(T_LIM + 16, true);
+	/* 15 */ T(3, false);
+	/* 16 */ T(T_LIM + 16, false);
+	/* 17 */ T(T_LIM * 4, true);
+	/* 18 */ T(T_LIM * 4 - (T_LIM - 1), true);
+	/* 19 */ T(10, false);
+	/* 20 */ T(T_LIM * 4 - T_LIM, false);
+	/* 21 */ T(T_LIM * 4 - (T_LIM + 1), false);
+	/* 22 */ T(T_LIM * 4 - (T_LIM - 2), true);
+	/* 23 */ T(T_LIM * 4 + 1 - T_LIM, false);
+	/* 24 */ T(0, false);
+	/* 25 */ T(REJECT_AFTER_MESSAGES, false);
+	/* 26 */ T(REJECT_AFTER_MESSAGES - 1, true);
+	/* 27 */ T(REJECT_AFTER_MESSAGES, false);
+	/* 28 */ T(REJECT_AFTER_MESSAGES - 1, false);
+	/* 29 */ T(REJECT_AFTER_MESSAGES - 2, true);
+	/* 30 */ T(REJECT_AFTER_MESSAGES + 1, false);
+	/* 31 */ T(REJECT_AFTER_MESSAGES + 2, false);
+	/* 32 */ T(REJECT_AFTER_MESSAGES - 2, false);
+	/* 33 */ T(REJECT_AFTER_MESSAGES - 3, true);
+	/* 34 */ T(0, false);
+
+	T_INIT;
+	for (i = 1; i <= COUNTER_WINDOW_SIZE; ++i)
+		T(i, true);
+	T(0, true);
+	T(0, false);
+
+	T_INIT;
+	for (i = 2; i <= COUNTER_WINDOW_SIZE + 1; ++i)
+		T(i, true);
+	T(1, true);
+	T(0, false);
+
+	T_INIT;
+	for (i = COUNTER_WINDOW_SIZE + 1; i-- > 0;)
+		T(i, true);
+
+	T_INIT;
+	for (i = COUNTER_WINDOW_SIZE + 2; i-- > 1;)
+		T(i, true);
+	T(0, false);
+
+	T_INIT;
+	for (i = COUNTER_WINDOW_SIZE + 1; i-- > 1;)
+		T(i, true);
+	T(COUNTER_WINDOW_SIZE + 1, true);
+	T(0, false);
+
+	T_INIT;
+	for (i = COUNTER_WINDOW_SIZE + 1; i-- > 1;)
+		T(i, true);
+	T(0, true);
+	T(COUNTER_WINDOW_SIZE + 1, true);
+
+#undef T
+#undef T_LIM
+#undef T_INIT
+
+	if (success)
+		pr_info("nonce counter self-tests: pass\n");
+	return success;
+}
+#endif
diff -Naur --no-dereference a/net/wireguard/selftest/ratelimiter.c b/net/wireguard/selftest/ratelimiter.c
--- a/net/wireguard/selftest/ratelimiter.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/selftest/ratelimiter.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,226 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#ifdef DEBUG
+
+#include <linux/jiffies.h>
+
+static const struct {
+	bool result;
+	unsigned int msec_to_sleep_before;
+} expected_results[] __initconst = {
+	[0 ... PACKETS_BURSTABLE - 1] = { true, 0 },
+	[PACKETS_BURSTABLE] = { false, 0 },
+	[PACKETS_BURSTABLE + 1] = { true, MSEC_PER_SEC / PACKETS_PER_SECOND },
+	[PACKETS_BURSTABLE + 2] = { false, 0 },
+	[PACKETS_BURSTABLE + 3] = { true, (MSEC_PER_SEC / PACKETS_PER_SECOND) * 2 },
+	[PACKETS_BURSTABLE + 4] = { true, 0 },
+	[PACKETS_BURSTABLE + 5] = { false, 0 }
+};
+
+static __init unsigned int maximum_jiffies_at_index(int index)
+{
+	unsigned int total_msecs = 2 * MSEC_PER_SEC / PACKETS_PER_SECOND / 3;
+	int i;
+
+	for (i = 0; i <= index; ++i)
+		total_msecs += expected_results[i].msec_to_sleep_before;
+	return msecs_to_jiffies(total_msecs);
+}
+
+static __init int timings_test(struct sk_buff *skb4, struct iphdr *hdr4,
+			       struct sk_buff *skb6, struct ipv6hdr *hdr6,
+			       int *test)
+{
+	unsigned long loop_start_time;
+	int i;
+
+	wg_ratelimiter_gc_entries(NULL);
+	rcu_barrier();
+	loop_start_time = jiffies;
+
+	for (i = 0; i < ARRAY_SIZE(expected_results); ++i) {
+		if (expected_results[i].msec_to_sleep_before)
+			msleep(expected_results[i].msec_to_sleep_before);
+
+		if (time_is_before_jiffies(loop_start_time +
+					   maximum_jiffies_at_index(i)))
+			return -ETIMEDOUT;
+		if (wg_ratelimiter_allow(skb4, &init_net) !=
+					expected_results[i].result)
+			return -EXFULL;
+		++(*test);
+
+		hdr4->saddr = htonl(ntohl(hdr4->saddr) + i + 1);
+		if (time_is_before_jiffies(loop_start_time +
+					   maximum_jiffies_at_index(i)))
+			return -ETIMEDOUT;
+		if (!wg_ratelimiter_allow(skb4, &init_net))
+			return -EXFULL;
+		++(*test);
+
+		hdr4->saddr = htonl(ntohl(hdr4->saddr) - i - 1);
+
+#if IS_ENABLED(CONFIG_IPV6)
+		hdr6->saddr.in6_u.u6_addr32[2] = htonl(i);
+		hdr6->saddr.in6_u.u6_addr32[3] = htonl(i);
+		if (time_is_before_jiffies(loop_start_time +
+					   maximum_jiffies_at_index(i)))
+			return -ETIMEDOUT;
+		if (wg_ratelimiter_allow(skb6, &init_net) !=
+					expected_results[i].result)
+			return -EXFULL;
+		++(*test);
+
+		hdr6->saddr.in6_u.u6_addr32[0] =
+			htonl(ntohl(hdr6->saddr.in6_u.u6_addr32[0]) + i + 1);
+		if (time_is_before_jiffies(loop_start_time +
+					   maximum_jiffies_at_index(i)))
+			return -ETIMEDOUT;
+		if (!wg_ratelimiter_allow(skb6, &init_net))
+			return -EXFULL;
+		++(*test);
+
+		hdr6->saddr.in6_u.u6_addr32[0] =
+			htonl(ntohl(hdr6->saddr.in6_u.u6_addr32[0]) - i - 1);
+
+		if (time_is_before_jiffies(loop_start_time +
+					   maximum_jiffies_at_index(i)))
+			return -ETIMEDOUT;
+#endif
+	}
+	return 0;
+}
+
+static __init int capacity_test(struct sk_buff *skb4, struct iphdr *hdr4,
+				int *test)
+{
+	int i;
+
+	wg_ratelimiter_gc_entries(NULL);
+	rcu_barrier();
+
+	if (atomic_read(&total_entries))
+		return -EXFULL;
+	++(*test);
+
+	for (i = 0; i <= max_entries; ++i) {
+		hdr4->saddr = htonl(i);
+		if (wg_ratelimiter_allow(skb4, &init_net) != (i != max_entries))
+			return -EXFULL;
+		++(*test);
+	}
+	return 0;
+}
+
+bool __init wg_ratelimiter_selftest(void)
+{
+	enum { TRIALS_BEFORE_GIVING_UP = 5000 };
+	bool success = false;
+	int test = 0, trials;
+	struct sk_buff *skb4, *skb6;
+	struct iphdr *hdr4;
+	struct ipv6hdr *hdr6;
+
+	if (IS_ENABLED(CONFIG_KASAN) || IS_ENABLED(CONFIG_UBSAN))
+		return true;
+
+	BUILD_BUG_ON(MSEC_PER_SEC % PACKETS_PER_SECOND != 0);
+
+	if (wg_ratelimiter_init())
+		goto out;
+	++test;
+	if (wg_ratelimiter_init()) {
+		wg_ratelimiter_uninit();
+		goto out;
+	}
+	++test;
+	if (wg_ratelimiter_init()) {
+		wg_ratelimiter_uninit();
+		wg_ratelimiter_uninit();
+		goto out;
+	}
+	++test;
+
+	skb4 = alloc_skb(sizeof(struct iphdr), GFP_KERNEL);
+	if (unlikely(!skb4))
+		goto err_nofree;
+	skb4->protocol = htons(ETH_P_IP);
+	hdr4 = (struct iphdr *)skb_put(skb4, sizeof(*hdr4));
+	hdr4->saddr = htonl(8182);
+	skb_reset_network_header(skb4);
+	++test;
+
+#if IS_ENABLED(CONFIG_IPV6)
+	skb6 = alloc_skb(sizeof(struct ipv6hdr), GFP_KERNEL);
+	if (unlikely(!skb6)) {
+		kfree_skb(skb4);
+		goto err_nofree;
+	}
+	skb6->protocol = htons(ETH_P_IPV6);
+	hdr6 = (struct ipv6hdr *)skb_put(skb6, sizeof(*hdr6));
+	hdr6->saddr.in6_u.u6_addr32[0] = htonl(1212);
+	hdr6->saddr.in6_u.u6_addr32[1] = htonl(289188);
+	skb_reset_network_header(skb6);
+	++test;
+#endif
+
+	for (trials = TRIALS_BEFORE_GIVING_UP;;) {
+		int test_count = 0, ret;
+
+		ret = timings_test(skb4, hdr4, skb6, hdr6, &test_count);
+		if (ret == -ETIMEDOUT) {
+			if (!trials--) {
+				test += test_count;
+				goto err;
+			}
+			msleep(500);
+			continue;
+		} else if (ret < 0) {
+			test += test_count;
+			goto err;
+		} else {
+			test += test_count;
+			break;
+		}
+	}
+
+	for (trials = TRIALS_BEFORE_GIVING_UP;;) {
+		int test_count = 0;
+
+		if (capacity_test(skb4, hdr4, &test_count) < 0) {
+			if (!trials--) {
+				test += test_count;
+				goto err;
+			}
+			msleep(50);
+			continue;
+		}
+		test += test_count;
+		break;
+	}
+
+	success = true;
+
+err:
+	kfree_skb(skb4);
+#if IS_ENABLED(CONFIG_IPV6)
+	kfree_skb(skb6);
+#endif
+err_nofree:
+	wg_ratelimiter_uninit();
+	wg_ratelimiter_uninit();
+	wg_ratelimiter_uninit();
+	/* Uninit one extra time to check underflow detection. */
+	wg_ratelimiter_uninit();
+out:
+	if (success)
+		pr_info("ratelimiter self-tests: pass\n");
+	else
+		pr_err("ratelimiter self-test %d: FAIL\n", test);
+
+	return success;
+}
+#endif
diff -Naur --no-dereference a/net/wireguard/send.c b/net/wireguard/send.c
--- a/net/wireguard/send.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/send.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,421 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#include "queueing.h"
+#include "timers.h"
+#include "device.h"
+#include "peer.h"
+#include "socket.h"
+#include "messages.h"
+#include "cookie.h"
+
+#include <linux/simd.h>
+#include <linux/uio.h>
+#include <linux/inetdevice.h>
+#include <linux/socket.h>
+#include <net/ip_tunnels.h>
+#include <net/udp.h>
+#include <net/sock.h>
+
+static void wg_packet_send_handshake_initiation(struct wg_peer *peer)
+{
+	struct message_handshake_initiation packet;
+
+	if (!wg_birthdate_has_expired(atomic64_read(&peer->last_sent_handshake),
+				      REKEY_TIMEOUT))
+		return; /* This function is rate limited. */
+
+	atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns());
+	net_dbg_ratelimited("%s: Sending handshake initiation to peer %llu (%pISpfsc)\n",
+			    peer->device->dev->name, peer->internal_id,
+			    &peer->endpoint.addr);
+
+	if (wg_noise_handshake_create_initiation(&packet, &peer->handshake)) {
+		wg_cookie_add_mac_to_packet(&packet, sizeof(packet), peer);
+		wg_timers_any_authenticated_packet_traversal(peer);
+		wg_timers_any_authenticated_packet_sent(peer);
+		atomic64_set(&peer->last_sent_handshake,
+			     ktime_get_coarse_boottime_ns());
+		wg_socket_send_buffer_to_peer(peer, &packet, sizeof(packet),
+					      HANDSHAKE_DSCP);
+		wg_timers_handshake_initiated(peer);
+	}
+}
+
+void wg_packet_handshake_send_worker(struct work_struct *work)
+{
+	struct wg_peer *peer = container_of(work, struct wg_peer,
+					    transmit_handshake_work);
+
+	wg_packet_send_handshake_initiation(peer);
+	wg_peer_put(peer);
+}
+
+void wg_packet_send_queued_handshake_initiation(struct wg_peer *peer,
+						bool is_retry)
+{
+	if (!is_retry)
+		peer->timer_handshake_attempts = 0;
+
+	rcu_read_lock_bh();
+	/* We check last_sent_handshake here in addition to the actual function
+	 * we're queueing up, so that we don't queue things if not strictly
+	 * necessary:
+	 */
+	if (!wg_birthdate_has_expired(atomic64_read(&peer->last_sent_handshake),
+				      REKEY_TIMEOUT) ||
+			unlikely(READ_ONCE(peer->is_dead)))
+		goto out;
+
+	wg_peer_get(peer);
+	/* Queues up calling packet_send_queued_handshakes(peer), where we do a
+	 * peer_put(peer) after:
+	 */
+	if (!queue_work(peer->device->handshake_send_wq,
+			&peer->transmit_handshake_work))
+		/* If the work was already queued, we want to drop the
+		 * extra reference:
+		 */
+		wg_peer_put(peer);
+out:
+	rcu_read_unlock_bh();
+}
+
+void wg_packet_send_handshake_response(struct wg_peer *peer)
+{
+	struct message_handshake_response packet;
+
+	atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns());
+	net_dbg_ratelimited("%s: Sending handshake response to peer %llu (%pISpfsc)\n",
+			    peer->device->dev->name, peer->internal_id,
+			    &peer->endpoint.addr);
+
+	if (wg_noise_handshake_create_response(&packet, &peer->handshake)) {
+		wg_cookie_add_mac_to_packet(&packet, sizeof(packet), peer);
+		if (wg_noise_handshake_begin_session(&peer->handshake,
+						     &peer->keypairs)) {
+			wg_timers_session_derived(peer);
+			wg_timers_any_authenticated_packet_traversal(peer);
+			wg_timers_any_authenticated_packet_sent(peer);
+			atomic64_set(&peer->last_sent_handshake,
+				     ktime_get_coarse_boottime_ns());
+			wg_socket_send_buffer_to_peer(peer, &packet,
+						      sizeof(packet),
+						      HANDSHAKE_DSCP);
+		}
+	}
+}
+
+void wg_packet_send_handshake_cookie(struct wg_device *wg,
+				     struct sk_buff *initiating_skb,
+				     __le32 sender_index)
+{
+	struct message_handshake_cookie packet;
+
+	net_dbg_skb_ratelimited("%s: Sending cookie response for denied handshake message for %pISpfsc\n",
+				wg->dev->name, initiating_skb);
+	wg_cookie_message_create(&packet, initiating_skb, sender_index,
+				 &wg->cookie_checker);
+	wg_socket_send_buffer_as_reply_to_skb(wg, initiating_skb, &packet,
+					      sizeof(packet));
+}
+
+static void keep_key_fresh(struct wg_peer *peer)
+{
+	struct noise_keypair *keypair;
+	bool send = false;
+
+	rcu_read_lock_bh();
+	keypair = rcu_dereference_bh(peer->keypairs.current_keypair);
+	if (likely(keypair && READ_ONCE(keypair->sending.is_valid)) &&
+	    (unlikely(atomic64_read(&keypair->sending.counter.counter) >
+		      REKEY_AFTER_MESSAGES) ||
+	     (keypair->i_am_the_initiator &&
+	      unlikely(wg_birthdate_has_expired(keypair->sending.birthdate,
+						REKEY_AFTER_TIME)))))
+		send = true;
+	rcu_read_unlock_bh();
+
+	if (send)
+		wg_packet_send_queued_handshake_initiation(peer, false);
+}
+
+static unsigned int calculate_skb_padding(struct sk_buff *skb)
+{
+	/* We do this modulo business with the MTU, just in case the networking
+	 * layer gives us a packet that's bigger than the MTU. In that case, we
+	 * wouldn't want the final subtraction to overflow in the case of the
+	 * padded_size being clamped.
+	 */
+	unsigned int last_unit = skb->len % PACKET_CB(skb)->mtu;
+	unsigned int padded_size = ALIGN(last_unit, MESSAGE_PADDING_MULTIPLE);
+
+	if (padded_size > PACKET_CB(skb)->mtu)
+		padded_size = PACKET_CB(skb)->mtu;
+	return padded_size - last_unit;
+}
+
+static bool encrypt_packet(struct sk_buff *skb, struct noise_keypair *keypair,
+			   simd_context_t *simd_context)
+{
+	unsigned int padding_len, plaintext_len, trailer_len;
+	struct scatterlist sg[MAX_SKB_FRAGS + 8];
+	struct message_data *header;
+	struct sk_buff *trailer;
+	int num_frags;
+
+	/* Calculate lengths. */
+	padding_len = calculate_skb_padding(skb);
+	trailer_len = padding_len + noise_encrypted_len(0);
+	plaintext_len = skb->len + padding_len;
+
+	/* Expand data section to have room for padding and auth tag. */
+	num_frags = skb_cow_data(skb, trailer_len, &trailer);
+	if (unlikely(num_frags < 0 || num_frags > ARRAY_SIZE(sg)))
+		return false;
+
+	/* Set the padding to zeros, and make sure it and the auth tag are part
+	 * of the skb.
+	 */
+	memset(skb_tail_pointer(trailer), 0, padding_len);
+
+	/* Expand head section to have room for our header and the network
+	 * stack's headers.
+	 */
+	if (unlikely(skb_cow_head(skb, DATA_PACKET_HEAD_ROOM) < 0))
+		return false;
+
+	/* Finalize checksum calculation for the inner packet, if required. */
+	if (unlikely(skb->ip_summed == CHECKSUM_PARTIAL &&
+		     skb_checksum_help(skb)))
+		return false;
+
+	/* Only after checksumming can we safely add on the padding at the end
+	 * and the header.
+	 */
+	skb_set_inner_network_header(skb, 0);
+	header = (struct message_data *)skb_push(skb, sizeof(*header));
+	header->header.type = cpu_to_le32(MESSAGE_DATA);
+	header->key_idx = keypair->remote_index;
+	header->counter = cpu_to_le64(PACKET_CB(skb)->nonce);
+	pskb_put(skb, trailer, trailer_len);
+
+	/* Now we can encrypt the scattergather segments */
+	sg_init_table(sg, num_frags);
+	if (skb_to_sgvec(skb, sg, sizeof(struct message_data),
+			 noise_encrypted_len(plaintext_len)) <= 0)
+		return false;
+	return chacha20poly1305_encrypt_sg_inplace(sg, plaintext_len, NULL, 0,
+						   PACKET_CB(skb)->nonce,
+						   keypair->sending.key,
+						   simd_context);
+}
+
+void wg_packet_send_keepalive(struct wg_peer *peer)
+{
+	struct sk_buff *skb;
+
+	if (skb_queue_empty(&peer->staged_packet_queue)) {
+		skb = alloc_skb(DATA_PACKET_HEAD_ROOM + MESSAGE_MINIMUM_LENGTH,
+				GFP_ATOMIC);
+		if (unlikely(!skb))
+			return;
+		skb_reserve(skb, DATA_PACKET_HEAD_ROOM);
+		skb->dev = peer->device->dev;
+		PACKET_CB(skb)->mtu = skb->dev->mtu;
+		skb_queue_tail(&peer->staged_packet_queue, skb);
+		net_dbg_ratelimited("%s: Sending keepalive packet to peer %llu (%pISpfsc)\n",
+				    peer->device->dev->name, peer->internal_id,
+				    &peer->endpoint.addr);
+	}
+
+	wg_packet_send_staged_packets(peer);
+}
+
+static void wg_packet_create_data_done(struct sk_buff *first,
+				       struct wg_peer *peer)
+{
+	struct sk_buff *skb, *next;
+	bool is_keepalive, data_sent = false;
+
+	wg_timers_any_authenticated_packet_traversal(peer);
+	wg_timers_any_authenticated_packet_sent(peer);
+	skb_list_walk_safe(first, skb, next) {
+		is_keepalive = skb->len == message_data_len(0);
+		if (likely(!wg_socket_send_skb_to_peer(peer, skb,
+				PACKET_CB(skb)->ds) && !is_keepalive))
+			data_sent = true;
+	}
+
+	if (likely(data_sent))
+		wg_timers_data_sent(peer);
+
+	keep_key_fresh(peer);
+}
+
+void wg_packet_tx_worker(struct work_struct *work)
+{
+	struct crypt_queue *queue = container_of(work, struct crypt_queue,
+						 work);
+	struct noise_keypair *keypair;
+	enum packet_state state;
+	struct sk_buff *first;
+	struct wg_peer *peer;
+
+	while ((first = __ptr_ring_peek(&queue->ring)) != NULL &&
+	       (state = atomic_read_acquire(&PACKET_CB(first)->state)) !=
+		       PACKET_STATE_UNCRYPTED) {
+		__ptr_ring_discard_one(&queue->ring);
+		peer = PACKET_PEER(first);
+		keypair = PACKET_CB(first)->keypair;
+
+		if (likely(state == PACKET_STATE_CRYPTED))
+			wg_packet_create_data_done(first, peer);
+		else
+			kfree_skb_list(first);
+
+		wg_noise_keypair_put(keypair, false);
+		wg_peer_put(peer);
+	}
+}
+
+void wg_packet_encrypt_worker(struct work_struct *work)
+{
+	struct crypt_queue *queue = container_of(work, struct multicore_worker,
+						 work)->ptr;
+	struct sk_buff *first, *skb, *next;
+	simd_context_t simd_context;
+
+	simd_get(&simd_context);
+	while ((first = ptr_ring_consume_bh(&queue->ring)) != NULL) {
+		enum packet_state state = PACKET_STATE_CRYPTED;
+
+		skb_list_walk_safe(first, skb, next) {
+			if (likely(encrypt_packet(skb,
+						  PACKET_CB(first)->keypair,
+						  &simd_context))) {
+				wg_reset_packet(skb);
+			} else {
+				state = PACKET_STATE_DEAD;
+				break;
+			}
+		}
+		wg_queue_enqueue_per_peer(&PACKET_PEER(first)->tx_queue, first,
+					  state);
+
+		simd_relax(&simd_context);
+	}
+	simd_put(&simd_context);
+}
+
+static void wg_packet_create_data(struct sk_buff *first)
+{
+	struct wg_peer *peer = PACKET_PEER(first);
+	struct wg_device *wg = peer->device;
+	int ret = -EINVAL;
+
+	rcu_read_lock_bh();
+	if (unlikely(READ_ONCE(peer->is_dead)))
+		goto err;
+
+	ret = wg_queue_enqueue_per_device_and_peer(&wg->encrypt_queue,
+						   &peer->tx_queue, first,
+						   wg->packet_crypt_wq,
+						   &wg->encrypt_queue.last_cpu);
+	if (unlikely(ret == -EPIPE))
+		wg_queue_enqueue_per_peer(&peer->tx_queue, first,
+					  PACKET_STATE_DEAD);
+err:
+	rcu_read_unlock_bh();
+	if (likely(!ret || ret == -EPIPE))
+		return;
+	wg_noise_keypair_put(PACKET_CB(first)->keypair, false);
+	wg_peer_put(peer);
+	kfree_skb_list(first);
+}
+
+void wg_packet_purge_staged_packets(struct wg_peer *peer)
+{
+	spin_lock_bh(&peer->staged_packet_queue.lock);
+	peer->device->dev->stats.tx_dropped += peer->staged_packet_queue.qlen;
+	__skb_queue_purge(&peer->staged_packet_queue);
+	spin_unlock_bh(&peer->staged_packet_queue.lock);
+}
+
+void wg_packet_send_staged_packets(struct wg_peer *peer)
+{
+	struct noise_symmetric_key *key;
+	struct noise_keypair *keypair;
+	struct sk_buff_head packets;
+	struct sk_buff *skb;
+
+	/* Steal the current queue into our local one. */
+	__skb_queue_head_init(&packets);
+	spin_lock_bh(&peer->staged_packet_queue.lock);
+	skb_queue_splice_init(&peer->staged_packet_queue, &packets);
+	spin_unlock_bh(&peer->staged_packet_queue.lock);
+	if (unlikely(skb_queue_empty(&packets)))
+		return;
+
+	/* First we make sure we have a valid reference to a valid key. */
+	rcu_read_lock_bh();
+	keypair = wg_noise_keypair_get(
+		rcu_dereference_bh(peer->keypairs.current_keypair));
+	rcu_read_unlock_bh();
+	if (unlikely(!keypair))
+		goto out_nokey;
+	key = &keypair->sending;
+	if (unlikely(!READ_ONCE(key->is_valid)))
+		goto out_nokey;
+	if (unlikely(wg_birthdate_has_expired(key->birthdate,
+					      REJECT_AFTER_TIME)))
+		goto out_invalid;
+
+	/* After we know we have a somewhat valid key, we now try to assign
+	 * nonces to all of the packets in the queue. If we can't assign nonces
+	 * for all of them, we just consider it a failure and wait for the next
+	 * handshake.
+	 */
+	skb_queue_walk(&packets, skb) {
+		/* 0 for no outer TOS: no leak. TODO: at some later point, we
+		 * might consider using flowi->tos as outer instead.
+		 */
+		PACKET_CB(skb)->ds = ip_tunnel_ecn_encap(0, ip_hdr(skb), skb);
+		PACKET_CB(skb)->nonce =
+				atomic64_inc_return(&key->counter.counter) - 1;
+		if (unlikely(PACKET_CB(skb)->nonce >= REJECT_AFTER_MESSAGES))
+			goto out_invalid;
+	}
+
+	packets.prev->next = NULL;
+	wg_peer_get(keypair->entry.peer);
+	PACKET_CB(packets.next)->keypair = keypair;
+	wg_packet_create_data(packets.next);
+	return;
+
+out_invalid:
+	WRITE_ONCE(key->is_valid, false);
+out_nokey:
+	wg_noise_keypair_put(keypair, false);
+
+	/* We orphan the packets if we're waiting on a handshake, so that they
+	 * don't block a socket's pool.
+	 */
+	skb_queue_walk(&packets, skb)
+		skb_orphan(skb);
+	/* Then we put them back on the top of the queue. We're not too
+	 * concerned about accidentally getting things a little out of order if
+	 * packets are being added really fast, because this queue is for before
+	 * packets can even be sent and it's small anyway.
+	 */
+	spin_lock_bh(&peer->staged_packet_queue.lock);
+	skb_queue_splice(&packets, &peer->staged_packet_queue);
+	spin_unlock_bh(&peer->staged_packet_queue.lock);
+
+	/* If we're exiting because there's something wrong with the key, it
+	 * means we should initiate a new handshake.
+	 */
+	wg_packet_send_queued_handshake_initiation(peer, false);
+}
diff -Naur --no-dereference a/net/wireguard/socket.c b/net/wireguard/socket.c
--- a/net/wireguard/socket.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/socket.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,437 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#include "device.h"
+#include "peer.h"
+#include "socket.h"
+#include "queueing.h"
+#include "messages.h"
+
+#include <linux/ctype.h>
+#include <linux/net.h>
+#include <linux/if_vlan.h>
+#include <linux/if_ether.h>
+#include <linux/inetdevice.h>
+#include <net/udp_tunnel.h>
+#include <net/ipv6.h>
+
+static int send4(struct wg_device *wg, struct sk_buff *skb,
+		 struct endpoint *endpoint, u8 ds, struct dst_cache *cache)
+{
+	struct flowi4 fl = {
+		.saddr = endpoint->src4.s_addr,
+		.daddr = endpoint->addr4.sin_addr.s_addr,
+		.fl4_dport = endpoint->addr4.sin_port,
+		.flowi4_mark = wg->fwmark,
+		.flowi4_proto = IPPROTO_UDP
+	};
+	struct rtable *rt = NULL;
+	struct sock *sock;
+	int ret = 0;
+
+	skb_mark_not_on_list(skb);
+	skb->dev = wg->dev;
+	skb->mark = wg->fwmark;
+
+	rcu_read_lock_bh();
+	sock = rcu_dereference_bh(wg->sock4);
+
+	if (unlikely(!sock)) {
+		ret = -ENONET;
+		goto err;
+	}
+
+	fl.fl4_sport = inet_sk(sock)->inet_sport;
+
+	if (cache)
+		rt = dst_cache_get_ip4(cache, &fl.saddr);
+
+	if (!rt) {
+		security_sk_classify_flow(sock, flowi4_to_flowi(&fl));
+		if (unlikely(!inet_confirm_addr(sock_net(sock), NULL, 0,
+						fl.saddr, RT_SCOPE_HOST))) {
+			endpoint->src4.s_addr = 0;
+			*(__force __be32 *)&endpoint->src_if4 = 0;
+			fl.saddr = 0;
+			if (cache)
+				dst_cache_reset(cache);
+		}
+		rt = ip_route_output_flow(sock_net(sock), &fl, sock);
+		if (unlikely(endpoint->src_if4 && ((IS_ERR(rt) &&
+			     PTR_ERR(rt) == -EINVAL) || (!IS_ERR(rt) &&
+			     rt->dst.dev->ifindex != endpoint->src_if4)))) {
+			endpoint->src4.s_addr = 0;
+			*(__force __be32 *)&endpoint->src_if4 = 0;
+			fl.saddr = 0;
+			if (cache)
+				dst_cache_reset(cache);
+			if (!IS_ERR(rt))
+				ip_rt_put(rt);
+			rt = ip_route_output_flow(sock_net(sock), &fl, sock);
+		}
+		if (unlikely(IS_ERR(rt))) {
+			ret = PTR_ERR(rt);
+			net_dbg_ratelimited("%s: No route to %pISpfsc, error %d\n",
+					    wg->dev->name, &endpoint->addr, ret);
+			goto err;
+		} else if (unlikely(rt->dst.dev == skb->dev)) {
+			ip_rt_put(rt);
+			ret = -ELOOP;
+			net_dbg_ratelimited("%s: Avoiding routing loop to %pISpfsc\n",
+					    wg->dev->name, &endpoint->addr);
+			goto err;
+		}
+		if (cache)
+			dst_cache_set_ip4(cache, &rt->dst, fl.saddr);
+	}
+
+	skb->ignore_df = 1;
+	udp_tunnel_xmit_skb(rt, sock, skb, fl.saddr, fl.daddr, ds,
+			    ip4_dst_hoplimit(&rt->dst), 0, fl.fl4_sport,
+			    fl.fl4_dport, false, false);
+	goto out;
+
+err:
+	kfree_skb(skb);
+out:
+	rcu_read_unlock_bh();
+	return ret;
+}
+
+static int send6(struct wg_device *wg, struct sk_buff *skb,
+		 struct endpoint *endpoint, u8 ds, struct dst_cache *cache)
+{
+#if IS_ENABLED(CONFIG_IPV6)
+	struct flowi6 fl = {
+		.saddr = endpoint->src6,
+		.daddr = endpoint->addr6.sin6_addr,
+		.fl6_dport = endpoint->addr6.sin6_port,
+		.flowi6_mark = wg->fwmark,
+		.flowi6_oif = endpoint->addr6.sin6_scope_id,
+		.flowi6_proto = IPPROTO_UDP
+		/* TODO: addr->sin6_flowinfo */
+	};
+	struct dst_entry *dst = NULL;
+	struct sock *sock;
+	int ret = 0;
+
+	skb_mark_not_on_list(skb);
+	skb->dev = wg->dev;
+	skb->mark = wg->fwmark;
+
+	rcu_read_lock_bh();
+	sock = rcu_dereference_bh(wg->sock6);
+
+	if (unlikely(!sock)) {
+		ret = -ENONET;
+		goto err;
+	}
+
+	fl.fl6_sport = inet_sk(sock)->inet_sport;
+
+	if (cache)
+		dst = dst_cache_get_ip6(cache, &fl.saddr);
+
+	if (!dst) {
+		security_sk_classify_flow(sock, flowi6_to_flowi(&fl));
+		if (unlikely(!ipv6_addr_any(&fl.saddr) &&
+			     !ipv6_chk_addr(sock_net(sock), &fl.saddr, NULL, 0))) {
+			endpoint->src6 = fl.saddr = in6addr_any;
+			if (cache)
+				dst_cache_reset(cache);
+		}
+		dst = ipv6_stub->ipv6_dst_lookup_flow(sock_net(sock), sock, &fl,
+						      NULL);
+		if (unlikely(IS_ERR(dst))) {
+			ret = PTR_ERR(dst);
+			net_dbg_ratelimited("%s: No route to %pISpfsc, error %d\n",
+					    wg->dev->name, &endpoint->addr, ret);
+			goto err;
+		} else if (unlikely(dst->dev == skb->dev)) {
+			dst_release(dst);
+			ret = -ELOOP;
+			net_dbg_ratelimited("%s: Avoiding routing loop to %pISpfsc\n",
+					    wg->dev->name, &endpoint->addr);
+			goto err;
+		}
+		if (cache)
+			dst_cache_set_ip6(cache, dst, &fl.saddr);
+	}
+
+	skb->ignore_df = 1;
+	udp_tunnel6_xmit_skb(dst, sock, skb, skb->dev, &fl.saddr, &fl.daddr, ds,
+			     ip6_dst_hoplimit(dst), 0, fl.fl6_sport,
+			     fl.fl6_dport, false);
+	goto out;
+
+err:
+	kfree_skb(skb);
+out:
+	rcu_read_unlock_bh();
+	return ret;
+#else
+	return -EAFNOSUPPORT;
+#endif
+}
+
+int wg_socket_send_skb_to_peer(struct wg_peer *peer, struct sk_buff *skb, u8 ds)
+{
+	size_t skb_len = skb->len;
+	int ret = -EAFNOSUPPORT;
+
+	read_lock_bh(&peer->endpoint_lock);
+	if (peer->endpoint.addr.sa_family == AF_INET)
+		ret = send4(peer->device, skb, &peer->endpoint, ds,
+			    &peer->endpoint_cache);
+	else if (peer->endpoint.addr.sa_family == AF_INET6)
+		ret = send6(peer->device, skb, &peer->endpoint, ds,
+			    &peer->endpoint_cache);
+	else
+		dev_kfree_skb(skb);
+	if (likely(!ret))
+		peer->tx_bytes += skb_len;
+	read_unlock_bh(&peer->endpoint_lock);
+
+	return ret;
+}
+
+int wg_socket_send_buffer_to_peer(struct wg_peer *peer, void *buffer,
+				  size_t len, u8 ds)
+{
+	struct sk_buff *skb = alloc_skb(len + SKB_HEADER_LEN, GFP_ATOMIC);
+
+	if (unlikely(!skb))
+		return -ENOMEM;
+
+	skb_reserve(skb, SKB_HEADER_LEN);
+	skb_set_inner_network_header(skb, 0);
+	skb_put_data(skb, buffer, len);
+	return wg_socket_send_skb_to_peer(peer, skb, ds);
+}
+
+int wg_socket_send_buffer_as_reply_to_skb(struct wg_device *wg,
+					  struct sk_buff *in_skb, void *buffer,
+					  size_t len)
+{
+	int ret = 0;
+	struct sk_buff *skb;
+	struct endpoint endpoint;
+
+	if (unlikely(!in_skb))
+		return -EINVAL;
+	ret = wg_socket_endpoint_from_skb(&endpoint, in_skb);
+	if (unlikely(ret < 0))
+		return ret;
+
+	skb = alloc_skb(len + SKB_HEADER_LEN, GFP_ATOMIC);
+	if (unlikely(!skb))
+		return -ENOMEM;
+	skb_reserve(skb, SKB_HEADER_LEN);
+	skb_set_inner_network_header(skb, 0);
+	skb_put_data(skb, buffer, len);
+
+	if (endpoint.addr.sa_family == AF_INET)
+		ret = send4(wg, skb, &endpoint, 0, NULL);
+	else if (endpoint.addr.sa_family == AF_INET6)
+		ret = send6(wg, skb, &endpoint, 0, NULL);
+	/* No other possibilities if the endpoint is valid, which it is,
+	 * as we checked above.
+	 */
+
+	return ret;
+}
+
+int wg_socket_endpoint_from_skb(struct endpoint *endpoint,
+				const struct sk_buff *skb)
+{
+	memset(endpoint, 0, sizeof(*endpoint));
+	if (skb->protocol == htons(ETH_P_IP)) {
+		endpoint->addr4.sin_family = AF_INET;
+		endpoint->addr4.sin_port = udp_hdr(skb)->source;
+		endpoint->addr4.sin_addr.s_addr = ip_hdr(skb)->saddr;
+		endpoint->src4.s_addr = ip_hdr(skb)->daddr;
+		endpoint->src_if4 = skb->skb_iif;
+	} else if (skb->protocol == htons(ETH_P_IPV6)) {
+		endpoint->addr6.sin6_family = AF_INET6;
+		endpoint->addr6.sin6_port = udp_hdr(skb)->source;
+		endpoint->addr6.sin6_addr = ipv6_hdr(skb)->saddr;
+		endpoint->addr6.sin6_scope_id = ipv6_iface_scope_id(
+			&ipv6_hdr(skb)->saddr, skb->skb_iif);
+		endpoint->src6 = ipv6_hdr(skb)->daddr;
+	} else {
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static bool endpoint_eq(const struct endpoint *a, const struct endpoint *b)
+{
+	return (a->addr.sa_family == AF_INET && b->addr.sa_family == AF_INET &&
+		a->addr4.sin_port == b->addr4.sin_port &&
+		a->addr4.sin_addr.s_addr == b->addr4.sin_addr.s_addr &&
+		a->src4.s_addr == b->src4.s_addr && a->src_if4 == b->src_if4) ||
+	       (a->addr.sa_family == AF_INET6 &&
+		b->addr.sa_family == AF_INET6 &&
+		a->addr6.sin6_port == b->addr6.sin6_port &&
+		ipv6_addr_equal(&a->addr6.sin6_addr, &b->addr6.sin6_addr) &&
+		a->addr6.sin6_scope_id == b->addr6.sin6_scope_id &&
+		ipv6_addr_equal(&a->src6, &b->src6)) ||
+	       unlikely(!a->addr.sa_family && !b->addr.sa_family);
+}
+
+void wg_socket_set_peer_endpoint(struct wg_peer *peer,
+				 const struct endpoint *endpoint)
+{
+	/* First we check unlocked, in order to optimize, since it's pretty rare
+	 * that an endpoint will change. If we happen to be mid-write, and two
+	 * CPUs wind up writing the same thing or something slightly different,
+	 * it doesn't really matter much either.
+	 */
+	if (endpoint_eq(endpoint, &peer->endpoint))
+		return;
+	write_lock_bh(&peer->endpoint_lock);
+	if (endpoint->addr.sa_family == AF_INET) {
+		peer->endpoint.addr4 = endpoint->addr4;
+		peer->endpoint.src4 = endpoint->src4;
+		peer->endpoint.src_if4 = endpoint->src_if4;
+	} else if (endpoint->addr.sa_family == AF_INET6) {
+		peer->endpoint.addr6 = endpoint->addr6;
+		peer->endpoint.src6 = endpoint->src6;
+	} else {
+		goto out;
+	}
+	dst_cache_reset(&peer->endpoint_cache);
+out:
+	write_unlock_bh(&peer->endpoint_lock);
+}
+
+void wg_socket_set_peer_endpoint_from_skb(struct wg_peer *peer,
+					  const struct sk_buff *skb)
+{
+	struct endpoint endpoint;
+
+	if (!wg_socket_endpoint_from_skb(&endpoint, skb))
+		wg_socket_set_peer_endpoint(peer, &endpoint);
+}
+
+void wg_socket_clear_peer_endpoint_src(struct wg_peer *peer)
+{
+	write_lock_bh(&peer->endpoint_lock);
+	memset(&peer->endpoint.src6, 0, sizeof(peer->endpoint.src6));
+	dst_cache_reset(&peer->endpoint_cache);
+	write_unlock_bh(&peer->endpoint_lock);
+}
+
+static int wg_receive(struct sock *sk, struct sk_buff *skb)
+{
+	struct wg_device *wg;
+
+	if (unlikely(!sk))
+		goto err;
+	wg = sk->sk_user_data;
+	if (unlikely(!wg))
+		goto err;
+	wg_packet_receive(wg, skb);
+	return 0;
+
+err:
+	kfree_skb(skb);
+	return 0;
+}
+
+static void sock_free(struct sock *sock)
+{
+	if (unlikely(!sock))
+		return;
+	sk_clear_memalloc(sock);
+	udp_tunnel_sock_release(sock->sk_socket);
+}
+
+static void set_sock_opts(struct socket *sock)
+{
+	sock->sk->sk_allocation = GFP_ATOMIC;
+	sock->sk->sk_sndbuf = INT_MAX;
+	sk_set_memalloc(sock->sk);
+}
+
+int wg_socket_init(struct wg_device *wg, u16 port)
+{
+	int ret;
+	struct udp_tunnel_sock_cfg cfg = {
+		.sk_user_data = wg,
+		.encap_type = 1,
+		.encap_rcv = wg_receive
+	};
+	struct socket *new4 = NULL, *new6 = NULL;
+	struct udp_port_cfg port4 = {
+		.family = AF_INET,
+		.local_ip.s_addr = htonl(INADDR_ANY),
+		.local_udp_port = htons(port),
+		.use_udp_checksums = true
+	};
+#if IS_ENABLED(CONFIG_IPV6)
+	int retries = 0;
+	struct udp_port_cfg port6 = {
+		.family = AF_INET6,
+		.local_ip6 = IN6ADDR_ANY_INIT,
+		.use_udp6_tx_checksums = true,
+		.use_udp6_rx_checksums = true,
+		.ipv6_v6only = true
+	};
+#endif
+
+#if IS_ENABLED(CONFIG_IPV6)
+retry:
+#endif
+
+	ret = udp_sock_create(wg->creating_net, &port4, &new4);
+	if (ret < 0) {
+		pr_err("%s: Could not create IPv4 socket\n", wg->dev->name);
+		return ret;
+	}
+	set_sock_opts(new4);
+	setup_udp_tunnel_sock(wg->creating_net, new4, &cfg);
+
+#if IS_ENABLED(CONFIG_IPV6)
+	if (ipv6_mod_enabled()) {
+		port6.local_udp_port = inet_sk(new4->sk)->inet_sport;
+		ret = udp_sock_create(wg->creating_net, &port6, &new6);
+		if (ret < 0) {
+			udp_tunnel_sock_release(new4);
+			if (ret == -EADDRINUSE && !port && retries++ < 100)
+				goto retry;
+			pr_err("%s: Could not create IPv6 socket\n",
+			       wg->dev->name);
+			return ret;
+		}
+		set_sock_opts(new6);
+		setup_udp_tunnel_sock(wg->creating_net, new6, &cfg);
+	}
+#endif
+
+	wg_socket_reinit(wg, new4->sk, new6 ? new6->sk : NULL);
+	return 0;
+}
+
+void wg_socket_reinit(struct wg_device *wg, struct sock *new4,
+		      struct sock *new6)
+{
+	struct sock *old4, *old6;
+
+	mutex_lock(&wg->socket_update_lock);
+	old4 = rcu_dereference_protected(wg->sock4,
+				lockdep_is_held(&wg->socket_update_lock));
+	old6 = rcu_dereference_protected(wg->sock6,
+				lockdep_is_held(&wg->socket_update_lock));
+	rcu_assign_pointer(wg->sock4, new4);
+	rcu_assign_pointer(wg->sock6, new6);
+	if (new4)
+		wg->incoming_port = ntohs(inet_sk(new4)->inet_sport);
+	mutex_unlock(&wg->socket_update_lock);
+	synchronize_rcu();
+	synchronize_net();
+	sock_free(old4);
+	sock_free(old6);
+}
diff -Naur --no-dereference a/net/wireguard/socket.h b/net/wireguard/socket.h
--- a/net/wireguard/socket.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/socket.h	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#ifndef _WG_SOCKET_H
+#define _WG_SOCKET_H
+
+#include <linux/netdevice.h>
+#include <linux/udp.h>
+#include <linux/if_vlan.h>
+#include <linux/if_ether.h>
+
+int wg_socket_init(struct wg_device *wg, u16 port);
+void wg_socket_reinit(struct wg_device *wg, struct sock *new4,
+		      struct sock *new6);
+int wg_socket_send_buffer_to_peer(struct wg_peer *peer, void *data,
+				  size_t len, u8 ds);
+int wg_socket_send_skb_to_peer(struct wg_peer *peer, struct sk_buff *skb,
+			       u8 ds);
+int wg_socket_send_buffer_as_reply_to_skb(struct wg_device *wg,
+					  struct sk_buff *in_skb,
+					  void *out_buffer, size_t len);
+
+int wg_socket_endpoint_from_skb(struct endpoint *endpoint,
+				const struct sk_buff *skb);
+void wg_socket_set_peer_endpoint(struct wg_peer *peer,
+				 const struct endpoint *endpoint);
+void wg_socket_set_peer_endpoint_from_skb(struct wg_peer *peer,
+					  const struct sk_buff *skb);
+void wg_socket_clear_peer_endpoint_src(struct wg_peer *peer);
+
+#if defined(CONFIG_DYNAMIC_DEBUG) || defined(DEBUG)
+#define net_dbg_skb_ratelimited(fmt, dev, skb, ...) do {                       \
+		struct endpoint __endpoint;                                    \
+		wg_socket_endpoint_from_skb(&__endpoint, skb);                 \
+		net_dbg_ratelimited(fmt, dev, &__endpoint.addr,                \
+				    ##__VA_ARGS__);                            \
+	} while (0)
+#else
+#define net_dbg_skb_ratelimited(fmt, skb, ...)
+#endif
+
+#endif /* _WG_SOCKET_H */
diff -Naur --no-dereference a/net/wireguard/timers.c b/net/wireguard/timers.c
--- a/net/wireguard/timers.c	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/timers.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,243 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#include "timers.h"
+#include "device.h"
+#include "peer.h"
+#include "queueing.h"
+#include "socket.h"
+
+/*
+ * - Timer for retransmitting the handshake if we don't hear back after
+ * `REKEY_TIMEOUT + jitter` ms.
+ *
+ * - Timer for sending empty packet if we have received a packet but after have
+ * not sent one for `KEEPALIVE_TIMEOUT` ms.
+ *
+ * - Timer for initiating new handshake if we have sent a packet but after have
+ * not received one (even empty) for `(KEEPALIVE_TIMEOUT + REKEY_TIMEOUT) +
+ * jitter` ms.
+ *
+ * - Timer for zeroing out all ephemeral keys after `(REJECT_AFTER_TIME * 3)` ms
+ * if no new keys have been received.
+ *
+ * - Timer for, if enabled, sending an empty authenticated packet every user-
+ * specified seconds.
+ */
+
+static inline void mod_peer_timer(struct wg_peer *peer,
+				  struct timer_list *timer,
+				  unsigned long expires)
+{
+	rcu_read_lock_bh();
+	if (likely(netif_running(peer->device->dev) &&
+		   !READ_ONCE(peer->is_dead)))
+		mod_timer(timer, expires);
+	rcu_read_unlock_bh();
+}
+
+static void wg_expired_retransmit_handshake(struct timer_list *timer)
+{
+	struct wg_peer *peer = from_timer(peer, timer,
+					  timer_retransmit_handshake);
+
+	if (peer->timer_handshake_attempts > MAX_TIMER_HANDSHAKES) {
+		pr_debug("%s: Handshake for peer %llu (%pISpfsc) did not complete after %d attempts, giving up\n",
+			 peer->device->dev->name, peer->internal_id,
+			 &peer->endpoint.addr, MAX_TIMER_HANDSHAKES + 2);
+
+		del_timer(&peer->timer_send_keepalive);
+		/* We drop all packets without a keypair and don't try again,
+		 * if we try unsuccessfully for too long to make a handshake.
+		 */
+		wg_packet_purge_staged_packets(peer);
+
+		/* We set a timer for destroying any residue that might be left
+		 * of a partial exchange.
+		 */
+		if (!timer_pending(&peer->timer_zero_key_material))
+			mod_peer_timer(peer, &peer->timer_zero_key_material,
+				       jiffies + REJECT_AFTER_TIME * 3 * HZ);
+	} else {
+		++peer->timer_handshake_attempts;
+		pr_debug("%s: Handshake for peer %llu (%pISpfsc) did not complete after %d seconds, retrying (try %d)\n",
+			 peer->device->dev->name, peer->internal_id,
+			 &peer->endpoint.addr, REKEY_TIMEOUT,
+			 peer->timer_handshake_attempts + 1);
+
+		/* We clear the endpoint address src address, in case this is
+		 * the cause of trouble.
+		 */
+		wg_socket_clear_peer_endpoint_src(peer);
+
+		wg_packet_send_queued_handshake_initiation(peer, true);
+	}
+}
+
+static void wg_expired_send_keepalive(struct timer_list *timer)
+{
+	struct wg_peer *peer = from_timer(peer, timer, timer_send_keepalive);
+
+	wg_packet_send_keepalive(peer);
+	if (peer->timer_need_another_keepalive) {
+		peer->timer_need_another_keepalive = false;
+		mod_peer_timer(peer, &peer->timer_send_keepalive,
+			       jiffies + KEEPALIVE_TIMEOUT * HZ);
+	}
+}
+
+static void wg_expired_new_handshake(struct timer_list *timer)
+{
+	struct wg_peer *peer = from_timer(peer, timer, timer_new_handshake);
+
+	pr_debug("%s: Retrying handshake with peer %llu (%pISpfsc) because we stopped hearing back after %d seconds\n",
+		 peer->device->dev->name, peer->internal_id,
+		 &peer->endpoint.addr, KEEPALIVE_TIMEOUT + REKEY_TIMEOUT);
+	/* We clear the endpoint address src address, in case this is the cause
+	 * of trouble.
+	 */
+	wg_socket_clear_peer_endpoint_src(peer);
+	wg_packet_send_queued_handshake_initiation(peer, false);
+}
+
+static void wg_expired_zero_key_material(struct timer_list *timer)
+{
+	struct wg_peer *peer = from_timer(peer, timer, timer_zero_key_material);
+
+	rcu_read_lock_bh();
+	if (!READ_ONCE(peer->is_dead)) {
+		wg_peer_get(peer);
+		if (!queue_work(peer->device->handshake_send_wq,
+				&peer->clear_peer_work))
+			/* If the work was already on the queue, we want to drop
+			 * the extra reference.
+			 */
+			wg_peer_put(peer);
+	}
+	rcu_read_unlock_bh();
+}
+
+static void wg_queued_expired_zero_key_material(struct work_struct *work)
+{
+	struct wg_peer *peer = container_of(work, struct wg_peer,
+					    clear_peer_work);
+
+	pr_debug("%s: Zeroing out all keys for peer %llu (%pISpfsc), since we haven't received a new one in %d seconds\n",
+		 peer->device->dev->name, peer->internal_id,
+		 &peer->endpoint.addr, REJECT_AFTER_TIME * 3);
+	wg_noise_handshake_clear(&peer->handshake);
+	wg_noise_keypairs_clear(&peer->keypairs);
+	wg_peer_put(peer);
+}
+
+static void wg_expired_send_persistent_keepalive(struct timer_list *timer)
+{
+	struct wg_peer *peer = from_timer(peer, timer,
+					  timer_persistent_keepalive);
+
+	if (likely(peer->persistent_keepalive_interval))
+		wg_packet_send_keepalive(peer);
+}
+
+/* Should be called after an authenticated data packet is sent. */
+void wg_timers_data_sent(struct wg_peer *peer)
+{
+	if (!timer_pending(&peer->timer_new_handshake))
+		mod_peer_timer(peer, &peer->timer_new_handshake,
+			jiffies + (KEEPALIVE_TIMEOUT + REKEY_TIMEOUT) * HZ +
+			prandom_u32_max(REKEY_TIMEOUT_JITTER_MAX_JIFFIES));
+}
+
+/* Should be called after an authenticated data packet is received. */
+void wg_timers_data_received(struct wg_peer *peer)
+{
+	if (likely(netif_running(peer->device->dev))) {
+		if (!timer_pending(&peer->timer_send_keepalive))
+			mod_peer_timer(peer, &peer->timer_send_keepalive,
+				       jiffies + KEEPALIVE_TIMEOUT * HZ);
+		else
+			peer->timer_need_another_keepalive = true;
+	}
+}
+
+/* Should be called after any type of authenticated packet is sent, whether
+ * keepalive, data, or handshake.
+ */
+void wg_timers_any_authenticated_packet_sent(struct wg_peer *peer)
+{
+	del_timer(&peer->timer_send_keepalive);
+}
+
+/* Should be called after any type of authenticated packet is received, whether
+ * keepalive, data, or handshake.
+ */
+void wg_timers_any_authenticated_packet_received(struct wg_peer *peer)
+{
+	del_timer(&peer->timer_new_handshake);
+}
+
+/* Should be called after a handshake initiation message is sent. */
+void wg_timers_handshake_initiated(struct wg_peer *peer)
+{
+	mod_peer_timer(peer, &peer->timer_retransmit_handshake,
+		       jiffies + REKEY_TIMEOUT * HZ +
+		       prandom_u32_max(REKEY_TIMEOUT_JITTER_MAX_JIFFIES));
+}
+
+/* Should be called after a handshake response message is received and processed
+ * or when getting key confirmation via the first data message.
+ */
+void wg_timers_handshake_complete(struct wg_peer *peer)
+{
+	del_timer(&peer->timer_retransmit_handshake);
+	peer->timer_handshake_attempts = 0;
+	peer->sent_lastminute_handshake = false;
+	ktime_get_real_ts64(&peer->walltime_last_handshake);
+}
+
+/* Should be called after an ephemeral key is created, which is before sending a
+ * handshake response or after receiving a handshake response.
+ */
+void wg_timers_session_derived(struct wg_peer *peer)
+{
+	mod_peer_timer(peer, &peer->timer_zero_key_material,
+		       jiffies + REJECT_AFTER_TIME * 3 * HZ);
+}
+
+/* Should be called before a packet with authentication, whether
+ * keepalive, data, or handshakem is sent, or after one is received.
+ */
+void wg_timers_any_authenticated_packet_traversal(struct wg_peer *peer)
+{
+	if (peer->persistent_keepalive_interval)
+		mod_peer_timer(peer, &peer->timer_persistent_keepalive,
+			jiffies + peer->persistent_keepalive_interval * HZ);
+}
+
+void wg_timers_init(struct wg_peer *peer)
+{
+	timer_setup(&peer->timer_retransmit_handshake,
+		    wg_expired_retransmit_handshake, 0);
+	timer_setup(&peer->timer_send_keepalive, wg_expired_send_keepalive, 0);
+	timer_setup(&peer->timer_new_handshake, wg_expired_new_handshake, 0);
+	timer_setup(&peer->timer_zero_key_material,
+		    wg_expired_zero_key_material, 0);
+	timer_setup(&peer->timer_persistent_keepalive,
+		    wg_expired_send_persistent_keepalive, 0);
+	INIT_WORK(&peer->clear_peer_work, wg_queued_expired_zero_key_material);
+	peer->timer_handshake_attempts = 0;
+	peer->sent_lastminute_handshake = false;
+	peer->timer_need_another_keepalive = false;
+}
+
+void wg_timers_stop(struct wg_peer *peer)
+{
+	del_timer_sync(&peer->timer_retransmit_handshake);
+	del_timer_sync(&peer->timer_send_keepalive);
+	del_timer_sync(&peer->timer_new_handshake);
+	del_timer_sync(&peer->timer_zero_key_material);
+	del_timer_sync(&peer->timer_persistent_keepalive);
+	flush_work(&peer->clear_peer_work);
+}
diff -Naur --no-dereference a/net/wireguard/timers.h b/net/wireguard/timers.h
--- a/net/wireguard/timers.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/timers.h	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#ifndef _WG_TIMERS_H
+#define _WG_TIMERS_H
+
+#include <linux/ktime.h>
+
+struct wg_peer;
+
+void wg_timers_init(struct wg_peer *peer);
+void wg_timers_stop(struct wg_peer *peer);
+void wg_timers_data_sent(struct wg_peer *peer);
+void wg_timers_data_received(struct wg_peer *peer);
+void wg_timers_any_authenticated_packet_sent(struct wg_peer *peer);
+void wg_timers_any_authenticated_packet_received(struct wg_peer *peer);
+void wg_timers_handshake_initiated(struct wg_peer *peer);
+void wg_timers_handshake_complete(struct wg_peer *peer);
+void wg_timers_session_derived(struct wg_peer *peer);
+void wg_timers_any_authenticated_packet_traversal(struct wg_peer *peer);
+
+static inline bool wg_birthdate_has_expired(u64 birthday_nanoseconds,
+					    u64 expiration_seconds)
+{
+	return (s64)(birthday_nanoseconds + expiration_seconds * NSEC_PER_SEC)
+		<= (s64)ktime_get_coarse_boottime_ns();
+}
+
+#endif /* _WG_TIMERS_H */
diff -Naur --no-dereference a/net/wireguard/uapi/wireguard.h b/net/wireguard/uapi/wireguard.h
--- a/net/wireguard/uapi/wireguard.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/uapi/wireguard.h	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,196 @@
+/* SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR MIT */
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ *
+ * Documentation
+ * =============
+ *
+ * The below enums and macros are for interfacing with WireGuard, using generic
+ * netlink, with family WG_GENL_NAME and version WG_GENL_VERSION. It defines two
+ * methods: get and set. Note that while they share many common attributes,
+ * these two functions actually accept a slightly different set of inputs and
+ * outputs.
+ *
+ * WG_CMD_GET_DEVICE
+ * -----------------
+ *
+ * May only be called via NLM_F_REQUEST | NLM_F_DUMP. The command should contain
+ * one but not both of:
+ *
+ *    WGDEVICE_A_IFINDEX: NLA_U32
+ *    WGDEVICE_A_IFNAME: NLA_NUL_STRING, maxlen IFNAMSIZ - 1
+ *
+ * The kernel will then return several messages (NLM_F_MULTI) containing the
+ * following tree of nested items:
+ *
+ *    WGDEVICE_A_IFINDEX: NLA_U32
+ *    WGDEVICE_A_IFNAME: NLA_NUL_STRING, maxlen IFNAMSIZ - 1
+ *    WGDEVICE_A_PRIVATE_KEY: NLA_EXACT_LEN, len WG_KEY_LEN
+ *    WGDEVICE_A_PUBLIC_KEY: NLA_EXACT_LEN, len WG_KEY_LEN
+ *    WGDEVICE_A_LISTEN_PORT: NLA_U16
+ *    WGDEVICE_A_FWMARK: NLA_U32
+ *    WGDEVICE_A_PEERS: NLA_NESTED
+ *        0: NLA_NESTED
+ *            WGPEER_A_PUBLIC_KEY: NLA_EXACT_LEN, len WG_KEY_LEN
+ *            WGPEER_A_PRESHARED_KEY: NLA_EXACT_LEN, len WG_KEY_LEN
+ *            WGPEER_A_ENDPOINT: NLA_MIN_LEN(struct sockaddr), struct sockaddr_in or struct sockaddr_in6
+ *            WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL: NLA_U16
+ *            WGPEER_A_LAST_HANDSHAKE_TIME: NLA_EXACT_LEN, struct __kernel_timespec
+ *            WGPEER_A_RX_BYTES: NLA_U64
+ *            WGPEER_A_TX_BYTES: NLA_U64
+ *            WGPEER_A_ALLOWEDIPS: NLA_NESTED
+ *                0: NLA_NESTED
+ *                    WGALLOWEDIP_A_FAMILY: NLA_U16
+ *                    WGALLOWEDIP_A_IPADDR: NLA_MIN_LEN(struct in_addr), struct in_addr or struct in6_addr
+ *                    WGALLOWEDIP_A_CIDR_MASK: NLA_U8
+ *                0: NLA_NESTED
+ *                    ...
+ *                0: NLA_NESTED
+ *                    ...
+ *                ...
+ *            WGPEER_A_PROTOCOL_VERSION: NLA_U32
+ *        0: NLA_NESTED
+ *            ...
+ *        ...
+ *
+ * It is possible that all of the allowed IPs of a single peer will not
+ * fit within a single netlink message. In that case, the same peer will
+ * be written in the following message, except it will only contain
+ * WGPEER_A_PUBLIC_KEY and WGPEER_A_ALLOWEDIPS. This may occur several
+ * times in a row for the same peer. It is then up to the receiver to
+ * coalesce adjacent peers. Likewise, it is possible that all peers will
+ * not fit within a single message. So, subsequent peers will be sent
+ * in following messages, except those will only contain WGDEVICE_A_IFNAME
+ * and WGDEVICE_A_PEERS. It is then up to the receiver to coalesce these
+ * messages to form the complete list of peers.
+ *
+ * Since this is an NLA_F_DUMP command, the final message will always be
+ * NLMSG_DONE, even if an error occurs. However, this NLMSG_DONE message
+ * contains an integer error code. It is either zero or a negative error
+ * code corresponding to the errno.
+ *
+ * WG_CMD_SET_DEVICE
+ * -----------------
+ *
+ * May only be called via NLM_F_REQUEST. The command should contain the
+ * following tree of nested items, containing one but not both of
+ * WGDEVICE_A_IFINDEX and WGDEVICE_A_IFNAME:
+ *
+ *    WGDEVICE_A_IFINDEX: NLA_U32
+ *    WGDEVICE_A_IFNAME: NLA_NUL_STRING, maxlen IFNAMSIZ - 1
+ *    WGDEVICE_A_FLAGS: NLA_U32, 0 or WGDEVICE_F_REPLACE_PEERS if all current
+ *                      peers should be removed prior to adding the list below.
+ *    WGDEVICE_A_PRIVATE_KEY: len WG_KEY_LEN, all zeros to remove
+ *    WGDEVICE_A_LISTEN_PORT: NLA_U16, 0 to choose randomly
+ *    WGDEVICE_A_FWMARK: NLA_U32, 0 to disable
+ *    WGDEVICE_A_PEERS: NLA_NESTED
+ *        0: NLA_NESTED
+ *            WGPEER_A_PUBLIC_KEY: len WG_KEY_LEN
+ *            WGPEER_A_FLAGS: NLA_U32, 0 and/or WGPEER_F_REMOVE_ME if the
+ *                            specified peer should not exist at the end of the
+ *                            operation, rather than added/updated and/or
+ *                            WGPEER_F_REPLACE_ALLOWEDIPS if all current allowed
+ *                            IPs of this peer should be removed prior to adding
+ *                            the list below and/or WGPEER_F_UPDATE_ONLY if the
+ *                            peer should only be set if it already exists.
+ *            WGPEER_A_PRESHARED_KEY: len WG_KEY_LEN, all zeros to remove
+ *            WGPEER_A_ENDPOINT: struct sockaddr_in or struct sockaddr_in6
+ *            WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL: NLA_U16, 0 to disable
+ *            WGPEER_A_ALLOWEDIPS: NLA_NESTED
+ *                0: NLA_NESTED
+ *                    WGALLOWEDIP_A_FAMILY: NLA_U16
+ *                    WGALLOWEDIP_A_IPADDR: struct in_addr or struct in6_addr
+ *                    WGALLOWEDIP_A_CIDR_MASK: NLA_U8
+ *                0: NLA_NESTED
+ *                    ...
+ *                0: NLA_NESTED
+ *                    ...
+ *                ...
+ *            WGPEER_A_PROTOCOL_VERSION: NLA_U32, should not be set or used at
+ *                                       all by most users of this API, as the
+ *                                       most recent protocol will be used when
+ *                                       this is unset. Otherwise, must be set
+ *                                       to 1.
+ *        0: NLA_NESTED
+ *            ...
+ *        ...
+ *
+ * It is possible that the amount of configuration data exceeds that of
+ * the maximum message length accepted by the kernel. In that case, several
+ * messages should be sent one after another, with each successive one
+ * filling in information not contained in the prior. Note that if
+ * WGDEVICE_F_REPLACE_PEERS is specified in the first message, it probably
+ * should not be specified in fragments that come after, so that the list
+ * of peers is only cleared the first time but appended after. Likewise for
+ * peers, if WGPEER_F_REPLACE_ALLOWEDIPS is specified in the first message
+ * of a peer, it likely should not be specified in subsequent fragments.
+ *
+ * If an error occurs, NLMSG_ERROR will reply containing an errno.
+ */
+
+#ifndef _WG_UAPI_WIREGUARD_H
+#define _WG_UAPI_WIREGUARD_H
+
+#define WG_GENL_NAME "wireguard"
+#define WG_GENL_VERSION 1
+
+#define WG_KEY_LEN 32
+
+enum wg_cmd {
+	WG_CMD_GET_DEVICE,
+	WG_CMD_SET_DEVICE,
+	__WG_CMD_MAX
+};
+#define WG_CMD_MAX (__WG_CMD_MAX - 1)
+
+enum wgdevice_flag {
+	WGDEVICE_F_REPLACE_PEERS = 1U << 0,
+	__WGDEVICE_F_ALL = WGDEVICE_F_REPLACE_PEERS
+};
+enum wgdevice_attribute {
+	WGDEVICE_A_UNSPEC,
+	WGDEVICE_A_IFINDEX,
+	WGDEVICE_A_IFNAME,
+	WGDEVICE_A_PRIVATE_KEY,
+	WGDEVICE_A_PUBLIC_KEY,
+	WGDEVICE_A_FLAGS,
+	WGDEVICE_A_LISTEN_PORT,
+	WGDEVICE_A_FWMARK,
+	WGDEVICE_A_PEERS,
+	__WGDEVICE_A_LAST
+};
+#define WGDEVICE_A_MAX (__WGDEVICE_A_LAST - 1)
+
+enum wgpeer_flag {
+	WGPEER_F_REMOVE_ME = 1U << 0,
+	WGPEER_F_REPLACE_ALLOWEDIPS = 1U << 1,
+	WGPEER_F_UPDATE_ONLY = 1U << 2,
+	__WGPEER_F_ALL = WGPEER_F_REMOVE_ME | WGPEER_F_REPLACE_ALLOWEDIPS |
+			 WGPEER_F_UPDATE_ONLY
+};
+enum wgpeer_attribute {
+	WGPEER_A_UNSPEC,
+	WGPEER_A_PUBLIC_KEY,
+	WGPEER_A_PRESHARED_KEY,
+	WGPEER_A_FLAGS,
+	WGPEER_A_ENDPOINT,
+	WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL,
+	WGPEER_A_LAST_HANDSHAKE_TIME,
+	WGPEER_A_RX_BYTES,
+	WGPEER_A_TX_BYTES,
+	WGPEER_A_ALLOWEDIPS,
+	WGPEER_A_PROTOCOL_VERSION,
+	__WGPEER_A_LAST
+};
+#define WGPEER_A_MAX (__WGPEER_A_LAST - 1)
+
+enum wgallowedip_attribute {
+	WGALLOWEDIP_A_UNSPEC,
+	WGALLOWEDIP_A_FAMILY,
+	WGALLOWEDIP_A_IPADDR,
+	WGALLOWEDIP_A_CIDR_MASK,
+	__WGALLOWEDIP_A_LAST
+};
+#define WGALLOWEDIP_A_MAX (__WGALLOWEDIP_A_LAST - 1)
+
+#endif /* _WG_UAPI_WIREGUARD_H */
diff -Naur --no-dereference a/net/wireguard/version.h b/net/wireguard/version.h
--- a/net/wireguard/version.h	1969-12-31 19:00:00.000000000 -0500
+++ b/net/wireguard/version.h	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1 @@
+#define WIREGUARD_VERSION "0.0.20191219"
diff -Naur --no-dereference a/net/wireless/nl80211.c b/net/wireless/nl80211.c
--- a/net/wireless/nl80211.c	2020-01-09 04:19:10.000000000 -0500
+++ b/net/wireless/nl80211.c	2020-06-19 21:00:31.436227717 -0400
@@ -456,6 +456,8 @@
 	[NL80211_ATTR_FILS_CACHE_ID] = { .len = 2 },
 	[NL80211_ATTR_PMK] = { .type = NLA_BINARY, .len = PMK_MAX_LEN },
 	[NL80211_ATTR_SCHED_SCAN_MULTI] = { .type = NLA_FLAG },
+	[NL80211_ATTR_SAE_PASSWORD] = { .type = NLA_BINARY,
+					.len = SAE_PASSWORD_MAX_LEN },
 	[NL80211_ATTR_EXTERNAL_AUTH_SUPPORT] = { .type = NLA_FLAG },
 
 	[NL80211_ATTR_TXQ_LIMIT] = { .type = NLA_U32 },
@@ -4184,6 +4186,8 @@
 		return true;
 	case NL80211_CMD_CONNECT:
 		if (!(rdev->wiphy.features & NL80211_FEATURE_SAE) &&
+		    !wiphy_ext_feature_isset(&rdev->wiphy,
+					     NL80211_EXT_FEATURE_SAE_OFFLOAD) &&
 		    auth_type == NL80211_AUTHTYPE_SAE)
 			return false;
 
@@ -4198,8 +4202,9 @@
 			return false;
 		return true;
 	case NL80211_CMD_START_AP:
-		/* SAE not supported yet */
-		if (auth_type == NL80211_AUTHTYPE_SAE)
+		if (!wiphy_ext_feature_isset(&rdev->wiphy,
+					     NL80211_EXT_FEATURE_SAE_OFFLOAD) &&
+		    auth_type == NL80211_AUTHTYPE_SAE)
 			return false;
 		/* FILS not supported yet */
 		if (auth_type == NL80211_AUTHTYPE_FILS_SK ||
@@ -8406,7 +8411,8 @@
 static bool nl80211_valid_wpa_versions(u32 wpa_versions)
 {
 	return !(wpa_versions & ~(NL80211_WPA_VERSION_1 |
-				  NL80211_WPA_VERSION_2));
+				  NL80211_WPA_VERSION_2 |
+				  NL80211_WPA_VERSION_3));
 }
 
 static int nl80211_authenticate(struct sk_buff *skb, struct genl_info *info)
@@ -8640,11 +8646,23 @@
 		if (nla_len(info->attrs[NL80211_ATTR_PMK]) != WLAN_PMK_LEN)
 			return -EINVAL;
 		if (!wiphy_ext_feature_isset(&rdev->wiphy,
-					     NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK))
+					     NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK) &&
+		    !wiphy_ext_feature_isset(&rdev->wiphy,
+					     NL80211_EXT_FEATURE_4WAY_HANDSHAKE_AP_PSK))
 			return -EINVAL;
 		settings->psk = nla_data(info->attrs[NL80211_ATTR_PMK]);
 	}
 
+	if (info->attrs[NL80211_ATTR_SAE_PASSWORD]) {
+		if (!wiphy_ext_feature_isset(&rdev->wiphy,
+					     NL80211_EXT_FEATURE_SAE_OFFLOAD))
+			return -EINVAL;
+		settings->sae_pwd =
+			nla_data(info->attrs[NL80211_ATTR_SAE_PASSWORD]);
+		settings->sae_pwd_len =
+			nla_len(info->attrs[NL80211_ATTR_SAE_PASSWORD]);
+	}
+
 	return 0;
 }
 
@@ -14465,6 +14483,8 @@
 	     (nla_put_flag(msg, NL80211_ATTR_TIMED_OUT) ||
 	      nla_put_u32(msg, NL80211_ATTR_TIMEOUT_REASON,
 			  cr->timeout_reason))) ||
+	    (cr->authorized &&
+	     nla_put_flag(msg, NL80211_ATTR_PORT_AUTHORIZED)) ||
 	    (cr->req_ie &&
 	     nla_put(msg, NL80211_ATTR_REQ_IE, cr->req_ie_len, cr->req_ie)) ||
 	    (cr->resp_ie &&
@@ -14522,6 +14542,8 @@
 	    (info->resp_ie &&
 	     nla_put(msg, NL80211_ATTR_RESP_IE, info->resp_ie_len,
 		     info->resp_ie)) ||
+	    (info->authorized &&
+	     nla_put_flag(msg, NL80211_ATTR_PORT_AUTHORIZED)) ||
 	    (info->fils.update_erp_next_seq_num &&
 	     nla_put_u16(msg, NL80211_ATTR_FILS_ERP_NEXT_SEQ_NUM,
 			 info->fils.erp_next_seq_num)) ||
@@ -14560,7 +14582,8 @@
 		return;
 	}
 
-	if (nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN, bssid))
+	if (nla_put_u32(msg, NL80211_ATTR_IFINDEX, netdev->ifindex) ||
+	    nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN, bssid))
 		goto nla_put_failure;
 
 	genlmsg_end(msg, hdr);
diff -Naur --no-dereference a/net/wireless/sme.c b/net/wireless/sme.c
--- a/net/wireless/sme.c	2020-01-09 04:19:10.000000000 -0500
+++ b/net/wireless/sme.c	2020-06-19 21:00:31.436227717 -0400
@@ -863,6 +863,7 @@
 	ev->cr.bss = params->bss;
 	ev->cr.status = params->status;
 	ev->cr.timeout_reason = params->timeout_reason;
+	ev->cr.authorized = params->authorized;
 
 	spin_lock_irqsave(&wdev->event_lock, flags);
 	list_add_tail(&ev->list, &wdev->event_list);
@@ -996,6 +997,7 @@
 	if (info->fils.update_erp_next_seq_num)
 		ev->rm.fils.erp_next_seq_num = info->fils.erp_next_seq_num;
 	ev->rm.bss = info->bss;
+	ev->rm.authorized = info->authorized;
 
 	spin_lock_irqsave(&wdev->event_lock, flags);
 	list_add_tail(&ev->list, &wdev->event_list);
diff -Naur --no-dereference a/README.md b/README.md
--- a/README.md	1969-12-31 19:00:00.000000000 -0500
+++ b/README.md	2020-06-19 21:00:31.404227715 -0400
@@ -0,0 +1,7 @@
+| kernel | normal | rt |
+|:---------:|:--------:|:-------:|
+|4.4  | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.4)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.4/)   | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.4-rt)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.4-rt/)   |
+|4.9  | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.9)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.9/)   | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.9-rt)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.9-rt/)   |
+|4.14 | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.14)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.14/) | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.14-rt)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.14-rt/) |
+|4.19 | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.19)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.19/) | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.19-rt)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.19-rt/) |
+|5.4  | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/5.4)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/5.4/)   | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/5.4-rt)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/5.4-rt/)   |
diff -Naur --no-dereference a/samples/rpmsg/rpmsg_client_sample.c b/samples/rpmsg/rpmsg_client_sample.c
--- a/samples/rpmsg/rpmsg_client_sample.c	2020-01-09 04:19:10.000000000 -0500
+++ b/samples/rpmsg/rpmsg_client_sample.c	2020-06-19 21:00:31.436227717 -0400
@@ -22,7 +22,9 @@
 #include <linux/rpmsg.h>
 
 #define MSG		"hello world!"
-#define MSG_LIMIT	100
+
+static int count = 100;
+module_param(count, int, 0644);
 
 struct instance_data {
 	int rx_count;
@@ -37,11 +39,11 @@
 	dev_info(&rpdev->dev, "incoming msg %d (src: 0x%x)\n",
 		 ++idata->rx_count, src);
 
-	print_hex_dump(KERN_DEBUG, __func__, DUMP_PREFIX_NONE, 16, 1,
-		       data, len,  true);
+	print_hex_dump_debug(__func__, DUMP_PREFIX_NONE, 16, 1, data, len,
+			     true);
 
 	/* samples should not live forever */
-	if (idata->rx_count >= MSG_LIMIT) {
+	if (idata->rx_count >= count) {
 		dev_info(&rpdev->dev, "goodbye!\n");
 		return 0;
 	}
@@ -85,6 +87,7 @@
 
 static struct rpmsg_device_id rpmsg_driver_sample_id_table[] = {
 	{ .name	= "rpmsg-client-sample" },
+	{ .name	= "ti.ipc4.ping-pong" },
 	{ },
 };
 MODULE_DEVICE_TABLE(rpmsg, rpmsg_driver_sample_id_table);
diff -Naur --no-dereference a/samples/seccomp/Makefile b/samples/seccomp/Makefile
--- a/samples/seccomp/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/samples/seccomp/Makefile	2020-06-19 21:00:31.436227717 -0400
@@ -18,6 +18,7 @@
 
 # Try to match the kernel target.
 ifndef CONFIG_64BIT
+ifndef CONFIG_ARM
 
 # s390 has -m31 flag to build 31 bit binaries
 ifndef CONFIG_S390
@@ -36,3 +37,4 @@
 endif
 always := $(hostprogs-m)
 endif
+endif
diff -Naur --no-dereference a/scripts/dtb-merge b/scripts/dtb-merge
--- a/scripts/dtb-merge	1969-12-31 19:00:00.000000000 -0500
+++ b/scripts/dtb-merge	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,110 @@
+#!/usr/bin/perl
+use strict;
+
+my $debug = 0;
+
+die "Usage: $ARGV[0] <srctree> <objtree> <dtb-file> <overlay-tool> <its-dir>\n"
+		if @ARGV != 5;
+
+my $srctree = shift @ARGV;
+my $objtree = shift @ARGV;
+my $dtb_tgt = shift @ARGV;
+my $fdtoverlay = shift @ARGV;
+my $dir = shift @ARGV;
+
+print "  DTC-M   $dtb_tgt\n";
+
+# Find all .its files
+my @its_files = glob "$srctree/$dir/*.its";
+
+my $tgt;
+
+# Generate target config string
+if ($dtb_tgt =~ /\/([^\/]+)\.dtb$/) {
+  $tgt = $1;
+}
+
+my @dtbs;
+
+# Parse all .its files to find corresponding config
+foreach my $its_file (@its_files) {
+  open my $in,"<$its_file" or die "Unable to open $its_file\n";
+
+  my $conf = 0;
+  my $image = 0;
+
+  my %conf_map = ();
+  my %image_map = ();
+
+  foreach (<$in>) {
+    if (/^\s*images \{/) {
+      $image = 1;
+      next;
+    }
+    if ($image && /(\S*) \{/) {
+      $image = $1;
+    }
+    if ($image && /data\s*=\s*\/incbin\// && /\"(.*)\"/) {
+      $image_map{$image} = $1;
+    }
+    if (/^\s*configurations \{/) {
+      $conf = 1;
+      $image = 0;
+      next;
+    }
+    if ($conf && /(\S*) \{/) {
+      $conf = $1;
+      $conf =~ s/\.dtb$//;
+    }
+    if ($conf && /fdt = (.*);/) {
+      my $fdts = $1;
+      $fdts =~ s/ |"//g;
+      $conf_map{$conf} = { fdts => [] };
+      foreach my $fdt (split ",", $fdts) {
+        push @{ $conf_map{$conf}->{fdts} }, $image_map{$fdt};
+      }
+    }
+  }
+
+  close $in;
+
+  # Try matching the config
+  @dtbs = find_config(\%conf_map, $tgt);
+  last if @dtbs;
+}
+
+die "No config found for $tgt\n" if !@dtbs;
+
+print "Found config for $tgt:\n" if $debug;
+
+# Generate the merged dtb file
+my $cmd = "$fdtoverlay";
+$cmd .= " -v" if $debug;
+$cmd .= " -o $dtb_tgt -i";
+
+foreach my $dtb (@dtbs) {
+  $cmd .= " $objtree/$dir/$dtb";
+}
+
+exit system($cmd);
+
+sub find_config {
+  my $map = shift @_;
+  my $conf = shift @_;
+
+  my @confs = split("-",$conf);
+
+  for (my $i = @confs - 1; $i >= 0; $i--) {
+    my $tmp = join("-",@confs[0 .. $i]);
+    if ($map->{$tmp}) {
+      if ($i == @confs - 1) {
+        return @{ $map->{$tmp}->{fdts} };
+      } else {
+        my @res = find_config($map, join("-",@confs[$i + 1 .. @confs - 1]));
+        return @{ $map->{$tmp}->{fdts} }, @res if @res;
+      }
+    }
+  }
+
+  return ();
+}
diff -Naur --no-dereference a/scripts/dtc/fdtoverlay.c b/scripts/dtc/fdtoverlay.c
--- a/scripts/dtc/fdtoverlay.c	1969-12-31 19:00:00.000000000 -0500
+++ b/scripts/dtc/fdtoverlay.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,175 @@
+/*
+ * Copyright (c) 2017 Konsulko Group Inc. All rights reserved.
+ *
+ * Author:
+ *	 Pantelis Antoniou <pantelis.antoniou@konsulko.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+ * MA 02111-1307 USA
+ */
+
+#include <assert.h>
+#include <ctype.h>
+#include <getopt.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <alloca.h>
+#include <inttypes.h>
+
+#include <libfdt.h>
+
+#include "util.h"
+
+/* Usage related data. */
+static const char usage_synopsis[] =
+	"apply a number of overlays to a base blob\n"
+	"	fdtoverlay <options> [<overlay.dtbo> [<overlay.dtbo>]]\n"
+	"\n"
+	USAGE_TYPE_MSG;
+static const char usage_short_opts[] = "i:o:v" USAGE_COMMON_SHORT_OPTS;
+static struct option const usage_long_opts[] = {
+	{"input",            required_argument, NULL, 'i'},
+	{"output",	     required_argument, NULL, 'o'},
+	{"verbose",	           no_argument, NULL, 'v'},
+	USAGE_COMMON_LONG_OPTS,
+};
+static const char * const usage_opts_help[] = {
+	"Input base DT blob",
+	"Output DT blob",
+	"Verbose messages",
+	USAGE_COMMON_OPTS_HELP
+};
+
+int verbose = 0;
+
+static int do_fdtoverlay(const char *input_filename,
+			 const char *output_filename,
+			 int argc, char *argv[])
+{
+	char *blob = NULL;
+	char **ovblob = NULL;
+	off_t blob_len, ov_len, total_len;
+	int i, ret = -1;
+
+	blob = utilfdt_read_len(input_filename, &blob_len);
+	if (!blob) {
+		fprintf(stderr, "\nFailed to read base blob %s\n",
+				input_filename);
+		goto out_err;
+	}
+	if (fdt_totalsize(blob) > blob_len) {
+		fprintf(stderr,
+ "\nBase blob is incomplete (%lu / %" PRIu32 " bytes read)\n",
+			(unsigned long)blob_len, fdt_totalsize(blob));
+		goto out_err;
+	}
+	ret = 0;
+
+	/* allocate blob pointer array */
+	ovblob = alloca(sizeof(*ovblob) * argc);
+	memset(ovblob, 0, sizeof(*ovblob) * argc);
+
+	/* read and keep track of the overlay blobs */
+	total_len = 0;
+	for (i = 0; i < argc; i++) {
+		ovblob[i] = utilfdt_read_len(argv[i], &ov_len);
+		if (!ovblob[i]) {
+			fprintf(stderr, "\nFailed to read overlay %s\n",
+					argv[i]);
+			goto out_err;
+		}
+		total_len += ov_len;
+	}
+
+	/* grow the blob to worst case */
+	blob_len = fdt_totalsize(blob) + total_len;
+	blob = xrealloc(blob, blob_len);
+	fdt_open_into(blob, blob, blob_len);
+
+	/* apply the overlays in sequence */
+	for (i = 0; i < argc; i++) {
+		ret = fdt_overlay_apply(blob, ovblob[i]);
+		if (ret) {
+			fprintf(stderr, "\nFailed to apply %s (%d)\n",
+					argv[i], ret);
+			goto out_err;
+		}
+	}
+
+	fdt_pack(blob);
+	ret = utilfdt_write(output_filename, blob);
+	if (ret)
+		fprintf(stderr, "\nFailed to write output blob %s\n",
+				output_filename);
+
+out_err:
+	if (ovblob) {
+		for (i = 0; i < argc; i++) {
+			if (ovblob[i])
+				free(ovblob[i]);
+		}
+	}
+	free(blob);
+
+	return ret;
+}
+
+int main(int argc, char *argv[])
+{
+	int opt, i;
+	char *input_filename = NULL;
+	char *output_filename = NULL;
+
+	while ((opt = util_getopt_long()) != EOF) {
+		switch (opt) {
+		case_USAGE_COMMON_FLAGS
+
+		case 'i':
+			input_filename = optarg;
+			break;
+		case 'o':
+			output_filename = optarg;
+			break;
+		case 'v':
+			verbose = 1;
+			break;
+		}
+	}
+
+	if (!input_filename)
+		usage("missing input file");
+
+	if (!output_filename)
+		usage("missing output file");
+
+	argv += optind;
+	argc -= optind;
+
+	if (argc <= 0)
+		usage("missing overlay file(s)");
+
+	if (verbose) {
+		printf("input  = %s\n", input_filename);
+		printf("output = %s\n", output_filename);
+		for (i = 0; i < argc; i++)
+			printf("overlay[%d] = %s\n", i, argv[i]);
+	}
+
+	if (do_fdtoverlay(input_filename, output_filename, argc, argv))
+		return 1;
+
+	return 0;
+}
diff -Naur --no-dereference a/scripts/dtc/.gitignore b/scripts/dtc/.gitignore
--- a/scripts/dtc/.gitignore	2020-01-09 04:19:10.000000000 -0500
+++ b/scripts/dtc/.gitignore	2020-06-19 21:00:31.436227717 -0400
@@ -1 +1,2 @@
 dtc
+fdtoverlay
diff -Naur --no-dereference a/scripts/dtc/Makefile b/scripts/dtc/Makefile
--- a/scripts/dtc/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/scripts/dtc/Makefile	2020-06-19 21:00:31.436227717 -0400
@@ -1,19 +1,39 @@
 # SPDX-License-Identifier: GPL-2.0
 # scripts/dtc makefile
 
-hostprogs-y	:= dtc
+hostprogs-y	:= dtc fdtoverlay
 always		:= $(hostprogs-y)
 
 dtc-objs	:= dtc.o flattree.o fstree.o data.o livetree.o treesource.o \
 		   srcpos.o checks.o util.o
 dtc-objs	+= dtc-lexer.lex.o dtc-parser.tab.o
 
+fdtoverlay-objs := fdtoverlay.o util.o
+fdtoverlay-objs += $(subst $(srctree)/$(src)/,,$(patsubst %.c,%.o,$(wildcard $(srctree)/$(src)/libfdt/*.c)))
+
 # Source files need to get at the userspace version of libfdt_env.h to compile
 HOST_EXTRACFLAGS := -I$(src)/libfdt
 
 # Generated files need one more search path to include headers in source tree
 HOSTCFLAGS_dtc-lexer.lex.o := -I$(src)
 HOSTCFLAGS_dtc-parser.tab.o := -I$(src)
+HOSTCFLAGS_DTC := -I$(src) -I$(src)/libfdt
+
+HOSTCFLAGS_checks.o := $(HOSTCFLAGS_DTC)
+HOSTCFLAGS_data.o := $(HOSTCFLAGS_DTC)
+HOSTCFLAGS_dtc.o := $(HOSTCFLAGS_DTC)
+HOSTCFLAGS_flattree.o := $(HOSTCFLAGS_DTC)
+HOSTCFLAGS_fstree.o := $(HOSTCFLAGS_DTC)
+HOSTCFLAGS_livetree.o := $(HOSTCFLAGS_DTC)
+HOSTCFLAGS_srcpos.o := $(HOSTCFLAGS_DTC)
+HOSTCFLAGS_treesource.o := $(HOSTCFLAGS_DTC)
+HOSTCFLAGS_util.o := $(HOSTCFLAGS_DTC)
+HOSTCFLAGS_fdtoverlay.o := $(HOSTCFLAGS_DTC)
+
+HOSTCFLAGS_dtc-lexer.lex.o := $(HOSTCFLAGS_DTC)
+HOSTCFLAGS_dtc-parser.tab.o := $(HOSTCFLAGS_DTC)
+
+HOSTCFLAGS += $(HOSTCFLAGS_DTC)
 
 # dependencies on generated files need to be listed explicitly
 $(obj)/dtc-lexer.lex.o: $(obj)/dtc-parser.tab.h
diff -Naur --no-dereference a/scripts/Makefile.dtbinst b/scripts/Makefile.dtbinst
--- a/scripts/Makefile.dtbinst	2020-01-09 04:19:10.000000000 -0500
+++ b/scripts/Makefile.dtbinst	2020-06-19 21:00:31.436227717 -0400
@@ -28,13 +28,22 @@
 
 install-dir = $(patsubst $(dtbinst_root)%,$(INSTALL_DTBS_PATH)%,$(obj))
 
-$(dtbinst-files): %.dtb: $(obj)/%.dtb
+$(filter %.dtb,$(dtbinst-files)): %.dtb: $(obj)/%.dtb
+	$(call cmd,dtb_install,$(install-dir))
+
+$(filter %.dtbo,$(dtbinst-files)): %.dtbo: $(obj)/%.dtbo
+	$(call cmd,dtb_install,$(install-dir))
+
+$(filter %.itb,$(dtbinst-files)): %.itb: $(obj)/%.itb
 	$(call cmd,dtb_install,$(install-dir))
 
 $(dtbinst-dirs):
 	$(Q)$(MAKE) $(dtbinst)=$(obj)/$@
 
-PHONY += $(dtbinst-files) $(dtbinst-dirs)
-__dtbs_install: $(dtbinst-files) $(dtbinst-dirs)
+dummy:
+	@echo > /dev/null
+
+PHONY += $(dtbinst-files) $(dtbinst-dirs) dummy
+__dtbs_install: $(dtbinst-files) $(dtbinst-dirs) dummy
 
 .PHONY: $(PHONY)
diff -Naur --no-dereference a/scripts/package/builddeb b/scripts/package/builddeb
--- a/scripts/package/builddeb	2020-01-09 04:19:10.000000000 -0500
+++ b/scripts/package/builddeb	2020-06-19 21:00:31.436227717 -0400
@@ -12,6 +12,18 @@
 
 set -e
 
+is_enabled() {
+	grep -q "^$1=y" include/config/auto.conf
+}
+
+if_enabled_echo() {
+	if is_enabled "$1"; then
+		echo -n "$2"
+	elif [ $# -ge 3 ]; then
+		echo -n "$3"
+	fi
+}
+
 create_package() {
 	local pname="$1" pdir="$2"
 
@@ -62,7 +74,7 @@
 	installed_image_path="boot/vmlinuz-$version"
 esac
 
-BUILD_DEBUG="$(grep -s '^CONFIG_DEBUG_INFO=y' $KCONFIG_CONFIG || true)"
+BUILD_DEBUG=$(if_enabled_echo CONFIG_DEBUG_INFO Yes)
 
 # Setup the directory structure
 rm -rf "$tmpdir" "$kernel_headers_dir" "$libc_headers_dir" "$dbg_dir" $objtree/debian/files
@@ -83,15 +95,16 @@
 fi
 cp "$($MAKE -s -f $srctree/Makefile image_name)" "$tmpdir/$installed_image_path"
 
-if grep -q "^CONFIG_OF=y" $KCONFIG_CONFIG ; then
+if is_enabled CONFIG_OF_EARLY_FLATTREE; then
 	# Only some architectures with OF support have this target
-	if grep -q dtbs_install "${srctree}/arch/$SRCARCH/Makefile"; then
-		$MAKE KBUILD_SRC= INSTALL_DTBS_PATH="$tmpdir/usr/lib/$packagename" dtbs_install
+	if [ -d "${srctree}/arch/$SRCARCH/boot/dts" ]; then
+		$MAKE -f $srctree/Makefile INSTALL_DTBS_PATH="$tmpdir/usr/lib/$packagename" dtbs_install
+		$MAKE -f $srctree/Makefile INSTALL_DTBS_PATH="$tmpdir/boot/dtbs/$version" dtbs_install
 	fi
 fi
 
-if grep -q '^CONFIG_MODULES=y' $KCONFIG_CONFIG ; then
-	INSTALL_MOD_PATH="$tmpdir" $MAKE KBUILD_SRC= modules_install
+if is_enabled CONFIG_MODULES; then
+	INSTALL_MOD_PATH="$tmpdir" $MAKE -f $srctree/Makefile modules_install
 	rm -f "$tmpdir/lib/modules/$version/build"
 	rm -f "$tmpdir/lib/modules/$version/source"
 	if [ "$ARCH" = "um" ] ; then
@@ -111,16 +124,15 @@
 		done
 
 		# resign stripped modules
-		MODULE_SIG_ALL="$(grep -s '^CONFIG_MODULE_SIG_ALL=y' $KCONFIG_CONFIG || true)"
-		if [ -n "$MODULE_SIG_ALL" ]; then
-			INSTALL_MOD_PATH="$tmpdir" $MAKE KBUILD_SRC= modules_sign
+		if is_enabled CONFIG_MODULE_SIG_ALL; then
+			INSTALL_MOD_PATH="$tmpdir" $MAKE -f $srctree/Makefile modules_sign
 		fi
 	fi
 fi
 
 if [ "$ARCH" != "um" ]; then
-	$MAKE headers_check KBUILD_SRC=
-	$MAKE headers_install KBUILD_SRC= INSTALL_HDR_PATH="$libc_headers_dir/usr"
+	$MAKE -f $srctree/Makefile headers_check
+	$MAKE -f $srctree/Makefile headers_install INSTALL_HDR_PATH="$libc_headers_dir/usr"
 fi
 
 # Install the maintainer scripts
@@ -129,11 +141,6 @@
 # make-kpkg sets $INITRD to indicate whether an initramfs is wanted, and
 # so do we; recent versions of dracut and initramfs-tools will obey this.
 debhookdir=${KDEB_HOOKDIR:-/etc/kernel}
-if grep -q '^CONFIG_BLK_DEV_INITRD=y' $KCONFIG_CONFIG; then
-	want_initrd=Yes
-else
-	want_initrd=No
-fi
 for script in postinst postrm preinst prerm ; do
 	mkdir -p "$tmpdir$debhookdir/$script.d"
 	cat <<EOF > "$tmpdir/DEBIAN/$script"
@@ -145,7 +152,7 @@
 export DEB_MAINT_PARAMS="\$*"
 
 # Tell initramfs builder whether it's wanted
-export INITRD=$want_initrd
+export INITRD=$(if_enabled_echo CONFIG_BLK_DEV_INITRD Yes No)
 
 test -d $debhookdir/$script.d && run-parts --arg="$version" --arg="/$installed_image_path" $debhookdir/$script.d
 exit 0
@@ -158,11 +165,11 @@
 (cd $srctree; find arch/*/include include scripts -type f -o -type l) >> "$objtree/debian/hdrsrcfiles"
 (cd $srctree; find arch/$SRCARCH -name module.lds -o -name Kbuild.platforms -o -name Platform) >> "$objtree/debian/hdrsrcfiles"
 (cd $srctree; find $(find arch/$SRCARCH -name include -o -name scripts -type d) -type f) >> "$objtree/debian/hdrsrcfiles"
-if grep -q '^CONFIG_STACK_VALIDATION=y' $KCONFIG_CONFIG ; then
+if is_enabled CONFIG_STACK_VALIDATION; then
 	(cd $objtree; find tools/objtool -type f -executable) >> "$objtree/debian/hdrobjfiles"
 fi
 (cd $objtree; find arch/$SRCARCH/include Module.symvers include scripts -type f) >> "$objtree/debian/hdrobjfiles"
-if grep -q '^CONFIG_GCC_PLUGINS=y' $KCONFIG_CONFIG ; then
+if is_enabled CONFIG_GCC_PLUGINS; then
 	(cd $objtree; find scripts/gcc-plugins -name \*.so -o -name gcc-common.h) >> "$objtree/debian/hdrobjfiles"
 fi
 destdir=$kernel_headers_dir/usr/src/linux-headers-$version
diff -Naur --no-dereference a/scripts/package/buildtar b/scripts/package/buildtar
--- a/scripts/package/buildtar	2020-01-09 04:19:10.000000000 -0500
+++ b/scripts/package/buildtar	2020-06-19 21:00:31.436227717 -0400
@@ -56,8 +56,8 @@
 #
 # Try to install modules
 #
-if grep -q '^CONFIG_MODULES=y' "${KCONFIG_CONFIG}"; then
-	make ARCH="${ARCH}" O="${objtree}" KBUILD_SRC= INSTALL_MOD_PATH="${tmpdir}" modules_install
+if grep -q '^CONFIG_MODULES=y' include/config/auto.conf; then
+	make ARCH="${ARCH}" -f ${srctree}/Makefile INSTALL_MOD_PATH="${tmpdir}" modules_install
 	dirs="$dirs lib"
 fi
 
diff -Naur --no-dereference a/scripts/package/Makefile b/scripts/package/Makefile
--- a/scripts/package/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/scripts/package/Makefile	2020-06-19 21:00:31.436227717 -0400
@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0-only
 # Makefile for the different targets used to generate full packages of a kernel
 # It uses the generic clean infrastructure of kbuild
 
@@ -33,7 +34,6 @@
 
 quiet_cmd_src_tar = TAR     $(2).tar.gz
       cmd_src_tar = \
-set -e; \
 if test "$(objtree)" != "$(srctree)"; then \
 	echo >&2; \
 	echo >&2 "  ERROR:"; \
@@ -60,7 +60,7 @@
 # binrpm-pkg
 # ---------------------------------------------------------------------------
 binrpm-pkg: FORCE
-	$(MAKE) KBUILD_SRC=
+	$(MAKE) -f $(srctree)/Makefile
 	$(CONFIG_SHELL) $(MKSPEC) prebuilt > $(objtree)/binkernel.spec
 	+rpmbuild $(RPMOPTS) --define "_builddir $(objtree)" --target \
 		$(UTS_MACHINE) -bb $(objtree)/binkernel.spec
@@ -73,11 +73,11 @@
 	$(call cmd,src_tar,$(KDEB_SOURCENAME))
 	origversion=$$(dpkg-parsechangelog -SVersion |sed 's/-[^-]*$$//');\
 		mv $(KDEB_SOURCENAME).tar.gz ../$(KDEB_SOURCENAME)_$${origversion}.orig.tar.gz
-	+dpkg-buildpackage -r$(KBUILD_PKG_ROOTCMD) -a$$(cat debian/arch) -i.git -us -uc
+	+dpkg-buildpackage -r$(KBUILD_PKG_ROOTCMD) -a$$(cat debian/arch) $(DPKG_FLAGS) -i.git -us -uc
 
 bindeb-pkg: FORCE
 	$(CONFIG_SHELL) $(srctree)/scripts/package/mkdebian
-	+dpkg-buildpackage -r$(KBUILD_PKG_ROOTCMD) -a$$(cat debian/arch) -b -nc -uc
+	+dpkg-buildpackage -r$(KBUILD_PKG_ROOTCMD) -a$$(cat debian/arch) $(DPKG_FLAGS) -b -nc -uc
 
 intdeb-pkg: FORCE
 	+$(CONFIG_SHELL) $(srctree)/scripts/package/builddeb
@@ -103,8 +103,8 @@
 # tarball targets
 # ---------------------------------------------------------------------------
 tar%pkg: FORCE
-	$(MAKE) KBUILD_SRC=
-	$(CONFIG_SHELL) $(srctree)/scripts/package/buildtar $@
+	$(MAKE) -f $(srctree)/Makefile
+	+$(CONFIG_SHELL) $(srctree)/scripts/package/buildtar $@
 
 clean-dirs += $(objtree)/tar-install/
 
diff -Naur --no-dereference a/scripts/package/mkdebian b/scripts/package/mkdebian
--- a/scripts/package/mkdebian	2020-01-09 04:19:10.000000000 -0500
+++ b/scripts/package/mkdebian	2020-06-19 21:00:31.436227717 -0400
@@ -7,7 +7,7 @@
 set -e
 
 is_enabled() {
-	grep -q "^CONFIG_$1=y" $KCONFIG_CONFIG
+	grep -q "^$1=y" include/config/auto.conf
 }
 
 if_enabled_echo() {
@@ -31,23 +31,23 @@
 	x86_64)
 		debarch=amd64 ;;
 	sparc*)
-		debarch=sparc$(if_enabled_echo 64BIT 64) ;;
+		debarch=sparc$(if_enabled_echo CONFIG_64BIT 64) ;;
 	s390*)
 		debarch=s390x ;;
 	ppc*)
-		if is_enabled 64BIT; then
-			debarch=ppc64$(if_enabled_echo CPU_LITTLE_ENDIAN el)
+		if is_enabled CONFIG_64BIT; then
+			debarch=ppc64$(if_enabled_echo CONFIG_CPU_LITTLE_ENDIAN el)
 		else
-			debarch=powerpc$(if_enabled_echo SPE spe)
+			debarch=powerpc$(if_enabled_echo CONFIG_SPE spe)
 		fi
 		;;
 	parisc*)
 		debarch=hppa ;;
 	mips*)
-		if is_enabled CPU_LITTLE_ENDIAN; then
-			debarch=mips$(if_enabled_echo 64BIT 64)$(if_enabled_echo CPU_MIPSR6 r6)el
-		elif is_enabled CPU_MIPSR6; then
-			debarch=mips$(if_enabled_echo 64BIT 64)r6
+		if is_enabled CONFIG_CPU_LITTLE_ENDIAN; then
+			debarch=mips$(if_enabled_echo CONFIG_64BIT 64)$(if_enabled_echo CONFIG_CPU_MIPSR6 r6)el
+		elif is_enabled CONFIG_CPU_MIPSR6; then
+			debarch=mips$(if_enabled_echo CONFIG_64BIT 64)r6
 		else
 			debarch=mips
 		fi
@@ -55,8 +55,8 @@
 	aarch64|arm64)
 		debarch=arm64 ;;
 	arm*)
-		if is_enabled AEABI; then
-			debarch=arm$(if_enabled_echo VFP hf el)
+		if is_enabled CONFIG_AEABI; then
+			debarch=arm$(if_enabled_echo CONFIG_VFP hf el)
 		else
 			debarch=arm
 		fi
@@ -64,10 +64,10 @@
 	openrisc)
 		debarch=or1k ;;
 	sh)
-		if is_enabled CPU_SH3; then
-			debarch=sh3$(if_enabled_echo CPU_BIG_ENDIAN eb)
-		elif is_enabled CPU_SH4; then
-			debarch=sh4$(if_enabled_echo CPU_BIG_ENDIAN eb)
+		if is_enabled CONFIG_CPU_SH3; then
+			debarch=sh3$(if_enabled_echo CONFIG_CPU_BIG_ENDIAN eb)
+		elif is_enabled CONFIG_CPU_SH4; then
+			debarch=sh4$(if_enabled_echo CONFIG_CPU_BIG_ENDIAN eb)
 		fi
 		;;
 	esac
@@ -132,8 +132,12 @@
         echo >&2 "Install lsb-release or set \$KDEB_CHANGELOG_DIST explicitly"
 fi
 
-mkdir -p debian/
+mkdir -p debian/source/
+echo "1.0" > debian/source/format
+
 echo $debarch > debian/arch
+extra_build_depends=", $(if_enabled_echo CONFIG_UNWINDER_ORC libelf-dev)"
+extra_build_depends="$extra_build_depends, $(if_enabled_echo CONFIG_SYSTEM_TRUSTED_KEYRING libssl-dev:native)"
 
 # Generate a simple changelog template
 cat <<EOF > debian/changelog
@@ -170,7 +174,7 @@
 Section: kernel
 Priority: optional
 Maintainer: $maintainer
-Build-Depends: bc, kmod, cpio
+Build-Depends: bc, kmod, cpio, bison, flex | flex:native $extra_build_depends
 Homepage: http://www.kernel.org/
 
 Package: $packagename
@@ -205,13 +209,15 @@
 cat <<EOF > debian/rules
 #!$(command -v $MAKE) -f
 
+srctree ?= .
+
 build:
 	\$(MAKE) KERNELRELEASE=${version} ARCH=${ARCH} \
-	KBUILD_BUILD_VERSION=${revision} KBUILD_SRC=
+	KBUILD_BUILD_VERSION=${revision} -f \$(srctree)/Makefile
 
 binary-arch:
 	\$(MAKE) KERNELRELEASE=${version} ARCH=${ARCH} \
-	KBUILD_BUILD_VERSION=${revision} KBUILD_SRC= intdeb-pkg
+	KBUILD_BUILD_VERSION=${revision} -f \$(srctree)/Makefile intdeb-pkg
 
 clean:
 	rm -rf debian/*tmp debian/files
@@ -219,5 +225,6 @@
 
 binary: binary-arch
 EOF
+chmod +x debian/rules
 
 exit 0
diff -Naur --no-dereference a/scripts/package/mkspec b/scripts/package/mkspec
--- a/scripts/package/mkspec	2020-01-09 04:19:10.000000000 -0500
+++ b/scripts/package/mkspec	2020-06-19 21:00:31.436227717 -0400
@@ -12,6 +12,7 @@
 # how we were called determines which rpms we build and how we build them
 if [ "$1" = prebuilt ]; then
 	S=DEL
+	MAKE="$MAKE -f $srctree/Makefile"
 else
 	S=
 fi
@@ -78,19 +79,19 @@
 $S	%setup -q
 $S
 $S	%build
-$S	make %{?_smp_mflags} KBUILD_BUILD_VERSION=%{release}
+$S	$MAKE %{?_smp_mflags} KBUILD_BUILD_VERSION=%{release}
 $S
 	%install
 	mkdir -p %{buildroot}/boot
 	%ifarch ia64
 	mkdir -p %{buildroot}/boot/efi
-	cp \$(make image_name) %{buildroot}/boot/efi/vmlinuz-$KERNELRELEASE
+	cp \$($MAKE image_name) %{buildroot}/boot/efi/vmlinuz-$KERNELRELEASE
 	ln -s efi/vmlinuz-$KERNELRELEASE %{buildroot}/boot/
 	%else
-	cp \$(make image_name) %{buildroot}/boot/vmlinuz-$KERNELRELEASE
+	cp \$($MAKE image_name) %{buildroot}/boot/vmlinuz-$KERNELRELEASE
 	%endif
-$M	make %{?_smp_mflags} INSTALL_MOD_PATH=%{buildroot} KBUILD_SRC= modules_install
-	make %{?_smp_mflags} INSTALL_HDR_PATH=%{buildroot}/usr KBUILD_SRC= headers_install
+$M	$MAKE %{?_smp_mflags} INSTALL_MOD_PATH=%{buildroot} modules_install
+	$MAKE %{?_smp_mflags} INSTALL_HDR_PATH=%{buildroot}/usr headers_install
 	cp System.map %{buildroot}/boot/System.map-$KERNELRELEASE
 	cp .config %{buildroot}/boot/config-$KERNELRELEASE
 	bzip2 -9 --keep vmlinux
diff -Naur --no-dereference a/security/device_cgroup.c b/security/device_cgroup.c
--- a/security/device_cgroup.c	2020-01-09 04:19:10.000000000 -0500
+++ b/security/device_cgroup.c	2020-06-19 21:00:31.436227717 -0400
@@ -824,3 +824,4 @@
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(__devcgroup_check_permission);
diff -Naur --no-dereference a/security/security.c b/security/security.c
--- a/security/security.c	2020-01-09 04:19:10.000000000 -0500
+++ b/security/security.c	2020-06-19 21:00:31.436227717 -0400
@@ -542,6 +542,7 @@
 		return 0;
 	return call_int_hook(path_rmdir, 0, dir, dentry);
 }
+EXPORT_SYMBOL_GPL(security_path_rmdir);
 
 int security_path_unlink(const struct path *dir, struct dentry *dentry)
 {
@@ -558,6 +559,7 @@
 		return 0;
 	return call_int_hook(path_symlink, 0, dir, dentry, old_name);
 }
+EXPORT_SYMBOL_GPL(security_path_symlink);
 
 int security_path_link(struct dentry *old_dentry, const struct path *new_dir,
 		       struct dentry *new_dentry)
@@ -566,6 +568,7 @@
 		return 0;
 	return call_int_hook(path_link, 0, old_dentry, new_dir, new_dentry);
 }
+EXPORT_SYMBOL_GPL(security_path_link);
 
 int security_path_rename(const struct path *old_dir, struct dentry *old_dentry,
 			 const struct path *new_dir, struct dentry *new_dentry,
@@ -593,6 +596,7 @@
 		return 0;
 	return call_int_hook(path_truncate, 0, path);
 }
+EXPORT_SYMBOL_GPL(security_path_truncate);
 
 int security_path_chmod(const struct path *path, umode_t mode)
 {
@@ -600,6 +604,7 @@
 		return 0;
 	return call_int_hook(path_chmod, 0, path, mode);
 }
+EXPORT_SYMBOL_GPL(security_path_chmod);
 
 int security_path_chown(const struct path *path, kuid_t uid, kgid_t gid)
 {
@@ -607,6 +612,7 @@
 		return 0;
 	return call_int_hook(path_chown, 0, path, uid, gid);
 }
+EXPORT_SYMBOL_GPL(security_path_chown);
 
 int security_path_chroot(const struct path *path)
 {
@@ -707,6 +713,7 @@
 		return 0;
 	return call_int_hook(inode_permission, 0, inode, mask);
 }
+EXPORT_SYMBOL_GPL(security_inode_permission);
 
 int security_inode_setattr(struct dentry *dentry, struct iattr *attr)
 {
@@ -878,6 +885,7 @@
 
 	return fsnotify_perm(file, mask);
 }
+EXPORT_SYMBOL_GPL(security_file_permission);
 
 int security_file_alloc(struct file *file)
 {
@@ -937,6 +945,7 @@
 		return ret;
 	return ima_file_mmap(file, prot);
 }
+EXPORT_SYMBOL_GPL(security_mmap_file);
 
 int security_mmap_addr(unsigned long addr)
 {
diff -Naur --no-dereference a/security/selinux/hooks.c b/security/selinux/hooks.c
--- a/security/selinux/hooks.c	2020-01-09 04:19:10.000000000 -0500
+++ b/security/selinux/hooks.c	2020-06-19 21:00:31.436227717 -0400
@@ -1496,7 +1496,9 @@
 			return SECCLASS_SMC_SOCKET;
 		case PF_XDP:
 			return SECCLASS_XDP_SOCKET;
-#if PF_MAX > 45
+		case PF_RPMSG:
+			return SECCLASS_RPMSG_SOCKET;
+#if PF_MAX > 46
 #error New address family defined, please update this function.
 #endif
 		}
diff -Naur --no-dereference a/security/selinux/include/classmap.h b/security/selinux/include/classmap.h
--- a/security/selinux/include/classmap.h	2020-01-09 04:19:10.000000000 -0500
+++ b/security/selinux/include/classmap.h	2020-06-19 21:00:31.436227717 -0400
@@ -235,6 +235,8 @@
 	  { COMMON_SOCK_PERMS, NULL } },
 	{ "smc_socket",
 	  { COMMON_SOCK_PERMS, NULL } },
+	{ "rpmsg_socket",
+	  { COMMON_SOCK_PERMS, NULL } },
 	{ "infiniband_pkey",
 	  { "access", NULL } },
 	{ "infiniband_endport",
@@ -246,6 +248,6 @@
 	{ NULL }
   };
 
-#if PF_MAX > 45
+#if PF_MAX > 46
 #error New address family defined, please update secclass_map.
 #endif
diff -Naur --no-dereference a/sound/core/init.c b/sound/core/init.c
--- a/sound/core/init.c	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/core/init.c	2020-06-19 21:00:31.436227717 -0400
@@ -58,6 +58,10 @@
 module_param_array(slots, charp, NULL, 0444);
 MODULE_PARM_DESC(slots, "Module names assigned to the slots.");
 
+static bool slots_reserved[SNDRV_CARDS];
+module_param_array(slots_reserved, bool, NULL, 0444);
+MODULE_PARM_DESC(slots_reserved, "Slots which are excluded from automatic assignment");
+
 /* return non-zero if the given index is reserved for the given
  * module via slots option
  */
@@ -142,7 +146,7 @@
 	for (slot = 0; slot < SNDRV_CARDS; slot++) {
 		if (slot < 32 && !(mask & (1U << slot)))
 			continue;
-		if (!test_bit(slot, snd_cards_lock)) {
+		if (!test_bit(slot, snd_cards_lock) && !slots_reserved[slot]) {
 			if (check(module, slot))
 				return slot; /* found */
 		}
diff -Naur --no-dereference a/sound/core/pcm_dmaengine.c b/sound/core/pcm_dmaengine.c
--- a/sound/core/pcm_dmaengine.c	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/core/pcm_dmaengine.c	2020-06-19 21:00:31.436227717 -0400
@@ -256,9 +256,16 @@
 
 	status = dmaengine_tx_status(prtd->dma_chan, prtd->cookie, &state);
 	if (status == DMA_IN_PROGRESS || status == DMA_PAUSED) {
+		struct snd_pcm_runtime *runtime = substream->runtime;
+		int sample_bytes = snd_pcm_format_physical_width(
+							runtime->format);
+
 		buf_size = snd_pcm_lib_buffer_bytes(substream);
 		if (state.residue > 0 && state.residue <= buf_size)
 			pos = buf_size - state.residue;
+
+		sample_bytes = (sample_bytes / 8) * runtime->channels;
+		runtime->delay = state.in_flight_bytes / sample_bytes;
 	}
 
 	return bytes_to_frames(substream->runtime, pos);
diff -Naur --no-dereference a/sound/soc/codecs/pcm3168a.c b/sound/soc/codecs/pcm3168a.c
--- a/sound/soc/codecs/pcm3168a.c	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/codecs/pcm3168a.c	2020-06-19 21:00:31.436227717 -0400
@@ -32,6 +32,8 @@
 #define PCM3168A_FMT_RIGHT_J_16		0x3
 #define PCM3168A_FMT_DSP_A		0x4
 #define PCM3168A_FMT_DSP_B		0x5
+#define PCM3168A_FMT_I2S_TDM		0x6
+#define PCM3168A_FMT_LEFT_J_TDM		0x7
 #define PCM3168A_FMT_DSP_MASK		0x4
 
 #define PCM3168A_NUM_SUPPLIES 6
@@ -44,15 +46,25 @@
 	"VCCDA2"
 };
 
+#define PCM3168A_DAI_DAC		0
+#define PCM3168A_DAI_ADC		1
+
+/* ADC/DAC side parameters */
+struct pcm3168a_io_params {
+	bool master_mode;
+	unsigned int fmt;
+	int tdm_slots;
+	u32 tdm_mask;
+	int slot_width;
+};
+
 struct pcm3168a_priv {
 	struct regulator_bulk_data supplies[PCM3168A_NUM_SUPPLIES];
 	struct regmap *regmap;
 	struct clk *scki;
-	bool adc_master_mode;
-	bool dac_master_mode;
 	unsigned long sysclk;
-	unsigned int adc_fmt;
-	unsigned int dac_fmt;
+
+	struct pcm3168a_io_params io_params[2];
 };
 
 static const char *const pcm3168a_roll_off[] = { "Sharp", "Slow" };
@@ -130,10 +142,6 @@
 	SOC_DOUBLE("DAC2 Invert Switch", PCM3168A_DAC_INV, 2, 3, 1, 0),
 	SOC_DOUBLE("DAC3 Invert Switch", PCM3168A_DAC_INV, 4, 5, 1, 0),
 	SOC_DOUBLE("DAC4 Invert Switch", PCM3168A_DAC_INV, 6, 7, 1, 0),
-	SOC_DOUBLE_STS("DAC1 Zero Flag", PCM3168A_DAC_ZERO, 0, 1, 1, 0),
-	SOC_DOUBLE_STS("DAC2 Zero Flag", PCM3168A_DAC_ZERO, 2, 3, 1, 0),
-	SOC_DOUBLE_STS("DAC3 Zero Flag", PCM3168A_DAC_ZERO, 4, 5, 1, 0),
-	SOC_DOUBLE_STS("DAC4 Zero Flag", PCM3168A_DAC_ZERO, 6, 7, 1, 0),
 	SOC_ENUM("DAC Volume Control Type", pcm3168a_dac_volume_type),
 	SOC_ENUM("DAC Volume Rate Multiplier", pcm3168a_dac_att_mult),
 	SOC_ENUM("DAC De-Emphasis", pcm3168a_dac_demp),
@@ -173,9 +181,6 @@
 	SOC_DOUBLE("ADC1 Mute Switch", PCM3168A_ADC_MUTE, 0, 1, 1, 0),
 	SOC_DOUBLE("ADC2 Mute Switch", PCM3168A_ADC_MUTE, 2, 3, 1, 0),
 	SOC_DOUBLE("ADC3 Mute Switch", PCM3168A_ADC_MUTE, 4, 5, 1, 0),
-	SOC_DOUBLE_STS("ADC1 Overflow Flag", PCM3168A_ADC_OV, 0, 1, 1, 0),
-	SOC_DOUBLE_STS("ADC2 Overflow Flag", PCM3168A_ADC_OV, 2, 3, 1, 0),
-	SOC_DOUBLE_STS("ADC3 Overflow Flag", PCM3168A_ADC_OV, 4, 5, 1, 0),
 	SOC_ENUM("ADC Volume Control Type", pcm3168a_adc_volume_type),
 	SOC_ENUM("ADC Volume Rate Multiplier", pcm3168a_adc_att_mult),
 	SOC_ENUM("ADC Overflow Flag Polarity", pcm3168a_adc_ov_pol),
@@ -312,8 +317,7 @@
 	return 0;
 }
 
-static int pcm3168a_set_dai_fmt(struct snd_soc_dai *dai,
-			       unsigned int format, bool dac)
+static int pcm3168a_set_dai_fmt(struct snd_soc_dai *dai, unsigned int format)
 {
 	struct snd_soc_component *component = dai->component;
 	struct pcm3168a_priv *pcm3168a = snd_soc_component_get_drvdata(component);
@@ -360,35 +364,55 @@
 		return -EINVAL;
 	}
 
-	if (dac) {
+	if (dai->id == PCM3168A_DAI_DAC) {
 		reg = PCM3168A_DAC_PWR_MST_FMT;
 		mask = PCM3168A_DAC_FMT_MASK;
 		shift = PCM3168A_DAC_FMT_SHIFT;
-		pcm3168a->dac_master_mode = master_mode;
-		pcm3168a->dac_fmt = fmt;
 	} else {
 		reg = PCM3168A_ADC_MST_FMT;
 		mask = PCM3168A_ADC_FMTAD_MASK;
 		shift = PCM3168A_ADC_FMTAD_SHIFT;
-		pcm3168a->adc_master_mode = master_mode;
-		pcm3168a->adc_fmt = fmt;
 	}
 
+	pcm3168a->io_params[dai->id].master_mode = master_mode;
+	pcm3168a->io_params[dai->id].fmt = fmt;
+
 	regmap_update_bits(pcm3168a->regmap, reg, mask, fmt << shift);
 
 	return 0;
 }
 
-static int pcm3168a_set_dai_fmt_dac(struct snd_soc_dai *dai,
-			       unsigned int format)
+static int pcm3168a_set_tdm_slot(struct snd_soc_dai *dai, unsigned int tx_mask,
+				 unsigned int rx_mask, int slots,
+				 int slot_width)
 {
-	return pcm3168a_set_dai_fmt(dai, format, true);
-}
+	struct snd_soc_component *component = dai->component;
+	struct pcm3168a_priv *pcm3168a = snd_soc_component_get_drvdata(component);
+	struct pcm3168a_io_params *io_params = &pcm3168a->io_params[dai->id];
 
-static int pcm3168a_set_dai_fmt_adc(struct snd_soc_dai *dai,
-			       unsigned int format)
-{
-	return pcm3168a_set_dai_fmt(dai, format, false);
+	if (tx_mask >= (1<<slots) || rx_mask >= (1<<slots)) {
+		dev_err(component->dev,
+			"Bad tdm mask tx: 0x%08x rx: 0x%08x slots %d\n",
+			tx_mask, rx_mask, slots);
+		return -EINVAL;
+	}
+
+	if (slot_width &&
+	    (slot_width != 16 && slot_width != 24 && slot_width != 32 )) {
+		dev_err(component->dev, "Unsupported slot_width %d\n",
+			slot_width);
+		return -EINVAL;
+	}
+
+	io_params->tdm_slots = slots;
+	io_params->slot_width = slot_width;
+	/* Ignore the not relevant mask for the DAI/direction */
+	if (dai->id == PCM3168A_DAI_DAC)
+		io_params->tdm_mask = tx_mask;
+	else
+		io_params->tdm_mask = rx_mask;
+
+	return 0;
 }
 
 static int pcm3168a_hw_params(struct snd_pcm_substream *substream,
@@ -397,32 +421,32 @@
 {
 	struct snd_soc_component *component = dai->component;
 	struct pcm3168a_priv *pcm3168a = snd_soc_component_get_drvdata(component);
-	bool tx, master_mode;
+	struct pcm3168a_io_params *io_params = &pcm3168a->io_params[dai->id];
+	bool master_mode;
 	u32 val, mask, shift, reg;
 	unsigned int rate, fmt, ratio, max_ratio;
-	int i, min_frame_size;
+	unsigned int tdm_slots;
+	int i, slot_width;
 
 	rate = params_rate(params);
 
 	ratio = pcm3168a->sysclk / rate;
 
-	tx = substream->stream == SNDRV_PCM_STREAM_PLAYBACK;
-	if (tx) {
+	if (dai->id == PCM3168A_DAI_DAC) {
 		max_ratio = PCM3168A_NUM_SCKI_RATIOS_DAC;
 		reg = PCM3168A_DAC_PWR_MST_FMT;
 		mask = PCM3168A_DAC_MSDA_MASK;
 		shift = PCM3168A_DAC_MSDA_SHIFT;
-		master_mode = pcm3168a->dac_master_mode;
-		fmt = pcm3168a->dac_fmt;
 	} else {
 		max_ratio = PCM3168A_NUM_SCKI_RATIOS_ADC;
 		reg = PCM3168A_ADC_MST_FMT;
 		mask = PCM3168A_ADC_MSAD_MASK;
 		shift = PCM3168A_ADC_MSAD_SHIFT;
-		master_mode = pcm3168a->adc_master_mode;
-		fmt = pcm3168a->adc_fmt;
 	}
 
+	master_mode = io_params->master_mode;
+	fmt = io_params->fmt;
+
 	for (i = 0; i < max_ratio; i++) {
 		if (pcm3168a_scki_ratios[i] == ratio)
 			break;
@@ -433,28 +457,62 @@
 		return -EINVAL;
 	}
 
-	min_frame_size = params_width(params) * 2;
-	switch (min_frame_size) {
-	case 32:
+	if (io_params->slot_width)
+		slot_width = io_params->slot_width;
+	else
+		slot_width = params_width(params);
+
+	switch (slot_width) {
+	case 16:
 		if (master_mode || (fmt != PCM3168A_FMT_RIGHT_J)) {
-			dev_err(component->dev, "32-bit frames are supported only for slave mode using right justified\n");
+			dev_err(component->dev, "16-bit slots are supported only for slave mode using right justified\n");
 			return -EINVAL;
 		}
 		fmt = PCM3168A_FMT_RIGHT_J_16;
 		break;
-	case 48:
+	case 24:
 		if (master_mode || (fmt & PCM3168A_FMT_DSP_MASK)) {
-			dev_err(component->dev, "48-bit frames not supported in master mode, or slave mode using DSP\n");
+			dev_err(component->dev, "24-bit slots not supported in master mode, or slave mode using DSP\n");
 			return -EINVAL;
 		}
 		break;
-	case 64:
+	case 32:
 		break;
 	default:
-		dev_err(component->dev, "unsupported frame size: %d\n", min_frame_size);
+		dev_err(component->dev, "unsupported frame size: %d\n", slot_width);
 		return -EINVAL;
 	}
 
+	if (io_params->tdm_slots)
+		tdm_slots = io_params->tdm_slots;
+	else
+		tdm_slots = params_channels(params);
+
+	/*
+	 * Switch the codec to TDM mode when more than 2 TDM slots are needed
+	 * for the stream.
+	 * If pcm3168a->tdm_slots is not set or set to more than 2 (8/6 usually)
+	 * then DIN1/DOUT1 is used in TDM mode.
+	 * If pcm3168a->tdm_slots is set to 2 then DIN1/2/3/4 and DOUT1/2/3 is
+	 * used in normal mode, no need to switch to TDM modes.
+	 */
+	if (tdm_slots > 2) {
+		switch (fmt) {
+		case PCM3168A_FMT_I2S:
+		case PCM3168A_FMT_DSP_A:
+			fmt = PCM3168A_FMT_I2S_TDM;
+			break;
+		case PCM3168A_FMT_LEFT_J:
+		case PCM3168A_FMT_DSP_B:
+			fmt = PCM3168A_FMT_LEFT_J_TDM;
+			break;
+		default:
+			dev_err(component->dev,
+				"TDM is supported under DSP/I2S/Left_J only\n");
+			return -EINVAL;
+		}
+	}
+
 	if (master_mode)
 		val = ((i + 1) << shift);
 	else
@@ -462,7 +520,7 @@
 
 	regmap_update_bits(pcm3168a->regmap, reg, mask, val);
 
-	if (tx) {
+	if (dai->id == PCM3168A_DAI_DAC) {
 		mask = PCM3168A_DAC_FMT_MASK;
 		shift = PCM3168A_DAC_FMT_SHIFT;
 	} else {
@@ -475,22 +533,74 @@
 	return 0;
 }
 
-static const struct snd_soc_dai_ops pcm3168a_dac_dai_ops = {
-	.set_fmt	= pcm3168a_set_dai_fmt_dac,
-	.set_sysclk	= pcm3168a_set_dai_sysclk,
-	.hw_params	= pcm3168a_hw_params,
-	.digital_mute	= pcm3168a_digital_mute
-};
+static int pcm3168a_startup(struct snd_pcm_substream *substream,
+			    struct snd_soc_dai *dai)
+{
+	struct snd_soc_component *component = dai->component;
+	struct pcm3168a_priv *pcm3168a = snd_soc_component_get_drvdata(component);
+	unsigned int sample_min;
+	unsigned int channel_max;
+	unsigned int channel_maxs[] = {
+		8, /* DAC */
+		6  /* ADC */
+	};
+
+	/*
+	 * Available Data Bits
+	 *
+	 * RIGHT_J : 24 / 16
+	 * LEFT_J  : 24
+	 * I2S     : 24
+	 *
+	 * TDM available
+	 *
+	 * I2S
+	 * LEFT_J
+	 */
+	switch (pcm3168a->io_params[dai->id].fmt) {
+	case PCM3168A_FMT_RIGHT_J:
+		sample_min  = 16;
+		channel_max =  2;
+		break;
+	case PCM3168A_FMT_LEFT_J:
+	case PCM3168A_FMT_I2S:
+	case PCM3168A_FMT_DSP_A:
+	case PCM3168A_FMT_DSP_B:
+		sample_min  = 24;
+		channel_max = channel_maxs[dai->id];
+		break;
+	default:
+		sample_min  = 24;
+		channel_max =  2;
+	}
+
+	snd_pcm_hw_constraint_minmax(substream->runtime,
+				     SNDRV_PCM_HW_PARAM_SAMPLE_BITS,
+				     sample_min, 32);
+
+	/* Allow all channels in multi DIN/DOUT mode */
+	if (pcm3168a->io_params[dai->id].tdm_slots == 2)
+		channel_max = channel_maxs[dai->id];
+
+	snd_pcm_hw_constraint_minmax(substream->runtime,
+				     SNDRV_PCM_HW_PARAM_CHANNELS,
+				     2, channel_max);
 
-static const struct snd_soc_dai_ops pcm3168a_adc_dai_ops = {
-	.set_fmt	= pcm3168a_set_dai_fmt_adc,
+	return 0;
+}
+static const struct snd_soc_dai_ops pcm3168a_dai_ops = {
+	.startup	= pcm3168a_startup,
+	.set_fmt	= pcm3168a_set_dai_fmt,
 	.set_sysclk	= pcm3168a_set_dai_sysclk,
-	.hw_params	= pcm3168a_hw_params
+	.hw_params	= pcm3168a_hw_params,
+	.digital_mute	= pcm3168a_digital_mute,
+	.set_tdm_slot	= pcm3168a_set_tdm_slot,
 };
 
 static struct snd_soc_dai_driver pcm3168a_dais[] = {
 	{
 		.name = "pcm3168a-dac",
+		.id = PCM3168A_DAI_DAC,
 		.playback = {
 			.stream_name = "Playback",
 			.channels_min = 1,
@@ -498,10 +608,11 @@
 			.rates = SNDRV_PCM_RATE_8000_192000,
 			.formats = PCM3168A_FORMATS
 		},
-		.ops = &pcm3168a_dac_dai_ops
+		.ops = &pcm3168a_dai_ops
 	},
 	{
 		.name = "pcm3168a-adc",
+		.id = PCM3168A_DAI_ADC,
 		.capture = {
 			.stream_name = "Capture",
 			.channels_min = 1,
@@ -509,7 +620,7 @@
 			.rates = SNDRV_PCM_RATE_8000_96000,
 			.formats = PCM3168A_FORMATS
 		},
-		.ops = &pcm3168a_adc_dai_ops
+		.ops = &pcm3168a_dai_ops
 	},
 };
 
diff -Naur --no-dereference a/sound/soc/codecs/tlv320aic31xx.c b/sound/soc/codecs/tlv320aic31xx.c
--- a/sound/soc/codecs/tlv320aic31xx.c	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/codecs/tlv320aic31xx.c	2020-06-19 21:00:31.436227717 -0400
@@ -167,6 +167,7 @@
 	u8 p_div;
 	int rate_div_line;
 	bool master_dapm_route_applied;
+	int irq;
 };
 
 struct aic31xx_rate_divs {
@@ -1391,6 +1392,69 @@
 MODULE_DEVICE_TABLE(acpi, aic31xx_acpi_match);
 #endif
 
+static irqreturn_t aic31xx_irq(int irq, void *data)
+{
+	struct aic31xx_priv *aic31xx = data;
+	struct device *dev = aic31xx->dev;
+	unsigned int value;
+	bool handled = false;
+	int ret;
+
+	ret = regmap_read(aic31xx->regmap, AIC31XX_INTRDACFLAG, &value);
+	if (ret) {
+		dev_err(dev, "Failed to read interrupt mask: %d\n", ret);
+		goto exit;
+	}
+
+	if (value)
+		handled = true;
+	else
+		goto read_overflow;
+
+	if (value & AIC31XX_HPLSCDETECT)
+		dev_err(dev, "Short circuit on Left output is detected\n");
+	if (value & AIC31XX_HPRSCDETECT)
+		dev_err(dev, "Short circuit on Right output is detected\n");
+	if (value & ~(AIC31XX_HPLSCDETECT |
+		      AIC31XX_HPRSCDETECT))
+		dev_err(dev, "Unknown DAC interrupt flags: 0x%08x\n", value);
+
+read_overflow:
+	ret = regmap_read(aic31xx->regmap, AIC31XX_OFFLAG, &value);
+	if (ret) {
+		dev_err(dev, "Failed to read overflow flag: %d\n", ret);
+		goto exit;
+	}
+
+	if (value)
+		handled = true;
+	else
+		goto exit;
+
+	if (value & AIC31XX_DAC_OF_LEFT)
+		dev_warn(dev, "Left-channel DAC overflow has occurred\n");
+	if (value & AIC31XX_DAC_OF_RIGHT)
+		dev_warn(dev, "Right-channel DAC overflow has occurred\n");
+	if (value & AIC31XX_DAC_OF_SHIFTER)
+		dev_warn(dev, "DAC barrel shifter overflow has occurred\n");
+	if (value & AIC31XX_ADC_OF)
+		dev_warn(dev, "ADC overflow has occurred\n");
+	if (value & AIC31XX_ADC_OF_SHIFTER)
+		dev_warn(dev, "ADC barrel shifter overflow has occurred\n");
+	if (value & ~(AIC31XX_DAC_OF_LEFT |
+		      AIC31XX_DAC_OF_RIGHT |
+		      AIC31XX_DAC_OF_SHIFTER |
+		      AIC31XX_ADC_OF |
+		      AIC31XX_ADC_OF_SHIFTER))
+		dev_warn(dev, "Unknown overflow interrupt flags: 0x%08x\n", value);
+
+exit:
+	if (handled)
+		return IRQ_HANDLED;
+	else
+		return IRQ_NONE;
+}
+
 static int aic31xx_i2c_probe(struct i2c_client *i2c,
 			     const struct i2c_device_id *id)
 {
@@ -1413,6 +1477,7 @@
 		return ret;
 	}
 	aic31xx->dev = &i2c->dev;
+	aic31xx->irq = i2c->irq;
 
 	aic31xx->codec_type = id->driver_data;
 
@@ -1459,6 +1524,26 @@
 		return ret;
 	}
 
+	if (aic31xx->irq > 0) {
+		regmap_update_bits(aic31xx->regmap, AIC31XX_GPIO1,
+				   AIC31XX_GPIO1_FUNC_MASK,
+				   AIC31XX_GPIO1_INT1 <<
+				   AIC31XX_GPIO1_FUNC_SHIFT);
+
+		regmap_write(aic31xx->regmap, AIC31XX_INT1CTRL,
+			     AIC31XX_SC |
+			     AIC31XX_ENGINE);
+
+		ret = devm_request_threaded_irq(aic31xx->dev, aic31xx->irq,
+						NULL, aic31xx_irq,
+						IRQF_ONESHOT, "aic31xx-irq",
+						aic31xx);
+		if (ret) {
+			dev_err(aic31xx->dev, "Unable to request IRQ\n");
+			return ret;
+		}
+	}
+
 	if (aic31xx->codec_type & DAC31XX_BIT)
 		return devm_snd_soc_register_component(&i2c->dev,
 				&soc_codec_driver_aic31xx,
diff -Naur --no-dereference a/sound/soc/codecs/tlv320aic31xx.h b/sound/soc/codecs/tlv320aic31xx.h
--- a/sound/soc/codecs/tlv320aic31xx.h	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/codecs/tlv320aic31xx.h	2020-06-19 21:00:31.436227717 -0400
@@ -173,6 +173,13 @@
 #define AIC31XX_HPRDRVPWRSTATUS_MASK	BIT(1)
 #define AIC31XX_SPRDRVPWRSTATUS_MASK	BIT(0)
 
+/* AIC31XX_OFFLAG */
+#define AIC31XX_DAC_OF_LEFT		BIT(7)
+#define AIC31XX_DAC_OF_RIGHT		BIT(6)
+#define AIC31XX_DAC_OF_SHIFTER		BIT(5)
+#define AIC31XX_ADC_OF			BIT(3)
+#define AIC31XX_ADC_OF_SHIFTER		BIT(1)
+
 /* AIC31XX_INTRDACFLAG */
 #define AIC31XX_HPLSCDETECT		BIT(7)
 #define AIC31XX_HPRSCDETECT		BIT(6)
@@ -191,6 +198,22 @@
 #define AIC31XX_SC			BIT(3)
 #define AIC31XX_ENGINE			BIT(2)
 
+/* AIC31XX_GPIO1 */
+#define AIC31XX_GPIO1_FUNC_MASK		GENMASK(5, 2)
+#define AIC31XX_GPIO1_FUNC_SHIFT	2
+#define AIC31XX_GPIO1_DISABLED		0x00
+#define AIC31XX_GPIO1_INPUT		0x01
+#define AIC31XX_GPIO1_GPI		0x02
+#define AIC31XX_GPIO1_GPO		0x03
+#define AIC31XX_GPIO1_CLKOUT		0x04
+#define AIC31XX_GPIO1_INT1		0x05
+#define AIC31XX_GPIO1_INT2		0x06
+#define AIC31XX_GPIO1_ADC_WCLK		0x07
+#define AIC31XX_GPIO1_SBCLK		0x08
+#define AIC31XX_GPIO1_SWCLK		0x09
+#define AIC31XX_GPIO1_ADC_MOD_CLK	0x10
+#define AIC31XX_GPIO1_SDOUT		0x11
+
 /* AIC31XX_DACSETUP */
 #define AIC31XX_SOFTSTEP_MASK		GENMASK(1, 0)
 
diff -Naur --no-dereference a/sound/soc/codecs/tlv320aic3x.c b/sound/soc/codecs/tlv320aic3x.c
--- a/sound/soc/codecs/tlv320aic3x.c	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/codecs/tlv320aic3x.c	2020-06-19 21:00:31.436227717 -0400
@@ -307,6 +307,13 @@
 static SOC_ENUM_SINGLE_DECL(aic3x_rampup_step_enum, HPOUT_POP_REDUCTION, 2,
 			    aic3x_rampup_step);
 
+static const char * const aic3x_output_pdown_mode[] = {
+	"Drive to a common-mode", "High-impedance mode"};
+static SOC_ENUM_SINGLE_DECL(aic3x_hpl_pdown_mode_enum, HPLOUT_CTRL, 2,
+			    aic3x_output_pdown_mode);
+static SOC_ENUM_SINGLE_DECL(aic3x_hpr_pdown_mode_enum, HPROUT_CTRL, 2,
+			    aic3x_output_pdown_mode);
+
 /*
  * DAC digital volumes. From -63.5 to 0 dB in 0.5 dB steps
  */
@@ -394,6 +401,9 @@
 	SOC_DOUBLE_R("HPCOM Playback Switch", HPLCOM_CTRL, HPRCOM_CTRL, 3,
 		     0x01, 0),
 
+	SOC_ENUM("Left HP Power Down mode", aic3x_hpl_pdown_mode_enum),
+	SOC_ENUM("Right HP Power Down mode", aic3x_hpr_pdown_mode_enum),
+
 	/*
 	 * Note: enable Automatic input Gain Controller with care. It can
 	 * adjust PGA to max value when ADC is on and will never go back.
@@ -1260,6 +1270,16 @@
 		aic3x->master = 0;
 		iface_areg &= ~(BIT_CLK_MASTER | WORD_CLK_MASTER);
 		break;
+	case SND_SOC_DAIFMT_CBM_CFS:
+		aic3x->master = 1;
+		iface_areg |= BIT_CLK_MASTER;
+		iface_areg &= ~WORD_CLK_MASTER;
+		break;
+	case SND_SOC_DAIFMT_CBS_CFM:
+		aic3x->master = 1;
+		iface_areg |= WORD_CLK_MASTER;
+		iface_areg &= ~BIT_CLK_MASTER;
+		break;
 	default:
 		return -EINVAL;
 	}
diff -Naur --no-dereference a/sound/soc/davinci/davinci-evm.c b/sound/soc/davinci/davinci-evm.c
--- a/sound/soc/davinci/davinci-evm.c	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/davinci/davinci-evm.c	1969-12-31 19:00:00.000000000 -0500
@@ -1,511 +0,0 @@
-/*
- * ASoC driver for TI DAVINCI EVM platform
- *
- * Author:      Vladimir Barinov, <vbarinov@embeddedalley.com>
- * Copyright:   (C) 2007 MontaVista Software, Inc., <source@mvista.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#include <linux/module.h>
-#include <linux/moduleparam.h>
-#include <linux/timer.h>
-#include <linux/interrupt.h>
-#include <linux/platform_device.h>
-#include <linux/i2c.h>
-#include <linux/of_platform.h>
-#include <linux/clk.h>
-#include <sound/core.h>
-#include <sound/pcm.h>
-#include <sound/soc.h>
-
-#include <asm/dma.h>
-#include <asm/mach-types.h>
-
-struct snd_soc_card_drvdata_davinci {
-	struct clk *mclk;
-	unsigned sysclk;
-};
-
-static int evm_startup(struct snd_pcm_substream *substream)
-{
-	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct snd_soc_card *soc_card = rtd->card;
-	struct snd_soc_card_drvdata_davinci *drvdata =
-		snd_soc_card_get_drvdata(soc_card);
-
-	if (drvdata->mclk)
-		return clk_prepare_enable(drvdata->mclk);
-
-	return 0;
-}
-
-static void evm_shutdown(struct snd_pcm_substream *substream)
-{
-	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct snd_soc_card *soc_card = rtd->card;
-	struct snd_soc_card_drvdata_davinci *drvdata =
-		snd_soc_card_get_drvdata(soc_card);
-
-	if (drvdata->mclk)
-		clk_disable_unprepare(drvdata->mclk);
-}
-
-static int evm_hw_params(struct snd_pcm_substream *substream,
-			 struct snd_pcm_hw_params *params)
-{
-	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct snd_soc_dai *codec_dai = rtd->codec_dai;
-	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
-	struct snd_soc_card *soc_card = rtd->card;
-	int ret = 0;
-	unsigned sysclk = ((struct snd_soc_card_drvdata_davinci *)
-			   snd_soc_card_get_drvdata(soc_card))->sysclk;
-
-	/* set the codec system clock */
-	ret = snd_soc_dai_set_sysclk(codec_dai, 0, sysclk, SND_SOC_CLOCK_OUT);
-	if (ret < 0)
-		return ret;
-
-	/* set the CPU system clock */
-	ret = snd_soc_dai_set_sysclk(cpu_dai, 0, sysclk, SND_SOC_CLOCK_OUT);
-	if (ret < 0)
-		return ret;
-
-	return 0;
-}
-
-static struct snd_soc_ops evm_ops = {
-	.startup = evm_startup,
-	.shutdown = evm_shutdown,
-	.hw_params = evm_hw_params,
-};
-
-/* davinci-evm machine dapm widgets */
-static const struct snd_soc_dapm_widget aic3x_dapm_widgets[] = {
-	SND_SOC_DAPM_HP("Headphone Jack", NULL),
-	SND_SOC_DAPM_LINE("Line Out", NULL),
-	SND_SOC_DAPM_MIC("Mic Jack", NULL),
-	SND_SOC_DAPM_LINE("Line In", NULL),
-};
-
-/* davinci-evm machine audio_mapnections to the codec pins */
-static const struct snd_soc_dapm_route audio_map[] = {
-	/* Headphone connected to HPLOUT, HPROUT */
-	{"Headphone Jack", NULL, "HPLOUT"},
-	{"Headphone Jack", NULL, "HPROUT"},
-
-	/* Line Out connected to LLOUT, RLOUT */
-	{"Line Out", NULL, "LLOUT"},
-	{"Line Out", NULL, "RLOUT"},
-
-	/* Mic connected to (MIC3L | MIC3R) */
-	{"MIC3L", NULL, "Mic Bias"},
-	{"MIC3R", NULL, "Mic Bias"},
-	{"Mic Bias", NULL, "Mic Jack"},
-
-	/* Line In connected to (LINE1L | LINE2L), (LINE1R | LINE2R) */
-	{"LINE1L", NULL, "Line In"},
-	{"LINE2L", NULL, "Line In"},
-	{"LINE1R", NULL, "Line In"},
-	{"LINE2R", NULL, "Line In"},
-};
-
-/* Logic for a aic3x as connected on a davinci-evm */
-static int evm_aic3x_init(struct snd_soc_pcm_runtime *rtd)
-{
-	struct snd_soc_card *card = rtd->card;
-	struct device_node *np = card->dev->of_node;
-	int ret;
-
-	/* Add davinci-evm specific widgets */
-	snd_soc_dapm_new_controls(&card->dapm, aic3x_dapm_widgets,
-				  ARRAY_SIZE(aic3x_dapm_widgets));
-
-	if (np) {
-		ret = snd_soc_of_parse_audio_routing(card, "ti,audio-routing");
-		if (ret)
-			return ret;
-	} else {
-		/* Set up davinci-evm specific audio path audio_map */
-		snd_soc_dapm_add_routes(&card->dapm, audio_map,
-					ARRAY_SIZE(audio_map));
-	}
-
-	/* not connected */
-	snd_soc_dapm_nc_pin(&card->dapm, "MONO_LOUT");
-	snd_soc_dapm_nc_pin(&card->dapm, "HPLCOM");
-	snd_soc_dapm_nc_pin(&card->dapm, "HPRCOM");
-
-	return 0;
-}
-
-/* davinci-evm digital audio interface glue - connects codec <--> CPU */
-static struct snd_soc_dai_link dm6446_evm_dai = {
-	.name = "TLV320AIC3X",
-	.stream_name = "AIC3X",
-	.cpu_dai_name = "davinci-mcbsp",
-	.codec_dai_name = "tlv320aic3x-hifi",
-	.codec_name = "tlv320aic3x-codec.1-001b",
-	.platform_name = "davinci-mcbsp",
-	.init = evm_aic3x_init,
-	.ops = &evm_ops,
-	.dai_fmt = SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_CBM_CFM |
-		   SND_SOC_DAIFMT_IB_NF,
-};
-
-static struct snd_soc_dai_link dm355_evm_dai = {
-	.name = "TLV320AIC3X",
-	.stream_name = "AIC3X",
-	.cpu_dai_name = "davinci-mcbsp.1",
-	.codec_dai_name = "tlv320aic3x-hifi",
-	.codec_name = "tlv320aic3x-codec.1-001b",
-	.platform_name = "davinci-mcbsp.1",
-	.init = evm_aic3x_init,
-	.ops = &evm_ops,
-	.dai_fmt = SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_CBM_CFM |
-		   SND_SOC_DAIFMT_IB_NF,
-};
-
-static struct snd_soc_dai_link dm365_evm_dai = {
-#ifdef CONFIG_SND_DM365_AIC3X_CODEC
-	.name = "TLV320AIC3X",
-	.stream_name = "AIC3X",
-	.cpu_dai_name = "davinci-mcbsp",
-	.codec_dai_name = "tlv320aic3x-hifi",
-	.codec_name = "tlv320aic3x-codec.1-0018",
-	.platform_name = "davinci-mcbsp",
-	.init = evm_aic3x_init,
-	.ops = &evm_ops,
-	.dai_fmt = SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_CBM_CFM |
-		   SND_SOC_DAIFMT_IB_NF,
-#elif defined(CONFIG_SND_DM365_VOICE_CODEC)
-	.name = "Voice Codec - CQ93VC",
-	.stream_name = "CQ93",
-	.cpu_dai_name = "davinci-vcif",
-	.codec_dai_name = "cq93vc-hifi",
-	.codec_name = "cq93vc-codec",
-	.platform_name = "davinci-vcif",
-#endif
-};
-
-static struct snd_soc_dai_link dm6467_evm_dai[] = {
-	{
-		.name = "TLV320AIC3X",
-		.stream_name = "AIC3X",
-		.cpu_dai_name= "davinci-mcasp.0",
-		.codec_dai_name = "tlv320aic3x-hifi",
-		.platform_name = "davinci-mcasp.0",
-		.codec_name = "tlv320aic3x-codec.0-001a",
-		.init = evm_aic3x_init,
-		.ops = &evm_ops,
-		.dai_fmt = SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_CBM_CFM |
-			   SND_SOC_DAIFMT_IB_NF,
-	},
-	{
-		.name = "McASP",
-		.stream_name = "spdif",
-		.cpu_dai_name= "davinci-mcasp.1",
-		.codec_dai_name = "dit-hifi",
-		.codec_name = "spdif_dit",
-		.platform_name = "davinci-mcasp.1",
-		.dai_fmt = SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_CBM_CFM |
-			   SND_SOC_DAIFMT_IB_NF,
-	},
-};
-
-static struct snd_soc_dai_link da830_evm_dai = {
-	.name = "TLV320AIC3X",
-	.stream_name = "AIC3X",
-	.cpu_dai_name = "davinci-mcasp.1",
-	.codec_dai_name = "tlv320aic3x-hifi",
-	.codec_name = "tlv320aic3x-codec.1-0018",
-	.platform_name = "davinci-mcasp.1",
-	.init = evm_aic3x_init,
-	.ops = &evm_ops,
-	.dai_fmt = SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_CBM_CFM |
-		   SND_SOC_DAIFMT_IB_NF,
-};
-
-static struct snd_soc_dai_link da850_evm_dai = {
-	.name = "TLV320AIC3X",
-	.stream_name = "AIC3X",
-	.cpu_dai_name= "davinci-mcasp.0",
-	.codec_dai_name = "tlv320aic3x-hifi",
-	.codec_name = "tlv320aic3x-codec.1-0018",
-	.platform_name = "davinci-mcasp.0",
-	.init = evm_aic3x_init,
-	.ops = &evm_ops,
-	.dai_fmt = SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_CBM_CFM |
-		   SND_SOC_DAIFMT_IB_NF,
-};
-
-/* davinci dm6446 evm audio machine driver */
-/*
- * ASP0 in DM6446 EVM is clocked by U55, as configured by
- * board-dm644x-evm.c using GPIOs from U18.  There are six
- * options; here we "know" we use a 48 KHz sample rate.
- */
-static struct snd_soc_card_drvdata_davinci dm6446_snd_soc_card_drvdata = {
-	.sysclk = 12288000,
-};
-
-static struct snd_soc_card dm6446_snd_soc_card_evm = {
-	.name = "DaVinci DM6446 EVM",
-	.owner = THIS_MODULE,
-	.dai_link = &dm6446_evm_dai,
-	.num_links = 1,
-	.drvdata = &dm6446_snd_soc_card_drvdata,
-};
-
-/* davinci dm355 evm audio machine driver */
-/* ASP1 on DM355 EVM is clocked by an external oscillator */
-static struct snd_soc_card_drvdata_davinci dm355_snd_soc_card_drvdata = {
-	.sysclk = 27000000,
-};
-
-static struct snd_soc_card dm355_snd_soc_card_evm = {
-	.name = "DaVinci DM355 EVM",
-	.owner = THIS_MODULE,
-	.dai_link = &dm355_evm_dai,
-	.num_links = 1,
-	.drvdata = &dm355_snd_soc_card_drvdata,
-};
-
-/* davinci dm365 evm audio machine driver */
-static struct snd_soc_card_drvdata_davinci dm365_snd_soc_card_drvdata = {
-	.sysclk = 27000000,
-};
-
-static struct snd_soc_card dm365_snd_soc_card_evm = {
-	.name = "DaVinci DM365 EVM",
-	.owner = THIS_MODULE,
-	.dai_link = &dm365_evm_dai,
-	.num_links = 1,
-	.drvdata = &dm365_snd_soc_card_drvdata,
-};
-
-/* davinci dm6467 evm audio machine driver */
-static struct snd_soc_card_drvdata_davinci dm6467_snd_soc_card_drvdata = {
-	.sysclk = 27000000,
-};
-
-static struct snd_soc_card dm6467_snd_soc_card_evm = {
-	.name = "DaVinci DM6467 EVM",
-	.owner = THIS_MODULE,
-	.dai_link = dm6467_evm_dai,
-	.num_links = ARRAY_SIZE(dm6467_evm_dai),
-	.drvdata = &dm6467_snd_soc_card_drvdata,
-};
-
-static struct snd_soc_card_drvdata_davinci da830_snd_soc_card_drvdata = {
-	.sysclk = 24576000,
-};
-
-static struct snd_soc_card da830_snd_soc_card = {
-	.name = "DA830/OMAP-L137 EVM",
-	.owner = THIS_MODULE,
-	.dai_link = &da830_evm_dai,
-	.num_links = 1,
-	.drvdata = &da830_snd_soc_card_drvdata,
-};
-
-static struct snd_soc_card_drvdata_davinci da850_snd_soc_card_drvdata = {
-	.sysclk = 24576000,
-};
-
-static struct snd_soc_card da850_snd_soc_card = {
-	.name = "DA850/OMAP-L138 EVM",
-	.owner = THIS_MODULE,
-	.dai_link = &da850_evm_dai,
-	.num_links = 1,
-	.drvdata = &da850_snd_soc_card_drvdata,
-};
-
-#if defined(CONFIG_OF)
-
-/*
- * The struct is used as place holder. It will be completely
- * filled with data from dt node.
- */
-static struct snd_soc_dai_link evm_dai_tlv320aic3x = {
-	.name		= "TLV320AIC3X",
-	.stream_name	= "AIC3X",
-	.codec_dai_name	= "tlv320aic3x-hifi",
-	.ops            = &evm_ops,
-	.init           = evm_aic3x_init,
-	.dai_fmt = SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_CBM_CFM |
-		   SND_SOC_DAIFMT_IB_NF,
-};
-
-static const struct of_device_id davinci_evm_dt_ids[] = {
-	{
-		.compatible = "ti,da830-evm-audio",
-		.data = (void *) &evm_dai_tlv320aic3x,
-	},
-	{ /* sentinel */ }
-};
-MODULE_DEVICE_TABLE(of, davinci_evm_dt_ids);
-
-/* davinci evm audio machine driver */
-static struct snd_soc_card evm_soc_card = {
-	.owner = THIS_MODULE,
-	.num_links = 1,
-};
-
-static int davinci_evm_probe(struct platform_device *pdev)
-{
-	struct device_node *np = pdev->dev.of_node;
-	const struct of_device_id *match;
-	struct snd_soc_dai_link *dai;
-	struct snd_soc_card_drvdata_davinci *drvdata = NULL;
-	struct clk *mclk;
-	int ret = 0;
-
-	match = of_match_device(of_match_ptr(davinci_evm_dt_ids), &pdev->dev);
-	if (!match) {
-		dev_err(&pdev->dev, "Error: No device match found\n");
-		return -ENODEV;
-	}
-
-	dai = (struct snd_soc_dai_link *) match->data;
-
-	evm_soc_card.dai_link = dai;
-
-	dai->codec_of_node = of_parse_phandle(np, "ti,audio-codec", 0);
-	if (!dai->codec_of_node)
-		return -EINVAL;
-
-	dai->cpu_of_node = of_parse_phandle(np, "ti,mcasp-controller", 0);
-	if (!dai->cpu_of_node)
-		return -EINVAL;
-
-	dai->platform_of_node = dai->cpu_of_node;
-
-	evm_soc_card.dev = &pdev->dev;
-	ret = snd_soc_of_parse_card_name(&evm_soc_card, "ti,model");
-	if (ret)
-		return ret;
-
-	mclk = devm_clk_get(&pdev->dev, "mclk");
-	if (PTR_ERR(mclk) == -EPROBE_DEFER) {
-		return -EPROBE_DEFER;
-	} else if (IS_ERR(mclk)) {
-		dev_dbg(&pdev->dev, "mclk not found.\n");
-		mclk = NULL;
-	}
-
-	drvdata = devm_kzalloc(&pdev->dev, sizeof(*drvdata), GFP_KERNEL);
-	if (!drvdata)
-		return -ENOMEM;
-
-	drvdata->mclk = mclk;
-
-	ret = of_property_read_u32(np, "ti,codec-clock-rate", &drvdata->sysclk);
-
-	if (ret < 0) {
-		if (!drvdata->mclk) {
-			dev_err(&pdev->dev,
-				"No clock or clock rate defined.\n");
-			return -EINVAL;
-		}
-		drvdata->sysclk = clk_get_rate(drvdata->mclk);
-	} else if (drvdata->mclk) {
-		unsigned int requestd_rate = drvdata->sysclk;
-		clk_set_rate(drvdata->mclk, drvdata->sysclk);
-		drvdata->sysclk = clk_get_rate(drvdata->mclk);
-		if (drvdata->sysclk != requestd_rate)
-			dev_warn(&pdev->dev,
-				 "Could not get requested rate %u using %u.\n",
-				 requestd_rate, drvdata->sysclk);
-	}
-
-	snd_soc_card_set_drvdata(&evm_soc_card, drvdata);
-	ret = devm_snd_soc_register_card(&pdev->dev, &evm_soc_card);
-
-	if (ret)
-		dev_err(&pdev->dev, "snd_soc_register_card failed (%d)\n", ret);
-
-	return ret;
-}
-
-static struct platform_driver davinci_evm_driver = {
-	.probe		= davinci_evm_probe,
-	.driver		= {
-		.name	= "davinci_evm",
-		.pm	= &snd_soc_pm_ops,
-		.of_match_table = of_match_ptr(davinci_evm_dt_ids),
-	},
-};
-#endif
-
-static struct platform_device *evm_snd_device;
-
-static int __init evm_init(void)
-{
-	struct snd_soc_card *evm_snd_dev_data;
-	int index;
-	int ret;
-
-	/*
-	 * If dtb is there, the devices will be created dynamically.
-	 * Only register platfrom driver structure.
-	 */
-#if defined(CONFIG_OF)
-	if (of_have_populated_dt())
-		return platform_driver_register(&davinci_evm_driver);
-#endif
-
-	if (machine_is_davinci_evm()) {
-		evm_snd_dev_data = &dm6446_snd_soc_card_evm;
-		index = 0;
-	} else if (machine_is_davinci_dm355_evm()) {
-		evm_snd_dev_data = &dm355_snd_soc_card_evm;
-		index = 1;
-	} else if (machine_is_davinci_dm365_evm()) {
-		evm_snd_dev_data = &dm365_snd_soc_card_evm;
-		index = 0;
-	} else if (machine_is_davinci_dm6467_evm()) {
-		evm_snd_dev_data = &dm6467_snd_soc_card_evm;
-		index = 0;
-	} else if (machine_is_davinci_da830_evm()) {
-		evm_snd_dev_data = &da830_snd_soc_card;
-		index = 1;
-	} else if (machine_is_davinci_da850_evm()) {
-		evm_snd_dev_data = &da850_snd_soc_card;
-		index = 0;
-	} else
-		return -EINVAL;
-
-	evm_snd_device = platform_device_alloc("soc-audio", index);
-	if (!evm_snd_device)
-		return -ENOMEM;
-
-	platform_set_drvdata(evm_snd_device, evm_snd_dev_data);
-	ret = platform_device_add(evm_snd_device);
-	if (ret)
-		platform_device_put(evm_snd_device);
-
-	return ret;
-}
-
-static void __exit evm_exit(void)
-{
-#if defined(CONFIG_OF)
-	if (of_have_populated_dt()) {
-		platform_driver_unregister(&davinci_evm_driver);
-		return;
-	}
-#endif
-
-	platform_device_unregister(evm_snd_device);
-}
-
-module_init(evm_init);
-module_exit(evm_exit);
-
-MODULE_AUTHOR("Vladimir Barinov");
-MODULE_DESCRIPTION("TI DAVINCI EVM ASoC driver");
-MODULE_LICENSE("GPL");
diff -Naur --no-dereference a/sound/soc/davinci/davinci-i2s.c b/sound/soc/davinci/davinci-i2s.c
--- a/sound/soc/davinci/davinci-i2s.c	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/davinci/davinci-i2s.c	1969-12-31 19:00:00.000000000 -0500
@@ -1,782 +0,0 @@
-/*
- * ALSA SoC I2S (McBSP) Audio Layer for TI DAVINCI processor
- *
- * Author:      Vladimir Barinov, <vbarinov@embeddedalley.com>
- * Copyright:   (C) 2007 MontaVista Software, Inc., <source@mvista.com>
- *
- * DT support	(c) 2016 Petr Kulhavy, Barix AG <petr@barix.com>
- *		based on davinci-mcasp.c DT support
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * TODO:
- * on DA850 implement HW FIFOs instead of DMA into DXR and DRR registers
- */
-
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/device.h>
-#include <linux/slab.h>
-#include <linux/delay.h>
-#include <linux/io.h>
-#include <linux/clk.h>
-#include <linux/platform_data/davinci_asp.h>
-
-#include <sound/core.h>
-#include <sound/pcm.h>
-#include <sound/pcm_params.h>
-#include <sound/initval.h>
-#include <sound/soc.h>
-#include <sound/dmaengine_pcm.h>
-
-#include "edma-pcm.h"
-#include "davinci-i2s.h"
-
-#define DRV_NAME "davinci-i2s"
-
-/*
- * NOTE:  terminology here is confusing.
- *
- *  - This driver supports the "Audio Serial Port" (ASP),
- *    found on dm6446, dm355, and other DaVinci chips.
- *
- *  - But it labels it a "Multi-channel Buffered Serial Port"
- *    (McBSP) as on older chips like the dm642 ... which was
- *    backward-compatible, possibly explaining that confusion.
- *
- *  - OMAP chips have a controller called McBSP, which is
- *    incompatible with the DaVinci flavor of McBSP.
- *
- *  - Newer DaVinci chips have a controller called McASP,
- *    incompatible with ASP and with either McBSP.
- *
- * In short:  this uses ASP to implement I2S, not McBSP.
- * And it won't be the only DaVinci implemention of I2S.
- */
-#define DAVINCI_MCBSP_DRR_REG	0x00
-#define DAVINCI_MCBSP_DXR_REG	0x04
-#define DAVINCI_MCBSP_SPCR_REG	0x08
-#define DAVINCI_MCBSP_RCR_REG	0x0c
-#define DAVINCI_MCBSP_XCR_REG	0x10
-#define DAVINCI_MCBSP_SRGR_REG	0x14
-#define DAVINCI_MCBSP_PCR_REG	0x24
-
-#define DAVINCI_MCBSP_SPCR_RRST		(1 << 0)
-#define DAVINCI_MCBSP_SPCR_RINTM(v)	((v) << 4)
-#define DAVINCI_MCBSP_SPCR_XRST		(1 << 16)
-#define DAVINCI_MCBSP_SPCR_XINTM(v)	((v) << 20)
-#define DAVINCI_MCBSP_SPCR_GRST		(1 << 22)
-#define DAVINCI_MCBSP_SPCR_FRST		(1 << 23)
-#define DAVINCI_MCBSP_SPCR_FREE		(1 << 25)
-
-#define DAVINCI_MCBSP_RCR_RWDLEN1(v)	((v) << 5)
-#define DAVINCI_MCBSP_RCR_RFRLEN1(v)	((v) << 8)
-#define DAVINCI_MCBSP_RCR_RDATDLY(v)	((v) << 16)
-#define DAVINCI_MCBSP_RCR_RFIG		(1 << 18)
-#define DAVINCI_MCBSP_RCR_RWDLEN2(v)	((v) << 21)
-#define DAVINCI_MCBSP_RCR_RFRLEN2(v)	((v) << 24)
-#define DAVINCI_MCBSP_RCR_RPHASE	BIT(31)
-
-#define DAVINCI_MCBSP_XCR_XWDLEN1(v)	((v) << 5)
-#define DAVINCI_MCBSP_XCR_XFRLEN1(v)	((v) << 8)
-#define DAVINCI_MCBSP_XCR_XDATDLY(v)	((v) << 16)
-#define DAVINCI_MCBSP_XCR_XFIG		(1 << 18)
-#define DAVINCI_MCBSP_XCR_XWDLEN2(v)	((v) << 21)
-#define DAVINCI_MCBSP_XCR_XFRLEN2(v)	((v) << 24)
-#define DAVINCI_MCBSP_XCR_XPHASE	BIT(31)
-
-#define DAVINCI_MCBSP_SRGR_FWID(v)	((v) << 8)
-#define DAVINCI_MCBSP_SRGR_FPER(v)	((v) << 16)
-#define DAVINCI_MCBSP_SRGR_FSGM		(1 << 28)
-#define DAVINCI_MCBSP_SRGR_CLKSM	BIT(29)
-
-#define DAVINCI_MCBSP_PCR_CLKRP		(1 << 0)
-#define DAVINCI_MCBSP_PCR_CLKXP		(1 << 1)
-#define DAVINCI_MCBSP_PCR_FSRP		(1 << 2)
-#define DAVINCI_MCBSP_PCR_FSXP		(1 << 3)
-#define DAVINCI_MCBSP_PCR_SCLKME	(1 << 7)
-#define DAVINCI_MCBSP_PCR_CLKRM		(1 << 8)
-#define DAVINCI_MCBSP_PCR_CLKXM		(1 << 9)
-#define DAVINCI_MCBSP_PCR_FSRM		(1 << 10)
-#define DAVINCI_MCBSP_PCR_FSXM		(1 << 11)
-
-enum {
-	DAVINCI_MCBSP_WORD_8 = 0,
-	DAVINCI_MCBSP_WORD_12,
-	DAVINCI_MCBSP_WORD_16,
-	DAVINCI_MCBSP_WORD_20,
-	DAVINCI_MCBSP_WORD_24,
-	DAVINCI_MCBSP_WORD_32,
-};
-
-static const unsigned char data_type[SNDRV_PCM_FORMAT_S32_LE + 1] = {
-	[SNDRV_PCM_FORMAT_S8]		= 1,
-	[SNDRV_PCM_FORMAT_S16_LE]	= 2,
-	[SNDRV_PCM_FORMAT_S32_LE]	= 4,
-};
-
-static const unsigned char asp_word_length[SNDRV_PCM_FORMAT_S32_LE + 1] = {
-	[SNDRV_PCM_FORMAT_S8]		= DAVINCI_MCBSP_WORD_8,
-	[SNDRV_PCM_FORMAT_S16_LE]	= DAVINCI_MCBSP_WORD_16,
-	[SNDRV_PCM_FORMAT_S32_LE]	= DAVINCI_MCBSP_WORD_32,
-};
-
-static const unsigned char double_fmt[SNDRV_PCM_FORMAT_S32_LE + 1] = {
-	[SNDRV_PCM_FORMAT_S8]		= SNDRV_PCM_FORMAT_S16_LE,
-	[SNDRV_PCM_FORMAT_S16_LE]	= SNDRV_PCM_FORMAT_S32_LE,
-};
-
-struct davinci_mcbsp_dev {
-	struct device *dev;
-	struct snd_dmaengine_dai_dma_data dma_data[2];
-	int dma_request[2];
-	void __iomem			*base;
-#define MOD_DSP_A	0
-#define MOD_DSP_B	1
-	int				mode;
-	u32				pcr;
-	struct clk			*clk;
-	/*
-	 * Combining both channels into 1 element will at least double the
-	 * amount of time between servicing the dma channel, increase
-	 * effiency, and reduce the chance of overrun/underrun. But,
-	 * it will result in the left & right channels being swapped.
-	 *
-	 * If relabeling the left and right channels is not possible,
-	 * you may want to let the codec know to swap them back.
-	 *
-	 * It may allow x10 the amount of time to service dma requests,
-	 * if the codec is master and is using an unnecessarily fast bit clock
-	 * (ie. tlvaic23b), independent of the sample rate. So, having an
-	 * entire frame at once means it can be serviced at the sample rate
-	 * instead of the bit clock rate.
-	 *
-	 * In the now unlikely case that an underrun still
-	 * occurs, both the left and right samples will be repeated
-	 * so that no pops are heard, and the left and right channels
-	 * won't end up being swapped because of the underrun.
-	 */
-	unsigned enable_channel_combine:1;
-
-	unsigned int fmt;
-	int clk_div;
-	int clk_input_pin;
-	bool i2s_accurate_sck;
-};
-
-static inline void davinci_mcbsp_write_reg(struct davinci_mcbsp_dev *dev,
-					   int reg, u32 val)
-{
-	__raw_writel(val, dev->base + reg);
-}
-
-static inline u32 davinci_mcbsp_read_reg(struct davinci_mcbsp_dev *dev, int reg)
-{
-	return __raw_readl(dev->base + reg);
-}
-
-static void toggle_clock(struct davinci_mcbsp_dev *dev, int playback)
-{
-	u32 m = playback ? DAVINCI_MCBSP_PCR_CLKXP : DAVINCI_MCBSP_PCR_CLKRP;
-	/* The clock needs to toggle to complete reset.
-	 * So, fake it by toggling the clk polarity.
-	 */
-	davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_PCR_REG, dev->pcr ^ m);
-	davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_PCR_REG, dev->pcr);
-}
-
-static void davinci_mcbsp_start(struct davinci_mcbsp_dev *dev,
-		struct snd_pcm_substream *substream)
-{
-	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct snd_soc_component *component = snd_soc_rtdcom_lookup(rtd, DRV_NAME);
-	int playback = (substream->stream == SNDRV_PCM_STREAM_PLAYBACK);
-	u32 spcr;
-	u32 mask = playback ? DAVINCI_MCBSP_SPCR_XRST : DAVINCI_MCBSP_SPCR_RRST;
-	spcr = davinci_mcbsp_read_reg(dev, DAVINCI_MCBSP_SPCR_REG);
-	if (spcr & mask) {
-		/* start off disabled */
-		davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_SPCR_REG,
-				spcr & ~mask);
-		toggle_clock(dev, playback);
-	}
-	if (dev->pcr & (DAVINCI_MCBSP_PCR_FSXM | DAVINCI_MCBSP_PCR_FSRM |
-			DAVINCI_MCBSP_PCR_CLKXM | DAVINCI_MCBSP_PCR_CLKRM)) {
-		/* Start the sample generator */
-		spcr |= DAVINCI_MCBSP_SPCR_GRST;
-		davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_SPCR_REG, spcr);
-	}
-
-	if (playback) {
-		/* Stop the DMA to avoid data loss */
-		/* while the transmitter is out of reset to handle XSYNCERR */
-		if (component->driver->ops->trigger) {
-			int ret = component->driver->ops->trigger(substream,
-				SNDRV_PCM_TRIGGER_STOP);
-			if (ret < 0)
-				printk(KERN_DEBUG "Playback DMA stop failed\n");
-		}
-
-		/* Enable the transmitter */
-		spcr = davinci_mcbsp_read_reg(dev, DAVINCI_MCBSP_SPCR_REG);
-		spcr |= DAVINCI_MCBSP_SPCR_XRST;
-		davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_SPCR_REG, spcr);
-
-		/* wait for any unexpected frame sync error to occur */
-		udelay(100);
-
-		/* Disable the transmitter to clear any outstanding XSYNCERR */
-		spcr = davinci_mcbsp_read_reg(dev, DAVINCI_MCBSP_SPCR_REG);
-		spcr &= ~DAVINCI_MCBSP_SPCR_XRST;
-		davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_SPCR_REG, spcr);
-		toggle_clock(dev, playback);
-
-		/* Restart the DMA */
-		if (component->driver->ops->trigger) {
-			int ret = component->driver->ops->trigger(substream,
-				SNDRV_PCM_TRIGGER_START);
-			if (ret < 0)
-				printk(KERN_DEBUG "Playback DMA start failed\n");
-		}
-	}
-
-	/* Enable transmitter or receiver */
-	spcr = davinci_mcbsp_read_reg(dev, DAVINCI_MCBSP_SPCR_REG);
-	spcr |= mask;
-
-	if (dev->pcr & (DAVINCI_MCBSP_PCR_FSXM | DAVINCI_MCBSP_PCR_FSRM)) {
-		/* Start frame sync */
-		spcr |= DAVINCI_MCBSP_SPCR_FRST;
-	}
-	davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_SPCR_REG, spcr);
-}
-
-static void davinci_mcbsp_stop(struct davinci_mcbsp_dev *dev, int playback)
-{
-	u32 spcr;
-
-	/* Reset transmitter/receiver and sample rate/frame sync generators */
-	spcr = davinci_mcbsp_read_reg(dev, DAVINCI_MCBSP_SPCR_REG);
-	spcr &= ~(DAVINCI_MCBSP_SPCR_GRST | DAVINCI_MCBSP_SPCR_FRST);
-	spcr &= playback ? ~DAVINCI_MCBSP_SPCR_XRST : ~DAVINCI_MCBSP_SPCR_RRST;
-	davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_SPCR_REG, spcr);
-	toggle_clock(dev, playback);
-}
-
-#define DEFAULT_BITPERSAMPLE	16
-
-static int davinci_i2s_set_dai_fmt(struct snd_soc_dai *cpu_dai,
-				   unsigned int fmt)
-{
-	struct davinci_mcbsp_dev *dev = snd_soc_dai_get_drvdata(cpu_dai);
-	unsigned int pcr;
-	unsigned int srgr;
-	bool inv_fs = false;
-	/* Attention srgr is updated by hw_params! */
-	srgr = DAVINCI_MCBSP_SRGR_FSGM |
-		DAVINCI_MCBSP_SRGR_FPER(DEFAULT_BITPERSAMPLE * 2 - 1) |
-		DAVINCI_MCBSP_SRGR_FWID(DEFAULT_BITPERSAMPLE - 1);
-
-	dev->fmt = fmt;
-	/* set master/slave audio interface */
-	switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) {
-	case SND_SOC_DAIFMT_CBS_CFS:
-		/* cpu is master */
-		pcr = DAVINCI_MCBSP_PCR_FSXM |
-			DAVINCI_MCBSP_PCR_FSRM |
-			DAVINCI_MCBSP_PCR_CLKXM |
-			DAVINCI_MCBSP_PCR_CLKRM;
-		break;
-	case SND_SOC_DAIFMT_CBM_CFS:
-		pcr = DAVINCI_MCBSP_PCR_FSRM | DAVINCI_MCBSP_PCR_FSXM;
-		/*
-		 * Selection of the clock input pin that is the
-		 * input for the Sample Rate Generator.
-		 * McBSP FSR and FSX are driven by the Sample Rate
-		 * Generator.
-		 */
-		switch (dev->clk_input_pin) {
-		case MCBSP_CLKS:
-			pcr |= DAVINCI_MCBSP_PCR_CLKXM |
-				DAVINCI_MCBSP_PCR_CLKRM;
-			break;
-		case MCBSP_CLKR:
-			pcr |= DAVINCI_MCBSP_PCR_SCLKME;
-			break;
-		default:
-			dev_err(dev->dev, "bad clk_input_pin\n");
-			return -EINVAL;
-		}
-
-		break;
-	case SND_SOC_DAIFMT_CBM_CFM:
-		/* codec is master */
-		pcr = 0;
-		break;
-	default:
-		printk(KERN_ERR "%s:bad master\n", __func__);
-		return -EINVAL;
-	}
-
-	/* interface format */
-	switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) {
-	case SND_SOC_DAIFMT_I2S:
-		/* Davinci doesn't support TRUE I2S, but some codecs will have
-		 * the left and right channels contiguous. This allows
-		 * dsp_a mode to be used with an inverted normal frame clk.
-		 * If your codec is master and does not have contiguous
-		 * channels, then you will have sound on only one channel.
-		 * Try using a different mode, or codec as slave.
-		 *
-		 * The TLV320AIC33 is an example of a codec where this works.
-		 * It has a variable bit clock frequency allowing it to have
-		 * valid data on every bit clock.
-		 *
-		 * The TLV320AIC23 is an example of a codec where this does not
-		 * work. It has a fixed bit clock frequency with progressively
-		 * more empty bit clock slots between channels as the sample
-		 * rate is lowered.
-		 */
-		inv_fs = true;
-		/* fall through */
-	case SND_SOC_DAIFMT_DSP_A:
-		dev->mode = MOD_DSP_A;
-		break;
-	case SND_SOC_DAIFMT_DSP_B:
-		dev->mode = MOD_DSP_B;
-		break;
-	default:
-		printk(KERN_ERR "%s:bad format\n", __func__);
-		return -EINVAL;
-	}
-
-	switch (fmt & SND_SOC_DAIFMT_INV_MASK) {
-	case SND_SOC_DAIFMT_NB_NF:
-		/* CLKRP Receive clock polarity,
-		 *	1 - sampled on rising edge of CLKR
-		 *	valid on rising edge
-		 * CLKXP Transmit clock polarity,
-		 *	1 - clocked on falling edge of CLKX
-		 *	valid on rising edge
-		 * FSRP  Receive frame sync pol, 0 - active high
-		 * FSXP  Transmit frame sync pol, 0 - active high
-		 */
-		pcr |= (DAVINCI_MCBSP_PCR_CLKXP | DAVINCI_MCBSP_PCR_CLKRP);
-		break;
-	case SND_SOC_DAIFMT_IB_IF:
-		/* CLKRP Receive clock polarity,
-		 *	0 - sampled on falling edge of CLKR
-		 *	valid on falling edge
-		 * CLKXP Transmit clock polarity,
-		 *	0 - clocked on rising edge of CLKX
-		 *	valid on falling edge
-		 * FSRP  Receive frame sync pol, 1 - active low
-		 * FSXP  Transmit frame sync pol, 1 - active low
-		 */
-		pcr |= (DAVINCI_MCBSP_PCR_FSXP | DAVINCI_MCBSP_PCR_FSRP);
-		break;
-	case SND_SOC_DAIFMT_NB_IF:
-		/* CLKRP Receive clock polarity,
-		 *	1 - sampled on rising edge of CLKR
-		 *	valid on rising edge
-		 * CLKXP Transmit clock polarity,
-		 *	1 - clocked on falling edge of CLKX
-		 *	valid on rising edge
-		 * FSRP  Receive frame sync pol, 1 - active low
-		 * FSXP  Transmit frame sync pol, 1 - active low
-		 */
-		pcr |= (DAVINCI_MCBSP_PCR_CLKXP | DAVINCI_MCBSP_PCR_CLKRP |
-			DAVINCI_MCBSP_PCR_FSXP | DAVINCI_MCBSP_PCR_FSRP);
-		break;
-	case SND_SOC_DAIFMT_IB_NF:
-		/* CLKRP Receive clock polarity,
-		 *	0 - sampled on falling edge of CLKR
-		 *	valid on falling edge
-		 * CLKXP Transmit clock polarity,
-		 *	0 - clocked on rising edge of CLKX
-		 *	valid on falling edge
-		 * FSRP  Receive frame sync pol, 0 - active high
-		 * FSXP  Transmit frame sync pol, 0 - active high
-		 */
-		break;
-	default:
-		return -EINVAL;
-	}
-	if (inv_fs == true)
-		pcr ^= (DAVINCI_MCBSP_PCR_FSXP | DAVINCI_MCBSP_PCR_FSRP);
-	davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_SRGR_REG, srgr);
-	dev->pcr = pcr;
-	davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_PCR_REG, pcr);
-	return 0;
-}
-
-static int davinci_i2s_dai_set_clkdiv(struct snd_soc_dai *cpu_dai,
-				int div_id, int div)
-{
-	struct davinci_mcbsp_dev *dev = snd_soc_dai_get_drvdata(cpu_dai);
-
-	if (div_id != DAVINCI_MCBSP_CLKGDV)
-		return -ENODEV;
-
-	dev->clk_div = div;
-	return 0;
-}
-
-static int davinci_i2s_hw_params(struct snd_pcm_substream *substream,
-				 struct snd_pcm_hw_params *params,
-				 struct snd_soc_dai *dai)
-{
-	struct davinci_mcbsp_dev *dev = snd_soc_dai_get_drvdata(dai);
-	struct snd_interval *i = NULL;
-	int mcbsp_word_length, master;
-	unsigned int rcr, xcr, srgr, clk_div, freq, framesize;
-	u32 spcr;
-	snd_pcm_format_t fmt;
-	unsigned element_cnt = 1;
-
-	/* general line settings */
-	spcr = davinci_mcbsp_read_reg(dev, DAVINCI_MCBSP_SPCR_REG);
-	if (substream->stream == SNDRV_PCM_STREAM_CAPTURE) {
-		spcr |= DAVINCI_MCBSP_SPCR_RINTM(3) | DAVINCI_MCBSP_SPCR_FREE;
-		davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_SPCR_REG, spcr);
-	} else {
-		spcr |= DAVINCI_MCBSP_SPCR_XINTM(3) | DAVINCI_MCBSP_SPCR_FREE;
-		davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_SPCR_REG, spcr);
-	}
-
-	master = dev->fmt & SND_SOC_DAIFMT_MASTER_MASK;
-	fmt = params_format(params);
-	mcbsp_word_length = asp_word_length[fmt];
-
-	switch (master) {
-	case SND_SOC_DAIFMT_CBS_CFS:
-		freq = clk_get_rate(dev->clk);
-		srgr = DAVINCI_MCBSP_SRGR_FSGM |
-		       DAVINCI_MCBSP_SRGR_CLKSM;
-		srgr |= DAVINCI_MCBSP_SRGR_FWID(mcbsp_word_length *
-						8 - 1);
-		if (dev->i2s_accurate_sck) {
-			clk_div = 256;
-			do {
-				framesize = (freq / (--clk_div)) /
-				params->rate_num *
-					params->rate_den;
-			} while (((framesize < 33) || (framesize > 4095)) &&
-				 (clk_div));
-			clk_div--;
-			srgr |= DAVINCI_MCBSP_SRGR_FPER(framesize - 1);
-		} else {
-			/* symmetric waveforms */
-			clk_div = freq / (mcbsp_word_length * 16) /
-				  params->rate_num * params->rate_den;
-			srgr |= DAVINCI_MCBSP_SRGR_FPER(mcbsp_word_length *
-							16 - 1);
-		}
-		clk_div &= 0xFF;
-		srgr |= clk_div;
-		break;
-	case SND_SOC_DAIFMT_CBM_CFS:
-		srgr = DAVINCI_MCBSP_SRGR_FSGM;
-		clk_div = dev->clk_div - 1;
-		srgr |= DAVINCI_MCBSP_SRGR_FWID(mcbsp_word_length * 8 - 1);
-		srgr |= DAVINCI_MCBSP_SRGR_FPER(mcbsp_word_length * 16 - 1);
-		clk_div &= 0xFF;
-		srgr |= clk_div;
-		break;
-	case SND_SOC_DAIFMT_CBM_CFM:
-		/* Clock and frame sync given from external sources */
-		i = hw_param_interval(params, SNDRV_PCM_HW_PARAM_SAMPLE_BITS);
-		srgr = DAVINCI_MCBSP_SRGR_FSGM;
-		srgr |= DAVINCI_MCBSP_SRGR_FWID(snd_interval_value(i) - 1);
-		pr_debug("%s - %d  FWID set: re-read srgr = %X\n",
-			__func__, __LINE__, snd_interval_value(i) - 1);
-
-		i = hw_param_interval(params, SNDRV_PCM_HW_PARAM_FRAME_BITS);
-		srgr |= DAVINCI_MCBSP_SRGR_FPER(snd_interval_value(i) - 1);
-		break;
-	default:
-		return -EINVAL;
-	}
-	davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_SRGR_REG, srgr);
-
-	rcr = DAVINCI_MCBSP_RCR_RFIG;
-	xcr = DAVINCI_MCBSP_XCR_XFIG;
-	if (dev->mode == MOD_DSP_B) {
-		rcr |= DAVINCI_MCBSP_RCR_RDATDLY(0);
-		xcr |= DAVINCI_MCBSP_XCR_XDATDLY(0);
-	} else {
-		rcr |= DAVINCI_MCBSP_RCR_RDATDLY(1);
-		xcr |= DAVINCI_MCBSP_XCR_XDATDLY(1);
-	}
-	/* Determine xfer data type */
-	fmt = params_format(params);
-	if ((fmt > SNDRV_PCM_FORMAT_S32_LE) || !data_type[fmt]) {
-		printk(KERN_WARNING "davinci-i2s: unsupported PCM format\n");
-		return -EINVAL;
-	}
-
-	if (params_channels(params) == 2) {
-		element_cnt = 2;
-		if (double_fmt[fmt] && dev->enable_channel_combine) {
-			element_cnt = 1;
-			fmt = double_fmt[fmt];
-		}
-		switch (master) {
-		case SND_SOC_DAIFMT_CBS_CFS:
-		case SND_SOC_DAIFMT_CBS_CFM:
-			rcr |= DAVINCI_MCBSP_RCR_RFRLEN2(0);
-			xcr |= DAVINCI_MCBSP_XCR_XFRLEN2(0);
-			rcr |= DAVINCI_MCBSP_RCR_RPHASE;
-			xcr |= DAVINCI_MCBSP_XCR_XPHASE;
-			break;
-		case SND_SOC_DAIFMT_CBM_CFM:
-		case SND_SOC_DAIFMT_CBM_CFS:
-			rcr |= DAVINCI_MCBSP_RCR_RFRLEN2(element_cnt - 1);
-			xcr |= DAVINCI_MCBSP_XCR_XFRLEN2(element_cnt - 1);
-			break;
-		default:
-			return -EINVAL;
-		}
-	}
-	mcbsp_word_length = asp_word_length[fmt];
-
-	switch (master) {
-	case SND_SOC_DAIFMT_CBS_CFS:
-	case SND_SOC_DAIFMT_CBS_CFM:
-		rcr |= DAVINCI_MCBSP_RCR_RFRLEN1(0);
-		xcr |= DAVINCI_MCBSP_XCR_XFRLEN1(0);
-		break;
-	case SND_SOC_DAIFMT_CBM_CFM:
-	case SND_SOC_DAIFMT_CBM_CFS:
-		rcr |= DAVINCI_MCBSP_RCR_RFRLEN1(element_cnt - 1);
-		xcr |= DAVINCI_MCBSP_XCR_XFRLEN1(element_cnt - 1);
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	rcr |= DAVINCI_MCBSP_RCR_RWDLEN1(mcbsp_word_length) |
-		DAVINCI_MCBSP_RCR_RWDLEN2(mcbsp_word_length);
-	xcr |= DAVINCI_MCBSP_XCR_XWDLEN1(mcbsp_word_length) |
-		DAVINCI_MCBSP_XCR_XWDLEN2(mcbsp_word_length);
-
-	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
-		davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_XCR_REG, xcr);
-	else
-		davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_RCR_REG, rcr);
-
-	pr_debug("%s - %d  srgr=%X\n", __func__, __LINE__, srgr);
-	pr_debug("%s - %d  xcr=%X\n", __func__, __LINE__, xcr);
-	pr_debug("%s - %d  rcr=%X\n", __func__, __LINE__, rcr);
-	return 0;
-}
-
-static int davinci_i2s_prepare(struct snd_pcm_substream *substream,
-		struct snd_soc_dai *dai)
-{
-	struct davinci_mcbsp_dev *dev = snd_soc_dai_get_drvdata(dai);
-	int playback = (substream->stream == SNDRV_PCM_STREAM_PLAYBACK);
-	davinci_mcbsp_stop(dev, playback);
-	return 0;
-}
-
-static int davinci_i2s_trigger(struct snd_pcm_substream *substream, int cmd,
-			       struct snd_soc_dai *dai)
-{
-	struct davinci_mcbsp_dev *dev = snd_soc_dai_get_drvdata(dai);
-	int ret = 0;
-	int playback = (substream->stream == SNDRV_PCM_STREAM_PLAYBACK);
-
-	switch (cmd) {
-	case SNDRV_PCM_TRIGGER_START:
-	case SNDRV_PCM_TRIGGER_RESUME:
-	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
-		davinci_mcbsp_start(dev, substream);
-		break;
-	case SNDRV_PCM_TRIGGER_STOP:
-	case SNDRV_PCM_TRIGGER_SUSPEND:
-	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
-		davinci_mcbsp_stop(dev, playback);
-		break;
-	default:
-		ret = -EINVAL;
-	}
-	return ret;
-}
-
-static void davinci_i2s_shutdown(struct snd_pcm_substream *substream,
-		struct snd_soc_dai *dai)
-{
-	struct davinci_mcbsp_dev *dev = snd_soc_dai_get_drvdata(dai);
-	int playback = (substream->stream == SNDRV_PCM_STREAM_PLAYBACK);
-	davinci_mcbsp_stop(dev, playback);
-}
-
-#define DAVINCI_I2S_RATES	SNDRV_PCM_RATE_8000_96000
-
-static const struct snd_soc_dai_ops davinci_i2s_dai_ops = {
-	.shutdown	= davinci_i2s_shutdown,
-	.prepare	= davinci_i2s_prepare,
-	.trigger	= davinci_i2s_trigger,
-	.hw_params	= davinci_i2s_hw_params,
-	.set_fmt	= davinci_i2s_set_dai_fmt,
-	.set_clkdiv	= davinci_i2s_dai_set_clkdiv,
-
-};
-
-static int davinci_i2s_dai_probe(struct snd_soc_dai *dai)
-{
-	struct davinci_mcbsp_dev *dev = snd_soc_dai_get_drvdata(dai);
-
-	dai->playback_dma_data = &dev->dma_data[SNDRV_PCM_STREAM_PLAYBACK];
-	dai->capture_dma_data = &dev->dma_data[SNDRV_PCM_STREAM_CAPTURE];
-
-	return 0;
-}
-
-static struct snd_soc_dai_driver davinci_i2s_dai = {
-	.probe = davinci_i2s_dai_probe,
-	.playback = {
-		.channels_min = 2,
-		.channels_max = 2,
-		.rates = DAVINCI_I2S_RATES,
-		.formats = SNDRV_PCM_FMTBIT_S16_LE,},
-	.capture = {
-		.channels_min = 2,
-		.channels_max = 2,
-		.rates = DAVINCI_I2S_RATES,
-		.formats = SNDRV_PCM_FMTBIT_S16_LE,},
-	.ops = &davinci_i2s_dai_ops,
-
-};
-
-static const struct snd_soc_component_driver davinci_i2s_component = {
-	.name		= DRV_NAME,
-};
-
-static int davinci_i2s_probe(struct platform_device *pdev)
-{
-	struct snd_dmaengine_dai_dma_data *dma_data;
-	struct davinci_mcbsp_dev *dev;
-	struct resource *mem, *res;
-	void __iomem *io_base;
-	int *dma;
-	int ret;
-
-	mem = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mpu");
-	if (!mem) {
-		dev_warn(&pdev->dev,
-			 "\"mpu\" mem resource not found, using index 0\n");
-		mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-		if (!mem) {
-			dev_err(&pdev->dev, "no mem resource?\n");
-			return -ENODEV;
-		}
-	}
-
-	io_base = devm_ioremap_resource(&pdev->dev, mem);
-	if (IS_ERR(io_base))
-		return PTR_ERR(io_base);
-
-	dev = devm_kzalloc(&pdev->dev, sizeof(struct davinci_mcbsp_dev),
-			   GFP_KERNEL);
-	if (!dev)
-		return -ENOMEM;
-
-	dev->base = io_base;
-
-	/* setup DMA, first TX, then RX */
-	dma_data = &dev->dma_data[SNDRV_PCM_STREAM_PLAYBACK];
-	dma_data->addr = (dma_addr_t)(mem->start + DAVINCI_MCBSP_DXR_REG);
-
-	res = platform_get_resource(pdev, IORESOURCE_DMA, 0);
-	if (res) {
-		dma = &dev->dma_request[SNDRV_PCM_STREAM_PLAYBACK];
-		*dma = res->start;
-		dma_data->filter_data = dma;
-	} else if (IS_ENABLED(CONFIG_OF) && pdev->dev.of_node) {
-		dma_data->filter_data = "tx";
-	} else {
-		dev_err(&pdev->dev, "Missing DMA tx resource\n");
-		return -ENODEV;
-	}
-
-	dma_data = &dev->dma_data[SNDRV_PCM_STREAM_CAPTURE];
-	dma_data->addr = (dma_addr_t)(mem->start + DAVINCI_MCBSP_DRR_REG);
-
-	res = platform_get_resource(pdev, IORESOURCE_DMA, 1);
-	if (res) {
-		dma = &dev->dma_request[SNDRV_PCM_STREAM_CAPTURE];
-		*dma = res->start;
-		dma_data->filter_data = dma;
-	} else if (IS_ENABLED(CONFIG_OF) && pdev->dev.of_node) {
-		dma_data->filter_data = "rx";
-	} else {
-		dev_err(&pdev->dev, "Missing DMA rx resource\n");
-		return -ENODEV;
-	}
-
-	dev->clk = clk_get(&pdev->dev, NULL);
-	if (IS_ERR(dev->clk))
-		return -ENODEV;
-	clk_enable(dev->clk);
-
-	dev->dev = &pdev->dev;
-	dev_set_drvdata(&pdev->dev, dev);
-
-	ret = snd_soc_register_component(&pdev->dev, &davinci_i2s_component,
-					 &davinci_i2s_dai, 1);
-	if (ret != 0)
-		goto err_release_clk;
-
-	ret = edma_pcm_platform_register(&pdev->dev);
-	if (ret) {
-		dev_err(&pdev->dev, "register PCM failed: %d\n", ret);
-		goto err_unregister_component;
-	}
-
-	return 0;
-
-err_unregister_component:
-	snd_soc_unregister_component(&pdev->dev);
-err_release_clk:
-	clk_disable(dev->clk);
-	clk_put(dev->clk);
-	return ret;
-}
-
-static int davinci_i2s_remove(struct platform_device *pdev)
-{
-	struct davinci_mcbsp_dev *dev = dev_get_drvdata(&pdev->dev);
-
-	snd_soc_unregister_component(&pdev->dev);
-
-	clk_disable(dev->clk);
-	clk_put(dev->clk);
-	dev->clk = NULL;
-
-	return 0;
-}
-
-static const struct of_device_id davinci_i2s_match[] = {
-	{ .compatible = "ti,da850-mcbsp" },
-	{},
-};
-MODULE_DEVICE_TABLE(of, davinci_i2s_match);
-
-static struct platform_driver davinci_mcbsp_driver = {
-	.probe		= davinci_i2s_probe,
-	.remove		= davinci_i2s_remove,
-	.driver		= {
-		.name	= "davinci-mcbsp",
-		.of_match_table = of_match_ptr(davinci_i2s_match),
-	},
-};
-
-module_platform_driver(davinci_mcbsp_driver);
-
-MODULE_AUTHOR("Vladimir Barinov");
-MODULE_DESCRIPTION("TI DAVINCI I2S (McBSP) SoC Interface");
-MODULE_LICENSE("GPL");
diff -Naur --no-dereference a/sound/soc/davinci/davinci-i2s.h b/sound/soc/davinci/davinci-i2s.h
--- a/sound/soc/davinci/davinci-i2s.h	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/davinci/davinci-i2s.h	1969-12-31 19:00:00.000000000 -0500
@@ -1,20 +0,0 @@
-/*
- * ALSA SoC I2S (McBSP) Audio Layer for TI DAVINCI processor
- *
- * Author:      Vladimir Barinov, <vbarinov@embeddedalley.com>
- * Copyright:   (C) 2007 MontaVista Software, Inc., <source@mvista.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#ifndef _DAVINCI_I2S_H
-#define _DAVINCI_I2S_H
-
-/* McBSP dividers */
-enum davinci_mcbsp_div {
-	DAVINCI_MCBSP_CLKGDV,              /* Sample rate generator divider */
-};
-
-#endif
diff -Naur --no-dereference a/sound/soc/davinci/davinci-mcasp.c b/sound/soc/davinci/davinci-mcasp.c
--- a/sound/soc/davinci/davinci-mcasp.c	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/davinci/davinci-mcasp.c	1969-12-31 19:00:00.000000000 -0500
@@ -1,2121 +0,0 @@
-/*
- * ALSA SoC McASP Audio Layer for TI DAVINCI processor
- *
- * Multi-channel Audio Serial Port Driver
- *
- * Author: Nirmal Pandey <n-pandey@ti.com>,
- *         Suresh Rajashekara <suresh.r@ti.com>
- *         Steve Chen <schen@.mvista.com>
- *
- * Copyright:   (C) 2009 MontaVista Software, Inc., <source@mvista.com>
- * Copyright:   (C) 2009  Texas Instruments, India
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/device.h>
-#include <linux/slab.h>
-#include <linux/delay.h>
-#include <linux/io.h>
-#include <linux/clk.h>
-#include <linux/pm_runtime.h>
-#include <linux/of.h>
-#include <linux/of_platform.h>
-#include <linux/of_device.h>
-#include <linux/platform_data/davinci_asp.h>
-#include <linux/math64.h>
-
-#include <sound/asoundef.h>
-#include <sound/core.h>
-#include <sound/pcm.h>
-#include <sound/pcm_params.h>
-#include <sound/initval.h>
-#include <sound/soc.h>
-#include <sound/dmaengine_pcm.h>
-
-#include "edma-pcm.h"
-#include "../omap/sdma-pcm.h"
-#include "davinci-mcasp.h"
-
-#define MCASP_MAX_AFIFO_DEPTH	64
-
-#ifdef CONFIG_PM
-static u32 context_regs[] = {
-	DAVINCI_MCASP_TXFMCTL_REG,
-	DAVINCI_MCASP_RXFMCTL_REG,
-	DAVINCI_MCASP_TXFMT_REG,
-	DAVINCI_MCASP_RXFMT_REG,
-	DAVINCI_MCASP_ACLKXCTL_REG,
-	DAVINCI_MCASP_ACLKRCTL_REG,
-	DAVINCI_MCASP_AHCLKXCTL_REG,
-	DAVINCI_MCASP_AHCLKRCTL_REG,
-	DAVINCI_MCASP_PDIR_REG,
-	DAVINCI_MCASP_RXMASK_REG,
-	DAVINCI_MCASP_TXMASK_REG,
-	DAVINCI_MCASP_RXTDM_REG,
-	DAVINCI_MCASP_TXTDM_REG,
-};
-
-struct davinci_mcasp_context {
-	u32	config_regs[ARRAY_SIZE(context_regs)];
-	u32	afifo_regs[2]; /* for read/write fifo control registers */
-	u32	*xrsr_regs; /* for serializer configuration */
-	bool	pm_state;
-};
-#endif
-
-struct davinci_mcasp_ruledata {
-	struct davinci_mcasp *mcasp;
-	int serializers;
-};
-
-struct davinci_mcasp {
-	struct snd_dmaengine_dai_dma_data dma_data[2];
-	void __iomem *base;
-	u32 fifo_base;
-	struct device *dev;
-	struct snd_pcm_substream *substreams[2];
-	unsigned int dai_fmt;
-
-	/* McASP specific data */
-	int	tdm_slots;
-	u32	tdm_mask[2];
-	int	slot_width;
-	u8	op_mode;
-	u8	num_serializer;
-	u8	*serial_dir;
-	u8	version;
-	u8	bclk_div;
-	int	streams;
-	u32	irq_request[2];
-	int	dma_request[2];
-
-	int	sysclk_freq;
-	bool	bclk_master;
-
-	/* McASP FIFO related */
-	u8	txnumevt;
-	u8	rxnumevt;
-
-	bool	dat_port;
-
-	/* Used for comstraint setting on the second stream */
-	u32	channels;
-
-#ifdef CONFIG_PM_SLEEP
-	struct davinci_mcasp_context context;
-#endif
-
-	struct davinci_mcasp_ruledata ruledata[2];
-	struct snd_pcm_hw_constraint_list chconstr[2];
-};
-
-static inline void mcasp_set_bits(struct davinci_mcasp *mcasp, u32 offset,
-				  u32 val)
-{
-	void __iomem *reg = mcasp->base + offset;
-	__raw_writel(__raw_readl(reg) | val, reg);
-}
-
-static inline void mcasp_clr_bits(struct davinci_mcasp *mcasp, u32 offset,
-				  u32 val)
-{
-	void __iomem *reg = mcasp->base + offset;
-	__raw_writel((__raw_readl(reg) & ~(val)), reg);
-}
-
-static inline void mcasp_mod_bits(struct davinci_mcasp *mcasp, u32 offset,
-				  u32 val, u32 mask)
-{
-	void __iomem *reg = mcasp->base + offset;
-	__raw_writel((__raw_readl(reg) & ~mask) | val, reg);
-}
-
-static inline void mcasp_set_reg(struct davinci_mcasp *mcasp, u32 offset,
-				 u32 val)
-{
-	__raw_writel(val, mcasp->base + offset);
-}
-
-static inline u32 mcasp_get_reg(struct davinci_mcasp *mcasp, u32 offset)
-{
-	return (u32)__raw_readl(mcasp->base + offset);
-}
-
-static void mcasp_set_ctl_reg(struct davinci_mcasp *mcasp, u32 ctl_reg, u32 val)
-{
-	int i = 0;
-
-	mcasp_set_bits(mcasp, ctl_reg, val);
-
-	/* programming GBLCTL needs to read back from GBLCTL and verfiy */
-	/* loop count is to avoid the lock-up */
-	for (i = 0; i < 1000; i++) {
-		if ((mcasp_get_reg(mcasp, ctl_reg) & val) == val)
-			break;
-	}
-
-	if (i == 1000 && ((mcasp_get_reg(mcasp, ctl_reg) & val) != val))
-		printk(KERN_ERR "GBLCTL write error\n");
-}
-
-static bool mcasp_is_synchronous(struct davinci_mcasp *mcasp)
-{
-	u32 rxfmctl = mcasp_get_reg(mcasp, DAVINCI_MCASP_RXFMCTL_REG);
-	u32 aclkxctl = mcasp_get_reg(mcasp, DAVINCI_MCASP_ACLKXCTL_REG);
-
-	return !(aclkxctl & TX_ASYNC) && rxfmctl & AFSRE;
-}
-
-static void mcasp_start_rx(struct davinci_mcasp *mcasp)
-{
-	if (mcasp->rxnumevt) {	/* enable FIFO */
-		u32 reg = mcasp->fifo_base + MCASP_RFIFOCTL_OFFSET;
-
-		mcasp_clr_bits(mcasp, reg, FIFO_ENABLE);
-		mcasp_set_bits(mcasp, reg, FIFO_ENABLE);
-	}
-
-	/* Start clocks */
-	mcasp_set_ctl_reg(mcasp, DAVINCI_MCASP_GBLCTLR_REG, RXHCLKRST);
-	mcasp_set_ctl_reg(mcasp, DAVINCI_MCASP_GBLCTLR_REG, RXCLKRST);
-	/*
-	 * When ASYNC == 0 the transmit and receive sections operate
-	 * synchronously from the transmit clock and frame sync. We need to make
-	 * sure that the TX signlas are enabled when starting reception.
-	 */
-	if (mcasp_is_synchronous(mcasp)) {
-		mcasp_set_ctl_reg(mcasp, DAVINCI_MCASP_GBLCTLX_REG, TXHCLKRST);
-		mcasp_set_ctl_reg(mcasp, DAVINCI_MCASP_GBLCTLX_REG, TXCLKRST);
-	}
-
-	/* Activate serializer(s) */
-	mcasp_set_ctl_reg(mcasp, DAVINCI_MCASP_GBLCTLR_REG, RXSERCLR);
-	/* Release RX state machine */
-	mcasp_set_ctl_reg(mcasp, DAVINCI_MCASP_GBLCTLR_REG, RXSMRST);
-	/* Release Frame Sync generator */
-	mcasp_set_ctl_reg(mcasp, DAVINCI_MCASP_GBLCTLR_REG, RXFSRST);
-	if (mcasp_is_synchronous(mcasp))
-		mcasp_set_ctl_reg(mcasp, DAVINCI_MCASP_GBLCTLX_REG, TXFSRST);
-
-	/* enable receive IRQs */
-	mcasp_set_bits(mcasp, DAVINCI_MCASP_EVTCTLR_REG,
-		       mcasp->irq_request[SNDRV_PCM_STREAM_CAPTURE]);
-}
-
-static void mcasp_start_tx(struct davinci_mcasp *mcasp)
-{
-	u32 cnt;
-
-	if (mcasp->txnumevt) {	/* enable FIFO */
-		u32 reg = mcasp->fifo_base + MCASP_WFIFOCTL_OFFSET;
-
-		mcasp_clr_bits(mcasp, reg, FIFO_ENABLE);
-		mcasp_set_bits(mcasp, reg, FIFO_ENABLE);
-	}
-
-	/* Start clocks */
-	mcasp_set_ctl_reg(mcasp, DAVINCI_MCASP_GBLCTLX_REG, TXHCLKRST);
-	mcasp_set_ctl_reg(mcasp, DAVINCI_MCASP_GBLCTLX_REG, TXCLKRST);
-	/* Activate serializer(s) */
-	mcasp_set_ctl_reg(mcasp, DAVINCI_MCASP_GBLCTLX_REG, TXSERCLR);
-
-	/* wait for XDATA to be cleared */
-	cnt = 0;
-	while ((mcasp_get_reg(mcasp, DAVINCI_MCASP_TXSTAT_REG) & XRDATA) &&
-	       (cnt < 100000))
-		cnt++;
-
-	/* Release TX state machine */
-	mcasp_set_ctl_reg(mcasp, DAVINCI_MCASP_GBLCTLX_REG, TXSMRST);
-	/* Release Frame Sync generator */
-	mcasp_set_ctl_reg(mcasp, DAVINCI_MCASP_GBLCTLX_REG, TXFSRST);
-
-	/* enable transmit IRQs */
-	mcasp_set_bits(mcasp, DAVINCI_MCASP_EVTCTLX_REG,
-		       mcasp->irq_request[SNDRV_PCM_STREAM_PLAYBACK]);
-}
-
-static void davinci_mcasp_start(struct davinci_mcasp *mcasp, int stream)
-{
-	mcasp->streams++;
-
-	if (stream == SNDRV_PCM_STREAM_PLAYBACK)
-		mcasp_start_tx(mcasp);
-	else
-		mcasp_start_rx(mcasp);
-}
-
-static void mcasp_stop_rx(struct davinci_mcasp *mcasp)
-{
-	/* disable IRQ sources */
-	mcasp_clr_bits(mcasp, DAVINCI_MCASP_EVTCTLR_REG,
-		       mcasp->irq_request[SNDRV_PCM_STREAM_CAPTURE]);
-
-	/*
-	 * In synchronous mode stop the TX clocks if no other stream is
-	 * running
-	 */
-	if (mcasp_is_synchronous(mcasp) && !mcasp->streams)
-		mcasp_set_reg(mcasp, DAVINCI_MCASP_GBLCTLX_REG, 0);
-
-	mcasp_set_reg(mcasp, DAVINCI_MCASP_GBLCTLR_REG, 0);
-	mcasp_set_reg(mcasp, DAVINCI_MCASP_RXSTAT_REG, 0xFFFFFFFF);
-
-	if (mcasp->rxnumevt) {	/* disable FIFO */
-		u32 reg = mcasp->fifo_base + MCASP_RFIFOCTL_OFFSET;
-
-		mcasp_clr_bits(mcasp, reg, FIFO_ENABLE);
-	}
-}
-
-static void mcasp_stop_tx(struct davinci_mcasp *mcasp)
-{
-	u32 val = 0;
-
-	/* disable IRQ sources */
-	mcasp_clr_bits(mcasp, DAVINCI_MCASP_EVTCTLX_REG,
-		       mcasp->irq_request[SNDRV_PCM_STREAM_PLAYBACK]);
-
-	/*
-	 * In synchronous mode keep TX clocks running if the capture stream is
-	 * still running.
-	 */
-	if (mcasp_is_synchronous(mcasp) && mcasp->streams)
-		val =  TXHCLKRST | TXCLKRST | TXFSRST;
-
-	mcasp_set_reg(mcasp, DAVINCI_MCASP_GBLCTLX_REG, val);
-	mcasp_set_reg(mcasp, DAVINCI_MCASP_TXSTAT_REG, 0xFFFFFFFF);
-
-	if (mcasp->txnumevt) {	/* disable FIFO */
-		u32 reg = mcasp->fifo_base + MCASP_WFIFOCTL_OFFSET;
-
-		mcasp_clr_bits(mcasp, reg, FIFO_ENABLE);
-	}
-}
-
-static void davinci_mcasp_stop(struct davinci_mcasp *mcasp, int stream)
-{
-	mcasp->streams--;
-
-	if (stream == SNDRV_PCM_STREAM_PLAYBACK)
-		mcasp_stop_tx(mcasp);
-	else
-		mcasp_stop_rx(mcasp);
-}
-
-static irqreturn_t davinci_mcasp_tx_irq_handler(int irq, void *data)
-{
-	struct davinci_mcasp *mcasp = (struct davinci_mcasp *)data;
-	struct snd_pcm_substream *substream;
-	u32 irq_mask = mcasp->irq_request[SNDRV_PCM_STREAM_PLAYBACK];
-	u32 handled_mask = 0;
-	u32 stat;
-
-	stat = mcasp_get_reg(mcasp, DAVINCI_MCASP_TXSTAT_REG);
-	if (stat & XUNDRN & irq_mask) {
-		dev_warn(mcasp->dev, "Transmit buffer underflow\n");
-		handled_mask |= XUNDRN;
-
-		substream = mcasp->substreams[SNDRV_PCM_STREAM_PLAYBACK];
-		if (substream)
-			snd_pcm_stop_xrun(substream);
-	}
-
-	if (!handled_mask)
-		dev_warn(mcasp->dev, "unhandled tx event. txstat: 0x%08x\n",
-			 stat);
-
-	if (stat & XRERR)
-		handled_mask |= XRERR;
-
-	/* Ack the handled event only */
-	mcasp_set_reg(mcasp, DAVINCI_MCASP_TXSTAT_REG, handled_mask);
-
-	return IRQ_RETVAL(handled_mask);
-}
-
-static irqreturn_t davinci_mcasp_rx_irq_handler(int irq, void *data)
-{
-	struct davinci_mcasp *mcasp = (struct davinci_mcasp *)data;
-	struct snd_pcm_substream *substream;
-	u32 irq_mask = mcasp->irq_request[SNDRV_PCM_STREAM_CAPTURE];
-	u32 handled_mask = 0;
-	u32 stat;
-
-	stat = mcasp_get_reg(mcasp, DAVINCI_MCASP_RXSTAT_REG);
-	if (stat & ROVRN & irq_mask) {
-		dev_warn(mcasp->dev, "Receive buffer overflow\n");
-		handled_mask |= ROVRN;
-
-		substream = mcasp->substreams[SNDRV_PCM_STREAM_CAPTURE];
-		if (substream)
-			snd_pcm_stop_xrun(substream);
-	}
-
-	if (!handled_mask)
-		dev_warn(mcasp->dev, "unhandled rx event. rxstat: 0x%08x\n",
-			 stat);
-
-	if (stat & XRERR)
-		handled_mask |= XRERR;
-
-	/* Ack the handled event only */
-	mcasp_set_reg(mcasp, DAVINCI_MCASP_RXSTAT_REG, handled_mask);
-
-	return IRQ_RETVAL(handled_mask);
-}
-
-static irqreturn_t davinci_mcasp_common_irq_handler(int irq, void *data)
-{
-	struct davinci_mcasp *mcasp = (struct davinci_mcasp *)data;
-	irqreturn_t ret = IRQ_NONE;
-
-	if (mcasp->substreams[SNDRV_PCM_STREAM_PLAYBACK])
-		ret = davinci_mcasp_tx_irq_handler(irq, data);
-
-	if (mcasp->substreams[SNDRV_PCM_STREAM_CAPTURE])
-		ret |= davinci_mcasp_rx_irq_handler(irq, data);
-
-	return ret;
-}
-
-static int davinci_mcasp_set_dai_fmt(struct snd_soc_dai *cpu_dai,
-					 unsigned int fmt)
-{
-	struct davinci_mcasp *mcasp = snd_soc_dai_get_drvdata(cpu_dai);
-	int ret = 0;
-	u32 data_delay;
-	bool fs_pol_rising;
-	bool inv_fs = false;
-
-	if (!fmt)
-		return 0;
-
-	pm_runtime_get_sync(mcasp->dev);
-	switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) {
-	case SND_SOC_DAIFMT_DSP_A:
-		mcasp_clr_bits(mcasp, DAVINCI_MCASP_TXFMCTL_REG, FSXDUR);
-		mcasp_clr_bits(mcasp, DAVINCI_MCASP_RXFMCTL_REG, FSRDUR);
-		/* 1st data bit occur one ACLK cycle after the frame sync */
-		data_delay = 1;
-		break;
-	case SND_SOC_DAIFMT_DSP_B:
-	case SND_SOC_DAIFMT_AC97:
-		mcasp_clr_bits(mcasp, DAVINCI_MCASP_TXFMCTL_REG, FSXDUR);
-		mcasp_clr_bits(mcasp, DAVINCI_MCASP_RXFMCTL_REG, FSRDUR);
-		/* No delay after FS */
-		data_delay = 0;
-		break;
-	case SND_SOC_DAIFMT_I2S:
-		/* configure a full-word SYNC pulse (LRCLK) */
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_TXFMCTL_REG, FSXDUR);
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_RXFMCTL_REG, FSRDUR);
-		/* 1st data bit occur one ACLK cycle after the frame sync */
-		data_delay = 1;
-		/* FS need to be inverted */
-		inv_fs = true;
-		break;
-	case SND_SOC_DAIFMT_LEFT_J:
-		/* configure a full-word SYNC pulse (LRCLK) */
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_TXFMCTL_REG, FSXDUR);
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_RXFMCTL_REG, FSRDUR);
-		/* No delay after FS */
-		data_delay = 0;
-		break;
-	default:
-		ret = -EINVAL;
-		goto out;
-	}
-
-	mcasp_mod_bits(mcasp, DAVINCI_MCASP_TXFMT_REG, FSXDLY(data_delay),
-		       FSXDLY(3));
-	mcasp_mod_bits(mcasp, DAVINCI_MCASP_RXFMT_REG, FSRDLY(data_delay),
-		       FSRDLY(3));
-
-	switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) {
-	case SND_SOC_DAIFMT_CBS_CFS:
-		/* codec is clock and frame slave */
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_ACLKXCTL_REG, ACLKXE);
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_TXFMCTL_REG, AFSXE);
-
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_ACLKRCTL_REG, ACLKRE);
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_RXFMCTL_REG, AFSRE);
-
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_PDIR_REG, ACLKX | ACLKR);
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_PDIR_REG, AFSX | AFSR);
-		mcasp->bclk_master = 1;
-		break;
-	case SND_SOC_DAIFMT_CBS_CFM:
-		/* codec is clock slave and frame master */
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_ACLKXCTL_REG, ACLKXE);
-		mcasp_clr_bits(mcasp, DAVINCI_MCASP_TXFMCTL_REG, AFSXE);
-
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_ACLKRCTL_REG, ACLKRE);
-		mcasp_clr_bits(mcasp, DAVINCI_MCASP_RXFMCTL_REG, AFSRE);
-
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_PDIR_REG, ACLKX | ACLKR);
-		mcasp_clr_bits(mcasp, DAVINCI_MCASP_PDIR_REG, AFSX | AFSR);
-		mcasp->bclk_master = 1;
-		break;
-	case SND_SOC_DAIFMT_CBM_CFS:
-		/* codec is clock master and frame slave */
-		mcasp_clr_bits(mcasp, DAVINCI_MCASP_ACLKXCTL_REG, ACLKXE);
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_TXFMCTL_REG, AFSXE);
-
-		mcasp_clr_bits(mcasp, DAVINCI_MCASP_ACLKRCTL_REG, ACLKRE);
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_RXFMCTL_REG, AFSRE);
-
-		mcasp_clr_bits(mcasp, DAVINCI_MCASP_PDIR_REG, ACLKX | ACLKR);
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_PDIR_REG, AFSX | AFSR);
-		mcasp->bclk_master = 0;
-		break;
-	case SND_SOC_DAIFMT_CBM_CFM:
-		/* codec is clock and frame master */
-		mcasp_clr_bits(mcasp, DAVINCI_MCASP_ACLKXCTL_REG, ACLKXE);
-		mcasp_clr_bits(mcasp, DAVINCI_MCASP_TXFMCTL_REG, AFSXE);
-
-		mcasp_clr_bits(mcasp, DAVINCI_MCASP_ACLKRCTL_REG, ACLKRE);
-		mcasp_clr_bits(mcasp, DAVINCI_MCASP_RXFMCTL_REG, AFSRE);
-
-		mcasp_clr_bits(mcasp, DAVINCI_MCASP_PDIR_REG,
-			       ACLKX | AFSX | ACLKR | AHCLKR | AFSR);
-		mcasp->bclk_master = 0;
-		break;
-	default:
-		ret = -EINVAL;
-		goto out;
-	}
-
-	switch (fmt & SND_SOC_DAIFMT_INV_MASK) {
-	case SND_SOC_DAIFMT_IB_NF:
-		mcasp_clr_bits(mcasp, DAVINCI_MCASP_ACLKXCTL_REG, ACLKXPOL);
-		mcasp_clr_bits(mcasp, DAVINCI_MCASP_ACLKRCTL_REG, ACLKRPOL);
-		fs_pol_rising = true;
-		break;
-	case SND_SOC_DAIFMT_NB_IF:
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_ACLKXCTL_REG, ACLKXPOL);
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_ACLKRCTL_REG, ACLKRPOL);
-		fs_pol_rising = false;
-		break;
-	case SND_SOC_DAIFMT_IB_IF:
-		mcasp_clr_bits(mcasp, DAVINCI_MCASP_ACLKXCTL_REG, ACLKXPOL);
-		mcasp_clr_bits(mcasp, DAVINCI_MCASP_ACLKRCTL_REG, ACLKRPOL);
-		fs_pol_rising = false;
-		break;
-	case SND_SOC_DAIFMT_NB_NF:
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_ACLKXCTL_REG, ACLKXPOL);
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_ACLKRCTL_REG, ACLKRPOL);
-		fs_pol_rising = true;
-		break;
-	default:
-		ret = -EINVAL;
-		goto out;
-	}
-
-	if (inv_fs)
-		fs_pol_rising = !fs_pol_rising;
-
-	if (fs_pol_rising) {
-		mcasp_clr_bits(mcasp, DAVINCI_MCASP_TXFMCTL_REG, FSXPOL);
-		mcasp_clr_bits(mcasp, DAVINCI_MCASP_RXFMCTL_REG, FSRPOL);
-	} else {
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_TXFMCTL_REG, FSXPOL);
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_RXFMCTL_REG, FSRPOL);
-	}
-
-	mcasp->dai_fmt = fmt;
-out:
-	pm_runtime_put(mcasp->dev);
-	return ret;
-}
-
-static int __davinci_mcasp_set_clkdiv(struct davinci_mcasp *mcasp, int div_id,
-				      int div, bool explicit)
-{
-	pm_runtime_get_sync(mcasp->dev);
-	switch (div_id) {
-	case MCASP_CLKDIV_AUXCLK:			/* MCLK divider */
-		mcasp_mod_bits(mcasp, DAVINCI_MCASP_AHCLKXCTL_REG,
-			       AHCLKXDIV(div - 1), AHCLKXDIV_MASK);
-		mcasp_mod_bits(mcasp, DAVINCI_MCASP_AHCLKRCTL_REG,
-			       AHCLKRDIV(div - 1), AHCLKRDIV_MASK);
-		break;
-
-	case MCASP_CLKDIV_BCLK:			/* BCLK divider */
-		mcasp_mod_bits(mcasp, DAVINCI_MCASP_ACLKXCTL_REG,
-			       ACLKXDIV(div - 1), ACLKXDIV_MASK);
-		mcasp_mod_bits(mcasp, DAVINCI_MCASP_ACLKRCTL_REG,
-			       ACLKRDIV(div - 1), ACLKRDIV_MASK);
-		if (explicit)
-			mcasp->bclk_div = div;
-		break;
-
-	case MCASP_CLKDIV_BCLK_FS_RATIO:
-		/*
-		 * BCLK/LRCLK ratio descries how many bit-clock cycles
-		 * fit into one frame. The clock ratio is given for a
-		 * full period of data (for I2S format both left and
-		 * right channels), so it has to be divided by number
-		 * of tdm-slots (for I2S - divided by 2).
-		 * Instead of storing this ratio, we calculate a new
-		 * tdm_slot width by dividing the the ratio by the
-		 * number of configured tdm slots.
-		 */
-		mcasp->slot_width = div / mcasp->tdm_slots;
-		if (div % mcasp->tdm_slots)
-			dev_warn(mcasp->dev,
-				 "%s(): BCLK/LRCLK %d is not divisible by %d tdm slots",
-				 __func__, div, mcasp->tdm_slots);
-		break;
-
-	default:
-		return -EINVAL;
-	}
-
-	pm_runtime_put(mcasp->dev);
-	return 0;
-}
-
-static int davinci_mcasp_set_clkdiv(struct snd_soc_dai *dai, int div_id,
-				    int div)
-{
-	struct davinci_mcasp *mcasp = snd_soc_dai_get_drvdata(dai);
-
-	return __davinci_mcasp_set_clkdiv(mcasp, div_id, div, 1);
-}
-
-static int davinci_mcasp_set_sysclk(struct snd_soc_dai *dai, int clk_id,
-				    unsigned int freq, int dir)
-{
-	struct davinci_mcasp *mcasp = snd_soc_dai_get_drvdata(dai);
-
-	pm_runtime_get_sync(mcasp->dev);
-	if (dir == SND_SOC_CLOCK_OUT) {
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_AHCLKXCTL_REG, AHCLKXE);
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_AHCLKRCTL_REG, AHCLKRE);
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_PDIR_REG, AHCLKX);
-	} else {
-		mcasp_clr_bits(mcasp, DAVINCI_MCASP_AHCLKXCTL_REG, AHCLKXE);
-		mcasp_clr_bits(mcasp, DAVINCI_MCASP_AHCLKRCTL_REG, AHCLKRE);
-		mcasp_clr_bits(mcasp, DAVINCI_MCASP_PDIR_REG, AHCLKX);
-	}
-
-	mcasp->sysclk_freq = freq;
-
-	pm_runtime_put(mcasp->dev);
-	return 0;
-}
-
-/* All serializers must have equal number of channels */
-static int davinci_mcasp_ch_constraint(struct davinci_mcasp *mcasp, int stream,
-				       int serializers)
-{
-	struct snd_pcm_hw_constraint_list *cl = &mcasp->chconstr[stream];
-	unsigned int *list = (unsigned int *) cl->list;
-	int slots = mcasp->tdm_slots;
-	int i, count = 0;
-
-	if (mcasp->tdm_mask[stream])
-		slots = hweight32(mcasp->tdm_mask[stream]);
-
-	for (i = 1; i <= slots; i++)
-		list[count++] = i;
-
-	for (i = 2; i <= serializers; i++)
-		list[count++] = i*slots;
-
-	cl->count = count;
-
-	return 0;
-}
-
-static int davinci_mcasp_set_ch_constraints(struct davinci_mcasp *mcasp)
-{
-	int rx_serializers = 0, tx_serializers = 0, ret, i;
-
-	for (i = 0; i < mcasp->num_serializer; i++)
-		if (mcasp->serial_dir[i] == TX_MODE)
-			tx_serializers++;
-		else if (mcasp->serial_dir[i] == RX_MODE)
-			rx_serializers++;
-
-	ret = davinci_mcasp_ch_constraint(mcasp, SNDRV_PCM_STREAM_PLAYBACK,
-					  tx_serializers);
-	if (ret)
-		return ret;
-
-	ret = davinci_mcasp_ch_constraint(mcasp, SNDRV_PCM_STREAM_CAPTURE,
-					  rx_serializers);
-
-	return ret;
-}
-
-
-static int davinci_mcasp_set_tdm_slot(struct snd_soc_dai *dai,
-				      unsigned int tx_mask,
-				      unsigned int rx_mask,
-				      int slots, int slot_width)
-{
-	struct davinci_mcasp *mcasp = snd_soc_dai_get_drvdata(dai);
-
-	dev_dbg(mcasp->dev,
-		 "%s() tx_mask 0x%08x rx_mask 0x%08x slots %d width %d\n",
-		 __func__, tx_mask, rx_mask, slots, slot_width);
-
-	if (tx_mask >= (1<<slots) || rx_mask >= (1<<slots)) {
-		dev_err(mcasp->dev,
-			"Bad tdm mask tx: 0x%08x rx: 0x%08x slots %d\n",
-			tx_mask, rx_mask, slots);
-		return -EINVAL;
-	}
-
-	if (slot_width &&
-	    (slot_width < 8 || slot_width > 32 || slot_width % 4 != 0)) {
-		dev_err(mcasp->dev, "%s: Unsupported slot_width %d\n",
-			__func__, slot_width);
-		return -EINVAL;
-	}
-
-	mcasp->tdm_slots = slots;
-	mcasp->tdm_mask[SNDRV_PCM_STREAM_PLAYBACK] = tx_mask;
-	mcasp->tdm_mask[SNDRV_PCM_STREAM_CAPTURE] = rx_mask;
-	mcasp->slot_width = slot_width;
-
-	return davinci_mcasp_set_ch_constraints(mcasp);
-}
-
-static int davinci_config_channel_size(struct davinci_mcasp *mcasp,
-				       int sample_width)
-{
-	u32 fmt;
-	u32 tx_rotate = (sample_width / 4) & 0x7;
-	u32 mask = (1ULL << sample_width) - 1;
-	u32 slot_width = sample_width;
-
-	/*
-	 * For captured data we should not rotate, inversion and masking is
-	 * enoguh to get the data to the right position:
-	 * Format	  data from bus		after reverse (XRBUF)
-	 * S16_LE:	|LSB|MSB|xxx|xxx|	|xxx|xxx|MSB|LSB|
-	 * S24_3LE:	|LSB|DAT|MSB|xxx|	|xxx|MSB|DAT|LSB|
-	 * S24_LE:	|LSB|DAT|MSB|xxx|	|xxx|MSB|DAT|LSB|
-	 * S32_LE:	|LSB|DAT|DAT|MSB|	|MSB|DAT|DAT|LSB|
-	 */
-	u32 rx_rotate = 0;
-
-	/*
-	 * Setting the tdm slot width either with set_clkdiv() or
-	 * set_tdm_slot() allows us to for example send 32 bits per
-	 * channel to the codec, while only 16 of them carry audio
-	 * payload.
-	 */
-	if (mcasp->slot_width) {
-		/*
-		 * When we have more bclk then it is needed for the
-		 * data, we need to use the rotation to move the
-		 * received samples to have correct alignment.
-		 */
-		slot_width = mcasp->slot_width;
-		rx_rotate = (slot_width - sample_width) / 4;
-	}
-
-	/* mapping of the XSSZ bit-field as described in the datasheet */
-	fmt = (slot_width >> 1) - 1;
-
-	if (mcasp->op_mode != DAVINCI_MCASP_DIT_MODE) {
-		mcasp_mod_bits(mcasp, DAVINCI_MCASP_RXFMT_REG, RXSSZ(fmt),
-			       RXSSZ(0x0F));
-		mcasp_mod_bits(mcasp, DAVINCI_MCASP_TXFMT_REG, TXSSZ(fmt),
-			       TXSSZ(0x0F));
-		mcasp_mod_bits(mcasp, DAVINCI_MCASP_TXFMT_REG, TXROT(tx_rotate),
-			       TXROT(7));
-		mcasp_mod_bits(mcasp, DAVINCI_MCASP_RXFMT_REG, RXROT(rx_rotate),
-			       RXROT(7));
-		mcasp_set_reg(mcasp, DAVINCI_MCASP_RXMASK_REG, mask);
-	}
-
-	mcasp_set_reg(mcasp, DAVINCI_MCASP_TXMASK_REG, mask);
-
-	return 0;
-}
-
-static int mcasp_common_hw_param(struct davinci_mcasp *mcasp, int stream,
-				 int period_words, int channels)
-{
-	struct snd_dmaengine_dai_dma_data *dma_data = &mcasp->dma_data[stream];
-	int i;
-	u8 tx_ser = 0;
-	u8 rx_ser = 0;
-	u8 slots = mcasp->tdm_slots;
-	u8 max_active_serializers = (channels + slots - 1) / slots;
-	int active_serializers, numevt;
-	u32 reg;
-	/* Default configuration */
-	if (mcasp->version < MCASP_VERSION_3)
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_PWREMUMGT_REG, MCASP_SOFT);
-
-	/* All PINS as McASP */
-	mcasp_set_reg(mcasp, DAVINCI_MCASP_PFUNC_REG, 0x00000000);
-
-	if (stream == SNDRV_PCM_STREAM_PLAYBACK) {
-		mcasp_set_reg(mcasp, DAVINCI_MCASP_TXSTAT_REG, 0xFFFFFFFF);
-		mcasp_clr_bits(mcasp, DAVINCI_MCASP_XEVTCTL_REG, TXDATADMADIS);
-	} else {
-		mcasp_set_reg(mcasp, DAVINCI_MCASP_RXSTAT_REG, 0xFFFFFFFF);
-		mcasp_clr_bits(mcasp, DAVINCI_MCASP_REVTCTL_REG, RXDATADMADIS);
-	}
-
-	for (i = 0; i < mcasp->num_serializer; i++) {
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_XRSRCTL_REG(i),
-			       mcasp->serial_dir[i]);
-		if (mcasp->serial_dir[i] == TX_MODE &&
-					tx_ser < max_active_serializers) {
-			mcasp_set_bits(mcasp, DAVINCI_MCASP_PDIR_REG, AXR(i));
-			mcasp_mod_bits(mcasp, DAVINCI_MCASP_XRSRCTL_REG(i),
-				       DISMOD_LOW, DISMOD_MASK);
-			tx_ser++;
-		} else if (mcasp->serial_dir[i] == RX_MODE &&
-					rx_ser < max_active_serializers) {
-			mcasp_clr_bits(mcasp, DAVINCI_MCASP_PDIR_REG, AXR(i));
-			rx_ser++;
-		} else if (mcasp->serial_dir[i] == INACTIVE_MODE) {
-			mcasp_mod_bits(mcasp, DAVINCI_MCASP_XRSRCTL_REG(i),
-				       SRMOD_INACTIVE, SRMOD_MASK);
-		}
-	}
-
-	if (stream == SNDRV_PCM_STREAM_PLAYBACK) {
-		active_serializers = tx_ser;
-		numevt = mcasp->txnumevt;
-		reg = mcasp->fifo_base + MCASP_WFIFOCTL_OFFSET;
-	} else {
-		active_serializers = rx_ser;
-		numevt = mcasp->rxnumevt;
-		reg = mcasp->fifo_base + MCASP_RFIFOCTL_OFFSET;
-	}
-
-	if (active_serializers < max_active_serializers) {
-		dev_warn(mcasp->dev, "stream has more channels (%d) than are "
-			 "enabled in mcasp (%d)\n", channels,
-			 active_serializers * slots);
-		return -EINVAL;
-	}
-
-	/* AFIFO is not in use */
-	if (!numevt) {
-		/* Configure the burst size for platform drivers */
-		if (active_serializers > 1) {
-			/*
-			 * If more than one serializers are in use we have one
-			 * DMA request to provide data for all serializers.
-			 * For example if three serializers are enabled the DMA
-			 * need to transfer three words per DMA request.
-			 */
-			dma_data->maxburst = active_serializers;
-		} else {
-			dma_data->maxburst = 0;
-		}
-		return 0;
-	}
-
-	if (period_words % active_serializers) {
-		dev_err(mcasp->dev, "Invalid combination of period words and "
-			"active serializers: %d, %d\n", period_words,
-			active_serializers);
-		return -EINVAL;
-	}
-
-	/*
-	 * Calculate the optimal AFIFO depth for platform side:
-	 * The number of words for numevt need to be in steps of active
-	 * serializers.
-	 */
-	numevt = (numevt / active_serializers) * active_serializers;
-
-	while (period_words % numevt && numevt > 0)
-		numevt -= active_serializers;
-	if (numevt <= 0)
-		numevt = active_serializers;
-
-	mcasp_mod_bits(mcasp, reg, active_serializers, NUMDMA_MASK);
-	mcasp_mod_bits(mcasp, reg, NUMEVT(numevt), NUMEVT_MASK);
-
-	/* Configure the burst size for platform drivers */
-	if (numevt == 1)
-		numevt = 0;
-	dma_data->maxburst = numevt;
-
-	return 0;
-}
-
-static int mcasp_i2s_hw_param(struct davinci_mcasp *mcasp, int stream,
-			      int channels)
-{
-	int i, active_slots;
-	int total_slots;
-	int active_serializers;
-	u32 mask = 0;
-	u32 busel = 0;
-
-	total_slots = mcasp->tdm_slots;
-
-	/*
-	 * If more than one serializer is needed, then use them with
-	 * all the specified tdm_slots. Otherwise, one serializer can
-	 * cope with the transaction using just as many slots as there
-	 * are channels in the stream.
-	 */
-	if (mcasp->tdm_mask[stream]) {
-		active_slots = hweight32(mcasp->tdm_mask[stream]);
-		active_serializers = (channels + active_slots - 1) /
-			active_slots;
-		if (active_serializers == 1) {
-			active_slots = channels;
-			for (i = 0; i < total_slots; i++) {
-				if ((1 << i) & mcasp->tdm_mask[stream]) {
-					mask |= (1 << i);
-					if (--active_slots <= 0)
-						break;
-				}
-			}
-		}
-	} else {
-		active_serializers = (channels + total_slots - 1) / total_slots;
-		if (active_serializers == 1)
-			active_slots = channels;
-		else
-			active_slots = total_slots;
-
-		for (i = 0; i < active_slots; i++)
-			mask |= (1 << i);
-	}
-	mcasp_clr_bits(mcasp, DAVINCI_MCASP_ACLKXCTL_REG, TX_ASYNC);
-
-	if (!mcasp->dat_port)
-		busel = TXSEL;
-
-	if (stream == SNDRV_PCM_STREAM_PLAYBACK) {
-		mcasp_set_reg(mcasp, DAVINCI_MCASP_TXTDM_REG, mask);
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_TXFMT_REG, busel | TXORD);
-		mcasp_mod_bits(mcasp, DAVINCI_MCASP_TXFMCTL_REG,
-			       FSXMOD(total_slots), FSXMOD(0x1FF));
-	} else if (stream == SNDRV_PCM_STREAM_CAPTURE) {
-		mcasp_set_reg(mcasp, DAVINCI_MCASP_RXTDM_REG, mask);
-		mcasp_set_bits(mcasp, DAVINCI_MCASP_RXFMT_REG, busel | RXORD);
-		mcasp_mod_bits(mcasp, DAVINCI_MCASP_RXFMCTL_REG,
-			       FSRMOD(total_slots), FSRMOD(0x1FF));
-		/*
-		 * If McASP is set to be TX/RX synchronous and the playback is
-		 * not running already we need to configure the TX slots in
-		 * order to have correct FSX on the bus
-		 */
-		if (mcasp_is_synchronous(mcasp) && !mcasp->channels)
-			mcasp_mod_bits(mcasp, DAVINCI_MCASP_TXFMCTL_REG,
-				       FSXMOD(total_slots), FSXMOD(0x1FF));
-	}
-
-	return 0;
-}
-
-/* S/PDIF */
-static int mcasp_dit_hw_param(struct davinci_mcasp *mcasp,
-			      unsigned int rate)
-{
-	u32 cs_value = 0;
-	u8 *cs_bytes = (u8*) &cs_value;
-
-	/* Set the TX format : 24 bit right rotation, 32 bit slot, Pad 0
-	   and LSB first */
-	mcasp_set_bits(mcasp, DAVINCI_MCASP_TXFMT_REG, TXROT(6) | TXSSZ(15));
-
-	/* Set TX frame synch : DIT Mode, 1 bit width, internal, rising edge */
-	mcasp_set_reg(mcasp, DAVINCI_MCASP_TXFMCTL_REG, AFSXE | FSXMOD(0x180));
-
-	/* Set the TX tdm : for all the slots */
-	mcasp_set_reg(mcasp, DAVINCI_MCASP_TXTDM_REG, 0xFFFFFFFF);
-
-	/* Set the TX clock controls : div = 1 and internal */
-	mcasp_set_bits(mcasp, DAVINCI_MCASP_ACLKXCTL_REG, ACLKXE | TX_ASYNC);
-
-	mcasp_clr_bits(mcasp, DAVINCI_MCASP_XEVTCTL_REG, TXDATADMADIS);
-
-	/* Only 44100 and 48000 are valid, both have the same setting */
-	mcasp_set_bits(mcasp, DAVINCI_MCASP_AHCLKXCTL_REG, AHCLKXDIV(3));
-
-	/* Enable the DIT */
-	mcasp_set_bits(mcasp, DAVINCI_MCASP_TXDITCTL_REG, DITEN);
-
-	/* Set S/PDIF channel status bits */
-	cs_bytes[0] = IEC958_AES0_CON_NOT_COPYRIGHT;
-	cs_bytes[1] = IEC958_AES1_CON_PCM_CODER;
-
-	switch (rate) {
-	case 22050:
-		cs_bytes[3] |= IEC958_AES3_CON_FS_22050;
-		break;
-	case 24000:
-		cs_bytes[3] |= IEC958_AES3_CON_FS_24000;
-		break;
-	case 32000:
-		cs_bytes[3] |= IEC958_AES3_CON_FS_32000;
-		break;
-	case 44100:
-		cs_bytes[3] |= IEC958_AES3_CON_FS_44100;
-		break;
-	case 48000:
-		cs_bytes[3] |= IEC958_AES3_CON_FS_48000;
-		break;
-	case 88200:
-		cs_bytes[3] |= IEC958_AES3_CON_FS_88200;
-		break;
-	case 96000:
-		cs_bytes[3] |= IEC958_AES3_CON_FS_96000;
-		break;
-	case 176400:
-		cs_bytes[3] |= IEC958_AES3_CON_FS_176400;
-		break;
-	case 192000:
-		cs_bytes[3] |= IEC958_AES3_CON_FS_192000;
-		break;
-	default:
-		printk(KERN_WARNING "unsupported sampling rate: %d\n", rate);
-		return -EINVAL;
-	}
-
-	mcasp_set_reg(mcasp, DAVINCI_MCASP_DITCSRA_REG, cs_value);
-	mcasp_set_reg(mcasp, DAVINCI_MCASP_DITCSRB_REG, cs_value);
-
-	return 0;
-}
-
-static int davinci_mcasp_calc_clk_div(struct davinci_mcasp *mcasp,
-				      unsigned int bclk_freq, bool set)
-{
-	int error_ppm;
-	unsigned int sysclk_freq = mcasp->sysclk_freq;
-	u32 reg = mcasp_get_reg(mcasp, DAVINCI_MCASP_AHCLKXCTL_REG);
-	int div = sysclk_freq / bclk_freq;
-	int rem = sysclk_freq % bclk_freq;
-	int aux_div = 1;
-
-	if (div > (ACLKXDIV_MASK + 1)) {
-		if (reg & AHCLKXE) {
-			aux_div = div / (ACLKXDIV_MASK + 1);
-			if (div % (ACLKXDIV_MASK + 1))
-				aux_div++;
-
-			sysclk_freq /= aux_div;
-			div = sysclk_freq / bclk_freq;
-			rem = sysclk_freq % bclk_freq;
-		} else if (set) {
-			dev_warn(mcasp->dev, "Too fast reference clock (%u)\n",
-				 sysclk_freq);
-		}
-	}
-
-	if (rem != 0) {
-		if (div == 0 ||
-		    ((sysclk_freq / div) - bclk_freq) >
-		    (bclk_freq - (sysclk_freq / (div+1)))) {
-			div++;
-			rem = rem - bclk_freq;
-		}
-	}
-	error_ppm = (div*1000000 + (int)div64_long(1000000LL*rem,
-		     (int)bclk_freq)) / div - 1000000;
-
-	if (set) {
-		if (error_ppm)
-			dev_info(mcasp->dev, "Sample-rate is off by %d PPM\n",
-				 error_ppm);
-
-		__davinci_mcasp_set_clkdiv(mcasp, MCASP_CLKDIV_BCLK, div, 0);
-		if (reg & AHCLKXE)
-			__davinci_mcasp_set_clkdiv(mcasp, MCASP_CLKDIV_AUXCLK,
-						   aux_div, 0);
-	}
-
-	return error_ppm;
-}
-
-static int davinci_mcasp_hw_params(struct snd_pcm_substream *substream,
-					struct snd_pcm_hw_params *params,
-					struct snd_soc_dai *cpu_dai)
-{
-	struct davinci_mcasp *mcasp = snd_soc_dai_get_drvdata(cpu_dai);
-	int word_length;
-	int channels = params_channels(params);
-	int period_size = params_period_size(params);
-	int ret;
-
-	ret = davinci_mcasp_set_dai_fmt(cpu_dai, mcasp->dai_fmt);
-	if (ret)
-		return ret;
-
-	/*
-	 * If mcasp is BCLK master, and a BCLK divider was not provided by
-	 * the machine driver, we need to calculate the ratio.
-	 */
-	if (mcasp->bclk_master && mcasp->bclk_div == 0 && mcasp->sysclk_freq) {
-		int slots = mcasp->tdm_slots;
-		int rate = params_rate(params);
-		int sbits = params_width(params);
-
-		if (mcasp->slot_width)
-			sbits = mcasp->slot_width;
-
-		davinci_mcasp_calc_clk_div(mcasp, rate * sbits * slots, true);
-	}
-
-	ret = mcasp_common_hw_param(mcasp, substream->stream,
-				    period_size * channels, channels);
-	if (ret)
-		return ret;
-
-	if (mcasp->op_mode == DAVINCI_MCASP_DIT_MODE)
-		ret = mcasp_dit_hw_param(mcasp, params_rate(params));
-	else
-		ret = mcasp_i2s_hw_param(mcasp, substream->stream,
-					 channels);
-
-	if (ret)
-		return ret;
-
-	switch (params_format(params)) {
-	case SNDRV_PCM_FORMAT_U8:
-	case SNDRV_PCM_FORMAT_S8:
-		word_length = 8;
-		break;
-
-	case SNDRV_PCM_FORMAT_U16_LE:
-	case SNDRV_PCM_FORMAT_S16_LE:
-		word_length = 16;
-		break;
-
-	case SNDRV_PCM_FORMAT_U24_3LE:
-	case SNDRV_PCM_FORMAT_S24_3LE:
-		word_length = 24;
-		break;
-
-	case SNDRV_PCM_FORMAT_U24_LE:
-	case SNDRV_PCM_FORMAT_S24_LE:
-		word_length = 24;
-		break;
-
-	case SNDRV_PCM_FORMAT_U32_LE:
-	case SNDRV_PCM_FORMAT_S32_LE:
-		word_length = 32;
-		break;
-
-	default:
-		printk(KERN_WARNING "davinci-mcasp: unsupported PCM format");
-		return -EINVAL;
-	}
-
-	davinci_config_channel_size(mcasp, word_length);
-
-	if (mcasp->op_mode == DAVINCI_MCASP_IIS_MODE)
-		mcasp->channels = channels;
-
-	return 0;
-}
-
-static int davinci_mcasp_trigger(struct snd_pcm_substream *substream,
-				     int cmd, struct snd_soc_dai *cpu_dai)
-{
-	struct davinci_mcasp *mcasp = snd_soc_dai_get_drvdata(cpu_dai);
-	int ret = 0;
-
-	switch (cmd) {
-	case SNDRV_PCM_TRIGGER_RESUME:
-	case SNDRV_PCM_TRIGGER_START:
-	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
-		davinci_mcasp_start(mcasp, substream->stream);
-		break;
-	case SNDRV_PCM_TRIGGER_SUSPEND:
-	case SNDRV_PCM_TRIGGER_STOP:
-	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
-		davinci_mcasp_stop(mcasp, substream->stream);
-		break;
-
-	default:
-		ret = -EINVAL;
-	}
-
-	return ret;
-}
-
-static int davinci_mcasp_hw_rule_slot_width(struct snd_pcm_hw_params *params,
-					    struct snd_pcm_hw_rule *rule)
-{
-	struct davinci_mcasp_ruledata *rd = rule->private;
-	struct snd_mask *fmt = hw_param_mask(params, SNDRV_PCM_HW_PARAM_FORMAT);
-	struct snd_mask nfmt;
-	int i, slot_width;
-
-	snd_mask_none(&nfmt);
-	slot_width = rd->mcasp->slot_width;
-
-	for (i = 0; i <= SNDRV_PCM_FORMAT_LAST; i++) {
-		if (snd_mask_test(fmt, i)) {
-			if (snd_pcm_format_width(i) <= slot_width) {
-				snd_mask_set(&nfmt, i);
-			}
-		}
-	}
-
-	return snd_mask_refine(fmt, &nfmt);
-}
-
-static const unsigned int davinci_mcasp_dai_rates[] = {
-	8000, 11025, 16000, 22050, 32000, 44100, 48000, 64000,
-	88200, 96000, 176400, 192000,
-};
-
-#define DAVINCI_MAX_RATE_ERROR_PPM 1000
-
-static int davinci_mcasp_hw_rule_rate(struct snd_pcm_hw_params *params,
-				      struct snd_pcm_hw_rule *rule)
-{
-	struct davinci_mcasp_ruledata *rd = rule->private;
-	struct snd_interval *ri =
-		hw_param_interval(params, SNDRV_PCM_HW_PARAM_RATE);
-	int sbits = params_width(params);
-	int slots = rd->mcasp->tdm_slots;
-	struct snd_interval range;
-	int i;
-
-	if (rd->mcasp->slot_width)
-		sbits = rd->mcasp->slot_width;
-
-	snd_interval_any(&range);
-	range.empty = 1;
-
-	for (i = 0; i < ARRAY_SIZE(davinci_mcasp_dai_rates); i++) {
-		if (snd_interval_test(ri, davinci_mcasp_dai_rates[i])) {
-			uint bclk_freq = sbits*slots*
-				davinci_mcasp_dai_rates[i];
-			int ppm;
-
-			ppm = davinci_mcasp_calc_clk_div(rd->mcasp, bclk_freq,
-							 false);
-			if (abs(ppm) < DAVINCI_MAX_RATE_ERROR_PPM) {
-				if (range.empty) {
-					range.min = davinci_mcasp_dai_rates[i];
-					range.empty = 0;
-				}
-				range.max = davinci_mcasp_dai_rates[i];
-			}
-		}
-	}
-
-	dev_dbg(rd->mcasp->dev,
-		"Frequencies %d-%d -> %d-%d for %d sbits and %d tdm slots\n",
-		ri->min, ri->max, range.min, range.max, sbits, slots);
-
-	return snd_interval_refine(hw_param_interval(params, rule->var),
-				   &range);
-}
-
-static int davinci_mcasp_hw_rule_format(struct snd_pcm_hw_params *params,
-					struct snd_pcm_hw_rule *rule)
-{
-	struct davinci_mcasp_ruledata *rd = rule->private;
-	struct snd_mask *fmt = hw_param_mask(params, SNDRV_PCM_HW_PARAM_FORMAT);
-	struct snd_mask nfmt;
-	int rate = params_rate(params);
-	int slots = rd->mcasp->tdm_slots;
-	int i, count = 0;
-
-	snd_mask_none(&nfmt);
-
-	for (i = 0; i <= SNDRV_PCM_FORMAT_LAST; i++) {
-		if (snd_mask_test(fmt, i)) {
-			uint sbits = snd_pcm_format_width(i);
-			int ppm;
-
-			if (rd->mcasp->slot_width)
-				sbits = rd->mcasp->slot_width;
-
-			ppm = davinci_mcasp_calc_clk_div(rd->mcasp,
-							 sbits * slots * rate,
-							 false);
-			if (abs(ppm) < DAVINCI_MAX_RATE_ERROR_PPM) {
-				snd_mask_set(&nfmt, i);
-				count++;
-			}
-		}
-	}
-	dev_dbg(rd->mcasp->dev,
-		"%d possible sample format for %d Hz and %d tdm slots\n",
-		count, rate, slots);
-
-	return snd_mask_refine(fmt, &nfmt);
-}
-
-static int davinci_mcasp_hw_rule_min_periodsize(
-		struct snd_pcm_hw_params *params, struct snd_pcm_hw_rule *rule)
-{
-	struct snd_interval *period_size = hw_param_interval(params,
-						SNDRV_PCM_HW_PARAM_PERIOD_SIZE);
-	struct snd_interval frames;
-
-	snd_interval_any(&frames);
-	frames.min = 64;
-	frames.integer = 1;
-
-	return snd_interval_refine(period_size, &frames);
-}
-
-static int davinci_mcasp_startup(struct snd_pcm_substream *substream,
-				 struct snd_soc_dai *cpu_dai)
-{
-	struct davinci_mcasp *mcasp = snd_soc_dai_get_drvdata(cpu_dai);
-	struct davinci_mcasp_ruledata *ruledata =
-					&mcasp->ruledata[substream->stream];
-	u32 max_channels = 0;
-	int i, dir, ret;
-	int tdm_slots = mcasp->tdm_slots;
-
-	/* Do not allow more then one stream per direction */
-	if (mcasp->substreams[substream->stream])
-		return -EBUSY;
-
-	mcasp->substreams[substream->stream] = substream;
-
-	if (mcasp->tdm_mask[substream->stream])
-		tdm_slots = hweight32(mcasp->tdm_mask[substream->stream]);
-
-	if (mcasp->op_mode == DAVINCI_MCASP_DIT_MODE)
-		return 0;
-
-	/*
-	 * Limit the maximum allowed channels for the first stream:
-	 * number of serializers for the direction * tdm slots per serializer
-	 */
-	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
-		dir = TX_MODE;
-	else
-		dir = RX_MODE;
-
-	for (i = 0; i < mcasp->num_serializer; i++) {
-		if (mcasp->serial_dir[i] == dir)
-			max_channels++;
-	}
-	ruledata->serializers = max_channels;
-	ruledata->mcasp = mcasp;
-	max_channels *= tdm_slots;
-	/*
-	 * If the already active stream has less channels than the calculated
-	 * limnit based on the seirializers * tdm_slots, we need to use that as
-	 * a constraint for the second stream.
-	 * Otherwise (first stream or less allowed channels) we use the
-	 * calculated constraint.
-	 */
-	if (mcasp->channels && mcasp->channels < max_channels)
-		max_channels = mcasp->channels;
-	/*
-	 * But we can always allow channels upto the amount of
-	 * the available tdm_slots.
-	 */
-	if (max_channels < tdm_slots)
-		max_channels = tdm_slots;
-
-	snd_pcm_hw_constraint_minmax(substream->runtime,
-				     SNDRV_PCM_HW_PARAM_CHANNELS,
-				     0, max_channels);
-
-	snd_pcm_hw_constraint_list(substream->runtime,
-				   0, SNDRV_PCM_HW_PARAM_CHANNELS,
-				   &mcasp->chconstr[substream->stream]);
-
-	if (mcasp->slot_width) {
-		/* Only allow formats require <= slot_width bits on the bus */
-		ret = snd_pcm_hw_rule_add(substream->runtime, 0,
-					  SNDRV_PCM_HW_PARAM_FORMAT,
-					  davinci_mcasp_hw_rule_slot_width,
-					  ruledata,
-					  SNDRV_PCM_HW_PARAM_FORMAT, -1);
-		if (ret)
-			return ret;
-	}
-
-	/*
-	 * If we rely on implicit BCLK divider setting we should
-	 * set constraints based on what we can provide.
-	 */
-	if (mcasp->bclk_master && mcasp->bclk_div == 0 && mcasp->sysclk_freq) {
-		ret = snd_pcm_hw_rule_add(substream->runtime, 0,
-					  SNDRV_PCM_HW_PARAM_RATE,
-					  davinci_mcasp_hw_rule_rate,
-					  ruledata,
-					  SNDRV_PCM_HW_PARAM_FORMAT, -1);
-		if (ret)
-			return ret;
-		ret = snd_pcm_hw_rule_add(substream->runtime, 0,
-					  SNDRV_PCM_HW_PARAM_FORMAT,
-					  davinci_mcasp_hw_rule_format,
-					  ruledata,
-					  SNDRV_PCM_HW_PARAM_RATE, -1);
-		if (ret)
-			return ret;
-	}
-
-	snd_pcm_hw_rule_add(substream->runtime, 0,
-			    SNDRV_PCM_HW_PARAM_PERIOD_SIZE,
-			    davinci_mcasp_hw_rule_min_periodsize, NULL,
-			    SNDRV_PCM_HW_PARAM_PERIOD_SIZE, -1);
-
-	return 0;
-}
-
-static void davinci_mcasp_shutdown(struct snd_pcm_substream *substream,
-				   struct snd_soc_dai *cpu_dai)
-{
-	struct davinci_mcasp *mcasp = snd_soc_dai_get_drvdata(cpu_dai);
-
-	mcasp->substreams[substream->stream] = NULL;
-
-	if (mcasp->op_mode == DAVINCI_MCASP_DIT_MODE)
-		return;
-
-	if (!cpu_dai->active)
-		mcasp->channels = 0;
-}
-
-static const struct snd_soc_dai_ops davinci_mcasp_dai_ops = {
-	.startup	= davinci_mcasp_startup,
-	.shutdown	= davinci_mcasp_shutdown,
-	.trigger	= davinci_mcasp_trigger,
-	.hw_params	= davinci_mcasp_hw_params,
-	.set_fmt	= davinci_mcasp_set_dai_fmt,
-	.set_clkdiv	= davinci_mcasp_set_clkdiv,
-	.set_sysclk	= davinci_mcasp_set_sysclk,
-	.set_tdm_slot	= davinci_mcasp_set_tdm_slot,
-};
-
-static int davinci_mcasp_dai_probe(struct snd_soc_dai *dai)
-{
-	struct davinci_mcasp *mcasp = snd_soc_dai_get_drvdata(dai);
-
-	dai->playback_dma_data = &mcasp->dma_data[SNDRV_PCM_STREAM_PLAYBACK];
-	dai->capture_dma_data = &mcasp->dma_data[SNDRV_PCM_STREAM_CAPTURE];
-
-	return 0;
-}
-
-#ifdef CONFIG_PM_SLEEP
-static int davinci_mcasp_suspend(struct snd_soc_dai *dai)
-{
-	struct davinci_mcasp *mcasp = snd_soc_dai_get_drvdata(dai);
-	struct davinci_mcasp_context *context = &mcasp->context;
-	u32 reg;
-	int i;
-
-	context->pm_state = pm_runtime_active(mcasp->dev);
-	if (!context->pm_state)
-		pm_runtime_get_sync(mcasp->dev);
-
-	for (i = 0; i < ARRAY_SIZE(context_regs); i++)
-		context->config_regs[i] = mcasp_get_reg(mcasp, context_regs[i]);
-
-	if (mcasp->txnumevt) {
-		reg = mcasp->fifo_base + MCASP_WFIFOCTL_OFFSET;
-		context->afifo_regs[0] = mcasp_get_reg(mcasp, reg);
-	}
-	if (mcasp->rxnumevt) {
-		reg = mcasp->fifo_base + MCASP_RFIFOCTL_OFFSET;
-		context->afifo_regs[1] = mcasp_get_reg(mcasp, reg);
-	}
-
-	for (i = 0; i < mcasp->num_serializer; i++)
-		context->xrsr_regs[i] = mcasp_get_reg(mcasp,
-						DAVINCI_MCASP_XRSRCTL_REG(i));
-
-	pm_runtime_put_sync(mcasp->dev);
-
-	return 0;
-}
-
-static int davinci_mcasp_resume(struct snd_soc_dai *dai)
-{
-	struct davinci_mcasp *mcasp = snd_soc_dai_get_drvdata(dai);
-	struct davinci_mcasp_context *context = &mcasp->context;
-	u32 reg;
-	int i;
-
-	pm_runtime_get_sync(mcasp->dev);
-
-	for (i = 0; i < ARRAY_SIZE(context_regs); i++)
-		mcasp_set_reg(mcasp, context_regs[i], context->config_regs[i]);
-
-	if (mcasp->txnumevt) {
-		reg = mcasp->fifo_base + MCASP_WFIFOCTL_OFFSET;
-		mcasp_set_reg(mcasp, reg, context->afifo_regs[0]);
-	}
-	if (mcasp->rxnumevt) {
-		reg = mcasp->fifo_base + MCASP_RFIFOCTL_OFFSET;
-		mcasp_set_reg(mcasp, reg, context->afifo_regs[1]);
-	}
-
-	for (i = 0; i < mcasp->num_serializer; i++)
-		mcasp_set_reg(mcasp, DAVINCI_MCASP_XRSRCTL_REG(i),
-			      context->xrsr_regs[i]);
-
-	if (!context->pm_state)
-		pm_runtime_put_sync(mcasp->dev);
-
-	return 0;
-}
-#else
-#define davinci_mcasp_suspend NULL
-#define davinci_mcasp_resume NULL
-#endif
-
-#define DAVINCI_MCASP_RATES	SNDRV_PCM_RATE_8000_192000
-
-#define DAVINCI_MCASP_PCM_FMTS (SNDRV_PCM_FMTBIT_S8 | \
-				SNDRV_PCM_FMTBIT_U8 | \
-				SNDRV_PCM_FMTBIT_S16_LE | \
-				SNDRV_PCM_FMTBIT_U16_LE | \
-				SNDRV_PCM_FMTBIT_S24_LE | \
-				SNDRV_PCM_FMTBIT_U24_LE | \
-				SNDRV_PCM_FMTBIT_S24_3LE | \
-				SNDRV_PCM_FMTBIT_U24_3LE | \
-				SNDRV_PCM_FMTBIT_S32_LE | \
-				SNDRV_PCM_FMTBIT_U32_LE)
-
-static struct snd_soc_dai_driver davinci_mcasp_dai[] = {
-	{
-		.name		= "davinci-mcasp.0",
-		.probe		= davinci_mcasp_dai_probe,
-		.suspend	= davinci_mcasp_suspend,
-		.resume		= davinci_mcasp_resume,
-		.playback	= {
-			.channels_min	= 1,
-			.channels_max	= 32 * 16,
-			.rates 		= DAVINCI_MCASP_RATES,
-			.formats	= DAVINCI_MCASP_PCM_FMTS,
-		},
-		.capture 	= {
-			.channels_min 	= 1,
-			.channels_max	= 32 * 16,
-			.rates 		= DAVINCI_MCASP_RATES,
-			.formats	= DAVINCI_MCASP_PCM_FMTS,
-		},
-		.ops 		= &davinci_mcasp_dai_ops,
-
-		.symmetric_samplebits	= 1,
-		.symmetric_rates	= 1,
-	},
-	{
-		.name		= "davinci-mcasp.1",
-		.probe		= davinci_mcasp_dai_probe,
-		.playback 	= {
-			.channels_min	= 1,
-			.channels_max	= 384,
-			.rates		= DAVINCI_MCASP_RATES,
-			.formats	= DAVINCI_MCASP_PCM_FMTS,
-		},
-		.ops 		= &davinci_mcasp_dai_ops,
-	},
-
-};
-
-static const struct snd_soc_component_driver davinci_mcasp_component = {
-	.name		= "davinci-mcasp",
-};
-
-/* Some HW specific values and defaults. The rest is filled in from DT. */
-static struct davinci_mcasp_pdata dm646x_mcasp_pdata = {
-	.tx_dma_offset = 0x400,
-	.rx_dma_offset = 0x400,
-	.version = MCASP_VERSION_1,
-};
-
-static struct davinci_mcasp_pdata da830_mcasp_pdata = {
-	.tx_dma_offset = 0x2000,
-	.rx_dma_offset = 0x2000,
-	.version = MCASP_VERSION_2,
-};
-
-static struct davinci_mcasp_pdata am33xx_mcasp_pdata = {
-	.tx_dma_offset = 0,
-	.rx_dma_offset = 0,
-	.version = MCASP_VERSION_3,
-};
-
-static struct davinci_mcasp_pdata dra7_mcasp_pdata = {
-	/* The CFG port offset will be calculated if it is needed */
-	.tx_dma_offset = 0,
-	.rx_dma_offset = 0,
-	.version = MCASP_VERSION_4,
-};
-
-static const struct of_device_id mcasp_dt_ids[] = {
-	{
-		.compatible = "ti,dm646x-mcasp-audio",
-		.data = &dm646x_mcasp_pdata,
-	},
-	{
-		.compatible = "ti,da830-mcasp-audio",
-		.data = &da830_mcasp_pdata,
-	},
-	{
-		.compatible = "ti,am33xx-mcasp-audio",
-		.data = &am33xx_mcasp_pdata,
-	},
-	{
-		.compatible = "ti,dra7-mcasp-audio",
-		.data = &dra7_mcasp_pdata,
-	},
-	{ /* sentinel */ }
-};
-MODULE_DEVICE_TABLE(of, mcasp_dt_ids);
-
-static int mcasp_reparent_fck(struct platform_device *pdev)
-{
-	struct device_node *node = pdev->dev.of_node;
-	struct clk *gfclk, *parent_clk;
-	const char *parent_name;
-	int ret;
-
-	if (!node)
-		return 0;
-
-	parent_name = of_get_property(node, "fck_parent", NULL);
-	if (!parent_name)
-		return 0;
-
-	dev_warn(&pdev->dev, "Update the bindings to use assigned-clocks!\n");
-
-	gfclk = clk_get(&pdev->dev, "fck");
-	if (IS_ERR(gfclk)) {
-		dev_err(&pdev->dev, "failed to get fck\n");
-		return PTR_ERR(gfclk);
-	}
-
-	parent_clk = clk_get(NULL, parent_name);
-	if (IS_ERR(parent_clk)) {
-		dev_err(&pdev->dev, "failed to get parent clock\n");
-		ret = PTR_ERR(parent_clk);
-		goto err1;
-	}
-
-	ret = clk_set_parent(gfclk, parent_clk);
-	if (ret) {
-		dev_err(&pdev->dev, "failed to reparent fck\n");
-		goto err2;
-	}
-
-err2:
-	clk_put(parent_clk);
-err1:
-	clk_put(gfclk);
-	return ret;
-}
-
-static struct davinci_mcasp_pdata *davinci_mcasp_set_pdata_from_of(
-						struct platform_device *pdev)
-{
-	struct device_node *np = pdev->dev.of_node;
-	struct davinci_mcasp_pdata *pdata = NULL;
-	const struct of_device_id *match =
-			of_match_device(mcasp_dt_ids, &pdev->dev);
-	struct of_phandle_args dma_spec;
-
-	const u32 *of_serial_dir32;
-	u32 val;
-	int i, ret = 0;
-
-	if (pdev->dev.platform_data) {
-		pdata = pdev->dev.platform_data;
-		return pdata;
-	} else if (match) {
-		pdata = devm_kmemdup(&pdev->dev, match->data, sizeof(*pdata),
-				     GFP_KERNEL);
-		if (!pdata) {
-			ret = -ENOMEM;
-			return pdata;
-		}
-	} else {
-		/* control shouldn't reach here. something is wrong */
-		ret = -EINVAL;
-		goto nodata;
-	}
-
-	ret = of_property_read_u32(np, "op-mode", &val);
-	if (ret >= 0)
-		pdata->op_mode = val;
-
-	ret = of_property_read_u32(np, "tdm-slots", &val);
-	if (ret >= 0) {
-		if (val < 2 || val > 32) {
-			dev_err(&pdev->dev,
-				"tdm-slots must be in rage [2-32]\n");
-			ret = -EINVAL;
-			goto nodata;
-		}
-
-		pdata->tdm_slots = val;
-	}
-
-	of_serial_dir32 = of_get_property(np, "serial-dir", &val);
-	val /= sizeof(u32);
-	if (of_serial_dir32) {
-		u8 *of_serial_dir = devm_kzalloc(&pdev->dev,
-						 (sizeof(*of_serial_dir) * val),
-						 GFP_KERNEL);
-		if (!of_serial_dir) {
-			ret = -ENOMEM;
-			goto nodata;
-		}
-
-		for (i = 0; i < val; i++)
-			of_serial_dir[i] = be32_to_cpup(&of_serial_dir32[i]);
-
-		pdata->num_serializer = val;
-		pdata->serial_dir = of_serial_dir;
-	}
-
-	ret = of_property_match_string(np, "dma-names", "tx");
-	if (ret < 0)
-		goto nodata;
-
-	ret = of_parse_phandle_with_args(np, "dmas", "#dma-cells", ret,
-					 &dma_spec);
-	if (ret < 0)
-		goto nodata;
-
-	pdata->tx_dma_channel = dma_spec.args[0];
-
-	/* RX is not valid in DIT mode */
-	if (pdata->op_mode != DAVINCI_MCASP_DIT_MODE) {
-		ret = of_property_match_string(np, "dma-names", "rx");
-		if (ret < 0)
-			goto nodata;
-
-		ret = of_parse_phandle_with_args(np, "dmas", "#dma-cells", ret,
-						 &dma_spec);
-		if (ret < 0)
-			goto nodata;
-
-		pdata->rx_dma_channel = dma_spec.args[0];
-	}
-
-	ret = of_property_read_u32(np, "tx-num-evt", &val);
-	if (ret >= 0)
-		pdata->txnumevt = val;
-
-	ret = of_property_read_u32(np, "rx-num-evt", &val);
-	if (ret >= 0)
-		pdata->rxnumevt = val;
-
-	ret = of_property_read_u32(np, "sram-size-playback", &val);
-	if (ret >= 0)
-		pdata->sram_size_playback = val;
-
-	ret = of_property_read_u32(np, "sram-size-capture", &val);
-	if (ret >= 0)
-		pdata->sram_size_capture = val;
-
-	return  pdata;
-
-nodata:
-	if (ret < 0) {
-		dev_err(&pdev->dev, "Error populating platform data, err %d\n",
-			ret);
-		pdata = NULL;
-	}
-	return  pdata;
-}
-
-enum {
-	PCM_EDMA,
-	PCM_SDMA,
-};
-static const char *sdma_prefix = "ti,omap";
-
-static int davinci_mcasp_get_dma_type(struct davinci_mcasp *mcasp)
-{
-	struct dma_chan *chan;
-	const char *tmp;
-	int ret = PCM_EDMA;
-
-	if (!mcasp->dev->of_node)
-		return PCM_EDMA;
-
-	tmp = mcasp->dma_data[SNDRV_PCM_STREAM_PLAYBACK].filter_data;
-	chan = dma_request_slave_channel_reason(mcasp->dev, tmp);
-	if (IS_ERR(chan)) {
-		if (PTR_ERR(chan) != -EPROBE_DEFER)
-			dev_err(mcasp->dev,
-				"Can't verify DMA configuration (%ld)\n",
-				PTR_ERR(chan));
-		return PTR_ERR(chan);
-	}
-	if (WARN_ON(!chan->device || !chan->device->dev))
-		return -EINVAL;
-
-	if (chan->device->dev->of_node)
-		ret = of_property_read_string(chan->device->dev->of_node,
-					      "compatible", &tmp);
-	else
-		dev_dbg(mcasp->dev, "DMA controller has no of-node\n");
-
-	dma_release_channel(chan);
-	if (ret)
-		return ret;
-
-	dev_dbg(mcasp->dev, "DMA controller compatible = \"%s\"\n", tmp);
-	if (!strncmp(tmp, sdma_prefix, strlen(sdma_prefix)))
-		return PCM_SDMA;
-
-	return PCM_EDMA;
-}
-
-static u32 davinci_mcasp_txdma_offset(struct davinci_mcasp_pdata *pdata)
-{
-	int i;
-	u32 offset = 0;
-
-	if (pdata->version != MCASP_VERSION_4)
-		return pdata->tx_dma_offset;
-
-	for (i = 0; i < pdata->num_serializer; i++) {
-		if (pdata->serial_dir[i] == TX_MODE) {
-			if (!offset) {
-				offset = DAVINCI_MCASP_TXBUF_REG(i);
-			} else {
-				pr_err("%s: Only one serializer allowed!\n",
-				       __func__);
-				break;
-			}
-		}
-	}
-
-	return offset;
-}
-
-static u32 davinci_mcasp_rxdma_offset(struct davinci_mcasp_pdata *pdata)
-{
-	int i;
-	u32 offset = 0;
-
-	if (pdata->version != MCASP_VERSION_4)
-		return pdata->rx_dma_offset;
-
-	for (i = 0; i < pdata->num_serializer; i++) {
-		if (pdata->serial_dir[i] == RX_MODE) {
-			if (!offset) {
-				offset = DAVINCI_MCASP_RXBUF_REG(i);
-			} else {
-				pr_err("%s: Only one serializer allowed!\n",
-				       __func__);
-				break;
-			}
-		}
-	}
-
-	return offset;
-}
-
-static int davinci_mcasp_probe(struct platform_device *pdev)
-{
-	struct snd_dmaengine_dai_dma_data *dma_data;
-	struct resource *mem, *res, *dat;
-	struct davinci_mcasp_pdata *pdata;
-	struct davinci_mcasp *mcasp;
-	char *irq_name;
-	int *dma;
-	int irq;
-	int ret;
-
-	if (!pdev->dev.platform_data && !pdev->dev.of_node) {
-		dev_err(&pdev->dev, "No platform data supplied\n");
-		return -EINVAL;
-	}
-
-	mcasp = devm_kzalloc(&pdev->dev, sizeof(struct davinci_mcasp),
-			   GFP_KERNEL);
-	if (!mcasp)
-		return	-ENOMEM;
-
-	pdata = davinci_mcasp_set_pdata_from_of(pdev);
-	if (!pdata) {
-		dev_err(&pdev->dev, "no platform data\n");
-		return -EINVAL;
-	}
-
-	mem = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mpu");
-	if (!mem) {
-		dev_warn(mcasp->dev,
-			 "\"mpu\" mem resource not found, using index 0\n");
-		mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-		if (!mem) {
-			dev_err(&pdev->dev, "no mem resource?\n");
-			return -ENODEV;
-		}
-	}
-
-	mcasp->base = devm_ioremap_resource(&pdev->dev, mem);
-	if (IS_ERR(mcasp->base))
-		return PTR_ERR(mcasp->base);
-
-	pm_runtime_enable(&pdev->dev);
-
-	mcasp->op_mode = pdata->op_mode;
-	/* sanity check for tdm slots parameter */
-	if (mcasp->op_mode == DAVINCI_MCASP_IIS_MODE) {
-		if (pdata->tdm_slots < 2) {
-			dev_err(&pdev->dev, "invalid tdm slots: %d\n",
-				pdata->tdm_slots);
-			mcasp->tdm_slots = 2;
-		} else if (pdata->tdm_slots > 32) {
-			dev_err(&pdev->dev, "invalid tdm slots: %d\n",
-				pdata->tdm_slots);
-			mcasp->tdm_slots = 32;
-		} else {
-			mcasp->tdm_slots = pdata->tdm_slots;
-		}
-	}
-
-	mcasp->num_serializer = pdata->num_serializer;
-#ifdef CONFIG_PM_SLEEP
-	mcasp->context.xrsr_regs = devm_kcalloc(&pdev->dev,
-					mcasp->num_serializer, sizeof(u32),
-					GFP_KERNEL);
-	if (!mcasp->context.xrsr_regs) {
-		ret = -ENOMEM;
-		goto err;
-	}
-#endif
-	mcasp->serial_dir = pdata->serial_dir;
-	mcasp->version = pdata->version;
-	mcasp->txnumevt = pdata->txnumevt;
-	mcasp->rxnumevt = pdata->rxnumevt;
-
-	mcasp->dev = &pdev->dev;
-
-	irq = platform_get_irq_byname(pdev, "common");
-	if (irq >= 0) {
-		irq_name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s_common",
-					  dev_name(&pdev->dev));
-		if (!irq_name) {
-			ret = -ENOMEM;
-			goto err;
-		}
-		ret = devm_request_threaded_irq(&pdev->dev, irq, NULL,
-						davinci_mcasp_common_irq_handler,
-						IRQF_ONESHOT | IRQF_SHARED,
-						irq_name, mcasp);
-		if (ret) {
-			dev_err(&pdev->dev, "common IRQ request failed\n");
-			goto err;
-		}
-
-		mcasp->irq_request[SNDRV_PCM_STREAM_PLAYBACK] = XUNDRN;
-		mcasp->irq_request[SNDRV_PCM_STREAM_CAPTURE] = ROVRN;
-	}
-
-	irq = platform_get_irq_byname(pdev, "rx");
-	if (irq >= 0) {
-		irq_name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s_rx",
-					  dev_name(&pdev->dev));
-		if (!irq_name) {
-			ret = -ENOMEM;
-			goto err;
-		}
-		ret = devm_request_threaded_irq(&pdev->dev, irq, NULL,
-						davinci_mcasp_rx_irq_handler,
-						IRQF_ONESHOT, irq_name, mcasp);
-		if (ret) {
-			dev_err(&pdev->dev, "RX IRQ request failed\n");
-			goto err;
-		}
-
-		mcasp->irq_request[SNDRV_PCM_STREAM_CAPTURE] = ROVRN;
-	}
-
-	irq = platform_get_irq_byname(pdev, "tx");
-	if (irq >= 0) {
-		irq_name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s_tx",
-					  dev_name(&pdev->dev));
-		if (!irq_name) {
-			ret = -ENOMEM;
-			goto err;
-		}
-		ret = devm_request_threaded_irq(&pdev->dev, irq, NULL,
-						davinci_mcasp_tx_irq_handler,
-						IRQF_ONESHOT, irq_name, mcasp);
-		if (ret) {
-			dev_err(&pdev->dev, "TX IRQ request failed\n");
-			goto err;
-		}
-
-		mcasp->irq_request[SNDRV_PCM_STREAM_PLAYBACK] = XUNDRN;
-	}
-
-	dat = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dat");
-	if (dat)
-		mcasp->dat_port = true;
-
-	dma_data = &mcasp->dma_data[SNDRV_PCM_STREAM_PLAYBACK];
-	if (dat)
-		dma_data->addr = dat->start;
-	else
-		dma_data->addr = mem->start + davinci_mcasp_txdma_offset(pdata);
-
-	dma = &mcasp->dma_request[SNDRV_PCM_STREAM_PLAYBACK];
-	res = platform_get_resource(pdev, IORESOURCE_DMA, 0);
-	if (res)
-		*dma = res->start;
-	else
-		*dma = pdata->tx_dma_channel;
-
-	/* dmaengine filter data for DT and non-DT boot */
-	if (pdev->dev.of_node)
-		dma_data->filter_data = "tx";
-	else
-		dma_data->filter_data = dma;
-
-	/* RX is not valid in DIT mode */
-	if (mcasp->op_mode != DAVINCI_MCASP_DIT_MODE) {
-		dma_data = &mcasp->dma_data[SNDRV_PCM_STREAM_CAPTURE];
-		if (dat)
-			dma_data->addr = dat->start;
-		else
-			dma_data->addr =
-				mem->start + davinci_mcasp_rxdma_offset(pdata);
-
-		dma = &mcasp->dma_request[SNDRV_PCM_STREAM_CAPTURE];
-		res = platform_get_resource(pdev, IORESOURCE_DMA, 1);
-		if (res)
-			*dma = res->start;
-		else
-			*dma = pdata->rx_dma_channel;
-
-		/* dmaengine filter data for DT and non-DT boot */
-		if (pdev->dev.of_node)
-			dma_data->filter_data = "rx";
-		else
-			dma_data->filter_data = dma;
-	}
-
-	if (mcasp->version < MCASP_VERSION_3) {
-		mcasp->fifo_base = DAVINCI_MCASP_V2_AFIFO_BASE;
-		/* dma_params->dma_addr is pointing to the data port address */
-		mcasp->dat_port = true;
-	} else {
-		mcasp->fifo_base = DAVINCI_MCASP_V3_AFIFO_BASE;
-	}
-
-	/* Allocate memory for long enough list for all possible
-	 * scenarios. Maximum number tdm slots is 32 and there cannot
-	 * be more serializers than given in the configuration.  The
-	 * serializer directions could be taken into account, but it
-	 * would make code much more complex and save only couple of
-	 * bytes.
-	 */
-	mcasp->chconstr[SNDRV_PCM_STREAM_PLAYBACK].list =
-		devm_kcalloc(mcasp->dev,
-			     32 + mcasp->num_serializer - 1,
-			     sizeof(unsigned int),
-			     GFP_KERNEL);
-
-	mcasp->chconstr[SNDRV_PCM_STREAM_CAPTURE].list =
-		devm_kcalloc(mcasp->dev,
-			     32 + mcasp->num_serializer - 1,
-			     sizeof(unsigned int),
-			     GFP_KERNEL);
-
-	if (!mcasp->chconstr[SNDRV_PCM_STREAM_PLAYBACK].list ||
-	    !mcasp->chconstr[SNDRV_PCM_STREAM_CAPTURE].list) {
-		ret = -ENOMEM;
-		goto err;
-	}
-
-	ret = davinci_mcasp_set_ch_constraints(mcasp);
-	if (ret)
-		goto err;
-
-	dev_set_drvdata(&pdev->dev, mcasp);
-
-	mcasp_reparent_fck(pdev);
-
-	ret = devm_snd_soc_register_component(&pdev->dev,
-					&davinci_mcasp_component,
-					&davinci_mcasp_dai[pdata->op_mode], 1);
-
-	if (ret != 0)
-		goto err;
-
-	ret = davinci_mcasp_get_dma_type(mcasp);
-	switch (ret) {
-	case PCM_EDMA:
-#if IS_BUILTIN(CONFIG_SND_EDMA_SOC) || \
-	(IS_MODULE(CONFIG_SND_DAVINCI_SOC_MCASP) && \
-	 IS_MODULE(CONFIG_SND_EDMA_SOC))
-		ret = edma_pcm_platform_register(&pdev->dev);
-#else
-		dev_err(&pdev->dev, "Missing SND_EDMA_SOC\n");
-		ret = -EINVAL;
-		goto err;
-#endif
-		break;
-	case PCM_SDMA:
-#if IS_BUILTIN(CONFIG_SND_SDMA_SOC) || \
-	(IS_MODULE(CONFIG_SND_DAVINCI_SOC_MCASP) && \
-	 IS_MODULE(CONFIG_SND_SDMA_SOC))
-		ret = sdma_pcm_platform_register(&pdev->dev, NULL, NULL);
-#else
-		dev_err(&pdev->dev, "Missing SND_SDMA_SOC\n");
-		ret = -EINVAL;
-		goto err;
-#endif
-		break;
-	default:
-		dev_err(&pdev->dev, "No DMA controller found (%d)\n", ret);
-	case -EPROBE_DEFER:
-		goto err;
-		break;
-	}
-
-	if (ret) {
-		dev_err(&pdev->dev, "register PCM failed: %d\n", ret);
-		goto err;
-	}
-
-	return 0;
-
-err:
-	pm_runtime_disable(&pdev->dev);
-	return ret;
-}
-
-static int davinci_mcasp_remove(struct platform_device *pdev)
-{
-	pm_runtime_disable(&pdev->dev);
-
-	return 0;
-}
-
-static struct platform_driver davinci_mcasp_driver = {
-	.probe		= davinci_mcasp_probe,
-	.remove		= davinci_mcasp_remove,
-	.driver		= {
-		.name	= "davinci-mcasp",
-		.of_match_table = mcasp_dt_ids,
-	},
-};
-
-module_platform_driver(davinci_mcasp_driver);
-
-MODULE_AUTHOR("Steve Chen");
-MODULE_DESCRIPTION("TI DAVINCI McASP SoC Interface");
-MODULE_LICENSE("GPL");
diff -Naur --no-dereference a/sound/soc/davinci/davinci-mcasp.h b/sound/soc/davinci/davinci-mcasp.h
--- a/sound/soc/davinci/davinci-mcasp.h	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/davinci/davinci-mcasp.h	1969-12-31 19:00:00.000000000 -0500
@@ -1,314 +0,0 @@
-/*
- * ALSA SoC McASP Audio Layer for TI DAVINCI processor
- *
- * MCASP related definitions
- *
- * Author: Nirmal Pandey <n-pandey@ti.com>,
- *         Suresh Rajashekara <suresh.r@ti.com>
- *         Steve Chen <schen@.mvista.com>
- *
- * Copyright:   (C) 2009 MontaVista Software, Inc., <source@mvista.com>
- * Copyright:   (C) 2009  Texas Instruments, India
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#ifndef DAVINCI_MCASP_H
-#define DAVINCI_MCASP_H
-
-/*
- * McASP register definitions
- */
-#define DAVINCI_MCASP_PID_REG		0x00
-#define DAVINCI_MCASP_PWREMUMGT_REG	0x04
-
-#define DAVINCI_MCASP_PFUNC_REG		0x10
-#define DAVINCI_MCASP_PDIR_REG		0x14
-#define DAVINCI_MCASP_PDOUT_REG		0x18
-#define DAVINCI_MCASP_PDSET_REG		0x1c
-
-#define DAVINCI_MCASP_PDCLR_REG		0x20
-
-#define DAVINCI_MCASP_TLGC_REG		0x30
-#define DAVINCI_MCASP_TLMR_REG		0x34
-
-#define DAVINCI_MCASP_GBLCTL_REG	0x44
-#define DAVINCI_MCASP_AMUTE_REG		0x48
-#define DAVINCI_MCASP_LBCTL_REG		0x4c
-
-#define DAVINCI_MCASP_TXDITCTL_REG	0x50
-
-#define DAVINCI_MCASP_GBLCTLR_REG	0x60
-#define DAVINCI_MCASP_RXMASK_REG	0x64
-#define DAVINCI_MCASP_RXFMT_REG		0x68
-#define DAVINCI_MCASP_RXFMCTL_REG	0x6c
-
-#define DAVINCI_MCASP_ACLKRCTL_REG	0x70
-#define DAVINCI_MCASP_AHCLKRCTL_REG	0x74
-#define DAVINCI_MCASP_RXTDM_REG		0x78
-#define DAVINCI_MCASP_EVTCTLR_REG	0x7c
-
-#define DAVINCI_MCASP_RXSTAT_REG	0x80
-#define DAVINCI_MCASP_RXTDMSLOT_REG	0x84
-#define DAVINCI_MCASP_RXCLKCHK_REG	0x88
-#define DAVINCI_MCASP_REVTCTL_REG	0x8c
-
-#define DAVINCI_MCASP_GBLCTLX_REG	0xa0
-#define DAVINCI_MCASP_TXMASK_REG	0xa4
-#define DAVINCI_MCASP_TXFMT_REG		0xa8
-#define DAVINCI_MCASP_TXFMCTL_REG	0xac
-
-#define DAVINCI_MCASP_ACLKXCTL_REG	0xb0
-#define DAVINCI_MCASP_AHCLKXCTL_REG	0xb4
-#define DAVINCI_MCASP_TXTDM_REG		0xb8
-#define DAVINCI_MCASP_EVTCTLX_REG	0xbc
-
-#define DAVINCI_MCASP_TXSTAT_REG	0xc0
-#define DAVINCI_MCASP_TXTDMSLOT_REG	0xc4
-#define DAVINCI_MCASP_TXCLKCHK_REG	0xc8
-#define DAVINCI_MCASP_XEVTCTL_REG	0xcc
-
-/* Left(even TDM Slot) Channel Status Register File */
-#define DAVINCI_MCASP_DITCSRA_REG	0x100
-/* Right(odd TDM slot) Channel Status Register File */
-#define DAVINCI_MCASP_DITCSRB_REG	0x118
-/* Left(even TDM slot) User Data Register File */
-#define DAVINCI_MCASP_DITUDRA_REG	0x130
-/* Right(odd TDM Slot) User Data Register File */
-#define DAVINCI_MCASP_DITUDRB_REG	0x148
-
-/* Serializer n Control Register */
-#define DAVINCI_MCASP_XRSRCTL_BASE_REG	0x180
-#define DAVINCI_MCASP_XRSRCTL_REG(n)	(DAVINCI_MCASP_XRSRCTL_BASE_REG + \
-						(n << 2))
-
-/* Transmit Buffer for Serializer n */
-#define DAVINCI_MCASP_TXBUF_REG(n)	(0x200 + (n << 2))
-/* Receive Buffer for Serializer n */
-#define DAVINCI_MCASP_RXBUF_REG(n)	(0x280 + (n << 2))
-
-/* McASP FIFO Registers */
-#define DAVINCI_MCASP_V2_AFIFO_BASE	(0x1010)
-#define DAVINCI_MCASP_V3_AFIFO_BASE	(0x1000)
-
-/* FIFO register offsets from AFIFO base */
-#define MCASP_WFIFOCTL_OFFSET		(0x0)
-#define MCASP_WFIFOSTS_OFFSET		(0x4)
-#define MCASP_RFIFOCTL_OFFSET		(0x8)
-#define MCASP_RFIFOSTS_OFFSET		(0xc)
-
-/*
- * DAVINCI_MCASP_PWREMUMGT_REG - Power Down and Emulation Management
- *     Register Bits
- */
-#define MCASP_FREE	BIT(0)
-#define MCASP_SOFT	BIT(1)
-
-/*
- * DAVINCI_MCASP_PFUNC_REG - Pin Function / GPIO Enable Register Bits
- */
-#define AXR(n)		(1<<n)
-#define PFUNC_AMUTE	BIT(25)
-#define ACLKX		BIT(26)
-#define AHCLKX		BIT(27)
-#define AFSX		BIT(28)
-#define ACLKR		BIT(29)
-#define AHCLKR		BIT(30)
-#define AFSR		BIT(31)
-
-/*
- * DAVINCI_MCASP_PDIR_REG - Pin Direction Register Bits
- */
-#define AXR(n)		(1<<n)
-#define PDIR_AMUTE	BIT(25)
-#define ACLKX		BIT(26)
-#define AHCLKX		BIT(27)
-#define AFSX		BIT(28)
-#define ACLKR		BIT(29)
-#define AHCLKR		BIT(30)
-#define AFSR		BIT(31)
-
-/*
- * DAVINCI_MCASP_TXDITCTL_REG - Transmit DIT Control Register Bits
- */
-#define DITEN	BIT(0)	/* Transmit DIT mode enable/disable */
-#define VA	BIT(2)
-#define VB	BIT(3)
-
-/*
- * DAVINCI_MCASP_TXFMT_REG - Transmit Bitstream Format Register Bits
- */
-#define TXROT(val)	(val)
-#define TXSEL		BIT(3)
-#define TXSSZ(val)	(val<<4)
-#define TXPBIT(val)	(val<<8)
-#define TXPAD(val)	(val<<13)
-#define TXORD		BIT(15)
-#define FSXDLY(val)	(val<<16)
-
-/*
- * DAVINCI_MCASP_RXFMT_REG - Receive Bitstream Format Register Bits
- */
-#define RXROT(val)	(val)
-#define RXSEL		BIT(3)
-#define RXSSZ(val)	(val<<4)
-#define RXPBIT(val)	(val<<8)
-#define RXPAD(val)	(val<<13)
-#define RXORD		BIT(15)
-#define FSRDLY(val)	(val<<16)
-
-/*
- * DAVINCI_MCASP_TXFMCTL_REG -  Transmit Frame Control Register Bits
- */
-#define FSXPOL		BIT(0)
-#define AFSXE		BIT(1)
-#define FSXDUR		BIT(4)
-#define FSXMOD(val)	(val<<7)
-
-/*
- * DAVINCI_MCASP_RXFMCTL_REG - Receive Frame Control Register Bits
- */
-#define FSRPOL		BIT(0)
-#define AFSRE		BIT(1)
-#define FSRDUR		BIT(4)
-#define FSRMOD(val)	(val<<7)
-
-/*
- * DAVINCI_MCASP_ACLKXCTL_REG - Transmit Clock Control Register Bits
- */
-#define ACLKXDIV(val)	(val)
-#define ACLKXE		BIT(5)
-#define TX_ASYNC	BIT(6)
-#define ACLKXPOL	BIT(7)
-#define ACLKXDIV_MASK	0x1f
-
-/*
- * DAVINCI_MCASP_ACLKRCTL_REG Receive Clock Control Register Bits
- */
-#define ACLKRDIV(val)	(val)
-#define ACLKRE		BIT(5)
-#define RX_ASYNC	BIT(6)
-#define ACLKRPOL	BIT(7)
-#define ACLKRDIV_MASK	0x1f
-
-/*
- * DAVINCI_MCASP_AHCLKXCTL_REG - High Frequency Transmit Clock Control
- *     Register Bits
- */
-#define AHCLKXDIV(val)	(val)
-#define AHCLKXPOL	BIT(14)
-#define AHCLKXE		BIT(15)
-#define AHCLKXDIV_MASK	0xfff
-
-/*
- * DAVINCI_MCASP_AHCLKRCTL_REG - High Frequency Receive Clock Control
- *     Register Bits
- */
-#define AHCLKRDIV(val)	(val)
-#define AHCLKRPOL	BIT(14)
-#define AHCLKRE		BIT(15)
-#define AHCLKRDIV_MASK	0xfff
-
-/*
- * DAVINCI_MCASP_XRSRCTL_BASE_REG -  Serializer Control Register Bits
- */
-#define MODE(val)	(val)
-#define DISMOD_3STATE	(0x0)
-#define DISMOD_LOW	(0x2 << 2)
-#define DISMOD_HIGH	(0x3 << 2)
-#define DISMOD_MASK	DISMOD_HIGH
-#define TXSTATE		BIT(4)
-#define RXSTATE		BIT(5)
-#define SRMOD_MASK	3
-#define SRMOD_INACTIVE	0
-
-/*
- * DAVINCI_MCASP_LBCTL_REG - Loop Back Control Register Bits
- */
-#define LBEN		BIT(0)
-#define LBORD		BIT(1)
-#define LBGENMODE(val)	(val<<2)
-
-/*
- * DAVINCI_MCASP_TXTDMSLOT_REG - Transmit TDM Slot Register configuration
- */
-#define TXTDMS(n)	(1<<n)
-
-/*
- * DAVINCI_MCASP_RXTDMSLOT_REG - Receive TDM Slot Register configuration
- */
-#define RXTDMS(n)	(1<<n)
-
-/*
- * DAVINCI_MCASP_GBLCTL_REG -  Global Control Register Bits
- */
-#define RXCLKRST	BIT(0)	/* Receiver Clock Divider Reset */
-#define RXHCLKRST	BIT(1)	/* Receiver High Frequency Clock Divider */
-#define RXSERCLR	BIT(2)	/* Receiver Serializer Clear */
-#define RXSMRST		BIT(3)	/* Receiver State Machine Reset */
-#define RXFSRST		BIT(4)	/* Frame Sync Generator Reset */
-#define TXCLKRST	BIT(8)	/* Transmitter Clock Divider Reset */
-#define TXHCLKRST	BIT(9)	/* Transmitter High Frequency Clock Divider*/
-#define TXSERCLR	BIT(10)	/* Transmit Serializer Clear */
-#define TXSMRST		BIT(11)	/* Transmitter State Machine Reset */
-#define TXFSRST		BIT(12)	/* Frame Sync Generator Reset */
-
-/*
- * DAVINCI_MCASP_TXSTAT_REG - Transmitter Status Register Bits
- * DAVINCI_MCASP_RXSTAT_REG - Receiver Status Register Bits
- */
-#define XRERR		BIT(8) /* Transmit/Receive error */
-#define XRDATA		BIT(5) /* Transmit/Receive data ready */
-
-/*
- * DAVINCI_MCASP_AMUTE_REG -  Mute Control Register Bits
- */
-#define MUTENA(val)	(val)
-#define MUTEINPOL	BIT(2)
-#define MUTEINENA	BIT(3)
-#define MUTEIN		BIT(4)
-#define MUTER		BIT(5)
-#define MUTEX		BIT(6)
-#define MUTEFSR		BIT(7)
-#define MUTEFSX		BIT(8)
-#define MUTEBADCLKR	BIT(9)
-#define MUTEBADCLKX	BIT(10)
-#define MUTERXDMAERR	BIT(11)
-#define MUTETXDMAERR	BIT(12)
-
-/*
- * DAVINCI_MCASP_REVTCTL_REG - Receiver DMA Event Control Register bits
- */
-#define RXDATADMADIS	BIT(0)
-
-/*
- * DAVINCI_MCASP_XEVTCTL_REG - Transmitter DMA Event Control Register bits
- */
-#define TXDATADMADIS	BIT(0)
-
-/*
- * DAVINCI_MCASP_EVTCTLR_REG - Receiver Interrupt Control Register Bits
- */
-#define ROVRN		BIT(0)
-
-/*
- * DAVINCI_MCASP_EVTCTLX_REG - Transmitter Interrupt Control Register Bits
- */
-#define XUNDRN		BIT(0)
-
-/*
- * DAVINCI_MCASP_W[R]FIFOCTL - Write/Read FIFO Control Register bits
- */
-#define FIFO_ENABLE	BIT(16)
-#define NUMEVT_MASK	(0xFF << 8)
-#define NUMEVT(x)	(((x) & 0xFF) << 8)
-#define NUMDMA_MASK	(0xFF)
-
-/* clock divider IDs */
-#define MCASP_CLKDIV_AUXCLK		0 /* HCLK divider from AUXCLK */
-#define MCASP_CLKDIV_BCLK		1 /* BCLK divider from HCLK */
-#define MCASP_CLKDIV_BCLK_FS_RATIO	2 /* to set BCLK FS ration */
-
-#endif	/* DAVINCI_MCASP_H */
diff -Naur --no-dereference a/sound/soc/davinci/davinci-vcif.c b/sound/soc/davinci/davinci-vcif.c
--- a/sound/soc/davinci/davinci-vcif.c	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/davinci/davinci-vcif.c	1969-12-31 19:00:00.000000000 -0500
@@ -1,259 +0,0 @@
-/*
- * ALSA SoC Voice Codec Interface for TI DAVINCI processor
- *
- * Copyright (C) 2010 Texas Instruments.
- *
- * Author: Miguel Aguilar <miguel.aguilar@ridgerun.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
- */
-
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/device.h>
-#include <linux/delay.h>
-#include <linux/slab.h>
-#include <linux/io.h>
-#include <linux/mfd/davinci_voicecodec.h>
-
-#include <sound/core.h>
-#include <sound/pcm.h>
-#include <sound/pcm_params.h>
-#include <sound/initval.h>
-#include <sound/soc.h>
-#include <sound/dmaengine_pcm.h>
-
-#include "edma-pcm.h"
-#include "davinci-i2s.h"
-
-#define MOD_REG_BIT(val, mask, set) do { \
-	if (set) { \
-		val |= mask; \
-	} else { \
-		val &= ~mask; \
-	} \
-} while (0)
-
-struct davinci_vcif_dev {
-	struct davinci_vc *davinci_vc;
-	struct snd_dmaengine_dai_dma_data dma_data[2];
-	int dma_request[2];
-};
-
-static void davinci_vcif_start(struct snd_pcm_substream *substream)
-{
-	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct davinci_vcif_dev *davinci_vcif_dev =
-			snd_soc_dai_get_drvdata(rtd->cpu_dai);
-	struct davinci_vc *davinci_vc = davinci_vcif_dev->davinci_vc;
-	u32 w;
-
-	/* Start the sample generator and enable transmitter/receiver */
-	w = readl(davinci_vc->base + DAVINCI_VC_CTRL);
-
-	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
-		MOD_REG_BIT(w, DAVINCI_VC_CTRL_RSTDAC, 0);
-	else
-		MOD_REG_BIT(w, DAVINCI_VC_CTRL_RSTADC, 0);
-
-	writel(w, davinci_vc->base + DAVINCI_VC_CTRL);
-}
-
-static void davinci_vcif_stop(struct snd_pcm_substream *substream)
-{
-	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct davinci_vcif_dev *davinci_vcif_dev =
-			snd_soc_dai_get_drvdata(rtd->cpu_dai);
-	struct davinci_vc *davinci_vc = davinci_vcif_dev->davinci_vc;
-	u32 w;
-
-	/* Reset transmitter/receiver and sample rate/frame sync generators */
-	w = readl(davinci_vc->base + DAVINCI_VC_CTRL);
-	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
-		MOD_REG_BIT(w, DAVINCI_VC_CTRL_RSTDAC, 1);
-	else
-		MOD_REG_BIT(w, DAVINCI_VC_CTRL_RSTADC, 1);
-
-	writel(w, davinci_vc->base + DAVINCI_VC_CTRL);
-}
-
-static int davinci_vcif_hw_params(struct snd_pcm_substream *substream,
-				  struct snd_pcm_hw_params *params,
-				  struct snd_soc_dai *dai)
-{
-	struct davinci_vcif_dev *davinci_vcif_dev = snd_soc_dai_get_drvdata(dai);
-	struct davinci_vc *davinci_vc = davinci_vcif_dev->davinci_vc;
-	u32 w;
-
-	/* Restart the codec before setup */
-	davinci_vcif_stop(substream);
-	davinci_vcif_start(substream);
-
-	/* General line settings */
-	writel(DAVINCI_VC_CTRL_MASK, davinci_vc->base + DAVINCI_VC_CTRL);
-
-	writel(DAVINCI_VC_INT_MASK, davinci_vc->base + DAVINCI_VC_INTCLR);
-
-	writel(DAVINCI_VC_INT_MASK, davinci_vc->base + DAVINCI_VC_INTEN);
-
-	w = readl(davinci_vc->base + DAVINCI_VC_CTRL);
-
-	/* Determine xfer data type */
-	switch (params_format(params)) {
-	case SNDRV_PCM_FORMAT_U8:
-		MOD_REG_BIT(w, DAVINCI_VC_CTRL_RD_BITS_8 |
-			    DAVINCI_VC_CTRL_RD_UNSIGNED |
-			    DAVINCI_VC_CTRL_WD_BITS_8 |
-			    DAVINCI_VC_CTRL_WD_UNSIGNED, 1);
-		break;
-	case SNDRV_PCM_FORMAT_S8:
-		MOD_REG_BIT(w, DAVINCI_VC_CTRL_RD_BITS_8 |
-			    DAVINCI_VC_CTRL_WD_BITS_8, 1);
-
-		MOD_REG_BIT(w, DAVINCI_VC_CTRL_RD_UNSIGNED |
-			    DAVINCI_VC_CTRL_WD_UNSIGNED, 0);
-		break;
-	case SNDRV_PCM_FORMAT_S16_LE:
-		MOD_REG_BIT(w, DAVINCI_VC_CTRL_RD_BITS_8 |
-			    DAVINCI_VC_CTRL_RD_UNSIGNED |
-			    DAVINCI_VC_CTRL_WD_BITS_8 |
-			    DAVINCI_VC_CTRL_WD_UNSIGNED, 0);
-		break;
-	default:
-		printk(KERN_WARNING "davinci-vcif: unsupported PCM format");
-		return -EINVAL;
-	}
-
-	writel(w, davinci_vc->base + DAVINCI_VC_CTRL);
-
-	return 0;
-}
-
-static int davinci_vcif_trigger(struct snd_pcm_substream *substream, int cmd,
-				struct snd_soc_dai *dai)
-{
-	int ret = 0;
-
-	switch (cmd) {
-	case SNDRV_PCM_TRIGGER_START:
-	case SNDRV_PCM_TRIGGER_RESUME:
-	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
-		davinci_vcif_start(substream);
-		break;
-	case SNDRV_PCM_TRIGGER_STOP:
-	case SNDRV_PCM_TRIGGER_SUSPEND:
-	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
-		davinci_vcif_stop(substream);
-		break;
-	default:
-		ret = -EINVAL;
-	}
-
-	return ret;
-}
-
-#define DAVINCI_VCIF_RATES	SNDRV_PCM_RATE_8000_48000
-
-static const struct snd_soc_dai_ops davinci_vcif_dai_ops = {
-	.trigger	= davinci_vcif_trigger,
-	.hw_params	= davinci_vcif_hw_params,
-};
-
-static int davinci_vcif_dai_probe(struct snd_soc_dai *dai)
-{
-	struct davinci_vcif_dev *dev = snd_soc_dai_get_drvdata(dai);
-
-	dai->playback_dma_data = &dev->dma_data[SNDRV_PCM_STREAM_PLAYBACK];
-	dai->capture_dma_data = &dev->dma_data[SNDRV_PCM_STREAM_CAPTURE];
-
-	return 0;
-}
-
-static struct snd_soc_dai_driver davinci_vcif_dai = {
-	.probe = davinci_vcif_dai_probe,
-	.playback = {
-		.channels_min = 1,
-		.channels_max = 2,
-		.rates = DAVINCI_VCIF_RATES,
-		.formats = SNDRV_PCM_FMTBIT_S16_LE,},
-	.capture = {
-		.channels_min = 1,
-		.channels_max = 2,
-		.rates = DAVINCI_VCIF_RATES,
-		.formats = SNDRV_PCM_FMTBIT_S16_LE,},
-	.ops = &davinci_vcif_dai_ops,
-
-};
-
-static const struct snd_soc_component_driver davinci_vcif_component = {
-	.name		= "davinci-vcif",
-};
-
-static int davinci_vcif_probe(struct platform_device *pdev)
-{
-	struct davinci_vc *davinci_vc = pdev->dev.platform_data;
-	struct davinci_vcif_dev *davinci_vcif_dev;
-	int ret;
-
-	davinci_vcif_dev = devm_kzalloc(&pdev->dev,
-					sizeof(struct davinci_vcif_dev),
-					GFP_KERNEL);
-	if (!davinci_vcif_dev)
-		return -ENOMEM;
-
-	/* DMA tx params */
-	davinci_vcif_dev->davinci_vc = davinci_vc;
-	davinci_vcif_dev->dma_data[SNDRV_PCM_STREAM_PLAYBACK].filter_data =
-				&davinci_vc->davinci_vcif.dma_tx_channel;
-	davinci_vcif_dev->dma_data[SNDRV_PCM_STREAM_PLAYBACK].addr =
-				davinci_vc->davinci_vcif.dma_tx_addr;
-
-	/* DMA rx params */
-	davinci_vcif_dev->dma_data[SNDRV_PCM_STREAM_CAPTURE].filter_data =
-				&davinci_vc->davinci_vcif.dma_rx_channel;
-	davinci_vcif_dev->dma_data[SNDRV_PCM_STREAM_CAPTURE].addr =
-				davinci_vc->davinci_vcif.dma_rx_addr;
-
-	dev_set_drvdata(&pdev->dev, davinci_vcif_dev);
-
-	ret = devm_snd_soc_register_component(&pdev->dev,
-					      &davinci_vcif_component,
-					      &davinci_vcif_dai, 1);
-	if (ret != 0) {
-		dev_err(&pdev->dev, "could not register dai\n");
-		return ret;
-	}
-
-	ret = edma_pcm_platform_register(&pdev->dev);
-	if (ret) {
-		dev_err(&pdev->dev, "register PCM failed: %d\n", ret);
-		return ret;
-	}
-
-	return 0;
-}
-
-static struct platform_driver davinci_vcif_driver = {
-	.probe		= davinci_vcif_probe,
-	.driver		= {
-		.name	= "davinci-vcif",
-	},
-};
-
-module_platform_driver(davinci_vcif_driver);
-
-MODULE_AUTHOR("Miguel Aguilar");
-MODULE_DESCRIPTION("Texas Instruments DaVinci ASoC Voice Codec Interface");
-MODULE_LICENSE("GPL");
diff -Naur --no-dereference a/sound/soc/davinci/edma-pcm.c b/sound/soc/davinci/edma-pcm.c
--- a/sound/soc/davinci/edma-pcm.c	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/davinci/edma-pcm.c	1969-12-31 19:00:00.000000000 -0500
@@ -1,59 +0,0 @@
-/*
- * edma-pcm.c - eDMA PCM driver using dmaengine for AM3xxx, AM4xxx
- *
- * Copyright (C) 2014 Texas Instruments, Inc.
- *
- * Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
- *
- * Based on: sound/soc/tegra/tegra_pcm.c
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- */
-
-#include <linux/module.h>
-#include <sound/core.h>
-#include <sound/pcm.h>
-#include <sound/pcm_params.h>
-#include <sound/soc.h>
-#include <sound/dmaengine_pcm.h>
-#include <linux/edma.h>
-
-#include "edma-pcm.h"
-
-static const struct snd_pcm_hardware edma_pcm_hardware = {
-	.info			= SNDRV_PCM_INFO_MMAP |
-				  SNDRV_PCM_INFO_MMAP_VALID |
-				  SNDRV_PCM_INFO_PAUSE | SNDRV_PCM_INFO_RESUME |
-				  SNDRV_PCM_INFO_NO_PERIOD_WAKEUP |
-				  SNDRV_PCM_INFO_INTERLEAVED,
-	.buffer_bytes_max	= 128 * 1024,
-	.period_bytes_min	= 32,
-	.period_bytes_max	= 64 * 1024,
-	.periods_min		= 2,
-	.periods_max		= 19, /* Limit by edma dmaengine driver */
-};
-
-static const struct snd_dmaengine_pcm_config edma_dmaengine_pcm_config = {
-	.pcm_hardware = &edma_pcm_hardware,
-	.prepare_slave_config = snd_dmaengine_pcm_prepare_slave_config,
-	.compat_filter_fn = edma_filter_fn,
-	.prealloc_buffer_size = 128 * 1024,
-};
-
-int edma_pcm_platform_register(struct device *dev)
-{
-	return devm_snd_dmaengine_pcm_register(dev, &edma_dmaengine_pcm_config,
-					SND_DMAENGINE_PCM_FLAG_COMPAT);
-}
-EXPORT_SYMBOL_GPL(edma_pcm_platform_register);
-
-MODULE_AUTHOR("Peter Ujfalusi <peter.ujfalusi@ti.com>");
-MODULE_DESCRIPTION("eDMA PCM ASoC platform driver");
-MODULE_LICENSE("GPL");
diff -Naur --no-dereference a/sound/soc/davinci/edma-pcm.h b/sound/soc/davinci/edma-pcm.h
--- a/sound/soc/davinci/edma-pcm.h	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/davinci/edma-pcm.h	1969-12-31 19:00:00.000000000 -0500
@@ -1,32 +0,0 @@
-/*
- * edma-pcm.h - eDMA PCM driver using dmaengine for AM3xxx, AM4xxx
- *
- * Copyright (C) 2014 Texas Instruments, Inc.
- *
- * Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
- *
- * Based on: sound/soc/tegra/tegra_pcm.h
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- */
-
-#ifndef __EDMA_PCM_H__
-#define __EDMA_PCM_H__
-
-#if IS_ENABLED(CONFIG_SND_EDMA_SOC)
-int edma_pcm_platform_register(struct device *dev);
-#else
-static inline int edma_pcm_platform_register(struct device *dev)
-{
-	return 0;
-}
-#endif /* CONFIG_SND_EDMA_SOC */
-
-#endif /* __EDMA_PCM_H__ */
diff -Naur --no-dereference a/sound/soc/davinci/Kconfig b/sound/soc/davinci/Kconfig
--- a/sound/soc/davinci/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/davinci/Kconfig	1969-12-31 19:00:00.000000000 -0500
@@ -1,106 +0,0 @@
-config SND_DAVINCI_SOC
-	tristate
-	depends on ARCH_DAVINCI
-	select SND_EDMA_SOC
-
-config SND_EDMA_SOC
-	tristate "SoC Audio for Texas Instruments chips using eDMA"
-	depends on TI_EDMA
-	select SND_SOC_GENERIC_DMAENGINE_PCM
-	help
-	  Say Y or M here if you want audio support for TI SoC which uses eDMA.
-	  The following line of SoCs are supported by this platform driver:
-	  - daVinci devices
-	  - AM335x
-	  - AM437x/AM438x
-	  - DRA7xx family
-
-config SND_DAVINCI_SOC_I2S
-	tristate "DaVinci Multichannel Buffered Serial Port (McBSP) support"
-	depends on SND_EDMA_SOC
-	help
-	  Say Y or M here if you want to have support for McBSP IP found in
-	  Texas Instruments DaVinci DA850 SoCs.
-
-config SND_DAVINCI_SOC_MCASP
-	tristate "Multichannel Audio Serial Port (McASP) support"
-	depends on SND_SDMA_SOC || SND_EDMA_SOC
-	help
-	  Say Y or M here if you want to have support for McASP IP found in
-	  various Texas Instruments SoCs like:
-	  - daVinci devices
-	  - Sitara line of SoCs (AM335x, AM438x, etc)
-	  - DRA7x devices
-
-config SND_DAVINCI_SOC_VCIF
-	tristate
-
-config SND_DAVINCI_SOC_GENERIC_EVM
-	tristate
-	select SND_SOC_TLV320AIC3X
-	select SND_DAVINCI_SOC_MCASP
-
-config SND_AM33XX_SOC_EVM
-	tristate "SoC Audio for the AM33XX chip based boards"
-	depends on SND_EDMA_SOC && SOC_AM33XX && I2C
-	select SND_DAVINCI_SOC_GENERIC_EVM
-	help
-	  Say Y or M if you want to add support for SoC audio on AM33XX
-	  boards using McASP and TLV320AIC3X codec. For example AM335X-EVM,
-	  AM335X-EVMSK, and BeagelBone with AudioCape boards have this
-	  setup.
-
-config SND_DAVINCI_SOC_EVM
-	tristate "SoC Audio support for DaVinci DM6446, DM355 or DM365 EVM"
-	depends on SND_EDMA_SOC && I2C
-	depends on MACH_DAVINCI_EVM || MACH_DAVINCI_DM355_EVM || MACH_DAVINCI_DM365_EVM
-	select SND_DAVINCI_SOC_GENERIC_EVM
-	help
-	  Say Y if you want to add support for SoC audio on TI
-	  DaVinci DM6446, DM355 or DM365 EVM platforms.
-
-choice
-	prompt "DM365 codec select"
-	depends on SND_DAVINCI_SOC_EVM
-	depends on MACH_DAVINCI_DM365_EVM
-
-config SND_DM365_AIC3X_CODEC
-	tristate "Audio Codec - AIC3101"
-	help
-	  Say Y if you want to add support for AIC3101 audio codec
-
-config SND_DM365_VOICE_CODEC
-	tristate "Voice Codec - CQ93VC"
-	select MFD_DAVINCI_VOICECODEC
-	select SND_DAVINCI_SOC_VCIF
-	select SND_SOC_CQ0093VC
-	help
-	  Say Y if you want to add support for SoC On-chip voice codec
-endchoice
-
-config  SND_DM6467_SOC_EVM
-	tristate "SoC Audio support for DaVinci DM6467 EVM"
-	depends on SND_EDMA_SOC && MACH_DAVINCI_DM6467_EVM && I2C
-	select SND_DAVINCI_SOC_GENERIC_EVM
-	select SND_SOC_SPDIF
-
-	help
-	  Say Y if you want to add support for SoC audio on TI
-
-config  SND_DA830_SOC_EVM
-	tristate "SoC Audio support for DA830/OMAP-L137 EVM"
-	depends on SND_EDMA_SOC && MACH_DAVINCI_DA830_EVM && I2C
-	select SND_DAVINCI_SOC_GENERIC_EVM
-
-	help
-	  Say Y if you want to add support for SoC audio on TI
-	  DA830/OMAP-L137 EVM
-
-config  SND_DA850_SOC_EVM
-	tristate "SoC Audio support for DA850/OMAP-L138 EVM"
-	depends on SND_EDMA_SOC && MACH_DAVINCI_DA850_EVM && I2C
-	select SND_DAVINCI_SOC_GENERIC_EVM
-	help
-	  Say Y if you want to add support for SoC audio on TI
-	  DA850/OMAP-L138 EVM
-
diff -Naur --no-dereference a/sound/soc/davinci/Makefile b/sound/soc/davinci/Makefile
--- a/sound/soc/davinci/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/davinci/Makefile	1969-12-31 19:00:00.000000000 -0500
@@ -1,16 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-# DAVINCI Platform Support
-snd-soc-edma-objs := edma-pcm.o
-snd-soc-davinci-i2s-objs := davinci-i2s.o
-snd-soc-davinci-mcasp-objs:= davinci-mcasp.o
-snd-soc-davinci-vcif-objs:= davinci-vcif.o
-
-obj-$(CONFIG_SND_EDMA_SOC) += snd-soc-edma.o
-obj-$(CONFIG_SND_DAVINCI_SOC_I2S) += snd-soc-davinci-i2s.o
-obj-$(CONFIG_SND_DAVINCI_SOC_MCASP) += snd-soc-davinci-mcasp.o
-obj-$(CONFIG_SND_DAVINCI_SOC_VCIF) += snd-soc-davinci-vcif.o
-
-# Generic DAVINCI/AM33xx Machine Support
-snd-soc-evm-objs := davinci-evm.o
-
-obj-$(CONFIG_SND_DAVINCI_SOC_GENERIC_EVM) += snd-soc-evm.o
diff -Naur --no-dereference a/sound/soc/generic/simple-card-utils.c b/sound/soc/generic/simple-card-utils.c
--- a/sound/soc/generic/simple-card-utils.c	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/generic/simple-card-utils.c	2020-06-19 21:00:31.436227717 -0400
@@ -200,8 +200,12 @@
 	if (of_property_read_bool(node, "system-clock-direction-out"))
 		simple_dai->clk_direction = SND_SOC_CLOCK_OUT;
 
-	dev_dbg(dev, "%s : sysclk = %d, direction %d\n", name,
-		simple_dai->sysclk, simple_dai->clk_direction);
+	if (!of_property_read_u32(node, "system-clock-id", &val))
+		simple_dai->sysclk_id = val;
+
+	dev_dbg(dev, "%s : sysclk-%d = %d, direction %d\n", name,
+		simple_dai->sysclk_id, simple_dai->sysclk,
+		simple_dai->clk_direction);
 
 	return 0;
 }
@@ -313,7 +317,8 @@
 	int ret;
 
 	if (simple_dai->sysclk) {
-		ret = snd_soc_dai_set_sysclk(dai, 0, simple_dai->sysclk,
+		ret = snd_soc_dai_set_sysclk(dai, simple_dai->sysclk_id,
+					     simple_dai->sysclk,
 					     simple_dai->clk_direction);
 		if (ret && ret != -ENOTSUPP) {
 			dev_err(dai->dev, "simple-card: set_sysclk error\n");
diff -Naur --no-dereference a/sound/soc/Kconfig b/sound/soc/Kconfig
--- a/sound/soc/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/Kconfig	2020-06-19 21:00:31.436227717 -0400
@@ -46,13 +46,11 @@
 source "sound/soc/au1x/Kconfig"
 source "sound/soc/bcm/Kconfig"
 source "sound/soc/cirrus/Kconfig"
-source "sound/soc/davinci/Kconfig"
 source "sound/soc/dwc/Kconfig"
 source "sound/soc/fsl/Kconfig"
 source "sound/soc/hisilicon/Kconfig"
 source "sound/soc/jz4740/Kconfig"
 source "sound/soc/nuc900/Kconfig"
-source "sound/soc/omap/Kconfig"
 source "sound/soc/kirkwood/Kconfig"
 source "sound/soc/img/Kconfig"
 source "sound/soc/intel/Kconfig"
@@ -70,6 +68,7 @@
 source "sound/soc/stm/Kconfig"
 source "sound/soc/sunxi/Kconfig"
 source "sound/soc/tegra/Kconfig"
+source "sound/soc/ti/Kconfig"
 source "sound/soc/txx9/Kconfig"
 source "sound/soc/uniphier/Kconfig"
 source "sound/soc/ux500/Kconfig"
diff -Naur --no-dereference a/sound/soc/Makefile b/sound/soc/Makefile
--- a/sound/soc/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/Makefile	2020-06-19 21:00:31.436227717 -0400
@@ -30,7 +30,6 @@
 obj-$(CONFIG_SND_SOC)	+= au1x/
 obj-$(CONFIG_SND_SOC)	+= bcm/
 obj-$(CONFIG_SND_SOC)	+= cirrus/
-obj-$(CONFIG_SND_SOC)	+= davinci/
 obj-$(CONFIG_SND_SOC)	+= dwc/
 obj-$(CONFIG_SND_SOC)	+= fsl/
 obj-$(CONFIG_SND_SOC)	+= hisilicon/
@@ -41,7 +40,6 @@
 obj-$(CONFIG_SND_SOC)	+= meson/
 obj-$(CONFIG_SND_SOC)	+= mxs/
 obj-$(CONFIG_SND_SOC)	+= nuc900/
-obj-$(CONFIG_SND_SOC)	+= omap/
 obj-$(CONFIG_SND_SOC)	+= kirkwood/
 obj-$(CONFIG_SND_SOC)	+= pxa/
 obj-$(CONFIG_SND_SOC)	+= qcom/
@@ -54,6 +52,7 @@
 obj-$(CONFIG_SND_SOC)	+= stm/
 obj-$(CONFIG_SND_SOC)	+= sunxi/
 obj-$(CONFIG_SND_SOC)	+= tegra/
+obj-$(CONFIG_SND_SOC)	+= ti/
 obj-$(CONFIG_SND_SOC)	+= txx9/
 obj-$(CONFIG_SND_SOC)	+= uniphier/
 obj-$(CONFIG_SND_SOC)	+= ux500/
diff -Naur --no-dereference a/sound/soc/omap/am3517evm.c b/sound/soc/omap/am3517evm.c
--- a/sound/soc/omap/am3517evm.c	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/omap/am3517evm.c	1969-12-31 19:00:00.000000000 -0500
@@ -1,141 +0,0 @@
-/*
- * am3517evm.c  -- ALSA SoC support for OMAP3517 / AM3517 EVM
- *
- * Author: Anuj Aggarwal <anuj.aggarwal@ti.com>
- *
- * Based on sound/soc/omap/beagle.c by Steve Sakoman
- *
- * Copyright (C) 2009 Texas Instruments Incorporated
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2.
- *
- * This program is distributed "as is" WITHOUT ANY WARRANTY of any kind,
- * whether express or implied; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- */
-
-#include <linux/clk.h>
-#include <linux/platform_device.h>
-#include <linux/module.h>
-#include <sound/core.h>
-#include <sound/pcm.h>
-#include <sound/soc.h>
-
-#include <asm/mach-types.h>
-#include <linux/platform_data/asoc-ti-mcbsp.h>
-
-#include "omap-mcbsp.h"
-
-#include "../codecs/tlv320aic23.h"
-
-#define CODEC_CLOCK 	12000000
-
-static int am3517evm_hw_params(struct snd_pcm_substream *substream,
-	struct snd_pcm_hw_params *params)
-{
-	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct snd_soc_dai *codec_dai = rtd->codec_dai;
-	int ret;
-
-	/* Set the codec system clock for DAC and ADC */
-	ret = snd_soc_dai_set_sysclk(codec_dai, 0,
-			CODEC_CLOCK, SND_SOC_CLOCK_IN);
-	if (ret < 0)
-		printk(KERN_ERR "can't set codec system clock\n");
-
-	return ret;
-}
-
-static const struct snd_soc_ops am3517evm_ops = {
-	.hw_params = am3517evm_hw_params,
-};
-
-/* am3517evm machine dapm widgets */
-static const struct snd_soc_dapm_widget tlv320aic23_dapm_widgets[] = {
-	SND_SOC_DAPM_HP("Line Out", NULL),
-	SND_SOC_DAPM_LINE("Line In", NULL),
-	SND_SOC_DAPM_MIC("Mic In", NULL),
-};
-
-static const struct snd_soc_dapm_route audio_map[] = {
-	/* Line Out connected to LLOUT, RLOUT */
-	{"Line Out", NULL, "LOUT"},
-	{"Line Out", NULL, "ROUT"},
-
-	{"LLINEIN", NULL, "Line In"},
-	{"RLINEIN", NULL, "Line In"},
-
-	{"MICIN", NULL, "Mic In"},
-};
-
-/* Digital audio interface glue - connects codec <--> CPU */
-static struct snd_soc_dai_link am3517evm_dai = {
-	.name = "TLV320AIC23",
-	.stream_name = "AIC23",
-	.cpu_dai_name = "omap-mcbsp.1",
-	.codec_dai_name = "tlv320aic23-hifi",
-	.platform_name = "omap-mcbsp.1",
-	.codec_name = "tlv320aic23-codec.2-001a",
-	.dai_fmt = SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_NB_NF |
-		   SND_SOC_DAIFMT_CBM_CFM,
-	.ops = &am3517evm_ops,
-};
-
-/* Audio machine driver */
-static struct snd_soc_card snd_soc_am3517evm = {
-	.name = "am3517evm",
-	.owner = THIS_MODULE,
-	.dai_link = &am3517evm_dai,
-	.num_links = 1,
-
-	.dapm_widgets = tlv320aic23_dapm_widgets,
-	.num_dapm_widgets = ARRAY_SIZE(tlv320aic23_dapm_widgets),
-	.dapm_routes = audio_map,
-	.num_dapm_routes = ARRAY_SIZE(audio_map),
-};
-
-static struct platform_device *am3517evm_snd_device;
-
-static int __init am3517evm_soc_init(void)
-{
-	int ret;
-
-	if (!machine_is_omap3517evm())
-		return -ENODEV;
-	pr_info("OMAP3517 / AM3517 EVM SoC init\n");
-
-	am3517evm_snd_device = platform_device_alloc("soc-audio", -1);
-	if (!am3517evm_snd_device) {
-		printk(KERN_ERR "Platform device allocation failed\n");
-		return -ENOMEM;
-	}
-
-	platform_set_drvdata(am3517evm_snd_device, &snd_soc_am3517evm);
-
-	ret = platform_device_add(am3517evm_snd_device);
-	if (ret)
-		goto err1;
-
-	return 0;
-
-err1:
-	printk(KERN_ERR "Unable to add platform device\n");
-	platform_device_put(am3517evm_snd_device);
-
-	return ret;
-}
-
-static void __exit am3517evm_soc_exit(void)
-{
-	platform_device_unregister(am3517evm_snd_device);
-}
-
-module_init(am3517evm_soc_init);
-module_exit(am3517evm_soc_exit);
-
-MODULE_AUTHOR("Anuj Aggarwal <anuj.aggarwal@ti.com>");
-MODULE_DESCRIPTION("ALSA SoC OMAP3517 / AM3517 EVM");
-MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/sound/soc/omap/ams-delta.c b/sound/soc/omap/ams-delta.c
--- a/sound/soc/omap/ams-delta.c	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/omap/ams-delta.c	1969-12-31 19:00:00.000000000 -0500
@@ -1,594 +0,0 @@
-/*
- * ams-delta.c  --  SoC audio for Amstrad E3 (Delta) videophone
- *
- * Copyright (C) 2009 Janusz Krzysztofik <jkrzyszt@tis.icnet.pl>
- *
- * Initially based on sound/soc/omap/osk5912.x
- * Copyright (C) 2008 Mistral Solutions
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
- * 02110-1301 USA
- *
- */
-
-#include <linux/gpio/consumer.h>
-#include <linux/spinlock.h>
-#include <linux/tty.h>
-#include <linux/module.h>
-
-#include <sound/soc.h>
-#include <sound/jack.h>
-
-#include <asm/mach-types.h>
-
-#include <linux/platform_data/asoc-ti-mcbsp.h>
-
-#include "omap-mcbsp.h"
-#include "../codecs/cx20442.h"
-
-/* Board specific DAPM widgets */
-static const struct snd_soc_dapm_widget ams_delta_dapm_widgets[] = {
-	/* Handset */
-	SND_SOC_DAPM_MIC("Mouthpiece", NULL),
-	SND_SOC_DAPM_HP("Earpiece", NULL),
-	/* Handsfree/Speakerphone */
-	SND_SOC_DAPM_MIC("Microphone", NULL),
-	SND_SOC_DAPM_SPK("Speaker", NULL),
-};
-
-/* How they are connected to codec pins */
-static const struct snd_soc_dapm_route ams_delta_audio_map[] = {
-	{"TELIN", NULL, "Mouthpiece"},
-	{"Earpiece", NULL, "TELOUT"},
-
-	{"MIC", NULL, "Microphone"},
-	{"Speaker", NULL, "SPKOUT"},
-};
-
-/*
- * Controls, functional after the modem line discipline is activated.
- */
-
-/* Virtual switch: audio input/output constellations */
-static const char *ams_delta_audio_mode[] =
-	{"Mixed", "Handset", "Handsfree", "Speakerphone"};
-
-/* Selection <-> pin translation */
-#define AMS_DELTA_MOUTHPIECE	0
-#define AMS_DELTA_EARPIECE	1
-#define AMS_DELTA_MICROPHONE	2
-#define AMS_DELTA_SPEAKER	3
-#define AMS_DELTA_AGC		4
-
-#define AMS_DELTA_MIXED		((1 << AMS_DELTA_EARPIECE) | \
-						(1 << AMS_DELTA_MICROPHONE))
-#define AMS_DELTA_HANDSET	((1 << AMS_DELTA_MOUTHPIECE) | \
-						(1 << AMS_DELTA_EARPIECE))
-#define AMS_DELTA_HANDSFREE	((1 << AMS_DELTA_MICROPHONE) | \
-						(1 << AMS_DELTA_SPEAKER))
-#define AMS_DELTA_SPEAKERPHONE	(AMS_DELTA_HANDSFREE | (1 << AMS_DELTA_AGC))
-
-static const unsigned short ams_delta_audio_mode_pins[] = {
-	AMS_DELTA_MIXED,
-	AMS_DELTA_HANDSET,
-	AMS_DELTA_HANDSFREE,
-	AMS_DELTA_SPEAKERPHONE,
-};
-
-static unsigned short ams_delta_audio_agc;
-
-/*
- * Used for passing a codec structure pointer
- * from the board initialization code to the tty line discipline.
- */
-static struct snd_soc_component *cx20442_codec;
-
-static int ams_delta_set_audio_mode(struct snd_kcontrol *kcontrol,
-					struct snd_ctl_elem_value *ucontrol)
-{
-	struct snd_soc_card *card = snd_kcontrol_chip(kcontrol);
-	struct snd_soc_dapm_context *dapm = &card->dapm;
-	struct soc_enum *control = (struct soc_enum *)kcontrol->private_value;
-	unsigned short pins;
-	int pin, changed = 0;
-
-	/* Refuse any mode changes if we are not able to control the codec. */
-	if (!cx20442_codec->card->pop_time)
-		return -EUNATCH;
-
-	if (ucontrol->value.enumerated.item[0] >= control->items)
-		return -EINVAL;
-
-	snd_soc_dapm_mutex_lock(dapm);
-
-	/* Translate selection to bitmap */
-	pins = ams_delta_audio_mode_pins[ucontrol->value.enumerated.item[0]];
-
-	/* Setup pins after corresponding bits if changed */
-	pin = !!(pins & (1 << AMS_DELTA_MOUTHPIECE));
-
-	if (pin != snd_soc_dapm_get_pin_status(dapm, "Mouthpiece")) {
-		changed = 1;
-		if (pin)
-			snd_soc_dapm_enable_pin_unlocked(dapm, "Mouthpiece");
-		else
-			snd_soc_dapm_disable_pin_unlocked(dapm, "Mouthpiece");
-	}
-	pin = !!(pins & (1 << AMS_DELTA_EARPIECE));
-	if (pin != snd_soc_dapm_get_pin_status(dapm, "Earpiece")) {
-		changed = 1;
-		if (pin)
-			snd_soc_dapm_enable_pin_unlocked(dapm, "Earpiece");
-		else
-			snd_soc_dapm_disable_pin_unlocked(dapm, "Earpiece");
-	}
-	pin = !!(pins & (1 << AMS_DELTA_MICROPHONE));
-	if (pin != snd_soc_dapm_get_pin_status(dapm, "Microphone")) {
-		changed = 1;
-		if (pin)
-			snd_soc_dapm_enable_pin_unlocked(dapm, "Microphone");
-		else
-			snd_soc_dapm_disable_pin_unlocked(dapm, "Microphone");
-	}
-	pin = !!(pins & (1 << AMS_DELTA_SPEAKER));
-	if (pin != snd_soc_dapm_get_pin_status(dapm, "Speaker")) {
-		changed = 1;
-		if (pin)
-			snd_soc_dapm_enable_pin_unlocked(dapm, "Speaker");
-		else
-			snd_soc_dapm_disable_pin_unlocked(dapm, "Speaker");
-	}
-	pin = !!(pins & (1 << AMS_DELTA_AGC));
-	if (pin != ams_delta_audio_agc) {
-		ams_delta_audio_agc = pin;
-		changed = 1;
-		if (pin)
-			snd_soc_dapm_enable_pin_unlocked(dapm, "AGCIN");
-		else
-			snd_soc_dapm_disable_pin_unlocked(dapm, "AGCIN");
-	}
-
-	if (changed)
-		snd_soc_dapm_sync_unlocked(dapm);
-
-	snd_soc_dapm_mutex_unlock(dapm);
-
-	return changed;
-}
-
-static int ams_delta_get_audio_mode(struct snd_kcontrol *kcontrol,
-					struct snd_ctl_elem_value *ucontrol)
-{
-	struct snd_soc_card *card = snd_kcontrol_chip(kcontrol);
-	struct snd_soc_dapm_context *dapm = &card->dapm;
-	unsigned short pins, mode;
-
-	pins = ((snd_soc_dapm_get_pin_status(dapm, "Mouthpiece") <<
-							AMS_DELTA_MOUTHPIECE) |
-			(snd_soc_dapm_get_pin_status(dapm, "Earpiece") <<
-							AMS_DELTA_EARPIECE));
-	if (pins)
-		pins |= (snd_soc_dapm_get_pin_status(dapm, "Microphone") <<
-							AMS_DELTA_MICROPHONE);
-	else
-		pins = ((snd_soc_dapm_get_pin_status(dapm, "Microphone") <<
-							AMS_DELTA_MICROPHONE) |
-			(snd_soc_dapm_get_pin_status(dapm, "Speaker") <<
-							AMS_DELTA_SPEAKER) |
-			(ams_delta_audio_agc << AMS_DELTA_AGC));
-
-	for (mode = 0; mode < ARRAY_SIZE(ams_delta_audio_mode); mode++)
-		if (pins == ams_delta_audio_mode_pins[mode])
-			break;
-
-	if (mode >= ARRAY_SIZE(ams_delta_audio_mode))
-		return -EINVAL;
-
-	ucontrol->value.enumerated.item[0] = mode;
-
-	return 0;
-}
-
-static const SOC_ENUM_SINGLE_EXT_DECL(ams_delta_audio_enum,
-				      ams_delta_audio_mode);
-
-static const struct snd_kcontrol_new ams_delta_audio_controls[] = {
-	SOC_ENUM_EXT("Audio Mode", ams_delta_audio_enum,
-			ams_delta_get_audio_mode, ams_delta_set_audio_mode),
-};
-
-/* Hook switch */
-static struct snd_soc_jack ams_delta_hook_switch;
-static struct snd_soc_jack_gpio ams_delta_hook_switch_gpios[] = {
-	{
-		.name = "hook_switch",
-		.report = SND_JACK_HEADSET,
-		.invert = 1,
-		.debounce_time = 150,
-	}
-};
-
-/* After we are able to control the codec over the modem,
- * the hook switch can be used for dynamic DAPM reconfiguration. */
-static struct snd_soc_jack_pin ams_delta_hook_switch_pins[] = {
-	/* Handset */
-	{
-		.pin = "Mouthpiece",
-		.mask = SND_JACK_MICROPHONE,
-	},
-	{
-		.pin = "Earpiece",
-		.mask = SND_JACK_HEADPHONE,
-	},
-	/* Handsfree */
-	{
-		.pin = "Microphone",
-		.mask = SND_JACK_MICROPHONE,
-		.invert = 1,
-	},
-	{
-		.pin = "Speaker",
-		.mask = SND_JACK_HEADPHONE,
-		.invert = 1,
-	},
-};
-
-
-/*
- * Modem line discipline, required for making above controls functional.
- * Activated from userspace with ldattach, possibly invoked from udev rule.
- */
-
-/* To actually apply any modem controlled configuration changes to the codec,
- * we must connect codec DAI pins to the modem for a moment.  Be careful not
- * to interfere with our digital mute function that shares the same hardware. */
-static struct timer_list cx81801_timer;
-static bool cx81801_cmd_pending;
-static bool ams_delta_muted;
-static DEFINE_SPINLOCK(ams_delta_lock);
-static struct gpio_desc *gpiod_modem_codec;
-
-static void cx81801_timeout(struct timer_list *unused)
-{
-	int muted;
-
-	spin_lock(&ams_delta_lock);
-	cx81801_cmd_pending = 0;
-	muted = ams_delta_muted;
-	spin_unlock(&ams_delta_lock);
-
-	/* Reconnect the codec DAI back from the modem to the CPU DAI
-	 * only if digital mute still off */
-	if (!muted)
-		gpiod_set_value(gpiod_modem_codec, 0);
-}
-
-/* Line discipline .open() */
-static int cx81801_open(struct tty_struct *tty)
-{
-	int ret;
-
-	if (!cx20442_codec)
-		return -ENODEV;
-
-	/*
-	 * Pass the codec structure pointer for use by other ldisc callbacks,
-	 * both the card and the codec specific parts.
-	 */
-	tty->disc_data = cx20442_codec;
-
-	ret = v253_ops.open(tty);
-
-	if (ret < 0)
-		tty->disc_data = NULL;
-
-	return ret;
-}
-
-/* Line discipline .close() */
-static void cx81801_close(struct tty_struct *tty)
-{
-	struct snd_soc_component *component = tty->disc_data;
-	struct snd_soc_dapm_context *dapm = &component->card->dapm;
-
-	del_timer_sync(&cx81801_timer);
-
-	/* Prevent the hook switch from further changing the DAPM pins */
-	INIT_LIST_HEAD(&ams_delta_hook_switch.pins);
-
-	if (!component)
-		return;
-
-	v253_ops.close(tty);
-
-	/* Revert back to default audio input/output constellation */
-	snd_soc_dapm_mutex_lock(dapm);
-
-	snd_soc_dapm_disable_pin_unlocked(dapm, "Mouthpiece");
-	snd_soc_dapm_enable_pin_unlocked(dapm, "Earpiece");
-	snd_soc_dapm_enable_pin_unlocked(dapm, "Microphone");
-	snd_soc_dapm_disable_pin_unlocked(dapm, "Speaker");
-	snd_soc_dapm_disable_pin_unlocked(dapm, "AGCIN");
-
-	snd_soc_dapm_sync_unlocked(dapm);
-
-	snd_soc_dapm_mutex_unlock(dapm);
-}
-
-/* Line discipline .hangup() */
-static int cx81801_hangup(struct tty_struct *tty)
-{
-	cx81801_close(tty);
-	return 0;
-}
-
-/* Line discipline .receive_buf() */
-static void cx81801_receive(struct tty_struct *tty,
-				const unsigned char *cp, char *fp, int count)
-{
-	struct snd_soc_component *component = tty->disc_data;
-	const unsigned char *c;
-	int apply, ret;
-
-	if (!component)
-		return;
-
-	if (!component->card->pop_time) {
-		/* First modem response, complete setup procedure */
-
-		/* Initialize timer used for config pulse generation */
-		timer_setup(&cx81801_timer, cx81801_timeout, 0);
-
-		v253_ops.receive_buf(tty, cp, fp, count);
-
-		/* Link hook switch to DAPM pins */
-		ret = snd_soc_jack_add_pins(&ams_delta_hook_switch,
-					ARRAY_SIZE(ams_delta_hook_switch_pins),
-					ams_delta_hook_switch_pins);
-		if (ret)
-			dev_warn(component->dev,
-				"Failed to link hook switch to DAPM pins, "
-				"will continue with hook switch unlinked.\n");
-
-		return;
-	}
-
-	v253_ops.receive_buf(tty, cp, fp, count);
-
-	for (c = &cp[count - 1]; c >= cp; c--) {
-		if (*c != '\r')
-			continue;
-		/* Complete modem response received, apply config to codec */
-
-		spin_lock_bh(&ams_delta_lock);
-		mod_timer(&cx81801_timer, jiffies + msecs_to_jiffies(150));
-		apply = !ams_delta_muted && !cx81801_cmd_pending;
-		cx81801_cmd_pending = 1;
-		spin_unlock_bh(&ams_delta_lock);
-
-		/* Apply config pulse by connecting the codec to the modem
-		 * if not already done */
-		if (apply)
-			gpiod_set_value(gpiod_modem_codec, 1);
-		break;
-	}
-}
-
-/* Line discipline .write_wakeup() */
-static void cx81801_wakeup(struct tty_struct *tty)
-{
-	v253_ops.write_wakeup(tty);
-}
-
-static struct tty_ldisc_ops cx81801_ops = {
-	.magic = TTY_LDISC_MAGIC,
-	.name = "cx81801",
-	.owner = THIS_MODULE,
-	.open = cx81801_open,
-	.close = cx81801_close,
-	.hangup = cx81801_hangup,
-	.receive_buf = cx81801_receive,
-	.write_wakeup = cx81801_wakeup,
-};
-
-
-/*
- * Even if not very useful, the sound card can still work without any of the
- * above functonality activated.  You can still control its audio input/output
- * constellation and speakerphone gain from userspace by issuing AT commands
- * over the modem port.
- */
-
-static struct snd_soc_ops ams_delta_ops;
-
-
-/* Digital mute implemented using modem/CPU multiplexer.
- * Shares hardware with codec config pulse generation */
-static bool ams_delta_muted = 1;
-
-static int ams_delta_digital_mute(struct snd_soc_dai *dai, int mute)
-{
-	int apply;
-
-	if (ams_delta_muted == mute)
-		return 0;
-
-	spin_lock_bh(&ams_delta_lock);
-	ams_delta_muted = mute;
-	apply = !cx81801_cmd_pending;
-	spin_unlock_bh(&ams_delta_lock);
-
-	if (apply)
-		gpiod_set_value(gpiod_modem_codec, !!mute);
-	return 0;
-}
-
-/* Our codec DAI probably doesn't have its own .ops structure */
-static const struct snd_soc_dai_ops ams_delta_dai_ops = {
-	.digital_mute = ams_delta_digital_mute,
-};
-
-/* Will be used if the codec ever has its own digital_mute function */
-static int ams_delta_startup(struct snd_pcm_substream *substream)
-{
-	return ams_delta_digital_mute(NULL, 0);
-}
-
-static void ams_delta_shutdown(struct snd_pcm_substream *substream)
-{
-	ams_delta_digital_mute(NULL, 1);
-}
-
-
-/*
- * Card initialization
- */
-
-static int ams_delta_cx20442_init(struct snd_soc_pcm_runtime *rtd)
-{
-	struct snd_soc_dai *codec_dai = rtd->codec_dai;
-	struct snd_soc_card *card = rtd->card;
-	struct snd_soc_dapm_context *dapm = &card->dapm;
-	int ret;
-	/* Codec is ready, now add/activate board specific controls */
-
-	/* Store a pointer to the codec structure for tty ldisc use */
-	cx20442_codec = rtd->codec_dai->component;
-
-	/* Add hook switch - can be used to control the codec from userspace
-	 * even if line discipline fails */
-	ret = snd_soc_card_jack_new(card, "hook_switch", SND_JACK_HEADSET,
-				    &ams_delta_hook_switch, NULL, 0);
-	if (ret)
-		dev_warn(card->dev,
-				"Failed to allocate resources for hook switch, "
-				"will continue without one.\n");
-	else {
-		ret = snd_soc_jack_add_gpiods(card->dev, &ams_delta_hook_switch,
-					ARRAY_SIZE(ams_delta_hook_switch_gpios),
-					ams_delta_hook_switch_gpios);
-		if (ret)
-			dev_warn(card->dev,
-				"Failed to set up hook switch GPIO line, "
-				"will continue with hook switch inactive.\n");
-	}
-
-	gpiod_modem_codec = devm_gpiod_get(card->dev, "modem_codec",
-					   GPIOD_OUT_HIGH);
-	if (IS_ERR(gpiod_modem_codec)) {
-		dev_warn(card->dev, "Failed to obtain modem_codec GPIO\n");
-		return 0;
-	}
-
-	/* Set up digital mute if not provided by the codec */
-	if (!codec_dai->driver->ops) {
-		codec_dai->driver->ops = &ams_delta_dai_ops;
-	} else {
-		ams_delta_ops.startup = ams_delta_startup;
-		ams_delta_ops.shutdown = ams_delta_shutdown;
-	}
-
-	/* Register optional line discipline for over the modem control */
-	ret = tty_register_ldisc(N_V253, &cx81801_ops);
-	if (ret) {
-		dev_warn(card->dev,
-				"Failed to register line discipline, "
-				"will continue without any controls.\n");
-		return 0;
-	}
-
-	/* Set up initial pin constellation */
-	snd_soc_dapm_disable_pin(dapm, "Mouthpiece");
-	snd_soc_dapm_disable_pin(dapm, "Speaker");
-	snd_soc_dapm_disable_pin(dapm, "AGCIN");
-	snd_soc_dapm_disable_pin(dapm, "AGCOUT");
-
-	return 0;
-}
-
-/* DAI glue - connects codec <--> CPU */
-static struct snd_soc_dai_link ams_delta_dai_link = {
-	.name = "CX20442",
-	.stream_name = "CX20442",
-	.cpu_dai_name = "omap-mcbsp.1",
-	.codec_dai_name = "cx20442-voice",
-	.init = ams_delta_cx20442_init,
-	.platform_name = "omap-mcbsp.1",
-	.codec_name = "cx20442-codec",
-	.ops = &ams_delta_ops,
-	.dai_fmt = SND_SOC_DAIFMT_DSP_A | SND_SOC_DAIFMT_NB_NF |
-		   SND_SOC_DAIFMT_CBM_CFM,
-};
-
-/* Audio card driver */
-static struct snd_soc_card ams_delta_audio_card = {
-	.name = "AMS_DELTA",
-	.owner = THIS_MODULE,
-	.dai_link = &ams_delta_dai_link,
-	.num_links = 1,
-
-	.controls = ams_delta_audio_controls,
-	.num_controls = ARRAY_SIZE(ams_delta_audio_controls),
-	.dapm_widgets = ams_delta_dapm_widgets,
-	.num_dapm_widgets = ARRAY_SIZE(ams_delta_dapm_widgets),
-	.dapm_routes = ams_delta_audio_map,
-	.num_dapm_routes = ARRAY_SIZE(ams_delta_audio_map),
-};
-
-/* Module init/exit */
-static int ams_delta_probe(struct platform_device *pdev)
-{
-	struct snd_soc_card *card = &ams_delta_audio_card;
-	int ret;
-
-	card->dev = &pdev->dev;
-
-	ret = snd_soc_register_card(card);
-	if (ret) {
-		dev_err(&pdev->dev, "snd_soc_register_card failed (%d)\n", ret);
-		card->dev = NULL;
-		return ret;
-	}
-	return 0;
-}
-
-static int ams_delta_remove(struct platform_device *pdev)
-{
-	struct snd_soc_card *card = platform_get_drvdata(pdev);
-
-	if (tty_unregister_ldisc(N_V253) != 0)
-		dev_warn(&pdev->dev,
-			"failed to unregister V253 line discipline\n");
-
-	snd_soc_unregister_card(card);
-	card->dev = NULL;
-	return 0;
-}
-
-#define DRV_NAME "ams-delta-audio"
-
-static struct platform_driver ams_delta_driver = {
-	.driver = {
-		.name = DRV_NAME,
-	},
-	.probe = ams_delta_probe,
-	.remove = ams_delta_remove,
-};
-
-module_platform_driver(ams_delta_driver);
-
-MODULE_AUTHOR("Janusz Krzysztofik <jkrzyszt@tis.icnet.pl>");
-MODULE_DESCRIPTION("ALSA SoC driver for Amstrad E3 (Delta) videophone");
-MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:" DRV_NAME);
diff -Naur --no-dereference a/sound/soc/omap/Kconfig b/sound/soc/omap/Kconfig
--- a/sound/soc/omap/Kconfig	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/omap/Kconfig	1969-12-31 19:00:00.000000000 -0500
@@ -1,129 +0,0 @@
-config SND_OMAP_SOC
-	tristate "SoC Audio for Texas Instruments OMAP chips (deprecated)"
-	depends on (ARCH_OMAP && DMA_OMAP) || (ARM && COMPILE_TEST)
-	select SND_SDMA_SOC
-
-config SND_SDMA_SOC
-	tristate "SoC Audio for Texas Instruments chips using sDMA"
-	depends on DMA_OMAP || COMPILE_TEST
-	select SND_SOC_GENERIC_DMAENGINE_PCM
-
-config SND_OMAP_SOC_DMIC
-	tristate
-
-config SND_OMAP_SOC_MCBSP
-	tristate
-
-config SND_OMAP_SOC_MCPDM
-	tristate
-
-config SND_OMAP_SOC_HDMI_AUDIO
-	tristate "HDMI audio support for OMAP4+ based SoCs"
-	depends on SND_SDMA_SOC
-	help
-	  For HDMI audio to work OMAPDSS HDMI support should be
-	  enabled.
-	  The hdmi audio driver implements cpu-dai component using the
-	  callbacks provided by OMAPDSS and registers the component
-	  under DSS HDMI device. Omap-pcm is registered for platform
-	  component also under DSS HDMI device. Dummy codec is used as
-	  as codec component. The hdmi audio driver implements also
-	  the card and registers it under its own platform device.
-	  The device for the driver is registered by OMAPDSS hdmi
-	  driver.
-
-config SND_OMAP_SOC_N810
-	tristate "SoC Audio support for Nokia N810"
-	depends on SND_SDMA_SOC && MACH_NOKIA_N810 && I2C
-	select SND_OMAP_SOC_MCBSP
-	select SND_SOC_TLV320AIC3X
-	help
-	  Say Y if you want to add support for SoC audio on Nokia N810.
-
-config SND_OMAP_SOC_RX51
-	tristate "SoC Audio support for Nokia N900 (RX-51)"
-	depends on SND_SDMA_SOC && ARM && I2C
-	select SND_OMAP_SOC_MCBSP
-	select SND_SOC_TLV320AIC3X
-	select SND_SOC_TPA6130A2
-	depends on GPIOLIB
-	help
-	  Say Y if you want to add support for SoC audio on Nokia N900
-	  cellphone.
-
-config SND_OMAP_SOC_AMS_DELTA
-	tristate "SoC Audio support for Amstrad E3 (Delta) videophone"
-	depends on SND_SDMA_SOC && MACH_AMS_DELTA && TTY
-	select SND_OMAP_SOC_MCBSP
-	select SND_SOC_CX20442
-	help
-	  Say Y  if you want to add support  for SoC audio device  connected to
-	  a handset and a speakerphone found on Amstrad E3 (Delta) videophone.
-
-	  Note that in order to get those devices fully supported,  you have to
-	  build  the kernel  with  standard  serial port  driver  included  and
-	  configured for at least 4 ports.  Then, from userspace, you must load
-	  a line discipline #19 on the modem (ttyS3) serial line.  The simplest
-	  way to achieve this is to install util-linux-ng  and use the included
-	  ldattach  utility.  This  can be  started  automatically  from  udev,
-	  a simple rule like this one should do the trick (it does for me):
-	  	ACTION=="add", KERNEL=="controlC0", \
-				RUN+="/usr/sbin/ldattach 19 /dev/ttyS3"
-
-config SND_OMAP_SOC_OSK5912
-	tristate "SoC Audio support for omap osk5912"
-	depends on SND_SDMA_SOC && MACH_OMAP_OSK && I2C
-	select SND_OMAP_SOC_MCBSP
-	select SND_SOC_TLV320AIC23_I2C
-	help
-	  Say Y if you want to add support for SoC audio on osk5912.
-
-config SND_OMAP_SOC_AM3517EVM
-	tristate "SoC Audio support for OMAP3517 / AM3517 EVM"
-	depends on SND_SDMA_SOC && MACH_OMAP3517EVM && I2C
-	select SND_OMAP_SOC_MCBSP
-	select SND_SOC_TLV320AIC23_I2C
-	help
-	  Say Y if you want to add support for SoC audio on the OMAP3517 / AM3517
-	  EVM.
-
-config SND_OMAP_SOC_OMAP_TWL4030
-	tristate "SoC Audio support for TI SoC based boards with twl4030 codec"
-	depends on TWL4030_CORE && SND_SDMA_SOC
-	select SND_OMAP_SOC_MCBSP
-	select SND_SOC_TWL4030
-	help
-	  Say Y if you want to add support for SoC audio on TI SoC based boards
-	  using twl4030 as c codec. This driver currently supports:
-	  - Beagleboard or Devkit8000
-	  - Gumstix Overo or CompuLab CM-T35/CM-T3730
-	  - IGEP v2
-	  - OMAP3EVM
-	  - SDP3430
-	  - Zoom2
-
-config SND_OMAP_SOC_OMAP_ABE_TWL6040
-	tristate "SoC Audio support for OMAP boards using ABE and twl6040 codec"
-	depends on TWL6040_CORE && SND_SDMA_SOC && COMMON_CLK
-	depends on ARCH_OMAP4 || (SOC_OMAP5 && MFD_PALMAS) || COMPILE_TEST
-	select SND_OMAP_SOC_DMIC
-	select SND_OMAP_SOC_MCPDM
-	select SND_SOC_TWL6040
-	select SND_SOC_DMIC
-	select COMMON_CLK_PALMAS if (SOC_OMAP5 && MFD_PALMAS)
-	select CLK_TWL6040
-	help
-	  Say Y if you want to add support for SoC audio on OMAP boards using
-	  ABE and twl6040 codec. This driver currently supports:
-	  - SDP4430/Blaze boards
-	  - PandaBoard (4430)
-	  - PandaBoardES (4460)
-	  - omap5-uevm (5432)
-
-config SND_OMAP_SOC_OMAP3_PANDORA
-	tristate "SoC Audio support for OMAP3 Pandora"
-	depends on TWL4030_CORE && SND_SDMA_SOC && MACH_OMAP3_PANDORA
-	select SND_OMAP_SOC_MCBSP
-	select SND_SOC_TWL4030
-	help
-	  Say Y if you want to add support for SoC audio on the OMAP3 Pandora.
diff -Naur --no-dereference a/sound/soc/omap/Makefile b/sound/soc/omap/Makefile
--- a/sound/soc/omap/Makefile	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/omap/Makefile	1969-12-31 19:00:00.000000000 -0500
@@ -1,32 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-# OMAP Platform Support
-snd-soc-sdma-objs := sdma-pcm.o
-snd-soc-omap-dmic-objs := omap-dmic.o
-snd-soc-omap-mcbsp-objs := omap-mcbsp.o mcbsp.o
-snd-soc-omap-mcpdm-objs := omap-mcpdm.o
-snd-soc-omap-hdmi-audio-objs := omap-hdmi-audio.o
-
-obj-$(CONFIG_SND_SDMA_SOC) += snd-soc-sdma.o
-obj-$(CONFIG_SND_OMAP_SOC_DMIC) += snd-soc-omap-dmic.o
-obj-$(CONFIG_SND_OMAP_SOC_MCBSP) += snd-soc-omap-mcbsp.o
-obj-$(CONFIG_SND_OMAP_SOC_MCPDM) += snd-soc-omap-mcpdm.o
-obj-$(CONFIG_SND_OMAP_SOC_HDMI_AUDIO) += snd-soc-omap-hdmi-audio.o
-
-# OMAP Machine Support
-snd-soc-n810-objs := n810.o
-snd-soc-rx51-objs := rx51.o
-snd-soc-ams-delta-objs := ams-delta.o
-snd-soc-osk5912-objs := osk5912.o
-snd-soc-am3517evm-objs := am3517evm.o
-snd-soc-omap-abe-twl6040-objs := omap-abe-twl6040.o
-snd-soc-omap-twl4030-objs := omap-twl4030.o
-snd-soc-omap3pandora-objs := omap3pandora.o
-
-obj-$(CONFIG_SND_OMAP_SOC_N810) += snd-soc-n810.o
-obj-$(CONFIG_SND_OMAP_SOC_RX51) += snd-soc-rx51.o
-obj-$(CONFIG_SND_OMAP_SOC_AMS_DELTA) += snd-soc-ams-delta.o
-obj-$(CONFIG_SND_OMAP_SOC_OSK5912) += snd-soc-osk5912.o
-obj-$(CONFIG_SND_OMAP_SOC_AM3517EVM) += snd-soc-am3517evm.o
-obj-$(CONFIG_SND_OMAP_SOC_OMAP_ABE_TWL6040) += snd-soc-omap-abe-twl6040.o
-obj-$(CONFIG_SND_OMAP_SOC_OMAP_TWL4030) += snd-soc-omap-twl4030.o
-obj-$(CONFIG_SND_OMAP_SOC_OMAP3_PANDORA) += snd-soc-omap3pandora.o
diff -Naur --no-dereference a/sound/soc/omap/mcbsp.c b/sound/soc/omap/mcbsp.c
--- a/sound/soc/omap/mcbsp.c	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/omap/mcbsp.c	1969-12-31 19:00:00.000000000 -0500
@@ -1,1104 +0,0 @@
-/*
- * sound/soc/omap/mcbsp.c
- *
- * Copyright (C) 2004 Nokia Corporation
- * Author: Samuel Ortiz <samuel.ortiz@nokia.com>
- *
- * Contact: Jarkko Nikula <jarkko.nikula@bitmer.com>
- *          Peter Ujfalusi <peter.ujfalusi@ti.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * Multichannel mode not supported.
- */
-
-#include <linux/module.h>
-#include <linux/init.h>
-#include <linux/device.h>
-#include <linux/platform_device.h>
-#include <linux/interrupt.h>
-#include <linux/err.h>
-#include <linux/clk.h>
-#include <linux/delay.h>
-#include <linux/io.h>
-#include <linux/slab.h>
-#include <linux/pm_runtime.h>
-
-#include <linux/platform_data/asoc-ti-mcbsp.h>
-
-#include "mcbsp.h"
-
-static void omap_mcbsp_write(struct omap_mcbsp *mcbsp, u16 reg, u32 val)
-{
-	void __iomem *addr = mcbsp->io_base + reg * mcbsp->pdata->reg_step;
-
-	if (mcbsp->pdata->reg_size == 2) {
-		((u16 *)mcbsp->reg_cache)[reg] = (u16)val;
-		writew_relaxed((u16)val, addr);
-	} else {
-		((u32 *)mcbsp->reg_cache)[reg] = val;
-		writel_relaxed(val, addr);
-	}
-}
-
-static int omap_mcbsp_read(struct omap_mcbsp *mcbsp, u16 reg, bool from_cache)
-{
-	void __iomem *addr = mcbsp->io_base + reg * mcbsp->pdata->reg_step;
-
-	if (mcbsp->pdata->reg_size == 2) {
-		return !from_cache ? readw_relaxed(addr) :
-				     ((u16 *)mcbsp->reg_cache)[reg];
-	} else {
-		return !from_cache ? readl_relaxed(addr) :
-				     ((u32 *)mcbsp->reg_cache)[reg];
-	}
-}
-
-static void omap_mcbsp_st_write(struct omap_mcbsp *mcbsp, u16 reg, u32 val)
-{
-	writel_relaxed(val, mcbsp->st_data->io_base_st + reg);
-}
-
-static int omap_mcbsp_st_read(struct omap_mcbsp *mcbsp, u16 reg)
-{
-	return readl_relaxed(mcbsp->st_data->io_base_st + reg);
-}
-
-#define MCBSP_READ(mcbsp, reg) \
-		omap_mcbsp_read(mcbsp, OMAP_MCBSP_REG_##reg, 0)
-#define MCBSP_WRITE(mcbsp, reg, val) \
-		omap_mcbsp_write(mcbsp, OMAP_MCBSP_REG_##reg, val)
-#define MCBSP_READ_CACHE(mcbsp, reg) \
-		omap_mcbsp_read(mcbsp, OMAP_MCBSP_REG_##reg, 1)
-
-#define MCBSP_ST_READ(mcbsp, reg) \
-			omap_mcbsp_st_read(mcbsp, OMAP_ST_REG_##reg)
-#define MCBSP_ST_WRITE(mcbsp, reg, val) \
-			omap_mcbsp_st_write(mcbsp, OMAP_ST_REG_##reg, val)
-
-static void omap_mcbsp_dump_reg(struct omap_mcbsp *mcbsp)
-{
-	dev_dbg(mcbsp->dev, "**** McBSP%d regs ****\n", mcbsp->id);
-	dev_dbg(mcbsp->dev, "DRR2:  0x%04x\n",
-			MCBSP_READ(mcbsp, DRR2));
-	dev_dbg(mcbsp->dev, "DRR1:  0x%04x\n",
-			MCBSP_READ(mcbsp, DRR1));
-	dev_dbg(mcbsp->dev, "DXR2:  0x%04x\n",
-			MCBSP_READ(mcbsp, DXR2));
-	dev_dbg(mcbsp->dev, "DXR1:  0x%04x\n",
-			MCBSP_READ(mcbsp, DXR1));
-	dev_dbg(mcbsp->dev, "SPCR2: 0x%04x\n",
-			MCBSP_READ(mcbsp, SPCR2));
-	dev_dbg(mcbsp->dev, "SPCR1: 0x%04x\n",
-			MCBSP_READ(mcbsp, SPCR1));
-	dev_dbg(mcbsp->dev, "RCR2:  0x%04x\n",
-			MCBSP_READ(mcbsp, RCR2));
-	dev_dbg(mcbsp->dev, "RCR1:  0x%04x\n",
-			MCBSP_READ(mcbsp, RCR1));
-	dev_dbg(mcbsp->dev, "XCR2:  0x%04x\n",
-			MCBSP_READ(mcbsp, XCR2));
-	dev_dbg(mcbsp->dev, "XCR1:  0x%04x\n",
-			MCBSP_READ(mcbsp, XCR1));
-	dev_dbg(mcbsp->dev, "SRGR2: 0x%04x\n",
-			MCBSP_READ(mcbsp, SRGR2));
-	dev_dbg(mcbsp->dev, "SRGR1: 0x%04x\n",
-			MCBSP_READ(mcbsp, SRGR1));
-	dev_dbg(mcbsp->dev, "PCR0:  0x%04x\n",
-			MCBSP_READ(mcbsp, PCR0));
-	dev_dbg(mcbsp->dev, "***********************\n");
-}
-
-static irqreturn_t omap_mcbsp_irq_handler(int irq, void *dev_id)
-{
-	struct omap_mcbsp *mcbsp = dev_id;
-	u16 irqst;
-
-	irqst = MCBSP_READ(mcbsp, IRQST);
-	dev_dbg(mcbsp->dev, "IRQ callback : 0x%x\n", irqst);
-
-	if (irqst & RSYNCERREN)
-		dev_err(mcbsp->dev, "RX Frame Sync Error!\n");
-	if (irqst & RFSREN)
-		dev_dbg(mcbsp->dev, "RX Frame Sync\n");
-	if (irqst & REOFEN)
-		dev_dbg(mcbsp->dev, "RX End Of Frame\n");
-	if (irqst & RRDYEN)
-		dev_dbg(mcbsp->dev, "RX Buffer Threshold Reached\n");
-	if (irqst & RUNDFLEN)
-		dev_err(mcbsp->dev, "RX Buffer Underflow!\n");
-	if (irqst & ROVFLEN)
-		dev_err(mcbsp->dev, "RX Buffer Overflow!\n");
-
-	if (irqst & XSYNCERREN)
-		dev_err(mcbsp->dev, "TX Frame Sync Error!\n");
-	if (irqst & XFSXEN)
-		dev_dbg(mcbsp->dev, "TX Frame Sync\n");
-	if (irqst & XEOFEN)
-		dev_dbg(mcbsp->dev, "TX End Of Frame\n");
-	if (irqst & XRDYEN)
-		dev_dbg(mcbsp->dev, "TX Buffer threshold Reached\n");
-	if (irqst & XUNDFLEN)
-		dev_err(mcbsp->dev, "TX Buffer Underflow!\n");
-	if (irqst & XOVFLEN)
-		dev_err(mcbsp->dev, "TX Buffer Overflow!\n");
-	if (irqst & XEMPTYEOFEN)
-		dev_dbg(mcbsp->dev, "TX Buffer empty at end of frame\n");
-
-	MCBSP_WRITE(mcbsp, IRQST, irqst);
-
-	return IRQ_HANDLED;
-}
-
-static irqreturn_t omap_mcbsp_tx_irq_handler(int irq, void *dev_id)
-{
-	struct omap_mcbsp *mcbsp_tx = dev_id;
-	u16 irqst_spcr2;
-
-	irqst_spcr2 = MCBSP_READ(mcbsp_tx, SPCR2);
-	dev_dbg(mcbsp_tx->dev, "TX IRQ callback : 0x%x\n", irqst_spcr2);
-
-	if (irqst_spcr2 & XSYNC_ERR) {
-		dev_err(mcbsp_tx->dev, "TX Frame Sync Error! : 0x%x\n",
-			irqst_spcr2);
-		/* Writing zero to XSYNC_ERR clears the IRQ */
-		MCBSP_WRITE(mcbsp_tx, SPCR2, MCBSP_READ_CACHE(mcbsp_tx, SPCR2));
-	}
-
-	return IRQ_HANDLED;
-}
-
-static irqreturn_t omap_mcbsp_rx_irq_handler(int irq, void *dev_id)
-{
-	struct omap_mcbsp *mcbsp_rx = dev_id;
-	u16 irqst_spcr1;
-
-	irqst_spcr1 = MCBSP_READ(mcbsp_rx, SPCR1);
-	dev_dbg(mcbsp_rx->dev, "RX IRQ callback : 0x%x\n", irqst_spcr1);
-
-	if (irqst_spcr1 & RSYNC_ERR) {
-		dev_err(mcbsp_rx->dev, "RX Frame Sync Error! : 0x%x\n",
-			irqst_spcr1);
-		/* Writing zero to RSYNC_ERR clears the IRQ */
-		MCBSP_WRITE(mcbsp_rx, SPCR1, MCBSP_READ_CACHE(mcbsp_rx, SPCR1));
-	}
-
-	return IRQ_HANDLED;
-}
-
-/*
- * omap_mcbsp_config simply write a config to the
- * appropriate McBSP.
- * You either call this function or set the McBSP registers
- * by yourself before calling omap_mcbsp_start().
- */
-void omap_mcbsp_config(struct omap_mcbsp *mcbsp,
-		       const struct omap_mcbsp_reg_cfg *config)
-{
-	dev_dbg(mcbsp->dev, "Configuring McBSP%d  phys_base: 0x%08lx\n",
-			mcbsp->id, mcbsp->phys_base);
-
-	/* We write the given config */
-	MCBSP_WRITE(mcbsp, SPCR2, config->spcr2);
-	MCBSP_WRITE(mcbsp, SPCR1, config->spcr1);
-	MCBSP_WRITE(mcbsp, RCR2, config->rcr2);
-	MCBSP_WRITE(mcbsp, RCR1, config->rcr1);
-	MCBSP_WRITE(mcbsp, XCR2, config->xcr2);
-	MCBSP_WRITE(mcbsp, XCR1, config->xcr1);
-	MCBSP_WRITE(mcbsp, SRGR2, config->srgr2);
-	MCBSP_WRITE(mcbsp, SRGR1, config->srgr1);
-	MCBSP_WRITE(mcbsp, MCR2, config->mcr2);
-	MCBSP_WRITE(mcbsp, MCR1, config->mcr1);
-	MCBSP_WRITE(mcbsp, PCR0, config->pcr0);
-	if (mcbsp->pdata->has_ccr) {
-		MCBSP_WRITE(mcbsp, XCCR, config->xccr);
-		MCBSP_WRITE(mcbsp, RCCR, config->rccr);
-	}
-	/* Enable wakeup behavior */
-	if (mcbsp->pdata->has_wakeup)
-		MCBSP_WRITE(mcbsp, WAKEUPEN, XRDYEN | RRDYEN);
-
-	/* Enable TX/RX sync error interrupts by default */
-	if (mcbsp->irq)
-		MCBSP_WRITE(mcbsp, IRQEN, RSYNCERREN | XSYNCERREN |
-			    RUNDFLEN | ROVFLEN | XUNDFLEN | XOVFLEN);
-}
-
-/**
- * omap_mcbsp_dma_reg_params - returns the address of mcbsp data register
- * @id - mcbsp id
- * @stream - indicates the direction of data flow (rx or tx)
- *
- * Returns the address of mcbsp data transmit register or data receive register
- * to be used by DMA for transferring/receiving data based on the value of
- * @stream for the requested mcbsp given by @id
- */
-static int omap_mcbsp_dma_reg_params(struct omap_mcbsp *mcbsp,
-				     unsigned int stream)
-{
-	int data_reg;
-
-	if (mcbsp->pdata->reg_size == 2) {
-		if (stream)
-			data_reg = OMAP_MCBSP_REG_DRR1;
-		else
-			data_reg = OMAP_MCBSP_REG_DXR1;
-	} else {
-		if (stream)
-			data_reg = OMAP_MCBSP_REG_DRR;
-		else
-			data_reg = OMAP_MCBSP_REG_DXR;
-	}
-
-	return mcbsp->phys_dma_base + data_reg * mcbsp->pdata->reg_step;
-}
-
-static void omap_st_on(struct omap_mcbsp *mcbsp)
-{
-	unsigned int w;
-
-	if (mcbsp->pdata->force_ick_on)
-		mcbsp->pdata->force_ick_on(mcbsp->st_data->mcbsp_iclk, true);
-
-	/* Disable Sidetone clock auto-gating for normal operation */
-	w = MCBSP_ST_READ(mcbsp, SYSCONFIG);
-	MCBSP_ST_WRITE(mcbsp, SYSCONFIG, w & ~(ST_AUTOIDLE));
-
-	/* Enable McBSP Sidetone */
-	w = MCBSP_READ(mcbsp, SSELCR);
-	MCBSP_WRITE(mcbsp, SSELCR, w | SIDETONEEN);
-
-	/* Enable Sidetone from Sidetone Core */
-	w = MCBSP_ST_READ(mcbsp, SSELCR);
-	MCBSP_ST_WRITE(mcbsp, SSELCR, w | ST_SIDETONEEN);
-}
-
-static void omap_st_off(struct omap_mcbsp *mcbsp)
-{
-	unsigned int w;
-
-	w = MCBSP_ST_READ(mcbsp, SSELCR);
-	MCBSP_ST_WRITE(mcbsp, SSELCR, w & ~(ST_SIDETONEEN));
-
-	w = MCBSP_READ(mcbsp, SSELCR);
-	MCBSP_WRITE(mcbsp, SSELCR, w & ~(SIDETONEEN));
-
-	/* Enable Sidetone clock auto-gating to reduce power consumption */
-	w = MCBSP_ST_READ(mcbsp, SYSCONFIG);
-	MCBSP_ST_WRITE(mcbsp, SYSCONFIG, w | ST_AUTOIDLE);
-
-	if (mcbsp->pdata->force_ick_on)
-		mcbsp->pdata->force_ick_on(mcbsp->st_data->mcbsp_iclk, false);
-}
-
-static void omap_st_fir_write(struct omap_mcbsp *mcbsp, s16 *fir)
-{
-	u16 val, i;
-
-	val = MCBSP_ST_READ(mcbsp, SSELCR);
-
-	if (val & ST_COEFFWREN)
-		MCBSP_ST_WRITE(mcbsp, SSELCR, val & ~(ST_COEFFWREN));
-
-	MCBSP_ST_WRITE(mcbsp, SSELCR, val | ST_COEFFWREN);
-
-	for (i = 0; i < 128; i++)
-		MCBSP_ST_WRITE(mcbsp, SFIRCR, fir[i]);
-
-	i = 0;
-
-	val = MCBSP_ST_READ(mcbsp, SSELCR);
-	while (!(val & ST_COEFFWRDONE) && (++i < 1000))
-		val = MCBSP_ST_READ(mcbsp, SSELCR);
-
-	MCBSP_ST_WRITE(mcbsp, SSELCR, val & ~(ST_COEFFWREN));
-
-	if (i == 1000)
-		dev_err(mcbsp->dev, "McBSP FIR load error!\n");
-}
-
-static void omap_st_chgain(struct omap_mcbsp *mcbsp)
-{
-	u16 w;
-	struct omap_mcbsp_st_data *st_data = mcbsp->st_data;
-
-	w = MCBSP_ST_READ(mcbsp, SSELCR);
-
-	MCBSP_ST_WRITE(mcbsp, SGAINCR, ST_CH0GAIN(st_data->ch0gain) | \
-		      ST_CH1GAIN(st_data->ch1gain));
-}
-
-int omap_st_set_chgain(struct omap_mcbsp *mcbsp, int channel, s16 chgain)
-{
-	struct omap_mcbsp_st_data *st_data = mcbsp->st_data;
-	int ret = 0;
-
-	if (!st_data)
-		return -ENOENT;
-
-	spin_lock_irq(&mcbsp->lock);
-	if (channel == 0)
-		st_data->ch0gain = chgain;
-	else if (channel == 1)
-		st_data->ch1gain = chgain;
-	else
-		ret = -EINVAL;
-
-	if (st_data->enabled)
-		omap_st_chgain(mcbsp);
-	spin_unlock_irq(&mcbsp->lock);
-
-	return ret;
-}
-
-int omap_st_get_chgain(struct omap_mcbsp *mcbsp, int channel, s16 *chgain)
-{
-	struct omap_mcbsp_st_data *st_data = mcbsp->st_data;
-	int ret = 0;
-
-	if (!st_data)
-		return -ENOENT;
-
-	spin_lock_irq(&mcbsp->lock);
-	if (channel == 0)
-		*chgain = st_data->ch0gain;
-	else if (channel == 1)
-		*chgain = st_data->ch1gain;
-	else
-		ret = -EINVAL;
-	spin_unlock_irq(&mcbsp->lock);
-
-	return ret;
-}
-
-static int omap_st_start(struct omap_mcbsp *mcbsp)
-{
-	struct omap_mcbsp_st_data *st_data = mcbsp->st_data;
-
-	if (st_data->enabled && !st_data->running) {
-		omap_st_fir_write(mcbsp, st_data->taps);
-		omap_st_chgain(mcbsp);
-
-		if (!mcbsp->free) {
-			omap_st_on(mcbsp);
-			st_data->running = 1;
-		}
-	}
-
-	return 0;
-}
-
-int omap_st_enable(struct omap_mcbsp *mcbsp)
-{
-	struct omap_mcbsp_st_data *st_data = mcbsp->st_data;
-
-	if (!st_data)
-		return -ENODEV;
-
-	spin_lock_irq(&mcbsp->lock);
-	st_data->enabled = 1;
-	omap_st_start(mcbsp);
-	spin_unlock_irq(&mcbsp->lock);
-
-	return 0;
-}
-
-static int omap_st_stop(struct omap_mcbsp *mcbsp)
-{
-	struct omap_mcbsp_st_data *st_data = mcbsp->st_data;
-
-	if (st_data->running) {
-		if (!mcbsp->free) {
-			omap_st_off(mcbsp);
-			st_data->running = 0;
-		}
-	}
-
-	return 0;
-}
-
-int omap_st_disable(struct omap_mcbsp *mcbsp)
-{
-	struct omap_mcbsp_st_data *st_data = mcbsp->st_data;
-	int ret = 0;
-
-	if (!st_data)
-		return -ENODEV;
-
-	spin_lock_irq(&mcbsp->lock);
-	omap_st_stop(mcbsp);
-	st_data->enabled = 0;
-	spin_unlock_irq(&mcbsp->lock);
-
-	return ret;
-}
-
-int omap_st_is_enabled(struct omap_mcbsp *mcbsp)
-{
-	struct omap_mcbsp_st_data *st_data = mcbsp->st_data;
-
-	if (!st_data)
-		return -ENODEV;
-
-	return st_data->enabled;
-}
-
-/*
- * omap_mcbsp_set_rx_threshold configures the transmit threshold in words.
- * The threshold parameter is 1 based, and it is converted (threshold - 1)
- * for the THRSH2 register.
- */
-void omap_mcbsp_set_tx_threshold(struct omap_mcbsp *mcbsp, u16 threshold)
-{
-	if (mcbsp->pdata->buffer_size == 0)
-		return;
-
-	if (threshold && threshold <= mcbsp->max_tx_thres)
-		MCBSP_WRITE(mcbsp, THRSH2, threshold - 1);
-}
-
-/*
- * omap_mcbsp_set_rx_threshold configures the receive threshold in words.
- * The threshold parameter is 1 based, and it is converted (threshold - 1)
- * for the THRSH1 register.
- */
-void omap_mcbsp_set_rx_threshold(struct omap_mcbsp *mcbsp, u16 threshold)
-{
-	if (mcbsp->pdata->buffer_size == 0)
-		return;
-
-	if (threshold && threshold <= mcbsp->max_rx_thres)
-		MCBSP_WRITE(mcbsp, THRSH1, threshold - 1);
-}
-
-/*
- * omap_mcbsp_get_tx_delay returns the number of used slots in the McBSP FIFO
- */
-u16 omap_mcbsp_get_tx_delay(struct omap_mcbsp *mcbsp)
-{
-	u16 buffstat;
-
-	if (mcbsp->pdata->buffer_size == 0)
-		return 0;
-
-	/* Returns the number of free locations in the buffer */
-	buffstat = MCBSP_READ(mcbsp, XBUFFSTAT);
-
-	/* Number of slots are different in McBSP ports */
-	return mcbsp->pdata->buffer_size - buffstat;
-}
-
-/*
- * omap_mcbsp_get_rx_delay returns the number of free slots in the McBSP FIFO
- * to reach the threshold value (when the DMA will be triggered to read it)
- */
-u16 omap_mcbsp_get_rx_delay(struct omap_mcbsp *mcbsp)
-{
-	u16 buffstat, threshold;
-
-	if (mcbsp->pdata->buffer_size == 0)
-		return 0;
-
-	/* Returns the number of used locations in the buffer */
-	buffstat = MCBSP_READ(mcbsp, RBUFFSTAT);
-	/* RX threshold */
-	threshold = MCBSP_READ(mcbsp, THRSH1);
-
-	/* Return the number of location till we reach the threshold limit */
-	if (threshold <= buffstat)
-		return 0;
-	else
-		return threshold - buffstat;
-}
-
-int omap_mcbsp_request(struct omap_mcbsp *mcbsp)
-{
-	void *reg_cache;
-	int err;
-
-	reg_cache = kzalloc(mcbsp->reg_cache_size, GFP_KERNEL);
-	if (!reg_cache) {
-		return -ENOMEM;
-	}
-
-	spin_lock(&mcbsp->lock);
-	if (!mcbsp->free) {
-		dev_err(mcbsp->dev, "McBSP%d is currently in use\n",
-			mcbsp->id);
-		err = -EBUSY;
-		goto err_kfree;
-	}
-
-	mcbsp->free = false;
-	mcbsp->reg_cache = reg_cache;
-	spin_unlock(&mcbsp->lock);
-
-	if (mcbsp->pdata && mcbsp->pdata->ops && mcbsp->pdata->ops->request)
-		mcbsp->pdata->ops->request(mcbsp->id - 1);
-
-	/*
-	 * Make sure that transmitter, receiver and sample-rate generator are
-	 * not running before activating IRQs.
-	 */
-	MCBSP_WRITE(mcbsp, SPCR1, 0);
-	MCBSP_WRITE(mcbsp, SPCR2, 0);
-
-	if (mcbsp->irq) {
-		err = request_irq(mcbsp->irq, omap_mcbsp_irq_handler, 0,
-				  "McBSP", (void *)mcbsp);
-		if (err != 0) {
-			dev_err(mcbsp->dev, "Unable to request IRQ\n");
-			goto err_clk_disable;
-		}
-	} else {
-		err = request_irq(mcbsp->tx_irq, omap_mcbsp_tx_irq_handler, 0,
-				  "McBSP TX", (void *)mcbsp);
-		if (err != 0) {
-			dev_err(mcbsp->dev, "Unable to request TX IRQ\n");
-			goto err_clk_disable;
-		}
-
-		err = request_irq(mcbsp->rx_irq, omap_mcbsp_rx_irq_handler, 0,
-				  "McBSP RX", (void *)mcbsp);
-		if (err != 0) {
-			dev_err(mcbsp->dev, "Unable to request RX IRQ\n");
-			goto err_free_irq;
-		}
-	}
-
-	return 0;
-err_free_irq:
-	free_irq(mcbsp->tx_irq, (void *)mcbsp);
-err_clk_disable:
-	if (mcbsp->pdata && mcbsp->pdata->ops && mcbsp->pdata->ops->free)
-		mcbsp->pdata->ops->free(mcbsp->id - 1);
-
-	/* Disable wakeup behavior */
-	if (mcbsp->pdata->has_wakeup)
-		MCBSP_WRITE(mcbsp, WAKEUPEN, 0);
-
-	spin_lock(&mcbsp->lock);
-	mcbsp->free = true;
-	mcbsp->reg_cache = NULL;
-err_kfree:
-	spin_unlock(&mcbsp->lock);
-	kfree(reg_cache);
-
-	return err;
-}
-
-void omap_mcbsp_free(struct omap_mcbsp *mcbsp)
-{
-	void *reg_cache;
-
-	if (mcbsp->pdata && mcbsp->pdata->ops && mcbsp->pdata->ops->free)
-		mcbsp->pdata->ops->free(mcbsp->id - 1);
-
-	/* Disable wakeup behavior */
-	if (mcbsp->pdata->has_wakeup)
-		MCBSP_WRITE(mcbsp, WAKEUPEN, 0);
-
-	/* Disable interrupt requests */
-	if (mcbsp->irq)
-		MCBSP_WRITE(mcbsp, IRQEN, 0);
-
-	if (mcbsp->irq) {
-		free_irq(mcbsp->irq, (void *)mcbsp);
-	} else {
-		free_irq(mcbsp->rx_irq, (void *)mcbsp);
-		free_irq(mcbsp->tx_irq, (void *)mcbsp);
-	}
-
-	reg_cache = mcbsp->reg_cache;
-
-	/*
-	 * Select CLKS source from internal source unconditionally before
-	 * marking the McBSP port as free.
-	 * If the external clock source via MCBSP_CLKS pin has been selected the
-	 * system will refuse to enter idle if the CLKS pin source is not reset
-	 * back to internal source.
-	 */
-	if (!mcbsp_omap1())
-		omap2_mcbsp_set_clks_src(mcbsp, MCBSP_CLKS_PRCM_SRC);
-
-	spin_lock(&mcbsp->lock);
-	if (mcbsp->free)
-		dev_err(mcbsp->dev, "McBSP%d was not reserved\n", mcbsp->id);
-	else
-		mcbsp->free = true;
-	mcbsp->reg_cache = NULL;
-	spin_unlock(&mcbsp->lock);
-
-	kfree(reg_cache);
-}
-
-/*
- * Here we start the McBSP, by enabling transmitter, receiver or both.
- * If no transmitter or receiver is active prior calling, then sample-rate
- * generator and frame sync are started.
- */
-void omap_mcbsp_start(struct omap_mcbsp *mcbsp, int tx, int rx)
-{
-	int enable_srg = 0;
-	u16 w;
-
-	if (mcbsp->st_data)
-		omap_st_start(mcbsp);
-
-	/* Only enable SRG, if McBSP is master */
-	w = MCBSP_READ_CACHE(mcbsp, PCR0);
-	if (w & (FSXM | FSRM | CLKXM | CLKRM))
-		enable_srg = !((MCBSP_READ_CACHE(mcbsp, SPCR2) |
-				MCBSP_READ_CACHE(mcbsp, SPCR1)) & 1);
-
-	if (enable_srg) {
-		/* Start the sample generator */
-		w = MCBSP_READ_CACHE(mcbsp, SPCR2);
-		MCBSP_WRITE(mcbsp, SPCR2, w | (1 << 6));
-	}
-
-	/* Enable transmitter and receiver */
-	tx &= 1;
-	w = MCBSP_READ_CACHE(mcbsp, SPCR2);
-	MCBSP_WRITE(mcbsp, SPCR2, w | tx);
-
-	rx &= 1;
-	w = MCBSP_READ_CACHE(mcbsp, SPCR1);
-	MCBSP_WRITE(mcbsp, SPCR1, w | rx);
-
-	/*
-	 * Worst case: CLKSRG*2 = 8000khz: (1/8000) * 2 * 2 usec
-	 * REVISIT: 100us may give enough time for two CLKSRG, however
-	 * due to some unknown PM related, clock gating etc. reason it
-	 * is now at 500us.
-	 */
-	udelay(500);
-
-	if (enable_srg) {
-		/* Start frame sync */
-		w = MCBSP_READ_CACHE(mcbsp, SPCR2);
-		MCBSP_WRITE(mcbsp, SPCR2, w | (1 << 7));
-	}
-
-	if (mcbsp->pdata->has_ccr) {
-		/* Release the transmitter and receiver */
-		w = MCBSP_READ_CACHE(mcbsp, XCCR);
-		w &= ~(tx ? XDISABLE : 0);
-		MCBSP_WRITE(mcbsp, XCCR, w);
-		w = MCBSP_READ_CACHE(mcbsp, RCCR);
-		w &= ~(rx ? RDISABLE : 0);
-		MCBSP_WRITE(mcbsp, RCCR, w);
-	}
-
-	/* Dump McBSP Regs */
-	omap_mcbsp_dump_reg(mcbsp);
-}
-
-void omap_mcbsp_stop(struct omap_mcbsp *mcbsp, int tx, int rx)
-{
-	int idle;
-	u16 w;
-
-	/* Reset transmitter */
-	tx &= 1;
-	if (mcbsp->pdata->has_ccr) {
-		w = MCBSP_READ_CACHE(mcbsp, XCCR);
-		w |= (tx ? XDISABLE : 0);
-		MCBSP_WRITE(mcbsp, XCCR, w);
-	}
-	w = MCBSP_READ_CACHE(mcbsp, SPCR2);
-	MCBSP_WRITE(mcbsp, SPCR2, w & ~tx);
-
-	/* Reset receiver */
-	rx &= 1;
-	if (mcbsp->pdata->has_ccr) {
-		w = MCBSP_READ_CACHE(mcbsp, RCCR);
-		w |= (rx ? RDISABLE : 0);
-		MCBSP_WRITE(mcbsp, RCCR, w);
-	}
-	w = MCBSP_READ_CACHE(mcbsp, SPCR1);
-	MCBSP_WRITE(mcbsp, SPCR1, w & ~rx);
-
-	idle = !((MCBSP_READ_CACHE(mcbsp, SPCR2) |
-			MCBSP_READ_CACHE(mcbsp, SPCR1)) & 1);
-
-	if (idle) {
-		/* Reset the sample rate generator */
-		w = MCBSP_READ_CACHE(mcbsp, SPCR2);
-		MCBSP_WRITE(mcbsp, SPCR2, w & ~(1 << 6));
-	}
-
-	if (mcbsp->st_data)
-		omap_st_stop(mcbsp);
-}
-
-int omap2_mcbsp_set_clks_src(struct omap_mcbsp *mcbsp, u8 fck_src_id)
-{
-	struct clk *fck_src;
-	const char *src;
-	int r;
-
-	if (fck_src_id == MCBSP_CLKS_PAD_SRC)
-		src = "pad_fck";
-	else if (fck_src_id == MCBSP_CLKS_PRCM_SRC)
-		src = "prcm_fck";
-	else
-		return -EINVAL;
-
-	fck_src = clk_get(mcbsp->dev, src);
-	if (IS_ERR(fck_src)) {
-		dev_err(mcbsp->dev, "CLKS: could not clk_get() %s\n", src);
-		return -EINVAL;
-	}
-
-	pm_runtime_put_sync(mcbsp->dev);
-
-	r = clk_set_parent(mcbsp->fclk, fck_src);
-	if (r) {
-		dev_err(mcbsp->dev, "CLKS: could not clk_set_parent() to %s\n",
-			src);
-		clk_put(fck_src);
-		return r;
-	}
-
-	pm_runtime_get_sync(mcbsp->dev);
-
-	clk_put(fck_src);
-
-	return 0;
-
-}
-
-#define max_thres(m)			(mcbsp->pdata->buffer_size)
-#define valid_threshold(m, val)		((val) <= max_thres(m))
-#define THRESHOLD_PROP_BUILDER(prop)					\
-static ssize_t prop##_show(struct device *dev,				\
-			struct device_attribute *attr, char *buf)	\
-{									\
-	struct omap_mcbsp *mcbsp = dev_get_drvdata(dev);		\
-									\
-	return sprintf(buf, "%u\n", mcbsp->prop);			\
-}									\
-									\
-static ssize_t prop##_store(struct device *dev,				\
-				struct device_attribute *attr,		\
-				const char *buf, size_t size)		\
-{									\
-	struct omap_mcbsp *mcbsp = dev_get_drvdata(dev);		\
-	unsigned long val;						\
-	int status;							\
-									\
-	status = kstrtoul(buf, 0, &val);				\
-	if (status)							\
-		return status;						\
-									\
-	if (!valid_threshold(mcbsp, val))				\
-		return -EDOM;						\
-									\
-	mcbsp->prop = val;						\
-	return size;							\
-}									\
-									\
-static DEVICE_ATTR(prop, 0644, prop##_show, prop##_store);
-
-THRESHOLD_PROP_BUILDER(max_tx_thres);
-THRESHOLD_PROP_BUILDER(max_rx_thres);
-
-static const char *dma_op_modes[] = {
-	"element", "threshold",
-};
-
-static ssize_t dma_op_mode_show(struct device *dev,
-			struct device_attribute *attr, char *buf)
-{
-	struct omap_mcbsp *mcbsp = dev_get_drvdata(dev);
-	int dma_op_mode, i = 0;
-	ssize_t len = 0;
-	const char * const *s;
-
-	dma_op_mode = mcbsp->dma_op_mode;
-
-	for (s = &dma_op_modes[i]; i < ARRAY_SIZE(dma_op_modes); s++, i++) {
-		if (dma_op_mode == i)
-			len += sprintf(buf + len, "[%s] ", *s);
-		else
-			len += sprintf(buf + len, "%s ", *s);
-	}
-	len += sprintf(buf + len, "\n");
-
-	return len;
-}
-
-static ssize_t dma_op_mode_store(struct device *dev,
-				struct device_attribute *attr,
-				const char *buf, size_t size)
-{
-	struct omap_mcbsp *mcbsp = dev_get_drvdata(dev);
-	int i;
-
-	i = sysfs_match_string(dma_op_modes, buf);
-	if (i < 0)
-		return i;
-
-	spin_lock_irq(&mcbsp->lock);
-	if (!mcbsp->free) {
-		size = -EBUSY;
-		goto unlock;
-	}
-	mcbsp->dma_op_mode = i;
-
-unlock:
-	spin_unlock_irq(&mcbsp->lock);
-
-	return size;
-}
-
-static DEVICE_ATTR_RW(dma_op_mode);
-
-static const struct attribute *additional_attrs[] = {
-	&dev_attr_max_tx_thres.attr,
-	&dev_attr_max_rx_thres.attr,
-	&dev_attr_dma_op_mode.attr,
-	NULL,
-};
-
-static const struct attribute_group additional_attr_group = {
-	.attrs = (struct attribute **)additional_attrs,
-};
-
-static ssize_t st_taps_show(struct device *dev,
-			    struct device_attribute *attr, char *buf)
-{
-	struct omap_mcbsp *mcbsp = dev_get_drvdata(dev);
-	struct omap_mcbsp_st_data *st_data = mcbsp->st_data;
-	ssize_t status = 0;
-	int i;
-
-	spin_lock_irq(&mcbsp->lock);
-	for (i = 0; i < st_data->nr_taps; i++)
-		status += sprintf(&buf[status], (i ? ", %d" : "%d"),
-				  st_data->taps[i]);
-	if (i)
-		status += sprintf(&buf[status], "\n");
-	spin_unlock_irq(&mcbsp->lock);
-
-	return status;
-}
-
-static ssize_t st_taps_store(struct device *dev,
-			     struct device_attribute *attr,
-			     const char *buf, size_t size)
-{
-	struct omap_mcbsp *mcbsp = dev_get_drvdata(dev);
-	struct omap_mcbsp_st_data *st_data = mcbsp->st_data;
-	int val, tmp, status, i = 0;
-
-	spin_lock_irq(&mcbsp->lock);
-	memset(st_data->taps, 0, sizeof(st_data->taps));
-	st_data->nr_taps = 0;
-
-	do {
-		status = sscanf(buf, "%d%n", &val, &tmp);
-		if (status < 0 || status == 0) {
-			size = -EINVAL;
-			goto out;
-		}
-		if (val < -32768 || val > 32767) {
-			size = -EINVAL;
-			goto out;
-		}
-		st_data->taps[i++] = val;
-		buf += tmp;
-		if (*buf != ',')
-			break;
-		buf++;
-	} while (1);
-
-	st_data->nr_taps = i;
-
-out:
-	spin_unlock_irq(&mcbsp->lock);
-
-	return size;
-}
-
-static DEVICE_ATTR_RW(st_taps);
-
-static const struct attribute *sidetone_attrs[] = {
-	&dev_attr_st_taps.attr,
-	NULL,
-};
-
-static const struct attribute_group sidetone_attr_group = {
-	.attrs = (struct attribute **)sidetone_attrs,
-};
-
-static int omap_st_add(struct omap_mcbsp *mcbsp, struct resource *res)
-{
-	struct omap_mcbsp_st_data *st_data;
-	int err;
-
-	st_data = devm_kzalloc(mcbsp->dev, sizeof(*mcbsp->st_data), GFP_KERNEL);
-	if (!st_data)
-		return -ENOMEM;
-
-	st_data->mcbsp_iclk = clk_get(mcbsp->dev, "ick");
-	if (IS_ERR(st_data->mcbsp_iclk)) {
-		dev_warn(mcbsp->dev,
-			 "Failed to get ick, sidetone might be broken\n");
-		st_data->mcbsp_iclk = NULL;
-	}
-
-	st_data->io_base_st = devm_ioremap(mcbsp->dev, res->start,
-					   resource_size(res));
-	if (!st_data->io_base_st)
-		return -ENOMEM;
-
-	err = sysfs_create_group(&mcbsp->dev->kobj, &sidetone_attr_group);
-	if (err)
-		return err;
-
-	mcbsp->st_data = st_data;
-	return 0;
-}
-
-/*
- * McBSP1 and McBSP3 are directly mapped on 1610 and 1510.
- * 730 has only 2 McBSP, and both of them are MPU peripherals.
- */
-int omap_mcbsp_init(struct platform_device *pdev)
-{
-	struct omap_mcbsp *mcbsp = platform_get_drvdata(pdev);
-	struct resource *res;
-	int ret = 0;
-
-	spin_lock_init(&mcbsp->lock);
-	mcbsp->free = true;
-
-	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mpu");
-	if (!res)
-		res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-
-	mcbsp->io_base = devm_ioremap_resource(&pdev->dev, res);
-	if (IS_ERR(mcbsp->io_base))
-		return PTR_ERR(mcbsp->io_base);
-
-	mcbsp->phys_base = res->start;
-	mcbsp->reg_cache_size = resource_size(res);
-
-	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dma");
-	if (!res)
-		mcbsp->phys_dma_base = mcbsp->phys_base;
-	else
-		mcbsp->phys_dma_base = res->start;
-
-	/*
-	 * OMAP1, 2 uses two interrupt lines: TX, RX
-	 * OMAP2430, OMAP3 SoC have combined IRQ line as well.
-	 * OMAP4 and newer SoC only have the combined IRQ line.
-	 * Use the combined IRQ if available since it gives better debugging
-	 * possibilities.
-	 */
-	mcbsp->irq = platform_get_irq_byname(pdev, "common");
-	if (mcbsp->irq == -ENXIO) {
-		mcbsp->tx_irq = platform_get_irq_byname(pdev, "tx");
-
-		if (mcbsp->tx_irq == -ENXIO) {
-			mcbsp->irq = platform_get_irq(pdev, 0);
-			mcbsp->tx_irq = 0;
-		} else {
-			mcbsp->rx_irq = platform_get_irq_byname(pdev, "rx");
-			mcbsp->irq = 0;
-		}
-	}
-
-	if (!pdev->dev.of_node) {
-		res = platform_get_resource_byname(pdev, IORESOURCE_DMA, "tx");
-		if (!res) {
-			dev_err(&pdev->dev, "invalid tx DMA channel\n");
-			return -ENODEV;
-		}
-		mcbsp->dma_req[0] = res->start;
-		mcbsp->dma_data[0].filter_data = &mcbsp->dma_req[0];
-
-		res = platform_get_resource_byname(pdev, IORESOURCE_DMA, "rx");
-		if (!res) {
-			dev_err(&pdev->dev, "invalid rx DMA channel\n");
-			return -ENODEV;
-		}
-		mcbsp->dma_req[1] = res->start;
-		mcbsp->dma_data[1].filter_data = &mcbsp->dma_req[1];
-	} else {
-		mcbsp->dma_data[0].filter_data = "tx";
-		mcbsp->dma_data[1].filter_data = "rx";
-	}
-
-	mcbsp->dma_data[0].addr = omap_mcbsp_dma_reg_params(mcbsp, 0);
-	mcbsp->dma_data[0].maxburst = 4;
-
-	mcbsp->dma_data[1].addr = omap_mcbsp_dma_reg_params(mcbsp, 1);
-	mcbsp->dma_data[1].maxburst = 4;
-
-	mcbsp->fclk = clk_get(&pdev->dev, "fck");
-	if (IS_ERR(mcbsp->fclk)) {
-		ret = PTR_ERR(mcbsp->fclk);
-		dev_err(mcbsp->dev, "unable to get fck: %d\n", ret);
-		return ret;
-	}
-
-	mcbsp->dma_op_mode = MCBSP_DMA_MODE_ELEMENT;
-	if (mcbsp->pdata->buffer_size) {
-		/*
-		 * Initially configure the maximum thresholds to a safe value.
-		 * The McBSP FIFO usage with these values should not go under
-		 * 16 locations.
-		 * If the whole FIFO without safety buffer is used, than there
-		 * is a possibility that the DMA will be not able to push the
-		 * new data on time, causing channel shifts in runtime.
-		 */
-		mcbsp->max_tx_thres = max_thres(mcbsp) - 0x10;
-		mcbsp->max_rx_thres = max_thres(mcbsp) - 0x10;
-
-		ret = sysfs_create_group(&mcbsp->dev->kobj,
-					 &additional_attr_group);
-		if (ret) {
-			dev_err(mcbsp->dev,
-				"Unable to create additional controls\n");
-			goto err_thres;
-		}
-	} else {
-		mcbsp->max_tx_thres = -EINVAL;
-		mcbsp->max_rx_thres = -EINVAL;
-	}
-
-	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "sidetone");
-	if (res) {
-		ret = omap_st_add(mcbsp, res);
-		if (ret) {
-			dev_err(mcbsp->dev,
-				"Unable to create sidetone controls\n");
-			goto err_st;
-		}
-	}
-
-	return 0;
-
-err_st:
-	if (mcbsp->pdata->buffer_size)
-		sysfs_remove_group(&mcbsp->dev->kobj, &additional_attr_group);
-err_thres:
-	clk_put(mcbsp->fclk);
-	return ret;
-}
-
-void omap_mcbsp_cleanup(struct omap_mcbsp *mcbsp)
-{
-	if (mcbsp->pdata->buffer_size)
-		sysfs_remove_group(&mcbsp->dev->kobj, &additional_attr_group);
-
-	if (mcbsp->st_data) {
-		sysfs_remove_group(&mcbsp->dev->kobj, &sidetone_attr_group);
-		clk_put(mcbsp->st_data->mcbsp_iclk);
-	}
-}
diff -Naur --no-dereference a/sound/soc/omap/mcbsp.h b/sound/soc/omap/mcbsp.h
--- a/sound/soc/omap/mcbsp.h	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/omap/mcbsp.h	1969-12-31 19:00:00.000000000 -0500
@@ -1,358 +0,0 @@
-/*
- * sound/soc/omap/mcbsp.h
- *
- * OMAP Multi-Channel Buffered Serial Port
- *
- * Contact: Jarkko Nikula <jarkko.nikula@bitmer.com>
- *          Peter Ujfalusi <peter.ujfalusi@ti.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
- *
- */
-#ifndef __ASOC_MCBSP_H
-#define __ASOC_MCBSP_H
-
-#ifdef CONFIG_ARCH_OMAP1
-#define mcbsp_omap1()	1
-#else
-#define mcbsp_omap1()	0
-#endif
-
-#include <sound/dmaengine_pcm.h>
-
-/* McBSP register numbers. Register address offset = num * reg_step */
-enum {
-	/* Common registers */
-	OMAP_MCBSP_REG_SPCR2 = 4,
-	OMAP_MCBSP_REG_SPCR1,
-	OMAP_MCBSP_REG_RCR2,
-	OMAP_MCBSP_REG_RCR1,
-	OMAP_MCBSP_REG_XCR2,
-	OMAP_MCBSP_REG_XCR1,
-	OMAP_MCBSP_REG_SRGR2,
-	OMAP_MCBSP_REG_SRGR1,
-	OMAP_MCBSP_REG_MCR2,
-	OMAP_MCBSP_REG_MCR1,
-	OMAP_MCBSP_REG_RCERA,
-	OMAP_MCBSP_REG_RCERB,
-	OMAP_MCBSP_REG_XCERA,
-	OMAP_MCBSP_REG_XCERB,
-	OMAP_MCBSP_REG_PCR0,
-	OMAP_MCBSP_REG_RCERC,
-	OMAP_MCBSP_REG_RCERD,
-	OMAP_MCBSP_REG_XCERC,
-	OMAP_MCBSP_REG_XCERD,
-	OMAP_MCBSP_REG_RCERE,
-	OMAP_MCBSP_REG_RCERF,
-	OMAP_MCBSP_REG_XCERE,
-	OMAP_MCBSP_REG_XCERF,
-	OMAP_MCBSP_REG_RCERG,
-	OMAP_MCBSP_REG_RCERH,
-	OMAP_MCBSP_REG_XCERG,
-	OMAP_MCBSP_REG_XCERH,
-
-	/* OMAP1-OMAP2420 registers */
-	OMAP_MCBSP_REG_DRR2 = 0,
-	OMAP_MCBSP_REG_DRR1,
-	OMAP_MCBSP_REG_DXR2,
-	OMAP_MCBSP_REG_DXR1,
-
-	/* OMAP2430 and onwards */
-	OMAP_MCBSP_REG_DRR = 0,
-	OMAP_MCBSP_REG_DXR = 2,
-	OMAP_MCBSP_REG_SYSCON =	35,
-	OMAP_MCBSP_REG_THRSH2,
-	OMAP_MCBSP_REG_THRSH1,
-	OMAP_MCBSP_REG_IRQST = 40,
-	OMAP_MCBSP_REG_IRQEN,
-	OMAP_MCBSP_REG_WAKEUPEN,
-	OMAP_MCBSP_REG_XCCR,
-	OMAP_MCBSP_REG_RCCR,
-	OMAP_MCBSP_REG_XBUFFSTAT,
-	OMAP_MCBSP_REG_RBUFFSTAT,
-	OMAP_MCBSP_REG_SSELCR,
-};
-
-/* OMAP3 sidetone control registers */
-#define OMAP_ST_REG_REV		0x00
-#define OMAP_ST_REG_SYSCONFIG	0x10
-#define OMAP_ST_REG_IRQSTATUS	0x18
-#define OMAP_ST_REG_IRQENABLE	0x1C
-#define OMAP_ST_REG_SGAINCR	0x24
-#define OMAP_ST_REG_SFIRCR	0x28
-#define OMAP_ST_REG_SSELCR	0x2C
-
-/************************** McBSP SPCR1 bit definitions ***********************/
-#define RRST			BIT(0)
-#define RRDY			BIT(1)
-#define RFULL			BIT(2)
-#define RSYNC_ERR		BIT(3)
-#define RINTM(value)		(((value) & 0x3) << 4)	/* bits 4:5 */
-#define ABIS			BIT(6)
-#define DXENA			BIT(7)
-#define CLKSTP(value)		(((value) & 0x3) << 11)	/* bits 11:12 */
-#define RJUST(value)		(((value) & 0x3) << 13)	/* bits 13:14 */
-#define ALB			BIT(15)
-#define DLB			BIT(15)
-
-/************************** McBSP SPCR2 bit definitions ***********************/
-#define XRST			BIT(0)
-#define XRDY			BIT(1)
-#define XEMPTY			BIT(2)
-#define XSYNC_ERR		BIT(3)
-#define XINTM(value)		(((value) & 0x3) << 4)	/* bits 4:5 */
-#define GRST			BIT(6)
-#define FRST			BIT(7)
-#define SOFT			BIT(8)
-#define FREE			BIT(9)
-
-/************************** McBSP PCR bit definitions *************************/
-#define CLKRP			BIT(0)
-#define CLKXP			BIT(1)
-#define FSRP			BIT(2)
-#define FSXP			BIT(3)
-#define DR_STAT			BIT(4)
-#define DX_STAT			BIT(5)
-#define CLKS_STAT		BIT(6)
-#define SCLKME			BIT(7)
-#define CLKRM			BIT(8)
-#define CLKXM			BIT(9)
-#define FSRM			BIT(10)
-#define FSXM			BIT(11)
-#define RIOEN			BIT(12)
-#define XIOEN			BIT(13)
-#define IDLE_EN			BIT(14)
-
-/************************** McBSP RCR1 bit definitions ************************/
-#define RWDLEN1(value)		(((value) & 0x7) << 5)	/* Bits 5:7 */
-#define RFRLEN1(value)		(((value) & 0x7f) << 8)	/* Bits 8:14 */
-
-/************************** McBSP XCR1 bit definitions ************************/
-#define XWDLEN1(value)		(((value) & 0x7) << 5)	/* Bits 5:7 */
-#define XFRLEN1(value)		(((value) & 0x7f) << 8)	/* Bits 8:14 */
-
-/*************************** McBSP RCR2 bit definitions ***********************/
-#define RDATDLY(value)		((value) & 0x3)		/* Bits 0:1 */
-#define RFIG			BIT(2)
-#define RCOMPAND(value)		(((value) & 0x3) << 3)	/* Bits 3:4 */
-#define RWDLEN2(value)		(((value) & 0x7) << 5)	/* Bits 5:7 */
-#define RFRLEN2(value)		(((value) & 0x7f) << 8)	/* Bits 8:14 */
-#define RPHASE			BIT(15)
-
-/*************************** McBSP XCR2 bit definitions ***********************/
-#define XDATDLY(value)		((value) & 0x3)		/* Bits 0:1 */
-#define XFIG			BIT(2)
-#define XCOMPAND(value)		(((value) & 0x3) << 3)	/* Bits 3:4 */
-#define XWDLEN2(value)		(((value) & 0x7) << 5)	/* Bits 5:7 */
-#define XFRLEN2(value)		(((value) & 0x7f) << 8)	/* Bits 8:14 */
-#define XPHASE			BIT(15)
-
-/************************* McBSP SRGR1 bit definitions ************************/
-#define CLKGDV(value)		((value) & 0x7f)		/* Bits 0:7 */
-#define FWID(value)		(((value) & 0xff) << 8)	/* Bits 8:15 */
-
-/************************* McBSP SRGR2 bit definitions ************************/
-#define FPER(value)		((value) & 0x0fff)	/* Bits 0:11 */
-#define FSGM			BIT(12)
-#define CLKSM			BIT(13)
-#define CLKSP			BIT(14)
-#define GSYNC			BIT(15)
-
-/************************* McBSP MCR1 bit definitions *************************/
-#define RMCM			BIT(0)
-#define RCBLK(value)		(((value) & 0x7) << 2)	/* Bits 2:4 */
-#define RPABLK(value)		(((value) & 0x3) << 5)	/* Bits 5:6 */
-#define RPBBLK(value)		(((value) & 0x3) << 7)	/* Bits 7:8 */
-
-/************************* McBSP MCR2 bit definitions *************************/
-#define XMCM(value)		((value) & 0x3)		/* Bits 0:1 */
-#define XCBLK(value)		(((value) & 0x7) << 2)	/* Bits 2:4 */
-#define XPABLK(value)		(((value) & 0x3) << 5)	/* Bits 5:6 */
-#define XPBBLK(value)		(((value) & 0x3) << 7)	/* Bits 7:8 */
-
-/*********************** McBSP XCCR bit definitions *************************/
-#define XDISABLE		BIT(0)
-#define XDMAEN			BIT(3)
-#define DILB			BIT(5)
-#define XFULL_CYCLE		BIT(11)
-#define DXENDLY(value)		(((value) & 0x3) << 12)	/* Bits 12:13 */
-#define PPCONNECT		BIT(14)
-#define EXTCLKGATE		BIT(15)
-
-/********************** McBSP RCCR bit definitions *************************/
-#define RDISABLE		BIT(0)
-#define RDMAEN			BIT(3)
-#define RFULL_CYCLE		BIT(11)
-
-/********************** McBSP SYSCONFIG bit definitions ********************/
-#define SOFTRST			BIT(1)
-#define ENAWAKEUP		BIT(2)
-#define SIDLEMODE(value)	(((value) & 0x3) << 3)
-#define CLOCKACTIVITY(value)	(((value) & 0x3) << 8)
-
-/********************** McBSP SSELCR bit definitions ***********************/
-#define SIDETONEEN		BIT(10)
-
-/********************** McBSP Sidetone SYSCONFIG bit definitions ***********/
-#define ST_AUTOIDLE		BIT(0)
-
-/********************** McBSP Sidetone SGAINCR bit definitions *************/
-#define ST_CH0GAIN(value)	((value) & 0xffff)	/* Bits 0:15 */
-#define ST_CH1GAIN(value)	(((value) & 0xffff) << 16) /* Bits 16:31 */
-
-/********************** McBSP Sidetone SFIRCR bit definitions **************/
-#define ST_FIRCOEFF(value)	((value) & 0xffff)	/* Bits 0:15 */
-
-/********************** McBSP Sidetone SSELCR bit definitions **************/
-#define ST_SIDETONEEN		BIT(0)
-#define ST_COEFFWREN		BIT(1)
-#define ST_COEFFWRDONE		BIT(2)
-
-/********************** McBSP DMA operating modes **************************/
-#define MCBSP_DMA_MODE_ELEMENT		0
-#define MCBSP_DMA_MODE_THRESHOLD	1
-
-/********************** McBSP WAKEUPEN/IRQST/IRQEN bit definitions *********/
-#define RSYNCERREN		BIT(0)
-#define RFSREN			BIT(1)
-#define REOFEN			BIT(2)
-#define RRDYEN			BIT(3)
-#define RUNDFLEN		BIT(4)
-#define ROVFLEN			BIT(5)
-#define XSYNCERREN		BIT(7)
-#define XFSXEN			BIT(8)
-#define XEOFEN			BIT(9)
-#define XRDYEN			BIT(10)
-#define XUNDFLEN		BIT(11)
-#define XOVFLEN			BIT(12)
-#define XEMPTYEOFEN		BIT(14)
-
-/* Clock signal muxing options */
-#define CLKR_SRC_CLKR		0 /* CLKR signal is from the CLKR pin */
-#define CLKR_SRC_CLKX		1 /* CLKR signal is from the CLKX pin */
-#define FSR_SRC_FSR		2 /* FSR signal is from the FSR pin */
-#define FSR_SRC_FSX		3 /* FSR signal is from the FSX pin */
-
-/* McBSP functional clock sources */
-#define MCBSP_CLKS_PRCM_SRC	0
-#define MCBSP_CLKS_PAD_SRC	1
-
-/* we don't do multichannel for now */
-struct omap_mcbsp_reg_cfg {
-	u16 spcr2;
-	u16 spcr1;
-	u16 rcr2;
-	u16 rcr1;
-	u16 xcr2;
-	u16 xcr1;
-	u16 srgr2;
-	u16 srgr1;
-	u16 mcr2;
-	u16 mcr1;
-	u16 pcr0;
-	u16 rcerc;
-	u16 rcerd;
-	u16 xcerc;
-	u16 xcerd;
-	u16 rcere;
-	u16 rcerf;
-	u16 xcere;
-	u16 xcerf;
-	u16 rcerg;
-	u16 rcerh;
-	u16 xcerg;
-	u16 xcerh;
-	u16 xccr;
-	u16 rccr;
-};
-
-struct omap_mcbsp_st_data {
-	void __iomem *io_base_st;
-	struct clk *mcbsp_iclk;
-	bool running;
-	bool enabled;
-	s16 taps[128];	/* Sidetone filter coefficients */
-	int nr_taps;	/* Number of filter coefficients in use */
-	s16 ch0gain;
-	s16 ch1gain;
-};
-
-struct omap_mcbsp {
-	struct device *dev;
-	struct clk *fclk;
-	spinlock_t lock;
-	unsigned long phys_base;
-	unsigned long phys_dma_base;
-	void __iomem *io_base;
-	u8 id;
-	/*
-	 * Flags indicating is the bus already activated and configured by
-	 * another substream
-	 */
-	int active;
-	int configured;
-	u8 free;
-
-	int irq;
-	int rx_irq;
-	int tx_irq;
-
-	/* Protect the field .free, while checking if the mcbsp is in use */
-	struct omap_mcbsp_platform_data *pdata;
-	struct omap_mcbsp_st_data *st_data;
-	struct omap_mcbsp_reg_cfg cfg_regs;
-	struct snd_dmaengine_dai_dma_data dma_data[2];
-	unsigned int dma_req[2];
-	int dma_op_mode;
-	u16 max_tx_thres;
-	u16 max_rx_thres;
-	void *reg_cache;
-	int reg_cache_size;
-
-	unsigned int fmt;
-	unsigned int in_freq;
-	unsigned int latency[2];
-	int clk_div;
-	int wlen;
-
-	struct pm_qos_request pm_qos_req;
-};
-
-void omap_mcbsp_config(struct omap_mcbsp *mcbsp,
-		       const struct omap_mcbsp_reg_cfg *config);
-void omap_mcbsp_set_tx_threshold(struct omap_mcbsp *mcbsp, u16 threshold);
-void omap_mcbsp_set_rx_threshold(struct omap_mcbsp *mcbsp, u16 threshold);
-u16 omap_mcbsp_get_tx_delay(struct omap_mcbsp *mcbsp);
-u16 omap_mcbsp_get_rx_delay(struct omap_mcbsp *mcbsp);
-int omap_mcbsp_get_dma_op_mode(struct omap_mcbsp *mcbsp);
-int omap_mcbsp_request(struct omap_mcbsp *mcbsp);
-void omap_mcbsp_free(struct omap_mcbsp *mcbsp);
-void omap_mcbsp_start(struct omap_mcbsp *mcbsp, int tx, int rx);
-void omap_mcbsp_stop(struct omap_mcbsp *mcbsp, int tx, int rx);
-
-/* McBSP functional clock source changing function */
-int omap2_mcbsp_set_clks_src(struct omap_mcbsp *mcbsp, u8 fck_src_id);
-
-/* Sidetone specific API */
-int omap_st_set_chgain(struct omap_mcbsp *mcbsp, int channel, s16 chgain);
-int omap_st_get_chgain(struct omap_mcbsp *mcbsp, int channel, s16 *chgain);
-int omap_st_enable(struct omap_mcbsp *mcbsp);
-int omap_st_disable(struct omap_mcbsp *mcbsp);
-int omap_st_is_enabled(struct omap_mcbsp *mcbsp);
-
-int omap_mcbsp_init(struct platform_device *pdev);
-void omap_mcbsp_cleanup(struct omap_mcbsp *mcbsp);
-
-#endif /* __ASOC_MCBSP_H */
diff -Naur --no-dereference a/sound/soc/omap/n810.c b/sound/soc/omap/n810.c
--- a/sound/soc/omap/n810.c	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/omap/n810.c	1969-12-31 19:00:00.000000000 -0500
@@ -1,378 +0,0 @@
-/*
- * n810.c  --  SoC audio for Nokia N810
- *
- * Copyright (C) 2008 Nokia Corporation
- *
- * Contact: Jarkko Nikula <jarkko.nikula@bitmer.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
- * 02110-1301 USA
- *
- */
-
-#include <linux/clk.h>
-#include <linux/i2c.h>
-#include <linux/platform_device.h>
-#include <sound/core.h>
-#include <sound/pcm.h>
-#include <sound/soc.h>
-
-#include <asm/mach-types.h>
-#include <linux/gpio.h>
-#include <linux/module.h>
-#include <linux/platform_data/asoc-ti-mcbsp.h>
-
-#include "omap-mcbsp.h"
-
-#define N810_HEADSET_AMP_GPIO	10
-#define N810_SPEAKER_AMP_GPIO	101
-
-enum {
-	N810_JACK_DISABLED,
-	N810_JACK_HP,
-	N810_JACK_HS,
-	N810_JACK_MIC,
-};
-
-static struct clk *sys_clkout2;
-static struct clk *sys_clkout2_src;
-static struct clk *func96m_clk;
-
-static int n810_spk_func;
-static int n810_jack_func;
-static int n810_dmic_func;
-
-static void n810_ext_control(struct snd_soc_dapm_context *dapm)
-{
-	int hp = 0, line1l = 0;
-
-	switch (n810_jack_func) {
-	case N810_JACK_HS:
-		line1l = 1;
-	case N810_JACK_HP:
-		hp = 1;
-		break;
-	case N810_JACK_MIC:
-		line1l = 1;
-		break;
-	}
-
-	snd_soc_dapm_mutex_lock(dapm);
-
-	if (n810_spk_func)
-		snd_soc_dapm_enable_pin_unlocked(dapm, "Ext Spk");
-	else
-		snd_soc_dapm_disable_pin_unlocked(dapm, "Ext Spk");
-
-	if (hp)
-		snd_soc_dapm_enable_pin_unlocked(dapm, "Headphone Jack");
-	else
-		snd_soc_dapm_disable_pin_unlocked(dapm, "Headphone Jack");
-	if (line1l)
-		snd_soc_dapm_enable_pin_unlocked(dapm, "HS Mic");
-	else
-		snd_soc_dapm_disable_pin_unlocked(dapm, "HS Mic");
-
-	if (n810_dmic_func)
-		snd_soc_dapm_enable_pin_unlocked(dapm, "DMic");
-	else
-		snd_soc_dapm_disable_pin_unlocked(dapm, "DMic");
-
-	snd_soc_dapm_sync_unlocked(dapm);
-
-	snd_soc_dapm_mutex_unlock(dapm);
-}
-
-static int n810_startup(struct snd_pcm_substream *substream)
-{
-	struct snd_pcm_runtime *runtime = substream->runtime;
-	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-
-	snd_pcm_hw_constraint_single(runtime, SNDRV_PCM_HW_PARAM_CHANNELS, 2);
-
-	n810_ext_control(&rtd->card->dapm);
-	return clk_prepare_enable(sys_clkout2);
-}
-
-static void n810_shutdown(struct snd_pcm_substream *substream)
-{
-	clk_disable_unprepare(sys_clkout2);
-}
-
-static int n810_hw_params(struct snd_pcm_substream *substream,
-	struct snd_pcm_hw_params *params)
-{
-	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct snd_soc_dai *codec_dai = rtd->codec_dai;
-	int err;
-
-	/* Set the codec system clock for DAC and ADC */
-	err = snd_soc_dai_set_sysclk(codec_dai, 0, 12000000,
-					    SND_SOC_CLOCK_IN);
-
-	return err;
-}
-
-static const struct snd_soc_ops n810_ops = {
-	.startup = n810_startup,
-	.hw_params = n810_hw_params,
-	.shutdown = n810_shutdown,
-};
-
-static int n810_get_spk(struct snd_kcontrol *kcontrol,
-			struct snd_ctl_elem_value *ucontrol)
-{
-	ucontrol->value.enumerated.item[0] = n810_spk_func;
-
-	return 0;
-}
-
-static int n810_set_spk(struct snd_kcontrol *kcontrol,
-			struct snd_ctl_elem_value *ucontrol)
-{
-	struct snd_soc_card *card =  snd_kcontrol_chip(kcontrol);
-
-	if (n810_spk_func == ucontrol->value.enumerated.item[0])
-		return 0;
-
-	n810_spk_func = ucontrol->value.enumerated.item[0];
-	n810_ext_control(&card->dapm);
-
-	return 1;
-}
-
-static int n810_get_jack(struct snd_kcontrol *kcontrol,
-			 struct snd_ctl_elem_value *ucontrol)
-{
-	ucontrol->value.enumerated.item[0] = n810_jack_func;
-
-	return 0;
-}
-
-static int n810_set_jack(struct snd_kcontrol *kcontrol,
-			 struct snd_ctl_elem_value *ucontrol)
-{
-	struct snd_soc_card *card =  snd_kcontrol_chip(kcontrol);
-
-	if (n810_jack_func == ucontrol->value.enumerated.item[0])
-		return 0;
-
-	n810_jack_func = ucontrol->value.enumerated.item[0];
-	n810_ext_control(&card->dapm);
-
-	return 1;
-}
-
-static int n810_get_input(struct snd_kcontrol *kcontrol,
-			  struct snd_ctl_elem_value *ucontrol)
-{
-	ucontrol->value.enumerated.item[0] = n810_dmic_func;
-
-	return 0;
-}
-
-static int n810_set_input(struct snd_kcontrol *kcontrol,
-			  struct snd_ctl_elem_value *ucontrol)
-{
-	struct snd_soc_card *card =  snd_kcontrol_chip(kcontrol);
-
-	if (n810_dmic_func == ucontrol->value.enumerated.item[0])
-		return 0;
-
-	n810_dmic_func = ucontrol->value.enumerated.item[0];
-	n810_ext_control(&card->dapm);
-
-	return 1;
-}
-
-static int n810_spk_event(struct snd_soc_dapm_widget *w,
-			  struct snd_kcontrol *k, int event)
-{
-	if (SND_SOC_DAPM_EVENT_ON(event))
-		gpio_set_value(N810_SPEAKER_AMP_GPIO, 1);
-	else
-		gpio_set_value(N810_SPEAKER_AMP_GPIO, 0);
-
-	return 0;
-}
-
-static int n810_jack_event(struct snd_soc_dapm_widget *w,
-			   struct snd_kcontrol *k, int event)
-{
-	if (SND_SOC_DAPM_EVENT_ON(event))
-		gpio_set_value(N810_HEADSET_AMP_GPIO, 1);
-	else
-		gpio_set_value(N810_HEADSET_AMP_GPIO, 0);
-
-	return 0;
-}
-
-static const struct snd_soc_dapm_widget aic33_dapm_widgets[] = {
-	SND_SOC_DAPM_SPK("Ext Spk", n810_spk_event),
-	SND_SOC_DAPM_HP("Headphone Jack", n810_jack_event),
-	SND_SOC_DAPM_MIC("DMic", NULL),
-	SND_SOC_DAPM_MIC("HS Mic", NULL),
-};
-
-static const struct snd_soc_dapm_route audio_map[] = {
-	{"Headphone Jack", NULL, "HPLOUT"},
-	{"Headphone Jack", NULL, "HPROUT"},
-
-	{"Ext Spk", NULL, "LLOUT"},
-	{"Ext Spk", NULL, "RLOUT"},
-
-	{"DMic Rate 64", NULL, "DMic"},
-	{"DMic", NULL, "Mic Bias"},
-
-	/*
-	 * Note that the mic bias is coming from Retu/Vilma and we don't have
-	 * control over it atm. The analog HS mic is not working. <- TODO
-	 */
-	{"LINE1L", NULL, "HS Mic"},
-};
-
-static const char *spk_function[] = {"Off", "On"};
-static const char *jack_function[] = {"Off", "Headphone", "Headset", "Mic"};
-static const char *input_function[] = {"ADC", "Digital Mic"};
-static const struct soc_enum n810_enum[] = {
-	SOC_ENUM_SINGLE_EXT(ARRAY_SIZE(spk_function), spk_function),
-	SOC_ENUM_SINGLE_EXT(ARRAY_SIZE(jack_function), jack_function),
-	SOC_ENUM_SINGLE_EXT(ARRAY_SIZE(input_function), input_function),
-};
-
-static const struct snd_kcontrol_new aic33_n810_controls[] = {
-	SOC_ENUM_EXT("Speaker Function", n810_enum[0],
-		     n810_get_spk, n810_set_spk),
-	SOC_ENUM_EXT("Jack Function", n810_enum[1],
-		     n810_get_jack, n810_set_jack),
-	SOC_ENUM_EXT("Input Select",  n810_enum[2],
-		     n810_get_input, n810_set_input),
-};
-
-/* Digital audio interface glue - connects codec <--> CPU */
-static struct snd_soc_dai_link n810_dai = {
-	.name = "TLV320AIC33",
-	.stream_name = "AIC33",
-	.cpu_dai_name = "48076000.mcbsp",
-	.platform_name = "48076000.mcbsp",
-	.codec_name = "tlv320aic3x-codec.1-0018",
-	.codec_dai_name = "tlv320aic3x-hifi",
-	.dai_fmt = SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF |
-		   SND_SOC_DAIFMT_CBM_CFM,
-	.ops = &n810_ops,
-};
-
-/* Audio machine driver */
-static struct snd_soc_card snd_soc_n810 = {
-	.name = "N810",
-	.owner = THIS_MODULE,
-	.dai_link = &n810_dai,
-	.num_links = 1,
-
-	.controls = aic33_n810_controls,
-	.num_controls = ARRAY_SIZE(aic33_n810_controls),
-	.dapm_widgets = aic33_dapm_widgets,
-	.num_dapm_widgets = ARRAY_SIZE(aic33_dapm_widgets),
-	.dapm_routes = audio_map,
-	.num_dapm_routes = ARRAY_SIZE(audio_map),
-	.fully_routed = true,
-};
-
-static struct platform_device *n810_snd_device;
-
-static int __init n810_soc_init(void)
-{
-	int err;
-	struct device *dev;
-
-	if (!of_have_populated_dt() ||
-	    (!of_machine_is_compatible("nokia,n810") &&
-	     !of_machine_is_compatible("nokia,n810-wimax")))
-		return -ENODEV;
-
-	n810_snd_device = platform_device_alloc("soc-audio", -1);
-	if (!n810_snd_device)
-		return -ENOMEM;
-
-	platform_set_drvdata(n810_snd_device, &snd_soc_n810);
-	err = platform_device_add(n810_snd_device);
-	if (err)
-		goto err1;
-
-	dev = &n810_snd_device->dev;
-
-	sys_clkout2_src = clk_get(dev, "sys_clkout2_src");
-	if (IS_ERR(sys_clkout2_src)) {
-		dev_err(dev, "Could not get sys_clkout2_src clock\n");
-		err = PTR_ERR(sys_clkout2_src);
-		goto err2;
-	}
-	sys_clkout2 = clk_get(dev, "sys_clkout2");
-	if (IS_ERR(sys_clkout2)) {
-		dev_err(dev, "Could not get sys_clkout2\n");
-		err = PTR_ERR(sys_clkout2);
-		goto err3;
-	}
-	/*
-	 * Configure 12 MHz output on SYS_CLKOUT2. Therefore we must use
-	 * 96 MHz as its parent in order to get 12 MHz
-	 */
-	func96m_clk = clk_get(dev, "func_96m_ck");
-	if (IS_ERR(func96m_clk)) {
-		dev_err(dev, "Could not get func 96M clock\n");
-		err = PTR_ERR(func96m_clk);
-		goto err4;
-	}
-	clk_set_parent(sys_clkout2_src, func96m_clk);
-	clk_set_rate(sys_clkout2, 12000000);
-
-	if (WARN_ON((gpio_request(N810_HEADSET_AMP_GPIO, "hs_amp") < 0) ||
-		    (gpio_request(N810_SPEAKER_AMP_GPIO, "spk_amp") < 0))) {
-		err = -EINVAL;
-		goto err4;
-	}
-
-	gpio_direction_output(N810_HEADSET_AMP_GPIO, 0);
-	gpio_direction_output(N810_SPEAKER_AMP_GPIO, 0);
-
-	return 0;
-err4:
-	clk_put(sys_clkout2);
-err3:
-	clk_put(sys_clkout2_src);
-err2:
-	platform_device_del(n810_snd_device);
-err1:
-	platform_device_put(n810_snd_device);
-
-	return err;
-}
-
-static void __exit n810_soc_exit(void)
-{
-	gpio_free(N810_SPEAKER_AMP_GPIO);
-	gpio_free(N810_HEADSET_AMP_GPIO);
-	clk_put(sys_clkout2_src);
-	clk_put(sys_clkout2);
-	clk_put(func96m_clk);
-
-	platform_device_unregister(n810_snd_device);
-}
-
-module_init(n810_soc_init);
-module_exit(n810_soc_exit);
-
-MODULE_AUTHOR("Jarkko Nikula <jarkko.nikula@bitmer.com>");
-MODULE_DESCRIPTION("ALSA SoC Nokia N810");
-MODULE_LICENSE("GPL");
diff -Naur --no-dereference a/sound/soc/omap/omap3pandora.c b/sound/soc/omap/omap3pandora.c
--- a/sound/soc/omap/omap3pandora.c	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/omap/omap3pandora.c	1969-12-31 19:00:00.000000000 -0500
@@ -1,315 +0,0 @@
-/*
- * omap3pandora.c  --  SoC audio for Pandora Handheld Console
- *
- * Author: Gražvydas Ignotas <notasas@gmail.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
- * 02110-1301 USA
- *
- */
-
-#include <linux/clk.h>
-#include <linux/platform_device.h>
-#include <linux/gpio.h>
-#include <linux/delay.h>
-#include <linux/regulator/consumer.h>
-#include <linux/module.h>
-
-#include <sound/core.h>
-#include <sound/pcm.h>
-#include <sound/soc.h>
-
-#include <asm/mach-types.h>
-#include <linux/platform_data/asoc-ti-mcbsp.h>
-
-#include "omap-mcbsp.h"
-
-#define OMAP3_PANDORA_DAC_POWER_GPIO	118
-#define OMAP3_PANDORA_AMP_POWER_GPIO	14
-
-#define PREFIX "ASoC omap3pandora: "
-
-static struct regulator *omap3pandora_dac_reg;
-
-static int omap3pandora_hw_params(struct snd_pcm_substream *substream,
-	struct snd_pcm_hw_params *params)
-{
-	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct snd_soc_dai *codec_dai = rtd->codec_dai;
-	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
-	int ret;
-
-	/* Set the codec system clock for DAC and ADC */
-	ret = snd_soc_dai_set_sysclk(codec_dai, 0, 26000000,
-					    SND_SOC_CLOCK_IN);
-	if (ret < 0) {
-		pr_err(PREFIX "can't set codec system clock\n");
-		return ret;
-	}
-
-	/* Set McBSP clock to external */
-	ret = snd_soc_dai_set_sysclk(cpu_dai, OMAP_MCBSP_SYSCLK_CLKS_EXT,
-				     256 * params_rate(params),
-				     SND_SOC_CLOCK_IN);
-	if (ret < 0) {
-		pr_err(PREFIX "can't set cpu system clock\n");
-		return ret;
-	}
-
-	ret = snd_soc_dai_set_clkdiv(cpu_dai, OMAP_MCBSP_CLKGDV, 8);
-	if (ret < 0) {
-		pr_err(PREFIX "can't set SRG clock divider\n");
-		return ret;
-	}
-
-	return 0;
-}
-
-static int omap3pandora_dac_event(struct snd_soc_dapm_widget *w,
-	struct snd_kcontrol *k, int event)
-{
-	int ret;
-
-	/*
-	 * The PCM1773 DAC datasheet requires 1ms delay between switching
-	 * VCC power on/off and /PD pin high/low
-	 */
-	if (SND_SOC_DAPM_EVENT_ON(event)) {
-		ret = regulator_enable(omap3pandora_dac_reg);
-		if (ret) {
-			dev_err(w->dapm->dev, "Failed to power DAC: %d\n", ret);
-			return ret;
-		}
-		mdelay(1);
-		gpio_set_value(OMAP3_PANDORA_DAC_POWER_GPIO, 1);
-	} else {
-		gpio_set_value(OMAP3_PANDORA_DAC_POWER_GPIO, 0);
-		mdelay(1);
-		regulator_disable(omap3pandora_dac_reg);
-	}
-
-	return 0;
-}
-
-static int omap3pandora_hp_event(struct snd_soc_dapm_widget *w,
-	struct snd_kcontrol *k, int event)
-{
-	if (SND_SOC_DAPM_EVENT_ON(event))
-		gpio_set_value(OMAP3_PANDORA_AMP_POWER_GPIO, 1);
-	else
-		gpio_set_value(OMAP3_PANDORA_AMP_POWER_GPIO, 0);
-
-	return 0;
-}
-
-/*
- * Audio paths on Pandora board:
- *
- *  |O| ---> PCM DAC +-> AMP -> Headphone Jack
- *  |M|         A    +--------> Line Out
- *  |A| <~~clk~~+
- *  |P| <--- TWL4030 <--------- Line In and MICs
- */
-static const struct snd_soc_dapm_widget omap3pandora_dapm_widgets[] = {
-	SND_SOC_DAPM_DAC_E("PCM DAC", "HiFi Playback", SND_SOC_NOPM,
-			   0, 0, omap3pandora_dac_event,
-			   SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_PRE_PMD),
-	SND_SOC_DAPM_PGA_E("Headphone Amplifier", SND_SOC_NOPM,
-			   0, 0, NULL, 0, omap3pandora_hp_event,
-			   SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_PRE_PMD),
-	SND_SOC_DAPM_HP("Headphone Jack", NULL),
-	SND_SOC_DAPM_LINE("Line Out", NULL),
-
-	SND_SOC_DAPM_MIC("Mic (internal)", NULL),
-	SND_SOC_DAPM_MIC("Mic (external)", NULL),
-	SND_SOC_DAPM_LINE("Line In", NULL),
-};
-
-static const struct snd_soc_dapm_route omap3pandora_map[] = {
-	{"PCM DAC", NULL, "APLL Enable"},
-	{"Headphone Amplifier", NULL, "PCM DAC"},
-	{"Line Out", NULL, "PCM DAC"},
-	{"Headphone Jack", NULL, "Headphone Amplifier"},
-
-	{"AUXL", NULL, "Line In"},
-	{"AUXR", NULL, "Line In"},
-
-	{"MAINMIC", NULL, "Mic (internal)"},
-	{"Mic (internal)", NULL, "Mic Bias 1"},
-
-	{"SUBMIC", NULL, "Mic (external)"},
-	{"Mic (external)", NULL, "Mic Bias 2"},
-};
-
-static int omap3pandora_out_init(struct snd_soc_pcm_runtime *rtd)
-{
-	struct snd_soc_dapm_context *dapm = &rtd->card->dapm;
-
-	/* All TWL4030 output pins are floating */
-	snd_soc_dapm_nc_pin(dapm, "EARPIECE");
-	snd_soc_dapm_nc_pin(dapm, "PREDRIVEL");
-	snd_soc_dapm_nc_pin(dapm, "PREDRIVER");
-	snd_soc_dapm_nc_pin(dapm, "HSOL");
-	snd_soc_dapm_nc_pin(dapm, "HSOR");
-	snd_soc_dapm_nc_pin(dapm, "CARKITL");
-	snd_soc_dapm_nc_pin(dapm, "CARKITR");
-	snd_soc_dapm_nc_pin(dapm, "HFL");
-	snd_soc_dapm_nc_pin(dapm, "HFR");
-	snd_soc_dapm_nc_pin(dapm, "VIBRA");
-
-	return 0;
-}
-
-static int omap3pandora_in_init(struct snd_soc_pcm_runtime *rtd)
-{
-	struct snd_soc_dapm_context *dapm = &rtd->card->dapm;
-
-	/* Not comnnected */
-	snd_soc_dapm_nc_pin(dapm, "HSMIC");
-	snd_soc_dapm_nc_pin(dapm, "CARKITMIC");
-	snd_soc_dapm_nc_pin(dapm, "DIGIMIC0");
-	snd_soc_dapm_nc_pin(dapm, "DIGIMIC1");
-
-	return 0;
-}
-
-static const struct snd_soc_ops omap3pandora_ops = {
-	.hw_params = omap3pandora_hw_params,
-};
-
-/* Digital audio interface glue - connects codec <--> CPU */
-static struct snd_soc_dai_link omap3pandora_dai[] = {
-	{
-		.name = "PCM1773",
-		.stream_name = "HiFi Out",
-		.cpu_dai_name = "omap-mcbsp.2",
-		.codec_dai_name = "twl4030-hifi",
-		.platform_name = "omap-mcbsp.2",
-		.codec_name = "twl4030-codec",
-		.dai_fmt = SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF |
-			   SND_SOC_DAIFMT_CBS_CFS,
-		.ops = &omap3pandora_ops,
-		.init = omap3pandora_out_init,
-	}, {
-		.name = "TWL4030",
-		.stream_name = "Line/Mic In",
-		.cpu_dai_name = "omap-mcbsp.4",
-		.codec_dai_name = "twl4030-hifi",
-		.platform_name = "omap-mcbsp.4",
-		.codec_name = "twl4030-codec",
-		.dai_fmt = SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF |
-			   SND_SOC_DAIFMT_CBS_CFS,
-		.ops = &omap3pandora_ops,
-		.init = omap3pandora_in_init,
-	}
-};
-
-/* SoC card */
-static struct snd_soc_card snd_soc_card_omap3pandora = {
-	.name = "omap3pandora",
-	.owner = THIS_MODULE,
-	.dai_link = omap3pandora_dai,
-	.num_links = ARRAY_SIZE(omap3pandora_dai),
-
-	.dapm_widgets = omap3pandora_dapm_widgets,
-	.num_dapm_widgets = ARRAY_SIZE(omap3pandora_dapm_widgets),
-	.dapm_routes = omap3pandora_map,
-	.num_dapm_routes = ARRAY_SIZE(omap3pandora_map),
-};
-
-static struct platform_device *omap3pandora_snd_device;
-
-static int __init omap3pandora_soc_init(void)
-{
-	int ret;
-
-	if (!machine_is_omap3_pandora())
-		return -ENODEV;
-
-	pr_info("OMAP3 Pandora SoC init\n");
-
-	ret = gpio_request(OMAP3_PANDORA_DAC_POWER_GPIO, "dac_power");
-	if (ret) {
-		pr_err(PREFIX "Failed to get DAC power GPIO\n");
-		return ret;
-	}
-
-	ret = gpio_direction_output(OMAP3_PANDORA_DAC_POWER_GPIO, 0);
-	if (ret) {
-		pr_err(PREFIX "Failed to set DAC power GPIO direction\n");
-		goto fail0;
-	}
-
-	ret = gpio_request(OMAP3_PANDORA_AMP_POWER_GPIO, "amp_power");
-	if (ret) {
-		pr_err(PREFIX "Failed to get amp power GPIO\n");
-		goto fail0;
-	}
-
-	ret = gpio_direction_output(OMAP3_PANDORA_AMP_POWER_GPIO, 0);
-	if (ret) {
-		pr_err(PREFIX "Failed to set amp power GPIO direction\n");
-		goto fail1;
-	}
-
-	omap3pandora_snd_device = platform_device_alloc("soc-audio", -1);
-	if (omap3pandora_snd_device == NULL) {
-		pr_err(PREFIX "Platform device allocation failed\n");
-		ret = -ENOMEM;
-		goto fail1;
-	}
-
-	platform_set_drvdata(omap3pandora_snd_device, &snd_soc_card_omap3pandora);
-
-	ret = platform_device_add(omap3pandora_snd_device);
-	if (ret) {
-		pr_err(PREFIX "Unable to add platform device\n");
-		goto fail2;
-	}
-
-	omap3pandora_dac_reg = regulator_get(&omap3pandora_snd_device->dev, "vcc");
-	if (IS_ERR(omap3pandora_dac_reg)) {
-		pr_err(PREFIX "Failed to get DAC regulator from %s: %ld\n",
-			dev_name(&omap3pandora_snd_device->dev),
-			PTR_ERR(omap3pandora_dac_reg));
-		ret = PTR_ERR(omap3pandora_dac_reg);
-		goto fail3;
-	}
-
-	return 0;
-
-fail3:
-	platform_device_del(omap3pandora_snd_device);
-fail2:
-	platform_device_put(omap3pandora_snd_device);
-fail1:
-	gpio_free(OMAP3_PANDORA_AMP_POWER_GPIO);
-fail0:
-	gpio_free(OMAP3_PANDORA_DAC_POWER_GPIO);
-	return ret;
-}
-module_init(omap3pandora_soc_init);
-
-static void __exit omap3pandora_soc_exit(void)
-{
-	regulator_put(omap3pandora_dac_reg);
-	platform_device_unregister(omap3pandora_snd_device);
-	gpio_free(OMAP3_PANDORA_AMP_POWER_GPIO);
-	gpio_free(OMAP3_PANDORA_DAC_POWER_GPIO);
-}
-module_exit(omap3pandora_soc_exit);
-
-MODULE_AUTHOR("Grazvydas Ignotas <notasas@gmail.com>");
-MODULE_DESCRIPTION("ALSA SoC OMAP3 Pandora");
-MODULE_LICENSE("GPL");
diff -Naur --no-dereference a/sound/soc/omap/omap-abe-twl6040.c b/sound/soc/omap/omap-abe-twl6040.c
--- a/sound/soc/omap/omap-abe-twl6040.c	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/omap/omap-abe-twl6040.c	1969-12-31 19:00:00.000000000 -0500
@@ -1,353 +0,0 @@
-/*
- * omap-abe-twl6040.c  --  SoC audio for TI OMAP based boards with ABE and
- *			   twl6040 codec
- *
- * Author: Misael Lopez Cruz <misael.lopez@ti.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
- * 02110-1301 USA
- *
- */
-
-#include <linux/clk.h>
-#include <linux/platform_device.h>
-#include <linux/mfd/twl6040.h>
-#include <linux/module.h>
-#include <linux/of.h>
-
-#include <sound/core.h>
-#include <sound/pcm.h>
-#include <sound/soc.h>
-#include <sound/jack.h>
-
-#include "omap-dmic.h"
-#include "omap-mcpdm.h"
-#include "../codecs/twl6040.h"
-
-struct abe_twl6040 {
-	struct snd_soc_card card;
-	struct snd_soc_dai_link dai_links[2];
-	int	jack_detection;	/* board can detect jack events */
-	int	mclk_freq;	/* MCLK frequency speed for twl6040 */
-};
-
-static struct platform_device *dmic_codec_dev;
-
-static int omap_abe_hw_params(struct snd_pcm_substream *substream,
-	struct snd_pcm_hw_params *params)
-{
-	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct snd_soc_dai *codec_dai = rtd->codec_dai;
-	struct snd_soc_card *card = rtd->card;
-	struct abe_twl6040 *priv = snd_soc_card_get_drvdata(card);
-	int clk_id, freq;
-	int ret;
-
-	clk_id = twl6040_get_clk_id(codec_dai->component);
-	if (clk_id == TWL6040_SYSCLK_SEL_HPPLL)
-		freq = priv->mclk_freq;
-	else if (clk_id == TWL6040_SYSCLK_SEL_LPPLL)
-		freq = 32768;
-	else
-		return -EINVAL;
-
-	/* set the codec mclk */
-	ret = snd_soc_dai_set_sysclk(codec_dai, clk_id, freq,
-				SND_SOC_CLOCK_IN);
-	if (ret) {
-		printk(KERN_ERR "can't set codec system clock\n");
-		return ret;
-	}
-	return ret;
-}
-
-static const struct snd_soc_ops omap_abe_ops = {
-	.hw_params = omap_abe_hw_params,
-};
-
-static int omap_abe_dmic_hw_params(struct snd_pcm_substream *substream,
-	struct snd_pcm_hw_params *params)
-{
-	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
-	int ret = 0;
-
-	ret = snd_soc_dai_set_sysclk(cpu_dai, OMAP_DMIC_SYSCLK_PAD_CLKS,
-				     19200000, SND_SOC_CLOCK_IN);
-	if (ret < 0) {
-		printk(KERN_ERR "can't set DMIC cpu system clock\n");
-		return ret;
-	}
-	ret = snd_soc_dai_set_sysclk(cpu_dai, OMAP_DMIC_ABE_DMIC_CLK, 2400000,
-				     SND_SOC_CLOCK_OUT);
-	if (ret < 0) {
-		printk(KERN_ERR "can't set DMIC output clock\n");
-		return ret;
-	}
-	return 0;
-}
-
-static struct snd_soc_ops omap_abe_dmic_ops = {
-	.hw_params = omap_abe_dmic_hw_params,
-};
-
-/* Headset jack */
-static struct snd_soc_jack hs_jack;
-
-/*Headset jack detection DAPM pins */
-static struct snd_soc_jack_pin hs_jack_pins[] = {
-	{
-		.pin = "Headset Mic",
-		.mask = SND_JACK_MICROPHONE,
-	},
-	{
-		.pin = "Headset Stereophone",
-		.mask = SND_JACK_HEADPHONE,
-	},
-};
-
-/* SDP4430 machine DAPM */
-static const struct snd_soc_dapm_widget twl6040_dapm_widgets[] = {
-	/* Outputs */
-	SND_SOC_DAPM_HP("Headset Stereophone", NULL),
-	SND_SOC_DAPM_SPK("Earphone Spk", NULL),
-	SND_SOC_DAPM_SPK("Ext Spk", NULL),
-	SND_SOC_DAPM_LINE("Line Out", NULL),
-	SND_SOC_DAPM_SPK("Vibrator", NULL),
-
-	/* Inputs */
-	SND_SOC_DAPM_MIC("Headset Mic", NULL),
-	SND_SOC_DAPM_MIC("Main Handset Mic", NULL),
-	SND_SOC_DAPM_MIC("Sub Handset Mic", NULL),
-	SND_SOC_DAPM_LINE("Line In", NULL),
-
-	/* Digital microphones */
-	SND_SOC_DAPM_MIC("Digital Mic", NULL),
-};
-
-static const struct snd_soc_dapm_route audio_map[] = {
-	/* Routings for outputs */
-	{"Headset Stereophone", NULL, "HSOL"},
-	{"Headset Stereophone", NULL, "HSOR"},
-
-	{"Earphone Spk", NULL, "EP"},
-
-	{"Ext Spk", NULL, "HFL"},
-	{"Ext Spk", NULL, "HFR"},
-
-	{"Line Out", NULL, "AUXL"},
-	{"Line Out", NULL, "AUXR"},
-
-	{"Vibrator", NULL, "VIBRAL"},
-	{"Vibrator", NULL, "VIBRAR"},
-
-	/* Routings for inputs */
-	{"HSMIC", NULL, "Headset Mic"},
-	{"Headset Mic", NULL, "Headset Mic Bias"},
-
-	{"MAINMIC", NULL, "Main Handset Mic"},
-	{"Main Handset Mic", NULL, "Main Mic Bias"},
-
-	{"SUBMIC", NULL, "Sub Handset Mic"},
-	{"Sub Handset Mic", NULL, "Main Mic Bias"},
-
-	{"AFML", NULL, "Line In"},
-	{"AFMR", NULL, "Line In"},
-};
-
-static int omap_abe_twl6040_init(struct snd_soc_pcm_runtime *rtd)
-{
-	struct snd_soc_component *component = rtd->codec_dai->component;
-	struct snd_soc_card *card = rtd->card;
-	struct abe_twl6040 *priv = snd_soc_card_get_drvdata(card);
-	int hs_trim;
-	int ret = 0;
-
-	/*
-	 * Configure McPDM offset cancellation based on the HSOTRIM value from
-	 * twl6040.
-	 */
-	hs_trim = twl6040_get_trim_value(component, TWL6040_TRIM_HSOTRIM);
-	omap_mcpdm_configure_dn_offsets(rtd, TWL6040_HSF_TRIM_LEFT(hs_trim),
-					TWL6040_HSF_TRIM_RIGHT(hs_trim));
-
-	/* Headset jack detection only if it is supported */
-	if (priv->jack_detection) {
-		ret = snd_soc_card_jack_new(rtd->card, "Headset Jack",
-					    SND_JACK_HEADSET, &hs_jack,
-					    hs_jack_pins,
-					    ARRAY_SIZE(hs_jack_pins));
-		if (ret)
-			return ret;
-
-		twl6040_hs_jack_detect(component, &hs_jack, SND_JACK_HEADSET);
-	}
-
-	return 0;
-}
-
-static const struct snd_soc_dapm_route dmic_audio_map[] = {
-	{"DMic", NULL, "Digital Mic"},
-	{"Digital Mic", NULL, "Digital Mic1 Bias"},
-};
-
-static int omap_abe_dmic_init(struct snd_soc_pcm_runtime *rtd)
-{
-	struct snd_soc_dapm_context *dapm = &rtd->card->dapm;
-
-	return snd_soc_dapm_add_routes(dapm, dmic_audio_map,
-				ARRAY_SIZE(dmic_audio_map));
-}
-
-static int omap_abe_probe(struct platform_device *pdev)
-{
-	struct device_node *node = pdev->dev.of_node;
-	struct snd_soc_card *card;
-	struct device_node *dai_node;
-	struct abe_twl6040 *priv;
-	int num_links = 0;
-	int ret = 0;
-
-	if (!node) {
-		dev_err(&pdev->dev, "of node is missing.\n");
-		return -ENODEV;
-	}
-
-	priv = devm_kzalloc(&pdev->dev, sizeof(struct abe_twl6040), GFP_KERNEL);
-	if (priv == NULL)
-		return -ENOMEM;
-
-	card = &priv->card;
-	card->dev = &pdev->dev;
-	card->owner = THIS_MODULE;
-	card->dapm_widgets = twl6040_dapm_widgets;
-	card->num_dapm_widgets = ARRAY_SIZE(twl6040_dapm_widgets);
-	card->dapm_routes = audio_map;
-	card->num_dapm_routes = ARRAY_SIZE(audio_map);
-
-	if (snd_soc_of_parse_card_name(card, "ti,model")) {
-		dev_err(&pdev->dev, "Card name is not provided\n");
-		return -ENODEV;
-	}
-
-	ret = snd_soc_of_parse_audio_routing(card, "ti,audio-routing");
-	if (ret) {
-		dev_err(&pdev->dev, "Error while parsing DAPM routing\n");
-		return ret;
-	}
-
-	dai_node = of_parse_phandle(node, "ti,mcpdm", 0);
-	if (!dai_node) {
-		dev_err(&pdev->dev, "McPDM node is not provided\n");
-		return -EINVAL;
-	}
-
-	priv->dai_links[0].name = "DMIC";
-	priv->dai_links[0].stream_name = "TWL6040";
-	priv->dai_links[0].cpu_of_node = dai_node;
-	priv->dai_links[0].platform_of_node = dai_node;
-	priv->dai_links[0].codec_dai_name = "twl6040-legacy";
-	priv->dai_links[0].codec_name = "twl6040-codec";
-	priv->dai_links[0].init = omap_abe_twl6040_init;
-	priv->dai_links[0].ops = &omap_abe_ops;
-
-	dai_node = of_parse_phandle(node, "ti,dmic", 0);
-	if (dai_node) {
-		num_links = 2;
-		priv->dai_links[1].name = "TWL6040";
-		priv->dai_links[1].stream_name = "DMIC Capture";
-		priv->dai_links[1].cpu_of_node = dai_node;
-		priv->dai_links[1].platform_of_node = dai_node;
-		priv->dai_links[1].codec_dai_name = "dmic-hifi";
-		priv->dai_links[1].codec_name = "dmic-codec";
-		priv->dai_links[1].init = omap_abe_dmic_init;
-		priv->dai_links[1].ops = &omap_abe_dmic_ops;
-	} else {
-		num_links = 1;
-	}
-
-	priv->jack_detection = of_property_read_bool(node, "ti,jack-detection");
-	of_property_read_u32(node, "ti,mclk-freq", &priv->mclk_freq);
-	if (!priv->mclk_freq) {
-		dev_err(&pdev->dev, "MCLK frequency not provided\n");
-		return -EINVAL;
-	}
-
-	card->fully_routed = 1;
-
-	if (!priv->mclk_freq) {
-		dev_err(&pdev->dev, "MCLK frequency missing\n");
-		return -ENODEV;
-	}
-
-	card->dai_link = priv->dai_links;
-	card->num_links = num_links;
-
-	snd_soc_card_set_drvdata(card, priv);
-
-	ret = devm_snd_soc_register_card(&pdev->dev, card);
-	if (ret)
-		dev_err(&pdev->dev, "devm_snd_soc_register_card() failed: %d\n",
-			ret);
-
-	return ret;
-}
-
-static const struct of_device_id omap_abe_of_match[] = {
-	{.compatible = "ti,abe-twl6040", },
-	{ },
-};
-MODULE_DEVICE_TABLE(of, omap_abe_of_match);
-
-static struct platform_driver omap_abe_driver = {
-	.driver = {
-		.name = "omap-abe-twl6040",
-		.pm = &snd_soc_pm_ops,
-		.of_match_table = omap_abe_of_match,
-	},
-	.probe = omap_abe_probe,
-};
-
-static int __init omap_abe_init(void)
-{
-	int ret;
-
-	dmic_codec_dev = platform_device_register_simple("dmic-codec", -1, NULL,
-							 0);
-	if (IS_ERR(dmic_codec_dev)) {
-		pr_err("%s: dmic-codec device registration failed\n", __func__);
-		return PTR_ERR(dmic_codec_dev);
-	}
-
-	ret = platform_driver_register(&omap_abe_driver);
-	if (ret) {
-		pr_err("%s: platform driver registration failed\n", __func__);
-		platform_device_unregister(dmic_codec_dev);
-	}
-
-	return ret;
-}
-module_init(omap_abe_init);
-
-static void __exit omap_abe_exit(void)
-{
-	platform_driver_unregister(&omap_abe_driver);
-	platform_device_unregister(dmic_codec_dev);
-}
-module_exit(omap_abe_exit);
-
-MODULE_AUTHOR("Misael Lopez Cruz <misael.lopez@ti.com>");
-MODULE_DESCRIPTION("ALSA SoC for OMAP boards with ABE and twl6040 codec");
-MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:omap-abe-twl6040");
diff -Naur --no-dereference a/sound/soc/omap/omap-dmic.c b/sound/soc/omap/omap-dmic.c
--- a/sound/soc/omap/omap-dmic.c	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/omap/omap-dmic.c	1969-12-31 19:00:00.000000000 -0500
@@ -1,541 +0,0 @@
-/*
- * omap-dmic.c  --  OMAP ASoC DMIC DAI driver
- *
- * Copyright (C) 2010 - 2011 Texas Instruments
- *
- * Author: David Lambert <dlambert@ti.com>
- *	   Misael Lopez Cruz <misael.lopez@ti.com>
- *	   Liam Girdwood <lrg@ti.com>
- *	   Peter Ujfalusi <peter.ujfalusi@ti.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
- * 02110-1301 USA
- *
- */
-
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/platform_device.h>
-#include <linux/err.h>
-#include <linux/clk.h>
-#include <linux/io.h>
-#include <linux/slab.h>
-#include <linux/pm_runtime.h>
-#include <linux/of_device.h>
-
-#include <sound/core.h>
-#include <sound/pcm.h>
-#include <sound/pcm_params.h>
-#include <sound/initval.h>
-#include <sound/soc.h>
-#include <sound/dmaengine_pcm.h>
-
-#include "omap-dmic.h"
-#include "sdma-pcm.h"
-
-struct omap_dmic {
-	struct device *dev;
-	void __iomem *io_base;
-	struct clk *fclk;
-	struct pm_qos_request pm_qos_req;
-	int latency;
-	int fclk_freq;
-	int out_freq;
-	int clk_div;
-	int sysclk;
-	int threshold;
-	u32 ch_enabled;
-	bool active;
-	struct mutex mutex;
-
-	struct snd_dmaengine_dai_dma_data dma_data;
-};
-
-static inline void omap_dmic_write(struct omap_dmic *dmic, u16 reg, u32 val)
-{
-	writel_relaxed(val, dmic->io_base + reg);
-}
-
-static inline int omap_dmic_read(struct omap_dmic *dmic, u16 reg)
-{
-	return readl_relaxed(dmic->io_base + reg);
-}
-
-static inline void omap_dmic_start(struct omap_dmic *dmic)
-{
-	u32 ctrl = omap_dmic_read(dmic, OMAP_DMIC_CTRL_REG);
-
-	/* Configure DMA controller */
-	omap_dmic_write(dmic, OMAP_DMIC_DMAENABLE_SET_REG,
-			OMAP_DMIC_DMA_ENABLE);
-
-	omap_dmic_write(dmic, OMAP_DMIC_CTRL_REG, ctrl | dmic->ch_enabled);
-}
-
-static inline void omap_dmic_stop(struct omap_dmic *dmic)
-{
-	u32 ctrl = omap_dmic_read(dmic, OMAP_DMIC_CTRL_REG);
-	omap_dmic_write(dmic, OMAP_DMIC_CTRL_REG,
-			ctrl & ~OMAP_DMIC_UP_ENABLE_MASK);
-
-	/* Disable DMA request generation */
-	omap_dmic_write(dmic, OMAP_DMIC_DMAENABLE_CLR_REG,
-			OMAP_DMIC_DMA_ENABLE);
-
-}
-
-static inline int dmic_is_enabled(struct omap_dmic *dmic)
-{
-	return omap_dmic_read(dmic, OMAP_DMIC_CTRL_REG) &
-						OMAP_DMIC_UP_ENABLE_MASK;
-}
-
-static int omap_dmic_dai_startup(struct snd_pcm_substream *substream,
-				  struct snd_soc_dai *dai)
-{
-	struct omap_dmic *dmic = snd_soc_dai_get_drvdata(dai);
-	int ret = 0;
-
-	mutex_lock(&dmic->mutex);
-
-	if (!dai->active)
-		dmic->active = 1;
-	else
-		ret = -EBUSY;
-
-	mutex_unlock(&dmic->mutex);
-
-	return ret;
-}
-
-static void omap_dmic_dai_shutdown(struct snd_pcm_substream *substream,
-				    struct snd_soc_dai *dai)
-{
-	struct omap_dmic *dmic = snd_soc_dai_get_drvdata(dai);
-
-	mutex_lock(&dmic->mutex);
-
-	pm_qos_remove_request(&dmic->pm_qos_req);
-
-	if (!dai->active)
-		dmic->active = 0;
-
-	mutex_unlock(&dmic->mutex);
-}
-
-static int omap_dmic_select_divider(struct omap_dmic *dmic, int sample_rate)
-{
-	int divider = -EINVAL;
-
-	/*
-	 * 192KHz rate is only supported with 19.2MHz/3.84MHz clock
-	 * configuration.
-	 */
-	if (sample_rate == 192000) {
-		if (dmic->fclk_freq == 19200000 && dmic->out_freq == 3840000)
-			divider = 0x6; /* Divider: 5 (192KHz sampling rate) */
-		else
-			dev_err(dmic->dev,
-				"invalid clock configuration for 192KHz\n");
-
-		return divider;
-	}
-
-	switch (dmic->out_freq) {
-	case 1536000:
-		if (dmic->fclk_freq != 24576000)
-			goto div_err;
-		divider = 0x4; /* Divider: 16 */
-		break;
-	case 2400000:
-		switch (dmic->fclk_freq) {
-		case 12000000:
-			divider = 0x5; /* Divider: 5 */
-			break;
-		case 19200000:
-			divider = 0x0; /* Divider: 8 */
-			break;
-		case 24000000:
-			divider = 0x2; /* Divider: 10 */
-			break;
-		default:
-			goto div_err;
-		}
-		break;
-	case 3072000:
-		if (dmic->fclk_freq != 24576000)
-			goto div_err;
-		divider = 0x3; /* Divider: 8 */
-		break;
-	case 3840000:
-		if (dmic->fclk_freq != 19200000)
-			goto div_err;
-		divider = 0x1; /* Divider: 5 (96KHz sampling rate) */
-		break;
-	default:
-		dev_err(dmic->dev, "invalid out frequency: %dHz\n",
-			dmic->out_freq);
-		break;
-	}
-
-	return divider;
-
-div_err:
-	dev_err(dmic->dev, "invalid out frequency %dHz for %dHz input\n",
-		dmic->out_freq, dmic->fclk_freq);
-	return -EINVAL;
-}
-
-static int omap_dmic_dai_hw_params(struct snd_pcm_substream *substream,
-				    struct snd_pcm_hw_params *params,
-				    struct snd_soc_dai *dai)
-{
-	struct omap_dmic *dmic = snd_soc_dai_get_drvdata(dai);
-	struct snd_dmaengine_dai_dma_data *dma_data;
-	int channels;
-
-	dmic->clk_div = omap_dmic_select_divider(dmic, params_rate(params));
-	if (dmic->clk_div < 0) {
-		dev_err(dmic->dev, "no valid divider for %dHz from %dHz\n",
-			dmic->out_freq, dmic->fclk_freq);
-		return -EINVAL;
-	}
-
-	dmic->ch_enabled = 0;
-	channels = params_channels(params);
-	switch (channels) {
-	case 6:
-		dmic->ch_enabled |= OMAP_DMIC_UP3_ENABLE;
-		/* fall through */
-	case 4:
-		dmic->ch_enabled |= OMAP_DMIC_UP2_ENABLE;
-		/* fall through */
-	case 2:
-		dmic->ch_enabled |= OMAP_DMIC_UP1_ENABLE;
-		break;
-	default:
-		dev_err(dmic->dev, "invalid number of legacy channels\n");
-		return -EINVAL;
-	}
-
-	/* packet size is threshold * channels */
-	dma_data = snd_soc_dai_get_dma_data(dai, substream);
-	dma_data->maxburst = dmic->threshold * channels;
-	dmic->latency = (OMAP_DMIC_THRES_MAX - dmic->threshold) * USEC_PER_SEC /
-			params_rate(params);
-
-	return 0;
-}
-
-static int omap_dmic_dai_prepare(struct snd_pcm_substream *substream,
-				  struct snd_soc_dai *dai)
-{
-	struct omap_dmic *dmic = snd_soc_dai_get_drvdata(dai);
-	u32 ctrl;
-
-	if (pm_qos_request_active(&dmic->pm_qos_req))
-		pm_qos_update_request(&dmic->pm_qos_req, dmic->latency);
-
-	/* Configure uplink threshold */
-	omap_dmic_write(dmic, OMAP_DMIC_FIFO_CTRL_REG, dmic->threshold);
-
-	ctrl = omap_dmic_read(dmic, OMAP_DMIC_CTRL_REG);
-
-	/* Set dmic out format */
-	ctrl &= ~(OMAP_DMIC_FORMAT | OMAP_DMIC_POLAR_MASK);
-	ctrl |= (OMAP_DMICOUTFORMAT_LJUST | OMAP_DMIC_POLAR1 |
-		 OMAP_DMIC_POLAR2 | OMAP_DMIC_POLAR3);
-
-	/* Configure dmic clock divider */
-	ctrl &= ~OMAP_DMIC_CLK_DIV_MASK;
-	ctrl |= OMAP_DMIC_CLK_DIV(dmic->clk_div);
-
-	omap_dmic_write(dmic, OMAP_DMIC_CTRL_REG, ctrl);
-
-	omap_dmic_write(dmic, OMAP_DMIC_CTRL_REG,
-			ctrl | OMAP_DMICOUTFORMAT_LJUST | OMAP_DMIC_POLAR1 |
-			OMAP_DMIC_POLAR2 | OMAP_DMIC_POLAR3);
-
-	return 0;
-}
-
-static int omap_dmic_dai_trigger(struct snd_pcm_substream *substream,
-				  int cmd, struct snd_soc_dai *dai)
-{
-	struct omap_dmic *dmic = snd_soc_dai_get_drvdata(dai);
-
-	switch (cmd) {
-	case SNDRV_PCM_TRIGGER_START:
-		omap_dmic_start(dmic);
-		break;
-	case SNDRV_PCM_TRIGGER_STOP:
-		omap_dmic_stop(dmic);
-		break;
-	default:
-		break;
-	}
-
-	return 0;
-}
-
-static int omap_dmic_select_fclk(struct omap_dmic *dmic, int clk_id,
-				 unsigned int freq)
-{
-	struct clk *parent_clk, *mux;
-	char *parent_clk_name;
-	int ret = 0;
-
-	switch (freq) {
-	case 12000000:
-	case 19200000:
-	case 24000000:
-	case 24576000:
-		break;
-	default:
-		dev_err(dmic->dev, "invalid input frequency: %dHz\n", freq);
-		dmic->fclk_freq = 0;
-		return -EINVAL;
-	}
-
-	if (dmic->sysclk == clk_id) {
-		dmic->fclk_freq = freq;
-		return 0;
-	}
-
-	/* re-parent not allowed if a stream is ongoing */
-	if (dmic->active && dmic_is_enabled(dmic)) {
-		dev_err(dmic->dev, "can't re-parent when DMIC active\n");
-		return -EBUSY;
-	}
-
-	switch (clk_id) {
-	case OMAP_DMIC_SYSCLK_PAD_CLKS:
-		parent_clk_name = "pad_clks_ck";
-		break;
-	case OMAP_DMIC_SYSCLK_SLIMBLUS_CLKS:
-		parent_clk_name = "slimbus_clk";
-		break;
-	case OMAP_DMIC_SYSCLK_SYNC_MUX_CLKS:
-		parent_clk_name = "dmic_sync_mux_ck";
-		break;
-	default:
-		dev_err(dmic->dev, "fclk clk_id (%d) not supported\n", clk_id);
-		return -EINVAL;
-	}
-
-	parent_clk = clk_get(dmic->dev, parent_clk_name);
-	if (IS_ERR(parent_clk)) {
-		dev_err(dmic->dev, "can't get %s\n", parent_clk_name);
-		return -ENODEV;
-	}
-
-	mux = clk_get_parent(dmic->fclk);
-	if (IS_ERR(mux)) {
-		dev_err(dmic->dev, "can't get fck mux parent\n");
-		clk_put(parent_clk);
-		return -ENODEV;
-	}
-
-	mutex_lock(&dmic->mutex);
-	if (dmic->active) {
-		/* disable clock while reparenting */
-		pm_runtime_put_sync(dmic->dev);
-		ret = clk_set_parent(mux, parent_clk);
-		pm_runtime_get_sync(dmic->dev);
-	} else {
-		ret = clk_set_parent(mux, parent_clk);
-	}
-	mutex_unlock(&dmic->mutex);
-
-	if (ret < 0) {
-		dev_err(dmic->dev, "re-parent failed\n");
-		goto err_busy;
-	}
-
-	dmic->sysclk = clk_id;
-	dmic->fclk_freq = freq;
-
-err_busy:
-	clk_put(mux);
-	clk_put(parent_clk);
-
-	return ret;
-}
-
-static int omap_dmic_select_outclk(struct omap_dmic *dmic, int clk_id,
-				    unsigned int freq)
-{
-	int ret = 0;
-
-	if (clk_id != OMAP_DMIC_ABE_DMIC_CLK) {
-		dev_err(dmic->dev, "output clk_id (%d) not supported\n",
-			clk_id);
-		return -EINVAL;
-	}
-
-	switch (freq) {
-	case 1536000:
-	case 2400000:
-	case 3072000:
-	case 3840000:
-		dmic->out_freq = freq;
-		break;
-	default:
-		dev_err(dmic->dev, "invalid out frequency: %dHz\n", freq);
-		dmic->out_freq = 0;
-		ret = -EINVAL;
-	}
-
-	return ret;
-}
-
-static int omap_dmic_set_dai_sysclk(struct snd_soc_dai *dai, int clk_id,
-				    unsigned int freq, int dir)
-{
-	struct omap_dmic *dmic = snd_soc_dai_get_drvdata(dai);
-
-	if (dir == SND_SOC_CLOCK_IN)
-		return omap_dmic_select_fclk(dmic, clk_id, freq);
-	else if (dir == SND_SOC_CLOCK_OUT)
-		return omap_dmic_select_outclk(dmic, clk_id, freq);
-
-	dev_err(dmic->dev, "invalid clock direction (%d)\n", dir);
-	return -EINVAL;
-}
-
-static const struct snd_soc_dai_ops omap_dmic_dai_ops = {
-	.startup	= omap_dmic_dai_startup,
-	.shutdown	= omap_dmic_dai_shutdown,
-	.hw_params	= omap_dmic_dai_hw_params,
-	.prepare	= omap_dmic_dai_prepare,
-	.trigger	= omap_dmic_dai_trigger,
-	.set_sysclk	= omap_dmic_set_dai_sysclk,
-};
-
-static int omap_dmic_probe(struct snd_soc_dai *dai)
-{
-	struct omap_dmic *dmic = snd_soc_dai_get_drvdata(dai);
-
-	pm_runtime_enable(dmic->dev);
-
-	/* Disable lines while request is ongoing */
-	pm_runtime_get_sync(dmic->dev);
-	omap_dmic_write(dmic, OMAP_DMIC_CTRL_REG, 0x00);
-	pm_runtime_put_sync(dmic->dev);
-
-	/* Configure DMIC threshold value */
-	dmic->threshold = OMAP_DMIC_THRES_MAX - 3;
-
-	snd_soc_dai_init_dma_data(dai, NULL, &dmic->dma_data);
-
-	return 0;
-}
-
-static int omap_dmic_remove(struct snd_soc_dai *dai)
-{
-	struct omap_dmic *dmic = snd_soc_dai_get_drvdata(dai);
-
-	pm_runtime_disable(dmic->dev);
-
-	return 0;
-}
-
-static struct snd_soc_dai_driver omap_dmic_dai = {
-	.name = "omap-dmic",
-	.probe = omap_dmic_probe,
-	.remove = omap_dmic_remove,
-	.capture = {
-		.channels_min = 2,
-		.channels_max = 6,
-		.rates = SNDRV_PCM_RATE_96000 | SNDRV_PCM_RATE_192000,
-		.formats = SNDRV_PCM_FMTBIT_S32_LE,
-		.sig_bits = 24,
-	},
-	.ops = &omap_dmic_dai_ops,
-};
-
-static const struct snd_soc_component_driver omap_dmic_component = {
-	.name		= "omap-dmic",
-};
-
-static int asoc_dmic_probe(struct platform_device *pdev)
-{
-	struct omap_dmic *dmic;
-	struct resource *res;
-	int ret;
-
-	dmic = devm_kzalloc(&pdev->dev, sizeof(struct omap_dmic), GFP_KERNEL);
-	if (!dmic)
-		return -ENOMEM;
-
-	platform_set_drvdata(pdev, dmic);
-	dmic->dev = &pdev->dev;
-	dmic->sysclk = OMAP_DMIC_SYSCLK_SYNC_MUX_CLKS;
-
-	mutex_init(&dmic->mutex);
-
-	dmic->fclk = devm_clk_get(dmic->dev, "fck");
-	if (IS_ERR(dmic->fclk)) {
-		dev_err(dmic->dev, "cant get fck\n");
-		return -ENODEV;
-	}
-
-	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dma");
-	if (!res) {
-		dev_err(dmic->dev, "invalid dma memory resource\n");
-		return -ENODEV;
-	}
-	dmic->dma_data.addr = res->start + OMAP_DMIC_DATA_REG;
-
-	dmic->dma_data.filter_data = "up_link";
-
-	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mpu");
-	dmic->io_base = devm_ioremap_resource(&pdev->dev, res);
-	if (IS_ERR(dmic->io_base))
-		return PTR_ERR(dmic->io_base);
-
-
-	ret = devm_snd_soc_register_component(&pdev->dev,
-					      &omap_dmic_component,
-					      &omap_dmic_dai, 1);
-	if (ret)
-		return ret;
-
-	ret = sdma_pcm_platform_register(&pdev->dev, NULL, "up_link");
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
-static const struct of_device_id omap_dmic_of_match[] = {
-	{ .compatible = "ti,omap4-dmic", },
-	{ }
-};
-MODULE_DEVICE_TABLE(of, omap_dmic_of_match);
-
-static struct platform_driver asoc_dmic_driver = {
-	.driver = {
-		.name = "omap-dmic",
-		.of_match_table = omap_dmic_of_match,
-	},
-	.probe = asoc_dmic_probe,
-};
-
-module_platform_driver(asoc_dmic_driver);
-
-MODULE_ALIAS("platform:omap-dmic");
-MODULE_AUTHOR("Peter Ujfalusi <peter.ujfalusi@ti.com>");
-MODULE_DESCRIPTION("OMAP DMIC ASoC Interface");
-MODULE_LICENSE("GPL");
diff -Naur --no-dereference a/sound/soc/omap/omap-dmic.h b/sound/soc/omap/omap-dmic.h
--- a/sound/soc/omap/omap-dmic.h	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/omap/omap-dmic.h	1969-12-31 19:00:00.000000000 -0500
@@ -1,69 +0,0 @@
-/*
- * omap-dmic.h  --  OMAP Digital Microphone Controller
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#ifndef _OMAP_DMIC_H
-#define _OMAP_DMIC_H
-
-#define OMAP_DMIC_REVISION_REG		0x00
-#define OMAP_DMIC_SYSCONFIG_REG		0x10
-#define OMAP_DMIC_IRQSTATUS_RAW_REG	0x24
-#define OMAP_DMIC_IRQSTATUS_REG		0x28
-#define OMAP_DMIC_IRQENABLE_SET_REG	0x2C
-#define OMAP_DMIC_IRQENABLE_CLR_REG	0x30
-#define OMAP_DMIC_IRQWAKE_EN_REG	0x34
-#define OMAP_DMIC_DMAENABLE_SET_REG	0x38
-#define OMAP_DMIC_DMAENABLE_CLR_REG	0x3C
-#define OMAP_DMIC_DMAWAKEEN_REG		0x40
-#define OMAP_DMIC_CTRL_REG		0x44
-#define OMAP_DMIC_DATA_REG		0x48
-#define OMAP_DMIC_FIFO_CTRL_REG		0x4C
-#define OMAP_DMIC_FIFO_DMIC1R_DATA_REG	0x50
-#define OMAP_DMIC_FIFO_DMIC1L_DATA_REG	0x54
-#define OMAP_DMIC_FIFO_DMIC2R_DATA_REG	0x58
-#define OMAP_DMIC_FIFO_DMIC2L_DATA_REG	0x5C
-#define OMAP_DMIC_FIFO_DMIC3R_DATA_REG	0x60
-#define OMAP_DMIC_FIFO_DMIC3L_DATA_REG	0x64
-
-/* IRQSTATUS_RAW, IRQSTATUS, IRQENABLE_SET, IRQENABLE_CLR bit fields */
-#define OMAP_DMIC_IRQ			(1 << 0)
-#define OMAP_DMIC_IRQ_FULL		(1 << 1)
-#define OMAP_DMIC_IRQ_ALMST_EMPTY	(1 << 2)
-#define OMAP_DMIC_IRQ_EMPTY		(1 << 3)
-#define OMAP_DMIC_IRQ_MASK		0x07
-
-/* DMIC_DMAENABLE bit fields */
-#define OMAP_DMIC_DMA_ENABLE		0x1
-
-/* DMIC_CTRL bit fields */
-#define OMAP_DMIC_UP1_ENABLE		(1 << 0)
-#define OMAP_DMIC_UP2_ENABLE		(1 << 1)
-#define OMAP_DMIC_UP3_ENABLE		(1 << 2)
-#define OMAP_DMIC_UP_ENABLE_MASK	0x7
-#define OMAP_DMIC_FORMAT		(1 << 3)
-#define OMAP_DMIC_POLAR1		(1 << 4)
-#define OMAP_DMIC_POLAR2		(1 << 5)
-#define OMAP_DMIC_POLAR3		(1 << 6)
-#define OMAP_DMIC_POLAR_MASK		(0x7 << 4)
-#define OMAP_DMIC_CLK_DIV(x)		(((x) & 0x7) << 7)
-#define OMAP_DMIC_CLK_DIV_MASK		(0x7 << 7)
-#define	OMAP_DMIC_RESET			(1 << 10)
-
-#define OMAP_DMICOUTFORMAT_LJUST	(0 << 3)
-#define OMAP_DMICOUTFORMAT_RJUST	(1 << 3)
-
-/* DMIC_FIFO_CTRL bit fields */
-#define OMAP_DMIC_THRES_MAX		0xF
-
-enum omap_dmic_clk {
-	OMAP_DMIC_SYSCLK_PAD_CLKS,		/* PAD_CLKS */
-	OMAP_DMIC_SYSCLK_SLIMBLUS_CLKS,		/* SLIMBUS_CLK */
-	OMAP_DMIC_SYSCLK_SYNC_MUX_CLKS,		/* DMIC_SYNC_MUX_CLK */
-	OMAP_DMIC_ABE_DMIC_CLK,			/* abe_dmic_clk */
-};
-
-#endif
diff -Naur --no-dereference a/sound/soc/omap/omap-hdmi-audio.c b/sound/soc/omap/omap-hdmi-audio.c
--- a/sound/soc/omap/omap-hdmi-audio.c	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/omap/omap-hdmi-audio.c	1969-12-31 19:00:00.000000000 -0500
@@ -1,420 +0,0 @@
-/*
- * omap-hdmi-audio.c -- OMAP4+ DSS HDMI audio support library
- *
- * Copyright (C) 2014 Texas Instruments Incorporated - http://www.ti.com
- *
- * Author: Jyri Sarha <jsarha@ti.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- */
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/err.h>
-#include <linux/string.h>
-#include <linux/platform_device.h>
-#include <sound/soc.h>
-#include <sound/pcm_params.h>
-#include <sound/dmaengine_pcm.h>
-#include <uapi/sound/asound.h>
-#include <sound/asoundef.h>
-#include <sound/omap-hdmi-audio.h>
-
-#include "sdma-pcm.h"
-
-#define DRV_NAME "omap-hdmi-audio"
-
-struct hdmi_audio_data {
-	struct snd_soc_card *card;
-
-	const struct omap_hdmi_audio_ops *ops;
-	struct device *dssdev;
-	struct snd_dmaengine_dai_dma_data dma_data;
-	struct omap_dss_audio dss_audio;
-	struct snd_aes_iec958 iec;
-	struct snd_cea_861_aud_if cea;
-
-	struct mutex current_stream_lock;
-	struct snd_pcm_substream *current_stream;
-};
-
-static
-struct hdmi_audio_data *card_drvdata_substream(struct snd_pcm_substream *ss)
-{
-	struct snd_soc_pcm_runtime *rtd = ss->private_data;
-
-	return snd_soc_card_get_drvdata(rtd->card);
-}
-
-static void hdmi_dai_abort(struct device *dev)
-{
-	struct hdmi_audio_data *ad = dev_get_drvdata(dev);
-
-	mutex_lock(&ad->current_stream_lock);
-	if (ad->current_stream && ad->current_stream->runtime &&
-	    snd_pcm_running(ad->current_stream)) {
-		dev_err(dev, "HDMI display disabled, aborting playback\n");
-		snd_pcm_stream_lock_irq(ad->current_stream);
-		snd_pcm_stop(ad->current_stream, SNDRV_PCM_STATE_DISCONNECTED);
-		snd_pcm_stream_unlock_irq(ad->current_stream);
-	}
-	mutex_unlock(&ad->current_stream_lock);
-}
-
-static int hdmi_dai_startup(struct snd_pcm_substream *substream,
-			    struct snd_soc_dai *dai)
-{
-	struct hdmi_audio_data *ad = card_drvdata_substream(substream);
-	int ret;
-	/*
-	 * Make sure that the period bytes are multiple of the DMA packet size.
-	 * Largest packet size we use is 32 32-bit words = 128 bytes
-	 */
-	ret = snd_pcm_hw_constraint_step(substream->runtime, 0,
-					 SNDRV_PCM_HW_PARAM_PERIOD_BYTES, 128);
-	if (ret < 0) {
-		dev_err(dai->dev, "Could not apply period constraint: %d\n",
-			ret);
-		return ret;
-	}
-	ret = snd_pcm_hw_constraint_step(substream->runtime, 0,
-					 SNDRV_PCM_HW_PARAM_BUFFER_BYTES, 128);
-	if (ret < 0) {
-		dev_err(dai->dev, "Could not apply buffer constraint: %d\n",
-			ret);
-		return ret;
-	}
-
-	snd_soc_dai_set_dma_data(dai, substream, &ad->dma_data);
-
-	mutex_lock(&ad->current_stream_lock);
-	ad->current_stream = substream;
-	mutex_unlock(&ad->current_stream_lock);
-
-	ret = ad->ops->audio_startup(ad->dssdev, hdmi_dai_abort);
-
-	if (ret) {
-		mutex_lock(&ad->current_stream_lock);
-		ad->current_stream = NULL;
-		mutex_unlock(&ad->current_stream_lock);
-	}
-
-	return ret;
-}
-
-static int hdmi_dai_hw_params(struct snd_pcm_substream *substream,
-			      struct snd_pcm_hw_params *params,
-			      struct snd_soc_dai *dai)
-{
-	struct hdmi_audio_data *ad = card_drvdata_substream(substream);
-	struct snd_aes_iec958 *iec = &ad->iec;
-	struct snd_cea_861_aud_if *cea = &ad->cea;
-
-	WARN_ON(ad->current_stream != substream);
-
-	switch (params_format(params)) {
-	case SNDRV_PCM_FORMAT_S16_LE:
-		ad->dma_data.maxburst = 16;
-		break;
-	case SNDRV_PCM_FORMAT_S24_LE:
-		ad->dma_data.maxburst = 32;
-		break;
-	default:
-		dev_err(dai->dev, "format not supported!\n");
-		return -EINVAL;
-	}
-
-	ad->dss_audio.iec = iec;
-	ad->dss_audio.cea = cea;
-	/*
-	 * fill the IEC-60958 channel status word
-	 */
-	/* initialize the word bytes */
-	memset(iec->status, 0, sizeof(iec->status));
-
-	/* specify IEC-60958-3 (commercial use) */
-	iec->status[0] &= ~IEC958_AES0_PROFESSIONAL;
-
-	/* specify that the audio is LPCM*/
-	iec->status[0] &= ~IEC958_AES0_NONAUDIO;
-
-	iec->status[0] |= IEC958_AES0_CON_NOT_COPYRIGHT;
-
-	iec->status[0] |= IEC958_AES0_CON_EMPHASIS_NONE;
-
-	iec->status[1] = IEC958_AES1_CON_GENERAL;
-
-	iec->status[2] |= IEC958_AES2_CON_SOURCE_UNSPEC;
-
-	iec->status[2] |= IEC958_AES2_CON_CHANNEL_UNSPEC;
-
-	switch (params_rate(params)) {
-	case 32000:
-		iec->status[3] |= IEC958_AES3_CON_FS_32000;
-		break;
-	case 44100:
-		iec->status[3] |= IEC958_AES3_CON_FS_44100;
-		break;
-	case 48000:
-		iec->status[3] |= IEC958_AES3_CON_FS_48000;
-		break;
-	case 88200:
-		iec->status[3] |= IEC958_AES3_CON_FS_88200;
-		break;
-	case 96000:
-		iec->status[3] |= IEC958_AES3_CON_FS_96000;
-		break;
-	case 176400:
-		iec->status[3] |= IEC958_AES3_CON_FS_176400;
-		break;
-	case 192000:
-		iec->status[3] |= IEC958_AES3_CON_FS_192000;
-		break;
-	default:
-		dev_err(dai->dev, "rate not supported!\n");
-		return -EINVAL;
-	}
-
-	/* specify the clock accuracy */
-	iec->status[3] |= IEC958_AES3_CON_CLOCK_1000PPM;
-
-	/*
-	 * specify the word length. The same word length value can mean
-	 * two different lengths. Hence, we need to specify the maximum
-	 * word length as well.
-	 */
-	switch (params_format(params)) {
-	case SNDRV_PCM_FORMAT_S16_LE:
-		iec->status[4] |= IEC958_AES4_CON_WORDLEN_20_16;
-		iec->status[4] &= ~IEC958_AES4_CON_MAX_WORDLEN_24;
-		break;
-	case SNDRV_PCM_FORMAT_S24_LE:
-		iec->status[4] |= IEC958_AES4_CON_WORDLEN_24_20;
-		iec->status[4] |= IEC958_AES4_CON_MAX_WORDLEN_24;
-		break;
-	default:
-		dev_err(dai->dev, "format not supported!\n");
-		return -EINVAL;
-	}
-
-	/*
-	 * Fill the CEA-861 audio infoframe (see spec for details)
-	 */
-
-	cea->db1_ct_cc = (params_channels(params) - 1)
-		& CEA861_AUDIO_INFOFRAME_DB1CC;
-	cea->db1_ct_cc |= CEA861_AUDIO_INFOFRAME_DB1CT_FROM_STREAM;
-
-	cea->db2_sf_ss = CEA861_AUDIO_INFOFRAME_DB2SF_FROM_STREAM;
-	cea->db2_sf_ss |= CEA861_AUDIO_INFOFRAME_DB2SS_FROM_STREAM;
-
-	cea->db3 = 0; /* not used, all zeros */
-
-	if (params_channels(params) == 2)
-		cea->db4_ca = 0x0;
-	else if (params_channels(params) == 6)
-		cea->db4_ca = 0xb;
-	else
-		cea->db4_ca = 0x13;
-
-	if (cea->db4_ca == 0x00)
-		cea->db5_dminh_lsv = CEA861_AUDIO_INFOFRAME_DB5_DM_INH_PERMITTED;
-	else
-		cea->db5_dminh_lsv = CEA861_AUDIO_INFOFRAME_DB5_DM_INH_PROHIBITED;
-
-	/* the expression is trivial but makes clear what we are doing */
-	cea->db5_dminh_lsv |= (0 & CEA861_AUDIO_INFOFRAME_DB5_LSV);
-
-	return ad->ops->audio_config(ad->dssdev, &ad->dss_audio);
-}
-
-static int hdmi_dai_trigger(struct snd_pcm_substream *substream, int cmd,
-			    struct snd_soc_dai *dai)
-{
-	struct hdmi_audio_data *ad = card_drvdata_substream(substream);
-	int err = 0;
-
-	WARN_ON(ad->current_stream != substream);
-
-	switch (cmd) {
-	case SNDRV_PCM_TRIGGER_START:
-	case SNDRV_PCM_TRIGGER_RESUME:
-	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
-		err = ad->ops->audio_start(ad->dssdev);
-		break;
-	case SNDRV_PCM_TRIGGER_STOP:
-	case SNDRV_PCM_TRIGGER_SUSPEND:
-	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
-		ad->ops->audio_stop(ad->dssdev);
-		break;
-	default:
-		err = -EINVAL;
-	}
-	return err;
-}
-
-static void hdmi_dai_shutdown(struct snd_pcm_substream *substream,
-			      struct snd_soc_dai *dai)
-{
-	struct hdmi_audio_data *ad = card_drvdata_substream(substream);
-
-	WARN_ON(ad->current_stream != substream);
-
-	ad->ops->audio_shutdown(ad->dssdev);
-
-	mutex_lock(&ad->current_stream_lock);
-	ad->current_stream = NULL;
-	mutex_unlock(&ad->current_stream_lock);
-}
-
-static const struct snd_soc_dai_ops hdmi_dai_ops = {
-	.startup	= hdmi_dai_startup,
-	.hw_params	= hdmi_dai_hw_params,
-	.trigger	= hdmi_dai_trigger,
-	.shutdown	= hdmi_dai_shutdown,
-};
-
-static const struct snd_soc_component_driver omap_hdmi_component = {
-	.name = "omapdss_hdmi",
-};
-
-static struct snd_soc_dai_driver omap5_hdmi_dai = {
-	.name = "omap5-hdmi-dai",
-	.playback = {
-		.channels_min = 2,
-		.channels_max = 8,
-		.rates = (SNDRV_PCM_RATE_32000 | SNDRV_PCM_RATE_44100 |
-			  SNDRV_PCM_RATE_48000 | SNDRV_PCM_RATE_88200 |
-			  SNDRV_PCM_RATE_96000 | SNDRV_PCM_RATE_176400 |
-			  SNDRV_PCM_RATE_192000),
-		.formats = SNDRV_PCM_FMTBIT_S16_LE,
-	},
-	.ops = &hdmi_dai_ops,
-};
-
-static struct snd_soc_dai_driver omap4_hdmi_dai = {
-	.name = "omap4-hdmi-dai",
-	.playback = {
-		.channels_min = 2,
-		.channels_max = 8,
-		.rates = (SNDRV_PCM_RATE_32000 | SNDRV_PCM_RATE_44100 |
-			  SNDRV_PCM_RATE_48000 | SNDRV_PCM_RATE_88200 |
-			  SNDRV_PCM_RATE_96000 | SNDRV_PCM_RATE_176400 |
-			  SNDRV_PCM_RATE_192000),
-		.formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S24_LE,
-	},
-	.ops = &hdmi_dai_ops,
-};
-
-static int omap_hdmi_audio_probe(struct platform_device *pdev)
-{
-	struct omap_hdmi_audio_pdata *ha = pdev->dev.platform_data;
-	struct device *dev = &pdev->dev;
-	struct hdmi_audio_data *ad;
-	struct snd_soc_dai_driver *dai_drv;
-	struct snd_soc_card *card;
-	int ret;
-
-	if (!ha) {
-		dev_err(dev, "No platform data\n");
-		return -EINVAL;
-	}
-
-	ad = devm_kzalloc(dev, sizeof(*ad), GFP_KERNEL);
-	if (!ad)
-		return -ENOMEM;
-	ad->dssdev = ha->dev;
-	ad->ops = ha->ops;
-	ad->dma_data.addr = ha->audio_dma_addr;
-	ad->dma_data.filter_data = "audio_tx";
-	ad->dma_data.addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
-	mutex_init(&ad->current_stream_lock);
-
-	switch (ha->version) {
-	case 4:
-		dai_drv = &omap4_hdmi_dai;
-		break;
-	case 5:
-		dai_drv = &omap5_hdmi_dai;
-		break;
-	default:
-		return -EINVAL;
-	}
-	ret = snd_soc_register_component(ad->dssdev, &omap_hdmi_component,
-					 dai_drv, 1);
-	if (ret)
-		return ret;
-
-	ret = sdma_pcm_platform_register(ad->dssdev, "audio_tx", NULL);
-	if (ret)
-		return ret;
-
-	card = devm_kzalloc(dev, sizeof(*card), GFP_KERNEL);
-	if (!card)
-		return -ENOMEM;
-
-	card->name = devm_kasprintf(dev, GFP_KERNEL,
-				    "HDMI %s", dev_name(ad->dssdev));
-	if (!card->name)
-		return -ENOMEM;
-
-	card->owner = THIS_MODULE;
-	card->dai_link =
-		devm_kzalloc(dev, sizeof(*(card->dai_link)), GFP_KERNEL);
-	if (!card->dai_link)
-		return -ENOMEM;
-	card->dai_link->name = card->name;
-	card->dai_link->stream_name = card->name;
-	card->dai_link->cpu_dai_name = dev_name(ad->dssdev);
-	card->dai_link->platform_name = dev_name(ad->dssdev);
-	card->dai_link->codec_name = "snd-soc-dummy";
-	card->dai_link->codec_dai_name = "snd-soc-dummy-dai";
-	card->num_links = 1;
-	card->dev = dev;
-
-	ret = snd_soc_register_card(card);
-	if (ret) {
-		dev_err(dev, "snd_soc_register_card failed (%d)\n", ret);
-		snd_soc_unregister_component(ad->dssdev);
-		return ret;
-	}
-
-	ad->card = card;
-	snd_soc_card_set_drvdata(card, ad);
-
-	dev_set_drvdata(dev, ad);
-
-	return 0;
-}
-
-static int omap_hdmi_audio_remove(struct platform_device *pdev)
-{
-	struct hdmi_audio_data *ad = platform_get_drvdata(pdev);
-
-	snd_soc_unregister_card(ad->card);
-	snd_soc_unregister_component(ad->dssdev);
-	return 0;
-}
-
-static struct platform_driver hdmi_audio_driver = {
-	.driver = {
-		.name = DRV_NAME,
-	},
-	.probe = omap_hdmi_audio_probe,
-	.remove = omap_hdmi_audio_remove,
-};
-
-module_platform_driver(hdmi_audio_driver);
-
-MODULE_AUTHOR("Jyri Sarha <jsarha@ti.com>");
-MODULE_DESCRIPTION("OMAP HDMI Audio Driver");
-MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:" DRV_NAME);
diff -Naur --no-dereference a/sound/soc/omap/omap-mcbsp.c b/sound/soc/omap/omap-mcbsp.c
--- a/sound/soc/omap/omap-mcbsp.c	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/omap/omap-mcbsp.c	1969-12-31 19:00:00.000000000 -0500
@@ -1,906 +0,0 @@
-/*
- * omap-mcbsp.c  --  OMAP ALSA SoC DAI driver using McBSP port
- *
- * Copyright (C) 2008 Nokia Corporation
- *
- * Contact: Jarkko Nikula <jarkko.nikula@bitmer.com>
- *          Peter Ujfalusi <peter.ujfalusi@ti.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
- * 02110-1301 USA
- *
- */
-
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/device.h>
-#include <linux/pm_runtime.h>
-#include <linux/of.h>
-#include <linux/of_device.h>
-#include <sound/core.h>
-#include <sound/pcm.h>
-#include <sound/pcm_params.h>
-#include <sound/initval.h>
-#include <sound/soc.h>
-#include <sound/dmaengine_pcm.h>
-
-#include <linux/platform_data/asoc-ti-mcbsp.h>
-#include "mcbsp.h"
-#include "omap-mcbsp.h"
-#include "sdma-pcm.h"
-
-#define OMAP_MCBSP_RATES	(SNDRV_PCM_RATE_8000_96000)
-
-#define OMAP_MCBSP_SOC_SINGLE_S16_EXT(xname, xmin, xmax, \
-	xhandler_get, xhandler_put) \
-{	.iface = SNDRV_CTL_ELEM_IFACE_MIXER, .name = xname, \
-	.info = omap_mcbsp_st_info_volsw, \
-	.get = xhandler_get, .put = xhandler_put, \
-	.private_value = (unsigned long) &(struct soc_mixer_control) \
-	{.min = xmin, .max = xmax} }
-
-enum {
-	OMAP_MCBSP_WORD_8 = 0,
-	OMAP_MCBSP_WORD_12,
-	OMAP_MCBSP_WORD_16,
-	OMAP_MCBSP_WORD_20,
-	OMAP_MCBSP_WORD_24,
-	OMAP_MCBSP_WORD_32,
-};
-
-/*
- * Stream DMA parameters. DMA request line and port address are set runtime
- * since they are different between OMAP1 and later OMAPs
- */
-static void omap_mcbsp_set_threshold(struct snd_pcm_substream *substream,
-		unsigned int packet_size)
-{
-	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
-	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);
-	int words;
-
-	/*
-	 * Configure McBSP threshold based on either:
-	 * packet_size, when the sDMA is in packet mode, or based on the
-	 * period size in THRESHOLD mode, otherwise use McBSP threshold = 1
-	 * for mono streams.
-	 */
-	if (packet_size)
-		words = packet_size;
-	else
-		words = 1;
-
-	/* Configure McBSP internal buffer usage */
-	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
-		omap_mcbsp_set_tx_threshold(mcbsp, words);
-	else
-		omap_mcbsp_set_rx_threshold(mcbsp, words);
-}
-
-static int omap_mcbsp_hwrule_min_buffersize(struct snd_pcm_hw_params *params,
-				    struct snd_pcm_hw_rule *rule)
-{
-	struct snd_interval *buffer_size = hw_param_interval(params,
-					SNDRV_PCM_HW_PARAM_BUFFER_SIZE);
-	struct snd_interval *channels = hw_param_interval(params,
-					SNDRV_PCM_HW_PARAM_CHANNELS);
-	struct omap_mcbsp *mcbsp = rule->private;
-	struct snd_interval frames;
-	int size;
-
-	snd_interval_any(&frames);
-	size = mcbsp->pdata->buffer_size;
-
-	frames.min = size / channels->min;
-	frames.integer = 1;
-	return snd_interval_refine(buffer_size, &frames);
-}
-
-static int omap_mcbsp_dai_startup(struct snd_pcm_substream *substream,
-				  struct snd_soc_dai *cpu_dai)
-{
-	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);
-	int err = 0;
-
-	if (!cpu_dai->active)
-		err = omap_mcbsp_request(mcbsp);
-
-	/*
-	 * OMAP3 McBSP FIFO is word structured.
-	 * McBSP2 has 1024 + 256 = 1280 word long buffer,
-	 * McBSP1,3,4,5 has 128 word long buffer
-	 * This means that the size of the FIFO depends on the sample format.
-	 * For example on McBSP3:
-	 * 16bit samples: size is 128 * 2 = 256 bytes
-	 * 32bit samples: size is 128 * 4 = 512 bytes
-	 * It is simpler to place constraint for buffer and period based on
-	 * channels.
-	 * McBSP3 as example again (16 or 32 bit samples):
-	 * 1 channel (mono): size is 128 frames (128 words)
-	 * 2 channels (stereo): size is 128 / 2 = 64 frames (2 * 64 words)
-	 * 4 channels: size is 128 / 4 = 32 frames (4 * 32 words)
-	 */
-	if (mcbsp->pdata->buffer_size) {
-		/*
-		* Rule for the buffer size. We should not allow
-		* smaller buffer than the FIFO size to avoid underruns.
-		* This applies only for the playback stream.
-		*/
-		if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
-			snd_pcm_hw_rule_add(substream->runtime, 0,
-					    SNDRV_PCM_HW_PARAM_BUFFER_SIZE,
-					    omap_mcbsp_hwrule_min_buffersize,
-					    mcbsp,
-					    SNDRV_PCM_HW_PARAM_CHANNELS, -1);
-
-		/* Make sure, that the period size is always even */
-		snd_pcm_hw_constraint_step(substream->runtime, 0,
-					   SNDRV_PCM_HW_PARAM_PERIOD_SIZE, 2);
-	}
-
-	return err;
-}
-
-static void omap_mcbsp_dai_shutdown(struct snd_pcm_substream *substream,
-				    struct snd_soc_dai *cpu_dai)
-{
-	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);
-	int tx = (substream->stream == SNDRV_PCM_STREAM_PLAYBACK);
-	int stream1 = tx ? SNDRV_PCM_STREAM_PLAYBACK : SNDRV_PCM_STREAM_CAPTURE;
-	int stream2 = tx ? SNDRV_PCM_STREAM_CAPTURE : SNDRV_PCM_STREAM_PLAYBACK;
-
-	if (mcbsp->latency[stream2])
-		pm_qos_update_request(&mcbsp->pm_qos_req,
-				      mcbsp->latency[stream2]);
-	else if (mcbsp->latency[stream1])
-		pm_qos_remove_request(&mcbsp->pm_qos_req);
-
-	mcbsp->latency[stream1] = 0;
-
-	if (!cpu_dai->active) {
-		omap_mcbsp_free(mcbsp);
-		mcbsp->configured = 0;
-	}
-}
-
-static int omap_mcbsp_dai_prepare(struct snd_pcm_substream *substream,
-				  struct snd_soc_dai *cpu_dai)
-{
-	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);
-	struct pm_qos_request *pm_qos_req = &mcbsp->pm_qos_req;
-	int tx = (substream->stream == SNDRV_PCM_STREAM_PLAYBACK);
-	int stream1 = tx ? SNDRV_PCM_STREAM_PLAYBACK : SNDRV_PCM_STREAM_CAPTURE;
-	int stream2 = tx ? SNDRV_PCM_STREAM_CAPTURE : SNDRV_PCM_STREAM_PLAYBACK;
-	int latency = mcbsp->latency[stream2];
-
-	/* Prevent omap hardware from hitting off between FIFO fills */
-	if (!latency || mcbsp->latency[stream1] < latency)
-		latency = mcbsp->latency[stream1];
-
-	if (pm_qos_request_active(pm_qos_req))
-		pm_qos_update_request(pm_qos_req, latency);
-	else if (latency)
-		pm_qos_add_request(pm_qos_req, PM_QOS_CPU_DMA_LATENCY, latency);
-
-	return 0;
-}
-
-static int omap_mcbsp_dai_trigger(struct snd_pcm_substream *substream, int cmd,
-				  struct snd_soc_dai *cpu_dai)
-{
-	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);
-	int err = 0, play = (substream->stream == SNDRV_PCM_STREAM_PLAYBACK);
-
-	switch (cmd) {
-	case SNDRV_PCM_TRIGGER_START:
-	case SNDRV_PCM_TRIGGER_RESUME:
-	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
-		mcbsp->active++;
-		omap_mcbsp_start(mcbsp, play, !play);
-		break;
-
-	case SNDRV_PCM_TRIGGER_STOP:
-	case SNDRV_PCM_TRIGGER_SUSPEND:
-	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
-		omap_mcbsp_stop(mcbsp, play, !play);
-		mcbsp->active--;
-		break;
-	default:
-		err = -EINVAL;
-	}
-
-	return err;
-}
-
-static snd_pcm_sframes_t omap_mcbsp_dai_delay(
-			struct snd_pcm_substream *substream,
-			struct snd_soc_dai *dai)
-{
-	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
-	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);
-	u16 fifo_use;
-	snd_pcm_sframes_t delay;
-
-	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
-		fifo_use = omap_mcbsp_get_tx_delay(mcbsp);
-	else
-		fifo_use = omap_mcbsp_get_rx_delay(mcbsp);
-
-	/*
-	 * Divide the used locations with the channel count to get the
-	 * FIFO usage in samples (don't care about partial samples in the
-	 * buffer).
-	 */
-	delay = fifo_use / substream->runtime->channels;
-
-	return delay;
-}
-
-static int omap_mcbsp_dai_hw_params(struct snd_pcm_substream *substream,
-				    struct snd_pcm_hw_params *params,
-				    struct snd_soc_dai *cpu_dai)
-{
-	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);
-	struct omap_mcbsp_reg_cfg *regs = &mcbsp->cfg_regs;
-	struct snd_dmaengine_dai_dma_data *dma_data;
-	int wlen, channels, wpf;
-	int pkt_size = 0;
-	unsigned int format, div, framesize, master;
-	unsigned int buffer_size = mcbsp->pdata->buffer_size;
-
-	dma_data = snd_soc_dai_get_dma_data(cpu_dai, substream);
-	channels = params_channels(params);
-
-	switch (params_format(params)) {
-	case SNDRV_PCM_FORMAT_S16_LE:
-		wlen = 16;
-		break;
-	case SNDRV_PCM_FORMAT_S32_LE:
-		wlen = 32;
-		break;
-	default:
-		return -EINVAL;
-	}
-	if (buffer_size) {
-		int latency;
-
-		if (mcbsp->dma_op_mode == MCBSP_DMA_MODE_THRESHOLD) {
-			int period_words, max_thrsh;
-			int divider = 0;
-
-			period_words = params_period_bytes(params) / (wlen / 8);
-			if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
-				max_thrsh = mcbsp->max_tx_thres;
-			else
-				max_thrsh = mcbsp->max_rx_thres;
-			/*
-			 * Use sDMA packet mode if McBSP is in threshold mode:
-			 * If period words less than the FIFO size the packet
-			 * size is set to the number of period words, otherwise
-			 * Look for the biggest threshold value which divides
-			 * the period size evenly.
-			 */
-			divider = period_words / max_thrsh;
-			if (period_words % max_thrsh)
-				divider++;
-			while (period_words % divider &&
-				divider < period_words)
-				divider++;
-			if (divider == period_words)
-				return -EINVAL;
-
-			pkt_size = period_words / divider;
-		} else if (channels > 1) {
-			/* Use packet mode for non mono streams */
-			pkt_size = channels;
-		}
-
-		latency = (buffer_size - pkt_size) / channels;
-		latency = latency * USEC_PER_SEC /
-			  (params->rate_num / params->rate_den);
-		mcbsp->latency[substream->stream] = latency;
-
-		omap_mcbsp_set_threshold(substream, pkt_size);
-	}
-
-	dma_data->maxburst = pkt_size;
-
-	if (mcbsp->configured) {
-		/* McBSP already configured by another stream */
-		return 0;
-	}
-
-	regs->rcr2	&= ~(RPHASE | RFRLEN2(0x7f) | RWDLEN2(7));
-	regs->xcr2	&= ~(RPHASE | XFRLEN2(0x7f) | XWDLEN2(7));
-	regs->rcr1	&= ~(RFRLEN1(0x7f) | RWDLEN1(7));
-	regs->xcr1	&= ~(XFRLEN1(0x7f) | XWDLEN1(7));
-	format = mcbsp->fmt & SND_SOC_DAIFMT_FORMAT_MASK;
-	wpf = channels;
-	if (channels == 2 && (format == SND_SOC_DAIFMT_I2S ||
-			      format == SND_SOC_DAIFMT_LEFT_J)) {
-		/* Use dual-phase frames */
-		regs->rcr2	|= RPHASE;
-		regs->xcr2	|= XPHASE;
-		/* Set 1 word per (McBSP) frame for phase1 and phase2 */
-		wpf--;
-		regs->rcr2	|= RFRLEN2(wpf - 1);
-		regs->xcr2	|= XFRLEN2(wpf - 1);
-	}
-
-	regs->rcr1	|= RFRLEN1(wpf - 1);
-	regs->xcr1	|= XFRLEN1(wpf - 1);
-
-	switch (params_format(params)) {
-	case SNDRV_PCM_FORMAT_S16_LE:
-		/* Set word lengths */
-		regs->rcr2	|= RWDLEN2(OMAP_MCBSP_WORD_16);
-		regs->rcr1	|= RWDLEN1(OMAP_MCBSP_WORD_16);
-		regs->xcr2	|= XWDLEN2(OMAP_MCBSP_WORD_16);
-		regs->xcr1	|= XWDLEN1(OMAP_MCBSP_WORD_16);
-		break;
-	case SNDRV_PCM_FORMAT_S32_LE:
-		/* Set word lengths */
-		regs->rcr2	|= RWDLEN2(OMAP_MCBSP_WORD_32);
-		regs->rcr1	|= RWDLEN1(OMAP_MCBSP_WORD_32);
-		regs->xcr2	|= XWDLEN2(OMAP_MCBSP_WORD_32);
-		regs->xcr1	|= XWDLEN1(OMAP_MCBSP_WORD_32);
-		break;
-	default:
-		/* Unsupported PCM format */
-		return -EINVAL;
-	}
-
-	/* In McBSP master modes, FRAME (i.e. sample rate) is generated
-	 * by _counting_ BCLKs. Calculate frame size in BCLKs */
-	master = mcbsp->fmt & SND_SOC_DAIFMT_MASTER_MASK;
-	if (master ==	SND_SOC_DAIFMT_CBS_CFS) {
-		div = mcbsp->clk_div ? mcbsp->clk_div : 1;
-		framesize = (mcbsp->in_freq / div) / params_rate(params);
-
-		if (framesize < wlen * channels) {
-			printk(KERN_ERR "%s: not enough bandwidth for desired rate and "
-					"channels\n", __func__);
-			return -EINVAL;
-		}
-	} else
-		framesize = wlen * channels;
-
-	/* Set FS period and length in terms of bit clock periods */
-	regs->srgr2	&= ~FPER(0xfff);
-	regs->srgr1	&= ~FWID(0xff);
-	switch (format) {
-	case SND_SOC_DAIFMT_I2S:
-	case SND_SOC_DAIFMT_LEFT_J:
-		regs->srgr2	|= FPER(framesize - 1);
-		regs->srgr1	|= FWID((framesize >> 1) - 1);
-		break;
-	case SND_SOC_DAIFMT_DSP_A:
-	case SND_SOC_DAIFMT_DSP_B:
-		regs->srgr2	|= FPER(framesize - 1);
-		regs->srgr1	|= FWID(0);
-		break;
-	}
-
-	omap_mcbsp_config(mcbsp, &mcbsp->cfg_regs);
-	mcbsp->wlen = wlen;
-	mcbsp->configured = 1;
-
-	return 0;
-}
-
-/*
- * This must be called before _set_clkdiv and _set_sysclk since McBSP register
- * cache is initialized here
- */
-static int omap_mcbsp_dai_set_dai_fmt(struct snd_soc_dai *cpu_dai,
-				      unsigned int fmt)
-{
-	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);
-	struct omap_mcbsp_reg_cfg *regs = &mcbsp->cfg_regs;
-	bool inv_fs = false;
-
-	if (mcbsp->configured)
-		return 0;
-
-	mcbsp->fmt = fmt;
-	memset(regs, 0, sizeof(*regs));
-	/* Generic McBSP register settings */
-	regs->spcr2	|= XINTM(3) | FREE;
-	regs->spcr1	|= RINTM(3);
-	/* RFIG and XFIG are not defined in 2430 and on OMAP3+ */
-	if (!mcbsp->pdata->has_ccr) {
-		regs->rcr2	|= RFIG;
-		regs->xcr2	|= XFIG;
-	}
-
-	/* Configure XCCR/RCCR only for revisions which have ccr registers */
-	if (mcbsp->pdata->has_ccr) {
-		regs->xccr = DXENDLY(1) | XDMAEN | XDISABLE;
-		regs->rccr = RFULL_CYCLE | RDMAEN | RDISABLE;
-	}
-
-	switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) {
-	case SND_SOC_DAIFMT_I2S:
-		/* 1-bit data delay */
-		regs->rcr2	|= RDATDLY(1);
-		regs->xcr2	|= XDATDLY(1);
-		break;
-	case SND_SOC_DAIFMT_LEFT_J:
-		/* 0-bit data delay */
-		regs->rcr2	|= RDATDLY(0);
-		regs->xcr2	|= XDATDLY(0);
-		regs->spcr1	|= RJUST(2);
-		/* Invert FS polarity configuration */
-		inv_fs = true;
-		break;
-	case SND_SOC_DAIFMT_DSP_A:
-		/* 1-bit data delay */
-		regs->rcr2      |= RDATDLY(1);
-		regs->xcr2      |= XDATDLY(1);
-		/* Invert FS polarity configuration */
-		inv_fs = true;
-		break;
-	case SND_SOC_DAIFMT_DSP_B:
-		/* 0-bit data delay */
-		regs->rcr2      |= RDATDLY(0);
-		regs->xcr2      |= XDATDLY(0);
-		/* Invert FS polarity configuration */
-		inv_fs = true;
-		break;
-	default:
-		/* Unsupported data format */
-		return -EINVAL;
-	}
-
-	switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) {
-	case SND_SOC_DAIFMT_CBS_CFS:
-		/* McBSP master. Set FS and bit clocks as outputs */
-		regs->pcr0	|= FSXM | FSRM |
-				   CLKXM | CLKRM;
-		/* Sample rate generator drives the FS */
-		regs->srgr2	|= FSGM;
-		break;
-	case SND_SOC_DAIFMT_CBM_CFS:
-		/* McBSP slave. FS clock as output */
-		regs->srgr2	|= FSGM;
-		regs->pcr0	|= FSXM | FSRM;
-		break;
-	case SND_SOC_DAIFMT_CBM_CFM:
-		/* McBSP slave */
-		break;
-	default:
-		/* Unsupported master/slave configuration */
-		return -EINVAL;
-	}
-
-	/* Set bit clock (CLKX/CLKR) and FS polarities */
-	switch (fmt & SND_SOC_DAIFMT_INV_MASK) {
-	case SND_SOC_DAIFMT_NB_NF:
-		/*
-		 * Normal BCLK + FS.
-		 * FS active low. TX data driven on falling edge of bit clock
-		 * and RX data sampled on rising edge of bit clock.
-		 */
-		regs->pcr0	|= FSXP | FSRP |
-				   CLKXP | CLKRP;
-		break;
-	case SND_SOC_DAIFMT_NB_IF:
-		regs->pcr0	|= CLKXP | CLKRP;
-		break;
-	case SND_SOC_DAIFMT_IB_NF:
-		regs->pcr0	|= FSXP | FSRP;
-		break;
-	case SND_SOC_DAIFMT_IB_IF:
-		break;
-	default:
-		return -EINVAL;
-	}
-	if (inv_fs == true)
-		regs->pcr0 ^= FSXP | FSRP;
-
-	return 0;
-}
-
-static int omap_mcbsp_dai_set_clkdiv(struct snd_soc_dai *cpu_dai,
-				     int div_id, int div)
-{
-	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);
-	struct omap_mcbsp_reg_cfg *regs = &mcbsp->cfg_regs;
-
-	if (div_id != OMAP_MCBSP_CLKGDV)
-		return -ENODEV;
-
-	mcbsp->clk_div = div;
-	regs->srgr1	&= ~CLKGDV(0xff);
-	regs->srgr1	|= CLKGDV(div - 1);
-
-	return 0;
-}
-
-static int omap_mcbsp_dai_set_dai_sysclk(struct snd_soc_dai *cpu_dai,
-					 int clk_id, unsigned int freq,
-					 int dir)
-{
-	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);
-	struct omap_mcbsp_reg_cfg *regs = &mcbsp->cfg_regs;
-	int err = 0;
-
-	if (mcbsp->active) {
-		if (freq == mcbsp->in_freq)
-			return 0;
-		else
-			return -EBUSY;
-	}
-
-	mcbsp->in_freq = freq;
-	regs->srgr2 &= ~CLKSM;
-	regs->pcr0 &= ~SCLKME;
-
-	switch (clk_id) {
-	case OMAP_MCBSP_SYSCLK_CLK:
-		regs->srgr2	|= CLKSM;
-		break;
-	case OMAP_MCBSP_SYSCLK_CLKS_FCLK:
-		if (mcbsp_omap1()) {
-			err = -EINVAL;
-			break;
-		}
-		err = omap2_mcbsp_set_clks_src(mcbsp,
-					       MCBSP_CLKS_PRCM_SRC);
-		break;
-	case OMAP_MCBSP_SYSCLK_CLKS_EXT:
-		if (mcbsp_omap1()) {
-			err = 0;
-			break;
-		}
-		err = omap2_mcbsp_set_clks_src(mcbsp,
-					       MCBSP_CLKS_PAD_SRC);
-		break;
-
-	case OMAP_MCBSP_SYSCLK_CLKX_EXT:
-		regs->srgr2	|= CLKSM;
-		regs->pcr0	|= SCLKME;
-		/*
-		 * If McBSP is master but yet the CLKX/CLKR pin drives the SRG,
-		 * disable output on those pins. This enables to inject the
-		 * reference clock through CLKX/CLKR. For this to work
-		 * set_dai_sysclk() _needs_ to be called after set_dai_fmt().
-		 */
-		regs->pcr0	&= ~CLKXM;
-		break;
-	case OMAP_MCBSP_SYSCLK_CLKR_EXT:
-		regs->pcr0	|= SCLKME;
-		/* Disable ouput on CLKR pin in master mode */
-		regs->pcr0	&= ~CLKRM;
-		break;
-	default:
-		err = -ENODEV;
-	}
-
-	return err;
-}
-
-static const struct snd_soc_dai_ops mcbsp_dai_ops = {
-	.startup	= omap_mcbsp_dai_startup,
-	.shutdown	= omap_mcbsp_dai_shutdown,
-	.prepare	= omap_mcbsp_dai_prepare,
-	.trigger	= omap_mcbsp_dai_trigger,
-	.delay		= omap_mcbsp_dai_delay,
-	.hw_params	= omap_mcbsp_dai_hw_params,
-	.set_fmt	= omap_mcbsp_dai_set_dai_fmt,
-	.set_clkdiv	= omap_mcbsp_dai_set_clkdiv,
-	.set_sysclk	= omap_mcbsp_dai_set_dai_sysclk,
-};
-
-static int omap_mcbsp_probe(struct snd_soc_dai *dai)
-{
-	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(dai);
-
-	pm_runtime_enable(mcbsp->dev);
-
-	snd_soc_dai_init_dma_data(dai,
-				  &mcbsp->dma_data[SNDRV_PCM_STREAM_PLAYBACK],
-				  &mcbsp->dma_data[SNDRV_PCM_STREAM_CAPTURE]);
-
-	return 0;
-}
-
-static int omap_mcbsp_remove(struct snd_soc_dai *dai)
-{
-	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(dai);
-
-	pm_runtime_disable(mcbsp->dev);
-
-	return 0;
-}
-
-static struct snd_soc_dai_driver omap_mcbsp_dai = {
-	.probe = omap_mcbsp_probe,
-	.remove = omap_mcbsp_remove,
-	.playback = {
-		.channels_min = 1,
-		.channels_max = 16,
-		.rates = OMAP_MCBSP_RATES,
-		.formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S32_LE,
-	},
-	.capture = {
-		.channels_min = 1,
-		.channels_max = 16,
-		.rates = OMAP_MCBSP_RATES,
-		.formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S32_LE,
-	},
-	.ops = &mcbsp_dai_ops,
-};
-
-static const struct snd_soc_component_driver omap_mcbsp_component = {
-	.name		= "omap-mcbsp",
-};
-
-static int omap_mcbsp_st_info_volsw(struct snd_kcontrol *kcontrol,
-			struct snd_ctl_elem_info *uinfo)
-{
-	struct soc_mixer_control *mc =
-		(struct soc_mixer_control *)kcontrol->private_value;
-	int max = mc->max;
-	int min = mc->min;
-
-	uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER;
-	uinfo->count = 1;
-	uinfo->value.integer.min = min;
-	uinfo->value.integer.max = max;
-	return 0;
-}
-
-#define OMAP_MCBSP_ST_CHANNEL_VOLUME(channel)				\
-static int								\
-omap_mcbsp_set_st_ch##channel##_volume(struct snd_kcontrol *kc,		\
-					struct snd_ctl_elem_value *uc)	\
-{									\
-	struct snd_soc_dai *cpu_dai = snd_kcontrol_chip(kc);		\
-	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);	\
-	struct soc_mixer_control *mc =					\
-		(struct soc_mixer_control *)kc->private_value;		\
-	int max = mc->max;						\
-	int min = mc->min;						\
-	int val = uc->value.integer.value[0];				\
-									\
-	if (val < min || val > max)					\
-		return -EINVAL;						\
-									\
-	/* OMAP McBSP implementation uses index values 0..4 */		\
-	return omap_st_set_chgain(mcbsp, channel, val);			\
-}									\
-									\
-static int								\
-omap_mcbsp_get_st_ch##channel##_volume(struct snd_kcontrol *kc,		\
-					struct snd_ctl_elem_value *uc)	\
-{									\
-	struct snd_soc_dai *cpu_dai = snd_kcontrol_chip(kc);		\
-	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);	\
-	s16 chgain;							\
-									\
-	if (omap_st_get_chgain(mcbsp, channel, &chgain))		\
-		return -EAGAIN;						\
-									\
-	uc->value.integer.value[0] = chgain;				\
-	return 0;							\
-}
-
-OMAP_MCBSP_ST_CHANNEL_VOLUME(0)
-OMAP_MCBSP_ST_CHANNEL_VOLUME(1)
-
-static int omap_mcbsp_st_put_mode(struct snd_kcontrol *kcontrol,
-				struct snd_ctl_elem_value *ucontrol)
-{
-	struct snd_soc_dai *cpu_dai = snd_kcontrol_chip(kcontrol);
-	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);
-	u8 value = ucontrol->value.integer.value[0];
-
-	if (value == omap_st_is_enabled(mcbsp))
-		return 0;
-
-	if (value)
-		omap_st_enable(mcbsp);
-	else
-		omap_st_disable(mcbsp);
-
-	return 1;
-}
-
-static int omap_mcbsp_st_get_mode(struct snd_kcontrol *kcontrol,
-				struct snd_ctl_elem_value *ucontrol)
-{
-	struct snd_soc_dai *cpu_dai = snd_kcontrol_chip(kcontrol);
-	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);
-
-	ucontrol->value.integer.value[0] = omap_st_is_enabled(mcbsp);
-	return 0;
-}
-
-#define OMAP_MCBSP_ST_CONTROLS(port)					  \
-static const struct snd_kcontrol_new omap_mcbsp##port##_st_controls[] = { \
-SOC_SINGLE_EXT("McBSP" #port " Sidetone Switch", 1, 0, 1, 0,		  \
-	       omap_mcbsp_st_get_mode, omap_mcbsp_st_put_mode),		  \
-OMAP_MCBSP_SOC_SINGLE_S16_EXT("McBSP" #port " Sidetone Channel 0 Volume", \
-			      -32768, 32767,				  \
-			      omap_mcbsp_get_st_ch0_volume,		  \
-			      omap_mcbsp_set_st_ch0_volume),		  \
-OMAP_MCBSP_SOC_SINGLE_S16_EXT("McBSP" #port " Sidetone Channel 1 Volume", \
-			      -32768, 32767,				  \
-			      omap_mcbsp_get_st_ch1_volume,		  \
-			      omap_mcbsp_set_st_ch1_volume),		  \
-}
-
-OMAP_MCBSP_ST_CONTROLS(2);
-OMAP_MCBSP_ST_CONTROLS(3);
-
-int omap_mcbsp_st_add_controls(struct snd_soc_pcm_runtime *rtd, int port_id)
-{
-	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
-	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);
-
-	if (!mcbsp->st_data) {
-		dev_warn(mcbsp->dev, "No sidetone data for port\n");
-		return 0;
-	}
-
-	switch (port_id) {
-	case 2: /* McBSP 2 */
-		return snd_soc_add_dai_controls(cpu_dai,
-					omap_mcbsp2_st_controls,
-					ARRAY_SIZE(omap_mcbsp2_st_controls));
-	case 3: /* McBSP 3 */
-		return snd_soc_add_dai_controls(cpu_dai,
-					omap_mcbsp3_st_controls,
-					ARRAY_SIZE(omap_mcbsp3_st_controls));
-	default:
-		dev_err(mcbsp->dev, "Port %d not supported\n", port_id);
-		break;
-	}
-
-	return -EINVAL;
-}
-EXPORT_SYMBOL_GPL(omap_mcbsp_st_add_controls);
-
-static struct omap_mcbsp_platform_data omap2420_pdata = {
-	.reg_step = 4,
-	.reg_size = 2,
-};
-
-static struct omap_mcbsp_platform_data omap2430_pdata = {
-	.reg_step = 4,
-	.reg_size = 4,
-	.has_ccr = true,
-};
-
-static struct omap_mcbsp_platform_data omap3_pdata = {
-	.reg_step = 4,
-	.reg_size = 4,
-	.has_ccr = true,
-	.has_wakeup = true,
-};
-
-static struct omap_mcbsp_platform_data omap4_pdata = {
-	.reg_step = 4,
-	.reg_size = 4,
-	.has_ccr = true,
-	.has_wakeup = true,
-};
-
-static const struct of_device_id omap_mcbsp_of_match[] = {
-	{
-		.compatible = "ti,omap2420-mcbsp",
-		.data = &omap2420_pdata,
-	},
-	{
-		.compatible = "ti,omap2430-mcbsp",
-		.data = &omap2430_pdata,
-	},
-	{
-		.compatible = "ti,omap3-mcbsp",
-		.data = &omap3_pdata,
-	},
-	{
-		.compatible = "ti,omap4-mcbsp",
-		.data = &omap4_pdata,
-	},
-	{ },
-};
-MODULE_DEVICE_TABLE(of, omap_mcbsp_of_match);
-
-static int asoc_mcbsp_probe(struct platform_device *pdev)
-{
-	struct omap_mcbsp_platform_data *pdata = dev_get_platdata(&pdev->dev);
-	struct omap_mcbsp *mcbsp;
-	const struct of_device_id *match;
-	int ret;
-
-	match = of_match_device(omap_mcbsp_of_match, &pdev->dev);
-	if (match) {
-		struct device_node *node = pdev->dev.of_node;
-		struct omap_mcbsp_platform_data *pdata_quirk = pdata;
-		int buffer_size;
-
-		pdata = devm_kzalloc(&pdev->dev,
-				     sizeof(struct omap_mcbsp_platform_data),
-				     GFP_KERNEL);
-		if (!pdata)
-			return -ENOMEM;
-
-		memcpy(pdata, match->data, sizeof(*pdata));
-		if (!of_property_read_u32(node, "ti,buffer-size", &buffer_size))
-			pdata->buffer_size = buffer_size;
-		if (pdata_quirk)
-			pdata->force_ick_on = pdata_quirk->force_ick_on;
-	} else if (!pdata) {
-		dev_err(&pdev->dev, "missing platform data.\n");
-		return -EINVAL;
-	}
-	mcbsp = devm_kzalloc(&pdev->dev, sizeof(struct omap_mcbsp), GFP_KERNEL);
-	if (!mcbsp)
-		return -ENOMEM;
-
-	mcbsp->id = pdev->id;
-	mcbsp->pdata = pdata;
-	mcbsp->dev = &pdev->dev;
-	platform_set_drvdata(pdev, mcbsp);
-
-	ret = omap_mcbsp_init(pdev);
-	if (ret)
-		return ret;
-
-	ret = devm_snd_soc_register_component(&pdev->dev,
-					      &omap_mcbsp_component,
-					      &omap_mcbsp_dai, 1);
-	if (ret)
-		return ret;
-
-	return sdma_pcm_platform_register(&pdev->dev, NULL, NULL);
-}
-
-static int asoc_mcbsp_remove(struct platform_device *pdev)
-{
-	struct omap_mcbsp *mcbsp = platform_get_drvdata(pdev);
-
-	if (mcbsp->pdata->ops && mcbsp->pdata->ops->free)
-		mcbsp->pdata->ops->free(mcbsp->id);
-
-	if (pm_qos_request_active(&mcbsp->pm_qos_req))
-		pm_qos_remove_request(&mcbsp->pm_qos_req);
-
-	omap_mcbsp_cleanup(mcbsp);
-
-	clk_put(mcbsp->fclk);
-
-	return 0;
-}
-
-static struct platform_driver asoc_mcbsp_driver = {
-	.driver = {
-			.name = "omap-mcbsp",
-			.of_match_table = omap_mcbsp_of_match,
-	},
-
-	.probe = asoc_mcbsp_probe,
-	.remove = asoc_mcbsp_remove,
-};
-
-module_platform_driver(asoc_mcbsp_driver);
-
-MODULE_AUTHOR("Jarkko Nikula <jarkko.nikula@bitmer.com>");
-MODULE_DESCRIPTION("OMAP I2S SoC Interface");
-MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:omap-mcbsp");
diff -Naur --no-dereference a/sound/soc/omap/omap-mcbsp.h b/sound/soc/omap/omap-mcbsp.h
--- a/sound/soc/omap/omap-mcbsp.h	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/omap/omap-mcbsp.h	1969-12-31 19:00:00.000000000 -0500
@@ -1,44 +0,0 @@
-/*
- * omap-mcbsp.h
- *
- * Copyright (C) 2008 Nokia Corporation
- *
- * Contact: Jarkko Nikula <jarkko.nikula@bitmer.com>
- *          Peter Ujfalusi <peter.ujfalusi@ti.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
- * 02110-1301 USA
- *
- */
-
-#ifndef __OMAP_I2S_H__
-#define __OMAP_I2S_H__
-
-/* Source clocks for McBSP sample rate generator */
-enum omap_mcbsp_clksrg_clk {
-	OMAP_MCBSP_SYSCLK_CLKS_FCLK,	/* Internal FCLK */
-	OMAP_MCBSP_SYSCLK_CLKS_EXT,	/* External CLKS pin */
-	OMAP_MCBSP_SYSCLK_CLK,		/* Internal ICLK */
-	OMAP_MCBSP_SYSCLK_CLKX_EXT,	/* External CLKX pin */
-	OMAP_MCBSP_SYSCLK_CLKR_EXT,	/* External CLKR pin */
-};
-
-/* McBSP dividers */
-enum omap_mcbsp_div {
-	OMAP_MCBSP_CLKGDV,		/* Sample rate generator divider */
-};
-
-int omap_mcbsp_st_add_controls(struct snd_soc_pcm_runtime *rtd, int port_id);
-
-#endif
diff -Naur --no-dereference a/sound/soc/omap/omap-mcpdm.c b/sound/soc/omap/omap-mcpdm.c
--- a/sound/soc/omap/omap-mcpdm.c	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/omap/omap-mcpdm.c	1969-12-31 19:00:00.000000000 -0500
@@ -1,619 +0,0 @@
-/*
- * omap-mcpdm.c  --  OMAP ALSA SoC DAI driver using McPDM port
- *
- * Copyright (C) 2009 - 2011 Texas Instruments
- *
- * Author: Misael Lopez Cruz <misael.lopez@ti.com>
- * Contact: Jorge Eduardo Candelaria <x0107209@ti.com>
- *          Margarita Olaya <magi.olaya@ti.com>
- *          Peter Ujfalusi <peter.ujfalusi@ti.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
- * 02110-1301 USA
- *
- */
-
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/platform_device.h>
-#include <linux/interrupt.h>
-#include <linux/err.h>
-#include <linux/io.h>
-#include <linux/irq.h>
-#include <linux/slab.h>
-#include <linux/pm_runtime.h>
-#include <linux/of_device.h>
-
-#include <sound/core.h>
-#include <sound/pcm.h>
-#include <sound/pcm_params.h>
-#include <sound/soc.h>
-#include <sound/dmaengine_pcm.h>
-
-#include "omap-mcpdm.h"
-#include "sdma-pcm.h"
-
-struct mcpdm_link_config {
-	u32 link_mask; /* channel mask for the direction */
-	u32 threshold; /* FIFO threshold */
-};
-
-struct omap_mcpdm {
-	struct device *dev;
-	unsigned long phys_base;
-	void __iomem *io_base;
-	int irq;
-	struct pm_qos_request pm_qos_req;
-	int latency[2];
-
-	struct mutex mutex;
-
-	/* Playback/Capture configuration */
-	struct mcpdm_link_config config[2];
-
-	/* McPDM dn offsets for rx1, and 2 channels */
-	u32 dn_rx_offset;
-
-	/* McPDM needs to be restarted due to runtime reconfiguration */
-	bool restart;
-
-	/* pm state for suspend/resume handling */
-	int pm_active_count;
-
-	struct snd_dmaengine_dai_dma_data dma_data[2];
-};
-
-/*
- * Stream DMA parameters
- */
-
-static inline void omap_mcpdm_write(struct omap_mcpdm *mcpdm, u16 reg, u32 val)
-{
-	writel_relaxed(val, mcpdm->io_base + reg);
-}
-
-static inline int omap_mcpdm_read(struct omap_mcpdm *mcpdm, u16 reg)
-{
-	return readl_relaxed(mcpdm->io_base + reg);
-}
-
-#ifdef DEBUG
-static void omap_mcpdm_reg_dump(struct omap_mcpdm *mcpdm)
-{
-	dev_dbg(mcpdm->dev, "***********************\n");
-	dev_dbg(mcpdm->dev, "IRQSTATUS_RAW:  0x%04x\n",
-			omap_mcpdm_read(mcpdm, MCPDM_REG_IRQSTATUS_RAW));
-	dev_dbg(mcpdm->dev, "IRQSTATUS:  0x%04x\n",
-			omap_mcpdm_read(mcpdm, MCPDM_REG_IRQSTATUS));
-	dev_dbg(mcpdm->dev, "IRQENABLE_SET:  0x%04x\n",
-			omap_mcpdm_read(mcpdm, MCPDM_REG_IRQENABLE_SET));
-	dev_dbg(mcpdm->dev, "IRQENABLE_CLR:  0x%04x\n",
-			omap_mcpdm_read(mcpdm, MCPDM_REG_IRQENABLE_CLR));
-	dev_dbg(mcpdm->dev, "IRQWAKE_EN: 0x%04x\n",
-			omap_mcpdm_read(mcpdm, MCPDM_REG_IRQWAKE_EN));
-	dev_dbg(mcpdm->dev, "DMAENABLE_SET: 0x%04x\n",
-			omap_mcpdm_read(mcpdm, MCPDM_REG_DMAENABLE_SET));
-	dev_dbg(mcpdm->dev, "DMAENABLE_CLR:  0x%04x\n",
-			omap_mcpdm_read(mcpdm, MCPDM_REG_DMAENABLE_CLR));
-	dev_dbg(mcpdm->dev, "DMAWAKEEN:  0x%04x\n",
-			omap_mcpdm_read(mcpdm, MCPDM_REG_DMAWAKEEN));
-	dev_dbg(mcpdm->dev, "CTRL:  0x%04x\n",
-			omap_mcpdm_read(mcpdm, MCPDM_REG_CTRL));
-	dev_dbg(mcpdm->dev, "DN_DATA:  0x%04x\n",
-			omap_mcpdm_read(mcpdm, MCPDM_REG_DN_DATA));
-	dev_dbg(mcpdm->dev, "UP_DATA: 0x%04x\n",
-			omap_mcpdm_read(mcpdm, MCPDM_REG_UP_DATA));
-	dev_dbg(mcpdm->dev, "FIFO_CTRL_DN: 0x%04x\n",
-			omap_mcpdm_read(mcpdm, MCPDM_REG_FIFO_CTRL_DN));
-	dev_dbg(mcpdm->dev, "FIFO_CTRL_UP:  0x%04x\n",
-			omap_mcpdm_read(mcpdm, MCPDM_REG_FIFO_CTRL_UP));
-	dev_dbg(mcpdm->dev, "***********************\n");
-}
-#else
-static void omap_mcpdm_reg_dump(struct omap_mcpdm *mcpdm) {}
-#endif
-
-/*
- * Enables the transfer through the PDM interface to/from the Phoenix
- * codec by enabling the corresponding UP or DN channels.
- */
-static void omap_mcpdm_start(struct omap_mcpdm *mcpdm)
-{
-	u32 ctrl = omap_mcpdm_read(mcpdm, MCPDM_REG_CTRL);
-	u32 link_mask = mcpdm->config[0].link_mask | mcpdm->config[1].link_mask;
-
-	ctrl |= (MCPDM_SW_DN_RST | MCPDM_SW_UP_RST);
-	omap_mcpdm_write(mcpdm, MCPDM_REG_CTRL, ctrl);
-
-	ctrl |= link_mask;
-	omap_mcpdm_write(mcpdm, MCPDM_REG_CTRL, ctrl);
-
-	ctrl &= ~(MCPDM_SW_DN_RST | MCPDM_SW_UP_RST);
-	omap_mcpdm_write(mcpdm, MCPDM_REG_CTRL, ctrl);
-}
-
-/*
- * Disables the transfer through the PDM interface to/from the Phoenix
- * codec by disabling the corresponding UP or DN channels.
- */
-static void omap_mcpdm_stop(struct omap_mcpdm *mcpdm)
-{
-	u32 ctrl = omap_mcpdm_read(mcpdm, MCPDM_REG_CTRL);
-	u32 link_mask = MCPDM_PDM_DN_MASK | MCPDM_PDM_UP_MASK;
-
-	ctrl |= (MCPDM_SW_DN_RST | MCPDM_SW_UP_RST);
-	omap_mcpdm_write(mcpdm, MCPDM_REG_CTRL, ctrl);
-
-	ctrl &= ~(link_mask);
-	omap_mcpdm_write(mcpdm, MCPDM_REG_CTRL, ctrl);
-
-	ctrl &= ~(MCPDM_SW_DN_RST | MCPDM_SW_UP_RST);
-	omap_mcpdm_write(mcpdm, MCPDM_REG_CTRL, ctrl);
-
-}
-
-/*
- * Is the physical McPDM interface active.
- */
-static inline int omap_mcpdm_active(struct omap_mcpdm *mcpdm)
-{
-	return omap_mcpdm_read(mcpdm, MCPDM_REG_CTRL) &
-					(MCPDM_PDM_DN_MASK | MCPDM_PDM_UP_MASK);
-}
-
-/*
- * Configures McPDM uplink, and downlink for audio.
- * This function should be called before omap_mcpdm_start.
- */
-static void omap_mcpdm_open_streams(struct omap_mcpdm *mcpdm)
-{
-	u32 ctrl = omap_mcpdm_read(mcpdm, MCPDM_REG_CTRL);
-
-	omap_mcpdm_write(mcpdm, MCPDM_REG_CTRL, ctrl | MCPDM_WD_EN);
-
-	omap_mcpdm_write(mcpdm, MCPDM_REG_IRQENABLE_SET,
-			MCPDM_DN_IRQ_EMPTY | MCPDM_DN_IRQ_FULL |
-			MCPDM_UP_IRQ_EMPTY | MCPDM_UP_IRQ_FULL);
-
-	/* Enable DN RX1/2 offset cancellation feature, if configured */
-	if (mcpdm->dn_rx_offset) {
-		u32 dn_offset = mcpdm->dn_rx_offset;
-
-		omap_mcpdm_write(mcpdm, MCPDM_REG_DN_OFFSET, dn_offset);
-		dn_offset |= (MCPDM_DN_OFST_RX1_EN | MCPDM_DN_OFST_RX2_EN);
-		omap_mcpdm_write(mcpdm, MCPDM_REG_DN_OFFSET, dn_offset);
-	}
-
-	omap_mcpdm_write(mcpdm, MCPDM_REG_FIFO_CTRL_DN,
-			 mcpdm->config[SNDRV_PCM_STREAM_PLAYBACK].threshold);
-	omap_mcpdm_write(mcpdm, MCPDM_REG_FIFO_CTRL_UP,
-			 mcpdm->config[SNDRV_PCM_STREAM_CAPTURE].threshold);
-
-	omap_mcpdm_write(mcpdm, MCPDM_REG_DMAENABLE_SET,
-			MCPDM_DMA_DN_ENABLE | MCPDM_DMA_UP_ENABLE);
-}
-
-/*
- * Cleans McPDM uplink, and downlink configuration.
- * This function should be called when the stream is closed.
- */
-static void omap_mcpdm_close_streams(struct omap_mcpdm *mcpdm)
-{
-	/* Disable irq request generation for downlink */
-	omap_mcpdm_write(mcpdm, MCPDM_REG_IRQENABLE_CLR,
-			MCPDM_DN_IRQ_EMPTY | MCPDM_DN_IRQ_FULL);
-
-	/* Disable DMA request generation for downlink */
-	omap_mcpdm_write(mcpdm, MCPDM_REG_DMAENABLE_CLR, MCPDM_DMA_DN_ENABLE);
-
-	/* Disable irq request generation for uplink */
-	omap_mcpdm_write(mcpdm, MCPDM_REG_IRQENABLE_CLR,
-			MCPDM_UP_IRQ_EMPTY | MCPDM_UP_IRQ_FULL);
-
-	/* Disable DMA request generation for uplink */
-	omap_mcpdm_write(mcpdm, MCPDM_REG_DMAENABLE_CLR, MCPDM_DMA_UP_ENABLE);
-
-	/* Disable RX1/2 offset cancellation */
-	if (mcpdm->dn_rx_offset)
-		omap_mcpdm_write(mcpdm, MCPDM_REG_DN_OFFSET, 0);
-}
-
-static irqreturn_t omap_mcpdm_irq_handler(int irq, void *dev_id)
-{
-	struct omap_mcpdm *mcpdm = dev_id;
-	int irq_status;
-
-	irq_status = omap_mcpdm_read(mcpdm, MCPDM_REG_IRQSTATUS);
-
-	/* Acknowledge irq event */
-	omap_mcpdm_write(mcpdm, MCPDM_REG_IRQSTATUS, irq_status);
-
-	if (irq_status & MCPDM_DN_IRQ_FULL)
-		dev_dbg(mcpdm->dev, "DN (playback) FIFO Full\n");
-
-	if (irq_status & MCPDM_DN_IRQ_EMPTY)
-		dev_dbg(mcpdm->dev, "DN (playback) FIFO Empty\n");
-
-	if (irq_status & MCPDM_DN_IRQ)
-		dev_dbg(mcpdm->dev, "DN (playback) write request\n");
-
-	if (irq_status & MCPDM_UP_IRQ_FULL)
-		dev_dbg(mcpdm->dev, "UP (capture) FIFO Full\n");
-
-	if (irq_status & MCPDM_UP_IRQ_EMPTY)
-		dev_dbg(mcpdm->dev, "UP (capture) FIFO Empty\n");
-
-	if (irq_status & MCPDM_UP_IRQ)
-		dev_dbg(mcpdm->dev, "UP (capture) write request\n");
-
-	return IRQ_HANDLED;
-}
-
-static int omap_mcpdm_dai_startup(struct snd_pcm_substream *substream,
-				  struct snd_soc_dai *dai)
-{
-	struct omap_mcpdm *mcpdm = snd_soc_dai_get_drvdata(dai);
-
-	mutex_lock(&mcpdm->mutex);
-
-	if (!dai->active)
-		omap_mcpdm_open_streams(mcpdm);
-
-	mutex_unlock(&mcpdm->mutex);
-
-	return 0;
-}
-
-static void omap_mcpdm_dai_shutdown(struct snd_pcm_substream *substream,
-				  struct snd_soc_dai *dai)
-{
-	struct omap_mcpdm *mcpdm = snd_soc_dai_get_drvdata(dai);
-	int tx = (substream->stream == SNDRV_PCM_STREAM_PLAYBACK);
-	int stream1 = tx ? SNDRV_PCM_STREAM_PLAYBACK : SNDRV_PCM_STREAM_CAPTURE;
-	int stream2 = tx ? SNDRV_PCM_STREAM_CAPTURE : SNDRV_PCM_STREAM_PLAYBACK;
-
-	mutex_lock(&mcpdm->mutex);
-
-	if (!dai->active) {
-		if (omap_mcpdm_active(mcpdm)) {
-			omap_mcpdm_stop(mcpdm);
-			omap_mcpdm_close_streams(mcpdm);
-			mcpdm->config[0].link_mask = 0;
-			mcpdm->config[1].link_mask = 0;
-		}
-	}
-
-	if (mcpdm->latency[stream2])
-		pm_qos_update_request(&mcpdm->pm_qos_req,
-				      mcpdm->latency[stream2]);
-	else if (mcpdm->latency[stream1])
-		pm_qos_remove_request(&mcpdm->pm_qos_req);
-
-	mcpdm->latency[stream1] = 0;
-
-	mutex_unlock(&mcpdm->mutex);
-}
-
-static int omap_mcpdm_dai_hw_params(struct snd_pcm_substream *substream,
-				    struct snd_pcm_hw_params *params,
-				    struct snd_soc_dai *dai)
-{
-	struct omap_mcpdm *mcpdm = snd_soc_dai_get_drvdata(dai);
-	int stream = substream->stream;
-	struct snd_dmaengine_dai_dma_data *dma_data;
-	u32 threshold;
-	int channels, latency;
-	int link_mask = 0;
-
-	channels = params_channels(params);
-	switch (channels) {
-	case 5:
-		if (stream == SNDRV_PCM_STREAM_CAPTURE)
-			/* up to 3 channels for capture */
-			return -EINVAL;
-		link_mask |= 1 << 4;
-		/* fall through */
-	case 4:
-		if (stream == SNDRV_PCM_STREAM_CAPTURE)
-			/* up to 3 channels for capture */
-			return -EINVAL;
-		link_mask |= 1 << 3;
-		/* fall through */
-	case 3:
-		link_mask |= 1 << 2;
-		/* fall through */
-	case 2:
-		link_mask |= 1 << 1;
-		/* fall through */
-	case 1:
-		link_mask |= 1 << 0;
-		break;
-	default:
-		/* unsupported number of channels */
-		return -EINVAL;
-	}
-
-	dma_data = snd_soc_dai_get_dma_data(dai, substream);
-
-	threshold = mcpdm->config[stream].threshold;
-	/* Configure McPDM channels, and DMA packet size */
-	if (stream == SNDRV_PCM_STREAM_PLAYBACK) {
-		link_mask <<= 3;
-
-		/* If capture is not running assume a stereo stream to come */
-		if (!mcpdm->config[!stream].link_mask)
-			mcpdm->config[!stream].link_mask = 0x3;
-
-		dma_data->maxburst =
-				(MCPDM_DN_THRES_MAX - threshold) * channels;
-		latency = threshold;
-	} else {
-		/* If playback is not running assume a stereo stream to come */
-		if (!mcpdm->config[!stream].link_mask)
-			mcpdm->config[!stream].link_mask = (0x3 << 3);
-
-		dma_data->maxburst = threshold * channels;
-		latency = (MCPDM_DN_THRES_MAX - threshold);
-	}
-
-	/*
-	 * The DMA must act to a DMA request within latency time (usec) to avoid
-	 * under/overflow
-	 */
-	mcpdm->latency[stream] = latency * USEC_PER_SEC / params_rate(params);
-
-	if (!mcpdm->latency[stream])
-		mcpdm->latency[stream] = 10;
-
-	/* Check if we need to restart McPDM with this stream */
-	if (mcpdm->config[stream].link_mask &&
-	    mcpdm->config[stream].link_mask != link_mask)
-		mcpdm->restart = true;
-
-	mcpdm->config[stream].link_mask = link_mask;
-
-	return 0;
-}
-
-static int omap_mcpdm_prepare(struct snd_pcm_substream *substream,
-				  struct snd_soc_dai *dai)
-{
-	struct omap_mcpdm *mcpdm = snd_soc_dai_get_drvdata(dai);
-	struct pm_qos_request *pm_qos_req = &mcpdm->pm_qos_req;
-	int tx = (substream->stream == SNDRV_PCM_STREAM_PLAYBACK);
-	int stream1 = tx ? SNDRV_PCM_STREAM_PLAYBACK : SNDRV_PCM_STREAM_CAPTURE;
-	int stream2 = tx ? SNDRV_PCM_STREAM_CAPTURE : SNDRV_PCM_STREAM_PLAYBACK;
-	int latency = mcpdm->latency[stream2];
-
-	/* Prevent omap hardware from hitting off between FIFO fills */
-	if (!latency || mcpdm->latency[stream1] < latency)
-		latency = mcpdm->latency[stream1];
-
-	if (pm_qos_request_active(pm_qos_req))
-		pm_qos_update_request(pm_qos_req, latency);
-	else if (latency)
-		pm_qos_add_request(pm_qos_req, PM_QOS_CPU_DMA_LATENCY, latency);
-
-	if (!omap_mcpdm_active(mcpdm)) {
-		omap_mcpdm_start(mcpdm);
-		omap_mcpdm_reg_dump(mcpdm);
-	} else if (mcpdm->restart) {
-		omap_mcpdm_stop(mcpdm);
-		omap_mcpdm_start(mcpdm);
-		mcpdm->restart = false;
-		omap_mcpdm_reg_dump(mcpdm);
-	}
-
-	return 0;
-}
-
-static const struct snd_soc_dai_ops omap_mcpdm_dai_ops = {
-	.startup	= omap_mcpdm_dai_startup,
-	.shutdown	= omap_mcpdm_dai_shutdown,
-	.hw_params	= omap_mcpdm_dai_hw_params,
-	.prepare	= omap_mcpdm_prepare,
-};
-
-static int omap_mcpdm_probe(struct snd_soc_dai *dai)
-{
-	struct omap_mcpdm *mcpdm = snd_soc_dai_get_drvdata(dai);
-	int ret;
-
-	pm_runtime_enable(mcpdm->dev);
-
-	/* Disable lines while request is ongoing */
-	pm_runtime_get_sync(mcpdm->dev);
-	omap_mcpdm_write(mcpdm, MCPDM_REG_CTRL, 0x00);
-
-	ret = request_irq(mcpdm->irq, omap_mcpdm_irq_handler, 0, "McPDM",
-			  (void *)mcpdm);
-
-	pm_runtime_put_sync(mcpdm->dev);
-
-	if (ret) {
-		dev_err(mcpdm->dev, "Request for IRQ failed\n");
-		pm_runtime_disable(mcpdm->dev);
-	}
-
-	/* Configure McPDM threshold values */
-	mcpdm->config[SNDRV_PCM_STREAM_PLAYBACK].threshold = 2;
-	mcpdm->config[SNDRV_PCM_STREAM_CAPTURE].threshold =
-							MCPDM_UP_THRES_MAX - 3;
-
-	snd_soc_dai_init_dma_data(dai,
-				  &mcpdm->dma_data[SNDRV_PCM_STREAM_PLAYBACK],
-				  &mcpdm->dma_data[SNDRV_PCM_STREAM_CAPTURE]);
-
-	return ret;
-}
-
-static int omap_mcpdm_remove(struct snd_soc_dai *dai)
-{
-	struct omap_mcpdm *mcpdm = snd_soc_dai_get_drvdata(dai);
-
-	free_irq(mcpdm->irq, (void *)mcpdm);
-	pm_runtime_disable(mcpdm->dev);
-
-	if (pm_qos_request_active(&mcpdm->pm_qos_req))
-		pm_qos_remove_request(&mcpdm->pm_qos_req);
-
-	return 0;
-}
-
-#ifdef CONFIG_PM_SLEEP
-static int omap_mcpdm_suspend(struct snd_soc_dai *dai)
-{
-	struct omap_mcpdm *mcpdm = snd_soc_dai_get_drvdata(dai);
-
-	if (dai->active) {
-		omap_mcpdm_stop(mcpdm);
-		omap_mcpdm_close_streams(mcpdm);
-	}
-
-	mcpdm->pm_active_count = 0;
-	while (pm_runtime_active(mcpdm->dev)) {
-		pm_runtime_put_sync(mcpdm->dev);
-		mcpdm->pm_active_count++;
-	}
-
-	return 0;
-}
-
-static int omap_mcpdm_resume(struct snd_soc_dai *dai)
-{
-	struct omap_mcpdm *mcpdm = snd_soc_dai_get_drvdata(dai);
-
-	if (mcpdm->pm_active_count) {
-		while (mcpdm->pm_active_count--)
-			pm_runtime_get_sync(mcpdm->dev);
-
-		if (dai->active) {
-			omap_mcpdm_open_streams(mcpdm);
-			omap_mcpdm_start(mcpdm);
-		}
-	}
-
-
-	return 0;
-}
-#else
-#define omap_mcpdm_suspend NULL
-#define omap_mcpdm_resume NULL
-#endif
-
-#define OMAP_MCPDM_RATES	(SNDRV_PCM_RATE_88200 | SNDRV_PCM_RATE_96000)
-#define OMAP_MCPDM_FORMATS	SNDRV_PCM_FMTBIT_S32_LE
-
-static struct snd_soc_dai_driver omap_mcpdm_dai = {
-	.probe = omap_mcpdm_probe,
-	.remove = omap_mcpdm_remove,
-	.suspend = omap_mcpdm_suspend,
-	.resume = omap_mcpdm_resume,
-	.probe_order = SND_SOC_COMP_ORDER_LATE,
-	.remove_order = SND_SOC_COMP_ORDER_EARLY,
-	.playback = {
-		.channels_min = 1,
-		.channels_max = 5,
-		.rates = OMAP_MCPDM_RATES,
-		.formats = OMAP_MCPDM_FORMATS,
-		.sig_bits = 24,
-	},
-	.capture = {
-		.channels_min = 1,
-		.channels_max = 3,
-		.rates = OMAP_MCPDM_RATES,
-		.formats = OMAP_MCPDM_FORMATS,
-		.sig_bits = 24,
-	},
-	.ops = &omap_mcpdm_dai_ops,
-};
-
-static const struct snd_soc_component_driver omap_mcpdm_component = {
-	.name		= "omap-mcpdm",
-};
-
-void omap_mcpdm_configure_dn_offsets(struct snd_soc_pcm_runtime *rtd,
-				    u8 rx1, u8 rx2)
-{
-	struct omap_mcpdm *mcpdm = snd_soc_dai_get_drvdata(rtd->cpu_dai);
-
-	mcpdm->dn_rx_offset = MCPDM_DNOFST_RX1(rx1) | MCPDM_DNOFST_RX2(rx2);
-}
-EXPORT_SYMBOL_GPL(omap_mcpdm_configure_dn_offsets);
-
-static int asoc_mcpdm_probe(struct platform_device *pdev)
-{
-	struct omap_mcpdm *mcpdm;
-	struct resource *res;
-	int ret;
-
-	mcpdm = devm_kzalloc(&pdev->dev, sizeof(struct omap_mcpdm), GFP_KERNEL);
-	if (!mcpdm)
-		return -ENOMEM;
-
-	platform_set_drvdata(pdev, mcpdm);
-
-	mutex_init(&mcpdm->mutex);
-
-	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dma");
-	if (res == NULL)
-		return -ENOMEM;
-
-	mcpdm->dma_data[0].addr = res->start + MCPDM_REG_DN_DATA;
-	mcpdm->dma_data[1].addr = res->start + MCPDM_REG_UP_DATA;
-
-	mcpdm->dma_data[0].filter_data = "dn_link";
-	mcpdm->dma_data[1].filter_data = "up_link";
-
-	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mpu");
-	mcpdm->io_base = devm_ioremap_resource(&pdev->dev, res);
-	if (IS_ERR(mcpdm->io_base))
-		return PTR_ERR(mcpdm->io_base);
-
-	mcpdm->irq = platform_get_irq(pdev, 0);
-	if (mcpdm->irq < 0)
-		return mcpdm->irq;
-
-	mcpdm->dev = &pdev->dev;
-
-	ret =  devm_snd_soc_register_component(&pdev->dev,
-					       &omap_mcpdm_component,
-					       &omap_mcpdm_dai, 1);
-	if (ret)
-		return ret;
-
-	return sdma_pcm_platform_register(&pdev->dev, "dn_link", "up_link");
-}
-
-static const struct of_device_id omap_mcpdm_of_match[] = {
-	{ .compatible = "ti,omap4-mcpdm", },
-	{ }
-};
-MODULE_DEVICE_TABLE(of, omap_mcpdm_of_match);
-
-static struct platform_driver asoc_mcpdm_driver = {
-	.driver = {
-		.name	= "omap-mcpdm",
-		.of_match_table = omap_mcpdm_of_match,
-	},
-
-	.probe	= asoc_mcpdm_probe,
-};
-
-module_platform_driver(asoc_mcpdm_driver);
-
-MODULE_ALIAS("platform:omap-mcpdm");
-MODULE_AUTHOR("Misael Lopez Cruz <misael.lopez@ti.com>");
-MODULE_DESCRIPTION("OMAP PDM SoC Interface");
-MODULE_LICENSE("GPL");
diff -Naur --no-dereference a/sound/soc/omap/omap-mcpdm.h b/sound/soc/omap/omap-mcpdm.h
--- a/sound/soc/omap/omap-mcpdm.h	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/omap/omap-mcpdm.h	1969-12-31 19:00:00.000000000 -0500
@@ -1,107 +0,0 @@
-/*
- * omap-mcpdm.h
- *
- * Copyright (C) 2009 - 2011 Texas Instruments
- *
- * Contact: Misael Lopez Cruz <misael.lopez@ti.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
- * 02110-1301 USA
- *
- */
-
-#ifndef __OMAP_MCPDM_H__
-#define __OMAP_MCPDM_H__
-
-#define MCPDM_REG_REVISION		0x00
-#define MCPDM_REG_SYSCONFIG		0x10
-#define MCPDM_REG_IRQSTATUS_RAW		0x24
-#define MCPDM_REG_IRQSTATUS		0x28
-#define MCPDM_REG_IRQENABLE_SET		0x2C
-#define MCPDM_REG_IRQENABLE_CLR		0x30
-#define MCPDM_REG_IRQWAKE_EN		0x34
-#define MCPDM_REG_DMAENABLE_SET		0x38
-#define MCPDM_REG_DMAENABLE_CLR		0x3C
-#define MCPDM_REG_DMAWAKEEN		0x40
-#define MCPDM_REG_CTRL			0x44
-#define MCPDM_REG_DN_DATA		0x48
-#define MCPDM_REG_UP_DATA		0x4C
-#define MCPDM_REG_FIFO_CTRL_DN		0x50
-#define MCPDM_REG_FIFO_CTRL_UP		0x54
-#define MCPDM_REG_DN_OFFSET		0x58
-
-/*
- * MCPDM_IRQ bit fields
- * IRQSTATUS_RAW, IRQSTATUS, IRQENABLE_SET, IRQENABLE_CLR
- */
-
-#define MCPDM_DN_IRQ			(1 << 0)
-#define MCPDM_DN_IRQ_EMPTY		(1 << 1)
-#define MCPDM_DN_IRQ_ALMST_EMPTY	(1 << 2)
-#define MCPDM_DN_IRQ_FULL		(1 << 3)
-
-#define MCPDM_UP_IRQ			(1 << 8)
-#define MCPDM_UP_IRQ_EMPTY		(1 << 9)
-#define MCPDM_UP_IRQ_ALMST_FULL		(1 << 10)
-#define MCPDM_UP_IRQ_FULL		(1 << 11)
-
-#define MCPDM_DOWNLINK_IRQ_MASK		0x00F
-#define MCPDM_UPLINK_IRQ_MASK		0xF00
-
-/*
- * MCPDM_DMAENABLE bit fields
- */
-
-#define MCPDM_DMA_DN_ENABLE		(1 << 0)
-#define MCPDM_DMA_UP_ENABLE		(1 << 1)
-
-/*
- * MCPDM_CTRL bit fields
- */
-
-#define MCPDM_PDM_UPLINK_EN(x)		(1 << (x - 1)) /* ch1 is at bit 0 */
-#define MCPDM_PDM_DOWNLINK_EN(x)	(1 << (x + 2)) /* ch1 is at bit 3 */
-#define MCPDM_PDMOUTFORMAT		(1 << 8)
-#define MCPDM_CMD_INT			(1 << 9)
-#define MCPDM_STATUS_INT		(1 << 10)
-#define MCPDM_SW_UP_RST			(1 << 11)
-#define MCPDM_SW_DN_RST			(1 << 12)
-#define MCPDM_WD_EN			(1 << 14)
-#define MCPDM_PDM_UP_MASK		0x7
-#define MCPDM_PDM_DN_MASK		(0x1f << 3)
-
-
-#define MCPDM_PDMOUTFORMAT_LJUST	(0 << 8)
-#define MCPDM_PDMOUTFORMAT_RJUST	(1 << 8)
-
-/*
- * MCPDM_FIFO_CTRL bit fields
- */
-
-#define MCPDM_UP_THRES_MAX		0xF
-#define MCPDM_DN_THRES_MAX		0xF
-
-/*
- * MCPDM_DN_OFFSET bit fields
- */
-
-#define MCPDM_DN_OFST_RX1_EN		(1 << 0)
-#define MCPDM_DNOFST_RX1(x)		((x & 0x1f) << 1)
-#define MCPDM_DN_OFST_RX2_EN		(1 << 8)
-#define MCPDM_DNOFST_RX2(x)		((x & 0x1f) << 9)
-
-void omap_mcpdm_configure_dn_offsets(struct snd_soc_pcm_runtime *rtd,
-				    u8 rx1, u8 rx2);
-
-#endif	/* End of __OMAP_MCPDM_H__ */
diff -Naur --no-dereference a/sound/soc/omap/omap-twl4030.c b/sound/soc/omap/omap-twl4030.c
--- a/sound/soc/omap/omap-twl4030.c	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/omap/omap-twl4030.c	1969-12-31 19:00:00.000000000 -0500
@@ -1,353 +0,0 @@
-/*
- * omap-twl4030.c  --  SoC audio for TI SoC based boards with twl4030 codec
- *
- * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com
- * All rights reserved.
- *
- * Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
- *
- * This driver replaces the following machine drivers:
- * omap3beagle (Author: Steve Sakoman <steve@sakoman.com>)
- * omap3evm (Author: Anuj Aggarwal <anuj.aggarwal@ti.com>)
- * overo (Author: Steve Sakoman <steve@sakoman.com>)
- * igep0020 (Author: Enric Balletbo i Serra <eballetbo@iseebcn.com>)
- * zoom2 (Author: Misael Lopez Cruz <misael.lopez@ti.com>)
- * sdp3430 (Author: Misael Lopez Cruz <misael.lopez@ti.com>)
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
- * 02110-1301 USA
- *
- */
-
-#include <linux/platform_device.h>
-#include <linux/platform_data/omap-twl4030.h>
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/gpio.h>
-#include <linux/of_gpio.h>
-
-#include <sound/core.h>
-#include <sound/pcm.h>
-#include <sound/soc.h>
-#include <sound/jack.h>
-
-#include "omap-mcbsp.h"
-
-struct omap_twl4030 {
-	int jack_detect;	/* board can detect jack events */
-	struct snd_soc_jack hs_jack;
-};
-
-static int omap_twl4030_hw_params(struct snd_pcm_substream *substream,
-	struct snd_pcm_hw_params *params)
-{
-	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	unsigned int fmt;
-
-	switch (params_channels(params)) {
-	case 2: /* Stereo I2S mode */
-		fmt =	SND_SOC_DAIFMT_I2S |
-			SND_SOC_DAIFMT_NB_NF |
-			SND_SOC_DAIFMT_CBM_CFM;
-		break;
-	case 4: /* Four channel TDM mode */
-		fmt =	SND_SOC_DAIFMT_DSP_A |
-			SND_SOC_DAIFMT_IB_NF |
-			SND_SOC_DAIFMT_CBM_CFM;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	return snd_soc_runtime_set_dai_fmt(rtd, fmt);
-}
-
-static const struct snd_soc_ops omap_twl4030_ops = {
-	.hw_params = omap_twl4030_hw_params,
-};
-
-static const struct snd_soc_dapm_widget dapm_widgets[] = {
-	SND_SOC_DAPM_SPK("Earpiece Spk", NULL),
-	SND_SOC_DAPM_SPK("Handsfree Spk", NULL),
-	SND_SOC_DAPM_HP("Headset Stereophone", NULL),
-	SND_SOC_DAPM_SPK("Ext Spk", NULL),
-	SND_SOC_DAPM_SPK("Carkit Spk", NULL),
-
-	SND_SOC_DAPM_MIC("Main Mic", NULL),
-	SND_SOC_DAPM_MIC("Sub Mic", NULL),
-	SND_SOC_DAPM_MIC("Headset Mic", NULL),
-	SND_SOC_DAPM_MIC("Carkit Mic", NULL),
-	SND_SOC_DAPM_MIC("Digital0 Mic", NULL),
-	SND_SOC_DAPM_MIC("Digital1 Mic", NULL),
-	SND_SOC_DAPM_LINE("Line In", NULL),
-};
-
-static const struct snd_soc_dapm_route audio_map[] = {
-	/* Headset Stereophone:  HSOL, HSOR */
-	{"Headset Stereophone", NULL, "HSOL"},
-	{"Headset Stereophone", NULL, "HSOR"},
-	/* External Speakers: HFL, HFR */
-	{"Handsfree Spk", NULL, "HFL"},
-	{"Handsfree Spk", NULL, "HFR"},
-	/* External Speakers: PredrivL, PredrivR */
-	{"Ext Spk", NULL, "PREDRIVEL"},
-	{"Ext Spk", NULL, "PREDRIVER"},
-	/* Carkit speakers:  CARKITL, CARKITR */
-	{"Carkit Spk", NULL, "CARKITL"},
-	{"Carkit Spk", NULL, "CARKITR"},
-	/* Earpiece */
-	{"Earpiece Spk", NULL, "EARPIECE"},
-
-	/* External Mics: MAINMIC, SUBMIC with bias */
-	{"MAINMIC", NULL, "Main Mic"},
-	{"Main Mic", NULL, "Mic Bias 1"},
-	{"SUBMIC", NULL, "Sub Mic"},
-	{"Sub Mic", NULL, "Mic Bias 2"},
-	/* Headset Mic: HSMIC with bias */
-	{"HSMIC", NULL, "Headset Mic"},
-	{"Headset Mic", NULL, "Headset Mic Bias"},
-	/* Digital Mics: DIGIMIC0, DIGIMIC1 with bias */
-	{"DIGIMIC0", NULL, "Digital0 Mic"},
-	{"Digital0 Mic", NULL, "Mic Bias 1"},
-	{"DIGIMIC1", NULL, "Digital1 Mic"},
-	{"Digital1 Mic", NULL, "Mic Bias 2"},
-	/* Carkit In: CARKITMIC */
-	{"CARKITMIC", NULL, "Carkit Mic"},
-	/* Aux In: AUXL, AUXR */
-	{"AUXL", NULL, "Line In"},
-	{"AUXR", NULL, "Line In"},
-};
-
-/* Headset jack detection DAPM pins */
-static struct snd_soc_jack_pin hs_jack_pins[] = {
-	{
-		.pin = "Headset Mic",
-		.mask = SND_JACK_MICROPHONE,
-	},
-	{
-		.pin = "Headset Stereophone",
-		.mask = SND_JACK_HEADPHONE,
-	},
-};
-
-/* Headset jack detection gpios */
-static struct snd_soc_jack_gpio hs_jack_gpios[] = {
-	{
-		.name = "hsdet-gpio",
-		.report = SND_JACK_HEADSET,
-		.debounce_time = 200,
-	},
-};
-
-static inline void twl4030_disconnect_pin(struct snd_soc_dapm_context *dapm,
-					  int connected, char *pin)
-{
-	if (!connected)
-		snd_soc_dapm_disable_pin(dapm, pin);
-}
-
-static int omap_twl4030_init(struct snd_soc_pcm_runtime *rtd)
-{
-	struct snd_soc_card *card = rtd->card;
-	struct snd_soc_dapm_context *dapm = &card->dapm;
-	struct omap_tw4030_pdata *pdata = dev_get_platdata(card->dev);
-	struct omap_twl4030 *priv = snd_soc_card_get_drvdata(card);
-	int ret = 0;
-
-	/* Headset jack detection only if it is supported */
-	if (priv->jack_detect > 0) {
-		hs_jack_gpios[0].gpio = priv->jack_detect;
-
-		ret = snd_soc_card_jack_new(rtd->card, "Headset Jack",
-					    SND_JACK_HEADSET, &priv->hs_jack,
-					    hs_jack_pins,
-					    ARRAY_SIZE(hs_jack_pins));
-		if (ret)
-			return ret;
-
-		ret = snd_soc_jack_add_gpios(&priv->hs_jack,
-					     ARRAY_SIZE(hs_jack_gpios),
-					     hs_jack_gpios);
-		if (ret)
-			return ret;
-	}
-
-	/*
-	 * NULL pdata means we booted with DT. In this case the routing is
-	 * provided and the card is fully routed, no need to mark pins.
-	 */
-	if (!pdata || !pdata->custom_routing)
-		return ret;
-
-	/* Disable not connected paths if not used */
-	twl4030_disconnect_pin(dapm, pdata->has_ear, "Earpiece Spk");
-	twl4030_disconnect_pin(dapm, pdata->has_hf, "Handsfree Spk");
-	twl4030_disconnect_pin(dapm, pdata->has_hs, "Headset Stereophone");
-	twl4030_disconnect_pin(dapm, pdata->has_predriv, "Ext Spk");
-	twl4030_disconnect_pin(dapm, pdata->has_carkit, "Carkit Spk");
-
-	twl4030_disconnect_pin(dapm, pdata->has_mainmic, "Main Mic");
-	twl4030_disconnect_pin(dapm, pdata->has_submic, "Sub Mic");
-	twl4030_disconnect_pin(dapm, pdata->has_hsmic, "Headset Mic");
-	twl4030_disconnect_pin(dapm, pdata->has_carkitmic, "Carkit Mic");
-	twl4030_disconnect_pin(dapm, pdata->has_digimic0, "Digital0 Mic");
-	twl4030_disconnect_pin(dapm, pdata->has_digimic1, "Digital1 Mic");
-	twl4030_disconnect_pin(dapm, pdata->has_linein, "Line In");
-
-	return ret;
-}
-
-/* Digital audio interface glue - connects codec <--> CPU */
-static struct snd_soc_dai_link omap_twl4030_dai_links[] = {
-	{
-		.name = "TWL4030 HiFi",
-		.stream_name = "TWL4030 HiFi",
-		.cpu_dai_name = "omap-mcbsp.2",
-		.codec_dai_name = "twl4030-hifi",
-		.platform_name = "omap-mcbsp.2",
-		.codec_name = "twl4030-codec",
-		.init = omap_twl4030_init,
-		.ops = &omap_twl4030_ops,
-	},
-	{
-		.name = "TWL4030 Voice",
-		.stream_name = "TWL4030 Voice",
-		.cpu_dai_name = "omap-mcbsp.3",
-		.codec_dai_name = "twl4030-voice",
-		.platform_name = "omap-mcbsp.3",
-		.codec_name = "twl4030-codec",
-		.dai_fmt = SND_SOC_DAIFMT_DSP_A | SND_SOC_DAIFMT_IB_NF |
-			   SND_SOC_DAIFMT_CBM_CFM,
-	},
-};
-
-/* Audio machine driver */
-static struct snd_soc_card omap_twl4030_card = {
-	.owner = THIS_MODULE,
-	.dai_link = omap_twl4030_dai_links,
-	.num_links = ARRAY_SIZE(omap_twl4030_dai_links),
-
-	.dapm_widgets = dapm_widgets,
-	.num_dapm_widgets = ARRAY_SIZE(dapm_widgets),
-	.dapm_routes = audio_map,
-	.num_dapm_routes = ARRAY_SIZE(audio_map),
-};
-
-static int omap_twl4030_probe(struct platform_device *pdev)
-{
-	struct omap_tw4030_pdata *pdata = dev_get_platdata(&pdev->dev);
-	struct device_node *node = pdev->dev.of_node;
-	struct snd_soc_card *card = &omap_twl4030_card;
-	struct omap_twl4030 *priv;
-	int ret = 0;
-
-	card->dev = &pdev->dev;
-
-	priv = devm_kzalloc(&pdev->dev, sizeof(struct omap_twl4030), GFP_KERNEL);
-	if (priv == NULL)
-		return -ENOMEM;
-
-	if (node) {
-		struct device_node *dai_node;
-		struct property *prop;
-
-		if (snd_soc_of_parse_card_name(card, "ti,model")) {
-			dev_err(&pdev->dev, "Card name is not provided\n");
-			return -ENODEV;
-		}
-
-		dai_node = of_parse_phandle(node, "ti,mcbsp", 0);
-		if (!dai_node) {
-			dev_err(&pdev->dev, "McBSP node is not provided\n");
-			return -EINVAL;
-		}
-		omap_twl4030_dai_links[0].cpu_dai_name  = NULL;
-		omap_twl4030_dai_links[0].cpu_of_node = dai_node;
-
-		omap_twl4030_dai_links[0].platform_name  = NULL;
-		omap_twl4030_dai_links[0].platform_of_node = dai_node;
-
-		dai_node = of_parse_phandle(node, "ti,mcbsp-voice", 0);
-		if (!dai_node) {
-			card->num_links = 1;
-		} else {
-			omap_twl4030_dai_links[1].cpu_dai_name  = NULL;
-			omap_twl4030_dai_links[1].cpu_of_node = dai_node;
-
-			omap_twl4030_dai_links[1].platform_name  = NULL;
-			omap_twl4030_dai_links[1].platform_of_node = dai_node;
-		}
-
-		priv->jack_detect = of_get_named_gpio(node,
-						      "ti,jack-det-gpio", 0);
-
-		/* Optional: audio routing can be provided */
-		prop = of_find_property(node, "ti,audio-routing", NULL);
-		if (prop) {
-			ret = snd_soc_of_parse_audio_routing(card,
-							    "ti,audio-routing");
-			if (ret)
-				return ret;
-
-			card->fully_routed = 1;
-		}
-	} else if (pdata) {
-		if (pdata->card_name) {
-			card->name = pdata->card_name;
-		} else {
-			dev_err(&pdev->dev, "Card name is not provided\n");
-			return -ENODEV;
-		}
-
-		if (!pdata->voice_connected)
-			card->num_links = 1;
-
-		priv->jack_detect = pdata->jack_detect;
-	} else {
-		dev_err(&pdev->dev, "Missing pdata\n");
-		return -ENODEV;
-	}
-
-	snd_soc_card_set_drvdata(card, priv);
-	ret = devm_snd_soc_register_card(&pdev->dev, card);
-	if (ret) {
-		dev_err(&pdev->dev, "devm_snd_soc_register_card() failed: %d\n",
-			ret);
-		return ret;
-	}
-
-	return 0;
-}
-
-static const struct of_device_id omap_twl4030_of_match[] = {
-	{.compatible = "ti,omap-twl4030", },
-	{ },
-};
-MODULE_DEVICE_TABLE(of, omap_twl4030_of_match);
-
-static struct platform_driver omap_twl4030_driver = {
-	.driver = {
-		.name = "omap-twl4030",
-		.pm = &snd_soc_pm_ops,
-		.of_match_table = omap_twl4030_of_match,
-	},
-	.probe = omap_twl4030_probe,
-};
-
-module_platform_driver(omap_twl4030_driver);
-
-MODULE_AUTHOR("Peter Ujfalusi <peter.ujfalusi@ti.com>");
-MODULE_DESCRIPTION("ALSA SoC for TI SoC based boards with twl4030 codec");
-MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:omap-twl4030");
diff -Naur --no-dereference a/sound/soc/omap/osk5912.c b/sound/soc/omap/osk5912.c
--- a/sound/soc/omap/osk5912.c	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/omap/osk5912.c	1969-12-31 19:00:00.000000000 -0500
@@ -1,187 +0,0 @@
-/*
- * osk5912.c  --  SoC audio for OSK 5912
- *
- * Copyright (C) 2008 Mistral Solutions
- *
- * Contact: Arun KS  <arunks@mistralsolutions.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
- * 02110-1301 USA
- *
- */
-
-#include <linux/clk.h>
-#include <linux/platform_device.h>
-#include <sound/core.h>
-#include <sound/pcm.h>
-#include <sound/soc.h>
-
-#include <asm/mach-types.h>
-#include <linux/gpio.h>
-#include <linux/module.h>
-#include <linux/platform_data/asoc-ti-mcbsp.h>
-
-#include "omap-mcbsp.h"
-#include "../codecs/tlv320aic23.h"
-
-#define CODEC_CLOCK 	12000000
-
-static struct clk *tlv320aic23_mclk;
-
-static int osk_startup(struct snd_pcm_substream *substream)
-{
-	return clk_enable(tlv320aic23_mclk);
-}
-
-static void osk_shutdown(struct snd_pcm_substream *substream)
-{
-	clk_disable(tlv320aic23_mclk);
-}
-
-static int osk_hw_params(struct snd_pcm_substream *substream,
-			 struct snd_pcm_hw_params *params)
-{
-	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct snd_soc_dai *codec_dai = rtd->codec_dai;
-	int err;
-
-	/* Set the codec system clock for DAC and ADC */
-	err =
-	    snd_soc_dai_set_sysclk(codec_dai, 0, CODEC_CLOCK, SND_SOC_CLOCK_IN);
-
-	if (err < 0) {
-		printk(KERN_ERR "can't set codec system clock\n");
-		return err;
-	}
-
-	return err;
-}
-
-static const struct snd_soc_ops osk_ops = {
-	.startup = osk_startup,
-	.hw_params = osk_hw_params,
-	.shutdown = osk_shutdown,
-};
-
-static const struct snd_soc_dapm_widget tlv320aic23_dapm_widgets[] = {
-	SND_SOC_DAPM_HP("Headphone Jack", NULL),
-	SND_SOC_DAPM_LINE("Line In", NULL),
-	SND_SOC_DAPM_MIC("Mic Jack", NULL),
-};
-
-static const struct snd_soc_dapm_route audio_map[] = {
-	{"Headphone Jack", NULL, "LHPOUT"},
-	{"Headphone Jack", NULL, "RHPOUT"},
-
-	{"LLINEIN", NULL, "Line In"},
-	{"RLINEIN", NULL, "Line In"},
-
-	{"MICIN", NULL, "Mic Jack"},
-};
-
-/* Digital audio interface glue - connects codec <--> CPU */
-static struct snd_soc_dai_link osk_dai = {
-	.name = "TLV320AIC23",
-	.stream_name = "AIC23",
-	.cpu_dai_name = "omap-mcbsp.1",
-	.codec_dai_name = "tlv320aic23-hifi",
-	.platform_name = "omap-mcbsp.1",
-	.codec_name = "tlv320aic23-codec",
-	.dai_fmt = SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_NB_NF |
-		   SND_SOC_DAIFMT_CBM_CFM,
-	.ops = &osk_ops,
-};
-
-/* Audio machine driver */
-static struct snd_soc_card snd_soc_card_osk = {
-	.name = "OSK5912",
-	.owner = THIS_MODULE,
-	.dai_link = &osk_dai,
-	.num_links = 1,
-
-	.dapm_widgets = tlv320aic23_dapm_widgets,
-	.num_dapm_widgets = ARRAY_SIZE(tlv320aic23_dapm_widgets),
-	.dapm_routes = audio_map,
-	.num_dapm_routes = ARRAY_SIZE(audio_map),
-};
-
-static struct platform_device *osk_snd_device;
-
-static int __init osk_soc_init(void)
-{
-	int err;
-	u32 curRate;
-	struct device *dev;
-
-	if (!(machine_is_omap_osk()))
-		return -ENODEV;
-
-	osk_snd_device = platform_device_alloc("soc-audio", -1);
-	if (!osk_snd_device)
-		return -ENOMEM;
-
-	platform_set_drvdata(osk_snd_device, &snd_soc_card_osk);
-	err = platform_device_add(osk_snd_device);
-	if (err)
-		goto err1;
-
-	dev = &osk_snd_device->dev;
-
-	tlv320aic23_mclk = clk_get(dev, "mclk");
-	if (IS_ERR(tlv320aic23_mclk)) {
-		printk(KERN_ERR "Could not get mclk clock\n");
-		err = PTR_ERR(tlv320aic23_mclk);
-		goto err2;
-	}
-
-	/*
-	 * Configure 12 MHz output on MCLK.
-	 */
-	curRate = (uint) clk_get_rate(tlv320aic23_mclk);
-	if (curRate != CODEC_CLOCK) {
-		if (clk_set_rate(tlv320aic23_mclk, CODEC_CLOCK)) {
-			printk(KERN_ERR "Cannot set MCLK for AIC23 CODEC\n");
-			err = -ECANCELED;
-			goto err3;
-		}
-	}
-
-	printk(KERN_INFO "MCLK = %d [%d]\n",
-	       (uint) clk_get_rate(tlv320aic23_mclk), CODEC_CLOCK);
-
-	return 0;
-
-err3:
-	clk_put(tlv320aic23_mclk);
-err2:
-	platform_device_del(osk_snd_device);
-err1:
-	platform_device_put(osk_snd_device);
-
-	return err;
-
-}
-
-static void __exit osk_soc_exit(void)
-{
-	clk_put(tlv320aic23_mclk);
-	platform_device_unregister(osk_snd_device);
-}
-
-module_init(osk_soc_init);
-module_exit(osk_soc_exit);
-
-MODULE_AUTHOR("Arun KS <arunks@mistralsolutions.com>");
-MODULE_DESCRIPTION("ALSA SoC OSK 5912");
-MODULE_LICENSE("GPL");
diff -Naur --no-dereference a/sound/soc/omap/rx51.c b/sound/soc/omap/rx51.c
--- a/sound/soc/omap/rx51.c	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/omap/rx51.c	1969-12-31 19:00:00.000000000 -0500
@@ -1,493 +0,0 @@
-/*
- * rx51.c  --  SoC audio for Nokia RX-51
- *
- * Copyright (C) 2008 - 2009 Nokia Corporation
- *
- * Contact: Peter Ujfalusi <peter.ujfalusi@ti.com>
- *          Eduardo Valentin <eduardo.valentin@nokia.com>
- *          Jarkko Nikula <jarkko.nikula@bitmer.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
- * 02110-1301 USA
- *
- */
-
-#include <linux/delay.h>
-#include <linux/gpio.h>
-#include <linux/platform_device.h>
-#include <linux/gpio/consumer.h>
-#include <linux/module.h>
-#include <sound/core.h>
-#include <sound/jack.h>
-#include <sound/pcm.h>
-#include <sound/soc.h>
-#include <linux/platform_data/asoc-ti-mcbsp.h>
-
-#include <asm/mach-types.h>
-
-#include "omap-mcbsp.h"
-
-enum {
-	RX51_JACK_DISABLED,
-	RX51_JACK_TVOUT,		/* tv-out with stereo output */
-	RX51_JACK_HP,			/* headphone: stereo output, no mic */
-	RX51_JACK_HS,			/* headset: stereo output with mic */
-};
-
-struct rx51_audio_pdata {
-	struct gpio_desc *tvout_selection_gpio;
-	struct gpio_desc *jack_detection_gpio;
-	struct gpio_desc *eci_sw_gpio;
-	struct gpio_desc *speaker_amp_gpio;
-};
-
-static int rx51_spk_func;
-static int rx51_dmic_func;
-static int rx51_jack_func;
-
-static void rx51_ext_control(struct snd_soc_dapm_context *dapm)
-{
-	struct snd_soc_card *card = dapm->card;
-	struct rx51_audio_pdata *pdata = snd_soc_card_get_drvdata(card);
-	int hp = 0, hs = 0, tvout = 0;
-
-	switch (rx51_jack_func) {
-	case RX51_JACK_TVOUT:
-		tvout = 1;
-		hp = 1;
-		break;
-	case RX51_JACK_HS:
-		hs = 1;
-	case RX51_JACK_HP:
-		hp = 1;
-		break;
-	}
-
-	snd_soc_dapm_mutex_lock(dapm);
-
-	if (rx51_spk_func)
-		snd_soc_dapm_enable_pin_unlocked(dapm, "Ext Spk");
-	else
-		snd_soc_dapm_disable_pin_unlocked(dapm, "Ext Spk");
-	if (rx51_dmic_func)
-		snd_soc_dapm_enable_pin_unlocked(dapm, "DMic");
-	else
-		snd_soc_dapm_disable_pin_unlocked(dapm, "DMic");
-	if (hp)
-		snd_soc_dapm_enable_pin_unlocked(dapm, "Headphone Jack");
-	else
-		snd_soc_dapm_disable_pin_unlocked(dapm, "Headphone Jack");
-	if (hs)
-		snd_soc_dapm_enable_pin_unlocked(dapm, "HS Mic");
-	else
-		snd_soc_dapm_disable_pin_unlocked(dapm, "HS Mic");
-
-	gpiod_set_value(pdata->tvout_selection_gpio, tvout);
-
-	snd_soc_dapm_sync_unlocked(dapm);
-
-	snd_soc_dapm_mutex_unlock(dapm);
-}
-
-static int rx51_startup(struct snd_pcm_substream *substream)
-{
-	struct snd_pcm_runtime *runtime = substream->runtime;
-	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct snd_soc_card *card = rtd->card;
-
-	snd_pcm_hw_constraint_single(runtime, SNDRV_PCM_HW_PARAM_CHANNELS, 2);
-	rx51_ext_control(&card->dapm);
-
-	return 0;
-}
-
-static int rx51_hw_params(struct snd_pcm_substream *substream,
-	struct snd_pcm_hw_params *params)
-{
-	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct snd_soc_dai *codec_dai = rtd->codec_dai;
-
-	/* Set the codec system clock for DAC and ADC */
-	return snd_soc_dai_set_sysclk(codec_dai, 0, 19200000,
-				      SND_SOC_CLOCK_IN);
-}
-
-static const struct snd_soc_ops rx51_ops = {
-	.startup = rx51_startup,
-	.hw_params = rx51_hw_params,
-};
-
-static int rx51_get_spk(struct snd_kcontrol *kcontrol,
-			struct snd_ctl_elem_value *ucontrol)
-{
-	ucontrol->value.enumerated.item[0] = rx51_spk_func;
-
-	return 0;
-}
-
-static int rx51_set_spk(struct snd_kcontrol *kcontrol,
-			struct snd_ctl_elem_value *ucontrol)
-{
-	struct snd_soc_card *card = snd_kcontrol_chip(kcontrol);
-
-	if (rx51_spk_func == ucontrol->value.enumerated.item[0])
-		return 0;
-
-	rx51_spk_func = ucontrol->value.enumerated.item[0];
-	rx51_ext_control(&card->dapm);
-
-	return 1;
-}
-
-static int rx51_spk_event(struct snd_soc_dapm_widget *w,
-			  struct snd_kcontrol *k, int event)
-{
-	struct snd_soc_dapm_context *dapm = w->dapm;
-	struct snd_soc_card *card = dapm->card;
-	struct rx51_audio_pdata *pdata = snd_soc_card_get_drvdata(card);
-
-	gpiod_set_raw_value_cansleep(pdata->speaker_amp_gpio,
-				     !!SND_SOC_DAPM_EVENT_ON(event));
-
-	return 0;
-}
-
-static int rx51_get_input(struct snd_kcontrol *kcontrol,
-			  struct snd_ctl_elem_value *ucontrol)
-{
-	ucontrol->value.enumerated.item[0] = rx51_dmic_func;
-
-	return 0;
-}
-
-static int rx51_set_input(struct snd_kcontrol *kcontrol,
-			  struct snd_ctl_elem_value *ucontrol)
-{
-	struct snd_soc_card *card = snd_kcontrol_chip(kcontrol);
-
-	if (rx51_dmic_func == ucontrol->value.enumerated.item[0])
-		return 0;
-
-	rx51_dmic_func = ucontrol->value.enumerated.item[0];
-	rx51_ext_control(&card->dapm);
-
-	return 1;
-}
-
-static int rx51_get_jack(struct snd_kcontrol *kcontrol,
-			 struct snd_ctl_elem_value *ucontrol)
-{
-	ucontrol->value.enumerated.item[0] = rx51_jack_func;
-
-	return 0;
-}
-
-static int rx51_set_jack(struct snd_kcontrol *kcontrol,
-			 struct snd_ctl_elem_value *ucontrol)
-{
-	struct snd_soc_card *card = snd_kcontrol_chip(kcontrol);
-
-	if (rx51_jack_func == ucontrol->value.enumerated.item[0])
-		return 0;
-
-	rx51_jack_func = ucontrol->value.enumerated.item[0];
-	rx51_ext_control(&card->dapm);
-
-	return 1;
-}
-
-static struct snd_soc_jack rx51_av_jack;
-
-static struct snd_soc_jack_gpio rx51_av_jack_gpios[] = {
-	{
-		.name = "avdet-gpio",
-		.report = SND_JACK_HEADSET,
-		.invert = 1,
-		.debounce_time = 200,
-	},
-};
-
-static const struct snd_soc_dapm_widget aic34_dapm_widgets[] = {
-	SND_SOC_DAPM_SPK("Ext Spk", rx51_spk_event),
-	SND_SOC_DAPM_MIC("DMic", NULL),
-	SND_SOC_DAPM_HP("Headphone Jack", NULL),
-	SND_SOC_DAPM_MIC("HS Mic", NULL),
-	SND_SOC_DAPM_LINE("FM Transmitter", NULL),
-	SND_SOC_DAPM_SPK("Earphone", NULL),
-};
-
-static const struct snd_soc_dapm_route audio_map[] = {
-	{"Ext Spk", NULL, "HPLOUT"},
-	{"Ext Spk", NULL, "HPROUT"},
-	{"Ext Spk", NULL, "HPLCOM"},
-	{"Ext Spk", NULL, "HPRCOM"},
-	{"FM Transmitter", NULL, "LLOUT"},
-	{"FM Transmitter", NULL, "RLOUT"},
-
-	{"Headphone Jack", NULL, "TPA6130A2 HPLEFT"},
-	{"Headphone Jack", NULL, "TPA6130A2 HPRIGHT"},
-	{"TPA6130A2 LEFTIN", NULL, "LLOUT"},
-	{"TPA6130A2 RIGHTIN", NULL, "RLOUT"},
-
-	{"DMic Rate 64", NULL, "DMic"},
-	{"DMic", NULL, "Mic Bias"},
-
-	{"b LINE2R", NULL, "MONO_LOUT"},
-	{"Earphone", NULL, "b HPLOUT"},
-
-	{"LINE1L", NULL, "HS Mic"},
-	{"HS Mic", NULL, "b Mic Bias"},
-};
-
-static const char * const spk_function[] = {"Off", "On"};
-static const char * const input_function[] = {"ADC", "Digital Mic"};
-static const char * const jack_function[] = {
-	"Off", "TV-OUT", "Headphone", "Headset"
-};
-
-static const struct soc_enum rx51_enum[] = {
-	SOC_ENUM_SINGLE_EXT(ARRAY_SIZE(spk_function), spk_function),
-	SOC_ENUM_SINGLE_EXT(ARRAY_SIZE(input_function), input_function),
-	SOC_ENUM_SINGLE_EXT(ARRAY_SIZE(jack_function), jack_function),
-};
-
-static const struct snd_kcontrol_new aic34_rx51_controls[] = {
-	SOC_ENUM_EXT("Speaker Function", rx51_enum[0],
-		     rx51_get_spk, rx51_set_spk),
-	SOC_ENUM_EXT("Input Select",  rx51_enum[1],
-		     rx51_get_input, rx51_set_input),
-	SOC_ENUM_EXT("Jack Function", rx51_enum[2],
-		     rx51_get_jack, rx51_set_jack),
-	SOC_DAPM_PIN_SWITCH("FM Transmitter"),
-	SOC_DAPM_PIN_SWITCH("Earphone"),
-};
-
-static int rx51_aic34_init(struct snd_soc_pcm_runtime *rtd)
-{
-	struct snd_soc_card *card = rtd->card;
-	struct rx51_audio_pdata *pdata = snd_soc_card_get_drvdata(card);
-	int err;
-
-	snd_soc_limit_volume(card, "TPA6130A2 Headphone Playback Volume", 42);
-
-	err = omap_mcbsp_st_add_controls(rtd, 2);
-	if (err < 0) {
-		dev_err(card->dev, "Failed to add MCBSP controls\n");
-		return err;
-	}
-
-	/* AV jack detection */
-	err = snd_soc_card_jack_new(rtd->card, "AV Jack",
-				    SND_JACK_HEADSET | SND_JACK_VIDEOOUT,
-				    &rx51_av_jack, NULL, 0);
-	if (err) {
-		dev_err(card->dev, "Failed to add AV Jack\n");
-		return err;
-	}
-
-	/* prepare gpio for snd_soc_jack_add_gpios */
-	rx51_av_jack_gpios[0].gpio = desc_to_gpio(pdata->jack_detection_gpio);
-	devm_gpiod_put(card->dev, pdata->jack_detection_gpio);
-
-	err = snd_soc_jack_add_gpios(&rx51_av_jack,
-				     ARRAY_SIZE(rx51_av_jack_gpios),
-				     rx51_av_jack_gpios);
-	if (err) {
-		dev_err(card->dev, "Failed to add GPIOs\n");
-		return err;
-	}
-
-	return err;
-}
-
-/* Digital audio interface glue - connects codec <--> CPU */
-static struct snd_soc_dai_link rx51_dai[] = {
-	{
-		.name = "TLV320AIC34",
-		.stream_name = "AIC34",
-		.cpu_dai_name = "omap-mcbsp.2",
-		.codec_dai_name = "tlv320aic3x-hifi",
-		.platform_name = "omap-mcbsp.2",
-		.codec_name = "tlv320aic3x-codec.2-0018",
-		.dai_fmt = SND_SOC_DAIFMT_DSP_A | SND_SOC_DAIFMT_IB_NF |
-			   SND_SOC_DAIFMT_CBM_CFM,
-		.init = rx51_aic34_init,
-		.ops = &rx51_ops,
-	},
-};
-
-static struct snd_soc_aux_dev rx51_aux_dev[] = {
-	{
-		.name = "TLV320AIC34b",
-		.codec_name = "tlv320aic3x-codec.2-0019",
-	},
-	{
-		.name = "TPA61320A2",
-		.codec_name = "tpa6130a2.2-0060",
-	},
-};
-
-static struct snd_soc_codec_conf rx51_codec_conf[] = {
-	{
-		.dev_name = "tlv320aic3x-codec.2-0019",
-		.name_prefix = "b",
-	},
-	{
-		.dev_name = "tpa6130a2.2-0060",
-		.name_prefix = "TPA6130A2",
-	},
-};
-
-/* Audio card */
-static struct snd_soc_card rx51_sound_card = {
-	.name = "RX-51",
-	.owner = THIS_MODULE,
-	.dai_link = rx51_dai,
-	.num_links = ARRAY_SIZE(rx51_dai),
-	.aux_dev = rx51_aux_dev,
-	.num_aux_devs = ARRAY_SIZE(rx51_aux_dev),
-	.codec_conf = rx51_codec_conf,
-	.num_configs = ARRAY_SIZE(rx51_codec_conf),
-	.fully_routed = true,
-
-	.controls = aic34_rx51_controls,
-	.num_controls = ARRAY_SIZE(aic34_rx51_controls),
-	.dapm_widgets = aic34_dapm_widgets,
-	.num_dapm_widgets = ARRAY_SIZE(aic34_dapm_widgets),
-	.dapm_routes = audio_map,
-	.num_dapm_routes = ARRAY_SIZE(audio_map),
-};
-
-static int rx51_soc_probe(struct platform_device *pdev)
-{
-	struct rx51_audio_pdata *pdata;
-	struct device_node *np = pdev->dev.of_node;
-	struct snd_soc_card *card = &rx51_sound_card;
-	int err;
-
-	if (!machine_is_nokia_rx51() && !of_machine_is_compatible("nokia,omap3-n900"))
-		return -ENODEV;
-
-	card->dev = &pdev->dev;
-
-	if (np) {
-		struct device_node *dai_node;
-
-		dai_node = of_parse_phandle(np, "nokia,cpu-dai", 0);
-		if (!dai_node) {
-			dev_err(&pdev->dev, "McBSP node is not provided\n");
-			return -EINVAL;
-		}
-		rx51_dai[0].cpu_dai_name = NULL;
-		rx51_dai[0].platform_name = NULL;
-		rx51_dai[0].cpu_of_node = dai_node;
-		rx51_dai[0].platform_of_node = dai_node;
-
-		dai_node = of_parse_phandle(np, "nokia,audio-codec", 0);
-		if (!dai_node) {
-			dev_err(&pdev->dev, "Codec node is not provided\n");
-			return -EINVAL;
-		}
-		rx51_dai[0].codec_name = NULL;
-		rx51_dai[0].codec_of_node = dai_node;
-
-		dai_node = of_parse_phandle(np, "nokia,audio-codec", 1);
-		if (!dai_node) {
-			dev_err(&pdev->dev, "Auxiliary Codec node is not provided\n");
-			return -EINVAL;
-		}
-		rx51_aux_dev[0].codec_name = NULL;
-		rx51_aux_dev[0].codec_of_node = dai_node;
-		rx51_codec_conf[0].dev_name = NULL;
-		rx51_codec_conf[0].of_node = dai_node;
-
-		dai_node = of_parse_phandle(np, "nokia,headphone-amplifier", 0);
-		if (!dai_node) {
-			dev_err(&pdev->dev, "Headphone amplifier node is not provided\n");
-			return -EINVAL;
-		}
-		rx51_aux_dev[1].codec_name = NULL;
-		rx51_aux_dev[1].codec_of_node = dai_node;
-		rx51_codec_conf[1].dev_name = NULL;
-		rx51_codec_conf[1].of_node = dai_node;
-	}
-
-	pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
-	if (pdata == NULL)
-		return -ENOMEM;
-
-	snd_soc_card_set_drvdata(card, pdata);
-
-	pdata->tvout_selection_gpio = devm_gpiod_get(card->dev,
-						     "tvout-selection",
-						     GPIOD_OUT_LOW);
-	if (IS_ERR(pdata->tvout_selection_gpio)) {
-		dev_err(card->dev, "could not get tvout selection gpio\n");
-		return PTR_ERR(pdata->tvout_selection_gpio);
-	}
-
-	pdata->jack_detection_gpio = devm_gpiod_get(card->dev,
-						    "jack-detection",
-						    GPIOD_ASIS);
-	if (IS_ERR(pdata->jack_detection_gpio)) {
-		dev_err(card->dev, "could not get jack detection gpio\n");
-		return PTR_ERR(pdata->jack_detection_gpio);
-	}
-
-	pdata->eci_sw_gpio = devm_gpiod_get(card->dev, "eci-switch",
-					    GPIOD_OUT_HIGH);
-	if (IS_ERR(pdata->eci_sw_gpio)) {
-		dev_err(card->dev, "could not get eci switch gpio\n");
-		return PTR_ERR(pdata->eci_sw_gpio);
-	}
-
-	pdata->speaker_amp_gpio = devm_gpiod_get(card->dev,
-						 "speaker-amplifier",
-						 GPIOD_OUT_LOW);
-	if (IS_ERR(pdata->speaker_amp_gpio)) {
-		dev_err(card->dev, "could not get speaker enable gpio\n");
-		return PTR_ERR(pdata->speaker_amp_gpio);
-	}
-
-	err = devm_snd_soc_register_card(card->dev, card);
-	if (err) {
-		dev_err(&pdev->dev, "snd_soc_register_card failed (%d)\n", err);
-		return err;
-	}
-
-	return 0;
-}
-
-#if defined(CONFIG_OF)
-static const struct of_device_id rx51_audio_of_match[] = {
-	{ .compatible = "nokia,n900-audio", },
-	{},
-};
-MODULE_DEVICE_TABLE(of, rx51_audio_of_match);
-#endif
-
-static struct platform_driver rx51_soc_driver = {
-	.driver = {
-		.name = "rx51-audio",
-		.of_match_table = of_match_ptr(rx51_audio_of_match),
-	},
-	.probe = rx51_soc_probe,
-};
-
-module_platform_driver(rx51_soc_driver);
-
-MODULE_AUTHOR("Nokia Corporation");
-MODULE_DESCRIPTION("ALSA SoC Nokia RX-51");
-MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:rx51-audio");
diff -Naur --no-dereference a/sound/soc/omap/sdma-pcm.c b/sound/soc/omap/sdma-pcm.c
--- a/sound/soc/omap/sdma-pcm.c	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/omap/sdma-pcm.c	1969-12-31 19:00:00.000000000 -0500
@@ -1,74 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- *  Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
- *  Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
- */
-
-#include <linux/device.h>
-#include <linux/module.h>
-#include <sound/core.h>
-#include <sound/pcm.h>
-#include <sound/pcm_params.h>
-#include <sound/soc.h>
-#include <sound/dmaengine_pcm.h>
-#include <linux/omap-dmaengine.h>
-
-#include "sdma-pcm.h"
-
-static const struct snd_pcm_hardware sdma_pcm_hardware = {
-	.info			= SNDRV_PCM_INFO_MMAP |
-				  SNDRV_PCM_INFO_MMAP_VALID |
-				  SNDRV_PCM_INFO_PAUSE | SNDRV_PCM_INFO_RESUME |
-				  SNDRV_PCM_INFO_NO_PERIOD_WAKEUP |
-				  SNDRV_PCM_INFO_INTERLEAVED,
-	.period_bytes_min	= 32,
-	.period_bytes_max	= 64 * 1024,
-	.buffer_bytes_max	= 128 * 1024,
-	.periods_min		= 2,
-	.periods_max		= 255,
-};
-
-static const struct snd_dmaengine_pcm_config sdma_dmaengine_pcm_config = {
-	.pcm_hardware = &sdma_pcm_hardware,
-	.prepare_slave_config = snd_dmaengine_pcm_prepare_slave_config,
-	.compat_filter_fn = omap_dma_filter_fn,
-	.prealloc_buffer_size = 128 * 1024,
-};
-
-int sdma_pcm_platform_register(struct device *dev,
-			       char *txdmachan, char *rxdmachan)
-{
-	struct snd_dmaengine_pcm_config *config;
-	unsigned int flags = SND_DMAENGINE_PCM_FLAG_COMPAT;
-
-	/* Standard names for the directions: 'tx' and 'rx' */
-	if (!txdmachan && !rxdmachan)
-		return devm_snd_dmaengine_pcm_register(dev,
-						&sdma_dmaengine_pcm_config,
-						flags);
-
-	config = devm_kzalloc(dev, sizeof(*config), GFP_KERNEL);
-	if (!config)
-		return -ENOMEM;
-
-	*config = sdma_dmaengine_pcm_config;
-
-	if (!txdmachan || !rxdmachan) {
-		/* One direction only PCM */
-		flags |= SND_DMAENGINE_PCM_FLAG_HALF_DUPLEX;
-		if (!txdmachan) {
-			txdmachan = rxdmachan;
-			rxdmachan = NULL;
-		}
-	}
-
-	config->chan_names[0] = txdmachan;
-	config->chan_names[1] = rxdmachan;
-
-	return devm_snd_dmaengine_pcm_register(dev, config, flags);
-}
-EXPORT_SYMBOL_GPL(sdma_pcm_platform_register);
-
-MODULE_AUTHOR("Peter Ujfalusi <peter.ujfalusi@ti.com>");
-MODULE_DESCRIPTION("sDMA PCM ASoC platform driver");
-MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/sound/soc/omap/sdma-pcm.h b/sound/soc/omap/sdma-pcm.h
--- a/sound/soc/omap/sdma-pcm.h	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/omap/sdma-pcm.h	1969-12-31 19:00:00.000000000 -0500
@@ -1,21 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- *  Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
- *  Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
- */
-
-#ifndef __SDMA_PCM_H__
-#define __SDMA_PCM_H__
-
-#if IS_ENABLED(CONFIG_SND_SDMA_SOC)
-int sdma_pcm_platform_register(struct device *dev,
-			       char *txdmachan, char *rxdmachan);
-#else
-static inline int sdma_pcm_platform_register(struct device *dev,
-					     char *txdmachan, char *rxdmachan)
-{
-	return -ENODEV;
-}
-#endif /* CONFIG_SND_SDMA_SOC */
-
-#endif /* __SDMA_PCM_H__ */
diff -Naur --no-dereference a/sound/soc/soc-compress.c b/sound/soc/soc-compress.c
--- a/sound/soc/soc-compress.c	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/soc-compress.c	2020-06-19 21:00:31.436227717 -0400
@@ -80,7 +80,7 @@
 	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
 	int ret;
 
-	mutex_lock_nested(&rtd->pcm_mutex, rtd->pcm_subclass);
+	mutex_lock_nested(&rtd->card->pcm_mutex, rtd->card->pcm_subclass);
 
 	if (cpu_dai->driver->cops && cpu_dai->driver->cops->startup) {
 		ret = cpu_dai->driver->cops->startup(cstream, cpu_dai);
@@ -108,7 +108,7 @@
 
 	snd_soc_runtime_activate(rtd, cstream->direction);
 
-	mutex_unlock(&rtd->pcm_mutex);
+	mutex_unlock(&rtd->card->pcm_mutex);
 
 	return 0;
 
@@ -118,7 +118,7 @@
 	if (cpu_dai->driver->cops && cpu_dai->driver->cops->shutdown)
 		cpu_dai->driver->cops->shutdown(cstream, cpu_dai);
 out:
-	mutex_unlock(&rtd->pcm_mutex);
+	mutex_unlock(&rtd->card->pcm_mutex);
 	return ret;
 }
 
@@ -224,7 +224,7 @@
 			container_of(work, struct snd_soc_pcm_runtime, delayed_work.work);
 	struct snd_soc_dai *codec_dai = rtd->codec_dai;
 
-	mutex_lock_nested(&rtd->pcm_mutex, rtd->pcm_subclass);
+	mutex_lock_nested(&rtd->card->pcm_mutex, rtd->card->pcm_subclass);
 
 	dev_dbg(rtd->dev,
 		"Compress ASoC: pop wq checking: %s status: %s waiting: %s\n",
@@ -239,7 +239,7 @@
 					  SND_SOC_DAPM_STREAM_STOP);
 	}
 
-	mutex_unlock(&rtd->pcm_mutex);
+	mutex_unlock(&rtd->card->pcm_mutex);
 }
 
 static int soc_compr_free(struct snd_compr_stream *cstream)
@@ -249,7 +249,7 @@
 	struct snd_soc_dai *codec_dai = rtd->codec_dai;
 	int stream;
 
-	mutex_lock_nested(&rtd->pcm_mutex, rtd->pcm_subclass);
+	mutex_lock_nested(&rtd->card->pcm_mutex, rtd->card->pcm_subclass);
 
 	if (cstream->direction == SND_COMPRESS_PLAYBACK)
 		stream = SNDRV_PCM_STREAM_PLAYBACK;
@@ -292,7 +292,7 @@
 					  SND_SOC_DAPM_STREAM_STOP);
 	}
 
-	mutex_unlock(&rtd->pcm_mutex);
+	mutex_unlock(&rtd->card->pcm_mutex);
 	return 0;
 }
 
@@ -355,7 +355,7 @@
 	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
 	int ret = 0, __ret;
 
-	mutex_lock_nested(&rtd->pcm_mutex, rtd->pcm_subclass);
+	mutex_lock_nested(&rtd->card->pcm_mutex, rtd->card->pcm_subclass);
 
 	for_each_rtdcom(rtd, rtdcom) {
 		component = rtdcom->component;
@@ -384,7 +384,7 @@
 	}
 
 out:
-	mutex_unlock(&rtd->pcm_mutex);
+	mutex_unlock(&rtd->card->pcm_mutex);
 	return ret;
 }
 
@@ -474,7 +474,7 @@
 	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
 	int ret = 0, __ret;
 
-	mutex_lock_nested(&rtd->pcm_mutex, rtd->pcm_subclass);
+	mutex_lock_nested(&rtd->card->pcm_mutex, rtd->card->pcm_subclass);
 
 	/*
 	 * First we call set_params for the CPU DAI, then the component
@@ -518,14 +518,14 @@
 
 	/* cancel any delayed stream shutdown that is pending */
 	rtd->pop_wait = 0;
-	mutex_unlock(&rtd->pcm_mutex);
+	mutex_unlock(&rtd->card->pcm_mutex);
 
 	cancel_delayed_work_sync(&rtd->delayed_work);
 
 	return ret;
 
 err:
-	mutex_unlock(&rtd->pcm_mutex);
+	mutex_unlock(&rtd->card->pcm_mutex);
 	return ret;
 }
 
@@ -609,7 +609,7 @@
 	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
 	int ret = 0, __ret;
 
-	mutex_lock_nested(&rtd->pcm_mutex, rtd->pcm_subclass);
+	mutex_lock_nested(&rtd->card->pcm_mutex, rtd->card->pcm_subclass);
 
 	if (cpu_dai->driver->cops && cpu_dai->driver->cops->get_params) {
 		ret = cpu_dai->driver->cops->get_params(cstream, params, cpu_dai);
@@ -630,7 +630,7 @@
 	}
 
 err:
-	mutex_unlock(&rtd->pcm_mutex);
+	mutex_unlock(&rtd->card->pcm_mutex);
 	return ret;
 }
 
@@ -642,7 +642,7 @@
 	struct snd_soc_rtdcom_list *rtdcom;
 	int ret = 0, __ret;
 
-	mutex_lock_nested(&rtd->pcm_mutex, rtd->pcm_subclass);
+	mutex_lock_nested(&rtd->card->pcm_mutex, rtd->card->pcm_subclass);
 
 	for_each_rtdcom(rtd, rtdcom) {
 		component = rtdcom->component;
@@ -656,7 +656,7 @@
 			ret = __ret;
 	}
 
-	mutex_unlock(&rtd->pcm_mutex);
+	mutex_unlock(&rtd->card->pcm_mutex);
 	return ret;
 }
 
@@ -668,7 +668,7 @@
 	struct snd_soc_rtdcom_list *rtdcom;
 	int ret = 0, __ret;
 
-	mutex_lock_nested(&rtd->pcm_mutex, rtd->pcm_subclass);
+	mutex_lock_nested(&rtd->card->pcm_mutex, rtd->card->pcm_subclass);
 
 	for_each_rtdcom(rtd, rtdcom) {
 		component = rtdcom->component;
@@ -682,7 +682,7 @@
 			ret = __ret;
 	}
 
-	mutex_unlock(&rtd->pcm_mutex);
+	mutex_unlock(&rtd->card->pcm_mutex);
 	return ret;
 }
 
@@ -694,7 +694,7 @@
 	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
 	int ret = 0, __ret;
 
-	mutex_lock_nested(&rtd->pcm_mutex, rtd->pcm_subclass);
+	mutex_lock_nested(&rtd->card->pcm_mutex, rtd->card->pcm_subclass);
 
 	if (cpu_dai->driver->cops && cpu_dai->driver->cops->ack) {
 		ret = cpu_dai->driver->cops->ack(cstream, bytes, cpu_dai);
@@ -715,7 +715,7 @@
 	}
 
 err:
-	mutex_unlock(&rtd->pcm_mutex);
+	mutex_unlock(&rtd->card->pcm_mutex);
 	return ret;
 }
 
@@ -728,7 +728,7 @@
 	int ret = 0, __ret;
 	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
 
-	mutex_lock_nested(&rtd->pcm_mutex, rtd->pcm_subclass);
+	mutex_lock_nested(&rtd->card->pcm_mutex, rtd->card->pcm_subclass);
 
 	if (cpu_dai->driver->cops && cpu_dai->driver->cops->pointer)
 		cpu_dai->driver->cops->pointer(cstream, tstamp, cpu_dai);
@@ -745,7 +745,7 @@
 			ret = __ret;
 	}
 
-	mutex_unlock(&rtd->pcm_mutex);
+	mutex_unlock(&rtd->card->pcm_mutex);
 	return ret;
 }
 
@@ -757,7 +757,7 @@
 	struct snd_soc_rtdcom_list *rtdcom;
 	int ret = 0;
 
-	mutex_lock_nested(&rtd->pcm_mutex, rtd->pcm_subclass);
+	mutex_lock_nested(&rtd->card->pcm_mutex, rtd->card->pcm_subclass);
 
 	for_each_rtdcom(rtd, rtdcom) {
 		component = rtdcom->component;
@@ -770,7 +770,7 @@
 		break;
 	}
 
-	mutex_unlock(&rtd->pcm_mutex);
+	mutex_unlock(&rtd->card->pcm_mutex);
 	return ret;
 }
 
diff -Naur --no-dereference a/sound/soc/soc-core.c b/sound/soc/soc-core.c
--- a/sound/soc/soc-core.c	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/soc-core.c	2020-06-19 21:00:31.436227717 -0400
@@ -1352,7 +1352,6 @@
 	rtd->dev->groups = soc_dev_attr_groups;
 	dev_set_name(rtd->dev, "%s", name);
 	dev_set_drvdata(rtd->dev, rtd);
-	mutex_init(&rtd->pcm_mutex);
 	INIT_LIST_HEAD(&rtd->dpcm[SNDRV_PCM_STREAM_PLAYBACK].be_clients);
 	INIT_LIST_HEAD(&rtd->dpcm[SNDRV_PCM_STREAM_CAPTURE].be_clients);
 	INIT_LIST_HEAD(&rtd->dpcm[SNDRV_PCM_STREAM_PLAYBACK].fe_clients);
@@ -1957,6 +1956,7 @@
 	struct snd_soc_pcm_runtime *rtd;
 	struct snd_soc_dai_link *dai_link;
 	int ret, i, order;
+	int idx;
 
 	mutex_lock(&client_mutex);
 	mutex_lock_nested(&card->mutex, SND_SOC_CARD_CLASS_INIT);
@@ -1982,9 +1982,17 @@
 	for (i = 0; i < card->num_links; i++)
 		snd_soc_add_dai_link(card, card->dai_link+i);
 
+	if (card->dev->of_node) {
+		idx = of_alias_get_id(card->dev->of_node, "sound");
+		if (idx < 0)
+			idx = SNDRV_DEFAULT_IDX1;
+	} else {
+		idx = card->id_hint;
+	}
+
 	/* card bind complete so register a sound card */
-	ret = snd_card_new(card->dev, SNDRV_DEFAULT_IDX1, SNDRV_DEFAULT_STR1,
-			card->owner, 0, &card->snd_card);
+	ret = snd_card_new(card->dev, idx, SNDRV_DEFAULT_STR1, card->owner, 0,
+			   &card->snd_card);
 	if (ret < 0) {
 		dev_err(card->dev,
 			"ASoC: can't create sound card for card %s: %d\n",
@@ -2734,6 +2742,7 @@
 	card->instantiated = 0;
 	mutex_init(&card->mutex);
 	mutex_init(&card->dapm_mutex);
+	mutex_init(&card->pcm_mutex);
 
 	ret = snd_soc_instantiate_card(card);
 	if (ret != 0)
diff -Naur --no-dereference a/sound/soc/soc-pcm.c b/sound/soc/soc-pcm.c
--- a/sound/soc/soc-pcm.c	2020-01-09 04:19:10.000000000 -0500
+++ b/sound/soc/soc-pcm.c	2020-06-19 21:00:31.436227717 -0400
@@ -54,14 +54,14 @@
  * Increments the active count for all the DAIs and components attached to a PCM
  * runtime. Should typically be called when a stream is opened.
  *
- * Must be called with the rtd->pcm_mutex being held
+ * Must be called with the rtd->card->pcm_mutex being held
  */
 void snd_soc_runtime_activate(struct snd_soc_pcm_runtime *rtd, int stream)
 {
 	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
 	int i;
 
-	lockdep_assert_held(&rtd->pcm_mutex);
+	lockdep_assert_held(&rtd->card->pcm_mutex);
 
 	if (stream == SNDRV_PCM_STREAM_PLAYBACK) {
 		cpu_dai->playback_active++;
@@ -89,14 +89,14 @@
  * Decrements the active count for all the DAIs and components attached to a PCM
  * runtime. Should typically be called when a stream is closed.
  *
- * Must be called with the rtd->pcm_mutex being held
+ * Must be called with the rtd->card->pcm_mutex being held
  */
 void snd_soc_runtime_deactivate(struct snd_soc_pcm_runtime *rtd, int stream)
 {
 	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
 	int i;
 
-	lockdep_assert_held(&rtd->pcm_mutex);
+	lockdep_assert_held(&rtd->card->pcm_mutex);
 
 	if (stream == SNDRV_PCM_STREAM_PLAYBACK) {
 		cpu_dai->playback_active--;
@@ -491,7 +491,7 @@
 		pm_runtime_get_sync(component->dev);
 	}
 
-	mutex_lock_nested(&rtd->pcm_mutex, rtd->pcm_subclass);
+	mutex_lock_nested(&rtd->card->pcm_mutex, rtd->card->pcm_subclass);
 
 	/* startup the audio subsystem */
 	if (cpu_dai->driver->ops->startup) {
@@ -609,7 +609,7 @@
 
 	snd_soc_runtime_activate(rtd, substream->stream);
 
-	mutex_unlock(&rtd->pcm_mutex);
+	mutex_unlock(&rtd->card->pcm_mutex);
 	return 0;
 
 config_err:
@@ -632,7 +632,7 @@
 	if (cpu_dai->driver->ops->shutdown)
 		cpu_dai->driver->ops->shutdown(substream, cpu_dai);
 out:
-	mutex_unlock(&rtd->pcm_mutex);
+	mutex_unlock(&rtd->card->pcm_mutex);
 
 	for_each_rtdcom(rtd, rtdcom) {
 		component = rtdcom->component;
@@ -662,7 +662,7 @@
 			container_of(work, struct snd_soc_pcm_runtime, delayed_work.work);
 	struct snd_soc_dai *codec_dai = rtd->codec_dais[0];
 
-	mutex_lock_nested(&rtd->pcm_mutex, rtd->pcm_subclass);
+	mutex_lock_nested(&rtd->card->pcm_mutex, rtd->card->pcm_subclass);
 
 	dev_dbg(rtd->dev, "ASoC: pop wq checking: %s status: %s waiting: %s\n",
 		 codec_dai->driver->playback.stream_name,
@@ -676,7 +676,7 @@
 					  SND_SOC_DAPM_STREAM_STOP);
 	}
 
-	mutex_unlock(&rtd->pcm_mutex);
+	mutex_unlock(&rtd->card->pcm_mutex);
 }
 
 /*
@@ -693,7 +693,7 @@
 	struct snd_soc_dai *codec_dai;
 	int i;
 
-	mutex_lock_nested(&rtd->pcm_mutex, rtd->pcm_subclass);
+	mutex_lock_nested(&rtd->card->pcm_mutex, rtd->card->pcm_subclass);
 
 	snd_soc_runtime_deactivate(rtd, substream->stream);
 
@@ -742,7 +742,7 @@
 					  SND_SOC_DAPM_STREAM_STOP);
 	}
 
-	mutex_unlock(&rtd->pcm_mutex);
+	mutex_unlock(&rtd->card->pcm_mutex);
 
 	for_each_rtdcom(rtd, rtdcom) {
 		component = rtdcom->component;
@@ -775,7 +775,7 @@
 	struct snd_soc_dai *codec_dai;
 	int i, ret = 0;
 
-	mutex_lock_nested(&rtd->pcm_mutex, rtd->pcm_subclass);
+	mutex_lock_nested(&rtd->card->pcm_mutex, rtd->card->pcm_subclass);
 
 	if (rtd->dai_link->ops->prepare) {
 		ret = rtd->dai_link->ops->prepare(substream);
@@ -840,7 +840,7 @@
 	snd_soc_dai_digital_mute(cpu_dai, 0, substream->stream);
 
 out:
-	mutex_unlock(&rtd->pcm_mutex);
+	mutex_unlock(&rtd->card->pcm_mutex);
 	return ret;
 }
 
@@ -922,7 +922,7 @@
 	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
 	int i, ret = 0;
 
-	mutex_lock_nested(&rtd->pcm_mutex, rtd->pcm_subclass);
+	mutex_lock_nested(&rtd->card->pcm_mutex, rtd->card->pcm_subclass);
 	if (rtd->dai_link->ops->hw_params) {
 		ret = rtd->dai_link->ops->hw_params(substream, params);
 		if (ret < 0) {
@@ -1008,7 +1008,7 @@
         if (ret)
 		goto component_err;
 out:
-	mutex_unlock(&rtd->pcm_mutex);
+	mutex_unlock(&rtd->card->pcm_mutex);
 	return ret;
 
 component_err:
@@ -1031,7 +1031,7 @@
 	if (rtd->dai_link->ops->hw_free)
 		rtd->dai_link->ops->hw_free(substream);
 
-	mutex_unlock(&rtd->pcm_mutex);
+	mutex_unlock(&rtd->card->pcm_mutex);
 	return ret;
 }
 
@@ -1046,7 +1046,7 @@
 	bool playback = substream->stream == SNDRV_PCM_STREAM_PLAYBACK;
 	int i;
 
-	mutex_lock_nested(&rtd->pcm_mutex, rtd->pcm_subclass);
+	mutex_lock_nested(&rtd->card->pcm_mutex, rtd->card->pcm_subclass);
 
 	/* clear the corresponding DAIs parameters when going to be inactive */
 	if (cpu_dai->active == 1) {
@@ -1089,11 +1089,57 @@
 	if (cpu_dai->driver->ops->hw_free)
 		cpu_dai->driver->ops->hw_free(substream, cpu_dai);
 
-	mutex_unlock(&rtd->pcm_mutex);
+	mutex_unlock(&rtd->card->pcm_mutex);
 	return 0;
 }
 
-static int soc_pcm_trigger(struct snd_pcm_substream *substream, int cmd)
+static int soc_pcm_trigger_start(struct snd_pcm_substream *substream, int cmd)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_component *component;
+	struct snd_soc_rtdcom_list *rtdcom;
+	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
+	struct snd_soc_dai *codec_dai;
+	int i, ret;
+
+	if (rtd->dai_link->ops->trigger) {
+		ret = rtd->dai_link->ops->trigger(substream, cmd);
+		if (ret < 0)
+			return ret;
+	}
+
+	for_each_rtdcom(rtd, rtdcom) {
+		component = rtdcom->component;
+
+		if (!component->driver->ops ||
+		    !component->driver->ops->trigger)
+			continue;
+
+		ret = component->driver->ops->trigger(substream, cmd);
+		if (ret < 0)
+			return ret;
+	}
+
+	if (cpu_dai->driver->ops->trigger) {
+		ret = cpu_dai->driver->ops->trigger(substream, cmd, cpu_dai);
+		if (ret < 0)
+			return ret;
+	}
+
+	for (i = 0; i < rtd->num_codecs; i++) {
+		codec_dai = rtd->codec_dais[i];
+		if (codec_dai->driver->ops->trigger) {
+			ret = codec_dai->driver->ops->trigger(substream,
+							      cmd, codec_dai);
+			if (ret < 0)
+				return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int soc_pcm_trigger_stop(struct snd_pcm_substream *substream, int cmd)
 {
 	struct snd_soc_pcm_runtime *rtd = substream->private_data;
 	struct snd_soc_component *component;
@@ -1112,6 +1158,12 @@
 		}
 	}
 
+	if (cpu_dai->driver->ops->trigger) {
+		ret = cpu_dai->driver->ops->trigger(substream, cmd, cpu_dai);
+		if (ret < 0)
+			return ret;
+	}
+
 	for_each_rtdcom(rtd, rtdcom) {
 		component = rtdcom->component;
 
@@ -1124,12 +1176,6 @@
 			return ret;
 	}
 
-	if (cpu_dai->driver->ops->trigger) {
-		ret = cpu_dai->driver->ops->trigger(substream, cmd, cpu_dai);
-		if (ret < 0)
-			return ret;
-	}
-
 	if (rtd->dai_link->ops->trigger) {
 		ret = rtd->dai_link->ops->trigger(substream, cmd);
 		if (ret < 0)
@@ -1139,6 +1185,28 @@
 	return 0;
 }
 
+static int soc_pcm_trigger(struct snd_pcm_substream *substream, int cmd)
+{
+	int ret;
+
+	switch (cmd) {
+	case SNDRV_PCM_TRIGGER_START:
+	case SNDRV_PCM_TRIGGER_RESUME:
+	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
+		ret = soc_pcm_trigger_start(substream, cmd);
+		break;
+	case SNDRV_PCM_TRIGGER_STOP:
+	case SNDRV_PCM_TRIGGER_SUSPEND:
+	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
+		ret = soc_pcm_trigger_stop(substream, cmd);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return ret;
+}
+
 static int soc_pcm_bespoke_trigger(struct snd_pcm_substream *substream,
 				   int cmd)
 {
@@ -1211,7 +1279,7 @@
 	}
 	delay += codec_delay;
 
-	runtime->delay = delay;
+	runtime->delay += delay;
 
 	return offset;
 }
diff -Naur --no-dereference a/sound/soc/ti/ams-delta.c b/sound/soc/ti/ams-delta.c
--- a/sound/soc/ti/ams-delta.c	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/ams-delta.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,594 @@
+/*
+ * ams-delta.c  --  SoC audio for Amstrad E3 (Delta) videophone
+ *
+ * Copyright (C) 2009 Janusz Krzysztofik <jkrzyszt@tis.icnet.pl>
+ *
+ * Initially based on sound/soc/omap/osk5912.x
+ * Copyright (C) 2008 Mistral Solutions
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA
+ *
+ */
+
+#include <linux/gpio/consumer.h>
+#include <linux/spinlock.h>
+#include <linux/tty.h>
+#include <linux/module.h>
+
+#include <sound/soc.h>
+#include <sound/jack.h>
+
+#include <asm/mach-types.h>
+
+#include <linux/platform_data/asoc-ti-mcbsp.h>
+
+#include "omap-mcbsp.h"
+#include "../codecs/cx20442.h"
+
+/* Board specific DAPM widgets */
+static const struct snd_soc_dapm_widget ams_delta_dapm_widgets[] = {
+	/* Handset */
+	SND_SOC_DAPM_MIC("Mouthpiece", NULL),
+	SND_SOC_DAPM_HP("Earpiece", NULL),
+	/* Handsfree/Speakerphone */
+	SND_SOC_DAPM_MIC("Microphone", NULL),
+	SND_SOC_DAPM_SPK("Speaker", NULL),
+};
+
+/* How they are connected to codec pins */
+static const struct snd_soc_dapm_route ams_delta_audio_map[] = {
+	{"TELIN", NULL, "Mouthpiece"},
+	{"Earpiece", NULL, "TELOUT"},
+
+	{"MIC", NULL, "Microphone"},
+	{"Speaker", NULL, "SPKOUT"},
+};
+
+/*
+ * Controls, functional after the modem line discipline is activated.
+ */
+
+/* Virtual switch: audio input/output constellations */
+static const char *ams_delta_audio_mode[] =
+	{"Mixed", "Handset", "Handsfree", "Speakerphone"};
+
+/* Selection <-> pin translation */
+#define AMS_DELTA_MOUTHPIECE	0
+#define AMS_DELTA_EARPIECE	1
+#define AMS_DELTA_MICROPHONE	2
+#define AMS_DELTA_SPEAKER	3
+#define AMS_DELTA_AGC		4
+
+#define AMS_DELTA_MIXED		((1 << AMS_DELTA_EARPIECE) | \
+						(1 << AMS_DELTA_MICROPHONE))
+#define AMS_DELTA_HANDSET	((1 << AMS_DELTA_MOUTHPIECE) | \
+						(1 << AMS_DELTA_EARPIECE))
+#define AMS_DELTA_HANDSFREE	((1 << AMS_DELTA_MICROPHONE) | \
+						(1 << AMS_DELTA_SPEAKER))
+#define AMS_DELTA_SPEAKERPHONE	(AMS_DELTA_HANDSFREE | (1 << AMS_DELTA_AGC))
+
+static const unsigned short ams_delta_audio_mode_pins[] = {
+	AMS_DELTA_MIXED,
+	AMS_DELTA_HANDSET,
+	AMS_DELTA_HANDSFREE,
+	AMS_DELTA_SPEAKERPHONE,
+};
+
+static unsigned short ams_delta_audio_agc;
+
+/*
+ * Used for passing a codec structure pointer
+ * from the board initialization code to the tty line discipline.
+ */
+static struct snd_soc_component *cx20442_codec;
+
+static int ams_delta_set_audio_mode(struct snd_kcontrol *kcontrol,
+					struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_card *card = snd_kcontrol_chip(kcontrol);
+	struct snd_soc_dapm_context *dapm = &card->dapm;
+	struct soc_enum *control = (struct soc_enum *)kcontrol->private_value;
+	unsigned short pins;
+	int pin, changed = 0;
+
+	/* Refuse any mode changes if we are not able to control the codec. */
+	if (!cx20442_codec->card->pop_time)
+		return -EUNATCH;
+
+	if (ucontrol->value.enumerated.item[0] >= control->items)
+		return -EINVAL;
+
+	snd_soc_dapm_mutex_lock(dapm);
+
+	/* Translate selection to bitmap */
+	pins = ams_delta_audio_mode_pins[ucontrol->value.enumerated.item[0]];
+
+	/* Setup pins after corresponding bits if changed */
+	pin = !!(pins & (1 << AMS_DELTA_MOUTHPIECE));
+
+	if (pin != snd_soc_dapm_get_pin_status(dapm, "Mouthpiece")) {
+		changed = 1;
+		if (pin)
+			snd_soc_dapm_enable_pin_unlocked(dapm, "Mouthpiece");
+		else
+			snd_soc_dapm_disable_pin_unlocked(dapm, "Mouthpiece");
+	}
+	pin = !!(pins & (1 << AMS_DELTA_EARPIECE));
+	if (pin != snd_soc_dapm_get_pin_status(dapm, "Earpiece")) {
+		changed = 1;
+		if (pin)
+			snd_soc_dapm_enable_pin_unlocked(dapm, "Earpiece");
+		else
+			snd_soc_dapm_disable_pin_unlocked(dapm, "Earpiece");
+	}
+	pin = !!(pins & (1 << AMS_DELTA_MICROPHONE));
+	if (pin != snd_soc_dapm_get_pin_status(dapm, "Microphone")) {
+		changed = 1;
+		if (pin)
+			snd_soc_dapm_enable_pin_unlocked(dapm, "Microphone");
+		else
+			snd_soc_dapm_disable_pin_unlocked(dapm, "Microphone");
+	}
+	pin = !!(pins & (1 << AMS_DELTA_SPEAKER));
+	if (pin != snd_soc_dapm_get_pin_status(dapm, "Speaker")) {
+		changed = 1;
+		if (pin)
+			snd_soc_dapm_enable_pin_unlocked(dapm, "Speaker");
+		else
+			snd_soc_dapm_disable_pin_unlocked(dapm, "Speaker");
+	}
+	pin = !!(pins & (1 << AMS_DELTA_AGC));
+	if (pin != ams_delta_audio_agc) {
+		ams_delta_audio_agc = pin;
+		changed = 1;
+		if (pin)
+			snd_soc_dapm_enable_pin_unlocked(dapm, "AGCIN");
+		else
+			snd_soc_dapm_disable_pin_unlocked(dapm, "AGCIN");
+	}
+
+	if (changed)
+		snd_soc_dapm_sync_unlocked(dapm);
+
+	snd_soc_dapm_mutex_unlock(dapm);
+
+	return changed;
+}
+
+static int ams_delta_get_audio_mode(struct snd_kcontrol *kcontrol,
+					struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_card *card = snd_kcontrol_chip(kcontrol);
+	struct snd_soc_dapm_context *dapm = &card->dapm;
+	unsigned short pins, mode;
+
+	pins = ((snd_soc_dapm_get_pin_status(dapm, "Mouthpiece") <<
+							AMS_DELTA_MOUTHPIECE) |
+			(snd_soc_dapm_get_pin_status(dapm, "Earpiece") <<
+							AMS_DELTA_EARPIECE));
+	if (pins)
+		pins |= (snd_soc_dapm_get_pin_status(dapm, "Microphone") <<
+							AMS_DELTA_MICROPHONE);
+	else
+		pins = ((snd_soc_dapm_get_pin_status(dapm, "Microphone") <<
+							AMS_DELTA_MICROPHONE) |
+			(snd_soc_dapm_get_pin_status(dapm, "Speaker") <<
+							AMS_DELTA_SPEAKER) |
+			(ams_delta_audio_agc << AMS_DELTA_AGC));
+
+	for (mode = 0; mode < ARRAY_SIZE(ams_delta_audio_mode); mode++)
+		if (pins == ams_delta_audio_mode_pins[mode])
+			break;
+
+	if (mode >= ARRAY_SIZE(ams_delta_audio_mode))
+		return -EINVAL;
+
+	ucontrol->value.enumerated.item[0] = mode;
+
+	return 0;
+}
+
+static const SOC_ENUM_SINGLE_EXT_DECL(ams_delta_audio_enum,
+				      ams_delta_audio_mode);
+
+static const struct snd_kcontrol_new ams_delta_audio_controls[] = {
+	SOC_ENUM_EXT("Audio Mode", ams_delta_audio_enum,
+			ams_delta_get_audio_mode, ams_delta_set_audio_mode),
+};
+
+/* Hook switch */
+static struct snd_soc_jack ams_delta_hook_switch;
+static struct snd_soc_jack_gpio ams_delta_hook_switch_gpios[] = {
+	{
+		.name = "hook_switch",
+		.report = SND_JACK_HEADSET,
+		.invert = 1,
+		.debounce_time = 150,
+	}
+};
+
+/* After we are able to control the codec over the modem,
+ * the hook switch can be used for dynamic DAPM reconfiguration. */
+static struct snd_soc_jack_pin ams_delta_hook_switch_pins[] = {
+	/* Handset */
+	{
+		.pin = "Mouthpiece",
+		.mask = SND_JACK_MICROPHONE,
+	},
+	{
+		.pin = "Earpiece",
+		.mask = SND_JACK_HEADPHONE,
+	},
+	/* Handsfree */
+	{
+		.pin = "Microphone",
+		.mask = SND_JACK_MICROPHONE,
+		.invert = 1,
+	},
+	{
+		.pin = "Speaker",
+		.mask = SND_JACK_HEADPHONE,
+		.invert = 1,
+	},
+};
+
+
+/*
+ * Modem line discipline, required for making above controls functional.
+ * Activated from userspace with ldattach, possibly invoked from udev rule.
+ */
+
+/* To actually apply any modem controlled configuration changes to the codec,
+ * we must connect codec DAI pins to the modem for a moment.  Be careful not
+ * to interfere with our digital mute function that shares the same hardware. */
+static struct timer_list cx81801_timer;
+static bool cx81801_cmd_pending;
+static bool ams_delta_muted;
+static DEFINE_SPINLOCK(ams_delta_lock);
+static struct gpio_desc *gpiod_modem_codec;
+
+static void cx81801_timeout(struct timer_list *unused)
+{
+	int muted;
+
+	spin_lock(&ams_delta_lock);
+	cx81801_cmd_pending = 0;
+	muted = ams_delta_muted;
+	spin_unlock(&ams_delta_lock);
+
+	/* Reconnect the codec DAI back from the modem to the CPU DAI
+	 * only if digital mute still off */
+	if (!muted)
+		gpiod_set_value(gpiod_modem_codec, 0);
+}
+
+/* Line discipline .open() */
+static int cx81801_open(struct tty_struct *tty)
+{
+	int ret;
+
+	if (!cx20442_codec)
+		return -ENODEV;
+
+	/*
+	 * Pass the codec structure pointer for use by other ldisc callbacks,
+	 * both the card and the codec specific parts.
+	 */
+	tty->disc_data = cx20442_codec;
+
+	ret = v253_ops.open(tty);
+
+	if (ret < 0)
+		tty->disc_data = NULL;
+
+	return ret;
+}
+
+/* Line discipline .close() */
+static void cx81801_close(struct tty_struct *tty)
+{
+	struct snd_soc_component *component = tty->disc_data;
+	struct snd_soc_dapm_context *dapm = &component->card->dapm;
+
+	del_timer_sync(&cx81801_timer);
+
+	/* Prevent the hook switch from further changing the DAPM pins */
+	INIT_LIST_HEAD(&ams_delta_hook_switch.pins);
+
+	if (!component)
+		return;
+
+	v253_ops.close(tty);
+
+	/* Revert back to default audio input/output constellation */
+	snd_soc_dapm_mutex_lock(dapm);
+
+	snd_soc_dapm_disable_pin_unlocked(dapm, "Mouthpiece");
+	snd_soc_dapm_enable_pin_unlocked(dapm, "Earpiece");
+	snd_soc_dapm_enable_pin_unlocked(dapm, "Microphone");
+	snd_soc_dapm_disable_pin_unlocked(dapm, "Speaker");
+	snd_soc_dapm_disable_pin_unlocked(dapm, "AGCIN");
+
+	snd_soc_dapm_sync_unlocked(dapm);
+
+	snd_soc_dapm_mutex_unlock(dapm);
+}
+
+/* Line discipline .hangup() */
+static int cx81801_hangup(struct tty_struct *tty)
+{
+	cx81801_close(tty);
+	return 0;
+}
+
+/* Line discipline .receive_buf() */
+static void cx81801_receive(struct tty_struct *tty,
+				const unsigned char *cp, char *fp, int count)
+{
+	struct snd_soc_component *component = tty->disc_data;
+	const unsigned char *c;
+	int apply, ret;
+
+	if (!component)
+		return;
+
+	if (!component->card->pop_time) {
+		/* First modem response, complete setup procedure */
+
+		/* Initialize timer used for config pulse generation */
+		timer_setup(&cx81801_timer, cx81801_timeout, 0);
+
+		v253_ops.receive_buf(tty, cp, fp, count);
+
+		/* Link hook switch to DAPM pins */
+		ret = snd_soc_jack_add_pins(&ams_delta_hook_switch,
+					ARRAY_SIZE(ams_delta_hook_switch_pins),
+					ams_delta_hook_switch_pins);
+		if (ret)
+			dev_warn(component->dev,
+				"Failed to link hook switch to DAPM pins, "
+				"will continue with hook switch unlinked.\n");
+
+		return;
+	}
+
+	v253_ops.receive_buf(tty, cp, fp, count);
+
+	for (c = &cp[count - 1]; c >= cp; c--) {
+		if (*c != '\r')
+			continue;
+		/* Complete modem response received, apply config to codec */
+
+		spin_lock_bh(&ams_delta_lock);
+		mod_timer(&cx81801_timer, jiffies + msecs_to_jiffies(150));
+		apply = !ams_delta_muted && !cx81801_cmd_pending;
+		cx81801_cmd_pending = 1;
+		spin_unlock_bh(&ams_delta_lock);
+
+		/* Apply config pulse by connecting the codec to the modem
+		 * if not already done */
+		if (apply)
+			gpiod_set_value(gpiod_modem_codec, 1);
+		break;
+	}
+}
+
+/* Line discipline .write_wakeup() */
+static void cx81801_wakeup(struct tty_struct *tty)
+{
+	v253_ops.write_wakeup(tty);
+}
+
+static struct tty_ldisc_ops cx81801_ops = {
+	.magic = TTY_LDISC_MAGIC,
+	.name = "cx81801",
+	.owner = THIS_MODULE,
+	.open = cx81801_open,
+	.close = cx81801_close,
+	.hangup = cx81801_hangup,
+	.receive_buf = cx81801_receive,
+	.write_wakeup = cx81801_wakeup,
+};
+
+
+/*
+ * Even if not very useful, the sound card can still work without any of the
+ * above functonality activated.  You can still control its audio input/output
+ * constellation and speakerphone gain from userspace by issuing AT commands
+ * over the modem port.
+ */
+
+static struct snd_soc_ops ams_delta_ops;
+
+
+/* Digital mute implemented using modem/CPU multiplexer.
+ * Shares hardware with codec config pulse generation */
+static bool ams_delta_muted = 1;
+
+static int ams_delta_digital_mute(struct snd_soc_dai *dai, int mute)
+{
+	int apply;
+
+	if (ams_delta_muted == mute)
+		return 0;
+
+	spin_lock_bh(&ams_delta_lock);
+	ams_delta_muted = mute;
+	apply = !cx81801_cmd_pending;
+	spin_unlock_bh(&ams_delta_lock);
+
+	if (apply)
+		gpiod_set_value(gpiod_modem_codec, !!mute);
+	return 0;
+}
+
+/* Our codec DAI probably doesn't have its own .ops structure */
+static const struct snd_soc_dai_ops ams_delta_dai_ops = {
+	.digital_mute = ams_delta_digital_mute,
+};
+
+/* Will be used if the codec ever has its own digital_mute function */
+static int ams_delta_startup(struct snd_pcm_substream *substream)
+{
+	return ams_delta_digital_mute(NULL, 0);
+}
+
+static void ams_delta_shutdown(struct snd_pcm_substream *substream)
+{
+	ams_delta_digital_mute(NULL, 1);
+}
+
+
+/*
+ * Card initialization
+ */
+
+static int ams_delta_cx20442_init(struct snd_soc_pcm_runtime *rtd)
+{
+	struct snd_soc_dai *codec_dai = rtd->codec_dai;
+	struct snd_soc_card *card = rtd->card;
+	struct snd_soc_dapm_context *dapm = &card->dapm;
+	int ret;
+	/* Codec is ready, now add/activate board specific controls */
+
+	/* Store a pointer to the codec structure for tty ldisc use */
+	cx20442_codec = rtd->codec_dai->component;
+
+	/* Add hook switch - can be used to control the codec from userspace
+	 * even if line discipline fails */
+	ret = snd_soc_card_jack_new(card, "hook_switch", SND_JACK_HEADSET,
+				    &ams_delta_hook_switch, NULL, 0);
+	if (ret)
+		dev_warn(card->dev,
+				"Failed to allocate resources for hook switch, "
+				"will continue without one.\n");
+	else {
+		ret = snd_soc_jack_add_gpiods(card->dev, &ams_delta_hook_switch,
+					ARRAY_SIZE(ams_delta_hook_switch_gpios),
+					ams_delta_hook_switch_gpios);
+		if (ret)
+			dev_warn(card->dev,
+				"Failed to set up hook switch GPIO line, "
+				"will continue with hook switch inactive.\n");
+	}
+
+	gpiod_modem_codec = devm_gpiod_get(card->dev, "modem_codec",
+					   GPIOD_OUT_HIGH);
+	if (IS_ERR(gpiod_modem_codec)) {
+		dev_warn(card->dev, "Failed to obtain modem_codec GPIO\n");
+		return 0;
+	}
+
+	/* Set up digital mute if not provided by the codec */
+	if (!codec_dai->driver->ops) {
+		codec_dai->driver->ops = &ams_delta_dai_ops;
+	} else {
+		ams_delta_ops.startup = ams_delta_startup;
+		ams_delta_ops.shutdown = ams_delta_shutdown;
+	}
+
+	/* Register optional line discipline for over the modem control */
+	ret = tty_register_ldisc(N_V253, &cx81801_ops);
+	if (ret) {
+		dev_warn(card->dev,
+				"Failed to register line discipline, "
+				"will continue without any controls.\n");
+		return 0;
+	}
+
+	/* Set up initial pin constellation */
+	snd_soc_dapm_disable_pin(dapm, "Mouthpiece");
+	snd_soc_dapm_disable_pin(dapm, "Speaker");
+	snd_soc_dapm_disable_pin(dapm, "AGCIN");
+	snd_soc_dapm_disable_pin(dapm, "AGCOUT");
+
+	return 0;
+}
+
+/* DAI glue - connects codec <--> CPU */
+static struct snd_soc_dai_link ams_delta_dai_link = {
+	.name = "CX20442",
+	.stream_name = "CX20442",
+	.cpu_dai_name = "omap-mcbsp.1",
+	.codec_dai_name = "cx20442-voice",
+	.init = ams_delta_cx20442_init,
+	.platform_name = "omap-mcbsp.1",
+	.codec_name = "cx20442-codec",
+	.ops = &ams_delta_ops,
+	.dai_fmt = SND_SOC_DAIFMT_DSP_A | SND_SOC_DAIFMT_NB_NF |
+		   SND_SOC_DAIFMT_CBM_CFM,
+};
+
+/* Audio card driver */
+static struct snd_soc_card ams_delta_audio_card = {
+	.name = "AMS_DELTA",
+	.owner = THIS_MODULE,
+	.dai_link = &ams_delta_dai_link,
+	.num_links = 1,
+
+	.controls = ams_delta_audio_controls,
+	.num_controls = ARRAY_SIZE(ams_delta_audio_controls),
+	.dapm_widgets = ams_delta_dapm_widgets,
+	.num_dapm_widgets = ARRAY_SIZE(ams_delta_dapm_widgets),
+	.dapm_routes = ams_delta_audio_map,
+	.num_dapm_routes = ARRAY_SIZE(ams_delta_audio_map),
+};
+
+/* Module init/exit */
+static int ams_delta_probe(struct platform_device *pdev)
+{
+	struct snd_soc_card *card = &ams_delta_audio_card;
+	int ret;
+
+	card->dev = &pdev->dev;
+
+	ret = snd_soc_register_card(card);
+	if (ret) {
+		dev_err(&pdev->dev, "snd_soc_register_card failed (%d)\n", ret);
+		card->dev = NULL;
+		return ret;
+	}
+	return 0;
+}
+
+static int ams_delta_remove(struct platform_device *pdev)
+{
+	struct snd_soc_card *card = platform_get_drvdata(pdev);
+
+	if (tty_unregister_ldisc(N_V253) != 0)
+		dev_warn(&pdev->dev,
+			"failed to unregister V253 line discipline\n");
+
+	snd_soc_unregister_card(card);
+	card->dev = NULL;
+	return 0;
+}
+
+#define DRV_NAME "ams-delta-audio"
+
+static struct platform_driver ams_delta_driver = {
+	.driver = {
+		.name = DRV_NAME,
+	},
+	.probe = ams_delta_probe,
+	.remove = ams_delta_remove,
+};
+
+module_platform_driver(ams_delta_driver);
+
+MODULE_AUTHOR("Janusz Krzysztofik <jkrzyszt@tis.icnet.pl>");
+MODULE_DESCRIPTION("ALSA SoC driver for Amstrad E3 (Delta) videophone");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:" DRV_NAME);
diff -Naur --no-dereference a/sound/soc/ti/davinci-evm.c b/sound/soc/ti/davinci-evm.c
--- a/sound/soc/ti/davinci-evm.c	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/davinci-evm.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,511 @@
+/*
+ * ASoC driver for TI DAVINCI EVM platform
+ *
+ * Author:      Vladimir Barinov, <vbarinov@embeddedalley.com>
+ * Copyright:   (C) 2007 MontaVista Software, Inc., <source@mvista.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/timer.h>
+#include <linux/interrupt.h>
+#include <linux/platform_device.h>
+#include <linux/i2c.h>
+#include <linux/of_platform.h>
+#include <linux/clk.h>
+#include <sound/core.h>
+#include <sound/pcm.h>
+#include <sound/soc.h>
+
+#include <asm/dma.h>
+#include <asm/mach-types.h>
+
+struct snd_soc_card_drvdata_davinci {
+	struct clk *mclk;
+	unsigned sysclk;
+};
+
+static int evm_startup(struct snd_pcm_substream *substream)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_card *soc_card = rtd->card;
+	struct snd_soc_card_drvdata_davinci *drvdata =
+		snd_soc_card_get_drvdata(soc_card);
+
+	if (drvdata->mclk)
+		return clk_prepare_enable(drvdata->mclk);
+
+	return 0;
+}
+
+static void evm_shutdown(struct snd_pcm_substream *substream)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_card *soc_card = rtd->card;
+	struct snd_soc_card_drvdata_davinci *drvdata =
+		snd_soc_card_get_drvdata(soc_card);
+
+	if (drvdata->mclk)
+		clk_disable_unprepare(drvdata->mclk);
+}
+
+static int evm_hw_params(struct snd_pcm_substream *substream,
+			 struct snd_pcm_hw_params *params)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_dai *codec_dai = rtd->codec_dai;
+	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
+	struct snd_soc_card *soc_card = rtd->card;
+	int ret = 0;
+	unsigned sysclk = ((struct snd_soc_card_drvdata_davinci *)
+			   snd_soc_card_get_drvdata(soc_card))->sysclk;
+
+	/* set the codec system clock */
+	ret = snd_soc_dai_set_sysclk(codec_dai, 0, sysclk, SND_SOC_CLOCK_OUT);
+	if (ret < 0)
+		return ret;
+
+	/* set the CPU system clock */
+	ret = snd_soc_dai_set_sysclk(cpu_dai, 0, sysclk, SND_SOC_CLOCK_OUT);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static struct snd_soc_ops evm_ops = {
+	.startup = evm_startup,
+	.shutdown = evm_shutdown,
+	.hw_params = evm_hw_params,
+};
+
+/* davinci-evm machine dapm widgets */
+static const struct snd_soc_dapm_widget aic3x_dapm_widgets[] = {
+	SND_SOC_DAPM_HP("Headphone Jack", NULL),
+	SND_SOC_DAPM_LINE("Line Out", NULL),
+	SND_SOC_DAPM_MIC("Mic Jack", NULL),
+	SND_SOC_DAPM_LINE("Line In", NULL),
+};
+
+/* davinci-evm machine audio_mapnections to the codec pins */
+static const struct snd_soc_dapm_route audio_map[] = {
+	/* Headphone connected to HPLOUT, HPROUT */
+	{"Headphone Jack", NULL, "HPLOUT"},
+	{"Headphone Jack", NULL, "HPROUT"},
+
+	/* Line Out connected to LLOUT, RLOUT */
+	{"Line Out", NULL, "LLOUT"},
+	{"Line Out", NULL, "RLOUT"},
+
+	/* Mic connected to (MIC3L | MIC3R) */
+	{"MIC3L", NULL, "Mic Bias"},
+	{"MIC3R", NULL, "Mic Bias"},
+	{"Mic Bias", NULL, "Mic Jack"},
+
+	/* Line In connected to (LINE1L | LINE2L), (LINE1R | LINE2R) */
+	{"LINE1L", NULL, "Line In"},
+	{"LINE2L", NULL, "Line In"},
+	{"LINE1R", NULL, "Line In"},
+	{"LINE2R", NULL, "Line In"},
+};
+
+/* Logic for a aic3x as connected on a davinci-evm */
+static int evm_aic3x_init(struct snd_soc_pcm_runtime *rtd)
+{
+	struct snd_soc_card *card = rtd->card;
+	struct device_node *np = card->dev->of_node;
+	int ret;
+
+	/* Add davinci-evm specific widgets */
+	snd_soc_dapm_new_controls(&card->dapm, aic3x_dapm_widgets,
+				  ARRAY_SIZE(aic3x_dapm_widgets));
+
+	if (np) {
+		ret = snd_soc_of_parse_audio_routing(card, "ti,audio-routing");
+		if (ret)
+			return ret;
+	} else {
+		/* Set up davinci-evm specific audio path audio_map */
+		snd_soc_dapm_add_routes(&card->dapm, audio_map,
+					ARRAY_SIZE(audio_map));
+	}
+
+	/* not connected */
+	snd_soc_dapm_nc_pin(&card->dapm, "MONO_LOUT");
+	snd_soc_dapm_nc_pin(&card->dapm, "HPLCOM");
+	snd_soc_dapm_nc_pin(&card->dapm, "HPRCOM");
+
+	return 0;
+}
+
+/* davinci-evm digital audio interface glue - connects codec <--> CPU */
+static struct snd_soc_dai_link dm6446_evm_dai = {
+	.name = "TLV320AIC3X",
+	.stream_name = "AIC3X",
+	.cpu_dai_name = "davinci-mcbsp",
+	.codec_dai_name = "tlv320aic3x-hifi",
+	.codec_name = "tlv320aic3x-codec.1-001b",
+	.platform_name = "davinci-mcbsp",
+	.init = evm_aic3x_init,
+	.ops = &evm_ops,
+	.dai_fmt = SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_CBM_CFM |
+		   SND_SOC_DAIFMT_IB_NF,
+};
+
+static struct snd_soc_dai_link dm355_evm_dai = {
+	.name = "TLV320AIC3X",
+	.stream_name = "AIC3X",
+	.cpu_dai_name = "davinci-mcbsp.1",
+	.codec_dai_name = "tlv320aic3x-hifi",
+	.codec_name = "tlv320aic3x-codec.1-001b",
+	.platform_name = "davinci-mcbsp.1",
+	.init = evm_aic3x_init,
+	.ops = &evm_ops,
+	.dai_fmt = SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_CBM_CFM |
+		   SND_SOC_DAIFMT_IB_NF,
+};
+
+static struct snd_soc_dai_link dm365_evm_dai = {
+#ifdef CONFIG_SND_SOC_DM365_AIC3X_CODEC
+	.name = "TLV320AIC3X",
+	.stream_name = "AIC3X",
+	.cpu_dai_name = "davinci-mcbsp",
+	.codec_dai_name = "tlv320aic3x-hifi",
+	.codec_name = "tlv320aic3x-codec.1-0018",
+	.platform_name = "davinci-mcbsp",
+	.init = evm_aic3x_init,
+	.ops = &evm_ops,
+	.dai_fmt = SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_CBM_CFM |
+		   SND_SOC_DAIFMT_IB_NF,
+#elif defined(CONFIG_SND_SOC_DM365_VOICE_CODEC)
+	.name = "Voice Codec - CQ93VC",
+	.stream_name = "CQ93",
+	.cpu_dai_name = "davinci-vcif",
+	.codec_dai_name = "cq93vc-hifi",
+	.codec_name = "cq93vc-codec",
+	.platform_name = "davinci-vcif",
+#endif
+};
+
+static struct snd_soc_dai_link dm6467_evm_dai[] = {
+	{
+		.name = "TLV320AIC3X",
+		.stream_name = "AIC3X",
+		.cpu_dai_name= "davinci-mcasp.0",
+		.codec_dai_name = "tlv320aic3x-hifi",
+		.platform_name = "davinci-mcasp.0",
+		.codec_name = "tlv320aic3x-codec.0-001a",
+		.init = evm_aic3x_init,
+		.ops = &evm_ops,
+		.dai_fmt = SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_CBM_CFM |
+			   SND_SOC_DAIFMT_IB_NF,
+	},
+	{
+		.name = "McASP",
+		.stream_name = "spdif",
+		.cpu_dai_name= "davinci-mcasp.1",
+		.codec_dai_name = "dit-hifi",
+		.codec_name = "spdif_dit",
+		.platform_name = "davinci-mcasp.1",
+		.dai_fmt = SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_CBM_CFM |
+			   SND_SOC_DAIFMT_IB_NF,
+	},
+};
+
+static struct snd_soc_dai_link da830_evm_dai = {
+	.name = "TLV320AIC3X",
+	.stream_name = "AIC3X",
+	.cpu_dai_name = "davinci-mcasp.1",
+	.codec_dai_name = "tlv320aic3x-hifi",
+	.codec_name = "tlv320aic3x-codec.1-0018",
+	.platform_name = "davinci-mcasp.1",
+	.init = evm_aic3x_init,
+	.ops = &evm_ops,
+	.dai_fmt = SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_CBM_CFM |
+		   SND_SOC_DAIFMT_IB_NF,
+};
+
+static struct snd_soc_dai_link da850_evm_dai = {
+	.name = "TLV320AIC3X",
+	.stream_name = "AIC3X",
+	.cpu_dai_name= "davinci-mcasp.0",
+	.codec_dai_name = "tlv320aic3x-hifi",
+	.codec_name = "tlv320aic3x-codec.1-0018",
+	.platform_name = "davinci-mcasp.0",
+	.init = evm_aic3x_init,
+	.ops = &evm_ops,
+	.dai_fmt = SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_CBM_CFM |
+		   SND_SOC_DAIFMT_IB_NF,
+};
+
+/* davinci dm6446 evm audio machine driver */
+/*
+ * ASP0 in DM6446 EVM is clocked by U55, as configured by
+ * board-dm644x-evm.c using GPIOs from U18.  There are six
+ * options; here we "know" we use a 48 KHz sample rate.
+ */
+static struct snd_soc_card_drvdata_davinci dm6446_snd_soc_card_drvdata = {
+	.sysclk = 12288000,
+};
+
+static struct snd_soc_card dm6446_snd_soc_card_evm = {
+	.name = "DaVinci DM6446 EVM",
+	.owner = THIS_MODULE,
+	.dai_link = &dm6446_evm_dai,
+	.num_links = 1,
+	.drvdata = &dm6446_snd_soc_card_drvdata,
+};
+
+/* davinci dm355 evm audio machine driver */
+/* ASP1 on DM355 EVM is clocked by an external oscillator */
+static struct snd_soc_card_drvdata_davinci dm355_snd_soc_card_drvdata = {
+	.sysclk = 27000000,
+};
+
+static struct snd_soc_card dm355_snd_soc_card_evm = {
+	.name = "DaVinci DM355 EVM",
+	.owner = THIS_MODULE,
+	.dai_link = &dm355_evm_dai,
+	.num_links = 1,
+	.drvdata = &dm355_snd_soc_card_drvdata,
+};
+
+/* davinci dm365 evm audio machine driver */
+static struct snd_soc_card_drvdata_davinci dm365_snd_soc_card_drvdata = {
+	.sysclk = 27000000,
+};
+
+static struct snd_soc_card dm365_snd_soc_card_evm = {
+	.name = "DaVinci DM365 EVM",
+	.owner = THIS_MODULE,
+	.dai_link = &dm365_evm_dai,
+	.num_links = 1,
+	.drvdata = &dm365_snd_soc_card_drvdata,
+};
+
+/* davinci dm6467 evm audio machine driver */
+static struct snd_soc_card_drvdata_davinci dm6467_snd_soc_card_drvdata = {
+	.sysclk = 27000000,
+};
+
+static struct snd_soc_card dm6467_snd_soc_card_evm = {
+	.name = "DaVinci DM6467 EVM",
+	.owner = THIS_MODULE,
+	.dai_link = dm6467_evm_dai,
+	.num_links = ARRAY_SIZE(dm6467_evm_dai),
+	.drvdata = &dm6467_snd_soc_card_drvdata,
+};
+
+static struct snd_soc_card_drvdata_davinci da830_snd_soc_card_drvdata = {
+	.sysclk = 24576000,
+};
+
+static struct snd_soc_card da830_snd_soc_card = {
+	.name = "DA830/OMAP-L137 EVM",
+	.owner = THIS_MODULE,
+	.dai_link = &da830_evm_dai,
+	.num_links = 1,
+	.drvdata = &da830_snd_soc_card_drvdata,
+};
+
+static struct snd_soc_card_drvdata_davinci da850_snd_soc_card_drvdata = {
+	.sysclk = 24576000,
+};
+
+static struct snd_soc_card da850_snd_soc_card = {
+	.name = "DA850/OMAP-L138 EVM",
+	.owner = THIS_MODULE,
+	.dai_link = &da850_evm_dai,
+	.num_links = 1,
+	.drvdata = &da850_snd_soc_card_drvdata,
+};
+
+#if defined(CONFIG_OF)
+
+/*
+ * The struct is used as place holder. It will be completely
+ * filled with data from dt node.
+ */
+static struct snd_soc_dai_link evm_dai_tlv320aic3x = {
+	.name		= "TLV320AIC3X",
+	.stream_name	= "AIC3X",
+	.codec_dai_name	= "tlv320aic3x-hifi",
+	.ops            = &evm_ops,
+	.init           = evm_aic3x_init,
+	.dai_fmt = SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_CBM_CFM |
+		   SND_SOC_DAIFMT_IB_NF,
+};
+
+static const struct of_device_id davinci_evm_dt_ids[] = {
+	{
+		.compatible = "ti,da830-evm-audio",
+		.data = (void *) &evm_dai_tlv320aic3x,
+	},
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, davinci_evm_dt_ids);
+
+/* davinci evm audio machine driver */
+static struct snd_soc_card evm_soc_card = {
+	.owner = THIS_MODULE,
+	.num_links = 1,
+};
+
+static int davinci_evm_probe(struct platform_device *pdev)
+{
+	struct device_node *np = pdev->dev.of_node;
+	const struct of_device_id *match;
+	struct snd_soc_dai_link *dai;
+	struct snd_soc_card_drvdata_davinci *drvdata = NULL;
+	struct clk *mclk;
+	int ret = 0;
+
+	match = of_match_device(of_match_ptr(davinci_evm_dt_ids), &pdev->dev);
+	if (!match) {
+		dev_err(&pdev->dev, "Error: No device match found\n");
+		return -ENODEV;
+	}
+
+	dai = (struct snd_soc_dai_link *) match->data;
+
+	evm_soc_card.dai_link = dai;
+
+	dai->codec_of_node = of_parse_phandle(np, "ti,audio-codec", 0);
+	if (!dai->codec_of_node)
+		return -EINVAL;
+
+	dai->cpu_of_node = of_parse_phandle(np, "ti,mcasp-controller", 0);
+	if (!dai->cpu_of_node)
+		return -EINVAL;
+
+	dai->platform_of_node = dai->cpu_of_node;
+
+	evm_soc_card.dev = &pdev->dev;
+	ret = snd_soc_of_parse_card_name(&evm_soc_card, "ti,model");
+	if (ret)
+		return ret;
+
+	mclk = devm_clk_get(&pdev->dev, "mclk");
+	if (PTR_ERR(mclk) == -EPROBE_DEFER) {
+		return -EPROBE_DEFER;
+	} else if (IS_ERR(mclk)) {
+		dev_dbg(&pdev->dev, "mclk not found.\n");
+		mclk = NULL;
+	}
+
+	drvdata = devm_kzalloc(&pdev->dev, sizeof(*drvdata), GFP_KERNEL);
+	if (!drvdata)
+		return -ENOMEM;
+
+	drvdata->mclk = mclk;
+
+	ret = of_property_read_u32(np, "ti,codec-clock-rate", &drvdata->sysclk);
+
+	if (ret < 0) {
+		if (!drvdata->mclk) {
+			dev_err(&pdev->dev,
+				"No clock or clock rate defined.\n");
+			return -EINVAL;
+		}
+		drvdata->sysclk = clk_get_rate(drvdata->mclk);
+	} else if (drvdata->mclk) {
+		unsigned int requestd_rate = drvdata->sysclk;
+		clk_set_rate(drvdata->mclk, drvdata->sysclk);
+		drvdata->sysclk = clk_get_rate(drvdata->mclk);
+		if (drvdata->sysclk != requestd_rate)
+			dev_warn(&pdev->dev,
+				 "Could not get requested rate %u using %u.\n",
+				 requestd_rate, drvdata->sysclk);
+	}
+
+	snd_soc_card_set_drvdata(&evm_soc_card, drvdata);
+	ret = devm_snd_soc_register_card(&pdev->dev, &evm_soc_card);
+
+	if (ret)
+		dev_err(&pdev->dev, "snd_soc_register_card failed (%d)\n", ret);
+
+	return ret;
+}
+
+static struct platform_driver davinci_evm_driver = {
+	.probe		= davinci_evm_probe,
+	.driver		= {
+		.name	= "davinci_evm",
+		.pm	= &snd_soc_pm_ops,
+		.of_match_table = of_match_ptr(davinci_evm_dt_ids),
+	},
+};
+#endif
+
+static struct platform_device *evm_snd_device;
+
+static int __init evm_init(void)
+{
+	struct snd_soc_card *evm_snd_dev_data;
+	int index;
+	int ret;
+
+	/*
+	 * If dtb is there, the devices will be created dynamically.
+	 * Only register platfrom driver structure.
+	 */
+#if defined(CONFIG_OF)
+	if (of_have_populated_dt())
+		return platform_driver_register(&davinci_evm_driver);
+#endif
+
+	if (machine_is_davinci_evm()) {
+		evm_snd_dev_data = &dm6446_snd_soc_card_evm;
+		index = 0;
+	} else if (machine_is_davinci_dm355_evm()) {
+		evm_snd_dev_data = &dm355_snd_soc_card_evm;
+		index = 1;
+	} else if (machine_is_davinci_dm365_evm()) {
+		evm_snd_dev_data = &dm365_snd_soc_card_evm;
+		index = 0;
+	} else if (machine_is_davinci_dm6467_evm()) {
+		evm_snd_dev_data = &dm6467_snd_soc_card_evm;
+		index = 0;
+	} else if (machine_is_davinci_da830_evm()) {
+		evm_snd_dev_data = &da830_snd_soc_card;
+		index = 1;
+	} else if (machine_is_davinci_da850_evm()) {
+		evm_snd_dev_data = &da850_snd_soc_card;
+		index = 0;
+	} else
+		return -EINVAL;
+
+	evm_snd_device = platform_device_alloc("soc-audio", index);
+	if (!evm_snd_device)
+		return -ENOMEM;
+
+	platform_set_drvdata(evm_snd_device, evm_snd_dev_data);
+	ret = platform_device_add(evm_snd_device);
+	if (ret)
+		platform_device_put(evm_snd_device);
+
+	return ret;
+}
+
+static void __exit evm_exit(void)
+{
+#if defined(CONFIG_OF)
+	if (of_have_populated_dt()) {
+		platform_driver_unregister(&davinci_evm_driver);
+		return;
+	}
+#endif
+
+	platform_device_unregister(evm_snd_device);
+}
+
+module_init(evm_init);
+module_exit(evm_exit);
+
+MODULE_AUTHOR("Vladimir Barinov");
+MODULE_DESCRIPTION("TI DAVINCI EVM ASoC driver");
+MODULE_LICENSE("GPL");
diff -Naur --no-dereference a/sound/soc/ti/davinci-i2s.c b/sound/soc/ti/davinci-i2s.c
--- a/sound/soc/ti/davinci-i2s.c	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/davinci-i2s.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,782 @@
+/*
+ * ALSA SoC I2S (McBSP) Audio Layer for TI DAVINCI processor
+ *
+ * Author:      Vladimir Barinov, <vbarinov@embeddedalley.com>
+ * Copyright:   (C) 2007 MontaVista Software, Inc., <source@mvista.com>
+ *
+ * DT support	(c) 2016 Petr Kulhavy, Barix AG <petr@barix.com>
+ *		based on davinci-mcasp.c DT support
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * TODO:
+ * on DA850 implement HW FIFOs instead of DMA into DXR and DRR registers
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/slab.h>
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/clk.h>
+#include <linux/platform_data/davinci_asp.h>
+
+#include <sound/core.h>
+#include <sound/pcm.h>
+#include <sound/pcm_params.h>
+#include <sound/initval.h>
+#include <sound/soc.h>
+#include <sound/dmaengine_pcm.h>
+
+#include "edma-pcm.h"
+#include "davinci-i2s.h"
+
+#define DRV_NAME "davinci-i2s"
+
+/*
+ * NOTE:  terminology here is confusing.
+ *
+ *  - This driver supports the "Audio Serial Port" (ASP),
+ *    found on dm6446, dm355, and other DaVinci chips.
+ *
+ *  - But it labels it a "Multi-channel Buffered Serial Port"
+ *    (McBSP) as on older chips like the dm642 ... which was
+ *    backward-compatible, possibly explaining that confusion.
+ *
+ *  - OMAP chips have a controller called McBSP, which is
+ *    incompatible with the DaVinci flavor of McBSP.
+ *
+ *  - Newer DaVinci chips have a controller called McASP,
+ *    incompatible with ASP and with either McBSP.
+ *
+ * In short:  this uses ASP to implement I2S, not McBSP.
+ * And it won't be the only DaVinci implemention of I2S.
+ */
+#define DAVINCI_MCBSP_DRR_REG	0x00
+#define DAVINCI_MCBSP_DXR_REG	0x04
+#define DAVINCI_MCBSP_SPCR_REG	0x08
+#define DAVINCI_MCBSP_RCR_REG	0x0c
+#define DAVINCI_MCBSP_XCR_REG	0x10
+#define DAVINCI_MCBSP_SRGR_REG	0x14
+#define DAVINCI_MCBSP_PCR_REG	0x24
+
+#define DAVINCI_MCBSP_SPCR_RRST		(1 << 0)
+#define DAVINCI_MCBSP_SPCR_RINTM(v)	((v) << 4)
+#define DAVINCI_MCBSP_SPCR_XRST		(1 << 16)
+#define DAVINCI_MCBSP_SPCR_XINTM(v)	((v) << 20)
+#define DAVINCI_MCBSP_SPCR_GRST		(1 << 22)
+#define DAVINCI_MCBSP_SPCR_FRST		(1 << 23)
+#define DAVINCI_MCBSP_SPCR_FREE		(1 << 25)
+
+#define DAVINCI_MCBSP_RCR_RWDLEN1(v)	((v) << 5)
+#define DAVINCI_MCBSP_RCR_RFRLEN1(v)	((v) << 8)
+#define DAVINCI_MCBSP_RCR_RDATDLY(v)	((v) << 16)
+#define DAVINCI_MCBSP_RCR_RFIG		(1 << 18)
+#define DAVINCI_MCBSP_RCR_RWDLEN2(v)	((v) << 21)
+#define DAVINCI_MCBSP_RCR_RFRLEN2(v)	((v) << 24)
+#define DAVINCI_MCBSP_RCR_RPHASE	BIT(31)
+
+#define DAVINCI_MCBSP_XCR_XWDLEN1(v)	((v) << 5)
+#define DAVINCI_MCBSP_XCR_XFRLEN1(v)	((v) << 8)
+#define DAVINCI_MCBSP_XCR_XDATDLY(v)	((v) << 16)
+#define DAVINCI_MCBSP_XCR_XFIG		(1 << 18)
+#define DAVINCI_MCBSP_XCR_XWDLEN2(v)	((v) << 21)
+#define DAVINCI_MCBSP_XCR_XFRLEN2(v)	((v) << 24)
+#define DAVINCI_MCBSP_XCR_XPHASE	BIT(31)
+
+#define DAVINCI_MCBSP_SRGR_FWID(v)	((v) << 8)
+#define DAVINCI_MCBSP_SRGR_FPER(v)	((v) << 16)
+#define DAVINCI_MCBSP_SRGR_FSGM		(1 << 28)
+#define DAVINCI_MCBSP_SRGR_CLKSM	BIT(29)
+
+#define DAVINCI_MCBSP_PCR_CLKRP		(1 << 0)
+#define DAVINCI_MCBSP_PCR_CLKXP		(1 << 1)
+#define DAVINCI_MCBSP_PCR_FSRP		(1 << 2)
+#define DAVINCI_MCBSP_PCR_FSXP		(1 << 3)
+#define DAVINCI_MCBSP_PCR_SCLKME	(1 << 7)
+#define DAVINCI_MCBSP_PCR_CLKRM		(1 << 8)
+#define DAVINCI_MCBSP_PCR_CLKXM		(1 << 9)
+#define DAVINCI_MCBSP_PCR_FSRM		(1 << 10)
+#define DAVINCI_MCBSP_PCR_FSXM		(1 << 11)
+
+enum {
+	DAVINCI_MCBSP_WORD_8 = 0,
+	DAVINCI_MCBSP_WORD_12,
+	DAVINCI_MCBSP_WORD_16,
+	DAVINCI_MCBSP_WORD_20,
+	DAVINCI_MCBSP_WORD_24,
+	DAVINCI_MCBSP_WORD_32,
+};
+
+static const unsigned char data_type[SNDRV_PCM_FORMAT_S32_LE + 1] = {
+	[SNDRV_PCM_FORMAT_S8]		= 1,
+	[SNDRV_PCM_FORMAT_S16_LE]	= 2,
+	[SNDRV_PCM_FORMAT_S32_LE]	= 4,
+};
+
+static const unsigned char asp_word_length[SNDRV_PCM_FORMAT_S32_LE + 1] = {
+	[SNDRV_PCM_FORMAT_S8]		= DAVINCI_MCBSP_WORD_8,
+	[SNDRV_PCM_FORMAT_S16_LE]	= DAVINCI_MCBSP_WORD_16,
+	[SNDRV_PCM_FORMAT_S32_LE]	= DAVINCI_MCBSP_WORD_32,
+};
+
+static const unsigned char double_fmt[SNDRV_PCM_FORMAT_S32_LE + 1] = {
+	[SNDRV_PCM_FORMAT_S8]		= SNDRV_PCM_FORMAT_S16_LE,
+	[SNDRV_PCM_FORMAT_S16_LE]	= SNDRV_PCM_FORMAT_S32_LE,
+};
+
+struct davinci_mcbsp_dev {
+	struct device *dev;
+	struct snd_dmaengine_dai_dma_data dma_data[2];
+	int dma_request[2];
+	void __iomem			*base;
+#define MOD_DSP_A	0
+#define MOD_DSP_B	1
+	int				mode;
+	u32				pcr;
+	struct clk			*clk;
+	/*
+	 * Combining both channels into 1 element will at least double the
+	 * amount of time between servicing the dma channel, increase
+	 * effiency, and reduce the chance of overrun/underrun. But,
+	 * it will result in the left & right channels being swapped.
+	 *
+	 * If relabeling the left and right channels is not possible,
+	 * you may want to let the codec know to swap them back.
+	 *
+	 * It may allow x10 the amount of time to service dma requests,
+	 * if the codec is master and is using an unnecessarily fast bit clock
+	 * (ie. tlvaic23b), independent of the sample rate. So, having an
+	 * entire frame at once means it can be serviced at the sample rate
+	 * instead of the bit clock rate.
+	 *
+	 * In the now unlikely case that an underrun still
+	 * occurs, both the left and right samples will be repeated
+	 * so that no pops are heard, and the left and right channels
+	 * won't end up being swapped because of the underrun.
+	 */
+	unsigned enable_channel_combine:1;
+
+	unsigned int fmt;
+	int clk_div;
+	int clk_input_pin;
+	bool i2s_accurate_sck;
+};
+
+static inline void davinci_mcbsp_write_reg(struct davinci_mcbsp_dev *dev,
+					   int reg, u32 val)
+{
+	__raw_writel(val, dev->base + reg);
+}
+
+static inline u32 davinci_mcbsp_read_reg(struct davinci_mcbsp_dev *dev, int reg)
+{
+	return __raw_readl(dev->base + reg);
+}
+
+static void toggle_clock(struct davinci_mcbsp_dev *dev, int playback)
+{
+	u32 m = playback ? DAVINCI_MCBSP_PCR_CLKXP : DAVINCI_MCBSP_PCR_CLKRP;
+	/* The clock needs to toggle to complete reset.
+	 * So, fake it by toggling the clk polarity.
+	 */
+	davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_PCR_REG, dev->pcr ^ m);
+	davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_PCR_REG, dev->pcr);
+}
+
+static void davinci_mcbsp_start(struct davinci_mcbsp_dev *dev,
+		struct snd_pcm_substream *substream)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_component *component = snd_soc_rtdcom_lookup(rtd, DRV_NAME);
+	int playback = (substream->stream == SNDRV_PCM_STREAM_PLAYBACK);
+	u32 spcr;
+	u32 mask = playback ? DAVINCI_MCBSP_SPCR_XRST : DAVINCI_MCBSP_SPCR_RRST;
+	spcr = davinci_mcbsp_read_reg(dev, DAVINCI_MCBSP_SPCR_REG);
+	if (spcr & mask) {
+		/* start off disabled */
+		davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_SPCR_REG,
+				spcr & ~mask);
+		toggle_clock(dev, playback);
+	}
+	if (dev->pcr & (DAVINCI_MCBSP_PCR_FSXM | DAVINCI_MCBSP_PCR_FSRM |
+			DAVINCI_MCBSP_PCR_CLKXM | DAVINCI_MCBSP_PCR_CLKRM)) {
+		/* Start the sample generator */
+		spcr |= DAVINCI_MCBSP_SPCR_GRST;
+		davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_SPCR_REG, spcr);
+	}
+
+	if (playback) {
+		/* Stop the DMA to avoid data loss */
+		/* while the transmitter is out of reset to handle XSYNCERR */
+		if (component->driver->ops->trigger) {
+			int ret = component->driver->ops->trigger(substream,
+				SNDRV_PCM_TRIGGER_STOP);
+			if (ret < 0)
+				printk(KERN_DEBUG "Playback DMA stop failed\n");
+		}
+
+		/* Enable the transmitter */
+		spcr = davinci_mcbsp_read_reg(dev, DAVINCI_MCBSP_SPCR_REG);
+		spcr |= DAVINCI_MCBSP_SPCR_XRST;
+		davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_SPCR_REG, spcr);
+
+		/* wait for any unexpected frame sync error to occur */
+		udelay(100);
+
+		/* Disable the transmitter to clear any outstanding XSYNCERR */
+		spcr = davinci_mcbsp_read_reg(dev, DAVINCI_MCBSP_SPCR_REG);
+		spcr &= ~DAVINCI_MCBSP_SPCR_XRST;
+		davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_SPCR_REG, spcr);
+		toggle_clock(dev, playback);
+
+		/* Restart the DMA */
+		if (component->driver->ops->trigger) {
+			int ret = component->driver->ops->trigger(substream,
+				SNDRV_PCM_TRIGGER_START);
+			if (ret < 0)
+				printk(KERN_DEBUG "Playback DMA start failed\n");
+		}
+	}
+
+	/* Enable transmitter or receiver */
+	spcr = davinci_mcbsp_read_reg(dev, DAVINCI_MCBSP_SPCR_REG);
+	spcr |= mask;
+
+	if (dev->pcr & (DAVINCI_MCBSP_PCR_FSXM | DAVINCI_MCBSP_PCR_FSRM)) {
+		/* Start frame sync */
+		spcr |= DAVINCI_MCBSP_SPCR_FRST;
+	}
+	davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_SPCR_REG, spcr);
+}
+
+static void davinci_mcbsp_stop(struct davinci_mcbsp_dev *dev, int playback)
+{
+	u32 spcr;
+
+	/* Reset transmitter/receiver and sample rate/frame sync generators */
+	spcr = davinci_mcbsp_read_reg(dev, DAVINCI_MCBSP_SPCR_REG);
+	spcr &= ~(DAVINCI_MCBSP_SPCR_GRST | DAVINCI_MCBSP_SPCR_FRST);
+	spcr &= playback ? ~DAVINCI_MCBSP_SPCR_XRST : ~DAVINCI_MCBSP_SPCR_RRST;
+	davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_SPCR_REG, spcr);
+	toggle_clock(dev, playback);
+}
+
+#define DEFAULT_BITPERSAMPLE	16
+
+static int davinci_i2s_set_dai_fmt(struct snd_soc_dai *cpu_dai,
+				   unsigned int fmt)
+{
+	struct davinci_mcbsp_dev *dev = snd_soc_dai_get_drvdata(cpu_dai);
+	unsigned int pcr;
+	unsigned int srgr;
+	bool inv_fs = false;
+	/* Attention srgr is updated by hw_params! */
+	srgr = DAVINCI_MCBSP_SRGR_FSGM |
+		DAVINCI_MCBSP_SRGR_FPER(DEFAULT_BITPERSAMPLE * 2 - 1) |
+		DAVINCI_MCBSP_SRGR_FWID(DEFAULT_BITPERSAMPLE - 1);
+
+	dev->fmt = fmt;
+	/* set master/slave audio interface */
+	switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) {
+	case SND_SOC_DAIFMT_CBS_CFS:
+		/* cpu is master */
+		pcr = DAVINCI_MCBSP_PCR_FSXM |
+			DAVINCI_MCBSP_PCR_FSRM |
+			DAVINCI_MCBSP_PCR_CLKXM |
+			DAVINCI_MCBSP_PCR_CLKRM;
+		break;
+	case SND_SOC_DAIFMT_CBM_CFS:
+		pcr = DAVINCI_MCBSP_PCR_FSRM | DAVINCI_MCBSP_PCR_FSXM;
+		/*
+		 * Selection of the clock input pin that is the
+		 * input for the Sample Rate Generator.
+		 * McBSP FSR and FSX are driven by the Sample Rate
+		 * Generator.
+		 */
+		switch (dev->clk_input_pin) {
+		case MCBSP_CLKS:
+			pcr |= DAVINCI_MCBSP_PCR_CLKXM |
+				DAVINCI_MCBSP_PCR_CLKRM;
+			break;
+		case MCBSP_CLKR:
+			pcr |= DAVINCI_MCBSP_PCR_SCLKME;
+			break;
+		default:
+			dev_err(dev->dev, "bad clk_input_pin\n");
+			return -EINVAL;
+		}
+
+		break;
+	case SND_SOC_DAIFMT_CBM_CFM:
+		/* codec is master */
+		pcr = 0;
+		break;
+	default:
+		printk(KERN_ERR "%s:bad master\n", __func__);
+		return -EINVAL;
+	}
+
+	/* interface format */
+	switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) {
+	case SND_SOC_DAIFMT_I2S:
+		/* Davinci doesn't support TRUE I2S, but some codecs will have
+		 * the left and right channels contiguous. This allows
+		 * dsp_a mode to be used with an inverted normal frame clk.
+		 * If your codec is master and does not have contiguous
+		 * channels, then you will have sound on only one channel.
+		 * Try using a different mode, or codec as slave.
+		 *
+		 * The TLV320AIC33 is an example of a codec where this works.
+		 * It has a variable bit clock frequency allowing it to have
+		 * valid data on every bit clock.
+		 *
+		 * The TLV320AIC23 is an example of a codec where this does not
+		 * work. It has a fixed bit clock frequency with progressively
+		 * more empty bit clock slots between channels as the sample
+		 * rate is lowered.
+		 */
+		inv_fs = true;
+		/* fall through */
+	case SND_SOC_DAIFMT_DSP_A:
+		dev->mode = MOD_DSP_A;
+		break;
+	case SND_SOC_DAIFMT_DSP_B:
+		dev->mode = MOD_DSP_B;
+		break;
+	default:
+		printk(KERN_ERR "%s:bad format\n", __func__);
+		return -EINVAL;
+	}
+
+	switch (fmt & SND_SOC_DAIFMT_INV_MASK) {
+	case SND_SOC_DAIFMT_NB_NF:
+		/* CLKRP Receive clock polarity,
+		 *	1 - sampled on rising edge of CLKR
+		 *	valid on rising edge
+		 * CLKXP Transmit clock polarity,
+		 *	1 - clocked on falling edge of CLKX
+		 *	valid on rising edge
+		 * FSRP  Receive frame sync pol, 0 - active high
+		 * FSXP  Transmit frame sync pol, 0 - active high
+		 */
+		pcr |= (DAVINCI_MCBSP_PCR_CLKXP | DAVINCI_MCBSP_PCR_CLKRP);
+		break;
+	case SND_SOC_DAIFMT_IB_IF:
+		/* CLKRP Receive clock polarity,
+		 *	0 - sampled on falling edge of CLKR
+		 *	valid on falling edge
+		 * CLKXP Transmit clock polarity,
+		 *	0 - clocked on rising edge of CLKX
+		 *	valid on falling edge
+		 * FSRP  Receive frame sync pol, 1 - active low
+		 * FSXP  Transmit frame sync pol, 1 - active low
+		 */
+		pcr |= (DAVINCI_MCBSP_PCR_FSXP | DAVINCI_MCBSP_PCR_FSRP);
+		break;
+	case SND_SOC_DAIFMT_NB_IF:
+		/* CLKRP Receive clock polarity,
+		 *	1 - sampled on rising edge of CLKR
+		 *	valid on rising edge
+		 * CLKXP Transmit clock polarity,
+		 *	1 - clocked on falling edge of CLKX
+		 *	valid on rising edge
+		 * FSRP  Receive frame sync pol, 1 - active low
+		 * FSXP  Transmit frame sync pol, 1 - active low
+		 */
+		pcr |= (DAVINCI_MCBSP_PCR_CLKXP | DAVINCI_MCBSP_PCR_CLKRP |
+			DAVINCI_MCBSP_PCR_FSXP | DAVINCI_MCBSP_PCR_FSRP);
+		break;
+	case SND_SOC_DAIFMT_IB_NF:
+		/* CLKRP Receive clock polarity,
+		 *	0 - sampled on falling edge of CLKR
+		 *	valid on falling edge
+		 * CLKXP Transmit clock polarity,
+		 *	0 - clocked on rising edge of CLKX
+		 *	valid on falling edge
+		 * FSRP  Receive frame sync pol, 0 - active high
+		 * FSXP  Transmit frame sync pol, 0 - active high
+		 */
+		break;
+	default:
+		return -EINVAL;
+	}
+	if (inv_fs == true)
+		pcr ^= (DAVINCI_MCBSP_PCR_FSXP | DAVINCI_MCBSP_PCR_FSRP);
+	davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_SRGR_REG, srgr);
+	dev->pcr = pcr;
+	davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_PCR_REG, pcr);
+	return 0;
+}
+
+static int davinci_i2s_dai_set_clkdiv(struct snd_soc_dai *cpu_dai,
+				int div_id, int div)
+{
+	struct davinci_mcbsp_dev *dev = snd_soc_dai_get_drvdata(cpu_dai);
+
+	if (div_id != DAVINCI_MCBSP_CLKGDV)
+		return -ENODEV;
+
+	dev->clk_div = div;
+	return 0;
+}
+
+static int davinci_i2s_hw_params(struct snd_pcm_substream *substream,
+				 struct snd_pcm_hw_params *params,
+				 struct snd_soc_dai *dai)
+{
+	struct davinci_mcbsp_dev *dev = snd_soc_dai_get_drvdata(dai);
+	struct snd_interval *i = NULL;
+	int mcbsp_word_length, master;
+	unsigned int rcr, xcr, srgr, clk_div, freq, framesize;
+	u32 spcr;
+	snd_pcm_format_t fmt;
+	unsigned element_cnt = 1;
+
+	/* general line settings */
+	spcr = davinci_mcbsp_read_reg(dev, DAVINCI_MCBSP_SPCR_REG);
+	if (substream->stream == SNDRV_PCM_STREAM_CAPTURE) {
+		spcr |= DAVINCI_MCBSP_SPCR_RINTM(3) | DAVINCI_MCBSP_SPCR_FREE;
+		davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_SPCR_REG, spcr);
+	} else {
+		spcr |= DAVINCI_MCBSP_SPCR_XINTM(3) | DAVINCI_MCBSP_SPCR_FREE;
+		davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_SPCR_REG, spcr);
+	}
+
+	master = dev->fmt & SND_SOC_DAIFMT_MASTER_MASK;
+	fmt = params_format(params);
+	mcbsp_word_length = asp_word_length[fmt];
+
+	switch (master) {
+	case SND_SOC_DAIFMT_CBS_CFS:
+		freq = clk_get_rate(dev->clk);
+		srgr = DAVINCI_MCBSP_SRGR_FSGM |
+		       DAVINCI_MCBSP_SRGR_CLKSM;
+		srgr |= DAVINCI_MCBSP_SRGR_FWID(mcbsp_word_length *
+						8 - 1);
+		if (dev->i2s_accurate_sck) {
+			clk_div = 256;
+			do {
+				framesize = (freq / (--clk_div)) /
+				params->rate_num *
+					params->rate_den;
+			} while (((framesize < 33) || (framesize > 4095)) &&
+				 (clk_div));
+			clk_div--;
+			srgr |= DAVINCI_MCBSP_SRGR_FPER(framesize - 1);
+		} else {
+			/* symmetric waveforms */
+			clk_div = freq / (mcbsp_word_length * 16) /
+				  params->rate_num * params->rate_den;
+			srgr |= DAVINCI_MCBSP_SRGR_FPER(mcbsp_word_length *
+							16 - 1);
+		}
+		clk_div &= 0xFF;
+		srgr |= clk_div;
+		break;
+	case SND_SOC_DAIFMT_CBM_CFS:
+		srgr = DAVINCI_MCBSP_SRGR_FSGM;
+		clk_div = dev->clk_div - 1;
+		srgr |= DAVINCI_MCBSP_SRGR_FWID(mcbsp_word_length * 8 - 1);
+		srgr |= DAVINCI_MCBSP_SRGR_FPER(mcbsp_word_length * 16 - 1);
+		clk_div &= 0xFF;
+		srgr |= clk_div;
+		break;
+	case SND_SOC_DAIFMT_CBM_CFM:
+		/* Clock and frame sync given from external sources */
+		i = hw_param_interval(params, SNDRV_PCM_HW_PARAM_SAMPLE_BITS);
+		srgr = DAVINCI_MCBSP_SRGR_FSGM;
+		srgr |= DAVINCI_MCBSP_SRGR_FWID(snd_interval_value(i) - 1);
+		pr_debug("%s - %d  FWID set: re-read srgr = %X\n",
+			__func__, __LINE__, snd_interval_value(i) - 1);
+
+		i = hw_param_interval(params, SNDRV_PCM_HW_PARAM_FRAME_BITS);
+		srgr |= DAVINCI_MCBSP_SRGR_FPER(snd_interval_value(i) - 1);
+		break;
+	default:
+		return -EINVAL;
+	}
+	davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_SRGR_REG, srgr);
+
+	rcr = DAVINCI_MCBSP_RCR_RFIG;
+	xcr = DAVINCI_MCBSP_XCR_XFIG;
+	if (dev->mode == MOD_DSP_B) {
+		rcr |= DAVINCI_MCBSP_RCR_RDATDLY(0);
+		xcr |= DAVINCI_MCBSP_XCR_XDATDLY(0);
+	} else {
+		rcr |= DAVINCI_MCBSP_RCR_RDATDLY(1);
+		xcr |= DAVINCI_MCBSP_XCR_XDATDLY(1);
+	}
+	/* Determine xfer data type */
+	fmt = params_format(params);
+	if ((fmt > SNDRV_PCM_FORMAT_S32_LE) || !data_type[fmt]) {
+		printk(KERN_WARNING "davinci-i2s: unsupported PCM format\n");
+		return -EINVAL;
+	}
+
+	if (params_channels(params) == 2) {
+		element_cnt = 2;
+		if (double_fmt[fmt] && dev->enable_channel_combine) {
+			element_cnt = 1;
+			fmt = double_fmt[fmt];
+		}
+		switch (master) {
+		case SND_SOC_DAIFMT_CBS_CFS:
+		case SND_SOC_DAIFMT_CBS_CFM:
+			rcr |= DAVINCI_MCBSP_RCR_RFRLEN2(0);
+			xcr |= DAVINCI_MCBSP_XCR_XFRLEN2(0);
+			rcr |= DAVINCI_MCBSP_RCR_RPHASE;
+			xcr |= DAVINCI_MCBSP_XCR_XPHASE;
+			break;
+		case SND_SOC_DAIFMT_CBM_CFM:
+		case SND_SOC_DAIFMT_CBM_CFS:
+			rcr |= DAVINCI_MCBSP_RCR_RFRLEN2(element_cnt - 1);
+			xcr |= DAVINCI_MCBSP_XCR_XFRLEN2(element_cnt - 1);
+			break;
+		default:
+			return -EINVAL;
+		}
+	}
+	mcbsp_word_length = asp_word_length[fmt];
+
+	switch (master) {
+	case SND_SOC_DAIFMT_CBS_CFS:
+	case SND_SOC_DAIFMT_CBS_CFM:
+		rcr |= DAVINCI_MCBSP_RCR_RFRLEN1(0);
+		xcr |= DAVINCI_MCBSP_XCR_XFRLEN1(0);
+		break;
+	case SND_SOC_DAIFMT_CBM_CFM:
+	case SND_SOC_DAIFMT_CBM_CFS:
+		rcr |= DAVINCI_MCBSP_RCR_RFRLEN1(element_cnt - 1);
+		xcr |= DAVINCI_MCBSP_XCR_XFRLEN1(element_cnt - 1);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	rcr |= DAVINCI_MCBSP_RCR_RWDLEN1(mcbsp_word_length) |
+		DAVINCI_MCBSP_RCR_RWDLEN2(mcbsp_word_length);
+	xcr |= DAVINCI_MCBSP_XCR_XWDLEN1(mcbsp_word_length) |
+		DAVINCI_MCBSP_XCR_XWDLEN2(mcbsp_word_length);
+
+	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
+		davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_XCR_REG, xcr);
+	else
+		davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_RCR_REG, rcr);
+
+	pr_debug("%s - %d  srgr=%X\n", __func__, __LINE__, srgr);
+	pr_debug("%s - %d  xcr=%X\n", __func__, __LINE__, xcr);
+	pr_debug("%s - %d  rcr=%X\n", __func__, __LINE__, rcr);
+	return 0;
+}
+
+static int davinci_i2s_prepare(struct snd_pcm_substream *substream,
+		struct snd_soc_dai *dai)
+{
+	struct davinci_mcbsp_dev *dev = snd_soc_dai_get_drvdata(dai);
+	int playback = (substream->stream == SNDRV_PCM_STREAM_PLAYBACK);
+	davinci_mcbsp_stop(dev, playback);
+	return 0;
+}
+
+static int davinci_i2s_trigger(struct snd_pcm_substream *substream, int cmd,
+			       struct snd_soc_dai *dai)
+{
+	struct davinci_mcbsp_dev *dev = snd_soc_dai_get_drvdata(dai);
+	int ret = 0;
+	int playback = (substream->stream == SNDRV_PCM_STREAM_PLAYBACK);
+
+	switch (cmd) {
+	case SNDRV_PCM_TRIGGER_START:
+	case SNDRV_PCM_TRIGGER_RESUME:
+	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
+		davinci_mcbsp_start(dev, substream);
+		break;
+	case SNDRV_PCM_TRIGGER_STOP:
+	case SNDRV_PCM_TRIGGER_SUSPEND:
+	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
+		davinci_mcbsp_stop(dev, playback);
+		break;
+	default:
+		ret = -EINVAL;
+	}
+	return ret;
+}
+
+static void davinci_i2s_shutdown(struct snd_pcm_substream *substream,
+		struct snd_soc_dai *dai)
+{
+	struct davinci_mcbsp_dev *dev = snd_soc_dai_get_drvdata(dai);
+	int playback = (substream->stream == SNDRV_PCM_STREAM_PLAYBACK);
+	davinci_mcbsp_stop(dev, playback);
+}
+
+#define DAVINCI_I2S_RATES	SNDRV_PCM_RATE_8000_96000
+
+static const struct snd_soc_dai_ops davinci_i2s_dai_ops = {
+	.shutdown	= davinci_i2s_shutdown,
+	.prepare	= davinci_i2s_prepare,
+	.trigger	= davinci_i2s_trigger,
+	.hw_params	= davinci_i2s_hw_params,
+	.set_fmt	= davinci_i2s_set_dai_fmt,
+	.set_clkdiv	= davinci_i2s_dai_set_clkdiv,
+
+};
+
+static int davinci_i2s_dai_probe(struct snd_soc_dai *dai)
+{
+	struct davinci_mcbsp_dev *dev = snd_soc_dai_get_drvdata(dai);
+
+	dai->playback_dma_data = &dev->dma_data[SNDRV_PCM_STREAM_PLAYBACK];
+	dai->capture_dma_data = &dev->dma_data[SNDRV_PCM_STREAM_CAPTURE];
+
+	return 0;
+}
+
+static struct snd_soc_dai_driver davinci_i2s_dai = {
+	.probe = davinci_i2s_dai_probe,
+	.playback = {
+		.channels_min = 2,
+		.channels_max = 2,
+		.rates = DAVINCI_I2S_RATES,
+		.formats = SNDRV_PCM_FMTBIT_S16_LE,},
+	.capture = {
+		.channels_min = 2,
+		.channels_max = 2,
+		.rates = DAVINCI_I2S_RATES,
+		.formats = SNDRV_PCM_FMTBIT_S16_LE,},
+	.ops = &davinci_i2s_dai_ops,
+
+};
+
+static const struct snd_soc_component_driver davinci_i2s_component = {
+	.name		= DRV_NAME,
+};
+
+static int davinci_i2s_probe(struct platform_device *pdev)
+{
+	struct snd_dmaengine_dai_dma_data *dma_data;
+	struct davinci_mcbsp_dev *dev;
+	struct resource *mem, *res;
+	void __iomem *io_base;
+	int *dma;
+	int ret;
+
+	mem = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mpu");
+	if (!mem) {
+		dev_warn(&pdev->dev,
+			 "\"mpu\" mem resource not found, using index 0\n");
+		mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+		if (!mem) {
+			dev_err(&pdev->dev, "no mem resource?\n");
+			return -ENODEV;
+		}
+	}
+
+	io_base = devm_ioremap_resource(&pdev->dev, mem);
+	if (IS_ERR(io_base))
+		return PTR_ERR(io_base);
+
+	dev = devm_kzalloc(&pdev->dev, sizeof(struct davinci_mcbsp_dev),
+			   GFP_KERNEL);
+	if (!dev)
+		return -ENOMEM;
+
+	dev->base = io_base;
+
+	/* setup DMA, first TX, then RX */
+	dma_data = &dev->dma_data[SNDRV_PCM_STREAM_PLAYBACK];
+	dma_data->addr = (dma_addr_t)(mem->start + DAVINCI_MCBSP_DXR_REG);
+
+	res = platform_get_resource(pdev, IORESOURCE_DMA, 0);
+	if (res) {
+		dma = &dev->dma_request[SNDRV_PCM_STREAM_PLAYBACK];
+		*dma = res->start;
+		dma_data->filter_data = dma;
+	} else if (IS_ENABLED(CONFIG_OF) && pdev->dev.of_node) {
+		dma_data->filter_data = "tx";
+	} else {
+		dev_err(&pdev->dev, "Missing DMA tx resource\n");
+		return -ENODEV;
+	}
+
+	dma_data = &dev->dma_data[SNDRV_PCM_STREAM_CAPTURE];
+	dma_data->addr = (dma_addr_t)(mem->start + DAVINCI_MCBSP_DRR_REG);
+
+	res = platform_get_resource(pdev, IORESOURCE_DMA, 1);
+	if (res) {
+		dma = &dev->dma_request[SNDRV_PCM_STREAM_CAPTURE];
+		*dma = res->start;
+		dma_data->filter_data = dma;
+	} else if (IS_ENABLED(CONFIG_OF) && pdev->dev.of_node) {
+		dma_data->filter_data = "rx";
+	} else {
+		dev_err(&pdev->dev, "Missing DMA rx resource\n");
+		return -ENODEV;
+	}
+
+	dev->clk = clk_get(&pdev->dev, NULL);
+	if (IS_ERR(dev->clk))
+		return -ENODEV;
+	clk_enable(dev->clk);
+
+	dev->dev = &pdev->dev;
+	dev_set_drvdata(&pdev->dev, dev);
+
+	ret = snd_soc_register_component(&pdev->dev, &davinci_i2s_component,
+					 &davinci_i2s_dai, 1);
+	if (ret != 0)
+		goto err_release_clk;
+
+	ret = edma_pcm_platform_register(&pdev->dev);
+	if (ret) {
+		dev_err(&pdev->dev, "register PCM failed: %d\n", ret);
+		goto err_unregister_component;
+	}
+
+	return 0;
+
+err_unregister_component:
+	snd_soc_unregister_component(&pdev->dev);
+err_release_clk:
+	clk_disable(dev->clk);
+	clk_put(dev->clk);
+	return ret;
+}
+
+static int davinci_i2s_remove(struct platform_device *pdev)
+{
+	struct davinci_mcbsp_dev *dev = dev_get_drvdata(&pdev->dev);
+
+	snd_soc_unregister_component(&pdev->dev);
+
+	clk_disable(dev->clk);
+	clk_put(dev->clk);
+	dev->clk = NULL;
+
+	return 0;
+}
+
+static const struct of_device_id davinci_i2s_match[] = {
+	{ .compatible = "ti,da850-mcbsp" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, davinci_i2s_match);
+
+static struct platform_driver davinci_mcbsp_driver = {
+	.probe		= davinci_i2s_probe,
+	.remove		= davinci_i2s_remove,
+	.driver		= {
+		.name	= "davinci-mcbsp",
+		.of_match_table = of_match_ptr(davinci_i2s_match),
+	},
+};
+
+module_platform_driver(davinci_mcbsp_driver);
+
+MODULE_AUTHOR("Vladimir Barinov");
+MODULE_DESCRIPTION("TI DAVINCI I2S (McBSP) SoC Interface");
+MODULE_LICENSE("GPL");
diff -Naur --no-dereference a/sound/soc/ti/davinci-i2s.h b/sound/soc/ti/davinci-i2s.h
--- a/sound/soc/ti/davinci-i2s.h	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/davinci-i2s.h	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,20 @@
+/*
+ * ALSA SoC I2S (McBSP) Audio Layer for TI DAVINCI processor
+ *
+ * Author:      Vladimir Barinov, <vbarinov@embeddedalley.com>
+ * Copyright:   (C) 2007 MontaVista Software, Inc., <source@mvista.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef _DAVINCI_I2S_H
+#define _DAVINCI_I2S_H
+
+/* McBSP dividers */
+enum davinci_mcbsp_div {
+	DAVINCI_MCBSP_CLKGDV,              /* Sample rate generator divider */
+};
+
+#endif
diff -Naur --no-dereference a/sound/soc/ti/davinci-mcasp.c b/sound/soc/ti/davinci-mcasp.c
--- a/sound/soc/ti/davinci-mcasp.c	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/davinci-mcasp.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,2535 @@
+/*
+ * ALSA SoC McASP Audio Layer for TI DAVINCI processor
+ *
+ * Multi-channel Audio Serial Port Driver
+ *
+ * Author: Nirmal Pandey <n-pandey@ti.com>,
+ *         Suresh Rajashekara <suresh.r@ti.com>
+ *         Steve Chen <schen@.mvista.com>
+ *
+ * Copyright:   (C) 2009 MontaVista Software, Inc., <source@mvista.com>
+ * Copyright:   (C) 2009  Texas Instruments, India
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/slab.h>
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/clk.h>
+#include <linux/pm_runtime.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/of_device.h>
+#include <linux/platform_data/davinci_asp.h>
+#include <linux/math64.h>
+#include <linux/bitmap.h>
+#include <linux/gpio/driver.h>
+
+#include <sound/asoundef.h>
+#include <sound/core.h>
+#include <sound/pcm.h>
+#include <sound/pcm_params.h>
+#include <sound/initval.h>
+#include <sound/soc.h>
+#include <sound/dmaengine_pcm.h>
+#include <dt-bindings/sound/ti-mcasp.h>
+
+#include "edma-pcm.h"
+#include "sdma-pcm.h"
+#include "udma-pcm.h"
+#include "davinci-mcasp.h"
+
+#define MCASP_MAX_AFIFO_DEPTH	64
+
+#ifdef CONFIG_PM
+static u32 context_regs[] = {
+	DAVINCI_MCASP_TXFMCTL_REG,
+	DAVINCI_MCASP_RXFMCTL_REG,
+	DAVINCI_MCASP_TXFMT_REG,
+	DAVINCI_MCASP_RXFMT_REG,
+	DAVINCI_MCASP_ACLKXCTL_REG,
+	DAVINCI_MCASP_ACLKRCTL_REG,
+	DAVINCI_MCASP_AHCLKXCTL_REG,
+	DAVINCI_MCASP_AHCLKRCTL_REG,
+	DAVINCI_MCASP_PDIR_REG,
+	DAVINCI_MCASP_PFUNC_REG,
+	DAVINCI_MCASP_RXMASK_REG,
+	DAVINCI_MCASP_TXMASK_REG,
+	DAVINCI_MCASP_RXTDM_REG,
+	DAVINCI_MCASP_TXTDM_REG,
+};
+
+struct davinci_mcasp_context {
+	u32	config_regs[ARRAY_SIZE(context_regs)];
+	u32	afifo_regs[2]; /* for read/write fifo control registers */
+	u32	*xrsr_regs; /* for serializer configuration */
+	bool	pm_state;
+};
+#endif
+
+struct davinci_mcasp_ruledata {
+	struct davinci_mcasp *mcasp;
+	int serializers;
+};
+
+struct davinci_mcasp {
+	struct snd_dmaengine_dai_dma_data dma_data[2];
+	void __iomem *base;
+	u32 fifo_base;
+	struct device *dev;
+	struct snd_pcm_substream *substreams[2];
+	unsigned int dai_fmt;
+
+	/* McASP specific data */
+	int	tdm_slots;
+	u32	tdm_mask[2];
+	int	slot_width;
+	u8	op_mode;
+	u8	dismod;
+	u8	num_serializer;
+	u8	*serial_dir;
+	u8	version;
+	u8	bclk_div;
+	int	streams;
+	u32	irq_request[2];
+	int	dma_request[2];
+
+	int	sysclk_freq;
+	bool	bclk_master;
+	u32	auxclk_fs_ratio;
+
+	unsigned long pdir; /* Pin direction bitfield */
+
+	/* McASP FIFO related */
+	u8	txnumevt;
+	u8	rxnumevt;
+
+	bool	dat_port;
+
+	/* Used for comstraint setting on the second stream */
+	u32	channels;
+	int	max_format_width;
+	u8	active_serializers[2];
+
+#ifdef CONFIG_GPIOLIB
+	struct gpio_chip gpio_chip;
+#endif
+
+#ifdef CONFIG_PM
+	struct davinci_mcasp_context context;
+#endif
+
+	struct davinci_mcasp_ruledata ruledata[2];
+	struct snd_pcm_hw_constraint_list chconstr[2];
+};
+
+static inline void mcasp_set_bits(struct davinci_mcasp *mcasp, u32 offset,
+				  u32 val)
+{
+	void __iomem *reg = mcasp->base + offset;
+	__raw_writel(__raw_readl(reg) | val, reg);
+}
+
+static inline void mcasp_clr_bits(struct davinci_mcasp *mcasp, u32 offset,
+				  u32 val)
+{
+	void __iomem *reg = mcasp->base + offset;
+	__raw_writel((__raw_readl(reg) & ~(val)), reg);
+}
+
+static inline void mcasp_mod_bits(struct davinci_mcasp *mcasp, u32 offset,
+				  u32 val, u32 mask)
+{
+	void __iomem *reg = mcasp->base + offset;
+	__raw_writel((__raw_readl(reg) & ~mask) | val, reg);
+}
+
+static inline void mcasp_set_reg(struct davinci_mcasp *mcasp, u32 offset,
+				 u32 val)
+{
+	__raw_writel(val, mcasp->base + offset);
+}
+
+static inline u32 mcasp_get_reg(struct davinci_mcasp *mcasp, u32 offset)
+{
+	return (u32)__raw_readl(mcasp->base + offset);
+}
+
+static void mcasp_set_ctl_reg(struct davinci_mcasp *mcasp, u32 ctl_reg, u32 val)
+{
+	int i = 0;
+
+	mcasp_set_bits(mcasp, ctl_reg, val);
+
+	/* programming GBLCTL needs to read back from GBLCTL and verfiy */
+	/* loop count is to avoid the lock-up */
+	for (i = 0; i < 1000; i++) {
+		if ((mcasp_get_reg(mcasp, ctl_reg) & val) == val)
+			break;
+	}
+
+	if (i == 1000 && ((mcasp_get_reg(mcasp, ctl_reg) & val) != val))
+		printk(KERN_ERR "GBLCTL write error\n");
+}
+
+static bool mcasp_is_synchronous(struct davinci_mcasp *mcasp)
+{
+	u32 rxfmctl = mcasp_get_reg(mcasp, DAVINCI_MCASP_RXFMCTL_REG);
+	u32 aclkxctl = mcasp_get_reg(mcasp, DAVINCI_MCASP_ACLKXCTL_REG);
+
+	return !(aclkxctl & TX_ASYNC) && rxfmctl & AFSRE;
+}
+
+static inline void mcasp_set_clk_pdir(struct davinci_mcasp *mcasp, bool enable)
+{
+	u32 bit = PIN_BIT_AMUTE;
+
+	for_each_set_bit_from(bit, &mcasp->pdir, PIN_BIT_AFSR + 1) {
+		if (enable)
+			mcasp_set_bits(mcasp, DAVINCI_MCASP_PDIR_REG, BIT(bit));
+		else
+			mcasp_clr_bits(mcasp, DAVINCI_MCASP_PDIR_REG, BIT(bit));
+	}
+}
+
+static inline void mcasp_set_axr_pdir(struct davinci_mcasp *mcasp, bool enable)
+{
+	u32 bit;
+
+	for_each_set_bit(bit, &mcasp->pdir, PIN_BIT_AMUTE) {
+		if (enable)
+			mcasp_set_bits(mcasp, DAVINCI_MCASP_PDIR_REG, BIT(bit));
+		else
+			mcasp_clr_bits(mcasp, DAVINCI_MCASP_PDIR_REG, BIT(bit));
+	}
+}
+
+static void mcasp_start_rx(struct davinci_mcasp *mcasp)
+{
+	if (mcasp->rxnumevt) {	/* enable FIFO */
+		u32 reg = mcasp->fifo_base + MCASP_RFIFOCTL_OFFSET;
+
+		mcasp_clr_bits(mcasp, reg, FIFO_ENABLE);
+		mcasp_set_bits(mcasp, reg, FIFO_ENABLE);
+	}
+
+	/* Start clocks */
+	mcasp_set_ctl_reg(mcasp, DAVINCI_MCASP_GBLCTLR_REG, RXHCLKRST);
+	mcasp_set_ctl_reg(mcasp, DAVINCI_MCASP_GBLCTLR_REG, RXCLKRST);
+	/*
+	 * When ASYNC == 0 the transmit and receive sections operate
+	 * synchronously from the transmit clock and frame sync. We need to make
+	 * sure that the TX signlas are enabled when starting reception.
+	 */
+	if (mcasp_is_synchronous(mcasp)) {
+		mcasp_set_ctl_reg(mcasp, DAVINCI_MCASP_GBLCTLX_REG, TXHCLKRST);
+		mcasp_set_ctl_reg(mcasp, DAVINCI_MCASP_GBLCTLX_REG, TXCLKRST);
+		mcasp_set_clk_pdir(mcasp, true);
+	}
+
+	/* Activate serializer(s) */
+	mcasp_set_reg(mcasp, DAVINCI_MCASP_RXSTAT_REG, 0xFFFFFFFF);
+	mcasp_set_ctl_reg(mcasp, DAVINCI_MCASP_GBLCTLR_REG, RXSERCLR);
+	/* Release RX state machine */
+	mcasp_set_ctl_reg(mcasp, DAVINCI_MCASP_GBLCTLR_REG, RXSMRST);
+	/* Release Frame Sync generator */
+	mcasp_set_ctl_reg(mcasp, DAVINCI_MCASP_GBLCTLR_REG, RXFSRST);
+	if (mcasp_is_synchronous(mcasp))
+		mcasp_set_ctl_reg(mcasp, DAVINCI_MCASP_GBLCTLX_REG, TXFSRST);
+
+	/* enable receive IRQs */
+	mcasp_set_bits(mcasp, DAVINCI_MCASP_EVTCTLR_REG,
+		       mcasp->irq_request[SNDRV_PCM_STREAM_CAPTURE]);
+}
+
+static void mcasp_start_tx(struct davinci_mcasp *mcasp)
+{
+	u32 cnt;
+
+	if (mcasp->txnumevt) {	/* enable FIFO */
+		u32 reg = mcasp->fifo_base + MCASP_WFIFOCTL_OFFSET;
+
+		mcasp_clr_bits(mcasp, reg, FIFO_ENABLE);
+		mcasp_set_bits(mcasp, reg, FIFO_ENABLE);
+	}
+
+	/* Start clocks */
+	mcasp_set_ctl_reg(mcasp, DAVINCI_MCASP_GBLCTLX_REG, TXHCLKRST);
+	mcasp_set_ctl_reg(mcasp, DAVINCI_MCASP_GBLCTLX_REG, TXCLKRST);
+	mcasp_set_clk_pdir(mcasp, true);
+
+	/* Activate serializer(s) */
+	mcasp_set_reg(mcasp, DAVINCI_MCASP_TXSTAT_REG, 0xFFFFFFFF);
+	mcasp_set_ctl_reg(mcasp, DAVINCI_MCASP_GBLCTLX_REG, TXSERCLR);
+
+	/* wait for XDATA to be cleared */
+	cnt = 0;
+	while ((mcasp_get_reg(mcasp, DAVINCI_MCASP_TXSTAT_REG) & XRDATA) &&
+	       (cnt < 100000))
+		cnt++;
+
+	mcasp_set_axr_pdir(mcasp, true);
+
+	/* Release TX state machine */
+	mcasp_set_ctl_reg(mcasp, DAVINCI_MCASP_GBLCTLX_REG, TXSMRST);
+	/* Release Frame Sync generator */
+	mcasp_set_ctl_reg(mcasp, DAVINCI_MCASP_GBLCTLX_REG, TXFSRST);
+
+	/* enable transmit IRQs */
+	mcasp_set_bits(mcasp, DAVINCI_MCASP_EVTCTLX_REG,
+		       mcasp->irq_request[SNDRV_PCM_STREAM_PLAYBACK]);
+}
+
+static void davinci_mcasp_start(struct davinci_mcasp *mcasp, int stream)
+{
+	mcasp->streams++;
+
+	if (stream == SNDRV_PCM_STREAM_PLAYBACK)
+		mcasp_start_tx(mcasp);
+	else
+		mcasp_start_rx(mcasp);
+}
+
+static void mcasp_stop_rx(struct davinci_mcasp *mcasp)
+{
+	/* disable IRQ sources */
+	mcasp_clr_bits(mcasp, DAVINCI_MCASP_EVTCTLR_REG,
+		       mcasp->irq_request[SNDRV_PCM_STREAM_CAPTURE]);
+
+	/*
+	 * In synchronous mode stop the TX clocks if no other stream is
+	 * running
+	 */
+	if (mcasp_is_synchronous(mcasp) && !mcasp->streams) {
+		mcasp_set_clk_pdir(mcasp, false);
+		mcasp_set_reg(mcasp, DAVINCI_MCASP_GBLCTLX_REG, 0);
+	}
+
+	mcasp_set_reg(mcasp, DAVINCI_MCASP_GBLCTLR_REG, 0);
+	mcasp_set_reg(mcasp, DAVINCI_MCASP_RXSTAT_REG, 0xFFFFFFFF);
+
+	if (mcasp->rxnumevt) {	/* disable FIFO */
+		u32 reg = mcasp->fifo_base + MCASP_RFIFOCTL_OFFSET;
+
+		mcasp_clr_bits(mcasp, reg, FIFO_ENABLE);
+	}
+}
+
+static void mcasp_stop_tx(struct davinci_mcasp *mcasp)
+{
+	u32 val = 0;
+
+	/* disable IRQ sources */
+	mcasp_clr_bits(mcasp, DAVINCI_MCASP_EVTCTLX_REG,
+		       mcasp->irq_request[SNDRV_PCM_STREAM_PLAYBACK]);
+
+	/*
+	 * In synchronous mode keep TX clocks running if the capture stream is
+	 * still running.
+	 */
+	if (mcasp_is_synchronous(mcasp) && mcasp->streams)
+		val =  TXHCLKRST | TXCLKRST | TXFSRST;
+	else
+		mcasp_set_clk_pdir(mcasp, false);
+
+
+	mcasp_set_reg(mcasp, DAVINCI_MCASP_GBLCTLX_REG, val);
+	mcasp_set_reg(mcasp, DAVINCI_MCASP_TXSTAT_REG, 0xFFFFFFFF);
+
+	if (mcasp->txnumevt) {	/* disable FIFO */
+		u32 reg = mcasp->fifo_base + MCASP_WFIFOCTL_OFFSET;
+
+		mcasp_clr_bits(mcasp, reg, FIFO_ENABLE);
+	}
+
+	mcasp_set_axr_pdir(mcasp, false);
+}
+
+static void davinci_mcasp_stop(struct davinci_mcasp *mcasp, int stream)
+{
+	mcasp->streams--;
+
+	if (stream == SNDRV_PCM_STREAM_PLAYBACK)
+		mcasp_stop_tx(mcasp);
+	else
+		mcasp_stop_rx(mcasp);
+}
+
+static irqreturn_t davinci_mcasp_tx_irq_handler(int irq, void *data)
+{
+	struct davinci_mcasp *mcasp = (struct davinci_mcasp *)data;
+	struct snd_pcm_substream *substream;
+	u32 irq_mask = mcasp->irq_request[SNDRV_PCM_STREAM_PLAYBACK];
+	u32 handled_mask = 0;
+	u32 stat;
+
+	stat = mcasp_get_reg(mcasp, DAVINCI_MCASP_TXSTAT_REG);
+	if (stat & XUNDRN & irq_mask) {
+		dev_warn(mcasp->dev, "Transmit buffer underflow\n");
+		handled_mask |= XUNDRN;
+
+		substream = mcasp->substreams[SNDRV_PCM_STREAM_PLAYBACK];
+		if (substream)
+			snd_pcm_stop_xrun(substream);
+	}
+
+	if (!handled_mask)
+		dev_warn(mcasp->dev, "unhandled tx event. txstat: 0x%08x\n",
+			 stat);
+
+	if (stat & XRERR)
+		handled_mask |= XRERR;
+
+	/* Ack the handled event only */
+	mcasp_set_reg(mcasp, DAVINCI_MCASP_TXSTAT_REG, handled_mask);
+
+	return IRQ_RETVAL(handled_mask);
+}
+
+static irqreturn_t davinci_mcasp_rx_irq_handler(int irq, void *data)
+{
+	struct davinci_mcasp *mcasp = (struct davinci_mcasp *)data;
+	struct snd_pcm_substream *substream;
+	u32 irq_mask = mcasp->irq_request[SNDRV_PCM_STREAM_CAPTURE];
+	u32 handled_mask = 0;
+	u32 stat;
+
+	stat = mcasp_get_reg(mcasp, DAVINCI_MCASP_RXSTAT_REG);
+	if (stat & ROVRN & irq_mask) {
+		dev_warn(mcasp->dev, "Receive buffer overflow\n");
+		handled_mask |= ROVRN;
+
+		substream = mcasp->substreams[SNDRV_PCM_STREAM_CAPTURE];
+		if (substream)
+			snd_pcm_stop_xrun(substream);
+	}
+
+	if (!handled_mask)
+		dev_warn(mcasp->dev, "unhandled rx event. rxstat: 0x%08x\n",
+			 stat);
+
+	if (stat & XRERR)
+		handled_mask |= XRERR;
+
+	/* Ack the handled event only */
+	mcasp_set_reg(mcasp, DAVINCI_MCASP_RXSTAT_REG, handled_mask);
+
+	return IRQ_RETVAL(handled_mask);
+}
+
+static irqreturn_t davinci_mcasp_common_irq_handler(int irq, void *data)
+{
+	struct davinci_mcasp *mcasp = (struct davinci_mcasp *)data;
+	irqreturn_t ret = IRQ_NONE;
+
+	if (mcasp->substreams[SNDRV_PCM_STREAM_PLAYBACK])
+		ret = davinci_mcasp_tx_irq_handler(irq, data);
+
+	if (mcasp->substreams[SNDRV_PCM_STREAM_CAPTURE])
+		ret |= davinci_mcasp_rx_irq_handler(irq, data);
+
+	return ret;
+}
+
+static int davinci_mcasp_set_dai_fmt(struct snd_soc_dai *cpu_dai,
+					 unsigned int fmt)
+{
+	struct davinci_mcasp *mcasp = snd_soc_dai_get_drvdata(cpu_dai);
+	int ret = 0;
+	u32 data_delay;
+	bool fs_pol_rising;
+	bool inv_fs = false;
+
+	if (!fmt)
+		return 0;
+
+	pm_runtime_get_sync(mcasp->dev);
+	switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) {
+	case SND_SOC_DAIFMT_DSP_A:
+		mcasp_clr_bits(mcasp, DAVINCI_MCASP_TXFMCTL_REG, FSXDUR);
+		mcasp_clr_bits(mcasp, DAVINCI_MCASP_RXFMCTL_REG, FSRDUR);
+		/* 1st data bit occur one ACLK cycle after the frame sync */
+		data_delay = 1;
+		break;
+	case SND_SOC_DAIFMT_DSP_B:
+	case SND_SOC_DAIFMT_AC97:
+		mcasp_clr_bits(mcasp, DAVINCI_MCASP_TXFMCTL_REG, FSXDUR);
+		mcasp_clr_bits(mcasp, DAVINCI_MCASP_RXFMCTL_REG, FSRDUR);
+		/* No delay after FS */
+		data_delay = 0;
+		break;
+	case SND_SOC_DAIFMT_I2S:
+		/* configure a full-word SYNC pulse (LRCLK) */
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_TXFMCTL_REG, FSXDUR);
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_RXFMCTL_REG, FSRDUR);
+		/* 1st data bit occur one ACLK cycle after the frame sync */
+		data_delay = 1;
+		/* FS need to be inverted */
+		inv_fs = true;
+		break;
+	case SND_SOC_DAIFMT_RIGHT_J:
+	case SND_SOC_DAIFMT_LEFT_J:
+		/* configure a full-word SYNC pulse (LRCLK) */
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_TXFMCTL_REG, FSXDUR);
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_RXFMCTL_REG, FSRDUR);
+		/* No delay after FS */
+		data_delay = 0;
+		break;
+	default:
+		ret = -EINVAL;
+		goto out;
+	}
+
+	mcasp_mod_bits(mcasp, DAVINCI_MCASP_TXFMT_REG, FSXDLY(data_delay),
+		       FSXDLY(3));
+	mcasp_mod_bits(mcasp, DAVINCI_MCASP_RXFMT_REG, FSRDLY(data_delay),
+		       FSRDLY(3));
+
+	switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) {
+	case SND_SOC_DAIFMT_CBS_CFS:
+		/* codec is clock and frame slave */
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_ACLKXCTL_REG, ACLKXE);
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_TXFMCTL_REG, AFSXE);
+
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_ACLKRCTL_REG, ACLKRE);
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_RXFMCTL_REG, AFSRE);
+
+		/* BCLK */
+		set_bit(PIN_BIT_ACLKX, &mcasp->pdir);
+		set_bit(PIN_BIT_ACLKR, &mcasp->pdir);
+		/* Frame Sync */
+		set_bit(PIN_BIT_AFSX, &mcasp->pdir);
+		set_bit(PIN_BIT_AFSR, &mcasp->pdir);
+
+		mcasp->bclk_master = 1;
+		break;
+	case SND_SOC_DAIFMT_CBS_CFM:
+		/* codec is clock slave and frame master */
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_ACLKXCTL_REG, ACLKXE);
+		mcasp_clr_bits(mcasp, DAVINCI_MCASP_TXFMCTL_REG, AFSXE);
+
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_ACLKRCTL_REG, ACLKRE);
+		mcasp_clr_bits(mcasp, DAVINCI_MCASP_RXFMCTL_REG, AFSRE);
+
+		/* BCLK */
+		set_bit(PIN_BIT_ACLKX, &mcasp->pdir);
+		set_bit(PIN_BIT_ACLKR, &mcasp->pdir);
+		/* Frame Sync */
+		clear_bit(PIN_BIT_AFSX, &mcasp->pdir);
+		clear_bit(PIN_BIT_AFSR, &mcasp->pdir);
+
+		mcasp->bclk_master = 1;
+		break;
+	case SND_SOC_DAIFMT_CBM_CFS:
+		/* codec is clock master and frame slave */
+		mcasp_clr_bits(mcasp, DAVINCI_MCASP_ACLKXCTL_REG, ACLKXE);
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_TXFMCTL_REG, AFSXE);
+
+		mcasp_clr_bits(mcasp, DAVINCI_MCASP_ACLKRCTL_REG, ACLKRE);
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_RXFMCTL_REG, AFSRE);
+
+		/* BCLK */
+		clear_bit(PIN_BIT_ACLKX, &mcasp->pdir);
+		clear_bit(PIN_BIT_ACLKR, &mcasp->pdir);
+		/* Frame Sync */
+		set_bit(PIN_BIT_AFSX, &mcasp->pdir);
+		set_bit(PIN_BIT_AFSR, &mcasp->pdir);
+
+		mcasp->bclk_master = 0;
+		break;
+	case SND_SOC_DAIFMT_CBM_CFM:
+		/* codec is clock and frame master */
+		mcasp_clr_bits(mcasp, DAVINCI_MCASP_ACLKXCTL_REG, ACLKXE);
+		mcasp_clr_bits(mcasp, DAVINCI_MCASP_TXFMCTL_REG, AFSXE);
+
+		mcasp_clr_bits(mcasp, DAVINCI_MCASP_ACLKRCTL_REG, ACLKRE);
+		mcasp_clr_bits(mcasp, DAVINCI_MCASP_RXFMCTL_REG, AFSRE);
+
+		/* BCLK */
+		clear_bit(PIN_BIT_ACLKX, &mcasp->pdir);
+		clear_bit(PIN_BIT_ACLKR, &mcasp->pdir);
+		/* Frame Sync */
+		clear_bit(PIN_BIT_AFSX, &mcasp->pdir);
+		clear_bit(PIN_BIT_AFSR, &mcasp->pdir);
+
+		mcasp->bclk_master = 0;
+		break;
+	default:
+		ret = -EINVAL;
+		goto out;
+	}
+
+	switch (fmt & SND_SOC_DAIFMT_INV_MASK) {
+	case SND_SOC_DAIFMT_IB_NF:
+		mcasp_clr_bits(mcasp, DAVINCI_MCASP_ACLKXCTL_REG, ACLKXPOL);
+		mcasp_clr_bits(mcasp, DAVINCI_MCASP_ACLKRCTL_REG, ACLKRPOL);
+		fs_pol_rising = true;
+		break;
+	case SND_SOC_DAIFMT_NB_IF:
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_ACLKXCTL_REG, ACLKXPOL);
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_ACLKRCTL_REG, ACLKRPOL);
+		fs_pol_rising = false;
+		break;
+	case SND_SOC_DAIFMT_IB_IF:
+		mcasp_clr_bits(mcasp, DAVINCI_MCASP_ACLKXCTL_REG, ACLKXPOL);
+		mcasp_clr_bits(mcasp, DAVINCI_MCASP_ACLKRCTL_REG, ACLKRPOL);
+		fs_pol_rising = false;
+		break;
+	case SND_SOC_DAIFMT_NB_NF:
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_ACLKXCTL_REG, ACLKXPOL);
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_ACLKRCTL_REG, ACLKRPOL);
+		fs_pol_rising = true;
+		break;
+	default:
+		ret = -EINVAL;
+		goto out;
+	}
+
+	if (inv_fs)
+		fs_pol_rising = !fs_pol_rising;
+
+	if (fs_pol_rising) {
+		mcasp_clr_bits(mcasp, DAVINCI_MCASP_TXFMCTL_REG, FSXPOL);
+		mcasp_clr_bits(mcasp, DAVINCI_MCASP_RXFMCTL_REG, FSRPOL);
+	} else {
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_TXFMCTL_REG, FSXPOL);
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_RXFMCTL_REG, FSRPOL);
+	}
+
+	mcasp->dai_fmt = fmt;
+out:
+	pm_runtime_put(mcasp->dev);
+	return ret;
+}
+
+static int __davinci_mcasp_set_clkdiv(struct davinci_mcasp *mcasp, int div_id,
+				      int div, bool explicit)
+{
+	pm_runtime_get_sync(mcasp->dev);
+	switch (div_id) {
+	case MCASP_CLKDIV_AUXCLK:			/* MCLK divider */
+		mcasp_mod_bits(mcasp, DAVINCI_MCASP_AHCLKXCTL_REG,
+			       AHCLKXDIV(div - 1), AHCLKXDIV_MASK);
+		mcasp_mod_bits(mcasp, DAVINCI_MCASP_AHCLKRCTL_REG,
+			       AHCLKRDIV(div - 1), AHCLKRDIV_MASK);
+		break;
+
+	case MCASP_CLKDIV_BCLK:			/* BCLK divider */
+		mcasp_mod_bits(mcasp, DAVINCI_MCASP_ACLKXCTL_REG,
+			       ACLKXDIV(div - 1), ACLKXDIV_MASK);
+		mcasp_mod_bits(mcasp, DAVINCI_MCASP_ACLKRCTL_REG,
+			       ACLKRDIV(div - 1), ACLKRDIV_MASK);
+		if (explicit)
+			mcasp->bclk_div = div;
+		break;
+
+	case MCASP_CLKDIV_BCLK_FS_RATIO:
+		/*
+		 * BCLK/LRCLK ratio descries how many bit-clock cycles
+		 * fit into one frame. The clock ratio is given for a
+		 * full period of data (for I2S format both left and
+		 * right channels), so it has to be divided by number
+		 * of tdm-slots (for I2S - divided by 2).
+		 * Instead of storing this ratio, we calculate a new
+		 * tdm_slot width by dividing the the ratio by the
+		 * number of configured tdm slots.
+		 */
+		mcasp->slot_width = div / mcasp->tdm_slots;
+		if (div % mcasp->tdm_slots)
+			dev_warn(mcasp->dev,
+				 "%s(): BCLK/LRCLK %d is not divisible by %d tdm slots",
+				 __func__, div, mcasp->tdm_slots);
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	pm_runtime_put(mcasp->dev);
+	return 0;
+}
+
+static int davinci_mcasp_set_clkdiv(struct snd_soc_dai *dai, int div_id,
+				    int div)
+{
+	struct davinci_mcasp *mcasp = snd_soc_dai_get_drvdata(dai);
+
+	return __davinci_mcasp_set_clkdiv(mcasp, div_id, div, 1);
+}
+
+static int davinci_mcasp_set_sysclk(struct snd_soc_dai *dai, int clk_id,
+				    unsigned int freq, int dir)
+{
+	struct davinci_mcasp *mcasp = snd_soc_dai_get_drvdata(dai);
+
+	pm_runtime_get_sync(mcasp->dev);
+
+	if (dir == SND_SOC_CLOCK_IN) {
+		switch (clk_id) {
+		case MCASP_CLK_HCLK_AHCLK:
+			mcasp_clr_bits(mcasp, DAVINCI_MCASP_AHCLKXCTL_REG,
+				       AHCLKXE);
+			mcasp_clr_bits(mcasp, DAVINCI_MCASP_AHCLKRCTL_REG,
+				       AHCLKRE);
+			clear_bit(PIN_BIT_AHCLKX, &mcasp->pdir);
+			break;
+		case MCASP_CLK_HCLK_AUXCLK:
+			mcasp_set_bits(mcasp, DAVINCI_MCASP_AHCLKXCTL_REG,
+				       AHCLKXE);
+			mcasp_set_bits(mcasp, DAVINCI_MCASP_AHCLKRCTL_REG,
+				       AHCLKRE);
+			set_bit(PIN_BIT_AHCLKX, &mcasp->pdir);
+			break;
+		default:
+			dev_err(mcasp->dev, "Invalid clk id: %d\n", clk_id);
+			goto out;
+		}
+	} else {
+		/* Select AUXCLK as HCLK */
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_AHCLKXCTL_REG, AHCLKXE);
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_AHCLKRCTL_REG, AHCLKRE);
+		set_bit(PIN_BIT_AHCLKX, &mcasp->pdir);
+	}
+	/*
+	 * When AHCLK X/R is selected to be output it means that the HCLK is
+	 * the same clock - coming via AUXCLK.
+	 */
+	mcasp->sysclk_freq = freq;
+out:
+	pm_runtime_put(mcasp->dev);
+	return 0;
+}
+
+/* All serializers must have equal number of channels */
+static int davinci_mcasp_ch_constraint(struct davinci_mcasp *mcasp, int stream,
+				       int serializers)
+{
+	struct snd_pcm_hw_constraint_list *cl = &mcasp->chconstr[stream];
+	unsigned int *list = (unsigned int *) cl->list;
+	int slots = mcasp->tdm_slots;
+	int i, count = 0;
+
+	if (mcasp->tdm_mask[stream])
+		slots = hweight32(mcasp->tdm_mask[stream]);
+
+	for (i = 1; i <= slots; i++)
+		list[count++] = i;
+
+	for (i = 2; i <= serializers; i++)
+		list[count++] = i*slots;
+
+	cl->count = count;
+
+	return 0;
+}
+
+static int davinci_mcasp_set_ch_constraints(struct davinci_mcasp *mcasp)
+{
+	int rx_serializers = 0, tx_serializers = 0, ret, i;
+
+	for (i = 0; i < mcasp->num_serializer; i++)
+		if (mcasp->serial_dir[i] == TX_MODE)
+			tx_serializers++;
+		else if (mcasp->serial_dir[i] == RX_MODE)
+			rx_serializers++;
+
+	ret = davinci_mcasp_ch_constraint(mcasp, SNDRV_PCM_STREAM_PLAYBACK,
+					  tx_serializers);
+	if (ret)
+		return ret;
+
+	ret = davinci_mcasp_ch_constraint(mcasp, SNDRV_PCM_STREAM_CAPTURE,
+					  rx_serializers);
+
+	return ret;
+}
+
+
+static int davinci_mcasp_set_tdm_slot(struct snd_soc_dai *dai,
+				      unsigned int tx_mask,
+				      unsigned int rx_mask,
+				      int slots, int slot_width)
+{
+	struct davinci_mcasp *mcasp = snd_soc_dai_get_drvdata(dai);
+
+	dev_dbg(mcasp->dev,
+		 "%s() tx_mask 0x%08x rx_mask 0x%08x slots %d width %d\n",
+		 __func__, tx_mask, rx_mask, slots, slot_width);
+
+	if (tx_mask >= (1<<slots) || rx_mask >= (1<<slots)) {
+		dev_err(mcasp->dev,
+			"Bad tdm mask tx: 0x%08x rx: 0x%08x slots %d\n",
+			tx_mask, rx_mask, slots);
+		return -EINVAL;
+	}
+
+	if (slot_width &&
+	    (slot_width < 8 || slot_width > 32 || slot_width % 4 != 0)) {
+		dev_err(mcasp->dev, "%s: Unsupported slot_width %d\n",
+			__func__, slot_width);
+		return -EINVAL;
+	}
+
+	mcasp->tdm_slots = slots;
+	mcasp->tdm_mask[SNDRV_PCM_STREAM_PLAYBACK] = tx_mask;
+	mcasp->tdm_mask[SNDRV_PCM_STREAM_CAPTURE] = rx_mask;
+	mcasp->slot_width = slot_width;
+
+	return davinci_mcasp_set_ch_constraints(mcasp);
+}
+
+static int davinci_config_channel_size(struct davinci_mcasp *mcasp,
+				       int sample_width)
+{
+	u32 fmt;
+	u32 tx_rotate, rx_rotate, slot_width;
+	u32 mask = (1ULL << sample_width) - 1;
+
+	if (mcasp->slot_width)
+		slot_width = mcasp->slot_width;
+	else if (mcasp->max_format_width)
+		slot_width = mcasp->max_format_width;
+	else
+		slot_width = sample_width;
+	/*
+	 * TX rotation:
+	 * right aligned formats: rotate w/ slot_width
+	 * left aligned formats: rotate w/ sample_width
+	 *
+	 * RX rotation:
+	 * right aligned formats: no rotation needed
+	 * left aligned formats: rotate w/ (slot_width - sample_width)
+	 */
+	if ((mcasp->dai_fmt & SND_SOC_DAIFMT_FORMAT_MASK) ==
+	    SND_SOC_DAIFMT_RIGHT_J) {
+		tx_rotate = (slot_width / 4) & 0x7;
+		rx_rotate = 0;
+	} else {
+		tx_rotate = (sample_width / 4) & 0x7;
+		rx_rotate = (slot_width - sample_width) / 4;
+	}
+
+	/* mapping of the XSSZ bit-field as described in the datasheet */
+	fmt = (slot_width >> 1) - 1;
+
+	if (mcasp->op_mode != DAVINCI_MCASP_DIT_MODE) {
+		mcasp_mod_bits(mcasp, DAVINCI_MCASP_RXFMT_REG, RXSSZ(fmt),
+			       RXSSZ(0x0F));
+		mcasp_mod_bits(mcasp, DAVINCI_MCASP_TXFMT_REG, TXSSZ(fmt),
+			       TXSSZ(0x0F));
+		mcasp_mod_bits(mcasp, DAVINCI_MCASP_TXFMT_REG, TXROT(tx_rotate),
+			       TXROT(7));
+		mcasp_mod_bits(mcasp, DAVINCI_MCASP_RXFMT_REG, RXROT(rx_rotate),
+			       RXROT(7));
+		mcasp_set_reg(mcasp, DAVINCI_MCASP_RXMASK_REG, mask);
+	}
+
+	mcasp_set_reg(mcasp, DAVINCI_MCASP_TXMASK_REG, mask);
+
+	return 0;
+}
+
+static int mcasp_common_hw_param(struct davinci_mcasp *mcasp, int stream,
+				 int period_words, int channels)
+{
+	struct snd_dmaengine_dai_dma_data *dma_data = &mcasp->dma_data[stream];
+	int i;
+	u8 tx_ser = 0;
+	u8 rx_ser = 0;
+	u8 slots = mcasp->tdm_slots;
+	u8 max_active_serializers = (channels + slots - 1) / slots;
+	u8 max_rx_serializers, max_tx_serializers;
+	int active_serializers, numevt;
+	u32 reg;
+	/* Default configuration */
+	if (mcasp->version < MCASP_VERSION_3)
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_PWREMUMGT_REG, MCASP_SOFT);
+
+	if (stream == SNDRV_PCM_STREAM_PLAYBACK) {
+		mcasp_set_reg(mcasp, DAVINCI_MCASP_TXSTAT_REG, 0xFFFFFFFF);
+		mcasp_clr_bits(mcasp, DAVINCI_MCASP_XEVTCTL_REG, TXDATADMADIS);
+		max_tx_serializers = max_active_serializers;
+		max_rx_serializers =
+			mcasp->active_serializers[SNDRV_PCM_STREAM_CAPTURE];
+	} else {
+		mcasp_set_reg(mcasp, DAVINCI_MCASP_RXSTAT_REG, 0xFFFFFFFF);
+		mcasp_clr_bits(mcasp, DAVINCI_MCASP_REVTCTL_REG, RXDATADMADIS);
+		max_tx_serializers =
+			mcasp->active_serializers[SNDRV_PCM_STREAM_PLAYBACK];
+		max_rx_serializers = max_active_serializers;
+	}
+
+	for (i = 0; i < mcasp->num_serializer; i++) {
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_XRSRCTL_REG(i),
+			       mcasp->serial_dir[i]);
+		if (mcasp->serial_dir[i] == TX_MODE &&
+					tx_ser < max_tx_serializers) {
+			mcasp_mod_bits(mcasp, DAVINCI_MCASP_XRSRCTL_REG(i),
+				       mcasp->dismod, DISMOD_MASK);
+			set_bit(PIN_BIT_AXR(i), &mcasp->pdir);
+			tx_ser++;
+		} else if (mcasp->serial_dir[i] == RX_MODE &&
+					rx_ser < max_rx_serializers) {
+			clear_bit(PIN_BIT_AXR(i), &mcasp->pdir);
+			rx_ser++;
+		} else {
+			/* Inactive or unused pin, set it to inactive */
+			mcasp_mod_bits(mcasp, DAVINCI_MCASP_XRSRCTL_REG(i),
+				       SRMOD_INACTIVE, SRMOD_MASK);
+			/* If unused, set DISMOD for the pin */
+			if (mcasp->serial_dir[i] != INACTIVE_MODE)
+				mcasp_mod_bits(mcasp,
+					       DAVINCI_MCASP_XRSRCTL_REG(i),
+					       mcasp->dismod, DISMOD_MASK);
+			clear_bit(PIN_BIT_AXR(i), &mcasp->pdir);
+		}
+	}
+
+	if (stream == SNDRV_PCM_STREAM_PLAYBACK) {
+		active_serializers = tx_ser;
+		numevt = mcasp->txnumevt;
+		reg = mcasp->fifo_base + MCASP_WFIFOCTL_OFFSET;
+	} else {
+		active_serializers = rx_ser;
+		numevt = mcasp->rxnumevt;
+		reg = mcasp->fifo_base + MCASP_RFIFOCTL_OFFSET;
+	}
+
+	if (active_serializers < max_active_serializers) {
+		dev_warn(mcasp->dev, "stream has more channels (%d) than are "
+			 "enabled in mcasp (%d)\n", channels,
+			 active_serializers * slots);
+		return -EINVAL;
+	}
+
+	/* AFIFO is not in use */
+	if (!numevt) {
+		/* Configure the burst size for platform drivers */
+		if (active_serializers > 1) {
+			/*
+			 * If more than one serializers are in use we have one
+			 * DMA request to provide data for all serializers.
+			 * For example if three serializers are enabled the DMA
+			 * need to transfer three words per DMA request.
+			 */
+			dma_data->maxburst = active_serializers;
+		} else {
+			dma_data->maxburst = 0;
+		}
+
+		goto out;
+	}
+
+	if (period_words % active_serializers) {
+		dev_err(mcasp->dev, "Invalid combination of period words and "
+			"active serializers: %d, %d\n", period_words,
+			active_serializers);
+		return -EINVAL;
+	}
+
+	/*
+	 * Calculate the optimal AFIFO depth for platform side:
+	 * The number of words for numevt need to be in steps of active
+	 * serializers.
+	 */
+	numevt = (numevt / active_serializers) * active_serializers;
+
+	while (period_words % numevt && numevt > 0)
+		numevt -= active_serializers;
+	if (numevt <= 0)
+		numevt = active_serializers;
+
+	mcasp_mod_bits(mcasp, reg, active_serializers, NUMDMA_MASK);
+	mcasp_mod_bits(mcasp, reg, NUMEVT(numevt), NUMEVT_MASK);
+
+	/* Configure the burst size for platform drivers */
+	if (numevt == 1)
+		numevt = 0;
+	dma_data->maxburst = numevt;
+
+out:
+	mcasp->active_serializers[stream] = active_serializers;
+
+	return 0;
+}
+
+static int mcasp_i2s_hw_param(struct davinci_mcasp *mcasp, int stream,
+			      int channels)
+{
+	int i, active_slots;
+	int total_slots;
+	int active_serializers;
+	u32 mask = 0;
+	u32 busel = 0;
+
+	total_slots = mcasp->tdm_slots;
+
+	/*
+	 * If more than one serializer is needed, then use them with
+	 * all the specified tdm_slots. Otherwise, one serializer can
+	 * cope with the transaction using just as many slots as there
+	 * are channels in the stream.
+	 */
+	if (mcasp->tdm_mask[stream]) {
+		active_slots = hweight32(mcasp->tdm_mask[stream]);
+		active_serializers = (channels + active_slots - 1) /
+			active_slots;
+		if (active_serializers == 1)
+			active_slots = channels;
+		for (i = 0; i < total_slots; i++) {
+			if ((1 << i) & mcasp->tdm_mask[stream]) {
+				mask |= (1 << i);
+				if (--active_slots <= 0)
+					break;
+			}
+		}
+	} else {
+		active_serializers = (channels + total_slots - 1) / total_slots;
+		if (active_serializers == 1)
+			active_slots = channels;
+		else
+			active_slots = total_slots;
+
+		for (i = 0; i < active_slots; i++)
+			mask |= (1 << i);
+	}
+
+	mcasp_clr_bits(mcasp, DAVINCI_MCASP_ACLKXCTL_REG, TX_ASYNC);
+
+	if (!mcasp->dat_port)
+		busel = TXSEL;
+
+	if (stream == SNDRV_PCM_STREAM_PLAYBACK) {
+		mcasp_set_reg(mcasp, DAVINCI_MCASP_TXTDM_REG, mask);
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_TXFMT_REG, busel | TXORD);
+		mcasp_mod_bits(mcasp, DAVINCI_MCASP_TXFMCTL_REG,
+			       FSXMOD(total_slots), FSXMOD(0x1FF));
+	} else if (stream == SNDRV_PCM_STREAM_CAPTURE) {
+		mcasp_set_reg(mcasp, DAVINCI_MCASP_RXTDM_REG, mask);
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_RXFMT_REG, busel | RXORD);
+		mcasp_mod_bits(mcasp, DAVINCI_MCASP_RXFMCTL_REG,
+			       FSRMOD(total_slots), FSRMOD(0x1FF));
+		/*
+		 * If McASP is set to be TX/RX synchronous and the playback is
+		 * not running already we need to configure the TX slots in
+		 * order to have correct FSX on the bus
+		 */
+		if (mcasp_is_synchronous(mcasp) && !mcasp->channels)
+			mcasp_mod_bits(mcasp, DAVINCI_MCASP_TXFMCTL_REG,
+				       FSXMOD(total_slots), FSXMOD(0x1FF));
+	}
+
+	return 0;
+}
+
+/* S/PDIF */
+static int mcasp_dit_hw_param(struct davinci_mcasp *mcasp,
+			      unsigned int rate)
+{
+	u32 cs_value = 0;
+	u8 *cs_bytes = (u8*) &cs_value;
+
+	/* Set the TX format : 24 bit right rotation, 32 bit slot, Pad 0
+	   and LSB first */
+	mcasp_set_bits(mcasp, DAVINCI_MCASP_TXFMT_REG, TXROT(6) | TXSSZ(15));
+
+	/* Set TX frame synch : DIT Mode, 1 bit width, internal, rising edge */
+	mcasp_set_reg(mcasp, DAVINCI_MCASP_TXFMCTL_REG, AFSXE | FSXMOD(0x180));
+
+	/* Set the TX tdm : for all the slots */
+	mcasp_set_reg(mcasp, DAVINCI_MCASP_TXTDM_REG, 0xFFFFFFFF);
+
+	/* Set the TX clock controls : div = 1 and internal */
+	mcasp_set_bits(mcasp, DAVINCI_MCASP_ACLKXCTL_REG, ACLKXE | TX_ASYNC);
+
+	mcasp_clr_bits(mcasp, DAVINCI_MCASP_XEVTCTL_REG, TXDATADMADIS);
+
+	/* Only 44100 and 48000 are valid, both have the same setting */
+	mcasp_set_bits(mcasp, DAVINCI_MCASP_AHCLKXCTL_REG, AHCLKXDIV(3));
+
+	/* Enable the DIT */
+	mcasp_set_bits(mcasp, DAVINCI_MCASP_TXDITCTL_REG, DITEN);
+
+	/* Set S/PDIF channel status bits */
+	cs_bytes[0] = IEC958_AES0_CON_NOT_COPYRIGHT;
+	cs_bytes[1] = IEC958_AES1_CON_PCM_CODER;
+
+	switch (rate) {
+	case 22050:
+		cs_bytes[3] |= IEC958_AES3_CON_FS_22050;
+		break;
+	case 24000:
+		cs_bytes[3] |= IEC958_AES3_CON_FS_24000;
+		break;
+	case 32000:
+		cs_bytes[3] |= IEC958_AES3_CON_FS_32000;
+		break;
+	case 44100:
+		cs_bytes[3] |= IEC958_AES3_CON_FS_44100;
+		break;
+	case 48000:
+		cs_bytes[3] |= IEC958_AES3_CON_FS_48000;
+		break;
+	case 88200:
+		cs_bytes[3] |= IEC958_AES3_CON_FS_88200;
+		break;
+	case 96000:
+		cs_bytes[3] |= IEC958_AES3_CON_FS_96000;
+		break;
+	case 176400:
+		cs_bytes[3] |= IEC958_AES3_CON_FS_176400;
+		break;
+	case 192000:
+		cs_bytes[3] |= IEC958_AES3_CON_FS_192000;
+		break;
+	default:
+		printk(KERN_WARNING "unsupported sampling rate: %d\n", rate);
+		return -EINVAL;
+	}
+
+	mcasp_set_reg(mcasp, DAVINCI_MCASP_DITCSRA_REG, cs_value);
+	mcasp_set_reg(mcasp, DAVINCI_MCASP_DITCSRB_REG, cs_value);
+
+	return 0;
+}
+
+static int davinci_mcasp_calc_clk_div(struct davinci_mcasp *mcasp,
+				      unsigned int sysclk_freq,
+				      unsigned int bclk_freq, bool set)
+{
+	u32 reg = mcasp_get_reg(mcasp, DAVINCI_MCASP_AHCLKXCTL_REG);
+	int div = sysclk_freq / bclk_freq;
+	int rem = sysclk_freq % bclk_freq;
+	int error_ppm;
+	int aux_div = 1;
+
+	if (div > (ACLKXDIV_MASK + 1)) {
+		if (reg & AHCLKXE) {
+			aux_div = div / (ACLKXDIV_MASK + 1);
+			if (div % (ACLKXDIV_MASK + 1))
+				aux_div++;
+
+			sysclk_freq /= aux_div;
+			div = sysclk_freq / bclk_freq;
+			rem = sysclk_freq % bclk_freq;
+		} else if (set) {
+			dev_warn(mcasp->dev, "Too fast reference clock (%u)\n",
+				 sysclk_freq);
+		}
+	}
+
+	if (rem != 0) {
+		if (div == 0 ||
+		    ((sysclk_freq / div) - bclk_freq) >
+		    (bclk_freq - (sysclk_freq / (div+1)))) {
+			div++;
+			rem = rem - bclk_freq;
+		}
+	}
+	error_ppm = (div*1000000 + (int)div64_long(1000000LL*rem,
+		     (int)bclk_freq)) / div - 1000000;
+
+	if (set) {
+		if (error_ppm)
+			dev_info(mcasp->dev, "Sample-rate is off by %d PPM\n",
+				 error_ppm);
+
+		__davinci_mcasp_set_clkdiv(mcasp, MCASP_CLKDIV_BCLK, div, 0);
+		if (reg & AHCLKXE)
+			__davinci_mcasp_set_clkdiv(mcasp, MCASP_CLKDIV_AUXCLK,
+						   aux_div, 0);
+	}
+
+	return error_ppm;
+}
+
+static inline u32 davinci_mcasp_tx_delay(struct davinci_mcasp *mcasp)
+{
+	if (!mcasp->txnumevt)
+		return 0;
+
+	return mcasp_get_reg(mcasp, mcasp->fifo_base + MCASP_WFIFOSTS_OFFSET);
+}
+
+static inline u32 davinci_mcasp_rx_delay(struct davinci_mcasp *mcasp)
+{
+	if (!mcasp->rxnumevt)
+		return 0;
+
+	return mcasp_get_reg(mcasp, mcasp->fifo_base + MCASP_RFIFOSTS_OFFSET);
+}
+
+static snd_pcm_sframes_t davinci_mcasp_delay(
+			struct snd_pcm_substream *substream,
+			struct snd_soc_dai *cpu_dai)
+{
+	struct davinci_mcasp *mcasp = snd_soc_dai_get_drvdata(cpu_dai);
+	u32 fifo_use;
+
+	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
+		fifo_use = davinci_mcasp_tx_delay(mcasp);
+	else
+		fifo_use = davinci_mcasp_rx_delay(mcasp);
+
+	/*
+	 * Divide the used locations with the channel count to get the
+	 * FIFO usage in samples (don't care about partial samples in the
+	 * buffer).
+	 */
+	return fifo_use / substream->runtime->channels;
+}
+
+static int davinci_mcasp_hw_params(struct snd_pcm_substream *substream,
+					struct snd_pcm_hw_params *params,
+					struct snd_soc_dai *cpu_dai)
+{
+	struct davinci_mcasp *mcasp = snd_soc_dai_get_drvdata(cpu_dai);
+	int word_length;
+	int channels = params_channels(params);
+	int period_size = params_period_size(params);
+	int ret;
+
+	switch (params_format(params)) {
+	case SNDRV_PCM_FORMAT_U8:
+	case SNDRV_PCM_FORMAT_S8:
+		word_length = 8;
+		break;
+
+	case SNDRV_PCM_FORMAT_U16_LE:
+	case SNDRV_PCM_FORMAT_S16_LE:
+		word_length = 16;
+		break;
+
+	case SNDRV_PCM_FORMAT_U24_3LE:
+	case SNDRV_PCM_FORMAT_S24_3LE:
+		word_length = 24;
+		break;
+
+	case SNDRV_PCM_FORMAT_U24_LE:
+	case SNDRV_PCM_FORMAT_S24_LE:
+		word_length = 24;
+		break;
+
+	case SNDRV_PCM_FORMAT_U32_LE:
+	case SNDRV_PCM_FORMAT_S32_LE:
+		word_length = 32;
+		break;
+
+	default:
+		printk(KERN_WARNING "davinci-mcasp: unsupported PCM format");
+		return -EINVAL;
+	}
+
+	ret = davinci_mcasp_set_dai_fmt(cpu_dai, mcasp->dai_fmt);
+	if (ret)
+		return ret;
+
+	/*
+	 * If mcasp is BCLK master, and a BCLK divider was not provided by
+	 * the machine driver, we need to calculate the ratio.
+	 */
+	if (mcasp->bclk_master && mcasp->bclk_div == 0 && mcasp->sysclk_freq) {
+		int slots = mcasp->tdm_slots;
+		int rate = params_rate(params);
+		int sbits = params_width(params);
+
+		if (mcasp->slot_width)
+			sbits = mcasp->slot_width;
+
+		davinci_mcasp_calc_clk_div(mcasp, mcasp->sysclk_freq,
+					   rate * sbits * slots, true);
+	}
+
+	ret = mcasp_common_hw_param(mcasp, substream->stream,
+				    period_size * channels, channels);
+	if (ret)
+		return ret;
+
+	if (mcasp->op_mode == DAVINCI_MCASP_DIT_MODE)
+		ret = mcasp_dit_hw_param(mcasp, params_rate(params));
+	else
+		ret = mcasp_i2s_hw_param(mcasp, substream->stream,
+					 channels);
+
+	if (ret)
+		return ret;
+
+	davinci_config_channel_size(mcasp, word_length);
+
+	if (mcasp->op_mode == DAVINCI_MCASP_IIS_MODE) {
+		mcasp->channels = channels;
+		if (!mcasp->max_format_width)
+			mcasp->max_format_width = word_length;
+	}
+
+	return 0;
+}
+
+static int davinci_mcasp_trigger(struct snd_pcm_substream *substream,
+				     int cmd, struct snd_soc_dai *cpu_dai)
+{
+	struct davinci_mcasp *mcasp = snd_soc_dai_get_drvdata(cpu_dai);
+	int ret = 0;
+
+	switch (cmd) {
+	case SNDRV_PCM_TRIGGER_RESUME:
+	case SNDRV_PCM_TRIGGER_START:
+	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
+		davinci_mcasp_start(mcasp, substream->stream);
+		break;
+	case SNDRV_PCM_TRIGGER_SUSPEND:
+	case SNDRV_PCM_TRIGGER_STOP:
+	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
+		davinci_mcasp_stop(mcasp, substream->stream);
+		break;
+
+	default:
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
+static int davinci_mcasp_hw_rule_slot_width(struct snd_pcm_hw_params *params,
+					    struct snd_pcm_hw_rule *rule)
+{
+	struct davinci_mcasp_ruledata *rd = rule->private;
+	struct snd_mask *fmt = hw_param_mask(params, SNDRV_PCM_HW_PARAM_FORMAT);
+	struct snd_mask nfmt;
+	int i, slot_width;
+
+	snd_mask_none(&nfmt);
+	slot_width = rd->mcasp->slot_width;
+
+	for (i = 0; i <= SNDRV_PCM_FORMAT_LAST; i++) {
+		if (snd_mask_test(fmt, i)) {
+			if (snd_pcm_format_width(i) <= slot_width) {
+				snd_mask_set(&nfmt, i);
+			}
+		}
+	}
+
+	return snd_mask_refine(fmt, &nfmt);
+}
+
+static int davinci_mcasp_hw_rule_format_width(struct snd_pcm_hw_params *params,
+					      struct snd_pcm_hw_rule *rule)
+{
+	struct davinci_mcasp_ruledata *rd = rule->private;
+	struct snd_mask *fmt = hw_param_mask(params, SNDRV_PCM_HW_PARAM_FORMAT);
+	struct snd_mask nfmt;
+	int i, format_width;
+
+	snd_mask_none(&nfmt);
+	format_width = rd->mcasp->max_format_width;
+
+	for (i = 0; i <= SNDRV_PCM_FORMAT_LAST; i++) {
+		if (snd_mask_test(fmt, i)) {
+			if (snd_pcm_format_width(i) == format_width) {
+				snd_mask_set(&nfmt, i);
+			}
+		}
+	}
+
+	return snd_mask_refine(fmt, &nfmt);
+}
+
+static const unsigned int davinci_mcasp_dai_rates[] = {
+	8000, 11025, 16000, 22050, 32000, 44100, 48000, 64000,
+	88200, 96000, 176400, 192000,
+};
+
+#define DAVINCI_MAX_RATE_ERROR_PPM 1000
+
+static int davinci_mcasp_hw_rule_rate(struct snd_pcm_hw_params *params,
+				      struct snd_pcm_hw_rule *rule)
+{
+	struct davinci_mcasp_ruledata *rd = rule->private;
+	struct snd_interval *ri =
+		hw_param_interval(params, SNDRV_PCM_HW_PARAM_RATE);
+	int sbits = params_width(params);
+	int slots = rd->mcasp->tdm_slots;
+	struct snd_interval range;
+	int i;
+
+	if (rd->mcasp->slot_width)
+		sbits = rd->mcasp->slot_width;
+
+	snd_interval_any(&range);
+	range.empty = 1;
+
+	for (i = 0; i < ARRAY_SIZE(davinci_mcasp_dai_rates); i++) {
+		if (snd_interval_test(ri, davinci_mcasp_dai_rates[i])) {
+			uint bclk_freq = sbits * slots *
+					 davinci_mcasp_dai_rates[i];
+			unsigned int sysclk_freq;
+			int ppm;
+
+			if (rd->mcasp->auxclk_fs_ratio)
+				sysclk_freq =  davinci_mcasp_dai_rates[i] *
+					       rd->mcasp->auxclk_fs_ratio;
+			else
+				sysclk_freq = rd->mcasp->sysclk_freq;
+
+			ppm = davinci_mcasp_calc_clk_div(rd->mcasp, sysclk_freq,
+							 bclk_freq, false);
+			if (abs(ppm) < DAVINCI_MAX_RATE_ERROR_PPM) {
+				if (range.empty) {
+					range.min = davinci_mcasp_dai_rates[i];
+					range.empty = 0;
+				}
+				range.max = davinci_mcasp_dai_rates[i];
+			}
+		}
+	}
+
+	dev_dbg(rd->mcasp->dev,
+		"Frequencies %d-%d -> %d-%d for %d sbits and %d tdm slots\n",
+		ri->min, ri->max, range.min, range.max, sbits, slots);
+
+	return snd_interval_refine(hw_param_interval(params, rule->var),
+				   &range);
+}
+
+static int davinci_mcasp_hw_rule_format(struct snd_pcm_hw_params *params,
+					struct snd_pcm_hw_rule *rule)
+{
+	struct davinci_mcasp_ruledata *rd = rule->private;
+	struct snd_mask *fmt = hw_param_mask(params, SNDRV_PCM_HW_PARAM_FORMAT);
+	struct snd_mask nfmt;
+	int rate = params_rate(params);
+	int slots = rd->mcasp->tdm_slots;
+	int i, count = 0;
+
+	snd_mask_none(&nfmt);
+
+	for (i = 0; i <= SNDRV_PCM_FORMAT_LAST; i++) {
+		if (snd_mask_test(fmt, i)) {
+			uint sbits = snd_pcm_format_width(i);
+			unsigned int sysclk_freq;
+			int ppm;
+
+			if (rd->mcasp->auxclk_fs_ratio)
+				sysclk_freq =  rate *
+					       rd->mcasp->auxclk_fs_ratio;
+			else
+				sysclk_freq = rd->mcasp->sysclk_freq;
+
+			if (rd->mcasp->slot_width)
+				sbits = rd->mcasp->slot_width;
+
+			ppm = davinci_mcasp_calc_clk_div(rd->mcasp, sysclk_freq,
+							 sbits * slots * rate,
+							 false);
+			if (abs(ppm) < DAVINCI_MAX_RATE_ERROR_PPM) {
+				snd_mask_set(&nfmt, i);
+				count++;
+			}
+		}
+	}
+	dev_dbg(rd->mcasp->dev,
+		"%d possible sample format for %d Hz and %d tdm slots\n",
+		count, rate, slots);
+
+	return snd_mask_refine(fmt, &nfmt);
+}
+
+static int davinci_mcasp_hw_rule_min_periodsize(
+		struct snd_pcm_hw_params *params, struct snd_pcm_hw_rule *rule)
+{
+	struct snd_interval *period_size = hw_param_interval(params,
+						SNDRV_PCM_HW_PARAM_PERIOD_SIZE);
+	struct snd_interval frames;
+
+	snd_interval_any(&frames);
+	frames.min = 64;
+	frames.integer = 1;
+
+	return snd_interval_refine(period_size, &frames);
+}
+
+static int davinci_mcasp_startup(struct snd_pcm_substream *substream,
+				 struct snd_soc_dai *cpu_dai)
+{
+	struct davinci_mcasp *mcasp = snd_soc_dai_get_drvdata(cpu_dai);
+	struct davinci_mcasp_ruledata *ruledata =
+					&mcasp->ruledata[substream->stream];
+	u32 max_channels = 0;
+	int i, dir, ret;
+	int tdm_slots = mcasp->tdm_slots;
+
+	/* Do not allow more then one stream per direction */
+	if (mcasp->substreams[substream->stream])
+		return -EBUSY;
+
+	mcasp->substreams[substream->stream] = substream;
+
+	if (mcasp->tdm_mask[substream->stream])
+		tdm_slots = hweight32(mcasp->tdm_mask[substream->stream]);
+
+	if (mcasp->op_mode == DAVINCI_MCASP_DIT_MODE)
+		return 0;
+
+	/*
+	 * Limit the maximum allowed channels for the first stream:
+	 * number of serializers for the direction * tdm slots per serializer
+	 */
+	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
+		dir = TX_MODE;
+	else
+		dir = RX_MODE;
+
+	for (i = 0; i < mcasp->num_serializer; i++) {
+		if (mcasp->serial_dir[i] == dir)
+			max_channels++;
+	}
+	ruledata->serializers = max_channels;
+	ruledata->mcasp = mcasp;
+	max_channels *= tdm_slots;
+	/*
+	 * If the already active stream has less channels than the calculated
+	 * limit based on the seirializers * tdm_slots, and only one serializer
+	 * is in use we need to use that as a constraint for the second stream.
+	 * Otherwise (first stream or less allowed channels or more than one
+	 * serializer in use) we use the calculated constraint.
+	 */
+	if (mcasp->channels && mcasp->channels < max_channels &&
+	    ruledata->serializers == 1)
+		max_channels = mcasp->channels;
+	/*
+	 * But we can always allow channels upto the amount of
+	 * the available tdm_slots.
+	 */
+	if (max_channels < tdm_slots)
+		max_channels = tdm_slots;
+
+	snd_pcm_hw_constraint_minmax(substream->runtime,
+				     SNDRV_PCM_HW_PARAM_CHANNELS,
+				     0, max_channels);
+
+	snd_pcm_hw_constraint_list(substream->runtime,
+				   0, SNDRV_PCM_HW_PARAM_CHANNELS,
+				   &mcasp->chconstr[substream->stream]);
+
+	if (mcasp->max_format_width) {
+		/*
+		 * Only allow formats which require same amount of bits on the
+		 * bus as the currently running stream
+		 */
+		ret = snd_pcm_hw_rule_add(substream->runtime, 0,
+					  SNDRV_PCM_HW_PARAM_FORMAT,
+					  davinci_mcasp_hw_rule_format_width,
+					  ruledata,
+					  SNDRV_PCM_HW_PARAM_FORMAT, -1);
+		if (ret)
+			return ret;
+	}
+	else if (mcasp->slot_width) {
+		/* Only allow formats require <= slot_width bits on the bus */
+		ret = snd_pcm_hw_rule_add(substream->runtime, 0,
+					  SNDRV_PCM_HW_PARAM_FORMAT,
+					  davinci_mcasp_hw_rule_slot_width,
+					  ruledata,
+					  SNDRV_PCM_HW_PARAM_FORMAT, -1);
+		if (ret)
+			return ret;
+	}
+
+	/*
+	 * If we rely on implicit BCLK divider setting we should
+	 * set constraints based on what we can provide.
+	 */
+	if (mcasp->bclk_master && mcasp->bclk_div == 0 && mcasp->sysclk_freq) {
+		ret = snd_pcm_hw_rule_add(substream->runtime, 0,
+					  SNDRV_PCM_HW_PARAM_RATE,
+					  davinci_mcasp_hw_rule_rate,
+					  ruledata,
+					  SNDRV_PCM_HW_PARAM_FORMAT, -1);
+		if (ret)
+			return ret;
+		ret = snd_pcm_hw_rule_add(substream->runtime, 0,
+					  SNDRV_PCM_HW_PARAM_FORMAT,
+					  davinci_mcasp_hw_rule_format,
+					  ruledata,
+					  SNDRV_PCM_HW_PARAM_RATE, -1);
+		if (ret)
+			return ret;
+	}
+
+	snd_pcm_hw_rule_add(substream->runtime, 0,
+			    SNDRV_PCM_HW_PARAM_PERIOD_SIZE,
+			    davinci_mcasp_hw_rule_min_periodsize, NULL,
+			    SNDRV_PCM_HW_PARAM_PERIOD_SIZE, -1);
+
+	return 0;
+}
+
+static void davinci_mcasp_shutdown(struct snd_pcm_substream *substream,
+				   struct snd_soc_dai *cpu_dai)
+{
+	struct davinci_mcasp *mcasp = snd_soc_dai_get_drvdata(cpu_dai);
+
+	mcasp->substreams[substream->stream] = NULL;
+	mcasp->active_serializers[substream->stream] = 0;
+
+	if (mcasp->op_mode == DAVINCI_MCASP_DIT_MODE)
+		return;
+
+	if (!cpu_dai->active) {
+		mcasp->channels = 0;
+		mcasp->max_format_width = 0;
+	}
+}
+
+static const struct snd_soc_dai_ops davinci_mcasp_dai_ops = {
+	.startup	= davinci_mcasp_startup,
+	.shutdown	= davinci_mcasp_shutdown,
+	.trigger	= davinci_mcasp_trigger,
+	.delay		= davinci_mcasp_delay,
+	.hw_params	= davinci_mcasp_hw_params,
+	.set_fmt	= davinci_mcasp_set_dai_fmt,
+	.set_clkdiv	= davinci_mcasp_set_clkdiv,
+	.set_sysclk	= davinci_mcasp_set_sysclk,
+	.set_tdm_slot	= davinci_mcasp_set_tdm_slot,
+};
+
+static int davinci_mcasp_dai_probe(struct snd_soc_dai *dai)
+{
+	struct davinci_mcasp *mcasp = snd_soc_dai_get_drvdata(dai);
+
+	dai->playback_dma_data = &mcasp->dma_data[SNDRV_PCM_STREAM_PLAYBACK];
+	dai->capture_dma_data = &mcasp->dma_data[SNDRV_PCM_STREAM_CAPTURE];
+
+	return 0;
+}
+
+#define DAVINCI_MCASP_RATES	SNDRV_PCM_RATE_8000_192000
+
+#define DAVINCI_MCASP_PCM_FMTS (SNDRV_PCM_FMTBIT_S8 | \
+				SNDRV_PCM_FMTBIT_U8 | \
+				SNDRV_PCM_FMTBIT_S16_LE | \
+				SNDRV_PCM_FMTBIT_U16_LE | \
+				SNDRV_PCM_FMTBIT_S24_LE | \
+				SNDRV_PCM_FMTBIT_U24_LE | \
+				SNDRV_PCM_FMTBIT_S24_3LE | \
+				SNDRV_PCM_FMTBIT_U24_3LE | \
+				SNDRV_PCM_FMTBIT_S32_LE | \
+				SNDRV_PCM_FMTBIT_U32_LE)
+
+static struct snd_soc_dai_driver davinci_mcasp_dai[] = {
+	{
+		.name		= "davinci-mcasp.0",
+		.probe		= davinci_mcasp_dai_probe,
+		.playback	= {
+			.channels_min	= 1,
+			.channels_max	= 32 * 16,
+			.rates 		= DAVINCI_MCASP_RATES,
+			.formats	= DAVINCI_MCASP_PCM_FMTS,
+		},
+		.capture 	= {
+			.channels_min 	= 1,
+			.channels_max	= 32 * 16,
+			.rates 		= DAVINCI_MCASP_RATES,
+			.formats	= DAVINCI_MCASP_PCM_FMTS,
+		},
+		.ops 		= &davinci_mcasp_dai_ops,
+
+		.symmetric_rates	= 1,
+	},
+	{
+		.name		= "davinci-mcasp.1",
+		.probe		= davinci_mcasp_dai_probe,
+		.playback 	= {
+			.channels_min	= 1,
+			.channels_max	= 384,
+			.rates		= DAVINCI_MCASP_RATES,
+			.formats	= DAVINCI_MCASP_PCM_FMTS,
+		},
+		.ops 		= &davinci_mcasp_dai_ops,
+	},
+
+};
+
+static const struct snd_soc_component_driver davinci_mcasp_component = {
+	.name		= "davinci-mcasp",
+};
+
+/* Some HW specific values and defaults. The rest is filled in from DT. */
+static struct davinci_mcasp_pdata dm646x_mcasp_pdata = {
+	.tx_dma_offset = 0x400,
+	.rx_dma_offset = 0x400,
+	.version = MCASP_VERSION_1,
+};
+
+static struct davinci_mcasp_pdata da830_mcasp_pdata = {
+	.tx_dma_offset = 0x2000,
+	.rx_dma_offset = 0x2000,
+	.version = MCASP_VERSION_2,
+};
+
+static struct davinci_mcasp_pdata am33xx_mcasp_pdata = {
+	.tx_dma_offset = 0,
+	.rx_dma_offset = 0,
+	.version = MCASP_VERSION_3,
+};
+
+static struct davinci_mcasp_pdata dra7_mcasp_pdata = {
+	/* The CFG port offset will be calculated if it is needed */
+	.tx_dma_offset = 0,
+	.rx_dma_offset = 0,
+	.version = MCASP_VERSION_4,
+};
+
+static const struct of_device_id mcasp_dt_ids[] = {
+	{
+		.compatible = "ti,dm646x-mcasp-audio",
+		.data = &dm646x_mcasp_pdata,
+	},
+	{
+		.compatible = "ti,da830-mcasp-audio",
+		.data = &da830_mcasp_pdata,
+	},
+	{
+		.compatible = "ti,am33xx-mcasp-audio",
+		.data = &am33xx_mcasp_pdata,
+	},
+	{
+		.compatible = "ti,dra7-mcasp-audio",
+		.data = &dra7_mcasp_pdata,
+	},
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, mcasp_dt_ids);
+
+static int mcasp_reparent_fck(struct platform_device *pdev)
+{
+	struct device_node *node = pdev->dev.of_node;
+	struct clk *gfclk, *parent_clk;
+	const char *parent_name;
+	int ret;
+
+	if (!node)
+		return 0;
+
+	parent_name = of_get_property(node, "fck_parent", NULL);
+	if (!parent_name)
+		return 0;
+
+	dev_warn(&pdev->dev, "Update the bindings to use assigned-clocks!\n");
+
+	gfclk = clk_get(&pdev->dev, "fck");
+	if (IS_ERR(gfclk)) {
+		dev_err(&pdev->dev, "failed to get fck\n");
+		return PTR_ERR(gfclk);
+	}
+
+	parent_clk = clk_get(NULL, parent_name);
+	if (IS_ERR(parent_clk)) {
+		dev_err(&pdev->dev, "failed to get parent clock\n");
+		ret = PTR_ERR(parent_clk);
+		goto err1;
+	}
+
+	ret = clk_set_parent(gfclk, parent_clk);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to reparent fck\n");
+		goto err2;
+	}
+
+err2:
+	clk_put(parent_clk);
+err1:
+	clk_put(gfclk);
+	return ret;
+}
+
+static struct davinci_mcasp_pdata *davinci_mcasp_set_pdata_from_of(
+						struct platform_device *pdev)
+{
+	struct device_node *np = pdev->dev.of_node;
+	struct davinci_mcasp_pdata *pdata = NULL;
+	const struct of_device_id *match =
+			of_match_device(mcasp_dt_ids, &pdev->dev);
+	struct of_phandle_args dma_spec;
+
+	const u32 *of_serial_dir32;
+	u32 val;
+	int i, ret = 0;
+
+	if (pdev->dev.platform_data) {
+		pdata = pdev->dev.platform_data;
+		pdata->dismod = DISMOD_LOW;
+		return pdata;
+	} else if (match) {
+		pdata = devm_kmemdup(&pdev->dev, match->data, sizeof(*pdata),
+				     GFP_KERNEL);
+		if (!pdata) {
+			ret = -ENOMEM;
+			return pdata;
+		}
+	} else {
+		/* control shouldn't reach here. something is wrong */
+		ret = -EINVAL;
+		goto nodata;
+	}
+
+	ret = of_property_read_u32(np, "op-mode", &val);
+	if (ret >= 0)
+		pdata->op_mode = val;
+
+	ret = of_property_read_u32(np, "tdm-slots", &val);
+	if (ret >= 0) {
+		if (val < 2 || val > 32) {
+			dev_err(&pdev->dev,
+				"tdm-slots must be in rage [2-32]\n");
+			ret = -EINVAL;
+			goto nodata;
+		}
+
+		pdata->tdm_slots = val;
+	}
+
+	of_serial_dir32 = of_get_property(np, "serial-dir", &val);
+	val /= sizeof(u32);
+	if (of_serial_dir32) {
+		u8 *of_serial_dir = devm_kzalloc(&pdev->dev,
+						 (sizeof(*of_serial_dir) * val),
+						 GFP_KERNEL);
+		if (!of_serial_dir) {
+			ret = -ENOMEM;
+			goto nodata;
+		}
+
+		for (i = 0; i < val; i++)
+			of_serial_dir[i] = be32_to_cpup(&of_serial_dir32[i]);
+
+		pdata->num_serializer = val;
+		pdata->serial_dir = of_serial_dir;
+	}
+
+	ret = of_property_match_string(np, "dma-names", "tx");
+	if (ret < 0)
+		goto nodata;
+
+	ret = of_parse_phandle_with_args(np, "dmas", "#dma-cells", ret,
+					 &dma_spec);
+	if (ret < 0)
+		goto nodata;
+
+	pdata->tx_dma_channel = dma_spec.args[0];
+
+	/* RX is not valid in DIT mode */
+	if (pdata->op_mode != DAVINCI_MCASP_DIT_MODE) {
+		ret = of_property_match_string(np, "dma-names", "rx");
+		if (ret < 0)
+			goto nodata;
+
+		ret = of_parse_phandle_with_args(np, "dmas", "#dma-cells", ret,
+						 &dma_spec);
+		if (ret < 0)
+			goto nodata;
+
+		pdata->rx_dma_channel = dma_spec.args[0];
+	}
+
+	ret = of_property_read_u32(np, "tx-num-evt", &val);
+	if (ret >= 0)
+		pdata->txnumevt = val;
+
+	ret = of_property_read_u32(np, "rx-num-evt", &val);
+	if (ret >= 0)
+		pdata->rxnumevt = val;
+
+	ret = of_property_read_u32(np, "sram-size-playback", &val);
+	if (ret >= 0)
+		pdata->sram_size_playback = val;
+
+	ret = of_property_read_u32(np, "sram-size-capture", &val);
+	if (ret >= 0)
+		pdata->sram_size_capture = val;
+
+	ret = of_property_read_u32(np, "dismod", &val);
+	if (ret >= 0) {
+		if (val == 0 || val == 2 || val == 3) {
+			pdata->dismod = DISMOD_VAL(val);
+		} else {
+			dev_warn(&pdev->dev, "Invalid dismod value: %u\n", val);
+			pdata->dismod = DISMOD_LOW;
+		}
+	} else {
+		pdata->dismod = DISMOD_LOW;
+	}
+
+	return  pdata;
+
+nodata:
+	if (ret < 0) {
+		dev_err(&pdev->dev, "Error populating platform data, err %d\n",
+			ret);
+		pdata = NULL;
+	}
+	return  pdata;
+}
+
+enum {
+	PCM_EDMA,
+	PCM_SDMA,
+	PCM_UDMA,
+};
+static const char *sdma_prefix = "ti,omap";
+
+static int davinci_mcasp_get_dma_type(struct davinci_mcasp *mcasp)
+{
+	struct dma_chan *chan;
+	const char *tmp;
+	int ret = PCM_EDMA;
+
+	if (!mcasp->dev->of_node)
+		return PCM_EDMA;
+
+	tmp = mcasp->dma_data[SNDRV_PCM_STREAM_PLAYBACK].filter_data;
+	chan = dma_request_slave_channel_reason(mcasp->dev, tmp);
+	if (IS_ERR(chan)) {
+		if (PTR_ERR(chan) != -EPROBE_DEFER)
+			dev_err(mcasp->dev,
+				"Can't verify DMA configuration (%ld)\n",
+				PTR_ERR(chan));
+		return PTR_ERR(chan);
+	}
+	if (WARN_ON(!chan->device || !chan->device->dev))
+		return -EINVAL;
+
+	if (chan->device->dev->of_node)
+		ret = of_property_read_string(chan->device->dev->of_node,
+					      "compatible", &tmp);
+	else
+		dev_dbg(mcasp->dev, "DMA controller has no of-node\n");
+
+	dma_release_channel(chan);
+	if (ret)
+		return ret;
+
+	dev_dbg(mcasp->dev, "DMA controller compatible = \"%s\"\n", tmp);
+	if (!strncmp(tmp, sdma_prefix, strlen(sdma_prefix)))
+		return PCM_SDMA;
+	else if (strstr(tmp, "udmap"))
+		return PCM_UDMA;
+
+	return PCM_EDMA;
+}
+
+static u32 davinci_mcasp_txdma_offset(struct davinci_mcasp_pdata *pdata)
+{
+	int i;
+	u32 offset = 0;
+
+	if (pdata->version != MCASP_VERSION_4)
+		return pdata->tx_dma_offset;
+
+	for (i = 0; i < pdata->num_serializer; i++) {
+		if (pdata->serial_dir[i] == TX_MODE) {
+			if (!offset) {
+				offset = DAVINCI_MCASP_TXBUF_REG(i);
+			} else {
+				pr_err("%s: Only one serializer allowed!\n",
+				       __func__);
+				break;
+			}
+		}
+	}
+
+	return offset;
+}
+
+static u32 davinci_mcasp_rxdma_offset(struct davinci_mcasp_pdata *pdata)
+{
+	int i;
+	u32 offset = 0;
+
+	if (pdata->version != MCASP_VERSION_4)
+		return pdata->rx_dma_offset;
+
+	for (i = 0; i < pdata->num_serializer; i++) {
+		if (pdata->serial_dir[i] == RX_MODE) {
+			if (!offset) {
+				offset = DAVINCI_MCASP_RXBUF_REG(i);
+			} else {
+				pr_err("%s: Only one serializer allowed!\n",
+				       __func__);
+				break;
+			}
+		}
+	}
+
+	return offset;
+}
+
+#ifdef CONFIG_GPIOLIB
+static int davinci_mcasp_gpio_request(struct gpio_chip *chip, unsigned offset)
+{
+	struct davinci_mcasp *mcasp = gpiochip_get_data(chip);
+
+	if (mcasp->num_serializer && offset < mcasp->num_serializer &&
+	    mcasp->serial_dir[offset] != INACTIVE_MODE) {
+		dev_err(mcasp->dev, "AXR%u pin is  used for audio\n", offset);
+		return -EBUSY;
+	}
+
+	/* Do not change the PIN yet */
+
+	return pm_runtime_get_sync(mcasp->dev);
+}
+
+static void davinci_mcasp_gpio_free(struct gpio_chip *chip, unsigned offset)
+{
+	struct davinci_mcasp *mcasp = gpiochip_get_data(chip);
+
+	/* Set the direction to input */
+	mcasp_clr_bits(mcasp, DAVINCI_MCASP_PDIR_REG, BIT(offset));
+
+	/* Set the pin as McASP pin */
+	mcasp_clr_bits(mcasp, DAVINCI_MCASP_PFUNC_REG, BIT(offset));
+
+	pm_runtime_put_sync(mcasp->dev);
+}
+
+static int davinci_mcasp_gpio_direction_out(struct gpio_chip *chip,
+					    unsigned offset, int value)
+{
+	struct davinci_mcasp *mcasp = gpiochip_get_data(chip);
+	u32 val;
+
+	if (value)
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_PDOUT_REG, BIT(offset));
+	else
+		mcasp_clr_bits(mcasp, DAVINCI_MCASP_PDOUT_REG, BIT(offset));
+
+	val = mcasp_get_reg(mcasp, DAVINCI_MCASP_PFUNC_REG);
+	if (!(val & BIT(offset))) {
+		/* Set the pin as GPIO pin */
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_PFUNC_REG, BIT(offset));
+
+		/* Set the direction to output */
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_PDIR_REG, BIT(offset));
+	}
+
+	return 0;
+}
+
+static void davinci_mcasp_gpio_set(struct gpio_chip *chip, unsigned offset,
+				  int value)
+{
+	struct davinci_mcasp *mcasp = gpiochip_get_data(chip);
+
+	if (value)
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_PDOUT_REG, BIT(offset));
+	else
+		mcasp_clr_bits(mcasp, DAVINCI_MCASP_PDOUT_REG, BIT(offset));
+}
+
+static int davinci_mcasp_gpio_direction_in(struct gpio_chip *chip,
+					   unsigned offset)
+{
+	struct davinci_mcasp *mcasp = gpiochip_get_data(chip);
+	u32 val;
+
+	val = mcasp_get_reg(mcasp, DAVINCI_MCASP_PFUNC_REG);
+	if (!(val & BIT(offset))) {
+		/* Set the direction to input */
+		mcasp_clr_bits(mcasp, DAVINCI_MCASP_PDIR_REG, BIT(offset));
+
+		/* Set the pin as GPIO pin */
+		mcasp_set_bits(mcasp, DAVINCI_MCASP_PFUNC_REG, BIT(offset));
+	}
+
+	return 0;
+}
+
+static int davinci_mcasp_gpio_get(struct gpio_chip *chip, unsigned offset)
+{
+	struct davinci_mcasp *mcasp = gpiochip_get_data(chip);
+	u32 val;
+
+	val = mcasp_get_reg(mcasp, DAVINCI_MCASP_PDSET_REG);
+	if (val & BIT(offset))
+		return 1;
+
+	return 0;
+}
+
+static int davinci_mcasp_gpio_get_direction(struct gpio_chip *chip,
+					    unsigned offset)
+{
+	struct davinci_mcasp *mcasp = gpiochip_get_data(chip);
+	u32 val;
+
+	val = mcasp_get_reg(mcasp, DAVINCI_MCASP_PDIR_REG);
+	if (val & BIT(offset))
+		return 0;
+
+	return 1;
+}
+
+static const struct gpio_chip davinci_mcasp_template_chip = {
+	.owner			= THIS_MODULE,
+	.request		= davinci_mcasp_gpio_request,
+	.free			= davinci_mcasp_gpio_free,
+	.direction_output	= davinci_mcasp_gpio_direction_out,
+	.set			= davinci_mcasp_gpio_set,
+	.direction_input	= davinci_mcasp_gpio_direction_in,
+	.get			= davinci_mcasp_gpio_get,
+	.get_direction		= davinci_mcasp_gpio_get_direction,
+	.base			= -1,
+	.ngpio			= 32,
+};
+
+static int davinci_mcasp_init_gpiochip(struct davinci_mcasp *mcasp)
+{
+	if (!of_property_read_bool(mcasp->dev->of_node, "gpio-controller"))
+		return 0;
+
+	mcasp->gpio_chip = davinci_mcasp_template_chip;
+	mcasp->gpio_chip.label = dev_name(mcasp->dev);
+	mcasp->gpio_chip.parent = mcasp->dev;
+#ifdef CONFIG_OF_GPIO
+	mcasp->gpio_chip.of_node = mcasp->dev->of_node;
+#endif
+
+	return devm_gpiochip_add_data(mcasp->dev, &mcasp->gpio_chip, mcasp);
+}
+
+#else /* CONFIG_GPIOLIB */
+static inline int davinci_mcasp_init_gpiochip(struct davinci_mcasp *mcasp)
+{
+	return 0;
+}
+#endif /* CONFIG_GPIOLIB */
+
+static int davinci_mcasp_get_dt_params(struct davinci_mcasp *mcasp)
+{
+	struct device_node *np = mcasp->dev->of_node;
+	int ret;
+	u32 val;
+
+	if (!np)
+		return 0;
+
+	ret = of_property_read_u32(np, "auxclk-fs-ratio", &val);
+	if (ret >= 0)
+		mcasp->auxclk_fs_ratio = val;
+
+	return 0;
+}
+
+static int davinci_mcasp_probe(struct platform_device *pdev)
+{
+	struct snd_dmaengine_dai_dma_data *dma_data;
+	struct resource *mem, *res, *dat;
+	struct davinci_mcasp_pdata *pdata;
+	struct davinci_mcasp *mcasp;
+	char *irq_name;
+	int *dma;
+	int irq;
+	int ret;
+
+	if (!pdev->dev.platform_data && !pdev->dev.of_node) {
+		dev_err(&pdev->dev, "No platform data supplied\n");
+		return -EINVAL;
+	}
+
+	mcasp = devm_kzalloc(&pdev->dev, sizeof(struct davinci_mcasp),
+			   GFP_KERNEL);
+	if (!mcasp)
+		return	-ENOMEM;
+
+	pdata = davinci_mcasp_set_pdata_from_of(pdev);
+	if (!pdata) {
+		dev_err(&pdev->dev, "no platform data\n");
+		return -EINVAL;
+	}
+
+	mem = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mpu");
+	if (!mem) {
+		dev_warn(mcasp->dev,
+			 "\"mpu\" mem resource not found, using index 0\n");
+		mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+		if (!mem) {
+			dev_err(&pdev->dev, "no mem resource?\n");
+			return -ENODEV;
+		}
+	}
+
+	mcasp->base = devm_ioremap_resource(&pdev->dev, mem);
+	if (IS_ERR(mcasp->base))
+		return PTR_ERR(mcasp->base);
+
+	pm_runtime_enable(&pdev->dev);
+
+	mcasp->op_mode = pdata->op_mode;
+	/* sanity check for tdm slots parameter */
+	if (mcasp->op_mode == DAVINCI_MCASP_IIS_MODE) {
+		if (pdata->tdm_slots < 2) {
+			dev_err(&pdev->dev, "invalid tdm slots: %d\n",
+				pdata->tdm_slots);
+			mcasp->tdm_slots = 2;
+		} else if (pdata->tdm_slots > 32) {
+			dev_err(&pdev->dev, "invalid tdm slots: %d\n",
+				pdata->tdm_slots);
+			mcasp->tdm_slots = 32;
+		} else {
+			mcasp->tdm_slots = pdata->tdm_slots;
+		}
+	}
+
+	mcasp->num_serializer = pdata->num_serializer;
+#ifdef CONFIG_PM
+	mcasp->context.xrsr_regs = devm_kcalloc(&pdev->dev,
+					mcasp->num_serializer, sizeof(u32),
+					GFP_KERNEL);
+	if (!mcasp->context.xrsr_regs) {
+		ret = -ENOMEM;
+		goto err;
+	}
+#endif
+	mcasp->serial_dir = pdata->serial_dir;
+	mcasp->version = pdata->version;
+	mcasp->txnumevt = pdata->txnumevt;
+	mcasp->rxnumevt = pdata->rxnumevt;
+	mcasp->dismod = pdata->dismod;
+
+	mcasp->dev = &pdev->dev;
+
+	irq = platform_get_irq_byname(pdev, "common");
+	if (irq >= 0) {
+		irq_name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s_common",
+					  dev_name(&pdev->dev));
+		if (!irq_name) {
+			ret = -ENOMEM;
+			goto err;
+		}
+		ret = devm_request_threaded_irq(&pdev->dev, irq, NULL,
+						davinci_mcasp_common_irq_handler,
+						IRQF_ONESHOT | IRQF_SHARED,
+						irq_name, mcasp);
+		if (ret) {
+			dev_err(&pdev->dev, "common IRQ request failed\n");
+			goto err;
+		}
+
+		mcasp->irq_request[SNDRV_PCM_STREAM_PLAYBACK] = XUNDRN;
+		mcasp->irq_request[SNDRV_PCM_STREAM_CAPTURE] = ROVRN;
+	}
+
+	irq = platform_get_irq_byname(pdev, "rx");
+	if (irq >= 0) {
+		irq_name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s_rx",
+					  dev_name(&pdev->dev));
+		if (!irq_name) {
+			ret = -ENOMEM;
+			goto err;
+		}
+		ret = devm_request_threaded_irq(&pdev->dev, irq, NULL,
+						davinci_mcasp_rx_irq_handler,
+						IRQF_ONESHOT, irq_name, mcasp);
+		if (ret) {
+			dev_err(&pdev->dev, "RX IRQ request failed\n");
+			goto err;
+		}
+
+		mcasp->irq_request[SNDRV_PCM_STREAM_CAPTURE] = ROVRN;
+	}
+
+	irq = platform_get_irq_byname(pdev, "tx");
+	if (irq >= 0) {
+		irq_name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s_tx",
+					  dev_name(&pdev->dev));
+		if (!irq_name) {
+			ret = -ENOMEM;
+			goto err;
+		}
+		ret = devm_request_threaded_irq(&pdev->dev, irq, NULL,
+						davinci_mcasp_tx_irq_handler,
+						IRQF_ONESHOT, irq_name, mcasp);
+		if (ret) {
+			dev_err(&pdev->dev, "TX IRQ request failed\n");
+			goto err;
+		}
+
+		mcasp->irq_request[SNDRV_PCM_STREAM_PLAYBACK] = XUNDRN;
+	}
+
+	dat = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dat");
+	if (dat)
+		mcasp->dat_port = true;
+
+	dma_data = &mcasp->dma_data[SNDRV_PCM_STREAM_PLAYBACK];
+	if (dat)
+		dma_data->addr = dat->start;
+	else
+		dma_data->addr = mem->start + davinci_mcasp_txdma_offset(pdata);
+
+	dma = &mcasp->dma_request[SNDRV_PCM_STREAM_PLAYBACK];
+	res = platform_get_resource(pdev, IORESOURCE_DMA, 0);
+	if (res)
+		*dma = res->start;
+	else
+		*dma = pdata->tx_dma_channel;
+
+	/* dmaengine filter data for DT and non-DT boot */
+	if (pdev->dev.of_node)
+		dma_data->filter_data = "tx";
+	else
+		dma_data->filter_data = dma;
+
+	/* RX is not valid in DIT mode */
+	if (mcasp->op_mode != DAVINCI_MCASP_DIT_MODE) {
+		dma_data = &mcasp->dma_data[SNDRV_PCM_STREAM_CAPTURE];
+		if (dat)
+			dma_data->addr = dat->start;
+		else
+			dma_data->addr =
+				mem->start + davinci_mcasp_rxdma_offset(pdata);
+
+		dma = &mcasp->dma_request[SNDRV_PCM_STREAM_CAPTURE];
+		res = platform_get_resource(pdev, IORESOURCE_DMA, 1);
+		if (res)
+			*dma = res->start;
+		else
+			*dma = pdata->rx_dma_channel;
+
+		/* dmaengine filter data for DT and non-DT boot */
+		if (pdev->dev.of_node)
+			dma_data->filter_data = "rx";
+		else
+			dma_data->filter_data = dma;
+	}
+
+	if (mcasp->version < MCASP_VERSION_3) {
+		mcasp->fifo_base = DAVINCI_MCASP_V2_AFIFO_BASE;
+		/* dma_params->dma_addr is pointing to the data port address */
+		mcasp->dat_port = true;
+	} else {
+		mcasp->fifo_base = DAVINCI_MCASP_V3_AFIFO_BASE;
+	}
+
+	/* Allocate memory for long enough list for all possible
+	 * scenarios. Maximum number tdm slots is 32 and there cannot
+	 * be more serializers than given in the configuration.  The
+	 * serializer directions could be taken into account, but it
+	 * would make code much more complex and save only couple of
+	 * bytes.
+	 */
+	mcasp->chconstr[SNDRV_PCM_STREAM_PLAYBACK].list =
+		devm_kcalloc(mcasp->dev,
+			     32 + mcasp->num_serializer - 1,
+			     sizeof(unsigned int),
+			     GFP_KERNEL);
+
+	mcasp->chconstr[SNDRV_PCM_STREAM_CAPTURE].list =
+		devm_kcalloc(mcasp->dev,
+			     32 + mcasp->num_serializer - 1,
+			     sizeof(unsigned int),
+			     GFP_KERNEL);
+
+	if (!mcasp->chconstr[SNDRV_PCM_STREAM_PLAYBACK].list ||
+	    !mcasp->chconstr[SNDRV_PCM_STREAM_CAPTURE].list) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	ret = davinci_mcasp_set_ch_constraints(mcasp);
+	if (ret)
+		goto err;
+
+	dev_set_drvdata(&pdev->dev, mcasp);
+
+	mcasp_reparent_fck(pdev);
+
+	if (mcasp->version == MCASP_VERSION_4) {
+		u32 rev;
+
+		pm_runtime_get_sync(mcasp->dev);
+		rev = mcasp_get_reg(mcasp, DAVINCI_MCASP_PID_REG) &
+				    MCASP_V4_REV_MASK;
+		pm_runtime_put(mcasp->dev);
+
+		if (rev < MCASP_V4_REV(3, 3)) {
+			/*
+			 * ERRATA i868: to avoid race condition between DMA and
+			 * AFIFO events the R/WNUMEVT need to be set to be
+			 * less-than-equal to 32 words.
+			 */
+			if (mcasp->txnumevt)
+				mcasp->txnumevt = 32;
+			if (mcasp->rxnumevt)
+				mcasp->rxnumevt = 32;
+
+			if (mcasp->txnumevt || mcasp->rxnumevt)
+				dev_info(&pdev->dev,
+					 "ERRATA i868 workaround is enabled\n");
+		}
+	}
+
+	/* All PINS as McASP */
+	pm_runtime_get_sync(mcasp->dev);
+	mcasp_set_reg(mcasp, DAVINCI_MCASP_PFUNC_REG, 0x00000000);
+	pm_runtime_put(mcasp->dev);
+
+	ret = davinci_mcasp_init_gpiochip(mcasp);
+	if (ret)
+		goto err;
+
+	ret = davinci_mcasp_get_dt_params(mcasp);
+	if (ret)
+		return -EINVAL;
+
+	ret = devm_snd_soc_register_component(&pdev->dev,
+					&davinci_mcasp_component,
+					&davinci_mcasp_dai[pdata->op_mode], 1);
+
+	if (ret != 0)
+		goto err;
+
+	ret = davinci_mcasp_get_dma_type(mcasp);
+	switch (ret) {
+	case PCM_EDMA:
+#if IS_BUILTIN(CONFIG_SND_SOC_TI_EDMA_PCM) || \
+	(IS_MODULE(CONFIG_SND_SOC_DAVINCI_MCASP) && \
+	 IS_MODULE(CONFIG_SND_SOC_TI_EDMA_PCM))
+		ret = edma_pcm_platform_register(&pdev->dev);
+#else
+		dev_err(&pdev->dev, "Missing SND_EDMA_SOC\n");
+		ret = -EINVAL;
+		goto err;
+#endif
+		break;
+	case PCM_SDMA:
+#if IS_BUILTIN(CONFIG_SND_SOC_TI_SDMA_PCM) || \
+	(IS_MODULE(CONFIG_SND_SOC_DAVINCI_MCASP) && \
+	 IS_MODULE(CONFIG_SND_SOC_TI_SDMA_PCM))
+		ret = sdma_pcm_platform_register(&pdev->dev, NULL, NULL);
+#else
+		dev_err(&pdev->dev, "Missing SND_SDMA_SOC\n");
+		ret = -EINVAL;
+		goto err;
+#endif
+		break;
+	case PCM_UDMA:
+#if IS_BUILTIN(CONFIG_SND_SOC_TI_UDMA_PCM) || \
+	(IS_MODULE(CONFIG_SND_SOC_DAVINCI_MCASP) && \
+	 IS_MODULE(CONFIG_SND_SOC_TI_UDMA_PCM))
+		ret = udma_pcm_platform_register(&pdev->dev);
+#else
+		dev_err(&pdev->dev, "Missing SND_SOC_TI_UDMA_PCM\n");
+		ret = -EINVAL;
+		goto err;
+#endif
+		break;
+	default:
+		dev_err(&pdev->dev, "No DMA controller found (%d)\n", ret);
+	case -EPROBE_DEFER:
+		goto err;
+		break;
+	}
+
+	if (ret) {
+		dev_err(&pdev->dev, "register PCM failed: %d\n", ret);
+		goto err;
+	}
+
+	return 0;
+
+err:
+	pm_runtime_disable(&pdev->dev);
+	return ret;
+}
+
+static int davinci_mcasp_remove(struct platform_device *pdev)
+{
+	pm_runtime_disable(&pdev->dev);
+
+	return 0;
+}
+
+#ifdef CONFIG_PM
+static int davinci_mcasp_runtime_suspend(struct device *dev)
+{
+	struct davinci_mcasp *mcasp = dev_get_drvdata(dev);
+	struct davinci_mcasp_context *context = &mcasp->context;
+	u32 reg;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(context_regs); i++)
+		context->config_regs[i] = mcasp_get_reg(mcasp, context_regs[i]);
+
+	if (mcasp->txnumevt) {
+		reg = mcasp->fifo_base + MCASP_WFIFOCTL_OFFSET;
+		context->afifo_regs[0] = mcasp_get_reg(mcasp, reg);
+	}
+	if (mcasp->rxnumevt) {
+		reg = mcasp->fifo_base + MCASP_RFIFOCTL_OFFSET;
+		context->afifo_regs[1] = mcasp_get_reg(mcasp, reg);
+	}
+
+	for (i = 0; i < mcasp->num_serializer; i++)
+		context->xrsr_regs[i] = mcasp_get_reg(mcasp,
+						DAVINCI_MCASP_XRSRCTL_REG(i));
+
+	return 0;
+}
+
+static int davinci_mcasp_runtime_resume(struct device *dev)
+{
+	struct davinci_mcasp *mcasp = dev_get_drvdata(dev);
+	struct davinci_mcasp_context *context = &mcasp->context;
+	u32 reg;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(context_regs); i++)
+		mcasp_set_reg(mcasp, context_regs[i], context->config_regs[i]);
+
+	if (mcasp->txnumevt) {
+		reg = mcasp->fifo_base + MCASP_WFIFOCTL_OFFSET;
+		mcasp_set_reg(mcasp, reg, context->afifo_regs[0]);
+	}
+	if (mcasp->rxnumevt) {
+		reg = mcasp->fifo_base + MCASP_RFIFOCTL_OFFSET;
+		mcasp_set_reg(mcasp, reg, context->afifo_regs[1]);
+	}
+
+	for (i = 0; i < mcasp->num_serializer; i++)
+		mcasp_set_reg(mcasp, DAVINCI_MCASP_XRSRCTL_REG(i),
+			      context->xrsr_regs[i]);
+
+	return 0;
+}
+
+#endif
+
+static const struct dev_pm_ops davinci_mcasp_pm_ops = {
+	SET_RUNTIME_PM_OPS(davinci_mcasp_runtime_suspend,
+			   davinci_mcasp_runtime_resume,
+			   NULL)
+};
+
+static struct platform_driver davinci_mcasp_driver = {
+	.probe		= davinci_mcasp_probe,
+	.remove		= davinci_mcasp_remove,
+	.driver		= {
+		.name	= "davinci-mcasp",
+		.pm     = &davinci_mcasp_pm_ops,
+		.of_match_table = mcasp_dt_ids,
+	},
+};
+
+module_platform_driver(davinci_mcasp_driver);
+
+MODULE_AUTHOR("Steve Chen");
+MODULE_DESCRIPTION("TI DAVINCI McASP SoC Interface");
+MODULE_LICENSE("GPL");
diff -Naur --no-dereference a/sound/soc/ti/davinci-mcasp.h b/sound/soc/ti/davinci-mcasp.h
--- a/sound/soc/ti/davinci-mcasp.h	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/davinci-mcasp.h	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,308 @@
+/*
+ * ALSA SoC McASP Audio Layer for TI DAVINCI processor
+ *
+ * MCASP related definitions
+ *
+ * Author: Nirmal Pandey <n-pandey@ti.com>,
+ *         Suresh Rajashekara <suresh.r@ti.com>
+ *         Steve Chen <schen@.mvista.com>
+ *
+ * Copyright:   (C) 2009 MontaVista Software, Inc., <source@mvista.com>
+ * Copyright:   (C) 2009  Texas Instruments, India
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef DAVINCI_MCASP_H
+#define DAVINCI_MCASP_H
+
+/*
+ * McASP register definitions
+ */
+#define DAVINCI_MCASP_PID_REG		0x00
+#define DAVINCI_MCASP_PWREMUMGT_REG	0x04
+
+#define DAVINCI_MCASP_PFUNC_REG		0x10
+#define DAVINCI_MCASP_PDIR_REG		0x14
+#define DAVINCI_MCASP_PDOUT_REG		0x18
+#define DAVINCI_MCASP_PDSET_REG		0x1c
+
+#define DAVINCI_MCASP_PDCLR_REG		0x20
+
+#define DAVINCI_MCASP_TLGC_REG		0x30
+#define DAVINCI_MCASP_TLMR_REG		0x34
+
+#define DAVINCI_MCASP_GBLCTL_REG	0x44
+#define DAVINCI_MCASP_AMUTE_REG		0x48
+#define DAVINCI_MCASP_LBCTL_REG		0x4c
+
+#define DAVINCI_MCASP_TXDITCTL_REG	0x50
+
+#define DAVINCI_MCASP_GBLCTLR_REG	0x60
+#define DAVINCI_MCASP_RXMASK_REG	0x64
+#define DAVINCI_MCASP_RXFMT_REG		0x68
+#define DAVINCI_MCASP_RXFMCTL_REG	0x6c
+
+#define DAVINCI_MCASP_ACLKRCTL_REG	0x70
+#define DAVINCI_MCASP_AHCLKRCTL_REG	0x74
+#define DAVINCI_MCASP_RXTDM_REG		0x78
+#define DAVINCI_MCASP_EVTCTLR_REG	0x7c
+
+#define DAVINCI_MCASP_RXSTAT_REG	0x80
+#define DAVINCI_MCASP_RXTDMSLOT_REG	0x84
+#define DAVINCI_MCASP_RXCLKCHK_REG	0x88
+#define DAVINCI_MCASP_REVTCTL_REG	0x8c
+
+#define DAVINCI_MCASP_GBLCTLX_REG	0xa0
+#define DAVINCI_MCASP_TXMASK_REG	0xa4
+#define DAVINCI_MCASP_TXFMT_REG		0xa8
+#define DAVINCI_MCASP_TXFMCTL_REG	0xac
+
+#define DAVINCI_MCASP_ACLKXCTL_REG	0xb0
+#define DAVINCI_MCASP_AHCLKXCTL_REG	0xb4
+#define DAVINCI_MCASP_TXTDM_REG		0xb8
+#define DAVINCI_MCASP_EVTCTLX_REG	0xbc
+
+#define DAVINCI_MCASP_TXSTAT_REG	0xc0
+#define DAVINCI_MCASP_TXTDMSLOT_REG	0xc4
+#define DAVINCI_MCASP_TXCLKCHK_REG	0xc8
+#define DAVINCI_MCASP_XEVTCTL_REG	0xcc
+
+/* Left(even TDM Slot) Channel Status Register File */
+#define DAVINCI_MCASP_DITCSRA_REG	0x100
+/* Right(odd TDM slot) Channel Status Register File */
+#define DAVINCI_MCASP_DITCSRB_REG	0x118
+/* Left(even TDM slot) User Data Register File */
+#define DAVINCI_MCASP_DITUDRA_REG	0x130
+/* Right(odd TDM Slot) User Data Register File */
+#define DAVINCI_MCASP_DITUDRB_REG	0x148
+
+/* Serializer n Control Register */
+#define DAVINCI_MCASP_XRSRCTL_BASE_REG	0x180
+#define DAVINCI_MCASP_XRSRCTL_REG(n)	(DAVINCI_MCASP_XRSRCTL_BASE_REG + \
+						(n << 2))
+
+/* Transmit Buffer for Serializer n */
+#define DAVINCI_MCASP_TXBUF_REG(n)	(0x200 + (n << 2))
+/* Receive Buffer for Serializer n */
+#define DAVINCI_MCASP_RXBUF_REG(n)	(0x280 + (n << 2))
+
+/* McASP FIFO Registers */
+#define DAVINCI_MCASP_V2_AFIFO_BASE	(0x1010)
+#define DAVINCI_MCASP_V3_AFIFO_BASE	(0x1000)
+
+/* FIFO register offsets from AFIFO base */
+#define MCASP_WFIFOCTL_OFFSET		(0x0)
+#define MCASP_WFIFOSTS_OFFSET		(0x4)
+#define MCASP_RFIFOCTL_OFFSET		(0x8)
+#define MCASP_RFIFOSTS_OFFSET		(0xc)
+
+/* DAVINCI_MCASP_PID_REG - Peripheral Identification Register Bits */
+#define MCASP_V4_REVMINOR_MASK		(0x3f)
+#define MCASP_V4_REVMAJOR_MASK		(0x7 << 8)
+#define MCASP_V4_REV_MASK		(MCASP_V4_REVMAJOR_MASK | \
+					 MCASP_V4_REVMINOR_MASK)
+#define MCASP_V4_REV(maj, min)		((maj) << 8 | (min))
+
+/*
+ * DAVINCI_MCASP_PWREMUMGT_REG - Power Down and Emulation Management
+ *     Register Bits
+ */
+#define MCASP_FREE	BIT(0)
+#define MCASP_SOFT	BIT(1)
+
+/*
+ * DAVINCI_MCASP_PFUNC_REG - Pin Function / GPIO Enable Register Bits
+ * DAVINCI_MCASP_PDIR_REG - Pin Direction Register Bits
+ * DAVINCI_MCASP_PDOUT_REG - Pin output in GPIO mode
+ * DAVINCI_MCASP_PDSET_REG - Pin input in GPIO mode
+ */
+#define PIN_BIT_AXR(n)	(n)
+#define PIN_BIT_AMUTE	25
+#define PIN_BIT_ACLKX	26
+#define PIN_BIT_AHCLKX	27
+#define PIN_BIT_AFSX	28
+#define PIN_BIT_ACLKR	29
+#define PIN_BIT_AHCLKR	30
+#define PIN_BIT_AFSR	31
+
+/*
+ * DAVINCI_MCASP_TXDITCTL_REG - Transmit DIT Control Register Bits
+ */
+#define DITEN	BIT(0)	/* Transmit DIT mode enable/disable */
+#define VA	BIT(2)
+#define VB	BIT(3)
+
+/*
+ * DAVINCI_MCASP_TXFMT_REG - Transmit Bitstream Format Register Bits
+ */
+#define TXROT(val)	(val)
+#define TXSEL		BIT(3)
+#define TXSSZ(val)	(val<<4)
+#define TXPBIT(val)	(val<<8)
+#define TXPAD(val)	(val<<13)
+#define TXORD		BIT(15)
+#define FSXDLY(val)	(val<<16)
+
+/*
+ * DAVINCI_MCASP_RXFMT_REG - Receive Bitstream Format Register Bits
+ */
+#define RXROT(val)	(val)
+#define RXSEL		BIT(3)
+#define RXSSZ(val)	(val<<4)
+#define RXPBIT(val)	(val<<8)
+#define RXPAD(val)	(val<<13)
+#define RXORD		BIT(15)
+#define FSRDLY(val)	(val<<16)
+
+/*
+ * DAVINCI_MCASP_TXFMCTL_REG -  Transmit Frame Control Register Bits
+ */
+#define FSXPOL		BIT(0)
+#define AFSXE		BIT(1)
+#define FSXDUR		BIT(4)
+#define FSXMOD(val)	(val<<7)
+
+/*
+ * DAVINCI_MCASP_RXFMCTL_REG - Receive Frame Control Register Bits
+ */
+#define FSRPOL		BIT(0)
+#define AFSRE		BIT(1)
+#define FSRDUR		BIT(4)
+#define FSRMOD(val)	(val<<7)
+
+/*
+ * DAVINCI_MCASP_ACLKXCTL_REG - Transmit Clock Control Register Bits
+ */
+#define ACLKXDIV(val)	(val)
+#define ACLKXE		BIT(5)
+#define TX_ASYNC	BIT(6)
+#define ACLKXPOL	BIT(7)
+#define ACLKXDIV_MASK	0x1f
+
+/*
+ * DAVINCI_MCASP_ACLKRCTL_REG Receive Clock Control Register Bits
+ */
+#define ACLKRDIV(val)	(val)
+#define ACLKRE		BIT(5)
+#define RX_ASYNC	BIT(6)
+#define ACLKRPOL	BIT(7)
+#define ACLKRDIV_MASK	0x1f
+
+/*
+ * DAVINCI_MCASP_AHCLKXCTL_REG - High Frequency Transmit Clock Control
+ *     Register Bits
+ */
+#define AHCLKXDIV(val)	(val)
+#define AHCLKXPOL	BIT(14)
+#define AHCLKXE		BIT(15)
+#define AHCLKXDIV_MASK	0xfff
+
+/*
+ * DAVINCI_MCASP_AHCLKRCTL_REG - High Frequency Receive Clock Control
+ *     Register Bits
+ */
+#define AHCLKRDIV(val)	(val)
+#define AHCLKRPOL	BIT(14)
+#define AHCLKRE		BIT(15)
+#define AHCLKRDIV_MASK	0xfff
+
+/*
+ * DAVINCI_MCASP_XRSRCTL_BASE_REG -  Serializer Control Register Bits
+ */
+#define MODE(val)	(val)
+#define DISMOD_3STATE	(0x0)
+#define DISMOD_LOW	(0x2 << 2)
+#define DISMOD_HIGH	(0x3 << 2)
+#define DISMOD_VAL(x)	((x) << 2)
+#define DISMOD_MASK	DISMOD_HIGH
+#define TXSTATE		BIT(4)
+#define RXSTATE		BIT(5)
+#define SRMOD_MASK	3
+#define SRMOD_INACTIVE	0
+
+/*
+ * DAVINCI_MCASP_LBCTL_REG - Loop Back Control Register Bits
+ */
+#define LBEN		BIT(0)
+#define LBORD		BIT(1)
+#define LBGENMODE(val)	(val<<2)
+
+/*
+ * DAVINCI_MCASP_TXTDMSLOT_REG - Transmit TDM Slot Register configuration
+ */
+#define TXTDMS(n)	(1<<n)
+
+/*
+ * DAVINCI_MCASP_RXTDMSLOT_REG - Receive TDM Slot Register configuration
+ */
+#define RXTDMS(n)	(1<<n)
+
+/*
+ * DAVINCI_MCASP_GBLCTL_REG -  Global Control Register Bits
+ */
+#define RXCLKRST	BIT(0)	/* Receiver Clock Divider Reset */
+#define RXHCLKRST	BIT(1)	/* Receiver High Frequency Clock Divider */
+#define RXSERCLR	BIT(2)	/* Receiver Serializer Clear */
+#define RXSMRST		BIT(3)	/* Receiver State Machine Reset */
+#define RXFSRST		BIT(4)	/* Frame Sync Generator Reset */
+#define TXCLKRST	BIT(8)	/* Transmitter Clock Divider Reset */
+#define TXHCLKRST	BIT(9)	/* Transmitter High Frequency Clock Divider*/
+#define TXSERCLR	BIT(10)	/* Transmit Serializer Clear */
+#define TXSMRST		BIT(11)	/* Transmitter State Machine Reset */
+#define TXFSRST		BIT(12)	/* Frame Sync Generator Reset */
+
+/*
+ * DAVINCI_MCASP_TXSTAT_REG - Transmitter Status Register Bits
+ * DAVINCI_MCASP_RXSTAT_REG - Receiver Status Register Bits
+ */
+#define XRERR		BIT(8) /* Transmit/Receive error */
+#define XRDATA		BIT(5) /* Transmit/Receive data ready */
+
+/*
+ * DAVINCI_MCASP_AMUTE_REG -  Mute Control Register Bits
+ */
+#define MUTENA(val)	(val)
+#define MUTEINPOL	BIT(2)
+#define MUTEINENA	BIT(3)
+#define MUTEIN		BIT(4)
+#define MUTER		BIT(5)
+#define MUTEX		BIT(6)
+#define MUTEFSR		BIT(7)
+#define MUTEFSX		BIT(8)
+#define MUTEBADCLKR	BIT(9)
+#define MUTEBADCLKX	BIT(10)
+#define MUTERXDMAERR	BIT(11)
+#define MUTETXDMAERR	BIT(12)
+
+/*
+ * DAVINCI_MCASP_REVTCTL_REG - Receiver DMA Event Control Register bits
+ */
+#define RXDATADMADIS	BIT(0)
+
+/*
+ * DAVINCI_MCASP_XEVTCTL_REG - Transmitter DMA Event Control Register bits
+ */
+#define TXDATADMADIS	BIT(0)
+
+/*
+ * DAVINCI_MCASP_EVTCTLR_REG - Receiver Interrupt Control Register Bits
+ */
+#define ROVRN		BIT(0)
+
+/*
+ * DAVINCI_MCASP_EVTCTLX_REG - Transmitter Interrupt Control Register Bits
+ */
+#define XUNDRN		BIT(0)
+
+/*
+ * DAVINCI_MCASP_W[R]FIFOCTL - Write/Read FIFO Control Register bits
+ */
+#define FIFO_ENABLE	BIT(16)
+#define NUMEVT_MASK	(0xFF << 8)
+#define NUMEVT(x)	(((x) & 0xFF) << 8)
+#define NUMDMA_MASK	(0xFF)
+
+#endif	/* DAVINCI_MCASP_H */
diff -Naur --no-dereference a/sound/soc/ti/davinci-vcif.c b/sound/soc/ti/davinci-vcif.c
--- a/sound/soc/ti/davinci-vcif.c	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/davinci-vcif.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,259 @@
+/*
+ * ALSA SoC Voice Codec Interface for TI DAVINCI processor
+ *
+ * Copyright (C) 2010 Texas Instruments.
+ *
+ * Author: Miguel Aguilar <miguel.aguilar@ridgerun.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/delay.h>
+#include <linux/slab.h>
+#include <linux/io.h>
+#include <linux/mfd/davinci_voicecodec.h>
+
+#include <sound/core.h>
+#include <sound/pcm.h>
+#include <sound/pcm_params.h>
+#include <sound/initval.h>
+#include <sound/soc.h>
+#include <sound/dmaengine_pcm.h>
+
+#include "edma-pcm.h"
+#include "davinci-i2s.h"
+
+#define MOD_REG_BIT(val, mask, set) do { \
+	if (set) { \
+		val |= mask; \
+	} else { \
+		val &= ~mask; \
+	} \
+} while (0)
+
+struct davinci_vcif_dev {
+	struct davinci_vc *davinci_vc;
+	struct snd_dmaengine_dai_dma_data dma_data[2];
+	int dma_request[2];
+};
+
+static void davinci_vcif_start(struct snd_pcm_substream *substream)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct davinci_vcif_dev *davinci_vcif_dev =
+			snd_soc_dai_get_drvdata(rtd->cpu_dai);
+	struct davinci_vc *davinci_vc = davinci_vcif_dev->davinci_vc;
+	u32 w;
+
+	/* Start the sample generator and enable transmitter/receiver */
+	w = readl(davinci_vc->base + DAVINCI_VC_CTRL);
+
+	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
+		MOD_REG_BIT(w, DAVINCI_VC_CTRL_RSTDAC, 0);
+	else
+		MOD_REG_BIT(w, DAVINCI_VC_CTRL_RSTADC, 0);
+
+	writel(w, davinci_vc->base + DAVINCI_VC_CTRL);
+}
+
+static void davinci_vcif_stop(struct snd_pcm_substream *substream)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct davinci_vcif_dev *davinci_vcif_dev =
+			snd_soc_dai_get_drvdata(rtd->cpu_dai);
+	struct davinci_vc *davinci_vc = davinci_vcif_dev->davinci_vc;
+	u32 w;
+
+	/* Reset transmitter/receiver and sample rate/frame sync generators */
+	w = readl(davinci_vc->base + DAVINCI_VC_CTRL);
+	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
+		MOD_REG_BIT(w, DAVINCI_VC_CTRL_RSTDAC, 1);
+	else
+		MOD_REG_BIT(w, DAVINCI_VC_CTRL_RSTADC, 1);
+
+	writel(w, davinci_vc->base + DAVINCI_VC_CTRL);
+}
+
+static int davinci_vcif_hw_params(struct snd_pcm_substream *substream,
+				  struct snd_pcm_hw_params *params,
+				  struct snd_soc_dai *dai)
+{
+	struct davinci_vcif_dev *davinci_vcif_dev = snd_soc_dai_get_drvdata(dai);
+	struct davinci_vc *davinci_vc = davinci_vcif_dev->davinci_vc;
+	u32 w;
+
+	/* Restart the codec before setup */
+	davinci_vcif_stop(substream);
+	davinci_vcif_start(substream);
+
+	/* General line settings */
+	writel(DAVINCI_VC_CTRL_MASK, davinci_vc->base + DAVINCI_VC_CTRL);
+
+	writel(DAVINCI_VC_INT_MASK, davinci_vc->base + DAVINCI_VC_INTCLR);
+
+	writel(DAVINCI_VC_INT_MASK, davinci_vc->base + DAVINCI_VC_INTEN);
+
+	w = readl(davinci_vc->base + DAVINCI_VC_CTRL);
+
+	/* Determine xfer data type */
+	switch (params_format(params)) {
+	case SNDRV_PCM_FORMAT_U8:
+		MOD_REG_BIT(w, DAVINCI_VC_CTRL_RD_BITS_8 |
+			    DAVINCI_VC_CTRL_RD_UNSIGNED |
+			    DAVINCI_VC_CTRL_WD_BITS_8 |
+			    DAVINCI_VC_CTRL_WD_UNSIGNED, 1);
+		break;
+	case SNDRV_PCM_FORMAT_S8:
+		MOD_REG_BIT(w, DAVINCI_VC_CTRL_RD_BITS_8 |
+			    DAVINCI_VC_CTRL_WD_BITS_8, 1);
+
+		MOD_REG_BIT(w, DAVINCI_VC_CTRL_RD_UNSIGNED |
+			    DAVINCI_VC_CTRL_WD_UNSIGNED, 0);
+		break;
+	case SNDRV_PCM_FORMAT_S16_LE:
+		MOD_REG_BIT(w, DAVINCI_VC_CTRL_RD_BITS_8 |
+			    DAVINCI_VC_CTRL_RD_UNSIGNED |
+			    DAVINCI_VC_CTRL_WD_BITS_8 |
+			    DAVINCI_VC_CTRL_WD_UNSIGNED, 0);
+		break;
+	default:
+		printk(KERN_WARNING "davinci-vcif: unsupported PCM format");
+		return -EINVAL;
+	}
+
+	writel(w, davinci_vc->base + DAVINCI_VC_CTRL);
+
+	return 0;
+}
+
+static int davinci_vcif_trigger(struct snd_pcm_substream *substream, int cmd,
+				struct snd_soc_dai *dai)
+{
+	int ret = 0;
+
+	switch (cmd) {
+	case SNDRV_PCM_TRIGGER_START:
+	case SNDRV_PCM_TRIGGER_RESUME:
+	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
+		davinci_vcif_start(substream);
+		break;
+	case SNDRV_PCM_TRIGGER_STOP:
+	case SNDRV_PCM_TRIGGER_SUSPEND:
+	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
+		davinci_vcif_stop(substream);
+		break;
+	default:
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
+#define DAVINCI_VCIF_RATES	SNDRV_PCM_RATE_8000_48000
+
+static const struct snd_soc_dai_ops davinci_vcif_dai_ops = {
+	.trigger	= davinci_vcif_trigger,
+	.hw_params	= davinci_vcif_hw_params,
+};
+
+static int davinci_vcif_dai_probe(struct snd_soc_dai *dai)
+{
+	struct davinci_vcif_dev *dev = snd_soc_dai_get_drvdata(dai);
+
+	dai->playback_dma_data = &dev->dma_data[SNDRV_PCM_STREAM_PLAYBACK];
+	dai->capture_dma_data = &dev->dma_data[SNDRV_PCM_STREAM_CAPTURE];
+
+	return 0;
+}
+
+static struct snd_soc_dai_driver davinci_vcif_dai = {
+	.probe = davinci_vcif_dai_probe,
+	.playback = {
+		.channels_min = 1,
+		.channels_max = 2,
+		.rates = DAVINCI_VCIF_RATES,
+		.formats = SNDRV_PCM_FMTBIT_S16_LE,},
+	.capture = {
+		.channels_min = 1,
+		.channels_max = 2,
+		.rates = DAVINCI_VCIF_RATES,
+		.formats = SNDRV_PCM_FMTBIT_S16_LE,},
+	.ops = &davinci_vcif_dai_ops,
+
+};
+
+static const struct snd_soc_component_driver davinci_vcif_component = {
+	.name		= "davinci-vcif",
+};
+
+static int davinci_vcif_probe(struct platform_device *pdev)
+{
+	struct davinci_vc *davinci_vc = pdev->dev.platform_data;
+	struct davinci_vcif_dev *davinci_vcif_dev;
+	int ret;
+
+	davinci_vcif_dev = devm_kzalloc(&pdev->dev,
+					sizeof(struct davinci_vcif_dev),
+					GFP_KERNEL);
+	if (!davinci_vcif_dev)
+		return -ENOMEM;
+
+	/* DMA tx params */
+	davinci_vcif_dev->davinci_vc = davinci_vc;
+	davinci_vcif_dev->dma_data[SNDRV_PCM_STREAM_PLAYBACK].filter_data =
+				&davinci_vc->davinci_vcif.dma_tx_channel;
+	davinci_vcif_dev->dma_data[SNDRV_PCM_STREAM_PLAYBACK].addr =
+				davinci_vc->davinci_vcif.dma_tx_addr;
+
+	/* DMA rx params */
+	davinci_vcif_dev->dma_data[SNDRV_PCM_STREAM_CAPTURE].filter_data =
+				&davinci_vc->davinci_vcif.dma_rx_channel;
+	davinci_vcif_dev->dma_data[SNDRV_PCM_STREAM_CAPTURE].addr =
+				davinci_vc->davinci_vcif.dma_rx_addr;
+
+	dev_set_drvdata(&pdev->dev, davinci_vcif_dev);
+
+	ret = devm_snd_soc_register_component(&pdev->dev,
+					      &davinci_vcif_component,
+					      &davinci_vcif_dai, 1);
+	if (ret != 0) {
+		dev_err(&pdev->dev, "could not register dai\n");
+		return ret;
+	}
+
+	ret = edma_pcm_platform_register(&pdev->dev);
+	if (ret) {
+		dev_err(&pdev->dev, "register PCM failed: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static struct platform_driver davinci_vcif_driver = {
+	.probe		= davinci_vcif_probe,
+	.driver		= {
+		.name	= "davinci-vcif",
+	},
+};
+
+module_platform_driver(davinci_vcif_driver);
+
+MODULE_AUTHOR("Miguel Aguilar");
+MODULE_DESCRIPTION("Texas Instruments DaVinci ASoC Voice Codec Interface");
+MODULE_LICENSE("GPL");
diff -Naur --no-dereference a/sound/soc/ti/edma-pcm.c b/sound/soc/ti/edma-pcm.c
--- a/sound/soc/ti/edma-pcm.c	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/edma-pcm.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,59 @@
+/*
+ * edma-pcm.c - eDMA PCM driver using dmaengine for AM3xxx, AM4xxx
+ *
+ * Copyright (C) 2014 Texas Instruments, Inc.
+ *
+ * Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
+ *
+ * Based on: sound/soc/tegra/tegra_pcm.c
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+
+#include <linux/module.h>
+#include <sound/core.h>
+#include <sound/pcm.h>
+#include <sound/pcm_params.h>
+#include <sound/soc.h>
+#include <sound/dmaengine_pcm.h>
+#include <linux/edma.h>
+
+#include "edma-pcm.h"
+
+static const struct snd_pcm_hardware edma_pcm_hardware = {
+	.info			= SNDRV_PCM_INFO_MMAP |
+				  SNDRV_PCM_INFO_MMAP_VALID |
+				  SNDRV_PCM_INFO_PAUSE | SNDRV_PCM_INFO_RESUME |
+				  SNDRV_PCM_INFO_NO_PERIOD_WAKEUP |
+				  SNDRV_PCM_INFO_INTERLEAVED,
+	.buffer_bytes_max	= 128 * 1024,
+	.period_bytes_min	= 32,
+	.period_bytes_max	= 64 * 1024,
+	.periods_min		= 2,
+	.periods_max		= 19, /* Limit by edma dmaengine driver */
+};
+
+static const struct snd_dmaengine_pcm_config edma_dmaengine_pcm_config = {
+	.pcm_hardware = &edma_pcm_hardware,
+	.prepare_slave_config = snd_dmaengine_pcm_prepare_slave_config,
+	.compat_filter_fn = edma_filter_fn,
+	.prealloc_buffer_size = 128 * 1024,
+};
+
+int edma_pcm_platform_register(struct device *dev)
+{
+	return devm_snd_dmaengine_pcm_register(dev, &edma_dmaengine_pcm_config,
+					SND_DMAENGINE_PCM_FLAG_COMPAT);
+}
+EXPORT_SYMBOL_GPL(edma_pcm_platform_register);
+
+MODULE_AUTHOR("Peter Ujfalusi <peter.ujfalusi@ti.com>");
+MODULE_DESCRIPTION("eDMA PCM ASoC platform driver");
+MODULE_LICENSE("GPL");
diff -Naur --no-dereference a/sound/soc/ti/edma-pcm.h b/sound/soc/ti/edma-pcm.h
--- a/sound/soc/ti/edma-pcm.h	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/edma-pcm.h	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,32 @@
+/*
+ * edma-pcm.h - eDMA PCM driver using dmaengine for AM3xxx, AM4xxx
+ *
+ * Copyright (C) 2014 Texas Instruments, Inc.
+ *
+ * Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
+ *
+ * Based on: sound/soc/tegra/tegra_pcm.h
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+
+#ifndef __EDMA_PCM_H__
+#define __EDMA_PCM_H__
+
+#if IS_ENABLED(CONFIG_SND_SOC_TI_EDMA_PCM)
+int edma_pcm_platform_register(struct device *dev);
+#else
+static inline int edma_pcm_platform_register(struct device *dev)
+{
+	return 0;
+}
+#endif /* CONFIG_SND_SOC_TI_EDMA_PCM */
+
+#endif /* __EDMA_PCM_H__ */
diff -Naur --no-dereference a/sound/soc/ti/j721e-evm.c b/sound/soc/ti/j721e-evm.c
--- a/sound/soc/ti/j721e-evm.c	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/j721e-evm.c	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,495 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
+ *  Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/platform_device.h>
+#include <linux/module.h>
+#include <linux/of.h>
+
+#include <sound/pcm_params.h>
+#include <sound/core.h>
+#include <sound/pcm.h>
+#include <sound/soc.h>
+#include <dt-bindings/sound/ti-mcasp.h>
+
+#define J721E_CPD_DAI_CNT	2
+
+#define J721E_CLK_PARENT_48000	0
+#define J721E_CLK_PARENT_44100	1
+
+#define J721E_MAX_CLK_HSDIV	128
+#define PCM1368A_MAX_SYSCLK	36864000
+
+#define J721E_DAI_FMT		(SND_SOC_DAIFMT_RIGHT_J | \
+				 SND_SOC_DAIFMT_NB_NF |   \
+				 SND_SOC_DAIFMT_CBS_CFS)
+
+static unsigned int ratios_for_pcm3168a[] = {
+	256,
+	512,
+	768,
+};
+
+struct j721e_audio_clocks {
+	struct clk *target;
+	struct clk *parent[2];
+};
+
+struct j721e_priv {
+	struct device *dev;
+	struct snd_soc_card cpb_card;
+	struct snd_soc_dai_link cpb_dai_links[J721E_CPD_DAI_CNT];
+	struct snd_soc_codec_conf codec_conf;
+	struct snd_interval rate_range;
+
+	struct j721e_audio_clocks audio_refclk2;
+	struct j721e_audio_clocks cpb_mcasp;
+	u32 pll_rates[2];
+	unsigned int current_cpb_ref_rate;
+	int current_cpb_ref_parent;
+
+	int active;
+	unsigned int rate;
+	struct mutex mutex;
+};
+
+static int j721e_configure_refclk(struct j721e_priv *priv, unsigned int rate)
+{
+	unsigned int scki;
+	int ret = -EINVAL;
+	int i, clk_id;
+
+	if (!(rate % 8000))
+		clk_id = J721E_CLK_PARENT_48000;
+	else if (!(rate % 11025))
+		clk_id = J721E_CLK_PARENT_44100;
+	else
+		return ret;
+
+	for (i = 0; i < ARRAY_SIZE(ratios_for_pcm3168a); i++) {
+		scki = ratios_for_pcm3168a[i] * rate;
+
+		if (priv->pll_rates[clk_id] / scki <= J721E_MAX_CLK_HSDIV) {
+			ret = 0;
+			break;
+		}
+	}
+
+	if (ret) {
+		dev_err(priv->dev, "No valid clock configuration for %u Hz\n",
+			rate);
+		return ret;
+	}
+
+	if (priv->current_cpb_ref_rate != scki) {
+		dev_dbg(priv->dev,
+			"Configuration for %u Hz: %s, %dxFS (SCKI: %u Hz)\n",
+			rate,
+			clk_id == J721E_CLK_PARENT_48000 ? "PLL4" : "PLL15",
+			ratios_for_pcm3168a[i], scki);
+
+		if (priv->current_cpb_ref_parent != clk_id) {
+			ret = clk_set_parent(priv->audio_refclk2.target,
+					priv->audio_refclk2.parent[clk_id]);
+			if (ret)
+				return ret;
+
+			ret = clk_set_parent(priv->cpb_mcasp.target,
+					priv->cpb_mcasp.parent[clk_id]);
+			if (ret)
+				return ret;
+
+			priv->current_cpb_ref_parent = clk_id;
+		}
+
+		ret = clk_set_rate(priv->audio_refclk2.target, scki);
+		if (ret)
+			return ret;
+
+		ret = clk_set_rate(priv->cpb_mcasp.target, scki);
+		if (!ret)
+			priv->current_cpb_ref_rate = scki;
+	}
+
+	return ret;
+}
+
+static int j721e_rule_rate(struct snd_pcm_hw_params *params,
+			   struct snd_pcm_hw_rule *rule)
+{
+	struct snd_interval *t = rule->private;
+
+	return snd_interval_refine(hw_param_interval(params, rule->var), t);
+}
+
+static int j721e_audio_startup(struct snd_pcm_substream *substream)
+{
+	struct 	snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct j721e_priv *priv = snd_soc_card_get_drvdata(rtd->card);
+	int ret = 0;
+
+	mutex_lock(&priv->mutex);
+	priv->active++;
+
+	if (priv->rate)
+		ret = snd_pcm_hw_constraint_single(substream->runtime,
+						   SNDRV_PCM_HW_PARAM_RATE,
+						   priv->rate);
+	else
+		ret = snd_pcm_hw_rule_add(substream->runtime, 0,
+					  SNDRV_PCM_HW_PARAM_RATE,
+					  j721e_rule_rate, &priv->rate_range,
+					  SNDRV_PCM_HW_PARAM_RATE, -1);
+
+	mutex_unlock(&priv->mutex);
+
+	if (ret)
+		return ret;
+
+	/* Reset TDM slots to 32 */
+	ret = snd_soc_dai_set_tdm_slot(rtd->cpu_dai, 0x3, 0x3, 2, 32);
+	if (ret)
+		return ret;
+
+	ret = snd_soc_dai_set_tdm_slot(rtd->codec_dai, 0x3, 0x3, 2, 32);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int j721e_audio_hw_params(struct snd_pcm_substream *substream,
+				 struct snd_pcm_hw_params *params)
+{
+	struct 	snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_card *card = rtd->card;
+	struct j721e_priv *priv = snd_soc_card_get_drvdata(card);
+	int slot_width = 32;
+	int ret;
+
+	mutex_lock(&priv->mutex);
+
+	if (priv->rate && priv->rate != params_rate(params)) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	if (params_width(params) == 16)
+		slot_width = 16;
+
+	ret = snd_soc_dai_set_tdm_slot(rtd->cpu_dai, 0x3, 0x3, 2, slot_width);
+	if (ret)
+		goto out;
+
+	ret = snd_soc_dai_set_tdm_slot(rtd->codec_dai, 0x3, 0x3, 2, slot_width);
+	if (ret)
+		goto out;
+
+	ret = j721e_configure_refclk(priv, params_rate(params));
+	if (ret)
+		goto out;
+
+	ret = snd_soc_dai_set_sysclk(rtd->codec_dai, 0,
+				     priv->current_cpb_ref_rate,
+				     SND_SOC_CLOCK_IN);
+	if (ret)
+		goto out;
+
+	ret = snd_soc_dai_set_sysclk(rtd->cpu_dai, MCASP_CLK_HCLK_AUXCLK,
+				     priv->current_cpb_ref_rate,
+				     SND_SOC_CLOCK_IN);
+
+	if (!ret)
+		priv->rate = params_rate(params);
+
+out:
+	mutex_unlock(&priv->mutex);
+	return ret;
+}
+
+static void j721e_audio_shutdown(struct snd_pcm_substream *substream)
+{
+	struct 	snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct j721e_priv *priv = snd_soc_card_get_drvdata(rtd->card);
+
+	mutex_lock(&priv->mutex);
+
+	priv->active--;
+	if (!priv->active)
+		priv->rate = 0;
+
+	mutex_unlock(&priv->mutex);
+}
+
+static const struct snd_soc_ops j721e_audio_ops = {
+	.startup = j721e_audio_startup,
+	.hw_params = j721e_audio_hw_params,
+	.shutdown = j721e_audio_shutdown,
+};
+
+static int j721e_audio_init(struct snd_soc_pcm_runtime *rtd)
+{
+	struct j721e_priv *priv = snd_soc_card_get_drvdata(rtd->card);
+	int ret;
+
+	/* Set up initial clock configuration */
+	ret = j721e_configure_refclk(priv, 48000);
+	if (ret)
+		return ret;
+
+	ret = snd_soc_dai_set_sysclk(rtd->codec_dai, 0,
+				     priv->current_cpb_ref_rate,
+				     SND_SOC_CLOCK_IN);
+	if (ret)
+		return ret;
+
+	ret = snd_soc_dai_set_sysclk(rtd->cpu_dai, MCASP_CLK_HCLK_AUXCLK,
+				     priv->current_cpb_ref_rate,
+				     SND_SOC_CLOCK_IN);
+	if (ret)
+		return ret;
+
+	/* Set initial tdm slots */
+	ret = snd_soc_dai_set_tdm_slot(rtd->cpu_dai, 0x3, 0x3, 2, 32);
+	if (ret)
+		return ret;
+
+	ret = snd_soc_dai_set_tdm_slot(rtd->codec_dai, 0x3, 0x3, 2, 32);
+
+	return ret;
+}
+
+static const struct snd_soc_dapm_widget j721e_cpb_dapm_widgets[] = {
+	SND_SOC_DAPM_HP("CPB Stereo HP 1", NULL),
+	SND_SOC_DAPM_HP("CPB Stereo HP 2", NULL),
+	SND_SOC_DAPM_HP("CPB Stereo HP 3", NULL),
+	SND_SOC_DAPM_LINE("CPB Line Out", NULL),
+	SND_SOC_DAPM_MIC("CPB Stereo Mic 1", NULL),
+	SND_SOC_DAPM_MIC("CPB Stereo Mic 2", NULL),
+	SND_SOC_DAPM_LINE("CPB Line In", NULL),
+};
+
+static const struct snd_soc_dapm_route j721e_cpb_dapm_routes[] = {
+	{"CPB Stereo HP 1", NULL, "codec1 AOUT1L"},
+	{"CPB Stereo HP 1", NULL, "codec1 AOUT1R"},
+	{"CPB Stereo HP 2", NULL, "codec1 AOUT2L"},
+	{"CPB Stereo HP 2", NULL, "codec1 AOUT2R"},
+	{"CPB Stereo HP 3", NULL, "codec1 AOUT3L"},
+	{"CPB Stereo HP 3", NULL, "codec1 AOUT3R"},
+	{"CPB Line Out", NULL, "codec1 AOUT4L"},
+	{"CPB Line Out", NULL, "codec1 AOUT4R"},
+
+	{"codec1 AIN1L", NULL, "CPB Stereo Mic 1"},
+	{"codec1 AIN1R", NULL, "CPB Stereo Mic 1"},
+	{"codec1 AIN2L", NULL, "CPB Stereo Mic 2"},
+	{"codec1 AIN2R", NULL, "CPB Stereo Mic 2"},
+	{"codec1 AIN3L", NULL, "CPB Line In"},
+	{"codec1 AIN3R", NULL, "CPB Line In"},
+};
+
+static int j721e_get_clocks(struct platform_device *pdev,
+			    struct j721e_audio_clocks *clocks, char *prefix)
+{
+	struct clk *parent;
+	char *clk_name;
+	int ret;
+
+	clocks->target = devm_clk_get(&pdev->dev, prefix);
+	if (IS_ERR(clocks->target)) {
+		ret = PTR_ERR(clocks->target);
+		if (ret != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "failed to acquire %s': %d\n",
+				prefix, ret);
+		return ret;
+	}
+
+	clk_name = kasprintf(GFP_KERNEL, "%s-48000", prefix);
+	if (clk_name) {
+		parent = devm_clk_get(&pdev->dev, clk_name);
+		kfree(clk_name);
+		if (IS_ERR(parent)) {
+			ret = PTR_ERR(parent);
+			if (ret != -EPROBE_DEFER)
+				dev_err(&pdev->dev, "failed to acquire %s': %d\n",
+					prefix, ret);
+			return ret;
+		}
+		clocks->parent[J721E_CLK_PARENT_48000] = parent;
+	} else {
+		return -ENOMEM;
+	}
+
+	clk_name = kasprintf(GFP_KERNEL, "%s-44100", prefix);
+	if (clk_name) {
+		parent = devm_clk_get(&pdev->dev, clk_name);
+		kfree(clk_name);
+		if (IS_ERR(parent)) {
+			ret = PTR_ERR(parent);
+			if (ret != -EPROBE_DEFER)
+				dev_err(&pdev->dev, "failed to acquire %s': %d\n",
+					prefix, ret);
+			return ret;
+		}
+		clocks->parent[J721E_CLK_PARENT_44100] = parent;
+	} else {
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static void j721e_calculate_rate_range(struct j721e_priv *priv)
+{
+	unsigned int min_rate, max_rate, pll_rate;
+
+	pll_rate = priv->pll_rates[J721E_CLK_PARENT_44100];
+	min_rate = pll_rate / J721E_MAX_CLK_HSDIV;
+	min_rate /= ratios_for_pcm3168a[ARRAY_SIZE(ratios_for_pcm3168a) - 1];
+
+	pll_rate = priv->pll_rates[J721E_CLK_PARENT_48000];
+	if (pll_rate > PCM1368A_MAX_SYSCLK)
+		pll_rate = PCM1368A_MAX_SYSCLK;
+
+	max_rate = pll_rate / ratios_for_pcm3168a[0];
+
+	snd_interval_any(&priv->rate_range);
+	priv->rate_range.min = min_rate;
+	priv->rate_range.max = max_rate;
+}
+
+static int j721e_soc_probe(struct platform_device *pdev)
+{
+	struct device_node *node = pdev->dev.of_node;
+	struct snd_soc_card *card;
+	struct device_node *cpb_dai_node, *cpb_codec_node;
+	struct j721e_priv *priv;
+	int ret;
+
+	if (!node) {
+		dev_err(&pdev->dev, "of node is missing.\n");
+		return -ENODEV;
+	}
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (priv == NULL)
+		return -ENOMEM;
+
+	priv->current_cpb_ref_parent = -1;
+	priv->dev = &pdev->dev;
+	card = &priv->cpb_card;
+	card->dev = &pdev->dev;
+	card->owner = THIS_MODULE;
+	card->dapm_widgets = j721e_cpb_dapm_widgets;
+	card->num_dapm_widgets = ARRAY_SIZE(j721e_cpb_dapm_widgets);
+	card->dapm_routes = j721e_cpb_dapm_routes;
+	card->num_dapm_routes = ARRAY_SIZE(j721e_cpb_dapm_routes);
+	card->fully_routed = 1;
+
+	if (snd_soc_of_parse_card_name(card, "ti,model")) {
+		dev_err(&pdev->dev, "Card name is not provided\n");
+		return -ENODEV;
+	}
+
+	cpb_dai_node = of_parse_phandle(node, "ti,cpb-mcasp", 0);
+	if (!cpb_dai_node) {
+		dev_err(&pdev->dev, "CPB McASP node is not provided\n");
+		return -EINVAL;
+	}
+
+	cpb_codec_node = of_parse_phandle(node, "ti,cpb-codec", 0);
+	if (!cpb_codec_node) {
+		dev_err(&pdev->dev, "CPB codec node is not provided\n");
+		return -EINVAL;
+	}
+
+	ret = j721e_get_clocks(pdev, &priv->audio_refclk2, "audio-refclk2");
+	if (ret)
+		return ret;
+
+	ret = j721e_get_clocks(pdev, &priv->cpb_mcasp, "cpb-mcasp");
+	if (ret)
+		return ret;
+
+	ret = of_property_read_u32(node, "pll4-rate",
+				   &priv->pll_rates[J721E_CLK_PARENT_48000]);
+	if (ret)
+		return ret;
+
+	ret = of_property_read_u32(node, "pll15-rate",
+				   &priv->pll_rates[J721E_CLK_PARENT_44100]);
+	if (ret)
+		return ret;
+	
+	priv->cpb_dai_links[0].name = "CPB pcm3168a DAC";
+	priv->cpb_dai_links[0].stream_name = "cpb pcm3168a Playback";
+	priv->cpb_dai_links[0].cpu_of_node = cpb_dai_node;
+	priv->cpb_dai_links[0].platform_of_node = cpb_dai_node;
+	priv->cpb_dai_links[0].codec_of_node = cpb_codec_node;
+	priv->cpb_dai_links[0].codec_dai_name = "pcm3168a-dac";
+	priv->cpb_dai_links[0].playback_only = 1;
+	priv->cpb_dai_links[0].dai_fmt = J721E_DAI_FMT;
+	priv->cpb_dai_links[0].init = j721e_audio_init;
+	priv->cpb_dai_links[0].ops = &j721e_audio_ops;
+
+	priv->cpb_dai_links[1].name = "CPB pcm3168a ADC";
+	priv->cpb_dai_links[1].stream_name = "cpb pcm3168a Capture";
+	priv->cpb_dai_links[1].cpu_of_node = cpb_dai_node;
+	priv->cpb_dai_links[1].platform_of_node = cpb_dai_node;
+	priv->cpb_dai_links[1].codec_of_node = cpb_codec_node;
+	priv->cpb_dai_links[1].codec_dai_name = "pcm3168a-adc";
+	priv->cpb_dai_links[1].capture_only = 1;
+	priv->cpb_dai_links[1].dai_fmt = J721E_DAI_FMT;
+	priv->cpb_dai_links[1].init = j721e_audio_init;
+	priv->cpb_dai_links[1].ops = &j721e_audio_ops;
+
+	card->dai_link = priv->cpb_dai_links;
+	card->num_links = ARRAY_SIZE(priv->cpb_dai_links);
+
+	priv->codec_conf.of_node = cpb_codec_node;
+	priv->codec_conf.name_prefix = "codec1";
+	if (!priv->codec_conf.name_prefix)
+		return -ENOMEM;
+
+	card->codec_conf = &priv->codec_conf;
+	card->num_configs = 1;
+
+	j721e_calculate_rate_range(priv);
+
+	snd_soc_card_set_drvdata(card, priv);
+
+	mutex_init(&priv->mutex);
+	ret = devm_snd_soc_register_card(&pdev->dev, card);
+	if (ret)
+		dev_err(&pdev->dev, "devm_snd_soc_register_card() failed: %d\n",
+			ret);
+
+	return ret;
+
+	return 0;
+}
+
+#if defined(CONFIG_OF)
+static const struct of_device_id j721e_audio_of_match[] = {
+	{ .compatible = "ti,j721e-cpb-audio", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, j721e_audio_of_match);
+#endif
+
+static struct platform_driver j721e_soc_driver = {
+	.driver = {
+		.name = "j721e-audio",
+		.pm = &snd_soc_pm_ops,
+		.of_match_table = of_match_ptr(j721e_audio_of_match),
+	},
+	.probe = j721e_soc_probe,
+};
+
+module_platform_driver(j721e_soc_driver);
+
+MODULE_AUTHOR("Peter Ujfalusi <peter.ujfalusi@ti.com>");
+MODULE_DESCRIPTION("ASoC machine driver for j721e Common Processor Board");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/sound/soc/ti/Kconfig b/sound/soc/ti/Kconfig
--- a/sound/soc/ti/Kconfig	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/Kconfig	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,224 @@
+menu "Audio support for Texas Instruments SoCs"
+depends on DMA_OMAP || TI_EDMA || TI_K3_UDMA || COMPILE_TEST
+
+config SND_SOC_TI_EDMA_PCM
+	tristate
+	select SND_SOC_GENERIC_DMAENGINE_PCM
+
+config SND_SOC_TI_SDMA_PCM
+	tristate
+	select SND_SOC_GENERIC_DMAENGINE_PCM
+
+config SND_SOC_TI_UDMA_PCM
+	tristate
+	select SND_SOC_GENERIC_DMAENGINE_PCM
+
+comment "Texas Instruments DAI support for:"
+config SND_SOC_DAVINCI_ASP
+	tristate "daVinci Audio Serial Port (ASP) or McBSP suport"
+	depends on ARCH_DAVINCI || COMPILE_TEST
+	select SND_SOC_TI_EDMA_PCM
+	help
+	  Say Y or M here if you want audio support via daVinci ASP or McBSP.
+	  The driver only implements the ASP support which is a subset of
+	  daVinci McBSP (w/o the multichannel support).
+
+config SND_SOC_DAVINCI_MCASP
+	tristate "Multichannel Audio Serial Port (McASP) support"
+	select SND_SOC_TI_EDMA_PCM if TI_EDMA
+	select SND_SOC_TI_SDMA_PCM if DMA_OMAP
+	select SND_SOC_TI_UDMA_PCM if TI_K3_UDMA
+	select SND_SOC_PCM5102A
+	help
+	  Say Y or M here if you want to have support for McASP IP found in
+	  various Texas Instruments SoCs like:
+	  - daVinci devices
+	  - Sitara line of SoCs (AM335x, AM438x, etc)
+	  - DRA7x devices
+	  - Keystone devices
+	  - AM65x devices
+
+config SND_SOC_DAVINCI_VCIF
+	tristate "daVinci Voice Interface (VCIF) suport"
+	depends on ARCH_DAVINCI || COMPILE_TEST
+	select SND_SOC_TI_EDMA_PCM
+	help
+	  Say Y or M here if you want audio support via daVinci VCIF.
+
+config SND_SOC_OMAP_DMIC
+	tristate "Digital Microphone Module (DMIC) support"
+	depends on ARCH_OMAP4 || SOC_OMAP5 || COMPILE_TEST
+	select SND_SOC_TI_SDMA_PCM
+	help
+	  Say Y or M here if you want to have support for DMIC IP found in
+	  OMAP4 and OMAP5.
+
+config SND_SOC_OMAP_MCBSP
+	tristate "Multichannel Buffered Serial Port (McBSP) support"
+	depends on ARCH_OMAP || ARCH_OMAP1 || COMPILE_TEST
+	select SND_SOC_TI_SDMA_PCM
+	help
+	  Say Y or M here if you want to have support for McBSP IP found in
+	  Texas Instruments OMAP1/2/3/4/5 SoCs.
+
+config SND_SOC_OMAP_MCPDM
+	tristate "Multichannel PDM Controller (McPDM) support"
+	depends on ARCH_OMAP4 || SOC_OMAP5 || COMPILE_TEST
+	select SND_SOC_TI_SDMA_PCM
+	help
+	  Say Y or M here if you want to have support for McPDM IP found in
+	  OMAP4 and OMAP5.
+
+comment "Audio support for boards with Texas Instruments SoCs"
+config SND_SOC_NOKIA_N810
+	tristate "SoC Audio support for Nokia N810"
+	depends on MACH_NOKIA_N810 && I2C
+	select SND_SOC_OMAP_MCBSP
+	select SND_SOC_TLV320AIC3X
+	help
+	  Say Y or M if you want to add support for SoC audio on Nokia N810.
+
+config SND_SOC_NOKIA_RX51
+	tristate "SoC Audio support for Nokia RX-51"
+	depends on ARCH_OMAP3 && I2C && GPIOLIB
+	select SND_SOC_OMAP_MCBSP
+	select SND_SOC_TLV320AIC3X
+	select SND_SOC_TPA6130A2
+	help
+	  Say Y or M if you want to add support for SoC audio on Nokia RX-51
+	  hardware. This is also known as Nokia N900 product.
+
+config SND_SOC_OMAP3_PANDORA
+	tristate "SoC Audio support for OMAP3 Pandora"
+	depends on ARCH_OMAP3
+	depends on TWL4030_CORE
+	select SND_SOC_OMAP_MCBSP
+	select SND_SOC_TWL4030
+	help
+	  Say Y or M if you want to add support for SoC audio on the OMAP3 Pandora.
+
+config SND_SOC_OMAP3_TWL4030
+	tristate "SoC Audio support for OMAP3 based boards with twl4030 codec"
+	depends on ARCH_OMAP3 || COMPILE_TEST
+	depends on TWL4030_CORE
+	select SND_SOC_OMAP_MCBSP
+	select SND_SOC_TWL4030
+	help
+	  Say Y or M if you want to add support for SoC audio on OMAP3 based
+	  boards using twl4030 as codec. This driver currently supports:
+	  - Beagleboard or Devkit8000
+	  - Gumstix Overo or CompuLab CM-T35/CM-T3730
+	  - IGEP v2
+	  - OMAP3EVM
+	  - SDP3430
+	  - Zoom2
+
+config SND_SOC_OMAP_ABE_TWL6040
+	tristate "SoC Audio support for OMAP boards using ABE and twl6040 codec"
+	depends on TWL6040_CORE && COMMON_CLK
+	depends on ARCH_OMAP4 || (SOC_OMAP5 && MFD_PALMAS) || COMPILE_TEST
+	select SND_SOC_OMAP_DMIC
+	select SND_SOC_OMAP_MCPDM
+	select SND_SOC_TWL6040
+	help
+	  Say Y or M if you want to add support for SoC audio on OMAP boards
+	  using ABE and twl6040 codec. This driver currently supports:
+	  - SDP4430/Blaze boards
+	  - PandaBoard (4430)
+	  - PandaBoardES (4460)
+	  - OMAP5 uEVM
+
+config SND_SOC_OMAP_AMS_DELTA
+	tristate "SoC Audio support for Amstrad E3 (Delta) videophone"
+	depends on MACH_AMS_DELTA && TTY
+	select SND_SOC_OMAP_MCBSP
+	select SND_SOC_CX20442
+	help
+	  Say Y  or M if you want to add support  for SoC audio device
+	  connected to a handset and a speakerphone found on Amstrad E3 (Delta)
+	  videophone.
+
+	  Note that in order to get those devices fully supported,  you have to
+	  build  the kernel  with  standard  serial port  driver  included  and
+	  configured for at least 4 ports.  Then, from userspace, you must load
+	  a line discipline #19 on the modem (ttyS3) serial line.  The simplest
+	  way to achieve this is to install util-linux-ng  and use the included
+	  ldattach  utility.  This  can be  started  automatically  from  udev,
+	  a simple rule like this one should do the trick (it does for me):
+		ACTION=="add", KERNEL=="controlC0", \
+				RUN+="/usr/sbin/ldattach 19 /dev/ttyS3"
+
+config SND_SOC_OMAP_HDMI
+	tristate "OMAP4/5 HDMI audio support"
+	depends on OMAP4_DSS_HDMI || OMAP5_DSS_HDMI || COMPILE_TEST
+	select SND_SOC_TI_SDMA_PCM
+	help
+	  For HDMI audio to work OMAPDSS HDMI support should be
+	  enabled.
+	  The hdmi audio driver implements cpu-dai component using the
+	  callbacks provided by OMAPDSS and registers the component
+	  under DSS HDMI device. Omap-pcm is registered for platform
+	  component also under DSS HDMI device. Dummy codec is used as
+	  as codec component. The hdmi audio driver implements also
+	  the card and registers it under its own platform device.
+	  The device for the driver is registered by OMAPDSS hdmi
+	  driver.
+
+config SND_SOC_OMAP_OSK5912
+	tristate "SoC Audio support for omap osk5912"
+	depends on MACH_OMAP_OSK && I2C
+	select SND_SOC_OMAP_MCBSP
+	select SND_SOC_TLV320AIC23_I2C
+	help
+	  Say Y or M if you want to add support for SoC audio on osk5912.
+
+config SND_SOC_DAVINCI_EVM
+	tristate "SoC Audio support for DaVinci EVMs"
+	depends on ARCH_DAVINCI && I2C
+	select SND_SOC_DAVINCI_ASP if MACH_DAVINCI_DM355_EVM
+	select SND_SOC_DAVINCI_ASP if SND_SOC_DM365_AIC3X_CODEC
+	select SND_SOC_DAVINCI_VCIF if SND_SOC_DM365_VOICE_CODEC
+	select SND_SOC_DAVINCI_ASP if MACH_DAVINCI_EVM # DM6446
+	select SND_SOC_DAVINCI_MCASP if MACH_DAVINCI_DM6467_EVM
+	select SND_SOC_SPDIF if MACH_DAVINCI_DM6467_EVM
+	select SND_SOC_DAVINCI_MCASP if MACH_DAVINCI_DA830_EVM
+	select SND_SOC_DAVINCI_MCASP if MACH_DAVINCI_DA850_EVM
+	select SND_SOC_TLV320AIC3X
+	help
+	  Say Y if you want to add support for SoC audio on the following TI
+	  DaVinci EVM platforms:
+	  - DM355
+	  - DM365
+	  - DM6446
+	  - DM6447
+	  - DM830
+	  - DM850
+
+choice
+	prompt "DM365 codec select"
+	depends on SND_SOC_DAVINCI_EVM
+	depends on MACH_DAVINCI_DM365_EVM
+
+config SND_SOC_DM365_AIC3X_CODEC
+	bool "Audio Codec - AIC3101"
+	help
+	  Say Y if you want to add support for AIC3101 audio codec
+
+config SND_SOC_DM365_VOICE_CODEC
+	bool "Voice Codec - CQ93VC"
+	select MFD_DAVINCI_VOICECODEC
+	select SND_SOC_CQ0093VC
+	help
+	  Say Y if you want to add support for SoC On-chip voice codec
+endchoice
+
+config SND_SOC_J721E_EVM
+	tristate "SoC Audio support for j721e EVM"
+	depends on ARCH_K3_J721E_SOC || COMPILE_TEST
+	select SND_SOC_PCM3168A_I2C
+	select SND_SOC_DAVINCI_MCASP
+	help
+	  Say Y if you want to add support for SoC audio on j721e Common
+	  Processor Board.
+endmenu
+
diff -Naur --no-dereference a/sound/soc/ti/Makefile b/sound/soc/ti/Makefile
--- a/sound/soc/ti/Makefile	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/Makefile	2020-06-19 21:00:31.436227717 -0400
@@ -0,0 +1,48 @@
+# SPDX-License-Identifier: GPL-2.0
+
+# Platform drivers
+snd-soc-ti-edma-objs := edma-pcm.o
+snd-soc-ti-sdma-objs := sdma-pcm.o
+snd-soc-ti-udma-objs := udma-pcm.o
+
+obj-$(CONFIG_SND_SOC_TI_EDMA_PCM) += snd-soc-ti-edma.o
+obj-$(CONFIG_SND_SOC_TI_SDMA_PCM) += snd-soc-ti-sdma.o
+obj-$(CONFIG_SND_SOC_TI_UDMA_PCM) += snd-soc-ti-udma.o
+
+# CPU DAI drivers
+snd-soc-davinci-asp-objs := davinci-i2s.o
+snd-soc-davinci-mcasp-objs := davinci-mcasp.o
+snd-soc-davinci-vcif-objs := davinci-vcif.o
+snd-soc-omap-dmic-objs := omap-dmic.o
+snd-soc-omap-mcbsp-objs := omap-mcbsp.o omap-mcbsp-st.o
+snd-soc-omap-mcpdm-objs := omap-mcpdm.o
+
+obj-$(CONFIG_SND_SOC_DAVINCI_ASP) += snd-soc-davinci-asp.o
+obj-$(CONFIG_SND_SOC_DAVINCI_MCASP) += snd-soc-davinci-mcasp.o
+obj-$(CONFIG_SND_SOC_DAVINCI_VCIF) += snd-soc-davinci-vcif.o
+obj-$(CONFIG_SND_SOC_OMAP_DMIC) += snd-soc-omap-dmic.o
+obj-$(CONFIG_SND_SOC_OMAP_MCBSP) += snd-soc-omap-mcbsp.o
+obj-$(CONFIG_SND_SOC_OMAP_MCPDM) += snd-soc-omap-mcpdm.o
+
+# Machine drivers
+snd-soc-davinci-evm-objs := davinci-evm.o
+snd-soc-n810-objs := n810.o
+snd-soc-rx51-objs := rx51.o
+snd-soc-omap3pandora-objs := omap3pandora.o
+snd-soc-omap-twl4030-objs := omap-twl4030.o
+snd-soc-omap-abe-twl6040-objs := omap-abe-twl6040.o
+snd-soc-ams-delta-objs := ams-delta.o
+snd-soc-omap-hdmi-objs := omap-hdmi.o
+snd-soc-osk5912-objs := osk5912.o
+snd-soc-j721e-evm-objs := j721e-evm.o
+
+obj-$(CONFIG_SND_SOC_DAVINCI_EVM) += snd-soc-davinci-evm.o
+obj-$(CONFIG_SND_SOC_NOKIA_N810) += snd-soc-n810.o
+obj-$(CONFIG_SND_SOC_NOKIA_RX51) += snd-soc-rx51.o
+obj-$(CONFIG_SND_SOC_OMAP3_PANDORA) += snd-soc-omap3pandora.o
+obj-$(CONFIG_SND_SOC_OMAP3_TWL4030) += snd-soc-omap-twl4030.o
+obj-$(CONFIG_SND_SOC_OMAP_ABE_TWL6040) += snd-soc-omap-abe-twl6040.o
+obj-$(CONFIG_SND_SOC_OMAP_AMS_DELTA) += snd-soc-ams-delta.o
+obj-$(CONFIG_SND_SOC_OMAP_HDMI) += snd-soc-omap-hdmi.o
+obj-$(CONFIG_SND_SOC_OMAP_OSK5912) += snd-soc-osk5912.o
+obj-$(CONFIG_SND_SOC_J721E_EVM) += snd-soc-j721e-evm.o
diff -Naur --no-dereference a/sound/soc/ti/n810.c b/sound/soc/ti/n810.c
--- a/sound/soc/ti/n810.c	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/n810.c	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,378 @@
+/*
+ * n810.c  --  SoC audio for Nokia N810
+ *
+ * Copyright (C) 2008 Nokia Corporation
+ *
+ * Contact: Jarkko Nikula <jarkko.nikula@bitmer.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA
+ *
+ */
+
+#include <linux/clk.h>
+#include <linux/i2c.h>
+#include <linux/platform_device.h>
+#include <sound/core.h>
+#include <sound/pcm.h>
+#include <sound/soc.h>
+
+#include <asm/mach-types.h>
+#include <linux/gpio.h>
+#include <linux/module.h>
+#include <linux/platform_data/asoc-ti-mcbsp.h>
+
+#include "omap-mcbsp.h"
+
+#define N810_HEADSET_AMP_GPIO	10
+#define N810_SPEAKER_AMP_GPIO	101
+
+enum {
+	N810_JACK_DISABLED,
+	N810_JACK_HP,
+	N810_JACK_HS,
+	N810_JACK_MIC,
+};
+
+static struct clk *sys_clkout2;
+static struct clk *sys_clkout2_src;
+static struct clk *func96m_clk;
+
+static int n810_spk_func;
+static int n810_jack_func;
+static int n810_dmic_func;
+
+static void n810_ext_control(struct snd_soc_dapm_context *dapm)
+{
+	int hp = 0, line1l = 0;
+
+	switch (n810_jack_func) {
+	case N810_JACK_HS:
+		line1l = 1;
+	case N810_JACK_HP:
+		hp = 1;
+		break;
+	case N810_JACK_MIC:
+		line1l = 1;
+		break;
+	}
+
+	snd_soc_dapm_mutex_lock(dapm);
+
+	if (n810_spk_func)
+		snd_soc_dapm_enable_pin_unlocked(dapm, "Ext Spk");
+	else
+		snd_soc_dapm_disable_pin_unlocked(dapm, "Ext Spk");
+
+	if (hp)
+		snd_soc_dapm_enable_pin_unlocked(dapm, "Headphone Jack");
+	else
+		snd_soc_dapm_disable_pin_unlocked(dapm, "Headphone Jack");
+	if (line1l)
+		snd_soc_dapm_enable_pin_unlocked(dapm, "HS Mic");
+	else
+		snd_soc_dapm_disable_pin_unlocked(dapm, "HS Mic");
+
+	if (n810_dmic_func)
+		snd_soc_dapm_enable_pin_unlocked(dapm, "DMic");
+	else
+		snd_soc_dapm_disable_pin_unlocked(dapm, "DMic");
+
+	snd_soc_dapm_sync_unlocked(dapm);
+
+	snd_soc_dapm_mutex_unlock(dapm);
+}
+
+static int n810_startup(struct snd_pcm_substream *substream)
+{
+	struct snd_pcm_runtime *runtime = substream->runtime;
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+
+	snd_pcm_hw_constraint_single(runtime, SNDRV_PCM_HW_PARAM_CHANNELS, 2);
+
+	n810_ext_control(&rtd->card->dapm);
+	return clk_prepare_enable(sys_clkout2);
+}
+
+static void n810_shutdown(struct snd_pcm_substream *substream)
+{
+	clk_disable_unprepare(sys_clkout2);
+}
+
+static int n810_hw_params(struct snd_pcm_substream *substream,
+	struct snd_pcm_hw_params *params)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_dai *codec_dai = rtd->codec_dai;
+	int err;
+
+	/* Set the codec system clock for DAC and ADC */
+	err = snd_soc_dai_set_sysclk(codec_dai, 0, 12000000,
+					    SND_SOC_CLOCK_IN);
+
+	return err;
+}
+
+static const struct snd_soc_ops n810_ops = {
+	.startup = n810_startup,
+	.hw_params = n810_hw_params,
+	.shutdown = n810_shutdown,
+};
+
+static int n810_get_spk(struct snd_kcontrol *kcontrol,
+			struct snd_ctl_elem_value *ucontrol)
+{
+	ucontrol->value.enumerated.item[0] = n810_spk_func;
+
+	return 0;
+}
+
+static int n810_set_spk(struct snd_kcontrol *kcontrol,
+			struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_card *card =  snd_kcontrol_chip(kcontrol);
+
+	if (n810_spk_func == ucontrol->value.enumerated.item[0])
+		return 0;
+
+	n810_spk_func = ucontrol->value.enumerated.item[0];
+	n810_ext_control(&card->dapm);
+
+	return 1;
+}
+
+static int n810_get_jack(struct snd_kcontrol *kcontrol,
+			 struct snd_ctl_elem_value *ucontrol)
+{
+	ucontrol->value.enumerated.item[0] = n810_jack_func;
+
+	return 0;
+}
+
+static int n810_set_jack(struct snd_kcontrol *kcontrol,
+			 struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_card *card =  snd_kcontrol_chip(kcontrol);
+
+	if (n810_jack_func == ucontrol->value.enumerated.item[0])
+		return 0;
+
+	n810_jack_func = ucontrol->value.enumerated.item[0];
+	n810_ext_control(&card->dapm);
+
+	return 1;
+}
+
+static int n810_get_input(struct snd_kcontrol *kcontrol,
+			  struct snd_ctl_elem_value *ucontrol)
+{
+	ucontrol->value.enumerated.item[0] = n810_dmic_func;
+
+	return 0;
+}
+
+static int n810_set_input(struct snd_kcontrol *kcontrol,
+			  struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_card *card =  snd_kcontrol_chip(kcontrol);
+
+	if (n810_dmic_func == ucontrol->value.enumerated.item[0])
+		return 0;
+
+	n810_dmic_func = ucontrol->value.enumerated.item[0];
+	n810_ext_control(&card->dapm);
+
+	return 1;
+}
+
+static int n810_spk_event(struct snd_soc_dapm_widget *w,
+			  struct snd_kcontrol *k, int event)
+{
+	if (SND_SOC_DAPM_EVENT_ON(event))
+		gpio_set_value(N810_SPEAKER_AMP_GPIO, 1);
+	else
+		gpio_set_value(N810_SPEAKER_AMP_GPIO, 0);
+
+	return 0;
+}
+
+static int n810_jack_event(struct snd_soc_dapm_widget *w,
+			   struct snd_kcontrol *k, int event)
+{
+	if (SND_SOC_DAPM_EVENT_ON(event))
+		gpio_set_value(N810_HEADSET_AMP_GPIO, 1);
+	else
+		gpio_set_value(N810_HEADSET_AMP_GPIO, 0);
+
+	return 0;
+}
+
+static const struct snd_soc_dapm_widget aic33_dapm_widgets[] = {
+	SND_SOC_DAPM_SPK("Ext Spk", n810_spk_event),
+	SND_SOC_DAPM_HP("Headphone Jack", n810_jack_event),
+	SND_SOC_DAPM_MIC("DMic", NULL),
+	SND_SOC_DAPM_MIC("HS Mic", NULL),
+};
+
+static const struct snd_soc_dapm_route audio_map[] = {
+	{"Headphone Jack", NULL, "HPLOUT"},
+	{"Headphone Jack", NULL, "HPROUT"},
+
+	{"Ext Spk", NULL, "LLOUT"},
+	{"Ext Spk", NULL, "RLOUT"},
+
+	{"DMic Rate 64", NULL, "DMic"},
+	{"DMic", NULL, "Mic Bias"},
+
+	/*
+	 * Note that the mic bias is coming from Retu/Vilma and we don't have
+	 * control over it atm. The analog HS mic is not working. <- TODO
+	 */
+	{"LINE1L", NULL, "HS Mic"},
+};
+
+static const char *spk_function[] = {"Off", "On"};
+static const char *jack_function[] = {"Off", "Headphone", "Headset", "Mic"};
+static const char *input_function[] = {"ADC", "Digital Mic"};
+static const struct soc_enum n810_enum[] = {
+	SOC_ENUM_SINGLE_EXT(ARRAY_SIZE(spk_function), spk_function),
+	SOC_ENUM_SINGLE_EXT(ARRAY_SIZE(jack_function), jack_function),
+	SOC_ENUM_SINGLE_EXT(ARRAY_SIZE(input_function), input_function),
+};
+
+static const struct snd_kcontrol_new aic33_n810_controls[] = {
+	SOC_ENUM_EXT("Speaker Function", n810_enum[0],
+		     n810_get_spk, n810_set_spk),
+	SOC_ENUM_EXT("Jack Function", n810_enum[1],
+		     n810_get_jack, n810_set_jack),
+	SOC_ENUM_EXT("Input Select",  n810_enum[2],
+		     n810_get_input, n810_set_input),
+};
+
+/* Digital audio interface glue - connects codec <--> CPU */
+static struct snd_soc_dai_link n810_dai = {
+	.name = "TLV320AIC33",
+	.stream_name = "AIC33",
+	.cpu_dai_name = "48076000.mcbsp",
+	.platform_name = "48076000.mcbsp",
+	.codec_name = "tlv320aic3x-codec.1-0018",
+	.codec_dai_name = "tlv320aic3x-hifi",
+	.dai_fmt = SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF |
+		   SND_SOC_DAIFMT_CBM_CFM,
+	.ops = &n810_ops,
+};
+
+/* Audio machine driver */
+static struct snd_soc_card snd_soc_n810 = {
+	.name = "N810",
+	.owner = THIS_MODULE,
+	.dai_link = &n810_dai,
+	.num_links = 1,
+
+	.controls = aic33_n810_controls,
+	.num_controls = ARRAY_SIZE(aic33_n810_controls),
+	.dapm_widgets = aic33_dapm_widgets,
+	.num_dapm_widgets = ARRAY_SIZE(aic33_dapm_widgets),
+	.dapm_routes = audio_map,
+	.num_dapm_routes = ARRAY_SIZE(audio_map),
+	.fully_routed = true,
+};
+
+static struct platform_device *n810_snd_device;
+
+static int __init n810_soc_init(void)
+{
+	int err;
+	struct device *dev;
+
+	if (!of_have_populated_dt() ||
+	    (!of_machine_is_compatible("nokia,n810") &&
+	     !of_machine_is_compatible("nokia,n810-wimax")))
+		return -ENODEV;
+
+	n810_snd_device = platform_device_alloc("soc-audio", -1);
+	if (!n810_snd_device)
+		return -ENOMEM;
+
+	platform_set_drvdata(n810_snd_device, &snd_soc_n810);
+	err = platform_device_add(n810_snd_device);
+	if (err)
+		goto err1;
+
+	dev = &n810_snd_device->dev;
+
+	sys_clkout2_src = clk_get(dev, "sys_clkout2_src");
+	if (IS_ERR(sys_clkout2_src)) {
+		dev_err(dev, "Could not get sys_clkout2_src clock\n");
+		err = PTR_ERR(sys_clkout2_src);
+		goto err2;
+	}
+	sys_clkout2 = clk_get(dev, "sys_clkout2");
+	if (IS_ERR(sys_clkout2)) {
+		dev_err(dev, "Could not get sys_clkout2\n");
+		err = PTR_ERR(sys_clkout2);
+		goto err3;
+	}
+	/*
+	 * Configure 12 MHz output on SYS_CLKOUT2. Therefore we must use
+	 * 96 MHz as its parent in order to get 12 MHz
+	 */
+	func96m_clk = clk_get(dev, "func_96m_ck");
+	if (IS_ERR(func96m_clk)) {
+		dev_err(dev, "Could not get func 96M clock\n");
+		err = PTR_ERR(func96m_clk);
+		goto err4;
+	}
+	clk_set_parent(sys_clkout2_src, func96m_clk);
+	clk_set_rate(sys_clkout2, 12000000);
+
+	if (WARN_ON((gpio_request(N810_HEADSET_AMP_GPIO, "hs_amp") < 0) ||
+		    (gpio_request(N810_SPEAKER_AMP_GPIO, "spk_amp") < 0))) {
+		err = -EINVAL;
+		goto err4;
+	}
+
+	gpio_direction_output(N810_HEADSET_AMP_GPIO, 0);
+	gpio_direction_output(N810_SPEAKER_AMP_GPIO, 0);
+
+	return 0;
+err4:
+	clk_put(sys_clkout2);
+err3:
+	clk_put(sys_clkout2_src);
+err2:
+	platform_device_del(n810_snd_device);
+err1:
+	platform_device_put(n810_snd_device);
+
+	return err;
+}
+
+static void __exit n810_soc_exit(void)
+{
+	gpio_free(N810_SPEAKER_AMP_GPIO);
+	gpio_free(N810_HEADSET_AMP_GPIO);
+	clk_put(sys_clkout2_src);
+	clk_put(sys_clkout2);
+	clk_put(func96m_clk);
+
+	platform_device_unregister(n810_snd_device);
+}
+
+module_init(n810_soc_init);
+module_exit(n810_soc_exit);
+
+MODULE_AUTHOR("Jarkko Nikula <jarkko.nikula@bitmer.com>");
+MODULE_DESCRIPTION("ALSA SoC Nokia N810");
+MODULE_LICENSE("GPL");
diff -Naur --no-dereference a/sound/soc/ti/omap3pandora.c b/sound/soc/ti/omap3pandora.c
--- a/sound/soc/ti/omap3pandora.c	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/omap3pandora.c	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,315 @@
+/*
+ * omap3pandora.c  --  SoC audio for Pandora Handheld Console
+ *
+ * Author: Gražvydas Ignotas <notasas@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA
+ *
+ */
+
+#include <linux/clk.h>
+#include <linux/platform_device.h>
+#include <linux/gpio.h>
+#include <linux/delay.h>
+#include <linux/regulator/consumer.h>
+#include <linux/module.h>
+
+#include <sound/core.h>
+#include <sound/pcm.h>
+#include <sound/soc.h>
+
+#include <asm/mach-types.h>
+#include <linux/platform_data/asoc-ti-mcbsp.h>
+
+#include "omap-mcbsp.h"
+
+#define OMAP3_PANDORA_DAC_POWER_GPIO	118
+#define OMAP3_PANDORA_AMP_POWER_GPIO	14
+
+#define PREFIX "ASoC omap3pandora: "
+
+static struct regulator *omap3pandora_dac_reg;
+
+static int omap3pandora_hw_params(struct snd_pcm_substream *substream,
+	struct snd_pcm_hw_params *params)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_dai *codec_dai = rtd->codec_dai;
+	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
+	int ret;
+
+	/* Set the codec system clock for DAC and ADC */
+	ret = snd_soc_dai_set_sysclk(codec_dai, 0, 26000000,
+					    SND_SOC_CLOCK_IN);
+	if (ret < 0) {
+		pr_err(PREFIX "can't set codec system clock\n");
+		return ret;
+	}
+
+	/* Set McBSP clock to external */
+	ret = snd_soc_dai_set_sysclk(cpu_dai, OMAP_MCBSP_SYSCLK_CLKS_EXT,
+				     256 * params_rate(params),
+				     SND_SOC_CLOCK_IN);
+	if (ret < 0) {
+		pr_err(PREFIX "can't set cpu system clock\n");
+		return ret;
+	}
+
+	ret = snd_soc_dai_set_clkdiv(cpu_dai, OMAP_MCBSP_CLKGDV, 8);
+	if (ret < 0) {
+		pr_err(PREFIX "can't set SRG clock divider\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int omap3pandora_dac_event(struct snd_soc_dapm_widget *w,
+	struct snd_kcontrol *k, int event)
+{
+	int ret;
+
+	/*
+	 * The PCM1773 DAC datasheet requires 1ms delay between switching
+	 * VCC power on/off and /PD pin high/low
+	 */
+	if (SND_SOC_DAPM_EVENT_ON(event)) {
+		ret = regulator_enable(omap3pandora_dac_reg);
+		if (ret) {
+			dev_err(w->dapm->dev, "Failed to power DAC: %d\n", ret);
+			return ret;
+		}
+		mdelay(1);
+		gpio_set_value(OMAP3_PANDORA_DAC_POWER_GPIO, 1);
+	} else {
+		gpio_set_value(OMAP3_PANDORA_DAC_POWER_GPIO, 0);
+		mdelay(1);
+		regulator_disable(omap3pandora_dac_reg);
+	}
+
+	return 0;
+}
+
+static int omap3pandora_hp_event(struct snd_soc_dapm_widget *w,
+	struct snd_kcontrol *k, int event)
+{
+	if (SND_SOC_DAPM_EVENT_ON(event))
+		gpio_set_value(OMAP3_PANDORA_AMP_POWER_GPIO, 1);
+	else
+		gpio_set_value(OMAP3_PANDORA_AMP_POWER_GPIO, 0);
+
+	return 0;
+}
+
+/*
+ * Audio paths on Pandora board:
+ *
+ *  |O| ---> PCM DAC +-> AMP -> Headphone Jack
+ *  |M|         A    +--------> Line Out
+ *  |A| <~~clk~~+
+ *  |P| <--- TWL4030 <--------- Line In and MICs
+ */
+static const struct snd_soc_dapm_widget omap3pandora_dapm_widgets[] = {
+	SND_SOC_DAPM_DAC_E("PCM DAC", "HiFi Playback", SND_SOC_NOPM,
+			   0, 0, omap3pandora_dac_event,
+			   SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_PRE_PMD),
+	SND_SOC_DAPM_PGA_E("Headphone Amplifier", SND_SOC_NOPM,
+			   0, 0, NULL, 0, omap3pandora_hp_event,
+			   SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_PRE_PMD),
+	SND_SOC_DAPM_HP("Headphone Jack", NULL),
+	SND_SOC_DAPM_LINE("Line Out", NULL),
+
+	SND_SOC_DAPM_MIC("Mic (internal)", NULL),
+	SND_SOC_DAPM_MIC("Mic (external)", NULL),
+	SND_SOC_DAPM_LINE("Line In", NULL),
+};
+
+static const struct snd_soc_dapm_route omap3pandora_map[] = {
+	{"PCM DAC", NULL, "APLL Enable"},
+	{"Headphone Amplifier", NULL, "PCM DAC"},
+	{"Line Out", NULL, "PCM DAC"},
+	{"Headphone Jack", NULL, "Headphone Amplifier"},
+
+	{"AUXL", NULL, "Line In"},
+	{"AUXR", NULL, "Line In"},
+
+	{"MAINMIC", NULL, "Mic (internal)"},
+	{"Mic (internal)", NULL, "Mic Bias 1"},
+
+	{"SUBMIC", NULL, "Mic (external)"},
+	{"Mic (external)", NULL, "Mic Bias 2"},
+};
+
+static int omap3pandora_out_init(struct snd_soc_pcm_runtime *rtd)
+{
+	struct snd_soc_dapm_context *dapm = &rtd->card->dapm;
+
+	/* All TWL4030 output pins are floating */
+	snd_soc_dapm_nc_pin(dapm, "EARPIECE");
+	snd_soc_dapm_nc_pin(dapm, "PREDRIVEL");
+	snd_soc_dapm_nc_pin(dapm, "PREDRIVER");
+	snd_soc_dapm_nc_pin(dapm, "HSOL");
+	snd_soc_dapm_nc_pin(dapm, "HSOR");
+	snd_soc_dapm_nc_pin(dapm, "CARKITL");
+	snd_soc_dapm_nc_pin(dapm, "CARKITR");
+	snd_soc_dapm_nc_pin(dapm, "HFL");
+	snd_soc_dapm_nc_pin(dapm, "HFR");
+	snd_soc_dapm_nc_pin(dapm, "VIBRA");
+
+	return 0;
+}
+
+static int omap3pandora_in_init(struct snd_soc_pcm_runtime *rtd)
+{
+	struct snd_soc_dapm_context *dapm = &rtd->card->dapm;
+
+	/* Not comnnected */
+	snd_soc_dapm_nc_pin(dapm, "HSMIC");
+	snd_soc_dapm_nc_pin(dapm, "CARKITMIC");
+	snd_soc_dapm_nc_pin(dapm, "DIGIMIC0");
+	snd_soc_dapm_nc_pin(dapm, "DIGIMIC1");
+
+	return 0;
+}
+
+static const struct snd_soc_ops omap3pandora_ops = {
+	.hw_params = omap3pandora_hw_params,
+};
+
+/* Digital audio interface glue - connects codec <--> CPU */
+static struct snd_soc_dai_link omap3pandora_dai[] = {
+	{
+		.name = "PCM1773",
+		.stream_name = "HiFi Out",
+		.cpu_dai_name = "omap-mcbsp.2",
+		.codec_dai_name = "twl4030-hifi",
+		.platform_name = "omap-mcbsp.2",
+		.codec_name = "twl4030-codec",
+		.dai_fmt = SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF |
+			   SND_SOC_DAIFMT_CBS_CFS,
+		.ops = &omap3pandora_ops,
+		.init = omap3pandora_out_init,
+	}, {
+		.name = "TWL4030",
+		.stream_name = "Line/Mic In",
+		.cpu_dai_name = "omap-mcbsp.4",
+		.codec_dai_name = "twl4030-hifi",
+		.platform_name = "omap-mcbsp.4",
+		.codec_name = "twl4030-codec",
+		.dai_fmt = SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF |
+			   SND_SOC_DAIFMT_CBS_CFS,
+		.ops = &omap3pandora_ops,
+		.init = omap3pandora_in_init,
+	}
+};
+
+/* SoC card */
+static struct snd_soc_card snd_soc_card_omap3pandora = {
+	.name = "omap3pandora",
+	.owner = THIS_MODULE,
+	.dai_link = omap3pandora_dai,
+	.num_links = ARRAY_SIZE(omap3pandora_dai),
+
+	.dapm_widgets = omap3pandora_dapm_widgets,
+	.num_dapm_widgets = ARRAY_SIZE(omap3pandora_dapm_widgets),
+	.dapm_routes = omap3pandora_map,
+	.num_dapm_routes = ARRAY_SIZE(omap3pandora_map),
+};
+
+static struct platform_device *omap3pandora_snd_device;
+
+static int __init omap3pandora_soc_init(void)
+{
+	int ret;
+
+	if (!machine_is_omap3_pandora())
+		return -ENODEV;
+
+	pr_info("OMAP3 Pandora SoC init\n");
+
+	ret = gpio_request(OMAP3_PANDORA_DAC_POWER_GPIO, "dac_power");
+	if (ret) {
+		pr_err(PREFIX "Failed to get DAC power GPIO\n");
+		return ret;
+	}
+
+	ret = gpio_direction_output(OMAP3_PANDORA_DAC_POWER_GPIO, 0);
+	if (ret) {
+		pr_err(PREFIX "Failed to set DAC power GPIO direction\n");
+		goto fail0;
+	}
+
+	ret = gpio_request(OMAP3_PANDORA_AMP_POWER_GPIO, "amp_power");
+	if (ret) {
+		pr_err(PREFIX "Failed to get amp power GPIO\n");
+		goto fail0;
+	}
+
+	ret = gpio_direction_output(OMAP3_PANDORA_AMP_POWER_GPIO, 0);
+	if (ret) {
+		pr_err(PREFIX "Failed to set amp power GPIO direction\n");
+		goto fail1;
+	}
+
+	omap3pandora_snd_device = platform_device_alloc("soc-audio", -1);
+	if (omap3pandora_snd_device == NULL) {
+		pr_err(PREFIX "Platform device allocation failed\n");
+		ret = -ENOMEM;
+		goto fail1;
+	}
+
+	platform_set_drvdata(omap3pandora_snd_device, &snd_soc_card_omap3pandora);
+
+	ret = platform_device_add(omap3pandora_snd_device);
+	if (ret) {
+		pr_err(PREFIX "Unable to add platform device\n");
+		goto fail2;
+	}
+
+	omap3pandora_dac_reg = regulator_get(&omap3pandora_snd_device->dev, "vcc");
+	if (IS_ERR(omap3pandora_dac_reg)) {
+		pr_err(PREFIX "Failed to get DAC regulator from %s: %ld\n",
+			dev_name(&omap3pandora_snd_device->dev),
+			PTR_ERR(omap3pandora_dac_reg));
+		ret = PTR_ERR(omap3pandora_dac_reg);
+		goto fail3;
+	}
+
+	return 0;
+
+fail3:
+	platform_device_del(omap3pandora_snd_device);
+fail2:
+	platform_device_put(omap3pandora_snd_device);
+fail1:
+	gpio_free(OMAP3_PANDORA_AMP_POWER_GPIO);
+fail0:
+	gpio_free(OMAP3_PANDORA_DAC_POWER_GPIO);
+	return ret;
+}
+module_init(omap3pandora_soc_init);
+
+static void __exit omap3pandora_soc_exit(void)
+{
+	regulator_put(omap3pandora_dac_reg);
+	platform_device_unregister(omap3pandora_snd_device);
+	gpio_free(OMAP3_PANDORA_AMP_POWER_GPIO);
+	gpio_free(OMAP3_PANDORA_DAC_POWER_GPIO);
+}
+module_exit(omap3pandora_soc_exit);
+
+MODULE_AUTHOR("Grazvydas Ignotas <notasas@gmail.com>");
+MODULE_DESCRIPTION("ALSA SoC OMAP3 Pandora");
+MODULE_LICENSE("GPL");
diff -Naur --no-dereference a/sound/soc/ti/omap-abe-twl6040.c b/sound/soc/ti/omap-abe-twl6040.c
--- a/sound/soc/ti/omap-abe-twl6040.c	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/omap-abe-twl6040.c	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,353 @@
+/*
+ * omap-abe-twl6040.c  --  SoC audio for TI OMAP based boards with ABE and
+ *			   twl6040 codec
+ *
+ * Author: Misael Lopez Cruz <misael.lopez@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA
+ *
+ */
+
+#include <linux/clk.h>
+#include <linux/platform_device.h>
+#include <linux/mfd/twl6040.h>
+#include <linux/module.h>
+#include <linux/of.h>
+
+#include <sound/core.h>
+#include <sound/pcm.h>
+#include <sound/soc.h>
+#include <sound/jack.h>
+
+#include "omap-dmic.h"
+#include "omap-mcpdm.h"
+#include "../codecs/twl6040.h"
+
+struct abe_twl6040 {
+	struct snd_soc_card card;
+	struct snd_soc_dai_link dai_links[2];
+	int	jack_detection;	/* board can detect jack events */
+	int	mclk_freq;	/* MCLK frequency speed for twl6040 */
+};
+
+static struct platform_device *dmic_codec_dev;
+
+static int omap_abe_hw_params(struct snd_pcm_substream *substream,
+	struct snd_pcm_hw_params *params)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_dai *codec_dai = rtd->codec_dai;
+	struct snd_soc_card *card = rtd->card;
+	struct abe_twl6040 *priv = snd_soc_card_get_drvdata(card);
+	int clk_id, freq;
+	int ret;
+
+	clk_id = twl6040_get_clk_id(codec_dai->component);
+	if (clk_id == TWL6040_SYSCLK_SEL_HPPLL)
+		freq = priv->mclk_freq;
+	else if (clk_id == TWL6040_SYSCLK_SEL_LPPLL)
+		freq = 32768;
+	else
+		return -EINVAL;
+
+	/* set the codec mclk */
+	ret = snd_soc_dai_set_sysclk(codec_dai, clk_id, freq,
+				SND_SOC_CLOCK_IN);
+	if (ret) {
+		printk(KERN_ERR "can't set codec system clock\n");
+		return ret;
+	}
+	return ret;
+}
+
+static const struct snd_soc_ops omap_abe_ops = {
+	.hw_params = omap_abe_hw_params,
+};
+
+static int omap_abe_dmic_hw_params(struct snd_pcm_substream *substream,
+	struct snd_pcm_hw_params *params)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
+	int ret = 0;
+
+	ret = snd_soc_dai_set_sysclk(cpu_dai, OMAP_DMIC_SYSCLK_PAD_CLKS,
+				     19200000, SND_SOC_CLOCK_IN);
+	if (ret < 0) {
+		printk(KERN_ERR "can't set DMIC cpu system clock\n");
+		return ret;
+	}
+	ret = snd_soc_dai_set_sysclk(cpu_dai, OMAP_DMIC_ABE_DMIC_CLK, 2400000,
+				     SND_SOC_CLOCK_OUT);
+	if (ret < 0) {
+		printk(KERN_ERR "can't set DMIC output clock\n");
+		return ret;
+	}
+	return 0;
+}
+
+static struct snd_soc_ops omap_abe_dmic_ops = {
+	.hw_params = omap_abe_dmic_hw_params,
+};
+
+/* Headset jack */
+static struct snd_soc_jack hs_jack;
+
+/*Headset jack detection DAPM pins */
+static struct snd_soc_jack_pin hs_jack_pins[] = {
+	{
+		.pin = "Headset Mic",
+		.mask = SND_JACK_MICROPHONE,
+	},
+	{
+		.pin = "Headset Stereophone",
+		.mask = SND_JACK_HEADPHONE,
+	},
+};
+
+/* SDP4430 machine DAPM */
+static const struct snd_soc_dapm_widget twl6040_dapm_widgets[] = {
+	/* Outputs */
+	SND_SOC_DAPM_HP("Headset Stereophone", NULL),
+	SND_SOC_DAPM_SPK("Earphone Spk", NULL),
+	SND_SOC_DAPM_SPK("Ext Spk", NULL),
+	SND_SOC_DAPM_LINE("Line Out", NULL),
+	SND_SOC_DAPM_SPK("Vibrator", NULL),
+
+	/* Inputs */
+	SND_SOC_DAPM_MIC("Headset Mic", NULL),
+	SND_SOC_DAPM_MIC("Main Handset Mic", NULL),
+	SND_SOC_DAPM_MIC("Sub Handset Mic", NULL),
+	SND_SOC_DAPM_LINE("Line In", NULL),
+
+	/* Digital microphones */
+	SND_SOC_DAPM_MIC("Digital Mic", NULL),
+};
+
+static const struct snd_soc_dapm_route audio_map[] = {
+	/* Routings for outputs */
+	{"Headset Stereophone", NULL, "HSOL"},
+	{"Headset Stereophone", NULL, "HSOR"},
+
+	{"Earphone Spk", NULL, "EP"},
+
+	{"Ext Spk", NULL, "HFL"},
+	{"Ext Spk", NULL, "HFR"},
+
+	{"Line Out", NULL, "AUXL"},
+	{"Line Out", NULL, "AUXR"},
+
+	{"Vibrator", NULL, "VIBRAL"},
+	{"Vibrator", NULL, "VIBRAR"},
+
+	/* Routings for inputs */
+	{"HSMIC", NULL, "Headset Mic"},
+	{"Headset Mic", NULL, "Headset Mic Bias"},
+
+	{"MAINMIC", NULL, "Main Handset Mic"},
+	{"Main Handset Mic", NULL, "Main Mic Bias"},
+
+	{"SUBMIC", NULL, "Sub Handset Mic"},
+	{"Sub Handset Mic", NULL, "Main Mic Bias"},
+
+	{"AFML", NULL, "Line In"},
+	{"AFMR", NULL, "Line In"},
+};
+
+static int omap_abe_twl6040_init(struct snd_soc_pcm_runtime *rtd)
+{
+	struct snd_soc_component *component = rtd->codec_dai->component;
+	struct snd_soc_card *card = rtd->card;
+	struct abe_twl6040 *priv = snd_soc_card_get_drvdata(card);
+	int hs_trim;
+	int ret = 0;
+
+	/*
+	 * Configure McPDM offset cancellation based on the HSOTRIM value from
+	 * twl6040.
+	 */
+	hs_trim = twl6040_get_trim_value(component, TWL6040_TRIM_HSOTRIM);
+	omap_mcpdm_configure_dn_offsets(rtd, TWL6040_HSF_TRIM_LEFT(hs_trim),
+					TWL6040_HSF_TRIM_RIGHT(hs_trim));
+
+	/* Headset jack detection only if it is supported */
+	if (priv->jack_detection) {
+		ret = snd_soc_card_jack_new(rtd->card, "Headset Jack",
+					    SND_JACK_HEADSET, &hs_jack,
+					    hs_jack_pins,
+					    ARRAY_SIZE(hs_jack_pins));
+		if (ret)
+			return ret;
+
+		twl6040_hs_jack_detect(component, &hs_jack, SND_JACK_HEADSET);
+	}
+
+	return 0;
+}
+
+static const struct snd_soc_dapm_route dmic_audio_map[] = {
+	{"DMic", NULL, "Digital Mic"},
+	{"Digital Mic", NULL, "Digital Mic1 Bias"},
+};
+
+static int omap_abe_dmic_init(struct snd_soc_pcm_runtime *rtd)
+{
+	struct snd_soc_dapm_context *dapm = &rtd->card->dapm;
+
+	return snd_soc_dapm_add_routes(dapm, dmic_audio_map,
+				ARRAY_SIZE(dmic_audio_map));
+}
+
+static int omap_abe_probe(struct platform_device *pdev)
+{
+	struct device_node *node = pdev->dev.of_node;
+	struct snd_soc_card *card;
+	struct device_node *dai_node;
+	struct abe_twl6040 *priv;
+	int num_links = 0;
+	int ret = 0;
+
+	if (!node) {
+		dev_err(&pdev->dev, "of node is missing.\n");
+		return -ENODEV;
+	}
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(struct abe_twl6040), GFP_KERNEL);
+	if (priv == NULL)
+		return -ENOMEM;
+
+	card = &priv->card;
+	card->dev = &pdev->dev;
+	card->owner = THIS_MODULE;
+	card->dapm_widgets = twl6040_dapm_widgets;
+	card->num_dapm_widgets = ARRAY_SIZE(twl6040_dapm_widgets);
+	card->dapm_routes = audio_map;
+	card->num_dapm_routes = ARRAY_SIZE(audio_map);
+
+	if (snd_soc_of_parse_card_name(card, "ti,model")) {
+		dev_err(&pdev->dev, "Card name is not provided\n");
+		return -ENODEV;
+	}
+
+	ret = snd_soc_of_parse_audio_routing(card, "ti,audio-routing");
+	if (ret) {
+		dev_err(&pdev->dev, "Error while parsing DAPM routing\n");
+		return ret;
+	}
+
+	dai_node = of_parse_phandle(node, "ti,mcpdm", 0);
+	if (!dai_node) {
+		dev_err(&pdev->dev, "McPDM node is not provided\n");
+		return -EINVAL;
+	}
+
+	priv->dai_links[0].name = "DMIC";
+	priv->dai_links[0].stream_name = "TWL6040";
+	priv->dai_links[0].cpu_of_node = dai_node;
+	priv->dai_links[0].platform_of_node = dai_node;
+	priv->dai_links[0].codec_dai_name = "twl6040-legacy";
+	priv->dai_links[0].codec_name = "twl6040-codec";
+	priv->dai_links[0].init = omap_abe_twl6040_init;
+	priv->dai_links[0].ops = &omap_abe_ops;
+
+	dai_node = of_parse_phandle(node, "ti,dmic", 0);
+	if (dai_node) {
+		num_links = 2;
+		priv->dai_links[1].name = "TWL6040";
+		priv->dai_links[1].stream_name = "DMIC Capture";
+		priv->dai_links[1].cpu_of_node = dai_node;
+		priv->dai_links[1].platform_of_node = dai_node;
+		priv->dai_links[1].codec_dai_name = "dmic-hifi";
+		priv->dai_links[1].codec_name = "dmic-codec";
+		priv->dai_links[1].init = omap_abe_dmic_init;
+		priv->dai_links[1].ops = &omap_abe_dmic_ops;
+	} else {
+		num_links = 1;
+	}
+
+	priv->jack_detection = of_property_read_bool(node, "ti,jack-detection");
+	of_property_read_u32(node, "ti,mclk-freq", &priv->mclk_freq);
+	if (!priv->mclk_freq) {
+		dev_err(&pdev->dev, "MCLK frequency not provided\n");
+		return -EINVAL;
+	}
+
+	card->fully_routed = 1;
+
+	if (!priv->mclk_freq) {
+		dev_err(&pdev->dev, "MCLK frequency missing\n");
+		return -ENODEV;
+	}
+
+	card->dai_link = priv->dai_links;
+	card->num_links = num_links;
+
+	snd_soc_card_set_drvdata(card, priv);
+
+	ret = devm_snd_soc_register_card(&pdev->dev, card);
+	if (ret)
+		dev_err(&pdev->dev, "devm_snd_soc_register_card() failed: %d\n",
+			ret);
+
+	return ret;
+}
+
+static const struct of_device_id omap_abe_of_match[] = {
+	{.compatible = "ti,abe-twl6040", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, omap_abe_of_match);
+
+static struct platform_driver omap_abe_driver = {
+	.driver = {
+		.name = "omap-abe-twl6040",
+		.pm = &snd_soc_pm_ops,
+		.of_match_table = omap_abe_of_match,
+	},
+	.probe = omap_abe_probe,
+};
+
+static int __init omap_abe_init(void)
+{
+	int ret;
+
+	dmic_codec_dev = platform_device_register_simple("dmic-codec", -1, NULL,
+							 0);
+	if (IS_ERR(dmic_codec_dev)) {
+		pr_err("%s: dmic-codec device registration failed\n", __func__);
+		return PTR_ERR(dmic_codec_dev);
+	}
+
+	ret = platform_driver_register(&omap_abe_driver);
+	if (ret) {
+		pr_err("%s: platform driver registration failed\n", __func__);
+		platform_device_unregister(dmic_codec_dev);
+	}
+
+	return ret;
+}
+module_init(omap_abe_init);
+
+static void __exit omap_abe_exit(void)
+{
+	platform_driver_unregister(&omap_abe_driver);
+	platform_device_unregister(dmic_codec_dev);
+}
+module_exit(omap_abe_exit);
+
+MODULE_AUTHOR("Misael Lopez Cruz <misael.lopez@ti.com>");
+MODULE_DESCRIPTION("ALSA SoC for OMAP boards with ABE and twl6040 codec");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:omap-abe-twl6040");
diff -Naur --no-dereference a/sound/soc/ti/omap-dmic.c b/sound/soc/ti/omap-dmic.c
--- a/sound/soc/ti/omap-dmic.c	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/omap-dmic.c	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,541 @@
+/*
+ * omap-dmic.c  --  OMAP ASoC DMIC DAI driver
+ *
+ * Copyright (C) 2010 - 2011 Texas Instruments
+ *
+ * Author: David Lambert <dlambert@ti.com>
+ *	   Misael Lopez Cruz <misael.lopez@ti.com>
+ *	   Liam Girdwood <lrg@ti.com>
+ *	   Peter Ujfalusi <peter.ujfalusi@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA
+ *
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/err.h>
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/slab.h>
+#include <linux/pm_runtime.h>
+#include <linux/of_device.h>
+
+#include <sound/core.h>
+#include <sound/pcm.h>
+#include <sound/pcm_params.h>
+#include <sound/initval.h>
+#include <sound/soc.h>
+#include <sound/dmaengine_pcm.h>
+
+#include "omap-dmic.h"
+#include "sdma-pcm.h"
+
+struct omap_dmic {
+	struct device *dev;
+	void __iomem *io_base;
+	struct clk *fclk;
+	struct pm_qos_request pm_qos_req;
+	int latency;
+	int fclk_freq;
+	int out_freq;
+	int clk_div;
+	int sysclk;
+	int threshold;
+	u32 ch_enabled;
+	bool active;
+	struct mutex mutex;
+
+	struct snd_dmaengine_dai_dma_data dma_data;
+};
+
+static inline void omap_dmic_write(struct omap_dmic *dmic, u16 reg, u32 val)
+{
+	writel_relaxed(val, dmic->io_base + reg);
+}
+
+static inline int omap_dmic_read(struct omap_dmic *dmic, u16 reg)
+{
+	return readl_relaxed(dmic->io_base + reg);
+}
+
+static inline void omap_dmic_start(struct omap_dmic *dmic)
+{
+	u32 ctrl = omap_dmic_read(dmic, OMAP_DMIC_CTRL_REG);
+
+	/* Configure DMA controller */
+	omap_dmic_write(dmic, OMAP_DMIC_DMAENABLE_SET_REG,
+			OMAP_DMIC_DMA_ENABLE);
+
+	omap_dmic_write(dmic, OMAP_DMIC_CTRL_REG, ctrl | dmic->ch_enabled);
+}
+
+static inline void omap_dmic_stop(struct omap_dmic *dmic)
+{
+	u32 ctrl = omap_dmic_read(dmic, OMAP_DMIC_CTRL_REG);
+	omap_dmic_write(dmic, OMAP_DMIC_CTRL_REG,
+			ctrl & ~OMAP_DMIC_UP_ENABLE_MASK);
+
+	/* Disable DMA request generation */
+	omap_dmic_write(dmic, OMAP_DMIC_DMAENABLE_CLR_REG,
+			OMAP_DMIC_DMA_ENABLE);
+
+}
+
+static inline int dmic_is_enabled(struct omap_dmic *dmic)
+{
+	return omap_dmic_read(dmic, OMAP_DMIC_CTRL_REG) &
+						OMAP_DMIC_UP_ENABLE_MASK;
+}
+
+static int omap_dmic_dai_startup(struct snd_pcm_substream *substream,
+				  struct snd_soc_dai *dai)
+{
+	struct omap_dmic *dmic = snd_soc_dai_get_drvdata(dai);
+	int ret = 0;
+
+	mutex_lock(&dmic->mutex);
+
+	if (!dai->active)
+		dmic->active = 1;
+	else
+		ret = -EBUSY;
+
+	mutex_unlock(&dmic->mutex);
+
+	return ret;
+}
+
+static void omap_dmic_dai_shutdown(struct snd_pcm_substream *substream,
+				    struct snd_soc_dai *dai)
+{
+	struct omap_dmic *dmic = snd_soc_dai_get_drvdata(dai);
+
+	mutex_lock(&dmic->mutex);
+
+	pm_qos_remove_request(&dmic->pm_qos_req);
+
+	if (!dai->active)
+		dmic->active = 0;
+
+	mutex_unlock(&dmic->mutex);
+}
+
+static int omap_dmic_select_divider(struct omap_dmic *dmic, int sample_rate)
+{
+	int divider = -EINVAL;
+
+	/*
+	 * 192KHz rate is only supported with 19.2MHz/3.84MHz clock
+	 * configuration.
+	 */
+	if (sample_rate == 192000) {
+		if (dmic->fclk_freq == 19200000 && dmic->out_freq == 3840000)
+			divider = 0x6; /* Divider: 5 (192KHz sampling rate) */
+		else
+			dev_err(dmic->dev,
+				"invalid clock configuration for 192KHz\n");
+
+		return divider;
+	}
+
+	switch (dmic->out_freq) {
+	case 1536000:
+		if (dmic->fclk_freq != 24576000)
+			goto div_err;
+		divider = 0x4; /* Divider: 16 */
+		break;
+	case 2400000:
+		switch (dmic->fclk_freq) {
+		case 12000000:
+			divider = 0x5; /* Divider: 5 */
+			break;
+		case 19200000:
+			divider = 0x0; /* Divider: 8 */
+			break;
+		case 24000000:
+			divider = 0x2; /* Divider: 10 */
+			break;
+		default:
+			goto div_err;
+		}
+		break;
+	case 3072000:
+		if (dmic->fclk_freq != 24576000)
+			goto div_err;
+		divider = 0x3; /* Divider: 8 */
+		break;
+	case 3840000:
+		if (dmic->fclk_freq != 19200000)
+			goto div_err;
+		divider = 0x1; /* Divider: 5 (96KHz sampling rate) */
+		break;
+	default:
+		dev_err(dmic->dev, "invalid out frequency: %dHz\n",
+			dmic->out_freq);
+		break;
+	}
+
+	return divider;
+
+div_err:
+	dev_err(dmic->dev, "invalid out frequency %dHz for %dHz input\n",
+		dmic->out_freq, dmic->fclk_freq);
+	return -EINVAL;
+}
+
+static int omap_dmic_dai_hw_params(struct snd_pcm_substream *substream,
+				    struct snd_pcm_hw_params *params,
+				    struct snd_soc_dai *dai)
+{
+	struct omap_dmic *dmic = snd_soc_dai_get_drvdata(dai);
+	struct snd_dmaengine_dai_dma_data *dma_data;
+	int channels;
+
+	dmic->clk_div = omap_dmic_select_divider(dmic, params_rate(params));
+	if (dmic->clk_div < 0) {
+		dev_err(dmic->dev, "no valid divider for %dHz from %dHz\n",
+			dmic->out_freq, dmic->fclk_freq);
+		return -EINVAL;
+	}
+
+	dmic->ch_enabled = 0;
+	channels = params_channels(params);
+	switch (channels) {
+	case 6:
+		dmic->ch_enabled |= OMAP_DMIC_UP3_ENABLE;
+		/* fall through */
+	case 4:
+		dmic->ch_enabled |= OMAP_DMIC_UP2_ENABLE;
+		/* fall through */
+	case 2:
+		dmic->ch_enabled |= OMAP_DMIC_UP1_ENABLE;
+		break;
+	default:
+		dev_err(dmic->dev, "invalid number of legacy channels\n");
+		return -EINVAL;
+	}
+
+	/* packet size is threshold * channels */
+	dma_data = snd_soc_dai_get_dma_data(dai, substream);
+	dma_data->maxburst = dmic->threshold * channels;
+	dmic->latency = (OMAP_DMIC_THRES_MAX - dmic->threshold) * USEC_PER_SEC /
+			params_rate(params);
+
+	return 0;
+}
+
+static int omap_dmic_dai_prepare(struct snd_pcm_substream *substream,
+				  struct snd_soc_dai *dai)
+{
+	struct omap_dmic *dmic = snd_soc_dai_get_drvdata(dai);
+	u32 ctrl;
+
+	if (pm_qos_request_active(&dmic->pm_qos_req))
+		pm_qos_update_request(&dmic->pm_qos_req, dmic->latency);
+
+	/* Configure uplink threshold */
+	omap_dmic_write(dmic, OMAP_DMIC_FIFO_CTRL_REG, dmic->threshold);
+
+	ctrl = omap_dmic_read(dmic, OMAP_DMIC_CTRL_REG);
+
+	/* Set dmic out format */
+	ctrl &= ~(OMAP_DMIC_FORMAT | OMAP_DMIC_POLAR_MASK);
+	ctrl |= (OMAP_DMICOUTFORMAT_LJUST | OMAP_DMIC_POLAR1 |
+		 OMAP_DMIC_POLAR2 | OMAP_DMIC_POLAR3);
+
+	/* Configure dmic clock divider */
+	ctrl &= ~OMAP_DMIC_CLK_DIV_MASK;
+	ctrl |= OMAP_DMIC_CLK_DIV(dmic->clk_div);
+
+	omap_dmic_write(dmic, OMAP_DMIC_CTRL_REG, ctrl);
+
+	omap_dmic_write(dmic, OMAP_DMIC_CTRL_REG,
+			ctrl | OMAP_DMICOUTFORMAT_LJUST | OMAP_DMIC_POLAR1 |
+			OMAP_DMIC_POLAR2 | OMAP_DMIC_POLAR3);
+
+	return 0;
+}
+
+static int omap_dmic_dai_trigger(struct snd_pcm_substream *substream,
+				  int cmd, struct snd_soc_dai *dai)
+{
+	struct omap_dmic *dmic = snd_soc_dai_get_drvdata(dai);
+
+	switch (cmd) {
+	case SNDRV_PCM_TRIGGER_START:
+		omap_dmic_start(dmic);
+		break;
+	case SNDRV_PCM_TRIGGER_STOP:
+		omap_dmic_stop(dmic);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static int omap_dmic_select_fclk(struct omap_dmic *dmic, int clk_id,
+				 unsigned int freq)
+{
+	struct clk *parent_clk, *mux;
+	char *parent_clk_name;
+	int ret = 0;
+
+	switch (freq) {
+	case 12000000:
+	case 19200000:
+	case 24000000:
+	case 24576000:
+		break;
+	default:
+		dev_err(dmic->dev, "invalid input frequency: %dHz\n", freq);
+		dmic->fclk_freq = 0;
+		return -EINVAL;
+	}
+
+	if (dmic->sysclk == clk_id) {
+		dmic->fclk_freq = freq;
+		return 0;
+	}
+
+	/* re-parent not allowed if a stream is ongoing */
+	if (dmic->active && dmic_is_enabled(dmic)) {
+		dev_err(dmic->dev, "can't re-parent when DMIC active\n");
+		return -EBUSY;
+	}
+
+	switch (clk_id) {
+	case OMAP_DMIC_SYSCLK_PAD_CLKS:
+		parent_clk_name = "pad_clks_ck";
+		break;
+	case OMAP_DMIC_SYSCLK_SLIMBLUS_CLKS:
+		parent_clk_name = "slimbus_clk";
+		break;
+	case OMAP_DMIC_SYSCLK_SYNC_MUX_CLKS:
+		parent_clk_name = "dmic_sync_mux_ck";
+		break;
+	default:
+		dev_err(dmic->dev, "fclk clk_id (%d) not supported\n", clk_id);
+		return -EINVAL;
+	}
+
+	parent_clk = clk_get(dmic->dev, parent_clk_name);
+	if (IS_ERR(parent_clk)) {
+		dev_err(dmic->dev, "can't get %s\n", parent_clk_name);
+		return -ENODEV;
+	}
+
+	mux = clk_get_parent(dmic->fclk);
+	if (IS_ERR(mux)) {
+		dev_err(dmic->dev, "can't get fck mux parent\n");
+		clk_put(parent_clk);
+		return -ENODEV;
+	}
+
+	mutex_lock(&dmic->mutex);
+	if (dmic->active) {
+		/* disable clock while reparenting */
+		pm_runtime_put_sync(dmic->dev);
+		ret = clk_set_parent(mux, parent_clk);
+		pm_runtime_get_sync(dmic->dev);
+	} else {
+		ret = clk_set_parent(mux, parent_clk);
+	}
+	mutex_unlock(&dmic->mutex);
+
+	if (ret < 0) {
+		dev_err(dmic->dev, "re-parent failed\n");
+		goto err_busy;
+	}
+
+	dmic->sysclk = clk_id;
+	dmic->fclk_freq = freq;
+
+err_busy:
+	clk_put(mux);
+	clk_put(parent_clk);
+
+	return ret;
+}
+
+static int omap_dmic_select_outclk(struct omap_dmic *dmic, int clk_id,
+				    unsigned int freq)
+{
+	int ret = 0;
+
+	if (clk_id != OMAP_DMIC_ABE_DMIC_CLK) {
+		dev_err(dmic->dev, "output clk_id (%d) not supported\n",
+			clk_id);
+		return -EINVAL;
+	}
+
+	switch (freq) {
+	case 1536000:
+	case 2400000:
+	case 3072000:
+	case 3840000:
+		dmic->out_freq = freq;
+		break;
+	default:
+		dev_err(dmic->dev, "invalid out frequency: %dHz\n", freq);
+		dmic->out_freq = 0;
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
+static int omap_dmic_set_dai_sysclk(struct snd_soc_dai *dai, int clk_id,
+				    unsigned int freq, int dir)
+{
+	struct omap_dmic *dmic = snd_soc_dai_get_drvdata(dai);
+
+	if (dir == SND_SOC_CLOCK_IN)
+		return omap_dmic_select_fclk(dmic, clk_id, freq);
+	else if (dir == SND_SOC_CLOCK_OUT)
+		return omap_dmic_select_outclk(dmic, clk_id, freq);
+
+	dev_err(dmic->dev, "invalid clock direction (%d)\n", dir);
+	return -EINVAL;
+}
+
+static const struct snd_soc_dai_ops omap_dmic_dai_ops = {
+	.startup	= omap_dmic_dai_startup,
+	.shutdown	= omap_dmic_dai_shutdown,
+	.hw_params	= omap_dmic_dai_hw_params,
+	.prepare	= omap_dmic_dai_prepare,
+	.trigger	= omap_dmic_dai_trigger,
+	.set_sysclk	= omap_dmic_set_dai_sysclk,
+};
+
+static int omap_dmic_probe(struct snd_soc_dai *dai)
+{
+	struct omap_dmic *dmic = snd_soc_dai_get_drvdata(dai);
+
+	pm_runtime_enable(dmic->dev);
+
+	/* Disable lines while request is ongoing */
+	pm_runtime_get_sync(dmic->dev);
+	omap_dmic_write(dmic, OMAP_DMIC_CTRL_REG, 0x00);
+	pm_runtime_put_sync(dmic->dev);
+
+	/* Configure DMIC threshold value */
+	dmic->threshold = OMAP_DMIC_THRES_MAX - 3;
+
+	snd_soc_dai_init_dma_data(dai, NULL, &dmic->dma_data);
+
+	return 0;
+}
+
+static int omap_dmic_remove(struct snd_soc_dai *dai)
+{
+	struct omap_dmic *dmic = snd_soc_dai_get_drvdata(dai);
+
+	pm_runtime_disable(dmic->dev);
+
+	return 0;
+}
+
+static struct snd_soc_dai_driver omap_dmic_dai = {
+	.name = "omap-dmic",
+	.probe = omap_dmic_probe,
+	.remove = omap_dmic_remove,
+	.capture = {
+		.channels_min = 2,
+		.channels_max = 6,
+		.rates = SNDRV_PCM_RATE_96000 | SNDRV_PCM_RATE_192000,
+		.formats = SNDRV_PCM_FMTBIT_S32_LE,
+		.sig_bits = 24,
+	},
+	.ops = &omap_dmic_dai_ops,
+};
+
+static const struct snd_soc_component_driver omap_dmic_component = {
+	.name		= "omap-dmic",
+};
+
+static int asoc_dmic_probe(struct platform_device *pdev)
+{
+	struct omap_dmic *dmic;
+	struct resource *res;
+	int ret;
+
+	dmic = devm_kzalloc(&pdev->dev, sizeof(struct omap_dmic), GFP_KERNEL);
+	if (!dmic)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, dmic);
+	dmic->dev = &pdev->dev;
+	dmic->sysclk = OMAP_DMIC_SYSCLK_SYNC_MUX_CLKS;
+
+	mutex_init(&dmic->mutex);
+
+	dmic->fclk = devm_clk_get(dmic->dev, "fck");
+	if (IS_ERR(dmic->fclk)) {
+		dev_err(dmic->dev, "cant get fck\n");
+		return -ENODEV;
+	}
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dma");
+	if (!res) {
+		dev_err(dmic->dev, "invalid dma memory resource\n");
+		return -ENODEV;
+	}
+	dmic->dma_data.addr = res->start + OMAP_DMIC_DATA_REG;
+
+	dmic->dma_data.filter_data = "up_link";
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mpu");
+	dmic->io_base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(dmic->io_base))
+		return PTR_ERR(dmic->io_base);
+
+
+	ret = devm_snd_soc_register_component(&pdev->dev,
+					      &omap_dmic_component,
+					      &omap_dmic_dai, 1);
+	if (ret)
+		return ret;
+
+	ret = sdma_pcm_platform_register(&pdev->dev, NULL, "up_link");
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static const struct of_device_id omap_dmic_of_match[] = {
+	{ .compatible = "ti,omap4-dmic", },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, omap_dmic_of_match);
+
+static struct platform_driver asoc_dmic_driver = {
+	.driver = {
+		.name = "omap-dmic",
+		.of_match_table = omap_dmic_of_match,
+	},
+	.probe = asoc_dmic_probe,
+};
+
+module_platform_driver(asoc_dmic_driver);
+
+MODULE_ALIAS("platform:omap-dmic");
+MODULE_AUTHOR("Peter Ujfalusi <peter.ujfalusi@ti.com>");
+MODULE_DESCRIPTION("OMAP DMIC ASoC Interface");
+MODULE_LICENSE("GPL");
diff -Naur --no-dereference a/sound/soc/ti/omap-dmic.h b/sound/soc/ti/omap-dmic.h
--- a/sound/soc/ti/omap-dmic.h	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/omap-dmic.h	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,69 @@
+/*
+ * omap-dmic.h  --  OMAP Digital Microphone Controller
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef _OMAP_DMIC_H
+#define _OMAP_DMIC_H
+
+#define OMAP_DMIC_REVISION_REG		0x00
+#define OMAP_DMIC_SYSCONFIG_REG		0x10
+#define OMAP_DMIC_IRQSTATUS_RAW_REG	0x24
+#define OMAP_DMIC_IRQSTATUS_REG		0x28
+#define OMAP_DMIC_IRQENABLE_SET_REG	0x2C
+#define OMAP_DMIC_IRQENABLE_CLR_REG	0x30
+#define OMAP_DMIC_IRQWAKE_EN_REG	0x34
+#define OMAP_DMIC_DMAENABLE_SET_REG	0x38
+#define OMAP_DMIC_DMAENABLE_CLR_REG	0x3C
+#define OMAP_DMIC_DMAWAKEEN_REG		0x40
+#define OMAP_DMIC_CTRL_REG		0x44
+#define OMAP_DMIC_DATA_REG		0x48
+#define OMAP_DMIC_FIFO_CTRL_REG		0x4C
+#define OMAP_DMIC_FIFO_DMIC1R_DATA_REG	0x50
+#define OMAP_DMIC_FIFO_DMIC1L_DATA_REG	0x54
+#define OMAP_DMIC_FIFO_DMIC2R_DATA_REG	0x58
+#define OMAP_DMIC_FIFO_DMIC2L_DATA_REG	0x5C
+#define OMAP_DMIC_FIFO_DMIC3R_DATA_REG	0x60
+#define OMAP_DMIC_FIFO_DMIC3L_DATA_REG	0x64
+
+/* IRQSTATUS_RAW, IRQSTATUS, IRQENABLE_SET, IRQENABLE_CLR bit fields */
+#define OMAP_DMIC_IRQ			(1 << 0)
+#define OMAP_DMIC_IRQ_FULL		(1 << 1)
+#define OMAP_DMIC_IRQ_ALMST_EMPTY	(1 << 2)
+#define OMAP_DMIC_IRQ_EMPTY		(1 << 3)
+#define OMAP_DMIC_IRQ_MASK		0x07
+
+/* DMIC_DMAENABLE bit fields */
+#define OMAP_DMIC_DMA_ENABLE		0x1
+
+/* DMIC_CTRL bit fields */
+#define OMAP_DMIC_UP1_ENABLE		(1 << 0)
+#define OMAP_DMIC_UP2_ENABLE		(1 << 1)
+#define OMAP_DMIC_UP3_ENABLE		(1 << 2)
+#define OMAP_DMIC_UP_ENABLE_MASK	0x7
+#define OMAP_DMIC_FORMAT		(1 << 3)
+#define OMAP_DMIC_POLAR1		(1 << 4)
+#define OMAP_DMIC_POLAR2		(1 << 5)
+#define OMAP_DMIC_POLAR3		(1 << 6)
+#define OMAP_DMIC_POLAR_MASK		(0x7 << 4)
+#define OMAP_DMIC_CLK_DIV(x)		(((x) & 0x7) << 7)
+#define OMAP_DMIC_CLK_DIV_MASK		(0x7 << 7)
+#define	OMAP_DMIC_RESET			(1 << 10)
+
+#define OMAP_DMICOUTFORMAT_LJUST	(0 << 3)
+#define OMAP_DMICOUTFORMAT_RJUST	(1 << 3)
+
+/* DMIC_FIFO_CTRL bit fields */
+#define OMAP_DMIC_THRES_MAX		0xF
+
+enum omap_dmic_clk {
+	OMAP_DMIC_SYSCLK_PAD_CLKS,		/* PAD_CLKS */
+	OMAP_DMIC_SYSCLK_SLIMBLUS_CLKS,		/* SLIMBUS_CLK */
+	OMAP_DMIC_SYSCLK_SYNC_MUX_CLKS,		/* DMIC_SYNC_MUX_CLK */
+	OMAP_DMIC_ABE_DMIC_CLK,			/* abe_dmic_clk */
+};
+
+#endif
diff -Naur --no-dereference a/sound/soc/ti/omap-hdmi.c b/sound/soc/ti/omap-hdmi.c
--- a/sound/soc/ti/omap-hdmi.c	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/omap-hdmi.c	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,428 @@
+/*
+ * omap-hdmi-audio.c -- OMAP4+ DSS HDMI audio support library
+ *
+ * Copyright (C) 2014 Texas Instruments Incorporated - http://www.ti.com
+ *
+ * Author: Jyri Sarha <jsarha@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/err.h>
+#include <linux/string.h>
+#include <linux/platform_device.h>
+#include <sound/soc.h>
+#include <sound/pcm_params.h>
+#include <sound/dmaengine_pcm.h>
+#include <uapi/sound/asound.h>
+#include <sound/asoundef.h>
+#include <sound/omap-hdmi-audio.h>
+#include <sound/initval.h>
+
+#include "sdma-pcm.h"
+
+#define DRV_NAME "omap-hdmi-audio"
+
+struct hdmi_audio_data {
+	struct snd_soc_card *card;
+
+	const struct omap_hdmi_audio_ops *ops;
+	struct device *dssdev;
+	struct snd_dmaengine_dai_dma_data dma_data;
+	struct omap_dss_audio dss_audio;
+	struct snd_aes_iec958 iec;
+	struct snd_cea_861_aud_if cea;
+
+	struct mutex current_stream_lock;
+	struct snd_pcm_substream *current_stream;
+};
+
+static
+struct hdmi_audio_data *card_drvdata_substream(struct snd_pcm_substream *ss)
+{
+	struct snd_soc_pcm_runtime *rtd = ss->private_data;
+
+	return snd_soc_card_get_drvdata(rtd->card);
+}
+
+static void hdmi_dai_abort(struct device *dev)
+{
+	struct hdmi_audio_data *ad = dev_get_drvdata(dev);
+
+	mutex_lock(&ad->current_stream_lock);
+	if (ad->current_stream && ad->current_stream->runtime &&
+	    snd_pcm_running(ad->current_stream)) {
+		dev_err(dev, "HDMI display disabled, aborting playback\n");
+		snd_pcm_stream_lock_irq(ad->current_stream);
+		snd_pcm_stop(ad->current_stream, SNDRV_PCM_STATE_DISCONNECTED);
+		snd_pcm_stream_unlock_irq(ad->current_stream);
+	}
+	mutex_unlock(&ad->current_stream_lock);
+}
+
+static int hdmi_dai_startup(struct snd_pcm_substream *substream,
+			    struct snd_soc_dai *dai)
+{
+	struct hdmi_audio_data *ad = card_drvdata_substream(substream);
+	int ret;
+	/*
+	 * Make sure that the period bytes are multiple of the DMA packet size.
+	 * Largest packet size we use is 32 32-bit words = 128 bytes
+	 */
+	ret = snd_pcm_hw_constraint_step(substream->runtime, 0,
+					 SNDRV_PCM_HW_PARAM_PERIOD_BYTES, 128);
+	if (ret < 0) {
+		dev_err(dai->dev, "Could not apply period constraint: %d\n",
+			ret);
+		return ret;
+	}
+	ret = snd_pcm_hw_constraint_step(substream->runtime, 0,
+					 SNDRV_PCM_HW_PARAM_BUFFER_BYTES, 128);
+	if (ret < 0) {
+		dev_err(dai->dev, "Could not apply buffer constraint: %d\n",
+			ret);
+		return ret;
+	}
+
+	snd_soc_dai_set_dma_data(dai, substream, &ad->dma_data);
+
+	mutex_lock(&ad->current_stream_lock);
+	ad->current_stream = substream;
+	mutex_unlock(&ad->current_stream_lock);
+
+	ret = ad->ops->audio_startup(ad->dssdev, hdmi_dai_abort);
+
+	if (ret) {
+		mutex_lock(&ad->current_stream_lock);
+		ad->current_stream = NULL;
+		mutex_unlock(&ad->current_stream_lock);
+	}
+
+	return ret;
+}
+
+static int hdmi_dai_hw_params(struct snd_pcm_substream *substream,
+			      struct snd_pcm_hw_params *params,
+			      struct snd_soc_dai *dai)
+{
+	struct hdmi_audio_data *ad = card_drvdata_substream(substream);
+	struct snd_aes_iec958 *iec = &ad->iec;
+	struct snd_cea_861_aud_if *cea = &ad->cea;
+
+	WARN_ON(ad->current_stream != substream);
+
+	switch (params_format(params)) {
+	case SNDRV_PCM_FORMAT_S16_LE:
+		ad->dma_data.maxburst = 16;
+		break;
+	case SNDRV_PCM_FORMAT_S24_LE:
+		ad->dma_data.maxburst = 32;
+		break;
+	default:
+		dev_err(dai->dev, "format not supported!\n");
+		return -EINVAL;
+	}
+
+	ad->dss_audio.iec = iec;
+	ad->dss_audio.cea = cea;
+	/*
+	 * fill the IEC-60958 channel status word
+	 */
+	/* initialize the word bytes */
+	memset(iec->status, 0, sizeof(iec->status));
+
+	/* specify IEC-60958-3 (commercial use) */
+	iec->status[0] &= ~IEC958_AES0_PROFESSIONAL;
+
+	/* specify that the audio is LPCM*/
+	iec->status[0] &= ~IEC958_AES0_NONAUDIO;
+
+	iec->status[0] |= IEC958_AES0_CON_NOT_COPYRIGHT;
+
+	iec->status[0] |= IEC958_AES0_CON_EMPHASIS_NONE;
+
+	iec->status[1] = IEC958_AES1_CON_GENERAL;
+
+	iec->status[2] |= IEC958_AES2_CON_SOURCE_UNSPEC;
+
+	iec->status[2] |= IEC958_AES2_CON_CHANNEL_UNSPEC;
+
+	switch (params_rate(params)) {
+	case 32000:
+		iec->status[3] |= IEC958_AES3_CON_FS_32000;
+		break;
+	case 44100:
+		iec->status[3] |= IEC958_AES3_CON_FS_44100;
+		break;
+	case 48000:
+		iec->status[3] |= IEC958_AES3_CON_FS_48000;
+		break;
+	case 88200:
+		iec->status[3] |= IEC958_AES3_CON_FS_88200;
+		break;
+	case 96000:
+		iec->status[3] |= IEC958_AES3_CON_FS_96000;
+		break;
+	case 176400:
+		iec->status[3] |= IEC958_AES3_CON_FS_176400;
+		break;
+	case 192000:
+		iec->status[3] |= IEC958_AES3_CON_FS_192000;
+		break;
+	default:
+		dev_err(dai->dev, "rate not supported!\n");
+		return -EINVAL;
+	}
+
+	/* specify the clock accuracy */
+	iec->status[3] |= IEC958_AES3_CON_CLOCK_1000PPM;
+
+	/*
+	 * specify the word length. The same word length value can mean
+	 * two different lengths. Hence, we need to specify the maximum
+	 * word length as well.
+	 */
+	switch (params_format(params)) {
+	case SNDRV_PCM_FORMAT_S16_LE:
+		iec->status[4] |= IEC958_AES4_CON_WORDLEN_20_16;
+		iec->status[4] &= ~IEC958_AES4_CON_MAX_WORDLEN_24;
+		break;
+	case SNDRV_PCM_FORMAT_S24_LE:
+		iec->status[4] |= IEC958_AES4_CON_WORDLEN_24_20;
+		iec->status[4] |= IEC958_AES4_CON_MAX_WORDLEN_24;
+		break;
+	default:
+		dev_err(dai->dev, "format not supported!\n");
+		return -EINVAL;
+	}
+
+	/*
+	 * Fill the CEA-861 audio infoframe (see spec for details)
+	 */
+
+	cea->db1_ct_cc = (params_channels(params) - 1)
+		& CEA861_AUDIO_INFOFRAME_DB1CC;
+	cea->db1_ct_cc |= CEA861_AUDIO_INFOFRAME_DB1CT_FROM_STREAM;
+
+	cea->db2_sf_ss = CEA861_AUDIO_INFOFRAME_DB2SF_FROM_STREAM;
+	cea->db2_sf_ss |= CEA861_AUDIO_INFOFRAME_DB2SS_FROM_STREAM;
+
+	cea->db3 = 0; /* not used, all zeros */
+
+	if (params_channels(params) == 2)
+		cea->db4_ca = 0x0;
+	else if (params_channels(params) == 6)
+		cea->db4_ca = 0xb;
+	else
+		cea->db4_ca = 0x13;
+
+	if (cea->db4_ca == 0x00)
+		cea->db5_dminh_lsv = CEA861_AUDIO_INFOFRAME_DB5_DM_INH_PERMITTED;
+	else
+		cea->db5_dminh_lsv = CEA861_AUDIO_INFOFRAME_DB5_DM_INH_PROHIBITED;
+
+	/* the expression is trivial but makes clear what we are doing */
+	cea->db5_dminh_lsv |= (0 & CEA861_AUDIO_INFOFRAME_DB5_LSV);
+
+	return ad->ops->audio_config(ad->dssdev, &ad->dss_audio);
+}
+
+static int hdmi_dai_trigger(struct snd_pcm_substream *substream, int cmd,
+			    struct snd_soc_dai *dai)
+{
+	struct hdmi_audio_data *ad = card_drvdata_substream(substream);
+	int err = 0;
+
+	WARN_ON(ad->current_stream != substream);
+
+	switch (cmd) {
+	case SNDRV_PCM_TRIGGER_START:
+	case SNDRV_PCM_TRIGGER_RESUME:
+	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
+		err = ad->ops->audio_start(ad->dssdev);
+		break;
+	case SNDRV_PCM_TRIGGER_STOP:
+	case SNDRV_PCM_TRIGGER_SUSPEND:
+	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
+		ad->ops->audio_stop(ad->dssdev);
+		break;
+	default:
+		err = -EINVAL;
+	}
+	return err;
+}
+
+static void hdmi_dai_shutdown(struct snd_pcm_substream *substream,
+			      struct snd_soc_dai *dai)
+{
+	struct hdmi_audio_data *ad = card_drvdata_substream(substream);
+
+	WARN_ON(ad->current_stream != substream);
+
+	ad->ops->audio_shutdown(ad->dssdev);
+
+	mutex_lock(&ad->current_stream_lock);
+	ad->current_stream = NULL;
+	mutex_unlock(&ad->current_stream_lock);
+}
+
+static const struct snd_soc_dai_ops hdmi_dai_ops = {
+	.startup	= hdmi_dai_startup,
+	.hw_params	= hdmi_dai_hw_params,
+	.trigger	= hdmi_dai_trigger,
+	.shutdown	= hdmi_dai_shutdown,
+};
+
+static const struct snd_soc_component_driver omap_hdmi_component = {
+	.name = "omapdss_hdmi",
+};
+
+static struct snd_soc_dai_driver omap5_hdmi_dai = {
+	.name = "omap5-hdmi-dai",
+	.playback = {
+		.channels_min = 2,
+		.channels_max = 8,
+		.rates = (SNDRV_PCM_RATE_32000 | SNDRV_PCM_RATE_44100 |
+			  SNDRV_PCM_RATE_48000 | SNDRV_PCM_RATE_88200 |
+			  SNDRV_PCM_RATE_96000 | SNDRV_PCM_RATE_176400 |
+			  SNDRV_PCM_RATE_192000),
+		.formats = SNDRV_PCM_FMTBIT_S16_LE,
+	},
+	.ops = &hdmi_dai_ops,
+};
+
+static struct snd_soc_dai_driver omap4_hdmi_dai = {
+	.name = "omap4-hdmi-dai",
+	.playback = {
+		.channels_min = 2,
+		.channels_max = 8,
+		.rates = (SNDRV_PCM_RATE_32000 | SNDRV_PCM_RATE_44100 |
+			  SNDRV_PCM_RATE_48000 | SNDRV_PCM_RATE_88200 |
+			  SNDRV_PCM_RATE_96000 | SNDRV_PCM_RATE_176400 |
+			  SNDRV_PCM_RATE_192000),
+		.formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S24_LE,
+	},
+	.ops = &hdmi_dai_ops,
+};
+
+static int omap_hdmi_audio_probe(struct platform_device *pdev)
+{
+	struct omap_hdmi_audio_pdata *ha = pdev->dev.platform_data;
+	struct device *dev = &pdev->dev;
+	struct hdmi_audio_data *ad;
+	struct snd_soc_dai_driver *dai_drv;
+	struct snd_soc_card *card;
+	int ret;
+	int id = SNDRV_DEFAULT_IDX1;
+
+	if (!ha) {
+		dev_err(dev, "No platform data\n");
+		return -EINVAL;
+	}
+
+	/* Get the id of the parent (the HDMI HW IP) */
+	if (ha->dev->of_node) {
+		id = of_alias_get_id(ha->dev->of_node, "sound");
+		if (id < 0)
+			id = SNDRV_DEFAULT_IDX1;
+	}
+
+	ad = devm_kzalloc(dev, sizeof(*ad), GFP_KERNEL);
+	if (!ad)
+		return -ENOMEM;
+	ad->dssdev = ha->dev;
+	ad->ops = ha->ops;
+	ad->dma_data.addr = ha->audio_dma_addr;
+	ad->dma_data.filter_data = "audio_tx";
+	ad->dma_data.addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+	mutex_init(&ad->current_stream_lock);
+
+	switch (ha->version) {
+	case 4:
+		dai_drv = &omap4_hdmi_dai;
+		break;
+	case 5:
+		dai_drv = &omap5_hdmi_dai;
+		break;
+	default:
+		return -EINVAL;
+	}
+	ret = devm_snd_soc_register_component(ad->dssdev, &omap_hdmi_component,
+					 dai_drv, 1);
+	if (ret)
+		return ret;
+
+	ret = sdma_pcm_platform_register(ad->dssdev, "audio_tx", NULL);
+	if (ret)
+		return ret;
+
+	card = devm_kzalloc(dev, sizeof(*card), GFP_KERNEL);
+	if (!card)
+		return -ENOMEM;
+
+	card->name = devm_kasprintf(dev, GFP_KERNEL,
+				    "HDMI %s", dev_name(ad->dssdev));
+	if (!card->name)
+		return -ENOMEM;
+
+	card->id_hint = id;
+	card->owner = THIS_MODULE;
+	card->dai_link =
+		devm_kzalloc(dev, sizeof(*(card->dai_link)), GFP_KERNEL);
+	if (!card->dai_link)
+		return -ENOMEM;
+	card->dai_link->name = card->name;
+	card->dai_link->stream_name = card->name;
+	card->dai_link->cpu_dai_name = dev_name(ad->dssdev);
+	card->dai_link->platform_name = dev_name(ad->dssdev);
+	card->dai_link->codec_name = "snd-soc-dummy";
+	card->dai_link->codec_dai_name = "snd-soc-dummy-dai";
+	card->num_links = 1;
+	card->dev = dev;
+
+	ret = snd_soc_register_card(card);
+	if (ret) {
+		dev_err(dev, "snd_soc_register_card failed (%d)\n", ret);
+		return ret;
+	}
+
+	ad->card = card;
+	snd_soc_card_set_drvdata(card, ad);
+
+	dev_set_drvdata(dev, ad);
+
+	return 0;
+}
+
+static int omap_hdmi_audio_remove(struct platform_device *pdev)
+{
+	struct hdmi_audio_data *ad = platform_get_drvdata(pdev);
+
+	snd_soc_unregister_card(ad->card);
+	return 0;
+}
+
+static struct platform_driver hdmi_audio_driver = {
+	.driver = {
+		.name = DRV_NAME,
+	},
+	.probe = omap_hdmi_audio_probe,
+	.remove = omap_hdmi_audio_remove,
+};
+
+module_platform_driver(hdmi_audio_driver);
+
+MODULE_AUTHOR("Jyri Sarha <jsarha@ti.com>");
+MODULE_DESCRIPTION("OMAP HDMI Audio Driver");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:" DRV_NAME);
diff -Naur --no-dereference a/sound/soc/ti/omap-mcbsp.c b/sound/soc/ti/omap-mcbsp.c
--- a/sound/soc/ti/omap-mcbsp.c	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/omap-mcbsp.c	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,1479 @@
+/*
+ * omap-mcbsp.c  --  OMAP ALSA SoC DAI driver using McBSP port
+ *
+ * Copyright (C) 2008 Nokia Corporation
+ *
+ * Contact: Jarkko Nikula <jarkko.nikula@bitmer.com>
+ *          Peter Ujfalusi <peter.ujfalusi@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA
+ *
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/pm_runtime.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <sound/core.h>
+#include <sound/pcm.h>
+#include <sound/pcm_params.h>
+#include <sound/initval.h>
+#include <sound/soc.h>
+#include <sound/dmaengine_pcm.h>
+
+#include "omap-mcbsp-priv.h"
+#include "omap-mcbsp.h"
+#include "sdma-pcm.h"
+
+#define OMAP_MCBSP_RATES	(SNDRV_PCM_RATE_8000_96000)
+
+enum {
+	OMAP_MCBSP_WORD_8 = 0,
+	OMAP_MCBSP_WORD_12,
+	OMAP_MCBSP_WORD_16,
+	OMAP_MCBSP_WORD_20,
+	OMAP_MCBSP_WORD_24,
+	OMAP_MCBSP_WORD_32,
+};
+
+static void omap_mcbsp_dump_reg(struct omap_mcbsp *mcbsp)
+{
+	dev_dbg(mcbsp->dev, "**** McBSP%d regs ****\n", mcbsp->id);
+	dev_dbg(mcbsp->dev, "DRR2:  0x%04x\n", MCBSP_READ(mcbsp, DRR2));
+	dev_dbg(mcbsp->dev, "DRR1:  0x%04x\n", MCBSP_READ(mcbsp, DRR1));
+	dev_dbg(mcbsp->dev, "DXR2:  0x%04x\n", MCBSP_READ(mcbsp, DXR2));
+	dev_dbg(mcbsp->dev, "DXR1:  0x%04x\n", MCBSP_READ(mcbsp, DXR1));
+	dev_dbg(mcbsp->dev, "SPCR2: 0x%04x\n", MCBSP_READ(mcbsp, SPCR2));
+	dev_dbg(mcbsp->dev, "SPCR1: 0x%04x\n", MCBSP_READ(mcbsp, SPCR1));
+	dev_dbg(mcbsp->dev, "RCR2:  0x%04x\n", MCBSP_READ(mcbsp, RCR2));
+	dev_dbg(mcbsp->dev, "RCR1:  0x%04x\n", MCBSP_READ(mcbsp, RCR1));
+	dev_dbg(mcbsp->dev, "XCR2:  0x%04x\n", MCBSP_READ(mcbsp, XCR2));
+	dev_dbg(mcbsp->dev, "XCR1:  0x%04x\n", MCBSP_READ(mcbsp, XCR1));
+	dev_dbg(mcbsp->dev, "SRGR2: 0x%04x\n", MCBSP_READ(mcbsp, SRGR2));
+	dev_dbg(mcbsp->dev, "SRGR1: 0x%04x\n", MCBSP_READ(mcbsp, SRGR1));
+	dev_dbg(mcbsp->dev, "PCR0:  0x%04x\n", MCBSP_READ(mcbsp, PCR0));
+	dev_dbg(mcbsp->dev, "***********************\n");
+}
+
+static int omap2_mcbsp_set_clks_src(struct omap_mcbsp *mcbsp, u8 fck_src_id)
+{
+	struct clk *fck_src;
+	const char *src;
+	int r;
+
+	if (fck_src_id == MCBSP_CLKS_PAD_SRC)
+		src = "pad_fck";
+	else if (fck_src_id == MCBSP_CLKS_PRCM_SRC)
+		src = "prcm_fck";
+	else
+		return -EINVAL;
+
+	fck_src = clk_get(mcbsp->dev, src);
+	if (IS_ERR(fck_src)) {
+		dev_err(mcbsp->dev, "CLKS: could not clk_get() %s\n", src);
+		return -EINVAL;
+	}
+
+	pm_runtime_put_sync(mcbsp->dev);
+
+	r = clk_set_parent(mcbsp->fclk, fck_src);
+	if (r) {
+		dev_err(mcbsp->dev, "CLKS: could not clk_set_parent() to %s\n",
+			src);
+		clk_put(fck_src);
+		return r;
+	}
+
+	pm_runtime_get_sync(mcbsp->dev);
+
+	clk_put(fck_src);
+
+	return 0;
+}
+
+static irqreturn_t omap_mcbsp_irq_handler(int irq, void *data)
+{
+	struct omap_mcbsp *mcbsp = data;
+	u16 irqst;
+
+	irqst = MCBSP_READ(mcbsp, IRQST);
+	dev_dbg(mcbsp->dev, "IRQ callback : 0x%x\n", irqst);
+
+	if (irqst & RSYNCERREN)
+		dev_err(mcbsp->dev, "RX Frame Sync Error!\n");
+	if (irqst & RFSREN)
+		dev_dbg(mcbsp->dev, "RX Frame Sync\n");
+	if (irqst & REOFEN)
+		dev_dbg(mcbsp->dev, "RX End Of Frame\n");
+	if (irqst & RRDYEN)
+		dev_dbg(mcbsp->dev, "RX Buffer Threshold Reached\n");
+	if (irqst & RUNDFLEN)
+		dev_err(mcbsp->dev, "RX Buffer Underflow!\n");
+	if (irqst & ROVFLEN)
+		dev_err(mcbsp->dev, "RX Buffer Overflow!\n");
+
+	if (irqst & XSYNCERREN)
+		dev_err(mcbsp->dev, "TX Frame Sync Error!\n");
+	if (irqst & XFSXEN)
+		dev_dbg(mcbsp->dev, "TX Frame Sync\n");
+	if (irqst & XEOFEN)
+		dev_dbg(mcbsp->dev, "TX End Of Frame\n");
+	if (irqst & XRDYEN)
+		dev_dbg(mcbsp->dev, "TX Buffer threshold Reached\n");
+	if (irqst & XUNDFLEN)
+		dev_err(mcbsp->dev, "TX Buffer Underflow!\n");
+	if (irqst & XOVFLEN)
+		dev_err(mcbsp->dev, "TX Buffer Overflow!\n");
+	if (irqst & XEMPTYEOFEN)
+		dev_dbg(mcbsp->dev, "TX Buffer empty at end of frame\n");
+
+	MCBSP_WRITE(mcbsp, IRQST, irqst);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t omap_mcbsp_tx_irq_handler(int irq, void *data)
+{
+	struct omap_mcbsp *mcbsp = data;
+	u16 irqst_spcr2;
+
+	irqst_spcr2 = MCBSP_READ(mcbsp, SPCR2);
+	dev_dbg(mcbsp->dev, "TX IRQ callback : 0x%x\n", irqst_spcr2);
+
+	if (irqst_spcr2 & XSYNC_ERR) {
+		dev_err(mcbsp->dev, "TX Frame Sync Error! : 0x%x\n",
+			irqst_spcr2);
+		/* Writing zero to XSYNC_ERR clears the IRQ */
+		MCBSP_WRITE(mcbsp, SPCR2, MCBSP_READ_CACHE(mcbsp, SPCR2));
+	}
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t omap_mcbsp_rx_irq_handler(int irq, void *data)
+{
+	struct omap_mcbsp *mcbsp = data;
+	u16 irqst_spcr1;
+
+	irqst_spcr1 = MCBSP_READ(mcbsp, SPCR1);
+	dev_dbg(mcbsp->dev, "RX IRQ callback : 0x%x\n", irqst_spcr1);
+
+	if (irqst_spcr1 & RSYNC_ERR) {
+		dev_err(mcbsp->dev, "RX Frame Sync Error! : 0x%x\n",
+			irqst_spcr1);
+		/* Writing zero to RSYNC_ERR clears the IRQ */
+		MCBSP_WRITE(mcbsp, SPCR1, MCBSP_READ_CACHE(mcbsp, SPCR1));
+	}
+
+	return IRQ_HANDLED;
+}
+
+/*
+ * omap_mcbsp_config simply write a config to the
+ * appropriate McBSP.
+ * You either call this function or set the McBSP registers
+ * by yourself before calling omap_mcbsp_start().
+ */
+static void omap_mcbsp_config(struct omap_mcbsp *mcbsp,
+			      const struct omap_mcbsp_reg_cfg *config)
+{
+	dev_dbg(mcbsp->dev, "Configuring McBSP%d  phys_base: 0x%08lx\n",
+		mcbsp->id, mcbsp->phys_base);
+
+	/* We write the given config */
+	MCBSP_WRITE(mcbsp, SPCR2, config->spcr2);
+	MCBSP_WRITE(mcbsp, SPCR1, config->spcr1);
+	MCBSP_WRITE(mcbsp, RCR2, config->rcr2);
+	MCBSP_WRITE(mcbsp, RCR1, config->rcr1);
+	MCBSP_WRITE(mcbsp, XCR2, config->xcr2);
+	MCBSP_WRITE(mcbsp, XCR1, config->xcr1);
+	MCBSP_WRITE(mcbsp, SRGR2, config->srgr2);
+	MCBSP_WRITE(mcbsp, SRGR1, config->srgr1);
+	MCBSP_WRITE(mcbsp, MCR2, config->mcr2);
+	MCBSP_WRITE(mcbsp, MCR1, config->mcr1);
+	MCBSP_WRITE(mcbsp, PCR0, config->pcr0);
+	if (mcbsp->pdata->has_ccr) {
+		MCBSP_WRITE(mcbsp, XCCR, config->xccr);
+		MCBSP_WRITE(mcbsp, RCCR, config->rccr);
+	}
+	/* Enable wakeup behavior */
+	if (mcbsp->pdata->has_wakeup)
+		MCBSP_WRITE(mcbsp, WAKEUPEN, XRDYEN | RRDYEN);
+
+	/* Enable TX/RX sync error interrupts by default */
+	if (mcbsp->irq)
+		MCBSP_WRITE(mcbsp, IRQEN, RSYNCERREN | XSYNCERREN |
+			    RUNDFLEN | ROVFLEN | XUNDFLEN | XOVFLEN);
+}
+
+/**
+ * omap_mcbsp_dma_reg_params - returns the address of mcbsp data register
+ * @mcbsp: omap_mcbsp struct for the McBSP instance
+ * @stream: Stream direction (playback/capture)
+ *
+ * Returns the address of mcbsp data transmit register or data receive register
+ * to be used by DMA for transferring/receiving data
+ */
+static int omap_mcbsp_dma_reg_params(struct omap_mcbsp *mcbsp,
+				     unsigned int stream)
+{
+	int data_reg;
+
+	if (stream == SNDRV_PCM_STREAM_PLAYBACK) {
+		if (mcbsp->pdata->reg_size == 2)
+			data_reg = OMAP_MCBSP_REG_DXR1;
+		else
+			data_reg = OMAP_MCBSP_REG_DXR;
+	} else {
+		if (mcbsp->pdata->reg_size == 2)
+			data_reg = OMAP_MCBSP_REG_DRR1;
+		else
+			data_reg = OMAP_MCBSP_REG_DRR;
+	}
+
+	return mcbsp->phys_dma_base + data_reg * mcbsp->pdata->reg_step;
+}
+
+/*
+ * omap_mcbsp_set_rx_threshold configures the transmit threshold in words.
+ * The threshold parameter is 1 based, and it is converted (threshold - 1)
+ * for the THRSH2 register.
+ */
+static void omap_mcbsp_set_tx_threshold(struct omap_mcbsp *mcbsp, u16 threshold)
+{
+	if (threshold && threshold <= mcbsp->max_tx_thres)
+		MCBSP_WRITE(mcbsp, THRSH2, threshold - 1);
+}
+
+/*
+ * omap_mcbsp_set_rx_threshold configures the receive threshold in words.
+ * The threshold parameter is 1 based, and it is converted (threshold - 1)
+ * for the THRSH1 register.
+ */
+static void omap_mcbsp_set_rx_threshold(struct omap_mcbsp *mcbsp, u16 threshold)
+{
+	if (threshold && threshold <= mcbsp->max_rx_thres)
+		MCBSP_WRITE(mcbsp, THRSH1, threshold - 1);
+}
+
+/*
+ * omap_mcbsp_get_tx_delay returns the number of used slots in the McBSP FIFO
+ */
+static u16 omap_mcbsp_get_tx_delay(struct omap_mcbsp *mcbsp)
+{
+	u16 buffstat;
+
+	/* Returns the number of free locations in the buffer */
+	buffstat = MCBSP_READ(mcbsp, XBUFFSTAT);
+
+	/* Number of slots are different in McBSP ports */
+	return mcbsp->pdata->buffer_size - buffstat;
+}
+
+/*
+ * omap_mcbsp_get_rx_delay returns the number of free slots in the McBSP FIFO
+ * to reach the threshold value (when the DMA will be triggered to read it)
+ */
+static u16 omap_mcbsp_get_rx_delay(struct omap_mcbsp *mcbsp)
+{
+	u16 buffstat, threshold;
+
+	/* Returns the number of used locations in the buffer */
+	buffstat = MCBSP_READ(mcbsp, RBUFFSTAT);
+	/* RX threshold */
+	threshold = MCBSP_READ(mcbsp, THRSH1);
+
+	/* Return the number of location till we reach the threshold limit */
+	if (threshold <= buffstat)
+		return 0;
+	else
+		return threshold - buffstat;
+}
+
+static int omap_mcbsp_request(struct omap_mcbsp *mcbsp)
+{
+	void *reg_cache;
+	int err;
+
+	reg_cache = kzalloc(mcbsp->reg_cache_size, GFP_KERNEL);
+	if (!reg_cache)
+		return -ENOMEM;
+
+	spin_lock(&mcbsp->lock);
+	if (!mcbsp->free) {
+		dev_err(mcbsp->dev, "McBSP%d is currently in use\n", mcbsp->id);
+		err = -EBUSY;
+		goto err_kfree;
+	}
+
+	mcbsp->free = false;
+	mcbsp->reg_cache = reg_cache;
+	spin_unlock(&mcbsp->lock);
+
+	if(mcbsp->pdata->ops && mcbsp->pdata->ops->request)
+		mcbsp->pdata->ops->request(mcbsp->id - 1);
+
+	/*
+	 * Make sure that transmitter, receiver and sample-rate generator are
+	 * not running before activating IRQs.
+	 */
+	MCBSP_WRITE(mcbsp, SPCR1, 0);
+	MCBSP_WRITE(mcbsp, SPCR2, 0);
+
+	if (mcbsp->irq) {
+		err = request_irq(mcbsp->irq, omap_mcbsp_irq_handler, 0,
+				  "McBSP", (void *)mcbsp);
+		if (err != 0) {
+			dev_err(mcbsp->dev, "Unable to request IRQ\n");
+			goto err_clk_disable;
+		}
+	} else {
+		err = request_irq(mcbsp->tx_irq, omap_mcbsp_tx_irq_handler, 0,
+				  "McBSP TX", (void *)mcbsp);
+		if (err != 0) {
+			dev_err(mcbsp->dev, "Unable to request TX IRQ\n");
+			goto err_clk_disable;
+		}
+
+		err = request_irq(mcbsp->rx_irq, omap_mcbsp_rx_irq_handler, 0,
+				  "McBSP RX", (void *)mcbsp);
+		if (err != 0) {
+			dev_err(mcbsp->dev, "Unable to request RX IRQ\n");
+			goto err_free_irq;
+		}
+	}
+
+	return 0;
+err_free_irq:
+	free_irq(mcbsp->tx_irq, (void *)mcbsp);
+err_clk_disable:
+	if(mcbsp->pdata->ops && mcbsp->pdata->ops->free)
+		mcbsp->pdata->ops->free(mcbsp->id - 1);
+
+	/* Disable wakeup behavior */
+	if (mcbsp->pdata->has_wakeup)
+		MCBSP_WRITE(mcbsp, WAKEUPEN, 0);
+
+	spin_lock(&mcbsp->lock);
+	mcbsp->free = true;
+	mcbsp->reg_cache = NULL;
+err_kfree:
+	spin_unlock(&mcbsp->lock);
+	kfree(reg_cache);
+
+	return err;
+}
+
+static void omap_mcbsp_free(struct omap_mcbsp *mcbsp)
+{
+	void *reg_cache;
+
+	if(mcbsp->pdata->ops && mcbsp->pdata->ops->free)
+		mcbsp->pdata->ops->free(mcbsp->id - 1);
+
+	/* Disable wakeup behavior */
+	if (mcbsp->pdata->has_wakeup)
+		MCBSP_WRITE(mcbsp, WAKEUPEN, 0);
+
+	/* Disable interrupt requests */
+	if (mcbsp->irq)
+		MCBSP_WRITE(mcbsp, IRQEN, 0);
+
+	if (mcbsp->irq) {
+		free_irq(mcbsp->irq, (void *)mcbsp);
+	} else {
+		free_irq(mcbsp->rx_irq, (void *)mcbsp);
+		free_irq(mcbsp->tx_irq, (void *)mcbsp);
+	}
+
+	reg_cache = mcbsp->reg_cache;
+
+	/*
+	 * Select CLKS source from internal source unconditionally before
+	 * marking the McBSP port as free.
+	 * If the external clock source via MCBSP_CLKS pin has been selected the
+	 * system will refuse to enter idle if the CLKS pin source is not reset
+	 * back to internal source.
+	 */
+	if (!mcbsp_omap1())
+		omap2_mcbsp_set_clks_src(mcbsp, MCBSP_CLKS_PRCM_SRC);
+
+	spin_lock(&mcbsp->lock);
+	if (mcbsp->free)
+		dev_err(mcbsp->dev, "McBSP%d was not reserved\n", mcbsp->id);
+	else
+		mcbsp->free = true;
+	mcbsp->reg_cache = NULL;
+	spin_unlock(&mcbsp->lock);
+
+	kfree(reg_cache);
+}
+
+/*
+ * Here we start the McBSP, by enabling transmitter, receiver or both.
+ * If no transmitter or receiver is active prior calling, then sample-rate
+ * generator and frame sync are started.
+ */
+static void omap_mcbsp_start(struct omap_mcbsp *mcbsp, int stream)
+{
+	int tx = (stream == SNDRV_PCM_STREAM_PLAYBACK);
+	int rx = !tx;
+	int enable_srg = 0;
+	u16 w;
+
+	if (mcbsp->st_data)
+		omap_mcbsp_st_start(mcbsp);
+
+	/* Only enable SRG, if McBSP is master */
+	w = MCBSP_READ_CACHE(mcbsp, PCR0);
+	if (w & (FSXM | FSRM | CLKXM | CLKRM))
+		enable_srg = !((MCBSP_READ_CACHE(mcbsp, SPCR2) |
+				MCBSP_READ_CACHE(mcbsp, SPCR1)) & 1);
+
+	if (enable_srg) {
+		/* Start the sample generator */
+		w = MCBSP_READ_CACHE(mcbsp, SPCR2);
+		MCBSP_WRITE(mcbsp, SPCR2, w | (1 << 6));
+	}
+
+	/* Enable transmitter and receiver */
+	tx &= 1;
+	w = MCBSP_READ_CACHE(mcbsp, SPCR2);
+	MCBSP_WRITE(mcbsp, SPCR2, w | tx);
+
+	rx &= 1;
+	w = MCBSP_READ_CACHE(mcbsp, SPCR1);
+	MCBSP_WRITE(mcbsp, SPCR1, w | rx);
+
+	/*
+	 * Worst case: CLKSRG*2 = 8000khz: (1/8000) * 2 * 2 usec
+	 * REVISIT: 100us may give enough time for two CLKSRG, however
+	 * due to some unknown PM related, clock gating etc. reason it
+	 * is now at 500us.
+	 */
+	udelay(500);
+
+	if (enable_srg) {
+		/* Start frame sync */
+		w = MCBSP_READ_CACHE(mcbsp, SPCR2);
+		MCBSP_WRITE(mcbsp, SPCR2, w | (1 << 7));
+	}
+
+	if (mcbsp->pdata->has_ccr) {
+		/* Release the transmitter and receiver */
+		w = MCBSP_READ_CACHE(mcbsp, XCCR);
+		w &= ~(tx ? XDISABLE : 0);
+		MCBSP_WRITE(mcbsp, XCCR, w);
+		w = MCBSP_READ_CACHE(mcbsp, RCCR);
+		w &= ~(rx ? RDISABLE : 0);
+		MCBSP_WRITE(mcbsp, RCCR, w);
+	}
+
+	/* Dump McBSP Regs */
+	omap_mcbsp_dump_reg(mcbsp);
+}
+
+static void omap_mcbsp_stop(struct omap_mcbsp *mcbsp, int stream)
+{
+	int tx = (stream == SNDRV_PCM_STREAM_PLAYBACK);
+	int rx = !tx;
+	int idle;
+	u16 w;
+
+	/* Reset transmitter */
+	tx &= 1;
+	if (mcbsp->pdata->has_ccr) {
+		w = MCBSP_READ_CACHE(mcbsp, XCCR);
+		w |= (tx ? XDISABLE : 0);
+		MCBSP_WRITE(mcbsp, XCCR, w);
+	}
+	w = MCBSP_READ_CACHE(mcbsp, SPCR2);
+	MCBSP_WRITE(mcbsp, SPCR2, w & ~tx);
+
+	/* Reset receiver */
+	rx &= 1;
+	if (mcbsp->pdata->has_ccr) {
+		w = MCBSP_READ_CACHE(mcbsp, RCCR);
+		w |= (rx ? RDISABLE : 0);
+		MCBSP_WRITE(mcbsp, RCCR, w);
+	}
+	w = MCBSP_READ_CACHE(mcbsp, SPCR1);
+	MCBSP_WRITE(mcbsp, SPCR1, w & ~rx);
+
+	idle = !((MCBSP_READ_CACHE(mcbsp, SPCR2) |
+			MCBSP_READ_CACHE(mcbsp, SPCR1)) & 1);
+
+	if (idle) {
+		/* Reset the sample rate generator */
+		w = MCBSP_READ_CACHE(mcbsp, SPCR2);
+		MCBSP_WRITE(mcbsp, SPCR2, w & ~(1 << 6));
+	}
+
+	if (mcbsp->st_data)
+		omap_mcbsp_st_stop(mcbsp);
+}
+
+#define max_thres(m)			(mcbsp->pdata->buffer_size)
+#define valid_threshold(m, val)		((val) <= max_thres(m))
+#define THRESHOLD_PROP_BUILDER(prop)					\
+static ssize_t prop##_show(struct device *dev,				\
+			struct device_attribute *attr, char *buf)	\
+{									\
+	struct omap_mcbsp *mcbsp = dev_get_drvdata(dev);		\
+									\
+	return sprintf(buf, "%u\n", mcbsp->prop);			\
+}									\
+									\
+static ssize_t prop##_store(struct device *dev,				\
+				struct device_attribute *attr,		\
+				const char *buf, size_t size)		\
+{									\
+	struct omap_mcbsp *mcbsp = dev_get_drvdata(dev);		\
+	unsigned long val;						\
+	int status;							\
+									\
+	status = kstrtoul(buf, 0, &val);				\
+	if (status)							\
+		return status;						\
+									\
+	if (!valid_threshold(mcbsp, val))				\
+		return -EDOM;						\
+									\
+	mcbsp->prop = val;						\
+	return size;							\
+}									\
+									\
+static DEVICE_ATTR(prop, 0644, prop##_show, prop##_store)
+
+THRESHOLD_PROP_BUILDER(max_tx_thres);
+THRESHOLD_PROP_BUILDER(max_rx_thres);
+
+static const char * const dma_op_modes[] = {
+	"element", "threshold",
+};
+
+static ssize_t dma_op_mode_show(struct device *dev,
+				struct device_attribute *attr, char *buf)
+{
+	struct omap_mcbsp *mcbsp = dev_get_drvdata(dev);
+	int dma_op_mode, i = 0;
+	ssize_t len = 0;
+	const char * const *s;
+
+	dma_op_mode = mcbsp->dma_op_mode;
+
+	for (s = &dma_op_modes[i]; i < ARRAY_SIZE(dma_op_modes); s++, i++) {
+		if (dma_op_mode == i)
+			len += sprintf(buf + len, "[%s] ", *s);
+		else
+			len += sprintf(buf + len, "%s ", *s);
+	}
+	len += sprintf(buf + len, "\n");
+
+	return len;
+}
+
+static ssize_t dma_op_mode_store(struct device *dev,
+				 struct device_attribute *attr, const char *buf,
+				 size_t size)
+{
+	struct omap_mcbsp *mcbsp = dev_get_drvdata(dev);
+	int i;
+
+	i = sysfs_match_string(dma_op_modes, buf);
+	if (i < 0)
+		return i;
+
+	spin_lock_irq(&mcbsp->lock);
+	if (!mcbsp->free) {
+		size = -EBUSY;
+		goto unlock;
+	}
+	mcbsp->dma_op_mode = i;
+
+unlock:
+	spin_unlock_irq(&mcbsp->lock);
+
+	return size;
+}
+
+static DEVICE_ATTR_RW(dma_op_mode);
+
+static const struct attribute *additional_attrs[] = {
+	&dev_attr_max_tx_thres.attr,
+	&dev_attr_max_rx_thres.attr,
+	&dev_attr_dma_op_mode.attr,
+	NULL,
+};
+
+static const struct attribute_group additional_attr_group = {
+	.attrs = (struct attribute **)additional_attrs,
+};
+
+/*
+ * McBSP1 and McBSP3 are directly mapped on 1610 and 1510.
+ * 730 has only 2 McBSP, and both of them are MPU peripherals.
+ */
+static int omap_mcbsp_init(struct platform_device *pdev)
+{
+	struct omap_mcbsp *mcbsp = platform_get_drvdata(pdev);
+	struct resource *res;
+	int ret = 0;
+
+	spin_lock_init(&mcbsp->lock);
+	mcbsp->free = true;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mpu");
+	if (!res)
+		res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+
+	mcbsp->io_base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(mcbsp->io_base))
+		return PTR_ERR(mcbsp->io_base);
+
+	mcbsp->phys_base = res->start;
+	mcbsp->reg_cache_size = resource_size(res);
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dma");
+	if (!res)
+		mcbsp->phys_dma_base = mcbsp->phys_base;
+	else
+		mcbsp->phys_dma_base = res->start;
+
+	/*
+	 * OMAP1, 2 uses two interrupt lines: TX, RX
+	 * OMAP2430, OMAP3 SoC have combined IRQ line as well.
+	 * OMAP4 and newer SoC only have the combined IRQ line.
+	 * Use the combined IRQ if available since it gives better debugging
+	 * possibilities.
+	 */
+	mcbsp->irq = platform_get_irq_byname(pdev, "common");
+	if (mcbsp->irq == -ENXIO) {
+		mcbsp->tx_irq = platform_get_irq_byname(pdev, "tx");
+
+		if (mcbsp->tx_irq == -ENXIO) {
+			mcbsp->irq = platform_get_irq(pdev, 0);
+			mcbsp->tx_irq = 0;
+		} else {
+			mcbsp->rx_irq = platform_get_irq_byname(pdev, "rx");
+			mcbsp->irq = 0;
+		}
+	}
+
+	if (!pdev->dev.of_node) {
+		res = platform_get_resource_byname(pdev, IORESOURCE_DMA, "tx");
+		if (!res) {
+			dev_err(&pdev->dev, "invalid tx DMA channel\n");
+			return -ENODEV;
+		}
+		mcbsp->dma_req[0] = res->start;
+		mcbsp->dma_data[0].filter_data = &mcbsp->dma_req[0];
+
+		res = platform_get_resource_byname(pdev, IORESOURCE_DMA, "rx");
+		if (!res) {
+			dev_err(&pdev->dev, "invalid rx DMA channel\n");
+			return -ENODEV;
+		}
+		mcbsp->dma_req[1] = res->start;
+		mcbsp->dma_data[1].filter_data = &mcbsp->dma_req[1];
+	} else {
+		mcbsp->dma_data[0].filter_data = "tx";
+		mcbsp->dma_data[1].filter_data = "rx";
+	}
+
+	mcbsp->dma_data[0].addr = omap_mcbsp_dma_reg_params(mcbsp,
+						SNDRV_PCM_STREAM_PLAYBACK);
+	mcbsp->dma_data[1].addr = omap_mcbsp_dma_reg_params(mcbsp,
+						SNDRV_PCM_STREAM_CAPTURE);
+
+	mcbsp->fclk = clk_get(&pdev->dev, "fck");
+	if (IS_ERR(mcbsp->fclk)) {
+		ret = PTR_ERR(mcbsp->fclk);
+		dev_err(mcbsp->dev, "unable to get fck: %d\n", ret);
+		return ret;
+	}
+
+	mcbsp->dma_op_mode = MCBSP_DMA_MODE_ELEMENT;
+	if (mcbsp->pdata->buffer_size) {
+		/*
+		 * Initially configure the maximum thresholds to a safe value.
+		 * The McBSP FIFO usage with these values should not go under
+		 * 16 locations.
+		 * If the whole FIFO without safety buffer is used, than there
+		 * is a possibility that the DMA will be not able to push the
+		 * new data on time, causing channel shifts in runtime.
+		 */
+		mcbsp->max_tx_thres = max_thres(mcbsp) - 0x10;
+		mcbsp->max_rx_thres = max_thres(mcbsp) - 0x10;
+
+		ret = sysfs_create_group(&mcbsp->dev->kobj,
+					 &additional_attr_group);
+		if (ret) {
+			dev_err(mcbsp->dev,
+				"Unable to create additional controls\n");
+			goto err_thres;
+		}
+	}
+
+	ret = omap_mcbsp_st_init(pdev);
+	if (ret)
+		goto err_st;
+
+	return 0;
+
+err_st:
+	if (mcbsp->pdata->buffer_size)
+		sysfs_remove_group(&mcbsp->dev->kobj, &additional_attr_group);
+err_thres:
+	clk_put(mcbsp->fclk);
+	return ret;
+}
+
+/*
+ * Stream DMA parameters. DMA request line and port address are set runtime
+ * since they are different between OMAP1 and later OMAPs
+ */
+static void omap_mcbsp_set_threshold(struct snd_pcm_substream *substream,
+		unsigned int packet_size)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
+	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);
+	int words;
+
+	/* No need to proceed further if McBSP does not have FIFO */
+	if (mcbsp->pdata->buffer_size == 0)
+		return;
+
+	/*
+	 * Configure McBSP threshold based on either:
+	 * packet_size, when the sDMA is in packet mode, or based on the
+	 * period size in THRESHOLD mode, otherwise use McBSP threshold = 1
+	 * for mono streams.
+	 */
+	if (packet_size)
+		words = packet_size;
+	else
+		words = 1;
+
+	/* Configure McBSP internal buffer usage */
+	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
+		omap_mcbsp_set_tx_threshold(mcbsp, words);
+	else
+		omap_mcbsp_set_rx_threshold(mcbsp, words);
+}
+
+static int omap_mcbsp_hwrule_min_buffersize(struct snd_pcm_hw_params *params,
+				    struct snd_pcm_hw_rule *rule)
+{
+	struct snd_interval *buffer_size = hw_param_interval(params,
+					SNDRV_PCM_HW_PARAM_BUFFER_SIZE);
+	struct snd_interval *channels = hw_param_interval(params,
+					SNDRV_PCM_HW_PARAM_CHANNELS);
+	struct omap_mcbsp *mcbsp = rule->private;
+	struct snd_interval frames;
+	int size;
+
+	snd_interval_any(&frames);
+	size = mcbsp->pdata->buffer_size;
+
+	frames.min = size / channels->min;
+	frames.integer = 1;
+	return snd_interval_refine(buffer_size, &frames);
+}
+
+static int omap_mcbsp_dai_startup(struct snd_pcm_substream *substream,
+				  struct snd_soc_dai *cpu_dai)
+{
+	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);
+	int err = 0;
+
+	if (!cpu_dai->active)
+		err = omap_mcbsp_request(mcbsp);
+
+	/*
+	 * OMAP3 McBSP FIFO is word structured.
+	 * McBSP2 has 1024 + 256 = 1280 word long buffer,
+	 * McBSP1,3,4,5 has 128 word long buffer
+	 * This means that the size of the FIFO depends on the sample format.
+	 * For example on McBSP3:
+	 * 16bit samples: size is 128 * 2 = 256 bytes
+	 * 32bit samples: size is 128 * 4 = 512 bytes
+	 * It is simpler to place constraint for buffer and period based on
+	 * channels.
+	 * McBSP3 as example again (16 or 32 bit samples):
+	 * 1 channel (mono): size is 128 frames (128 words)
+	 * 2 channels (stereo): size is 128 / 2 = 64 frames (2 * 64 words)
+	 * 4 channels: size is 128 / 4 = 32 frames (4 * 32 words)
+	 */
+	if (mcbsp->pdata->buffer_size) {
+		/*
+		* Rule for the buffer size. We should not allow
+		* smaller buffer than the FIFO size to avoid underruns.
+		* This applies only for the playback stream.
+		*/
+		if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
+			snd_pcm_hw_rule_add(substream->runtime, 0,
+					    SNDRV_PCM_HW_PARAM_BUFFER_SIZE,
+					    omap_mcbsp_hwrule_min_buffersize,
+					    mcbsp,
+					    SNDRV_PCM_HW_PARAM_CHANNELS, -1);
+
+		/* Make sure, that the period size is always even */
+		snd_pcm_hw_constraint_step(substream->runtime, 0,
+					   SNDRV_PCM_HW_PARAM_PERIOD_SIZE, 2);
+	}
+
+	return err;
+}
+
+static void omap_mcbsp_dai_shutdown(struct snd_pcm_substream *substream,
+				    struct snd_soc_dai *cpu_dai)
+{
+	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);
+	int tx = (substream->stream == SNDRV_PCM_STREAM_PLAYBACK);
+	int stream1 = tx ? SNDRV_PCM_STREAM_PLAYBACK : SNDRV_PCM_STREAM_CAPTURE;
+	int stream2 = tx ? SNDRV_PCM_STREAM_CAPTURE : SNDRV_PCM_STREAM_PLAYBACK;
+
+	if (mcbsp->latency[stream2])
+		pm_qos_update_request(&mcbsp->pm_qos_req,
+				      mcbsp->latency[stream2]);
+	else if (mcbsp->latency[stream1])
+		pm_qos_remove_request(&mcbsp->pm_qos_req);
+
+	mcbsp->latency[stream1] = 0;
+
+	if (!cpu_dai->active) {
+		omap_mcbsp_free(mcbsp);
+		mcbsp->configured = 0;
+	}
+}
+
+static int omap_mcbsp_dai_prepare(struct snd_pcm_substream *substream,
+				  struct snd_soc_dai *cpu_dai)
+{
+	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);
+	struct pm_qos_request *pm_qos_req = &mcbsp->pm_qos_req;
+	int tx = (substream->stream == SNDRV_PCM_STREAM_PLAYBACK);
+	int stream1 = tx ? SNDRV_PCM_STREAM_PLAYBACK : SNDRV_PCM_STREAM_CAPTURE;
+	int stream2 = tx ? SNDRV_PCM_STREAM_CAPTURE : SNDRV_PCM_STREAM_PLAYBACK;
+	int latency = mcbsp->latency[stream2];
+
+	/* Prevent omap hardware from hitting off between FIFO fills */
+	if (!latency || mcbsp->latency[stream1] < latency)
+		latency = mcbsp->latency[stream1];
+
+	if (pm_qos_request_active(pm_qos_req))
+		pm_qos_update_request(pm_qos_req, latency);
+	else if (latency)
+		pm_qos_add_request(pm_qos_req, PM_QOS_CPU_DMA_LATENCY, latency);
+
+	return 0;
+}
+
+static int omap_mcbsp_dai_trigger(struct snd_pcm_substream *substream, int cmd,
+				  struct snd_soc_dai *cpu_dai)
+{
+	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);
+
+	switch (cmd) {
+	case SNDRV_PCM_TRIGGER_START:
+	case SNDRV_PCM_TRIGGER_RESUME:
+	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
+		mcbsp->active++;
+		omap_mcbsp_start(mcbsp, substream->stream);
+		break;
+
+	case SNDRV_PCM_TRIGGER_STOP:
+	case SNDRV_PCM_TRIGGER_SUSPEND:
+	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
+		omap_mcbsp_stop(mcbsp, substream->stream);
+		mcbsp->active--;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static snd_pcm_sframes_t omap_mcbsp_dai_delay(
+			struct snd_pcm_substream *substream,
+			struct snd_soc_dai *dai)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
+	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);
+	u16 fifo_use;
+	snd_pcm_sframes_t delay;
+
+	/* No need to proceed further if McBSP does not have FIFO */
+	if (mcbsp->pdata->buffer_size == 0)
+		return 0;
+
+	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
+		fifo_use = omap_mcbsp_get_tx_delay(mcbsp);
+	else
+		fifo_use = omap_mcbsp_get_rx_delay(mcbsp);
+
+	/*
+	 * Divide the used locations with the channel count to get the
+	 * FIFO usage in samples (don't care about partial samples in the
+	 * buffer).
+	 */
+	delay = fifo_use / substream->runtime->channels;
+
+	return delay;
+}
+
+static int omap_mcbsp_dai_hw_params(struct snd_pcm_substream *substream,
+				    struct snd_pcm_hw_params *params,
+				    struct snd_soc_dai *cpu_dai)
+{
+	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);
+	struct omap_mcbsp_reg_cfg *regs = &mcbsp->cfg_regs;
+	struct snd_dmaengine_dai_dma_data *dma_data;
+	int wlen, channels, wpf;
+	int pkt_size = 0;
+	unsigned int format, div, framesize, master;
+	unsigned int buffer_size = mcbsp->pdata->buffer_size;
+
+	dma_data = snd_soc_dai_get_dma_data(cpu_dai, substream);
+	channels = params_channels(params);
+
+	switch (params_format(params)) {
+	case SNDRV_PCM_FORMAT_S16_LE:
+		wlen = 16;
+		break;
+	case SNDRV_PCM_FORMAT_S32_LE:
+		wlen = 32;
+		break;
+	default:
+		return -EINVAL;
+	}
+	if (buffer_size) {
+		int latency;
+
+		if (mcbsp->dma_op_mode == MCBSP_DMA_MODE_THRESHOLD) {
+			int period_words, max_thrsh;
+			int divider = 0;
+
+			period_words = params_period_bytes(params) / (wlen / 8);
+			if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
+				max_thrsh = mcbsp->max_tx_thres;
+			else
+				max_thrsh = mcbsp->max_rx_thres;
+			/*
+			 * Use sDMA packet mode if McBSP is in threshold mode:
+			 * If period words less than the FIFO size the packet
+			 * size is set to the number of period words, otherwise
+			 * Look for the biggest threshold value which divides
+			 * the period size evenly.
+			 */
+			divider = period_words / max_thrsh;
+			if (period_words % max_thrsh)
+				divider++;
+			while (period_words % divider &&
+				divider < period_words)
+				divider++;
+			if (divider == period_words)
+				return -EINVAL;
+
+			pkt_size = period_words / divider;
+		} else if (channels > 1) {
+			/* Use packet mode for non mono streams */
+			pkt_size = channels;
+		}
+
+		latency = (buffer_size - pkt_size) / channels;
+		latency = latency * USEC_PER_SEC /
+			  (params->rate_num / params->rate_den);
+		mcbsp->latency[substream->stream] = latency;
+
+		omap_mcbsp_set_threshold(substream, pkt_size);
+	}
+
+	dma_data->maxburst = pkt_size;
+
+	if (mcbsp->configured) {
+		/* McBSP already configured by another stream */
+		return 0;
+	}
+
+	regs->rcr2	&= ~(RPHASE | RFRLEN2(0x7f) | RWDLEN2(7));
+	regs->xcr2	&= ~(RPHASE | XFRLEN2(0x7f) | XWDLEN2(7));
+	regs->rcr1	&= ~(RFRLEN1(0x7f) | RWDLEN1(7));
+	regs->xcr1	&= ~(XFRLEN1(0x7f) | XWDLEN1(7));
+	format = mcbsp->fmt & SND_SOC_DAIFMT_FORMAT_MASK;
+	wpf = channels;
+	if (channels == 2 && (format == SND_SOC_DAIFMT_I2S ||
+			      format == SND_SOC_DAIFMT_LEFT_J)) {
+		/* Use dual-phase frames */
+		regs->rcr2	|= RPHASE;
+		regs->xcr2	|= XPHASE;
+		/* Set 1 word per (McBSP) frame for phase1 and phase2 */
+		wpf--;
+		regs->rcr2	|= RFRLEN2(wpf - 1);
+		regs->xcr2	|= XFRLEN2(wpf - 1);
+	}
+
+	regs->rcr1	|= RFRLEN1(wpf - 1);
+	regs->xcr1	|= XFRLEN1(wpf - 1);
+
+	switch (params_format(params)) {
+	case SNDRV_PCM_FORMAT_S16_LE:
+		/* Set word lengths */
+		regs->rcr2	|= RWDLEN2(OMAP_MCBSP_WORD_16);
+		regs->rcr1	|= RWDLEN1(OMAP_MCBSP_WORD_16);
+		regs->xcr2	|= XWDLEN2(OMAP_MCBSP_WORD_16);
+		regs->xcr1	|= XWDLEN1(OMAP_MCBSP_WORD_16);
+		break;
+	case SNDRV_PCM_FORMAT_S32_LE:
+		/* Set word lengths */
+		regs->rcr2	|= RWDLEN2(OMAP_MCBSP_WORD_32);
+		regs->rcr1	|= RWDLEN1(OMAP_MCBSP_WORD_32);
+		regs->xcr2	|= XWDLEN2(OMAP_MCBSP_WORD_32);
+		regs->xcr1	|= XWDLEN1(OMAP_MCBSP_WORD_32);
+		break;
+	default:
+		/* Unsupported PCM format */
+		return -EINVAL;
+	}
+
+	/* In McBSP master modes, FRAME (i.e. sample rate) is generated
+	 * by _counting_ BCLKs. Calculate frame size in BCLKs */
+	master = mcbsp->fmt & SND_SOC_DAIFMT_MASTER_MASK;
+	if (master ==	SND_SOC_DAIFMT_CBS_CFS) {
+		div = mcbsp->clk_div ? mcbsp->clk_div : 1;
+		framesize = (mcbsp->in_freq / div) / params_rate(params);
+
+		if (framesize < wlen * channels) {
+			printk(KERN_ERR "%s: not enough bandwidth for desired rate and "
+					"channels\n", __func__);
+			return -EINVAL;
+		}
+	} else
+		framesize = wlen * channels;
+
+	/* Set FS period and length in terms of bit clock periods */
+	regs->srgr2	&= ~FPER(0xfff);
+	regs->srgr1	&= ~FWID(0xff);
+	switch (format) {
+	case SND_SOC_DAIFMT_I2S:
+	case SND_SOC_DAIFMT_LEFT_J:
+		regs->srgr2	|= FPER(framesize - 1);
+		regs->srgr1	|= FWID((framesize >> 1) - 1);
+		break;
+	case SND_SOC_DAIFMT_DSP_A:
+	case SND_SOC_DAIFMT_DSP_B:
+		regs->srgr2	|= FPER(framesize - 1);
+		regs->srgr1	|= FWID(0);
+		break;
+	}
+
+	omap_mcbsp_config(mcbsp, &mcbsp->cfg_regs);
+	mcbsp->wlen = wlen;
+	mcbsp->configured = 1;
+
+	return 0;
+}
+
+/*
+ * This must be called before _set_clkdiv and _set_sysclk since McBSP register
+ * cache is initialized here
+ */
+static int omap_mcbsp_dai_set_dai_fmt(struct snd_soc_dai *cpu_dai,
+				      unsigned int fmt)
+{
+	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);
+	struct omap_mcbsp_reg_cfg *regs = &mcbsp->cfg_regs;
+	bool inv_fs = false;
+
+	if (mcbsp->configured)
+		return 0;
+
+	mcbsp->fmt = fmt;
+	memset(regs, 0, sizeof(*regs));
+	/* Generic McBSP register settings */
+	regs->spcr2	|= XINTM(3) | FREE;
+	regs->spcr1	|= RINTM(3);
+	/* RFIG and XFIG are not defined in 2430 and on OMAP3+ */
+	if (!mcbsp->pdata->has_ccr) {
+		regs->rcr2	|= RFIG;
+		regs->xcr2	|= XFIG;
+	}
+
+	/* Configure XCCR/RCCR only for revisions which have ccr registers */
+	if (mcbsp->pdata->has_ccr) {
+		regs->xccr = DXENDLY(1) | XDMAEN | XDISABLE;
+		regs->rccr = RFULL_CYCLE | RDMAEN | RDISABLE;
+	}
+
+	switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) {
+	case SND_SOC_DAIFMT_I2S:
+		/* 1-bit data delay */
+		regs->rcr2	|= RDATDLY(1);
+		regs->xcr2	|= XDATDLY(1);
+		break;
+	case SND_SOC_DAIFMT_LEFT_J:
+		/* 0-bit data delay */
+		regs->rcr2	|= RDATDLY(0);
+		regs->xcr2	|= XDATDLY(0);
+		regs->spcr1	|= RJUST(2);
+		/* Invert FS polarity configuration */
+		inv_fs = true;
+		break;
+	case SND_SOC_DAIFMT_DSP_A:
+		/* 1-bit data delay */
+		regs->rcr2      |= RDATDLY(1);
+		regs->xcr2      |= XDATDLY(1);
+		/* Invert FS polarity configuration */
+		inv_fs = true;
+		break;
+	case SND_SOC_DAIFMT_DSP_B:
+		/* 0-bit data delay */
+		regs->rcr2      |= RDATDLY(0);
+		regs->xcr2      |= XDATDLY(0);
+		/* Invert FS polarity configuration */
+		inv_fs = true;
+		break;
+	default:
+		/* Unsupported data format */
+		return -EINVAL;
+	}
+
+	switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) {
+	case SND_SOC_DAIFMT_CBS_CFS:
+		/* McBSP master. Set FS and bit clocks as outputs */
+		regs->pcr0	|= FSXM | FSRM |
+				   CLKXM | CLKRM;
+		/* Sample rate generator drives the FS */
+		regs->srgr2	|= FSGM;
+		break;
+	case SND_SOC_DAIFMT_CBM_CFS:
+		/* McBSP slave. FS clock as output */
+		regs->srgr2	|= FSGM;
+		regs->pcr0	|= FSXM | FSRM;
+		break;
+	case SND_SOC_DAIFMT_CBM_CFM:
+		/* McBSP slave */
+		break;
+	default:
+		/* Unsupported master/slave configuration */
+		return -EINVAL;
+	}
+
+	/* Set bit clock (CLKX/CLKR) and FS polarities */
+	switch (fmt & SND_SOC_DAIFMT_INV_MASK) {
+	case SND_SOC_DAIFMT_NB_NF:
+		/*
+		 * Normal BCLK + FS.
+		 * FS active low. TX data driven on falling edge of bit clock
+		 * and RX data sampled on rising edge of bit clock.
+		 */
+		regs->pcr0	|= FSXP | FSRP |
+				   CLKXP | CLKRP;
+		break;
+	case SND_SOC_DAIFMT_NB_IF:
+		regs->pcr0	|= CLKXP | CLKRP;
+		break;
+	case SND_SOC_DAIFMT_IB_NF:
+		regs->pcr0	|= FSXP | FSRP;
+		break;
+	case SND_SOC_DAIFMT_IB_IF:
+		break;
+	default:
+		return -EINVAL;
+	}
+	if (inv_fs == true)
+		regs->pcr0 ^= FSXP | FSRP;
+
+	return 0;
+}
+
+static int omap_mcbsp_dai_set_clkdiv(struct snd_soc_dai *cpu_dai,
+				     int div_id, int div)
+{
+	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);
+	struct omap_mcbsp_reg_cfg *regs = &mcbsp->cfg_regs;
+
+	if (div_id != OMAP_MCBSP_CLKGDV)
+		return -ENODEV;
+
+	mcbsp->clk_div = div;
+	regs->srgr1	&= ~CLKGDV(0xff);
+	regs->srgr1	|= CLKGDV(div - 1);
+
+	return 0;
+}
+
+static int omap_mcbsp_dai_set_dai_sysclk(struct snd_soc_dai *cpu_dai,
+					 int clk_id, unsigned int freq,
+					 int dir)
+{
+	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);
+	struct omap_mcbsp_reg_cfg *regs = &mcbsp->cfg_regs;
+	int err = 0;
+
+	if (mcbsp->active) {
+		if (freq == mcbsp->in_freq)
+			return 0;
+		else
+			return -EBUSY;
+	}
+
+	mcbsp->in_freq = freq;
+	regs->srgr2 &= ~CLKSM;
+	regs->pcr0 &= ~SCLKME;
+
+	switch (clk_id) {
+	case OMAP_MCBSP_SYSCLK_CLK:
+		regs->srgr2	|= CLKSM;
+		break;
+	case OMAP_MCBSP_SYSCLK_CLKS_FCLK:
+		if (mcbsp_omap1()) {
+			err = -EINVAL;
+			break;
+		}
+		err = omap2_mcbsp_set_clks_src(mcbsp,
+					       MCBSP_CLKS_PRCM_SRC);
+		break;
+	case OMAP_MCBSP_SYSCLK_CLKS_EXT:
+		if (mcbsp_omap1()) {
+			err = 0;
+			break;
+		}
+		err = omap2_mcbsp_set_clks_src(mcbsp,
+					       MCBSP_CLKS_PAD_SRC);
+		break;
+
+	case OMAP_MCBSP_SYSCLK_CLKX_EXT:
+		regs->srgr2	|= CLKSM;
+		regs->pcr0	|= SCLKME;
+		/*
+		 * If McBSP is master but yet the CLKX/CLKR pin drives the SRG,
+		 * disable output on those pins. This enables to inject the
+		 * reference clock through CLKX/CLKR. For this to work
+		 * set_dai_sysclk() _needs_ to be called after set_dai_fmt().
+		 */
+		regs->pcr0	&= ~CLKXM;
+		break;
+	case OMAP_MCBSP_SYSCLK_CLKR_EXT:
+		regs->pcr0	|= SCLKME;
+		/* Disable ouput on CLKR pin in master mode */
+		regs->pcr0	&= ~CLKRM;
+		break;
+	default:
+		err = -ENODEV;
+	}
+
+	return err;
+}
+
+static const struct snd_soc_dai_ops mcbsp_dai_ops = {
+	.startup	= omap_mcbsp_dai_startup,
+	.shutdown	= omap_mcbsp_dai_shutdown,
+	.prepare	= omap_mcbsp_dai_prepare,
+	.trigger	= omap_mcbsp_dai_trigger,
+	.delay		= omap_mcbsp_dai_delay,
+	.hw_params	= omap_mcbsp_dai_hw_params,
+	.set_fmt	= omap_mcbsp_dai_set_dai_fmt,
+	.set_clkdiv	= omap_mcbsp_dai_set_clkdiv,
+	.set_sysclk	= omap_mcbsp_dai_set_dai_sysclk,
+};
+
+static int omap_mcbsp_probe(struct snd_soc_dai *dai)
+{
+	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(dai);
+
+	pm_runtime_enable(mcbsp->dev);
+
+	snd_soc_dai_init_dma_data(dai,
+				  &mcbsp->dma_data[SNDRV_PCM_STREAM_PLAYBACK],
+				  &mcbsp->dma_data[SNDRV_PCM_STREAM_CAPTURE]);
+
+	return 0;
+}
+
+static int omap_mcbsp_remove(struct snd_soc_dai *dai)
+{
+	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(dai);
+
+	pm_runtime_disable(mcbsp->dev);
+
+	return 0;
+}
+
+static struct snd_soc_dai_driver omap_mcbsp_dai = {
+	.probe = omap_mcbsp_probe,
+	.remove = omap_mcbsp_remove,
+	.playback = {
+		.channels_min = 1,
+		.channels_max = 16,
+		.rates = OMAP_MCBSP_RATES,
+		.formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S32_LE,
+	},
+	.capture = {
+		.channels_min = 1,
+		.channels_max = 16,
+		.rates = OMAP_MCBSP_RATES,
+		.formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S32_LE,
+	},
+	.ops = &mcbsp_dai_ops,
+};
+
+static const struct snd_soc_component_driver omap_mcbsp_component = {
+	.name		= "omap-mcbsp",
+};
+
+static struct omap_mcbsp_platform_data omap2420_pdata = {
+	.reg_step = 4,
+	.reg_size = 2,
+};
+
+static struct omap_mcbsp_platform_data omap2430_pdata = {
+	.reg_step = 4,
+	.reg_size = 4,
+	.has_ccr = true,
+};
+
+static struct omap_mcbsp_platform_data omap3_pdata = {
+	.reg_step = 4,
+	.reg_size = 4,
+	.has_ccr = true,
+	.has_wakeup = true,
+};
+
+static struct omap_mcbsp_platform_data omap4_pdata = {
+	.reg_step = 4,
+	.reg_size = 4,
+	.has_ccr = true,
+	.has_wakeup = true,
+};
+
+static const struct of_device_id omap_mcbsp_of_match[] = {
+	{
+		.compatible = "ti,omap2420-mcbsp",
+		.data = &omap2420_pdata,
+	},
+	{
+		.compatible = "ti,omap2430-mcbsp",
+		.data = &omap2430_pdata,
+	},
+	{
+		.compatible = "ti,omap3-mcbsp",
+		.data = &omap3_pdata,
+	},
+	{
+		.compatible = "ti,omap4-mcbsp",
+		.data = &omap4_pdata,
+	},
+	{ },
+};
+MODULE_DEVICE_TABLE(of, omap_mcbsp_of_match);
+
+static int asoc_mcbsp_probe(struct platform_device *pdev)
+{
+	struct omap_mcbsp_platform_data *pdata = dev_get_platdata(&pdev->dev);
+	struct omap_mcbsp *mcbsp;
+	const struct of_device_id *match;
+	int ret;
+
+	match = of_match_device(omap_mcbsp_of_match, &pdev->dev);
+	if (match) {
+		struct device_node *node = pdev->dev.of_node;
+		struct omap_mcbsp_platform_data *pdata_quirk = pdata;
+		int buffer_size;
+
+		pdata = devm_kzalloc(&pdev->dev,
+				     sizeof(struct omap_mcbsp_platform_data),
+				     GFP_KERNEL);
+		if (!pdata)
+			return -ENOMEM;
+
+		memcpy(pdata, match->data, sizeof(*pdata));
+		if (!of_property_read_u32(node, "ti,buffer-size", &buffer_size))
+			pdata->buffer_size = buffer_size;
+		if (pdata_quirk)
+			pdata->force_ick_on = pdata_quirk->force_ick_on;
+	} else if (!pdata) {
+		dev_err(&pdev->dev, "missing platform data.\n");
+		return -EINVAL;
+	}
+	mcbsp = devm_kzalloc(&pdev->dev, sizeof(struct omap_mcbsp), GFP_KERNEL);
+	if (!mcbsp)
+		return -ENOMEM;
+
+	mcbsp->id = pdev->id;
+	mcbsp->pdata = pdata;
+	mcbsp->dev = &pdev->dev;
+	platform_set_drvdata(pdev, mcbsp);
+
+	ret = omap_mcbsp_init(pdev);
+	if (ret)
+		return ret;
+
+	if (mcbsp->pdata->reg_size == 2) {
+		omap_mcbsp_dai.playback.formats = SNDRV_PCM_FMTBIT_S16_LE;
+		omap_mcbsp_dai.capture.formats = SNDRV_PCM_FMTBIT_S16_LE;
+	}
+
+	ret = devm_snd_soc_register_component(&pdev->dev,
+					      &omap_mcbsp_component,
+					      &omap_mcbsp_dai, 1);
+	if (ret)
+		return ret;
+
+	return sdma_pcm_platform_register(&pdev->dev, NULL, NULL);
+}
+
+static int asoc_mcbsp_remove(struct platform_device *pdev)
+{
+	struct omap_mcbsp *mcbsp = platform_get_drvdata(pdev);
+
+	if (mcbsp->pdata->ops && mcbsp->pdata->ops->free)
+		mcbsp->pdata->ops->free(mcbsp->id);
+
+	if (pm_qos_request_active(&mcbsp->pm_qos_req))
+		pm_qos_remove_request(&mcbsp->pm_qos_req);
+
+	if (mcbsp->pdata->buffer_size)
+		sysfs_remove_group(&mcbsp->dev->kobj, &additional_attr_group);
+
+	omap_mcbsp_st_cleanup(pdev);
+
+	clk_put(mcbsp->fclk);
+
+	return 0;
+}
+
+static struct platform_driver asoc_mcbsp_driver = {
+	.driver = {
+			.name = "omap-mcbsp",
+			.of_match_table = omap_mcbsp_of_match,
+	},
+
+	.probe = asoc_mcbsp_probe,
+	.remove = asoc_mcbsp_remove,
+};
+
+module_platform_driver(asoc_mcbsp_driver);
+
+MODULE_AUTHOR("Jarkko Nikula <jarkko.nikula@bitmer.com>");
+MODULE_DESCRIPTION("OMAP I2S SoC Interface");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:omap-mcbsp");
diff -Naur --no-dereference a/sound/soc/ti/omap-mcbsp.h b/sound/soc/ti/omap-mcbsp.h
--- a/sound/soc/ti/omap-mcbsp.h	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/omap-mcbsp.h	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,46 @@
+/*
+ * omap-mcbsp.h
+ *
+ * Copyright (C) 2008 Nokia Corporation
+ *
+ * Contact: Jarkko Nikula <jarkko.nikula@bitmer.com>
+ *          Peter Ujfalusi <peter.ujfalusi@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA
+ *
+ */
+
+#ifndef __OMAP_MCBSP_H__
+#define __OMAP_MCBSP_H__
+
+#include <sound/dmaengine_pcm.h>
+
+/* Source clocks for McBSP sample rate generator */
+enum omap_mcbsp_clksrg_clk {
+	OMAP_MCBSP_SYSCLK_CLKS_FCLK,	/* Internal FCLK */
+	OMAP_MCBSP_SYSCLK_CLKS_EXT,	/* External CLKS pin */
+	OMAP_MCBSP_SYSCLK_CLK,		/* Internal ICLK */
+	OMAP_MCBSP_SYSCLK_CLKX_EXT,	/* External CLKX pin */
+	OMAP_MCBSP_SYSCLK_CLKR_EXT,	/* External CLKR pin */
+};
+
+/* McBSP dividers */
+enum omap_mcbsp_div {
+	OMAP_MCBSP_CLKGDV,		/* Sample rate generator divider */
+};
+
+int omap_mcbsp_st_add_controls(struct snd_soc_pcm_runtime *rtd, int port_id);
+
+#endif /* __OMAP_MCBSP_H__ */
diff -Naur --no-dereference a/sound/soc/ti/omap-mcbsp-priv.h b/sound/soc/ti/omap-mcbsp-priv.h
--- a/sound/soc/ti/omap-mcbsp-priv.h	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/omap-mcbsp-priv.h	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,324 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * OMAP Multi-Channel Buffered Serial Port
+ *
+ * Contact: Jarkko Nikula <jarkko.nikula@bitmer.com>
+ *          Peter Ujfalusi <peter.ujfalusi@ti.com>
+ */
+
+#ifndef __OMAP_MCBSP_PRIV_H__
+#define __OMAP_MCBSP_PRIV_H__
+
+#include <linux/platform_data/asoc-ti-mcbsp.h>
+
+#ifdef CONFIG_ARCH_OMAP1
+#define mcbsp_omap1()	1
+#else
+#define mcbsp_omap1()	0
+#endif
+
+/* McBSP register numbers. Register address offset = num * reg_step */
+enum {
+	/* Common registers */
+	OMAP_MCBSP_REG_SPCR2 = 4,
+	OMAP_MCBSP_REG_SPCR1,
+	OMAP_MCBSP_REG_RCR2,
+	OMAP_MCBSP_REG_RCR1,
+	OMAP_MCBSP_REG_XCR2,
+	OMAP_MCBSP_REG_XCR1,
+	OMAP_MCBSP_REG_SRGR2,
+	OMAP_MCBSP_REG_SRGR1,
+	OMAP_MCBSP_REG_MCR2,
+	OMAP_MCBSP_REG_MCR1,
+	OMAP_MCBSP_REG_RCERA,
+	OMAP_MCBSP_REG_RCERB,
+	OMAP_MCBSP_REG_XCERA,
+	OMAP_MCBSP_REG_XCERB,
+	OMAP_MCBSP_REG_PCR0,
+	OMAP_MCBSP_REG_RCERC,
+	OMAP_MCBSP_REG_RCERD,
+	OMAP_MCBSP_REG_XCERC,
+	OMAP_MCBSP_REG_XCERD,
+	OMAP_MCBSP_REG_RCERE,
+	OMAP_MCBSP_REG_RCERF,
+	OMAP_MCBSP_REG_XCERE,
+	OMAP_MCBSP_REG_XCERF,
+	OMAP_MCBSP_REG_RCERG,
+	OMAP_MCBSP_REG_RCERH,
+	OMAP_MCBSP_REG_XCERG,
+	OMAP_MCBSP_REG_XCERH,
+
+	/* OMAP1-OMAP2420 registers */
+	OMAP_MCBSP_REG_DRR2 = 0,
+	OMAP_MCBSP_REG_DRR1,
+	OMAP_MCBSP_REG_DXR2,
+	OMAP_MCBSP_REG_DXR1,
+
+	/* OMAP2430 and onwards */
+	OMAP_MCBSP_REG_DRR = 0,
+	OMAP_MCBSP_REG_DXR = 2,
+	OMAP_MCBSP_REG_SYSCON =	35,
+	OMAP_MCBSP_REG_THRSH2,
+	OMAP_MCBSP_REG_THRSH1,
+	OMAP_MCBSP_REG_IRQST = 40,
+	OMAP_MCBSP_REG_IRQEN,
+	OMAP_MCBSP_REG_WAKEUPEN,
+	OMAP_MCBSP_REG_XCCR,
+	OMAP_MCBSP_REG_RCCR,
+	OMAP_MCBSP_REG_XBUFFSTAT,
+	OMAP_MCBSP_REG_RBUFFSTAT,
+	OMAP_MCBSP_REG_SSELCR,
+};
+
+/************************** McBSP SPCR1 bit definitions ***********************/
+#define RRST			BIT(0)
+#define RRDY			BIT(1)
+#define RFULL			BIT(2)
+#define RSYNC_ERR		BIT(3)
+#define RINTM(value)		(((value) & 0x3) << 4)	/* bits 4:5 */
+#define ABIS			BIT(6)
+#define DXENA			BIT(7)
+#define CLKSTP(value)		(((value) & 0x3) << 11)	/* bits 11:12 */
+#define RJUST(value)		(((value) & 0x3) << 13)	/* bits 13:14 */
+#define ALB			BIT(15)
+#define DLB			BIT(15)
+
+/************************** McBSP SPCR2 bit definitions ***********************/
+#define XRST			BIT(0)
+#define XRDY			BIT(1)
+#define XEMPTY			BIT(2)
+#define XSYNC_ERR		BIT(3)
+#define XINTM(value)		(((value) & 0x3) << 4)	/* bits 4:5 */
+#define GRST			BIT(6)
+#define FRST			BIT(7)
+#define SOFT			BIT(8)
+#define FREE			BIT(9)
+
+/************************** McBSP PCR bit definitions *************************/
+#define CLKRP			BIT(0)
+#define CLKXP			BIT(1)
+#define FSRP			BIT(2)
+#define FSXP			BIT(3)
+#define DR_STAT			BIT(4)
+#define DX_STAT			BIT(5)
+#define CLKS_STAT		BIT(6)
+#define SCLKME			BIT(7)
+#define CLKRM			BIT(8)
+#define CLKXM			BIT(9)
+#define FSRM			BIT(10)
+#define FSXM			BIT(11)
+#define RIOEN			BIT(12)
+#define XIOEN			BIT(13)
+#define IDLE_EN			BIT(14)
+
+/************************** McBSP RCR1 bit definitions ************************/
+#define RWDLEN1(value)		(((value) & 0x7) << 5)	/* Bits 5:7 */
+#define RFRLEN1(value)		(((value) & 0x7f) << 8)	/* Bits 8:14 */
+
+/************************** McBSP XCR1 bit definitions ************************/
+#define XWDLEN1(value)		(((value) & 0x7) << 5)	/* Bits 5:7 */
+#define XFRLEN1(value)		(((value) & 0x7f) << 8)	/* Bits 8:14 */
+
+/*************************** McBSP RCR2 bit definitions ***********************/
+#define RDATDLY(value)		((value) & 0x3)		/* Bits 0:1 */
+#define RFIG			BIT(2)
+#define RCOMPAND(value)		(((value) & 0x3) << 3)	/* Bits 3:4 */
+#define RWDLEN2(value)		(((value) & 0x7) << 5)	/* Bits 5:7 */
+#define RFRLEN2(value)		(((value) & 0x7f) << 8)	/* Bits 8:14 */
+#define RPHASE			BIT(15)
+
+/*************************** McBSP XCR2 bit definitions ***********************/
+#define XDATDLY(value)		((value) & 0x3)		/* Bits 0:1 */
+#define XFIG			BIT(2)
+#define XCOMPAND(value)		(((value) & 0x3) << 3)	/* Bits 3:4 */
+#define XWDLEN2(value)		(((value) & 0x7) << 5)	/* Bits 5:7 */
+#define XFRLEN2(value)		(((value) & 0x7f) << 8)	/* Bits 8:14 */
+#define XPHASE			BIT(15)
+
+/************************* McBSP SRGR1 bit definitions ************************/
+#define CLKGDV(value)		((value) & 0x7f)		/* Bits 0:7 */
+#define FWID(value)		(((value) & 0xff) << 8)	/* Bits 8:15 */
+
+/************************* McBSP SRGR2 bit definitions ************************/
+#define FPER(value)		((value) & 0x0fff)	/* Bits 0:11 */
+#define FSGM			BIT(12)
+#define CLKSM			BIT(13)
+#define CLKSP			BIT(14)
+#define GSYNC			BIT(15)
+
+/************************* McBSP MCR1 bit definitions *************************/
+#define RMCM			BIT(0)
+#define RCBLK(value)		(((value) & 0x7) << 2)	/* Bits 2:4 */
+#define RPABLK(value)		(((value) & 0x3) << 5)	/* Bits 5:6 */
+#define RPBBLK(value)		(((value) & 0x3) << 7)	/* Bits 7:8 */
+
+/************************* McBSP MCR2 bit definitions *************************/
+#define XMCM(value)		((value) & 0x3)		/* Bits 0:1 */
+#define XCBLK(value)		(((value) & 0x7) << 2)	/* Bits 2:4 */
+#define XPABLK(value)		(((value) & 0x3) << 5)	/* Bits 5:6 */
+#define XPBBLK(value)		(((value) & 0x3) << 7)	/* Bits 7:8 */
+
+/*********************** McBSP XCCR bit definitions *************************/
+#define XDISABLE		BIT(0)
+#define XDMAEN			BIT(3)
+#define DILB			BIT(5)
+#define XFULL_CYCLE		BIT(11)
+#define DXENDLY(value)		(((value) & 0x3) << 12)	/* Bits 12:13 */
+#define PPCONNECT		BIT(14)
+#define EXTCLKGATE		BIT(15)
+
+/********************** McBSP RCCR bit definitions *************************/
+#define RDISABLE		BIT(0)
+#define RDMAEN			BIT(3)
+#define RFULL_CYCLE		BIT(11)
+
+/********************** McBSP SYSCONFIG bit definitions ********************/
+#define SOFTRST			BIT(1)
+#define ENAWAKEUP		BIT(2)
+#define SIDLEMODE(value)	(((value) & 0x3) << 3)
+#define CLOCKACTIVITY(value)	(((value) & 0x3) << 8)
+
+/********************** McBSP DMA operating modes **************************/
+#define MCBSP_DMA_MODE_ELEMENT		0
+#define MCBSP_DMA_MODE_THRESHOLD	1
+
+/********************** McBSP WAKEUPEN/IRQST/IRQEN bit definitions *********/
+#define RSYNCERREN		BIT(0)
+#define RFSREN			BIT(1)
+#define REOFEN			BIT(2)
+#define RRDYEN			BIT(3)
+#define RUNDFLEN		BIT(4)
+#define ROVFLEN			BIT(5)
+#define XSYNCERREN		BIT(7)
+#define XFSXEN			BIT(8)
+#define XEOFEN			BIT(9)
+#define XRDYEN			BIT(10)
+#define XUNDFLEN		BIT(11)
+#define XOVFLEN			BIT(12)
+#define XEMPTYEOFEN		BIT(14)
+
+/* Clock signal muxing options */
+#define CLKR_SRC_CLKR		0 /* CLKR signal is from the CLKR pin */
+#define CLKR_SRC_CLKX		1 /* CLKR signal is from the CLKX pin */
+#define FSR_SRC_FSR		2 /* FSR signal is from the FSR pin */
+#define FSR_SRC_FSX		3 /* FSR signal is from the FSX pin */
+
+/* McBSP functional clock sources */
+#define MCBSP_CLKS_PRCM_SRC	0
+#define MCBSP_CLKS_PAD_SRC	1
+
+/* we don't do multichannel for now */
+struct omap_mcbsp_reg_cfg {
+	u16 spcr2;
+	u16 spcr1;
+	u16 rcr2;
+	u16 rcr1;
+	u16 xcr2;
+	u16 xcr1;
+	u16 srgr2;
+	u16 srgr1;
+	u16 mcr2;
+	u16 mcr1;
+	u16 pcr0;
+	u16 rcerc;
+	u16 rcerd;
+	u16 xcerc;
+	u16 xcerd;
+	u16 rcere;
+	u16 rcerf;
+	u16 xcere;
+	u16 xcerf;
+	u16 rcerg;
+	u16 rcerh;
+	u16 xcerg;
+	u16 xcerh;
+	u16 xccr;
+	u16 rccr;
+};
+
+struct omap_mcbsp_st_data;
+
+struct omap_mcbsp {
+	struct device *dev;
+	struct clk *fclk;
+	spinlock_t lock;
+	unsigned long phys_base;
+	unsigned long phys_dma_base;
+	void __iomem *io_base;
+	u8 id;
+	/*
+	 * Flags indicating is the bus already activated and configured by
+	 * another substream
+	 */
+	int active;
+	int configured;
+	u8 free;
+
+	int irq;
+	int rx_irq;
+	int tx_irq;
+
+	/* Protect the field .free, while checking if the mcbsp is in use */
+	struct omap_mcbsp_platform_data *pdata;
+	struct omap_mcbsp_st_data *st_data;
+	struct omap_mcbsp_reg_cfg cfg_regs;
+	struct snd_dmaengine_dai_dma_data dma_data[2];
+	unsigned int dma_req[2];
+	int dma_op_mode;
+	u16 max_tx_thres;
+	u16 max_rx_thres;
+	void *reg_cache;
+	int reg_cache_size;
+
+	unsigned int fmt;
+	unsigned int in_freq;
+	unsigned int latency[2];
+	int clk_div;
+	int wlen;
+
+	struct pm_qos_request pm_qos_req;
+};
+
+static inline void omap_mcbsp_write(struct omap_mcbsp *mcbsp, u16 reg, u32 val)
+{
+	void __iomem *addr = mcbsp->io_base + reg * mcbsp->pdata->reg_step;
+
+	if (mcbsp->pdata->reg_size == 2) {
+		((u16 *)mcbsp->reg_cache)[reg] = (u16)val;
+		writew_relaxed((u16)val, addr);
+	} else {
+		((u32 *)mcbsp->reg_cache)[reg] = val;
+		writel_relaxed(val, addr);
+	}
+}
+
+static inline int omap_mcbsp_read(struct omap_mcbsp *mcbsp, u16 reg,
+				  bool from_cache)
+{
+	void __iomem *addr = mcbsp->io_base + reg * mcbsp->pdata->reg_step;
+
+	if (mcbsp->pdata->reg_size == 2) {
+		return !from_cache ? readw_relaxed(addr) :
+				     ((u16 *)mcbsp->reg_cache)[reg];
+	} else {
+		return !from_cache ? readl_relaxed(addr) :
+				     ((u32 *)mcbsp->reg_cache)[reg];
+	}
+}
+
+#define MCBSP_READ(mcbsp, reg) \
+		omap_mcbsp_read(mcbsp, OMAP_MCBSP_REG_##reg, 0)
+#define MCBSP_WRITE(mcbsp, reg, val) \
+		omap_mcbsp_write(mcbsp, OMAP_MCBSP_REG_##reg, val)
+#define MCBSP_READ_CACHE(mcbsp, reg) \
+		omap_mcbsp_read(mcbsp, OMAP_MCBSP_REG_##reg, 1)
+
+
+/* Sidetone specific API */
+int omap_mcbsp_st_init(struct platform_device *pdev);
+void omap_mcbsp_st_cleanup(struct platform_device *pdev);
+
+int omap_mcbsp_st_start(struct omap_mcbsp *mcbsp);
+int omap_mcbsp_st_stop(struct omap_mcbsp *mcbsp);
+
+#endif /* __OMAP_MCBSP_PRIV_H__ */
diff -Naur --no-dereference a/sound/soc/ti/omap-mcbsp-st.c b/sound/soc/ti/omap-mcbsp-st.c
--- a/sound/soc/ti/omap-mcbsp-st.c	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/omap-mcbsp-st.c	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,516 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * McBSP Sidetone support
+ *
+ * Copyright (C) 2004 Nokia Corporation
+ * Author: Samuel Ortiz <samuel.ortiz@nokia.com>
+ *
+ * Contact: Jarkko Nikula <jarkko.nikula@bitmer.com>
+ *          Peter Ujfalusi <peter.ujfalusi@ti.com>
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/device.h>
+#include <linux/platform_device.h>
+#include <linux/interrupt.h>
+#include <linux/err.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/slab.h>
+#include <linux/pm_runtime.h>
+
+#include "omap-mcbsp.h"
+#include "omap-mcbsp-priv.h"
+
+/* OMAP3 sidetone control registers */
+#define OMAP_ST_REG_REV		0x00
+#define OMAP_ST_REG_SYSCONFIG	0x10
+#define OMAP_ST_REG_IRQSTATUS	0x18
+#define OMAP_ST_REG_IRQENABLE	0x1C
+#define OMAP_ST_REG_SGAINCR	0x24
+#define OMAP_ST_REG_SFIRCR	0x28
+#define OMAP_ST_REG_SSELCR	0x2C
+
+/********************** McBSP SSELCR bit definitions ***********************/
+#define SIDETONEEN		BIT(10)
+
+/********************** McBSP Sidetone SYSCONFIG bit definitions ***********/
+#define ST_AUTOIDLE		BIT(0)
+
+/********************** McBSP Sidetone SGAINCR bit definitions *************/
+#define ST_CH0GAIN(value)	((value) & 0xffff)	/* Bits 0:15 */
+#define ST_CH1GAIN(value)	(((value) & 0xffff) << 16) /* Bits 16:31 */
+
+/********************** McBSP Sidetone SFIRCR bit definitions **************/
+#define ST_FIRCOEFF(value)	((value) & 0xffff)	/* Bits 0:15 */
+
+/********************** McBSP Sidetone SSELCR bit definitions **************/
+#define ST_SIDETONEEN		BIT(0)
+#define ST_COEFFWREN		BIT(1)
+#define ST_COEFFWRDONE		BIT(2)
+
+struct omap_mcbsp_st_data {
+	void __iomem *io_base_st;
+	struct clk *mcbsp_iclk;
+	bool running;
+	bool enabled;
+	s16 taps[128];	/* Sidetone filter coefficients */
+	int nr_taps;	/* Number of filter coefficients in use */
+	s16 ch0gain;
+	s16 ch1gain;
+};
+
+static void omap_mcbsp_st_write(struct omap_mcbsp *mcbsp, u16 reg, u32 val)
+{
+	writel_relaxed(val, mcbsp->st_data->io_base_st + reg);
+}
+
+static int omap_mcbsp_st_read(struct omap_mcbsp *mcbsp, u16 reg)
+{
+	return readl_relaxed(mcbsp->st_data->io_base_st + reg);
+}
+
+#define MCBSP_ST_READ(mcbsp, reg) omap_mcbsp_st_read(mcbsp, OMAP_ST_REG_##reg)
+#define MCBSP_ST_WRITE(mcbsp, reg, val) \
+			omap_mcbsp_st_write(mcbsp, OMAP_ST_REG_##reg, val)
+
+static void omap_mcbsp_st_on(struct omap_mcbsp *mcbsp)
+{
+	unsigned int w;
+
+	if (mcbsp->pdata->force_ick_on)
+		mcbsp->pdata->force_ick_on(mcbsp->st_data->mcbsp_iclk, true);
+
+	/* Disable Sidetone clock auto-gating for normal operation */
+	w = MCBSP_ST_READ(mcbsp, SYSCONFIG);
+	MCBSP_ST_WRITE(mcbsp, SYSCONFIG, w & ~(ST_AUTOIDLE));
+
+	/* Enable McBSP Sidetone */
+	w = MCBSP_READ(mcbsp, SSELCR);
+	MCBSP_WRITE(mcbsp, SSELCR, w | SIDETONEEN);
+
+	/* Enable Sidetone from Sidetone Core */
+	w = MCBSP_ST_READ(mcbsp, SSELCR);
+	MCBSP_ST_WRITE(mcbsp, SSELCR, w | ST_SIDETONEEN);
+}
+
+static void omap_mcbsp_st_off(struct omap_mcbsp *mcbsp)
+{
+	unsigned int w;
+
+	w = MCBSP_ST_READ(mcbsp, SSELCR);
+	MCBSP_ST_WRITE(mcbsp, SSELCR, w & ~(ST_SIDETONEEN));
+
+	w = MCBSP_READ(mcbsp, SSELCR);
+	MCBSP_WRITE(mcbsp, SSELCR, w & ~(SIDETONEEN));
+
+	/* Enable Sidetone clock auto-gating to reduce power consumption */
+	w = MCBSP_ST_READ(mcbsp, SYSCONFIG);
+	MCBSP_ST_WRITE(mcbsp, SYSCONFIG, w | ST_AUTOIDLE);
+
+	if (mcbsp->pdata->force_ick_on)
+		mcbsp->pdata->force_ick_on(mcbsp->st_data->mcbsp_iclk, false);
+}
+
+static void omap_mcbsp_st_fir_write(struct omap_mcbsp *mcbsp, s16 *fir)
+{
+	u16 val, i;
+
+	val = MCBSP_ST_READ(mcbsp, SSELCR);
+
+	if (val & ST_COEFFWREN)
+		MCBSP_ST_WRITE(mcbsp, SSELCR, val & ~(ST_COEFFWREN));
+
+	MCBSP_ST_WRITE(mcbsp, SSELCR, val | ST_COEFFWREN);
+
+	for (i = 0; i < 128; i++)
+		MCBSP_ST_WRITE(mcbsp, SFIRCR, fir[i]);
+
+	i = 0;
+
+	val = MCBSP_ST_READ(mcbsp, SSELCR);
+	while (!(val & ST_COEFFWRDONE) && (++i < 1000))
+		val = MCBSP_ST_READ(mcbsp, SSELCR);
+
+	MCBSP_ST_WRITE(mcbsp, SSELCR, val & ~(ST_COEFFWREN));
+
+	if (i == 1000)
+		dev_err(mcbsp->dev, "McBSP FIR load error!\n");
+}
+
+static void omap_mcbsp_st_chgain(struct omap_mcbsp *mcbsp)
+{
+	u16 w;
+	struct omap_mcbsp_st_data *st_data = mcbsp->st_data;
+
+	w = MCBSP_ST_READ(mcbsp, SSELCR);
+
+	MCBSP_ST_WRITE(mcbsp, SGAINCR, ST_CH0GAIN(st_data->ch0gain) |
+		       ST_CH1GAIN(st_data->ch1gain));
+}
+
+static int omap_mcbsp_st_set_chgain(struct omap_mcbsp *mcbsp, int channel,
+				    s16 chgain)
+{
+	struct omap_mcbsp_st_data *st_data = mcbsp->st_data;
+	int ret = 0;
+
+	if (!st_data)
+		return -ENOENT;
+
+	spin_lock_irq(&mcbsp->lock);
+	if (channel == 0)
+		st_data->ch0gain = chgain;
+	else if (channel == 1)
+		st_data->ch1gain = chgain;
+	else
+		ret = -EINVAL;
+
+	if (st_data->enabled)
+		omap_mcbsp_st_chgain(mcbsp);
+	spin_unlock_irq(&mcbsp->lock);
+
+	return ret;
+}
+
+static int omap_mcbsp_st_get_chgain(struct omap_mcbsp *mcbsp, int channel,
+				    s16 *chgain)
+{
+	struct omap_mcbsp_st_data *st_data = mcbsp->st_data;
+	int ret = 0;
+
+	if (!st_data)
+		return -ENOENT;
+
+	spin_lock_irq(&mcbsp->lock);
+	if (channel == 0)
+		*chgain = st_data->ch0gain;
+	else if (channel == 1)
+		*chgain = st_data->ch1gain;
+	else
+		ret = -EINVAL;
+	spin_unlock_irq(&mcbsp->lock);
+
+	return ret;
+}
+
+static int omap_mcbsp_st_enable(struct omap_mcbsp *mcbsp)
+{
+	struct omap_mcbsp_st_data *st_data = mcbsp->st_data;
+
+	if (!st_data)
+		return -ENODEV;
+
+	spin_lock_irq(&mcbsp->lock);
+	st_data->enabled = 1;
+	omap_mcbsp_st_start(mcbsp);
+	spin_unlock_irq(&mcbsp->lock);
+
+	return 0;
+}
+
+static int omap_mcbsp_st_disable(struct omap_mcbsp *mcbsp)
+{
+	struct omap_mcbsp_st_data *st_data = mcbsp->st_data;
+	int ret = 0;
+
+	if (!st_data)
+		return -ENODEV;
+
+	spin_lock_irq(&mcbsp->lock);
+	omap_mcbsp_st_stop(mcbsp);
+	st_data->enabled = 0;
+	spin_unlock_irq(&mcbsp->lock);
+
+	return ret;
+}
+
+static int omap_mcbsp_st_is_enabled(struct omap_mcbsp *mcbsp)
+{
+	struct omap_mcbsp_st_data *st_data = mcbsp->st_data;
+
+	if (!st_data)
+		return -ENODEV;
+
+	return st_data->enabled;
+}
+
+static ssize_t st_taps_show(struct device *dev,
+			    struct device_attribute *attr, char *buf)
+{
+	struct omap_mcbsp *mcbsp = dev_get_drvdata(dev);
+	struct omap_mcbsp_st_data *st_data = mcbsp->st_data;
+	ssize_t status = 0;
+	int i;
+
+	spin_lock_irq(&mcbsp->lock);
+	for (i = 0; i < st_data->nr_taps; i++)
+		status += sprintf(&buf[status], (i ? ", %d" : "%d"),
+				  st_data->taps[i]);
+	if (i)
+		status += sprintf(&buf[status], "\n");
+	spin_unlock_irq(&mcbsp->lock);
+
+	return status;
+}
+
+static ssize_t st_taps_store(struct device *dev,
+			     struct device_attribute *attr,
+			     const char *buf, size_t size)
+{
+	struct omap_mcbsp *mcbsp = dev_get_drvdata(dev);
+	struct omap_mcbsp_st_data *st_data = mcbsp->st_data;
+	int val, tmp, status, i = 0;
+
+	spin_lock_irq(&mcbsp->lock);
+	memset(st_data->taps, 0, sizeof(st_data->taps));
+	st_data->nr_taps = 0;
+
+	do {
+		status = sscanf(buf, "%d%n", &val, &tmp);
+		if (status < 0 || status == 0) {
+			size = -EINVAL;
+			goto out;
+		}
+		if (val < -32768 || val > 32767) {
+			size = -EINVAL;
+			goto out;
+		}
+		st_data->taps[i++] = val;
+		buf += tmp;
+		if (*buf != ',')
+			break;
+		buf++;
+	} while (1);
+
+	st_data->nr_taps = i;
+
+out:
+	spin_unlock_irq(&mcbsp->lock);
+
+	return size;
+}
+
+static DEVICE_ATTR_RW(st_taps);
+
+static const struct attribute *sidetone_attrs[] = {
+	&dev_attr_st_taps.attr,
+	NULL,
+};
+
+static const struct attribute_group sidetone_attr_group = {
+	.attrs = (struct attribute **)sidetone_attrs,
+};
+
+int omap_mcbsp_st_start(struct omap_mcbsp *mcbsp)
+{
+	struct omap_mcbsp_st_data *st_data = mcbsp->st_data;
+
+	if (st_data->enabled && !st_data->running) {
+		omap_mcbsp_st_fir_write(mcbsp, st_data->taps);
+		omap_mcbsp_st_chgain(mcbsp);
+
+		if (!mcbsp->free) {
+			omap_mcbsp_st_on(mcbsp);
+			st_data->running = 1;
+		}
+	}
+
+	return 0;
+}
+
+int omap_mcbsp_st_stop(struct omap_mcbsp *mcbsp)
+{
+	struct omap_mcbsp_st_data *st_data = mcbsp->st_data;
+
+	if (st_data->running) {
+		if (!mcbsp->free) {
+			omap_mcbsp_st_off(mcbsp);
+			st_data->running = 0;
+		}
+	}
+
+	return 0;
+}
+
+int omap_mcbsp_st_init(struct platform_device *pdev)
+{
+	struct omap_mcbsp *mcbsp = platform_get_drvdata(pdev);
+	struct omap_mcbsp_st_data *st_data;
+	struct resource *res;
+	int ret;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "sidetone");
+	if (!res)
+		return 0;
+
+	st_data = devm_kzalloc(mcbsp->dev, sizeof(*mcbsp->st_data), GFP_KERNEL);
+	if (!st_data)
+		return -ENOMEM;
+
+	st_data->mcbsp_iclk = clk_get(mcbsp->dev, "ick");
+	if (IS_ERR(st_data->mcbsp_iclk)) {
+		dev_warn(mcbsp->dev,
+			 "Failed to get ick, sidetone might be broken\n");
+		st_data->mcbsp_iclk = NULL;
+	}
+
+	st_data->io_base_st = devm_ioremap(mcbsp->dev, res->start,
+					   resource_size(res));
+	if (!st_data->io_base_st)
+		return -ENOMEM;
+
+	ret = sysfs_create_group(&mcbsp->dev->kobj, &sidetone_attr_group);
+	if (ret)
+		return ret;
+
+	mcbsp->st_data = st_data;
+
+	return 0;
+}
+
+void omap_mcbsp_st_cleanup(struct platform_device *pdev)
+{
+	struct omap_mcbsp *mcbsp = platform_get_drvdata(pdev);
+
+	if (mcbsp->st_data) {
+		sysfs_remove_group(&mcbsp->dev->kobj, &sidetone_attr_group);
+		clk_put(mcbsp->st_data->mcbsp_iclk);
+	}
+}
+
+static int omap_mcbsp_st_info_volsw(struct snd_kcontrol *kcontrol,
+				    struct snd_ctl_elem_info *uinfo)
+{
+	struct soc_mixer_control *mc =
+		(struct soc_mixer_control *)kcontrol->private_value;
+	int max = mc->max;
+	int min = mc->min;
+
+	uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER;
+	uinfo->count = 1;
+	uinfo->value.integer.min = min;
+	uinfo->value.integer.max = max;
+	return 0;
+}
+
+#define OMAP_MCBSP_ST_CHANNEL_VOLUME(channel)				\
+static int								\
+omap_mcbsp_set_st_ch##channel##_volume(struct snd_kcontrol *kc,		\
+				       struct snd_ctl_elem_value *uc)	\
+{									\
+	struct snd_soc_dai *cpu_dai = snd_kcontrol_chip(kc);		\
+	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);	\
+	struct soc_mixer_control *mc =					\
+		(struct soc_mixer_control *)kc->private_value;		\
+	int max = mc->max;						\
+	int min = mc->min;						\
+	int val = uc->value.integer.value[0];				\
+									\
+	if (val < min || val > max)					\
+		return -EINVAL;						\
+									\
+	/* OMAP McBSP implementation uses index values 0..4 */		\
+	return omap_mcbsp_st_set_chgain(mcbsp, channel, val);		\
+}									\
+									\
+static int								\
+omap_mcbsp_get_st_ch##channel##_volume(struct snd_kcontrol *kc,		\
+				       struct snd_ctl_elem_value *uc)	\
+{									\
+	struct snd_soc_dai *cpu_dai = snd_kcontrol_chip(kc);		\
+	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);	\
+	s16 chgain;							\
+									\
+	if (omap_mcbsp_st_get_chgain(mcbsp, channel, &chgain))		\
+		return -EAGAIN;						\
+									\
+	uc->value.integer.value[0] = chgain;				\
+	return 0;							\
+}
+
+OMAP_MCBSP_ST_CHANNEL_VOLUME(0)
+OMAP_MCBSP_ST_CHANNEL_VOLUME(1)
+
+static int omap_mcbsp_st_put_mode(struct snd_kcontrol *kcontrol,
+				  struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_dai *cpu_dai = snd_kcontrol_chip(kcontrol);
+	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);
+	u8 value = ucontrol->value.integer.value[0];
+
+	if (value == omap_mcbsp_st_is_enabled(mcbsp))
+		return 0;
+
+	if (value)
+		omap_mcbsp_st_enable(mcbsp);
+	else
+		omap_mcbsp_st_disable(mcbsp);
+
+	return 1;
+}
+
+static int omap_mcbsp_st_get_mode(struct snd_kcontrol *kcontrol,
+				  struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_dai *cpu_dai = snd_kcontrol_chip(kcontrol);
+	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);
+
+	ucontrol->value.integer.value[0] = omap_mcbsp_st_is_enabled(mcbsp);
+	return 0;
+}
+
+#define OMAP_MCBSP_SOC_SINGLE_S16_EXT(xname, xmin, xmax,		\
+				      xhandler_get, xhandler_put)	\
+{	.iface = SNDRV_CTL_ELEM_IFACE_MIXER, .name = xname,		\
+	.info = omap_mcbsp_st_info_volsw,				\
+	.get = xhandler_get, .put = xhandler_put,			\
+	.private_value = (unsigned long)&(struct soc_mixer_control)	\
+	{.min = xmin, .max = xmax} }
+
+#define OMAP_MCBSP_ST_CONTROLS(port)					  \
+static const struct snd_kcontrol_new omap_mcbsp##port##_st_controls[] = { \
+SOC_SINGLE_EXT("McBSP" #port " Sidetone Switch", 1, 0, 1, 0,		  \
+	       omap_mcbsp_st_get_mode, omap_mcbsp_st_put_mode),		  \
+OMAP_MCBSP_SOC_SINGLE_S16_EXT("McBSP" #port " Sidetone Channel 0 Volume", \
+			      -32768, 32767,				  \
+			      omap_mcbsp_get_st_ch0_volume,		  \
+			      omap_mcbsp_set_st_ch0_volume),		  \
+OMAP_MCBSP_SOC_SINGLE_S16_EXT("McBSP" #port " Sidetone Channel 1 Volume", \
+			      -32768, 32767,				  \
+			      omap_mcbsp_get_st_ch1_volume,		  \
+			      omap_mcbsp_set_st_ch1_volume),		  \
+}
+
+OMAP_MCBSP_ST_CONTROLS(2);
+OMAP_MCBSP_ST_CONTROLS(3);
+
+int omap_mcbsp_st_add_controls(struct snd_soc_pcm_runtime *rtd, int port_id)
+{
+	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
+	struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai);
+
+	if (!mcbsp->st_data) {
+		dev_warn(mcbsp->dev, "No sidetone data for port\n");
+		return 0;
+	}
+
+	switch (port_id) {
+	case 2: /* McBSP 2 */
+		return snd_soc_add_dai_controls(cpu_dai,
+					omap_mcbsp2_st_controls,
+					ARRAY_SIZE(omap_mcbsp2_st_controls));
+	case 3: /* McBSP 3 */
+		return snd_soc_add_dai_controls(cpu_dai,
+					omap_mcbsp3_st_controls,
+					ARRAY_SIZE(omap_mcbsp3_st_controls));
+	default:
+		dev_err(mcbsp->dev, "Port %d not supported\n", port_id);
+		break;
+	}
+
+	return -EINVAL;
+}
+EXPORT_SYMBOL_GPL(omap_mcbsp_st_add_controls);
diff -Naur --no-dereference a/sound/soc/ti/omap-mcpdm.c b/sound/soc/ti/omap-mcpdm.c
--- a/sound/soc/ti/omap-mcpdm.c	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/omap-mcpdm.c	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,643 @@
+/*
+ * omap-mcpdm.c  --  OMAP ALSA SoC DAI driver using McPDM port
+ *
+ * Copyright (C) 2009 - 2011 Texas Instruments
+ *
+ * Author: Misael Lopez Cruz <misael.lopez@ti.com>
+ * Contact: Jorge Eduardo Candelaria <x0107209@ti.com>
+ *          Margarita Olaya <magi.olaya@ti.com>
+ *          Peter Ujfalusi <peter.ujfalusi@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA
+ *
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/interrupt.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/irq.h>
+#include <linux/clk.h>
+#include <linux/slab.h>
+#include <linux/pm_runtime.h>
+#include <linux/of_device.h>
+
+#include <sound/core.h>
+#include <sound/pcm.h>
+#include <sound/pcm_params.h>
+#include <sound/soc.h>
+#include <sound/dmaengine_pcm.h>
+
+#include "omap-mcpdm.h"
+#include "sdma-pcm.h"
+
+struct mcpdm_link_config {
+	u32 link_mask; /* channel mask for the direction */
+	u32 threshold; /* FIFO threshold */
+};
+
+struct omap_mcpdm {
+	struct device *dev;
+	unsigned long phys_base;
+	void __iomem *io_base;
+	int irq;
+	struct pm_qos_request pm_qos_req;
+	int latency[2];
+	struct clk *pdmclk;
+
+	struct mutex mutex;
+
+	/* Playback/Capture configuration */
+	struct mcpdm_link_config config[2];
+
+	/* McPDM dn offsets for rx1, and 2 channels */
+	u32 dn_rx_offset;
+
+	/* McPDM needs to be restarted due to runtime reconfiguration */
+	bool restart;
+
+	/* pm state for suspend/resume handling */
+	int pm_active_count;
+
+	struct snd_dmaengine_dai_dma_data dma_data[2];
+};
+
+/*
+ * Stream DMA parameters
+ */
+
+static inline void omap_mcpdm_write(struct omap_mcpdm *mcpdm, u16 reg, u32 val)
+{
+	writel_relaxed(val, mcpdm->io_base + reg);
+}
+
+static inline int omap_mcpdm_read(struct omap_mcpdm *mcpdm, u16 reg)
+{
+	return readl_relaxed(mcpdm->io_base + reg);
+}
+
+#ifdef DEBUG
+static void omap_mcpdm_reg_dump(struct omap_mcpdm *mcpdm)
+{
+	dev_dbg(mcpdm->dev, "***********************\n");
+	dev_dbg(mcpdm->dev, "IRQSTATUS_RAW:  0x%04x\n",
+			omap_mcpdm_read(mcpdm, MCPDM_REG_IRQSTATUS_RAW));
+	dev_dbg(mcpdm->dev, "IRQSTATUS:  0x%04x\n",
+			omap_mcpdm_read(mcpdm, MCPDM_REG_IRQSTATUS));
+	dev_dbg(mcpdm->dev, "IRQENABLE_SET:  0x%04x\n",
+			omap_mcpdm_read(mcpdm, MCPDM_REG_IRQENABLE_SET));
+	dev_dbg(mcpdm->dev, "IRQENABLE_CLR:  0x%04x\n",
+			omap_mcpdm_read(mcpdm, MCPDM_REG_IRQENABLE_CLR));
+	dev_dbg(mcpdm->dev, "IRQWAKE_EN: 0x%04x\n",
+			omap_mcpdm_read(mcpdm, MCPDM_REG_IRQWAKE_EN));
+	dev_dbg(mcpdm->dev, "DMAENABLE_SET: 0x%04x\n",
+			omap_mcpdm_read(mcpdm, MCPDM_REG_DMAENABLE_SET));
+	dev_dbg(mcpdm->dev, "DMAENABLE_CLR:  0x%04x\n",
+			omap_mcpdm_read(mcpdm, MCPDM_REG_DMAENABLE_CLR));
+	dev_dbg(mcpdm->dev, "DMAWAKEEN:  0x%04x\n",
+			omap_mcpdm_read(mcpdm, MCPDM_REG_DMAWAKEEN));
+	dev_dbg(mcpdm->dev, "CTRL:  0x%04x\n",
+			omap_mcpdm_read(mcpdm, MCPDM_REG_CTRL));
+	dev_dbg(mcpdm->dev, "DN_DATA:  0x%04x\n",
+			omap_mcpdm_read(mcpdm, MCPDM_REG_DN_DATA));
+	dev_dbg(mcpdm->dev, "UP_DATA: 0x%04x\n",
+			omap_mcpdm_read(mcpdm, MCPDM_REG_UP_DATA));
+	dev_dbg(mcpdm->dev, "FIFO_CTRL_DN: 0x%04x\n",
+			omap_mcpdm_read(mcpdm, MCPDM_REG_FIFO_CTRL_DN));
+	dev_dbg(mcpdm->dev, "FIFO_CTRL_UP:  0x%04x\n",
+			omap_mcpdm_read(mcpdm, MCPDM_REG_FIFO_CTRL_UP));
+	dev_dbg(mcpdm->dev, "***********************\n");
+}
+#else
+static void omap_mcpdm_reg_dump(struct omap_mcpdm *mcpdm) {}
+#endif
+
+/*
+ * Enables the transfer through the PDM interface to/from the Phoenix
+ * codec by enabling the corresponding UP or DN channels.
+ */
+static void omap_mcpdm_start(struct omap_mcpdm *mcpdm)
+{
+	u32 ctrl = omap_mcpdm_read(mcpdm, MCPDM_REG_CTRL);
+	u32 link_mask = mcpdm->config[0].link_mask | mcpdm->config[1].link_mask;
+
+	ctrl |= (MCPDM_SW_DN_RST | MCPDM_SW_UP_RST);
+	omap_mcpdm_write(mcpdm, MCPDM_REG_CTRL, ctrl);
+
+	ctrl |= link_mask;
+	omap_mcpdm_write(mcpdm, MCPDM_REG_CTRL, ctrl);
+
+	ctrl &= ~(MCPDM_SW_DN_RST | MCPDM_SW_UP_RST);
+	omap_mcpdm_write(mcpdm, MCPDM_REG_CTRL, ctrl);
+}
+
+/*
+ * Disables the transfer through the PDM interface to/from the Phoenix
+ * codec by disabling the corresponding UP or DN channels.
+ */
+static void omap_mcpdm_stop(struct omap_mcpdm *mcpdm)
+{
+	u32 ctrl = omap_mcpdm_read(mcpdm, MCPDM_REG_CTRL);
+	u32 link_mask = MCPDM_PDM_DN_MASK | MCPDM_PDM_UP_MASK;
+
+	ctrl |= (MCPDM_SW_DN_RST | MCPDM_SW_UP_RST);
+	omap_mcpdm_write(mcpdm, MCPDM_REG_CTRL, ctrl);
+
+	ctrl &= ~(link_mask);
+	omap_mcpdm_write(mcpdm, MCPDM_REG_CTRL, ctrl);
+
+	ctrl &= ~(MCPDM_SW_DN_RST | MCPDM_SW_UP_RST);
+	omap_mcpdm_write(mcpdm, MCPDM_REG_CTRL, ctrl);
+
+}
+
+/*
+ * Is the physical McPDM interface active.
+ */
+static inline int omap_mcpdm_active(struct omap_mcpdm *mcpdm)
+{
+	return omap_mcpdm_read(mcpdm, MCPDM_REG_CTRL) &
+					(MCPDM_PDM_DN_MASK | MCPDM_PDM_UP_MASK);
+}
+
+/*
+ * Configures McPDM uplink, and downlink for audio.
+ * This function should be called before omap_mcpdm_start.
+ */
+static void omap_mcpdm_open_streams(struct omap_mcpdm *mcpdm)
+{
+	u32 ctrl = omap_mcpdm_read(mcpdm, MCPDM_REG_CTRL);
+
+	omap_mcpdm_write(mcpdm, MCPDM_REG_CTRL, ctrl | MCPDM_WD_EN);
+
+	omap_mcpdm_write(mcpdm, MCPDM_REG_IRQENABLE_SET,
+			MCPDM_DN_IRQ_EMPTY | MCPDM_DN_IRQ_FULL |
+			MCPDM_UP_IRQ_EMPTY | MCPDM_UP_IRQ_FULL);
+
+	/* Enable DN RX1/2 offset cancellation feature, if configured */
+	if (mcpdm->dn_rx_offset) {
+		u32 dn_offset = mcpdm->dn_rx_offset;
+
+		omap_mcpdm_write(mcpdm, MCPDM_REG_DN_OFFSET, dn_offset);
+		dn_offset |= (MCPDM_DN_OFST_RX1_EN | MCPDM_DN_OFST_RX2_EN);
+		omap_mcpdm_write(mcpdm, MCPDM_REG_DN_OFFSET, dn_offset);
+	}
+
+	omap_mcpdm_write(mcpdm, MCPDM_REG_FIFO_CTRL_DN,
+			 mcpdm->config[SNDRV_PCM_STREAM_PLAYBACK].threshold);
+	omap_mcpdm_write(mcpdm, MCPDM_REG_FIFO_CTRL_UP,
+			 mcpdm->config[SNDRV_PCM_STREAM_CAPTURE].threshold);
+
+	omap_mcpdm_write(mcpdm, MCPDM_REG_DMAENABLE_SET,
+			MCPDM_DMA_DN_ENABLE | MCPDM_DMA_UP_ENABLE);
+}
+
+/*
+ * Cleans McPDM uplink, and downlink configuration.
+ * This function should be called when the stream is closed.
+ */
+static void omap_mcpdm_close_streams(struct omap_mcpdm *mcpdm)
+{
+	/* Disable irq request generation for downlink */
+	omap_mcpdm_write(mcpdm, MCPDM_REG_IRQENABLE_CLR,
+			MCPDM_DN_IRQ_EMPTY | MCPDM_DN_IRQ_FULL);
+
+	/* Disable DMA request generation for downlink */
+	omap_mcpdm_write(mcpdm, MCPDM_REG_DMAENABLE_CLR, MCPDM_DMA_DN_ENABLE);
+
+	/* Disable irq request generation for uplink */
+	omap_mcpdm_write(mcpdm, MCPDM_REG_IRQENABLE_CLR,
+			MCPDM_UP_IRQ_EMPTY | MCPDM_UP_IRQ_FULL);
+
+	/* Disable DMA request generation for uplink */
+	omap_mcpdm_write(mcpdm, MCPDM_REG_DMAENABLE_CLR, MCPDM_DMA_UP_ENABLE);
+
+	/* Disable RX1/2 offset cancellation */
+	if (mcpdm->dn_rx_offset)
+		omap_mcpdm_write(mcpdm, MCPDM_REG_DN_OFFSET, 0);
+}
+
+static irqreturn_t omap_mcpdm_irq_handler(int irq, void *dev_id)
+{
+	struct omap_mcpdm *mcpdm = dev_id;
+	int irq_status;
+
+	irq_status = omap_mcpdm_read(mcpdm, MCPDM_REG_IRQSTATUS);
+
+	/* Acknowledge irq event */
+	omap_mcpdm_write(mcpdm, MCPDM_REG_IRQSTATUS, irq_status);
+
+	if (irq_status & MCPDM_DN_IRQ_FULL)
+		dev_dbg(mcpdm->dev, "DN (playback) FIFO Full\n");
+
+	if (irq_status & MCPDM_DN_IRQ_EMPTY)
+		dev_dbg(mcpdm->dev, "DN (playback) FIFO Empty\n");
+
+	if (irq_status & MCPDM_DN_IRQ)
+		dev_dbg(mcpdm->dev, "DN (playback) write request\n");
+
+	if (irq_status & MCPDM_UP_IRQ_FULL)
+		dev_dbg(mcpdm->dev, "UP (capture) FIFO Full\n");
+
+	if (irq_status & MCPDM_UP_IRQ_EMPTY)
+		dev_dbg(mcpdm->dev, "UP (capture) FIFO Empty\n");
+
+	if (irq_status & MCPDM_UP_IRQ)
+		dev_dbg(mcpdm->dev, "UP (capture) write request\n");
+
+	return IRQ_HANDLED;
+}
+
+static int omap_mcpdm_dai_startup(struct snd_pcm_substream *substream,
+				  struct snd_soc_dai *dai)
+{
+	struct omap_mcpdm *mcpdm = snd_soc_dai_get_drvdata(dai);
+
+	mutex_lock(&mcpdm->mutex);
+
+	if (!dai->active)
+		omap_mcpdm_open_streams(mcpdm);
+
+	mutex_unlock(&mcpdm->mutex);
+
+	return 0;
+}
+
+static void omap_mcpdm_dai_shutdown(struct snd_pcm_substream *substream,
+				  struct snd_soc_dai *dai)
+{
+	struct omap_mcpdm *mcpdm = snd_soc_dai_get_drvdata(dai);
+	int tx = (substream->stream == SNDRV_PCM_STREAM_PLAYBACK);
+	int stream1 = tx ? SNDRV_PCM_STREAM_PLAYBACK : SNDRV_PCM_STREAM_CAPTURE;
+	int stream2 = tx ? SNDRV_PCM_STREAM_CAPTURE : SNDRV_PCM_STREAM_PLAYBACK;
+
+	mutex_lock(&mcpdm->mutex);
+
+	if (!dai->active) {
+		if (omap_mcpdm_active(mcpdm)) {
+			omap_mcpdm_stop(mcpdm);
+			omap_mcpdm_close_streams(mcpdm);
+			mcpdm->config[0].link_mask = 0;
+			mcpdm->config[1].link_mask = 0;
+		}
+	}
+
+	if (mcpdm->latency[stream2])
+		pm_qos_update_request(&mcpdm->pm_qos_req,
+				      mcpdm->latency[stream2]);
+	else if (mcpdm->latency[stream1])
+		pm_qos_remove_request(&mcpdm->pm_qos_req);
+
+	mcpdm->latency[stream1] = 0;
+
+	mutex_unlock(&mcpdm->mutex);
+}
+
+static int omap_mcpdm_dai_hw_params(struct snd_pcm_substream *substream,
+				    struct snd_pcm_hw_params *params,
+				    struct snd_soc_dai *dai)
+{
+	struct omap_mcpdm *mcpdm = snd_soc_dai_get_drvdata(dai);
+	int stream = substream->stream;
+	struct snd_dmaengine_dai_dma_data *dma_data;
+	u32 threshold;
+	int channels, latency;
+	int link_mask = 0;
+
+	channels = params_channels(params);
+	switch (channels) {
+	case 5:
+		if (stream == SNDRV_PCM_STREAM_CAPTURE)
+			/* up to 3 channels for capture */
+			return -EINVAL;
+		link_mask |= 1 << 4;
+		/* fall through */
+	case 4:
+		if (stream == SNDRV_PCM_STREAM_CAPTURE)
+			/* up to 3 channels for capture */
+			return -EINVAL;
+		link_mask |= 1 << 3;
+		/* fall through */
+	case 3:
+		link_mask |= 1 << 2;
+		/* fall through */
+	case 2:
+		link_mask |= 1 << 1;
+		/* fall through */
+	case 1:
+		link_mask |= 1 << 0;
+		break;
+	default:
+		/* unsupported number of channels */
+		return -EINVAL;
+	}
+
+	dma_data = snd_soc_dai_get_dma_data(dai, substream);
+
+	threshold = mcpdm->config[stream].threshold;
+	/* Configure McPDM channels, and DMA packet size */
+	if (stream == SNDRV_PCM_STREAM_PLAYBACK) {
+		link_mask <<= 3;
+
+		/* If capture is not running assume a stereo stream to come */
+		if (!mcpdm->config[!stream].link_mask)
+			mcpdm->config[!stream].link_mask = 0x3;
+
+		dma_data->maxburst =
+				(MCPDM_DN_THRES_MAX - threshold) * channels;
+		latency = threshold;
+	} else {
+		/* If playback is not running assume a stereo stream to come */
+		if (!mcpdm->config[!stream].link_mask)
+			mcpdm->config[!stream].link_mask = (0x3 << 3);
+
+		dma_data->maxburst = threshold * channels;
+		latency = (MCPDM_DN_THRES_MAX - threshold);
+	}
+
+	/*
+	 * The DMA must act to a DMA request within latency time (usec) to avoid
+	 * under/overflow
+	 */
+	mcpdm->latency[stream] = latency * USEC_PER_SEC / params_rate(params);
+
+	if (!mcpdm->latency[stream])
+		mcpdm->latency[stream] = 10;
+
+	/* Check if we need to restart McPDM with this stream */
+	if (mcpdm->config[stream].link_mask &&
+	    mcpdm->config[stream].link_mask != link_mask)
+		mcpdm->restart = true;
+
+	mcpdm->config[stream].link_mask = link_mask;
+
+	return 0;
+}
+
+static int omap_mcpdm_prepare(struct snd_pcm_substream *substream,
+				  struct snd_soc_dai *dai)
+{
+	struct omap_mcpdm *mcpdm = snd_soc_dai_get_drvdata(dai);
+	struct pm_qos_request *pm_qos_req = &mcpdm->pm_qos_req;
+	int tx = (substream->stream == SNDRV_PCM_STREAM_PLAYBACK);
+	int stream1 = tx ? SNDRV_PCM_STREAM_PLAYBACK : SNDRV_PCM_STREAM_CAPTURE;
+	int stream2 = tx ? SNDRV_PCM_STREAM_CAPTURE : SNDRV_PCM_STREAM_PLAYBACK;
+	int latency = mcpdm->latency[stream2];
+
+	/* Prevent omap hardware from hitting off between FIFO fills */
+	if (!latency || mcpdm->latency[stream1] < latency)
+		latency = mcpdm->latency[stream1];
+
+	if (pm_qos_request_active(pm_qos_req))
+		pm_qos_update_request(pm_qos_req, latency);
+	else if (latency)
+		pm_qos_add_request(pm_qos_req, PM_QOS_CPU_DMA_LATENCY, latency);
+
+	if (!omap_mcpdm_active(mcpdm)) {
+		omap_mcpdm_start(mcpdm);
+		omap_mcpdm_reg_dump(mcpdm);
+	} else if (mcpdm->restart) {
+		omap_mcpdm_stop(mcpdm);
+		omap_mcpdm_start(mcpdm);
+		mcpdm->restart = false;
+		omap_mcpdm_reg_dump(mcpdm);
+	}
+
+	return 0;
+}
+
+static const struct snd_soc_dai_ops omap_mcpdm_dai_ops = {
+	.startup	= omap_mcpdm_dai_startup,
+	.shutdown	= omap_mcpdm_dai_shutdown,
+	.hw_params	= omap_mcpdm_dai_hw_params,
+	.prepare	= omap_mcpdm_prepare,
+};
+
+static int omap_mcpdm_probe(struct snd_soc_dai *dai)
+{
+	struct omap_mcpdm *mcpdm = snd_soc_dai_get_drvdata(dai);
+	int ret;
+
+	ret = clk_prepare_enable(mcpdm->pdmclk);
+	if (ret)
+		return ret;
+
+	pm_runtime_enable(mcpdm->dev);
+
+	/* Disable lines while request is ongoing */
+	pm_runtime_get_sync(mcpdm->dev);
+	omap_mcpdm_write(mcpdm, MCPDM_REG_CTRL, 0x00);
+
+	ret = request_irq(mcpdm->irq, omap_mcpdm_irq_handler, 0, "McPDM",
+			  (void *)mcpdm);
+
+	pm_runtime_put_sync(mcpdm->dev);
+
+	if (ret) {
+		dev_err(mcpdm->dev, "Request for IRQ failed\n");
+		pm_runtime_disable(mcpdm->dev);
+	}
+
+	/* Configure McPDM threshold values */
+	mcpdm->config[SNDRV_PCM_STREAM_PLAYBACK].threshold = 2;
+	mcpdm->config[SNDRV_PCM_STREAM_CAPTURE].threshold =
+							MCPDM_UP_THRES_MAX - 3;
+
+	snd_soc_dai_init_dma_data(dai,
+				  &mcpdm->dma_data[SNDRV_PCM_STREAM_PLAYBACK],
+				  &mcpdm->dma_data[SNDRV_PCM_STREAM_CAPTURE]);
+
+	return ret;
+}
+
+static int omap_mcpdm_remove(struct snd_soc_dai *dai)
+{
+	struct omap_mcpdm *mcpdm = snd_soc_dai_get_drvdata(dai);
+
+	free_irq(mcpdm->irq, (void *)mcpdm);
+	pm_runtime_disable(mcpdm->dev);
+
+	if (pm_qos_request_active(&mcpdm->pm_qos_req))
+		pm_qos_remove_request(&mcpdm->pm_qos_req);
+
+	clk_disable_unprepare(mcpdm->pdmclk);
+
+	return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int omap_mcpdm_suspend(struct snd_soc_dai *dai)
+{
+	struct omap_mcpdm *mcpdm = snd_soc_dai_get_drvdata(dai);
+
+	if (dai->active) {
+		omap_mcpdm_stop(mcpdm);
+		omap_mcpdm_close_streams(mcpdm);
+	}
+
+	mcpdm->pm_active_count = 0;
+	while (pm_runtime_active(mcpdm->dev)) {
+		pm_runtime_put_sync(mcpdm->dev);
+		mcpdm->pm_active_count++;
+	}
+
+	clk_disable_unprepare(mcpdm->pdmclk);
+
+	return 0;
+}
+
+static int omap_mcpdm_resume(struct snd_soc_dai *dai)
+{
+	struct omap_mcpdm *mcpdm = snd_soc_dai_get_drvdata(dai);
+	int ret;
+
+	ret = clk_prepare_enable(mcpdm->pdmclk);
+	if (ret)
+		return ret;
+
+	if (mcpdm->pm_active_count) {
+		while (mcpdm->pm_active_count--)
+			pm_runtime_get_sync(mcpdm->dev);
+
+		if (dai->active) {
+			omap_mcpdm_open_streams(mcpdm);
+			omap_mcpdm_start(mcpdm);
+		}
+	}
+
+
+	return 0;
+}
+#else
+#define omap_mcpdm_suspend NULL
+#define omap_mcpdm_resume NULL
+#endif
+
+#define OMAP_MCPDM_RATES	(SNDRV_PCM_RATE_88200 | SNDRV_PCM_RATE_96000)
+#define OMAP_MCPDM_FORMATS	SNDRV_PCM_FMTBIT_S32_LE
+
+static struct snd_soc_dai_driver omap_mcpdm_dai = {
+	.probe = omap_mcpdm_probe,
+	.remove = omap_mcpdm_remove,
+	.suspend = omap_mcpdm_suspend,
+	.resume = omap_mcpdm_resume,
+	.probe_order = SND_SOC_COMP_ORDER_LATE,
+	.remove_order = SND_SOC_COMP_ORDER_EARLY,
+	.playback = {
+		.channels_min = 1,
+		.channels_max = 5,
+		.rates = OMAP_MCPDM_RATES,
+		.formats = OMAP_MCPDM_FORMATS,
+		.sig_bits = 24,
+	},
+	.capture = {
+		.channels_min = 1,
+		.channels_max = 3,
+		.rates = OMAP_MCPDM_RATES,
+		.formats = OMAP_MCPDM_FORMATS,
+		.sig_bits = 24,
+	},
+	.ops = &omap_mcpdm_dai_ops,
+};
+
+static const struct snd_soc_component_driver omap_mcpdm_component = {
+	.name		= "omap-mcpdm",
+};
+
+void omap_mcpdm_configure_dn_offsets(struct snd_soc_pcm_runtime *rtd,
+				    u8 rx1, u8 rx2)
+{
+	struct omap_mcpdm *mcpdm = snd_soc_dai_get_drvdata(rtd->cpu_dai);
+
+	mcpdm->dn_rx_offset = MCPDM_DNOFST_RX1(rx1) | MCPDM_DNOFST_RX2(rx2);
+}
+EXPORT_SYMBOL_GPL(omap_mcpdm_configure_dn_offsets);
+
+static int asoc_mcpdm_probe(struct platform_device *pdev)
+{
+	struct omap_mcpdm *mcpdm;
+	struct resource *res;
+	int ret;
+
+	mcpdm = devm_kzalloc(&pdev->dev, sizeof(struct omap_mcpdm), GFP_KERNEL);
+	if (!mcpdm)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, mcpdm);
+
+	mutex_init(&mcpdm->mutex);
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dma");
+	if (res == NULL)
+		return -ENOMEM;
+
+	mcpdm->dma_data[0].addr = res->start + MCPDM_REG_DN_DATA;
+	mcpdm->dma_data[1].addr = res->start + MCPDM_REG_UP_DATA;
+
+	mcpdm->dma_data[0].filter_data = "dn_link";
+	mcpdm->dma_data[1].filter_data = "up_link";
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mpu");
+	mcpdm->io_base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(mcpdm->io_base))
+		return PTR_ERR(mcpdm->io_base);
+
+	mcpdm->irq = platform_get_irq(pdev, 0);
+	if (mcpdm->irq < 0)
+		return mcpdm->irq;
+
+	mcpdm->dev = &pdev->dev;
+
+	mcpdm->pdmclk = devm_clk_get(&pdev->dev, "pdmclk");
+	if (IS_ERR(mcpdm->pdmclk)) {
+		if (PTR_ERR(mcpdm->pdmclk) == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
+		dev_warn(&pdev->dev, "Error getting pdmclk (%ld)!\n",
+			 PTR_ERR(mcpdm->pdmclk));
+		mcpdm->pdmclk = NULL;
+	}
+
+	ret =  devm_snd_soc_register_component(&pdev->dev,
+					       &omap_mcpdm_component,
+					       &omap_mcpdm_dai, 1);
+	if (ret)
+		return ret;
+
+	return sdma_pcm_platform_register(&pdev->dev, "dn_link", "up_link");
+}
+
+static const struct of_device_id omap_mcpdm_of_match[] = {
+	{ .compatible = "ti,omap4-mcpdm", },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, omap_mcpdm_of_match);
+
+static struct platform_driver asoc_mcpdm_driver = {
+	.driver = {
+		.name	= "omap-mcpdm",
+		.of_match_table = omap_mcpdm_of_match,
+	},
+
+	.probe	= asoc_mcpdm_probe,
+};
+
+module_platform_driver(asoc_mcpdm_driver);
+
+MODULE_ALIAS("platform:omap-mcpdm");
+MODULE_AUTHOR("Misael Lopez Cruz <misael.lopez@ti.com>");
+MODULE_DESCRIPTION("OMAP PDM SoC Interface");
+MODULE_LICENSE("GPL");
diff -Naur --no-dereference a/sound/soc/ti/omap-mcpdm.h b/sound/soc/ti/omap-mcpdm.h
--- a/sound/soc/ti/omap-mcpdm.h	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/omap-mcpdm.h	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,107 @@
+/*
+ * omap-mcpdm.h
+ *
+ * Copyright (C) 2009 - 2011 Texas Instruments
+ *
+ * Contact: Misael Lopez Cruz <misael.lopez@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA
+ *
+ */
+
+#ifndef __OMAP_MCPDM_H__
+#define __OMAP_MCPDM_H__
+
+#define MCPDM_REG_REVISION		0x00
+#define MCPDM_REG_SYSCONFIG		0x10
+#define MCPDM_REG_IRQSTATUS_RAW		0x24
+#define MCPDM_REG_IRQSTATUS		0x28
+#define MCPDM_REG_IRQENABLE_SET		0x2C
+#define MCPDM_REG_IRQENABLE_CLR		0x30
+#define MCPDM_REG_IRQWAKE_EN		0x34
+#define MCPDM_REG_DMAENABLE_SET		0x38
+#define MCPDM_REG_DMAENABLE_CLR		0x3C
+#define MCPDM_REG_DMAWAKEEN		0x40
+#define MCPDM_REG_CTRL			0x44
+#define MCPDM_REG_DN_DATA		0x48
+#define MCPDM_REG_UP_DATA		0x4C
+#define MCPDM_REG_FIFO_CTRL_DN		0x50
+#define MCPDM_REG_FIFO_CTRL_UP		0x54
+#define MCPDM_REG_DN_OFFSET		0x58
+
+/*
+ * MCPDM_IRQ bit fields
+ * IRQSTATUS_RAW, IRQSTATUS, IRQENABLE_SET, IRQENABLE_CLR
+ */
+
+#define MCPDM_DN_IRQ			(1 << 0)
+#define MCPDM_DN_IRQ_EMPTY		(1 << 1)
+#define MCPDM_DN_IRQ_ALMST_EMPTY	(1 << 2)
+#define MCPDM_DN_IRQ_FULL		(1 << 3)
+
+#define MCPDM_UP_IRQ			(1 << 8)
+#define MCPDM_UP_IRQ_EMPTY		(1 << 9)
+#define MCPDM_UP_IRQ_ALMST_FULL		(1 << 10)
+#define MCPDM_UP_IRQ_FULL		(1 << 11)
+
+#define MCPDM_DOWNLINK_IRQ_MASK		0x00F
+#define MCPDM_UPLINK_IRQ_MASK		0xF00
+
+/*
+ * MCPDM_DMAENABLE bit fields
+ */
+
+#define MCPDM_DMA_DN_ENABLE		(1 << 0)
+#define MCPDM_DMA_UP_ENABLE		(1 << 1)
+
+/*
+ * MCPDM_CTRL bit fields
+ */
+
+#define MCPDM_PDM_UPLINK_EN(x)		(1 << (x - 1)) /* ch1 is at bit 0 */
+#define MCPDM_PDM_DOWNLINK_EN(x)	(1 << (x + 2)) /* ch1 is at bit 3 */
+#define MCPDM_PDMOUTFORMAT		(1 << 8)
+#define MCPDM_CMD_INT			(1 << 9)
+#define MCPDM_STATUS_INT		(1 << 10)
+#define MCPDM_SW_UP_RST			(1 << 11)
+#define MCPDM_SW_DN_RST			(1 << 12)
+#define MCPDM_WD_EN			(1 << 14)
+#define MCPDM_PDM_UP_MASK		0x7
+#define MCPDM_PDM_DN_MASK		(0x1f << 3)
+
+
+#define MCPDM_PDMOUTFORMAT_LJUST	(0 << 8)
+#define MCPDM_PDMOUTFORMAT_RJUST	(1 << 8)
+
+/*
+ * MCPDM_FIFO_CTRL bit fields
+ */
+
+#define MCPDM_UP_THRES_MAX		0xF
+#define MCPDM_DN_THRES_MAX		0xF
+
+/*
+ * MCPDM_DN_OFFSET bit fields
+ */
+
+#define MCPDM_DN_OFST_RX1_EN		(1 << 0)
+#define MCPDM_DNOFST_RX1(x)		((x & 0x1f) << 1)
+#define MCPDM_DN_OFST_RX2_EN		(1 << 8)
+#define MCPDM_DNOFST_RX2(x)		((x & 0x1f) << 9)
+
+void omap_mcpdm_configure_dn_offsets(struct snd_soc_pcm_runtime *rtd,
+				    u8 rx1, u8 rx2);
+
+#endif	/* End of __OMAP_MCPDM_H__ */
diff -Naur --no-dereference a/sound/soc/ti/omap-twl4030.c b/sound/soc/ti/omap-twl4030.c
--- a/sound/soc/ti/omap-twl4030.c	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/omap-twl4030.c	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,353 @@
+/*
+ * omap-twl4030.c  --  SoC audio for TI SoC based boards with twl4030 codec
+ *
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com
+ * All rights reserved.
+ *
+ * Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
+ *
+ * This driver replaces the following machine drivers:
+ * omap3beagle (Author: Steve Sakoman <steve@sakoman.com>)
+ * omap3evm (Author: Anuj Aggarwal <anuj.aggarwal@ti.com>)
+ * overo (Author: Steve Sakoman <steve@sakoman.com>)
+ * igep0020 (Author: Enric Balletbo i Serra <eballetbo@iseebcn.com>)
+ * zoom2 (Author: Misael Lopez Cruz <misael.lopez@ti.com>)
+ * sdp3430 (Author: Misael Lopez Cruz <misael.lopez@ti.com>)
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA
+ *
+ */
+
+#include <linux/platform_device.h>
+#include <linux/platform_data/omap-twl4030.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/gpio.h>
+#include <linux/of_gpio.h>
+
+#include <sound/core.h>
+#include <sound/pcm.h>
+#include <sound/soc.h>
+#include <sound/jack.h>
+
+#include "omap-mcbsp.h"
+
+struct omap_twl4030 {
+	int jack_detect;	/* board can detect jack events */
+	struct snd_soc_jack hs_jack;
+};
+
+static int omap_twl4030_hw_params(struct snd_pcm_substream *substream,
+	struct snd_pcm_hw_params *params)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	unsigned int fmt;
+
+	switch (params_channels(params)) {
+	case 2: /* Stereo I2S mode */
+		fmt =	SND_SOC_DAIFMT_I2S |
+			SND_SOC_DAIFMT_NB_NF |
+			SND_SOC_DAIFMT_CBM_CFM;
+		break;
+	case 4: /* Four channel TDM mode */
+		fmt =	SND_SOC_DAIFMT_DSP_A |
+			SND_SOC_DAIFMT_IB_NF |
+			SND_SOC_DAIFMT_CBM_CFM;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return snd_soc_runtime_set_dai_fmt(rtd, fmt);
+}
+
+static const struct snd_soc_ops omap_twl4030_ops = {
+	.hw_params = omap_twl4030_hw_params,
+};
+
+static const struct snd_soc_dapm_widget dapm_widgets[] = {
+	SND_SOC_DAPM_SPK("Earpiece Spk", NULL),
+	SND_SOC_DAPM_SPK("Handsfree Spk", NULL),
+	SND_SOC_DAPM_HP("Headset Stereophone", NULL),
+	SND_SOC_DAPM_SPK("Ext Spk", NULL),
+	SND_SOC_DAPM_SPK("Carkit Spk", NULL),
+
+	SND_SOC_DAPM_MIC("Main Mic", NULL),
+	SND_SOC_DAPM_MIC("Sub Mic", NULL),
+	SND_SOC_DAPM_MIC("Headset Mic", NULL),
+	SND_SOC_DAPM_MIC("Carkit Mic", NULL),
+	SND_SOC_DAPM_MIC("Digital0 Mic", NULL),
+	SND_SOC_DAPM_MIC("Digital1 Mic", NULL),
+	SND_SOC_DAPM_LINE("Line In", NULL),
+};
+
+static const struct snd_soc_dapm_route audio_map[] = {
+	/* Headset Stereophone:  HSOL, HSOR */
+	{"Headset Stereophone", NULL, "HSOL"},
+	{"Headset Stereophone", NULL, "HSOR"},
+	/* External Speakers: HFL, HFR */
+	{"Handsfree Spk", NULL, "HFL"},
+	{"Handsfree Spk", NULL, "HFR"},
+	/* External Speakers: PredrivL, PredrivR */
+	{"Ext Spk", NULL, "PREDRIVEL"},
+	{"Ext Spk", NULL, "PREDRIVER"},
+	/* Carkit speakers:  CARKITL, CARKITR */
+	{"Carkit Spk", NULL, "CARKITL"},
+	{"Carkit Spk", NULL, "CARKITR"},
+	/* Earpiece */
+	{"Earpiece Spk", NULL, "EARPIECE"},
+
+	/* External Mics: MAINMIC, SUBMIC with bias */
+	{"MAINMIC", NULL, "Main Mic"},
+	{"Main Mic", NULL, "Mic Bias 1"},
+	{"SUBMIC", NULL, "Sub Mic"},
+	{"Sub Mic", NULL, "Mic Bias 2"},
+	/* Headset Mic: HSMIC with bias */
+	{"HSMIC", NULL, "Headset Mic"},
+	{"Headset Mic", NULL, "Headset Mic Bias"},
+	/* Digital Mics: DIGIMIC0, DIGIMIC1 with bias */
+	{"DIGIMIC0", NULL, "Digital0 Mic"},
+	{"Digital0 Mic", NULL, "Mic Bias 1"},
+	{"DIGIMIC1", NULL, "Digital1 Mic"},
+	{"Digital1 Mic", NULL, "Mic Bias 2"},
+	/* Carkit In: CARKITMIC */
+	{"CARKITMIC", NULL, "Carkit Mic"},
+	/* Aux In: AUXL, AUXR */
+	{"AUXL", NULL, "Line In"},
+	{"AUXR", NULL, "Line In"},
+};
+
+/* Headset jack detection DAPM pins */
+static struct snd_soc_jack_pin hs_jack_pins[] = {
+	{
+		.pin = "Headset Mic",
+		.mask = SND_JACK_MICROPHONE,
+	},
+	{
+		.pin = "Headset Stereophone",
+		.mask = SND_JACK_HEADPHONE,
+	},
+};
+
+/* Headset jack detection gpios */
+static struct snd_soc_jack_gpio hs_jack_gpios[] = {
+	{
+		.name = "hsdet-gpio",
+		.report = SND_JACK_HEADSET,
+		.debounce_time = 200,
+	},
+};
+
+static inline void twl4030_disconnect_pin(struct snd_soc_dapm_context *dapm,
+					  int connected, char *pin)
+{
+	if (!connected)
+		snd_soc_dapm_disable_pin(dapm, pin);
+}
+
+static int omap_twl4030_init(struct snd_soc_pcm_runtime *rtd)
+{
+	struct snd_soc_card *card = rtd->card;
+	struct snd_soc_dapm_context *dapm = &card->dapm;
+	struct omap_tw4030_pdata *pdata = dev_get_platdata(card->dev);
+	struct omap_twl4030 *priv = snd_soc_card_get_drvdata(card);
+	int ret = 0;
+
+	/* Headset jack detection only if it is supported */
+	if (priv->jack_detect > 0) {
+		hs_jack_gpios[0].gpio = priv->jack_detect;
+
+		ret = snd_soc_card_jack_new(rtd->card, "Headset Jack",
+					    SND_JACK_HEADSET, &priv->hs_jack,
+					    hs_jack_pins,
+					    ARRAY_SIZE(hs_jack_pins));
+		if (ret)
+			return ret;
+
+		ret = snd_soc_jack_add_gpios(&priv->hs_jack,
+					     ARRAY_SIZE(hs_jack_gpios),
+					     hs_jack_gpios);
+		if (ret)
+			return ret;
+	}
+
+	/*
+	 * NULL pdata means we booted with DT. In this case the routing is
+	 * provided and the card is fully routed, no need to mark pins.
+	 */
+	if (!pdata || !pdata->custom_routing)
+		return ret;
+
+	/* Disable not connected paths if not used */
+	twl4030_disconnect_pin(dapm, pdata->has_ear, "Earpiece Spk");
+	twl4030_disconnect_pin(dapm, pdata->has_hf, "Handsfree Spk");
+	twl4030_disconnect_pin(dapm, pdata->has_hs, "Headset Stereophone");
+	twl4030_disconnect_pin(dapm, pdata->has_predriv, "Ext Spk");
+	twl4030_disconnect_pin(dapm, pdata->has_carkit, "Carkit Spk");
+
+	twl4030_disconnect_pin(dapm, pdata->has_mainmic, "Main Mic");
+	twl4030_disconnect_pin(dapm, pdata->has_submic, "Sub Mic");
+	twl4030_disconnect_pin(dapm, pdata->has_hsmic, "Headset Mic");
+	twl4030_disconnect_pin(dapm, pdata->has_carkitmic, "Carkit Mic");
+	twl4030_disconnect_pin(dapm, pdata->has_digimic0, "Digital0 Mic");
+	twl4030_disconnect_pin(dapm, pdata->has_digimic1, "Digital1 Mic");
+	twl4030_disconnect_pin(dapm, pdata->has_linein, "Line In");
+
+	return ret;
+}
+
+/* Digital audio interface glue - connects codec <--> CPU */
+static struct snd_soc_dai_link omap_twl4030_dai_links[] = {
+	{
+		.name = "TWL4030 HiFi",
+		.stream_name = "TWL4030 HiFi",
+		.cpu_dai_name = "omap-mcbsp.2",
+		.codec_dai_name = "twl4030-hifi",
+		.platform_name = "omap-mcbsp.2",
+		.codec_name = "twl4030-codec",
+		.init = omap_twl4030_init,
+		.ops = &omap_twl4030_ops,
+	},
+	{
+		.name = "TWL4030 Voice",
+		.stream_name = "TWL4030 Voice",
+		.cpu_dai_name = "omap-mcbsp.3",
+		.codec_dai_name = "twl4030-voice",
+		.platform_name = "omap-mcbsp.3",
+		.codec_name = "twl4030-codec",
+		.dai_fmt = SND_SOC_DAIFMT_DSP_A | SND_SOC_DAIFMT_IB_NF |
+			   SND_SOC_DAIFMT_CBM_CFM,
+	},
+};
+
+/* Audio machine driver */
+static struct snd_soc_card omap_twl4030_card = {
+	.owner = THIS_MODULE,
+	.dai_link = omap_twl4030_dai_links,
+	.num_links = ARRAY_SIZE(omap_twl4030_dai_links),
+
+	.dapm_widgets = dapm_widgets,
+	.num_dapm_widgets = ARRAY_SIZE(dapm_widgets),
+	.dapm_routes = audio_map,
+	.num_dapm_routes = ARRAY_SIZE(audio_map),
+};
+
+static int omap_twl4030_probe(struct platform_device *pdev)
+{
+	struct omap_tw4030_pdata *pdata = dev_get_platdata(&pdev->dev);
+	struct device_node *node = pdev->dev.of_node;
+	struct snd_soc_card *card = &omap_twl4030_card;
+	struct omap_twl4030 *priv;
+	int ret = 0;
+
+	card->dev = &pdev->dev;
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(struct omap_twl4030), GFP_KERNEL);
+	if (priv == NULL)
+		return -ENOMEM;
+
+	if (node) {
+		struct device_node *dai_node;
+		struct property *prop;
+
+		if (snd_soc_of_parse_card_name(card, "ti,model")) {
+			dev_err(&pdev->dev, "Card name is not provided\n");
+			return -ENODEV;
+		}
+
+		dai_node = of_parse_phandle(node, "ti,mcbsp", 0);
+		if (!dai_node) {
+			dev_err(&pdev->dev, "McBSP node is not provided\n");
+			return -EINVAL;
+		}
+		omap_twl4030_dai_links[0].cpu_dai_name  = NULL;
+		omap_twl4030_dai_links[0].cpu_of_node = dai_node;
+
+		omap_twl4030_dai_links[0].platform_name  = NULL;
+		omap_twl4030_dai_links[0].platform_of_node = dai_node;
+
+		dai_node = of_parse_phandle(node, "ti,mcbsp-voice", 0);
+		if (!dai_node) {
+			card->num_links = 1;
+		} else {
+			omap_twl4030_dai_links[1].cpu_dai_name  = NULL;
+			omap_twl4030_dai_links[1].cpu_of_node = dai_node;
+
+			omap_twl4030_dai_links[1].platform_name  = NULL;
+			omap_twl4030_dai_links[1].platform_of_node = dai_node;
+		}
+
+		priv->jack_detect = of_get_named_gpio(node,
+						      "ti,jack-det-gpio", 0);
+
+		/* Optional: audio routing can be provided */
+		prop = of_find_property(node, "ti,audio-routing", NULL);
+		if (prop) {
+			ret = snd_soc_of_parse_audio_routing(card,
+							    "ti,audio-routing");
+			if (ret)
+				return ret;
+
+			card->fully_routed = 1;
+		}
+	} else if (pdata) {
+		if (pdata->card_name) {
+			card->name = pdata->card_name;
+		} else {
+			dev_err(&pdev->dev, "Card name is not provided\n");
+			return -ENODEV;
+		}
+
+		if (!pdata->voice_connected)
+			card->num_links = 1;
+
+		priv->jack_detect = pdata->jack_detect;
+	} else {
+		dev_err(&pdev->dev, "Missing pdata\n");
+		return -ENODEV;
+	}
+
+	snd_soc_card_set_drvdata(card, priv);
+	ret = devm_snd_soc_register_card(&pdev->dev, card);
+	if (ret) {
+		dev_err(&pdev->dev, "devm_snd_soc_register_card() failed: %d\n",
+			ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static const struct of_device_id omap_twl4030_of_match[] = {
+	{.compatible = "ti,omap-twl4030", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, omap_twl4030_of_match);
+
+static struct platform_driver omap_twl4030_driver = {
+	.driver = {
+		.name = "omap-twl4030",
+		.pm = &snd_soc_pm_ops,
+		.of_match_table = omap_twl4030_of_match,
+	},
+	.probe = omap_twl4030_probe,
+};
+
+module_platform_driver(omap_twl4030_driver);
+
+MODULE_AUTHOR("Peter Ujfalusi <peter.ujfalusi@ti.com>");
+MODULE_DESCRIPTION("ALSA SoC for TI SoC based boards with twl4030 codec");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:omap-twl4030");
diff -Naur --no-dereference a/sound/soc/ti/osk5912.c b/sound/soc/ti/osk5912.c
--- a/sound/soc/ti/osk5912.c	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/osk5912.c	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,187 @@
+/*
+ * osk5912.c  --  SoC audio for OSK 5912
+ *
+ * Copyright (C) 2008 Mistral Solutions
+ *
+ * Contact: Arun KS  <arunks@mistralsolutions.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA
+ *
+ */
+
+#include <linux/clk.h>
+#include <linux/platform_device.h>
+#include <sound/core.h>
+#include <sound/pcm.h>
+#include <sound/soc.h>
+
+#include <asm/mach-types.h>
+#include <linux/gpio.h>
+#include <linux/module.h>
+#include <linux/platform_data/asoc-ti-mcbsp.h>
+
+#include "omap-mcbsp.h"
+#include "../codecs/tlv320aic23.h"
+
+#define CODEC_CLOCK 	12000000
+
+static struct clk *tlv320aic23_mclk;
+
+static int osk_startup(struct snd_pcm_substream *substream)
+{
+	return clk_enable(tlv320aic23_mclk);
+}
+
+static void osk_shutdown(struct snd_pcm_substream *substream)
+{
+	clk_disable(tlv320aic23_mclk);
+}
+
+static int osk_hw_params(struct snd_pcm_substream *substream,
+			 struct snd_pcm_hw_params *params)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_dai *codec_dai = rtd->codec_dai;
+	int err;
+
+	/* Set the codec system clock for DAC and ADC */
+	err =
+	    snd_soc_dai_set_sysclk(codec_dai, 0, CODEC_CLOCK, SND_SOC_CLOCK_IN);
+
+	if (err < 0) {
+		printk(KERN_ERR "can't set codec system clock\n");
+		return err;
+	}
+
+	return err;
+}
+
+static const struct snd_soc_ops osk_ops = {
+	.startup = osk_startup,
+	.hw_params = osk_hw_params,
+	.shutdown = osk_shutdown,
+};
+
+static const struct snd_soc_dapm_widget tlv320aic23_dapm_widgets[] = {
+	SND_SOC_DAPM_HP("Headphone Jack", NULL),
+	SND_SOC_DAPM_LINE("Line In", NULL),
+	SND_SOC_DAPM_MIC("Mic Jack", NULL),
+};
+
+static const struct snd_soc_dapm_route audio_map[] = {
+	{"Headphone Jack", NULL, "LHPOUT"},
+	{"Headphone Jack", NULL, "RHPOUT"},
+
+	{"LLINEIN", NULL, "Line In"},
+	{"RLINEIN", NULL, "Line In"},
+
+	{"MICIN", NULL, "Mic Jack"},
+};
+
+/* Digital audio interface glue - connects codec <--> CPU */
+static struct snd_soc_dai_link osk_dai = {
+	.name = "TLV320AIC23",
+	.stream_name = "AIC23",
+	.cpu_dai_name = "omap-mcbsp.1",
+	.codec_dai_name = "tlv320aic23-hifi",
+	.platform_name = "omap-mcbsp.1",
+	.codec_name = "tlv320aic23-codec",
+	.dai_fmt = SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_NB_NF |
+		   SND_SOC_DAIFMT_CBM_CFM,
+	.ops = &osk_ops,
+};
+
+/* Audio machine driver */
+static struct snd_soc_card snd_soc_card_osk = {
+	.name = "OSK5912",
+	.owner = THIS_MODULE,
+	.dai_link = &osk_dai,
+	.num_links = 1,
+
+	.dapm_widgets = tlv320aic23_dapm_widgets,
+	.num_dapm_widgets = ARRAY_SIZE(tlv320aic23_dapm_widgets),
+	.dapm_routes = audio_map,
+	.num_dapm_routes = ARRAY_SIZE(audio_map),
+};
+
+static struct platform_device *osk_snd_device;
+
+static int __init osk_soc_init(void)
+{
+	int err;
+	u32 curRate;
+	struct device *dev;
+
+	if (!(machine_is_omap_osk()))
+		return -ENODEV;
+
+	osk_snd_device = platform_device_alloc("soc-audio", -1);
+	if (!osk_snd_device)
+		return -ENOMEM;
+
+	platform_set_drvdata(osk_snd_device, &snd_soc_card_osk);
+	err = platform_device_add(osk_snd_device);
+	if (err)
+		goto err1;
+
+	dev = &osk_snd_device->dev;
+
+	tlv320aic23_mclk = clk_get(dev, "mclk");
+	if (IS_ERR(tlv320aic23_mclk)) {
+		printk(KERN_ERR "Could not get mclk clock\n");
+		err = PTR_ERR(tlv320aic23_mclk);
+		goto err2;
+	}
+
+	/*
+	 * Configure 12 MHz output on MCLK.
+	 */
+	curRate = (uint) clk_get_rate(tlv320aic23_mclk);
+	if (curRate != CODEC_CLOCK) {
+		if (clk_set_rate(tlv320aic23_mclk, CODEC_CLOCK)) {
+			printk(KERN_ERR "Cannot set MCLK for AIC23 CODEC\n");
+			err = -ECANCELED;
+			goto err3;
+		}
+	}
+
+	printk(KERN_INFO "MCLK = %d [%d]\n",
+	       (uint) clk_get_rate(tlv320aic23_mclk), CODEC_CLOCK);
+
+	return 0;
+
+err3:
+	clk_put(tlv320aic23_mclk);
+err2:
+	platform_device_del(osk_snd_device);
+err1:
+	platform_device_put(osk_snd_device);
+
+	return err;
+
+}
+
+static void __exit osk_soc_exit(void)
+{
+	clk_put(tlv320aic23_mclk);
+	platform_device_unregister(osk_snd_device);
+}
+
+module_init(osk_soc_init);
+module_exit(osk_soc_exit);
+
+MODULE_AUTHOR("Arun KS <arunks@mistralsolutions.com>");
+MODULE_DESCRIPTION("ALSA SoC OSK 5912");
+MODULE_LICENSE("GPL");
diff -Naur --no-dereference a/sound/soc/ti/rx51.c b/sound/soc/ti/rx51.c
--- a/sound/soc/ti/rx51.c	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/rx51.c	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,493 @@
+/*
+ * rx51.c  --  SoC audio for Nokia RX-51
+ *
+ * Copyright (C) 2008 - 2009 Nokia Corporation
+ *
+ * Contact: Peter Ujfalusi <peter.ujfalusi@ti.com>
+ *          Eduardo Valentin <eduardo.valentin@nokia.com>
+ *          Jarkko Nikula <jarkko.nikula@bitmer.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA
+ *
+ */
+
+#include <linux/delay.h>
+#include <linux/gpio.h>
+#include <linux/platform_device.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <sound/core.h>
+#include <sound/jack.h>
+#include <sound/pcm.h>
+#include <sound/soc.h>
+#include <linux/platform_data/asoc-ti-mcbsp.h>
+
+#include <asm/mach-types.h>
+
+#include "omap-mcbsp.h"
+
+enum {
+	RX51_JACK_DISABLED,
+	RX51_JACK_TVOUT,		/* tv-out with stereo output */
+	RX51_JACK_HP,			/* headphone: stereo output, no mic */
+	RX51_JACK_HS,			/* headset: stereo output with mic */
+};
+
+struct rx51_audio_pdata {
+	struct gpio_desc *tvout_selection_gpio;
+	struct gpio_desc *jack_detection_gpio;
+	struct gpio_desc *eci_sw_gpio;
+	struct gpio_desc *speaker_amp_gpio;
+};
+
+static int rx51_spk_func;
+static int rx51_dmic_func;
+static int rx51_jack_func;
+
+static void rx51_ext_control(struct snd_soc_dapm_context *dapm)
+{
+	struct snd_soc_card *card = dapm->card;
+	struct rx51_audio_pdata *pdata = snd_soc_card_get_drvdata(card);
+	int hp = 0, hs = 0, tvout = 0;
+
+	switch (rx51_jack_func) {
+	case RX51_JACK_TVOUT:
+		tvout = 1;
+		hp = 1;
+		break;
+	case RX51_JACK_HS:
+		hs = 1;
+	case RX51_JACK_HP:
+		hp = 1;
+		break;
+	}
+
+	snd_soc_dapm_mutex_lock(dapm);
+
+	if (rx51_spk_func)
+		snd_soc_dapm_enable_pin_unlocked(dapm, "Ext Spk");
+	else
+		snd_soc_dapm_disable_pin_unlocked(dapm, "Ext Spk");
+	if (rx51_dmic_func)
+		snd_soc_dapm_enable_pin_unlocked(dapm, "DMic");
+	else
+		snd_soc_dapm_disable_pin_unlocked(dapm, "DMic");
+	if (hp)
+		snd_soc_dapm_enable_pin_unlocked(dapm, "Headphone Jack");
+	else
+		snd_soc_dapm_disable_pin_unlocked(dapm, "Headphone Jack");
+	if (hs)
+		snd_soc_dapm_enable_pin_unlocked(dapm, "HS Mic");
+	else
+		snd_soc_dapm_disable_pin_unlocked(dapm, "HS Mic");
+
+	gpiod_set_value(pdata->tvout_selection_gpio, tvout);
+
+	snd_soc_dapm_sync_unlocked(dapm);
+
+	snd_soc_dapm_mutex_unlock(dapm);
+}
+
+static int rx51_startup(struct snd_pcm_substream *substream)
+{
+	struct snd_pcm_runtime *runtime = substream->runtime;
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_card *card = rtd->card;
+
+	snd_pcm_hw_constraint_single(runtime, SNDRV_PCM_HW_PARAM_CHANNELS, 2);
+	rx51_ext_control(&card->dapm);
+
+	return 0;
+}
+
+static int rx51_hw_params(struct snd_pcm_substream *substream,
+	struct snd_pcm_hw_params *params)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_dai *codec_dai = rtd->codec_dai;
+
+	/* Set the codec system clock for DAC and ADC */
+	return snd_soc_dai_set_sysclk(codec_dai, 0, 19200000,
+				      SND_SOC_CLOCK_IN);
+}
+
+static const struct snd_soc_ops rx51_ops = {
+	.startup = rx51_startup,
+	.hw_params = rx51_hw_params,
+};
+
+static int rx51_get_spk(struct snd_kcontrol *kcontrol,
+			struct snd_ctl_elem_value *ucontrol)
+{
+	ucontrol->value.enumerated.item[0] = rx51_spk_func;
+
+	return 0;
+}
+
+static int rx51_set_spk(struct snd_kcontrol *kcontrol,
+			struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_card *card = snd_kcontrol_chip(kcontrol);
+
+	if (rx51_spk_func == ucontrol->value.enumerated.item[0])
+		return 0;
+
+	rx51_spk_func = ucontrol->value.enumerated.item[0];
+	rx51_ext_control(&card->dapm);
+
+	return 1;
+}
+
+static int rx51_spk_event(struct snd_soc_dapm_widget *w,
+			  struct snd_kcontrol *k, int event)
+{
+	struct snd_soc_dapm_context *dapm = w->dapm;
+	struct snd_soc_card *card = dapm->card;
+	struct rx51_audio_pdata *pdata = snd_soc_card_get_drvdata(card);
+
+	gpiod_set_raw_value_cansleep(pdata->speaker_amp_gpio,
+				     !!SND_SOC_DAPM_EVENT_ON(event));
+
+	return 0;
+}
+
+static int rx51_get_input(struct snd_kcontrol *kcontrol,
+			  struct snd_ctl_elem_value *ucontrol)
+{
+	ucontrol->value.enumerated.item[0] = rx51_dmic_func;
+
+	return 0;
+}
+
+static int rx51_set_input(struct snd_kcontrol *kcontrol,
+			  struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_card *card = snd_kcontrol_chip(kcontrol);
+
+	if (rx51_dmic_func == ucontrol->value.enumerated.item[0])
+		return 0;
+
+	rx51_dmic_func = ucontrol->value.enumerated.item[0];
+	rx51_ext_control(&card->dapm);
+
+	return 1;
+}
+
+static int rx51_get_jack(struct snd_kcontrol *kcontrol,
+			 struct snd_ctl_elem_value *ucontrol)
+{
+	ucontrol->value.enumerated.item[0] = rx51_jack_func;
+
+	return 0;
+}
+
+static int rx51_set_jack(struct snd_kcontrol *kcontrol,
+			 struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_card *card = snd_kcontrol_chip(kcontrol);
+
+	if (rx51_jack_func == ucontrol->value.enumerated.item[0])
+		return 0;
+
+	rx51_jack_func = ucontrol->value.enumerated.item[0];
+	rx51_ext_control(&card->dapm);
+
+	return 1;
+}
+
+static struct snd_soc_jack rx51_av_jack;
+
+static struct snd_soc_jack_gpio rx51_av_jack_gpios[] = {
+	{
+		.name = "avdet-gpio",
+		.report = SND_JACK_HEADSET,
+		.invert = 1,
+		.debounce_time = 200,
+	},
+};
+
+static const struct snd_soc_dapm_widget aic34_dapm_widgets[] = {
+	SND_SOC_DAPM_SPK("Ext Spk", rx51_spk_event),
+	SND_SOC_DAPM_MIC("DMic", NULL),
+	SND_SOC_DAPM_HP("Headphone Jack", NULL),
+	SND_SOC_DAPM_MIC("HS Mic", NULL),
+	SND_SOC_DAPM_LINE("FM Transmitter", NULL),
+	SND_SOC_DAPM_SPK("Earphone", NULL),
+};
+
+static const struct snd_soc_dapm_route audio_map[] = {
+	{"Ext Spk", NULL, "HPLOUT"},
+	{"Ext Spk", NULL, "HPROUT"},
+	{"Ext Spk", NULL, "HPLCOM"},
+	{"Ext Spk", NULL, "HPRCOM"},
+	{"FM Transmitter", NULL, "LLOUT"},
+	{"FM Transmitter", NULL, "RLOUT"},
+
+	{"Headphone Jack", NULL, "TPA6130A2 HPLEFT"},
+	{"Headphone Jack", NULL, "TPA6130A2 HPRIGHT"},
+	{"TPA6130A2 LEFTIN", NULL, "LLOUT"},
+	{"TPA6130A2 RIGHTIN", NULL, "RLOUT"},
+
+	{"DMic Rate 64", NULL, "DMic"},
+	{"DMic", NULL, "Mic Bias"},
+
+	{"b LINE2R", NULL, "MONO_LOUT"},
+	{"Earphone", NULL, "b HPLOUT"},
+
+	{"LINE1L", NULL, "HS Mic"},
+	{"HS Mic", NULL, "b Mic Bias"},
+};
+
+static const char * const spk_function[] = {"Off", "On"};
+static const char * const input_function[] = {"ADC", "Digital Mic"};
+static const char * const jack_function[] = {
+	"Off", "TV-OUT", "Headphone", "Headset"
+};
+
+static const struct soc_enum rx51_enum[] = {
+	SOC_ENUM_SINGLE_EXT(ARRAY_SIZE(spk_function), spk_function),
+	SOC_ENUM_SINGLE_EXT(ARRAY_SIZE(input_function), input_function),
+	SOC_ENUM_SINGLE_EXT(ARRAY_SIZE(jack_function), jack_function),
+};
+
+static const struct snd_kcontrol_new aic34_rx51_controls[] = {
+	SOC_ENUM_EXT("Speaker Function", rx51_enum[0],
+		     rx51_get_spk, rx51_set_spk),
+	SOC_ENUM_EXT("Input Select",  rx51_enum[1],
+		     rx51_get_input, rx51_set_input),
+	SOC_ENUM_EXT("Jack Function", rx51_enum[2],
+		     rx51_get_jack, rx51_set_jack),
+	SOC_DAPM_PIN_SWITCH("FM Transmitter"),
+	SOC_DAPM_PIN_SWITCH("Earphone"),
+};
+
+static int rx51_aic34_init(struct snd_soc_pcm_runtime *rtd)
+{
+	struct snd_soc_card *card = rtd->card;
+	struct rx51_audio_pdata *pdata = snd_soc_card_get_drvdata(card);
+	int err;
+
+	snd_soc_limit_volume(card, "TPA6130A2 Headphone Playback Volume", 42);
+
+	err = omap_mcbsp_st_add_controls(rtd, 2);
+	if (err < 0) {
+		dev_err(card->dev, "Failed to add MCBSP controls\n");
+		return err;
+	}
+
+	/* AV jack detection */
+	err = snd_soc_card_jack_new(rtd->card, "AV Jack",
+				    SND_JACK_HEADSET | SND_JACK_VIDEOOUT,
+				    &rx51_av_jack, NULL, 0);
+	if (err) {
+		dev_err(card->dev, "Failed to add AV Jack\n");
+		return err;
+	}
+
+	/* prepare gpio for snd_soc_jack_add_gpios */
+	rx51_av_jack_gpios[0].gpio = desc_to_gpio(pdata->jack_detection_gpio);
+	devm_gpiod_put(card->dev, pdata->jack_detection_gpio);
+
+	err = snd_soc_jack_add_gpios(&rx51_av_jack,
+				     ARRAY_SIZE(rx51_av_jack_gpios),
+				     rx51_av_jack_gpios);
+	if (err) {
+		dev_err(card->dev, "Failed to add GPIOs\n");
+		return err;
+	}
+
+	return err;
+}
+
+/* Digital audio interface glue - connects codec <--> CPU */
+static struct snd_soc_dai_link rx51_dai[] = {
+	{
+		.name = "TLV320AIC34",
+		.stream_name = "AIC34",
+		.cpu_dai_name = "omap-mcbsp.2",
+		.codec_dai_name = "tlv320aic3x-hifi",
+		.platform_name = "omap-mcbsp.2",
+		.codec_name = "tlv320aic3x-codec.2-0018",
+		.dai_fmt = SND_SOC_DAIFMT_DSP_A | SND_SOC_DAIFMT_IB_NF |
+			   SND_SOC_DAIFMT_CBM_CFM,
+		.init = rx51_aic34_init,
+		.ops = &rx51_ops,
+	},
+};
+
+static struct snd_soc_aux_dev rx51_aux_dev[] = {
+	{
+		.name = "TLV320AIC34b",
+		.codec_name = "tlv320aic3x-codec.2-0019",
+	},
+	{
+		.name = "TPA61320A2",
+		.codec_name = "tpa6130a2.2-0060",
+	},
+};
+
+static struct snd_soc_codec_conf rx51_codec_conf[] = {
+	{
+		.dev_name = "tlv320aic3x-codec.2-0019",
+		.name_prefix = "b",
+	},
+	{
+		.dev_name = "tpa6130a2.2-0060",
+		.name_prefix = "TPA6130A2",
+	},
+};
+
+/* Audio card */
+static struct snd_soc_card rx51_sound_card = {
+	.name = "RX-51",
+	.owner = THIS_MODULE,
+	.dai_link = rx51_dai,
+	.num_links = ARRAY_SIZE(rx51_dai),
+	.aux_dev = rx51_aux_dev,
+	.num_aux_devs = ARRAY_SIZE(rx51_aux_dev),
+	.codec_conf = rx51_codec_conf,
+	.num_configs = ARRAY_SIZE(rx51_codec_conf),
+	.fully_routed = true,
+
+	.controls = aic34_rx51_controls,
+	.num_controls = ARRAY_SIZE(aic34_rx51_controls),
+	.dapm_widgets = aic34_dapm_widgets,
+	.num_dapm_widgets = ARRAY_SIZE(aic34_dapm_widgets),
+	.dapm_routes = audio_map,
+	.num_dapm_routes = ARRAY_SIZE(audio_map),
+};
+
+static int rx51_soc_probe(struct platform_device *pdev)
+{
+	struct rx51_audio_pdata *pdata;
+	struct device_node *np = pdev->dev.of_node;
+	struct snd_soc_card *card = &rx51_sound_card;
+	int err;
+
+	if (!machine_is_nokia_rx51() && !of_machine_is_compatible("nokia,omap3-n900"))
+		return -ENODEV;
+
+	card->dev = &pdev->dev;
+
+	if (np) {
+		struct device_node *dai_node;
+
+		dai_node = of_parse_phandle(np, "nokia,cpu-dai", 0);
+		if (!dai_node) {
+			dev_err(&pdev->dev, "McBSP node is not provided\n");
+			return -EINVAL;
+		}
+		rx51_dai[0].cpu_dai_name = NULL;
+		rx51_dai[0].platform_name = NULL;
+		rx51_dai[0].cpu_of_node = dai_node;
+		rx51_dai[0].platform_of_node = dai_node;
+
+		dai_node = of_parse_phandle(np, "nokia,audio-codec", 0);
+		if (!dai_node) {
+			dev_err(&pdev->dev, "Codec node is not provided\n");
+			return -EINVAL;
+		}
+		rx51_dai[0].codec_name = NULL;
+		rx51_dai[0].codec_of_node = dai_node;
+
+		dai_node = of_parse_phandle(np, "nokia,audio-codec", 1);
+		if (!dai_node) {
+			dev_err(&pdev->dev, "Auxiliary Codec node is not provided\n");
+			return -EINVAL;
+		}
+		rx51_aux_dev[0].codec_name = NULL;
+		rx51_aux_dev[0].codec_of_node = dai_node;
+		rx51_codec_conf[0].dev_name = NULL;
+		rx51_codec_conf[0].of_node = dai_node;
+
+		dai_node = of_parse_phandle(np, "nokia,headphone-amplifier", 0);
+		if (!dai_node) {
+			dev_err(&pdev->dev, "Headphone amplifier node is not provided\n");
+			return -EINVAL;
+		}
+		rx51_aux_dev[1].codec_name = NULL;
+		rx51_aux_dev[1].codec_of_node = dai_node;
+		rx51_codec_conf[1].dev_name = NULL;
+		rx51_codec_conf[1].of_node = dai_node;
+	}
+
+	pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
+	if (pdata == NULL)
+		return -ENOMEM;
+
+	snd_soc_card_set_drvdata(card, pdata);
+
+	pdata->tvout_selection_gpio = devm_gpiod_get(card->dev,
+						     "tvout-selection",
+						     GPIOD_OUT_LOW);
+	if (IS_ERR(pdata->tvout_selection_gpio)) {
+		dev_err(card->dev, "could not get tvout selection gpio\n");
+		return PTR_ERR(pdata->tvout_selection_gpio);
+	}
+
+	pdata->jack_detection_gpio = devm_gpiod_get(card->dev,
+						    "jack-detection",
+						    GPIOD_ASIS);
+	if (IS_ERR(pdata->jack_detection_gpio)) {
+		dev_err(card->dev, "could not get jack detection gpio\n");
+		return PTR_ERR(pdata->jack_detection_gpio);
+	}
+
+	pdata->eci_sw_gpio = devm_gpiod_get(card->dev, "eci-switch",
+					    GPIOD_OUT_HIGH);
+	if (IS_ERR(pdata->eci_sw_gpio)) {
+		dev_err(card->dev, "could not get eci switch gpio\n");
+		return PTR_ERR(pdata->eci_sw_gpio);
+	}
+
+	pdata->speaker_amp_gpio = devm_gpiod_get(card->dev,
+						 "speaker-amplifier",
+						 GPIOD_OUT_LOW);
+	if (IS_ERR(pdata->speaker_amp_gpio)) {
+		dev_err(card->dev, "could not get speaker enable gpio\n");
+		return PTR_ERR(pdata->speaker_amp_gpio);
+	}
+
+	err = devm_snd_soc_register_card(card->dev, card);
+	if (err) {
+		dev_err(&pdev->dev, "snd_soc_register_card failed (%d)\n", err);
+		return err;
+	}
+
+	return 0;
+}
+
+#if defined(CONFIG_OF)
+static const struct of_device_id rx51_audio_of_match[] = {
+	{ .compatible = "nokia,n900-audio", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, rx51_audio_of_match);
+#endif
+
+static struct platform_driver rx51_soc_driver = {
+	.driver = {
+		.name = "rx51-audio",
+		.of_match_table = of_match_ptr(rx51_audio_of_match),
+	},
+	.probe = rx51_soc_probe,
+};
+
+module_platform_driver(rx51_soc_driver);
+
+MODULE_AUTHOR("Nokia Corporation");
+MODULE_DESCRIPTION("ALSA SoC Nokia RX-51");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:rx51-audio");
diff -Naur --no-dereference a/sound/soc/ti/sdma-pcm.c b/sound/soc/ti/sdma-pcm.c
--- a/sound/soc/ti/sdma-pcm.c	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/sdma-pcm.c	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,74 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
+ *  Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
+ */
+
+#include <linux/device.h>
+#include <linux/module.h>
+#include <sound/core.h>
+#include <sound/pcm.h>
+#include <sound/pcm_params.h>
+#include <sound/soc.h>
+#include <sound/dmaengine_pcm.h>
+#include <linux/omap-dmaengine.h>
+
+#include "sdma-pcm.h"
+
+static const struct snd_pcm_hardware sdma_pcm_hardware = {
+	.info			= SNDRV_PCM_INFO_MMAP |
+				  SNDRV_PCM_INFO_MMAP_VALID |
+				  SNDRV_PCM_INFO_PAUSE | SNDRV_PCM_INFO_RESUME |
+				  SNDRV_PCM_INFO_NO_PERIOD_WAKEUP |
+				  SNDRV_PCM_INFO_INTERLEAVED,
+	.period_bytes_min	= 32,
+	.period_bytes_max	= 64 * 1024,
+	.buffer_bytes_max	= 128 * 1024,
+	.periods_min		= 2,
+	.periods_max		= 255,
+};
+
+static const struct snd_dmaengine_pcm_config sdma_dmaengine_pcm_config = {
+	.pcm_hardware = &sdma_pcm_hardware,
+	.prepare_slave_config = snd_dmaengine_pcm_prepare_slave_config,
+	.compat_filter_fn = omap_dma_filter_fn,
+	.prealloc_buffer_size = 128 * 1024,
+};
+
+int sdma_pcm_platform_register(struct device *dev,
+			       char *txdmachan, char *rxdmachan)
+{
+	struct snd_dmaengine_pcm_config *config;
+	unsigned int flags = SND_DMAENGINE_PCM_FLAG_COMPAT;
+
+	/* Standard names for the directions: 'tx' and 'rx' */
+	if (!txdmachan && !rxdmachan)
+		return devm_snd_dmaengine_pcm_register(dev,
+						&sdma_dmaengine_pcm_config,
+						flags);
+
+	config = devm_kzalloc(dev, sizeof(*config), GFP_KERNEL);
+	if (!config)
+		return -ENOMEM;
+
+	*config = sdma_dmaengine_pcm_config;
+
+	if (!txdmachan || !rxdmachan) {
+		/* One direction only PCM */
+		flags |= SND_DMAENGINE_PCM_FLAG_HALF_DUPLEX;
+		if (!txdmachan) {
+			txdmachan = rxdmachan;
+			rxdmachan = NULL;
+		}
+	}
+
+	config->chan_names[0] = txdmachan;
+	config->chan_names[1] = rxdmachan;
+
+	return devm_snd_dmaengine_pcm_register(dev, config, flags);
+}
+EXPORT_SYMBOL_GPL(sdma_pcm_platform_register);
+
+MODULE_AUTHOR("Peter Ujfalusi <peter.ujfalusi@ti.com>");
+MODULE_DESCRIPTION("sDMA PCM ASoC platform driver");
+MODULE_LICENSE("GPL v2");
diff -Naur --no-dereference a/sound/soc/ti/sdma-pcm.h b/sound/soc/ti/sdma-pcm.h
--- a/sound/soc/ti/sdma-pcm.h	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/sdma-pcm.h	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ *  Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
+ *  Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
+ */
+
+#ifndef __SDMA_PCM_H__
+#define __SDMA_PCM_H__
+
+#if IS_ENABLED(CONFIG_SND_SOC_TI_SDMA_PCM)
+int sdma_pcm_platform_register(struct device *dev,
+			       char *txdmachan, char *rxdmachan);
+#else
+static inline int sdma_pcm_platform_register(struct device *dev,
+					     char *txdmachan, char *rxdmachan)
+{
+	return -ENODEV;
+}
+#endif /* CONFIG_SND_SOC_TI_SDMA_PCM */
+
+#endif /* __SDMA_PCM_H__ */
diff -Naur --no-dereference a/sound/soc/ti/udma-pcm.c b/sound/soc/ti/udma-pcm.c
--- a/sound/soc/ti/udma-pcm.c	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/udma-pcm.c	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,43 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
+ *  Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
+ */
+
+#include <linux/module.h>
+#include <sound/core.h>
+#include <sound/pcm.h>
+#include <sound/pcm_params.h>
+#include <sound/soc.h>
+#include <sound/dmaengine_pcm.h>
+
+#include "udma-pcm.h"
+
+static const struct snd_pcm_hardware udma_pcm_hardware = {
+	.info			= SNDRV_PCM_INFO_MMAP |
+				  SNDRV_PCM_INFO_MMAP_VALID |
+				  SNDRV_PCM_INFO_PAUSE | SNDRV_PCM_INFO_RESUME |
+				  SNDRV_PCM_INFO_NO_PERIOD_WAKEUP |
+				  SNDRV_PCM_INFO_INTERLEAVED,
+	.buffer_bytes_max	= SIZE_MAX,
+	.period_bytes_min	= 32,
+	.period_bytes_max	= SZ_64K,
+	.periods_min		= 2,
+	.periods_max		= UINT_MAX,
+};
+
+static const struct snd_dmaengine_pcm_config udma_dmaengine_pcm_config = {
+	.pcm_hardware = &udma_pcm_hardware,
+	.prepare_slave_config = snd_dmaengine_pcm_prepare_slave_config,
+};
+
+int udma_pcm_platform_register(struct device *dev)
+{
+	return devm_snd_dmaengine_pcm_register(dev, &udma_dmaengine_pcm_config,
+					       0);
+}
+EXPORT_SYMBOL_GPL(udma_pcm_platform_register);
+
+MODULE_AUTHOR("Peter Ujfalusi <peter.ujfalusi@ti.com>");
+MODULE_DESCRIPTION("UDMA PCM ASoC platform driver");
+MODULE_LICENSE("GPL");
diff -Naur --no-dereference a/sound/soc/ti/udma-pcm.h b/sound/soc/ti/udma-pcm.h
--- a/sound/soc/ti/udma-pcm.h	1969-12-31 19:00:00.000000000 -0500
+++ b/sound/soc/ti/udma-pcm.h	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ *  Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
+ */
+
+#ifndef __UDMA_PCM_H__
+#define __UDMA_PCM_H__
+
+#if IS_ENABLED(CONFIG_SND_SOC_TI_UDMA_PCM)
+int udma_pcm_platform_register(struct device *dev);
+#else
+static inline int udma_pcm_platform_register(struct device *dev)
+{
+	return 0;
+}
+#endif /* CONFIG_SND_SOC_TI_UDMA_PCM */
+
+#endif /* __UDMA_PCM_H__ */
diff -Naur --no-dereference a/ti_config_fragments/am33xx_only.cfg b/ti_config_fragments/am33xx_only.cfg
--- a/ti_config_fragments/am33xx_only.cfg	1969-12-31 19:00:00.000000000 -0500
+++ b/ti_config_fragments/am33xx_only.cfg	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,19 @@
+##################################################
+# TI AM33XX specific config options
+##################################################
+
+# Disable Socs other than AM33xx
+CONFIG_ARCH_OMAP2=n
+CONFIG_ARCH_OMAP3=n
+CONFIG_ARCH_OMAP4=n
+CONFIG_SOC_OMAP5=n
+CONFIG_SOC_AM43XX=n
+CONFIG_SOC_DRA7XX=n
+CONFIG_ARCH_KEYSTONE=n
+
+#Disable CONFIG_SMP
+CONFIG_SMP=n
+CONFIG_CPUSETS=n
+
+# Increase CMA for HDMI
+CONFIG_CMA_SIZE_MBYTES=48
diff -Naur --no-dereference a/ti_config_fragments/am43xx_only.cfg b/ti_config_fragments/am43xx_only.cfg
--- a/ti_config_fragments/am43xx_only.cfg	1969-12-31 19:00:00.000000000 -0500
+++ b/ti_config_fragments/am43xx_only.cfg	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,19 @@
+##################################################
+# TI AM43XX specific config options
+##################################################
+
+# Disable Socs other than AM43xx
+CONFIG_ARCH_OMAP2=n
+CONFIG_ARCH_OMAP3=n
+CONFIG_ARCH_OMAP4=n
+CONFIG_SOC_OMAP5=n
+CONFIG_SOC_AM33XX=n
+CONFIG_SOC_DRA7XX=n
+CONFIG_ARCH_KEYSTONE=n
+
+#Disable CONFIG_SMP
+CONFIG_SMP=n
+CONFIG_CPUSETS=n
+
+# Increase CMA for HDMI
+CONFIG_CMA_SIZE_MBYTES=48
diff -Naur --no-dereference a/ti_config_fragments/arm64_prune.cfg b/ti_config_fragments/arm64_prune.cfg
--- a/ti_config_fragments/arm64_prune.cfg	1969-12-31 19:00:00.000000000 -0500
+++ b/ti_config_fragments/arm64_prune.cfg	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,78 @@
+CONFIG_ARCH_SUNXI=n
+CONFIG_ARCH_ALPINE=n
+CONFIG_ARCH_BCM2835=n
+CONFIG_ARCH_BCM_IPROC=n
+CONFIG_ARCH_BERLIN=n
+CONFIG_ARCH_BRCMSTB=n
+CONFIG_ARCH_EXYNOS=n
+CONFIG_ARCH_LAYERSCAPE=n
+CONFIG_ARCH_LG1K=n
+CONFIG_ARCH_HISI=n
+CONFIG_ARCH_MEDIATEK=n
+CONFIG_ARCH_MESON=n
+CONFIG_ARCH_MVEBU=n
+CONFIG_ARCH_QCOM=n
+CONFIG_ARCH_ROCKCHIP=n
+CONFIG_ARCH_SEATTLE=n
+CONFIG_ARCH_RENESAS=n
+CONFIG_ARCH_R8A7795=n
+CONFIG_ARCH_R8A7796=n
+CONFIG_ARCH_STRATIX10=n
+CONFIG_ARCH_SYNQUACER=n
+CONFIG_ARCH_TEGRA=n
+CONFIG_ARCH_SPRD=n
+CONFIG_ARCH_THUNDER=n
+CONFIG_ARCH_THUNDER2=n
+CONFIG_ARCH_UNIPHIER=n
+CONFIG_ARCH_VEXPRESS=n
+CONFIG_ARCH_XGENE=n
+CONFIG_ARCH_ZX=n
+CONFIG_ARCH_ZYNQMP=n
+CONFIG_ARCH_TEGRA_132_SOC=n
+CONFIG_ARCH_TEGRA_210_SOC=n
+CONFIG_ARCH_TEGRA_186_SOC=n
+
+CONFIG_QCOM_FALKOR_ERRATUM_1003=n
+CONFIG_QCOM_FALKOR_ERRATUM_1009=n
+CONFIG_QCOM_QDF2400_ERRATUM_0065=n
+CONFIG_QCOM_FALKOR_ERRATUM_E1041=n
+
+CONFIG_CAVIUM_ERRATUM_22375=n
+CONFIG_CAVIUM_ERRATUM_23144=n
+CONFIG_CAVIUM_ERRATUM_23154=n
+CONFIG_CAVIUM_ERRATUM_27456=n
+CONFIG_CAVIUM_ERRATUM_30115=n
+
+CONFIG_HISILICON_ERRATUM_161600802=n
+
+CONFIG_SERIAL_8250_FSL=n
+CONFIG_SERIAL_8250_DW=n
+CONFIG_SERIAL_AMBA_PL011=n
+CONFIG_SERIAL_AMBA_PL011_CONSOLE=n
+CONFIG_SERIAL_XILINX_PS_UART=n
+CONFIG_SERIAL_XILINX_PS_UART_CONSOLE=n
+
+CONFIG_ARM_SCPI_PROTOCOL=n
+CONFIG_ARM_SCPI_POWER_DOMAIN=n
+CONFIG_ARM_SCPI_CPUFREQ=n
+CONFIG_SENSORS_ARM_SCPI=n
+CONFIG_COMMON_CLK_SCPI=n
+
+CONFIG_ARM_MHU=n
+CONFIG_PLATFORM_MHU=n
+
+CONFIG_ACPI=n
+CONFIG_EFI=n
+
+CONFIG_NUMA=n
+
+CONFIG_XEN=n
+CONFIG_KVM=n
+
+CONFIG_ARM_BIG_LITTLE_CPUFREQ=n
+
+CONFIG_PCI_XGENE=n
+CONFIG_PCI_HISI=n
+CONFIG_PCIE_KIRIN=n
+
+CONFIG_DEBUG_INFO=n
diff -Naur --no-dereference a/ti_config_fragments/audio_display.cfg b/ti_config_fragments/audio_display.cfg
--- a/ti_config_fragments/audio_display.cfg	1969-12-31 19:00:00.000000000 -0500
+++ b/ti_config_fragments/audio_display.cfg	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,165 @@
+##################################################
+# TI Audio/Display config options
+##################################################
+
+CONFIG_CMA=y
+CONFIG_DMA_CMA=y
+CONFIG_CMA_SIZE_MBYTES=24
+
+# backlight
+
+CONFIG_LEDS_CLASS=y
+CONFIG_LEDS_TLC591XX=y
+
+CONFIG_BACKLIGHT_PWM=y
+CONFIG_BACKLIGHT_GPIO=y
+CONFIG_BACKLIGHT_LED=y
+
+# fbdev
+
+CONFIG_FB_SSD1307=y
+CONFIG_FRAMEBUFFER_CONSOLE=y
+
+# drm
+
+CONFIG_DRM=y
+CONFIG_DRM_SII902X=y
+CONFIG_DRM_TI_TFP410=y
+CONFIG_DRM_TOSHIBA_TC358767=y
+CONFIG_DRM_TOSHIBA_TC358768=y
+CONFIG_DRM_PANEL_SIMPLE=y
+CONFIG_DRM_PANEL_OSD_OSD101T2587_53TS=y
+# Firmware loading only works if built as module
+CONFIG_DRM_CDNS_MHDP=m
+CONFIG_DRM_CDNS_DSI=n
+
+CONFIG_PHY_J721E_WIZ=y
+CONFIG_PHY_CADENCE_TORRENT=y
+
+# SGX driver needs legacy support
+CONFIG_DRM_LEGACY=y
+CONFIG_DRM_VM=y
+
+# tilcdc
+
+CONFIG_DRM_I2C_NXP_TDA998X=y
+CONFIG_DRM_TILCDC=y
+
+# omapdrm
+
+CONFIG_DRM_OMAP=y
+CONFIG_DRM_OMAP_WB=y
+CONFIG_OMAP2_DSS=y
+CONFIG_OMAP2_DSS_DEBUGFS=y
+CONFIG_OMAP2_DSS_DPI=y
+CONFIG_OMAP2_DSS_VENC=n
+CONFIG_OMAP4_DSS_HDMI=y
+CONFIG_OMAP5_DSS_HDMI=y
+CONFIG_OMAP2_DSS_SDI=n
+CONFIG_OMAP2_DSS_DSI=n
+
+CONFIG_DRM_OMAP_CONNECTOR_HDMI=y
+CONFIG_DRM_OMAP_ENCODER_TPD12S015=y
+CONFIG_DRM_OMAP_DRA7EVM_ENCODER_TPD12S015=y
+CONFIG_DRM_OMAP_ENCODER_TFP410=y
+CONFIG_DRM_OMAP_CONNECTOR_DVI=y
+CONFIG_DRM_OMAP_PANEL_DPI=y
+
+# tidss
+CONFIG_DRM_TIDSS=y
+CONFIG_DRM_TIDSS_WB=y
+CONFIG_DRM_TIDSS_DSS6=y
+CONFIG_DRM_TIDSS_DSS7=y
+
+# Disable unneeded features
+
+CONFIG_VGA_ARB=n
+CONFIG_DRM_I2C_ADV7511=n
+CONFIG_DRM_NOUVEAU=n
+CONFIG_DRM_EXYNOS=n
+CONFIG_DRM_PANEL_SAMSUNG_S6E8AA0=n
+CONFIG_DRM_NXP_PTN3460=n
+CONFIG_DRM_PARADE_PS8622=n
+CONFIG_DRM_STI=n
+CONFIG_DRM_PANEL_SAMSUNG_LD9040=n
+CONFIG_DRM_PANEL_SAMSUNG_S6E63J0X03=n
+CONFIG_DRM_DUMB_VGA_DAC=n
+CONFIG_LCD_CLASS_DEVICE=n
+CONFIG_FB_EFI=n
+CONFIG_FB_SIMPLE=n
+CONFIG_FRAMEBUFFER_CONSOLE_ROTATION=n
+CONFIG_DRM_HISI_KIRIN=n
+CONFIG_FB_ARMCLCD=n
+CONFIG_XEN_FBDEV_FRONTEND=n
+CONFIG_DRM_RCAR_LVDS=n
+CONFIG_DRM_FSL_DCU=n
+CONFIG_DRM_SII9234=n
+CONFIG_DRM_MXSFB=n
+CONFIG_DRM_HISI_HIBMC=n
+
+# Touchscreen
+
+CONFIG_INPUT_TOUCHSCREEN=y
+CONFIG_TOUCHSCREEN_EDT_FT5X06=m
+CONFIG_TOUCHSCREEN_PIXCIR=m
+CONFIG_TOUCHSCREEN_GOODIX=m
+
+CONFIG_HID_MULTITOUCH=m
+
+# V4L2
+
+CONFIG_MEDIA_SUPPORT=y
+CONFIG_MEDIA_CAMERA_SUPPORT=y
+CONFIG_MEDIA_CONTROLLER=y
+CONFIG_VIDEO_V4L2_SUBDEV_API=y
+
+CONFIG_V4L_PLATFORM_DRIVERS=y
+CONFIG_VIDEO_AM437X_VPFE=m
+CONFIG_VIDEO_TI_CAL=m
+CONFIG_VIDEO_TI_VIP=m
+
+CONFIG_V4L_MEM2MEM_DRIVERS=y
+CONFIG_VIDEO_TI_VPE=m
+
+CONFIG_MEDIA_SUBDRV_AUTOSELECT=n
+CONFIG_VIDEO_OV2659=m
+CONFIG_VIDEO_OV1063X=m
+CONFIG_VIDEO_MT9T11X=m
+CONFIG_VIDEO_OV490=m
+CONFIG_VIDEO_OV5640=m
+
+CONFIG_MEDIA_USB_SUPPORT=y
+CONFIG_USB_VIDEO_CLASS=m
+CONFIG_USB_VIDEO_CLASS_INPUT_EVDEV=y
+
+# sound - START
+CONFIG_SND_ARM=n
+CONFIG_SND_SPI=n
+CONFIG_SND_ATMEL_SOC=n
+CONFIG_SND_SOC_FSL_SAI=n
+CONFIG_SND_SOC_AK4642=n
+CONFIG_SND_SOC_CPCAP=n
+CONFIG_SND_SOC_SGTL5000=n
+CONFIG_SND_SOC_STI_SAS=n
+CONFIG_SND_SOC_WM8978=n
+
+CONFIG_SOUND=y
+CONFIG_SND=y
+CONFIG_SND_SOC=y
+CONFIG_SND_SIMPLE_CARD=m
+CONFIG_SND_SIMPLE_SCU_CARD=m
+CONFIG_SND_AUDIO_GRAPH_CARD=m
+CONFIG_SND_AUDIO_GRAPH_SCU_CARD=m
+
+CONFIG_SND_SOC_DAVINCI_MCASP=y
+CONFIG_SND_SOC_OMAP_HDMI=m
+CONFIG_SND_SOC_J721E_EVM=m
+
+CONFIG_SND_SOC_TLV320AIC3X=m
+CONFIG_SND_SOC_TLV320AIC31XX=m
+CONFIG_SND_SOC_PCM3168A_I2C=m
+CONFIG_SND_SOC_HDMI_CODEC=m
+# sound - END
+
+# display sharing
+CONFIG_RPMSG_KDRV_DISPLAY=y
diff -Naur --no-dereference a/ti_config_fragments/baseport.cfg b/ti_config_fragments/baseport.cfg
--- a/ti_config_fragments/baseport.cfg	1969-12-31 19:00:00.000000000 -0500
+++ b/ti_config_fragments/baseport.cfg	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,593 @@
+##################################################
+# TI Baseport Config Options
+##################################################
+
+# Kernel compression
+CONFIG_KERNEL_GZIP=n
+CONFIG_KERNEL_LZMA=y
+CONFIG_KERNEL_LZ4=n
+CONFIG_KERNEL_XZ=n
+CONFIG_KERNEL_LZO=n
+
+# Enable process accounting
+CONFIG_BSD_PROCESS_ACCT=y
+
+# Have some way to pick up kernel config later on
+# Always useful to look at /proc/config.gz
+CONFIG_IKCONFIG=y
+CONFIG_IKCONFIG_PROC=y
+
+# Add Bin2c
+CONFIG_BUILD_BIN2C=y
+
+# Add base Cgroups functions
+CONFIG_CGROUPS=y
+CONFIG_CGROUP_DEBUG=n
+CONFIG_CGROUP_FREEZER=y
+CONFIG_CGROUP_PIDS=y
+CONFIG_CGROUP_DEVICE=y
+CONFIG_CPUSETS=y
+CONFIG_PROC_PID_CPUSET=y
+CONFIG_CGROUP_CPUACCT=y
+CONFIG_PAGE_COUNTER=y
+CONFIG_MEMCG=y
+CONFIG_MEMCG_SWAP=y
+CONFIG_MEMCG_SWAP_ENABLED=y
+CONFIG_CGROUP_PERF=y
+CONFIG_CGROUP_SCHED=y
+CONFIG_FAIR_GROUP_SCHED=y
+CONFIG_CFS_BANDWIDTH=y
+CONFIG_RT_GROUP_SCHED=y
+CONFIG_BLK_CGROUP=y
+CONFIG_DEBUG_BLK_CGROUP=n
+CONFIG_CGROUP_WRITEBACK=y
+
+# Choose CONFIG_EMBEDDED
+CONFIG_NAMESPACES=n
+CONFIG_EMBEDDED=y
+CONFIG_EXPERT=y
+CONFIG_ARM_PATCH_PHYS_VIRT=y
+
+# Enable all kernel symbols please
+CONFIG_KALLSYMS=y
+CONFIG_KALLSYMS_ALL=y
+
+# Enable AEABI
+CONFIG_AEABI=y
+
+# How do we want kernel Modules to work?
+CONFIG_BASE_FULL=y
+CONFIG_BASE_SMALL=0
+CONFIG_MODULES=y
+CONFIG_MODULE_FORCE_LOAD=y
+CONFIG_MODULE_UNLOAD=y
+CONFIG_MODULE_FORCE_UNLOAD=y
+CONFIG_MODVERSIONS=y
+CONFIG_MODULE_SRCVERSION_ALL=y
+CONFIG_MODULE_SIG=n
+CONFIG_MODULE_COMPRESS=n
+CONFIG_MODULES_TREE_LOOKUP=y
+
+# Boot options
+CONFIG_USE_OF=y
+CONFIG_ATAGS=y
+CONFIG_DEPRECATED_PARAM_STRUCT=n
+CONFIG_ZBOOT_ROM_TEXT=0x0
+CONFIG_ZBOOT_ROM_BSS=0x0
+CONFIG_ARM_APPENDED_DTB=y
+CONFIG_ARM_ATAG_DTB_COMPAT=y
+CONFIG_ARM_ATAG_DTB_COMPAT_CMDLINE_FROM_BOOTLOADER=y
+CONFIG_ARM_ATAG_DTB_COMPAT_CMDLINE_EXTEND=n
+CONFIG_CMDLINE=""
+CONFIG_CMDLINE_FROM_BOOTLOADER=y
+CONFIG_CMDLINE_EXTEND=n
+CONFIG_CMDLINE_FORCE=n
+CONFIG_ATAGS_PROC=y
+CONFIG_CRASH_DUMP=n
+CONFIG_AUTO_ZRELADDR=y
+
+# Mem allocator
+CONFIG_SLAB=n
+CONFIG_SLUB=y
+
+# Enable NEON
+CONFIG_VFP=y
+CONFIG_NEON=y
+CONFIG_KERNEL_MODE_NEON=y
+
+# POSIX Message queue
+CONFIG_POSIX_MQUEUE=y
+
+# No Multi Cluster systems in TI yet..
+CONFIG_MCPM=n
+
+# Serial
+CONFIG_SERIAL_8250_OMAP=y
+CONFIG_SERIAL_8250_OMAP_TTYO_FIXUP=y
+CONFIG_SERIAL_8250_NR_UARTS=10
+CONFIG_SERIAL_8250_RUNTIME_UARTS=10
+CONFIG_SERIAL_8250_DMA=y
+CONFIG_SERIAL_OMAP=n
+CONFIG_SERIAL_8250_DW=n
+CONFIG_SERIAL_8250_EM=n
+CONFIG_SERIAL_AMBA_PL011=n
+CONFIG_SERIAL_XILINX_PS_UART=n
+CONFIG_SERIAL_FSL_LPUART=n
+CONFIG_SERIAL_CONEXANT_DIGICOLOR=n
+CONFIG_SERIAL_ST_ASC=n
+
+CONFIG_JUMP_LABEL=y
+
+# Disable Extra debug options
+CONFIG_SCHEDSTATS=n
+CONFIG_TIMER_STATS=n
+CONFIG_DEBUG_SPINLOCK=n
+CONFIG_DEBUG_MUTEXES=n
+CONFIG_DEBUG_LOCK_ALLOC=n
+CONFIG_PROVE_LOCKING=n
+CONFIG_LOCKDEP=n
+CONFIG_STACKTRACE=n
+CONFIG_SCHED_DEBUG=n
+CONFIG_FTRACE=n
+CONFIG_ARM_UNWIND=n
+CONFIG_DEBUG_PREEMPT=n
+CONFIG_SLUB_DEBUG=n
+CONFIG_LOCKUP_DETECTOR=n
+CONFIG_DETECT_HUNG_TASK=n
+CONFIG_IRQ_DOMAIN_DEBUG=n
+CONFIG_DEBUG_BUGVERBOSE=n
+
+CONFIG_PREEMPT=y
+CONFIG_DEBUG_FS=y
+
+# Enable System V IPC
+CONFIG_SYSVIPC=y
+
+# Power management options
+CONFIG_SUSPEND=y
+CONFIG_SUSPEND_FREEZER=y
+CONFIG_SUSPEND_SKIP_SYNC=n
+CONFIG_HIBERNATION=n
+CONFIG_PM_SLEEP=y
+CONFIG_PM_SLEEP_SMP=y
+CONFIG_PM_AUTOSLEEP=n
+CONFIG_PM_WAKELOCKS=n
+CONFIG_PM=y
+CONFIG_PM_DEBUG=y
+CONFIG_APM_EMULATION=n
+CONFIG_PM_OPP=y
+CONFIG_PM_CLK=y
+CONFIG_PM_GENERIC_DOMAINS=y
+CONFIG_WQ_POWER_EFFICIENT_DEFAULT=n
+CONFIG_PM_GENERIC_DOMAINS_SLEEP=y
+CONFIG_PM_GENERIC_DOMAINS_OF=y
+CONFIG_CPU_PM=y
+CONFIG_ARCH_SUSPEND_POSSIBLE=y
+CONFIG_ARM_CPU_SUSPEND=y
+CONFIG_ARCH_HIBERNATION_POSSIBLE=y
+
+# Clock framework stuff we need
+CONFIG_CLKDEV_LOOKUP=y
+CONFIG_HAVE_CLK_PREPARE=y
+CONFIG_COMMON_CLK=y
+CONFIG_COMMON_CLK_SI5351=n
+CONFIG_COMMON_CLK_SI514=n
+CONFIG_COMMON_CLK_SI570=n
+CONFIG_COMMON_CLK_CDCE925=n
+CONFIG_CLK_TWL6040=n
+CONFIG_CLK_QORIQ=n
+CONFIG_COMMON_CLK_PALMAS=y
+CONFIG_COMMON_CLK_PWM=n
+CONFIG_COMMON_CLK_PXA=n
+CONFIG_COMMON_CLK_CDCE706=n
+CONFIG_CLKSRC_OF=y
+CONFIG_CLKSRC_PROBE=y
+CONFIG_CLKSRC_MMIO=y
+CONFIG_CLKSRC_TI_32K=y
+CONFIG_ARM_ARCH_TIMER=y
+CONFIG_ARM_ARCH_TIMER_EVTSTREAM=y
+CONFIG_ARM_GLOBAL_TIMER=n
+CONFIG_ARM_TIMER_SP804=n
+CONFIG_TI_SYSCON_CLK=y
+# Clock cleanup please
+CONFIG_OMAP_RESET_CLOCKS=y
+
+# CPU Idle
+CONFIG_CPU_IDLE=y
+CONFIG_CPU_IDLE_GOV_LADDER=y
+CONFIG_CPU_IDLE_GOV_MENU=y
+CONFIG_DT_IDLE_STATES=y
+
+# ARM CPU Idle Drivers
+CONFIG_ARM_CPUIDLE=y
+CONFIG_ARCH_NEEDS_CPU_IDLE_COUPLED=n
+
+# CPU Frequency scaling
+CONFIG_CPU_FREQ=y
+CONFIG_CPU_FREQ_GOV_COMMON=y
+CONFIG_CPU_FREQ_STAT=y
+CONFIG_CPU_FREQ_STAT_DETAILS=y
+CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND=y
+CONFIG_CPU_FREQ_GOV_PERFORMANCE=y
+CONFIG_CPU_FREQ_GOV_POWERSAVE=y
+CONFIG_CPU_FREQ_GOV_USERSPACE=y
+CONFIG_CPU_FREQ_GOV_ONDEMAND=y
+CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y
+
+# CPUFreq Driver Options
+CONFIG_CPUFREQ_DT=y
+CONFIG_ARM_BIG_LITTLE_CPUFREQ=n
+CONFIG_ARM_KIRKWOOD_CPUFREQ=n
+CONFIG_ARM_OMAP2PLUS_CPUFREQ=n
+CONFIG_ARM_TI_CPUFREQ=y
+CONFIG_QORIQ_CPUFREQ=n
+
+# AMx3 Power Config Options
+CONFIG_MAILBOX=y
+CONFIG_OMAP2PLUS_MBOX=y
+CONFIG_WKUP_M3_RPROC=m
+CONFIG_SOC_TI=y
+CONFIG_SOC_BRCMSTB=n
+CONFIG_WKUP_M3_IPC=m
+CONFIG_AMX3_PM=m
+CONFIG_SRAM=y
+CONFIG_TI_EMIF_SRAM=m
+CONFIG_SUNXI_SRAM=n
+
+CONFIG_PM_DEVFREQ=n
+
+# K2G Power config options
+# CONFIG_MAILBOX is enabled in AMx3
+CONFIG_TI_MESSAGE_MANAGER=y
+CONFIG_TI_SCI_PROTOCOL=y
+CONFIG_TI_SCI_PM_DOMAINS=y
+CONFIG_TI_SCI_CLK=y
+
+# Enable Reset Controllers
+CONFIG_RESET_CONTROLLER=y
+CONFIG_RESET_TI_SYSCON=y
+CONFIG_RESET_TI_SCI=y
+
+# CPUFreq and OPP drivers
+CONFIG_OPP_DOMAIN_TI=y
+
+# Thermal
+CONFIG_THERMAL=y
+CONFIG_THERMAL_OF=y
+CONFIG_THERMAL_WRITABLE_TRIPS=y
+CONFIG_THERMAL_DEFAULT_GOV_STEP_WISE=y
+CONFIG_THERMAL_GOV_FAIR_SHARE=y
+CONFIG_THERMAL_GOV_STEP_WISE=y
+CONFIG_THERMAL_GOV_BANG_BANG=y
+CONFIG_THERMAL_GOV_USER_SPACE=y
+CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y
+CONFIG_CPU_THERMAL=y
+CONFIG_TI_SOC_THERMAL=y
+CONFIG_TI_THERMAL=y
+CONFIG_OMAP4_THERMAL=y
+CONFIG_OMAP5_THERMAL=y
+CONFIG_DRA752_THERMAL=y
+
+# Since HWMON is needed by Sensors..
+CONFIG_HWMON=y
+CONFIG_SENSORS_TMP102=y
+CONFIG_SENSORS_GPIO_FAN=y
+
+# Enable the reset framework
+CONFIG_POWER_RESET=y
+CONFIG_POWER_SUPPLY=y
+
+# Pinctrl
+CONFIG_PINCTRL_TI_IODELAY=y
+CONFIG_PINCTRL_SINGLE=y
+
+#
+# Multifunction device drivers
+CONFIG_MFD_CORE=y
+CONFIG_MFD_AS3711=n
+CONFIG_MFD_AS3722=n
+CONFIG_PMIC_ADP5520=n
+CONFIG_MFD_AAT2870_CORE=n
+CONFIG_MFD_ATMEL_FLEXCOM=n
+CONFIG_MFD_ATMEL_HLCDC=n
+CONFIG_MFD_BCM590XX=n
+CONFIG_MFD_AXP20X=n
+CONFIG_MFD_AXP20X_I2C=n
+CONFIG_MFD_CROS_EC=n
+CONFIG_MFD_CROS_EC_I2C=n
+CONFIG_MFD_CROS_EC_SPI=n
+CONFIG_MFD_ASIC3=n
+CONFIG_PMIC_DA903X=n
+CONFIG_MFD_DA9052_SPI=n
+CONFIG_MFD_DA9052_I2C=n
+CONFIG_MFD_DA9055=n
+CONFIG_MFD_DA9062=n
+CONFIG_MFD_DA9063=n
+CONFIG_MFD_DA9150=n
+CONFIG_MFD_DLN2=n
+CONFIG_MFD_MC13XXX_SPI=n
+CONFIG_MFD_MC13XXX_I2C=n
+CONFIG_MFD_HI6421_PMIC=n
+CONFIG_HTC_EGPIO=n
+CONFIG_HTC_PASIC3=n
+CONFIG_HTC_I2CPLD=n
+CONFIG_LPC_ICH=n
+CONFIG_LPC_SCH=n
+CONFIG_INTEL_SOC_PMIC=n
+CONFIG_MFD_JANZ_CMODIO=n
+CONFIG_MFD_KEMPLD=n
+CONFIG_MFD_88PM800=n
+CONFIG_MFD_88PM805=n
+CONFIG_MFD_88PM860X=n
+CONFIG_MFD_MAX14577=n
+CONFIG_MFD_MAX77686=n
+CONFIG_MFD_MAX77693=n
+CONFIG_MFD_MAX77843=n
+CONFIG_MFD_MAX8907=n
+CONFIG_MFD_MAX8925=n
+CONFIG_MFD_MAX8997=n
+CONFIG_MFD_MAX8998=n
+CONFIG_MFD_MT6397=n
+CONFIG_MFD_MENF21BMC=n
+CONFIG_EZX_PCAP=n
+CONFIG_MFD_VIPERBOARD=n
+CONFIG_MFD_RETU=n
+CONFIG_MFD_PCF50633=n
+CONFIG_MFD_PM8XXX=n
+CONFIG_MFD_PM8921_CORE=n
+CONFIG_MFD_RDC321X=n
+CONFIG_MFD_RTSX_PCI=n
+CONFIG_MFD_RT5033=n
+CONFIG_MFD_RTSX_USB=n
+CONFIG_MFD_RC5T583=n
+CONFIG_MFD_RK808=n
+CONFIG_MFD_RN5T618=n
+CONFIG_MFD_SEC_CORE=n
+CONFIG_MFD_SI476X_CORE=n
+CONFIG_MFD_SM501=n
+CONFIG_MFD_SKY81452=n
+CONFIG_MFD_SMSC=n
+CONFIG_ABX500_CORE=n
+CONFIG_AB3100_CORE=n
+CONFIG_MFD_STMPE=n
+CONFIG_STMPE_I2C=n
+CONFIG_STMPE_SPI=n
+CONFIG_MFD_SYSCON=y
+CONFIG_MFD_TI_AM335X_TSCADC=m
+CONFIG_MFD_LP3943=n
+CONFIG_MFD_LP8788=n
+CONFIG_MFD_OMAP_USB_HOST=y
+CONFIG_MFD_PALMAS=y
+CONFIG_TPS6105X=n
+CONFIG_TPS65010=n
+CONFIG_TPS6507X=n
+CONFIG_MFD_TPS65086=n
+CONFIG_MFD_TPS65090=y
+CONFIG_MFD_TPS65217=y
+CONFIG_MFD_TPS65218=y
+CONFIG_MFD_TPS6586X=y
+CONFIG_MFD_TPS65910=y
+CONFIG_MFD_TPS65912_I2C=n
+CONFIG_MFD_TPS65912_SPI=n
+CONFIG_MFD_TPS80031=n
+CONFIG_TWL4030_CORE=n
+CONFIG_TWL4030_POWER=n
+CONFIG_MFD_TWL4030_AUDIO=n
+CONFIG_TWL6040_CORE=y
+CONFIG_MFD_WL1273_CORE=n
+CONFIG_MFD_LM3533=n
+CONFIG_MFD_TI_LP873X=y
+CONFIG_MFD_TI_LP87565=y
+CONFIG_MFD_TC3589X=n
+CONFIG_MFD_TMIO=n
+CONFIG_MFD_T7L66XB=n
+CONFIG_MFD_TC6387XB=n
+CONFIG_MFD_TC6393XB=n
+CONFIG_MFD_VX855=n
+CONFIG_MFD_ARIZONA_I2C=n
+CONFIG_MFD_ARIZONA_SPI=n
+CONFIG_MFD_WM8400=n
+CONFIG_MFD_WM831X_I2C=n
+CONFIG_MFD_WM831X_SPI=n
+CONFIG_MFD_WM8350_I2C=n
+CONFIG_MFD_WM8994=n
+CONFIG_MFD_VEXPRESS_SYSREG=n
+
+# Regulators
+CONFIG_REGULATOR=y
+CONFIG_REGULATOR_FIXED_VOLTAGE=y
+CONFIG_REGULATOR_DEBUG=n
+CONFIG_REGULATOR_VIRTUAL_CONSUMER=n
+CONFIG_REGULATOR_USERSPACE_CONSUMER=n
+CONFIG_REGULATOR_ACT8865=n
+CONFIG_REGULATOR_AD5398=n
+CONFIG_REGULATOR_ANATOP=n
+CONFIG_REGULATOR_AS3711=n
+CONFIG_REGULATOR_AS3722=n
+CONFIG_REGULATOR_AXP20X=n
+CONFIG_REGULATOR_BCM590XX=n
+CONFIG_REGULATOR_DA9210=n
+CONFIG_REGULATOR_DA9211=n
+CONFIG_REGULATOR_FAN53555=n
+CONFIG_REGULATOR_GPIO=y
+CONFIG_REGULATOR_ISL9305=n
+CONFIG_REGULATOR_ISL6271A=n
+CONFIG_REGULATOR_LP3971=n
+CONFIG_REGULATOR_LP3972=n
+CONFIG_REGULATOR_LP872X=n
+CONFIG_REGULATOR_LP8755=n
+CONFIG_REGULATOR_LP873X=y
+CONFIG_REGULATOR_LP87565=y
+CONFIG_REGULATOR_LTC3589=n
+CONFIG_REGULATOR_MAX14577=n
+CONFIG_REGULATOR_MAX1586=n
+CONFIG_REGULATOR_MAX8649=n
+CONFIG_REGULATOR_MAX8660=n
+CONFIG_REGULATOR_MAX8907=n
+CONFIG_REGULATOR_MAX8952=n
+CONFIG_REGULATOR_MAX8973=n
+CONFIG_REGULATOR_MAX77686=n
+CONFIG_REGULATOR_MAX77693=n
+CONFIG_REGULATOR_MAX77802=n
+CONFIG_REGULATOR_MT6311=n
+CONFIG_REGULATOR_PALMAS=y
+CONFIG_REGULATOR_PBIAS=y
+CONFIG_REGULATOR_PFUZE100=n
+CONFIG_REGULATOR_PWM=n
+CONFIG_REGULATOR_RK808=n
+CONFIG_REGULATOR_S2MPA01=n
+CONFIG_REGULATOR_S2MPS11=n
+CONFIG_REGULATOR_S5M8767=n
+CONFIG_REGULATOR_TI_ABB=y
+CONFIG_REGULATOR_TPS51632=n
+CONFIG_REGULATOR_TPS62360=y
+CONFIG_REGULATOR_TPS65023=y
+CONFIG_REGULATOR_TPS6507X=y
+CONFIG_REGULATOR_TPS65090=y
+CONFIG_REGULATOR_TPS65217=y
+CONFIG_REGULATOR_TPS65218=y
+CONFIG_REGULATOR_TPS6524X=y
+CONFIG_REGULATOR_TPS6586X=y
+CONFIG_REGULATOR_TPS65910=y
+CONFIG_REGULATOR_TWL4030=n
+CONFIG_REGULATOR_VEXPRESS=n
+
+# Crypto Modules
+CONFIG_CRYPTO_DEV_OMAP=m
+CONFIG_CRYPTO_DEV_OMAP_SHAM=m
+CONFIG_CRYPTO_DEV_OMAP_AES=m
+CONFIG_CRYPTO_DEV_OMAP_DES=m
+CONFIG_CRYPTO_USER_API_HASH=m
+CONFIG_CRYPTO_USER_API_SKCIPHER=m
+CONFIG_CRYPTO_SHA1_ARM_NEON=m
+CONFIG_CRYPTO_SHA256_ARM=m
+CONFIG_CRYPTO_SHA512_ARM=m
+CONFIG_CRYPTO_AES_ARM_BS=m
+
+#
+# RTC drivers
+#
+CONFIG_RTC_DRV_ABB5ZES3=n
+CONFIG_RTC_DRV_ABX80X=n
+CONFIG_RTC_DRV_DS1307=m
+CONFIG_RTC_DRV_DS1374=n
+CONFIG_RTC_DRV_DS1672=n
+CONFIG_RTC_DRV_DS3232=n
+CONFIG_RTC_DRV_HYM8563=n
+CONFIG_RTC_DRV_MAX6900=n
+CONFIG_RTC_DRV_RS5C372=n
+CONFIG_RTC_DRV_ISL1208=n
+CONFIG_RTC_DRV_ISL12022=n
+CONFIG_RTC_DRV_ISL12057=n
+CONFIG_RTC_DRV_X1205=n
+CONFIG_RTC_DRV_PALMAS=m
+CONFIG_RTC_DRV_PCF2127=n
+CONFIG_RTC_DRV_PCF8523=n
+CONFIG_RTC_DRV_PCF8563=n
+CONFIG_RTC_DRV_PCF85063=n
+CONFIG_RTC_DRV_PCF8583=n
+CONFIG_RTC_DRV_M41T80=n
+CONFIG_RTC_DRV_BQ32K=n
+CONFIG_RTC_DRV_TPS6586X=m
+CONFIG_RTC_DRV_TPS65910=m
+CONFIG_RTC_DRV_TWL92330=y
+CONFIG_RTC_DRV_TWL4030=n
+CONFIG_RTC_DRV_S35390A=n
+CONFIG_RTC_DRV_FM3130=n
+CONFIG_RTC_DRV_RX8581=n
+CONFIG_RTC_DRV_RX8025=n
+CONFIG_RTC_DRV_EM3027=n
+CONFIG_RTC_DRV_RV3029C2=n
+CONFIG_RTC_DRV_RV8803=n
+
+CONFIG_RTC_DRV_M41T93=n
+CONFIG_RTC_DRV_M41T94=n
+CONFIG_RTC_DRV_DS1305=n
+CONFIG_RTC_DRV_DS1343=n
+CONFIG_RTC_DRV_DS1347=n
+CONFIG_RTC_DRV_DS1390=n
+CONFIG_RTC_DRV_MAX6902=n
+CONFIG_RTC_DRV_R9701=n
+CONFIG_RTC_DRV_RS5C348=n
+CONFIG_RTC_DRV_DS3234=n
+CONFIG_RTC_DRV_PCF2123=n
+CONFIG_RTC_DRV_RX4581=n
+CONFIG_RTC_DRV_MCP795=n
+
+CONFIG_RTC_DRV_CMOS=n
+CONFIG_RTC_DRV_DS1286=n
+CONFIG_RTC_DRV_DS1511=n
+CONFIG_RTC_DRV_DS1553=n
+CONFIG_RTC_DRV_DS1685_FAMILY=n
+CONFIG_RTC_DRV_DS1742=n
+CONFIG_RTC_DRV_DS2404=n
+CONFIG_RTC_DRV_STK17TA8=n
+CONFIG_RTC_DRV_M48T86=n
+CONFIG_RTC_DRV_M48T35=n
+CONFIG_RTC_DRV_M48T59=n
+CONFIG_RTC_DRV_MSM6242=n
+CONFIG_RTC_DRV_BQ4802=n
+CONFIG_RTC_DRV_RP5C01=n
+CONFIG_RTC_DRV_V3020=n
+CONFIG_RTC_DRV_ZYNQMP=n
+
+CONFIG_RTC_DRV_OMAP=m
+CONFIG_RTC_DRV_SNVS=n
+
+CONFIG_RTC_DRV_HID_SENSOR_TIME=n
+
+# WatchDog
+CONFIG_WATCHDOG=y
+CONFIG_WATCHDOG_CORE=y
+CONFIG_WATCHDOG_NOWAYOUT=n
+
+# Watchdog Device Drivers
+CONFIG_SOFT_WATCHDOG=n
+CONFIG_GPIO_WATCHDOG=n
+CONFIG_XILINX_WATCHDOG=n
+CONFIG_CADENCE_WATCHDOG=n
+CONFIG_DW_WATCHDOG=n
+CONFIG_OMAP_WATCHDOG=m
+CONFIG_TWL4030_WATCHDOG=m
+CONFIG_MAX63XX_WATCHDOG=n
+CONFIG_ALIM7101_WDT=n
+CONFIG_I6300ESB_WDT=n
+CONFIG_BCM7038_WDT=n
+CONFIG_MEN_A21_WDT=n
+
+# GPIO
+CONFIG_GPIOLIB=y
+CONFIG_GPIO_SYSFS=y
+CONFIG_DEBUG_GPIO=y
+CONFIG_GPIO_DAVINCI=y
+
+# GPIO disable unused options
+CONFIG_GPIO_DWAPB=n
+CONFIG_GPIO_EM=n
+CONFIG_GPIO_GENERIC_PLATFORM=n
+CONFIG_GPIO_XILINX=n
+CONFIG_GPIO_TPS6586X=n
+
+# Enable options to facilitate testing
+CONFIG_CRYPTO_TEST=m
+CONFIG_RTC_DEBUG=y
+CONFIG_THERMAL_EMULATION=y
+
+# OPTEE Driver
+CONFIG_TEE=y
+CONFIG_OPTEE=y
+
+#K3 Thermal
+CONFIG_K3_THERMAL=y
+
+# ION
+CONFIG_STAGING=y
+CONFIG_ANDROID=y
+CONFIG_ION=y
+CONFIG_ION_SYSTEM_HEAP=y
+CONFIG_ION_CARVEOUT_HEAP=y
+CONFIG_ION_CHUNK_HEAP=y
+CONFIG_ION_TI=y
+
+# DMA-BUF exporter
+CONFIG_DMA_BUF_PHYS=y
diff -Naur --no-dereference a/ti_config_fragments/connectivity.cfg b/ti_config_fragments/connectivity.cfg
--- a/ti_config_fragments/connectivity.cfg	1969-12-31 19:00:00.000000000 -0500
+++ b/ti_config_fragments/connectivity.cfg	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,477 @@
+##################################################
+# TI Connectivity Configs
+##################################################
+
+# Disable unused I2C options
+CONFIG_I2C_MUX=n
+CONFIG_I2C_DESIGNWARE_PLATFORM=n
+CONFIG_I2C_GPIO=n
+CONFIG_I2C_RK3X=n
+CONFIG_I2C_XILINX=n
+
+# I2C controllers
+CONFIG_I2C=y
+CONFIG_I2C_OMAP=y
+
+# I2C GPIO expanders
+CONFIG_GPIO_PCA953X=y
+CONFIG_GPIO_PCA953X_IRQ=y
+CONFIG_GPIO_PCF857X=y
+CONFIG_GPIO_TPIC2810=m
+
+#I2C EEPROMS
+CONFIG_EEPROM_AT24=m
+
+#Networking drivers
+CONFIG_NET_VENDOR_TI=y
+CONFIG_KEYSTONE_NAVIGATOR_QMSS=y
+CONFIG_KEYSTONE_NAVIGATOR_DMA=y
+CONFIG_TI_KEYSTONE_NETCP=y
+CONFIG_TI_KEYSTONE_NETCP_ETHSS=y
+CONFIG_TI_DAVINCI_EMAC=y
+CONFIG_TI_DAVINCI_MDIO=y
+CONFIG_TI_DAVINCI_CPDMA=y
+CONFIG_TI_CPSW_PHY_SEL=y
+CONFIG_TI_CPSW_ALE=y
+CONFIG_TI_CPSW=y
+CONFIG_TI_CPTS=y
+CONFIG_TI_PRUETH=m
+CONFIG_TI_ICSSG_PRUETH=m
+CONFIG_TI_RDEV_ETH_SWITCH_VIRT_EMAC=m
+
+# non-TI Net vendors
+CONFIG_NET_DSA_BCM_SF2=n
+CONFIG_B53=n
+CONFIG_SYSTEMPORT=n
+CONFIG_NET_VENDOR_3COM=n
+CONFIG_NET_VENDOR_ADAPTEC=n
+CONFIG_NET_VENDOR_AGERE=n
+CONFIG_NET_VENDOR_ALTEON=n
+CONFIG_NET_VENDOR_AMAZON=n
+CONFIG_NET_VENDOR_AMD=n
+CONFIG_NET_VENDOR_ARC=n
+CONFIG_NET_VENDOR_ATHEROS=n
+CONFIG_NET_VENDOR_BROCADE=n
+CONFIG_NET_VENDOR_CAVIUM=n
+CONFIG_NET_VENDOR_CHELSIO=n
+CONFIG_BCMGENET=n
+CONFIG_NET_VENDOR_CIRRUS=n
+CONFIG_NET_VENDOR_CISCO=n
+CONFIG_NET_VENDOR_DEC=n
+CONFIG_NET_VENDOR_DLINK=n
+CONFIG_NET_VENDOR_EMULEX=n
+CONFIG_NET_VENDOR_EZCHIP=n
+CONFIG_NET_VENDOR_FARADAY=n
+CONFIG_NET_VENDOR_HISILICON=n
+CONFIG_NET_VENDOR_HP=n
+CONFIG_IGB=n
+CONFIG_NET_VENDOR_I825XX=n
+CONFIG_NET_VENDOR_MELLANOX=n
+CONFIG_NET_VENDOR_MICROCHIP=n
+CONFIG_NET_VENDOR_MYRI=n
+CONFIG_NET_VENDOR_NATSEMI=n
+CONFIG_NET_VENDOR_NETRONOME=n
+CONFIG_NET_VENDOR_NVIDIA=n
+CONFIG_NET_VENDOR_OKI=n
+CONFIG_NET_VENDOR_8390=n
+CONFIG_NET_VENDOR_QLOGIC=n
+CONFIG_NET_VENDOR_QUALCOMM=n
+CONFIG_NET_VENDOR_REALTEK=n
+CONFIG_NET_VENDOR_RENESAS=n
+CONFIG_NET_VENDOR_RDC=n
+CONFIG_NET_VENDOR_ROCKER=n
+CONFIG_NET_VENDOR_SAMSUNG=n
+CONFIG_NET_VENDOR_SILAN=n
+CONFIG_NET_VENDOR_SIS=n
+CONFIG_NET_VENDOR_SEEQ=n
+CONFIG_NET_VENDOR_STMICRO=n
+CONFIG_NET_VENDOR_SUN=n
+CONFIG_NET_VENDOR_SYNOPSYS=n
+CONFIG_NET_VENDOR_TEHUTI=n
+CONFIG_NET_VENDOR_VIA=n
+CONFIG_NET_VENDOR_WIZNET=n
+#Wireless LAN
+CONFIG_WLCORE=m
+CONFIG_WLCORE_SDIO=m
+CONFIG_WL18XX=m
+CONFIG_NL80211_TESTMODE=y
+
+#MDIO phys
+CONFIG_MARVELL_PHY=y
+CONFIG_MICREL_PHY=y
+# unused PHY drivers
+CONFIG_BROADCOM_PHY=n
+CONFIG_ICPLUS_PHY=n
+#PRU MII PHYs for Industrial Boards
+CONFIG_DP83848_PHY=y
+# Enable phy for DRA72 evm
+CONFIG_DP83867_PHY=y
+
+#MTD
+CONFIG_MTD=y
+CONFIG_MEMORY=y
+CONFIG_TI_AEMIF=y
+CONFIG_OMAP_GPMC=y
+CONFIG_MTD_NAND_OMAP2=y
+CONFIG_MTD_NAND_OMAP_BCH=y
+CONFIG_MTD_NAND_OMAP_BCH_BUILD=y
+CONFIG_MTD_NAND_DAVINCI=y
+CONFIG_MTD_TESTS=m
+CONFIG_MTD_CMDLINE_PARTS=y
+CONFIG_MTD_UBI=y
+CONFIG_MTD_HYPERBUS=y
+CONFIG_HBMC_AM654=y
+
+#PCIE
+CONFIG_SOCIONEXT_SYNQUACER_PREITS=y
+CONFIG_PCI=y
+CONFIG_PCI_MSI=y
+CONFIG_PCI_ENDPOINT=y
+CONFIG_PCI_ENDPOINT_CONFIGFS=y
+CONFIG_PCI_EPF_TEST=y
+CONFIG_PCI_EPF_NTB=y
+CONFIG_PCI_ENDPOINT_TEST=m
+CONFIG_PCI_DRA7XX=y
+CONFIG_PCI_DRA7XX_HOST=y
+CONFIG_PCI_DRA7XX_EP=y
+CONFIG_PCI_KEYSTONE=y
+CONFIG_PCI_KEYSTONE_EP=y
+CONFIG_PCI_J721E=y
+CONFIG_PCIE_CADENCE=y
+CONFIG_PCIE_CADENCE_HOST=y
+CONFIG_PCIE_CADENCE_EP=y
+
+#NTB
+CONFIG_NTB=m
+CONFIG_NTB_EPF=m
+CONFIG_NTB_TRANSPORT=m
+CONFIG_NTB_NETDEV=m
+
+#NVME
+CONFIG_NVME_CORE=m
+CONFIG_BLK_DEV_NVME=m
+
+#These drivers have been used with DRA7x/AM57x PCIe RC with some success
+CONFIG_NET_VENDOR_BROADCOM=y
+CONFIG_NET_VENDOR_MARVELL=y
+CONFIG_NET_VENDOR_INTEL=y
+CONFIG_TIGON3=m
+CONFIG_SKGE=m
+CONFIG_E1000=m
+CONFIG_E1000E=m
+CONFIG_IWLWIFI=m
+CONFIG_IWLDVM=m
+CONFIG_IWLMVM=m
+CONFIG_B43=m
+#Generic Phys
+CONFIG_PHY_TI_KEYSTONE_SERDES=y
+CONFIG_PHY_AM654_SERDES=y
+CONFIG_PHY_J721E_WIZ=y
+CONFIG_PHY_CADENCE_SIERRA=y
+
+# Networking
+CONFIG_NF_CONNTRACK=m
+CONFIG_IP_NF_IPTABLES=m
+CONFIG_IP_NF_ARPTABLES=m
+CONFIG_IP_NF_ARPFILTER=m
+CONFIG_IP_NF_ARP_MANGLE=m
+CONFIG_IP_NF_FILTER=m
+CONFIG_IP_NF_NAT=m
+CONFIG_NF_NAT_IPV4=m
+CONFIG_IP_NF_TARGET_MASQUERADE=m
+CONFIG_BRIDGE=m
+CONFIG_XFRM_USER=m
+CONFIG_NET_KEY=m
+CONFIG_INET=y
+CONFIG_INET_AH=m
+CONFIG_INET6_AH=m
+CONFIG_INET6_IPCOMP=m
+CONFIG_INET6_XFRM_TUNNEL=m
+CONFIG_INET6_TUNNEL=m
+CONFIG_INET_ESP=m
+CONFIG_INET_IPCOMP=m
+CONFIG_INET_XFRM_MODE_TRANSPORT=m
+CONFIG_INET_XFRM_MODE_TUNNEL=m
+CONFIG_INET_XFRM_MODE_BEET=m
+CONFIG_IPV6_TUNNEL=m
+CONFIG_NETFILTER=y
+CONFIG_NETFILTER_XTABLES=m
+CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m
+CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m
+CONFIG_NETFILTER_XT_TARGET_MARK=m
+CONFIG_NETFILTER_XT_MATCH_COMMENT=m
+CONFIG_NETFILTER_XT_MATCH_CPU=m
+CONFIG_NETFILTER_XT_MATCH_IPRANGE=m
+CONFIG_NETFILTER_XT_MATCH_LENGTH=m
+CONFIG_NETFILTER_XT_MATCH_LIMIT=m
+CONFIG_NETFILTER_XT_MATCH_MAC=m
+CONFIG_NETFILTER_XT_MATCH_MARK=m
+CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m
+CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m
+CONFIG_NETFILTER_XT_MATCH_POLICY=m
+CONFIG_NETFILTER_XT_MATCH_SCTP=m
+CONFIG_BRIDGE_NF_EBTABLES=m
+CONFIG_BRIDGE_EBT_BROUTE=m
+CONFIG_BRIDGE_EBT_T_FILTER=m
+CONFIG_BRIDGE_EBT_T_NAT=m
+CONFIG_BRIDGE_EBT_802_3=m
+CONFIG_BRIDGE_EBT_AMONG=m
+CONFIG_BRIDGE_EBT_ARP=m
+CONFIG_BRIDGE_EBT_IP=m
+CONFIG_BRIDGE_EBT_IP6=m
+CONFIG_BRIDGE_EBT_LIMIT=m
+CONFIG_BRIDGE_EBT_MARK=m
+CONFIG_BRIDGE_EBT_PKTTYPE=m
+CONFIG_BRIDGE_EBT_STP=m
+CONFIG_BRIDGE_EBT_VLAN=m
+CONFIG_BRIDGE_EBT_ARPREPLY=m
+CONFIG_BRIDGE_EBT_DNAT=m
+CONFIG_BRIDGE_EBT_MARK_T=m
+CONFIG_BRIDGE_EBT_REDIRECT=m
+CONFIG_BRIDGE_EBT_SNAT=m
+CONFIG_BRIDGE_EBT_LOG=m
+CONFIG_BRIDGE_EBT_NFLOG=m
+CONFIG_NET_SCHED=y
+CONFIG_NET_SCH_CBQ=m
+CONFIG_NET_SCH_HTB=m
+CONFIG_NET_SCH_HFSC=m
+CONFIG_NET_SCH_PRIO=m
+CONFIG_NET_SCH_MULTIQ=m
+CONFIG_NET_SCH_RED=m
+CONFIG_NET_SCH_SFB=m
+CONFIG_NET_SCH_SFQ=m
+CONFIG_NET_SCH_TEQL=m
+CONFIG_NET_SCH_TBF=m
+CONFIG_NET_SCH_GRED=m
+CONFIG_NET_SCH_DSMARK=m
+CONFIG_NET_SCH_NETEM=m
+CONFIG_NET_SCH_DRR=m
+CONFIG_NET_SCH_MQPRIO=m
+CONFIG_NET_SCH_CHOKE=m
+CONFIG_NET_SCH_QFQ=m
+CONFIG_NET_SCH_CODEL=m
+CONFIG_NET_SCH_FQ_CODEL=m
+CONFIG_NET_SCH_INGRESS=m
+CONFIG_NET_CLS=y
+CONFIG_NET_CLS_BASIC=m
+CONFIG_NET_CLS_TCINDEX=m
+CONFIG_NET_CLS_ROUTE4=m
+CONFIG_NET_CLS_FW=m
+CONFIG_NET_CLS_U32=m
+CONFIG_CLS_U32_MARK=y
+CONFIG_NET_CLS_RSVP=m
+CONFIG_NET_CLS_RSVP6=m
+CONFIG_NET_CLS_FLOW=m
+CONFIG_NET_EMATCH=y
+CONFIG_NET_EMATCH_STACK=32
+CONFIG_NET_EMATCH_CMP=m
+CONFIG_NET_EMATCH_NBYTE=m
+CONFIG_NET_EMATCH_U32=m
+CONFIG_NET_EMATCH_META=m
+CONFIG_NET_EMATCH_TEXT=m
+CONFIG_NET_CLS_ACT=y
+CONFIG_NET_ACT_POLICE=m
+CONFIG_NET_ACT_GACT=m
+CONFIG_GACT_PROB=y
+CONFIG_NET_ACT_MIRRED=m
+CONFIG_NET_ACT_IPT=m
+CONFIG_NET_ACT_NAT=m
+CONFIG_NET_ACT_PEDIT=m
+CONFIG_NET_ACT_SIMP=m
+CONFIG_NET_ACT_SKBEDIT=m
+CONFIG_NET_ACT_CSUM=m
+CONFIG_NET_CLS_IND=y
+CONFIG_NET_SCH_FIFO=y
+CONFIG_IP_SCTP=m
+CONFIG_VLAN_8021Q=m
+CONFIG_IP_MULTICAST=y
+
+# Ethernet drivers
+CONFIG_TI_K3_UDMA_DESC_POOL=y
+CONFIG_TI_AM65_CPSW_NUSS=y
+CONFIG_TI_AM65_CPTS=y
+
+#MMC/SD support
+CONFIG_MMC=y
+CONFIG_MMC_SDHCI=y
+CONFIG_MMC_SDHCI_PLTFM=y
+CONFIG_MMC_OMAP_HS=y
+CONFIG_MMC_SDHCI_OMAP=y
+CONFIG_MMC_DW=n
+CONFIG_MMC_SDHCI_AM654=y
+
+#SPI
+CONFIG_SPI_CADENCE_QUADSPI=y
+CONFIG_SPI_TI_QSPI=y
+CONFIG_SPI_OMAP24XX=y
+#Disable unused SPI controllers
+CONFIG_SPI_BITBANG=n
+CONFIG_SPI_CADENCE=n
+CONFIG_SPI_ROCKCHIP=n
+CONFIG_SPI_XILINX=n
+CONFIG_SPI_SPIDEV=n
+
+#Disable SPI NOR 4K SECTORS
+CONFIG_MTD_SPI_NOR_USE_4K_SECTORS=n
+
+#SPI GPIO expanders
+CONFIG_GPIO_PISOSR=m
+
+#SATA
+CONFIG_SATA_AHCI_PLATFORM=m
+CONFIG_SATA_AHCI=m
+CONFIG_ATA=m
+
+#USB PHY
+CONFIG_OMAP_USB2=m
+
+#USB gadgets
+CONFIG_USB_GADGET=m
+CONFIG_USB_AUDIO=m
+CONFIG_USB_ETH=m
+CONFIG_USB_G_NCM=m
+CONFIG_USB_GADGETFS=m
+CONFIG_USB_FUNCTIONFS=m
+CONFIG_USB_FUNCTIONFS_ETH=y
+CONFIG_USB_FUNCTIONFS_RNDIS=y
+CONFIG_USB_FUNCTIONFS_GENERIC=y
+CONFIG_USB_MASS_STORAGE=m
+CONFIG_USB_G_SERIAL=m
+CONFIG_USB_MIDI_GADGET=m
+CONFIG_USB_G_PRINTER=m
+CONFIG_USB_CDC_COMPOSITE=m
+CONFIG_USB_G_ACM_MS=m
+CONFIG_USB_G_MULTI=m
+CONFIG_USB_G_MULTI_CDC=y
+CONFIG_USB_G_HID=m
+CONFIG_USB_G_DBGP=m
+CONFIG_USB_G_WEBCAM=m
+CONFIG_USB_GADGET_STORAGE_NUM_BUFFERS=32
+CONFIG_USB_ZERO=m
+CONFIG_USB_CONFIGFS=m
+CONFIG_USB_CONFIGFS_SERIAL=y
+CONFIG_USB_CONFIGFS_ACM=y
+CONFIG_USB_CONFIGFS_OBEX=y
+CONFIG_USB_CONFIGFS_NCM=y
+CONFIG_USB_CONFIGFS_ECM=y
+CONFIG_USB_CONFIGFS_ECM_SUBSET=y
+CONFIG_USB_CONFIGFS_RNDIS=y
+CONFIG_USB_CONFIGFS_EEM=y
+CONFIG_USB_CONFIGFS_MASS_STORAGE=y
+CONFIG_USB_CONFIGFS_F_LB_SS=y
+CONFIG_USB_CONFIGFS_F_FS=y
+CONFIG_USB_CONFIGFS_F_UAC1=y
+CONFIG_USB_CONFIGFS_F_UAC2=y
+CONFIG_USB_CONFIGFS_F_MIDI=y
+CONFIG_USB_CONFIGFS_F_HID=y
+CONFIG_USB_CONFIGFS_F_UVC=y
+CONFIG_USB_CONFIGFS_F_PRINTER=y
+
+# USB DWC3
+CONFIG_USB_DWC3=m
+CONFIG_USB_DWC3_DUAL_ROLE=y
+CONFIG_USB_DWC3_OMAP=m
+CONFIG_USB_DWC3_KEYSTONE=m
+CONFIG_USB_DWC3_PCI=n
+CONFIG_USB_DWC2=n
+CONFIG_USB_CHIPIDEA=n
+
+# USB CDNS3
+CONFIG_USB_CDNS3=m
+CONFIG_USB_CDNS3_GADGET=y
+CONFIG_USB_CDNS3_HOST=y
+CONFIG_USB_CDNS3_TI=m
+
+#USB PHY
+CONFIG_NOP_USB_XCEIV=m
+
+#USB MUSB
+CONFIG_USB_MUSB_HDRC=m
+CONFIG_USB_MUSB_DUAL_ROLE=y
+CONFIG_USB_MUSB_OMAP2PLUS=m
+CONFIG_USB_MUSB_AM35X=m
+CONFIG_USB_MUSB_DSPS=m
+CONFIG_USB_MUSB_AM335X_CHILD=m
+CONFIG_TI_CPPI41=y
+CONFIG_USB_TI_CPPI41_DMA=y
+CONFIG_AM335X_CONTROL_USB=y
+CONFIG_AM335X_PHY_USB=y
+
+#USB EHCI
+CONFIG_USB=m
+CONFIG_USB_EHCI_HCD=m
+CONFIG_USB_EHCI_PCI=m
+CONFIG_USB_EHCI_HCD_OMAP=m
+
+#USB Networking
+CONFIG_USB_USBNET=m
+CONFIG_USB_NET_DRIVERS=m
+CONFIG_USB_NET_SMSC95XX=m
+
+#USB testing
+CONFIG_USB_TEST=m
+CONFIG_USB_ANNOUNCE_NEW_DEVICES=y
+
+#USB Serial
+CONFIG_USB_ACM=m
+CONFIG_USB_SERIAL=m
+CONFIG_USB_SERIAL_CP210X=m
+CONFIG_USB_SERIAL_FTDI_SIO=m
+CONFIG_USB_SERIAL_PL2303=m
+CONFIG_USB_SERIAL_OPTION=m
+
+# Extcon drivers
+CONFIG_EXTCON=m
+CONFIG_EXTCON_PALMAS=m
+CONFIG_EXTCON_USB_GPIO=m
+
+# PWM
+CONFIG_PWM_TIECAP=y
+CONFIG_PWM_TIEHRPWM=y
+CONFIG_PWM_TIPWMSS=y
+CONFIG_PWM_PRU=m
+
+# 1-wire Bus Masters
+CONFIG_W1=m
+CONFIG_HDQ_MASTER_OMAP=m
+
+# Matrix keypad
+CONFIG_KEYBOARD_MATRIX=m
+
+#Touchscreen/ADC
+CONFIG_TOUCHSCREEN_PIXCIR=m
+CONFIG_MFD_TI_AM335X_TSCADC=m
+CONFIG_TOUCHSCREEN_TI_AM335X_TSC=m
+CONFIG_TI_AM335X_ADC=m
+
+#CAN
+CONFIG_CAN=m
+CONFIG_CAN_C_CAN=m
+CONFIG_CAN_C_CAN_PLATFORM=m
+CONFIG_CAN_M_CAN=m
+
+# Rotary Encoder
+CONFIG_INPUT_GPIO_DECODER=m
+
+# Filesystem extra options
+CONFIG_EXT4_FS_POSIX_ACL=y
+CONFIG_EXT4_FS_SECURITY=y
+CONFIG_CIFS=m
+CONFIG_CIFS_STATS=y
+CONFIG_CIFS_XATTR=y
+CONFIG_CIFS_POSIX=y
+CONFIG_CIFS_ACL=y
+CONFIG_UBIFS_FS=y
+
+# HD-Audio
+CONFIG_SND_USB=y
+CONFIG_SND_USB_AUDIO=m
+
+#UFS
+CONFIG_SCSI=y
+CONFIG_BLK_DEV_SD=y
+CONFIG_SCSI_UFSHCD=y
+CONFIG_SCSI_UFS_BSG=y
+CONFIG_SCSI_UFSHCD_PLATFORM=y
+CONFIG_SCSI_UFS_CDNS_PLATFORM=y
+CONFIG_SCSI_UFS_TI_J721E=y
diff -Naur --no-dereference a/ti_config_fragments/debug_ftrace.cfg b/ti_config_fragments/debug_ftrace.cfg
--- a/ti_config_fragments/debug_ftrace.cfg	1969-12-31 19:00:00.000000000 -0500
+++ b/ti_config_fragments/debug_ftrace.cfg	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,14 @@
+##################################################
+# Debug FTRACE options
+##################################################
+
+CONFIG_FTRACE=y
+CONFIG_DYNAMIC_FTRACE=y
+
+CONFIG_PREEMPT_TRACER=y
+CONFIG_IRQSOFF_TRACER=y
+CONFIG_FUNCTION_TRACER=y
+CONFIG_FUNCTION_PROFILER=y
+CONFIG_PREEMPT_TRACER=y
+CONFIG_SCHED_TRACER=y
+CONFIG_STACK_TRACER=y
diff -Naur --no-dereference a/ti_config_fragments/debug_options.cfg b/ti_config_fragments/debug_options.cfg
--- a/ti_config_fragments/debug_options.cfg	1969-12-31 19:00:00.000000000 -0500
+++ b/ti_config_fragments/debug_options.cfg	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,94 @@
+##################################################
+# Common Debug options considered generically
+# useful for kernel debug. This is not meant to
+# be an exhaustive list of options and will probably
+# have performance impact by using the same.
+#
+# IMPORTANT NOTE: Do not collect reference performance
+# data with this configuration.
+##################################################
+
+CONFIG_DYNAMIC_DEBUG=y
+
+CONFIG_DEBUG_PAGEALLOC=y
+CONFIG_WANT_PAGE_DEBUG_FLAGS=y
+CONFIG_PAGE_POISONING=y
+CONFIG_DEBUG_OBJECTS=y
+CONFIG_DEBUG_OBJECTS_SELFTEST=n
+CONFIG_DEBUG_OBJECTS_FREE=n
+CONFIG_DEBUG_OBJECTS_TIMERS=n
+CONFIG_DEBUG_OBJECTS_WORK=n
+CONFIG_DEBUG_OBJECTS_RCU_HEAD=y
+CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER=n
+CONFIG_DEBUG_OBJECTS_ENABLE_DEFAULT=1
+CONFIG_DEBUG_SLAB=y
+CONFIG_DEBUG_SLAB_LEAK=y
+CONFIG_SLUB_DEBUG=y
+
+CONFIG_DEBUG_KMEMLEAK=y
+CONFIG_DEBUG_KMEMLEAK_EARLY_LOG_SIZE=8000
+CONFIG_DEBUG_KMEMLEAK_TEST=n
+CONFIG_DEBUG_KMEMLEAK_DEFAULT_OFF=n
+CONFIG_DEBUG_INFO=y
+
+CONFIG_DEBUG_SHIRQ=n
+CONFIG_DEBUG_PREEMPT=y
+
+CONFIG_LOCKUP_DETECTOR=y
+CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=y
+CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC_VALUE=1
+CONFIG_DETECT_HUNG_TASK=y
+CONFIG_DEFAULT_HUNG_TASK_TIMEOUT=300
+CONFIG_BOOTPARAM_HUNG_TASK_PANIC=y
+CONFIG_BOOTPARAM_HUNG_TASK_PANIC_VALUE=1
+CONFIG_PANIC_ON_OOPS=y
+CONFIG_PANIC_ON_OOPS_VALUE=1
+
+CONFIG_DEBUG_RT_MUTEXES=y
+CONFIG_DEBUG_PI_LIST=y
+CONFIG_RT_MUTEX_TESTER=n
+CONFIG_DEBUG_SPINLOCK=y
+CONFIG_DEBUG_MUTEXES=y
+CONFIG_DEBUG_WW_MUTEX_SLOWPATH=y
+CONFIG_DEBUG_LOCK_ALLOC=y
+CONFIG_PROVE_LOCKING=y
+CONFIG_LOCKDEP=y
+CONFIG_LOCK_STAT=n
+CONFIG_DEBUG_LOCKDEP=y
+CONFIG_DEBUG_ATOMIC_SLEEP=y
+CONFIG_DEBUG_LOCKING_API_SELFTESTS=n
+CONFIG_STACKTRACE=y
+CONFIG_DEBUG_KOBJECT=n
+CONFIG_DEBUG_BUGVERBOSE=y
+CONFIG_DEBUG_WRITECOUNT=n
+CONFIG_DEBUG_LIST=y
+CONFIG_DEBUG_SG=y
+CONFIG_DEBUG_NOTIFIERS=y
+CONFIG_DEBUG_CREDENTIALS=n
+
+CONFIG_SPARSE_RCU_POINTER=y
+CONFIG_RCU_TORTURE_TEST=n
+CONFIG_RCU_CPU_STALL_TIMEOUT=60
+CONFIG_RCU_CPU_STALL_INFO=y
+CONFIG_RCU_TRACE=y
+CONFIG_PROVE_RCU=y
+CONFIG_PROVE_RCU_REPEATEDLY=y
+
+CONFIG_DMA_API_DEBUG=y
+
+CONFIG_ARM_PTDUMP=y
+CONFIG_ARM_UNWIND=y
+CONFIG_DEBUG_USER=y
+
+# Cgroups debug
+CONFIG_CGROUP_DEBUG=y
+CONFIG_DEBUG_BLK_CGROUP=y
+
+# IRQ Domain debug
+CONFIG_IRQ_DOMAIN_DEBUG=y
+
+# Debug HighMem
+CONFIG_DEBUG_HIGHMEM=y
+
+# Crypto self-tests
+CONFIG_CRYPTO_MANAGER_DISABLE_TESTS=n
diff -Naur --no-dereference a/ti_config_fragments/defconfig_builder.sh b/ti_config_fragments/defconfig_builder.sh
--- a/ti_config_fragments/defconfig_builder.sh	1969-12-31 19:00:00.000000000 -0500
+++ b/ti_config_fragments/defconfig_builder.sh	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,479 @@
+#!/bin/bash
+#
+#  defconfig_builder.sh
+#
+#  Copyright (C) 2015 Texas Instruments Incorporated - http://www.ti.com/
+#  ALL RIGHTS RESERVED
+#
+#  This script will perform a merge of config fragment files into a defconfig
+#  based on a map file.  The map file defines the defconfig options that have
+#  been tested by TI to boot and compile.
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  For more information type defconfig_builder.sh -?
+#
+# IMPORTANT NOTE: All modifications to this script must be verified by running
+# 'shellcheck --shell=bash defconfig_builder.sh' See 'Installing' section in:
+# https://github.com/koalaman/shellcheck/blob/master/README.md
+# Also note that shellcheck must be at least at revision 0.3.3
+
+DEBUG_CONFIG_TAG="debug_options"
+CONFIG_FRAGMENT_TAG="config-fragment="
+DISCLAIMER="\n*Please be advised that the Debug Option defconfigs may\nimpact \
+performance and should only be used for debugging.\n"
+
+# Template for temporary build files.. use PID to differentiate
+TMP_PREFIX=ti_defconfig_builder_$$
+TMP_TEMPLATE="$TMP_PREFIX"_XXXXX.tmp
+
+set_working_directory() {
+	# Sanity checkup kernel build location.
+	if [ ! -d "$WORKING_PATH" ]; then
+		WORKING_PATH=$(pwd)
+	fi
+
+	ORIGINAL_DIR=$(pwd)
+	if [ "$ORIGINAL_DIR" != "$WORKING_PATH" ]; then
+		cd "$WORKING_PATH"
+		WORKING_PATH=$(pwd)
+	fi
+
+	SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+	TI_WORKING_PATH=$SCRIPT_DIR
+	WORKING_PATH=$(cd "$SCRIPT_DIR""/.." && pwd)
+}
+
+prepare_for_exit() {
+	D=$(dirname "$PROCESSOR_FILE")
+	rm -f "$PROCESSOR_FILE"
+	rm -f "$BUILD_TYPE_FILE"
+	rm -f "$TEMP_TYPE_FILE"
+	if [ -s "$OLD_CONFIG" ]; then
+		mv "$OLD_CONFIG" "$WORKING_PATH"/.config
+	fi
+	# Clean everyone else up if we missed any
+	rm -f "$D"/"$TMP_PREFIX"*.tmp
+	exit
+}
+
+check_for_config_existance() {
+	# Check to make sure that the config fragments exist
+	TEMP_EXTRA_CONFIG_FILE=$(echo "$BUILD_DETAILS" | cut -d: -f6)
+	if [ -z "$TEMP_EXTRA_CONFIG_FILE" ]; then
+		CONFIG_FRAGMENTS=
+	else
+		for CONFIG_FRAGMENT_FILE in $TEMP_EXTRA_CONFIG_FILE;
+		do
+			# If we do already point to existing file, we are good.
+			if [ -e "$CONFIG_FRAGMENT_FILE" ]; then
+				CONFIG_FRAG="$CONFIG_FRAGMENT_FILE"
+			else
+				# Assume it is present in TI working path
+				CONFIG_FRAG="$TI_WORKING_PATH/$CONFIG_FRAGMENT_FILE"
+			fi
+			if [ ! -e "$CONFIG_FRAG" ]; then
+				CONFIG_FRAGMENTS="N/A"
+			fi
+		done
+	fi
+
+	if ! grep -qc "$BT_TEMP" "$BUILD_TYPE_FILE"; then
+		# If the config file and config fragments are available
+		# add it to the list.
+		CONFIG_FILE=$(echo "$BUILD_DETAILS" | awk '{print$8}')
+		if [ "$CONFIG_FILE" = "None" ]; then
+			CONFIG_FILE=
+		else
+			if [ -e "$TI_WORKING_PATH""/""$CONFIG_FILE" ]; then
+				CONFIG_FILE=
+			fi
+		fi
+		# If the check for the config file and the config fragments
+		# pass then these two variables should be empty.  If
+		# they fail then they should be N/A.
+		if [ -z "$CONFIG_FILE" -a -z "$CONFIG_FRAGMENTS" ]; then
+			max_configs=$((max_configs+1))
+			echo -e '\t'"$max_configs". "$BT_TEMP" >> "$BUILD_TYPE_FILE"
+		fi
+	fi
+}
+
+choose_build_type() {
+	TEMP_BT_FILE=$(mktemp -t $TMP_TEMPLATE)
+	TEMP_BUILD_FILE=$(mktemp -t $TMP_TEMPLATE)
+
+	grep "$DEFCONFIG_FILTER" "$DEFCONFIG_MAP_FILE" | grep "^classification:" | awk '{print$4}' > "$TEMP_BUILD_FILE"
+
+	max_configs=0
+	while true;
+	do
+		CONFIG_FILE=
+		CONFIG_FRAGMENTS=
+
+		BT_TEMP=$(head -n 1 "$TEMP_BUILD_FILE")
+		if [ -z "$BT_TEMP" ]; then
+			break
+		fi
+		BUILD_DETAILS=$(grep -w "$BT_TEMP" "$DEFCONFIG_MAP_FILE")
+		check_for_config_existance
+		sed -i "1d" "$TEMP_BUILD_FILE"
+	done
+
+	NUM_OF_BUILDS=$(wc -l "$BUILD_TYPE_FILE" | awk '{print$1}')
+	if [ "$NUM_OF_BUILDS" -eq 0 ]; then
+		echo "Sorry no build targets for this configuration.  Are you on the right branch?"
+		prepare_for_exit
+	fi
+
+	# Force the user to answer.  Maybe the user does not want to continue
+	while true;
+	do
+		echo -e "Available ""$DEFCONFIG_FILTER"" defconfig build options:\n"
+		cat "$BUILD_TYPE_FILE"
+		echo ""
+		read -p "Please enter the number of the defconfig to build or 'q' to exit: " REPLY
+
+		if [ "$REPLY" = "q" -o "$REPLY" = "Q" ]; then
+			prepare_for_exit
+		elif ! [[ "$REPLY" =~ ^[0-9]+$ ]]; then
+			echo -e "\n$REPLY is not a number of the defconfig.  Please try again!\n"
+			continue
+		elif [ "$REPLY" -gt '0' -a "$REPLY" -le "$NUM_OF_BUILDS" ]; then
+			CHOSEN_BUILD_TYPE=$(grep -w "$REPLY" "$BUILD_TYPE_FILE" | awk '{print$2}')
+			break
+		else
+			echo -e "\n'$REPLY' is not a valid choice. Please \
+choose a value between '1' and '$max_configs':\n"
+		fi
+	done
+	rm "$TEMP_BT_FILE"
+	rm "$TEMP_BUILD_FILE"
+}
+
+list_all_targets() {
+
+	TMP_MAP=$(mktemp -t $TMP_TEMPLATE)
+
+	count=0
+	max_types=0
+	while [ "x${SUPPORTED_ARCH[max_types]}" != "x" ]
+	do
+		DEFCONFIG_MAP_FILE=${SUPPORTED_ARCH[(count * 3) + 2]}
+		count=$(( $count + 1 ))
+		max_types=$(( $max_types + 3 ))
+		if [ ! -e "$DEFCONFIG_MAP_FILE" ]; then
+			continue
+		fi
+
+		cat "$DEFCONFIG_MAP_FILE" > "$TMP_MAP"
+		while true;
+		do
+			CONFIG_FILE=
+			CONFIG_FRAGMENTS=
+
+			BT_TEMP=$(head -n 1 "$TMP_MAP" | awk '{print$4}')
+			BUILD_DETAILS=$(head -n 1 "$TMP_MAP")
+			if [ -z "$BUILD_DETAILS" ]; then
+				break
+			fi
+			check_for_config_existance
+			sed -i "1d" "$TMP_MAP"
+		done
+		rm "$TMP_MAP"
+	done
+	cat "$BUILD_TYPE_FILE"
+}
+
+get_build_details() {
+
+	count=0
+	max_types=0
+	while [ "x${SUPPORTED_ARCH[max_types]}" != "x" ]
+	do
+		DEFCONFIG_MAP_FILE=${SUPPORTED_ARCH[(count * 3) + 2]}
+		if [ -e "$DEFCONFIG_MAP_FILE" ]; then
+			BUILD_DETAILS=$(grep -w "$CHOSEN_BUILD_TYPE" "$DEFCONFIG_MAP_FILE")
+			if [ ! -z "$BUILD_DETAILS" ]; then
+				DEFCONFIG_KERNEL_PATH=${SUPPORTED_ARCH[(count * 3) + 1]}
+				break
+			fi
+		fi
+
+		count=$(( $count + 1 ))
+		max_types=$(( $max_types + 3 ))
+	done
+
+	if [ -z "$BUILD_DETAILS" ]; then
+		echo "Cannot find the build type or a match for $CHOSEN_BUILD_TYPE"
+		TEMP_BUILD_FILE=$(mktemp -t $TMP_TEMPLATE)
+		grep "$CHOSEN_BUILD_TYPE" "$DEFCONFIG_MAP_FILE" > "$TEMP_BUILD_FILE"
+		while true;
+		do
+			CONFIG_FILE=
+			CONFIG_FRAGMENTS=
+
+			BT_TEMP=$(head -n 1 "$TEMP_BUILD_FILE" | awk '{print$4}')
+			if [ -z "$BT_TEMP" ]; then
+				break
+			fi
+			BUILD_DETAILS=$(grep -w "$BT_TEMP" "$DEFCONFIG_MAP_FILE")
+			check_for_config_existance
+			sed -i "1d" "$TEMP_BUILD_FILE"
+		done
+		rm -rf "$TEMP_BUILD_FILE"
+
+		NUM_OF_BUILDS=$(wc -l "$BUILD_TYPE_FILE" | awk '{print$1}')
+		if [ "$NUM_OF_BUILDS" -eq 0 ]; then
+			echo "Maybe try one of the following:"
+			list_all_targets
+		else
+			echo "Did you mean any of the following?"
+			cat "$BUILD_TYPE_FILE"
+		fi
+
+		return 1
+	fi
+
+	DEFCONFIG=$(echo "$BUILD_DETAILS" | awk '{print$6}')
+	DEFCONFIG="$DEFCONFIG_KERNEL_PATH""/""$DEFCONFIG"
+	CONFIG_FILE=$(echo "$BUILD_DETAILS" | awk '{print$8}')
+	# There may be a need to just build with the config fragments themselves
+	if [ "$CONFIG_FILE" = "None" ]; then
+		CONFIG_FILE=
+	fi
+
+	if [ ! -e "$TI_WORKING_PATH/$CONFIG_FILE" ]; then
+		echo "$TI_WORKING_PATH/$CONFIG_FILE does not exist"
+		return 1
+	fi
+
+	TEMP_EXTRA_CONFIG_FILE=$(echo "$BUILD_DETAILS" | cut -d: -f6)
+	for CONFIG_FRAGMENT_FILE in $TEMP_EXTRA_CONFIG_FILE;
+	do
+		# If we do already point to existing file, we are good.
+		if [ -e "$CONFIG_FRAGMENT_FILE" ]; then
+			CONFIG_FRAG="$CONFIG_FRAGMENT_FILE"
+		else
+			# Assume it is present in TI working path
+			CONFIG_FRAG="$TI_WORKING_PATH/$CONFIG_FRAGMENT_FILE"
+		fi
+		if [ -e "$CONFIG_FRAG" ]; then
+			EXTRA_CONFIG_FILE="$EXTRA_CONFIG_FILE $CONFIG_FRAG"
+		else
+			echo "$CONFIG_FRAG" does not exist
+		fi
+	done
+}
+
+build_defconfig() {
+
+	if [ ! -z "$CONFIG_FILE" -a -e "$TI_WORKING_PATH/$CONFIG_FILE" ]; then
+		CONFIGS=$(grep "$CONFIG_FRAGMENT_TAG" "$TI_WORKING_PATH/$CONFIG_FILE" | cut -d= -f2)
+	fi
+
+	"$WORKING_PATH"/scripts/kconfig/merge_config.sh -m -r "$DEFCONFIG" \
+		"$CONFIGS" "$EXTRA_CONFIG_FILE" > /dev/null
+
+	if [ "$?" = "0" ];then
+		echo "Creating defconfig file ""$DEFCONFIG_KERNEL_PATH/""$CHOSEN_BUILD_TYPE"_defconfig
+		mv .config "$DEFCONFIG_KERNEL_PATH"/"$CHOSEN_BUILD_TYPE"_defconfig
+	else
+		echo "Defconfig creation failed"
+		return 1
+	fi
+}
+
+choose_defconfig_type() {
+
+	TEMP_TYPE_FILE=$(mktemp -t $TMP_TEMPLATE)
+
+	TYPE_FILE=$(awk '{print$2}' "$DEFCONFIG_MAP_FILE" | sort -u | grep -i "SDK_")
+
+	max_types=0
+	for TYPE_TMP in $TYPE_FILE;
+	do
+		max_types=$((max_types+1))
+		echo -e '\t' "$max_types." "$TYPE_TMP" >> "$TEMP_TYPE_FILE"
+	done
+	echo >> "$TEMP_TYPE_FILE"
+
+	while true;
+	do
+		cat "$TEMP_TYPE_FILE"
+		read -p "Please choose a defconfig type to build for or 'q' to exit: " REPLY
+		if [ "$REPLY" = "q" -o "$REPLY" = "Q" ]; then
+			prepare_for_exit
+		elif ! [[ "$REPLY" =~ ^[0-9]+$ ]]; then
+			echo -e "\n'$REPLY' is not a number for the build type.  Please try again!\n"
+			continue
+		elif [ "$REPLY" -gt '0' -a "$REPLY" -le "$max_types" ]; then
+			REPLY="$REPLY""."
+			DEFCONFIG_FILTER=$(awk '{if ($1 == "'"$REPLY"'") print $2;}' "$TEMP_TYPE_FILE")
+			break
+		else
+			echo -e "\n'$REPLY' is not a valid choice. Please \
+choose a value between '1' and '$max_types':\n"
+		fi
+	done
+
+	DEBUG_BUILD=$(grep "$DEFCONFIG_FILTER" "$DEFCONFIG_MAP_FILE" | grep -wc "$DEBUG_CONFIG_TAG" )
+	if [ "$DEBUG_BUILD" -gt '0' ]; then
+		echo -e "$DISCLAIMER"
+	fi
+}
+
+choose_architecture() {
+
+	TEMP_ARCH_FILE=$(mktemp -t $TMP_TEMPLATE)
+
+	max_types=0
+	count=0
+	while [ "x${SUPPORTED_ARCH[max_types]}" != "x" ]
+	do
+		ARCH_TYPE=${SUPPORTED_ARCH[max_types]}
+		if [ -e "${SUPPORTED_ARCH[max_types + 2]}" ]; then
+			count=$(( $count + 1 ))
+			echo -e '\t' "$count." "$ARCH_TYPE" >> "$TEMP_ARCH_FILE"
+		fi
+		max_types=$(( $max_types + 3 ))
+	done
+
+	if [ "$count" -eq 1 ]; then
+		REPLY=$count
+	else
+		while true;
+		do
+			cat "$TEMP_ARCH_FILE"
+			read -p "Please choose an architecture to build for or 'q' to exit: " REPLY
+			if [ "$REPLY" = "q" -o "$REPLY" = "Q" ]; then
+				prepare_for_exit
+			elif ! [[ "$REPLY" =~ ^[0-9]+$ ]]; then
+				echo -e "\n'$REPLY' is not a number for the build type.  Please try again!\n"
+				continue
+			elif [ "$REPLY" -gt '0' -a "$REPLY" -le "$max_types" ]; then
+				REPLY_DISP="$REPLY""."
+				ARCH_TO_BUILD=$(awk '{if ($1 == "'"$REPLY_DISP"'") print $2;}' "$TEMP_ARCH_FILE")
+				break
+			else
+				echo -e "\n'$REPLY' is not a valid choice. Please \
+	choose a value between '1' and '$max_types':\n"
+			fi
+		done
+	fi
+
+	max_types=0
+	while [ "x${SUPPORTED_ARCH[max_types]}" != "x" ]
+	do
+		ARCH_TYPE=${SUPPORTED_ARCH[max_types]}
+		ARCH_COUNTER=$(grep -c "$ARCH_TO_BUILD" <<< $ARCH_TYPE)
+		if [ "$ARCH_COUNTER" -gt 0 ]; then
+			break
+		fi
+		max_types=$(( $max_types + 3 ))
+	done
+
+	DEFCONFIG_KERNEL_PATH=${SUPPORTED_ARCH[max_types + 1]}
+	DEFCONFIG_MAP_FILE=${SUPPORTED_ARCH[max_types + 2]}
+}
+
+usage() {
+cat << EOF
+
+This script will perform a merge of config fragment files into a defconfig
+based on a map file.  The map file defines the defconfig options that have
+been tested by TI to boot and compile.
+
+Optional:
+	-w - Location of the TI Linux kernel
+	-t - Indicates the type of defconfig to build.  This will force the
+	     defconfig to build without user interaction.
+	-l - List all buildable defconfig options
+
+Command line example to generate the TI SDK AM335x processor defconfig automatically
+without user interaction:
+
+	ti_config_fragments/defconfig_builder.sh -t ti_sdk_am3x_release
+
+Command line Example if building from the ti_config_fragments directory:
+	defconfig_builder.sh -w ../.
+
+User interactive command line example:
+	ti_config_fragments/defconfig_builder.sh
+EOF
+}
+
+#########################################
+# Script Start
+#########################################
+while getopts "?w:t:l" OPTION
+do
+	case $OPTION in
+		w)
+			WORKING_PATH=$OPTARG;;
+		t)
+			CHOSEN_BUILD_TYPE=$OPTARG;;
+		l)
+			LIST_TARGETS="y";;
+		?)
+			usage
+			exit;;
+     esac
+done
+
+trap prepare_for_exit SIGHUP EXIT SIGINT SIGTERM
+
+set_working_directory
+
+SUPPORTED_ARCH=(
+"v7 ARM Architecture" "$WORKING_PATH/arch/arm/configs" "$TI_WORKING_PATH/defconfig_map.txt"
+"v8 ARM Architecture" "$WORKING_PATH/arch/arm64/configs" "$TI_WORKING_PATH/v8_defconfig_map.txt")
+
+BUILD_TYPE_FILE=$(mktemp -t $TMP_TEMPLATE)
+
+if [ ! -z "$LIST_TARGETS" ]; then
+	echo "The following are a list of buildable defconfigs:"
+	list_all_targets
+	exit 0
+fi
+
+
+PROCESSOR_FILE=$(mktemp -t $TMP_TEMPLATE)
+OLD_CONFIG=$(mktemp -t $TMP_TEMPLATE)
+if [ -f "$WORKING_PATH"/.config ]; then
+	mv "$WORKING_PATH"/.config "$OLD_CONFIG"
+fi
+
+if [ ! -z "$CHOSEN_BUILD_TYPE" ]; then
+	get_build_details
+	if [ "$?" -gt 0 ]; then
+		exit 1
+	fi
+
+	build_defconfig
+	if [ "$?" -gt 0 ]; then
+		exit 1
+	fi
+	exit 0
+fi
+
+choose_architecture
+
+if [ ! -e "$DEFCONFIG_MAP_FILE" ]; then
+	echo "No defconfig map file found"
+	exit 1
+fi
+
+choose_defconfig_type
+
+choose_build_type
+get_build_details
+
+build_defconfig
diff -Naur --no-dereference a/ti_config_fragments/defconfig_map.txt b/ti_config_fragments/defconfig_map.txt
--- a/ti_config_fragments/defconfig_map.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/ti_config_fragments/defconfig_map.txt	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,34 @@
+# Release Defconfigs
+classification: SDK_Release_Defconfigs type: ti_sdk_am3x_release defconfig: multi_v7_defconfig config_file: None extra_configs: multi_v7_prune.cfg baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg omap_soc.cfg am33xx_only.cfg systemd.cfg
+classification: SDK_Release_Defconfigs type: ti_sdk_am4x_release defconfig: multi_v7_defconfig config_file: None extra_configs: multi_v7_prune.cfg baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg omap_soc.cfg am43xx_only.cfg systemd.cfg
+classification: SDK_Release_Defconfigs type: ti_sdk_dra7x_release defconfig: multi_v7_defconfig config_file: None extra_configs: multi_v7_prune.cfg baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg omap_soc.cfg lpae.cfg dra7_only.cfg systemd.cfg
+classification: SDK_Release_Defconfigs type: ti_sdk_omap2_release defconfig: multi_v7_defconfig config_file: None extra_configs: multi_v7_prune.cfg baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg omap_soc.cfg systemd.cfg
+classification: SDK_Release_Defconfigs type: ti_sdk_k2g_release defconfig: multi_v7_defconfig config_file: None extra_configs: multi_v7_prune.cfg baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg keystone.cfg lpae.cfg k2g_only.cfg systemd.cfg
+classification: SDK_Release_Defconfigs type: ti_sdk_keystone_release defconfig: multi_v7_defconfig config_file: None extra_configs: multi_v7_prune.cfg baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg keystone.cfg lpae.cfg systemd.cfg
+classification: SDK_Release_Defconfigs type: ti_sdk_omapl138_release defconfig: davinci_all_defconfig config_file: None extra_configs: baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg systemd.cfg
+# Debug Defconfigs
+classification: SDK_Debug_Defconfigs type: ti_sdk_am3x_debug defconfig: multi_v7_defconfig config_file: None extra_configs: multi_v7_prune.cfg baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg omap_soc.cfg am33xx_only.cfg systemd.cfg debug_options.cfg
+classification: SDK_Debug_Defconfigs type: ti_sdk_am4x_debug defconfig: multi_v7_defconfig config_file: None extra_configs: multi_v7_prune.cfg baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg omap_soc.cfg am43xx_only.cfg systemd.cfg debug_options.cfg
+classification: SDK_Debug_Defconfigs type: ti_sdk_dra7x_debug defconfig: multi_v7_defconfig config_file: None extra_configs: multi_v7_prune.cfg baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg omap_soc.cfg lpae.cfg dra7_only.cfg systemd.cfg debug_options.cfg
+classification: SDK_Debug_Defconfigs type: ti_sdk_omap2_debug defconfig: multi_v7_defconfig config_file: None extra_configs: multi_v7_prune.cfg baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg omap_soc.cfg systemd.cfg debug_options.cfg
+classification: SDK_Debug_Defconfigs type: ti_sdk_k2g_debug defconfig: multi_v7_defconfig config_file: None extra_configs: multi_v7_prune.cfg baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg keystone.cfg lpae.cfg k2g_only.cfg systemd.cfg debug_options.cfg
+classification: SDK_Debug_Defconfigs type: ti_sdk_keystone_debug defconfig: multi_v7_defconfig config_file: None extra_configs: multi_v7_prune.cfg baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg keystone.cfg lpae.cfg systemd.cfg debug_options.cfg
+classification: SDK_Debug_Defconfigs type: ti_sdk_omapl138_debug defconfig: davinci_all_defconfig config_file: None extra_configs: baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg systemd.cfg debug_options.cfg
+# RT Defconfigs
+classification: SDK_Release_Defconfigs type: ti_sdk_am3x_rt_release defconfig: multi_v7_defconfig config_file: None extra_configs: multi_v7_prune.cfg baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg omap_soc.cfg am33xx_only.cfg real_time.cfg systemd.cfg
+classification: SDK_Release_Defconfigs type: ti_sdk_am4x_rt_release defconfig: multi_v7_defconfig config_file: None extra_configs: multi_v7_prune.cfg baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg omap_soc.cfg am43xx_only.cfg real_time.cfg systemd.cfg
+classification: SDK_Release_Defconfigs type: ti_sdk_dra7x_rt_release defconfig: multi_v7_defconfig config_file: None extra_configs: multi_v7_prune.cfg baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg omap_soc.cfg lpae.cfg dra7_only.cfg real_time.cfg systemd.cfg
+classification: SDK_Release_Defconfigs type: ti_sdk_omap2_rt_release defconfig: multi_v7_defconfig config_file: None extra_configs: multi_v7_prune.cfg baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg omap_soc.cfg real_time.cfg systemd.cfg
+classification: SDK_Release_Defconfigs type: ti_sdk_k2g_rt_release defconfig: multi_v7_defconfig config_file: None extra_configs: multi_v7_prune.cfg baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg  keystone.cfg lpae.cfg k2g_only.cfg real_time.cfg systemd.cfg
+classification: SDK_Release_Defconfigs type: ti_sdk_keystone_rt_release defconfig: multi_v7_defconfig config_file: None extra_configs: multi_v7_prune.cfg baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg keystone.cfg lpae.cfg real_time.cfg systemd.cfg
+# RT Debug Defconfigs
+classification: SDK_Debug_Defconfigs type: ti_sdk_am3x_rt_debug defconfig: multi_v7_defconfig config_file: None extra_configs: multi_v7_prune.cfg baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg  omap_soc.cfg am33xx_only.cfg real_time.cfg systemd.cfg debug_options.cfg
+classification: SDK_Debug_Defconfigs type: ti_sdk_am4x_rt_debug defconfig: multi_v7_defconfig config_file: None extra_configs: multi_v7_prune.cfg baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg  omap_soc.cfg am43xx_only.cfg real_time.cfg systemd.cfg debug_options.cfg
+classification: SDK_Debug_Defconfigs type: ti_sdk_dra7x_rt_debug defconfig: multi_v7_defconfig config_file: None extra_configs: multi_v7_prune.cfg baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg  omap_soc.cfg lpae.cfg dra7_only.cfg real_time.cfg systemd.cfg debug_options.cfg
+classification: SDK_Debug_Defconfigs type: ti_sdk_omap2_rt_debug defconfig: multi_v7_defconfig config_file: None extra_configs: multi_v7_prune.cfg baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg omap_soc.cfg real_time.cfg systemd.cfg debug_options.cfg
+classification: SDK_Debug_Defconfigs type: ti_sdk_k2g_rt_debug defconfig: multi_v7_defconfig config_file: None extra_configs: multi_v7_prune.cfg baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg keystone.cfg lpae.cfg k2g_only.cfg real_time.cfg systemd.cfg debug_options.cfg
+classification: SDK_Debug_Defconfigs type: ti_sdk_keystone_rt_debug defconfig: multi_v7_defconfig config_file: None extra_configs:  multi_v7_prune.cfg baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg keystone.cfg lpae.cfg real_time.cfg systemd.cfg debug_options.cfg
+# Android Defconfigs
+classification: SDK_Release_Defconfigs type: ti_sdk_am57x_android_release defconfig: multi_v7_defconfig config_file: None extra_configs: multi_v7_prune.cfg baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg omap_soc.cfg lpae.cfg dra7_only.cfg systemd.cfg ../kernel/configs/android-base.config ../kernel/configs/android-recommended.config android.cfg
+# Android Debug Defconfigs
+classification: SDK_Debug_Defconfigs type: ti_sdk_am57x_android_debug defconfig: multi_v7_defconfig config_file: None extra_configs: multi_v7_prune.cfg baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg omap_soc.cfg lpae.cfg dra7_only.cfg systemd.cfg debug_options.cfg ../kernel/configs/android-base.config ../kernel/configs/android-recommended.config android.cfg
diff -Naur --no-dereference a/ti_config_fragments/dra7_only.cfg b/ti_config_fragments/dra7_only.cfg
--- a/ti_config_fragments/dra7_only.cfg	1969-12-31 19:00:00.000000000 -0500
+++ b/ti_config_fragments/dra7_only.cfg	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,25 @@
+##################################################
+# TI DRA7 specific config options
+##################################################
+
+# Supported ARM CPUs
+CONFIG_ARCH_MULTI_V6=n
+CONFIG_ARCH_MULTI_V7=y
+CONFIG_ARCH_MULTI_V6_V7=n
+CONFIG_CPU_V6=n
+
+# Enable CONFIG_SMP
+CONFIG_SMP=y
+
+# Disable Socs other than DRA7
+CONFIG_ARCH_OMAP2=n
+CONFIG_ARCH_OMAP3=n
+CONFIG_ARCH_OMAP4=n
+CONFIG_SOC_OMAP5=n
+CONFIG_SOC_AM33XX=n
+CONFIG_SOC_AM43XX=n
+CONFIG_SOC_DRA7XX=y
+
+CONFIG_ARCH_KEYSTONE=n
+
+CONFIG_ARM_LPAE=y
diff -Naur --no-dereference a/ti_config_fragments/ipc.cfg b/ti_config_fragments/ipc.cfg
--- a/ti_config_fragments/ipc.cfg	1969-12-31 19:00:00.000000000 -0500
+++ b/ti_config_fragments/ipc.cfg	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,37 @@
+##################################################
+# TI RPMsg/IPC Config Options
+##################################################
+# HwSpinLock
+CONFIG_HWSPINLOCK=y
+CONFIG_HWSPINLOCK_OMAP=y
+
+# Mailbox
+CONFIG_MAILBOX=y
+CONFIG_OMAP2PLUS_MBOX=y
+
+# IOMMU
+CONFIG_IOMMU_SUPPORT=y
+CONFIG_OMAP_IOMMU=y
+CONFIG_OMAP_IOMMU_DEBUG=y
+
+# SoC
+CONFIG_TI_PRUSS=m
+
+# Remoteproc
+CONFIG_REMOTEPROC=m
+CONFIG_OMAP_REMOTEPROC=m
+CONFIG_OMAP_REMOTEPROC_WATCHDOG=y
+CONFIG_KEYSTONE_REMOTEPROC=m
+CONFIG_PRU_REMOTEPROC=m
+
+# RPMsg
+CONFIG_RPMSG_VIRTIO=m
+CONFIG_RPMSG_PROTO=m
+CONFIG_RPMSG_RPC=m
+CONFIG_RPMSG_PRU=m
+
+# DSP Memory Mapper for Keystone MPM
+CONFIG_KEYSTONE_DSP_MEM=m
+
+# UIO Module
+CONFIG_UIO=m
diff -Naur --no-dereference a/ti_config_fragments/k2g_only.cfg b/ti_config_fragments/k2g_only.cfg
--- a/ti_config_fragments/k2g_only.cfg	1969-12-31 19:00:00.000000000 -0500
+++ b/ti_config_fragments/k2g_only.cfg	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,11 @@
+##################################################
+# TI K2G (UP) specific config options
+##################################################
+
+# HACK: Enable SMP for working around Coherency problems
+CONFIG_SMP=y
+CONFIG_CPUSETS=n
+CONFIG_POWER_SUPPLY=n
+
+# Increase CMA for HDMI
+CONFIG_CMA_SIZE_MBYTES=48
diff -Naur --no-dereference a/ti_config_fragments/k3_soc.cfg b/ti_config_fragments/k3_soc.cfg
--- a/ti_config_fragments/k3_soc.cfg	1969-12-31 19:00:00.000000000 -0500
+++ b/ti_config_fragments/k3_soc.cfg	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,7 @@
+##################################################
+# TI Base K3 SoCs
+##################################################
+
+CONFIG_ARCH_K3=y
+CONFIG_SOC_TI=y
+CONFIG_ARCH_K3_AM6_SOC=y
diff -Naur --no-dereference a/ti_config_fragments/keystone.cfg b/ti_config_fragments/keystone.cfg
--- a/ti_config_fragments/keystone.cfg	1969-12-31 19:00:00.000000000 -0500
+++ b/ti_config_fragments/keystone.cfg	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,29 @@
+##################################################
+# TI Base Keystone generation SoCs generic default
+##################################################
+
+# EDAC
+CONFIG_RAS=y
+CONFIG_EDAC=y
+CONFIG_EDAC_TI=m
+
+CONFIG_ARCH_MULTI_V7=y
+CONFIG_ARCH_MULTI_V6_V7=y
+CONFIG_ARCH_KEYSTONE=y
+CONFIG_KEYSTONE2_DMA_COHERENT=y
+CONFIG_ARCH_OMAP=n
+CONFIG_ARCH_OMAP2PLUS=n
+CONFIG_ARCH_OMAP2=n
+CONFIG_ARCH_OMAP3=n
+CONFIG_ARCH_OMAP4=n
+CONFIG_SOC_OMAP5=n
+CONFIG_SOC_AM33XX=n
+CONFIG_SOC_AM43XX=n
+CONFIG_SOC_DRA7XX=n
+CONFIG_ARM_PSCI=y
+CONFIG_POWER_RESET_KEYSTONE=y
+
+# Maximum Number of processors
+CONFIG_NR_CPUS=8
+
+CONFIG_HIGHMEM=y
diff -Naur --no-dereference a/ti_config_fragments/lpae.cfg b/ti_config_fragments/lpae.cfg
--- a/ti_config_fragments/lpae.cfg	1969-12-31 19:00:00.000000000 -0500
+++ b/ti_config_fragments/lpae.cfg	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1 @@
+CONFIG_ARM_LPAE=y
diff -Naur --no-dereference a/ti_config_fragments/multimedia.cfg b/ti_config_fragments/multimedia.cfg
--- a/ti_config_fragments/multimedia.cfg	1969-12-31 19:00:00.000000000 -0500
+++ b/ti_config_fragments/multimedia.cfg	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,10 @@
+##################################################
+# TI Multimedia (Video Decode, Encode) Configs
+##################################################
+
+# IMG D5500 v4l2 Driver Config Options
+CONFIG_MEDIA_SUPPORT=y
+CONFIG_VIDEO_IMG_VXD_DEC=m
+
+# IMG VXE384 v4l2 Driver Config Options
+CONFIG_VIDEO_IMG_VXE_ENC=m
diff -Naur --no-dereference a/ti_config_fragments/multi_v7_prune.cfg b/ti_config_fragments/multi_v7_prune.cfg
--- a/ti_config_fragments/multi_v7_prune.cfg	1969-12-31 19:00:00.000000000 -0500
+++ b/ti_config_fragments/multi_v7_prune.cfg	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,62 @@
+# Add config flags here that only appear in the multi_v7_defconfig
+CONFIG_ARCH_ALPINE=n
+CONFIG_ARCH_ARTPEC=n
+CONFIG_ARCH_AT91=n
+CONFIG_ARCH_BCM=n
+CONFIG_ARCH_BCM_CYGNUS=n
+CONFIG_ARCH_BCM_NSP=n
+CONFIG_ARCH_BCM_5301X=n
+CONFIG_ARCH_BCM_281XX=n
+CONFIG_ARCH_BCM_21664=n
+CONFIG_ARCH_BRCMSTB=n
+CONFIG_ARCH_BERLIN=n
+CONFIG_ARCH_DIGICOLOR=n
+CONFIG_ARCH_HIGHBANK=n
+CONFIG_ARCH_HISI=n
+CONFIG_ARCH_HI3xxx=n
+CONFIG_ARCH_HIP01=n
+CONFIG_ARCH_HIP04=n
+CONFIG_ARCH_HIX5HD2=n
+CONFIG_ARCH_MESON=n
+CONFIG_ARCH_MXC=n
+CONFIG_ARCH_MEDIATEK=n
+CONFIG_ARCH_QCOM=n
+CONFIG_ARCH_MSM8X60=n
+CONFIG_ARCH_MSM8960=n
+CONFIG_ARCH_MSM8974=n
+CONFIG_ARCH_ROCKCHIP=n
+CONFIG_ARCH_SOCFPGA=n
+CONFIG_ARCH_SPEAR13XX=n
+CONFIG_ARCH_STI=n
+CONFIG_ARCH_EXYNOS=n
+CONFIG_ARCH_SHMOBILE_MULTI=n
+CONFIG_ARCH_EMEV2=n
+CONFIG_ARCH_R7S72100=n
+CONFIG_ARCH_R8A73A4=n
+CONFIG_ARCH_R8A7740=n
+CONFIG_ARCH_R8A7778=n
+CONFIG_ARCH_R8A7779=n
+CONFIG_ARCH_R8A7790=n
+CONFIG_ARCH_R8A7791=n
+CONFIG_ARCH_R8A7793=n
+CONFIG_ARCH_R8A7794=n
+CONFIG_ARCH_RENESAS=n
+CONFIG_ARCH_SH73A0=n
+CONFIG_ARCH_SUNXI=n
+CONFIG_ARCH_STM32=n
+CONFIG_ARCH_SIRF=n
+CONFIG_ARCH_TEGRA=n
+CONFIG_ARCH_TEGRA_2x_SOC=n
+CONFIG_ARCH_TEGRA_3x_SOC=n
+CONFIG_ARCH_TEGRA_114_SOC=n
+CONFIG_ARCH_TEGRA_124_SOC=n
+CONFIG_ARCH_UNIPHIER=n
+CONFIG_ARCH_U8500=n
+CONFIG_ARCH_VEXPRESS=n
+CONFIG_ARCH_VEXPRESS_TC2_PM=n
+CONFIG_ARCH_WM8850=n
+CONFIG_ARCH_ZYNQ=n
+CONFIG_ARCH_VIRT=n
+CONFIG_ARCH_MVEBU=n
+CONFIG_PLAT_SPEAR=n
+CONFIG_CHROME_PLATFORMS=n
diff -Naur --no-dereference a/ti_config_fragments/omap5_soc.cfg b/ti_config_fragments/omap5_soc.cfg
--- a/ti_config_fragments/omap5_soc.cfg	1969-12-31 19:00:00.000000000 -0500
+++ b/ti_config_fragments/omap5_soc.cfg	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,14 @@
+##################################################
+# TI OMAP5 EVM specific config options
+##################################################
+
+#ETHERNET
+CONFIG_USB_USBNET=y
+CONFIG_USB_NET_DRIVERS=y
+CONFIG_USB_NET_SMSC95XX=y
+
+#USB HOST
+CONFIG_USB=y
+CONFIG_USB_EHCI_HCD=y
+CONFIG_USB_EHCI_PCI=y
+CONFIG_USB_EHCI_HCD_OMAP=y
diff -Naur --no-dereference a/ti_config_fragments/omap_soc.cfg b/ti_config_fragments/omap_soc.cfg
--- a/ti_config_fragments/omap_soc.cfg	1969-12-31 19:00:00.000000000 -0500
+++ b/ti_config_fragments/omap_soc.cfg	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,31 @@
+##################################################
+# TI Base OMAP generation SoCs generic default
+##################################################
+
+CONFIG_ARCH_MULTI_V7=y
+CONFIG_ARCH_MULTI_V6_V7=y
+CONFIG_ARCH_KEYSTONE=n
+CONFIG_KEYSTONE2_DMA_COHERENT=n
+CONFIG_ARCH_OMAP=y
+
+CONFIG_ARCH_OMAP2=n
+CONFIG_ARCH_OMAP3=n
+CONFIG_ARCH_OMAP4=n
+CONFIG_SOC_OMAP5=y
+CONFIG_SOC_AM33XX=y
+CONFIG_SOC_AM43XX=y
+CONFIG_SOC_DRA7XX=y
+CONFIG_ARCH_OMAP2PLUS=y
+CONFIG_OMAP_INTERCONNECT_BARRIER=y
+CONFIG_ARM_PSCI=n
+
+# Maximum Number of processors
+CONFIG_NR_CPUS=2
+
+# Kexec
+CONFIG_KEXEC=y
+
+# EDAC
+CONFIG_RAS=y
+CONFIG_EDAC=y
+CONFIG_EDAC_TI=m
diff -Naur --no-dereference a/ti_config_fragments/README b/ti_config_fragments/README
--- a/ti_config_fragments/README	1969-12-31 19:00:00.000000000 -0500
+++ b/ti_config_fragments/README	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,170 @@
+##################################
+# Introduction
+##################################
+
+Using config fragments in the Linux kernel is a supported model in the
+Linux kernel main line.  TI has taken advantage of this model to be able
+to take a base defconfig and add or remove configuration flags based contained
+within a config fragment.  This allows the base defconfig to be left in tact
+and helps facilitate and understanding of what config flags can be enabled
+or disabled.  The config fragments drastically reduces the number of merge
+conflicts when the Linux stable dot release (LTS) is merged back to the
+integration point as well as merges for TI domain trees.
+
+But alas this creates another issue as with the config fragments there is no
+direct defconfig for customers to build just many, many, many options.
+
+Therefore the defconfig_builder script was derived to take a defconfig map file
+and present a command line interface to the customer to to create a standard
+defconfig based off the TI integration branches.  The customer is walked through
+steps and if successful a defconfig will be created in the arch/arm/configs directory
+that can be built.
+
+##################################
+# Requirements
+##################################
+
+* awk v1.2 or greater.
+* bash shell 4.3.11(1)-release or greater.
+* Coreutils package v8.1 or greater.
+* grep (GNU grep) v2.16 or greater.
+* Linux kernel with merge_config script.
+* sed (GNU sed) v4.2.2 or greater.
+
+##################################
+# Usage
+##################################
+
+There is only one option for this script.  This option defines the working
+path to where the Linux kernel resides.
+
+	-w - Location of the TI Linux kernel
+	-t - Indicates the type of defconfig to build.  This will force the
+	     defconfig to build without user interaction.
+	-l - List all buildable defconfig options
+
+Command line example to generate the TI SDK AM335x processor defconfig automatically
+without user interaction:
+
+	ti_config_fragments/defconfig_builder.sh -t ti_sdk_am3x_release
+
+Command line Example if building from the ti_config_fragments directory:
+	defconfig_builder.sh -w ../.
+
+User interactive command line example:
+
+##################################
+# Defconfig Map File
+##################################
+
+The defconfig map file contains the mapping of the defconfig options and the
+assembly instructions for the defconfig to be created.
+
+The file contains keywords and all are *required* to be filled out.
+
+classification: - Defines the class of the defconfig which can be Release, Debug, System Test.
+type: - Defines the type of build.  Like debug build, LSK build or RT build
+defconfig: - Defines the base defconfig config to append the config fragments to
+config_file: - Defines a file that contains a list of config fragments.  If there is not a list then declare "None" for this field.
+extra_configs: - These are individual config fragments that can be appended to the defconfig by themselves or in addition to
+        the config_file.
+
+A Note on ordering of "extra_configs":
+
+In general, config fragments should be listed such that the fragment
+with more restrictive options goes towards the end of the list. This
+makes sure that those options are not inadvertently overridden by more
+generically applicable options. More concretely, the following ordering
+is recommended:
+
+1) Pruning of non-TI features
+2) Introduce domain specific features
+3) Optimize for an SoC family
+4) Optimize for a particular SoC
+5) Introduce distribution specific features. (example: Android)
+
+Example:
+
+This is an example of an AM335x entry that will create a defconfig only for AM335x
+processors, based on the omap2plus_defconfig.  The defconfig_fragment file adds features
+that have been enabled in the TI integrated kernel.  The extra_configs will add in the
+debug options as well as undefine all processors that are not AM335x.
+
+classification: SDK_release type: ti_sdk_am3x_release defconfig: multi_v7_defconfig config_file: None extra_configs: multi_v7_prune.cfg baseport.cfg ipc.cfg connectivity.cfg audio_display.cfg omap_soc.cfg am33xx_only.cfg systemd.cfg
+
+##################################
+# Defconfig Mapping
+##################################
+
+#Processor Specific Defconfig options
+
+These defconfig options enable the Linux kernel for the targer processor only.  Other processor platforms
+are disabled.  The base for each of these defconfigs is the omap2plus_defconfig contained
+in the arch/arm/configs directory of the kernel.
+
+	TI Base Integrated Kernel:
+		am335x_only - Adds the configs from the files contained in the defconfig_fragment file.
+		am335x_debug_only - Adds the configs from the files contained in the defconfig_fragment file and adds in the debug options from the debug_options.cfg
+		am43x_only - Adds the configs from the files contained in the defconfig_fragment file.
+		am43x_debug_only - Adds the configs from the files contained in the defconfig_fragment file and adds in the debug options from the debug_options.cfg
+		dra7xx_only - Adds the configs from the files contained in the defconfig_fragment file.
+		dra7xx_debug_only - Adds the configs from the files contained in the defconfig_fragment file and adds in the debug options from the debug_options.cfg
+
+	Linaro Stable Kernel builds: (These configurations are only available on the TI LSK branch.)
+		am335x_lsk_only - Adds the configs from the files contained in the defconfig_fragment file and adds in the linaro options from the linaro/configs directory
+		am335x_lsk_debug_only - Adds the configs from the files contained in the defconfig_fragment file and adds in the debug options from the debug_options.cfg as
+		       well as the linaro options from the linaro/configs directory
+		am43x_lsk_only - Adds the configs from the files contained in the defconfig_fragment file and adds in the linaro options from the linaro/configs directory
+		am43x_lsk_debug_only - Adds the configs from the files contained in the defconfig_fragment file and adds in the debug options from the debug_options.cfg as
+		       well as the linaro options from the linaro/configs directory
+		dra7xx_lsk_only - Adds the configs from the files contained in the defconfig_fragment file and adds in the linaro options from the linaro/configs directory
+		dra7xx_lsk_debug_only - Adds the configs from the files contained in the defconfig_fragment file and adds in the debug options from the debug_options.cfg as
+		       well as the linaro options from the linaro/configs directory
+
+	TI Based Real Time Kernel: (These configurations are only available on the TI RT branch.)
+		am335x_rt_only - Adds the configs from the files contained in the rt_defconfig_fragment file.
+		am335x_rt_debug_only - Adds the configs from the files contained in the rt_defconfig_fragment file and adds in the debug options from the debug_options.cfg.
+		am43x_rt_only - Adds the configs from the files contained in the rt_defconfig_fragment file.
+		am43x_rt_debug_only - Adds the configs from the files contained in the rt_defconfig_fragment file and adds in the debug options from the debug_options.cfg.
+		dra7xx_rt_only - Adds the configs from the files contained in the rt_defconfig_fragment file.
+		dra7xx_rt_debug_only - Adds the configs from the files contained in the rt_defconfig_fragment file and adds in the debug options from the debug_options.cfg.
+
+#OMAP2+ Defconfig Options
+
+These defconfig options use the omap2plus_defconfig in the arch/arm/configs directory as the base defconfig.
+These options do not disable any processor's or platforms but add additional features that
+
+	TI Base Integrated Kernel:
+		omap2plus_debug - Takes the omap2plus_defconfig and only adds the debug options from the debug_options.cfg.
+		ti_omap2plus - Adds the configs from the files contained in the defconfig_fragment file.
+		ti_omap2plus_debug - Adds the configs from the files contained in the defconfig_fragment file and adds in the debug options from the debug_options.cfg
+
+	Linaro Stable Kernel builds: (These configurations are only available on the TI LSK branch.)
+		ti_lsk_omap2plus - Adds the configs from the files contained in the defconfig_fragment file and adds in the linaro options from the linaro/configs directory.
+		ti_lsk_omap2plus_debug - Adds the configs from the files contained in the defconfig_fragment file and adds in the debug options from the debug_options.cfg as
+		       well as the linaro options from the linaro/configs directory
+
+	TI Based Real Time Kernel: (These configurations are only available on the TI RT branch.)
+		ti_rt_omap2plus - Adds the configs from the files contained in the rt_defconfig_fragment file.
+		ti_rt_omap2plus_debug - Adds the configs from the files contained in the rt_defconfig_fragment file and adds in the debug options from the debug_options.cfg.
+
+#Keystone Defconfig Options
+
+These defconfig options use the keystone_defconfig in the arch/arm/configs directory as the base defconfig.
+These options do not disable any processor's or platforms but add additional features that
+
+	TI Base Integrated Kernel:
+		base_keystone_debug - Takes the keystone_defconfig and only adds the debug options from the debug_options.cfg.
+		ti_keystone defconfig - Adds the configs from the files contained in the multi_core_defconfig_fragment file.
+		ti_keystone_debug - Adds the configs from the files contained in the multi_core_defconfig_fragment file and
+                       adds in the debug options from the debug_options.cfg.
+
+	Linaro Stable Kernel builds: (These configurations are only available on the TI LSK branch.)
+		ti_lsk_keystone  - Adds the configs from the files contained in the multi_core_defconfig_fragment file and adds in the
+                       linaro options from the linaro/configs directory
+		ti_lsk_keystone_debug_only - Adds the configs from the files contained in the multi_core_defconfig_fragment file and adds in the
+                       debug options from the debug_options.cfg as well as the linaro options from the linaro/configs directory
+
+	TI Based Real Time Kernel: (These configurations are only available on the TI RT branch.)
+		ti_rt_keystone - Adds the configs from the files contained in the rt_defconfig_fragment file.
+		ti_rt_keystone_debug  - Adds the configs from the files contained in the rt_defconfig_fragment file and adds in the debug options from the debug_options.cfg.
diff -Naur --no-dereference a/ti_config_fragments/systemd.cfg b/ti_config_fragments/systemd.cfg
--- a/ti_config_fragments/systemd.cfg	1969-12-31 19:00:00.000000000 -0500
+++ b/ti_config_fragments/systemd.cfg	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,88 @@
+##############################################
+#
+# Kernel options needed for systemd enabled TI SDKs
+# See https://cgit.freedesktop.org/systemd/systemd/tree/README#n38 for details
+#
+##############################################
+CONFIG_TMPFS=y
+CONFIG_DEVTMPFS=y
+CONFIG_DEVTMPFS_MOUNT=y
+CONFIG_CGROUPS=y
+CONFIG_CGROUP_FREEZER=y
+CONFIG_CGROUP_DEVICE=y
+CONFIG_CGROUP_CPUACCT=y
+CONFIG_BLK_CGROUP=y
+CONFIG_CGROUP_PIDS=y
+CONFIG_CGROUP_PERF=y
+
+CONFIG_INOTIFY_USER=y
+CONFIG_SIGNALFD=y
+CONFIG_TIMERFD=y
+CONFIG_EPOLL=y
+CONFIG_NET=y
+CONFIG_SYSFS=y
+CONFIG_PROC_FS=y
+CONFIG_FHANDLE=y
+
+# udev will fail to work with the legacy sysfs layout:
+CONFIG_SYSFS_DEPRECATED=n
+
+# Legacy hotplug slows down the system and confuses udev:
+CONFIG_UEVENT_HELPER_PATH=""
+
+# Userspace firmware loading is not supported and should
+# be disabled in the kernel:
+CONFIG_FW_LOADER_USER_HELPER=n
+
+# Some udev rules and virtualization detection relies on it:
+# Only for UEFI based systems
+# http://cateee.net/lkddb/web-lkddb/DMI.html
+CONFIG_DMIID=n
+
+# Support for some SCSI devices serial number retrieval, to
+# create additional symlinks in /dev/disk/ and /dev/tape:
+CONFIG_BLK_DEV_BSG=y
+
+# Required for PrivateNetwork and PrivateDevices in service units:
+# Note that systemd-localed.service and other systemd units use
+# PrivateNetwork and PrivateDevices so this is effectively required.
+CONFIG_NAMESPACES=y
+CONFIG_NET_NS=y
+CONFIG_DEVPTS_MULTIPLE_INSTANCES=y
+
+# Optional but strongly recommended options: those are nice to have and
+# indeed recommended, but not necessarily systemd required. These to be
+# enabled in corresponding domain fragments since they are not specific
+# to supporting systemd.
+
+CONFIG_SECCOMP=y
+# for kcmp syscall
+CONFIG_CHECKPOINT_RESTORE=y
+
+# Required for CPUShares= in resource control unit settings
+CONFIG_CGROUP_SCHED=y
+CONFIG_FAIR_GROUP_SCHED=y
+
+# Required for CPUQuota= in resource control unit settings
+CONFIG_CFS_BANDWIDTH=y
+
+# For systemd-bootchart, several proc debug interfaces are required:
+# Systemd-debug.cfg?
+CONFIG_SCHEDSTATS=y
+CONFIG_SCHED_DEBUG=y
+
+# We recommend to turn off Real-Time group scheduling in the
+# kernel when using systemd. RT group scheduling effectively
+# makes RT scheduling unavailable for most userspace, since it
+# requires explicit assignment of RT budgets to each unit whose
+# processes making use of RT. As there's no sensible way to
+# assign these budgets automatically this cannot really be
+# fixed, and it's best to disable group scheduling hence.
+CONFIG_RT_GROUP_SCHED=n
+
+# Note that kernel auditing is broken when used with systemd's
+# container code. When using systemd in conjunction with
+# containers, please make sure to either turn off auditing at
+# runtime using the kernel command line option "audit=0", or
+# turn it off at kernel compile time using:
+CONFIG_AUDIT=n
diff -Naur --no-dereference a/ti_config_fragments/v8_baseport.cfg b/ti_config_fragments/v8_baseport.cfg
--- a/ti_config_fragments/v8_baseport.cfg	1969-12-31 19:00:00.000000000 -0500
+++ b/ti_config_fragments/v8_baseport.cfg	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,161 @@
+##################################################
+# TI Baseport Config Options
+##################################################
+
+# ARM configuration
+CONFIG_ARM64_4K_PAGES=n
+CONFIG_ARM64_64K_PAGES=y
+CONFIG_ARM64_VA_BITS_48=y
+CONFIG_ARM64_VA_BITS=48
+
+# Serial
+CONFIG_SERIAL_8250_OMAP=y
+CONFIG_SERIAL_8250_NR_UARTS=10
+CONFIG_SERIAL_8250_RUNTIME_UARTS=10
+
+# K3 Power config options
+CONFIG_MAILBOX=y
+CONFIG_PM=y
+CONFIG_TI_MESSAGE_MANAGER=y
+CONFIG_TI_SCI_PROTOCOL=y
+CONFIG_TI_SCI_PM_DOMAINS=y
+CONFIG_TI_SCI_CLK=y
+
+# Enable Reset Controllers
+CONFIG_RESET_CONTROLLER=y
+CONFIG_RESET_TI_SYSCON=y
+CONFIG_RESET_TI_SCI=y
+
+# Enable Interrupt Controllers
+CONFIG_TI_SCI_INTR_IRQCHIP=y
+CONFIG_TI_SCI_INTA_IRQCHIP=y
+
+# Enable options for NAVSS (DMA support)
+CONFIG_TI_K3_RINGACC=y
+CONFIG_TI_K3_UDMA=y
+CONFIG_TI_K3_UDMA_GLUE_LAYER=y
+
+# OPTEE Driver
+CONFIG_TEE=y
+CONFIG_OPTEE=y
+
+#-----
+# Generic Kernel Options
+#-----
+
+# Enable process accounting
+CONFIG_BSD_PROCESS_ACCT=y
+
+# Enable support for perf
+CONFIG_PERF_EVENTS=y
+
+# Have some way to pick up kernel config later on
+# Always useful to look at /proc/config.gz
+CONFIG_IKCONFIG=y
+CONFIG_IKCONFIG_PROC=y
+
+# Add Bin2c
+CONFIG_BUILD_BIN2C=y
+
+# Add base Cgroups functions
+CONFIG_CGROUPS=y
+CONFIG_CGROUP_DEBUG=n
+CONFIG_CGROUP_FREEZER=y
+CONFIG_CGROUP_PIDS=y
+CONFIG_CGROUP_DEVICE=y
+CONFIG_CPUSETS=y
+CONFIG_PROC_PID_CPUSET=y
+CONFIG_CGROUP_CPUACCT=y
+CONFIG_PAGE_COUNTER=y
+CONFIG_MEMCG=y
+CONFIG_MEMCG_SWAP=y
+CONFIG_MEMCG_SWAP_ENABLED=y
+CONFIG_CGROUP_PERF=y
+CONFIG_CGROUP_SCHED=y
+CONFIG_FAIR_GROUP_SCHED=y
+CONFIG_CFS_BANDWIDTH=y
+CONFIG_RT_GROUP_SCHED=y
+CONFIG_BLK_CGROUP=y
+CONFIG_DEBUG_BLK_CGROUP=n
+CONFIG_CGROUP_WRITEBACK=y
+
+# Choose CONFIG_EMBEDDED
+CONFIG_NAMESPACES=n
+CONFIG_EMBEDDED=y
+CONFIG_EXPERT=y
+
+# Enable all kernel symbols please
+CONFIG_KALLSYMS=y
+CONFIG_KALLSYMS_ALL=y
+
+# How do we want kernel Modules to work?
+CONFIG_BASE_FULL=y
+CONFIG_BASE_SMALL=0
+CONFIG_MODULES=y
+CONFIG_MODULE_FORCE_LOAD=y
+CONFIG_MODULE_UNLOAD=y
+CONFIG_MODULE_FORCE_UNLOAD=y
+CONFIG_MODVERSIONS=y
+CONFIG_MODULE_SRCVERSION_ALL=y
+CONFIG_MODULE_SIG=n
+CONFIG_MODULE_COMPRESS=n
+CONFIG_MODULES_TREE_LOOKUP=y
+
+# Boot options
+CONFIG_CMDLINE=""
+CONFIG_CMDLINE_FORCE=n
+CONFIG_CRASH_DUMP=n
+CONFIG_AUTO_ZRELADDR=y
+
+# Mem allocator
+CONFIG_SLAB=n
+CONFIG_SLUB=y
+CONFIG_SLUB_DEBUG=n
+
+# Pinctrl
+CONFIG_PINCTRL=y
+CONFIG_PINCTRL_SINGLE=y
+
+# GPIO
+CONFIG_GPIOLIB=y
+CONFIG_GPIO_SYSFS=y
+CONFIG_GPIO_DAVINCI=y
+
+# Enable options to facilitate testing
+CONFIG_CRYPTO_TEST=m
+
+# Pseudo filesystems
+CONFIG_TMPFS=y
+
+# Thermal
+CONFIG_K3_THERMAL=y
+
+# Disable extra debug options
+CONFIG_PROFILING=n
+
+# Block Layer
+CONFIG_BLK_DEV_RAM=y
+
+# ION
+CONFIG_STAGING=y
+CONFIG_ANDROID=y
+CONFIG_ION=y
+CONFIG_ION_SYSTEM_HEAP=y
+CONFIG_ION_CARVEOUT_HEAP=y
+CONFIG_ION_CHUNK_HEAP=y
+CONFIG_ION_TI=y
+
+# DMA-BUF exporter
+CONFIG_DMA_BUF_PHYS=y
+
+# PMIC
+CONFIG_MFD_PALMAS=y
+CONFIG_REGULATOR=y
+CONFIG_REGULATOR_PALMAS=y
+CONFIG_REGULATOR_FIXED_VOLTAGE=y
+
+# TI PAT
+CONFIG_TI_PAT=y
+
+# Watchdog support
+CONFIG_K3_RTI_WATCHDOG=m
diff -Naur --no-dereference a/ti_config_fragments/v8_defconfig_map.txt b/ti_config_fragments/v8_defconfig_map.txt
--- a/ti_config_fragments/v8_defconfig_map.txt	1969-12-31 19:00:00.000000000 -0500
+++ b/ti_config_fragments/v8_defconfig_map.txt	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,12 @@
+# Release Defconfigs
+classification: SDK_Release_Defconfigs type: ti_sdk_arm64_release defconfig: defconfig config_file: None extra_configs: arm64_prune.cfg v8_baseport.cfg v8_ipc.cfg connectivity.cfg audio_display.cfg k3_soc.cfg systemd.cfg multimedia.cfg
+# Debug Defconfigs
+classification: SDK_Debug_Defconfigs type: ti_sdk_arm64_debug defconfig: defconfig config_file: None extra_configs: arm64_prune.cfg v8_baseport.cfg v8_ipc.cfg connectivity.cfg audio_display.cfg k3_soc.cfg systemd.cfg debug_options.cfg multimedia.cfg
+# RT Defconfigs
+classification: SDK_Release_Defconfigs type: ti_sdk_arm64_rt_release defconfig: defconfig config_file: None extra_configs: arm64_prune.cfg v8_baseport.cfg v8_ipc.cfg connectivity.cfg audio_display.cfg k3_soc.cfg real_time.cfg systemd.cfg multimedia.cfg
+# RT Debug Defconfigs
+classification: SDK_Debug_Defconfigs type: ti_sdk_arm64_rt_debug defconfig: defconfig config_file: None extra_configs: arm64_prune.cfg v8_baseport.cfg v8_ipc.cfg connectivity.cfg audio_display.cfg k3_soc.cfg real_time.cfg systemd.cfg debug_options.cfg multimedia.cfg
+# Android Defconfigs
+classification: SDK_Release_Defconfigs type: ti_sdk_arm64_android_release defconfig: defconfig config_file: None extra_configs: arm64_prune.cfg v8_baseport.cfg v8_ipc.cfg connectivity.cfg audio_display.cfg k3_soc.cfg ../kernel/configs/android-base.config ../kernel/configs/android-recommended.config v8_android.cfg
+# Android Debug Defconfigs
+classification: SDK_Debug_Defconfigs type: ti_sdk_arm64_android_debug defconfig: defconfig config_file: None extra_configs: arm64_prune.cfg v8_baseport.cfg v8_ipc.cfg connectivity.cfg audio_display.cfg k3_soc.cfg debug_options.cfg ../kernel/configs/android-base.config ../kernel/configs/android-recommended.config v8_android.cfg
diff -Naur --no-dereference a/ti_config_fragments/v8_ipc.cfg b/ti_config_fragments/v8_ipc.cfg
--- a/ti_config_fragments/v8_ipc.cfg	1969-12-31 19:00:00.000000000 -0500
+++ b/ti_config_fragments/v8_ipc.cfg	2020-06-19 21:00:31.440227717 -0400
@@ -0,0 +1,28 @@
+##################################################
+# TI RPMsg/IPC Config Options
+##################################################
+# HwSpinLock
+CONFIG_HWSPINLOCK=y
+CONFIG_HWSPINLOCK_OMAP=y
+
+# Mailbox
+CONFIG_MAILBOX=y
+CONFIG_OMAP2PLUS_MBOX=y
+
+# SoC Drivers
+CONFIG_TI_PRUSS=m
+
+# Remoteproc
+CONFIG_REMOTEPROC=m
+CONFIG_PRU_REMOTEPROC=m
+CONFIG_TI_K3_R5_REMOTEPROC=m
+CONFIG_TI_K3_DSP_REMOTEPROC=m
+
+# RPMsg
+CONFIG_RPMSG_VIRTIO=m
+CONFIG_RPMSG_PROTO=m
+CONFIG_RPMSG_PRU=m
+
+# RPMsg Samples
+CONFIG_SAMPLES=y
+CONFIG_SAMPLE_RPMSG_CLIENT=m
diff -Naur --no-dereference a/tools/pci/pcitest.c b/tools/pci/pcitest.c
--- a/tools/pci/pcitest.c	2020-01-09 04:19:10.000000000 -0500
+++ b/tools/pci/pcitest.c	2020-06-19 21:00:31.440227717 -0400
@@ -41,6 +41,7 @@
 	int		irqtype;
 	bool		set_irqtype;
 	bool		get_irqtype;
+	bool		clear_irq;
 	bool		read;
 	bool		write;
 	bool		copy;
@@ -85,6 +86,15 @@
 			fprintf(stdout, "%s\n", irq[ret]);
 	}
 
+	if (test->clear_irq) {
+		ret = ioctl(fd, PCITEST_CLEAR_IRQ);
+		fprintf(stdout, "CLEAR IRQ:\t\t");
+		if (ret < 0)
+			fprintf(stdout, "FAILED\n");
+		else
+			fprintf(stdout, "%s\n", result[ret]);
+	}
+
 	if (test->legacyirq) {
 		ret = ioctl(fd, PCITEST_LEGACY_IRQ, 0);
 		fprintf(stdout, "LEGACY IRQ:\t");
@@ -162,7 +172,7 @@
 	/* set default endpoint device */
 	test->device = "/dev/pci-endpoint-test.0";
 
-	while ((c = getopt(argc, argv, "D:b:m:x:i:Ilrwcs:")) != EOF)
+	while ((c = getopt(argc, argv, "D:b:m:x:i:eIlhrwcs:")) != EOF)
 	switch (c) {
 	case 'D':
 		test->device = optarg;
@@ -203,10 +213,12 @@
 	case 'c':
 		test->copy = true;
 		continue;
+	case 'e':
+		test->clear_irq = true;
+		continue;
 	case 's':
 		test->size = strtoul(optarg, NULL, 0);
 		continue;
-	case '?':
 	case 'h':
 	default:
 usage:
@@ -218,12 +230,14 @@
 			"\t-m <msi num>		MSI test (msi number between 1..32)\n"
 			"\t-x <msix num>	\tMSI-X test (msix number between 1..2048)\n"
 			"\t-i <irq type>	\tSet IRQ type (0 - Legacy, 1 - MSI, 2 - MSI-X)\n"
+			"\t-e			Clear IRQ\n"
 			"\t-I			Get current IRQ type configured\n"
 			"\t-l			Legacy IRQ test\n"
 			"\t-r			Read buffer test\n"
 			"\t-w			Write buffer test\n"
 			"\t-c			Copy buffer test\n"
 			"\t-s <size>		Size of buffer {default: 100KB}\n",
+			"\t-h			Print this help message\n",
 			argv[0]);
 		return -EINVAL;
 	}
diff -Naur --no-dereference a/tools/perf/util/stat-shadow.c b/tools/perf/util/stat-shadow.c
--- a/tools/perf/util/stat-shadow.c	2020-01-09 04:19:10.000000000 -0500
+++ b/tools/perf/util/stat-shadow.c	2020-06-19 21:00:31.440227717 -0400
@@ -209,12 +209,12 @@
 				    int cpu, struct runtime_stat *st)
 {
 	int ctx = evsel_context(counter);
+	u64 count_ns = count;
 
 	count *= counter->scale;
 
-	if (perf_evsel__match(counter, SOFTWARE, SW_TASK_CLOCK) ||
-	    perf_evsel__match(counter, SOFTWARE, SW_CPU_CLOCK))
-		update_runtime_stat(st, STAT_NSECS, 0, cpu, count);
+	if (perf_evsel__is_clock(counter))
+		update_runtime_stat(st, STAT_NSECS, 0, cpu, count_ns);
 	else if (perf_evsel__match(counter, HARDWARE, HW_CPU_CYCLES))
 		update_runtime_stat(st, STAT_CYCLES, ctx, cpu, count);
 	else if (perf_stat_evsel__is(counter, CYCLES_IN_TX))
